// SPDX-License-Identifier: GPL-2.0
/*
* Implementation of the diskquota system for the LINUX operating system. QUOTA
* is implemented using the BSD system call interface as the means of
* communication with the user level. This file contains the generic routines
* called by the different filesystems on allocation of an inode or block.
* These routines take care of the administration needed to have a consistent
* diskquota tracking system. The ideas of both user and group quotas are based
* on the Melbourne quota system as used on BSD derived systems. The internal
* implementation is based on one of the several variants of the LINUX
* inode-subsystem with added complexity of the diskquota system.
*
* Author: Marco van Wieringen <mvw@planets.elm.net>
*
* Fixes: Dmitry Gorodchanin <pgmdsg@ibi.com>, 11 Feb 96
*
* Revised list management to avoid races
* -- Bill Hawes, <whawes@star.net>, 9/98
*
* Fixed races in dquot_transfer(), dqget() and dquot_alloc_...().
* As the consequence the locking was moved from dquot_decr_...(),
* dquot_incr_...() to calling functions.
* invalidate_dquots() now writes modified dquots.
* Serialized quota_off() and quota_on() for mount point.
* Fixed a few bugs in grow_dquots().
* Fixed deadlock in write_dquot() - we no longer account quotas on
* quota files
* remove_dquot_ref() moved to inode.c - it now traverses through inodes
* add_dquot_ref() restarts after blocking
* Added check for bogus uid and fixed check for group in quotactl.
* Jan Kara, <jack@suse.cz>, sponsored by SuSE CR, 10-11/99
*
* Used struct list_head instead of own list struct
* Invalidation of referenced dquots is no longer possible
* Improved free_dquots list management
* Quota and i_blocks are now updated in one place to avoid races
* Warnings are now delayed so we won't block in critical section
* Write updated not to require dquot lock
* Jan Kara, <jack@suse.cz>, 9/2000
*
* Added dynamic quota structure allocation
* Jan Kara <jack@suse.cz> 12/2000
*
* Rewritten quota interface. Implemented new quota format and
* formats registering.
* Jan Kara, <jack@suse.cz>, 2001,2002
*
* New SMP locking.
* Jan Kara, <jack@suse.cz>, 10/2002
*
* Added journalled quota support, fix lock inversion problems
* Jan Kara, <jack@suse.cz>, 2003,2004
*
* (C) Copyright 1994 - 1997 Marco van Wieringen
*/
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/mm.h>
#include <linux/time.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/fcntl.h>
#include <linux/stat.h>
#include <linux/tty.h>
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/sysctl.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/security.h>
#include <linux/sched.h>
#include <linux/cred.h>
#include <linux/kmod.h>
#include <linux/namei.h>
#include <linux/capability.h>
#include <linux/quotaops.h>
#include <linux/blkdev.h>
#include <linux/sched/mm.h>
#include "../internal.h" /* ugh */
#include <linux/uaccess.h>
/*
* There are five quota SMP locks:
* * dq_list_lock protects all lists with quotas and quota formats.
* * dquot->dq_dqb_lock protects data from dq_dqb
* * inode->i_lock protects inode->i_blocks, i_bytes and also guards
* consistency of dquot->dq_dqb with inode->i_blocks, i_bytes so that
* dquot_transfer() can stabilize amount it transfers
* * dq_data_lock protects mem_dqinfo structures and modifications of dquot
* pointers in the inode
* * dq_state_lock protects modifications of quota state (on quotaon and
* quotaoff) and readers who care about latest values take it as well.
*
* The spinlock ordering is hence:
* dq_data_lock > dq_list_lock > i_lock > dquot->dq_dqb_lock,
* dq_list_lock > dq_state_lock
*
* Note that some things (eg. sb pointer, type, id) doesn't change during
* the life of the dquot structure and so needn't to be protected by a lock
*
* Operation accessing dquots via inode pointers are protected by dquot_srcu.
* Operation of reading pointer needs srcu_read_lock(&dquot_srcu), and
* synchronize_srcu(&dquot_srcu) is called after clearing pointers from
* inode and before dropping dquot references to avoid use of dquots after
* they are freed. dq_data_lock is used to serialize the pointer setting and
* clearing operations.
* Special care needs to be taken about S_NOQUOTA inode flag (marking that
* inode is a quota file). Functions adding pointers from inode to dquots have
* to check this flag under dq_data_lock and then (if S_NOQUOTA is not set) they
* have to do all pointer modifications before dropping dq_data_lock. This makes
* sure they cannot race with quotaon which first sets S_NOQUOTA flag and
* then drops all pointers to dquots from an inode.
*
* Each dquot has its dq_lock mutex. Dquot is locked when it is being read to
* memory (or space for it is being allocated) on the first dqget(), when it is
* being written out, and when it is being released on the last dqput(). The
* allocation and release operations are serialized by the dq_lock and by
* checking the use count in dquot_release().
*
* Lock ordering (including related VFS locks) is the following:
* s_umount > i_mutex > journal_lock > dquot->dq_lock > dqio_sem
*/
static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dq_list_lock);
static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dq_state_lock);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(dq_data_lock);
EXPORT_SYMBOL(dq_data_lock);
DEFINE_STATIC_SRCU(dquot_srcu);
static DECLARE_WAIT_QUEUE_HEAD(dquot_ref_wq);
void __quota_error(struct super_block *sb, const char *func,
const char *fmt, ...)
{
if (printk_ratelimit()) {
va_list args;
struct va_format vaf;
va_start(args, fmt);
vaf.fmt = fmt;
vaf.va = &args;
printk(KERN_ERR "Quota error (device %s): %s: %pV\n",
sb->s_id, func, &vaf);
va_end(args);
}
}
EXPORT_SYMBOL(__quota_error);
#if defined(CONFIG_QUOTA_DEBUG) || defined(CONFIG_PRINT_QUOTA_WARNING)
static char *quotatypes[] = INITQFNAMES;
#endif
static struct quota_format_type *quota_formats; /* List of registered formats */
static struct quota_module_name module_names[] = INIT_QUOTA_MODULE_NAMES;
/* SLAB cache for dquot structures */
static struct kmem_cache *dquot_cachep;
int register_quota_format(struct quota_format_type *fmt)
{
spin_lock(&dq_list_lock);
fmt->qf_next = quota_formats;
quota_formats = fmt;
spin_unlock(&dq_list_lock);
return 0;
}
EXPORT_SYMBOL(register_quota_format);
void unregister_quota_format(struct quota_format_type *fmt)
{
struct quota_format_type **actqf;
spin_lock(&dq_list_lock);
for (actqf = "a_formats; *actqf && *actqf != fmt;
actqf = &(*actqf)->qf_next)
;
if (*actqf)
*actqf = (*actqf)->qf_next;
spin_unlock(&dq_list_lock);
}
EXPORT_SYMBOL(unregister_quota_format);
static struct quota_format_type *find_quota_format(int id)
{
struct quota_format_type *actqf;
spin_lock(&dq_list_lock);
for (actqf = quota_formats; actqf && actqf->qf_fmt_id != id;
actqf = actqf->qf_next)
;
if (!actqf || !try_module_get(actqf->qf_owner)) {
int qm;
spin_unlock(&dq_list_lock);
for (qm = 0; module_names[qm].qm_fmt_id &&
module_names[qm].qm_fmt_id != id; qm++)
;
if (!module_names[qm].qm_fmt_id ||
request_module(module_names[qm].qm_mod_name))
return NULL;
spin_lock(&dq_list_lock);
for (actqf = quota_formats; actqf && actqf->qf_fmt_id != id;
actqf = actqf->qf_next)
;
if (actqf && !try_module_get(actqf->qf_owner))
actqf = NULL;
}
spin_unlock(&dq_list_lock);
return actqf;
}
static void put_quota_format(struct quota_format_type *fmt)
{
module_put(fmt->qf_owner);
}
/*
* Dquot List Management:
* The quota code uses five lists for dquot management: the inuse_list,
* releasing_dquots, free_dquots, dqi_dirty_list, and dquot_hash[] array.
* A single dquot structure may be on some of those lists, depending on
* its current state.
*
* All dquots are placed to the end of inuse_list when first created, and this
* list is used for invalidate operation, which must look at every dquot.
*
* When the last reference of a dquot is dropped, the dquot is added to
* releasing_dquots. We'll then queue work item which will call
* synchronize_srcu() and after that perform the final cleanup of all the
* dquots on the list. Each cleaned up dquot is moved to free_dquots list.
* Both releasing_dquots and free_dquots use the dq_free list_head in the dquot
* struct.
*
* Unused and cleaned up dquots are in the free_dquots list and this list is
* searched whenever we need an available dquot. Dquots are removed from the
* list as soon as they are used again and dqstats.free_dquots gives the number
* of dquots on the list. When dquot is invalidated it's completely released
* from memory.
*
* Dirty dquots are added to the dqi_dirty_list of quota_info when mark
* dirtied, and this list is searched when writing dirty dquots back to
* quota file. Note that some filesystems do dirty dquot tracking on their
* own (e.g. in a journal) and thus don't use dqi_dirty_list.
*
* Dquots with a specific identity (device, type and id) are placed on
* one of the dquot_hash[] hash chains. The provides an efficient search
* mechanism to locate a specific dquot.
*/
static LIST_HEAD(inuse_list);
static LIST_HEAD(free_dquots);
static LIST_HEAD(releasing_dquots);
static unsigned int dq_hash_bits, dq_hash_mask;
static struct hlist_head *dquot_hash;
struct dqstats dqstats;
EXPORT_SYMBOL(dqstats);
static qsize_t inode_get_rsv_space(struct inode *inode);
static qsize_t __inode_get_rsv_space(struct inode *inode);
static int __dquot_initialize(struct inode *inode, int type);
static void quota_release_workfn(struct work_struct *work);
static DECLARE_DELAYED_WORK(quota_release_work, quota_release_workfn);
static inline unsigned int
hashfn(const struct super_block *sb, struct kqid qid)
{
unsigned int id = from_kqid(&init_user_ns, qid);
int type = qid.type;
unsigned long tmp;
tmp = (((unsigned long)sb>>L1_CACHE_SHIFT) ^ id) * (MAXQUOTAS - type);
return (tmp + (tmp >> dq_hash_bits)) & dq_hash_mask;
}
/*
* Following list functions expect dq_list_lock to be held
*/
static inline void insert_dquot_hash(struct dquot *dquot)
{
struct hlist_head *head;
head = dquot_hash + hashfn(dquot->dq_sb, dquot->dq_id);
hlist_add_head(&dquot->dq_hash, head);
}
static inline void remove_dquot_hash(struct dquot *dquot)
{
hlist_del_init(&dquot->dq_hash);
}
static struct dquot *find_dquot(unsigned int hashent, struct super_block *sb,
struct kqid qid)
{
struct dquot *dquot;
hlist_for_each_entry(dquot, dquot_hash+hashent, dq_hash)
if (dquot->dq_sb == sb && qid_eq(dquot->dq_id, qid))
return dquot;
return NULL;
}
/* Add a dquot to the tail of the free list */
static inline void put_dquot_last(struct dquot *dquot)
{
list_add_tail(&dquot->dq_free, &free_dquots);
dqstats_inc(DQST_FREE_DQUOTS);
}
static inline void put_releasing_dquots(struct dquot *dquot)
{
list_add_tail(&dquot->dq_free, &releasing_dquots);
set_bit(DQ_RELEASING_B, &dquot->dq_flags);
}
static inline void remove_free_dquot(struct dquot *dquot)
{
if (list_empty(&dquot->dq_free))
return;
list_del_init(&dquot->dq_free);
if (!test_bit(DQ_RELEASING_B, &dquot->dq_flags))
dqstats_dec(DQST_FREE_DQUOTS);
else
clear_bit(DQ_RELEASING_B, &dquot->dq_flags);
}
static inline void put_inuse(struct dquot *dquot)
{
/* We add to the back of inuse list so we don't have to restart
* when traversing this list and we block */
list_add_tail(&dquot->dq_inuse, &inuse_list);
dqstats_inc(DQST_ALLOC_DQUOTS);
}
static inline void remove_inuse(struct dquot *dquot)
{
dqstats_dec(DQST_ALLOC_DQUOTS);
list_del(&dquot->dq_inuse);
}
/*
* End of list functions needing dq_list_lock
*/
static void wait_on_dquot(struct dquot *dquot)
{
mutex_lock(&dquot->dq_lock);
mutex_unlock(&dquot->dq_lock);
}
static inline int dquot_active(struct dquot *dquot)
{
return test_bit(DQ_ACTIVE_B, &dquot->dq_flags);
}
static inline int dquot_dirty(struct dquot *dquot)
{
return test_bit(DQ_MOD_B, &dquot->dq_flags);
}
static inline int mark_dquot_dirty(struct dquot *dquot)
{
return dquot->dq_sb->dq_op->mark_dirty(dquot);
}
/* Mark dquot dirty in atomic manner, and return it's old dirty flag state */
int dquot_mark_dquot_dirty(struct dquot *dquot)
{
int ret = 1;
if (!dquot_active(dquot))
return 0;
if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NOLIST_DIRTY)
return test_and_set_bit(DQ_MOD_B, &dquot->dq_flags);
/* If quota is dirty already, we don't have to acquire dq_list_lock */
if (dquot_dirty(dquot))
return 1;
spin_lock(&dq_list_lock);
if (!test_and_set_bit(DQ_MOD_B, &dquot->dq_flags)) {
list_add(&dquot->dq_dirty, &sb_dqopt(dquot->dq_sb)->
info[dquot->dq_id.type].dqi_dirty_list);
ret = 0;
}
spin_unlock(&dq_list_lock);
return ret;
}
EXPORT_SYMBOL(dquot_mark_dquot_dirty);
/* Dirtify all the dquots - this can block when journalling */
static inline int mark_all_dquot_dirty(struct dquot __rcu * const *dquots)
{
int ret, err, cnt;
struct dquot *dquot;
ret = err = 0;
for (cnt = 0; cnt < MAXQUOTAS; cnt++) { dquot = srcu_dereference(dquots[cnt], &dquot_srcu); if (dquot)
/* Even in case of error we have to continue */
ret = mark_dquot_dirty(dquot); if (!err)
err = ret;
}
return err;
}
static inline void dqput_all(struct dquot **dquot)
{
unsigned int cnt;
for (cnt = 0; cnt < MAXQUOTAS; cnt++) dqput(dquot[cnt]);
}
static inline int clear_dquot_dirty(struct dquot *dquot)
{
if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NOLIST_DIRTY)
return test_and_clear_bit(DQ_MOD_B, &dquot->dq_flags);
spin_lock(&dq_list_lock);
if (!test_and_clear_bit(DQ_MOD_B, &dquot->dq_flags)) {
spin_unlock(&dq_list_lock);
return 0;
}
list_del_init(&dquot->dq_dirty);
spin_unlock(&dq_list_lock);
return 1;
}
void mark_info_dirty(struct super_block *sb, int type)
{
spin_lock(&dq_data_lock);
sb_dqopt(sb)->info[type].dqi_flags |= DQF_INFO_DIRTY;
spin_unlock(&dq_data_lock);
}
EXPORT_SYMBOL(mark_info_dirty);
/*
* Read dquot from disk and alloc space for it
*/
int dquot_acquire(struct dquot *dquot)
{
int ret = 0, ret2 = 0;
unsigned int memalloc;
struct quota_info *dqopt = sb_dqopt(dquot->dq_sb);
mutex_lock(&dquot->dq_lock);
memalloc = memalloc_nofs_save();
if (!test_bit(DQ_READ_B, &dquot->dq_flags)) {
ret = dqopt->ops[dquot->dq_id.type]->read_dqblk(dquot);
if (ret < 0)
goto out_iolock;
}
/* Make sure flags update is visible after dquot has been filled */
smp_mb__before_atomic();
set_bit(DQ_READ_B, &dquot->dq_flags);
/* Instantiate dquot if needed */
if (!dquot_active(dquot) && !dquot->dq_off) {
ret = dqopt->ops[dquot->dq_id.type]->commit_dqblk(dquot);
/* Write the info if needed */
if (info_dirty(&dqopt->info[dquot->dq_id.type])) {
ret2 = dqopt->ops[dquot->dq_id.type]->write_file_info(
dquot->dq_sb, dquot->dq_id.type);
}
if (ret < 0)
goto out_iolock;
if (ret2 < 0) {
ret = ret2;
goto out_iolock;
}
}
/*
* Make sure flags update is visible after on-disk struct has been
* allocated. Paired with smp_rmb() in dqget().
*/
smp_mb__before_atomic();
set_bit(DQ_ACTIVE_B, &dquot->dq_flags);
out_iolock:
memalloc_nofs_restore(memalloc);
mutex_unlock(&dquot->dq_lock);
return ret;
}
EXPORT_SYMBOL(dquot_acquire);
/*
* Write dquot to disk
*/
int dquot_commit(struct dquot *dquot)
{
int ret = 0;
unsigned int memalloc;
struct quota_info *dqopt = sb_dqopt(dquot->dq_sb);
mutex_lock(&dquot->dq_lock);
memalloc = memalloc_nofs_save();
if (!clear_dquot_dirty(dquot))
goto out_lock;
/* Inactive dquot can be only if there was error during read/init
* => we have better not writing it */
if (dquot_active(dquot))
ret = dqopt->ops[dquot->dq_id.type]->commit_dqblk(dquot);
else
ret = -EIO;
out_lock:
memalloc_nofs_restore(memalloc);
mutex_unlock(&dquot->dq_lock);
return ret;
}
EXPORT_SYMBOL(dquot_commit);
/*
* Release dquot
*/
int dquot_release(struct dquot *dquot)
{
int ret = 0, ret2 = 0;
unsigned int memalloc;
struct quota_info *dqopt = sb_dqopt(dquot->dq_sb);
mutex_lock(&dquot->dq_lock);
memalloc = memalloc_nofs_save();
/* Check whether we are not racing with some other dqget() */
if (dquot_is_busy(dquot))
goto out_dqlock;
if (dqopt->ops[dquot->dq_id.type]->release_dqblk) {
ret = dqopt->ops[dquot->dq_id.type]->release_dqblk(dquot);
/* Write the info */
if (info_dirty(&dqopt->info[dquot->dq_id.type])) {
ret2 = dqopt->ops[dquot->dq_id.type]->write_file_info(
dquot->dq_sb, dquot->dq_id.type);
}
if (ret >= 0)
ret = ret2;
}
clear_bit(DQ_ACTIVE_B, &dquot->dq_flags);
out_dqlock:
memalloc_nofs_restore(memalloc);
mutex_unlock(&dquot->dq_lock);
return ret;
}
EXPORT_SYMBOL(dquot_release);
void dquot_destroy(struct dquot *dquot)
{
kmem_cache_free(dquot_cachep, dquot);
}
EXPORT_SYMBOL(dquot_destroy);
static inline void do_destroy_dquot(struct dquot *dquot)
{
dquot->dq_sb->dq_op->destroy_dquot(dquot);
}
/* Invalidate all dquots on the list. Note that this function is called after
* quota is disabled and pointers from inodes removed so there cannot be new
* quota users. There can still be some users of quotas due to inodes being
* just deleted or pruned by prune_icache() (those are not attached to any
* list) or parallel quotactl call. We have to wait for such users.
*/
static void invalidate_dquots(struct super_block *sb, int type)
{
struct dquot *dquot, *tmp;
restart:
flush_delayed_work("a_release_work);
spin_lock(&dq_list_lock);
list_for_each_entry_safe(dquot, tmp, &inuse_list, dq_inuse) {
if (dquot->dq_sb != sb)
continue;
if (dquot->dq_id.type != type)
continue;
/* Wait for dquot users */
if (atomic_read(&dquot->dq_count)) {
atomic_inc(&dquot->dq_count);
spin_unlock(&dq_list_lock);
/*
* Once dqput() wakes us up, we know it's time to free
* the dquot.
* IMPORTANT: we rely on the fact that there is always
* at most one process waiting for dquot to free.
* Otherwise dq_count would be > 1 and we would never
* wake up.
*/
wait_event(dquot_ref_wq,
atomic_read(&dquot->dq_count) == 1);
dqput(dquot);
/* At this moment dquot() need not exist (it could be
* reclaimed by prune_dqcache(). Hence we must
* restart. */
goto restart;
}
/*
* The last user already dropped its reference but dquot didn't
* get fully cleaned up yet. Restart the scan which flushes the
* work cleaning up released dquots.
*/
if (test_bit(DQ_RELEASING_B, &dquot->dq_flags)) {
spin_unlock(&dq_list_lock);
goto restart;
}
/*
* Quota now has no users and it has been written on last
* dqput()
*/
remove_dquot_hash(dquot);
remove_free_dquot(dquot);
remove_inuse(dquot);
do_destroy_dquot(dquot);
}
spin_unlock(&dq_list_lock);
}
/* Call callback for every active dquot on given filesystem */
int dquot_scan_active(struct super_block *sb,
int (*fn)(struct dquot *dquot, unsigned long priv),
unsigned long priv)
{
struct dquot *dquot, *old_dquot = NULL;
int ret = 0;
WARN_ON_ONCE(!rwsem_is_locked(&sb->s_umount));
spin_lock(&dq_list_lock);
list_for_each_entry(dquot, &inuse_list, dq_inuse) {
if (!dquot_active(dquot))
continue;
if (dquot->dq_sb != sb)
continue;
/* Now we have active dquot so we can just increase use count */
atomic_inc(&dquot->dq_count);
spin_unlock(&dq_list_lock);
dqput(old_dquot);
old_dquot = dquot;
/*
* ->release_dquot() can be racing with us. Our reference
* protects us from new calls to it so just wait for any
* outstanding call and recheck the DQ_ACTIVE_B after that.
*/
wait_on_dquot(dquot);
if (dquot_active(dquot)) {
ret = fn(dquot, priv);
if (ret < 0)
goto out;
}
spin_lock(&dq_list_lock);
/* We are safe to continue now because our dquot could not
* be moved out of the inuse list while we hold the reference */
}
spin_unlock(&dq_list_lock);
out:
dqput(old_dquot);
return ret;
}
EXPORT_SYMBOL(dquot_scan_active);
static inline int dquot_write_dquot(struct dquot *dquot)
{
int ret = dquot->dq_sb->dq_op->write_dquot(dquot);
if (ret < 0) {
quota_error(dquot->dq_sb, "Can't write quota structure "
"(error %d). Quota may get out of sync!", ret);
/* Clear dirty bit anyway to avoid infinite loop. */
clear_dquot_dirty(dquot);
}
return ret;
}
/* Write all dquot structures to quota files */
int dquot_writeback_dquots(struct super_block *sb, int type)
{
struct list_head dirty;
struct dquot *dquot;
struct quota_info *dqopt = sb_dqopt(sb);
int cnt;
int err, ret = 0;
WARN_ON_ONCE(!rwsem_is_locked(&sb->s_umount));
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
if (type != -1 && cnt != type)
continue;
if (!sb_has_quota_active(sb, cnt))
continue;
spin_lock(&dq_list_lock);
/* Move list away to avoid livelock. */
list_replace_init(&dqopt->info[cnt].dqi_dirty_list, &dirty);
while (!list_empty(&dirty)) {
dquot = list_first_entry(&dirty, struct dquot,
dq_dirty);
WARN_ON(!dquot_active(dquot));
/* If the dquot is releasing we should not touch it */
if (test_bit(DQ_RELEASING_B, &dquot->dq_flags)) {
spin_unlock(&dq_list_lock);
flush_delayed_work("a_release_work);
spin_lock(&dq_list_lock);
continue;
}
/* Now we have active dquot from which someone is
* holding reference so we can safely just increase
* use count */
dqgrab(dquot);
spin_unlock(&dq_list_lock);
err = dquot_write_dquot(dquot);
if (err && !ret)
ret = err;
dqput(dquot);
spin_lock(&dq_list_lock);
}
spin_unlock(&dq_list_lock);
}
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
if ((cnt == type || type == -1) && sb_has_quota_active(sb, cnt)
&& info_dirty(&dqopt->info[cnt]))
sb->dq_op->write_info(sb, cnt);
dqstats_inc(DQST_SYNCS);
return ret;
}
EXPORT_SYMBOL(dquot_writeback_dquots);
/* Write all dquot structures to disk and make them visible from userspace */
int dquot_quota_sync(struct super_block *sb, int type)
{
struct quota_info *dqopt = sb_dqopt(sb);
int cnt;
int ret;
ret = dquot_writeback_dquots(sb, type);
if (ret)
return ret;
if (dqopt->flags & DQUOT_QUOTA_SYS_FILE)
return 0;
/* This is not very clever (and fast) but currently I don't know about
* any other simple way of getting quota data to disk and we must get
* them there for userspace to be visible... */
if (sb->s_op->sync_fs) {
ret = sb->s_op->sync_fs(sb, 1);
if (ret)
return ret;
}
ret = sync_blockdev(sb->s_bdev);
if (ret)
return ret;
/*
* Now when everything is written we can discard the pagecache so
* that userspace sees the changes.
*/
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
if (type != -1 && cnt != type)
continue;
if (!sb_has_quota_active(sb, cnt))
continue;
inode_lock(dqopt->files[cnt]);
truncate_inode_pages(&dqopt->files[cnt]->i_data, 0);
inode_unlock(dqopt->files[cnt]);
}
return 0;
}
EXPORT_SYMBOL(dquot_quota_sync);
static unsigned long
dqcache_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
{
struct dquot *dquot;
unsigned long freed = 0;
spin_lock(&dq_list_lock);
while (!list_empty(&free_dquots) && sc->nr_to_scan) {
dquot = list_first_entry(&free_dquots, struct dquot, dq_free);
remove_dquot_hash(dquot);
remove_free_dquot(dquot);
remove_inuse(dquot);
do_destroy_dquot(dquot);
sc->nr_to_scan--;
freed++;
}
spin_unlock(&dq_list_lock);
return freed;
}
static unsigned long
dqcache_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
return vfs_pressure_ratio(
percpu_counter_read_positive(&dqstats.counter[DQST_FREE_DQUOTS]));
}
/*
* Safely release dquot and put reference to dquot.
*/
static void quota_release_workfn(struct work_struct *work)
{
struct dquot *dquot;
struct list_head rls_head;
spin_lock(&dq_list_lock);
/* Exchange the list head to avoid livelock. */
list_replace_init(&releasing_dquots, &rls_head);
spin_unlock(&dq_list_lock);
synchronize_srcu(&dquot_srcu);
restart:
spin_lock(&dq_list_lock);
while (!list_empty(&rls_head)) {
dquot = list_first_entry(&rls_head, struct dquot, dq_free);
WARN_ON_ONCE(atomic_read(&dquot->dq_count));
/*
* Note that DQ_RELEASING_B protects us from racing with
* invalidate_dquots() calls so we are safe to work with the
* dquot even after we drop dq_list_lock.
*/
if (dquot_dirty(dquot)) {
spin_unlock(&dq_list_lock);
/* Commit dquot before releasing */
dquot_write_dquot(dquot);
goto restart;
}
if (dquot_active(dquot)) {
spin_unlock(&dq_list_lock);
dquot->dq_sb->dq_op->release_dquot(dquot);
goto restart;
}
/* Dquot is inactive and clean, now move it to free list */
remove_free_dquot(dquot);
put_dquot_last(dquot);
}
spin_unlock(&dq_list_lock);
}
/*
* Put reference to dquot
*/
void dqput(struct dquot *dquot)
{
if (!dquot)
return;
#ifdef CONFIG_QUOTA_DEBUG
if (!atomic_read(&dquot->dq_count)) {
quota_error(dquot->dq_sb, "trying to free free dquot of %s %d",
quotatypes[dquot->dq_id.type],
from_kqid(&init_user_ns, dquot->dq_id));
BUG();
}
#endif
dqstats_inc(DQST_DROPS);
spin_lock(&dq_list_lock);
if (atomic_read(&dquot->dq_count) > 1) {
/* We have more than one user... nothing to do */
atomic_dec(&dquot->dq_count);
/* Releasing dquot during quotaoff phase? */
if (!sb_has_quota_active(dquot->dq_sb, dquot->dq_id.type) &&
atomic_read(&dquot->dq_count) == 1)
wake_up(&dquot_ref_wq);
spin_unlock(&dq_list_lock);
return;
}
/* Need to release dquot? */
WARN_ON_ONCE(!list_empty(&dquot->dq_free));
put_releasing_dquots(dquot);
atomic_dec(&dquot->dq_count);
spin_unlock(&dq_list_lock);
queue_delayed_work(system_unbound_wq, "a_release_work, 1);
}
EXPORT_SYMBOL(dqput);
struct dquot *dquot_alloc(struct super_block *sb, int type)
{
return kmem_cache_zalloc(dquot_cachep, GFP_NOFS);
}
EXPORT_SYMBOL(dquot_alloc);
static struct dquot *get_empty_dquot(struct super_block *sb, int type)
{
struct dquot *dquot;
dquot = sb->dq_op->alloc_dquot(sb, type);
if(!dquot)
return NULL;
mutex_init(&dquot->dq_lock);
INIT_LIST_HEAD(&dquot->dq_free);
INIT_LIST_HEAD(&dquot->dq_inuse);
INIT_HLIST_NODE(&dquot->dq_hash);
INIT_LIST_HEAD(&dquot->dq_dirty);
dquot->dq_sb = sb;
dquot->dq_id = make_kqid_invalid(type);
atomic_set(&dquot->dq_count, 1);
spin_lock_init(&dquot->dq_dqb_lock);
return dquot;
}
/*
* Get reference to dquot
*
* Locking is slightly tricky here. We are guarded from parallel quotaoff()
* destroying our dquot by:
* a) checking for quota flags under dq_list_lock and
* b) getting a reference to dquot before we release dq_list_lock
*/
struct dquot *dqget(struct super_block *sb, struct kqid qid)
{
unsigned int hashent = hashfn(sb, qid);
struct dquot *dquot, *empty = NULL;
if (!qid_has_mapping(sb->s_user_ns, qid))
return ERR_PTR(-EINVAL);
if (!sb_has_quota_active(sb, qid.type))
return ERR_PTR(-ESRCH);
we_slept:
spin_lock(&dq_list_lock);
spin_lock(&dq_state_lock);
if (!sb_has_quota_active(sb, qid.type)) {
spin_unlock(&dq_state_lock);
spin_unlock(&dq_list_lock);
dquot = ERR_PTR(-ESRCH);
goto out;
}
spin_unlock(&dq_state_lock);
dquot = find_dquot(hashent, sb, qid);
if (!dquot) {
if (!empty) {
spin_unlock(&dq_list_lock);
empty = get_empty_dquot(sb, qid.type);
if (!empty)
schedule(); /* Try to wait for a moment... */
goto we_slept;
}
dquot = empty;
empty = NULL;
dquot->dq_id = qid;
/* all dquots go on the inuse_list */
put_inuse(dquot);
/* hash it first so it can be found */
insert_dquot_hash(dquot);
spin_unlock(&dq_list_lock);
dqstats_inc(DQST_LOOKUPS);
} else {
if (!atomic_read(&dquot->dq_count))
remove_free_dquot(dquot);
atomic_inc(&dquot->dq_count);
spin_unlock(&dq_list_lock);
dqstats_inc(DQST_CACHE_HITS);
dqstats_inc(DQST_LOOKUPS);
}
/* Wait for dq_lock - after this we know that either dquot_release() is
* already finished or it will be canceled due to dq_count > 0 test */
wait_on_dquot(dquot);
/* Read the dquot / allocate space in quota file */
if (!dquot_active(dquot)) {
int err;
err = sb->dq_op->acquire_dquot(dquot);
if (err < 0) {
dqput(dquot);
dquot = ERR_PTR(err);
goto out;
}
}
/*
* Make sure following reads see filled structure - paired with
* smp_mb__before_atomic() in dquot_acquire().
*/
smp_rmb();
/* Has somebody invalidated entry under us? */
WARN_ON_ONCE(hlist_unhashed(&dquot->dq_hash));
out:
if (empty)
do_destroy_dquot(empty);
return dquot;
}
EXPORT_SYMBOL(dqget);
static inline struct dquot __rcu **i_dquot(struct inode *inode)
{
return inode->i_sb->s_op->get_dquots(inode);
}
static int dqinit_needed(struct inode *inode, int type)
{
struct dquot __rcu * const *dquots;
int cnt;
if (IS_NOQUOTA(inode))
return 0;
dquots = i_dquot(inode);
if (type != -1)
return !dquots[type];
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
if (!dquots[cnt])
return 1;
return 0;
}
/* This routine is guarded by s_umount semaphore */
static int add_dquot_ref(struct super_block *sb, int type)
{
struct inode *inode, *old_inode = NULL;
#ifdef CONFIG_QUOTA_DEBUG
int reserved = 0;
#endif
int err = 0;
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
spin_lock(&inode->i_lock);
if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
!atomic_read(&inode->i_writecount) ||
!dqinit_needed(inode, type)) {
spin_unlock(&inode->i_lock);
continue;
}
__iget(inode);
spin_unlock(&inode->i_lock);
spin_unlock(&sb->s_inode_list_lock);
#ifdef CONFIG_QUOTA_DEBUG
if (unlikely(inode_get_rsv_space(inode) > 0))
reserved = 1;
#endif
iput(old_inode);
err = __dquot_initialize(inode, type);
if (err) {
iput(inode);
goto out;
}
/*
* We hold a reference to 'inode' so it couldn't have been
* removed from s_inodes list while we dropped the
* s_inode_list_lock. We cannot iput the inode now as we can be
* holding the last reference and we cannot iput it under
* s_inode_list_lock. So we keep the reference and iput it
* later.
*/
old_inode = inode;
cond_resched();
spin_lock(&sb->s_inode_list_lock);
}
spin_unlock(&sb->s_inode_list_lock);
iput(old_inode);
out:
#ifdef CONFIG_QUOTA_DEBUG
if (reserved) {
quota_error(sb, "Writes happened before quota was turned on "
"thus quota information is probably inconsistent. "
"Please run quotacheck(8)");
}
#endif
return err;
}
static void remove_dquot_ref(struct super_block *sb, int type)
{
struct inode *inode;
#ifdef CONFIG_QUOTA_DEBUG
int reserved = 0;
#endif
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
/*
* We have to scan also I_NEW inodes because they can already
* have quota pointer initialized. Luckily, we need to touch
* only quota pointers and these have separate locking
* (dq_data_lock).
*/
spin_lock(&dq_data_lock);
if (!IS_NOQUOTA(inode)) {
struct dquot __rcu **dquots = i_dquot(inode);
struct dquot *dquot = srcu_dereference_check(
dquots[type], &dquot_srcu,
lockdep_is_held(&dq_data_lock));
#ifdef CONFIG_QUOTA_DEBUG
if (unlikely(inode_get_rsv_space(inode) > 0))
reserved = 1;
#endif
rcu_assign_pointer(dquots[type], NULL);
if (dquot)
dqput(dquot);
}
spin_unlock(&dq_data_lock);
}
spin_unlock(&sb->s_inode_list_lock);
#ifdef CONFIG_QUOTA_DEBUG
if (reserved) {
printk(KERN_WARNING "VFS (%s): Writes happened after quota"
" was disabled thus quota information is probably "
"inconsistent. Please run quotacheck(8).\n", sb->s_id);
}
#endif
}
/* Gather all references from inodes and drop them */
static void drop_dquot_ref(struct super_block *sb, int type)
{
if (sb->dq_op)
remove_dquot_ref(sb, type);
}
static inline
void dquot_free_reserved_space(struct dquot *dquot, qsize_t number)
{
if (dquot->dq_dqb.dqb_rsvspace >= number) dquot->dq_dqb.dqb_rsvspace -= number;
else {
WARN_ON_ONCE(1); dquot->dq_dqb.dqb_rsvspace = 0;
}
if (dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace <=
dquot->dq_dqb.dqb_bsoftlimit)
dquot->dq_dqb.dqb_btime = (time64_t) 0; clear_bit(DQ_BLKS_B, &dquot->dq_flags);
}
static void dquot_decr_inodes(struct dquot *dquot, qsize_t number)
{
if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NEGATIVE_USAGE ||
dquot->dq_dqb.dqb_curinodes >= number)
dquot->dq_dqb.dqb_curinodes -= number;
else
dquot->dq_dqb.dqb_curinodes = 0;
if (dquot->dq_dqb.dqb_curinodes <= dquot->dq_dqb.dqb_isoftlimit)
dquot->dq_dqb.dqb_itime = (time64_t) 0;
clear_bit(DQ_INODES_B, &dquot->dq_flags);
}
static void dquot_decr_space(struct dquot *dquot, qsize_t number)
{
if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NEGATIVE_USAGE ||
dquot->dq_dqb.dqb_curspace >= number)
dquot->dq_dqb.dqb_curspace -= number;
else
dquot->dq_dqb.dqb_curspace = 0;
if (dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace <=
dquot->dq_dqb.dqb_bsoftlimit)
dquot->dq_dqb.dqb_btime = (time64_t) 0;
clear_bit(DQ_BLKS_B, &dquot->dq_flags);
}
struct dquot_warn {
struct super_block *w_sb;
struct kqid w_dq_id;
short w_type;
};
static int warning_issued(struct dquot *dquot, const int warntype)
{
int flag = (warntype == QUOTA_NL_BHARDWARN ||
warntype == QUOTA_NL_BSOFTLONGWARN) ? DQ_BLKS_B :
((warntype == QUOTA_NL_IHARDWARN ||
warntype == QUOTA_NL_ISOFTLONGWARN) ? DQ_INODES_B : 0);
if (!flag)
return 0;
return test_and_set_bit(flag, &dquot->dq_flags);
}
#ifdef CONFIG_PRINT_QUOTA_WARNING
static int flag_print_warnings = 1;
static int need_print_warning(struct dquot_warn *warn)
{
if (!flag_print_warnings)
return 0;
switch (warn->w_dq_id.type) {
case USRQUOTA:
return uid_eq(current_fsuid(), warn->w_dq_id.uid);
case GRPQUOTA:
return in_group_p(warn->w_dq_id.gid);
case PRJQUOTA:
return 1;
}
return 0;
}
/* Print warning to user which exceeded quota */
static void print_warning(struct dquot_warn *warn)
{
char *msg = NULL;
struct tty_struct *tty;
int warntype = warn->w_type;
if (warntype == QUOTA_NL_IHARDBELOW ||
warntype == QUOTA_NL_ISOFTBELOW ||
warntype == QUOTA_NL_BHARDBELOW ||
warntype == QUOTA_NL_BSOFTBELOW || !need_print_warning(warn))
return;
tty = get_current_tty();
if (!tty)
return;
tty_write_message(tty, warn->w_sb->s_id);
if (warntype == QUOTA_NL_ISOFTWARN || warntype == QUOTA_NL_BSOFTWARN)
tty_write_message(tty, ": warning, ");
else
tty_write_message(tty, ": write failed, ");
tty_write_message(tty, quotatypes[warn->w_dq_id.type]);
switch (warntype) {
case QUOTA_NL_IHARDWARN:
msg = " file limit reached.\r\n";
break;
case QUOTA_NL_ISOFTLONGWARN:
msg = " file quota exceeded too long.\r\n";
break;
case QUOTA_NL_ISOFTWARN:
msg = " file quota exceeded.\r\n";
break;
case QUOTA_NL_BHARDWARN:
msg = " block limit reached.\r\n";
break;
case QUOTA_NL_BSOFTLONGWARN:
msg = " block quota exceeded too long.\r\n";
break;
case QUOTA_NL_BSOFTWARN:
msg = " block quota exceeded.\r\n";
break;
}
tty_write_message(tty, msg);
tty_kref_put(tty);
}
#endif
static void prepare_warning(struct dquot_warn *warn, struct dquot *dquot,
int warntype)
{
if (warning_issued(dquot, warntype))
return;
warn->w_type = warntype;
warn->w_sb = dquot->dq_sb;
warn->w_dq_id = dquot->dq_id;
}
/*
* Write warnings to the console and send warning messages over netlink.
*
* Note that this function can call into tty and networking code.
*/
static void flush_warnings(struct dquot_warn *warn)
{
int i;
for (i = 0; i < MAXQUOTAS; i++) { if (warn[i].w_type == QUOTA_NL_NOWARN)
continue;
#ifdef CONFIG_PRINT_QUOTA_WARNING
print_warning(&warn[i]);
#endif
quota_send_warning(warn[i].w_dq_id,
warn[i].w_sb->s_dev, warn[i].w_type);
}
}
static int ignore_hardlimit(struct dquot *dquot)
{
struct mem_dqinfo *info = &sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type];
return capable(CAP_SYS_RESOURCE) &&
(info->dqi_format->qf_fmt_id != QFMT_VFS_OLD ||
!(info->dqi_flags & DQF_ROOT_SQUASH));
}
static int dquot_add_inodes(struct dquot *dquot, qsize_t inodes,
struct dquot_warn *warn)
{
qsize_t newinodes;
int ret = 0;
spin_lock(&dquot->dq_dqb_lock);
newinodes = dquot->dq_dqb.dqb_curinodes + inodes;
if (!sb_has_quota_limits_enabled(dquot->dq_sb, dquot->dq_id.type) ||
test_bit(DQ_FAKE_B, &dquot->dq_flags))
goto add;
if (dquot->dq_dqb.dqb_ihardlimit &&
newinodes > dquot->dq_dqb.dqb_ihardlimit &&
!ignore_hardlimit(dquot)) {
prepare_warning(warn, dquot, QUOTA_NL_IHARDWARN);
ret = -EDQUOT;
goto out;
}
if (dquot->dq_dqb.dqb_isoftlimit &&
newinodes > dquot->dq_dqb.dqb_isoftlimit &&
dquot->dq_dqb.dqb_itime &&
ktime_get_real_seconds() >= dquot->dq_dqb.dqb_itime &&
!ignore_hardlimit(dquot)) {
prepare_warning(warn, dquot, QUOTA_NL_ISOFTLONGWARN);
ret = -EDQUOT;
goto out;
}
if (dquot->dq_dqb.dqb_isoftlimit &&
newinodes > dquot->dq_dqb.dqb_isoftlimit &&
dquot->dq_dqb.dqb_itime == 0) {
prepare_warning(warn, dquot, QUOTA_NL_ISOFTWARN);
dquot->dq_dqb.dqb_itime = ktime_get_real_seconds() +
sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type].dqi_igrace;
}
add:
dquot->dq_dqb.dqb_curinodes = newinodes;
out:
spin_unlock(&dquot->dq_dqb_lock);
return ret;
}
static int dquot_add_space(struct dquot *dquot, qsize_t space,
qsize_t rsv_space, unsigned int flags,
struct dquot_warn *warn)
{
qsize_t tspace;
struct super_block *sb = dquot->dq_sb;
int ret = 0;
spin_lock(&dquot->dq_dqb_lock);
if (!sb_has_quota_limits_enabled(sb, dquot->dq_id.type) ||
test_bit(DQ_FAKE_B, &dquot->dq_flags))
goto finish;
tspace = dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace
+ space + rsv_space;
if (dquot->dq_dqb.dqb_bhardlimit &&
tspace > dquot->dq_dqb.dqb_bhardlimit &&
!ignore_hardlimit(dquot)) {
if (flags & DQUOT_SPACE_WARN)
prepare_warning(warn, dquot, QUOTA_NL_BHARDWARN);
ret = -EDQUOT;
goto finish;
}
if (dquot->dq_dqb.dqb_bsoftlimit &&
tspace > dquot->dq_dqb.dqb_bsoftlimit &&
dquot->dq_dqb.dqb_btime &&
ktime_get_real_seconds() >= dquot->dq_dqb.dqb_btime &&
!ignore_hardlimit(dquot)) {
if (flags & DQUOT_SPACE_WARN)
prepare_warning(warn, dquot, QUOTA_NL_BSOFTLONGWARN);
ret = -EDQUOT;
goto finish;
}
if (dquot->dq_dqb.dqb_bsoftlimit &&
tspace > dquot->dq_dqb.dqb_bsoftlimit &&
dquot->dq_dqb.dqb_btime == 0) {
if (flags & DQUOT_SPACE_WARN) {
prepare_warning(warn, dquot, QUOTA_NL_BSOFTWARN);
dquot->dq_dqb.dqb_btime = ktime_get_real_seconds() +
sb_dqopt(sb)->info[dquot->dq_id.type].dqi_bgrace;
} else {
/*
* We don't allow preallocation to exceed softlimit so exceeding will
* be always printed
*/
ret = -EDQUOT;
goto finish;
}
}
finish:
/*
* We have to be careful and go through warning generation & grace time
* setting even if DQUOT_SPACE_NOFAIL is set. That's why we check it
* only here...
*/
if (flags & DQUOT_SPACE_NOFAIL)
ret = 0;
if (!ret) {
dquot->dq_dqb.dqb_rsvspace += rsv_space;
dquot->dq_dqb.dqb_curspace += space;
}
spin_unlock(&dquot->dq_dqb_lock);
return ret;
}
static int info_idq_free(struct dquot *dquot, qsize_t inodes)
{
qsize_t newinodes;
if (test_bit(DQ_FAKE_B, &dquot->dq_flags) ||
dquot->dq_dqb.dqb_curinodes <= dquot->dq_dqb.dqb_isoftlimit ||
!sb_has_quota_limits_enabled(dquot->dq_sb, dquot->dq_id.type))
return QUOTA_NL_NOWARN;
newinodes = dquot->dq_dqb.dqb_curinodes - inodes;
if (newinodes <= dquot->dq_dqb.dqb_isoftlimit)
return QUOTA_NL_ISOFTBELOW;
if (dquot->dq_dqb.dqb_curinodes >= dquot->dq_dqb.dqb_ihardlimit &&
newinodes < dquot->dq_dqb.dqb_ihardlimit)
return QUOTA_NL_IHARDBELOW;
return QUOTA_NL_NOWARN;
}
static int info_bdq_free(struct dquot *dquot, qsize_t space)
{
qsize_t tspace;
tspace = dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace;
if (test_bit(DQ_FAKE_B, &dquot->dq_flags) ||
tspace <= dquot->dq_dqb.dqb_bsoftlimit)
return QUOTA_NL_NOWARN;
if (tspace - space <= dquot->dq_dqb.dqb_bsoftlimit)
return QUOTA_NL_BSOFTBELOW;
if (tspace >= dquot->dq_dqb.dqb_bhardlimit &&
tspace - space < dquot->dq_dqb.dqb_bhardlimit)
return QUOTA_NL_BHARDBELOW;
return QUOTA_NL_NOWARN;
}
static int inode_quota_active(const struct inode *inode)
{
struct super_block *sb = inode->i_sb;
if (IS_NOQUOTA(inode))
return 0;
return sb_any_quota_loaded(sb) & ~sb_any_quota_suspended(sb);
}
/*
* Initialize quota pointers in inode
*
* It is better to call this function outside of any transaction as it
* might need a lot of space in journal for dquot structure allocation.
*/
static int __dquot_initialize(struct inode *inode, int type)
{
int cnt, init_needed = 0;
struct dquot __rcu **dquots;
struct dquot *got[MAXQUOTAS] = {};
struct super_block *sb = inode->i_sb;
qsize_t rsv;
int ret = 0;
if (!inode_quota_active(inode)) return 0; dquots = i_dquot(inode);
/* First get references to structures we might need. */
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
struct kqid qid;
kprojid_t projid;
int rc;
struct dquot *dquot;
if (type != -1 && cnt != type)
continue;
/*
* The i_dquot should have been initialized in most cases,
* we check it without locking here to avoid unnecessary
* dqget()/dqput() calls.
*/
if (dquots[cnt])
continue;
if (!sb_has_quota_active(sb, cnt))
continue;
init_needed = 1;
switch (cnt) {
case USRQUOTA:
qid = make_kqid_uid(inode->i_uid); break;
case GRPQUOTA:
qid = make_kqid_gid(inode->i_gid);
break;
case PRJQUOTA:
rc = inode->i_sb->dq_op->get_projid(inode, &projid);
if (rc)
continue;
qid = make_kqid_projid(projid);
break;
}
dquot = dqget(sb, qid);
if (IS_ERR(dquot)) {
/* We raced with somebody turning quotas off... */
if (PTR_ERR(dquot) != -ESRCH) {
ret = PTR_ERR(dquot);
goto out_put;
}
dquot = NULL;
}
got[cnt] = dquot;
}
/* All required i_dquot has been initialized */
if (!init_needed)
return 0;
spin_lock(&dq_data_lock);
if (IS_NOQUOTA(inode))
goto out_lock;
for (cnt = 0; cnt < MAXQUOTAS; cnt++) { if (type != -1 && cnt != type)
continue;
/* Avoid races with quotaoff() */
if (!sb_has_quota_active(sb, cnt))
continue;
/* We could race with quotaon or dqget() could have failed */
if (!got[cnt])
continue;
if (!dquots[cnt]) { rcu_assign_pointer(dquots[cnt], got[cnt]);
got[cnt] = NULL;
/*
* Make quota reservation system happy if someone
* did a write before quota was turned on
*/
rsv = inode_get_rsv_space(inode);
if (unlikely(rsv)) {
struct dquot *dquot = srcu_dereference_check(
dquots[cnt], &dquot_srcu,
lockdep_is_held(&dq_data_lock));
spin_lock(&inode->i_lock);
/* Get reservation again under proper lock */
rsv = __inode_get_rsv_space(inode); spin_lock(&dquot->dq_dqb_lock);
dquot->dq_dqb.dqb_rsvspace += rsv;
spin_unlock(&dquot->dq_dqb_lock);
spin_unlock(&inode->i_lock);
}
}
}
out_lock:
spin_unlock(&dq_data_lock);
out_put:
/* Drop unused references */
dqput_all(got);
return ret;
}
int dquot_initialize(struct inode *inode)
{
return __dquot_initialize(inode, -1);
}
EXPORT_SYMBOL(dquot_initialize);
bool dquot_initialize_needed(struct inode *inode)
{
struct dquot __rcu **dquots;
int i;
if (!inode_quota_active(inode))
return false;
dquots = i_dquot(inode);
for (i = 0; i < MAXQUOTAS; i++)
if (!dquots[i] && sb_has_quota_active(inode->i_sb, i))
return true;
return false;
}
EXPORT_SYMBOL(dquot_initialize_needed);
/*
* Release all quotas referenced by inode.
*
* This function only be called on inode free or converting
* a file to quota file, no other users for the i_dquot in
* both cases, so we needn't call synchronize_srcu() after
* clearing i_dquot.
*/
static void __dquot_drop(struct inode *inode)
{
int cnt;
struct dquot __rcu **dquots = i_dquot(inode);
struct dquot *put[MAXQUOTAS];
spin_lock(&dq_data_lock);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
put[cnt] = srcu_dereference_check(dquots[cnt], &dquot_srcu,
lockdep_is_held(&dq_data_lock));
rcu_assign_pointer(dquots[cnt], NULL);
}
spin_unlock(&dq_data_lock);
dqput_all(put);
}
void dquot_drop(struct inode *inode)
{
struct dquot __rcu * const *dquots;
int cnt;
if (IS_NOQUOTA(inode))
return;
/*
* Test before calling to rule out calls from proc and such
* where we are not allowed to block. Note that this is
* actually reliable test even without the lock - the caller
* must assure that nobody can come after the DQUOT_DROP and
* add quota pointers back anyway.
*/
dquots = i_dquot(inode);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
if (dquots[cnt])
break;
}
if (cnt < MAXQUOTAS)
__dquot_drop(inode);
}
EXPORT_SYMBOL(dquot_drop);
/*
* inode_reserved_space is managed internally by quota, and protected by
* i_lock similar to i_blocks+i_bytes.
*/
static qsize_t *inode_reserved_space(struct inode * inode)
{
/* Filesystem must explicitly define it's own method in order to use
* quota reservation interface */
BUG_ON(!inode->i_sb->dq_op->get_reserved_space); return inode->i_sb->dq_op->get_reserved_space(inode);
}
static qsize_t __inode_get_rsv_space(struct inode *inode)
{
if (!inode->i_sb->dq_op->get_reserved_space)
return 0;
return *inode_reserved_space(inode);
}
static qsize_t inode_get_rsv_space(struct inode *inode)
{
qsize_t ret;
if (!inode->i_sb->dq_op->get_reserved_space)
return 0;
spin_lock(&inode->i_lock); ret = __inode_get_rsv_space(inode); spin_unlock(&inode->i_lock);
return ret;
}
/*
* This functions updates i_blocks+i_bytes fields and quota information
* (together with appropriate checks).
*
* NOTE: We absolutely rely on the fact that caller dirties the inode
* (usually helpers in quotaops.h care about this) and holds a handle for
* the current transaction so that dquot write and inode write go into the
* same transaction.
*/
/*
* This operation can block, but only after everything is updated
*/
int __dquot_alloc_space(struct inode *inode, qsize_t number, int flags)
{
int cnt, ret = 0, index;
struct dquot_warn warn[MAXQUOTAS];
int reserve = flags & DQUOT_SPACE_RESERVE;
struct dquot __rcu **dquots;
struct dquot *dquot;
if (!inode_quota_active(inode)) { if (reserve) { spin_lock(&inode->i_lock); *inode_reserved_space(inode) += number;
spin_unlock(&inode->i_lock);
} else {
inode_add_bytes(inode, number);
}
goto out;
}
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
warn[cnt].w_type = QUOTA_NL_NOWARN;
dquots = i_dquot(inode);
index = srcu_read_lock(&dquot_srcu);
spin_lock(&inode->i_lock);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) { dquot = srcu_dereference(dquots[cnt], &dquot_srcu); if (!dquot)
continue;
if (reserve) { ret = dquot_add_space(dquot, 0, number, flags, &warn[cnt]);
} else {
ret = dquot_add_space(dquot, number, 0, flags, &warn[cnt]);
}
if (ret) {
/* Back out changes we already did */
for (cnt--; cnt >= 0; cnt--) { dquot = srcu_dereference(dquots[cnt], &dquot_srcu); if (!dquot)
continue;
spin_lock(&dquot->dq_dqb_lock);
if (reserve)
dquot_free_reserved_space(dquot, number);
else
dquot_decr_space(dquot, number); spin_unlock(&dquot->dq_dqb_lock);
}
spin_unlock(&inode->i_lock);
goto out_flush_warn;
}
}
if (reserve) *inode_reserved_space(inode) += number;
else
__inode_add_bytes(inode, number);
spin_unlock(&inode->i_lock);
if (reserve)
goto out_flush_warn;
mark_all_dquot_dirty(dquots);
out_flush_warn:
srcu_read_unlock(&dquot_srcu, index); flush_warnings(warn);
out:
return ret;
}
EXPORT_SYMBOL(__dquot_alloc_space);
/*
* This operation can block, but only after everything is updated
*/
int dquot_alloc_inode(struct inode *inode)
{
int cnt, ret = 0, index;
struct dquot_warn warn[MAXQUOTAS];
struct dquot __rcu * const *dquots;
struct dquot *dquot;
if (!inode_quota_active(inode))
return 0;
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
warn[cnt].w_type = QUOTA_NL_NOWARN;
dquots = i_dquot(inode);
index = srcu_read_lock(&dquot_srcu);
spin_lock(&inode->i_lock);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
dquot = srcu_dereference(dquots[cnt], &dquot_srcu);
if (!dquot)
continue;
ret = dquot_add_inodes(dquot, 1, &warn[cnt]);
if (ret) {
for (cnt--; cnt >= 0; cnt--) {
dquot = srcu_dereference(dquots[cnt], &dquot_srcu);
if (!dquot)
continue;
/* Back out changes we already did */
spin_lock(&dquot->dq_dqb_lock);
dquot_decr_inodes(dquot, 1);
spin_unlock(&dquot->dq_dqb_lock);
}
goto warn_put_all;
}
}
warn_put_all:
spin_unlock(&inode->i_lock);
if (ret == 0)
mark_all_dquot_dirty(dquots);
srcu_read_unlock(&dquot_srcu, index);
flush_warnings(warn);
return ret;
}
EXPORT_SYMBOL(dquot_alloc_inode);
/*
* Convert in-memory reserved quotas to real consumed quotas
*/
void dquot_claim_space_nodirty(struct inode *inode, qsize_t number)
{
struct dquot __rcu **dquots;
struct dquot *dquot;
int cnt, index;
if (!inode_quota_active(inode)) {
spin_lock(&inode->i_lock);
*inode_reserved_space(inode) -= number;
__inode_add_bytes(inode, number);
spin_unlock(&inode->i_lock);
return;
}
dquots = i_dquot(inode);
index = srcu_read_lock(&dquot_srcu);
spin_lock(&inode->i_lock);
/* Claim reserved quotas to allocated quotas */
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
dquot = srcu_dereference(dquots[cnt], &dquot_srcu);
if (dquot) {
spin_lock(&dquot->dq_dqb_lock);
if (WARN_ON_ONCE(dquot->dq_dqb.dqb_rsvspace < number))
number = dquot->dq_dqb.dqb_rsvspace;
dquot->dq_dqb.dqb_curspace += number;
dquot->dq_dqb.dqb_rsvspace -= number;
spin_unlock(&dquot->dq_dqb_lock);
}
}
/* Update inode bytes */
*inode_reserved_space(inode) -= number;
__inode_add_bytes(inode, number);
spin_unlock(&inode->i_lock);
mark_all_dquot_dirty(dquots);
srcu_read_unlock(&dquot_srcu, index);
return;
}
EXPORT_SYMBOL(dquot_claim_space_nodirty);
/*
* Convert allocated space back to in-memory reserved quotas
*/
void dquot_reclaim_space_nodirty(struct inode *inode, qsize_t number)
{
struct dquot __rcu **dquots;
struct dquot *dquot;
int cnt, index;
if (!inode_quota_active(inode)) {
spin_lock(&inode->i_lock);
*inode_reserved_space(inode) += number;
__inode_sub_bytes(inode, number);
spin_unlock(&inode->i_lock);
return;
}
dquots = i_dquot(inode);
index = srcu_read_lock(&dquot_srcu);
spin_lock(&inode->i_lock);
/* Claim reserved quotas to allocated quotas */
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
dquot = srcu_dereference(dquots[cnt], &dquot_srcu);
if (dquot) {
spin_lock(&dquot->dq_dqb_lock);
if (WARN_ON_ONCE(dquot->dq_dqb.dqb_curspace < number))
number = dquot->dq_dqb.dqb_curspace;
dquot->dq_dqb.dqb_rsvspace += number;
dquot->dq_dqb.dqb_curspace -= number;
spin_unlock(&dquot->dq_dqb_lock);
}
}
/* Update inode bytes */
*inode_reserved_space(inode) += number;
__inode_sub_bytes(inode, number);
spin_unlock(&inode->i_lock);
mark_all_dquot_dirty(dquots);
srcu_read_unlock(&dquot_srcu, index);
return;
}
EXPORT_SYMBOL(dquot_reclaim_space_nodirty);
/*
* This operation can block, but only after everything is updated
*/
void __dquot_free_space(struct inode *inode, qsize_t number, int flags)
{
unsigned int cnt;
struct dquot_warn warn[MAXQUOTAS];
struct dquot __rcu **dquots;
struct dquot *dquot;
int reserve = flags & DQUOT_SPACE_RESERVE, index;
if (!inode_quota_active(inode)) { if (reserve) { spin_lock(&inode->i_lock); *inode_reserved_space(inode) -= number;
spin_unlock(&inode->i_lock);
} else {
inode_sub_bytes(inode, number);
}
return;
}
dquots = i_dquot(inode);
index = srcu_read_lock(&dquot_srcu);
spin_lock(&inode->i_lock);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
int wtype;
warn[cnt].w_type = QUOTA_NL_NOWARN; dquot = srcu_dereference(dquots[cnt], &dquot_srcu); if (!dquot)
continue;
spin_lock(&dquot->dq_dqb_lock);
wtype = info_bdq_free(dquot, number);
if (wtype != QUOTA_NL_NOWARN)
prepare_warning(&warn[cnt], dquot, wtype); if (reserve) dquot_free_reserved_space(dquot, number);
else
dquot_decr_space(dquot, number); spin_unlock(&dquot->dq_dqb_lock);
}
if (reserve) *inode_reserved_space(inode) -= number;
else
__inode_sub_bytes(inode, number);
spin_unlock(&inode->i_lock);
if (reserve)
goto out_unlock;
mark_all_dquot_dirty(dquots);
out_unlock:
srcu_read_unlock(&dquot_srcu, index); flush_warnings(warn);
}
EXPORT_SYMBOL(__dquot_free_space);
/*
* This operation can block, but only after everything is updated
*/
void dquot_free_inode(struct inode *inode)
{
unsigned int cnt;
struct dquot_warn warn[MAXQUOTAS];
struct dquot __rcu * const *dquots;
struct dquot *dquot;
int index;
if (!inode_quota_active(inode))
return;
dquots = i_dquot(inode);
index = srcu_read_lock(&dquot_srcu);
spin_lock(&inode->i_lock);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
int wtype;
warn[cnt].w_type = QUOTA_NL_NOWARN;
dquot = srcu_dereference(dquots[cnt], &dquot_srcu);
if (!dquot)
continue;
spin_lock(&dquot->dq_dqb_lock);
wtype = info_idq_free(dquot, 1);
if (wtype != QUOTA_NL_NOWARN)
prepare_warning(&warn[cnt], dquot, wtype);
dquot_decr_inodes(dquot, 1);
spin_unlock(&dquot->dq_dqb_lock);
}
spin_unlock(&inode->i_lock);
mark_all_dquot_dirty(dquots);
srcu_read_unlock(&dquot_srcu, index);
flush_warnings(warn);
}
EXPORT_SYMBOL(dquot_free_inode);
/*
* Transfer the number of inode and blocks from one diskquota to an other.
* On success, dquot references in transfer_to are consumed and references
* to original dquots that need to be released are placed there. On failure,
* references are kept untouched.
*
* This operation can block, but only after everything is updated
* A transaction must be started when entering this function.
*
* We are holding reference on transfer_from & transfer_to, no need to
* protect them by srcu_read_lock().
*/
int __dquot_transfer(struct inode *inode, struct dquot **transfer_to)
{
qsize_t cur_space;
qsize_t rsv_space = 0;
qsize_t inode_usage = 1;
struct dquot __rcu **dquots;
struct dquot *transfer_from[MAXQUOTAS] = {};
int cnt, index, ret = 0;
char is_valid[MAXQUOTAS] = {};
struct dquot_warn warn_to[MAXQUOTAS];
struct dquot_warn warn_from_inodes[MAXQUOTAS];
struct dquot_warn warn_from_space[MAXQUOTAS];
if (IS_NOQUOTA(inode))
return 0;
if (inode->i_sb->dq_op->get_inode_usage) {
ret = inode->i_sb->dq_op->get_inode_usage(inode, &inode_usage);
if (ret)
return ret;
}
/* Initialize the arrays */
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
warn_to[cnt].w_type = QUOTA_NL_NOWARN;
warn_from_inodes[cnt].w_type = QUOTA_NL_NOWARN;
warn_from_space[cnt].w_type = QUOTA_NL_NOWARN;
}
spin_lock(&dq_data_lock);
spin_lock(&inode->i_lock);
if (IS_NOQUOTA(inode)) { /* File without quota accounting? */
spin_unlock(&inode->i_lock);
spin_unlock(&dq_data_lock);
return 0;
}
cur_space = __inode_get_bytes(inode);
rsv_space = __inode_get_rsv_space(inode);
dquots = i_dquot(inode);
/*
* Build the transfer_from list, check limits, and update usage in
* the target structures.
*/
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
/*
* Skip changes for same uid or gid or for turned off quota-type.
*/
if (!transfer_to[cnt])
continue;
/* Avoid races with quotaoff() */
if (!sb_has_quota_active(inode->i_sb, cnt))
continue;
is_valid[cnt] = 1;
transfer_from[cnt] = srcu_dereference_check(dquots[cnt],
&dquot_srcu, lockdep_is_held(&dq_data_lock));
ret = dquot_add_inodes(transfer_to[cnt], inode_usage,
&warn_to[cnt]);
if (ret)
goto over_quota;
ret = dquot_add_space(transfer_to[cnt], cur_space, rsv_space,
DQUOT_SPACE_WARN, &warn_to[cnt]);
if (ret) {
spin_lock(&transfer_to[cnt]->dq_dqb_lock);
dquot_decr_inodes(transfer_to[cnt], inode_usage);
spin_unlock(&transfer_to[cnt]->dq_dqb_lock);
goto over_quota;
}
}
/* Decrease usage for source structures and update quota pointers */
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
if (!is_valid[cnt])
continue;
/* Due to IO error we might not have transfer_from[] structure */
if (transfer_from[cnt]) {
int wtype;
spin_lock(&transfer_from[cnt]->dq_dqb_lock);
wtype = info_idq_free(transfer_from[cnt], inode_usage);
if (wtype != QUOTA_NL_NOWARN)
prepare_warning(&warn_from_inodes[cnt],
transfer_from[cnt], wtype);
wtype = info_bdq_free(transfer_from[cnt],
cur_space + rsv_space);
if (wtype != QUOTA_NL_NOWARN)
prepare_warning(&warn_from_space[cnt],
transfer_from[cnt], wtype);
dquot_decr_inodes(transfer_from[cnt], inode_usage);
dquot_decr_space(transfer_from[cnt], cur_space);
dquot_free_reserved_space(transfer_from[cnt],
rsv_space);
spin_unlock(&transfer_from[cnt]->dq_dqb_lock);
}
rcu_assign_pointer(dquots[cnt], transfer_to[cnt]);
}
spin_unlock(&inode->i_lock);
spin_unlock(&dq_data_lock);
/*
* These arrays are local and we hold dquot references so we don't need
* the srcu protection but still take dquot_srcu to avoid warning in
* mark_all_dquot_dirty().
*/
index = srcu_read_lock(&dquot_srcu);
mark_all_dquot_dirty((struct dquot __rcu **)transfer_from);
mark_all_dquot_dirty((struct dquot __rcu **)transfer_to);
srcu_read_unlock(&dquot_srcu, index);
flush_warnings(warn_to);
flush_warnings(warn_from_inodes);
flush_warnings(warn_from_space);
/* Pass back references to put */
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
if (is_valid[cnt])
transfer_to[cnt] = transfer_from[cnt];
return 0;
over_quota:
/* Back out changes we already did */
for (cnt--; cnt >= 0; cnt--) {
if (!is_valid[cnt])
continue;
spin_lock(&transfer_to[cnt]->dq_dqb_lock);
dquot_decr_inodes(transfer_to[cnt], inode_usage);
dquot_decr_space(transfer_to[cnt], cur_space);
dquot_free_reserved_space(transfer_to[cnt], rsv_space);
spin_unlock(&transfer_to[cnt]->dq_dqb_lock);
}
spin_unlock(&inode->i_lock);
spin_unlock(&dq_data_lock);
flush_warnings(warn_to);
return ret;
}
EXPORT_SYMBOL(__dquot_transfer);
/* Wrapper for transferring ownership of an inode for uid/gid only
* Called from FSXXX_setattr()
*/
int dquot_transfer(struct mnt_idmap *idmap, struct inode *inode,
struct iattr *iattr)
{
struct dquot *transfer_to[MAXQUOTAS] = {};
struct dquot *dquot;
struct super_block *sb = inode->i_sb;
int ret;
if (!inode_quota_active(inode))
return 0;
if (i_uid_needs_update(idmap, iattr, inode)) {
kuid_t kuid = from_vfsuid(idmap, i_user_ns(inode),
iattr->ia_vfsuid);
dquot = dqget(sb, make_kqid_uid(kuid));
if (IS_ERR(dquot)) {
if (PTR_ERR(dquot) != -ESRCH) {
ret = PTR_ERR(dquot);
goto out_put;
}
dquot = NULL;
}
transfer_to[USRQUOTA] = dquot;
}
if (i_gid_needs_update(idmap, iattr, inode)) {
kgid_t kgid = from_vfsgid(idmap, i_user_ns(inode),
iattr->ia_vfsgid);
dquot = dqget(sb, make_kqid_gid(kgid));
if (IS_ERR(dquot)) {
if (PTR_ERR(dquot) != -ESRCH) {
ret = PTR_ERR(dquot);
goto out_put;
}
dquot = NULL;
}
transfer_to[GRPQUOTA] = dquot;
}
ret = __dquot_transfer(inode, transfer_to);
out_put:
dqput_all(transfer_to);
return ret;
}
EXPORT_SYMBOL(dquot_transfer);
/*
* Write info of quota file to disk
*/
int dquot_commit_info(struct super_block *sb, int type)
{
struct quota_info *dqopt = sb_dqopt(sb);
return dqopt->ops[type]->write_file_info(sb, type);
}
EXPORT_SYMBOL(dquot_commit_info);
int dquot_get_next_id(struct super_block *sb, struct kqid *qid)
{
struct quota_info *dqopt = sb_dqopt(sb);
if (!sb_has_quota_active(sb, qid->type))
return -ESRCH;
if (!dqopt->ops[qid->type]->get_next_id)
return -ENOSYS;
return dqopt->ops[qid->type]->get_next_id(sb, qid);
}
EXPORT_SYMBOL(dquot_get_next_id);
/*
* Definitions of diskquota operations.
*/
const struct dquot_operations dquot_operations = {
.write_dquot = dquot_commit,
.acquire_dquot = dquot_acquire,
.release_dquot = dquot_release,
.mark_dirty = dquot_mark_dquot_dirty,
.write_info = dquot_commit_info,
.alloc_dquot = dquot_alloc,
.destroy_dquot = dquot_destroy,
.get_next_id = dquot_get_next_id,
};
EXPORT_SYMBOL(dquot_operations);
/*
* Generic helper for ->open on filesystems supporting disk quotas.
*/
int dquot_file_open(struct inode *inode, struct file *file)
{
int error;
error = generic_file_open(inode, file);
if (!error && (file->f_mode & FMODE_WRITE))
error = dquot_initialize(inode);
return error;
}
EXPORT_SYMBOL(dquot_file_open);
static void vfs_cleanup_quota_inode(struct super_block *sb, int type)
{
struct quota_info *dqopt = sb_dqopt(sb);
struct inode *inode = dqopt->files[type];
if (!inode)
return;
if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) {
inode_lock(inode);
inode->i_flags &= ~S_NOQUOTA;
inode_unlock(inode);
}
dqopt->files[type] = NULL;
iput(inode);
}
/*
* Turn quota off on a device. type == -1 ==> quotaoff for all types (umount)
*/
int dquot_disable(struct super_block *sb, int type, unsigned int flags)
{
int cnt;
struct quota_info *dqopt = sb_dqopt(sb);
/* s_umount should be held in exclusive mode */
if (WARN_ON_ONCE(down_read_trylock(&sb->s_umount)))
up_read(&sb->s_umount);
/* Cannot turn off usage accounting without turning off limits, or
* suspend quotas and simultaneously turn quotas off. */
if ((flags & DQUOT_USAGE_ENABLED && !(flags & DQUOT_LIMITS_ENABLED))
|| (flags & DQUOT_SUSPENDED && flags & (DQUOT_LIMITS_ENABLED |
DQUOT_USAGE_ENABLED)))
return -EINVAL;
/*
* Skip everything if there's nothing to do. We have to do this because
* sometimes we are called when fill_super() failed and calling
* sync_fs() in such cases does no good.
*/
if (!sb_any_quota_loaded(sb))
return 0;
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
if (type != -1 && cnt != type)
continue;
if (!sb_has_quota_loaded(sb, cnt))
continue;
if (flags & DQUOT_SUSPENDED) {
spin_lock(&dq_state_lock);
dqopt->flags |=
dquot_state_flag(DQUOT_SUSPENDED, cnt);
spin_unlock(&dq_state_lock);
} else {
spin_lock(&dq_state_lock);
dqopt->flags &= ~dquot_state_flag(flags, cnt);
/* Turning off suspended quotas? */
if (!sb_has_quota_loaded(sb, cnt) &&
sb_has_quota_suspended(sb, cnt)) {
dqopt->flags &= ~dquot_state_flag(
DQUOT_SUSPENDED, cnt);
spin_unlock(&dq_state_lock);
vfs_cleanup_quota_inode(sb, cnt);
continue;
}
spin_unlock(&dq_state_lock);
}
/* We still have to keep quota loaded? */
if (sb_has_quota_loaded(sb, cnt) && !(flags & DQUOT_SUSPENDED))
continue;
/* Note: these are blocking operations */
drop_dquot_ref(sb, cnt);
invalidate_dquots(sb, cnt);
/*
* Now all dquots should be invalidated, all writes done so we
* should be only users of the info. No locks needed.
*/
if (info_dirty(&dqopt->info[cnt]))
sb->dq_op->write_info(sb, cnt);
if (dqopt->ops[cnt]->free_file_info)
dqopt->ops[cnt]->free_file_info(sb, cnt);
put_quota_format(dqopt->info[cnt].dqi_format);
dqopt->info[cnt].dqi_flags = 0;
dqopt->info[cnt].dqi_igrace = 0;
dqopt->info[cnt].dqi_bgrace = 0;
dqopt->ops[cnt] = NULL;
}
/* Skip syncing and setting flags if quota files are hidden */
if (dqopt->flags & DQUOT_QUOTA_SYS_FILE)
goto put_inodes;
/* Sync the superblock so that buffers with quota data are written to
* disk (and so userspace sees correct data afterwards). */
if (sb->s_op->sync_fs)
sb->s_op->sync_fs(sb, 1);
sync_blockdev(sb->s_bdev);
/* Now the quota files are just ordinary files and we can set the
* inode flags back. Moreover we discard the pagecache so that
* userspace sees the writes we did bypassing the pagecache. We
* must also discard the blockdev buffers so that we see the
* changes done by userspace on the next quotaon() */
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
if (!sb_has_quota_loaded(sb, cnt) && dqopt->files[cnt]) {
inode_lock(dqopt->files[cnt]);
truncate_inode_pages(&dqopt->files[cnt]->i_data, 0);
inode_unlock(dqopt->files[cnt]);
}
if (sb->s_bdev)
invalidate_bdev(sb->s_bdev);
put_inodes:
/* We are done when suspending quotas */
if (flags & DQUOT_SUSPENDED)
return 0;
for (cnt = 0; cnt < MAXQUOTAS; cnt++)
if (!sb_has_quota_loaded(sb, cnt))
vfs_cleanup_quota_inode(sb, cnt);
return 0;
}
EXPORT_SYMBOL(dquot_disable);
int dquot_quota_off(struct super_block *sb, int type)
{
return dquot_disable(sb, type,
DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED);
}
EXPORT_SYMBOL(dquot_quota_off);
/*
* Turn quotas on on a device
*/
static int vfs_setup_quota_inode(struct inode *inode, int type)
{
struct super_block *sb = inode->i_sb;
struct quota_info *dqopt = sb_dqopt(sb);
if (is_bad_inode(inode))
return -EUCLEAN;
if (!S_ISREG(inode->i_mode))
return -EACCES;
if (IS_RDONLY(inode))
return -EROFS;
if (sb_has_quota_loaded(sb, type))
return -EBUSY;
/*
* Quota files should never be encrypted. They should be thought of as
* filesystem metadata, not user data. New-style internal quota files
* cannot be encrypted by users anyway, but old-style external quota
* files could potentially be incorrectly created in an encrypted
* directory, hence this explicit check. Some reasons why encrypted
* quota files don't work include: (1) some filesystems that support
* encryption don't handle it in their quota_read and quota_write, and
* (2) cleaning up encrypted quota files at unmount would need special
* consideration, as quota files are cleaned up later than user files.
*/
if (IS_ENCRYPTED(inode))
return -EINVAL;
dqopt->files[type] = igrab(inode);
if (!dqopt->files[type])
return -EIO;
if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) {
/* We don't want quota and atime on quota files (deadlocks
* possible) Also nobody should write to the file - we use
* special IO operations which ignore the immutable bit. */
inode_lock(inode);
inode->i_flags |= S_NOQUOTA;
inode_unlock(inode);
/*
* When S_NOQUOTA is set, remove dquot references as no more
* references can be added
*/
__dquot_drop(inode);
}
return 0;
}
int dquot_load_quota_sb(struct super_block *sb, int type, int format_id,
unsigned int flags)
{
struct quota_format_type *fmt = find_quota_format(format_id);
struct quota_info *dqopt = sb_dqopt(sb);
int error;
lockdep_assert_held_write(&sb->s_umount);
/* Just unsuspend quotas? */
if (WARN_ON_ONCE(flags & DQUOT_SUSPENDED))
return -EINVAL;
if (!fmt)
return -ESRCH;
if (!sb->dq_op || !sb->s_qcop ||
(type == PRJQUOTA && sb->dq_op->get_projid == NULL)) {
error = -EINVAL;
goto out_fmt;
}
/* Filesystems outside of init_user_ns not yet supported */
if (sb->s_user_ns != &init_user_ns) {
error = -EINVAL;
goto out_fmt;
}
/* Usage always has to be set... */
if (!(flags & DQUOT_USAGE_ENABLED)) {
error = -EINVAL;
goto out_fmt;
}
if (sb_has_quota_loaded(sb, type)) {
error = -EBUSY;
goto out_fmt;
}
if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) {
/* As we bypass the pagecache we must now flush all the
* dirty data and invalidate caches so that kernel sees
* changes from userspace. It is not enough to just flush
* the quota file since if blocksize < pagesize, invalidation
* of the cache could fail because of other unrelated dirty
* data */
sync_filesystem(sb);
invalidate_bdev(sb->s_bdev);
}
error = -EINVAL;
if (!fmt->qf_ops->check_quota_file(sb, type))
goto out_fmt;
dqopt->ops[type] = fmt->qf_ops;
dqopt->info[type].dqi_format = fmt;
dqopt->info[type].dqi_fmt_id = format_id;
INIT_LIST_HEAD(&dqopt->info[type].dqi_dirty_list);
error = dqopt->ops[type]->read_file_info(sb, type);
if (error < 0)
goto out_fmt;
if (dqopt->flags & DQUOT_QUOTA_SYS_FILE) {
spin_lock(&dq_data_lock);
dqopt->info[type].dqi_flags |= DQF_SYS_FILE;
spin_unlock(&dq_data_lock);
}
spin_lock(&dq_state_lock);
dqopt->flags |= dquot_state_flag(flags, type);
spin_unlock(&dq_state_lock);
error = add_dquot_ref(sb, type);
if (error)
dquot_disable(sb, type,
DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED);
return error;
out_fmt:
put_quota_format(fmt);
return error;
}
EXPORT_SYMBOL(dquot_load_quota_sb);
/*
* More powerful function for turning on quotas on given quota inode allowing
* setting of individual quota flags
*/
int dquot_load_quota_inode(struct inode *inode, int type, int format_id,
unsigned int flags)
{
int err;
err = vfs_setup_quota_inode(inode, type);
if (err < 0)
return err;
err = dquot_load_quota_sb(inode->i_sb, type, format_id, flags);
if (err < 0)
vfs_cleanup_quota_inode(inode->i_sb, type);
return err;
}
EXPORT_SYMBOL(dquot_load_quota_inode);
/* Reenable quotas on remount RW */
int dquot_resume(struct super_block *sb, int type)
{
struct quota_info *dqopt = sb_dqopt(sb);
int ret = 0, cnt;
unsigned int flags;
/* s_umount should be held in exclusive mode */
if (WARN_ON_ONCE(down_read_trylock(&sb->s_umount)))
up_read(&sb->s_umount);
for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
if (type != -1 && cnt != type)
continue;
if (!sb_has_quota_suspended(sb, cnt))
continue;
spin_lock(&dq_state_lock);
flags = dqopt->flags & dquot_state_flag(DQUOT_USAGE_ENABLED |
DQUOT_LIMITS_ENABLED,
cnt);
dqopt->flags &= ~dquot_state_flag(DQUOT_STATE_FLAGS, cnt);
spin_unlock(&dq_state_lock);
flags = dquot_generic_flag(flags, cnt);
ret = dquot_load_quota_sb(sb, cnt, dqopt->info[cnt].dqi_fmt_id,
flags);
if (ret < 0)
vfs_cleanup_quota_inode(sb, cnt);
}
return ret;
}
EXPORT_SYMBOL(dquot_resume);
int dquot_quota_on(struct super_block *sb, int type, int format_id,
const struct path *path)
{
int error = security_quota_on(path->dentry);
if (error)
return error;
/* Quota file not on the same filesystem? */
if (path->dentry->d_sb != sb)
error = -EXDEV;
else
error = dquot_load_quota_inode(d_inode(path->dentry), type,
format_id, DQUOT_USAGE_ENABLED |
DQUOT_LIMITS_ENABLED);
return error;
}
EXPORT_SYMBOL(dquot_quota_on);
/*
* This function is used when filesystem needs to initialize quotas
* during mount time.
*/
int dquot_quota_on_mount(struct super_block *sb, char *qf_name,
int format_id, int type)
{
struct dentry *dentry;
int error;
dentry = lookup_positive_unlocked(qf_name, sb->s_root, strlen(qf_name));
if (IS_ERR(dentry))
return PTR_ERR(dentry);
error = security_quota_on(dentry);
if (!error)
error = dquot_load_quota_inode(d_inode(dentry), type, format_id,
DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED);
dput(dentry);
return error;
}
EXPORT_SYMBOL(dquot_quota_on_mount);
static int dquot_quota_enable(struct super_block *sb, unsigned int flags)
{
int ret;
int type;
struct quota_info *dqopt = sb_dqopt(sb);
if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE))
return -ENOSYS;
/* Accounting cannot be turned on while fs is mounted */
flags &= ~(FS_QUOTA_UDQ_ACCT | FS_QUOTA_GDQ_ACCT | FS_QUOTA_PDQ_ACCT);
if (!flags)
return -EINVAL;
for (type = 0; type < MAXQUOTAS; type++) {
if (!(flags & qtype_enforce_flag(type)))
continue;
/* Can't enforce without accounting */
if (!sb_has_quota_usage_enabled(sb, type)) {
ret = -EINVAL;
goto out_err;
}
if (sb_has_quota_limits_enabled(sb, type)) {
ret = -EBUSY;
goto out_err;
}
spin_lock(&dq_state_lock);
dqopt->flags |= dquot_state_flag(DQUOT_LIMITS_ENABLED, type);
spin_unlock(&dq_state_lock);
}
return 0;
out_err:
/* Backout enforcement enablement we already did */
for (type--; type >= 0; type--) {
if (flags & qtype_enforce_flag(type))
dquot_disable(sb, type, DQUOT_LIMITS_ENABLED);
}
/* Error code translation for better compatibility with XFS */
if (ret == -EBUSY)
ret = -EEXIST;
return ret;
}
static int dquot_quota_disable(struct super_block *sb, unsigned int flags)
{
int ret;
int type;
struct quota_info *dqopt = sb_dqopt(sb);
if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE))
return -ENOSYS;
/*
* We don't support turning off accounting via quotactl. In principle
* quota infrastructure can do this but filesystems don't expect
* userspace to be able to do it.
*/
if (flags &
(FS_QUOTA_UDQ_ACCT | FS_QUOTA_GDQ_ACCT | FS_QUOTA_PDQ_ACCT))
return -EOPNOTSUPP;
/* Filter out limits not enabled */
for (type = 0; type < MAXQUOTAS; type++)
if (!sb_has_quota_limits_enabled(sb, type))
flags &= ~qtype_enforce_flag(type);
/* Nothing left? */
if (!flags)
return -EEXIST;
for (type = 0; type < MAXQUOTAS; type++) {
if (flags & qtype_enforce_flag(type)) {
ret = dquot_disable(sb, type, DQUOT_LIMITS_ENABLED);
if (ret < 0)
goto out_err;
}
}
return 0;
out_err:
/* Backout enforcement disabling we already did */
for (type--; type >= 0; type--) {
if (flags & qtype_enforce_flag(type)) {
spin_lock(&dq_state_lock);
dqopt->flags |=
dquot_state_flag(DQUOT_LIMITS_ENABLED, type);
spin_unlock(&dq_state_lock);
}
}
return ret;
}
/* Generic routine for getting common part of quota structure */
static void do_get_dqblk(struct dquot *dquot, struct qc_dqblk *di)
{
struct mem_dqblk *dm = &dquot->dq_dqb;
memset(di, 0, sizeof(*di));
spin_lock(&dquot->dq_dqb_lock);
di->d_spc_hardlimit = dm->dqb_bhardlimit;
di->d_spc_softlimit = dm->dqb_bsoftlimit;
di->d_ino_hardlimit = dm->dqb_ihardlimit;
di->d_ino_softlimit = dm->dqb_isoftlimit;
di->d_space = dm->dqb_curspace + dm->dqb_rsvspace;
di->d_ino_count = dm->dqb_curinodes;
di->d_spc_timer = dm->dqb_btime;
di->d_ino_timer = dm->dqb_itime;
spin_unlock(&dquot->dq_dqb_lock);
}
int dquot_get_dqblk(struct super_block *sb, struct kqid qid,
struct qc_dqblk *di)
{
struct dquot *dquot;
dquot = dqget(sb, qid);
if (IS_ERR(dquot))
return PTR_ERR(dquot);
do_get_dqblk(dquot, di);
dqput(dquot);
return 0;
}
EXPORT_SYMBOL(dquot_get_dqblk);
int dquot_get_next_dqblk(struct super_block *sb, struct kqid *qid,
struct qc_dqblk *di)
{
struct dquot *dquot;
int err;
if (!sb->dq_op->get_next_id)
return -ENOSYS;
err = sb->dq_op->get_next_id(sb, qid);
if (err < 0)
return err;
dquot = dqget(sb, *qid);
if (IS_ERR(dquot))
return PTR_ERR(dquot);
do_get_dqblk(dquot, di);
dqput(dquot);
return 0;
}
EXPORT_SYMBOL(dquot_get_next_dqblk);
#define VFS_QC_MASK \
(QC_SPACE | QC_SPC_SOFT | QC_SPC_HARD | \
QC_INO_COUNT | QC_INO_SOFT | QC_INO_HARD | \
QC_SPC_TIMER | QC_INO_TIMER)
/* Generic routine for setting common part of quota structure */
static int do_set_dqblk(struct dquot *dquot, struct qc_dqblk *di)
{
struct mem_dqblk *dm = &dquot->dq_dqb;
int check_blim = 0, check_ilim = 0;
struct mem_dqinfo *dqi = &sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type];
if (di->d_fieldmask & ~VFS_QC_MASK)
return -EINVAL;
if (((di->d_fieldmask & QC_SPC_SOFT) &&
di->d_spc_softlimit > dqi->dqi_max_spc_limit) ||
((di->d_fieldmask & QC_SPC_HARD) &&
di->d_spc_hardlimit > dqi->dqi_max_spc_limit) ||
((di->d_fieldmask & QC_INO_SOFT) &&
(di->d_ino_softlimit > dqi->dqi_max_ino_limit)) ||
((di->d_fieldmask & QC_INO_HARD) &&
(di->d_ino_hardlimit > dqi->dqi_max_ino_limit)))
return -ERANGE;
spin_lock(&dquot->dq_dqb_lock);
if (di->d_fieldmask & QC_SPACE) {
dm->dqb_curspace = di->d_space - dm->dqb_rsvspace;
check_blim = 1;
set_bit(DQ_LASTSET_B + QIF_SPACE_B, &dquot->dq_flags);
}
if (di->d_fieldmask & QC_SPC_SOFT)
dm->dqb_bsoftlimit = di->d_spc_softlimit;
if (di->d_fieldmask & QC_SPC_HARD)
dm->dqb_bhardlimit = di->d_spc_hardlimit;
if (di->d_fieldmask & (QC_SPC_SOFT | QC_SPC_HARD)) {
check_blim = 1;
set_bit(DQ_LASTSET_B + QIF_BLIMITS_B, &dquot->dq_flags);
}
if (di->d_fieldmask & QC_INO_COUNT) {
dm->dqb_curinodes = di->d_ino_count;
check_ilim = 1;
set_bit(DQ_LASTSET_B + QIF_INODES_B, &dquot->dq_flags);
}
if (di->d_fieldmask & QC_INO_SOFT)
dm->dqb_isoftlimit = di->d_ino_softlimit;
if (di->d_fieldmask & QC_INO_HARD)
dm->dqb_ihardlimit = di->d_ino_hardlimit;
if (di->d_fieldmask & (QC_INO_SOFT | QC_INO_HARD)) {
check_ilim = 1;
set_bit(DQ_LASTSET_B + QIF_ILIMITS_B, &dquot->dq_flags);
}
if (di->d_fieldmask & QC_SPC_TIMER) {
dm->dqb_btime = di->d_spc_timer;
check_blim = 1;
set_bit(DQ_LASTSET_B + QIF_BTIME_B, &dquot->dq_flags);
}
if (di->d_fieldmask & QC_INO_TIMER) {
dm->dqb_itime = di->d_ino_timer;
check_ilim = 1;
set_bit(DQ_LASTSET_B + QIF_ITIME_B, &dquot->dq_flags);
}
if (check_blim) {
if (!dm->dqb_bsoftlimit ||
dm->dqb_curspace + dm->dqb_rsvspace <= dm->dqb_bsoftlimit) {
dm->dqb_btime = 0;
clear_bit(DQ_BLKS_B, &dquot->dq_flags);
} else if (!(di->d_fieldmask & QC_SPC_TIMER))
/* Set grace only if user hasn't provided his own... */
dm->dqb_btime = ktime_get_real_seconds() + dqi->dqi_bgrace;
}
if (check_ilim) {
if (!dm->dqb_isoftlimit ||
dm->dqb_curinodes <= dm->dqb_isoftlimit) {
dm->dqb_itime = 0;
clear_bit(DQ_INODES_B, &dquot->dq_flags);
} else if (!(di->d_fieldmask & QC_INO_TIMER))
/* Set grace only if user hasn't provided his own... */
dm->dqb_itime = ktime_get_real_seconds() + dqi->dqi_igrace;
}
if (dm->dqb_bhardlimit || dm->dqb_bsoftlimit || dm->dqb_ihardlimit ||
dm->dqb_isoftlimit)
clear_bit(DQ_FAKE_B, &dquot->dq_flags);
else
set_bit(DQ_FAKE_B, &dquot->dq_flags);
spin_unlock(&dquot->dq_dqb_lock);
mark_dquot_dirty(dquot);
return 0;
}
int dquot_set_dqblk(struct super_block *sb, struct kqid qid,
struct qc_dqblk *di)
{
struct dquot *dquot;
int rc;
dquot = dqget(sb, qid);
if (IS_ERR(dquot)) {
rc = PTR_ERR(dquot);
goto out;
}
rc = do_set_dqblk(dquot, di);
dqput(dquot);
out:
return rc;
}
EXPORT_SYMBOL(dquot_set_dqblk);
/* Generic routine for getting common part of quota file information */
int dquot_get_state(struct super_block *sb, struct qc_state *state)
{
struct mem_dqinfo *mi;
struct qc_type_state *tstate;
struct quota_info *dqopt = sb_dqopt(sb);
int type;
memset(state, 0, sizeof(*state));
for (type = 0; type < MAXQUOTAS; type++) {
if (!sb_has_quota_active(sb, type))
continue;
tstate = state->s_state + type;
mi = sb_dqopt(sb)->info + type;
tstate->flags = QCI_ACCT_ENABLED;
spin_lock(&dq_data_lock);
if (mi->dqi_flags & DQF_SYS_FILE)
tstate->flags |= QCI_SYSFILE;
if (mi->dqi_flags & DQF_ROOT_SQUASH)
tstate->flags |= QCI_ROOT_SQUASH;
if (sb_has_quota_limits_enabled(sb, type))
tstate->flags |= QCI_LIMITS_ENFORCED;
tstate->spc_timelimit = mi->dqi_bgrace;
tstate->ino_timelimit = mi->dqi_igrace;
if (dqopt->files[type]) {
tstate->ino = dqopt->files[type]->i_ino;
tstate->blocks = dqopt->files[type]->i_blocks;
}
tstate->nextents = 1; /* We don't know... */
spin_unlock(&dq_data_lock);
}
return 0;
}
EXPORT_SYMBOL(dquot_get_state);
/* Generic routine for setting common part of quota file information */
int dquot_set_dqinfo(struct super_block *sb, int type, struct qc_info *ii)
{
struct mem_dqinfo *mi;
if ((ii->i_fieldmask & QC_WARNS_MASK) ||
(ii->i_fieldmask & QC_RT_SPC_TIMER))
return -EINVAL;
if (!sb_has_quota_active(sb, type))
return -ESRCH;
mi = sb_dqopt(sb)->info + type;
if (ii->i_fieldmask & QC_FLAGS) {
if ((ii->i_flags & QCI_ROOT_SQUASH &&
mi->dqi_format->qf_fmt_id != QFMT_VFS_OLD))
return -EINVAL;
}
spin_lock(&dq_data_lock);
if (ii->i_fieldmask & QC_SPC_TIMER)
mi->dqi_bgrace = ii->i_spc_timelimit;
if (ii->i_fieldmask & QC_INO_TIMER)
mi->dqi_igrace = ii->i_ino_timelimit;
if (ii->i_fieldmask & QC_FLAGS) {
if (ii->i_flags & QCI_ROOT_SQUASH)
mi->dqi_flags |= DQF_ROOT_SQUASH;
else
mi->dqi_flags &= ~DQF_ROOT_SQUASH;
}
spin_unlock(&dq_data_lock);
mark_info_dirty(sb, type);
/* Force write to disk */
return sb->dq_op->write_info(sb, type);
}
EXPORT_SYMBOL(dquot_set_dqinfo);
const struct quotactl_ops dquot_quotactl_sysfile_ops = {
.quota_enable = dquot_quota_enable,
.quota_disable = dquot_quota_disable,
.quota_sync = dquot_quota_sync,
.get_state = dquot_get_state,
.set_info = dquot_set_dqinfo,
.get_dqblk = dquot_get_dqblk,
.get_nextdqblk = dquot_get_next_dqblk,
.set_dqblk = dquot_set_dqblk
};
EXPORT_SYMBOL(dquot_quotactl_sysfile_ops);
static int do_proc_dqstats(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
unsigned int type = (unsigned long *)table->data - dqstats.stat;
s64 value = percpu_counter_sum(&dqstats.counter[type]);
/* Filter negative values for non-monotonic counters */
if (value < 0 && (type == DQST_ALLOC_DQUOTS ||
type == DQST_FREE_DQUOTS))
value = 0;
/* Update global table */
dqstats.stat[type] = value;
return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
}
static struct ctl_table fs_dqstats_table[] = {
{
.procname = "lookups",
.data = &dqstats.stat[DQST_LOOKUPS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "drops",
.data = &dqstats.stat[DQST_DROPS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "reads",
.data = &dqstats.stat[DQST_READS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "writes",
.data = &dqstats.stat[DQST_WRITES],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "cache_hits",
.data = &dqstats.stat[DQST_CACHE_HITS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "allocated_dquots",
.data = &dqstats.stat[DQST_ALLOC_DQUOTS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "free_dquots",
.data = &dqstats.stat[DQST_FREE_DQUOTS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
{
.procname = "syncs",
.data = &dqstats.stat[DQST_SYNCS],
.maxlen = sizeof(unsigned long),
.mode = 0444,
.proc_handler = do_proc_dqstats,
},
#ifdef CONFIG_PRINT_QUOTA_WARNING
{
.procname = "warnings",
.data = &flag_print_warnings,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
#endif
};
static int __init dquot_init(void)
{
int i, ret;
unsigned long nr_hash, order;
struct shrinker *dqcache_shrinker;
printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__);
register_sysctl_init("fs/quota", fs_dqstats_table);
dquot_cachep = kmem_cache_create("dquot",
sizeof(struct dquot), sizeof(unsigned long) * 4,
(SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
SLAB_PANIC),
NULL);
order = 0;
dquot_hash = (struct hlist_head *)__get_free_pages(GFP_KERNEL, order);
if (!dquot_hash)
panic("Cannot create dquot hash table");
for (i = 0; i < _DQST_DQSTAT_LAST; i++) {
ret = percpu_counter_init(&dqstats.counter[i], 0, GFP_KERNEL);
if (ret)
panic("Cannot create dquot stat counters");
}
/* Find power-of-two hlist_heads which can fit into allocation */
nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
dq_hash_bits = ilog2(nr_hash);
nr_hash = 1UL << dq_hash_bits;
dq_hash_mask = nr_hash - 1;
for (i = 0; i < nr_hash; i++)
INIT_HLIST_HEAD(dquot_hash + i);
pr_info("VFS: Dquot-cache hash table entries: %ld (order %ld,"
" %ld bytes)\n", nr_hash, order, (PAGE_SIZE << order));
dqcache_shrinker = shrinker_alloc(0, "dquota-cache");
if (!dqcache_shrinker)
panic("Cannot allocate dquot shrinker");
dqcache_shrinker->count_objects = dqcache_shrink_count;
dqcache_shrinker->scan_objects = dqcache_shrink_scan;
shrinker_register(dqcache_shrinker);
return 0;
}
fs_initcall(dquot_init);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __IEEE802154_CORE_H
#define __IEEE802154_CORE_H
#include <net/cfg802154.h>
struct cfg802154_registered_device {
const struct cfg802154_ops *ops;
struct list_head list;
/* wpan_phy index, internal only */
int wpan_phy_idx;
/* also protected by devlist_mtx */
int opencount;
wait_queue_head_t dev_wait;
/* protected by RTNL only */
int num_running_ifaces;
/* associated wpan interfaces, protected by rtnl or RCU */
struct list_head wpan_dev_list;
int devlist_generation, wpan_dev_id;
/* must be last because of the way we do wpan_phy_priv(),
* and it should at least be aligned to NETDEV_ALIGN
*/
struct wpan_phy wpan_phy __aligned(NETDEV_ALIGN);
};
static inline struct cfg802154_registered_device *
wpan_phy_to_rdev(struct wpan_phy *wpan_phy)
{
BUG_ON(!wpan_phy); return container_of(wpan_phy, struct cfg802154_registered_device,
wpan_phy);
}
extern struct list_head cfg802154_rdev_list;
extern int cfg802154_rdev_list_generation;
int cfg802154_switch_netns(struct cfg802154_registered_device *rdev,
struct net *net);
/* free object */
void cfg802154_dev_free(struct cfg802154_registered_device *rdev);
struct cfg802154_registered_device *
cfg802154_rdev_by_wpan_phy_idx(int wpan_phy_idx);
struct wpan_phy *wpan_phy_idx_to_wpan_phy(int wpan_phy_idx);
#endif /* __IEEE802154_CORE_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* kernel/workqueue_internal.h
*
* Workqueue internal header file. Only to be included by workqueue and
* core kernel subsystems.
*/
#ifndef _KERNEL_WORKQUEUE_INTERNAL_H
#define _KERNEL_WORKQUEUE_INTERNAL_H
#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/preempt.h>
struct worker_pool;
/*
* The poor guys doing the actual heavy lifting. All on-duty workers are
* either serving the manager role, on idle list or on busy hash. For
* details on the locking annotation (L, I, X...), refer to workqueue.c.
*
* Only to be used in workqueue and async.
*/
struct worker {
/* on idle list while idle, on busy hash table while busy */
union {
struct list_head entry; /* L: while idle */
struct hlist_node hentry; /* L: while busy */
};
struct work_struct *current_work; /* K: work being processed and its */
work_func_t current_func; /* K: function */
struct pool_workqueue *current_pwq; /* K: pwq */
u64 current_at; /* K: runtime at start or last wakeup */
unsigned int current_color; /* K: color */
int sleeping; /* S: is worker sleeping? */
/* used by the scheduler to determine a worker's last known identity */
work_func_t last_func; /* K: last work's fn */
struct list_head scheduled; /* L: scheduled works */
struct task_struct *task; /* I: worker task */
struct worker_pool *pool; /* A: the associated pool */
/* L: for rescuers */
struct list_head node; /* A: anchored at pool->workers */
/* A: runs through worker->node */
unsigned long last_active; /* K: last active timestamp */
unsigned int flags; /* L: flags */
int id; /* I: worker id */
/*
* Opaque string set with work_set_desc(). Printed out with task
* dump for debugging - WARN, BUG, panic or sysrq.
*/
char desc[WORKER_DESC_LEN];
/* used only by rescuers to point to the target workqueue */
struct workqueue_struct *rescue_wq; /* I: the workqueue to rescue */
};
/**
* current_wq_worker - return struct worker if %current is a workqueue worker
*/
static inline struct worker *current_wq_worker(void)
{
if (in_task() && (current->flags & PF_WQ_WORKER)) return kthread_data(current);
return NULL;
}
/*
* Scheduler hooks for concurrency managed workqueue. Only to be used from
* sched/ and workqueue.c.
*/
void wq_worker_running(struct task_struct *task);
void wq_worker_sleeping(struct task_struct *task);
void wq_worker_tick(struct task_struct *task);
work_func_t wq_worker_last_func(struct task_struct *task);
#endif /* _KERNEL_WORKQUEUE_INTERNAL_H */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* NET3 Protocol independent device support routines.
*
* Derived from the non IP parts of dev.c 1.0.19
* Authors: Ross Biro
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
* Mark Evans, <evansmp@uhura.aston.ac.uk>
*
* Additional Authors:
* Florian la Roche <rzsfl@rz.uni-sb.de>
* Alan Cox <gw4pts@gw4pts.ampr.org>
* David Hinds <dahinds@users.sourceforge.net>
* Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
* Adam Sulmicki <adam@cfar.umd.edu>
* Pekka Riikonen <priikone@poesidon.pspt.fi>
*
* Changes:
* D.J. Barrow : Fixed bug where dev->refcnt gets set
* to 2 if register_netdev gets called
* before net_dev_init & also removed a
* few lines of code in the process.
* Alan Cox : device private ioctl copies fields back.
* Alan Cox : Transmit queue code does relevant
* stunts to keep the queue safe.
* Alan Cox : Fixed double lock.
* Alan Cox : Fixed promisc NULL pointer trap
* ???????? : Support the full private ioctl range
* Alan Cox : Moved ioctl permission check into
* drivers
* Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
* Alan Cox : 100 backlog just doesn't cut it when
* you start doing multicast video 8)
* Alan Cox : Rewrote net_bh and list manager.
* Alan Cox : Fix ETH_P_ALL echoback lengths.
* Alan Cox : Took out transmit every packet pass
* Saved a few bytes in the ioctl handler
* Alan Cox : Network driver sets packet type before
* calling netif_rx. Saves a function
* call a packet.
* Alan Cox : Hashed net_bh()
* Richard Kooijman: Timestamp fixes.
* Alan Cox : Wrong field in SIOCGIFDSTADDR
* Alan Cox : Device lock protection.
* Alan Cox : Fixed nasty side effect of device close
* changes.
* Rudi Cilibrasi : Pass the right thing to
* set_mac_address()
* Dave Miller : 32bit quantity for the device lock to
* make it work out on a Sparc.
* Bjorn Ekwall : Added KERNELD hack.
* Alan Cox : Cleaned up the backlog initialise.
* Craig Metz : SIOCGIFCONF fix if space for under
* 1 device.
* Thomas Bogendoerfer : Return ENODEV for dev_open, if there
* is no device open function.
* Andi Kleen : Fix error reporting for SIOCGIFCONF
* Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
* Cyrus Durgin : Cleaned for KMOD
* Adam Sulmicki : Bug Fix : Network Device Unload
* A network device unload needs to purge
* the backlog queue.
* Paul Rusty Russell : SIOCSIFNAME
* Pekka Riikonen : Netdev boot-time settings code
* Andrew Morton : Make unregister_netdevice wait
* indefinitely on dev->refcnt
* J Hadi Salim : - Backlog queue sampling
* - netif_rx() feedback
*/
#include <linux/uaccess.h>
#include <linux/bitmap.h>
#include <linux/capability.h>
#include <linux/cpu.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/hash.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/if_ether.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/skbuff.h>
#include <linux/kthread.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/busy_poll.h>
#include <linux/rtnetlink.h>
#include <linux/stat.h>
#include <net/dsa.h>
#include <net/dst.h>
#include <net/dst_metadata.h>
#include <net/gro.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/checksum.h>
#include <net/xfrm.h>
#include <net/tcx.h>
#include <linux/highmem.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netpoll.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>
#include <net/iw_handler.h>
#include <asm/current.h>
#include <linux/audit.h>
#include <linux/dmaengine.h>
#include <linux/err.h>
#include <linux/ctype.h>
#include <linux/if_arp.h>
#include <linux/if_vlan.h>
#include <linux/ip.h>
#include <net/ip.h>
#include <net/mpls.h>
#include <linux/ipv6.h>
#include <linux/in.h>
#include <linux/jhash.h>
#include <linux/random.h>
#include <trace/events/napi.h>
#include <trace/events/net.h>
#include <trace/events/skb.h>
#include <trace/events/qdisc.h>
#include <trace/events/xdp.h>
#include <linux/inetdevice.h>
#include <linux/cpu_rmap.h>
#include <linux/static_key.h>
#include <linux/hashtable.h>
#include <linux/vmalloc.h>
#include <linux/if_macvlan.h>
#include <linux/errqueue.h>
#include <linux/hrtimer.h>
#include <linux/netfilter_netdev.h>
#include <linux/crash_dump.h>
#include <linux/sctp.h>
#include <net/udp_tunnel.h>
#include <linux/net_namespace.h>
#include <linux/indirect_call_wrapper.h>
#include <net/devlink.h>
#include <linux/pm_runtime.h>
#include <linux/prandom.h>
#include <linux/once_lite.h>
#include <net/netdev_rx_queue.h>
#include <net/page_pool/types.h>
#include <net/page_pool/helpers.h>
#include <net/rps.h>
#include "dev.h"
#include "net-sysfs.h"
static DEFINE_SPINLOCK(ptype_lock);
struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
static int netif_rx_internal(struct sk_buff *skb);
static int call_netdevice_notifiers_extack(unsigned long val,
struct net_device *dev,
struct netlink_ext_ack *extack);
static DEFINE_MUTEX(ifalias_mutex);
/* protects napi_hash addition/deletion and napi_gen_id */
static DEFINE_SPINLOCK(napi_hash_lock);
static unsigned int napi_gen_id = NR_CPUS;
static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
static DECLARE_RWSEM(devnet_rename_sem);
static inline void dev_base_seq_inc(struct net *net)
{
unsigned int val = net->dev_base_seq + 1;
WRITE_ONCE(net->dev_base_seq, val ?: 1);
}
static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
{
unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
}
static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
{
return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
}
static inline void rps_lock_irqsave(struct softnet_data *sd,
unsigned long *flags)
{
if (IS_ENABLED(CONFIG_RPS))
spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
local_irq_save(*flags);
}
static inline void rps_lock_irq_disable(struct softnet_data *sd)
{
if (IS_ENABLED(CONFIG_RPS))
spin_lock_irq(&sd->input_pkt_queue.lock);
else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
local_irq_disable();
}
static inline void rps_unlock_irq_restore(struct softnet_data *sd,
unsigned long *flags)
{
if (IS_ENABLED(CONFIG_RPS))
spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
local_irq_restore(*flags);
}
static inline void rps_unlock_irq_enable(struct softnet_data *sd)
{
if (IS_ENABLED(CONFIG_RPS))
spin_unlock_irq(&sd->input_pkt_queue.lock);
else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
local_irq_enable();
}
static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
const char *name)
{
struct netdev_name_node *name_node;
name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
if (!name_node)
return NULL;
INIT_HLIST_NODE(&name_node->hlist);
name_node->dev = dev;
name_node->name = name;
return name_node;
}
static struct netdev_name_node *
netdev_name_node_head_alloc(struct net_device *dev)
{
struct netdev_name_node *name_node;
name_node = netdev_name_node_alloc(dev, dev->name);
if (!name_node)
return NULL;
INIT_LIST_HEAD(&name_node->list);
return name_node;
}
static void netdev_name_node_free(struct netdev_name_node *name_node)
{
kfree(name_node);
}
static void netdev_name_node_add(struct net *net,
struct netdev_name_node *name_node)
{
hlist_add_head_rcu(&name_node->hlist,
dev_name_hash(net, name_node->name));
}
static void netdev_name_node_del(struct netdev_name_node *name_node)
{
hlist_del_rcu(&name_node->hlist);
}
static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
const char *name)
{
struct hlist_head *head = dev_name_hash(net, name);
struct netdev_name_node *name_node;
hlist_for_each_entry(name_node, head, hlist)
if (!strcmp(name_node->name, name))
return name_node;
return NULL;
}
static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
const char *name)
{
struct hlist_head *head = dev_name_hash(net, name);
struct netdev_name_node *name_node;
hlist_for_each_entry_rcu(name_node, head, hlist)
if (!strcmp(name_node->name, name))
return name_node;
return NULL;
}
bool netdev_name_in_use(struct net *net, const char *name)
{
return netdev_name_node_lookup(net, name);
}
EXPORT_SYMBOL(netdev_name_in_use);
int netdev_name_node_alt_create(struct net_device *dev, const char *name)
{
struct netdev_name_node *name_node;
struct net *net = dev_net(dev);
name_node = netdev_name_node_lookup(net, name);
if (name_node)
return -EEXIST;
name_node = netdev_name_node_alloc(dev, name);
if (!name_node)
return -ENOMEM;
netdev_name_node_add(net, name_node);
/* The node that holds dev->name acts as a head of per-device list. */
list_add_tail_rcu(&name_node->list, &dev->name_node->list);
return 0;
}
static void netdev_name_node_alt_free(struct rcu_head *head)
{
struct netdev_name_node *name_node =
container_of(head, struct netdev_name_node, rcu);
kfree(name_node->name);
netdev_name_node_free(name_node);
}
static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
{
netdev_name_node_del(name_node);
list_del(&name_node->list);
call_rcu(&name_node->rcu, netdev_name_node_alt_free);
}
int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
{
struct netdev_name_node *name_node;
struct net *net = dev_net(dev);
name_node = netdev_name_node_lookup(net, name);
if (!name_node)
return -ENOENT;
/* lookup might have found our primary name or a name belonging
* to another device.
*/
if (name_node == dev->name_node || name_node->dev != dev)
return -EINVAL;
__netdev_name_node_alt_destroy(name_node);
return 0;
}
static void netdev_name_node_alt_flush(struct net_device *dev)
{
struct netdev_name_node *name_node, *tmp;
list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) {
list_del(&name_node->list);
netdev_name_node_alt_free(&name_node->rcu);
}
}
/* Device list insertion */
static void list_netdevice(struct net_device *dev)
{
struct netdev_name_node *name_node;
struct net *net = dev_net(dev);
ASSERT_RTNL();
list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
netdev_name_node_add(net, dev->name_node);
hlist_add_head_rcu(&dev->index_hlist,
dev_index_hash(net, dev->ifindex));
netdev_for_each_altname(dev, name_node)
netdev_name_node_add(net, name_node);
/* We reserved the ifindex, this can't fail */
WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));
dev_base_seq_inc(net);
}
/* Device list removal
* caller must respect a RCU grace period before freeing/reusing dev
*/
static void unlist_netdevice(struct net_device *dev)
{
struct netdev_name_node *name_node;
struct net *net = dev_net(dev); ASSERT_RTNL(); xa_erase(&net->dev_by_index, dev->ifindex);
netdev_for_each_altname(dev, name_node)
netdev_name_node_del(name_node);
/* Unlink dev from the device chain */
list_del_rcu(&dev->dev_list); netdev_name_node_del(dev->name_node); hlist_del_rcu(&dev->index_hlist);
dev_base_seq_inc(dev_net(dev));
}
/*
* Our notifier list
*/
static RAW_NOTIFIER_HEAD(netdev_chain);
/*
* Device drivers call our routines to queue packets here. We empty the
* queue in the local softnet handler.
*/
DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
EXPORT_PER_CPU_SYMBOL(softnet_data);
/* Page_pool has a lockless array/stack to alloc/recycle pages.
* PP consumers must pay attention to run APIs in the appropriate context
* (e.g. NAPI context).
*/
static DEFINE_PER_CPU(struct page_pool *, system_page_pool);
#ifdef CONFIG_LOCKDEP
/*
* register_netdevice() inits txq->_xmit_lock and sets lockdep class
* according to dev->type
*/
static const unsigned short netdev_lock_type[] = {
ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
static const char *const netdev_lock_name[] = {
"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
"_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
"_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
static inline unsigned short netdev_lock_pos(unsigned short dev_type)
{
int i;
for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
if (netdev_lock_type[i] == dev_type)
return i;
/* the last key is used by default */
return ARRAY_SIZE(netdev_lock_type) - 1;
}
static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
unsigned short dev_type)
{
int i;
i = netdev_lock_pos(dev_type);
lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
netdev_lock_name[i]);
}
static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
{
int i;
i = netdev_lock_pos(dev->type);
lockdep_set_class_and_name(&dev->addr_list_lock,
&netdev_addr_lock_key[i],
netdev_lock_name[i]);
}
#else
static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
unsigned short dev_type)
{
}
static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
{
}
#endif
/*******************************************************************************
*
* Protocol management and registration routines
*
*******************************************************************************/
/*
* Add a protocol ID to the list. Now that the input handler is
* smarter we can dispense with all the messy stuff that used to be
* here.
*
* BEWARE!!! Protocol handlers, mangling input packets,
* MUST BE last in hash buckets and checking protocol handlers
* MUST start from promiscuous ptype_all chain in net_bh.
* It is true now, do not change it.
* Explanation follows: if protocol handler, mangling packet, will
* be the first on list, it is not able to sense, that packet
* is cloned and should be copied-on-write, so that it will
* change it and subsequent readers will get broken packet.
* --ANK (980803)
*/
static inline struct list_head *ptype_head(const struct packet_type *pt)
{
if (pt->type == htons(ETH_P_ALL))
return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
else
return pt->dev ? &pt->dev->ptype_specific : &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
}
/**
* dev_add_pack - add packet handler
* @pt: packet type declaration
*
* Add a protocol handler to the networking stack. The passed &packet_type
* is linked into kernel lists and may not be freed until it has been
* removed from the kernel lists.
*
* This call does not sleep therefore it can not
* guarantee all CPU's that are in middle of receiving packets
* will see the new packet type (until the next received packet).
*/
void dev_add_pack(struct packet_type *pt)
{
struct list_head *head = ptype_head(pt);
spin_lock(&ptype_lock);
list_add_rcu(&pt->list, head);
spin_unlock(&ptype_lock);
}
EXPORT_SYMBOL(dev_add_pack);
/**
* __dev_remove_pack - remove packet handler
* @pt: packet type declaration
*
* Remove a protocol handler that was previously added to the kernel
* protocol handlers by dev_add_pack(). The passed &packet_type is removed
* from the kernel lists and can be freed or reused once this function
* returns.
*
* The packet type might still be in use by receivers
* and must not be freed until after all the CPU's have gone
* through a quiescent state.
*/
void __dev_remove_pack(struct packet_type *pt)
{ struct list_head *head = ptype_head(pt);
struct packet_type *pt1;
spin_lock(&ptype_lock); list_for_each_entry(pt1, head, list) { if (pt == pt1) { list_del_rcu(&pt->list);
goto out;
}
}
pr_warn("dev_remove_pack: %p not found\n", pt);
out:
spin_unlock(&ptype_lock);
}
EXPORT_SYMBOL(__dev_remove_pack);
/**
* dev_remove_pack - remove packet handler
* @pt: packet type declaration
*
* Remove a protocol handler that was previously added to the kernel
* protocol handlers by dev_add_pack(). The passed &packet_type is removed
* from the kernel lists and can be freed or reused once this function
* returns.
*
* This call sleeps to guarantee that no CPU is looking at the packet
* type after return.
*/
void dev_remove_pack(struct packet_type *pt)
{
__dev_remove_pack(pt);
synchronize_net();
}
EXPORT_SYMBOL(dev_remove_pack);
/*******************************************************************************
*
* Device Interface Subroutines
*
*******************************************************************************/
/**
* dev_get_iflink - get 'iflink' value of a interface
* @dev: targeted interface
*
* Indicates the ifindex the interface is linked to.
* Physical interfaces have the same 'ifindex' and 'iflink' values.
*/
int dev_get_iflink(const struct net_device *dev)
{
if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) return dev->netdev_ops->ndo_get_iflink(dev); return READ_ONCE(dev->ifindex);
}
EXPORT_SYMBOL(dev_get_iflink);
/**
* dev_fill_metadata_dst - Retrieve tunnel egress information.
* @dev: targeted interface
* @skb: The packet.
*
* For better visibility of tunnel traffic OVS needs to retrieve
* egress tunnel information for a packet. Following API allows
* user to get this info.
*/
int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
{
struct ip_tunnel_info *info;
if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
return -EINVAL;
info = skb_tunnel_info_unclone(skb);
if (!info)
return -ENOMEM;
if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
return -EINVAL;
return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
}
EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
{
int k = stack->num_paths++;
if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
return NULL;
return &stack->path[k];
}
int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
struct net_device_path_stack *stack)
{
const struct net_device *last_dev;
struct net_device_path_ctx ctx = {
.dev = dev,
};
struct net_device_path *path;
int ret = 0;
memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
stack->num_paths = 0;
while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
last_dev = ctx.dev;
path = dev_fwd_path(stack);
if (!path)
return -1;
memset(path, 0, sizeof(struct net_device_path));
ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
if (ret < 0)
return -1;
if (WARN_ON_ONCE(last_dev == ctx.dev))
return -1;
}
if (!ctx.dev)
return ret;
path = dev_fwd_path(stack);
if (!path)
return -1;
path->type = DEV_PATH_ETHERNET;
path->dev = ctx.dev;
return ret;
}
EXPORT_SYMBOL_GPL(dev_fill_forward_path);
/**
* __dev_get_by_name - find a device by its name
* @net: the applicable net namespace
* @name: name to find
*
* Find an interface by name. Must be called under RTNL semaphore.
* If the name is found a pointer to the device is returned.
* If the name is not found then %NULL is returned. The
* reference counters are not incremented so the caller must be
* careful with locks.
*/
struct net_device *__dev_get_by_name(struct net *net, const char *name)
{
struct netdev_name_node *node_name;
node_name = netdev_name_node_lookup(net, name);
return node_name ? node_name->dev : NULL;
}
EXPORT_SYMBOL(__dev_get_by_name);
/**
* dev_get_by_name_rcu - find a device by its name
* @net: the applicable net namespace
* @name: name to find
*
* Find an interface by name.
* If the name is found a pointer to the device is returned.
* If the name is not found then %NULL is returned.
* The reference counters are not incremented so the caller must be
* careful with locks. The caller must hold RCU lock.
*/
struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
{
struct netdev_name_node *node_name;
node_name = netdev_name_node_lookup_rcu(net, name);
return node_name ? node_name->dev : NULL;
}
EXPORT_SYMBOL(dev_get_by_name_rcu);
/* Deprecated for new users, call netdev_get_by_name() instead */
struct net_device *dev_get_by_name(struct net *net, const char *name)
{
struct net_device *dev;
rcu_read_lock();
dev = dev_get_by_name_rcu(net, name);
dev_hold(dev);
rcu_read_unlock();
return dev;
}
EXPORT_SYMBOL(dev_get_by_name);
/**
* netdev_get_by_name() - find a device by its name
* @net: the applicable net namespace
* @name: name to find
* @tracker: tracking object for the acquired reference
* @gfp: allocation flags for the tracker
*
* Find an interface by name. This can be called from any
* context and does its own locking. The returned handle has
* the usage count incremented and the caller must use netdev_put() to
* release it when it is no longer needed. %NULL is returned if no
* matching device is found.
*/
struct net_device *netdev_get_by_name(struct net *net, const char *name,
netdevice_tracker *tracker, gfp_t gfp)
{
struct net_device *dev;
dev = dev_get_by_name(net, name);
if (dev)
netdev_tracker_alloc(dev, tracker, gfp);
return dev;
}
EXPORT_SYMBOL(netdev_get_by_name);
/**
* __dev_get_by_index - find a device by its ifindex
* @net: the applicable net namespace
* @ifindex: index of device
*
* Search for an interface by index. Returns %NULL if the device
* is not found or a pointer to the device. The device has not
* had its reference counter increased so the caller must be careful
* about locking. The caller must hold the RTNL semaphore.
*/
struct net_device *__dev_get_by_index(struct net *net, int ifindex)
{
struct net_device *dev;
struct hlist_head *head = dev_index_hash(net, ifindex);
hlist_for_each_entry(dev, head, index_hlist)
if (dev->ifindex == ifindex)
return dev;
return NULL;
}
EXPORT_SYMBOL(__dev_get_by_index);
/**
* dev_get_by_index_rcu - find a device by its ifindex
* @net: the applicable net namespace
* @ifindex: index of device
*
* Search for an interface by index. Returns %NULL if the device
* is not found or a pointer to the device. The device has not
* had its reference counter increased so the caller must be careful
* about locking. The caller must hold RCU lock.
*/
struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
{
struct net_device *dev;
struct hlist_head *head = dev_index_hash(net, ifindex);
hlist_for_each_entry_rcu(dev, head, index_hlist)
if (dev->ifindex == ifindex)
return dev;
return NULL;
}
EXPORT_SYMBOL(dev_get_by_index_rcu);
/* Deprecated for new users, call netdev_get_by_index() instead */
struct net_device *dev_get_by_index(struct net *net, int ifindex)
{
struct net_device *dev;
rcu_read_lock();
dev = dev_get_by_index_rcu(net, ifindex);
dev_hold(dev);
rcu_read_unlock();
return dev;
}
EXPORT_SYMBOL(dev_get_by_index);
/**
* netdev_get_by_index() - find a device by its ifindex
* @net: the applicable net namespace
* @ifindex: index of device
* @tracker: tracking object for the acquired reference
* @gfp: allocation flags for the tracker
*
* Search for an interface by index. Returns NULL if the device
* is not found or a pointer to the device. The device returned has
* had a reference added and the pointer is safe until the user calls
* netdev_put() to indicate they have finished with it.
*/
struct net_device *netdev_get_by_index(struct net *net, int ifindex,
netdevice_tracker *tracker, gfp_t gfp)
{
struct net_device *dev;
dev = dev_get_by_index(net, ifindex);
if (dev)
netdev_tracker_alloc(dev, tracker, gfp);
return dev;
}
EXPORT_SYMBOL(netdev_get_by_index);
/**
* dev_get_by_napi_id - find a device by napi_id
* @napi_id: ID of the NAPI struct
*
* Search for an interface by NAPI ID. Returns %NULL if the device
* is not found or a pointer to the device. The device has not had
* its reference counter increased so the caller must be careful
* about locking. The caller must hold RCU lock.
*/
struct net_device *dev_get_by_napi_id(unsigned int napi_id)
{
struct napi_struct *napi;
WARN_ON_ONCE(!rcu_read_lock_held());
if (napi_id < MIN_NAPI_ID)
return NULL;
napi = napi_by_id(napi_id);
return napi ? napi->dev : NULL;
}
EXPORT_SYMBOL(dev_get_by_napi_id);
/**
* netdev_get_name - get a netdevice name, knowing its ifindex.
* @net: network namespace
* @name: a pointer to the buffer where the name will be stored.
* @ifindex: the ifindex of the interface to get the name from.
*/
int netdev_get_name(struct net *net, char *name, int ifindex)
{
struct net_device *dev;
int ret;
down_read(&devnet_rename_sem);
rcu_read_lock();
dev = dev_get_by_index_rcu(net, ifindex);
if (!dev) {
ret = -ENODEV;
goto out;
}
strcpy(name, dev->name);
ret = 0;
out:
rcu_read_unlock();
up_read(&devnet_rename_sem);
return ret;
}
/**
* dev_getbyhwaddr_rcu - find a device by its hardware address
* @net: the applicable net namespace
* @type: media type of device
* @ha: hardware address
*
* Search for an interface by MAC address. Returns NULL if the device
* is not found or a pointer to the device.
* The caller must hold RCU or RTNL.
* The returned device has not had its ref count increased
* and the caller must therefore be careful about locking
*
*/
struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
const char *ha)
{
struct net_device *dev;
for_each_netdev_rcu(net, dev)
if (dev->type == type &&
!memcmp(dev->dev_addr, ha, dev->addr_len))
return dev;
return NULL;
}
EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
{
struct net_device *dev, *ret = NULL;
rcu_read_lock();
for_each_netdev_rcu(net, dev)
if (dev->type == type) {
dev_hold(dev);
ret = dev;
break;
}
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL(dev_getfirstbyhwtype);
/**
* __dev_get_by_flags - find any device with given flags
* @net: the applicable net namespace
* @if_flags: IFF_* values
* @mask: bitmask of bits in if_flags to check
*
* Search for any interface with the given flags. Returns NULL if a device
* is not found or a pointer to the device. Must be called inside
* rtnl_lock(), and result refcount is unchanged.
*/
struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
unsigned short mask)
{
struct net_device *dev, *ret;
ASSERT_RTNL();
ret = NULL;
for_each_netdev(net, dev) {
if (((dev->flags ^ if_flags) & mask) == 0) {
ret = dev;
break;
}
}
return ret;
}
EXPORT_SYMBOL(__dev_get_by_flags);
/**
* dev_valid_name - check if name is okay for network device
* @name: name string
*
* Network device names need to be valid file names to
* allow sysfs to work. We also disallow any kind of
* whitespace.
*/
bool dev_valid_name(const char *name)
{
if (*name == '\0')
return false;
if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
return false;
if (!strcmp(name, ".") || !strcmp(name, ".."))
return false;
while (*name) {
if (*name == '/' || *name == ':' || isspace(*name))
return false;
name++;
}
return true;
}
EXPORT_SYMBOL(dev_valid_name);
/**
* __dev_alloc_name - allocate a name for a device
* @net: network namespace to allocate the device name in
* @name: name format string
* @res: result name string
*
* Passed a format string - eg "lt%d" it will try and find a suitable
* id. It scans list of devices to build up a free map, then chooses
* the first empty slot. The caller must hold the dev_base or rtnl lock
* while allocating the name and adding the device in order to avoid
* duplicates.
* Limited to bits_per_byte * page size devices (ie 32K on most platforms).
* Returns the number of the unit assigned or a negative errno code.
*/
static int __dev_alloc_name(struct net *net, const char *name, char *res)
{
int i = 0;
const char *p;
const int max_netdevices = 8*PAGE_SIZE;
unsigned long *inuse;
struct net_device *d;
char buf[IFNAMSIZ];
/* Verify the string as this thing may have come from the user.
* There must be one "%d" and no other "%" characters.
*/
p = strchr(name, '%');
if (!p || p[1] != 'd' || strchr(p + 2, '%'))
return -EINVAL;
/* Use one page as a bit array of possible slots */
inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC);
if (!inuse)
return -ENOMEM;
for_each_netdev(net, d) {
struct netdev_name_node *name_node;
netdev_for_each_altname(d, name_node) {
if (!sscanf(name_node->name, name, &i))
continue;
if (i < 0 || i >= max_netdevices)
continue;
/* avoid cases where sscanf is not exact inverse of printf */
snprintf(buf, IFNAMSIZ, name, i);
if (!strncmp(buf, name_node->name, IFNAMSIZ))
__set_bit(i, inuse);
}
if (!sscanf(d->name, name, &i))
continue;
if (i < 0 || i >= max_netdevices)
continue;
/* avoid cases where sscanf is not exact inverse of printf */
snprintf(buf, IFNAMSIZ, name, i);
if (!strncmp(buf, d->name, IFNAMSIZ))
__set_bit(i, inuse);
}
i = find_first_zero_bit(inuse, max_netdevices);
bitmap_free(inuse);
if (i == max_netdevices)
return -ENFILE;
/* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */
strscpy(buf, name, IFNAMSIZ);
snprintf(res, IFNAMSIZ, buf, i);
return i;
}
/* Returns negative errno or allocated unit id (see __dev_alloc_name()) */
static int dev_prep_valid_name(struct net *net, struct net_device *dev,
const char *want_name, char *out_name,
int dup_errno)
{
if (!dev_valid_name(want_name))
return -EINVAL;
if (strchr(want_name, '%'))
return __dev_alloc_name(net, want_name, out_name);
if (netdev_name_in_use(net, want_name))
return -dup_errno;
if (out_name != want_name)
strscpy(out_name, want_name, IFNAMSIZ);
return 0;
}
/**
* dev_alloc_name - allocate a name for a device
* @dev: device
* @name: name format string
*
* Passed a format string - eg "lt%d" it will try and find a suitable
* id. It scans list of devices to build up a free map, then chooses
* the first empty slot. The caller must hold the dev_base or rtnl lock
* while allocating the name and adding the device in order to avoid
* duplicates.
* Limited to bits_per_byte * page size devices (ie 32K on most platforms).
* Returns the number of the unit assigned or a negative errno code.
*/
int dev_alloc_name(struct net_device *dev, const char *name)
{
return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE);
}
EXPORT_SYMBOL(dev_alloc_name);
static int dev_get_valid_name(struct net *net, struct net_device *dev,
const char *name)
{
int ret;
ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST);
return ret < 0 ? ret : 0;
}
/**
* dev_change_name - change name of a device
* @dev: device
* @newname: name (or format string) must be at least IFNAMSIZ
*
* Change name of a device, can pass format strings "eth%d".
* for wildcarding.
*/
int dev_change_name(struct net_device *dev, const char *newname)
{
unsigned char old_assign_type;
char oldname[IFNAMSIZ];
int err = 0;
int ret;
struct net *net;
ASSERT_RTNL();
BUG_ON(!dev_net(dev));
net = dev_net(dev);
down_write(&devnet_rename_sem);
if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
up_write(&devnet_rename_sem);
return 0;
}
memcpy(oldname, dev->name, IFNAMSIZ);
err = dev_get_valid_name(net, dev, newname);
if (err < 0) {
up_write(&devnet_rename_sem);
return err;
}
if (oldname[0] && !strchr(oldname, '%'))
netdev_info(dev, "renamed from %s%s\n", oldname,
dev->flags & IFF_UP ? " (while UP)" : "");
old_assign_type = dev->name_assign_type;
WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED);
rollback:
ret = device_rename(&dev->dev, dev->name);
if (ret) {
memcpy(dev->name, oldname, IFNAMSIZ);
WRITE_ONCE(dev->name_assign_type, old_assign_type);
up_write(&devnet_rename_sem);
return ret;
}
up_write(&devnet_rename_sem);
netdev_adjacent_rename_links(dev, oldname);
netdev_name_node_del(dev->name_node);
synchronize_net();
netdev_name_node_add(net, dev->name_node);
ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
ret = notifier_to_errno(ret);
if (ret) {
/* err >= 0 after dev_alloc_name() or stores the first errno */
if (err >= 0) {
err = ret;
down_write(&devnet_rename_sem);
memcpy(dev->name, oldname, IFNAMSIZ);
memcpy(oldname, newname, IFNAMSIZ);
WRITE_ONCE(dev->name_assign_type, old_assign_type);
old_assign_type = NET_NAME_RENAMED;
goto rollback;
} else {
netdev_err(dev, "name change rollback failed: %d\n",
ret);
}
}
return err;
}
/**
* dev_set_alias - change ifalias of a device
* @dev: device
* @alias: name up to IFALIASZ
* @len: limit of bytes to copy from info
*
* Set ifalias for a device,
*/
int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
{
struct dev_ifalias *new_alias = NULL;
if (len >= IFALIASZ)
return -EINVAL;
if (len) {
new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
if (!new_alias)
return -ENOMEM;
memcpy(new_alias->ifalias, alias, len);
new_alias->ifalias[len] = 0;
}
mutex_lock(&ifalias_mutex);
new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
mutex_is_locked(&ifalias_mutex));
mutex_unlock(&ifalias_mutex);
if (new_alias)
kfree_rcu(new_alias, rcuhead);
return len;
}
EXPORT_SYMBOL(dev_set_alias);
/**
* dev_get_alias - get ifalias of a device
* @dev: device
* @name: buffer to store name of ifalias
* @len: size of buffer
*
* get ifalias for a device. Caller must make sure dev cannot go
* away, e.g. rcu read lock or own a reference count to device.
*/
int dev_get_alias(const struct net_device *dev, char *name, size_t len)
{
const struct dev_ifalias *alias;
int ret = 0;
rcu_read_lock(); alias = rcu_dereference(dev->ifalias); if (alias) ret = snprintf(name, len, "%s", alias->ifalias); rcu_read_unlock();
return ret;
}
/**
* netdev_features_change - device changes features
* @dev: device to cause notification
*
* Called to indicate a device has changed features.
*/
void netdev_features_change(struct net_device *dev)
{
call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
}
EXPORT_SYMBOL(netdev_features_change);
/**
* netdev_state_change - device changes state
* @dev: device to cause notification
*
* Called to indicate a device has changed state. This function calls
* the notifier chains for netdev_chain and sends a NEWLINK message
* to the routing socket.
*/
void netdev_state_change(struct net_device *dev)
{
if (dev->flags & IFF_UP) {
struct netdev_notifier_change_info change_info = {
.info.dev = dev,
};
call_netdevice_notifiers_info(NETDEV_CHANGE,
&change_info.info);
rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
}
}
EXPORT_SYMBOL(netdev_state_change);
/**
* __netdev_notify_peers - notify network peers about existence of @dev,
* to be called when rtnl lock is already held.
* @dev: network device
*
* Generate traffic such that interested network peers are aware of
* @dev, such as by generating a gratuitous ARP. This may be used when
* a device wants to inform the rest of the network about some sort of
* reconfiguration such as a failover event or virtual machine
* migration.
*/
void __netdev_notify_peers(struct net_device *dev)
{
ASSERT_RTNL();
call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
}
EXPORT_SYMBOL(__netdev_notify_peers);
/**
* netdev_notify_peers - notify network peers about existence of @dev
* @dev: network device
*
* Generate traffic such that interested network peers are aware of
* @dev, such as by generating a gratuitous ARP. This may be used when
* a device wants to inform the rest of the network about some sort of
* reconfiguration such as a failover event or virtual machine
* migration.
*/
void netdev_notify_peers(struct net_device *dev)
{
rtnl_lock();
__netdev_notify_peers(dev);
rtnl_unlock();
}
EXPORT_SYMBOL(netdev_notify_peers);
static int napi_threaded_poll(void *data);
static int napi_kthread_create(struct napi_struct *n)
{
int err = 0;
/* Create and wake up the kthread once to put it in
* TASK_INTERRUPTIBLE mode to avoid the blocked task
* warning and work with loadavg.
*/
n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
n->dev->name, n->napi_id);
if (IS_ERR(n->thread)) {
err = PTR_ERR(n->thread);
pr_err("kthread_run failed with err %d\n", err);
n->thread = NULL;
}
return err;
}
static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
{
const struct net_device_ops *ops = dev->netdev_ops;
int ret;
ASSERT_RTNL();
dev_addr_check(dev);
if (!netif_device_present(dev)) {
/* may be detached because parent is runtime-suspended */
if (dev->dev.parent)
pm_runtime_resume(dev->dev.parent);
if (!netif_device_present(dev))
return -ENODEV;
}
/* Block netpoll from trying to do any rx path servicing.
* If we don't do this there is a chance ndo_poll_controller
* or ndo_poll may be running while we open the device
*/
netpoll_poll_disable(dev);
ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
ret = notifier_to_errno(ret);
if (ret)
return ret;
set_bit(__LINK_STATE_START, &dev->state);
if (ops->ndo_validate_addr)
ret = ops->ndo_validate_addr(dev);
if (!ret && ops->ndo_open)
ret = ops->ndo_open(dev);
netpoll_poll_enable(dev);
if (ret)
clear_bit(__LINK_STATE_START, &dev->state);
else {
dev->flags |= IFF_UP;
dev_set_rx_mode(dev);
dev_activate(dev);
add_device_randomness(dev->dev_addr, dev->addr_len);
}
return ret;
}
/**
* dev_open - prepare an interface for use.
* @dev: device to open
* @extack: netlink extended ack
*
* Takes a device from down to up state. The device's private open
* function is invoked and then the multicast lists are loaded. Finally
* the device is moved into the up state and a %NETDEV_UP message is
* sent to the netdev notifier chain.
*
* Calling this function on an active interface is a nop. On a failure
* a negative errno code is returned.
*/
int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
{
int ret;
if (dev->flags & IFF_UP)
return 0;
ret = __dev_open(dev, extack);
if (ret < 0)
return ret;
rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
call_netdevice_notifiers(NETDEV_UP, dev);
return ret;
}
EXPORT_SYMBOL(dev_open);
static void __dev_close_many(struct list_head *head)
{
struct net_device *dev;
ASSERT_RTNL(); might_sleep();
list_for_each_entry(dev, head, close_list) {
/* Temporarily disable netpoll until the interface is down */
netpoll_poll_disable(dev);
call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
clear_bit(__LINK_STATE_START, &dev->state);
/* Synchronize to scheduled poll. We cannot touch poll list, it
* can be even on different cpu. So just clear netif_running().
*
* dev->stop() will invoke napi_disable() on all of it's
* napi_struct instances on this device.
*/
smp_mb__after_atomic(); /* Commit netif_running(). */
}
dev_deactivate_many(head);
list_for_each_entry(dev, head, close_list) {
const struct net_device_ops *ops = dev->netdev_ops;
/*
* Call the device specific close. This cannot fail.
* Only if device is UP
*
* We allow it to be called even after a DETACH hot-plug
* event.
*/
if (ops->ndo_stop)
ops->ndo_stop(dev); dev->flags &= ~IFF_UP;
netpoll_poll_enable(dev);
}
}
static void __dev_close(struct net_device *dev)
{
LIST_HEAD(single);
list_add(&dev->close_list, &single);
__dev_close_many(&single);
list_del(&single);
}
void dev_close_many(struct list_head *head, bool unlink)
{
struct net_device *dev, *tmp;
/* Remove the devices that don't need to be closed */
list_for_each_entry_safe(dev, tmp, head, close_list) if (!(dev->flags & IFF_UP)) list_del_init(&dev->close_list); __dev_close_many(head); list_for_each_entry_safe(dev, tmp, head, close_list) { rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
call_netdevice_notifiers(NETDEV_DOWN, dev);
if (unlink)
list_del_init(&dev->close_list);
}
}
EXPORT_SYMBOL(dev_close_many);
/**
* dev_close - shutdown an interface.
* @dev: device to shutdown
*
* This function moves an active device into down state. A
* %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
* is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
* chain.
*/
void dev_close(struct net_device *dev)
{
if (dev->flags & IFF_UP) {
LIST_HEAD(single);
list_add(&dev->close_list, &single);
dev_close_many(&single, true);
list_del(&single);
}
}
EXPORT_SYMBOL(dev_close);
/**
* dev_disable_lro - disable Large Receive Offload on a device
* @dev: device
*
* Disable Large Receive Offload (LRO) on a net device. Must be
* called under RTNL. This is needed if received packets may be
* forwarded to another interface.
*/
void dev_disable_lro(struct net_device *dev)
{
struct net_device *lower_dev;
struct list_head *iter;
dev->wanted_features &= ~NETIF_F_LRO;
netdev_update_features(dev);
if (unlikely(dev->features & NETIF_F_LRO))
netdev_WARN(dev, "failed to disable LRO!\n");
netdev_for_each_lower_dev(dev, lower_dev, iter)
dev_disable_lro(lower_dev);
}
EXPORT_SYMBOL(dev_disable_lro);
/**
* dev_disable_gro_hw - disable HW Generic Receive Offload on a device
* @dev: device
*
* Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
* called under RTNL. This is needed if Generic XDP is installed on
* the device.
*/
static void dev_disable_gro_hw(struct net_device *dev)
{
dev->wanted_features &= ~NETIF_F_GRO_HW;
netdev_update_features(dev);
if (unlikely(dev->features & NETIF_F_GRO_HW))
netdev_WARN(dev, "failed to disable GRO_HW!\n");
}
const char *netdev_cmd_to_name(enum netdev_cmd cmd)
{
#define N(val) \
case NETDEV_##val: \
return "NETDEV_" __stringify(val);
switch (cmd) {
N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
N(XDP_FEAT_CHANGE)
}
#undef N
return "UNKNOWN_NETDEV_EVENT";
}
EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
struct net_device *dev)
{
struct netdev_notifier_info info = {
.dev = dev,
};
return nb->notifier_call(nb, val, &info);
}
static int call_netdevice_register_notifiers(struct notifier_block *nb,
struct net_device *dev)
{
int err;
err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
err = notifier_to_errno(err);
if (err)
return err;
if (!(dev->flags & IFF_UP))
return 0;
call_netdevice_notifier(nb, NETDEV_UP, dev);
return 0;
}
static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
struct net_device *dev)
{
if (dev->flags & IFF_UP) {
call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
dev);
call_netdevice_notifier(nb, NETDEV_DOWN, dev);
}
call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
}
static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
struct net *net)
{
struct net_device *dev;
int err;
for_each_netdev(net, dev) {
err = call_netdevice_register_notifiers(nb, dev);
if (err)
goto rollback;
}
return 0;
rollback:
for_each_netdev_continue_reverse(net, dev)
call_netdevice_unregister_notifiers(nb, dev);
return err;
}
static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
struct net *net)
{
struct net_device *dev;
for_each_netdev(net, dev)
call_netdevice_unregister_notifiers(nb, dev);
}
static int dev_boot_phase = 1;
/**
* register_netdevice_notifier - register a network notifier block
* @nb: notifier
*
* Register a notifier to be called when network device events occur.
* The notifier passed is linked into the kernel structures and must
* not be reused until it has been unregistered. A negative errno code
* is returned on a failure.
*
* When registered all registration and up events are replayed
* to the new notifier to allow device to have a race free
* view of the network device list.
*/
int register_netdevice_notifier(struct notifier_block *nb)
{
struct net *net;
int err;
/* Close race with setup_net() and cleanup_net() */
down_write(&pernet_ops_rwsem);
rtnl_lock();
err = raw_notifier_chain_register(&netdev_chain, nb);
if (err)
goto unlock;
if (dev_boot_phase)
goto unlock;
for_each_net(net) {
err = call_netdevice_register_net_notifiers(nb, net);
if (err)
goto rollback;
}
unlock:
rtnl_unlock();
up_write(&pernet_ops_rwsem);
return err;
rollback:
for_each_net_continue_reverse(net)
call_netdevice_unregister_net_notifiers(nb, net);
raw_notifier_chain_unregister(&netdev_chain, nb);
goto unlock;
}
EXPORT_SYMBOL(register_netdevice_notifier);
/**
* unregister_netdevice_notifier - unregister a network notifier block
* @nb: notifier
*
* Unregister a notifier previously registered by
* register_netdevice_notifier(). The notifier is unlinked into the
* kernel structures and may then be reused. A negative errno code
* is returned on a failure.
*
* After unregistering unregister and down device events are synthesized
* for all devices on the device list to the removed notifier to remove
* the need for special case cleanup code.
*/
int unregister_netdevice_notifier(struct notifier_block *nb)
{
struct net *net;
int err;
/* Close race with setup_net() and cleanup_net() */
down_write(&pernet_ops_rwsem);
rtnl_lock();
err = raw_notifier_chain_unregister(&netdev_chain, nb);
if (err)
goto unlock;
for_each_net(net)
call_netdevice_unregister_net_notifiers(nb, net);
unlock:
rtnl_unlock();
up_write(&pernet_ops_rwsem);
return err;
}
EXPORT_SYMBOL(unregister_netdevice_notifier);
static int __register_netdevice_notifier_net(struct net *net,
struct notifier_block *nb,
bool ignore_call_fail)
{
int err;
err = raw_notifier_chain_register(&net->netdev_chain, nb);
if (err)
return err;
if (dev_boot_phase)
return 0;
err = call_netdevice_register_net_notifiers(nb, net);
if (err && !ignore_call_fail)
goto chain_unregister;
return 0;
chain_unregister:
raw_notifier_chain_unregister(&net->netdev_chain, nb);
return err;
}
static int __unregister_netdevice_notifier_net(struct net *net,
struct notifier_block *nb)
{
int err;
err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
if (err)
return err;
call_netdevice_unregister_net_notifiers(nb, net);
return 0;
}
/**
* register_netdevice_notifier_net - register a per-netns network notifier block
* @net: network namespace
* @nb: notifier
*
* Register a notifier to be called when network device events occur.
* The notifier passed is linked into the kernel structures and must
* not be reused until it has been unregistered. A negative errno code
* is returned on a failure.
*
* When registered all registration and up events are replayed
* to the new notifier to allow device to have a race free
* view of the network device list.
*/
int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
{
int err;
rtnl_lock();
err = __register_netdevice_notifier_net(net, nb, false);
rtnl_unlock();
return err;
}
EXPORT_SYMBOL(register_netdevice_notifier_net);
/**
* unregister_netdevice_notifier_net - unregister a per-netns
* network notifier block
* @net: network namespace
* @nb: notifier
*
* Unregister a notifier previously registered by
* register_netdevice_notifier_net(). The notifier is unlinked from the
* kernel structures and may then be reused. A negative errno code
* is returned on a failure.
*
* After unregistering unregister and down device events are synthesized
* for all devices on the device list to the removed notifier to remove
* the need for special case cleanup code.
*/
int unregister_netdevice_notifier_net(struct net *net,
struct notifier_block *nb)
{
int err;
rtnl_lock();
err = __unregister_netdevice_notifier_net(net, nb);
rtnl_unlock();
return err;
}
EXPORT_SYMBOL(unregister_netdevice_notifier_net);
static void __move_netdevice_notifier_net(struct net *src_net,
struct net *dst_net,
struct notifier_block *nb)
{
__unregister_netdevice_notifier_net(src_net, nb);
__register_netdevice_notifier_net(dst_net, nb, true);
}
int register_netdevice_notifier_dev_net(struct net_device *dev,
struct notifier_block *nb,
struct netdev_net_notifier *nn)
{
int err;
rtnl_lock();
err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
if (!err) {
nn->nb = nb;
list_add(&nn->list, &dev->net_notifier_list);
}
rtnl_unlock();
return err;
}
EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
int unregister_netdevice_notifier_dev_net(struct net_device *dev,
struct notifier_block *nb,
struct netdev_net_notifier *nn)
{
int err;
rtnl_lock();
list_del(&nn->list);
err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
rtnl_unlock();
return err;
}
EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
static void move_netdevice_notifiers_dev_net(struct net_device *dev,
struct net *net)
{
struct netdev_net_notifier *nn;
list_for_each_entry(nn, &dev->net_notifier_list, list)
__move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
}
/**
* call_netdevice_notifiers_info - call all network notifier blocks
* @val: value passed unmodified to notifier function
* @info: notifier information data
*
* Call all network notifier blocks. Parameters and return value
* are as for raw_notifier_call_chain().
*/
int call_netdevice_notifiers_info(unsigned long val,
struct netdev_notifier_info *info)
{
struct net *net = dev_net(info->dev);
int ret;
ASSERT_RTNL();
/* Run per-netns notifier block chain first, then run the global one.
* Hopefully, one day, the global one is going to be removed after
* all notifier block registrators get converted to be per-netns.
*/
ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
if (ret & NOTIFY_STOP_MASK)
return ret;
return raw_notifier_call_chain(&netdev_chain, val, info);
}
/**
* call_netdevice_notifiers_info_robust - call per-netns notifier blocks
* for and rollback on error
* @val_up: value passed unmodified to notifier function
* @val_down: value passed unmodified to the notifier function when
* recovering from an error on @val_up
* @info: notifier information data
*
* Call all per-netns network notifier blocks, but not notifier blocks on
* the global notifier chain. Parameters and return value are as for
* raw_notifier_call_chain_robust().
*/
static int
call_netdevice_notifiers_info_robust(unsigned long val_up,
unsigned long val_down,
struct netdev_notifier_info *info)
{
struct net *net = dev_net(info->dev);
ASSERT_RTNL();
return raw_notifier_call_chain_robust(&net->netdev_chain,
val_up, val_down, info);
}
static int call_netdevice_notifiers_extack(unsigned long val,
struct net_device *dev,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_info info = {
.dev = dev,
.extack = extack,
};
return call_netdevice_notifiers_info(val, &info);
}
/**
* call_netdevice_notifiers - call all network notifier blocks
* @val: value passed unmodified to notifier function
* @dev: net_device pointer passed unmodified to notifier function
*
* Call all network notifier blocks. Parameters and return value
* are as for raw_notifier_call_chain().
*/
int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
{
return call_netdevice_notifiers_extack(val, dev, NULL);
}
EXPORT_SYMBOL(call_netdevice_notifiers);
/**
* call_netdevice_notifiers_mtu - call all network notifier blocks
* @val: value passed unmodified to notifier function
* @dev: net_device pointer passed unmodified to notifier function
* @arg: additional u32 argument passed to the notifier function
*
* Call all network notifier blocks. Parameters and return value
* are as for raw_notifier_call_chain().
*/
static int call_netdevice_notifiers_mtu(unsigned long val,
struct net_device *dev, u32 arg)
{
struct netdev_notifier_info_ext info = {
.info.dev = dev,
.ext.mtu = arg,
};
BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
return call_netdevice_notifiers_info(val, &info.info);
}
#ifdef CONFIG_NET_INGRESS
static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
void net_inc_ingress_queue(void)
{
static_branch_inc(&ingress_needed_key);
}
EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
void net_dec_ingress_queue(void)
{
static_branch_dec(&ingress_needed_key);
}
EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
#endif
#ifdef CONFIG_NET_EGRESS
static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
void net_inc_egress_queue(void)
{
static_branch_inc(&egress_needed_key);
}
EXPORT_SYMBOL_GPL(net_inc_egress_queue);
void net_dec_egress_queue(void)
{
static_branch_dec(&egress_needed_key);
}
EXPORT_SYMBOL_GPL(net_dec_egress_queue);
#endif
DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
EXPORT_SYMBOL(netstamp_needed_key);
#ifdef CONFIG_JUMP_LABEL
static atomic_t netstamp_needed_deferred;
static atomic_t netstamp_wanted;
static void netstamp_clear(struct work_struct *work)
{
int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
int wanted;
wanted = atomic_add_return(deferred, &netstamp_wanted);
if (wanted > 0)
static_branch_enable(&netstamp_needed_key);
else
static_branch_disable(&netstamp_needed_key);
}
static DECLARE_WORK(netstamp_work, netstamp_clear);
#endif
void net_enable_timestamp(void)
{
#ifdef CONFIG_JUMP_LABEL
int wanted = atomic_read(&netstamp_wanted);
while (wanted > 0) {
if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
return;
}
atomic_inc(&netstamp_needed_deferred);
schedule_work(&netstamp_work);
#else
static_branch_inc(&netstamp_needed_key);
#endif
}
EXPORT_SYMBOL(net_enable_timestamp);
void net_disable_timestamp(void)
{
#ifdef CONFIG_JUMP_LABEL
int wanted = atomic_read(&netstamp_wanted);
while (wanted > 1) {
if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
return;
}
atomic_dec(&netstamp_needed_deferred);
schedule_work(&netstamp_work);
#else
static_branch_dec(&netstamp_needed_key);
#endif
}
EXPORT_SYMBOL(net_disable_timestamp);
static inline void net_timestamp_set(struct sk_buff *skb)
{
skb->tstamp = 0;
skb->mono_delivery_time = 0;
if (static_branch_unlikely(&netstamp_needed_key))
skb->tstamp = ktime_get_real();
}
#define net_timestamp_check(COND, SKB) \
if (static_branch_unlikely(&netstamp_needed_key)) { \
if ((COND) && !(SKB)->tstamp) \
(SKB)->tstamp = ktime_get_real(); \
} \
bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
{
return __is_skb_forwardable(dev, skb, true);
}
EXPORT_SYMBOL_GPL(is_skb_forwardable);
static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
bool check_mtu)
{
int ret = ____dev_forward_skb(dev, skb, check_mtu);
if (likely(!ret)) {
skb->protocol = eth_type_trans(skb, dev);
skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
}
return ret;
}
int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
{
return __dev_forward_skb2(dev, skb, true);
}
EXPORT_SYMBOL_GPL(__dev_forward_skb);
/**
* dev_forward_skb - loopback an skb to another netif
*
* @dev: destination network device
* @skb: buffer to forward
*
* return values:
* NET_RX_SUCCESS (no congestion)
* NET_RX_DROP (packet was dropped, but freed)
*
* dev_forward_skb can be used for injecting an skb from the
* start_xmit function of one device into the receive queue
* of another device.
*
* The receiving device may be in another namespace, so
* we have to clear all information in the skb that could
* impact namespace isolation.
*/
int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
{
return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
}
EXPORT_SYMBOL_GPL(dev_forward_skb);
int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
{
return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
}
static inline int deliver_skb(struct sk_buff *skb,
struct packet_type *pt_prev,
struct net_device *orig_dev)
{
if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
return -ENOMEM;
refcount_inc(&skb->users);
return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
}
static inline void deliver_ptype_list_skb(struct sk_buff *skb,
struct packet_type **pt,
struct net_device *orig_dev,
__be16 type,
struct list_head *ptype_list)
{
struct packet_type *ptype, *pt_prev = *pt;
list_for_each_entry_rcu(ptype, ptype_list, list) {
if (ptype->type != type)
continue;
if (pt_prev)
deliver_skb(skb, pt_prev, orig_dev);
pt_prev = ptype;
}
*pt = pt_prev;
}
static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
{
if (!ptype->af_packet_priv || !skb->sk)
return false;
if (ptype->id_match)
return ptype->id_match(ptype, skb->sk);
else if ((struct sock *)ptype->af_packet_priv == skb->sk)
return true;
return false;
}
/**
* dev_nit_active - return true if any network interface taps are in use
*
* @dev: network device to check for the presence of taps
*/
bool dev_nit_active(struct net_device *dev)
{
return !list_empty(&net_hotdata.ptype_all) ||
!list_empty(&dev->ptype_all);
}
EXPORT_SYMBOL_GPL(dev_nit_active);
/*
* Support routine. Sends outgoing frames to any network
* taps currently in use.
*/
void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
{
struct list_head *ptype_list = &net_hotdata.ptype_all;
struct packet_type *ptype, *pt_prev = NULL;
struct sk_buff *skb2 = NULL;
rcu_read_lock();
again:
list_for_each_entry_rcu(ptype, ptype_list, list) {
if (READ_ONCE(ptype->ignore_outgoing))
continue;
/* Never send packets back to the socket
* they originated from - MvS (miquels@drinkel.ow.org)
*/
if (skb_loop_sk(ptype, skb))
continue;
if (pt_prev) {
deliver_skb(skb2, pt_prev, skb->dev);
pt_prev = ptype;
continue;
}
/* need to clone skb, done only once */
skb2 = skb_clone(skb, GFP_ATOMIC);
if (!skb2)
goto out_unlock;
net_timestamp_set(skb2);
/* skb->nh should be correctly
* set by sender, so that the second statement is
* just protection against buggy protocols.
*/
skb_reset_mac_header(skb2);
if (skb_network_header(skb2) < skb2->data ||
skb_network_header(skb2) > skb_tail_pointer(skb2)) {
net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
ntohs(skb2->protocol),
dev->name);
skb_reset_network_header(skb2);
}
skb2->transport_header = skb2->network_header;
skb2->pkt_type = PACKET_OUTGOING;
pt_prev = ptype;
}
if (ptype_list == &net_hotdata.ptype_all) {
ptype_list = &dev->ptype_all;
goto again;
}
out_unlock:
if (pt_prev) {
if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
else
kfree_skb(skb2);
}
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
/**
* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
* @dev: Network device
* @txq: number of queues available
*
* If real_num_tx_queues is changed the tc mappings may no longer be
* valid. To resolve this verify the tc mapping remains valid and if
* not NULL the mapping. With no priorities mapping to this
* offset/count pair it will no longer be used. In the worst case TC0
* is invalid nothing can be done so disable priority mappings. If is
* expected that drivers will fix this mapping if they can before
* calling netif_set_real_num_tx_queues.
*/
static void netif_setup_tc(struct net_device *dev, unsigned int txq)
{
int i;
struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
/* If TC0 is invalidated disable TC mapping */
if (tc->offset + tc->count > txq) {
netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
dev->num_tc = 0;
return;
}
/* Invalidated prio to tc mappings set to TC0 */
for (i = 1; i < TC_BITMASK + 1; i++) {
int q = netdev_get_prio_tc_map(dev, i);
tc = &dev->tc_to_txq[q];
if (tc->offset + tc->count > txq) {
netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
i, q);
netdev_set_prio_tc_map(dev, i, 0);
}
}
}
int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
{
if (dev->num_tc) {
struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
int i;
/* walk through the TCs and see if it falls into any of them */
for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
if ((txq - tc->offset) < tc->count)
return i;
}
/* didn't find it, just return -1 to indicate no match */
return -1;
}
return 0;
}
EXPORT_SYMBOL(netdev_txq_to_tc);
#ifdef CONFIG_XPS
static struct static_key xps_needed __read_mostly;
static struct static_key xps_rxqs_needed __read_mostly;
static DEFINE_MUTEX(xps_map_mutex);
#define xmap_dereference(P) \
rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
struct xps_dev_maps *old_maps, int tci, u16 index)
{
struct xps_map *map = NULL;
int pos;
map = xmap_dereference(dev_maps->attr_map[tci]);
if (!map)
return false;
for (pos = map->len; pos--;) {
if (map->queues[pos] != index)
continue;
if (map->len > 1) {
map->queues[pos] = map->queues[--map->len];
break;
}
if (old_maps)
RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
kfree_rcu(map, rcu);
return false;
}
return true;
}
static bool remove_xps_queue_cpu(struct net_device *dev,
struct xps_dev_maps *dev_maps,
int cpu, u16 offset, u16 count)
{
int num_tc = dev_maps->num_tc;
bool active = false;
int tci;
for (tci = cpu * num_tc; num_tc--; tci++) {
int i, j;
for (i = count, j = offset; i--; j++) {
if (!remove_xps_queue(dev_maps, NULL, tci, j))
break;
}
active |= i < 0;
}
return active;
}
static void reset_xps_maps(struct net_device *dev,
struct xps_dev_maps *dev_maps,
enum xps_map_type type)
{
static_key_slow_dec_cpuslocked(&xps_needed);
if (type == XPS_RXQS)
static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
RCU_INIT_POINTER(dev->xps_maps[type], NULL);
kfree_rcu(dev_maps, rcu);
}
static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
u16 offset, u16 count)
{
struct xps_dev_maps *dev_maps;
bool active = false;
int i, j;
dev_maps = xmap_dereference(dev->xps_maps[type]);
if (!dev_maps)
return;
for (j = 0; j < dev_maps->nr_ids; j++)
active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
if (!active)
reset_xps_maps(dev, dev_maps, type);
if (type == XPS_CPUS) {
for (i = offset + (count - 1); count--; i--)
netdev_queue_numa_node_write(
netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
}
}
static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
u16 count)
{
if (!static_key_false(&xps_needed))
return;
cpus_read_lock();
mutex_lock(&xps_map_mutex);
if (static_key_false(&xps_rxqs_needed))
clean_xps_maps(dev, XPS_RXQS, offset, count);
clean_xps_maps(dev, XPS_CPUS, offset, count);
mutex_unlock(&xps_map_mutex);
cpus_read_unlock();
}
static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
{
netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
}
static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
u16 index, bool is_rxqs_map)
{
struct xps_map *new_map;
int alloc_len = XPS_MIN_MAP_ALLOC;
int i, pos;
for (pos = 0; map && pos < map->len; pos++) {
if (map->queues[pos] != index)
continue;
return map;
}
/* Need to add tx-queue to this CPU's/rx-queue's existing map */
if (map) {
if (pos < map->alloc_len)
return map;
alloc_len = map->alloc_len * 2;
}
/* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
* map
*/
if (is_rxqs_map)
new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
else
new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
cpu_to_node(attr_index));
if (!new_map)
return NULL;
for (i = 0; i < pos; i++)
new_map->queues[i] = map->queues[i];
new_map->alloc_len = alloc_len;
new_map->len = pos;
return new_map;
}
/* Copy xps maps at a given index */
static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
struct xps_dev_maps *new_dev_maps, int index,
int tc, bool skip_tc)
{
int i, tci = index * dev_maps->num_tc;
struct xps_map *map;
/* copy maps belonging to foreign traffic classes */
for (i = 0; i < dev_maps->num_tc; i++, tci++) {
if (i == tc && skip_tc)
continue;
/* fill in the new device map from the old device map */
map = xmap_dereference(dev_maps->attr_map[tci]);
RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
}
}
/* Must be called under cpus_read_lock */
int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
u16 index, enum xps_map_type type)
{
struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
const unsigned long *online_mask = NULL;
bool active = false, copy = false;
int i, j, tci, numa_node_id = -2;
int maps_sz, num_tc = 1, tc = 0;
struct xps_map *map, *new_map;
unsigned int nr_ids;
WARN_ON_ONCE(index >= dev->num_tx_queues);
if (dev->num_tc) {
/* Do not allow XPS on subordinate device directly */
num_tc = dev->num_tc;
if (num_tc < 0)
return -EINVAL;
/* If queue belongs to subordinate dev use its map */
dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
tc = netdev_txq_to_tc(dev, index);
if (tc < 0)
return -EINVAL;
}
mutex_lock(&xps_map_mutex);
dev_maps = xmap_dereference(dev->xps_maps[type]);
if (type == XPS_RXQS) {
maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
nr_ids = dev->num_rx_queues;
} else {
maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
if (num_possible_cpus() > 1)
online_mask = cpumask_bits(cpu_online_mask);
nr_ids = nr_cpu_ids;
}
if (maps_sz < L1_CACHE_BYTES)
maps_sz = L1_CACHE_BYTES;
/* The old dev_maps could be larger or smaller than the one we're
* setting up now, as dev->num_tc or nr_ids could have been updated in
* between. We could try to be smart, but let's be safe instead and only
* copy foreign traffic classes if the two map sizes match.
*/
if (dev_maps &&
dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
copy = true;
/* allocate memory for queue storage */
for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
j < nr_ids;) {
if (!new_dev_maps) {
new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
if (!new_dev_maps) {
mutex_unlock(&xps_map_mutex);
return -ENOMEM;
}
new_dev_maps->nr_ids = nr_ids;
new_dev_maps->num_tc = num_tc;
}
tci = j * num_tc + tc;
map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
map = expand_xps_map(map, j, index, type == XPS_RXQS);
if (!map)
goto error;
RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
}
if (!new_dev_maps)
goto out_no_new_maps;
if (!dev_maps) {
/* Increment static keys at most once per type */
static_key_slow_inc_cpuslocked(&xps_needed);
if (type == XPS_RXQS)
static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
}
for (j = 0; j < nr_ids; j++) {
bool skip_tc = false;
tci = j * num_tc + tc;
if (netif_attr_test_mask(j, mask, nr_ids) &&
netif_attr_test_online(j, online_mask, nr_ids)) {
/* add tx-queue to CPU/rx-queue maps */
int pos = 0;
skip_tc = true;
map = xmap_dereference(new_dev_maps->attr_map[tci]);
while ((pos < map->len) && (map->queues[pos] != index))
pos++;
if (pos == map->len)
map->queues[map->len++] = index;
#ifdef CONFIG_NUMA
if (type == XPS_CPUS) {
if (numa_node_id == -2)
numa_node_id = cpu_to_node(j);
else if (numa_node_id != cpu_to_node(j))
numa_node_id = -1;
}
#endif
}
if (copy)
xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
skip_tc);
}
rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
/* Cleanup old maps */
if (!dev_maps)
goto out_no_old_maps;
for (j = 0; j < dev_maps->nr_ids; j++) {
for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
map = xmap_dereference(dev_maps->attr_map[tci]);
if (!map)
continue;
if (copy) {
new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
if (map == new_map)
continue;
}
RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
kfree_rcu(map, rcu);
}
}
old_dev_maps = dev_maps;
out_no_old_maps:
dev_maps = new_dev_maps;
active = true;
out_no_new_maps:
if (type == XPS_CPUS)
/* update Tx queue numa node */
netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
(numa_node_id >= 0) ?
numa_node_id : NUMA_NO_NODE);
if (!dev_maps)
goto out_no_maps;
/* removes tx-queue from unused CPUs/rx-queues */
for (j = 0; j < dev_maps->nr_ids; j++) {
tci = j * dev_maps->num_tc;
for (i = 0; i < dev_maps->num_tc; i++, tci++) {
if (i == tc &&
netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
continue;
active |= remove_xps_queue(dev_maps,
copy ? old_dev_maps : NULL,
tci, index);
}
}
if (old_dev_maps)
kfree_rcu(old_dev_maps, rcu);
/* free map if not active */
if (!active)
reset_xps_maps(dev, dev_maps, type);
out_no_maps:
mutex_unlock(&xps_map_mutex);
return 0;
error:
/* remove any maps that we added */
for (j = 0; j < nr_ids; j++) {
for (i = num_tc, tci = j * num_tc; i--; tci++) {
new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
map = copy ?
xmap_dereference(dev_maps->attr_map[tci]) :
NULL;
if (new_map && new_map != map)
kfree(new_map);
}
}
mutex_unlock(&xps_map_mutex);
kfree(new_dev_maps);
return -ENOMEM;
}
EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
u16 index)
{
int ret;
cpus_read_lock();
ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
cpus_read_unlock();
return ret;
}
EXPORT_SYMBOL(netif_set_xps_queue);
#endif
static void netdev_unbind_all_sb_channels(struct net_device *dev)
{
struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
/* Unbind any subordinate channels */
while (txq-- != &dev->_tx[0]) {
if (txq->sb_dev)
netdev_unbind_sb_channel(dev, txq->sb_dev);
}
}
void netdev_reset_tc(struct net_device *dev)
{
#ifdef CONFIG_XPS
netif_reset_xps_queues_gt(dev, 0);
#endif
netdev_unbind_all_sb_channels(dev);
/* Reset TC configuration of device */
dev->num_tc = 0;
memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
}
EXPORT_SYMBOL(netdev_reset_tc);
int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
{
if (tc >= dev->num_tc)
return -EINVAL;
#ifdef CONFIG_XPS
netif_reset_xps_queues(dev, offset, count);
#endif
dev->tc_to_txq[tc].count = count;
dev->tc_to_txq[tc].offset = offset;
return 0;
}
EXPORT_SYMBOL(netdev_set_tc_queue);
int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
{
if (num_tc > TC_MAX_QUEUE)
return -EINVAL;
#ifdef CONFIG_XPS
netif_reset_xps_queues_gt(dev, 0);
#endif
netdev_unbind_all_sb_channels(dev);
dev->num_tc = num_tc;
return 0;
}
EXPORT_SYMBOL(netdev_set_num_tc);
void netdev_unbind_sb_channel(struct net_device *dev,
struct net_device *sb_dev)
{
struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
#ifdef CONFIG_XPS
netif_reset_xps_queues_gt(sb_dev, 0);
#endif
memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
while (txq-- != &dev->_tx[0]) {
if (txq->sb_dev == sb_dev)
txq->sb_dev = NULL;
}
}
EXPORT_SYMBOL(netdev_unbind_sb_channel);
int netdev_bind_sb_channel_queue(struct net_device *dev,
struct net_device *sb_dev,
u8 tc, u16 count, u16 offset)
{
/* Make certain the sb_dev and dev are already configured */
if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
return -EINVAL;
/* We cannot hand out queues we don't have */
if ((offset + count) > dev->real_num_tx_queues)
return -EINVAL;
/* Record the mapping */
sb_dev->tc_to_txq[tc].count = count;
sb_dev->tc_to_txq[tc].offset = offset;
/* Provide a way for Tx queue to find the tc_to_txq map or
* XPS map for itself.
*/
while (count--)
netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
return 0;
}
EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
int netdev_set_sb_channel(struct net_device *dev, u16 channel)
{
/* Do not use a multiqueue device to represent a subordinate channel */
if (netif_is_multiqueue(dev))
return -ENODEV;
/* We allow channels 1 - 32767 to be used for subordinate channels.
* Channel 0 is meant to be "native" mode and used only to represent
* the main root device. We allow writing 0 to reset the device back
* to normal mode after being used as a subordinate channel.
*/
if (channel > S16_MAX)
return -EINVAL;
dev->num_tc = -channel;
return 0;
}
EXPORT_SYMBOL(netdev_set_sb_channel);
/*
* Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
* greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
*/
int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
{
bool disabling;
int rc;
disabling = txq < dev->real_num_tx_queues;
if (txq < 1 || txq > dev->num_tx_queues)
return -EINVAL;
if (dev->reg_state == NETREG_REGISTERED ||
dev->reg_state == NETREG_UNREGISTERING) {
ASSERT_RTNL();
rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
txq);
if (rc)
return rc;
if (dev->num_tc)
netif_setup_tc(dev, txq);
dev_qdisc_change_real_num_tx(dev, txq);
dev->real_num_tx_queues = txq;
if (disabling) {
synchronize_net();
qdisc_reset_all_tx_gt(dev, txq);
#ifdef CONFIG_XPS
netif_reset_xps_queues_gt(dev, txq);
#endif
}
} else {
dev->real_num_tx_queues = txq;
}
return 0;
}
EXPORT_SYMBOL(netif_set_real_num_tx_queues);
#ifdef CONFIG_SYSFS
/**
* netif_set_real_num_rx_queues - set actual number of RX queues used
* @dev: Network device
* @rxq: Actual number of RX queues
*
* This must be called either with the rtnl_lock held or before
* registration of the net device. Returns 0 on success, or a
* negative error code. If called before registration, it always
* succeeds.
*/
int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
{
int rc;
if (rxq < 1 || rxq > dev->num_rx_queues)
return -EINVAL;
if (dev->reg_state == NETREG_REGISTERED) {
ASSERT_RTNL();
rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
rxq);
if (rc)
return rc;
}
dev->real_num_rx_queues = rxq;
return 0;
}
EXPORT_SYMBOL(netif_set_real_num_rx_queues);
#endif
/**
* netif_set_real_num_queues - set actual number of RX and TX queues used
* @dev: Network device
* @txq: Actual number of TX queues
* @rxq: Actual number of RX queues
*
* Set the real number of both TX and RX queues.
* Does nothing if the number of queues is already correct.
*/
int netif_set_real_num_queues(struct net_device *dev,
unsigned int txq, unsigned int rxq)
{
unsigned int old_rxq = dev->real_num_rx_queues;
int err;
if (txq < 1 || txq > dev->num_tx_queues ||
rxq < 1 || rxq > dev->num_rx_queues)
return -EINVAL;
/* Start from increases, so the error path only does decreases -
* decreases can't fail.
*/
if (rxq > dev->real_num_rx_queues) {
err = netif_set_real_num_rx_queues(dev, rxq);
if (err)
return err;
}
if (txq > dev->real_num_tx_queues) {
err = netif_set_real_num_tx_queues(dev, txq);
if (err)
goto undo_rx;
}
if (rxq < dev->real_num_rx_queues)
WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
if (txq < dev->real_num_tx_queues)
WARN_ON(netif_set_real_num_tx_queues(dev, txq));
return 0;
undo_rx:
WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
return err;
}
EXPORT_SYMBOL(netif_set_real_num_queues);
/**
* netif_set_tso_max_size() - set the max size of TSO frames supported
* @dev: netdev to update
* @size: max skb->len of a TSO frame
*
* Set the limit on the size of TSO super-frames the device can handle.
* Unless explicitly set the stack will assume the value of
* %GSO_LEGACY_MAX_SIZE.
*/
void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
{
dev->tso_max_size = min(GSO_MAX_SIZE, size);
if (size < READ_ONCE(dev->gso_max_size))
netif_set_gso_max_size(dev, size);
if (size < READ_ONCE(dev->gso_ipv4_max_size))
netif_set_gso_ipv4_max_size(dev, size);
}
EXPORT_SYMBOL(netif_set_tso_max_size);
/**
* netif_set_tso_max_segs() - set the max number of segs supported for TSO
* @dev: netdev to update
* @segs: max number of TCP segments
*
* Set the limit on the number of TCP segments the device can generate from
* a single TSO super-frame.
* Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
*/
void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
{
dev->tso_max_segs = segs;
if (segs < READ_ONCE(dev->gso_max_segs))
netif_set_gso_max_segs(dev, segs);
}
EXPORT_SYMBOL(netif_set_tso_max_segs);
/**
* netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
* @to: netdev to update
* @from: netdev from which to copy the limits
*/
void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
{
netif_set_tso_max_size(to, from->tso_max_size);
netif_set_tso_max_segs(to, from->tso_max_segs);
}
EXPORT_SYMBOL(netif_inherit_tso_max);
/**
* netif_get_num_default_rss_queues - default number of RSS queues
*
* Default value is the number of physical cores if there are only 1 or 2, or
* divided by 2 if there are more.
*/
int netif_get_num_default_rss_queues(void)
{
cpumask_var_t cpus;
int cpu, count = 0;
if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
return 1;
cpumask_copy(cpus, cpu_online_mask);
for_each_cpu(cpu, cpus) {
++count;
cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
}
free_cpumask_var(cpus);
return count > 2 ? DIV_ROUND_UP(count, 2) : count;
}
EXPORT_SYMBOL(netif_get_num_default_rss_queues);
static void __netif_reschedule(struct Qdisc *q)
{
struct softnet_data *sd;
unsigned long flags;
local_irq_save(flags); sd = this_cpu_ptr(&softnet_data);
q->next_sched = NULL;
*sd->output_queue_tailp = q;
sd->output_queue_tailp = &q->next_sched;
raise_softirq_irqoff(NET_TX_SOFTIRQ);
local_irq_restore(flags);
}
void __netif_schedule(struct Qdisc *q)
{
if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) __netif_reschedule(q);
}
EXPORT_SYMBOL(__netif_schedule);
struct dev_kfree_skb_cb {
enum skb_drop_reason reason;
};
static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
{
return (struct dev_kfree_skb_cb *)skb->cb;
}
void netif_schedule_queue(struct netdev_queue *txq)
{
rcu_read_lock(); if (!netif_xmit_stopped(txq)) { struct Qdisc *q = rcu_dereference(txq->qdisc); __netif_schedule(q);
}
rcu_read_unlock();
}
EXPORT_SYMBOL(netif_schedule_queue);
void netif_tx_wake_queue(struct netdev_queue *dev_queue)
{
if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
struct Qdisc *q;
rcu_read_lock();
q = rcu_dereference(dev_queue->qdisc);
__netif_schedule(q);
rcu_read_unlock();
}
}
EXPORT_SYMBOL(netif_tx_wake_queue);
void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
unsigned long flags;
if (unlikely(!skb))
return;
if (likely(refcount_read(&skb->users) == 1)) {
smp_rmb();
refcount_set(&skb->users, 0);
} else if (likely(!refcount_dec_and_test(&skb->users))) {
return;
}
get_kfree_skb_cb(skb)->reason = reason;
local_irq_save(flags);
skb->next = __this_cpu_read(softnet_data.completion_queue);
__this_cpu_write(softnet_data.completion_queue, skb);
raise_softirq_irqoff(NET_TX_SOFTIRQ);
local_irq_restore(flags);
}
EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
if (in_hardirq() || irqs_disabled())
dev_kfree_skb_irq_reason(skb, reason);
else
kfree_skb_reason(skb, reason);
}
EXPORT_SYMBOL(dev_kfree_skb_any_reason);
/**
* netif_device_detach - mark device as removed
* @dev: network device
*
* Mark device as removed from system and therefore no longer available.
*/
void netif_device_detach(struct net_device *dev)
{
if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
netif_running(dev)) {
netif_tx_stop_all_queues(dev);
}
}
EXPORT_SYMBOL(netif_device_detach);
/**
* netif_device_attach - mark device as attached
* @dev: network device
*
* Mark device as attached from system and restart if needed.
*/
void netif_device_attach(struct net_device *dev)
{
if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
netif_running(dev)) {
netif_tx_wake_all_queues(dev);
__netdev_watchdog_up(dev);
}
}
EXPORT_SYMBOL(netif_device_attach);
/*
* Returns a Tx hash based on the given packet descriptor a Tx queues' number
* to be used as a distribution range.
*/
static u16 skb_tx_hash(const struct net_device *dev,
const struct net_device *sb_dev,
struct sk_buff *skb)
{
u32 hash;
u16 qoffset = 0;
u16 qcount = dev->real_num_tx_queues;
if (dev->num_tc) {
u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
qoffset = sb_dev->tc_to_txq[tc].offset;
qcount = sb_dev->tc_to_txq[tc].count;
if (unlikely(!qcount)) {
net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
sb_dev->name, qoffset, tc);
qoffset = 0;
qcount = dev->real_num_tx_queues;
}
}
if (skb_rx_queue_recorded(skb)) {
DEBUG_NET_WARN_ON_ONCE(qcount == 0);
hash = skb_get_rx_queue(skb);
if (hash >= qoffset)
hash -= qoffset;
while (unlikely(hash >= qcount))
hash -= qcount;
return hash + qoffset;
}
return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
}
void skb_warn_bad_offload(const struct sk_buff *skb)
{
static const netdev_features_t null_features;
struct net_device *dev = skb->dev;
const char *name = "";
if (!net_ratelimit())
return;
if (dev) {
if (dev->dev.parent)
name = dev_driver_string(dev->dev.parent);
else
name = netdev_name(dev);
}
skb_dump(KERN_WARNING, skb, false);
WARN(1, "%s: caps=(%pNF, %pNF)\n",
name, dev ? &dev->features : &null_features,
skb->sk ? &skb->sk->sk_route_caps : &null_features);
}
/*
* Invalidate hardware checksum when packet is to be mangled, and
* complete checksum manually on outgoing path.
*/
int skb_checksum_help(struct sk_buff *skb)
{
__wsum csum;
int ret = 0, offset;
if (skb->ip_summed == CHECKSUM_COMPLETE)
goto out_set_summed;
if (unlikely(skb_is_gso(skb))) {
skb_warn_bad_offload(skb);
return -EINVAL;
}
/* Before computing a checksum, we should make sure no frag could
* be modified by an external entity : checksum could be wrong.
*/
if (skb_has_shared_frag(skb)) {
ret = __skb_linearize(skb);
if (ret)
goto out;
}
offset = skb_checksum_start_offset(skb);
ret = -EINVAL;
if (unlikely(offset >= skb_headlen(skb))) {
DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
offset, skb_headlen(skb));
goto out;
}
csum = skb_checksum(skb, offset, skb->len - offset, 0);
offset += skb->csum_offset;
if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
offset + sizeof(__sum16), skb_headlen(skb));
goto out;
}
ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
if (ret)
goto out;
*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
out_set_summed:
skb->ip_summed = CHECKSUM_NONE;
out:
return ret;
}
EXPORT_SYMBOL(skb_checksum_help);
int skb_crc32c_csum_help(struct sk_buff *skb)
{
__le32 crc32c_csum;
int ret = 0, offset, start;
if (skb->ip_summed != CHECKSUM_PARTIAL)
goto out;
if (unlikely(skb_is_gso(skb)))
goto out;
/* Before computing a checksum, we should make sure no frag could
* be modified by an external entity : checksum could be wrong.
*/
if (unlikely(skb_has_shared_frag(skb))) {
ret = __skb_linearize(skb);
if (ret)
goto out;
}
start = skb_checksum_start_offset(skb);
offset = start + offsetof(struct sctphdr, checksum);
if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
ret = -EINVAL;
goto out;
}
ret = skb_ensure_writable(skb, offset + sizeof(__le32));
if (ret)
goto out;
crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
skb->len - start, ~(__u32)0,
crc32c_csum_stub));
*(__le32 *)(skb->data + offset) = crc32c_csum;
skb_reset_csum_not_inet(skb);
out:
return ret;
}
__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
{
__be16 type = skb->protocol;
/* Tunnel gso handlers can set protocol to ethernet. */
if (type == htons(ETH_P_TEB)) {
struct ethhdr *eth;
if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
return 0;
eth = (struct ethhdr *)skb->data;
type = eth->h_proto;
}
return vlan_get_protocol_and_depth(skb, type, depth);
}
/* Take action when hardware reception checksum errors are detected. */
#ifdef CONFIG_BUG
static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
{
netdev_err(dev, "hw csum failure\n");
skb_dump(KERN_ERR, skb, true);
dump_stack();
}
void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
{
DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
}
EXPORT_SYMBOL(netdev_rx_csum_fault);
#endif
/* XXX: check that highmem exists at all on the given machine. */
static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
{
#ifdef CONFIG_HIGHMEM
int i;
if (!(dev->features & NETIF_F_HIGHDMA)) {
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
if (PageHighMem(skb_frag_page(frag)))
return 1;
}
}
#endif
return 0;
}
/* If MPLS offload request, verify we are testing hardware MPLS features
* instead of standard features for the netdev.
*/
#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
static netdev_features_t net_mpls_features(struct sk_buff *skb,
netdev_features_t features,
__be16 type)
{
if (eth_p_mpls(type))
features &= skb->dev->mpls_features;
return features;
}
#else
static netdev_features_t net_mpls_features(struct sk_buff *skb,
netdev_features_t features,
__be16 type)
{
return features;
}
#endif
static netdev_features_t harmonize_features(struct sk_buff *skb,
netdev_features_t features)
{
__be16 type;
type = skb_network_protocol(skb, NULL);
features = net_mpls_features(skb, features, type);
if (skb->ip_summed != CHECKSUM_NONE &&
!can_checksum_protocol(features, type)) {
features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
}
if (illegal_highdma(skb->dev, skb))
features &= ~NETIF_F_SG;
return features;
}
netdev_features_t passthru_features_check(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features)
{
return features;
}
EXPORT_SYMBOL(passthru_features_check);
static netdev_features_t dflt_features_check(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features)
{
return vlan_features_check(skb, features);
}
static netdev_features_t gso_features_check(const struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features)
{
u16 gso_segs = skb_shinfo(skb)->gso_segs;
if (gso_segs > READ_ONCE(dev->gso_max_segs))
return features & ~NETIF_F_GSO_MASK;
if (unlikely(skb->len >= READ_ONCE(dev->gso_max_size)))
return features & ~NETIF_F_GSO_MASK;
if (!skb_shinfo(skb)->gso_type) {
skb_warn_bad_offload(skb);
return features & ~NETIF_F_GSO_MASK;
}
/* Support for GSO partial features requires software
* intervention before we can actually process the packets
* so we need to strip support for any partial features now
* and we can pull them back in after we have partially
* segmented the frame.
*/
if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
features &= ~dev->gso_partial_features;
/* Make sure to clear the IPv4 ID mangling feature if the
* IPv4 header has the potential to be fragmented.
*/
if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
struct iphdr *iph = skb->encapsulation ?
inner_ip_hdr(skb) : ip_hdr(skb);
if (!(iph->frag_off & htons(IP_DF)))
features &= ~NETIF_F_TSO_MANGLEID;
}
return features;
}
netdev_features_t netif_skb_features(struct sk_buff *skb)
{
struct net_device *dev = skb->dev;
netdev_features_t features = dev->features;
if (skb_is_gso(skb))
features = gso_features_check(skb, dev, features);
/* If encapsulation offload request, verify we are testing
* hardware encapsulation features instead of standard
* features for the netdev
*/
if (skb->encapsulation)
features &= dev->hw_enc_features;
if (skb_vlan_tagged(skb))
features = netdev_intersect_features(features,
dev->vlan_features |
NETIF_F_HW_VLAN_CTAG_TX |
NETIF_F_HW_VLAN_STAG_TX);
if (dev->netdev_ops->ndo_features_check)
features &= dev->netdev_ops->ndo_features_check(skb, dev,
features);
else
features &= dflt_features_check(skb, dev, features);
return harmonize_features(skb, features);
}
EXPORT_SYMBOL(netif_skb_features);
static int xmit_one(struct sk_buff *skb, struct net_device *dev,
struct netdev_queue *txq, bool more)
{
unsigned int len;
int rc;
if (dev_nit_active(dev))
dev_queue_xmit_nit(skb, dev);
len = skb->len;
trace_net_dev_start_xmit(skb, dev);
rc = netdev_start_xmit(skb, dev, txq, more);
trace_net_dev_xmit(skb, rc, dev, len);
return rc;
}
struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
struct netdev_queue *txq, int *ret)
{
struct sk_buff *skb = first;
int rc = NETDEV_TX_OK;
while (skb) {
struct sk_buff *next = skb->next;
skb_mark_not_on_list(skb);
rc = xmit_one(skb, dev, txq, next != NULL);
if (unlikely(!dev_xmit_complete(rc))) {
skb->next = next;
goto out;
}
skb = next;
if (netif_tx_queue_stopped(txq) && skb) {
rc = NETDEV_TX_BUSY;
break;
}
}
out:
*ret = rc;
return skb;
}
static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
netdev_features_t features)
{
if (skb_vlan_tag_present(skb) &&
!vlan_hw_offload_capable(features, skb->vlan_proto))
skb = __vlan_hwaccel_push_inside(skb);
return skb;
}
int skb_csum_hwoffload_help(struct sk_buff *skb,
const netdev_features_t features)
{
if (unlikely(skb_csum_is_sctp(skb)))
return !!(features & NETIF_F_SCTP_CRC) ? 0 :
skb_crc32c_csum_help(skb);
if (features & NETIF_F_HW_CSUM)
return 0;
if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
switch (skb->csum_offset) {
case offsetof(struct tcphdr, check):
case offsetof(struct udphdr, check):
return 0;
}
}
return skb_checksum_help(skb);
}
EXPORT_SYMBOL(skb_csum_hwoffload_help);
static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
{
netdev_features_t features;
features = netif_skb_features(skb);
skb = validate_xmit_vlan(skb, features);
if (unlikely(!skb))
goto out_null;
skb = sk_validate_xmit_skb(skb, dev);
if (unlikely(!skb))
goto out_null;
if (netif_needs_gso(skb, features)) {
struct sk_buff *segs;
segs = skb_gso_segment(skb, features);
if (IS_ERR(segs)) {
goto out_kfree_skb;
} else if (segs) {
consume_skb(skb);
skb = segs;
}
} else {
if (skb_needs_linearize(skb, features) &&
__skb_linearize(skb))
goto out_kfree_skb;
/* If packet is not checksummed and device does not
* support checksumming for this protocol, complete
* checksumming here.
*/
if (skb->ip_summed == CHECKSUM_PARTIAL) {
if (skb->encapsulation)
skb_set_inner_transport_header(skb,
skb_checksum_start_offset(skb));
else
skb_set_transport_header(skb,
skb_checksum_start_offset(skb));
if (skb_csum_hwoffload_help(skb, features))
goto out_kfree_skb;
}
}
skb = validate_xmit_xfrm(skb, features, again);
return skb;
out_kfree_skb:
kfree_skb(skb);
out_null:
dev_core_stats_tx_dropped_inc(dev);
return NULL;
}
struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
{
struct sk_buff *next, *head = NULL, *tail;
for (; skb != NULL; skb = next) {
next = skb->next;
skb_mark_not_on_list(skb);
/* in case skb wont be segmented, point to itself */
skb->prev = skb;
skb = validate_xmit_skb(skb, dev, again);
if (!skb)
continue;
if (!head)
head = skb;
else
tail->next = skb;
/* If skb was segmented, skb->prev points to
* the last segment. If not, it still contains skb.
*/
tail = skb->prev;
}
return head;
}
EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
static void qdisc_pkt_len_init(struct sk_buff *skb)
{
const struct skb_shared_info *shinfo = skb_shinfo(skb);
qdisc_skb_cb(skb)->pkt_len = skb->len;
/* To get more precise estimation of bytes sent on wire,
* we add to pkt_len the headers size of all segments
*/
if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
u16 gso_segs = shinfo->gso_segs;
unsigned int hdr_len;
/* mac layer + network layer */
hdr_len = skb_transport_offset(skb);
/* + transport layer */
if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
const struct tcphdr *th;
struct tcphdr _tcphdr;
th = skb_header_pointer(skb, hdr_len,
sizeof(_tcphdr), &_tcphdr);
if (likely(th))
hdr_len += __tcp_hdrlen(th);
} else {
struct udphdr _udphdr;
if (skb_header_pointer(skb, hdr_len,
sizeof(_udphdr), &_udphdr))
hdr_len += sizeof(struct udphdr);
}
if (shinfo->gso_type & SKB_GSO_DODGY)
gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
shinfo->gso_size);
qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
}
}
static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
struct sk_buff **to_free,
struct netdev_queue *txq)
{
int rc;
rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
if (rc == NET_XMIT_SUCCESS)
trace_qdisc_enqueue(q, txq, skb);
return rc;
}
static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
struct net_device *dev,
struct netdev_queue *txq)
{
spinlock_t *root_lock = qdisc_lock(q);
struct sk_buff *to_free = NULL;
bool contended;
int rc;
qdisc_calculate_pkt_len(skb, q);
tcf_set_drop_reason(skb, SKB_DROP_REASON_QDISC_DROP);
if (q->flags & TCQ_F_NOLOCK) {
if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
qdisc_run_begin(q)) {
/* Retest nolock_qdisc_is_empty() within the protection
* of q->seqlock to protect from racing with requeuing.
*/
if (unlikely(!nolock_qdisc_is_empty(q))) {
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
__qdisc_run(q);
qdisc_run_end(q);
goto no_lock_out;
}
qdisc_bstats_cpu_update(q, skb);
if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
!nolock_qdisc_is_empty(q))
__qdisc_run(q);
qdisc_run_end(q);
return NET_XMIT_SUCCESS;
}
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
qdisc_run(q);
no_lock_out:
if (unlikely(to_free))
kfree_skb_list_reason(to_free,
tcf_get_drop_reason(to_free));
return rc;
}
if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) {
kfree_skb_reason(skb, SKB_DROP_REASON_TC_RECLASSIFY_LOOP);
return NET_XMIT_DROP;
}
/*
* Heuristic to force contended enqueues to serialize on a
* separate lock before trying to get qdisc main lock.
* This permits qdisc->running owner to get the lock more
* often and dequeue packets faster.
* On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
* and then other tasks will only enqueue packets. The packets will be
* sent after the qdisc owner is scheduled again. To prevent this
* scenario the task always serialize on the lock.
*/
contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
if (unlikely(contended))
spin_lock(&q->busylock);
spin_lock(root_lock);
if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
__qdisc_drop(skb, &to_free);
rc = NET_XMIT_DROP;
} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
qdisc_run_begin(q)) {
/*
* This is a work-conserving queue; there are no old skbs
* waiting to be sent out; and the qdisc is not running -
* xmit the skb directly.
*/
qdisc_bstats_update(q, skb);
if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
if (unlikely(contended)) {
spin_unlock(&q->busylock);
contended = false;
}
__qdisc_run(q);
}
qdisc_run_end(q);
rc = NET_XMIT_SUCCESS;
} else {
WRITE_ONCE(q->owner, smp_processor_id());
rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
WRITE_ONCE(q->owner, -1);
if (qdisc_run_begin(q)) {
if (unlikely(contended)) {
spin_unlock(&q->busylock);
contended = false;
}
__qdisc_run(q);
qdisc_run_end(q);
}
}
spin_unlock(root_lock);
if (unlikely(to_free))
kfree_skb_list_reason(to_free,
tcf_get_drop_reason(to_free));
if (unlikely(contended))
spin_unlock(&q->busylock);
return rc;
}
#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
static void skb_update_prio(struct sk_buff *skb)
{
const struct netprio_map *map;
const struct sock *sk;
unsigned int prioidx;
if (skb->priority)
return;
map = rcu_dereference_bh(skb->dev->priomap);
if (!map)
return;
sk = skb_to_full_sk(skb);
if (!sk)
return;
prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
if (prioidx < map->priomap_len)
skb->priority = map->priomap[prioidx];
}
#else
#define skb_update_prio(skb)
#endif
/**
* dev_loopback_xmit - loop back @skb
* @net: network namespace this loopback is happening in
* @sk: sk needed to be a netfilter okfn
* @skb: buffer to transmit
*/
int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
{
skb_reset_mac_header(skb);
__skb_pull(skb, skb_network_offset(skb));
skb->pkt_type = PACKET_LOOPBACK;
if (skb->ip_summed == CHECKSUM_NONE)
skb->ip_summed = CHECKSUM_UNNECESSARY;
DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
skb_dst_force(skb);
netif_rx(skb);
return 0;
}
EXPORT_SYMBOL(dev_loopback_xmit);
#ifdef CONFIG_NET_EGRESS
static struct netdev_queue *
netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
{
int qm = skb_get_queue_mapping(skb);
return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
}
static bool netdev_xmit_txqueue_skipped(void)
{
return __this_cpu_read(softnet_data.xmit.skip_txqueue);
}
void netdev_xmit_skip_txqueue(bool skip)
{
__this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
}
EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
#endif /* CONFIG_NET_EGRESS */
#ifdef CONFIG_NET_XGRESS
static int tc_run(struct tcx_entry *entry, struct sk_buff *skb,
enum skb_drop_reason *drop_reason)
{
int ret = TC_ACT_UNSPEC;
#ifdef CONFIG_NET_CLS_ACT
struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq);
struct tcf_result res;
if (!miniq)
return ret;
tc_skb_cb(skb)->mru = 0;
tc_skb_cb(skb)->post_ct = false;
tcf_set_drop_reason(skb, *drop_reason);
mini_qdisc_bstats_cpu_update(miniq, skb);
ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false);
/* Only tcf related quirks below. */
switch (ret) {
case TC_ACT_SHOT:
*drop_reason = tcf_get_drop_reason(skb);
mini_qdisc_qstats_cpu_drop(miniq);
break;
case TC_ACT_OK:
case TC_ACT_RECLASSIFY:
skb->tc_index = TC_H_MIN(res.classid);
break;
}
#endif /* CONFIG_NET_CLS_ACT */
return ret;
}
static DEFINE_STATIC_KEY_FALSE(tcx_needed_key);
void tcx_inc(void)
{
static_branch_inc(&tcx_needed_key);
}
void tcx_dec(void)
{
static_branch_dec(&tcx_needed_key);
}
static __always_inline enum tcx_action_base
tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
const bool needs_mac)
{
const struct bpf_mprog_fp *fp;
const struct bpf_prog *prog;
int ret = TCX_NEXT;
if (needs_mac)
__skb_push(skb, skb->mac_len);
bpf_mprog_foreach_prog(entry, fp, prog) {
bpf_compute_data_pointers(skb);
ret = bpf_prog_run(prog, skb);
if (ret != TCX_NEXT)
break;
}
if (needs_mac)
__skb_pull(skb, skb->mac_len);
return tcx_action_code(skb, ret);
}
static __always_inline struct sk_buff *
sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
struct net_device *orig_dev, bool *another)
{
struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS;
int sch_ret;
if (!entry)
return skb;
if (*pt_prev) {
*ret = deliver_skb(skb, *pt_prev, orig_dev);
*pt_prev = NULL;
}
qdisc_skb_cb(skb)->pkt_len = skb->len;
tcx_set_ingress(skb, true);
if (static_branch_unlikely(&tcx_needed_key)) {
sch_ret = tcx_run(entry, skb, true);
if (sch_ret != TC_ACT_UNSPEC)
goto ingress_verdict;
}
sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
ingress_verdict:
switch (sch_ret) {
case TC_ACT_REDIRECT:
/* skb_mac_header check was done by BPF, so we can safely
* push the L2 header back before redirecting to another
* netdev.
*/
__skb_push(skb, skb->mac_len);
if (skb_do_redirect(skb) == -EAGAIN) {
__skb_pull(skb, skb->mac_len);
*another = true;
break;
}
*ret = NET_RX_SUCCESS;
return NULL;
case TC_ACT_SHOT:
kfree_skb_reason(skb, drop_reason);
*ret = NET_RX_DROP;
return NULL;
/* used by tc_run */
case TC_ACT_STOLEN:
case TC_ACT_QUEUED:
case TC_ACT_TRAP:
consume_skb(skb);
fallthrough;
case TC_ACT_CONSUMED:
*ret = NET_RX_SUCCESS;
return NULL;
}
return skb;
}
static __always_inline struct sk_buff *
sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
{
struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS;
int sch_ret;
if (!entry)
return skb;
/* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
* already set by the caller.
*/
if (static_branch_unlikely(&tcx_needed_key)) {
sch_ret = tcx_run(entry, skb, false);
if (sch_ret != TC_ACT_UNSPEC)
goto egress_verdict;
}
sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
egress_verdict:
switch (sch_ret) {
case TC_ACT_REDIRECT:
/* No need to push/pop skb's mac_header here on egress! */
skb_do_redirect(skb);
*ret = NET_XMIT_SUCCESS;
return NULL;
case TC_ACT_SHOT:
kfree_skb_reason(skb, drop_reason);
*ret = NET_XMIT_DROP;
return NULL;
/* used by tc_run */
case TC_ACT_STOLEN:
case TC_ACT_QUEUED:
case TC_ACT_TRAP:
consume_skb(skb);
fallthrough;
case TC_ACT_CONSUMED:
*ret = NET_XMIT_SUCCESS;
return NULL;
}
return skb;
}
#else
static __always_inline struct sk_buff *
sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
struct net_device *orig_dev, bool *another)
{
return skb;
}
static __always_inline struct sk_buff *
sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
{
return skb;
}
#endif /* CONFIG_NET_XGRESS */
#ifdef CONFIG_XPS
static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
struct xps_dev_maps *dev_maps, unsigned int tci)
{
int tc = netdev_get_prio_tc_map(dev, skb->priority);
struct xps_map *map;
int queue_index = -1;
if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
return queue_index;
tci *= dev_maps->num_tc;
tci += tc;
map = rcu_dereference(dev_maps->attr_map[tci]);
if (map) {
if (map->len == 1)
queue_index = map->queues[0];
else
queue_index = map->queues[reciprocal_scale(
skb_get_hash(skb), map->len)];
if (unlikely(queue_index >= dev->real_num_tx_queues))
queue_index = -1;
}
return queue_index;
}
#endif
static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
struct sk_buff *skb)
{
#ifdef CONFIG_XPS
struct xps_dev_maps *dev_maps;
struct sock *sk = skb->sk;
int queue_index = -1;
if (!static_key_false(&xps_needed))
return -1;
rcu_read_lock();
if (!static_key_false(&xps_rxqs_needed))
goto get_cpus_map;
dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
if (dev_maps) {
int tci = sk_rx_queue_get(sk);
if (tci >= 0)
queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
tci);
}
get_cpus_map:
if (queue_index < 0) {
dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
if (dev_maps) {
unsigned int tci = skb->sender_cpu - 1;
queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
tci);
}
}
rcu_read_unlock();
return queue_index;
#else
return -1;
#endif
}
u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
struct net_device *sb_dev)
{
return 0;
}
EXPORT_SYMBOL(dev_pick_tx_zero);
u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
struct net_device *sb_dev)
{
return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
}
EXPORT_SYMBOL(dev_pick_tx_cpu_id);
u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
struct net_device *sb_dev)
{
struct sock *sk = skb->sk;
int queue_index = sk_tx_queue_get(sk);
sb_dev = sb_dev ? : dev;
if (queue_index < 0 || skb->ooo_okay ||
queue_index >= dev->real_num_tx_queues) {
int new_index = get_xps_queue(dev, sb_dev, skb);
if (new_index < 0)
new_index = skb_tx_hash(dev, sb_dev, skb);
if (queue_index != new_index && sk &&
sk_fullsock(sk) &&
rcu_access_pointer(sk->sk_dst_cache))
sk_tx_queue_set(sk, new_index);
queue_index = new_index;
}
return queue_index;
}
EXPORT_SYMBOL(netdev_pick_tx);
struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
struct sk_buff *skb,
struct net_device *sb_dev)
{
int queue_index = 0;
#ifdef CONFIG_XPS
u32 sender_cpu = skb->sender_cpu - 1;
if (sender_cpu >= (u32)NR_CPUS)
skb->sender_cpu = raw_smp_processor_id() + 1;
#endif
if (dev->real_num_tx_queues != 1) {
const struct net_device_ops *ops = dev->netdev_ops;
if (ops->ndo_select_queue)
queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
else
queue_index = netdev_pick_tx(dev, skb, sb_dev);
queue_index = netdev_cap_txqueue(dev, queue_index);
}
skb_set_queue_mapping(skb, queue_index);
return netdev_get_tx_queue(dev, queue_index);
}
/**
* __dev_queue_xmit() - transmit a buffer
* @skb: buffer to transmit
* @sb_dev: suboordinate device used for L2 forwarding offload
*
* Queue a buffer for transmission to a network device. The caller must
* have set the device and priority and built the buffer before calling
* this function. The function can be called from an interrupt.
*
* When calling this method, interrupts MUST be enabled. This is because
* the BH enable code must have IRQs enabled so that it will not deadlock.
*
* Regardless of the return value, the skb is consumed, so it is currently
* difficult to retry a send to this method. (You can bump the ref count
* before sending to hold a reference for retry if you are careful.)
*
* Return:
* * 0 - buffer successfully transmitted
* * positive qdisc return code - NET_XMIT_DROP etc.
* * negative errno - other errors
*/
int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
{
struct net_device *dev = skb->dev;
struct netdev_queue *txq = NULL;
struct Qdisc *q;
int rc = -ENOMEM;
bool again = false;
skb_reset_mac_header(skb);
skb_assert_len(skb);
if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
/* Disable soft irqs for various locks below. Also
* stops preemption for RCU.
*/
rcu_read_lock_bh();
skb_update_prio(skb);
qdisc_pkt_len_init(skb);
tcx_set_ingress(skb, false);
#ifdef CONFIG_NET_EGRESS
if (static_branch_unlikely(&egress_needed_key)) {
if (nf_hook_egress_active()) {
skb = nf_hook_egress(skb, &rc, dev);
if (!skb)
goto out;
}
netdev_xmit_skip_txqueue(false);
nf_skip_egress(skb, true);
skb = sch_handle_egress(skb, &rc, dev);
if (!skb)
goto out;
nf_skip_egress(skb, false);
if (netdev_xmit_txqueue_skipped())
txq = netdev_tx_queue_mapping(dev, skb);
}
#endif
/* If device/qdisc don't need skb->dst, release it right now while
* its hot in this cpu cache.
*/
if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
skb_dst_drop(skb);
else
skb_dst_force(skb);
if (!txq)
txq = netdev_core_pick_tx(dev, skb, sb_dev);
q = rcu_dereference_bh(txq->qdisc);
trace_net_dev_queue(skb);
if (q->enqueue) {
rc = __dev_xmit_skb(skb, q, dev, txq);
goto out;
}
/* The device has no queue. Common case for software devices:
* loopback, all the sorts of tunnels...
* Really, it is unlikely that netif_tx_lock protection is necessary
* here. (f.e. loopback and IP tunnels are clean ignoring statistics
* counters.)
* However, it is possible, that they rely on protection
* made by us here.
* Check this and shot the lock. It is not prone from deadlocks.
*Either shot noqueue qdisc, it is even simpler 8)
*/
if (dev->flags & IFF_UP) {
int cpu = smp_processor_id(); /* ok because BHs are off */
/* Other cpus might concurrently change txq->xmit_lock_owner
* to -1 or to their cpu id, but not to our id.
*/
if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
if (dev_xmit_recursion())
goto recursion_alert;
skb = validate_xmit_skb(skb, dev, &again);
if (!skb)
goto out;
HARD_TX_LOCK(dev, txq, cpu);
if (!netif_xmit_stopped(txq)) {
dev_xmit_recursion_inc();
skb = dev_hard_start_xmit(skb, dev, txq, &rc);
dev_xmit_recursion_dec();
if (dev_xmit_complete(rc)) {
HARD_TX_UNLOCK(dev, txq);
goto out;
}
}
HARD_TX_UNLOCK(dev, txq);
net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
dev->name);
} else {
/* Recursion is detected! It is possible,
* unfortunately
*/
recursion_alert:
net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
dev->name);
}
}
rc = -ENETDOWN;
rcu_read_unlock_bh();
dev_core_stats_tx_dropped_inc(dev);
kfree_skb_list(skb);
return rc;
out:
rcu_read_unlock_bh();
return rc;
}
EXPORT_SYMBOL(__dev_queue_xmit);
int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
{
struct net_device *dev = skb->dev;
struct sk_buff *orig_skb = skb;
struct netdev_queue *txq;
int ret = NETDEV_TX_BUSY;
bool again = false;
if (unlikely(!netif_running(dev) ||
!netif_carrier_ok(dev)))
goto drop;
skb = validate_xmit_skb_list(skb, dev, &again);
if (skb != orig_skb)
goto drop;
skb_set_queue_mapping(skb, queue_id);
txq = skb_get_tx_queue(dev, skb);
local_bh_disable();
dev_xmit_recursion_inc();
HARD_TX_LOCK(dev, txq, smp_processor_id());
if (!netif_xmit_frozen_or_drv_stopped(txq))
ret = netdev_start_xmit(skb, dev, txq, false);
HARD_TX_UNLOCK(dev, txq);
dev_xmit_recursion_dec();
local_bh_enable();
return ret;
drop:
dev_core_stats_tx_dropped_inc(dev);
kfree_skb_list(skb);
return NET_XMIT_DROP;
}
EXPORT_SYMBOL(__dev_direct_xmit);
/*************************************************************************
* Receiver routines
*************************************************************************/
unsigned int sysctl_skb_defer_max __read_mostly = 64;
int weight_p __read_mostly = 64; /* old backlog weight */
int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
/* Called with irq disabled */
static inline void ____napi_schedule(struct softnet_data *sd,
struct napi_struct *napi)
{
struct task_struct *thread;
lockdep_assert_irqs_disabled(); if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
/* Paired with smp_mb__before_atomic() in
* napi_enable()/dev_set_threaded().
* Use READ_ONCE() to guarantee a complete
* read on napi->thread. Only call
* wake_up_process() when it's not NULL.
*/
thread = READ_ONCE(napi->thread);
if (thread) {
/* Avoid doing set_bit() if the thread is in
* INTERRUPTIBLE state, cause napi_thread_wait()
* makes sure to proceed with napi polling
* if the thread is explicitly woken from here.
*/
if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE) set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); wake_up_process(thread);
return;
}
}
list_add_tail(&napi->poll_list, &sd->poll_list); WRITE_ONCE(napi->list_owner, smp_processor_id());
/* If not called from net_rx_action()
* we have to raise NET_RX_SOFTIRQ.
*/
if (!sd->in_net_rx_action)
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
}
#ifdef CONFIG_RPS
struct static_key_false rps_needed __read_mostly;
EXPORT_SYMBOL(rps_needed);
struct static_key_false rfs_needed __read_mostly;
EXPORT_SYMBOL(rfs_needed);
static struct rps_dev_flow *
set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
struct rps_dev_flow *rflow, u16 next_cpu)
{
if (next_cpu < nr_cpu_ids) {
#ifdef CONFIG_RFS_ACCEL
struct netdev_rx_queue *rxqueue;
struct rps_dev_flow_table *flow_table;
struct rps_dev_flow *old_rflow;
u32 flow_id;
u16 rxq_index;
int rc;
/* Should we steer this flow to a different hardware queue? */
if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
!(dev->features & NETIF_F_NTUPLE))
goto out;
rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
if (rxq_index == skb_get_rx_queue(skb))
goto out;
rxqueue = dev->_rx + rxq_index;
flow_table = rcu_dereference(rxqueue->rps_flow_table);
if (!flow_table)
goto out;
flow_id = skb_get_hash(skb) & flow_table->mask;
rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
rxq_index, flow_id);
if (rc < 0)
goto out;
old_rflow = rflow;
rflow = &flow_table->flows[flow_id];
rflow->filter = rc;
if (old_rflow->filter == rflow->filter)
old_rflow->filter = RPS_NO_FILTER;
out:
#endif
rflow->last_qtail =
per_cpu(softnet_data, next_cpu).input_queue_head;
}
rflow->cpu = next_cpu;
return rflow;
}
/*
* get_rps_cpu is called from netif_receive_skb and returns the target
* CPU from the RPS map of the receiving queue for a given skb.
* rcu_read_lock must be held on entry.
*/
static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
struct rps_dev_flow **rflowp)
{
const struct rps_sock_flow_table *sock_flow_table;
struct netdev_rx_queue *rxqueue = dev->_rx;
struct rps_dev_flow_table *flow_table;
struct rps_map *map;
int cpu = -1;
u32 tcpu;
u32 hash;
if (skb_rx_queue_recorded(skb)) {
u16 index = skb_get_rx_queue(skb);
if (unlikely(index >= dev->real_num_rx_queues)) {
WARN_ONCE(dev->real_num_rx_queues > 1,
"%s received packet on queue %u, but number "
"of RX queues is %u\n",
dev->name, index, dev->real_num_rx_queues);
goto done;
}
rxqueue += index;
}
/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
flow_table = rcu_dereference(rxqueue->rps_flow_table);
map = rcu_dereference(rxqueue->rps_map);
if (!flow_table && !map)
goto done;
skb_reset_network_header(skb);
hash = skb_get_hash(skb);
if (!hash)
goto done;
sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
if (flow_table && sock_flow_table) {
struct rps_dev_flow *rflow;
u32 next_cpu;
u32 ident;
/* First check into global flow table if there is a match.
* This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
*/
ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
goto try_rps;
next_cpu = ident & net_hotdata.rps_cpu_mask;
/* OK, now we know there is a match,
* we can look at the local (per receive queue) flow table
*/
rflow = &flow_table->flows[hash & flow_table->mask];
tcpu = rflow->cpu;
/*
* If the desired CPU (where last recvmsg was done) is
* different from current CPU (one in the rx-queue flow
* table entry), switch if one of the following holds:
* - Current CPU is unset (>= nr_cpu_ids).
* - Current CPU is offline.
* - The current CPU's queue tail has advanced beyond the
* last packet that was enqueued using this table entry.
* This guarantees that all previous packets for the flow
* have been dequeued, thus preserving in order delivery.
*/
if (unlikely(tcpu != next_cpu) &&
(tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
((int)(per_cpu(softnet_data, tcpu).input_queue_head -
rflow->last_qtail)) >= 0)) {
tcpu = next_cpu;
rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
}
if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
*rflowp = rflow;
cpu = tcpu;
goto done;
}
}
try_rps:
if (map) {
tcpu = map->cpus[reciprocal_scale(hash, map->len)];
if (cpu_online(tcpu)) {
cpu = tcpu;
goto done;
}
}
done:
return cpu;
}
#ifdef CONFIG_RFS_ACCEL
/**
* rps_may_expire_flow - check whether an RFS hardware filter may be removed
* @dev: Device on which the filter was set
* @rxq_index: RX queue index
* @flow_id: Flow ID passed to ndo_rx_flow_steer()
* @filter_id: Filter ID returned by ndo_rx_flow_steer()
*
* Drivers that implement ndo_rx_flow_steer() should periodically call
* this function for each installed filter and remove the filters for
* which it returns %true.
*/
bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
u32 flow_id, u16 filter_id)
{
struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
struct rps_dev_flow_table *flow_table;
struct rps_dev_flow *rflow;
bool expire = true;
unsigned int cpu;
rcu_read_lock();
flow_table = rcu_dereference(rxqueue->rps_flow_table);
if (flow_table && flow_id <= flow_table->mask) {
rflow = &flow_table->flows[flow_id];
cpu = READ_ONCE(rflow->cpu);
if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
((int)(per_cpu(softnet_data, cpu).input_queue_head -
rflow->last_qtail) <
(int)(10 * flow_table->mask)))
expire = false;
}
rcu_read_unlock();
return expire;
}
EXPORT_SYMBOL(rps_may_expire_flow);
#endif /* CONFIG_RFS_ACCEL */
/* Called from hardirq (IPI) context */
static void rps_trigger_softirq(void *data)
{
struct softnet_data *sd = data;
____napi_schedule(sd, &sd->backlog);
sd->received_rps++;
}
#endif /* CONFIG_RPS */
/* Called from hardirq (IPI) context */
static void trigger_rx_softirq(void *data)
{
struct softnet_data *sd = data;
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
smp_store_release(&sd->defer_ipi_scheduled, 0);
}
/*
* After we queued a packet into sd->input_pkt_queue,
* we need to make sure this queue is serviced soon.
*
* - If this is another cpu queue, link it to our rps_ipi_list,
* and make sure we will process rps_ipi_list from net_rx_action().
*
* - If this is our own queue, NAPI schedule our backlog.
* Note that this also raises NET_RX_SOFTIRQ.
*/
static void napi_schedule_rps(struct softnet_data *sd)
{
struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
#ifdef CONFIG_RPS
if (sd != mysd) {
sd->rps_ipi_next = mysd->rps_ipi_list;
mysd->rps_ipi_list = sd;
/* If not called from net_rx_action() or napi_threaded_poll()
* we have to raise NET_RX_SOFTIRQ.
*/
if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
return;
}
#endif /* CONFIG_RPS */
__napi_schedule_irqoff(&mysd->backlog);
}
#ifdef CONFIG_NET_FLOW_LIMIT
int netdev_flow_limit_table_len __read_mostly = (1 << 12);
#endif
static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
{
#ifdef CONFIG_NET_FLOW_LIMIT
struct sd_flow_limit *fl;
struct softnet_data *sd;
unsigned int old_flow, new_flow;
if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1))
return false;
sd = this_cpu_ptr(&softnet_data);
rcu_read_lock();
fl = rcu_dereference(sd->flow_limit);
if (fl) {
new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
old_flow = fl->history[fl->history_head];
fl->history[fl->history_head] = new_flow;
fl->history_head++;
fl->history_head &= FLOW_LIMIT_HISTORY - 1;
if (likely(fl->buckets[old_flow]))
fl->buckets[old_flow]--;
if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
fl->count++;
rcu_read_unlock();
return true;
}
}
rcu_read_unlock();
#endif
return false;
}
/*
* enqueue_to_backlog is called to queue an skb to a per CPU backlog
* queue (may be a remote CPU queue).
*/
static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
unsigned int *qtail)
{
enum skb_drop_reason reason;
struct softnet_data *sd;
unsigned long flags;
unsigned int qlen;
reason = SKB_DROP_REASON_NOT_SPECIFIED;
sd = &per_cpu(softnet_data, cpu);
rps_lock_irqsave(sd, &flags);
if (!netif_running(skb->dev))
goto drop;
qlen = skb_queue_len(&sd->input_pkt_queue);
if (qlen <= READ_ONCE(net_hotdata.max_backlog) &&
!skb_flow_limit(skb, qlen)) {
if (qlen) {
enqueue:
__skb_queue_tail(&sd->input_pkt_queue, skb);
input_queue_tail_incr_save(sd, qtail);
rps_unlock_irq_restore(sd, &flags);
return NET_RX_SUCCESS;
}
/* Schedule NAPI for backlog device
* We can use non atomic operation since we own the queue lock
*/
if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
napi_schedule_rps(sd);
goto enqueue;
}
reason = SKB_DROP_REASON_CPU_BACKLOG;
drop:
sd->dropped++;
rps_unlock_irq_restore(sd, &flags);
dev_core_stats_rx_dropped_inc(skb->dev);
kfree_skb_reason(skb, reason);
return NET_RX_DROP;
}
static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
{
struct net_device *dev = skb->dev;
struct netdev_rx_queue *rxqueue;
rxqueue = dev->_rx;
if (skb_rx_queue_recorded(skb)) {
u16 index = skb_get_rx_queue(skb);
if (unlikely(index >= dev->real_num_rx_queues)) {
WARN_ONCE(dev->real_num_rx_queues > 1,
"%s received packet on queue %u, but number "
"of RX queues is %u\n",
dev->name, index, dev->real_num_rx_queues);
return rxqueue; /* Return first rxqueue */
}
rxqueue += index;
}
return rxqueue;
}
u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
struct bpf_prog *xdp_prog)
{
void *orig_data, *orig_data_end, *hard_start;
struct netdev_rx_queue *rxqueue;
bool orig_bcast, orig_host;
u32 mac_len, frame_sz;
__be16 orig_eth_type;
struct ethhdr *eth;
u32 metalen, act;
int off;
/* The XDP program wants to see the packet starting at the MAC
* header.
*/
mac_len = skb->data - skb_mac_header(skb);
hard_start = skb->data - skb_headroom(skb);
/* SKB "head" area always have tailroom for skb_shared_info */
frame_sz = (void *)skb_end_pointer(skb) - hard_start;
frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
rxqueue = netif_get_rxqueue(skb);
xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
skb_headlen(skb) + mac_len, true);
if (skb_is_nonlinear(skb)) {
skb_shinfo(skb)->xdp_frags_size = skb->data_len;
xdp_buff_set_frags_flag(xdp);
} else {
xdp_buff_clear_frags_flag(xdp);
}
orig_data_end = xdp->data_end;
orig_data = xdp->data;
eth = (struct ethhdr *)xdp->data;
orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
orig_eth_type = eth->h_proto;
act = bpf_prog_run_xdp(xdp_prog, xdp);
/* check if bpf_xdp_adjust_head was used */
off = xdp->data - orig_data;
if (off) {
if (off > 0)
__skb_pull(skb, off);
else if (off < 0)
__skb_push(skb, -off);
skb->mac_header += off;
skb_reset_network_header(skb);
}
/* check if bpf_xdp_adjust_tail was used */
off = xdp->data_end - orig_data_end;
if (off != 0) {
skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
skb->len += off; /* positive on grow, negative on shrink */
}
/* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
* (e.g. bpf_xdp_adjust_tail), we need to update data_len here.
*/
if (xdp_buff_has_frags(xdp))
skb->data_len = skb_shinfo(skb)->xdp_frags_size;
else
skb->data_len = 0;
/* check if XDP changed eth hdr such SKB needs update */
eth = (struct ethhdr *)xdp->data;
if ((orig_eth_type != eth->h_proto) ||
(orig_host != ether_addr_equal_64bits(eth->h_dest,
skb->dev->dev_addr)) ||
(orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
__skb_push(skb, ETH_HLEN);
skb->pkt_type = PACKET_HOST;
skb->protocol = eth_type_trans(skb, skb->dev);
}
/* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
* before calling us again on redirect path. We do not call do_redirect
* as we leave that up to the caller.
*
* Caller is responsible for managing lifetime of skb (i.e. calling
* kfree_skb in response to actions it cannot handle/XDP_DROP).
*/
switch (act) {
case XDP_REDIRECT:
case XDP_TX:
__skb_push(skb, mac_len);
break;
case XDP_PASS:
metalen = xdp->data - xdp->data_meta;
if (metalen)
skb_metadata_set(skb, metalen);
break;
}
return act;
}
static int
netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog)
{
struct sk_buff *skb = *pskb;
int err, hroom, troom;
if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog))
return 0;
/* In case we have to go down the path and also linearize,
* then lets do the pskb_expand_head() work just once here.
*/
hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
troom = skb->tail + skb->data_len - skb->end;
err = pskb_expand_head(skb,
hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
troom > 0 ? troom + 128 : 0, GFP_ATOMIC);
if (err)
return err;
return skb_linearize(skb);
}
static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
struct xdp_buff *xdp,
struct bpf_prog *xdp_prog)
{
struct sk_buff *skb = *pskb;
u32 mac_len, act = XDP_DROP;
/* Reinjected packets coming from act_mirred or similar should
* not get XDP generic processing.
*/
if (skb_is_redirected(skb))
return XDP_PASS;
/* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM
* bytes. This is the guarantee that also native XDP provides,
* thus we need to do it here as well.
*/
mac_len = skb->data - skb_mac_header(skb);
__skb_push(skb, mac_len);
if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
skb_headroom(skb) < XDP_PACKET_HEADROOM) {
if (netif_skb_check_for_xdp(pskb, xdp_prog))
goto do_drop;
}
__skb_pull(*pskb, mac_len);
act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog);
switch (act) {
case XDP_REDIRECT:
case XDP_TX:
case XDP_PASS:
break;
default:
bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception((*pskb)->dev, xdp_prog, act);
fallthrough;
case XDP_DROP:
do_drop:
kfree_skb(*pskb);
break;
}
return act;
}
/* When doing generic XDP we have to bypass the qdisc layer and the
* network taps in order to match in-driver-XDP behavior. This also means
* that XDP packets are able to starve other packets going through a qdisc,
* and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
* queues, so they do not have this starvation issue.
*/
void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
{
struct net_device *dev = skb->dev;
struct netdev_queue *txq;
bool free_skb = true;
int cpu, rc;
txq = netdev_core_pick_tx(dev, skb, NULL);
cpu = smp_processor_id();
HARD_TX_LOCK(dev, txq, cpu);
if (!netif_xmit_frozen_or_drv_stopped(txq)) {
rc = netdev_start_xmit(skb, dev, txq, 0);
if (dev_xmit_complete(rc))
free_skb = false;
}
HARD_TX_UNLOCK(dev, txq);
if (free_skb) {
trace_xdp_exception(dev, xdp_prog, XDP_TX);
dev_core_stats_tx_dropped_inc(dev);
kfree_skb(skb);
}
}
static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb)
{
if (xdp_prog) {
struct xdp_buff xdp;
u32 act;
int err;
act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog);
if (act != XDP_PASS) {
switch (act) {
case XDP_REDIRECT:
err = xdp_do_generic_redirect((*pskb)->dev, *pskb,
&xdp, xdp_prog);
if (err)
goto out_redir;
break;
case XDP_TX:
generic_xdp_tx(*pskb, xdp_prog);
break;
}
return XDP_DROP;
}
}
return XDP_PASS;
out_redir:
kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP);
return XDP_DROP;
}
EXPORT_SYMBOL_GPL(do_xdp_generic);
static int netif_rx_internal(struct sk_buff *skb)
{
int ret;
net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
trace_netif_rx(skb);
#ifdef CONFIG_RPS
if (static_branch_unlikely(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
if (cpu < 0)
cpu = smp_processor_id();
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
} else
#endif
{
unsigned int qtail;
ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
}
return ret;
}
/**
* __netif_rx - Slightly optimized version of netif_rx
* @skb: buffer to post
*
* This behaves as netif_rx except that it does not disable bottom halves.
* As a result this function may only be invoked from the interrupt context
* (either hard or soft interrupt).
*/
int __netif_rx(struct sk_buff *skb)
{
int ret;
lockdep_assert_once(hardirq_count() | softirq_count());
trace_netif_rx_entry(skb);
ret = netif_rx_internal(skb);
trace_netif_rx_exit(ret);
return ret;
}
EXPORT_SYMBOL(__netif_rx);
/**
* netif_rx - post buffer to the network code
* @skb: buffer to post
*
* This function receives a packet from a device driver and queues it for
* the upper (protocol) levels to process via the backlog NAPI device. It
* always succeeds. The buffer may be dropped during processing for
* congestion control or by the protocol layers.
* The network buffer is passed via the backlog NAPI device. Modern NIC
* driver should use NAPI and GRO.
* This function can used from interrupt and from process context. The
* caller from process context must not disable interrupts before invoking
* this function.
*
* return values:
* NET_RX_SUCCESS (no congestion)
* NET_RX_DROP (packet was dropped)
*
*/
int netif_rx(struct sk_buff *skb)
{
bool need_bh_off = !(hardirq_count() | softirq_count());
int ret;
if (need_bh_off)
local_bh_disable();
trace_netif_rx_entry(skb);
ret = netif_rx_internal(skb);
trace_netif_rx_exit(ret);
if (need_bh_off)
local_bh_enable();
return ret;
}
EXPORT_SYMBOL(netif_rx);
static __latent_entropy void net_tx_action(struct softirq_action *h)
{
struct softnet_data *sd = this_cpu_ptr(&softnet_data);
if (sd->completion_queue) {
struct sk_buff *clist;
local_irq_disable();
clist = sd->completion_queue;
sd->completion_queue = NULL;
local_irq_enable();
while (clist) {
struct sk_buff *skb = clist;
clist = clist->next;
WARN_ON(refcount_read(&skb->users));
if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
trace_consume_skb(skb, net_tx_action);
else
trace_kfree_skb(skb, net_tx_action,
get_kfree_skb_cb(skb)->reason);
if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
__kfree_skb(skb);
else
__napi_kfree_skb(skb,
get_kfree_skb_cb(skb)->reason);
}
}
if (sd->output_queue) {
struct Qdisc *head;
local_irq_disable();
head = sd->output_queue;
sd->output_queue = NULL;
sd->output_queue_tailp = &sd->output_queue;
local_irq_enable();
rcu_read_lock();
while (head) {
struct Qdisc *q = head;
spinlock_t *root_lock = NULL;
head = head->next_sched;
/* We need to make sure head->next_sched is read
* before clearing __QDISC_STATE_SCHED
*/
smp_mb__before_atomic();
if (!(q->flags & TCQ_F_NOLOCK)) {
root_lock = qdisc_lock(q);
spin_lock(root_lock);
} else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
&q->state))) {
/* There is a synchronize_net() between
* STATE_DEACTIVATED flag being set and
* qdisc_reset()/some_qdisc_is_busy() in
* dev_deactivate(), so we can safely bail out
* early here to avoid data race between
* qdisc_deactivate() and some_qdisc_is_busy()
* for lockless qdisc.
*/
clear_bit(__QDISC_STATE_SCHED, &q->state);
continue;
}
clear_bit(__QDISC_STATE_SCHED, &q->state);
qdisc_run(q);
if (root_lock)
spin_unlock(root_lock);
}
rcu_read_unlock();
}
xfrm_dev_backlog(sd);
}
#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
/* This hook is defined here for ATM LANE */
int (*br_fdb_test_addr_hook)(struct net_device *dev,
unsigned char *addr) __read_mostly;
EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
#endif
/**
* netdev_is_rx_handler_busy - check if receive handler is registered
* @dev: device to check
*
* Check if a receive handler is already registered for a given device.
* Return true if there one.
*
* The caller must hold the rtnl_mutex.
*/
bool netdev_is_rx_handler_busy(struct net_device *dev)
{
ASSERT_RTNL();
return dev && rtnl_dereference(dev->rx_handler);
}
EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
/**
* netdev_rx_handler_register - register receive handler
* @dev: device to register a handler for
* @rx_handler: receive handler to register
* @rx_handler_data: data pointer that is used by rx handler
*
* Register a receive handler for a device. This handler will then be
* called from __netif_receive_skb. A negative errno code is returned
* on a failure.
*
* The caller must hold the rtnl_mutex.
*
* For a general description of rx_handler, see enum rx_handler_result.
*/
int netdev_rx_handler_register(struct net_device *dev,
rx_handler_func_t *rx_handler,
void *rx_handler_data)
{
if (netdev_is_rx_handler_busy(dev))
return -EBUSY;
if (dev->priv_flags & IFF_NO_RX_HANDLER)
return -EINVAL;
/* Note: rx_handler_data must be set before rx_handler */
rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
rcu_assign_pointer(dev->rx_handler, rx_handler);
return 0;
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
/**
* netdev_rx_handler_unregister - unregister receive handler
* @dev: device to unregister a handler from
*
* Unregister a receive handler from a device.
*
* The caller must hold the rtnl_mutex.
*/
void netdev_rx_handler_unregister(struct net_device *dev)
{
ASSERT_RTNL();
RCU_INIT_POINTER(dev->rx_handler, NULL);
/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
* section has a guarantee to see a non NULL rx_handler_data
* as well.
*/
synchronize_net();
RCU_INIT_POINTER(dev->rx_handler_data, NULL);
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
/*
* Limit the use of PFMEMALLOC reserves to those protocols that implement
* the special handling of PFMEMALLOC skbs.
*/
static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
{
switch (skb->protocol) {
case htons(ETH_P_ARP):
case htons(ETH_P_IP):
case htons(ETH_P_IPV6):
case htons(ETH_P_8021Q):
case htons(ETH_P_8021AD):
return true;
default:
return false;
}
}
static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
int *ret, struct net_device *orig_dev)
{
if (nf_hook_ingress_active(skb)) {
int ingress_retval;
if (*pt_prev) {
*ret = deliver_skb(skb, *pt_prev, orig_dev);
*pt_prev = NULL;
}
rcu_read_lock();
ingress_retval = nf_hook_ingress(skb);
rcu_read_unlock();
return ingress_retval;
}
return 0;
}
static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
struct packet_type **ppt_prev)
{
struct packet_type *ptype, *pt_prev;
rx_handler_func_t *rx_handler;
struct sk_buff *skb = *pskb;
struct net_device *orig_dev;
bool deliver_exact = false;
int ret = NET_RX_DROP;
__be16 type;
net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
trace_netif_receive_skb(skb);
orig_dev = skb->dev;
skb_reset_network_header(skb);
if (!skb_transport_header_was_set(skb))
skb_reset_transport_header(skb);
skb_reset_mac_len(skb);
pt_prev = NULL;
another_round:
skb->skb_iif = skb->dev->ifindex;
__this_cpu_inc(softnet_data.processed);
if (static_branch_unlikely(&generic_xdp_needed_key)) {
int ret2;
migrate_disable();
ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog),
&skb);
migrate_enable();
if (ret2 != XDP_PASS) {
ret = NET_RX_DROP;
goto out;
}
}
if (eth_type_vlan(skb->protocol)) {
skb = skb_vlan_untag(skb);
if (unlikely(!skb))
goto out;
}
if (skb_skip_tc_classify(skb))
goto skip_classify;
if (pfmemalloc)
goto skip_taps;
list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
if (pt_prev)
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = ptype;
}
list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
if (pt_prev)
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = ptype;
}
skip_taps:
#ifdef CONFIG_NET_INGRESS
if (static_branch_unlikely(&ingress_needed_key)) {
bool another = false;
nf_skip_egress(skb, true);
skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
&another);
if (another)
goto another_round;
if (!skb)
goto out;
nf_skip_egress(skb, false);
if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
goto out;
}
#endif
skb_reset_redirect(skb);
skip_classify:
if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
goto drop;
if (skb_vlan_tag_present(skb)) {
if (pt_prev) {
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = NULL;
}
if (vlan_do_receive(&skb))
goto another_round;
else if (unlikely(!skb))
goto out;
}
rx_handler = rcu_dereference(skb->dev->rx_handler);
if (rx_handler) {
if (pt_prev) {
ret = deliver_skb(skb, pt_prev, orig_dev);
pt_prev = NULL;
}
switch (rx_handler(&skb)) {
case RX_HANDLER_CONSUMED:
ret = NET_RX_SUCCESS;
goto out;
case RX_HANDLER_ANOTHER:
goto another_round;
case RX_HANDLER_EXACT:
deliver_exact = true;
break;
case RX_HANDLER_PASS:
break;
default:
BUG();
}
}
if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
check_vlan_id:
if (skb_vlan_tag_get_id(skb)) {
/* Vlan id is non 0 and vlan_do_receive() above couldn't
* find vlan device.
*/
skb->pkt_type = PACKET_OTHERHOST;
} else if (eth_type_vlan(skb->protocol)) {
/* Outer header is 802.1P with vlan 0, inner header is
* 802.1Q or 802.1AD and vlan_do_receive() above could
* not find vlan dev for vlan id 0.
*/
__vlan_hwaccel_clear_tag(skb);
skb = skb_vlan_untag(skb);
if (unlikely(!skb))
goto out;
if (vlan_do_receive(&skb))
/* After stripping off 802.1P header with vlan 0
* vlan dev is found for inner header.
*/
goto another_round;
else if (unlikely(!skb))
goto out;
else
/* We have stripped outer 802.1P vlan 0 header.
* But could not find vlan dev.
* check again for vlan id to set OTHERHOST.
*/
goto check_vlan_id;
}
/* Note: we might in the future use prio bits
* and set skb->priority like in vlan_do_receive()
* For the time being, just ignore Priority Code Point
*/
__vlan_hwaccel_clear_tag(skb);
}
type = skb->protocol;
/* deliver only exact match when indicated */
if (likely(!deliver_exact)) {
deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
&ptype_base[ntohs(type) &
PTYPE_HASH_MASK]);
}
deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
&orig_dev->ptype_specific);
if (unlikely(skb->dev != orig_dev)) {
deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
&skb->dev->ptype_specific);
}
if (pt_prev) {
if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
goto drop;
*ppt_prev = pt_prev;
} else {
drop:
if (!deliver_exact)
dev_core_stats_rx_dropped_inc(skb->dev);
else
dev_core_stats_rx_nohandler_inc(skb->dev);
kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
/* Jamal, now you will not able to escape explaining
* me how you were going to use this. :-)
*/
ret = NET_RX_DROP;
}
out:
/* The invariant here is that if *ppt_prev is not NULL
* then skb should also be non-NULL.
*
* Apparently *ppt_prev assignment above holds this invariant due to
* skb dereferencing near it.
*/
*pskb = skb;
return ret;
}
static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
{
struct net_device *orig_dev = skb->dev;
struct packet_type *pt_prev = NULL;
int ret;
ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
if (pt_prev)
ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
skb->dev, pt_prev, orig_dev);
return ret;
}
/**
* netif_receive_skb_core - special purpose version of netif_receive_skb
* @skb: buffer to process
*
* More direct receive version of netif_receive_skb(). It should
* only be used by callers that have a need to skip RPS and Generic XDP.
* Caller must also take care of handling if ``(page_is_)pfmemalloc``.
*
* This function may only be called from softirq context and interrupts
* should be enabled.
*
* Return values (usually ignored):
* NET_RX_SUCCESS: no congestion
* NET_RX_DROP: packet was dropped
*/
int netif_receive_skb_core(struct sk_buff *skb)
{
int ret;
rcu_read_lock();
ret = __netif_receive_skb_one_core(skb, false);
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL(netif_receive_skb_core);
static inline void __netif_receive_skb_list_ptype(struct list_head *head,
struct packet_type *pt_prev,
struct net_device *orig_dev)
{
struct sk_buff *skb, *next;
if (!pt_prev)
return;
if (list_empty(head))
return;
if (pt_prev->list_func != NULL)
INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
ip_list_rcv, head, pt_prev, orig_dev);
else
list_for_each_entry_safe(skb, next, head, list) {
skb_list_del_init(skb);
pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
}
}
static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
{
/* Fast-path assumptions:
* - There is no RX handler.
* - Only one packet_type matches.
* If either of these fails, we will end up doing some per-packet
* processing in-line, then handling the 'last ptype' for the whole
* sublist. This can't cause out-of-order delivery to any single ptype,
* because the 'last ptype' must be constant across the sublist, and all
* other ptypes are handled per-packet.
*/
/* Current (common) ptype of sublist */
struct packet_type *pt_curr = NULL;
/* Current (common) orig_dev of sublist */
struct net_device *od_curr = NULL;
struct list_head sublist;
struct sk_buff *skb, *next;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
struct net_device *orig_dev = skb->dev;
struct packet_type *pt_prev = NULL;
skb_list_del_init(skb);
__netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
if (!pt_prev)
continue;
if (pt_curr != pt_prev || od_curr != orig_dev) {
/* dispatch old sublist */
__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
/* start new sublist */
INIT_LIST_HEAD(&sublist);
pt_curr = pt_prev;
od_curr = orig_dev;
}
list_add_tail(&skb->list, &sublist);
}
/* dispatch final sublist */
__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
}
static int __netif_receive_skb(struct sk_buff *skb)
{
int ret;
if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
unsigned int noreclaim_flag;
/*
* PFMEMALLOC skbs are special, they should
* - be delivered to SOCK_MEMALLOC sockets only
* - stay away from userspace
* - have bounded memory usage
*
* Use PF_MEMALLOC as this saves us from propagating the allocation
* context down to all allocation sites.
*/
noreclaim_flag = memalloc_noreclaim_save();
ret = __netif_receive_skb_one_core(skb, true);
memalloc_noreclaim_restore(noreclaim_flag);
} else
ret = __netif_receive_skb_one_core(skb, false);
return ret;
}
static void __netif_receive_skb_list(struct list_head *head)
{
unsigned long noreclaim_flag = 0;
struct sk_buff *skb, *next;
bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
list_for_each_entry_safe(skb, next, head, list) {
if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
struct list_head sublist;
/* Handle the previous sublist */
list_cut_before(&sublist, head, &skb->list);
if (!list_empty(&sublist))
__netif_receive_skb_list_core(&sublist, pfmemalloc);
pfmemalloc = !pfmemalloc;
/* See comments in __netif_receive_skb */
if (pfmemalloc)
noreclaim_flag = memalloc_noreclaim_save();
else
memalloc_noreclaim_restore(noreclaim_flag);
}
}
/* Handle the remaining sublist */
if (!list_empty(head))
__netif_receive_skb_list_core(head, pfmemalloc);
/* Restore pflags */
if (pfmemalloc)
memalloc_noreclaim_restore(noreclaim_flag);
}
static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
{
struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
struct bpf_prog *new = xdp->prog;
int ret = 0;
switch (xdp->command) {
case XDP_SETUP_PROG:
rcu_assign_pointer(dev->xdp_prog, new);
if (old)
bpf_prog_put(old);
if (old && !new) {
static_branch_dec(&generic_xdp_needed_key);
} else if (new && !old) {
static_branch_inc(&generic_xdp_needed_key);
dev_disable_lro(dev);
dev_disable_gro_hw(dev);
}
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
static int netif_receive_skb_internal(struct sk_buff *skb)
{
int ret;
net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
if (skb_defer_rx_timestamp(skb))
return NET_RX_SUCCESS;
rcu_read_lock();
#ifdef CONFIG_RPS
if (static_branch_unlikely(&rps_needed)) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu = get_rps_cpu(skb->dev, skb, &rflow);
if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
return ret;
}
}
#endif
ret = __netif_receive_skb(skb);
rcu_read_unlock();
return ret;
}
void netif_receive_skb_list_internal(struct list_head *head)
{
struct sk_buff *skb, *next;
struct list_head sublist;
INIT_LIST_HEAD(&sublist);
list_for_each_entry_safe(skb, next, head, list) {
net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
skb);
skb_list_del_init(skb);
if (!skb_defer_rx_timestamp(skb))
list_add_tail(&skb->list, &sublist);
}
list_splice_init(&sublist, head);
rcu_read_lock();
#ifdef CONFIG_RPS
if (static_branch_unlikely(&rps_needed)) {
list_for_each_entry_safe(skb, next, head, list) {
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu = get_rps_cpu(skb->dev, skb, &rflow);
if (cpu >= 0) {
/* Will be handled, remove from list */
skb_list_del_init(skb);
enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
}
}
}
#endif
__netif_receive_skb_list(head);
rcu_read_unlock();
}
/**
* netif_receive_skb - process receive buffer from network
* @skb: buffer to process
*
* netif_receive_skb() is the main receive data processing function.
* It always succeeds. The buffer may be dropped during processing
* for congestion control or by the protocol layers.
*
* This function may only be called from softirq context and interrupts
* should be enabled.
*
* Return values (usually ignored):
* NET_RX_SUCCESS: no congestion
* NET_RX_DROP: packet was dropped
*/
int netif_receive_skb(struct sk_buff *skb)
{
int ret;
trace_netif_receive_skb_entry(skb);
ret = netif_receive_skb_internal(skb);
trace_netif_receive_skb_exit(ret);
return ret;
}
EXPORT_SYMBOL(netif_receive_skb);
/**
* netif_receive_skb_list - process many receive buffers from network
* @head: list of skbs to process.
*
* Since return value of netif_receive_skb() is normally ignored, and
* wouldn't be meaningful for a list, this function returns void.
*
* This function may only be called from softirq context and interrupts
* should be enabled.
*/
void netif_receive_skb_list(struct list_head *head)
{
struct sk_buff *skb;
if (list_empty(head))
return;
if (trace_netif_receive_skb_list_entry_enabled()) {
list_for_each_entry(skb, head, list)
trace_netif_receive_skb_list_entry(skb);
}
netif_receive_skb_list_internal(head);
trace_netif_receive_skb_list_exit(0);
}
EXPORT_SYMBOL(netif_receive_skb_list);
static DEFINE_PER_CPU(struct work_struct, flush_works);
/* Network device is going away, flush any packets still pending */
static void flush_backlog(struct work_struct *work)
{
struct sk_buff *skb, *tmp;
struct softnet_data *sd;
local_bh_disable();
sd = this_cpu_ptr(&softnet_data);
rps_lock_irq_disable(sd);
skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
if (skb->dev->reg_state == NETREG_UNREGISTERING) {
__skb_unlink(skb, &sd->input_pkt_queue);
dev_kfree_skb_irq(skb);
input_queue_head_incr(sd);
}
}
rps_unlock_irq_enable(sd);
skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
if (skb->dev->reg_state == NETREG_UNREGISTERING) {
__skb_unlink(skb, &sd->process_queue);
kfree_skb(skb);
input_queue_head_incr(sd);
}
}
local_bh_enable();
}
static bool flush_required(int cpu)
{
#if IS_ENABLED(CONFIG_RPS)
struct softnet_data *sd = &per_cpu(softnet_data, cpu);
bool do_flush;
rps_lock_irq_disable(sd);
/* as insertion into process_queue happens with the rps lock held,
* process_queue access may race only with dequeue
*/
do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
!skb_queue_empty_lockless(&sd->process_queue); rps_unlock_irq_enable(sd);
return do_flush;
#endif
/* without RPS we can't safely check input_pkt_queue: during a
* concurrent remote skb_queue_splice() we can detect as empty both
* input_pkt_queue and process_queue even if the latter could end-up
* containing a lot of packets.
*/
return true;
}
static void flush_all_backlogs(void)
{
static cpumask_t flush_cpus;
unsigned int cpu;
/* since we are under rtnl lock protection we can use static data
* for the cpumask and avoid allocating on stack the possibly
* large mask
*/
ASSERT_RTNL(); cpus_read_lock();
cpumask_clear(&flush_cpus);
for_each_online_cpu(cpu) { if (flush_required(cpu)) {
queue_work_on(cpu, system_highpri_wq,
per_cpu_ptr(&flush_works, cpu));
cpumask_set_cpu(cpu, &flush_cpus);
}
}
/* we can have in flight packet[s] on the cpus we are not flushing,
* synchronize_net() in unregister_netdevice_many() will take care of
* them
*/
for_each_cpu(cpu, &flush_cpus) flush_work(per_cpu_ptr(&flush_works, cpu)); cpus_read_unlock();
}
static void net_rps_send_ipi(struct softnet_data *remsd)
{
#ifdef CONFIG_RPS
while (remsd) {
struct softnet_data *next = remsd->rps_ipi_next;
if (cpu_online(remsd->cpu))
smp_call_function_single_async(remsd->cpu, &remsd->csd);
remsd = next;
}
#endif
}
/*
* net_rps_action_and_irq_enable sends any pending IPI's for rps.
* Note: called with local irq disabled, but exits with local irq enabled.
*/
static void net_rps_action_and_irq_enable(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
struct softnet_data *remsd = sd->rps_ipi_list;
if (remsd) {
sd->rps_ipi_list = NULL;
local_irq_enable();
/* Send pending IPI's to kick RPS processing on remote cpus. */
net_rps_send_ipi(remsd);
} else
#endif
local_irq_enable();
}
static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
return sd->rps_ipi_list != NULL;
#else
return false;
#endif
}
static int process_backlog(struct napi_struct *napi, int quota)
{
struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
bool again = true;
int work = 0;
/* Check if we have pending ipi, its better to send them now,
* not waiting net_rx_action() end.
*/
if (sd_has_rps_ipi_waiting(sd)) {
local_irq_disable();
net_rps_action_and_irq_enable(sd);
}
napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
while (again) {
struct sk_buff *skb;
while ((skb = __skb_dequeue(&sd->process_queue))) {
rcu_read_lock();
__netif_receive_skb(skb);
rcu_read_unlock();
input_queue_head_incr(sd);
if (++work >= quota)
return work;
}
rps_lock_irq_disable(sd);
if (skb_queue_empty(&sd->input_pkt_queue)) {
/*
* Inline a custom version of __napi_complete().
* only current cpu owns and manipulates this napi,
* and NAPI_STATE_SCHED is the only possible flag set
* on backlog.
* We can use a plain write instead of clear_bit(),
* and we dont need an smp_mb() memory barrier.
*/
napi->state = 0;
again = false;
} else {
skb_queue_splice_tail_init(&sd->input_pkt_queue,
&sd->process_queue);
}
rps_unlock_irq_enable(sd);
}
return work;
}
/**
* __napi_schedule - schedule for receive
* @n: entry to schedule
*
* The entry's receive function will be scheduled to run.
* Consider using __napi_schedule_irqoff() if hard irqs are masked.
*/
void __napi_schedule(struct napi_struct *n)
{
unsigned long flags;
local_irq_save(flags); ____napi_schedule(this_cpu_ptr(&softnet_data), n); local_irq_restore(flags);
}
EXPORT_SYMBOL(__napi_schedule);
/**
* napi_schedule_prep - check if napi can be scheduled
* @n: napi context
*
* Test if NAPI routine is already running, and if not mark
* it as running. This is used as a condition variable to
* insure only one NAPI poll instance runs. We also make
* sure there is no pending NAPI disable.
*/
bool napi_schedule_prep(struct napi_struct *n)
{
unsigned long new, val = READ_ONCE(n->state);
do {
if (unlikely(val & NAPIF_STATE_DISABLE))
return false;
new = val | NAPIF_STATE_SCHED;
/* Sets STATE_MISSED bit if STATE_SCHED was already set
* This was suggested by Alexander Duyck, as compiler
* emits better code than :
* if (val & NAPIF_STATE_SCHED)
* new |= NAPIF_STATE_MISSED;
*/
new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
NAPIF_STATE_MISSED;
} while (!try_cmpxchg(&n->state, &val, new));
return !(val & NAPIF_STATE_SCHED);
}
EXPORT_SYMBOL(napi_schedule_prep);
/**
* __napi_schedule_irqoff - schedule for receive
* @n: entry to schedule
*
* Variant of __napi_schedule() assuming hard irqs are masked.
*
* On PREEMPT_RT enabled kernels this maps to __napi_schedule()
* because the interrupt disabled assumption might not be true
* due to force-threaded interrupts and spinlock substitution.
*/
void __napi_schedule_irqoff(struct napi_struct *n)
{
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
____napi_schedule(this_cpu_ptr(&softnet_data), n);
else
__napi_schedule(n);
}
EXPORT_SYMBOL(__napi_schedule_irqoff);
bool napi_complete_done(struct napi_struct *n, int work_done)
{
unsigned long flags, val, new, timeout = 0;
bool ret = true;
/*
* 1) Don't let napi dequeue from the cpu poll list
* just in case its running on a different cpu.
* 2) If we are busy polling, do nothing here, we have
* the guarantee we will be called later.
*/
if (unlikely(n->state & (NAPIF_STATE_NPSVC |
NAPIF_STATE_IN_BUSY_POLL)))
return false;
if (work_done) {
if (n->gro_bitmask)
timeout = READ_ONCE(n->dev->gro_flush_timeout);
n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
}
if (n->defer_hard_irqs_count > 0) {
n->defer_hard_irqs_count--;
timeout = READ_ONCE(n->dev->gro_flush_timeout);
if (timeout)
ret = false;
}
if (n->gro_bitmask) {
/* When the NAPI instance uses a timeout and keeps postponing
* it, we need to bound somehow the time packets are kept in
* the GRO layer
*/
napi_gro_flush(n, !!timeout);
}
gro_normal_list(n);
if (unlikely(!list_empty(&n->poll_list))) {
/* If n->poll_list is not empty, we need to mask irqs */
local_irq_save(flags);
list_del_init(&n->poll_list);
local_irq_restore(flags);
}
WRITE_ONCE(n->list_owner, -1);
val = READ_ONCE(n->state);
do {
WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
NAPIF_STATE_SCHED_THREADED |
NAPIF_STATE_PREFER_BUSY_POLL);
/* If STATE_MISSED was set, leave STATE_SCHED set,
* because we will call napi->poll() one more time.
* This C code was suggested by Alexander Duyck to help gcc.
*/
new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
NAPIF_STATE_SCHED;
} while (!try_cmpxchg(&n->state, &val, new));
if (unlikely(val & NAPIF_STATE_MISSED)) {
__napi_schedule(n);
return false;
}
if (timeout)
hrtimer_start(&n->timer, ns_to_ktime(timeout),
HRTIMER_MODE_REL_PINNED);
return ret;
}
EXPORT_SYMBOL(napi_complete_done);
/* must be called under rcu_read_lock(), as we dont take a reference */
struct napi_struct *napi_by_id(unsigned int napi_id)
{
unsigned int hash = napi_id % HASH_SIZE(napi_hash);
struct napi_struct *napi;
hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
if (napi->napi_id == napi_id)
return napi;
return NULL;
}
static void skb_defer_free_flush(struct softnet_data *sd)
{
struct sk_buff *skb, *next;
/* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
if (!READ_ONCE(sd->defer_list))
return;
spin_lock(&sd->defer_lock);
skb = sd->defer_list;
sd->defer_list = NULL;
sd->defer_count = 0;
spin_unlock(&sd->defer_lock);
while (skb != NULL) {
next = skb->next;
napi_consume_skb(skb, 1);
skb = next;
}
}
#if defined(CONFIG_NET_RX_BUSY_POLL)
static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
{
if (!skip_schedule) {
gro_normal_list(napi);
__napi_schedule(napi);
return;
}
if (napi->gro_bitmask) {
/* flush too old packets
* If HZ < 1000, flush all packets.
*/
napi_gro_flush(napi, HZ >= 1000);
}
gro_normal_list(napi);
clear_bit(NAPI_STATE_SCHED, &napi->state);
}
enum {
NAPI_F_PREFER_BUSY_POLL = 1,
NAPI_F_END_ON_RESCHED = 2,
};
static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
unsigned flags, u16 budget)
{
bool skip_schedule = false;
unsigned long timeout;
int rc;
/* Busy polling means there is a high chance device driver hard irq
* could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
* set in napi_schedule_prep().
* Since we are about to call napi->poll() once more, we can safely
* clear NAPI_STATE_MISSED.
*
* Note: x86 could use a single "lock and ..." instruction
* to perform these two clear_bit()
*/
clear_bit(NAPI_STATE_MISSED, &napi->state);
clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
local_bh_disable();
if (flags & NAPI_F_PREFER_BUSY_POLL) {
napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
timeout = READ_ONCE(napi->dev->gro_flush_timeout);
if (napi->defer_hard_irqs_count && timeout) {
hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
skip_schedule = true;
}
}
/* All we really want here is to re-enable device interrupts.
* Ideally, a new ndo_busy_poll_stop() could avoid another round.
*/
rc = napi->poll(napi, budget);
/* We can't gro_normal_list() here, because napi->poll() might have
* rearmed the napi (napi_complete_done()) in which case it could
* already be running on another CPU.
*/
trace_napi_poll(napi, rc, budget);
netpoll_poll_unlock(have_poll_lock);
if (rc == budget)
__busy_poll_stop(napi, skip_schedule);
local_bh_enable();
}
static void __napi_busy_loop(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, unsigned flags, u16 budget)
{
unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
int (*napi_poll)(struct napi_struct *napi, int budget);
void *have_poll_lock = NULL;
struct napi_struct *napi;
WARN_ON_ONCE(!rcu_read_lock_held());
restart:
napi_poll = NULL;
napi = napi_by_id(napi_id);
if (!napi)
return;
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_disable();
for (;;) {
int work = 0;
local_bh_disable();
if (!napi_poll) {
unsigned long val = READ_ONCE(napi->state);
/* If multiple threads are competing for this napi,
* we avoid dirtying napi->state as much as we can.
*/
if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
NAPIF_STATE_IN_BUSY_POLL)) {
if (flags & NAPI_F_PREFER_BUSY_POLL)
set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
goto count;
}
if (cmpxchg(&napi->state, val,
val | NAPIF_STATE_IN_BUSY_POLL |
NAPIF_STATE_SCHED) != val) {
if (flags & NAPI_F_PREFER_BUSY_POLL)
set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
goto count;
}
have_poll_lock = netpoll_poll_lock(napi);
napi_poll = napi->poll;
}
work = napi_poll(napi, budget);
trace_napi_poll(napi, work, budget);
gro_normal_list(napi);
count:
if (work > 0)
__NET_ADD_STATS(dev_net(napi->dev),
LINUX_MIB_BUSYPOLLRXPACKETS, work);
skb_defer_free_flush(this_cpu_ptr(&softnet_data));
local_bh_enable();
if (!loop_end || loop_end(loop_end_arg, start_time))
break;
if (unlikely(need_resched())) {
if (flags & NAPI_F_END_ON_RESCHED)
break;
if (napi_poll)
busy_poll_stop(napi, have_poll_lock, flags, budget);
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_enable();
rcu_read_unlock();
cond_resched();
rcu_read_lock();
if (loop_end(loop_end_arg, start_time))
return;
goto restart;
}
cpu_relax();
}
if (napi_poll)
busy_poll_stop(napi, have_poll_lock, flags, budget);
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_enable();
}
void napi_busy_loop_rcu(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget)
{
unsigned flags = NAPI_F_END_ON_RESCHED;
if (prefer_busy_poll)
flags |= NAPI_F_PREFER_BUSY_POLL;
__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
}
void napi_busy_loop(unsigned int napi_id,
bool (*loop_end)(void *, unsigned long),
void *loop_end_arg, bool prefer_busy_poll, u16 budget)
{
unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
rcu_read_lock();
__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
rcu_read_unlock();
}
EXPORT_SYMBOL(napi_busy_loop);
#endif /* CONFIG_NET_RX_BUSY_POLL */
static void napi_hash_add(struct napi_struct *napi)
{
if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
return;
spin_lock(&napi_hash_lock);
/* 0..NR_CPUS range is reserved for sender_cpu use */
do {
if (unlikely(++napi_gen_id < MIN_NAPI_ID))
napi_gen_id = MIN_NAPI_ID;
} while (napi_by_id(napi_gen_id));
napi->napi_id = napi_gen_id;
hlist_add_head_rcu(&napi->napi_hash_node,
&napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
spin_unlock(&napi_hash_lock);
}
/* Warning : caller is responsible to make sure rcu grace period
* is respected before freeing memory containing @napi
*/
static void napi_hash_del(struct napi_struct *napi)
{
spin_lock(&napi_hash_lock); hlist_del_init_rcu(&napi->napi_hash_node); spin_unlock(&napi_hash_lock);
}
static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
{
struct napi_struct *napi;
napi = container_of(timer, struct napi_struct, timer);
/* Note : we use a relaxed variant of napi_schedule_prep() not setting
* NAPI_STATE_MISSED, since we do not react to a device IRQ.
*/
if (!napi_disable_pending(napi) &&
!test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
__napi_schedule_irqoff(napi);
}
return HRTIMER_NORESTART;
}
static void init_gro_hash(struct napi_struct *napi)
{
int i;
for (i = 0; i < GRO_HASH_BUCKETS; i++) {
INIT_LIST_HEAD(&napi->gro_hash[i].list);
napi->gro_hash[i].count = 0;
}
napi->gro_bitmask = 0;
}
int dev_set_threaded(struct net_device *dev, bool threaded)
{
struct napi_struct *napi;
int err = 0;
if (dev->threaded == threaded)
return 0;
if (threaded) {
list_for_each_entry(napi, &dev->napi_list, dev_list) {
if (!napi->thread) {
err = napi_kthread_create(napi);
if (err) {
threaded = false;
break;
}
}
}
}
dev->threaded = threaded;
/* Make sure kthread is created before THREADED bit
* is set.
*/
smp_mb__before_atomic();
/* Setting/unsetting threaded mode on a napi might not immediately
* take effect, if the current napi instance is actively being
* polled. In this case, the switch between threaded mode and
* softirq mode will happen in the next round of napi_schedule().
* This should not cause hiccups/stalls to the live traffic.
*/
list_for_each_entry(napi, &dev->napi_list, dev_list)
assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
return err;
}
EXPORT_SYMBOL(dev_set_threaded);
/**
* netif_queue_set_napi - Associate queue with the napi
* @dev: device to which NAPI and queue belong
* @queue_index: Index of queue
* @type: queue type as RX or TX
* @napi: NAPI context, pass NULL to clear previously set NAPI
*
* Set queue with its corresponding napi context. This should be done after
* registering the NAPI handler for the queue-vector and the queues have been
* mapped to the corresponding interrupt vector.
*/
void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
enum netdev_queue_type type, struct napi_struct *napi)
{
struct netdev_rx_queue *rxq;
struct netdev_queue *txq;
if (WARN_ON_ONCE(napi && !napi->dev))
return;
if (dev->reg_state >= NETREG_REGISTERED)
ASSERT_RTNL();
switch (type) {
case NETDEV_QUEUE_TYPE_RX:
rxq = __netif_get_rx_queue(dev, queue_index);
rxq->napi = napi;
return;
case NETDEV_QUEUE_TYPE_TX:
txq = netdev_get_tx_queue(dev, queue_index);
txq->napi = napi;
return;
default:
return;
}
}
EXPORT_SYMBOL(netif_queue_set_napi);
void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
int (*poll)(struct napi_struct *, int), int weight)
{
if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
return;
INIT_LIST_HEAD(&napi->poll_list);
INIT_HLIST_NODE(&napi->napi_hash_node);
hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
napi->timer.function = napi_watchdog;
init_gro_hash(napi);
napi->skb = NULL;
INIT_LIST_HEAD(&napi->rx_list);
napi->rx_count = 0;
napi->poll = poll;
if (weight > NAPI_POLL_WEIGHT)
netdev_err_once(dev, "%s() called with weight %d\n", __func__,
weight);
napi->weight = weight;
napi->dev = dev;
#ifdef CONFIG_NETPOLL
napi->poll_owner = -1;
#endif
napi->list_owner = -1;
set_bit(NAPI_STATE_SCHED, &napi->state);
set_bit(NAPI_STATE_NPSVC, &napi->state);
list_add_rcu(&napi->dev_list, &dev->napi_list);
napi_hash_add(napi);
napi_get_frags_check(napi);
/* Create kthread for this napi if dev->threaded is set.
* Clear dev->threaded if kthread creation failed so that
* threaded mode will not be enabled in napi_enable().
*/
if (dev->threaded && napi_kthread_create(napi))
dev->threaded = 0;
netif_napi_set_irq(napi, -1);
}
EXPORT_SYMBOL(netif_napi_add_weight);
void napi_disable(struct napi_struct *n)
{
unsigned long val, new;
might_sleep();
set_bit(NAPI_STATE_DISABLE, &n->state);
val = READ_ONCE(n->state);
do {
while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
usleep_range(20, 200);
val = READ_ONCE(n->state);
}
new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
} while (!try_cmpxchg(&n->state, &val, new));
hrtimer_cancel(&n->timer);
clear_bit(NAPI_STATE_DISABLE, &n->state);
}
EXPORT_SYMBOL(napi_disable);
/**
* napi_enable - enable NAPI scheduling
* @n: NAPI context
*
* Resume NAPI from being scheduled on this context.
* Must be paired with napi_disable.
*/
void napi_enable(struct napi_struct *n)
{
unsigned long new, val = READ_ONCE(n->state);
do {
BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
if (n->dev->threaded && n->thread)
new |= NAPIF_STATE_THREADED;
} while (!try_cmpxchg(&n->state, &val, new));
}
EXPORT_SYMBOL(napi_enable);
static void flush_gro_hash(struct napi_struct *napi)
{
int i;
for (i = 0; i < GRO_HASH_BUCKETS; i++) {
struct sk_buff *skb, *n;
list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list) kfree_skb(skb); napi->gro_hash[i].count = 0;
}
}
/* Must be called in process context */
void __netif_napi_del(struct napi_struct *napi)
{
if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
return;
napi_hash_del(napi); list_del_rcu(&napi->dev_list);
napi_free_frags(napi);
flush_gro_hash(napi); napi->gro_bitmask = 0;
if (napi->thread) {
kthread_stop(napi->thread);
napi->thread = NULL;
}
}
EXPORT_SYMBOL(__netif_napi_del);
static int __napi_poll(struct napi_struct *n, bool *repoll)
{
int work, weight;
weight = n->weight;
/* This NAPI_STATE_SCHED test is for avoiding a race
* with netpoll's poll_napi(). Only the entity which
* obtains the lock and sees NAPI_STATE_SCHED set will
* actually make the ->poll() call. Therefore we avoid
* accidentally calling ->poll() when NAPI is not scheduled.
*/
work = 0;
if (napi_is_scheduled(n)) {
work = n->poll(n, weight);
trace_napi_poll(n, work, weight);
xdp_do_check_flushed(n);
}
if (unlikely(work > weight))
netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
n->poll, work, weight);
if (likely(work < weight))
return work;
/* Drivers must not modify the NAPI state if they
* consume the entire weight. In such cases this code
* still "owns" the NAPI instance and therefore can
* move the instance around on the list at-will.
*/
if (unlikely(napi_disable_pending(n))) {
napi_complete(n);
return work;
}
/* The NAPI context has more processing work, but busy-polling
* is preferred. Exit early.
*/
if (napi_prefer_busy_poll(n)) {
if (napi_complete_done(n, work)) {
/* If timeout is not set, we need to make sure
* that the NAPI is re-scheduled.
*/
napi_schedule(n);
}
return work;
}
if (n->gro_bitmask) {
/* flush too old packets
* If HZ < 1000, flush all packets.
*/
napi_gro_flush(n, HZ >= 1000);
}
gro_normal_list(n);
/* Some drivers may have called napi_schedule
* prior to exhausting their budget.
*/
if (unlikely(!list_empty(&n->poll_list))) {
pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
n->dev ? n->dev->name : "backlog");
return work;
}
*repoll = true;
return work;
}
static int napi_poll(struct napi_struct *n, struct list_head *repoll)
{
bool do_repoll = false;
void *have;
int work;
list_del_init(&n->poll_list);
have = netpoll_poll_lock(n);
work = __napi_poll(n, &do_repoll);
if (do_repoll)
list_add_tail(&n->poll_list, repoll);
netpoll_poll_unlock(have);
return work;
}
static int napi_thread_wait(struct napi_struct *napi)
{
bool woken = false;
set_current_state(TASK_INTERRUPTIBLE);
while (!kthread_should_stop()) {
/* Testing SCHED_THREADED bit here to make sure the current
* kthread owns this napi and could poll on this napi.
* Testing SCHED bit is not enough because SCHED bit might be
* set by some other busy poll thread or by napi_disable().
*/
if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
WARN_ON(!list_empty(&napi->poll_list));
__set_current_state(TASK_RUNNING);
return 0;
}
schedule();
/* woken being true indicates this thread owns this napi. */
woken = true;
set_current_state(TASK_INTERRUPTIBLE);
}
__set_current_state(TASK_RUNNING);
return -1;
}
static int napi_threaded_poll(void *data)
{
struct napi_struct *napi = data;
struct softnet_data *sd;
void *have;
while (!napi_thread_wait(napi)) {
unsigned long last_qs = jiffies;
for (;;) {
bool repoll = false;
local_bh_disable();
sd = this_cpu_ptr(&softnet_data);
sd->in_napi_threaded_poll = true;
have = netpoll_poll_lock(napi);
__napi_poll(napi, &repoll);
netpoll_poll_unlock(have);
sd->in_napi_threaded_poll = false;
barrier();
if (sd_has_rps_ipi_waiting(sd)) {
local_irq_disable();
net_rps_action_and_irq_enable(sd);
}
skb_defer_free_flush(sd);
local_bh_enable();
if (!repoll)
break;
rcu_softirq_qs_periodic(last_qs);
cond_resched();
}
}
return 0;
}
static __latent_entropy void net_rx_action(struct softirq_action *h)
{
struct softnet_data *sd = this_cpu_ptr(&softnet_data);
unsigned long time_limit = jiffies +
usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
int budget = READ_ONCE(net_hotdata.netdev_budget);
LIST_HEAD(list);
LIST_HEAD(repoll);
start:
sd->in_net_rx_action = true;
local_irq_disable();
list_splice_init(&sd->poll_list, &list);
local_irq_enable();
for (;;) {
struct napi_struct *n;
skb_defer_free_flush(sd);
if (list_empty(&list)) {
if (list_empty(&repoll)) {
sd->in_net_rx_action = false;
barrier();
/* We need to check if ____napi_schedule()
* had refilled poll_list while
* sd->in_net_rx_action was true.
*/
if (!list_empty(&sd->poll_list))
goto start;
if (!sd_has_rps_ipi_waiting(sd))
goto end;
}
break;
}
n = list_first_entry(&list, struct napi_struct, poll_list);
budget -= napi_poll(n, &repoll);
/* If softirq window is exhausted then punt.
* Allow this to run for 2 jiffies since which will allow
* an average latency of 1.5/HZ.
*/
if (unlikely(budget <= 0 ||
time_after_eq(jiffies, time_limit))) {
sd->time_squeeze++;
break;
}
}
local_irq_disable();
list_splice_tail_init(&sd->poll_list, &list);
list_splice_tail(&repoll, &list);
list_splice(&list, &sd->poll_list);
if (!list_empty(&sd->poll_list))
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
else
sd->in_net_rx_action = false;
net_rps_action_and_irq_enable(sd);
end:;
}
struct netdev_adjacent {
struct net_device *dev;
netdevice_tracker dev_tracker;
/* upper master flag, there can only be one master device per list */
bool master;
/* lookup ignore flag */
bool ignore;
/* counter for the number of times this device was added to us */
u16 ref_nr;
/* private field for the users */
void *private;
struct list_head list;
struct rcu_head rcu;
};
static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
struct list_head *adj_list)
{
struct netdev_adjacent *adj;
list_for_each_entry(adj, adj_list, list) {
if (adj->dev == adj_dev)
return adj;
}
return NULL;
}
static int ____netdev_has_upper_dev(struct net_device *upper_dev,
struct netdev_nested_priv *priv)
{
struct net_device *dev = (struct net_device *)priv->data;
return upper_dev == dev;
}
/**
* netdev_has_upper_dev - Check if device is linked to an upper device
* @dev: device
* @upper_dev: upper device to check
*
* Find out if a device is linked to specified upper device and return true
* in case it is. Note that this checks only immediate upper device,
* not through a complete stack of devices. The caller must hold the RTNL lock.
*/
bool netdev_has_upper_dev(struct net_device *dev,
struct net_device *upper_dev)
{
struct netdev_nested_priv priv = {
.data = (void *)upper_dev,
};
ASSERT_RTNL();
return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
&priv);
}
EXPORT_SYMBOL(netdev_has_upper_dev);
/**
* netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
* @dev: device
* @upper_dev: upper device to check
*
* Find out if a device is linked to specified upper device and return true
* in case it is. Note that this checks the entire upper device chain.
* The caller must hold rcu lock.
*/
bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
struct net_device *upper_dev)
{
struct netdev_nested_priv priv = {
.data = (void *)upper_dev,
};
return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
&priv);
}
EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
/**
* netdev_has_any_upper_dev - Check if device is linked to some device
* @dev: device
*
* Find out if a device is linked to an upper device and return true in case
* it is. The caller must hold the RTNL lock.
*/
bool netdev_has_any_upper_dev(struct net_device *dev)
{
ASSERT_RTNL(); return !list_empty(&dev->adj_list.upper);
}
EXPORT_SYMBOL(netdev_has_any_upper_dev);
/**
* netdev_master_upper_dev_get - Get master upper device
* @dev: device
*
* Find a master upper device and return pointer to it or NULL in case
* it's not there. The caller must hold the RTNL lock.
*/
struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
{
struct netdev_adjacent *upper;
ASSERT_RTNL();
if (list_empty(&dev->adj_list.upper))
return NULL;
upper = list_first_entry(&dev->adj_list.upper,
struct netdev_adjacent, list);
if (likely(upper->master))
return upper->dev;
return NULL;
}
EXPORT_SYMBOL(netdev_master_upper_dev_get);
static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
{
struct netdev_adjacent *upper;
ASSERT_RTNL();
if (list_empty(&dev->adj_list.upper))
return NULL;
upper = list_first_entry(&dev->adj_list.upper,
struct netdev_adjacent, list);
if (likely(upper->master) && !upper->ignore)
return upper->dev;
return NULL;
}
/**
* netdev_has_any_lower_dev - Check if device is linked to some device
* @dev: device
*
* Find out if a device is linked to a lower device and return true in case
* it is. The caller must hold the RTNL lock.
*/
static bool netdev_has_any_lower_dev(struct net_device *dev)
{
ASSERT_RTNL(); return !list_empty(&dev->adj_list.lower);
}
void *netdev_adjacent_get_private(struct list_head *adj_list)
{
struct netdev_adjacent *adj;
adj = list_entry(adj_list, struct netdev_adjacent, list);
return adj->private;
}
EXPORT_SYMBOL(netdev_adjacent_get_private);
/**
* netdev_upper_get_next_dev_rcu - Get the next dev from upper list
* @dev: device
* @iter: list_head ** of the current position
*
* Gets the next device from the dev's upper list, starting from iter
* position. The caller must hold RCU read lock.
*/
struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
struct list_head **iter)
{
struct netdev_adjacent *upper;
WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
if (&upper->list == &dev->adj_list.upper)
return NULL;
*iter = &upper->list;
return upper->dev;
}
EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
struct list_head **iter,
bool *ignore)
{
struct netdev_adjacent *upper;
upper = list_entry((*iter)->next, struct netdev_adjacent, list);
if (&upper->list == &dev->adj_list.upper)
return NULL;
*iter = &upper->list;
*ignore = upper->ignore;
return upper->dev;
}
static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
struct list_head **iter)
{
struct netdev_adjacent *upper;
WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
if (&upper->list == &dev->adj_list.upper)
return NULL;
*iter = &upper->list;
return upper->dev;
}
static int __netdev_walk_all_upper_dev(struct net_device *dev,
int (*fn)(struct net_device *dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv)
{
struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
int ret, cur = 0;
bool ignore;
now = dev;
iter = &dev->adj_list.upper;
while (1) {
if (now != dev) {
ret = fn(now, priv);
if (ret)
return ret;
}
next = NULL;
while (1) {
udev = __netdev_next_upper_dev(now, &iter, &ignore);
if (!udev)
break;
if (ignore)
continue;
next = udev;
niter = &udev->adj_list.upper;
dev_stack[cur] = now;
iter_stack[cur++] = iter;
break;
}
if (!next) {
if (!cur)
return 0;
next = dev_stack[--cur];
niter = iter_stack[cur];
}
now = next;
iter = niter;
}
return 0;
}
int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv)
{
struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
int ret, cur = 0;
now = dev;
iter = &dev->adj_list.upper;
while (1) {
if (now != dev) {
ret = fn(now, priv);
if (ret)
return ret;
}
next = NULL;
while (1) {
udev = netdev_next_upper_dev_rcu(now, &iter);
if (!udev)
break;
next = udev;
niter = &udev->adj_list.upper;
dev_stack[cur] = now;
iter_stack[cur++] = iter;
break;
}
if (!next) {
if (!cur)
return 0;
next = dev_stack[--cur];
niter = iter_stack[cur];
}
now = next;
iter = niter;
}
return 0;
}
EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
static bool __netdev_has_upper_dev(struct net_device *dev,
struct net_device *upper_dev)
{
struct netdev_nested_priv priv = {
.flags = 0,
.data = (void *)upper_dev,
};
ASSERT_RTNL();
return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
&priv);
}
/**
* netdev_lower_get_next_private - Get the next ->private from the
* lower neighbour list
* @dev: device
* @iter: list_head ** of the current position
*
* Gets the next netdev_adjacent->private from the dev's lower neighbour
* list, starting from iter position. The caller must hold either hold the
* RTNL lock or its own locking that guarantees that the neighbour lower
* list will remain unchanged.
*/
void *netdev_lower_get_next_private(struct net_device *dev,
struct list_head **iter)
{
struct netdev_adjacent *lower;
lower = list_entry(*iter, struct netdev_adjacent, list);
if (&lower->list == &dev->adj_list.lower)
return NULL;
*iter = lower->list.next;
return lower->private;
}
EXPORT_SYMBOL(netdev_lower_get_next_private);
/**
* netdev_lower_get_next_private_rcu - Get the next ->private from the
* lower neighbour list, RCU
* variant
* @dev: device
* @iter: list_head ** of the current position
*
* Gets the next netdev_adjacent->private from the dev's lower neighbour
* list, starting from iter position. The caller must hold RCU read lock.
*/
void *netdev_lower_get_next_private_rcu(struct net_device *dev,
struct list_head **iter)
{
struct netdev_adjacent *lower;
WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
if (&lower->list == &dev->adj_list.lower)
return NULL;
*iter = &lower->list;
return lower->private;
}
EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
/**
* netdev_lower_get_next - Get the next device from the lower neighbour
* list
* @dev: device
* @iter: list_head ** of the current position
*
* Gets the next netdev_adjacent from the dev's lower neighbour
* list, starting from iter position. The caller must hold RTNL lock or
* its own locking that guarantees that the neighbour lower
* list will remain unchanged.
*/
void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
{
struct netdev_adjacent *lower;
lower = list_entry(*iter, struct netdev_adjacent, list);
if (&lower->list == &dev->adj_list.lower)
return NULL;
*iter = lower->list.next;
return lower->dev;
}
EXPORT_SYMBOL(netdev_lower_get_next);
static struct net_device *netdev_next_lower_dev(struct net_device *dev,
struct list_head **iter)
{
struct netdev_adjacent *lower;
lower = list_entry((*iter)->next, struct netdev_adjacent, list);
if (&lower->list == &dev->adj_list.lower)
return NULL;
*iter = &lower->list;
return lower->dev;
}
static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
struct list_head **iter,
bool *ignore)
{
struct netdev_adjacent *lower;
lower = list_entry((*iter)->next, struct netdev_adjacent, list);
if (&lower->list == &dev->adj_list.lower)
return NULL;
*iter = &lower->list;
*ignore = lower->ignore;
return lower->dev;
}
int netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv)
{
struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
int ret, cur = 0;
now = dev;
iter = &dev->adj_list.lower;
while (1) {
if (now != dev) {
ret = fn(now, priv);
if (ret)
return ret;
}
next = NULL;
while (1) {
ldev = netdev_next_lower_dev(now, &iter);
if (!ldev)
break;
next = ldev;
niter = &ldev->adj_list.lower;
dev_stack[cur] = now;
iter_stack[cur++] = iter;
break;
}
if (!next) {
if (!cur)
return 0;
next = dev_stack[--cur];
niter = iter_stack[cur];
}
now = next;
iter = niter;
}
return 0;
}
EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
static int __netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv)
{
struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
int ret, cur = 0;
bool ignore;
now = dev;
iter = &dev->adj_list.lower;
while (1) {
if (now != dev) {
ret = fn(now, priv);
if (ret)
return ret;
}
next = NULL;
while (1) {
ldev = __netdev_next_lower_dev(now, &iter, &ignore);
if (!ldev)
break;
if (ignore)
continue;
next = ldev;
niter = &ldev->adj_list.lower;
dev_stack[cur] = now;
iter_stack[cur++] = iter;
break;
}
if (!next) {
if (!cur)
return 0;
next = dev_stack[--cur];
niter = iter_stack[cur];
}
now = next;
iter = niter;
}
return 0;
}
struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
struct list_head **iter)
{
struct netdev_adjacent *lower;
lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
if (&lower->list == &dev->adj_list.lower)
return NULL;
*iter = &lower->list;
return lower->dev;
}
EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
static u8 __netdev_upper_depth(struct net_device *dev)
{
struct net_device *udev;
struct list_head *iter;
u8 max_depth = 0;
bool ignore;
for (iter = &dev->adj_list.upper,
udev = __netdev_next_upper_dev(dev, &iter, &ignore);
udev;
udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
if (ignore)
continue;
if (max_depth < udev->upper_level)
max_depth = udev->upper_level;
}
return max_depth;
}
static u8 __netdev_lower_depth(struct net_device *dev)
{
struct net_device *ldev;
struct list_head *iter;
u8 max_depth = 0;
bool ignore;
for (iter = &dev->adj_list.lower,
ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
ldev;
ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
if (ignore)
continue;
if (max_depth < ldev->lower_level)
max_depth = ldev->lower_level;
}
return max_depth;
}
static int __netdev_update_upper_level(struct net_device *dev,
struct netdev_nested_priv *__unused)
{
dev->upper_level = __netdev_upper_depth(dev) + 1;
return 0;
}
#ifdef CONFIG_LOCKDEP
static LIST_HEAD(net_unlink_list);
static void net_unlink_todo(struct net_device *dev)
{
if (list_empty(&dev->unlink_list))
list_add_tail(&dev->unlink_list, &net_unlink_list);
}
#endif
static int __netdev_update_lower_level(struct net_device *dev,
struct netdev_nested_priv *priv)
{
dev->lower_level = __netdev_lower_depth(dev) + 1;
#ifdef CONFIG_LOCKDEP
if (!priv)
return 0;
if (priv->flags & NESTED_SYNC_IMM)
dev->nested_level = dev->lower_level - 1;
if (priv->flags & NESTED_SYNC_TODO)
net_unlink_todo(dev);
#endif
return 0;
}
int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv)
{
struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
int ret, cur = 0;
now = dev;
iter = &dev->adj_list.lower;
while (1) {
if (now != dev) {
ret = fn(now, priv);
if (ret)
return ret;
}
next = NULL;
while (1) {
ldev = netdev_next_lower_dev_rcu(now, &iter);
if (!ldev)
break;
next = ldev;
niter = &ldev->adj_list.lower;
dev_stack[cur] = now;
iter_stack[cur++] = iter;
break;
}
if (!next) {
if (!cur)
return 0;
next = dev_stack[--cur];
niter = iter_stack[cur];
}
now = next;
iter = niter;
}
return 0;
}
EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
/**
* netdev_lower_get_first_private_rcu - Get the first ->private from the
* lower neighbour list, RCU
* variant
* @dev: device
*
* Gets the first netdev_adjacent->private from the dev's lower neighbour
* list. The caller must hold RCU read lock.
*/
void *netdev_lower_get_first_private_rcu(struct net_device *dev)
{
struct netdev_adjacent *lower;
lower = list_first_or_null_rcu(&dev->adj_list.lower,
struct netdev_adjacent, list);
if (lower)
return lower->private;
return NULL;
}
EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
/**
* netdev_master_upper_dev_get_rcu - Get master upper device
* @dev: device
*
* Find a master upper device and return pointer to it or NULL in case
* it's not there. The caller must hold the RCU read lock.
*/
struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
{
struct netdev_adjacent *upper;
upper = list_first_or_null_rcu(&dev->adj_list.upper,
struct netdev_adjacent, list);
if (upper && likely(upper->master)) return upper->dev;
return NULL;
}
EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
static int netdev_adjacent_sysfs_add(struct net_device *dev,
struct net_device *adj_dev,
struct list_head *dev_list)
{
char linkname[IFNAMSIZ+7];
sprintf(linkname, dev_list == &dev->adj_list.upper ?
"upper_%s" : "lower_%s", adj_dev->name);
return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
linkname);
}
static void netdev_adjacent_sysfs_del(struct net_device *dev,
char *name,
struct list_head *dev_list)
{
char linkname[IFNAMSIZ+7];
sprintf(linkname, dev_list == &dev->adj_list.upper ?
"upper_%s" : "lower_%s", name);
sysfs_remove_link(&(dev->dev.kobj), linkname);
}
static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
struct net_device *adj_dev,
struct list_head *dev_list)
{
return (dev_list == &dev->adj_list.upper ||
dev_list == &dev->adj_list.lower) &&
net_eq(dev_net(dev), dev_net(adj_dev));
}
static int __netdev_adjacent_dev_insert(struct net_device *dev,
struct net_device *adj_dev,
struct list_head *dev_list,
void *private, bool master)
{
struct netdev_adjacent *adj;
int ret;
adj = __netdev_find_adj(adj_dev, dev_list);
if (adj) {
adj->ref_nr += 1;
pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
dev->name, adj_dev->name, adj->ref_nr);
return 0;
}
adj = kmalloc(sizeof(*adj), GFP_KERNEL);
if (!adj)
return -ENOMEM;
adj->dev = adj_dev;
adj->master = master;
adj->ref_nr = 1;
adj->private = private;
adj->ignore = false;
netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
if (ret)
goto free_adj;
}
/* Ensure that master link is always the first item in list. */
if (master) {
ret = sysfs_create_link(&(dev->dev.kobj),
&(adj_dev->dev.kobj), "master");
if (ret)
goto remove_symlinks;
list_add_rcu(&adj->list, dev_list);
} else {
list_add_tail_rcu(&adj->list, dev_list);
}
return 0;
remove_symlinks:
if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
free_adj:
netdev_put(adj_dev, &adj->dev_tracker);
kfree(adj);
return ret;
}
static void __netdev_adjacent_dev_remove(struct net_device *dev,
struct net_device *adj_dev,
u16 ref_nr,
struct list_head *dev_list)
{
struct netdev_adjacent *adj;
pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
dev->name, adj_dev->name, ref_nr);
adj = __netdev_find_adj(adj_dev, dev_list);
if (!adj) {
pr_err("Adjacency does not exist for device %s from %s\n",
dev->name, adj_dev->name);
WARN_ON(1);
return;
}
if (adj->ref_nr > ref_nr) {
pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
dev->name, adj_dev->name, ref_nr,
adj->ref_nr - ref_nr);
adj->ref_nr -= ref_nr;
return;
}
if (adj->master)
sysfs_remove_link(&(dev->dev.kobj), "master");
if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
list_del_rcu(&adj->list);
pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
adj_dev->name, dev->name, adj_dev->name);
netdev_put(adj_dev, &adj->dev_tracker);
kfree_rcu(adj, rcu);
}
static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
struct net_device *upper_dev,
struct list_head *up_list,
struct list_head *down_list,
void *private, bool master)
{
int ret;
ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
private, master);
if (ret)
return ret;
ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
private, false);
if (ret) {
__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
return ret;
}
return 0;
}
static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
struct net_device *upper_dev,
u16 ref_nr,
struct list_head *up_list,
struct list_head *down_list)
{
__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
}
static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
struct net_device *upper_dev,
void *private, bool master)
{
return __netdev_adjacent_dev_link_lists(dev, upper_dev,
&dev->adj_list.upper,
&upper_dev->adj_list.lower,
private, master);
}
static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
struct net_device *upper_dev)
{
__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
&dev->adj_list.upper,
&upper_dev->adj_list.lower);
}
static int __netdev_upper_dev_link(struct net_device *dev,
struct net_device *upper_dev, bool master,
void *upper_priv, void *upper_info,
struct netdev_nested_priv *priv,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_changeupper_info changeupper_info = {
.info = {
.dev = dev,
.extack = extack,
},
.upper_dev = upper_dev,
.master = master,
.linking = true,
.upper_info = upper_info,
};
struct net_device *master_dev;
int ret = 0;
ASSERT_RTNL();
if (dev == upper_dev)
return -EBUSY;
/* To prevent loops, check if dev is not upper device to upper_dev. */
if (__netdev_has_upper_dev(upper_dev, dev))
return -EBUSY;
if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
return -EMLINK;
if (!master) {
if (__netdev_has_upper_dev(dev, upper_dev))
return -EEXIST;
} else {
master_dev = __netdev_master_upper_dev_get(dev);
if (master_dev)
return master_dev == upper_dev ? -EEXIST : -EBUSY;
}
ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
&changeupper_info.info);
ret = notifier_to_errno(ret);
if (ret)
return ret;
ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
master);
if (ret)
return ret;
ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
&changeupper_info.info);
ret = notifier_to_errno(ret);
if (ret)
goto rollback;
__netdev_update_upper_level(dev, NULL);
__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
__netdev_update_lower_level(upper_dev, priv);
__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
priv);
return 0;
rollback:
__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
return ret;
}
/**
* netdev_upper_dev_link - Add a link to the upper device
* @dev: device
* @upper_dev: new upper device
* @extack: netlink extended ack
*
* Adds a link to device which is upper to this one. The caller must hold
* the RTNL lock. On a failure a negative errno code is returned.
* On success the reference counts are adjusted and the function
* returns zero.
*/
int netdev_upper_dev_link(struct net_device *dev,
struct net_device *upper_dev,
struct netlink_ext_ack *extack)
{
struct netdev_nested_priv priv = {
.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
.data = NULL,
};
return __netdev_upper_dev_link(dev, upper_dev, false,
NULL, NULL, &priv, extack);
}
EXPORT_SYMBOL(netdev_upper_dev_link);
/**
* netdev_master_upper_dev_link - Add a master link to the upper device
* @dev: device
* @upper_dev: new upper device
* @upper_priv: upper device private
* @upper_info: upper info to be passed down via notifier
* @extack: netlink extended ack
*
* Adds a link to device which is upper to this one. In this case, only
* one master upper device can be linked, although other non-master devices
* might be linked as well. The caller must hold the RTNL lock.
* On a failure a negative errno code is returned. On success the reference
* counts are adjusted and the function returns zero.
*/
int netdev_master_upper_dev_link(struct net_device *dev,
struct net_device *upper_dev,
void *upper_priv, void *upper_info,
struct netlink_ext_ack *extack)
{
struct netdev_nested_priv priv = {
.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
.data = NULL,
};
return __netdev_upper_dev_link(dev, upper_dev, true,
upper_priv, upper_info, &priv, extack);
}
EXPORT_SYMBOL(netdev_master_upper_dev_link);
static void __netdev_upper_dev_unlink(struct net_device *dev,
struct net_device *upper_dev,
struct netdev_nested_priv *priv)
{
struct netdev_notifier_changeupper_info changeupper_info = {
.info = {
.dev = dev,
},
.upper_dev = upper_dev,
.linking = false,
};
ASSERT_RTNL();
changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
&changeupper_info.info);
__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
&changeupper_info.info);
__netdev_update_upper_level(dev, NULL);
__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
__netdev_update_lower_level(upper_dev, priv);
__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
priv);
}
/**
* netdev_upper_dev_unlink - Removes a link to upper device
* @dev: device
* @upper_dev: new upper device
*
* Removes a link to device which is upper to this one. The caller must hold
* the RTNL lock.
*/
void netdev_upper_dev_unlink(struct net_device *dev,
struct net_device *upper_dev)
{
struct netdev_nested_priv priv = {
.flags = NESTED_SYNC_TODO,
.data = NULL,
};
__netdev_upper_dev_unlink(dev, upper_dev, &priv);
}
EXPORT_SYMBOL(netdev_upper_dev_unlink);
static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
struct net_device *lower_dev,
bool val)
{
struct netdev_adjacent *adj;
adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
if (adj)
adj->ignore = val;
adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
if (adj)
adj->ignore = val;
}
static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
struct net_device *lower_dev)
{
__netdev_adjacent_dev_set(upper_dev, lower_dev, true);
}
static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
struct net_device *lower_dev)
{
__netdev_adjacent_dev_set(upper_dev, lower_dev, false);
}
int netdev_adjacent_change_prepare(struct net_device *old_dev,
struct net_device *new_dev,
struct net_device *dev,
struct netlink_ext_ack *extack)
{
struct netdev_nested_priv priv = {
.flags = 0,
.data = NULL,
};
int err;
if (!new_dev)
return 0;
if (old_dev && new_dev != old_dev)
netdev_adjacent_dev_disable(dev, old_dev);
err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
extack);
if (err) {
if (old_dev && new_dev != old_dev)
netdev_adjacent_dev_enable(dev, old_dev);
return err;
}
return 0;
}
EXPORT_SYMBOL(netdev_adjacent_change_prepare);
void netdev_adjacent_change_commit(struct net_device *old_dev,
struct net_device *new_dev,
struct net_device *dev)
{
struct netdev_nested_priv priv = {
.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
.data = NULL,
};
if (!new_dev || !old_dev)
return;
if (new_dev == old_dev)
return;
netdev_adjacent_dev_enable(dev, old_dev);
__netdev_upper_dev_unlink(old_dev, dev, &priv);
}
EXPORT_SYMBOL(netdev_adjacent_change_commit);
void netdev_adjacent_change_abort(struct net_device *old_dev,
struct net_device *new_dev,
struct net_device *dev)
{
struct netdev_nested_priv priv = {
.flags = 0,
.data = NULL,
};
if (!new_dev)
return;
if (old_dev && new_dev != old_dev)
netdev_adjacent_dev_enable(dev, old_dev);
__netdev_upper_dev_unlink(new_dev, dev, &priv);
}
EXPORT_SYMBOL(netdev_adjacent_change_abort);
/**
* netdev_bonding_info_change - Dispatch event about slave change
* @dev: device
* @bonding_info: info to dispatch
*
* Send NETDEV_BONDING_INFO to netdev notifiers with info.
* The caller must hold the RTNL lock.
*/
void netdev_bonding_info_change(struct net_device *dev,
struct netdev_bonding_info *bonding_info)
{
struct netdev_notifier_bonding_info info = {
.info.dev = dev,
};
memcpy(&info.bonding_info, bonding_info,
sizeof(struct netdev_bonding_info));
call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
&info.info);
}
EXPORT_SYMBOL(netdev_bonding_info_change);
static int netdev_offload_xstats_enable_l3(struct net_device *dev,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_offload_xstats_info info = {
.info.dev = dev,
.info.extack = extack,
.type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
};
int err;
int rc;
dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
GFP_KERNEL);
if (!dev->offload_xstats_l3)
return -ENOMEM;
rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
NETDEV_OFFLOAD_XSTATS_DISABLE,
&info.info);
err = notifier_to_errno(rc);
if (err)
goto free_stats;
return 0;
free_stats:
kfree(dev->offload_xstats_l3);
dev->offload_xstats_l3 = NULL;
return err;
}
int netdev_offload_xstats_enable(struct net_device *dev,
enum netdev_offload_xstats_type type,
struct netlink_ext_ack *extack)
{
ASSERT_RTNL();
if (netdev_offload_xstats_enabled(dev, type))
return -EALREADY;
switch (type) {
case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
return netdev_offload_xstats_enable_l3(dev, extack);
}
WARN_ON(1);
return -EINVAL;
}
EXPORT_SYMBOL(netdev_offload_xstats_enable);
static void netdev_offload_xstats_disable_l3(struct net_device *dev)
{
struct netdev_notifier_offload_xstats_info info = {
.info.dev = dev,
.type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
};
call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
&info.info);
kfree(dev->offload_xstats_l3);
dev->offload_xstats_l3 = NULL;
}
int netdev_offload_xstats_disable(struct net_device *dev,
enum netdev_offload_xstats_type type)
{
ASSERT_RTNL(); if (!netdev_offload_xstats_enabled(dev, type))
return -EALREADY;
switch (type) {
case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
netdev_offload_xstats_disable_l3(dev); return 0;
}
WARN_ON(1);
return -EINVAL;
}
EXPORT_SYMBOL(netdev_offload_xstats_disable);
static void netdev_offload_xstats_disable_all(struct net_device *dev)
{
netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
}
static struct rtnl_hw_stats64 *
netdev_offload_xstats_get_ptr(const struct net_device *dev,
enum netdev_offload_xstats_type type)
{
switch (type) {
case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
return dev->offload_xstats_l3;
}
WARN_ON(1);
return NULL;
}
bool netdev_offload_xstats_enabled(const struct net_device *dev,
enum netdev_offload_xstats_type type)
{
ASSERT_RTNL(); return netdev_offload_xstats_get_ptr(dev, type);
}
EXPORT_SYMBOL(netdev_offload_xstats_enabled);
struct netdev_notifier_offload_xstats_ru {
bool used;
};
struct netdev_notifier_offload_xstats_rd {
struct rtnl_hw_stats64 stats;
bool used;
};
static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
const struct rtnl_hw_stats64 *src)
{
dest->rx_packets += src->rx_packets;
dest->tx_packets += src->tx_packets;
dest->rx_bytes += src->rx_bytes;
dest->tx_bytes += src->tx_bytes;
dest->rx_errors += src->rx_errors;
dest->tx_errors += src->tx_errors;
dest->rx_dropped += src->rx_dropped;
dest->tx_dropped += src->tx_dropped;
dest->multicast += src->multicast;
}
static int netdev_offload_xstats_get_used(struct net_device *dev,
enum netdev_offload_xstats_type type,
bool *p_used,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_offload_xstats_ru report_used = {};
struct netdev_notifier_offload_xstats_info info = {
.info.dev = dev,
.info.extack = extack,
.type = type,
.report_used = &report_used,
};
int rc;
WARN_ON(!netdev_offload_xstats_enabled(dev, type));
rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
&info.info);
*p_used = report_used.used;
return notifier_to_errno(rc);
}
static int netdev_offload_xstats_get_stats(struct net_device *dev,
enum netdev_offload_xstats_type type,
struct rtnl_hw_stats64 *p_stats,
bool *p_used,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_offload_xstats_rd report_delta = {};
struct netdev_notifier_offload_xstats_info info = {
.info.dev = dev,
.info.extack = extack,
.type = type,
.report_delta = &report_delta,
};
struct rtnl_hw_stats64 *stats;
int rc;
stats = netdev_offload_xstats_get_ptr(dev, type);
if (WARN_ON(!stats))
return -EINVAL;
rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
&info.info);
/* Cache whatever we got, even if there was an error, otherwise the
* successful stats retrievals would get lost.
*/
netdev_hw_stats64_add(stats, &report_delta.stats);
if (p_stats)
*p_stats = *stats;
*p_used = report_delta.used;
return notifier_to_errno(rc);
}
int netdev_offload_xstats_get(struct net_device *dev,
enum netdev_offload_xstats_type type,
struct rtnl_hw_stats64 *p_stats, bool *p_used,
struct netlink_ext_ack *extack)
{
ASSERT_RTNL();
if (p_stats)
return netdev_offload_xstats_get_stats(dev, type, p_stats,
p_used, extack);
else
return netdev_offload_xstats_get_used(dev, type, p_used,
extack);
}
EXPORT_SYMBOL(netdev_offload_xstats_get);
void
netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
const struct rtnl_hw_stats64 *stats)
{
report_delta->used = true;
netdev_hw_stats64_add(&report_delta->stats, stats);
}
EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
void
netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
{
report_used->used = true;
}
EXPORT_SYMBOL(netdev_offload_xstats_report_used);
void netdev_offload_xstats_push_delta(struct net_device *dev,
enum netdev_offload_xstats_type type,
const struct rtnl_hw_stats64 *p_stats)
{
struct rtnl_hw_stats64 *stats;
ASSERT_RTNL();
stats = netdev_offload_xstats_get_ptr(dev, type);
if (WARN_ON(!stats))
return;
netdev_hw_stats64_add(stats, p_stats);
}
EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
/**
* netdev_get_xmit_slave - Get the xmit slave of master device
* @dev: device
* @skb: The packet
* @all_slaves: assume all the slaves are active
*
* The reference counters are not incremented so the caller must be
* careful with locks. The caller must hold RCU lock.
* %NULL is returned if no slave is found.
*/
struct net_device *netdev_get_xmit_slave(struct net_device *dev,
struct sk_buff *skb,
bool all_slaves)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (!ops->ndo_get_xmit_slave)
return NULL;
return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
}
EXPORT_SYMBOL(netdev_get_xmit_slave);
static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
struct sock *sk)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (!ops->ndo_sk_get_lower_dev)
return NULL;
return ops->ndo_sk_get_lower_dev(dev, sk);
}
/**
* netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
* @dev: device
* @sk: the socket
*
* %NULL is returned if no lower device is found.
*/
struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
struct sock *sk)
{
struct net_device *lower;
lower = netdev_sk_get_lower_dev(dev, sk);
while (lower) {
dev = lower;
lower = netdev_sk_get_lower_dev(dev, sk);
}
return dev;
}
EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
static void netdev_adjacent_add_links(struct net_device *dev)
{
struct netdev_adjacent *iter;
struct net *net = dev_net(dev);
list_for_each_entry(iter, &dev->adj_list.upper, list) {
if (!net_eq(net, dev_net(iter->dev)))
continue;
netdev_adjacent_sysfs_add(iter->dev, dev,
&iter->dev->adj_list.lower);
netdev_adjacent_sysfs_add(dev, iter->dev,
&dev->adj_list.upper);
}
list_for_each_entry(iter, &dev->adj_list.lower, list) {
if (!net_eq(net, dev_net(iter->dev)))
continue;
netdev_adjacent_sysfs_add(iter->dev, dev,
&iter->dev->adj_list.upper);
netdev_adjacent_sysfs_add(dev, iter->dev,
&dev->adj_list.lower);
}
}
static void netdev_adjacent_del_links(struct net_device *dev)
{
struct netdev_adjacent *iter;
struct net *net = dev_net(dev);
list_for_each_entry(iter, &dev->adj_list.upper, list) {
if (!net_eq(net, dev_net(iter->dev)))
continue;
netdev_adjacent_sysfs_del(iter->dev, dev->name,
&iter->dev->adj_list.lower);
netdev_adjacent_sysfs_del(dev, iter->dev->name,
&dev->adj_list.upper);
}
list_for_each_entry(iter, &dev->adj_list.lower, list) {
if (!net_eq(net, dev_net(iter->dev)))
continue;
netdev_adjacent_sysfs_del(iter->dev, dev->name,
&iter->dev->adj_list.upper);
netdev_adjacent_sysfs_del(dev, iter->dev->name,
&dev->adj_list.lower);
}
}
void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
{
struct netdev_adjacent *iter;
struct net *net = dev_net(dev);
list_for_each_entry(iter, &dev->adj_list.upper, list) {
if (!net_eq(net, dev_net(iter->dev)))
continue;
netdev_adjacent_sysfs_del(iter->dev, oldname,
&iter->dev->adj_list.lower);
netdev_adjacent_sysfs_add(iter->dev, dev,
&iter->dev->adj_list.lower);
}
list_for_each_entry(iter, &dev->adj_list.lower, list) {
if (!net_eq(net, dev_net(iter->dev)))
continue;
netdev_adjacent_sysfs_del(iter->dev, oldname,
&iter->dev->adj_list.upper);
netdev_adjacent_sysfs_add(iter->dev, dev,
&iter->dev->adj_list.upper);
}
}
void *netdev_lower_dev_get_private(struct net_device *dev,
struct net_device *lower_dev)
{
struct netdev_adjacent *lower;
if (!lower_dev)
return NULL;
lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
if (!lower)
return NULL;
return lower->private;
}
EXPORT_SYMBOL(netdev_lower_dev_get_private);
/**
* netdev_lower_state_changed - Dispatch event about lower device state change
* @lower_dev: device
* @lower_state_info: state to dispatch
*
* Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
* The caller must hold the RTNL lock.
*/
void netdev_lower_state_changed(struct net_device *lower_dev,
void *lower_state_info)
{
struct netdev_notifier_changelowerstate_info changelowerstate_info = {
.info.dev = lower_dev,
};
ASSERT_RTNL();
changelowerstate_info.lower_state_info = lower_state_info;
call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
&changelowerstate_info.info);
}
EXPORT_SYMBOL(netdev_lower_state_changed);
static void dev_change_rx_flags(struct net_device *dev, int flags)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (ops->ndo_change_rx_flags)
ops->ndo_change_rx_flags(dev, flags);
}
static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
{
unsigned int old_flags = dev->flags;
kuid_t uid;
kgid_t gid;
ASSERT_RTNL();
dev->flags |= IFF_PROMISC;
dev->promiscuity += inc;
if (dev->promiscuity == 0) {
/*
* Avoid overflow.
* If inc causes overflow, untouch promisc and return error.
*/
if (inc < 0)
dev->flags &= ~IFF_PROMISC;
else {
dev->promiscuity -= inc;
netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
return -EOVERFLOW;
}
}
if (dev->flags != old_flags) {
netdev_info(dev, "%s promiscuous mode\n",
dev->flags & IFF_PROMISC ? "entered" : "left");
if (audit_enabled) {
current_uid_gid(&uid, &gid);
audit_log(audit_context(), GFP_ATOMIC,
AUDIT_ANOM_PROMISCUOUS,
"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
dev->name, (dev->flags & IFF_PROMISC),
(old_flags & IFF_PROMISC),
from_kuid(&init_user_ns, audit_get_loginuid(current)),
from_kuid(&init_user_ns, uid),
from_kgid(&init_user_ns, gid),
audit_get_sessionid(current));
}
dev_change_rx_flags(dev, IFF_PROMISC);
}
if (notify)
__dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
return 0;
}
/**
* dev_set_promiscuity - update promiscuity count on a device
* @dev: device
* @inc: modifier
*
* Add or remove promiscuity from a device. While the count in the device
* remains above zero the interface remains promiscuous. Once it hits zero
* the device reverts back to normal filtering operation. A negative inc
* value is used to drop promiscuity on the device.
* Return 0 if successful or a negative errno code on error.
*/
int dev_set_promiscuity(struct net_device *dev, int inc)
{
unsigned int old_flags = dev->flags;
int err;
err = __dev_set_promiscuity(dev, inc, true);
if (err < 0)
return err;
if (dev->flags != old_flags)
dev_set_rx_mode(dev);
return err;
}
EXPORT_SYMBOL(dev_set_promiscuity);
static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
{
unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
ASSERT_RTNL();
dev->flags |= IFF_ALLMULTI;
dev->allmulti += inc;
if (dev->allmulti == 0) {
/*
* Avoid overflow.
* If inc causes overflow, untouch allmulti and return error.
*/
if (inc < 0)
dev->flags &= ~IFF_ALLMULTI;
else {
dev->allmulti -= inc;
netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
return -EOVERFLOW;
}
}
if (dev->flags ^ old_flags) {
netdev_info(dev, "%s allmulticast mode\n",
dev->flags & IFF_ALLMULTI ? "entered" : "left");
dev_change_rx_flags(dev, IFF_ALLMULTI);
dev_set_rx_mode(dev);
if (notify)
__dev_notify_flags(dev, old_flags,
dev->gflags ^ old_gflags, 0, NULL);
}
return 0;
}
/**
* dev_set_allmulti - update allmulti count on a device
* @dev: device
* @inc: modifier
*
* Add or remove reception of all multicast frames to a device. While the
* count in the device remains above zero the interface remains listening
* to all interfaces. Once it hits zero the device reverts back to normal
* filtering operation. A negative @inc value is used to drop the counter
* when releasing a resource needing all multicasts.
* Return 0 if successful or a negative errno code on error.
*/
int dev_set_allmulti(struct net_device *dev, int inc)
{
return __dev_set_allmulti(dev, inc, true);
}
EXPORT_SYMBOL(dev_set_allmulti);
/*
* Upload unicast and multicast address lists to device and
* configure RX filtering. When the device doesn't support unicast
* filtering it is put in promiscuous mode while unicast addresses
* are present.
*/
void __dev_set_rx_mode(struct net_device *dev)
{
const struct net_device_ops *ops = dev->netdev_ops;
/* dev_open will call this function so the list will stay sane. */
if (!(dev->flags&IFF_UP))
return;
if (!netif_device_present(dev))
return;
if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
/* Unicast addresses changes may only happen under the rtnl,
* therefore calling __dev_set_promiscuity here is safe.
*/
if (!netdev_uc_empty(dev) && !dev->uc_promisc) { __dev_set_promiscuity(dev, 1, false);
dev->uc_promisc = true;
} else if (netdev_uc_empty(dev) && dev->uc_promisc) { __dev_set_promiscuity(dev, -1, false);
dev->uc_promisc = false;
}
}
if (ops->ndo_set_rx_mode) ops->ndo_set_rx_mode(dev);
}
void dev_set_rx_mode(struct net_device *dev)
{
netif_addr_lock_bh(dev);
__dev_set_rx_mode(dev);
netif_addr_unlock_bh(dev);
}
/**
* dev_get_flags - get flags reported to userspace
* @dev: device
*
* Get the combination of flag bits exported through APIs to userspace.
*/
unsigned int dev_get_flags(const struct net_device *dev)
{
unsigned int flags;
flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC |
IFF_ALLMULTI |
IFF_RUNNING |
IFF_LOWER_UP |
IFF_DORMANT)) |
(READ_ONCE(dev->gflags) & (IFF_PROMISC |
IFF_ALLMULTI));
if (netif_running(dev)) {
if (netif_oper_up(dev)) flags |= IFF_RUNNING; if (netif_carrier_ok(dev)) flags |= IFF_LOWER_UP; if (netif_dormant(dev)) flags |= IFF_DORMANT;
}
return flags;
}
EXPORT_SYMBOL(dev_get_flags);
int __dev_change_flags(struct net_device *dev, unsigned int flags,
struct netlink_ext_ack *extack)
{
unsigned int old_flags = dev->flags;
int ret;
ASSERT_RTNL();
/*
* Set the flags on our device.
*/
dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
IFF_AUTOMEDIA)) |
(dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
IFF_ALLMULTI));
/*
* Load in the correct multicast list now the flags have changed.
*/
if ((old_flags ^ flags) & IFF_MULTICAST)
dev_change_rx_flags(dev, IFF_MULTICAST);
dev_set_rx_mode(dev);
/*
* Have we downed the interface. We handle IFF_UP ourselves
* according to user attempts to set it, rather than blindly
* setting it.
*/
ret = 0;
if ((old_flags ^ flags) & IFF_UP) {
if (old_flags & IFF_UP)
__dev_close(dev);
else
ret = __dev_open(dev, extack);
}
if ((flags ^ dev->gflags) & IFF_PROMISC) {
int inc = (flags & IFF_PROMISC) ? 1 : -1;
unsigned int old_flags = dev->flags;
dev->gflags ^= IFF_PROMISC;
if (__dev_set_promiscuity(dev, inc, false) >= 0)
if (dev->flags != old_flags)
dev_set_rx_mode(dev);
}
/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
* is important. Some (broken) drivers set IFF_PROMISC, when
* IFF_ALLMULTI is requested not asking us and not reporting.
*/
if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
dev->gflags ^= IFF_ALLMULTI;
__dev_set_allmulti(dev, inc, false);
}
return ret;
}
void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
unsigned int gchanges, u32 portid,
const struct nlmsghdr *nlh)
{
unsigned int changes = dev->flags ^ old_flags;
if (gchanges)
rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
if (changes & IFF_UP) {
if (dev->flags & IFF_UP)
call_netdevice_notifiers(NETDEV_UP, dev);
else
call_netdevice_notifiers(NETDEV_DOWN, dev);
}
if (dev->flags & IFF_UP &&
(changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
struct netdev_notifier_change_info change_info = {
.info = {
.dev = dev,
},
.flags_changed = changes,
};
call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
}
}
/**
* dev_change_flags - change device settings
* @dev: device
* @flags: device state flags
* @extack: netlink extended ack
*
* Change settings on device based state flags. The flags are
* in the userspace exported format.
*/
int dev_change_flags(struct net_device *dev, unsigned int flags,
struct netlink_ext_ack *extack)
{
int ret;
unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
ret = __dev_change_flags(dev, flags, extack);
if (ret < 0)
return ret;
changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
__dev_notify_flags(dev, old_flags, changes, 0, NULL);
return ret;
}
EXPORT_SYMBOL(dev_change_flags);
int __dev_set_mtu(struct net_device *dev, int new_mtu)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (ops->ndo_change_mtu)
return ops->ndo_change_mtu(dev, new_mtu);
/* Pairs with all the lockless reads of dev->mtu in the stack */
WRITE_ONCE(dev->mtu, new_mtu);
return 0;
}
EXPORT_SYMBOL(__dev_set_mtu);
int dev_validate_mtu(struct net_device *dev, int new_mtu,
struct netlink_ext_ack *extack)
{
/* MTU must be positive, and in range */
if (new_mtu < 0 || new_mtu < dev->min_mtu) {
NL_SET_ERR_MSG(extack, "mtu less than device minimum");
return -EINVAL;
}
if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
return -EINVAL;
}
return 0;
}
/**
* dev_set_mtu_ext - Change maximum transfer unit
* @dev: device
* @new_mtu: new transfer unit
* @extack: netlink extended ack
*
* Change the maximum transfer size of the network device.
*/
int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
struct netlink_ext_ack *extack)
{
int err, orig_mtu;
if (new_mtu == dev->mtu)
return 0;
err = dev_validate_mtu(dev, new_mtu, extack);
if (err)
return err;
if (!netif_device_present(dev))
return -ENODEV;
err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
err = notifier_to_errno(err);
if (err)
return err;
orig_mtu = dev->mtu;
err = __dev_set_mtu(dev, new_mtu);
if (!err) {
err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
orig_mtu);
err = notifier_to_errno(err);
if (err) {
/* setting mtu back and notifying everyone again,
* so that they have a chance to revert changes.
*/
__dev_set_mtu(dev, orig_mtu);
call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
new_mtu);
}
}
return err;
}
int dev_set_mtu(struct net_device *dev, int new_mtu)
{
struct netlink_ext_ack extack;
int err;
memset(&extack, 0, sizeof(extack));
err = dev_set_mtu_ext(dev, new_mtu, &extack);
if (err && extack._msg)
net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
return err;
}
EXPORT_SYMBOL(dev_set_mtu);
/**
* dev_change_tx_queue_len - Change TX queue length of a netdevice
* @dev: device
* @new_len: new tx queue length
*/
int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
{
unsigned int orig_len = dev->tx_queue_len;
int res;
if (new_len != (unsigned int)new_len)
return -ERANGE;
if (new_len != orig_len) {
dev->tx_queue_len = new_len;
res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
res = notifier_to_errno(res);
if (res)
goto err_rollback;
res = dev_qdisc_change_tx_queue_len(dev);
if (res)
goto err_rollback;
}
return 0;
err_rollback:
netdev_err(dev, "refused to change device tx_queue_len\n");
dev->tx_queue_len = orig_len;
return res;
}
/**
* dev_set_group - Change group this device belongs to
* @dev: device
* @new_group: group this device should belong to
*/
void dev_set_group(struct net_device *dev, int new_group)
{
dev->group = new_group;
}
/**
* dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
* @dev: device
* @addr: new address
* @extack: netlink extended ack
*/
int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_pre_changeaddr_info info = {
.info.dev = dev,
.info.extack = extack,
.dev_addr = addr,
};
int rc;
rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
return notifier_to_errno(rc);
}
EXPORT_SYMBOL(dev_pre_changeaddr_notify);
/**
* dev_set_mac_address - Change Media Access Control Address
* @dev: device
* @sa: new address
* @extack: netlink extended ack
*
* Change the hardware (MAC) address of the device
*/
int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
struct netlink_ext_ack *extack)
{
const struct net_device_ops *ops = dev->netdev_ops;
int err;
if (!ops->ndo_set_mac_address)
return -EOPNOTSUPP;
if (sa->sa_family != dev->type)
return -EINVAL;
if (!netif_device_present(dev))
return -ENODEV;
err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
if (err)
return err;
if (memcmp(dev->dev_addr, sa->sa_data, dev->addr_len)) {
err = ops->ndo_set_mac_address(dev, sa);
if (err)
return err;
}
dev->addr_assign_type = NET_ADDR_SET;
call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
add_device_randomness(dev->dev_addr, dev->addr_len);
return 0;
}
EXPORT_SYMBOL(dev_set_mac_address);
DECLARE_RWSEM(dev_addr_sem);
int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
struct netlink_ext_ack *extack)
{
int ret;
down_write(&dev_addr_sem);
ret = dev_set_mac_address(dev, sa, extack);
up_write(&dev_addr_sem);
return ret;
}
EXPORT_SYMBOL(dev_set_mac_address_user);
int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
{
size_t size = sizeof(sa->sa_data_min);
struct net_device *dev;
int ret = 0;
down_read(&dev_addr_sem);
rcu_read_lock();
dev = dev_get_by_name_rcu(net, dev_name);
if (!dev) {
ret = -ENODEV;
goto unlock;
}
if (!dev->addr_len)
memset(sa->sa_data, 0, size);
else
memcpy(sa->sa_data, dev->dev_addr,
min_t(size_t, size, dev->addr_len));
sa->sa_family = dev->type;
unlock:
rcu_read_unlock();
up_read(&dev_addr_sem);
return ret;
}
EXPORT_SYMBOL(dev_get_mac_address);
/**
* dev_change_carrier - Change device carrier
* @dev: device
* @new_carrier: new value
*
* Change device carrier
*/
int dev_change_carrier(struct net_device *dev, bool new_carrier)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (!ops->ndo_change_carrier)
return -EOPNOTSUPP;
if (!netif_device_present(dev))
return -ENODEV;
return ops->ndo_change_carrier(dev, new_carrier);
}
/**
* dev_get_phys_port_id - Get device physical port ID
* @dev: device
* @ppid: port ID
*
* Get device physical port ID
*/
int dev_get_phys_port_id(struct net_device *dev,
struct netdev_phys_item_id *ppid)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (!ops->ndo_get_phys_port_id)
return -EOPNOTSUPP;
return ops->ndo_get_phys_port_id(dev, ppid);
}
/**
* dev_get_phys_port_name - Get device physical port name
* @dev: device
* @name: port name
* @len: limit of bytes to copy to name
*
* Get device physical port name
*/
int dev_get_phys_port_name(struct net_device *dev,
char *name, size_t len)
{
const struct net_device_ops *ops = dev->netdev_ops;
int err;
if (ops->ndo_get_phys_port_name) {
err = ops->ndo_get_phys_port_name(dev, name, len); if (err != -EOPNOTSUPP)
return err;
}
return devlink_compat_phys_port_name_get(dev, name, len);
}
/**
* dev_get_port_parent_id - Get the device's port parent identifier
* @dev: network device
* @ppid: pointer to a storage for the port's parent identifier
* @recurse: allow/disallow recursion to lower devices
*
* Get the devices's port parent identifier
*/
int dev_get_port_parent_id(struct net_device *dev,
struct netdev_phys_item_id *ppid,
bool recurse)
{
const struct net_device_ops *ops = dev->netdev_ops;
struct netdev_phys_item_id first = { };
struct net_device *lower_dev;
struct list_head *iter;
int err;
if (ops->ndo_get_port_parent_id) {
err = ops->ndo_get_port_parent_id(dev, ppid);
if (err != -EOPNOTSUPP)
return err;
}
err = devlink_compat_switch_id_get(dev, ppid);
if (!recurse || err != -EOPNOTSUPP)
return err;
netdev_for_each_lower_dev(dev, lower_dev, iter) { err = dev_get_port_parent_id(lower_dev, ppid, true);
if (err)
break;
if (!first.id_len) first = *ppid; else if (memcmp(&first, ppid, sizeof(*ppid)))
return -EOPNOTSUPP;
}
return err;
}
EXPORT_SYMBOL(dev_get_port_parent_id);
/**
* netdev_port_same_parent_id - Indicate if two network devices have
* the same port parent identifier
* @a: first network device
* @b: second network device
*/
bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
{
struct netdev_phys_item_id a_id = { };
struct netdev_phys_item_id b_id = { };
if (dev_get_port_parent_id(a, &a_id, true) ||
dev_get_port_parent_id(b, &b_id, true))
return false;
return netdev_phys_item_id_same(&a_id, &b_id);
}
EXPORT_SYMBOL(netdev_port_same_parent_id);
/**
* dev_change_proto_down - set carrier according to proto_down.
*
* @dev: device
* @proto_down: new value
*/
int dev_change_proto_down(struct net_device *dev, bool proto_down)
{
if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN))
return -EOPNOTSUPP;
if (!netif_device_present(dev))
return -ENODEV;
if (proto_down)
netif_carrier_off(dev);
else
netif_carrier_on(dev);
dev->proto_down = proto_down;
return 0;
}
/**
* dev_change_proto_down_reason - proto down reason
*
* @dev: device
* @mask: proto down mask
* @value: proto down value
*/
void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
u32 value)
{
int b;
if (!mask) {
dev->proto_down_reason = value;
} else {
for_each_set_bit(b, &mask, 32) {
if (value & (1 << b))
dev->proto_down_reason |= BIT(b);
else
dev->proto_down_reason &= ~BIT(b);
}
}
}
struct bpf_xdp_link {
struct bpf_link link;
struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
int flags;
};
static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
{
if (flags & XDP_FLAGS_HW_MODE)
return XDP_MODE_HW;
if (flags & XDP_FLAGS_DRV_MODE)
return XDP_MODE_DRV;
if (flags & XDP_FLAGS_SKB_MODE)
return XDP_MODE_SKB;
return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
}
static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
{
switch (mode) {
case XDP_MODE_SKB:
return generic_xdp_install;
case XDP_MODE_DRV:
case XDP_MODE_HW:
return dev->netdev_ops->ndo_bpf;
default:
return NULL;
}
}
static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
enum bpf_xdp_mode mode)
{
return dev->xdp_state[mode].link;
}
static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
enum bpf_xdp_mode mode)
{
struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
if (link)
return link->link.prog; return dev->xdp_state[mode].prog;
}
u8 dev_xdp_prog_count(struct net_device *dev)
{
u8 count = 0;
int i;
for (i = 0; i < __MAX_XDP_MODE; i++)
if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
count++;
return count;
}
EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
{
struct bpf_prog *prog = dev_xdp_prog(dev, mode); return prog ? prog->aux->id : 0;
}
static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
struct bpf_xdp_link *link)
{
dev->xdp_state[mode].link = link;
dev->xdp_state[mode].prog = NULL;
}
static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
struct bpf_prog *prog)
{
dev->xdp_state[mode].link = NULL;
dev->xdp_state[mode].prog = prog;
}
static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
bpf_op_t bpf_op, struct netlink_ext_ack *extack,
u32 flags, struct bpf_prog *prog)
{
struct netdev_bpf xdp;
int err;
memset(&xdp, 0, sizeof(xdp));
xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
xdp.extack = extack;
xdp.flags = flags;
xdp.prog = prog;
/* Drivers assume refcnt is already incremented (i.e, prog pointer is
* "moved" into driver), so they don't increment it on their own, but
* they do decrement refcnt when program is detached or replaced.
* Given net_device also owns link/prog, we need to bump refcnt here
* to prevent drivers from underflowing it.
*/
if (prog)
bpf_prog_inc(prog);
err = bpf_op(dev, &xdp);
if (err) {
if (prog)
bpf_prog_put(prog);
return err;
}
if (mode != XDP_MODE_HW)
bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
return 0;
}
static void dev_xdp_uninstall(struct net_device *dev)
{
struct bpf_xdp_link *link;
struct bpf_prog *prog;
enum bpf_xdp_mode mode;
bpf_op_t bpf_op;
ASSERT_RTNL(); for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) { prog = dev_xdp_prog(dev, mode); if (!prog)
continue;
bpf_op = dev_xdp_bpf_op(dev, mode);
if (!bpf_op)
continue;
WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
/* auto-detach link from net device */
link = dev_xdp_link(dev, mode);
if (link)
link->dev = NULL;
else
bpf_prog_put(prog); dev_xdp_set_link(dev, mode, NULL);
}
}
static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
struct bpf_xdp_link *link, struct bpf_prog *new_prog,
struct bpf_prog *old_prog, u32 flags)
{
unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
struct bpf_prog *cur_prog;
struct net_device *upper;
struct list_head *iter;
enum bpf_xdp_mode mode;
bpf_op_t bpf_op;
int err;
ASSERT_RTNL();
/* either link or prog attachment, never both */
if (link && (new_prog || old_prog))
return -EINVAL;
/* link supports only XDP mode flags */
if (link && (flags & ~XDP_FLAGS_MODES)) {
NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
return -EINVAL;
}
/* just one XDP mode bit should be set, zero defaults to drv/skb mode */
if (num_modes > 1) {
NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
return -EINVAL;
}
/* avoid ambiguity if offload + drv/skb mode progs are both loaded */
if (!num_modes && dev_xdp_prog_count(dev) > 1) {
NL_SET_ERR_MSG(extack,
"More than one program loaded, unset mode is ambiguous");
return -EINVAL;
}
/* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
return -EINVAL;
}
mode = dev_xdp_mode(dev, flags);
/* can't replace attached link */
if (dev_xdp_link(dev, mode)) {
NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
return -EBUSY;
}
/* don't allow if an upper device already has a program */
netdev_for_each_upper_dev_rcu(dev, upper, iter) {
if (dev_xdp_prog_count(upper) > 0) {
NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
return -EEXIST;
}
}
cur_prog = dev_xdp_prog(dev, mode);
/* can't replace attached prog with link */
if (link && cur_prog) {
NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
return -EBUSY;
}
if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
NL_SET_ERR_MSG(extack, "Active program does not match expected");
return -EEXIST;
}
/* put effective new program into new_prog */
if (link)
new_prog = link->link.prog;
if (new_prog) {
bool offload = mode == XDP_MODE_HW;
enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
? XDP_MODE_DRV : XDP_MODE_SKB;
if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
NL_SET_ERR_MSG(extack, "XDP program already attached");
return -EBUSY;
}
if (!offload && dev_xdp_prog(dev, other_mode)) {
NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
return -EEXIST;
}
if (!offload && bpf_prog_is_offloaded(new_prog->aux)) {
NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
return -EINVAL;
}
if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) {
NL_SET_ERR_MSG(extack, "Program bound to different device");
return -EINVAL;
}
if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
return -EINVAL;
}
if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
return -EINVAL;
}
}
/* don't call drivers if the effective program didn't change */
if (new_prog != cur_prog) {
bpf_op = dev_xdp_bpf_op(dev, mode);
if (!bpf_op) {
NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
return -EOPNOTSUPP;
}
err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
if (err)
return err;
}
if (link)
dev_xdp_set_link(dev, mode, link);
else
dev_xdp_set_prog(dev, mode, new_prog);
if (cur_prog)
bpf_prog_put(cur_prog);
return 0;
}
static int dev_xdp_attach_link(struct net_device *dev,
struct netlink_ext_ack *extack,
struct bpf_xdp_link *link)
{
return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
}
static int dev_xdp_detach_link(struct net_device *dev,
struct netlink_ext_ack *extack,
struct bpf_xdp_link *link)
{
enum bpf_xdp_mode mode;
bpf_op_t bpf_op;
ASSERT_RTNL();
mode = dev_xdp_mode(dev, link->flags);
if (dev_xdp_link(dev, mode) != link)
return -EINVAL;
bpf_op = dev_xdp_bpf_op(dev, mode);
WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
dev_xdp_set_link(dev, mode, NULL);
return 0;
}
static void bpf_xdp_link_release(struct bpf_link *link)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
rtnl_lock();
/* if racing with net_device's tear down, xdp_link->dev might be
* already NULL, in which case link was already auto-detached
*/
if (xdp_link->dev) {
WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
xdp_link->dev = NULL;
}
rtnl_unlock();
}
static int bpf_xdp_link_detach(struct bpf_link *link)
{
bpf_xdp_link_release(link);
return 0;
}
static void bpf_xdp_link_dealloc(struct bpf_link *link)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
kfree(xdp_link);
}
static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
struct seq_file *seq)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
u32 ifindex = 0;
rtnl_lock();
if (xdp_link->dev)
ifindex = xdp_link->dev->ifindex;
rtnl_unlock();
seq_printf(seq, "ifindex:\t%u\n", ifindex);
}
static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
struct bpf_link_info *info)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
u32 ifindex = 0;
rtnl_lock();
if (xdp_link->dev)
ifindex = xdp_link->dev->ifindex;
rtnl_unlock();
info->xdp.ifindex = ifindex;
return 0;
}
static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
struct bpf_prog *old_prog)
{
struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
enum bpf_xdp_mode mode;
bpf_op_t bpf_op;
int err = 0;
rtnl_lock();
/* link might have been auto-released already, so fail */
if (!xdp_link->dev) {
err = -ENOLINK;
goto out_unlock;
}
if (old_prog && link->prog != old_prog) {
err = -EPERM;
goto out_unlock;
}
old_prog = link->prog;
if (old_prog->type != new_prog->type ||
old_prog->expected_attach_type != new_prog->expected_attach_type) {
err = -EINVAL;
goto out_unlock;
}
if (old_prog == new_prog) {
/* no-op, don't disturb drivers */
bpf_prog_put(new_prog);
goto out_unlock;
}
mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
xdp_link->flags, new_prog);
if (err)
goto out_unlock;
old_prog = xchg(&link->prog, new_prog);
bpf_prog_put(old_prog);
out_unlock:
rtnl_unlock();
return err;
}
static const struct bpf_link_ops bpf_xdp_link_lops = {
.release = bpf_xdp_link_release,
.dealloc = bpf_xdp_link_dealloc,
.detach = bpf_xdp_link_detach,
.show_fdinfo = bpf_xdp_link_show_fdinfo,
.fill_link_info = bpf_xdp_link_fill_link_info,
.update_prog = bpf_xdp_link_update,
};
int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
{
struct net *net = current->nsproxy->net_ns;
struct bpf_link_primer link_primer;
struct netlink_ext_ack extack = {};
struct bpf_xdp_link *link;
struct net_device *dev;
int err, fd;
rtnl_lock();
dev = dev_get_by_index(net, attr->link_create.target_ifindex);
if (!dev) {
rtnl_unlock();
return -EINVAL;
}
link = kzalloc(sizeof(*link), GFP_USER);
if (!link) {
err = -ENOMEM;
goto unlock;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
link->dev = dev;
link->flags = attr->link_create.flags;
err = bpf_link_prime(&link->link, &link_primer);
if (err) {
kfree(link);
goto unlock;
}
err = dev_xdp_attach_link(dev, &extack, link);
rtnl_unlock();
if (err) {
link->dev = NULL;
bpf_link_cleanup(&link_primer);
trace_bpf_xdp_link_attach_failed(extack._msg);
goto out_put_dev;
}
fd = bpf_link_settle(&link_primer);
/* link itself doesn't hold dev's refcnt to not complicate shutdown */
dev_put(dev);
return fd;
unlock:
rtnl_unlock();
out_put_dev:
dev_put(dev);
return err;
}
/**
* dev_change_xdp_fd - set or clear a bpf program for a device rx path
* @dev: device
* @extack: netlink extended ack
* @fd: new program fd or negative value to clear
* @expected_fd: old program fd that userspace expects to replace or clear
* @flags: xdp-related flags
*
* Set or clear a bpf program for a device
*/
int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
int fd, int expected_fd, u32 flags)
{
enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
struct bpf_prog *new_prog = NULL, *old_prog = NULL;
int err;
ASSERT_RTNL();
if (fd >= 0) {
new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
mode != XDP_MODE_SKB);
if (IS_ERR(new_prog))
return PTR_ERR(new_prog);
}
if (expected_fd >= 0) {
old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
mode != XDP_MODE_SKB);
if (IS_ERR(old_prog)) {
err = PTR_ERR(old_prog);
old_prog = NULL;
goto err_out;
}
}
err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
err_out:
if (err && new_prog)
bpf_prog_put(new_prog);
if (old_prog)
bpf_prog_put(old_prog);
return err;
}
/**
* dev_index_reserve() - allocate an ifindex in a namespace
* @net: the applicable net namespace
* @ifindex: requested ifindex, pass %0 to get one allocated
*
* Allocate a ifindex for a new device. Caller must either use the ifindex
* to store the device (via list_netdevice()) or call dev_index_release()
* to give the index up.
*
* Return: a suitable unique value for a new device interface number or -errno.
*/
static int dev_index_reserve(struct net *net, u32 ifindex)
{
int err;
if (ifindex > INT_MAX) {
DEBUG_NET_WARN_ON_ONCE(1);
return -EINVAL;
}
if (!ifindex)
err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL,
xa_limit_31b, &net->ifindex, GFP_KERNEL);
else
err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL);
if (err < 0)
return err;
return ifindex;
}
static void dev_index_release(struct net *net, int ifindex)
{
/* Expect only unused indexes, unlist_netdevice() removes the used */
WARN_ON(xa_erase(&net->dev_by_index, ifindex));
}
/* Delayed registration/unregisteration */
LIST_HEAD(net_todo_list);
DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
atomic_t dev_unreg_count = ATOMIC_INIT(0);
static void net_set_todo(struct net_device *dev)
{
list_add_tail(&dev->todo_list, &net_todo_list);
}
static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
struct net_device *upper, netdev_features_t features)
{
netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
netdev_features_t feature;
int feature_bit;
for_each_netdev_feature(upper_disables, feature_bit) {
feature = __NETIF_F_BIT(feature_bit);
if (!(upper->wanted_features & feature)
&& (features & feature)) {
netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
&feature, upper->name);
features &= ~feature;
}
}
return features;
}
static void netdev_sync_lower_features(struct net_device *upper,
struct net_device *lower, netdev_features_t features)
{
netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
netdev_features_t feature;
int feature_bit;
for_each_netdev_feature(upper_disables, feature_bit) {
feature = __NETIF_F_BIT(feature_bit);
if (!(features & feature) && (lower->features & feature)) {
netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
&feature, lower->name);
lower->wanted_features &= ~feature;
__netdev_update_features(lower);
if (unlikely(lower->features & feature))
netdev_WARN(upper, "failed to disable %pNF on %s!\n",
&feature, lower->name);
else
netdev_features_change(lower);
}
}
}
static netdev_features_t netdev_fix_features(struct net_device *dev,
netdev_features_t features)
{
/* Fix illegal checksum combinations */
if ((features & NETIF_F_HW_CSUM) &&
(features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
netdev_warn(dev, "mixed HW and IP checksum settings.\n");
features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
}
/* TSO requires that SG is present as well. */
if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
features &= ~NETIF_F_ALL_TSO;
}
if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
!(features & NETIF_F_IP_CSUM)) {
netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
features &= ~NETIF_F_TSO;
features &= ~NETIF_F_TSO_ECN;
}
if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
!(features & NETIF_F_IPV6_CSUM)) {
netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
features &= ~NETIF_F_TSO6;
}
/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
features &= ~NETIF_F_TSO_MANGLEID;
/* TSO ECN requires that TSO is present as well. */
if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
features &= ~NETIF_F_TSO_ECN;
/* Software GSO depends on SG. */
if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
features &= ~NETIF_F_GSO;
}
/* GSO partial features require GSO partial be set */
if ((features & dev->gso_partial_features) &&
!(features & NETIF_F_GSO_PARTIAL)) {
netdev_dbg(dev,
"Dropping partially supported GSO features since no GSO partial.\n");
features &= ~dev->gso_partial_features;
}
if (!(features & NETIF_F_RXCSUM)) {
/* NETIF_F_GRO_HW implies doing RXCSUM since every packet
* successfully merged by hardware must also have the
* checksum verified by hardware. If the user does not
* want to enable RXCSUM, logically, we should disable GRO_HW.
*/
if (features & NETIF_F_GRO_HW) {
netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
features &= ~NETIF_F_GRO_HW;
}
}
/* LRO/HW-GRO features cannot be combined with RX-FCS */
if (features & NETIF_F_RXFCS) {
if (features & NETIF_F_LRO) {
netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
features &= ~NETIF_F_LRO;
}
if (features & NETIF_F_GRO_HW) {
netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
features &= ~NETIF_F_GRO_HW;
}
}
if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
features &= ~NETIF_F_LRO;
}
if (features & NETIF_F_HW_TLS_TX) {
bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
bool hw_csum = features & NETIF_F_HW_CSUM;
if (!ip_csum && !hw_csum) {
netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
features &= ~NETIF_F_HW_TLS_TX;
}
}
if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
features &= ~NETIF_F_HW_TLS_RX;
}
return features;
}
int __netdev_update_features(struct net_device *dev)
{
struct net_device *upper, *lower;
netdev_features_t features;
struct list_head *iter;
int err = -1;
ASSERT_RTNL();
features = netdev_get_wanted_features(dev);
if (dev->netdev_ops->ndo_fix_features)
features = dev->netdev_ops->ndo_fix_features(dev, features);
/* driver might be less strict about feature dependencies */
features = netdev_fix_features(dev, features);
/* some features can't be enabled if they're off on an upper device */
netdev_for_each_upper_dev_rcu(dev, upper, iter)
features = netdev_sync_upper_features(dev, upper, features);
if (dev->features == features)
goto sync_lower;
netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
&dev->features, &features);
if (dev->netdev_ops->ndo_set_features)
err = dev->netdev_ops->ndo_set_features(dev, features);
else
err = 0;
if (unlikely(err < 0)) {
netdev_err(dev,
"set_features() failed (%d); wanted %pNF, left %pNF\n",
err, &features, &dev->features);
/* return non-0 since some features might have changed and
* it's better to fire a spurious notification than miss it
*/
return -1;
}
sync_lower:
/* some features must be disabled on lower devices when disabled
* on an upper device (think: bonding master or bridge)
*/
netdev_for_each_lower_dev(dev, lower, iter)
netdev_sync_lower_features(dev, lower, features);
if (!err) {
netdev_features_t diff = features ^ dev->features;
if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
/* udp_tunnel_{get,drop}_rx_info both need
* NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
* device, or they won't do anything.
* Thus we need to update dev->features
* *before* calling udp_tunnel_get_rx_info,
* but *after* calling udp_tunnel_drop_rx_info.
*/
if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
dev->features = features;
udp_tunnel_get_rx_info(dev);
} else {
udp_tunnel_drop_rx_info(dev);
}
}
if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
dev->features = features;
err |= vlan_get_rx_ctag_filter_info(dev);
} else {
vlan_drop_rx_ctag_filter_info(dev);
}
}
if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
dev->features = features;
err |= vlan_get_rx_stag_filter_info(dev);
} else {
vlan_drop_rx_stag_filter_info(dev);
}
}
dev->features = features;
}
return err < 0 ? 0 : 1;
}
/**
* netdev_update_features - recalculate device features
* @dev: the device to check
*
* Recalculate dev->features set and send notifications if it
* has changed. Should be called after driver or hardware dependent
* conditions might have changed that influence the features.
*/
void netdev_update_features(struct net_device *dev)
{
if (__netdev_update_features(dev))
netdev_features_change(dev);
}
EXPORT_SYMBOL(netdev_update_features);
/**
* netdev_change_features - recalculate device features
* @dev: the device to check
*
* Recalculate dev->features set and send notifications even
* if they have not changed. Should be called instead of
* netdev_update_features() if also dev->vlan_features might
* have changed to allow the changes to be propagated to stacked
* VLAN devices.
*/
void netdev_change_features(struct net_device *dev)
{
__netdev_update_features(dev);
netdev_features_change(dev);
}
EXPORT_SYMBOL(netdev_change_features);
/**
* netif_stacked_transfer_operstate - transfer operstate
* @rootdev: the root or lower level device to transfer state from
* @dev: the device to transfer operstate to
*
* Transfer operational state from root to device. This is normally
* called when a stacking relationship exists between the root
* device and the device(a leaf device).
*/
void netif_stacked_transfer_operstate(const struct net_device *rootdev,
struct net_device *dev)
{
if (rootdev->operstate == IF_OPER_DORMANT)
netif_dormant_on(dev);
else
netif_dormant_off(dev);
if (rootdev->operstate == IF_OPER_TESTING)
netif_testing_on(dev);
else
netif_testing_off(dev);
if (netif_carrier_ok(rootdev))
netif_carrier_on(dev);
else
netif_carrier_off(dev);
}
EXPORT_SYMBOL(netif_stacked_transfer_operstate);
static int netif_alloc_rx_queues(struct net_device *dev)
{
unsigned int i, count = dev->num_rx_queues;
struct netdev_rx_queue *rx;
size_t sz = count * sizeof(*rx);
int err = 0;
BUG_ON(count < 1);
rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
if (!rx)
return -ENOMEM;
dev->_rx = rx;
for (i = 0; i < count; i++) {
rx[i].dev = dev;
/* XDP RX-queue setup */
err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
if (err < 0)
goto err_rxq_info;
}
return 0;
err_rxq_info:
/* Rollback successful reg's and free other resources */
while (i--)
xdp_rxq_info_unreg(&rx[i].xdp_rxq);
kvfree(dev->_rx);
dev->_rx = NULL;
return err;
}
static void netif_free_rx_queues(struct net_device *dev)
{
unsigned int i, count = dev->num_rx_queues;
/* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
if (!dev->_rx)
return;
for (i = 0; i < count; i++)
xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
kvfree(dev->_rx);
}
static void netdev_init_one_queue(struct net_device *dev,
struct netdev_queue *queue, void *_unused)
{
/* Initialize queue lock */
spin_lock_init(&queue->_xmit_lock);
netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
queue->xmit_lock_owner = -1;
netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
queue->dev = dev;
#ifdef CONFIG_BQL
dql_init(&queue->dql, HZ);
#endif
}
static void netif_free_tx_queues(struct net_device *dev)
{
kvfree(dev->_tx);
}
static int netif_alloc_netdev_queues(struct net_device *dev)
{
unsigned int count = dev->num_tx_queues;
struct netdev_queue *tx;
size_t sz = count * sizeof(*tx);
if (count < 1 || count > 0xffff)
return -EINVAL;
tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
if (!tx)
return -ENOMEM;
dev->_tx = tx;
netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
spin_lock_init(&dev->tx_global_lock);
return 0;
}
void netif_tx_stop_all_queues(struct net_device *dev)
{
unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
netif_tx_stop_queue(txq);
}
}
EXPORT_SYMBOL(netif_tx_stop_all_queues);
static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
{
void __percpu *v;
/* Drivers implementing ndo_get_peer_dev must support tstat
* accounting, so that skb_do_redirect() can bump the dev's
* RX stats upon network namespace switch.
*/
if (dev->netdev_ops->ndo_get_peer_dev &&
dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
return -EOPNOTSUPP;
switch (dev->pcpu_stat_type) {
case NETDEV_PCPU_STAT_NONE:
return 0;
case NETDEV_PCPU_STAT_LSTATS:
v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
break;
case NETDEV_PCPU_STAT_TSTATS:
v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
break;
case NETDEV_PCPU_STAT_DSTATS:
v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
break;
default:
return -EINVAL;
}
return v ? 0 : -ENOMEM;
}
static void netdev_do_free_pcpu_stats(struct net_device *dev)
{
switch (dev->pcpu_stat_type) {
case NETDEV_PCPU_STAT_NONE:
return;
case NETDEV_PCPU_STAT_LSTATS:
free_percpu(dev->lstats);
break;
case NETDEV_PCPU_STAT_TSTATS:
free_percpu(dev->tstats);
break;
case NETDEV_PCPU_STAT_DSTATS:
free_percpu(dev->dstats);
break;
}
}
/**
* register_netdevice() - register a network device
* @dev: device to register
*
* Take a prepared network device structure and make it externally accessible.
* A %NETDEV_REGISTER message is sent to the netdev notifier chain.
* Callers must hold the rtnl lock - you may want register_netdev()
* instead of this.
*/
int register_netdevice(struct net_device *dev)
{
int ret;
struct net *net = dev_net(dev);
BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
NETDEV_FEATURE_COUNT);
BUG_ON(dev_boot_phase);
ASSERT_RTNL();
might_sleep();
/* When net_device's are persistent, this will be fatal. */
BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
BUG_ON(!net);
ret = ethtool_check_ops(dev->ethtool_ops);
if (ret)
return ret;
spin_lock_init(&dev->addr_list_lock);
netdev_set_addr_lockdep_class(dev);
ret = dev_get_valid_name(net, dev, dev->name);
if (ret < 0)
goto out;
ret = -ENOMEM;
dev->name_node = netdev_name_node_head_alloc(dev);
if (!dev->name_node)
goto out;
/* Init, if this function is available */
if (dev->netdev_ops->ndo_init) {
ret = dev->netdev_ops->ndo_init(dev);
if (ret) {
if (ret > 0)
ret = -EIO;
goto err_free_name;
}
}
if (((dev->hw_features | dev->features) &
NETIF_F_HW_VLAN_CTAG_FILTER) &&
(!dev->netdev_ops->ndo_vlan_rx_add_vid ||
!dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
ret = -EINVAL;
goto err_uninit;
}
ret = netdev_do_alloc_pcpu_stats(dev);
if (ret)
goto err_uninit;
ret = dev_index_reserve(net, dev->ifindex);
if (ret < 0)
goto err_free_pcpu;
dev->ifindex = ret;
/* Transfer changeable features to wanted_features and enable
* software offloads (GSO and GRO).
*/
dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
dev->features |= NETIF_F_SOFT_FEATURES;
if (dev->udp_tunnel_nic_info) {
dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
}
dev->wanted_features = dev->features & dev->hw_features;
if (!(dev->flags & IFF_LOOPBACK))
dev->hw_features |= NETIF_F_NOCACHE_COPY;
/* If IPv4 TCP segmentation offload is supported we should also
* allow the device to enable segmenting the frame with the option
* of ignoring a static IP ID value. This doesn't enable the
* feature itself but allows the user to enable it later.
*/
if (dev->hw_features & NETIF_F_TSO)
dev->hw_features |= NETIF_F_TSO_MANGLEID;
if (dev->vlan_features & NETIF_F_TSO)
dev->vlan_features |= NETIF_F_TSO_MANGLEID;
if (dev->mpls_features & NETIF_F_TSO)
dev->mpls_features |= NETIF_F_TSO_MANGLEID;
if (dev->hw_enc_features & NETIF_F_TSO)
dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
*/
dev->vlan_features |= NETIF_F_HIGHDMA;
/* Make NETIF_F_SG inheritable to tunnel devices.
*/
dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
/* Make NETIF_F_SG inheritable to MPLS.
*/
dev->mpls_features |= NETIF_F_SG;
ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
ret = notifier_to_errno(ret);
if (ret)
goto err_ifindex_release;
ret = netdev_register_kobject(dev);
WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED);
if (ret)
goto err_uninit_notify;
__netdev_update_features(dev);
/*
* Default initial state at registry is that the
* device is present.
*/
set_bit(__LINK_STATE_PRESENT, &dev->state);
linkwatch_init_dev(dev);
dev_init_scheduler(dev);
netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
list_netdevice(dev);
add_device_randomness(dev->dev_addr, dev->addr_len);
/* If the device has permanent device address, driver should
* set dev_addr and also addr_assign_type should be set to
* NET_ADDR_PERM (default value).
*/
if (dev->addr_assign_type == NET_ADDR_PERM)
memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
/* Notify protocols, that a new device appeared. */
ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
ret = notifier_to_errno(ret);
if (ret) {
/* Expect explicit free_netdev() on failure */
dev->needs_free_netdev = false;
unregister_netdevice_queue(dev, NULL);
goto out;
}
/*
* Prevent userspace races by waiting until the network
* device is fully setup before sending notifications.
*/
if (!dev->rtnl_link_ops ||
dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
out:
return ret;
err_uninit_notify:
call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
err_ifindex_release:
dev_index_release(net, dev->ifindex);
err_free_pcpu:
netdev_do_free_pcpu_stats(dev);
err_uninit:
if (dev->netdev_ops->ndo_uninit)
dev->netdev_ops->ndo_uninit(dev);
if (dev->priv_destructor)
dev->priv_destructor(dev);
err_free_name:
netdev_name_node_free(dev->name_node);
goto out;
}
EXPORT_SYMBOL(register_netdevice);
/**
* init_dummy_netdev - init a dummy network device for NAPI
* @dev: device to init
*
* This takes a network device structure and initialize the minimum
* amount of fields so it can be used to schedule NAPI polls without
* registering a full blown interface. This is to be used by drivers
* that need to tie several hardware interfaces to a single NAPI
* poll scheduler due to HW limitations.
*/
void init_dummy_netdev(struct net_device *dev)
{
/* Clear everything. Note we don't initialize spinlocks
* are they aren't supposed to be taken by any of the
* NAPI code and this dummy netdev is supposed to be
* only ever used for NAPI polls
*/
memset(dev, 0, sizeof(struct net_device));
/* make sure we BUG if trying to hit standard
* register/unregister code path
*/
dev->reg_state = NETREG_DUMMY;
/* NAPI wants this */
INIT_LIST_HEAD(&dev->napi_list);
/* a dummy interface is started by default */
set_bit(__LINK_STATE_PRESENT, &dev->state);
set_bit(__LINK_STATE_START, &dev->state);
/* napi_busy_loop stats accounting wants this */
dev_net_set(dev, &init_net);
/* Note : We dont allocate pcpu_refcnt for dummy devices,
* because users of this 'device' dont need to change
* its refcount.
*/
}
EXPORT_SYMBOL_GPL(init_dummy_netdev);
/**
* register_netdev - register a network device
* @dev: device to register
*
* Take a completed network device structure and add it to the kernel
* interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
* chain. 0 is returned on success. A negative errno code is returned
* on a failure to set up the device, or if the name is a duplicate.
*
* This is a wrapper around register_netdevice that takes the rtnl semaphore
* and expands the device name if you passed a format string to
* alloc_netdev.
*/
int register_netdev(struct net_device *dev)
{
int err;
if (rtnl_lock_killable())
return -EINTR;
err = register_netdevice(dev);
rtnl_unlock();
return err;
}
EXPORT_SYMBOL(register_netdev);
int netdev_refcnt_read(const struct net_device *dev)
{
#ifdef CONFIG_PCPU_DEV_REFCNT
int i, refcnt = 0;
for_each_possible_cpu(i)
refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
return refcnt;
#else
return refcount_read(&dev->dev_refcnt);
#endif
}
EXPORT_SYMBOL(netdev_refcnt_read);
int netdev_unregister_timeout_secs __read_mostly = 10;
#define WAIT_REFS_MIN_MSECS 1
#define WAIT_REFS_MAX_MSECS 250
/**
* netdev_wait_allrefs_any - wait until all references are gone.
* @list: list of net_devices to wait on
*
* This is called when unregistering network devices.
*
* Any protocol or device that holds a reference should register
* for netdevice notification, and cleanup and put back the
* reference if they receive an UNREGISTER event.
* We can get stuck here if buggy protocols don't correctly
* call dev_put.
*/
static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
{
unsigned long rebroadcast_time, warning_time;
struct net_device *dev;
int wait = 0;
rebroadcast_time = warning_time = jiffies;
list_for_each_entry(dev, list, todo_list)
if (netdev_refcnt_read(dev) == 1)
return dev;
while (true) {
if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
rtnl_lock();
/* Rebroadcast unregister notification */
list_for_each_entry(dev, list, todo_list)
call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
__rtnl_unlock();
rcu_barrier();
rtnl_lock();
list_for_each_entry(dev, list, todo_list)
if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
&dev->state)) {
/* We must not have linkwatch events
* pending on unregister. If this
* happens, we simply run the queue
* unscheduled, resulting in a noop
* for this device.
*/
linkwatch_run_queue();
break;
}
__rtnl_unlock();
rebroadcast_time = jiffies;
}
if (!wait) {
rcu_barrier();
wait = WAIT_REFS_MIN_MSECS;
} else {
msleep(wait);
wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
}
list_for_each_entry(dev, list, todo_list)
if (netdev_refcnt_read(dev) == 1)
return dev;
if (time_after(jiffies, warning_time +
READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
list_for_each_entry(dev, list, todo_list) {
pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
dev->name, netdev_refcnt_read(dev));
ref_tracker_dir_print(&dev->refcnt_tracker, 10);
}
warning_time = jiffies;
}
}
}
/* The sequence is:
*
* rtnl_lock();
* ...
* register_netdevice(x1);
* register_netdevice(x2);
* ...
* unregister_netdevice(y1);
* unregister_netdevice(y2);
* ...
* rtnl_unlock();
* free_netdev(y1);
* free_netdev(y2);
*
* We are invoked by rtnl_unlock().
* This allows us to deal with problems:
* 1) We can delete sysfs objects which invoke hotplug
* without deadlocking with linkwatch via keventd.
* 2) Since we run with the RTNL semaphore not held, we can sleep
* safely in order to wait for the netdev refcnt to drop to zero.
*
* We must not return until all unregister events added during
* the interval the lock was held have been completed.
*/
void netdev_run_todo(void)
{
struct net_device *dev, *tmp;
struct list_head list;
int cnt;
#ifdef CONFIG_LOCKDEP
struct list_head unlink_list;
list_replace_init(&net_unlink_list, &unlink_list);
while (!list_empty(&unlink_list)) {
struct net_device *dev = list_first_entry(&unlink_list,
struct net_device,
unlink_list);
list_del_init(&dev->unlink_list);
dev->nested_level = dev->lower_level - 1;
}
#endif
/* Snapshot list, allow later requests */
list_replace_init(&net_todo_list, &list);
__rtnl_unlock();
/* Wait for rcu callbacks to finish before next phase */
if (!list_empty(&list))
rcu_barrier();
list_for_each_entry_safe(dev, tmp, &list, todo_list) {
if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
netdev_WARN(dev, "run_todo but not unregistering\n");
list_del(&dev->todo_list);
continue;
}
WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED);
linkwatch_sync_dev(dev);
}
cnt = 0;
while (!list_empty(&list)) {
dev = netdev_wait_allrefs_any(&list);
list_del(&dev->todo_list);
/* paranoia */
BUG_ON(netdev_refcnt_read(dev) != 1);
BUG_ON(!list_empty(&dev->ptype_all));
BUG_ON(!list_empty(&dev->ptype_specific));
WARN_ON(rcu_access_pointer(dev->ip_ptr));
WARN_ON(rcu_access_pointer(dev->ip6_ptr));
netdev_do_free_pcpu_stats(dev);
if (dev->priv_destructor)
dev->priv_destructor(dev);
if (dev->needs_free_netdev)
free_netdev(dev);
cnt++;
/* Free network device */
kobject_put(&dev->dev.kobj);
}
if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count))
wake_up(&netdev_unregistering_wq);
}
/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
* all the same fields in the same order as net_device_stats, with only
* the type differing, but rtnl_link_stats64 may have additional fields
* at the end for newer counters.
*/
void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
const struct net_device_stats *netdev_stats)
{
size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
const atomic_long_t *src = (atomic_long_t *)netdev_stats;
u64 *dst = (u64 *)stats64;
BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
for (i = 0; i < n; i++)
dst[i] = (unsigned long)atomic_long_read(&src[i]);
/* zero out counters that only exist in rtnl_link_stats64 */
memset((char *)stats64 + n * sizeof(u64), 0,
sizeof(*stats64) - n * sizeof(u64));
}
EXPORT_SYMBOL(netdev_stats_to_stats64);
static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc(
struct net_device *dev)
{
struct net_device_core_stats __percpu *p;
p = alloc_percpu_gfp(struct net_device_core_stats,
GFP_ATOMIC | __GFP_NOWARN);
if (p && cmpxchg(&dev->core_stats, NULL, p))
free_percpu(p);
/* This READ_ONCE() pairs with the cmpxchg() above */
return READ_ONCE(dev->core_stats);
}
noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
{
/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
unsigned long __percpu *field;
if (unlikely(!p)) {
p = netdev_core_stats_alloc(dev);
if (!p)
return;
}
field = (__force unsigned long __percpu *)((__force void *)p + offset);
this_cpu_inc(*field);
}
EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
/**
* dev_get_stats - get network device statistics
* @dev: device to get statistics from
* @storage: place to store stats
*
* Get network statistics from device. Return @storage.
* The device driver may provide its own method by setting
* dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
* otherwise the internal statistics structure is used.
*/
struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
struct rtnl_link_stats64 *storage)
{
const struct net_device_ops *ops = dev->netdev_ops;
const struct net_device_core_stats __percpu *p;
if (ops->ndo_get_stats64) {
memset(storage, 0, sizeof(*storage));
ops->ndo_get_stats64(dev, storage);
} else if (ops->ndo_get_stats) { netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { dev_get_tstats64(dev, storage);
} else {
netdev_stats_to_stats64(storage, &dev->stats);
}
/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
p = READ_ONCE(dev->core_stats);
if (p) {
const struct net_device_core_stats *core_stats;
int i;
for_each_possible_cpu(i) { core_stats = per_cpu_ptr(p, i);
storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
}
}
return storage;
}
EXPORT_SYMBOL(dev_get_stats);
/**
* dev_fetch_sw_netstats - get per-cpu network device statistics
* @s: place to store stats
* @netstats: per-cpu network stats to read from
*
* Read per-cpu network statistics and populate the related fields in @s.
*/
void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
const struct pcpu_sw_netstats __percpu *netstats)
{
int cpu;
for_each_possible_cpu(cpu) {
u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
const struct pcpu_sw_netstats *stats;
unsigned int start;
stats = per_cpu_ptr(netstats, cpu);
do {
start = u64_stats_fetch_begin(&stats->syncp);
rx_packets = u64_stats_read(&stats->rx_packets);
rx_bytes = u64_stats_read(&stats->rx_bytes);
tx_packets = u64_stats_read(&stats->tx_packets);
tx_bytes = u64_stats_read(&stats->tx_bytes);
} while (u64_stats_fetch_retry(&stats->syncp, start));
s->rx_packets += rx_packets;
s->rx_bytes += rx_bytes;
s->tx_packets += tx_packets;
s->tx_bytes += tx_bytes;
}
}
EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
/**
* dev_get_tstats64 - ndo_get_stats64 implementation
* @dev: device to get statistics from
* @s: place to store stats
*
* Populate @s from dev->stats and dev->tstats. Can be used as
* ndo_get_stats64() callback.
*/
void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
{
netdev_stats_to_stats64(s, &dev->stats);
dev_fetch_sw_netstats(s, dev->tstats);
}
EXPORT_SYMBOL_GPL(dev_get_tstats64);
struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
{
struct netdev_queue *queue = dev_ingress_queue(dev);
#ifdef CONFIG_NET_CLS_ACT
if (queue)
return queue;
queue = kzalloc(sizeof(*queue), GFP_KERNEL);
if (!queue)
return NULL;
netdev_init_one_queue(dev, queue, NULL);
RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
rcu_assign_pointer(dev->ingress_queue, queue);
#endif
return queue;
}
static const struct ethtool_ops default_ethtool_ops;
void netdev_set_default_ethtool_ops(struct net_device *dev,
const struct ethtool_ops *ops)
{
if (dev->ethtool_ops == &default_ethtool_ops)
dev->ethtool_ops = ops;
}
EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
/**
* netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
* @dev: netdev to enable the IRQ coalescing on
*
* Sets a conservative default for SW IRQ coalescing. Users can use
* sysfs attributes to override the default values.
*/
void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
{
WARN_ON(dev->reg_state == NETREG_REGISTERED);
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
dev->gro_flush_timeout = 20000;
dev->napi_defer_hard_irqs = 1;
}
}
EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
void netdev_freemem(struct net_device *dev)
{
char *addr = (char *)dev - dev->padded;
kvfree(addr);
}
/**
* alloc_netdev_mqs - allocate network device
* @sizeof_priv: size of private data to allocate space for
* @name: device name format string
* @name_assign_type: origin of device name
* @setup: callback to initialize device
* @txqs: the number of TX subqueues to allocate
* @rxqs: the number of RX subqueues to allocate
*
* Allocates a struct net_device with private data area for driver use
* and performs basic initialization. Also allocates subqueue structs
* for each queue on the device.
*/
struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
unsigned char name_assign_type,
void (*setup)(struct net_device *),
unsigned int txqs, unsigned int rxqs)
{
struct net_device *dev;
unsigned int alloc_size;
struct net_device *p;
BUG_ON(strlen(name) >= sizeof(dev->name));
if (txqs < 1) {
pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
return NULL;
}
if (rxqs < 1) {
pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
return NULL;
}
alloc_size = sizeof(struct net_device);
if (sizeof_priv) {
/* ensure 32-byte alignment of private area */
alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
alloc_size += sizeof_priv;
}
/* ensure 32-byte alignment of whole construct */
alloc_size += NETDEV_ALIGN - 1;
p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
if (!p)
return NULL;
dev = PTR_ALIGN(p, NETDEV_ALIGN);
dev->padded = (char *)dev - (char *)p;
ref_tracker_dir_init(&dev->refcnt_tracker, 128, name);
#ifdef CONFIG_PCPU_DEV_REFCNT
dev->pcpu_refcnt = alloc_percpu(int);
if (!dev->pcpu_refcnt)
goto free_dev;
__dev_hold(dev);
#else
refcount_set(&dev->dev_refcnt, 1);
#endif
if (dev_addr_init(dev))
goto free_pcpu;
dev_mc_init(dev);
dev_uc_init(dev);
dev_net_set(dev, &init_net);
dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
dev->xdp_zc_max_segs = 1;
dev->gso_max_segs = GSO_MAX_SEGS;
dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
dev->tso_max_segs = TSO_MAX_SEGS;
dev->upper_level = 1;
dev->lower_level = 1;
#ifdef CONFIG_LOCKDEP
dev->nested_level = 0;
INIT_LIST_HEAD(&dev->unlink_list);
#endif
INIT_LIST_HEAD(&dev->napi_list);
INIT_LIST_HEAD(&dev->unreg_list);
INIT_LIST_HEAD(&dev->close_list);
INIT_LIST_HEAD(&dev->link_watch_list);
INIT_LIST_HEAD(&dev->adj_list.upper);
INIT_LIST_HEAD(&dev->adj_list.lower);
INIT_LIST_HEAD(&dev->ptype_all);
INIT_LIST_HEAD(&dev->ptype_specific);
INIT_LIST_HEAD(&dev->net_notifier_list);
#ifdef CONFIG_NET_SCHED
hash_init(dev->qdisc_hash);
#endif
dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
setup(dev);
if (!dev->tx_queue_len) {
dev->priv_flags |= IFF_NO_QUEUE;
dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
}
dev->num_tx_queues = txqs;
dev->real_num_tx_queues = txqs;
if (netif_alloc_netdev_queues(dev))
goto free_all;
dev->num_rx_queues = rxqs;
dev->real_num_rx_queues = rxqs;
if (netif_alloc_rx_queues(dev))
goto free_all;
strcpy(dev->name, name);
dev->name_assign_type = name_assign_type;
dev->group = INIT_NETDEV_GROUP;
if (!dev->ethtool_ops)
dev->ethtool_ops = &default_ethtool_ops;
nf_hook_netdev_init(dev);
return dev;
free_all:
free_netdev(dev);
return NULL;
free_pcpu:
#ifdef CONFIG_PCPU_DEV_REFCNT
free_percpu(dev->pcpu_refcnt);
free_dev:
#endif
netdev_freemem(dev);
return NULL;
}
EXPORT_SYMBOL(alloc_netdev_mqs);
/**
* free_netdev - free network device
* @dev: device
*
* This function does the last stage of destroying an allocated device
* interface. The reference to the device object is released. If this
* is the last reference then it will be freed.Must be called in process
* context.
*/
void free_netdev(struct net_device *dev)
{
struct napi_struct *p, *n;
might_sleep();
/* When called immediately after register_netdevice() failed the unwind
* handling may still be dismantling the device. Handle that case by
* deferring the free.
*/
if (dev->reg_state == NETREG_UNREGISTERING) {
ASSERT_RTNL();
dev->needs_free_netdev = true;
return;
}
netif_free_tx_queues(dev);
netif_free_rx_queues(dev);
kfree(rcu_dereference_protected(dev->ingress_queue, 1));
/* Flush device addresses */
dev_addr_flush(dev);
list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
netif_napi_del(p);
ref_tracker_dir_exit(&dev->refcnt_tracker);
#ifdef CONFIG_PCPU_DEV_REFCNT
free_percpu(dev->pcpu_refcnt);
dev->pcpu_refcnt = NULL;
#endif
free_percpu(dev->core_stats);
dev->core_stats = NULL;
free_percpu(dev->xdp_bulkq);
dev->xdp_bulkq = NULL;
/* Compatibility with error handling in drivers */
if (dev->reg_state == NETREG_UNINITIALIZED) {
netdev_freemem(dev);
return;
}
BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
WRITE_ONCE(dev->reg_state, NETREG_RELEASED);
/* will free via device release */
put_device(&dev->dev);
}
EXPORT_SYMBOL(free_netdev);
/**
* synchronize_net - Synchronize with packet receive processing
*
* Wait for packets currently being received to be done.
* Does not block later packets from starting.
*/
void synchronize_net(void)
{
might_sleep();
if (rtnl_is_locked())
synchronize_rcu_expedited();
else
synchronize_rcu();
}
EXPORT_SYMBOL(synchronize_net);
/**
* unregister_netdevice_queue - remove device from the kernel
* @dev: device
* @head: list
*
* This function shuts down a device interface and removes it
* from the kernel tables.
* If head not NULL, device is queued to be unregistered later.
*
* Callers must hold the rtnl semaphore. You may want
* unregister_netdev() instead of this.
*/
void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
{
ASSERT_RTNL();
if (head) {
list_move_tail(&dev->unreg_list, head);
} else {
LIST_HEAD(single); list_add(&dev->unreg_list, &single); unregister_netdevice_many(&single);
}
}
EXPORT_SYMBOL(unregister_netdevice_queue);
void unregister_netdevice_many_notify(struct list_head *head,
u32 portid, const struct nlmsghdr *nlh)
{
struct net_device *dev, *tmp;
LIST_HEAD(close_head);
int cnt = 0;
BUG_ON(dev_boot_phase); ASSERT_RTNL(); if (list_empty(head))
return;
list_for_each_entry_safe(dev, tmp, head, unreg_list) {
/* Some devices call without registering
* for initialization unwind. Remove those
* devices and proceed with the remaining.
*/
if (dev->reg_state == NETREG_UNINITIALIZED) { pr_debug("unregister_netdevice: device %s/%p never was registered\n",
dev->name, dev);
WARN_ON(1); list_del(&dev->unreg_list);
continue;
}
dev->dismantle = true; BUG_ON(dev->reg_state != NETREG_REGISTERED);
}
/* If device is running, close it first. */
list_for_each_entry(dev, head, unreg_list) list_add_tail(&dev->close_list, &close_head); dev_close_many(&close_head, true);
list_for_each_entry(dev, head, unreg_list) {
/* And unlink it from device chain. */
unlist_netdevice(dev);
WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING);
}
flush_all_backlogs(); synchronize_net(); list_for_each_entry(dev, head, unreg_list) {
struct sk_buff *skb = NULL;
/* Shutdown queueing discipline. */
dev_shutdown(dev); dev_tcx_uninstall(dev); dev_xdp_uninstall(dev); bpf_dev_bound_netdev_unregister(dev);
netdev_offload_xstats_disable_all(dev);
/* Notify protocols, that we are about to destroy
* this device. They should clean all the things.
*/
call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
if (!dev->rtnl_link_ops ||
dev->rtnl_link_state == RTNL_LINK_INITIALIZED) skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
GFP_KERNEL, NULL, 0,
portid, nlh);
/*
* Flush the unicast and multicast chains
*/
dev_uc_flush(dev);
dev_mc_flush(dev);
netdev_name_node_alt_flush(dev); netdev_name_node_free(dev->name_node);
call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
if (dev->netdev_ops->ndo_uninit)
dev->netdev_ops->ndo_uninit(dev); if (skb) rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
/* Notifier chain MUST detach us all upper devices. */
WARN_ON(netdev_has_any_upper_dev(dev)); WARN_ON(netdev_has_any_lower_dev(dev));
/* Remove entries from kobject tree */
netdev_unregister_kobject(dev);
#ifdef CONFIG_XPS
/* Remove XPS queueing entries */
netif_reset_xps_queues_gt(dev, 0);
#endif
}
synchronize_net(); list_for_each_entry(dev, head, unreg_list) { netdev_put(dev, &dev->dev_registered_tracker); net_set_todo(dev); cnt++;
}
atomic_add(cnt, &dev_unreg_count); list_del(head);
}
/**
* unregister_netdevice_many - unregister many devices
* @head: list of devices
*
* Note: As most callers use a stack allocated list_head,
* we force a list_del() to make sure stack wont be corrupted later.
*/
void unregister_netdevice_many(struct list_head *head)
{
unregister_netdevice_many_notify(head, 0, NULL);
}
EXPORT_SYMBOL(unregister_netdevice_many);
/**
* unregister_netdev - remove device from the kernel
* @dev: device
*
* This function shuts down a device interface and removes it
* from the kernel tables.
*
* This is just a wrapper for unregister_netdevice that takes
* the rtnl semaphore. In general you want to use this and not
* unregister_netdevice.
*/
void unregister_netdev(struct net_device *dev)
{
rtnl_lock();
unregister_netdevice(dev);
rtnl_unlock();
}
EXPORT_SYMBOL(unregister_netdev);
/**
* __dev_change_net_namespace - move device to different nethost namespace
* @dev: device
* @net: network namespace
* @pat: If not NULL name pattern to try if the current device name
* is already taken in the destination network namespace.
* @new_ifindex: If not zero, specifies device index in the target
* namespace.
*
* This function shuts down a device interface and moves it
* to a new network namespace. On success 0 is returned, on
* a failure a netagive errno code is returned.
*
* Callers must hold the rtnl semaphore.
*/
int __dev_change_net_namespace(struct net_device *dev, struct net *net,
const char *pat, int new_ifindex)
{
struct netdev_name_node *name_node;
struct net *net_old = dev_net(dev);
char new_name[IFNAMSIZ] = {};
int err, new_nsid;
ASSERT_RTNL();
/* Don't allow namespace local devices to be moved. */
err = -EINVAL;
if (dev->features & NETIF_F_NETNS_LOCAL)
goto out;
/* Ensure the device has been registrered */
if (dev->reg_state != NETREG_REGISTERED)
goto out;
/* Get out if there is nothing todo */
err = 0;
if (net_eq(net_old, net))
goto out;
/* Pick the destination device name, and ensure
* we can use it in the destination network namespace.
*/
err = -EEXIST;
if (netdev_name_in_use(net, dev->name)) {
/* We get here if we can't use the current device name */
if (!pat)
goto out;
err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST);
if (err < 0)
goto out;
}
/* Check that none of the altnames conflicts. */
err = -EEXIST;
netdev_for_each_altname(dev, name_node)
if (netdev_name_in_use(net, name_node->name))
goto out;
/* Check that new_ifindex isn't used yet. */
if (new_ifindex) {
err = dev_index_reserve(net, new_ifindex);
if (err < 0)
goto out;
} else {
/* If there is an ifindex conflict assign a new one */
err = dev_index_reserve(net, dev->ifindex);
if (err == -EBUSY)
err = dev_index_reserve(net, 0);
if (err < 0)
goto out;
new_ifindex = err;
}
/*
* And now a mini version of register_netdevice unregister_netdevice.
*/
/* If device is running close it first. */
dev_close(dev);
/* And unlink it from device chain */
unlist_netdevice(dev);
synchronize_net();
/* Shutdown queueing discipline. */
dev_shutdown(dev);
/* Notify protocols, that we are about to destroy
* this device. They should clean all the things.
*
* Note that dev->reg_state stays at NETREG_REGISTERED.
* This is wanted because this way 8021q and macvlan know
* the device is just moving and can keep their slaves up.
*/
call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
rcu_barrier();
new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
new_ifindex);
/*
* Flush the unicast and multicast chains
*/
dev_uc_flush(dev);
dev_mc_flush(dev);
/* Send a netdev-removed uevent to the old namespace */
kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
netdev_adjacent_del_links(dev);
/* Move per-net netdevice notifiers that are following the netdevice */
move_netdevice_notifiers_dev_net(dev, net);
/* Actually switch the network namespace */
dev_net_set(dev, net);
dev->ifindex = new_ifindex;
if (new_name[0]) /* Rename the netdev to prepared name */
strscpy(dev->name, new_name, IFNAMSIZ);
/* Fixup kobjects */
dev_set_uevent_suppress(&dev->dev, 1);
err = device_rename(&dev->dev, dev->name);
dev_set_uevent_suppress(&dev->dev, 0);
WARN_ON(err);
/* Send a netdev-add uevent to the new namespace */
kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
netdev_adjacent_add_links(dev);
/* Adapt owner in case owning user namespace of target network
* namespace is different from the original one.
*/
err = netdev_change_owner(dev, net_old, net);
WARN_ON(err);
/* Add the device back in the hashes */
list_netdevice(dev);
/* Notify protocols, that a new device appeared. */
call_netdevice_notifiers(NETDEV_REGISTER, dev);
/*
* Prevent userspace races by waiting until the network
* device is fully setup before sending notifications.
*/
rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
synchronize_net();
err = 0;
out:
return err;
}
EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
static int dev_cpu_dead(unsigned int oldcpu)
{
struct sk_buff **list_skb;
struct sk_buff *skb;
unsigned int cpu;
struct softnet_data *sd, *oldsd, *remsd = NULL;
local_irq_disable();
cpu = smp_processor_id();
sd = &per_cpu(softnet_data, cpu);
oldsd = &per_cpu(softnet_data, oldcpu);
/* Find end of our completion_queue. */
list_skb = &sd->completion_queue;
while (*list_skb)
list_skb = &(*list_skb)->next;
/* Append completion queue from offline CPU. */
*list_skb = oldsd->completion_queue;
oldsd->completion_queue = NULL;
/* Append output queue from offline CPU. */
if (oldsd->output_queue) {
*sd->output_queue_tailp = oldsd->output_queue;
sd->output_queue_tailp = oldsd->output_queue_tailp;
oldsd->output_queue = NULL;
oldsd->output_queue_tailp = &oldsd->output_queue;
}
/* Append NAPI poll list from offline CPU, with one exception :
* process_backlog() must be called by cpu owning percpu backlog.
* We properly handle process_queue & input_pkt_queue later.
*/
while (!list_empty(&oldsd->poll_list)) {
struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
struct napi_struct,
poll_list);
list_del_init(&napi->poll_list);
if (napi->poll == process_backlog)
napi->state = 0;
else
____napi_schedule(sd, napi);
}
raise_softirq_irqoff(NET_TX_SOFTIRQ);
local_irq_enable();
#ifdef CONFIG_RPS
remsd = oldsd->rps_ipi_list;
oldsd->rps_ipi_list = NULL;
#endif
/* send out pending IPI's on offline CPU */
net_rps_send_ipi(remsd);
/* Process offline CPU's input_pkt_queue */
while ((skb = __skb_dequeue(&oldsd->process_queue))) {
netif_rx(skb);
input_queue_head_incr(oldsd);
}
while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
netif_rx(skb);
input_queue_head_incr(oldsd);
}
return 0;
}
/**
* netdev_increment_features - increment feature set by one
* @all: current feature set
* @one: new feature set
* @mask: mask feature set
*
* Computes a new feature set after adding a device with feature set
* @one to the master device with current feature set @all. Will not
* enable anything that is off in @mask. Returns the new feature set.
*/
netdev_features_t netdev_increment_features(netdev_features_t all,
netdev_features_t one, netdev_features_t mask)
{
if (mask & NETIF_F_HW_CSUM)
mask |= NETIF_F_CSUM_MASK;
mask |= NETIF_F_VLAN_CHALLENGED;
all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
all &= one | ~NETIF_F_ALL_FOR_ALL;
/* If one device supports hw checksumming, set for all. */
if (all & NETIF_F_HW_CSUM)
all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
return all;
}
EXPORT_SYMBOL(netdev_increment_features);
static struct hlist_head * __net_init netdev_create_hash(void)
{
int i;
struct hlist_head *hash;
hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
if (hash != NULL)
for (i = 0; i < NETDEV_HASHENTRIES; i++)
INIT_HLIST_HEAD(&hash[i]);
return hash;
}
/* Initialize per network namespace state */
static int __net_init netdev_init(struct net *net)
{
BUILD_BUG_ON(GRO_HASH_BUCKETS >
8 * sizeof_field(struct napi_struct, gro_bitmask));
INIT_LIST_HEAD(&net->dev_base_head);
net->dev_name_head = netdev_create_hash();
if (net->dev_name_head == NULL)
goto err_name;
net->dev_index_head = netdev_create_hash();
if (net->dev_index_head == NULL)
goto err_idx;
xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1);
RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
return 0;
err_idx:
kfree(net->dev_name_head);
err_name:
return -ENOMEM;
}
/**
* netdev_drivername - network driver for the device
* @dev: network device
*
* Determine network driver for device.
*/
const char *netdev_drivername(const struct net_device *dev)
{
const struct device_driver *driver;
const struct device *parent;
const char *empty = "";
parent = dev->dev.parent;
if (!parent)
return empty;
driver = parent->driver;
if (driver && driver->name)
return driver->name;
return empty;
}
static void __netdev_printk(const char *level, const struct net_device *dev,
struct va_format *vaf)
{
if (dev && dev->dev.parent) { dev_printk_emit(level[1] - '0',
dev->dev.parent,
"%s %s %s%s: %pV",
dev_driver_string(dev->dev.parent),
dev_name(dev->dev.parent),
netdev_name(dev), netdev_reg_state(dev),
vaf);
} else if (dev) {
printk("%s%s%s: %pV",
level, netdev_name(dev), netdev_reg_state(dev), vaf);
} else {
printk("%s(NULL net_device): %pV", level, vaf);
}
}
void netdev_printk(const char *level, const struct net_device *dev,
const char *format, ...)
{
struct va_format vaf;
va_list args;
va_start(args, format);
vaf.fmt = format;
vaf.va = &args;
__netdev_printk(level, dev, &vaf);
va_end(args);
}
EXPORT_SYMBOL(netdev_printk);
#define define_netdev_printk_level(func, level) \
void func(const struct net_device *dev, const char *fmt, ...) \
{ \
struct va_format vaf; \
va_list args; \
\
va_start(args, fmt); \
\
vaf.fmt = fmt; \
vaf.va = &args; \
\
__netdev_printk(level, dev, &vaf); \
\
va_end(args); \
} \
EXPORT_SYMBOL(func);
define_netdev_printk_level(netdev_emerg, KERN_EMERG);
define_netdev_printk_level(netdev_alert, KERN_ALERT);
define_netdev_printk_level(netdev_crit, KERN_CRIT);
define_netdev_printk_level(netdev_err, KERN_ERR);
define_netdev_printk_level(netdev_warn, KERN_WARNING);
define_netdev_printk_level(netdev_notice, KERN_NOTICE);
define_netdev_printk_level(netdev_info, KERN_INFO);
static void __net_exit netdev_exit(struct net *net)
{
kfree(net->dev_name_head);
kfree(net->dev_index_head);
xa_destroy(&net->dev_by_index);
if (net != &init_net)
WARN_ON_ONCE(!list_empty(&net->dev_base_head));
}
static struct pernet_operations __net_initdata netdev_net_ops = {
.init = netdev_init,
.exit = netdev_exit,
};
static void __net_exit default_device_exit_net(struct net *net)
{
struct netdev_name_node *name_node, *tmp;
struct net_device *dev, *aux;
/*
* Push all migratable network devices back to the
* initial network namespace
*/
ASSERT_RTNL();
for_each_netdev_safe(net, dev, aux) {
int err;
char fb_name[IFNAMSIZ];
/* Ignore unmoveable devices (i.e. loopback) */
if (dev->features & NETIF_F_NETNS_LOCAL)
continue;
/* Leave virtual devices for the generic cleanup */
if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
continue;
/* Push remaining network devices to init_net */
snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
if (netdev_name_in_use(&init_net, fb_name))
snprintf(fb_name, IFNAMSIZ, "dev%%d");
netdev_for_each_altname_safe(dev, name_node, tmp)
if (netdev_name_in_use(&init_net, name_node->name))
__netdev_name_node_alt_destroy(name_node);
err = dev_change_net_namespace(dev, &init_net, fb_name);
if (err) {
pr_emerg("%s: failed to move %s to init_net: %d\n",
__func__, dev->name, err);
BUG();
}
}
}
static void __net_exit default_device_exit_batch(struct list_head *net_list)
{
/* At exit all network devices most be removed from a network
* namespace. Do this in the reverse order of registration.
* Do this across as many network namespaces as possible to
* improve batching efficiency.
*/
struct net_device *dev;
struct net *net;
LIST_HEAD(dev_kill_list);
rtnl_lock();
list_for_each_entry(net, net_list, exit_list) {
default_device_exit_net(net);
cond_resched();
}
list_for_each_entry(net, net_list, exit_list) {
for_each_netdev_reverse(net, dev) {
if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
else
unregister_netdevice_queue(dev, &dev_kill_list);
}
}
unregister_netdevice_many(&dev_kill_list);
rtnl_unlock();
}
static struct pernet_operations __net_initdata default_device_ops = {
.exit_batch = default_device_exit_batch,
};
static void __init net_dev_struct_check(void)
{
/* TX read-mostly hotpath */
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq);
#ifdef CONFIG_XPS
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps);
#endif
#ifdef CONFIG_NETFILTER_EGRESS
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress);
#endif
#ifdef CONFIG_NET_XGRESS
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress);
#endif
CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160);
/* TXRX read-mostly hotpath */
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr);
CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46);
/* RX read-mostly hotpath */
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_flush_timeout);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, napi_defer_hard_irqs);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data);
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net);
#ifdef CONFIG_NETPOLL
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo);
#endif
#ifdef CONFIG_NET_XGRESS
CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress);
#endif
CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 104);
}
/*
* Initialize the DEV module. At boot time this walks the device list and
* unhooks any devices that fail to initialise (normally hardware not
* present) and leaves us with a valid list of present and active devices.
*
*/
/* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */
#define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE)
static int net_page_pool_create(int cpuid)
{
#if IS_ENABLED(CONFIG_PAGE_POOL)
struct page_pool_params page_pool_params = {
.pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE,
.flags = PP_FLAG_SYSTEM_POOL,
.nid = NUMA_NO_NODE,
};
struct page_pool *pp_ptr;
pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid);
if (IS_ERR(pp_ptr))
return -ENOMEM;
per_cpu(system_page_pool, cpuid) = pp_ptr;
#endif
return 0;
}
/*
* This is called single threaded during boot, so no need
* to take the rtnl semaphore.
*/
static int __init net_dev_init(void)
{
int i, rc = -ENOMEM;
BUG_ON(!dev_boot_phase);
net_dev_struct_check();
if (dev_proc_init())
goto out;
if (netdev_kobject_init())
goto out;
for (i = 0; i < PTYPE_HASH_SIZE; i++)
INIT_LIST_HEAD(&ptype_base[i]);
if (register_pernet_subsys(&netdev_net_ops))
goto out;
/*
* Initialise the packet receive queues.
*/
for_each_possible_cpu(i) {
struct work_struct *flush = per_cpu_ptr(&flush_works, i);
struct softnet_data *sd = &per_cpu(softnet_data, i);
INIT_WORK(flush, flush_backlog);
skb_queue_head_init(&sd->input_pkt_queue);
skb_queue_head_init(&sd->process_queue);
#ifdef CONFIG_XFRM_OFFLOAD
skb_queue_head_init(&sd->xfrm_backlog);
#endif
INIT_LIST_HEAD(&sd->poll_list);
sd->output_queue_tailp = &sd->output_queue;
#ifdef CONFIG_RPS
INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
sd->cpu = i;
#endif
INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
spin_lock_init(&sd->defer_lock);
init_gro_hash(&sd->backlog);
sd->backlog.poll = process_backlog;
sd->backlog.weight = weight_p;
if (net_page_pool_create(i))
goto out;
}
dev_boot_phase = 0;
/* The loopback device is special if any other network devices
* is present in a network namespace the loopback device must
* be present. Since we now dynamically allocate and free the
* loopback device ensure this invariant is maintained by
* keeping the loopback device as the first device on the
* list of network devices. Ensuring the loopback devices
* is the first device that appears and the last network device
* that disappears.
*/
if (register_pernet_device(&loopback_net_ops))
goto out;
if (register_pernet_device(&default_device_ops))
goto out;
open_softirq(NET_TX_SOFTIRQ, net_tx_action);
open_softirq(NET_RX_SOFTIRQ, net_rx_action);
rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
NULL, dev_cpu_dead);
WARN_ON(rc < 0);
rc = 0;
out:
if (rc < 0) {
for_each_possible_cpu(i) {
struct page_pool *pp_ptr;
pp_ptr = per_cpu(system_page_pool, i);
if (!pp_ptr)
continue;
page_pool_destroy(pp_ptr);
per_cpu(system_page_pool, i) = NULL;
}
}
return rc;
}
subsys_initcall(net_dev_init);
// SPDX-License-Identifier: GPL-2.0
/*
* Workingset detection
*
* Copyright (C) 2013 Red Hat, Inc., Johannes Weiner
*/
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/writeback.h>
#include <linux/shmem_fs.h>
#include <linux/pagemap.h>
#include <linux/atomic.h>
#include <linux/module.h>
#include <linux/swap.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include "internal.h"
/*
* Double CLOCK lists
*
* Per node, two clock lists are maintained for file pages: the
* inactive and the active list. Freshly faulted pages start out at
* the head of the inactive list and page reclaim scans pages from the
* tail. Pages that are accessed multiple times on the inactive list
* are promoted to the active list, to protect them from reclaim,
* whereas active pages are demoted to the inactive list when the
* active list grows too big.
*
* fault ------------------------+
* |
* +--------------+ | +-------------+
* reclaim <- | inactive | <-+-- demotion | active | <--+
* +--------------+ +-------------+ |
* | |
* +-------------- promotion ------------------+
*
*
* Access frequency and refault distance
*
* A workload is thrashing when its pages are frequently used but they
* are evicted from the inactive list every time before another access
* would have promoted them to the active list.
*
* In cases where the average access distance between thrashing pages
* is bigger than the size of memory there is nothing that can be
* done - the thrashing set could never fit into memory under any
* circumstance.
*
* However, the average access distance could be bigger than the
* inactive list, yet smaller than the size of memory. In this case,
* the set could fit into memory if it weren't for the currently
* active pages - which may be used more, hopefully less frequently:
*
* +-memory available to cache-+
* | |
* +-inactive------+-active----+
* a b | c d e f g h i | J K L M N |
* +---------------+-----------+
*
* It is prohibitively expensive to accurately track access frequency
* of pages. But a reasonable approximation can be made to measure
* thrashing on the inactive list, after which refaulting pages can be
* activated optimistically to compete with the existing active pages.
*
* Approximating inactive page access frequency - Observations:
*
* 1. When a page is accessed for the first time, it is added to the
* head of the inactive list, slides every existing inactive page
* towards the tail by one slot, and pushes the current tail page
* out of memory.
*
* 2. When a page is accessed for the second time, it is promoted to
* the active list, shrinking the inactive list by one slot. This
* also slides all inactive pages that were faulted into the cache
* more recently than the activated page towards the tail of the
* inactive list.
*
* Thus:
*
* 1. The sum of evictions and activations between any two points in
* time indicate the minimum number of inactive pages accessed in
* between.
*
* 2. Moving one inactive page N page slots towards the tail of the
* list requires at least N inactive page accesses.
*
* Combining these:
*
* 1. When a page is finally evicted from memory, the number of
* inactive pages accessed while the page was in cache is at least
* the number of page slots on the inactive list.
*
* 2. In addition, measuring the sum of evictions and activations (E)
* at the time of a page's eviction, and comparing it to another
* reading (R) at the time the page faults back into memory tells
* the minimum number of accesses while the page was not cached.
* This is called the refault distance.
*
* Because the first access of the page was the fault and the second
* access the refault, we combine the in-cache distance with the
* out-of-cache distance to get the complete minimum access distance
* of this page:
*
* NR_inactive + (R - E)
*
* And knowing the minimum access distance of a page, we can easily
* tell if the page would be able to stay in cache assuming all page
* slots in the cache were available:
*
* NR_inactive + (R - E) <= NR_inactive + NR_active
*
* If we have swap we should consider about NR_inactive_anon and
* NR_active_anon, so for page cache and anonymous respectively:
*
* NR_inactive_file + (R - E) <= NR_inactive_file + NR_active_file
* + NR_inactive_anon + NR_active_anon
*
* NR_inactive_anon + (R - E) <= NR_inactive_anon + NR_active_anon
* + NR_inactive_file + NR_active_file
*
* Which can be further simplified to:
*
* (R - E) <= NR_active_file + NR_inactive_anon + NR_active_anon
*
* (R - E) <= NR_active_anon + NR_inactive_file + NR_active_file
*
* Put into words, the refault distance (out-of-cache) can be seen as
* a deficit in inactive list space (in-cache). If the inactive list
* had (R - E) more page slots, the page would not have been evicted
* in between accesses, but activated instead. And on a full system,
* the only thing eating into inactive list space is active pages.
*
*
* Refaulting inactive pages
*
* All that is known about the active list is that the pages have been
* accessed more than once in the past. This means that at any given
* time there is actually a good chance that pages on the active list
* are no longer in active use.
*
* So when a refault distance of (R - E) is observed and there are at
* least (R - E) pages in the userspace workingset, the refaulting page
* is activated optimistically in the hope that (R - E) pages are actually
* used less frequently than the refaulting page - or even not used at
* all anymore.
*
* That means if inactive cache is refaulting with a suitable refault
* distance, we assume the cache workingset is transitioning and put
* pressure on the current workingset.
*
* If this is wrong and demotion kicks in, the pages which are truly
* used more frequently will be reactivated while the less frequently
* used once will be evicted from memory.
*
* But if this is right, the stale pages will be pushed out of memory
* and the used pages get to stay in cache.
*
* Refaulting active pages
*
* If on the other hand the refaulting pages have recently been
* deactivated, it means that the active list is no longer protecting
* actively used cache from reclaim. The cache is NOT transitioning to
* a different workingset; the existing workingset is thrashing in the
* space allocated to the page cache.
*
*
* Implementation
*
* For each node's LRU lists, a counter for inactive evictions and
* activations is maintained (node->nonresident_age).
*
* On eviction, a snapshot of this counter (along with some bits to
* identify the node) is stored in the now empty page cache
* slot of the evicted page. This is called a shadow entry.
*
* On cache misses for which there are shadow entries, an eligible
* refault distance will immediately activate the refaulting page.
*/
#define WORKINGSET_SHIFT 1
#define EVICTION_SHIFT ((BITS_PER_LONG - BITS_PER_XA_VALUE) + \
WORKINGSET_SHIFT + NODES_SHIFT + \
MEM_CGROUP_ID_SHIFT)
#define EVICTION_MASK (~0UL >> EVICTION_SHIFT)
/*
* Eviction timestamps need to be able to cover the full range of
* actionable refaults. However, bits are tight in the xarray
* entry, and after storing the identifier for the lruvec there might
* not be enough left to represent every single actionable refault. In
* that case, we have to sacrifice granularity for distance, and group
* evictions into coarser buckets by shaving off lower timestamp bits.
*/
static unsigned int bucket_order __read_mostly;
static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
bool workingset)
{
eviction &= EVICTION_MASK;
eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
eviction = (eviction << WORKINGSET_SHIFT) | workingset;
return xa_mk_value(eviction);
}
static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
unsigned long *evictionp, bool *workingsetp)
{
unsigned long entry = xa_to_value(shadow);
int memcgid, nid;
bool workingset;
workingset = entry & ((1UL << WORKINGSET_SHIFT) - 1);
entry >>= WORKINGSET_SHIFT;
nid = entry & ((1UL << NODES_SHIFT) - 1);
entry >>= NODES_SHIFT;
memcgid = entry & ((1UL << MEM_CGROUP_ID_SHIFT) - 1);
entry >>= MEM_CGROUP_ID_SHIFT;
*memcgidp = memcgid;
*pgdat = NODE_DATA(nid);
*evictionp = entry;
*workingsetp = workingset;
}
#ifdef CONFIG_LRU_GEN
static void *lru_gen_eviction(struct folio *folio)
{
int hist;
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
struct lru_gen_folio *lrugen;
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
int refs = folio_lru_refs(folio);
int tier = lru_tier_from_refs(refs);
struct mem_cgroup *memcg = folio_memcg(folio);
struct pglist_data *pgdat = folio_pgdat(folio);
BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);
lruvec = mem_cgroup_lruvec(memcg, pgdat);
lrugen = &lruvec->lrugen;
min_seq = READ_ONCE(lrugen->min_seq[type]);
token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
hist = lru_hist_from_seq(min_seq);
atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs);
}
/*
* Tests if the shadow entry is for a folio that was recently evicted.
* Fills in @lruvec, @token, @workingset with the values unpacked from shadow.
*/
static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec,
unsigned long *token, bool *workingset)
{
int memcg_id;
unsigned long min_seq;
struct mem_cgroup *memcg;
struct pglist_data *pgdat;
unpack_shadow(shadow, &memcg_id, &pgdat, token, workingset);
memcg = mem_cgroup_from_id(memcg_id);
*lruvec = mem_cgroup_lruvec(memcg, pgdat);
min_seq = READ_ONCE((*lruvec)->lrugen.min_seq[file]);
return (*token >> LRU_REFS_WIDTH) == (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH));
}
static void lru_gen_refault(struct folio *folio, void *shadow)
{
bool recent;
int hist, tier, refs;
bool workingset;
unsigned long token;
struct lruvec *lruvec;
struct lru_gen_folio *lrugen;
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
rcu_read_lock();
recent = lru_gen_test_recent(shadow, type, &lruvec, &token, &workingset);
if (lruvec != folio_lruvec(folio))
goto unlock;
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
if (!recent)
goto unlock;
lrugen = &lruvec->lrugen;
hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
/* see the comment in folio_lru_refs() */
refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset;
tier = lru_tier_from_refs(refs);
atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
/*
* Count the following two cases as stalls:
* 1. For pages accessed through page tables, hotter pages pushed out
* hot pages which refaulted immediately.
* 2. For pages accessed multiple times through file descriptors,
* they would have been protected by sort_folio().
*/
if (lru_gen_in_fault() || refs >= BIT(LRU_REFS_WIDTH) - 1) {
set_mask_bits(&folio->flags, 0, LRU_REFS_MASK | BIT(PG_workingset));
mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
}
unlock:
rcu_read_unlock();
}
#else /* !CONFIG_LRU_GEN */
static void *lru_gen_eviction(struct folio *folio)
{
return NULL;
}
static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec,
unsigned long *token, bool *workingset)
{
return false;
}
static void lru_gen_refault(struct folio *folio, void *shadow)
{
}
#endif /* CONFIG_LRU_GEN */
/**
* workingset_age_nonresident - age non-resident entries as LRU ages
* @lruvec: the lruvec that was aged
* @nr_pages: the number of pages to count
*
* As in-memory pages are aged, non-resident pages need to be aged as
* well, in order for the refault distances later on to be comparable
* to the in-memory dimensions. This function allows reclaim and LRU
* operations to drive the non-resident aging along in parallel.
*/
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages)
{
/*
* Reclaiming a cgroup means reclaiming all its children in a
* round-robin fashion. That means that each cgroup has an LRU
* order that is composed of the LRU orders of its child
* cgroups; and every page has an LRU position not just in the
* cgroup that owns it, but in all of that group's ancestors.
*
* So when the physical inactive list of a leaf cgroup ages,
* the virtual inactive lists of all its parents, including
* the root cgroup's, age as well.
*/
do {
atomic_long_add(nr_pages, &lruvec->nonresident_age); } while ((lruvec = parent_lruvec(lruvec)));
}
/**
* workingset_eviction - note the eviction of a folio from memory
* @target_memcg: the cgroup that is causing the reclaim
* @folio: the folio being evicted
*
* Return: a shadow entry to be stored in @folio->mapping->i_pages in place
* of the evicted @folio so that a later refault can be detected.
*/
void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg)
{
struct pglist_data *pgdat = folio_pgdat(folio);
unsigned long eviction;
struct lruvec *lruvec;
int memcgid;
/* Folio is fully exclusive and pins folio's memory cgroup pointer */
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (lru_gen_enabled())
return lru_gen_eviction(folio);
lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
/* XXX: target_memcg can be NULL, go through lruvec */
memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
eviction = atomic_long_read(&lruvec->nonresident_age);
eviction >>= bucket_order;
workingset_age_nonresident(lruvec, folio_nr_pages(folio));
return pack_shadow(memcgid, pgdat, eviction,
folio_test_workingset(folio));
}
/**
* workingset_test_recent - tests if the shadow entry is for a folio that was
* recently evicted. Also fills in @workingset with the value unpacked from
* shadow.
* @shadow: the shadow entry to be tested.
* @file: whether the corresponding folio is from the file lru.
* @workingset: where the workingset value unpacked from shadow should
* be stored.
*
* Return: true if the shadow is for a recently evicted folio; false otherwise.
*/
bool workingset_test_recent(void *shadow, bool file, bool *workingset)
{
struct mem_cgroup *eviction_memcg;
struct lruvec *eviction_lruvec;
unsigned long refault_distance;
unsigned long workingset_size;
unsigned long refault;
int memcgid;
struct pglist_data *pgdat;
unsigned long eviction;
rcu_read_lock();
if (lru_gen_enabled()) {
bool recent = lru_gen_test_recent(shadow, file,
&eviction_lruvec, &eviction, workingset);
rcu_read_unlock();
return recent;
}
unpack_shadow(shadow, &memcgid, &pgdat, &eviction, workingset);
eviction <<= bucket_order;
/*
* Look up the memcg associated with the stored ID. It might
* have been deleted since the folio's eviction.
*
* Note that in rare events the ID could have been recycled
* for a new cgroup that refaults a shared folio. This is
* impossible to tell from the available data. However, this
* should be a rare and limited disturbance, and activations
* are always speculative anyway. Ultimately, it's the aging
* algorithm's job to shake out the minimum access frequency
* for the active cache.
*
* XXX: On !CONFIG_MEMCG, this will always return NULL; it
* would be better if the root_mem_cgroup existed in all
* configurations instead.
*/
eviction_memcg = mem_cgroup_from_id(memcgid);
if (!mem_cgroup_disabled() &&
(!eviction_memcg || !mem_cgroup_tryget(eviction_memcg))) {
rcu_read_unlock();
return false;
}
rcu_read_unlock();
/*
* Flush stats (and potentially sleep) outside the RCU read section.
* XXX: With per-memcg flushing and thresholding, is ratelimiting
* still needed here?
*/
mem_cgroup_flush_stats_ratelimited(eviction_memcg);
eviction_lruvec = mem_cgroup_lruvec(eviction_memcg, pgdat);
refault = atomic_long_read(&eviction_lruvec->nonresident_age);
/*
* Calculate the refault distance
*
* The unsigned subtraction here gives an accurate distance
* across nonresident_age overflows in most cases. There is a
* special case: usually, shadow entries have a short lifetime
* and are either refaulted or reclaimed along with the inode
* before they get too old. But it is not impossible for the
* nonresident_age to lap a shadow entry in the field, which
* can then result in a false small refault distance, leading
* to a false activation should this old entry actually
* refault again. However, earlier kernels used to deactivate
* unconditionally with *every* reclaim invocation for the
* longest time, so the occasional inappropriate activation
* leading to pressure on the active list is not a problem.
*/
refault_distance = (refault - eviction) & EVICTION_MASK;
/*
* Compare the distance to the existing workingset size. We
* don't activate pages that couldn't stay resident even if
* all the memory was available to the workingset. Whether
* workingset competition needs to consider anon or not depends
* on having free swap space.
*/
workingset_size = lruvec_page_state(eviction_lruvec, NR_ACTIVE_FILE);
if (!file) {
workingset_size += lruvec_page_state(eviction_lruvec,
NR_INACTIVE_FILE);
}
if (mem_cgroup_get_nr_swap_pages(eviction_memcg) > 0) {
workingset_size += lruvec_page_state(eviction_lruvec,
NR_ACTIVE_ANON);
if (file) {
workingset_size += lruvec_page_state(eviction_lruvec,
NR_INACTIVE_ANON);
}
}
mem_cgroup_put(eviction_memcg);
return refault_distance <= workingset_size;
}
/**
* workingset_refault - Evaluate the refault of a previously evicted folio.
* @folio: The freshly allocated replacement folio.
* @shadow: Shadow entry of the evicted folio.
*
* Calculates and evaluates the refault distance of the previously
* evicted folio in the context of the node and the memcg whose memory
* pressure caused the eviction.
*/
void workingset_refault(struct folio *folio, void *shadow)
{
bool file = folio_is_file_lru(folio);
struct pglist_data *pgdat;
struct mem_cgroup *memcg;
struct lruvec *lruvec;
bool workingset;
long nr;
if (lru_gen_enabled()) {
lru_gen_refault(folio, shadow);
return;
}
/*
* The activation decision for this folio is made at the level
* where the eviction occurred, as that is where the LRU order
* during folio reclaim is being determined.
*
* However, the cgroup that will own the folio is the one that
* is actually experiencing the refault event. Make sure the folio is
* locked to guarantee folio_memcg() stability throughout.
*/
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
nr = folio_nr_pages(folio);
memcg = folio_memcg(folio);
pgdat = folio_pgdat(folio);
lruvec = mem_cgroup_lruvec(memcg, pgdat);
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr);
if (!workingset_test_recent(shadow, file, &workingset))
return;
folio_set_active(folio);
workingset_age_nonresident(lruvec, nr);
mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + file, nr);
/* Folio was active prior to eviction */
if (workingset) {
folio_set_workingset(folio);
/*
* XXX: Move to folio_add_lru() when it supports new vs
* putback
*/
lru_note_cost_refault(folio);
mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
}
}
/**
* workingset_activation - note a page activation
* @folio: Folio that is being activated.
*/
void workingset_activation(struct folio *folio)
{
struct mem_cgroup *memcg;
rcu_read_lock();
/*
* Filter non-memcg pages here, e.g. unmap can call
* mark_page_accessed() on VDSO pages.
*
* XXX: See workingset_refault() - this should return
* root_mem_cgroup even for !CONFIG_MEMCG.
*/
memcg = folio_memcg_rcu(folio); if (!mem_cgroup_disabled() && !memcg)
goto out;
workingset_age_nonresident(folio_lruvec(folio), folio_nr_pages(folio));
out:
rcu_read_unlock();
}
/*
* Shadow entries reflect the share of the working set that does not
* fit into memory, so their number depends on the access pattern of
* the workload. In most cases, they will refault or get reclaimed
* along with the inode, but a (malicious) workload that streams
* through files with a total size several times that of available
* memory, while preventing the inodes from being reclaimed, can
* create excessive amounts of shadow nodes. To keep a lid on this,
* track shadow nodes and reclaim them when they grow way past the
* point where they would still be useful.
*/
struct list_lru shadow_nodes;
void workingset_update_node(struct xa_node *node)
{
struct address_space *mapping;
/*
* Track non-empty nodes that contain only shadow entries;
* unlink those that contain pages or are being freed.
*
* Avoid acquiring the list_lru lock when the nodes are
* already where they should be. The list_empty() test is safe
* as node->private_list is protected by the i_pages lock.
*/
mapping = container_of(node->array, struct address_space, i_pages);
lockdep_assert_held(&mapping->i_pages.xa_lock);
if (node->count && node->count == node->nr_values) {
if (list_empty(&node->private_list)) {
list_lru_add_obj(&shadow_nodes, &node->private_list);
__inc_lruvec_kmem_state(node, WORKINGSET_NODES);
}
} else {
if (!list_empty(&node->private_list)) {
list_lru_del_obj(&shadow_nodes, &node->private_list);
__dec_lruvec_kmem_state(node, WORKINGSET_NODES);
}
}
}
static unsigned long count_shadow_nodes(struct shrinker *shrinker,
struct shrink_control *sc)
{
unsigned long max_nodes;
unsigned long nodes;
unsigned long pages;
nodes = list_lru_shrink_count(&shadow_nodes, sc);
if (!nodes)
return SHRINK_EMPTY;
/*
* Approximate a reasonable limit for the nodes
* containing shadow entries. We don't need to keep more
* shadow entries than possible pages on the active list,
* since refault distances bigger than that are dismissed.
*
* The size of the active list converges toward 100% of
* overall page cache as memory grows, with only a tiny
* inactive list. Assume the total cache size for that.
*
* Nodes might be sparsely populated, with only one shadow
* entry in the extreme case. Obviously, we cannot keep one
* node for every eligible shadow entry, so compromise on a
* worst-case density of 1/8th. Below that, not all eligible
* refaults can be detected anymore.
*
* On 64-bit with 7 xa_nodes per page and 64 slots
* each, this will reclaim shadow entries when they consume
* ~1.8% of available memory:
*
* PAGE_SIZE / xa_nodes / node_entries * 8 / PAGE_SIZE
*/
#ifdef CONFIG_MEMCG
if (sc->memcg) {
struct lruvec *lruvec;
int i;
mem_cgroup_flush_stats_ratelimited(sc->memcg);
lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
pages += lruvec_page_state_local(lruvec,
NR_LRU_BASE + i);
pages += lruvec_page_state_local(
lruvec, NR_SLAB_RECLAIMABLE_B) >> PAGE_SHIFT;
pages += lruvec_page_state_local(
lruvec, NR_SLAB_UNRECLAIMABLE_B) >> PAGE_SHIFT;
} else
#endif
pages = node_present_pages(sc->nid);
max_nodes = pages >> (XA_CHUNK_SHIFT - 3);
if (nodes <= max_nodes)
return 0;
return nodes - max_nodes;
}
static enum lru_status shadow_lru_isolate(struct list_head *item,
struct list_lru_one *lru,
spinlock_t *lru_lock,
void *arg) __must_hold(lru_lock)
{
struct xa_node *node = container_of(item, struct xa_node, private_list);
struct address_space *mapping;
int ret;
/*
* Page cache insertions and deletions synchronously maintain
* the shadow node LRU under the i_pages lock and the
* lru_lock. Because the page cache tree is emptied before
* the inode can be destroyed, holding the lru_lock pins any
* address_space that has nodes on the LRU.
*
* We can then safely transition to the i_pages lock to
* pin only the address_space of the particular node we want
* to reclaim, take the node off-LRU, and drop the lru_lock.
*/
mapping = container_of(node->array, struct address_space, i_pages);
/* Coming from the list, invert the lock order */
if (!xa_trylock(&mapping->i_pages)) {
spin_unlock_irq(lru_lock);
ret = LRU_RETRY;
goto out;
}
/* For page cache we need to hold i_lock */
if (mapping->host != NULL) {
if (!spin_trylock(&mapping->host->i_lock)) {
xa_unlock(&mapping->i_pages);
spin_unlock_irq(lru_lock);
ret = LRU_RETRY;
goto out;
}
}
list_lru_isolate(lru, item);
__dec_lruvec_kmem_state(node, WORKINGSET_NODES);
spin_unlock(lru_lock);
/*
* The nodes should only contain one or more shadow entries,
* no pages, so we expect to be able to remove them all and
* delete and free the empty node afterwards.
*/
if (WARN_ON_ONCE(!node->nr_values))
goto out_invalid;
if (WARN_ON_ONCE(node->count != node->nr_values))
goto out_invalid;
xa_delete_node(node, workingset_update_node);
__inc_lruvec_kmem_state(node, WORKINGSET_NODERECLAIM);
out_invalid:
xa_unlock_irq(&mapping->i_pages);
if (mapping->host != NULL) {
if (mapping_shrinkable(mapping))
inode_add_lru(mapping->host);
spin_unlock(&mapping->host->i_lock);
}
ret = LRU_REMOVED_RETRY;
out:
cond_resched();
spin_lock_irq(lru_lock);
return ret;
}
static unsigned long scan_shadow_nodes(struct shrinker *shrinker,
struct shrink_control *sc)
{
/* list_lru lock nests inside the IRQ-safe i_pages lock */
return list_lru_shrink_walk_irq(&shadow_nodes, sc, shadow_lru_isolate,
NULL);
}
/*
* Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe
* i_pages lock.
*/
static struct lock_class_key shadow_nodes_key;
static int __init workingset_init(void)
{
struct shrinker *workingset_shadow_shrinker;
unsigned int timestamp_bits;
unsigned int max_order;
int ret = -ENOMEM;
BUILD_BUG_ON(BITS_PER_LONG < EVICTION_SHIFT);
/*
* Calculate the eviction bucket size to cover the longest
* actionable refault distance, which is currently half of
* memory (totalram_pages/2). However, memory hotplug may add
* some more pages at runtime, so keep working with up to
* double the initial memory by using totalram_pages as-is.
*/
timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT;
max_order = fls_long(totalram_pages() - 1);
if (max_order > timestamp_bits)
bucket_order = max_order - timestamp_bits;
pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
timestamp_bits, max_order, bucket_order);
workingset_shadow_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE |
SHRINKER_MEMCG_AWARE,
"mm-shadow");
if (!workingset_shadow_shrinker)
goto err;
ret = __list_lru_init(&shadow_nodes, true, &shadow_nodes_key,
workingset_shadow_shrinker);
if (ret)
goto err_list_lru;
workingset_shadow_shrinker->count_objects = count_shadow_nodes;
workingset_shadow_shrinker->scan_objects = scan_shadow_nodes;
/* ->count reports only fully expendable nodes */
workingset_shadow_shrinker->seeks = 0;
shrinker_register(workingset_shadow_shrinker);
return 0;
err_list_lru:
shrinker_free(workingset_shadow_shrinker);
err:
return ret;
}
module_init(workingset_init);
// SPDX-License-Identifier: GPL-2.0+
/*
* (C) Copyright Linus Torvalds 1999
* (C) Copyright Johannes Erdfelt 1999-2001
* (C) Copyright Andreas Gal 1999
* (C) Copyright Gregory P. Smith 1999
* (C) Copyright Deti Fliegl 1999
* (C) Copyright Randy Dunlap 2000
* (C) Copyright David Brownell 2000-2002
*/
#include <linux/bcd.h>
#include <linux/module.h>
#include <linux/version.h>
#include <linux/kernel.h>
#include <linux/sched/task_stack.h>
#include <linux/slab.h>
#include <linux/completion.h>
#include <linux/utsname.h>
#include <linux/mm.h>
#include <asm/io.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/mutex.h>
#include <asm/irq.h>
#include <asm/byteorder.h>
#include <asm/unaligned.h>
#include <linux/platform_device.h>
#include <linux/workqueue.h>
#include <linux/pm_runtime.h>
#include <linux/types.h>
#include <linux/genalloc.h>
#include <linux/io.h>
#include <linux/kcov.h>
#include <linux/phy/phy.h>
#include <linux/usb.h>
#include <linux/usb/hcd.h>
#include <linux/usb/otg.h>
#include "usb.h"
#include "phy.h"
/*-------------------------------------------------------------------------*/
/*
* USB Host Controller Driver framework
*
* Plugs into usbcore (usb_bus) and lets HCDs share code, minimizing
* HCD-specific behaviors/bugs.
*
* This does error checks, tracks devices and urbs, and delegates to a
* "hc_driver" only for code (and data) that really needs to know about
* hardware differences. That includes root hub registers, i/o queues,
* and so on ... but as little else as possible.
*
* Shared code includes most of the "root hub" code (these are emulated,
* though each HC's hardware works differently) and PCI glue, plus request
* tracking overhead. The HCD code should only block on spinlocks or on
* hardware handshaking; blocking on software events (such as other kernel
* threads releasing resources, or completing actions) is all generic.
*
* Happens the USB 2.0 spec says this would be invisible inside the "USBD",
* and includes mostly a "HCDI" (HCD Interface) along with some APIs used
* only by the hub driver ... and that neither should be seen or used by
* usb client device drivers.
*
* Contributors of ideas or unattributed patches include: David Brownell,
* Roman Weissgaerber, Rory Bolt, Greg Kroah-Hartman, ...
*
* HISTORY:
* 2002-02-21 Pull in most of the usb_bus support from usb.c; some
* associated cleanup. "usb_hcd" still != "usb_bus".
* 2001-12-12 Initial patch version for Linux 2.5.1 kernel.
*/
/*-------------------------------------------------------------------------*/
/* Keep track of which host controller drivers are loaded */
unsigned long usb_hcds_loaded;
EXPORT_SYMBOL_GPL(usb_hcds_loaded);
/* host controllers we manage */
DEFINE_IDR (usb_bus_idr);
EXPORT_SYMBOL_GPL (usb_bus_idr);
/* used when allocating bus numbers */
#define USB_MAXBUS 64
/* used when updating list of hcds */
DEFINE_MUTEX(usb_bus_idr_lock); /* exported only for usbfs */
EXPORT_SYMBOL_GPL (usb_bus_idr_lock);
/* used for controlling access to virtual root hubs */
static DEFINE_SPINLOCK(hcd_root_hub_lock);
/* used when updating an endpoint's URB list */
static DEFINE_SPINLOCK(hcd_urb_list_lock);
/* used to protect against unlinking URBs after the device is gone */
static DEFINE_SPINLOCK(hcd_urb_unlink_lock);
/* wait queue for synchronous unlinks */
DECLARE_WAIT_QUEUE_HEAD(usb_kill_urb_queue);
/*-------------------------------------------------------------------------*/
/*
* Sharable chunks of root hub code.
*/
/*-------------------------------------------------------------------------*/
#define KERNEL_REL bin2bcd(LINUX_VERSION_MAJOR)
#define KERNEL_VER bin2bcd(LINUX_VERSION_PATCHLEVEL)
/* usb 3.1 root hub device descriptor */
static const u8 usb31_rh_dev_descriptor[18] = {
0x12, /* __u8 bLength; */
USB_DT_DEVICE, /* __u8 bDescriptorType; Device */
0x10, 0x03, /* __le16 bcdUSB; v3.1 */
0x09, /* __u8 bDeviceClass; HUB_CLASSCODE */
0x00, /* __u8 bDeviceSubClass; */
0x03, /* __u8 bDeviceProtocol; USB 3 hub */
0x09, /* __u8 bMaxPacketSize0; 2^9 = 512 Bytes */
0x6b, 0x1d, /* __le16 idVendor; Linux Foundation 0x1d6b */
0x03, 0x00, /* __le16 idProduct; device 0x0003 */
KERNEL_VER, KERNEL_REL, /* __le16 bcdDevice */
0x03, /* __u8 iManufacturer; */
0x02, /* __u8 iProduct; */
0x01, /* __u8 iSerialNumber; */
0x01 /* __u8 bNumConfigurations; */
};
/* usb 3.0 root hub device descriptor */
static const u8 usb3_rh_dev_descriptor[18] = {
0x12, /* __u8 bLength; */
USB_DT_DEVICE, /* __u8 bDescriptorType; Device */
0x00, 0x03, /* __le16 bcdUSB; v3.0 */
0x09, /* __u8 bDeviceClass; HUB_CLASSCODE */
0x00, /* __u8 bDeviceSubClass; */
0x03, /* __u8 bDeviceProtocol; USB 3.0 hub */
0x09, /* __u8 bMaxPacketSize0; 2^9 = 512 Bytes */
0x6b, 0x1d, /* __le16 idVendor; Linux Foundation 0x1d6b */
0x03, 0x00, /* __le16 idProduct; device 0x0003 */
KERNEL_VER, KERNEL_REL, /* __le16 bcdDevice */
0x03, /* __u8 iManufacturer; */
0x02, /* __u8 iProduct; */
0x01, /* __u8 iSerialNumber; */
0x01 /* __u8 bNumConfigurations; */
};
/* usb 2.0 root hub device descriptor */
static const u8 usb2_rh_dev_descriptor[18] = {
0x12, /* __u8 bLength; */
USB_DT_DEVICE, /* __u8 bDescriptorType; Device */
0x00, 0x02, /* __le16 bcdUSB; v2.0 */
0x09, /* __u8 bDeviceClass; HUB_CLASSCODE */
0x00, /* __u8 bDeviceSubClass; */
0x00, /* __u8 bDeviceProtocol; [ usb 2.0 no TT ] */
0x40, /* __u8 bMaxPacketSize0; 64 Bytes */
0x6b, 0x1d, /* __le16 idVendor; Linux Foundation 0x1d6b */
0x02, 0x00, /* __le16 idProduct; device 0x0002 */
KERNEL_VER, KERNEL_REL, /* __le16 bcdDevice */
0x03, /* __u8 iManufacturer; */
0x02, /* __u8 iProduct; */
0x01, /* __u8 iSerialNumber; */
0x01 /* __u8 bNumConfigurations; */
};
/* no usb 2.0 root hub "device qualifier" descriptor: one speed only */
/* usb 1.1 root hub device descriptor */
static const u8 usb11_rh_dev_descriptor[18] = {
0x12, /* __u8 bLength; */
USB_DT_DEVICE, /* __u8 bDescriptorType; Device */
0x10, 0x01, /* __le16 bcdUSB; v1.1 */
0x09, /* __u8 bDeviceClass; HUB_CLASSCODE */
0x00, /* __u8 bDeviceSubClass; */
0x00, /* __u8 bDeviceProtocol; [ low/full speeds only ] */
0x40, /* __u8 bMaxPacketSize0; 64 Bytes */
0x6b, 0x1d, /* __le16 idVendor; Linux Foundation 0x1d6b */
0x01, 0x00, /* __le16 idProduct; device 0x0001 */
KERNEL_VER, KERNEL_REL, /* __le16 bcdDevice */
0x03, /* __u8 iManufacturer; */
0x02, /* __u8 iProduct; */
0x01, /* __u8 iSerialNumber; */
0x01 /* __u8 bNumConfigurations; */
};
/*-------------------------------------------------------------------------*/
/* Configuration descriptors for our root hubs */
static const u8 fs_rh_config_descriptor[] = {
/* one configuration */
0x09, /* __u8 bLength; */
USB_DT_CONFIG, /* __u8 bDescriptorType; Configuration */
0x19, 0x00, /* __le16 wTotalLength; */
0x01, /* __u8 bNumInterfaces; (1) */
0x01, /* __u8 bConfigurationValue; */
0x00, /* __u8 iConfiguration; */
0xc0, /* __u8 bmAttributes;
Bit 7: must be set,
6: Self-powered,
5: Remote wakeup,
4..0: resvd */
0x00, /* __u8 MaxPower; */
/* USB 1.1:
* USB 2.0, single TT organization (mandatory):
* one interface, protocol 0
*
* USB 2.0, multiple TT organization (optional):
* two interfaces, protocols 1 (like single TT)
* and 2 (multiple TT mode) ... config is
* sometimes settable
* NOT IMPLEMENTED
*/
/* one interface */
0x09, /* __u8 if_bLength; */
USB_DT_INTERFACE, /* __u8 if_bDescriptorType; Interface */
0x00, /* __u8 if_bInterfaceNumber; */
0x00, /* __u8 if_bAlternateSetting; */
0x01, /* __u8 if_bNumEndpoints; */
0x09, /* __u8 if_bInterfaceClass; HUB_CLASSCODE */
0x00, /* __u8 if_bInterfaceSubClass; */
0x00, /* __u8 if_bInterfaceProtocol; [usb1.1 or single tt] */
0x00, /* __u8 if_iInterface; */
/* one endpoint (status change endpoint) */
0x07, /* __u8 ep_bLength; */
USB_DT_ENDPOINT, /* __u8 ep_bDescriptorType; Endpoint */
0x81, /* __u8 ep_bEndpointAddress; IN Endpoint 1 */
0x03, /* __u8 ep_bmAttributes; Interrupt */
0x02, 0x00, /* __le16 ep_wMaxPacketSize; 1 + (MAX_ROOT_PORTS / 8) */
0xff /* __u8 ep_bInterval; (255ms -- usb 2.0 spec) */
};
static const u8 hs_rh_config_descriptor[] = {
/* one configuration */
0x09, /* __u8 bLength; */
USB_DT_CONFIG, /* __u8 bDescriptorType; Configuration */
0x19, 0x00, /* __le16 wTotalLength; */
0x01, /* __u8 bNumInterfaces; (1) */
0x01, /* __u8 bConfigurationValue; */
0x00, /* __u8 iConfiguration; */
0xc0, /* __u8 bmAttributes;
Bit 7: must be set,
6: Self-powered,
5: Remote wakeup,
4..0: resvd */
0x00, /* __u8 MaxPower; */
/* USB 1.1:
* USB 2.0, single TT organization (mandatory):
* one interface, protocol 0
*
* USB 2.0, multiple TT organization (optional):
* two interfaces, protocols 1 (like single TT)
* and 2 (multiple TT mode) ... config is
* sometimes settable
* NOT IMPLEMENTED
*/
/* one interface */
0x09, /* __u8 if_bLength; */
USB_DT_INTERFACE, /* __u8 if_bDescriptorType; Interface */
0x00, /* __u8 if_bInterfaceNumber; */
0x00, /* __u8 if_bAlternateSetting; */
0x01, /* __u8 if_bNumEndpoints; */
0x09, /* __u8 if_bInterfaceClass; HUB_CLASSCODE */
0x00, /* __u8 if_bInterfaceSubClass; */
0x00, /* __u8 if_bInterfaceProtocol; [usb1.1 or single tt] */
0x00, /* __u8 if_iInterface; */
/* one endpoint (status change endpoint) */
0x07, /* __u8 ep_bLength; */
USB_DT_ENDPOINT, /* __u8 ep_bDescriptorType; Endpoint */
0x81, /* __u8 ep_bEndpointAddress; IN Endpoint 1 */
0x03, /* __u8 ep_bmAttributes; Interrupt */
/* __le16 ep_wMaxPacketSize; 1 + (MAX_ROOT_PORTS / 8)
* see hub.c:hub_configure() for details. */
(USB_MAXCHILDREN + 1 + 7) / 8, 0x00,
0x0c /* __u8 ep_bInterval; (256ms -- usb 2.0 spec) */
};
static const u8 ss_rh_config_descriptor[] = {
/* one configuration */
0x09, /* __u8 bLength; */
USB_DT_CONFIG, /* __u8 bDescriptorType; Configuration */
0x1f, 0x00, /* __le16 wTotalLength; */
0x01, /* __u8 bNumInterfaces; (1) */
0x01, /* __u8 bConfigurationValue; */
0x00, /* __u8 iConfiguration; */
0xc0, /* __u8 bmAttributes;
Bit 7: must be set,
6: Self-powered,
5: Remote wakeup,
4..0: resvd */
0x00, /* __u8 MaxPower; */
/* one interface */
0x09, /* __u8 if_bLength; */
USB_DT_INTERFACE, /* __u8 if_bDescriptorType; Interface */
0x00, /* __u8 if_bInterfaceNumber; */
0x00, /* __u8 if_bAlternateSetting; */
0x01, /* __u8 if_bNumEndpoints; */
0x09, /* __u8 if_bInterfaceClass; HUB_CLASSCODE */
0x00, /* __u8 if_bInterfaceSubClass; */
0x00, /* __u8 if_bInterfaceProtocol; */
0x00, /* __u8 if_iInterface; */
/* one endpoint (status change endpoint) */
0x07, /* __u8 ep_bLength; */
USB_DT_ENDPOINT, /* __u8 ep_bDescriptorType; Endpoint */
0x81, /* __u8 ep_bEndpointAddress; IN Endpoint 1 */
0x03, /* __u8 ep_bmAttributes; Interrupt */
/* __le16 ep_wMaxPacketSize; 1 + (MAX_ROOT_PORTS / 8)
* see hub.c:hub_configure() for details. */
(USB_MAXCHILDREN + 1 + 7) / 8, 0x00,
0x0c, /* __u8 ep_bInterval; (256ms -- usb 2.0 spec) */
/* one SuperSpeed endpoint companion descriptor */
0x06, /* __u8 ss_bLength */
USB_DT_SS_ENDPOINT_COMP, /* __u8 ss_bDescriptorType; SuperSpeed EP */
/* Companion */
0x00, /* __u8 ss_bMaxBurst; allows 1 TX between ACKs */
0x00, /* __u8 ss_bmAttributes; 1 packet per service interval */
0x02, 0x00 /* __le16 ss_wBytesPerInterval; 15 bits for max 15 ports */
};
/* authorized_default behaviour:
* -1 is authorized for all devices (leftover from wireless USB)
* 0 is unauthorized for all devices
* 1 is authorized for all devices
* 2 is authorized for internal devices
*/
#define USB_AUTHORIZE_WIRED -1
#define USB_AUTHORIZE_NONE 0
#define USB_AUTHORIZE_ALL 1
#define USB_AUTHORIZE_INTERNAL 2
static int authorized_default = CONFIG_USB_DEFAULT_AUTHORIZATION_MODE;
module_param(authorized_default, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(authorized_default,
"Default USB device authorization: 0 is not authorized, 1 is authorized (default), 2 is authorized for internal devices, -1 is authorized (same as 1)");
/*-------------------------------------------------------------------------*/
/**
* ascii2desc() - Helper routine for producing UTF-16LE string descriptors
* @s: Null-terminated ASCII (actually ISO-8859-1) string
* @buf: Buffer for USB string descriptor (header + UTF-16LE)
* @len: Length (in bytes; may be odd) of descriptor buffer.
*
* Return: The number of bytes filled in: 2 + 2*strlen(s) or @len,
* whichever is less.
*
* Note:
* USB String descriptors can contain at most 126 characters; input
* strings longer than that are truncated.
*/
static unsigned
ascii2desc(char const *s, u8 *buf, unsigned len)
{
unsigned n, t = 2 + 2*strlen(s);
if (t > 254)
t = 254; /* Longest possible UTF string descriptor */
if (len > t)
len = t;
t += USB_DT_STRING << 8; /* Now t is first 16 bits to store */
n = len;
while (n--) { *buf++ = t;
if (!n--)
break;
*buf++ = t >> 8;
t = (unsigned char)*s++;
}
return len;
}
/**
* rh_string() - provides string descriptors for root hub
* @id: the string ID number (0: langids, 1: serial #, 2: product, 3: vendor)
* @hcd: the host controller for this root hub
* @data: buffer for output packet
* @len: length of the provided buffer
*
* Produces either a manufacturer, product or serial number string for the
* virtual root hub device.
*
* Return: The number of bytes filled in: the length of the descriptor or
* of the provided buffer, whichever is less.
*/
static unsigned
rh_string(int id, struct usb_hcd const *hcd, u8 *data, unsigned len)
{
char buf[100];
char const *s;
static char const langids[4] = {4, USB_DT_STRING, 0x09, 0x04};
/* language ids */
switch (id) {
case 0:
/* Array of LANGID codes (0x0409 is MSFT-speak for "en-us") */
/* See http://www.usb.org/developers/docs/USB_LANGIDs.pdf */
if (len > 4)
len = 4;
memcpy(data, langids, len);
return len;
case 1:
/* Serial number */
s = hcd->self.bus_name;
break;
case 2:
/* Product name */
s = hcd->product_desc;
break;
case 3:
/* Manufacturer */
snprintf (buf, sizeof buf, "%s %s %s", init_utsname()->sysname,
init_utsname()->release, hcd->driver->description);
s = buf;
break;
default:
/* Can't happen; caller guarantees it */
return 0;
}
return ascii2desc(s, data, len);
}
/* Root hub control transfers execute synchronously */
static int rh_call_control (struct usb_hcd *hcd, struct urb *urb)
{
struct usb_ctrlrequest *cmd;
u16 typeReq, wValue, wIndex, wLength;
u8 *ubuf = urb->transfer_buffer;
unsigned len = 0;
int status;
u8 patch_wakeup = 0;
u8 patch_protocol = 0;
u16 tbuf_size;
u8 *tbuf = NULL;
const u8 *bufp;
might_sleep();
spin_lock_irq(&hcd_root_hub_lock);
status = usb_hcd_link_urb_to_ep(hcd, urb);
spin_unlock_irq(&hcd_root_hub_lock);
if (status)
return status;
urb->hcpriv = hcd; /* Indicate it's queued */
cmd = (struct usb_ctrlrequest *) urb->setup_packet;
typeReq = (cmd->bRequestType << 8) | cmd->bRequest;
wValue = le16_to_cpu (cmd->wValue);
wIndex = le16_to_cpu (cmd->wIndex);
wLength = le16_to_cpu (cmd->wLength);
if (wLength > urb->transfer_buffer_length)
goto error;
/*
* tbuf should be at least as big as the
* USB hub descriptor.
*/
tbuf_size = max_t(u16, sizeof(struct usb_hub_descriptor), wLength);
tbuf = kzalloc(tbuf_size, GFP_KERNEL);
if (!tbuf) {
status = -ENOMEM;
goto err_alloc;
}
bufp = tbuf;
urb->actual_length = 0;
switch (typeReq) {
/* DEVICE REQUESTS */
/* The root hub's remote wakeup enable bit is implemented using
* driver model wakeup flags. If this system supports wakeup
* through USB, userspace may change the default "allow wakeup"
* policy through sysfs or these calls.
*
* Most root hubs support wakeup from downstream devices, for
* runtime power management (disabling USB clocks and reducing
* VBUS power usage). However, not all of them do so; silicon,
* board, and BIOS bugs here are not uncommon, so these can't
* be treated quite like external hubs.
*
* Likewise, not all root hubs will pass wakeup events upstream,
* to wake up the whole system. So don't assume root hub and
* controller capabilities are identical.
*/
case DeviceRequest | USB_REQ_GET_STATUS:
tbuf[0] = (device_may_wakeup(&hcd->self.root_hub->dev)
<< USB_DEVICE_REMOTE_WAKEUP)
| (1 << USB_DEVICE_SELF_POWERED);
tbuf[1] = 0;
len = 2;
break;
case DeviceOutRequest | USB_REQ_CLEAR_FEATURE:
if (wValue == USB_DEVICE_REMOTE_WAKEUP) device_set_wakeup_enable(&hcd->self.root_hub->dev, 0);
else
goto error;
break;
case DeviceOutRequest | USB_REQ_SET_FEATURE:
if (device_can_wakeup(&hcd->self.root_hub->dev) && wValue == USB_DEVICE_REMOTE_WAKEUP) device_set_wakeup_enable(&hcd->self.root_hub->dev, 1);
else
goto error;
break;
case DeviceRequest | USB_REQ_GET_CONFIGURATION:
tbuf[0] = 1;
len = 1;
fallthrough;
case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
break;
case DeviceRequest | USB_REQ_GET_DESCRIPTOR:
switch (wValue & 0xff00) {
case USB_DT_DEVICE << 8:
switch (hcd->speed) {
case HCD_USB32:
case HCD_USB31:
bufp = usb31_rh_dev_descriptor;
break;
case HCD_USB3:
bufp = usb3_rh_dev_descriptor;
break;
case HCD_USB2:
bufp = usb2_rh_dev_descriptor;
break;
case HCD_USB11:
bufp = usb11_rh_dev_descriptor;
break;
default:
goto error;
}
len = 18;
if (hcd->has_tt)
patch_protocol = 1;
break;
case USB_DT_CONFIG << 8:
switch (hcd->speed) {
case HCD_USB32:
case HCD_USB31:
case HCD_USB3:
bufp = ss_rh_config_descriptor;
len = sizeof ss_rh_config_descriptor;
break;
case HCD_USB2:
bufp = hs_rh_config_descriptor;
len = sizeof hs_rh_config_descriptor;
break;
case HCD_USB11:
bufp = fs_rh_config_descriptor;
len = sizeof fs_rh_config_descriptor;
break;
default:
goto error;
}
if (device_can_wakeup(&hcd->self.root_hub->dev))
patch_wakeup = 1;
break;
case USB_DT_STRING << 8:
if ((wValue & 0xff) < 4) urb->actual_length = rh_string(wValue & 0xff,
hcd, ubuf, wLength);
else /* unsupported IDs --> "protocol stall" */
goto error;
break;
case USB_DT_BOS << 8:
goto nongeneric;
default:
goto error;
}
break;
case DeviceRequest | USB_REQ_GET_INTERFACE:
tbuf[0] = 0;
len = 1;
fallthrough;
case DeviceOutRequest | USB_REQ_SET_INTERFACE:
break;
case DeviceOutRequest | USB_REQ_SET_ADDRESS:
/* wValue == urb->dev->devaddr */
dev_dbg (hcd->self.controller, "root hub device address %d\n",
wValue);
break;
/* INTERFACE REQUESTS (no defined feature/status flags) */
/* ENDPOINT REQUESTS */
case EndpointRequest | USB_REQ_GET_STATUS:
/* ENDPOINT_HALT flag */
tbuf[0] = 0;
tbuf[1] = 0;
len = 2;
fallthrough;
case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
case EndpointOutRequest | USB_REQ_SET_FEATURE:
dev_dbg (hcd->self.controller, "no endpoint features yet\n");
break;
/* CLASS REQUESTS (and errors) */
default:
nongeneric:
/* non-generic request */
switch (typeReq) {
case GetHubStatus:
len = 4;
break;
case GetPortStatus:
if (wValue == HUB_PORT_STATUS)
len = 4;
else
/* other port status types return 8 bytes */
len = 8;
break;
case GetHubDescriptor:
len = sizeof (struct usb_hub_descriptor);
break;
case DeviceRequest | USB_REQ_GET_DESCRIPTOR:
/* len is returned by hub_control */
break;
}
status = hcd->driver->hub_control (hcd,
typeReq, wValue, wIndex,
tbuf, wLength);
if (typeReq == GetHubDescriptor)
usb_hub_adjust_deviceremovable(hcd->self.root_hub,
(struct usb_hub_descriptor *)tbuf);
break;
error:
/* "protocol stall" on error */
status = -EPIPE;
}
if (status < 0) {
len = 0;
if (status != -EPIPE) { dev_dbg (hcd->self.controller,
"CTRL: TypeReq=0x%x val=0x%x "
"idx=0x%x len=%d ==> %d\n",
typeReq, wValue, wIndex,
wLength, status);
}
} else if (status > 0) {
/* hub_control may return the length of data copied. */
len = status;
status = 0;
}
if (len) { if (urb->transfer_buffer_length < len)
len = urb->transfer_buffer_length;
urb->actual_length = len;
/* always USB_DIR_IN, toward host */
memcpy (ubuf, bufp, len);
/* report whether RH hardware supports remote wakeup */
if (patch_wakeup &&
len > offsetof (struct usb_config_descriptor,
bmAttributes))
((struct usb_config_descriptor *)ubuf)->bmAttributes
|= USB_CONFIG_ATT_WAKEUP;
/* report whether RH hardware has an integrated TT */
if (patch_protocol &&
len > offsetof(struct usb_device_descriptor,
bDeviceProtocol))
((struct usb_device_descriptor *) ubuf)->
bDeviceProtocol = USB_HUB_PR_HS_SINGLE_TT;
}
kfree(tbuf);
err_alloc:
/* any errors get returned through the urb completion */
spin_lock_irq(&hcd_root_hub_lock); usb_hcd_unlink_urb_from_ep(hcd, urb);
usb_hcd_giveback_urb(hcd, urb, status);
spin_unlock_irq(&hcd_root_hub_lock);
return 0;
}
/*-------------------------------------------------------------------------*/
/*
* Root Hub interrupt transfers are polled using a timer if the
* driver requests it; otherwise the driver is responsible for
* calling usb_hcd_poll_rh_status() when an event occurs.
*
* Completion handler may not sleep. See usb_hcd_giveback_urb() for details.
*/
void usb_hcd_poll_rh_status(struct usb_hcd *hcd)
{
struct urb *urb;
int length;
int status;
unsigned long flags;
char buffer[6]; /* Any root hubs with > 31 ports? */
if (unlikely(!hcd->rh_pollable))
return;
if (!hcd->uses_new_polling && !hcd->status_urb)
return;
length = hcd->driver->hub_status_data(hcd, buffer);
if (length > 0) {
/* try to complete the status urb */
spin_lock_irqsave(&hcd_root_hub_lock, flags);
urb = hcd->status_urb;
if (urb) {
clear_bit(HCD_FLAG_POLL_PENDING, &hcd->flags);
hcd->status_urb = NULL;
if (urb->transfer_buffer_length >= length) {
status = 0;
} else {
status = -EOVERFLOW;
length = urb->transfer_buffer_length;
}
urb->actual_length = length; memcpy(urb->transfer_buffer, buffer, length); usb_hcd_unlink_urb_from_ep(hcd, urb);
usb_hcd_giveback_urb(hcd, urb, status);
} else {
length = 0;
set_bit(HCD_FLAG_POLL_PENDING, &hcd->flags);
}
spin_unlock_irqrestore(&hcd_root_hub_lock, flags);
}
/* The USB 2.0 spec says 256 ms. This is close enough and won't
* exceed that limit if HZ is 100. The math is more clunky than
* maybe expected, this is to make sure that all timers for USB devices
* fire at the same time to give the CPU a break in between */
if (hcd->uses_new_polling ? HCD_POLL_RH(hcd) : (length == 0 && hcd->status_urb != NULL)) mod_timer (&hcd->rh_timer, (jiffies/(HZ/4) + 1) * (HZ/4));
}
EXPORT_SYMBOL_GPL(usb_hcd_poll_rh_status);
/* timer callback */
static void rh_timer_func (struct timer_list *t)
{
struct usb_hcd *_hcd = from_timer(_hcd, t, rh_timer);
usb_hcd_poll_rh_status(_hcd);
}
/*-------------------------------------------------------------------------*/
static int rh_queue_status (struct usb_hcd *hcd, struct urb *urb)
{
int retval;
unsigned long flags;
unsigned len = 1 + (urb->dev->maxchild / 8);
spin_lock_irqsave (&hcd_root_hub_lock, flags);
if (hcd->status_urb || urb->transfer_buffer_length < len) { dev_dbg (hcd->self.controller, "not queuing rh status urb\n");
retval = -EINVAL;
goto done;
}
retval = usb_hcd_link_urb_to_ep(hcd, urb);
if (retval)
goto done;
hcd->status_urb = urb;
urb->hcpriv = hcd; /* indicate it's queued */
if (!hcd->uses_new_polling)
mod_timer(&hcd->rh_timer, (jiffies/(HZ/4) + 1) * (HZ/4));
/* If a status change has already occurred, report it ASAP */
else if (HCD_POLL_PENDING(hcd)) mod_timer(&hcd->rh_timer, jiffies);
retval = 0;
done:
spin_unlock_irqrestore (&hcd_root_hub_lock, flags);
return retval;
}
static int rh_urb_enqueue (struct usb_hcd *hcd, struct urb *urb)
{
if (usb_endpoint_xfer_int(&urb->ep->desc)) return rh_queue_status (hcd, urb); if (usb_endpoint_xfer_control(&urb->ep->desc)) return rh_call_control (hcd, urb);
return -EINVAL;
}
/*-------------------------------------------------------------------------*/
/* Unlinks of root-hub control URBs are legal, but they don't do anything
* since these URBs always execute synchronously.
*/
static int usb_rh_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
{
unsigned long flags;
int rc;
spin_lock_irqsave(&hcd_root_hub_lock, flags); rc = usb_hcd_check_unlink_urb(hcd, urb, status);
if (rc)
goto done;
if (usb_endpoint_num(&urb->ep->desc) == 0) { /* Control URB */
; /* Do nothing */
} else { /* Status URB */
if (!hcd->uses_new_polling) del_timer (&hcd->rh_timer); if (urb == hcd->status_urb) { hcd->status_urb = NULL; usb_hcd_unlink_urb_from_ep(hcd, urb);
usb_hcd_giveback_urb(hcd, urb, status);
}
}
done:
spin_unlock_irqrestore(&hcd_root_hub_lock, flags);
return rc;
}
/*-------------------------------------------------------------------------*/
/**
* usb_bus_init - shared initialization code
* @bus: the bus structure being initialized
*
* This code is used to initialize a usb_bus structure, memory for which is
* separately managed.
*/
static void usb_bus_init (struct usb_bus *bus)
{
memset(&bus->devmap, 0, sizeof(bus->devmap));
bus->devnum_next = 1;
bus->root_hub = NULL;
bus->busnum = -1;
bus->bandwidth_allocated = 0;
bus->bandwidth_int_reqs = 0;
bus->bandwidth_isoc_reqs = 0;
mutex_init(&bus->devnum_next_mutex);
}
/*-------------------------------------------------------------------------*/
/**
* usb_register_bus - registers the USB host controller with the usb core
* @bus: pointer to the bus to register
*
* Context: task context, might sleep.
*
* Assigns a bus number, and links the controller into usbcore data
* structures so that it can be seen by scanning the bus list.
*
* Return: 0 if successful. A negative error code otherwise.
*/
static int usb_register_bus(struct usb_bus *bus)
{
int result = -E2BIG;
int busnum;
mutex_lock(&usb_bus_idr_lock);
busnum = idr_alloc(&usb_bus_idr, bus, 1, USB_MAXBUS, GFP_KERNEL);
if (busnum < 0) {
pr_err("%s: failed to get bus number\n", usbcore_name);
goto error_find_busnum;
}
bus->busnum = busnum;
mutex_unlock(&usb_bus_idr_lock);
usb_notify_add_bus(bus);
dev_info (bus->controller, "new USB bus registered, assigned bus "
"number %d\n", bus->busnum);
return 0;
error_find_busnum:
mutex_unlock(&usb_bus_idr_lock);
return result;
}
/**
* usb_deregister_bus - deregisters the USB host controller
* @bus: pointer to the bus to deregister
*
* Context: task context, might sleep.
*
* Recycles the bus number, and unlinks the controller from usbcore data
* structures so that it won't be seen by scanning the bus list.
*/
static void usb_deregister_bus (struct usb_bus *bus)
{
dev_info (bus->controller, "USB bus %d deregistered\n", bus->busnum);
/*
* NOTE: make sure that all the devices are removed by the
* controller code, as well as having it call this when cleaning
* itself up
*/
mutex_lock(&usb_bus_idr_lock);
idr_remove(&usb_bus_idr, bus->busnum);
mutex_unlock(&usb_bus_idr_lock);
usb_notify_remove_bus(bus);
}
/**
* register_root_hub - called by usb_add_hcd() to register a root hub
* @hcd: host controller for this root hub
*
* This function registers the root hub with the USB subsystem. It sets up
* the device properly in the device tree and then calls usb_new_device()
* to register the usb device. It also assigns the root hub's USB address
* (always 1).
*
* Return: 0 if successful. A negative error code otherwise.
*/
static int register_root_hub(struct usb_hcd *hcd)
{
struct device *parent_dev = hcd->self.controller;
struct usb_device *usb_dev = hcd->self.root_hub;
struct usb_device_descriptor *descr;
const int devnum = 1;
int retval;
usb_dev->devnum = devnum;
usb_dev->bus->devnum_next = devnum + 1;
set_bit(devnum, usb_dev->bus->devmap);
usb_set_device_state(usb_dev, USB_STATE_ADDRESS);
mutex_lock(&usb_bus_idr_lock);
usb_dev->ep0.desc.wMaxPacketSize = cpu_to_le16(64);
descr = usb_get_device_descriptor(usb_dev);
if (IS_ERR(descr)) {
retval = PTR_ERR(descr);
mutex_unlock(&usb_bus_idr_lock);
dev_dbg (parent_dev, "can't read %s device descriptor %d\n",
dev_name(&usb_dev->dev), retval);
return retval;
}
usb_dev->descriptor = *descr;
kfree(descr);
if (le16_to_cpu(usb_dev->descriptor.bcdUSB) >= 0x0201) {
retval = usb_get_bos_descriptor(usb_dev);
if (!retval) {
usb_dev->lpm_capable = usb_device_supports_lpm(usb_dev);
} else if (usb_dev->speed >= USB_SPEED_SUPER) {
mutex_unlock(&usb_bus_idr_lock);
dev_dbg(parent_dev, "can't read %s bos descriptor %d\n",
dev_name(&usb_dev->dev), retval);
return retval;
}
}
retval = usb_new_device (usb_dev);
if (retval) {
dev_err (parent_dev, "can't register root hub for %s, %d\n",
dev_name(&usb_dev->dev), retval);
} else {
spin_lock_irq (&hcd_root_hub_lock);
hcd->rh_registered = 1;
spin_unlock_irq (&hcd_root_hub_lock);
/* Did the HC die before the root hub was registered? */
if (HCD_DEAD(hcd))
usb_hc_died (hcd); /* This time clean up */
}
mutex_unlock(&usb_bus_idr_lock);
return retval;
}
/*
* usb_hcd_start_port_resume - a root-hub port is sending a resume signal
* @bus: the bus which the root hub belongs to
* @portnum: the port which is being resumed
*
* HCDs should call this function when they know that a resume signal is
* being sent to a root-hub port. The root hub will be prevented from
* going into autosuspend until usb_hcd_end_port_resume() is called.
*
* The bus's private lock must be held by the caller.
*/
void usb_hcd_start_port_resume(struct usb_bus *bus, int portnum)
{
unsigned bit = 1 << portnum;
if (!(bus->resuming_ports & bit)) {
bus->resuming_ports |= bit;
pm_runtime_get_noresume(&bus->root_hub->dev);
}
}
EXPORT_SYMBOL_GPL(usb_hcd_start_port_resume);
/*
* usb_hcd_end_port_resume - a root-hub port has stopped sending a resume signal
* @bus: the bus which the root hub belongs to
* @portnum: the port which is being resumed
*
* HCDs should call this function when they know that a resume signal has
* stopped being sent to a root-hub port. The root hub will be allowed to
* autosuspend again.
*
* The bus's private lock must be held by the caller.
*/
void usb_hcd_end_port_resume(struct usb_bus *bus, int portnum)
{
unsigned bit = 1 << portnum;
if (bus->resuming_ports & bit) {
bus->resuming_ports &= ~bit;
pm_runtime_put_noidle(&bus->root_hub->dev);
}
}
EXPORT_SYMBOL_GPL(usb_hcd_end_port_resume);
/*-------------------------------------------------------------------------*/
/**
* usb_calc_bus_time - approximate periodic transaction time in nanoseconds
* @speed: from dev->speed; USB_SPEED_{LOW,FULL,HIGH}
* @is_input: true iff the transaction sends data to the host
* @isoc: true for isochronous transactions, false for interrupt ones
* @bytecount: how many bytes in the transaction.
*
* Return: Approximate bus time in nanoseconds for a periodic transaction.
*
* Note:
* See USB 2.0 spec section 5.11.3; only periodic transfers need to be
* scheduled in software, this function is only used for such scheduling.
*/
long usb_calc_bus_time (int speed, int is_input, int isoc, int bytecount)
{
unsigned long tmp;
switch (speed) {
case USB_SPEED_LOW: /* INTR only */
if (is_input) {
tmp = (67667L * (31L + 10L * BitTime (bytecount))) / 1000L;
return 64060L + (2 * BW_HUB_LS_SETUP) + BW_HOST_DELAY + tmp;
} else {
tmp = (66700L * (31L + 10L * BitTime (bytecount))) / 1000L;
return 64107L + (2 * BW_HUB_LS_SETUP) + BW_HOST_DELAY + tmp;
}
case USB_SPEED_FULL: /* ISOC or INTR */
if (isoc) {
tmp = (8354L * (31L + 10L * BitTime (bytecount))) / 1000L;
return ((is_input) ? 7268L : 6265L) + BW_HOST_DELAY + tmp;
} else {
tmp = (8354L * (31L + 10L * BitTime (bytecount))) / 1000L;
return 9107L + BW_HOST_DELAY + tmp;
}
case USB_SPEED_HIGH: /* ISOC or INTR */
/* FIXME adjust for input vs output */
if (isoc)
tmp = HS_NSECS_ISO (bytecount);
else
tmp = HS_NSECS (bytecount);
return tmp;
default:
pr_debug ("%s: bogus device speed!\n", usbcore_name);
return -1;
}
}
EXPORT_SYMBOL_GPL(usb_calc_bus_time);
/*-------------------------------------------------------------------------*/
/*
* Generic HC operations.
*/
/*-------------------------------------------------------------------------*/
/**
* usb_hcd_link_urb_to_ep - add an URB to its endpoint queue
* @hcd: host controller to which @urb was submitted
* @urb: URB being submitted
*
* Host controller drivers should call this routine in their enqueue()
* method. The HCD's private spinlock must be held and interrupts must
* be disabled. The actions carried out here are required for URB
* submission, as well as for endpoint shutdown and for usb_kill_urb.
*
* Return: 0 for no error, otherwise a negative error code (in which case
* the enqueue() method must fail). If no error occurs but enqueue() fails
* anyway, it must call usb_hcd_unlink_urb_from_ep() before releasing
* the private spinlock and returning.
*/
int usb_hcd_link_urb_to_ep(struct usb_hcd *hcd, struct urb *urb)
{
int rc = 0;
spin_lock(&hcd_urb_list_lock);
/* Check that the URB isn't being killed */
if (unlikely(atomic_read(&urb->reject))) {
rc = -EPERM;
goto done;
}
if (unlikely(!urb->ep->enabled)) {
rc = -ENOENT;
goto done;
}
if (unlikely(!urb->dev->can_submit)) {
rc = -EHOSTUNREACH;
goto done;
}
/*
* Check the host controller's state and add the URB to the
* endpoint's queue.
*/
if (HCD_RH_RUNNING(hcd)) { urb->unlinked = 0; list_add_tail(&urb->urb_list, &urb->ep->urb_list);
} else {
rc = -ESHUTDOWN;
goto done;
}
done:
spin_unlock(&hcd_urb_list_lock);
return rc;
}
EXPORT_SYMBOL_GPL(usb_hcd_link_urb_to_ep);
/**
* usb_hcd_check_unlink_urb - check whether an URB may be unlinked
* @hcd: host controller to which @urb was submitted
* @urb: URB being checked for unlinkability
* @status: error code to store in @urb if the unlink succeeds
*
* Host controller drivers should call this routine in their dequeue()
* method. The HCD's private spinlock must be held and interrupts must
* be disabled. The actions carried out here are required for making
* sure than an unlink is valid.
*
* Return: 0 for no error, otherwise a negative error code (in which case
* the dequeue() method must fail). The possible error codes are:
*
* -EIDRM: @urb was not submitted or has already completed.
* The completion function may not have been called yet.
*
* -EBUSY: @urb has already been unlinked.
*/
int usb_hcd_check_unlink_urb(struct usb_hcd *hcd, struct urb *urb,
int status)
{
struct list_head *tmp;
/* insist the urb is still queued */
list_for_each(tmp, &urb->ep->urb_list) { if (tmp == &urb->urb_list)
break;
}
if (tmp != &urb->urb_list)
return -EIDRM;
/* Any status except -EINPROGRESS means something already started to
* unlink this URB from the hardware. So there's no more work to do.
*/
if (urb->unlinked)
return -EBUSY;
urb->unlinked = status; return 0;
}
EXPORT_SYMBOL_GPL(usb_hcd_check_unlink_urb);
/**
* usb_hcd_unlink_urb_from_ep - remove an URB from its endpoint queue
* @hcd: host controller to which @urb was submitted
* @urb: URB being unlinked
*
* Host controller drivers should call this routine before calling
* usb_hcd_giveback_urb(). The HCD's private spinlock must be held and
* interrupts must be disabled. The actions carried out here are required
* for URB completion.
*/
void usb_hcd_unlink_urb_from_ep(struct usb_hcd *hcd, struct urb *urb)
{
/* clear all state linking urb to this dev (and hcd) */
spin_lock(&hcd_urb_list_lock);
list_del_init(&urb->urb_list);
spin_unlock(&hcd_urb_list_lock);
}
EXPORT_SYMBOL_GPL(usb_hcd_unlink_urb_from_ep);
/*
* Some usb host controllers can only perform dma using a small SRAM area,
* or have restrictions on addressable DRAM.
* The usb core itself is however optimized for host controllers that can dma
* using regular system memory - like pci devices doing bus mastering.
*
* To support host controllers with limited dma capabilities we provide dma
* bounce buffers. This feature can be enabled by initializing
* hcd->localmem_pool using usb_hcd_setup_local_mem().
*
* The initialized hcd->localmem_pool then tells the usb code to allocate all
* data for dma using the genalloc API.
*
* So, to summarize...
*
* - We need "local" memory, canonical example being
* a small SRAM on a discrete controller being the
* only memory that the controller can read ...
* (a) "normal" kernel memory is no good, and
* (b) there's not enough to share
*
* - So we use that, even though the primary requirement
* is that the memory be "local" (hence addressable
* by that device), not "coherent".
*
*/
static int hcd_alloc_coherent(struct usb_bus *bus,
gfp_t mem_flags, dma_addr_t *dma_handle,
void **vaddr_handle, size_t size,
enum dma_data_direction dir)
{
unsigned char *vaddr;
if (*vaddr_handle == NULL) {
WARN_ON_ONCE(1);
return -EFAULT;
}
vaddr = hcd_buffer_alloc(bus, size + sizeof(unsigned long),
mem_flags, dma_handle);
if (!vaddr)
return -ENOMEM;
/*
* Store the virtual address of the buffer at the end
* of the allocated dma buffer. The size of the buffer
* may be uneven so use unaligned functions instead
* of just rounding up. It makes sense to optimize for
* memory footprint over access speed since the amount
* of memory available for dma may be limited.
*/
put_unaligned((unsigned long)*vaddr_handle,
(unsigned long *)(vaddr + size));
if (dir == DMA_TO_DEVICE)
memcpy(vaddr, *vaddr_handle, size);
*vaddr_handle = vaddr;
return 0;
}
static void hcd_free_coherent(struct usb_bus *bus, dma_addr_t *dma_handle,
void **vaddr_handle, size_t size,
enum dma_data_direction dir)
{
unsigned char *vaddr = *vaddr_handle;
vaddr = (void *)get_unaligned((unsigned long *)(vaddr + size));
if (dir == DMA_FROM_DEVICE)
memcpy(vaddr, *vaddr_handle, size); hcd_buffer_free(bus, size + sizeof(vaddr), *vaddr_handle, *dma_handle);
*vaddr_handle = vaddr;
*dma_handle = 0;
}
void usb_hcd_unmap_urb_setup_for_dma(struct usb_hcd *hcd, struct urb *urb)
{
if (IS_ENABLED(CONFIG_HAS_DMA) &&
(urb->transfer_flags & URB_SETUP_MAP_SINGLE)) dma_unmap_single(hcd->self.sysdev,
urb->setup_dma,
sizeof(struct usb_ctrlrequest),
DMA_TO_DEVICE);
else if (urb->transfer_flags & URB_SETUP_MAP_LOCAL) hcd_free_coherent(urb->dev->bus,
&urb->setup_dma,
(void **) &urb->setup_packet,
sizeof(struct usb_ctrlrequest),
DMA_TO_DEVICE);
/* Make it safe to call this routine more than once */
urb->transfer_flags &= ~(URB_SETUP_MAP_SINGLE | URB_SETUP_MAP_LOCAL);
}
EXPORT_SYMBOL_GPL(usb_hcd_unmap_urb_setup_for_dma);
static void unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
{
if (hcd->driver->unmap_urb_for_dma) hcd->driver->unmap_urb_for_dma(hcd, urb);
else
usb_hcd_unmap_urb_for_dma(hcd, urb);
}
void usb_hcd_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
{
enum dma_data_direction dir;
usb_hcd_unmap_urb_setup_for_dma(hcd, urb);
dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
if (IS_ENABLED(CONFIG_HAS_DMA) &&
(urb->transfer_flags & URB_DMA_MAP_SG)) dma_unmap_sg(hcd->self.sysdev,
urb->sg,
urb->num_sgs,
dir);
else if (IS_ENABLED(CONFIG_HAS_DMA) &&
(urb->transfer_flags & URB_DMA_MAP_PAGE)) dma_unmap_page(hcd->self.sysdev,
urb->transfer_dma,
urb->transfer_buffer_length,
dir);
else if (IS_ENABLED(CONFIG_HAS_DMA) &&
(urb->transfer_flags & URB_DMA_MAP_SINGLE)) dma_unmap_single(hcd->self.sysdev,
urb->transfer_dma,
urb->transfer_buffer_length,
dir);
else if (urb->transfer_flags & URB_MAP_LOCAL) hcd_free_coherent(urb->dev->bus,
&urb->transfer_dma,
&urb->transfer_buffer,
urb->transfer_buffer_length,
dir);
/* Make it safe to call this routine more than once */
urb->transfer_flags &= ~(URB_DMA_MAP_SG | URB_DMA_MAP_PAGE |
URB_DMA_MAP_SINGLE | URB_MAP_LOCAL);
}
EXPORT_SYMBOL_GPL(usb_hcd_unmap_urb_for_dma);
static int map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
gfp_t mem_flags)
{
if (hcd->driver->map_urb_for_dma) return hcd->driver->map_urb_for_dma(hcd, urb, mem_flags);
else
return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
}
int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
gfp_t mem_flags)
{
enum dma_data_direction dir;
int ret = 0;
/* Map the URB's buffers for DMA access.
* Lower level HCD code should use *_dma exclusively,
* unless it uses pio or talks to another transport,
* or uses the provided scatter gather list for bulk.
*/
if (usb_endpoint_xfer_control(&urb->ep->desc)) { if (hcd->self.uses_pio_for_control)
return ret;
if (hcd->localmem_pool) {
ret = hcd_alloc_coherent(
urb->dev->bus, mem_flags,
&urb->setup_dma,
(void **)&urb->setup_packet,
sizeof(struct usb_ctrlrequest),
DMA_TO_DEVICE);
if (ret)
return ret;
urb->transfer_flags |= URB_SETUP_MAP_LOCAL; } else if (hcd_uses_dma(hcd)) { if (object_is_on_stack(urb->setup_packet)) { WARN_ONCE(1, "setup packet is on stack\n");
return -EAGAIN;
}
urb->setup_dma = dma_map_single(
hcd->self.sysdev,
urb->setup_packet,
sizeof(struct usb_ctrlrequest),
DMA_TO_DEVICE);
if (dma_mapping_error(hcd->self.sysdev,
urb->setup_dma))
return -EAGAIN;
urb->transfer_flags |= URB_SETUP_MAP_SINGLE;
}
}
dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; if (urb->transfer_buffer_length != 0 && !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) { if (hcd->localmem_pool) { ret = hcd_alloc_coherent(
urb->dev->bus, mem_flags,
&urb->transfer_dma,
&urb->transfer_buffer,
urb->transfer_buffer_length,
dir);
if (ret == 0)
urb->transfer_flags |= URB_MAP_LOCAL; } else if (hcd_uses_dma(hcd)) { if (urb->num_sgs) {
int n;
/* We don't support sg for isoc transfers ! */
if (usb_endpoint_xfer_isoc(&urb->ep->desc)) { WARN_ON(1);
return -EINVAL;
}
n = dma_map_sg(
hcd->self.sysdev,
urb->sg,
urb->num_sgs,
dir);
if (!n)
ret = -EAGAIN;
else
urb->transfer_flags |= URB_DMA_MAP_SG; urb->num_mapped_sgs = n;
if (n != urb->num_sgs)
urb->transfer_flags |=
URB_DMA_SG_COMBINED;
} else if (urb->sg) {
struct scatterlist *sg = urb->sg;
urb->transfer_dma = dma_map_page(
hcd->self.sysdev,
sg_page(sg),
sg->offset,
urb->transfer_buffer_length,
dir);
if (dma_mapping_error(hcd->self.sysdev,
urb->transfer_dma))
ret = -EAGAIN;
else
urb->transfer_flags |= URB_DMA_MAP_PAGE; } else if (object_is_on_stack(urb->transfer_buffer)) { WARN_ONCE(1, "transfer buffer is on stack\n");
ret = -EAGAIN;
} else {
urb->transfer_dma = dma_map_single(
hcd->self.sysdev,
urb->transfer_buffer,
urb->transfer_buffer_length,
dir);
if (dma_mapping_error(hcd->self.sysdev,
urb->transfer_dma))
ret = -EAGAIN;
else
urb->transfer_flags |= URB_DMA_MAP_SINGLE;
}
}
if (ret && (urb->transfer_flags & (URB_SETUP_MAP_SINGLE |
URB_SETUP_MAP_LOCAL)))
usb_hcd_unmap_urb_for_dma(hcd, urb);
}
return ret;
}
EXPORT_SYMBOL_GPL(usb_hcd_map_urb_for_dma);
/*-------------------------------------------------------------------------*/
/* may be called in any context with a valid urb->dev usecount
* caller surrenders "ownership" of urb
* expects usb_submit_urb() to have sanity checked and conditioned all
* inputs in the urb
*/
int usb_hcd_submit_urb (struct urb *urb, gfp_t mem_flags)
{
int status;
struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus);
/* increment urb's reference count as part of giving it to the HCD
* (which will control it). HCD guarantees that it either returns
* an error or calls giveback(), but not both.
*/
usb_get_urb(urb);
atomic_inc(&urb->use_count);
atomic_inc(&urb->dev->urbnum);
usbmon_urb_submit(&hcd->self, urb);
/* NOTE requirements on root-hub callers (usbfs and the hub
* driver, for now): URBs' urb->transfer_buffer must be
* valid and usb_buffer_{sync,unmap}() not be needed, since
* they could clobber root hub response data. Also, control
* URBs must be submitted in process context with interrupts
* enabled.
*/
if (is_root_hub(urb->dev)) { status = rh_urb_enqueue(hcd, urb);
} else {
status = map_urb_for_dma(hcd, urb, mem_flags); if (likely(status == 0)) { status = hcd->driver->urb_enqueue(hcd, urb, mem_flags);
if (unlikely(status))
unmap_urb_for_dma(hcd, urb);
}
}
if (unlikely(status)) {
usbmon_urb_submit_error(&hcd->self, urb, status); urb->hcpriv = NULL;
INIT_LIST_HEAD(&urb->urb_list);
atomic_dec(&urb->use_count);
/*
* Order the write of urb->use_count above before the read
* of urb->reject below. Pairs with the memory barriers in
* usb_kill_urb() and usb_poison_urb().
*/
smp_mb__after_atomic();
atomic_dec(&urb->dev->urbnum);
if (atomic_read(&urb->reject))
wake_up(&usb_kill_urb_queue); usb_put_urb(urb);
}
return status;
}
/*-------------------------------------------------------------------------*/
/* this makes the hcd giveback() the urb more quickly, by kicking it
* off hardware queues (which may take a while) and returning it as
* soon as practical. we've already set up the urb's return status,
* but we can't know if the callback completed already.
*/
static int unlink1(struct usb_hcd *hcd, struct urb *urb, int status)
{
int value;
if (is_root_hub(urb->dev)) value = usb_rh_urb_dequeue(hcd, urb, status);
else {
/* The only reason an HCD might fail this call is if
* it has not yet fully queued the urb to begin with.
* Such failures should be harmless. */
value = hcd->driver->urb_dequeue(hcd, urb, status);
}
return value;
}
/*
* called in any context
*
* caller guarantees urb won't be recycled till both unlink()
* and the urb's completion function return
*/
int usb_hcd_unlink_urb (struct urb *urb, int status)
{
struct usb_hcd *hcd;
struct usb_device *udev = urb->dev;
int retval = -EIDRM;
unsigned long flags;
/* Prevent the device and bus from going away while
* the unlink is carried out. If they are already gone
* then urb->use_count must be 0, since disconnected
* devices can't have any active URBs.
*/
spin_lock_irqsave(&hcd_urb_unlink_lock, flags);
if (atomic_read(&urb->use_count) > 0) {
retval = 0;
usb_get_dev(udev);
}
spin_unlock_irqrestore(&hcd_urb_unlink_lock, flags);
if (retval == 0) {
hcd = bus_to_hcd(urb->dev->bus);
retval = unlink1(hcd, urb, status);
if (retval == 0)
retval = -EINPROGRESS;
else if (retval != -EIDRM && retval != -EBUSY) dev_dbg(&udev->dev, "hcd_unlink_urb %pK fail %d\n",
urb, retval);
usb_put_dev(udev);
}
return retval;
}
/*-------------------------------------------------------------------------*/
static void __usb_hcd_giveback_urb(struct urb *urb)
{
struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus);
struct usb_anchor *anchor = urb->anchor;
int status = urb->unlinked;
urb->hcpriv = NULL;
if (unlikely((urb->transfer_flags & URB_SHORT_NOT_OK) &&
urb->actual_length < urb->transfer_buffer_length &&
!status))
status = -EREMOTEIO;
unmap_urb_for_dma(hcd, urb); usbmon_urb_complete(&hcd->self, urb, status); usb_anchor_suspend_wakeups(anchor);
usb_unanchor_urb(urb);
if (likely(status == 0))
usb_led_activity(USB_LED_EVENT_HOST);
/* pass ownership to the completion handler */
urb->status = status;
/*
* This function can be called in task context inside another remote
* coverage collection section, but kcov doesn't support that kind of
* recursion yet. Only collect coverage in softirq context for now.
*/
kcov_remote_start_usb_softirq((u64)urb->dev->bus->busnum); urb->complete(urb); kcov_remote_stop_softirq(); usb_anchor_resume_wakeups(anchor);
atomic_dec(&urb->use_count);
/*
* Order the write of urb->use_count above before the read
* of urb->reject below. Pairs with the memory barriers in
* usb_kill_urb() and usb_poison_urb().
*/
smp_mb__after_atomic();
if (unlikely(atomic_read(&urb->reject)))
wake_up(&usb_kill_urb_queue); usb_put_urb(urb);
}
static void usb_giveback_urb_bh(struct work_struct *work)
{
struct giveback_urb_bh *bh =
container_of(work, struct giveback_urb_bh, bh);
struct list_head local_list;
spin_lock_irq(&bh->lock);
bh->running = true;
list_replace_init(&bh->head, &local_list);
spin_unlock_irq(&bh->lock);
while (!list_empty(&local_list)) {
struct urb *urb;
urb = list_entry(local_list.next, struct urb, urb_list);
list_del_init(&urb->urb_list);
bh->completing_ep = urb->ep;
__usb_hcd_giveback_urb(urb);
bh->completing_ep = NULL;
}
/*
* giveback new URBs next time to prevent this function
* from not exiting for a long time.
*/
spin_lock_irq(&bh->lock);
if (!list_empty(&bh->head)) {
if (bh->high_prio)
queue_work(system_bh_highpri_wq, &bh->bh);
else
queue_work(system_bh_wq, &bh->bh);
}
bh->running = false;
spin_unlock_irq(&bh->lock);
}
/**
* usb_hcd_giveback_urb - return URB from HCD to device driver
* @hcd: host controller returning the URB
* @urb: urb being returned to the USB device driver.
* @status: completion status code for the URB.
*
* Context: atomic. The completion callback is invoked in caller's context.
* For HCDs with HCD_BH flag set, the completion callback is invoked in BH
* context (except for URBs submitted to the root hub which always complete in
* caller's context).
*
* This hands the URB from HCD to its USB device driver, using its
* completion function. The HCD has freed all per-urb resources
* (and is done using urb->hcpriv). It also released all HCD locks;
* the device driver won't cause problems if it frees, modifies,
* or resubmits this URB.
*
* If @urb was unlinked, the value of @status will be overridden by
* @urb->unlinked. Erroneous short transfers are detected in case
* the HCD hasn't checked for them.
*/
void usb_hcd_giveback_urb(struct usb_hcd *hcd, struct urb *urb, int status)
{
struct giveback_urb_bh *bh;
bool running;
/* pass status to BH via unlinked */
if (likely(!urb->unlinked)) urb->unlinked = status; if (!hcd_giveback_urb_in_bh(hcd) && !is_root_hub(urb->dev)) { __usb_hcd_giveback_urb(urb);
return;
}
if (usb_pipeisoc(urb->pipe) || usb_pipeint(urb->pipe)) bh = &hcd->high_prio_bh;
else
bh = &hcd->low_prio_bh; spin_lock(&bh->lock); list_add_tail(&urb->urb_list, &bh->head); running = bh->running;
spin_unlock(&bh->lock);
if (running)
;
else if (bh->high_prio) queue_work(system_bh_highpri_wq, &bh->bh);
else
queue_work(system_bh_wq, &bh->bh);
}
EXPORT_SYMBOL_GPL(usb_hcd_giveback_urb);
/*-------------------------------------------------------------------------*/
/* Cancel all URBs pending on this endpoint and wait for the endpoint's
* queue to drain completely. The caller must first insure that no more
* URBs can be submitted for this endpoint.
*/
void usb_hcd_flush_endpoint(struct usb_device *udev,
struct usb_host_endpoint *ep)
{
struct usb_hcd *hcd;
struct urb *urb;
if (!ep)
return;
might_sleep();
hcd = bus_to_hcd(udev->bus);
/* No more submits can occur */
spin_lock_irq(&hcd_urb_list_lock);
rescan:
list_for_each_entry_reverse(urb, &ep->urb_list, urb_list) {
int is_in;
if (urb->unlinked)
continue;
usb_get_urb (urb);
is_in = usb_urb_dir_in(urb);
spin_unlock(&hcd_urb_list_lock);
/* kick hcd */
unlink1(hcd, urb, -ESHUTDOWN);
dev_dbg (hcd->self.controller,
"shutdown urb %pK ep%d%s-%s\n",
urb, usb_endpoint_num(&ep->desc),
is_in ? "in" : "out",
usb_ep_type_string(usb_endpoint_type(&ep->desc)));
usb_put_urb (urb);
/* list contents may have changed */
spin_lock(&hcd_urb_list_lock);
goto rescan;
}
spin_unlock_irq(&hcd_urb_list_lock);
/* Wait until the endpoint queue is completely empty */
while (!list_empty (&ep->urb_list)) { spin_lock_irq(&hcd_urb_list_lock);
/* The list may have changed while we acquired the spinlock */
urb = NULL;
if (!list_empty (&ep->urb_list)) {
urb = list_entry (ep->urb_list.prev, struct urb,
urb_list);
usb_get_urb (urb);
}
spin_unlock_irq(&hcd_urb_list_lock);
if (urb) {
usb_kill_urb (urb);
usb_put_urb (urb);
}
}
}
/**
* usb_hcd_alloc_bandwidth - check whether a new bandwidth setting exceeds
* the bus bandwidth
* @udev: target &usb_device
* @new_config: new configuration to install
* @cur_alt: the current alternate interface setting
* @new_alt: alternate interface setting that is being installed
*
* To change configurations, pass in the new configuration in new_config,
* and pass NULL for cur_alt and new_alt.
*
* To reset a device's configuration (put the device in the ADDRESSED state),
* pass in NULL for new_config, cur_alt, and new_alt.
*
* To change alternate interface settings, pass in NULL for new_config,
* pass in the current alternate interface setting in cur_alt,
* and pass in the new alternate interface setting in new_alt.
*
* Return: An error if the requested bandwidth change exceeds the
* bus bandwidth or host controller internal resources.
*/
int usb_hcd_alloc_bandwidth(struct usb_device *udev,
struct usb_host_config *new_config,
struct usb_host_interface *cur_alt,
struct usb_host_interface *new_alt)
{
int num_intfs, i, j;
struct usb_host_interface *alt = NULL;
int ret = 0;
struct usb_hcd *hcd;
struct usb_host_endpoint *ep;
hcd = bus_to_hcd(udev->bus);
if (!hcd->driver->check_bandwidth)
return 0;
/* Configuration is being removed - set configuration 0 */
if (!new_config && !cur_alt) {
for (i = 1; i < 16; ++i) {
ep = udev->ep_out[i];
if (ep)
hcd->driver->drop_endpoint(hcd, udev, ep);
ep = udev->ep_in[i];
if (ep)
hcd->driver->drop_endpoint(hcd, udev, ep);
}
hcd->driver->check_bandwidth(hcd, udev);
return 0;
}
/* Check if the HCD says there's enough bandwidth. Enable all endpoints
* each interface's alt setting 0 and ask the HCD to check the bandwidth
* of the bus. There will always be bandwidth for endpoint 0, so it's
* ok to exclude it.
*/
if (new_config) {
num_intfs = new_config->desc.bNumInterfaces;
/* Remove endpoints (except endpoint 0, which is always on the
* schedule) from the old config from the schedule
*/
for (i = 1; i < 16; ++i) {
ep = udev->ep_out[i];
if (ep) {
ret = hcd->driver->drop_endpoint(hcd, udev, ep);
if (ret < 0)
goto reset;
}
ep = udev->ep_in[i];
if (ep) {
ret = hcd->driver->drop_endpoint(hcd, udev, ep);
if (ret < 0)
goto reset;
}
}
for (i = 0; i < num_intfs; ++i) {
struct usb_host_interface *first_alt;
int iface_num;
first_alt = &new_config->intf_cache[i]->altsetting[0];
iface_num = first_alt->desc.bInterfaceNumber;
/* Set up endpoints for alternate interface setting 0 */
alt = usb_find_alt_setting(new_config, iface_num, 0);
if (!alt)
/* No alt setting 0? Pick the first setting. */
alt = first_alt;
for (j = 0; j < alt->desc.bNumEndpoints; j++) {
ret = hcd->driver->add_endpoint(hcd, udev, &alt->endpoint[j]);
if (ret < 0)
goto reset;
}
}
}
if (cur_alt && new_alt) {
struct usb_interface *iface = usb_ifnum_to_if(udev,
cur_alt->desc.bInterfaceNumber);
if (!iface)
return -EINVAL;
if (iface->resetting_device) {
/*
* The USB core just reset the device, so the xHCI host
* and the device will think alt setting 0 is installed.
* However, the USB core will pass in the alternate
* setting installed before the reset as cur_alt. Dig
* out the alternate setting 0 structure, or the first
* alternate setting if a broken device doesn't have alt
* setting 0.
*/
cur_alt = usb_altnum_to_altsetting(iface, 0);
if (!cur_alt)
cur_alt = &iface->altsetting[0];
}
/* Drop all the endpoints in the current alt setting */
for (i = 0; i < cur_alt->desc.bNumEndpoints; i++) {
ret = hcd->driver->drop_endpoint(hcd, udev,
&cur_alt->endpoint[i]);
if (ret < 0)
goto reset;
}
/* Add all the endpoints in the new alt setting */
for (i = 0; i < new_alt->desc.bNumEndpoints; i++) {
ret = hcd->driver->add_endpoint(hcd, udev,
&new_alt->endpoint[i]);
if (ret < 0)
goto reset;
}
}
ret = hcd->driver->check_bandwidth(hcd, udev);
reset:
if (ret < 0)
hcd->driver->reset_bandwidth(hcd, udev);
return ret;
}
/* Disables the endpoint: synchronizes with the hcd to make sure all
* endpoint state is gone from hardware. usb_hcd_flush_endpoint() must
* have been called previously. Use for set_configuration, set_interface,
* driver removal, physical disconnect.
*
* example: a qh stored in ep->hcpriv, holding state related to endpoint
* type, maxpacket size, toggle, halt status, and scheduling.
*/
void usb_hcd_disable_endpoint(struct usb_device *udev,
struct usb_host_endpoint *ep)
{
struct usb_hcd *hcd;
might_sleep();
hcd = bus_to_hcd(udev->bus);
if (hcd->driver->endpoint_disable)
hcd->driver->endpoint_disable(hcd, ep);
}
/**
* usb_hcd_reset_endpoint - reset host endpoint state
* @udev: USB device.
* @ep: the endpoint to reset.
*
* Resets any host endpoint state such as the toggle bit, sequence
* number and current window.
*/
void usb_hcd_reset_endpoint(struct usb_device *udev,
struct usb_host_endpoint *ep)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
if (hcd->driver->endpoint_reset)
hcd->driver->endpoint_reset(hcd, ep);
else {
int epnum = usb_endpoint_num(&ep->desc);
int is_out = usb_endpoint_dir_out(&ep->desc);
int is_control = usb_endpoint_xfer_control(&ep->desc);
usb_settoggle(udev, epnum, is_out, 0);
if (is_control)
usb_settoggle(udev, epnum, !is_out, 0);
}
}
/**
* usb_alloc_streams - allocate bulk endpoint stream IDs.
* @interface: alternate setting that includes all endpoints.
* @eps: array of endpoints that need streams.
* @num_eps: number of endpoints in the array.
* @num_streams: number of streams to allocate.
* @mem_flags: flags hcd should use to allocate memory.
*
* Sets up a group of bulk endpoints to have @num_streams stream IDs available.
* Drivers may queue multiple transfers to different stream IDs, which may
* complete in a different order than they were queued.
*
* Return: On success, the number of allocated streams. On failure, a negative
* error code.
*/
int usb_alloc_streams(struct usb_interface *interface,
struct usb_host_endpoint **eps, unsigned int num_eps,
unsigned int num_streams, gfp_t mem_flags)
{
struct usb_hcd *hcd;
struct usb_device *dev;
int i, ret;
dev = interface_to_usbdev(interface);
hcd = bus_to_hcd(dev->bus);
if (!hcd->driver->alloc_streams || !hcd->driver->free_streams)
return -EINVAL;
if (dev->speed < USB_SPEED_SUPER)
return -EINVAL;
if (dev->state < USB_STATE_CONFIGURED)
return -ENODEV;
for (i = 0; i < num_eps; i++) {
/* Streams only apply to bulk endpoints. */
if (!usb_endpoint_xfer_bulk(&eps[i]->desc))
return -EINVAL;
/* Re-alloc is not allowed */
if (eps[i]->streams)
return -EINVAL;
}
ret = hcd->driver->alloc_streams(hcd, dev, eps, num_eps,
num_streams, mem_flags);
if (ret < 0)
return ret;
for (i = 0; i < num_eps; i++)
eps[i]->streams = ret;
return ret;
}
EXPORT_SYMBOL_GPL(usb_alloc_streams);
/**
* usb_free_streams - free bulk endpoint stream IDs.
* @interface: alternate setting that includes all endpoints.
* @eps: array of endpoints to remove streams from.
* @num_eps: number of endpoints in the array.
* @mem_flags: flags hcd should use to allocate memory.
*
* Reverts a group of bulk endpoints back to not using stream IDs.
* Can fail if we are given bad arguments, or HCD is broken.
*
* Return: 0 on success. On failure, a negative error code.
*/
int usb_free_streams(struct usb_interface *interface,
struct usb_host_endpoint **eps, unsigned int num_eps,
gfp_t mem_flags)
{
struct usb_hcd *hcd;
struct usb_device *dev;
int i, ret;
dev = interface_to_usbdev(interface);
hcd = bus_to_hcd(dev->bus);
if (dev->speed < USB_SPEED_SUPER)
return -EINVAL;
/* Double-free is not allowed */
for (i = 0; i < num_eps; i++)
if (!eps[i] || !eps[i]->streams)
return -EINVAL;
ret = hcd->driver->free_streams(hcd, dev, eps, num_eps, mem_flags);
if (ret < 0)
return ret;
for (i = 0; i < num_eps; i++)
eps[i]->streams = 0;
return ret;
}
EXPORT_SYMBOL_GPL(usb_free_streams);
/* Protect against drivers that try to unlink URBs after the device
* is gone, by waiting until all unlinks for @udev are finished.
* Since we don't currently track URBs by device, simply wait until
* nothing is running in the locked region of usb_hcd_unlink_urb().
*/
void usb_hcd_synchronize_unlinks(struct usb_device *udev)
{
spin_lock_irq(&hcd_urb_unlink_lock);
spin_unlock_irq(&hcd_urb_unlink_lock);
}
/*-------------------------------------------------------------------------*/
/* called in any context */
int usb_hcd_get_frame_number (struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
if (!HCD_RH_RUNNING(hcd))
return -ESHUTDOWN;
return hcd->driver->get_frame_number (hcd);
}
/*-------------------------------------------------------------------------*/
#ifdef CONFIG_USB_HCD_TEST_MODE
static void usb_ehset_completion(struct urb *urb)
{
struct completion *done = urb->context;
complete(done);
}
/*
* Allocate and initialize a control URB. This request will be used by the
* EHSET SINGLE_STEP_SET_FEATURE test in which the DATA and STATUS stages
* of the GetDescriptor request are sent 15 seconds after the SETUP stage.
* Return NULL if failed.
*/
static struct urb *request_single_step_set_feature_urb(
struct usb_device *udev,
void *dr,
void *buf,
struct completion *done)
{
struct urb *urb;
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
urb = usb_alloc_urb(0, GFP_KERNEL);
if (!urb)
return NULL;
urb->pipe = usb_rcvctrlpipe(udev, 0);
urb->ep = &udev->ep0;
urb->dev = udev;
urb->setup_packet = (void *)dr;
urb->transfer_buffer = buf;
urb->transfer_buffer_length = USB_DT_DEVICE_SIZE;
urb->complete = usb_ehset_completion;
urb->status = -EINPROGRESS;
urb->actual_length = 0;
urb->transfer_flags = URB_DIR_IN;
usb_get_urb(urb);
atomic_inc(&urb->use_count);
atomic_inc(&urb->dev->urbnum);
if (map_urb_for_dma(hcd, urb, GFP_KERNEL)) {
usb_put_urb(urb);
usb_free_urb(urb);
return NULL;
}
urb->context = done;
return urb;
}
int ehset_single_step_set_feature(struct usb_hcd *hcd, int port)
{
int retval = -ENOMEM;
struct usb_ctrlrequest *dr;
struct urb *urb;
struct usb_device *udev;
struct usb_device_descriptor *buf;
DECLARE_COMPLETION_ONSTACK(done);
/* Obtain udev of the rhub's child port */
udev = usb_hub_find_child(hcd->self.root_hub, port);
if (!udev) {
dev_err(hcd->self.controller, "No device attached to the RootHub\n");
return -ENODEV;
}
buf = kmalloc(USB_DT_DEVICE_SIZE, GFP_KERNEL);
if (!buf)
return -ENOMEM;
dr = kmalloc(sizeof(struct usb_ctrlrequest), GFP_KERNEL);
if (!dr) {
kfree(buf);
return -ENOMEM;
}
/* Fill Setup packet for GetDescriptor */
dr->bRequestType = USB_DIR_IN;
dr->bRequest = USB_REQ_GET_DESCRIPTOR;
dr->wValue = cpu_to_le16(USB_DT_DEVICE << 8);
dr->wIndex = 0;
dr->wLength = cpu_to_le16(USB_DT_DEVICE_SIZE);
urb = request_single_step_set_feature_urb(udev, dr, buf, &done);
if (!urb)
goto cleanup;
/* Submit just the SETUP stage */
retval = hcd->driver->submit_single_step_set_feature(hcd, urb, 1);
if (retval)
goto out1;
if (!wait_for_completion_timeout(&done, msecs_to_jiffies(2000))) {
usb_kill_urb(urb);
retval = -ETIMEDOUT;
dev_err(hcd->self.controller,
"%s SETUP stage timed out on ep0\n", __func__);
goto out1;
}
msleep(15 * 1000);
/* Complete remaining DATA and STATUS stages using the same URB */
urb->status = -EINPROGRESS;
usb_get_urb(urb);
atomic_inc(&urb->use_count);
atomic_inc(&urb->dev->urbnum);
retval = hcd->driver->submit_single_step_set_feature(hcd, urb, 0);
if (!retval && !wait_for_completion_timeout(&done,
msecs_to_jiffies(2000))) {
usb_kill_urb(urb);
retval = -ETIMEDOUT;
dev_err(hcd->self.controller,
"%s IN stage timed out on ep0\n", __func__);
}
out1:
usb_free_urb(urb);
cleanup:
kfree(dr);
kfree(buf);
return retval;
}
EXPORT_SYMBOL_GPL(ehset_single_step_set_feature);
#endif /* CONFIG_USB_HCD_TEST_MODE */
/*-------------------------------------------------------------------------*/
#ifdef CONFIG_PM
int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg)
{
struct usb_hcd *hcd = bus_to_hcd(rhdev->bus);
int status;
int old_state = hcd->state;
dev_dbg(&rhdev->dev, "bus %ssuspend, wakeup %d\n",
(PMSG_IS_AUTO(msg) ? "auto-" : ""),
rhdev->do_remote_wakeup);
if (HCD_DEAD(hcd)) {
dev_dbg(&rhdev->dev, "skipped %s of dead bus\n", "suspend");
return 0;
}
if (!hcd->driver->bus_suspend) {
status = -ENOENT;
} else {
clear_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
hcd->state = HC_STATE_QUIESCING;
status = hcd->driver->bus_suspend(hcd);
}
if (status == 0) {
usb_set_device_state(rhdev, USB_STATE_SUSPENDED);
hcd->state = HC_STATE_SUSPENDED;
if (!PMSG_IS_AUTO(msg))
usb_phy_roothub_suspend(hcd->self.sysdev,
hcd->phy_roothub);
/* Did we race with a root-hub wakeup event? */
if (rhdev->do_remote_wakeup) {
char buffer[6];
status = hcd->driver->hub_status_data(hcd, buffer);
if (status != 0) {
dev_dbg(&rhdev->dev, "suspend raced with wakeup event\n");
hcd_bus_resume(rhdev, PMSG_AUTO_RESUME);
status = -EBUSY;
}
}
} else {
spin_lock_irq(&hcd_root_hub_lock);
if (!HCD_DEAD(hcd)) {
set_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
hcd->state = old_state;
}
spin_unlock_irq(&hcd_root_hub_lock);
dev_dbg(&rhdev->dev, "bus %s fail, err %d\n",
"suspend", status);
}
return status;
}
int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg)
{
struct usb_hcd *hcd = bus_to_hcd(rhdev->bus);
int status;
int old_state = hcd->state;
dev_dbg(&rhdev->dev, "usb %sresume\n",
(PMSG_IS_AUTO(msg) ? "auto-" : ""));
if (HCD_DEAD(hcd)) { dev_dbg(&rhdev->dev, "skipped %s of dead bus\n", "resume"); return 0;
}
if (!PMSG_IS_AUTO(msg)) { status = usb_phy_roothub_resume(hcd->self.sysdev,
hcd->phy_roothub);
if (status)
return status;
}
if (!hcd->driver->bus_resume)
return -ENOENT;
if (HCD_RH_RUNNING(hcd))
return 0;
hcd->state = HC_STATE_RESUMING;
status = hcd->driver->bus_resume(hcd);
clear_bit(HCD_FLAG_WAKEUP_PENDING, &hcd->flags);
if (status == 0)
status = usb_phy_roothub_calibrate(hcd->phy_roothub);
if (status == 0) {
struct usb_device *udev;
int port1;
spin_lock_irq(&hcd_root_hub_lock);
if (!HCD_DEAD(hcd)) {
usb_set_device_state(rhdev, rhdev->actconfig
? USB_STATE_CONFIGURED
: USB_STATE_ADDRESS);
set_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
hcd->state = HC_STATE_RUNNING;
}
spin_unlock_irq(&hcd_root_hub_lock);
/*
* Check whether any of the enabled ports on the root hub are
* unsuspended. If they are then a TRSMRCY delay is needed
* (this is what the USB-2 spec calls a "global resume").
* Otherwise we can skip the delay.
*/
usb_hub_for_each_child(rhdev, port1, udev) { if (udev->state != USB_STATE_NOTATTACHED && !udev->port_is_suspended) { usleep_range(10000, 11000); /* TRSMRCY */
break;
}
}
} else {
hcd->state = old_state;
usb_phy_roothub_suspend(hcd->self.sysdev, hcd->phy_roothub);
dev_dbg(&rhdev->dev, "bus %s fail, err %d\n",
"resume", status);
if (status != -ESHUTDOWN) usb_hc_died(hcd);
}
return status;
}
/* Workqueue routine for root-hub remote wakeup */
static void hcd_resume_work(struct work_struct *work)
{
struct usb_hcd *hcd = container_of(work, struct usb_hcd, wakeup_work);
struct usb_device *udev = hcd->self.root_hub;
usb_remote_wakeup(udev);
}
/**
* usb_hcd_resume_root_hub - called by HCD to resume its root hub
* @hcd: host controller for this root hub
*
* The USB host controller calls this function when its root hub is
* suspended (with the remote wakeup feature enabled) and a remote
* wakeup request is received. The routine submits a workqueue request
* to resume the root hub (that is, manage its downstream ports again).
*/
void usb_hcd_resume_root_hub (struct usb_hcd *hcd)
{
unsigned long flags;
spin_lock_irqsave (&hcd_root_hub_lock, flags);
if (hcd->rh_registered) {
pm_wakeup_event(&hcd->self.root_hub->dev, 0);
set_bit(HCD_FLAG_WAKEUP_PENDING, &hcd->flags);
queue_work(pm_wq, &hcd->wakeup_work);
}
spin_unlock_irqrestore (&hcd_root_hub_lock, flags);
}
EXPORT_SYMBOL_GPL(usb_hcd_resume_root_hub);
#endif /* CONFIG_PM */
/*-------------------------------------------------------------------------*/
#ifdef CONFIG_USB_OTG
/**
* usb_bus_start_enum - start immediate enumeration (for OTG)
* @bus: the bus (must use hcd framework)
* @port_num: 1-based number of port; usually bus->otg_port
* Context: atomic
*
* Starts enumeration, with an immediate reset followed later by
* hub_wq identifying and possibly configuring the device.
* This is needed by OTG controller drivers, where it helps meet
* HNP protocol timing requirements for starting a port reset.
*
* Return: 0 if successful.
*/
int usb_bus_start_enum(struct usb_bus *bus, unsigned port_num)
{
struct usb_hcd *hcd;
int status = -EOPNOTSUPP;
/* NOTE: since HNP can't start by grabbing the bus's address0_sem,
* boards with root hubs hooked up to internal devices (instead of
* just the OTG port) may need more attention to resetting...
*/
hcd = bus_to_hcd(bus);
if (port_num && hcd->driver->start_port_reset)
status = hcd->driver->start_port_reset(hcd, port_num);
/* allocate hub_wq shortly after (first) root port reset finishes;
* it may issue others, until at least 50 msecs have passed.
*/
if (status == 0)
mod_timer(&hcd->rh_timer, jiffies + msecs_to_jiffies(10));
return status;
}
EXPORT_SYMBOL_GPL(usb_bus_start_enum);
#endif
/*-------------------------------------------------------------------------*/
/**
* usb_hcd_irq - hook IRQs to HCD framework (bus glue)
* @irq: the IRQ being raised
* @__hcd: pointer to the HCD whose IRQ is being signaled
*
* If the controller isn't HALTed, calls the driver's irq handler.
* Checks whether the controller is now dead.
*
* Return: %IRQ_HANDLED if the IRQ was handled. %IRQ_NONE otherwise.
*/
irqreturn_t usb_hcd_irq (int irq, void *__hcd)
{
struct usb_hcd *hcd = __hcd;
irqreturn_t rc;
if (unlikely(HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)))
rc = IRQ_NONE;
else if (hcd->driver->irq(hcd) == IRQ_NONE)
rc = IRQ_NONE;
else
rc = IRQ_HANDLED;
return rc;
}
EXPORT_SYMBOL_GPL(usb_hcd_irq);
/*-------------------------------------------------------------------------*/
/* Workqueue routine for when the root-hub has died. */
static void hcd_died_work(struct work_struct *work)
{
struct usb_hcd *hcd = container_of(work, struct usb_hcd, died_work);
static char *env[] = {
"ERROR=DEAD",
NULL
};
/* Notify user space that the host controller has died */
kobject_uevent_env(&hcd->self.root_hub->dev.kobj, KOBJ_OFFLINE, env);
}
/**
* usb_hc_died - report abnormal shutdown of a host controller (bus glue)
* @hcd: pointer to the HCD representing the controller
*
* This is called by bus glue to report a USB host controller that died
* while operations may still have been pending. It's called automatically
* by the PCI glue, so only glue for non-PCI busses should need to call it.
*
* Only call this function with the primary HCD.
*/
void usb_hc_died (struct usb_hcd *hcd)
{
unsigned long flags;
dev_err (hcd->self.controller, "HC died; cleaning up\n");
spin_lock_irqsave (&hcd_root_hub_lock, flags);
clear_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
set_bit(HCD_FLAG_DEAD, &hcd->flags);
if (hcd->rh_registered) {
clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
/* make hub_wq clean up old urbs and devices */
usb_set_device_state (hcd->self.root_hub,
USB_STATE_NOTATTACHED);
usb_kick_hub_wq(hcd->self.root_hub);
}
if (usb_hcd_is_primary_hcd(hcd) && hcd->shared_hcd) {
hcd = hcd->shared_hcd;
clear_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
set_bit(HCD_FLAG_DEAD, &hcd->flags);
if (hcd->rh_registered) {
clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
/* make hub_wq clean up old urbs and devices */
usb_set_device_state(hcd->self.root_hub,
USB_STATE_NOTATTACHED);
usb_kick_hub_wq(hcd->self.root_hub);
}
}
/* Handle the case where this function gets called with a shared HCD */
if (usb_hcd_is_primary_hcd(hcd))
schedule_work(&hcd->died_work);
else
schedule_work(&hcd->primary_hcd->died_work);
spin_unlock_irqrestore (&hcd_root_hub_lock, flags);
/* Make sure that the other roothub is also deallocated. */
}
EXPORT_SYMBOL_GPL (usb_hc_died);
/*-------------------------------------------------------------------------*/
static void init_giveback_urb_bh(struct giveback_urb_bh *bh)
{
spin_lock_init(&bh->lock);
INIT_LIST_HEAD(&bh->head);
INIT_WORK(&bh->bh, usb_giveback_urb_bh);
}
struct usb_hcd *__usb_create_hcd(const struct hc_driver *driver,
struct device *sysdev, struct device *dev, const char *bus_name,
struct usb_hcd *primary_hcd)
{
struct usb_hcd *hcd;
hcd = kzalloc(sizeof(*hcd) + driver->hcd_priv_size, GFP_KERNEL);
if (!hcd)
return NULL;
if (primary_hcd == NULL) {
hcd->address0_mutex = kmalloc(sizeof(*hcd->address0_mutex),
GFP_KERNEL);
if (!hcd->address0_mutex) {
kfree(hcd);
dev_dbg(dev, "hcd address0 mutex alloc failed\n");
return NULL;
}
mutex_init(hcd->address0_mutex);
hcd->bandwidth_mutex = kmalloc(sizeof(*hcd->bandwidth_mutex),
GFP_KERNEL);
if (!hcd->bandwidth_mutex) {
kfree(hcd->address0_mutex);
kfree(hcd);
dev_dbg(dev, "hcd bandwidth mutex alloc failed\n");
return NULL;
}
mutex_init(hcd->bandwidth_mutex);
dev_set_drvdata(dev, hcd);
} else {
mutex_lock(&usb_port_peer_mutex);
hcd->address0_mutex = primary_hcd->address0_mutex;
hcd->bandwidth_mutex = primary_hcd->bandwidth_mutex;
hcd->primary_hcd = primary_hcd;
primary_hcd->primary_hcd = primary_hcd;
hcd->shared_hcd = primary_hcd;
primary_hcd->shared_hcd = hcd;
mutex_unlock(&usb_port_peer_mutex);
}
kref_init(&hcd->kref);
usb_bus_init(&hcd->self);
hcd->self.controller = dev;
hcd->self.sysdev = sysdev;
hcd->self.bus_name = bus_name;
timer_setup(&hcd->rh_timer, rh_timer_func, 0);
#ifdef CONFIG_PM
INIT_WORK(&hcd->wakeup_work, hcd_resume_work);
#endif
INIT_WORK(&hcd->died_work, hcd_died_work);
hcd->driver = driver;
hcd->speed = driver->flags & HCD_MASK;
hcd->product_desc = (driver->product_desc) ? driver->product_desc :
"USB Host Controller";
return hcd;
}
EXPORT_SYMBOL_GPL(__usb_create_hcd);
/**
* usb_create_shared_hcd - create and initialize an HCD structure
* @driver: HC driver that will use this hcd
* @dev: device for this HC, stored in hcd->self.controller
* @bus_name: value to store in hcd->self.bus_name
* @primary_hcd: a pointer to the usb_hcd structure that is sharing the
* PCI device. Only allocate certain resources for the primary HCD
*
* Context: task context, might sleep.
*
* Allocate a struct usb_hcd, with extra space at the end for the
* HC driver's private data. Initialize the generic members of the
* hcd structure.
*
* Return: On success, a pointer to the created and initialized HCD structure.
* On failure (e.g. if memory is unavailable), %NULL.
*/
struct usb_hcd *usb_create_shared_hcd(const struct hc_driver *driver,
struct device *dev, const char *bus_name,
struct usb_hcd *primary_hcd)
{
return __usb_create_hcd(driver, dev, dev, bus_name, primary_hcd);
}
EXPORT_SYMBOL_GPL(usb_create_shared_hcd);
/**
* usb_create_hcd - create and initialize an HCD structure
* @driver: HC driver that will use this hcd
* @dev: device for this HC, stored in hcd->self.controller
* @bus_name: value to store in hcd->self.bus_name
*
* Context: task context, might sleep.
*
* Allocate a struct usb_hcd, with extra space at the end for the
* HC driver's private data. Initialize the generic members of the
* hcd structure.
*
* Return: On success, a pointer to the created and initialized HCD
* structure. On failure (e.g. if memory is unavailable), %NULL.
*/
struct usb_hcd *usb_create_hcd(const struct hc_driver *driver,
struct device *dev, const char *bus_name)
{
return __usb_create_hcd(driver, dev, dev, bus_name, NULL);
}
EXPORT_SYMBOL_GPL(usb_create_hcd);
/*
* Roothubs that share one PCI device must also share the bandwidth mutex.
* Don't deallocate the bandwidth_mutex until the last shared usb_hcd is
* deallocated.
*
* Make sure to deallocate the bandwidth_mutex only when the last HCD is
* freed. When hcd_release() is called for either hcd in a peer set,
* invalidate the peer's ->shared_hcd and ->primary_hcd pointers.
*/
static void hcd_release(struct kref *kref)
{
struct usb_hcd *hcd = container_of (kref, struct usb_hcd, kref);
mutex_lock(&usb_port_peer_mutex);
if (hcd->shared_hcd) {
struct usb_hcd *peer = hcd->shared_hcd;
peer->shared_hcd = NULL;
peer->primary_hcd = NULL;
} else {
kfree(hcd->address0_mutex);
kfree(hcd->bandwidth_mutex);
}
mutex_unlock(&usb_port_peer_mutex);
kfree(hcd);
}
struct usb_hcd *usb_get_hcd (struct usb_hcd *hcd)
{
if (hcd) kref_get (&hcd->kref); return hcd;
}
EXPORT_SYMBOL_GPL(usb_get_hcd);
void usb_put_hcd (struct usb_hcd *hcd)
{
if (hcd) kref_put (&hcd->kref, hcd_release);
}
EXPORT_SYMBOL_GPL(usb_put_hcd);
int usb_hcd_is_primary_hcd(struct usb_hcd *hcd)
{
if (!hcd->primary_hcd)
return 1;
return hcd == hcd->primary_hcd;
}
EXPORT_SYMBOL_GPL(usb_hcd_is_primary_hcd);
int usb_hcd_find_raw_port_number(struct usb_hcd *hcd, int port1)
{
if (!hcd->driver->find_raw_port_number)
return port1;
return hcd->driver->find_raw_port_number(hcd, port1);
}
static int usb_hcd_request_irqs(struct usb_hcd *hcd,
unsigned int irqnum, unsigned long irqflags)
{
int retval;
if (hcd->driver->irq) {
snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
hcd->driver->description, hcd->self.busnum);
retval = request_irq(irqnum, &usb_hcd_irq, irqflags,
hcd->irq_descr, hcd);
if (retval != 0) {
dev_err(hcd->self.controller,
"request interrupt %d failed\n",
irqnum);
return retval;
}
hcd->irq = irqnum;
dev_info(hcd->self.controller, "irq %d, %s 0x%08llx\n", irqnum,
(hcd->driver->flags & HCD_MEMORY) ?
"io mem" : "io port",
(unsigned long long)hcd->rsrc_start);
} else {
hcd->irq = 0;
if (hcd->rsrc_start)
dev_info(hcd->self.controller, "%s 0x%08llx\n",
(hcd->driver->flags & HCD_MEMORY) ?
"io mem" : "io port",
(unsigned long long)hcd->rsrc_start);
}
return 0;
}
/*
* Before we free this root hub, flush in-flight peering attempts
* and disable peer lookups
*/
static void usb_put_invalidate_rhdev(struct usb_hcd *hcd)
{
struct usb_device *rhdev;
mutex_lock(&usb_port_peer_mutex);
rhdev = hcd->self.root_hub;
hcd->self.root_hub = NULL;
mutex_unlock(&usb_port_peer_mutex);
usb_put_dev(rhdev);
}
/**
* usb_stop_hcd - Halt the HCD
* @hcd: the usb_hcd that has to be halted
*
* Stop the root-hub polling timer and invoke the HCD's ->stop callback.
*/
static void usb_stop_hcd(struct usb_hcd *hcd)
{
hcd->rh_pollable = 0;
clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
del_timer_sync(&hcd->rh_timer);
hcd->driver->stop(hcd);
hcd->state = HC_STATE_HALT;
/* In case the HCD restarted the timer, stop it again. */
clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
del_timer_sync(&hcd->rh_timer);
}
/**
* usb_add_hcd - finish generic HCD structure initialization and register
* @hcd: the usb_hcd structure to initialize
* @irqnum: Interrupt line to allocate
* @irqflags: Interrupt type flags
*
* Finish the remaining parts of generic HCD initialization: allocate the
* buffers of consistent memory, register the bus, request the IRQ line,
* and call the driver's reset() and start() routines.
*/
int usb_add_hcd(struct usb_hcd *hcd,
unsigned int irqnum, unsigned long irqflags)
{
int retval;
struct usb_device *rhdev;
struct usb_hcd *shared_hcd;
if (!hcd->skip_phy_initialization) {
if (usb_hcd_is_primary_hcd(hcd)) {
hcd->phy_roothub = usb_phy_roothub_alloc(hcd->self.sysdev);
if (IS_ERR(hcd->phy_roothub))
return PTR_ERR(hcd->phy_roothub);
} else {
hcd->phy_roothub = usb_phy_roothub_alloc_usb3_phy(hcd->self.sysdev);
if (IS_ERR(hcd->phy_roothub))
return PTR_ERR(hcd->phy_roothub);
}
retval = usb_phy_roothub_init(hcd->phy_roothub);
if (retval)
return retval;
retval = usb_phy_roothub_set_mode(hcd->phy_roothub,
PHY_MODE_USB_HOST_SS);
if (retval)
retval = usb_phy_roothub_set_mode(hcd->phy_roothub,
PHY_MODE_USB_HOST);
if (retval)
goto err_usb_phy_roothub_power_on;
retval = usb_phy_roothub_power_on(hcd->phy_roothub);
if (retval)
goto err_usb_phy_roothub_power_on;
}
dev_info(hcd->self.controller, "%s\n", hcd->product_desc);
switch (authorized_default) {
case USB_AUTHORIZE_NONE:
hcd->dev_policy = USB_DEVICE_AUTHORIZE_NONE;
break;
case USB_AUTHORIZE_INTERNAL:
hcd->dev_policy = USB_DEVICE_AUTHORIZE_INTERNAL;
break;
case USB_AUTHORIZE_ALL:
case USB_AUTHORIZE_WIRED:
default:
hcd->dev_policy = USB_DEVICE_AUTHORIZE_ALL;
break;
}
set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
/* per default all interfaces are authorized */
set_bit(HCD_FLAG_INTF_AUTHORIZED, &hcd->flags);
/* HC is in reset state, but accessible. Now do the one-time init,
* bottom up so that hcds can customize the root hubs before hub_wq
* starts talking to them. (Note, bus id is assigned early too.)
*/
retval = hcd_buffer_create(hcd);
if (retval != 0) {
dev_dbg(hcd->self.sysdev, "pool alloc failed\n");
goto err_create_buf;
}
retval = usb_register_bus(&hcd->self);
if (retval < 0)
goto err_register_bus;
rhdev = usb_alloc_dev(NULL, &hcd->self, 0);
if (rhdev == NULL) {
dev_err(hcd->self.sysdev, "unable to allocate root hub\n");
retval = -ENOMEM;
goto err_allocate_root_hub;
}
mutex_lock(&usb_port_peer_mutex);
hcd->self.root_hub = rhdev;
mutex_unlock(&usb_port_peer_mutex);
rhdev->rx_lanes = 1;
rhdev->tx_lanes = 1;
rhdev->ssp_rate = USB_SSP_GEN_UNKNOWN;
switch (hcd->speed) {
case HCD_USB11:
rhdev->speed = USB_SPEED_FULL;
break;
case HCD_USB2:
rhdev->speed = USB_SPEED_HIGH;
break;
case HCD_USB3:
rhdev->speed = USB_SPEED_SUPER;
break;
case HCD_USB32:
rhdev->rx_lanes = 2;
rhdev->tx_lanes = 2;
rhdev->ssp_rate = USB_SSP_GEN_2x2;
rhdev->speed = USB_SPEED_SUPER_PLUS;
break;
case HCD_USB31:
rhdev->ssp_rate = USB_SSP_GEN_2x1;
rhdev->speed = USB_SPEED_SUPER_PLUS;
break;
default:
retval = -EINVAL;
goto err_set_rh_speed;
}
/* wakeup flag init defaults to "everything works" for root hubs,
* but drivers can override it in reset() if needed, along with
* recording the overall controller's system wakeup capability.
*/
device_set_wakeup_capable(&rhdev->dev, 1);
/* HCD_FLAG_RH_RUNNING doesn't matter until the root hub is
* registered. But since the controller can die at any time,
* let's initialize the flag before touching the hardware.
*/
set_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
/* "reset" is misnamed; its role is now one-time init. the controller
* should already have been reset (and boot firmware kicked off etc).
*/
if (hcd->driver->reset) {
retval = hcd->driver->reset(hcd);
if (retval < 0) {
dev_err(hcd->self.controller, "can't setup: %d\n",
retval);
goto err_hcd_driver_setup;
}
}
hcd->rh_pollable = 1;
retval = usb_phy_roothub_calibrate(hcd->phy_roothub);
if (retval)
goto err_hcd_driver_setup;
/* NOTE: root hub and controller capabilities may not be the same */
if (device_can_wakeup(hcd->self.controller)
&& device_can_wakeup(&hcd->self.root_hub->dev))
dev_dbg(hcd->self.controller, "supports USB remote wakeup\n");
/* initialize BHs */
init_giveback_urb_bh(&hcd->high_prio_bh);
hcd->high_prio_bh.high_prio = true;
init_giveback_urb_bh(&hcd->low_prio_bh);
/* enable irqs just before we start the controller,
* if the BIOS provides legacy PCI irqs.
*/
if (usb_hcd_is_primary_hcd(hcd) && irqnum) {
retval = usb_hcd_request_irqs(hcd, irqnum, irqflags);
if (retval)
goto err_request_irq;
}
hcd->state = HC_STATE_RUNNING;
retval = hcd->driver->start(hcd);
if (retval < 0) {
dev_err(hcd->self.controller, "startup error %d\n", retval);
goto err_hcd_driver_start;
}
/* starting here, usbcore will pay attention to the shared HCD roothub */
shared_hcd = hcd->shared_hcd;
if (!usb_hcd_is_primary_hcd(hcd) && shared_hcd && HCD_DEFER_RH_REGISTER(shared_hcd)) {
retval = register_root_hub(shared_hcd);
if (retval != 0)
goto err_register_root_hub;
if (shared_hcd->uses_new_polling && HCD_POLL_RH(shared_hcd))
usb_hcd_poll_rh_status(shared_hcd);
}
/* starting here, usbcore will pay attention to this root hub */
if (!HCD_DEFER_RH_REGISTER(hcd)) {
retval = register_root_hub(hcd);
if (retval != 0)
goto err_register_root_hub;
if (hcd->uses_new_polling && HCD_POLL_RH(hcd))
usb_hcd_poll_rh_status(hcd);
}
return retval;
err_register_root_hub:
usb_stop_hcd(hcd);
err_hcd_driver_start:
if (usb_hcd_is_primary_hcd(hcd) && hcd->irq > 0)
free_irq(irqnum, hcd);
err_request_irq:
err_hcd_driver_setup:
err_set_rh_speed:
usb_put_invalidate_rhdev(hcd);
err_allocate_root_hub:
usb_deregister_bus(&hcd->self);
err_register_bus:
hcd_buffer_destroy(hcd);
err_create_buf:
usb_phy_roothub_power_off(hcd->phy_roothub);
err_usb_phy_roothub_power_on:
usb_phy_roothub_exit(hcd->phy_roothub);
return retval;
}
EXPORT_SYMBOL_GPL(usb_add_hcd);
/**
* usb_remove_hcd - shutdown processing for generic HCDs
* @hcd: the usb_hcd structure to remove
*
* Context: task context, might sleep.
*
* Disconnects the root hub, then reverses the effects of usb_add_hcd(),
* invoking the HCD's stop() method.
*/
void usb_remove_hcd(struct usb_hcd *hcd)
{
struct usb_device *rhdev;
bool rh_registered;
if (!hcd) {
pr_debug("%s: hcd is NULL\n", __func__);
return;
}
rhdev = hcd->self.root_hub;
dev_info(hcd->self.controller, "remove, state %x\n", hcd->state);
usb_get_dev(rhdev);
clear_bit(HCD_FLAG_RH_RUNNING, &hcd->flags);
if (HC_IS_RUNNING (hcd->state))
hcd->state = HC_STATE_QUIESCING;
dev_dbg(hcd->self.controller, "roothub graceful disconnect\n");
spin_lock_irq (&hcd_root_hub_lock);
rh_registered = hcd->rh_registered;
hcd->rh_registered = 0;
spin_unlock_irq (&hcd_root_hub_lock);
#ifdef CONFIG_PM
cancel_work_sync(&hcd->wakeup_work);
#endif
cancel_work_sync(&hcd->died_work);
mutex_lock(&usb_bus_idr_lock);
if (rh_registered)
usb_disconnect(&rhdev); /* Sets rhdev to NULL */
mutex_unlock(&usb_bus_idr_lock);
/*
* flush_work() isn't needed here because:
* - driver's disconnect() called from usb_disconnect() should
* make sure its URBs are completed during the disconnect()
* callback
*
* - it is too late to run complete() here since driver may have
* been removed already now
*/
/* Prevent any more root-hub status calls from the timer.
* The HCD might still restart the timer (if a port status change
* interrupt occurs), but usb_hcd_poll_rh_status() won't invoke
* the hub_status_data() callback.
*/
usb_stop_hcd(hcd);
if (usb_hcd_is_primary_hcd(hcd)) {
if (hcd->irq > 0)
free_irq(hcd->irq, hcd);
}
usb_deregister_bus(&hcd->self);
hcd_buffer_destroy(hcd);
usb_phy_roothub_power_off(hcd->phy_roothub);
usb_phy_roothub_exit(hcd->phy_roothub);
usb_put_invalidate_rhdev(hcd);
hcd->flags = 0;
}
EXPORT_SYMBOL_GPL(usb_remove_hcd);
void
usb_hcd_platform_shutdown(struct platform_device *dev)
{
struct usb_hcd *hcd = platform_get_drvdata(dev);
/* No need for pm_runtime_put(), we're shutting down */
pm_runtime_get_sync(&dev->dev);
if (hcd->driver->shutdown)
hcd->driver->shutdown(hcd);
}
EXPORT_SYMBOL_GPL(usb_hcd_platform_shutdown);
int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr,
dma_addr_t dma, size_t size)
{
int err;
void *local_mem;
hcd->localmem_pool = devm_gen_pool_create(hcd->self.sysdev, 4,
dev_to_node(hcd->self.sysdev),
dev_name(hcd->self.sysdev));
if (IS_ERR(hcd->localmem_pool))
return PTR_ERR(hcd->localmem_pool);
/*
* if a physical SRAM address was passed, map it, otherwise
* allocate system memory as a buffer.
*/
if (phys_addr)
local_mem = devm_memremap(hcd->self.sysdev, phys_addr,
size, MEMREMAP_WC);
else
local_mem = dmam_alloc_attrs(hcd->self.sysdev, size, &dma,
GFP_KERNEL,
DMA_ATTR_WRITE_COMBINE);
if (IS_ERR_OR_NULL(local_mem)) {
if (!local_mem)
return -ENOMEM;
return PTR_ERR(local_mem);
}
/*
* Here we pass a dma_addr_t but the arg type is a phys_addr_t.
* It's not backed by system memory and thus there's no kernel mapping
* for it.
*/
err = gen_pool_add_virt(hcd->localmem_pool, (unsigned long)local_mem,
dma, size, dev_to_node(hcd->self.sysdev));
if (err < 0) {
dev_err(hcd->self.sysdev, "gen_pool_add_virt failed with %d\n",
err);
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_hcd_setup_local_mem);
/*-------------------------------------------------------------------------*/
#if IS_ENABLED(CONFIG_USB_MON)
const struct usb_mon_operations *mon_ops;
/*
* The registration is unlocked.
* We do it this way because we do not want to lock in hot paths.
*
* Notice that the code is minimally error-proof. Because usbmon needs
* symbols from usbcore, usbcore gets referenced and cannot be unloaded first.
*/
int usb_mon_register(const struct usb_mon_operations *ops)
{
if (mon_ops)
return -EBUSY;
mon_ops = ops;
mb();
return 0;
}
EXPORT_SYMBOL_GPL (usb_mon_register);
void usb_mon_deregister (void)
{
if (mon_ops == NULL) {
printk(KERN_ERR "USB: monitor was not registered\n");
return;
}
mon_ops = NULL;
mb();
}
EXPORT_SYMBOL_GPL (usb_mon_deregister);
#endif /* CONFIG_USB_MON || CONFIG_USB_MON_MODULE */
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _LINUX_RCUREF_H
#define _LINUX_RCUREF_H
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/limits.h>
#include <linux/lockdep.h>
#include <linux/preempt.h>
#include <linux/rcupdate.h>
#define RCUREF_ONEREF 0x00000000U
#define RCUREF_MAXREF 0x7FFFFFFFU
#define RCUREF_SATURATED 0xA0000000U
#define RCUREF_RELEASED 0xC0000000U
#define RCUREF_DEAD 0xE0000000U
#define RCUREF_NOREF 0xFFFFFFFFU
/**
* rcuref_init - Initialize a rcuref reference count with the given reference count
* @ref: Pointer to the reference count
* @cnt: The initial reference count typically '1'
*/
static inline void rcuref_init(rcuref_t *ref, unsigned int cnt)
{
atomic_set(&ref->refcnt, cnt - 1);
}
/**
* rcuref_read - Read the number of held reference counts of a rcuref
* @ref: Pointer to the reference count
*
* Return: The number of held references (0 ... N)
*/
static inline unsigned int rcuref_read(rcuref_t *ref)
{
unsigned int c = atomic_read(&ref->refcnt);
/* Return 0 if within the DEAD zone. */
return c >= RCUREF_RELEASED ? 0 : c + 1;
}
extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);
/**
* rcuref_get - Acquire one reference on a rcuref reference count
* @ref: Pointer to the reference count
*
* Similar to atomic_inc_not_zero() but saturates at RCUREF_MAXREF.
*
* Provides no memory ordering, it is assumed the caller has guaranteed the
* object memory to be stable (RCU, etc.). It does provide a control dependency
* and thereby orders future stores. See documentation in lib/rcuref.c
*
* Return:
* False if the attempt to acquire a reference failed. This happens
* when the last reference has been put already
*
* True if a reference was successfully acquired
*/
static inline __must_check bool rcuref_get(rcuref_t *ref)
{
/*
* Unconditionally increase the reference count. The saturation and
* dead zones provide enough tolerance for this.
*/
if (likely(!atomic_add_negative_relaxed(1, &ref->refcnt)))
return true;
/* Handle the cases inside the saturation and dead zones */
return rcuref_get_slowpath(ref);
}
extern __must_check bool rcuref_put_slowpath(rcuref_t *ref);
/*
* Internal helper. Do not invoke directly.
*/
static __always_inline __must_check bool __rcuref_put(rcuref_t *ref)
{
RCU_LOCKDEP_WARN(!rcu_read_lock_held() && preemptible(),
"suspicious rcuref_put_rcusafe() usage");
/*
* Unconditionally decrease the reference count. The saturation and
* dead zones provide enough tolerance for this.
*/
if (likely(!atomic_add_negative_release(-1, &ref->refcnt)))
return false;
/*
* Handle the last reference drop and cases inside the saturation
* and dead zones.
*/
return rcuref_put_slowpath(ref);
}
/**
* rcuref_put_rcusafe -- Release one reference for a rcuref reference count RCU safe
* @ref: Pointer to the reference count
*
* Provides release memory ordering, such that prior loads and stores are done
* before, and provides an acquire ordering on success such that free()
* must come after.
*
* Can be invoked from contexts, which guarantee that no grace period can
* happen which would free the object concurrently if the decrement drops
* the last reference and the slowpath races against a concurrent get() and
* put() pair. rcu_read_lock()'ed and atomic contexts qualify.
*
* Return:
* True if this was the last reference with no future references
* possible. This signals the caller that it can safely release the
* object which is protected by the reference counter.
*
* False if there are still active references or the put() raced
* with a concurrent get()/put() pair. Caller is not allowed to
* release the protected object.
*/
static inline __must_check bool rcuref_put_rcusafe(rcuref_t *ref)
{
return __rcuref_put(ref);
}
/**
* rcuref_put -- Release one reference for a rcuref reference count
* @ref: Pointer to the reference count
*
* Can be invoked from any context.
*
* Provides release memory ordering, such that prior loads and stores are done
* before, and provides an acquire ordering on success such that free()
* must come after.
*
* Return:
*
* True if this was the last reference with no future references
* possible. This signals the caller that it can safely schedule the
* object, which is protected by the reference counter, for
* deconstruction.
*
* False if there are still active references or the put() raced
* with a concurrent get()/put() pair. Caller is not allowed to
* deconstruct the protected object.
*/
static inline __must_check bool rcuref_put(rcuref_t *ref)
{
bool released;
preempt_disable();
released = __rcuref_put(ref);
preempt_enable();
return released;
}
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/kernel/profile.c
* Simple profiling. Manages a direct-mapped profile hit count buffer,
* with configurable resolution, support for restricting the cpus on
* which profiling is done, and switching between cpu time and
* schedule() calls via kernel command line parameters passed at boot.
*
* Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
* Red Hat, July 2004
* Consolidation of architecture support code for profiling,
* Nadia Yvette Chambers, Oracle, July 2004
* Amortized hit count accounting via per-cpu open-addressed hashtables
* to resolve timer interrupt livelocks, Nadia Yvette Chambers,
* Oracle, 2004
*/
#include <linux/export.h>
#include <linux/profile.h>
#include <linux/memblock.h>
#include <linux/notifier.h>
#include <linux/mm.h>
#include <linux/cpumask.h>
#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/sched/stat.h>
#include <asm/sections.h>
#include <asm/irq_regs.h>
#include <asm/ptrace.h>
struct profile_hit {
u32 pc, hits;
};
#define PROFILE_GRPSHIFT 3
#define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
#define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
#define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
static atomic_t *prof_buffer;
static unsigned long prof_len;
static unsigned short int prof_shift;
int prof_on __read_mostly;
EXPORT_SYMBOL_GPL(prof_on);
static cpumask_var_t prof_cpu_mask;
#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
static DEFINE_PER_CPU(int, cpu_profile_flip);
static DEFINE_MUTEX(profile_flip_mutex);
#endif /* CONFIG_SMP */
int profile_setup(char *str)
{
static const char schedstr[] = "schedule";
static const char sleepstr[] = "sleep";
static const char kvmstr[] = "kvm";
const char *select = NULL;
int par;
if (!strncmp(str, sleepstr, strlen(sleepstr))) {
#ifdef CONFIG_SCHEDSTATS
force_schedstat_enabled();
prof_on = SLEEP_PROFILING;
select = sleepstr;
#else
pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
#endif /* CONFIG_SCHEDSTATS */
} else if (!strncmp(str, schedstr, strlen(schedstr))) {
prof_on = SCHED_PROFILING;
select = schedstr;
} else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
prof_on = KVM_PROFILING;
select = kvmstr;
} else if (get_option(&str, &par)) {
prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
prof_on = CPU_PROFILING;
pr_info("kernel profiling enabled (shift: %u)\n",
prof_shift);
}
if (select) {
if (str[strlen(select)] == ',')
str += strlen(select) + 1;
if (get_option(&str, &par))
prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
pr_info("kernel %s profiling enabled (shift: %u)\n",
select, prof_shift);
}
return 1;
}
__setup("profile=", profile_setup);
int __ref profile_init(void)
{
int buffer_bytes;
if (!prof_on)
return 0;
/* only text is profiled */
prof_len = (_etext - _stext) >> prof_shift;
if (!prof_len) {
pr_warn("profiling shift: %u too large\n", prof_shift);
prof_on = 0;
return -EINVAL;
}
buffer_bytes = prof_len*sizeof(atomic_t);
if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
return -ENOMEM;
cpumask_copy(prof_cpu_mask, cpu_possible_mask);
prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
if (prof_buffer)
return 0;
prof_buffer = alloc_pages_exact(buffer_bytes,
GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
if (prof_buffer)
return 0;
prof_buffer = vzalloc(buffer_bytes);
if (prof_buffer)
return 0;
free_cpumask_var(prof_cpu_mask);
return -ENOMEM;
}
#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
/*
* Each cpu has a pair of open-addressed hashtables for pending
* profile hits. read_profile() IPI's all cpus to request them
* to flip buffers and flushes their contents to prof_buffer itself.
* Flip requests are serialized by the profile_flip_mutex. The sole
* use of having a second hashtable is for avoiding cacheline
* contention that would otherwise happen during flushes of pending
* profile hits required for the accuracy of reported profile hits
* and so resurrect the interrupt livelock issue.
*
* The open-addressed hashtables are indexed by profile buffer slot
* and hold the number of pending hits to that profile buffer slot on
* a cpu in an entry. When the hashtable overflows, all pending hits
* are accounted to their corresponding profile buffer slots with
* atomic_add() and the hashtable emptied. As numerous pending hits
* may be accounted to a profile buffer slot in a hashtable entry,
* this amortizes a number of atomic profile buffer increments likely
* to be far larger than the number of entries in the hashtable,
* particularly given that the number of distinct profile buffer
* positions to which hits are accounted during short intervals (e.g.
* several seconds) is usually very small. Exclusion from buffer
* flipping is provided by interrupt disablement (note that for
* SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
* process context).
* The hash function is meant to be lightweight as opposed to strong,
* and was vaguely inspired by ppc64 firmware-supported inverted
* pagetable hash functions, but uses a full hashtable full of finite
* collision chains, not just pairs of them.
*
* -- nyc
*/
static void __profile_flip_buffers(void *unused)
{
int cpu = smp_processor_id();
per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
}
static void profile_flip_buffers(void)
{
int i, j, cpu;
mutex_lock(&profile_flip_mutex);
j = per_cpu(cpu_profile_flip, get_cpu());
put_cpu();
on_each_cpu(__profile_flip_buffers, NULL, 1);
for_each_online_cpu(cpu) {
struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
for (i = 0; i < NR_PROFILE_HIT; ++i) {
if (!hits[i].hits) {
if (hits[i].pc)
hits[i].pc = 0;
continue;
}
atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
hits[i].hits = hits[i].pc = 0;
}
}
mutex_unlock(&profile_flip_mutex);
}
static void profile_discard_flip_buffers(void)
{
int i, cpu;
mutex_lock(&profile_flip_mutex);
i = per_cpu(cpu_profile_flip, get_cpu());
put_cpu();
on_each_cpu(__profile_flip_buffers, NULL, 1);
for_each_online_cpu(cpu) {
struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
}
mutex_unlock(&profile_flip_mutex);
}
static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
{
unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
int i, j, cpu;
struct profile_hit *hits;
pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
cpu = get_cpu();
hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
if (!hits) {
put_cpu();
return;
}
/*
* We buffer the global profiler buffer into a per-CPU
* queue and thus reduce the number of global (and possibly
* NUMA-alien) accesses. The write-queue is self-coalescing:
*/
local_irq_save(flags);
do {
for (j = 0; j < PROFILE_GRPSZ; ++j) {
if (hits[i + j].pc == pc) {
hits[i + j].hits += nr_hits;
goto out;
} else if (!hits[i + j].hits) {
hits[i + j].pc = pc;
hits[i + j].hits = nr_hits;
goto out;
}
}
i = (i + secondary) & (NR_PROFILE_HIT - 1);
} while (i != primary);
/*
* Add the current hit(s) and flush the write-queue out
* to the global buffer:
*/
atomic_add(nr_hits, &prof_buffer[pc]);
for (i = 0; i < NR_PROFILE_HIT; ++i) {
atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
hits[i].pc = hits[i].hits = 0;
}
out:
local_irq_restore(flags);
put_cpu();
}
static int profile_dead_cpu(unsigned int cpu)
{
struct page *page;
int i;
if (cpumask_available(prof_cpu_mask))
cpumask_clear_cpu(cpu, prof_cpu_mask);
for (i = 0; i < 2; i++) {
if (per_cpu(cpu_profile_hits, cpu)[i]) {
page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
per_cpu(cpu_profile_hits, cpu)[i] = NULL;
__free_page(page);
}
}
return 0;
}
static int profile_prepare_cpu(unsigned int cpu)
{
int i, node = cpu_to_mem(cpu);
struct page *page;
per_cpu(cpu_profile_flip, cpu) = 0;
for (i = 0; i < 2; i++) {
if (per_cpu(cpu_profile_hits, cpu)[i])
continue;
page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
if (!page) {
profile_dead_cpu(cpu);
return -ENOMEM;
}
per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);
}
return 0;
}
static int profile_online_cpu(unsigned int cpu)
{
if (cpumask_available(prof_cpu_mask))
cpumask_set_cpu(cpu, prof_cpu_mask);
return 0;
}
#else /* !CONFIG_SMP */
#define profile_flip_buffers() do { } while (0)
#define profile_discard_flip_buffers() do { } while (0)
static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
{
unsigned long pc;
pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
}
#endif /* !CONFIG_SMP */
void profile_hits(int type, void *__pc, unsigned int nr_hits)
{
if (prof_on != type || !prof_buffer)
return;
do_profile_hits(type, __pc, nr_hits);
}
EXPORT_SYMBOL_GPL(profile_hits);
void profile_tick(int type)
{
struct pt_regs *regs = get_irq_regs();
if (!user_mode(regs) && cpumask_available(prof_cpu_mask) &&
cpumask_test_cpu(smp_processor_id(), prof_cpu_mask)) profile_hit(type, (void *)profile_pc(regs));
}
#ifdef CONFIG_PROC_FS
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
/*
* This function accesses profiling information. The returned data is
* binary: the sampling step and the actual contents of the profile
* buffer. Use of the program readprofile is recommended in order to
* get meaningful info out of these data.
*/
static ssize_t
read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
unsigned long p = *ppos;
ssize_t read;
char *pnt;
unsigned long sample_step = 1UL << prof_shift;
profile_flip_buffers();
if (p >= (prof_len+1)*sizeof(unsigned int))
return 0;
if (count > (prof_len+1)*sizeof(unsigned int) - p)
count = (prof_len+1)*sizeof(unsigned int) - p;
read = 0;
while (p < sizeof(unsigned int) && count > 0) {
if (put_user(*((char *)(&sample_step)+p), buf))
return -EFAULT;
buf++; p++; count--; read++;
}
pnt = (char *)prof_buffer + p - sizeof(atomic_t);
if (copy_to_user(buf, (void *)pnt, count))
return -EFAULT;
read += count;
*ppos += read;
return read;
}
/* default is to not implement this call */
int __weak setup_profiling_timer(unsigned mult)
{
return -EINVAL;
}
/*
* Writing to /proc/profile resets the counters
*
* Writing a 'profiling multiplier' value into it also re-sets the profiling
* interrupt frequency, on architectures that support this.
*/
static ssize_t write_profile(struct file *file, const char __user *buf,
size_t count, loff_t *ppos)
{
#ifdef CONFIG_SMP
if (count == sizeof(int)) {
unsigned int multiplier;
if (copy_from_user(&multiplier, buf, sizeof(int)))
return -EFAULT;
if (setup_profiling_timer(multiplier))
return -EINVAL;
}
#endif
profile_discard_flip_buffers();
memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
return count;
}
static const struct proc_ops profile_proc_ops = {
.proc_read = read_profile,
.proc_write = write_profile,
.proc_lseek = default_llseek,
};
int __ref create_proc_profile(void)
{
struct proc_dir_entry *entry;
#ifdef CONFIG_SMP
enum cpuhp_state online_state;
#endif
int err = 0;
if (!prof_on)
return 0;
#ifdef CONFIG_SMP
err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
profile_prepare_cpu, profile_dead_cpu);
if (err)
return err;
err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
profile_online_cpu, NULL);
if (err < 0)
goto err_state_prep;
online_state = err;
err = 0;
#endif
entry = proc_create("profile", S_IWUSR | S_IRUGO,
NULL, &profile_proc_ops);
if (!entry)
goto err_state_onl;
proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
return err;
err_state_onl:
#ifdef CONFIG_SMP
cpuhp_remove_state(online_state);
err_state_prep:
cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
#endif
return err;
}
subsys_initcall(create_proc_profile);
#endif /* CONFIG_PROC_FS */
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Queued spinlock
*
* A 'generic' spinlock implementation that is based on MCS locks. For an
* architecture that's looking for a 'generic' spinlock, please first consider
* ticket-lock.h and only come looking here when you've considered all the
* constraints below and can show your hardware does actually perform better
* with qspinlock.
*
* qspinlock relies on atomic_*_release()/atomic_*_acquire() to be RCsc (or no
* weaker than RCtso if you're power), where regular code only expects atomic_t
* to be RCpc.
*
* qspinlock relies on a far greater (compared to asm-generic/spinlock.h) set
* of atomic operations to behave well together, please audit them carefully to
* ensure they all have forward progress. Many atomic operations may default to
* cmpxchg() loops which will not have good forward progress properties on
* LL/SC architectures.
*
* One notable example is atomic_fetch_or_acquire(), which x86 cannot (cheaply)
* do. Carefully read the patches that introduced
* queued_fetch_set_pending_acquire().
*
* qspinlock also heavily relies on mixed size atomic operations, in specific
* it requires architectures to have xchg16; something which many LL/SC
* architectures need to implement as a 32bit and+or in order to satisfy the
* forward progress guarantees mentioned above.
*
* Further reading on mixed size atomics that might be relevant:
*
* http://www.cl.cam.ac.uk/~pes20/popl17/mixed-size.pdf
*
* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
* (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
*
* Authors: Waiman Long <waiman.long@hpe.com>
*/
#ifndef __ASM_GENERIC_QSPINLOCK_H
#define __ASM_GENERIC_QSPINLOCK_H
#include <asm-generic/qspinlock_types.h>
#include <linux/atomic.h>
#ifndef queued_spin_is_locked
/**
* queued_spin_is_locked - is the spinlock locked?
* @lock: Pointer to queued spinlock structure
* Return: 1 if it is locked, 0 otherwise
*/
static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
{
/*
* Any !0 state indicates it is locked, even if _Q_LOCKED_VAL
* isn't immediately observable.
*/
return atomic_read(&lock->val);
}
#endif
/**
* queued_spin_value_unlocked - is the spinlock structure unlocked?
* @lock: queued spinlock structure
* Return: 1 if it is unlocked, 0 otherwise
*
* N.B. Whenever there are tasks waiting for the lock, it is considered
* locked wrt the lockref code to avoid lock stealing by the lockref
* code and change things underneath the lock. This also allows some
* optimizations to be applied without conflict with lockref.
*/
static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
{
return !lock.val.counter;
}
/**
* queued_spin_is_contended - check if the lock is contended
* @lock : Pointer to queued spinlock structure
* Return: 1 if lock contended, 0 otherwise
*/
static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
{
return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
}
/**
* queued_spin_trylock - try to acquire the queued spinlock
* @lock : Pointer to queued spinlock structure
* Return: 1 if lock acquired, 0 if failed
*/
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
{
int val = atomic_read(&lock->val);
if (unlikely(val))
return 0;
return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
}
extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
#ifndef queued_spin_lock
/**
* queued_spin_lock - acquire a queued spinlock
* @lock: Pointer to queued spinlock structure
*/
static __always_inline void queued_spin_lock(struct qspinlock *lock)
{
int val = 0;
if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
return;
queued_spin_lock_slowpath(lock, val);
}
#endif
#ifndef queued_spin_unlock
/**
* queued_spin_unlock - release a queued spinlock
* @lock : Pointer to queued spinlock structure
*/
static __always_inline void queued_spin_unlock(struct qspinlock *lock)
{
/*
* unlock() needs release semantics:
*/
smp_store_release(&lock->locked, 0);
}
#endif
#ifndef virt_spin_lock
static __always_inline bool virt_spin_lock(struct qspinlock *lock)
{
return false;
}
#endif
/*
* Remapping spinlock architecture specific functions to the corresponding
* queued spinlock functions.
*/
#define arch_spin_is_locked(l) queued_spin_is_locked(l)
#define arch_spin_is_contended(l) queued_spin_is_contended(l)
#define arch_spin_value_unlocked(l) queued_spin_value_unlocked(l)
#define arch_spin_lock(l) queued_spin_lock(l)
#define arch_spin_trylock(l) queued_spin_trylock(l)
#define arch_spin_unlock(l) queued_spin_unlock(l)
#endif /* __ASM_GENERIC_QSPINLOCK_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PGTABLE_H
#define _LINUX_PGTABLE_H
#include <linux/pfn.h>
#include <asm/pgtable.h>
#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
#define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT)
#ifndef __ASSEMBLY__
#ifdef CONFIG_MMU
#include <linux/mm_types.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <asm-generic/pgtable_uffd.h>
#include <linux/page_table_check.h>
#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \
defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS
#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
#endif
/*
* On almost all architectures and configurations, 0 can be used as the
* upper ceiling to free_pgtables(): on many architectures it has the same
* effect as using TASK_SIZE. However, there is one configuration which
* must impose a more careful limit, to avoid freeing kernel pgtables.
*/
#ifndef USER_PGTABLES_CEILING
#define USER_PGTABLES_CEILING 0UL
#endif
/*
* This defines the first usable user address. Platforms
* can override its value with custom FIRST_USER_ADDRESS
* defined in their respective <asm/pgtable.h>.
*/
#ifndef FIRST_USER_ADDRESS
#define FIRST_USER_ADDRESS 0UL
#endif
/*
* This defines the generic helper for accessing PMD page
* table page. Although platforms can still override this
* via their respective <asm/pgtable.h>.
*/
#ifndef pmd_pgtable
#define pmd_pgtable(pmd) pmd_page(pmd)
#endif
/*
* A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD]
*
* The pXx_index() functions return the index of the entry in the page
* table page which would control the given virtual address
*
* As these functions may be used by the same code for different levels of
* the page table folding, they are always available, regardless of
* CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0
* because in such cases PTRS_PER_PxD equals 1.
*/
static inline unsigned long pte_index(unsigned long address)
{
return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
}
#ifndef pmd_index
static inline unsigned long pmd_index(unsigned long address)
{
return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
}
#define pmd_index pmd_index
#endif
#ifndef pud_index
static inline unsigned long pud_index(unsigned long address)
{
return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
}
#define pud_index pud_index
#endif
#ifndef pgd_index
/* Must be a compile-time constant, so implement it as a macro */
#define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
#endif
#ifndef pte_offset_kernel
static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
{
return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
}
#define pte_offset_kernel pte_offset_kernel
#endif
#ifdef CONFIG_HIGHPTE
#define __pte_map(pmd, address) \
((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address)))
#define pte_unmap(pte) do { \
kunmap_local((pte)); \
rcu_read_unlock(); \
} while (0)
#else
static inline pte_t *__pte_map(pmd_t *pmd, unsigned long address)
{
return pte_offset_kernel(pmd, address);
}
static inline void pte_unmap(pte_t *pte)
{
rcu_read_unlock();
}
#endif
void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable);
/* Find an entry in the second-level page table.. */
#ifndef pmd_offset
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
{
return pud_pgtable(*pud) + pmd_index(address);
}
#define pmd_offset pmd_offset
#endif
#ifndef pud_offset
static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
{
return p4d_pgtable(*p4d) + pud_index(address);
}
#define pud_offset pud_offset
#endif
static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address)
{
return (pgd + pgd_index(address));
};
/*
* a shortcut to get a pgd_t in a given mm
*/
#ifndef pgd_offset
#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
#endif
/*
* a shortcut which implies the use of the kernel's pgd, instead
* of a process's
*/
#ifndef pgd_offset_k
#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
#endif
/*
* In many cases it is known that a virtual address is mapped at PMD or PTE
* level, so instead of traversing all the page table levels, we can get a
* pointer to the PMD entry in user or kernel page table or translate a virtual
* address to the pointer in the PTE in the kernel page tables with simple
* helpers.
*/
static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va)
{
return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va);
}
static inline pmd_t *pmd_off_k(unsigned long va)
{
return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va);
}
static inline pte_t *virt_to_kpte(unsigned long vaddr)
{
pmd_t *pmd = pmd_off_k(vaddr);
return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);
}
#ifndef pmd_young
static inline int pmd_young(pmd_t pmd)
{
return 0;
}
#endif
#ifndef pmd_dirty
static inline int pmd_dirty(pmd_t pmd)
{
return 0;
}
#endif
/*
* A facility to provide lazy MMU batching. This allows PTE updates and
* page invalidations to be delayed until a call to leave lazy MMU mode
* is issued. Some architectures may benefit from doing this, and it is
* beneficial for both shadow and direct mode hypervisors, which may batch
* the PTE updates which happen during this window. Note that using this
* interface requires that read hazards be removed from the code. A read
* hazard could result in the direct mode hypervisor case, since the actual
* write to the page tables may not yet have taken place, so reads though
* a raw PTE pointer after it has been modified are not guaranteed to be
* up to date. This mode can only be entered and left under the protection of
* the page table locks for all page tables which may be modified. In the UP
* case, this is required so that preemption is disabled, and in the SMP case,
* it must synchronize the delayed page table writes properly on other CPUs.
*/
#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
#define arch_enter_lazy_mmu_mode() do {} while (0)
#define arch_leave_lazy_mmu_mode() do {} while (0)
#define arch_flush_lazy_mmu_mode() do {} while (0)
#endif
#ifndef pte_batch_hint
/**
* pte_batch_hint - Number of pages that can be added to batch without scanning.
* @ptep: Page table pointer for the entry.
* @pte: Page table entry.
*
* Some architectures know that a set of contiguous ptes all map the same
* contiguous memory with the same permissions. In this case, it can provide a
* hint to aid pte batching without the core code needing to scan every pte.
*
* An architecture implementation may ignore the PTE accessed state. Further,
* the dirty state must apply atomically to all the PTEs described by the hint.
*
* May be overridden by the architecture, else pte_batch_hint is always 1.
*/
static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)
{
return 1;
}
#endif
#ifndef pte_advance_pfn
static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
{
return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT));
}
#endif
#define pte_next_pfn(pte) pte_advance_pfn(pte, 1)
#ifndef set_ptes
/**
* set_ptes - Map consecutive pages to a contiguous range of addresses.
* @mm: Address space to map the pages into.
* @addr: Address to map the first page at.
* @ptep: Page table pointer for the first entry.
* @pte: Page table entry for the first page.
* @nr: Number of pages to map.
*
* When nr==1, initial state of pte may be present or not present, and new state
* may be present or not present. When nr>1, initial state of all ptes must be
* not present, and new state must be present.
*
* May be overridden by the architecture, or the architecture can define
* set_pte() and PFN_PTE_SHIFT.
*
* Context: The caller holds the page table lock. The pages all belong
* to the same folio. The PTEs are all in the same PMD.
*/
static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte, unsigned int nr)
{
page_table_check_ptes_set(mm, ptep, pte, nr);
arch_enter_lazy_mmu_mode();
for (;;) {
set_pte(ptep, pte);
if (--nr == 0)
break;
ptep++; pte = pte_next_pfn(pte);
}
arch_leave_lazy_mmu_mode();
}
#endif
#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1)
#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
extern int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep,
pte_t entry, int dirty);
#endif
#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern int pmdp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp,
pmd_t entry, int dirty);
extern int pudp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp,
pud_t entry, int dirty);
#else
static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp,
pmd_t entry, int dirty)
{
BUILD_BUG();
return 0;
}
static inline int pudp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp,
pud_t entry, int dirty)
{
BUILD_BUG();
return 0;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
#ifndef ptep_get
static inline pte_t ptep_get(pte_t *ptep)
{
return READ_ONCE(*ptep);
}
#endif
#ifndef pmdp_get
static inline pmd_t pmdp_get(pmd_t *pmdp)
{
return READ_ONCE(*pmdp);
}
#endif
#ifndef pudp_get
static inline pud_t pudp_get(pud_t *pudp)
{
return READ_ONCE(*pudp);
}
#endif
#ifndef p4dp_get
static inline p4d_t p4dp_get(p4d_t *p4dp)
{
return READ_ONCE(*p4dp);
}
#endif
#ifndef pgdp_get
static inline pgd_t pgdp_get(pgd_t *pgdp)
{
return READ_ONCE(*pgdp);
}
#endif
#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
unsigned long address,
pte_t *ptep)
{
pte_t pte = ptep_get(ptep);
int r = 1;
if (!pte_young(pte))
r = 0;
else
set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
return r;
}
#endif
#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long address,
pmd_t *pmdp)
{
pmd_t pmd = *pmdp;
int r = 1;
if (!pmd_young(pmd))
r = 0;
else
set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
return r;
}
#else
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long address,
pmd_t *pmdp)
{
BUILD_BUG();
return 0;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep);
#endif
#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp);
#else
/*
* Despite relevant to THP only, this API is called from generic rmap code
* under PageTransHuge(), hence needs a dummy implementation for !THP
*/
static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
BUILD_BUG();
return 0;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
#ifndef arch_has_hw_nonleaf_pmd_young
/*
* Return whether the accessed bit in non-leaf PMD entries is supported on the
* local CPU.
*/
static inline bool arch_has_hw_nonleaf_pmd_young(void)
{
return IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG);
}
#endif
#ifndef arch_has_hw_pte_young
/*
* Return whether the accessed bit is supported on the local CPU.
*
* This stub assumes accessing through an old PTE triggers a page fault.
* Architectures that automatically set the access bit should overwrite it.
*/
static inline bool arch_has_hw_pte_young(void)
{
return IS_ENABLED(CONFIG_ARCH_HAS_HW_PTE_YOUNG);
}
#endif
#ifndef arch_check_zapped_pte
static inline void arch_check_zapped_pte(struct vm_area_struct *vma,
pte_t pte)
{
}
#endif
#ifndef arch_check_zapped_pmd
static inline void arch_check_zapped_pmd(struct vm_area_struct *vma,
pmd_t pmd)
{
}
#endif
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
unsigned long address,
pte_t *ptep)
{
pte_t pte = ptep_get(ptep);
pte_clear(mm, address, ptep);
page_table_check_pte_clear(mm, pte);
return pte;
}
#endif
static inline void ptep_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
ptep_get_and_clear(mm, addr, ptep);
}
#ifdef CONFIG_GUP_GET_PXX_LOW_HIGH
/*
* For walking the pagetables without holding any locks. Some architectures
* (eg x86-32 PAE) cannot load the entries atomically without using expensive
* instructions. We are guaranteed that a PTE will only either go from not
* present to present, or present to not present -- it will not switch to a
* completely different present page without a TLB flush inbetween; which we
* are blocking by holding interrupts off.
*
* Setting ptes from not present to present goes:
*
* ptep->pte_high = h;
* smp_wmb();
* ptep->pte_low = l;
*
* And present to not present goes:
*
* ptep->pte_low = 0;
* smp_wmb();
* ptep->pte_high = 0;
*
* We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
* We load pte_high *after* loading pte_low, which ensures we don't see an older
* value of pte_high. *Then* we recheck pte_low, which ensures that we haven't
* picked up a changed pte high. We might have gotten rubbish values from
* pte_low and pte_high, but we are guaranteed that pte_low will not have the
* present bit set *unless* it is 'l'. Because get_user_pages_fast() only
* operates on present ptes we're safe.
*/
static inline pte_t ptep_get_lockless(pte_t *ptep)
{
pte_t pte;
do {
pte.pte_low = ptep->pte_low;
smp_rmb();
pte.pte_high = ptep->pte_high;
smp_rmb();
} while (unlikely(pte.pte_low != ptep->pte_low));
return pte;
}
#define ptep_get_lockless ptep_get_lockless
#if CONFIG_PGTABLE_LEVELS > 2
static inline pmd_t pmdp_get_lockless(pmd_t *pmdp)
{
pmd_t pmd;
do {
pmd.pmd_low = pmdp->pmd_low;
smp_rmb();
pmd.pmd_high = pmdp->pmd_high;
smp_rmb();
} while (unlikely(pmd.pmd_low != pmdp->pmd_low));
return pmd;
}
#define pmdp_get_lockless pmdp_get_lockless
#define pmdp_get_lockless_sync() tlb_remove_table_sync_one()
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
#endif /* CONFIG_GUP_GET_PXX_LOW_HIGH */
/*
* We require that the PTE can be read atomically.
*/
#ifndef ptep_get_lockless
static inline pte_t ptep_get_lockless(pte_t *ptep)
{
return ptep_get(ptep);
}
#endif
#ifndef pmdp_get_lockless
static inline pmd_t pmdp_get_lockless(pmd_t *pmdp)
{
return pmdp_get(pmdp);
}
static inline void pmdp_get_lockless_sync(void)
{
}
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address,
pmd_t *pmdp)
{
pmd_t pmd = *pmdp;
pmd_clear(pmdp);
page_table_check_pmd_clear(mm, pmd);
return pmd;
}
#endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */
#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address,
pud_t *pudp)
{
pud_t pud = *pudp;
pud_clear(pudp);
page_table_check_pud_clear(mm, pud);
return pud;
}
#endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp,
int full)
{
return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
}
#endif
#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL
static inline pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp,
int full)
{
return pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
}
#endif
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
unsigned long address, pte_t *ptep,
int full)
{
return ptep_get_and_clear(mm, address, ptep);
}
#endif
#ifndef get_and_clear_full_ptes
/**
* get_and_clear_full_ptes - Clear present PTEs that map consecutive pages of
* the same folio, collecting dirty/accessed bits.
* @mm: Address space the pages are mapped into.
* @addr: Address the first page is mapped at.
* @ptep: Page table pointer for the first entry.
* @nr: Number of entries to clear.
* @full: Whether we are clearing a full mm.
*
* May be overridden by the architecture; otherwise, implemented as a simple
* loop over ptep_get_and_clear_full(), merging dirty/accessed bits into the
* returned PTE.
*
* Note that PTE bits in the PTE range besides the PFN can differ. For example,
* some PTEs might be write-protected.
*
* Context: The caller holds the page table lock. The PTEs map consecutive
* pages that belong to the same folio. The PTEs are all in the same PMD.
*/
static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, unsigned int nr, int full)
{
pte_t pte, tmp_pte;
pte = ptep_get_and_clear_full(mm, addr, ptep, full);
while (--nr) {
ptep++;
addr += PAGE_SIZE;
tmp_pte = ptep_get_and_clear_full(mm, addr, ptep, full);
if (pte_dirty(tmp_pte))
pte = pte_mkdirty(pte);
if (pte_young(tmp_pte))
pte = pte_mkyoung(pte);
}
return pte;
}
#endif
#ifndef clear_full_ptes
/**
* clear_full_ptes - Clear present PTEs that map consecutive pages of the same
* folio.
* @mm: Address space the pages are mapped into.
* @addr: Address the first page is mapped at.
* @ptep: Page table pointer for the first entry.
* @nr: Number of entries to clear.
* @full: Whether we are clearing a full mm.
*
* May be overridden by the architecture; otherwise, implemented as a simple
* loop over ptep_get_and_clear_full().
*
* Note that PTE bits in the PTE range besides the PFN can differ. For example,
* some PTEs might be write-protected.
*
* Context: The caller holds the page table lock. The PTEs map consecutive
* pages that belong to the same folio. The PTEs are all in the same PMD.
*/
static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned int nr, int full)
{
for (;;) {
ptep_get_and_clear_full(mm, addr, ptep, full);
if (--nr == 0)
break;
ptep++;
addr += PAGE_SIZE;
}
}
#endif
/*
* If two threads concurrently fault at the same page, the thread that
* won the race updates the PTE and its local TLB/Cache. The other thread
* gives up, simply does nothing, and continues; on architectures where
* software can update TLB, local TLB can be updated here to avoid next page
* fault. This function updates TLB only, do nothing with cache or others.
* It is the difference with function update_mmu_cache.
*/
#ifndef __HAVE_ARCH_UPDATE_MMU_TLB
static inline void update_mmu_tlb(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
}
#define __HAVE_ARCH_UPDATE_MMU_TLB
#endif
/*
* Some architectures may be able to avoid expensive synchronization
* primitives when modifications are made to PTE's which are already
* not present, or in the process of an address space destruction.
*/
#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
static inline void pte_clear_not_present_full(struct mm_struct *mm,
unsigned long address,
pte_t *ptep,
int full)
{
pte_clear(mm, address, ptep);
}
#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
unsigned long address,
pte_t *ptep);
#endif
#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
unsigned long address,
pmd_t *pmdp);
extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma,
unsigned long address,
pud_t *pudp);
#endif
#ifndef pte_mkwrite
static inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
{
return pte_mkwrite_novma(pte);
}
#endif
#if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite)
static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
{
return pmd_mkwrite_novma(pmd);
}
#endif
#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
struct mm_struct;
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
{
pte_t old_pte = ptep_get(ptep);
set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
}
#endif
#ifndef wrprotect_ptes
/**
* wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same
* folio.
* @mm: Address space the pages are mapped into.
* @addr: Address the first page is mapped at.
* @ptep: Page table pointer for the first entry.
* @nr: Number of entries to write-protect.
*
* May be overridden by the architecture; otherwise, implemented as a simple
* loop over ptep_set_wrprotect().
*
* Note that PTE bits in the PTE range besides the PFN can differ. For example,
* some PTEs might be write-protected.
*
* Context: The caller holds the page table lock. The PTEs map consecutive
* pages that belong to the same folio. The PTEs are all in the same PMD.
*/
static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned int nr)
{
for (;;) {
ptep_set_wrprotect(mm, addr, ptep);
if (--nr == 0)
break;
ptep++;
addr += PAGE_SIZE;
}
}
#endif
/*
* On some architectures hardware does not set page access bit when accessing
* memory page, it is responsibility of software setting this bit. It brings
* out extra page fault penalty to track page access bit. For optimization page
* access bit can be set during all page fault flow on these arches.
* To be differentiate with macro pte_mkyoung, this macro is used on platforms
* where software maintains page access bit.
*/
#ifndef pte_sw_mkyoung
static inline pte_t pte_sw_mkyoung(pte_t pte)
{
return pte;
}
#define pte_sw_mkyoung pte_sw_mkyoung
#endif
#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp)
{
pmd_t old_pmd = *pmdp;
set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
}
#else
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp)
{
BUILD_BUG();
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void pudp_set_wrprotect(struct mm_struct *mm,
unsigned long address, pud_t *pudp)
{
pud_t old_pud = *pudp;
set_pud_at(mm, address, pudp, pud_wrprotect(old_pud));
}
#else
static inline void pudp_set_wrprotect(struct mm_struct *mm,
unsigned long address, pud_t *pudp)
{
BUILD_BUG();
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
#endif
#ifndef pmdp_collapse_flush
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp);
#else
static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address,
pmd_t *pmdp)
{
BUILD_BUG();
return *pmdp;
}
#define pmdp_collapse_flush pmdp_collapse_flush
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pgtable);
#endif
#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
#endif
#ifndef arch_needs_pgtable_deposit
#define arch_needs_pgtable_deposit() (false)
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
* This is an implementation of pmdp_establish() that is only suitable for an
* architecture that doesn't have hardware dirty/accessed bits. In this case we
* can't race with CPU which sets these bits and non-atomic approach is fine.
*/
static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp, pmd_t pmd)
{
pmd_t old_pmd = *pmdp;
set_pmd_at(vma->vm_mm, address, pmdp, pmd);
return old_pmd;
}
#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATE
extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp);
#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
/*
* pmdp_invalidate_ad() invalidates the PMD while changing a transparent
* hugepage mapping in the page tables. This function is similar to
* pmdp_invalidate(), but should only be used if the access and dirty bits would
* not be cleared by the software in the new PMD value. The function ensures
* that hardware changes of the access and dirty bits updates would not be lost.
*
* Doing so can allow in certain architectures to avoid a TLB flush in most
* cases. Yet, another TLB flush might be necessary later if the PMD update
* itself requires such flush (e.g., if protection was set to be stricter). Yet,
* even when a TLB flush is needed because of the update, the caller may be able
* to batch these TLB flushing operations, so fewer TLB flush operations are
* needed.
*/
extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp);
#endif
#ifndef __HAVE_ARCH_PTE_SAME
static inline int pte_same(pte_t pte_a, pte_t pte_b)
{
return pte_val(pte_a) == pte_val(pte_b);
}
#endif
#ifndef __HAVE_ARCH_PTE_UNUSED
/*
* Some architectures provide facilities to virtualization guests
* so that they can flag allocated pages as unused. This allows the
* host to transparently reclaim unused pages. This function returns
* whether the pte's page is unused.
*/
static inline int pte_unused(pte_t pte)
{
return 0;
}
#endif
#ifndef pte_access_permitted
#define pte_access_permitted(pte, write) \
(pte_present(pte) && (!(write) || pte_write(pte)))
#endif
#ifndef pmd_access_permitted
#define pmd_access_permitted(pmd, write) \
(pmd_present(pmd) && (!(write) || pmd_write(pmd)))
#endif
#ifndef pud_access_permitted
#define pud_access_permitted(pud, write) \
(pud_present(pud) && (!(write) || pud_write(pud)))
#endif
#ifndef p4d_access_permitted
#define p4d_access_permitted(p4d, write) \
(p4d_present(p4d) && (!(write) || p4d_write(p4d)))
#endif
#ifndef pgd_access_permitted
#define pgd_access_permitted(pgd, write) \
(pgd_present(pgd) && (!(write) || pgd_write(pgd)))
#endif
#ifndef __HAVE_ARCH_PMD_SAME
static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
{
return pmd_val(pmd_a) == pmd_val(pmd_b);
}
#endif
#ifndef pud_same
static inline int pud_same(pud_t pud_a, pud_t pud_b)
{
return pud_val(pud_a) == pud_val(pud_b);
}
#define pud_same pud_same
#endif
#ifndef __HAVE_ARCH_P4D_SAME
static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b)
{
return p4d_val(p4d_a) == p4d_val(p4d_b);
}
#endif
#ifndef __HAVE_ARCH_PGD_SAME
static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b)
{
return pgd_val(pgd_a) == pgd_val(pgd_b);
}
#endif
/*
* Use set_p*_safe(), and elide TLB flushing, when confident that *no*
* TLB flush will be required as a result of the "set". For example, use
* in scenarios where it is known ahead of time that the routine is
* setting non-present entries, or re-setting an existing entry to the
* same value. Otherwise, use the typical "set" helpers and flush the
* TLB.
*/
#define set_pte_safe(ptep, pte) \
({ \
WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
set_pte(ptep, pte); \
})
#define set_pmd_safe(pmdp, pmd) \
({ \
WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
set_pmd(pmdp, pmd); \
})
#define set_pud_safe(pudp, pud) \
({ \
WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
set_pud(pudp, pud); \
})
#define set_p4d_safe(p4dp, p4d) \
({ \
WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
set_p4d(p4dp, p4d); \
})
#define set_pgd_safe(pgdp, pgd) \
({ \
WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
set_pgd(pgdp, pgd); \
})
#ifndef __HAVE_ARCH_DO_SWAP_PAGE
/*
* Some architectures support metadata associated with a page. When a
* page is being swapped out, this metadata must be saved so it can be
* restored when the page is swapped back in. SPARC M7 and newer
* processors support an ADI (Application Data Integrity) tag for the
* page as metadata for the page. arch_do_swap_page() can restore this
* metadata when a page is swapped back in.
*/
static inline void arch_do_swap_page(struct mm_struct *mm,
struct vm_area_struct *vma,
unsigned long addr,
pte_t pte, pte_t oldpte)
{
}
#endif
#ifndef __HAVE_ARCH_UNMAP_ONE
/*
* Some architectures support metadata associated with a page. When a
* page is being swapped out, this metadata must be saved so it can be
* restored when the page is swapped back in. SPARC M7 and newer
* processors support an ADI (Application Data Integrity) tag for the
* page as metadata for the page. arch_unmap_one() can save this
* metadata on a swap-out of a page.
*/
static inline int arch_unmap_one(struct mm_struct *mm,
struct vm_area_struct *vma,
unsigned long addr,
pte_t orig_pte)
{
return 0;
}
#endif
/*
* Allow architectures to preserve additional metadata associated with
* swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function
* prototypes must be defined in the arch-specific asm/pgtable.h file.
*/
#ifndef __HAVE_ARCH_PREPARE_TO_SWAP
static inline int arch_prepare_to_swap(struct page *page)
{
return 0;
}
#endif
#ifndef __HAVE_ARCH_SWAP_INVALIDATE
static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
{
}
static inline void arch_swap_invalidate_area(int type)
{
}
#endif
#ifndef __HAVE_ARCH_SWAP_RESTORE
static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
{
}
#endif
#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
#endif
#ifndef __HAVE_ARCH_MOVE_PTE
#define move_pte(pte, prot, old_addr, new_addr) (pte)
#endif
#ifndef pte_accessible
# define pte_accessible(mm, pte) ((void)(pte), 1)
#endif
#ifndef flush_tlb_fix_spurious_fault
#define flush_tlb_fix_spurious_fault(vma, address, ptep) flush_tlb_page(vma, address)
#endif
/*
* When walking page tables, get the address of the next boundary,
* or the end address of the range if that comes earlier. Although no
* vma end wraps to 0, rounded up __boundary may wrap to 0 throughout.
*/
#define pgd_addr_end(addr, end) \
({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
(__boundary - 1 < (end) - 1)? __boundary: (end); \
})
#ifndef p4d_addr_end
#define p4d_addr_end(addr, end) \
({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \
(__boundary - 1 < (end) - 1)? __boundary: (end); \
})
#endif
#ifndef pud_addr_end
#define pud_addr_end(addr, end) \
({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
(__boundary - 1 < (end) - 1)? __boundary: (end); \
})
#endif
#ifndef pmd_addr_end
#define pmd_addr_end(addr, end) \
({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
(__boundary - 1 < (end) - 1)? __boundary: (end); \
})
#endif
/*
* When walking page tables, we usually want to skip any p?d_none entries;
* and any p?d_bad entries - reporting the error before resetting to none.
* Do the tests inline, but report and clear the bad entry in mm/memory.c.
*/
void pgd_clear_bad(pgd_t *);
#ifndef __PAGETABLE_P4D_FOLDED
void p4d_clear_bad(p4d_t *);
#else
#define p4d_clear_bad(p4d) do { } while (0)
#endif
#ifndef __PAGETABLE_PUD_FOLDED
void pud_clear_bad(pud_t *);
#else
#define pud_clear_bad(p4d) do { } while (0)
#endif
void pmd_clear_bad(pmd_t *);
static inline int pgd_none_or_clear_bad(pgd_t *pgd)
{
if (pgd_none(*pgd))
return 1;
if (unlikely(pgd_bad(*pgd))) {
pgd_clear_bad(pgd);
return 1;
}
return 0;
}
static inline int p4d_none_or_clear_bad(p4d_t *p4d)
{
if (p4d_none(*p4d))
return 1;
if (unlikely(p4d_bad(*p4d))) {
p4d_clear_bad(p4d);
return 1;
}
return 0;
}
static inline int pud_none_or_clear_bad(pud_t *pud)
{
if (pud_none(*pud))
return 1;
if (unlikely(pud_bad(*pud))) { pud_clear_bad(pud);
return 1;
}
return 0;
}
static inline int pmd_none_or_clear_bad(pmd_t *pmd)
{
if (pmd_none(*pmd))
return 1;
if (unlikely(pmd_bad(*pmd))) { pmd_clear_bad(pmd);
return 1;
}
return 0;
}
static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma,
unsigned long addr,
pte_t *ptep)
{
/*
* Get the current pte state, but zero it out to make it
* non-present, preventing the hardware from asynchronously
* updating it.
*/
return ptep_get_and_clear(vma->vm_mm, addr, ptep);
}
static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma,
unsigned long addr,
pte_t *ptep, pte_t pte)
{
/*
* The pte is non-present, so there's no hardware state to
* preserve.
*/
set_pte_at(vma->vm_mm, addr, ptep, pte);
}
#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
/*
* Start a pte protection read-modify-write transaction, which
* protects against asynchronous hardware modifications to the pte.
* The intention is not to prevent the hardware from making pte
* updates, but to prevent any updates it may make from being lost.
*
* This does not protect against other software modifications of the
* pte; the appropriate pte lock must be held over the transaction.
*
* Note that this interface is intended to be batchable, meaning that
* ptep_modify_prot_commit may not actually update the pte, but merely
* queue the update to be done at some later time. The update must be
* actually committed before the pte lock is released, however.
*/
static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
unsigned long addr,
pte_t *ptep)
{
return __ptep_modify_prot_start(vma, addr, ptep);
}
/*
* Commit an update to a pte, leaving any hardware-controlled bits in
* the PTE unmodified.
*/
static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
unsigned long addr,
pte_t *ptep, pte_t old_pte, pte_t pte)
{
__ptep_modify_prot_commit(vma, addr, ptep, pte);
}
#endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
#endif /* CONFIG_MMU */
/*
* No-op macros that just return the current protection value. Defined here
* because these macros can be used even if CONFIG_MMU is not defined.
*/
#ifndef pgprot_nx
#define pgprot_nx(prot) (prot)
#endif
#ifndef pgprot_noncached
#define pgprot_noncached(prot) (prot)
#endif
#ifndef pgprot_writecombine
#define pgprot_writecombine pgprot_noncached
#endif
#ifndef pgprot_writethrough
#define pgprot_writethrough pgprot_noncached
#endif
#ifndef pgprot_device
#define pgprot_device pgprot_noncached
#endif
#ifndef pgprot_mhp
#define pgprot_mhp(prot) (prot)
#endif
#ifdef CONFIG_MMU
#ifndef pgprot_modify
#define pgprot_modify pgprot_modify
static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
{
if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
newprot = pgprot_noncached(newprot);
if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
newprot = pgprot_writecombine(newprot);
if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
newprot = pgprot_device(newprot);
return newprot;
}
#endif
#endif /* CONFIG_MMU */
#ifndef pgprot_encrypted
#define pgprot_encrypted(prot) (prot)
#endif
#ifndef pgprot_decrypted
#define pgprot_decrypted(prot) (prot)
#endif
/*
* A facility to provide batching of the reload of page tables and
* other process state with the actual context switch code for
* paravirtualized guests. By convention, only one of the batched
* update (lazy) modes (CPU, MMU) should be active at any given time,
* entry should never be nested, and entry and exits should always be
* paired. This is for sanity of maintaining and reasoning about the
* kernel code. In this case, the exit (end of the context switch) is
* in architecture-specific code, and so doesn't need a generic
* definition.
*/
#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
#define arch_start_context_switch(prev) do {} while (0)
#endif
#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION
static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
{
return pmd;
}
static inline int pmd_swp_soft_dirty(pmd_t pmd)
{
return 0;
}
static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
{
return pmd;
}
#endif
#else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */
static inline int pte_soft_dirty(pte_t pte)
{
return 0;
}
static inline int pmd_soft_dirty(pmd_t pmd)
{
return 0;
}
static inline pte_t pte_mksoft_dirty(pte_t pte)
{
return pte;
}
static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
{
return pmd;
}
static inline pte_t pte_clear_soft_dirty(pte_t pte)
{
return pte;
}
static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
{
return pmd;
}
static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
{
return pte;
}
static inline int pte_swp_soft_dirty(pte_t pte)
{
return 0;
}
static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
{
return pte;
}
static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
{
return pmd;
}
static inline int pmd_swp_soft_dirty(pmd_t pmd)
{
return 0;
}
static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
{
return pmd;
}
#endif
#ifndef __HAVE_PFNMAP_TRACKING
/*
* Interfaces that can be used by architecture code to keep track of
* memory type of pfn mappings specified by the remap_pfn_range,
* vmf_insert_pfn.
*/
/*
* track_pfn_remap is called when a _new_ pfn mapping is being established
* by remap_pfn_range() for physical range indicated by pfn and size.
*/
static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
unsigned long pfn, unsigned long addr,
unsigned long size)
{
return 0;
}
/*
* track_pfn_insert is called when a _new_ single pfn is established
* by vmf_insert_pfn().
*/
static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
pfn_t pfn)
{
}
/*
* track_pfn_copy is called when vma that is covering the pfnmap gets
* copied through copy_page_range().
*/
static inline int track_pfn_copy(struct vm_area_struct *vma)
{
return 0;
}
/*
* untrack_pfn is called while unmapping a pfnmap for a region.
* untrack can be called for a specific region indicated by pfn and size or
* can be for the entire vma (in which case pfn, size are zero).
*/
static inline void untrack_pfn(struct vm_area_struct *vma,
unsigned long pfn, unsigned long size,
bool mm_wr_locked)
{
}
/*
* untrack_pfn_clear is called while mremapping a pfnmap for a new region
* or fails to copy pgtable during duplicate vm area.
*/
static inline void untrack_pfn_clear(struct vm_area_struct *vma)
{
}
#else
extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
unsigned long pfn, unsigned long addr,
unsigned long size);
extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
pfn_t pfn);
extern int track_pfn_copy(struct vm_area_struct *vma);
extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
unsigned long size, bool mm_wr_locked);
extern void untrack_pfn_clear(struct vm_area_struct *vma);
#endif
#ifdef CONFIG_MMU
#ifdef __HAVE_COLOR_ZERO_PAGE
static inline int is_zero_pfn(unsigned long pfn)
{
extern unsigned long zero_pfn;
unsigned long offset_from_zero_pfn = pfn - zero_pfn;
return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
}
#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
#else
static inline int is_zero_pfn(unsigned long pfn)
{
extern unsigned long zero_pfn;
return pfn == zero_pfn;
}
static inline unsigned long my_zero_pfn(unsigned long addr)
{
extern unsigned long zero_pfn;
return zero_pfn;
}
#endif
#else
static inline int is_zero_pfn(unsigned long pfn)
{
return 0;
}
static inline unsigned long my_zero_pfn(unsigned long addr)
{
return 0;
}
#endif /* CONFIG_MMU */
#ifdef CONFIG_MMU
#ifndef CONFIG_TRANSPARENT_HUGEPAGE
static inline int pmd_trans_huge(pmd_t pmd)
{
return 0;
}
#ifndef pmd_write
static inline int pmd_write(pmd_t pmd)
{
BUG();
return 0;
}
#endif /* pmd_write */
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifndef pud_write
static inline int pud_write(pud_t pud)
{
BUG();
return 0;
}
#endif /* pud_write */
#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
static inline int pmd_devmap(pmd_t pmd)
{
return 0;
}
static inline int pud_devmap(pud_t pud)
{
return 0;
}
static inline int pgd_devmap(pgd_t pgd)
{
return 0;
}
#endif
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
!defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
static inline int pud_trans_huge(pud_t pud)
{
return 0;
}
#endif
static inline int pud_trans_unstable(pud_t *pud)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
pud_t pudval = READ_ONCE(*pud);
if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval))
return 1;
if (unlikely(pud_bad(pudval))) {
pud_clear_bad(pud);
return 1;
}
#endif
return 0;
}
#ifndef CONFIG_NUMA_BALANCING
/*
* In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is
* perfectly valid to indicate "no" in that case, which is why our default
* implementation defaults to "always no".
*
* In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE
* page protection due to NUMA hinting. NUMA hinting faults only apply in
* accessible VMAs.
*
* So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault,
* looking at the VMA accessibility is sufficient.
*/
static inline int pte_protnone(pte_t pte)
{
return 0;
}
static inline int pmd_protnone(pmd_t pmd)
{
return 0;
}
#endif /* CONFIG_NUMA_BALANCING */
#endif /* CONFIG_MMU */
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
#ifndef __PAGETABLE_P4D_FOLDED
int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
void p4d_clear_huge(p4d_t *p4d);
#else
static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
{
return 0;
}
static inline void p4d_clear_huge(p4d_t *p4d) { }
#endif /* !__PAGETABLE_P4D_FOLDED */
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
int pud_clear_huge(pud_t *pud);
int pmd_clear_huge(pmd_t *pmd);
int p4d_free_pud_page(p4d_t *p4d, unsigned long addr);
int pud_free_pmd_page(pud_t *pud, unsigned long addr);
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
{
return 0;
}
static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
{
return 0;
}
static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
{
return 0;
}
static inline void p4d_clear_huge(p4d_t *p4d) { }
static inline int pud_clear_huge(pud_t *pud)
{
return 0;
}
static inline int pmd_clear_huge(pmd_t *pmd)
{
return 0;
}
static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)
{
return 0;
}
static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr)
{
return 0;
}
static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
{
return 0;
}
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
* ARCHes with special requirements for evicting THP backing TLB entries can
* implement this. Otherwise also, it can help optimize normal TLB flush in
* THP regime. Stock flush_tlb_range() typically has optimization to nuke the
* entire TLB if flush span is greater than a threshold, which will
* likely be true for a single huge page. Thus a single THP flush will
* invalidate the entire TLB which is not desirable.
* e.g. see arch/arc: flush_pmd_tlb_range
*/
#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
#else
#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()
#endif
#endif
struct file;
int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
unsigned long size, pgprot_t *vma_prot);
#ifndef CONFIG_X86_ESPFIX64
static inline void init_espfix_bsp(void) { }
#endif
extern void __init pgtable_cache_init(void);
#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED
static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot)
{
return true;
}
static inline bool arch_has_pfn_modify_check(void)
{
return false;
}
#endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */
/*
* Architecture PAGE_KERNEL_* fallbacks
*
* Some architectures don't define certain PAGE_KERNEL_* flags. This is either
* because they really don't support them, or the port needs to be updated to
* reflect the required functionality. Below are a set of relatively safe
* fallbacks, as best effort, which we can count on in lieu of the architectures
* not defining them on their own yet.
*/
#ifndef PAGE_KERNEL_RO
# define PAGE_KERNEL_RO PAGE_KERNEL
#endif
#ifndef PAGE_KERNEL_EXEC
# define PAGE_KERNEL_EXEC PAGE_KERNEL
#endif
/*
* Page Table Modification bits for pgtbl_mod_mask.
*
* These are used by the p?d_alloc_track*() set of functions an in the generic
* vmalloc/ioremap code to track at which page-table levels entries have been
* modified. Based on that the code can better decide when vmalloc and ioremap
* mapping changes need to be synchronized to other page-tables in the system.
*/
#define __PGTBL_PGD_MODIFIED 0
#define __PGTBL_P4D_MODIFIED 1
#define __PGTBL_PUD_MODIFIED 2
#define __PGTBL_PMD_MODIFIED 3
#define __PGTBL_PTE_MODIFIED 4
#define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED)
#define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED)
#define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED)
#define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED)
#define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED)
/* Page-Table Modification Mask */
typedef unsigned int pgtbl_mod_mask;
#endif /* !__ASSEMBLY__ */
#if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
#ifdef CONFIG_PHYS_ADDR_T_64BIT
/*
* ZSMALLOC needs to know the highest PFN on 32-bit architectures
* with physical address space extension, but falls back to
* BITS_PER_LONG otherwise.
*/
#error Missing MAX_POSSIBLE_PHYSMEM_BITS definition
#else
#define MAX_POSSIBLE_PHYSMEM_BITS 32
#endif
#endif
#ifndef has_transparent_hugepage
#define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)
#endif
#ifndef has_transparent_pud_hugepage
#define has_transparent_pud_hugepage() IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
#endif
/*
* On some architectures it depends on the mm if the p4d/pud or pmd
* layer of the page table hierarchy is folded or not.
*/
#ifndef mm_p4d_folded
#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED)
#endif
#ifndef mm_pud_folded
#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED)
#endif
#ifndef mm_pmd_folded
#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)
#endif
#ifndef p4d_offset_lockless
#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)
#endif
#ifndef pud_offset_lockless
#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)
#endif
#ifndef pmd_offset_lockless
#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)
#endif
/*
* p?d_leaf() - true if this entry is a final mapping to a physical address.
* This differs from p?d_huge() by the fact that they are always available (if
* the architecture supports large pages at the appropriate level) even
* if CONFIG_HUGETLB_PAGE is not defined.
* Only meaningful when called on a valid entry.
*/
#ifndef pgd_leaf
#define pgd_leaf(x) false
#endif
#ifndef p4d_leaf
#define p4d_leaf(x) false
#endif
#ifndef pud_leaf
#define pud_leaf(x) false
#endif
#ifndef pmd_leaf
#define pmd_leaf(x) false
#endif
#ifndef pgd_leaf_size
#define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT)
#endif
#ifndef p4d_leaf_size
#define p4d_leaf_size(x) P4D_SIZE
#endif
#ifndef pud_leaf_size
#define pud_leaf_size(x) PUD_SIZE
#endif
#ifndef pmd_leaf_size
#define pmd_leaf_size(x) PMD_SIZE
#endif
#ifndef pte_leaf_size
#define pte_leaf_size(x) PAGE_SIZE
#endif
/*
* Some architectures have MMUs that are configurable or selectable at boot
* time. These lead to variable PTRS_PER_x. For statically allocated arrays it
* helps to have a static maximum value.
*/
#ifndef MAX_PTRS_PER_PTE
#define MAX_PTRS_PER_PTE PTRS_PER_PTE
#endif
#ifndef MAX_PTRS_PER_PMD
#define MAX_PTRS_PER_PMD PTRS_PER_PMD
#endif
#ifndef MAX_PTRS_PER_PUD
#define MAX_PTRS_PER_PUD PTRS_PER_PUD
#endif
#ifndef MAX_PTRS_PER_P4D
#define MAX_PTRS_PER_P4D PTRS_PER_P4D
#endif
/* description of effects of mapping type and prot in current implementation.
* this is due to the limited x86 page protection hardware. The expected
* behavior is in parens:
*
* map_type prot
* PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
* MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
* w: (no) no w: (no) no w: (yes) yes w: (no) no
* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
*
* MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
* w: (no) no w: (no) no w: (copy) copy w: (no) no
* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
*
* On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and
* MAP_PRIVATE (with Enhanced PAN supported):
* r: (no) no
* w: (no) no
* x: (yes) yes
*/
#define DECLARE_VM_GET_PAGE_PROT \
pgprot_t vm_get_page_prot(unsigned long vm_flags) \
{ \
return protection_map[vm_flags & \
(VM_READ | VM_WRITE | VM_EXEC | VM_SHARED)]; \
} \
EXPORT_SYMBOL(vm_get_page_prot);
#endif /* _LINUX_PGTABLE_H */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
*/
#include <linux/dcache.h>
#include <linux/fs.h>
#include <linux/gfp.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/srcu.h>
#include <linux/fsnotify_backend.h>
#include "fsnotify.h"
/*
* Clear all of the marks on an inode when it is being evicted from core
*/
void __fsnotify_inode_delete(struct inode *inode)
{
fsnotify_clear_marks_by_inode(inode);
}
EXPORT_SYMBOL_GPL(__fsnotify_inode_delete);
void __fsnotify_vfsmount_delete(struct vfsmount *mnt)
{
fsnotify_clear_marks_by_mount(mnt);
}
/**
* fsnotify_unmount_inodes - an sb is unmounting. handle any watched inodes.
* @sb: superblock being unmounted.
*
* Called during unmount with no locks held, so needs to be safe against
* concurrent modifiers. We temporarily drop sb->s_inode_list_lock and CAN block.
*/
static void fsnotify_unmount_inodes(struct super_block *sb)
{
struct inode *inode, *iput_inode = NULL;
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
/*
* We cannot __iget() an inode in state I_FREEING,
* I_WILL_FREE, or I_NEW which is fine because by that point
* the inode cannot have any associated watches.
*/
spin_lock(&inode->i_lock);
if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) {
spin_unlock(&inode->i_lock);
continue;
}
/*
* If i_count is zero, the inode cannot have any watches and
* doing an __iget/iput with SB_ACTIVE clear would actually
* evict all inodes with zero i_count from icache which is
* unnecessarily violent and may in fact be illegal to do.
* However, we should have been called /after/ evict_inodes
* removed all zero refcount inodes, in any case. Test to
* be sure.
*/
if (!atomic_read(&inode->i_count)) {
spin_unlock(&inode->i_lock);
continue;
}
__iget(inode);
spin_unlock(&inode->i_lock);
spin_unlock(&sb->s_inode_list_lock);
iput(iput_inode);
/* for each watch, send FS_UNMOUNT and then remove it */
fsnotify_inode(inode, FS_UNMOUNT);
fsnotify_inode_delete(inode);
iput_inode = inode;
cond_resched();
spin_lock(&sb->s_inode_list_lock);
}
spin_unlock(&sb->s_inode_list_lock);
iput(iput_inode);
}
void fsnotify_sb_delete(struct super_block *sb)
{
fsnotify_unmount_inodes(sb);
fsnotify_clear_marks_by_sb(sb);
/* Wait for outstanding object references from connectors */
wait_var_event(&sb->s_fsnotify_connectors,
!atomic_long_read(&sb->s_fsnotify_connectors));
}
/*
* Given an inode, first check if we care what happens to our children. Inotify
* and dnotify both tell their parents about events. If we care about any event
* on a child we run all of our children and set a dentry flag saying that the
* parent cares. Thus when an event happens on a child it can quickly tell
* if there is a need to find a parent and send the event to the parent.
*/
void __fsnotify_update_child_dentry_flags(struct inode *inode)
{
struct dentry *alias;
int watched;
if (!S_ISDIR(inode->i_mode))
return;
/* determine if the children should tell inode about their events */
watched = fsnotify_inode_watches_children(inode);
spin_lock(&inode->i_lock);
/* run all of the dentries associated with this inode. Since this is a
* directory, there damn well better only be one item on this list */
hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
struct dentry *child;
/* run all of the children of the original inode and fix their
* d_flags to indicate parental interest (their parent is the
* original inode) */
spin_lock(&alias->d_lock);
hlist_for_each_entry(child, &alias->d_children, d_sib) {
if (!child->d_inode)
continue;
spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED);
if (watched)
child->d_flags |= DCACHE_FSNOTIFY_PARENT_WATCHED;
else
child->d_flags &= ~DCACHE_FSNOTIFY_PARENT_WATCHED;
spin_unlock(&child->d_lock);
}
spin_unlock(&alias->d_lock);
}
spin_unlock(&inode->i_lock);
}
/* Are inode/sb/mount interested in parent and name info with this event? */
static bool fsnotify_event_needs_parent(struct inode *inode, __u32 mnt_mask,
__u32 mask)
{
__u32 marks_mask = 0;
/* We only send parent/name to inode/sb/mount for events on non-dir */
if (mask & FS_ISDIR)
return false;
/*
* All events that are possible on child can also may be reported with
* parent/name info to inode/sb/mount. Otherwise, a watching parent
* could result in events reported with unexpected name info to sb/mount.
*/
BUILD_BUG_ON(FS_EVENTS_POSS_ON_CHILD & ~FS_EVENTS_POSS_TO_PARENT);
/* Did either inode/sb/mount subscribe for events with parent/name? */
marks_mask |= fsnotify_parent_needed_mask(inode->i_fsnotify_mask); marks_mask |= fsnotify_parent_needed_mask(inode->i_sb->s_fsnotify_mask); marks_mask |= fsnotify_parent_needed_mask(mnt_mask);
/* Did they subscribe for this event with parent/name info? */
return mask & marks_mask;
}
/* Are there any inode/mount/sb objects that are interested in this event? */
static inline bool fsnotify_object_watched(struct inode *inode, __u32 mnt_mask,
__u32 mask)
{
__u32 marks_mask = inode->i_fsnotify_mask | mnt_mask | inode->i_sb->s_fsnotify_mask;
return mask & marks_mask & ALL_FSNOTIFY_EVENTS;
}
/*
* Notify this dentry's parent about a child's events with child name info
* if parent is watching or if inode/sb/mount are interested in events with
* parent and name info.
*
* Notify only the child without name info if parent is not watching and
* inode/sb/mount are not interested in events with parent and name info.
*/
int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
int data_type)
{
const struct path *path = fsnotify_data_path(data, data_type); __u32 mnt_mask = path ? real_mount(path->mnt)->mnt_fsnotify_mask : 0; struct inode *inode = d_inode(dentry);
struct dentry *parent;
bool parent_watched = dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED;
bool parent_needed, parent_interested;
__u32 p_mask;
struct inode *p_inode = NULL;
struct name_snapshot name;
struct qstr *file_name = NULL;
int ret = 0;
/* Optimize the likely case of nobody watching this path */
if (likely(!parent_watched &&
!fsnotify_object_watched(inode, mnt_mask, mask)))
return 0;
parent = NULL;
parent_needed = fsnotify_event_needs_parent(inode, mnt_mask, mask); if (!parent_watched && !parent_needed)
goto notify;
/* Does parent inode care about events on children? */
parent = dget_parent(dentry);
p_inode = parent->d_inode;
p_mask = fsnotify_inode_watches_children(p_inode); if (unlikely(parent_watched && !p_mask)) __fsnotify_update_child_dentry_flags(p_inode);
/*
* Include parent/name in notification either if some notification
* groups require parent info or the parent is interested in this event.
*/
parent_interested = mask & p_mask & ALL_FSNOTIFY_EVENTS; if (parent_needed || parent_interested) {
/* When notifying parent, child should be passed as data */
WARN_ON_ONCE(inode != fsnotify_data_inode(data, data_type));
/* Notify both parent and child with child name info */
take_dentry_name_snapshot(&name, dentry);
file_name = &name.name;
if (parent_interested)
mask |= FS_EVENT_ON_CHILD;
}
notify:
ret = fsnotify(mask, data, data_type, p_inode, file_name, inode, 0);
if (file_name)
release_dentry_name_snapshot(&name);
dput(parent);
return ret;
}
EXPORT_SYMBOL_GPL(__fsnotify_parent);
static int fsnotify_handle_inode_event(struct fsnotify_group *group,
struct fsnotify_mark *inode_mark,
u32 mask, const void *data, int data_type,
struct inode *dir, const struct qstr *name,
u32 cookie)
{
const struct path *path = fsnotify_data_path(data, data_type);
struct inode *inode = fsnotify_data_inode(data, data_type);
const struct fsnotify_ops *ops = group->ops;
if (WARN_ON_ONCE(!ops->handle_inode_event))
return 0;
if (WARN_ON_ONCE(!inode && !dir))
return 0;
if ((inode_mark->flags & FSNOTIFY_MARK_FLAG_EXCL_UNLINK) &&
path && d_unlinked(path->dentry))
return 0;
/* Check interest of this mark in case event was sent with two marks */
if (!(mask & inode_mark->mask & ALL_FSNOTIFY_EVENTS))
return 0;
return ops->handle_inode_event(inode_mark, mask, inode, dir, name, cookie);
}
static int fsnotify_handle_event(struct fsnotify_group *group, __u32 mask,
const void *data, int data_type,
struct inode *dir, const struct qstr *name,
u32 cookie, struct fsnotify_iter_info *iter_info)
{
struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info); struct fsnotify_mark *parent_mark = fsnotify_iter_parent_mark(iter_info);
int ret;
if (WARN_ON_ONCE(fsnotify_iter_sb_mark(iter_info)) || WARN_ON_ONCE(fsnotify_iter_vfsmount_mark(iter_info)))
return 0;
/*
* For FS_RENAME, 'dir' is old dir and 'data' is new dentry.
* The only ->handle_inode_event() backend that supports FS_RENAME is
* dnotify, where it means file was renamed within same parent.
*/
if (mask & FS_RENAME) { struct dentry *moved = fsnotify_data_dentry(data, data_type); if (dir != moved->d_parent->d_inode)
return 0;
}
if (parent_mark) { ret = fsnotify_handle_inode_event(group, parent_mark, mask,
data, data_type, dir, name, 0);
if (ret)
return ret;
}
if (!inode_mark)
return 0;
if (mask & FS_EVENT_ON_CHILD) {
/*
* Some events can be sent on both parent dir and child marks
* (e.g. FS_ATTRIB). If both parent dir and child are
* watching, report the event once to parent dir with name (if
* interested) and once to child without name (if interested).
* The child watcher is expecting an event without a file name
* and without the FS_EVENT_ON_CHILD flag.
*/
mask &= ~FS_EVENT_ON_CHILD;
dir = NULL;
name = NULL;
}
return fsnotify_handle_inode_event(group, inode_mark, mask, data, data_type,
dir, name, cookie);
}
static int send_to_group(__u32 mask, const void *data, int data_type,
struct inode *dir, const struct qstr *file_name,
u32 cookie, struct fsnotify_iter_info *iter_info)
{
struct fsnotify_group *group = NULL;
__u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS);
__u32 marks_mask = 0;
__u32 marks_ignore_mask = 0;
bool is_dir = mask & FS_ISDIR;
struct fsnotify_mark *mark;
int type;
if (!iter_info->report_mask)
return 0;
/* clear ignored on inode modification */
if (mask & FS_MODIFY) { fsnotify_foreach_iter_mark_type(iter_info, mark, type) { if (!(mark->flags &
FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY))
mark->ignore_mask = 0;
}
}
/* Are any of the group marks interested in this event? */
fsnotify_foreach_iter_mark_type(iter_info, mark, type) { group = mark->group;
marks_mask |= mark->mask;
marks_ignore_mask |= fsnotify_effective_ignore_mask(mark, is_dir, type);
}
pr_debug("%s: group=%p mask=%x marks_mask=%x marks_ignore_mask=%x data=%p data_type=%d dir=%p cookie=%d\n",
__func__, group, mask, marks_mask, marks_ignore_mask,
data, data_type, dir, cookie);
if (!(test_mask & marks_mask & ~marks_ignore_mask))
return 0;
if (group->ops->handle_event) { return group->ops->handle_event(group, mask, data, data_type, dir,
file_name, cookie, iter_info);
}
return fsnotify_handle_event(group, mask, data, data_type, dir,
file_name, cookie, iter_info);
}
static struct fsnotify_mark *fsnotify_first_mark(struct fsnotify_mark_connector **connp)
{
struct fsnotify_mark_connector *conn;
struct hlist_node *node = NULL;
conn = srcu_dereference(*connp, &fsnotify_mark_srcu);
if (conn)
node = srcu_dereference(conn->list.first, &fsnotify_mark_srcu);
return hlist_entry_safe(node, struct fsnotify_mark, obj_list);
}
static struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark)
{
struct hlist_node *node = NULL;
if (mark)
node = srcu_dereference(mark->obj_list.next,
&fsnotify_mark_srcu);
return hlist_entry_safe(node, struct fsnotify_mark, obj_list);
}
/*
* iter_info is a multi head priority queue of marks.
* Pick a subset of marks from queue heads, all with the same group
* and set the report_mask to a subset of the selected marks.
* Returns false if there are no more groups to iterate.
*/
static bool fsnotify_iter_select_report_types(
struct fsnotify_iter_info *iter_info)
{
struct fsnotify_group *max_prio_group = NULL;
struct fsnotify_mark *mark;
int type;
/* Choose max prio group among groups of all queue heads */
fsnotify_foreach_iter_type(type) { mark = iter_info->marks[type];
if (mark &&
fsnotify_compare_groups(max_prio_group, mark->group) > 0) max_prio_group = mark->group;
}
if (!max_prio_group)
return false;
/* Set the report mask for marks from same group as max prio group */
iter_info->current_group = max_prio_group;
iter_info->report_mask = 0;
fsnotify_foreach_iter_type(type) { mark = iter_info->marks[type]; if (mark && mark->group == iter_info->current_group) {
/*
* FSNOTIFY_ITER_TYPE_PARENT indicates that this inode
* is watching children and interested in this event,
* which is an event possible on child.
* But is *this mark* watching children?
*/
if (type == FSNOTIFY_ITER_TYPE_PARENT && !(mark->mask & FS_EVENT_ON_CHILD) && !(fsnotify_ignore_mask(mark) & FS_EVENT_ON_CHILD))
continue;
fsnotify_iter_set_report_type(iter_info, type);
}
}
return true;
}
/*
* Pop from iter_info multi head queue, the marks that belong to the group of
* current iteration step.
*/
static void fsnotify_iter_next(struct fsnotify_iter_info *iter_info)
{
struct fsnotify_mark *mark;
int type;
/*
* We cannot use fsnotify_foreach_iter_mark_type() here because we
* may need to advance a mark of type X that belongs to current_group
* but was not selected for reporting.
*/
fsnotify_foreach_iter_type(type) { mark = iter_info->marks[type]; if (mark && mark->group == iter_info->current_group) iter_info->marks[type] = fsnotify_next_mark(iter_info->marks[type]);
}
}
/*
* fsnotify - This is the main call to fsnotify.
*
* The VFS calls into hook specific functions in linux/fsnotify.h.
* Those functions then in turn call here. Here will call out to all of the
* registered fsnotify_group. Those groups can then use the notification event
* in whatever means they feel necessary.
*
* @mask: event type and flags
* @data: object that event happened on
* @data_type: type of object for fanotify_data_XXX() accessors
* @dir: optional directory associated with event -
* if @file_name is not NULL, this is the directory that
* @file_name is relative to
* @file_name: optional file name associated with event
* @inode: optional inode associated with event -
* If @dir and @inode are both non-NULL, event may be
* reported to both.
* @cookie: inotify rename cookie
*/
int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir,
const struct qstr *file_name, struct inode *inode, u32 cookie)
{
const struct path *path = fsnotify_data_path(data, data_type); struct super_block *sb = fsnotify_data_sb(data, data_type); struct fsnotify_iter_info iter_info = {};
struct mount *mnt = NULL;
struct inode *inode2 = NULL;
struct dentry *moved;
int inode2_type;
int ret = 0;
__u32 test_mask, marks_mask;
if (path)
mnt = real_mount(path->mnt); if (!inode) {
/* Dirent event - report on TYPE_INODE to dir */
inode = dir;
/* For FS_RENAME, inode is old_dir and inode2 is new_dir */
if (mask & FS_RENAME) { moved = fsnotify_data_dentry(data, data_type); inode2 = moved->d_parent->d_inode;
inode2_type = FSNOTIFY_ITER_TYPE_INODE2;
}
} else if (mask & FS_EVENT_ON_CHILD) {
/*
* Event on child - report on TYPE_PARENT to dir if it is
* watching children and on TYPE_INODE to child.
*/
inode2 = dir;
inode2_type = FSNOTIFY_ITER_TYPE_PARENT;
}
/*
* Optimization: srcu_read_lock() has a memory barrier which can
* be expensive. It protects walking the *_fsnotify_marks lists.
* However, if we do not walk the lists, we do not have to do
* SRCU because we have no references to any objects and do not
* need SRCU to keep them "alive".
*/
if (!sb->s_fsnotify_marks && (!mnt || !mnt->mnt_fsnotify_marks) && (!inode || !inode->i_fsnotify_marks) && (!inode2 || !inode2->i_fsnotify_marks))
return 0;
marks_mask = sb->s_fsnotify_mask;
if (mnt)
marks_mask |= mnt->mnt_fsnotify_mask; if (inode) marks_mask |= inode->i_fsnotify_mask; if (inode2) marks_mask |= inode2->i_fsnotify_mask;
/*
* If this is a modify event we may need to clear some ignore masks.
* In that case, the object with ignore masks will have the FS_MODIFY
* event in its mask.
* Otherwise, return if none of the marks care about this type of event.
*/
test_mask = (mask & ALL_FSNOTIFY_EVENTS);
if (!(test_mask & marks_mask))
return 0;
iter_info.srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
iter_info.marks[FSNOTIFY_ITER_TYPE_SB] =
fsnotify_first_mark(&sb->s_fsnotify_marks);
if (mnt) {
iter_info.marks[FSNOTIFY_ITER_TYPE_VFSMOUNT] =
fsnotify_first_mark(&mnt->mnt_fsnotify_marks);
}
if (inode) {
iter_info.marks[FSNOTIFY_ITER_TYPE_INODE] =
fsnotify_first_mark(&inode->i_fsnotify_marks);
}
if (inode2) {
iter_info.marks[inode2_type] =
fsnotify_first_mark(&inode2->i_fsnotify_marks);
}
/*
* We need to merge inode/vfsmount/sb mark lists so that e.g. inode mark
* ignore masks are properly reflected for mount/sb mark notifications.
* That's why this traversal is so complicated...
*/
while (fsnotify_iter_select_report_types(&iter_info)) { ret = send_to_group(mask, data, data_type, dir, file_name,
cookie, &iter_info);
if (ret && (mask & ALL_FSNOTIFY_PERM_EVENTS))
goto out;
fsnotify_iter_next(&iter_info);
}
ret = 0;
out:
srcu_read_unlock(&fsnotify_mark_srcu, iter_info.srcu_idx);
return ret;
}
EXPORT_SYMBOL_GPL(fsnotify);
static __init int fsnotify_init(void)
{
int ret;
BUILD_BUG_ON(HWEIGHT32(ALL_FSNOTIFY_BITS) != 23);
ret = init_srcu_struct(&fsnotify_mark_srcu);
if (ret)
panic("initializing fsnotify_mark_srcu");
fsnotify_mark_connector_cachep = KMEM_CACHE(fsnotify_mark_connector,
SLAB_PANIC);
return 0;
}
core_initcall(fsnotify_init);
// SPDX-License-Identifier: GPL-2.0-or-later
/* Virtio ring implementation.
*
* Copyright 2007 Rusty Russell IBM Corporation
*/
#include <linux/virtio.h>
#include <linux/virtio_ring.h>
#include <linux/virtio_config.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/hrtimer.h>
#include <linux/dma-mapping.h>
#include <linux/kmsan.h>
#include <linux/spinlock.h>
#include <xen/xen.h>
#ifdef DEBUG
/* For development, we want to crash whenever the ring is screwed. */
#define BAD_RING(_vq, fmt, args...) \
do { \
dev_err(&(_vq)->vq.vdev->dev, \
"%s:"fmt, (_vq)->vq.name, ##args); \
BUG(); \
} while (0)
/* Caller is supposed to guarantee no reentry. */
#define START_USE(_vq) \
do { \
if ((_vq)->in_use) \
panic("%s:in_use = %i\n", \
(_vq)->vq.name, (_vq)->in_use); \
(_vq)->in_use = __LINE__; \
} while (0)
#define END_USE(_vq) \
do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
#define LAST_ADD_TIME_UPDATE(_vq) \
do { \
ktime_t now = ktime_get(); \
\
/* No kick or get, with .1 second between? Warn. */ \
if ((_vq)->last_add_time_valid) \
WARN_ON(ktime_to_ms(ktime_sub(now, \
(_vq)->last_add_time)) > 100); \
(_vq)->last_add_time = now; \
(_vq)->last_add_time_valid = true; \
} while (0)
#define LAST_ADD_TIME_CHECK(_vq) \
do { \
if ((_vq)->last_add_time_valid) { \
WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
(_vq)->last_add_time)) > 100); \
} \
} while (0)
#define LAST_ADD_TIME_INVALID(_vq) \
((_vq)->last_add_time_valid = false)
#else
#define BAD_RING(_vq, fmt, args...) \
do { \
dev_err(&_vq->vq.vdev->dev, \
"%s:"fmt, (_vq)->vq.name, ##args); \
(_vq)->broken = true; \
} while (0)
#define START_USE(vq)
#define END_USE(vq)
#define LAST_ADD_TIME_UPDATE(vq)
#define LAST_ADD_TIME_CHECK(vq)
#define LAST_ADD_TIME_INVALID(vq)
#endif
struct vring_desc_state_split {
void *data; /* Data for callback. */
struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
};
struct vring_desc_state_packed {
void *data; /* Data for callback. */
struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
u16 num; /* Descriptor list length. */
u16 last; /* The last desc state in a list. */
};
struct vring_desc_extra {
dma_addr_t addr; /* Descriptor DMA addr. */
u32 len; /* Descriptor length. */
u16 flags; /* Descriptor flags. */
u16 next; /* The next desc state in a list. */
};
struct vring_virtqueue_split {
/* Actual memory layout for this queue. */
struct vring vring;
/* Last written value to avail->flags */
u16 avail_flags_shadow;
/*
* Last written value to avail->idx in
* guest byte order.
*/
u16 avail_idx_shadow;
/* Per-descriptor state. */
struct vring_desc_state_split *desc_state;
struct vring_desc_extra *desc_extra;
/* DMA address and size information */
dma_addr_t queue_dma_addr;
size_t queue_size_in_bytes;
/*
* The parameters for creating vrings are reserved for creating new
* vring.
*/
u32 vring_align;
bool may_reduce_num;
};
struct vring_virtqueue_packed {
/* Actual memory layout for this queue. */
struct {
unsigned int num;
struct vring_packed_desc *desc;
struct vring_packed_desc_event *driver;
struct vring_packed_desc_event *device;
} vring;
/* Driver ring wrap counter. */
bool avail_wrap_counter;
/* Avail used flags. */
u16 avail_used_flags;
/* Index of the next avail descriptor. */
u16 next_avail_idx;
/*
* Last written value to driver->flags in
* guest byte order.
*/
u16 event_flags_shadow;
/* Per-descriptor state. */
struct vring_desc_state_packed *desc_state;
struct vring_desc_extra *desc_extra;
/* DMA address and size information */
dma_addr_t ring_dma_addr;
dma_addr_t driver_event_dma_addr;
dma_addr_t device_event_dma_addr;
size_t ring_size_in_bytes;
size_t event_size_in_bytes;
};
struct vring_virtqueue {
struct virtqueue vq;
/* Is this a packed ring? */
bool packed_ring;
/* Is DMA API used? */
bool use_dma_api;
/* Can we use weak barriers? */
bool weak_barriers;
/* Other side has made a mess, don't try any more. */
bool broken;
/* Host supports indirect buffers */
bool indirect;
/* Host publishes avail event idx */
bool event;
/* Do DMA mapping by driver */
bool premapped;
/* Do unmap or not for desc. Just when premapped is False and
* use_dma_api is true, this is true.
*/
bool do_unmap;
/* Head of free buffer list. */
unsigned int free_head;
/* Number we've added since last sync. */
unsigned int num_added;
/* Last used index we've seen.
* for split ring, it just contains last used index
* for packed ring:
* bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index.
* bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter.
*/
u16 last_used_idx;
/* Hint for event idx: already triggered no need to disable. */
bool event_triggered;
union {
/* Available for split ring */
struct vring_virtqueue_split split;
/* Available for packed ring */
struct vring_virtqueue_packed packed;
};
/* How to notify other side. FIXME: commonalize hcalls! */
bool (*notify)(struct virtqueue *vq);
/* DMA, allocation, and size information */
bool we_own_ring;
/* Device used for doing DMA */
struct device *dma_dev;
#ifdef DEBUG
/* They're supposed to lock for us. */
unsigned int in_use;
/* Figure out if their kicks are too delayed. */
bool last_add_time_valid;
ktime_t last_add_time;
#endif
};
static struct virtqueue *__vring_new_virtqueue(unsigned int index,
struct vring_virtqueue_split *vring_split,
struct virtio_device *vdev,
bool weak_barriers,
bool context,
bool (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name,
struct device *dma_dev);
static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
static void vring_free(struct virtqueue *_vq);
/*
* Helpers.
*/
#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
unsigned int total_sg)
{
/*
* If the host supports indirect descriptor tables, and we have multiple
* buffers, then go indirect. FIXME: tune this threshold
*/
return (vq->indirect && total_sg > 1 && vq->vq.num_free);
}
/*
* Modern virtio devices have feature bits to specify whether they need a
* quirk and bypass the IOMMU. If not there, just use the DMA API.
*
* If there, the interaction between virtio and DMA API is messy.
*
* On most systems with virtio, physical addresses match bus addresses,
* and it doesn't particularly matter whether we use the DMA API.
*
* On some systems, including Xen and any system with a physical device
* that speaks virtio behind a physical IOMMU, we must use the DMA API
* for virtio DMA to work at all.
*
* On other systems, including SPARC and PPC64, virtio-pci devices are
* enumerated as though they are behind an IOMMU, but the virtio host
* ignores the IOMMU, so we must either pretend that the IOMMU isn't
* there or somehow map everything as the identity.
*
* For the time being, we preserve historic behavior and bypass the DMA
* API.
*
* TODO: install a per-device DMA ops structure that does the right thing
* taking into account all the above quirks, and use the DMA API
* unconditionally on data path.
*/
static bool vring_use_dma_api(const struct virtio_device *vdev)
{
if (!virtio_has_dma_quirk(vdev))
return true;
/* Otherwise, we are left to guess. */
/*
* In theory, it's possible to have a buggy QEMU-supposed
* emulated Q35 IOMMU and Xen enabled at the same time. On
* such a configuration, virtio has never worked and will
* not work without an even larger kludge. Instead, enable
* the DMA API if we're a Xen guest, which at least allows
* all of the sensible Xen configurations to work correctly.
*/
if (xen_domain())
return true;
return false;
}
size_t virtio_max_dma_size(const struct virtio_device *vdev)
{
size_t max_segment_size = SIZE_MAX;
if (vring_use_dma_api(vdev))
max_segment_size = dma_max_mapping_size(vdev->dev.parent);
return max_segment_size;
}
EXPORT_SYMBOL_GPL(virtio_max_dma_size);
static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
struct device *dma_dev)
{
if (vring_use_dma_api(vdev)) {
return dma_alloc_coherent(dma_dev, size,
dma_handle, flag);
} else {
void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
if (queue) {
phys_addr_t phys_addr = virt_to_phys(queue);
*dma_handle = (dma_addr_t)phys_addr;
/*
* Sanity check: make sure we dind't truncate
* the address. The only arches I can find that
* have 64-bit phys_addr_t but 32-bit dma_addr_t
* are certain non-highmem MIPS and x86
* configurations, but these configurations
* should never allocate physical pages above 32
* bits, so this is fine. Just in case, throw a
* warning and abort if we end up with an
* unrepresentable address.
*/
if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
free_pages_exact(queue, PAGE_ALIGN(size));
return NULL;
}
}
return queue;
}
}
static void vring_free_queue(struct virtio_device *vdev, size_t size,
void *queue, dma_addr_t dma_handle,
struct device *dma_dev)
{
if (vring_use_dma_api(vdev))
dma_free_coherent(dma_dev, size, queue, dma_handle);
else
free_pages_exact(queue, PAGE_ALIGN(size));
}
/*
* The DMA ops on various arches are rather gnarly right now, and
* making all of the arch DMA ops work on the vring device itself
* is a mess.
*/
static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
{
return vq->dma_dev;
}
/* Map one sg entry. */
static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg,
enum dma_data_direction direction, dma_addr_t *addr)
{
if (vq->premapped) {
*addr = sg_dma_address(sg);
return 0;
}
if (!vq->use_dma_api) {
/*
* If DMA is not used, KMSAN doesn't know that the scatterlist
* is initialized by the hardware. Explicitly check/unpoison it
* depending on the direction.
*/
kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction);
*addr = (dma_addr_t)sg_phys(sg);
return 0;
}
/*
* We can't use dma_map_sg, because we don't use scatterlists in
* the way it expects (we don't guarantee that the scatterlist
* will exist for the lifetime of the mapping).
*/
*addr = dma_map_page(vring_dma_dev(vq),
sg_page(sg), sg->offset, sg->length,
direction);
if (dma_mapping_error(vring_dma_dev(vq), *addr))
return -ENOMEM;
return 0;
}
static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
void *cpu_addr, size_t size,
enum dma_data_direction direction)
{
if (!vq->use_dma_api)
return (dma_addr_t)virt_to_phys(cpu_addr);
return dma_map_single(vring_dma_dev(vq),
cpu_addr, size, direction);
}
static int vring_mapping_error(const struct vring_virtqueue *vq,
dma_addr_t addr)
{
if (!vq->use_dma_api)
return 0;
return dma_mapping_error(vring_dma_dev(vq), addr);
}
static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
{
vq->vq.num_free = num;
if (vq->packed_ring)
vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
else
vq->last_used_idx = 0;
vq->event_triggered = false;
vq->num_added = 0;
#ifdef DEBUG
vq->in_use = false;
vq->last_add_time_valid = false;
#endif
}
/*
* Split ring specific functions - *_split().
*/
static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
const struct vring_desc *desc)
{
u16 flags;
if (!vq->do_unmap)
return;
flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
dma_unmap_page(vring_dma_dev(vq),
virtio64_to_cpu(vq->vq.vdev, desc->addr),
virtio32_to_cpu(vq->vq.vdev, desc->len),
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
}
static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
unsigned int i)
{
struct vring_desc_extra *extra = vq->split.desc_extra;
u16 flags;
flags = extra[i].flags;
if (flags & VRING_DESC_F_INDIRECT) {
if (!vq->use_dma_api)
goto out;
dma_unmap_single(vring_dma_dev(vq),
extra[i].addr,
extra[i].len,
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
} else {
if (!vq->do_unmap)
goto out;
dma_unmap_page(vring_dma_dev(vq),
extra[i].addr,
extra[i].len,
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
}
out:
return extra[i].next;
}
static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
unsigned int total_sg,
gfp_t gfp)
{
struct vring_desc *desc;
unsigned int i;
/*
* We require lowmem mappings for the descriptors because
* otherwise virt_to_phys will give us bogus addresses in the
* virtqueue.
*/
gfp &= ~__GFP_HIGHMEM;
desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
if (!desc)
return NULL;
for (i = 0; i < total_sg; i++)
desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
return desc;
}
static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
struct vring_desc *desc,
unsigned int i,
dma_addr_t addr,
unsigned int len,
u16 flags,
bool indirect)
{
struct vring_virtqueue *vring = to_vvq(vq);
struct vring_desc_extra *extra = vring->split.desc_extra;
u16 next;
desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
desc[i].len = cpu_to_virtio32(vq->vdev, len);
if (!indirect) {
next = extra[i].next;
desc[i].next = cpu_to_virtio16(vq->vdev, next);
extra[i].addr = addr;
extra[i].len = len;
extra[i].flags = flags;
} else
next = virtio16_to_cpu(vq->vdev, desc[i].next);
return next;
}
static inline int virtqueue_add_split(struct virtqueue *_vq,
struct scatterlist *sgs[],
unsigned int total_sg,
unsigned int out_sgs,
unsigned int in_sgs,
void *data,
void *ctx,
gfp_t gfp)
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct scatterlist *sg;
struct vring_desc *desc;
unsigned int i, n, avail, descs_used, prev, err_idx;
int head;
bool indirect;
START_USE(vq);
BUG_ON(data == NULL);
BUG_ON(ctx && vq->indirect);
if (unlikely(vq->broken)) {
END_USE(vq);
return -EIO;
}
LAST_ADD_TIME_UPDATE(vq);
BUG_ON(total_sg == 0);
head = vq->free_head;
if (virtqueue_use_indirect(vq, total_sg))
desc = alloc_indirect_split(_vq, total_sg, gfp);
else {
desc = NULL;
WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
}
if (desc) {
/* Use a single buffer which doesn't continue */
indirect = true;
/* Set up rest to use this indirect table. */
i = 0;
descs_used = 1;
} else {
indirect = false;
desc = vq->split.vring.desc;
i = head;
descs_used = total_sg;
}
if (unlikely(vq->vq.num_free < descs_used)) {
pr_debug("Can't add buf len %i - avail = %i\n",
descs_used, vq->vq.num_free);
/* FIXME: for historical reasons, we force a notify here if
* there are outgoing parts to the buffer. Presumably the
* host should service the ring ASAP. */
if (out_sgs)
vq->notify(&vq->vq);
if (indirect)
kfree(desc);
END_USE(vq);
return -ENOSPC;
}
for (n = 0; n < out_sgs; n++) {
for (sg = sgs[n]; sg; sg = sg_next(sg)) {
dma_addr_t addr;
if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr))
goto unmap_release;
prev = i;
/* Note that we trust indirect descriptor
* table since it use stream DMA mapping.
*/
i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
VRING_DESC_F_NEXT,
indirect);
}
}
for (; n < (out_sgs + in_sgs); n++) {
for (sg = sgs[n]; sg; sg = sg_next(sg)) {
dma_addr_t addr;
if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr))
goto unmap_release;
prev = i;
/* Note that we trust indirect descriptor
* table since it use stream DMA mapping.
*/
i = virtqueue_add_desc_split(_vq, desc, i, addr,
sg->length,
VRING_DESC_F_NEXT |
VRING_DESC_F_WRITE,
indirect);
}
}
/* Last one doesn't continue. */
desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
if (!indirect && vq->do_unmap)
vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
~VRING_DESC_F_NEXT;
if (indirect) {
/* Now that the indirect table is filled in, map it. */
dma_addr_t addr = vring_map_single(
vq, desc, total_sg * sizeof(struct vring_desc),
DMA_TO_DEVICE);
if (vring_mapping_error(vq, addr)) {
if (vq->premapped)
goto free_indirect;
goto unmap_release;
}
virtqueue_add_desc_split(_vq, vq->split.vring.desc,
head, addr,
total_sg * sizeof(struct vring_desc),
VRING_DESC_F_INDIRECT,
false);
}
/* We're using some buffers from the free list. */
vq->vq.num_free -= descs_used;
/* Update free pointer */
if (indirect)
vq->free_head = vq->split.desc_extra[head].next;
else
vq->free_head = i;
/* Store token and indirect buffer state. */
vq->split.desc_state[head].data = data;
if (indirect)
vq->split.desc_state[head].indir_desc = desc;
else
vq->split.desc_state[head].indir_desc = ctx;
/* Put entry in available array (but don't update avail->idx until they
* do sync). */
avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
/* Descriptors and available array need to be set before we expose the
* new available array entries. */
virtio_wmb(vq->weak_barriers);
vq->split.avail_idx_shadow++;
vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
vq->split.avail_idx_shadow);
vq->num_added++;
pr_debug("Added buffer head %i to %p\n", head, vq);
END_USE(vq);
/* This is very unlikely, but theoretically possible. Kick
* just in case. */
if (unlikely(vq->num_added == (1 << 16) - 1))
virtqueue_kick(_vq);
return 0;
unmap_release:
err_idx = i;
if (indirect)
i = 0;
else
i = head;
for (n = 0; n < total_sg; n++) {
if (i == err_idx)
break;
if (indirect) {
vring_unmap_one_split_indirect(vq, &desc[i]);
i = virtio16_to_cpu(_vq->vdev, desc[i].next);
} else
i = vring_unmap_one_split(vq, i);
}
free_indirect:
if (indirect)
kfree(desc);
END_USE(vq);
return -ENOMEM;
}
static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 new, old;
bool needs_kick;
START_USE(vq);
/* We need to expose available array entries before checking avail
* event. */
virtio_mb(vq->weak_barriers);
old = vq->split.avail_idx_shadow - vq->num_added;
new = vq->split.avail_idx_shadow;
vq->num_added = 0;
LAST_ADD_TIME_CHECK(vq);
LAST_ADD_TIME_INVALID(vq);
if (vq->event) {
needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
vring_avail_event(&vq->split.vring)),
new, old);
} else {
needs_kick = !(vq->split.vring.used->flags &
cpu_to_virtio16(_vq->vdev,
VRING_USED_F_NO_NOTIFY));
}
END_USE(vq);
return needs_kick;
}
static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
void **ctx)
{
unsigned int i, j;
__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
/* Clear data ptr. */
vq->split.desc_state[head].data = NULL;
/* Put back on free list: unmap first-level descriptors and find end */
i = head;
while (vq->split.vring.desc[i].flags & nextflag) {
vring_unmap_one_split(vq, i);
i = vq->split.desc_extra[i].next;
vq->vq.num_free++;
}
vring_unmap_one_split(vq, i);
vq->split.desc_extra[i].next = vq->free_head;
vq->free_head = head;
/* Plus final descriptor */
vq->vq.num_free++;
if (vq->indirect) {
struct vring_desc *indir_desc =
vq->split.desc_state[head].indir_desc;
u32 len;
/* Free the indirect table, if any, now that it's unmapped. */
if (!indir_desc)
return;
len = vq->split.desc_extra[head].len;
BUG_ON(!(vq->split.desc_extra[head].flags &
VRING_DESC_F_INDIRECT));
BUG_ON(len == 0 || len % sizeof(struct vring_desc));
if (vq->do_unmap) {
for (j = 0; j < len / sizeof(struct vring_desc); j++)
vring_unmap_one_split_indirect(vq, &indir_desc[j]);
}
kfree(indir_desc);
vq->split.desc_state[head].indir_desc = NULL;
} else if (ctx) {
*ctx = vq->split.desc_state[head].indir_desc;
}
}
static bool more_used_split(const struct vring_virtqueue *vq)
{
return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
vq->split.vring.used->idx);
}
static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
unsigned int *len,
void **ctx)
{
struct vring_virtqueue *vq = to_vvq(_vq);
void *ret;
unsigned int i;
u16 last_used;
START_USE(vq);
if (unlikely(vq->broken)) {
END_USE(vq);
return NULL;
}
if (!more_used_split(vq)) {
pr_debug("No more buffers in queue\n");
END_USE(vq);
return NULL;
}
/* Only get used array entries after they have been exposed by host. */
virtio_rmb(vq->weak_barriers);
last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
i = virtio32_to_cpu(_vq->vdev,
vq->split.vring.used->ring[last_used].id);
*len = virtio32_to_cpu(_vq->vdev,
vq->split.vring.used->ring[last_used].len);
if (unlikely(i >= vq->split.vring.num)) {
BAD_RING(vq, "id %u out of range\n", i);
return NULL;
}
if (unlikely(!vq->split.desc_state[i].data)) {
BAD_RING(vq, "id %u is not a head!\n", i);
return NULL;
}
/* detach_buf_split clears data, so grab it now. */
ret = vq->split.desc_state[i].data;
detach_buf_split(vq, i, ctx);
vq->last_used_idx++;
/* If we expect an interrupt for the next entry, tell host
* by writing event index and flush out the write before
* the read in the next get_buf call. */
if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
virtio_store_mb(vq->weak_barriers,
&vring_used_event(&vq->split.vring),
cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
LAST_ADD_TIME_INVALID(vq);
END_USE(vq);
return ret;
}
static void virtqueue_disable_cb_split(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
/*
* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
if (vq->event)
/* TODO: this is a hack. Figure out a cleaner value to write. */
vring_used_event(&vq->split.vring) = 0x0;
else
vq->split.vring.avail->flags =
cpu_to_virtio16(_vq->vdev,
vq->split.avail_flags_shadow);
}
}
static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 last_used_idx;
START_USE(vq);
/* We optimistically turn back on interrupts, then check if there was
* more to do. */
/* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
* either clear the flags bit or point the event index at the next
* entry. Always do both to keep code simple. */
if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event)
vq->split.vring.avail->flags =
cpu_to_virtio16(_vq->vdev,
vq->split.avail_flags_shadow);
}
vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
last_used_idx = vq->last_used_idx);
END_USE(vq);
return last_used_idx;
}
static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
{
struct vring_virtqueue *vq = to_vvq(_vq);
return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
vq->split.vring.used->idx);
}
static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 bufs;
START_USE(vq);
/* We optimistically turn back on interrupts, then check if there was
* more to do. */
/* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
* either clear the flags bit or point the event index at the next
* entry. Always update the event index to keep code simple. */
if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event)
vq->split.vring.avail->flags =
cpu_to_virtio16(_vq->vdev,
vq->split.avail_flags_shadow);
}
/* TODO: tune this threshold */
bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
virtio_store_mb(vq->weak_barriers,
&vring_used_event(&vq->split.vring),
cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
- vq->last_used_idx) > bufs)) {
END_USE(vq);
return false;
}
END_USE(vq);
return true;
}
static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
unsigned int i;
void *buf;
START_USE(vq);
for (i = 0; i < vq->split.vring.num; i++) {
if (!vq->split.desc_state[i].data)
continue;
/* detach_buf_split clears data, so grab it now. */
buf = vq->split.desc_state[i].data;
detach_buf_split(vq, i, NULL);
vq->split.avail_idx_shadow--;
vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
vq->split.avail_idx_shadow);
END_USE(vq);
return buf;
}
/* That should have freed everything. */
BUG_ON(vq->vq.num_free != vq->split.vring.num);
END_USE(vq);
return NULL;
}
static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
struct vring_virtqueue *vq)
{
struct virtio_device *vdev;
vdev = vq->vq.vdev;
vring_split->avail_flags_shadow = 0;
vring_split->avail_idx_shadow = 0;
/* No callback? Tell other side not to bother us. */
if (!vq->vq.callback) {
vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
if (!vq->event)
vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
vring_split->avail_flags_shadow);
}
}
static void virtqueue_reinit_split(struct vring_virtqueue *vq)
{
int num;
num = vq->split.vring.num;
vq->split.vring.avail->flags = 0;
vq->split.vring.avail->idx = 0;
/* reset avail event */
vq->split.vring.avail->ring[num] = 0;
vq->split.vring.used->flags = 0;
vq->split.vring.used->idx = 0;
/* reset used event */
*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;
virtqueue_init(vq, num);
virtqueue_vring_init_split(&vq->split, vq);
}
static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
struct vring_virtqueue_split *vring_split)
{
vq->split = *vring_split;
/* Put everything in free lists. */
vq->free_head = 0;
}
static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
{
struct vring_desc_state_split *state;
struct vring_desc_extra *extra;
u32 num = vring_split->vring.num;
state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL);
if (!state)
goto err_state;
extra = vring_alloc_desc_extra(num);
if (!extra)
goto err_extra;
memset(state, 0, num * sizeof(struct vring_desc_state_split));
vring_split->desc_state = state;
vring_split->desc_extra = extra;
return 0;
err_extra:
kfree(state);
err_state:
return -ENOMEM;
}
static void vring_free_split(struct vring_virtqueue_split *vring_split,
struct virtio_device *vdev, struct device *dma_dev)
{
vring_free_queue(vdev, vring_split->queue_size_in_bytes,
vring_split->vring.desc,
vring_split->queue_dma_addr,
dma_dev);
kfree(vring_split->desc_state);
kfree(vring_split->desc_extra);
}
static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
struct virtio_device *vdev,
u32 num,
unsigned int vring_align,
bool may_reduce_num,
struct device *dma_dev)
{
void *queue = NULL;
dma_addr_t dma_addr;
/* We assume num is a power of 2. */
if (!is_power_of_2(num)) {
dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
return -EINVAL;
}
/* TODO: allocate each queue chunk individually */
for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
&dma_addr,
GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
dma_dev);
if (queue)
break;
if (!may_reduce_num)
return -ENOMEM;
}
if (!num)
return -ENOMEM;
if (!queue) {
/* Try to get a single page. You are my only hope! */
queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
&dma_addr, GFP_KERNEL | __GFP_ZERO,
dma_dev);
}
if (!queue)
return -ENOMEM;
vring_init(&vring_split->vring, num, queue, vring_align);
vring_split->queue_dma_addr = dma_addr;
vring_split->queue_size_in_bytes = vring_size(num, vring_align);
vring_split->vring_align = vring_align;
vring_split->may_reduce_num = may_reduce_num;
return 0;
}
static struct virtqueue *vring_create_virtqueue_split(
unsigned int index,
unsigned int num,
unsigned int vring_align,
struct virtio_device *vdev,
bool weak_barriers,
bool may_reduce_num,
bool context,
bool (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name,
struct device *dma_dev)
{
struct vring_virtqueue_split vring_split = {};
struct virtqueue *vq;
int err;
err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
may_reduce_num, dma_dev);
if (err)
return NULL;
vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
context, notify, callback, name, dma_dev);
if (!vq) {
vring_free_split(&vring_split, vdev, dma_dev);
return NULL;
}
to_vvq(vq)->we_own_ring = true;
return vq;
}
static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
{
struct vring_virtqueue_split vring_split = {};
struct vring_virtqueue *vq = to_vvq(_vq);
struct virtio_device *vdev = _vq->vdev;
int err;
err = vring_alloc_queue_split(&vring_split, vdev, num,
vq->split.vring_align,
vq->split.may_reduce_num,
vring_dma_dev(vq));
if (err)
goto err;
err = vring_alloc_state_extra_split(&vring_split);
if (err)
goto err_state_extra;
vring_free(&vq->vq);
virtqueue_vring_init_split(&vring_split, vq);
virtqueue_init(vq, vring_split.vring.num);
virtqueue_vring_attach_split(vq, &vring_split);
return 0;
err_state_extra:
vring_free_split(&vring_split, vdev, vring_dma_dev(vq));
err:
virtqueue_reinit_split(vq);
return -ENOMEM;
}
/*
* Packed ring specific functions - *_packed().
*/
static bool packed_used_wrap_counter(u16 last_used_idx)
{
return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
}
static u16 packed_last_used(u16 last_used_idx)
{
return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
}
static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
const struct vring_desc_extra *extra)
{
u16 flags;
flags = extra->flags;
if (flags & VRING_DESC_F_INDIRECT) {
if (!vq->use_dma_api)
return;
dma_unmap_single(vring_dma_dev(vq),
extra->addr, extra->len,
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
} else {
if (!vq->do_unmap)
return;
dma_unmap_page(vring_dma_dev(vq),
extra->addr, extra->len,
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
}
}
static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
const struct vring_packed_desc *desc)
{
u16 flags;
if (!vq->do_unmap)
return;
flags = le16_to_cpu(desc->flags);
dma_unmap_page(vring_dma_dev(vq),
le64_to_cpu(desc->addr),
le32_to_cpu(desc->len),
(flags & VRING_DESC_F_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE);
}
static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
gfp_t gfp)
{
struct vring_packed_desc *desc;
/*
* We require lowmem mappings for the descriptors because
* otherwise virt_to_phys will give us bogus addresses in the
* virtqueue.
*/
gfp &= ~__GFP_HIGHMEM;
desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
return desc;
}
static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
struct scatterlist *sgs[],
unsigned int total_sg,
unsigned int out_sgs,
unsigned int in_sgs,
void *data,
gfp_t gfp)
{
struct vring_packed_desc *desc;
struct scatterlist *sg;
unsigned int i, n, err_idx;
u16 head, id;
dma_addr_t addr;
head = vq->packed.next_avail_idx;
desc = alloc_indirect_packed(total_sg, gfp);
if (!desc)
return -ENOMEM;
if (unlikely(vq->vq.num_free < 1)) {
pr_debug("Can't add buf len 1 - avail = 0\n");
kfree(desc);
END_USE(vq);
return -ENOSPC;
}
i = 0;
id = vq->free_head;
BUG_ON(id == vq->packed.vring.num);
for (n = 0; n < out_sgs + in_sgs; n++) {
for (sg = sgs[n]; sg; sg = sg_next(sg)) {
if (vring_map_one_sg(vq, sg, n < out_sgs ?
DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr))
goto unmap_release;
desc[i].flags = cpu_to_le16(n < out_sgs ?
0 : VRING_DESC_F_WRITE);
desc[i].addr = cpu_to_le64(addr);
desc[i].len = cpu_to_le32(sg->length);
i++;
}
}
/* Now that the indirect table is filled in, map it. */
addr = vring_map_single(vq, desc,
total_sg * sizeof(struct vring_packed_desc),
DMA_TO_DEVICE);
if (vring_mapping_error(vq, addr)) {
if (vq->premapped)
goto free_desc;
goto unmap_release;
}
vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
sizeof(struct vring_packed_desc));
vq->packed.vring.desc[head].id = cpu_to_le16(id);
if (vq->use_dma_api) {
vq->packed.desc_extra[id].addr = addr;
vq->packed.desc_extra[id].len = total_sg *
sizeof(struct vring_packed_desc);
vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
vq->packed.avail_used_flags;
}
/*
* A driver MUST NOT make the first descriptor in the list
* available before all subsequent descriptors comprising
* the list are made available.
*/
virtio_wmb(vq->weak_barriers);
vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
vq->packed.avail_used_flags);
/* We're using some buffers from the free list. */
vq->vq.num_free -= 1;
/* Update free pointer */
n = head + 1;
if (n >= vq->packed.vring.num) {
n = 0;
vq->packed.avail_wrap_counter ^= 1;
vq->packed.avail_used_flags ^=
1 << VRING_PACKED_DESC_F_AVAIL |
1 << VRING_PACKED_DESC_F_USED;
}
vq->packed.next_avail_idx = n;
vq->free_head = vq->packed.desc_extra[id].next;
/* Store token and indirect buffer state. */
vq->packed.desc_state[id].num = 1;
vq->packed.desc_state[id].data = data;
vq->packed.desc_state[id].indir_desc = desc;
vq->packed.desc_state[id].last = id;
vq->num_added += 1;
pr_debug("Added buffer head %i to %p\n", head, vq);
END_USE(vq);
return 0;
unmap_release:
err_idx = i;
for (i = 0; i < err_idx; i++)
vring_unmap_desc_packed(vq, &desc[i]);
free_desc:
kfree(desc);
END_USE(vq);
return -ENOMEM;
}
static inline int virtqueue_add_packed(struct virtqueue *_vq,
struct scatterlist *sgs[],
unsigned int total_sg,
unsigned int out_sgs,
unsigned int in_sgs,
void *data,
void *ctx,
gfp_t gfp)
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct vring_packed_desc *desc;
struct scatterlist *sg;
unsigned int i, n, c, descs_used, err_idx;
__le16 head_flags, flags;
u16 head, id, prev, curr, avail_used_flags;
int err;
START_USE(vq);
BUG_ON(data == NULL);
BUG_ON(ctx && vq->indirect);
if (unlikely(vq->broken)) {
END_USE(vq);
return -EIO;
}
LAST_ADD_TIME_UPDATE(vq);
BUG_ON(total_sg == 0);
if (virtqueue_use_indirect(vq, total_sg)) {
err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
in_sgs, data, gfp);
if (err != -ENOMEM) {
END_USE(vq);
return err;
}
/* fall back on direct */
}
head = vq->packed.next_avail_idx;
avail_used_flags = vq->packed.avail_used_flags;
WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
desc = vq->packed.vring.desc;
i = head;
descs_used = total_sg;
if (unlikely(vq->vq.num_free < descs_used)) {
pr_debug("Can't add buf len %i - avail = %i\n",
descs_used, vq->vq.num_free);
END_USE(vq);
return -ENOSPC;
}
id = vq->free_head;
BUG_ON(id == vq->packed.vring.num);
curr = id;
c = 0;
for (n = 0; n < out_sgs + in_sgs; n++) {
for (sg = sgs[n]; sg; sg = sg_next(sg)) {
dma_addr_t addr;
if (vring_map_one_sg(vq, sg, n < out_sgs ?
DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr))
goto unmap_release;
flags = cpu_to_le16(vq->packed.avail_used_flags |
(++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
(n < out_sgs ? 0 : VRING_DESC_F_WRITE));
if (i == head)
head_flags = flags;
else
desc[i].flags = flags;
desc[i].addr = cpu_to_le64(addr);
desc[i].len = cpu_to_le32(sg->length);
desc[i].id = cpu_to_le16(id);
if (unlikely(vq->use_dma_api)) {
vq->packed.desc_extra[curr].addr = addr;
vq->packed.desc_extra[curr].len = sg->length;
vq->packed.desc_extra[curr].flags =
le16_to_cpu(flags);
}
prev = curr;
curr = vq->packed.desc_extra[curr].next;
if ((unlikely(++i >= vq->packed.vring.num))) {
i = 0;
vq->packed.avail_used_flags ^=
1 << VRING_PACKED_DESC_F_AVAIL |
1 << VRING_PACKED_DESC_F_USED;
}
}
}
if (i <= head)
vq->packed.avail_wrap_counter ^= 1;
/* We're using some buffers from the free list. */
vq->vq.num_free -= descs_used;
/* Update free pointer */
vq->packed.next_avail_idx = i;
vq->free_head = curr;
/* Store token. */
vq->packed.desc_state[id].num = descs_used;
vq->packed.desc_state[id].data = data;
vq->packed.desc_state[id].indir_desc = ctx;
vq->packed.desc_state[id].last = prev;
/*
* A driver MUST NOT make the first descriptor in the list
* available before all subsequent descriptors comprising
* the list are made available.
*/
virtio_wmb(vq->weak_barriers);
vq->packed.vring.desc[head].flags = head_flags;
vq->num_added += descs_used;
pr_debug("Added buffer head %i to %p\n", head, vq);
END_USE(vq);
return 0;
unmap_release:
err_idx = i;
i = head;
curr = vq->free_head;
vq->packed.avail_used_flags = avail_used_flags;
for (n = 0; n < total_sg; n++) {
if (i == err_idx)
break;
vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
curr = vq->packed.desc_extra[curr].next;
i++;
if (i >= vq->packed.vring.num)
i = 0;
}
END_USE(vq);
return -EIO;
}
static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 new, old, off_wrap, flags, wrap_counter, event_idx;
bool needs_kick;
union {
struct {
__le16 off_wrap;
__le16 flags;
};
u32 u32;
} snapshot;
START_USE(vq);
/*
* We need to expose the new flags value before checking notification
* suppressions.
*/
virtio_mb(vq->weak_barriers);
old = vq->packed.next_avail_idx - vq->num_added;
new = vq->packed.next_avail_idx;
vq->num_added = 0;
snapshot.u32 = *(u32 *)vq->packed.vring.device;
flags = le16_to_cpu(snapshot.flags);
LAST_ADD_TIME_CHECK(vq);
LAST_ADD_TIME_INVALID(vq);
if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
goto out;
}
off_wrap = le16_to_cpu(snapshot.off_wrap);
wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
if (wrap_counter != vq->packed.avail_wrap_counter)
event_idx -= vq->packed.vring.num;
needs_kick = vring_need_event(event_idx, new, old);
out:
END_USE(vq);
return needs_kick;
}
static void detach_buf_packed(struct vring_virtqueue *vq,
unsigned int id, void **ctx)
{
struct vring_desc_state_packed *state = NULL;
struct vring_packed_desc *desc;
unsigned int i, curr;
state = &vq->packed.desc_state[id];
/* Clear data ptr. */
state->data = NULL;
vq->packed.desc_extra[state->last].next = vq->free_head;
vq->free_head = id;
vq->vq.num_free += state->num;
if (unlikely(vq->use_dma_api)) {
curr = id;
for (i = 0; i < state->num; i++) {
vring_unmap_extra_packed(vq,
&vq->packed.desc_extra[curr]);
curr = vq->packed.desc_extra[curr].next;
}
}
if (vq->indirect) {
u32 len;
/* Free the indirect table, if any, now that it's unmapped. */
desc = state->indir_desc;
if (!desc)
return;
if (vq->do_unmap) {
len = vq->packed.desc_extra[id].len;
for (i = 0; i < len / sizeof(struct vring_packed_desc);
i++)
vring_unmap_desc_packed(vq, &desc[i]);
}
kfree(desc);
state->indir_desc = NULL;
} else if (ctx) {
*ctx = state->indir_desc;
}
}
static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
u16 idx, bool used_wrap_counter)
{
bool avail, used;
u16 flags;
flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
return avail == used && used == used_wrap_counter;
}
static bool more_used_packed(const struct vring_virtqueue *vq)
{
u16 last_used;
u16 last_used_idx;
bool used_wrap_counter;
last_used_idx = READ_ONCE(vq->last_used_idx);
last_used = packed_last_used(last_used_idx);
used_wrap_counter = packed_used_wrap_counter(last_used_idx);
return is_used_desc_packed(vq, last_used, used_wrap_counter);
}
static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
unsigned int *len,
void **ctx)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 last_used, id, last_used_idx;
bool used_wrap_counter;
void *ret;
START_USE(vq);
if (unlikely(vq->broken)) {
END_USE(vq);
return NULL;
}
if (!more_used_packed(vq)) {
pr_debug("No more buffers in queue\n");
END_USE(vq);
return NULL;
}
/* Only get used elements after they have been exposed by host. */
virtio_rmb(vq->weak_barriers);
last_used_idx = READ_ONCE(vq->last_used_idx);
used_wrap_counter = packed_used_wrap_counter(last_used_idx);
last_used = packed_last_used(last_used_idx);
id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
if (unlikely(id >= vq->packed.vring.num)) {
BAD_RING(vq, "id %u out of range\n", id);
return NULL;
}
if (unlikely(!vq->packed.desc_state[id].data)) {
BAD_RING(vq, "id %u is not a head!\n", id);
return NULL;
}
/* detach_buf_packed clears data, so grab it now. */
ret = vq->packed.desc_state[id].data;
detach_buf_packed(vq, id, ctx);
last_used += vq->packed.desc_state[id].num;
if (unlikely(last_used >= vq->packed.vring.num)) {
last_used -= vq->packed.vring.num;
used_wrap_counter ^= 1;
}
last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
WRITE_ONCE(vq->last_used_idx, last_used);
/*
* If we expect an interrupt for the next entry, tell host
* by writing event index and flush out the write before
* the read in the next get_buf call.
*/
if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
virtio_store_mb(vq->weak_barriers,
&vq->packed.vring.driver->off_wrap,
cpu_to_le16(vq->last_used_idx));
LAST_ADD_TIME_INVALID(vq);
END_USE(vq);
return ret;
}
static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
/*
* If device triggered an event already it won't trigger one again:
* no need to disable.
*/
if (vq->event_triggered)
return;
vq->packed.vring.driver->flags =
cpu_to_le16(vq->packed.event_flags_shadow);
}
}
static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
START_USE(vq);
/*
* We optimistically turn back on interrupts, then check if there was
* more to do.
*/
if (vq->event) {
vq->packed.vring.driver->off_wrap =
cpu_to_le16(vq->last_used_idx);
/*
* We need to update event offset and event wrap
* counter first before updating event flags.
*/
virtio_wmb(vq->weak_barriers);
}
if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
vq->packed.event_flags_shadow = vq->event ?
VRING_PACKED_EVENT_FLAG_DESC :
VRING_PACKED_EVENT_FLAG_ENABLE;
vq->packed.vring.driver->flags =
cpu_to_le16(vq->packed.event_flags_shadow);
}
END_USE(vq);
return vq->last_used_idx;
}
static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
{
struct vring_virtqueue *vq = to_vvq(_vq);
bool wrap_counter;
u16 used_idx;
wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
return is_used_desc_packed(vq, used_idx, wrap_counter);
}
static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 used_idx, wrap_counter, last_used_idx;
u16 bufs;
START_USE(vq);
/*
* We optimistically turn back on interrupts, then check if there was
* more to do.
*/
if (vq->event) {
/* TODO: tune this threshold */
bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
last_used_idx = READ_ONCE(vq->last_used_idx);
wrap_counter = packed_used_wrap_counter(last_used_idx);
used_idx = packed_last_used(last_used_idx) + bufs;
if (used_idx >= vq->packed.vring.num) {
used_idx -= vq->packed.vring.num;
wrap_counter ^= 1;
}
vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
(wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
/*
* We need to update event offset and event wrap
* counter first before updating event flags.
*/
virtio_wmb(vq->weak_barriers);
}
if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
vq->packed.event_flags_shadow = vq->event ?
VRING_PACKED_EVENT_FLAG_DESC :
VRING_PACKED_EVENT_FLAG_ENABLE;
vq->packed.vring.driver->flags =
cpu_to_le16(vq->packed.event_flags_shadow);
}
/*
* We need to update event suppression structure first
* before re-checking for more used buffers.
*/
virtio_mb(vq->weak_barriers);
last_used_idx = READ_ONCE(vq->last_used_idx);
wrap_counter = packed_used_wrap_counter(last_used_idx);
used_idx = packed_last_used(last_used_idx);
if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
END_USE(vq);
return false;
}
END_USE(vq);
return true;
}
static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
unsigned int i;
void *buf;
START_USE(vq);
for (i = 0; i < vq->packed.vring.num; i++) {
if (!vq->packed.desc_state[i].data)
continue;
/* detach_buf clears data, so grab it now. */
buf = vq->packed.desc_state[i].data;
detach_buf_packed(vq, i, NULL);
END_USE(vq);
return buf;
}
/* That should have freed everything. */
BUG_ON(vq->vq.num_free != vq->packed.vring.num);
END_USE(vq);
return NULL;
}
static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
{
struct vring_desc_extra *desc_extra;
unsigned int i;
desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra),
GFP_KERNEL);
if (!desc_extra)
return NULL;
memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
for (i = 0; i < num - 1; i++)
desc_extra[i].next = i + 1;
return desc_extra;
}
static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
struct virtio_device *vdev,
struct device *dma_dev)
{
if (vring_packed->vring.desc)
vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
vring_packed->vring.desc,
vring_packed->ring_dma_addr,
dma_dev);
if (vring_packed->vring.driver)
vring_free_queue(vdev, vring_packed->event_size_in_bytes,
vring_packed->vring.driver,
vring_packed->driver_event_dma_addr,
dma_dev);
if (vring_packed->vring.device)
vring_free_queue(vdev, vring_packed->event_size_in_bytes,
vring_packed->vring.device,
vring_packed->device_event_dma_addr,
dma_dev);
kfree(vring_packed->desc_state);
kfree(vring_packed->desc_extra);
}
static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
struct virtio_device *vdev,
u32 num, struct device *dma_dev)
{
struct vring_packed_desc *ring;
struct vring_packed_desc_event *driver, *device;
dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
size_t ring_size_in_bytes, event_size_in_bytes;
ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
ring = vring_alloc_queue(vdev, ring_size_in_bytes,
&ring_dma_addr,
GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
dma_dev);
if (!ring)
goto err;
vring_packed->vring.desc = ring;
vring_packed->ring_dma_addr = ring_dma_addr;
vring_packed->ring_size_in_bytes = ring_size_in_bytes;
event_size_in_bytes = sizeof(struct vring_packed_desc_event);
driver = vring_alloc_queue(vdev, event_size_in_bytes,
&driver_event_dma_addr,
GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
dma_dev);
if (!driver)
goto err;
vring_packed->vring.driver = driver;
vring_packed->event_size_in_bytes = event_size_in_bytes;
vring_packed->driver_event_dma_addr = driver_event_dma_addr;
device = vring_alloc_queue(vdev, event_size_in_bytes,
&device_event_dma_addr,
GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
dma_dev);
if (!device)
goto err;
vring_packed->vring.device = device;
vring_packed->device_event_dma_addr = device_event_dma_addr;
vring_packed->vring.num = num;
return 0;
err:
vring_free_packed(vring_packed, vdev, dma_dev);
return -ENOMEM;
}
static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
{
struct vring_desc_state_packed *state;
struct vring_desc_extra *extra;
u32 num = vring_packed->vring.num;
state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL);
if (!state)
goto err_desc_state;
memset(state, 0, num * sizeof(struct vring_desc_state_packed));
extra = vring_alloc_desc_extra(num);
if (!extra)
goto err_desc_extra;
vring_packed->desc_state = state;
vring_packed->desc_extra = extra;
return 0;
err_desc_extra:
kfree(state);
err_desc_state:
return -ENOMEM;
}
static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
bool callback)
{
vring_packed->next_avail_idx = 0;
vring_packed->avail_wrap_counter = 1;
vring_packed->event_flags_shadow = 0;
vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
/* No callback? Tell other side not to bother us. */
if (!callback) {
vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
vring_packed->vring.driver->flags =
cpu_to_le16(vring_packed->event_flags_shadow);
}
}
static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
struct vring_virtqueue_packed *vring_packed)
{
vq->packed = *vring_packed;
/* Put everything in free lists. */
vq->free_head = 0;
}
static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
{
memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
/* we need to reset the desc.flags. For more, see is_used_desc_packed() */
memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
virtqueue_init(vq, vq->packed.vring.num);
virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
}
static struct virtqueue *vring_create_virtqueue_packed(
unsigned int index,
unsigned int num,
unsigned int vring_align,
struct virtio_device *vdev,
bool weak_barriers,
bool may_reduce_num,
bool context,
bool (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name,
struct device *dma_dev)
{
struct vring_virtqueue_packed vring_packed = {};
struct vring_virtqueue *vq;
int err;
if (vring_alloc_queue_packed(&vring_packed, vdev, num, dma_dev))
goto err_ring;
vq = kmalloc(sizeof(*vq), GFP_KERNEL);
if (!vq)
goto err_vq;
vq->vq.callback = callback;
vq->vq.vdev = vdev;
vq->vq.name = name;
vq->vq.index = index;
vq->vq.reset = false;
vq->we_own_ring = true;
vq->notify = notify;
vq->weak_barriers = weak_barriers;
#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
vq->broken = true;
#else
vq->broken = false;
#endif
vq->packed_ring = true;
vq->dma_dev = dma_dev;
vq->use_dma_api = vring_use_dma_api(vdev);
vq->premapped = false;
vq->do_unmap = vq->use_dma_api;
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
err = vring_alloc_state_extra_packed(&vring_packed);
if (err)
goto err_state_extra;
virtqueue_vring_init_packed(&vring_packed, !!callback);
virtqueue_init(vq, num);
virtqueue_vring_attach_packed(vq, &vring_packed);
spin_lock(&vdev->vqs_list_lock);
list_add_tail(&vq->vq.list, &vdev->vqs);
spin_unlock(&vdev->vqs_list_lock);
return &vq->vq;
err_state_extra:
kfree(vq);
err_vq:
vring_free_packed(&vring_packed, vdev, dma_dev);
err_ring:
return NULL;
}
static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
{
struct vring_virtqueue_packed vring_packed = {};
struct vring_virtqueue *vq = to_vvq(_vq);
struct virtio_device *vdev = _vq->vdev;
int err;
if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq)))
goto err_ring;
err = vring_alloc_state_extra_packed(&vring_packed);
if (err)
goto err_state_extra;
vring_free(&vq->vq);
virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
virtqueue_init(vq, vring_packed.vring.num);
virtqueue_vring_attach_packed(vq, &vring_packed);
return 0;
err_state_extra:
vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq));
err_ring:
virtqueue_reinit_packed(vq);
return -ENOMEM;
}
static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
void (*recycle)(struct virtqueue *vq, void *buf))
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct virtio_device *vdev = vq->vq.vdev;
void *buf;
int err;
if (!vq->we_own_ring)
return -EPERM;
if (!vdev->config->disable_vq_and_reset)
return -ENOENT;
if (!vdev->config->enable_vq_after_reset)
return -ENOENT;
err = vdev->config->disable_vq_and_reset(_vq);
if (err)
return err;
while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
recycle(_vq, buf);
return 0;
}
static int virtqueue_enable_after_reset(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct virtio_device *vdev = vq->vq.vdev;
if (vdev->config->enable_vq_after_reset(_vq))
return -EBUSY;
return 0;
}
/*
* Generic functions and exported symbols.
*/
static inline int virtqueue_add(struct virtqueue *_vq,
struct scatterlist *sgs[],
unsigned int total_sg,
unsigned int out_sgs,
unsigned int in_sgs,
void *data,
void *ctx,
gfp_t gfp)
{
struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
out_sgs, in_sgs, data, ctx, gfp) :
virtqueue_add_split(_vq, sgs, total_sg,
out_sgs, in_sgs, data, ctx, gfp);
}
/**
* virtqueue_add_sgs - expose buffers to other end
* @_vq: the struct virtqueue we're talking about.
* @sgs: array of terminated scatterlists.
* @out_sgs: the number of scatterlists readable by other side
* @in_sgs: the number of scatterlists which are writable (after readable ones)
* @data: the token identifying the buffer.
* @gfp: how to do memory allocations (if necessary).
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
*/
int virtqueue_add_sgs(struct virtqueue *_vq,
struct scatterlist *sgs[],
unsigned int out_sgs,
unsigned int in_sgs,
void *data,
gfp_t gfp)
{
unsigned int i, total_sg = 0;
/* Count them first. */
for (i = 0; i < out_sgs + in_sgs; i++) {
struct scatterlist *sg;
for (sg = sgs[i]; sg; sg = sg_next(sg))
total_sg++;
}
return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
data, NULL, gfp);
}
EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
/**
* virtqueue_add_outbuf - expose output buffers to other end
* @vq: the struct virtqueue we're talking about.
* @sg: scatterlist (must be well-formed and terminated!)
* @num: the number of entries in @sg readable by other side
* @data: the token identifying the buffer.
* @gfp: how to do memory allocations (if necessary).
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
*/
int virtqueue_add_outbuf(struct virtqueue *vq,
struct scatterlist *sg, unsigned int num,
void *data,
gfp_t gfp)
{
return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
}
EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
/**
* virtqueue_add_inbuf - expose input buffers to other end
* @vq: the struct virtqueue we're talking about.
* @sg: scatterlist (must be well-formed and terminated!)
* @num: the number of entries in @sg writable by other side
* @data: the token identifying the buffer.
* @gfp: how to do memory allocations (if necessary).
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
*/
int virtqueue_add_inbuf(struct virtqueue *vq,
struct scatterlist *sg, unsigned int num,
void *data,
gfp_t gfp)
{
return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
}
EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
/**
* virtqueue_add_inbuf_ctx - expose input buffers to other end
* @vq: the struct virtqueue we're talking about.
* @sg: scatterlist (must be well-formed and terminated!)
* @num: the number of entries in @sg writable by other side
* @data: the token identifying the buffer.
* @ctx: extra context for the token
* @gfp: how to do memory allocations (if necessary).
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
*/
int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
struct scatterlist *sg, unsigned int num,
void *data,
void *ctx,
gfp_t gfp)
{
return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
}
EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
/**
* virtqueue_dma_dev - get the dma dev
* @_vq: the struct virtqueue we're talking about.
*
* Returns the dma dev. That can been used for dma api.
*/
struct device *virtqueue_dma_dev(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (vq->use_dma_api)
return vring_dma_dev(vq);
else
return NULL;
}
EXPORT_SYMBOL_GPL(virtqueue_dma_dev);
/**
* virtqueue_kick_prepare - first half of split virtqueue_kick call.
* @_vq: the struct virtqueue
*
* Instead of virtqueue_kick(), you can do:
* if (virtqueue_kick_prepare(vq))
* virtqueue_notify(vq);
*
* This is sometimes useful because the virtqueue_kick_prepare() needs
* to be serialized, but the actual virtqueue_notify() call does not.
*/
bool virtqueue_kick_prepare(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
virtqueue_kick_prepare_split(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
/**
* virtqueue_notify - second half of split virtqueue_kick call.
* @_vq: the struct virtqueue
*
* This does not need to be serialized.
*
* Returns false if host notify failed or queue is broken, otherwise true.
*/
bool virtqueue_notify(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (unlikely(vq->broken))
return false;
/* Prod other side to tell it about changes. */
if (!vq->notify(_vq)) {
vq->broken = true;
return false;
}
return true;
}
EXPORT_SYMBOL_GPL(virtqueue_notify);
/**
* virtqueue_kick - update after add_buf
* @vq: the struct virtqueue
*
* After one or more virtqueue_add_* calls, invoke this to kick
* the other side.
*
* Caller must ensure we don't call this with other virtqueue
* operations at the same time (except where noted).
*
* Returns false if kick failed, otherwise true.
*/
bool virtqueue_kick(struct virtqueue *vq)
{
if (virtqueue_kick_prepare(vq))
return virtqueue_notify(vq);
return true;
}
EXPORT_SYMBOL_GPL(virtqueue_kick);
/**
* virtqueue_get_buf_ctx - get the next used buffer
* @_vq: the struct virtqueue we're talking about.
* @len: the length written into the buffer
* @ctx: extra context for the token
*
* If the device wrote data into the buffer, @len will be set to the
* amount written. This means you don't need to clear the buffer
* beforehand to ensure there's no data leakage in the case of short
* writes.
*
* Caller must ensure we don't call this with other virtqueue
* operations at the same time (except where noted).
*
* Returns NULL if there are no used buffers, or the "data" token
* handed to virtqueue_add_*().
*/
void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
void **ctx)
{
struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
virtqueue_get_buf_ctx_split(_vq, len, ctx);
}
EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
{
return virtqueue_get_buf_ctx(_vq, len, NULL);
}
EXPORT_SYMBOL_GPL(virtqueue_get_buf);
/**
* virtqueue_disable_cb - disable callbacks
* @_vq: the struct virtqueue we're talking about.
*
* Note that this is not necessarily synchronous, hence unreliable and only
* useful as an optimization.
*
* Unlike other operations, this need not be serialized.
*/
void virtqueue_disable_cb(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (vq->packed_ring) virtqueue_disable_cb_packed(_vq);
else
virtqueue_disable_cb_split(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
/**
* virtqueue_enable_cb_prepare - restart callbacks after disable_cb
* @_vq: the struct virtqueue we're talking about.
*
* This re-enables callbacks; it returns current queue state
* in an opaque unsigned value. This value should be later tested by
* virtqueue_poll, to detect a possible race between the driver checking for
* more work, and enabling callbacks.
*
* Caller must ensure we don't call this with other virtqueue
* operations at the same time (except where noted).
*/
unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (vq->event_triggered)
vq->event_triggered = false;
return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
virtqueue_enable_cb_prepare_split(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
/**
* virtqueue_poll - query pending used buffers
* @_vq: the struct virtqueue we're talking about.
* @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
*
* Returns "true" if there are pending used buffers in the queue.
*
* This does not need to be serialized.
*/
bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (unlikely(vq->broken))
return false;
virtio_mb(vq->weak_barriers);
return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
virtqueue_poll_split(_vq, last_used_idx);
}
EXPORT_SYMBOL_GPL(virtqueue_poll);
/**
* virtqueue_enable_cb - restart callbacks after disable_cb.
* @_vq: the struct virtqueue we're talking about.
*
* This re-enables callbacks; it returns "false" if there are pending
* buffers in the queue, to detect a possible race between the driver
* checking for more work, and enabling callbacks.
*
* Caller must ensure we don't call this with other virtqueue
* operations at the same time (except where noted).
*/
bool virtqueue_enable_cb(struct virtqueue *_vq)
{
unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
return !virtqueue_poll(_vq, last_used_idx);
}
EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
/**
* virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
* @_vq: the struct virtqueue we're talking about.
*
* This re-enables callbacks but hints to the other side to delay
* interrupts until most of the available buffers have been processed;
* it returns "false" if there are many pending buffers in the queue,
* to detect a possible race between the driver checking for more work,
* and enabling callbacks.
*
* Caller must ensure we don't call this with other virtqueue
* operations at the same time (except where noted).
*/
bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (vq->event_triggered)
vq->event_triggered = false;
return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
virtqueue_enable_cb_delayed_split(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
/**
* virtqueue_detach_unused_buf - detach first unused buffer
* @_vq: the struct virtqueue we're talking about.
*
* Returns NULL or the "data" token handed to virtqueue_add_*().
* This is not valid on an active queue; it is useful for device
* shutdown or the reset queue.
*/
void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
virtqueue_detach_unused_buf_split(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
static inline bool more_used(const struct vring_virtqueue *vq)
{
return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
}
/**
* vring_interrupt - notify a virtqueue on an interrupt
* @irq: the IRQ number (ignored)
* @_vq: the struct virtqueue to notify
*
* Calls the callback function of @_vq to process the virtqueue
* notification.
*/
irqreturn_t vring_interrupt(int irq, void *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!more_used(vq)) { pr_debug("virtqueue interrupt with no work for %p\n", vq); return IRQ_NONE;
}
if (unlikely(vq->broken)) {
#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
dev_warn_once(&vq->vq.vdev->dev,
"virtio vring IRQ raised before DRIVER_OK");
return IRQ_NONE;
#else
return IRQ_HANDLED;
#endif
}
/* Just a hint for performance: so it's ok that this can be racy! */
if (vq->event) vq->event_triggered = true; pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); if (vq->vq.callback) vq->vq.callback(&vq->vq);
return IRQ_HANDLED;
}
EXPORT_SYMBOL_GPL(vring_interrupt);
/* Only available for split ring */
static struct virtqueue *__vring_new_virtqueue(unsigned int index,
struct vring_virtqueue_split *vring_split,
struct virtio_device *vdev,
bool weak_barriers,
bool context,
bool (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name,
struct device *dma_dev)
{
struct vring_virtqueue *vq;
int err;
if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
return NULL;
vq = kmalloc(sizeof(*vq), GFP_KERNEL);
if (!vq)
return NULL;
vq->packed_ring = false;
vq->vq.callback = callback;
vq->vq.vdev = vdev;
vq->vq.name = name;
vq->vq.index = index;
vq->vq.reset = false;
vq->we_own_ring = false;
vq->notify = notify;
vq->weak_barriers = weak_barriers;
#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
vq->broken = true;
#else
vq->broken = false;
#endif
vq->dma_dev = dma_dev;
vq->use_dma_api = vring_use_dma_api(vdev);
vq->premapped = false;
vq->do_unmap = vq->use_dma_api;
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
err = vring_alloc_state_extra_split(vring_split);
if (err) {
kfree(vq);
return NULL;
}
virtqueue_vring_init_split(vring_split, vq);
virtqueue_init(vq, vring_split->vring.num);
virtqueue_vring_attach_split(vq, vring_split);
spin_lock(&vdev->vqs_list_lock);
list_add_tail(&vq->vq.list, &vdev->vqs);
spin_unlock(&vdev->vqs_list_lock);
return &vq->vq;
}
struct virtqueue *vring_create_virtqueue(
unsigned int index,
unsigned int num,
unsigned int vring_align,
struct virtio_device *vdev,
bool weak_barriers,
bool may_reduce_num,
bool context,
bool (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name)
{
if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
return vring_create_virtqueue_packed(index, num, vring_align,
vdev, weak_barriers, may_reduce_num,
context, notify, callback, name, vdev->dev.parent);
return vring_create_virtqueue_split(index, num, vring_align,
vdev, weak_barriers, may_reduce_num,
context, notify, callback, name, vdev->dev.parent);
}
EXPORT_SYMBOL_GPL(vring_create_virtqueue);
struct virtqueue *vring_create_virtqueue_dma(
unsigned int index,
unsigned int num,
unsigned int vring_align,
struct virtio_device *vdev,
bool weak_barriers,
bool may_reduce_num,
bool context,
bool (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name,
struct device *dma_dev)
{
if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
return vring_create_virtqueue_packed(index, num, vring_align,
vdev, weak_barriers, may_reduce_num,
context, notify, callback, name, dma_dev);
return vring_create_virtqueue_split(index, num, vring_align,
vdev, weak_barriers, may_reduce_num,
context, notify, callback, name, dma_dev);
}
EXPORT_SYMBOL_GPL(vring_create_virtqueue_dma);
/**
* virtqueue_resize - resize the vring of vq
* @_vq: the struct virtqueue we're talking about.
* @num: new ring num
* @recycle: callback to recycle unused buffers
*
* When it is really necessary to create a new vring, it will set the current vq
* into the reset state. Then call the passed callback to recycle the buffer
* that is no longer used. Only after the new vring is successfully created, the
* old vring will be released.
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error.
* 0: success.
* -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
* vq can still work normally
* -EBUSY: Failed to sync with device, vq may not work properly
* -ENOENT: Transport or device not supported
* -E2BIG/-EINVAL: num error
* -EPERM: Operation not permitted
*
*/
int virtqueue_resize(struct virtqueue *_vq, u32 num,
void (*recycle)(struct virtqueue *vq, void *buf))
{
struct vring_virtqueue *vq = to_vvq(_vq);
int err;
if (num > vq->vq.num_max)
return -E2BIG;
if (!num)
return -EINVAL;
if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
return 0;
err = virtqueue_disable_and_recycle(_vq, recycle);
if (err)
return err;
if (vq->packed_ring)
err = virtqueue_resize_packed(_vq, num);
else
err = virtqueue_resize_split(_vq, num);
return virtqueue_enable_after_reset(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_resize);
/**
* virtqueue_set_dma_premapped - set the vring premapped mode
* @_vq: the struct virtqueue we're talking about.
*
* Enable the premapped mode of the vq.
*
* The vring in premapped mode does not do dma internally, so the driver must
* do dma mapping in advance. The driver must pass the dma_address through
* dma_address of scatterlist. When the driver got a used buffer from
* the vring, it has to unmap the dma address.
*
* This function must be called immediately after creating the vq, or after vq
* reset, and before adding any buffers to it.
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error.
* 0: success.
* -EINVAL: vring does not use the dma api, so we can not enable premapped mode.
*/
int virtqueue_set_dma_premapped(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u32 num;
START_USE(vq);
num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
if (num != vq->vq.num_free) {
END_USE(vq);
return -EINVAL;
}
if (!vq->use_dma_api) {
END_USE(vq);
return -EINVAL;
}
vq->premapped = true;
vq->do_unmap = false;
END_USE(vq);
return 0;
}
EXPORT_SYMBOL_GPL(virtqueue_set_dma_premapped);
/**
* virtqueue_reset - detach and recycle all unused buffers
* @_vq: the struct virtqueue we're talking about.
* @recycle: callback to recycle unused buffers
*
* Caller must ensure we don't call this with other virtqueue operations
* at the same time (except where noted).
*
* Returns zero or a negative error.
* 0: success.
* -EBUSY: Failed to sync with device, vq may not work properly
* -ENOENT: Transport or device not supported
* -EPERM: Operation not permitted
*/
int virtqueue_reset(struct virtqueue *_vq,
void (*recycle)(struct virtqueue *vq, void *buf))
{
struct vring_virtqueue *vq = to_vvq(_vq);
int err;
err = virtqueue_disable_and_recycle(_vq, recycle);
if (err)
return err;
if (vq->packed_ring)
virtqueue_reinit_packed(vq);
else
virtqueue_reinit_split(vq);
return virtqueue_enable_after_reset(_vq);
}
EXPORT_SYMBOL_GPL(virtqueue_reset);
/* Only available for split ring */
struct virtqueue *vring_new_virtqueue(unsigned int index,
unsigned int num,
unsigned int vring_align,
struct virtio_device *vdev,
bool weak_barriers,
bool context,
void *pages,
bool (*notify)(struct virtqueue *vq),
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct vring_virtqueue_split vring_split = {};
if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
return NULL;
vring_init(&vring_split.vring, num, pages, vring_align);
return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
context, notify, callback, name,
vdev->dev.parent);
}
EXPORT_SYMBOL_GPL(vring_new_virtqueue);
static void vring_free(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (vq->we_own_ring) {
if (vq->packed_ring) {
vring_free_queue(vq->vq.vdev,
vq->packed.ring_size_in_bytes,
vq->packed.vring.desc,
vq->packed.ring_dma_addr,
vring_dma_dev(vq));
vring_free_queue(vq->vq.vdev,
vq->packed.event_size_in_bytes,
vq->packed.vring.driver,
vq->packed.driver_event_dma_addr,
vring_dma_dev(vq));
vring_free_queue(vq->vq.vdev,
vq->packed.event_size_in_bytes,
vq->packed.vring.device,
vq->packed.device_event_dma_addr,
vring_dma_dev(vq));
kfree(vq->packed.desc_state);
kfree(vq->packed.desc_extra);
} else {
vring_free_queue(vq->vq.vdev,
vq->split.queue_size_in_bytes,
vq->split.vring.desc,
vq->split.queue_dma_addr,
vring_dma_dev(vq));
}
}
if (!vq->packed_ring) {
kfree(vq->split.desc_state);
kfree(vq->split.desc_extra);
}
}
void vring_del_virtqueue(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
spin_lock(&vq->vq.vdev->vqs_list_lock);
list_del(&_vq->list);
spin_unlock(&vq->vq.vdev->vqs_list_lock);
vring_free(_vq);
kfree(vq);
}
EXPORT_SYMBOL_GPL(vring_del_virtqueue);
u32 vring_notification_data(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
u16 next;
if (vq->packed_ring)
next = (vq->packed.next_avail_idx &
~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
vq->packed.avail_wrap_counter <<
VRING_PACKED_EVENT_F_WRAP_CTR;
else
next = vq->split.avail_idx_shadow;
return next << 16 | _vq->index;
}
EXPORT_SYMBOL_GPL(vring_notification_data);
/* Manipulates transport-specific feature bits. */
void vring_transport_features(struct virtio_device *vdev)
{
unsigned int i;
for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
switch (i) {
case VIRTIO_RING_F_INDIRECT_DESC:
break;
case VIRTIO_RING_F_EVENT_IDX:
break;
case VIRTIO_F_VERSION_1:
break;
case VIRTIO_F_ACCESS_PLATFORM:
break;
case VIRTIO_F_RING_PACKED:
break;
case VIRTIO_F_ORDER_PLATFORM:
break;
case VIRTIO_F_NOTIFICATION_DATA:
break;
default:
/* We don't understand this bit. */
__virtio_clear_bit(vdev, i);
}
}
}
EXPORT_SYMBOL_GPL(vring_transport_features);
/**
* virtqueue_get_vring_size - return the size of the virtqueue's vring
* @_vq: the struct virtqueue containing the vring of interest.
*
* Returns the size of the vring. This is mainly used for boasting to
* userspace. Unlike other operations, this need not be serialized.
*/
unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
}
EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
/*
* This function should only be called by the core, not directly by the driver.
*/
void __virtqueue_break(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
WRITE_ONCE(vq->broken, true);
}
EXPORT_SYMBOL_GPL(__virtqueue_break);
/*
* This function should only be called by the core, not directly by the driver.
*/
void __virtqueue_unbreak(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
WRITE_ONCE(vq->broken, false);
}
EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
bool virtqueue_is_broken(const struct virtqueue *_vq)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
return READ_ONCE(vq->broken);
}
EXPORT_SYMBOL_GPL(virtqueue_is_broken);
/*
* This should prevent the device from being used, allowing drivers to
* recover. You may need to grab appropriate locks to flush.
*/
void virtio_break_device(struct virtio_device *dev)
{
struct virtqueue *_vq;
spin_lock(&dev->vqs_list_lock);
list_for_each_entry(_vq, &dev->vqs, list) {
struct vring_virtqueue *vq = to_vvq(_vq);
/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
WRITE_ONCE(vq->broken, true);
}
spin_unlock(&dev->vqs_list_lock);
}
EXPORT_SYMBOL_GPL(virtio_break_device);
/*
* This should allow the device to be used by the driver. You may
* need to grab appropriate locks to flush the write to
* vq->broken. This should only be used in some specific case e.g
* (probing and restoring). This function should only be called by the
* core, not directly by the driver.
*/
void __virtio_unbreak_device(struct virtio_device *dev)
{
struct virtqueue *_vq;
spin_lock(&dev->vqs_list_lock);
list_for_each_entry(_vq, &dev->vqs, list) {
struct vring_virtqueue *vq = to_vvq(_vq);
/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
WRITE_ONCE(vq->broken, false);
}
spin_unlock(&dev->vqs_list_lock);
}
EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring);
if (vq->packed_ring)
return vq->packed.ring_dma_addr;
return vq->split.queue_dma_addr;
}
EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring);
if (vq->packed_ring)
return vq->packed.driver_event_dma_addr;
return vq->split.queue_dma_addr +
((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
}
EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
{
const struct vring_virtqueue *vq = to_vvq(_vq);
BUG_ON(!vq->we_own_ring);
if (vq->packed_ring)
return vq->packed.device_event_dma_addr;
return vq->split.queue_dma_addr +
((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
}
EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
/* Only available for split ring */
const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
{
return &to_vvq(vq)->split.vring;
}
EXPORT_SYMBOL_GPL(virtqueue_get_vring);
/**
* virtqueue_dma_map_single_attrs - map DMA for _vq
* @_vq: the struct virtqueue we're talking about.
* @ptr: the pointer of the buffer to do dma
* @size: the size of the buffer to do dma
* @dir: DMA direction
* @attrs: DMA Attrs
*
* The caller calls this to do dma mapping in advance. The DMA address can be
* passed to this _vq when it is in pre-mapped mode.
*
* return DMA address. Caller should check that by virtqueue_dma_mapping_error().
*/
dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr,
size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!vq->use_dma_api)
return (dma_addr_t)virt_to_phys(ptr);
return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs);
}
EXPORT_SYMBOL_GPL(virtqueue_dma_map_single_attrs);
/**
* virtqueue_dma_unmap_single_attrs - unmap DMA for _vq
* @_vq: the struct virtqueue we're talking about.
* @addr: the dma address to unmap
* @size: the size of the buffer
* @dir: DMA direction
* @attrs: DMA Attrs
*
* Unmap the address that is mapped by the virtqueue_dma_map_* APIs.
*
*/
void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!vq->use_dma_api)
return;
dma_unmap_single_attrs(vring_dma_dev(vq), addr, size, dir, attrs);
}
EXPORT_SYMBOL_GPL(virtqueue_dma_unmap_single_attrs);
/**
* virtqueue_dma_mapping_error - check dma address
* @_vq: the struct virtqueue we're talking about.
* @addr: DMA address
*
* Returns 0 means dma valid. Other means invalid dma address.
*/
int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!vq->use_dma_api)
return 0;
return dma_mapping_error(vring_dma_dev(vq), addr);
}
EXPORT_SYMBOL_GPL(virtqueue_dma_mapping_error);
/**
* virtqueue_dma_need_sync - check a dma address needs sync
* @_vq: the struct virtqueue we're talking about.
* @addr: DMA address
*
* Check if the dma address mapped by the virtqueue_dma_map_* APIs needs to be
* synchronized
*
* return bool
*/
bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr)
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!vq->use_dma_api)
return false;
return dma_need_sync(vring_dma_dev(vq), addr);
}
EXPORT_SYMBOL_GPL(virtqueue_dma_need_sync);
/**
* virtqueue_dma_sync_single_range_for_cpu - dma sync for cpu
* @_vq: the struct virtqueue we're talking about.
* @addr: DMA address
* @offset: DMA address offset
* @size: buf size for sync
* @dir: DMA direction
*
* Before calling this function, use virtqueue_dma_need_sync() to confirm that
* the DMA address really needs to be synchronized
*
*/
void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq,
dma_addr_t addr,
unsigned long offset, size_t size,
enum dma_data_direction dir)
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct device *dev = vring_dma_dev(vq);
if (!vq->use_dma_api)
return;
dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
}
EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_cpu);
/**
* virtqueue_dma_sync_single_range_for_device - dma sync for device
* @_vq: the struct virtqueue we're talking about.
* @addr: DMA address
* @offset: DMA address offset
* @size: buf size for sync
* @dir: DMA direction
*
* Before calling this function, use virtqueue_dma_need_sync() to confirm that
* the DMA address really needs to be synchronized
*/
void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq,
dma_addr_t addr,
unsigned long offset, size_t size,
enum dma_data_direction dir)
{
struct vring_virtqueue *vq = to_vvq(_vq);
struct device *dev = vring_dma_dev(vq);
if (!vq->use_dma_api)
return;
dma_sync_single_range_for_device(dev, addr, offset, size, dir);
}
EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device);
MODULE_LICENSE("GPL");
/*
* videobuf2-core.c - video buffer 2 core framework
*
* Copyright (C) 2010 Samsung Electronics
*
* Author: Pawel Osciak <pawel@osciak.com>
* Marek Szyprowski <m.szyprowski@samsung.com>
*
* The vb2_thread implementation was based on code from videobuf-dvb.c:
* (c) 2004 Gerd Knorr <kraxel@bytesex.org> [SUSE Labs]
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <media/videobuf2-core.h>
#include <media/v4l2-mc.h>
#include <trace/events/vb2.h>
#define PLANE_INDEX_BITS 3
#define PLANE_INDEX_SHIFT (PAGE_SHIFT + PLANE_INDEX_BITS)
#define PLANE_INDEX_MASK (BIT_MASK(PLANE_INDEX_BITS) - 1)
#define MAX_BUFFER_INDEX BIT_MASK(30 - PLANE_INDEX_SHIFT)
#define BUFFER_INDEX_MASK (MAX_BUFFER_INDEX - 1)
#if BIT(PLANE_INDEX_BITS) != VIDEO_MAX_PLANES
#error PLANE_INDEX_BITS order must be equal to VIDEO_MAX_PLANES
#endif
static int debug;
module_param(debug, int, 0644);
#define dprintk(q, level, fmt, arg...) \
do { \
if (debug >= level) \
pr_info("[%s] %s: " fmt, (q)->name, __func__, \
## arg); \
} while (0)
#ifdef CONFIG_VIDEO_ADV_DEBUG
/*
* If advanced debugging is on, then count how often each op is called
* successfully, which can either be per-buffer or per-queue.
*
* This makes it easy to check that the 'init' and 'cleanup'
* (and variations thereof) stay balanced.
*/
#define log_memop(vb, op) \
dprintk((vb)->vb2_queue, 2, "call_memop(%d, %s)%s\n", \
(vb)->index, #op, \
(vb)->vb2_queue->mem_ops->op ? "" : " (nop)")
#define call_memop(vb, op, args...) \
({ \
struct vb2_queue *_q = (vb)->vb2_queue; \
int err; \
\
log_memop(vb, op); \
err = _q->mem_ops->op ? _q->mem_ops->op(args) : 0; \
if (!err) \
(vb)->cnt_mem_ ## op++; \
err; \
})
#define call_ptr_memop(op, vb, args...) \
({ \
struct vb2_queue *_q = (vb)->vb2_queue; \
void *ptr; \
\
log_memop(vb, op); \
ptr = _q->mem_ops->op ? _q->mem_ops->op(vb, args) : NULL; \
if (!IS_ERR_OR_NULL(ptr)) \
(vb)->cnt_mem_ ## op++; \
ptr; \
})
#define call_void_memop(vb, op, args...) \
({ \
struct vb2_queue *_q = (vb)->vb2_queue; \
\
log_memop(vb, op); \
if (_q->mem_ops->op) \
_q->mem_ops->op(args); \
(vb)->cnt_mem_ ## op++; \
})
#define log_qop(q, op) \
dprintk(q, 2, "call_qop(%s)%s\n", #op, \
(q)->ops->op ? "" : " (nop)")
#define call_qop(q, op, args...) \
({ \
int err; \
\
log_qop(q, op); \
err = (q)->ops->op ? (q)->ops->op(args) : 0; \
if (!err) \
(q)->cnt_ ## op++; \
err; \
})
#define call_void_qop(q, op, args...) \
({ \
log_qop(q, op); \
if ((q)->ops->op) \
(q)->ops->op(args); \
(q)->cnt_ ## op++; \
})
#define log_vb_qop(vb, op, args...) \
dprintk((vb)->vb2_queue, 2, "call_vb_qop(%d, %s)%s\n", \
(vb)->index, #op, \
(vb)->vb2_queue->ops->op ? "" : " (nop)")
#define call_vb_qop(vb, op, args...) \
({ \
int err; \
\
log_vb_qop(vb, op); \
err = (vb)->vb2_queue->ops->op ? \
(vb)->vb2_queue->ops->op(args) : 0; \
if (!err) \
(vb)->cnt_ ## op++; \
err; \
})
#define call_void_vb_qop(vb, op, args...) \
({ \
log_vb_qop(vb, op); \
if ((vb)->vb2_queue->ops->op) \
(vb)->vb2_queue->ops->op(args); \
(vb)->cnt_ ## op++; \
})
#else
#define call_memop(vb, op, args...) \
((vb)->vb2_queue->mem_ops->op ? \
(vb)->vb2_queue->mem_ops->op(args) : 0)
#define call_ptr_memop(op, vb, args...) \
((vb)->vb2_queue->mem_ops->op ? \
(vb)->vb2_queue->mem_ops->op(vb, args) : NULL)
#define call_void_memop(vb, op, args...) \
do { \
if ((vb)->vb2_queue->mem_ops->op) \
(vb)->vb2_queue->mem_ops->op(args); \
} while (0)
#define call_qop(q, op, args...) \
((q)->ops->op ? (q)->ops->op(args) : 0)
#define call_void_qop(q, op, args...) \
do { \
if ((q)->ops->op) \
(q)->ops->op(args); \
} while (0)
#define call_vb_qop(vb, op, args...) \
((vb)->vb2_queue->ops->op ? (vb)->vb2_queue->ops->op(args) : 0)
#define call_void_vb_qop(vb, op, args...) \
do { \
if ((vb)->vb2_queue->ops->op) \
(vb)->vb2_queue->ops->op(args); \
} while (0)
#endif
#define call_bufop(q, op, args...) \
({ \
int ret = 0; \
if (q && q->buf_ops && q->buf_ops->op) \
ret = q->buf_ops->op(args); \
ret; \
})
#define call_void_bufop(q, op, args...) \
({ \
if (q && q->buf_ops && q->buf_ops->op) \
q->buf_ops->op(args); \
})
static void __vb2_queue_cancel(struct vb2_queue *q);
static void __enqueue_in_driver(struct vb2_buffer *vb);
static const char *vb2_state_name(enum vb2_buffer_state s)
{
static const char * const state_names[] = {
[VB2_BUF_STATE_DEQUEUED] = "dequeued",
[VB2_BUF_STATE_IN_REQUEST] = "in request",
[VB2_BUF_STATE_PREPARING] = "preparing",
[VB2_BUF_STATE_QUEUED] = "queued",
[VB2_BUF_STATE_ACTIVE] = "active",
[VB2_BUF_STATE_DONE] = "done",
[VB2_BUF_STATE_ERROR] = "error",
};
if ((unsigned int)(s) < ARRAY_SIZE(state_names))
return state_names[s];
return "unknown";
}
/*
* __vb2_buf_mem_alloc() - allocate video memory for the given buffer
*/
static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
{
struct vb2_queue *q = vb->vb2_queue;
void *mem_priv;
int plane;
int ret = -ENOMEM;
/*
* Allocate memory for all planes in this buffer
* NOTE: mmapped areas should be page aligned
*/
for (plane = 0; plane < vb->num_planes; ++plane) {
/* Memops alloc requires size to be page aligned. */
unsigned long size = PAGE_ALIGN(vb->planes[plane].length);
/* Did it wrap around? */
if (size < vb->planes[plane].length)
goto free;
mem_priv = call_ptr_memop(alloc,
vb,
q->alloc_devs[plane] ? : q->dev,
size);
if (IS_ERR_OR_NULL(mem_priv)) {
if (mem_priv)
ret = PTR_ERR(mem_priv);
goto free;
}
/* Associate allocator private data with this plane */
vb->planes[plane].mem_priv = mem_priv;
}
return 0;
free:
/* Free already allocated memory if one of the allocations failed */
for (; plane > 0; --plane) {
call_void_memop(vb, put, vb->planes[plane - 1].mem_priv);
vb->planes[plane - 1].mem_priv = NULL;
}
return ret;
}
/*
* __vb2_buf_mem_free() - free memory of the given buffer
*/
static void __vb2_buf_mem_free(struct vb2_buffer *vb)
{
unsigned int plane;
for (plane = 0; plane < vb->num_planes; ++plane) {
call_void_memop(vb, put, vb->planes[plane].mem_priv);
vb->planes[plane].mem_priv = NULL;
dprintk(vb->vb2_queue, 3, "freed plane %d of buffer %d\n",
plane, vb->index);
}
}
/*
* __vb2_buf_userptr_put() - release userspace memory associated with
* a USERPTR buffer
*/
static void __vb2_buf_userptr_put(struct vb2_buffer *vb)
{
unsigned int plane;
for (plane = 0; plane < vb->num_planes; ++plane) {
if (vb->planes[plane].mem_priv)
call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
vb->planes[plane].mem_priv = NULL;
}
}
/*
* __vb2_plane_dmabuf_put() - release memory associated with
* a DMABUF shared plane
*/
static void __vb2_plane_dmabuf_put(struct vb2_buffer *vb, struct vb2_plane *p)
{
if (!p->mem_priv)
return;
if (p->dbuf_mapped)
call_void_memop(vb, unmap_dmabuf, p->mem_priv);
call_void_memop(vb, detach_dmabuf, p->mem_priv);
dma_buf_put(p->dbuf);
p->mem_priv = NULL;
p->dbuf = NULL;
p->dbuf_mapped = 0;
}
/*
* __vb2_buf_dmabuf_put() - release memory associated with
* a DMABUF shared buffer
*/
static void __vb2_buf_dmabuf_put(struct vb2_buffer *vb)
{
unsigned int plane;
for (plane = 0; plane < vb->num_planes; ++plane)
__vb2_plane_dmabuf_put(vb, &vb->planes[plane]);
}
/*
* __vb2_buf_mem_prepare() - call ->prepare() on buffer's private memory
* to sync caches
*/
static void __vb2_buf_mem_prepare(struct vb2_buffer *vb)
{
unsigned int plane;
if (vb->synced)
return;
vb->synced = 1;
for (plane = 0; plane < vb->num_planes; ++plane)
call_void_memop(vb, prepare, vb->planes[plane].mem_priv);
}
/*
* __vb2_buf_mem_finish() - call ->finish on buffer's private memory
* to sync caches
*/
static void __vb2_buf_mem_finish(struct vb2_buffer *vb)
{
unsigned int plane;
if (!vb->synced)
return;
vb->synced = 0;
for (plane = 0; plane < vb->num_planes; ++plane)
call_void_memop(vb, finish, vb->planes[plane].mem_priv);
}
/*
* __setup_offsets() - setup unique offsets ("cookies") for every plane in
* the buffer.
*/
static void __setup_offsets(struct vb2_buffer *vb)
{
struct vb2_queue *q = vb->vb2_queue;
unsigned int plane;
unsigned long offset = 0;
/*
* The offset "cookie" value has the following constraints:
* - a buffer can have up to 8 planes.
* - v4l2 mem2mem uses bit 30 to distinguish between
* OUTPUT (aka "source", bit 30 is 0) and
* CAPTURE (aka "destination", bit 30 is 1) buffers.
* - must be page aligned
* That led to this bit mapping when PAGE_SHIFT = 12:
* |30 |29 15|14 12|11 0|
* |DST_QUEUE_OFF_BASE|buffer index|plane index| 0 |
* where there are 15 bits to store the buffer index.
* Depending on PAGE_SHIFT value we can have fewer bits
* to store the buffer index.
*/
offset = vb->index << PLANE_INDEX_SHIFT;
for (plane = 0; plane < vb->num_planes; ++plane) {
vb->planes[plane].m.offset = offset + (plane << PAGE_SHIFT);
dprintk(q, 3, "buffer %d, plane %d offset 0x%08lx\n",
vb->index, plane, offset);
}
}
static void init_buffer_cache_hints(struct vb2_queue *q, struct vb2_buffer *vb)
{
/*
* DMA exporter should take care of cache syncs, so we can avoid
* explicit ->prepare()/->finish() syncs. For other ->memory types
* we always need ->prepare() or/and ->finish() cache sync.
*/
if (q->memory == VB2_MEMORY_DMABUF) {
vb->skip_cache_sync_on_finish = 1;
vb->skip_cache_sync_on_prepare = 1;
return;
}
/*
* ->finish() cache sync can be avoided when queue direction is
* TO_DEVICE.
*/
if (q->dma_dir == DMA_TO_DEVICE)
vb->skip_cache_sync_on_finish = 1;
}
/**
* vb2_queue_add_buffer() - add a buffer to a queue
* @q: pointer to &struct vb2_queue with videobuf2 queue.
* @vb: pointer to &struct vb2_buffer to be added to the queue.
* @index: index where add vb2_buffer in the queue
*/
static void vb2_queue_add_buffer(struct vb2_queue *q, struct vb2_buffer *vb, unsigned int index)
{
WARN_ON(index >= q->max_num_buffers || q->bufs[index] || vb->vb2_queue);
q->bufs[index] = vb;
vb->index = index;
vb->vb2_queue = q;
}
/**
* vb2_queue_remove_buffer() - remove a buffer from a queue
* @vb: pointer to &struct vb2_buffer to be removed from the queue.
*/
static void vb2_queue_remove_buffer(struct vb2_buffer *vb)
{
vb->vb2_queue->bufs[vb->index] = NULL;
vb->vb2_queue = NULL;
}
/*
* __vb2_queue_alloc() - allocate vb2 buffer structures and (for MMAP type)
* video buffer memory for all buffers/planes on the queue and initializes the
* queue
*
* Returns the number of buffers successfully allocated.
*/
static int __vb2_queue_alloc(struct vb2_queue *q, enum vb2_memory memory,
unsigned int num_buffers, unsigned int num_planes,
const unsigned plane_sizes[VB2_MAX_PLANES])
{
unsigned int q_num_buffers = vb2_get_num_buffers(q);
unsigned int buffer, plane;
struct vb2_buffer *vb;
int ret;
/*
* Ensure that the number of already queue + the number of buffers already
* in the queue is below q->max_num_buffers
*/
num_buffers = min_t(unsigned int, num_buffers,
q->max_num_buffers - q_num_buffers);
for (buffer = 0; buffer < num_buffers; ++buffer) {
/* Allocate vb2 buffer structures */
vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
if (!vb) {
dprintk(q, 1, "memory alloc for buffer struct failed\n");
break;
}
vb->state = VB2_BUF_STATE_DEQUEUED;
vb->num_planes = num_planes;
vb->type = q->type;
vb->memory = memory;
init_buffer_cache_hints(q, vb);
for (plane = 0; plane < num_planes; ++plane) {
vb->planes[plane].length = plane_sizes[plane];
vb->planes[plane].min_length = plane_sizes[plane];
}
vb2_queue_add_buffer(q, vb, q_num_buffers + buffer);
call_void_bufop(q, init_buffer, vb);
/* Allocate video buffer memory for the MMAP type */
if (memory == VB2_MEMORY_MMAP) {
ret = __vb2_buf_mem_alloc(vb);
if (ret) {
dprintk(q, 1, "failed allocating memory for buffer %d\n",
buffer);
vb2_queue_remove_buffer(vb);
kfree(vb);
break;
}
__setup_offsets(vb);
/*
* Call the driver-provided buffer initialization
* callback, if given. An error in initialization
* results in queue setup failure.
*/
ret = call_vb_qop(vb, buf_init, vb);
if (ret) {
dprintk(q, 1, "buffer %d %p initialization failed\n",
buffer, vb);
__vb2_buf_mem_free(vb);
vb2_queue_remove_buffer(vb);
kfree(vb);
break;
}
}
}
dprintk(q, 3, "allocated %d buffers, %d plane(s) each\n",
buffer, num_planes);
return buffer;
}
/*
* __vb2_free_mem() - release all video buffer memory for a given queue
*/
static void __vb2_free_mem(struct vb2_queue *q, unsigned int buffers)
{
unsigned int buffer;
struct vb2_buffer *vb;
unsigned int q_num_buffers = vb2_get_num_buffers(q);
for (buffer = q_num_buffers - buffers; buffer < q_num_buffers;
++buffer) {
vb = vb2_get_buffer(q, buffer);
if (!vb)
continue;
/* Free MMAP buffers or release USERPTR buffers */
if (q->memory == VB2_MEMORY_MMAP)
__vb2_buf_mem_free(vb);
else if (q->memory == VB2_MEMORY_DMABUF)
__vb2_buf_dmabuf_put(vb);
else
__vb2_buf_userptr_put(vb);
}
}
/*
* __vb2_queue_free() - free buffers at the end of the queue - video memory and
* related information, if no buffers are left return the queue to an
* uninitialized state. Might be called even if the queue has already been freed.
*/
static void __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
{
unsigned int buffer;
unsigned int q_num_buffers = vb2_get_num_buffers(q);
lockdep_assert_held(&q->mmap_lock);
/* Call driver-provided cleanup function for each buffer, if provided */
for (buffer = q_num_buffers - buffers; buffer < q_num_buffers;
++buffer) {
struct vb2_buffer *vb = vb2_get_buffer(q, buffer);
if (vb && vb->planes[0].mem_priv)
call_void_vb_qop(vb, buf_cleanup, vb);
}
/* Release video buffer memory */
__vb2_free_mem(q, buffers);
#ifdef CONFIG_VIDEO_ADV_DEBUG
/*
* Check that all the calls were balanced during the life-time of this
* queue. If not then dump the counters to the kernel log.
*/
if (q_num_buffers) {
bool unbalanced = q->cnt_start_streaming != q->cnt_stop_streaming ||
q->cnt_prepare_streaming != q->cnt_unprepare_streaming ||
q->cnt_wait_prepare != q->cnt_wait_finish;
if (unbalanced) {
pr_info("unbalanced counters for queue %p:\n", q);
if (q->cnt_start_streaming != q->cnt_stop_streaming)
pr_info(" setup: %u start_streaming: %u stop_streaming: %u\n",
q->cnt_queue_setup, q->cnt_start_streaming,
q->cnt_stop_streaming);
if (q->cnt_prepare_streaming != q->cnt_unprepare_streaming)
pr_info(" prepare_streaming: %u unprepare_streaming: %u\n",
q->cnt_prepare_streaming, q->cnt_unprepare_streaming);
if (q->cnt_wait_prepare != q->cnt_wait_finish)
pr_info(" wait_prepare: %u wait_finish: %u\n",
q->cnt_wait_prepare, q->cnt_wait_finish);
}
q->cnt_queue_setup = 0;
q->cnt_wait_prepare = 0;
q->cnt_wait_finish = 0;
q->cnt_prepare_streaming = 0;
q->cnt_start_streaming = 0;
q->cnt_stop_streaming = 0;
q->cnt_unprepare_streaming = 0;
}
for (buffer = 0; buffer < vb2_get_num_buffers(q); buffer++) {
struct vb2_buffer *vb = vb2_get_buffer(q, buffer);
bool unbalanced;
if (!vb)
continue;
unbalanced = vb->cnt_mem_alloc != vb->cnt_mem_put ||
vb->cnt_mem_prepare != vb->cnt_mem_finish ||
vb->cnt_mem_get_userptr != vb->cnt_mem_put_userptr ||
vb->cnt_mem_attach_dmabuf != vb->cnt_mem_detach_dmabuf ||
vb->cnt_mem_map_dmabuf != vb->cnt_mem_unmap_dmabuf ||
vb->cnt_buf_queue != vb->cnt_buf_done ||
vb->cnt_buf_prepare != vb->cnt_buf_finish ||
vb->cnt_buf_init != vb->cnt_buf_cleanup;
if (unbalanced) {
pr_info("unbalanced counters for queue %p, buffer %d:\n",
q, buffer);
if (vb->cnt_buf_init != vb->cnt_buf_cleanup)
pr_info(" buf_init: %u buf_cleanup: %u\n",
vb->cnt_buf_init, vb->cnt_buf_cleanup);
if (vb->cnt_buf_prepare != vb->cnt_buf_finish)
pr_info(" buf_prepare: %u buf_finish: %u\n",
vb->cnt_buf_prepare, vb->cnt_buf_finish);
if (vb->cnt_buf_queue != vb->cnt_buf_done)
pr_info(" buf_out_validate: %u buf_queue: %u buf_done: %u buf_request_complete: %u\n",
vb->cnt_buf_out_validate, vb->cnt_buf_queue,
vb->cnt_buf_done, vb->cnt_buf_request_complete);
if (vb->cnt_mem_alloc != vb->cnt_mem_put)
pr_info(" alloc: %u put: %u\n",
vb->cnt_mem_alloc, vb->cnt_mem_put);
if (vb->cnt_mem_prepare != vb->cnt_mem_finish)
pr_info(" prepare: %u finish: %u\n",
vb->cnt_mem_prepare, vb->cnt_mem_finish);
if (vb->cnt_mem_get_userptr != vb->cnt_mem_put_userptr)
pr_info(" get_userptr: %u put_userptr: %u\n",
vb->cnt_mem_get_userptr, vb->cnt_mem_put_userptr);
if (vb->cnt_mem_attach_dmabuf != vb->cnt_mem_detach_dmabuf)
pr_info(" attach_dmabuf: %u detach_dmabuf: %u\n",
vb->cnt_mem_attach_dmabuf, vb->cnt_mem_detach_dmabuf);
if (vb->cnt_mem_map_dmabuf != vb->cnt_mem_unmap_dmabuf)
pr_info(" map_dmabuf: %u unmap_dmabuf: %u\n",
vb->cnt_mem_map_dmabuf, vb->cnt_mem_unmap_dmabuf);
pr_info(" get_dmabuf: %u num_users: %u\n",
vb->cnt_mem_get_dmabuf,
vb->cnt_mem_num_users);
}
}
#endif
/* Free vb2 buffers */
for (buffer = q_num_buffers - buffers; buffer < q_num_buffers;
++buffer) {
struct vb2_buffer *vb = vb2_get_buffer(q, buffer);
if (!vb)
continue;
vb2_queue_remove_buffer(vb);
kfree(vb);
}
q->num_buffers -= buffers;
if (!vb2_get_num_buffers(q)) {
q->memory = VB2_MEMORY_UNKNOWN;
INIT_LIST_HEAD(&q->queued_list);
}
}
bool vb2_buffer_in_use(struct vb2_queue *q, struct vb2_buffer *vb)
{
unsigned int plane;
for (plane = 0; plane < vb->num_planes; ++plane) {
void *mem_priv = vb->planes[plane].mem_priv;
/*
* If num_users() has not been provided, call_memop
* will return 0, apparently nobody cares about this
* case anyway. If num_users() returns more than 1,
* we are not the only user of the plane's memory.
*/
if (mem_priv && call_memop(vb, num_users, mem_priv) > 1)
return true;
}
return false;
}
EXPORT_SYMBOL(vb2_buffer_in_use);
/*
* __buffers_in_use() - return true if any buffers on the queue are in use and
* the queue cannot be freed (by the means of REQBUFS(0)) call
*/
static bool __buffers_in_use(struct vb2_queue *q)
{
unsigned int buffer;
for (buffer = 0; buffer < vb2_get_num_buffers(q); ++buffer) {
struct vb2_buffer *vb = vb2_get_buffer(q, buffer);
if (!vb)
continue;
if (vb2_buffer_in_use(q, vb))
return true;
}
return false;
}
void vb2_core_querybuf(struct vb2_queue *q, struct vb2_buffer *vb, void *pb)
{
call_void_bufop(q, fill_user_buffer, vb, pb);
}
EXPORT_SYMBOL_GPL(vb2_core_querybuf);
/*
* __verify_userptr_ops() - verify that all memory operations required for
* USERPTR queue type have been provided
*/
static int __verify_userptr_ops(struct vb2_queue *q)
{
if (!(q->io_modes & VB2_USERPTR) || !q->mem_ops->get_userptr ||
!q->mem_ops->put_userptr)
return -EINVAL;
return 0;
}
/*
* __verify_mmap_ops() - verify that all memory operations required for
* MMAP queue type have been provided
*/
static int __verify_mmap_ops(struct vb2_queue *q)
{
if (!(q->io_modes & VB2_MMAP) || !q->mem_ops->alloc ||
!q->mem_ops->put || !q->mem_ops->mmap)
return -EINVAL;
return 0;
}
/*
* __verify_dmabuf_ops() - verify that all memory operations required for
* DMABUF queue type have been provided
*/
static int __verify_dmabuf_ops(struct vb2_queue *q)
{
if (!(q->io_modes & VB2_DMABUF) || !q->mem_ops->attach_dmabuf ||
!q->mem_ops->detach_dmabuf || !q->mem_ops->map_dmabuf ||
!q->mem_ops->unmap_dmabuf)
return -EINVAL;
return 0;
}
int vb2_verify_memory_type(struct vb2_queue *q,
enum vb2_memory memory, unsigned int type)
{
if (memory != VB2_MEMORY_MMAP && memory != VB2_MEMORY_USERPTR &&
memory != VB2_MEMORY_DMABUF) {
dprintk(q, 1, "unsupported memory type\n");
return -EINVAL;
}
if (type != q->type) {
dprintk(q, 1, "requested type is incorrect\n");
return -EINVAL;
}
/*
* Make sure all the required memory ops for given memory type
* are available.
*/
if (memory == VB2_MEMORY_MMAP && __verify_mmap_ops(q)) {
dprintk(q, 1, "MMAP for current setup unsupported\n");
return -EINVAL;
}
if (memory == VB2_MEMORY_USERPTR && __verify_userptr_ops(q)) {
dprintk(q, 1, "USERPTR for current setup unsupported\n");
return -EINVAL;
}
if (memory == VB2_MEMORY_DMABUF && __verify_dmabuf_ops(q)) {
dprintk(q, 1, "DMABUF for current setup unsupported\n");
return -EINVAL;
}
/*
* Place the busy tests at the end: -EBUSY can be ignored when
* create_bufs is called with count == 0, but count == 0 should still
* do the memory and type validation.
*/
if (vb2_fileio_is_active(q)) {
dprintk(q, 1, "file io in progress\n");
return -EBUSY;
}
return 0;
}
EXPORT_SYMBOL(vb2_verify_memory_type);
static void set_queue_coherency(struct vb2_queue *q, bool non_coherent_mem)
{
q->non_coherent_mem = 0;
if (!vb2_queue_allows_cache_hints(q))
return;
q->non_coherent_mem = non_coherent_mem;
}
static bool verify_coherency_flags(struct vb2_queue *q, bool non_coherent_mem)
{
if (non_coherent_mem != q->non_coherent_mem) {
dprintk(q, 1, "memory coherency model mismatch\n");
return false;
}
return true;
}
int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
unsigned int flags, unsigned int *count)
{
unsigned int num_buffers, allocated_buffers, num_planes = 0;
unsigned int q_num_bufs = vb2_get_num_buffers(q);
unsigned plane_sizes[VB2_MAX_PLANES] = { };
bool non_coherent_mem = flags & V4L2_MEMORY_FLAG_NON_COHERENT;
unsigned int i;
int ret = 0;
if (q->streaming) {
dprintk(q, 1, "streaming active\n");
return -EBUSY;
}
if (q->waiting_in_dqbuf && *count) {
dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
return -EBUSY;
}
if (*count == 0 || q_num_bufs != 0 ||
(q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory) ||
!verify_coherency_flags(q, non_coherent_mem)) {
/*
* We already have buffers allocated, so first check if they
* are not in use and can be freed.
*/
mutex_lock(&q->mmap_lock);
if (debug && q->memory == VB2_MEMORY_MMAP &&
__buffers_in_use(q))
dprintk(q, 1, "memory in use, orphaning buffers\n");
/*
* Call queue_cancel to clean up any buffers in the
* QUEUED state which is possible if buffers were prepared or
* queued without ever calling STREAMON.
*/
__vb2_queue_cancel(q);
__vb2_queue_free(q, q_num_bufs);
mutex_unlock(&q->mmap_lock);
/*
* In case of REQBUFS(0) return immediately without calling
* driver's queue_setup() callback and allocating resources.
*/
if (*count == 0)
return 0;
}
/*
* Make sure the requested values and current defaults are sane.
*/
num_buffers = max_t(unsigned int, *count, q->min_queued_buffers);
num_buffers = min_t(unsigned int, num_buffers, q->max_num_buffers);
memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
/*
* Set this now to ensure that drivers see the correct q->memory value
* in the queue_setup op.
*/
mutex_lock(&q->mmap_lock);
if (!q->bufs)
q->bufs = kcalloc(q->max_num_buffers, sizeof(*q->bufs), GFP_KERNEL);
if (!q->bufs)
ret = -ENOMEM;
q->memory = memory;
mutex_unlock(&q->mmap_lock);
if (ret)
return ret;
set_queue_coherency(q, non_coherent_mem);
/*
* Ask the driver how many buffers and planes per buffer it requires.
* Driver also sets the size and allocator context for each plane.
*/
ret = call_qop(q, queue_setup, q, &num_buffers, &num_planes,
plane_sizes, q->alloc_devs);
if (ret)
goto error;
/* Check that driver has set sane values */
if (WARN_ON(!num_planes)) {
ret = -EINVAL;
goto error;
}
for (i = 0; i < num_planes; i++)
if (WARN_ON(!plane_sizes[i])) {
ret = -EINVAL;
goto error;
}
/* Finally, allocate buffers and video memory */
allocated_buffers =
__vb2_queue_alloc(q, memory, num_buffers, num_planes, plane_sizes);
if (allocated_buffers == 0) {
dprintk(q, 1, "memory allocation failed\n");
ret = -ENOMEM;
goto error;
}
/*
* There is no point in continuing if we can't allocate the minimum
* number of buffers needed by this vb2_queue.
*/
if (allocated_buffers < q->min_queued_buffers)
ret = -ENOMEM;
/*
* Check if driver can handle the allocated number of buffers.
*/
if (!ret && allocated_buffers < num_buffers) {
num_buffers = allocated_buffers;
/*
* num_planes is set by the previous queue_setup(), but since it
* signals to queue_setup() whether it is called from create_bufs()
* vs reqbufs() we zero it here to signal that queue_setup() is
* called for the reqbufs() case.
*/
num_planes = 0;
ret = call_qop(q, queue_setup, q, &num_buffers,
&num_planes, plane_sizes, q->alloc_devs);
if (!ret && allocated_buffers < num_buffers)
ret = -ENOMEM;
/*
* Either the driver has accepted a smaller number of buffers,
* or .queue_setup() returned an error
*/
}
mutex_lock(&q->mmap_lock);
q->num_buffers = allocated_buffers;
if (ret < 0) {
/*
* Note: __vb2_queue_free() will subtract 'allocated_buffers'
* from already queued buffers and it will reset q->memory to
* VB2_MEMORY_UNKNOWN.
*/
__vb2_queue_free(q, allocated_buffers);
mutex_unlock(&q->mmap_lock);
return ret;
}
mutex_unlock(&q->mmap_lock);
/*
* Return the number of successfully allocated buffers
* to the userspace.
*/
*count = allocated_buffers;
q->waiting_for_buffers = !q->is_output;
return 0;
error:
mutex_lock(&q->mmap_lock);
q->memory = VB2_MEMORY_UNKNOWN;
mutex_unlock(&q->mmap_lock);
return ret;
}
EXPORT_SYMBOL_GPL(vb2_core_reqbufs);
int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
unsigned int flags, unsigned int *count,
unsigned int requested_planes,
const unsigned int requested_sizes[])
{
unsigned int num_planes = 0, num_buffers, allocated_buffers;
unsigned plane_sizes[VB2_MAX_PLANES] = { };
bool non_coherent_mem = flags & V4L2_MEMORY_FLAG_NON_COHERENT;
unsigned int q_num_bufs = vb2_get_num_buffers(q);
bool no_previous_buffers = !q_num_bufs;
int ret = 0;
if (q_num_bufs == q->max_num_buffers) {
dprintk(q, 1, "maximum number of buffers already allocated\n");
return -ENOBUFS;
}
if (no_previous_buffers) {
if (q->waiting_in_dqbuf && *count) {
dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
return -EBUSY;
}
memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
/*
* Set this now to ensure that drivers see the correct q->memory
* value in the queue_setup op.
*/
mutex_lock(&q->mmap_lock);
q->memory = memory;
if (!q->bufs)
q->bufs = kcalloc(q->max_num_buffers, sizeof(*q->bufs), GFP_KERNEL);
if (!q->bufs)
ret = -ENOMEM;
mutex_unlock(&q->mmap_lock);
if (ret)
return ret;
q->waiting_for_buffers = !q->is_output;
set_queue_coherency(q, non_coherent_mem);
} else {
if (q->memory != memory) {
dprintk(q, 1, "memory model mismatch\n");
return -EINVAL;
}
if (!verify_coherency_flags(q, non_coherent_mem))
return -EINVAL;
}
num_buffers = min(*count, q->max_num_buffers - q_num_bufs);
if (requested_planes && requested_sizes) {
num_planes = requested_planes;
memcpy(plane_sizes, requested_sizes, sizeof(plane_sizes));
}
/*
* Ask the driver, whether the requested number of buffers, planes per
* buffer and their sizes are acceptable
*/
ret = call_qop(q, queue_setup, q, &num_buffers,
&num_planes, plane_sizes, q->alloc_devs);
if (ret)
goto error;
/* Finally, allocate buffers and video memory */
allocated_buffers = __vb2_queue_alloc(q, memory, num_buffers,
num_planes, plane_sizes);
if (allocated_buffers == 0) {
dprintk(q, 1, "memory allocation failed\n");
ret = -ENOMEM;
goto error;
}
/*
* Check if driver can handle the so far allocated number of buffers.
*/
if (allocated_buffers < num_buffers) {
num_buffers = allocated_buffers;
/*
* num_buffers contains the total number of buffers, that the
* queue driver has set up
*/
ret = call_qop(q, queue_setup, q, &num_buffers,
&num_planes, plane_sizes, q->alloc_devs);
if (!ret && allocated_buffers < num_buffers)
ret = -ENOMEM;
/*
* Either the driver has accepted a smaller number of buffers,
* or .queue_setup() returned an error
*/
}
mutex_lock(&q->mmap_lock);
q->num_buffers += allocated_buffers;
if (ret < 0) {
/*
* Note: __vb2_queue_free() will subtract 'allocated_buffers'
* from already queued buffers and it will reset q->memory to
* VB2_MEMORY_UNKNOWN.
*/
__vb2_queue_free(q, allocated_buffers);
mutex_unlock(&q->mmap_lock);
return -ENOMEM;
}
mutex_unlock(&q->mmap_lock);
/*
* Return the number of successfully allocated buffers
* to the userspace.
*/
*count = allocated_buffers;
return 0;
error:
if (no_previous_buffers) {
mutex_lock(&q->mmap_lock);
q->memory = VB2_MEMORY_UNKNOWN;
mutex_unlock(&q->mmap_lock);
}
return ret;
}
EXPORT_SYMBOL_GPL(vb2_core_create_bufs);
void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
{
if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
return NULL;
return call_ptr_memop(vaddr, vb, vb->planes[plane_no].mem_priv);
}
EXPORT_SYMBOL_GPL(vb2_plane_vaddr);
void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
{
if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
return NULL;
return call_ptr_memop(cookie, vb, vb->planes[plane_no].mem_priv);
}
EXPORT_SYMBOL_GPL(vb2_plane_cookie);
void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
{
struct vb2_queue *q = vb->vb2_queue;
unsigned long flags;
if (WARN_ON(vb->state != VB2_BUF_STATE_ACTIVE))
return;
if (WARN_ON(state != VB2_BUF_STATE_DONE &&
state != VB2_BUF_STATE_ERROR &&
state != VB2_BUF_STATE_QUEUED))
state = VB2_BUF_STATE_ERROR;
#ifdef CONFIG_VIDEO_ADV_DEBUG
/*
* Although this is not a callback, it still does have to balance
* with the buf_queue op. So update this counter manually.
*/
vb->cnt_buf_done++;
#endif
dprintk(q, 4, "done processing on buffer %d, state: %s\n",
vb->index, vb2_state_name(state));
if (state != VB2_BUF_STATE_QUEUED)
__vb2_buf_mem_finish(vb);
spin_lock_irqsave(&q->done_lock, flags);
if (state == VB2_BUF_STATE_QUEUED) {
vb->state = VB2_BUF_STATE_QUEUED;
} else {
/* Add the buffer to the done buffers list */
list_add_tail(&vb->done_entry, &q->done_list);
vb->state = state;
}
atomic_dec(&q->owned_by_drv_count);
if (state != VB2_BUF_STATE_QUEUED && vb->req_obj.req) {
media_request_object_unbind(&vb->req_obj);
media_request_object_put(&vb->req_obj);
}
spin_unlock_irqrestore(&q->done_lock, flags);
trace_vb2_buf_done(q, vb);
switch (state) {
case VB2_BUF_STATE_QUEUED:
return;
default:
/* Inform any processes that may be waiting for buffers */
wake_up(&q->done_wq);
break;
}
}
EXPORT_SYMBOL_GPL(vb2_buffer_done);
void vb2_discard_done(struct vb2_queue *q)
{
struct vb2_buffer *vb;
unsigned long flags;
spin_lock_irqsave(&q->done_lock, flags);
list_for_each_entry(vb, &q->done_list, done_entry)
vb->state = VB2_BUF_STATE_ERROR;
spin_unlock_irqrestore(&q->done_lock, flags);
}
EXPORT_SYMBOL_GPL(vb2_discard_done);
/*
* __prepare_mmap() - prepare an MMAP buffer
*/
static int __prepare_mmap(struct vb2_buffer *vb)
{
int ret = 0;
ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
vb, vb->planes);
return ret ? ret : call_vb_qop(vb, buf_prepare, vb);
}
/*
* __prepare_userptr() - prepare a USERPTR buffer
*/
static int __prepare_userptr(struct vb2_buffer *vb)
{
struct vb2_plane planes[VB2_MAX_PLANES];
struct vb2_queue *q = vb->vb2_queue;
void *mem_priv;
unsigned int plane;
int ret = 0;
bool reacquired = vb->planes[0].mem_priv == NULL;
memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
/* Copy relevant information provided by the userspace */
ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
vb, planes);
if (ret)
return ret;
for (plane = 0; plane < vb->num_planes; ++plane) {
/* Skip the plane if already verified */
if (vb->planes[plane].m.userptr &&
vb->planes[plane].m.userptr == planes[plane].m.userptr
&& vb->planes[plane].length == planes[plane].length)
continue;
dprintk(q, 3, "userspace address for plane %d changed, reacquiring memory\n",
plane);
/* Check if the provided plane buffer is large enough */
if (planes[plane].length < vb->planes[plane].min_length) {
dprintk(q, 1, "provided buffer size %u is less than setup size %u for plane %d\n",
planes[plane].length,
vb->planes[plane].min_length,
plane);
ret = -EINVAL;
goto err;
}
/* Release previously acquired memory if present */
if (vb->planes[plane].mem_priv) {
if (!reacquired) {
reacquired = true;
vb->copied_timestamp = 0;
call_void_vb_qop(vb, buf_cleanup, vb);
}
call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
}
vb->planes[plane].mem_priv = NULL;
vb->planes[plane].bytesused = 0;
vb->planes[plane].length = 0;
vb->planes[plane].m.userptr = 0;
vb->planes[plane].data_offset = 0;
/* Acquire each plane's memory */
mem_priv = call_ptr_memop(get_userptr,
vb,
q->alloc_devs[plane] ? : q->dev,
planes[plane].m.userptr,
planes[plane].length);
if (IS_ERR(mem_priv)) {
dprintk(q, 1, "failed acquiring userspace memory for plane %d\n",
plane);
ret = PTR_ERR(mem_priv);
goto err;
}
vb->planes[plane].mem_priv = mem_priv;
}
/*
* Now that everything is in order, copy relevant information
* provided by userspace.
*/
for (plane = 0; plane < vb->num_planes; ++plane) {
vb->planes[plane].bytesused = planes[plane].bytesused;
vb->planes[plane].length = planes[plane].length;
vb->planes[plane].m.userptr = planes[plane].m.userptr;
vb->planes[plane].data_offset = planes[plane].data_offset;
}
if (reacquired) {
/*
* One or more planes changed, so we must call buf_init to do
* the driver-specific initialization on the newly acquired
* buffer, if provided.
*/
ret = call_vb_qop(vb, buf_init, vb);
if (ret) {
dprintk(q, 1, "buffer initialization failed\n");
goto err;
}
}
ret = call_vb_qop(vb, buf_prepare, vb);
if (ret) {
dprintk(q, 1, "buffer preparation failed\n");
call_void_vb_qop(vb, buf_cleanup, vb);
goto err;
}
return 0;
err:
/* In case of errors, release planes that were already acquired */
for (plane = 0; plane < vb->num_planes; ++plane) {
if (vb->planes[plane].mem_priv)
call_void_memop(vb, put_userptr,
vb->planes[plane].mem_priv);
vb->planes[plane].mem_priv = NULL;
vb->planes[plane].m.userptr = 0;
vb->planes[plane].length = 0;
}
return ret;
}
/*
* __prepare_dmabuf() - prepare a DMABUF buffer
*/
static int __prepare_dmabuf(struct vb2_buffer *vb)
{
struct vb2_plane planes[VB2_MAX_PLANES];
struct vb2_queue *q = vb->vb2_queue;
void *mem_priv;
unsigned int plane;
int ret = 0;
bool reacquired = vb->planes[0].mem_priv == NULL;
memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
/* Copy relevant information provided by the userspace */
ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
vb, planes);
if (ret)
return ret;
for (plane = 0; plane < vb->num_planes; ++plane) {
struct dma_buf *dbuf = dma_buf_get(planes[plane].m.fd);
if (IS_ERR_OR_NULL(dbuf)) {
dprintk(q, 1, "invalid dmabuf fd for plane %d\n",
plane);
ret = -EINVAL;
goto err;
}
/* use DMABUF size if length is not provided */
if (planes[plane].length == 0)
planes[plane].length = dbuf->size;
if (planes[plane].length < vb->planes[plane].min_length) {
dprintk(q, 1, "invalid dmabuf length %u for plane %d, minimum length %u\n",
planes[plane].length, plane,
vb->planes[plane].min_length);
dma_buf_put(dbuf);
ret = -EINVAL;
goto err;
}
/* Skip the plane if already verified */
if (dbuf == vb->planes[plane].dbuf &&
vb->planes[plane].length == planes[plane].length) {
dma_buf_put(dbuf);
continue;
}
dprintk(q, 3, "buffer for plane %d changed\n", plane);
if (!reacquired) {
reacquired = true;
vb->copied_timestamp = 0;
call_void_vb_qop(vb, buf_cleanup, vb);
}
/* Release previously acquired memory if present */
__vb2_plane_dmabuf_put(vb, &vb->planes[plane]);
vb->planes[plane].bytesused = 0;
vb->planes[plane].length = 0;
vb->planes[plane].m.fd = 0;
vb->planes[plane].data_offset = 0;
/* Acquire each plane's memory */
mem_priv = call_ptr_memop(attach_dmabuf,
vb,
q->alloc_devs[plane] ? : q->dev,
dbuf,
planes[plane].length);
if (IS_ERR(mem_priv)) {
dprintk(q, 1, "failed to attach dmabuf\n");
ret = PTR_ERR(mem_priv);
dma_buf_put(dbuf);
goto err;
}
vb->planes[plane].dbuf = dbuf;
vb->planes[plane].mem_priv = mem_priv;
}
/*
* This pins the buffer(s) with dma_buf_map_attachment()). It's done
* here instead just before the DMA, while queueing the buffer(s) so
* userspace knows sooner rather than later if the dma-buf map fails.
*/
for (plane = 0; plane < vb->num_planes; ++plane) {
if (vb->planes[plane].dbuf_mapped)
continue;
ret = call_memop(vb, map_dmabuf, vb->planes[plane].mem_priv);
if (ret) {
dprintk(q, 1, "failed to map dmabuf for plane %d\n",
plane);
goto err;
}
vb->planes[plane].dbuf_mapped = 1;
}
/*
* Now that everything is in order, copy relevant information
* provided by userspace.
*/
for (plane = 0; plane < vb->num_planes; ++plane) {
vb->planes[plane].bytesused = planes[plane].bytesused;
vb->planes[plane].length = planes[plane].length;
vb->planes[plane].m.fd = planes[plane].m.fd;
vb->planes[plane].data_offset = planes[plane].data_offset;
}
if (reacquired) {
/*
* Call driver-specific initialization on the newly acquired buffer,
* if provided.
*/
ret = call_vb_qop(vb, buf_init, vb);
if (ret) {
dprintk(q, 1, "buffer initialization failed\n");
goto err;
}
}
ret = call_vb_qop(vb, buf_prepare, vb);
if (ret) {
dprintk(q, 1, "buffer preparation failed\n");
call_void_vb_qop(vb, buf_cleanup, vb);
goto err;
}
return 0;
err:
/* In case of errors, release planes that were already acquired */
__vb2_buf_dmabuf_put(vb);
return ret;
}
/*
* __enqueue_in_driver() - enqueue a vb2_buffer in driver for processing
*/
static void __enqueue_in_driver(struct vb2_buffer *vb)
{
struct vb2_queue *q = vb->vb2_queue;
vb->state = VB2_BUF_STATE_ACTIVE;
atomic_inc(&q->owned_by_drv_count);
trace_vb2_buf_queue(q, vb);
call_void_vb_qop(vb, buf_queue, vb);
}
static int __buf_prepare(struct vb2_buffer *vb)
{
struct vb2_queue *q = vb->vb2_queue;
enum vb2_buffer_state orig_state = vb->state;
int ret;
if (q->error) {
dprintk(q, 1, "fatal error occurred on queue\n");
return -EIO;
}
if (vb->prepared)
return 0;
WARN_ON(vb->synced);
if (q->is_output) {
ret = call_vb_qop(vb, buf_out_validate, vb);
if (ret) {
dprintk(q, 1, "buffer validation failed\n");
return ret;
}
}
vb->state = VB2_BUF_STATE_PREPARING;
switch (q->memory) {
case VB2_MEMORY_MMAP:
ret = __prepare_mmap(vb);
break;
case VB2_MEMORY_USERPTR:
ret = __prepare_userptr(vb);
break;
case VB2_MEMORY_DMABUF:
ret = __prepare_dmabuf(vb);
break;
default:
WARN(1, "Invalid queue type\n");
ret = -EINVAL;
break;
}
if (ret) {
dprintk(q, 1, "buffer preparation failed: %d\n", ret);
vb->state = orig_state;
return ret;
}
__vb2_buf_mem_prepare(vb);
vb->prepared = 1;
vb->state = orig_state;
return 0;
}
static int vb2_req_prepare(struct media_request_object *obj)
{
struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
int ret;
if (WARN_ON(vb->state != VB2_BUF_STATE_IN_REQUEST))
return -EINVAL;
mutex_lock(vb->vb2_queue->lock);
ret = __buf_prepare(vb);
mutex_unlock(vb->vb2_queue->lock);
return ret;
}
static void __vb2_dqbuf(struct vb2_buffer *vb);
static void vb2_req_unprepare(struct media_request_object *obj)
{
struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
mutex_lock(vb->vb2_queue->lock);
__vb2_dqbuf(vb);
vb->state = VB2_BUF_STATE_IN_REQUEST;
mutex_unlock(vb->vb2_queue->lock);
WARN_ON(!vb->req_obj.req);
}
static void vb2_req_queue(struct media_request_object *obj)
{
struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
int err;
mutex_lock(vb->vb2_queue->lock);
/*
* There is no method to propagate an error from vb2_core_qbuf(),
* so if this returns a non-0 value, then WARN.
*
* The only exception is -EIO which is returned if q->error is
* set. We just ignore that, and expect this will be caught the
* next time vb2_req_prepare() is called.
*/
err = vb2_core_qbuf(vb->vb2_queue, vb, NULL, NULL);
WARN_ON_ONCE(err && err != -EIO);
mutex_unlock(vb->vb2_queue->lock);
}
static void vb2_req_unbind(struct media_request_object *obj)
{
struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
if (vb->state == VB2_BUF_STATE_IN_REQUEST)
call_void_bufop(vb->vb2_queue, init_buffer, vb);
}
static void vb2_req_release(struct media_request_object *obj)
{
struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
if (vb->state == VB2_BUF_STATE_IN_REQUEST) {
vb->state = VB2_BUF_STATE_DEQUEUED;
if (vb->request)
media_request_put(vb->request);
vb->request = NULL;
}
}
static const struct media_request_object_ops vb2_core_req_ops = {
.prepare = vb2_req_prepare,
.unprepare = vb2_req_unprepare,
.queue = vb2_req_queue,
.unbind = vb2_req_unbind,
.release = vb2_req_release,
};
bool vb2_request_object_is_buffer(struct media_request_object *obj)
{
return obj->ops == &vb2_core_req_ops;
}
EXPORT_SYMBOL_GPL(vb2_request_object_is_buffer);
unsigned int vb2_request_buffer_cnt(struct media_request *req)
{
struct media_request_object *obj;
unsigned long flags;
unsigned int buffer_cnt = 0;
spin_lock_irqsave(&req->lock, flags);
list_for_each_entry(obj, &req->objects, list)
if (vb2_request_object_is_buffer(obj))
buffer_cnt++;
spin_unlock_irqrestore(&req->lock, flags);
return buffer_cnt;
}
EXPORT_SYMBOL_GPL(vb2_request_buffer_cnt);
int vb2_core_prepare_buf(struct vb2_queue *q, struct vb2_buffer *vb, void *pb)
{
int ret;
if (vb->state != VB2_BUF_STATE_DEQUEUED) {
dprintk(q, 1, "invalid buffer state %s\n",
vb2_state_name(vb->state));
return -EINVAL;
}
if (vb->prepared) {
dprintk(q, 1, "buffer already prepared\n");
return -EINVAL;
}
ret = __buf_prepare(vb);
if (ret)
return ret;
/* Fill buffer information for the userspace */
call_void_bufop(q, fill_user_buffer, vb, pb);
dprintk(q, 2, "prepare of buffer %d succeeded\n", vb->index);
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_prepare_buf);
/*
* vb2_start_streaming() - Attempt to start streaming.
* @q: videobuf2 queue
*
* Attempt to start streaming. When this function is called there must be
* at least q->min_queued_buffers queued up (i.e. the minimum
* number of buffers required for the DMA engine to function). If the
* @start_streaming op fails it is supposed to return all the driver-owned
* buffers back to vb2 in state QUEUED. Check if that happened and if
* not warn and reclaim them forcefully.
*/
static int vb2_start_streaming(struct vb2_queue *q)
{
struct vb2_buffer *vb;
int ret;
/*
* If any buffers were queued before streamon,
* we can now pass them to driver for processing.
*/
list_for_each_entry(vb, &q->queued_list, queued_entry)
__enqueue_in_driver(vb);
/* Tell the driver to start streaming */
q->start_streaming_called = 1;
ret = call_qop(q, start_streaming, q,
atomic_read(&q->owned_by_drv_count));
if (!ret)
return 0;
q->start_streaming_called = 0;
dprintk(q, 1, "driver refused to start streaming\n");
/*
* If you see this warning, then the driver isn't cleaning up properly
* after a failed start_streaming(). See the start_streaming()
* documentation in videobuf2-core.h for more information how buffers
* should be returned to vb2 in start_streaming().
*/
if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
unsigned i;
/*
* Forcefully reclaim buffers if the driver did not
* correctly return them to vb2.
*/
for (i = 0; i < vb2_get_num_buffers(q); ++i) {
vb = vb2_get_buffer(q, i);
if (!vb)
continue;
if (vb->state == VB2_BUF_STATE_ACTIVE)
vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED);
}
/* Must be zero now */
WARN_ON(atomic_read(&q->owned_by_drv_count));
}
/*
* If done_list is not empty, then start_streaming() didn't call
* vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED) but STATE_ERROR or
* STATE_DONE.
*/
WARN_ON(!list_empty(&q->done_list));
return ret;
}
int vb2_core_qbuf(struct vb2_queue *q, struct vb2_buffer *vb, void *pb,
struct media_request *req)
{
enum vb2_buffer_state orig_state;
int ret;
if (q->error) {
dprintk(q, 1, "fatal error occurred on queue\n");
return -EIO;
}
if (!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
q->requires_requests) {
dprintk(q, 1, "qbuf requires a request\n");
return -EBADR;
}
if ((req && q->uses_qbuf) ||
(!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
q->uses_requests)) {
dprintk(q, 1, "queue in wrong mode (qbuf vs requests)\n");
return -EBUSY;
}
if (req) {
int ret;
q->uses_requests = 1;
if (vb->state != VB2_BUF_STATE_DEQUEUED) {
dprintk(q, 1, "buffer %d not in dequeued state\n",
vb->index);
return -EINVAL;
}
if (q->is_output && !vb->prepared) {
ret = call_vb_qop(vb, buf_out_validate, vb);
if (ret) {
dprintk(q, 1, "buffer validation failed\n");
return ret;
}
}
media_request_object_init(&vb->req_obj);
/* Make sure the request is in a safe state for updating. */
ret = media_request_lock_for_update(req);
if (ret)
return ret;
ret = media_request_object_bind(req, &vb2_core_req_ops,
q, true, &vb->req_obj);
media_request_unlock_for_update(req);
if (ret)
return ret;
vb->state = VB2_BUF_STATE_IN_REQUEST;
/*
* Increment the refcount and store the request.
* The request refcount is decremented again when the
* buffer is dequeued. This is to prevent vb2_buffer_done()
* from freeing the request from interrupt context, which can
* happen if the application closed the request fd after
* queueing the request.
*/
media_request_get(req);
vb->request = req;
/* Fill buffer information for the userspace */
if (pb) {
call_void_bufop(q, copy_timestamp, vb, pb);
call_void_bufop(q, fill_user_buffer, vb, pb);
}
dprintk(q, 2, "qbuf of buffer %d succeeded\n", vb->index);
return 0;
}
if (vb->state != VB2_BUF_STATE_IN_REQUEST)
q->uses_qbuf = 1;
switch (vb->state) {
case VB2_BUF_STATE_DEQUEUED:
case VB2_BUF_STATE_IN_REQUEST:
if (!vb->prepared) {
ret = __buf_prepare(vb);
if (ret)
return ret;
}
break;
case VB2_BUF_STATE_PREPARING:
dprintk(q, 1, "buffer still being prepared\n");
return -EINVAL;
default:
dprintk(q, 1, "invalid buffer state %s\n",
vb2_state_name(vb->state));
return -EINVAL;
}
/*
* Add to the queued buffers list, a buffer will stay on it until
* dequeued in dqbuf.
*/
orig_state = vb->state;
list_add_tail(&vb->queued_entry, &q->queued_list);
q->queued_count++;
q->waiting_for_buffers = false;
vb->state = VB2_BUF_STATE_QUEUED;
if (pb)
call_void_bufop(q, copy_timestamp, vb, pb);
trace_vb2_qbuf(q, vb);
/*
* If already streaming, give the buffer to driver for processing.
* If not, the buffer will be given to driver on next streamon.
*/
if (q->start_streaming_called)
__enqueue_in_driver(vb);
/* Fill buffer information for the userspace */
if (pb)
call_void_bufop(q, fill_user_buffer, vb, pb);
/*
* If streamon has been called, and we haven't yet called
* start_streaming() since not enough buffers were queued, and
* we now have reached the minimum number of queued buffers,
* then we can finally call start_streaming().
*/
if (q->streaming && !q->start_streaming_called &&
q->queued_count >= q->min_queued_buffers) {
ret = vb2_start_streaming(q);
if (ret) {
/*
* Since vb2_core_qbuf will return with an error,
* we should return it to state DEQUEUED since
* the error indicates that the buffer wasn't queued.
*/
list_del(&vb->queued_entry);
q->queued_count--;
vb->state = orig_state;
return ret;
}
}
dprintk(q, 2, "qbuf of buffer %d succeeded\n", vb->index);
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_qbuf);
/*
* __vb2_wait_for_done_vb() - wait for a buffer to become available
* for dequeuing
*
* Will sleep if required for nonblocking == false.
*/
static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
{
/*
* All operations on vb_done_list are performed under done_lock
* spinlock protection. However, buffers may be removed from
* it and returned to userspace only while holding both driver's
* lock and the done_lock spinlock. Thus we can be sure that as
* long as we hold the driver's lock, the list will remain not
* empty if list_empty() check succeeds.
*/
for (;;) {
int ret;
if (q->waiting_in_dqbuf) {
dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
return -EBUSY;
}
if (!q->streaming) {
dprintk(q, 1, "streaming off, will not wait for buffers\n");
return -EINVAL;
}
if (q->error) {
dprintk(q, 1, "Queue in error state, will not wait for buffers\n");
return -EIO;
}
if (q->last_buffer_dequeued) {
dprintk(q, 3, "last buffer dequeued already, will not wait for buffers\n");
return -EPIPE;
}
if (!list_empty(&q->done_list)) {
/*
* Found a buffer that we were waiting for.
*/
break;
}
if (nonblocking) {
dprintk(q, 3, "nonblocking and no buffers to dequeue, will not wait\n");
return -EAGAIN;
}
q->waiting_in_dqbuf = 1;
/*
* We are streaming and blocking, wait for another buffer to
* become ready or for streamoff. Driver's lock is released to
* allow streamoff or qbuf to be called while waiting.
*/
call_void_qop(q, wait_prepare, q);
/*
* All locks have been released, it is safe to sleep now.
*/
dprintk(q, 3, "will sleep waiting for buffers\n");
ret = wait_event_interruptible(q->done_wq,
!list_empty(&q->done_list) || !q->streaming ||
q->error);
/*
* We need to reevaluate both conditions again after reacquiring
* the locks or return an error if one occurred.
*/
call_void_qop(q, wait_finish, q);
q->waiting_in_dqbuf = 0;
if (ret) {
dprintk(q, 1, "sleep was interrupted\n");
return ret;
}
}
return 0;
}
/*
* __vb2_get_done_vb() - get a buffer ready for dequeuing
*
* Will sleep if required for nonblocking == false.
*/
static int __vb2_get_done_vb(struct vb2_queue *q, struct vb2_buffer **vb,
void *pb, int nonblocking)
{
unsigned long flags;
int ret = 0;
/*
* Wait for at least one buffer to become available on the done_list.
*/
ret = __vb2_wait_for_done_vb(q, nonblocking);
if (ret)
return ret;
/*
* Driver's lock has been held since we last verified that done_list
* is not empty, so no need for another list_empty(done_list) check.
*/
spin_lock_irqsave(&q->done_lock, flags);
*vb = list_first_entry(&q->done_list, struct vb2_buffer, done_entry);
/*
* Only remove the buffer from done_list if all planes can be
* handled. Some cases such as V4L2 file I/O and DVB have pb
* == NULL; skip the check then as there's nothing to verify.
*/
if (pb)
ret = call_bufop(q, verify_planes_array, *vb, pb);
if (!ret)
list_del(&(*vb)->done_entry);
spin_unlock_irqrestore(&q->done_lock, flags);
return ret;
}
int vb2_wait_for_all_buffers(struct vb2_queue *q)
{
if (!q->streaming) {
dprintk(q, 1, "streaming off, will not wait for buffers\n");
return -EINVAL;
}
if (q->start_streaming_called)
wait_event(q->done_wq, !atomic_read(&q->owned_by_drv_count));
return 0;
}
EXPORT_SYMBOL_GPL(vb2_wait_for_all_buffers);
/*
* __vb2_dqbuf() - bring back the buffer to the DEQUEUED state
*/
static void __vb2_dqbuf(struct vb2_buffer *vb)
{
struct vb2_queue *q = vb->vb2_queue;
/* nothing to do if the buffer is already dequeued */
if (vb->state == VB2_BUF_STATE_DEQUEUED)
return;
vb->state = VB2_BUF_STATE_DEQUEUED;
call_void_bufop(q, init_buffer, vb);
}
int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb,
bool nonblocking)
{
struct vb2_buffer *vb = NULL;
int ret;
ret = __vb2_get_done_vb(q, &vb, pb, nonblocking);
if (ret < 0)
return ret;
switch (vb->state) {
case VB2_BUF_STATE_DONE:
dprintk(q, 3, "returning done buffer\n");
break;
case VB2_BUF_STATE_ERROR:
dprintk(q, 3, "returning done buffer with errors\n");
break;
default:
dprintk(q, 1, "invalid buffer state %s\n",
vb2_state_name(vb->state));
return -EINVAL;
}
call_void_vb_qop(vb, buf_finish, vb);
vb->prepared = 0;
if (pindex)
*pindex = vb->index;
/* Fill buffer information for the userspace */
if (pb)
call_void_bufop(q, fill_user_buffer, vb, pb);
/* Remove from vb2 queue */
list_del(&vb->queued_entry);
q->queued_count--;
trace_vb2_dqbuf(q, vb);
/* go back to dequeued state */
__vb2_dqbuf(vb);
if (WARN_ON(vb->req_obj.req)) {
media_request_object_unbind(&vb->req_obj);
media_request_object_put(&vb->req_obj);
}
if (vb->request)
media_request_put(vb->request);
vb->request = NULL;
dprintk(q, 2, "dqbuf of buffer %d, state: %s\n",
vb->index, vb2_state_name(vb->state));
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_dqbuf);
/*
* __vb2_queue_cancel() - cancel and stop (pause) streaming
*
* Removes all queued buffers from driver's queue and all buffers queued by
* userspace from vb2's queue. Returns to state after reqbufs.
*/
static void __vb2_queue_cancel(struct vb2_queue *q)
{
unsigned int i;
/*
* Tell driver to stop all transactions and release all queued
* buffers.
*/
if (q->start_streaming_called)
call_void_qop(q, stop_streaming, q);
if (q->streaming)
call_void_qop(q, unprepare_streaming, q);
/*
* If you see this warning, then the driver isn't cleaning up properly
* in stop_streaming(). See the stop_streaming() documentation in
* videobuf2-core.h for more information how buffers should be returned
* to vb2 in stop_streaming().
*/
if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
for (i = 0; i < vb2_get_num_buffers(q); i++) {
struct vb2_buffer *vb = vb2_get_buffer(q, i);
if (!vb)
continue;
if (vb->state == VB2_BUF_STATE_ACTIVE) {
pr_warn("driver bug: stop_streaming operation is leaving buffer %u in active state\n",
vb->index);
vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
}
}
/* Must be zero now */
WARN_ON(atomic_read(&q->owned_by_drv_count));
}
q->streaming = 0;
q->start_streaming_called = 0;
q->queued_count = 0;
q->error = 0;
q->uses_requests = 0;
q->uses_qbuf = 0;
/*
* Remove all buffers from vb2's list...
*/
INIT_LIST_HEAD(&q->queued_list);
/*
* ...and done list; userspace will not receive any buffers it
* has not already dequeued before initiating cancel.
*/
INIT_LIST_HEAD(&q->done_list);
atomic_set(&q->owned_by_drv_count, 0);
wake_up_all(&q->done_wq);
/*
* Reinitialize all buffers for next use.
* Make sure to call buf_finish for any queued buffers. Normally
* that's done in dqbuf, but that's not going to happen when we
* cancel the whole queue. Note: this code belongs here, not in
* __vb2_dqbuf() since in vb2_core_dqbuf() there is a critical
* call to __fill_user_buffer() after buf_finish(). That order can't
* be changed, so we can't move the buf_finish() to __vb2_dqbuf().
*/
for (i = 0; i < vb2_get_num_buffers(q); i++) {
struct vb2_buffer *vb;
struct media_request *req;
vb = vb2_get_buffer(q, i);
if (!vb)
continue;
req = vb->req_obj.req;
/*
* If a request is associated with this buffer, then
* call buf_request_cancel() to give the driver to complete()
* related request objects. Otherwise those objects would
* never complete.
*/
if (req) {
enum media_request_state state;
unsigned long flags;
spin_lock_irqsave(&req->lock, flags);
state = req->state;
spin_unlock_irqrestore(&req->lock, flags);
if (state == MEDIA_REQUEST_STATE_QUEUED)
call_void_vb_qop(vb, buf_request_complete, vb);
}
__vb2_buf_mem_finish(vb);
if (vb->prepared) {
call_void_vb_qop(vb, buf_finish, vb);
vb->prepared = 0;
}
__vb2_dqbuf(vb);
if (vb->req_obj.req) {
media_request_object_unbind(&vb->req_obj);
media_request_object_put(&vb->req_obj);
}
if (vb->request)
media_request_put(vb->request);
vb->request = NULL;
vb->copied_timestamp = 0;
}
}
int vb2_core_streamon(struct vb2_queue *q, unsigned int type)
{
unsigned int q_num_bufs = vb2_get_num_buffers(q);
int ret;
if (type != q->type) {
dprintk(q, 1, "invalid stream type\n");
return -EINVAL;
}
if (q->streaming) {
dprintk(q, 3, "already streaming\n");
return 0;
}
if (!q_num_bufs) {
dprintk(q, 1, "no buffers have been allocated\n");
return -EINVAL;
}
if (q_num_bufs < q->min_queued_buffers) {
dprintk(q, 1, "need at least %u queued buffers\n",
q->min_queued_buffers);
return -EINVAL;
}
ret = call_qop(q, prepare_streaming, q);
if (ret)
return ret;
/*
* Tell driver to start streaming provided sufficient buffers
* are available.
*/
if (q->queued_count >= q->min_queued_buffers) {
ret = vb2_start_streaming(q);
if (ret)
goto unprepare;
}
q->streaming = 1;
dprintk(q, 3, "successful\n");
return 0;
unprepare:
call_void_qop(q, unprepare_streaming, q);
return ret;
}
EXPORT_SYMBOL_GPL(vb2_core_streamon);
void vb2_queue_error(struct vb2_queue *q)
{
q->error = 1;
wake_up_all(&q->done_wq);
}
EXPORT_SYMBOL_GPL(vb2_queue_error);
int vb2_core_streamoff(struct vb2_queue *q, unsigned int type)
{
if (type != q->type) {
dprintk(q, 1, "invalid stream type\n");
return -EINVAL;
}
/*
* Cancel will pause streaming and remove all buffers from the driver
* and vb2, effectively returning control over them to userspace.
*
* Note that we do this even if q->streaming == 0: if you prepare or
* queue buffers, and then call streamoff without ever having called
* streamon, you would still expect those buffers to be returned to
* their normal dequeued state.
*/
__vb2_queue_cancel(q);
q->waiting_for_buffers = !q->is_output;
q->last_buffer_dequeued = false;
dprintk(q, 3, "successful\n");
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_streamoff);
/*
* __find_plane_by_offset() - find plane associated with the given offset
*/
static int __find_plane_by_offset(struct vb2_queue *q, unsigned long offset,
struct vb2_buffer **vb, unsigned int *plane)
{
unsigned int buffer;
/*
* Sanity checks to ensure the lock is held, MEMORY_MMAP is
* used and fileio isn't active.
*/
lockdep_assert_held(&q->mmap_lock);
if (q->memory != VB2_MEMORY_MMAP) {
dprintk(q, 1, "queue is not currently set up for mmap\n");
return -EINVAL;
}
if (vb2_fileio_is_active(q)) {
dprintk(q, 1, "file io in progress\n");
return -EBUSY;
}
/* Get buffer and plane from the offset */
buffer = (offset >> PLANE_INDEX_SHIFT) & BUFFER_INDEX_MASK;
*plane = (offset >> PAGE_SHIFT) & PLANE_INDEX_MASK;
*vb = vb2_get_buffer(q, buffer);
if (!*vb)
return -EINVAL;
if (*plane >= (*vb)->num_planes)
return -EINVAL;
return 0;
}
int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
struct vb2_buffer *vb, unsigned int plane, unsigned int flags)
{
struct vb2_plane *vb_plane;
int ret;
struct dma_buf *dbuf;
if (q->memory != VB2_MEMORY_MMAP) {
dprintk(q, 1, "queue is not currently set up for mmap\n");
return -EINVAL;
}
if (!q->mem_ops->get_dmabuf) {
dprintk(q, 1, "queue does not support DMA buffer exporting\n");
return -EINVAL;
}
if (flags & ~(O_CLOEXEC | O_ACCMODE)) {
dprintk(q, 1, "queue does support only O_CLOEXEC and access mode flags\n");
return -EINVAL;
}
if (type != q->type) {
dprintk(q, 1, "invalid buffer type\n");
return -EINVAL;
}
if (plane >= vb->num_planes) {
dprintk(q, 1, "buffer plane out of range\n");
return -EINVAL;
}
if (vb2_fileio_is_active(q)) {
dprintk(q, 1, "expbuf: file io in progress\n");
return -EBUSY;
}
vb_plane = &vb->planes[plane];
dbuf = call_ptr_memop(get_dmabuf,
vb,
vb_plane->mem_priv,
flags & O_ACCMODE);
if (IS_ERR_OR_NULL(dbuf)) {
dprintk(q, 1, "failed to export buffer %d, plane %d\n",
vb->index, plane);
return -EINVAL;
}
ret = dma_buf_fd(dbuf, flags & ~O_ACCMODE);
if (ret < 0) {
dprintk(q, 3, "buffer %d, plane %d failed to export (%d)\n",
vb->index, plane, ret);
dma_buf_put(dbuf);
return ret;
}
dprintk(q, 3, "buffer %d, plane %d exported as %d descriptor\n",
vb->index, plane, ret);
*fd = ret;
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_expbuf);
int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma)
{
unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
struct vb2_buffer *vb;
unsigned int plane = 0;
int ret;
unsigned long length;
/*
* Check memory area access mode.
*/
if (!(vma->vm_flags & VM_SHARED)) {
dprintk(q, 1, "invalid vma flags, VM_SHARED needed\n");
return -EINVAL;
}
if (q->is_output) {
if (!(vma->vm_flags & VM_WRITE)) {
dprintk(q, 1, "invalid vma flags, VM_WRITE needed\n");
return -EINVAL;
}
} else {
if (!(vma->vm_flags & VM_READ)) {
dprintk(q, 1, "invalid vma flags, VM_READ needed\n");
return -EINVAL;
}
}
mutex_lock(&q->mmap_lock);
/*
* Find the plane corresponding to the offset passed by userspace. This
* will return an error if not MEMORY_MMAP or file I/O is in progress.
*/
ret = __find_plane_by_offset(q, offset, &vb, &plane);
if (ret)
goto unlock;
/*
* MMAP requires page_aligned buffers.
* The buffer length was page_aligned at __vb2_buf_mem_alloc(),
* so, we need to do the same here.
*/
length = PAGE_ALIGN(vb->planes[plane].length);
if (length < (vma->vm_end - vma->vm_start)) {
dprintk(q, 1,
"MMAP invalid, as it would overflow buffer length\n");
ret = -EINVAL;
goto unlock;
}
/*
* vm_pgoff is treated in V4L2 API as a 'cookie' to select a buffer,
* not as a in-buffer offset. We always want to mmap a whole buffer
* from its beginning.
*/
vma->vm_pgoff = 0;
ret = call_memop(vb, mmap, vb->planes[plane].mem_priv, vma);
unlock:
mutex_unlock(&q->mmap_lock);
if (ret)
return ret;
dprintk(q, 3, "buffer %u, plane %d successfully mapped\n", vb->index, plane);
return 0;
}
EXPORT_SYMBOL_GPL(vb2_mmap);
#ifndef CONFIG_MMU
unsigned long vb2_get_unmapped_area(struct vb2_queue *q,
unsigned long addr,
unsigned long len,
unsigned long pgoff,
unsigned long flags)
{
unsigned long offset = pgoff << PAGE_SHIFT;
struct vb2_buffer *vb;
unsigned int plane;
void *vaddr;
int ret;
mutex_lock(&q->mmap_lock);
/*
* Find the plane corresponding to the offset passed by userspace. This
* will return an error if not MEMORY_MMAP or file I/O is in progress.
*/
ret = __find_plane_by_offset(q, offset, &vb, &plane);
if (ret)
goto unlock;
vaddr = vb2_plane_vaddr(vb, plane);
mutex_unlock(&q->mmap_lock);
return vaddr ? (unsigned long)vaddr : -EINVAL;
unlock:
mutex_unlock(&q->mmap_lock);
return ret;
}
EXPORT_SYMBOL_GPL(vb2_get_unmapped_area);
#endif
int vb2_core_queue_init(struct vb2_queue *q)
{
/*
* Sanity check
*/
/*
* For drivers who don't support max_num_buffers ensure
* a backward compatibility.
*/
if (!q->max_num_buffers)
q->max_num_buffers = VB2_MAX_FRAME;
/* The maximum is limited by offset cookie encoding pattern */
q->max_num_buffers = min_t(unsigned int, q->max_num_buffers, MAX_BUFFER_INDEX);
if (WARN_ON(!q) ||
WARN_ON(!q->ops) ||
WARN_ON(!q->mem_ops) ||
WARN_ON(!q->type) ||
WARN_ON(!q->io_modes) ||
WARN_ON(!q->ops->queue_setup) ||
WARN_ON(!q->ops->buf_queue))
return -EINVAL;
if (WARN_ON(q->max_num_buffers > MAX_BUFFER_INDEX) ||
WARN_ON(q->min_queued_buffers > q->max_num_buffers))
return -EINVAL;
if (WARN_ON(q->requires_requests && !q->supports_requests))
return -EINVAL;
/*
* This combination is not allowed since a non-zero value of
* q->min_queued_buffers can cause vb2_core_qbuf() to fail if
* it has to call start_streaming(), and the Request API expects
* that queueing a request (and thus queueing a buffer contained
* in that request) will always succeed. There is no method of
* propagating an error back to userspace.
*/
if (WARN_ON(q->supports_requests && q->min_queued_buffers))
return -EINVAL;
INIT_LIST_HEAD(&q->queued_list);
INIT_LIST_HEAD(&q->done_list);
spin_lock_init(&q->done_lock);
mutex_init(&q->mmap_lock);
init_waitqueue_head(&q->done_wq);
q->memory = VB2_MEMORY_UNKNOWN;
if (q->buf_struct_size == 0)
q->buf_struct_size = sizeof(struct vb2_buffer);
if (q->bidirectional)
q->dma_dir = DMA_BIDIRECTIONAL;
else
q->dma_dir = q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
if (q->name[0] == '\0')
snprintf(q->name, sizeof(q->name), "%s-%p",
q->is_output ? "out" : "cap", q);
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_queue_init);
static int __vb2_init_fileio(struct vb2_queue *q, int read);
static int __vb2_cleanup_fileio(struct vb2_queue *q);
void vb2_core_queue_release(struct vb2_queue *q)
{
__vb2_cleanup_fileio(q);
__vb2_queue_cancel(q);
mutex_lock(&q->mmap_lock);
__vb2_queue_free(q, vb2_get_num_buffers(q));
kfree(q->bufs);
q->bufs = NULL;
mutex_unlock(&q->mmap_lock);
}
EXPORT_SYMBOL_GPL(vb2_core_queue_release);
__poll_t vb2_core_poll(struct vb2_queue *q, struct file *file,
poll_table *wait)
{
__poll_t req_events = poll_requested_events(wait);
struct vb2_buffer *vb = NULL;
unsigned long flags;
/*
* poll_wait() MUST be called on the first invocation on all the
* potential queues of interest, even if we are not interested in their
* events during this first call. Failure to do so will result in
* queue's events to be ignored because the poll_table won't be capable
* of adding new wait queues thereafter.
*/
poll_wait(file, &q->done_wq, wait);
if (!q->is_output && !(req_events & (EPOLLIN | EPOLLRDNORM)))
return 0;
if (q->is_output && !(req_events & (EPOLLOUT | EPOLLWRNORM)))
return 0;
/*
* Start file I/O emulator only if streaming API has not been used yet.
*/
if (vb2_get_num_buffers(q) == 0 && !vb2_fileio_is_active(q)) {
if (!q->is_output && (q->io_modes & VB2_READ) &&
(req_events & (EPOLLIN | EPOLLRDNORM))) {
if (__vb2_init_fileio(q, 1))
return EPOLLERR;
}
if (q->is_output && (q->io_modes & VB2_WRITE) &&
(req_events & (EPOLLOUT | EPOLLWRNORM))) {
if (__vb2_init_fileio(q, 0))
return EPOLLERR;
/*
* Write to OUTPUT queue can be done immediately.
*/
return EPOLLOUT | EPOLLWRNORM;
}
}
/*
* There is nothing to wait for if the queue isn't streaming, or if the
* error flag is set.
*/
if (!vb2_is_streaming(q) || q->error)
return EPOLLERR;
/*
* If this quirk is set and QBUF hasn't been called yet then
* return EPOLLERR as well. This only affects capture queues, output
* queues will always initialize waiting_for_buffers to false.
* This quirk is set by V4L2 for backwards compatibility reasons.
*/
if (q->quirk_poll_must_check_waiting_for_buffers &&
q->waiting_for_buffers && (req_events & (EPOLLIN | EPOLLRDNORM)))
return EPOLLERR;
/*
* For output streams you can call write() as long as there are fewer
* buffers queued than there are buffers available.
*/
if (q->is_output && q->fileio && q->queued_count < vb2_get_num_buffers(q))
return EPOLLOUT | EPOLLWRNORM;
if (list_empty(&q->done_list)) {
/*
* If the last buffer was dequeued from a capture queue,
* return immediately. DQBUF will return -EPIPE.
*/
if (q->last_buffer_dequeued)
return EPOLLIN | EPOLLRDNORM;
}
/*
* Take first buffer available for dequeuing.
*/
spin_lock_irqsave(&q->done_lock, flags);
if (!list_empty(&q->done_list))
vb = list_first_entry(&q->done_list, struct vb2_buffer,
done_entry);
spin_unlock_irqrestore(&q->done_lock, flags);
if (vb && (vb->state == VB2_BUF_STATE_DONE
|| vb->state == VB2_BUF_STATE_ERROR)) {
return (q->is_output) ?
EPOLLOUT | EPOLLWRNORM :
EPOLLIN | EPOLLRDNORM;
}
return 0;
}
EXPORT_SYMBOL_GPL(vb2_core_poll);
/*
* struct vb2_fileio_buf - buffer context used by file io emulator
*
* vb2 provides a compatibility layer and emulator of file io (read and
* write) calls on top of streaming API. This structure is used for
* tracking context related to the buffers.
*/
struct vb2_fileio_buf {
void *vaddr;
unsigned int size;
unsigned int pos;
unsigned int queued:1;
};
/*
* struct vb2_fileio_data - queue context used by file io emulator
*
* @cur_index: the index of the buffer currently being read from or
* written to. If equal to number of buffers in the vb2_queue
* then a new buffer must be dequeued.
* @initial_index: in the read() case all buffers are queued up immediately
* in __vb2_init_fileio() and __vb2_perform_fileio() just cycles
* buffers. However, in the write() case no buffers are initially
* queued, instead whenever a buffer is full it is queued up by
* __vb2_perform_fileio(). Only once all available buffers have
* been queued up will __vb2_perform_fileio() start to dequeue
* buffers. This means that initially __vb2_perform_fileio()
* needs to know what buffer index to use when it is queuing up
* the buffers for the first time. That initial index is stored
* in this field. Once it is equal to number of buffers in the
* vb2_queue all available buffers have been queued and
* __vb2_perform_fileio() should start the normal dequeue/queue cycle.
*
* vb2 provides a compatibility layer and emulator of file io (read and
* write) calls on top of streaming API. For proper operation it required
* this structure to save the driver state between each call of the read
* or write function.
*/
struct vb2_fileio_data {
unsigned int count;
unsigned int type;
unsigned int memory;
struct vb2_fileio_buf bufs[VB2_MAX_FRAME];
unsigned int cur_index;
unsigned int initial_index;
unsigned int q_count;
unsigned int dq_count;
unsigned read_once:1;
unsigned write_immediately:1;
};
/*
* __vb2_init_fileio() - initialize file io emulator
* @q: videobuf2 queue
* @read: mode selector (1 means read, 0 means write)
*/
static int __vb2_init_fileio(struct vb2_queue *q, int read)
{
struct vb2_fileio_data *fileio;
struct vb2_buffer *vb;
int i, ret;
unsigned int count = 0;
/*
* Sanity check
*/
if (WARN_ON((read && !(q->io_modes & VB2_READ)) ||
(!read && !(q->io_modes & VB2_WRITE))))
return -EINVAL;
/*
* Check if device supports mapping buffers to kernel virtual space.
*/
if (!q->mem_ops->vaddr)
return -EBUSY;
/*
* Check if streaming api has not been already activated.
*/
if (q->streaming || vb2_get_num_buffers(q) > 0)
return -EBUSY;
/*
* Start with q->min_queued_buffers + 1, driver can increase it in
* queue_setup()
*
* 'min_queued_buffers' buffers need to be queued up before you
* can start streaming, plus 1 for userspace (or in this case,
* kernelspace) processing.
*/
count = max(2, q->min_queued_buffers + 1);
dprintk(q, 3, "setting up file io: mode %s, count %d, read_once %d, write_immediately %d\n",
(read) ? "read" : "write", count, q->fileio_read_once,
q->fileio_write_immediately);
fileio = kzalloc(sizeof(*fileio), GFP_KERNEL);
if (fileio == NULL)
return -ENOMEM;
fileio->read_once = q->fileio_read_once;
fileio->write_immediately = q->fileio_write_immediately;
/*
* Request buffers and use MMAP type to force driver
* to allocate buffers by itself.
*/
fileio->count = count;
fileio->memory = VB2_MEMORY_MMAP;
fileio->type = q->type;
q->fileio = fileio;
ret = vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
if (ret)
goto err_kfree;
/*
* Userspace can never add or delete buffers later, so there
* will never be holes. It is safe to assume that vb2_get_buffer(q, 0)
* will always return a valid vb pointer
*/
vb = vb2_get_buffer(q, 0);
/*
* Check if plane_count is correct
* (multiplane buffers are not supported).
*/
if (vb->num_planes != 1) {
ret = -EBUSY;
goto err_reqbufs;
}
/*
* Get kernel address of each buffer.
*/
for (i = 0; i < vb2_get_num_buffers(q); i++) {
/* vb can never be NULL when using fileio. */
vb = vb2_get_buffer(q, i);
fileio->bufs[i].vaddr = vb2_plane_vaddr(vb, 0);
if (fileio->bufs[i].vaddr == NULL) {
ret = -EINVAL;
goto err_reqbufs;
}
fileio->bufs[i].size = vb2_plane_size(vb, 0);
}
/*
* Read mode requires pre queuing of all buffers.
*/
if (read) {
/*
* Queue all buffers.
*/
for (i = 0; i < vb2_get_num_buffers(q); i++) {
struct vb2_buffer *vb2 = vb2_get_buffer(q, i);
if (!vb2)
continue;
ret = vb2_core_qbuf(q, vb2, NULL, NULL);
if (ret)
goto err_reqbufs;
fileio->bufs[i].queued = 1;
}
/*
* All buffers have been queued, so mark that by setting
* initial_index to the number of buffers in the vb2_queue
*/
fileio->initial_index = vb2_get_num_buffers(q);
fileio->cur_index = fileio->initial_index;
}
/*
* Start streaming.
*/
ret = vb2_core_streamon(q, q->type);
if (ret)
goto err_reqbufs;
return ret;
err_reqbufs:
fileio->count = 0;
vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
err_kfree:
q->fileio = NULL;
kfree(fileio);
return ret;
}
/*
* __vb2_cleanup_fileio() - free resourced used by file io emulator
* @q: videobuf2 queue
*/
static int __vb2_cleanup_fileio(struct vb2_queue *q)
{
struct vb2_fileio_data *fileio = q->fileio;
if (fileio) {
vb2_core_streamoff(q, q->type);
q->fileio = NULL;
fileio->count = 0;
vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
kfree(fileio);
dprintk(q, 3, "file io emulator closed\n");
}
return 0;
}
/*
* __vb2_perform_fileio() - perform a single file io (read or write) operation
* @q: videobuf2 queue
* @data: pointed to target userspace buffer
* @count: number of bytes to read or write
* @ppos: file handle position tracking pointer
* @nonblock: mode selector (1 means blocking calls, 0 means nonblocking)
* @read: access mode selector (1 means read, 0 means write)
*/
static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
loff_t *ppos, int nonblock, int read)
{
struct vb2_fileio_data *fileio;
struct vb2_fileio_buf *buf;
bool is_multiplanar = q->is_multiplanar;
/*
* When using write() to write data to an output video node the vb2 core
* should copy timestamps if V4L2_BUF_FLAG_TIMESTAMP_COPY is set. Nobody
* else is able to provide this information with the write() operation.
*/
bool copy_timestamp = !read && q->copy_timestamp;
unsigned index;
int ret;
dprintk(q, 3, "mode %s, offset %ld, count %zd, %sblocking\n",
read ? "read" : "write", (long)*ppos, count,
nonblock ? "non" : "");
if (!data)
return -EINVAL;
if (q->waiting_in_dqbuf) {
dprintk(q, 3, "another dup()ped fd is %s\n",
read ? "reading" : "writing");
return -EBUSY;
}
/*
* Initialize emulator on first call.
*/
if (!vb2_fileio_is_active(q)) {
ret = __vb2_init_fileio(q, read);
dprintk(q, 3, "vb2_init_fileio result: %d\n", ret);
if (ret)
return ret;
}
fileio = q->fileio;
/*
* Check if we need to dequeue the buffer.
*/
index = fileio->cur_index;
if (index >= vb2_get_num_buffers(q)) {
struct vb2_buffer *b;
/*
* Call vb2_dqbuf to get buffer back.
*/
ret = vb2_core_dqbuf(q, &index, NULL, nonblock);
dprintk(q, 5, "vb2_dqbuf result: %d\n", ret);
if (ret)
return ret;
fileio->dq_count += 1;
fileio->cur_index = index;
buf = &fileio->bufs[index];
/* b can never be NULL when using fileio. */
b = vb2_get_buffer(q, index);
/*
* Get number of bytes filled by the driver
*/
buf->pos = 0;
buf->queued = 0;
buf->size = read ? vb2_get_plane_payload(b, 0)
: vb2_plane_size(b, 0);
/* Compensate for data_offset on read in the multiplanar case. */
if (is_multiplanar && read &&
b->planes[0].data_offset < buf->size) {
buf->pos = b->planes[0].data_offset;
buf->size -= buf->pos;
}
} else {
buf = &fileio->bufs[index];
}
/*
* Limit count on last few bytes of the buffer.
*/
if (buf->pos + count > buf->size) {
count = buf->size - buf->pos;
dprintk(q, 5, "reducing read count: %zd\n", count);
}
/*
* Transfer data to userspace.
*/
dprintk(q, 3, "copying %zd bytes - buffer %d, offset %u\n",
count, index, buf->pos);
if (read)
ret = copy_to_user(data, buf->vaddr + buf->pos, count);
else
ret = copy_from_user(buf->vaddr + buf->pos, data, count);
if (ret) {
dprintk(q, 3, "error copying data\n");
return -EFAULT;
}
/*
* Update counters.
*/
buf->pos += count;
*ppos += count;
/*
* Queue next buffer if required.
*/
if (buf->pos == buf->size || (!read && fileio->write_immediately)) {
/* b can never be NULL when using fileio. */
struct vb2_buffer *b = vb2_get_buffer(q, index);
/*
* Check if this is the last buffer to read.
*/
if (read && fileio->read_once && fileio->dq_count == 1) {
dprintk(q, 3, "read limit reached\n");
return __vb2_cleanup_fileio(q);
}
/*
* Call vb2_qbuf and give buffer to the driver.
*/
b->planes[0].bytesused = buf->pos;
if (copy_timestamp)
b->timestamp = ktime_get_ns();
ret = vb2_core_qbuf(q, b, NULL, NULL);
dprintk(q, 5, "vb2_qbuf result: %d\n", ret);
if (ret)
return ret;
/*
* Buffer has been queued, update the status
*/
buf->pos = 0;
buf->queued = 1;
buf->size = vb2_plane_size(b, 0);
fileio->q_count += 1;
/*
* If we are queuing up buffers for the first time, then
* increase initial_index by one.
*/
if (fileio->initial_index < vb2_get_num_buffers(q))
fileio->initial_index++;
/*
* The next buffer to use is either a buffer that's going to be
* queued for the first time (initial_index < number of buffers in the vb2_queue)
* or it is equal to the number of buffers in the vb2_queue,
* meaning that the next time we need to dequeue a buffer since
* we've now queued up all the 'first time' buffers.
*/
fileio->cur_index = fileio->initial_index;
}
/*
* Return proper number of bytes processed.
*/
if (ret == 0)
ret = count;
return ret;
}
size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
loff_t *ppos, int nonblocking)
{
return __vb2_perform_fileio(q, data, count, ppos, nonblocking, 1);
}
EXPORT_SYMBOL_GPL(vb2_read);
size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
loff_t *ppos, int nonblocking)
{
return __vb2_perform_fileio(q, (char __user *) data, count,
ppos, nonblocking, 0);
}
EXPORT_SYMBOL_GPL(vb2_write);
struct vb2_threadio_data {
struct task_struct *thread;
vb2_thread_fnc fnc;
void *priv;
bool stop;
};
static int vb2_thread(void *data)
{
struct vb2_queue *q = data;
struct vb2_threadio_data *threadio = q->threadio;
bool copy_timestamp = false;
unsigned prequeue = 0;
unsigned index = 0;
int ret = 0;
if (q->is_output) {
prequeue = vb2_get_num_buffers(q);
copy_timestamp = q->copy_timestamp;
}
set_freezable();
for (;;) {
struct vb2_buffer *vb;
/*
* Call vb2_dqbuf to get buffer back.
*/
if (prequeue) {
vb = vb2_get_buffer(q, index++);
if (!vb)
continue;
prequeue--;
} else {
call_void_qop(q, wait_finish, q);
if (!threadio->stop)
ret = vb2_core_dqbuf(q, &index, NULL, 0);
call_void_qop(q, wait_prepare, q);
dprintk(q, 5, "file io: vb2_dqbuf result: %d\n", ret);
if (!ret)
vb = vb2_get_buffer(q, index);
}
if (ret || threadio->stop)
break;
try_to_freeze();
if (vb->state != VB2_BUF_STATE_ERROR)
if (threadio->fnc(vb, threadio->priv))
break;
call_void_qop(q, wait_finish, q);
if (copy_timestamp)
vb->timestamp = ktime_get_ns();
if (!threadio->stop)
ret = vb2_core_qbuf(q, vb, NULL, NULL);
call_void_qop(q, wait_prepare, q);
if (ret || threadio->stop)
break;
}
/* Hmm, linux becomes *very* unhappy without this ... */
while (!kthread_should_stop()) {
set_current_state(TASK_INTERRUPTIBLE);
schedule();
}
return 0;
}
/*
* This function should not be used for anything else but the videobuf2-dvb
* support. If you think you have another good use-case for this, then please
* contact the linux-media mailinglist first.
*/
int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv,
const char *thread_name)
{
struct vb2_threadio_data *threadio;
int ret = 0;
if (q->threadio)
return -EBUSY;
if (vb2_is_busy(q))
return -EBUSY;
if (WARN_ON(q->fileio))
return -EBUSY;
threadio = kzalloc(sizeof(*threadio), GFP_KERNEL);
if (threadio == NULL)
return -ENOMEM;
threadio->fnc = fnc;
threadio->priv = priv;
ret = __vb2_init_fileio(q, !q->is_output);
dprintk(q, 3, "file io: vb2_init_fileio result: %d\n", ret);
if (ret)
goto nomem;
q->threadio = threadio;
threadio->thread = kthread_run(vb2_thread, q, "vb2-%s", thread_name);
if (IS_ERR(threadio->thread)) {
ret = PTR_ERR(threadio->thread);
threadio->thread = NULL;
goto nothread;
}
return 0;
nothread:
__vb2_cleanup_fileio(q);
nomem:
kfree(threadio);
return ret;
}
EXPORT_SYMBOL_GPL(vb2_thread_start);
int vb2_thread_stop(struct vb2_queue *q)
{
struct vb2_threadio_data *threadio = q->threadio;
int err;
if (threadio == NULL)
return 0;
threadio->stop = true;
/* Wake up all pending sleeps in the thread */
vb2_queue_error(q);
err = kthread_stop(threadio->thread);
__vb2_cleanup_fileio(q);
threadio->thread = NULL;
kfree(threadio);
q->threadio = NULL;
return err;
}
EXPORT_SYMBOL_GPL(vb2_thread_stop);
MODULE_DESCRIPTION("Media buffer core framework");
MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>, Marek Szyprowski");
MODULE_LICENSE("GPL");
MODULE_IMPORT_NS(DMA_BUF);
// SPDX-License-Identifier: GPL-2.0
/*
* device.h - generic, centralized driver model
*
* Copyright (c) 2001-2003 Patrick Mochel <mochel@osdl.org>
* Copyright (c) 2004-2009 Greg Kroah-Hartman <gregkh@suse.de>
* Copyright (c) 2008-2009 Novell Inc.
*
* See Documentation/driver-api/driver-model/ for more information.
*/
#ifndef _DEVICE_H_
#define _DEVICE_H_
#include <linux/dev_printk.h>
#include <linux/energy_model.h>
#include <linux/ioport.h>
#include <linux/kobject.h>
#include <linux/klist.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/compiler.h>
#include <linux/types.h>
#include <linux/mutex.h>
#include <linux/pm.h>
#include <linux/atomic.h>
#include <linux/uidgid.h>
#include <linux/gfp.h>
#include <linux/overflow.h>
#include <linux/device/bus.h>
#include <linux/device/class.h>
#include <linux/device/driver.h>
#include <linux/cleanup.h>
#include <asm/device.h>
struct device;
struct device_private;
struct device_driver;
struct driver_private;
struct module;
struct class;
struct subsys_private;
struct device_node;
struct fwnode_handle;
struct iommu_group;
struct dev_pin_info;
struct dev_iommu;
struct msi_device_data;
/**
* struct subsys_interface - interfaces to device functions
* @name: name of the device function
* @subsys: subsystem of the devices to attach to
* @node: the list of functions registered at the subsystem
* @add_dev: device hookup to device function handler
* @remove_dev: device hookup to device function handler
*
* Simple interfaces attached to a subsystem. Multiple interfaces can
* attach to a subsystem and its devices. Unlike drivers, they do not
* exclusively claim or control devices. Interfaces usually represent
* a specific functionality of a subsystem/class of devices.
*/
struct subsys_interface {
const char *name;
const struct bus_type *subsys;
struct list_head node;
int (*add_dev)(struct device *dev, struct subsys_interface *sif);
void (*remove_dev)(struct device *dev, struct subsys_interface *sif);
};
int subsys_interface_register(struct subsys_interface *sif);
void subsys_interface_unregister(struct subsys_interface *sif);
int subsys_system_register(const struct bus_type *subsys,
const struct attribute_group **groups);
int subsys_virtual_register(const struct bus_type *subsys,
const struct attribute_group **groups);
/*
* The type of device, "struct device" is embedded in. A class
* or bus can contain devices of different types
* like "partitions" and "disks", "mouse" and "event".
* This identifies the device type and carries type-specific
* information, equivalent to the kobj_type of a kobject.
* If "name" is specified, the uevent will contain it in
* the DEVTYPE variable.
*/
struct device_type {
const char *name;
const struct attribute_group **groups;
int (*uevent)(const struct device *dev, struct kobj_uevent_env *env);
char *(*devnode)(const struct device *dev, umode_t *mode,
kuid_t *uid, kgid_t *gid);
void (*release)(struct device *dev);
const struct dev_pm_ops *pm;
};
/**
* struct device_attribute - Interface for exporting device attributes.
* @attr: sysfs attribute definition.
* @show: Show handler.
* @store: Store handler.
*/
struct device_attribute {
struct attribute attr;
ssize_t (*show)(struct device *dev, struct device_attribute *attr,
char *buf);
ssize_t (*store)(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count);
};
/**
* struct dev_ext_attribute - Exported device attribute with extra context.
* @attr: Exported device attribute.
* @var: Pointer to context.
*/
struct dev_ext_attribute {
struct device_attribute attr;
void *var;
};
ssize_t device_show_ulong(struct device *dev, struct device_attribute *attr,
char *buf);
ssize_t device_store_ulong(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count);
ssize_t device_show_int(struct device *dev, struct device_attribute *attr,
char *buf);
ssize_t device_store_int(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count);
ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
char *buf);
ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count);
/**
* DEVICE_ATTR - Define a device attribute.
* @_name: Attribute name.
* @_mode: File mode.
* @_show: Show handler. Optional, but mandatory if attribute is readable.
* @_store: Store handler. Optional, but mandatory if attribute is writable.
*
* Convenience macro for defining a struct device_attribute.
*
* For example, ``DEVICE_ATTR(foo, 0644, foo_show, foo_store);`` expands to:
*
* .. code-block:: c
*
* struct device_attribute dev_attr_foo = {
* .attr = { .name = "foo", .mode = 0644 },
* .show = foo_show,
* .store = foo_store,
* };
*/
#define DEVICE_ATTR(_name, _mode, _show, _store) \
struct device_attribute dev_attr_##_name = __ATTR(_name, _mode, _show, _store)
/**
* DEVICE_ATTR_PREALLOC - Define a preallocated device attribute.
* @_name: Attribute name.
* @_mode: File mode.
* @_show: Show handler. Optional, but mandatory if attribute is readable.
* @_store: Store handler. Optional, but mandatory if attribute is writable.
*
* Like DEVICE_ATTR(), but ``SYSFS_PREALLOC`` is set on @_mode.
*/
#define DEVICE_ATTR_PREALLOC(_name, _mode, _show, _store) \
struct device_attribute dev_attr_##_name = \
__ATTR_PREALLOC(_name, _mode, _show, _store)
/**
* DEVICE_ATTR_RW - Define a read-write device attribute.
* @_name: Attribute name.
*
* Like DEVICE_ATTR(), but @_mode is 0644, @_show is <_name>_show,
* and @_store is <_name>_store.
*/
#define DEVICE_ATTR_RW(_name) \
struct device_attribute dev_attr_##_name = __ATTR_RW(_name)
/**
* DEVICE_ATTR_ADMIN_RW - Define an admin-only read-write device attribute.
* @_name: Attribute name.
*
* Like DEVICE_ATTR_RW(), but @_mode is 0600.
*/
#define DEVICE_ATTR_ADMIN_RW(_name) \
struct device_attribute dev_attr_##_name = __ATTR_RW_MODE(_name, 0600)
/**
* DEVICE_ATTR_RO - Define a readable device attribute.
* @_name: Attribute name.
*
* Like DEVICE_ATTR(), but @_mode is 0444 and @_show is <_name>_show.
*/
#define DEVICE_ATTR_RO(_name) \
struct device_attribute dev_attr_##_name = __ATTR_RO(_name)
/**
* DEVICE_ATTR_ADMIN_RO - Define an admin-only readable device attribute.
* @_name: Attribute name.
*
* Like DEVICE_ATTR_RO(), but @_mode is 0400.
*/
#define DEVICE_ATTR_ADMIN_RO(_name) \
struct device_attribute dev_attr_##_name = __ATTR_RO_MODE(_name, 0400)
/**
* DEVICE_ATTR_WO - Define an admin-only writable device attribute.
* @_name: Attribute name.
*
* Like DEVICE_ATTR(), but @_mode is 0200 and @_store is <_name>_store.
*/
#define DEVICE_ATTR_WO(_name) \
struct device_attribute dev_attr_##_name = __ATTR_WO(_name)
/**
* DEVICE_ULONG_ATTR - Define a device attribute backed by an unsigned long.
* @_name: Attribute name.
* @_mode: File mode.
* @_var: Identifier of unsigned long.
*
* Like DEVICE_ATTR(), but @_show and @_store are automatically provided
* such that reads and writes to the attribute from userspace affect @_var.
*/
#define DEVICE_ULONG_ATTR(_name, _mode, _var) \
struct dev_ext_attribute dev_attr_##_name = \
{ __ATTR(_name, _mode, device_show_ulong, device_store_ulong), &(_var) }
/**
* DEVICE_INT_ATTR - Define a device attribute backed by an int.
* @_name: Attribute name.
* @_mode: File mode.
* @_var: Identifier of int.
*
* Like DEVICE_ULONG_ATTR(), but @_var is an int.
*/
#define DEVICE_INT_ATTR(_name, _mode, _var) \
struct dev_ext_attribute dev_attr_##_name = \
{ __ATTR(_name, _mode, device_show_int, device_store_int), &(_var) }
/**
* DEVICE_BOOL_ATTR - Define a device attribute backed by a bool.
* @_name: Attribute name.
* @_mode: File mode.
* @_var: Identifier of bool.
*
* Like DEVICE_ULONG_ATTR(), but @_var is a bool.
*/
#define DEVICE_BOOL_ATTR(_name, _mode, _var) \
struct dev_ext_attribute dev_attr_##_name = \
{ __ATTR(_name, _mode, device_show_bool, device_store_bool), &(_var) }
#define DEVICE_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \
struct device_attribute dev_attr_##_name = \
__ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store)
int device_create_file(struct device *device,
const struct device_attribute *entry);
void device_remove_file(struct device *dev,
const struct device_attribute *attr);
bool device_remove_file_self(struct device *dev,
const struct device_attribute *attr);
int __must_check device_create_bin_file(struct device *dev,
const struct bin_attribute *attr);
void device_remove_bin_file(struct device *dev,
const struct bin_attribute *attr);
/* device resource management */
typedef void (*dr_release_t)(struct device *dev, void *res);
typedef int (*dr_match_t)(struct device *dev, void *res, void *match_data);
void *__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp,
int nid, const char *name) __malloc;
#define devres_alloc(release, size, gfp) \
__devres_alloc_node(release, size, gfp, NUMA_NO_NODE, #release)
#define devres_alloc_node(release, size, gfp, nid) \
__devres_alloc_node(release, size, gfp, nid, #release)
void devres_for_each_res(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data,
void (*fn)(struct device *, void *, void *),
void *data);
void devres_free(void *res);
void devres_add(struct device *dev, void *res);
void *devres_find(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data);
void *devres_get(struct device *dev, void *new_res,
dr_match_t match, void *match_data);
void *devres_remove(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data);
int devres_destroy(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data);
int devres_release(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data);
/* devres group */
void * __must_check devres_open_group(struct device *dev, void *id, gfp_t gfp);
void devres_close_group(struct device *dev, void *id);
void devres_remove_group(struct device *dev, void *id);
int devres_release_group(struct device *dev, void *id);
/* managed devm_k.alloc/kfree for device drivers */
void *devm_kmalloc(struct device *dev, size_t size, gfp_t gfp) __alloc_size(2);
void *devm_krealloc(struct device *dev, void *ptr, size_t size,
gfp_t gfp) __must_check __realloc_size(3);
__printf(3, 0) char *devm_kvasprintf(struct device *dev, gfp_t gfp,
const char *fmt, va_list ap) __malloc;
__printf(3, 4) char *devm_kasprintf(struct device *dev, gfp_t gfp,
const char *fmt, ...) __malloc;
static inline void *devm_kzalloc(struct device *dev, size_t size, gfp_t gfp)
{
return devm_kmalloc(dev, size, gfp | __GFP_ZERO);
}
static inline void *devm_kmalloc_array(struct device *dev,
size_t n, size_t size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
return devm_kmalloc(dev, bytes, flags);
}
static inline void *devm_kcalloc(struct device *dev,
size_t n, size_t size, gfp_t flags)
{
return devm_kmalloc_array(dev, n, size, flags | __GFP_ZERO);
}
static inline __realloc_size(3, 4) void * __must_check
devm_krealloc_array(struct device *dev, void *p, size_t new_n, size_t new_size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
return NULL;
return devm_krealloc(dev, p, bytes, flags);
}
void devm_kfree(struct device *dev, const void *p);
char *devm_kstrdup(struct device *dev, const char *s, gfp_t gfp) __malloc;
const char *devm_kstrdup_const(struct device *dev, const char *s, gfp_t gfp);
void *devm_kmemdup(struct device *dev, const void *src, size_t len, gfp_t gfp)
__realloc_size(3);
unsigned long devm_get_free_pages(struct device *dev,
gfp_t gfp_mask, unsigned int order);
void devm_free_pages(struct device *dev, unsigned long addr);
#ifdef CONFIG_HAS_IOMEM
void __iomem *devm_ioremap_resource(struct device *dev,
const struct resource *res);
void __iomem *devm_ioremap_resource_wc(struct device *dev,
const struct resource *res);
void __iomem *devm_of_iomap(struct device *dev,
struct device_node *node, int index,
resource_size_t *size);
#else
static inline
void __iomem *devm_ioremap_resource(struct device *dev,
const struct resource *res)
{
return ERR_PTR(-EINVAL);
}
static inline
void __iomem *devm_ioremap_resource_wc(struct device *dev,
const struct resource *res)
{
return ERR_PTR(-EINVAL);
}
static inline
void __iomem *devm_of_iomap(struct device *dev,
struct device_node *node, int index,
resource_size_t *size)
{
return ERR_PTR(-EINVAL);
}
#endif
/* allows to add/remove a custom action to devres stack */
void devm_remove_action(struct device *dev, void (*action)(void *), void *data);
void devm_release_action(struct device *dev, void (*action)(void *), void *data);
int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name);
#define devm_add_action(dev, action, data) \
__devm_add_action(dev, action, data, #action)
static inline int __devm_add_action_or_reset(struct device *dev, void (*action)(void *),
void *data, const char *name)
{
int ret;
ret = __devm_add_action(dev, action, data, name);
if (ret)
action(data);
return ret;
}
#define devm_add_action_or_reset(dev, action, data) \
__devm_add_action_or_reset(dev, action, data, #action)
/**
* devm_alloc_percpu - Resource-managed alloc_percpu
* @dev: Device to allocate per-cpu memory for
* @type: Type to allocate per-cpu memory for
*
* Managed alloc_percpu. Per-cpu memory allocated with this function is
* automatically freed on driver detach.
*
* RETURNS:
* Pointer to allocated memory on success, NULL on failure.
*/
#define devm_alloc_percpu(dev, type) \
((typeof(type) __percpu *)__devm_alloc_percpu((dev), sizeof(type), \
__alignof__(type)))
void __percpu *__devm_alloc_percpu(struct device *dev, size_t size,
size_t align);
void devm_free_percpu(struct device *dev, void __percpu *pdata);
struct device_dma_parameters {
/*
* a low level driver may set these to teach IOMMU code about
* sg limitations.
*/
unsigned int max_segment_size;
unsigned int min_align_mask;
unsigned long segment_boundary_mask;
};
/**
* enum device_link_state - Device link states.
* @DL_STATE_NONE: The presence of the drivers is not being tracked.
* @DL_STATE_DORMANT: None of the supplier/consumer drivers is present.
* @DL_STATE_AVAILABLE: The supplier driver is present, but the consumer is not.
* @DL_STATE_CONSUMER_PROBE: The consumer is probing (supplier driver present).
* @DL_STATE_ACTIVE: Both the supplier and consumer drivers are present.
* @DL_STATE_SUPPLIER_UNBIND: The supplier driver is unbinding.
*/
enum device_link_state {
DL_STATE_NONE = -1,
DL_STATE_DORMANT = 0,
DL_STATE_AVAILABLE,
DL_STATE_CONSUMER_PROBE,
DL_STATE_ACTIVE,
DL_STATE_SUPPLIER_UNBIND,
};
/*
* Device link flags.
*
* STATELESS: The core will not remove this link automatically.
* AUTOREMOVE_CONSUMER: Remove the link automatically on consumer driver unbind.
* PM_RUNTIME: If set, the runtime PM framework will use this link.
* RPM_ACTIVE: Run pm_runtime_get_sync() on the supplier during link creation.
* AUTOREMOVE_SUPPLIER: Remove the link automatically on supplier driver unbind.
* AUTOPROBE_CONSUMER: Probe consumer driver automatically after supplier binds.
* MANAGED: The core tracks presence of supplier/consumer drivers (internal).
* SYNC_STATE_ONLY: Link only affects sync_state() behavior.
* INFERRED: Inferred from data (eg: firmware) and not from driver actions.
*/
#define DL_FLAG_STATELESS BIT(0)
#define DL_FLAG_AUTOREMOVE_CONSUMER BIT(1)
#define DL_FLAG_PM_RUNTIME BIT(2)
#define DL_FLAG_RPM_ACTIVE BIT(3)
#define DL_FLAG_AUTOREMOVE_SUPPLIER BIT(4)
#define DL_FLAG_AUTOPROBE_CONSUMER BIT(5)
#define DL_FLAG_MANAGED BIT(6)
#define DL_FLAG_SYNC_STATE_ONLY BIT(7)
#define DL_FLAG_INFERRED BIT(8)
#define DL_FLAG_CYCLE BIT(9)
/**
* enum dl_dev_state - Device driver presence tracking information.
* @DL_DEV_NO_DRIVER: There is no driver attached to the device.
* @DL_DEV_PROBING: A driver is probing.
* @DL_DEV_DRIVER_BOUND: The driver has been bound to the device.
* @DL_DEV_UNBINDING: The driver is unbinding from the device.
*/
enum dl_dev_state {
DL_DEV_NO_DRIVER = 0,
DL_DEV_PROBING,
DL_DEV_DRIVER_BOUND,
DL_DEV_UNBINDING,
};
/**
* enum device_removable - Whether the device is removable. The criteria for a
* device to be classified as removable is determined by its subsystem or bus.
* @DEVICE_REMOVABLE_NOT_SUPPORTED: This attribute is not supported for this
* device (default).
* @DEVICE_REMOVABLE_UNKNOWN: Device location is Unknown.
* @DEVICE_FIXED: Device is not removable by the user.
* @DEVICE_REMOVABLE: Device is removable by the user.
*/
enum device_removable {
DEVICE_REMOVABLE_NOT_SUPPORTED = 0, /* must be 0 */
DEVICE_REMOVABLE_UNKNOWN,
DEVICE_FIXED,
DEVICE_REMOVABLE,
};
/**
* struct dev_links_info - Device data related to device links.
* @suppliers: List of links to supplier devices.
* @consumers: List of links to consumer devices.
* @defer_sync: Hook to global list of devices that have deferred sync_state.
* @status: Driver status information.
*/
struct dev_links_info {
struct list_head suppliers;
struct list_head consumers;
struct list_head defer_sync;
enum dl_dev_state status;
};
/**
* struct dev_msi_info - Device data related to MSI
* @domain: The MSI interrupt domain associated to the device
* @data: Pointer to MSI device data
*/
struct dev_msi_info {
#ifdef CONFIG_GENERIC_MSI_IRQ
struct irq_domain *domain;
struct msi_device_data *data;
#endif
};
/**
* enum device_physical_location_panel - Describes which panel surface of the
* system's housing the device connection point resides on.
* @DEVICE_PANEL_TOP: Device connection point is on the top panel.
* @DEVICE_PANEL_BOTTOM: Device connection point is on the bottom panel.
* @DEVICE_PANEL_LEFT: Device connection point is on the left panel.
* @DEVICE_PANEL_RIGHT: Device connection point is on the right panel.
* @DEVICE_PANEL_FRONT: Device connection point is on the front panel.
* @DEVICE_PANEL_BACK: Device connection point is on the back panel.
* @DEVICE_PANEL_UNKNOWN: The panel with device connection point is unknown.
*/
enum device_physical_location_panel {
DEVICE_PANEL_TOP,
DEVICE_PANEL_BOTTOM,
DEVICE_PANEL_LEFT,
DEVICE_PANEL_RIGHT,
DEVICE_PANEL_FRONT,
DEVICE_PANEL_BACK,
DEVICE_PANEL_UNKNOWN,
};
/**
* enum device_physical_location_vertical_position - Describes vertical
* position of the device connection point on the panel surface.
* @DEVICE_VERT_POS_UPPER: Device connection point is at upper part of panel.
* @DEVICE_VERT_POS_CENTER: Device connection point is at center part of panel.
* @DEVICE_VERT_POS_LOWER: Device connection point is at lower part of panel.
*/
enum device_physical_location_vertical_position {
DEVICE_VERT_POS_UPPER,
DEVICE_VERT_POS_CENTER,
DEVICE_VERT_POS_LOWER,
};
/**
* enum device_physical_location_horizontal_position - Describes horizontal
* position of the device connection point on the panel surface.
* @DEVICE_HORI_POS_LEFT: Device connection point is at left part of panel.
* @DEVICE_HORI_POS_CENTER: Device connection point is at center part of panel.
* @DEVICE_HORI_POS_RIGHT: Device connection point is at right part of panel.
*/
enum device_physical_location_horizontal_position {
DEVICE_HORI_POS_LEFT,
DEVICE_HORI_POS_CENTER,
DEVICE_HORI_POS_RIGHT,
};
/**
* struct device_physical_location - Device data related to physical location
* of the device connection point.
* @panel: Panel surface of the system's housing that the device connection
* point resides on.
* @vertical_position: Vertical position of the device connection point within
* the panel.
* @horizontal_position: Horizontal position of the device connection point
* within the panel.
* @dock: Set if the device connection point resides in a docking station or
* port replicator.
* @lid: Set if this device connection point resides on the lid of laptop
* system.
*/
struct device_physical_location {
enum device_physical_location_panel panel;
enum device_physical_location_vertical_position vertical_position;
enum device_physical_location_horizontal_position horizontal_position;
bool dock;
bool lid;
};
/**
* struct device - The basic device structure
* @parent: The device's "parent" device, the device to which it is attached.
* In most cases, a parent device is some sort of bus or host
* controller. If parent is NULL, the device, is a top-level device,
* which is not usually what you want.
* @p: Holds the private data of the driver core portions of the device.
* See the comment of the struct device_private for detail.
* @kobj: A top-level, abstract class from which other classes are derived.
* @init_name: Initial name of the device.
* @type: The type of device.
* This identifies the device type and carries type-specific
* information.
* @mutex: Mutex to synchronize calls to its driver.
* @bus: Type of bus device is on.
* @driver: Which driver has allocated this
* @platform_data: Platform data specific to the device.
* Example: For devices on custom boards, as typical of embedded
* and SOC based hardware, Linux often uses platform_data to point
* to board-specific structures describing devices and how they
* are wired. That can include what ports are available, chip
* variants, which GPIO pins act in what additional roles, and so
* on. This shrinks the "Board Support Packages" (BSPs) and
* minimizes board-specific #ifdefs in drivers.
* @driver_data: Private pointer for driver specific info.
* @links: Links to suppliers and consumers of this device.
* @power: For device power management.
* See Documentation/driver-api/pm/devices.rst for details.
* @pm_domain: Provide callbacks that are executed during system suspend,
* hibernation, system resume and during runtime PM transitions
* along with subsystem-level and driver-level callbacks.
* @em_pd: device's energy model performance domain
* @pins: For device pin management.
* See Documentation/driver-api/pin-control.rst for details.
* @msi: MSI related data
* @numa_node: NUMA node this device is close to.
* @dma_ops: DMA mapping operations for this device.
* @dma_mask: Dma mask (if dma'ble device).
* @coherent_dma_mask: Like dma_mask, but for alloc_coherent mapping as not all
* hardware supports 64-bit addresses for consistent allocations
* such descriptors.
* @bus_dma_limit: Limit of an upstream bridge or bus which imposes a smaller
* DMA limit than the device itself supports.
* @dma_range_map: map for DMA memory ranges relative to that of RAM
* @dma_parms: A low level driver may set these to teach IOMMU code about
* segment limitations.
* @dma_pools: Dma pools (if dma'ble device).
* @dma_mem: Internal for coherent mem override.
* @cma_area: Contiguous memory area for dma allocations
* @dma_io_tlb_mem: Software IO TLB allocator. Not for driver use.
* @dma_io_tlb_pools: List of transient swiotlb memory pools.
* @dma_io_tlb_lock: Protects changes to the list of active pools.
* @dma_uses_io_tlb: %true if device has used the software IO TLB.
* @archdata: For arch-specific additions.
* @of_node: Associated device tree node.
* @fwnode: Associated device node supplied by platform firmware.
* @devt: For creating the sysfs "dev".
* @id: device instance
* @devres_lock: Spinlock to protect the resource of the device.
* @devres_head: The resources list of the device.
* @class: The class of the device.
* @groups: Optional attribute groups.
* @release: Callback to free the device after all references have
* gone away. This should be set by the allocator of the
* device (i.e. the bus driver that discovered the device).
* @iommu_group: IOMMU group the device belongs to.
* @iommu: Per device generic IOMMU runtime data
* @physical_location: Describes physical location of the device connection
* point in the system housing.
* @removable: Whether the device can be removed from the system. This
* should be set by the subsystem / bus driver that discovered
* the device.
*
* @offline_disabled: If set, the device is permanently online.
* @offline: Set after successful invocation of bus type's .offline().
* @of_node_reused: Set if the device-tree node is shared with an ancestor
* device.
* @state_synced: The hardware state of this device has been synced to match
* the software state of this device by calling the driver/bus
* sync_state() callback.
* @can_match: The device has matched with a driver at least once or it is in
* a bus (like AMBA) which can't check for matching drivers until
* other devices probe successfully.
* @dma_coherent: this particular device is dma coherent, even if the
* architecture supports non-coherent devices.
* @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
* streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
* and optionall (if the coherent mask is large enough) also
* for dma allocations. This flag is managed by the dma ops
* instance from ->dma_supported.
*
* At the lowest level, every device in a Linux system is represented by an
* instance of struct device. The device structure contains the information
* that the device model core needs to model the system. Most subsystems,
* however, track additional information about the devices they host. As a
* result, it is rare for devices to be represented by bare device structures;
* instead, that structure, like kobject structures, is usually embedded within
* a higher-level representation of the device.
*/
struct device {
struct kobject kobj;
struct device *parent;
struct device_private *p;
const char *init_name; /* initial name of the device */
const struct device_type *type;
const struct bus_type *bus; /* type of bus device is on */
struct device_driver *driver; /* which driver has allocated this
device */
void *platform_data; /* Platform specific data, device
core doesn't touch it */
void *driver_data; /* Driver data, set and get with
dev_set_drvdata/dev_get_drvdata */
struct mutex mutex; /* mutex to synchronize calls to
* its driver.
*/
struct dev_links_info links;
struct dev_pm_info power;
struct dev_pm_domain *pm_domain;
#ifdef CONFIG_ENERGY_MODEL
struct em_perf_domain *em_pd;
#endif
#ifdef CONFIG_PINCTRL
struct dev_pin_info *pins;
#endif
struct dev_msi_info msi;
#ifdef CONFIG_DMA_OPS
const struct dma_map_ops *dma_ops;
#endif
u64 *dma_mask; /* dma mask (if dma'able device) */
u64 coherent_dma_mask;/* Like dma_mask, but for
alloc_coherent mappings as
not all hardware supports
64 bit addresses for consistent
allocations such descriptors. */
u64 bus_dma_limit; /* upstream dma constraint */
const struct bus_dma_region *dma_range_map;
struct device_dma_parameters *dma_parms;
struct list_head dma_pools; /* dma pools (if dma'ble) */
#ifdef CONFIG_DMA_DECLARE_COHERENT
struct dma_coherent_mem *dma_mem; /* internal for coherent mem
override */
#endif
#ifdef CONFIG_DMA_CMA
struct cma *cma_area; /* contiguous memory area for dma
allocations */
#endif
#ifdef CONFIG_SWIOTLB
struct io_tlb_mem *dma_io_tlb_mem;
#endif
#ifdef CONFIG_SWIOTLB_DYNAMIC
struct list_head dma_io_tlb_pools;
spinlock_t dma_io_tlb_lock;
bool dma_uses_io_tlb;
#endif
/* arch specific additions */
struct dev_archdata archdata;
struct device_node *of_node; /* associated device tree node */
struct fwnode_handle *fwnode; /* firmware device node */
#ifdef CONFIG_NUMA
int numa_node; /* NUMA node this device is close to */
#endif
dev_t devt; /* dev_t, creates the sysfs "dev" */
u32 id; /* device instance */
spinlock_t devres_lock;
struct list_head devres_head;
const struct class *class;
const struct attribute_group **groups; /* optional groups */
void (*release)(struct device *dev);
struct iommu_group *iommu_group;
struct dev_iommu *iommu;
struct device_physical_location *physical_location;
enum device_removable removable;
bool offline_disabled:1;
bool offline:1;
bool of_node_reused:1;
bool state_synced:1;
bool can_match:1;
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
bool dma_coherent:1;
#endif
#ifdef CONFIG_DMA_OPS_BYPASS
bool dma_ops_bypass : 1;
#endif
};
/**
* struct device_link - Device link representation.
* @supplier: The device on the supplier end of the link.
* @s_node: Hook to the supplier device's list of links to consumers.
* @consumer: The device on the consumer end of the link.
* @c_node: Hook to the consumer device's list of links to suppliers.
* @link_dev: device used to expose link details in sysfs
* @status: The state of the link (with respect to the presence of drivers).
* @flags: Link flags.
* @rpm_active: Whether or not the consumer device is runtime-PM-active.
* @kref: Count repeated addition of the same link.
* @rm_work: Work structure used for removing the link.
* @supplier_preactivated: Supplier has been made active before consumer probe.
*/
struct device_link {
struct device *supplier;
struct list_head s_node;
struct device *consumer;
struct list_head c_node;
struct device link_dev;
enum device_link_state status;
u32 flags;
refcount_t rpm_active;
struct kref kref;
struct work_struct rm_work;
bool supplier_preactivated; /* Owned by consumer probe. */
};
#define kobj_to_dev(__kobj) container_of_const(__kobj, struct device, kobj)
/**
* device_iommu_mapped - Returns true when the device DMA is translated
* by an IOMMU
* @dev: Device to perform the check on
*/
static inline bool device_iommu_mapped(struct device *dev)
{
return (dev->iommu_group != NULL);
}
/* Get the wakeup routines, which depend on struct device */
#include <linux/pm_wakeup.h>
/**
* dev_name - Return a device's name.
* @dev: Device with name to get.
* Return: The kobject name of the device, or its initial name if unavailable.
*/
static inline const char *dev_name(const struct device *dev)
{
/* Use the init name until the kobject becomes available */
if (dev->init_name)
return dev->init_name;
return kobject_name(&dev->kobj);
}
/**
* dev_bus_name - Return a device's bus/class name, if at all possible
* @dev: struct device to get the bus/class name of
*
* Will return the name of the bus/class the device is attached to. If it is
* not attached to a bus/class, an empty string will be returned.
*/
static inline const char *dev_bus_name(const struct device *dev)
{
return dev->bus ? dev->bus->name : (dev->class ? dev->class->name : "");
}
__printf(2, 3) int dev_set_name(struct device *dev, const char *name, ...);
#ifdef CONFIG_NUMA
static inline int dev_to_node(struct device *dev)
{
return dev->numa_node;
}
static inline void set_dev_node(struct device *dev, int node)
{
dev->numa_node = node;
}
#else
static inline int dev_to_node(struct device *dev)
{
return NUMA_NO_NODE;
}
static inline void set_dev_node(struct device *dev, int node)
{
}
#endif
static inline struct irq_domain *dev_get_msi_domain(const struct device *dev)
{
#ifdef CONFIG_GENERIC_MSI_IRQ
return dev->msi.domain;
#else
return NULL;
#endif
}
static inline void dev_set_msi_domain(struct device *dev, struct irq_domain *d)
{
#ifdef CONFIG_GENERIC_MSI_IRQ
dev->msi.domain = d;
#endif
}
static inline void *dev_get_drvdata(const struct device *dev)
{
return dev->driver_data;
}
static inline void dev_set_drvdata(struct device *dev, void *data)
{
dev->driver_data = data;
}
static inline struct pm_subsys_data *dev_to_psd(struct device *dev)
{
return dev ? dev->power.subsys_data : NULL;
}
static inline unsigned int dev_get_uevent_suppress(const struct device *dev)
{
return dev->kobj.uevent_suppress;
}
static inline void dev_set_uevent_suppress(struct device *dev, int val)
{
dev->kobj.uevent_suppress = val;
}
static inline int device_is_registered(struct device *dev)
{
return dev->kobj.state_in_sysfs;
}
static inline void device_enable_async_suspend(struct device *dev)
{
if (!dev->power.is_prepared) dev->power.async_suspend = true;
}
static inline void device_disable_async_suspend(struct device *dev)
{
if (!dev->power.is_prepared)
dev->power.async_suspend = false;
}
static inline bool device_async_suspend_enabled(struct device *dev)
{
return !!dev->power.async_suspend;
}
static inline bool device_pm_not_required(struct device *dev)
{
return dev->power.no_pm;
}
static inline void device_set_pm_not_required(struct device *dev)
{
dev->power.no_pm = true;
}
static inline void dev_pm_syscore_device(struct device *dev, bool val)
{
#ifdef CONFIG_PM_SLEEP
dev->power.syscore = val;
#endif
}
static inline void dev_pm_set_driver_flags(struct device *dev, u32 flags)
{
dev->power.driver_flags = flags;
}
static inline bool dev_pm_test_driver_flags(struct device *dev, u32 flags)
{
return !!(dev->power.driver_flags & flags);
}
static inline void device_lock(struct device *dev)
{
mutex_lock(&dev->mutex);
}
static inline int device_lock_interruptible(struct device *dev)
{
return mutex_lock_interruptible(&dev->mutex);
}
static inline int device_trylock(struct device *dev)
{
return mutex_trylock(&dev->mutex);
}
static inline void device_unlock(struct device *dev)
{
mutex_unlock(&dev->mutex);
}
DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T))
static inline void device_lock_assert(struct device *dev)
{
lockdep_assert_held(&dev->mutex);
}
static inline struct device_node *dev_of_node(struct device *dev)
{
if (!IS_ENABLED(CONFIG_OF) || !dev)
return NULL;
return dev->of_node;
}
static inline bool dev_has_sync_state(struct device *dev)
{
if (!dev)
return false;
if (dev->driver && dev->driver->sync_state)
return true;
if (dev->bus && dev->bus->sync_state)
return true;
return false;
}
static inline void dev_set_removable(struct device *dev,
enum device_removable removable)
{
dev->removable = removable;
}
static inline bool dev_is_removable(struct device *dev)
{
return dev->removable == DEVICE_REMOVABLE;
}
static inline bool dev_removable_is_valid(struct device *dev)
{
return dev->removable != DEVICE_REMOVABLE_NOT_SUPPORTED;
}
/*
* High level routines for use by the bus drivers
*/
int __must_check device_register(struct device *dev);
void device_unregister(struct device *dev);
void device_initialize(struct device *dev);
int __must_check device_add(struct device *dev);
void device_del(struct device *dev);
DEFINE_FREE(device_del, struct device *, if (_T) device_del(_T))
int device_for_each_child(struct device *dev, void *data,
int (*fn)(struct device *dev, void *data));
int device_for_each_child_reverse(struct device *dev, void *data,
int (*fn)(struct device *dev, void *data));
struct device *device_find_child(struct device *dev, void *data,
int (*match)(struct device *dev, void *data));
struct device *device_find_child_by_name(struct device *parent,
const char *name);
struct device *device_find_any_child(struct device *parent);
int device_rename(struct device *dev, const char *new_name);
int device_move(struct device *dev, struct device *new_parent,
enum dpm_order dpm_order);
int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid);
static inline bool device_supports_offline(struct device *dev)
{
return dev->bus && dev->bus->offline && dev->bus->online;
}
#define __device_lock_set_class(dev, name, key) \
do { \
struct device *__d2 __maybe_unused = dev; \
lock_set_class(&__d2->mutex.dep_map, name, key, 0, _THIS_IP_); \
} while (0)
/**
* device_lock_set_class - Specify a temporary lock class while a device
* is attached to a driver
* @dev: device to modify
* @key: lock class key data
*
* This must be called with the device_lock() already held, for example
* from driver ->probe(). Take care to only override the default
* lockdep_no_validate class.
*/
#ifdef CONFIG_LOCKDEP
#define device_lock_set_class(dev, key) \
do { \
struct device *__d = dev; \
dev_WARN_ONCE(__d, !lockdep_match_class(&__d->mutex, \
&__lockdep_no_validate__), \
"overriding existing custom lock class\n"); \
__device_lock_set_class(__d, #key, key); \
} while (0)
#else
#define device_lock_set_class(dev, key) __device_lock_set_class(dev, #key, key)
#endif
/**
* device_lock_reset_class - Return a device to the default lockdep novalidate state
* @dev: device to modify
*
* This must be called with the device_lock() already held, for example
* from driver ->remove().
*/
#define device_lock_reset_class(dev) \
do { \
struct device *__d __maybe_unused = dev; \
lock_set_novalidate_class(&__d->mutex.dep_map, "&dev->mutex", \
_THIS_IP_); \
} while (0)
void lock_device_hotplug(void);
void unlock_device_hotplug(void);
int lock_device_hotplug_sysfs(void);
int device_offline(struct device *dev);
int device_online(struct device *dev);
void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode);
void set_secondary_fwnode(struct device *dev, struct fwnode_handle *fwnode);
void device_set_of_node_from_dev(struct device *dev, const struct device *dev2);
void device_set_node(struct device *dev, struct fwnode_handle *fwnode);
static inline int dev_num_vf(struct device *dev)
{
if (dev->bus && dev->bus->num_vf) return dev->bus->num_vf(dev);
return 0;
}
/*
* Root device objects for grouping under /sys/devices
*/
struct device *__root_device_register(const char *name, struct module *owner);
/* This is a macro to avoid include problems with THIS_MODULE */
#define root_device_register(name) \
__root_device_register(name, THIS_MODULE)
void root_device_unregister(struct device *root);
static inline void *dev_get_platdata(const struct device *dev)
{
return dev->platform_data;
}
/*
* Manual binding of a device to driver. See drivers/base/bus.c
* for information on use.
*/
int __must_check device_driver_attach(struct device_driver *drv,
struct device *dev);
int __must_check device_bind_driver(struct device *dev);
void device_release_driver(struct device *dev);
int __must_check device_attach(struct device *dev);
int __must_check driver_attach(struct device_driver *drv);
void device_initial_probe(struct device *dev);
int __must_check device_reprobe(struct device *dev);
bool device_is_bound(struct device *dev);
/*
* Easy functions for dynamically creating devices on the fly
*/
__printf(5, 6) struct device *
device_create(const struct class *cls, struct device *parent, dev_t devt,
void *drvdata, const char *fmt, ...);
__printf(6, 7) struct device *
device_create_with_groups(const struct class *cls, struct device *parent, dev_t devt,
void *drvdata, const struct attribute_group **groups,
const char *fmt, ...);
void device_destroy(const struct class *cls, dev_t devt);
int __must_check device_add_groups(struct device *dev,
const struct attribute_group **groups);
void device_remove_groups(struct device *dev,
const struct attribute_group **groups);
static inline int __must_check device_add_group(struct device *dev,
const struct attribute_group *grp)
{
const struct attribute_group *groups[] = { grp, NULL };
return device_add_groups(dev, groups);
}
static inline void device_remove_group(struct device *dev,
const struct attribute_group *grp)
{
const struct attribute_group *groups[] = { grp, NULL };
return device_remove_groups(dev, groups);
}
int __must_check devm_device_add_groups(struct device *dev,
const struct attribute_group **groups);
int __must_check devm_device_add_group(struct device *dev,
const struct attribute_group *grp);
/*
* Platform "fixup" functions - allow the platform to have their say
* about devices and actions that the general device layer doesn't
* know about.
*/
/* Notify platform of device discovery */
extern int (*platform_notify)(struct device *dev);
extern int (*platform_notify_remove)(struct device *dev);
/*
* get_device - atomically increment the reference count for the device.
*
*/
struct device *get_device(struct device *dev);
void put_device(struct device *dev);
DEFINE_FREE(put_device, struct device *, if (_T) put_device(_T))
bool kill_device(struct device *dev);
#ifdef CONFIG_DEVTMPFS
int devtmpfs_mount(void);
#else
static inline int devtmpfs_mount(void) { return 0; }
#endif
/* drivers/base/power/shutdown.c */
void device_shutdown(void);
/* debugging and troubleshooting/diagnostic helpers. */
const char *dev_driver_string(const struct device *dev);
/* Device links interface. */
struct device_link *device_link_add(struct device *consumer,
struct device *supplier, u32 flags);
void device_link_del(struct device_link *link);
void device_link_remove(void *consumer, struct device *supplier);
void device_links_supplier_sync_state_pause(void);
void device_links_supplier_sync_state_resume(void);
void device_link_wait_removal(void);
/* Create alias, so I can be autoloaded. */
#define MODULE_ALIAS_CHARDEV(major,minor) \
MODULE_ALIAS("char-major-" __stringify(major) "-" __stringify(minor))
#define MODULE_ALIAS_CHARDEV_MAJOR(major) \
MODULE_ALIAS("char-major-" __stringify(major) "-*")
#endif /* _DEVICE_H_ */
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/hugetlb.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/fixmap.h>
#include <asm/mtrr.h>
#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1;
EXPORT_SYMBOL(physical_mask);
#endif
#ifdef CONFIG_HIGHPTE
#define PGTABLE_HIGHMEM __GFP_HIGHMEM
#else
#define PGTABLE_HIGHMEM 0
#endif
#ifndef CONFIG_PARAVIRT
static inline
void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
{
tlb_remove_page(tlb, table);
}
#endif
gfp_t __userpte_alloc_gfp = GFP_PGTABLE_USER | PGTABLE_HIGHMEM;
pgtable_t pte_alloc_one(struct mm_struct *mm)
{
return __pte_alloc_one(mm, __userpte_alloc_gfp);
}
static int __init setup_userpte(char *arg)
{
if (!arg)
return -EINVAL;
/*
* "userpte=nohigh" disables allocation of user pagetables in
* high memory.
*/
if (strcmp(arg, "nohigh") == 0)
__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
else
return -EINVAL;
return 0;
}
early_param("userpte", setup_userpte);
void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
pagetable_pte_dtor(page_ptdesc(pte));
paravirt_release_pte(page_to_pfn(pte));
paravirt_tlb_remove_table(tlb, pte);
}
#if CONFIG_PGTABLE_LEVELS > 2
void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
/*
* NOTE! For PAE, any changes to the top page-directory-pointer-table
* entries need a full cr3 reload to flush.
*/
#ifdef CONFIG_X86_PAE
tlb->need_flush_all = 1;
#endif
pagetable_pmd_dtor(ptdesc);
paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc));
}
#if CONFIG_PGTABLE_LEVELS > 3
void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
pagetable_pud_dtor(ptdesc);
paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
paravirt_tlb_remove_table(tlb, virt_to_page(pud));
}
#if CONFIG_PGTABLE_LEVELS > 4
void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d)
{
paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT);
paravirt_tlb_remove_table(tlb, virt_to_page(p4d));
}
#endif /* CONFIG_PGTABLE_LEVELS > 4 */
#endif /* CONFIG_PGTABLE_LEVELS > 3 */
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
static inline void pgd_list_add(pgd_t *pgd)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
list_add(&ptdesc->pt_list, &pgd_list);
}
static inline void pgd_list_del(pgd_t *pgd)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
list_del(&ptdesc->pt_list);
}
#define UNSHARED_PTRS_PER_PGD \
(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
#define MAX_UNSHARED_PTRS_PER_PGD \
max_t(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD)
static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
{
virt_to_ptdesc(pgd)->pt_mm = mm;
}
struct mm_struct *pgd_page_get_mm(struct page *page)
{
return page_ptdesc(page)->pt_mm;
}
static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
{
/* If the pgd points to a shared pagetable level (either the
ptes in non-PAE, or shared PMD in PAE), then just copy the
references from swapper_pg_dir. */
if (CONFIG_PGTABLE_LEVELS == 2 ||
(CONFIG_PGTABLE_LEVELS == 3 && SHARED_KERNEL_PMD) ||
CONFIG_PGTABLE_LEVELS >= 4) {
clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
swapper_pg_dir + KERNEL_PGD_BOUNDARY,
KERNEL_PGD_PTRS);
}
/* list required to sync kernel mapping updates */
if (!SHARED_KERNEL_PMD) {
pgd_set_mm(pgd, mm);
pgd_list_add(pgd);
}
}
static void pgd_dtor(pgd_t *pgd)
{
if (SHARED_KERNEL_PMD)
return;
spin_lock(&pgd_lock);
pgd_list_del(pgd);
spin_unlock(&pgd_lock);
}
/*
* List of all pgd's needed for non-PAE so it can invalidate entries
* in both cached and uncached pgd's; not needed for PAE since the
* kernel pmd is shared. If PAE were not to share the pmd a similar
* tactic would be needed. This is essentially codepath-based locking
* against pageattr.c; it is the unique case in which a valid change
* of kernel pagetables can't be lazily synchronized by vmalloc faults.
* vmalloc faults work because attached pagetables are never freed.
* -- nyc
*/
#ifdef CONFIG_X86_PAE
/*
* In PAE mode, we need to do a cr3 reload (=tlb flush) when
* updating the top-level pagetable entries to guarantee the
* processor notices the update. Since this is expensive, and
* all 4 top-level entries are used almost immediately in a
* new process's life, we just pre-populate them here.
*
* Also, if we're in a paravirt environment where the kernel pmd is
* not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
* and initialize the kernel pmds here.
*/
#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
#define MAX_PREALLOCATED_PMDS MAX_UNSHARED_PTRS_PER_PGD
/*
* We allocate separate PMDs for the kernel part of the user page-table
* when PTI is enabled. We need them to map the per-process LDT into the
* user-space page-table.
*/
#define PREALLOCATED_USER_PMDS (boot_cpu_has(X86_FEATURE_PTI) ? \
KERNEL_PGD_PTRS : 0)
#define MAX_PREALLOCATED_USER_PMDS KERNEL_PGD_PTRS
void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
{
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
/* Note: almost everything apart from _PAGE_PRESENT is
reserved at the pmd (PDPT) level. */
set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
/*
* According to Intel App note "TLBs, Paging-Structure Caches,
* and Their Invalidation", April 2007, document 317080-001,
* section 8.1: in PAE mode we explicitly have to flush the
* TLB via cr3 if the top-level pgd is changed...
*/
flush_tlb_mm(mm);
}
#else /* !CONFIG_X86_PAE */
/* No need to prepopulate any pagetable entries in non-PAE modes. */
#define PREALLOCATED_PMDS 0
#define MAX_PREALLOCATED_PMDS 0
#define PREALLOCATED_USER_PMDS 0
#define MAX_PREALLOCATED_USER_PMDS 0
#endif /* CONFIG_X86_PAE */
static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
{
int i;
struct ptdesc *ptdesc;
for (i = 0; i < count; i++)
if (pmds[i]) {
ptdesc = virt_to_ptdesc(pmds[i]);
pagetable_pmd_dtor(ptdesc);
pagetable_free(ptdesc);
mm_dec_nr_pmds(mm);
}
}
static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
{
int i;
bool failed = false;
gfp_t gfp = GFP_PGTABLE_USER;
if (mm == &init_mm)
gfp &= ~__GFP_ACCOUNT;
gfp &= ~__GFP_HIGHMEM;
for (i = 0; i < count; i++) {
pmd_t *pmd = NULL;
struct ptdesc *ptdesc = pagetable_alloc(gfp, 0);
if (!ptdesc)
failed = true;
if (ptdesc && !pagetable_pmd_ctor(ptdesc)) {
pagetable_free(ptdesc);
ptdesc = NULL;
failed = true;
}
if (ptdesc) {
mm_inc_nr_pmds(mm);
pmd = ptdesc_address(ptdesc);
}
pmds[i] = pmd;
}
if (failed) {
free_pmds(mm, pmds, count);
return -ENOMEM;
}
return 0;
}
/*
* Mop up any pmd pages which may still be attached to the pgd.
* Normally they will be freed by munmap/exit_mmap, but any pmd we
* preallocate which never got a corresponding vma will need to be
* freed manually.
*/
static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp)
{
pgd_t pgd = *pgdp;
if (pgd_val(pgd) != 0) {
pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);
pgd_clear(pgdp);
paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
pmd_free(mm, pmd);
mm_dec_nr_pmds(mm);
}
}
static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
{
int i;
for (i = 0; i < PREALLOCATED_PMDS; i++)
mop_up_one_pmd(mm, &pgdp[i]);
#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
if (!boot_cpu_has(X86_FEATURE_PTI))
return;
pgdp = kernel_to_user_pgdp(pgdp);
for (i = 0; i < PREALLOCATED_USER_PMDS; i++)
mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]);
#endif
}
static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
{
p4d_t *p4d;
pud_t *pud;
int i;
p4d = p4d_offset(pgd, 0);
pud = pud_offset(p4d, 0);
for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
pmd_t *pmd = pmds[i];
if (i >= KERNEL_PGD_BOUNDARY)
memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
sizeof(pmd_t) * PTRS_PER_PMD);
pud_populate(mm, pud, pmd);
}
}
#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
static void pgd_prepopulate_user_pmd(struct mm_struct *mm,
pgd_t *k_pgd, pmd_t *pmds[])
{
pgd_t *s_pgd = kernel_to_user_pgdp(swapper_pg_dir);
pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd);
p4d_t *u_p4d;
pud_t *u_pud;
int i;
u_p4d = p4d_offset(u_pgd, 0);
u_pud = pud_offset(u_p4d, 0);
s_pgd += KERNEL_PGD_BOUNDARY;
u_pud += KERNEL_PGD_BOUNDARY;
for (i = 0; i < PREALLOCATED_USER_PMDS; i++, u_pud++, s_pgd++) {
pmd_t *pmd = pmds[i];
memcpy(pmd, (pmd_t *)pgd_page_vaddr(*s_pgd),
sizeof(pmd_t) * PTRS_PER_PMD);
pud_populate(mm, u_pud, pmd);
}
}
#else
static void pgd_prepopulate_user_pmd(struct mm_struct *mm,
pgd_t *k_pgd, pmd_t *pmds[])
{
}
#endif
/*
* Xen paravirt assumes pgd table should be in one page. 64 bit kernel also
* assumes that pgd should be in one page.
*
* But kernel with PAE paging that is not running as a Xen domain
* only needs to allocate 32 bytes for pgd instead of one page.
*/
#ifdef CONFIG_X86_PAE
#include <linux/slab.h>
#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
#define PGD_ALIGN 32
static struct kmem_cache *pgd_cache;
void __init pgtable_cache_init(void)
{
/*
* When PAE kernel is running as a Xen domain, it does not use
* shared kernel pmd. And this requires a whole page for pgd.
*/
if (!SHARED_KERNEL_PMD)
return;
/*
* when PAE kernel is not running as a Xen domain, it uses
* shared kernel pmd. Shared kernel pmd does not require a whole
* page for pgd. We are able to just allocate a 32-byte for pgd.
* During boot time, we create a 32-byte slab for pgd table allocation.
*/
pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN,
SLAB_PANIC, NULL);
}
static inline pgd_t *_pgd_alloc(void)
{
/*
* If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain.
* We allocate one page for pgd.
*/
if (!SHARED_KERNEL_PMD)
return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER,
PGD_ALLOCATION_ORDER);
/*
* Now PAE kernel is not running as a Xen domain. We can allocate
* a 32-byte slab for pgd to save memory space.
*/
return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER);
}
static inline void _pgd_free(pgd_t *pgd)
{
if (!SHARED_KERNEL_PMD)
free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER);
else
kmem_cache_free(pgd_cache, pgd);
}
#else
static inline pgd_t *_pgd_alloc(void)
{
return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER,
PGD_ALLOCATION_ORDER);
}
static inline void _pgd_free(pgd_t *pgd)
{
free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER);
}
#endif /* CONFIG_X86_PAE */
pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *pgd;
pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS];
pmd_t *pmds[MAX_PREALLOCATED_PMDS];
pgd = _pgd_alloc();
if (pgd == NULL)
goto out;
mm->pgd = pgd;
if (sizeof(pmds) != 0 &&
preallocate_pmds(mm, pmds, PREALLOCATED_PMDS) != 0)
goto out_free_pgd;
if (sizeof(u_pmds) != 0 &&
preallocate_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS) != 0)
goto out_free_pmds;
if (paravirt_pgd_alloc(mm) != 0)
goto out_free_user_pmds;
/*
* Make sure that pre-populating the pmds is atomic with
* respect to anything walking the pgd_list, so that they
* never see a partially populated pgd.
*/
spin_lock(&pgd_lock);
pgd_ctor(mm, pgd);
if (sizeof(pmds) != 0)
pgd_prepopulate_pmd(mm, pgd, pmds);
if (sizeof(u_pmds) != 0)
pgd_prepopulate_user_pmd(mm, pgd, u_pmds);
spin_unlock(&pgd_lock);
return pgd;
out_free_user_pmds:
if (sizeof(u_pmds) != 0)
free_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS);
out_free_pmds:
if (sizeof(pmds) != 0)
free_pmds(mm, pmds, PREALLOCATED_PMDS);
out_free_pgd:
_pgd_free(pgd);
out:
return NULL;
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
pgd_mop_up_pmds(mm, pgd); pgd_dtor(pgd);
paravirt_pgd_free(mm, pgd);
_pgd_free(pgd);
}
/*
* Used to set accessed or dirty bits in the page table entries
* on other architectures. On x86, the accessed and dirty bits
* are tracked by hardware. However, do_wp_page calls this function
* to also make the pte writeable at the same time the dirty bit is
* set. In that case we do actually need to write the PTE.
*/
int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep,
pte_t entry, int dirty)
{
int changed = !pte_same(*ptep, entry); if (changed && dirty) set_pte(ptep, entry); return changed;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp,
pmd_t entry, int dirty)
{
int changed = !pmd_same(*pmdp, entry);
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
if (changed && dirty) {
set_pmd(pmdp, entry);
/*
* We had a write-protection fault here and changed the pmd
* to to more permissive. No need to flush the TLB for that,
* #PF is architecturally guaranteed to do that and in the
* worst-case we'll generate a spurious fault.
*/
}
return changed;
}
int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
pud_t *pudp, pud_t entry, int dirty)
{
int changed = !pud_same(*pudp, entry);
VM_BUG_ON(address & ~HPAGE_PUD_MASK);
if (changed && dirty) {
set_pud(pudp, entry);
/*
* We had a write-protection fault here and changed the pud
* to to more permissive. No need to flush the TLB for that,
* #PF is architecturally guaranteed to do that and in the
* worst-case we'll generate a spurious fault.
*/
}
return changed;
}
#endif
int ptep_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
int ret = 0;
if (pte_young(*ptep))
ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
(unsigned long *) &ptep->pte);
return ret;
}
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pmd_t *pmdp)
{
int ret = 0;
if (pmd_young(*pmdp))
ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
(unsigned long *)pmdp);
return ret;
}
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pudp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pud_t *pudp)
{
int ret = 0;
if (pud_young(*pudp))
ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
(unsigned long *)pudp);
return ret;
}
#endif
int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
/*
* On x86 CPUs, clearing the accessed bit without a TLB flush
* doesn't cause data corruption. [ It could cause incorrect
* page aging and the (mistaken) reclaim of hot pages, but the
* chance of that should be relatively low. ]
*
* So as a performance optimization don't flush the TLB when
* clearing the accessed bit, it will eventually be flushed by
* a context switch or a VM operation anyway. [ In the rare
* event of it not getting flushed for a long time the delay
* shouldn't really matter because there's no real memory
* pressure for swapout to react to. ]
*/
return ptep_test_and_clear_young(vma, address, ptep);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
int young;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
young = pmdp_test_and_clear_young(vma, address, pmdp);
if (young)
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return young;
}
pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
/*
* No flush is necessary. Once an invalid PTE is established, the PTE's
* access and dirty bits cannot be updated.
*/
return pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
}
#endif
/**
* reserve_top_address - reserves a hole in the top of kernel address space
* @reserve - size of hole to reserve
*
* Can be used to relocate the fixmap area and poke a hole in the top
* of kernel address space to make room for a hypervisor.
*/
void __init reserve_top_address(unsigned long reserve)
{
#ifdef CONFIG_X86_32
BUG_ON(fixmaps_set > 0);
__FIXADDR_TOP = round_down(-reserve, 1 << PMD_SHIFT) - PAGE_SIZE;
printk(KERN_INFO "Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n",
-reserve, __FIXADDR_TOP + PAGE_SIZE);
#endif
}
int fixmaps_set;
void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
{
unsigned long address = __fix_to_virt(idx);
#ifdef CONFIG_X86_64
/*
* Ensure that the static initial page tables are covering the
* fixmap completely.
*/
BUILD_BUG_ON(__end_of_permanent_fixed_addresses >
(FIXMAP_PMD_NUM * PTRS_PER_PTE));
#endif
if (idx >= __end_of_fixed_addresses) {
BUG();
return;
}
set_pte_vaddr(address, pte);
fixmaps_set++;
}
void native_set_fixmap(unsigned /* enum fixed_addresses */ idx,
phys_addr_t phys, pgprot_t flags)
{
/* Sanitize 'prot' against any unsupported bits: */
pgprot_val(flags) &= __default_kernel_pte_mask;
__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
}
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
#ifdef CONFIG_X86_5LEVEL
/**
* p4d_set_huge - setup kernel P4D mapping
*
* No 512GB pages yet -- always return 0
*/
int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
{
return 0;
}
/**
* p4d_clear_huge - clear kernel P4D mapping when it is set
*
* No 512GB pages yet -- always return 0
*/
void p4d_clear_huge(p4d_t *p4d)
{
}
#endif
/**
* pud_set_huge - setup kernel PUD mapping
*
* MTRRs can override PAT memory types with 4KiB granularity. Therefore, this
* function sets up a huge page only if the complete range has the same MTRR
* caching mode.
*
* Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger
* page mapping attempt fails.
*
* Returns 1 on success and 0 on failure.
*/
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
{
u8 uniform;
mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform);
if (!uniform)
return 0;
/* Bail out if we are we on a populated non-leaf entry: */
if (pud_present(*pud) && !pud_huge(*pud))
return 0;
set_pte((pte_t *)pud, pfn_pte(
(u64)addr >> PAGE_SHIFT,
__pgprot(protval_4k_2_large(pgprot_val(prot)) | _PAGE_PSE)));
return 1;
}
/**
* pmd_set_huge - setup kernel PMD mapping
*
* See text over pud_set_huge() above.
*
* Returns 1 on success and 0 on failure.
*/
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
{
u8 uniform;
mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform);
if (!uniform) {
pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n",
__func__, addr, addr + PMD_SIZE);
return 0;
}
/* Bail out if we are we on a populated non-leaf entry: */
if (pmd_present(*pmd) && !pmd_huge(*pmd))
return 0;
set_pte((pte_t *)pmd, pfn_pte(
(u64)addr >> PAGE_SHIFT,
__pgprot(protval_4k_2_large(pgprot_val(prot)) | _PAGE_PSE)));
return 1;
}
/**
* pud_clear_huge - clear kernel PUD mapping when it is set
*
* Returns 1 on success and 0 on failure (no PUD map is found).
*/
int pud_clear_huge(pud_t *pud)
{ if (pud_leaf(*pud)) { pud_clear(pud); return 1;
}
return 0;
}
/**
* pmd_clear_huge - clear kernel PMD mapping when it is set
*
* Returns 1 on success and 0 on failure (no PMD map is found).
*/
int pmd_clear_huge(pmd_t *pmd)
{ if (pmd_leaf(*pmd)) { pmd_clear(pmd); return 1;
}
return 0;
}
#ifdef CONFIG_X86_64
/**
* pud_free_pmd_page - Clear pud entry and free pmd page.
* @pud: Pointer to a PUD.
* @addr: Virtual address associated with pud.
*
* Context: The pud range has been unmapped and TLB purged.
* Return: 1 if clearing the entry succeeded. 0 otherwise.
*
* NOTE: Callers must allow a single page allocation.
*/
int pud_free_pmd_page(pud_t *pud, unsigned long addr)
{
pmd_t *pmd, *pmd_sv;
pte_t *pte;
int i;
pmd = pud_pgtable(*pud);
pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL);
if (!pmd_sv)
return 0;
for (i = 0; i < PTRS_PER_PMD; i++) {
pmd_sv[i] = pmd[i];
if (!pmd_none(pmd[i]))
pmd_clear(&pmd[i]);
}
pud_clear(pud);
/* INVLPG to clear all paging-structure caches */
flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1);
for (i = 0; i < PTRS_PER_PMD; i++) {
if (!pmd_none(pmd_sv[i])) {
pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]);
free_page((unsigned long)pte);
}
}
free_page((unsigned long)pmd_sv);
pagetable_pmd_dtor(virt_to_ptdesc(pmd));
free_page((unsigned long)pmd);
return 1;
}
/**
* pmd_free_pte_page - Clear pmd entry and free pte page.
* @pmd: Pointer to a PMD.
* @addr: Virtual address associated with pmd.
*
* Context: The pmd range has been unmapped and TLB purged.
* Return: 1 if clearing the entry succeeded. 0 otherwise.
*/
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
{
pte_t *pte;
pte = (pte_t *)pmd_page_vaddr(*pmd);
pmd_clear(pmd);
/* INVLPG to clear all paging-structure caches */
flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1);
free_page((unsigned long)pte);
return 1;
}
#else /* !CONFIG_X86_64 */
/*
* Disable free page handling on x86-PAE. This assures that ioremap()
* does not update sync'd pmd entries. See vmalloc_sync_one().
*/
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
{
return pmd_none(*pmd);
}
#endif /* CONFIG_X86_64 */
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
{
if (vma->vm_flags & VM_SHADOW_STACK)
return pte_mkwrite_shstk(pte);
pte = pte_mkwrite_novma(pte);
return pte_clear_saveddirty(pte);
}
pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
{
if (vma->vm_flags & VM_SHADOW_STACK)
return pmd_mkwrite_shstk(pmd);
pmd = pmd_mkwrite_novma(pmd);
return pmd_clear_saveddirty(pmd);
}
void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte)
{
/*
* Hardware before shadow stack can (rarely) set Dirty=1
* on a Write=0 PTE. So the below condition
* only indicates a software bug when shadow stack is
* supported by the HW. This checking is covered in
* pte_shstk().
*/
VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
pte_shstk(pte));
}
void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd)
{
/* See note in arch_check_zapped_pte() */
VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
pmd_shstk(pmd));
}
/*
* Copyright (c) 2016 Intel Corporation
*
* Permission to use, copy, modify, distribute, and sell this software and its
* documentation for any purpose is hereby granted without fee, provided that
* the above copyright notice appear in all copies and that both that copyright
* notice and this permission notice appear in supporting documentation, and
* that the name of the copyright holders not be used in advertising or
* publicity pertaining to distribution of the software without specific,
* written prior permission. The copyright holders make no representations
* about the suitability of this software for any purpose. It is provided "as
* is" without express or implied warranty.
*
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
* EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
* DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
* TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
* OF THIS SOFTWARE.
*/
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <drm/drm_plane.h>
#include <drm/drm_drv.h>
#include <drm/drm_print.h>
#include <drm/drm_framebuffer.h>
#include <drm/drm_file.h>
#include <drm/drm_crtc.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_managed.h>
#include <drm/drm_vblank.h>
#include "drm_crtc_internal.h"
/**
* DOC: overview
*
* A plane represents an image source that can be blended with or overlaid on
* top of a CRTC during the scanout process. Planes take their input data from a
* &drm_framebuffer object. The plane itself specifies the cropping and scaling
* of that image, and where it is placed on the visible area of a display
* pipeline, represented by &drm_crtc. A plane can also have additional
* properties that specify how the pixels are positioned and blended, like
* rotation or Z-position. All these properties are stored in &drm_plane_state.
*
* Unless explicitly specified (via CRTC property or otherwise), the active area
* of a CRTC will be black by default. This means portions of the active area
* which are not covered by a plane will be black, and alpha blending of any
* planes with the CRTC background will blend with black at the lowest zpos.
*
* To create a plane, a KMS drivers allocates and zeroes an instances of
* &struct drm_plane (possibly as part of a larger structure) and registers it
* with a call to drm_universal_plane_init().
*
* Each plane has a type, see enum drm_plane_type. A plane can be compatible
* with multiple CRTCs, see &drm_plane.possible_crtcs.
*
* Each CRTC must have a unique primary plane userspace can attach to enable
* the CRTC. In other words, userspace must be able to attach a different
* primary plane to each CRTC at the same time. Primary planes can still be
* compatible with multiple CRTCs. There must be exactly as many primary planes
* as there are CRTCs.
*
* Legacy uAPI doesn't expose the primary and cursor planes directly. DRM core
* relies on the driver to set the primary and optionally the cursor plane used
* for legacy IOCTLs. This is done by calling drm_crtc_init_with_planes(). All
* drivers must provide one primary plane per CRTC to avoid surprising legacy
* userspace too much.
*/
/**
* DOC: standard plane properties
*
* DRM planes have a few standardized properties:
*
* type:
* Immutable property describing the type of the plane.
*
* For user-space which has enabled the &DRM_CLIENT_CAP_ATOMIC capability,
* the plane type is just a hint and is mostly superseded by atomic
* test-only commits. The type hint can still be used to come up more
* easily with a plane configuration accepted by the driver.
*
* The value of this property can be one of the following:
*
* "Primary":
* To light up a CRTC, attaching a primary plane is the most likely to
* work if it covers the whole CRTC and doesn't have scaling or
* cropping set up.
*
* Drivers may support more features for the primary plane, user-space
* can find out with test-only atomic commits.
*
* Some primary planes are implicitly used by the kernel in the legacy
* IOCTLs &DRM_IOCTL_MODE_SETCRTC and &DRM_IOCTL_MODE_PAGE_FLIP.
* Therefore user-space must not mix explicit usage of any primary
* plane (e.g. through an atomic commit) with these legacy IOCTLs.
*
* "Cursor":
* To enable this plane, using a framebuffer configured without scaling
* or cropping and with the following properties is the most likely to
* work:
*
* - If the driver provides the capabilities &DRM_CAP_CURSOR_WIDTH and
* &DRM_CAP_CURSOR_HEIGHT, create the framebuffer with this size.
* Otherwise, create a framebuffer with the size 64x64.
* - If the driver doesn't support modifiers, create a framebuffer with
* a linear layout. Otherwise, use the IN_FORMATS plane property.
*
* Drivers may support more features for the cursor plane, user-space
* can find out with test-only atomic commits.
*
* Some cursor planes are implicitly used by the kernel in the legacy
* IOCTLs &DRM_IOCTL_MODE_CURSOR and &DRM_IOCTL_MODE_CURSOR2.
* Therefore user-space must not mix explicit usage of any cursor
* plane (e.g. through an atomic commit) with these legacy IOCTLs.
*
* Some drivers may support cursors even if no cursor plane is exposed.
* In this case, the legacy cursor IOCTLs can be used to configure the
* cursor.
*
* "Overlay":
* Neither primary nor cursor.
*
* Overlay planes are the only planes exposed when the
* &DRM_CLIENT_CAP_UNIVERSAL_PLANES capability is disabled.
*
* IN_FORMATS:
* Blob property which contains the set of buffer format and modifier
* pairs supported by this plane. The blob is a struct
* drm_format_modifier_blob. Without this property the plane doesn't
* support buffers with modifiers. Userspace cannot change this property.
*
* Note that userspace can check the &DRM_CAP_ADDFB2_MODIFIERS driver
* capability for general modifier support. If this flag is set then every
* plane will have the IN_FORMATS property, even when it only supports
* DRM_FORMAT_MOD_LINEAR. Before linux kernel release v5.1 there have been
* various bugs in this area with inconsistencies between the capability
* flag and per-plane properties.
*/
static unsigned int drm_num_planes(struct drm_device *dev)
{
unsigned int num = 0;
struct drm_plane *tmp;
drm_for_each_plane(tmp, dev) {
num++;
}
return num;
}
static inline u32 *
formats_ptr(struct drm_format_modifier_blob *blob)
{
return (u32 *)(((char *)blob) + blob->formats_offset);
}
static inline struct drm_format_modifier *
modifiers_ptr(struct drm_format_modifier_blob *blob)
{
return (struct drm_format_modifier *)(((char *)blob) + blob->modifiers_offset);
}
static int create_in_format_blob(struct drm_device *dev, struct drm_plane *plane)
{
const struct drm_mode_config *config = &dev->mode_config;
struct drm_property_blob *blob;
struct drm_format_modifier *mod;
size_t blob_size, formats_size, modifiers_size;
struct drm_format_modifier_blob *blob_data;
unsigned int i, j;
formats_size = sizeof(__u32) * plane->format_count;
if (WARN_ON(!formats_size)) {
/* 0 formats are never expected */
return 0;
}
modifiers_size =
sizeof(struct drm_format_modifier) * plane->modifier_count;
blob_size = sizeof(struct drm_format_modifier_blob);
/* Modifiers offset is a pointer to a struct with a 64 bit field so it
* should be naturally aligned to 8B.
*/
BUILD_BUG_ON(sizeof(struct drm_format_modifier_blob) % 8);
blob_size += ALIGN(formats_size, 8);
blob_size += modifiers_size;
blob = drm_property_create_blob(dev, blob_size, NULL);
if (IS_ERR(blob))
return -1;
blob_data = blob->data;
blob_data->version = FORMAT_BLOB_CURRENT;
blob_data->count_formats = plane->format_count;
blob_data->formats_offset = sizeof(struct drm_format_modifier_blob);
blob_data->count_modifiers = plane->modifier_count;
blob_data->modifiers_offset =
ALIGN(blob_data->formats_offset + formats_size, 8);
memcpy(formats_ptr(blob_data), plane->format_types, formats_size);
mod = modifiers_ptr(blob_data);
for (i = 0; i < plane->modifier_count; i++) {
for (j = 0; j < plane->format_count; j++) {
if (!plane->funcs->format_mod_supported ||
plane->funcs->format_mod_supported(plane,
plane->format_types[j],
plane->modifiers[i])) {
mod->formats |= 1ULL << j;
}
}
mod->modifier = plane->modifiers[i];
mod->offset = 0;
mod->pad = 0;
mod++;
}
drm_object_attach_property(&plane->base, config->modifiers_property,
blob->base.id);
return 0;
}
/**
* DOC: hotspot properties
*
* HOTSPOT_X: property to set mouse hotspot x offset.
* HOTSPOT_Y: property to set mouse hotspot y offset.
*
* When the plane is being used as a cursor image to display a mouse pointer,
* the "hotspot" is the offset within the cursor image where mouse events
* are expected to go.
*
* Positive values move the hotspot from the top-left corner of the cursor
* plane towards the right and bottom.
*
* Most display drivers do not need this information because the
* hotspot is not actually connected to anything visible on screen.
* However, this is necessary for display drivers like the para-virtualized
* drivers (eg qxl, vbox, virtio, vmwgfx), that are attached to a user console
* with a mouse pointer. Since these consoles are often being remoted over a
* network, they would otherwise have to wait to display the pointer movement to
* the user until a full network round-trip has occurred. New mouse events have
* to be sent from the user's console, over the network to the virtual input
* devices, forwarded to the desktop for processing, and then the cursor plane's
* position can be updated and sent back to the user's console over the network.
* Instead, with the hotspot information, the console can anticipate the new
* location, and draw the mouse cursor there before the confirmation comes in.
* To do that correctly, the user's console must be able predict how the
* desktop will process mouse events, which normally requires the desktop's
* mouse topology information, ie where each CRTC sits in the mouse coordinate
* space. This is typically sent to the para-virtualized drivers using some
* driver-specific method, and the driver then forwards it to the console by
* way of the virtual display device or hypervisor.
*
* The assumption is generally made that there is only one cursor plane being
* used this way at a time, and that the desktop is feeding all mouse devices
* into the same global pointer. Para-virtualized drivers that require this
* should only be exposing a single cursor plane, or find some other way
* to coordinate with a userspace desktop that supports multiple pointers.
* If the hotspot properties are set, the cursor plane is therefore assumed to be
* used only for displaying a mouse cursor image, and the position of the combined
* cursor plane + offset can therefore be used for coordinating with input from a
* mouse device.
*
* The cursor will then be drawn either at the location of the plane in the CRTC
* console, or as a free-floating cursor plane on the user's console
* corresponding to their desktop mouse position.
*
* DRM clients which would like to work correctly on drivers which expose
* hotspot properties should advertise DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT.
* Setting this property on drivers which do not special case
* cursor planes will return EOPNOTSUPP, which can be used by userspace to
* gauge requirements of the hardware/drivers they're running on. Advertising
* DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT implies that the userspace client will be
* correctly setting the hotspot properties.
*/
/**
* drm_plane_create_hotspot_properties - creates the mouse hotspot
* properties and attaches them to the given cursor plane
*
* @plane: drm cursor plane
*
* This function enables the mouse hotspot property on a given
* cursor plane. Look at the documentation for hotspot properties
* to get a better understanding for what they're used for.
*
* RETURNS:
* Zero for success or -errno
*/
static int drm_plane_create_hotspot_properties(struct drm_plane *plane)
{
struct drm_property *prop_x;
struct drm_property *prop_y;
drm_WARN_ON(plane->dev,
!drm_core_check_feature(plane->dev,
DRIVER_CURSOR_HOTSPOT));
prop_x = drm_property_create_signed_range(plane->dev, 0, "HOTSPOT_X",
INT_MIN, INT_MAX);
if (IS_ERR(prop_x))
return PTR_ERR(prop_x);
prop_y = drm_property_create_signed_range(plane->dev, 0, "HOTSPOT_Y",
INT_MIN, INT_MAX);
if (IS_ERR(prop_y)) {
drm_property_destroy(plane->dev, prop_x);
return PTR_ERR(prop_y);
}
drm_object_attach_property(&plane->base, prop_x, 0);
drm_object_attach_property(&plane->base, prop_y, 0);
plane->hotspot_x_property = prop_x;
plane->hotspot_y_property = prop_y;
return 0;
}
__printf(9, 0)
static int __drm_universal_plane_init(struct drm_device *dev,
struct drm_plane *plane,
uint32_t possible_crtcs,
const struct drm_plane_funcs *funcs,
const uint32_t *formats,
unsigned int format_count,
const uint64_t *format_modifiers,
enum drm_plane_type type,
const char *name, va_list ap)
{
struct drm_mode_config *config = &dev->mode_config;
static const uint64_t default_modifiers[] = {
DRM_FORMAT_MOD_LINEAR,
};
unsigned int format_modifier_count = 0;
int ret;
/* plane index is used with 32bit bitmasks */
if (WARN_ON(config->num_total_plane >= 32))
return -EINVAL;
/*
* First driver to need more than 64 formats needs to fix this. Each
* format is encoded as a bit and the current code only supports a u64.
*/
if (WARN_ON(format_count > 64))
return -EINVAL;
WARN_ON(drm_drv_uses_atomic_modeset(dev) &&
(!funcs->atomic_destroy_state ||
!funcs->atomic_duplicate_state));
ret = drm_mode_object_add(dev, &plane->base, DRM_MODE_OBJECT_PLANE);
if (ret)
return ret;
drm_modeset_lock_init(&plane->mutex);
plane->base.properties = &plane->properties;
plane->dev = dev;
plane->funcs = funcs;
plane->format_types = kmalloc_array(format_count, sizeof(uint32_t),
GFP_KERNEL);
if (!plane->format_types) {
DRM_DEBUG_KMS("out of memory when allocating plane\n");
drm_mode_object_unregister(dev, &plane->base);
return -ENOMEM;
}
if (format_modifiers) {
const uint64_t *temp_modifiers = format_modifiers;
while (*temp_modifiers++ != DRM_FORMAT_MOD_INVALID)
format_modifier_count++;
} else {
if (!dev->mode_config.fb_modifiers_not_supported) {
format_modifiers = default_modifiers;
format_modifier_count = ARRAY_SIZE(default_modifiers);
}
}
/* autoset the cap and check for consistency across all planes */
drm_WARN_ON(dev, config->fb_modifiers_not_supported &&
format_modifier_count);
plane->modifier_count = format_modifier_count;
plane->modifiers = kmalloc_array(format_modifier_count,
sizeof(format_modifiers[0]),
GFP_KERNEL);
if (format_modifier_count && !plane->modifiers) {
DRM_DEBUG_KMS("out of memory when allocating plane\n");
kfree(plane->format_types);
drm_mode_object_unregister(dev, &plane->base);
return -ENOMEM;
}
if (name) {
plane->name = kvasprintf(GFP_KERNEL, name, ap);
} else {
plane->name = kasprintf(GFP_KERNEL, "plane-%d",
drm_num_planes(dev));
}
if (!plane->name) {
kfree(plane->format_types);
kfree(plane->modifiers);
drm_mode_object_unregister(dev, &plane->base);
return -ENOMEM;
}
memcpy(plane->format_types, formats, format_count * sizeof(uint32_t));
plane->format_count = format_count;
memcpy(plane->modifiers, format_modifiers,
format_modifier_count * sizeof(format_modifiers[0]));
plane->possible_crtcs = possible_crtcs;
plane->type = type;
list_add_tail(&plane->head, &config->plane_list);
plane->index = config->num_total_plane++;
drm_object_attach_property(&plane->base,
config->plane_type_property,
plane->type);
if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
drm_object_attach_property(&plane->base, config->prop_fb_id, 0);
drm_object_attach_property(&plane->base, config->prop_in_fence_fd, -1);
drm_object_attach_property(&plane->base, config->prop_crtc_id, 0);
drm_object_attach_property(&plane->base, config->prop_crtc_x, 0);
drm_object_attach_property(&plane->base, config->prop_crtc_y, 0);
drm_object_attach_property(&plane->base, config->prop_crtc_w, 0);
drm_object_attach_property(&plane->base, config->prop_crtc_h, 0);
drm_object_attach_property(&plane->base, config->prop_src_x, 0);
drm_object_attach_property(&plane->base, config->prop_src_y, 0);
drm_object_attach_property(&plane->base, config->prop_src_w, 0);
drm_object_attach_property(&plane->base, config->prop_src_h, 0);
}
if (drm_core_check_feature(dev, DRIVER_CURSOR_HOTSPOT) &&
type == DRM_PLANE_TYPE_CURSOR) {
drm_plane_create_hotspot_properties(plane);
}
if (format_modifier_count)
create_in_format_blob(dev, plane);
return 0;
}
/**
* drm_universal_plane_init - Initialize a new universal plane object
* @dev: DRM device
* @plane: plane object to init
* @possible_crtcs: bitmask of possible CRTCs
* @funcs: callbacks for the new plane
* @formats: array of supported formats (DRM_FORMAT\_\*)
* @format_count: number of elements in @formats
* @format_modifiers: array of struct drm_format modifiers terminated by
* DRM_FORMAT_MOD_INVALID
* @type: type of plane (overlay, primary, cursor)
* @name: printf style format string for the plane name, or NULL for default name
*
* Initializes a plane object of type @type. The &drm_plane_funcs.destroy hook
* should call drm_plane_cleanup() and kfree() the plane structure. The plane
* structure should not be allocated with devm_kzalloc().
*
* Note: consider using drmm_universal_plane_alloc() instead of
* drm_universal_plane_init() to let the DRM managed resource infrastructure
* take care of cleanup and deallocation.
*
* Drivers that only support the DRM_FORMAT_MOD_LINEAR modifier support may set
* @format_modifiers to NULL. The plane will advertise the linear modifier.
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
uint32_t possible_crtcs,
const struct drm_plane_funcs *funcs,
const uint32_t *formats, unsigned int format_count,
const uint64_t *format_modifiers,
enum drm_plane_type type,
const char *name, ...)
{
va_list ap;
int ret;
WARN_ON(!funcs->destroy);
va_start(ap, name);
ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
formats, format_count, format_modifiers,
type, name, ap);
va_end(ap);
return ret;
}
EXPORT_SYMBOL(drm_universal_plane_init);
static void drmm_universal_plane_alloc_release(struct drm_device *dev, void *ptr)
{
struct drm_plane *plane = ptr;
if (WARN_ON(!plane->dev))
return;
drm_plane_cleanup(plane);
}
void *__drmm_universal_plane_alloc(struct drm_device *dev, size_t size,
size_t offset, uint32_t possible_crtcs,
const struct drm_plane_funcs *funcs,
const uint32_t *formats, unsigned int format_count,
const uint64_t *format_modifiers,
enum drm_plane_type type,
const char *name, ...)
{
void *container;
struct drm_plane *plane;
va_list ap;
int ret;
if (WARN_ON(!funcs || funcs->destroy))
return ERR_PTR(-EINVAL);
container = drmm_kzalloc(dev, size, GFP_KERNEL);
if (!container)
return ERR_PTR(-ENOMEM);
plane = container + offset;
va_start(ap, name);
ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
formats, format_count, format_modifiers,
type, name, ap);
va_end(ap);
if (ret)
return ERR_PTR(ret);
ret = drmm_add_action_or_reset(dev, drmm_universal_plane_alloc_release,
plane);
if (ret)
return ERR_PTR(ret);
return container;
}
EXPORT_SYMBOL(__drmm_universal_plane_alloc);
void *__drm_universal_plane_alloc(struct drm_device *dev, size_t size,
size_t offset, uint32_t possible_crtcs,
const struct drm_plane_funcs *funcs,
const uint32_t *formats, unsigned int format_count,
const uint64_t *format_modifiers,
enum drm_plane_type type,
const char *name, ...)
{
void *container;
struct drm_plane *plane;
va_list ap;
int ret;
if (drm_WARN_ON(dev, !funcs))
return ERR_PTR(-EINVAL);
container = kzalloc(size, GFP_KERNEL);
if (!container)
return ERR_PTR(-ENOMEM);
plane = container + offset;
va_start(ap, name);
ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
formats, format_count, format_modifiers,
type, name, ap);
va_end(ap);
if (ret)
goto err_kfree;
return container;
err_kfree:
kfree(container);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(__drm_universal_plane_alloc);
int drm_plane_register_all(struct drm_device *dev)
{
unsigned int num_planes = 0;
unsigned int num_zpos = 0;
struct drm_plane *plane;
int ret = 0;
drm_for_each_plane(plane, dev) {
if (plane->funcs->late_register)
ret = plane->funcs->late_register(plane);
if (ret)
return ret;
if (plane->zpos_property)
num_zpos++;
num_planes++;
}
drm_WARN(dev, num_zpos && num_planes != num_zpos,
"Mixing planes with and without zpos property is invalid\n");
return 0;
}
void drm_plane_unregister_all(struct drm_device *dev)
{
struct drm_plane *plane;
drm_for_each_plane(plane, dev) { if (plane->funcs->early_unregister) plane->funcs->early_unregister(plane);
}
}
/**
* drm_plane_cleanup - Clean up the core plane usage
* @plane: plane to cleanup
*
* This function cleans up @plane and removes it from the DRM mode setting
* core. Note that the function does *not* free the plane structure itself,
* this is the responsibility of the caller.
*/
void drm_plane_cleanup(struct drm_plane *plane)
{
struct drm_device *dev = plane->dev;
drm_modeset_lock_fini(&plane->mutex);
kfree(plane->format_types);
kfree(plane->modifiers);
drm_mode_object_unregister(dev, &plane->base);
BUG_ON(list_empty(&plane->head));
/* Note that the plane_list is considered to be static; should we
* remove the drm_plane at runtime we would have to decrement all
* the indices on the drm_plane after us in the plane_list.
*/
list_del(&plane->head);
dev->mode_config.num_total_plane--;
WARN_ON(plane->state && !plane->funcs->atomic_destroy_state);
if (plane->state && plane->funcs->atomic_destroy_state)
plane->funcs->atomic_destroy_state(plane, plane->state);
kfree(plane->name);
memset(plane, 0, sizeof(*plane));
}
EXPORT_SYMBOL(drm_plane_cleanup);
/**
* drm_plane_from_index - find the registered plane at an index
* @dev: DRM device
* @idx: index of registered plane to find for
*
* Given a plane index, return the registered plane from DRM device's
* list of planes with matching index. This is the inverse of drm_plane_index().
*/
struct drm_plane *
drm_plane_from_index(struct drm_device *dev, int idx)
{
struct drm_plane *plane;
drm_for_each_plane(plane, dev)
if (idx == plane->index)
return plane;
return NULL;
}
EXPORT_SYMBOL(drm_plane_from_index);
/**
* drm_plane_force_disable - Forcibly disable a plane
* @plane: plane to disable
*
* Forces the plane to be disabled.
*
* Used when the plane's current framebuffer is destroyed,
* and when restoring fbdev mode.
*
* Note that this function is not suitable for atomic drivers, since it doesn't
* wire through the lock acquisition context properly and hence can't handle
* retries or driver private locks. You probably want to use
* drm_atomic_helper_disable_plane() or
* drm_atomic_helper_disable_planes_on_crtc() instead.
*/
void drm_plane_force_disable(struct drm_plane *plane)
{
int ret;
if (!plane->fb)
return;
WARN_ON(drm_drv_uses_atomic_modeset(plane->dev));
plane->old_fb = plane->fb;
ret = plane->funcs->disable_plane(plane, NULL);
if (ret) {
DRM_ERROR("failed to disable plane with busy fb\n");
plane->old_fb = NULL;
return;
}
/* disconnect the plane from the fb and crtc: */
drm_framebuffer_put(plane->old_fb);
plane->old_fb = NULL;
plane->fb = NULL;
plane->crtc = NULL;
}
EXPORT_SYMBOL(drm_plane_force_disable);
/**
* drm_mode_plane_set_obj_prop - set the value of a property
* @plane: drm plane object to set property value for
* @property: property to set
* @value: value the property should be set to
*
* This functions sets a given property on a given plane object. This function
* calls the driver's ->set_property callback and changes the software state of
* the property if the callback succeeds.
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_mode_plane_set_obj_prop(struct drm_plane *plane,
struct drm_property *property,
uint64_t value)
{
int ret = -EINVAL;
struct drm_mode_object *obj = &plane->base;
if (plane->funcs->set_property)
ret = plane->funcs->set_property(plane, property, value);
if (!ret)
drm_object_property_set_value(obj, property, value);
return ret;
}
EXPORT_SYMBOL(drm_mode_plane_set_obj_prop);
int drm_mode_getplane_res(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_mode_get_plane_res *plane_resp = data;
struct drm_plane *plane;
uint32_t __user *plane_ptr;
int count = 0;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
plane_ptr = u64_to_user_ptr(plane_resp->plane_id_ptr);
/*
* This ioctl is called twice, once to determine how much space is
* needed, and the 2nd time to fill it.
*/
drm_for_each_plane(plane, dev) {
/*
* Unless userspace set the 'universal planes'
* capability bit, only advertise overlays.
*/
if (plane->type != DRM_PLANE_TYPE_OVERLAY &&
!file_priv->universal_planes)
continue;
/*
* If we're running on a virtualized driver then,
* unless userspace advertizes support for the
* virtualized cursor plane, disable cursor planes
* because they'll be broken due to missing cursor
* hotspot info.
*/
if (plane->type == DRM_PLANE_TYPE_CURSOR &&
drm_core_check_feature(dev, DRIVER_CURSOR_HOTSPOT) &&
file_priv->atomic &&
!file_priv->supports_virtualized_cursor_plane)
continue;
if (drm_lease_held(file_priv, plane->base.id)) {
if (count < plane_resp->count_planes &&
put_user(plane->base.id, plane_ptr + count))
return -EFAULT;
count++;
}
}
plane_resp->count_planes = count;
return 0;
}
int drm_mode_getplane(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_mode_get_plane *plane_resp = data;
struct drm_plane *plane;
uint32_t __user *format_ptr;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
plane = drm_plane_find(dev, file_priv, plane_resp->plane_id);
if (!plane)
return -ENOENT;
drm_modeset_lock(&plane->mutex, NULL);
if (plane->state && plane->state->crtc && drm_lease_held(file_priv, plane->state->crtc->base.id))
plane_resp->crtc_id = plane->state->crtc->base.id;
else if (!plane->state && plane->crtc && drm_lease_held(file_priv, plane->crtc->base.id))
plane_resp->crtc_id = plane->crtc->base.id;
else
plane_resp->crtc_id = 0;
if (plane->state && plane->state->fb)
plane_resp->fb_id = plane->state->fb->base.id;
else if (!plane->state && plane->fb)
plane_resp->fb_id = plane->fb->base.id;
else
plane_resp->fb_id = 0;
drm_modeset_unlock(&plane->mutex);
plane_resp->plane_id = plane->base.id;
plane_resp->possible_crtcs = drm_lease_filter_crtcs(file_priv,
plane->possible_crtcs);
plane_resp->gamma_size = 0;
/*
* This ioctl is called twice, once to determine how much space is
* needed, and the 2nd time to fill it.
*/
if (plane->format_count &&
(plane_resp->count_format_types >= plane->format_count)) {
format_ptr = (uint32_t __user *)(unsigned long)plane_resp->format_type_ptr;
if (copy_to_user(format_ptr,
plane->format_types,
sizeof(uint32_t) * plane->format_count)) {
return -EFAULT;
}
}
plane_resp->count_format_types = plane->format_count;
return 0;
}
int drm_plane_check_pixel_format(struct drm_plane *plane,
u32 format, u64 modifier)
{
unsigned int i;
for (i = 0; i < plane->format_count; i++) {
if (format == plane->format_types[i])
break;
}
if (i == plane->format_count)
return -EINVAL;
if (plane->funcs->format_mod_supported) {
if (!plane->funcs->format_mod_supported(plane, format, modifier))
return -EINVAL;
} else {
if (!plane->modifier_count)
return 0;
for (i = 0; i < plane->modifier_count; i++) {
if (modifier == plane->modifiers[i])
break;
}
if (i == plane->modifier_count)
return -EINVAL;
}
return 0;
}
static int __setplane_check(struct drm_plane *plane,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
int32_t crtc_x, int32_t crtc_y,
uint32_t crtc_w, uint32_t crtc_h,
uint32_t src_x, uint32_t src_y,
uint32_t src_w, uint32_t src_h)
{
int ret;
/* Check whether this plane is usable on this CRTC */
if (!(plane->possible_crtcs & drm_crtc_mask(crtc))) {
DRM_DEBUG_KMS("Invalid crtc for plane\n");
return -EINVAL;
}
/* Check whether this plane supports the fb pixel format. */
ret = drm_plane_check_pixel_format(plane, fb->format->format,
fb->modifier);
if (ret) {
DRM_DEBUG_KMS("Invalid pixel format %p4cc, modifier 0x%llx\n",
&fb->format->format, fb->modifier);
return ret;
}
/* Give drivers some help against integer overflows */
if (crtc_w > INT_MAX ||
crtc_x > INT_MAX - (int32_t) crtc_w ||
crtc_h > INT_MAX ||
crtc_y > INT_MAX - (int32_t) crtc_h) {
DRM_DEBUG_KMS("Invalid CRTC coordinates %ux%u+%d+%d\n",
crtc_w, crtc_h, crtc_x, crtc_y);
return -ERANGE;
}
ret = drm_framebuffer_check_src_coords(src_x, src_y, src_w, src_h, fb);
if (ret)
return ret;
return 0;
}
/**
* drm_any_plane_has_format - Check whether any plane supports this format and modifier combination
* @dev: DRM device
* @format: pixel format (DRM_FORMAT_*)
* @modifier: data layout modifier
*
* Returns:
* Whether at least one plane supports the specified format and modifier combination.
*/
bool drm_any_plane_has_format(struct drm_device *dev,
u32 format, u64 modifier)
{
struct drm_plane *plane;
drm_for_each_plane(plane, dev) {
if (drm_plane_check_pixel_format(plane, format, modifier) == 0)
return true;
}
return false;
}
EXPORT_SYMBOL(drm_any_plane_has_format);
/*
* __setplane_internal - setplane handler for internal callers
*
* This function will take a reference on the new fb for the plane
* on success.
*
* src_{x,y,w,h} are provided in 16.16 fixed point format
*/
static int __setplane_internal(struct drm_plane *plane,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
int32_t crtc_x, int32_t crtc_y,
uint32_t crtc_w, uint32_t crtc_h,
/* src_{x,y,w,h} values are 16.16 fixed point */
uint32_t src_x, uint32_t src_y,
uint32_t src_w, uint32_t src_h,
struct drm_modeset_acquire_ctx *ctx)
{
int ret = 0;
WARN_ON(drm_drv_uses_atomic_modeset(plane->dev));
/* No fb means shut it down */
if (!fb) {
plane->old_fb = plane->fb;
ret = plane->funcs->disable_plane(plane, ctx);
if (!ret) {
plane->crtc = NULL;
plane->fb = NULL;
} else {
plane->old_fb = NULL;
}
goto out;
}
ret = __setplane_check(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
src_x, src_y, src_w, src_h);
if (ret)
goto out;
plane->old_fb = plane->fb;
ret = plane->funcs->update_plane(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
src_x, src_y, src_w, src_h, ctx);
if (!ret) {
plane->crtc = crtc;
plane->fb = fb;
drm_framebuffer_get(plane->fb);
} else {
plane->old_fb = NULL;
}
out:
if (plane->old_fb)
drm_framebuffer_put(plane->old_fb);
plane->old_fb = NULL;
return ret;
}
static int __setplane_atomic(struct drm_plane *plane,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
int32_t crtc_x, int32_t crtc_y,
uint32_t crtc_w, uint32_t crtc_h,
uint32_t src_x, uint32_t src_y,
uint32_t src_w, uint32_t src_h,
struct drm_modeset_acquire_ctx *ctx)
{
int ret;
WARN_ON(!drm_drv_uses_atomic_modeset(plane->dev));
/* No fb means shut it down */
if (!fb)
return plane->funcs->disable_plane(plane, ctx);
/*
* FIXME: This is redundant with drm_atomic_plane_check(),
* but the legacy cursor/"async" .update_plane() tricks
* don't call that so we still need this here. Should remove
* this when all .update_plane() implementations have been
* fixed to call drm_atomic_plane_check().
*/
ret = __setplane_check(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
src_x, src_y, src_w, src_h);
if (ret)
return ret;
return plane->funcs->update_plane(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
src_x, src_y, src_w, src_h, ctx);
}
static int setplane_internal(struct drm_plane *plane,
struct drm_crtc *crtc,
struct drm_framebuffer *fb,
int32_t crtc_x, int32_t crtc_y,
uint32_t crtc_w, uint32_t crtc_h,
/* src_{x,y,w,h} values are 16.16 fixed point */
uint32_t src_x, uint32_t src_y,
uint32_t src_w, uint32_t src_h)
{
struct drm_modeset_acquire_ctx ctx;
int ret;
DRM_MODESET_LOCK_ALL_BEGIN(plane->dev, ctx,
DRM_MODESET_ACQUIRE_INTERRUPTIBLE, ret);
if (drm_drv_uses_atomic_modeset(plane->dev))
ret = __setplane_atomic(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
src_x, src_y, src_w, src_h, &ctx);
else
ret = __setplane_internal(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
src_x, src_y, src_w, src_h, &ctx);
DRM_MODESET_LOCK_ALL_END(plane->dev, ctx, ret);
return ret;
}
int drm_mode_setplane(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_mode_set_plane *plane_req = data;
struct drm_plane *plane;
struct drm_crtc *crtc = NULL;
struct drm_framebuffer *fb = NULL;
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
/*
* First, find the plane, crtc, and fb objects. If not available,
* we don't bother to call the driver.
*/
plane = drm_plane_find(dev, file_priv, plane_req->plane_id);
if (!plane) {
DRM_DEBUG_KMS("Unknown plane ID %d\n",
plane_req->plane_id);
return -ENOENT;
}
if (plane_req->fb_id) {
fb = drm_framebuffer_lookup(dev, file_priv, plane_req->fb_id);
if (!fb) {
DRM_DEBUG_KMS("Unknown framebuffer ID %d\n",
plane_req->fb_id);
return -ENOENT;
}
crtc = drm_crtc_find(dev, file_priv, plane_req->crtc_id);
if (!crtc) {
drm_framebuffer_put(fb);
DRM_DEBUG_KMS("Unknown crtc ID %d\n",
plane_req->crtc_id);
return -ENOENT;
}
}
ret = setplane_internal(plane, crtc, fb,
plane_req->crtc_x, plane_req->crtc_y,
plane_req->crtc_w, plane_req->crtc_h,
plane_req->src_x, plane_req->src_y,
plane_req->src_w, plane_req->src_h);
if (fb)
drm_framebuffer_put(fb);
return ret;
}
static int drm_mode_cursor_universal(struct drm_crtc *crtc,
struct drm_mode_cursor2 *req,
struct drm_file *file_priv,
struct drm_modeset_acquire_ctx *ctx)
{
struct drm_device *dev = crtc->dev;
struct drm_plane *plane = crtc->cursor;
struct drm_framebuffer *fb = NULL;
struct drm_mode_fb_cmd2 fbreq = {
.width = req->width,
.height = req->height,
.pixel_format = DRM_FORMAT_ARGB8888,
.pitches = { req->width * 4 },
.handles = { req->handle },
};
int32_t crtc_x, crtc_y;
uint32_t crtc_w = 0, crtc_h = 0;
uint32_t src_w = 0, src_h = 0;
int ret = 0;
BUG_ON(!plane);
WARN_ON(plane->crtc != crtc && plane->crtc != NULL);
/*
* Obtain fb we'll be using (either new or existing) and take an extra
* reference to it if fb != null. setplane will take care of dropping
* the reference if the plane update fails.
*/
if (req->flags & DRM_MODE_CURSOR_BO) {
if (req->handle) {
fb = drm_internal_framebuffer_create(dev, &fbreq, file_priv);
if (IS_ERR(fb)) {
DRM_DEBUG_KMS("failed to wrap cursor buffer in drm framebuffer\n");
return PTR_ERR(fb);
}
if (plane->hotspot_x_property && plane->state)
plane->state->hotspot_x = req->hot_x;
if (plane->hotspot_y_property && plane->state)
plane->state->hotspot_y = req->hot_y;
} else {
fb = NULL;
}
} else {
if (plane->state)
fb = plane->state->fb;
else
fb = plane->fb;
if (fb)
drm_framebuffer_get(fb);
}
if (req->flags & DRM_MODE_CURSOR_MOVE) {
crtc_x = req->x;
crtc_y = req->y;
} else {
crtc_x = crtc->cursor_x;
crtc_y = crtc->cursor_y;
}
if (fb) {
crtc_w = fb->width;
crtc_h = fb->height;
src_w = fb->width << 16;
src_h = fb->height << 16;
}
if (drm_drv_uses_atomic_modeset(dev))
ret = __setplane_atomic(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
0, 0, src_w, src_h, ctx);
else
ret = __setplane_internal(plane, crtc, fb,
crtc_x, crtc_y, crtc_w, crtc_h,
0, 0, src_w, src_h, ctx);
if (fb)
drm_framebuffer_put(fb);
/* Update successful; save new cursor position, if necessary */
if (ret == 0 && req->flags & DRM_MODE_CURSOR_MOVE) {
crtc->cursor_x = req->x;
crtc->cursor_y = req->y;
}
return ret;
}
static int drm_mode_cursor_common(struct drm_device *dev,
struct drm_mode_cursor2 *req,
struct drm_file *file_priv)
{
struct drm_crtc *crtc;
struct drm_modeset_acquire_ctx ctx;
int ret = 0;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
if (!req->flags || (~DRM_MODE_CURSOR_FLAGS & req->flags))
return -EINVAL;
crtc = drm_crtc_find(dev, file_priv, req->crtc_id);
if (!crtc) {
DRM_DEBUG_KMS("Unknown CRTC ID %d\n", req->crtc_id);
return -ENOENT;
}
drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
retry:
ret = drm_modeset_lock(&crtc->mutex, &ctx);
if (ret)
goto out;
/*
* If this crtc has a universal cursor plane, call that plane's update
* handler rather than using legacy cursor handlers.
*/
if (crtc->cursor) {
ret = drm_modeset_lock(&crtc->cursor->mutex, &ctx);
if (ret)
goto out;
if (!drm_lease_held(file_priv, crtc->cursor->base.id)) {
ret = -EACCES;
goto out;
}
ret = drm_mode_cursor_universal(crtc, req, file_priv, &ctx);
goto out;
}
if (req->flags & DRM_MODE_CURSOR_BO) {
if (!crtc->funcs->cursor_set && !crtc->funcs->cursor_set2) {
ret = -ENXIO;
goto out;
}
/* Turns off the cursor if handle is 0 */
if (crtc->funcs->cursor_set2)
ret = crtc->funcs->cursor_set2(crtc, file_priv, req->handle,
req->width, req->height, req->hot_x, req->hot_y);
else
ret = crtc->funcs->cursor_set(crtc, file_priv, req->handle,
req->width, req->height);
}
if (req->flags & DRM_MODE_CURSOR_MOVE) {
if (crtc->funcs->cursor_move) {
ret = crtc->funcs->cursor_move(crtc, req->x, req->y);
} else {
ret = -EFAULT;
goto out;
}
}
out:
if (ret == -EDEADLK) {
ret = drm_modeset_backoff(&ctx);
if (!ret)
goto retry;
}
drm_modeset_drop_locks(&ctx);
drm_modeset_acquire_fini(&ctx);
return ret;
}
int drm_mode_cursor_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv)
{
struct drm_mode_cursor *req = data;
struct drm_mode_cursor2 new_req;
memcpy(&new_req, req, sizeof(struct drm_mode_cursor));
new_req.hot_x = new_req.hot_y = 0;
return drm_mode_cursor_common(dev, &new_req, file_priv);
}
/*
* Set the cursor configuration based on user request. This implements the 2nd
* version of the cursor ioctl, which allows userspace to additionally specify
* the hotspot of the pointer.
*/
int drm_mode_cursor2_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv)
{
struct drm_mode_cursor2 *req = data;
return drm_mode_cursor_common(dev, req, file_priv);
}
int drm_mode_page_flip_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv)
{
struct drm_mode_crtc_page_flip_target *page_flip = data;
struct drm_crtc *crtc;
struct drm_plane *plane;
struct drm_framebuffer *fb = NULL, *old_fb;
struct drm_pending_vblank_event *e = NULL;
u32 target_vblank = page_flip->sequence;
struct drm_modeset_acquire_ctx ctx;
int ret = -EINVAL;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
if (page_flip->flags & ~DRM_MODE_PAGE_FLIP_FLAGS)
return -EINVAL;
if (page_flip->sequence != 0 && !(page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET))
return -EINVAL;
/* Only one of the DRM_MODE_PAGE_FLIP_TARGET_ABSOLUTE/RELATIVE flags
* can be specified
*/
if ((page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET) == DRM_MODE_PAGE_FLIP_TARGET)
return -EINVAL;
if ((page_flip->flags & DRM_MODE_PAGE_FLIP_ASYNC) && !dev->mode_config.async_page_flip)
return -EINVAL;
crtc = drm_crtc_find(dev, file_priv, page_flip->crtc_id);
if (!crtc)
return -ENOENT;
plane = crtc->primary;
if (!drm_lease_held(file_priv, plane->base.id))
return -EACCES;
if (crtc->funcs->page_flip_target) {
u32 current_vblank;
int r;
r = drm_crtc_vblank_get(crtc);
if (r)
return r;
current_vblank = (u32)drm_crtc_vblank_count(crtc);
switch (page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET) {
case DRM_MODE_PAGE_FLIP_TARGET_ABSOLUTE:
if ((int)(target_vblank - current_vblank) > 1) {
DRM_DEBUG("Invalid absolute flip target %u, "
"must be <= %u\n", target_vblank,
current_vblank + 1);
drm_crtc_vblank_put(crtc);
return -EINVAL;
}
break;
case DRM_MODE_PAGE_FLIP_TARGET_RELATIVE:
if (target_vblank != 0 && target_vblank != 1) {
DRM_DEBUG("Invalid relative flip target %u, "
"must be 0 or 1\n", target_vblank);
drm_crtc_vblank_put(crtc);
return -EINVAL;
}
target_vblank += current_vblank;
break;
default:
target_vblank = current_vblank +
!(page_flip->flags & DRM_MODE_PAGE_FLIP_ASYNC);
break;
}
} else if (crtc->funcs->page_flip == NULL ||
(page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET)) {
return -EINVAL;
}
drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
retry:
ret = drm_modeset_lock(&crtc->mutex, &ctx);
if (ret)
goto out;
ret = drm_modeset_lock(&plane->mutex, &ctx);
if (ret)
goto out;
if (plane->state)
old_fb = plane->state->fb;
else
old_fb = plane->fb;
if (old_fb == NULL) {
/* The framebuffer is currently unbound, presumably
* due to a hotplug event, that userspace has not
* yet discovered.
*/
ret = -EBUSY;
goto out;
}
fb = drm_framebuffer_lookup(dev, file_priv, page_flip->fb_id);
if (!fb) {
ret = -ENOENT;
goto out;
}
if (plane->state) {
const struct drm_plane_state *state = plane->state;
ret = drm_framebuffer_check_src_coords(state->src_x,
state->src_y,
state->src_w,
state->src_h,
fb);
} else {
ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y,
&crtc->mode, fb);
}
if (ret)
goto out;
/*
* Only check the FOURCC format code, excluding modifiers. This is
* enough for all legacy drivers. Atomic drivers have their own
* checks in their ->atomic_check implementation, which will
* return -EINVAL if any hw or driver constraint is violated due
* to modifier changes.
*/
if (old_fb->format->format != fb->format->format) {
DRM_DEBUG_KMS("Page flip is not allowed to change frame buffer format.\n");
ret = -EINVAL;
goto out;
}
if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT) {
e = kzalloc(sizeof *e, GFP_KERNEL);
if (!e) {
ret = -ENOMEM;
goto out;
}
e->event.base.type = DRM_EVENT_FLIP_COMPLETE;
e->event.base.length = sizeof(e->event);
e->event.vbl.user_data = page_flip->user_data;
e->event.vbl.crtc_id = crtc->base.id;
ret = drm_event_reserve_init(dev, file_priv, &e->base, &e->event.base);
if (ret) {
kfree(e);
e = NULL;
goto out;
}
}
plane->old_fb = plane->fb;
if (crtc->funcs->page_flip_target)
ret = crtc->funcs->page_flip_target(crtc, fb, e,
page_flip->flags,
target_vblank,
&ctx);
else
ret = crtc->funcs->page_flip(crtc, fb, e, page_flip->flags,
&ctx);
if (ret) {
if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT)
drm_event_cancel_free(dev, &e->base);
/* Keep the old fb, don't unref it. */
plane->old_fb = NULL;
} else {
if (!plane->state) {
plane->fb = fb;
drm_framebuffer_get(fb);
}
}
out:
if (fb)
drm_framebuffer_put(fb);
fb = NULL;
if (plane->old_fb)
drm_framebuffer_put(plane->old_fb);
plane->old_fb = NULL;
if (ret == -EDEADLK) {
ret = drm_modeset_backoff(&ctx);
if (!ret)
goto retry;
}
drm_modeset_drop_locks(&ctx);
drm_modeset_acquire_fini(&ctx);
if (ret && crtc->funcs->page_flip_target)
drm_crtc_vblank_put(crtc);
return ret;
}
/**
* DOC: damage tracking
*
* FB_DAMAGE_CLIPS is an optional plane property which provides a means to
* specify a list of damage rectangles on a plane in framebuffer coordinates of
* the framebuffer attached to the plane. In current context damage is the area
* of plane framebuffer that has changed since last plane update (also called
* page-flip), irrespective of whether currently attached framebuffer is same as
* framebuffer attached during last plane update or not.
*
* FB_DAMAGE_CLIPS is a hint to kernel which could be helpful for some drivers
* to optimize internally especially for virtual devices where each framebuffer
* change needs to be transmitted over network, usb, etc.
*
* Since FB_DAMAGE_CLIPS is a hint so it is an optional property. User-space can
* ignore damage clips property and in that case driver will do a full plane
* update. In case damage clips are provided then it is guaranteed that the area
* inside damage clips will be updated to plane. For efficiency driver can do
* full update or can update more than specified in damage clips. Since driver
* is free to read more, user-space must always render the entire visible
* framebuffer. Otherwise there can be corruptions. Also, if a user-space
* provides damage clips which doesn't encompass the actual damage to
* framebuffer (since last plane update) can result in incorrect rendering.
*
* FB_DAMAGE_CLIPS is a blob property with the layout of blob data is simply an
* array of &drm_mode_rect. Unlike plane &drm_plane_state.src coordinates,
* damage clips are not in 16.16 fixed point. Similar to plane src in
* framebuffer, damage clips cannot be negative. In damage clip, x1/y1 are
* inclusive and x2/y2 are exclusive. While kernel does not error for overlapped
* damage clips, it is strongly discouraged.
*
* Drivers that are interested in damage interface for plane should enable
* FB_DAMAGE_CLIPS property by calling drm_plane_enable_fb_damage_clips().
* Drivers implementing damage can use drm_atomic_helper_damage_iter_init() and
* drm_atomic_helper_damage_iter_next() helper iterator function to get damage
* rectangles clipped to &drm_plane_state.src.
*
* Note that there are two types of damage handling: frame damage and buffer
* damage, the type of damage handling implemented depends on a driver's upload
* target. Drivers implementing a per-plane or per-CRTC upload target need to
* handle frame damage, while drivers implementing a per-buffer upload target
* need to handle buffer damage.
*
* The existing damage helpers only support the frame damage type, there is no
* buffer age support or similar damage accumulation algorithm implemented yet.
*
* Only drivers handling frame damage can use the mentioned damage helpers to
* iterate over the damaged regions. Drivers that handle buffer damage, must set
* &drm_plane_state.ignore_damage_clips for drm_atomic_helper_damage_iter_init()
* to know that damage clips should be ignored and return &drm_plane_state.src
* as the damage rectangle, to force a full plane update.
*
* Drivers with a per-buffer upload target could compare the &drm_plane_state.fb
* of the old and new plane states to determine if the framebuffer attached to a
* plane has changed or not since the last plane update. If &drm_plane_state.fb
* has changed, then &drm_plane_state.ignore_damage_clips must be set to true.
*
* That is because drivers with a per-plane upload target, expect the backing
* storage buffer to not change for a given plane. If the upload buffer changes
* between page flips, the new upload buffer has to be updated as a whole. This
* can be improved in the future if support for frame damage is added to the DRM
* damage helpers, similarly to how user-space already handle this case as it is
* explained in the following documents:
*
* https://registry.khronos.org/EGL/extensions/KHR/EGL_KHR_swap_buffers_with_damage.txt
* https://emersion.fr/blog/2019/intro-to-damage-tracking/
*/
/**
* drm_plane_enable_fb_damage_clips - Enables plane fb damage clips property.
* @plane: Plane on which to enable damage clips property.
*
* This function lets driver to enable the damage clips property on a plane.
*/
void drm_plane_enable_fb_damage_clips(struct drm_plane *plane)
{
struct drm_device *dev = plane->dev;
struct drm_mode_config *config = &dev->mode_config;
drm_object_attach_property(&plane->base, config->prop_fb_damage_clips,
0);
}
EXPORT_SYMBOL(drm_plane_enable_fb_damage_clips);
/**
* drm_plane_get_damage_clips_count - Returns damage clips count.
* @state: Plane state.
*
* Simple helper to get the number of &drm_mode_rect clips set by user-space
* during plane update.
*
* Return: Number of clips in plane fb_damage_clips blob property.
*/
unsigned int
drm_plane_get_damage_clips_count(const struct drm_plane_state *state)
{
return (state && state->fb_damage_clips) ?
state->fb_damage_clips->length/sizeof(struct drm_mode_rect) : 0;
}
EXPORT_SYMBOL(drm_plane_get_damage_clips_count);
struct drm_mode_rect *
__drm_plane_get_damage_clips(const struct drm_plane_state *state)
{
return (struct drm_mode_rect *)((state && state->fb_damage_clips) ?
state->fb_damage_clips->data : NULL);
}
/**
* drm_plane_get_damage_clips - Returns damage clips.
* @state: Plane state.
*
* Note that this function returns uapi type &drm_mode_rect. Drivers might want
* to use the helper functions drm_atomic_helper_damage_iter_init() and
* drm_atomic_helper_damage_iter_next() or drm_atomic_helper_damage_merged() if
* the driver can only handle a single damage region at most.
*
* Return: Damage clips in plane fb_damage_clips blob property.
*/
struct drm_mode_rect *
drm_plane_get_damage_clips(const struct drm_plane_state *state)
{
struct drm_device *dev = state->plane->dev;
struct drm_mode_config *config = &dev->mode_config;
/* check that drm_plane_enable_fb_damage_clips() was called */
if (!drm_mode_obj_find_prop_id(&state->plane->base,
config->prop_fb_damage_clips->base.id))
drm_warn_once(dev, "drm_plane_enable_fb_damage_clips() not called\n");
return __drm_plane_get_damage_clips(state);
}
EXPORT_SYMBOL(drm_plane_get_damage_clips);
struct drm_property *
drm_create_scaling_filter_prop(struct drm_device *dev,
unsigned int supported_filters)
{
struct drm_property *prop;
static const struct drm_prop_enum_list props[] = {
{ DRM_SCALING_FILTER_DEFAULT, "Default" },
{ DRM_SCALING_FILTER_NEAREST_NEIGHBOR, "Nearest Neighbor" },
};
unsigned int valid_mode_mask = BIT(DRM_SCALING_FILTER_DEFAULT) |
BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR);
int i;
if (WARN_ON((supported_filters & ~valid_mode_mask) ||
((supported_filters & BIT(DRM_SCALING_FILTER_DEFAULT)) == 0)))
return ERR_PTR(-EINVAL);
prop = drm_property_create(dev, DRM_MODE_PROP_ENUM,
"SCALING_FILTER",
hweight32(supported_filters));
if (!prop)
return ERR_PTR(-ENOMEM);
for (i = 0; i < ARRAY_SIZE(props); i++) {
int ret;
if (!(BIT(props[i].type) & supported_filters))
continue;
ret = drm_property_add_enum(prop, props[i].type,
props[i].name);
if (ret) {
drm_property_destroy(dev, prop);
return ERR_PTR(ret);
}
}
return prop;
}
/**
* drm_plane_create_scaling_filter_property - create a new scaling filter
* property
*
* @plane: drm plane
* @supported_filters: bitmask of supported scaling filters, must include
* BIT(DRM_SCALING_FILTER_DEFAULT).
*
* This function lets driver to enable the scaling filter property on a given
* plane.
*
* RETURNS:
* Zero for success or -errno
*/
int drm_plane_create_scaling_filter_property(struct drm_plane *plane,
unsigned int supported_filters)
{
struct drm_property *prop =
drm_create_scaling_filter_prop(plane->dev, supported_filters);
if (IS_ERR(prop))
return PTR_ERR(prop);
drm_object_attach_property(&plane->base, prop,
DRM_SCALING_FILTER_DEFAULT);
plane->scaling_filter_property = prop;
return 0;
}
EXPORT_SYMBOL(drm_plane_create_scaling_filter_property);
// SPDX-License-Identifier: GPL-2.0
/*
* kobject.h - generic kernel object infrastructure.
*
* Copyright (c) 2002-2003 Patrick Mochel
* Copyright (c) 2002-2003 Open Source Development Labs
* Copyright (c) 2006-2008 Greg Kroah-Hartman <greg@kroah.com>
* Copyright (c) 2006-2008 Novell Inc.
*
* Please read Documentation/core-api/kobject.rst before using the kobject
* interface, ESPECIALLY the parts about reference counts and object
* destructors.
*/
#ifndef _KOBJECT_H_
#define _KOBJECT_H_
#include <linux/types.h>
#include <linux/list.h>
#include <linux/sysfs.h>
#include <linux/compiler.h>
#include <linux/container_of.h>
#include <linux/spinlock.h>
#include <linux/kref.h>
#include <linux/kobject_ns.h>
#include <linux/wait.h>
#include <linux/atomic.h>
#include <linux/workqueue.h>
#include <linux/uidgid.h>
#define UEVENT_HELPER_PATH_LEN 256
#define UEVENT_NUM_ENVP 64 /* number of env pointers */
#define UEVENT_BUFFER_SIZE 2048 /* buffer for the variables */
#ifdef CONFIG_UEVENT_HELPER
/* path to the userspace helper executed on an event */
extern char uevent_helper[];
#endif
/* counter to tag the uevent, read only except for the kobject core */
extern atomic64_t uevent_seqnum;
/*
* The actions here must match the index to the string array
* in lib/kobject_uevent.c
*
* Do not add new actions here without checking with the driver-core
* maintainers. Action strings are not meant to express subsystem
* or device specific properties. In most cases you want to send a
* kobject_uevent_env(kobj, KOBJ_CHANGE, env) with additional event
* specific variables added to the event environment.
*/
enum kobject_action {
KOBJ_ADD,
KOBJ_REMOVE,
KOBJ_CHANGE,
KOBJ_MOVE,
KOBJ_ONLINE,
KOBJ_OFFLINE,
KOBJ_BIND,
KOBJ_UNBIND,
};
struct kobject {
const char *name;
struct list_head entry;
struct kobject *parent;
struct kset *kset;
const struct kobj_type *ktype;
struct kernfs_node *sd; /* sysfs directory entry */
struct kref kref;
unsigned int state_initialized:1;
unsigned int state_in_sysfs:1;
unsigned int state_add_uevent_sent:1;
unsigned int state_remove_uevent_sent:1;
unsigned int uevent_suppress:1;
#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
struct delayed_work release;
#endif
};
__printf(2, 3) int kobject_set_name(struct kobject *kobj, const char *name, ...);
__printf(2, 0) int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list vargs);
static inline const char *kobject_name(const struct kobject *kobj)
{
return kobj->name;
}
void kobject_init(struct kobject *kobj, const struct kobj_type *ktype);
__printf(3, 4) __must_check int kobject_add(struct kobject *kobj,
struct kobject *parent,
const char *fmt, ...);
__printf(4, 5) __must_check int kobject_init_and_add(struct kobject *kobj,
const struct kobj_type *ktype,
struct kobject *parent,
const char *fmt, ...);
void kobject_del(struct kobject *kobj);
struct kobject * __must_check kobject_create_and_add(const char *name, struct kobject *parent);
int __must_check kobject_rename(struct kobject *, const char *new_name);
int __must_check kobject_move(struct kobject *, struct kobject *);
struct kobject *kobject_get(struct kobject *kobj);
struct kobject * __must_check kobject_get_unless_zero(struct kobject *kobj);
void kobject_put(struct kobject *kobj);
const void *kobject_namespace(const struct kobject *kobj);
void kobject_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid);
char *kobject_get_path(const struct kobject *kobj, gfp_t flag);
struct kobj_type {
void (*release)(struct kobject *kobj);
const struct sysfs_ops *sysfs_ops;
const struct attribute_group **default_groups;
const struct kobj_ns_type_operations *(*child_ns_type)(const struct kobject *kobj);
const void *(*namespace)(const struct kobject *kobj);
void (*get_ownership)(const struct kobject *kobj, kuid_t *uid, kgid_t *gid);
};
struct kobj_uevent_env {
char *argv[3];
char *envp[UEVENT_NUM_ENVP];
int envp_idx;
char buf[UEVENT_BUFFER_SIZE];
int buflen;
};
struct kset_uevent_ops {
int (* const filter)(const struct kobject *kobj);
const char *(* const name)(const struct kobject *kobj);
int (* const uevent)(const struct kobject *kobj, struct kobj_uevent_env *env);
};
struct kobj_attribute {
struct attribute attr;
ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr,
char *buf);
ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr,
const char *buf, size_t count);
};
extern const struct sysfs_ops kobj_sysfs_ops;
struct sock;
/**
* struct kset - a set of kobjects of a specific type, belonging to a specific subsystem.
*
* A kset defines a group of kobjects. They can be individually
* different "types" but overall these kobjects all want to be grouped
* together and operated on in the same manner. ksets are used to
* define the attribute callbacks and other common events that happen to
* a kobject.
*
* @list: the list of all kobjects for this kset
* @list_lock: a lock for iterating over the kobjects
* @kobj: the embedded kobject for this kset (recursion, isn't it fun...)
* @uevent_ops: the set of uevent operations for this kset. These are
* called whenever a kobject has something happen to it so that the kset
* can add new environment variables, or filter out the uevents if so
* desired.
*/
struct kset {
struct list_head list;
spinlock_t list_lock;
struct kobject kobj;
const struct kset_uevent_ops *uevent_ops;
} __randomize_layout;
void kset_init(struct kset *kset);
int __must_check kset_register(struct kset *kset);
void kset_unregister(struct kset *kset);
struct kset * __must_check kset_create_and_add(const char *name, const struct kset_uevent_ops *u,
struct kobject *parent_kobj);
static inline struct kset *to_kset(struct kobject *kobj)
{
return kobj ? container_of(kobj, struct kset, kobj) : NULL;
}
static inline struct kset *kset_get(struct kset *k)
{
return k ? to_kset(kobject_get(&k->kobj)) : NULL;
}
static inline void kset_put(struct kset *k)
{
kobject_put(&k->kobj);
}
static inline const struct kobj_type *get_ktype(const struct kobject *kobj)
{
return kobj->ktype;
}
struct kobject *kset_find_obj(struct kset *, const char *);
/* The global /sys/kernel/ kobject for people to chain off of */
extern struct kobject *kernel_kobj;
/* The global /sys/kernel/mm/ kobject for people to chain off of */
extern struct kobject *mm_kobj;
/* The global /sys/hypervisor/ kobject for people to chain off of */
extern struct kobject *hypervisor_kobj;
/* The global /sys/power/ kobject for people to chain off of */
extern struct kobject *power_kobj;
/* The global /sys/firmware/ kobject for people to chain off of */
extern struct kobject *firmware_kobj;
int kobject_uevent(struct kobject *kobj, enum kobject_action action);
int kobject_uevent_env(struct kobject *kobj, enum kobject_action action,
char *envp[]);
int kobject_synth_uevent(struct kobject *kobj, const char *buf, size_t count);
__printf(2, 3)
int add_uevent_var(struct kobj_uevent_env *env, const char *format, ...);
#endif /* _KOBJECT_H_ */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/mm/memory.c
*
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
*/
/*
* demand-loading started 01.12.91 - seems it is high on the list of
* things wanted, and it should be easy to implement. - Linus
*/
/*
* Ok, demand-loading was easy, shared pages a little bit tricker. Shared
* pages started 02.12.91, seems to work. - Linus.
*
* Tested sharing by executing about 30 /bin/sh: under the old kernel it
* would have taken more than the 6M I have free, but it worked well as
* far as I could see.
*
* Also corrected some "invalidate()"s - I wasn't doing enough of them.
*/
/*
* Real VM (paging to/from disk) started 18.12.91. Much more work and
* thought has to go into this. Oh, well..
* 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
* Found it. Everything seems to work now.
* 20.12.91 - Ok, making the swap-device changeable like the root.
*/
/*
* 05.04.94 - Multi-page memory management added for v1.1.
* Idea by Alex Bligh (alex@cconcepts.co.uk)
*
* 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
* (Gerhard.Wichert@pdb.siemens.de)
*
* Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
*/
#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
#include <linux/kmsan.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
#include <linux/delayacct.h>
#include <linux/init.h>
#include <linux/pfn_t.h>
#include <linux/writeback.h>
#include <linux/memcontrol.h>
#include <linux/mmu_notifier.h>
#include <linux/swapops.h>
#include <linux/elf.h>
#include <linux/gfp.h>
#include <linux/migrate.h>
#include <linux/string.h>
#include <linux/memory-tiers.h>
#include <linux/debugfs.h>
#include <linux/userfaultfd_k.h>
#include <linux/dax.h>
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/perf_event.h>
#include <linux/ptrace.h>
#include <linux/vmalloc.h>
#include <linux/sched/sysctl.h>
#include <trace/events/kmem.h>
#include <asm/io.h>
#include <asm/mmu_context.h>
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include "pgalloc-track.h"
#include "internal.h"
#include "swap.h"
#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
#endif
#ifndef CONFIG_NUMA
unsigned long max_mapnr;
EXPORT_SYMBOL(max_mapnr);
struct page *mem_map;
EXPORT_SYMBOL(mem_map);
#endif
static vm_fault_t do_fault(struct vm_fault *vmf);
static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
static bool vmf_pte_changed(struct vm_fault *vmf);
/*
* Return true if the original pte was a uffd-wp pte marker (so the pte was
* wr-protected).
*/
static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
{
if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
return false;
return pte_marker_uffd_wp(vmf->orig_pte);
}
/*
* A number of key systems in x86 including ioremap() rely on the assumption
* that high_memory defines the upper bound on direct map memory, then end
* of ZONE_NORMAL.
*/
void *high_memory;
EXPORT_SYMBOL(high_memory);
/*
* Randomize the address space (stacks, mmaps, brk, etc.).
*
* ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
* as ancient (libc5 based) binaries can segfault. )
*/
int randomize_va_space __read_mostly =
#ifdef CONFIG_COMPAT_BRK
1;
#else
2;
#endif
#ifndef arch_wants_old_prefaulted_pte
static inline bool arch_wants_old_prefaulted_pte(void)
{
/*
* Transitioning a PTE from 'old' to 'young' can be expensive on
* some architectures, even if it's performed in hardware. By
* default, "false" means prefaulted entries will be 'young'.
*/
return false;
}
#endif
static int __init disable_randmaps(char *s)
{
randomize_va_space = 0;
return 1;
}
__setup("norandmaps", disable_randmaps);
unsigned long zero_pfn __read_mostly;
EXPORT_SYMBOL(zero_pfn);
unsigned long highest_memmap_pfn __read_mostly;
/*
* CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
*/
static int __init init_zero_pfn(void)
{
zero_pfn = page_to_pfn(ZERO_PAGE(0));
return 0;
}
early_initcall(init_zero_pfn);
void mm_trace_rss_stat(struct mm_struct *mm, int member)
{
trace_rss_stat(mm, member);
}
/*
* Note: this doesn't free the actual pages themselves. That
* has been handled earlier when unmapping all the memory regions.
*/
static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
unsigned long addr)
{
pgtable_t token = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free_tlb(tlb, token, addr);
mm_dec_nr_ptes(tlb->mm);
}
static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
pmd_t *pmd;
unsigned long next;
unsigned long start;
start = addr;
pmd = pmd_offset(pud, addr);
do {
next = pmd_addr_end(addr, end);
if (pmd_none_or_clear_bad(pmd))
continue;
free_pte_range(tlb, pmd, addr);
} while (pmd++, addr = next, addr != end);
start &= PUD_MASK;
if (start < floor)
return;
if (ceiling) {
ceiling &= PUD_MASK;
if (!ceiling)
return;
}
if (end - 1 > ceiling - 1)
return;
pmd = pmd_offset(pud, start);
pud_clear(pud);
pmd_free_tlb(tlb, pmd, start);
mm_dec_nr_pmds(tlb->mm);
}
static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
pud_t *pud;
unsigned long next;
unsigned long start;
start = addr;
pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
continue;
free_pmd_range(tlb, pud, addr, next, floor, ceiling);
} while (pud++, addr = next, addr != end);
start &= P4D_MASK;
if (start < floor)
return;
if (ceiling) {
ceiling &= P4D_MASK;
if (!ceiling)
return;
}
if (end - 1 > ceiling - 1)
return;
pud = pud_offset(p4d, start);
p4d_clear(p4d);
pud_free_tlb(tlb, pud, start);
mm_dec_nr_puds(tlb->mm);
}
static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
p4d_t *p4d;
unsigned long next;
unsigned long start;
start = addr;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (p4d_none_or_clear_bad(p4d))
continue;
free_pud_range(tlb, p4d, addr, next, floor, ceiling);
} while (p4d++, addr = next, addr != end);
start &= PGDIR_MASK;
if (start < floor)
return;
if (ceiling) {
ceiling &= PGDIR_MASK;
if (!ceiling)
return;
}
if (end - 1 > ceiling - 1)
return;
p4d = p4d_offset(pgd, start);
pgd_clear(pgd);
p4d_free_tlb(tlb, p4d, start);
}
/*
* This function frees user-level page tables of a process.
*/
void free_pgd_range(struct mmu_gather *tlb,
unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling)
{
pgd_t *pgd;
unsigned long next;
/*
* The next few lines have given us lots of grief...
*
* Why are we testing PMD* at this top level? Because often
* there will be no work to do at all, and we'd prefer not to
* go all the way down to the bottom just to discover that.
*
* Why all these "- 1"s? Because 0 represents both the bottom
* of the address space and the top of it (using -1 for the
* top wouldn't help much: the masks would do the wrong thing).
* The rule is that addr 0 and floor 0 refer to the bottom of
* the address space, but end 0 and ceiling 0 refer to the top
* Comparisons need to use "end - 1" and "ceiling - 1" (though
* that end 0 case should be mythical).
*
* Wherever addr is brought up or ceiling brought down, we must
* be careful to reject "the opposite 0" before it confuses the
* subsequent tests. But what about where end is brought down
* by PMD_SIZE below? no, end can't go down to 0 there.
*
* Whereas we round start (addr) and ceiling down, by different
* masks at different levels, in order to test whether a table
* now has no other vmas using it, so can be freed, we don't
* bother to round floor or end up - the tests don't need that.
*/
addr &= PMD_MASK;
if (addr < floor) {
addr += PMD_SIZE;
if (!addr)
return;
}
if (ceiling) {
ceiling &= PMD_MASK;
if (!ceiling)
return;
}
if (end - 1 > ceiling - 1)
end -= PMD_SIZE;
if (addr > end - 1)
return;
/*
* We add page table cache pages with PAGE_SIZE,
* (see pte_free_tlb()), flush the tlb if we need
*/
tlb_change_page_size(tlb, PAGE_SIZE);
pgd = pgd_offset(tlb->mm, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
} while (pgd++, addr = next, addr != end);
}
void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
struct vm_area_struct *vma, unsigned long floor,
unsigned long ceiling, bool mm_wr_locked)
{
do {
unsigned long addr = vma->vm_start;
struct vm_area_struct *next;
/*
* Note: USER_PGTABLES_CEILING may be passed as ceiling and may
* be 0. This will underflow and is okay.
*/
next = mas_find(mas, ceiling - 1);
if (unlikely(xa_is_zero(next)))
next = NULL;
/*
* Hide vma from rmap and truncate_pagecache before freeing
* pgtables
*/
if (mm_wr_locked)
vma_start_write(vma);
unlink_anon_vmas(vma);
unlink_file_vma(vma);
if (is_vm_hugetlb_page(vma)) {
hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
floor, next ? next->vm_start : ceiling);
} else {
/*
* Optimization: gather nearby vmas into one call down
*/
while (next && next->vm_start <= vma->vm_end + PMD_SIZE
&& !is_vm_hugetlb_page(next)) {
vma = next;
next = mas_find(mas, ceiling - 1);
if (unlikely(xa_is_zero(next)))
next = NULL;
if (mm_wr_locked)
vma_start_write(vma);
unlink_anon_vmas(vma);
unlink_file_vma(vma);
}
free_pgd_range(tlb, addr, vma->vm_end,
floor, next ? next->vm_start : ceiling);
}
vma = next;
} while (vma);
}
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
{
spinlock_t *ptl = pmd_lock(mm, pmd);
if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
mm_inc_nr_ptes(mm);
/*
* Ensure all pte setup (eg. pte page lock and page clearing) are
* visible before the pte is made visible to other CPUs by being
* put into page tables.
*
* The other side of the story is the pointer chasing in the page
* table walking code (when walking the page table without locking;
* ie. most of the time). Fortunately, these data accesses consist
* of a chain of data-dependent loads, meaning most CPUs (alpha
* being the notable exception) will already guarantee loads are
* seen in-order. See the alpha page table accessors for the
* smp_rmb() barriers in page table walking code.
*/
smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
pmd_populate(mm, pmd, *pte);
*pte = NULL;
}
spin_unlock(ptl);
}
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
{
pgtable_t new = pte_alloc_one(mm);
if (!new)
return -ENOMEM;
pmd_install(mm, pmd, &new);
if (new)
pte_free(mm, new); return 0;
}
int __pte_alloc_kernel(pmd_t *pmd)
{
pte_t *new = pte_alloc_one_kernel(&init_mm);
if (!new)
return -ENOMEM;
spin_lock(&init_mm.page_table_lock);
if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
smp_wmb(); /* See comment in pmd_install() */
pmd_populate_kernel(&init_mm, pmd, new);
new = NULL;
}
spin_unlock(&init_mm.page_table_lock);
if (new)
pte_free_kernel(&init_mm, new);
return 0;
}
static inline void init_rss_vec(int *rss)
{
memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
}
static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
{
int i;
for (i = 0; i < NR_MM_COUNTERS; i++)
if (rss[i])
add_mm_counter(mm, i, rss[i]);
}
/*
* This function is called to print an error when a bad pte
* is found. For example, we might have a PFN-mapped pte in
* a region that doesn't allow it.
*
* The calling function must still handle the error.
*/
static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
pte_t pte, struct page *page)
{
pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
p4d_t *p4d = p4d_offset(pgd, addr);
pud_t *pud = pud_offset(p4d, addr);
pmd_t *pmd = pmd_offset(pud, addr);
struct address_space *mapping;
pgoff_t index;
static unsigned long resume;
static unsigned long nr_shown;
static unsigned long nr_unshown;
/*
* Allow a burst of 60 reports, then keep quiet for that minute;
* or allow a steady drip of one report per second.
*/
if (nr_shown == 60) {
if (time_before(jiffies, resume)) {
nr_unshown++;
return;
}
if (nr_unshown) {
pr_alert("BUG: Bad page map: %lu messages suppressed\n",
nr_unshown);
nr_unshown = 0;
}
nr_shown = 0;
}
if (nr_shown++ == 0)
resume = jiffies + 60 * HZ;
mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
index = linear_page_index(vma, addr);
pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
current->comm,
(long long)pte_val(pte), (long long)pmd_val(*pmd));
if (page)
dump_page(page, "bad pte");
pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
(void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
vma->vm_file,
vma->vm_ops ? vma->vm_ops->fault : NULL,
vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
mapping ? mapping->a_ops->read_folio : NULL);
dump_stack();
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
}
/*
* vm_normal_page -- This function gets the "struct page" associated with a pte.
*
* "Special" mappings do not wish to be associated with a "struct page" (either
* it doesn't exist, or it exists but they don't want to touch it). In this
* case, NULL is returned here. "Normal" mappings do have a struct page.
*
* There are 2 broad cases. Firstly, an architecture may define a pte_special()
* pte bit, in which case this function is trivial. Secondly, an architecture
* may not have a spare pte bit, which requires a more complicated scheme,
* described below.
*
* A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
* special mapping (even if there are underlying and valid "struct pages").
* COWed pages of a VM_PFNMAP are always normal.
*
* The way we recognize COWed pages within VM_PFNMAP mappings is through the
* rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
* set, and the vm_pgoff will point to the first PFN mapped: thus every special
* mapping will always honor the rule
*
* pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
*
* And for normal mappings this is false.
*
* This restricts such mappings to be a linear translation from virtual address
* to pfn. To get around this restriction, we allow arbitrary mappings so long
* as the vma is not a COW mapping; in that case, we know that all ptes are
* special (because none can have been COWed).
*
*
* In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
*
* VM_MIXEDMAP mappings can likewise contain memory with or without "struct
* page" backing, however the difference is that _all_ pages with a struct
* page (that is, those where pfn_valid is true) are refcounted and considered
* normal pages by the VM. The disadvantage is that pages are refcounted
* (which can be slower and simply not an option for some PFNMAP users). The
* advantage is that we don't have to follow the strict linearity rule of
* PFNMAP mappings in order to support COWable mappings.
*
*/
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
pte_t pte)
{
unsigned long pfn = pte_pfn(pte);
if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
if (likely(!pte_special(pte)))
goto check_pfn;
if (vma->vm_ops && vma->vm_ops->find_special_page) return vma->vm_ops->find_special_page(vma, addr); if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
return NULL;
if (is_zero_pfn(pfn))
return NULL;
if (pte_devmap(pte))
/*
* NOTE: New users of ZONE_DEVICE will not set pte_devmap()
* and will have refcounts incremented on their struct pages
* when they are inserted into PTEs, thus they are safe to
* return here. Legacy ZONE_DEVICE pages that set pte_devmap()
* do not have refcounts. Example of legacy ZONE_DEVICE is
* MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
*/
return NULL;
print_bad_pte(vma, addr, pte, NULL);
return NULL;
}
/* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
if (vma->vm_flags & VM_MIXEDMAP) {
if (!pfn_valid(pfn))
return NULL;
goto out;
} else {
unsigned long off;
off = (addr - vma->vm_start) >> PAGE_SHIFT;
if (pfn == vma->vm_pgoff + off)
return NULL;
if (!is_cow_mapping(vma->vm_flags))
return NULL;
}
}
if (is_zero_pfn(pfn))
return NULL;
check_pfn:
if (unlikely(pfn > highest_memmap_pfn)) {
print_bad_pte(vma, addr, pte, NULL);
return NULL;
}
/*
* NOTE! We still have PageReserved() pages in the page tables.
* eg. VDSO mappings can cause them to exist.
*/
out:
return pfn_to_page(pfn);
}
struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
pte_t pte)
{
struct page *page = vm_normal_page(vma, addr, pte);
if (page)
return page_folio(page);
return NULL;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
pmd_t pmd)
{
unsigned long pfn = pmd_pfn(pmd);
/*
* There is no pmd_special() but there may be special pmds, e.g.
* in a direct-access (dax) mapping, so let's just replicate the
* !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
*/
if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
if (vma->vm_flags & VM_MIXEDMAP) {
if (!pfn_valid(pfn))
return NULL;
goto out;
} else {
unsigned long off;
off = (addr - vma->vm_start) >> PAGE_SHIFT;
if (pfn == vma->vm_pgoff + off)
return NULL;
if (!is_cow_mapping(vma->vm_flags))
return NULL;
}
}
if (pmd_devmap(pmd))
return NULL;
if (is_huge_zero_pmd(pmd))
return NULL;
if (unlikely(pfn > highest_memmap_pfn))
return NULL;
/*
* NOTE! We still have PageReserved() pages in the page tables.
* eg. VDSO mappings can cause them to exist.
*/
out:
return pfn_to_page(pfn);
}
struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
unsigned long addr, pmd_t pmd)
{
struct page *page = vm_normal_page_pmd(vma, addr, pmd);
if (page)
return page_folio(page);
return NULL;
}
#endif
static void restore_exclusive_pte(struct vm_area_struct *vma,
struct page *page, unsigned long address,
pte_t *ptep)
{
struct folio *folio = page_folio(page);
pte_t orig_pte;
pte_t pte;
swp_entry_t entry;
orig_pte = ptep_get(ptep);
pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
if (pte_swp_soft_dirty(orig_pte))
pte = pte_mksoft_dirty(pte);
entry = pte_to_swp_entry(orig_pte);
if (pte_swp_uffd_wp(orig_pte))
pte = pte_mkuffd_wp(pte);
else if (is_writable_device_exclusive_entry(entry))
pte = maybe_mkwrite(pte_mkdirty(pte), vma);
VM_BUG_ON_FOLIO(pte_write(pte) && (!folio_test_anon(folio) &&
PageAnonExclusive(page)), folio);
/*
* No need to take a page reference as one was already
* created when the swap entry was made.
*/
if (folio_test_anon(folio))
folio_add_anon_rmap_pte(folio, page, vma, address, RMAP_NONE);
else
/*
* Currently device exclusive access only supports anonymous
* memory so the entry shouldn't point to a filebacked page.
*/
WARN_ON_ONCE(1);
set_pte_at(vma->vm_mm, address, ptep, pte);
/*
* No need to invalidate - it was non-present before. However
* secondary CPUs may have mappings that need invalidating.
*/
update_mmu_cache(vma, address, ptep);
}
/*
* Tries to restore an exclusive pte if the page lock can be acquired without
* sleeping.
*/
static int
try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
unsigned long addr)
{
swp_entry_t entry = pte_to_swp_entry(ptep_get(src_pte));
struct page *page = pfn_swap_entry_to_page(entry);
if (trylock_page(page)) {
restore_exclusive_pte(vma, page, addr, src_pte);
unlock_page(page);
return 0;
}
return -EBUSY;
}
/*
* copy one vm_area from one task to the other. Assumes the page tables
* already present in the new task to be cleared in the whole range
* covered by this vma.
*/
static unsigned long
copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
struct vm_area_struct *src_vma, unsigned long addr, int *rss)
{
unsigned long vm_flags = dst_vma->vm_flags;
pte_t orig_pte = ptep_get(src_pte);
pte_t pte = orig_pte;
struct folio *folio;
struct page *page;
swp_entry_t entry = pte_to_swp_entry(orig_pte);
if (likely(!non_swap_entry(entry))) {
if (swap_duplicate(entry) < 0)
return -EIO;
/* make sure dst_mm is on swapoff's mmlist. */
if (unlikely(list_empty(&dst_mm->mmlist))) {
spin_lock(&mmlist_lock);
if (list_empty(&dst_mm->mmlist))
list_add(&dst_mm->mmlist,
&src_mm->mmlist);
spin_unlock(&mmlist_lock);
}
/* Mark the swap entry as shared. */
if (pte_swp_exclusive(orig_pte)) {
pte = pte_swp_clear_exclusive(orig_pte);
set_pte_at(src_mm, addr, src_pte, pte);
}
rss[MM_SWAPENTS]++;
} else if (is_migration_entry(entry)) {
folio = pfn_swap_entry_folio(entry);
rss[mm_counter(folio)]++;
if (!is_readable_migration_entry(entry) &&
is_cow_mapping(vm_flags)) {
/*
* COW mappings require pages in both parent and child
* to be set to read. A previously exclusive entry is
* now shared.
*/
entry = make_readable_migration_entry(
swp_offset(entry));
pte = swp_entry_to_pte(entry);
if (pte_swp_soft_dirty(orig_pte))
pte = pte_swp_mksoft_dirty(pte);
if (pte_swp_uffd_wp(orig_pte))
pte = pte_swp_mkuffd_wp(pte);
set_pte_at(src_mm, addr, src_pte, pte);
}
} else if (is_device_private_entry(entry)) {
page = pfn_swap_entry_to_page(entry);
folio = page_folio(page);
/*
* Update rss count even for unaddressable pages, as
* they should treated just like normal pages in this
* respect.
*
* We will likely want to have some new rss counters
* for unaddressable pages, at some point. But for now
* keep things as they are.
*/
folio_get(folio);
rss[mm_counter(folio)]++;
/* Cannot fail as these pages cannot get pinned. */
folio_try_dup_anon_rmap_pte(folio, page, src_vma);
/*
* We do not preserve soft-dirty information, because so
* far, checkpoint/restore is the only feature that
* requires that. And checkpoint/restore does not work
* when a device driver is involved (you cannot easily
* save and restore device driver state).
*/
if (is_writable_device_private_entry(entry) &&
is_cow_mapping(vm_flags)) {
entry = make_readable_device_private_entry(
swp_offset(entry));
pte = swp_entry_to_pte(entry);
if (pte_swp_uffd_wp(orig_pte))
pte = pte_swp_mkuffd_wp(pte);
set_pte_at(src_mm, addr, src_pte, pte);
}
} else if (is_device_exclusive_entry(entry)) {
/*
* Make device exclusive entries present by restoring the
* original entry then copying as for a present pte. Device
* exclusive entries currently only support private writable
* (ie. COW) mappings.
*/
VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
if (try_restore_exclusive_pte(src_pte, src_vma, addr))
return -EBUSY;
return -ENOENT;
} else if (is_pte_marker_entry(entry)) {
pte_marker marker = copy_pte_marker(entry, dst_vma);
if (marker)
set_pte_at(dst_mm, addr, dst_pte,
make_pte_marker(marker));
return 0;
}
if (!userfaultfd_wp(dst_vma))
pte = pte_swp_clear_uffd_wp(pte);
set_pte_at(dst_mm, addr, dst_pte, pte);
return 0;
}
/*
* Copy a present and normal page.
*
* NOTE! The usual case is that this isn't required;
* instead, the caller can just increase the page refcount
* and re-use the pte the traditional way.
*
* And if we need a pre-allocated page but don't yet have
* one, return a negative error to let the preallocation
* code know so that it can do so outside the page table
* lock.
*/
static inline int
copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
struct folio **prealloc, struct page *page)
{
struct folio *new_folio;
pte_t pte;
new_folio = *prealloc;
if (!new_folio)
return -EAGAIN;
/*
* We have a prealloc page, all good! Take it
* over and copy the page & arm it.
*/
*prealloc = NULL;
copy_user_highpage(&new_folio->page, page, addr, src_vma);
__folio_mark_uptodate(new_folio);
folio_add_new_anon_rmap(new_folio, dst_vma, addr);
folio_add_lru_vma(new_folio, dst_vma);
rss[MM_ANONPAGES]++;
/* All done, just insert the new page copy in the child */
pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
/* Uffd-wp needs to be delivered to dest pte as well */
pte = pte_mkuffd_wp(pte);
set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
return 0;
}
static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte,
pte_t pte, unsigned long addr, int nr)
{
struct mm_struct *src_mm = src_vma->vm_mm;
/* If it's a COW mapping, write protect it both processes. */
if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) {
wrprotect_ptes(src_mm, addr, src_pte, nr);
pte = pte_wrprotect(pte);
}
/* If it's a shared mapping, mark it clean in the child. */
if (src_vma->vm_flags & VM_SHARED)
pte = pte_mkclean(pte);
pte = pte_mkold(pte);
if (!userfaultfd_wp(dst_vma))
pte = pte_clear_uffd_wp(pte);
set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
}
/*
* Copy one present PTE, trying to batch-process subsequent PTEs that map
* consecutive pages of the same folio by copying them as well.
*
* Returns -EAGAIN if one preallocated page is required to copy the next PTE.
* Otherwise, returns the number of copied PTEs (at least 1).
*/
static inline int
copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr,
int max_nr, int *rss, struct folio **prealloc)
{
struct page *page;
struct folio *folio;
bool any_writable;
fpb_t flags = 0;
int err, nr;
page = vm_normal_page(src_vma, addr, pte);
if (unlikely(!page))
goto copy_pte;
folio = page_folio(page);
/*
* If we likely have to copy, just don't bother with batching. Make
* sure that the common "small folio" case is as fast as possible
* by keeping the batching logic separate.
*/
if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {
if (src_vma->vm_flags & VM_SHARED)
flags |= FPB_IGNORE_DIRTY;
if (!vma_soft_dirty_enabled(src_vma))
flags |= FPB_IGNORE_SOFT_DIRTY;
nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,
&any_writable);
folio_ref_add(folio, nr);
if (folio_test_anon(folio)) {
if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
nr, src_vma))) {
folio_ref_sub(folio, nr);
return -EAGAIN;
}
rss[MM_ANONPAGES] += nr;
VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
} else {
folio_dup_file_rmap_ptes(folio, page, nr);
rss[mm_counter_file(folio)] += nr;
}
if (any_writable)
pte = pte_mkwrite(pte, src_vma);
__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
addr, nr);
return nr;
}
folio_get(folio);
if (folio_test_anon(folio)) {
/*
* If this page may have been pinned by the parent process,
* copy the page immediately for the child so that we'll always
* guarantee the pinned page won't be randomly replaced in the
* future.
*/
if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, src_vma))) {
/* Page may be pinned, we have to copy. */
folio_put(folio);
err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
addr, rss, prealloc, page);
return err ? err : 1;
}
rss[MM_ANONPAGES]++;
VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
} else {
folio_dup_file_rmap_pte(folio, page);
rss[mm_counter_file(folio)]++;
}
copy_pte:
__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
return 1;
}
static inline struct folio *folio_prealloc(struct mm_struct *src_mm,
struct vm_area_struct *vma, unsigned long addr, bool need_zero)
{
struct folio *new_folio;
if (need_zero)
new_folio = vma_alloc_zeroed_movable_folio(vma, addr);
else
new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma,
addr, false);
if (!new_folio)
return NULL;
if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) { folio_put(new_folio);
return NULL;
}
folio_throttle_swaprate(new_folio, GFP_KERNEL);
return new_folio;
}
static int
copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
unsigned long end)
{
struct mm_struct *dst_mm = dst_vma->vm_mm;
struct mm_struct *src_mm = src_vma->vm_mm;
pte_t *orig_src_pte, *orig_dst_pte;
pte_t *src_pte, *dst_pte;
pte_t ptent;
spinlock_t *src_ptl, *dst_ptl;
int progress, max_nr, ret = 0;
int rss[NR_MM_COUNTERS];
swp_entry_t entry = (swp_entry_t){0};
struct folio *prealloc = NULL;
int nr;
again:
progress = 0;
init_rss_vec(rss);
/*
* copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
* error handling here, assume that exclusive mmap_lock on dst and src
* protects anon from unexpected THP transitions; with shmem and file
* protected by mmap_lock-less collapse skipping areas with anon_vma
* (whereas vma_needs_copy() skips areas without anon_vma). A rework
* can remove such assumptions later, but this is good enough for now.
*/
dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
if (!dst_pte) {
ret = -ENOMEM;
goto out;
}
src_pte = pte_offset_map_nolock(src_mm, src_pmd, addr, &src_ptl);
if (!src_pte) {
pte_unmap_unlock(dst_pte, dst_ptl);
/* ret == 0 */
goto out;
}
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
orig_src_pte = src_pte;
orig_dst_pte = dst_pte;
arch_enter_lazy_mmu_mode();
do {
nr = 1;
/*
* We are holding two locks at this point - either of them
* could generate latencies in another task on another CPU.
*/
if (progress >= 32) {
progress = 0;
if (need_resched() ||
spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
break;
}
ptent = ptep_get(src_pte);
if (pte_none(ptent)) {
progress++;
continue;
}
if (unlikely(!pte_present(ptent))) {
ret = copy_nonpresent_pte(dst_mm, src_mm,
dst_pte, src_pte,
dst_vma, src_vma,
addr, rss);
if (ret == -EIO) {
entry = pte_to_swp_entry(ptep_get(src_pte));
break;
} else if (ret == -EBUSY) {
break;
} else if (!ret) {
progress += 8;
continue;
}
ptent = ptep_get(src_pte);
VM_WARN_ON_ONCE(!pte_present(ptent));
/*
* Device exclusive entry restored, continue by copying
* the now present pte.
*/
WARN_ON_ONCE(ret != -ENOENT);
}
/* copy_present_ptes() will clear `*prealloc' if consumed */
max_nr = (end - addr) / PAGE_SIZE;
ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
ptent, addr, max_nr, rss, &prealloc);
/*
* If we need a pre-allocated page for this pte, drop the
* locks, allocate, and try again.
*/
if (unlikely(ret == -EAGAIN))
break;
if (unlikely(prealloc)) {
/*
* pre-alloc page cannot be reused by next time so as
* to strictly follow mempolicy (e.g., alloc_page_vma()
* will allocate page according to address). This
* could only happen if one pinned pte changed.
*/
folio_put(prealloc);
prealloc = NULL;
}
nr = ret;
progress += 8 * nr;
} while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr,
addr != end);
arch_leave_lazy_mmu_mode();
pte_unmap_unlock(orig_src_pte, src_ptl);
add_mm_rss_vec(dst_mm, rss);
pte_unmap_unlock(orig_dst_pte, dst_ptl);
cond_resched();
if (ret == -EIO) {
VM_WARN_ON_ONCE(!entry.val);
if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
ret = -ENOMEM;
goto out;
}
entry.val = 0;
} else if (ret == -EBUSY) {
goto out;
} else if (ret == -EAGAIN) {
prealloc = folio_prealloc(src_mm, src_vma, addr, false);
if (!prealloc)
return -ENOMEM;
} else if (ret < 0) {
VM_WARN_ON_ONCE(1);
}
/* We've captured and resolved the error. Reset, try again. */
ret = 0;
if (addr != end)
goto again;
out:
if (unlikely(prealloc))
folio_put(prealloc);
return ret;
}
static inline int
copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
unsigned long end)
{
struct mm_struct *dst_mm = dst_vma->vm_mm;
struct mm_struct *src_mm = src_vma->vm_mm;
pmd_t *src_pmd, *dst_pmd;
unsigned long next;
dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
if (!dst_pmd)
return -ENOMEM;
src_pmd = pmd_offset(src_pud, addr);
do {
next = pmd_addr_end(addr, end);
if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
|| pmd_devmap(*src_pmd)) {
int err;
VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
addr, dst_vma, src_vma);
if (err == -ENOMEM)
return -ENOMEM;
if (!err)
continue;
/* fall through */
}
if (pmd_none_or_clear_bad(src_pmd))
continue;
if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
addr, next))
return -ENOMEM;
} while (dst_pmd++, src_pmd++, addr = next, addr != end);
return 0;
}
static inline int
copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
unsigned long end)
{
struct mm_struct *dst_mm = dst_vma->vm_mm;
struct mm_struct *src_mm = src_vma->vm_mm;
pud_t *src_pud, *dst_pud;
unsigned long next;
dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
if (!dst_pud)
return -ENOMEM;
src_pud = pud_offset(src_p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
int err;
VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
err = copy_huge_pud(dst_mm, src_mm,
dst_pud, src_pud, addr, src_vma);
if (err == -ENOMEM)
return -ENOMEM;
if (!err)
continue;
/* fall through */
}
if (pud_none_or_clear_bad(src_pud))
continue;
if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
addr, next))
return -ENOMEM;
} while (dst_pud++, src_pud++, addr = next, addr != end);
return 0;
}
static inline int
copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
unsigned long end)
{
struct mm_struct *dst_mm = dst_vma->vm_mm;
p4d_t *src_p4d, *dst_p4d;
unsigned long next;
dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
if (!dst_p4d)
return -ENOMEM;
src_p4d = p4d_offset(src_pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (p4d_none_or_clear_bad(src_p4d))
continue;
if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
addr, next))
return -ENOMEM;
} while (dst_p4d++, src_p4d++, addr = next, addr != end);
return 0;
}
/*
* Return true if the vma needs to copy the pgtable during this fork(). Return
* false when we can speed up fork() by allowing lazy page faults later until
* when the child accesses the memory range.
*/
static bool
vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
/*
* Always copy pgtables when dst_vma has uffd-wp enabled even if it's
* file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
* contains uffd-wp protection information, that's something we can't
* retrieve from page cache, and skip copying will lose those info.
*/
if (userfaultfd_wp(dst_vma))
return true;
if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
return true;
if (src_vma->anon_vma)
return true;
/*
* Don't copy ptes where a page fault will fill them correctly. Fork
* becomes much lighter when there are big shared or private readonly
* mappings. The tradeoff is that copy_page_range is more efficient
* than faulting.
*/
return false;
}
int
copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
pgd_t *src_pgd, *dst_pgd;
unsigned long next;
unsigned long addr = src_vma->vm_start;
unsigned long end = src_vma->vm_end;
struct mm_struct *dst_mm = dst_vma->vm_mm;
struct mm_struct *src_mm = src_vma->vm_mm;
struct mmu_notifier_range range;
bool is_cow;
int ret;
if (!vma_needs_copy(dst_vma, src_vma))
return 0;
if (is_vm_hugetlb_page(src_vma))
return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
/*
* We do not free on error cases below as remove_vma
* gets called on error from higher level routine
*/
ret = track_pfn_copy(src_vma);
if (ret)
return ret;
}
/*
* We need to invalidate the secondary MMU mappings only when
* there could be a permission downgrade on the ptes of the
* parent mm. And a permission downgrade will only happen if
* is_cow_mapping() returns true.
*/
is_cow = is_cow_mapping(src_vma->vm_flags);
if (is_cow) {
mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
0, src_mm, addr, end);
mmu_notifier_invalidate_range_start(&range);
/*
* Disabling preemption is not needed for the write side, as
* the read side doesn't spin, but goes to the mmap_lock.
*
* Use the raw variant of the seqcount_t write API to avoid
* lockdep complaining about preemptibility.
*/
vma_assert_write_locked(src_vma);
raw_write_seqcount_begin(&src_mm->write_protect_seq);
}
ret = 0;
dst_pgd = pgd_offset(dst_mm, addr);
src_pgd = pgd_offset(src_mm, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(src_pgd))
continue;
if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
addr, next))) {
untrack_pfn_clear(dst_vma);
ret = -ENOMEM;
break;
}
} while (dst_pgd++, src_pgd++, addr = next, addr != end);
if (is_cow) {
raw_write_seqcount_end(&src_mm->write_protect_seq);
mmu_notifier_invalidate_range_end(&range);
}
return ret;
}
/* Whether we should zap all COWed (private) pages too */
static inline bool should_zap_cows(struct zap_details *details)
{
/* By default, zap all pages */
if (!details)
return true;
/* Or, we zap COWed pages only if the caller wants to */
return details->even_cows;
}
/* Decides whether we should zap this folio with the folio pointer specified */
static inline bool should_zap_folio(struct zap_details *details,
struct folio *folio)
{
/* If we can make a decision without *folio.. */
if (should_zap_cows(details))
return true;
/* Otherwise we should only zap non-anon folios */
return !folio_test_anon(folio);
}
static inline bool zap_drop_file_uffd_wp(struct zap_details *details)
{
if (!details)
return false;
return details->zap_flags & ZAP_FLAG_DROP_MARKER;
}
/*
* This function makes sure that we'll replace the none pte with an uffd-wp
* swap special pte marker when necessary. Must be with the pgtable lock held.
*/
static inline void
zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
unsigned long addr, pte_t *pte, int nr,
struct zap_details *details, pte_t pteval)
{
/* Zap on anonymous always means dropping everything */
if (vma_is_anonymous(vma))
return;
if (zap_drop_file_uffd_wp(details))
return;
for (;;) {
/* the PFN in the PTE is irrelevant. */
pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
if (--nr == 0)
break;
pte++;
addr += PAGE_SIZE;
}
}
static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
struct vm_area_struct *vma, struct folio *folio,
struct page *page, pte_t *pte, pte_t ptent, unsigned int nr,
unsigned long addr, struct zap_details *details, int *rss,
bool *force_flush, bool *force_break)
{
struct mm_struct *mm = tlb->mm;
bool delay_rmap = false;
if (!folio_test_anon(folio)) {
ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
if (pte_dirty(ptent)) {
folio_mark_dirty(folio);
if (tlb_delay_rmap(tlb)) {
delay_rmap = true;
*force_flush = true;
}
}
if (pte_young(ptent) && likely(vma_has_recency(vma)))
folio_mark_accessed(folio);
rss[mm_counter(folio)] -= nr;
} else {
/* We don't need up-to-date accessed/dirty bits. */
clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
rss[MM_ANONPAGES] -= nr;
}
/* Checking a single PTE in a batch is sufficient. */
arch_check_zapped_pte(vma, ptent);
tlb_remove_tlb_entries(tlb, pte, nr, addr);
if (unlikely(userfaultfd_pte_wp(vma, ptent)))
zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details,
ptent);
if (!delay_rmap) {
folio_remove_rmap_ptes(folio, page, nr, vma);
/* Only sanity-check the first page in a batch. */
if (unlikely(page_mapcount(page) < 0))
print_bad_pte(vma, addr, ptent, page);
}
if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
*force_flush = true;
*force_break = true;
}
}
/*
* Zap or skip at least one present PTE, trying to batch-process subsequent
* PTEs that map consecutive pages of the same folio.
*
* Returns the number of processed (skipped or zapped) PTEs (at least 1).
*/
static inline int zap_present_ptes(struct mmu_gather *tlb,
struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
unsigned int max_nr, unsigned long addr,
struct zap_details *details, int *rss, bool *force_flush,
bool *force_break)
{
const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
struct mm_struct *mm = tlb->mm;
struct folio *folio;
struct page *page;
int nr;
page = vm_normal_page(vma, addr, ptent);
if (!page) {
/* We don't need up-to-date accessed/dirty bits. */
ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);
arch_check_zapped_pte(vma, ptent);
tlb_remove_tlb_entry(tlb, pte, addr);
if (userfaultfd_pte_wp(vma, ptent))
zap_install_uffd_wp_if_needed(vma, addr, pte, 1,
details, ptent);
ksm_might_unmap_zero_page(mm, ptent);
return 1;
}
folio = page_folio(page);
if (unlikely(!should_zap_folio(details, folio)))
return 1;
/*
* Make sure that the common "small folio" case is as fast as possible
* by keeping the batching logic separate.
*/
if (unlikely(folio_test_large(folio) && max_nr != 1)) {
nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags,
NULL);
zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,
addr, details, rss, force_flush,
force_break);
return nr;
}
zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr,
details, rss, force_flush, force_break);
return 1;
}
static unsigned long zap_pte_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end,
struct zap_details *details)
{
bool force_flush = false, force_break = false;
struct mm_struct *mm = tlb->mm;
int rss[NR_MM_COUNTERS];
spinlock_t *ptl;
pte_t *start_pte;
pte_t *pte;
swp_entry_t entry;
int nr;
tlb_change_page_size(tlb, PAGE_SIZE);
init_rss_vec(rss);
start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
if (!pte)
return addr;
flush_tlb_batched_pending(mm);
arch_enter_lazy_mmu_mode();
do {
pte_t ptent = ptep_get(pte);
struct folio *folio;
struct page *page;
int max_nr;
nr = 1;
if (pte_none(ptent))
continue;
if (need_resched())
break;
if (pte_present(ptent)) {
max_nr = (end - addr) / PAGE_SIZE;
nr = zap_present_ptes(tlb, vma, pte, ptent, max_nr,
addr, details, rss, &force_flush,
&force_break);
if (unlikely(force_break)) {
addr += nr * PAGE_SIZE;
break;
}
continue;
}
entry = pte_to_swp_entry(ptent);
if (is_device_private_entry(entry) ||
is_device_exclusive_entry(entry)) {
page = pfn_swap_entry_to_page(entry);
folio = page_folio(page);
if (unlikely(!should_zap_folio(details, folio)))
continue;
/*
* Both device private/exclusive mappings should only
* work with anonymous page so far, so we don't need to
* consider uffd-wp bit when zap. For more information,
* see zap_install_uffd_wp_if_needed().
*/
WARN_ON_ONCE(!vma_is_anonymous(vma));
rss[mm_counter(folio)]--;
if (is_device_private_entry(entry))
folio_remove_rmap_pte(folio, page, vma);
folio_put(folio);
} else if (!non_swap_entry(entry)) {
/* Genuine swap entry, hence a private anon page */
if (!should_zap_cows(details))
continue;
rss[MM_SWAPENTS]--;
if (unlikely(!free_swap_and_cache(entry)))
print_bad_pte(vma, addr, ptent, NULL);
} else if (is_migration_entry(entry)) {
folio = pfn_swap_entry_folio(entry);
if (!should_zap_folio(details, folio))
continue;
rss[mm_counter(folio)]--;
} else if (pte_marker_entry_uffd_wp(entry)) {
/*
* For anon: always drop the marker; for file: only
* drop the marker if explicitly requested.
*/
if (!vma_is_anonymous(vma) &&
!zap_drop_file_uffd_wp(details))
continue;
} else if (is_hwpoison_entry(entry) ||
is_poisoned_swp_entry(entry)) {
if (!should_zap_cows(details))
continue;
} else {
/* We should have covered all the swap entry types */
pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
WARN_ON_ONCE(1);
}
pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
zap_install_uffd_wp_if_needed(vma, addr, pte, 1, details, ptent);
} while (pte += nr, addr += PAGE_SIZE * nr, addr != end);
add_mm_rss_vec(mm, rss);
arch_leave_lazy_mmu_mode();
/* Do the actual TLB flush before dropping ptl */
if (force_flush) {
tlb_flush_mmu_tlbonly(tlb);
tlb_flush_rmaps(tlb, vma);
}
pte_unmap_unlock(start_pte, ptl);
/*
* If we forced a TLB flush (either due to running out of
* batch buffers or because we needed to flush dirty TLB
* entries before releasing the ptl), free the batched
* memory too. Come back again if we didn't do everything.
*/
if (force_flush)
tlb_flush_mmu(tlb);
return addr;
}
static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, pud_t *pud,
unsigned long addr, unsigned long end,
struct zap_details *details)
{
pmd_t *pmd;
unsigned long next;
pmd = pmd_offset(pud, addr);
do {
next = pmd_addr_end(addr, end);
if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
if (next - addr != HPAGE_PMD_SIZE)
__split_huge_pmd(vma, pmd, addr, false, NULL);
else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
addr = next;
continue;
}
/* fall through */
} else if (details && details->single_folio &&
folio_test_pmd_mappable(details->single_folio) &&
next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
/*
* Take and drop THP pmd lock so that we cannot return
* prematurely, while zap_huge_pmd() has cleared *pmd,
* but not yet decremented compound_mapcount().
*/
spin_unlock(ptl);
}
if (pmd_none(*pmd)) {
addr = next;
continue;
}
addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
if (addr != next)
pmd--;
} while (pmd++, cond_resched(), addr != end);
return addr;
}
static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, p4d_t *p4d,
unsigned long addr, unsigned long end,
struct zap_details *details)
{
pud_t *pud;
unsigned long next;
pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
if (next - addr != HPAGE_PUD_SIZE) {
mmap_assert_locked(tlb->mm);
split_huge_pud(vma, pud, addr);
} else if (zap_huge_pud(tlb, vma, pud, addr))
goto next;
/* fall through */
}
if (pud_none_or_clear_bad(pud))
continue;
next = zap_pmd_range(tlb, vma, pud, addr, next, details);
next:
cond_resched();
} while (pud++, addr = next, addr != end);
return addr;
}
static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
struct vm_area_struct *vma, pgd_t *pgd,
unsigned long addr, unsigned long end,
struct zap_details *details)
{
p4d_t *p4d;
unsigned long next;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (p4d_none_or_clear_bad(p4d))
continue;
next = zap_pud_range(tlb, vma, p4d, addr, next, details);
} while (p4d++, addr = next, addr != end);
return addr;
}
void unmap_page_range(struct mmu_gather *tlb,
struct vm_area_struct *vma,
unsigned long addr, unsigned long end,
struct zap_details *details)
{
pgd_t *pgd;
unsigned long next;
BUG_ON(addr >= end);
tlb_start_vma(tlb, vma);
pgd = pgd_offset(vma->vm_mm, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
} while (pgd++, addr = next, addr != end);
tlb_end_vma(tlb, vma);
}
static void unmap_single_vma(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long start_addr,
unsigned long end_addr,
struct zap_details *details, bool mm_wr_locked)
{
unsigned long start = max(vma->vm_start, start_addr);
unsigned long end;
if (start >= vma->vm_end)
return;
end = min(vma->vm_end, end_addr);
if (end <= vma->vm_start)
return;
if (vma->vm_file)
uprobe_munmap(vma, start, end);
if (unlikely(vma->vm_flags & VM_PFNMAP))
untrack_pfn(vma, 0, 0, mm_wr_locked);
if (start != end) {
if (unlikely(is_vm_hugetlb_page(vma))) {
/*
* It is undesirable to test vma->vm_file as it
* should be non-null for valid hugetlb area.
* However, vm_file will be NULL in the error
* cleanup path of mmap_region. When
* hugetlbfs ->mmap method fails,
* mmap_region() nullifies vma->vm_file
* before calling this function to clean up.
* Since no pte has actually been setup, it is
* safe to do nothing in this case.
*/
if (vma->vm_file) {
zap_flags_t zap_flags = details ?
details->zap_flags : 0;
__unmap_hugepage_range(tlb, vma, start, end,
NULL, zap_flags);
}
} else
unmap_page_range(tlb, vma, start, end, details);
}
}
/**
* unmap_vmas - unmap a range of memory covered by a list of vma's
* @tlb: address of the caller's struct mmu_gather
* @mas: the maple state
* @vma: the starting vma
* @start_addr: virtual address at which to start unmapping
* @end_addr: virtual address at which to end unmapping
* @tree_end: The maximum index to check
* @mm_wr_locked: lock flag
*
* Unmap all pages in the vma list.
*
* Only addresses between `start' and `end' will be unmapped.
*
* The VMA list must be sorted in ascending virtual address order.
*
* unmap_vmas() assumes that the caller will flush the whole unmapped address
* range after unmap_vmas() returns. So the only responsibility here is to
* ensure that any thus-far unmapped pages are flushed before unmap_vmas()
* drops the lock and schedules.
*/
void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
struct vm_area_struct *vma, unsigned long start_addr,
unsigned long end_addr, unsigned long tree_end,
bool mm_wr_locked)
{
struct mmu_notifier_range range;
struct zap_details details = {
.zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
/* Careful - we need to zap private pages too! */
.even_cows = true,
};
mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
start_addr, end_addr);
mmu_notifier_invalidate_range_start(&range);
do {
unsigned long start = start_addr;
unsigned long end = end_addr;
hugetlb_zap_begin(vma, &start, &end);
unmap_single_vma(tlb, vma, start, end, &details,
mm_wr_locked);
hugetlb_zap_end(vma, &details);
vma = mas_find(mas, tree_end - 1);
} while (vma && likely(!xa_is_zero(vma)));
mmu_notifier_invalidate_range_end(&range);
}
/**
* zap_page_range_single - remove user pages in a given range
* @vma: vm_area_struct holding the applicable pages
* @address: starting address of pages to zap
* @size: number of bytes to zap
* @details: details of shared cache invalidation
*
* The range must fit into one VMA.
*/
void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
unsigned long size, struct zap_details *details)
{
const unsigned long end = address + size;
struct mmu_notifier_range range;
struct mmu_gather tlb;
lru_add_drain();
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
address, end);
hugetlb_zap_begin(vma, &range.start, &range.end);
tlb_gather_mmu(&tlb, vma->vm_mm);
update_hiwater_rss(vma->vm_mm);
mmu_notifier_invalidate_range_start(&range);
/*
* unmap 'address-end' not 'range.start-range.end' as range
* could have been expanded for hugetlb pmd sharing.
*/
unmap_single_vma(&tlb, vma, address, end, details, false);
mmu_notifier_invalidate_range_end(&range);
tlb_finish_mmu(&tlb);
hugetlb_zap_end(vma, details);
}
/**
* zap_vma_ptes - remove ptes mapping the vma
* @vma: vm_area_struct holding ptes to be zapped
* @address: starting address of pages to zap
* @size: number of bytes to zap
*
* This function only unmaps ptes assigned to VM_PFNMAP vmas.
*
* The entire address range must be fully contained within the vma.
*
*/
void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
unsigned long size)
{
if (!range_in_vma(vma, address, address + size) ||
!(vma->vm_flags & VM_PFNMAP))
return;
zap_page_range_single(vma, address, size, NULL);
}
EXPORT_SYMBOL_GPL(zap_vma_ptes);
static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
p4d = p4d_alloc(mm, pgd, addr);
if (!p4d)
return NULL;
pud = pud_alloc(mm, p4d, addr);
if (!pud)
return NULL;
pmd = pmd_alloc(mm, pud, addr);
if (!pmd)
return NULL;
VM_BUG_ON(pmd_trans_huge(*pmd));
return pmd;
}
pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
spinlock_t **ptl)
{
pmd_t *pmd = walk_to_pmd(mm, addr);
if (!pmd)
return NULL;
return pte_alloc_map_lock(mm, pmd, addr, ptl);
}
static int validate_page_before_insert(struct page *page)
{
struct folio *folio = page_folio(page);
if (folio_test_anon(folio) || folio_test_slab(folio) ||
page_has_type(page))
return -EINVAL;
flush_dcache_folio(folio);
return 0;
}
static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
unsigned long addr, struct page *page, pgprot_t prot)
{
struct folio *folio = page_folio(page);
if (!pte_none(ptep_get(pte)))
return -EBUSY;
/* Ok, finally just insert the thing.. */
folio_get(folio);
inc_mm_counter(vma->vm_mm, mm_counter_file(folio));
folio_add_file_rmap_pte(folio, page, vma);
set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
return 0;
}
/*
* This is the old fallback for page remapping.
*
* For historical reasons, it only allows reserved pages. Only
* old drivers should use this, and they needed to mark their
* pages reserved for the old functions anyway.
*/
static int insert_page(struct vm_area_struct *vma, unsigned long addr,
struct page *page, pgprot_t prot)
{
int retval;
pte_t *pte;
spinlock_t *ptl;
retval = validate_page_before_insert(page);
if (retval)
goto out;
retval = -ENOMEM;
pte = get_locked_pte(vma->vm_mm, addr, &ptl);
if (!pte)
goto out;
retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
pte_unmap_unlock(pte, ptl);
out:
return retval;
}
static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
unsigned long addr, struct page *page, pgprot_t prot)
{
int err;
if (!page_count(page))
return -EINVAL;
err = validate_page_before_insert(page);
if (err)
return err;
return insert_page_into_pte_locked(vma, pte, addr, page, prot);
}
/* insert_pages() amortizes the cost of spinlock operations
* when inserting pages in a loop.
*/
static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
struct page **pages, unsigned long *num, pgprot_t prot)
{
pmd_t *pmd = NULL;
pte_t *start_pte, *pte;
spinlock_t *pte_lock;
struct mm_struct *const mm = vma->vm_mm;
unsigned long curr_page_idx = 0;
unsigned long remaining_pages_total = *num;
unsigned long pages_to_write_in_pmd;
int ret;
more:
ret = -EFAULT;
pmd = walk_to_pmd(mm, addr);
if (!pmd)
goto out;
pages_to_write_in_pmd = min_t(unsigned long,
remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
/* Allocate the PTE if necessary; takes PMD lock once only. */
ret = -ENOMEM;
if (pte_alloc(mm, pmd))
goto out;
while (pages_to_write_in_pmd) {
int pte_idx = 0;
const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
if (!start_pte) {
ret = -EFAULT;
goto out;
}
for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
int err = insert_page_in_batch_locked(vma, pte,
addr, pages[curr_page_idx], prot);
if (unlikely(err)) {
pte_unmap_unlock(start_pte, pte_lock);
ret = err;
remaining_pages_total -= pte_idx;
goto out;
}
addr += PAGE_SIZE;
++curr_page_idx;
}
pte_unmap_unlock(start_pte, pte_lock);
pages_to_write_in_pmd -= batch_size;
remaining_pages_total -= batch_size;
}
if (remaining_pages_total)
goto more;
ret = 0;
out:
*num = remaining_pages_total;
return ret;
}
/**
* vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
* @vma: user vma to map to
* @addr: target start user address of these pages
* @pages: source kernel pages
* @num: in: number of pages to map. out: number of pages that were *not*
* mapped. (0 means all pages were successfully mapped).
*
* Preferred over vm_insert_page() when inserting multiple pages.
*
* In case of error, we may have mapped a subset of the provided
* pages. It is the caller's responsibility to account for this case.
*
* The same restrictions apply as in vm_insert_page().
*/
int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
struct page **pages, unsigned long *num)
{
const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
if (addr < vma->vm_start || end_addr >= vma->vm_end)
return -EFAULT;
if (!(vma->vm_flags & VM_MIXEDMAP)) {
BUG_ON(mmap_read_trylock(vma->vm_mm));
BUG_ON(vma->vm_flags & VM_PFNMAP);
vm_flags_set(vma, VM_MIXEDMAP);
}
/* Defer page refcount checking till we're about to map that page. */
return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_insert_pages);
/**
* vm_insert_page - insert single page into user vma
* @vma: user vma to map to
* @addr: target user address of this page
* @page: source kernel page
*
* This allows drivers to insert individual pages they've allocated
* into a user vma.
*
* The page has to be a nice clean _individual_ kernel allocation.
* If you allocate a compound page, you need to have marked it as
* such (__GFP_COMP), or manually just split the page up yourself
* (see split_page()).
*
* NOTE! Traditionally this was done with "remap_pfn_range()" which
* took an arbitrary page protection parameter. This doesn't allow
* that. Your vma protection will have to be set up correctly, which
* means that if you want a shared writable mapping, you'd better
* ask for a shared writable mapping!
*
* The page does not need to be reserved.
*
* Usually this function is called from f_op->mmap() handler
* under mm->mmap_lock write-lock, so it can change vma->vm_flags.
* Caller must set VM_MIXEDMAP on vma if it wants to call this
* function from other places, for example from page-fault handler.
*
* Return: %0 on success, negative error code otherwise.
*/
int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
struct page *page)
{
if (addr < vma->vm_start || addr >= vma->vm_end)
return -EFAULT;
if (!page_count(page))
return -EINVAL;
if (!(vma->vm_flags & VM_MIXEDMAP)) {
BUG_ON(mmap_read_trylock(vma->vm_mm));
BUG_ON(vma->vm_flags & VM_PFNMAP);
vm_flags_set(vma, VM_MIXEDMAP);
}
return insert_page(vma, addr, page, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_insert_page);
/*
* __vm_map_pages - maps range of kernel pages into user vma
* @vma: user vma to map to
* @pages: pointer to array of source kernel pages
* @num: number of pages in page array
* @offset: user's requested vm_pgoff
*
* This allows drivers to map range of kernel pages into a user vma.
*
* Return: 0 on success and error code otherwise.
*/
static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
unsigned long num, unsigned long offset)
{
unsigned long count = vma_pages(vma);
unsigned long uaddr = vma->vm_start;
int ret, i;
/* Fail if the user requested offset is beyond the end of the object */
if (offset >= num)
return -ENXIO;
/* Fail if the user requested size exceeds available object size */
if (count > num - offset)
return -ENXIO;
for (i = 0; i < count; i++) {
ret = vm_insert_page(vma, uaddr, pages[offset + i]);
if (ret < 0)
return ret;
uaddr += PAGE_SIZE;
}
return 0;
}
/**
* vm_map_pages - maps range of kernel pages starts with non zero offset
* @vma: user vma to map to
* @pages: pointer to array of source kernel pages
* @num: number of pages in page array
*
* Maps an object consisting of @num pages, catering for the user's
* requested vm_pgoff
*
* If we fail to insert any page into the vma, the function will return
* immediately leaving any previously inserted pages present. Callers
* from the mmap handler may immediately return the error as their caller
* will destroy the vma, removing any successfully inserted pages. Other
* callers should make their own arrangements for calling unmap_region().
*
* Context: Process context. Called by mmap handlers.
* Return: 0 on success and error code otherwise.
*/
int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
unsigned long num)
{
return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
}
EXPORT_SYMBOL(vm_map_pages);
/**
* vm_map_pages_zero - map range of kernel pages starts with zero offset
* @vma: user vma to map to
* @pages: pointer to array of source kernel pages
* @num: number of pages in page array
*
* Similar to vm_map_pages(), except that it explicitly sets the offset
* to 0. This function is intended for the drivers that did not consider
* vm_pgoff.
*
* Context: Process context. Called by mmap handlers.
* Return: 0 on success and error code otherwise.
*/
int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
unsigned long num)
{
return __vm_map_pages(vma, pages, num, 0);
}
EXPORT_SYMBOL(vm_map_pages_zero);
static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
pfn_t pfn, pgprot_t prot, bool mkwrite)
{
struct mm_struct *mm = vma->vm_mm;
pte_t *pte, entry;
spinlock_t *ptl;
pte = get_locked_pte(mm, addr, &ptl);
if (!pte)
return VM_FAULT_OOM;
entry = ptep_get(pte);
if (!pte_none(entry)) {
if (mkwrite) {
/*
* For read faults on private mappings the PFN passed
* in may not match the PFN we have mapped if the
* mapped PFN is a writeable COW page. In the mkwrite
* case we are creating a writable PTE for a shared
* mapping and we expect the PFNs to match. If they
* don't match, we are likely racing with block
* allocation and mapping invalidation so just skip the
* update.
*/
if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
goto out_unlock;
}
entry = pte_mkyoung(entry);
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
if (ptep_set_access_flags(vma, addr, pte, entry, 1))
update_mmu_cache(vma, addr, pte);
}
goto out_unlock;
}
/* Ok, finally just insert the thing.. */
if (pfn_t_devmap(pfn))
entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
else
entry = pte_mkspecial(pfn_t_pte(pfn, prot));
if (mkwrite) {
entry = pte_mkyoung(entry);
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
}
set_pte_at(mm, addr, pte, entry);
update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
out_unlock:
pte_unmap_unlock(pte, ptl);
return VM_FAULT_NOPAGE;
}
/**
* vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
* @vma: user vma to map to
* @addr: target user address of this page
* @pfn: source kernel pfn
* @pgprot: pgprot flags for the inserted page
*
* This is exactly like vmf_insert_pfn(), except that it allows drivers
* to override pgprot on a per-page basis.
*
* This only makes sense for IO mappings, and it makes no sense for
* COW mappings. In general, using multiple vmas is preferable;
* vmf_insert_pfn_prot should only be used if using multiple VMAs is
* impractical.
*
* pgprot typically only differs from @vma->vm_page_prot when drivers set
* caching- and encryption bits different than those of @vma->vm_page_prot,
* because the caching- or encryption mode may not be known at mmap() time.
*
* This is ok as long as @vma->vm_page_prot is not used by the core vm
* to set caching and encryption bits for those vmas (except for COW pages).
* This is ensured by core vm only modifying these page table entries using
* functions that don't touch caching- or encryption bits, using pte_modify()
* if needed. (See for example mprotect()).
*
* Also when new page-table entries are created, this is only done using the
* fault() callback, and never using the value of vma->vm_page_prot,
* except for page-table entries that point to anonymous pages as the result
* of COW.
*
* Context: Process context. May allocate using %GFP_KERNEL.
* Return: vm_fault_t value.
*/
vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t pgprot)
{
/*
* Technically, architectures with pte_special can avoid all these
* restrictions (same for remap_pfn_range). However we would like
* consistency in testing and feature parity among all, so we should
* try to keep these invariants in place for everybody.
*/
BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
(VM_PFNMAP|VM_MIXEDMAP));
BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
if (addr < vma->vm_start || addr >= vma->vm_end)
return VM_FAULT_SIGBUS;
if (!pfn_modify_allowed(pfn, pgprot))
return VM_FAULT_SIGBUS;
track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
false);
}
EXPORT_SYMBOL(vmf_insert_pfn_prot);
/**
* vmf_insert_pfn - insert single pfn into user vma
* @vma: user vma to map to
* @addr: target user address of this page
* @pfn: source kernel pfn
*
* Similar to vm_insert_page, this allows drivers to insert individual pages
* they've allocated into a user vma. Same comments apply.
*
* This function should only be called from a vm_ops->fault handler, and
* in that case the handler should return the result of this function.
*
* vma cannot be a COW mapping.
*
* As this is called only for pages that do not currently exist, we
* do not need to flush old virtual caches or the TLB.
*
* Context: Process context. May allocate using %GFP_KERNEL.
* Return: vm_fault_t value.
*/
vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn)
{
return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
}
EXPORT_SYMBOL(vmf_insert_pfn);
static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
{
/* these checks mirror the abort conditions in vm_normal_page */
if (vma->vm_flags & VM_MIXEDMAP)
return true;
if (pfn_t_devmap(pfn))
return true;
if (pfn_t_special(pfn))
return true;
if (is_zero_pfn(pfn_t_to_pfn(pfn)))
return true;
return false;
}
static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
unsigned long addr, pfn_t pfn, bool mkwrite)
{
pgprot_t pgprot = vma->vm_page_prot;
int err;
BUG_ON(!vm_mixed_ok(vma, pfn));
if (addr < vma->vm_start || addr >= vma->vm_end)
return VM_FAULT_SIGBUS;
track_pfn_insert(vma, &pgprot, pfn);
if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
return VM_FAULT_SIGBUS;
/*
* If we don't have pte special, then we have to use the pfn_valid()
* based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
* refcount the page if pfn_valid is true (hence insert_page rather
* than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
* without pte special, it would there be refcounted as a normal page.
*/
if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
!pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
struct page *page;
/*
* At this point we are committed to insert_page()
* regardless of whether the caller specified flags that
* result in pfn_t_has_page() == false.
*/
page = pfn_to_page(pfn_t_to_pfn(pfn));
err = insert_page(vma, addr, page, pgprot);
} else {
return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
}
if (err == -ENOMEM)
return VM_FAULT_OOM;
if (err < 0 && err != -EBUSY)
return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;
}
vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
pfn_t pfn)
{
return __vm_insert_mixed(vma, addr, pfn, false);
}
EXPORT_SYMBOL(vmf_insert_mixed);
/*
* If the insertion of PTE failed because someone else already added a
* different entry in the mean time, we treat that as success as we assume
* the same entry was actually inserted.
*/
vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
unsigned long addr, pfn_t pfn)
{
return __vm_insert_mixed(vma, addr, pfn, true);
}
EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
/*
* maps a range of physical memory into the requested pages. the old
* mappings are removed. any references to nonexistent pages results
* in null mappings (currently treated as "copy-on-access")
*/
static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, unsigned long end,
unsigned long pfn, pgprot_t prot)
{
pte_t *pte, *mapped_pte;
spinlock_t *ptl;
int err = 0;
mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
if (!pte)
return -ENOMEM;
arch_enter_lazy_mmu_mode();
do {
BUG_ON(!pte_none(ptep_get(pte)));
if (!pfn_modify_allowed(pfn, prot)) {
err = -EACCES;
break;
}
set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
pfn++;
} while (pte++, addr += PAGE_SIZE, addr != end);
arch_leave_lazy_mmu_mode();
pte_unmap_unlock(mapped_pte, ptl);
return err;
}
static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
unsigned long addr, unsigned long end,
unsigned long pfn, pgprot_t prot)
{
pmd_t *pmd;
unsigned long next;
int err;
pfn -= addr >> PAGE_SHIFT;
pmd = pmd_alloc(mm, pud, addr);
if (!pmd)
return -ENOMEM;
VM_BUG_ON(pmd_trans_huge(*pmd));
do {
next = pmd_addr_end(addr, end);
err = remap_pte_range(mm, pmd, addr, next,
pfn + (addr >> PAGE_SHIFT), prot);
if (err)
return err;
} while (pmd++, addr = next, addr != end);
return 0;
}
static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned long pfn, pgprot_t prot)
{
pud_t *pud;
unsigned long next;
int err;
pfn -= addr >> PAGE_SHIFT;
pud = pud_alloc(mm, p4d, addr);
if (!pud)
return -ENOMEM;
do {
next = pud_addr_end(addr, end);
err = remap_pmd_range(mm, pud, addr, next,
pfn + (addr >> PAGE_SHIFT), prot);
if (err)
return err;
} while (pud++, addr = next, addr != end);
return 0;
}
static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
unsigned long addr, unsigned long end,
unsigned long pfn, pgprot_t prot)
{
p4d_t *p4d;
unsigned long next;
int err;
pfn -= addr >> PAGE_SHIFT;
p4d = p4d_alloc(mm, pgd, addr);
if (!p4d)
return -ENOMEM;
do {
next = p4d_addr_end(addr, end);
err = remap_pud_range(mm, p4d, addr, next,
pfn + (addr >> PAGE_SHIFT), prot);
if (err)
return err;
} while (p4d++, addr = next, addr != end);
return 0;
}
/*
* Variant of remap_pfn_range that does not call track_pfn_remap. The caller
* must have pre-validated the caching bits of the pgprot_t.
*/
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t prot)
{
pgd_t *pgd;
unsigned long next;
unsigned long end = addr + PAGE_ALIGN(size);
struct mm_struct *mm = vma->vm_mm;
int err;
if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
return -EINVAL;
/*
* Physically remapped pages are special. Tell the
* rest of the world about it:
* VM_IO tells people not to look at these pages
* (accesses can have side effects).
* VM_PFNMAP tells the core MM that the base pages are just
* raw PFN mappings, and do not have a "struct page" associated
* with them.
* VM_DONTEXPAND
* Disable vma merging and expanding with mremap().
* VM_DONTDUMP
* Omit vma from core dump, even when VM_IO turned off.
*
* There's a horrible special case to handle copy-on-write
* behaviour that some programs depend on. We mark the "original"
* un-COW'ed pages by matching them up with "vma->vm_pgoff".
* See vm_normal_page() for details.
*/
if (is_cow_mapping(vma->vm_flags)) {
if (addr != vma->vm_start || end != vma->vm_end)
return -EINVAL;
vma->vm_pgoff = pfn;
}
vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
BUG_ON(addr >= end);
pfn -= addr >> PAGE_SHIFT;
pgd = pgd_offset(mm, addr);
flush_cache_range(vma, addr, end);
do {
next = pgd_addr_end(addr, end);
err = remap_p4d_range(mm, pgd, addr, next,
pfn + (addr >> PAGE_SHIFT), prot);
if (err)
return err;
} while (pgd++, addr = next, addr != end);
return 0;
}
/**
* remap_pfn_range - remap kernel memory to userspace
* @vma: user vma to map to
* @addr: target page aligned user address to start at
* @pfn: page frame number of kernel physical memory address
* @size: size of mapping area
* @prot: page protection flags for this mapping
*
* Note: this is only safe if the mm semaphore is held when called.
*
* Return: %0 on success, negative error code otherwise.
*/
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t prot)
{
int err;
err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
if (err)
return -EINVAL;
err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
if (err)
untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
return err;
}
EXPORT_SYMBOL(remap_pfn_range);
/**
* vm_iomap_memory - remap memory to userspace
* @vma: user vma to map to
* @start: start of the physical memory to be mapped
* @len: size of area
*
* This is a simplified io_remap_pfn_range() for common driver use. The
* driver just needs to give us the physical memory range to be mapped,
* we'll figure out the rest from the vma information.
*
* NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
* whatever write-combining details or similar.
*
* Return: %0 on success, negative error code otherwise.
*/
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
{
unsigned long vm_len, pfn, pages;
/* Check that the physical memory area passed in looks valid */
if (start + len < start)
return -EINVAL;
/*
* You *really* shouldn't map things that aren't page-aligned,
* but we've historically allowed it because IO memory might
* just have smaller alignment.
*/
len += start & ~PAGE_MASK;
pfn = start >> PAGE_SHIFT;
pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
if (pfn + pages < pfn)
return -EINVAL;
/* We start the mapping 'vm_pgoff' pages into the area */
if (vma->vm_pgoff > pages)
return -EINVAL;
pfn += vma->vm_pgoff;
pages -= vma->vm_pgoff;
/* Can we fit all of the mapping? */
vm_len = vma->vm_end - vma->vm_start;
if (vm_len >> PAGE_SHIFT > pages)
return -EINVAL;
/* Ok, let it rip */
return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_iomap_memory);
static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, unsigned long end,
pte_fn_t fn, void *data, bool create,
pgtbl_mod_mask *mask)
{
pte_t *pte, *mapped_pte;
int err = 0;
spinlock_t *ptl;
if (create) {
mapped_pte = pte = (mm == &init_mm) ? pte_alloc_kernel_track(pmd, addr, mask) : pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte)
return -ENOMEM;
} else {
mapped_pte = pte = (mm == &init_mm) ?
pte_offset_kernel(pmd, addr) : pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte)
return -EINVAL;
}
arch_enter_lazy_mmu_mode();
if (fn) {
do {
if (create || !pte_none(ptep_get(pte))) { err = fn(pte++, addr, data);
if (err)
break;
}
} while (addr += PAGE_SIZE, addr != end);
}
*mask |= PGTBL_PTE_MODIFIED;
arch_leave_lazy_mmu_mode();
if (mm != &init_mm) pte_unmap_unlock(mapped_pte, ptl);
return err;
}
static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
unsigned long addr, unsigned long end,
pte_fn_t fn, void *data, bool create,
pgtbl_mod_mask *mask)
{
pmd_t *pmd;
unsigned long next;
int err = 0;
BUG_ON(pud_huge(*pud)); if (create) { pmd = pmd_alloc_track(mm, pud, addr, mask);
if (!pmd)
return -ENOMEM;
} else {
pmd = pmd_offset(pud, addr);
}
do {
next = pmd_addr_end(addr, end); if (pmd_none(*pmd) && !create)
continue;
if (WARN_ON_ONCE(pmd_leaf(*pmd))) return -EINVAL; if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
if (!create)
continue;
pmd_clear_bad(pmd);
}
err = apply_to_pte_range(mm, pmd, addr, next,
fn, data, create, mask);
if (err)
break;
} while (pmd++, addr = next, addr != end);
return err;
}
static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
unsigned long addr, unsigned long end,
pte_fn_t fn, void *data, bool create,
pgtbl_mod_mask *mask)
{
pud_t *pud;
unsigned long next;
int err = 0;
if (create) {
pud = pud_alloc_track(mm, p4d, addr, mask);
if (!pud)
return -ENOMEM;
} else {
pud = pud_offset(p4d, addr);
}
do {
next = pud_addr_end(addr, end); if (pud_none(*pud) && !create)
continue;
if (WARN_ON_ONCE(pud_leaf(*pud)))
return -EINVAL;
if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
if (!create)
continue;
pud_clear_bad(pud);
}
err = apply_to_pmd_range(mm, pud, addr, next,
fn, data, create, mask);
if (err)
break;
} while (pud++, addr = next, addr != end);
return err;
}
static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
unsigned long addr, unsigned long end,
pte_fn_t fn, void *data, bool create,
pgtbl_mod_mask *mask)
{
p4d_t *p4d;
unsigned long next;
int err = 0;
if (create) {
p4d = p4d_alloc_track(mm, pgd, addr, mask);
if (!p4d)
return -ENOMEM;
} else {
p4d = p4d_offset(pgd, addr);
}
do {
next = p4d_addr_end(addr, end);
if (p4d_none(*p4d) && !create)
continue;
if (WARN_ON_ONCE(p4d_leaf(*p4d)))
return -EINVAL;
if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) { if (!create)
continue;
p4d_clear_bad(p4d);
}
err = apply_to_pud_range(mm, p4d, addr, next,
fn, data, create, mask);
if (err)
break;
} while (p4d++, addr = next, addr != end);
return err;
}
static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
unsigned long size, pte_fn_t fn,
void *data, bool create)
{
pgd_t *pgd;
unsigned long start = addr, next;
unsigned long end = addr + size;
pgtbl_mod_mask mask = 0;
int err = 0;
if (WARN_ON(addr >= end))
return -EINVAL;
pgd = pgd_offset(mm, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_none(*pgd) && !create)
continue;
if (WARN_ON_ONCE(pgd_leaf(*pgd)))
return -EINVAL;
if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
if (!create)
continue;
pgd_clear_bad(pgd);
}
err = apply_to_p4d_range(mm, pgd, addr, next,
fn, data, create, &mask);
if (err)
break;
} while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, start + size);
return err;
}
/*
* Scan a region of virtual memory, filling in page tables as necessary
* and calling a provided function on each leaf page table.
*/
int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
unsigned long size, pte_fn_t fn, void *data)
{
return __apply_to_page_range(mm, addr, size, fn, data, true);
}
EXPORT_SYMBOL_GPL(apply_to_page_range);
/*
* Scan a region of virtual memory, calling a provided function on
* each leaf page table where it exists.
*
* Unlike apply_to_page_range, this does _not_ fill in page tables
* where they are absent.
*/
int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
unsigned long size, pte_fn_t fn, void *data)
{
return __apply_to_page_range(mm, addr, size, fn, data, false);
}
EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
/*
* handle_pte_fault chooses page fault handler according to an entry which was
* read non-atomically. Before making any commitment, on those architectures
* or configurations (e.g. i386 with PAE) which might give a mix of unmatched
* parts, do_swap_page must check under lock before unmapping the pte and
* proceeding (but do_wp_page is only called after already making such a check;
* and do_anonymous_page can safely check later on).
*/
static inline int pte_unmap_same(struct vm_fault *vmf)
{
int same = 1;
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
if (sizeof(pte_t) > sizeof(unsigned long)) {
spin_lock(vmf->ptl);
same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
spin_unlock(vmf->ptl);
}
#endif
pte_unmap(vmf->pte);
vmf->pte = NULL;
return same;
}
/*
* Return:
* 0: copied succeeded
* -EHWPOISON: copy failed due to hwpoison in source page
* -EAGAIN: copied failed (some other reason)
*/
static inline int __wp_page_copy_user(struct page *dst, struct page *src,
struct vm_fault *vmf)
{
int ret;
void *kaddr;
void __user *uaddr;
struct vm_area_struct *vma = vmf->vma;
struct mm_struct *mm = vma->vm_mm;
unsigned long addr = vmf->address;
if (likely(src)) {
if (copy_mc_user_highpage(dst, src, addr, vma)) {
memory_failure_queue(page_to_pfn(src), 0);
return -EHWPOISON;
}
return 0;
}
/*
* If the source page was a PFN mapping, we don't have
* a "struct page" for it. We do a best-effort copy by
* just copying from the original user address. If that
* fails, we just zero-fill it. Live with it.
*/
kaddr = kmap_local_page(dst);
pagefault_disable();
uaddr = (void __user *)(addr & PAGE_MASK);
/*
* On architectures with software "accessed" bits, we would
* take a double page fault, so mark it accessed here.
*/
vmf->pte = NULL;
if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
pte_t entry;
vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
/*
* Other thread has already handled the fault
* and update local tlb only
*/
if (vmf->pte)
update_mmu_tlb(vma, addr, vmf->pte);
ret = -EAGAIN;
goto pte_unlock;
}
entry = pte_mkyoung(vmf->orig_pte);
if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
}
/*
* This really shouldn't fail, because the page is there
* in the page tables. But it might just be unreadable,
* in which case we just give up and fill the result with
* zeroes.
*/
if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
if (vmf->pte)
goto warn;
/* Re-validate under PTL if the page is still mapped */
vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
/* The PTE changed under us, update local tlb */
if (vmf->pte)
update_mmu_tlb(vma, addr, vmf->pte);
ret = -EAGAIN;
goto pte_unlock;
}
/*
* The same page can be mapped back since last copy attempt.
* Try to copy again under PTL.
*/
if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
/*
* Give a warn in case there can be some obscure
* use-case
*/
warn:
WARN_ON_ONCE(1);
clear_page(kaddr);
}
}
ret = 0;
pte_unlock:
if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl); pagefault_enable();
kunmap_local(kaddr);
flush_dcache_page(dst);
return ret;
}
static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
{
struct file *vm_file = vma->vm_file;
if (vm_file)
return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
/*
* Special mappings (e.g. VDSO) do not have any file so fake
* a default GFP_KERNEL for them.
*/
return GFP_KERNEL;
}
/*
* Notify the address space that the page is about to become writable so that
* it can prohibit this or wait for the page to get into an appropriate state.
*
* We do this without the lock held, so that it can sleep if it needs to.
*/
static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
{
vm_fault_t ret;
unsigned int old_flags = vmf->flags;
vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
if (vmf->vma->vm_file &&
IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
return VM_FAULT_SIGBUS;
ret = vmf->vma->vm_ops->page_mkwrite(vmf);
/* Restore original flags so that caller is not surprised */
vmf->flags = old_flags;
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
return ret;
if (unlikely(!(ret & VM_FAULT_LOCKED))) {
folio_lock(folio);
if (!folio->mapping) {
folio_unlock(folio);
return 0; /* retry */
}
ret |= VM_FAULT_LOCKED;
} else
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
return ret;
}
/*
* Handle dirtying of a page in shared file mapping on a write fault.
*
* The function expects the page to be locked and unlocks it.
*/
static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct address_space *mapping;
struct folio *folio = page_folio(vmf->page);
bool dirtied;
bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
dirtied = folio_mark_dirty(folio);
VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
/*
* Take a local copy of the address_space - folio.mapping may be zeroed
* by truncate after folio_unlock(). The address_space itself remains
* pinned by vma->vm_file's reference. We rely on folio_unlock()'s
* release semantics to prevent the compiler from undoing this copying.
*/
mapping = folio_raw_mapping(folio);
folio_unlock(folio);
if (!page_mkwrite)
file_update_time(vma->vm_file);
/*
* Throttle page dirtying rate down to writeback speed.
*
* mapping may be NULL here because some device drivers do not
* set page.mapping but still dirty their pages
*
* Drop the mmap_lock before waiting on IO, if we can. The file
* is pinning the mapping, as per above.
*/
if ((dirtied || page_mkwrite) && mapping) {
struct file *fpin;
fpin = maybe_unlock_mmap_for_io(vmf, NULL);
balance_dirty_pages_ratelimited(mapping);
if (fpin) {
fput(fpin);
return VM_FAULT_COMPLETED;
}
}
return 0;
}
/*
* Handle write page faults for pages that can be reused in the current vma
*
* This can happen either due to the mapping being with the VM_SHARED flag,
* or due to us being the last reference standing to the page. In either
* case, all we need to do here is to mark the page as writable and update
* any related book-keeping.
*/
static inline void wp_page_reuse(struct vm_fault *vmf, struct folio *folio)
__releases(vmf->ptl)
{
struct vm_area_struct *vma = vmf->vma;
pte_t entry;
VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
if (folio) {
VM_BUG_ON(folio_test_anon(folio) &&
!PageAnonExclusive(vmf->page));
/*
* Clear the folio's cpupid information as the existing
* information potentially belongs to a now completely
* unrelated process.
*/
folio_xchg_last_cpupid(folio, (1 << LAST_CPUPID_SHIFT) - 1);
}
flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
entry = pte_mkyoung(vmf->orig_pte);
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
pte_unmap_unlock(vmf->pte, vmf->ptl);
count_vm_event(PGREUSE);
}
/*
* We could add a bitflag somewhere, but for now, we know that all
* vm_ops that have a ->map_pages have been audited and don't need
* the mmap_lock to be held.
*/
static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma; if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK))
return 0;
vma_end_read(vma);
return VM_FAULT_RETRY;
}
vm_fault_t vmf_anon_prepare(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma; if (likely(vma->anon_vma))
return 0;
if (vmf->flags & FAULT_FLAG_VMA_LOCK) { vma_end_read(vma);
return VM_FAULT_RETRY;
}
if (__anon_vma_prepare(vma))
return VM_FAULT_OOM;
return 0;
}
/*
* Handle the case of a page which we actually need to copy to a new page,
* either due to COW or unsharing.
*
* Called with mmap_lock locked and the old page referenced, but
* without the ptl held.
*
* High level logic flow:
*
* - Allocate a page, copy the content of the old page to the new one.
* - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
* - Take the PTL. If the pte changed, bail out and release the allocated page
* - If the pte is still the way we remember it, update the page table and all
* relevant references. This includes dropping the reference the page-table
* held to the old page, as well as updating the rmap.
* - In any case, unlock the PTL and drop the reference we took to the old page.
*/
static vm_fault_t wp_page_copy(struct vm_fault *vmf)
{
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
struct vm_area_struct *vma = vmf->vma;
struct mm_struct *mm = vma->vm_mm;
struct folio *old_folio = NULL;
struct folio *new_folio = NULL;
pte_t entry;
int page_copied = 0;
struct mmu_notifier_range range;
vm_fault_t ret;
bool pfn_is_zero;
delayacct_wpcopy_start(); if (vmf->page) old_folio = page_folio(vmf->page); ret = vmf_anon_prepare(vmf);
if (unlikely(ret))
goto out;
pfn_is_zero = is_zero_pfn(pte_pfn(vmf->orig_pte)); new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero);
if (!new_folio)
goto oom;
if (!pfn_is_zero) {
int err;
err = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
if (err) {
/*
* COW failed, if the fault was solved by other,
* it's fine. If not, userspace would re-fault on
* the same address and we will handle the fault
* from the second attempt.
* The -EHWPOISON case will not be retried.
*/
folio_put(new_folio);
if (old_folio)
folio_put(old_folio); delayacct_wpcopy_end();
return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
}
kmsan_copy_page_meta(&new_folio->page, vmf->page);
}
__folio_mark_uptodate(new_folio);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
vmf->address & PAGE_MASK,
(vmf->address & PAGE_MASK) + PAGE_SIZE);
mmu_notifier_invalidate_range_start(&range);
/*
* Re-check the pte - we dropped the lock
*/
vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { if (old_folio) { if (!folio_test_anon(old_folio)) { dec_mm_counter(mm, mm_counter_file(old_folio));
inc_mm_counter(mm, MM_ANONPAGES);
}
} else {
ksm_might_unmap_zero_page(mm, vmf->orig_pte);
inc_mm_counter(mm, MM_ANONPAGES);
}
flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
entry = mk_pte(&new_folio->page, vma->vm_page_prot);
entry = pte_sw_mkyoung(entry);
if (unlikely(unshare)) {
if (pte_soft_dirty(vmf->orig_pte))
entry = pte_mksoft_dirty(entry);
if (pte_uffd_wp(vmf->orig_pte))
entry = pte_mkuffd_wp(entry);
} else {
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
}
/*
* Clear the pte entry and flush it first, before updating the
* pte with the new entry, to keep TLBs on different CPUs in
* sync. This code used to set the new PTE then flush TLBs, but
* that left a window where the new PTE could be loaded into
* some TLBs while the old PTE remains in others.
*/
ptep_clear_flush(vma, vmf->address, vmf->pte);
folio_add_new_anon_rmap(new_folio, vma, vmf->address);
folio_add_lru_vma(new_folio, vma);
/*
* We call the notify macro here because, when using secondary
* mmu page tables (such as kvm shadow page tables), we want the
* new page to be mapped directly into the secondary page table.
*/
BUG_ON(unshare && pte_write(entry)); set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
if (old_folio) {
/*
* Only after switching the pte to the new page may
* we remove the mapcount here. Otherwise another
* process may come and find the rmap count decremented
* before the pte is switched to the new page, and
* "reuse" the old page writing into it while our pte
* here still points into it and can be read by other
* threads.
*
* The critical issue is to order this
* folio_remove_rmap_pte() with the ptp_clear_flush
* above. Those stores are ordered by (if nothing else,)
* the barrier present in the atomic_add_negative
* in folio_remove_rmap_pte();
*
* Then the TLB flush in ptep_clear_flush ensures that
* no process can access the old page before the
* decremented mapcount is visible. And the old page
* cannot be reused until after the decremented
* mapcount is visible. So transitively, TLBs to
* old page will be flushed before it can be reused.
*/
folio_remove_rmap_pte(old_folio, vmf->page, vma);
}
/* Free the old page.. */
new_folio = old_folio;
page_copied = 1;
pte_unmap_unlock(vmf->pte, vmf->ptl);
} else if (vmf->pte) {
update_mmu_tlb(vma, vmf->address, vmf->pte);
pte_unmap_unlock(vmf->pte, vmf->ptl);
}
mmu_notifier_invalidate_range_end(&range); if (new_folio) folio_put(new_folio); if (old_folio) { if (page_copied) free_swap_cache(old_folio); folio_put(old_folio);
}
delayacct_wpcopy_end(); return 0;
oom:
ret = VM_FAULT_OOM;
out:
if (old_folio) folio_put(old_folio); delayacct_wpcopy_end();
return ret;
}
/**
* finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
* writeable once the page is prepared
*
* @vmf: structure describing the fault
* @folio: the folio of vmf->page
*
* This function handles all that is needed to finish a write page fault in a
* shared mapping due to PTE being read-only once the mapped page is prepared.
* It handles locking of PTE and modifying it.
*
* The function expects the page to be locked or other protection against
* concurrent faults / writeback (such as DAX radix tree locks).
*
* Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
* we acquired PTE lock.
*/
static vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf, struct folio *folio)
{
WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
&vmf->ptl);
if (!vmf->pte)
return VM_FAULT_NOPAGE;
/*
* We might have raced with another page fault while we released the
* pte_offset_map_lock.
*/
if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
pte_unmap_unlock(vmf->pte, vmf->ptl);
return VM_FAULT_NOPAGE;
}
wp_page_reuse(vmf, folio);
return 0;
}
/*
* Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
* mapping
*/
static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma; if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
vm_fault_t ret;
pte_unmap_unlock(vmf->pte, vmf->ptl); ret = vmf_can_call_fault(vmf);
if (ret)
return ret;
vmf->flags |= FAULT_FLAG_MKWRITE;
ret = vma->vm_ops->pfn_mkwrite(vmf);
if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
return ret;
return finish_mkwrite_fault(vmf, NULL);
}
wp_page_reuse(vmf, NULL);
return 0;
}
static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
__releases(vmf->ptl)
{
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret = 0;
folio_get(folio); if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
vm_fault_t tmp;
pte_unmap_unlock(vmf->pte, vmf->ptl); tmp = vmf_can_call_fault(vmf);
if (tmp) {
folio_put(folio);
return tmp;
}
tmp = do_page_mkwrite(vmf, folio); if (unlikely(!tmp || (tmp &
(VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
folio_put(folio);
return tmp;
}
tmp = finish_mkwrite_fault(vmf, folio);
if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
folio_unlock(folio);
folio_put(folio);
return tmp;
}
} else {
wp_page_reuse(vmf, folio); folio_lock(folio);
}
ret |= fault_dirty_shared_page(vmf); folio_put(folio);
return ret;
}
static bool wp_can_reuse_anon_folio(struct folio *folio,
struct vm_area_struct *vma)
{
/*
* We could currently only reuse a subpage of a large folio if no
* other subpages of the large folios are still mapped. However,
* let's just consistently not reuse subpages even if we could
* reuse in that scenario, and give back a large folio a bit
* sooner.
*/
if (folio_test_large(folio))
return false;
/*
* We have to verify under folio lock: these early checks are
* just an optimization to avoid locking the folio and freeing
* the swapcache if there is little hope that we can reuse.
*
* KSM doesn't necessarily raise the folio refcount.
*/
if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
return false;
if (!folio_test_lru(folio))
/*
* We cannot easily detect+handle references from
* remote LRU caches or references to LRU folios.
*/
lru_add_drain(); if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
return false;
if (!folio_trylock(folio))
return false;
if (folio_test_swapcache(folio)) folio_free_swap(folio); if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) { folio_unlock(folio);
return false;
}
/*
* Ok, we've got the only folio reference from our mapping
* and the folio is locked, it's dark out, and we're wearing
* sunglasses. Hit it.
*/
folio_move_anon_rmap(folio, vma);
folio_unlock(folio);
return true;
}
/*
* This routine handles present pages, when
* * users try to write to a shared page (FAULT_FLAG_WRITE)
* * GUP wants to take a R/O pin on a possibly shared anonymous page
* (FAULT_FLAG_UNSHARE)
*
* It is done by copying the page to a new address and decrementing the
* shared-page counter for the old page.
*
* Note that this routine assumes that the protection checks have been
* done by the caller (the low-level page fault routine in most cases).
* Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
* done any necessary COW.
*
* In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
* though the page will change only once the write actually happens. This
* avoids a few races, and potentially makes it more efficient.
*
* We enter with non-exclusive mmap_lock (to exclude vma changes,
* but allow concurrent faults), with pte both mapped and locked.
* We return with mmap_lock still held, but pte unmapped and unlocked.
*/
static vm_fault_t do_wp_page(struct vm_fault *vmf)
__releases(vmf->ptl)
{
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
struct vm_area_struct *vma = vmf->vma;
struct folio *folio = NULL;
pte_t pte;
if (likely(!unshare)) {
if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
if (!userfaultfd_wp_async(vma)) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
return handle_userfault(vmf, VM_UFFD_WP);
}
/*
* Nothing needed (cache flush, TLB invalidations,
* etc.) because we're only removing the uffd-wp bit,
* which is completely invisible to the user.
*/
pte = pte_clear_uffd_wp(ptep_get(vmf->pte));
set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
/*
* Update this to be prepared for following up CoW
* handling
*/
vmf->orig_pte = pte;
}
/*
* Userfaultfd write-protect can defer flushes. Ensure the TLB
* is flushed in this case before copying.
*/
if (unlikely(userfaultfd_wp(vmf->vma) &&
mm_tlb_flush_pending(vmf->vma->vm_mm)))
flush_tlb_page(vmf->vma, vmf->address);
}
vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
if (vmf->page)
folio = page_folio(vmf->page);
/*
* Shared mapping: we are guaranteed to have VM_WRITE and
* FAULT_FLAG_WRITE set at this point.
*/
if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
/*
* VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
* VM_PFNMAP VMA.
*
* We should not cow pages in a shared writeable mapping.
* Just mark the pages writable and/or call ops->pfn_mkwrite.
*/
if (!vmf->page) return wp_pfn_shared(vmf); return wp_page_shared(vmf, folio);
}
/*
* Private mapping: create an exclusive anonymous page copy if reuse
* is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
*
* If we encounter a page that is marked exclusive, we must reuse
* the page without further checks.
*/
if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { if (!PageAnonExclusive(vmf->page)) SetPageAnonExclusive(vmf->page); if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl);
return 0;
}
wp_page_reuse(vmf, folio);
return 0;
}
/*
* Ok, we need to copy. Oh, well..
*/
if (folio)
folio_get(folio); pte_unmap_unlock(vmf->pte, vmf->ptl);
#ifdef CONFIG_KSM
if (folio && folio_test_ksm(folio))
count_vm_event(COW_KSM);
#endif
return wp_page_copy(vmf);
}
static void unmap_mapping_range_vma(struct vm_area_struct *vma,
unsigned long start_addr, unsigned long end_addr,
struct zap_details *details)
{
zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
}
static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
pgoff_t first_index,
pgoff_t last_index,
struct zap_details *details)
{
struct vm_area_struct *vma;
pgoff_t vba, vea, zba, zea;
vma_interval_tree_foreach(vma, root, first_index, last_index) {
vba = vma->vm_pgoff;
vea = vba + vma_pages(vma) - 1;
zba = max(first_index, vba);
zea = min(last_index, vea);
unmap_mapping_range_vma(vma,
((zba - vba) << PAGE_SHIFT) + vma->vm_start,
((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
details);
}
}
/**
* unmap_mapping_folio() - Unmap single folio from processes.
* @folio: The locked folio to be unmapped.
*
* Unmap this folio from any userspace process which still has it mmaped.
* Typically, for efficiency, the range of nearby pages has already been
* unmapped by unmap_mapping_pages() or unmap_mapping_range(). But once
* truncation or invalidation holds the lock on a folio, it may find that
* the page has been remapped again: and then uses unmap_mapping_folio()
* to unmap it finally.
*/
void unmap_mapping_folio(struct folio *folio)
{
struct address_space *mapping = folio->mapping;
struct zap_details details = { };
pgoff_t first_index;
pgoff_t last_index;
VM_BUG_ON(!folio_test_locked(folio));
first_index = folio->index;
last_index = folio_next_index(folio) - 1;
details.even_cows = false;
details.single_folio = folio;
details.zap_flags = ZAP_FLAG_DROP_MARKER;
i_mmap_lock_read(mapping);
if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
unmap_mapping_range_tree(&mapping->i_mmap, first_index,
last_index, &details);
i_mmap_unlock_read(mapping);
}
/**
* unmap_mapping_pages() - Unmap pages from processes.
* @mapping: The address space containing pages to be unmapped.
* @start: Index of first page to be unmapped.
* @nr: Number of pages to be unmapped. 0 to unmap to end of file.
* @even_cows: Whether to unmap even private COWed pages.
*
* Unmap the pages in this address space from any userspace process which
* has them mmaped. Generally, you want to remove COWed pages as well when
* a file is being truncated, but not when invalidating pages from the page
* cache.
*/
void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
pgoff_t nr, bool even_cows)
{
struct zap_details details = { };
pgoff_t first_index = start;
pgoff_t last_index = start + nr - 1;
details.even_cows = even_cows;
if (last_index < first_index)
last_index = ULONG_MAX;
i_mmap_lock_read(mapping);
if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
unmap_mapping_range_tree(&mapping->i_mmap, first_index,
last_index, &details);
i_mmap_unlock_read(mapping);
}
EXPORT_SYMBOL_GPL(unmap_mapping_pages);
/**
* unmap_mapping_range - unmap the portion of all mmaps in the specified
* address_space corresponding to the specified byte range in the underlying
* file.
*
* @mapping: the address space containing mmaps to be unmapped.
* @holebegin: byte in first page to unmap, relative to the start of
* the underlying file. This will be rounded down to a PAGE_SIZE
* boundary. Note that this is different from truncate_pagecache(), which
* must keep the partial page. In contrast, we must get rid of
* partial pages.
* @holelen: size of prospective hole in bytes. This will be rounded
* up to a PAGE_SIZE boundary. A holelen of zero truncates to the
* end of the file.
* @even_cows: 1 when truncating a file, unmap even private COWed pages;
* but 0 when invalidating pagecache, don't throw away private data.
*/
void unmap_mapping_range(struct address_space *mapping,
loff_t const holebegin, loff_t const holelen, int even_cows)
{
pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
/* Check for overflow. */
if (sizeof(holelen) > sizeof(hlen)) {
long long holeend =
(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (holeend & ~(long long)ULONG_MAX)
hlen = ULONG_MAX - hba + 1;
}
unmap_mapping_pages(mapping, hba, hlen, even_cows);
}
EXPORT_SYMBOL(unmap_mapping_range);
/*
* Restore a potential device exclusive pte to a working pte entry
*/
static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
{
struct folio *folio = page_folio(vmf->page);
struct vm_area_struct *vma = vmf->vma;
struct mmu_notifier_range range;
vm_fault_t ret;
/*
* We need a reference to lock the folio because we don't hold
* the PTL so a racing thread can remove the device-exclusive
* entry and unmap it. If the folio is free the entry must
* have been removed already. If it happens to have already
* been re-allocated after being freed all we do is lock and
* unlock it.
*/
if (!folio_try_get(folio))
return 0;
ret = folio_lock_or_retry(folio, vmf);
if (ret) {
folio_put(folio);
return ret;
}
mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
vma->vm_mm, vmf->address & PAGE_MASK,
(vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
mmu_notifier_invalidate_range_start(&range);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
&vmf->ptl);
if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
if (vmf->pte)
pte_unmap_unlock(vmf->pte, vmf->ptl);
folio_unlock(folio);
folio_put(folio);
mmu_notifier_invalidate_range_end(&range);
return 0;
}
static inline bool should_try_to_free_swap(struct folio *folio,
struct vm_area_struct *vma,
unsigned int fault_flags)
{
if (!folio_test_swapcache(folio))
return false;
if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
folio_test_mlocked(folio))
return true;
/*
* If we want to map a page that's in the swapcache writable, we
* have to detect via the refcount if we're really the exclusive
* user. Try freeing the swapcache to get rid of the swapcache
* reference only in case it's likely that we'll be the exlusive user.
*/
return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
folio_ref_count(folio) == 2;
}
static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
{
vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (!vmf->pte)
return 0;
/*
* Be careful so that we will only recover a special uffd-wp pte into a
* none pte. Otherwise it means the pte could have changed, so retry.
*
* This should also cover the case where e.g. the pte changed
* quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
* So is_pte_marker() check is not enough to safely drop the pte.
*/
if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
pte_unmap_unlock(vmf->pte, vmf->ptl);
return 0;
}
static vm_fault_t do_pte_missing(struct vm_fault *vmf)
{
if (vma_is_anonymous(vmf->vma)) return do_anonymous_page(vmf);
else
return do_fault(vmf);
}
/*
* This is actually a page-missing access, but with uffd-wp special pte
* installed. It means this pte was wr-protected before being unmapped.
*/
static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
{
/*
* Just in case there're leftover special ptes even after the region
* got unregistered - we can simply clear them.
*/
if (unlikely(!userfaultfd_wp(vmf->vma)))
return pte_marker_clear(vmf);
return do_pte_missing(vmf);
}
static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
{
swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
unsigned long marker = pte_marker_get(entry);
/*
* PTE markers should never be empty. If anything weird happened,
* the best thing to do is to kill the process along with its mm.
*/
if (WARN_ON_ONCE(!marker))
return VM_FAULT_SIGBUS;
/* Higher priority than uffd-wp when data corrupted */
if (marker & PTE_MARKER_POISONED)
return VM_FAULT_HWPOISON;
if (pte_marker_entry_uffd_wp(entry))
return pte_marker_handle_uffd_wp(vmf);
/* This is an unknown pte marker */
return VM_FAULT_SIGBUS;
}
/*
* We enter with non-exclusive mmap_lock (to exclude vma changes,
* but allow concurrent faults), and pte mapped but not yet locked.
* We return with pte unmapped and unlocked.
*
* We return with the mmap_lock locked or unlocked in the same cases
* as does filemap_fault().
*/
vm_fault_t do_swap_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct folio *swapcache, *folio = NULL;
struct page *page;
struct swap_info_struct *si = NULL;
rmap_t rmap_flags = RMAP_NONE;
bool need_clear_cache = false;
bool exclusive = false;
swp_entry_t entry;
pte_t pte;
vm_fault_t ret = 0;
void *shadow = NULL;
if (!pte_unmap_same(vmf))
goto out;
entry = pte_to_swp_entry(vmf->orig_pte);
if (unlikely(non_swap_entry(entry))) {
if (is_migration_entry(entry)) {
migration_entry_wait(vma->vm_mm, vmf->pmd,
vmf->address);
} else if (is_device_exclusive_entry(entry)) {
vmf->page = pfn_swap_entry_to_page(entry);
ret = remove_device_exclusive_entry(vmf);
} else if (is_device_private_entry(entry)) {
if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
/*
* migrate_to_ram is not yet ready to operate
* under VMA lock.
*/
vma_end_read(vma);
ret = VM_FAULT_RETRY;
goto out;
}
vmf->page = pfn_swap_entry_to_page(entry);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (unlikely(!vmf->pte ||
!pte_same(ptep_get(vmf->pte),
vmf->orig_pte)))
goto unlock;
/*
* Get a page reference while we know the page can't be
* freed.
*/
get_page(vmf->page);
pte_unmap_unlock(vmf->pte, vmf->ptl);
ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
put_page(vmf->page);
} else if (is_hwpoison_entry(entry)) {
ret = VM_FAULT_HWPOISON;
} else if (is_pte_marker_entry(entry)) {
ret = handle_pte_marker(vmf);
} else {
print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
ret = VM_FAULT_SIGBUS;
}
goto out;
}
/* Prevent swapoff from happening to us. */
si = get_swap_device(entry);
if (unlikely(!si))
goto out;
folio = swap_cache_get_folio(entry, vma, vmf->address);
if (folio)
page = folio_file_page(folio, swp_offset(entry));
swapcache = folio;
if (!folio) {
if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
__swap_count(entry) == 1) {
/*
* Prevent parallel swapin from proceeding with
* the cache flag. Otherwise, another thread may
* finish swapin first, free the entry, and swapout
* reusing the same entry. It's undetectable as
* pte_same() returns true due to entry reuse.
*/
if (swapcache_prepare(entry)) {
/* Relax a bit to prevent rapid repeated page faults */
schedule_timeout_uninterruptible(1);
goto out;
}
need_clear_cache = true;
/* skip swapcache */
folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0,
vma, vmf->address, false);
page = &folio->page;
if (folio) {
__folio_set_locked(folio);
__folio_set_swapbacked(folio);
if (mem_cgroup_swapin_charge_folio(folio,
vma->vm_mm, GFP_KERNEL,
entry)) {
ret = VM_FAULT_OOM;
goto out_page;
}
mem_cgroup_swapin_uncharge_swap(entry);
shadow = get_shadow_from_swap_cache(entry);
if (shadow)
workingset_refault(folio, shadow);
folio_add_lru(folio);
/* To provide entry to swap_read_folio() */
folio->swap = entry;
swap_read_folio(folio, true, NULL);
folio->private = NULL;
}
} else {
page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
vmf);
if (page)
folio = page_folio(page);
swapcache = folio;
}
if (!folio) {
/*
* Back out if somebody else faulted in this pte
* while we released the pte lock.
*/
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (likely(vmf->pte &&
pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
ret = VM_FAULT_OOM;
goto unlock;
}
/* Had to read the page from swap area: Major fault */
ret = VM_FAULT_MAJOR;
count_vm_event(PGMAJFAULT);
count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
} else if (PageHWPoison(page)) {
/*
* hwpoisoned dirty swapcache pages are kept for killing
* owner processes (which may be unknown at hwpoison time)
*/
ret = VM_FAULT_HWPOISON;
goto out_release;
}
ret |= folio_lock_or_retry(folio, vmf);
if (ret & VM_FAULT_RETRY)
goto out_release;
if (swapcache) {
/*
* Make sure folio_free_swap() or swapoff did not release the
* swapcache from under us. The page pin, and pte_same test
* below, are not enough to exclude that. Even if it is still
* swapcache, we need to check that the page's swap has not
* changed.
*/
if (unlikely(!folio_test_swapcache(folio) ||
page_swap_entry(page).val != entry.val))
goto out_page;
/*
* KSM sometimes has to copy on read faults, for example, if
* page->index of !PageKSM() pages would be nonlinear inside the
* anon VMA -- PageKSM() is lost on actual swapout.
*/
folio = ksm_might_need_to_copy(folio, vma, vmf->address);
if (unlikely(!folio)) {
ret = VM_FAULT_OOM;
folio = swapcache;
goto out_page;
} else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
ret = VM_FAULT_HWPOISON;
folio = swapcache;
goto out_page;
}
if (folio != swapcache)
page = folio_page(folio, 0);
/*
* If we want to map a page that's in the swapcache writable, we
* have to detect via the refcount if we're really the exclusive
* owner. Try removing the extra reference from the local LRU
* caches if required.
*/
if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
!folio_test_ksm(folio) && !folio_test_lru(folio))
lru_add_drain();
}
folio_throttle_swaprate(folio, GFP_KERNEL);
/*
* Back out if somebody else already faulted in this pte.
*/
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
&vmf->ptl);
if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
goto out_nomap;
if (unlikely(!folio_test_uptodate(folio))) {
ret = VM_FAULT_SIGBUS;
goto out_nomap;
}
/*
* PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
* must never point at an anonymous page in the swapcache that is
* PG_anon_exclusive. Sanity check that this holds and especially, that
* no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
* check after taking the PT lock and making sure that nobody
* concurrently faulted in this page and set PG_anon_exclusive.
*/
BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
/*
* Check under PT lock (to protect against concurrent fork() sharing
* the swap entry concurrently) for certainly exclusive pages.
*/
if (!folio_test_ksm(folio)) {
exclusive = pte_swp_exclusive(vmf->orig_pte);
if (folio != swapcache) {
/*
* We have a fresh page that is not exposed to the
* swapcache -> certainly exclusive.
*/
exclusive = true;
} else if (exclusive && folio_test_writeback(folio) &&
data_race(si->flags & SWP_STABLE_WRITES)) {
/*
* This is tricky: not all swap backends support
* concurrent page modifications while under writeback.
*
* So if we stumble over such a page in the swapcache
* we must not set the page exclusive, otherwise we can
* map it writable without further checks and modify it
* while still under writeback.
*
* For these problematic swap backends, simply drop the
* exclusive marker: this is perfectly fine as we start
* writeback only if we fully unmapped the page and
* there are no unexpected references on the page after
* unmapping succeeded. After fully unmapped, no
* further GUP references (FOLL_GET and FOLL_PIN) can
* appear, so dropping the exclusive marker and mapping
* it only R/O is fine.
*/
exclusive = false;
}
}
/*
* Some architectures may have to restore extra metadata to the page
* when reading from swap. This metadata may be indexed by swap entry
* so this must be called before swap_free().
*/
arch_swap_restore(entry, folio);
/*
* Remove the swap entry and conditionally try to free up the swapcache.
* We're already holding a reference on the page but haven't mapped it
* yet.
*/
swap_free(entry);
if (should_try_to_free_swap(folio, vma, vmf->flags))
folio_free_swap(folio);
inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
pte = mk_pte(page, vma->vm_page_prot);
/*
* Same logic as in do_wp_page(); however, optimize for pages that are
* certainly not shared either because we just allocated them without
* exposing them to the swapcache or because the swap entry indicates
* exclusivity.
*/
if (!folio_test_ksm(folio) &&
(exclusive || folio_ref_count(folio) == 1)) {
if (vmf->flags & FAULT_FLAG_WRITE) {
pte = maybe_mkwrite(pte_mkdirty(pte), vma);
vmf->flags &= ~FAULT_FLAG_WRITE;
}
rmap_flags |= RMAP_EXCLUSIVE;
}
flush_icache_page(vma, page);
if (pte_swp_soft_dirty(vmf->orig_pte))
pte = pte_mksoft_dirty(pte);
if (pte_swp_uffd_wp(vmf->orig_pte))
pte = pte_mkuffd_wp(pte);
vmf->orig_pte = pte;
/* ksm created a completely new copy */
if (unlikely(folio != swapcache && swapcache)) {
folio_add_new_anon_rmap(folio, vma, vmf->address);
folio_add_lru_vma(folio, vma);
} else {
folio_add_anon_rmap_pte(folio, page, vma, vmf->address,
rmap_flags);
}
VM_BUG_ON(!folio_test_anon(folio) ||
(pte_write(pte) && !PageAnonExclusive(page)));
set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
folio_unlock(folio);
if (folio != swapcache && swapcache) {
/*
* Hold the lock to avoid the swap entry to be reused
* until we take the PT lock for the pte_same() check
* (to avoid false positives from pte_same). For
* further safety release the lock after the swap_free
* so that the swap count won't change under a
* parallel locked swapcache.
*/
folio_unlock(swapcache);
folio_put(swapcache);
}
if (vmf->flags & FAULT_FLAG_WRITE) {
ret |= do_wp_page(vmf);
if (ret & VM_FAULT_ERROR)
ret &= VM_FAULT_ERROR;
goto out;
}
/* No need to invalidate - it was non-present before */
update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
unlock:
if (vmf->pte)
pte_unmap_unlock(vmf->pte, vmf->ptl);
out:
/* Clear the swap cache pin for direct swapin after PTL unlock */
if (need_clear_cache)
swapcache_clear(si, entry);
if (si)
put_swap_device(si);
return ret;
out_nomap:
if (vmf->pte)
pte_unmap_unlock(vmf->pte, vmf->ptl);
out_page:
folio_unlock(folio);
out_release:
folio_put(folio);
if (folio != swapcache && swapcache) {
folio_unlock(swapcache);
folio_put(swapcache);
}
if (need_clear_cache)
swapcache_clear(si, entry);
if (si)
put_swap_device(si);
return ret;
}
static bool pte_range_none(pte_t *pte, int nr_pages)
{
int i;
for (i = 0; i < nr_pages; i++) { if (!pte_none(ptep_get_lockless(pte + i)))
return false;
}
return true;
}
static struct folio *alloc_anon_folio(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
unsigned long orders;
struct folio *folio;
unsigned long addr;
pte_t *pte;
gfp_t gfp;
int order;
/*
* If uffd is active for the vma we need per-page fault fidelity to
* maintain the uffd semantics.
*/
if (unlikely(userfaultfd_armed(vma)))
goto fallback;
/*
* Get a list of all the (large) orders below PMD_ORDER that are enabled
* for this vma. Then filter out the orders that can't be allocated over
* the faulting address and still be fully contained in the vma.
*/
orders = thp_vma_allowable_orders(vma, vma->vm_flags, false, true, true,
BIT(PMD_ORDER) - 1);
orders = thp_vma_suitable_orders(vma, vmf->address, orders);
if (!orders)
goto fallback;
pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK);
if (!pte)
return ERR_PTR(-EAGAIN);
/*
* Find the highest order where the aligned range is completely
* pte_none(). Note that all remaining orders will be completely
* pte_none().
*/
order = highest_order(orders);
while (orders) {
addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
if (pte_range_none(pte + pte_index(addr), 1 << order))
break;
order = next_order(&orders, order);
}
pte_unmap(pte);
/* Try allocating the highest of the remaining orders. */
gfp = vma_thp_gfp_mask(vma);
while (orders) {
addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
folio = vma_alloc_folio(gfp, order, vma, addr, true);
if (folio) {
if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
folio_put(folio);
goto next;
}
folio_throttle_swaprate(folio, gfp);
clear_huge_page(&folio->page, vmf->address, 1 << order);
return folio;
}
next:
order = next_order(&orders, order);
}
fallback:
#endif
return folio_prealloc(vma->vm_mm, vma, vmf->address, true);
}
/*
* We enter with non-exclusive mmap_lock (to exclude vma changes,
* but allow concurrent faults), and pte mapped but not yet locked.
* We return with mmap_lock still held, but pte unmapped and unlocked.
*/
static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
{
bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
struct vm_area_struct *vma = vmf->vma;
unsigned long addr = vmf->address;
struct folio *folio;
vm_fault_t ret = 0;
int nr_pages = 1;
pte_t entry;
int i;
/* File mapping without ->vm_ops ? */
if (vma->vm_flags & VM_SHARED)
return VM_FAULT_SIGBUS;
/*
* Use pte_alloc() instead of pte_alloc_map(), so that OOM can
* be distinguished from a transient failure of pte_offset_map().
*/
if (pte_alloc(vma->vm_mm, vmf->pmd))
return VM_FAULT_OOM;
/* Use the zero-page for reads */
if (!(vmf->flags & FAULT_FLAG_WRITE) &&
!mm_forbids_zeropage(vma->vm_mm)) {
entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
vma->vm_page_prot));
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (!vmf->pte)
goto unlock;
if (vmf_pte_changed(vmf)) {
update_mmu_tlb(vma, vmf->address, vmf->pte);
goto unlock;
}
ret = check_stable_address_space(vma->vm_mm);
if (ret)
goto unlock;
/* Deliver the page fault to userland, check inside PT lock */
if (userfaultfd_missing(vma)) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
return handle_userfault(vmf, VM_UFFD_MISSING);
}
goto setpte;
}
/* Allocate our own private page. */
if (unlikely(anon_vma_prepare(vma)))
goto oom;
/* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */
folio = alloc_anon_folio(vmf); if (IS_ERR(folio))
return 0;
if (!folio)
goto oom;
nr_pages = folio_nr_pages(folio);
addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
/*
* The memory barrier inside __folio_mark_uptodate makes sure that
* preceding stores to the page contents become visible before
* the set_pte_at() write.
*/
__folio_mark_uptodate(folio);
entry = mk_pte(&folio->page, vma->vm_page_prot);
entry = pte_sw_mkyoung(entry);
if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
if (!vmf->pte)
goto release;
if (nr_pages == 1 && vmf_pte_changed(vmf)) {
update_mmu_tlb(vma, addr, vmf->pte);
goto release;
} else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
for (i = 0; i < nr_pages; i++)
update_mmu_tlb(vma, addr + PAGE_SIZE * i, vmf->pte + i);
goto release;
}
ret = check_stable_address_space(vma->vm_mm);
if (ret)
goto release;
/* Deliver the page fault to userland, check inside PT lock */
if (userfaultfd_missing(vma)) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
folio_put(folio);
return handle_userfault(vmf, VM_UFFD_MISSING);
}
folio_ref_add(folio, nr_pages - 1);
add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
folio_add_new_anon_rmap(folio, vma, addr);
folio_add_lru_vma(folio, vma);
setpte:
if (uffd_wp)
entry = pte_mkuffd_wp(entry);
set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages);
/* No need to invalidate - it was non-present before */
update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr_pages);
unlock:
if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl);
return ret;
release:
folio_put(folio);
goto unlock;
oom:
return VM_FAULT_OOM;
}
/*
* The mmap_lock must have been held on entry, and may have been
* released depending on flags and vma->vm_ops->fault() return value.
* See filemap_fault() and __lock_page_retry().
*/
static vm_fault_t __do_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct folio *folio;
vm_fault_t ret;
/*
* Preallocate pte before we take page_lock because this might lead to
* deadlocks for memcg reclaim which waits for pages under writeback:
* lock_page(A)
* SetPageWriteback(A)
* unlock_page(A)
* lock_page(B)
* lock_page(B)
* pte_alloc_one
* shrink_page_list
* wait_on_page_writeback(A)
* SetPageWriteback(B)
* unlock_page(B)
* # flush A, B to clear the writeback
*/
if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
if (!vmf->prealloc_pte)
return VM_FAULT_OOM;
}
ret = vma->vm_ops->fault(vmf);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
VM_FAULT_DONE_COW)))
return ret;
folio = page_folio(vmf->page);
if (unlikely(PageHWPoison(vmf->page))) {
vm_fault_t poisonret = VM_FAULT_HWPOISON;
if (ret & VM_FAULT_LOCKED) {
if (page_mapped(vmf->page))
unmap_mapping_folio(folio);
/* Retry if a clean folio was removed from the cache. */
if (mapping_evict_folio(folio->mapping, folio))
poisonret = VM_FAULT_NOPAGE;
folio_unlock(folio);
}
folio_put(folio);
vmf->page = NULL;
return poisonret;
}
if (unlikely(!(ret & VM_FAULT_LOCKED)))
folio_lock(folio);
else
VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);
return ret;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void deposit_prealloc_pte(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
/*
* We are going to consume the prealloc table,
* count that as nr_ptes.
*/
mm_inc_nr_ptes(vma->vm_mm);
vmf->prealloc_pte = NULL;
}
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
{
struct folio *folio = page_folio(page);
struct vm_area_struct *vma = vmf->vma;
bool write = vmf->flags & FAULT_FLAG_WRITE;
unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
pmd_t entry;
vm_fault_t ret = VM_FAULT_FALLBACK;
if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
return ret;
if (page != &folio->page || folio_order(folio) != HPAGE_PMD_ORDER)
return ret;
/*
* Just backoff if any subpage of a THP is corrupted otherwise
* the corrupted page may mapped by PMD silently to escape the
* check. This kind of THP just can be PTE mapped. Access to
* the corrupted subpage should trigger SIGBUS as expected.
*/
if (unlikely(folio_test_has_hwpoisoned(folio)))
return ret;
/*
* Archs like ppc64 need additional space to store information
* related to pte entry. Use the preallocated table for that.
*/
if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
if (!vmf->prealloc_pte)
return VM_FAULT_OOM;
}
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
if (unlikely(!pmd_none(*vmf->pmd)))
goto out;
flush_icache_pages(vma, page, HPAGE_PMD_NR);
entry = mk_huge_pmd(page, vma->vm_page_prot);
if (write)
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR);
folio_add_file_rmap_pmd(folio, page, vma);
/*
* deposit and withdraw with pmd lock held
*/
if (arch_needs_pgtable_deposit())
deposit_prealloc_pte(vmf);
set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
update_mmu_cache_pmd(vma, haddr, vmf->pmd);
/* fault is handled */
ret = 0;
count_vm_event(THP_FILE_MAPPED);
out:
spin_unlock(vmf->ptl);
return ret;
}
#else
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
{
return VM_FAULT_FALLBACK;
}
#endif
/**
* set_pte_range - Set a range of PTEs to point to pages in a folio.
* @vmf: Fault decription.
* @folio: The folio that contains @page.
* @page: The first page to create a PTE for.
* @nr: The number of PTEs to create.
* @addr: The first address to create a PTE for.
*/
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
struct page *page, unsigned int nr, unsigned long addr)
{
struct vm_area_struct *vma = vmf->vma;
bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
bool write = vmf->flags & FAULT_FLAG_WRITE;
bool prefault = in_range(vmf->address, addr, nr * PAGE_SIZE);
pte_t entry;
flush_icache_pages(vma, page, nr);
entry = mk_pte(page, vma->vm_page_prot);
if (prefault && arch_wants_old_prefaulted_pte())
entry = pte_mkold(entry);
else
entry = pte_sw_mkyoung(entry);
if (write) entry = maybe_mkwrite(pte_mkdirty(entry), vma);
if (unlikely(uffd_wp))
entry = pte_mkuffd_wp(entry);
/* copy-on-write page */
if (write && !(vma->vm_flags & VM_SHARED)) { add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr); VM_BUG_ON_FOLIO(nr != 1, folio); folio_add_new_anon_rmap(folio, vma, addr);
folio_add_lru_vma(folio, vma);
} else {
add_mm_counter(vma->vm_mm, mm_counter_file(folio), nr);
folio_add_file_rmap_ptes(folio, page, nr, vma);
}
set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
/* no need to invalidate: a not-present page won't be cached */
update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
}
static bool vmf_pte_changed(struct vm_fault *vmf)
{
if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
return !pte_none(ptep_get(vmf->pte));
}
/**
* finish_fault - finish page fault once we have prepared the page to fault
*
* @vmf: structure describing the fault
*
* This function handles all that is needed to finish a page fault once the
* page to fault in is prepared. It handles locking of PTEs, inserts PTE for
* given page, adds reverse page mapping, handles memcg charges and LRU
* addition.
*
* The function expects the page to be locked and on success it consumes a
* reference of a page being mapped (for the PTE which maps it).
*
* Return: %0 on success, %VM_FAULT_ code in case of error.
*/
vm_fault_t finish_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct page *page;
vm_fault_t ret;
/* Did we COW the page? */
if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
page = vmf->cow_page;
else
page = vmf->page;
/*
* check even for read faults because we might have lost our CoWed
* page
*/
if (!(vma->vm_flags & VM_SHARED)) {
ret = check_stable_address_space(vma->vm_mm);
if (ret)
return ret;
}
if (pmd_none(*vmf->pmd)) {
if (PageTransCompound(page)) {
ret = do_set_pmd(vmf, page);
if (ret != VM_FAULT_FALLBACK)
return ret;
}
if (vmf->prealloc_pte)
pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
return VM_FAULT_OOM;
}
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (!vmf->pte)
return VM_FAULT_NOPAGE;
/* Re-check under ptl */
if (likely(!vmf_pte_changed(vmf))) {
struct folio *folio = page_folio(page);
set_pte_range(vmf, folio, page, 1, vmf->address);
ret = 0;
} else {
update_mmu_tlb(vma, vmf->address, vmf->pte);
ret = VM_FAULT_NOPAGE;
}
pte_unmap_unlock(vmf->pte, vmf->ptl);
return ret;
}
static unsigned long fault_around_pages __read_mostly =
65536 >> PAGE_SHIFT;
#ifdef CONFIG_DEBUG_FS
static int fault_around_bytes_get(void *data, u64 *val)
{
*val = fault_around_pages << PAGE_SHIFT;
return 0;
}
/*
* fault_around_bytes must be rounded down to the nearest page order as it's
* what do_fault_around() expects to see.
*/
static int fault_around_bytes_set(void *data, u64 val)
{
if (val / PAGE_SIZE > PTRS_PER_PTE)
return -EINVAL;
/*
* The minimum value is 1 page, however this results in no fault-around
* at all. See should_fault_around().
*/
val = max(val, PAGE_SIZE);
fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
static int __init fault_around_debugfs(void)
{
debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
&fault_around_bytes_fops);
return 0;
}
late_initcall(fault_around_debugfs);
#endif
/*
* do_fault_around() tries to map few pages around the fault address. The hope
* is that the pages will be needed soon and this will lower the number of
* faults to handle.
*
* It uses vm_ops->map_pages() to map the pages, which skips the page if it's
* not ready to be mapped: not up-to-date, locked, etc.
*
* This function doesn't cross VMA or page table boundaries, in order to call
* map_pages() and acquire a PTE lock only once.
*
* fault_around_pages defines how many pages we'll try to map.
* do_fault_around() expects it to be set to a power of two less than or equal
* to PTRS_PER_PTE.
*
* The virtual address of the area that we map is naturally aligned to
* fault_around_pages * PAGE_SIZE rounded down to the machine page size
* (and therefore to page order). This way it's easier to guarantee
* that we don't cross page table boundaries.
*/
static vm_fault_t do_fault_around(struct vm_fault *vmf)
{
pgoff_t nr_pages = READ_ONCE(fault_around_pages);
pgoff_t pte_off = pte_index(vmf->address);
/* The page offset of vmf->address within the VMA. */
pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
pgoff_t from_pte, to_pte;
vm_fault_t ret;
/* The PTE offset of the start address, clamped to the VMA. */
from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
pte_off - min(pte_off, vma_off));
/* The PTE offset of the end address, clamped to the VMA and PTE. */
to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
pte_off + vma_pages(vmf->vma) - vma_off) - 1;
if (pmd_none(*vmf->pmd)) {
vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
if (!vmf->prealloc_pte)
return VM_FAULT_OOM;
}
rcu_read_lock(); ret = vmf->vma->vm_ops->map_pages(vmf,
vmf->pgoff + from_pte - pte_off,
vmf->pgoff + to_pte - pte_off);
rcu_read_unlock();
return ret;
}
/* Return true if we should do read fault-around, false otherwise */
static inline bool should_fault_around(struct vm_fault *vmf)
{
/* No ->map_pages? No way to fault around... */
if (!vmf->vma->vm_ops->map_pages)
return false;
if (uffd_disable_fault_around(vmf->vma))
return false;
/* A single page implies no faulting 'around' at all. */
return fault_around_pages > 1;
}
static vm_fault_t do_read_fault(struct vm_fault *vmf)
{
vm_fault_t ret = 0;
struct folio *folio;
/*
* Let's call ->map_pages() first and use ->fault() as fallback
* if page by the offset is not ready to be mapped (cold cache or
* something).
*/
if (should_fault_around(vmf)) { ret = do_fault_around(vmf);
if (ret)
return ret;
}
ret = vmf_can_call_fault(vmf);
if (ret)
return ret;
ret = __do_fault(vmf);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
return ret;
ret |= finish_fault(vmf);
folio = page_folio(vmf->page);
folio_unlock(folio);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
folio_put(folio);
return ret;
}
static vm_fault_t do_cow_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct folio *folio;
vm_fault_t ret;
ret = vmf_can_call_fault(vmf);
if (!ret)
ret = vmf_anon_prepare(vmf);
if (ret)
return ret;
folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false);
if (!folio)
return VM_FAULT_OOM;
vmf->cow_page = &folio->page;
ret = __do_fault(vmf);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
goto uncharge_out;
if (ret & VM_FAULT_DONE_COW)
return ret;
copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
__folio_mark_uptodate(folio);
ret |= finish_fault(vmf);
unlock_page(vmf->page);
put_page(vmf->page);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
goto uncharge_out;
return ret;
uncharge_out:
folio_put(folio);
return ret;
}
static vm_fault_t do_shared_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret, tmp;
struct folio *folio;
ret = vmf_can_call_fault(vmf);
if (ret)
return ret;
ret = __do_fault(vmf);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
return ret;
folio = page_folio(vmf->page);
/*
* Check if the backing address space wants to know that the page is
* about to become writable
*/
if (vma->vm_ops->page_mkwrite) {
folio_unlock(folio);
tmp = do_page_mkwrite(vmf, folio);
if (unlikely(!tmp ||
(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
folio_put(folio);
return tmp;
}
}
ret |= finish_fault(vmf);
if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
VM_FAULT_RETRY))) {
folio_unlock(folio);
folio_put(folio);
return ret;
}
ret |= fault_dirty_shared_page(vmf);
return ret;
}
/*
* We enter with non-exclusive mmap_lock (to exclude vma changes,
* but allow concurrent faults).
* The mmap_lock may have been released depending on flags and our
* return value. See filemap_fault() and __folio_lock_or_retry().
* If mmap_lock is released, vma may become invalid (for example
* by other thread calling munmap()).
*/
static vm_fault_t do_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct mm_struct *vm_mm = vma->vm_mm;
vm_fault_t ret;
/*
* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
*/
if (!vma->vm_ops->fault) {
vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (unlikely(!vmf->pte))
ret = VM_FAULT_SIGBUS;
else {
/*
* Make sure this is not a temporary clearing of pte
* by holding ptl and checking again. A R/M/W update
* of pte involves: take ptl, clearing the pte so that
* we don't have concurrent modification by hardware
* followed by an update.
*/
if (unlikely(pte_none(ptep_get(vmf->pte))))
ret = VM_FAULT_SIGBUS;
else
ret = VM_FAULT_NOPAGE;
pte_unmap_unlock(vmf->pte, vmf->ptl);
}
} else if (!(vmf->flags & FAULT_FLAG_WRITE)) ret = do_read_fault(vmf); else if (!(vma->vm_flags & VM_SHARED)) ret = do_cow_fault(vmf);
else
ret = do_shared_fault(vmf);
/* preallocated pagetable is unused: free it */
if (vmf->prealloc_pte) { pte_free(vm_mm, vmf->prealloc_pte);
vmf->prealloc_pte = NULL;
}
return ret;
}
int numa_migrate_prep(struct folio *folio, struct vm_area_struct *vma,
unsigned long addr, int page_nid, int *flags)
{
folio_get(folio);
/* Record the current PID acceesing VMA */
vma_set_access_pid_bit(vma);
count_vm_numa_event(NUMA_HINT_FAULTS);
if (page_nid == numa_node_id()) {
count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
*flags |= TNF_FAULT_LOCAL;
}
return mpol_misplaced(folio, vma, addr);
}
static vm_fault_t do_numa_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct folio *folio = NULL;
int nid = NUMA_NO_NODE;
bool writable = false;
int last_cpupid;
int target_nid;
pte_t pte, old_pte;
int flags = 0;
/*
* The pte cannot be used safely until we verify, while holding the page
* table lock, that its contents have not changed during fault handling.
*/
spin_lock(vmf->ptl);
/* Read the live PTE from the page tables: */
old_pte = ptep_get(vmf->pte);
if (unlikely(!pte_same(old_pte, vmf->orig_pte))) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
goto out;
}
pte = pte_modify(old_pte, vma->vm_page_prot);
/*
* Detect now whether the PTE could be writable; this information
* is only valid while holding the PT lock.
*/
writable = pte_write(pte);
if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
can_change_pte_writable(vma, vmf->address, pte))
writable = true;
folio = vm_normal_folio(vma, vmf->address, pte);
if (!folio || folio_is_zone_device(folio))
goto out_map;
/* TODO: handle PTE-mapped THP */
if (folio_test_large(folio))
goto out_map;
/*
* Avoid grouping on RO pages in general. RO pages shouldn't hurt as
* much anyway since they can be in shared cache state. This misses
* the case where a mapping is writable but the process never writes
* to it but pte_write gets cleared during protection updates and
* pte_dirty has unpredictable behaviour between PTE scan updates,
* background writeback, dirty balancing and application behaviour.
*/
if (!writable)
flags |= TNF_NO_GROUP;
/*
* Flag if the folio is shared between multiple address spaces. This
* is later used when determining whether to group tasks together
*/
if (folio_estimated_sharers(folio) > 1 && (vma->vm_flags & VM_SHARED))
flags |= TNF_SHARED;
nid = folio_nid(folio);
/*
* For memory tiering mode, cpupid of slow memory page is used
* to record page access time. So use default value.
*/
if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
!node_is_toptier(nid))
last_cpupid = (-1 & LAST_CPUPID_MASK);
else
last_cpupid = folio_last_cpupid(folio);
target_nid = numa_migrate_prep(folio, vma, vmf->address, nid, &flags);
if (target_nid == NUMA_NO_NODE) {
folio_put(folio);
goto out_map;
}
pte_unmap_unlock(vmf->pte, vmf->ptl);
writable = false;
/* Migrate to the requested node */
if (migrate_misplaced_folio(folio, vma, target_nid)) {
nid = target_nid;
flags |= TNF_MIGRATED;
} else {
flags |= TNF_MIGRATE_FAIL;
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (unlikely(!vmf->pte))
goto out;
if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
goto out;
}
goto out_map;
}
out:
if (nid != NUMA_NO_NODE)
task_numa_fault(last_cpupid, nid, 1, flags);
return 0;
out_map:
/*
* Make it present again, depending on how arch implements
* non-accessible ptes, some can allow access by kernel mode.
*/
old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
pte = pte_modify(old_pte, vma->vm_page_prot);
pte = pte_mkyoung(pte);
if (writable)
pte = pte_mkwrite(pte, vma);
ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
pte_unmap_unlock(vmf->pte, vmf->ptl);
goto out;
}
static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
if (vma_is_anonymous(vma))
return do_huge_pmd_anonymous_page(vmf);
if (vma->vm_ops->huge_fault)
return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
return VM_FAULT_FALLBACK;
}
/* `inline' is required to avoid gcc 4.1.2 build error */
static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
vm_fault_t ret;
if (vma_is_anonymous(vma)) {
if (likely(!unshare) &&
userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
if (userfaultfd_wp_async(vmf->vma))
goto split;
return handle_userfault(vmf, VM_UFFD_WP);
}
return do_huge_pmd_wp_page(vmf);
}
if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
if (vma->vm_ops->huge_fault) {
ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
}
}
split:
/* COW or write-notify handled on pte level: split pmd. */
__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
return VM_FAULT_FALLBACK;
}
static vm_fault_t create_huge_pud(struct vm_fault *vmf)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
struct vm_area_struct *vma = vmf->vma;
/* No support for anonymous transparent PUD pages yet */
if (vma_is_anonymous(vma))
return VM_FAULT_FALLBACK;
if (vma->vm_ops->huge_fault)
return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
return VM_FAULT_FALLBACK;
}
static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret;
/* No support for anonymous transparent PUD pages yet */
if (vma_is_anonymous(vma))
goto split;
if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
if (vma->vm_ops->huge_fault) {
ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
}
}
split:
/* COW or write-notify not handled on PUD level: split pud.*/
__split_huge_pud(vma, vmf->pud, vmf->address);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
return VM_FAULT_FALLBACK;
}
/*
* These routines also need to handle stuff like marking pages dirty
* and/or accessed for architectures that don't do it in hardware (most
* RISC architectures). The early dirtying is also good on the i386.
*
* There is also a hook called "update_mmu_cache()" that architectures
* with external mmu caches can use to update those (ie the Sparc or
* PowerPC hashed page tables that act as extended TLBs).
*
* We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
* concurrent faults).
*
* The mmap_lock may have been released depending on flags and our return value.
* See filemap_fault() and __folio_lock_or_retry().
*/
static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
{
pte_t entry;
if (unlikely(pmd_none(*vmf->pmd))) {
/*
* Leave __pte_alloc() until later: because vm_ops->fault may
* want to allocate huge page, and if we expose page table
* for an instant, it will be difficult to retract from
* concurrent faults and from rmap lookups.
*/
vmf->pte = NULL;
vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
} else {
/*
* A regular pmd is established and it can't morph into a huge
* pmd by anon khugepaged, since that takes mmap_lock in write
* mode; but shmem or file collapse to THP could still morph
* it into a huge pmd: just retry later if so.
*/
vmf->pte = pte_offset_map_nolock(vmf->vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
if (unlikely(!vmf->pte))
return 0;
vmf->orig_pte = ptep_get_lockless(vmf->pte);
vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
if (pte_none(vmf->orig_pte)) {
pte_unmap(vmf->pte);
vmf->pte = NULL;
}
}
if (!vmf->pte)
return do_pte_missing(vmf); if (!pte_present(vmf->orig_pte)) return do_swap_page(vmf);
if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
return do_numa_page(vmf);
spin_lock(vmf->ptl);
entry = vmf->orig_pte;
if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
goto unlock;
}
if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) { if (!pte_write(entry)) return do_wp_page(vmf); else if (likely(vmf->flags & FAULT_FLAG_WRITE)) entry = pte_mkdirty(entry);
}
entry = pte_mkyoung(entry);
if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
vmf->flags & FAULT_FLAG_WRITE)) {
update_mmu_cache_range(vmf, vmf->vma, vmf->address,
vmf->pte, 1);
} else {
/* Skip spurious TLB flush for retried page fault */
if (vmf->flags & FAULT_FLAG_TRIED)
goto unlock;
/*
* This is needed only for protection faults but the arch code
* is not yet telling us if this is a protection fault or not.
* This still avoids useless tlb flushes for .text page faults
* with threads.
*/
if (vmf->flags & FAULT_FLAG_WRITE)
flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
vmf->pte);
}
unlock:
pte_unmap_unlock(vmf->pte, vmf->ptl);
return 0;
}
/*
* On entry, we hold either the VMA lock or the mmap_lock
* (FAULT_FLAG_VMA_LOCK tells you which). If VM_FAULT_RETRY is set in
* the result, the mmap_lock is not held on exit. See filemap_fault()
* and __folio_lock_or_retry().
*/
static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
unsigned long address, unsigned int flags)
{
struct vm_fault vmf = {
.vma = vma,
.address = address & PAGE_MASK,
.real_address = address,
.flags = flags,
.pgoff = linear_page_index(vma, address),
.gfp_mask = __get_fault_gfp_mask(vma),
};
struct mm_struct *mm = vma->vm_mm;
unsigned long vm_flags = vma->vm_flags;
pgd_t *pgd;
p4d_t *p4d;
vm_fault_t ret;
pgd = pgd_offset(mm, address);
p4d = p4d_alloc(mm, pgd, address);
if (!p4d)
return VM_FAULT_OOM; vmf.pud = pud_alloc(mm, p4d, address);
if (!vmf.pud)
return VM_FAULT_OOM;
retry_pud:
if (pud_none(*vmf.pud) &&
thp_vma_allowable_order(vma, vm_flags, false, true, true, PUD_ORDER)) {
ret = create_huge_pud(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
pud_t orig_pud = *vmf.pud;
barrier();
if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
/*
* TODO once we support anonymous PUDs: NUMA case and
* FAULT_FLAG_UNSHARE handling.
*/
if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
ret = wp_huge_pud(&vmf, orig_pud);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
huge_pud_set_accessed(&vmf, orig_pud);
return 0;
}
}
}
vmf.pmd = pmd_alloc(mm, vmf.pud, address);
if (!vmf.pmd)
return VM_FAULT_OOM;
/* Huge pud page fault raced with pmd_alloc? */
if (pud_trans_unstable(vmf.pud))
goto retry_pud;
if (pmd_none(*vmf.pmd) &&
thp_vma_allowable_order(vma, vm_flags, false, true, true, PMD_ORDER)) {
ret = create_huge_pmd(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
VM_BUG_ON(thp_migration_supported() &&
!is_pmd_migration_entry(vmf.orig_pmd));
if (is_pmd_migration_entry(vmf.orig_pmd))
pmd_migration_entry_wait(mm, vmf.pmd);
return 0;
}
if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
return do_huge_pmd_numa_page(&vmf);
if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
!pmd_write(vmf.orig_pmd)) {
ret = wp_huge_pmd(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
huge_pmd_set_accessed(&vmf);
return 0;
}
}
}
return handle_pte_fault(&vmf);
}
/**
* mm_account_fault - Do page fault accounting
* @mm: mm from which memcg should be extracted. It can be NULL.
* @regs: the pt_regs struct pointer. When set to NULL, will skip accounting
* of perf event counters, but we'll still do the per-task accounting to
* the task who triggered this page fault.
* @address: the faulted address.
* @flags: the fault flags.
* @ret: the fault retcode.
*
* This will take care of most of the page fault accounting. Meanwhile, it
* will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
* updates. However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
* still be in per-arch page fault handlers at the entry of page fault.
*/
static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
unsigned long address, unsigned int flags,
vm_fault_t ret)
{
bool major;
/* Incomplete faults will be accounted upon completion. */
if (ret & VM_FAULT_RETRY)
return;
/*
* To preserve the behavior of older kernels, PGFAULT counters record
* both successful and failed faults, as opposed to perf counters,
* which ignore failed cases.
*/
count_vm_event(PGFAULT); count_memcg_event_mm(mm, PGFAULT);
/*
* Do not account for unsuccessful faults (e.g. when the address wasn't
* valid). That includes arch_vma_access_permitted() failing before
* reaching here. So this is not a "this many hardware page faults"
* counter. We should use the hw profiling for that.
*/
if (ret & VM_FAULT_ERROR)
return;
/*
* We define the fault as a major fault when the final successful fault
* is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
* handle it immediately previously).
*/
major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
if (major)
current->maj_flt++;
else
current->min_flt++;
/*
* If the fault is done for GUP, regs will be NULL. We only do the
* accounting for the per thread fault counters who triggered the
* fault, and we skip the perf event updates.
*/
if (!regs)
return;
if (major)
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
else
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
}
#ifdef CONFIG_LRU_GEN
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
/* the LRU algorithm only applies to accesses with recency */
current->in_lru_fault = vma_has_recency(vma);
}
static void lru_gen_exit_fault(void)
{
current->in_lru_fault = false;
}
#else
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
}
static void lru_gen_exit_fault(void)
{
}
#endif /* CONFIG_LRU_GEN */
static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
unsigned int *flags)
{
if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
return VM_FAULT_SIGSEGV;
/*
* FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
* just treat it like an ordinary read-fault otherwise.
*/
if (!is_cow_mapping(vma->vm_flags)) *flags &= ~FAULT_FLAG_UNSHARE; } else if (*flags & FAULT_FLAG_WRITE) {
/* Write faults on read-only mappings are impossible ... */
if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
return VM_FAULT_SIGSEGV;
/* ... and FOLL_FORCE only applies to COW mappings. */
if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
!is_cow_mapping(vma->vm_flags)))
return VM_FAULT_SIGSEGV;
}
#ifdef CONFIG_PER_VMA_LOCK
/*
* Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
* the assumption that lock is dropped on VM_FAULT_RETRY.
*/
if (WARN_ON_ONCE((*flags &
(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
return VM_FAULT_SIGSEGV;
#endif
return 0;
}
/*
* By the time we get here, we already hold the mm semaphore
*
* The mmap_lock may have been released depending on flags and our
* return value. See filemap_fault() and __folio_lock_or_retry().
*/
vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
unsigned int flags, struct pt_regs *regs)
{
/* If the fault handler drops the mmap_lock, vma may be freed */
struct mm_struct *mm = vma->vm_mm;
vm_fault_t ret;
__set_current_state(TASK_RUNNING);
ret = sanitize_fault_flags(vma, &flags);
if (ret)
goto out;
if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
flags & FAULT_FLAG_INSTRUCTION,
flags & FAULT_FLAG_REMOTE)) {
ret = VM_FAULT_SIGSEGV;
goto out;
}
/*
* Enable the memcg OOM handling for faults triggered in user
* space. Kernel faults are handled more gracefully.
*/
if (flags & FAULT_FLAG_USER) mem_cgroup_enter_user_fault();
lru_gen_enter_fault(vma);
if (unlikely(is_vm_hugetlb_page(vma))) ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
else
ret = __handle_mm_fault(vma, address, flags);
lru_gen_exit_fault();
if (flags & FAULT_FLAG_USER) { mem_cgroup_exit_user_fault();
/*
* The task may have entered a memcg OOM situation but
* if the allocation error was handled gracefully (no
* VM_FAULT_OOM), there is no need to kill anything.
* Just clean up the OOM state peacefully.
*/
if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM)) mem_cgroup_oom_synchronize(false);
}
out:
mm_account_fault(mm, regs, address, flags, ret); return ret;
}
EXPORT_SYMBOL_GPL(handle_mm_fault);
#ifdef CONFIG_LOCK_MM_AND_FIND_VMA
#include <linux/extable.h>
static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
{
if (likely(mmap_read_trylock(mm)))
return true;
if (regs && !user_mode(regs)) { unsigned long ip = exception_ip(regs);
if (!search_exception_tables(ip))
return false;
}
return !mmap_read_lock_killable(mm);
}
static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
{
/*
* We don't have this operation yet.
*
* It should be easy enough to do: it's basically a
* atomic_long_try_cmpxchg_acquire()
* from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
* it also needs the proper lockdep magic etc.
*/
return false;
}
static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
{
mmap_read_unlock(mm); if (regs && !user_mode(regs)) { unsigned long ip = exception_ip(regs);
if (!search_exception_tables(ip))
return false;
}
return !mmap_write_lock_killable(mm);
}
/*
* Helper for page fault handling.
*
* This is kind of equivalend to "mmap_read_lock()" followed
* by "find_extend_vma()", except it's a lot more careful about
* the locking (and will drop the lock on failure).
*
* For example, if we have a kernel bug that causes a page
* fault, we don't want to just use mmap_read_lock() to get
* the mm lock, because that would deadlock if the bug were
* to happen while we're holding the mm lock for writing.
*
* So this checks the exception tables on kernel faults in
* order to only do this all for instructions that are actually
* expected to fault.
*
* We can also actually take the mm lock for writing if we
* need to extend the vma, which helps the VM layer a lot.
*/
struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
unsigned long addr, struct pt_regs *regs)
{
struct vm_area_struct *vma;
if (!get_mmap_lock_carefully(mm, regs))
return NULL;
vma = find_vma(mm, addr); if (likely(vma && (vma->vm_start <= addr)))
return vma;
/*
* Well, dang. We might still be successful, but only
* if we can extend a vma to do so.
*/
if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) { mmap_read_unlock(mm);
return NULL;
}
/*
* We can try to upgrade the mmap lock atomically,
* in which case we can continue to use the vma
* we already looked up.
*
* Otherwise we'll have to drop the mmap lock and
* re-take it, and also look up the vma again,
* re-checking it.
*/
if (!mmap_upgrade_trylock(mm)) {
if (!upgrade_mmap_lock_carefully(mm, regs))
return NULL;
vma = find_vma(mm, addr);
if (!vma)
goto fail;
if (vma->vm_start <= addr)
goto success;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto fail;
}
if (expand_stack_locked(vma, addr))
goto fail;
success:
mmap_write_downgrade(mm);
return vma;
fail:
mmap_write_unlock(mm);
return NULL;
}
#endif
#ifdef CONFIG_PER_VMA_LOCK
/*
* Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
* stable and not isolated. If the VMA is not found or is being modified the
* function returns NULL.
*/
struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
unsigned long address)
{
MA_STATE(mas, &mm->mm_mt, address, address);
struct vm_area_struct *vma;
rcu_read_lock();
retry:
vma = mas_walk(&mas);
if (!vma)
goto inval;
if (!vma_start_read(vma))
goto inval;
/*
* find_mergeable_anon_vma uses adjacent vmas which are not locked.
* This check must happen after vma_start_read(); otherwise, a
* concurrent mremap() with MREMAP_DONTUNMAP could dissociate the VMA
* from its anon_vma.
*/
if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma))
goto inval_end_read;
/* Check since vm_start/vm_end might change before we lock the VMA */
if (unlikely(address < vma->vm_start || address >= vma->vm_end))
goto inval_end_read;
/* Check if the VMA got isolated after we found it */
if (vma->detached) { vma_end_read(vma);
count_vm_vma_lock_event(VMA_LOCK_MISS);
/* The area was replaced with another one */
goto retry;
}
rcu_read_unlock();
return vma;
inval_end_read:
vma_end_read(vma);
inval:
rcu_read_unlock();
count_vm_vma_lock_event(VMA_LOCK_ABORT);
return NULL;
}
#endif /* CONFIG_PER_VMA_LOCK */
#ifndef __PAGETABLE_P4D_FOLDED
/*
* Allocate p4d page table.
* We've already handled the fast-path in-line.
*/
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
{
p4d_t *new = p4d_alloc_one(mm, address);
if (!new)
return -ENOMEM;
spin_lock(&mm->page_table_lock);
if (pgd_present(*pgd)) { /* Another has populated it */
p4d_free(mm, new);
} else {
smp_wmb(); /* See comment in pmd_install() */
pgd_populate(mm, pgd, new);
}
spin_unlock(&mm->page_table_lock);
return 0;
}
#endif /* __PAGETABLE_P4D_FOLDED */
#ifndef __PAGETABLE_PUD_FOLDED
/*
* Allocate page upper directory.
* We've already handled the fast-path in-line.
*/
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
{
pud_t *new = pud_alloc_one(mm, address);
if (!new)
return -ENOMEM;
spin_lock(&mm->page_table_lock);
if (!p4d_present(*p4d)) {
mm_inc_nr_puds(mm);
smp_wmb(); /* See comment in pmd_install() */
p4d_populate(mm, p4d, new);
} else /* Another has populated it */
pud_free(mm, new);
spin_unlock(&mm->page_table_lock);
return 0;
}
#endif /* __PAGETABLE_PUD_FOLDED */
#ifndef __PAGETABLE_PMD_FOLDED
/*
* Allocate page middle directory.
* We've already handled the fast-path in-line.
*/
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
spinlock_t *ptl;
pmd_t *new = pmd_alloc_one(mm, address);
if (!new)
return -ENOMEM;
ptl = pud_lock(mm, pud); if (!pud_present(*pud)) { mm_inc_nr_pmds(mm);
smp_wmb(); /* See comment in pmd_install() */
pud_populate(mm, pud, new);
} else { /* Another has populated it */
pmd_free(mm, new);
}
spin_unlock(ptl); return 0;
}
#endif /* __PAGETABLE_PMD_FOLDED */
/**
* follow_pte - look up PTE at a user virtual address
* @mm: the mm_struct of the target address space
* @address: user virtual address
* @ptepp: location to store found PTE
* @ptlp: location to store the lock for the PTE
*
* On a successful return, the pointer to the PTE is stored in @ptepp;
* the corresponding lock is taken and its location is stored in @ptlp.
* The contents of the PTE are only stable until @ptlp is released;
* any further use, if any, must be protected against invalidation
* with MMU notifiers.
*
* Only IO mappings and raw PFN mappings are allowed. The mmap semaphore
* should be taken for read.
*
* KVM uses this function. While it is arguably less bad than ``follow_pfn``,
* it is not a good general-purpose API.
*
* Return: zero on success, -ve otherwise.
*/
int follow_pte(struct mm_struct *mm, unsigned long address,
pte_t **ptepp, spinlock_t **ptlp)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *ptep;
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
goto out;
p4d = p4d_offset(pgd, address);
if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
goto out;
pud = pud_offset(p4d, address);
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
goto out;
pmd = pmd_offset(pud, address);
VM_BUG_ON(pmd_trans_huge(*pmd));
ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
if (!ptep)
goto out;
if (!pte_present(ptep_get(ptep)))
goto unlock;
*ptepp = ptep;
return 0;
unlock:
pte_unmap_unlock(ptep, *ptlp);
out:
return -EINVAL;
}
EXPORT_SYMBOL_GPL(follow_pte);
/**
* follow_pfn - look up PFN at a user virtual address
* @vma: memory mapping
* @address: user virtual address
* @pfn: location to store found PFN
*
* Only IO mappings and raw PFN mappings are allowed.
*
* This function does not allow the caller to read the permissions
* of the PTE. Do not use it.
*
* Return: zero and the pfn at @pfn on success, -ve otherwise.
*/
int follow_pfn(struct vm_area_struct *vma, unsigned long address,
unsigned long *pfn)
{
int ret = -EINVAL;
spinlock_t *ptl;
pte_t *ptep;
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
return ret;
ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
if (ret)
return ret;
*pfn = pte_pfn(ptep_get(ptep));
pte_unmap_unlock(ptep, ptl);
return 0;
}
EXPORT_SYMBOL(follow_pfn);
#ifdef CONFIG_HAVE_IOREMAP_PROT
int follow_phys(struct vm_area_struct *vma,
unsigned long address, unsigned int flags,
unsigned long *prot, resource_size_t *phys)
{
int ret = -EINVAL;
pte_t *ptep, pte;
spinlock_t *ptl;
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
goto out;
if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
goto out;
pte = ptep_get(ptep);
/* Never return PFNs of anon folios in COW mappings. */
if (vm_normal_folio(vma, address, pte))
goto unlock;
if ((flags & FOLL_WRITE) && !pte_write(pte))
goto unlock;
*prot = pgprot_val(pte_pgprot(pte));
*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
ret = 0;
unlock:
pte_unmap_unlock(ptep, ptl);
out:
return ret;
}
/**
* generic_access_phys - generic implementation for iomem mmap access
* @vma: the vma to access
* @addr: userspace address, not relative offset within @vma
* @buf: buffer to read/write
* @len: length of transfer
* @write: set to FOLL_WRITE when writing, otherwise reading
*
* This is a generic implementation for &vm_operations_struct.access for an
* iomem mapping. This callback is used by access_process_vm() when the @vma is
* not page based.
*/
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write)
{
resource_size_t phys_addr;
unsigned long prot = 0;
void __iomem *maddr;
pte_t *ptep, pte;
spinlock_t *ptl;
int offset = offset_in_page(addr);
int ret = -EINVAL;
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
return -EINVAL;
retry:
if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
return -EINVAL;
pte = ptep_get(ptep);
pte_unmap_unlock(ptep, ptl);
prot = pgprot_val(pte_pgprot(pte));
phys_addr = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
if ((write & FOLL_WRITE) && !pte_write(pte))
return -EINVAL;
maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
if (!maddr)
return -ENOMEM;
if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
goto out_unmap;
if (!pte_same(pte, ptep_get(ptep))) {
pte_unmap_unlock(ptep, ptl);
iounmap(maddr);
goto retry;
}
if (write)
memcpy_toio(maddr + offset, buf, len);
else
memcpy_fromio(buf, maddr + offset, len);
ret = len;
pte_unmap_unlock(ptep, ptl);
out_unmap:
iounmap(maddr);
return ret;
}
EXPORT_SYMBOL_GPL(generic_access_phys);
#endif
/*
* Access another process' address space as given in mm.
*/
static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
void *buf, int len, unsigned int gup_flags)
{
void *old_buf = buf;
int write = gup_flags & FOLL_WRITE;
if (mmap_read_lock_killable(mm))
return 0;
/* Untag the address before looking up the VMA */
addr = untagged_addr_remote(mm, addr);
/* Avoid triggering the temporary warning in __get_user_pages */
if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
return 0;
/* ignore errors, just check how much was successfully transferred */
while (len) {
int bytes, offset;
void *maddr;
struct vm_area_struct *vma = NULL;
struct page *page = get_user_page_vma_remote(mm, addr,
gup_flags, &vma);
if (IS_ERR(page)) {
/* We might need to expand the stack to access it */
vma = vma_lookup(mm, addr);
if (!vma) {
vma = expand_stack(mm, addr);
/* mmap_lock was dropped on failure */
if (!vma)
return buf - old_buf;
/* Try again if stack expansion worked */
continue;
}
/*
* Check if this is a VM_IO | VM_PFNMAP VMA, which
* we can access using slightly different code.
*/
bytes = 0;
#ifdef CONFIG_HAVE_IOREMAP_PROT
if (vma->vm_ops && vma->vm_ops->access)
bytes = vma->vm_ops->access(vma, addr, buf,
len, write);
#endif
if (bytes <= 0)
break;
} else {
bytes = len;
offset = addr & (PAGE_SIZE-1);
if (bytes > PAGE_SIZE-offset)
bytes = PAGE_SIZE-offset;
maddr = kmap_local_page(page);
if (write) {
copy_to_user_page(vma, page, addr,
maddr + offset, buf, bytes);
set_page_dirty_lock(page);
} else {
copy_from_user_page(vma, page, addr,
buf, maddr + offset, bytes);
}
unmap_and_put_page(page, maddr);
}
len -= bytes;
buf += bytes;
addr += bytes;
}
mmap_read_unlock(mm);
return buf - old_buf;
}
/**
* access_remote_vm - access another process' address space
* @mm: the mm_struct of the target address space
* @addr: start address to access
* @buf: source or destination buffer
* @len: number of bytes to transfer
* @gup_flags: flags modifying lookup behaviour
*
* The caller must hold a reference on @mm.
*
* Return: number of bytes copied from source to destination.
*/
int access_remote_vm(struct mm_struct *mm, unsigned long addr,
void *buf, int len, unsigned int gup_flags)
{
return __access_remote_vm(mm, addr, buf, len, gup_flags);
}
/*
* Access another process' address space.
* Source/target buffer must be kernel space,
* Do not walk the page table directly, use get_user_pages
*/
int access_process_vm(struct task_struct *tsk, unsigned long addr,
void *buf, int len, unsigned int gup_flags)
{
struct mm_struct *mm;
int ret;
mm = get_task_mm(tsk);
if (!mm)
return 0;
ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
mmput(mm);
return ret;
}
EXPORT_SYMBOL_GPL(access_process_vm);
/*
* Print the name of a VMA.
*/
void print_vma_addr(char *prefix, unsigned long ip)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
/*
* we might be running from an atomic context so we cannot sleep
*/
if (!mmap_read_trylock(mm))
return;
vma = find_vma(mm, ip);
if (vma && vma->vm_file) {
struct file *f = vma->vm_file;
char *buf = (char *)__get_free_page(GFP_NOWAIT);
if (buf) {
char *p;
p = file_path(f, buf, PAGE_SIZE);
if (IS_ERR(p))
p = "?";
printk("%s%s[%lx+%lx]", prefix, kbasename(p),
vma->vm_start,
vma->vm_end - vma->vm_start);
free_page((unsigned long)buf);
}
}
mmap_read_unlock(mm);
}
#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
void __might_fault(const char *file, int line)
{
if (pagefault_disabled())
return;
__might_sleep(file, line);
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
if (current->mm)
might_lock_read(¤t->mm->mmap_lock);
#endif
}
EXPORT_SYMBOL(__might_fault);
#endif
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
/*
* Process all subpages of the specified huge page with the specified
* operation. The target subpage will be processed last to keep its
* cache lines hot.
*/
static inline int process_huge_page(
unsigned long addr_hint, unsigned int pages_per_huge_page,
int (*process_subpage)(unsigned long addr, int idx, void *arg),
void *arg)
{
int i, n, base, l, ret;
unsigned long addr = addr_hint &
~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
/* Process target subpage last to keep its cache lines hot */
might_sleep();
n = (addr_hint - addr) / PAGE_SIZE;
if (2 * n <= pages_per_huge_page) {
/* If target subpage in first half of huge page */
base = 0;
l = n;
/* Process subpages at the end of huge page */
for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
cond_resched();
ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
if (ret)
return ret;
}
} else {
/* If target subpage in second half of huge page */
base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
l = pages_per_huge_page - n;
/* Process subpages at the begin of huge page */
for (i = 0; i < base; i++) {
cond_resched();
ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
if (ret)
return ret;
}
}
/*
* Process remaining subpages in left-right-left-right pattern
* towards the target subpage
*/
for (i = 0; i < l; i++) {
int left_idx = base + i;
int right_idx = base + 2 * l - 1 - i;
cond_resched();
ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
if (ret)
return ret;
cond_resched();
ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
if (ret)
return ret;
}
return 0;
}
static void clear_gigantic_page(struct page *page,
unsigned long addr,
unsigned int pages_per_huge_page)
{
int i;
struct page *p;
might_sleep();
for (i = 0; i < pages_per_huge_page; i++) {
p = nth_page(page, i);
cond_resched();
clear_user_highpage(p, addr + i * PAGE_SIZE);
}
}
static int clear_subpage(unsigned long addr, int idx, void *arg)
{
struct page *page = arg;
clear_user_highpage(nth_page(page, idx), addr);
return 0;
}
void clear_huge_page(struct page *page,
unsigned long addr_hint, unsigned int pages_per_huge_page)
{
unsigned long addr = addr_hint &
~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
clear_gigantic_page(page, addr, pages_per_huge_page);
return;
}
process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
}
static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
unsigned long addr,
struct vm_area_struct *vma,
unsigned int pages_per_huge_page)
{
int i;
struct page *dst_page;
struct page *src_page;
for (i = 0; i < pages_per_huge_page; i++) {
dst_page = folio_page(dst, i);
src_page = folio_page(src, i);
cond_resched();
if (copy_mc_user_highpage(dst_page, src_page,
addr + i*PAGE_SIZE, vma)) {
memory_failure_queue(page_to_pfn(src_page), 0);
return -EHWPOISON;
}
}
return 0;
}
struct copy_subpage_arg {
struct page *dst;
struct page *src;
struct vm_area_struct *vma;
};
static int copy_subpage(unsigned long addr, int idx, void *arg)
{
struct copy_subpage_arg *copy_arg = arg;
struct page *dst = nth_page(copy_arg->dst, idx);
struct page *src = nth_page(copy_arg->src, idx);
if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma)) {
memory_failure_queue(page_to_pfn(src), 0);
return -EHWPOISON;
}
return 0;
}
int copy_user_large_folio(struct folio *dst, struct folio *src,
unsigned long addr_hint, struct vm_area_struct *vma)
{
unsigned int pages_per_huge_page = folio_nr_pages(dst);
unsigned long addr = addr_hint &
~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
struct copy_subpage_arg arg = {
.dst = &dst->page,
.src = &src->page,
.vma = vma,
};
if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES))
return copy_user_gigantic_page(dst, src, addr, vma,
pages_per_huge_page);
return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
}
long copy_folio_from_user(struct folio *dst_folio,
const void __user *usr_src,
bool allow_pagefault)
{
void *kaddr;
unsigned long i, rc = 0;
unsigned int nr_pages = folio_nr_pages(dst_folio);
unsigned long ret_val = nr_pages * PAGE_SIZE;
struct page *subpage;
for (i = 0; i < nr_pages; i++) {
subpage = folio_page(dst_folio, i);
kaddr = kmap_local_page(subpage);
if (!allow_pagefault)
pagefault_disable();
rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
if (!allow_pagefault)
pagefault_enable();
kunmap_local(kaddr);
ret_val -= (PAGE_SIZE - rc);
if (rc)
break;
flush_dcache_page(subpage);
cond_resched();
}
return ret_val;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
static struct kmem_cache *page_ptl_cachep;
void __init ptlock_cache_init(void)
{
page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
SLAB_PANIC, NULL);
}
bool ptlock_alloc(struct ptdesc *ptdesc)
{
spinlock_t *ptl;
ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
if (!ptl)
return false;
ptdesc->ptl = ptl; return true;
}
void ptlock_free(struct ptdesc *ptdesc)
{
kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
}
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* Power Management Quality of Service (PM QoS) support base.
*
* Copyright (C) 2020 Intel Corporation
*
* Authors:
* Mark Gross <mgross@linux.intel.com>
* Rafael J. Wysocki <rafael.j.wysocki@intel.com>
*
* Provided here is an interface for specifying PM QoS dependencies. It allows
* entities depending on QoS constraints to register their requests which are
* aggregated as appropriate to produce effective constraints (target values)
* that can be monitored by entities needing to respect them, either by polling
* or through a built-in notification mechanism.
*
* In addition to the basic functionality, more specific interfaces for managing
* global CPU latency QoS requests and frequency QoS requests are provided.
*/
/*#define DEBUG*/
#include <linux/pm_qos.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/miscdevice.h>
#include <linux/string.h>
#include <linux/platform_device.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
#include <linux/export.h>
#include <trace/events/power.h>
/*
* locking rule: all changes to constraints or notifiers lists
* or pm_qos_object list and pm_qos_objects need to happen with pm_qos_lock
* held, taken with _irqsave. One lock to rule them all
*/
static DEFINE_SPINLOCK(pm_qos_lock);
/**
* pm_qos_read_value - Return the current effective constraint value.
* @c: List of PM QoS constraint requests.
*/
s32 pm_qos_read_value(struct pm_qos_constraints *c)
{
return READ_ONCE(c->target_value);
}
static int pm_qos_get_value(struct pm_qos_constraints *c)
{
if (plist_head_empty(&c->list))
return c->no_constraint_value;
switch (c->type) {
case PM_QOS_MIN:
return plist_first(&c->list)->prio;
case PM_QOS_MAX:
return plist_last(&c->list)->prio;
default:
WARN(1, "Unknown PM QoS type in %s\n", __func__);
return PM_QOS_DEFAULT_VALUE;
}
}
static void pm_qos_set_value(struct pm_qos_constraints *c, s32 value)
{
WRITE_ONCE(c->target_value, value);
}
/**
* pm_qos_update_target - Update a list of PM QoS constraint requests.
* @c: List of PM QoS requests.
* @node: Target list entry.
* @action: Action to carry out (add, update or remove).
* @value: New request value for the target list entry.
*
* Update the given list of PM QoS constraint requests, @c, by carrying an
* @action involving the @node list entry and @value on it.
*
* The recognized values of @action are PM_QOS_ADD_REQ (store @value in @node
* and add it to the list), PM_QOS_UPDATE_REQ (remove @node from the list, store
* @value in it and add it to the list again), and PM_QOS_REMOVE_REQ (remove
* @node from the list, ignore @value).
*
* Return: 1 if the aggregate constraint value has changed, 0 otherwise.
*/
int pm_qos_update_target(struct pm_qos_constraints *c, struct plist_node *node,
enum pm_qos_req_action action, int value)
{
int prev_value, curr_value, new_value;
unsigned long flags;
spin_lock_irqsave(&pm_qos_lock, flags);
prev_value = pm_qos_get_value(c);
if (value == PM_QOS_DEFAULT_VALUE)
new_value = c->default_value;
else
new_value = value;
switch (action) {
case PM_QOS_REMOVE_REQ:
plist_del(node, &c->list);
break;
case PM_QOS_UPDATE_REQ:
/*
* To change the list, atomically remove, reinit with new value
* and add, then see if the aggregate has changed.
*/
plist_del(node, &c->list);
fallthrough;
case PM_QOS_ADD_REQ:
plist_node_init(node, new_value);
plist_add(node, &c->list);
break;
default:
/* no action */
;
}
curr_value = pm_qos_get_value(c);
pm_qos_set_value(c, curr_value);
spin_unlock_irqrestore(&pm_qos_lock, flags);
trace_pm_qos_update_target(action, prev_value, curr_value);
if (prev_value == curr_value)
return 0;
if (c->notifiers)
blocking_notifier_call_chain(c->notifiers, curr_value, NULL);
return 1;
}
/**
* pm_qos_flags_remove_req - Remove device PM QoS flags request.
* @pqf: Device PM QoS flags set to remove the request from.
* @req: Request to remove from the set.
*/
static void pm_qos_flags_remove_req(struct pm_qos_flags *pqf,
struct pm_qos_flags_request *req)
{
s32 val = 0;
list_del(&req->node);
list_for_each_entry(req, &pqf->list, node)
val |= req->flags; pqf->effective_flags = val;
}
/**
* pm_qos_update_flags - Update a set of PM QoS flags.
* @pqf: Set of PM QoS flags to update.
* @req: Request to add to the set, to modify, or to remove from the set.
* @action: Action to take on the set.
* @val: Value of the request to add or modify.
*
* Return: 1 if the aggregate constraint value has changed, 0 otherwise.
*/
bool pm_qos_update_flags(struct pm_qos_flags *pqf,
struct pm_qos_flags_request *req,
enum pm_qos_req_action action, s32 val)
{
unsigned long irqflags;
s32 prev_value, curr_value;
spin_lock_irqsave(&pm_qos_lock, irqflags); prev_value = list_empty(&pqf->list) ? 0 : pqf->effective_flags; switch (action) {
case PM_QOS_REMOVE_REQ:
pm_qos_flags_remove_req(pqf, req);
break;
case PM_QOS_UPDATE_REQ:
pm_qos_flags_remove_req(pqf, req);
fallthrough;
case PM_QOS_ADD_REQ:
req->flags = val;
INIT_LIST_HEAD(&req->node);
list_add_tail(&req->node, &pqf->list); pqf->effective_flags |= val;
break;
default:
/* no action */
;
}
curr_value = list_empty(&pqf->list) ? 0 : pqf->effective_flags; spin_unlock_irqrestore(&pm_qos_lock, irqflags); trace_pm_qos_update_flags(action, prev_value, curr_value); return prev_value != curr_value;
}
#ifdef CONFIG_CPU_IDLE
/* Definitions related to the CPU latency QoS. */
static struct pm_qos_constraints cpu_latency_constraints = {
.list = PLIST_HEAD_INIT(cpu_latency_constraints.list),
.target_value = PM_QOS_CPU_LATENCY_DEFAULT_VALUE,
.default_value = PM_QOS_CPU_LATENCY_DEFAULT_VALUE,
.no_constraint_value = PM_QOS_CPU_LATENCY_DEFAULT_VALUE,
.type = PM_QOS_MIN,
};
static inline bool cpu_latency_qos_value_invalid(s32 value)
{
return value < 0 && value != PM_QOS_DEFAULT_VALUE;
}
/**
* cpu_latency_qos_limit - Return current system-wide CPU latency QoS limit.
*/
s32 cpu_latency_qos_limit(void)
{
return pm_qos_read_value(&cpu_latency_constraints);
}
/**
* cpu_latency_qos_request_active - Check the given PM QoS request.
* @req: PM QoS request to check.
*
* Return: 'true' if @req has been added to the CPU latency QoS list, 'false'
* otherwise.
*/
bool cpu_latency_qos_request_active(struct pm_qos_request *req)
{
return req->qos == &cpu_latency_constraints;
}
EXPORT_SYMBOL_GPL(cpu_latency_qos_request_active);
static void cpu_latency_qos_apply(struct pm_qos_request *req,
enum pm_qos_req_action action, s32 value)
{
int ret = pm_qos_update_target(req->qos, &req->node, action, value);
if (ret > 0)
wake_up_all_idle_cpus();
}
/**
* cpu_latency_qos_add_request - Add new CPU latency QoS request.
* @req: Pointer to a preallocated handle.
* @value: Requested constraint value.
*
* Use @value to initialize the request handle pointed to by @req, insert it as
* a new entry to the CPU latency QoS list and recompute the effective QoS
* constraint for that list.
*
* Callers need to save the handle for later use in updates and removal of the
* QoS request represented by it.
*/
void cpu_latency_qos_add_request(struct pm_qos_request *req, s32 value)
{
if (!req || cpu_latency_qos_value_invalid(value))
return;
if (cpu_latency_qos_request_active(req)) {
WARN(1, KERN_ERR "%s called for already added request\n", __func__);
return;
}
trace_pm_qos_add_request(value);
req->qos = &cpu_latency_constraints;
cpu_latency_qos_apply(req, PM_QOS_ADD_REQ, value);
}
EXPORT_SYMBOL_GPL(cpu_latency_qos_add_request);
/**
* cpu_latency_qos_update_request - Modify existing CPU latency QoS request.
* @req : QoS request to update.
* @new_value: New requested constraint value.
*
* Use @new_value to update the QoS request represented by @req in the CPU
* latency QoS list along with updating the effective constraint value for that
* list.
*/
void cpu_latency_qos_update_request(struct pm_qos_request *req, s32 new_value)
{
if (!req || cpu_latency_qos_value_invalid(new_value))
return;
if (!cpu_latency_qos_request_active(req)) {
WARN(1, KERN_ERR "%s called for unknown object\n", __func__);
return;
}
trace_pm_qos_update_request(new_value);
if (new_value == req->node.prio)
return;
cpu_latency_qos_apply(req, PM_QOS_UPDATE_REQ, new_value);
}
EXPORT_SYMBOL_GPL(cpu_latency_qos_update_request);
/**
* cpu_latency_qos_remove_request - Remove existing CPU latency QoS request.
* @req: QoS request to remove.
*
* Remove the CPU latency QoS request represented by @req from the CPU latency
* QoS list along with updating the effective constraint value for that list.
*/
void cpu_latency_qos_remove_request(struct pm_qos_request *req)
{
if (!req)
return;
if (!cpu_latency_qos_request_active(req)) {
WARN(1, KERN_ERR "%s called for unknown object\n", __func__);
return;
}
trace_pm_qos_remove_request(PM_QOS_DEFAULT_VALUE);
cpu_latency_qos_apply(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
EXPORT_SYMBOL_GPL(cpu_latency_qos_remove_request);
/* User space interface to the CPU latency QoS via misc device. */
static int cpu_latency_qos_open(struct inode *inode, struct file *filp)
{
struct pm_qos_request *req;
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (!req)
return -ENOMEM;
cpu_latency_qos_add_request(req, PM_QOS_DEFAULT_VALUE);
filp->private_data = req;
return 0;
}
static int cpu_latency_qos_release(struct inode *inode, struct file *filp)
{
struct pm_qos_request *req = filp->private_data;
filp->private_data = NULL;
cpu_latency_qos_remove_request(req);
kfree(req);
return 0;
}
static ssize_t cpu_latency_qos_read(struct file *filp, char __user *buf,
size_t count, loff_t *f_pos)
{
struct pm_qos_request *req = filp->private_data;
unsigned long flags;
s32 value;
if (!req || !cpu_latency_qos_request_active(req))
return -EINVAL;
spin_lock_irqsave(&pm_qos_lock, flags);
value = pm_qos_get_value(&cpu_latency_constraints);
spin_unlock_irqrestore(&pm_qos_lock, flags);
return simple_read_from_buffer(buf, count, f_pos, &value, sizeof(s32));
}
static ssize_t cpu_latency_qos_write(struct file *filp, const char __user *buf,
size_t count, loff_t *f_pos)
{
s32 value;
if (count == sizeof(s32)) {
if (copy_from_user(&value, buf, sizeof(s32)))
return -EFAULT;
} else {
int ret;
ret = kstrtos32_from_user(buf, count, 16, &value);
if (ret)
return ret;
}
cpu_latency_qos_update_request(filp->private_data, value);
return count;
}
static const struct file_operations cpu_latency_qos_fops = {
.write = cpu_latency_qos_write,
.read = cpu_latency_qos_read,
.open = cpu_latency_qos_open,
.release = cpu_latency_qos_release,
.llseek = noop_llseek,
};
static struct miscdevice cpu_latency_qos_miscdev = {
.minor = MISC_DYNAMIC_MINOR,
.name = "cpu_dma_latency",
.fops = &cpu_latency_qos_fops,
};
static int __init cpu_latency_qos_init(void)
{
int ret;
ret = misc_register(&cpu_latency_qos_miscdev);
if (ret < 0)
pr_err("%s: %s setup failed\n", __func__,
cpu_latency_qos_miscdev.name);
return ret;
}
late_initcall(cpu_latency_qos_init);
#endif /* CONFIG_CPU_IDLE */
/* Definitions related to the frequency QoS below. */
static inline bool freq_qos_value_invalid(s32 value)
{
return value < 0 && value != PM_QOS_DEFAULT_VALUE;
}
/**
* freq_constraints_init - Initialize frequency QoS constraints.
* @qos: Frequency QoS constraints to initialize.
*/
void freq_constraints_init(struct freq_constraints *qos)
{
struct pm_qos_constraints *c;
c = &qos->min_freq;
plist_head_init(&c->list);
c->target_value = FREQ_QOS_MIN_DEFAULT_VALUE;
c->default_value = FREQ_QOS_MIN_DEFAULT_VALUE;
c->no_constraint_value = FREQ_QOS_MIN_DEFAULT_VALUE;
c->type = PM_QOS_MAX;
c->notifiers = &qos->min_freq_notifiers;
BLOCKING_INIT_NOTIFIER_HEAD(c->notifiers);
c = &qos->max_freq;
plist_head_init(&c->list);
c->target_value = FREQ_QOS_MAX_DEFAULT_VALUE;
c->default_value = FREQ_QOS_MAX_DEFAULT_VALUE;
c->no_constraint_value = FREQ_QOS_MAX_DEFAULT_VALUE;
c->type = PM_QOS_MIN;
c->notifiers = &qos->max_freq_notifiers;
BLOCKING_INIT_NOTIFIER_HEAD(c->notifiers);
}
/**
* freq_qos_read_value - Get frequency QoS constraint for a given list.
* @qos: Constraints to evaluate.
* @type: QoS request type.
*/
s32 freq_qos_read_value(struct freq_constraints *qos,
enum freq_qos_req_type type)
{
s32 ret;
switch (type) {
case FREQ_QOS_MIN:
ret = IS_ERR_OR_NULL(qos) ?
FREQ_QOS_MIN_DEFAULT_VALUE :
pm_qos_read_value(&qos->min_freq);
break;
case FREQ_QOS_MAX:
ret = IS_ERR_OR_NULL(qos) ?
FREQ_QOS_MAX_DEFAULT_VALUE :
pm_qos_read_value(&qos->max_freq);
break;
default:
WARN_ON(1);
ret = 0;
}
return ret;
}
/**
* freq_qos_apply - Add/modify/remove frequency QoS request.
* @req: Constraint request to apply.
* @action: Action to perform (add/update/remove).
* @value: Value to assign to the QoS request.
*
* This is only meant to be called from inside pm_qos, not drivers.
*/
int freq_qos_apply(struct freq_qos_request *req,
enum pm_qos_req_action action, s32 value)
{
int ret;
switch(req->type) {
case FREQ_QOS_MIN:
ret = pm_qos_update_target(&req->qos->min_freq, &req->pnode,
action, value);
break;
case FREQ_QOS_MAX:
ret = pm_qos_update_target(&req->qos->max_freq, &req->pnode,
action, value);
break;
default:
ret = -EINVAL;
}
return ret;
}
/**
* freq_qos_add_request - Insert new frequency QoS request into a given list.
* @qos: Constraints to update.
* @req: Preallocated request object.
* @type: Request type.
* @value: Request value.
*
* Insert a new entry into the @qos list of requests, recompute the effective
* QoS constraint value for that list and initialize the @req object. The
* caller needs to save that object for later use in updates and removal.
*
* Return 1 if the effective constraint value has changed, 0 if the effective
* constraint value has not changed, or a negative error code on failures.
*/
int freq_qos_add_request(struct freq_constraints *qos,
struct freq_qos_request *req,
enum freq_qos_req_type type, s32 value)
{
int ret;
if (IS_ERR_OR_NULL(qos) || !req || freq_qos_value_invalid(value))
return -EINVAL;
if (WARN(freq_qos_request_active(req),
"%s() called for active request\n", __func__))
return -EINVAL;
req->qos = qos;
req->type = type;
ret = freq_qos_apply(req, PM_QOS_ADD_REQ, value);
if (ret < 0) {
req->qos = NULL;
req->type = 0;
}
return ret;
}
EXPORT_SYMBOL_GPL(freq_qos_add_request);
/**
* freq_qos_update_request - Modify existing frequency QoS request.
* @req: Request to modify.
* @new_value: New request value.
*
* Update an existing frequency QoS request along with the effective constraint
* value for the list of requests it belongs to.
*
* Return 1 if the effective constraint value has changed, 0 if the effective
* constraint value has not changed, or a negative error code on failures.
*/
int freq_qos_update_request(struct freq_qos_request *req, s32 new_value)
{
if (!req || freq_qos_value_invalid(new_value))
return -EINVAL;
if (WARN(!freq_qos_request_active(req),
"%s() called for unknown object\n", __func__))
return -EINVAL;
if (req->pnode.prio == new_value)
return 0;
return freq_qos_apply(req, PM_QOS_UPDATE_REQ, new_value);
}
EXPORT_SYMBOL_GPL(freq_qos_update_request);
/**
* freq_qos_remove_request - Remove frequency QoS request from its list.
* @req: Request to remove.
*
* Remove the given frequency QoS request from the list of constraints it
* belongs to and recompute the effective constraint value for that list.
*
* Return 1 if the effective constraint value has changed, 0 if the effective
* constraint value has not changed, or a negative error code on failures.
*/
int freq_qos_remove_request(struct freq_qos_request *req)
{
int ret;
if (!req)
return -EINVAL;
if (WARN(!freq_qos_request_active(req),
"%s() called for unknown object\n", __func__))
return -EINVAL;
ret = freq_qos_apply(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
req->qos = NULL;
req->type = 0;
return ret;
}
EXPORT_SYMBOL_GPL(freq_qos_remove_request);
/**
* freq_qos_add_notifier - Add frequency QoS change notifier.
* @qos: List of requests to add the notifier to.
* @type: Request type.
* @notifier: Notifier block to add.
*/
int freq_qos_add_notifier(struct freq_constraints *qos,
enum freq_qos_req_type type,
struct notifier_block *notifier)
{
int ret;
if (IS_ERR_OR_NULL(qos) || !notifier)
return -EINVAL;
switch (type) {
case FREQ_QOS_MIN:
ret = blocking_notifier_chain_register(qos->min_freq.notifiers,
notifier);
break;
case FREQ_QOS_MAX:
ret = blocking_notifier_chain_register(qos->max_freq.notifiers,
notifier);
break;
default:
WARN_ON(1);
ret = -EINVAL;
}
return ret;
}
EXPORT_SYMBOL_GPL(freq_qos_add_notifier);
/**
* freq_qos_remove_notifier - Remove frequency QoS change notifier.
* @qos: List of requests to remove the notifier from.
* @type: Request type.
* @notifier: Notifier block to remove.
*/
int freq_qos_remove_notifier(struct freq_constraints *qos,
enum freq_qos_req_type type,
struct notifier_block *notifier)
{
int ret;
if (IS_ERR_OR_NULL(qos) || !notifier)
return -EINVAL;
switch (type) {
case FREQ_QOS_MIN:
ret = blocking_notifier_chain_unregister(qos->min_freq.notifiers,
notifier);
break;
case FREQ_QOS_MAX:
ret = blocking_notifier_chain_unregister(qos->max_freq.notifiers,
notifier);
break;
default:
WARN_ON(1);
ret = -EINVAL;
}
return ret;
}
EXPORT_SYMBOL_GPL(freq_qos_remove_notifier);
// SPDX-License-Identifier: GPL-2.0-only
/*
* Media entity
*
* Copyright (C) 2010 Nokia Corporation
*
* Contacts: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
* Sakari Ailus <sakari.ailus@iki.fi>
*/
#include <linux/bitmap.h>
#include <linux/list.h>
#include <linux/property.h>
#include <linux/slab.h>
#include <media/media-entity.h>
#include <media/media-device.h>
static inline const char *intf_type(struct media_interface *intf)
{
switch (intf->type) {
case MEDIA_INTF_T_DVB_FE:
return "dvb-frontend";
case MEDIA_INTF_T_DVB_DEMUX:
return "dvb-demux";
case MEDIA_INTF_T_DVB_DVR:
return "dvb-dvr";
case MEDIA_INTF_T_DVB_CA:
return "dvb-ca";
case MEDIA_INTF_T_DVB_NET:
return "dvb-net";
case MEDIA_INTF_T_V4L_VIDEO:
return "v4l-video";
case MEDIA_INTF_T_V4L_VBI:
return "v4l-vbi";
case MEDIA_INTF_T_V4L_RADIO:
return "v4l-radio";
case MEDIA_INTF_T_V4L_SUBDEV:
return "v4l-subdev";
case MEDIA_INTF_T_V4L_SWRADIO:
return "v4l-swradio";
case MEDIA_INTF_T_V4L_TOUCH:
return "v4l-touch";
default:
return "unknown-intf";
}
};
static inline const char *link_type_name(struct media_link *link)
{
switch (link->flags & MEDIA_LNK_FL_LINK_TYPE) {
case MEDIA_LNK_FL_DATA_LINK:
return "data";
case MEDIA_LNK_FL_INTERFACE_LINK:
return "interface";
case MEDIA_LNK_FL_ANCILLARY_LINK:
return "ancillary";
default:
return "unknown";
}
}
__must_check int media_entity_enum_init(struct media_entity_enum *ent_enum,
struct media_device *mdev)
{
int idx_max;
idx_max = ALIGN(mdev->entity_internal_idx_max + 1, BITS_PER_LONG);
ent_enum->bmap = bitmap_zalloc(idx_max, GFP_KERNEL);
if (!ent_enum->bmap)
return -ENOMEM;
ent_enum->idx_max = idx_max;
return 0;
}
EXPORT_SYMBOL_GPL(media_entity_enum_init);
void media_entity_enum_cleanup(struct media_entity_enum *ent_enum)
{
bitmap_free(ent_enum->bmap);
}
EXPORT_SYMBOL_GPL(media_entity_enum_cleanup);
/**
* dev_dbg_obj - Prints in debug mode a change on some object
*
* @event_name: Name of the event to report. Could be __func__
* @gobj: Pointer to the object
*
* Enabled only if DEBUG or CONFIG_DYNAMIC_DEBUG. Otherwise, it
* won't produce any code.
*/
static void dev_dbg_obj(const char *event_name, struct media_gobj *gobj)
{
#if defined(DEBUG) || defined (CONFIG_DYNAMIC_DEBUG)
switch (media_type(gobj)) {
case MEDIA_GRAPH_ENTITY:
dev_dbg(gobj->mdev->dev,
"%s id %u: entity '%s'\n",
event_name, media_id(gobj),
gobj_to_entity(gobj)->name);
break;
case MEDIA_GRAPH_LINK:
{
struct media_link *link = gobj_to_link(gobj);
dev_dbg(gobj->mdev->dev,
"%s id %u: %s link id %u ==> id %u\n",
event_name, media_id(gobj), link_type_name(link),
media_id(link->gobj0),
media_id(link->gobj1));
break;
}
case MEDIA_GRAPH_PAD:
{
struct media_pad *pad = gobj_to_pad(gobj);
dev_dbg(gobj->mdev->dev,
"%s id %u: %s%spad '%s':%d\n",
event_name, media_id(gobj),
pad->flags & MEDIA_PAD_FL_SINK ? "sink " : "",
pad->flags & MEDIA_PAD_FL_SOURCE ? "source " : "",
pad->entity->name, pad->index);
break;
}
case MEDIA_GRAPH_INTF_DEVNODE:
{
struct media_interface *intf = gobj_to_intf(gobj);
struct media_intf_devnode *devnode = intf_to_devnode(intf);
dev_dbg(gobj->mdev->dev,
"%s id %u: intf_devnode %s - major: %d, minor: %d\n",
event_name, media_id(gobj),
intf_type(intf),
devnode->major, devnode->minor);
break;
}
}
#endif
}
void media_gobj_create(struct media_device *mdev,
enum media_gobj_type type,
struct media_gobj *gobj)
{
BUG_ON(!mdev);
gobj->mdev = mdev;
/* Create a per-type unique object ID */
gobj->id = media_gobj_gen_id(type, ++mdev->id);
switch (type) {
case MEDIA_GRAPH_ENTITY:
list_add_tail(&gobj->list, &mdev->entities);
break;
case MEDIA_GRAPH_PAD:
list_add_tail(&gobj->list, &mdev->pads);
break;
case MEDIA_GRAPH_LINK:
list_add_tail(&gobj->list, &mdev->links);
break;
case MEDIA_GRAPH_INTF_DEVNODE:
list_add_tail(&gobj->list, &mdev->interfaces);
break;
}
mdev->topology_version++;
dev_dbg_obj(__func__, gobj);
}
void media_gobj_destroy(struct media_gobj *gobj)
{
/* Do nothing if the object is not linked. */
if (gobj->mdev == NULL)
return;
dev_dbg_obj(__func__, gobj);
gobj->mdev->topology_version++;
/* Remove the object from mdev list */
list_del(&gobj->list);
gobj->mdev = NULL;
}
/*
* TODO: Get rid of this.
*/
#define MEDIA_ENTITY_MAX_PADS 512
int media_entity_pads_init(struct media_entity *entity, u16 num_pads,
struct media_pad *pads)
{
struct media_device *mdev = entity->graph_obj.mdev;
struct media_pad *iter;
unsigned int i = 0;
int ret = 0;
if (num_pads >= MEDIA_ENTITY_MAX_PADS)
return -E2BIG;
entity->num_pads = num_pads;
entity->pads = pads;
if (mdev)
mutex_lock(&mdev->graph_mutex);
media_entity_for_each_pad(entity, iter) {
iter->entity = entity;
iter->index = i++;
if (hweight32(iter->flags & (MEDIA_PAD_FL_SINK |
MEDIA_PAD_FL_SOURCE)) != 1) {
ret = -EINVAL;
break;
}
if (mdev)
media_gobj_create(mdev, MEDIA_GRAPH_PAD,
&iter->graph_obj);
}
if (ret && mdev) {
media_entity_for_each_pad(entity, iter)
media_gobj_destroy(&iter->graph_obj);
}
if (mdev)
mutex_unlock(&mdev->graph_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(media_entity_pads_init);
/* -----------------------------------------------------------------------------
* Graph traversal
*/
/**
* media_entity_has_pad_interdep - Check interdependency between two pads
*
* @entity: The entity
* @pad0: The first pad index
* @pad1: The second pad index
*
* This function checks the interdependency inside the entity between @pad0
* and @pad1. If two pads are interdependent they are part of the same pipeline
* and enabling one of the pads means that the other pad will become "locked"
* and doesn't allow configuration changes.
*
* This function uses the &media_entity_operations.has_pad_interdep() operation
* to check the dependency inside the entity between @pad0 and @pad1. If the
* has_pad_interdep operation is not implemented, all pads of the entity are
* considered to be interdependent.
*
* One of @pad0 and @pad1 must be a sink pad and the other one a source pad.
* The function returns false if both pads are sinks or sources.
*
* The caller must hold entity->graph_obj.mdev->mutex.
*
* Return: true if the pads are connected internally and false otherwise.
*/
static bool media_entity_has_pad_interdep(struct media_entity *entity,
unsigned int pad0, unsigned int pad1)
{
if (pad0 >= entity->num_pads || pad1 >= entity->num_pads)
return false;
if (entity->pads[pad0].flags & entity->pads[pad1].flags &
(MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_SOURCE))
return false;
if (!entity->ops || !entity->ops->has_pad_interdep)
return true;
return entity->ops->has_pad_interdep(entity, pad0, pad1);
}
static struct media_entity *
media_entity_other(struct media_entity *entity, struct media_link *link)
{
if (link->source->entity == entity)
return link->sink->entity;
else
return link->source->entity;
}
/* push an entity to traversal stack */
static void stack_push(struct media_graph *graph,
struct media_entity *entity)
{
if (graph->top == MEDIA_ENTITY_ENUM_MAX_DEPTH - 1) {
WARN_ON(1);
return;
}
graph->top++;
graph->stack[graph->top].link = entity->links.next;
graph->stack[graph->top].entity = entity;
}
static struct media_entity *stack_pop(struct media_graph *graph)
{
struct media_entity *entity;
entity = graph->stack[graph->top].entity;
graph->top--;
return entity;
}
#define link_top(en) ((en)->stack[(en)->top].link)
#define stack_top(en) ((en)->stack[(en)->top].entity)
/**
* media_graph_walk_init - Allocate resources for graph walk
* @graph: Media graph structure that will be used to walk the graph
* @mdev: Media device
*
* Reserve resources for graph walk in media device's current
* state. The memory must be released using
* media_graph_walk_cleanup().
*
* Returns error on failure, zero on success.
*/
__must_check int media_graph_walk_init(
struct media_graph *graph, struct media_device *mdev)
{
return media_entity_enum_init(&graph->ent_enum, mdev);
}
EXPORT_SYMBOL_GPL(media_graph_walk_init);
/**
* media_graph_walk_cleanup - Release resources related to graph walking
* @graph: Media graph structure that was used to walk the graph
*/
void media_graph_walk_cleanup(struct media_graph *graph)
{
media_entity_enum_cleanup(&graph->ent_enum);
}
EXPORT_SYMBOL_GPL(media_graph_walk_cleanup);
void media_graph_walk_start(struct media_graph *graph,
struct media_entity *entity)
{
media_entity_enum_zero(&graph->ent_enum);
media_entity_enum_set(&graph->ent_enum, entity);
graph->top = 0;
graph->stack[graph->top].entity = NULL;
stack_push(graph, entity);
dev_dbg(entity->graph_obj.mdev->dev,
"begin graph walk at '%s'\n", entity->name);
}
EXPORT_SYMBOL_GPL(media_graph_walk_start);
static void media_graph_walk_iter(struct media_graph *graph)
{
struct media_entity *entity = stack_top(graph);
struct media_link *link;
struct media_entity *next;
link = list_entry(link_top(graph), typeof(*link), list);
/* If the link is not a data link, don't follow it */
if ((link->flags & MEDIA_LNK_FL_LINK_TYPE) != MEDIA_LNK_FL_DATA_LINK) {
link_top(graph) = link_top(graph)->next;
return;
}
/* The link is not enabled so we do not follow. */
if (!(link->flags & MEDIA_LNK_FL_ENABLED)) {
link_top(graph) = link_top(graph)->next;
dev_dbg(entity->graph_obj.mdev->dev,
"walk: skipping disabled link '%s':%u -> '%s':%u\n",
link->source->entity->name, link->source->index,
link->sink->entity->name, link->sink->index);
return;
}
/* Get the entity at the other end of the link. */
next = media_entity_other(entity, link);
/* Has the entity already been visited? */
if (media_entity_enum_test_and_set(&graph->ent_enum, next)) {
link_top(graph) = link_top(graph)->next;
dev_dbg(entity->graph_obj.mdev->dev,
"walk: skipping entity '%s' (already seen)\n",
next->name);
return;
}
/* Push the new entity to stack and start over. */
link_top(graph) = link_top(graph)->next;
stack_push(graph, next);
dev_dbg(entity->graph_obj.mdev->dev, "walk: pushing '%s' on stack\n",
next->name);
lockdep_assert_held(&entity->graph_obj.mdev->graph_mutex);
}
struct media_entity *media_graph_walk_next(struct media_graph *graph)
{
struct media_entity *entity;
if (stack_top(graph) == NULL)
return NULL;
/*
* Depth first search. Push entity to stack and continue from
* top of the stack until no more entities on the level can be
* found.
*/
while (link_top(graph) != &stack_top(graph)->links)
media_graph_walk_iter(graph);
entity = stack_pop(graph);
dev_dbg(entity->graph_obj.mdev->dev,
"walk: returning entity '%s'\n", entity->name);
return entity;
}
EXPORT_SYMBOL_GPL(media_graph_walk_next);
/* -----------------------------------------------------------------------------
* Pipeline management
*/
/*
* The pipeline traversal stack stores pads that are reached during graph
* traversal, with a list of links to be visited to continue the traversal.
* When a new pad is reached, an entry is pushed on the top of the stack and
* points to the incoming pad and the first link of the entity.
*
* To find further pads in the pipeline, the traversal algorithm follows
* internal pad dependencies in the entity, and then links in the graph. It
* does so by iterating over all links of the entity, and following enabled
* links that originate from a pad that is internally connected to the incoming
* pad, as reported by the media_entity_has_pad_interdep() function.
*/
/**
* struct media_pipeline_walk_entry - Entry in the pipeline traversal stack
*
* @pad: The media pad being visited
* @links: Links left to be visited
*/
struct media_pipeline_walk_entry {
struct media_pad *pad;
struct list_head *links;
};
/**
* struct media_pipeline_walk - State used by the media pipeline traversal
* algorithm
*
* @mdev: The media device
* @stack: Depth-first search stack
* @stack.size: Number of allocated entries in @stack.entries
* @stack.top: Index of the top stack entry (-1 if the stack is empty)
* @stack.entries: Stack entries
*/
struct media_pipeline_walk {
struct media_device *mdev;
struct {
unsigned int size;
int top;
struct media_pipeline_walk_entry *entries;
} stack;
};
#define MEDIA_PIPELINE_STACK_GROW_STEP 16
static struct media_pipeline_walk_entry *
media_pipeline_walk_top(struct media_pipeline_walk *walk)
{
return &walk->stack.entries[walk->stack.top];
}
static bool media_pipeline_walk_empty(struct media_pipeline_walk *walk)
{
return walk->stack.top == -1;
}
/* Increase the stack size by MEDIA_PIPELINE_STACK_GROW_STEP elements. */
static int media_pipeline_walk_resize(struct media_pipeline_walk *walk)
{
struct media_pipeline_walk_entry *entries;
unsigned int new_size;
/* Safety check, to avoid stack overflows in case of bugs. */
if (walk->stack.size >= 256)
return -E2BIG;
new_size = walk->stack.size + MEDIA_PIPELINE_STACK_GROW_STEP;
entries = krealloc(walk->stack.entries,
new_size * sizeof(*walk->stack.entries),
GFP_KERNEL);
if (!entries)
return -ENOMEM;
walk->stack.entries = entries;
walk->stack.size = new_size;
return 0;
}
/* Push a new entry on the stack. */
static int media_pipeline_walk_push(struct media_pipeline_walk *walk,
struct media_pad *pad)
{
struct media_pipeline_walk_entry *entry;
int ret;
if (walk->stack.top + 1 >= walk->stack.size) {
ret = media_pipeline_walk_resize(walk);
if (ret)
return ret;
}
walk->stack.top++;
entry = media_pipeline_walk_top(walk);
entry->pad = pad;
entry->links = pad->entity->links.next;
dev_dbg(walk->mdev->dev,
"media pipeline: pushed entry %u: '%s':%u\n",
walk->stack.top, pad->entity->name, pad->index);
return 0;
}
/*
* Move the top entry link cursor to the next link. If all links of the entry
* have been visited, pop the entry itself. Return true if the entry has been
* popped.
*/
static bool media_pipeline_walk_pop(struct media_pipeline_walk *walk)
{
struct media_pipeline_walk_entry *entry;
if (WARN_ON(walk->stack.top < 0))
return false;
entry = media_pipeline_walk_top(walk);
if (entry->links->next == &entry->pad->entity->links) {
dev_dbg(walk->mdev->dev,
"media pipeline: entry %u has no more links, popping\n",
walk->stack.top);
walk->stack.top--;
return true;
}
entry->links = entry->links->next;
dev_dbg(walk->mdev->dev,
"media pipeline: moved entry %u to next link\n",
walk->stack.top);
return false;
}
/* Free all memory allocated while walking the pipeline. */
static void media_pipeline_walk_destroy(struct media_pipeline_walk *walk)
{
kfree(walk->stack.entries);
}
/* Add a pad to the pipeline and push it to the stack. */
static int media_pipeline_add_pad(struct media_pipeline *pipe,
struct media_pipeline_walk *walk,
struct media_pad *pad)
{
struct media_pipeline_pad *ppad;
list_for_each_entry(ppad, &pipe->pads, list) {
if (ppad->pad == pad) {
dev_dbg(pad->graph_obj.mdev->dev,
"media pipeline: already contains pad '%s':%u\n",
pad->entity->name, pad->index);
return 0;
}
}
ppad = kzalloc(sizeof(*ppad), GFP_KERNEL);
if (!ppad)
return -ENOMEM;
ppad->pipe = pipe;
ppad->pad = pad;
list_add_tail(&ppad->list, &pipe->pads);
dev_dbg(pad->graph_obj.mdev->dev,
"media pipeline: added pad '%s':%u\n",
pad->entity->name, pad->index);
return media_pipeline_walk_push(walk, pad);
}
/* Explore the next link of the entity at the top of the stack. */
static int media_pipeline_explore_next_link(struct media_pipeline *pipe,
struct media_pipeline_walk *walk)
{
struct media_pipeline_walk_entry *entry = media_pipeline_walk_top(walk);
struct media_pad *origin;
struct media_link *link;
struct media_pad *local;
struct media_pad *remote;
bool last_link;
int ret;
origin = entry->pad;
link = list_entry(entry->links, typeof(*link), list);
last_link = media_pipeline_walk_pop(walk);
dev_dbg(walk->mdev->dev,
"media pipeline: exploring link '%s':%u -> '%s':%u\n",
link->source->entity->name, link->source->index,
link->sink->entity->name, link->sink->index);
/* Get the local pad and remote pad. */
if (link->source->entity == origin->entity) {
local = link->source;
remote = link->sink;
} else {
local = link->sink;
remote = link->source;
}
/*
* Skip links that originate from a different pad than the incoming pad
* that is not connected internally in the entity to the incoming pad.
*/
if (origin != local &&
!media_entity_has_pad_interdep(origin->entity, origin->index,
local->index)) {
dev_dbg(walk->mdev->dev,
"media pipeline: skipping link (no route)\n");
goto done;
}
/*
* Add the local pad of the link to the pipeline and push it to the
* stack, if not already present.
*/
ret = media_pipeline_add_pad(pipe, walk, local);
if (ret)
return ret;
/* Similarly, add the remote pad, but only if the link is enabled. */
if (!(link->flags & MEDIA_LNK_FL_ENABLED)) {
dev_dbg(walk->mdev->dev,
"media pipeline: skipping link (disabled)\n");
goto done;
}
ret = media_pipeline_add_pad(pipe, walk, remote);
if (ret)
return ret;
done:
/*
* If we're done iterating over links, iterate over pads of the entity.
* This is necessary to discover pads that are not connected with any
* link. Those are dead ends from a pipeline exploration point of view,
* but are still part of the pipeline and need to be added to enable
* proper validation.
*/
if (!last_link)
return 0;
dev_dbg(walk->mdev->dev,
"media pipeline: adding unconnected pads of '%s'\n",
local->entity->name);
media_entity_for_each_pad(origin->entity, local) {
/*
* Skip the origin pad (already handled), pad that have links
* (already discovered through iterating over links) and pads
* not internally connected.
*/
if (origin == local || !local->num_links ||
!media_entity_has_pad_interdep(origin->entity, origin->index,
local->index))
continue;
ret = media_pipeline_add_pad(pipe, walk, local);
if (ret)
return ret;
}
return 0;
}
static void media_pipeline_cleanup(struct media_pipeline *pipe)
{
while (!list_empty(&pipe->pads)) {
struct media_pipeline_pad *ppad;
ppad = list_first_entry(&pipe->pads, typeof(*ppad), list);
list_del(&ppad->list);
kfree(ppad);
}
}
static int media_pipeline_populate(struct media_pipeline *pipe,
struct media_pad *pad)
{
struct media_pipeline_walk walk = { };
struct media_pipeline_pad *ppad;
int ret;
/*
* Populate the media pipeline by walking the media graph, starting
* from @pad.
*/
INIT_LIST_HEAD(&pipe->pads);
pipe->mdev = pad->graph_obj.mdev;
walk.mdev = pipe->mdev;
walk.stack.top = -1;
ret = media_pipeline_add_pad(pipe, &walk, pad);
if (ret)
goto done;
/*
* Use a depth-first search algorithm: as long as the stack is not
* empty, explore the next link of the top entry. The
* media_pipeline_explore_next_link() function will either move to the
* next link, pop the entry if fully visited, or add new entries on
* top.
*/
while (!media_pipeline_walk_empty(&walk)) {
ret = media_pipeline_explore_next_link(pipe, &walk);
if (ret)
goto done;
}
dev_dbg(pad->graph_obj.mdev->dev,
"media pipeline populated, found pads:\n");
list_for_each_entry(ppad, &pipe->pads, list)
dev_dbg(pad->graph_obj.mdev->dev, "- '%s':%u\n",
ppad->pad->entity->name, ppad->pad->index);
WARN_ON(walk.stack.top != -1);
ret = 0;
done:
media_pipeline_walk_destroy(&walk);
if (ret)
media_pipeline_cleanup(pipe);
return ret;
}
__must_check int __media_pipeline_start(struct media_pad *pad,
struct media_pipeline *pipe)
{
struct media_device *mdev = pad->graph_obj.mdev;
struct media_pipeline_pad *err_ppad;
struct media_pipeline_pad *ppad;
int ret;
lockdep_assert_held(&mdev->graph_mutex);
/*
* If the pad is already part of a pipeline, that pipeline must be the
* same as the pipe given to media_pipeline_start().
*/
if (WARN_ON(pad->pipe && pad->pipe != pipe))
return -EINVAL;
/*
* If the pipeline has already been started, it is guaranteed to be
* valid, so just increase the start count.
*/
if (pipe->start_count) {
pipe->start_count++;
return 0;
}
/*
* Populate the pipeline. This populates the media_pipeline pads list
* with media_pipeline_pad instances for each pad found during graph
* walk.
*/
ret = media_pipeline_populate(pipe, pad);
if (ret)
return ret;
/*
* Now that all the pads in the pipeline have been gathered, perform
* the validation steps.
*/
list_for_each_entry(ppad, &pipe->pads, list) {
struct media_pad *pad = ppad->pad;
struct media_entity *entity = pad->entity;
bool has_enabled_link = false;
struct media_link *link;
dev_dbg(mdev->dev, "Validating pad '%s':%u\n", pad->entity->name,
pad->index);
/*
* 1. Ensure that the pad doesn't already belong to a different
* pipeline.
*/
if (pad->pipe) {
dev_dbg(mdev->dev, "Failed to start pipeline: pad '%s':%u busy\n",
pad->entity->name, pad->index);
ret = -EBUSY;
goto error;
}
/*
* 2. Validate all active links whose sink is the current pad.
* Validation of the source pads is performed in the context of
* the connected sink pad to avoid duplicating checks.
*/
for_each_media_entity_data_link(entity, link) {
/* Skip links unrelated to the current pad. */
if (link->sink != pad && link->source != pad)
continue;
/* Record if the pad has links and enabled links. */
if (link->flags & MEDIA_LNK_FL_ENABLED)
has_enabled_link = true;
/*
* Validate the link if it's enabled and has the
* current pad as its sink.
*/
if (!(link->flags & MEDIA_LNK_FL_ENABLED))
continue;
if (link->sink != pad)
continue;
if (!entity->ops || !entity->ops->link_validate)
continue;
ret = entity->ops->link_validate(link);
if (ret) {
dev_dbg(mdev->dev,
"Link '%s':%u -> '%s':%u failed validation: %d\n",
link->source->entity->name,
link->source->index,
link->sink->entity->name,
link->sink->index, ret);
goto error;
}
dev_dbg(mdev->dev,
"Link '%s':%u -> '%s':%u is valid\n",
link->source->entity->name,
link->source->index,
link->sink->entity->name,
link->sink->index);
}
/*
* 3. If the pad has the MEDIA_PAD_FL_MUST_CONNECT flag set,
* ensure that it has either no link or an enabled link.
*/
if ((pad->flags & MEDIA_PAD_FL_MUST_CONNECT) &&
!has_enabled_link) {
dev_dbg(mdev->dev,
"Pad '%s':%u must be connected by an enabled link\n",
pad->entity->name, pad->index);
ret = -ENOLINK;
goto error;
}
/* Validation passed, store the pipe pointer in the pad. */
pad->pipe = pipe;
}
pipe->start_count++;
return 0;
error:
/*
* Link validation on graph failed. We revert what we did and
* return the error.
*/
list_for_each_entry(err_ppad, &pipe->pads, list) {
if (err_ppad == ppad)
break;
err_ppad->pad->pipe = NULL;
}
media_pipeline_cleanup(pipe);
return ret;
}
EXPORT_SYMBOL_GPL(__media_pipeline_start);
__must_check int media_pipeline_start(struct media_pad *pad,
struct media_pipeline *pipe)
{
struct media_device *mdev = pad->graph_obj.mdev;
int ret;
mutex_lock(&mdev->graph_mutex);
ret = __media_pipeline_start(pad, pipe);
mutex_unlock(&mdev->graph_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(media_pipeline_start);
void __media_pipeline_stop(struct media_pad *pad)
{
struct media_pipeline *pipe = pad->pipe;
struct media_pipeline_pad *ppad;
/*
* If the following check fails, the driver has performed an
* unbalanced call to media_pipeline_stop()
*/
if (WARN_ON(!pipe))
return;
if (--pipe->start_count)
return;
list_for_each_entry(ppad, &pipe->pads, list)
ppad->pad->pipe = NULL;
media_pipeline_cleanup(pipe);
if (pipe->allocated)
kfree(pipe);
}
EXPORT_SYMBOL_GPL(__media_pipeline_stop);
void media_pipeline_stop(struct media_pad *pad)
{
struct media_device *mdev = pad->graph_obj.mdev;
mutex_lock(&mdev->graph_mutex);
__media_pipeline_stop(pad);
mutex_unlock(&mdev->graph_mutex);
}
EXPORT_SYMBOL_GPL(media_pipeline_stop);
__must_check int media_pipeline_alloc_start(struct media_pad *pad)
{
struct media_device *mdev = pad->graph_obj.mdev;
struct media_pipeline *new_pipe = NULL;
struct media_pipeline *pipe;
int ret;
mutex_lock(&mdev->graph_mutex);
/*
* Is the pad already part of a pipeline? If not, we need to allocate
* a pipe.
*/
pipe = media_pad_pipeline(pad);
if (!pipe) {
new_pipe = kzalloc(sizeof(*new_pipe), GFP_KERNEL);
if (!new_pipe) {
ret = -ENOMEM;
goto out;
}
pipe = new_pipe;
pipe->allocated = true;
}
ret = __media_pipeline_start(pad, pipe);
if (ret)
kfree(new_pipe);
out:
mutex_unlock(&mdev->graph_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(media_pipeline_alloc_start);
struct media_pad *
__media_pipeline_pad_iter_next(struct media_pipeline *pipe,
struct media_pipeline_pad_iter *iter,
struct media_pad *pad)
{
if (!pad)
iter->cursor = pipe->pads.next;
if (iter->cursor == &pipe->pads)
return NULL;
pad = list_entry(iter->cursor, struct media_pipeline_pad, list)->pad;
iter->cursor = iter->cursor->next;
return pad;
}
EXPORT_SYMBOL_GPL(__media_pipeline_pad_iter_next);
int media_pipeline_entity_iter_init(struct media_pipeline *pipe,
struct media_pipeline_entity_iter *iter)
{
return media_entity_enum_init(&iter->ent_enum, pipe->mdev);
}
EXPORT_SYMBOL_GPL(media_pipeline_entity_iter_init);
void media_pipeline_entity_iter_cleanup(struct media_pipeline_entity_iter *iter)
{
media_entity_enum_cleanup(&iter->ent_enum);
}
EXPORT_SYMBOL_GPL(media_pipeline_entity_iter_cleanup);
struct media_entity *
__media_pipeline_entity_iter_next(struct media_pipeline *pipe,
struct media_pipeline_entity_iter *iter,
struct media_entity *entity)
{
if (!entity)
iter->cursor = pipe->pads.next;
while (iter->cursor != &pipe->pads) {
struct media_pipeline_pad *ppad;
struct media_entity *entity;
ppad = list_entry(iter->cursor, struct media_pipeline_pad, list);
entity = ppad->pad->entity;
iter->cursor = iter->cursor->next;
if (!media_entity_enum_test_and_set(&iter->ent_enum, entity))
return entity;
}
return NULL;
}
EXPORT_SYMBOL_GPL(__media_pipeline_entity_iter_next);
/* -----------------------------------------------------------------------------
* Links management
*/
static struct media_link *media_add_link(struct list_head *head)
{
struct media_link *link;
link = kzalloc(sizeof(*link), GFP_KERNEL);
if (link == NULL)
return NULL;
list_add_tail(&link->list, head);
return link;
}
static void __media_entity_remove_link(struct media_entity *entity,
struct media_link *link)
{
struct media_link *rlink, *tmp;
struct media_entity *remote;
/* Remove the reverse links for a data link. */
if ((link->flags & MEDIA_LNK_FL_LINK_TYPE) == MEDIA_LNK_FL_DATA_LINK) {
link->source->num_links--;
link->sink->num_links--;
if (link->source->entity == entity)
remote = link->sink->entity;
else
remote = link->source->entity;
list_for_each_entry_safe(rlink, tmp, &remote->links, list) {
if (rlink != link->reverse)
continue;
if (link->source->entity == entity)
remote->num_backlinks--;
/* Remove the remote link */
list_del(&rlink->list);
media_gobj_destroy(&rlink->graph_obj);
kfree(rlink);
if (--remote->num_links == 0)
break;
}
}
list_del(&link->list);
media_gobj_destroy(&link->graph_obj);
kfree(link);
}
int media_get_pad_index(struct media_entity *entity, u32 pad_type,
enum media_pad_signal_type sig_type)
{
unsigned int i;
if (!entity)
return -EINVAL;
for (i = 0; i < entity->num_pads; i++) {
if ((entity->pads[i].flags &
(MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_SOURCE)) != pad_type)
continue;
if (entity->pads[i].sig_type == sig_type)
return i;
}
return -EINVAL;
}
EXPORT_SYMBOL_GPL(media_get_pad_index);
int
media_create_pad_link(struct media_entity *source, u16 source_pad,
struct media_entity *sink, u16 sink_pad, u32 flags)
{
struct media_link *link;
struct media_link *backlink;
if (flags & MEDIA_LNK_FL_LINK_TYPE)
return -EINVAL;
flags |= MEDIA_LNK_FL_DATA_LINK;
if (WARN_ON(!source || !sink) ||
WARN_ON(source_pad >= source->num_pads) ||
WARN_ON(sink_pad >= sink->num_pads))
return -EINVAL;
if (WARN_ON(!(source->pads[source_pad].flags & MEDIA_PAD_FL_SOURCE)))
return -EINVAL;
if (WARN_ON(!(sink->pads[sink_pad].flags & MEDIA_PAD_FL_SINK)))
return -EINVAL;
link = media_add_link(&source->links);
if (link == NULL)
return -ENOMEM;
link->source = &source->pads[source_pad];
link->sink = &sink->pads[sink_pad];
link->flags = flags;
/* Initialize graph object embedded at the new link */
media_gobj_create(source->graph_obj.mdev, MEDIA_GRAPH_LINK,
&link->graph_obj);
/* Create the backlink. Backlinks are used to help graph traversal and
* are not reported to userspace.
*/
backlink = media_add_link(&sink->links);
if (backlink == NULL) {
__media_entity_remove_link(source, link);
return -ENOMEM;
}
backlink->source = &source->pads[source_pad];
backlink->sink = &sink->pads[sink_pad];
backlink->flags = flags;
backlink->is_backlink = true;
/* Initialize graph object embedded at the new link */
media_gobj_create(sink->graph_obj.mdev, MEDIA_GRAPH_LINK,
&backlink->graph_obj);
link->reverse = backlink;
backlink->reverse = link;
sink->num_backlinks++;
sink->num_links++;
source->num_links++;
link->source->num_links++;
link->sink->num_links++;
return 0;
}
EXPORT_SYMBOL_GPL(media_create_pad_link);
int media_create_pad_links(const struct media_device *mdev,
const u32 source_function,
struct media_entity *source,
const u16 source_pad,
const u32 sink_function,
struct media_entity *sink,
const u16 sink_pad,
u32 flags,
const bool allow_both_undefined)
{
struct media_entity *entity;
unsigned function;
int ret;
/* Trivial case: 1:1 relation */
if (source && sink)
return media_create_pad_link(source, source_pad,
sink, sink_pad, flags);
/* Worse case scenario: n:n relation */
if (!source && !sink) {
if (!allow_both_undefined)
return 0;
media_device_for_each_entity(source, mdev) {
if (source->function != source_function)
continue;
media_device_for_each_entity(sink, mdev) {
if (sink->function != sink_function)
continue;
ret = media_create_pad_link(source, source_pad,
sink, sink_pad,
flags);
if (ret)
return ret;
flags &= ~(MEDIA_LNK_FL_ENABLED |
MEDIA_LNK_FL_IMMUTABLE);
}
}
return 0;
}
/* Handle 1:n and n:1 cases */
if (source)
function = sink_function;
else
function = source_function;
media_device_for_each_entity(entity, mdev) {
if (entity->function != function)
continue;
if (source)
ret = media_create_pad_link(source, source_pad,
entity, sink_pad, flags);
else
ret = media_create_pad_link(entity, source_pad,
sink, sink_pad, flags);
if (ret)
return ret;
flags &= ~(MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE);
}
return 0;
}
EXPORT_SYMBOL_GPL(media_create_pad_links);
void __media_entity_remove_links(struct media_entity *entity)
{
struct media_link *link, *tmp;
list_for_each_entry_safe(link, tmp, &entity->links, list) __media_entity_remove_link(entity, link); entity->num_links = 0;
entity->num_backlinks = 0;
}
EXPORT_SYMBOL_GPL(__media_entity_remove_links);
void media_entity_remove_links(struct media_entity *entity)
{
struct media_device *mdev = entity->graph_obj.mdev;
/* Do nothing if the entity is not registered. */
if (mdev == NULL)
return;
mutex_lock(&mdev->graph_mutex);
__media_entity_remove_links(entity);
mutex_unlock(&mdev->graph_mutex);
}
EXPORT_SYMBOL_GPL(media_entity_remove_links);
static int __media_entity_setup_link_notify(struct media_link *link, u32 flags)
{
int ret;
/* Notify both entities. */
ret = media_entity_call(link->source->entity, link_setup,
link->source, link->sink, flags);
if (ret < 0 && ret != -ENOIOCTLCMD)
return ret;
ret = media_entity_call(link->sink->entity, link_setup,
link->sink, link->source, flags);
if (ret < 0 && ret != -ENOIOCTLCMD) {
media_entity_call(link->source->entity, link_setup,
link->source, link->sink, link->flags);
return ret;
}
link->flags = flags;
link->reverse->flags = link->flags;
return 0;
}
int __media_entity_setup_link(struct media_link *link, u32 flags)
{
const u32 mask = MEDIA_LNK_FL_ENABLED;
struct media_device *mdev;
struct media_pad *source, *sink;
int ret = -EBUSY;
if (link == NULL)
return -EINVAL;
/* The non-modifiable link flags must not be modified. */
if ((link->flags & ~mask) != (flags & ~mask))
return -EINVAL;
if (link->flags & MEDIA_LNK_FL_IMMUTABLE)
return link->flags == flags ? 0 : -EINVAL;
if (link->flags == flags)
return 0;
source = link->source;
sink = link->sink;
if (!(link->flags & MEDIA_LNK_FL_DYNAMIC) &&
(media_pad_is_streaming(source) || media_pad_is_streaming(sink)))
return -EBUSY;
mdev = source->graph_obj.mdev;
if (mdev->ops && mdev->ops->link_notify) {
ret = mdev->ops->link_notify(link, flags,
MEDIA_DEV_NOTIFY_PRE_LINK_CH);
if (ret < 0)
return ret;
}
ret = __media_entity_setup_link_notify(link, flags);
if (mdev->ops && mdev->ops->link_notify)
mdev->ops->link_notify(link, flags,
MEDIA_DEV_NOTIFY_POST_LINK_CH);
return ret;
}
EXPORT_SYMBOL_GPL(__media_entity_setup_link);
int media_entity_setup_link(struct media_link *link, u32 flags)
{
int ret;
mutex_lock(&link->graph_obj.mdev->graph_mutex);
ret = __media_entity_setup_link(link, flags);
mutex_unlock(&link->graph_obj.mdev->graph_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(media_entity_setup_link);
struct media_link *
media_entity_find_link(struct media_pad *source, struct media_pad *sink)
{
struct media_link *link;
for_each_media_entity_data_link(source->entity, link) {
if (link->source->entity == source->entity &&
link->source->index == source->index &&
link->sink->entity == sink->entity &&
link->sink->index == sink->index)
return link;
}
return NULL;
}
EXPORT_SYMBOL_GPL(media_entity_find_link);
struct media_pad *media_pad_remote_pad_first(const struct media_pad *pad)
{
struct media_link *link;
for_each_media_entity_data_link(pad->entity, link) {
if (!(link->flags & MEDIA_LNK_FL_ENABLED))
continue;
if (link->source == pad)
return link->sink;
if (link->sink == pad)
return link->source;
}
return NULL;
}
EXPORT_SYMBOL_GPL(media_pad_remote_pad_first);
struct media_pad *
media_entity_remote_pad_unique(const struct media_entity *entity,
unsigned int type)
{
struct media_pad *pad = NULL;
struct media_link *link;
list_for_each_entry(link, &entity->links, list) {
struct media_pad *local_pad;
struct media_pad *remote_pad;
if (((link->flags & MEDIA_LNK_FL_LINK_TYPE) !=
MEDIA_LNK_FL_DATA_LINK) ||
!(link->flags & MEDIA_LNK_FL_ENABLED))
continue;
if (type == MEDIA_PAD_FL_SOURCE) {
local_pad = link->sink;
remote_pad = link->source;
} else {
local_pad = link->source;
remote_pad = link->sink;
}
if (local_pad->entity == entity) {
if (pad)
return ERR_PTR(-ENOTUNIQ);
pad = remote_pad;
}
}
if (!pad)
return ERR_PTR(-ENOLINK);
return pad;
}
EXPORT_SYMBOL_GPL(media_entity_remote_pad_unique);
struct media_pad *media_pad_remote_pad_unique(const struct media_pad *pad)
{
struct media_pad *found_pad = NULL;
struct media_link *link;
list_for_each_entry(link, &pad->entity->links, list) {
struct media_pad *remote_pad;
if (!(link->flags & MEDIA_LNK_FL_ENABLED))
continue;
if (link->sink == pad)
remote_pad = link->source;
else if (link->source == pad)
remote_pad = link->sink;
else
continue;
if (found_pad)
return ERR_PTR(-ENOTUNIQ);
found_pad = remote_pad;
}
if (!found_pad)
return ERR_PTR(-ENOLINK);
return found_pad;
}
EXPORT_SYMBOL_GPL(media_pad_remote_pad_unique);
int media_entity_get_fwnode_pad(struct media_entity *entity,
const struct fwnode_handle *fwnode,
unsigned long direction_flags)
{
struct fwnode_endpoint endpoint;
unsigned int i;
int ret;
if (!entity->ops || !entity->ops->get_fwnode_pad) {
for (i = 0; i < entity->num_pads; i++) {
if (entity->pads[i].flags & direction_flags)
return i;
}
return -ENXIO;
}
ret = fwnode_graph_parse_endpoint(fwnode, &endpoint);
if (ret)
return ret;
ret = entity->ops->get_fwnode_pad(entity, &endpoint);
if (ret < 0)
return ret;
if (ret >= entity->num_pads)
return -ENXIO;
if (!(entity->pads[ret].flags & direction_flags))
return -ENXIO;
return ret;
}
EXPORT_SYMBOL_GPL(media_entity_get_fwnode_pad);
struct media_pipeline *media_entity_pipeline(struct media_entity *entity)
{
struct media_pad *pad;
media_entity_for_each_pad(entity, pad) {
if (pad->pipe)
return pad->pipe;
}
return NULL;
}
EXPORT_SYMBOL_GPL(media_entity_pipeline);
struct media_pipeline *media_pad_pipeline(struct media_pad *pad)
{
return pad->pipe;
}
EXPORT_SYMBOL_GPL(media_pad_pipeline);
static void media_interface_init(struct media_device *mdev,
struct media_interface *intf,
u32 gobj_type,
u32 intf_type, u32 flags)
{
intf->type = intf_type;
intf->flags = flags;
INIT_LIST_HEAD(&intf->links);
media_gobj_create(mdev, gobj_type, &intf->graph_obj);
}
/* Functions related to the media interface via device nodes */
struct media_intf_devnode *media_devnode_create(struct media_device *mdev,
u32 type, u32 flags,
u32 major, u32 minor)
{
struct media_intf_devnode *devnode;
devnode = kzalloc(sizeof(*devnode), GFP_KERNEL);
if (!devnode)
return NULL;
devnode->major = major;
devnode->minor = minor;
media_interface_init(mdev, &devnode->intf, MEDIA_GRAPH_INTF_DEVNODE,
type, flags);
return devnode;
}
EXPORT_SYMBOL_GPL(media_devnode_create);
void media_devnode_remove(struct media_intf_devnode *devnode)
{
media_remove_intf_links(&devnode->intf); media_gobj_destroy(&devnode->intf.graph_obj); kfree(devnode);
}
EXPORT_SYMBOL_GPL(media_devnode_remove);
struct media_link *media_create_intf_link(struct media_entity *entity,
struct media_interface *intf,
u32 flags)
{
struct media_link *link;
link = media_add_link(&intf->links);
if (link == NULL)
return NULL;
link->intf = intf;
link->entity = entity;
link->flags = flags | MEDIA_LNK_FL_INTERFACE_LINK;
/* Initialize graph object embedded at the new link */
media_gobj_create(intf->graph_obj.mdev, MEDIA_GRAPH_LINK,
&link->graph_obj);
return link;
}
EXPORT_SYMBOL_GPL(media_create_intf_link);
void __media_remove_intf_link(struct media_link *link)
{
list_del(&link->list); media_gobj_destroy(&link->graph_obj); kfree(link);
}
EXPORT_SYMBOL_GPL(__media_remove_intf_link);
void media_remove_intf_link(struct media_link *link)
{
struct media_device *mdev = link->graph_obj.mdev;
/* Do nothing if the intf is not registered. */
if (mdev == NULL)
return;
mutex_lock(&mdev->graph_mutex);
__media_remove_intf_link(link);
mutex_unlock(&mdev->graph_mutex);
}
EXPORT_SYMBOL_GPL(media_remove_intf_link);
void __media_remove_intf_links(struct media_interface *intf)
{
struct media_link *link, *tmp;
list_for_each_entry_safe(link, tmp, &intf->links, list) __media_remove_intf_link(link);
}
EXPORT_SYMBOL_GPL(__media_remove_intf_links);
void media_remove_intf_links(struct media_interface *intf)
{
struct media_device *mdev = intf->graph_obj.mdev;
/* Do nothing if the intf is not registered. */
if (mdev == NULL)
return;
mutex_lock(&mdev->graph_mutex);
__media_remove_intf_links(intf);
mutex_unlock(&mdev->graph_mutex);
}
EXPORT_SYMBOL_GPL(media_remove_intf_links);
struct media_link *media_create_ancillary_link(struct media_entity *primary,
struct media_entity *ancillary)
{
struct media_link *link;
link = media_add_link(&primary->links);
if (!link)
return ERR_PTR(-ENOMEM);
link->gobj0 = &primary->graph_obj;
link->gobj1 = &ancillary->graph_obj;
link->flags = MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED |
MEDIA_LNK_FL_ANCILLARY_LINK;
/* Initialize graph object embedded in the new link */
media_gobj_create(primary->graph_obj.mdev, MEDIA_GRAPH_LINK,
&link->graph_obj);
return link;
}
EXPORT_SYMBOL_GPL(media_create_ancillary_link);
struct media_link *__media_entity_next_link(struct media_entity *entity,
struct media_link *link,
unsigned long link_type)
{
link = link ? list_next_entry(link, list)
: list_first_entry(&entity->links, typeof(*link), list);
list_for_each_entry_from(link, &entity->links, list)
if ((link->flags & MEDIA_LNK_FL_LINK_TYPE) == link_type)
return link;
return NULL;
}
EXPORT_SYMBOL_GPL(__media_entity_next_link);
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* AppArmor security module
*
* This file contains AppArmor contexts used to associate "labels" to objects.
*
* Copyright (C) 1998-2008 Novell/SUSE
* Copyright 2009-2010 Canonical Ltd.
*/
#ifndef __AA_CONTEXT_H
#define __AA_CONTEXT_H
#include <linux/cred.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include "label.h"
#include "policy_ns.h"
#include "task.h"
static inline struct aa_label *cred_label(const struct cred *cred)
{
struct aa_label **blob = cred->security + apparmor_blob_sizes.lbs_cred;
AA_BUG(!blob); return *blob;
}
static inline void set_cred_label(const struct cred *cred,
struct aa_label *label)
{
struct aa_label **blob = cred->security + apparmor_blob_sizes.lbs_cred;
AA_BUG(!blob);
*blob = label;
}
/**
* aa_cred_raw_label - obtain cred's label
* @cred: cred to obtain label from (NOT NULL)
*
* Returns: confining label
*
* does NOT increment reference count
*/
static inline struct aa_label *aa_cred_raw_label(const struct cred *cred)
{
struct aa_label *label = cred_label(cred); AA_BUG(!label);
return label;
}
/**
* aa_get_newest_cred_label - obtain the newest label on a cred
* @cred: cred to obtain label from (NOT NULL)
*
* Returns: newest version of confining label
*/
static inline struct aa_label *aa_get_newest_cred_label(const struct cred *cred)
{
return aa_get_newest_label(aa_cred_raw_label(cred));
}
/**
* aa_current_raw_label - find the current tasks confining label
*
* Returns: up to date confining label or the ns unconfined label (NOT NULL)
*
* This fn will not update the tasks cred to the most up to date version
* of the label so it is safe to call when inside of locks.
*/
static inline struct aa_label *aa_current_raw_label(void)
{
return aa_cred_raw_label(current_cred());
}
/**
* aa_get_current_label - get the newest version of the current tasks label
*
* Returns: newest version of confining label (NOT NULL)
*
* This fn will not update the tasks cred, so it is safe inside of locks
*
* The returned reference must be put with aa_put_label()
*/
static inline struct aa_label *aa_get_current_label(void)
{
struct aa_label *l = aa_current_raw_label();
if (label_is_stale(l))
return aa_get_newest_label(l);
return aa_get_label(l);
}
#define __end_current_label_crit_section(X) end_current_label_crit_section(X)
/**
* end_label_crit_section - put a reference found with begin_current_label..
* @label: label reference to put
*
* Should only be used with a reference obtained with
* begin_current_label_crit_section and never used in situations where the
* task cred may be updated
*/
static inline void end_current_label_crit_section(struct aa_label *label)
{
if (label != aa_current_raw_label()) aa_put_label(label);
}
/**
* __begin_current_label_crit_section - current's confining label
*
* Returns: up to date confining label or the ns unconfined label (NOT NULL)
*
* safe to call inside locks
*
* The returned reference must be put with __end_current_label_crit_section()
* This must NOT be used if the task cred could be updated within the
* critical section between __begin_current_label_crit_section() ..
* __end_current_label_crit_section()
*/
static inline struct aa_label *__begin_current_label_crit_section(void)
{
struct aa_label *label = aa_current_raw_label(); if (label_is_stale(label)) label = aa_get_newest_label(label);
return label;
}
/**
* begin_current_label_crit_section - current's confining label and update it
*
* Returns: up to date confining label or the ns unconfined label (NOT NULL)
*
* Not safe to call inside locks
*
* The returned reference must be put with end_current_label_crit_section()
* This must NOT be used if the task cred could be updated within the
* critical section between begin_current_label_crit_section() ..
* end_current_label_crit_section()
*/
static inline struct aa_label *begin_current_label_crit_section(void)
{
struct aa_label *label = aa_current_raw_label(); might_sleep();
if (label_is_stale(label)) {
label = aa_get_newest_label(label);
if (aa_replace_current_label(label) == 0)
/* task cred will keep the reference */
aa_put_label(label);
}
return label;
}
static inline struct aa_ns *aa_get_current_ns(void)
{
struct aa_label *label;
struct aa_ns *ns;
label = __begin_current_label_crit_section();
ns = aa_get_ns(labels_ns(label));
__end_current_label_crit_section(label);
return ns;
}
#endif /* __AA_CONTEXT_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _KERNEL_PRINTK_RINGBUFFER_H
#define _KERNEL_PRINTK_RINGBUFFER_H
#include <linux/atomic.h>
#include <linux/dev_printk.h>
/*
* Meta information about each stored message.
*
* All fields are set by the printk code except for @seq, which is
* set by the ringbuffer code.
*/
struct printk_info {
u64 seq; /* sequence number */
u64 ts_nsec; /* timestamp in nanoseconds */
u16 text_len; /* length of text message */
u8 facility; /* syslog facility */
u8 flags:5; /* internal record flags */
u8 level:3; /* syslog level */
u32 caller_id; /* thread id or processor id */
struct dev_printk_info dev_info;
};
/*
* A structure providing the buffers, used by writers and readers.
*
* Writers:
* Using prb_rec_init_wr(), a writer sets @text_buf_size before calling
* prb_reserve(). On success, prb_reserve() sets @info and @text_buf to
* buffers reserved for that writer.
*
* Readers:
* Using prb_rec_init_rd(), a reader sets all fields before calling
* prb_read_valid(). Note that the reader provides the @info and @text_buf,
* buffers. On success, the struct pointed to by @info will be filled and
* the char array pointed to by @text_buf will be filled with text data.
*/
struct printk_record {
struct printk_info *info;
char *text_buf;
unsigned int text_buf_size;
};
/* Specifies the logical position and span of a data block. */
struct prb_data_blk_lpos {
unsigned long begin;
unsigned long next;
};
/*
* A descriptor: the complete meta-data for a record.
*
* @state_var: A bitwise combination of descriptor ID and descriptor state.
*/
struct prb_desc {
atomic_long_t state_var;
struct prb_data_blk_lpos text_blk_lpos;
};
/* A ringbuffer of "ID + data" elements. */
struct prb_data_ring {
unsigned int size_bits;
char *data;
atomic_long_t head_lpos;
atomic_long_t tail_lpos;
};
/* A ringbuffer of "struct prb_desc" elements. */
struct prb_desc_ring {
unsigned int count_bits;
struct prb_desc *descs;
struct printk_info *infos;
atomic_long_t head_id;
atomic_long_t tail_id;
atomic_long_t last_finalized_seq;
};
/*
* The high level structure representing the printk ringbuffer.
*
* @fail: Count of failed prb_reserve() calls where not even a data-less
* record was created.
*/
struct printk_ringbuffer {
struct prb_desc_ring desc_ring;
struct prb_data_ring text_data_ring;
atomic_long_t fail;
};
/*
* Used by writers as a reserve/commit handle.
*
* @rb: Ringbuffer where the entry is reserved.
* @irqflags: Saved irq flags to restore on entry commit.
* @id: ID of the reserved descriptor.
* @text_space: Total occupied buffer space in the text data ring, including
* ID, alignment padding, and wrapping data blocks.
*
* This structure is an opaque handle for writers. Its contents are only
* to be used by the ringbuffer implementation.
*/
struct prb_reserved_entry {
struct printk_ringbuffer *rb;
unsigned long irqflags;
unsigned long id;
unsigned int text_space;
};
/* The possible responses of a descriptor state-query. */
enum desc_state {
desc_miss = -1, /* ID mismatch (pseudo state) */
desc_reserved = 0x0, /* reserved, in use by writer */
desc_committed = 0x1, /* committed by writer, could get reopened */
desc_finalized = 0x2, /* committed, no further modification allowed */
desc_reusable = 0x3, /* free, not yet used by any writer */
};
#define _DATA_SIZE(sz_bits) (1UL << (sz_bits))
#define _DESCS_COUNT(ct_bits) (1U << (ct_bits))
#define DESC_SV_BITS (sizeof(unsigned long) * 8)
#define DESC_FLAGS_SHIFT (DESC_SV_BITS - 2)
#define DESC_FLAGS_MASK (3UL << DESC_FLAGS_SHIFT)
#define DESC_STATE(sv) (3UL & (sv >> DESC_FLAGS_SHIFT))
#define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id)
#define DESC_ID_MASK (~DESC_FLAGS_MASK)
#define DESC_ID(sv) ((sv) & DESC_ID_MASK)
/*
* Special data block logical position values (for fields of
* @prb_desc.text_blk_lpos).
*
* - Bit0 is used to identify if the record has no data block. (Implemented in
* the LPOS_DATALESS() macro.)
*
* - Bit1 specifies the reason for not having a data block.
*
* These special values could never be real lpos values because of the
* meta data and alignment padding of data blocks. (See to_blk_size() for
* details.)
*/
#define FAILED_LPOS 0x1
#define EMPTY_LINE_LPOS 0x3
#define FAILED_BLK_LPOS \
{ \
.begin = FAILED_LPOS, \
.next = FAILED_LPOS, \
}
/*
* Descriptor Bootstrap
*
* The descriptor array is minimally initialized to allow immediate usage
* by readers and writers. The requirements that the descriptor array
* initialization must satisfy:
*
* Req1
* The tail must point to an existing (committed or reusable) descriptor.
* This is required by the implementation of prb_first_seq().
*
* Req2
* Readers must see that the ringbuffer is initially empty.
*
* Req3
* The first record reserved by a writer is assigned sequence number 0.
*
* To satisfy Req1, the tail initially points to a descriptor that is
* minimally initialized (having no data block, i.e. data-less with the
* data block's lpos @begin and @next values set to FAILED_LPOS).
*
* To satisfy Req2, the initial tail descriptor is initialized to the
* reusable state. Readers recognize reusable descriptors as existing
* records, but skip over them.
*
* To satisfy Req3, the last descriptor in the array is used as the initial
* head (and tail) descriptor. This allows the first record reserved by a
* writer (head + 1) to be the first descriptor in the array. (Only the first
* descriptor in the array could have a valid sequence number of 0.)
*
* The first time a descriptor is reserved, it is assigned a sequence number
* with the value of the array index. A "first time reserved" descriptor can
* be recognized because it has a sequence number of 0 but does not have an
* index of 0. (Only the first descriptor in the array could have a valid
* sequence number of 0.) After the first reservation, all future reservations
* (recycling) simply involve incrementing the sequence number by the array
* count.
*
* Hack #1
* Only the first descriptor in the array is allowed to have the sequence
* number 0. In this case it is not possible to recognize if it is being
* reserved the first time (set to index value) or has been reserved
* previously (increment by the array count). This is handled by _always_
* incrementing the sequence number by the array count when reserving the
* first descriptor in the array. In order to satisfy Req3, the sequence
* number of the first descriptor in the array is initialized to minus
* the array count. Then, upon the first reservation, it is incremented
* to 0, thus satisfying Req3.
*
* Hack #2
* prb_first_seq() can be called at any time by readers to retrieve the
* sequence number of the tail descriptor. However, due to Req2 and Req3,
* initially there are no records to report the sequence number of
* (sequence numbers are u64 and there is nothing less than 0). To handle
* this, the sequence number of the initial tail descriptor is initialized
* to 0. Technically this is incorrect, because there is no record with
* sequence number 0 (yet) and the tail descriptor is not the first
* descriptor in the array. But it allows prb_read_valid() to correctly
* report the existence of a record for _any_ given sequence number at all
* times. Bootstrapping is complete when the tail is pushed the first
* time, thus finally pointing to the first descriptor reserved by a
* writer, which has the assigned sequence number 0.
*/
/*
* Initiating Logical Value Overflows
*
* Both logical position (lpos) and ID values can be mapped to array indexes
* but may experience overflows during the lifetime of the system. To ensure
* that printk_ringbuffer can handle the overflows for these types, initial
* values are chosen that map to the correct initial array indexes, but will
* result in overflows soon.
*
* BLK0_LPOS
* The initial @head_lpos and @tail_lpos for data rings. It is at index
* 0 and the lpos value is such that it will overflow on the first wrap.
*
* DESC0_ID
* The initial @head_id and @tail_id for the desc ring. It is at the last
* index of the descriptor array (see Req3 above) and the ID value is such
* that it will overflow on the second wrap.
*/
#define BLK0_LPOS(sz_bits) (-(_DATA_SIZE(sz_bits)))
#define DESC0_ID(ct_bits) DESC_ID(-(_DESCS_COUNT(ct_bits) + 1))
#define DESC0_SV(ct_bits) DESC_SV(DESC0_ID(ct_bits), desc_reusable)
/*
* Define a ringbuffer with an external text data buffer. The same as
* DEFINE_PRINTKRB() but requires specifying an external buffer for the
* text data.
*
* Note: The specified external buffer must be of the size:
* 2 ^ (descbits + avgtextbits)
*/
#define _DEFINE_PRINTKRB(name, descbits, avgtextbits, text_buf) \
static struct prb_desc _##name##_descs[_DESCS_COUNT(descbits)] = { \
/* the initial head and tail */ \
[_DESCS_COUNT(descbits) - 1] = { \
/* reusable */ \
.state_var = ATOMIC_INIT(DESC0_SV(descbits)), \
/* no associated data block */ \
.text_blk_lpos = FAILED_BLK_LPOS, \
}, \
}; \
static struct printk_info _##name##_infos[_DESCS_COUNT(descbits)] = { \
/* this will be the first record reserved by a writer */ \
[0] = { \
/* will be incremented to 0 on the first reservation */ \
.seq = -(u64)_DESCS_COUNT(descbits), \
}, \
/* the initial head and tail */ \
[_DESCS_COUNT(descbits) - 1] = { \
/* reports the first seq value during the bootstrap phase */ \
.seq = 0, \
}, \
}; \
static struct printk_ringbuffer name = { \
.desc_ring = { \
.count_bits = descbits, \
.descs = &_##name##_descs[0], \
.infos = &_##name##_infos[0], \
.head_id = ATOMIC_INIT(DESC0_ID(descbits)), \
.tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \
.last_finalized_seq = ATOMIC_INIT(0), \
}, \
.text_data_ring = { \
.size_bits = (avgtextbits) + (descbits), \
.data = text_buf, \
.head_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \
.tail_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \
}, \
.fail = ATOMIC_LONG_INIT(0), \
}
/**
* DEFINE_PRINTKRB() - Define a ringbuffer.
*
* @name: The name of the ringbuffer variable.
* @descbits: The number of descriptors as a power-of-2 value.
* @avgtextbits: The average text data size per record as a power-of-2 value.
*
* This is a macro for defining a ringbuffer and all internal structures
* such that it is ready for immediate use. See _DEFINE_PRINTKRB() for a
* variant where the text data buffer can be specified externally.
*/
#define DEFINE_PRINTKRB(name, descbits, avgtextbits) \
static char _##name##_text[1U << ((avgtextbits) + (descbits))] \
__aligned(__alignof__(unsigned long)); \
_DEFINE_PRINTKRB(name, descbits, avgtextbits, &_##name##_text[0])
/* Writer Interface */
/**
* prb_rec_init_wr() - Initialize a buffer for writing records.
*
* @r: The record to initialize.
* @text_buf_size: The needed text buffer size.
*/
static inline void prb_rec_init_wr(struct printk_record *r,
unsigned int text_buf_size)
{
r->info = NULL;
r->text_buf = NULL;
r->text_buf_size = text_buf_size;
}
bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
struct printk_record *r);
bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
struct printk_record *r, u32 caller_id, unsigned int max_size);
void prb_commit(struct prb_reserved_entry *e);
void prb_final_commit(struct prb_reserved_entry *e);
void prb_init(struct printk_ringbuffer *rb,
char *text_buf, unsigned int text_buf_size,
struct prb_desc *descs, unsigned int descs_count_bits,
struct printk_info *infos);
unsigned int prb_record_text_space(struct prb_reserved_entry *e);
/* Reader Interface */
/**
* prb_rec_init_rd() - Initialize a buffer for reading records.
*
* @r: The record to initialize.
* @info: A buffer to store record meta-data.
* @text_buf: A buffer to store text data.
* @text_buf_size: The size of @text_buf.
*
* Initialize all the fields that a reader is interested in. All arguments
* (except @r) are optional. Only record data for arguments that are
* non-NULL or non-zero will be read.
*/
static inline void prb_rec_init_rd(struct printk_record *r,
struct printk_info *info,
char *text_buf, unsigned int text_buf_size)
{
r->info = info;
r->text_buf = text_buf;
r->text_buf_size = text_buf_size;
}
/**
* prb_for_each_record() - Iterate over the records of a ringbuffer.
*
* @from: The sequence number to begin with.
* @rb: The ringbuffer to iterate over.
* @s: A u64 to store the sequence number on each iteration.
* @r: A printk_record to store the record on each iteration.
*
* This is a macro for conveniently iterating over a ringbuffer.
* Note that @s may not be the sequence number of the record on each
* iteration. For the sequence number, @r->info->seq should be checked.
*
* Context: Any context.
*/
#define prb_for_each_record(from, rb, s, r) \
for ((s) = from; prb_read_valid(rb, s, r); (s) = (r)->info->seq + 1)
/**
* prb_for_each_info() - Iterate over the meta data of a ringbuffer.
*
* @from: The sequence number to begin with.
* @rb: The ringbuffer to iterate over.
* @s: A u64 to store the sequence number on each iteration.
* @i: A printk_info to store the record meta data on each iteration.
* @lc: An unsigned int to store the text line count of each record.
*
* This is a macro for conveniently iterating over a ringbuffer.
* Note that @s may not be the sequence number of the record on each
* iteration. For the sequence number, @r->info->seq should be checked.
*
* Context: Any context.
*/
#define prb_for_each_info(from, rb, s, i, lc) \
for ((s) = from; prb_read_valid_info(rb, s, i, lc); (s) = (i)->seq + 1)
bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
struct printk_record *r);
bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
struct printk_info *info, unsigned int *line_count);
u64 prb_first_seq(struct printk_ringbuffer *rb);
u64 prb_first_valid_seq(struct printk_ringbuffer *rb);
u64 prb_next_seq(struct printk_ringbuffer *rb);
u64 prb_next_reserve_seq(struct printk_ringbuffer *rb);
#ifdef CONFIG_64BIT
#define __u64seq_to_ulseq(u64seq) (u64seq)
#define __ulseq_to_u64seq(rb, ulseq) (ulseq)
#else /* CONFIG_64BIT */
#define __u64seq_to_ulseq(u64seq) ((u32)u64seq)
static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq)
{
u64 rb_first_seq = prb_first_seq(rb);
u64 seq;
/*
* The provided sequence is only the lower 32 bits of the ringbuffer
* sequence. It needs to be expanded to 64bit. Get the first sequence
* number from the ringbuffer and fold it.
*
* Having a 32bit representation in the console is sufficient.
* If a console ever gets more than 2^31 records behind
* the ringbuffer then this is the least of the problems.
*
* Also the access to the ring buffer is always safe.
*/
seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq);
return seq;
}
#endif /* CONFIG_64BIT */
#endif /* _KERNEL_PRINTK_RINGBUFFER_H */
// SPDX-License-Identifier: GPL-2.0-only
#include "cgroup-internal.h"
#include <linux/sched/cputime.h>
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
static DEFINE_SPINLOCK(cgroup_rstat_lock);
static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock);
static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)
{
return per_cpu_ptr(cgrp->rstat_cpu, cpu);
}
/**
* cgroup_rstat_updated - keep track of updated rstat_cpu
* @cgrp: target cgroup
* @cpu: cpu on which rstat_cpu was updated
*
* @cgrp's rstat_cpu on @cpu was updated. Put it on the parent's matching
* rstat_cpu->updated_children list. See the comment on top of
* cgroup_rstat_cpu definition for details.
*/
__bpf_kfunc void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
{
raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
unsigned long flags;
/*
* Speculative already-on-list test. This may race leading to
* temporary inaccuracies, which is fine.
*
* Because @parent's updated_children is terminated with @parent
* instead of NULL, we can tell whether @cgrp is on the list by
* testing the next pointer for NULL.
*/
if (data_race(cgroup_rstat_cpu(cgrp, cpu)->updated_next))
return;
raw_spin_lock_irqsave(cpu_lock, flags);
/* put @cgrp and all ancestors on the corresponding updated lists */
while (true) {
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_rstat_cpu *prstatc;
/*
* Both additions and removals are bottom-up. If a cgroup
* is already in the tree, all ancestors are.
*/
if (rstatc->updated_next)
break;
/* Root has no parent to link it to, but mark it busy */
if (!parent) { rstatc->updated_next = cgrp;
break;
}
prstatc = cgroup_rstat_cpu(parent, cpu);
rstatc->updated_next = prstatc->updated_children;
prstatc->updated_children = cgrp;
cgrp = parent;
}
raw_spin_unlock_irqrestore(cpu_lock, flags);
}
/**
* cgroup_rstat_push_children - push children cgroups into the given list
* @head: current head of the list (= subtree root)
* @child: first child of the root
* @cpu: target cpu
* Return: A new singly linked list of cgroups to be flush
*
* Iteratively traverse down the cgroup_rstat_cpu updated tree level by
* level and push all the parents first before their next level children
* into a singly linked list built from the tail backward like "pushing"
* cgroups into a stack. The root is pushed by the caller.
*/
static struct cgroup *cgroup_rstat_push_children(struct cgroup *head,
struct cgroup *child, int cpu)
{
struct cgroup *chead = child; /* Head of child cgroup level */
struct cgroup *ghead = NULL; /* Head of grandchild cgroup level */
struct cgroup *parent, *grandchild;
struct cgroup_rstat_cpu *crstatc;
child->rstat_flush_next = NULL;
next_level:
while (chead) {
child = chead;
chead = child->rstat_flush_next;
parent = cgroup_parent(child);
/* updated_next is parent cgroup terminated */
while (child != parent) {
child->rstat_flush_next = head;
head = child;
crstatc = cgroup_rstat_cpu(child, cpu);
grandchild = crstatc->updated_children;
if (grandchild != child) {
/* Push the grand child to the next level */
crstatc->updated_children = child;
grandchild->rstat_flush_next = ghead;
ghead = grandchild;
}
child = crstatc->updated_next;
crstatc->updated_next = NULL;
}
}
if (ghead) {
chead = ghead;
ghead = NULL;
goto next_level;
}
return head;
}
/**
* cgroup_rstat_updated_list - return a list of updated cgroups to be flushed
* @root: root of the cgroup subtree to traverse
* @cpu: target cpu
* Return: A singly linked list of cgroups to be flushed
*
* Walks the updated rstat_cpu tree on @cpu from @root. During traversal,
* each returned cgroup is unlinked from the updated tree.
*
* The only ordering guarantee is that, for a parent and a child pair
* covered by a given traversal, the child is before its parent in
* the list.
*
* Note that updated_children is self terminated and points to a list of
* child cgroups if not empty. Whereas updated_next is like a sibling link
* within the children list and terminated by the parent cgroup. An exception
* here is the cgroup root whose updated_next can be self terminated.
*/
static struct cgroup *cgroup_rstat_updated_list(struct cgroup *root, int cpu)
{
raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(root, cpu);
struct cgroup *head = NULL, *parent, *child;
unsigned long flags;
/*
* The _irqsave() is needed because cgroup_rstat_lock is
* spinlock_t which is a sleeping lock on PREEMPT_RT. Acquiring
* this lock with the _irq() suffix only disables interrupts on
* a non-PREEMPT_RT kernel. The raw_spinlock_t below disables
* interrupts on both configurations. The _irqsave() ensures
* that interrupts are always disabled and later restored.
*/
raw_spin_lock_irqsave(cpu_lock, flags);
/* Return NULL if this subtree is not on-list */
if (!rstatc->updated_next)
goto unlock_ret;
/*
* Unlink @root from its parent. As the updated_children list is
* singly linked, we have to walk it to find the removal point.
*/
parent = cgroup_parent(root);
if (parent) {
struct cgroup_rstat_cpu *prstatc;
struct cgroup **nextp;
prstatc = cgroup_rstat_cpu(parent, cpu);
nextp = &prstatc->updated_children;
while (*nextp != root) {
struct cgroup_rstat_cpu *nrstatc;
nrstatc = cgroup_rstat_cpu(*nextp, cpu);
WARN_ON_ONCE(*nextp == parent);
nextp = &nrstatc->updated_next;
}
*nextp = rstatc->updated_next;
}
rstatc->updated_next = NULL;
/* Push @root to the list first before pushing the children */
head = root;
root->rstat_flush_next = NULL;
child = rstatc->updated_children;
rstatc->updated_children = root;
if (child != root)
head = cgroup_rstat_push_children(head, child, cpu);
unlock_ret:
raw_spin_unlock_irqrestore(cpu_lock, flags);
return head;
}
/*
* A hook for bpf stat collectors to attach to and flush their stats.
* Together with providing bpf kfuncs for cgroup_rstat_updated() and
* cgroup_rstat_flush(), this enables a complete workflow where bpf progs that
* collect cgroup stats can integrate with rstat for efficient flushing.
*
* A static noinline declaration here could cause the compiler to optimize away
* the function. A global noinline declaration will keep the definition, but may
* optimize away the callsite. Therefore, __weak is needed to ensure that the
* call is still emitted, by telling the compiler that we don't know what the
* function might eventually be.
*/
__bpf_hook_start();
__weak noinline void bpf_rstat_flush(struct cgroup *cgrp,
struct cgroup *parent, int cpu)
{
}
__bpf_hook_end();
/* see cgroup_rstat_flush() */
static void cgroup_rstat_flush_locked(struct cgroup *cgrp)
__releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock)
{
int cpu;
lockdep_assert_held(&cgroup_rstat_lock);
for_each_possible_cpu(cpu) {
struct cgroup *pos = cgroup_rstat_updated_list(cgrp, cpu);
for (; pos; pos = pos->rstat_flush_next) {
struct cgroup_subsys_state *css;
cgroup_base_stat_flush(pos, cpu);
bpf_rstat_flush(pos, cgroup_parent(pos), cpu);
rcu_read_lock();
list_for_each_entry_rcu(css, &pos->rstat_css_list,
rstat_css_node)
css->ss->css_rstat_flush(css, cpu);
rcu_read_unlock();
}
/* play nice and yield if necessary */
if (need_resched() || spin_needbreak(&cgroup_rstat_lock)) {
spin_unlock_irq(&cgroup_rstat_lock);
if (!cond_resched())
cpu_relax();
spin_lock_irq(&cgroup_rstat_lock);
}
}
}
/**
* cgroup_rstat_flush - flush stats in @cgrp's subtree
* @cgrp: target cgroup
*
* Collect all per-cpu stats in @cgrp's subtree into the global counters
* and propagate them upwards. After this function returns, all cgroups in
* the subtree have up-to-date ->stat.
*
* This also gets all cgroups in the subtree including @cgrp off the
* ->updated_children lists.
*
* This function may block.
*/
__bpf_kfunc void cgroup_rstat_flush(struct cgroup *cgrp)
{
might_sleep();
spin_lock_irq(&cgroup_rstat_lock);
cgroup_rstat_flush_locked(cgrp);
spin_unlock_irq(&cgroup_rstat_lock);
}
/**
* cgroup_rstat_flush_hold - flush stats in @cgrp's subtree and hold
* @cgrp: target cgroup
*
* Flush stats in @cgrp's subtree and prevent further flushes. Must be
* paired with cgroup_rstat_flush_release().
*
* This function may block.
*/
void cgroup_rstat_flush_hold(struct cgroup *cgrp)
__acquires(&cgroup_rstat_lock)
{
might_sleep();
spin_lock_irq(&cgroup_rstat_lock);
cgroup_rstat_flush_locked(cgrp);
}
/**
* cgroup_rstat_flush_release - release cgroup_rstat_flush_hold()
*/
void cgroup_rstat_flush_release(void)
__releases(&cgroup_rstat_lock)
{
spin_unlock_irq(&cgroup_rstat_lock);
}
int cgroup_rstat_init(struct cgroup *cgrp)
{
int cpu;
/* the root cgrp has rstat_cpu preallocated */
if (!cgrp->rstat_cpu) {
cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);
if (!cgrp->rstat_cpu)
return -ENOMEM;
}
/* ->updated_children list is self terminated */
for_each_possible_cpu(cpu) {
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
rstatc->updated_children = cgrp;
u64_stats_init(&rstatc->bsync);
}
return 0;
}
void cgroup_rstat_exit(struct cgroup *cgrp)
{
int cpu;
cgroup_rstat_flush(cgrp);
/* sanity check */
for_each_possible_cpu(cpu) {
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||
WARN_ON_ONCE(rstatc->updated_next))
return;
}
free_percpu(cgrp->rstat_cpu);
cgrp->rstat_cpu = NULL;
}
void __init cgroup_rstat_boot(void)
{
int cpu;
for_each_possible_cpu(cpu)
raw_spin_lock_init(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu));
}
/*
* Functions for cgroup basic resource statistics implemented on top of
* rstat.
*/
static void cgroup_base_stat_add(struct cgroup_base_stat *dst_bstat,
struct cgroup_base_stat *src_bstat)
{
dst_bstat->cputime.utime += src_bstat->cputime.utime;
dst_bstat->cputime.stime += src_bstat->cputime.stime;
dst_bstat->cputime.sum_exec_runtime += src_bstat->cputime.sum_exec_runtime;
#ifdef CONFIG_SCHED_CORE
dst_bstat->forceidle_sum += src_bstat->forceidle_sum;
#endif
}
static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat,
struct cgroup_base_stat *src_bstat)
{
dst_bstat->cputime.utime -= src_bstat->cputime.utime;
dst_bstat->cputime.stime -= src_bstat->cputime.stime;
dst_bstat->cputime.sum_exec_runtime -= src_bstat->cputime.sum_exec_runtime;
#ifdef CONFIG_SCHED_CORE
dst_bstat->forceidle_sum -= src_bstat->forceidle_sum;
#endif
}
static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
{
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_rstat_cpu *prstatc;
struct cgroup_base_stat delta;
unsigned seq;
/* Root-level stats are sourced from system-wide CPU stats */
if (!parent)
return;
/* fetch the current per-cpu values */
do {
seq = __u64_stats_fetch_begin(&rstatc->bsync);
delta = rstatc->bstat;
} while (__u64_stats_fetch_retry(&rstatc->bsync, seq));
/* propagate per-cpu delta to cgroup and per-cpu global statistics */
cgroup_base_stat_sub(&delta, &rstatc->last_bstat);
cgroup_base_stat_add(&cgrp->bstat, &delta);
cgroup_base_stat_add(&rstatc->last_bstat, &delta);
cgroup_base_stat_add(&rstatc->subtree_bstat, &delta);
/* propagate cgroup and per-cpu global delta to parent (unless that's root) */
if (cgroup_parent(parent)) {
delta = cgrp->bstat;
cgroup_base_stat_sub(&delta, &cgrp->last_bstat);
cgroup_base_stat_add(&parent->bstat, &delta);
cgroup_base_stat_add(&cgrp->last_bstat, &delta);
delta = rstatc->subtree_bstat;
prstatc = cgroup_rstat_cpu(parent, cpu);
cgroup_base_stat_sub(&delta, &rstatc->last_subtree_bstat);
cgroup_base_stat_add(&prstatc->subtree_bstat, &delta);
cgroup_base_stat_add(&rstatc->last_subtree_bstat, &delta);
}
}
static struct cgroup_rstat_cpu *
cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)
{
struct cgroup_rstat_cpu *rstatc;
rstatc = get_cpu_ptr(cgrp->rstat_cpu);
*flags = u64_stats_update_begin_irqsave(&rstatc->bsync);
return rstatc;
}
static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
struct cgroup_rstat_cpu *rstatc,
unsigned long flags)
{
u64_stats_update_end_irqrestore(&rstatc->bsync, flags);
cgroup_rstat_updated(cgrp, smp_processor_id()); put_cpu_ptr(rstatc);
}
void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)
{
struct cgroup_rstat_cpu *rstatc;
unsigned long flags;
rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
rstatc->bstat.cputime.sum_exec_runtime += delta_exec;
cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);
}
void __cgroup_account_cputime_field(struct cgroup *cgrp,
enum cpu_usage_stat index, u64 delta_exec)
{
struct cgroup_rstat_cpu *rstatc;
unsigned long flags;
rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
switch (index) {
case CPUTIME_USER:
case CPUTIME_NICE:
rstatc->bstat.cputime.utime += delta_exec;
break;
case CPUTIME_SYSTEM:
case CPUTIME_IRQ:
case CPUTIME_SOFTIRQ:
rstatc->bstat.cputime.stime += delta_exec;
break;
#ifdef CONFIG_SCHED_CORE
case CPUTIME_FORCEIDLE:
rstatc->bstat.forceidle_sum += delta_exec;
break;
#endif
default:
break;
}
cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);
}
/*
* compute the cputime for the root cgroup by getting the per cpu data
* at a global level, then categorizing the fields in a manner consistent
* with how it is done by __cgroup_account_cputime_field for each bit of
* cpu time attributed to a cgroup.
*/
static void root_cgroup_cputime(struct cgroup_base_stat *bstat)
{
struct task_cputime *cputime = &bstat->cputime;
int i;
memset(bstat, 0, sizeof(*bstat));
for_each_possible_cpu(i) {
struct kernel_cpustat kcpustat;
u64 *cpustat = kcpustat.cpustat;
u64 user = 0;
u64 sys = 0;
kcpustat_cpu_fetch(&kcpustat, i);
user += cpustat[CPUTIME_USER];
user += cpustat[CPUTIME_NICE];
cputime->utime += user;
sys += cpustat[CPUTIME_SYSTEM];
sys += cpustat[CPUTIME_IRQ];
sys += cpustat[CPUTIME_SOFTIRQ];
cputime->stime += sys;
cputime->sum_exec_runtime += user;
cputime->sum_exec_runtime += sys;
cputime->sum_exec_runtime += cpustat[CPUTIME_STEAL];
#ifdef CONFIG_SCHED_CORE
bstat->forceidle_sum += cpustat[CPUTIME_FORCEIDLE];
#endif
}
}
void cgroup_base_stat_cputime_show(struct seq_file *seq)
{
struct cgroup *cgrp = seq_css(seq)->cgroup;
u64 usage, utime, stime;
struct cgroup_base_stat bstat;
#ifdef CONFIG_SCHED_CORE
u64 forceidle_time;
#endif
if (cgroup_parent(cgrp)) {
cgroup_rstat_flush_hold(cgrp);
usage = cgrp->bstat.cputime.sum_exec_runtime;
cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,
&utime, &stime);
#ifdef CONFIG_SCHED_CORE
forceidle_time = cgrp->bstat.forceidle_sum;
#endif
cgroup_rstat_flush_release();
} else {
root_cgroup_cputime(&bstat);
usage = bstat.cputime.sum_exec_runtime;
utime = bstat.cputime.utime;
stime = bstat.cputime.stime;
#ifdef CONFIG_SCHED_CORE
forceidle_time = bstat.forceidle_sum;
#endif
}
do_div(usage, NSEC_PER_USEC);
do_div(utime, NSEC_PER_USEC);
do_div(stime, NSEC_PER_USEC);
#ifdef CONFIG_SCHED_CORE
do_div(forceidle_time, NSEC_PER_USEC);
#endif
seq_printf(seq, "usage_usec %llu\n"
"user_usec %llu\n"
"system_usec %llu\n",
usage, utime, stime);
#ifdef CONFIG_SCHED_CORE
seq_printf(seq, "core_sched.force_idle_usec %llu\n", forceidle_time);
#endif
}
/* Add bpf kfuncs for cgroup_rstat_updated() and cgroup_rstat_flush() */
BTF_KFUNCS_START(bpf_rstat_kfunc_ids)
BTF_ID_FLAGS(func, cgroup_rstat_updated)
BTF_ID_FLAGS(func, cgroup_rstat_flush, KF_SLEEPABLE)
BTF_KFUNCS_END(bpf_rstat_kfunc_ids)
static const struct btf_kfunc_id_set bpf_rstat_kfunc_set = {
.owner = THIS_MODULE,
.set = &bpf_rstat_kfunc_ids,
};
static int __init bpf_rstat_kfunc_init(void)
{
return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
&bpf_rstat_kfunc_set);
}
late_initcall(bpf_rstat_kfunc_init);
// SPDX-License-Identifier: GPL-2.0
/*
* Native support for the I/O-Warrior USB devices
*
* Copyright (c) 2003-2005, 2020 Code Mercenaries GmbH
* written by Christian Lucht <lucht@codemercs.com> and
* Christoph Jung <jung@codemercs.com>
*
* based on
* usb-skeleton.c by Greg Kroah-Hartman <greg@kroah.com>
* brlvger.c by Stephane Dalton <sdalton@videotron.ca>
* and Stephane Doyon <s.doyon@videotron.ca>
*
* Released under the GPLv2.
*/
#include <linux/module.h>
#include <linux/usb.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/usb/iowarrior.h>
#define DRIVER_AUTHOR "Christian Lucht <lucht@codemercs.com>"
#define DRIVER_DESC "USB IO-Warrior driver"
#define USB_VENDOR_ID_CODEMERCS 1984
/* low speed iowarrior */
#define USB_DEVICE_ID_CODEMERCS_IOW40 0x1500
#define USB_DEVICE_ID_CODEMERCS_IOW24 0x1501
#define USB_DEVICE_ID_CODEMERCS_IOWPV1 0x1511
#define USB_DEVICE_ID_CODEMERCS_IOWPV2 0x1512
/* full speed iowarrior */
#define USB_DEVICE_ID_CODEMERCS_IOW56 0x1503
/* fuller speed iowarrior */
#define USB_DEVICE_ID_CODEMERCS_IOW28 0x1504
#define USB_DEVICE_ID_CODEMERCS_IOW28L 0x1505
#define USB_DEVICE_ID_CODEMERCS_IOW100 0x1506
/* OEMed devices */
#define USB_DEVICE_ID_CODEMERCS_IOW24SAG 0x158a
#define USB_DEVICE_ID_CODEMERCS_IOW56AM 0x158b
/* Get a minor range for your devices from the usb maintainer */
#ifdef CONFIG_USB_DYNAMIC_MINORS
#define IOWARRIOR_MINOR_BASE 0
#else
#define IOWARRIOR_MINOR_BASE 208 // SKELETON_MINOR_BASE 192 + 16, not official yet
#endif
/* interrupt input queue size */
#define MAX_INTERRUPT_BUFFER 16
/*
maximum number of urbs that are submitted for writes at the same time,
this applies to the IOWarrior56 only!
IOWarrior24 and IOWarrior40 use synchronous usb_control_msg calls.
*/
#define MAX_WRITES_IN_FLIGHT 4
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");
static struct usb_driver iowarrior_driver;
/*--------------*/
/* data */
/*--------------*/
/* Structure to hold all of our device specific stuff */
struct iowarrior {
struct mutex mutex; /* locks this structure */
struct usb_device *udev; /* save off the usb device pointer */
struct usb_interface *interface; /* the interface for this device */
unsigned char minor; /* the starting minor number for this device */
struct usb_endpoint_descriptor *int_out_endpoint; /* endpoint for reading (needed for IOW56 only) */
struct usb_endpoint_descriptor *int_in_endpoint; /* endpoint for reading */
struct urb *int_in_urb; /* the urb for reading data */
unsigned char *int_in_buffer; /* buffer for data to be read */
unsigned char serial_number; /* to detect lost packages */
unsigned char *read_queue; /* size is MAX_INTERRUPT_BUFFER * packet size */
wait_queue_head_t read_wait;
wait_queue_head_t write_wait; /* wait-queue for writing to the device */
atomic_t write_busy; /* number of write-urbs submitted */
atomic_t read_idx;
atomic_t intr_idx;
atomic_t overflow_flag; /* signals an index 'rollover' */
int present; /* this is 1 as long as the device is connected */
int opened; /* this is 1 if the device is currently open */
char chip_serial[9]; /* the serial number string of the chip connected */
int report_size; /* number of bytes in a report */
u16 product_id;
struct usb_anchor submitted;
};
/*--------------*/
/* globals */
/*--------------*/
#define USB_REQ_GET_REPORT 0x01
//#if 0
static int usb_get_report(struct usb_device *dev,
struct usb_host_interface *inter, unsigned char type,
unsigned char id, void *buf, int size)
{
return usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
USB_REQ_GET_REPORT,
USB_DIR_IN | USB_TYPE_CLASS |
USB_RECIP_INTERFACE, (type << 8) + id,
inter->desc.bInterfaceNumber, buf, size,
USB_CTRL_GET_TIMEOUT);
}
//#endif
#define USB_REQ_SET_REPORT 0x09
static int usb_set_report(struct usb_interface *intf, unsigned char type,
unsigned char id, void *buf, int size)
{
return usb_control_msg(interface_to_usbdev(intf),
usb_sndctrlpipe(interface_to_usbdev(intf), 0),
USB_REQ_SET_REPORT,
USB_TYPE_CLASS | USB_RECIP_INTERFACE,
(type << 8) + id,
intf->cur_altsetting->desc.bInterfaceNumber, buf,
size, 1000);
}
/*---------------------*/
/* driver registration */
/*---------------------*/
/* table of devices that work with this driver */
static const struct usb_device_id iowarrior_ids[] = {
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW40)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW24)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOWPV1)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOWPV2)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW56)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW24SAG)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW56AM)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW28)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW28L)},
{USB_DEVICE(USB_VENDOR_ID_CODEMERCS, USB_DEVICE_ID_CODEMERCS_IOW100)},
{} /* Terminating entry */
};
MODULE_DEVICE_TABLE(usb, iowarrior_ids);
/*
* USB callback handler for reading data
*/
static void iowarrior_callback(struct urb *urb)
{
struct iowarrior *dev = urb->context;
int intr_idx;
int read_idx;
int aux_idx;
int offset;
int status = urb->status;
int retval;
switch (status) {
case 0:
/* success */
break;
case -ECONNRESET:
case -ENOENT:
case -ESHUTDOWN:
return;
default:
goto exit;
}
intr_idx = atomic_read(&dev->intr_idx);
/* aux_idx become previous intr_idx */
aux_idx = (intr_idx == 0) ? (MAX_INTERRUPT_BUFFER - 1) : (intr_idx - 1);
read_idx = atomic_read(&dev->read_idx);
/* queue is not empty and it's interface 0 */
if ((intr_idx != read_idx)
&& (dev->interface->cur_altsetting->desc.bInterfaceNumber == 0)) {
/* + 1 for serial number */
offset = aux_idx * (dev->report_size + 1);
if (!memcmp
(dev->read_queue + offset, urb->transfer_buffer,
dev->report_size)) {
/* equal values on interface 0 will be ignored */
goto exit;
}
}
/* aux_idx become next intr_idx */
aux_idx = (intr_idx == (MAX_INTERRUPT_BUFFER - 1)) ? 0 : (intr_idx + 1);
if (read_idx == aux_idx) {
/* queue full, dropping oldest input */
read_idx = (++read_idx == MAX_INTERRUPT_BUFFER) ? 0 : read_idx;
atomic_set(&dev->read_idx, read_idx);
atomic_set(&dev->overflow_flag, 1);
}
/* +1 for serial number */
offset = intr_idx * (dev->report_size + 1);
memcpy(dev->read_queue + offset, urb->transfer_buffer,
dev->report_size);
*(dev->read_queue + offset + (dev->report_size)) = dev->serial_number++;
atomic_set(&dev->intr_idx, aux_idx);
/* tell the blocking read about the new data */
wake_up_interruptible(&dev->read_wait);
exit:
retval = usb_submit_urb(urb, GFP_ATOMIC);
if (retval)
dev_err(&dev->interface->dev, "%s - usb_submit_urb failed with result %d\n",
__func__, retval);
}
/*
* USB Callback handler for write-ops
*/
static void iowarrior_write_callback(struct urb *urb)
{
struct iowarrior *dev;
int status = urb->status;
dev = urb->context;
/* sync/async unlink faults aren't errors */
if (status &&
!(status == -ENOENT ||
status == -ECONNRESET || status == -ESHUTDOWN)) {
dev_dbg(&dev->interface->dev,
"nonzero write bulk status received: %d\n", status);
}
/* free up our allocated buffer */
usb_free_coherent(urb->dev, urb->transfer_buffer_length,
urb->transfer_buffer, urb->transfer_dma);
/* tell a waiting writer the interrupt-out-pipe is available again */
atomic_dec(&dev->write_busy);
wake_up_interruptible(&dev->write_wait);
}
/*
* iowarrior_delete
*/
static inline void iowarrior_delete(struct iowarrior *dev)
{
dev_dbg(&dev->interface->dev, "minor %d\n", dev->minor); kfree(dev->int_in_buffer);
usb_free_urb(dev->int_in_urb);
kfree(dev->read_queue);
usb_put_intf(dev->interface);
kfree(dev);
}
/*---------------------*/
/* fops implementation */
/*---------------------*/
static int read_index(struct iowarrior *dev)
{
int intr_idx, read_idx;
read_idx = atomic_read(&dev->read_idx);
intr_idx = atomic_read(&dev->intr_idx);
return (read_idx == intr_idx ? -1 : read_idx);
}
/*
* iowarrior_read
*/
static ssize_t iowarrior_read(struct file *file, char __user *buffer,
size_t count, loff_t *ppos)
{
struct iowarrior *dev;
int read_idx;
int offset;
dev = file->private_data;
/* verify that the device wasn't unplugged */
if (!dev || !dev->present)
return -ENODEV;
dev_dbg(&dev->interface->dev, "minor %d, count = %zd\n",
dev->minor, count);
/* read count must be packet size (+ time stamp) */
if ((count != dev->report_size)
&& (count != (dev->report_size + 1)))
return -EINVAL;
/* repeat until no buffer overrun in callback handler occur */
do {
atomic_set(&dev->overflow_flag, 0);
if ((read_idx = read_index(dev)) == -1) {
/* queue empty */
if (file->f_flags & O_NONBLOCK)
return -EAGAIN;
else {
//next line will return when there is either new data, or the device is unplugged
int r = wait_event_interruptible(dev->read_wait,
(!dev->present
|| (read_idx =
read_index
(dev)) !=
-1));
if (r) {
//we were interrupted by a signal
return -ERESTART;
}
if (!dev->present) {
//The device was unplugged
return -ENODEV;
}
if (read_idx == -1) {
// Can this happen ???
return 0;
}
}
}
offset = read_idx * (dev->report_size + 1);
if (copy_to_user(buffer, dev->read_queue + offset, count)) {
return -EFAULT;
}
} while (atomic_read(&dev->overflow_flag));
read_idx = ++read_idx == MAX_INTERRUPT_BUFFER ? 0 : read_idx;
atomic_set(&dev->read_idx, read_idx);
return count;
}
/*
* iowarrior_write
*/
static ssize_t iowarrior_write(struct file *file,
const char __user *user_buffer,
size_t count, loff_t *ppos)
{
struct iowarrior *dev;
int retval = 0;
char *buf = NULL; /* for IOW24 and IOW56 we need a buffer */
struct urb *int_out_urb = NULL;
dev = file->private_data;
mutex_lock(&dev->mutex);
/* verify that the device wasn't unplugged */
if (!dev->present) {
retval = -ENODEV;
goto exit;
}
dev_dbg(&dev->interface->dev, "minor %d, count = %zd\n",
dev->minor, count);
/* if count is 0 we're already done */
if (count == 0) {
retval = 0;
goto exit;
}
/* We only accept full reports */
if (count != dev->report_size) {
retval = -EINVAL;
goto exit;
}
switch (dev->product_id) {
case USB_DEVICE_ID_CODEMERCS_IOW24:
case USB_DEVICE_ID_CODEMERCS_IOW24SAG:
case USB_DEVICE_ID_CODEMERCS_IOWPV1:
case USB_DEVICE_ID_CODEMERCS_IOWPV2:
case USB_DEVICE_ID_CODEMERCS_IOW40:
/* IOW24 and IOW40 use a synchronous call */
buf = memdup_user(user_buffer, count);
if (IS_ERR(buf)) {
retval = PTR_ERR(buf);
goto exit;
}
retval = usb_set_report(dev->interface, 2, 0, buf, count);
kfree(buf);
goto exit;
case USB_DEVICE_ID_CODEMERCS_IOW56:
case USB_DEVICE_ID_CODEMERCS_IOW56AM:
case USB_DEVICE_ID_CODEMERCS_IOW28:
case USB_DEVICE_ID_CODEMERCS_IOW28L:
case USB_DEVICE_ID_CODEMERCS_IOW100:
/* The IOW56 uses asynchronous IO and more urbs */
if (atomic_read(&dev->write_busy) == MAX_WRITES_IN_FLIGHT) {
/* Wait until we are below the limit for submitted urbs */
if (file->f_flags & O_NONBLOCK) {
retval = -EAGAIN;
goto exit;
} else {
retval = wait_event_interruptible(dev->write_wait,
(!dev->present || (atomic_read (&dev-> write_busy) < MAX_WRITES_IN_FLIGHT)));
if (retval) {
/* we were interrupted by a signal */
retval = -ERESTART;
goto exit;
}
if (!dev->present) {
/* The device was unplugged */
retval = -ENODEV;
goto exit;
}
if (!dev->opened) {
/* We were closed while waiting for an URB */
retval = -ENODEV;
goto exit;
}
}
}
atomic_inc(&dev->write_busy);
int_out_urb = usb_alloc_urb(0, GFP_KERNEL);
if (!int_out_urb) {
retval = -ENOMEM;
goto error_no_urb;
}
buf = usb_alloc_coherent(dev->udev, dev->report_size,
GFP_KERNEL, &int_out_urb->transfer_dma);
if (!buf) {
retval = -ENOMEM;
dev_dbg(&dev->interface->dev,
"Unable to allocate buffer\n");
goto error_no_buffer;
}
usb_fill_int_urb(int_out_urb, dev->udev,
usb_sndintpipe(dev->udev,
dev->int_out_endpoint->bEndpointAddress),
buf, dev->report_size,
iowarrior_write_callback, dev,
dev->int_out_endpoint->bInterval);
int_out_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
if (copy_from_user(buf, user_buffer, count)) {
retval = -EFAULT;
goto error;
}
usb_anchor_urb(int_out_urb, &dev->submitted);
retval = usb_submit_urb(int_out_urb, GFP_KERNEL);
if (retval) {
dev_dbg(&dev->interface->dev,
"submit error %d for urb nr.%d\n",
retval, atomic_read(&dev->write_busy));
usb_unanchor_urb(int_out_urb);
goto error;
}
/* submit was ok */
retval = count;
usb_free_urb(int_out_urb);
goto exit;
default:
/* what do we have here ? An unsupported Product-ID ? */
dev_err(&dev->interface->dev, "%s - not supported for product=0x%x\n",
__func__, dev->product_id);
retval = -EFAULT;
goto exit;
}
error:
usb_free_coherent(dev->udev, dev->report_size, buf,
int_out_urb->transfer_dma);
error_no_buffer:
usb_free_urb(int_out_urb);
error_no_urb:
atomic_dec(&dev->write_busy);
wake_up_interruptible(&dev->write_wait);
exit:
mutex_unlock(&dev->mutex);
return retval;
}
/*
* iowarrior_ioctl
*/
static long iowarrior_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct iowarrior *dev = NULL;
__u8 *buffer;
__u8 __user *user_buffer;
int retval;
int io_res; /* checks for bytes read/written and copy_to/from_user results */
dev = file->private_data;
if (!dev)
return -ENODEV;
buffer = kzalloc(dev->report_size, GFP_KERNEL);
if (!buffer)
return -ENOMEM;
mutex_lock(&dev->mutex);
/* verify that the device wasn't unplugged */
if (!dev->present) {
retval = -ENODEV;
goto error_out;
}
dev_dbg(&dev->interface->dev, "minor %d, cmd 0x%.4x, arg %ld\n",
dev->minor, cmd, arg);
retval = 0;
switch (cmd) {
case IOW_WRITE:
if (dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW24 ||
dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW24SAG ||
dev->product_id == USB_DEVICE_ID_CODEMERCS_IOWPV1 ||
dev->product_id == USB_DEVICE_ID_CODEMERCS_IOWPV2 ||
dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW40) {
user_buffer = (__u8 __user *)arg;
io_res = copy_from_user(buffer, user_buffer,
dev->report_size);
if (io_res) {
retval = -EFAULT;
} else {
io_res = usb_set_report(dev->interface, 2, 0,
buffer,
dev->report_size);
if (io_res < 0)
retval = io_res;
}
} else {
retval = -EINVAL;
dev_err(&dev->interface->dev,
"ioctl 'IOW_WRITE' is not supported for product=0x%x.\n",
dev->product_id);
}
break;
case IOW_READ:
user_buffer = (__u8 __user *)arg;
io_res = usb_get_report(dev->udev,
dev->interface->cur_altsetting, 1, 0,
buffer, dev->report_size);
if (io_res < 0)
retval = io_res;
else {
io_res = copy_to_user(user_buffer, buffer, dev->report_size);
if (io_res)
retval = -EFAULT;
}
break;
case IOW_GETINFO:
{
/* Report available information for the device */
struct iowarrior_info info;
/* needed for power consumption */
struct usb_config_descriptor *cfg_descriptor = &dev->udev->actconfig->desc;
memset(&info, 0, sizeof(info));
/* directly from the descriptor */
info.vendor = le16_to_cpu(dev->udev->descriptor.idVendor);
info.product = dev->product_id;
info.revision = le16_to_cpu(dev->udev->descriptor.bcdDevice);
/* 0==UNKNOWN, 1==LOW(usb1.1) ,2=FULL(usb1.1), 3=HIGH(usb2.0) */
info.speed = dev->udev->speed;
info.if_num = dev->interface->cur_altsetting->desc.bInterfaceNumber;
info.report_size = dev->report_size;
/* serial number string has been read earlier 8 chars or empty string */
memcpy(info.serial, dev->chip_serial,
sizeof(dev->chip_serial));
if (cfg_descriptor == NULL) {
info.power = -1; /* no information available */
} else {
/* the MaxPower is stored in units of 2mA to make it fit into a byte-value */
info.power = cfg_descriptor->bMaxPower * 2;
}
io_res = copy_to_user((struct iowarrior_info __user *)arg, &info,
sizeof(struct iowarrior_info));
if (io_res)
retval = -EFAULT;
break;
}
default:
/* return that we did not understand this ioctl call */
retval = -ENOTTY;
break;
}
error_out:
/* unlock the device */
mutex_unlock(&dev->mutex);
kfree(buffer);
return retval;
}
/*
* iowarrior_open
*/
static int iowarrior_open(struct inode *inode, struct file *file)
{
struct iowarrior *dev = NULL;
struct usb_interface *interface;
int subminor;
int retval = 0;
subminor = iminor(inode);
interface = usb_find_interface(&iowarrior_driver, subminor);
if (!interface) {
pr_err("%s - error, can't find device for minor %d\n",
__func__, subminor);
return -ENODEV;
}
dev = usb_get_intfdata(interface);
if (!dev)
return -ENODEV;
mutex_lock(&dev->mutex);
/* Only one process can open each device, no sharing. */
if (dev->opened) {
retval = -EBUSY;
goto out;
}
/* setup interrupt handler for receiving values */
if ((retval = usb_submit_urb(dev->int_in_urb, GFP_KERNEL)) < 0) { dev_err(&interface->dev, "Error %d while submitting URB\n", retval);
retval = -EFAULT;
goto out;
}
/* increment our usage count for the driver */
++dev->opened;
/* save our object in the file's private structure */
file->private_data = dev;
retval = 0;
out:
mutex_unlock(&dev->mutex); return retval;
}
/*
* iowarrior_release
*/
static int iowarrior_release(struct inode *inode, struct file *file)
{
struct iowarrior *dev;
int retval = 0;
dev = file->private_data;
if (!dev)
return -ENODEV;
dev_dbg(&dev->interface->dev, "minor %d\n", dev->minor);
/* lock our device */
mutex_lock(&dev->mutex);
if (dev->opened <= 0) {
retval = -ENODEV; /* close called more than once */
mutex_unlock(&dev->mutex);
} else {
dev->opened = 0; /* we're closing now */
retval = 0;
if (dev->present) {
/*
The device is still connected so we only shutdown
pending read-/write-ops.
*/
usb_kill_urb(dev->int_in_urb);
wake_up_interruptible(&dev->read_wait);
wake_up_interruptible(&dev->write_wait);
mutex_unlock(&dev->mutex);
} else {
/* The device was unplugged, cleanup resources */
mutex_unlock(&dev->mutex);
iowarrior_delete(dev);
}
}
return retval;
}
static __poll_t iowarrior_poll(struct file *file, poll_table * wait)
{
struct iowarrior *dev = file->private_data;
__poll_t mask = 0;
if (!dev->present)
return EPOLLERR | EPOLLHUP;
poll_wait(file, &dev->read_wait, wait);
poll_wait(file, &dev->write_wait, wait);
if (!dev->present)
return EPOLLERR | EPOLLHUP;
if (read_index(dev) != -1)
mask |= EPOLLIN | EPOLLRDNORM;
if (atomic_read(&dev->write_busy) < MAX_WRITES_IN_FLIGHT)
mask |= EPOLLOUT | EPOLLWRNORM;
return mask;
}
/*
* File operations needed when we register this driver.
* This assumes that this driver NEEDS file operations,
* of course, which means that the driver is expected
* to have a node in the /dev directory. If the USB
* device were for a network interface then the driver
* would use "struct net_driver" instead, and a serial
* device would use "struct tty_driver".
*/
static const struct file_operations iowarrior_fops = {
.owner = THIS_MODULE,
.write = iowarrior_write,
.read = iowarrior_read,
.unlocked_ioctl = iowarrior_ioctl,
.open = iowarrior_open,
.release = iowarrior_release,
.poll = iowarrior_poll,
.llseek = noop_llseek,
};
static char *iowarrior_devnode(const struct device *dev, umode_t *mode)
{
return kasprintf(GFP_KERNEL, "usb/%s", dev_name(dev));
}
/*
* usb class driver info in order to get a minor number from the usb core,
* and to have the device registered with devfs and the driver core
*/
static struct usb_class_driver iowarrior_class = {
.name = "iowarrior%d",
.devnode = iowarrior_devnode,
.fops = &iowarrior_fops,
.minor_base = IOWARRIOR_MINOR_BASE,
};
/*---------------------------------*/
/* probe and disconnect functions */
/*---------------------------------*/
/*
* iowarrior_probe
*
* Called by the usb core when a new device is connected that it thinks
* this driver might be interested in.
*/
static int iowarrior_probe(struct usb_interface *interface,
const struct usb_device_id *id)
{
struct usb_device *udev = interface_to_usbdev(interface);
struct iowarrior *dev = NULL;
struct usb_host_interface *iface_desc;
int retval = -ENOMEM;
int res;
/* allocate memory for our device state and initialize it */
dev = kzalloc(sizeof(struct iowarrior), GFP_KERNEL);
if (!dev)
return retval;
mutex_init(&dev->mutex);
atomic_set(&dev->intr_idx, 0);
atomic_set(&dev->read_idx, 0);
atomic_set(&dev->overflow_flag, 0);
init_waitqueue_head(&dev->read_wait);
atomic_set(&dev->write_busy, 0);
init_waitqueue_head(&dev->write_wait);
dev->udev = udev;
dev->interface = usb_get_intf(interface);
iface_desc = interface->cur_altsetting;
dev->product_id = le16_to_cpu(udev->descriptor.idProduct);
init_usb_anchor(&dev->submitted);
res = usb_find_last_int_in_endpoint(iface_desc, &dev->int_in_endpoint);
if (res) {
dev_err(&interface->dev, "no interrupt-in endpoint found\n");
retval = res;
goto error;
}
if ((dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW56) ||
(dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW56AM) ||
(dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW28) ||
(dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW28L) ||
(dev->product_id == USB_DEVICE_ID_CODEMERCS_IOW100)) {
res = usb_find_last_int_out_endpoint(iface_desc,
&dev->int_out_endpoint);
if (res) {
dev_err(&interface->dev, "no interrupt-out endpoint found\n");
retval = res;
goto error;
}
}
/* we have to check the report_size often, so remember it in the endianness suitable for our machine */
dev->report_size = usb_endpoint_maxp(dev->int_in_endpoint);
/*
* Some devices need the report size to be different than the
* endpoint size.
*/
if (dev->interface->cur_altsetting->desc.bInterfaceNumber == 0) {
switch (dev->product_id) {
case USB_DEVICE_ID_CODEMERCS_IOW56:
case USB_DEVICE_ID_CODEMERCS_IOW56AM:
dev->report_size = 7;
break;
case USB_DEVICE_ID_CODEMERCS_IOW28:
case USB_DEVICE_ID_CODEMERCS_IOW28L:
dev->report_size = 4;
break;
case USB_DEVICE_ID_CODEMERCS_IOW100:
dev->report_size = 12;
break;
}
}
/* create the urb and buffer for reading */
dev->int_in_urb = usb_alloc_urb(0, GFP_KERNEL);
if (!dev->int_in_urb)
goto error;
dev->int_in_buffer = kmalloc(dev->report_size, GFP_KERNEL);
if (!dev->int_in_buffer)
goto error;
usb_fill_int_urb(dev->int_in_urb, dev->udev,
usb_rcvintpipe(dev->udev,
dev->int_in_endpoint->bEndpointAddress),
dev->int_in_buffer, dev->report_size,
iowarrior_callback, dev,
dev->int_in_endpoint->bInterval);
/* create an internal buffer for interrupt data from the device */
dev->read_queue =
kmalloc_array(dev->report_size + 1, MAX_INTERRUPT_BUFFER,
GFP_KERNEL);
if (!dev->read_queue)
goto error;
/* Get the serial-number of the chip */
memset(dev->chip_serial, 0x00, sizeof(dev->chip_serial));
usb_string(udev, udev->descriptor.iSerialNumber, dev->chip_serial,
sizeof(dev->chip_serial));
if (strlen(dev->chip_serial) != 8)
memset(dev->chip_serial, 0x00, sizeof(dev->chip_serial));
/* Set the idle timeout to 0, if this is interface 0 */
if (dev->interface->cur_altsetting->desc.bInterfaceNumber == 0) {
usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
0x0A,
USB_TYPE_CLASS | USB_RECIP_INTERFACE, 0,
0, NULL, 0, USB_CTRL_SET_TIMEOUT);
}
/* allow device read and ioctl */
dev->present = 1;
/* we can register the device now, as it is ready */
usb_set_intfdata(interface, dev);
retval = usb_register_dev(interface, &iowarrior_class);
if (retval) {
/* something prevented us from registering this driver */
dev_err(&interface->dev, "Not able to get a minor for this device.\n");
goto error;
}
dev->minor = interface->minor;
/* let the user know what node this device is now attached to */
dev_info(&interface->dev, "IOWarrior product=0x%x, serial=%s interface=%d "
"now attached to iowarrior%d\n", dev->product_id, dev->chip_serial,
iface_desc->desc.bInterfaceNumber, dev->minor - IOWARRIOR_MINOR_BASE);
return retval;
error:
iowarrior_delete(dev);
return retval;
}
/*
* iowarrior_disconnect
*
* Called by the usb core when the device is removed from the system.
*/
static void iowarrior_disconnect(struct usb_interface *interface)
{
struct iowarrior *dev = usb_get_intfdata(interface);
int minor = dev->minor;
usb_deregister_dev(interface, &iowarrior_class);
mutex_lock(&dev->mutex);
/* prevent device read, write and ioctl */
dev->present = 0;
if (dev->opened) {
/* There is a process that holds a filedescriptor to the device ,
so we only shutdown read-/write-ops going on.
Deleting the device is postponed until close() was called.
*/
usb_kill_urb(dev->int_in_urb);
usb_kill_anchored_urbs(&dev->submitted);
wake_up_interruptible(&dev->read_wait);
wake_up_interruptible(&dev->write_wait);
mutex_unlock(&dev->mutex);
} else {
/* no process is using the device, cleanup now */
mutex_unlock(&dev->mutex); iowarrior_delete(dev);
}
dev_info(&interface->dev, "I/O-Warror #%d now disconnected\n",
minor - IOWARRIOR_MINOR_BASE);
}
/* usb specific object needed to register this driver with the usb subsystem */
static struct usb_driver iowarrior_driver = {
.name = "iowarrior",
.probe = iowarrior_probe,
.disconnect = iowarrior_disconnect,
.id_table = iowarrior_ids,
};
module_usb_driver(iowarrior_driver);
// SPDX-License-Identifier: GPL-2.0-or-later
/* Common capabilities, needed by capability.o.
*/
#include <linux/capability.h>
#include <linux/audit.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/lsm_hooks.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/ptrace.h>
#include <linux/xattr.h>
#include <linux/hugetlb.h>
#include <linux/mount.h>
#include <linux/sched.h>
#include <linux/prctl.h>
#include <linux/securebits.h>
#include <linux/user_namespace.h>
#include <linux/binfmts.h>
#include <linux/personality.h>
#include <linux/mnt_idmapping.h>
#include <uapi/linux/lsm.h>
/*
* If a non-root user executes a setuid-root binary in
* !secure(SECURE_NOROOT) mode, then we raise capabilities.
* However if fE is also set, then the intent is for only
* the file capabilities to be applied, and the setuid-root
* bit is left on either to change the uid (plausible) or
* to get full privilege on a kernel without file capabilities
* support. So in that case we do not raise capabilities.
*
* Warn if that happens, once per boot.
*/
static void warn_setuid_and_fcaps_mixed(const char *fname)
{
static int warned;
if (!warned) {
printk(KERN_INFO "warning: `%s' has both setuid-root and"
" effective capabilities. Therefore not raising all"
" capabilities.\n", fname);
warned = 1;
}
}
/**
* cap_capable - Determine whether a task has a particular effective capability
* @cred: The credentials to use
* @targ_ns: The user namespace in which we need the capability
* @cap: The capability to check for
* @opts: Bitmask of options defined in include/linux/security.h
*
* Determine whether the nominated task has the specified capability amongst
* its effective set, returning 0 if it does, -ve if it does not.
*
* NOTE WELL: cap_has_capability() cannot be used like the kernel's capable()
* and has_capability() functions. That is, it has the reverse semantics:
* cap_has_capability() returns 0 when a task has a capability, but the
* kernel's capable() and has_capability() returns 1 for this case.
*/
int cap_capable(const struct cred *cred, struct user_namespace *targ_ns,
int cap, unsigned int opts)
{
struct user_namespace *ns = targ_ns;
/* See if cred has the capability in the target user namespace
* by examining the target user namespace and all of the target
* user namespace's parents.
*/
for (;;) {
/* Do we have the necessary capabilities? */
if (ns == cred->user_ns) return cap_raised(cred->cap_effective, cap) ? 0 : -EPERM;
/*
* If we're already at a lower level than we're looking for,
* we're done searching.
*/
if (ns->level <= cred->user_ns->level)
return -EPERM;
/*
* The owner of the user namespace in the parent of the
* user namespace has all caps.
*/
if ((ns->parent == cred->user_ns) && uid_eq(ns->owner, cred->euid))
return 0;
/*
* If you have a capability in a parent user ns, then you have
* it over all children user namespaces as well.
*/
ns = ns->parent;
}
/* We never get here */
}
/**
* cap_settime - Determine whether the current process may set the system clock
* @ts: The time to set
* @tz: The timezone to set
*
* Determine whether the current process may set the system clock and timezone
* information, returning 0 if permission granted, -ve if denied.
*/
int cap_settime(const struct timespec64 *ts, const struct timezone *tz)
{
if (!capable(CAP_SYS_TIME))
return -EPERM;
return 0;
}
/**
* cap_ptrace_access_check - Determine whether the current process may access
* another
* @child: The process to be accessed
* @mode: The mode of attachment.
*
* If we are in the same or an ancestor user_ns and have all the target
* task's capabilities, then ptrace access is allowed.
* If we have the ptrace capability to the target user_ns, then ptrace
* access is allowed.
* Else denied.
*
* Determine whether a process may access another, returning 0 if permission
* granted, -ve if denied.
*/
int cap_ptrace_access_check(struct task_struct *child, unsigned int mode)
{
int ret = 0;
const struct cred *cred, *child_cred;
const kernel_cap_t *caller_caps;
rcu_read_lock();
cred = current_cred();
child_cred = __task_cred(child);
if (mode & PTRACE_MODE_FSCREDS)
caller_caps = &cred->cap_effective;
else
caller_caps = &cred->cap_permitted;
if (cred->user_ns == child_cred->user_ns &&
cap_issubset(child_cred->cap_permitted, *caller_caps))
goto out;
if (ns_capable(child_cred->user_ns, CAP_SYS_PTRACE))
goto out;
ret = -EPERM;
out:
rcu_read_unlock();
return ret;
}
/**
* cap_ptrace_traceme - Determine whether another process may trace the current
* @parent: The task proposed to be the tracer
*
* If parent is in the same or an ancestor user_ns and has all current's
* capabilities, then ptrace access is allowed.
* If parent has the ptrace capability to current's user_ns, then ptrace
* access is allowed.
* Else denied.
*
* Determine whether the nominated task is permitted to trace the current
* process, returning 0 if permission is granted, -ve if denied.
*/
int cap_ptrace_traceme(struct task_struct *parent)
{
int ret = 0;
const struct cred *cred, *child_cred;
rcu_read_lock();
cred = __task_cred(parent);
child_cred = current_cred();
if (cred->user_ns == child_cred->user_ns &&
cap_issubset(child_cred->cap_permitted, cred->cap_permitted))
goto out;
if (has_ns_capability(parent, child_cred->user_ns, CAP_SYS_PTRACE))
goto out;
ret = -EPERM;
out:
rcu_read_unlock();
return ret;
}
/**
* cap_capget - Retrieve a task's capability sets
* @target: The task from which to retrieve the capability sets
* @effective: The place to record the effective set
* @inheritable: The place to record the inheritable set
* @permitted: The place to record the permitted set
*
* This function retrieves the capabilities of the nominated task and returns
* them to the caller.
*/
int cap_capget(const struct task_struct *target, kernel_cap_t *effective,
kernel_cap_t *inheritable, kernel_cap_t *permitted)
{
const struct cred *cred;
/* Derived from kernel/capability.c:sys_capget. */
rcu_read_lock();
cred = __task_cred(target);
*effective = cred->cap_effective;
*inheritable = cred->cap_inheritable;
*permitted = cred->cap_permitted;
rcu_read_unlock();
return 0;
}
/*
* Determine whether the inheritable capabilities are limited to the old
* permitted set. Returns 1 if they are limited, 0 if they are not.
*/
static inline int cap_inh_is_capped(void)
{
/* they are so limited unless the current task has the CAP_SETPCAP
* capability
*/
if (cap_capable(current_cred(), current_cred()->user_ns,
CAP_SETPCAP, CAP_OPT_NONE) == 0)
return 0;
return 1;
}
/**
* cap_capset - Validate and apply proposed changes to current's capabilities
* @new: The proposed new credentials; alterations should be made here
* @old: The current task's current credentials
* @effective: A pointer to the proposed new effective capabilities set
* @inheritable: A pointer to the proposed new inheritable capabilities set
* @permitted: A pointer to the proposed new permitted capabilities set
*
* This function validates and applies a proposed mass change to the current
* process's capability sets. The changes are made to the proposed new
* credentials, and assuming no error, will be committed by the caller of LSM.
*/
int cap_capset(struct cred *new,
const struct cred *old,
const kernel_cap_t *effective,
const kernel_cap_t *inheritable,
const kernel_cap_t *permitted)
{
if (cap_inh_is_capped() &&
!cap_issubset(*inheritable,
cap_combine(old->cap_inheritable,
old->cap_permitted)))
/* incapable of using this inheritable set */
return -EPERM;
if (!cap_issubset(*inheritable,
cap_combine(old->cap_inheritable,
old->cap_bset)))
/* no new pI capabilities outside bounding set */
return -EPERM;
/* verify restrictions on target's new Permitted set */
if (!cap_issubset(*permitted, old->cap_permitted))
return -EPERM;
/* verify the _new_Effective_ is a subset of the _new_Permitted_ */
if (!cap_issubset(*effective, *permitted))
return -EPERM;
new->cap_effective = *effective;
new->cap_inheritable = *inheritable;
new->cap_permitted = *permitted;
/*
* Mask off ambient bits that are no longer both permitted and
* inheritable.
*/
new->cap_ambient = cap_intersect(new->cap_ambient,
cap_intersect(*permitted,
*inheritable));
if (WARN_ON(!cap_ambient_invariant_ok(new)))
return -EINVAL;
return 0;
}
/**
* cap_inode_need_killpriv - Determine if inode change affects privileges
* @dentry: The inode/dentry in being changed with change marked ATTR_KILL_PRIV
*
* Determine if an inode having a change applied that's marked ATTR_KILL_PRIV
* affects the security markings on that inode, and if it is, should
* inode_killpriv() be invoked or the change rejected.
*
* Return: 1 if security.capability has a value, meaning inode_killpriv()
* is required, 0 otherwise, meaning inode_killpriv() is not required.
*/
int cap_inode_need_killpriv(struct dentry *dentry)
{
struct inode *inode = d_backing_inode(dentry);
int error;
error = __vfs_getxattr(dentry, inode, XATTR_NAME_CAPS, NULL, 0);
return error > 0;
}
/**
* cap_inode_killpriv - Erase the security markings on an inode
*
* @idmap: idmap of the mount the inode was found from
* @dentry: The inode/dentry to alter
*
* Erase the privilege-enhancing security markings on an inode.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*
* Return: 0 if successful, -ve on error.
*/
int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry)
{
int error;
error = __vfs_removexattr(idmap, dentry, XATTR_NAME_CAPS);
if (error == -EOPNOTSUPP)
error = 0;
return error;
}
static bool rootid_owns_currentns(vfsuid_t rootvfsuid)
{
struct user_namespace *ns;
kuid_t kroot;
if (!vfsuid_valid(rootvfsuid))
return false;
kroot = vfsuid_into_kuid(rootvfsuid);
for (ns = current_user_ns();; ns = ns->parent) {
if (from_kuid(ns, kroot) == 0)
return true;
if (ns == &init_user_ns)
break;
}
return false;
}
static __u32 sansflags(__u32 m)
{
return m & ~VFS_CAP_FLAGS_EFFECTIVE;
}
static bool is_v2header(int size, const struct vfs_cap_data *cap)
{
if (size != XATTR_CAPS_SZ_2)
return false;
return sansflags(le32_to_cpu(cap->magic_etc)) == VFS_CAP_REVISION_2;
}
static bool is_v3header(int size, const struct vfs_cap_data *cap)
{
if (size != XATTR_CAPS_SZ_3)
return false;
return sansflags(le32_to_cpu(cap->magic_etc)) == VFS_CAP_REVISION_3;
}
/*
* getsecurity: We are called for security.* before any attempt to read the
* xattr from the inode itself.
*
* This gives us a chance to read the on-disk value and convert it. If we
* return -EOPNOTSUPP, then vfs_getxattr() will call the i_op handler.
*
* Note we are not called by vfs_getxattr_alloc(), but that is only called
* by the integrity subsystem, which really wants the unconverted values -
* so that's good.
*/
int cap_inode_getsecurity(struct mnt_idmap *idmap,
struct inode *inode, const char *name, void **buffer,
bool alloc)
{
int size;
kuid_t kroot;
vfsuid_t vfsroot;
u32 nsmagic, magic;
uid_t root, mappedroot;
char *tmpbuf = NULL;
struct vfs_cap_data *cap;
struct vfs_ns_cap_data *nscap = NULL;
struct dentry *dentry;
struct user_namespace *fs_ns;
if (strcmp(name, "capability") != 0)
return -EOPNOTSUPP;
dentry = d_find_any_alias(inode);
if (!dentry)
return -EINVAL;
size = vfs_getxattr_alloc(idmap, dentry, XATTR_NAME_CAPS, &tmpbuf,
sizeof(struct vfs_ns_cap_data), GFP_NOFS);
dput(dentry);
/* gcc11 complains if we don't check for !tmpbuf */
if (size < 0 || !tmpbuf)
goto out_free;
fs_ns = inode->i_sb->s_user_ns;
cap = (struct vfs_cap_data *) tmpbuf;
if (is_v2header(size, cap)) {
root = 0;
} else if (is_v3header(size, cap)) {
nscap = (struct vfs_ns_cap_data *) tmpbuf;
root = le32_to_cpu(nscap->rootid);
} else {
size = -EINVAL;
goto out_free;
}
kroot = make_kuid(fs_ns, root);
/* If this is an idmapped mount shift the kuid. */
vfsroot = make_vfsuid(idmap, fs_ns, kroot);
/* If the root kuid maps to a valid uid in current ns, then return
* this as a nscap. */
mappedroot = from_kuid(current_user_ns(), vfsuid_into_kuid(vfsroot));
if (mappedroot != (uid_t)-1 && mappedroot != (uid_t)0) {
size = sizeof(struct vfs_ns_cap_data);
if (alloc) {
if (!nscap) {
/* v2 -> v3 conversion */
nscap = kzalloc(size, GFP_ATOMIC);
if (!nscap) {
size = -ENOMEM;
goto out_free;
}
nsmagic = VFS_CAP_REVISION_3;
magic = le32_to_cpu(cap->magic_etc);
if (magic & VFS_CAP_FLAGS_EFFECTIVE)
nsmagic |= VFS_CAP_FLAGS_EFFECTIVE;
memcpy(&nscap->data, &cap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
nscap->magic_etc = cpu_to_le32(nsmagic);
} else {
/* use allocated v3 buffer */
tmpbuf = NULL;
}
nscap->rootid = cpu_to_le32(mappedroot);
*buffer = nscap;
}
goto out_free;
}
if (!rootid_owns_currentns(vfsroot)) {
size = -EOVERFLOW;
goto out_free;
}
/* This comes from a parent namespace. Return as a v2 capability */
size = sizeof(struct vfs_cap_data);
if (alloc) {
if (nscap) {
/* v3 -> v2 conversion */
cap = kzalloc(size, GFP_ATOMIC);
if (!cap) {
size = -ENOMEM;
goto out_free;
}
magic = VFS_CAP_REVISION_2;
nsmagic = le32_to_cpu(nscap->magic_etc);
if (nsmagic & VFS_CAP_FLAGS_EFFECTIVE)
magic |= VFS_CAP_FLAGS_EFFECTIVE;
memcpy(&cap->data, &nscap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
cap->magic_etc = cpu_to_le32(magic);
} else {
/* use unconverted v2 */
tmpbuf = NULL;
}
*buffer = cap;
}
out_free:
kfree(tmpbuf);
return size;
}
/**
* rootid_from_xattr - translate root uid of vfs caps
*
* @value: vfs caps value which may be modified by this function
* @size: size of @ivalue
* @task_ns: user namespace of the caller
*/
static vfsuid_t rootid_from_xattr(const void *value, size_t size,
struct user_namespace *task_ns)
{
const struct vfs_ns_cap_data *nscap = value;
uid_t rootid = 0;
if (size == XATTR_CAPS_SZ_3)
rootid = le32_to_cpu(nscap->rootid);
return VFSUIDT_INIT(make_kuid(task_ns, rootid));
}
static bool validheader(size_t size, const struct vfs_cap_data *cap)
{
return is_v2header(size, cap) || is_v3header(size, cap);
}
/**
* cap_convert_nscap - check vfs caps
*
* @idmap: idmap of the mount the inode was found from
* @dentry: used to retrieve inode to check permissions on
* @ivalue: vfs caps value which may be modified by this function
* @size: size of @ivalue
*
* User requested a write of security.capability. If needed, update the
* xattr to change from v2 to v3, or to fixup the v3 rootid.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*
* Return: On success, return the new size; on error, return < 0.
*/
int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry,
const void **ivalue, size_t size)
{
struct vfs_ns_cap_data *nscap;
uid_t nsrootid;
const struct vfs_cap_data *cap = *ivalue;
__u32 magic, nsmagic;
struct inode *inode = d_backing_inode(dentry);
struct user_namespace *task_ns = current_user_ns(),
*fs_ns = inode->i_sb->s_user_ns;
kuid_t rootid;
vfsuid_t vfsrootid;
size_t newsize;
if (!*ivalue)
return -EINVAL;
if (!validheader(size, cap))
return -EINVAL;
if (!capable_wrt_inode_uidgid(idmap, inode, CAP_SETFCAP))
return -EPERM;
if (size == XATTR_CAPS_SZ_2 && (idmap == &nop_mnt_idmap))
if (ns_capable(inode->i_sb->s_user_ns, CAP_SETFCAP))
/* user is privileged, just write the v2 */
return size;
vfsrootid = rootid_from_xattr(*ivalue, size, task_ns);
if (!vfsuid_valid(vfsrootid))
return -EINVAL;
rootid = from_vfsuid(idmap, fs_ns, vfsrootid);
if (!uid_valid(rootid))
return -EINVAL;
nsrootid = from_kuid(fs_ns, rootid);
if (nsrootid == -1)
return -EINVAL;
newsize = sizeof(struct vfs_ns_cap_data);
nscap = kmalloc(newsize, GFP_ATOMIC);
if (!nscap)
return -ENOMEM;
nscap->rootid = cpu_to_le32(nsrootid);
nsmagic = VFS_CAP_REVISION_3;
magic = le32_to_cpu(cap->magic_etc);
if (magic & VFS_CAP_FLAGS_EFFECTIVE)
nsmagic |= VFS_CAP_FLAGS_EFFECTIVE;
nscap->magic_etc = cpu_to_le32(nsmagic);
memcpy(&nscap->data, &cap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
*ivalue = nscap;
return newsize;
}
/*
* Calculate the new process capability sets from the capability sets attached
* to a file.
*/
static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps,
struct linux_binprm *bprm,
bool *effective,
bool *has_fcap)
{
struct cred *new = bprm->cred;
int ret = 0;
if (caps->magic_etc & VFS_CAP_FLAGS_EFFECTIVE)
*effective = true;
if (caps->magic_etc & VFS_CAP_REVISION_MASK)
*has_fcap = true;
/*
* pP' = (X & fP) | (pI & fI)
* The addition of pA' is handled later.
*/
new->cap_permitted.val =
(new->cap_bset.val & caps->permitted.val) |
(new->cap_inheritable.val & caps->inheritable.val);
if (caps->permitted.val & ~new->cap_permitted.val)
/* insufficient to execute correctly */
ret = -EPERM;
/*
* For legacy apps, with no internal support for recognizing they
* do not have enough capabilities, we return an error if they are
* missing some "forced" (aka file-permitted) capabilities.
*/
return *effective ? ret : 0;
}
/**
* get_vfs_caps_from_disk - retrieve vfs caps from disk
*
* @idmap: idmap of the mount the inode was found from
* @dentry: dentry from which @inode is retrieved
* @cpu_caps: vfs capabilities
*
* Extract the on-exec-apply capability sets for an executable file.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*/
int get_vfs_caps_from_disk(struct mnt_idmap *idmap,
const struct dentry *dentry,
struct cpu_vfs_cap_data *cpu_caps)
{
struct inode *inode = d_backing_inode(dentry);
__u32 magic_etc;
int size;
struct vfs_ns_cap_data data, *nscaps = &data;
struct vfs_cap_data *caps = (struct vfs_cap_data *) &data;
kuid_t rootkuid;
vfsuid_t rootvfsuid;
struct user_namespace *fs_ns;
memset(cpu_caps, 0, sizeof(struct cpu_vfs_cap_data));
if (!inode)
return -ENODATA;
fs_ns = inode->i_sb->s_user_ns;
size = __vfs_getxattr((struct dentry *)dentry, inode,
XATTR_NAME_CAPS, &data, XATTR_CAPS_SZ);
if (size == -ENODATA || size == -EOPNOTSUPP)
/* no data, that's ok */
return -ENODATA;
if (size < 0)
return size;
if (size < sizeof(magic_etc))
return -EINVAL;
cpu_caps->magic_etc = magic_etc = le32_to_cpu(caps->magic_etc);
rootkuid = make_kuid(fs_ns, 0);
switch (magic_etc & VFS_CAP_REVISION_MASK) {
case VFS_CAP_REVISION_1:
if (size != XATTR_CAPS_SZ_1)
return -EINVAL;
break;
case VFS_CAP_REVISION_2:
if (size != XATTR_CAPS_SZ_2)
return -EINVAL;
break;
case VFS_CAP_REVISION_3:
if (size != XATTR_CAPS_SZ_3)
return -EINVAL;
rootkuid = make_kuid(fs_ns, le32_to_cpu(nscaps->rootid));
break;
default:
return -EINVAL;
}
rootvfsuid = make_vfsuid(idmap, fs_ns, rootkuid);
if (!vfsuid_valid(rootvfsuid))
return -ENODATA;
/* Limit the caps to the mounter of the filesystem
* or the more limited uid specified in the xattr.
*/
if (!rootid_owns_currentns(rootvfsuid))
return -ENODATA;
cpu_caps->permitted.val = le32_to_cpu(caps->data[0].permitted);
cpu_caps->inheritable.val = le32_to_cpu(caps->data[0].inheritable);
/*
* Rev1 had just a single 32-bit word, later expanded
* to a second one for the high bits
*/
if ((magic_etc & VFS_CAP_REVISION_MASK) != VFS_CAP_REVISION_1) {
cpu_caps->permitted.val += (u64)le32_to_cpu(caps->data[1].permitted) << 32;
cpu_caps->inheritable.val += (u64)le32_to_cpu(caps->data[1].inheritable) << 32;
}
cpu_caps->permitted.val &= CAP_VALID_MASK;
cpu_caps->inheritable.val &= CAP_VALID_MASK;
cpu_caps->rootid = vfsuid_into_kuid(rootvfsuid);
return 0;
}
/*
* Attempt to get the on-exec apply capability sets for an executable file from
* its xattrs and, if present, apply them to the proposed credentials being
* constructed by execve().
*/
static int get_file_caps(struct linux_binprm *bprm, const struct file *file,
bool *effective, bool *has_fcap)
{
int rc = 0;
struct cpu_vfs_cap_data vcaps;
cap_clear(bprm->cred->cap_permitted);
if (!file_caps_enabled)
return 0;
if (!mnt_may_suid(file->f_path.mnt))
return 0;
/*
* This check is redundant with mnt_may_suid() but is kept to make
* explicit that capability bits are limited to s_user_ns and its
* descendants.
*/
if (!current_in_userns(file->f_path.mnt->mnt_sb->s_user_ns))
return 0;
rc = get_vfs_caps_from_disk(file_mnt_idmap(file),
file->f_path.dentry, &vcaps);
if (rc < 0) {
if (rc == -EINVAL)
printk(KERN_NOTICE "Invalid argument reading file caps for %s\n",
bprm->filename);
else if (rc == -ENODATA)
rc = 0;
goto out;
}
rc = bprm_caps_from_vfs_caps(&vcaps, bprm, effective, has_fcap);
out:
if (rc)
cap_clear(bprm->cred->cap_permitted);
return rc;
}
static inline bool root_privileged(void) { return !issecure(SECURE_NOROOT); }
static inline bool __is_real(kuid_t uid, struct cred *cred)
{ return uid_eq(cred->uid, uid); }
static inline bool __is_eff(kuid_t uid, struct cred *cred)
{ return uid_eq(cred->euid, uid); }
static inline bool __is_suid(kuid_t uid, struct cred *cred)
{ return !__is_real(uid, cred) && __is_eff(uid, cred); }
/*
* handle_privileged_root - Handle case of privileged root
* @bprm: The execution parameters, including the proposed creds
* @has_fcap: Are any file capabilities set?
* @effective: Do we have effective root privilege?
* @root_uid: This namespace' root UID WRT initial USER namespace
*
* Handle the case where root is privileged and hasn't been neutered by
* SECURE_NOROOT. If file capabilities are set, they won't be combined with
* set UID root and nothing is changed. If we are root, cap_permitted is
* updated. If we have become set UID root, the effective bit is set.
*/
static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap,
bool *effective, kuid_t root_uid)
{
const struct cred *old = current_cred();
struct cred *new = bprm->cred;
if (!root_privileged())
return;
/*
* If the legacy file capability is set, then don't set privs
* for a setuid root binary run by a non-root user. Do set it
* for a root user just to cause least surprise to an admin.
*/
if (has_fcap && __is_suid(root_uid, new)) {
warn_setuid_and_fcaps_mixed(bprm->filename);
return;
}
/*
* To support inheritance of root-permissions and suid-root
* executables under compatibility mode, we override the
* capability sets for the file.
*/
if (__is_eff(root_uid, new) || __is_real(root_uid, new)) {
/* pP' = (cap_bset & ~0) | (pI & ~0) */
new->cap_permitted = cap_combine(old->cap_bset,
old->cap_inheritable);
}
/*
* If only the real uid is 0, we do not set the effective bit.
*/
if (__is_eff(root_uid, new))
*effective = true;
}
#define __cap_gained(field, target, source) \
!cap_issubset(target->cap_##field, source->cap_##field)
#define __cap_grew(target, source, cred) \
!cap_issubset(cred->cap_##target, cred->cap_##source)
#define __cap_full(field, cred) \
cap_issubset(CAP_FULL_SET, cred->cap_##field)
static inline bool __is_setuid(struct cred *new, const struct cred *old)
{ return !uid_eq(new->euid, old->uid); }
static inline bool __is_setgid(struct cred *new, const struct cred *old)
{ return !gid_eq(new->egid, old->gid); }
/*
* 1) Audit candidate if current->cap_effective is set
*
* We do not bother to audit if 3 things are true:
* 1) cap_effective has all caps
* 2) we became root *OR* are were already root
* 3) root is supposed to have all caps (SECURE_NOROOT)
* Since this is just a normal root execing a process.
*
* Number 1 above might fail if you don't have a full bset, but I think
* that is interesting information to audit.
*
* A number of other conditions require logging:
* 2) something prevented setuid root getting all caps
* 3) non-setuid root gets fcaps
* 4) non-setuid root gets ambient
*/
static inline bool nonroot_raised_pE(struct cred *new, const struct cred *old,
kuid_t root, bool has_fcap)
{
bool ret = false;
if ((__cap_grew(effective, ambient, new) &&
!(__cap_full(effective, new) &&
(__is_eff(root, new) || __is_real(root, new)) &&
root_privileged())) ||
(root_privileged() &&
__is_suid(root, new) &&
!__cap_full(effective, new)) ||
(!__is_setuid(new, old) &&
((has_fcap &&
__cap_gained(permitted, new, old)) ||
__cap_gained(ambient, new, old))))
ret = true;
return ret;
}
/**
* cap_bprm_creds_from_file - Set up the proposed credentials for execve().
* @bprm: The execution parameters, including the proposed creds
* @file: The file to pull the credentials from
*
* Set up the proposed credentials for a new execution context being
* constructed by execve(). The proposed creds in @bprm->cred is altered,
* which won't take effect immediately.
*
* Return: 0 if successful, -ve on error.
*/
int cap_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file)
{
/* Process setpcap binaries and capabilities for uid 0 */
const struct cred *old = current_cred();
struct cred *new = bprm->cred;
bool effective = false, has_fcap = false, is_setid;
int ret;
kuid_t root_uid;
if (WARN_ON(!cap_ambient_invariant_ok(old)))
return -EPERM;
ret = get_file_caps(bprm, file, &effective, &has_fcap);
if (ret < 0)
return ret;
root_uid = make_kuid(new->user_ns, 0);
handle_privileged_root(bprm, has_fcap, &effective, root_uid);
/* if we have fs caps, clear dangerous personality flags */
if (__cap_gained(permitted, new, old))
bprm->per_clear |= PER_CLEAR_ON_SETID;
/* Don't let someone trace a set[ug]id/setpcap binary with the revised
* credentials unless they have the appropriate permit.
*
* In addition, if NO_NEW_PRIVS, then ensure we get no new privs.
*/
is_setid = __is_setuid(new, old) || __is_setgid(new, old);
if ((is_setid || __cap_gained(permitted, new, old)) &&
((bprm->unsafe & ~LSM_UNSAFE_PTRACE) ||
!ptracer_capable(current, new->user_ns))) {
/* downgrade; they get no more than they had, and maybe less */
if (!ns_capable(new->user_ns, CAP_SETUID) ||
(bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) {
new->euid = new->uid;
new->egid = new->gid;
}
new->cap_permitted = cap_intersect(new->cap_permitted,
old->cap_permitted);
}
new->suid = new->fsuid = new->euid;
new->sgid = new->fsgid = new->egid;
/* File caps or setid cancels ambient. */
if (has_fcap || is_setid)
cap_clear(new->cap_ambient);
/*
* Now that we've computed pA', update pP' to give:
* pP' = (X & fP) | (pI & fI) | pA'
*/
new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient);
/*
* Set pE' = (fE ? pP' : pA'). Because pA' is zero if fE is set,
* this is the same as pE' = (fE ? pP' : 0) | pA'.
*/
if (effective)
new->cap_effective = new->cap_permitted;
else
new->cap_effective = new->cap_ambient;
if (WARN_ON(!cap_ambient_invariant_ok(new)))
return -EPERM;
if (nonroot_raised_pE(new, old, root_uid, has_fcap)) {
ret = audit_log_bprm_fcaps(bprm, new, old);
if (ret < 0)
return ret;
}
new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);
if (WARN_ON(!cap_ambient_invariant_ok(new)))
return -EPERM;
/* Check for privilege-elevated exec. */
if (is_setid ||
(!__is_real(root_uid, new) &&
(effective ||
__cap_grew(permitted, ambient, new))))
bprm->secureexec = 1;
return 0;
}
/**
* cap_inode_setxattr - Determine whether an xattr may be altered
* @dentry: The inode/dentry being altered
* @name: The name of the xattr to be changed
* @value: The value that the xattr will be changed to
* @size: The size of value
* @flags: The replacement flag
*
* Determine whether an xattr may be altered or set on an inode, returning 0 if
* permission is granted, -ve if denied.
*
* This is used to make sure security xattrs don't get updated or set by those
* who aren't privileged to do so.
*/
int cap_inode_setxattr(struct dentry *dentry, const char *name,
const void *value, size_t size, int flags)
{
struct user_namespace *user_ns = dentry->d_sb->s_user_ns;
/* Ignore non-security xattrs */
if (strncmp(name, XATTR_SECURITY_PREFIX,
XATTR_SECURITY_PREFIX_LEN) != 0)
return 0;
/*
* For XATTR_NAME_CAPS the check will be done in
* cap_convert_nscap(), called by setxattr()
*/
if (strcmp(name, XATTR_NAME_CAPS) == 0)
return 0;
if (!ns_capable(user_ns, CAP_SYS_ADMIN))
return -EPERM;
return 0;
}
/**
* cap_inode_removexattr - Determine whether an xattr may be removed
*
* @idmap: idmap of the mount the inode was found from
* @dentry: The inode/dentry being altered
* @name: The name of the xattr to be changed
*
* Determine whether an xattr may be removed from an inode, returning 0 if
* permission is granted, -ve if denied.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*
* This is used to make sure security xattrs don't get removed by those who
* aren't privileged to remove them.
*/
int cap_inode_removexattr(struct mnt_idmap *idmap,
struct dentry *dentry, const char *name)
{
struct user_namespace *user_ns = dentry->d_sb->s_user_ns;
/* Ignore non-security xattrs */
if (strncmp(name, XATTR_SECURITY_PREFIX,
XATTR_SECURITY_PREFIX_LEN) != 0)
return 0;
if (strcmp(name, XATTR_NAME_CAPS) == 0) {
/* security.capability gets namespaced */
struct inode *inode = d_backing_inode(dentry);
if (!inode)
return -EINVAL;
if (!capable_wrt_inode_uidgid(idmap, inode, CAP_SETFCAP))
return -EPERM;
return 0;
}
if (!ns_capable(user_ns, CAP_SYS_ADMIN))
return -EPERM;
return 0;
}
/*
* cap_emulate_setxuid() fixes the effective / permitted capabilities of
* a process after a call to setuid, setreuid, or setresuid.
*
* 1) When set*uiding _from_ one of {r,e,s}uid == 0 _to_ all of
* {r,e,s}uid != 0, the permitted and effective capabilities are
* cleared.
*
* 2) When set*uiding _from_ euid == 0 _to_ euid != 0, the effective
* capabilities of the process are cleared.
*
* 3) When set*uiding _from_ euid != 0 _to_ euid == 0, the effective
* capabilities are set to the permitted capabilities.
*
* fsuid is handled elsewhere. fsuid == 0 and {r,e,s}uid!= 0 should
* never happen.
*
* -astor
*
* cevans - New behaviour, Oct '99
* A process may, via prctl(), elect to keep its capabilities when it
* calls setuid() and switches away from uid==0. Both permitted and
* effective sets will be retained.
* Without this change, it was impossible for a daemon to drop only some
* of its privilege. The call to setuid(!=0) would drop all privileges!
* Keeping uid 0 is not an option because uid 0 owns too many vital
* files..
* Thanks to Olaf Kirch and Peter Benie for spotting this.
*/
static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old)
{
kuid_t root_uid = make_kuid(old->user_ns, 0);
if ((uid_eq(old->uid, root_uid) ||
uid_eq(old->euid, root_uid) ||
uid_eq(old->suid, root_uid)) &&
(!uid_eq(new->uid, root_uid) &&
!uid_eq(new->euid, root_uid) &&
!uid_eq(new->suid, root_uid))) {
if (!issecure(SECURE_KEEP_CAPS)) {
cap_clear(new->cap_permitted);
cap_clear(new->cap_effective);
}
/*
* Pre-ambient programs expect setresuid to nonroot followed
* by exec to drop capabilities. We should make sure that
* this remains the case.
*/
cap_clear(new->cap_ambient);
}
if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid))
cap_clear(new->cap_effective);
if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid))
new->cap_effective = new->cap_permitted;
}
/**
* cap_task_fix_setuid - Fix up the results of setuid() call
* @new: The proposed credentials
* @old: The current task's current credentials
* @flags: Indications of what has changed
*
* Fix up the results of setuid() call before the credential changes are
* actually applied.
*
* Return: 0 to grant the changes, -ve to deny them.
*/
int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags)
{
switch (flags) {
case LSM_SETID_RE:
case LSM_SETID_ID:
case LSM_SETID_RES:
/* juggle the capabilities to follow [RES]UID changes unless
* otherwise suppressed */
if (!issecure(SECURE_NO_SETUID_FIXUP))
cap_emulate_setxuid(new, old);
break;
case LSM_SETID_FS:
/* juggle the capabilities to follow FSUID changes, unless
* otherwise suppressed
*
* FIXME - is fsuser used for all CAP_FS_MASK capabilities?
* if not, we might be a bit too harsh here.
*/
if (!issecure(SECURE_NO_SETUID_FIXUP)) {
kuid_t root_uid = make_kuid(old->user_ns, 0);
if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid))
new->cap_effective =
cap_drop_fs_set(new->cap_effective);
if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid))
new->cap_effective =
cap_raise_fs_set(new->cap_effective,
new->cap_permitted);
}
break;
default:
return -EINVAL;
}
return 0;
}
/*
* Rationale: code calling task_setscheduler, task_setioprio, and
* task_setnice, assumes that
* . if capable(cap_sys_nice), then those actions should be allowed
* . if not capable(cap_sys_nice), but acting on your own processes,
* then those actions should be allowed
* This is insufficient now since you can call code without suid, but
* yet with increased caps.
* So we check for increased caps on the target process.
*/
static int cap_safe_nice(struct task_struct *p)
{
int is_subset, ret = 0;
rcu_read_lock();
is_subset = cap_issubset(__task_cred(p)->cap_permitted,
current_cred()->cap_permitted);
if (!is_subset && !ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE))
ret = -EPERM;
rcu_read_unlock();
return ret;
}
/**
* cap_task_setscheduler - Determine if scheduler policy change is permitted
* @p: The task to affect
*
* Determine if the requested scheduler policy change is permitted for the
* specified task.
*
* Return: 0 if permission is granted, -ve if denied.
*/
int cap_task_setscheduler(struct task_struct *p)
{
return cap_safe_nice(p);
}
/**
* cap_task_setioprio - Determine if I/O priority change is permitted
* @p: The task to affect
* @ioprio: The I/O priority to set
*
* Determine if the requested I/O priority change is permitted for the specified
* task.
*
* Return: 0 if permission is granted, -ve if denied.
*/
int cap_task_setioprio(struct task_struct *p, int ioprio)
{
return cap_safe_nice(p);
}
/**
* cap_task_setnice - Determine if task priority change is permitted
* @p: The task to affect
* @nice: The nice value to set
*
* Determine if the requested task priority change is permitted for the
* specified task.
*
* Return: 0 if permission is granted, -ve if denied.
*/
int cap_task_setnice(struct task_struct *p, int nice)
{
return cap_safe_nice(p);
}
/*
* Implement PR_CAPBSET_DROP. Attempt to remove the specified capability from
* the current task's bounding set. Returns 0 on success, -ve on error.
*/
static int cap_prctl_drop(unsigned long cap)
{
struct cred *new;
if (!ns_capable(current_user_ns(), CAP_SETPCAP))
return -EPERM;
if (!cap_valid(cap))
return -EINVAL;
new = prepare_creds();
if (!new)
return -ENOMEM;
cap_lower(new->cap_bset, cap);
return commit_creds(new);
}
/**
* cap_task_prctl - Implement process control functions for this security module
* @option: The process control function requested
* @arg2: The argument data for this function
* @arg3: The argument data for this function
* @arg4: The argument data for this function
* @arg5: The argument data for this function
*
* Allow process control functions (sys_prctl()) to alter capabilities; may
* also deny access to other functions not otherwise implemented here.
*
* Return: 0 or +ve on success, -ENOSYS if this function is not implemented
* here, other -ve on error. If -ENOSYS is returned, sys_prctl() and other LSM
* modules will consider performing the function.
*/
int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
unsigned long arg4, unsigned long arg5)
{
const struct cred *old = current_cred();
struct cred *new;
switch (option) {
case PR_CAPBSET_READ:
if (!cap_valid(arg2))
return -EINVAL;
return !!cap_raised(old->cap_bset, arg2);
case PR_CAPBSET_DROP:
return cap_prctl_drop(arg2);
/*
* The next four prctl's remain to assist with transitioning a
* system from legacy UID=0 based privilege (when filesystem
* capabilities are not in use) to a system using filesystem
* capabilities only - as the POSIX.1e draft intended.
*
* Note:
*
* PR_SET_SECUREBITS =
* issecure_mask(SECURE_KEEP_CAPS_LOCKED)
* | issecure_mask(SECURE_NOROOT)
* | issecure_mask(SECURE_NOROOT_LOCKED)
* | issecure_mask(SECURE_NO_SETUID_FIXUP)
* | issecure_mask(SECURE_NO_SETUID_FIXUP_LOCKED)
*
* will ensure that the current process and all of its
* children will be locked into a pure
* capability-based-privilege environment.
*/
case PR_SET_SECUREBITS:
if ((((old->securebits & SECURE_ALL_LOCKS) >> 1)
& (old->securebits ^ arg2)) /*[1]*/
|| ((old->securebits & SECURE_ALL_LOCKS & ~arg2)) /*[2]*/
|| (arg2 & ~(SECURE_ALL_LOCKS | SECURE_ALL_BITS)) /*[3]*/
|| (cap_capable(current_cred(),
current_cred()->user_ns,
CAP_SETPCAP,
CAP_OPT_NONE) != 0) /*[4]*/
/*
* [1] no changing of bits that are locked
* [2] no unlocking of locks
* [3] no setting of unsupported bits
* [4] doing anything requires privilege (go read about
* the "sendmail capabilities bug")
*/
)
/* cannot change a locked bit */
return -EPERM;
new = prepare_creds();
if (!new)
return -ENOMEM;
new->securebits = arg2;
return commit_creds(new);
case PR_GET_SECUREBITS:
return old->securebits;
case PR_GET_KEEPCAPS:
return !!issecure(SECURE_KEEP_CAPS);
case PR_SET_KEEPCAPS:
if (arg2 > 1) /* Note, we rely on arg2 being unsigned here */
return -EINVAL;
if (issecure(SECURE_KEEP_CAPS_LOCKED))
return -EPERM;
new = prepare_creds();
if (!new)
return -ENOMEM;
if (arg2)
new->securebits |= issecure_mask(SECURE_KEEP_CAPS);
else
new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);
return commit_creds(new);
case PR_CAP_AMBIENT:
if (arg2 == PR_CAP_AMBIENT_CLEAR_ALL) {
if (arg3 | arg4 | arg5)
return -EINVAL;
new = prepare_creds();
if (!new)
return -ENOMEM;
cap_clear(new->cap_ambient);
return commit_creds(new);
}
if (((!cap_valid(arg3)) | arg4 | arg5))
return -EINVAL;
if (arg2 == PR_CAP_AMBIENT_IS_SET) {
return !!cap_raised(current_cred()->cap_ambient, arg3);
} else if (arg2 != PR_CAP_AMBIENT_RAISE &&
arg2 != PR_CAP_AMBIENT_LOWER) {
return -EINVAL;
} else {
if (arg2 == PR_CAP_AMBIENT_RAISE &&
(!cap_raised(current_cred()->cap_permitted, arg3) ||
!cap_raised(current_cred()->cap_inheritable,
arg3) ||
issecure(SECURE_NO_CAP_AMBIENT_RAISE)))
return -EPERM;
new = prepare_creds();
if (!new)
return -ENOMEM;
if (arg2 == PR_CAP_AMBIENT_RAISE)
cap_raise(new->cap_ambient, arg3);
else
cap_lower(new->cap_ambient, arg3);
return commit_creds(new);
}
default:
/* No functionality available - continue with default */
return -ENOSYS;
}
}
/**
* cap_vm_enough_memory - Determine whether a new virtual mapping is permitted
* @mm: The VM space in which the new mapping is to be made
* @pages: The size of the mapping
*
* Determine whether the allocation of a new virtual mapping by the current
* task is permitted.
*
* Return: 1 if permission is granted, 0 if not.
*/
int cap_vm_enough_memory(struct mm_struct *mm, long pages)
{
int cap_sys_admin = 0;
if (cap_capable(current_cred(), &init_user_ns,
CAP_SYS_ADMIN, CAP_OPT_NOAUDIT) == 0)
cap_sys_admin = 1;
return cap_sys_admin;
}
/**
* cap_mmap_addr - check if able to map given addr
* @addr: address attempting to be mapped
*
* If the process is attempting to map memory below dac_mmap_min_addr they need
* CAP_SYS_RAWIO. The other parameters to this function are unused by the
* capability security module.
*
* Return: 0 if this mapping should be allowed or -EPERM if not.
*/
int cap_mmap_addr(unsigned long addr)
{
int ret = 0;
if (addr < dac_mmap_min_addr) {
ret = cap_capable(current_cred(), &init_user_ns, CAP_SYS_RAWIO,
CAP_OPT_NONE);
/* set PF_SUPERPRIV if it turns out we allow the low mmap */
if (ret == 0)
current->flags |= PF_SUPERPRIV;
}
return ret;
}
int cap_mmap_file(struct file *file, unsigned long reqprot,
unsigned long prot, unsigned long flags)
{
return 0;
}
#ifdef CONFIG_SECURITY
static const struct lsm_id capability_lsmid = {
.name = "capability",
.id = LSM_ID_CAPABILITY,
};
static struct security_hook_list capability_hooks[] __ro_after_init = {
LSM_HOOK_INIT(capable, cap_capable),
LSM_HOOK_INIT(settime, cap_settime),
LSM_HOOK_INIT(ptrace_access_check, cap_ptrace_access_check),
LSM_HOOK_INIT(ptrace_traceme, cap_ptrace_traceme),
LSM_HOOK_INIT(capget, cap_capget),
LSM_HOOK_INIT(capset, cap_capset),
LSM_HOOK_INIT(bprm_creds_from_file, cap_bprm_creds_from_file),
LSM_HOOK_INIT(inode_need_killpriv, cap_inode_need_killpriv),
LSM_HOOK_INIT(inode_killpriv, cap_inode_killpriv),
LSM_HOOK_INIT(inode_getsecurity, cap_inode_getsecurity),
LSM_HOOK_INIT(mmap_addr, cap_mmap_addr),
LSM_HOOK_INIT(mmap_file, cap_mmap_file),
LSM_HOOK_INIT(task_fix_setuid, cap_task_fix_setuid),
LSM_HOOK_INIT(task_prctl, cap_task_prctl),
LSM_HOOK_INIT(task_setscheduler, cap_task_setscheduler),
LSM_HOOK_INIT(task_setioprio, cap_task_setioprio),
LSM_HOOK_INIT(task_setnice, cap_task_setnice),
LSM_HOOK_INIT(vm_enough_memory, cap_vm_enough_memory),
};
static int __init capability_init(void)
{
security_add_hooks(capability_hooks, ARRAY_SIZE(capability_hooks),
&capability_lsmid);
return 0;
}
DEFINE_LSM(capability) = {
.name = "capability",
.order = LSM_ORDER_FIRST,
.init = capability_init,
};
#endif /* CONFIG_SECURITY */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 1999 Eric Youngdale
* Copyright (C) 2014 Christoph Hellwig
*
* SCSI queueing library.
* Initial versions: Eric Youngdale (eric@andante.org).
* Based upon conversations with large numbers
* of people at Linux Expo.
*/
#include <linux/bio.h>
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/completion.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/hardirq.h>
#include <linux/scatterlist.h>
#include <linux/blk-mq.h>
#include <linux/blk-integrity.h>
#include <linux/ratelimit.h>
#include <asm/unaligned.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport.h> /* __scsi_init_queue() */
#include <scsi/scsi_dh.h>
#include <trace/events/scsi.h>
#include "scsi_debugfs.h"
#include "scsi_priv.h"
#include "scsi_logging.h"
/*
* Size of integrity metadata is usually small, 1 inline sg should
* cover normal cases.
*/
#ifdef CONFIG_ARCH_NO_SG_CHAIN
#define SCSI_INLINE_PROT_SG_CNT 0
#define SCSI_INLINE_SG_CNT 0
#else
#define SCSI_INLINE_PROT_SG_CNT 1
#define SCSI_INLINE_SG_CNT 2
#endif
static struct kmem_cache *scsi_sense_cache;
static DEFINE_MUTEX(scsi_sense_cache_mutex);
static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd);
int scsi_init_sense_cache(struct Scsi_Host *shost)
{
int ret = 0;
mutex_lock(&scsi_sense_cache_mutex);
if (!scsi_sense_cache) {
scsi_sense_cache =
kmem_cache_create_usercopy("scsi_sense_cache",
SCSI_SENSE_BUFFERSIZE, 0, SLAB_HWCACHE_ALIGN,
0, SCSI_SENSE_BUFFERSIZE, NULL);
if (!scsi_sense_cache)
ret = -ENOMEM;
}
mutex_unlock(&scsi_sense_cache_mutex);
return ret;
}
static void
scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
{
struct Scsi_Host *host = cmd->device->host;
struct scsi_device *device = cmd->device;
struct scsi_target *starget = scsi_target(device);
/*
* Set the appropriate busy bit for the device/host.
*
* If the host/device isn't busy, assume that something actually
* completed, and that we should be able to queue a command now.
*
* Note that the prior mid-layer assumption that any host could
* always queue at least one command is now broken. The mid-layer
* will implement a user specifiable stall (see
* scsi_host.max_host_blocked and scsi_device.max_device_blocked)
* if a command is requeued with no other commands outstanding
* either for the device or for the host.
*/
switch (reason) {
case SCSI_MLQUEUE_HOST_BUSY:
atomic_set(&host->host_blocked, host->max_host_blocked);
break;
case SCSI_MLQUEUE_DEVICE_BUSY:
case SCSI_MLQUEUE_EH_RETRY:
atomic_set(&device->device_blocked,
device->max_device_blocked);
break;
case SCSI_MLQUEUE_TARGET_BUSY:
atomic_set(&starget->target_blocked,
starget->max_target_blocked);
break;
}
}
static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd, unsigned long msecs)
{
struct request *rq = scsi_cmd_to_rq(cmd);
if (rq->rq_flags & RQF_DONTPREP) {
rq->rq_flags &= ~RQF_DONTPREP;
scsi_mq_uninit_cmd(cmd);
} else {
WARN_ON_ONCE(true);
}
blk_mq_requeue_request(rq, false);
if (!scsi_host_in_recovery(cmd->device->host))
blk_mq_delay_kick_requeue_list(rq->q, msecs);
}
/**
* __scsi_queue_insert - private queue insertion
* @cmd: The SCSI command being requeued
* @reason: The reason for the requeue
* @unbusy: Whether the queue should be unbusied
*
* This is a private queue insertion. The public interface
* scsi_queue_insert() always assumes the queue should be unbusied
* because it's always called before the completion. This function is
* for a requeue after completion, which should only occur in this
* file.
*/
static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)
{
struct scsi_device *device = cmd->device;
SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
"Inserting command %p into mlqueue\n", cmd));
scsi_set_blocked(cmd, reason);
/*
* Decrement the counters, since these commands are no longer
* active on the host/device.
*/
if (unbusy)
scsi_device_unbusy(device, cmd);
/*
* Requeue this command. It will go before all other commands
* that are already in the queue. Schedule requeue work under
* lock such that the kblockd_schedule_work() call happens
* before blk_mq_destroy_queue() finishes.
*/
cmd->result = 0;
blk_mq_requeue_request(scsi_cmd_to_rq(cmd),
!scsi_host_in_recovery(cmd->device->host));
}
/**
* scsi_queue_insert - Reinsert a command in the queue.
* @cmd: command that we are adding to queue.
* @reason: why we are inserting command to queue.
*
* We do this for one of two cases. Either the host is busy and it cannot accept
* any more commands for the time being, or the device returned QUEUE_FULL and
* can accept no more commands.
*
* Context: This could be called either from an interrupt context or a normal
* process context.
*/
void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
{
__scsi_queue_insert(cmd, reason, true);
}
void scsi_failures_reset_retries(struct scsi_failures *failures)
{
struct scsi_failure *failure;
failures->total_retries = 0;
for (failure = failures->failure_definitions; failure->result;
failure++)
failure->retries = 0;
}
EXPORT_SYMBOL_GPL(scsi_failures_reset_retries);
/**
* scsi_check_passthrough - Determine if passthrough scsi_cmnd needs a retry.
* @scmd: scsi_cmnd to check.
* @failures: scsi_failures struct that lists failures to check for.
*
* Returns -EAGAIN if the caller should retry else 0.
*/
static int scsi_check_passthrough(struct scsi_cmnd *scmd,
struct scsi_failures *failures)
{
struct scsi_failure *failure;
struct scsi_sense_hdr sshdr;
enum sam_status status;
if (!failures)
return 0;
for (failure = failures->failure_definitions; failure->result;
failure++) {
if (failure->result == SCMD_FAILURE_RESULT_ANY)
goto maybe_retry;
if (host_byte(scmd->result) &&
host_byte(scmd->result) == host_byte(failure->result))
goto maybe_retry;
status = status_byte(scmd->result);
if (!status)
continue;
if (failure->result == SCMD_FAILURE_STAT_ANY &&
!scsi_status_is_good(scmd->result))
goto maybe_retry;
if (status != status_byte(failure->result))
continue;
if (status_byte(failure->result) != SAM_STAT_CHECK_CONDITION ||
failure->sense == SCMD_FAILURE_SENSE_ANY)
goto maybe_retry;
if (!scsi_command_normalize_sense(scmd, &sshdr))
return 0;
if (failure->sense != sshdr.sense_key)
continue;
if (failure->asc == SCMD_FAILURE_ASC_ANY)
goto maybe_retry;
if (failure->asc != sshdr.asc)
continue;
if (failure->ascq == SCMD_FAILURE_ASCQ_ANY ||
failure->ascq == sshdr.ascq)
goto maybe_retry;
}
return 0;
maybe_retry:
if (failure->allowed) {
if (failure->allowed == SCMD_FAILURE_NO_LIMIT ||
++failure->retries <= failure->allowed)
return -EAGAIN;
} else {
if (failures->total_allowed == SCMD_FAILURE_NO_LIMIT ||
++failures->total_retries <= failures->total_allowed)
return -EAGAIN;
}
return 0;
}
/**
* scsi_execute_cmd - insert request and wait for the result
* @sdev: scsi_device
* @cmd: scsi command
* @opf: block layer request cmd_flags
* @buffer: data buffer
* @bufflen: len of buffer
* @timeout: request timeout in HZ
* @ml_retries: number of times SCSI midlayer will retry request
* @args: Optional args. See struct definition for field descriptions
*
* Returns the scsi_cmnd result field if a command was executed, or a negative
* Linux error code if we didn't get that far.
*/
int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd,
blk_opf_t opf, void *buffer, unsigned int bufflen,
int timeout, int ml_retries,
const struct scsi_exec_args *args)
{
static const struct scsi_exec_args default_args;
struct request *req;
struct scsi_cmnd *scmd;
int ret;
if (!args)
args = &default_args;
else if (WARN_ON_ONCE(args->sense &&
args->sense_len != SCSI_SENSE_BUFFERSIZE))
return -EINVAL;
retry:
req = scsi_alloc_request(sdev->request_queue, opf, args->req_flags);
if (IS_ERR(req))
return PTR_ERR(req);
if (bufflen) {
ret = blk_rq_map_kern(sdev->request_queue, req,
buffer, bufflen, GFP_NOIO);
if (ret)
goto out;
}
scmd = blk_mq_rq_to_pdu(req);
scmd->cmd_len = COMMAND_SIZE(cmd[0]);
memcpy(scmd->cmnd, cmd, scmd->cmd_len);
scmd->allowed = ml_retries;
scmd->flags |= args->scmd_flags;
req->timeout = timeout;
req->rq_flags |= RQF_QUIET;
/*
* head injection *required* here otherwise quiesce won't work
*/
blk_execute_rq(req, true);
if (scsi_check_passthrough(scmd, args->failures) == -EAGAIN) {
blk_mq_free_request(req);
goto retry;
}
/*
* Some devices (USB mass-storage in particular) may transfer
* garbage data together with a residue indicating that the data
* is invalid. Prevent the garbage from being misinterpreted
* and prevent security leaks by zeroing out the excess data.
*/
if (unlikely(scmd->resid_len > 0 && scmd->resid_len <= bufflen))
memset(buffer + bufflen - scmd->resid_len, 0, scmd->resid_len);
if (args->resid)
*args->resid = scmd->resid_len;
if (args->sense)
memcpy(args->sense, scmd->sense_buffer, SCSI_SENSE_BUFFERSIZE);
if (args->sshdr)
scsi_normalize_sense(scmd->sense_buffer, scmd->sense_len,
args->sshdr);
ret = scmd->result;
out:
blk_mq_free_request(req);
return ret;
}
EXPORT_SYMBOL(scsi_execute_cmd);
/*
* Wake up the error handler if necessary. Avoid as follows that the error
* handler is not woken up if host in-flight requests number ==
* shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
* with an RCU read lock in this function to ensure that this function in
* its entirety either finishes before scsi_eh_scmd_add() increases the
* host_failed counter or that it notices the shost state change made by
* scsi_eh_scmd_add().
*/
static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
{
unsigned long flags;
rcu_read_lock();
__clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);
if (unlikely(scsi_host_in_recovery(shost))) {
unsigned int busy = scsi_host_busy(shost);
spin_lock_irqsave(shost->host_lock, flags);
if (shost->host_failed || shost->host_eh_scheduled)
scsi_eh_wakeup(shost, busy);
spin_unlock_irqrestore(shost->host_lock, flags);
}
rcu_read_unlock();
}
void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)
{
struct Scsi_Host *shost = sdev->host;
struct scsi_target *starget = scsi_target(sdev);
scsi_dec_host_busy(shost, cmd);
if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
sbitmap_put(&sdev->budget_map, cmd->budget_token);
cmd->budget_token = -1;
}
/*
* Kick the queue of SCSI device @sdev if @sdev != current_sdev. Called with
* interrupts disabled.
*/
static void scsi_kick_sdev_queue(struct scsi_device *sdev, void *data)
{
struct scsi_device *current_sdev = data;
if (sdev != current_sdev)
blk_mq_run_hw_queues(sdev->request_queue, true);
}
/*
* Called for single_lun devices on IO completion. Clear starget_sdev_user,
* and call blk_run_queue for all the scsi_devices on the target -
* including current_sdev first.
*
* Called with *no* scsi locks held.
*/
static void scsi_single_lun_run(struct scsi_device *current_sdev)
{
struct Scsi_Host *shost = current_sdev->host;
struct scsi_target *starget = scsi_target(current_sdev);
unsigned long flags;
spin_lock_irqsave(shost->host_lock, flags);
starget->starget_sdev_user = NULL;
spin_unlock_irqrestore(shost->host_lock, flags);
/*
* Call blk_run_queue for all LUNs on the target, starting with
* current_sdev. We race with others (to set starget_sdev_user),
* but in most cases, we will be first. Ideally, each LU on the
* target would get some limited time or requests on the target.
*/
blk_mq_run_hw_queues(current_sdev->request_queue,
shost->queuecommand_may_block);
spin_lock_irqsave(shost->host_lock, flags);
if (!starget->starget_sdev_user)
__starget_for_each_device(starget, current_sdev,
scsi_kick_sdev_queue);
spin_unlock_irqrestore(shost->host_lock, flags);
}
static inline bool scsi_device_is_busy(struct scsi_device *sdev)
{
if (scsi_device_busy(sdev) >= sdev->queue_depth)
return true;
if (atomic_read(&sdev->device_blocked) > 0)
return true;
return false;
}
static inline bool scsi_target_is_busy(struct scsi_target *starget)
{
if (starget->can_queue > 0) {
if (atomic_read(&starget->target_busy) >= starget->can_queue)
return true;
if (atomic_read(&starget->target_blocked) > 0)
return true;
}
return false;
}
static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
{
if (atomic_read(&shost->host_blocked) > 0)
return true;
if (shost->host_self_blocked)
return true;
return false;
}
static void scsi_starved_list_run(struct Scsi_Host *shost)
{
LIST_HEAD(starved_list);
struct scsi_device *sdev;
unsigned long flags;
spin_lock_irqsave(shost->host_lock, flags);
list_splice_init(&shost->starved_list, &starved_list);
while (!list_empty(&starved_list)) {
struct request_queue *slq;
/*
* As long as shost is accepting commands and we have
* starved queues, call blk_run_queue. scsi_request_fn
* drops the queue_lock and can add us back to the
* starved_list.
*
* host_lock protects the starved_list and starved_entry.
* scsi_request_fn must get the host_lock before checking
* or modifying starved_list or starved_entry.
*/
if (scsi_host_is_busy(shost))
break;
sdev = list_entry(starved_list.next,
struct scsi_device, starved_entry);
list_del_init(&sdev->starved_entry);
if (scsi_target_is_busy(scsi_target(sdev))) {
list_move_tail(&sdev->starved_entry,
&shost->starved_list);
continue;
}
/*
* Once we drop the host lock, a racing scsi_remove_device()
* call may remove the sdev from the starved list and destroy
* it and the queue. Mitigate by taking a reference to the
* queue and never touching the sdev again after we drop the
* host lock. Note: if __scsi_remove_device() invokes
* blk_mq_destroy_queue() before the queue is run from this
* function then blk_run_queue() will return immediately since
* blk_mq_destroy_queue() marks the queue with QUEUE_FLAG_DYING.
*/
slq = sdev->request_queue;
if (!blk_get_queue(slq))
continue;
spin_unlock_irqrestore(shost->host_lock, flags);
blk_mq_run_hw_queues(slq, false);
blk_put_queue(slq);
spin_lock_irqsave(shost->host_lock, flags);
}
/* put any unprocessed entries back */
list_splice(&starved_list, &shost->starved_list);
spin_unlock_irqrestore(shost->host_lock, flags);
}
/**
* scsi_run_queue - Select a proper request queue to serve next.
* @q: last request's queue
*
* The previous command was completely finished, start a new one if possible.
*/
static void scsi_run_queue(struct request_queue *q)
{
struct scsi_device *sdev = q->queuedata;
if (scsi_target(sdev)->single_lun)
scsi_single_lun_run(sdev);
if (!list_empty(&sdev->host->starved_list))
scsi_starved_list_run(sdev->host);
/* Note: blk_mq_kick_requeue_list() runs the queue asynchronously. */
blk_mq_kick_requeue_list(q);
}
void scsi_requeue_run_queue(struct work_struct *work)
{
struct scsi_device *sdev;
struct request_queue *q;
sdev = container_of(work, struct scsi_device, requeue_work);
q = sdev->request_queue;
scsi_run_queue(q);
}
void scsi_run_host_queues(struct Scsi_Host *shost)
{
struct scsi_device *sdev;
shost_for_each_device(sdev, shost)
scsi_run_queue(sdev->request_queue);
}
static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
{
if (!blk_rq_is_passthrough(scsi_cmd_to_rq(cmd))) {
struct scsi_driver *drv = scsi_cmd_to_driver(cmd);
if (drv->uninit_command)
drv->uninit_command(cmd);
}
}
void scsi_free_sgtables(struct scsi_cmnd *cmd)
{
if (cmd->sdb.table.nents)
sg_free_table_chained(&cmd->sdb.table,
SCSI_INLINE_SG_CNT);
if (scsi_prot_sg_count(cmd))
sg_free_table_chained(&cmd->prot_sdb->table,
SCSI_INLINE_PROT_SG_CNT);
}
EXPORT_SYMBOL_GPL(scsi_free_sgtables);
static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
{
scsi_free_sgtables(cmd);
scsi_uninit_cmd(cmd);
}
static void scsi_run_queue_async(struct scsi_device *sdev)
{
if (scsi_host_in_recovery(sdev->host))
return;
if (scsi_target(sdev)->single_lun ||
!list_empty(&sdev->host->starved_list)) {
kblockd_schedule_work(&sdev->requeue_work);
} else {
/*
* smp_mb() present in sbitmap_queue_clear() or implied in
* .end_io is for ordering writing .device_busy in
* scsi_device_unbusy() and reading sdev->restarts.
*/
int old = atomic_read(&sdev->restarts);
/*
* ->restarts has to be kept as non-zero if new budget
* contention occurs.
*
* No need to run queue when either another re-run
* queue wins in updating ->restarts or a new budget
* contention occurs.
*/
if (old && atomic_cmpxchg(&sdev->restarts, old, 0) == old)
blk_mq_run_hw_queues(sdev->request_queue, true);
}
}
/* Returns false when no more bytes to process, true if there are more */
static bool scsi_end_request(struct request *req, blk_status_t error,
unsigned int bytes)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
struct scsi_device *sdev = cmd->device;
struct request_queue *q = sdev->request_queue;
if (blk_update_request(req, error, bytes))
return true;
// XXX:
if (blk_queue_add_random(q))
add_disk_randomness(req->q->disk);
WARN_ON_ONCE(!blk_rq_is_passthrough(req) &&
!(cmd->flags & SCMD_INITIALIZED));
cmd->flags = 0;
/*
* Calling rcu_barrier() is not necessary here because the
* SCSI error handler guarantees that the function called by
* call_rcu() has been called before scsi_end_request() is
* called.
*/
destroy_rcu_head(&cmd->rcu);
/*
* In the MQ case the command gets freed by __blk_mq_end_request,
* so we have to do all cleanup that depends on it earlier.
*
* We also can't kick the queues from irq context, so we
* will have to defer it to a workqueue.
*/
scsi_mq_uninit_cmd(cmd);
/*
* queue is still alive, so grab the ref for preventing it
* from being cleaned up during running queue.
*/
percpu_ref_get(&q->q_usage_counter);
__blk_mq_end_request(req, error);
scsi_run_queue_async(sdev);
percpu_ref_put(&q->q_usage_counter);
return false;
}
/**
* scsi_result_to_blk_status - translate a SCSI result code into blk_status_t
* @result: scsi error code
*
* Translate a SCSI result code into a blk_status_t value.
*/
static blk_status_t scsi_result_to_blk_status(int result)
{
/*
* Check the scsi-ml byte first in case we converted a host or status
* byte.
*/
switch (scsi_ml_byte(result)) {
case SCSIML_STAT_OK:
break;
case SCSIML_STAT_RESV_CONFLICT:
return BLK_STS_RESV_CONFLICT;
case SCSIML_STAT_NOSPC:
return BLK_STS_NOSPC;
case SCSIML_STAT_MED_ERROR:
return BLK_STS_MEDIUM;
case SCSIML_STAT_TGT_FAILURE:
return BLK_STS_TARGET;
case SCSIML_STAT_DL_TIMEOUT:
return BLK_STS_DURATION_LIMIT;
}
switch (host_byte(result)) {
case DID_OK:
if (scsi_status_is_good(result))
return BLK_STS_OK;
return BLK_STS_IOERR;
case DID_TRANSPORT_FAILFAST:
case DID_TRANSPORT_MARGINAL:
return BLK_STS_TRANSPORT;
default:
return BLK_STS_IOERR;
}
}
/**
* scsi_rq_err_bytes - determine number of bytes till the next failure boundary
* @rq: request to examine
*
* Description:
* A request could be merge of IOs which require different failure
* handling. This function determines the number of bytes which
* can be failed from the beginning of the request without
* crossing into area which need to be retried further.
*
* Return:
* The number of bytes to fail.
*/
static unsigned int scsi_rq_err_bytes(const struct request *rq)
{
blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
unsigned int bytes = 0;
struct bio *bio;
if (!(rq->rq_flags & RQF_MIXED_MERGE))
return blk_rq_bytes(rq);
/*
* Currently the only 'mixing' which can happen is between
* different fastfail types. We can safely fail portions
* which have all the failfast bits that the first one has -
* the ones which are at least as eager to fail as the first
* one.
*/
for (bio = rq->bio; bio; bio = bio->bi_next) {
if ((bio->bi_opf & ff) != ff)
break;
bytes += bio->bi_iter.bi_size;
}
/* this could lead to infinite loop */
BUG_ON(blk_rq_bytes(rq) && !bytes);
return bytes;
}
static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd)
{
struct request *req = scsi_cmd_to_rq(cmd);
unsigned long wait_for;
if (cmd->allowed == SCSI_CMD_RETRIES_NO_LIMIT)
return false;
wait_for = (cmd->allowed + 1) * req->timeout;
if (time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
scmd_printk(KERN_ERR, cmd, "timing out command, waited %lus\n",
wait_for/HZ);
return true;
}
return false;
}
/*
* When ALUA transition state is returned, reprep the cmd to
* use the ALUA handler's transition timeout. Delay the reprep
* 1 sec to avoid aggressive retries of the target in that
* state.
*/
#define ALUA_TRANSITION_REPREP_DELAY 1000
/* Helper for scsi_io_completion() when special action required. */
static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)
{
struct request *req = scsi_cmd_to_rq(cmd);
int level = 0;
enum {ACTION_FAIL, ACTION_REPREP, ACTION_DELAYED_REPREP,
ACTION_RETRY, ACTION_DELAYED_RETRY} action;
struct scsi_sense_hdr sshdr;
bool sense_valid;
bool sense_current = true; /* false implies "deferred sense" */
blk_status_t blk_stat;
sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
if (sense_valid)
sense_current = !scsi_sense_is_deferred(&sshdr);
blk_stat = scsi_result_to_blk_status(result);
if (host_byte(result) == DID_RESET) {
/* Third party bus reset or reset for error recovery
* reasons. Just retry the command and see what
* happens.
*/
action = ACTION_RETRY;
} else if (sense_valid && sense_current) {
switch (sshdr.sense_key) {
case UNIT_ATTENTION:
if (cmd->device->removable) {
/* Detected disc change. Set a bit
* and quietly refuse further access.
*/
cmd->device->changed = 1;
action = ACTION_FAIL;
} else {
/* Must have been a power glitch, or a
* bus reset. Could not have been a
* media change, so we just retry the
* command and see what happens.
*/
action = ACTION_RETRY;
}
break;
case ILLEGAL_REQUEST:
/* If we had an ILLEGAL REQUEST returned, then
* we may have performed an unsupported
* command. The only thing this should be
* would be a ten byte read where only a six
* byte read was supported. Also, on a system
* where READ CAPACITY failed, we may have
* read past the end of the disk.
*/
if ((cmd->device->use_10_for_rw &&
sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
(cmd->cmnd[0] == READ_10 ||
cmd->cmnd[0] == WRITE_10)) {
/* This will issue a new 6-byte command. */
cmd->device->use_10_for_rw = 0;
action = ACTION_REPREP;
} else if (sshdr.asc == 0x10) /* DIX */ {
action = ACTION_FAIL;
blk_stat = BLK_STS_PROTECTION;
/* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
} else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
action = ACTION_FAIL;
blk_stat = BLK_STS_TARGET;
} else
action = ACTION_FAIL;
break;
case ABORTED_COMMAND:
action = ACTION_FAIL;
if (sshdr.asc == 0x10) /* DIF */
blk_stat = BLK_STS_PROTECTION;
break;
case NOT_READY:
/* If the device is in the process of becoming
* ready, or has a temporary blockage, retry.
*/
if (sshdr.asc == 0x04) {
switch (sshdr.ascq) {
case 0x01: /* becoming ready */
case 0x04: /* format in progress */
case 0x05: /* rebuild in progress */
case 0x06: /* recalculation in progress */
case 0x07: /* operation in progress */
case 0x08: /* Long write in progress */
case 0x09: /* self test in progress */
case 0x11: /* notify (enable spinup) required */
case 0x14: /* space allocation in progress */
case 0x1a: /* start stop unit in progress */
case 0x1b: /* sanitize in progress */
case 0x1d: /* configuration in progress */
case 0x24: /* depopulation in progress */
case 0x25: /* depopulation restore in progress */
action = ACTION_DELAYED_RETRY;
break;
case 0x0a: /* ALUA state transition */
action = ACTION_DELAYED_REPREP;
break;
default:
action = ACTION_FAIL;
break;
}
} else
action = ACTION_FAIL;
break;
case VOLUME_OVERFLOW:
/* See SSC3rXX or current. */
action = ACTION_FAIL;
break;
case DATA_PROTECT:
action = ACTION_FAIL;
if ((sshdr.asc == 0x0C && sshdr.ascq == 0x12) ||
(sshdr.asc == 0x55 &&
(sshdr.ascq == 0x0E || sshdr.ascq == 0x0F))) {
/* Insufficient zone resources */
blk_stat = BLK_STS_ZONE_OPEN_RESOURCE;
}
break;
case COMPLETED:
fallthrough;
default:
action = ACTION_FAIL;
break;
}
} else
action = ACTION_FAIL;
if (action != ACTION_FAIL && scsi_cmd_runtime_exceeced(cmd))
action = ACTION_FAIL;
switch (action) {
case ACTION_FAIL:
/* Give up and fail the remainder of the request */
if (!(req->rq_flags & RQF_QUIET)) {
static DEFINE_RATELIMIT_STATE(_rs,
DEFAULT_RATELIMIT_INTERVAL,
DEFAULT_RATELIMIT_BURST);
if (unlikely(scsi_logging_level))
level =
SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,
SCSI_LOG_MLCOMPLETE_BITS);
/*
* if logging is enabled the failure will be printed
* in scsi_log_completion(), so avoid duplicate messages
*/
if (!level && __ratelimit(&_rs)) {
scsi_print_result(cmd, NULL, FAILED);
if (sense_valid)
scsi_print_sense(cmd);
scsi_print_command(cmd);
}
}
if (!scsi_end_request(req, blk_stat, scsi_rq_err_bytes(req)))
return;
fallthrough;
case ACTION_REPREP:
scsi_mq_requeue_cmd(cmd, 0);
break;
case ACTION_DELAYED_REPREP:
scsi_mq_requeue_cmd(cmd, ALUA_TRANSITION_REPREP_DELAY);
break;
case ACTION_RETRY:
/* Retry the same command immediately */
__scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, false);
break;
case ACTION_DELAYED_RETRY:
/* Retry the same command after a delay */
__scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, false);
break;
}
}
/*
* Helper for scsi_io_completion() when cmd->result is non-zero. Returns a
* new result that may suppress further error checking. Also modifies
* *blk_statp in some cases.
*/
static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,
blk_status_t *blk_statp)
{
bool sense_valid;
bool sense_current = true; /* false implies "deferred sense" */
struct request *req = scsi_cmd_to_rq(cmd);
struct scsi_sense_hdr sshdr;
sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
if (sense_valid)
sense_current = !scsi_sense_is_deferred(&sshdr);
if (blk_rq_is_passthrough(req)) {
if (sense_valid) {
/*
* SG_IO wants current and deferred errors
*/
cmd->sense_len = min(8 + cmd->sense_buffer[7],
SCSI_SENSE_BUFFERSIZE);
}
if (sense_current)
*blk_statp = scsi_result_to_blk_status(result);
} else if (blk_rq_bytes(req) == 0 && sense_current) {
/*
* Flush commands do not transfers any data, and thus cannot use
* good_bytes != blk_rq_bytes(req) as the signal for an error.
* This sets *blk_statp explicitly for the problem case.
*/
*blk_statp = scsi_result_to_blk_status(result);
}
/*
* Recovered errors need reporting, but they're always treated as
* success, so fiddle the result code here. For passthrough requests
* we already took a copy of the original into sreq->result which
* is what gets returned to the user
*/
if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
bool do_print = true;
/*
* if ATA PASS-THROUGH INFORMATION AVAILABLE [0x0, 0x1d]
* skip print since caller wants ATA registers. Only occurs
* on SCSI ATA PASS_THROUGH commands when CK_COND=1
*/
if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
do_print = false;
else if (req->rq_flags & RQF_QUIET)
do_print = false;
if (do_print)
scsi_print_sense(cmd);
result = 0;
/* for passthrough, *blk_statp may be set */
*blk_statp = BLK_STS_OK;
}
/*
* Another corner case: the SCSI status byte is non-zero but 'good'.
* Example: PRE-FETCH command returns SAM_STAT_CONDITION_MET when
* it is able to fit nominated LBs in its cache (and SAM_STAT_GOOD
* if it can't fit). Treat SAM_STAT_CONDITION_MET and the related
* intermediate statuses (both obsolete in SAM-4) as good.
*/
if ((result & 0xff) && scsi_status_is_good(result)) {
result = 0;
*blk_statp = BLK_STS_OK;
}
return result;
}
/**
* scsi_io_completion - Completion processing for SCSI commands.
* @cmd: command that is finished.
* @good_bytes: number of processed bytes.
*
* We will finish off the specified number of sectors. If we are done, the
* command block will be released and the queue function will be goosed. If we
* are not done then we have to figure out what to do next:
*
* a) We can call scsi_mq_requeue_cmd(). The request will be
* unprepared and put back on the queue. Then a new command will
* be created for it. This should be used if we made forward
* progress, or if we want to switch from READ(10) to READ(6) for
* example.
*
* b) We can call scsi_io_completion_action(). The request will be
* put back on the queue and retried using the same command as
* before, possibly after a delay.
*
* c) We can call scsi_end_request() with blk_stat other than
* BLK_STS_OK, to fail the remainder of the request.
*/
void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
{
int result = cmd->result;
struct request *req = scsi_cmd_to_rq(cmd);
blk_status_t blk_stat = BLK_STS_OK;
if (unlikely(result)) /* a nz result may or may not be an error */
result = scsi_io_completion_nz_result(cmd, result, &blk_stat);
/*
* Next deal with any sectors which we were able to correctly
* handle.
*/
SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,
"%u sectors total, %d bytes done.\n",
blk_rq_sectors(req), good_bytes));
/*
* Failed, zero length commands always need to drop down
* to retry code. Fast path should return in this block.
*/
if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {
if (likely(!scsi_end_request(req, blk_stat, good_bytes)))
return; /* no bytes remaining */
}
/* Kill remainder if no retries. */
if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) {
if (scsi_end_request(req, blk_stat, blk_rq_bytes(req)))
WARN_ONCE(true,
"Bytes remaining after failed, no-retry command");
return;
}
/*
* If there had been no error, but we have leftover bytes in the
* request just queue the command up again.
*/
if (likely(result == 0))
scsi_mq_requeue_cmd(cmd, 0);
else
scsi_io_completion_action(cmd, result);
}
static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev,
struct request *rq)
{
return sdev->dma_drain_len && blk_rq_is_passthrough(rq) &&
!op_is_write(req_op(rq)) &&
sdev->host->hostt->dma_need_drain(rq);
}
/**
* scsi_alloc_sgtables - Allocate and initialize data and integrity scatterlists
* @cmd: SCSI command data structure to initialize.
*
* Initializes @cmd->sdb and also @cmd->prot_sdb if data integrity is enabled
* for @cmd.
*
* Returns:
* * BLK_STS_OK - on success
* * BLK_STS_RESOURCE - if the failure is retryable
* * BLK_STS_IOERR - if the failure is fatal
*/
blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd)
{
struct scsi_device *sdev = cmd->device;
struct request *rq = scsi_cmd_to_rq(cmd);
unsigned short nr_segs = blk_rq_nr_phys_segments(rq);
struct scatterlist *last_sg = NULL;
blk_status_t ret;
bool need_drain = scsi_cmd_needs_dma_drain(sdev, rq);
int count;
if (WARN_ON_ONCE(!nr_segs))
return BLK_STS_IOERR;
/*
* Make sure there is space for the drain. The driver must adjust
* max_hw_segments to be prepared for this.
*/
if (need_drain)
nr_segs++;
/*
* If sg table allocation fails, requeue request later.
*/
if (unlikely(sg_alloc_table_chained(&cmd->sdb.table, nr_segs,
cmd->sdb.table.sgl, SCSI_INLINE_SG_CNT)))
return BLK_STS_RESOURCE;
/*
* Next, walk the list, and fill in the addresses and sizes of
* each segment.
*/
count = __blk_rq_map_sg(rq->q, rq, cmd->sdb.table.sgl, &last_sg);
if (blk_rq_bytes(rq) & rq->q->dma_pad_mask) {
unsigned int pad_len =
(rq->q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
last_sg->length += pad_len;
cmd->extra_len += pad_len;
}
if (need_drain) {
sg_unmark_end(last_sg);
last_sg = sg_next(last_sg);
sg_set_buf(last_sg, sdev->dma_drain_buf, sdev->dma_drain_len);
sg_mark_end(last_sg);
cmd->extra_len += sdev->dma_drain_len;
count++;
}
BUG_ON(count > cmd->sdb.table.nents);
cmd->sdb.table.nents = count;
cmd->sdb.length = blk_rq_payload_bytes(rq);
if (blk_integrity_rq(rq)) {
struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
int ivecs;
if (WARN_ON_ONCE(!prot_sdb)) {
/*
* This can happen if someone (e.g. multipath)
* queues a command to a device on an adapter
* that does not support DIX.
*/
ret = BLK_STS_IOERR;
goto out_free_sgtables;
}
ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
if (sg_alloc_table_chained(&prot_sdb->table, ivecs,
prot_sdb->table.sgl,
SCSI_INLINE_PROT_SG_CNT)) {
ret = BLK_STS_RESOURCE;
goto out_free_sgtables;
}
count = blk_rq_map_integrity_sg(rq->q, rq->bio,
prot_sdb->table.sgl);
BUG_ON(count > ivecs);
BUG_ON(count > queue_max_integrity_segments(rq->q));
cmd->prot_sdb = prot_sdb;
cmd->prot_sdb->table.nents = count;
}
return BLK_STS_OK;
out_free_sgtables:
scsi_free_sgtables(cmd);
return ret;
}
EXPORT_SYMBOL(scsi_alloc_sgtables);
/**
* scsi_initialize_rq - initialize struct scsi_cmnd partially
* @rq: Request associated with the SCSI command to be initialized.
*
* This function initializes the members of struct scsi_cmnd that must be
* initialized before request processing starts and that won't be
* reinitialized if a SCSI command is requeued.
*/
static void scsi_initialize_rq(struct request *rq)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
memset(cmd->cmnd, 0, sizeof(cmd->cmnd));
cmd->cmd_len = MAX_COMMAND_SIZE;
cmd->sense_len = 0;
init_rcu_head(&cmd->rcu);
cmd->jiffies_at_alloc = jiffies;
cmd->retries = 0;
}
struct request *scsi_alloc_request(struct request_queue *q, blk_opf_t opf,
blk_mq_req_flags_t flags)
{
struct request *rq;
rq = blk_mq_alloc_request(q, opf, flags);
if (!IS_ERR(rq))
scsi_initialize_rq(rq);
return rq;
}
EXPORT_SYMBOL_GPL(scsi_alloc_request);
/*
* Only called when the request isn't completed by SCSI, and not freed by
* SCSI
*/
static void scsi_cleanup_rq(struct request *rq)
{
if (rq->rq_flags & RQF_DONTPREP) {
scsi_mq_uninit_cmd(blk_mq_rq_to_pdu(rq));
rq->rq_flags &= ~RQF_DONTPREP;
}
}
/* Called before a request is prepared. See also scsi_mq_prep_fn(). */
void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)
{
struct request *rq = scsi_cmd_to_rq(cmd);
if (!blk_rq_is_passthrough(rq) && !(cmd->flags & SCMD_INITIALIZED)) {
cmd->flags |= SCMD_INITIALIZED;
scsi_initialize_rq(rq);
}
cmd->device = dev;
INIT_LIST_HEAD(&cmd->eh_entry);
INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
}
static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev,
struct request *req)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
/*
* Passthrough requests may transfer data, in which case they must
* a bio attached to them. Or they might contain a SCSI command
* that does not transfer data, in which case they may optionally
* submit a request without an attached bio.
*/
if (req->bio) {
blk_status_t ret = scsi_alloc_sgtables(cmd);
if (unlikely(ret != BLK_STS_OK))
return ret;
} else {
BUG_ON(blk_rq_bytes(req));
memset(&cmd->sdb, 0, sizeof(cmd->sdb));
}
cmd->transfersize = blk_rq_bytes(req);
return BLK_STS_OK;
}
static blk_status_t
scsi_device_state_check(struct scsi_device *sdev, struct request *req)
{
switch (sdev->sdev_state) {
case SDEV_CREATED:
return BLK_STS_OK;
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
/*
* If the device is offline we refuse to process any
* commands. The device must be brought online
* before trying any recovery commands.
*/
if (!sdev->offline_already) {
sdev->offline_already = true;
sdev_printk(KERN_ERR, sdev,
"rejecting I/O to offline device\n");
}
return BLK_STS_IOERR;
case SDEV_DEL:
/*
* If the device is fully deleted, we refuse to
* process any commands as well.
*/
sdev_printk(KERN_ERR, sdev,
"rejecting I/O to dead device\n");
return BLK_STS_IOERR;
case SDEV_BLOCK:
case SDEV_CREATED_BLOCK:
return BLK_STS_RESOURCE;
case SDEV_QUIESCE:
/*
* If the device is blocked we only accept power management
* commands.
*/
if (req && WARN_ON_ONCE(!(req->rq_flags & RQF_PM)))
return BLK_STS_RESOURCE;
return BLK_STS_OK;
default:
/*
* For any other not fully online state we only allow
* power management commands.
*/
if (req && !(req->rq_flags & RQF_PM))
return BLK_STS_OFFLINE;
return BLK_STS_OK;
}
}
/*
* scsi_dev_queue_ready: if we can send requests to sdev, assign one token
* and return the token else return -1.
*/
static inline int scsi_dev_queue_ready(struct request_queue *q,
struct scsi_device *sdev)
{
int token;
token = sbitmap_get(&sdev->budget_map);
if (token < 0)
return -1;
if (!atomic_read(&sdev->device_blocked))
return token;
/*
* Only unblock if no other commands are pending and
* if device_blocked has decreased to zero
*/
if (scsi_device_busy(sdev) > 1 ||
atomic_dec_return(&sdev->device_blocked) > 0) {
sbitmap_put(&sdev->budget_map, token);
return -1;
}
SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,
"unblocking device at zero depth\n"));
return token;
}
/*
* scsi_target_queue_ready: checks if there we can send commands to target
* @sdev: scsi device on starget to check.
*/
static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
struct scsi_device *sdev)
{
struct scsi_target *starget = scsi_target(sdev);
unsigned int busy;
if (starget->single_lun) {
spin_lock_irq(shost->host_lock);
if (starget->starget_sdev_user &&
starget->starget_sdev_user != sdev) {
spin_unlock_irq(shost->host_lock);
return 0;
}
starget->starget_sdev_user = sdev;
spin_unlock_irq(shost->host_lock);
}
if (starget->can_queue <= 0)
return 1;
busy = atomic_inc_return(&starget->target_busy) - 1;
if (atomic_read(&starget->target_blocked) > 0) {
if (busy)
goto starved;
/*
* unblock after target_blocked iterates to zero
*/
if (atomic_dec_return(&starget->target_blocked) > 0)
goto out_dec;
SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
"unblocking target at zero depth\n"));
}
if (busy >= starget->can_queue)
goto starved;
return 1;
starved:
spin_lock_irq(shost->host_lock);
list_move_tail(&sdev->starved_entry, &shost->starved_list);
spin_unlock_irq(shost->host_lock);
out_dec:
if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
return 0;
}
/*
* scsi_host_queue_ready: if we can send requests to shost, return 1 else
* return 0. We must end up running the queue again whenever 0 is
* returned, else IO can hang.
*/
static inline int scsi_host_queue_ready(struct request_queue *q,
struct Scsi_Host *shost,
struct scsi_device *sdev,
struct scsi_cmnd *cmd)
{
if (atomic_read(&shost->host_blocked) > 0) {
if (scsi_host_busy(shost) > 0)
goto starved;
/*
* unblock after host_blocked iterates to zero
*/
if (atomic_dec_return(&shost->host_blocked) > 0)
goto out_dec;
SCSI_LOG_MLQUEUE(3,
shost_printk(KERN_INFO, shost,
"unblocking host at zero depth\n"));
}
if (shost->host_self_blocked)
goto starved;
/* We're OK to process the command, so we can't be starved */
if (!list_empty(&sdev->starved_entry)) {
spin_lock_irq(shost->host_lock);
if (!list_empty(&sdev->starved_entry))
list_del_init(&sdev->starved_entry);
spin_unlock_irq(shost->host_lock);
}
__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
return 1;
starved:
spin_lock_irq(shost->host_lock);
if (list_empty(&sdev->starved_entry))
list_add_tail(&sdev->starved_entry, &shost->starved_list);
spin_unlock_irq(shost->host_lock);
out_dec:
scsi_dec_host_busy(shost, cmd);
return 0;
}
/*
* Busy state exporting function for request stacking drivers.
*
* For efficiency, no lock is taken to check the busy state of
* shost/starget/sdev, since the returned value is not guaranteed and
* may be changed after request stacking drivers call the function,
* regardless of taking lock or not.
*
* When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi
* needs to return 'not busy'. Otherwise, request stacking drivers
* may hold requests forever.
*/
static bool scsi_mq_lld_busy(struct request_queue *q)
{
struct scsi_device *sdev = q->queuedata;
struct Scsi_Host *shost;
if (blk_queue_dying(q))
return false;
shost = sdev->host;
/*
* Ignore host/starget busy state.
* Since block layer does not have a concept of fairness across
* multiple queues, congestion of host/starget needs to be handled
* in SCSI layer.
*/
if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))
return true;
return false;
}
/*
* Block layer request completion callback. May be called from interrupt
* context.
*/
static void scsi_complete(struct request *rq)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
enum scsi_disposition disposition;
INIT_LIST_HEAD(&cmd->eh_entry);
atomic_inc(&cmd->device->iodone_cnt);
if (cmd->result)
atomic_inc(&cmd->device->ioerr_cnt);
disposition = scsi_decide_disposition(cmd);
if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd))
disposition = SUCCESS;
scsi_log_completion(cmd, disposition);
switch (disposition) {
case SUCCESS:
scsi_finish_command(cmd);
break;
case NEEDS_RETRY:
scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
break;
case ADD_TO_MLQUEUE:
scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
break;
default:
scsi_eh_scmd_add(cmd);
break;
}
}
/**
* scsi_dispatch_cmd - Dispatch a command to the low-level driver.
* @cmd: command block we are dispatching.
*
* Return: nonzero return request was rejected and device's queue needs to be
* plugged.
*/
static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
{
struct Scsi_Host *host = cmd->device->host;
int rtn = 0;
atomic_inc(&cmd->device->iorequest_cnt);
/* check if the device is still usable */
if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
/* in SDEV_DEL we error all commands. DID_NO_CONNECT
* returns an immediate error upwards, and signals
* that the device is no longer present */
cmd->result = DID_NO_CONNECT << 16;
goto done;
}
/* Check to see if the scsi lld made this device blocked. */
if (unlikely(scsi_device_blocked(cmd->device))) {
/*
* in blocked state, the command is just put back on
* the device queue. The suspend state has already
* blocked the queue so future requests should not
* occur until the device transitions out of the
* suspend state.
*/
SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
"queuecommand : device blocked\n"));
atomic_dec(&cmd->device->iorequest_cnt);
return SCSI_MLQUEUE_DEVICE_BUSY;
}
/* Store the LUN value in cmnd, if needed. */
if (cmd->device->lun_in_cdb)
cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) |
(cmd->device->lun << 5 & 0xe0);
scsi_log_send(cmd);
/*
* Before we queue this command, check if the command
* length exceeds what the host adapter can handle.
*/
if (cmd->cmd_len > cmd->device->host->max_cmd_len) {
SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
"queuecommand : command too long. "
"cdb_size=%d host->max_cmd_len=%d\n",
cmd->cmd_len, cmd->device->host->max_cmd_len));
cmd->result = (DID_ABORT << 16);
goto done;
}
if (unlikely(host->shost_state == SHOST_DEL)) {
cmd->result = (DID_NO_CONNECT << 16);
goto done;
}
trace_scsi_dispatch_cmd_start(cmd);
rtn = host->hostt->queuecommand(host, cmd);
if (rtn) {
atomic_dec(&cmd->device->iorequest_cnt);
trace_scsi_dispatch_cmd_error(cmd, rtn);
if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
rtn != SCSI_MLQUEUE_TARGET_BUSY)
rtn = SCSI_MLQUEUE_HOST_BUSY;
SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
"queuecommand : request rejected\n"));
}
return rtn;
done:
scsi_done(cmd);
return 0;
}
/* Size in bytes of the sg-list stored in the scsi-mq command-private data. */
static unsigned int scsi_mq_inline_sgl_size(struct Scsi_Host *shost)
{
return min_t(unsigned int, shost->sg_tablesize, SCSI_INLINE_SG_CNT) *
sizeof(struct scatterlist);
}
static blk_status_t scsi_prepare_cmd(struct request *req)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
struct scsi_device *sdev = req->q->queuedata;
struct Scsi_Host *shost = sdev->host;
bool in_flight = test_bit(SCMD_STATE_INFLIGHT, &cmd->state);
struct scatterlist *sg;
scsi_init_command(sdev, cmd);
cmd->eh_eflags = 0;
cmd->prot_type = 0;
cmd->prot_flags = 0;
cmd->submitter = 0;
memset(&cmd->sdb, 0, sizeof(cmd->sdb));
cmd->underflow = 0;
cmd->transfersize = 0;
cmd->host_scribble = NULL;
cmd->result = 0;
cmd->extra_len = 0;
cmd->state = 0;
if (in_flight)
__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
/*
* Only clear the driver-private command data if the LLD does not supply
* a function to initialize that data.
*/
if (!shost->hostt->init_cmd_priv)
memset(cmd + 1, 0, shost->hostt->cmd_size);
cmd->prot_op = SCSI_PROT_NORMAL;
if (blk_rq_bytes(req))
cmd->sc_data_direction = rq_dma_dir(req);
else
cmd->sc_data_direction = DMA_NONE;
sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
cmd->sdb.table.sgl = sg;
if (scsi_host_get_prot(shost)) {
memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer));
cmd->prot_sdb->table.sgl =
(struct scatterlist *)(cmd->prot_sdb + 1);
}
/*
* Special handling for passthrough commands, which don't go to the ULP
* at all:
*/
if (blk_rq_is_passthrough(req))
return scsi_setup_scsi_cmnd(sdev, req);
if (sdev->handler && sdev->handler->prep_fn) {
blk_status_t ret = sdev->handler->prep_fn(sdev, req);
if (ret != BLK_STS_OK)
return ret;
}
/* Usually overridden by the ULP */
cmd->allowed = 0;
memset(cmd->cmnd, 0, sizeof(cmd->cmnd));
return scsi_cmd_to_driver(cmd)->init_command(cmd);
}
static void scsi_done_internal(struct scsi_cmnd *cmd, bool complete_directly)
{
struct request *req = scsi_cmd_to_rq(cmd);
switch (cmd->submitter) {
case SUBMITTED_BY_BLOCK_LAYER:
break;
case SUBMITTED_BY_SCSI_ERROR_HANDLER:
return scsi_eh_done(cmd);
case SUBMITTED_BY_SCSI_RESET_IOCTL:
return;
}
if (unlikely(blk_should_fake_timeout(scsi_cmd_to_rq(cmd)->q)))
return;
if (unlikely(test_and_set_bit(SCMD_STATE_COMPLETE, &cmd->state)))
return;
trace_scsi_dispatch_cmd_done(cmd);
if (complete_directly)
blk_mq_complete_request_direct(req, scsi_complete);
else
blk_mq_complete_request(req);
}
void scsi_done(struct scsi_cmnd *cmd)
{
scsi_done_internal(cmd, false);
}
EXPORT_SYMBOL(scsi_done);
void scsi_done_direct(struct scsi_cmnd *cmd)
{
scsi_done_internal(cmd, true);
}
EXPORT_SYMBOL(scsi_done_direct);
static void scsi_mq_put_budget(struct request_queue *q, int budget_token)
{
struct scsi_device *sdev = q->queuedata;
sbitmap_put(&sdev->budget_map, budget_token);
}
/*
* When to reinvoke queueing after a resource shortage. It's 3 msecs to
* not change behaviour from the previous unplug mechanism, experimentation
* may prove this needs changing.
*/
#define SCSI_QUEUE_DELAY 3
static int scsi_mq_get_budget(struct request_queue *q)
{
struct scsi_device *sdev = q->queuedata;
int token = scsi_dev_queue_ready(q, sdev);
if (token >= 0)
return token;
atomic_inc(&sdev->restarts);
/*
* Orders atomic_inc(&sdev->restarts) and atomic_read(&sdev->device_busy).
* .restarts must be incremented before .device_busy is read because the
* code in scsi_run_queue_async() depends on the order of these operations.
*/
smp_mb__after_atomic();
/*
* If all in-flight requests originated from this LUN are completed
* before reading .device_busy, sdev->device_busy will be observed as
* zero, then blk_mq_delay_run_hw_queues() will dispatch this request
* soon. Otherwise, completion of one of these requests will observe
* the .restarts flag, and the request queue will be run for handling
* this request, see scsi_end_request().
*/
if (unlikely(scsi_device_busy(sdev) == 0 &&
!scsi_device_blocked(sdev)))
blk_mq_delay_run_hw_queues(sdev->request_queue, SCSI_QUEUE_DELAY);
return -1;
}
static void scsi_mq_set_rq_budget_token(struct request *req, int token)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
cmd->budget_token = token;
}
static int scsi_mq_get_rq_budget_token(struct request *req)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
return cmd->budget_token;
}
static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
{
struct request *req = bd->rq;
struct request_queue *q = req->q;
struct scsi_device *sdev = q->queuedata;
struct Scsi_Host *shost = sdev->host;
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
blk_status_t ret;
int reason;
WARN_ON_ONCE(cmd->budget_token < 0);
/*
* If the device is not in running state we will reject some or all
* commands.
*/
if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
ret = scsi_device_state_check(sdev, req);
if (ret != BLK_STS_OK)
goto out_put_budget;
}
ret = BLK_STS_RESOURCE;
if (!scsi_target_queue_ready(shost, sdev))
goto out_put_budget;
if (unlikely(scsi_host_in_recovery(shost))) {
if (cmd->flags & SCMD_FAIL_IF_RECOVERING)
ret = BLK_STS_OFFLINE;
goto out_dec_target_busy;
}
if (!scsi_host_queue_ready(q, shost, sdev, cmd))
goto out_dec_target_busy;
if (!(req->rq_flags & RQF_DONTPREP)) {
ret = scsi_prepare_cmd(req);
if (ret != BLK_STS_OK)
goto out_dec_host_busy;
req->rq_flags |= RQF_DONTPREP;
} else {
clear_bit(SCMD_STATE_COMPLETE, &cmd->state);
}
cmd->flags &= SCMD_PRESERVED_FLAGS;
if (sdev->simple_tags)
cmd->flags |= SCMD_TAGGED;
if (bd->last)
cmd->flags |= SCMD_LAST;
scsi_set_resid(cmd, 0);
memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
cmd->submitter = SUBMITTED_BY_BLOCK_LAYER;
blk_mq_start_request(req);
reason = scsi_dispatch_cmd(cmd);
if (reason) {
scsi_set_blocked(cmd, reason);
ret = BLK_STS_RESOURCE;
goto out_dec_host_busy;
}
return BLK_STS_OK;
out_dec_host_busy:
scsi_dec_host_busy(shost, cmd);
out_dec_target_busy:
if (scsi_target(sdev)->can_queue > 0)
atomic_dec(&scsi_target(sdev)->target_busy);
out_put_budget:
scsi_mq_put_budget(q, cmd->budget_token);
cmd->budget_token = -1;
switch (ret) {
case BLK_STS_OK:
break;
case BLK_STS_RESOURCE:
case BLK_STS_ZONE_RESOURCE:
if (scsi_device_blocked(sdev))
ret = BLK_STS_DEV_RESOURCE;
break;
case BLK_STS_AGAIN:
cmd->result = DID_BUS_BUSY << 16;
if (req->rq_flags & RQF_DONTPREP)
scsi_mq_uninit_cmd(cmd);
break;
default:
if (unlikely(!scsi_device_online(sdev)))
cmd->result = DID_NO_CONNECT << 16;
else
cmd->result = DID_ERROR << 16;
/*
* Make sure to release all allocated resources when
* we hit an error, as we will never see this command
* again.
*/
if (req->rq_flags & RQF_DONTPREP)
scsi_mq_uninit_cmd(cmd);
scsi_run_queue_async(sdev);
break;
}
return ret;
}
static int scsi_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
unsigned int hctx_idx, unsigned int numa_node)
{
struct Scsi_Host *shost = set->driver_data;
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
struct scatterlist *sg;
int ret = 0;
cmd->sense_buffer =
kmem_cache_alloc_node(scsi_sense_cache, GFP_KERNEL, numa_node);
if (!cmd->sense_buffer)
return -ENOMEM;
if (scsi_host_get_prot(shost)) {
sg = (void *)cmd + sizeof(struct scsi_cmnd) +
shost->hostt->cmd_size;
cmd->prot_sdb = (void *)sg + scsi_mq_inline_sgl_size(shost);
}
if (shost->hostt->init_cmd_priv) {
ret = shost->hostt->init_cmd_priv(shost, cmd);
if (ret < 0)
kmem_cache_free(scsi_sense_cache, cmd->sense_buffer);
}
return ret;
}
static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq,
unsigned int hctx_idx)
{
struct Scsi_Host *shost = set->driver_data; struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
if (shost->hostt->exit_cmd_priv)
shost->hostt->exit_cmd_priv(shost, cmd);
kmem_cache_free(scsi_sense_cache, cmd->sense_buffer);
}
static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
{
struct Scsi_Host *shost = hctx->driver_data;
if (shost->hostt->mq_poll)
return shost->hostt->mq_poll(shost, hctx->queue_num);
return 0;
}
static int scsi_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
unsigned int hctx_idx)
{
struct Scsi_Host *shost = data;
hctx->driver_data = shost;
return 0;
}
static void scsi_map_queues(struct blk_mq_tag_set *set)
{
struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set);
if (shost->hostt->map_queues)
return shost->hostt->map_queues(shost);
blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
}
void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
{
struct device *dev = shost->dma_dev;
/*
* this limit is imposed by hardware restrictions
*/
blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
SG_MAX_SEGMENTS));
if (scsi_host_prot_dma(shost)) {
shost->sg_prot_tablesize =
min_not_zero(shost->sg_prot_tablesize,
(unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
}
blk_queue_max_hw_sectors(q, shost->max_sectors);
blk_queue_segment_boundary(q, shost->dma_boundary);
dma_set_seg_boundary(dev, shost->dma_boundary);
blk_queue_max_segment_size(q, shost->max_segment_size);
blk_queue_virt_boundary(q, shost->virt_boundary_mask);
dma_set_max_seg_size(dev, queue_max_segment_size(q));
/*
* Set a reasonable default alignment: The larger of 32-byte (dword),
* which is a common minimum for HBAs, and the minimum DMA alignment,
* which is set by the platform.
*
* Devices that require a bigger alignment can increase it later.
*/
blk_queue_dma_alignment(q, max(4, dma_get_cache_alignment()) - 1);
}
EXPORT_SYMBOL_GPL(__scsi_init_queue);
static const struct blk_mq_ops scsi_mq_ops_no_commit = {
.get_budget = scsi_mq_get_budget,
.put_budget = scsi_mq_put_budget,
.queue_rq = scsi_queue_rq,
.complete = scsi_complete,
.timeout = scsi_timeout,
#ifdef CONFIG_BLK_DEBUG_FS
.show_rq = scsi_show_rq,
#endif
.init_request = scsi_mq_init_request,
.exit_request = scsi_mq_exit_request,
.cleanup_rq = scsi_cleanup_rq,
.busy = scsi_mq_lld_busy,
.map_queues = scsi_map_queues,
.init_hctx = scsi_init_hctx,
.poll = scsi_mq_poll,
.set_rq_budget_token = scsi_mq_set_rq_budget_token,
.get_rq_budget_token = scsi_mq_get_rq_budget_token,
};
static void scsi_commit_rqs(struct blk_mq_hw_ctx *hctx)
{
struct Scsi_Host *shost = hctx->driver_data;
shost->hostt->commit_rqs(shost, hctx->queue_num);
}
static const struct blk_mq_ops scsi_mq_ops = {
.get_budget = scsi_mq_get_budget,
.put_budget = scsi_mq_put_budget,
.queue_rq = scsi_queue_rq,
.commit_rqs = scsi_commit_rqs,
.complete = scsi_complete,
.timeout = scsi_timeout,
#ifdef CONFIG_BLK_DEBUG_FS
.show_rq = scsi_show_rq,
#endif
.init_request = scsi_mq_init_request,
.exit_request = scsi_mq_exit_request,
.cleanup_rq = scsi_cleanup_rq,
.busy = scsi_mq_lld_busy,
.map_queues = scsi_map_queues,
.init_hctx = scsi_init_hctx,
.poll = scsi_mq_poll,
.set_rq_budget_token = scsi_mq_set_rq_budget_token,
.get_rq_budget_token = scsi_mq_get_rq_budget_token,
};
int scsi_mq_setup_tags(struct Scsi_Host *shost)
{
unsigned int cmd_size, sgl_size;
struct blk_mq_tag_set *tag_set = &shost->tag_set;
sgl_size = max_t(unsigned int, sizeof(struct scatterlist),
scsi_mq_inline_sgl_size(shost));
cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size;
if (scsi_host_get_prot(shost))
cmd_size += sizeof(struct scsi_data_buffer) +
sizeof(struct scatterlist) * SCSI_INLINE_PROT_SG_CNT;
memset(tag_set, 0, sizeof(*tag_set));
if (shost->hostt->commit_rqs)
tag_set->ops = &scsi_mq_ops;
else
tag_set->ops = &scsi_mq_ops_no_commit;
tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;
tag_set->nr_maps = shost->nr_maps ? : 1;
tag_set->queue_depth = shost->can_queue;
tag_set->cmd_size = cmd_size;
tag_set->numa_node = dev_to_node(shost->dma_dev);
tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
tag_set->flags |=
BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);
if (shost->queuecommand_may_block)
tag_set->flags |= BLK_MQ_F_BLOCKING;
tag_set->driver_data = shost;
if (shost->host_tagset)
tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
return blk_mq_alloc_tag_set(tag_set);
}
void scsi_mq_free_tags(struct kref *kref)
{
struct Scsi_Host *shost = container_of(kref, typeof(*shost),
tagset_refcnt);
blk_mq_free_tag_set(&shost->tag_set);
complete(&shost->tagset_freed);
}
/**
* scsi_device_from_queue - return sdev associated with a request_queue
* @q: The request queue to return the sdev from
*
* Return the sdev associated with a request queue or NULL if the
* request_queue does not reference a SCSI device.
*/
struct scsi_device *scsi_device_from_queue(struct request_queue *q)
{
struct scsi_device *sdev = NULL;
if (q->mq_ops == &scsi_mq_ops_no_commit ||
q->mq_ops == &scsi_mq_ops)
sdev = q->queuedata;
if (!sdev || !get_device(&sdev->sdev_gendev))
sdev = NULL;
return sdev;
}
/*
* pktcdvd should have been integrated into the SCSI layers, but for historical
* reasons like the old IDE driver it isn't. This export allows it to safely
* probe if a given device is a SCSI one and only attach to that.
*/
#ifdef CONFIG_CDROM_PKTCDVD_MODULE
EXPORT_SYMBOL_GPL(scsi_device_from_queue);
#endif
/**
* scsi_block_requests - Utility function used by low-level drivers to prevent
* further commands from being queued to the device.
* @shost: host in question
*
* There is no timer nor any other means by which the requests get unblocked
* other than the low-level driver calling scsi_unblock_requests().
*/
void scsi_block_requests(struct Scsi_Host *shost)
{
shost->host_self_blocked = 1;
}
EXPORT_SYMBOL(scsi_block_requests);
/**
* scsi_unblock_requests - Utility function used by low-level drivers to allow
* further commands to be queued to the device.
* @shost: host in question
*
* There is no timer nor any other means by which the requests get unblocked
* other than the low-level driver calling scsi_unblock_requests(). This is done
* as an API function so that changes to the internals of the scsi mid-layer
* won't require wholesale changes to drivers that use this feature.
*/
void scsi_unblock_requests(struct Scsi_Host *shost)
{
shost->host_self_blocked = 0;
scsi_run_host_queues(shost);
}
EXPORT_SYMBOL(scsi_unblock_requests);
void scsi_exit_queue(void)
{
kmem_cache_destroy(scsi_sense_cache);
}
/**
* scsi_mode_select - issue a mode select
* @sdev: SCSI device to be queried
* @pf: Page format bit (1 == standard, 0 == vendor specific)
* @sp: Save page bit (0 == don't save, 1 == save)
* @buffer: request buffer (may not be smaller than eight bytes)
* @len: length of request buffer.
* @timeout: command timeout
* @retries: number of retries before failing
* @data: returns a structure abstracting the mode header data
* @sshdr: place to put sense data (or NULL if no sense to be collected).
* must be SCSI_SENSE_BUFFERSIZE big.
*
* Returns zero if successful; negative error number or scsi
* status on error
*
*/
int scsi_mode_select(struct scsi_device *sdev, int pf, int sp,
unsigned char *buffer, int len, int timeout, int retries,
struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
{
unsigned char cmd[10];
unsigned char *real_buffer;
const struct scsi_exec_args exec_args = {
.sshdr = sshdr,
};
int ret;
memset(cmd, 0, sizeof(cmd));
cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
/*
* Use MODE SELECT(10) if the device asked for it or if the mode page
* and the mode select header cannot fit within the maximumm 255 bytes
* of the MODE SELECT(6) command.
*/
if (sdev->use_10_for_ms ||
len + 4 > 255 ||
data->block_descriptor_length > 255) {
if (len > 65535 - 8)
return -EINVAL;
real_buffer = kmalloc(8 + len, GFP_KERNEL);
if (!real_buffer)
return -ENOMEM;
memcpy(real_buffer + 8, buffer, len);
len += 8;
real_buffer[0] = 0;
real_buffer[1] = 0;
real_buffer[2] = data->medium_type;
real_buffer[3] = data->device_specific;
real_buffer[4] = data->longlba ? 0x01 : 0;
real_buffer[5] = 0;
put_unaligned_be16(data->block_descriptor_length,
&real_buffer[6]);
cmd[0] = MODE_SELECT_10;
put_unaligned_be16(len, &cmd[7]);
} else {
if (data->longlba)
return -EINVAL;
real_buffer = kmalloc(4 + len, GFP_KERNEL);
if (!real_buffer)
return -ENOMEM;
memcpy(real_buffer + 4, buffer, len);
len += 4;
real_buffer[0] = 0;
real_buffer[1] = data->medium_type;
real_buffer[2] = data->device_specific;
real_buffer[3] = data->block_descriptor_length;
cmd[0] = MODE_SELECT;
cmd[4] = len;
}
ret = scsi_execute_cmd(sdev, cmd, REQ_OP_DRV_OUT, real_buffer, len,
timeout, retries, &exec_args);
kfree(real_buffer);
return ret;
}
EXPORT_SYMBOL_GPL(scsi_mode_select);
/**
* scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
* @sdev: SCSI device to be queried
* @dbd: set to prevent mode sense from returning block descriptors
* @modepage: mode page being requested
* @subpage: sub-page of the mode page being requested
* @buffer: request buffer (may not be smaller than eight bytes)
* @len: length of request buffer.
* @timeout: command timeout
* @retries: number of retries before failing
* @data: returns a structure abstracting the mode header data
* @sshdr: place to put sense data (or NULL if no sense to be collected).
* must be SCSI_SENSE_BUFFERSIZE big.
*
* Returns zero if successful, or a negative error number on failure
*/
int
scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, int subpage,
unsigned char *buffer, int len, int timeout, int retries,
struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
{
unsigned char cmd[12];
int use_10_for_ms;
int header_length;
int result;
struct scsi_sense_hdr my_sshdr;
struct scsi_failure failure_defs[] = {
{
.sense = UNIT_ATTENTION,
.asc = SCMD_FAILURE_ASC_ANY,
.ascq = SCMD_FAILURE_ASCQ_ANY,
.allowed = retries,
.result = SAM_STAT_CHECK_CONDITION,
},
{}
};
struct scsi_failures failures = {
.failure_definitions = failure_defs,
};
const struct scsi_exec_args exec_args = {
/* caller might not be interested in sense, but we need it */
.sshdr = sshdr ? : &my_sshdr,
.failures = &failures,
};
memset(data, 0, sizeof(*data));
memset(&cmd[0], 0, 12);
dbd = sdev->set_dbd_for_ms ? 8 : dbd;
cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */
cmd[2] = modepage;
cmd[3] = subpage;
sshdr = exec_args.sshdr;
retry:
use_10_for_ms = sdev->use_10_for_ms || len > 255;
if (use_10_for_ms) {
if (len < 8 || len > 65535)
return -EINVAL;
cmd[0] = MODE_SENSE_10;
put_unaligned_be16(len, &cmd[7]);
header_length = 8;
} else {
if (len < 4)
return -EINVAL;
cmd[0] = MODE_SENSE;
cmd[4] = len;
header_length = 4;
}
memset(buffer, 0, len);
result = scsi_execute_cmd(sdev, cmd, REQ_OP_DRV_IN, buffer, len,
timeout, retries, &exec_args);
if (result < 0)
return result;
/* This code looks awful: what it's doing is making sure an
* ILLEGAL REQUEST sense return identifies the actual command
* byte as the problem. MODE_SENSE commands can return
* ILLEGAL REQUEST if the code page isn't supported */
if (!scsi_status_is_good(result)) {
if (scsi_sense_valid(sshdr)) {
if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
(sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
/*
* Invalid command operation code: retry using
* MODE SENSE(6) if this was a MODE SENSE(10)
* request, except if the request mode page is
* too large for MODE SENSE single byte
* allocation length field.
*/
if (use_10_for_ms) {
if (len > 255)
return -EIO;
sdev->use_10_for_ms = 0;
goto retry;
}
}
}
return -EIO;
}
if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
(modepage == 6 || modepage == 8))) {
/* Initio breakage? */
header_length = 0;
data->length = 13;
data->medium_type = 0;
data->device_specific = 0;
data->longlba = 0;
data->block_descriptor_length = 0;
} else if (use_10_for_ms) {
data->length = get_unaligned_be16(&buffer[0]) + 2;
data->medium_type = buffer[2];
data->device_specific = buffer[3];
data->longlba = buffer[4] & 0x01;
data->block_descriptor_length = get_unaligned_be16(&buffer[6]);
} else {
data->length = buffer[0] + 1;
data->medium_type = buffer[1];
data->device_specific = buffer[2];
data->block_descriptor_length = buffer[3];
}
data->header_length = header_length;
return 0;
}
EXPORT_SYMBOL(scsi_mode_sense);
/**
* scsi_test_unit_ready - test if unit is ready
* @sdev: scsi device to change the state of.
* @timeout: command timeout
* @retries: number of retries before failing
* @sshdr: outpout pointer for decoded sense information.
*
* Returns zero if unsuccessful or an error if TUR failed. For
* removable media, UNIT_ATTENTION sets ->changed flag.
**/
int
scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
struct scsi_sense_hdr *sshdr)
{
char cmd[] = {
TEST_UNIT_READY, 0, 0, 0, 0, 0,
};
const struct scsi_exec_args exec_args = {
.sshdr = sshdr,
};
int result;
/* try to eat the UNIT_ATTENTION if there are enough retries */
do {
result = scsi_execute_cmd(sdev, cmd, REQ_OP_DRV_IN, NULL, 0,
timeout, 1, &exec_args);
if (sdev->removable && result > 0 && scsi_sense_valid(sshdr) &&
sshdr->sense_key == UNIT_ATTENTION)
sdev->changed = 1;
} while (result > 0 && scsi_sense_valid(sshdr) &&
sshdr->sense_key == UNIT_ATTENTION && --retries);
return result;
}
EXPORT_SYMBOL(scsi_test_unit_ready);
/**
* scsi_device_set_state - Take the given device through the device state model.
* @sdev: scsi device to change the state of.
* @state: state to change to.
*
* Returns zero if successful or an error if the requested
* transition is illegal.
*/
int
scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
{
enum scsi_device_state oldstate = sdev->sdev_state;
if (state == oldstate)
return 0;
switch (state) {
case SDEV_CREATED:
switch (oldstate) {
case SDEV_CREATED_BLOCK:
break;
default:
goto illegal;
}
break;
case SDEV_RUNNING:
switch (oldstate) {
case SDEV_CREATED:
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
case SDEV_QUIESCE:
case SDEV_BLOCK:
break;
default:
goto illegal;
}
break;
case SDEV_QUIESCE:
switch (oldstate) {
case SDEV_RUNNING:
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
break;
default:
goto illegal;
}
break;
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
switch (oldstate) {
case SDEV_CREATED:
case SDEV_RUNNING:
case SDEV_QUIESCE:
case SDEV_BLOCK:
break;
default:
goto illegal;
}
break;
case SDEV_BLOCK:
switch (oldstate) {
case SDEV_RUNNING:
case SDEV_CREATED_BLOCK:
case SDEV_QUIESCE:
case SDEV_OFFLINE:
break;
default:
goto illegal;
}
break;
case SDEV_CREATED_BLOCK:
switch (oldstate) {
case SDEV_CREATED:
break;
default:
goto illegal;
}
break;
case SDEV_CANCEL:
switch (oldstate) {
case SDEV_CREATED:
case SDEV_RUNNING:
case SDEV_QUIESCE:
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
break;
default:
goto illegal;
}
break;
case SDEV_DEL:
switch (oldstate) {
case SDEV_CREATED:
case SDEV_RUNNING:
case SDEV_OFFLINE:
case SDEV_TRANSPORT_OFFLINE:
case SDEV_CANCEL:
case SDEV_BLOCK:
case SDEV_CREATED_BLOCK:
break;
default:
goto illegal;
}
break;
}
sdev->offline_already = false;
sdev->sdev_state = state;
return 0;
illegal:
SCSI_LOG_ERROR_RECOVERY(1,
sdev_printk(KERN_ERR, sdev,
"Illegal state transition %s->%s",
scsi_device_state_name(oldstate),
scsi_device_state_name(state))
);
return -EINVAL;
}
EXPORT_SYMBOL(scsi_device_set_state);
/**
* scsi_evt_emit - emit a single SCSI device uevent
* @sdev: associated SCSI device
* @evt: event to emit
*
* Send a single uevent (scsi_event) to the associated scsi_device.
*/
static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)
{
int idx = 0;
char *envp[3];
switch (evt->evt_type) {
case SDEV_EVT_MEDIA_CHANGE:
envp[idx++] = "SDEV_MEDIA_CHANGE=1";
break;
case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
scsi_rescan_device(sdev);
envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED";
break;
case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED";
break;
case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED";
break;
case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED";
break;
case SDEV_EVT_LUN_CHANGE_REPORTED:
envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED";
break;
case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED:
envp[idx++] = "SDEV_UA=ASYMMETRIC_ACCESS_STATE_CHANGED";
break;
case SDEV_EVT_POWER_ON_RESET_OCCURRED:
envp[idx++] = "SDEV_UA=POWER_ON_RESET_OCCURRED";
break;
default:
/* do nothing */
break;
}
envp[idx++] = NULL;
kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);
}
/**
* scsi_evt_thread - send a uevent for each scsi event
* @work: work struct for scsi_device
*
* Dispatch queued events to their associated scsi_device kobjects
* as uevents.
*/
void scsi_evt_thread(struct work_struct *work)
{
struct scsi_device *sdev;
enum scsi_device_event evt_type;
LIST_HEAD(event_list);
sdev = container_of(work, struct scsi_device, event_work);
for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++)
if (test_and_clear_bit(evt_type, sdev->pending_events))
sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL);
while (1) {
struct scsi_event *evt;
struct list_head *this, *tmp;
unsigned long flags;
spin_lock_irqsave(&sdev->list_lock, flags);
list_splice_init(&sdev->event_list, &event_list);
spin_unlock_irqrestore(&sdev->list_lock, flags);
if (list_empty(&event_list))
break;
list_for_each_safe(this, tmp, &event_list) {
evt = list_entry(this, struct scsi_event, node);
list_del(&evt->node);
scsi_evt_emit(sdev, evt);
kfree(evt);
}
}
}
/**
* sdev_evt_send - send asserted event to uevent thread
* @sdev: scsi_device event occurred on
* @evt: event to send
*
* Assert scsi device event asynchronously.
*/
void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
{
unsigned long flags;
#if 0
/* FIXME: currently this check eliminates all media change events
* for polled devices. Need to update to discriminate between AN
* and polled events */
if (!test_bit(evt->evt_type, sdev->supported_events)) {
kfree(evt);
return;
}
#endif
spin_lock_irqsave(&sdev->list_lock, flags);
list_add_tail(&evt->node, &sdev->event_list);
schedule_work(&sdev->event_work);
spin_unlock_irqrestore(&sdev->list_lock, flags);
}
EXPORT_SYMBOL_GPL(sdev_evt_send);
/**
* sdev_evt_alloc - allocate a new scsi event
* @evt_type: type of event to allocate
* @gfpflags: GFP flags for allocation
*
* Allocates and returns a new scsi_event.
*/
struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,
gfp_t gfpflags)
{
struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);
if (!evt)
return NULL;
evt->evt_type = evt_type;
INIT_LIST_HEAD(&evt->node);
/* evt_type-specific initialization, if any */
switch (evt_type) {
case SDEV_EVT_MEDIA_CHANGE:
case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
case SDEV_EVT_LUN_CHANGE_REPORTED:
case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED:
case SDEV_EVT_POWER_ON_RESET_OCCURRED:
default:
/* do nothing */
break;
}
return evt;
}
EXPORT_SYMBOL_GPL(sdev_evt_alloc);
/**
* sdev_evt_send_simple - send asserted event to uevent thread
* @sdev: scsi_device event occurred on
* @evt_type: type of event to send
* @gfpflags: GFP flags for allocation
*
* Assert scsi device event asynchronously, given an event type.
*/
void sdev_evt_send_simple(struct scsi_device *sdev,
enum scsi_device_event evt_type, gfp_t gfpflags)
{
struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);
if (!evt) {
sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",
evt_type);
return;
}
sdev_evt_send(sdev, evt);
}
EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
/**
* scsi_device_quiesce - Block all commands except power management.
* @sdev: scsi device to quiesce.
*
* This works by trying to transition to the SDEV_QUIESCE state
* (which must be a legal transition). When the device is in this
* state, only power management requests will be accepted, all others will
* be deferred.
*
* Must be called with user context, may sleep.
*
* Returns zero if unsuccessful or an error if not.
*/
int
scsi_device_quiesce(struct scsi_device *sdev)
{
struct request_queue *q = sdev->request_queue;
int err;
/*
* It is allowed to call scsi_device_quiesce() multiple times from
* the same context but concurrent scsi_device_quiesce() calls are
* not allowed.
*/
WARN_ON_ONCE(sdev->quiesced_by && sdev->quiesced_by != current);
if (sdev->quiesced_by == current)
return 0;
blk_set_pm_only(q);
blk_mq_freeze_queue(q);
/*
* Ensure that the effect of blk_set_pm_only() will be visible
* for percpu_ref_tryget() callers that occur after the queue
* unfreeze even if the queue was already frozen before this function
* was called. See also https://lwn.net/Articles/573497/.
*/
synchronize_rcu();
blk_mq_unfreeze_queue(q);
mutex_lock(&sdev->state_mutex);
err = scsi_device_set_state(sdev, SDEV_QUIESCE);
if (err == 0)
sdev->quiesced_by = current;
else
blk_clear_pm_only(q);
mutex_unlock(&sdev->state_mutex);
return err;
}
EXPORT_SYMBOL(scsi_device_quiesce);
/**
* scsi_device_resume - Restart user issued commands to a quiesced device.
* @sdev: scsi device to resume.
*
* Moves the device from quiesced back to running and restarts the
* queues.
*
* Must be called with user context, may sleep.
*/
void scsi_device_resume(struct scsi_device *sdev)
{
/* check if the device state was mutated prior to resume, and if
* so assume the state is being managed elsewhere (for example
* device deleted during suspend)
*/
mutex_lock(&sdev->state_mutex);
if (sdev->sdev_state == SDEV_QUIESCE)
scsi_device_set_state(sdev, SDEV_RUNNING);
if (sdev->quiesced_by) {
sdev->quiesced_by = NULL;
blk_clear_pm_only(sdev->request_queue);
}
mutex_unlock(&sdev->state_mutex);
}
EXPORT_SYMBOL(scsi_device_resume);
static void
device_quiesce_fn(struct scsi_device *sdev, void *data)
{
scsi_device_quiesce(sdev);
}
void
scsi_target_quiesce(struct scsi_target *starget)
{
starget_for_each_device(starget, NULL, device_quiesce_fn);
}
EXPORT_SYMBOL(scsi_target_quiesce);
static void
device_resume_fn(struct scsi_device *sdev, void *data)
{
scsi_device_resume(sdev);
}
void
scsi_target_resume(struct scsi_target *starget)
{
starget_for_each_device(starget, NULL, device_resume_fn);
}
EXPORT_SYMBOL(scsi_target_resume);
static int __scsi_internal_device_block_nowait(struct scsi_device *sdev)
{
if (scsi_device_set_state(sdev, SDEV_BLOCK))
return scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);
return 0;
}
void scsi_start_queue(struct scsi_device *sdev)
{
if (cmpxchg(&sdev->queue_stopped, 1, 0))
blk_mq_unquiesce_queue(sdev->request_queue);
}
static void scsi_stop_queue(struct scsi_device *sdev)
{
/*
* The atomic variable of ->queue_stopped covers that
* blk_mq_quiesce_queue* is balanced with blk_mq_unquiesce_queue.
*
* The caller needs to wait until quiesce is done.
*/
if (!cmpxchg(&sdev->queue_stopped, 0, 1))
blk_mq_quiesce_queue_nowait(sdev->request_queue);
}
/**
* scsi_internal_device_block_nowait - try to transition to the SDEV_BLOCK state
* @sdev: device to block
*
* Pause SCSI command processing on the specified device. Does not sleep.
*
* Returns zero if successful or a negative error code upon failure.
*
* Notes:
* This routine transitions the device to the SDEV_BLOCK state (which must be
* a legal transition). When the device is in this state, command processing
* is paused until the device leaves the SDEV_BLOCK state. See also
* scsi_internal_device_unblock_nowait().
*/
int scsi_internal_device_block_nowait(struct scsi_device *sdev)
{
int ret = __scsi_internal_device_block_nowait(sdev);
/*
* The device has transitioned to SDEV_BLOCK. Stop the
* block layer from calling the midlayer with this device's
* request queue.
*/
if (!ret)
scsi_stop_queue(sdev);
return ret;
}
EXPORT_SYMBOL_GPL(scsi_internal_device_block_nowait);
/**
* scsi_device_block - try to transition to the SDEV_BLOCK state
* @sdev: device to block
* @data: dummy argument, ignored
*
* Pause SCSI command processing on the specified device. Callers must wait
* until all ongoing scsi_queue_rq() calls have finished after this function
* returns.
*
* Note:
* This routine transitions the device to the SDEV_BLOCK state (which must be
* a legal transition). When the device is in this state, command processing
* is paused until the device leaves the SDEV_BLOCK state. See also
* scsi_internal_device_unblock().
*/
static void scsi_device_block(struct scsi_device *sdev, void *data)
{
int err;
enum scsi_device_state state;
mutex_lock(&sdev->state_mutex);
err = __scsi_internal_device_block_nowait(sdev);
state = sdev->sdev_state;
if (err == 0)
/*
* scsi_stop_queue() must be called with the state_mutex
* held. Otherwise a simultaneous scsi_start_queue() call
* might unquiesce the queue before we quiesce it.
*/
scsi_stop_queue(sdev);
mutex_unlock(&sdev->state_mutex);
WARN_ONCE(err, "%s: failed to block %s in state %d\n",
__func__, dev_name(&sdev->sdev_gendev), state);
}
/**
* scsi_internal_device_unblock_nowait - resume a device after a block request
* @sdev: device to resume
* @new_state: state to set the device to after unblocking
*
* Restart the device queue for a previously suspended SCSI device. Does not
* sleep.
*
* Returns zero if successful or a negative error code upon failure.
*
* Notes:
* This routine transitions the device to the SDEV_RUNNING state or to one of
* the offline states (which must be a legal transition) allowing the midlayer
* to goose the queue for this device.
*/
int scsi_internal_device_unblock_nowait(struct scsi_device *sdev,
enum scsi_device_state new_state)
{
switch (new_state) {
case SDEV_RUNNING:
case SDEV_TRANSPORT_OFFLINE:
break;
default:
return -EINVAL;
}
/*
* Try to transition the scsi device to SDEV_RUNNING or one of the
* offlined states and goose the device queue if successful.
*/
switch (sdev->sdev_state) {
case SDEV_BLOCK:
case SDEV_TRANSPORT_OFFLINE:
sdev->sdev_state = new_state;
break;
case SDEV_CREATED_BLOCK:
if (new_state == SDEV_TRANSPORT_OFFLINE ||
new_state == SDEV_OFFLINE)
sdev->sdev_state = new_state;
else
sdev->sdev_state = SDEV_CREATED;
break;
case SDEV_CANCEL:
case SDEV_OFFLINE:
break;
default:
return -EINVAL;
}
scsi_start_queue(sdev);
return 0;
}
EXPORT_SYMBOL_GPL(scsi_internal_device_unblock_nowait);
/**
* scsi_internal_device_unblock - resume a device after a block request
* @sdev: device to resume
* @new_state: state to set the device to after unblocking
*
* Restart the device queue for a previously suspended SCSI device. May sleep.
*
* Returns zero if successful or a negative error code upon failure.
*
* Notes:
* This routine transitions the device to the SDEV_RUNNING state or to one of
* the offline states (which must be a legal transition) allowing the midlayer
* to goose the queue for this device.
*/
static int scsi_internal_device_unblock(struct scsi_device *sdev,
enum scsi_device_state new_state)
{
int ret;
mutex_lock(&sdev->state_mutex);
ret = scsi_internal_device_unblock_nowait(sdev, new_state);
mutex_unlock(&sdev->state_mutex);
return ret;
}
static int
target_block(struct device *dev, void *data)
{
if (scsi_is_target_device(dev))
starget_for_each_device(to_scsi_target(dev), NULL,
scsi_device_block);
return 0;
}
/**
* scsi_block_targets - transition all SCSI child devices to SDEV_BLOCK state
* @dev: a parent device of one or more scsi_target devices
* @shost: the Scsi_Host to which this device belongs
*
* Iterate over all children of @dev, which should be scsi_target devices,
* and switch all subordinate scsi devices to SDEV_BLOCK state. Wait for
* ongoing scsi_queue_rq() calls to finish. May sleep.
*
* Note:
* @dev must not itself be a scsi_target device.
*/
void
scsi_block_targets(struct Scsi_Host *shost, struct device *dev)
{
WARN_ON_ONCE(scsi_is_target_device(dev));
device_for_each_child(dev, NULL, target_block);
blk_mq_wait_quiesce_done(&shost->tag_set);
}
EXPORT_SYMBOL_GPL(scsi_block_targets);
static void
device_unblock(struct scsi_device *sdev, void *data)
{
scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data);
}
static int
target_unblock(struct device *dev, void *data)
{
if (scsi_is_target_device(dev))
starget_for_each_device(to_scsi_target(dev), data,
device_unblock);
return 0;
}
void
scsi_target_unblock(struct device *dev, enum scsi_device_state new_state)
{
if (scsi_is_target_device(dev))
starget_for_each_device(to_scsi_target(dev), &new_state,
device_unblock);
else
device_for_each_child(dev, &new_state, target_unblock);
}
EXPORT_SYMBOL_GPL(scsi_target_unblock);
/**
* scsi_host_block - Try to transition all logical units to the SDEV_BLOCK state
* @shost: device to block
*
* Pause SCSI command processing for all logical units associated with the SCSI
* host and wait until pending scsi_queue_rq() calls have finished.
*
* Returns zero if successful or a negative error code upon failure.
*/
int
scsi_host_block(struct Scsi_Host *shost)
{
struct scsi_device *sdev;
int ret;
/*
* Call scsi_internal_device_block_nowait so we can avoid
* calling synchronize_rcu() for each LUN.
*/
shost_for_each_device(sdev, shost) {
mutex_lock(&sdev->state_mutex);
ret = scsi_internal_device_block_nowait(sdev);
mutex_unlock(&sdev->state_mutex);
if (ret) {
scsi_device_put(sdev);
return ret;
}
}
/* Wait for ongoing scsi_queue_rq() calls to finish. */
blk_mq_wait_quiesce_done(&shost->tag_set);
return 0;
}
EXPORT_SYMBOL_GPL(scsi_host_block);
int
scsi_host_unblock(struct Scsi_Host *shost, int new_state)
{
struct scsi_device *sdev;
int ret = 0;
shost_for_each_device(sdev, shost) {
ret = scsi_internal_device_unblock(sdev, new_state);
if (ret) {
scsi_device_put(sdev);
break;
}
}
return ret;
}
EXPORT_SYMBOL_GPL(scsi_host_unblock);
/**
* scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
* @sgl: scatter-gather list
* @sg_count: number of segments in sg
* @offset: offset in bytes into sg, on return offset into the mapped area
* @len: bytes to map, on return number of bytes mapped
*
* Returns virtual address of the start of the mapped page
*/
void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
size_t *offset, size_t *len)
{
int i;
size_t sg_len = 0, len_complete = 0;
struct scatterlist *sg;
struct page *page;
WARN_ON(!irqs_disabled());
for_each_sg(sgl, sg, sg_count, i) {
len_complete = sg_len; /* Complete sg-entries */
sg_len += sg->length;
if (sg_len > *offset)
break;
}
if (unlikely(i == sg_count)) {
printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
"elements %d\n",
__func__, sg_len, *offset, sg_count);
WARN_ON(1);
return NULL;
}
/* Offset starting from the beginning of first page in this sg-entry */
*offset = *offset - len_complete + sg->offset;
/* Assumption: contiguous pages can be accessed as "page + i" */
page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));
*offset &= ~PAGE_MASK;
/* Bytes in this sg-entry from *offset to the end of the page */
sg_len = PAGE_SIZE - *offset;
if (*len > sg_len)
*len = sg_len;
return kmap_atomic(page);
}
EXPORT_SYMBOL(scsi_kmap_atomic_sg);
/**
* scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg
* @virt: virtual address to be unmapped
*/
void scsi_kunmap_atomic_sg(void *virt)
{
kunmap_atomic(virt);
}
EXPORT_SYMBOL(scsi_kunmap_atomic_sg);
void sdev_disable_disk_events(struct scsi_device *sdev)
{
atomic_inc(&sdev->disk_events_disable_depth);
}
EXPORT_SYMBOL(sdev_disable_disk_events);
void sdev_enable_disk_events(struct scsi_device *sdev)
{
if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0))
return;
atomic_dec(&sdev->disk_events_disable_depth);
}
EXPORT_SYMBOL(sdev_enable_disk_events);
static unsigned char designator_prio(const unsigned char *d)
{
if (d[1] & 0x30)
/* not associated with LUN */
return 0;
if (d[3] == 0)
/* invalid length */
return 0;
/*
* Order of preference for lun descriptor:
* - SCSI name string
* - NAA IEEE Registered Extended
* - EUI-64 based 16-byte
* - EUI-64 based 12-byte
* - NAA IEEE Registered
* - NAA IEEE Extended
* - EUI-64 based 8-byte
* - SCSI name string (truncated)
* - T10 Vendor ID
* as longer descriptors reduce the likelyhood
* of identification clashes.
*/
switch (d[1] & 0xf) {
case 8:
/* SCSI name string, variable-length UTF-8 */
return 9;
case 3:
switch (d[4] >> 4) {
case 6:
/* NAA registered extended */
return 8;
case 5:
/* NAA registered */
return 5;
case 4:
/* NAA extended */
return 4;
case 3:
/* NAA locally assigned */
return 1;
default:
break;
}
break;
case 2:
switch (d[3]) {
case 16:
/* EUI64-based, 16 byte */
return 7;
case 12:
/* EUI64-based, 12 byte */
return 6;
case 8:
/* EUI64-based, 8 byte */
return 3;
default:
break;
}
break;
case 1:
/* T10 vendor ID */
return 1;
default:
break;
}
return 0;
}
/**
* scsi_vpd_lun_id - return a unique device identification
* @sdev: SCSI device
* @id: buffer for the identification
* @id_len: length of the buffer
*
* Copies a unique device identification into @id based
* on the information in the VPD page 0x83 of the device.
* The string will be formatted as a SCSI name string.
*
* Returns the length of the identification or error on failure.
* If the identifier is longer than the supplied buffer the actual
* identifier length is returned and the buffer is not zero-padded.
*/
int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len)
{
u8 cur_id_prio = 0;
u8 cur_id_size = 0;
const unsigned char *d, *cur_id_str;
const struct scsi_vpd *vpd_pg83;
int id_size = -EINVAL;
rcu_read_lock();
vpd_pg83 = rcu_dereference(sdev->vpd_pg83);
if (!vpd_pg83) {
rcu_read_unlock();
return -ENXIO;
}
/* The id string must be at least 20 bytes + terminating NULL byte */
if (id_len < 21) {
rcu_read_unlock();
return -EINVAL;
}
memset(id, 0, id_len);
for (d = vpd_pg83->data + 4;
d < vpd_pg83->data + vpd_pg83->len;
d += d[3] + 4) {
u8 prio = designator_prio(d);
if (prio == 0 || cur_id_prio > prio)
continue;
switch (d[1] & 0xf) {
case 0x1:
/* T10 Vendor ID */
if (cur_id_size > d[3])
break;
cur_id_prio = prio;
cur_id_size = d[3];
if (cur_id_size + 4 > id_len)
cur_id_size = id_len - 4;
cur_id_str = d + 4;
id_size = snprintf(id, id_len, "t10.%*pE",
cur_id_size, cur_id_str);
break;
case 0x2:
/* EUI-64 */
cur_id_prio = prio;
cur_id_size = d[3];
cur_id_str = d + 4;
switch (cur_id_size) {
case 8:
id_size = snprintf(id, id_len,
"eui.%8phN",
cur_id_str);
break;
case 12:
id_size = snprintf(id, id_len,
"eui.%12phN",
cur_id_str);
break;
case 16:
id_size = snprintf(id, id_len,
"eui.%16phN",
cur_id_str);
break;
default:
break;
}
break;
case 0x3:
/* NAA */
cur_id_prio = prio;
cur_id_size = d[3];
cur_id_str = d + 4;
switch (cur_id_size) {
case 8:
id_size = snprintf(id, id_len,
"naa.%8phN",
cur_id_str);
break;
case 16:
id_size = snprintf(id, id_len,
"naa.%16phN",
cur_id_str);
break;
default:
break;
}
break;
case 0x8:
/* SCSI name string */
if (cur_id_size > d[3])
break;
/* Prefer others for truncated descriptor */
if (d[3] > id_len) {
prio = 2;
if (cur_id_prio > prio)
break;
}
cur_id_prio = prio;
cur_id_size = id_size = d[3];
cur_id_str = d + 4;
if (cur_id_size >= id_len)
cur_id_size = id_len - 1;
memcpy(id, cur_id_str, cur_id_size);
break;
default:
break;
}
}
rcu_read_unlock();
return id_size;
}
EXPORT_SYMBOL(scsi_vpd_lun_id);
/*
* scsi_vpd_tpg_id - return a target port group identifier
* @sdev: SCSI device
*
* Returns the Target Port Group identifier from the information
* froom VPD page 0x83 of the device.
*
* Returns the identifier or error on failure.
*/
int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id)
{
const unsigned char *d;
const struct scsi_vpd *vpd_pg83;
int group_id = -EAGAIN, rel_port = -1;
rcu_read_lock();
vpd_pg83 = rcu_dereference(sdev->vpd_pg83);
if (!vpd_pg83) {
rcu_read_unlock();
return -ENXIO;
}
d = vpd_pg83->data + 4;
while (d < vpd_pg83->data + vpd_pg83->len) {
switch (d[1] & 0xf) {
case 0x4:
/* Relative target port */
rel_port = get_unaligned_be16(&d[6]);
break;
case 0x5:
/* Target port group */
group_id = get_unaligned_be16(&d[6]);
break;
default:
break;
}
d += d[3] + 4;
}
rcu_read_unlock();
if (group_id >= 0 && rel_id && rel_port != -1)
*rel_id = rel_port;
return group_id;
}
EXPORT_SYMBOL(scsi_vpd_tpg_id);
/**
* scsi_build_sense - build sense data for a command
* @scmd: scsi command for which the sense should be formatted
* @desc: Sense format (non-zero == descriptor format,
* 0 == fixed format)
* @key: Sense key
* @asc: Additional sense code
* @ascq: Additional sense code qualifier
*
**/
void scsi_build_sense(struct scsi_cmnd *scmd, int desc, u8 key, u8 asc, u8 ascq)
{
scsi_build_sense_buffer(desc, scmd->sense_buffer, key, asc, ascq);
scmd->result = SAM_STAT_CHECK_CONDITION;
}
EXPORT_SYMBOL_GPL(scsi_build_sense);
#ifdef CONFIG_SCSI_LIB_KUNIT_TEST
#include "scsi_lib_test.c"
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/core.c - core driver model code (device registration, etc)
*
* Copyright (c) 2002-3 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
* Copyright (c) 2006 Greg Kroah-Hartman <gregkh@suse.de>
* Copyright (c) 2006 Novell, Inc.
*/
#include <linux/acpi.h>
#include <linux/cpufreq.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/fwnode.h>
#include <linux/init.h>
#include <linux/kstrtox.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/kdev_t.h>
#include <linux/notifier.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/blkdev.h>
#include <linux/mutex.h>
#include <linux/pm_runtime.h>
#include <linux/netdevice.h>
#include <linux/sched/signal.h>
#include <linux/sched/mm.h>
#include <linux/string_helpers.h>
#include <linux/swiotlb.h>
#include <linux/sysfs.h>
#include <linux/dma-map-ops.h> /* for dma_default_coherent */
#include "base.h"
#include "physical_location.h"
#include "power/power.h"
/* Device links support. */
static LIST_HEAD(deferred_sync);
static unsigned int defer_sync_state_count = 1;
static DEFINE_MUTEX(fwnode_link_lock);
static bool fw_devlink_is_permissive(void);
static void __fw_devlink_link_to_consumers(struct device *dev);
static bool fw_devlink_drv_reg_done;
static bool fw_devlink_best_effort;
static struct workqueue_struct *device_link_wq;
/**
* __fwnode_link_add - Create a link between two fwnode_handles.
* @con: Consumer end of the link.
* @sup: Supplier end of the link.
* @flags: Link flags.
*
* Create a fwnode link between fwnode handles @con and @sup. The fwnode link
* represents the detail that the firmware lists @sup fwnode as supplying a
* resource to @con.
*
* The driver core will use the fwnode link to create a device link between the
* two device objects corresponding to @con and @sup when they are created. The
* driver core will automatically delete the fwnode link between @con and @sup
* after doing that.
*
* Attempts to create duplicate links between the same pair of fwnode handles
* are ignored and there is no reference counting.
*/
static int __fwnode_link_add(struct fwnode_handle *con,
struct fwnode_handle *sup, u8 flags)
{
struct fwnode_link *link;
list_for_each_entry(link, &sup->consumers, s_hook)
if (link->consumer == con) {
link->flags |= flags;
return 0;
}
link = kzalloc(sizeof(*link), GFP_KERNEL);
if (!link)
return -ENOMEM;
link->supplier = sup;
INIT_LIST_HEAD(&link->s_hook);
link->consumer = con;
INIT_LIST_HEAD(&link->c_hook);
link->flags = flags;
list_add(&link->s_hook, &sup->consumers);
list_add(&link->c_hook, &con->suppliers);
pr_debug("%pfwf Linked as a fwnode consumer to %pfwf\n",
con, sup);
return 0;
}
int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup,
u8 flags)
{
int ret;
mutex_lock(&fwnode_link_lock);
ret = __fwnode_link_add(con, sup, flags);
mutex_unlock(&fwnode_link_lock);
return ret;
}
/**
* __fwnode_link_del - Delete a link between two fwnode_handles.
* @link: the fwnode_link to be deleted
*
* The fwnode_link_lock needs to be held when this function is called.
*/
static void __fwnode_link_del(struct fwnode_link *link)
{
pr_debug("%pfwf Dropping the fwnode link to %pfwf\n",
link->consumer, link->supplier);
list_del(&link->s_hook);
list_del(&link->c_hook);
kfree(link);
}
/**
* __fwnode_link_cycle - Mark a fwnode link as being part of a cycle.
* @link: the fwnode_link to be marked
*
* The fwnode_link_lock needs to be held when this function is called.
*/
static void __fwnode_link_cycle(struct fwnode_link *link)
{
pr_debug("%pfwf: cycle: depends on %pfwf\n",
link->consumer, link->supplier);
link->flags |= FWLINK_FLAG_CYCLE;
}
/**
* fwnode_links_purge_suppliers - Delete all supplier links of fwnode_handle.
* @fwnode: fwnode whose supplier links need to be deleted
*
* Deletes all supplier links connecting directly to @fwnode.
*/
static void fwnode_links_purge_suppliers(struct fwnode_handle *fwnode)
{
struct fwnode_link *link, *tmp;
mutex_lock(&fwnode_link_lock);
list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook)
__fwnode_link_del(link); mutex_unlock(&fwnode_link_lock);
}
/**
* fwnode_links_purge_consumers - Delete all consumer links of fwnode_handle.
* @fwnode: fwnode whose consumer links need to be deleted
*
* Deletes all consumer links connecting directly to @fwnode.
*/
static void fwnode_links_purge_consumers(struct fwnode_handle *fwnode)
{
struct fwnode_link *link, *tmp;
mutex_lock(&fwnode_link_lock);
list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook)
__fwnode_link_del(link);
mutex_unlock(&fwnode_link_lock);
}
/**
* fwnode_links_purge - Delete all links connected to a fwnode_handle.
* @fwnode: fwnode whose links needs to be deleted
*
* Deletes all links connecting directly to a fwnode.
*/
void fwnode_links_purge(struct fwnode_handle *fwnode)
{
fwnode_links_purge_suppliers(fwnode);
fwnode_links_purge_consumers(fwnode);
}
void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode)
{
struct fwnode_handle *child;
/* Don't purge consumer links of an added child */
if (fwnode->dev)
return;
fwnode->flags |= FWNODE_FLAG_NOT_DEVICE;
fwnode_links_purge_consumers(fwnode);
fwnode_for_each_available_child_node(fwnode, child)
fw_devlink_purge_absent_suppliers(child);
}
EXPORT_SYMBOL_GPL(fw_devlink_purge_absent_suppliers);
/**
* __fwnode_links_move_consumers - Move consumer from @from to @to fwnode_handle
* @from: move consumers away from this fwnode
* @to: move consumers to this fwnode
*
* Move all consumer links from @from fwnode to @to fwnode.
*/
static void __fwnode_links_move_consumers(struct fwnode_handle *from,
struct fwnode_handle *to)
{
struct fwnode_link *link, *tmp;
list_for_each_entry_safe(link, tmp, &from->consumers, s_hook) {
__fwnode_link_add(link->consumer, to, link->flags);
__fwnode_link_del(link);
}
}
/**
* __fw_devlink_pickup_dangling_consumers - Pick up dangling consumers
* @fwnode: fwnode from which to pick up dangling consumers
* @new_sup: fwnode of new supplier
*
* If the @fwnode has a corresponding struct device and the device supports
* probing (that is, added to a bus), then we want to let fw_devlink create
* MANAGED device links to this device, so leave @fwnode and its descendant's
* fwnode links alone.
*
* Otherwise, move its consumers to the new supplier @new_sup.
*/
static void __fw_devlink_pickup_dangling_consumers(struct fwnode_handle *fwnode,
struct fwnode_handle *new_sup)
{
struct fwnode_handle *child;
if (fwnode->dev && fwnode->dev->bus)
return;
fwnode->flags |= FWNODE_FLAG_NOT_DEVICE;
__fwnode_links_move_consumers(fwnode, new_sup);
fwnode_for_each_available_child_node(fwnode, child)
__fw_devlink_pickup_dangling_consumers(child, new_sup);
}
static DEFINE_MUTEX(device_links_lock);
DEFINE_STATIC_SRCU(device_links_srcu);
static inline void device_links_write_lock(void)
{
mutex_lock(&device_links_lock);
}
static inline void device_links_write_unlock(void)
{
mutex_unlock(&device_links_lock);
}
int device_links_read_lock(void) __acquires(&device_links_srcu)
{
return srcu_read_lock(&device_links_srcu);
}
void device_links_read_unlock(int idx) __releases(&device_links_srcu)
{
srcu_read_unlock(&device_links_srcu, idx);
}
int device_links_read_lock_held(void)
{
return srcu_read_lock_held(&device_links_srcu);
}
static void device_link_synchronize_removal(void)
{
synchronize_srcu(&device_links_srcu);
}
static void device_link_remove_from_lists(struct device_link *link)
{
list_del_rcu(&link->s_node);
list_del_rcu(&link->c_node);
}
static bool device_is_ancestor(struct device *dev, struct device *target)
{
while (target->parent) {
target = target->parent;
if (dev == target)
return true;
}
return false;
}
#define DL_MARKER_FLAGS (DL_FLAG_INFERRED | \
DL_FLAG_CYCLE | \
DL_FLAG_MANAGED)
static inline bool device_link_flag_is_sync_state_only(u32 flags)
{
return (flags & ~DL_MARKER_FLAGS) == DL_FLAG_SYNC_STATE_ONLY;
}
/**
* device_is_dependent - Check if one device depends on another one
* @dev: Device to check dependencies for.
* @target: Device to check against.
*
* Check if @target depends on @dev or any device dependent on it (its child or
* its consumer etc). Return 1 if that is the case or 0 otherwise.
*/
static int device_is_dependent(struct device *dev, void *target)
{
struct device_link *link;
int ret;
/*
* The "ancestors" check is needed to catch the case when the target
* device has not been completely initialized yet and it is still
* missing from the list of children of its parent device.
*/
if (dev == target || device_is_ancestor(dev, target))
return 1;
ret = device_for_each_child(dev, target, device_is_dependent);
if (ret)
return ret;
list_for_each_entry(link, &dev->links.consumers, s_node) {
if (device_link_flag_is_sync_state_only(link->flags))
continue;
if (link->consumer == target)
return 1;
ret = device_is_dependent(link->consumer, target);
if (ret)
break;
}
return ret;
}
static void device_link_init_status(struct device_link *link,
struct device *consumer,
struct device *supplier)
{
switch (supplier->links.status) {
case DL_DEV_PROBING:
switch (consumer->links.status) {
case DL_DEV_PROBING:
/*
* A consumer driver can create a link to a supplier
* that has not completed its probing yet as long as it
* knows that the supplier is already functional (for
* example, it has just acquired some resources from the
* supplier).
*/
link->status = DL_STATE_CONSUMER_PROBE;
break;
default:
link->status = DL_STATE_DORMANT;
break;
}
break;
case DL_DEV_DRIVER_BOUND:
switch (consumer->links.status) {
case DL_DEV_PROBING:
link->status = DL_STATE_CONSUMER_PROBE;
break;
case DL_DEV_DRIVER_BOUND:
link->status = DL_STATE_ACTIVE;
break;
default:
link->status = DL_STATE_AVAILABLE;
break;
}
break;
case DL_DEV_UNBINDING:
link->status = DL_STATE_SUPPLIER_UNBIND;
break;
default:
link->status = DL_STATE_DORMANT;
break;
}
}
static int device_reorder_to_tail(struct device *dev, void *not_used)
{
struct device_link *link;
/*
* Devices that have not been registered yet will be put to the ends
* of the lists during the registration, so skip them here.
*/
if (device_is_registered(dev))
devices_kset_move_last(dev);
if (device_pm_initialized(dev))
device_pm_move_last(dev);
device_for_each_child(dev, NULL, device_reorder_to_tail);
list_for_each_entry(link, &dev->links.consumers, s_node) {
if (device_link_flag_is_sync_state_only(link->flags))
continue;
device_reorder_to_tail(link->consumer, NULL);
}
return 0;
}
/**
* device_pm_move_to_tail - Move set of devices to the end of device lists
* @dev: Device to move
*
* This is a device_reorder_to_tail() wrapper taking the requisite locks.
*
* It moves the @dev along with all of its children and all of its consumers
* to the ends of the device_kset and dpm_list, recursively.
*/
void device_pm_move_to_tail(struct device *dev)
{
int idx;
idx = device_links_read_lock();
device_pm_lock();
device_reorder_to_tail(dev, NULL);
device_pm_unlock();
device_links_read_unlock(idx);
}
#define to_devlink(dev) container_of((dev), struct device_link, link_dev)
static ssize_t status_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
const char *output;
switch (to_devlink(dev)->status) {
case DL_STATE_NONE:
output = "not tracked";
break;
case DL_STATE_DORMANT:
output = "dormant";
break;
case DL_STATE_AVAILABLE:
output = "available";
break;
case DL_STATE_CONSUMER_PROBE:
output = "consumer probing";
break;
case DL_STATE_ACTIVE:
output = "active";
break;
case DL_STATE_SUPPLIER_UNBIND:
output = "supplier unbinding";
break;
default:
output = "unknown";
break;
}
return sysfs_emit(buf, "%s\n", output);
}
static DEVICE_ATTR_RO(status);
static ssize_t auto_remove_on_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct device_link *link = to_devlink(dev);
const char *output;
if (link->flags & DL_FLAG_AUTOREMOVE_SUPPLIER)
output = "supplier unbind";
else if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER)
output = "consumer unbind";
else
output = "never";
return sysfs_emit(buf, "%s\n", output);
}
static DEVICE_ATTR_RO(auto_remove_on);
static ssize_t runtime_pm_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct device_link *link = to_devlink(dev);
return sysfs_emit(buf, "%d\n", !!(link->flags & DL_FLAG_PM_RUNTIME));
}
static DEVICE_ATTR_RO(runtime_pm);
static ssize_t sync_state_only_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct device_link *link = to_devlink(dev);
return sysfs_emit(buf, "%d\n",
!!(link->flags & DL_FLAG_SYNC_STATE_ONLY));
}
static DEVICE_ATTR_RO(sync_state_only);
static struct attribute *devlink_attrs[] = {
&dev_attr_status.attr,
&dev_attr_auto_remove_on.attr,
&dev_attr_runtime_pm.attr,
&dev_attr_sync_state_only.attr,
NULL,
};
ATTRIBUTE_GROUPS(devlink);
static void device_link_release_fn(struct work_struct *work)
{
struct device_link *link = container_of(work, struct device_link, rm_work);
/* Ensure that all references to the link object have been dropped. */
device_link_synchronize_removal();
pm_runtime_release_supplier(link);
/*
* If supplier_preactivated is set, the link has been dropped between
* the pm_runtime_get_suppliers() and pm_runtime_put_suppliers() calls
* in __driver_probe_device(). In that case, drop the supplier's
* PM-runtime usage counter to remove the reference taken by
* pm_runtime_get_suppliers().
*/
if (link->supplier_preactivated)
pm_runtime_put_noidle(link->supplier);
pm_request_idle(link->supplier);
put_device(link->consumer);
put_device(link->supplier);
kfree(link);
}
static void devlink_dev_release(struct device *dev)
{
struct device_link *link = to_devlink(dev);
INIT_WORK(&link->rm_work, device_link_release_fn);
/*
* It may take a while to complete this work because of the SRCU
* synchronization in device_link_release_fn() and if the consumer or
* supplier devices get deleted when it runs, so put it into the
* dedicated workqueue.
*/
queue_work(device_link_wq, &link->rm_work);
}
/**
* device_link_wait_removal - Wait for ongoing devlink removal jobs to terminate
*/
void device_link_wait_removal(void)
{
/*
* devlink removal jobs are queued in the dedicated work queue.
* To be sure that all removal jobs are terminated, ensure that any
* scheduled work has run to completion.
*/
flush_workqueue(device_link_wq);
}
EXPORT_SYMBOL_GPL(device_link_wait_removal);
static struct class devlink_class = {
.name = "devlink",
.dev_groups = devlink_groups,
.dev_release = devlink_dev_release,
};
static int devlink_add_symlinks(struct device *dev)
{
int ret;
size_t len;
struct device_link *link = to_devlink(dev);
struct device *sup = link->supplier;
struct device *con = link->consumer;
char *buf;
len = max(strlen(dev_bus_name(sup)) + strlen(dev_name(sup)),
strlen(dev_bus_name(con)) + strlen(dev_name(con)));
len += strlen(":");
len += strlen("supplier:") + 1;
buf = kzalloc(len, GFP_KERNEL);
if (!buf)
return -ENOMEM;
ret = sysfs_create_link(&link->link_dev.kobj, &sup->kobj, "supplier");
if (ret)
goto out;
ret = sysfs_create_link(&link->link_dev.kobj, &con->kobj, "consumer");
if (ret)
goto err_con;
snprintf(buf, len, "consumer:%s:%s", dev_bus_name(con), dev_name(con));
ret = sysfs_create_link(&sup->kobj, &link->link_dev.kobj, buf);
if (ret)
goto err_con_dev;
snprintf(buf, len, "supplier:%s:%s", dev_bus_name(sup), dev_name(sup));
ret = sysfs_create_link(&con->kobj, &link->link_dev.kobj, buf);
if (ret)
goto err_sup_dev;
goto out;
err_sup_dev:
snprintf(buf, len, "consumer:%s:%s", dev_bus_name(con), dev_name(con));
sysfs_remove_link(&sup->kobj, buf);
err_con_dev:
sysfs_remove_link(&link->link_dev.kobj, "consumer");
err_con:
sysfs_remove_link(&link->link_dev.kobj, "supplier");
out:
kfree(buf);
return ret;
}
static void devlink_remove_symlinks(struct device *dev)
{
struct device_link *link = to_devlink(dev);
size_t len;
struct device *sup = link->supplier;
struct device *con = link->consumer;
char *buf;
sysfs_remove_link(&link->link_dev.kobj, "consumer");
sysfs_remove_link(&link->link_dev.kobj, "supplier");
len = max(strlen(dev_bus_name(sup)) + strlen(dev_name(sup)),
strlen(dev_bus_name(con)) + strlen(dev_name(con)));
len += strlen(":");
len += strlen("supplier:") + 1;
buf = kzalloc(len, GFP_KERNEL);
if (!buf) {
WARN(1, "Unable to properly free device link symlinks!\n");
return;
}
if (device_is_registered(con)) {
snprintf(buf, len, "supplier:%s:%s", dev_bus_name(sup), dev_name(sup));
sysfs_remove_link(&con->kobj, buf);
}
snprintf(buf, len, "consumer:%s:%s", dev_bus_name(con), dev_name(con));
sysfs_remove_link(&sup->kobj, buf);
kfree(buf);
}
static struct class_interface devlink_class_intf = {
.class = &devlink_class,
.add_dev = devlink_add_symlinks,
.remove_dev = devlink_remove_symlinks,
};
static int __init devlink_class_init(void)
{
int ret;
ret = class_register(&devlink_class);
if (ret)
return ret;
ret = class_interface_register(&devlink_class_intf);
if (ret)
class_unregister(&devlink_class);
return ret;
}
postcore_initcall(devlink_class_init);
#define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \
DL_FLAG_AUTOREMOVE_SUPPLIER | \
DL_FLAG_AUTOPROBE_CONSUMER | \
DL_FLAG_SYNC_STATE_ONLY | \
DL_FLAG_INFERRED | \
DL_FLAG_CYCLE)
#define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \
DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)
/**
* device_link_add - Create a link between two devices.
* @consumer: Consumer end of the link.
* @supplier: Supplier end of the link.
* @flags: Link flags.
*
* The caller is responsible for the proper synchronization of the link creation
* with runtime PM. First, setting the DL_FLAG_PM_RUNTIME flag will cause the
* runtime PM framework to take the link into account. Second, if the
* DL_FLAG_RPM_ACTIVE flag is set in addition to it, the supplier devices will
* be forced into the active meta state and reference-counted upon the creation
* of the link. If DL_FLAG_PM_RUNTIME is not set, DL_FLAG_RPM_ACTIVE will be
* ignored.
*
* If DL_FLAG_STATELESS is set in @flags, the caller of this function is
* expected to release the link returned by it directly with the help of either
* device_link_del() or device_link_remove().
*
* If that flag is not set, however, the caller of this function is handing the
* management of the link over to the driver core entirely and its return value
* can only be used to check whether or not the link is present. In that case,
* the DL_FLAG_AUTOREMOVE_CONSUMER and DL_FLAG_AUTOREMOVE_SUPPLIER device link
* flags can be used to indicate to the driver core when the link can be safely
* deleted. Namely, setting one of them in @flags indicates to the driver core
* that the link is not going to be used (by the given caller of this function)
* after unbinding the consumer or supplier driver, respectively, from its
* device, so the link can be deleted at that point. If none of them is set,
* the link will be maintained until one of the devices pointed to by it (either
* the consumer or the supplier) is unregistered.
*
* Also, if DL_FLAG_STATELESS, DL_FLAG_AUTOREMOVE_CONSUMER and
* DL_FLAG_AUTOREMOVE_SUPPLIER are not set in @flags (that is, a persistent
* managed device link is being added), the DL_FLAG_AUTOPROBE_CONSUMER flag can
* be used to request the driver core to automatically probe for a consumer
* driver after successfully binding a driver to the supplier device.
*
* The combination of DL_FLAG_STATELESS and one of DL_FLAG_AUTOREMOVE_CONSUMER,
* DL_FLAG_AUTOREMOVE_SUPPLIER, or DL_FLAG_AUTOPROBE_CONSUMER set in @flags at
* the same time is invalid and will cause NULL to be returned upfront.
* However, if a device link between the given @consumer and @supplier pair
* exists already when this function is called for them, the existing link will
* be returned regardless of its current type and status (the link's flags may
* be modified then). The caller of this function is then expected to treat
* the link as though it has just been created, so (in particular) if
* DL_FLAG_STATELESS was passed in @flags, the link needs to be released
* explicitly when not needed any more (as stated above).
*
* A side effect of the link creation is re-ordering of dpm_list and the
* devices_kset list by moving the consumer device and all devices depending
* on it to the ends of these lists (that does not happen to devices that have
* not been registered when this function is called).
*
* The supplier device is required to be registered when this function is called
* and NULL will be returned if that is not the case. The consumer device need
* not be registered, however.
*/
struct device_link *device_link_add(struct device *consumer,
struct device *supplier, u32 flags)
{
struct device_link *link;
if (!consumer || !supplier || consumer == supplier ||
flags & ~DL_ADD_VALID_FLAGS ||
(flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) ||
(flags & DL_FLAG_AUTOPROBE_CONSUMER &&
flags & (DL_FLAG_AUTOREMOVE_CONSUMER |
DL_FLAG_AUTOREMOVE_SUPPLIER)))
return NULL;
if (flags & DL_FLAG_PM_RUNTIME && flags & DL_FLAG_RPM_ACTIVE) {
if (pm_runtime_get_sync(supplier) < 0) {
pm_runtime_put_noidle(supplier);
return NULL;
}
}
if (!(flags & DL_FLAG_STATELESS))
flags |= DL_FLAG_MANAGED;
if (flags & DL_FLAG_SYNC_STATE_ONLY &&
!device_link_flag_is_sync_state_only(flags))
return NULL;
device_links_write_lock();
device_pm_lock();
/*
* If the supplier has not been fully registered yet or there is a
* reverse (non-SYNC_STATE_ONLY) dependency between the consumer and
* the supplier already in the graph, return NULL. If the link is a
* SYNC_STATE_ONLY link, we don't check for reverse dependencies
* because it only affects sync_state() callbacks.
*/
if (!device_pm_initialized(supplier)
|| (!(flags & DL_FLAG_SYNC_STATE_ONLY) &&
device_is_dependent(consumer, supplier))) {
link = NULL;
goto out;
}
/*
* SYNC_STATE_ONLY links are useless once a consumer device has probed.
* So, only create it if the consumer hasn't probed yet.
*/
if (flags & DL_FLAG_SYNC_STATE_ONLY &&
consumer->links.status != DL_DEV_NO_DRIVER &&
consumer->links.status != DL_DEV_PROBING) {
link = NULL;
goto out;
}
/*
* DL_FLAG_AUTOREMOVE_SUPPLIER indicates that the link will be needed
* longer than for DL_FLAG_AUTOREMOVE_CONSUMER and setting them both
* together doesn't make sense, so prefer DL_FLAG_AUTOREMOVE_SUPPLIER.
*/
if (flags & DL_FLAG_AUTOREMOVE_SUPPLIER)
flags &= ~DL_FLAG_AUTOREMOVE_CONSUMER;
list_for_each_entry(link, &supplier->links.consumers, s_node) {
if (link->consumer != consumer)
continue;
if (link->flags & DL_FLAG_INFERRED &&
!(flags & DL_FLAG_INFERRED))
link->flags &= ~DL_FLAG_INFERRED;
if (flags & DL_FLAG_PM_RUNTIME) {
if (!(link->flags & DL_FLAG_PM_RUNTIME)) {
pm_runtime_new_link(consumer);
link->flags |= DL_FLAG_PM_RUNTIME;
}
if (flags & DL_FLAG_RPM_ACTIVE)
refcount_inc(&link->rpm_active);
}
if (flags & DL_FLAG_STATELESS) {
kref_get(&link->kref);
if (link->flags & DL_FLAG_SYNC_STATE_ONLY &&
!(link->flags & DL_FLAG_STATELESS)) {
link->flags |= DL_FLAG_STATELESS;
goto reorder;
} else {
link->flags |= DL_FLAG_STATELESS;
goto out;
}
}
/*
* If the life time of the link following from the new flags is
* longer than indicated by the flags of the existing link,
* update the existing link to stay around longer.
*/
if (flags & DL_FLAG_AUTOREMOVE_SUPPLIER) {
if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER) {
link->flags &= ~DL_FLAG_AUTOREMOVE_CONSUMER;
link->flags |= DL_FLAG_AUTOREMOVE_SUPPLIER;
}
} else if (!(flags & DL_FLAG_AUTOREMOVE_CONSUMER)) {
link->flags &= ~(DL_FLAG_AUTOREMOVE_CONSUMER |
DL_FLAG_AUTOREMOVE_SUPPLIER);
}
if (!(link->flags & DL_FLAG_MANAGED)) {
kref_get(&link->kref);
link->flags |= DL_FLAG_MANAGED;
device_link_init_status(link, consumer, supplier);
}
if (link->flags & DL_FLAG_SYNC_STATE_ONLY &&
!(flags & DL_FLAG_SYNC_STATE_ONLY)) {
link->flags &= ~DL_FLAG_SYNC_STATE_ONLY;
goto reorder;
}
goto out;
}
link = kzalloc(sizeof(*link), GFP_KERNEL);
if (!link)
goto out;
refcount_set(&link->rpm_active, 1);
get_device(supplier);
link->supplier = supplier;
INIT_LIST_HEAD(&link->s_node);
get_device(consumer);
link->consumer = consumer;
INIT_LIST_HEAD(&link->c_node);
link->flags = flags;
kref_init(&link->kref);
link->link_dev.class = &devlink_class;
device_set_pm_not_required(&link->link_dev);
dev_set_name(&link->link_dev, "%s:%s--%s:%s",
dev_bus_name(supplier), dev_name(supplier),
dev_bus_name(consumer), dev_name(consumer));
if (device_register(&link->link_dev)) {
put_device(&link->link_dev);
link = NULL;
goto out;
}
if (flags & DL_FLAG_PM_RUNTIME) {
if (flags & DL_FLAG_RPM_ACTIVE)
refcount_inc(&link->rpm_active);
pm_runtime_new_link(consumer);
}
/* Determine the initial link state. */
if (flags & DL_FLAG_STATELESS)
link->status = DL_STATE_NONE;
else
device_link_init_status(link, consumer, supplier);
/*
* Some callers expect the link creation during consumer driver probe to
* resume the supplier even without DL_FLAG_RPM_ACTIVE.
*/
if (link->status == DL_STATE_CONSUMER_PROBE &&
flags & DL_FLAG_PM_RUNTIME)
pm_runtime_resume(supplier);
list_add_tail_rcu(&link->s_node, &supplier->links.consumers);
list_add_tail_rcu(&link->c_node, &consumer->links.suppliers);
if (flags & DL_FLAG_SYNC_STATE_ONLY) {
dev_dbg(consumer,
"Linked as a sync state only consumer to %s\n",
dev_name(supplier));
goto out;
}
reorder:
/*
* Move the consumer and all of the devices depending on it to the end
* of dpm_list and the devices_kset list.
*
* It is necessary to hold dpm_list locked throughout all that or else
* we may end up suspending with a wrong ordering of it.
*/
device_reorder_to_tail(consumer, NULL);
dev_dbg(consumer, "Linked as a consumer to %s\n", dev_name(supplier));
out:
device_pm_unlock();
device_links_write_unlock();
if ((flags & DL_FLAG_PM_RUNTIME && flags & DL_FLAG_RPM_ACTIVE) && !link)
pm_runtime_put(supplier);
return link;
}
EXPORT_SYMBOL_GPL(device_link_add);
static void __device_link_del(struct kref *kref)
{
struct device_link *link = container_of(kref, struct device_link, kref);
dev_dbg(link->consumer, "Dropping the link to %s\n",
dev_name(link->supplier));
pm_runtime_drop_link(link);
device_link_remove_from_lists(link);
device_unregister(&link->link_dev);
}
static void device_link_put_kref(struct device_link *link)
{
if (link->flags & DL_FLAG_STATELESS)
kref_put(&link->kref, __device_link_del);
else if (!device_is_registered(link->consumer))
__device_link_del(&link->kref);
else
WARN(1, "Unable to drop a managed device link reference\n");
}
/**
* device_link_del - Delete a stateless link between two devices.
* @link: Device link to delete.
*
* The caller must ensure proper synchronization of this function with runtime
* PM. If the link was added multiple times, it needs to be deleted as often.
* Care is required for hotplugged devices: Their links are purged on removal
* and calling device_link_del() is then no longer allowed.
*/
void device_link_del(struct device_link *link)
{
device_links_write_lock();
device_link_put_kref(link);
device_links_write_unlock();
}
EXPORT_SYMBOL_GPL(device_link_del);
/**
* device_link_remove - Delete a stateless link between two devices.
* @consumer: Consumer end of the link.
* @supplier: Supplier end of the link.
*
* The caller must ensure proper synchronization of this function with runtime
* PM.
*/
void device_link_remove(void *consumer, struct device *supplier)
{
struct device_link *link;
if (WARN_ON(consumer == supplier))
return;
device_links_write_lock();
list_for_each_entry(link, &supplier->links.consumers, s_node) {
if (link->consumer == consumer) {
device_link_put_kref(link);
break;
}
}
device_links_write_unlock();
}
EXPORT_SYMBOL_GPL(device_link_remove);
static void device_links_missing_supplier(struct device *dev)
{
struct device_link *link;
list_for_each_entry(link, &dev->links.suppliers, c_node) { if (link->status != DL_STATE_CONSUMER_PROBE)
continue;
if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) { WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
} else {
WARN_ON(!(link->flags & DL_FLAG_SYNC_STATE_ONLY)); WRITE_ONCE(link->status, DL_STATE_DORMANT);
}
}
}
static bool dev_is_best_effort(struct device *dev)
{
return (fw_devlink_best_effort && dev->can_match) || (dev->fwnode && (dev->fwnode->flags & FWNODE_FLAG_BEST_EFFORT));
}
static struct fwnode_handle *fwnode_links_check_suppliers(
struct fwnode_handle *fwnode)
{
struct fwnode_link *link;
if (!fwnode || fw_devlink_is_permissive())
return NULL;
list_for_each_entry(link, &fwnode->suppliers, c_hook) if (!(link->flags &
(FWLINK_FLAG_CYCLE | FWLINK_FLAG_IGNORE)))
return link->supplier;
return NULL;
}
/**
* device_links_check_suppliers - Check presence of supplier drivers.
* @dev: Consumer device.
*
* Check links from this device to any suppliers. Walk the list of the device's
* links to suppliers and see if all of them are available. If not, simply
* return -EPROBE_DEFER.
*
* We need to guarantee that the supplier will not go away after the check has
* been positive here. It only can go away in __device_release_driver() and
* that function checks the device's links to consumers. This means we need to
* mark the link as "consumer probe in progress" to make the supplier removal
* wait for us to complete (or bad things may happen).
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
int device_links_check_suppliers(struct device *dev)
{
struct device_link *link;
int ret = 0, fwnode_ret = 0;
struct fwnode_handle *sup_fw;
/*
* Device waiting for supplier to become available is not allowed to
* probe.
*/
mutex_lock(&fwnode_link_lock); sup_fw = fwnode_links_check_suppliers(dev->fwnode);
if (sup_fw) {
if (!dev_is_best_effort(dev)) {
fwnode_ret = -EPROBE_DEFER;
dev_err_probe(dev, -EPROBE_DEFER,
"wait for supplier %pfwf\n", sup_fw);
} else {
fwnode_ret = -EAGAIN;
}
}
mutex_unlock(&fwnode_link_lock);
if (fwnode_ret == -EPROBE_DEFER)
return fwnode_ret; device_links_write_lock(); list_for_each_entry(link, &dev->links.suppliers, c_node) { if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->status != DL_STATE_AVAILABLE &&
!(link->flags & DL_FLAG_SYNC_STATE_ONLY)) {
if (dev_is_best_effort(dev) &&
link->flags & DL_FLAG_INFERRED &&
!link->supplier->can_match) {
ret = -EAGAIN;
continue;
}
device_links_missing_supplier(dev); dev_err_probe(dev, -EPROBE_DEFER,
"supplier %s not ready\n",
dev_name(link->supplier));
ret = -EPROBE_DEFER;
break;
}
WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE);
}
dev->links.status = DL_DEV_PROBING;
device_links_write_unlock();
return ret ? ret : fwnode_ret;
}
/**
* __device_links_queue_sync_state - Queue a device for sync_state() callback
* @dev: Device to call sync_state() on
* @list: List head to queue the @dev on
*
* Queues a device for a sync_state() callback when the device links write lock
* isn't held. This allows the sync_state() execution flow to use device links
* APIs. The caller must ensure this function is called with
* device_links_write_lock() held.
*
* This function does a get_device() to make sure the device is not freed while
* on this list.
*
* So the caller must also ensure that device_links_flush_sync_list() is called
* as soon as the caller releases device_links_write_lock(). This is necessary
* to make sure the sync_state() is called in a timely fashion and the
* put_device() is called on this device.
*/
static void __device_links_queue_sync_state(struct device *dev,
struct list_head *list)
{
struct device_link *link;
if (!dev_has_sync_state(dev))
return;
if (dev->state_synced)
return;
list_for_each_entry(link, &dev->links.consumers, s_node) { if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->status != DL_STATE_ACTIVE)
return;
}
/*
* Set the flag here to avoid adding the same device to a list more
* than once. This can happen if new consumers get added to the device
* and probed before the list is flushed.
*/
dev->state_synced = true; if (WARN_ON(!list_empty(&dev->links.defer_sync)))
return;
get_device(dev); list_add_tail(&dev->links.defer_sync, list);}
/**
* device_links_flush_sync_list - Call sync_state() on a list of devices
* @list: List of devices to call sync_state() on
* @dont_lock_dev: Device for which lock is already held by the caller
*
* Calls sync_state() on all the devices that have been queued for it. This
* function is used in conjunction with __device_links_queue_sync_state(). The
* @dont_lock_dev parameter is useful when this function is called from a
* context where a device lock is already held.
*/
static void device_links_flush_sync_list(struct list_head *list,
struct device *dont_lock_dev)
{
struct device *dev, *tmp;
list_for_each_entry_safe(dev, tmp, list, links.defer_sync) { list_del_init(&dev->links.defer_sync);
if (dev != dont_lock_dev)
device_lock(dev); dev_sync_state(dev); if (dev != dont_lock_dev) device_unlock(dev); put_device(dev);
}
}
void device_links_supplier_sync_state_pause(void)
{
device_links_write_lock();
defer_sync_state_count++;
device_links_write_unlock();
}
void device_links_supplier_sync_state_resume(void)
{
struct device *dev, *tmp;
LIST_HEAD(sync_list);
device_links_write_lock();
if (!defer_sync_state_count) {
WARN(true, "Unmatched sync_state pause/resume!");
goto out;
}
defer_sync_state_count--;
if (defer_sync_state_count)
goto out;
list_for_each_entry_safe(dev, tmp, &deferred_sync, links.defer_sync) {
/*
* Delete from deferred_sync list before queuing it to
* sync_list because defer_sync is used for both lists.
*/
list_del_init(&dev->links.defer_sync);
__device_links_queue_sync_state(dev, &sync_list);
}
out:
device_links_write_unlock();
device_links_flush_sync_list(&sync_list, NULL);
}
static int sync_state_resume_initcall(void)
{
device_links_supplier_sync_state_resume();
return 0;
}
late_initcall(sync_state_resume_initcall);
static void __device_links_supplier_defer_sync(struct device *sup)
{
if (list_empty(&sup->links.defer_sync) && dev_has_sync_state(sup))
list_add_tail(&sup->links.defer_sync, &deferred_sync);
}
static void device_link_drop_managed(struct device_link *link)
{
link->flags &= ~DL_FLAG_MANAGED;
WRITE_ONCE(link->status, DL_STATE_NONE);
kref_put(&link->kref, __device_link_del);
}
static ssize_t waiting_for_supplier_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
bool val;
device_lock(dev);
mutex_lock(&fwnode_link_lock);
val = !!fwnode_links_check_suppliers(dev->fwnode);
mutex_unlock(&fwnode_link_lock);
device_unlock(dev);
return sysfs_emit(buf, "%u\n", val);
}
static DEVICE_ATTR_RO(waiting_for_supplier);
/**
* device_links_force_bind - Prepares device to be force bound
* @dev: Consumer device.
*
* device_bind_driver() force binds a device to a driver without calling any
* driver probe functions. So the consumer really isn't going to wait for any
* supplier before it's bound to the driver. We still want the device link
* states to be sensible when this happens.
*
* In preparation for device_bind_driver(), this function goes through each
* supplier device links and checks if the supplier is bound. If it is, then
* the device link status is set to CONSUMER_PROBE. Otherwise, the device link
* is dropped. Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_force_bind(struct device *dev)
{
struct device_link *link, *ln;
device_links_write_lock();
list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) {
if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->status != DL_STATE_AVAILABLE) {
device_link_drop_managed(link);
continue;
}
WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE);
}
dev->links.status = DL_DEV_PROBING;
device_links_write_unlock();
}
/**
* device_links_driver_bound - Update device links after probing its driver.
* @dev: Device to update the links for.
*
* The probe has been successful, so update links from this device to any
* consumers by changing their status to "available".
*
* Also change the status of @dev's links to suppliers to "active".
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_driver_bound(struct device *dev)
{
struct device_link *link, *ln;
LIST_HEAD(sync_list);
/*
* If a device binds successfully, it's expected to have created all
* the device links it needs to or make new device links as it needs
* them. So, fw_devlink no longer needs to create device links to any
* of the device's suppliers.
*
* Also, if a child firmware node of this bound device is not added as a
* device by now, assume it is never going to be added. Make this bound
* device the fallback supplier to the dangling consumers of the child
* firmware node because this bound device is probably implementing the
* child firmware node functionality and we don't want the dangling
* consumers to defer probe indefinitely waiting for a device for the
* child firmware node.
*/
if (dev->fwnode && dev->fwnode->dev == dev) {
struct fwnode_handle *child;
fwnode_links_purge_suppliers(dev->fwnode);
mutex_lock(&fwnode_link_lock);
fwnode_for_each_available_child_node(dev->fwnode, child)
__fw_devlink_pickup_dangling_consumers(child,
dev->fwnode);
__fw_devlink_link_to_consumers(dev);
mutex_unlock(&fwnode_link_lock);
}
device_remove_file(dev, &dev_attr_waiting_for_supplier); device_links_write_lock(); list_for_each_entry(link, &dev->links.consumers, s_node) { if (!(link->flags & DL_FLAG_MANAGED))
continue;
/*
* Links created during consumer probe may be in the "consumer
* probe" state to start with if the supplier is still probing
* when they are created and they may become "active" if the
* consumer probe returns first. Skip them here.
*/
if (link->status == DL_STATE_CONSUMER_PROBE ||
link->status == DL_STATE_ACTIVE)
continue;
WARN_ON(link->status != DL_STATE_DORMANT); WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
if (link->flags & DL_FLAG_AUTOPROBE_CONSUMER)
driver_deferred_probe_add(link->consumer);
}
if (defer_sync_state_count) __device_links_supplier_defer_sync(dev);
else
__device_links_queue_sync_state(dev, &sync_list); list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) {
struct device *supplier;
if (!(link->flags & DL_FLAG_MANAGED))
continue;
supplier = link->supplier;
if (link->flags & DL_FLAG_SYNC_STATE_ONLY) {
/*
* When DL_FLAG_SYNC_STATE_ONLY is set, it means no
* other DL_MANAGED_LINK_FLAGS have been set. So, it's
* save to drop the managed link completely.
*/
device_link_drop_managed(link);
} else if (dev_is_best_effort(dev) &&
link->flags & DL_FLAG_INFERRED &&
link->status != DL_STATE_CONSUMER_PROBE && !link->supplier->can_match) {
/*
* When dev_is_best_effort() is true, we ignore device
* links to suppliers that don't have a driver. If the
* consumer device still managed to probe, there's no
* point in maintaining a device link in a weird state
* (consumer probed before supplier). So delete it.
*/
device_link_drop_managed(link);
} else {
WARN_ON(link->status != DL_STATE_CONSUMER_PROBE); WRITE_ONCE(link->status, DL_STATE_ACTIVE);
}
/*
* This needs to be done even for the deleted
* DL_FLAG_SYNC_STATE_ONLY device link in case it was the last
* device link that was preventing the supplier from getting a
* sync_state() call.
*/
if (defer_sync_state_count) __device_links_supplier_defer_sync(supplier);
else
__device_links_queue_sync_state(supplier, &sync_list);
}
dev->links.status = DL_DEV_DRIVER_BOUND;
device_links_write_unlock();
device_links_flush_sync_list(&sync_list, dev);
}
/**
* __device_links_no_driver - Update links of a device without a driver.
* @dev: Device without a drvier.
*
* Delete all non-persistent links from this device to any suppliers.
*
* Persistent links stay around, but their status is changed to "available",
* unless they already are in the "supplier unbind in progress" state in which
* case they need not be updated.
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
static void __device_links_no_driver(struct device *dev)
{
struct device_link *link, *ln;
list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) { if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER) { device_link_drop_managed(link);
continue;
}
if (link->status != DL_STATE_CONSUMER_PROBE &&
link->status != DL_STATE_ACTIVE)
continue;
if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) { WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
} else {
WARN_ON(!(link->flags & DL_FLAG_SYNC_STATE_ONLY)); WRITE_ONCE(link->status, DL_STATE_DORMANT);
}
}
dev->links.status = DL_DEV_NO_DRIVER;
}
/**
* device_links_no_driver - Update links after failing driver probe.
* @dev: Device whose driver has just failed to probe.
*
* Clean up leftover links to consumers for @dev and invoke
* %__device_links_no_driver() to update links to suppliers for it as
* appropriate.
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_no_driver(struct device *dev)
{
struct device_link *link;
device_links_write_lock();
list_for_each_entry(link, &dev->links.consumers, s_node) {
if (!(link->flags & DL_FLAG_MANAGED))
continue;
/*
* The probe has failed, so if the status of the link is
* "consumer probe" or "active", it must have been added by
* a probing consumer while this device was still probing.
* Change its state to "dormant", as it represents a valid
* relationship, but it is not functionally meaningful.
*/
if (link->status == DL_STATE_CONSUMER_PROBE ||
link->status == DL_STATE_ACTIVE)
WRITE_ONCE(link->status, DL_STATE_DORMANT);
}
__device_links_no_driver(dev);
device_links_write_unlock();
}
/**
* device_links_driver_cleanup - Update links after driver removal.
* @dev: Device whose driver has just gone away.
*
* Update links to consumers for @dev by changing their status to "dormant" and
* invoke %__device_links_no_driver() to update links to suppliers for it as
* appropriate.
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_driver_cleanup(struct device *dev)
{
struct device_link *link, *ln;
device_links_write_lock(); list_for_each_entry_safe(link, ln, &dev->links.consumers, s_node) { if (!(link->flags & DL_FLAG_MANAGED))
continue;
WARN_ON(link->flags & DL_FLAG_AUTOREMOVE_CONSUMER); WARN_ON(link->status != DL_STATE_SUPPLIER_UNBIND);
/*
* autoremove the links between this @dev and its consumer
* devices that are not active, i.e. where the link state
* has moved to DL_STATE_SUPPLIER_UNBIND.
*/
if (link->status == DL_STATE_SUPPLIER_UNBIND &&
link->flags & DL_FLAG_AUTOREMOVE_SUPPLIER)
device_link_drop_managed(link); WRITE_ONCE(link->status, DL_STATE_DORMANT);
}
list_del_init(&dev->links.defer_sync);
__device_links_no_driver(dev);
device_links_write_unlock();
}
/**
* device_links_busy - Check if there are any busy links to consumers.
* @dev: Device to check.
*
* Check each consumer of the device and return 'true' if its link's status
* is one of "consumer probe" or "active" (meaning that the given consumer is
* probing right now or its driver is present). Otherwise, change the link
* state to "supplier unbind" to prevent the consumer from being probed
* successfully going forward.
*
* Return 'false' if there are no probing or active consumers.
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
bool device_links_busy(struct device *dev)
{
struct device_link *link;
bool ret = false;
device_links_write_lock(); list_for_each_entry(link, &dev->links.consumers, s_node) { if (!(link->flags & DL_FLAG_MANAGED))
continue;
if (link->status == DL_STATE_CONSUMER_PROBE
|| link->status == DL_STATE_ACTIVE) {
ret = true;
break;
}
WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND);
}
dev->links.status = DL_DEV_UNBINDING;
device_links_write_unlock();
return ret;
}
/**
* device_links_unbind_consumers - Force unbind consumers of the given device.
* @dev: Device to unbind the consumers of.
*
* Walk the list of links to consumers for @dev and if any of them is in the
* "consumer probe" state, wait for all device probes in progress to complete
* and start over.
*
* If that's not the case, change the status of the link to "supplier unbind"
* and check if the link was in the "active" state. If so, force the consumer
* driver to unbind and start over (the consumer will not re-probe as we have
* changed the state of the link already).
*
* Links without the DL_FLAG_MANAGED flag set are ignored.
*/
void device_links_unbind_consumers(struct device *dev)
{
struct device_link *link;
start:
device_links_write_lock();
list_for_each_entry(link, &dev->links.consumers, s_node) {
enum device_link_state status;
if (!(link->flags & DL_FLAG_MANAGED) ||
link->flags & DL_FLAG_SYNC_STATE_ONLY)
continue;
status = link->status;
if (status == DL_STATE_CONSUMER_PROBE) {
device_links_write_unlock();
wait_for_device_probe();
goto start;
}
WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND);
if (status == DL_STATE_ACTIVE) {
struct device *consumer = link->consumer;
get_device(consumer);
device_links_write_unlock();
device_release_driver_internal(consumer, NULL,
consumer->parent);
put_device(consumer);
goto start;
}
}
device_links_write_unlock();
}
/**
* device_links_purge - Delete existing links to other devices.
* @dev: Target device.
*/
static void device_links_purge(struct device *dev)
{
struct device_link *link, *ln;
if (dev->class == &devlink_class)
return;
/*
* Delete all of the remaining links from this device to any other
* devices (either consumers or suppliers).
*/
device_links_write_lock();
list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) {
WARN_ON(link->status == DL_STATE_ACTIVE); __device_link_del(&link->kref);
}
list_for_each_entry_safe_reverse(link, ln, &dev->links.consumers, s_node) { WARN_ON(link->status != DL_STATE_DORMANT &&
link->status != DL_STATE_NONE);
__device_link_del(&link->kref);
}
device_links_write_unlock();
}
#define FW_DEVLINK_FLAGS_PERMISSIVE (DL_FLAG_INFERRED | \
DL_FLAG_SYNC_STATE_ONLY)
#define FW_DEVLINK_FLAGS_ON (DL_FLAG_INFERRED | \
DL_FLAG_AUTOPROBE_CONSUMER)
#define FW_DEVLINK_FLAGS_RPM (FW_DEVLINK_FLAGS_ON | \
DL_FLAG_PM_RUNTIME)
static u32 fw_devlink_flags = FW_DEVLINK_FLAGS_RPM;
static int __init fw_devlink_setup(char *arg)
{
if (!arg)
return -EINVAL;
if (strcmp(arg, "off") == 0) {
fw_devlink_flags = 0;
} else if (strcmp(arg, "permissive") == 0) {
fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE;
} else if (strcmp(arg, "on") == 0) {
fw_devlink_flags = FW_DEVLINK_FLAGS_ON;
} else if (strcmp(arg, "rpm") == 0) {
fw_devlink_flags = FW_DEVLINK_FLAGS_RPM;
}
return 0;
}
early_param("fw_devlink", fw_devlink_setup);
static bool fw_devlink_strict;
static int __init fw_devlink_strict_setup(char *arg)
{
return kstrtobool(arg, &fw_devlink_strict);
}
early_param("fw_devlink.strict", fw_devlink_strict_setup);
#define FW_DEVLINK_SYNC_STATE_STRICT 0
#define FW_DEVLINK_SYNC_STATE_TIMEOUT 1
#ifndef CONFIG_FW_DEVLINK_SYNC_STATE_TIMEOUT
static int fw_devlink_sync_state;
#else
static int fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT;
#endif
static int __init fw_devlink_sync_state_setup(char *arg)
{
if (!arg)
return -EINVAL;
if (strcmp(arg, "strict") == 0) {
fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_STRICT;
return 0;
} else if (strcmp(arg, "timeout") == 0) {
fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT;
return 0;
}
return -EINVAL;
}
early_param("fw_devlink.sync_state", fw_devlink_sync_state_setup);
static inline u32 fw_devlink_get_flags(u8 fwlink_flags)
{
if (fwlink_flags & FWLINK_FLAG_CYCLE)
return FW_DEVLINK_FLAGS_PERMISSIVE | DL_FLAG_CYCLE;
return fw_devlink_flags;
}
static bool fw_devlink_is_permissive(void)
{
return fw_devlink_flags == FW_DEVLINK_FLAGS_PERMISSIVE;
}
bool fw_devlink_is_strict(void)
{
return fw_devlink_strict && !fw_devlink_is_permissive();
}
static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode)
{
if (fwnode->flags & FWNODE_FLAG_LINKS_ADDED)
return;
fwnode_call_int_op(fwnode, add_links);
fwnode->flags |= FWNODE_FLAG_LINKS_ADDED;
}
static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode)
{
struct fwnode_handle *child = NULL;
fw_devlink_parse_fwnode(fwnode);
while ((child = fwnode_get_next_available_child_node(fwnode, child)))
fw_devlink_parse_fwtree(child);
}
static void fw_devlink_relax_link(struct device_link *link)
{
if (!(link->flags & DL_FLAG_INFERRED))
return;
if (device_link_flag_is_sync_state_only(link->flags))
return;
pm_runtime_drop_link(link);
link->flags = DL_FLAG_MANAGED | FW_DEVLINK_FLAGS_PERMISSIVE;
dev_dbg(link->consumer, "Relaxing link with %s\n",
dev_name(link->supplier));
}
static int fw_devlink_no_driver(struct device *dev, void *data)
{
struct device_link *link = to_devlink(dev);
if (!link->supplier->can_match)
fw_devlink_relax_link(link);
return 0;
}
void fw_devlink_drivers_done(void)
{
fw_devlink_drv_reg_done = true;
device_links_write_lock();
class_for_each_device(&devlink_class, NULL, NULL,
fw_devlink_no_driver);
device_links_write_unlock();
}
static int fw_devlink_dev_sync_state(struct device *dev, void *data)
{
struct device_link *link = to_devlink(dev);
struct device *sup = link->supplier;
if (!(link->flags & DL_FLAG_MANAGED) ||
link->status == DL_STATE_ACTIVE || sup->state_synced ||
!dev_has_sync_state(sup))
return 0;
if (fw_devlink_sync_state == FW_DEVLINK_SYNC_STATE_STRICT) {
dev_warn(sup, "sync_state() pending due to %s\n",
dev_name(link->consumer));
return 0;
}
if (!list_empty(&sup->links.defer_sync))
return 0;
dev_warn(sup, "Timed out. Forcing sync_state()\n");
sup->state_synced = true;
get_device(sup);
list_add_tail(&sup->links.defer_sync, data);
return 0;
}
void fw_devlink_probing_done(void)
{
LIST_HEAD(sync_list);
device_links_write_lock();
class_for_each_device(&devlink_class, NULL, &sync_list,
fw_devlink_dev_sync_state);
device_links_write_unlock();
device_links_flush_sync_list(&sync_list, NULL);
}
/**
* wait_for_init_devices_probe - Try to probe any device needed for init
*
* Some devices might need to be probed and bound successfully before the kernel
* boot sequence can finish and move on to init/userspace. For example, a
* network interface might need to be bound to be able to mount a NFS rootfs.
*
* With fw_devlink=on by default, some of these devices might be blocked from
* probing because they are waiting on a optional supplier that doesn't have a
* driver. While fw_devlink will eventually identify such devices and unblock
* the probing automatically, it might be too late by the time it unblocks the
* probing of devices. For example, the IP4 autoconfig might timeout before
* fw_devlink unblocks probing of the network interface.
*
* This function is available to temporarily try and probe all devices that have
* a driver even if some of their suppliers haven't been added or don't have
* drivers.
*
* The drivers can then decide which of the suppliers are optional vs mandatory
* and probe the device if possible. By the time this function returns, all such
* "best effort" probes are guaranteed to be completed. If a device successfully
* probes in this mode, we delete all fw_devlink discovered dependencies of that
* device where the supplier hasn't yet probed successfully because they have to
* be optional dependencies.
*
* Any devices that didn't successfully probe go back to being treated as if
* this function was never called.
*
* This also means that some devices that aren't needed for init and could have
* waited for their optional supplier to probe (when the supplier's module is
* loaded later on) would end up probing prematurely with limited functionality.
* So call this function only when boot would fail without it.
*/
void __init wait_for_init_devices_probe(void)
{
if (!fw_devlink_flags || fw_devlink_is_permissive())
return;
/*
* Wait for all ongoing probes to finish so that the "best effort" is
* only applied to devices that can't probe otherwise.
*/
wait_for_device_probe();
pr_info("Trying to probe devices needed for running init ...\n");
fw_devlink_best_effort = true;
driver_deferred_probe_trigger();
/*
* Wait for all "best effort" probes to finish before going back to
* normal enforcement.
*/
wait_for_device_probe();
fw_devlink_best_effort = false;
}
static void fw_devlink_unblock_consumers(struct device *dev)
{
struct device_link *link;
if (!fw_devlink_flags || fw_devlink_is_permissive())
return;
device_links_write_lock(); list_for_each_entry(link, &dev->links.consumers, s_node) fw_devlink_relax_link(link); device_links_write_unlock();
}
#define get_dev_from_fwnode(fwnode) get_device((fwnode)->dev)
static bool fwnode_init_without_drv(struct fwnode_handle *fwnode)
{
struct device *dev;
bool ret;
if (!(fwnode->flags & FWNODE_FLAG_INITIALIZED))
return false;
dev = get_dev_from_fwnode(fwnode);
ret = !dev || dev->links.status == DL_DEV_NO_DRIVER;
put_device(dev);
return ret;
}
static bool fwnode_ancestor_init_without_drv(struct fwnode_handle *fwnode)
{
struct fwnode_handle *parent;
fwnode_for_each_parent_node(fwnode, parent) {
if (fwnode_init_without_drv(parent)) {
fwnode_handle_put(parent);
return true;
}
}
return false;
}
/**
* fwnode_is_ancestor_of - Test if @ancestor is ancestor of @child
* @ancestor: Firmware which is tested for being an ancestor
* @child: Firmware which is tested for being the child
*
* A node is considered an ancestor of itself too.
*
* Return: true if @ancestor is an ancestor of @child. Otherwise, returns false.
*/
static bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor,
const struct fwnode_handle *child)
{
struct fwnode_handle *parent;
if (IS_ERR_OR_NULL(ancestor))
return false;
if (child == ancestor)
return true;
fwnode_for_each_parent_node(child, parent) {
if (parent == ancestor) {
fwnode_handle_put(parent);
return true;
}
}
return false;
}
/**
* fwnode_get_next_parent_dev - Find device of closest ancestor fwnode
* @fwnode: firmware node
*
* Given a firmware node (@fwnode), this function finds its closest ancestor
* firmware node that has a corresponding struct device and returns that struct
* device.
*
* The caller is responsible for calling put_device() on the returned device
* pointer.
*
* Return: a pointer to the device of the @fwnode's closest ancestor.
*/
static struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode)
{
struct fwnode_handle *parent;
struct device *dev;
fwnode_for_each_parent_node(fwnode, parent) {
dev = get_dev_from_fwnode(parent);
if (dev) {
fwnode_handle_put(parent);
return dev;
}
}
return NULL;
}
/**
* __fw_devlink_relax_cycles - Relax and mark dependency cycles.
* @con: Potential consumer device.
* @sup_handle: Potential supplier's fwnode.
*
* Needs to be called with fwnode_lock and device link lock held.
*
* Check if @sup_handle or any of its ancestors or suppliers direct/indirectly
* depend on @con. This function can detect multiple cyles between @sup_handle
* and @con. When such dependency cycles are found, convert all device links
* created solely by fw_devlink into SYNC_STATE_ONLY device links. Also, mark
* all fwnode links in the cycle with FWLINK_FLAG_CYCLE so that when they are
* converted into a device link in the future, they are created as
* SYNC_STATE_ONLY device links. This is the equivalent of doing
* fw_devlink=permissive just between the devices in the cycle. We need to do
* this because, at this point, fw_devlink can't tell which of these
* dependencies is not a real dependency.
*
* Return true if one or more cycles were found. Otherwise, return false.
*/
static bool __fw_devlink_relax_cycles(struct device *con,
struct fwnode_handle *sup_handle)
{
struct device *sup_dev = NULL, *par_dev = NULL;
struct fwnode_link *link;
struct device_link *dev_link;
bool ret = false;
if (!sup_handle)
return false;
/*
* We aren't trying to find all cycles. Just a cycle between con and
* sup_handle.
*/
if (sup_handle->flags & FWNODE_FLAG_VISITED)
return false;
sup_handle->flags |= FWNODE_FLAG_VISITED;
sup_dev = get_dev_from_fwnode(sup_handle);
/* Termination condition. */
if (sup_dev == con) {
pr_debug("----- cycle: start -----\n");
ret = true;
goto out;
}
/*
* If sup_dev is bound to a driver and @con hasn't started binding to a
* driver, sup_dev can't be a consumer of @con. So, no need to check
* further.
*/
if (sup_dev && sup_dev->links.status == DL_DEV_DRIVER_BOUND &&
con->links.status == DL_DEV_NO_DRIVER) {
ret = false;
goto out;
}
list_for_each_entry(link, &sup_handle->suppliers, c_hook) {
if (link->flags & FWLINK_FLAG_IGNORE)
continue;
if (__fw_devlink_relax_cycles(con, link->supplier)) {
__fwnode_link_cycle(link);
ret = true;
}
}
/*
* Give priority to device parent over fwnode parent to account for any
* quirks in how fwnodes are converted to devices.
*/
if (sup_dev)
par_dev = get_device(sup_dev->parent);
else
par_dev = fwnode_get_next_parent_dev(sup_handle);
if (par_dev && __fw_devlink_relax_cycles(con, par_dev->fwnode)) {
pr_debug("%pfwf: cycle: child of %pfwf\n", sup_handle,
par_dev->fwnode);
ret = true;
}
if (!sup_dev)
goto out;
list_for_each_entry(dev_link, &sup_dev->links.suppliers, c_node) {
/*
* Ignore a SYNC_STATE_ONLY flag only if it wasn't marked as
* such due to a cycle.
*/
if (device_link_flag_is_sync_state_only(dev_link->flags) &&
!(dev_link->flags & DL_FLAG_CYCLE))
continue;
if (__fw_devlink_relax_cycles(con,
dev_link->supplier->fwnode)) {
pr_debug("%pfwf: cycle: depends on %pfwf\n", sup_handle,
dev_link->supplier->fwnode);
fw_devlink_relax_link(dev_link);
dev_link->flags |= DL_FLAG_CYCLE;
ret = true;
}
}
out:
sup_handle->flags &= ~FWNODE_FLAG_VISITED;
put_device(sup_dev);
put_device(par_dev);
return ret;
}
/**
* fw_devlink_create_devlink - Create a device link from a consumer to fwnode
* @con: consumer device for the device link
* @sup_handle: fwnode handle of supplier
* @link: fwnode link that's being converted to a device link
*
* This function will try to create a device link between the consumer device
* @con and the supplier device represented by @sup_handle.
*
* The supplier has to be provided as a fwnode because incorrect cycles in
* fwnode links can sometimes cause the supplier device to never be created.
* This function detects such cases and returns an error if it cannot create a
* device link from the consumer to a missing supplier.
*
* Returns,
* 0 on successfully creating a device link
* -EINVAL if the device link cannot be created as expected
* -EAGAIN if the device link cannot be created right now, but it may be
* possible to do that in the future
*/
static int fw_devlink_create_devlink(struct device *con,
struct fwnode_handle *sup_handle,
struct fwnode_link *link)
{
struct device *sup_dev;
int ret = 0;
u32 flags;
if (link->flags & FWLINK_FLAG_IGNORE)
return 0;
if (con->fwnode == link->consumer)
flags = fw_devlink_get_flags(link->flags);
else
flags = FW_DEVLINK_FLAGS_PERMISSIVE;
/*
* In some cases, a device P might also be a supplier to its child node
* C. However, this would defer the probe of C until the probe of P
* completes successfully. This is perfectly fine in the device driver
* model. device_add() doesn't guarantee probe completion of the device
* by the time it returns.
*
* However, there are a few drivers that assume C will finish probing
* as soon as it's added and before P finishes probing. So, we provide
* a flag to let fw_devlink know not to delay the probe of C until the
* probe of P completes successfully.
*
* When such a flag is set, we can't create device links where P is the
* supplier of C as that would delay the probe of C.
*/
if (sup_handle->flags & FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD &&
fwnode_is_ancestor_of(sup_handle, con->fwnode))
return -EINVAL;
/*
* SYNC_STATE_ONLY device links don't block probing and supports cycles.
* So, one might expect that cycle detection isn't necessary for them.
* However, if the device link was marked as SYNC_STATE_ONLY because
* it's part of a cycle, then we still need to do cycle detection. This
* is because the consumer and supplier might be part of multiple cycles
* and we need to detect all those cycles.
*/
if (!device_link_flag_is_sync_state_only(flags) ||
flags & DL_FLAG_CYCLE) {
device_links_write_lock();
if (__fw_devlink_relax_cycles(con, sup_handle)) {
__fwnode_link_cycle(link);
flags = fw_devlink_get_flags(link->flags);
pr_debug("----- cycle: end -----\n");
dev_info(con, "Fixed dependency cycle(s) with %pfwf\n",
sup_handle);
}
device_links_write_unlock();
}
if (sup_handle->flags & FWNODE_FLAG_NOT_DEVICE)
sup_dev = fwnode_get_next_parent_dev(sup_handle);
else
sup_dev = get_dev_from_fwnode(sup_handle);
if (sup_dev) {
/*
* If it's one of those drivers that don't actually bind to
* their device using driver core, then don't wait on this
* supplier device indefinitely.
*/
if (sup_dev->links.status == DL_DEV_NO_DRIVER &&
sup_handle->flags & FWNODE_FLAG_INITIALIZED) {
dev_dbg(con,
"Not linking %pfwf - dev might never probe\n",
sup_handle);
ret = -EINVAL;
goto out;
}
if (con != sup_dev && !device_link_add(con, sup_dev, flags)) {
dev_err(con, "Failed to create device link (0x%x) with %s\n",
flags, dev_name(sup_dev));
ret = -EINVAL;
}
goto out;
}
/*
* Supplier or supplier's ancestor already initialized without a struct
* device or being probed by a driver.
*/
if (fwnode_init_without_drv(sup_handle) ||
fwnode_ancestor_init_without_drv(sup_handle)) {
dev_dbg(con, "Not linking %pfwf - might never become dev\n",
sup_handle);
return -EINVAL;
}
ret = -EAGAIN;
out:
put_device(sup_dev);
return ret;
}
/**
* __fw_devlink_link_to_consumers - Create device links to consumers of a device
* @dev: Device that needs to be linked to its consumers
*
* This function looks at all the consumer fwnodes of @dev and creates device
* links between the consumer device and @dev (supplier).
*
* If the consumer device has not been added yet, then this function creates a
* SYNC_STATE_ONLY link between @dev (supplier) and the closest ancestor device
* of the consumer fwnode. This is necessary to make sure @dev doesn't get a
* sync_state() callback before the real consumer device gets to be added and
* then probed.
*
* Once device links are created from the real consumer to @dev (supplier), the
* fwnode links are deleted.
*/
static void __fw_devlink_link_to_consumers(struct device *dev)
{
struct fwnode_handle *fwnode = dev->fwnode;
struct fwnode_link *link, *tmp;
list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) {
struct device *con_dev;
bool own_link = true;
int ret;
con_dev = get_dev_from_fwnode(link->consumer);
/*
* If consumer device is not available yet, make a "proxy"
* SYNC_STATE_ONLY link from the consumer's parent device to
* the supplier device. This is necessary to make sure the
* supplier doesn't get a sync_state() callback before the real
* consumer can create a device link to the supplier.
*
* This proxy link step is needed to handle the case where the
* consumer's parent device is added before the supplier.
*/
if (!con_dev) {
con_dev = fwnode_get_next_parent_dev(link->consumer);
/*
* However, if the consumer's parent device is also the
* parent of the supplier, don't create a
* consumer-supplier link from the parent to its child
* device. Such a dependency is impossible.
*/
if (con_dev &&
fwnode_is_ancestor_of(con_dev->fwnode, fwnode)) {
put_device(con_dev);
con_dev = NULL;
} else {
own_link = false;
}
}
if (!con_dev)
continue;
ret = fw_devlink_create_devlink(con_dev, fwnode, link);
put_device(con_dev);
if (!own_link || ret == -EAGAIN)
continue;
__fwnode_link_del(link);
}
}
/**
* __fw_devlink_link_to_suppliers - Create device links to suppliers of a device
* @dev: The consumer device that needs to be linked to its suppliers
* @fwnode: Root of the fwnode tree that is used to create device links
*
* This function looks at all the supplier fwnodes of fwnode tree rooted at
* @fwnode and creates device links between @dev (consumer) and all the
* supplier devices of the entire fwnode tree at @fwnode.
*
* The function creates normal (non-SYNC_STATE_ONLY) device links between @dev
* and the real suppliers of @dev. Once these device links are created, the
* fwnode links are deleted.
*
* In addition, it also looks at all the suppliers of the entire fwnode tree
* because some of the child devices of @dev that have not been added yet
* (because @dev hasn't probed) might already have their suppliers added to
* driver core. So, this function creates SYNC_STATE_ONLY device links between
* @dev (consumer) and these suppliers to make sure they don't execute their
* sync_state() callbacks before these child devices have a chance to create
* their device links. The fwnode links that correspond to the child devices
* aren't delete because they are needed later to create the device links
* between the real consumer and supplier devices.
*/
static void __fw_devlink_link_to_suppliers(struct device *dev,
struct fwnode_handle *fwnode)
{
bool own_link = (dev->fwnode == fwnode);
struct fwnode_link *link, *tmp;
struct fwnode_handle *child = NULL;
list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) {
int ret;
struct fwnode_handle *sup = link->supplier;
ret = fw_devlink_create_devlink(dev, sup, link);
if (!own_link || ret == -EAGAIN)
continue;
__fwnode_link_del(link);
}
/*
* Make "proxy" SYNC_STATE_ONLY device links to represent the needs of
* all the descendants. This proxy link step is needed to handle the
* case where the supplier is added before the consumer's parent device
* (@dev).
*/
while ((child = fwnode_get_next_available_child_node(fwnode, child)))
__fw_devlink_link_to_suppliers(dev, child);
}
static void fw_devlink_link_device(struct device *dev)
{
struct fwnode_handle *fwnode = dev->fwnode;
if (!fw_devlink_flags)
return;
fw_devlink_parse_fwtree(fwnode);
mutex_lock(&fwnode_link_lock);
__fw_devlink_link_to_consumers(dev);
__fw_devlink_link_to_suppliers(dev, fwnode);
mutex_unlock(&fwnode_link_lock);
}
/* Device links support end. */
int (*platform_notify)(struct device *dev) = NULL;
int (*platform_notify_remove)(struct device *dev) = NULL;
static struct kobject *dev_kobj;
/* /sys/dev/char */
static struct kobject *sysfs_dev_char_kobj;
/* /sys/dev/block */
static struct kobject *sysfs_dev_block_kobj;
static DEFINE_MUTEX(device_hotplug_lock);
void lock_device_hotplug(void)
{
mutex_lock(&device_hotplug_lock);
}
void unlock_device_hotplug(void)
{
mutex_unlock(&device_hotplug_lock);
}
int lock_device_hotplug_sysfs(void)
{
if (mutex_trylock(&device_hotplug_lock))
return 0;
/* Avoid busy looping (5 ms of sleep should do). */
msleep(5);
return restart_syscall();
}
#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{
return !(dev->type == &part_type);
}
#else
static inline int device_is_not_partition(struct device *dev)
{
return 1;
}
#endif
static void device_platform_notify(struct device *dev)
{
acpi_device_notify(dev);
software_node_notify(dev);
if (platform_notify)
platform_notify(dev);
}
static void device_platform_notify_remove(struct device *dev)
{
if (platform_notify_remove) platform_notify_remove(dev); software_node_notify_remove(dev);
acpi_device_notify_remove(dev);
}
/**
* dev_driver_string - Return a device's driver name, if at all possible
* @dev: struct device to get the name of
*
* Will return the device's driver's name if it is bound to a device. If
* the device is not bound to a driver, it will return the name of the bus
* it is attached to. If it is not attached to a bus either, an empty
* string will be returned.
*/
const char *dev_driver_string(const struct device *dev)
{
struct device_driver *drv;
/* dev->driver can change to NULL underneath us because of unbinding,
* so be careful about accessing it. dev->bus and dev->class should
* never change once they are set, so they don't need special care.
*/
drv = READ_ONCE(dev->driver); return drv ? drv->name : dev_bus_name(dev);
}
EXPORT_SYMBOL(dev_driver_string);
#define to_dev_attr(_attr) container_of(_attr, struct device_attribute, attr)
static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct device_attribute *dev_attr = to_dev_attr(attr);
struct device *dev = kobj_to_dev(kobj);
ssize_t ret = -EIO;
if (dev_attr->show)
ret = dev_attr->show(dev, dev_attr, buf);
if (ret >= (ssize_t)PAGE_SIZE) {
printk("dev_attr_show: %pS returned bad count\n",
dev_attr->show);
}
return ret;
}
static ssize_t dev_attr_store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t count)
{
struct device_attribute *dev_attr = to_dev_attr(attr);
struct device *dev = kobj_to_dev(kobj);
ssize_t ret = -EIO;
if (dev_attr->store)
ret = dev_attr->store(dev, dev_attr, buf, count);
return ret;
}
static const struct sysfs_ops dev_sysfs_ops = {
.show = dev_attr_show,
.store = dev_attr_store,
};
#define to_ext_attr(x) container_of(x, struct dev_ext_attribute, attr)
ssize_t device_store_ulong(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t size)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
int ret;
unsigned long new;
ret = kstrtoul(buf, 0, &new);
if (ret)
return ret;
*(unsigned long *)(ea->var) = new;
/* Always return full write size even if we didn't consume all */
return size;
}
EXPORT_SYMBOL_GPL(device_store_ulong);
ssize_t device_show_ulong(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%lx\n", *(unsigned long *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_ulong);
ssize_t device_store_int(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t size)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
int ret;
long new;
ret = kstrtol(buf, 0, &new);
if (ret)
return ret;
if (new > INT_MAX || new < INT_MIN)
return -EINVAL;
*(int *)(ea->var) = new;
/* Always return full write size even if we didn't consume all */
return size;
}
EXPORT_SYMBOL_GPL(device_store_int);
ssize_t device_show_int(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%d\n", *(int *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_int);
ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
const char *buf, size_t size)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
if (kstrtobool(buf, ea->var) < 0)
return -EINVAL;
return size;
}
EXPORT_SYMBOL_GPL(device_store_bool);
ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct dev_ext_attribute *ea = to_ext_attr(attr);
return sysfs_emit(buf, "%d\n", *(bool *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_bool);
/**
* device_release - free device structure.
* @kobj: device's kobject.
*
* This is called once the reference count for the object
* reaches 0. We forward the call to the device's release
* method, which should handle actually freeing the structure.
*/
static void device_release(struct kobject *kobj)
{
struct device *dev = kobj_to_dev(kobj);
struct device_private *p = dev->p;
/*
* Some platform devices are driven without driver attached
* and managed resources may have been acquired. Make sure
* all resources are released.
*
* Drivers still can add resources into device after device
* is deleted but alive, so release devres here to avoid
* possible memory leak.
*/
devres_release_all(dev);
kfree(dev->dma_range_map);
if (dev->release)
dev->release(dev);
else if (dev->type && dev->type->release)
dev->type->release(dev);
else if (dev->class && dev->class->dev_release) dev->class->dev_release(dev);
else
WARN(1, KERN_ERR "Device '%s' does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n",
dev_name(dev));
kfree(p);
}
static const void *device_namespace(const struct kobject *kobj)
{
const struct device *dev = kobj_to_dev(kobj);
const void *ns = NULL;
if (dev->class && dev->class->ns_type)
ns = dev->class->namespace(dev);
return ns;
}
static void device_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid)
{
const struct device *dev = kobj_to_dev(kobj);
if (dev->class && dev->class->get_ownership) dev->class->get_ownership(dev, uid, gid);
}
static const struct kobj_type device_ktype = {
.release = device_release,
.sysfs_ops = &dev_sysfs_ops,
.namespace = device_namespace,
.get_ownership = device_get_ownership,
};
static int dev_uevent_filter(const struct kobject *kobj)
{
const struct kobj_type *ktype = get_ktype(kobj); if (ktype == &device_ktype) {
const struct device *dev = kobj_to_dev(kobj);
if (dev->bus)
return 1;
if (dev->class)
return 1;
}
return 0;
}
static const char *dev_uevent_name(const struct kobject *kobj)
{
const struct device *dev = kobj_to_dev(kobj);
if (dev->bus) return dev->bus->name; if (dev->class) return dev->class->name;
return NULL;
}
static int dev_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)
{
const struct device *dev = kobj_to_dev(kobj);
int retval = 0;
/* add device node properties if present */
if (MAJOR(dev->devt)) { const char *tmp;
const char *name;
umode_t mode = 0;
kuid_t uid = GLOBAL_ROOT_UID;
kgid_t gid = GLOBAL_ROOT_GID;
add_uevent_var(env, "MAJOR=%u", MAJOR(dev->devt));
add_uevent_var(env, "MINOR=%u", MINOR(dev->devt));
name = device_get_devnode(dev, &mode, &uid, &gid, &tmp);
if (name) {
add_uevent_var(env, "DEVNAME=%s", name);
if (mode)
add_uevent_var(env, "DEVMODE=%#o", mode & 0777); if (!uid_eq(uid, GLOBAL_ROOT_UID)) add_uevent_var(env, "DEVUID=%u", from_kuid(&init_user_ns, uid)); if (!gid_eq(gid, GLOBAL_ROOT_GID)) add_uevent_var(env, "DEVGID=%u", from_kgid(&init_user_ns, gid)); kfree(tmp);
}
}
if (dev->type && dev->type->name) add_uevent_var(env, "DEVTYPE=%s", dev->type->name); if (dev->driver) add_uevent_var(env, "DRIVER=%s", dev->driver->name);
/* Add common DT information about the device */
of_device_uevent(dev, env);
/* have the bus specific function add its stuff */
if (dev->bus && dev->bus->uevent) { retval = dev->bus->uevent(dev, env);
if (retval)
pr_debug("device: '%s': %s: bus uevent() returned %d\n",
dev_name(dev), __func__, retval);
}
/* have the class specific function add its stuff */
if (dev->class && dev->class->dev_uevent) { retval = dev->class->dev_uevent(dev, env);
if (retval)
pr_debug("device: '%s': %s: class uevent() "
"returned %d\n", dev_name(dev),
__func__, retval);
}
/* have the device type specific function add its stuff */
if (dev->type && dev->type->uevent) { retval = dev->type->uevent(dev, env);
if (retval)
pr_debug("device: '%s': %s: dev_type uevent() "
"returned %d\n", dev_name(dev),
__func__, retval);
}
return retval;
}
static const struct kset_uevent_ops device_uevent_ops = {
.filter = dev_uevent_filter,
.name = dev_uevent_name,
.uevent = dev_uevent,
};
static ssize_t uevent_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct kobject *top_kobj;
struct kset *kset;
struct kobj_uevent_env *env = NULL;
int i;
int len = 0;
int retval;
/* search the kset, the device belongs to */
top_kobj = &dev->kobj;
while (!top_kobj->kset && top_kobj->parent)
top_kobj = top_kobj->parent;
if (!top_kobj->kset)
goto out;
kset = top_kobj->kset;
if (!kset->uevent_ops || !kset->uevent_ops->uevent)
goto out;
/* respect filter */
if (kset->uevent_ops && kset->uevent_ops->filter)
if (!kset->uevent_ops->filter(&dev->kobj))
goto out;
env = kzalloc(sizeof(struct kobj_uevent_env), GFP_KERNEL);
if (!env)
return -ENOMEM;
/* let the kset specific function add its keys */
retval = kset->uevent_ops->uevent(&dev->kobj, env);
if (retval)
goto out;
/* copy keys to file */
for (i = 0; i < env->envp_idx; i++)
len += sysfs_emit_at(buf, len, "%s\n", env->envp[i]);
out:
kfree(env);
return len;
}
static ssize_t uevent_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
int rc;
rc = kobject_synth_uevent(&dev->kobj, buf, count);
if (rc) {
dev_err(dev, "uevent: failed to send synthetic uevent: %d\n", rc);
return rc;
}
return count;
}
static DEVICE_ATTR_RW(uevent);
static ssize_t online_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
bool val;
device_lock(dev);
val = !dev->offline;
device_unlock(dev);
return sysfs_emit(buf, "%u\n", val);
}
static ssize_t online_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
bool val;
int ret;
ret = kstrtobool(buf, &val);
if (ret < 0)
return ret;
ret = lock_device_hotplug_sysfs();
if (ret)
return ret;
ret = val ? device_online(dev) : device_offline(dev);
unlock_device_hotplug();
return ret < 0 ? ret : count;
}
static DEVICE_ATTR_RW(online);
static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
const char *loc;
switch (dev->removable) {
case DEVICE_REMOVABLE:
loc = "removable";
break;
case DEVICE_FIXED:
loc = "fixed";
break;
default:
loc = "unknown";
}
return sysfs_emit(buf, "%s\n", loc);
}
static DEVICE_ATTR_RO(removable);
int device_add_groups(struct device *dev, const struct attribute_group **groups)
{
return sysfs_create_groups(&dev->kobj, groups);
}
EXPORT_SYMBOL_GPL(device_add_groups);
void device_remove_groups(struct device *dev,
const struct attribute_group **groups)
{
sysfs_remove_groups(&dev->kobj, groups);
}
EXPORT_SYMBOL_GPL(device_remove_groups);
union device_attr_group_devres {
const struct attribute_group *group;
const struct attribute_group **groups;
};
static void devm_attr_group_remove(struct device *dev, void *res)
{
union device_attr_group_devres *devres = res;
const struct attribute_group *group = devres->group;
dev_dbg(dev, "%s: removing group %p\n", __func__, group);
sysfs_remove_group(&dev->kobj, group);
}
static void devm_attr_groups_remove(struct device *dev, void *res)
{
union device_attr_group_devres *devres = res;
const struct attribute_group **groups = devres->groups;
dev_dbg(dev, "%s: removing groups %p\n", __func__, groups);
sysfs_remove_groups(&dev->kobj, groups);
}
/**
* devm_device_add_group - given a device, create a managed attribute group
* @dev: The device to create the group for
* @grp: The attribute group to create
*
* This function creates a group for the first time. It will explicitly
* warn and error if any of the attribute files being created already exist.
*
* Returns 0 on success or error code on failure.
*/
int devm_device_add_group(struct device *dev, const struct attribute_group *grp)
{
union device_attr_group_devres *devres;
int error;
devres = devres_alloc(devm_attr_group_remove,
sizeof(*devres), GFP_KERNEL);
if (!devres)
return -ENOMEM;
error = sysfs_create_group(&dev->kobj, grp);
if (error) {
devres_free(devres);
return error;
}
devres->group = grp;
devres_add(dev, devres);
return 0;
}
EXPORT_SYMBOL_GPL(devm_device_add_group);
/**
* devm_device_add_groups - create a bunch of managed attribute groups
* @dev: The device to create the group for
* @groups: The attribute groups to create, NULL terminated
*
* This function creates a bunch of managed attribute groups. If an error
* occurs when creating a group, all previously created groups will be
* removed, unwinding everything back to the original state when this
* function was called. It will explicitly warn and error if any of the
* attribute files being created already exist.
*
* Returns 0 on success or error code from sysfs_create_group on failure.
*/
int devm_device_add_groups(struct device *dev,
const struct attribute_group **groups)
{
union device_attr_group_devres *devres;
int error;
devres = devres_alloc(devm_attr_groups_remove,
sizeof(*devres), GFP_KERNEL);
if (!devres)
return -ENOMEM;
error = sysfs_create_groups(&dev->kobj, groups);
if (error) {
devres_free(devres);
return error;
}
devres->groups = groups;
devres_add(dev, devres);
return 0;
}
EXPORT_SYMBOL_GPL(devm_device_add_groups);
static int device_add_attrs(struct device *dev)
{
const struct class *class = dev->class;
const struct device_type *type = dev->type;
int error;
if (class) {
error = device_add_groups(dev, class->dev_groups);
if (error)
return error;
}
if (type) { error = device_add_groups(dev, type->groups);
if (error)
goto err_remove_class_groups;
}
error = device_add_groups(dev, dev->groups);
if (error)
goto err_remove_type_groups;
if (device_supports_offline(dev) && !dev->offline_disabled) { error = device_create_file(dev, &dev_attr_online);
if (error)
goto err_remove_dev_groups;
}
if (fw_devlink_flags && !fw_devlink_is_permissive() && dev->fwnode) { error = device_create_file(dev, &dev_attr_waiting_for_supplier);
if (error)
goto err_remove_dev_online;
}
if (dev_removable_is_valid(dev)) { error = device_create_file(dev, &dev_attr_removable);
if (error)
goto err_remove_dev_waiting_for_supplier;
}
if (dev_add_physical_location(dev)) { error = device_add_group(dev,
&dev_attr_physical_location_group);
if (error)
goto err_remove_dev_removable;
}
return 0;
err_remove_dev_removable:
device_remove_file(dev, &dev_attr_removable);
err_remove_dev_waiting_for_supplier:
device_remove_file(dev, &dev_attr_waiting_for_supplier);
err_remove_dev_online:
device_remove_file(dev, &dev_attr_online);
err_remove_dev_groups:
device_remove_groups(dev, dev->groups);
err_remove_type_groups:
if (type)
device_remove_groups(dev, type->groups);
err_remove_class_groups:
if (class) device_remove_groups(dev, class->dev_groups);
return error;
}
static void device_remove_attrs(struct device *dev)
{
const struct class *class = dev->class;
const struct device_type *type = dev->type;
if (dev->physical_location) {
device_remove_group(dev, &dev_attr_physical_location_group);
kfree(dev->physical_location);
}
device_remove_file(dev, &dev_attr_removable);
device_remove_file(dev, &dev_attr_waiting_for_supplier);
device_remove_file(dev, &dev_attr_online);
device_remove_groups(dev, dev->groups);
if (type)
device_remove_groups(dev, type->groups); if (class) device_remove_groups(dev, class->dev_groups);}
static ssize_t dev_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
return print_dev_t(buf, dev->devt);
}
static DEVICE_ATTR_RO(dev);
/* /sys/devices/ */
struct kset *devices_kset;
/**
* devices_kset_move_before - Move device in the devices_kset's list.
* @deva: Device to move.
* @devb: Device @deva should come before.
*/
static void devices_kset_move_before(struct device *deva, struct device *devb)
{
if (!devices_kset)
return;
pr_debug("devices_kset: Moving %s before %s\n",
dev_name(deva), dev_name(devb));
spin_lock(&devices_kset->list_lock);
list_move_tail(&deva->kobj.entry, &devb->kobj.entry);
spin_unlock(&devices_kset->list_lock);
}
/**
* devices_kset_move_after - Move device in the devices_kset's list.
* @deva: Device to move
* @devb: Device @deva should come after.
*/
static void devices_kset_move_after(struct device *deva, struct device *devb)
{
if (!devices_kset)
return;
pr_debug("devices_kset: Moving %s after %s\n",
dev_name(deva), dev_name(devb));
spin_lock(&devices_kset->list_lock);
list_move(&deva->kobj.entry, &devb->kobj.entry);
spin_unlock(&devices_kset->list_lock);
}
/**
* devices_kset_move_last - move the device to the end of devices_kset's list.
* @dev: device to move
*/
void devices_kset_move_last(struct device *dev)
{
if (!devices_kset)
return;
pr_debug("devices_kset: Moving %s to end of list\n", dev_name(dev));
spin_lock(&devices_kset->list_lock);
list_move_tail(&dev->kobj.entry, &devices_kset->list);
spin_unlock(&devices_kset->list_lock);
}
/**
* device_create_file - create sysfs attribute file for device.
* @dev: device.
* @attr: device attribute descriptor.
*/
int device_create_file(struct device *dev,
const struct device_attribute *attr)
{
int error = 0;
if (dev) { WARN(((attr->attr.mode & S_IWUGO) && !attr->store),
"Attribute %s: write permission without 'store'\n",
attr->attr.name);
WARN(((attr->attr.mode & S_IRUGO) && !attr->show),
"Attribute %s: read permission without 'show'\n",
attr->attr.name);
error = sysfs_create_file(&dev->kobj, &attr->attr);
}
return error;
}
EXPORT_SYMBOL_GPL(device_create_file);
/**
* device_remove_file - remove sysfs attribute file.
* @dev: device.
* @attr: device attribute descriptor.
*/
void device_remove_file(struct device *dev,
const struct device_attribute *attr)
{
if (dev) sysfs_remove_file(&dev->kobj, &attr->attr);
}
EXPORT_SYMBOL_GPL(device_remove_file);
/**
* device_remove_file_self - remove sysfs attribute file from its own method.
* @dev: device.
* @attr: device attribute descriptor.
*
* See kernfs_remove_self() for details.
*/
bool device_remove_file_self(struct device *dev,
const struct device_attribute *attr)
{
if (dev)
return sysfs_remove_file_self(&dev->kobj, &attr->attr);
else
return false;
}
EXPORT_SYMBOL_GPL(device_remove_file_self);
/**
* device_create_bin_file - create sysfs binary attribute file for device.
* @dev: device.
* @attr: device binary attribute descriptor.
*/
int device_create_bin_file(struct device *dev,
const struct bin_attribute *attr)
{
int error = -EINVAL;
if (dev)
error = sysfs_create_bin_file(&dev->kobj, attr);
return error;
}
EXPORT_SYMBOL_GPL(device_create_bin_file);
/**
* device_remove_bin_file - remove sysfs binary attribute file
* @dev: device.
* @attr: device binary attribute descriptor.
*/
void device_remove_bin_file(struct device *dev,
const struct bin_attribute *attr)
{
if (dev)
sysfs_remove_bin_file(&dev->kobj, attr);
}
EXPORT_SYMBOL_GPL(device_remove_bin_file);
static void klist_children_get(struct klist_node *n)
{
struct device_private *p = to_device_private_parent(n);
struct device *dev = p->device;
get_device(dev);
}
static void klist_children_put(struct klist_node *n)
{
struct device_private *p = to_device_private_parent(n);
struct device *dev = p->device; put_device(dev);
}
/**
* device_initialize - init device structure.
* @dev: device.
*
* This prepares the device for use by other layers by initializing
* its fields.
* It is the first half of device_register(), if called by
* that function, though it can also be called separately, so one
* may use @dev's fields. In particular, get_device()/put_device()
* may be used for reference counting of @dev after calling this
* function.
*
* All fields in @dev must be initialized by the caller to 0, except
* for those explicitly set to some other value. The simplest
* approach is to use kzalloc() to allocate the structure containing
* @dev.
*
* NOTE: Use put_device() to give up your reference instead of freeing
* @dev directly once you have called this function.
*/
void device_initialize(struct device *dev)
{
dev->kobj.kset = devices_kset;
kobject_init(&dev->kobj, &device_ktype);
INIT_LIST_HEAD(&dev->dma_pools);
mutex_init(&dev->mutex);
lockdep_set_novalidate_class(&dev->mutex);
spin_lock_init(&dev->devres_lock);
INIT_LIST_HEAD(&dev->devres_head);
device_pm_init(dev);
set_dev_node(dev, NUMA_NO_NODE);
INIT_LIST_HEAD(&dev->links.consumers);
INIT_LIST_HEAD(&dev->links.suppliers);
INIT_LIST_HEAD(&dev->links.defer_sync);
dev->links.status = DL_DEV_NO_DRIVER;
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
dev->dma_coherent = dma_default_coherent;
#endif
swiotlb_dev_init(dev);
}
EXPORT_SYMBOL_GPL(device_initialize);
struct kobject *virtual_device_parent(struct device *dev)
{
static struct kobject *virtual_dir = NULL;
if (!virtual_dir)
virtual_dir = kobject_create_and_add("virtual",
&devices_kset->kobj);
return virtual_dir;
}
struct class_dir {
struct kobject kobj;
const struct class *class;
};
#define to_class_dir(obj) container_of(obj, struct class_dir, kobj)
static void class_dir_release(struct kobject *kobj)
{
struct class_dir *dir = to_class_dir(kobj);
kfree(dir);
}
static const
struct kobj_ns_type_operations *class_dir_child_ns_type(const struct kobject *kobj)
{
const struct class_dir *dir = to_class_dir(kobj);
return dir->class->ns_type;
}
static const struct kobj_type class_dir_ktype = {
.release = class_dir_release,
.sysfs_ops = &kobj_sysfs_ops,
.child_ns_type = class_dir_child_ns_type
};
static struct kobject *class_dir_create_and_add(struct subsys_private *sp,
struct kobject *parent_kobj)
{
struct class_dir *dir;
int retval;
dir = kzalloc(sizeof(*dir), GFP_KERNEL);
if (!dir)
return ERR_PTR(-ENOMEM);
dir->class = sp->class;
kobject_init(&dir->kobj, &class_dir_ktype);
dir->kobj.kset = &sp->glue_dirs;
retval = kobject_add(&dir->kobj, parent_kobj, "%s", sp->class->name);
if (retval < 0) {
kobject_put(&dir->kobj);
return ERR_PTR(retval);
}
return &dir->kobj;
}
static DEFINE_MUTEX(gdp_mutex);
static struct kobject *get_device_parent(struct device *dev,
struct device *parent)
{
struct subsys_private *sp = class_to_subsys(dev->class);
struct kobject *kobj = NULL;
if (sp) {
struct kobject *parent_kobj;
struct kobject *k;
/*
* If we have no parent, we live in "virtual".
* Class-devices with a non class-device as parent, live
* in a "glue" directory to prevent namespace collisions.
*/
if (parent == NULL)
parent_kobj = virtual_device_parent(dev); else if (parent->class && !dev->class->ns_type) { subsys_put(sp);
return &parent->kobj;
} else {
parent_kobj = &parent->kobj;
}
mutex_lock(&gdp_mutex);
/* find our class-directory at the parent and reference it */
spin_lock(&sp->glue_dirs.list_lock);
list_for_each_entry(k, &sp->glue_dirs.list, entry) if (k->parent == parent_kobj) { kobj = kobject_get(k);
break;
}
spin_unlock(&sp->glue_dirs.list_lock);
if (kobj) {
mutex_unlock(&gdp_mutex);
subsys_put(sp);
return kobj;
}
/* or create a new class-directory at the parent device */
k = class_dir_create_and_add(sp, parent_kobj);
/* do not emit an uevent for this simple "glue" directory */
mutex_unlock(&gdp_mutex);
subsys_put(sp);
return k;
}
/* subsystems can specify a default root directory for their devices */
if (!parent && dev->bus) { struct device *dev_root = bus_get_dev_root(dev->bus);
if (dev_root) {
kobj = &dev_root->kobj;
put_device(dev_root);
return kobj;
}
}
if (parent)
return &parent->kobj;
return NULL;
}
static inline bool live_in_glue_dir(struct kobject *kobj,
struct device *dev)
{
struct subsys_private *sp;
bool retval;
if (!kobj || !dev->class)
return false;
sp = class_to_subsys(dev->class);
if (!sp)
return false;
if (kobj->kset == &sp->glue_dirs)
retval = true;
else
retval = false;
subsys_put(sp);
return retval;
}
static inline struct kobject *get_glue_dir(struct device *dev)
{
return dev->kobj.parent;
}
/**
* kobject_has_children - Returns whether a kobject has children.
* @kobj: the object to test
*
* This will return whether a kobject has other kobjects as children.
*
* It does NOT account for the presence of attribute files, only sub
* directories. It also assumes there is no concurrent addition or
* removal of such children, and thus relies on external locking.
*/
static inline bool kobject_has_children(struct kobject *kobj)
{
WARN_ON_ONCE(kref_read(&kobj->kref) == 0); return kobj->sd && kobj->sd->dir.subdirs;
}
/*
* make sure cleaning up dir as the last step, we need to make
* sure .release handler of kobject is run with holding the
* global lock
*/
static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir)
{
unsigned int ref;
/* see if we live in a "glue" directory */
if (!live_in_glue_dir(glue_dir, dev))
return;
mutex_lock(&gdp_mutex);
/**
* There is a race condition between removing glue directory
* and adding a new device under the glue directory.
*
* CPU1: CPU2:
*
* device_add()
* get_device_parent()
* class_dir_create_and_add()
* kobject_add_internal()
* create_dir() // create glue_dir
*
* device_add()
* get_device_parent()
* kobject_get() // get glue_dir
*
* device_del()
* cleanup_glue_dir()
* kobject_del(glue_dir)
*
* kobject_add()
* kobject_add_internal()
* create_dir() // in glue_dir
* sysfs_create_dir_ns()
* kernfs_create_dir_ns(sd)
*
* sysfs_remove_dir() // glue_dir->sd=NULL
* sysfs_put() // free glue_dir->sd
*
* // sd is freed
* kernfs_new_node(sd)
* kernfs_get(glue_dir)
* kernfs_add_one()
* kernfs_put()
*
* Before CPU1 remove last child device under glue dir, if CPU2 add
* a new device under glue dir, the glue_dir kobject reference count
* will be increase to 2 in kobject_get(k). And CPU2 has been called
* kernfs_create_dir_ns(). Meanwhile, CPU1 call sysfs_remove_dir()
* and sysfs_put(). This result in glue_dir->sd is freed.
*
* Then the CPU2 will see a stale "empty" but still potentially used
* glue dir around in kernfs_new_node().
*
* In order to avoid this happening, we also should make sure that
* kernfs_node for glue_dir is released in CPU1 only when refcount
* for glue_dir kobj is 1.
*/
ref = kref_read(&glue_dir->kref);
if (!kobject_has_children(glue_dir) && !--ref) kobject_del(glue_dir); kobject_put(glue_dir);
mutex_unlock(&gdp_mutex);
}
static int device_add_class_symlinks(struct device *dev)
{
struct device_node *of_node = dev_of_node(dev);
struct subsys_private *sp;
int error;
if (of_node) {
error = sysfs_create_link(&dev->kobj, of_node_kobj(of_node), "of_node");
if (error)
dev_warn(dev, "Error %d creating of_node link\n",error);
/* An error here doesn't warrant bringing down the device */
}
sp = class_to_subsys(dev->class);
if (!sp)
return 0;
error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem");
if (error)
goto out_devnode;
if (dev->parent && device_is_not_partition(dev)) { error = sysfs_create_link(&dev->kobj, &dev->parent->kobj,
"device");
if (error)
goto out_subsys;
}
/* link in the class directory pointing to the device */
error = sysfs_create_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev));
if (error)
goto out_device;
goto exit;
out_device:
sysfs_remove_link(&dev->kobj, "device");
out_subsys:
sysfs_remove_link(&dev->kobj, "subsystem");
out_devnode:
sysfs_remove_link(&dev->kobj, "of_node");
exit:
subsys_put(sp);
return error;
}
static void device_remove_class_symlinks(struct device *dev)
{
struct subsys_private *sp = class_to_subsys(dev->class); if (dev_of_node(dev)) sysfs_remove_link(&dev->kobj, "of_node"); if (!sp)
return;
if (dev->parent && device_is_not_partition(dev)) sysfs_remove_link(&dev->kobj, "device"); sysfs_remove_link(&dev->kobj, "subsystem"); sysfs_delete_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev));
subsys_put(sp);
}
/**
* dev_set_name - set a device name
* @dev: device
* @fmt: format string for the device's name
*/
int dev_set_name(struct device *dev, const char *fmt, ...)
{
va_list vargs;
int err;
va_start(vargs, fmt);
err = kobject_set_name_vargs(&dev->kobj, fmt, vargs);
va_end(vargs);
return err;
}
EXPORT_SYMBOL_GPL(dev_set_name);
/* select a /sys/dev/ directory for the device */
static struct kobject *device_to_dev_kobj(struct device *dev)
{
if (is_blockdev(dev)) return sysfs_dev_block_kobj;
else
return sysfs_dev_char_kobj;
}
static int device_create_sys_dev_entry(struct device *dev)
{
struct kobject *kobj = device_to_dev_kobj(dev);
int error = 0;
char devt_str[15];
if (kobj) {
format_dev_t(devt_str, dev->devt);
error = sysfs_create_link(kobj, &dev->kobj, devt_str);
}
return error;
}
static void device_remove_sys_dev_entry(struct device *dev)
{
struct kobject *kobj = device_to_dev_kobj(dev); char devt_str[15];
if (kobj) {
format_dev_t(devt_str, dev->devt); sysfs_remove_link(kobj, devt_str);
}
}
static int device_private_init(struct device *dev)
{
dev->p = kzalloc(sizeof(*dev->p), GFP_KERNEL);
if (!dev->p)
return -ENOMEM;
dev->p->device = dev;
klist_init(&dev->p->klist_children, klist_children_get,
klist_children_put);
INIT_LIST_HEAD(&dev->p->deferred_probe);
return 0;
}
/**
* device_add - add device to device hierarchy.
* @dev: device.
*
* This is part 2 of device_register(), though may be called
* separately _iff_ device_initialize() has been called separately.
*
* This adds @dev to the kobject hierarchy via kobject_add(), adds it
* to the global and sibling lists for the device, then
* adds it to the other relevant subsystems of the driver model.
*
* Do not call this routine or device_register() more than once for
* any device structure. The driver model core is not designed to work
* with devices that get unregistered and then spring back to life.
* (Among other things, it's very hard to guarantee that all references
* to the previous incarnation of @dev have been dropped.) Allocate
* and register a fresh new struct device instead.
*
* NOTE: _Never_ directly free @dev after calling this function, even
* if it returned an error! Always use put_device() to give up your
* reference instead.
*
* Rule of thumb is: if device_add() succeeds, you should call
* device_del() when you want to get rid of it. If device_add() has
* *not* succeeded, use *only* put_device() to drop the reference
* count.
*/
int device_add(struct device *dev)
{
struct subsys_private *sp;
struct device *parent;
struct kobject *kobj;
struct class_interface *class_intf;
int error = -EINVAL;
struct kobject *glue_dir = NULL;
dev = get_device(dev);
if (!dev)
goto done;
if (!dev->p) { error = device_private_init(dev);
if (error)
goto done;
}
/*
* for statically allocated devices, which should all be converted
* some day, we need to initialize the name. We prevent reading back
* the name, and force the use of dev_name()
*/
if (dev->init_name) { error = dev_set_name(dev, "%s", dev->init_name);
dev->init_name = NULL;
}
if (dev_name(dev))
error = 0;
/* subsystems can specify simple device enumeration */
else if (dev->bus && dev->bus->dev_name) error = dev_set_name(dev, "%s%u", dev->bus->dev_name, dev->id);
else
error = -EINVAL;
if (error)
goto name_error;
pr_debug("device: '%s': %s\n", dev_name(dev), __func__); parent = get_device(dev->parent); kobj = get_device_parent(dev, parent);
if (IS_ERR(kobj)) {
error = PTR_ERR(kobj);
goto parent_error;
}
if (kobj) dev->kobj.parent = kobj;
/* use parent numa_node */
if (parent && (dev_to_node(dev) == NUMA_NO_NODE)) set_dev_node(dev, dev_to_node(parent));
/* first, register with generic layer. */
/* we require the name to be set before, and pass NULL */
error = kobject_add(&dev->kobj, dev->kobj.parent, NULL);
if (error) {
glue_dir = kobj;
goto Error;
}
/* notify platform of device entry */
device_platform_notify(dev); error = device_create_file(dev, &dev_attr_uevent);
if (error)
goto attrError;
error = device_add_class_symlinks(dev);
if (error)
goto SymlinkError;
error = device_add_attrs(dev);
if (error)
goto AttrsError;
error = bus_add_device(dev);
if (error)
goto BusError;
error = dpm_sysfs_add(dev);
if (error)
goto DPMError;
device_pm_add(dev);
if (MAJOR(dev->devt)) {
error = device_create_file(dev, &dev_attr_dev);
if (error)
goto DevAttrError;
error = device_create_sys_dev_entry(dev);
if (error)
goto SysEntryError;
devtmpfs_create_node(dev);
}
/* Notify clients of device addition. This call must come
* after dpm_sysfs_add() and before kobject_uevent().
*/
bus_notify(dev, BUS_NOTIFY_ADD_DEVICE);
kobject_uevent(&dev->kobj, KOBJ_ADD);
/*
* Check if any of the other devices (consumers) have been waiting for
* this device (supplier) to be added so that they can create a device
* link to it.
*
* This needs to happen after device_pm_add() because device_link_add()
* requires the supplier be registered before it's called.
*
* But this also needs to happen before bus_probe_device() to make sure
* waiting consumers can link to it before the driver is bound to the
* device and the driver sync_state callback is called for this device.
*/
if (dev->fwnode && !dev->fwnode->dev) { dev->fwnode->dev = dev; fw_devlink_link_device(dev);
}
bus_probe_device(dev);
/*
* If all driver registration is done and a newly added device doesn't
* match with any driver, don't block its consumers from probing in
* case the consumer device is able to operate without this supplier.
*/
if (dev->fwnode && fw_devlink_drv_reg_done && !dev->can_match) fw_devlink_unblock_consumers(dev); if (parent)
klist_add_tail(&dev->p->knode_parent,
&parent->p->klist_children); sp = class_to_subsys(dev->class);
if (sp) {
mutex_lock(&sp->mutex);
/* tie the class to the device */
klist_add_tail(&dev->p->knode_class, &sp->klist_devices);
/* notify any interfaces that the device is here */
list_for_each_entry(class_intf, &sp->interfaces, node) if (class_intf->add_dev) class_intf->add_dev(dev); mutex_unlock(&sp->mutex);
subsys_put(sp);
}
done:
put_device(dev); return error;
SysEntryError:
if (MAJOR(dev->devt)) device_remove_file(dev, &dev_attr_dev);
DevAttrError:
device_pm_remove(dev);
dpm_sysfs_remove(dev);
DPMError:
dev->driver = NULL;
bus_remove_device(dev);
BusError:
device_remove_attrs(dev);
AttrsError:
device_remove_class_symlinks(dev);
SymlinkError:
device_remove_file(dev, &dev_attr_uevent);
attrError:
device_platform_notify_remove(dev);
kobject_uevent(&dev->kobj, KOBJ_REMOVE);
glue_dir = get_glue_dir(dev);
kobject_del(&dev->kobj);
Error:
cleanup_glue_dir(dev, glue_dir);
parent_error:
put_device(parent);
name_error:
kfree(dev->p);
dev->p = NULL;
goto done;
}
EXPORT_SYMBOL_GPL(device_add);
/**
* device_register - register a device with the system.
* @dev: pointer to the device structure
*
* This happens in two clean steps - initialize the device
* and add it to the system. The two steps can be called
* separately, but this is the easiest and most common.
* I.e. you should only call the two helpers separately if
* have a clearly defined need to use and refcount the device
* before it is added to the hierarchy.
*
* For more information, see the kerneldoc for device_initialize()
* and device_add().
*
* NOTE: _Never_ directly free @dev after calling this function, even
* if it returned an error! Always use put_device() to give up the
* reference initialized in this function instead.
*/
int device_register(struct device *dev)
{
device_initialize(dev);
return device_add(dev);
}
EXPORT_SYMBOL_GPL(device_register);
/**
* get_device - increment reference count for device.
* @dev: device.
*
* This simply forwards the call to kobject_get(), though
* we do take care to provide for the case that we get a NULL
* pointer passed in.
*/
struct device *get_device(struct device *dev)
{
return dev ? kobj_to_dev(kobject_get(&dev->kobj)) : NULL;
}
EXPORT_SYMBOL_GPL(get_device);
/**
* put_device - decrement reference count.
* @dev: device in question.
*/
void put_device(struct device *dev)
{
/* might_sleep(); */
if (dev) kobject_put(&dev->kobj);
}
EXPORT_SYMBOL_GPL(put_device);
bool kill_device(struct device *dev)
{
/*
* Require the device lock and set the "dead" flag to guarantee that
* the update behavior is consistent with the other bitfields near
* it and that we cannot have an asynchronous probe routine trying
* to run while we are tearing out the bus/class/sysfs from
* underneath the device.
*/
device_lock_assert(dev); if (dev->p->dead)
return false;
dev->p->dead = true; return true;
}
EXPORT_SYMBOL_GPL(kill_device);
/**
* device_del - delete device from system.
* @dev: device.
*
* This is the first part of the device unregistration
* sequence. This removes the device from the lists we control
* from here, has it removed from the other driver model
* subsystems it was added to in device_add(), and removes it
* from the kobject hierarchy.
*
* NOTE: this should be called manually _iff_ device_add() was
* also called manually.
*/
void device_del(struct device *dev)
{
struct subsys_private *sp;
struct device *parent = dev->parent;
struct kobject *glue_dir = NULL;
struct class_interface *class_intf;
unsigned int noio_flag;
device_lock(dev);
kill_device(dev);
device_unlock(dev);
if (dev->fwnode && dev->fwnode->dev == dev) dev->fwnode->dev = NULL;
/* Notify clients of device removal. This call must come
* before dpm_sysfs_remove().
*/
noio_flag = memalloc_noio_save();
bus_notify(dev, BUS_NOTIFY_DEL_DEVICE);
dpm_sysfs_remove(dev);
if (parent)
klist_del(&dev->p->knode_parent); if (MAJOR(dev->devt)) { devtmpfs_delete_node(dev); device_remove_sys_dev_entry(dev); device_remove_file(dev, &dev_attr_dev);
}
sp = class_to_subsys(dev->class);
if (sp) {
device_remove_class_symlinks(dev);
mutex_lock(&sp->mutex);
/* notify any interfaces that the device is now gone */
list_for_each_entry(class_intf, &sp->interfaces, node) if (class_intf->remove_dev) class_intf->remove_dev(dev);
/* remove the device from the class list */
klist_del(&dev->p->knode_class);
mutex_unlock(&sp->mutex);
subsys_put(sp);
}
device_remove_file(dev, &dev_attr_uevent); device_remove_attrs(dev);
bus_remove_device(dev);
device_pm_remove(dev);
driver_deferred_probe_del(dev);
device_platform_notify_remove(dev); device_links_purge(dev);
/*
* If a device does not have a driver attached, we need to clean
* up any managed resources. We do this in device_release(), but
* it's never called (and we leak the device) if a managed
* resource holds a reference to the device. So release all
* managed resources here, like we do in driver_detach(). We
* still need to do so again in device_release() in case someone
* adds a new resource after this point, though.
*/
devres_release_all(dev);
bus_notify(dev, BUS_NOTIFY_REMOVED_DEVICE);
kobject_uevent(&dev->kobj, KOBJ_REMOVE);
glue_dir = get_glue_dir(dev);
kobject_del(&dev->kobj);
cleanup_glue_dir(dev, glue_dir);
memalloc_noio_restore(noio_flag);
put_device(parent);}
EXPORT_SYMBOL_GPL(device_del);
/**
* device_unregister - unregister device from system.
* @dev: device going away.
*
* We do this in two parts, like we do device_register(). First,
* we remove it from all the subsystems with device_del(), then
* we decrement the reference count via put_device(). If that
* is the final reference count, the device will be cleaned up
* via device_release() above. Otherwise, the structure will
* stick around until the final reference to the device is dropped.
*/
void device_unregister(struct device *dev)
{
pr_debug("device: '%s': %s\n", dev_name(dev), __func__); device_del(dev); put_device(dev);
}
EXPORT_SYMBOL_GPL(device_unregister);
static struct device *prev_device(struct klist_iter *i)
{
struct klist_node *n = klist_prev(i);
struct device *dev = NULL;
struct device_private *p;
if (n) {
p = to_device_private_parent(n);
dev = p->device;
}
return dev;
}
static struct device *next_device(struct klist_iter *i)
{
struct klist_node *n = klist_next(i);
struct device *dev = NULL;
struct device_private *p;
if (n) {
p = to_device_private_parent(n);
dev = p->device;
}
return dev;
}
/**
* device_get_devnode - path of device node file
* @dev: device
* @mode: returned file access mode
* @uid: returned file owner
* @gid: returned file group
* @tmp: possibly allocated string
*
* Return the relative path of a possible device node.
* Non-default names may need to allocate a memory to compose
* a name. This memory is returned in tmp and needs to be
* freed by the caller.
*/
const char *device_get_devnode(const struct device *dev,
umode_t *mode, kuid_t *uid, kgid_t *gid,
const char **tmp)
{
char *s;
*tmp = NULL;
/* the device type may provide a specific name */
if (dev->type && dev->type->devnode) *tmp = dev->type->devnode(dev, mode, uid, gid); if (*tmp)
return *tmp;
/* the class may provide a specific name */
if (dev->class && dev->class->devnode) *tmp = dev->class->devnode(dev, mode);
if (*tmp)
return *tmp;
/* return name without allocation, tmp == NULL */
if (strchr(dev_name(dev), '!') == NULL)
return dev_name(dev);
/* replace '!' in the name with '/' */
s = kstrdup_and_replace(dev_name(dev), '!', '/', GFP_KERNEL);
if (!s)
return NULL;
return *tmp = s;
}
/**
* device_for_each_child - device child iterator.
* @parent: parent struct device.
* @fn: function to be called for each device.
* @data: data for the callback.
*
* Iterate over @parent's child devices, and call @fn for each,
* passing it @data.
*
* We check the return of @fn each time. If it returns anything
* other than 0, we break out and return that value.
*/
int device_for_each_child(struct device *parent, void *data,
int (*fn)(struct device *dev, void *data))
{
struct klist_iter i;
struct device *child;
int error = 0;
if (!parent->p)
return 0;
klist_iter_init(&parent->p->klist_children, &i); while (!error && (child = next_device(&i))) error = fn(child, data); klist_iter_exit(&i); return error;
}
EXPORT_SYMBOL_GPL(device_for_each_child);
/**
* device_for_each_child_reverse - device child iterator in reversed order.
* @parent: parent struct device.
* @fn: function to be called for each device.
* @data: data for the callback.
*
* Iterate over @parent's child devices, and call @fn for each,
* passing it @data.
*
* We check the return of @fn each time. If it returns anything
* other than 0, we break out and return that value.
*/
int device_for_each_child_reverse(struct device *parent, void *data,
int (*fn)(struct device *dev, void *data))
{
struct klist_iter i;
struct device *child;
int error = 0;
if (!parent->p)
return 0;
klist_iter_init(&parent->p->klist_children, &i); while ((child = prev_device(&i)) && !error) error = fn(child, data); klist_iter_exit(&i); return error;
}
EXPORT_SYMBOL_GPL(device_for_each_child_reverse);
/**
* device_find_child - device iterator for locating a particular device.
* @parent: parent struct device
* @match: Callback function to check device
* @data: Data to pass to match function
*
* This is similar to the device_for_each_child() function above, but it
* returns a reference to a device that is 'found' for later use, as
* determined by the @match callback.
*
* The callback should return 0 if the device doesn't match and non-zero
* if it does. If the callback returns non-zero and a reference to the
* current device can be obtained, this function will return to the caller
* and not iterate over any more devices.
*
* NOTE: you will need to drop the reference with put_device() after use.
*/
struct device *device_find_child(struct device *parent, void *data,
int (*match)(struct device *dev, void *data))
{
struct klist_iter i;
struct device *child;
if (!parent)
return NULL;
klist_iter_init(&parent->p->klist_children, &i);
while ((child = next_device(&i)))
if (match(child, data) && get_device(child))
break;
klist_iter_exit(&i);
return child;
}
EXPORT_SYMBOL_GPL(device_find_child);
/**
* device_find_child_by_name - device iterator for locating a child device.
* @parent: parent struct device
* @name: name of the child device
*
* This is similar to the device_find_child() function above, but it
* returns a reference to a device that has the name @name.
*
* NOTE: you will need to drop the reference with put_device() after use.
*/
struct device *device_find_child_by_name(struct device *parent,
const char *name)
{
struct klist_iter i;
struct device *child;
if (!parent)
return NULL;
klist_iter_init(&parent->p->klist_children, &i);
while ((child = next_device(&i)))
if (sysfs_streq(dev_name(child), name) && get_device(child))
break;
klist_iter_exit(&i);
return child;
}
EXPORT_SYMBOL_GPL(device_find_child_by_name);
static int match_any(struct device *dev, void *unused)
{
return 1;
}
/**
* device_find_any_child - device iterator for locating a child device, if any.
* @parent: parent struct device
*
* This is similar to the device_find_child() function above, but it
* returns a reference to a child device, if any.
*
* NOTE: you will need to drop the reference with put_device() after use.
*/
struct device *device_find_any_child(struct device *parent)
{
return device_find_child(parent, NULL, match_any);
}
EXPORT_SYMBOL_GPL(device_find_any_child);
int __init devices_init(void)
{
devices_kset = kset_create_and_add("devices", &device_uevent_ops, NULL);
if (!devices_kset)
return -ENOMEM;
dev_kobj = kobject_create_and_add("dev", NULL);
if (!dev_kobj)
goto dev_kobj_err;
sysfs_dev_block_kobj = kobject_create_and_add("block", dev_kobj);
if (!sysfs_dev_block_kobj)
goto block_kobj_err;
sysfs_dev_char_kobj = kobject_create_and_add("char", dev_kobj);
if (!sysfs_dev_char_kobj)
goto char_kobj_err;
device_link_wq = alloc_workqueue("device_link_wq", 0, 0);
if (!device_link_wq)
goto wq_err;
return 0;
wq_err:
kobject_put(sysfs_dev_char_kobj);
char_kobj_err:
kobject_put(sysfs_dev_block_kobj);
block_kobj_err:
kobject_put(dev_kobj);
dev_kobj_err:
kset_unregister(devices_kset);
return -ENOMEM;
}
static int device_check_offline(struct device *dev, void *not_used)
{
int ret;
ret = device_for_each_child(dev, NULL, device_check_offline);
if (ret)
return ret;
return device_supports_offline(dev) && !dev->offline ? -EBUSY : 0;
}
/**
* device_offline - Prepare the device for hot-removal.
* @dev: Device to be put offline.
*
* Execute the device bus type's .offline() callback, if present, to prepare
* the device for a subsequent hot-removal. If that succeeds, the device must
* not be used until either it is removed or its bus type's .online() callback
* is executed.
*
* Call under device_hotplug_lock.
*/
int device_offline(struct device *dev)
{
int ret;
if (dev->offline_disabled)
return -EPERM;
ret = device_for_each_child(dev, NULL, device_check_offline);
if (ret)
return ret;
device_lock(dev);
if (device_supports_offline(dev)) {
if (dev->offline) {
ret = 1;
} else {
ret = dev->bus->offline(dev);
if (!ret) {
kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
dev->offline = true;
}
}
}
device_unlock(dev);
return ret;
}
/**
* device_online - Put the device back online after successful device_offline().
* @dev: Device to be put back online.
*
* If device_offline() has been successfully executed for @dev, but the device
* has not been removed subsequently, execute its bus type's .online() callback
* to indicate that the device can be used again.
*
* Call under device_hotplug_lock.
*/
int device_online(struct device *dev)
{
int ret = 0;
device_lock(dev);
if (device_supports_offline(dev)) {
if (dev->offline) {
ret = dev->bus->online(dev);
if (!ret) {
kobject_uevent(&dev->kobj, KOBJ_ONLINE);
dev->offline = false;
}
} else {
ret = 1;
}
}
device_unlock(dev);
return ret;
}
struct root_device {
struct device dev;
struct module *owner;
};
static inline struct root_device *to_root_device(struct device *d)
{
return container_of(d, struct root_device, dev);
}
static void root_device_release(struct device *dev)
{
kfree(to_root_device(dev));
}
/**
* __root_device_register - allocate and register a root device
* @name: root device name
* @owner: owner module of the root device, usually THIS_MODULE
*
* This function allocates a root device and registers it
* using device_register(). In order to free the returned
* device, use root_device_unregister().
*
* Root devices are dummy devices which allow other devices
* to be grouped under /sys/devices. Use this function to
* allocate a root device and then use it as the parent of
* any device which should appear under /sys/devices/{name}
*
* The /sys/devices/{name} directory will also contain a
* 'module' symlink which points to the @owner directory
* in sysfs.
*
* Returns &struct device pointer on success, or ERR_PTR() on error.
*
* Note: You probably want to use root_device_register().
*/
struct device *__root_device_register(const char *name, struct module *owner)
{
struct root_device *root;
int err = -ENOMEM;
root = kzalloc(sizeof(struct root_device), GFP_KERNEL);
if (!root)
return ERR_PTR(err);
err = dev_set_name(&root->dev, "%s", name);
if (err) {
kfree(root);
return ERR_PTR(err);
}
root->dev.release = root_device_release;
err = device_register(&root->dev);
if (err) {
put_device(&root->dev);
return ERR_PTR(err);
}
#ifdef CONFIG_MODULES /* gotta find a "cleaner" way to do this */
if (owner) {
struct module_kobject *mk = &owner->mkobj;
err = sysfs_create_link(&root->dev.kobj, &mk->kobj, "module");
if (err) {
device_unregister(&root->dev);
return ERR_PTR(err);
}
root->owner = owner;
}
#endif
return &root->dev;
}
EXPORT_SYMBOL_GPL(__root_device_register);
/**
* root_device_unregister - unregister and free a root device
* @dev: device going away
*
* This function unregisters and cleans up a device that was created by
* root_device_register().
*/
void root_device_unregister(struct device *dev)
{
struct root_device *root = to_root_device(dev);
if (root->owner)
sysfs_remove_link(&root->dev.kobj, "module");
device_unregister(dev);
}
EXPORT_SYMBOL_GPL(root_device_unregister);
static void device_create_release(struct device *dev)
{
pr_debug("device: '%s': %s\n", dev_name(dev), __func__); kfree(dev);
}
static __printf(6, 0) struct device *
device_create_groups_vargs(const struct class *class, struct device *parent,
dev_t devt, void *drvdata,
const struct attribute_group **groups,
const char *fmt, va_list args)
{
struct device *dev = NULL;
int retval = -ENODEV;
if (IS_ERR_OR_NULL(class))
goto error;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev) {
retval = -ENOMEM;
goto error;
}
device_initialize(dev);
dev->devt = devt;
dev->class = class;
dev->parent = parent;
dev->groups = groups;
dev->release = device_create_release;
dev_set_drvdata(dev, drvdata);
retval = kobject_set_name_vargs(&dev->kobj, fmt, args);
if (retval)
goto error;
retval = device_add(dev);
if (retval)
goto error;
return dev;
error:
put_device(dev);
return ERR_PTR(retval);
}
/**
* device_create - creates a device and registers it with sysfs
* @class: pointer to the struct class that this device should be registered to
* @parent: pointer to the parent struct device of this new device, if any
* @devt: the dev_t for the char device to be added
* @drvdata: the data to be added to the device for callbacks
* @fmt: string for the device's name
*
* This function can be used by char device classes. A struct device
* will be created in sysfs, registered to the specified class.
*
* A "dev" file will be created, showing the dev_t for the device, if
* the dev_t is not 0,0.
* If a pointer to a parent struct device is passed in, the newly created
* struct device will be a child of that device in sysfs.
* The pointer to the struct device will be returned from the call.
* Any further sysfs files that might be required can be created using this
* pointer.
*
* Returns &struct device pointer on success, or ERR_PTR() on error.
*/
struct device *device_create(const struct class *class, struct device *parent,
dev_t devt, void *drvdata, const char *fmt, ...)
{
va_list vargs;
struct device *dev;
va_start(vargs, fmt);
dev = device_create_groups_vargs(class, parent, devt, drvdata, NULL,
fmt, vargs);
va_end(vargs);
return dev;
}
EXPORT_SYMBOL_GPL(device_create);
/**
* device_create_with_groups - creates a device and registers it with sysfs
* @class: pointer to the struct class that this device should be registered to
* @parent: pointer to the parent struct device of this new device, if any
* @devt: the dev_t for the char device to be added
* @drvdata: the data to be added to the device for callbacks
* @groups: NULL-terminated list of attribute groups to be created
* @fmt: string for the device's name
*
* This function can be used by char device classes. A struct device
* will be created in sysfs, registered to the specified class.
* Additional attributes specified in the groups parameter will also
* be created automatically.
*
* A "dev" file will be created, showing the dev_t for the device, if
* the dev_t is not 0,0.
* If a pointer to a parent struct device is passed in, the newly created
* struct device will be a child of that device in sysfs.
* The pointer to the struct device will be returned from the call.
* Any further sysfs files that might be required can be created using this
* pointer.
*
* Returns &struct device pointer on success, or ERR_PTR() on error.
*/
struct device *device_create_with_groups(const struct class *class,
struct device *parent, dev_t devt,
void *drvdata,
const struct attribute_group **groups,
const char *fmt, ...)
{
va_list vargs;
struct device *dev;
va_start(vargs, fmt);
dev = device_create_groups_vargs(class, parent, devt, drvdata, groups,
fmt, vargs);
va_end(vargs);
return dev;
}
EXPORT_SYMBOL_GPL(device_create_with_groups);
/**
* device_destroy - removes a device that was created with device_create()
* @class: pointer to the struct class that this device was registered with
* @devt: the dev_t of the device that was previously registered
*
* This call unregisters and cleans up a device that was created with a
* call to device_create().
*/
void device_destroy(const struct class *class, dev_t devt)
{
struct device *dev;
dev = class_find_device_by_devt(class, devt);
if (dev) {
put_device(dev);
device_unregister(dev);
}
}
EXPORT_SYMBOL_GPL(device_destroy);
/**
* device_rename - renames a device
* @dev: the pointer to the struct device to be renamed
* @new_name: the new name of the device
*
* It is the responsibility of the caller to provide mutual
* exclusion between two different calls of device_rename
* on the same device to ensure that new_name is valid and
* won't conflict with other devices.
*
* Note: given that some subsystems (networking and infiniband) use this
* function, with no immediate plans for this to change, we cannot assume or
* require that this function not be called at all.
*
* However, if you're writing new code, do not call this function. The following
* text from Kay Sievers offers some insight:
*
* Renaming devices is racy at many levels, symlinks and other stuff are not
* replaced atomically, and you get a "move" uevent, but it's not easy to
* connect the event to the old and new device. Device nodes are not renamed at
* all, there isn't even support for that in the kernel now.
*
* In the meantime, during renaming, your target name might be taken by another
* driver, creating conflicts. Or the old name is taken directly after you
* renamed it -- then you get events for the same DEVPATH, before you even see
* the "move" event. It's just a mess, and nothing new should ever rely on
* kernel device renaming. Besides that, it's not even implemented now for
* other things than (driver-core wise very simple) network devices.
*
* Make up a "real" name in the driver before you register anything, or add
* some other attributes for userspace to find the device, or use udev to add
* symlinks -- but never rename kernel devices later, it's a complete mess. We
* don't even want to get into that and try to implement the missing pieces in
* the core. We really have other pieces to fix in the driver core mess. :)
*/
int device_rename(struct device *dev, const char *new_name)
{
struct kobject *kobj = &dev->kobj;
char *old_device_name = NULL;
int error;
dev = get_device(dev);
if (!dev)
return -EINVAL;
dev_dbg(dev, "renaming to %s\n", new_name);
old_device_name = kstrdup(dev_name(dev), GFP_KERNEL);
if (!old_device_name) {
error = -ENOMEM;
goto out;
}
if (dev->class) {
struct subsys_private *sp = class_to_subsys(dev->class);
if (!sp) {
error = -EINVAL;
goto out;
}
error = sysfs_rename_link_ns(&sp->subsys.kobj, kobj, old_device_name,
new_name, kobject_namespace(kobj));
subsys_put(sp);
if (error)
goto out;
}
error = kobject_rename(kobj, new_name);
if (error)
goto out;
out:
put_device(dev);
kfree(old_device_name);
return error;
}
EXPORT_SYMBOL_GPL(device_rename);
static int device_move_class_links(struct device *dev,
struct device *old_parent,
struct device *new_parent)
{
int error = 0;
if (old_parent)
sysfs_remove_link(&dev->kobj, "device");
if (new_parent)
error = sysfs_create_link(&dev->kobj, &new_parent->kobj,
"device");
return error;
}
/**
* device_move - moves a device to a new parent
* @dev: the pointer to the struct device to be moved
* @new_parent: the new parent of the device (can be NULL)
* @dpm_order: how to reorder the dpm_list
*/
int device_move(struct device *dev, struct device *new_parent,
enum dpm_order dpm_order)
{
int error;
struct device *old_parent;
struct kobject *new_parent_kobj;
dev = get_device(dev);
if (!dev)
return -EINVAL;
device_pm_lock();
new_parent = get_device(new_parent);
new_parent_kobj = get_device_parent(dev, new_parent);
if (IS_ERR(new_parent_kobj)) {
error = PTR_ERR(new_parent_kobj);
put_device(new_parent);
goto out;
}
pr_debug("device: '%s': %s: moving to '%s'\n", dev_name(dev),
__func__, new_parent ? dev_name(new_parent) : "<NULL>");
error = kobject_move(&dev->kobj, new_parent_kobj);
if (error) {
cleanup_glue_dir(dev, new_parent_kobj);
put_device(new_parent);
goto out;
}
old_parent = dev->parent;
dev->parent = new_parent;
if (old_parent)
klist_remove(&dev->p->knode_parent);
if (new_parent) {
klist_add_tail(&dev->p->knode_parent,
&new_parent->p->klist_children);
set_dev_node(dev, dev_to_node(new_parent));
}
if (dev->class) {
error = device_move_class_links(dev, old_parent, new_parent);
if (error) {
/* We ignore errors on cleanup since we're hosed anyway... */
device_move_class_links(dev, new_parent, old_parent);
if (!kobject_move(&dev->kobj, &old_parent->kobj)) {
if (new_parent)
klist_remove(&dev->p->knode_parent);
dev->parent = old_parent;
if (old_parent) {
klist_add_tail(&dev->p->knode_parent,
&old_parent->p->klist_children);
set_dev_node(dev, dev_to_node(old_parent));
}
}
cleanup_glue_dir(dev, new_parent_kobj);
put_device(new_parent);
goto out;
}
}
switch (dpm_order) {
case DPM_ORDER_NONE:
break;
case DPM_ORDER_DEV_AFTER_PARENT:
device_pm_move_after(dev, new_parent);
devices_kset_move_after(dev, new_parent);
break;
case DPM_ORDER_PARENT_BEFORE_DEV:
device_pm_move_before(new_parent, dev);
devices_kset_move_before(new_parent, dev);
break;
case DPM_ORDER_DEV_LAST:
device_pm_move_last(dev);
devices_kset_move_last(dev);
break;
}
put_device(old_parent);
out:
device_pm_unlock();
put_device(dev);
return error;
}
EXPORT_SYMBOL_GPL(device_move);
static int device_attrs_change_owner(struct device *dev, kuid_t kuid,
kgid_t kgid)
{
struct kobject *kobj = &dev->kobj;
const struct class *class = dev->class;
const struct device_type *type = dev->type;
int error;
if (class) {
/*
* Change the device groups of the device class for @dev to
* @kuid/@kgid.
*/
error = sysfs_groups_change_owner(kobj, class->dev_groups, kuid,
kgid);
if (error)
return error;
}
if (type) {
/*
* Change the device groups of the device type for @dev to
* @kuid/@kgid.
*/
error = sysfs_groups_change_owner(kobj, type->groups, kuid,
kgid);
if (error)
return error;
}
/* Change the device groups of @dev to @kuid/@kgid. */
error = sysfs_groups_change_owner(kobj, dev->groups, kuid, kgid);
if (error)
return error;
if (device_supports_offline(dev) && !dev->offline_disabled) {
/* Change online device attributes of @dev to @kuid/@kgid. */
error = sysfs_file_change_owner(kobj, dev_attr_online.attr.name,
kuid, kgid);
if (error)
return error;
}
return 0;
}
/**
* device_change_owner - change the owner of an existing device.
* @dev: device.
* @kuid: new owner's kuid
* @kgid: new owner's kgid
*
* This changes the owner of @dev and its corresponding sysfs entries to
* @kuid/@kgid. This function closely mirrors how @dev was added via driver
* core.
*
* Returns 0 on success or error code on failure.
*/
int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
{
int error;
struct kobject *kobj = &dev->kobj;
struct subsys_private *sp;
dev = get_device(dev);
if (!dev)
return -EINVAL;
/*
* Change the kobject and the default attributes and groups of the
* ktype associated with it to @kuid/@kgid.
*/
error = sysfs_change_owner(kobj, kuid, kgid);
if (error)
goto out;
/*
* Change the uevent file for @dev to the new owner. The uevent file
* was created in a separate step when @dev got added and we mirror
* that step here.
*/
error = sysfs_file_change_owner(kobj, dev_attr_uevent.attr.name, kuid,
kgid);
if (error)
goto out;
/*
* Change the device groups, the device groups associated with the
* device class, and the groups associated with the device type of @dev
* to @kuid/@kgid.
*/
error = device_attrs_change_owner(dev, kuid, kgid);
if (error)
goto out;
error = dpm_sysfs_change_owner(dev, kuid, kgid);
if (error)
goto out;
/*
* Change the owner of the symlink located in the class directory of
* the device class associated with @dev which points to the actual
* directory entry for @dev to @kuid/@kgid. This ensures that the
* symlink shows the same permissions as its target.
*/
sp = class_to_subsys(dev->class);
if (!sp) {
error = -EINVAL;
goto out;
}
error = sysfs_link_change_owner(&sp->subsys.kobj, &dev->kobj, dev_name(dev), kuid, kgid);
subsys_put(sp);
out:
put_device(dev);
return error;
}
EXPORT_SYMBOL_GPL(device_change_owner);
/**
* device_shutdown - call ->shutdown() on each device to shutdown.
*/
void device_shutdown(void)
{
struct device *dev, *parent;
wait_for_device_probe();
device_block_probing();
cpufreq_suspend();
spin_lock(&devices_kset->list_lock);
/*
* Walk the devices list backward, shutting down each in turn.
* Beware that device unplug events may also start pulling
* devices offline, even as the system is shutting down.
*/
while (!list_empty(&devices_kset->list)) {
dev = list_entry(devices_kset->list.prev, struct device,
kobj.entry);
/*
* hold reference count of device's parent to
* prevent it from being freed because parent's
* lock is to be held
*/
parent = get_device(dev->parent);
get_device(dev);
/*
* Make sure the device is off the kset list, in the
* event that dev->*->shutdown() doesn't remove it.
*/
list_del_init(&dev->kobj.entry);
spin_unlock(&devices_kset->list_lock);
/* hold lock to avoid race with probe/release */
if (parent)
device_lock(parent);
device_lock(dev);
/* Don't allow any more runtime suspends */
pm_runtime_get_noresume(dev);
pm_runtime_barrier(dev);
if (dev->class && dev->class->shutdown_pre) {
if (initcall_debug)
dev_info(dev, "shutdown_pre\n");
dev->class->shutdown_pre(dev);
}
if (dev->bus && dev->bus->shutdown) {
if (initcall_debug)
dev_info(dev, "shutdown\n");
dev->bus->shutdown(dev);
} else if (dev->driver && dev->driver->shutdown) {
if (initcall_debug)
dev_info(dev, "shutdown\n");
dev->driver->shutdown(dev);
}
device_unlock(dev);
if (parent)
device_unlock(parent);
put_device(dev);
put_device(parent);
spin_lock(&devices_kset->list_lock);
}
spin_unlock(&devices_kset->list_lock);
}
/*
* Device logging functions
*/
#ifdef CONFIG_PRINTK
static void
set_dev_info(const struct device *dev, struct dev_printk_info *dev_info)
{
const char *subsys;
memset(dev_info, 0, sizeof(*dev_info));
if (dev->class)
subsys = dev->class->name; else if (dev->bus) subsys = dev->bus->name;
else
return;
strscpy(dev_info->subsystem, subsys, sizeof(dev_info->subsystem));
/*
* Add device identifier DEVICE=:
* b12:8 block dev_t
* c127:3 char dev_t
* n8 netdev ifindex
* +sound:card0 subsystem:devname
*/
if (MAJOR(dev->devt)) {
char c;
if (strcmp(subsys, "block") == 0)
c = 'b';
else
c = 'c';
snprintf(dev_info->device, sizeof(dev_info->device),
"%c%u:%u", c, MAJOR(dev->devt), MINOR(dev->devt));
} else if (strcmp(subsys, "net") == 0) {
struct net_device *net = to_net_dev(dev);
snprintf(dev_info->device, sizeof(dev_info->device),
"n%u", net->ifindex);
} else {
snprintf(dev_info->device, sizeof(dev_info->device),
"+%s:%s", subsys, dev_name(dev));
}
}
int dev_vprintk_emit(int level, const struct device *dev,
const char *fmt, va_list args)
{
struct dev_printk_info dev_info;
set_dev_info(dev, &dev_info);
return vprintk_emit(0, level, &dev_info, fmt, args);
}
EXPORT_SYMBOL(dev_vprintk_emit);
int dev_printk_emit(int level, const struct device *dev, const char *fmt, ...)
{
va_list args;
int r;
va_start(args, fmt);
r = dev_vprintk_emit(level, dev, fmt, args);
va_end(args);
return r;
}
EXPORT_SYMBOL(dev_printk_emit);
static void __dev_printk(const char *level, const struct device *dev,
struct va_format *vaf)
{
if (dev) dev_printk_emit(level[1] - '0', dev, "%s %s: %pV",
dev_driver_string(dev), dev_name(dev), vaf);
else
printk("%s(NULL device *): %pV", level, vaf);
}
void _dev_printk(const char *level, const struct device *dev,
const char *fmt, ...)
{
struct va_format vaf;
va_list args;
va_start(args, fmt);
vaf.fmt = fmt;
vaf.va = &args;
__dev_printk(level, dev, &vaf);
va_end(args);
}
EXPORT_SYMBOL(_dev_printk);
#define define_dev_printk_level(func, kern_level) \
void func(const struct device *dev, const char *fmt, ...) \
{ \
struct va_format vaf; \
va_list args; \
\
va_start(args, fmt); \
\
vaf.fmt = fmt; \
vaf.va = &args; \
\
__dev_printk(kern_level, dev, &vaf); \
\
va_end(args); \
} \
EXPORT_SYMBOL(func);
define_dev_printk_level(_dev_emerg, KERN_EMERG);
define_dev_printk_level(_dev_alert, KERN_ALERT);
define_dev_printk_level(_dev_crit, KERN_CRIT);
define_dev_printk_level(_dev_err, KERN_ERR);define_dev_printk_level(_dev_warn, KERN_WARNING);define_dev_printk_level(_dev_notice, KERN_NOTICE);define_dev_printk_level(_dev_info, KERN_INFO);
#endif
/**
* dev_err_probe - probe error check and log helper
* @dev: the pointer to the struct device
* @err: error value to test
* @fmt: printf-style format string
* @...: arguments as specified in the format string
*
* This helper implements common pattern present in probe functions for error
* checking: print debug or error message depending if the error value is
* -EPROBE_DEFER and propagate error upwards.
* In case of -EPROBE_DEFER it sets also defer probe reason, which can be
* checked later by reading devices_deferred debugfs attribute.
* It replaces code sequence::
*
* if (err != -EPROBE_DEFER)
* dev_err(dev, ...);
* else
* dev_dbg(dev, ...);
* return err;
*
* with::
*
* return dev_err_probe(dev, err, ...);
*
* Using this helper in your probe function is totally fine even if @err is
* known to never be -EPROBE_DEFER.
* The benefit compared to a normal dev_err() is the standardized format
* of the error code, it being emitted symbolically (i.e. you get "EAGAIN"
* instead of "-35") and the fact that the error code is returned which allows
* more compact error paths.
*
* Returns @err.
*/
int dev_err_probe(const struct device *dev, int err, const char *fmt, ...)
{
struct va_format vaf;
va_list args;
va_start(args, fmt);
vaf.fmt = fmt;
vaf.va = &args;
if (err != -EPROBE_DEFER) {
dev_err(dev, "error %pe: %pV", ERR_PTR(err), &vaf);
} else {
device_set_deferred_probe_reason(dev, &vaf);
dev_dbg(dev, "error %pe: %pV", ERR_PTR(err), &vaf);
}
va_end(args);
return err;
}
EXPORT_SYMBOL_GPL(dev_err_probe);
static inline bool fwnode_is_primary(struct fwnode_handle *fwnode)
{
return fwnode && !IS_ERR(fwnode->secondary);
}
/**
* set_primary_fwnode - Change the primary firmware node of a given device.
* @dev: Device to handle.
* @fwnode: New primary firmware node of the device.
*
* Set the device's firmware node pointer to @fwnode, but if a secondary
* firmware node of the device is present, preserve it.
*
* Valid fwnode cases are:
* - primary --> secondary --> -ENODEV
* - primary --> NULL
* - secondary --> -ENODEV
* - NULL
*/
void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode)
{
struct device *parent = dev->parent;
struct fwnode_handle *fn = dev->fwnode;
if (fwnode) {
if (fwnode_is_primary(fn))
fn = fn->secondary;
if (fn) {
WARN_ON(fwnode->secondary);
fwnode->secondary = fn;
}
dev->fwnode = fwnode;
} else {
if (fwnode_is_primary(fn)) {
dev->fwnode = fn->secondary;
/* Skip nullifying fn->secondary if the primary is shared */
if (parent && fn == parent->fwnode)
return;
/* Set fn->secondary = NULL, so fn remains the primary fwnode */
fn->secondary = NULL;
} else {
dev->fwnode = NULL;
}
}
}
EXPORT_SYMBOL_GPL(set_primary_fwnode);
/**
* set_secondary_fwnode - Change the secondary firmware node of a given device.
* @dev: Device to handle.
* @fwnode: New secondary firmware node of the device.
*
* If a primary firmware node of the device is present, set its secondary
* pointer to @fwnode. Otherwise, set the device's firmware node pointer to
* @fwnode.
*/
void set_secondary_fwnode(struct device *dev, struct fwnode_handle *fwnode)
{
if (fwnode)
fwnode->secondary = ERR_PTR(-ENODEV);
if (fwnode_is_primary(dev->fwnode))
dev->fwnode->secondary = fwnode;
else
dev->fwnode = fwnode;
}
EXPORT_SYMBOL_GPL(set_secondary_fwnode);
/**
* device_set_of_node_from_dev - reuse device-tree node of another device
* @dev: device whose device-tree node is being set
* @dev2: device whose device-tree node is being reused
*
* Takes another reference to the new device-tree node after first dropping
* any reference held to the old node.
*/
void device_set_of_node_from_dev(struct device *dev, const struct device *dev2)
{
of_node_put(dev->of_node);
dev->of_node = of_node_get(dev2->of_node);
dev->of_node_reused = true;
}
EXPORT_SYMBOL_GPL(device_set_of_node_from_dev);
void device_set_node(struct device *dev, struct fwnode_handle *fwnode)
{
dev->fwnode = fwnode;
dev->of_node = to_of_node(fwnode);
}
EXPORT_SYMBOL_GPL(device_set_node);
int device_match_name(struct device *dev, const void *name)
{
return sysfs_streq(dev_name(dev), name);
}
EXPORT_SYMBOL_GPL(device_match_name);
int device_match_of_node(struct device *dev, const void *np)
{
return dev->of_node == np;
}
EXPORT_SYMBOL_GPL(device_match_of_node);
int device_match_fwnode(struct device *dev, const void *fwnode)
{
return dev_fwnode(dev) == fwnode;
}
EXPORT_SYMBOL_GPL(device_match_fwnode);
int device_match_devt(struct device *dev, const void *pdevt)
{
return dev->devt == *(dev_t *)pdevt;
}
EXPORT_SYMBOL_GPL(device_match_devt);
int device_match_acpi_dev(struct device *dev, const void *adev)
{
return ACPI_COMPANION(dev) == adev;
}
EXPORT_SYMBOL(device_match_acpi_dev);
int device_match_acpi_handle(struct device *dev, const void *handle)
{
return ACPI_HANDLE(dev) == handle;
}
EXPORT_SYMBOL(device_match_acpi_handle);
int device_match_any(struct device *dev, const void *unused)
{
return 1;
}
EXPORT_SYMBOL_GPL(device_match_any);
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1992 Krishna Balasubramanian and Linus Torvalds
* Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
* Copyright (C) 2002 Andi Kleen
*
* This handles calls from both 32bit and 64bit mode.
*
* Lock order:
* context.ldt_usr_sem
* mmap_lock
* context.lock
*/
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/syscalls.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/uaccess.h>
#include <asm/ldt.h>
#include <asm/tlb.h>
#include <asm/desc.h>
#include <asm/mmu_context.h>
#include <asm/pgtable_areas.h>
#include <xen/xen.h>
/* This is a multiple of PAGE_SIZE. */
#define LDT_SLOT_STRIDE (LDT_ENTRIES * LDT_ENTRY_SIZE)
static inline void *ldt_slot_va(int slot)
{
return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot);
}
void load_mm_ldt(struct mm_struct *mm)
{
struct ldt_struct *ldt;
/* READ_ONCE synchronizes with smp_store_release */
ldt = READ_ONCE(mm->context.ldt);
/*
* Any change to mm->context.ldt is followed by an IPI to all
* CPUs with the mm active. The LDT will not be freed until
* after the IPI is handled by all such CPUs. This means that
* if the ldt_struct changes before we return, the values we see
* will be safe, and the new values will be loaded before we run
* any user code.
*
* NB: don't try to convert this to use RCU without extreme care.
* We would still need IRQs off, because we don't want to change
* the local LDT after an IPI loaded a newer value than the one
* that we can see.
*/
if (unlikely(ldt)) {
if (static_cpu_has(X86_FEATURE_PTI)) {
if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) {
/*
* Whoops -- either the new LDT isn't mapped
* (if slot == -1) or is mapped into a bogus
* slot (if slot > 1).
*/
clear_LDT();
return;
}
/*
* If page table isolation is enabled, ldt->entries
* will not be mapped in the userspace pagetables.
* Tell the CPU to access the LDT through the alias
* at ldt_slot_va(ldt->slot).
*/
set_ldt(ldt_slot_va(ldt->slot), ldt->nr_entries);
} else {
set_ldt(ldt->entries, ldt->nr_entries);
}
} else {
clear_LDT();
}
}
void switch_ldt(struct mm_struct *prev, struct mm_struct *next)
{
/*
* Load the LDT if either the old or new mm had an LDT.
*
* An mm will never go from having an LDT to not having an LDT. Two
* mms never share an LDT, so we don't gain anything by checking to
* see whether the LDT changed. There's also no guarantee that
* prev->context.ldt actually matches LDTR, but, if LDTR is non-NULL,
* then prev->context.ldt will also be non-NULL.
*
* If we really cared, we could optimize the case where prev == next
* and we're exiting lazy mode. Most of the time, if this happens,
* we don't actually need to reload LDTR, but modify_ldt() is mostly
* used by legacy code and emulators where we don't need this level of
* performance.
*
* This uses | instead of || because it generates better code.
*/
if (unlikely((unsigned long)prev->context.ldt |
(unsigned long)next->context.ldt))
load_mm_ldt(next);
DEBUG_LOCKS_WARN_ON(preemptible());
}
static void refresh_ldt_segments(void)
{
#ifdef CONFIG_X86_64
unsigned short sel;
/*
* Make sure that the cached DS and ES descriptors match the updated
* LDT.
*/
savesegment(ds, sel);
if ((sel & SEGMENT_TI_MASK) == SEGMENT_LDT)
loadsegment(ds, sel);
savesegment(es, sel);
if ((sel & SEGMENT_TI_MASK) == SEGMENT_LDT)
loadsegment(es, sel);
#endif
}
/* context.lock is held by the task which issued the smp function call */
static void flush_ldt(void *__mm)
{
struct mm_struct *mm = __mm;
if (this_cpu_read(cpu_tlbstate.loaded_mm) != mm)
return;
load_mm_ldt(mm);
refresh_ldt_segments();
}
/* The caller must call finalize_ldt_struct on the result. LDT starts zeroed. */
static struct ldt_struct *alloc_ldt_struct(unsigned int num_entries)
{
struct ldt_struct *new_ldt;
unsigned int alloc_size;
if (num_entries > LDT_ENTRIES)
return NULL;
new_ldt = kmalloc(sizeof(struct ldt_struct), GFP_KERNEL_ACCOUNT);
if (!new_ldt)
return NULL;
BUILD_BUG_ON(LDT_ENTRY_SIZE != sizeof(struct desc_struct));
alloc_size = num_entries * LDT_ENTRY_SIZE;
/*
* Xen is very picky: it requires a page-aligned LDT that has no
* trailing nonzero bytes in any page that contains LDT descriptors.
* Keep it simple: zero the whole allocation and never allocate less
* than PAGE_SIZE.
*/
if (alloc_size > PAGE_SIZE)
new_ldt->entries = __vmalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
else
new_ldt->entries = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
if (!new_ldt->entries) {
kfree(new_ldt);
return NULL;
}
/* The new LDT isn't aliased for PTI yet. */
new_ldt->slot = -1;
new_ldt->nr_entries = num_entries;
return new_ldt;
}
#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
static void do_sanity_check(struct mm_struct *mm,
bool had_kernel_mapping,
bool had_user_mapping)
{
if (mm->context.ldt) {
/*
* We already had an LDT. The top-level entry should already
* have been allocated and synchronized with the usermode
* tables.
*/
WARN_ON(!had_kernel_mapping);
if (boot_cpu_has(X86_FEATURE_PTI))
WARN_ON(!had_user_mapping);
} else {
/*
* This is the first time we're mapping an LDT for this process.
* Sync the pgd to the usermode tables.
*/
WARN_ON(had_kernel_mapping);
if (boot_cpu_has(X86_FEATURE_PTI))
WARN_ON(had_user_mapping);
}
}
#ifdef CONFIG_X86_PAE
static pmd_t *pgd_to_pmd_walk(pgd_t *pgd, unsigned long va)
{
p4d_t *p4d;
pud_t *pud;
if (pgd->pgd == 0)
return NULL;
p4d = p4d_offset(pgd, va);
if (p4d_none(*p4d))
return NULL;
pud = pud_offset(p4d, va);
if (pud_none(*pud))
return NULL;
return pmd_offset(pud, va);
}
static void map_ldt_struct_to_user(struct mm_struct *mm)
{
pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR);
pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd);
pmd_t *k_pmd, *u_pmd;
k_pmd = pgd_to_pmd_walk(k_pgd, LDT_BASE_ADDR);
u_pmd = pgd_to_pmd_walk(u_pgd, LDT_BASE_ADDR);
if (boot_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt)
set_pmd(u_pmd, *k_pmd);
}
static void sanity_check_ldt_mapping(struct mm_struct *mm)
{
pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR);
pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd);
bool had_kernel, had_user;
pmd_t *k_pmd, *u_pmd;
k_pmd = pgd_to_pmd_walk(k_pgd, LDT_BASE_ADDR);
u_pmd = pgd_to_pmd_walk(u_pgd, LDT_BASE_ADDR);
had_kernel = (k_pmd->pmd != 0);
had_user = (u_pmd->pmd != 0);
do_sanity_check(mm, had_kernel, had_user);
}
#else /* !CONFIG_X86_PAE */
static void map_ldt_struct_to_user(struct mm_struct *mm)
{
pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR);
if (boot_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt)
set_pgd(kernel_to_user_pgdp(pgd), *pgd);
}
static void sanity_check_ldt_mapping(struct mm_struct *mm)
{
pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR);
bool had_kernel = (pgd->pgd != 0);
bool had_user = (kernel_to_user_pgdp(pgd)->pgd != 0);
do_sanity_check(mm, had_kernel, had_user);
}
#endif /* CONFIG_X86_PAE */
/*
* If PTI is enabled, this maps the LDT into the kernelmode and
* usermode tables for the given mm.
*/
static int
map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot)
{
unsigned long va;
bool is_vmalloc;
spinlock_t *ptl;
int i, nr_pages;
if (!boot_cpu_has(X86_FEATURE_PTI))
return 0;
/*
* Any given ldt_struct should have map_ldt_struct() called at most
* once.
*/
WARN_ON(ldt->slot != -1);
/* Check if the current mappings are sane */
sanity_check_ldt_mapping(mm);
is_vmalloc = is_vmalloc_addr(ldt->entries);
nr_pages = DIV_ROUND_UP(ldt->nr_entries * LDT_ENTRY_SIZE, PAGE_SIZE);
for (i = 0; i < nr_pages; i++) {
unsigned long offset = i << PAGE_SHIFT;
const void *src = (char *)ldt->entries + offset;
unsigned long pfn;
pgprot_t pte_prot;
pte_t pte, *ptep;
va = (unsigned long)ldt_slot_va(slot) + offset;
pfn = is_vmalloc ? vmalloc_to_pfn(src) :
page_to_pfn(virt_to_page(src));
/*
* Treat the PTI LDT range as a *userspace* range.
* get_locked_pte() will allocate all needed pagetables
* and account for them in this mm.
*/
ptep = get_locked_pte(mm, va, &ptl);
if (!ptep)
return -ENOMEM;
/*
* Map it RO so the easy to find address is not a primary
* target via some kernel interface which misses a
* permission check.
*/
pte_prot = __pgprot(__PAGE_KERNEL_RO & ~_PAGE_GLOBAL);
/* Filter out unsuppored __PAGE_KERNEL* bits: */
pgprot_val(pte_prot) &= __supported_pte_mask;
pte = pfn_pte(pfn, pte_prot);
set_pte_at(mm, va, ptep, pte);
pte_unmap_unlock(ptep, ptl);
}
/* Propagate LDT mapping to the user page-table */
map_ldt_struct_to_user(mm);
ldt->slot = slot;
return 0;
}
static void unmap_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt)
{
unsigned long va;
int i, nr_pages;
if (!ldt)
return;
/* LDT map/unmap is only required for PTI */
if (!boot_cpu_has(X86_FEATURE_PTI))
return;
nr_pages = DIV_ROUND_UP(ldt->nr_entries * LDT_ENTRY_SIZE, PAGE_SIZE);
for (i = 0; i < nr_pages; i++) {
unsigned long offset = i << PAGE_SHIFT;
spinlock_t *ptl;
pte_t *ptep;
va = (unsigned long)ldt_slot_va(ldt->slot) + offset;
ptep = get_locked_pte(mm, va, &ptl);
if (!WARN_ON_ONCE(!ptep)) {
pte_clear(mm, va, ptep);
pte_unmap_unlock(ptep, ptl);
}
}
va = (unsigned long)ldt_slot_va(ldt->slot);
flush_tlb_mm_range(mm, va, va + nr_pages * PAGE_SIZE, PAGE_SHIFT, false);
}
#else /* !CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */
static int
map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot)
{
return 0;
}
static void unmap_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt)
{
}
#endif /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */
static void free_ldt_pgtables(struct mm_struct *mm)
{
#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
struct mmu_gather tlb;
unsigned long start = LDT_BASE_ADDR;
unsigned long end = LDT_END_ADDR;
if (!boot_cpu_has(X86_FEATURE_PTI))
return;
/*
* Although free_pgd_range() is intended for freeing user
* page-tables, it also works out for kernel mappings on x86.
* We use tlb_gather_mmu_fullmm() to avoid confusing the
* range-tracking logic in __tlb_adjust_range().
*/
tlb_gather_mmu_fullmm(&tlb, mm);
free_pgd_range(&tlb, start, end, start, end);
tlb_finish_mmu(&tlb);
#endif
}
/* After calling this, the LDT is immutable. */
static void finalize_ldt_struct(struct ldt_struct *ldt)
{
paravirt_alloc_ldt(ldt->entries, ldt->nr_entries);
}
static void install_ldt(struct mm_struct *mm, struct ldt_struct *ldt)
{
mutex_lock(&mm->context.lock);
/* Synchronizes with READ_ONCE in load_mm_ldt. */
smp_store_release(&mm->context.ldt, ldt);
/* Activate the LDT for all CPUs using currents mm. */
on_each_cpu_mask(mm_cpumask(mm), flush_ldt, mm, true);
mutex_unlock(&mm->context.lock);
}
static void free_ldt_struct(struct ldt_struct *ldt)
{
if (likely(!ldt))
return;
paravirt_free_ldt(ldt->entries, ldt->nr_entries);
if (ldt->nr_entries * LDT_ENTRY_SIZE > PAGE_SIZE)
vfree_atomic(ldt->entries);
else
free_page((unsigned long)ldt->entries); kfree(ldt);
}
/*
* Called on fork from arch_dup_mmap(). Just copy the current LDT state,
* the new task is not running, so nothing can be installed.
*/
int ldt_dup_context(struct mm_struct *old_mm, struct mm_struct *mm)
{
struct ldt_struct *new_ldt;
int retval = 0;
if (!old_mm)
return 0;
mutex_lock(&old_mm->context.lock);
if (!old_mm->context.ldt)
goto out_unlock;
new_ldt = alloc_ldt_struct(old_mm->context.ldt->nr_entries);
if (!new_ldt) {
retval = -ENOMEM;
goto out_unlock;
}
memcpy(new_ldt->entries, old_mm->context.ldt->entries,
new_ldt->nr_entries * LDT_ENTRY_SIZE);
finalize_ldt_struct(new_ldt);
retval = map_ldt_struct(mm, new_ldt, 0);
if (retval) {
free_ldt_pgtables(mm);
free_ldt_struct(new_ldt);
goto out_unlock;
}
mm->context.ldt = new_ldt;
out_unlock:
mutex_unlock(&old_mm->context.lock);
return retval;
}
/*
* No need to lock the MM as we are the last user
*
* 64bit: Don't touch the LDT register - we're already in the next thread.
*/
void destroy_context_ldt(struct mm_struct *mm)
{
free_ldt_struct(mm->context.ldt); mm->context.ldt = NULL;
}
void ldt_arch_exit_mmap(struct mm_struct *mm)
{
free_ldt_pgtables(mm);
}
static int read_ldt(void __user *ptr, unsigned long bytecount)
{
struct mm_struct *mm = current->mm;
unsigned long entries_size;
int retval;
down_read(&mm->context.ldt_usr_sem);
if (!mm->context.ldt) {
retval = 0;
goto out_unlock;
}
if (bytecount > LDT_ENTRY_SIZE * LDT_ENTRIES)
bytecount = LDT_ENTRY_SIZE * LDT_ENTRIES;
entries_size = mm->context.ldt->nr_entries * LDT_ENTRY_SIZE;
if (entries_size > bytecount)
entries_size = bytecount;
if (copy_to_user(ptr, mm->context.ldt->entries, entries_size)) {
retval = -EFAULT;
goto out_unlock;
}
if (entries_size != bytecount) {
/* Zero-fill the rest and pretend we read bytecount bytes. */
if (clear_user(ptr + entries_size, bytecount - entries_size)) {
retval = -EFAULT;
goto out_unlock;
}
}
retval = bytecount;
out_unlock:
up_read(&mm->context.ldt_usr_sem);
return retval;
}
static int read_default_ldt(void __user *ptr, unsigned long bytecount)
{
/* CHECKME: Can we use _one_ random number ? */
#ifdef CONFIG_X86_32
unsigned long size = 5 * sizeof(struct desc_struct);
#else
unsigned long size = 128;
#endif
if (bytecount > size)
bytecount = size;
if (clear_user(ptr, bytecount))
return -EFAULT;
return bytecount;
}
static bool allow_16bit_segments(void)
{
if (!IS_ENABLED(CONFIG_X86_16BIT))
return false;
#ifdef CONFIG_XEN_PV
/*
* Xen PV does not implement ESPFIX64, which means that 16-bit
* segments will not work correctly. Until either Xen PV implements
* ESPFIX64 and can signal this fact to the guest or unless someone
* provides compelling evidence that allowing broken 16-bit segments
* is worthwhile, disallow 16-bit segments under Xen PV.
*/
if (xen_pv_domain()) {
pr_info_once("Warning: 16-bit segments do not work correctly in a Xen PV guest\n");
return false;
}
#endif
return true;
}
static int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode)
{
struct mm_struct *mm = current->mm;
struct ldt_struct *new_ldt, *old_ldt;
unsigned int old_nr_entries, new_nr_entries;
struct user_desc ldt_info;
struct desc_struct ldt;
int error;
error = -EINVAL;
if (bytecount != sizeof(ldt_info))
goto out;
error = -EFAULT;
if (copy_from_user(&ldt_info, ptr, sizeof(ldt_info)))
goto out;
error = -EINVAL;
if (ldt_info.entry_number >= LDT_ENTRIES)
goto out;
if (ldt_info.contents == 3) {
if (oldmode)
goto out;
if (ldt_info.seg_not_present == 0)
goto out;
}
if ((oldmode && !ldt_info.base_addr && !ldt_info.limit) ||
LDT_empty(&ldt_info)) {
/* The user wants to clear the entry. */
memset(&ldt, 0, sizeof(ldt));
} else {
if (!ldt_info.seg_32bit && !allow_16bit_segments()) {
error = -EINVAL;
goto out;
}
fill_ldt(&ldt, &ldt_info);
if (oldmode)
ldt.avl = 0;
}
if (down_write_killable(&mm->context.ldt_usr_sem))
return -EINTR;
old_ldt = mm->context.ldt;
old_nr_entries = old_ldt ? old_ldt->nr_entries : 0;
new_nr_entries = max(ldt_info.entry_number + 1, old_nr_entries);
error = -ENOMEM;
new_ldt = alloc_ldt_struct(new_nr_entries);
if (!new_ldt)
goto out_unlock;
if (old_ldt)
memcpy(new_ldt->entries, old_ldt->entries, old_nr_entries * LDT_ENTRY_SIZE);
new_ldt->entries[ldt_info.entry_number] = ldt;
finalize_ldt_struct(new_ldt);
/*
* If we are using PTI, map the new LDT into the userspace pagetables.
* If there is already an LDT, use the other slot so that other CPUs
* will continue to use the old LDT until install_ldt() switches
* them over to the new LDT.
*/
error = map_ldt_struct(mm, new_ldt, old_ldt ? !old_ldt->slot : 0);
if (error) {
/*
* This only can fail for the first LDT setup. If an LDT is
* already installed then the PTE page is already
* populated. Mop up a half populated page table.
*/
if (!WARN_ON_ONCE(old_ldt))
free_ldt_pgtables(mm);
free_ldt_struct(new_ldt);
goto out_unlock;
}
install_ldt(mm, new_ldt);
unmap_ldt_struct(mm, old_ldt);
free_ldt_struct(old_ldt);
error = 0;
out_unlock:
up_write(&mm->context.ldt_usr_sem);
out:
return error;
}
SYSCALL_DEFINE3(modify_ldt, int , func , void __user * , ptr ,
unsigned long , bytecount)
{
int ret = -ENOSYS;
switch (func) {
case 0:
ret = read_ldt(ptr, bytecount);
break;
case 1:
ret = write_ldt(ptr, bytecount, 1);
break;
case 2:
ret = read_default_ldt(ptr, bytecount);
break;
case 0x11:
ret = write_ldt(ptr, bytecount, 0);
break;
}
/*
* The SYSCALL_DEFINE() macros give us an 'unsigned long'
* return type, but the ABI for sys_modify_ldt() expects
* 'int'. This cast gives us an int-sized value in %rax
* for the return code. The 'unsigned' is necessary so
* the compiler does not try to sign-extend the negative
* return codes into the high half of the register when
* taking the value from int->long.
*/
return (unsigned int)ret;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __X86_KERNEL_FPU_XSTATE_H
#define __X86_KERNEL_FPU_XSTATE_H
#include <asm/cpufeature.h>
#include <asm/fpu/xstate.h>
#include <asm/fpu/xcr.h>
#ifdef CONFIG_X86_64
DECLARE_PER_CPU(u64, xfd_state);
#endif
static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask)
{
/*
* XRSTORS requires these bits set in xcomp_bv, or it will
* trigger #GP:
*/
if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED))
xsave->header.xcomp_bv = mask | XCOMP_BV_COMPACTED_FORMAT;
}
static inline u64 xstate_get_group_perm(bool guest)
{
struct fpu *fpu = ¤t->group_leader->thread.fpu;
struct fpu_state_perm *perm;
/* Pairs with WRITE_ONCE() in xstate_request_perm() */
perm = guest ? &fpu->guest_perm : &fpu->perm;
return READ_ONCE(perm->__state_perm);
}
static inline u64 xstate_get_host_group_perm(void)
{
return xstate_get_group_perm(false);
}
enum xstate_copy_mode {
XSTATE_COPY_FP,
XSTATE_COPY_FX,
XSTATE_COPY_XSAVE,
};
struct membuf;
extern void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate,
u64 xfeatures, u32 pkru_val,
enum xstate_copy_mode copy_mode);
extern void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,
enum xstate_copy_mode mode);
extern int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru);
extern int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, const void __user *ubuf);
extern void fpu__init_cpu_xstate(void);
extern void fpu__init_system_xstate(unsigned int legacy_size);
extern void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr);
static inline u64 xfeatures_mask_supervisor(void)
{
return fpu_kernel_cfg.max_features & XFEATURE_MASK_SUPERVISOR_SUPPORTED;
}
static inline u64 xfeatures_mask_independent(void)
{
if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
return XFEATURE_MASK_INDEPENDENT & ~XFEATURE_MASK_LBR;
return XFEATURE_MASK_INDEPENDENT;
}
/* XSAVE/XRSTOR wrapper functions */
#ifdef CONFIG_X86_64
#define REX_PREFIX "0x48, "
#else
#define REX_PREFIX
#endif
/* These macros all use (%edi)/(%rdi) as the single memory argument. */
#define XSAVE ".byte " REX_PREFIX "0x0f,0xae,0x27"
#define XSAVEOPT ".byte " REX_PREFIX "0x0f,0xae,0x37"
#define XSAVEC ".byte " REX_PREFIX "0x0f,0xc7,0x27"
#define XSAVES ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
#define XRSTOR ".byte " REX_PREFIX "0x0f,0xae,0x2f"
#define XRSTORS ".byte " REX_PREFIX "0x0f,0xc7,0x1f"
/*
* After this @err contains 0 on success or the trap number when the
* operation raises an exception.
*/
#define XSTATE_OP(op, st, lmask, hmask, err) \
asm volatile("1:" op "\n\t" \
"xor %[err], %[err]\n" \
"2:\n\t" \
_ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE) \
: [err] "=a" (err) \
: "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
: "memory")
/*
* If XSAVES is enabled, it replaces XSAVEC because it supports supervisor
* states in addition to XSAVEC.
*
* Otherwise if XSAVEC is enabled, it replaces XSAVEOPT because it supports
* compacted storage format in addition to XSAVEOPT.
*
* Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT
* supports modified optimization which is not supported by XSAVE.
*
* We use XSAVE as a fallback.
*
* The 661 label is defined in the ALTERNATIVE* macros as the address of the
* original instruction which gets replaced. We need to use it here as the
* address of the instruction where we might get an exception at.
*/
#define XSTATE_XSAVE(st, lmask, hmask, err) \
asm volatile(ALTERNATIVE_3(XSAVE, \
XSAVEOPT, X86_FEATURE_XSAVEOPT, \
XSAVEC, X86_FEATURE_XSAVEC, \
XSAVES, X86_FEATURE_XSAVES) \
"\n" \
"xor %[err], %[err]\n" \
"3:\n" \
_ASM_EXTABLE_TYPE_REG(661b, 3b, EX_TYPE_EFAULT_REG, %[err]) \
: [err] "=r" (err) \
: "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
: "memory")
/*
* Use XRSTORS to restore context if it is enabled. XRSTORS supports compact
* XSAVE area format.
*/
#define XSTATE_XRESTORE(st, lmask, hmask) \
asm volatile(ALTERNATIVE(XRSTOR, \
XRSTORS, X86_FEATURE_XSAVES) \
"\n" \
"3:\n" \
_ASM_EXTABLE_TYPE(661b, 3b, EX_TYPE_FPU_RESTORE) \
: \
: "D" (st), "m" (*st), "a" (lmask), "d" (hmask) \
: "memory")
#if defined(CONFIG_X86_64) && defined(CONFIG_X86_DEBUG_FPU)
extern void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor);
#else
static inline void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) { }
#endif
#ifdef CONFIG_X86_64
static inline void xfd_set_state(u64 xfd)
{
wrmsrl(MSR_IA32_XFD, xfd); __this_cpu_write(xfd_state, xfd);
}
static inline void xfd_update_state(struct fpstate *fpstate)
{
if (fpu_state_size_dynamic()) {
u64 xfd = fpstate->xfd;
if (__this_cpu_read(xfd_state) != xfd)
xfd_set_state(xfd);
}
}
extern int __xfd_enable_feature(u64 which, struct fpu_guest *guest_fpu);
#else
static inline void xfd_set_state(u64 xfd) { }
static inline void xfd_update_state(struct fpstate *fpstate) { }
static inline int __xfd_enable_feature(u64 which, struct fpu_guest *guest_fpu) {
return -EPERM;
}
#endif
/*
* Save processor xstate to xsave area.
*
* Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features
* and command line options. The choice is permanent until the next reboot.
*/
static inline void os_xsave(struct fpstate *fpstate)
{
u64 mask = fpstate->xfeatures;
u32 lmask = mask;
u32 hmask = mask >> 32;
int err;
WARN_ON_FPU(!alternatives_patched);
xfd_validate_state(fpstate, mask, false);
XSTATE_XSAVE(&fpstate->regs.xsave, lmask, hmask, err);
/* We should never fault when copying to a kernel buffer: */
WARN_ON_FPU(err);
}
/*
* Restore processor xstate from xsave area.
*
* Uses XRSTORS when XSAVES is used, XRSTOR otherwise.
*/
static inline void os_xrstor(struct fpstate *fpstate, u64 mask)
{
u32 lmask = mask;
u32 hmask = mask >> 32;
xfd_validate_state(fpstate, mask, true);
XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
}
/* Restore of supervisor state. Does not require XFD */
static inline void os_xrstor_supervisor(struct fpstate *fpstate)
{
u64 mask = xfeatures_mask_supervisor();
u32 lmask = mask;
u32 hmask = mask >> 32;
XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
}
/*
* XSAVE itself always writes all requested xfeatures. Removing features
* from the request bitmap reduces the features which are written.
* Generate a mask of features which must be written to a sigframe. The
* unset features can be optimized away and not written.
*
* This optimization is user-visible. Only use for states where
* uninitialized sigframe contents are tolerable, like dynamic features.
*
* Users of buffers produced with this optimization must check XSTATE_BV
* to determine which features have been optimized out.
*/
static inline u64 xfeatures_need_sigframe_write(void)
{
u64 xfeaures_to_write;
/* In-use features must be written: */
xfeaures_to_write = xfeatures_in_use();
/* Also write all non-optimizable sigframe features: */
xfeaures_to_write |= XFEATURE_MASK_USER_SUPPORTED &
~XFEATURE_MASK_SIGFRAME_INITOPT;
return xfeaures_to_write;
}
/*
* Save xstate to user space xsave area.
*
* We don't use modified optimization because xrstor/xrstors might track
* a different application.
*
* We don't use compacted format xsave area for backward compatibility for
* old applications which don't understand the compacted format of the
* xsave area.
*
* The caller has to zero buf::header before calling this because XSAVE*
* does not touch the reserved fields in the header.
*/
static inline int xsave_to_user_sigframe(struct xregs_state __user *buf)
{
/*
* Include the features which are not xsaved/rstored by the kernel
* internally, e.g. PKRU. That's user space ABI and also required
* to allow the signal handler to modify PKRU.
*/
struct fpstate *fpstate = current->thread.fpu.fpstate;
u64 mask = fpstate->user_xfeatures;
u32 lmask;
u32 hmask;
int err;
/* Optimize away writing unnecessary xfeatures: */
if (fpu_state_size_dynamic())
mask &= xfeatures_need_sigframe_write(); lmask = mask;
hmask = mask >> 32;
xfd_validate_state(fpstate, mask, false);
stac();
XSTATE_OP(XSAVE, buf, lmask, hmask, err);
clac();
return err;
}
/*
* Restore xstate from user space xsave area.
*/
static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask)
{
struct xregs_state *xstate = ((__force struct xregs_state *)buf);
u32 lmask = mask;
u32 hmask = mask >> 32;
int err;
xfd_validate_state(current->thread.fpu.fpstate, mask, true);
stac();
XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
clac();
return err;
}
/*
* Restore xstate from kernel space xsave area, return an error code instead of
* an exception.
*/
static inline int os_xrstor_safe(struct fpstate *fpstate, u64 mask)
{
struct xregs_state *xstate = &fpstate->regs.xsave;
u32 lmask = mask;
u32 hmask = mask >> 32;
int err;
/* Ensure that XFD is up to date */
xfd_update_state(fpstate);
if (cpu_feature_enabled(X86_FEATURE_XSAVES))
XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
else
XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
return err;
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_UACCESS_64_H
#define _ASM_X86_UACCESS_64_H
/*
* User space memory access functions
*/
#include <linux/compiler.h>
#include <linux/lockdep.h>
#include <linux/kasan-checks.h>
#include <asm/alternative.h>
#include <asm/cpufeatures.h>
#include <asm/page.h>
#include <asm/percpu.h>
#ifdef CONFIG_ADDRESS_MASKING
/*
* Mask out tag bits from the address.
*/
static inline unsigned long __untagged_addr(unsigned long addr)
{
asm (ALTERNATIVE("",
"and " __percpu_arg([mask]) ", %[addr]", X86_FEATURE_LAM)
: [addr] "+r" (addr)
: [mask] "m" (__my_cpu_var(tlbstate_untag_mask)));
return addr;
}
#define untagged_addr(addr) ({ \
unsigned long __addr = (__force unsigned long)(addr); \
(__force __typeof__(addr))__untagged_addr(__addr); \
})
static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
unsigned long addr)
{
mmap_assert_locked(mm);
return addr & (mm)->context.untag_mask;
}
#define untagged_addr_remote(mm, addr) ({ \
unsigned long __addr = (__force unsigned long)(addr); \
(__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
})
#endif
/*
* The virtual address space space is logically divided into a kernel
* half and a user half. When cast to a signed type, user pointers
* are positive and kernel pointers are negative.
*/
#define valid_user_address(x) ((__force long)(x) >= 0)
/*
* User pointers can have tag bits on x86-64. This scheme tolerates
* arbitrary values in those bits rather then masking them off.
*
* Enforce two rules:
* 1. 'ptr' must be in the user half of the address space
* 2. 'ptr+size' must not overflow into kernel addresses
*
* Note that addresses around the sign change are not valid addresses,
* and will GP-fault even with LAM enabled if the sign bit is set (see
* "CR3.LAM_SUP" that can narrow the canonicality check if we ever
* enable it, but not remove it entirely).
*
* So the "overflow into kernel addresses" does not imply some sudden
* exact boundary at the sign bit, and we can allow a lot of slop on the
* size check.
*
* In fact, we could probably remove the size check entirely, since
* any kernel accesses will be in increasing address order starting
* at 'ptr', and even if the end might be in kernel space, we'll
* hit the GP faults for non-canonical accesses before we ever get
* there.
*
* That's a separate optimization, for now just handle the small
* constant case.
*/
static inline bool __access_ok(const void __user *ptr, unsigned long size)
{
if (__builtin_constant_p(size <= PAGE_SIZE) && size <= PAGE_SIZE) {
return valid_user_address(ptr);
} else {
unsigned long sum = size + (__force unsigned long)ptr; return valid_user_address(sum) && sum >= (__force unsigned long)ptr;
}
}
#define __access_ok __access_ok
/*
* Copy To/From Userspace
*/
/* Handles exceptions in both to and from, but doesn't do access_ok */
__must_check unsigned long
rep_movs_alternative(void *to, const void *from, unsigned len);
static __always_inline __must_check unsigned long
copy_user_generic(void *to, const void *from, unsigned long len)
{
stac();
/*
* If CPU has FSRM feature, use 'rep movs'.
* Otherwise, use rep_movs_alternative.
*/
asm volatile(
"1:\n\t"
ALTERNATIVE("rep movsb",
"call rep_movs_alternative", ALT_NOT(X86_FEATURE_FSRM))
"2:\n"
_ASM_EXTABLE_UA(1b, 2b)
:"+c" (len), "+D" (to), "+S" (from), ASM_CALL_CONSTRAINT
: : "memory", "rax");
clac();
return len;
}
static __always_inline __must_check unsigned long
raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
{
return copy_user_generic(dst, (__force void *)src, size);
}
static __always_inline __must_check unsigned long
raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
{
return copy_user_generic((__force void *)dst, src, size);
}
extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
unsigned size)
{
long ret;
kasan_check_write(dst, size);
stac();
ret = __copy_user_nocache(dst, src, size);
clac();
return ret;
}
static inline int
__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
{
kasan_check_write(dst, size);
return __copy_user_flushcache(dst, src, size);
}
/*
* Zero Userspace.
*/
__must_check unsigned long
rep_stos_alternative(void __user *addr, unsigned long len);
static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size)
{
might_fault();
stac();
/*
* No memory constraint because it doesn't change any memory gcc
* knows about.
*/
asm volatile(
"1:\n\t"
ALTERNATIVE("rep stosb",
"call rep_stos_alternative", ALT_NOT(X86_FEATURE_FSRS))
"2:\n"
_ASM_EXTABLE_UA(1b, 2b)
: "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT
: "a" (0));
clac();
return size;
}
static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
{
if (__access_ok(to, n)) return __clear_user(to, n);
return n;
}
#endif /* _ASM_X86_UACCESS_64_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Tty port functions
*/
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/tty_flip.h>
#include <linux/serial.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/bitops.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/serdev.h>
#include "tty.h"
static size_t tty_port_default_receive_buf(struct tty_port *port, const u8 *p,
const u8 *f, size_t count)
{
struct tty_struct *tty;
struct tty_ldisc *ld;
tty = READ_ONCE(port->itty);
if (!tty)
return 0;
ld = tty_ldisc_ref(tty);
if (!ld)
return 0;
count = tty_ldisc_receive_buf(ld, p, f, count);
tty_ldisc_deref(ld);
return count;
}
static void tty_port_default_lookahead_buf(struct tty_port *port, const u8 *p,
const u8 *f, size_t count)
{
struct tty_struct *tty;
struct tty_ldisc *ld;
tty = READ_ONCE(port->itty);
if (!tty)
return;
ld = tty_ldisc_ref(tty);
if (!ld)
return;
if (ld->ops->lookahead_buf)
ld->ops->lookahead_buf(ld->tty, p, f, count);
tty_ldisc_deref(ld);
}
static void tty_port_default_wakeup(struct tty_port *port)
{
struct tty_struct *tty = tty_port_tty_get(port);
if (tty) {
tty_wakeup(tty);
tty_kref_put(tty);
}
}
const struct tty_port_client_operations tty_port_default_client_ops = {
.receive_buf = tty_port_default_receive_buf,
.lookahead_buf = tty_port_default_lookahead_buf,
.write_wakeup = tty_port_default_wakeup,
};
EXPORT_SYMBOL_GPL(tty_port_default_client_ops);
/**
* tty_port_init - initialize tty_port
* @port: tty_port to initialize
*
* Initializes the state of struct tty_port. When a port was initialized using
* this function, one has to destroy the port by tty_port_destroy(). Either
* indirectly by using &tty_port refcounting (tty_port_put()) or directly if
* refcounting is not used.
*/
void tty_port_init(struct tty_port *port)
{
memset(port, 0, sizeof(*port));
tty_buffer_init(port);
init_waitqueue_head(&port->open_wait);
init_waitqueue_head(&port->delta_msr_wait);
mutex_init(&port->mutex);
mutex_init(&port->buf_mutex);
spin_lock_init(&port->lock);
port->close_delay = (50 * HZ) / 100;
port->closing_wait = (3000 * HZ) / 100;
port->client_ops = &tty_port_default_client_ops;
kref_init(&port->kref);
}
EXPORT_SYMBOL(tty_port_init);
/**
* tty_port_link_device - link tty and tty_port
* @port: tty_port of the device
* @driver: tty_driver for this device
* @index: index of the tty
*
* Provide the tty layer with a link from a tty (specified by @index) to a
* tty_port (@port). Use this only if neither tty_port_register_device() nor
* tty_port_install() is used in the driver. If used, this has to be called
* before tty_register_driver().
*/
void tty_port_link_device(struct tty_port *port,
struct tty_driver *driver, unsigned index)
{
if (WARN_ON(index >= driver->num))
return;
driver->ports[index] = port;
}
EXPORT_SYMBOL_GPL(tty_port_link_device);
/**
* tty_port_register_device - register tty device
* @port: tty_port of the device
* @driver: tty_driver for this device
* @index: index of the tty
* @device: parent if exists, otherwise NULL
*
* It is the same as tty_register_device() except the provided @port is linked
* to a concrete tty specified by @index. Use this or tty_port_install() (or
* both). Call tty_port_link_device() as a last resort.
*/
struct device *tty_port_register_device(struct tty_port *port,
struct tty_driver *driver, unsigned index,
struct device *device)
{
return tty_port_register_device_attr(port, driver, index, device, NULL, NULL);
}
EXPORT_SYMBOL_GPL(tty_port_register_device);
/**
* tty_port_register_device_attr - register tty device
* @port: tty_port of the device
* @driver: tty_driver for this device
* @index: index of the tty
* @device: parent if exists, otherwise NULL
* @drvdata: Driver data to be set to device.
* @attr_grp: Attribute group to be set on device.
*
* It is the same as tty_register_device_attr() except the provided @port is
* linked to a concrete tty specified by @index. Use this or tty_port_install()
* (or both). Call tty_port_link_device() as a last resort.
*/
struct device *tty_port_register_device_attr(struct tty_port *port,
struct tty_driver *driver, unsigned index,
struct device *device, void *drvdata,
const struct attribute_group **attr_grp)
{
tty_port_link_device(port, driver, index);
return tty_register_device_attr(driver, index, device, drvdata,
attr_grp);
}
EXPORT_SYMBOL_GPL(tty_port_register_device_attr);
/**
* tty_port_register_device_attr_serdev - register tty or serdev device
* @port: tty_port of the device
* @driver: tty_driver for this device
* @index: index of the tty
* @host: serial port hardware device
* @parent: parent if exists, otherwise NULL
* @drvdata: driver data for the device
* @attr_grp: attribute group for the device
*
* Register a serdev or tty device depending on if the parent device has any
* defined serdev clients or not.
*/
struct device *tty_port_register_device_attr_serdev(struct tty_port *port,
struct tty_driver *driver, unsigned index,
struct device *host, struct device *parent, void *drvdata,
const struct attribute_group **attr_grp)
{
struct device *dev;
tty_port_link_device(port, driver, index);
dev = serdev_tty_port_register(port, host, parent, driver, index);
if (PTR_ERR(dev) != -ENODEV) {
/* Skip creating cdev if we registered a serdev device */
return dev;
}
return tty_register_device_attr(driver, index, parent, drvdata,
attr_grp);
}
EXPORT_SYMBOL_GPL(tty_port_register_device_attr_serdev);
/**
* tty_port_register_device_serdev - register tty or serdev device
* @port: tty_port of the device
* @driver: tty_driver for this device
* @index: index of the tty
* @host: serial port hardware controller device
* @parent: parent if exists, otherwise NULL
*
* Register a serdev or tty device depending on if the parent device has any
* defined serdev clients or not.
*/
struct device *tty_port_register_device_serdev(struct tty_port *port,
struct tty_driver *driver, unsigned index,
struct device *host, struct device *parent)
{
return tty_port_register_device_attr_serdev(port, driver, index,
host, parent, NULL, NULL);
}
EXPORT_SYMBOL_GPL(tty_port_register_device_serdev);
/**
* tty_port_unregister_device - deregister a tty or serdev device
* @port: tty_port of the device
* @driver: tty_driver for this device
* @index: index of the tty
*
* If a tty or serdev device is registered with a call to
* tty_port_register_device_serdev() then this function must be called when
* the device is gone.
*/
void tty_port_unregister_device(struct tty_port *port,
struct tty_driver *driver, unsigned index)
{
int ret;
ret = serdev_tty_port_unregister(port);
if (ret == 0)
return;
tty_unregister_device(driver, index);
}
EXPORT_SYMBOL_GPL(tty_port_unregister_device);
int tty_port_alloc_xmit_buf(struct tty_port *port)
{
/* We may sleep in get_zeroed_page() */
mutex_lock(&port->buf_mutex);
if (port->xmit_buf == NULL) {
port->xmit_buf = (u8 *)get_zeroed_page(GFP_KERNEL);
if (port->xmit_buf)
kfifo_init(&port->xmit_fifo, port->xmit_buf, PAGE_SIZE);
}
mutex_unlock(&port->buf_mutex);
if (port->xmit_buf == NULL)
return -ENOMEM;
return 0;
}
EXPORT_SYMBOL(tty_port_alloc_xmit_buf);
void tty_port_free_xmit_buf(struct tty_port *port)
{
mutex_lock(&port->buf_mutex);
free_page((unsigned long)port->xmit_buf);
port->xmit_buf = NULL;
INIT_KFIFO(port->xmit_fifo);
mutex_unlock(&port->buf_mutex);
}
EXPORT_SYMBOL(tty_port_free_xmit_buf);
/**
* tty_port_destroy - destroy inited port
* @port: tty port to be destroyed
*
* When a port was initialized using tty_port_init(), one has to destroy the
* port by this function. Either indirectly by using &tty_port refcounting
* (tty_port_put()) or directly if refcounting is not used.
*/
void tty_port_destroy(struct tty_port *port)
{
tty_buffer_cancel_work(port);
tty_buffer_free_all(port);
}
EXPORT_SYMBOL(tty_port_destroy);
static void tty_port_destructor(struct kref *kref)
{
struct tty_port *port = container_of(kref, struct tty_port, kref);
/* check if last port ref was dropped before tty release */
if (WARN_ON(port->itty))
return;
free_page((unsigned long)port->xmit_buf);
tty_port_destroy(port);
if (port->ops && port->ops->destruct)
port->ops->destruct(port);
else
kfree(port);
}
/**
* tty_port_put - drop a reference to tty_port
* @port: port to drop a reference of (can be NULL)
*
* The final put will destroy and free up the @port using
* @port->ops->destruct() hook, or using kfree() if not provided.
*/
void tty_port_put(struct tty_port *port)
{
if (port)
kref_put(&port->kref, tty_port_destructor);
}
EXPORT_SYMBOL(tty_port_put);
/**
* tty_port_tty_get - get a tty reference
* @port: tty port
*
* Return a refcount protected tty instance or %NULL if the port is not
* associated with a tty (eg due to close or hangup).
*/
struct tty_struct *tty_port_tty_get(struct tty_port *port)
{
unsigned long flags;
struct tty_struct *tty;
spin_lock_irqsave(&port->lock, flags); tty = tty_kref_get(port->tty); spin_unlock_irqrestore(&port->lock, flags);
return tty;
}
EXPORT_SYMBOL(tty_port_tty_get);
/**
* tty_port_tty_set - set the tty of a port
* @port: tty port
* @tty: the tty
*
* Associate the port and tty pair. Manages any internal refcounts. Pass %NULL
* to deassociate a port.
*/
void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty)
{
unsigned long flags;
spin_lock_irqsave(&port->lock, flags);
tty_kref_put(port->tty);
port->tty = tty_kref_get(tty);
spin_unlock_irqrestore(&port->lock, flags);
}
EXPORT_SYMBOL(tty_port_tty_set);
/**
* tty_port_shutdown - internal helper to shutdown the device
* @port: tty port to be shut down
* @tty: the associated tty
*
* It is used by tty_port_hangup() and tty_port_close(). Its task is to
* shutdown the device if it was initialized (note consoles remain
* functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes
* @port->ops->shutdown().
*/
static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty)
{
mutex_lock(&port->mutex);
if (port->console)
goto out;
if (tty_port_initialized(port)) {
tty_port_set_initialized(port, false);
/*
* Drop DTR/RTS if HUPCL is set. This causes any attached
* modem to hang up the line.
*/
if (tty && C_HUPCL(tty))
tty_port_lower_dtr_rts(port);
if (port->ops->shutdown)
port->ops->shutdown(port);
}
out:
mutex_unlock(&port->mutex);
}
/**
* tty_port_hangup - hangup helper
* @port: tty port
*
* Perform port level tty hangup flag and count changes. Drop the tty
* reference.
*
* Caller holds tty lock.
*/
void tty_port_hangup(struct tty_port *port)
{
struct tty_struct *tty;
unsigned long flags;
spin_lock_irqsave(&port->lock, flags);
port->count = 0;
tty = port->tty;
if (tty)
set_bit(TTY_IO_ERROR, &tty->flags);
port->tty = NULL;
spin_unlock_irqrestore(&port->lock, flags);
tty_port_set_active(port, false);
tty_port_shutdown(port, tty);
tty_kref_put(tty);
wake_up_interruptible(&port->open_wait);
wake_up_interruptible(&port->delta_msr_wait);
}
EXPORT_SYMBOL(tty_port_hangup);
/**
* tty_port_tty_hangup - helper to hang up a tty
* @port: tty port
* @check_clocal: hang only ttys with %CLOCAL unset?
*/
void tty_port_tty_hangup(struct tty_port *port, bool check_clocal)
{
struct tty_struct *tty = tty_port_tty_get(port);
if (tty && (!check_clocal || !C_CLOCAL(tty)))
tty_hangup(tty);
tty_kref_put(tty);
}
EXPORT_SYMBOL_GPL(tty_port_tty_hangup);
/**
* tty_port_tty_wakeup - helper to wake up a tty
* @port: tty port
*/
void tty_port_tty_wakeup(struct tty_port *port)
{
port->client_ops->write_wakeup(port);
}
EXPORT_SYMBOL_GPL(tty_port_tty_wakeup);
/**
* tty_port_carrier_raised - carrier raised check
* @port: tty port
*
* Wrapper for the carrier detect logic. For the moment this is used
* to hide some internal details. This will eventually become entirely
* internal to the tty port.
*/
bool tty_port_carrier_raised(struct tty_port *port)
{
if (port->ops->carrier_raised == NULL)
return true;
return port->ops->carrier_raised(port);
}
EXPORT_SYMBOL(tty_port_carrier_raised);
/**
* tty_port_raise_dtr_rts - Raise DTR/RTS
* @port: tty port
*
* Wrapper for the DTR/RTS raise logic. For the moment this is used to hide
* some internal details. This will eventually become entirely internal to the
* tty port.
*/
void tty_port_raise_dtr_rts(struct tty_port *port)
{
if (port->ops->dtr_rts)
port->ops->dtr_rts(port, true);
}
EXPORT_SYMBOL(tty_port_raise_dtr_rts);
/**
* tty_port_lower_dtr_rts - Lower DTR/RTS
* @port: tty port
*
* Wrapper for the DTR/RTS raise logic. For the moment this is used to hide
* some internal details. This will eventually become entirely internal to the
* tty port.
*/
void tty_port_lower_dtr_rts(struct tty_port *port)
{
if (port->ops->dtr_rts)
port->ops->dtr_rts(port, false);
}
EXPORT_SYMBOL(tty_port_lower_dtr_rts);
/**
* tty_port_block_til_ready - Waiting logic for tty open
* @port: the tty port being opened
* @tty: the tty device being bound
* @filp: the file pointer of the opener or %NULL
*
* Implement the core POSIX/SuS tty behaviour when opening a tty device.
* Handles:
*
* - hangup (both before and during)
* - non blocking open
* - rts/dtr/dcd
* - signals
* - port flags and counts
*
* The passed @port must implement the @port->ops->carrier_raised method if it
* can do carrier detect and the @port->ops->dtr_rts method if it supports
* software management of these lines. Note that the dtr/rts raise is done each
* iteration as a hangup may have previously dropped them while we wait.
*
* Caller holds tty lock.
*
* Note: May drop and reacquire tty lock when blocking, so @tty and @port may
* have changed state (eg., may have been hung up).
*/
int tty_port_block_til_ready(struct tty_port *port,
struct tty_struct *tty, struct file *filp)
{
int do_clocal = 0, retval;
unsigned long flags;
DEFINE_WAIT(wait);
/* if non-blocking mode is set we can pass directly to open unless
* the port has just hung up or is in another error state.
*/
if (tty_io_error(tty)) {
tty_port_set_active(port, true);
return 0;
}
if (filp == NULL || (filp->f_flags & O_NONBLOCK)) {
/* Indicate we are open */
if (C_BAUD(tty))
tty_port_raise_dtr_rts(port);
tty_port_set_active(port, true);
return 0;
}
if (C_CLOCAL(tty))
do_clocal = 1;
/* Block waiting until we can proceed. We may need to wait for the
* carrier, but we must also wait for any close that is in progress
* before the next open may complete.
*/
retval = 0;
/* The port lock protects the port counts */
spin_lock_irqsave(&port->lock, flags);
port->count--;
port->blocked_open++;
spin_unlock_irqrestore(&port->lock, flags);
while (1) {
/* Indicate we are open */
if (C_BAUD(tty) && tty_port_initialized(port))
tty_port_raise_dtr_rts(port);
prepare_to_wait(&port->open_wait, &wait, TASK_INTERRUPTIBLE);
/* Check for a hangup or uninitialised port.
* Return accordingly.
*/
if (tty_hung_up_p(filp) || !tty_port_initialized(port)) {
if (port->flags & ASYNC_HUP_NOTIFY)
retval = -EAGAIN;
else
retval = -ERESTARTSYS;
break;
}
/*
* Probe the carrier. For devices with no carrier detect
* tty_port_carrier_raised will always return true.
* Never ask drivers if CLOCAL is set, this causes troubles
* on some hardware.
*/
if (do_clocal || tty_port_carrier_raised(port))
break;
if (signal_pending(current)) {
retval = -ERESTARTSYS;
break;
}
tty_unlock(tty);
schedule();
tty_lock(tty);
}
finish_wait(&port->open_wait, &wait);
/* Update counts. A parallel hangup will have set count to zero and
* we must not mess that up further.
*/
spin_lock_irqsave(&port->lock, flags);
if (!tty_hung_up_p(filp))
port->count++;
port->blocked_open--;
spin_unlock_irqrestore(&port->lock, flags);
if (retval == 0)
tty_port_set_active(port, true);
return retval;
}
EXPORT_SYMBOL(tty_port_block_til_ready);
static void tty_port_drain_delay(struct tty_port *port, struct tty_struct *tty)
{
unsigned int bps = tty_get_baud_rate(tty);
long timeout;
if (bps > 1200) {
timeout = (HZ * 10 * port->drain_delay) / bps;
timeout = max_t(long, timeout, HZ / 10);
} else {
timeout = 2 * HZ;
}
schedule_timeout_interruptible(timeout);
}
/**
* tty_port_close_start - helper for tty->ops->close, part 1/2
* @port: tty_port of the device
* @tty: tty being closed
* @filp: passed file pointer
*
* Decrements and checks open count. Flushes the port if this is the last
* close. That means, dropping the data from the outpu buffer on the device and
* waiting for sending logic to finish. The rest of close handling is performed
* in tty_port_close_end().
*
* Locking: Caller holds tty lock.
*
* Return: 1 if this is the last close, otherwise 0
*/
int tty_port_close_start(struct tty_port *port,
struct tty_struct *tty, struct file *filp)
{
unsigned long flags;
if (tty_hung_up_p(filp))
return 0;
spin_lock_irqsave(&port->lock, flags);
if (tty->count == 1 && port->count != 1) {
tty_warn(tty, "%s: tty->count = 1 port count = %d\n", __func__,
port->count);
port->count = 1;
}
if (--port->count < 0) {
tty_warn(tty, "%s: bad port count (%d)\n", __func__,
port->count);
port->count = 0;
}
if (port->count) {
spin_unlock_irqrestore(&port->lock, flags);
return 0;
}
spin_unlock_irqrestore(&port->lock, flags);
tty->closing = 1;
if (tty_port_initialized(port)) {
/* Don't block on a stalled port, just pull the chain */
if (tty->flow.tco_stopped)
tty_driver_flush_buffer(tty);
if (port->closing_wait != ASYNC_CLOSING_WAIT_NONE)
tty_wait_until_sent(tty, port->closing_wait);
if (port->drain_delay)
tty_port_drain_delay(port, tty);
}
/* Flush the ldisc buffering */
tty_ldisc_flush(tty);
/* Report to caller this is the last port reference */
return 1;
}
EXPORT_SYMBOL(tty_port_close_start);
/**
* tty_port_close_end - helper for tty->ops->close, part 2/2
* @port: tty_port of the device
* @tty: tty being closed
*
* This is a continuation of the first part: tty_port_close_start(). This
* should be called after turning off the device. It flushes the data from the
* line discipline and delays the close by @port->close_delay.
*
* Locking: Caller holds tty lock.
*/
void tty_port_close_end(struct tty_port *port, struct tty_struct *tty)
{
unsigned long flags;
tty_ldisc_flush(tty);
tty->closing = 0;
spin_lock_irqsave(&port->lock, flags);
if (port->blocked_open) {
spin_unlock_irqrestore(&port->lock, flags);
if (port->close_delay)
msleep_interruptible(jiffies_to_msecs(port->close_delay));
spin_lock_irqsave(&port->lock, flags);
wake_up_interruptible(&port->open_wait);
}
spin_unlock_irqrestore(&port->lock, flags);
tty_port_set_active(port, false);
}
EXPORT_SYMBOL(tty_port_close_end);
/**
* tty_port_close - generic tty->ops->close handler
* @port: tty_port of the device
* @tty: tty being closed
* @filp: passed file pointer
*
* It is a generic helper to be used in driver's @tty->ops->close. It wraps a
* sequence of tty_port_close_start(), tty_port_shutdown(), and
* tty_port_close_end(). The latter two are called only if this is the last
* close. See the respective functions for the details.
*
* Locking: Caller holds tty lock
*/
void tty_port_close(struct tty_port *port, struct tty_struct *tty,
struct file *filp)
{
if (tty_port_close_start(port, tty, filp) == 0)
return;
tty_port_shutdown(port, tty);
if (!port->console)
set_bit(TTY_IO_ERROR, &tty->flags);
tty_port_close_end(port, tty);
tty_port_tty_set(port, NULL);
}
EXPORT_SYMBOL(tty_port_close);
/**
* tty_port_install - generic tty->ops->install handler
* @port: tty_port of the device
* @driver: tty_driver for this device
* @tty: tty to be installed
*
* It is the same as tty_standard_install() except the provided @port is linked
* to a concrete tty specified by @tty. Use this or tty_port_register_device()
* (or both). Call tty_port_link_device() as a last resort.
*/
int tty_port_install(struct tty_port *port, struct tty_driver *driver,
struct tty_struct *tty)
{
tty->port = port;
return tty_standard_install(driver, tty);
}
EXPORT_SYMBOL_GPL(tty_port_install);
/**
* tty_port_open - generic tty->ops->open handler
* @port: tty_port of the device
* @tty: tty to be opened
* @filp: passed file pointer
*
* It is a generic helper to be used in driver's @tty->ops->open. It activates
* the devices using @port->ops->activate if not active already. And waits for
* the device to be ready using tty_port_block_til_ready() (e.g. raises
* DTR/CTS and waits for carrier).
*
* Note that @port->ops->shutdown is not called when @port->ops->activate
* returns an error (on the contrary, @tty->ops->close is).
*
* Locking: Caller holds tty lock.
*
* Note: may drop and reacquire tty lock (in tty_port_block_til_ready()) so
* @tty and @port may have changed state (eg., may be hung up now).
*/
int tty_port_open(struct tty_port *port, struct tty_struct *tty,
struct file *filp)
{
spin_lock_irq(&port->lock);
++port->count;
spin_unlock_irq(&port->lock);
tty_port_tty_set(port, tty);
/*
* Do the device-specific open only if the hardware isn't
* already initialized. Serialize open and shutdown using the
* port mutex.
*/
mutex_lock(&port->mutex);
if (!tty_port_initialized(port)) {
clear_bit(TTY_IO_ERROR, &tty->flags);
if (port->ops->activate) {
int retval = port->ops->activate(port, tty);
if (retval) {
mutex_unlock(&port->mutex);
return retval;
}
}
tty_port_set_initialized(port, true);
}
mutex_unlock(&port->mutex);
return tty_port_block_til_ready(port, tty, filp);
}
EXPORT_SYMBOL(tty_port_open);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_NETLINK_H
#define __NET_NETLINK_H
#include <linux/types.h>
#include <linux/netlink.h>
#include <linux/jiffies.h>
#include <linux/in6.h>
/* ========================================================================
* Netlink Messages and Attributes Interface (As Seen On TV)
* ------------------------------------------------------------------------
* Messages Interface
* ------------------------------------------------------------------------
*
* Message Format:
* <--- nlmsg_total_size(payload) --->
* <-- nlmsg_msg_size(payload) ->
* +----------+- - -+-------------+- - -+-------- - -
* | nlmsghdr | Pad | Payload | Pad | nlmsghdr
* +----------+- - -+-------------+- - -+-------- - -
* nlmsg_data(nlh)---^ ^
* nlmsg_next(nlh)-----------------------+
*
* Payload Format:
* <---------------------- nlmsg_len(nlh) --------------------->
* <------ hdrlen ------> <- nlmsg_attrlen(nlh, hdrlen) ->
* +----------------------+- - -+--------------------------------+
* | Family Header | Pad | Attributes |
* +----------------------+- - -+--------------------------------+
* nlmsg_attrdata(nlh, hdrlen)---^
*
* Data Structures:
* struct nlmsghdr netlink message header
*
* Message Construction:
* nlmsg_new() create a new netlink message
* nlmsg_put() add a netlink message to an skb
* nlmsg_put_answer() callback based nlmsg_put()
* nlmsg_end() finalize netlink message
* nlmsg_get_pos() return current position in message
* nlmsg_trim() trim part of message
* nlmsg_cancel() cancel message construction
* nlmsg_free() free a netlink message
*
* Message Sending:
* nlmsg_multicast() multicast message to several groups
* nlmsg_unicast() unicast a message to a single socket
* nlmsg_notify() send notification message
*
* Message Length Calculations:
* nlmsg_msg_size(payload) length of message w/o padding
* nlmsg_total_size(payload) length of message w/ padding
* nlmsg_padlen(payload) length of padding at tail
*
* Message Payload Access:
* nlmsg_data(nlh) head of message payload
* nlmsg_len(nlh) length of message payload
* nlmsg_attrdata(nlh, hdrlen) head of attributes data
* nlmsg_attrlen(nlh, hdrlen) length of attributes data
*
* Message Parsing:
* nlmsg_ok(nlh, remaining) does nlh fit into remaining bytes?
* nlmsg_next(nlh, remaining) get next netlink message
* nlmsg_parse() parse attributes of a message
* nlmsg_find_attr() find an attribute in a message
* nlmsg_for_each_msg() loop over all messages
* nlmsg_validate() validate netlink message incl. attrs
* nlmsg_for_each_attr() loop over all attributes
*
* Misc:
* nlmsg_report() report back to application?
*
* ------------------------------------------------------------------------
* Attributes Interface
* ------------------------------------------------------------------------
*
* Attribute Format:
* <------- nla_total_size(payload) ------->
* <---- nla_attr_size(payload) ----->
* +----------+- - -+- - - - - - - - - +- - -+-------- - -
* | Header | Pad | Payload | Pad | Header
* +----------+- - -+- - - - - - - - - +- - -+-------- - -
* <- nla_len(nla) -> ^
* nla_data(nla)----^ |
* nla_next(nla)-----------------------------'
*
* Data Structures:
* struct nlattr netlink attribute header
*
* Attribute Construction:
* nla_reserve(skb, type, len) reserve room for an attribute
* nla_reserve_nohdr(skb, len) reserve room for an attribute w/o hdr
* nla_put(skb, type, len, data) add attribute to skb
* nla_put_nohdr(skb, len, data) add attribute w/o hdr
* nla_append(skb, len, data) append data to skb
*
* Attribute Construction for Basic Types:
* nla_put_u8(skb, type, value) add u8 attribute to skb
* nla_put_u16(skb, type, value) add u16 attribute to skb
* nla_put_u32(skb, type, value) add u32 attribute to skb
* nla_put_u64_64bit(skb, type,
* value, padattr) add u64 attribute to skb
* nla_put_s8(skb, type, value) add s8 attribute to skb
* nla_put_s16(skb, type, value) add s16 attribute to skb
* nla_put_s32(skb, type, value) add s32 attribute to skb
* nla_put_s64(skb, type, value,
* padattr) add s64 attribute to skb
* nla_put_string(skb, type, str) add string attribute to skb
* nla_put_flag(skb, type) add flag attribute to skb
* nla_put_msecs(skb, type, jiffies,
* padattr) add msecs attribute to skb
* nla_put_in_addr(skb, type, addr) add IPv4 address attribute to skb
* nla_put_in6_addr(skb, type, addr) add IPv6 address attribute to skb
*
* Nested Attributes Construction:
* nla_nest_start(skb, type) start a nested attribute
* nla_nest_end(skb, nla) finalize a nested attribute
* nla_nest_cancel(skb, nla) cancel nested attribute construction
*
* Attribute Length Calculations:
* nla_attr_size(payload) length of attribute w/o padding
* nla_total_size(payload) length of attribute w/ padding
* nla_padlen(payload) length of padding
*
* Attribute Payload Access:
* nla_data(nla) head of attribute payload
* nla_len(nla) length of attribute payload
*
* Attribute Payload Access for Basic Types:
* nla_get_uint(nla) get payload for a uint attribute
* nla_get_sint(nla) get payload for a sint attribute
* nla_get_u8(nla) get payload for a u8 attribute
* nla_get_u16(nla) get payload for a u16 attribute
* nla_get_u32(nla) get payload for a u32 attribute
* nla_get_u64(nla) get payload for a u64 attribute
* nla_get_s8(nla) get payload for a s8 attribute
* nla_get_s16(nla) get payload for a s16 attribute
* nla_get_s32(nla) get payload for a s32 attribute
* nla_get_s64(nla) get payload for a s64 attribute
* nla_get_flag(nla) return 1 if flag is true
* nla_get_msecs(nla) get payload for a msecs attribute
*
* Attribute Misc:
* nla_memcpy(dest, nla, count) copy attribute into memory
* nla_memcmp(nla, data, size) compare attribute with memory area
* nla_strscpy(dst, nla, size) copy attribute to a sized string
* nla_strcmp(nla, str) compare attribute with string
*
* Attribute Parsing:
* nla_ok(nla, remaining) does nla fit into remaining bytes?
* nla_next(nla, remaining) get next netlink attribute
* nla_validate() validate a stream of attributes
* nla_validate_nested() validate a stream of nested attributes
* nla_find() find attribute in stream of attributes
* nla_find_nested() find attribute in nested attributes
* nla_parse() parse and validate stream of attrs
* nla_parse_nested() parse nested attributes
* nla_for_each_attr() loop over all attributes
* nla_for_each_nested() loop over the nested attributes
*=========================================================================
*/
/**
* Standard attribute types to specify validation policy
*/
enum {
NLA_UNSPEC,
NLA_U8,
NLA_U16,
NLA_U32,
NLA_U64,
NLA_STRING,
NLA_FLAG,
NLA_MSECS,
NLA_NESTED,
NLA_NESTED_ARRAY,
NLA_NUL_STRING,
NLA_BINARY,
NLA_S8,
NLA_S16,
NLA_S32,
NLA_S64,
NLA_BITFIELD32,
NLA_REJECT,
NLA_BE16,
NLA_BE32,
NLA_SINT,
NLA_UINT,
__NLA_TYPE_MAX,
};
#define NLA_TYPE_MAX (__NLA_TYPE_MAX - 1)
struct netlink_range_validation {
u64 min, max;
};
struct netlink_range_validation_signed {
s64 min, max;
};
enum nla_policy_validation {
NLA_VALIDATE_NONE,
NLA_VALIDATE_RANGE,
NLA_VALIDATE_RANGE_WARN_TOO_LONG,
NLA_VALIDATE_MIN,
NLA_VALIDATE_MAX,
NLA_VALIDATE_MASK,
NLA_VALIDATE_RANGE_PTR,
NLA_VALIDATE_FUNCTION,
};
/**
* struct nla_policy - attribute validation policy
* @type: Type of attribute or NLA_UNSPEC
* @validation_type: type of attribute validation done in addition to
* type-specific validation (e.g. range, function call), see
* &enum nla_policy_validation
* @len: Type specific length of payload
*
* Policies are defined as arrays of this struct, the array must be
* accessible by attribute type up to the highest identifier to be expected.
*
* Meaning of `len' field:
* NLA_STRING Maximum length of string
* NLA_NUL_STRING Maximum length of string (excluding NUL)
* NLA_FLAG Unused
* NLA_BINARY Maximum length of attribute payload
* (but see also below with the validation type)
* NLA_NESTED,
* NLA_NESTED_ARRAY Length verification is done by checking len of
* nested header (or empty); len field is used if
* nested_policy is also used, for the max attr
* number in the nested policy.
* NLA_SINT, NLA_UINT,
* NLA_U8, NLA_U16,
* NLA_U32, NLA_U64,
* NLA_S8, NLA_S16,
* NLA_S32, NLA_S64,
* NLA_BE16, NLA_BE32,
* NLA_MSECS Leaving the length field zero will verify the
* given type fits, using it verifies minimum length
* just like "All other"
* NLA_BITFIELD32 Unused
* NLA_REJECT Unused
* All other Minimum length of attribute payload
*
* Meaning of validation union:
* NLA_BITFIELD32 This is a 32-bit bitmap/bitselector attribute and
* `bitfield32_valid' is the u32 value of valid flags
* NLA_REJECT This attribute is always rejected and `reject_message'
* may point to a string to report as the error instead
* of the generic one in extended ACK.
* NLA_NESTED `nested_policy' to a nested policy to validate, must
* also set `len' to the max attribute number. Use the
* provided NLA_POLICY_NESTED() macro.
* Note that nla_parse() will validate, but of course not
* parse, the nested sub-policies.
* NLA_NESTED_ARRAY `nested_policy' points to a nested policy to validate,
* must also set `len' to the max attribute number. Use
* the provided NLA_POLICY_NESTED_ARRAY() macro.
* The difference to NLA_NESTED is the structure:
* NLA_NESTED has the nested attributes directly inside
* while an array has the nested attributes at another
* level down and the attribute types directly in the
* nesting don't matter.
* NLA_UINT,
* NLA_U8,
* NLA_U16,
* NLA_U32,
* NLA_U64,
* NLA_BE16,
* NLA_BE32,
* NLA_SINT,
* NLA_S8,
* NLA_S16,
* NLA_S32,
* NLA_S64 The `min' and `max' fields are used depending on the
* validation_type field, if that is min/max/range then
* the min, max or both are used (respectively) to check
* the value of the integer attribute.
* Note that in the interest of code simplicity and
* struct size both limits are s16, so you cannot
* enforce a range that doesn't fall within the range
* of s16 - do that using the NLA_POLICY_FULL_RANGE()
* or NLA_POLICY_FULL_RANGE_SIGNED() macros instead.
* Use the NLA_POLICY_MIN(), NLA_POLICY_MAX() and
* NLA_POLICY_RANGE() macros.
* NLA_UINT,
* NLA_U8,
* NLA_U16,
* NLA_U32,
* NLA_U64 If the validation_type field instead is set to
* NLA_VALIDATE_RANGE_PTR, `range' must be a pointer
* to a struct netlink_range_validation that indicates
* the min/max values.
* Use NLA_POLICY_FULL_RANGE().
* NLA_SINT,
* NLA_S8,
* NLA_S16,
* NLA_S32,
* NLA_S64 If the validation_type field instead is set to
* NLA_VALIDATE_RANGE_PTR, `range_signed' must be a
* pointer to a struct netlink_range_validation_signed
* that indicates the min/max values.
* Use NLA_POLICY_FULL_RANGE_SIGNED().
*
* NLA_BINARY If the validation type is like the ones for integers
* above, then the min/max length (not value like for
* integers) of the attribute is enforced.
*
* All other Unused - but note that it's a union
*
* Meaning of `validate' field, use via NLA_POLICY_VALIDATE_FN:
* NLA_BINARY Validation function called for the attribute.
* All other Unused - but note that it's a union
*
* Example:
*
* static const u32 myvalidflags = 0xff231023;
*
* static const struct nla_policy my_policy[ATTR_MAX+1] = {
* [ATTR_FOO] = { .type = NLA_U16 },
* [ATTR_BAR] = { .type = NLA_STRING, .len = BARSIZ },
* [ATTR_BAZ] = NLA_POLICY_EXACT_LEN(sizeof(struct mystruct)),
* [ATTR_GOO] = NLA_POLICY_BITFIELD32(myvalidflags),
* };
*/
struct nla_policy {
u8 type;
u8 validation_type;
u16 len;
union {
/**
* @strict_start_type: first attribute to validate strictly
*
* This entry is special, and used for the attribute at index 0
* only, and specifies special data about the policy, namely it
* specifies the "boundary type" where strict length validation
* starts for any attribute types >= this value, also, strict
* nesting validation starts here.
*
* Additionally, it means that NLA_UNSPEC is actually NLA_REJECT
* for any types >= this, so need to use NLA_POLICY_MIN_LEN() to
* get the previous pure { .len = xyz } behaviour. The advantage
* of this is that types not specified in the policy will be
* rejected.
*
* For completely new families it should be set to 1 so that the
* validation is enforced for all attributes. For existing ones
* it should be set at least when new attributes are added to
* the enum used by the policy, and be set to the new value that
* was added to enforce strict validation from thereon.
*/
u16 strict_start_type;
/* private: use NLA_POLICY_*() to set */
const u32 bitfield32_valid;
const u32 mask;
const char *reject_message;
const struct nla_policy *nested_policy;
const struct netlink_range_validation *range;
const struct netlink_range_validation_signed *range_signed;
struct {
s16 min, max;
};
int (*validate)(const struct nlattr *attr,
struct netlink_ext_ack *extack);
};
};
#define NLA_POLICY_ETH_ADDR NLA_POLICY_EXACT_LEN(ETH_ALEN)
#define NLA_POLICY_ETH_ADDR_COMPAT NLA_POLICY_EXACT_LEN_WARN(ETH_ALEN)
#define _NLA_POLICY_NESTED(maxattr, policy) \
{ .type = NLA_NESTED, .nested_policy = policy, .len = maxattr }
#define _NLA_POLICY_NESTED_ARRAY(maxattr, policy) \
{ .type = NLA_NESTED_ARRAY, .nested_policy = policy, .len = maxattr }
#define NLA_POLICY_NESTED(policy) \
_NLA_POLICY_NESTED(ARRAY_SIZE(policy) - 1, policy)
#define NLA_POLICY_NESTED_ARRAY(policy) \
_NLA_POLICY_NESTED_ARRAY(ARRAY_SIZE(policy) - 1, policy)
#define NLA_POLICY_BITFIELD32(valid) \
{ .type = NLA_BITFIELD32, .bitfield32_valid = valid }
#define __NLA_IS_UINT_TYPE(tp) \
(tp == NLA_U8 || tp == NLA_U16 || tp == NLA_U32 || \
tp == NLA_U64 || tp == NLA_UINT || \
tp == NLA_BE16 || tp == NLA_BE32)
#define __NLA_IS_SINT_TYPE(tp) \
(tp == NLA_S8 || tp == NLA_S16 || tp == NLA_S32 || tp == NLA_S64 || \
tp == NLA_SINT)
#define __NLA_ENSURE(condition) BUILD_BUG_ON_ZERO(!(condition))
#define NLA_ENSURE_UINT_TYPE(tp) \
(__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp)) + tp)
#define NLA_ENSURE_UINT_OR_BINARY_TYPE(tp) \
(__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp) || \
tp == NLA_MSECS || \
tp == NLA_BINARY) + tp)
#define NLA_ENSURE_SINT_TYPE(tp) \
(__NLA_ENSURE(__NLA_IS_SINT_TYPE(tp)) + tp)
#define NLA_ENSURE_INT_OR_BINARY_TYPE(tp) \
(__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp) || \
__NLA_IS_SINT_TYPE(tp) || \
tp == NLA_MSECS || \
tp == NLA_BINARY) + tp)
#define NLA_ENSURE_NO_VALIDATION_PTR(tp) \
(__NLA_ENSURE(tp != NLA_BITFIELD32 && \
tp != NLA_REJECT && \
tp != NLA_NESTED && \
tp != NLA_NESTED_ARRAY) + tp)
#define NLA_POLICY_RANGE(tp, _min, _max) { \
.type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \
.validation_type = NLA_VALIDATE_RANGE, \
.min = _min, \
.max = _max \
}
#define NLA_POLICY_FULL_RANGE(tp, _range) { \
.type = NLA_ENSURE_UINT_OR_BINARY_TYPE(tp), \
.validation_type = NLA_VALIDATE_RANGE_PTR, \
.range = _range, \
}
#define NLA_POLICY_FULL_RANGE_SIGNED(tp, _range) { \
.type = NLA_ENSURE_SINT_TYPE(tp), \
.validation_type = NLA_VALIDATE_RANGE_PTR, \
.range_signed = _range, \
}
#define NLA_POLICY_MIN(tp, _min) { \
.type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \
.validation_type = NLA_VALIDATE_MIN, \
.min = _min, \
}
#define NLA_POLICY_MAX(tp, _max) { \
.type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \
.validation_type = NLA_VALIDATE_MAX, \
.max = _max, \
}
#define NLA_POLICY_MASK(tp, _mask) { \
.type = NLA_ENSURE_UINT_TYPE(tp), \
.validation_type = NLA_VALIDATE_MASK, \
.mask = _mask, \
}
#define NLA_POLICY_VALIDATE_FN(tp, fn, ...) { \
.type = NLA_ENSURE_NO_VALIDATION_PTR(tp), \
.validation_type = NLA_VALIDATE_FUNCTION, \
.validate = fn, \
.len = __VA_ARGS__ + 0, \
}
#define NLA_POLICY_EXACT_LEN(_len) NLA_POLICY_RANGE(NLA_BINARY, _len, _len)
#define NLA_POLICY_EXACT_LEN_WARN(_len) { \
.type = NLA_BINARY, \
.validation_type = NLA_VALIDATE_RANGE_WARN_TOO_LONG, \
.min = _len, \
.max = _len \
}
#define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len)
/**
* struct nl_info - netlink source information
* @nlh: Netlink message header of original request
* @nl_net: Network namespace
* @portid: Netlink PORTID of requesting application
* @skip_notify: Skip netlink notifications to user space
* @skip_notify_kernel: Skip selected in-kernel notifications
*/
struct nl_info {
struct nlmsghdr *nlh;
struct net *nl_net;
u32 portid;
u8 skip_notify:1,
skip_notify_kernel:1;
};
/**
* enum netlink_validation - netlink message/attribute validation levels
* @NL_VALIDATE_LIBERAL: Old-style "be liberal" validation, not caring about
* extra data at the end of the message, attributes being longer than
* they should be, or unknown attributes being present.
* @NL_VALIDATE_TRAILING: Reject junk data encountered after attribute parsing.
* @NL_VALIDATE_MAXTYPE: Reject attributes > max type; Together with _TRAILING
* this is equivalent to the old nla_parse_strict()/nlmsg_parse_strict().
* @NL_VALIDATE_UNSPEC: Reject attributes with NLA_UNSPEC in the policy.
* This can safely be set by the kernel when the given policy has no
* NLA_UNSPEC anymore, and can thus be used to ensure policy entries
* are enforced going forward.
* @NL_VALIDATE_STRICT_ATTRS: strict attribute policy parsing (e.g.
* U8, U16, U32 must have exact size, etc.)
* @NL_VALIDATE_NESTED: Check that NLA_F_NESTED is set for NLA_NESTED(_ARRAY)
* and unset for other policies.
*/
enum netlink_validation {
NL_VALIDATE_LIBERAL = 0,
NL_VALIDATE_TRAILING = BIT(0),
NL_VALIDATE_MAXTYPE = BIT(1),
NL_VALIDATE_UNSPEC = BIT(2),
NL_VALIDATE_STRICT_ATTRS = BIT(3),
NL_VALIDATE_NESTED = BIT(4),
};
#define NL_VALIDATE_DEPRECATED_STRICT (NL_VALIDATE_TRAILING |\
NL_VALIDATE_MAXTYPE)
#define NL_VALIDATE_STRICT (NL_VALIDATE_TRAILING |\
NL_VALIDATE_MAXTYPE |\
NL_VALIDATE_UNSPEC |\
NL_VALIDATE_STRICT_ATTRS |\
NL_VALIDATE_NESTED)
int netlink_rcv_skb(struct sk_buff *skb,
int (*cb)(struct sk_buff *, struct nlmsghdr *,
struct netlink_ext_ack *));
int nlmsg_notify(struct sock *sk, struct sk_buff *skb, u32 portid,
unsigned int group, int report, gfp_t flags);
int __nla_validate(const struct nlattr *head, int len, int maxtype,
const struct nla_policy *policy, unsigned int validate,
struct netlink_ext_ack *extack);
int __nla_parse(struct nlattr **tb, int maxtype, const struct nlattr *head,
int len, const struct nla_policy *policy, unsigned int validate,
struct netlink_ext_ack *extack);
int nla_policy_len(const struct nla_policy *, int);
struct nlattr *nla_find(const struct nlattr *head, int len, int attrtype);
ssize_t nla_strscpy(char *dst, const struct nlattr *nla, size_t dstsize);
char *nla_strdup(const struct nlattr *nla, gfp_t flags);
int nla_memcpy(void *dest, const struct nlattr *src, int count);
int nla_memcmp(const struct nlattr *nla, const void *data, size_t size);
int nla_strcmp(const struct nlattr *nla, const char *str);
struct nlattr *__nla_reserve(struct sk_buff *skb, int attrtype, int attrlen);
struct nlattr *__nla_reserve_64bit(struct sk_buff *skb, int attrtype,
int attrlen, int padattr);
void *__nla_reserve_nohdr(struct sk_buff *skb, int attrlen);
struct nlattr *nla_reserve(struct sk_buff *skb, int attrtype, int attrlen);
struct nlattr *nla_reserve_64bit(struct sk_buff *skb, int attrtype,
int attrlen, int padattr);
void *nla_reserve_nohdr(struct sk_buff *skb, int attrlen);
void __nla_put(struct sk_buff *skb, int attrtype, int attrlen,
const void *data);
void __nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen,
const void *data, int padattr);
void __nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data);
int nla_put(struct sk_buff *skb, int attrtype, int attrlen, const void *data);
int nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen,
const void *data, int padattr);
int nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data);
int nla_append(struct sk_buff *skb, int attrlen, const void *data);
/**************************************************************************
* Netlink Messages
**************************************************************************/
/**
* nlmsg_msg_size - length of netlink message not including padding
* @payload: length of message payload
*/
static inline int nlmsg_msg_size(int payload)
{
return NLMSG_HDRLEN + payload;
}
/**
* nlmsg_total_size - length of netlink message including padding
* @payload: length of message payload
*/
static inline int nlmsg_total_size(int payload)
{
return NLMSG_ALIGN(nlmsg_msg_size(payload));
}
/**
* nlmsg_padlen - length of padding at the message's tail
* @payload: length of message payload
*/
static inline int nlmsg_padlen(int payload)
{
return nlmsg_total_size(payload) - nlmsg_msg_size(payload);
}
/**
* nlmsg_data - head of message payload
* @nlh: netlink message header
*/
static inline void *nlmsg_data(const struct nlmsghdr *nlh)
{
return (unsigned char *) nlh + NLMSG_HDRLEN;
}
/**
* nlmsg_len - length of message payload
* @nlh: netlink message header
*/
static inline int nlmsg_len(const struct nlmsghdr *nlh)
{
return nlh->nlmsg_len - NLMSG_HDRLEN;
}
/**
* nlmsg_attrdata - head of attributes data
* @nlh: netlink message header
* @hdrlen: length of family specific header
*/
static inline struct nlattr *nlmsg_attrdata(const struct nlmsghdr *nlh,
int hdrlen)
{
unsigned char *data = nlmsg_data(nlh);
return (struct nlattr *) (data + NLMSG_ALIGN(hdrlen));
}
/**
* nlmsg_attrlen - length of attributes data
* @nlh: netlink message header
* @hdrlen: length of family specific header
*/
static inline int nlmsg_attrlen(const struct nlmsghdr *nlh, int hdrlen)
{
return nlmsg_len(nlh) - NLMSG_ALIGN(hdrlen);
}
/**
* nlmsg_ok - check if the netlink message fits into the remaining bytes
* @nlh: netlink message header
* @remaining: number of bytes remaining in message stream
*/
static inline int nlmsg_ok(const struct nlmsghdr *nlh, int remaining)
{
return (remaining >= (int) sizeof(struct nlmsghdr) &&
nlh->nlmsg_len >= sizeof(struct nlmsghdr) &&
nlh->nlmsg_len <= remaining);
}
/**
* nlmsg_next - next netlink message in message stream
* @nlh: netlink message header
* @remaining: number of bytes remaining in message stream
*
* Returns the next netlink message in the message stream and
* decrements remaining by the size of the current message.
*/
static inline struct nlmsghdr *
nlmsg_next(const struct nlmsghdr *nlh, int *remaining)
{
int totlen = NLMSG_ALIGN(nlh->nlmsg_len);
*remaining -= totlen;
return (struct nlmsghdr *) ((unsigned char *) nlh + totlen);
}
/**
* nla_parse - Parse a stream of attributes into a tb buffer
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @head: head of attribute stream
* @len: length of attribute stream
* @policy: validation policy
* @extack: extended ACK pointer
*
* Parses a stream of attributes and stores a pointer to each attribute in
* the tb array accessible via the attribute type. Attributes with a type
* exceeding maxtype will be rejected, policy must be specified, attributes
* will be validated in the strictest way possible.
*
* Returns 0 on success or a negative error code.
*/
static inline int nla_parse(struct nlattr **tb, int maxtype,
const struct nlattr *head, int len,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_parse(tb, maxtype, head, len, policy,
NL_VALIDATE_STRICT, extack);
}
/**
* nla_parse_deprecated - Parse a stream of attributes into a tb buffer
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @head: head of attribute stream
* @len: length of attribute stream
* @policy: validation policy
* @extack: extended ACK pointer
*
* Parses a stream of attributes and stores a pointer to each attribute in
* the tb array accessible via the attribute type. Attributes with a type
* exceeding maxtype will be ignored and attributes from the policy are not
* always strictly validated (only for new attributes).
*
* Returns 0 on success or a negative error code.
*/
static inline int nla_parse_deprecated(struct nlattr **tb, int maxtype,
const struct nlattr *head, int len,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_parse(tb, maxtype, head, len, policy,
NL_VALIDATE_LIBERAL, extack);
}
/**
* nla_parse_deprecated_strict - Parse a stream of attributes into a tb buffer
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @head: head of attribute stream
* @len: length of attribute stream
* @policy: validation policy
* @extack: extended ACK pointer
*
* Parses a stream of attributes and stores a pointer to each attribute in
* the tb array accessible via the attribute type. Attributes with a type
* exceeding maxtype will be rejected as well as trailing data, but the
* policy is not completely strictly validated (only for new attributes).
*
* Returns 0 on success or a negative error code.
*/
static inline int nla_parse_deprecated_strict(struct nlattr **tb, int maxtype,
const struct nlattr *head,
int len,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_parse(tb, maxtype, head, len, policy,
NL_VALIDATE_DEPRECATED_STRICT, extack);
}
/**
* __nlmsg_parse - parse attributes of a netlink message
* @nlh: netlink message header
* @hdrlen: length of family specific header
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @validate: validation strictness
* @extack: extended ACK report struct
*
* See nla_parse()
*/
static inline int __nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen,
struct nlattr *tb[], int maxtype,
const struct nla_policy *policy,
unsigned int validate,
struct netlink_ext_ack *extack)
{
if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen)) {
NL_SET_ERR_MSG(extack, "Invalid header length");
return -EINVAL;
}
return __nla_parse(tb, maxtype, nlmsg_attrdata(nlh, hdrlen),
nlmsg_attrlen(nlh, hdrlen), policy, validate,
extack);
}
/**
* nlmsg_parse - parse attributes of a netlink message
* @nlh: netlink message header
* @hdrlen: length of family specific header
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*
* See nla_parse()
*/
static inline int nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen,
struct nlattr *tb[], int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nlmsg_parse(nlh, hdrlen, tb, maxtype, policy,
NL_VALIDATE_STRICT, extack);
}
/**
* nlmsg_parse_deprecated - parse attributes of a netlink message
* @nlh: netlink message header
* @hdrlen: length of family specific header
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*
* See nla_parse_deprecated()
*/
static inline int nlmsg_parse_deprecated(const struct nlmsghdr *nlh, int hdrlen,
struct nlattr *tb[], int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nlmsg_parse(nlh, hdrlen, tb, maxtype, policy,
NL_VALIDATE_LIBERAL, extack);
}
/**
* nlmsg_parse_deprecated_strict - parse attributes of a netlink message
* @nlh: netlink message header
* @hdrlen: length of family specific header
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*
* See nla_parse_deprecated_strict()
*/
static inline int
nlmsg_parse_deprecated_strict(const struct nlmsghdr *nlh, int hdrlen,
struct nlattr *tb[], int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nlmsg_parse(nlh, hdrlen, tb, maxtype, policy,
NL_VALIDATE_DEPRECATED_STRICT, extack);
}
/**
* nlmsg_find_attr - find a specific attribute in a netlink message
* @nlh: netlink message header
* @hdrlen: length of familiy specific header
* @attrtype: type of attribute to look for
*
* Returns the first attribute which matches the specified type.
*/
static inline struct nlattr *nlmsg_find_attr(const struct nlmsghdr *nlh,
int hdrlen, int attrtype)
{
return nla_find(nlmsg_attrdata(nlh, hdrlen),
nlmsg_attrlen(nlh, hdrlen), attrtype);
}
/**
* nla_validate_deprecated - Validate a stream of attributes
* @head: head of attribute stream
* @len: length of attribute stream
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*
* Validates all attributes in the specified attribute stream against the
* specified policy. Validation is done in liberal mode.
* See documenation of struct nla_policy for more details.
*
* Returns 0 on success or a negative error code.
*/
static inline int nla_validate_deprecated(const struct nlattr *head, int len,
int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_validate(head, len, maxtype, policy, NL_VALIDATE_LIBERAL,
extack);
}
/**
* nla_validate - Validate a stream of attributes
* @head: head of attribute stream
* @len: length of attribute stream
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*
* Validates all attributes in the specified attribute stream against the
* specified policy. Validation is done in strict mode.
* See documenation of struct nla_policy for more details.
*
* Returns 0 on success or a negative error code.
*/
static inline int nla_validate(const struct nlattr *head, int len, int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_validate(head, len, maxtype, policy, NL_VALIDATE_STRICT,
extack);
}
/**
* nlmsg_validate_deprecated - validate a netlink message including attributes
* @nlh: netlinket message header
* @hdrlen: length of familiy specific header
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*/
static inline int nlmsg_validate_deprecated(const struct nlmsghdr *nlh,
int hdrlen, int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen))
return -EINVAL;
return __nla_validate(nlmsg_attrdata(nlh, hdrlen),
nlmsg_attrlen(nlh, hdrlen), maxtype,
policy, NL_VALIDATE_LIBERAL, extack);
}
/**
* nlmsg_report - need to report back to application?
* @nlh: netlink message header
*
* Returns 1 if a report back to the application is requested.
*/
static inline int nlmsg_report(const struct nlmsghdr *nlh)
{
return nlh ? !!(nlh->nlmsg_flags & NLM_F_ECHO) : 0;
}
/**
* nlmsg_seq - return the seq number of netlink message
* @nlh: netlink message header
*
* Returns 0 if netlink message is NULL
*/
static inline u32 nlmsg_seq(const struct nlmsghdr *nlh)
{
return nlh ? nlh->nlmsg_seq : 0;
}
/**
* nlmsg_for_each_attr - iterate over a stream of attributes
* @pos: loop counter, set to current attribute
* @nlh: netlink message header
* @hdrlen: length of familiy specific header
* @rem: initialized to len, holds bytes currently remaining in stream
*/
#define nlmsg_for_each_attr(pos, nlh, hdrlen, rem) \
nla_for_each_attr(pos, nlmsg_attrdata(nlh, hdrlen), \
nlmsg_attrlen(nlh, hdrlen), rem)
/**
* nlmsg_put - Add a new netlink message to an skb
* @skb: socket buffer to store message in
* @portid: netlink PORTID of requesting application
* @seq: sequence number of message
* @type: message type
* @payload: length of message payload
* @flags: message flags
*
* Returns NULL if the tailroom of the skb is insufficient to store
* the message header and payload.
*/
static inline struct nlmsghdr *nlmsg_put(struct sk_buff *skb, u32 portid, u32 seq,
int type, int payload, int flags)
{
if (unlikely(skb_tailroom(skb) < nlmsg_total_size(payload)))
return NULL;
return __nlmsg_put(skb, portid, seq, type, payload, flags);
}
/**
* nlmsg_append - Add more data to a nlmsg in a skb
* @skb: socket buffer to store message in
* @size: length of message payload
*
* Append data to an existing nlmsg, used when constructing a message
* with multiple fixed-format headers (which is rare).
* Returns NULL if the tailroom of the skb is insufficient to store
* the extra payload.
*/
static inline void *nlmsg_append(struct sk_buff *skb, u32 size)
{
if (unlikely(skb_tailroom(skb) < NLMSG_ALIGN(size)))
return NULL;
if (NLMSG_ALIGN(size) - size)
memset(skb_tail_pointer(skb) + size, 0,
NLMSG_ALIGN(size) - size);
return __skb_put(skb, NLMSG_ALIGN(size));
}
/**
* nlmsg_put_answer - Add a new callback based netlink message to an skb
* @skb: socket buffer to store message in
* @cb: netlink callback
* @type: message type
* @payload: length of message payload
* @flags: message flags
*
* Returns NULL if the tailroom of the skb is insufficient to store
* the message header and payload.
*/
static inline struct nlmsghdr *nlmsg_put_answer(struct sk_buff *skb,
struct netlink_callback *cb,
int type, int payload,
int flags)
{
return nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
type, payload, flags);
}
/**
* nlmsg_new - Allocate a new netlink message
* @payload: size of the message payload
* @flags: the type of memory to allocate.
*
* Use NLMSG_DEFAULT_SIZE if the size of the payload isn't known
* and a good default is needed.
*/
static inline struct sk_buff *nlmsg_new(size_t payload, gfp_t flags)
{
return alloc_skb(nlmsg_total_size(payload), flags);
}
/**
* nlmsg_new_large - Allocate a new netlink message with non-contiguous
* physical memory
* @payload: size of the message payload
*
* The allocated skb is unable to have frag page for shinfo->frags*,
* as the NULL setting for skb->head in netlink_skb_destructor() will
* bypass most of the handling in skb_release_data()
*/
static inline struct sk_buff *nlmsg_new_large(size_t payload)
{
return netlink_alloc_large_skb(nlmsg_total_size(payload), 0);
}
/**
* nlmsg_end - Finalize a netlink message
* @skb: socket buffer the message is stored in
* @nlh: netlink message header
*
* Corrects the netlink message header to include the appeneded
* attributes. Only necessary if attributes have been added to
* the message.
*/
static inline void nlmsg_end(struct sk_buff *skb, struct nlmsghdr *nlh)
{
nlh->nlmsg_len = skb_tail_pointer(skb) - (unsigned char *)nlh;
}
/**
* nlmsg_get_pos - return current position in netlink message
* @skb: socket buffer the message is stored in
*
* Returns a pointer to the current tail of the message.
*/
static inline void *nlmsg_get_pos(struct sk_buff *skb)
{
return skb_tail_pointer(skb);
}
/**
* nlmsg_trim - Trim message to a mark
* @skb: socket buffer the message is stored in
* @mark: mark to trim to
*
* Trims the message to the provided mark.
*/
static inline void nlmsg_trim(struct sk_buff *skb, const void *mark)
{
if (mark) {
WARN_ON((unsigned char *) mark < skb->data); skb_trim(skb, (unsigned char *) mark - skb->data);
}
}
/**
* nlmsg_cancel - Cancel construction of a netlink message
* @skb: socket buffer the message is stored in
* @nlh: netlink message header
*
* Removes the complete netlink message including all
* attributes from the socket buffer again.
*/
static inline void nlmsg_cancel(struct sk_buff *skb, struct nlmsghdr *nlh)
{
nlmsg_trim(skb, nlh);
}
/**
* nlmsg_free - free a netlink message
* @skb: socket buffer of netlink message
*/
static inline void nlmsg_free(struct sk_buff *skb)
{
kfree_skb(skb);
}
/**
* nlmsg_multicast_filtered - multicast a netlink message with filter function
* @sk: netlink socket to spread messages to
* @skb: netlink message as socket buffer
* @portid: own netlink portid to avoid sending to yourself
* @group: multicast group id
* @flags: allocation flags
* @filter: filter function
* @filter_data: filter function private data
*
* Return: 0 on success, negative error code for failure.
*/
static inline int nlmsg_multicast_filtered(struct sock *sk, struct sk_buff *skb,
u32 portid, unsigned int group,
gfp_t flags,
netlink_filter_fn filter,
void *filter_data)
{
int err;
NETLINK_CB(skb).dst_group = group;
err = netlink_broadcast_filtered(sk, skb, portid, group, flags,
filter, filter_data);
if (err > 0)
err = 0;
return err;
}
/**
* nlmsg_multicast - multicast a netlink message
* @sk: netlink socket to spread messages to
* @skb: netlink message as socket buffer
* @portid: own netlink portid to avoid sending to yourself
* @group: multicast group id
* @flags: allocation flags
*/
static inline int nlmsg_multicast(struct sock *sk, struct sk_buff *skb,
u32 portid, unsigned int group, gfp_t flags)
{
return nlmsg_multicast_filtered(sk, skb, portid, group, flags,
NULL, NULL);
}
/**
* nlmsg_unicast - unicast a netlink message
* @sk: netlink socket to spread message to
* @skb: netlink message as socket buffer
* @portid: netlink portid of the destination socket
*/
static inline int nlmsg_unicast(struct sock *sk, struct sk_buff *skb, u32 portid)
{
int err;
err = netlink_unicast(sk, skb, portid, MSG_DONTWAIT); if (err > 0)
err = 0;
return err;
}
/**
* nlmsg_for_each_msg - iterate over a stream of messages
* @pos: loop counter, set to current message
* @head: head of message stream
* @len: length of message stream
* @rem: initialized to len, holds bytes currently remaining in stream
*/
#define nlmsg_for_each_msg(pos, head, len, rem) \
for (pos = head, rem = len; \
nlmsg_ok(pos, rem); \
pos = nlmsg_next(pos, &(rem)))
/**
* nl_dump_check_consistent - check if sequence is consistent and advertise if not
* @cb: netlink callback structure that stores the sequence number
* @nlh: netlink message header to write the flag to
*
* This function checks if the sequence (generation) number changed during dump
* and if it did, advertises it in the netlink message header.
*
* The correct way to use it is to set cb->seq to the generation counter when
* all locks for dumping have been acquired, and then call this function for
* each message that is generated.
*
* Note that due to initialisation concerns, 0 is an invalid sequence number
* and must not be used by code that uses this functionality.
*/
static inline void
nl_dump_check_consistent(struct netlink_callback *cb,
struct nlmsghdr *nlh)
{
if (cb->prev_seq && cb->seq != cb->prev_seq)
nlh->nlmsg_flags |= NLM_F_DUMP_INTR;
cb->prev_seq = cb->seq;
}
/**************************************************************************
* Netlink Attributes
**************************************************************************/
/**
* nla_attr_size - length of attribute not including padding
* @payload: length of payload
*/
static inline int nla_attr_size(int payload)
{
return NLA_HDRLEN + payload;
}
/**
* nla_total_size - total length of attribute including padding
* @payload: length of payload
*/
static inline int nla_total_size(int payload)
{
return NLA_ALIGN(nla_attr_size(payload));
}
/**
* nla_padlen - length of padding at the tail of attribute
* @payload: length of payload
*/
static inline int nla_padlen(int payload)
{
return nla_total_size(payload) - nla_attr_size(payload);
}
/**
* nla_type - attribute type
* @nla: netlink attribute
*/
static inline int nla_type(const struct nlattr *nla)
{
return nla->nla_type & NLA_TYPE_MASK;
}
/**
* nla_data - head of payload
* @nla: netlink attribute
*/
static inline void *nla_data(const struct nlattr *nla)
{
return (char *) nla + NLA_HDRLEN;
}
/**
* nla_len - length of payload
* @nla: netlink attribute
*/
static inline u16 nla_len(const struct nlattr *nla)
{
return nla->nla_len - NLA_HDRLEN;
}
/**
* nla_ok - check if the netlink attribute fits into the remaining bytes
* @nla: netlink attribute
* @remaining: number of bytes remaining in attribute stream
*/
static inline int nla_ok(const struct nlattr *nla, int remaining)
{
return remaining >= (int) sizeof(*nla) &&
nla->nla_len >= sizeof(*nla) &&
nla->nla_len <= remaining;
}
/**
* nla_next - next netlink attribute in attribute stream
* @nla: netlink attribute
* @remaining: number of bytes remaining in attribute stream
*
* Returns the next netlink attribute in the attribute stream and
* decrements remaining by the size of the current attribute.
*/
static inline struct nlattr *nla_next(const struct nlattr *nla, int *remaining)
{
unsigned int totlen = NLA_ALIGN(nla->nla_len);
*remaining -= totlen;
return (struct nlattr *) ((char *) nla + totlen);
}
/**
* nla_find_nested - find attribute in a set of nested attributes
* @nla: attribute containing the nested attributes
* @attrtype: type of attribute to look for
*
* Returns the first attribute which matches the specified type.
*/
static inline struct nlattr *
nla_find_nested(const struct nlattr *nla, int attrtype)
{
return nla_find(nla_data(nla), nla_len(nla), attrtype);
}
/**
* nla_parse_nested - parse nested attributes
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @nla: attribute containing the nested attributes
* @policy: validation policy
* @extack: extended ACK report struct
*
* See nla_parse()
*/
static inline int nla_parse_nested(struct nlattr *tb[], int maxtype,
const struct nlattr *nla,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
if (!(nla->nla_type & NLA_F_NESTED)) {
NL_SET_ERR_MSG_ATTR(extack, nla, "NLA_F_NESTED is missing");
return -EINVAL;
}
return __nla_parse(tb, maxtype, nla_data(nla), nla_len(nla), policy,
NL_VALIDATE_STRICT, extack);
}
/**
* nla_parse_nested_deprecated - parse nested attributes
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @nla: attribute containing the nested attributes
* @policy: validation policy
* @extack: extended ACK report struct
*
* See nla_parse_deprecated()
*/
static inline int nla_parse_nested_deprecated(struct nlattr *tb[], int maxtype,
const struct nlattr *nla,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_parse(tb, maxtype, nla_data(nla), nla_len(nla), policy,
NL_VALIDATE_LIBERAL, extack);
}
/**
* nla_put_u8 - Add a u8 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_u8(struct sk_buff *skb, int attrtype, u8 value)
{
/* temporary variables to work around GCC PR81715 with asan-stack=1 */
u8 tmp = value;
return nla_put(skb, attrtype, sizeof(u8), &tmp);
}
/**
* nla_put_u16 - Add a u16 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_u16(struct sk_buff *skb, int attrtype, u16 value)
{
u16 tmp = value;
return nla_put(skb, attrtype, sizeof(u16), &tmp);
}
/**
* nla_put_be16 - Add a __be16 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_be16(struct sk_buff *skb, int attrtype, __be16 value)
{
__be16 tmp = value;
return nla_put(skb, attrtype, sizeof(__be16), &tmp);
}
/**
* nla_put_net16 - Add 16-bit network byte order netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_net16(struct sk_buff *skb, int attrtype, __be16 value)
{
__be16 tmp = value;
return nla_put_be16(skb, attrtype | NLA_F_NET_BYTEORDER, tmp);
}
/**
* nla_put_le16 - Add a __le16 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_le16(struct sk_buff *skb, int attrtype, __le16 value)
{
__le16 tmp = value;
return nla_put(skb, attrtype, sizeof(__le16), &tmp);
}
/**
* nla_put_u32 - Add a u32 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_u32(struct sk_buff *skb, int attrtype, u32 value)
{
u32 tmp = value;
return nla_put(skb, attrtype, sizeof(u32), &tmp);
}
/**
* nla_put_uint - Add a variable-size unsigned int to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_uint(struct sk_buff *skb, int attrtype, u64 value)
{
u64 tmp64 = value;
u32 tmp32 = value;
if (tmp64 == tmp32)
return nla_put_u32(skb, attrtype, tmp32);
return nla_put(skb, attrtype, sizeof(u64), &tmp64);
}
/**
* nla_put_be32 - Add a __be32 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_be32(struct sk_buff *skb, int attrtype, __be32 value)
{
__be32 tmp = value;
return nla_put(skb, attrtype, sizeof(__be32), &tmp);
}
/**
* nla_put_net32 - Add 32-bit network byte order netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_net32(struct sk_buff *skb, int attrtype, __be32 value)
{
__be32 tmp = value;
return nla_put_be32(skb, attrtype | NLA_F_NET_BYTEORDER, tmp);
}
/**
* nla_put_le32 - Add a __le32 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_le32(struct sk_buff *skb, int attrtype, __le32 value)
{
__le32 tmp = value;
return nla_put(skb, attrtype, sizeof(__le32), &tmp);
}
/**
* nla_put_u64_64bit - Add a u64 netlink attribute to a skb and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
* @padattr: attribute type for the padding
*/
static inline int nla_put_u64_64bit(struct sk_buff *skb, int attrtype,
u64 value, int padattr)
{
u64 tmp = value;
return nla_put_64bit(skb, attrtype, sizeof(u64), &tmp, padattr);
}
/**
* nla_put_be64 - Add a __be64 netlink attribute to a socket buffer and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
* @padattr: attribute type for the padding
*/
static inline int nla_put_be64(struct sk_buff *skb, int attrtype, __be64 value,
int padattr)
{
__be64 tmp = value;
return nla_put_64bit(skb, attrtype, sizeof(__be64), &tmp, padattr);
}
/**
* nla_put_net64 - Add 64-bit network byte order nlattr to a skb and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
* @padattr: attribute type for the padding
*/
static inline int nla_put_net64(struct sk_buff *skb, int attrtype, __be64 value,
int padattr)
{
__be64 tmp = value;
return nla_put_be64(skb, attrtype | NLA_F_NET_BYTEORDER, tmp,
padattr);
}
/**
* nla_put_le64 - Add a __le64 netlink attribute to a socket buffer and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
* @padattr: attribute type for the padding
*/
static inline int nla_put_le64(struct sk_buff *skb, int attrtype, __le64 value,
int padattr)
{
__le64 tmp = value;
return nla_put_64bit(skb, attrtype, sizeof(__le64), &tmp, padattr);
}
/**
* nla_put_s8 - Add a s8 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_s8(struct sk_buff *skb, int attrtype, s8 value)
{
s8 tmp = value;
return nla_put(skb, attrtype, sizeof(s8), &tmp);
}
/**
* nla_put_s16 - Add a s16 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_s16(struct sk_buff *skb, int attrtype, s16 value)
{
s16 tmp = value;
return nla_put(skb, attrtype, sizeof(s16), &tmp);
}
/**
* nla_put_s32 - Add a s32 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_s32(struct sk_buff *skb, int attrtype, s32 value)
{
s32 tmp = value;
return nla_put(skb, attrtype, sizeof(s32), &tmp);
}
/**
* nla_put_s64 - Add a s64 netlink attribute to a socket buffer and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
* @padattr: attribute type for the padding
*/
static inline int nla_put_s64(struct sk_buff *skb, int attrtype, s64 value,
int padattr)
{
s64 tmp = value;
return nla_put_64bit(skb, attrtype, sizeof(s64), &tmp, padattr);
}
/**
* nla_put_sint - Add a variable-size signed int to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: numeric value
*/
static inline int nla_put_sint(struct sk_buff *skb, int attrtype, s64 value)
{
s64 tmp64 = value;
s32 tmp32 = value;
if (tmp64 == tmp32)
return nla_put_s32(skb, attrtype, tmp32);
return nla_put(skb, attrtype, sizeof(s64), &tmp64);
}
/**
* nla_put_string - Add a string netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @str: NUL terminated string
*/
static inline int nla_put_string(struct sk_buff *skb, int attrtype,
const char *str)
{
return nla_put(skb, attrtype, strlen(str) + 1, str);
}
/**
* nla_put_flag - Add a flag netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
*/
static inline int nla_put_flag(struct sk_buff *skb, int attrtype)
{
return nla_put(skb, attrtype, 0, NULL);
}
/**
* nla_put_msecs - Add a msecs netlink attribute to a skb and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @njiffies: number of jiffies to convert to msecs
* @padattr: attribute type for the padding
*/
static inline int nla_put_msecs(struct sk_buff *skb, int attrtype,
unsigned long njiffies, int padattr)
{
u64 tmp = jiffies_to_msecs(njiffies);
return nla_put_64bit(skb, attrtype, sizeof(u64), &tmp, padattr);
}
/**
* nla_put_in_addr - Add an IPv4 address netlink attribute to a socket
* buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @addr: IPv4 address
*/
static inline int nla_put_in_addr(struct sk_buff *skb, int attrtype,
__be32 addr)
{
__be32 tmp = addr;
return nla_put_be32(skb, attrtype, tmp);
}
/**
* nla_put_in6_addr - Add an IPv6 address netlink attribute to a socket
* buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @addr: IPv6 address
*/
static inline int nla_put_in6_addr(struct sk_buff *skb, int attrtype,
const struct in6_addr *addr)
{
return nla_put(skb, attrtype, sizeof(*addr), addr);
}
/**
* nla_put_bitfield32 - Add a bitfield32 netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @value: value carrying bits
* @selector: selector of valid bits
*/
static inline int nla_put_bitfield32(struct sk_buff *skb, int attrtype,
__u32 value, __u32 selector)
{
struct nla_bitfield32 tmp = { value, selector, };
return nla_put(skb, attrtype, sizeof(tmp), &tmp);
}
/**
* nla_get_u32 - return payload of u32 attribute
* @nla: u32 netlink attribute
*/
static inline u32 nla_get_u32(const struct nlattr *nla)
{
return *(u32 *) nla_data(nla);
}
/**
* nla_get_be32 - return payload of __be32 attribute
* @nla: __be32 netlink attribute
*/
static inline __be32 nla_get_be32(const struct nlattr *nla)
{
return *(__be32 *) nla_data(nla);
}
/**
* nla_get_le32 - return payload of __le32 attribute
* @nla: __le32 netlink attribute
*/
static inline __le32 nla_get_le32(const struct nlattr *nla)
{
return *(__le32 *) nla_data(nla);
}
/**
* nla_get_u16 - return payload of u16 attribute
* @nla: u16 netlink attribute
*/
static inline u16 nla_get_u16(const struct nlattr *nla)
{
return *(u16 *) nla_data(nla);
}
/**
* nla_get_be16 - return payload of __be16 attribute
* @nla: __be16 netlink attribute
*/
static inline __be16 nla_get_be16(const struct nlattr *nla)
{
return *(__be16 *) nla_data(nla);
}
/**
* nla_get_le16 - return payload of __le16 attribute
* @nla: __le16 netlink attribute
*/
static inline __le16 nla_get_le16(const struct nlattr *nla)
{
return *(__le16 *) nla_data(nla);
}
/**
* nla_get_u8 - return payload of u8 attribute
* @nla: u8 netlink attribute
*/
static inline u8 nla_get_u8(const struct nlattr *nla)
{
return *(u8 *) nla_data(nla);
}
/**
* nla_get_u64 - return payload of u64 attribute
* @nla: u64 netlink attribute
*/
static inline u64 nla_get_u64(const struct nlattr *nla)
{
u64 tmp;
nla_memcpy(&tmp, nla, sizeof(tmp));
return tmp;
}
/**
* nla_get_uint - return payload of uint attribute
* @nla: uint netlink attribute
*/
static inline u64 nla_get_uint(const struct nlattr *nla)
{
if (nla_len(nla) == sizeof(u32))
return nla_get_u32(nla);
return nla_get_u64(nla);
}
/**
* nla_get_be64 - return payload of __be64 attribute
* @nla: __be64 netlink attribute
*/
static inline __be64 nla_get_be64(const struct nlattr *nla)
{
__be64 tmp;
nla_memcpy(&tmp, nla, sizeof(tmp));
return tmp;
}
/**
* nla_get_le64 - return payload of __le64 attribute
* @nla: __le64 netlink attribute
*/
static inline __le64 nla_get_le64(const struct nlattr *nla)
{
return *(__le64 *) nla_data(nla);
}
/**
* nla_get_s32 - return payload of s32 attribute
* @nla: s32 netlink attribute
*/
static inline s32 nla_get_s32(const struct nlattr *nla)
{
return *(s32 *) nla_data(nla);
}
/**
* nla_get_s16 - return payload of s16 attribute
* @nla: s16 netlink attribute
*/
static inline s16 nla_get_s16(const struct nlattr *nla)
{
return *(s16 *) nla_data(nla);
}
/**
* nla_get_s8 - return payload of s8 attribute
* @nla: s8 netlink attribute
*/
static inline s8 nla_get_s8(const struct nlattr *nla)
{
return *(s8 *) nla_data(nla);
}
/**
* nla_get_s64 - return payload of s64 attribute
* @nla: s64 netlink attribute
*/
static inline s64 nla_get_s64(const struct nlattr *nla)
{
s64 tmp;
nla_memcpy(&tmp, nla, sizeof(tmp));
return tmp;
}
/**
* nla_get_sint - return payload of uint attribute
* @nla: uint netlink attribute
*/
static inline s64 nla_get_sint(const struct nlattr *nla)
{
if (nla_len(nla) == sizeof(s32))
return nla_get_s32(nla);
return nla_get_s64(nla);
}
/**
* nla_get_flag - return payload of flag attribute
* @nla: flag netlink attribute
*/
static inline int nla_get_flag(const struct nlattr *nla)
{
return !!nla;
}
/**
* nla_get_msecs - return payload of msecs attribute
* @nla: msecs netlink attribute
*
* Returns the number of milliseconds in jiffies.
*/
static inline unsigned long nla_get_msecs(const struct nlattr *nla)
{
u64 msecs = nla_get_u64(nla);
return msecs_to_jiffies((unsigned long) msecs);
}
/**
* nla_get_in_addr - return payload of IPv4 address attribute
* @nla: IPv4 address netlink attribute
*/
static inline __be32 nla_get_in_addr(const struct nlattr *nla)
{
return *(__be32 *) nla_data(nla);
}
/**
* nla_get_in6_addr - return payload of IPv6 address attribute
* @nla: IPv6 address netlink attribute
*/
static inline struct in6_addr nla_get_in6_addr(const struct nlattr *nla)
{
struct in6_addr tmp;
nla_memcpy(&tmp, nla, sizeof(tmp));
return tmp;
}
/**
* nla_get_bitfield32 - return payload of 32 bitfield attribute
* @nla: nla_bitfield32 attribute
*/
static inline struct nla_bitfield32 nla_get_bitfield32(const struct nlattr *nla)
{
struct nla_bitfield32 tmp;
nla_memcpy(&tmp, nla, sizeof(tmp));
return tmp;
}
/**
* nla_memdup - duplicate attribute memory (kmemdup)
* @src: netlink attribute to duplicate from
* @gfp: GFP mask
*/
static inline void *nla_memdup(const struct nlattr *src, gfp_t gfp)
{
return kmemdup(nla_data(src), nla_len(src), gfp);
}
/**
* nla_nest_start_noflag - Start a new level of nested attributes
* @skb: socket buffer to add attributes to
* @attrtype: attribute type of container
*
* This function exists for backward compatibility to use in APIs which never
* marked their nest attributes with NLA_F_NESTED flag. New APIs should use
* nla_nest_start() which sets the flag.
*
* Returns the container attribute or NULL on error
*/
static inline struct nlattr *nla_nest_start_noflag(struct sk_buff *skb,
int attrtype)
{
struct nlattr *start = (struct nlattr *)skb_tail_pointer(skb); if (nla_put(skb, attrtype, 0, NULL) < 0)
return NULL;
return start;
}
/**
* nla_nest_start - Start a new level of nested attributes, with NLA_F_NESTED
* @skb: socket buffer to add attributes to
* @attrtype: attribute type of container
*
* Unlike nla_nest_start_noflag(), mark the nest attribute with NLA_F_NESTED
* flag. This is the preferred function to use in new code.
*
* Returns the container attribute or NULL on error
*/
static inline struct nlattr *nla_nest_start(struct sk_buff *skb, int attrtype)
{
return nla_nest_start_noflag(skb, attrtype | NLA_F_NESTED);
}
/**
* nla_nest_end - Finalize nesting of attributes
* @skb: socket buffer the attributes are stored in
* @start: container attribute
*
* Corrects the container attribute header to include the all
* appeneded attributes.
*
* Returns the total data length of the skb.
*/
static inline int nla_nest_end(struct sk_buff *skb, struct nlattr *start)
{
start->nla_len = skb_tail_pointer(skb) - (unsigned char *)start; return skb->len;
}
/**
* nla_nest_cancel - Cancel nesting of attributes
* @skb: socket buffer the message is stored in
* @start: container attribute
*
* Removes the container attribute and including all nested
* attributes. Returns -EMSGSIZE
*/
static inline void nla_nest_cancel(struct sk_buff *skb, struct nlattr *start)
{
nlmsg_trim(skb, start);
}
/**
* __nla_validate_nested - Validate a stream of nested attributes
* @start: container attribute
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @validate: validation strictness
* @extack: extended ACK report struct
*
* Validates all attributes in the nested attribute stream against the
* specified policy. Attributes with a type exceeding maxtype will be
* ignored. See documenation of struct nla_policy for more details.
*
* Returns 0 on success or a negative error code.
*/
static inline int __nla_validate_nested(const struct nlattr *start, int maxtype,
const struct nla_policy *policy,
unsigned int validate,
struct netlink_ext_ack *extack)
{
return __nla_validate(nla_data(start), nla_len(start), maxtype, policy,
validate, extack);
}
static inline int
nla_validate_nested(const struct nlattr *start, int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_validate_nested(start, maxtype, policy,
NL_VALIDATE_STRICT, extack);
}
static inline int
nla_validate_nested_deprecated(const struct nlattr *start, int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nla_validate_nested(start, maxtype, policy,
NL_VALIDATE_LIBERAL, extack);
}
/**
* nla_need_padding_for_64bit - test 64-bit alignment of the next attribute
* @skb: socket buffer the message is stored in
*
* Return true if padding is needed to align the next attribute (nla_data()) to
* a 64-bit aligned area.
*/
static inline bool nla_need_padding_for_64bit(struct sk_buff *skb)
{
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
/* The nlattr header is 4 bytes in size, that's why we test
* if the skb->data _is_ aligned. A NOP attribute, plus
* nlattr header for next attribute, will make nla_data()
* 8-byte aligned.
*/
if (IS_ALIGNED((unsigned long)skb_tail_pointer(skb), 8))
return true;
#endif
return false;
}
/**
* nla_align_64bit - 64-bit align the nla_data() of next attribute
* @skb: socket buffer the message is stored in
* @padattr: attribute type for the padding
*
* Conditionally emit a padding netlink attribute in order to make
* the next attribute we emit have a 64-bit aligned nla_data() area.
* This will only be done in architectures which do not have
* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS defined.
*
* Returns zero on success or a negative error code.
*/
static inline int nla_align_64bit(struct sk_buff *skb, int padattr)
{
if (nla_need_padding_for_64bit(skb) &&
!nla_reserve(skb, padattr, 0))
return -EMSGSIZE;
return 0;
}
/**
* nla_total_size_64bit - total length of attribute including padding
* @payload: length of payload
*/
static inline int nla_total_size_64bit(int payload)
{
return NLA_ALIGN(nla_attr_size(payload))
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ NLA_ALIGN(nla_attr_size(0))
#endif
;
}
/**
* nla_for_each_attr - iterate over a stream of attributes
* @pos: loop counter, set to current attribute
* @head: head of attribute stream
* @len: length of attribute stream
* @rem: initialized to len, holds bytes currently remaining in stream
*/
#define nla_for_each_attr(pos, head, len, rem) \
for (pos = head, rem = len; \
nla_ok(pos, rem); \
pos = nla_next(pos, &(rem)))
/**
* nla_for_each_nested - iterate over nested attributes
* @pos: loop counter, set to current attribute
* @nla: attribute containing the nested attributes
* @rem: initialized to len, holds bytes currently remaining in stream
*/
#define nla_for_each_nested(pos, nla, rem) \
nla_for_each_attr(pos, nla_data(nla), nla_len(nla), rem)
/**
* nla_is_last - Test if attribute is last in stream
* @nla: attribute to test
* @rem: bytes remaining in stream
*/
static inline bool nla_is_last(const struct nlattr *nla, int rem)
{
return nla->nla_len == rem;
}
void nla_get_range_unsigned(const struct nla_policy *pt,
struct netlink_range_validation *range);
void nla_get_range_signed(const struct nla_policy *pt,
struct netlink_range_validation_signed *range);
struct netlink_policy_dump_state;
int netlink_policy_dump_add_policy(struct netlink_policy_dump_state **pstate,
const struct nla_policy *policy,
unsigned int maxtype);
int netlink_policy_dump_get_policy_idx(struct netlink_policy_dump_state *state,
const struct nla_policy *policy,
unsigned int maxtype);
bool netlink_policy_dump_loop(struct netlink_policy_dump_state *state);
int netlink_policy_dump_write(struct sk_buff *skb,
struct netlink_policy_dump_state *state);
int netlink_policy_dump_attr_size_estimate(const struct nla_policy *pt);
int netlink_policy_dump_write_attr(struct sk_buff *skb,
const struct nla_policy *pt,
int nestattr);
void netlink_policy_dump_free(struct netlink_policy_dump_state *state);
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* mm/pgtable-generic.c
*
* Generic pgtable methods declared in linux/pgtable.h
*
* Copyright (C) 2010 Linus Torvalds
*/
#include <linux/pagemap.h>
#include <linux/hugetlb.h>
#include <linux/pgtable.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mm_inline.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
/*
* If a p?d_bad entry is found while walking page tables, report
* the error, before resetting entry to p?d_none. Usually (but
* very seldom) called out from the p?d_none_or_clear_bad macros.
*/
void pgd_clear_bad(pgd_t *pgd)
{
pgd_ERROR(*pgd);
pgd_clear(pgd);
}
#ifndef __PAGETABLE_P4D_FOLDED
void p4d_clear_bad(p4d_t *p4d)
{
p4d_ERROR(*p4d);
p4d_clear(p4d);
}
#endif
#ifndef __PAGETABLE_PUD_FOLDED
void pud_clear_bad(pud_t *pud)
{
pud_ERROR(*pud);
pud_clear(pud);
}
#endif
/*
* Note that the pmd variant below can't be stub'ed out just as for p4d/pud
* above. pmd folding is special and typically pmd_* macros refer to upper
* level even when folded
*/
void pmd_clear_bad(pmd_t *pmd)
{
pmd_ERROR(*pmd);
pmd_clear(pmd);
}
#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
* Only sets the access flags (dirty, accessed), as well as write
* permission. Furthermore, we know it always gets set to a "more
* permissive" setting, which allows most architectures to optimize
* this. We return whether the PTE actually changed, which in turn
* instructs the caller to do things like update__mmu_cache. This
* used to be done in the caller, but sparc needs minor faults to
* force that call on sun4c so we changed this macro slightly
*/
int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep,
pte_t entry, int dirty)
{
int changed = !pte_same(ptep_get(ptep), entry);
if (changed) {
set_pte_at(vma->vm_mm, address, ptep, entry);
flush_tlb_fix_spurious_fault(vma, address, ptep);
}
return changed;
}
#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
int young;
young = ptep_test_and_clear_young(vma, address, ptep);
if (young)
flush_tlb_page(vma, address);
return young;
}
#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
pte_t *ptep)
{
struct mm_struct *mm = (vma)->vm_mm;
pte_t pte;
pte = ptep_get_and_clear(mm, address, ptep); if (pte_accessible(mm, pte)) flush_tlb_page(vma, address); return pte;
}
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
int pmdp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp,
pmd_t entry, int dirty)
{
int changed = !pmd_same(*pmdp, entry);
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
if (changed) {
set_pmd_at(vma->vm_mm, address, pmdp, entry);
flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
return changed;
}
#endif
#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
int pmdp_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
int young;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
young = pmdp_test_and_clear_young(vma, address, pmdp);
if (young)
flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return young;
}
#endif
#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t pmd;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) &&
!pmd_devmap(*pmdp));
pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return pmd;
}
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
pud_t *pudp)
{
pud_t pud;
VM_BUG_ON(address & ~HPAGE_PUD_MASK);
VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp));
pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
return pud;
}
#endif
#endif
#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pgtable)
{
assert_spin_locked(pmd_lockptr(mm, pmdp));
/* FIFO */
if (!pmd_huge_pte(mm, pmdp))
INIT_LIST_HEAD(&pgtable->lru);
else
list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
pmd_huge_pte(mm, pmdp) = pgtable;
}
#endif
#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
/* no "address" argument so destroys page coloring of some arch */
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
pgtable_t pgtable;
assert_spin_locked(pmd_lockptr(mm, pmdp));
/* FIFO */
pgtable = pmd_huge_pte(mm, pmdp);
pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
struct page, lru);
if (pmd_huge_pte(mm, pmdp))
list_del(&pgtable->lru);
return pgtable;
}
#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATE
pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return old;
}
#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
return pmdp_invalidate(vma, address, pmdp);
}
#endif
#ifndef pmdp_collapse_flush
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
/*
* pmd and hugepage pte format are same. So we could
* use the same function.
*/
pmd_t pmd;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
VM_BUG_ON(pmd_trans_huge(*pmdp));
pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
/* collapse entails shooting down ptes not pmd */
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return pmd;
}
#endif
/* arch define pte_free_defer in asm/pgalloc.h for its own implementation */
#ifndef pte_free_defer
static void pte_free_now(struct rcu_head *head)
{
struct page *page;
page = container_of(head, struct page, rcu_head);
pte_free(NULL /* mm not passed and not used */, (pgtable_t)page);
}
void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
{
struct page *page;
page = pgtable;
call_rcu(&page->rcu_head, pte_free_now);
}
#endif /* pte_free_defer */
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \
(defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RCU))
/*
* See the comment above ptep_get_lockless() in include/linux/pgtable.h:
* the barriers in pmdp_get_lockless() cannot guarantee that the value in
* pmd_high actually belongs with the value in pmd_low; but holding interrupts
* off blocks the TLB flush between present updates, which guarantees that a
* successful __pte_offset_map() points to a page from matched halves.
*/
static unsigned long pmdp_get_lockless_start(void)
{
unsigned long irqflags;
local_irq_save(irqflags);
return irqflags;
}
static void pmdp_get_lockless_end(unsigned long irqflags)
{
local_irq_restore(irqflags);
}
#else
static unsigned long pmdp_get_lockless_start(void) { return 0; }
static void pmdp_get_lockless_end(unsigned long irqflags) { }
#endif
pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
{
unsigned long irqflags;
pmd_t pmdval;
rcu_read_lock();
irqflags = pmdp_get_lockless_start();
pmdval = pmdp_get_lockless(pmd);
pmdp_get_lockless_end(irqflags);
if (pmdvalp)
*pmdvalp = pmdval; if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval)))
goto nomap;
if (unlikely(pmd_trans_huge(pmdval) || pmd_devmap(pmdval)))
goto nomap;
if (unlikely(pmd_bad(pmdval))) { pmd_clear_bad(pmd);
goto nomap;
}
return __pte_map(&pmdval, addr);
nomap:
rcu_read_unlock();
return NULL;
}
pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp)
{
pmd_t pmdval;
pte_t *pte;
pte = __pte_offset_map(pmd, addr, &pmdval);
if (likely(pte))
*ptlp = pte_lockptr(mm, &pmdval); return pte;
}
/*
* pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
* __pte_offset_map_lock() below, is usually called with the pmd pointer for
* addr, reached by walking down the mm's pgd, p4d, pud for addr: either while
* holding mmap_lock or vma lock for read or for write; or in truncate or rmap
* context, while holding file's i_mmap_lock or anon_vma lock for read (or for
* write). In a few cases, it may be used with pmd pointing to a pmd_t already
* copied to or constructed on the stack.
*
* When successful, it returns the pte pointer for addr, with its page table
* kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent
* modification by software, with a pointer to that spinlock in ptlp (in some
* configs mm->page_table_lock, in SPLIT_PTLOCK configs a spinlock in table's
* struct page). pte_unmap_unlock(pte, ptl) to unlock and unmap afterwards.
*
* But it is unsuccessful, returning NULL with *ptlp unchanged, if there is no
* page table at *pmd: if, for example, the page table has just been removed,
* or replaced by the huge pmd of a THP. (When successful, *pmd is rechecked
* after acquiring the ptlock, and retried internally if it changed: so that a
* page table can be safely removed or replaced by THP while holding its lock.)
*
* pte_offset_map(pmd, addr), and its internal helper __pte_offset_map() above,
* just returns the pte pointer for addr, its page table kmapped if necessary;
* or NULL if there is no page table at *pmd. It does not attempt to lock the
* page table, so cannot normally be used when the page table is to be updated,
* or when entries read must be stable. But it does take rcu_read_lock(): so
* that even when page table is racily removed, it remains a valid though empty
* and disconnected table. Until pte_unmap(pte) unmaps and rcu_read_unlock()s
* afterwards.
*
* pte_offset_map_nolock(mm, pmd, addr, ptlp), above, is like pte_offset_map();
* but when successful, it also outputs a pointer to the spinlock in ptlp - as
* pte_offset_map_lock() does, but in this case without locking it. This helps
* the caller to avoid a later pte_lockptr(mm, *pmd), which might by that time
* act on a changed *pmd: pte_offset_map_nolock() provides the correct spinlock
* pointer for the page table that it returns. In principle, the caller should
* recheck *pmd once the lock is taken; in practice, no callsite needs that -
* either the mmap_lock for write, or pte_same() check on contents, is enough.
*
* Note that free_pgtables(), used after unmapping detached vmas, or when
* exiting the whole mm, does not take page table lock before freeing a page
* table, and may not use RCU at all: "outsiders" like khugepaged should avoid
* pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
*/
pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp)
{
spinlock_t *ptl;
pmd_t pmdval;
pte_t *pte;
again:
pte = __pte_offset_map(pmd, addr, &pmdval);
if (unlikely(!pte))
return pte;
ptl = pte_lockptr(mm, &pmdval);
spin_lock(ptl);
if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
*ptlp = ptl; return pte;
}
pte_unmap_unlock(pte, ptl);
goto again;
}
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/objtool.h>
#include <linux/module.h>
#include <linux/sort.h>
#include <asm/ptrace.h>
#include <asm/stacktrace.h>
#include <asm/unwind.h>
#include <asm/orc_types.h>
#include <asm/orc_lookup.h>
#include <asm/orc_header.h>
ORC_HEADER;
#define orc_warn(fmt, ...) \
printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
#define orc_warn_current(args...) \
({ \
static bool dumped_before; \
if (state->task == current && !state->error) { \
orc_warn(args); \
if (unwind_debug && !dumped_before) { \
dumped_before = true; \
unwind_dump(state); \
} \
} \
})
extern int __start_orc_unwind_ip[];
extern int __stop_orc_unwind_ip[];
extern struct orc_entry __start_orc_unwind[];
extern struct orc_entry __stop_orc_unwind[];
static bool orc_init __ro_after_init;
static bool unwind_debug __ro_after_init;
static unsigned int lookup_num_blocks __ro_after_init;
static int __init unwind_debug_cmdline(char *str)
{
unwind_debug = true;
return 0;
}
early_param("unwind_debug", unwind_debug_cmdline);
static void unwind_dump(struct unwind_state *state)
{
static bool dumped_before;
unsigned long word, *sp;
struct stack_info stack_info = {0};
unsigned long visit_mask = 0;
if (dumped_before)
return;
dumped_before = true;
printk_deferred("unwind stack type:%d next_sp:%p mask:0x%lx graph_idx:%d\n",
state->stack_info.type, state->stack_info.next_sp,
state->stack_mask, state->graph_idx);
for (sp = __builtin_frame_address(0); sp;
sp = PTR_ALIGN(stack_info.next_sp, sizeof(long))) {
if (get_stack_info(sp, state->task, &stack_info, &visit_mask))
break;
for (; sp < stack_info.end; sp++) {
word = READ_ONCE_NOCHECK(*sp);
printk_deferred("%0*lx: %0*lx (%pB)\n", BITS_PER_LONG/4,
(unsigned long)sp, BITS_PER_LONG/4,
word, (void *)word);
}
}
}
static inline unsigned long orc_ip(const int *ip)
{
return (unsigned long)ip + *ip;
}
static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
unsigned int num_entries, unsigned long ip)
{
int *first = ip_table;
int *last = ip_table + num_entries - 1;
int *mid, *found = first;
if (!num_entries)
return NULL;
/*
* Do a binary range search to find the rightmost duplicate of a given
* starting address. Some entries are section terminators which are
* "weak" entries for ensuring there are no gaps. They should be
* ignored when they conflict with a real entry.
*/
while (first <= last) { mid = first + ((last - first) / 2);
if (orc_ip(mid) <= ip) {
found = mid;
first = mid + 1;
} else
last = mid - 1;
}
return u_table + (found - ip_table);
}
#ifdef CONFIG_MODULES
static struct orc_entry *orc_module_find(unsigned long ip)
{
struct module *mod;
mod = __module_address(ip); if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip)
return NULL;
return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind,
mod->arch.num_orcs, ip);
}
#else
static struct orc_entry *orc_module_find(unsigned long ip)
{
return NULL;
}
#endif
#ifdef CONFIG_DYNAMIC_FTRACE
static struct orc_entry *orc_find(unsigned long ip);
/*
* Ftrace dynamic trampolines do not have orc entries of their own.
* But they are copies of the ftrace entries that are static and
* defined in ftrace_*.S, which do have orc entries.
*
* If the unwinder comes across a ftrace trampoline, then find the
* ftrace function that was used to create it, and use that ftrace
* function's orc entry, as the placement of the return code in
* the stack will be identical.
*/
static struct orc_entry *orc_ftrace_find(unsigned long ip)
{
struct ftrace_ops *ops;
unsigned long tramp_addr, offset;
ops = ftrace_ops_trampoline(ip);
if (!ops)
return NULL;
/* Set tramp_addr to the start of the code copied by the trampoline */
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
tramp_addr = (unsigned long)ftrace_regs_caller;
else
tramp_addr = (unsigned long)ftrace_caller;
/* Now place tramp_addr to the location within the trampoline ip is at */
offset = ip - ops->trampoline;
tramp_addr += offset;
/* Prevent unlikely recursion */
if (ip == tramp_addr)
return NULL;
return orc_find(tramp_addr);
}
#else
static struct orc_entry *orc_ftrace_find(unsigned long ip)
{
return NULL;
}
#endif
/*
* If we crash with IP==0, the last successfully executed instruction
* was probably an indirect function call with a NULL function pointer,
* and we don't have unwind information for NULL.
* This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
* pointer into its parent and then continue normally from there.
*/
static struct orc_entry null_orc_entry = {
.sp_offset = sizeof(long),
.sp_reg = ORC_REG_SP,
.bp_reg = ORC_REG_UNDEFINED,
.type = ORC_TYPE_CALL
};
/* Fake frame pointer entry -- used as a fallback for generated code */
static struct orc_entry orc_fp_entry = {
.type = ORC_TYPE_CALL,
.sp_reg = ORC_REG_BP,
.sp_offset = 16,
.bp_reg = ORC_REG_PREV_SP,
.bp_offset = -16,
};
static struct orc_entry *orc_find(unsigned long ip)
{
static struct orc_entry *orc;
if (ip == 0)
return &null_orc_entry;
/* For non-init vmlinux addresses, use the fast lookup table: */
if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) {
unsigned int idx, start, stop;
idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE;
if (unlikely((idx >= lookup_num_blocks-1))) {
orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n",
idx, lookup_num_blocks, (void *)ip);
return NULL;
}
start = orc_lookup[idx];
stop = orc_lookup[idx + 1] + 1;
if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) ||
(__start_orc_unwind + stop > __stop_orc_unwind))) {
orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n",
idx, lookup_num_blocks, start, stop, (void *)ip);
return NULL;
}
return __orc_find(__start_orc_unwind_ip + start,
__start_orc_unwind + start, stop - start, ip);
}
/* vmlinux .init slow lookup: */
if (is_kernel_inittext(ip))
return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
__stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
/* Module lookup: */
orc = orc_module_find(ip);
if (orc)
return orc;
return orc_ftrace_find(ip);
}
#ifdef CONFIG_MODULES
static DEFINE_MUTEX(sort_mutex);
static int *cur_orc_ip_table = __start_orc_unwind_ip;
static struct orc_entry *cur_orc_table = __start_orc_unwind;
static void orc_sort_swap(void *_a, void *_b, int size)
{
struct orc_entry *orc_a, *orc_b;
int *a = _a, *b = _b, tmp;
int delta = _b - _a;
/* Swap the .orc_unwind_ip entries: */
tmp = *a;
*a = *b + delta;
*b = tmp - delta;
/* Swap the corresponding .orc_unwind entries: */
orc_a = cur_orc_table + (a - cur_orc_ip_table);
orc_b = cur_orc_table + (b - cur_orc_ip_table);
swap(*orc_a, *orc_b);
}
static int orc_sort_cmp(const void *_a, const void *_b)
{
struct orc_entry *orc_a;
const int *a = _a, *b = _b;
unsigned long a_val = orc_ip(a);
unsigned long b_val = orc_ip(b);
if (a_val > b_val)
return 1;
if (a_val < b_val)
return -1;
/*
* The "weak" section terminator entries need to always be first
* to ensure the lookup code skips them in favor of real entries.
* These terminator entries exist to handle any gaps created by
* whitelisted .o files which didn't get objtool generation.
*/
orc_a = cur_orc_table + (a - cur_orc_ip_table);
return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
}
void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
void *_orc, size_t orc_size)
{
int *orc_ip = _orc_ip;
struct orc_entry *orc = _orc;
unsigned int num_entries = orc_ip_size / sizeof(int);
WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 ||
orc_size % sizeof(*orc) != 0 ||
num_entries != orc_size / sizeof(*orc));
/*
* The 'cur_orc_*' globals allow the orc_sort_swap() callback to
* associate an .orc_unwind_ip table entry with its corresponding
* .orc_unwind entry so they can both be swapped.
*/
mutex_lock(&sort_mutex);
cur_orc_ip_table = orc_ip;
cur_orc_table = orc;
sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
mutex_unlock(&sort_mutex);
mod->arch.orc_unwind_ip = orc_ip;
mod->arch.orc_unwind = orc;
mod->arch.num_orcs = num_entries;
}
#endif
void __init unwind_init(void)
{
size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip;
size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind;
size_t num_entries = orc_ip_size / sizeof(int);
struct orc_entry *orc;
int i;
if (!num_entries || orc_ip_size % sizeof(int) != 0 ||
orc_size % sizeof(struct orc_entry) != 0 ||
num_entries != orc_size / sizeof(struct orc_entry)) {
orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n");
return;
}
/*
* Note, the orc_unwind and orc_unwind_ip tables were already
* sorted at build time via the 'sorttable' tool.
* It's ready for binary search straight away, no need to sort it.
*/
/* Initialize the fast lookup table: */
lookup_num_blocks = orc_lookup_end - orc_lookup;
for (i = 0; i < lookup_num_blocks-1; i++) {
orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
num_entries,
LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i));
if (!orc) {
orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
return;
}
orc_lookup[i] = orc - __start_orc_unwind;
}
/* Initialize the ending block: */
orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries,
LOOKUP_STOP_IP);
if (!orc) {
orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
return;
}
orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind;
orc_init = true;
}
unsigned long unwind_get_return_address(struct unwind_state *state)
{
if (unwind_done(state))
return 0;
return __kernel_text_address(state->ip) ? state->ip : 0;
}
EXPORT_SYMBOL_GPL(unwind_get_return_address);
unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)
{
if (unwind_done(state))
return NULL;
if (state->regs)
return &state->regs->ip;
if (state->sp)
return (unsigned long *)state->sp - 1;
return NULL;
}
static bool stack_access_ok(struct unwind_state *state, unsigned long _addr,
size_t len)
{ struct stack_info *info = &state->stack_info;
void *addr = (void *)_addr;
if (on_stack(info, addr, len))
return true;
return !get_stack_info(addr, state->task, info, &state->stack_mask) && on_stack(info, addr, len);
}
static bool deref_stack_reg(struct unwind_state *state, unsigned long addr,
unsigned long *val)
{
if (!stack_access_ok(state, addr, sizeof(long)))
return false;
*val = READ_ONCE_NOCHECK(*(unsigned long *)addr);
return true;
}
static bool deref_stack_regs(struct unwind_state *state, unsigned long addr,
unsigned long *ip, unsigned long *sp)
{
struct pt_regs *regs = (struct pt_regs *)addr;
/* x86-32 support will be more complicated due to the ®s->sp hack */
BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32));
if (!stack_access_ok(state, addr, sizeof(struct pt_regs)))
return false;
*ip = READ_ONCE_NOCHECK(regs->ip);
*sp = READ_ONCE_NOCHECK(regs->sp);
return true;
}
static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr,
unsigned long *ip, unsigned long *sp)
{
struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET; if (!stack_access_ok(state, addr, IRET_FRAME_SIZE))
return false;
*ip = READ_ONCE_NOCHECK(regs->ip);
*sp = READ_ONCE_NOCHECK(regs->sp);
return true;
}
/*
* If state->regs is non-NULL, and points to a full pt_regs, just get the reg
* value from state->regs.
*
* Otherwise, if state->regs just points to IRET regs, and the previous frame
* had full regs, it's safe to get the value from the previous regs. This can
* happen when early/late IRQ entry code gets interrupted by an NMI.
*/
static bool get_reg(struct unwind_state *state, unsigned int reg_off,
unsigned long *val)
{
unsigned int reg = reg_off/8;
if (!state->regs)
return false;
if (state->full_regs) {
*val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]);
return true;
}
if (state->prev_regs) { *val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]);
return true;
}
return false;
}
bool unwind_next_frame(struct unwind_state *state)
{
unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp; enum stack_type prev_type = state->stack_info.type;
struct orc_entry *orc;
bool indirect = false;
if (unwind_done(state))
return false;
/* Don't let modules unload while we're reading their ORC data. */
preempt_disable();
/* End-of-stack check for user tasks: */
if (state->regs && user_mode(state->regs))
goto the_end;
/*
* Find the orc_entry associated with the text address.
*
* For a call frame (as opposed to a signal frame), state->ip points to
* the instruction after the call. That instruction's stack layout
* could be different from the call instruction's layout, for example
* if the call was to a noreturn function. So get the ORC data for the
* call instruction itself.
*/
orc = orc_find(state->signal ? state->ip : state->ip - 1); if (!orc) {
/*
* As a fallback, try to assume this code uses a frame pointer.
* This is useful for generated code, like BPF, which ORC
* doesn't know about. This is just a guess, so the rest of
* the unwind is no longer considered reliable.
*/
orc = &orc_fp_entry;
state->error = true;
} else {
if (orc->type == ORC_TYPE_UNDEFINED)
goto err;
if (orc->type == ORC_TYPE_END_OF_STACK)
goto the_end;
}
state->signal = orc->signal;
/* Find the previous frame's stack: */
switch (orc->sp_reg) {
case ORC_REG_SP:
sp = state->sp + orc->sp_offset;
break;
case ORC_REG_BP:
sp = state->bp + orc->sp_offset;
break;
case ORC_REG_SP_INDIRECT:
sp = state->sp;
indirect = true;
break;
case ORC_REG_BP_INDIRECT:
sp = state->bp + orc->sp_offset;
indirect = true;
break;
case ORC_REG_R10:
if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) { orc_warn_current("missing R10 value at %pB\n",
(void *)state->ip);
goto err;
}
break;
case ORC_REG_R13:
if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) { orc_warn_current("missing R13 value at %pB\n",
(void *)state->ip);
goto err;
}
break;
case ORC_REG_DI:
if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) { orc_warn_current("missing RDI value at %pB\n",
(void *)state->ip);
goto err;
}
break;
case ORC_REG_DX:
if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) { orc_warn_current("missing DX value at %pB\n",
(void *)state->ip);
goto err;
}
break;
default:
orc_warn("unknown SP base reg %d at %pB\n",
orc->sp_reg, (void *)state->ip);
goto err;
}
if (indirect) {
if (!deref_stack_reg(state, sp, &sp))
goto err;
if (orc->sp_reg == ORC_REG_SP_INDIRECT)
sp += orc->sp_offset;
}
/* Find IP, SP and possibly regs: */
switch (orc->type) {
case ORC_TYPE_CALL:
ip_p = sp - sizeof(long); if (!deref_stack_reg(state, ip_p, &state->ip))
goto err;
state->ip = unwind_recover_ret_addr(state, state->ip,
(unsigned long *)ip_p);
state->sp = sp;
state->regs = NULL;
state->prev_regs = NULL;
break;
case ORC_TYPE_REGS:
if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access registers at %pB\n",
(void *)orig_ip);
goto err;
}
/*
* There is a small chance to interrupt at the entry of
* arch_rethook_trampoline() where the ORC info doesn't exist.
* That point is right after the RET to arch_rethook_trampoline()
* which was modified return address.
* At that point, the @addr_p of the unwind_recover_rethook()
* (this has to point the address of the stack entry storing
* the modified return address) must be "SP - (a stack entry)"
* because SP is incremented by the RET.
*/
state->ip = unwind_recover_rethook(state, state->ip,
(unsigned long *)(state->sp - sizeof(long)));
state->regs = (struct pt_regs *)sp;
state->prev_regs = NULL;
state->full_regs = true;
break;
case ORC_TYPE_REGS_PARTIAL:
if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access iret registers at %pB\n",
(void *)orig_ip);
goto err;
}
/* See ORC_TYPE_REGS case comment. */
state->ip = unwind_recover_rethook(state, state->ip,
(unsigned long *)(state->sp - sizeof(long)));
if (state->full_regs)
state->prev_regs = state->regs; state->regs = (void *)sp - IRET_FRAME_OFFSET; state->full_regs = false;
break;
default:
orc_warn("unknown .orc_unwind entry type %d at %pB\n",
orc->type, (void *)orig_ip);
goto err;
}
/* Find BP: */
switch (orc->bp_reg) {
case ORC_REG_UNDEFINED:
if (get_reg(state, offsetof(struct pt_regs, bp), &tmp)) state->bp = tmp;
break;
case ORC_REG_PREV_SP:
if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp))
goto err;
break;
case ORC_REG_BP:
if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp))
goto err;
break;
default:
orc_warn("unknown BP base reg %d for ip %pB\n",
orc->bp_reg, (void *)orig_ip);
goto err;
}
/* Prevent a recursive loop due to bad ORC data: */
if (state->stack_info.type == prev_type && on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) &&
state->sp <= prev_sp) {
orc_warn_current("stack going in the wrong direction? at %pB\n",
(void *)orig_ip);
goto err;
}
preempt_enable();
return true;
err:
state->error = true;
the_end:
preempt_enable(); state->stack_info.type = STACK_TYPE_UNKNOWN;
return false;
}
EXPORT_SYMBOL_GPL(unwind_next_frame);
void __unwind_start(struct unwind_state *state, struct task_struct *task,
struct pt_regs *regs, unsigned long *first_frame)
{
memset(state, 0, sizeof(*state));
state->task = task;
if (!orc_init)
goto err;
/*
* Refuse to unwind the stack of a task while it's executing on another
* CPU. This check is racy, but that's ok: the unwinder has other
* checks to prevent it from going off the rails.
*/
if (task_on_another_cpu(task))
goto err;
if (regs) { if (user_mode(regs))
goto the_end;
state->ip = regs->ip;
state->sp = regs->sp;
state->bp = regs->bp;
state->regs = regs;
state->full_regs = true;
state->signal = true;
} else if (task == current) { asm volatile("lea (%%rip), %0\n\t"
"mov %%rsp, %1\n\t"
"mov %%rbp, %2\n\t"
: "=r" (state->ip), "=r" (state->sp),
"=r" (state->bp));
} else {
struct inactive_task_frame *frame = (void *)task->thread.sp;
state->sp = task->thread.sp + sizeof(*frame);
state->bp = READ_ONCE_NOCHECK(frame->bp);
state->ip = READ_ONCE_NOCHECK(frame->ret_addr);
state->signal = (void *)state->ip == ret_from_fork;
}
if (get_stack_info((unsigned long *)state->sp, state->task,
&state->stack_info, &state->stack_mask)) {
/*
* We weren't on a valid stack. It's possible that
* we overflowed a valid stack into a guard page.
* See if the next page up is valid so that we can
* generate some kind of backtrace if this happens.
*/
void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp);
state->error = true;
if (get_stack_info(next_page, state->task, &state->stack_info,
&state->stack_mask))
return;
}
/*
* The caller can provide the address of the first frame directly
* (first_frame) or indirectly (regs->sp) to indicate which stack frame
* to start unwinding at. Skip ahead until we reach it.
*/
/* When starting from regs, skip the regs frame: */
if (regs) { unwind_next_frame(state);
return;
}
/* Otherwise, skip ahead to the user-specified starting frame: */
while (!unwind_done(state) && (!on_stack(&state->stack_info, first_frame, sizeof(long)) || state->sp <= (unsigned long)first_frame)) unwind_next_frame(state);
return;
err:
state->error = true;
the_end:
state->stack_info.type = STACK_TYPE_UNKNOWN;
}
EXPORT_SYMBOL_GPL(__unwind_start);
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/power/runtime.c - Helper functions for device runtime PM
*
* Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
* Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu>
*/
#include <linux/sched/mm.h>
#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <linux/export.h>
#include <linux/pm_runtime.h>
#include <linux/pm_wakeirq.h>
#include <linux/rculist.h>
#include <trace/events/rpm.h>
#include "../base.h"
#include "power.h"
typedef int (*pm_callback_t)(struct device *);
static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
{
pm_callback_t cb;
const struct dev_pm_ops *ops;
if (dev->pm_domain) ops = &dev->pm_domain->ops; else if (dev->type && dev->type->pm)
ops = dev->type->pm;
else if (dev->class && dev->class->pm)
ops = dev->class->pm;
else if (dev->bus && dev->bus->pm)
ops = dev->bus->pm;
else
ops = NULL;
if (ops)
cb = *(pm_callback_t *)((void *)ops + cb_offset);
else
cb = NULL;
if (!cb && dev->driver && dev->driver->pm) cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset); return cb;
}
#define RPM_GET_CALLBACK(dev, callback) \
__rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
static int rpm_resume(struct device *dev, int rpmflags);
static int rpm_suspend(struct device *dev, int rpmflags);
/**
* update_pm_runtime_accounting - Update the time accounting of power states
* @dev: Device to update the accounting for
*
* In order to be able to have time accounting of the various power states
* (as used by programs such as PowerTOP to show the effectiveness of runtime
* PM), we need to track the time spent in each state.
* update_pm_runtime_accounting must be called each time before the
* runtime_status field is updated, to account the time in the old state
* correctly.
*/
static void update_pm_runtime_accounting(struct device *dev)
{
u64 now, last, delta;
if (dev->power.disable_depth > 0)
return;
last = dev->power.accounting_timestamp;
now = ktime_get_mono_fast_ns();
dev->power.accounting_timestamp = now;
/*
* Because ktime_get_mono_fast_ns() is not monotonic during
* timekeeping updates, ensure that 'now' is after the last saved
* timesptamp.
*/
if (now < last)
return;
delta = now - last;
if (dev->power.runtime_status == RPM_SUSPENDED)
dev->power.suspended_time += delta;
else
dev->power.active_time += delta;
}
static void __update_runtime_status(struct device *dev, enum rpm_status status)
{
update_pm_runtime_accounting(dev); trace_rpm_status(dev, status); dev->power.runtime_status = status;
}
static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
{
u64 time;
unsigned long flags;
spin_lock_irqsave(&dev->power.lock, flags);
update_pm_runtime_accounting(dev);
time = suspended ? dev->power.suspended_time : dev->power.active_time;
spin_unlock_irqrestore(&dev->power.lock, flags);
return time;
}
u64 pm_runtime_active_time(struct device *dev)
{
return rpm_get_accounted_time(dev, false);
}
u64 pm_runtime_suspended_time(struct device *dev)
{
return rpm_get_accounted_time(dev, true);
}
EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
/**
* pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
* @dev: Device to handle.
*/
static void pm_runtime_deactivate_timer(struct device *dev)
{
if (dev->power.timer_expires > 0) { hrtimer_try_to_cancel(&dev->power.suspend_timer);
dev->power.timer_expires = 0;
}
}
/**
* pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
* @dev: Device to handle.
*/
static void pm_runtime_cancel_pending(struct device *dev)
{
pm_runtime_deactivate_timer(dev);
/*
* In case there's a request pending, make sure its work function will
* return without doing anything.
*/
dev->power.request = RPM_REQ_NONE;
}
/*
* pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
* @dev: Device to handle.
*
* Compute the autosuspend-delay expiration time based on the device's
* power.last_busy time. If the delay has already expired or is disabled
* (negative) or the power.use_autosuspend flag isn't set, return 0.
* Otherwise return the expiration time in nanoseconds (adjusted to be nonzero).
*
* This function may be called either with or without dev->power.lock held.
* Either way it can be racy, since power.last_busy may be updated at any time.
*/
u64 pm_runtime_autosuspend_expiration(struct device *dev)
{
int autosuspend_delay;
u64 expires;
if (!dev->power.use_autosuspend)
return 0;
autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
if (autosuspend_delay < 0)
return 0;
expires = READ_ONCE(dev->power.last_busy);
expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
if (expires > ktime_get_mono_fast_ns())
return expires; /* Expires in the future */
return 0;
}
EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
static int dev_memalloc_noio(struct device *dev, void *data)
{
return dev->power.memalloc_noio;
}
/*
* pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
* @dev: Device to handle.
* @enable: True for setting the flag and False for clearing the flag.
*
* Set the flag for all devices in the path from the device to the
* root device in the device tree if @enable is true, otherwise clear
* the flag for devices in the path whose siblings don't set the flag.
*
* The function should only be called by block device, or network
* device driver for solving the deadlock problem during runtime
* resume/suspend:
*
* If memory allocation with GFP_KERNEL is called inside runtime
* resume/suspend callback of any one of its ancestors(or the
* block device itself), the deadlock may be triggered inside the
* memory allocation since it might not complete until the block
* device becomes active and the involed page I/O finishes. The
* situation is pointed out first by Alan Stern. Network device
* are involved in iSCSI kind of situation.
*
* The lock of dev_hotplug_mutex is held in the function for handling
* hotplug race because pm_runtime_set_memalloc_noio() may be called
* in async probe().
*
* The function should be called between device_add() and device_del()
* on the affected device(block/network device).
*/
void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
{
static DEFINE_MUTEX(dev_hotplug_mutex);
mutex_lock(&dev_hotplug_mutex);
for (;;) {
bool enabled;
/* hold power lock since bitfield is not SMP-safe. */
spin_lock_irq(&dev->power.lock);
enabled = dev->power.memalloc_noio;
dev->power.memalloc_noio = enable;
spin_unlock_irq(&dev->power.lock);
/*
* not need to enable ancestors any more if the device
* has been enabled.
*/
if (enabled && enable)
break;
dev = dev->parent;
/*
* clear flag of the parent device only if all the
* children don't set the flag because ancestor's
* flag was set by any one of the descendants.
*/
if (!dev || (!enable && device_for_each_child(dev, NULL, dev_memalloc_noio)))
break;
}
mutex_unlock(&dev_hotplug_mutex);
}
EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
/**
* rpm_check_suspend_allowed - Test whether a device may be suspended.
* @dev: Device to test.
*/
static int rpm_check_suspend_allowed(struct device *dev)
{
int retval = 0;
if (dev->power.runtime_error)
retval = -EINVAL;
else if (dev->power.disable_depth > 0)
retval = -EACCES;
else if (atomic_read(&dev->power.usage_count))
retval = -EAGAIN;
else if (!dev->power.ignore_children && atomic_read(&dev->power.child_count))
retval = -EBUSY;
/* Pending resume requests take precedence over suspends. */
else if ((dev->power.deferred_resume && dev->power.runtime_status == RPM_SUSPENDING) || (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME))
retval = -EAGAIN;
else if (__dev_pm_qos_resume_latency(dev) == 0)
retval = -EPERM;
else if (dev->power.runtime_status == RPM_SUSPENDED)
retval = 1;
return retval;
}
static int rpm_get_suppliers(struct device *dev)
{
struct device_link *link;
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
device_links_read_lock_held()) {
int retval;
if (!(link->flags & DL_FLAG_PM_RUNTIME))
continue;
retval = pm_runtime_get_sync(link->supplier);
/* Ignore suppliers with disabled runtime PM. */
if (retval < 0 && retval != -EACCES) { pm_runtime_put_noidle(link->supplier);
return retval;
}
refcount_inc(&link->rpm_active);
}
return 0;
}
/**
* pm_runtime_release_supplier - Drop references to device link's supplier.
* @link: Target device link.
*
* Drop all runtime PM references associated with @link to its supplier device.
*/
void pm_runtime_release_supplier(struct device_link *link)
{
struct device *supplier = link->supplier;
/*
* The additional power.usage_count check is a safety net in case
* the rpm_active refcount becomes saturated, in which case
* refcount_dec_not_one() would return true forever, but it is not
* strictly necessary.
*/
while (refcount_dec_not_one(&link->rpm_active) && atomic_read(&supplier->power.usage_count) > 0) pm_runtime_put_noidle(supplier);
}
static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
{
struct device_link *link;
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
device_links_read_lock_held()) {
pm_runtime_release_supplier(link); if (try_to_suspend) pm_request_idle(link->supplier);
}
}
static void rpm_put_suppliers(struct device *dev)
{
__rpm_put_suppliers(dev, true);
}
static void rpm_suspend_suppliers(struct device *dev)
{
struct device_link *link;
int idx = device_links_read_lock();
list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
device_links_read_lock_held())
pm_request_idle(link->supplier); device_links_read_unlock(idx);
}
/**
* __rpm_callback - Run a given runtime PM callback for a given device.
* @cb: Runtime PM callback to run.
* @dev: Device to run the callback for.
*/
static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
__releases(&dev->power.lock) __acquires(&dev->power.lock)
{
int retval = 0, idx;
bool use_links = dev->power.links_count > 0;
if (dev->power.irq_safe) {
spin_unlock(&dev->power.lock);
} else {
spin_unlock_irq(&dev->power.lock);
/*
* Resume suppliers if necessary.
*
* The device's runtime PM status cannot change until this
* routine returns, so it is safe to read the status outside of
* the lock.
*/
if (use_links && dev->power.runtime_status == RPM_RESUMING) { idx = device_links_read_lock();
retval = rpm_get_suppliers(dev);
if (retval) {
rpm_put_suppliers(dev);
goto fail;
}
device_links_read_unlock(idx);
}
}
if (cb) retval = cb(dev); if (dev->power.irq_safe) { spin_lock(&dev->power.lock);
} else {
/*
* If the device is suspending and the callback has returned
* success, drop the usage counters of the suppliers that have
* been reference counted on its resume.
*
* Do that if resume fails too.
*/
if (use_links && ((dev->power.runtime_status == RPM_SUSPENDING && !retval) || (dev->power.runtime_status == RPM_RESUMING && retval))) { idx = device_links_read_lock();
__rpm_put_suppliers(dev, false);
fail:
device_links_read_unlock(idx);
}
spin_lock_irq(&dev->power.lock);
}
return retval;
}
/**
* rpm_callback - Run a given runtime PM callback for a given device.
* @cb: Runtime PM callback to run.
* @dev: Device to run the callback for.
*/
static int rpm_callback(int (*cb)(struct device *), struct device *dev)
{
int retval;
if (dev->power.memalloc_noio) {
unsigned int noio_flag;
/*
* Deadlock might be caused if memory allocation with
* GFP_KERNEL happens inside runtime_suspend and
* runtime_resume callbacks of one block device's
* ancestor or the block device itself. Network
* device might be thought as part of iSCSI block
* device, so network device and its ancestor should
* be marked as memalloc_noio too.
*/
noio_flag = memalloc_noio_save();
retval = __rpm_callback(cb, dev);
memalloc_noio_restore(noio_flag);
} else {
retval = __rpm_callback(cb, dev);
}
dev->power.runtime_error = retval; return retval != -EACCES ? retval : -EIO;
}
/**
* rpm_idle - Notify device bus type if the device can be suspended.
* @dev: Device to notify the bus type about.
* @rpmflags: Flag bits.
*
* Check if the device's runtime PM status allows it to be suspended. If
* another idle notification has been started earlier, return immediately. If
* the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
* run the ->runtime_idle() callback directly. If the ->runtime_idle callback
* doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
*
* This function must be called under dev->power.lock with interrupts disabled.
*/
static int rpm_idle(struct device *dev, int rpmflags)
{
int (*callback)(struct device *);
int retval;
trace_rpm_idle(dev, rpmflags); retval = rpm_check_suspend_allowed(dev);
if (retval < 0)
; /* Conditions are wrong. */
/* Idle notifications are allowed only in the RPM_ACTIVE state. */
else if (dev->power.runtime_status != RPM_ACTIVE)
retval = -EAGAIN;
/*
* Any pending request other than an idle notification takes
* precedence over us, except that the timer may be running.
*/
else if (dev->power.request_pending && dev->power.request > RPM_REQ_IDLE)
retval = -EAGAIN;
/* Act as though RPM_NOWAIT is always set. */
else if (dev->power.idle_notification)
retval = -EINPROGRESS;
if (retval)
goto out;
/* Pending requests need to be canceled. */
dev->power.request = RPM_REQ_NONE;
callback = RPM_GET_CALLBACK(dev, runtime_idle);
/* If no callback assume success. */
if (!callback || dev->power.no_callbacks)
goto out;
/* Carry out an asynchronous or a synchronous idle notification. */
if (rpmflags & RPM_ASYNC) { dev->power.request = RPM_REQ_IDLE;
if (!dev->power.request_pending) {
dev->power.request_pending = true;
queue_work(pm_wq, &dev->power.work);
}
trace_rpm_return_int(dev, _THIS_IP_, 0);
return 0;
}
dev->power.idle_notification = true;
if (dev->power.irq_safe)
spin_unlock(&dev->power.lock);
else
spin_unlock_irq(&dev->power.lock); retval = callback(dev);
if (dev->power.irq_safe)
spin_lock(&dev->power.lock);
else
spin_lock_irq(&dev->power.lock); dev->power.idle_notification = false;
wake_up_all(&dev->power.wait_queue);
out:
trace_rpm_return_int(dev, _THIS_IP_, retval); return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
}
/**
* rpm_suspend - Carry out runtime suspend of given device.
* @dev: Device to suspend.
* @rpmflags: Flag bits.
*
* Check if the device's runtime PM status allows it to be suspended.
* Cancel a pending idle notification, autosuspend or suspend. If
* another suspend has been started earlier, either return immediately
* or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
* flags. If the RPM_ASYNC flag is set then queue a suspend request;
* otherwise run the ->runtime_suspend() callback directly. When
* ->runtime_suspend succeeded, if a deferred resume was requested while
* the callback was running then carry it out, otherwise send an idle
* notification for its parent (if the suspend succeeded and both
* ignore_children of parent->power and irq_safe of dev->power are not set).
* If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
* flag is set and the next autosuspend-delay expiration time is in the
* future, schedule another autosuspend attempt.
*
* This function must be called under dev->power.lock with interrupts disabled.
*/
static int rpm_suspend(struct device *dev, int rpmflags)
__releases(&dev->power.lock) __acquires(&dev->power.lock)
{
int (*callback)(struct device *);
struct device *parent = NULL;
int retval;
trace_rpm_suspend(dev, rpmflags);
repeat:
retval = rpm_check_suspend_allowed(dev);
if (retval < 0)
goto out; /* Conditions are wrong. */
/* Synchronous suspends are not allowed in the RPM_RESUMING state. */
if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
retval = -EAGAIN;
if (retval)
goto out;
/* If the autosuspend_delay time hasn't expired yet, reschedule. */
if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) { u64 expires = pm_runtime_autosuspend_expiration(dev);
if (expires != 0) {
/* Pending requests need to be canceled. */
dev->power.request = RPM_REQ_NONE;
/*
* Optimization: If the timer is already running and is
* set to expire at or before the autosuspend delay,
* avoid the overhead of resetting it. Just let it
* expire; pm_suspend_timer_fn() will take care of the
* rest.
*/
if (!(dev->power.timer_expires &&
dev->power.timer_expires <= expires)) {
/*
* We add a slack of 25% to gather wakeups
* without sacrificing the granularity.
*/
u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
(NSEC_PER_MSEC >> 2);
dev->power.timer_expires = expires;
hrtimer_start_range_ns(&dev->power.suspend_timer,
ns_to_ktime(expires),
slack,
HRTIMER_MODE_ABS);
}
dev->power.timer_autosuspends = 1;
goto out;
}
}
/* Other scheduled or pending requests need to be canceled. */
pm_runtime_cancel_pending(dev);
if (dev->power.runtime_status == RPM_SUSPENDING) {
DEFINE_WAIT(wait);
if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
retval = -EINPROGRESS;
goto out;
}
if (dev->power.irq_safe) { spin_unlock(&dev->power.lock);
cpu_relax();
spin_lock(&dev->power.lock);
goto repeat;
}
/* Wait for the other suspend running in parallel with us. */
for (;;) {
prepare_to_wait(&dev->power.wait_queue, &wait,
TASK_UNINTERRUPTIBLE);
if (dev->power.runtime_status != RPM_SUSPENDING)
break;
spin_unlock_irq(&dev->power.lock);
schedule();
spin_lock_irq(&dev->power.lock);
}
finish_wait(&dev->power.wait_queue, &wait);
goto repeat;
}
if (dev->power.no_callbacks)
goto no_callback; /* Assume success. */
/* Carry out an asynchronous or a synchronous suspend. */
if (rpmflags & RPM_ASYNC) { dev->power.request = (rpmflags & RPM_AUTO) ? RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
if (!dev->power.request_pending) {
dev->power.request_pending = true;
queue_work(pm_wq, &dev->power.work);
}
goto out;
}
__update_runtime_status(dev, RPM_SUSPENDING);
callback = RPM_GET_CALLBACK(dev, runtime_suspend);
dev_pm_enable_wake_irq_check(dev, true);
retval = rpm_callback(callback, dev);
if (retval)
goto fail;
dev_pm_enable_wake_irq_complete(dev);
no_callback:
__update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_deactivate_timer(dev); if (dev->parent) {
parent = dev->parent;
atomic_add_unless(&parent->power.child_count, -1, 0);
}
wake_up_all(&dev->power.wait_queue);
if (dev->power.deferred_resume) {
dev->power.deferred_resume = false;
rpm_resume(dev, 0);
retval = -EAGAIN;
goto out;
}
if (dev->power.irq_safe)
goto out;
/* Maybe the parent is now able to suspend. */
if (parent && !parent->power.ignore_children) { spin_unlock(&dev->power.lock);
spin_lock(&parent->power.lock);
rpm_idle(parent, RPM_ASYNC);
spin_unlock(&parent->power.lock);
spin_lock(&dev->power.lock);
}
/* Maybe the suppliers are now able to suspend. */
if (dev->power.links_count > 0) { spin_unlock_irq(&dev->power.lock); rpm_suspend_suppliers(dev);
spin_lock_irq(&dev->power.lock);
}
out:
trace_rpm_return_int(dev, _THIS_IP_, retval);
return retval;
fail:
dev_pm_disable_wake_irq_check(dev, true);
__update_runtime_status(dev, RPM_ACTIVE);
dev->power.deferred_resume = false;
wake_up_all(&dev->power.wait_queue);
if (retval == -EAGAIN || retval == -EBUSY) { dev->power.runtime_error = 0;
/*
* If the callback routine failed an autosuspend, and
* if the last_busy time has been updated so that there
* is a new autosuspend expiration time, automatically
* reschedule another autosuspend.
*/
if ((rpmflags & RPM_AUTO) &&
pm_runtime_autosuspend_expiration(dev) != 0)
goto repeat;
} else {
pm_runtime_cancel_pending(dev);
}
goto out;
}
/**
* rpm_resume - Carry out runtime resume of given device.
* @dev: Device to resume.
* @rpmflags: Flag bits.
*
* Check if the device's runtime PM status allows it to be resumed. Cancel
* any scheduled or pending requests. If another resume has been started
* earlier, either return immediately or wait for it to finish, depending on the
* RPM_NOWAIT and RPM_ASYNC flags. Similarly, if there's a suspend running in
* parallel with this function, either tell the other process to resume after
* suspending (deferred_resume) or wait for it to finish. If the RPM_ASYNC
* flag is set then queue a resume request; otherwise run the
* ->runtime_resume() callback directly. Queue an idle notification for the
* device if the resume succeeded.
*
* This function must be called under dev->power.lock with interrupts disabled.
*/
static int rpm_resume(struct device *dev, int rpmflags)
__releases(&dev->power.lock) __acquires(&dev->power.lock)
{
int (*callback)(struct device *);
struct device *parent = NULL;
int retval = 0;
trace_rpm_resume(dev, rpmflags);
repeat:
if (dev->power.runtime_error) {
retval = -EINVAL;
} else if (dev->power.disable_depth > 0) { if (dev->power.runtime_status == RPM_ACTIVE && dev->power.last_status == RPM_ACTIVE)
retval = 1;
else
retval = -EACCES;
}
if (retval)
goto out;
/*
* Other scheduled or pending requests need to be canceled. Small
* optimization: If an autosuspend timer is running, leave it running
* rather than cancelling it now only to restart it again in the near
* future.
*/
dev->power.request = RPM_REQ_NONE;
if (!dev->power.timer_autosuspends)
pm_runtime_deactivate_timer(dev); if (dev->power.runtime_status == RPM_ACTIVE) {
retval = 1;
goto out;
}
if (dev->power.runtime_status == RPM_RESUMING ||
dev->power.runtime_status == RPM_SUSPENDING) {
DEFINE_WAIT(wait);
if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
if (dev->power.runtime_status == RPM_SUSPENDING) { dev->power.deferred_resume = true;
if (rpmflags & RPM_NOWAIT)
retval = -EINPROGRESS;
} else {
retval = -EINPROGRESS;
}
goto out;
}
if (dev->power.irq_safe) { spin_unlock(&dev->power.lock);
cpu_relax();
spin_lock(&dev->power.lock);
goto repeat;
}
/* Wait for the operation carried out in parallel with us. */
for (;;) {
prepare_to_wait(&dev->power.wait_queue, &wait,
TASK_UNINTERRUPTIBLE);
if (dev->power.runtime_status != RPM_RESUMING &&
dev->power.runtime_status != RPM_SUSPENDING)
break;
spin_unlock_irq(&dev->power.lock);
schedule();
spin_lock_irq(&dev->power.lock);
}
finish_wait(&dev->power.wait_queue, &wait);
goto repeat;
}
/*
* See if we can skip waking up the parent. This is safe only if
* power.no_callbacks is set, because otherwise we don't know whether
* the resume will actually succeed.
*/
if (dev->power.no_callbacks && !parent && dev->parent) { spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
if (dev->parent->power.disable_depth > 0 ||
dev->parent->power.ignore_children ||
dev->parent->power.runtime_status == RPM_ACTIVE) { atomic_inc(&dev->parent->power.child_count);
spin_unlock(&dev->parent->power.lock);
retval = 1;
goto no_callback; /* Assume success. */
}
spin_unlock(&dev->parent->power.lock);
}
/* Carry out an asynchronous or a synchronous resume. */
if (rpmflags & RPM_ASYNC) { dev->power.request = RPM_REQ_RESUME;
if (!dev->power.request_pending) {
dev->power.request_pending = true;
queue_work(pm_wq, &dev->power.work);
}
retval = 0;
goto out;
}
if (!parent && dev->parent) {
/*
* Increment the parent's usage counter and resume it if
* necessary. Not needed if dev is irq-safe; then the
* parent is permanently resumed.
*/
parent = dev->parent;
if (dev->power.irq_safe)
goto skip_parent;
spin_unlock(&dev->power.lock);
pm_runtime_get_noresume(parent);
spin_lock(&parent->power.lock);
/*
* Resume the parent if it has runtime PM enabled and not been
* set to ignore its children.
*/
if (!parent->power.disable_depth &&
!parent->power.ignore_children) { rpm_resume(parent, 0);
if (parent->power.runtime_status != RPM_ACTIVE)
retval = -EBUSY;
}
spin_unlock(&parent->power.lock);
spin_lock(&dev->power.lock);
if (retval)
goto out;
goto repeat;
}
skip_parent:
if (dev->power.no_callbacks)
goto no_callback; /* Assume success. */
__update_runtime_status(dev, RPM_RESUMING);
callback = RPM_GET_CALLBACK(dev, runtime_resume);
dev_pm_disable_wake_irq_check(dev, false);
retval = rpm_callback(callback, dev);
if (retval) {
__update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_cancel_pending(dev);
dev_pm_enable_wake_irq_check(dev, false);
} else {
no_callback:
__update_runtime_status(dev, RPM_ACTIVE);
pm_runtime_mark_last_busy(dev);
if (parent)
atomic_inc(&parent->power.child_count);
}
wake_up_all(&dev->power.wait_queue);
if (retval >= 0)
rpm_idle(dev, RPM_ASYNC);
out:
if (parent && !dev->power.irq_safe) { spin_unlock_irq(&dev->power.lock);
pm_runtime_put(parent);
spin_lock_irq(&dev->power.lock);
}
trace_rpm_return_int(dev, _THIS_IP_, retval);
return retval;
}
/**
* pm_runtime_work - Universal runtime PM work function.
* @work: Work structure used for scheduling the execution of this function.
*
* Use @work to get the device object the work is to be done for, determine what
* is to be done and execute the appropriate runtime PM function.
*/
static void pm_runtime_work(struct work_struct *work)
{
struct device *dev = container_of(work, struct device, power.work);
enum rpm_request req;
spin_lock_irq(&dev->power.lock);
if (!dev->power.request_pending)
goto out;
req = dev->power.request;
dev->power.request = RPM_REQ_NONE;
dev->power.request_pending = false;
switch (req) {
case RPM_REQ_NONE:
break;
case RPM_REQ_IDLE:
rpm_idle(dev, RPM_NOWAIT);
break;
case RPM_REQ_SUSPEND:
rpm_suspend(dev, RPM_NOWAIT);
break;
case RPM_REQ_AUTOSUSPEND:
rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
break;
case RPM_REQ_RESUME:
rpm_resume(dev, RPM_NOWAIT);
break;
}
out:
spin_unlock_irq(&dev->power.lock);
}
/**
* pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
* @timer: hrtimer used by pm_schedule_suspend().
*
* Check if the time is right and queue a suspend request.
*/
static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer)
{
struct device *dev = container_of(timer, struct device, power.suspend_timer);
unsigned long flags;
u64 expires;
spin_lock_irqsave(&dev->power.lock, flags);
expires = dev->power.timer_expires;
/*
* If 'expires' is after the current time, we've been called
* too early.
*/
if (expires > 0 && expires < ktime_get_mono_fast_ns()) {
dev->power.timer_expires = 0;
rpm_suspend(dev, dev->power.timer_autosuspends ?
(RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
}
spin_unlock_irqrestore(&dev->power.lock, flags);
return HRTIMER_NORESTART;
}
/**
* pm_schedule_suspend - Set up a timer to submit a suspend request in future.
* @dev: Device to suspend.
* @delay: Time to wait before submitting a suspend request, in milliseconds.
*/
int pm_schedule_suspend(struct device *dev, unsigned int delay)
{
unsigned long flags;
u64 expires;
int retval;
spin_lock_irqsave(&dev->power.lock, flags);
if (!delay) {
retval = rpm_suspend(dev, RPM_ASYNC);
goto out;
}
retval = rpm_check_suspend_allowed(dev);
if (retval)
goto out;
/* Other scheduled or pending requests need to be canceled. */
pm_runtime_cancel_pending(dev);
expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
dev->power.timer_expires = expires;
dev->power.timer_autosuspends = 0;
hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);
out:
spin_unlock_irqrestore(&dev->power.lock, flags);
return retval;
}
EXPORT_SYMBOL_GPL(pm_schedule_suspend);
static int rpm_drop_usage_count(struct device *dev)
{
int ret;
ret = atomic_sub_return(1, &dev->power.usage_count); if (ret >= 0)
return ret;
/*
* Because rpm_resume() does not check the usage counter, it will resume
* the device even if the usage counter is 0 or negative, so it is
* sufficient to increment the usage counter here to reverse the change
* made above.
*/
atomic_inc(&dev->power.usage_count);
dev_warn(dev, "Runtime PM usage count underflow!\n");
return -EINVAL;
}
/**
* __pm_runtime_idle - Entry point for runtime idle operations.
* @dev: Device to send idle notification for.
* @rpmflags: Flag bits.
*
* If the RPM_GET_PUT flag is set, decrement the device's usage count and
* return immediately if it is larger than zero (if it becomes negative, log a
* warning, increment it, and return an error). Then carry out an idle
* notification, either synchronous or asynchronous.
*
* This routine may be called in atomic context if the RPM_ASYNC flag is set,
* or if pm_runtime_irq_safe() has been called.
*/
int __pm_runtime_idle(struct device *dev, int rpmflags)
{
unsigned long flags;
int retval;
if (rpmflags & RPM_GET_PUT) { retval = rpm_drop_usage_count(dev);
if (retval < 0) {
return retval;
} else if (retval > 0) { trace_rpm_usage(dev, rpmflags);
return 0;
}
}
might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); spin_lock_irqsave(&dev->power.lock, flags);
retval = rpm_idle(dev, rpmflags);
spin_unlock_irqrestore(&dev->power.lock, flags);
return retval;
}
EXPORT_SYMBOL_GPL(__pm_runtime_idle);
/**
* __pm_runtime_suspend - Entry point for runtime put/suspend operations.
* @dev: Device to suspend.
* @rpmflags: Flag bits.
*
* If the RPM_GET_PUT flag is set, decrement the device's usage count and
* return immediately if it is larger than zero (if it becomes negative, log a
* warning, increment it, and return an error). Then carry out a suspend,
* either synchronous or asynchronous.
*
* This routine may be called in atomic context if the RPM_ASYNC flag is set,
* or if pm_runtime_irq_safe() has been called.
*/
int __pm_runtime_suspend(struct device *dev, int rpmflags)
{
unsigned long flags;
int retval;
if (rpmflags & RPM_GET_PUT) { retval = rpm_drop_usage_count(dev);
if (retval < 0) {
return retval;
} else if (retval > 0) { trace_rpm_usage(dev, rpmflags);
return 0;
}
}
might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); spin_lock_irqsave(&dev->power.lock, flags);
retval = rpm_suspend(dev, rpmflags);
spin_unlock_irqrestore(&dev->power.lock, flags);
return retval;
}
EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
/**
* __pm_runtime_resume - Entry point for runtime resume operations.
* @dev: Device to resume.
* @rpmflags: Flag bits.
*
* If the RPM_GET_PUT flag is set, increment the device's usage count. Then
* carry out a resume, either synchronous or asynchronous.
*
* This routine may be called in atomic context if the RPM_ASYNC flag is set,
* or if pm_runtime_irq_safe() has been called.
*/
int __pm_runtime_resume(struct device *dev, int rpmflags)
{
unsigned long flags;
int retval;
might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
dev->power.runtime_status != RPM_ACTIVE);
if (rpmflags & RPM_GET_PUT) atomic_inc(&dev->power.usage_count); spin_lock_irqsave(&dev->power.lock, flags);
retval = rpm_resume(dev, rpmflags);
spin_unlock_irqrestore(&dev->power.lock, flags);
return retval;
}
EXPORT_SYMBOL_GPL(__pm_runtime_resume);
/**
* pm_runtime_get_conditional - Conditionally bump up device usage counter.
* @dev: Device to handle.
* @ign_usage_count: Whether or not to look at the current usage counter value.
*
* Return -EINVAL if runtime PM is disabled for @dev.
*
* Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either
* @ign_usage_count is %true or the runtime PM usage counter of @dev is not
* zero, increment the usage counter of @dev and return 1. Otherwise, return 0
* without changing the usage counter.
*
* If @ign_usage_count is %true, this function can be used to prevent suspending
* the device when its runtime PM status is %RPM_ACTIVE.
*
* If @ign_usage_count is %false, this function can be used to prevent
* suspending the device when both its runtime PM status is %RPM_ACTIVE and its
* runtime PM usage counter is not zero.
*
* The caller is responsible for decrementing the runtime PM usage counter of
* @dev after this function has returned a positive value for it.
*/
static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count)
{
unsigned long flags;
int retval;
spin_lock_irqsave(&dev->power.lock, flags);
if (dev->power.disable_depth > 0) {
retval = -EINVAL;
} else if (dev->power.runtime_status != RPM_ACTIVE) {
retval = 0;
} else if (ign_usage_count) {
retval = 1;
atomic_inc(&dev->power.usage_count);
} else {
retval = atomic_inc_not_zero(&dev->power.usage_count);
}
trace_rpm_usage(dev, 0);
spin_unlock_irqrestore(&dev->power.lock, flags);
return retval;
}
/**
* pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is
* in active state
* @dev: Target device.
*
* Increment the runtime PM usage counter of @dev if its runtime PM status is
* %RPM_ACTIVE, in which case it returns 1. If the device is in a different
* state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the
* device, in which case also the usage_count will remain unmodified.
*/
int pm_runtime_get_if_active(struct device *dev)
{
return pm_runtime_get_conditional(dev, true);
}
EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
/**
* pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter.
* @dev: Target device.
*
* Increment the runtime PM usage counter of @dev if its runtime PM status is
* %RPM_ACTIVE and its runtime PM usage counter is greater than 0, in which case
* it returns 1. If the device is in a different state or its usage_count is 0,
* 0 is returned. -EINVAL is returned if runtime PM is disabled for the device,
* in which case also the usage_count will remain unmodified.
*/
int pm_runtime_get_if_in_use(struct device *dev)
{
return pm_runtime_get_conditional(dev, false);
}
EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use);
/**
* __pm_runtime_set_status - Set runtime PM status of a device.
* @dev: Device to handle.
* @status: New runtime PM status of the device.
*
* If runtime PM of the device is disabled or its power.runtime_error field is
* different from zero, the status may be changed either to RPM_ACTIVE, or to
* RPM_SUSPENDED, as long as that reflects the actual state of the device.
* However, if the device has a parent and the parent is not active, and the
* parent's power.ignore_children flag is unset, the device's status cannot be
* set to RPM_ACTIVE, so -EBUSY is returned in that case.
*
* If successful, __pm_runtime_set_status() clears the power.runtime_error field
* and the device parent's counter of unsuspended children is modified to
* reflect the new status. If the new status is RPM_SUSPENDED, an idle
* notification request for the parent is submitted.
*
* If @dev has any suppliers (as reflected by device links to them), and @status
* is RPM_ACTIVE, they will be activated upfront and if the activation of one
* of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
* of the @status value) and the suppliers will be deacticated on exit. The
* error returned by the failing supplier activation will be returned in that
* case.
*/
int __pm_runtime_set_status(struct device *dev, unsigned int status)
{
struct device *parent = dev->parent;
bool notify_parent = false;
unsigned long flags;
int error = 0;
if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
return -EINVAL;
spin_lock_irqsave(&dev->power.lock, flags);
/*
* Prevent PM-runtime from being enabled for the device or return an
* error if it is enabled already and working.
*/
if (dev->power.runtime_error || dev->power.disable_depth) dev->power.disable_depth++;
else
error = -EAGAIN;
spin_unlock_irqrestore(&dev->power.lock, flags); if (error)
return error;
/*
* If the new status is RPM_ACTIVE, the suppliers can be activated
* upfront regardless of the current status, because next time
* rpm_put_suppliers() runs, the rpm_active refcounts of the links
* involved will be dropped down to one anyway.
*/
if (status == RPM_ACTIVE) {
int idx = device_links_read_lock();
error = rpm_get_suppliers(dev);
if (error)
status = RPM_SUSPENDED;
device_links_read_unlock(idx);
}
spin_lock_irqsave(&dev->power.lock, flags); if (dev->power.runtime_status == status || !parent)
goto out_set;
if (status == RPM_SUSPENDED) { atomic_add_unless(&parent->power.child_count, -1, 0); notify_parent = !parent->power.ignore_children;
} else {
spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
/*
* It is invalid to put an active child under a parent that is
* not active, has runtime PM enabled and the
* 'power.ignore_children' flag unset.
*/
if (!parent->power.disable_depth &&
!parent->power.ignore_children && parent->power.runtime_status != RPM_ACTIVE) { dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
dev_name(dev),
dev_name(parent));
error = -EBUSY;
} else if (dev->power.runtime_status == RPM_SUSPENDED) { atomic_inc(&parent->power.child_count);
}
spin_unlock(&parent->power.lock);
if (error) {
status = RPM_SUSPENDED;
goto out;
}
}
out_set:
__update_runtime_status(dev, status);
if (!error)
dev->power.runtime_error = 0;
out:
spin_unlock_irqrestore(&dev->power.lock, flags);
if (notify_parent)
pm_request_idle(parent); if (status == RPM_SUSPENDED) { int idx = device_links_read_lock();
rpm_put_suppliers(dev);
device_links_read_unlock(idx);
}
pm_runtime_enable(dev);
return error;
}
EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
/**
* __pm_runtime_barrier - Cancel pending requests and wait for completions.
* @dev: Device to handle.
*
* Flush all pending requests for the device from pm_wq and wait for all
* runtime PM operations involving the device in progress to complete.
*
* Should be called under dev->power.lock with interrupts disabled.
*/
static void __pm_runtime_barrier(struct device *dev)
{
pm_runtime_deactivate_timer(dev); if (dev->power.request_pending) { dev->power.request = RPM_REQ_NONE;
spin_unlock_irq(&dev->power.lock);
cancel_work_sync(&dev->power.work);
spin_lock_irq(&dev->power.lock);
dev->power.request_pending = false;
}
if (dev->power.runtime_status == RPM_SUSPENDING || dev->power.runtime_status == RPM_RESUMING ||
dev->power.idle_notification) {
DEFINE_WAIT(wait);
/* Suspend, wake-up or idle notification in progress. */
for (;;) {
prepare_to_wait(&dev->power.wait_queue, &wait,
TASK_UNINTERRUPTIBLE);
if (dev->power.runtime_status != RPM_SUSPENDING
&& dev->power.runtime_status != RPM_RESUMING
&& !dev->power.idle_notification)
break;
spin_unlock_irq(&dev->power.lock);
schedule();
spin_lock_irq(&dev->power.lock);
}
finish_wait(&dev->power.wait_queue, &wait);
}
}
/**
* pm_runtime_barrier - Flush pending requests and wait for completions.
* @dev: Device to handle.
*
* Prevent the device from being suspended by incrementing its usage counter and
* if there's a pending resume request for the device, wake the device up.
* Next, make sure that all pending requests for the device have been flushed
* from pm_wq and wait for all runtime PM operations involving the device in
* progress to complete.
*
* Return value:
* 1, if there was a resume request pending and the device had to be woken up,
* 0, otherwise
*/
int pm_runtime_barrier(struct device *dev)
{
int retval = 0;
pm_runtime_get_noresume(dev);
spin_lock_irq(&dev->power.lock);
if (dev->power.request_pending
&& dev->power.request == RPM_REQ_RESUME) { rpm_resume(dev, 0);
retval = 1;
}
__pm_runtime_barrier(dev);
spin_unlock_irq(&dev->power.lock);
pm_runtime_put_noidle(dev); return retval;
}
EXPORT_SYMBOL_GPL(pm_runtime_barrier);
/**
* __pm_runtime_disable - Disable runtime PM of a device.
* @dev: Device to handle.
* @check_resume: If set, check if there's a resume request for the device.
*
* Increment power.disable_depth for the device and if it was zero previously,
* cancel all pending runtime PM requests for the device and wait for all
* operations in progress to complete. The device can be either active or
* suspended after its runtime PM has been disabled.
*
* If @check_resume is set and there's a resume request pending when
* __pm_runtime_disable() is called and power.disable_depth is zero, the
* function will wake up the device before disabling its runtime PM.
*/
void __pm_runtime_disable(struct device *dev, bool check_resume)
{
spin_lock_irq(&dev->power.lock);
if (dev->power.disable_depth > 0) {
dev->power.disable_depth++;
goto out;
}
/*
* Wake up the device if there's a resume request pending, because that
* means there probably is some I/O to process and disabling runtime PM
* shouldn't prevent the device from processing the I/O.
*/
if (check_resume && dev->power.request_pending && dev->power.request == RPM_REQ_RESUME) {
/*
* Prevent suspends and idle notifications from being carried
* out after we have woken up the device.
*/
pm_runtime_get_noresume(dev);
rpm_resume(dev, 0);
pm_runtime_put_noidle(dev);
}
/* Update time accounting before disabling PM-runtime. */
update_pm_runtime_accounting(dev); if (!dev->power.disable_depth++) { __pm_runtime_barrier(dev);
dev->power.last_status = dev->power.runtime_status;
}
out:
spin_unlock_irq(&dev->power.lock);
}
EXPORT_SYMBOL_GPL(__pm_runtime_disable);
/**
* pm_runtime_enable - Enable runtime PM of a device.
* @dev: Device to handle.
*/
void pm_runtime_enable(struct device *dev)
{
unsigned long flags;
spin_lock_irqsave(&dev->power.lock, flags);
if (!dev->power.disable_depth) {
dev_warn(dev, "Unbalanced %s!\n", __func__);
goto out;
}
if (--dev->power.disable_depth > 0)
goto out;
dev->power.last_status = RPM_INVALID;
dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
if (dev->power.runtime_status == RPM_SUSPENDED &&
!dev->power.ignore_children && atomic_read(&dev->power.child_count) > 0) dev_warn(dev, "Enabling runtime PM for inactive device with active children\n");
out:
spin_unlock_irqrestore(&dev->power.lock, flags);
}
EXPORT_SYMBOL_GPL(pm_runtime_enable);
static void pm_runtime_disable_action(void *data)
{
pm_runtime_dont_use_autosuspend(data);
pm_runtime_disable(data);
}
/**
* devm_pm_runtime_enable - devres-enabled version of pm_runtime_enable.
*
* NOTE: this will also handle calling pm_runtime_dont_use_autosuspend() for
* you at driver exit time if needed.
*
* @dev: Device to handle.
*/
int devm_pm_runtime_enable(struct device *dev)
{
pm_runtime_enable(dev);
return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev);
}
EXPORT_SYMBOL_GPL(devm_pm_runtime_enable);
/**
* pm_runtime_forbid - Block runtime PM of a device.
* @dev: Device to handle.
*
* Increase the device's usage count and clear its power.runtime_auto flag,
* so that it cannot be suspended at run time until pm_runtime_allow() is called
* for it.
*/
void pm_runtime_forbid(struct device *dev)
{
spin_lock_irq(&dev->power.lock);
if (!dev->power.runtime_auto)
goto out;
dev->power.runtime_auto = false;
atomic_inc(&dev->power.usage_count);
rpm_resume(dev, 0);
out:
spin_unlock_irq(&dev->power.lock);
}
EXPORT_SYMBOL_GPL(pm_runtime_forbid);
/**
* pm_runtime_allow - Unblock runtime PM of a device.
* @dev: Device to handle.
*
* Decrease the device's usage count and set its power.runtime_auto flag.
*/
void pm_runtime_allow(struct device *dev)
{
int ret;
spin_lock_irq(&dev->power.lock);
if (dev->power.runtime_auto)
goto out;
dev->power.runtime_auto = true;
ret = rpm_drop_usage_count(dev);
if (ret == 0)
rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
else if (ret > 0)
trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC);
out:
spin_unlock_irq(&dev->power.lock);
}
EXPORT_SYMBOL_GPL(pm_runtime_allow);
/**
* pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
* @dev: Device to handle.
*
* Set the power.no_callbacks flag, which tells the PM core that this
* device is power-managed through its parent and has no runtime PM
* callbacks of its own. The runtime sysfs attributes will be removed.
*/
void pm_runtime_no_callbacks(struct device *dev)
{
spin_lock_irq(&dev->power.lock);
dev->power.no_callbacks = 1;
spin_unlock_irq(&dev->power.lock);
if (device_is_registered(dev))
rpm_sysfs_remove(dev);
}
EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
/**
* pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
* @dev: Device to handle
*
* Set the power.irq_safe flag, which tells the PM core that the
* ->runtime_suspend() and ->runtime_resume() callbacks for this device should
* always be invoked with the spinlock held and interrupts disabled. It also
* causes the parent's usage counter to be permanently incremented, preventing
* the parent from runtime suspending -- otherwise an irq-safe child might have
* to wait for a non-irq-safe parent.
*/
void pm_runtime_irq_safe(struct device *dev)
{
if (dev->parent)
pm_runtime_get_sync(dev->parent);
spin_lock_irq(&dev->power.lock);
dev->power.irq_safe = 1;
spin_unlock_irq(&dev->power.lock);
}
EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
/**
* update_autosuspend - Handle a change to a device's autosuspend settings.
* @dev: Device to handle.
* @old_delay: The former autosuspend_delay value.
* @old_use: The former use_autosuspend value.
*
* Prevent runtime suspend if the new delay is negative and use_autosuspend is
* set; otherwise allow it. Send an idle notification if suspends are allowed.
*
* This function must be called under dev->power.lock with interrupts disabled.
*/
static void update_autosuspend(struct device *dev, int old_delay, int old_use)
{
int delay = dev->power.autosuspend_delay;
/* Should runtime suspend be prevented now? */
if (dev->power.use_autosuspend && delay < 0) {
/* If it used to be allowed then prevent it. */
if (!old_use || old_delay >= 0) { atomic_inc(&dev->power.usage_count);
rpm_resume(dev, 0);
} else {
trace_rpm_usage(dev, 0);
}
}
/* Runtime suspend should be allowed now. */
else {
/* If it used to be prevented then allow it. */
if (old_use && old_delay < 0) atomic_dec(&dev->power.usage_count);
/* Maybe we can autosuspend now. */
rpm_idle(dev, RPM_AUTO);
}
}
/**
* pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
* @dev: Device to handle.
* @delay: Value of the new delay in milliseconds.
*
* Set the device's power.autosuspend_delay value. If it changes to negative
* and the power.use_autosuspend flag is set, prevent runtime suspends. If it
* changes the other way, allow runtime suspends.
*/
void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
{
int old_delay, old_use;
spin_lock_irq(&dev->power.lock);
old_delay = dev->power.autosuspend_delay;
old_use = dev->power.use_autosuspend;
dev->power.autosuspend_delay = delay;
update_autosuspend(dev, old_delay, old_use);
spin_unlock_irq(&dev->power.lock);
}
EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
/**
* __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
* @dev: Device to handle.
* @use: New value for use_autosuspend.
*
* Set the device's power.use_autosuspend flag, and allow or prevent runtime
* suspends as needed.
*/
void __pm_runtime_use_autosuspend(struct device *dev, bool use)
{
int old_delay, old_use;
spin_lock_irq(&dev->power.lock);
old_delay = dev->power.autosuspend_delay;
old_use = dev->power.use_autosuspend;
dev->power.use_autosuspend = use;
update_autosuspend(dev, old_delay, old_use);
spin_unlock_irq(&dev->power.lock);
}
EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
/**
* pm_runtime_init - Initialize runtime PM fields in given device object.
* @dev: Device object to initialize.
*/
void pm_runtime_init(struct device *dev)
{
dev->power.runtime_status = RPM_SUSPENDED;
dev->power.last_status = RPM_INVALID;
dev->power.idle_notification = false;
dev->power.disable_depth = 1;
atomic_set(&dev->power.usage_count, 0);
dev->power.runtime_error = 0;
atomic_set(&dev->power.child_count, 0);
pm_suspend_ignore_children(dev, false);
dev->power.runtime_auto = true;
dev->power.request_pending = false;
dev->power.request = RPM_REQ_NONE;
dev->power.deferred_resume = false;
dev->power.needs_force_resume = 0;
INIT_WORK(&dev->power.work, pm_runtime_work);
dev->power.timer_expires = 0;
hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
dev->power.suspend_timer.function = pm_suspend_timer_fn;
init_waitqueue_head(&dev->power.wait_queue);
}
/**
* pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
* @dev: Device object to re-initialize.
*/
void pm_runtime_reinit(struct device *dev)
{
if (!pm_runtime_enabled(dev)) { if (dev->power.runtime_status == RPM_ACTIVE) pm_runtime_set_suspended(dev); if (dev->power.irq_safe) { spin_lock_irq(&dev->power.lock);
dev->power.irq_safe = 0;
spin_unlock_irq(&dev->power.lock);
if (dev->parent)
pm_runtime_put(dev->parent);
}
}
}
/**
* pm_runtime_remove - Prepare for removing a device from device hierarchy.
* @dev: Device object being removed from device hierarchy.
*/
void pm_runtime_remove(struct device *dev)
{
__pm_runtime_disable(dev, false); pm_runtime_reinit(dev);
}
/**
* pm_runtime_get_suppliers - Resume and reference-count supplier devices.
* @dev: Consumer device.
*/
void pm_runtime_get_suppliers(struct device *dev)
{
struct device_link *link;
int idx;
idx = device_links_read_lock(); list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
device_links_read_lock_held())
if (link->flags & DL_FLAG_PM_RUNTIME) { link->supplier_preactivated = true;
pm_runtime_get_sync(link->supplier);
}
device_links_read_unlock(idx);
}
/**
* pm_runtime_put_suppliers - Drop references to supplier devices.
* @dev: Consumer device.
*/
void pm_runtime_put_suppliers(struct device *dev)
{
struct device_link *link;
int idx;
idx = device_links_read_lock(); list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
device_links_read_lock_held())
if (link->supplier_preactivated) { link->supplier_preactivated = false;
pm_runtime_put(link->supplier);
}
device_links_read_unlock(idx);
}
void pm_runtime_new_link(struct device *dev)
{
spin_lock_irq(&dev->power.lock);
dev->power.links_count++;
spin_unlock_irq(&dev->power.lock);
}
static void pm_runtime_drop_link_count(struct device *dev)
{
spin_lock_irq(&dev->power.lock);
WARN_ON(dev->power.links_count == 0);
dev->power.links_count--;
spin_unlock_irq(&dev->power.lock);
}
/**
* pm_runtime_drop_link - Prepare for device link removal.
* @link: Device link going away.
*
* Drop the link count of the consumer end of @link and decrement the supplier
* device's runtime PM usage counter as many times as needed to drop all of the
* PM runtime reference to it from the consumer.
*/
void pm_runtime_drop_link(struct device_link *link)
{
if (!(link->flags & DL_FLAG_PM_RUNTIME))
return;
pm_runtime_drop_link_count(link->consumer);
pm_runtime_release_supplier(link);
pm_request_idle(link->supplier);
}
static bool pm_runtime_need_not_resume(struct device *dev)
{
return atomic_read(&dev->power.usage_count) <= 1 &&
(atomic_read(&dev->power.child_count) == 0 ||
dev->power.ignore_children);
}
/**
* pm_runtime_force_suspend - Force a device into suspend state if needed.
* @dev: Device to suspend.
*
* Disable runtime PM so we safely can check the device's runtime PM status and
* if it is active, invoke its ->runtime_suspend callback to suspend it and
* change its runtime PM status field to RPM_SUSPENDED. Also, if the device's
* usage and children counters don't indicate that the device was in use before
* the system-wide transition under way, decrement its parent's children counter
* (if there is a parent). Keep runtime PM disabled to preserve the state
* unless we encounter errors.
*
* Typically this function may be invoked from a system suspend callback to make
* sure the device is put into low power state and it should only be used during
* system-wide PM transitions to sleep states. It assumes that the analogous
* pm_runtime_force_resume() will be used to resume the device.
*
* Do not use with DPM_FLAG_SMART_SUSPEND as this can lead to an inconsistent
* state where this function has called the ->runtime_suspend callback but the
* PM core marks the driver as runtime active.
*/
int pm_runtime_force_suspend(struct device *dev)
{
int (*callback)(struct device *);
int ret;
pm_runtime_disable(dev);
if (pm_runtime_status_suspended(dev))
return 0;
callback = RPM_GET_CALLBACK(dev, runtime_suspend);
dev_pm_enable_wake_irq_check(dev, true);
ret = callback ? callback(dev) : 0;
if (ret)
goto err;
dev_pm_enable_wake_irq_complete(dev);
/*
* If the device can stay in suspend after the system-wide transition
* to the working state that will follow, drop the children counter of
* its parent, but set its status to RPM_SUSPENDED anyway in case this
* function will be called again for it in the meantime.
*/
if (pm_runtime_need_not_resume(dev)) {
pm_runtime_set_suspended(dev);
} else {
__update_runtime_status(dev, RPM_SUSPENDED);
dev->power.needs_force_resume = 1;
}
return 0;
err:
dev_pm_disable_wake_irq_check(dev, true);
pm_runtime_enable(dev);
return ret;
}
EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
/**
* pm_runtime_force_resume - Force a device into resume state if needed.
* @dev: Device to resume.
*
* Prior invoking this function we expect the user to have brought the device
* into low power state by a call to pm_runtime_force_suspend(). Here we reverse
* those actions and bring the device into full power, if it is expected to be
* used on system resume. In the other case, we defer the resume to be managed
* via runtime PM.
*
* Typically this function may be invoked from a system resume callback.
*/
int pm_runtime_force_resume(struct device *dev)
{
int (*callback)(struct device *);
int ret = 0;
if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume)
goto out;
/*
* The value of the parent's children counter is correct already, so
* just update the status of the device.
*/
__update_runtime_status(dev, RPM_ACTIVE);
callback = RPM_GET_CALLBACK(dev, runtime_resume);
dev_pm_disable_wake_irq_check(dev, false);
ret = callback ? callback(dev) : 0;
if (ret) {
pm_runtime_set_suspended(dev);
dev_pm_enable_wake_irq_check(dev, false);
goto out;
}
pm_runtime_mark_last_busy(dev);
out:
dev->power.needs_force_resume = 0;
pm_runtime_enable(dev);
return ret;
}
EXPORT_SYMBOL_GPL(pm_runtime_force_resume);
/* SPDX-License-Identifier: GPL-2.0-or-later
*
* Copyright (C) 2005 David Brownell
*/
#ifndef __LINUX_SPI_H
#define __LINUX_SPI_H
#include <linux/acpi.h>
#include <linux/bits.h>
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/gpio/consumer.h>
#include <linux/kthread.h>
#include <linux/mod_devicetable.h>
#include <linux/overflow.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/u64_stats_sync.h>
#include <uapi/linux/spi/spi.h>
/* Max no. of CS supported per spi device */
#define SPI_CS_CNT_MAX 16
struct dma_chan;
struct software_node;
struct ptp_system_timestamp;
struct spi_controller;
struct spi_transfer;
struct spi_controller_mem_ops;
struct spi_controller_mem_caps;
struct spi_message;
/*
* INTERFACES between SPI master-side drivers and SPI slave protocol handlers,
* and SPI infrastructure.
*/
extern const struct bus_type spi_bus_type;
/**
* struct spi_statistics - statistics for spi transfers
* @syncp: seqcount to protect members in this struct for per-cpu update
* on 32-bit systems
*
* @messages: number of spi-messages handled
* @transfers: number of spi_transfers handled
* @errors: number of errors during spi_transfer
* @timedout: number of timeouts during spi_transfer
*
* @spi_sync: number of times spi_sync is used
* @spi_sync_immediate:
* number of times spi_sync is executed immediately
* in calling context without queuing and scheduling
* @spi_async: number of times spi_async is used
*
* @bytes: number of bytes transferred to/from device
* @bytes_tx: number of bytes sent to device
* @bytes_rx: number of bytes received from device
*
* @transfer_bytes_histo:
* transfer bytes histogram
*
* @transfers_split_maxsize:
* number of transfers that have been split because of
* maxsize limit
*/
struct spi_statistics {
struct u64_stats_sync syncp;
u64_stats_t messages;
u64_stats_t transfers;
u64_stats_t errors;
u64_stats_t timedout;
u64_stats_t spi_sync;
u64_stats_t spi_sync_immediate;
u64_stats_t spi_async;
u64_stats_t bytes;
u64_stats_t bytes_rx;
u64_stats_t bytes_tx;
#define SPI_STATISTICS_HISTO_SIZE 17
u64_stats_t transfer_bytes_histo[SPI_STATISTICS_HISTO_SIZE];
u64_stats_t transfers_split_maxsize;
};
#define SPI_STATISTICS_ADD_TO_FIELD(pcpu_stats, field, count) \
do { \
struct spi_statistics *__lstats; \
get_cpu(); \
__lstats = this_cpu_ptr(pcpu_stats); \
u64_stats_update_begin(&__lstats->syncp); \
u64_stats_add(&__lstats->field, count); \
u64_stats_update_end(&__lstats->syncp); \
put_cpu(); \
} while (0)
#define SPI_STATISTICS_INCREMENT_FIELD(pcpu_stats, field) \
do { \
struct spi_statistics *__lstats; \
get_cpu(); \
__lstats = this_cpu_ptr(pcpu_stats); \
u64_stats_update_begin(&__lstats->syncp); \
u64_stats_inc(&__lstats->field); \
u64_stats_update_end(&__lstats->syncp); \
put_cpu(); \
} while (0)
/**
* struct spi_delay - SPI delay information
* @value: Value for the delay
* @unit: Unit for the delay
*/
struct spi_delay {
#define SPI_DELAY_UNIT_USECS 0
#define SPI_DELAY_UNIT_NSECS 1
#define SPI_DELAY_UNIT_SCK 2
u16 value;
u8 unit;
};
extern int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer);
extern int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer);
extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg,
struct spi_transfer *xfer);
/**
* struct spi_device - Controller side proxy for an SPI slave device
* @dev: Driver model representation of the device.
* @controller: SPI controller used with the device.
* @max_speed_hz: Maximum clock rate to be used with this chip
* (on this board); may be changed by the device's driver.
* The spi_transfer.speed_hz can override this for each transfer.
* @chip_select: Array of physical chipselect, spi->chipselect[i] gives
* the corresponding physical CS for logical CS i.
* @mode: The spi mode defines how data is clocked out and in.
* This may be changed by the device's driver.
* The "active low" default for chipselect mode can be overridden
* (by specifying SPI_CS_HIGH) as can the "MSB first" default for
* each word in a transfer (by specifying SPI_LSB_FIRST).
* @bits_per_word: Data transfers involve one or more words; word sizes
* like eight or 12 bits are common. In-memory wordsizes are
* powers of two bytes (e.g. 20 bit samples use 32 bits).
* This may be changed by the device's driver, or left at the
* default (0) indicating protocol words are eight bit bytes.
* The spi_transfer.bits_per_word can override this for each transfer.
* @rt: Make the pump thread real time priority.
* @irq: Negative, or the number passed to request_irq() to receive
* interrupts from this device.
* @controller_state: Controller's runtime state
* @controller_data: Board-specific definitions for controller, such as
* FIFO initialization parameters; from board_info.controller_data
* @modalias: Name of the driver to use with this device, or an alias
* for that name. This appears in the sysfs "modalias" attribute
* for driver coldplugging, and in uevents used for hotplugging
* @driver_override: If the name of a driver is written to this attribute, then
* the device will bind to the named driver and only the named driver.
* Do not set directly, because core frees it; use driver_set_override() to
* set or clear it.
* @cs_gpiod: Array of GPIO descriptors of the corresponding chipselect lines
* (optional, NULL when not using a GPIO line)
* @word_delay: delay to be inserted between consecutive
* words of a transfer
* @cs_setup: delay to be introduced by the controller after CS is asserted
* @cs_hold: delay to be introduced by the controller before CS is deasserted
* @cs_inactive: delay to be introduced by the controller after CS is
* deasserted. If @cs_change_delay is used from @spi_transfer, then the
* two delays will be added up.
* @pcpu_statistics: statistics for the spi_device
* @cs_index_mask: Bit mask of the active chipselect(s) in the chipselect array
*
* A @spi_device is used to interchange data between an SPI slave
* (usually a discrete chip) and CPU memory.
*
* In @dev, the platform_data is used to hold information about this
* device that's meaningful to the device's protocol driver, but not
* to its controller. One example might be an identifier for a chip
* variant with slightly different functionality; another might be
* information about how this particular board wires the chip's pins.
*/
struct spi_device {
struct device dev;
struct spi_controller *controller;
u32 max_speed_hz;
u8 chip_select[SPI_CS_CNT_MAX];
u8 bits_per_word;
bool rt;
#define SPI_NO_TX BIT(31) /* No transmit wire */
#define SPI_NO_RX BIT(30) /* No receive wire */
/*
* TPM specification defines flow control over SPI. Client device
* can insert a wait state on MISO when address is transmitted by
* controller on MOSI. Detecting the wait state in software is only
* possible for full duplex controllers. For controllers that support
* only half-duplex, the wait state detection needs to be implemented
* in hardware. TPM devices would set this flag when hardware flow
* control is expected from SPI controller.
*/
#define SPI_TPM_HW_FLOW BIT(29) /* TPM HW flow control */
/*
* All bits defined above should be covered by SPI_MODE_KERNEL_MASK.
* The SPI_MODE_KERNEL_MASK has the SPI_MODE_USER_MASK counterpart,
* which is defined in 'include/uapi/linux/spi/spi.h'.
* The bits defined here are from bit 31 downwards, while in
* SPI_MODE_USER_MASK are from 0 upwards.
* These bits must not overlap. A static assert check should make sure of that.
* If adding extra bits, make sure to decrease the bit index below as well.
*/
#define SPI_MODE_KERNEL_MASK (~(BIT(29) - 1))
u32 mode;
int irq;
void *controller_state;
void *controller_data;
char modalias[SPI_NAME_SIZE];
const char *driver_override;
struct gpio_desc *cs_gpiod[SPI_CS_CNT_MAX]; /* Chip select gpio desc */
struct spi_delay word_delay; /* Inter-word delay */
/* CS delays */
struct spi_delay cs_setup;
struct spi_delay cs_hold;
struct spi_delay cs_inactive;
/* The statistics */
struct spi_statistics __percpu *pcpu_statistics;
/* Bit mask of the chipselect(s) that the driver need to use from
* the chipselect array.When the controller is capable to handle
* multiple chip selects & memories are connected in parallel
* then more than one bit need to be set in cs_index_mask.
*/
u32 cs_index_mask : SPI_CS_CNT_MAX;
/*
* Likely need more hooks for more protocol options affecting how
* the controller talks to each chip, like:
* - memory packing (12 bit samples into low bits, others zeroed)
* - priority
* - chipselect delays
* - ...
*/
};
/* Make sure that SPI_MODE_KERNEL_MASK & SPI_MODE_USER_MASK don't overlap */
static_assert((SPI_MODE_KERNEL_MASK & SPI_MODE_USER_MASK) == 0,
"SPI_MODE_USER_MASK & SPI_MODE_KERNEL_MASK must not overlap");
static inline struct spi_device *to_spi_device(const struct device *dev)
{
return dev ? container_of(dev, struct spi_device, dev) : NULL;
}
/* Most drivers won't need to care about device refcounting */
static inline struct spi_device *spi_dev_get(struct spi_device *spi)
{
return (spi && get_device(&spi->dev)) ? spi : NULL;
}
static inline void spi_dev_put(struct spi_device *spi)
{
if (spi)
put_device(&spi->dev);
}
/* ctldata is for the bus_controller driver's runtime state */
static inline void *spi_get_ctldata(const struct spi_device *spi)
{
return spi->controller_state;
}
static inline void spi_set_ctldata(struct spi_device *spi, void *state)
{
spi->controller_state = state;
}
/* Device driver data */
static inline void spi_set_drvdata(struct spi_device *spi, void *data)
{
dev_set_drvdata(&spi->dev, data);
}
static inline void *spi_get_drvdata(const struct spi_device *spi)
{
return dev_get_drvdata(&spi->dev);
}
static inline u8 spi_get_chipselect(const struct spi_device *spi, u8 idx)
{
return spi->chip_select[idx];
}
static inline void spi_set_chipselect(struct spi_device *spi, u8 idx, u8 chipselect)
{
spi->chip_select[idx] = chipselect;
}
static inline struct gpio_desc *spi_get_csgpiod(const struct spi_device *spi, u8 idx)
{
return spi->cs_gpiod[idx];
}
static inline void spi_set_csgpiod(struct spi_device *spi, u8 idx, struct gpio_desc *csgpiod)
{
spi->cs_gpiod[idx] = csgpiod;
}
static inline bool spi_is_csgpiod(struct spi_device *spi)
{
u8 idx;
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
if (spi_get_csgpiod(spi, idx))
return true;
}
return false;
}
/**
* struct spi_driver - Host side "protocol" driver
* @id_table: List of SPI devices supported by this driver
* @probe: Binds this driver to the SPI device. Drivers can verify
* that the device is actually present, and may need to configure
* characteristics (such as bits_per_word) which weren't needed for
* the initial configuration done during system setup.
* @remove: Unbinds this driver from the SPI device
* @shutdown: Standard shutdown callback used during system state
* transitions such as powerdown/halt and kexec
* @driver: SPI device drivers should initialize the name and owner
* field of this structure.
*
* This represents the kind of device driver that uses SPI messages to
* interact with the hardware at the other end of a SPI link. It's called
* a "protocol" driver because it works through messages rather than talking
* directly to SPI hardware (which is what the underlying SPI controller
* driver does to pass those messages). These protocols are defined in the
* specification for the device(s) supported by the driver.
*
* As a rule, those device protocols represent the lowest level interface
* supported by a driver, and it will support upper level interfaces too.
* Examples of such upper levels include frameworks like MTD, networking,
* MMC, RTC, filesystem character device nodes, and hardware monitoring.
*/
struct spi_driver {
const struct spi_device_id *id_table;
int (*probe)(struct spi_device *spi);
void (*remove)(struct spi_device *spi);
void (*shutdown)(struct spi_device *spi);
struct device_driver driver;
};
static inline struct spi_driver *to_spi_driver(struct device_driver *drv)
{
return drv ? container_of(drv, struct spi_driver, driver) : NULL;
}
extern int __spi_register_driver(struct module *owner, struct spi_driver *sdrv);
/**
* spi_unregister_driver - reverse effect of spi_register_driver
* @sdrv: the driver to unregister
* Context: can sleep
*/
static inline void spi_unregister_driver(struct spi_driver *sdrv)
{
if (sdrv)
driver_unregister(&sdrv->driver);
}
extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 chip_select);
/* Use a define to avoid include chaining to get THIS_MODULE */
#define spi_register_driver(driver) \
__spi_register_driver(THIS_MODULE, driver)
/**
* module_spi_driver() - Helper macro for registering a SPI driver
* @__spi_driver: spi_driver struct
*
* Helper macro for SPI drivers which do not do anything special in module
* init/exit. This eliminates a lot of boilerplate. Each module may only
* use this macro once, and calling it replaces module_init() and module_exit()
*/
#define module_spi_driver(__spi_driver) \
module_driver(__spi_driver, spi_register_driver, \
spi_unregister_driver)
/**
* struct spi_controller - interface to SPI master or slave controller
* @dev: device interface to this driver
* @list: link with the global spi_controller list
* @bus_num: board-specific (and often SOC-specific) identifier for a
* given SPI controller.
* @num_chipselect: chipselects are used to distinguish individual
* SPI slaves, and are numbered from zero to num_chipselects.
* each slave has a chipselect signal, but it's common that not
* every chipselect is connected to a slave.
* @dma_alignment: SPI controller constraint on DMA buffers alignment.
* @mode_bits: flags understood by this controller driver
* @buswidth_override_bits: flags to override for this controller driver
* @bits_per_word_mask: A mask indicating which values of bits_per_word are
* supported by the driver. Bit n indicates that a bits_per_word n+1 is
* supported. If set, the SPI core will reject any transfer with an
* unsupported bits_per_word. If not set, this value is simply ignored,
* and it's up to the individual driver to perform any validation.
* @min_speed_hz: Lowest supported transfer speed
* @max_speed_hz: Highest supported transfer speed
* @flags: other constraints relevant to this driver
* @slave: indicates that this is an SPI slave controller
* @target: indicates that this is an SPI target controller
* @devm_allocated: whether the allocation of this struct is devres-managed
* @max_transfer_size: function that returns the max transfer size for
* a &spi_device; may be %NULL, so the default %SIZE_MAX will be used.
* @max_message_size: function that returns the max message size for
* a &spi_device; may be %NULL, so the default %SIZE_MAX will be used.
* @io_mutex: mutex for physical bus access
* @add_lock: mutex to avoid adding devices to the same chipselect
* @bus_lock_spinlock: spinlock for SPI bus locking
* @bus_lock_mutex: mutex for exclusion of multiple callers
* @bus_lock_flag: indicates that the SPI bus is locked for exclusive use
* @setup: updates the device mode and clocking records used by a
* device's SPI controller; protocol code may call this. This
* must fail if an unrecognized or unsupported mode is requested.
* It's always safe to call this unless transfers are pending on
* the device whose settings are being modified.
* @set_cs_timing: optional hook for SPI devices to request SPI master
* controller for configuring specific CS setup time, hold time and inactive
* delay interms of clock counts
* @transfer: adds a message to the controller's transfer queue.
* @cleanup: frees controller-specific state
* @can_dma: determine whether this controller supports DMA
* @dma_map_dev: device which can be used for DMA mapping
* @cur_rx_dma_dev: device which is currently used for RX DMA mapping
* @cur_tx_dma_dev: device which is currently used for TX DMA mapping
* @queued: whether this controller is providing an internal message queue
* @kworker: pointer to thread struct for message pump
* @pump_messages: work struct for scheduling work to the message pump
* @queue_lock: spinlock to synchronise access to message queue
* @queue: message queue
* @cur_msg: the currently in-flight message
* @cur_msg_completion: a completion for the current in-flight message
* @cur_msg_incomplete: Flag used internally to opportunistically skip
* the @cur_msg_completion. This flag is used to check if the driver has
* already called spi_finalize_current_message().
* @cur_msg_need_completion: Flag used internally to opportunistically skip
* the @cur_msg_completion. This flag is used to signal the context that
* is running spi_finalize_current_message() that it needs to complete()
* @cur_msg_mapped: message has been mapped for DMA
* @fallback: fallback to PIO if DMA transfer return failure with
* SPI_TRANS_FAIL_NO_START.
* @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs.
* @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip
* selected
* @xfer_completion: used by core transfer_one_message()
* @busy: message pump is busy
* @running: message pump is running
* @rt: whether this queue is set to run as a realtime task
* @auto_runtime_pm: the core should ensure a runtime PM reference is held
* while the hardware is prepared, using the parent
* device for the spidev
* @max_dma_len: Maximum length of a DMA transfer for the device.
* @prepare_transfer_hardware: a message will soon arrive from the queue
* so the subsystem requests the driver to prepare the transfer hardware
* by issuing this call
* @transfer_one_message: the subsystem calls the driver to transfer a single
* message while queuing transfers that arrive in the meantime. When the
* driver is finished with this message, it must call
* spi_finalize_current_message() so the subsystem can issue the next
* message
* @unprepare_transfer_hardware: there are currently no more messages on the
* queue so the subsystem notifies the driver that it may relax the
* hardware by issuing this call
*
* @set_cs: set the logic level of the chip select line. May be called
* from interrupt context.
* @optimize_message: optimize the message for reuse
* @unoptimize_message: release resources allocated by optimize_message
* @prepare_message: set up the controller to transfer a single message,
* for example doing DMA mapping. Called from threaded
* context.
* @transfer_one: transfer a single spi_transfer.
*
* - return 0 if the transfer is finished,
* - return 1 if the transfer is still in progress. When
* the driver is finished with this transfer it must
* call spi_finalize_current_transfer() so the subsystem
* can issue the next transfer. If the transfer fails, the
* driver must set the flag SPI_TRANS_FAIL_IO to
* spi_transfer->error first, before calling
* spi_finalize_current_transfer().
* Note: transfer_one and transfer_one_message are mutually
* exclusive; when both are set, the generic subsystem does
* not call your transfer_one callback.
* @handle_err: the subsystem calls the driver to handle an error that occurs
* in the generic implementation of transfer_one_message().
* @mem_ops: optimized/dedicated operations for interactions with SPI memory.
* This field is optional and should only be implemented if the
* controller has native support for memory like operations.
* @mem_caps: controller capabilities for the handling of memory operations.
* @unprepare_message: undo any work done by prepare_message().
* @slave_abort: abort the ongoing transfer request on an SPI slave controller
* @target_abort: abort the ongoing transfer request on an SPI target controller
* @cs_gpiods: Array of GPIO descriptors to use as chip select lines; one per CS
* number. Any individual value may be NULL for CS lines that
* are not GPIOs (driven by the SPI controller itself).
* @use_gpio_descriptors: Turns on the code in the SPI core to parse and grab
* GPIO descriptors. This will fill in @cs_gpiods and SPI devices will have
* the cs_gpiod assigned if a GPIO line is found for the chipselect.
* @unused_native_cs: When cs_gpiods is used, spi_register_controller() will
* fill in this field with the first unused native CS, to be used by SPI
* controller drivers that need to drive a native CS when using GPIO CS.
* @max_native_cs: When cs_gpiods is used, and this field is filled in,
* spi_register_controller() will validate all native CS (including the
* unused native CS) against this value.
* @pcpu_statistics: statistics for the spi_controller
* @dma_tx: DMA transmit channel
* @dma_rx: DMA receive channel
* @dummy_rx: dummy receive buffer for full-duplex devices
* @dummy_tx: dummy transmit buffer for full-duplex devices
* @fw_translate_cs: If the boot firmware uses different numbering scheme
* what Linux expects, this optional hook can be used to translate
* between the two.
* @ptp_sts_supported: If the driver sets this to true, it must provide a
* time snapshot in @spi_transfer->ptp_sts as close as possible to the
* moment in time when @spi_transfer->ptp_sts_word_pre and
* @spi_transfer->ptp_sts_word_post were transmitted.
* If the driver does not set this, the SPI core takes the snapshot as
* close to the driver hand-over as possible.
* @irq_flags: Interrupt enable state during PTP system timestamping
* @queue_empty: signal green light for opportunistically skipping the queue
* for spi_sync transfers.
* @must_async: disable all fast paths in the core
*
* Each SPI controller can communicate with one or more @spi_device
* children. These make a small bus, sharing MOSI, MISO and SCK signals
* but not chip select signals. Each device may be configured to use a
* different clock rate, since those shared signals are ignored unless
* the chip is selected.
*
* The driver for an SPI controller manages access to those devices through
* a queue of spi_message transactions, copying data between CPU memory and
* an SPI slave device. For each such message it queues, it calls the
* message's completion function when the transaction completes.
*/
struct spi_controller {
struct device dev;
struct list_head list;
/*
* Other than negative (== assign one dynamically), bus_num is fully
* board-specific. Usually that simplifies to being SoC-specific.
* example: one SoC has three SPI controllers, numbered 0..2,
* and one board's schematics might show it using SPI-2. Software
* would normally use bus_num=2 for that controller.
*/
s16 bus_num;
/*
* Chipselects will be integral to many controllers; some others
* might use board-specific GPIOs.
*/
u16 num_chipselect;
/* Some SPI controllers pose alignment requirements on DMAable
* buffers; let protocol drivers know about these requirements.
*/
u16 dma_alignment;
/* spi_device.mode flags understood by this controller driver */
u32 mode_bits;
/* spi_device.mode flags override flags for this controller */
u32 buswidth_override_bits;
/* Bitmask of supported bits_per_word for transfers */
u32 bits_per_word_mask;
#define SPI_BPW_MASK(bits) BIT((bits) - 1)
#define SPI_BPW_RANGE_MASK(min, max) GENMASK((max) - 1, (min) - 1)
/* Limits on transfer speed */
u32 min_speed_hz;
u32 max_speed_hz;
/* Other constraints relevant to this driver */
u16 flags;
#define SPI_CONTROLLER_HALF_DUPLEX BIT(0) /* Can't do full duplex */
#define SPI_CONTROLLER_NO_RX BIT(1) /* Can't do buffer read */
#define SPI_CONTROLLER_NO_TX BIT(2) /* Can't do buffer write */
#define SPI_CONTROLLER_MUST_RX BIT(3) /* Requires rx */
#define SPI_CONTROLLER_MUST_TX BIT(4) /* Requires tx */
#define SPI_CONTROLLER_GPIO_SS BIT(5) /* GPIO CS must select slave */
#define SPI_CONTROLLER_SUSPENDED BIT(6) /* Currently suspended */
/*
* The spi-controller has multi chip select capability and can
* assert/de-assert more than one chip select at once.
*/
#define SPI_CONTROLLER_MULTI_CS BIT(7)
/* Flag indicating if the allocation of this struct is devres-managed */
bool devm_allocated;
union {
/* Flag indicating this is an SPI slave controller */
bool slave;
/* Flag indicating this is an SPI target controller */
bool target;
};
/*
* On some hardware transfer / message size may be constrained
* the limit may depend on device transfer settings.
*/
size_t (*max_transfer_size)(struct spi_device *spi);
size_t (*max_message_size)(struct spi_device *spi);
/* I/O mutex */
struct mutex io_mutex;
/* Used to avoid adding the same CS twice */
struct mutex add_lock;
/* Lock and mutex for SPI bus locking */
spinlock_t bus_lock_spinlock;
struct mutex bus_lock_mutex;
/* Flag indicating that the SPI bus is locked for exclusive use */
bool bus_lock_flag;
/*
* Setup mode and clock, etc (SPI driver may call many times).
*
* IMPORTANT: this may be called when transfers to another
* device are active. DO NOT UPDATE SHARED REGISTERS in ways
* which could break those transfers.
*/
int (*setup)(struct spi_device *spi);
/*
* set_cs_timing() method is for SPI controllers that supports
* configuring CS timing.
*
* This hook allows SPI client drivers to request SPI controllers
* to configure specific CS timing through spi_set_cs_timing() after
* spi_setup().
*/
int (*set_cs_timing)(struct spi_device *spi);
/*
* Bidirectional bulk transfers
*
* + The transfer() method may not sleep; its main role is
* just to add the message to the queue.
* + For now there's no remove-from-queue operation, or
* any other request management
* + To a given spi_device, message queueing is pure FIFO
*
* + The controller's main job is to process its message queue,
* selecting a chip (for masters), then transferring data
* + If there are multiple spi_device children, the i/o queue
* arbitration algorithm is unspecified (round robin, FIFO,
* priority, reservations, preemption, etc)
*
* + Chipselect stays active during the entire message
* (unless modified by spi_transfer.cs_change != 0).
* + The message transfers use clock and SPI mode parameters
* previously established by setup() for this device
*/
int (*transfer)(struct spi_device *spi,
struct spi_message *mesg);
/* Called on release() to free memory provided by spi_controller */
void (*cleanup)(struct spi_device *spi);
/*
* Used to enable core support for DMA handling, if can_dma()
* exists and returns true then the transfer will be mapped
* prior to transfer_one() being called. The driver should
* not modify or store xfer and dma_tx and dma_rx must be set
* while the device is prepared.
*/
bool (*can_dma)(struct spi_controller *ctlr,
struct spi_device *spi,
struct spi_transfer *xfer);
struct device *dma_map_dev;
struct device *cur_rx_dma_dev;
struct device *cur_tx_dma_dev;
/*
* These hooks are for drivers that want to use the generic
* controller transfer queueing mechanism. If these are used, the
* transfer() function above must NOT be specified by the driver.
* Over time we expect SPI drivers to be phased over to this API.
*/
bool queued;
struct kthread_worker *kworker;
struct kthread_work pump_messages;
spinlock_t queue_lock;
struct list_head queue;
struct spi_message *cur_msg;
struct completion cur_msg_completion;
bool cur_msg_incomplete;
bool cur_msg_need_completion;
bool busy;
bool running;
bool rt;
bool auto_runtime_pm;
bool cur_msg_mapped;
bool fallback;
bool last_cs_mode_high;
s8 last_cs[SPI_CS_CNT_MAX];
u32 last_cs_index_mask : SPI_CS_CNT_MAX;
struct completion xfer_completion;
size_t max_dma_len;
int (*optimize_message)(struct spi_message *msg);
int (*unoptimize_message)(struct spi_message *msg);
int (*prepare_transfer_hardware)(struct spi_controller *ctlr);
int (*transfer_one_message)(struct spi_controller *ctlr,
struct spi_message *mesg);
int (*unprepare_transfer_hardware)(struct spi_controller *ctlr);
int (*prepare_message)(struct spi_controller *ctlr,
struct spi_message *message);
int (*unprepare_message)(struct spi_controller *ctlr,
struct spi_message *message);
union {
int (*slave_abort)(struct spi_controller *ctlr);
int (*target_abort)(struct spi_controller *ctlr);
};
/*
* These hooks are for drivers that use a generic implementation
* of transfer_one_message() provided by the core.
*/
void (*set_cs)(struct spi_device *spi, bool enable);
int (*transfer_one)(struct spi_controller *ctlr, struct spi_device *spi,
struct spi_transfer *transfer);
void (*handle_err)(struct spi_controller *ctlr,
struct spi_message *message);
/* Optimized handlers for SPI memory-like operations. */
const struct spi_controller_mem_ops *mem_ops;
const struct spi_controller_mem_caps *mem_caps;
/* GPIO chip select */
struct gpio_desc **cs_gpiods;
bool use_gpio_descriptors;
s8 unused_native_cs;
s8 max_native_cs;
/* Statistics */
struct spi_statistics __percpu *pcpu_statistics;
/* DMA channels for use with core dmaengine helpers */
struct dma_chan *dma_tx;
struct dma_chan *dma_rx;
/* Dummy data for full duplex devices */
void *dummy_rx;
void *dummy_tx;
int (*fw_translate_cs)(struct spi_controller *ctlr, unsigned cs);
/*
* Driver sets this field to indicate it is able to snapshot SPI
* transfers (needed e.g. for reading the time of POSIX clocks)
*/
bool ptp_sts_supported;
/* Interrupt enable state during PTP system timestamping */
unsigned long irq_flags;
/* Flag for enabling opportunistic skipping of the queue in spi_sync */
bool queue_empty;
bool must_async;
};
static inline void *spi_controller_get_devdata(struct spi_controller *ctlr)
{
return dev_get_drvdata(&ctlr->dev);
}
static inline void spi_controller_set_devdata(struct spi_controller *ctlr,
void *data)
{
dev_set_drvdata(&ctlr->dev, data);
}
static inline struct spi_controller *spi_controller_get(struct spi_controller *ctlr)
{
if (!ctlr || !get_device(&ctlr->dev))
return NULL;
return ctlr;
}
static inline void spi_controller_put(struct spi_controller *ctlr)
{
if (ctlr)
put_device(&ctlr->dev);
}
static inline bool spi_controller_is_slave(struct spi_controller *ctlr)
{
return IS_ENABLED(CONFIG_SPI_SLAVE) && ctlr->slave;
}
static inline bool spi_controller_is_target(struct spi_controller *ctlr)
{
return IS_ENABLED(CONFIG_SPI_SLAVE) && ctlr->target;
}
/* PM calls that need to be issued by the driver */
extern int spi_controller_suspend(struct spi_controller *ctlr);
extern int spi_controller_resume(struct spi_controller *ctlr);
/* Calls the driver make to interact with the message queue */
extern struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr);
extern void spi_finalize_current_message(struct spi_controller *ctlr);
extern void spi_finalize_current_transfer(struct spi_controller *ctlr);
/* Helper calls for driver to timestamp transfer */
void spi_take_timestamp_pre(struct spi_controller *ctlr,
struct spi_transfer *xfer,
size_t progress, bool irqs_off);
void spi_take_timestamp_post(struct spi_controller *ctlr,
struct spi_transfer *xfer,
size_t progress, bool irqs_off);
/* The SPI driver core manages memory for the spi_controller classdev */
extern struct spi_controller *__spi_alloc_controller(struct device *host,
unsigned int size, bool slave);
static inline struct spi_controller *spi_alloc_master(struct device *host,
unsigned int size)
{
return __spi_alloc_controller(host, size, false);
}
static inline struct spi_controller *spi_alloc_slave(struct device *host,
unsigned int size)
{
if (!IS_ENABLED(CONFIG_SPI_SLAVE))
return NULL;
return __spi_alloc_controller(host, size, true);
}
static inline struct spi_controller *spi_alloc_host(struct device *dev,
unsigned int size)
{
return __spi_alloc_controller(dev, size, false);
}
static inline struct spi_controller *spi_alloc_target(struct device *dev,
unsigned int size)
{
if (!IS_ENABLED(CONFIG_SPI_SLAVE))
return NULL;
return __spi_alloc_controller(dev, size, true);
}
struct spi_controller *__devm_spi_alloc_controller(struct device *dev,
unsigned int size,
bool slave);
static inline struct spi_controller *devm_spi_alloc_master(struct device *dev,
unsigned int size)
{
return __devm_spi_alloc_controller(dev, size, false);
}
static inline struct spi_controller *devm_spi_alloc_slave(struct device *dev,
unsigned int size)
{
if (!IS_ENABLED(CONFIG_SPI_SLAVE))
return NULL;
return __devm_spi_alloc_controller(dev, size, true);
}
static inline struct spi_controller *devm_spi_alloc_host(struct device *dev,
unsigned int size)
{
return __devm_spi_alloc_controller(dev, size, false);
}
static inline struct spi_controller *devm_spi_alloc_target(struct device *dev,
unsigned int size)
{
if (!IS_ENABLED(CONFIG_SPI_SLAVE))
return NULL;
return __devm_spi_alloc_controller(dev, size, true);
}
extern int spi_register_controller(struct spi_controller *ctlr);
extern int devm_spi_register_controller(struct device *dev,
struct spi_controller *ctlr);
extern void spi_unregister_controller(struct spi_controller *ctlr);
#if IS_ENABLED(CONFIG_ACPI)
extern struct spi_controller *acpi_spi_find_controller_by_adev(struct acpi_device *adev);
extern struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr,
struct acpi_device *adev,
int index);
int acpi_spi_count_resources(struct acpi_device *adev);
#endif
/*
* SPI resource management while processing a SPI message
*/
typedef void (*spi_res_release_t)(struct spi_controller *ctlr,
struct spi_message *msg,
void *res);
/**
* struct spi_res - SPI resource management structure
* @entry: list entry
* @release: release code called prior to freeing this resource
* @data: extra data allocated for the specific use-case
*
* This is based on ideas from devres, but focused on life-cycle
* management during spi_message processing.
*/
struct spi_res {
struct list_head entry;
spi_res_release_t release;
unsigned long long data[]; /* Guarantee ull alignment */
};
/*---------------------------------------------------------------------------*/
/*
* I/O INTERFACE between SPI controller and protocol drivers
*
* Protocol drivers use a queue of spi_messages, each transferring data
* between the controller and memory buffers.
*
* The spi_messages themselves consist of a series of read+write transfer
* segments. Those segments always read the same number of bits as they
* write; but one or the other is easily ignored by passing a NULL buffer
* pointer. (This is unlike most types of I/O API, because SPI hardware
* is full duplex.)
*
* NOTE: Allocation of spi_transfer and spi_message memory is entirely
* up to the protocol driver, which guarantees the integrity of both (as
* well as the data buffers) for as long as the message is queued.
*/
/**
* struct spi_transfer - a read/write buffer pair
* @tx_buf: data to be written (DMA-safe memory), or NULL
* @rx_buf: data to be read (DMA-safe memory), or NULL
* @tx_dma: DMA address of tx_buf, if @spi_message.is_dma_mapped
* @rx_dma: DMA address of rx_buf, if @spi_message.is_dma_mapped
* @tx_nbits: number of bits used for writing. If 0 the default
* (SPI_NBITS_SINGLE) is used.
* @rx_nbits: number of bits used for reading. If 0 the default
* (SPI_NBITS_SINGLE) is used.
* @len: size of rx and tx buffers (in bytes)
* @speed_hz: Select a speed other than the device default for this
* transfer. If 0 the default (from @spi_device) is used.
* @bits_per_word: select a bits_per_word other than the device default
* for this transfer. If 0 the default (from @spi_device) is used.
* @dummy_data: indicates transfer is dummy bytes transfer.
* @cs_off: performs the transfer with chipselect off.
* @cs_change: affects chipselect after this transfer completes
* @cs_change_delay: delay between cs deassert and assert when
* @cs_change is set and @spi_transfer is not the last in @spi_message
* @delay: delay to be introduced after this transfer before
* (optionally) changing the chipselect status, then starting
* the next transfer or completing this @spi_message.
* @word_delay: inter word delay to be introduced after each word size
* (set by bits_per_word) transmission.
* @effective_speed_hz: the effective SCK-speed that was used to
* transfer this transfer. Set to 0 if the SPI bus driver does
* not support it.
* @transfer_list: transfers are sequenced through @spi_message.transfers
* @tx_sg: Scatterlist for transmit, currently not for client use
* @rx_sg: Scatterlist for receive, currently not for client use
* @ptp_sts_word_pre: The word (subject to bits_per_word semantics) offset
* within @tx_buf for which the SPI device is requesting that the time
* snapshot for this transfer begins. Upon completing the SPI transfer,
* this value may have changed compared to what was requested, depending
* on the available snapshotting resolution (DMA transfer,
* @ptp_sts_supported is false, etc).
* @ptp_sts_word_post: See @ptp_sts_word_post. The two can be equal (meaning
* that a single byte should be snapshotted).
* If the core takes care of the timestamp (if @ptp_sts_supported is false
* for this controller), it will set @ptp_sts_word_pre to 0, and
* @ptp_sts_word_post to the length of the transfer. This is done
* purposefully (instead of setting to spi_transfer->len - 1) to denote
* that a transfer-level snapshot taken from within the driver may still
* be of higher quality.
* @ptp_sts: Pointer to a memory location held by the SPI slave device where a
* PTP system timestamp structure may lie. If drivers use PIO or their
* hardware has some sort of assist for retrieving exact transfer timing,
* they can (and should) assert @ptp_sts_supported and populate this
* structure using the ptp_read_system_*ts helper functions.
* The timestamp must represent the time at which the SPI slave device has
* processed the word, i.e. the "pre" timestamp should be taken before
* transmitting the "pre" word, and the "post" timestamp after receiving
* transmit confirmation from the controller for the "post" word.
* @timestamped: true if the transfer has been timestamped
* @error: Error status logged by SPI controller driver.
*
* SPI transfers always write the same number of bytes as they read.
* Protocol drivers should always provide @rx_buf and/or @tx_buf.
* In some cases, they may also want to provide DMA addresses for
* the data being transferred; that may reduce overhead, when the
* underlying driver uses DMA.
*
* If the transmit buffer is NULL, zeroes will be shifted out
* while filling @rx_buf. If the receive buffer is NULL, the data
* shifted in will be discarded. Only "len" bytes shift out (or in).
* It's an error to try to shift out a partial word. (For example, by
* shifting out three bytes with word size of sixteen or twenty bits;
* the former uses two bytes per word, the latter uses four bytes.)
*
* In-memory data values are always in native CPU byte order, translated
* from the wire byte order (big-endian except with SPI_LSB_FIRST). So
* for example when bits_per_word is sixteen, buffers are 2N bytes long
* (@len = 2N) and hold N sixteen bit words in CPU byte order.
*
* When the word size of the SPI transfer is not a power-of-two multiple
* of eight bits, those in-memory words include extra bits. In-memory
* words are always seen by protocol drivers as right-justified, so the
* undefined (rx) or unused (tx) bits are always the most significant bits.
*
* All SPI transfers start with the relevant chipselect active. Normally
* it stays selected until after the last transfer in a message. Drivers
* can affect the chipselect signal using cs_change.
*
* (i) If the transfer isn't the last one in the message, this flag is
* used to make the chipselect briefly go inactive in the middle of the
* message. Toggling chipselect in this way may be needed to terminate
* a chip command, letting a single spi_message perform all of group of
* chip transactions together.
*
* (ii) When the transfer is the last one in the message, the chip may
* stay selected until the next transfer. On multi-device SPI busses
* with nothing blocking messages going to other devices, this is just
* a performance hint; starting a message to another device deselects
* this one. But in other cases, this can be used to ensure correctness.
* Some devices need protocol transactions to be built from a series of
* spi_message submissions, where the content of one message is determined
* by the results of previous messages and where the whole transaction
* ends when the chipselect goes inactive.
*
* When SPI can transfer in 1x,2x or 4x. It can get this transfer information
* from device through @tx_nbits and @rx_nbits. In Bi-direction, these
* two should both be set. User can set transfer mode with SPI_NBITS_SINGLE(1x)
* SPI_NBITS_DUAL(2x) and SPI_NBITS_QUAD(4x) to support these three transfer.
*
* The code that submits an spi_message (and its spi_transfers)
* to the lower layers is responsible for managing its memory.
* Zero-initialize every field you don't set up explicitly, to
* insulate against future API updates. After you submit a message
* and its transfers, ignore them until its completion callback.
*/
struct spi_transfer {
/*
* It's okay if tx_buf == rx_buf (right?).
* For MicroWire, one buffer must be NULL.
* Buffers must work with dma_*map_single() calls, unless
* spi_message.is_dma_mapped reports a pre-existing mapping.
*/
const void *tx_buf;
void *rx_buf;
unsigned len;
#define SPI_TRANS_FAIL_NO_START BIT(0)
#define SPI_TRANS_FAIL_IO BIT(1)
u16 error;
dma_addr_t tx_dma;
dma_addr_t rx_dma;
struct sg_table tx_sg;
struct sg_table rx_sg;
unsigned dummy_data:1;
unsigned cs_off:1;
unsigned cs_change:1;
unsigned tx_nbits:3;
unsigned rx_nbits:3;
unsigned timestamped:1;
#define SPI_NBITS_SINGLE 0x01 /* 1-bit transfer */
#define SPI_NBITS_DUAL 0x02 /* 2-bit transfer */
#define SPI_NBITS_QUAD 0x04 /* 4-bit transfer */
u8 bits_per_word;
struct spi_delay delay;
struct spi_delay cs_change_delay;
struct spi_delay word_delay;
u32 speed_hz;
u32 effective_speed_hz;
unsigned int ptp_sts_word_pre;
unsigned int ptp_sts_word_post;
struct ptp_system_timestamp *ptp_sts;
struct list_head transfer_list;
};
/**
* struct spi_message - one multi-segment SPI transaction
* @transfers: list of transfer segments in this transaction
* @spi: SPI device to which the transaction is queued
* @is_dma_mapped: if true, the caller provided both DMA and CPU virtual
* addresses for each transfer buffer
* @pre_optimized: peripheral driver pre-optimized the message
* @optimized: the message is in the optimized state
* @prepared: spi_prepare_message was called for the this message
* @status: zero for success, else negative errno
* @complete: called to report transaction completions
* @context: the argument to complete() when it's called
* @frame_length: the total number of bytes in the message
* @actual_length: the total number of bytes that were transferred in all
* successful segments
* @queue: for use by whichever driver currently owns the message
* @state: for use by whichever driver currently owns the message
* @opt_state: for use by whichever driver currently owns the message
* @resources: for resource management when the SPI message is processed
*
* A @spi_message is used to execute an atomic sequence of data transfers,
* each represented by a struct spi_transfer. The sequence is "atomic"
* in the sense that no other spi_message may use that SPI bus until that
* sequence completes. On some systems, many such sequences can execute as
* a single programmed DMA transfer. On all systems, these messages are
* queued, and might complete after transactions to other devices. Messages
* sent to a given spi_device are always executed in FIFO order.
*
* The code that submits an spi_message (and its spi_transfers)
* to the lower layers is responsible for managing its memory.
* Zero-initialize every field you don't set up explicitly, to
* insulate against future API updates. After you submit a message
* and its transfers, ignore them until its completion callback.
*/
struct spi_message {
struct list_head transfers;
struct spi_device *spi;
unsigned is_dma_mapped:1;
/* spi_optimize_message() was called for this message */
bool pre_optimized;
/* __spi_optimize_message() was called for this message */
bool optimized;
/* spi_prepare_message() was called for this message */
bool prepared;
/*
* REVISIT: we might want a flag affecting the behavior of the
* last transfer ... allowing things like "read 16 bit length L"
* immediately followed by "read L bytes". Basically imposing
* a specific message scheduling algorithm.
*
* Some controller drivers (message-at-a-time queue processing)
* could provide that as their default scheduling algorithm. But
* others (with multi-message pipelines) could need a flag to
* tell them about such special cases.
*/
/* Completion is reported through a callback */
int status;
void (*complete)(void *context);
void *context;
unsigned frame_length;
unsigned actual_length;
/*
* For optional use by whatever driver currently owns the
* spi_message ... between calls to spi_async and then later
* complete(), that's the spi_controller controller driver.
*/
struct list_head queue;
void *state;
/*
* Optional state for use by controller driver between calls to
* __spi_optimize_message() and __spi_unoptimize_message().
*/
void *opt_state;
/* List of spi_res resources when the SPI message is processed */
struct list_head resources;
};
static inline void spi_message_init_no_memset(struct spi_message *m)
{
INIT_LIST_HEAD(&m->transfers);
INIT_LIST_HEAD(&m->resources);
}
static inline void spi_message_init(struct spi_message *m)
{
memset(m, 0, sizeof *m);
spi_message_init_no_memset(m);
}
static inline void
spi_message_add_tail(struct spi_transfer *t, struct spi_message *m)
{
list_add_tail(&t->transfer_list, &m->transfers);
}
static inline void
spi_transfer_del(struct spi_transfer *t)
{
list_del(&t->transfer_list);
}
static inline int
spi_transfer_delay_exec(struct spi_transfer *t)
{
return spi_delay_exec(&t->delay, t);
}
/**
* spi_message_init_with_transfers - Initialize spi_message and append transfers
* @m: spi_message to be initialized
* @xfers: An array of SPI transfers
* @num_xfers: Number of items in the xfer array
*
* This function initializes the given spi_message and adds each spi_transfer in
* the given array to the message.
*/
static inline void
spi_message_init_with_transfers(struct spi_message *m,
struct spi_transfer *xfers, unsigned int num_xfers)
{
unsigned int i;
spi_message_init(m);
for (i = 0; i < num_xfers; ++i)
spi_message_add_tail(&xfers[i], m);
}
/*
* It's fine to embed message and transaction structures in other data
* structures so long as you don't free them while they're in use.
*/
static inline struct spi_message *spi_message_alloc(unsigned ntrans, gfp_t flags)
{
struct spi_message_with_transfers {
struct spi_message m;
struct spi_transfer t[];
} *mwt;
unsigned i;
mwt = kzalloc(struct_size(mwt, t, ntrans), flags);
if (!mwt)
return NULL;
spi_message_init_no_memset(&mwt->m);
for (i = 0; i < ntrans; i++)
spi_message_add_tail(&mwt->t[i], &mwt->m);
return &mwt->m;
}
static inline void spi_message_free(struct spi_message *m)
{
kfree(m);
}
extern int spi_optimize_message(struct spi_device *spi, struct spi_message *msg);
extern void spi_unoptimize_message(struct spi_message *msg);
extern int spi_setup(struct spi_device *spi);
extern int spi_async(struct spi_device *spi, struct spi_message *message);
extern int spi_slave_abort(struct spi_device *spi);
extern int spi_target_abort(struct spi_device *spi);
static inline size_t
spi_max_message_size(struct spi_device *spi)
{
struct spi_controller *ctlr = spi->controller;
if (!ctlr->max_message_size)
return SIZE_MAX;
return ctlr->max_message_size(spi);
}
static inline size_t
spi_max_transfer_size(struct spi_device *spi)
{
struct spi_controller *ctlr = spi->controller;
size_t tr_max = SIZE_MAX;
size_t msg_max = spi_max_message_size(spi);
if (ctlr->max_transfer_size)
tr_max = ctlr->max_transfer_size(spi);
/* Transfer size limit must not be greater than message size limit */
return min(tr_max, msg_max);
}
/**
* spi_is_bpw_supported - Check if bits per word is supported
* @spi: SPI device
* @bpw: Bits per word
*
* This function checks to see if the SPI controller supports @bpw.
*
* Returns:
* True if @bpw is supported, false otherwise.
*/
static inline bool spi_is_bpw_supported(struct spi_device *spi, u32 bpw)
{
u32 bpw_mask = spi->controller->bits_per_word_mask;
if (bpw == 8 || (bpw <= 32 && bpw_mask & SPI_BPW_MASK(bpw)))
return true;
return false;
}
/**
* spi_controller_xfer_timeout - Compute a suitable timeout value
* @ctlr: SPI device
* @xfer: Transfer descriptor
*
* Compute a relevant timeout value for the given transfer. We derive the time
* that it would take on a single data line and take twice this amount of time
* with a minimum of 500ms to avoid false positives on loaded systems.
*
* Returns: Transfer timeout value in milliseconds.
*/
static inline unsigned int spi_controller_xfer_timeout(struct spi_controller *ctlr,
struct spi_transfer *xfer)
{
return max(xfer->len * 8 * 2 / (xfer->speed_hz / 1000), 500U);
}
/*---------------------------------------------------------------------------*/
/* SPI transfer replacement methods which make use of spi_res */
struct spi_replaced_transfers;
typedef void (*spi_replaced_release_t)(struct spi_controller *ctlr,
struct spi_message *msg,
struct spi_replaced_transfers *res);
/**
* struct spi_replaced_transfers - structure describing the spi_transfer
* replacements that have occurred
* so that they can get reverted
* @release: some extra release code to get executed prior to
* releasing this structure
* @extradata: pointer to some extra data if requested or NULL
* @replaced_transfers: transfers that have been replaced and which need
* to get restored
* @replaced_after: the transfer after which the @replaced_transfers
* are to get re-inserted
* @inserted: number of transfers inserted
* @inserted_transfers: array of spi_transfers of array-size @inserted,
* that have been replacing replaced_transfers
*
* Note: that @extradata will point to @inserted_transfers[@inserted]
* if some extra allocation is requested, so alignment will be the same
* as for spi_transfers.
*/
struct spi_replaced_transfers {
spi_replaced_release_t release;
void *extradata;
struct list_head replaced_transfers;
struct list_head *replaced_after;
size_t inserted;
struct spi_transfer inserted_transfers[];
};
/*---------------------------------------------------------------------------*/
/* SPI transfer transformation methods */
extern int spi_split_transfers_maxsize(struct spi_controller *ctlr,
struct spi_message *msg,
size_t maxsize);
extern int spi_split_transfers_maxwords(struct spi_controller *ctlr,
struct spi_message *msg,
size_t maxwords);
/*---------------------------------------------------------------------------*/
/*
* All these synchronous SPI transfer routines are utilities layered
* over the core async transfer primitive. Here, "synchronous" means
* they will sleep uninterruptibly until the async transfer completes.
*/
extern int spi_sync(struct spi_device *spi, struct spi_message *message);
extern int spi_sync_locked(struct spi_device *spi, struct spi_message *message);
extern int spi_bus_lock(struct spi_controller *ctlr);
extern int spi_bus_unlock(struct spi_controller *ctlr);
/**
* spi_sync_transfer - synchronous SPI data transfer
* @spi: device with which data will be exchanged
* @xfers: An array of spi_transfers
* @num_xfers: Number of items in the xfer array
* Context: can sleep
*
* Does a synchronous SPI data transfer of the given spi_transfer array.
*
* For more specific semantics see spi_sync().
*
* Return: zero on success, else a negative error code.
*/
static inline int
spi_sync_transfer(struct spi_device *spi, struct spi_transfer *xfers,
unsigned int num_xfers)
{
struct spi_message msg;
spi_message_init_with_transfers(&msg, xfers, num_xfers);
return spi_sync(spi, &msg);
}
/**
* spi_write - SPI synchronous write
* @spi: device to which data will be written
* @buf: data buffer
* @len: data buffer size
* Context: can sleep
*
* This function writes the buffer @buf.
* Callable only from contexts that can sleep.
*
* Return: zero on success, else a negative error code.
*/
static inline int
spi_write(struct spi_device *spi, const void *buf, size_t len)
{
struct spi_transfer t = {
.tx_buf = buf,
.len = len,
};
return spi_sync_transfer(spi, &t, 1);
}
/**
* spi_read - SPI synchronous read
* @spi: device from which data will be read
* @buf: data buffer
* @len: data buffer size
* Context: can sleep
*
* This function reads the buffer @buf.
* Callable only from contexts that can sleep.
*
* Return: zero on success, else a negative error code.
*/
static inline int
spi_read(struct spi_device *spi, void *buf, size_t len)
{
struct spi_transfer t = {
.rx_buf = buf,
.len = len,
};
return spi_sync_transfer(spi, &t, 1);
}
/* This copies txbuf and rxbuf data; for small transfers only! */
extern int spi_write_then_read(struct spi_device *spi,
const void *txbuf, unsigned n_tx,
void *rxbuf, unsigned n_rx);
/**
* spi_w8r8 - SPI synchronous 8 bit write followed by 8 bit read
* @spi: device with which data will be exchanged
* @cmd: command to be written before data is read back
* Context: can sleep
*
* Callable only from contexts that can sleep.
*
* Return: the (unsigned) eight bit number returned by the
* device, or else a negative error code.
*/
static inline ssize_t spi_w8r8(struct spi_device *spi, u8 cmd)
{
ssize_t status;
u8 result;
status = spi_write_then_read(spi, &cmd, 1, &result, 1);
/* Return negative errno or unsigned value */
return (status < 0) ? status : result;
}
/**
* spi_w8r16 - SPI synchronous 8 bit write followed by 16 bit read
* @spi: device with which data will be exchanged
* @cmd: command to be written before data is read back
* Context: can sleep
*
* The number is returned in wire-order, which is at least sometimes
* big-endian.
*
* Callable only from contexts that can sleep.
*
* Return: the (unsigned) sixteen bit number returned by the
* device, or else a negative error code.
*/
static inline ssize_t spi_w8r16(struct spi_device *spi, u8 cmd)
{
ssize_t status;
u16 result;
status = spi_write_then_read(spi, &cmd, 1, &result, 2);
/* Return negative errno or unsigned value */
return (status < 0) ? status : result;
}
/**
* spi_w8r16be - SPI synchronous 8 bit write followed by 16 bit big-endian read
* @spi: device with which data will be exchanged
* @cmd: command to be written before data is read back
* Context: can sleep
*
* This function is similar to spi_w8r16, with the exception that it will
* convert the read 16 bit data word from big-endian to native endianness.
*
* Callable only from contexts that can sleep.
*
* Return: the (unsigned) sixteen bit number returned by the device in CPU
* endianness, or else a negative error code.
*/
static inline ssize_t spi_w8r16be(struct spi_device *spi, u8 cmd)
{
ssize_t status;
__be16 result;
status = spi_write_then_read(spi, &cmd, 1, &result, 2);
if (status < 0)
return status;
return be16_to_cpu(result);
}
/*---------------------------------------------------------------------------*/
/*
* INTERFACE between board init code and SPI infrastructure.
*
* No SPI driver ever sees these SPI device table segments, but
* it's how the SPI core (or adapters that get hotplugged) grows
* the driver model tree.
*
* As a rule, SPI devices can't be probed. Instead, board init code
* provides a table listing the devices which are present, with enough
* information to bind and set up the device's driver. There's basic
* support for non-static configurations too; enough to handle adding
* parport adapters, or microcontrollers acting as USB-to-SPI bridges.
*/
/**
* struct spi_board_info - board-specific template for a SPI device
* @modalias: Initializes spi_device.modalias; identifies the driver.
* @platform_data: Initializes spi_device.platform_data; the particular
* data stored there is driver-specific.
* @swnode: Software node for the device.
* @controller_data: Initializes spi_device.controller_data; some
* controllers need hints about hardware setup, e.g. for DMA.
* @irq: Initializes spi_device.irq; depends on how the board is wired.
* @max_speed_hz: Initializes spi_device.max_speed_hz; based on limits
* from the chip datasheet and board-specific signal quality issues.
* @bus_num: Identifies which spi_controller parents the spi_device; unused
* by spi_new_device(), and otherwise depends on board wiring.
* @chip_select: Initializes spi_device.chip_select; depends on how
* the board is wired.
* @mode: Initializes spi_device.mode; based on the chip datasheet, board
* wiring (some devices support both 3WIRE and standard modes), and
* possibly presence of an inverter in the chipselect path.
*
* When adding new SPI devices to the device tree, these structures serve
* as a partial device template. They hold information which can't always
* be determined by drivers. Information that probe() can establish (such
* as the default transfer wordsize) is not included here.
*
* These structures are used in two places. Their primary role is to
* be stored in tables of board-specific device descriptors, which are
* declared early in board initialization and then used (much later) to
* populate a controller's device tree after the that controller's driver
* initializes. A secondary (and atypical) role is as a parameter to
* spi_new_device() call, which happens after those controller drivers
* are active in some dynamic board configuration models.
*/
struct spi_board_info {
/*
* The device name and module name are coupled, like platform_bus;
* "modalias" is normally the driver name.
*
* platform_data goes to spi_device.dev.platform_data,
* controller_data goes to spi_device.controller_data,
* IRQ is copied too.
*/
char modalias[SPI_NAME_SIZE];
const void *platform_data;
const struct software_node *swnode;
void *controller_data;
int irq;
/* Slower signaling on noisy or low voltage boards */
u32 max_speed_hz;
/*
* bus_num is board specific and matches the bus_num of some
* spi_controller that will probably be registered later.
*
* chip_select reflects how this chip is wired to that master;
* it's less than num_chipselect.
*/
u16 bus_num;
u16 chip_select;
/*
* mode becomes spi_device.mode, and is essential for chips
* where the default of SPI_CS_HIGH = 0 is wrong.
*/
u32 mode;
/*
* ... may need additional spi_device chip config data here.
* avoid stuff protocol drivers can set; but include stuff
* needed to behave without being bound to a driver:
* - quirks like clock rate mattering when not selected
*/
};
#ifdef CONFIG_SPI
extern int
spi_register_board_info(struct spi_board_info const *info, unsigned n);
#else
/* Board init code may ignore whether SPI is configured or not */
static inline int
spi_register_board_info(struct spi_board_info const *info, unsigned n)
{ return 0; }
#endif
/*
* If you're hotplugging an adapter with devices (parport, USB, etc)
* use spi_new_device() to describe each device. You can also call
* spi_unregister_device() to start making that device vanish, but
* normally that would be handled by spi_unregister_controller().
*
* You can also use spi_alloc_device() and spi_add_device() to use a two
* stage registration sequence for each spi_device. This gives the caller
* some more control over the spi_device structure before it is registered,
* but requires that caller to initialize fields that would otherwise
* be defined using the board info.
*/
extern struct spi_device *
spi_alloc_device(struct spi_controller *ctlr);
extern int
spi_add_device(struct spi_device *spi);
extern struct spi_device *
spi_new_device(struct spi_controller *, struct spi_board_info *);
extern void spi_unregister_device(struct spi_device *spi);
extern const struct spi_device_id *
spi_get_device_id(const struct spi_device *sdev);
extern const void *
spi_get_device_match_data(const struct spi_device *sdev);
static inline bool
spi_transfer_is_last(struct spi_controller *ctlr, struct spi_transfer *xfer)
{
return list_is_last(&xfer->transfer_list, &ctlr->cur_msg->transfers);
}
#endif /* __LINUX_SPI_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* The input core
*
* Copyright (c) 1999-2002 Vojtech Pavlik
*/
#define pr_fmt(fmt) KBUILD_BASENAME ": " fmt
#include <linux/init.h>
#include <linux/types.h>
#include <linux/idr.h>
#include <linux/input/mt.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/major.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/pm.h>
#include <linux/poll.h>
#include <linux/device.h>
#include <linux/kstrtox.h>
#include <linux/mutex.h>
#include <linux/rcupdate.h>
#include "input-compat.h"
#include "input-core-private.h"
#include "input-poller.h"
MODULE_AUTHOR("Vojtech Pavlik <vojtech@suse.cz>");
MODULE_DESCRIPTION("Input core");
MODULE_LICENSE("GPL");
#define INPUT_MAX_CHAR_DEVICES 1024
#define INPUT_FIRST_DYNAMIC_DEV 256
static DEFINE_IDA(input_ida);
static LIST_HEAD(input_dev_list);
static LIST_HEAD(input_handler_list);
/*
* input_mutex protects access to both input_dev_list and input_handler_list.
* This also causes input_[un]register_device and input_[un]register_handler
* be mutually exclusive which simplifies locking in drivers implementing
* input handlers.
*/
static DEFINE_MUTEX(input_mutex);
static const struct input_value input_value_sync = { EV_SYN, SYN_REPORT, 1 };
static const unsigned int input_max_code[EV_CNT] = {
[EV_KEY] = KEY_MAX,
[EV_REL] = REL_MAX,
[EV_ABS] = ABS_MAX,
[EV_MSC] = MSC_MAX,
[EV_SW] = SW_MAX,
[EV_LED] = LED_MAX,
[EV_SND] = SND_MAX,
[EV_FF] = FF_MAX,
};
static inline int is_event_supported(unsigned int code,
unsigned long *bm, unsigned int max)
{
return code <= max && test_bit(code, bm);
}
static int input_defuzz_abs_event(int value, int old_val, int fuzz)
{
if (fuzz) {
if (value > old_val - fuzz / 2 && value < old_val + fuzz / 2)
return old_val;
if (value > old_val - fuzz && value < old_val + fuzz) return (old_val * 3 + value) / 4; if (value > old_val - fuzz * 2 && value < old_val + fuzz * 2) return (old_val + value) / 2;
}
return value;
}
static void input_start_autorepeat(struct input_dev *dev, int code)
{
if (test_bit(EV_REP, dev->evbit) &&
dev->rep[REP_PERIOD] && dev->rep[REP_DELAY] && dev->timer.function) { dev->repeat_key = code;
mod_timer(&dev->timer,
jiffies + msecs_to_jiffies(dev->rep[REP_DELAY]));
}
}
static void input_stop_autorepeat(struct input_dev *dev)
{
del_timer(&dev->timer);
}
/*
* Pass event first through all filters and then, if event has not been
* filtered out, through all open handles. This function is called with
* dev->event_lock held and interrupts disabled.
*/
static unsigned int input_to_handler(struct input_handle *handle,
struct input_value *vals, unsigned int count)
{
struct input_handler *handler = handle->handler;
struct input_value *end = vals;
struct input_value *v;
if (handler->filter) {
for (v = vals; v != vals + count; v++) { if (handler->filter(handle, v->type, v->code, v->value))
continue;
if (end != v) *end = *v; end++;
}
count = end - vals;
}
if (!count)
return 0;
if (handler->events) handler->events(handle, vals, count); else if (handler->event) for (v = vals; v != vals + count; v++) handler->event(handle, v->type, v->code, v->value);
return count;
}
/*
* Pass values first through all filters and then, if event has not been
* filtered out, through all open handles. This function is called with
* dev->event_lock held and interrupts disabled.
*/
static void input_pass_values(struct input_dev *dev,
struct input_value *vals, unsigned int count)
{
struct input_handle *handle;
struct input_value *v;
lockdep_assert_held(&dev->event_lock); if (!count)
return;
rcu_read_lock(); handle = rcu_dereference(dev->grab); if (handle) { count = input_to_handler(handle, vals, count);
} else {
list_for_each_entry_rcu(handle, &dev->h_list, d_node) if (handle->open) { count = input_to_handler(handle, vals, count);
if (!count)
break;
}
}
rcu_read_unlock();
/* trigger auto repeat for key events */
if (test_bit(EV_REP, dev->evbit) && test_bit(EV_KEY, dev->evbit)) { for (v = vals; v != vals + count; v++) { if (v->type == EV_KEY && v->value != 2) { if (v->value) input_start_autorepeat(dev, v->code);
else
input_stop_autorepeat(dev);
}
}
}
}
#define INPUT_IGNORE_EVENT 0
#define INPUT_PASS_TO_HANDLERS 1
#define INPUT_PASS_TO_DEVICE 2
#define INPUT_SLOT 4
#define INPUT_FLUSH 8
#define INPUT_PASS_TO_ALL (INPUT_PASS_TO_HANDLERS | INPUT_PASS_TO_DEVICE)
static int input_handle_abs_event(struct input_dev *dev,
unsigned int code, int *pval)
{
struct input_mt *mt = dev->mt;
bool is_new_slot = false;
bool is_mt_event;
int *pold;
if (code == ABS_MT_SLOT) {
/*
* "Stage" the event; we'll flush it later, when we
* get actual touch data.
*/
if (mt && *pval >= 0 && *pval < mt->num_slots) mt->slot = *pval;
return INPUT_IGNORE_EVENT;
}
is_mt_event = input_is_mt_value(code);
if (!is_mt_event) {
pold = &dev->absinfo[code].value; } else if (mt) { pold = &mt->slots[mt->slot].abs[code - ABS_MT_FIRST];
is_new_slot = mt->slot != dev->absinfo[ABS_MT_SLOT].value;
} else {
/*
* Bypass filtering for multi-touch events when
* not employing slots.
*/
pold = NULL;
}
if (pold) {
*pval = input_defuzz_abs_event(*pval, *pold,
dev->absinfo[code].fuzz);
if (*pold == *pval)
return INPUT_IGNORE_EVENT;
*pold = *pval;
}
/* Flush pending "slot" event */
if (is_new_slot) {
dev->absinfo[ABS_MT_SLOT].value = mt->slot;
return INPUT_PASS_TO_HANDLERS | INPUT_SLOT;
}
return INPUT_PASS_TO_HANDLERS;
}
static int input_get_disposition(struct input_dev *dev,
unsigned int type, unsigned int code, int *pval)
{
int disposition = INPUT_IGNORE_EVENT;
int value = *pval;
/* filter-out events from inhibited devices */
if (dev->inhibited)
return INPUT_IGNORE_EVENT;
switch (type) {
case EV_SYN:
switch (code) {
case SYN_CONFIG:
disposition = INPUT_PASS_TO_ALL;
break;
case SYN_REPORT:
disposition = INPUT_PASS_TO_HANDLERS | INPUT_FLUSH;
break;
case SYN_MT_REPORT:
disposition = INPUT_PASS_TO_HANDLERS;
break;
}
break;
case EV_KEY:
if (is_event_supported(code, dev->keybit, KEY_MAX)) {
/* auto-repeat bypasses state updates */
if (value == 2) {
disposition = INPUT_PASS_TO_HANDLERS;
break;
}
if (!!test_bit(code, dev->key) != !!value) { __change_bit(code, dev->key);
disposition = INPUT_PASS_TO_HANDLERS;
}
}
break;
case EV_SW:
if (is_event_supported(code, dev->swbit, SW_MAX) && !!test_bit(code, dev->sw) != !!value) { __change_bit(code, dev->sw);
disposition = INPUT_PASS_TO_HANDLERS;
}
break;
case EV_ABS:
if (is_event_supported(code, dev->absbit, ABS_MAX)) disposition = input_handle_abs_event(dev, code, &value);
break;
case EV_REL:
if (is_event_supported(code, dev->relbit, REL_MAX) && value)
disposition = INPUT_PASS_TO_HANDLERS;
break;
case EV_MSC:
if (is_event_supported(code, dev->mscbit, MSC_MAX))
disposition = INPUT_PASS_TO_ALL;
break;
case EV_LED:
if (is_event_supported(code, dev->ledbit, LED_MAX) && !!test_bit(code, dev->led) != !!value) { __change_bit(code, dev->led);
disposition = INPUT_PASS_TO_ALL;
}
break;
case EV_SND:
if (is_event_supported(code, dev->sndbit, SND_MAX)) { if (!!test_bit(code, dev->snd) != !!value) __change_bit(code, dev->snd);
disposition = INPUT_PASS_TO_ALL;
}
break;
case EV_REP:
if (code <= REP_MAX && value >= 0 && dev->rep[code] != value) { dev->rep[code] = value;
disposition = INPUT_PASS_TO_ALL;
}
break;
case EV_FF:
if (value >= 0)
disposition = INPUT_PASS_TO_ALL;
break;
case EV_PWR:
disposition = INPUT_PASS_TO_ALL;
break;
}
*pval = value;
return disposition;
}
static void input_event_dispose(struct input_dev *dev, int disposition,
unsigned int type, unsigned int code, int value)
{
if ((disposition & INPUT_PASS_TO_DEVICE) && dev->event) dev->event(dev, type, code, value); if (!dev->vals)
return;
if (disposition & INPUT_PASS_TO_HANDLERS) {
struct input_value *v;
if (disposition & INPUT_SLOT) { v = &dev->vals[dev->num_vals++];
v->type = EV_ABS;
v->code = ABS_MT_SLOT;
v->value = dev->mt->slot;
}
v = &dev->vals[dev->num_vals++];
v->type = type;
v->code = code;
v->value = value;
}
if (disposition & INPUT_FLUSH) { if (dev->num_vals >= 2) input_pass_values(dev, dev->vals, dev->num_vals); dev->num_vals = 0;
/*
* Reset the timestamp on flush so we won't end up
* with a stale one. Note we only need to reset the
* monolithic one as we use its presence when deciding
* whether to generate a synthetic timestamp.
*/
dev->timestamp[INPUT_CLK_MONO] = ktime_set(0, 0);
} else if (dev->num_vals >= dev->max_vals - 2) { dev->vals[dev->num_vals++] = input_value_sync;
input_pass_values(dev, dev->vals, dev->num_vals);
dev->num_vals = 0;
}
}
void input_handle_event(struct input_dev *dev,
unsigned int type, unsigned int code, int value)
{
int disposition;
lockdep_assert_held(&dev->event_lock); disposition = input_get_disposition(dev, type, code, &value);
if (disposition != INPUT_IGNORE_EVENT) {
if (type != EV_SYN)
add_input_randomness(type, code, value); input_event_dispose(dev, disposition, type, code, value);
}
}
/**
* input_event() - report new input event
* @dev: device that generated the event
* @type: type of the event
* @code: event code
* @value: value of the event
*
* This function should be used by drivers implementing various input
* devices to report input events. See also input_inject_event().
*
* NOTE: input_event() may be safely used right after input device was
* allocated with input_allocate_device(), even before it is registered
* with input_register_device(), but the event will not reach any of the
* input handlers. Such early invocation of input_event() may be used
* to 'seed' initial state of a switch or initial position of absolute
* axis, etc.
*/
void input_event(struct input_dev *dev,
unsigned int type, unsigned int code, int value)
{
unsigned long flags;
if (is_event_supported(type, dev->evbit, EV_MAX)) { spin_lock_irqsave(&dev->event_lock, flags);
input_handle_event(dev, type, code, value);
spin_unlock_irqrestore(&dev->event_lock, flags);
}
}
EXPORT_SYMBOL(input_event);
/**
* input_inject_event() - send input event from input handler
* @handle: input handle to send event through
* @type: type of the event
* @code: event code
* @value: value of the event
*
* Similar to input_event() but will ignore event if device is
* "grabbed" and handle injecting event is not the one that owns
* the device.
*/
void input_inject_event(struct input_handle *handle,
unsigned int type, unsigned int code, int value)
{
struct input_dev *dev = handle->dev;
struct input_handle *grab;
unsigned long flags;
if (is_event_supported(type, dev->evbit, EV_MAX)) { spin_lock_irqsave(&dev->event_lock, flags); rcu_read_lock(); grab = rcu_dereference(dev->grab); if (!grab || grab == handle) input_handle_event(dev, type, code, value); rcu_read_unlock();
spin_unlock_irqrestore(&dev->event_lock, flags);
}
}
EXPORT_SYMBOL(input_inject_event);
/**
* input_alloc_absinfo - allocates array of input_absinfo structs
* @dev: the input device emitting absolute events
*
* If the absinfo struct the caller asked for is already allocated, this
* functions will not do anything.
*/
void input_alloc_absinfo(struct input_dev *dev)
{
if (dev->absinfo)
return;
dev->absinfo = kcalloc(ABS_CNT, sizeof(*dev->absinfo), GFP_KERNEL);
if (!dev->absinfo) {
dev_err(dev->dev.parent ?: &dev->dev,
"%s: unable to allocate memory\n", __func__);
/*
* We will handle this allocation failure in
* input_register_device() when we refuse to register input
* device with ABS bits but without absinfo.
*/
}
}
EXPORT_SYMBOL(input_alloc_absinfo);
void input_set_abs_params(struct input_dev *dev, unsigned int axis,
int min, int max, int fuzz, int flat)
{
struct input_absinfo *absinfo;
__set_bit(EV_ABS, dev->evbit);
__set_bit(axis, dev->absbit);
input_alloc_absinfo(dev);
if (!dev->absinfo)
return;
absinfo = &dev->absinfo[axis];
absinfo->minimum = min;
absinfo->maximum = max;
absinfo->fuzz = fuzz;
absinfo->flat = flat;
}
EXPORT_SYMBOL(input_set_abs_params);
/**
* input_copy_abs - Copy absinfo from one input_dev to another
* @dst: Destination input device to copy the abs settings to
* @dst_axis: ABS_* value selecting the destination axis
* @src: Source input device to copy the abs settings from
* @src_axis: ABS_* value selecting the source axis
*
* Set absinfo for the selected destination axis by copying it from
* the specified source input device's source axis.
* This is useful to e.g. setup a pen/stylus input-device for combined
* touchscreen/pen hardware where the pen uses the same coordinates as
* the touchscreen.
*/
void input_copy_abs(struct input_dev *dst, unsigned int dst_axis,
const struct input_dev *src, unsigned int src_axis)
{
/* src must have EV_ABS and src_axis set */
if (WARN_ON(!(test_bit(EV_ABS, src->evbit) &&
test_bit(src_axis, src->absbit))))
return;
/*
* input_alloc_absinfo() may have failed for the source. Our caller is
* expected to catch this when registering the input devices, which may
* happen after the input_copy_abs() call.
*/
if (!src->absinfo)
return;
input_set_capability(dst, EV_ABS, dst_axis);
if (!dst->absinfo)
return;
dst->absinfo[dst_axis] = src->absinfo[src_axis];
}
EXPORT_SYMBOL(input_copy_abs);
/**
* input_grab_device - grabs device for exclusive use
* @handle: input handle that wants to own the device
*
* When a device is grabbed by an input handle all events generated by
* the device are delivered only to this handle. Also events injected
* by other input handles are ignored while device is grabbed.
*/
int input_grab_device(struct input_handle *handle)
{
struct input_dev *dev = handle->dev;
int retval;
retval = mutex_lock_interruptible(&dev->mutex);
if (retval)
return retval;
if (dev->grab) {
retval = -EBUSY;
goto out;
}
rcu_assign_pointer(dev->grab, handle);
out:
mutex_unlock(&dev->mutex); return retval;
}
EXPORT_SYMBOL(input_grab_device);
static void __input_release_device(struct input_handle *handle)
{
struct input_dev *dev = handle->dev;
struct input_handle *grabber;
grabber = rcu_dereference_protected(dev->grab,
lockdep_is_held(&dev->mutex));
if (grabber == handle) {
rcu_assign_pointer(dev->grab, NULL);
/* Make sure input_pass_values() notices that grab is gone */
synchronize_rcu();
list_for_each_entry(handle, &dev->h_list, d_node) if (handle->open && handle->handler->start) handle->handler->start(handle);
}
}
/**
* input_release_device - release previously grabbed device
* @handle: input handle that owns the device
*
* Releases previously grabbed device so that other input handles can
* start receiving input events. Upon release all handlers attached
* to the device have their start() method called so they have a change
* to synchronize device state with the rest of the system.
*/
void input_release_device(struct input_handle *handle)
{
struct input_dev *dev = handle->dev;
mutex_lock(&dev->mutex);
__input_release_device(handle);
mutex_unlock(&dev->mutex);
}
EXPORT_SYMBOL(input_release_device);
/**
* input_open_device - open input device
* @handle: handle through which device is being accessed
*
* This function should be called by input handlers when they
* want to start receive events from given input device.
*/
int input_open_device(struct input_handle *handle)
{
struct input_dev *dev = handle->dev;
int retval;
retval = mutex_lock_interruptible(&dev->mutex);
if (retval)
return retval;
if (dev->going_away) {
retval = -ENODEV;
goto out;
}
handle->open++; if (dev->users++ || dev->inhibited) {
/*
* Device is already opened and/or inhibited,
* so we can exit immediately and report success.
*/
goto out;
}
if (dev->open) { retval = dev->open(dev);
if (retval) {
dev->users--;
handle->open--;
/*
* Make sure we are not delivering any more events
* through this handle
*/
synchronize_rcu();
goto out;
}
}
if (dev->poller) input_dev_poller_start(dev->poller);
out:
mutex_unlock(&dev->mutex); return retval;
}
EXPORT_SYMBOL(input_open_device);
int input_flush_device(struct input_handle *handle, struct file *file)
{
struct input_dev *dev = handle->dev;
int retval;
retval = mutex_lock_interruptible(&dev->mutex);
if (retval)
return retval;
if (dev->flush) retval = dev->flush(dev, file); mutex_unlock(&dev->mutex); return retval;
}
EXPORT_SYMBOL(input_flush_device);
/**
* input_close_device - close input device
* @handle: handle through which device is being accessed
*
* This function should be called by input handlers when they
* want to stop receive events from given input device.
*/
void input_close_device(struct input_handle *handle)
{
struct input_dev *dev = handle->dev;
mutex_lock(&dev->mutex);
__input_release_device(handle);
if (!--dev->users && !dev->inhibited) { if (dev->poller) input_dev_poller_stop(dev->poller); if (dev->close) dev->close(dev);
}
if (!--handle->open) {
/*
* synchronize_rcu() makes sure that input_pass_values()
* completed and that no more input events are delivered
* through this handle
*/
synchronize_rcu();
}
mutex_unlock(&dev->mutex);
}
EXPORT_SYMBOL(input_close_device);
/*
* Simulate keyup events for all keys that are marked as pressed.
* The function must be called with dev->event_lock held.
*/
static bool input_dev_release_keys(struct input_dev *dev)
{
bool need_sync = false;
int code;
lockdep_assert_held(&dev->event_lock); if (is_event_supported(EV_KEY, dev->evbit, EV_MAX)) { for_each_set_bit(code, dev->key, KEY_CNT) { input_handle_event(dev, EV_KEY, code, 0);
need_sync = true;
}
}
return need_sync;
}
/*
* Prepare device for unregistering
*/
static void input_disconnect_device(struct input_dev *dev)
{
struct input_handle *handle;
/*
* Mark device as going away. Note that we take dev->mutex here
* not to protect access to dev->going_away but rather to ensure
* that there are no threads in the middle of input_open_device()
*/
mutex_lock(&dev->mutex);
dev->going_away = true;
mutex_unlock(&dev->mutex);
spin_lock_irq(&dev->event_lock);
/*
* Simulate keyup events for all pressed keys so that handlers
* are not left with "stuck" keys. The driver may continue
* generate events even after we done here but they will not
* reach any handlers.
*/
if (input_dev_release_keys(dev))
input_handle_event(dev, EV_SYN, SYN_REPORT, 1); list_for_each_entry(handle, &dev->h_list, d_node) handle->open = 0; spin_unlock_irq(&dev->event_lock);
}
/**
* input_scancode_to_scalar() - converts scancode in &struct input_keymap_entry
* @ke: keymap entry containing scancode to be converted.
* @scancode: pointer to the location where converted scancode should
* be stored.
*
* This function is used to convert scancode stored in &struct keymap_entry
* into scalar form understood by legacy keymap handling methods. These
* methods expect scancodes to be represented as 'unsigned int'.
*/
int input_scancode_to_scalar(const struct input_keymap_entry *ke,
unsigned int *scancode)
{
switch (ke->len) {
case 1:
*scancode = *((u8 *)ke->scancode);
break;
case 2:
*scancode = *((u16 *)ke->scancode);
break;
case 4:
*scancode = *((u32 *)ke->scancode);
break;
default:
return -EINVAL;
}
return 0;
}
EXPORT_SYMBOL(input_scancode_to_scalar);
/*
* Those routines handle the default case where no [gs]etkeycode() is
* defined. In this case, an array indexed by the scancode is used.
*/
static unsigned int input_fetch_keycode(struct input_dev *dev,
unsigned int index)
{
switch (dev->keycodesize) {
case 1:
return ((u8 *)dev->keycode)[index];
case 2:
return ((u16 *)dev->keycode)[index];
default:
return ((u32 *)dev->keycode)[index];
}
}
static int input_default_getkeycode(struct input_dev *dev,
struct input_keymap_entry *ke)
{
unsigned int index;
int error;
if (!dev->keycodesize)
return -EINVAL;
if (ke->flags & INPUT_KEYMAP_BY_INDEX) index = ke->index;
else {
error = input_scancode_to_scalar(ke, &index);
if (error)
return error;
}
if (index >= dev->keycodemax)
return -EINVAL;
ke->keycode = input_fetch_keycode(dev, index);
ke->index = index;
ke->len = sizeof(index);
memcpy(ke->scancode, &index, sizeof(index));
return 0;
}
static int input_default_setkeycode(struct input_dev *dev,
const struct input_keymap_entry *ke,
unsigned int *old_keycode)
{
unsigned int index;
int error;
int i;
if (!dev->keycodesize)
return -EINVAL;
if (ke->flags & INPUT_KEYMAP_BY_INDEX) { index = ke->index;
} else {
error = input_scancode_to_scalar(ke, &index);
if (error)
return error;
}
if (index >= dev->keycodemax)
return -EINVAL;
if (dev->keycodesize < sizeof(ke->keycode) && (ke->keycode >> (dev->keycodesize * 8)))
return -EINVAL;
switch (dev->keycodesize) {
case 1: {
u8 *k = (u8 *)dev->keycode;
*old_keycode = k[index];
k[index] = ke->keycode;
break;
}
case 2: {
u16 *k = (u16 *)dev->keycode;
*old_keycode = k[index];
k[index] = ke->keycode;
break;
}
default: {
u32 *k = (u32 *)dev->keycode;
*old_keycode = k[index];
k[index] = ke->keycode;
break;
}
}
if (*old_keycode <= KEY_MAX) { __clear_bit(*old_keycode, dev->keybit); for (i = 0; i < dev->keycodemax; i++) { if (input_fetch_keycode(dev, i) == *old_keycode) { __set_bit(*old_keycode, dev->keybit);
/* Setting the bit twice is useless, so break */
break;
}
}
}
__set_bit(ke->keycode, dev->keybit); return 0;
}
/**
* input_get_keycode - retrieve keycode currently mapped to a given scancode
* @dev: input device which keymap is being queried
* @ke: keymap entry
*
* This function should be called by anyone interested in retrieving current
* keymap. Presently evdev handlers use it.
*/
int input_get_keycode(struct input_dev *dev, struct input_keymap_entry *ke)
{
unsigned long flags;
int retval;
spin_lock_irqsave(&dev->event_lock, flags);
retval = dev->getkeycode(dev, ke);
spin_unlock_irqrestore(&dev->event_lock, flags);
return retval;
}
EXPORT_SYMBOL(input_get_keycode);
/**
* input_set_keycode - attribute a keycode to a given scancode
* @dev: input device which keymap is being updated
* @ke: new keymap entry
*
* This function should be called by anyone needing to update current
* keymap. Presently keyboard and evdev handlers use it.
*/
int input_set_keycode(struct input_dev *dev,
const struct input_keymap_entry *ke)
{
unsigned long flags;
unsigned int old_keycode;
int retval;
if (ke->keycode > KEY_MAX)
return -EINVAL;
spin_lock_irqsave(&dev->event_lock, flags);
retval = dev->setkeycode(dev, ke, &old_keycode);
if (retval)
goto out;
/* Make sure KEY_RESERVED did not get enabled. */
__clear_bit(KEY_RESERVED, dev->keybit);
/*
* Simulate keyup event if keycode is not present
* in the keymap anymore
*/
if (old_keycode > KEY_MAX) {
dev_warn(dev->dev.parent ?: &dev->dev,
"%s: got too big old keycode %#x\n",
__func__, old_keycode);
} else if (test_bit(EV_KEY, dev->evbit) && !is_event_supported(old_keycode, dev->keybit, KEY_MAX) && __test_and_clear_bit(old_keycode, dev->key)) {
/*
* We have to use input_event_dispose() here directly instead
* of input_handle_event() because the key we want to release
* here is considered no longer supported by the device and
* input_handle_event() will ignore it.
*/
input_event_dispose(dev, INPUT_PASS_TO_HANDLERS,
EV_KEY, old_keycode, 0);
input_event_dispose(dev, INPUT_PASS_TO_HANDLERS | INPUT_FLUSH,
EV_SYN, SYN_REPORT, 1);
}
out:
spin_unlock_irqrestore(&dev->event_lock, flags); return retval;
}
EXPORT_SYMBOL(input_set_keycode);
bool input_match_device_id(const struct input_dev *dev,
const struct input_device_id *id)
{
if (id->flags & INPUT_DEVICE_ID_MATCH_BUS)
if (id->bustype != dev->id.bustype)
return false;
if (id->flags & INPUT_DEVICE_ID_MATCH_VENDOR)
if (id->vendor != dev->id.vendor)
return false;
if (id->flags & INPUT_DEVICE_ID_MATCH_PRODUCT)
if (id->product != dev->id.product)
return false;
if (id->flags & INPUT_DEVICE_ID_MATCH_VERSION)
if (id->version != dev->id.version)
return false;
if (!bitmap_subset(id->evbit, dev->evbit, EV_MAX) ||
!bitmap_subset(id->keybit, dev->keybit, KEY_MAX) ||
!bitmap_subset(id->relbit, dev->relbit, REL_MAX) ||
!bitmap_subset(id->absbit, dev->absbit, ABS_MAX) ||
!bitmap_subset(id->mscbit, dev->mscbit, MSC_MAX) ||
!bitmap_subset(id->ledbit, dev->ledbit, LED_MAX) ||
!bitmap_subset(id->sndbit, dev->sndbit, SND_MAX) ||
!bitmap_subset(id->ffbit, dev->ffbit, FF_MAX) ||
!bitmap_subset(id->swbit, dev->swbit, SW_MAX) ||
!bitmap_subset(id->propbit, dev->propbit, INPUT_PROP_MAX)) {
return false;
}
return true;
}
EXPORT_SYMBOL(input_match_device_id);
static const struct input_device_id *input_match_device(struct input_handler *handler,
struct input_dev *dev)
{
const struct input_device_id *id;
for (id = handler->id_table; id->flags || id->driver_info; id++) {
if (input_match_device_id(dev, id) &&
(!handler->match || handler->match(handler, dev))) {
return id;
}
}
return NULL;
}
static int input_attach_handler(struct input_dev *dev, struct input_handler *handler)
{
const struct input_device_id *id;
int error;
id = input_match_device(handler, dev);
if (!id)
return -ENODEV;
error = handler->connect(handler, dev, id);
if (error && error != -ENODEV)
pr_err("failed to attach handler %s to device %s, error: %d\n",
handler->name, kobject_name(&dev->dev.kobj), error);
return error;
}
#ifdef CONFIG_COMPAT
static int input_bits_to_string(char *buf, int buf_size,
unsigned long bits, bool skip_empty)
{
int len = 0;
if (in_compat_syscall()) { u32 dword = bits >> 32; if (dword || !skip_empty) len += snprintf(buf, buf_size, "%x ", dword);
dword = bits & 0xffffffffUL;
if (dword || !skip_empty || len) len += snprintf(buf + len, max(buf_size - len, 0),
"%x", dword);
} else {
if (bits || !skip_empty) len += snprintf(buf, buf_size, "%lx", bits);
}
return len;
}
#else /* !CONFIG_COMPAT */
static int input_bits_to_string(char *buf, int buf_size,
unsigned long bits, bool skip_empty)
{
return bits || !skip_empty ?
snprintf(buf, buf_size, "%lx", bits) : 0;
}
#endif
#ifdef CONFIG_PROC_FS
static struct proc_dir_entry *proc_bus_input_dir;
static DECLARE_WAIT_QUEUE_HEAD(input_devices_poll_wait);
static int input_devices_state;
static inline void input_wakeup_procfs_readers(void)
{
input_devices_state++;
wake_up(&input_devices_poll_wait);
}
static __poll_t input_proc_devices_poll(struct file *file, poll_table *wait)
{
poll_wait(file, &input_devices_poll_wait, wait);
if (file->f_version != input_devices_state) {
file->f_version = input_devices_state;
return EPOLLIN | EPOLLRDNORM;
}
return 0;
}
union input_seq_state {
struct {
unsigned short pos;
bool mutex_acquired;
};
void *p;
};
static void *input_devices_seq_start(struct seq_file *seq, loff_t *pos)
{
union input_seq_state *state = (union input_seq_state *)&seq->private;
int error;
/* We need to fit into seq->private pointer */
BUILD_BUG_ON(sizeof(union input_seq_state) != sizeof(seq->private));
error = mutex_lock_interruptible(&input_mutex);
if (error) {
state->mutex_acquired = false;
return ERR_PTR(error);
}
state->mutex_acquired = true;
return seq_list_start(&input_dev_list, *pos);
}
static void *input_devices_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
return seq_list_next(v, &input_dev_list, pos);
}
static void input_seq_stop(struct seq_file *seq, void *v)
{
union input_seq_state *state = (union input_seq_state *)&seq->private;
if (state->mutex_acquired)
mutex_unlock(&input_mutex);
}
static void input_seq_print_bitmap(struct seq_file *seq, const char *name,
unsigned long *bitmap, int max)
{
int i;
bool skip_empty = true;
char buf[18];
seq_printf(seq, "B: %s=", name);
for (i = BITS_TO_LONGS(max) - 1; i >= 0; i--) {
if (input_bits_to_string(buf, sizeof(buf),
bitmap[i], skip_empty)) {
skip_empty = false;
seq_printf(seq, "%s%s", buf, i > 0 ? " " : "");
}
}
/*
* If no output was produced print a single 0.
*/
if (skip_empty)
seq_putc(seq, '0');
seq_putc(seq, '\n');
}
static int input_devices_seq_show(struct seq_file *seq, void *v)
{
struct input_dev *dev = container_of(v, struct input_dev, node);
const char *path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL);
struct input_handle *handle;
seq_printf(seq, "I: Bus=%04x Vendor=%04x Product=%04x Version=%04x\n",
dev->id.bustype, dev->id.vendor, dev->id.product, dev->id.version);
seq_printf(seq, "N: Name=\"%s\"\n", dev->name ? dev->name : "");
seq_printf(seq, "P: Phys=%s\n", dev->phys ? dev->phys : "");
seq_printf(seq, "S: Sysfs=%s\n", path ? path : "");
seq_printf(seq, "U: Uniq=%s\n", dev->uniq ? dev->uniq : "");
seq_puts(seq, "H: Handlers=");
list_for_each_entry(handle, &dev->h_list, d_node)
seq_printf(seq, "%s ", handle->name);
seq_putc(seq, '\n');
input_seq_print_bitmap(seq, "PROP", dev->propbit, INPUT_PROP_MAX);
input_seq_print_bitmap(seq, "EV", dev->evbit, EV_MAX);
if (test_bit(EV_KEY, dev->evbit))
input_seq_print_bitmap(seq, "KEY", dev->keybit, KEY_MAX);
if (test_bit(EV_REL, dev->evbit))
input_seq_print_bitmap(seq, "REL", dev->relbit, REL_MAX);
if (test_bit(EV_ABS, dev->evbit))
input_seq_print_bitmap(seq, "ABS", dev->absbit, ABS_MAX);
if (test_bit(EV_MSC, dev->evbit))
input_seq_print_bitmap(seq, "MSC", dev->mscbit, MSC_MAX);
if (test_bit(EV_LED, dev->evbit))
input_seq_print_bitmap(seq, "LED", dev->ledbit, LED_MAX);
if (test_bit(EV_SND, dev->evbit))
input_seq_print_bitmap(seq, "SND", dev->sndbit, SND_MAX);
if (test_bit(EV_FF, dev->evbit))
input_seq_print_bitmap(seq, "FF", dev->ffbit, FF_MAX);
if (test_bit(EV_SW, dev->evbit))
input_seq_print_bitmap(seq, "SW", dev->swbit, SW_MAX);
seq_putc(seq, '\n');
kfree(path);
return 0;
}
static const struct seq_operations input_devices_seq_ops = {
.start = input_devices_seq_start,
.next = input_devices_seq_next,
.stop = input_seq_stop,
.show = input_devices_seq_show,
};
static int input_proc_devices_open(struct inode *inode, struct file *file)
{
return seq_open(file, &input_devices_seq_ops);
}
static const struct proc_ops input_devices_proc_ops = {
.proc_open = input_proc_devices_open,
.proc_poll = input_proc_devices_poll,
.proc_read = seq_read,
.proc_lseek = seq_lseek,
.proc_release = seq_release,
};
static void *input_handlers_seq_start(struct seq_file *seq, loff_t *pos)
{
union input_seq_state *state = (union input_seq_state *)&seq->private;
int error;
/* We need to fit into seq->private pointer */
BUILD_BUG_ON(sizeof(union input_seq_state) != sizeof(seq->private));
error = mutex_lock_interruptible(&input_mutex);
if (error) {
state->mutex_acquired = false;
return ERR_PTR(error);
}
state->mutex_acquired = true;
state->pos = *pos;
return seq_list_start(&input_handler_list, *pos);
}
static void *input_handlers_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
union input_seq_state *state = (union input_seq_state *)&seq->private;
state->pos = *pos + 1;
return seq_list_next(v, &input_handler_list, pos);
}
static int input_handlers_seq_show(struct seq_file *seq, void *v)
{
struct input_handler *handler = container_of(v, struct input_handler, node);
union input_seq_state *state = (union input_seq_state *)&seq->private;
seq_printf(seq, "N: Number=%u Name=%s", state->pos, handler->name);
if (handler->filter)
seq_puts(seq, " (filter)");
if (handler->legacy_minors)
seq_printf(seq, " Minor=%d", handler->minor);
seq_putc(seq, '\n');
return 0;
}
static const struct seq_operations input_handlers_seq_ops = {
.start = input_handlers_seq_start,
.next = input_handlers_seq_next,
.stop = input_seq_stop,
.show = input_handlers_seq_show,
};
static int input_proc_handlers_open(struct inode *inode, struct file *file)
{
return seq_open(file, &input_handlers_seq_ops);
}
static const struct proc_ops input_handlers_proc_ops = {
.proc_open = input_proc_handlers_open,
.proc_read = seq_read,
.proc_lseek = seq_lseek,
.proc_release = seq_release,
};
static int __init input_proc_init(void)
{
struct proc_dir_entry *entry;
proc_bus_input_dir = proc_mkdir("bus/input", NULL);
if (!proc_bus_input_dir)
return -ENOMEM;
entry = proc_create("devices", 0, proc_bus_input_dir,
&input_devices_proc_ops);
if (!entry)
goto fail1;
entry = proc_create("handlers", 0, proc_bus_input_dir,
&input_handlers_proc_ops);
if (!entry)
goto fail2;
return 0;
fail2: remove_proc_entry("devices", proc_bus_input_dir);
fail1: remove_proc_entry("bus/input", NULL);
return -ENOMEM;
}
static void input_proc_exit(void)
{
remove_proc_entry("devices", proc_bus_input_dir);
remove_proc_entry("handlers", proc_bus_input_dir);
remove_proc_entry("bus/input", NULL);
}
#else /* !CONFIG_PROC_FS */
static inline void input_wakeup_procfs_readers(void) { }
static inline int input_proc_init(void) { return 0; }
static inline void input_proc_exit(void) { }
#endif
#define INPUT_DEV_STRING_ATTR_SHOW(name) \
static ssize_t input_dev_show_##name(struct device *dev, \
struct device_attribute *attr, \
char *buf) \
{ \
struct input_dev *input_dev = to_input_dev(dev); \
\
return sysfs_emit(buf, "%s\n", \
input_dev->name ? input_dev->name : ""); \
} \
static DEVICE_ATTR(name, S_IRUGO, input_dev_show_##name, NULL)
INPUT_DEV_STRING_ATTR_SHOW(name);
INPUT_DEV_STRING_ATTR_SHOW(phys);
INPUT_DEV_STRING_ATTR_SHOW(uniq);
static int input_print_modalias_bits(char *buf, int size,
char name, const unsigned long *bm,
unsigned int min_bit, unsigned int max_bit)
{
int len = 0, i;
len += snprintf(buf, max(size, 0), "%c", name);
for (i = min_bit; i < max_bit; i++) if (bm[BIT_WORD(i)] & BIT_MASK(i)) len += snprintf(buf + len, max(size - len, 0), "%X,", i);
return len;
}
static int input_print_modalias(char *buf, int size, const struct input_dev *id,
int add_cr)
{
int len;
len = snprintf(buf, max(size, 0),
"input:b%04Xv%04Xp%04Xe%04X-",
id->id.bustype, id->id.vendor,
id->id.product, id->id.version); len += input_print_modalias_bits(buf + len, size - len,
'e', id->evbit, 0, EV_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'k', id->keybit, KEY_MIN_INTERESTING, KEY_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'r', id->relbit, 0, REL_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'a', id->absbit, 0, ABS_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'm', id->mscbit, 0, MSC_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'l', id->ledbit, 0, LED_MAX);
len += input_print_modalias_bits(buf + len, size - len,
's', id->sndbit, 0, SND_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'f', id->ffbit, 0, FF_MAX);
len += input_print_modalias_bits(buf + len, size - len,
'w', id->swbit, 0, SW_MAX);
if (add_cr)
len += snprintf(buf + len, max(size - len, 0), "\n"); return len;
}
static ssize_t input_dev_show_modalias(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct input_dev *id = to_input_dev(dev);
ssize_t len;
len = input_print_modalias(buf, PAGE_SIZE, id, 1);
return min_t(int, len, PAGE_SIZE);
}
static DEVICE_ATTR(modalias, S_IRUGO, input_dev_show_modalias, NULL);
static int input_print_bitmap(char *buf, int buf_size, const unsigned long *bitmap,
int max, int add_cr);
static ssize_t input_dev_show_properties(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct input_dev *input_dev = to_input_dev(dev);
int len = input_print_bitmap(buf, PAGE_SIZE, input_dev->propbit,
INPUT_PROP_MAX, true);
return min_t(int, len, PAGE_SIZE);
}
static DEVICE_ATTR(properties, S_IRUGO, input_dev_show_properties, NULL);
static int input_inhibit_device(struct input_dev *dev);
static int input_uninhibit_device(struct input_dev *dev);
static ssize_t inhibited_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct input_dev *input_dev = to_input_dev(dev);
return sysfs_emit(buf, "%d\n", input_dev->inhibited);
}
static ssize_t inhibited_store(struct device *dev,
struct device_attribute *attr, const char *buf,
size_t len)
{
struct input_dev *input_dev = to_input_dev(dev);
ssize_t rv;
bool inhibited;
if (kstrtobool(buf, &inhibited))
return -EINVAL;
if (inhibited)
rv = input_inhibit_device(input_dev);
else
rv = input_uninhibit_device(input_dev);
if (rv != 0)
return rv;
return len;
}
static DEVICE_ATTR_RW(inhibited);
static struct attribute *input_dev_attrs[] = {
&dev_attr_name.attr,
&dev_attr_phys.attr,
&dev_attr_uniq.attr,
&dev_attr_modalias.attr,
&dev_attr_properties.attr,
&dev_attr_inhibited.attr,
NULL
};
static const struct attribute_group input_dev_attr_group = {
.attrs = input_dev_attrs,
};
#define INPUT_DEV_ID_ATTR(name) \
static ssize_t input_dev_show_id_##name(struct device *dev, \
struct device_attribute *attr, \
char *buf) \
{ \
struct input_dev *input_dev = to_input_dev(dev); \
return sysfs_emit(buf, "%04x\n", input_dev->id.name); \
} \
static DEVICE_ATTR(name, S_IRUGO, input_dev_show_id_##name, NULL)
INPUT_DEV_ID_ATTR(bustype);
INPUT_DEV_ID_ATTR(vendor);
INPUT_DEV_ID_ATTR(product);
INPUT_DEV_ID_ATTR(version);
static struct attribute *input_dev_id_attrs[] = {
&dev_attr_bustype.attr,
&dev_attr_vendor.attr,
&dev_attr_product.attr,
&dev_attr_version.attr,
NULL
};
static const struct attribute_group input_dev_id_attr_group = {
.name = "id",
.attrs = input_dev_id_attrs,
};
static int input_print_bitmap(char *buf, int buf_size, const unsigned long *bitmap,
int max, int add_cr)
{
int i;
int len = 0;
bool skip_empty = true;
for (i = BITS_TO_LONGS(max) - 1; i >= 0; i--) { len += input_bits_to_string(buf + len, max(buf_size - len, 0),
bitmap[i], skip_empty);
if (len) {
skip_empty = false;
if (i > 0) len += snprintf(buf + len, max(buf_size - len, 0), " ");
}
}
/*
* If no output was produced print a single 0.
*/
if (len == 0) len = snprintf(buf, buf_size, "%d", 0); if (add_cr) len += snprintf(buf + len, max(buf_size - len, 0), "\n"); return len;
}
#define INPUT_DEV_CAP_ATTR(ev, bm) \
static ssize_t input_dev_show_cap_##bm(struct device *dev, \
struct device_attribute *attr, \
char *buf) \
{ \
struct input_dev *input_dev = to_input_dev(dev); \
int len = input_print_bitmap(buf, PAGE_SIZE, \
input_dev->bm##bit, ev##_MAX, \
true); \
return min_t(int, len, PAGE_SIZE); \
} \
static DEVICE_ATTR(bm, S_IRUGO, input_dev_show_cap_##bm, NULL)
INPUT_DEV_CAP_ATTR(EV, ev);
INPUT_DEV_CAP_ATTR(KEY, key);
INPUT_DEV_CAP_ATTR(REL, rel);
INPUT_DEV_CAP_ATTR(ABS, abs);
INPUT_DEV_CAP_ATTR(MSC, msc);
INPUT_DEV_CAP_ATTR(LED, led);
INPUT_DEV_CAP_ATTR(SND, snd);
INPUT_DEV_CAP_ATTR(FF, ff);
INPUT_DEV_CAP_ATTR(SW, sw);
static struct attribute *input_dev_caps_attrs[] = {
&dev_attr_ev.attr,
&dev_attr_key.attr,
&dev_attr_rel.attr,
&dev_attr_abs.attr,
&dev_attr_msc.attr,
&dev_attr_led.attr,
&dev_attr_snd.attr,
&dev_attr_ff.attr,
&dev_attr_sw.attr,
NULL
};
static const struct attribute_group input_dev_caps_attr_group = {
.name = "capabilities",
.attrs = input_dev_caps_attrs,
};
static const struct attribute_group *input_dev_attr_groups[] = {
&input_dev_attr_group,
&input_dev_id_attr_group,
&input_dev_caps_attr_group,
&input_poller_attribute_group,
NULL
};
static void input_dev_release(struct device *device)
{
struct input_dev *dev = to_input_dev(device);
input_ff_destroy(dev);
input_mt_destroy_slots(dev);
kfree(dev->poller);
kfree(dev->absinfo);
kfree(dev->vals);
kfree(dev);
module_put(THIS_MODULE);
}
/*
* Input uevent interface - loading event handlers based on
* device bitfields.
*/
static int input_add_uevent_bm_var(struct kobj_uevent_env *env,
const char *name, const unsigned long *bitmap, int max)
{
int len;
if (add_uevent_var(env, "%s", name))
return -ENOMEM;
len = input_print_bitmap(&env->buf[env->buflen - 1],
sizeof(env->buf) - env->buflen,
bitmap, max, false);
if (len >= (sizeof(env->buf) - env->buflen))
return -ENOMEM;
env->buflen += len; return 0;
}
static int input_add_uevent_modalias_var(struct kobj_uevent_env *env,
const struct input_dev *dev)
{
int len;
if (add_uevent_var(env, "MODALIAS="))
return -ENOMEM;
len = input_print_modalias(&env->buf[env->buflen - 1],
sizeof(env->buf) - env->buflen,
dev, 0);
if (len >= (sizeof(env->buf) - env->buflen))
return -ENOMEM;
env->buflen += len;
return 0;
}
#define INPUT_ADD_HOTPLUG_VAR(fmt, val...) \
do { \
int err = add_uevent_var(env, fmt, val); \
if (err) \
return err; \
} while (0)
#define INPUT_ADD_HOTPLUG_BM_VAR(name, bm, max) \
do { \
int err = input_add_uevent_bm_var(env, name, bm, max); \
if (err) \
return err; \
} while (0)
#define INPUT_ADD_HOTPLUG_MODALIAS_VAR(dev) \
do { \
int err = input_add_uevent_modalias_var(env, dev); \
if (err) \
return err; \
} while (0)
static int input_dev_uevent(const struct device *device, struct kobj_uevent_env *env)
{
const struct input_dev *dev = to_input_dev(device);
INPUT_ADD_HOTPLUG_VAR("PRODUCT=%x/%x/%x/%x",
dev->id.bustype, dev->id.vendor,
dev->id.product, dev->id.version);
if (dev->name) INPUT_ADD_HOTPLUG_VAR("NAME=\"%s\"", dev->name); if (dev->phys) INPUT_ADD_HOTPLUG_VAR("PHYS=\"%s\"", dev->phys); if (dev->uniq) INPUT_ADD_HOTPLUG_VAR("UNIQ=\"%s\"", dev->uniq); INPUT_ADD_HOTPLUG_BM_VAR("PROP=", dev->propbit, INPUT_PROP_MAX); INPUT_ADD_HOTPLUG_BM_VAR("EV=", dev->evbit, EV_MAX); if (test_bit(EV_KEY, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("KEY=", dev->keybit, KEY_MAX); if (test_bit(EV_REL, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("REL=", dev->relbit, REL_MAX); if (test_bit(EV_ABS, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("ABS=", dev->absbit, ABS_MAX); if (test_bit(EV_MSC, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("MSC=", dev->mscbit, MSC_MAX); if (test_bit(EV_LED, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("LED=", dev->ledbit, LED_MAX); if (test_bit(EV_SND, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("SND=", dev->sndbit, SND_MAX); if (test_bit(EV_FF, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("FF=", dev->ffbit, FF_MAX); if (test_bit(EV_SW, dev->evbit)) INPUT_ADD_HOTPLUG_BM_VAR("SW=", dev->swbit, SW_MAX); INPUT_ADD_HOTPLUG_MODALIAS_VAR(dev);
return 0;
}
#define INPUT_DO_TOGGLE(dev, type, bits, on) \
do { \
int i; \
bool active; \
\
if (!test_bit(EV_##type, dev->evbit)) \
break; \
\
for_each_set_bit(i, dev->bits##bit, type##_CNT) { \
active = test_bit(i, dev->bits); \
if (!active && !on) \
continue; \
\
dev->event(dev, EV_##type, i, on ? active : 0); \
} \
} while (0)
static void input_dev_toggle(struct input_dev *dev, bool activate)
{
if (!dev->event)
return;
INPUT_DO_TOGGLE(dev, LED, led, activate);
INPUT_DO_TOGGLE(dev, SND, snd, activate);
if (activate && test_bit(EV_REP, dev->evbit)) {
dev->event(dev, EV_REP, REP_PERIOD, dev->rep[REP_PERIOD]);
dev->event(dev, EV_REP, REP_DELAY, dev->rep[REP_DELAY]);
}
}
/**
* input_reset_device() - reset/restore the state of input device
* @dev: input device whose state needs to be reset
*
* This function tries to reset the state of an opened input device and
* bring internal state and state if the hardware in sync with each other.
* We mark all keys as released, restore LED state, repeat rate, etc.
*/
void input_reset_device(struct input_dev *dev)
{
unsigned long flags;
mutex_lock(&dev->mutex);
spin_lock_irqsave(&dev->event_lock, flags);
input_dev_toggle(dev, true);
if (input_dev_release_keys(dev))
input_handle_event(dev, EV_SYN, SYN_REPORT, 1);
spin_unlock_irqrestore(&dev->event_lock, flags);
mutex_unlock(&dev->mutex);
}
EXPORT_SYMBOL(input_reset_device);
static int input_inhibit_device(struct input_dev *dev)
{
mutex_lock(&dev->mutex);
if (dev->inhibited)
goto out;
if (dev->users) {
if (dev->close)
dev->close(dev);
if (dev->poller)
input_dev_poller_stop(dev->poller);
}
spin_lock_irq(&dev->event_lock);
input_mt_release_slots(dev);
input_dev_release_keys(dev);
input_handle_event(dev, EV_SYN, SYN_REPORT, 1);
input_dev_toggle(dev, false);
spin_unlock_irq(&dev->event_lock);
dev->inhibited = true;
out:
mutex_unlock(&dev->mutex);
return 0;
}
static int input_uninhibit_device(struct input_dev *dev)
{
int ret = 0;
mutex_lock(&dev->mutex);
if (!dev->inhibited)
goto out;
if (dev->users) {
if (dev->open) {
ret = dev->open(dev);
if (ret)
goto out;
}
if (dev->poller)
input_dev_poller_start(dev->poller);
}
dev->inhibited = false;
spin_lock_irq(&dev->event_lock);
input_dev_toggle(dev, true);
spin_unlock_irq(&dev->event_lock);
out:
mutex_unlock(&dev->mutex);
return ret;
}
static int input_dev_suspend(struct device *dev)
{
struct input_dev *input_dev = to_input_dev(dev);
spin_lock_irq(&input_dev->event_lock);
/*
* Keys that are pressed now are unlikely to be
* still pressed when we resume.
*/
if (input_dev_release_keys(input_dev))
input_handle_event(input_dev, EV_SYN, SYN_REPORT, 1);
/* Turn off LEDs and sounds, if any are active. */
input_dev_toggle(input_dev, false);
spin_unlock_irq(&input_dev->event_lock);
return 0;
}
static int input_dev_resume(struct device *dev)
{
struct input_dev *input_dev = to_input_dev(dev);
spin_lock_irq(&input_dev->event_lock);
/* Restore state of LEDs and sounds, if any were active. */
input_dev_toggle(input_dev, true);
spin_unlock_irq(&input_dev->event_lock);
return 0;
}
static int input_dev_freeze(struct device *dev)
{
struct input_dev *input_dev = to_input_dev(dev);
spin_lock_irq(&input_dev->event_lock);
/*
* Keys that are pressed now are unlikely to be
* still pressed when we resume.
*/
if (input_dev_release_keys(input_dev))
input_handle_event(input_dev, EV_SYN, SYN_REPORT, 1);
spin_unlock_irq(&input_dev->event_lock);
return 0;
}
static int input_dev_poweroff(struct device *dev)
{
struct input_dev *input_dev = to_input_dev(dev);
spin_lock_irq(&input_dev->event_lock);
/* Turn off LEDs and sounds, if any are active. */
input_dev_toggle(input_dev, false);
spin_unlock_irq(&input_dev->event_lock);
return 0;
}
static const struct dev_pm_ops input_dev_pm_ops = {
.suspend = input_dev_suspend,
.resume = input_dev_resume,
.freeze = input_dev_freeze,
.poweroff = input_dev_poweroff,
.restore = input_dev_resume,
};
static const struct device_type input_dev_type = {
.groups = input_dev_attr_groups,
.release = input_dev_release,
.uevent = input_dev_uevent,
.pm = pm_sleep_ptr(&input_dev_pm_ops),
};
static char *input_devnode(const struct device *dev, umode_t *mode)
{
return kasprintf(GFP_KERNEL, "input/%s", dev_name(dev));
}
const struct class input_class = {
.name = "input",
.devnode = input_devnode,
};
EXPORT_SYMBOL_GPL(input_class);
/**
* input_allocate_device - allocate memory for new input device
*
* Returns prepared struct input_dev or %NULL.
*
* NOTE: Use input_free_device() to free devices that have not been
* registered; input_unregister_device() should be used for already
* registered devices.
*/
struct input_dev *input_allocate_device(void)
{
static atomic_t input_no = ATOMIC_INIT(-1);
struct input_dev *dev;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (dev) {
dev->dev.type = &input_dev_type;
dev->dev.class = &input_class;
device_initialize(&dev->dev);
mutex_init(&dev->mutex);
spin_lock_init(&dev->event_lock);
timer_setup(&dev->timer, NULL, 0);
INIT_LIST_HEAD(&dev->h_list);
INIT_LIST_HEAD(&dev->node);
dev_set_name(&dev->dev, "input%lu",
(unsigned long)atomic_inc_return(&input_no));
__module_get(THIS_MODULE);
}
return dev;
}
EXPORT_SYMBOL(input_allocate_device);
struct input_devres {
struct input_dev *input;
};
static int devm_input_device_match(struct device *dev, void *res, void *data)
{
struct input_devres *devres = res;
return devres->input == data;
}
static void devm_input_device_release(struct device *dev, void *res)
{
struct input_devres *devres = res;
struct input_dev *input = devres->input;
dev_dbg(dev, "%s: dropping reference to %s\n",
__func__, dev_name(&input->dev));
input_put_device(input);
}
/**
* devm_input_allocate_device - allocate managed input device
* @dev: device owning the input device being created
*
* Returns prepared struct input_dev or %NULL.
*
* Managed input devices do not need to be explicitly unregistered or
* freed as it will be done automatically when owner device unbinds from
* its driver (or binding fails). Once managed input device is allocated,
* it is ready to be set up and registered in the same fashion as regular
* input device. There are no special devm_input_device_[un]register()
* variants, regular ones work with both managed and unmanaged devices,
* should you need them. In most cases however, managed input device need
* not be explicitly unregistered or freed.
*
* NOTE: the owner device is set up as parent of input device and users
* should not override it.
*/
struct input_dev *devm_input_allocate_device(struct device *dev)
{
struct input_dev *input;
struct input_devres *devres;
devres = devres_alloc(devm_input_device_release,
sizeof(*devres), GFP_KERNEL);
if (!devres)
return NULL;
input = input_allocate_device();
if (!input) {
devres_free(devres);
return NULL;
}
input->dev.parent = dev;
input->devres_managed = true;
devres->input = input;
devres_add(dev, devres);
return input;
}
EXPORT_SYMBOL(devm_input_allocate_device);
/**
* input_free_device - free memory occupied by input_dev structure
* @dev: input device to free
*
* This function should only be used if input_register_device()
* was not called yet or if it failed. Once device was registered
* use input_unregister_device() and memory will be freed once last
* reference to the device is dropped.
*
* Device should be allocated by input_allocate_device().
*
* NOTE: If there are references to the input device then memory
* will not be freed until last reference is dropped.
*/
void input_free_device(struct input_dev *dev)
{
if (dev) { if (dev->devres_managed) WARN_ON(devres_destroy(dev->dev.parent,
devm_input_device_release,
devm_input_device_match,
dev));
input_put_device(dev);
}
}
EXPORT_SYMBOL(input_free_device);
/**
* input_set_timestamp - set timestamp for input events
* @dev: input device to set timestamp for
* @timestamp: the time at which the event has occurred
* in CLOCK_MONOTONIC
*
* This function is intended to provide to the input system a more
* accurate time of when an event actually occurred. The driver should
* call this function as soon as a timestamp is acquired ensuring
* clock conversions in input_set_timestamp are done correctly.
*
* The system entering suspend state between timestamp acquisition and
* calling input_set_timestamp can result in inaccurate conversions.
*/
void input_set_timestamp(struct input_dev *dev, ktime_t timestamp)
{
dev->timestamp[INPUT_CLK_MONO] = timestamp;
dev->timestamp[INPUT_CLK_REAL] = ktime_mono_to_real(timestamp);
dev->timestamp[INPUT_CLK_BOOT] = ktime_mono_to_any(timestamp,
TK_OFFS_BOOT);
}
EXPORT_SYMBOL(input_set_timestamp);
/**
* input_get_timestamp - get timestamp for input events
* @dev: input device to get timestamp from
*
* A valid timestamp is a timestamp of non-zero value.
*/
ktime_t *input_get_timestamp(struct input_dev *dev)
{
const ktime_t invalid_timestamp = ktime_set(0, 0);
if (!ktime_compare(dev->timestamp[INPUT_CLK_MONO], invalid_timestamp)) input_set_timestamp(dev, ktime_get()); return dev->timestamp;
}
EXPORT_SYMBOL(input_get_timestamp);
/**
* input_set_capability - mark device as capable of a certain event
* @dev: device that is capable of emitting or accepting event
* @type: type of the event (EV_KEY, EV_REL, etc...)
* @code: event code
*
* In addition to setting up corresponding bit in appropriate capability
* bitmap the function also adjusts dev->evbit.
*/
void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int code)
{
if (type < EV_CNT && input_max_code[type] &&
code > input_max_code[type]) {
pr_err("%s: invalid code %u for type %u\n", __func__, code,
type);
dump_stack();
return;
}
switch (type) {
case EV_KEY:
__set_bit(code, dev->keybit);
break;
case EV_REL:
__set_bit(code, dev->relbit);
break;
case EV_ABS:
input_alloc_absinfo(dev);
__set_bit(code, dev->absbit);
break;
case EV_MSC:
__set_bit(code, dev->mscbit);
break;
case EV_SW:
__set_bit(code, dev->swbit);
break;
case EV_LED:
__set_bit(code, dev->ledbit);
break;
case EV_SND:
__set_bit(code, dev->sndbit);
break;
case EV_FF:
__set_bit(code, dev->ffbit);
break;
case EV_PWR:
/* do nothing */
break;
default:
pr_err("%s: unknown type %u (code %u)\n", __func__, type, code);
dump_stack();
return;
}
__set_bit(type, dev->evbit);
}
EXPORT_SYMBOL(input_set_capability);
static unsigned int input_estimate_events_per_packet(struct input_dev *dev)
{
int mt_slots;
int i;
unsigned int events;
if (dev->mt) {
mt_slots = dev->mt->num_slots;
} else if (test_bit(ABS_MT_TRACKING_ID, dev->absbit)) {
mt_slots = dev->absinfo[ABS_MT_TRACKING_ID].maximum -
dev->absinfo[ABS_MT_TRACKING_ID].minimum + 1,
mt_slots = clamp(mt_slots, 2, 32);
} else if (test_bit(ABS_MT_POSITION_X, dev->absbit)) {
mt_slots = 2;
} else {
mt_slots = 0;
}
events = mt_slots + 1; /* count SYN_MT_REPORT and SYN_REPORT */
if (test_bit(EV_ABS, dev->evbit))
for_each_set_bit(i, dev->absbit, ABS_CNT)
events += input_is_mt_axis(i) ? mt_slots : 1;
if (test_bit(EV_REL, dev->evbit))
events += bitmap_weight(dev->relbit, REL_CNT);
/* Make room for KEY and MSC events */
events += 7;
return events;
}
#define INPUT_CLEANSE_BITMASK(dev, type, bits) \
do { \
if (!test_bit(EV_##type, dev->evbit)) \
memset(dev->bits##bit, 0, \
sizeof(dev->bits##bit)); \
} while (0)
static void input_cleanse_bitmasks(struct input_dev *dev)
{
INPUT_CLEANSE_BITMASK(dev, KEY, key);
INPUT_CLEANSE_BITMASK(dev, REL, rel);
INPUT_CLEANSE_BITMASK(dev, ABS, abs);
INPUT_CLEANSE_BITMASK(dev, MSC, msc);
INPUT_CLEANSE_BITMASK(dev, LED, led);
INPUT_CLEANSE_BITMASK(dev, SND, snd);
INPUT_CLEANSE_BITMASK(dev, FF, ff);
INPUT_CLEANSE_BITMASK(dev, SW, sw);
}
static void __input_unregister_device(struct input_dev *dev)
{
struct input_handle *handle, *next;
input_disconnect_device(dev);
mutex_lock(&input_mutex);
list_for_each_entry_safe(handle, next, &dev->h_list, d_node)
handle->handler->disconnect(handle); WARN_ON(!list_empty(&dev->h_list)); del_timer_sync(&dev->timer); list_del_init(&dev->node);
input_wakeup_procfs_readers();
mutex_unlock(&input_mutex);
device_del(&dev->dev);
}
static void devm_input_device_unregister(struct device *dev, void *res)
{
struct input_devres *devres = res;
struct input_dev *input = devres->input; dev_dbg(dev, "%s: unregistering device %s\n",
__func__, dev_name(&input->dev));
__input_unregister_device(input);
}
/*
* Generate software autorepeat event. Note that we take
* dev->event_lock here to avoid racing with input_event
* which may cause keys get "stuck".
*/
static void input_repeat_key(struct timer_list *t)
{
struct input_dev *dev = from_timer(dev, t, timer);
unsigned long flags;
spin_lock_irqsave(&dev->event_lock, flags);
if (!dev->inhibited &&
test_bit(dev->repeat_key, dev->key) &&
is_event_supported(dev->repeat_key, dev->keybit, KEY_MAX)) {
input_set_timestamp(dev, ktime_get());
input_handle_event(dev, EV_KEY, dev->repeat_key, 2);
input_handle_event(dev, EV_SYN, SYN_REPORT, 1);
if (dev->rep[REP_PERIOD])
mod_timer(&dev->timer, jiffies +
msecs_to_jiffies(dev->rep[REP_PERIOD]));
}
spin_unlock_irqrestore(&dev->event_lock, flags);
}
/**
* input_enable_softrepeat - enable software autorepeat
* @dev: input device
* @delay: repeat delay
* @period: repeat period
*
* Enable software autorepeat on the input device.
*/
void input_enable_softrepeat(struct input_dev *dev, int delay, int period)
{
dev->timer.function = input_repeat_key;
dev->rep[REP_DELAY] = delay;
dev->rep[REP_PERIOD] = period;
}
EXPORT_SYMBOL(input_enable_softrepeat);
bool input_device_enabled(struct input_dev *dev)
{
lockdep_assert_held(&dev->mutex);
return !dev->inhibited && dev->users > 0;
}
EXPORT_SYMBOL_GPL(input_device_enabled);
/**
* input_register_device - register device with input core
* @dev: device to be registered
*
* This function registers device with input core. The device must be
* allocated with input_allocate_device() and all it's capabilities
* set up before registering.
* If function fails the device must be freed with input_free_device().
* Once device has been successfully registered it can be unregistered
* with input_unregister_device(); input_free_device() should not be
* called in this case.
*
* Note that this function is also used to register managed input devices
* (ones allocated with devm_input_allocate_device()). Such managed input
* devices need not be explicitly unregistered or freed, their tear down
* is controlled by the devres infrastructure. It is also worth noting
* that tear down of managed input devices is internally a 2-step process:
* registered managed input device is first unregistered, but stays in
* memory and can still handle input_event() calls (although events will
* not be delivered anywhere). The freeing of managed input device will
* happen later, when devres stack is unwound to the point where device
* allocation was made.
*/
int input_register_device(struct input_dev *dev)
{
struct input_devres *devres = NULL;
struct input_handler *handler;
unsigned int packet_size;
const char *path;
int error;
if (test_bit(EV_ABS, dev->evbit) && !dev->absinfo) {
dev_err(&dev->dev,
"Absolute device without dev->absinfo, refusing to register\n");
return -EINVAL;
}
if (dev->devres_managed) {
devres = devres_alloc(devm_input_device_unregister,
sizeof(*devres), GFP_KERNEL);
if (!devres)
return -ENOMEM;
devres->input = dev;
}
/* Every input device generates EV_SYN/SYN_REPORT events. */
__set_bit(EV_SYN, dev->evbit);
/* KEY_RESERVED is not supposed to be transmitted to userspace. */
__clear_bit(KEY_RESERVED, dev->keybit);
/* Make sure that bitmasks not mentioned in dev->evbit are clean. */
input_cleanse_bitmasks(dev);
packet_size = input_estimate_events_per_packet(dev);
if (dev->hint_events_per_packet < packet_size)
dev->hint_events_per_packet = packet_size;
dev->max_vals = dev->hint_events_per_packet + 2;
dev->vals = kcalloc(dev->max_vals, sizeof(*dev->vals), GFP_KERNEL);
if (!dev->vals) {
error = -ENOMEM;
goto err_devres_free;
}
/*
* If delay and period are pre-set by the driver, then autorepeating
* is handled by the driver itself and we don't do it in input.c.
*/
if (!dev->rep[REP_DELAY] && !dev->rep[REP_PERIOD])
input_enable_softrepeat(dev, 250, 33);
if (!dev->getkeycode)
dev->getkeycode = input_default_getkeycode;
if (!dev->setkeycode)
dev->setkeycode = input_default_setkeycode;
if (dev->poller)
input_dev_poller_finalize(dev->poller);
error = device_add(&dev->dev);
if (error)
goto err_free_vals;
path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL);
pr_info("%s as %s\n",
dev->name ? dev->name : "Unspecified device",
path ? path : "N/A");
kfree(path);
error = mutex_lock_interruptible(&input_mutex);
if (error)
goto err_device_del;
list_add_tail(&dev->node, &input_dev_list);
list_for_each_entry(handler, &input_handler_list, node)
input_attach_handler(dev, handler);
input_wakeup_procfs_readers();
mutex_unlock(&input_mutex);
if (dev->devres_managed) {
dev_dbg(dev->dev.parent, "%s: registering %s with devres.\n",
__func__, dev_name(&dev->dev));
devres_add(dev->dev.parent, devres);
}
return 0;
err_device_del:
device_del(&dev->dev);
err_free_vals:
kfree(dev->vals);
dev->vals = NULL;
err_devres_free:
devres_free(devres);
return error;
}
EXPORT_SYMBOL(input_register_device);
/**
* input_unregister_device - unregister previously registered device
* @dev: device to be unregistered
*
* This function unregisters an input device. Once device is unregistered
* the caller should not try to access it as it may get freed at any moment.
*/
void input_unregister_device(struct input_dev *dev)
{
if (dev->devres_managed) { WARN_ON(devres_destroy(dev->dev.parent,
devm_input_device_unregister,
devm_input_device_match,
dev));
__input_unregister_device(dev);
/*
* We do not do input_put_device() here because it will be done
* when 2nd devres fires up.
*/
} else {
__input_unregister_device(dev); input_put_device(dev);
}
}
EXPORT_SYMBOL(input_unregister_device);
/**
* input_register_handler - register a new input handler
* @handler: handler to be registered
*
* This function registers a new input handler (interface) for input
* devices in the system and attaches it to all input devices that
* are compatible with the handler.
*/
int input_register_handler(struct input_handler *handler)
{
struct input_dev *dev;
int error;
error = mutex_lock_interruptible(&input_mutex);
if (error)
return error;
INIT_LIST_HEAD(&handler->h_list);
list_add_tail(&handler->node, &input_handler_list);
list_for_each_entry(dev, &input_dev_list, node)
input_attach_handler(dev, handler);
input_wakeup_procfs_readers();
mutex_unlock(&input_mutex);
return 0;
}
EXPORT_SYMBOL(input_register_handler);
/**
* input_unregister_handler - unregisters an input handler
* @handler: handler to be unregistered
*
* This function disconnects a handler from its input devices and
* removes it from lists of known handlers.
*/
void input_unregister_handler(struct input_handler *handler)
{
struct input_handle *handle, *next;
mutex_lock(&input_mutex);
list_for_each_entry_safe(handle, next, &handler->h_list, h_node)
handler->disconnect(handle);
WARN_ON(!list_empty(&handler->h_list));
list_del_init(&handler->node);
input_wakeup_procfs_readers();
mutex_unlock(&input_mutex);
}
EXPORT_SYMBOL(input_unregister_handler);
/**
* input_handler_for_each_handle - handle iterator
* @handler: input handler to iterate
* @data: data for the callback
* @fn: function to be called for each handle
*
* Iterate over @bus's list of devices, and call @fn for each, passing
* it @data and stop when @fn returns a non-zero value. The function is
* using RCU to traverse the list and therefore may be using in atomic
* contexts. The @fn callback is invoked from RCU critical section and
* thus must not sleep.
*/
int input_handler_for_each_handle(struct input_handler *handler, void *data,
int (*fn)(struct input_handle *, void *))
{
struct input_handle *handle;
int retval = 0;
rcu_read_lock();
list_for_each_entry_rcu(handle, &handler->h_list, h_node) {
retval = fn(handle, data);
if (retval)
break;
}
rcu_read_unlock();
return retval;
}
EXPORT_SYMBOL(input_handler_for_each_handle);
/**
* input_register_handle - register a new input handle
* @handle: handle to register
*
* This function puts a new input handle onto device's
* and handler's lists so that events can flow through
* it once it is opened using input_open_device().
*
* This function is supposed to be called from handler's
* connect() method.
*/
int input_register_handle(struct input_handle *handle)
{
struct input_handler *handler = handle->handler;
struct input_dev *dev = handle->dev;
int error;
/*
* We take dev->mutex here to prevent race with
* input_release_device().
*/
error = mutex_lock_interruptible(&dev->mutex);
if (error)
return error;
/*
* Filters go to the head of the list, normal handlers
* to the tail.
*/
if (handler->filter)
list_add_rcu(&handle->d_node, &dev->h_list);
else
list_add_tail_rcu(&handle->d_node, &dev->h_list);
mutex_unlock(&dev->mutex);
/*
* Since we are supposed to be called from ->connect()
* which is mutually exclusive with ->disconnect()
* we can't be racing with input_unregister_handle()
* and so separate lock is not needed here.
*/
list_add_tail_rcu(&handle->h_node, &handler->h_list);
if (handler->start)
handler->start(handle);
return 0;
}
EXPORT_SYMBOL(input_register_handle);
/**
* input_unregister_handle - unregister an input handle
* @handle: handle to unregister
*
* This function removes input handle from device's
* and handler's lists.
*
* This function is supposed to be called from handler's
* disconnect() method.
*/
void input_unregister_handle(struct input_handle *handle)
{
struct input_dev *dev = handle->dev; list_del_rcu(&handle->h_node);
/*
* Take dev->mutex to prevent race with input_release_device().
*/
mutex_lock(&dev->mutex);
list_del_rcu(&handle->d_node);
mutex_unlock(&dev->mutex);
synchronize_rcu();
}
EXPORT_SYMBOL(input_unregister_handle);
/**
* input_get_new_minor - allocates a new input minor number
* @legacy_base: beginning or the legacy range to be searched
* @legacy_num: size of legacy range
* @allow_dynamic: whether we can also take ID from the dynamic range
*
* This function allocates a new device minor for from input major namespace.
* Caller can request legacy minor by specifying @legacy_base and @legacy_num
* parameters and whether ID can be allocated from dynamic range if there are
* no free IDs in legacy range.
*/
int input_get_new_minor(int legacy_base, unsigned int legacy_num,
bool allow_dynamic)
{
/*
* This function should be called from input handler's ->connect()
* methods, which are serialized with input_mutex, so no additional
* locking is needed here.
*/
if (legacy_base >= 0) {
int minor = ida_alloc_range(&input_ida, legacy_base,
legacy_base + legacy_num - 1,
GFP_KERNEL);
if (minor >= 0 || !allow_dynamic)
return minor;
}
return ida_alloc_range(&input_ida, INPUT_FIRST_DYNAMIC_DEV,
INPUT_MAX_CHAR_DEVICES - 1, GFP_KERNEL);
}
EXPORT_SYMBOL(input_get_new_minor);
/**
* input_free_minor - release previously allocated minor
* @minor: minor to be released
*
* This function releases previously allocated input minor so that it can be
* reused later.
*/
void input_free_minor(unsigned int minor)
{
ida_free(&input_ida, minor);
}
EXPORT_SYMBOL(input_free_minor);
static int __init input_init(void)
{
int err;
err = class_register(&input_class);
if (err) {
pr_err("unable to register input_dev class\n");
return err;
}
err = input_proc_init();
if (err)
goto fail1;
err = register_chrdev_region(MKDEV(INPUT_MAJOR, 0),
INPUT_MAX_CHAR_DEVICES, "input");
if (err) {
pr_err("unable to register char major %d", INPUT_MAJOR);
goto fail2;
}
return 0;
fail2: input_proc_exit();
fail1: class_unregister(&input_class);
return err;
}
static void __exit input_exit(void)
{
input_proc_exit();
unregister_chrdev_region(MKDEV(INPUT_MAJOR, 0),
INPUT_MAX_CHAR_DEVICES);
class_unregister(&input_class);
}
subsys_initcall(input_init);
module_exit(input_exit);
// SPDX-License-Identifier: GPL-2.0-only
/*
* mm/percpu-vm.c - vmalloc area based chunk allocation
*
* Copyright (C) 2010 SUSE Linux Products GmbH
* Copyright (C) 2010 Tejun Heo <tj@kernel.org>
*
* Chunks are mapped into vmalloc areas and populated page by page.
* This is the default chunk allocator.
*/
#include "internal.h"
static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
unsigned int cpu, int page_idx)
{
/* must not be used on pre-mapped chunk */
WARN_ON(chunk->immutable);
return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
}
/**
* pcpu_get_pages - get temp pages array
*
* Returns pointer to array of pointers to struct page which can be indexed
* with pcpu_page_idx(). Note that there is only one array and accesses
* should be serialized by pcpu_alloc_mutex.
*
* RETURNS:
* Pointer to temp pages array on success.
*/
static struct page **pcpu_get_pages(void)
{
static struct page **pages;
size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
lockdep_assert_held(&pcpu_alloc_mutex);
if (!pages)
pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL);
return pages;
}
/**
* pcpu_free_pages - free pages which were allocated for @chunk
* @chunk: chunk pages were allocated for
* @pages: array of pages to be freed, indexed by pcpu_page_idx()
* @page_start: page index of the first page to be freed
* @page_end: page index of the last page to be freed + 1
*
* Free pages [@page_start and @page_end) in @pages for all units.
* The pages were allocated for @chunk.
*/
static void pcpu_free_pages(struct pcpu_chunk *chunk,
struct page **pages, int page_start, int page_end)
{
unsigned int cpu;
int i;
for_each_possible_cpu(cpu) {
for (i = page_start; i < page_end; i++) {
struct page *page = pages[pcpu_page_idx(cpu, i)];
if (page)
__free_page(page);
}
}
}
/**
* pcpu_alloc_pages - allocates pages for @chunk
* @chunk: target chunk
* @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
* @page_start: page index of the first page to be allocated
* @page_end: page index of the last page to be allocated + 1
* @gfp: allocation flags passed to the underlying allocator
*
* Allocate pages [@page_start,@page_end) into @pages for all units.
* The allocation is for @chunk. Percpu core doesn't care about the
* content of @pages and will pass it verbatim to pcpu_map_pages().
*/
static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
struct page **pages, int page_start, int page_end,
gfp_t gfp)
{
unsigned int cpu, tcpu;
int i;
gfp |= __GFP_HIGHMEM;
for_each_possible_cpu(cpu) {
for (i = page_start; i < page_end; i++) {
struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
*pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
if (!*pagep)
goto err;
}
}
return 0;
err:
while (--i >= page_start)
__free_page(pages[pcpu_page_idx(cpu, i)]);
for_each_possible_cpu(tcpu) {
if (tcpu == cpu)
break;
for (i = page_start; i < page_end; i++)
__free_page(pages[pcpu_page_idx(tcpu, i)]);
}
return -ENOMEM;
}
/**
* pcpu_pre_unmap_flush - flush cache prior to unmapping
* @chunk: chunk the regions to be flushed belongs to
* @page_start: page index of the first page to be flushed
* @page_end: page index of the last page to be flushed + 1
*
* Pages in [@page_start,@page_end) of @chunk are about to be
* unmapped. Flush cache. As each flushing trial can be very
* expensive, issue flush on the whole region at once rather than
* doing it for each cpu. This could be an overkill but is more
* scalable.
*/
static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
int page_start, int page_end)
{
flush_cache_vunmap(
pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
}
static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
{
vunmap_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT));
}
/**
* pcpu_unmap_pages - unmap pages out of a pcpu_chunk
* @chunk: chunk of interest
* @pages: pages array which can be used to pass information to free
* @page_start: page index of the first page to unmap
* @page_end: page index of the last page to unmap + 1
*
* For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
* Corresponding elements in @pages were cleared by the caller and can
* be used to carry information to pcpu_free_pages() which will be
* called after all unmaps are finished. The caller should call
* proper pre/post flush functions.
*/
static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
struct page **pages, int page_start, int page_end)
{
unsigned int cpu;
int i;
for_each_possible_cpu(cpu) {
for (i = page_start; i < page_end; i++) {
struct page *page;
page = pcpu_chunk_page(chunk, cpu, i);
WARN_ON(!page);
pages[pcpu_page_idx(cpu, i)] = page;
}
__pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
page_end - page_start);
}
}
/**
* pcpu_post_unmap_tlb_flush - flush TLB after unmapping
* @chunk: pcpu_chunk the regions to be flushed belong to
* @page_start: page index of the first page to be flushed
* @page_end: page index of the last page to be flushed + 1
*
* Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
* TLB for the regions. This can be skipped if the area is to be
* returned to vmalloc as vmalloc will handle TLB flushing lazily.
*
* As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
* for the whole region.
*/
static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
int page_start, int page_end)
{
flush_tlb_kernel_range(
pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
}
static int __pcpu_map_pages(unsigned long addr, struct page **pages,
int nr_pages)
{
return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT),
PAGE_KERNEL, pages, PAGE_SHIFT);
}
/**
* pcpu_map_pages - map pages into a pcpu_chunk
* @chunk: chunk of interest
* @pages: pages array containing pages to be mapped
* @page_start: page index of the first page to map
* @page_end: page index of the last page to map + 1
*
* For each cpu, map pages [@page_start,@page_end) into @chunk. The
* caller is responsible for calling pcpu_post_map_flush() after all
* mappings are complete.
*
* This function is responsible for setting up whatever is necessary for
* reverse lookup (addr -> chunk).
*/
static int pcpu_map_pages(struct pcpu_chunk *chunk,
struct page **pages, int page_start, int page_end)
{
unsigned int cpu, tcpu;
int i, err;
for_each_possible_cpu(cpu) {
err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
&pages[pcpu_page_idx(cpu, page_start)],
page_end - page_start);
if (err < 0)
goto err;
for (i = page_start; i < page_end; i++)
pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
chunk);
}
return 0;
err:
for_each_possible_cpu(tcpu) {
if (tcpu == cpu)
break;
__pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
page_end - page_start);
}
pcpu_post_unmap_tlb_flush(chunk, page_start, page_end);
return err;
}
/**
* pcpu_post_map_flush - flush cache after mapping
* @chunk: pcpu_chunk the regions to be flushed belong to
* @page_start: page index of the first page to be flushed
* @page_end: page index of the last page to be flushed + 1
*
* Pages [@page_start,@page_end) of @chunk have been mapped. Flush
* cache.
*
* As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
* for the whole region.
*/
static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
int page_start, int page_end)
{
flush_cache_vmap(
pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
}
/**
* pcpu_populate_chunk - populate and map an area of a pcpu_chunk
* @chunk: chunk of interest
* @page_start: the start page
* @page_end: the end page
* @gfp: allocation flags passed to the underlying memory allocator
*
* For each cpu, populate and map pages [@page_start,@page_end) into
* @chunk.
*
* CONTEXT:
* pcpu_alloc_mutex, does GFP_KERNEL allocation.
*/
static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
int page_start, int page_end, gfp_t gfp)
{
struct page **pages;
pages = pcpu_get_pages();
if (!pages)
return -ENOMEM;
if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))
return -ENOMEM;
if (pcpu_map_pages(chunk, pages, page_start, page_end)) {
pcpu_free_pages(chunk, pages, page_start, page_end);
return -ENOMEM;
}
pcpu_post_map_flush(chunk, page_start, page_end);
return 0;
}
/**
* pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
* @chunk: chunk to depopulate
* @page_start: the start page
* @page_end: the end page
*
* For each cpu, depopulate and unmap pages [@page_start,@page_end)
* from @chunk.
*
* Caller is required to call pcpu_post_unmap_tlb_flush() if not returning the
* region back to vmalloc() which will lazily flush the tlb.
*
* CONTEXT:
* pcpu_alloc_mutex.
*/
static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
int page_start, int page_end)
{
struct page **pages;
/*
* If control reaches here, there must have been at least one
* successful population attempt so the temp pages array must
* be available now.
*/
pages = pcpu_get_pages();
BUG_ON(!pages);
/* unmap and free */
pcpu_pre_unmap_flush(chunk, page_start, page_end);
pcpu_unmap_pages(chunk, pages, page_start, page_end);
pcpu_free_pages(chunk, pages, page_start, page_end);
}
static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
{
struct pcpu_chunk *chunk;
struct vm_struct **vms;
chunk = pcpu_alloc_chunk(gfp);
if (!chunk)
return NULL;
vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
pcpu_nr_groups, pcpu_atom_size);
if (!vms) {
pcpu_free_chunk(chunk);
return NULL;
}
chunk->data = vms;
chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
pcpu_stats_chunk_alloc();
trace_percpu_create_chunk(chunk->base_addr);
return chunk;
}
static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
{
if (!chunk)
return;
pcpu_stats_chunk_dealloc();
trace_percpu_destroy_chunk(chunk->base_addr);
if (chunk->data)
pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
pcpu_free_chunk(chunk);
}
static struct page *pcpu_addr_to_page(void *addr)
{
return vmalloc_to_page(addr);
}
static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
{
/* no extra restriction */
return 0;
}
/**
* pcpu_should_reclaim_chunk - determine if a chunk should go into reclaim
* @chunk: chunk of interest
*
* This is the entry point for percpu reclaim. If a chunk qualifies, it is then
* isolated and managed in separate lists at the back of pcpu_slot: sidelined
* and to_depopulate respectively. The to_depopulate list holds chunks slated
* for depopulation. They no longer contribute to pcpu_nr_empty_pop_pages once
* they are on this list. Once depopulated, they are moved onto the sidelined
* list which enables them to be pulled back in for allocation if no other chunk
* can suffice the allocation.
*/
static bool pcpu_should_reclaim_chunk(struct pcpu_chunk *chunk)
{
/* do not reclaim either the first chunk or reserved chunk */
if (chunk == pcpu_first_chunk || chunk == pcpu_reserved_chunk)
return false;
/*
* If it is isolated, it may be on the sidelined list so move it back to
* the to_depopulate list. If we hit at least 1/4 pages empty pages AND
* there is no system-wide shortage of empty pages aside from this
* chunk, move it to the to_depopulate list.
*/
return ((chunk->isolated && chunk->nr_empty_pop_pages) ||
(pcpu_nr_empty_pop_pages >
(PCPU_EMPTY_POP_PAGES_HIGH + chunk->nr_empty_pop_pages) && chunk->nr_empty_pop_pages >= chunk->nr_pages / 4));
}
/*
* Copyright (c) 1982, 1986 Regents of the University of California.
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Robert Elz at The University of Melbourne.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _LINUX_QUOTA_
#define _LINUX_QUOTA_
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/spinlock.h>
#include <linux/wait.h>
#include <linux/percpu_counter.h>
#include <linux/dqblk_xfs.h>
#include <linux/dqblk_v1.h>
#include <linux/dqblk_v2.h>
#include <linux/atomic.h>
#include <linux/uidgid.h>
#include <linux/projid.h>
#include <uapi/linux/quota.h>
#undef USRQUOTA
#undef GRPQUOTA
#undef PRJQUOTA
enum quota_type {
USRQUOTA = 0, /* element used for user quotas */
GRPQUOTA = 1, /* element used for group quotas */
PRJQUOTA = 2, /* element used for project quotas */
};
/* Masks for quota types when used as a bitmask */
#define QTYPE_MASK_USR (1 << USRQUOTA)
#define QTYPE_MASK_GRP (1 << GRPQUOTA)
#define QTYPE_MASK_PRJ (1 << PRJQUOTA)
typedef __kernel_uid32_t qid_t; /* Type in which we store ids in memory */
typedef long long qsize_t; /* Type in which we store sizes */
struct kqid { /* Type in which we store the quota identifier */
union {
kuid_t uid;
kgid_t gid;
kprojid_t projid;
};
enum quota_type type; /* USRQUOTA (uid) or GRPQUOTA (gid) or PRJQUOTA (projid) */
};
extern bool qid_eq(struct kqid left, struct kqid right);
extern bool qid_lt(struct kqid left, struct kqid right);
extern qid_t from_kqid(struct user_namespace *to, struct kqid qid);
extern qid_t from_kqid_munged(struct user_namespace *to, struct kqid qid);
extern bool qid_valid(struct kqid qid);
/**
* make_kqid - Map a user-namespace, type, qid tuple into a kqid.
* @from: User namespace that the qid is in
* @type: The type of quota
* @qid: Quota identifier
*
* Maps a user-namespace, type qid tuple into a kernel internal
* kqid, and returns that kqid.
*
* When there is no mapping defined for the user-namespace, type,
* qid tuple an invalid kqid is returned. Callers are expected to
* test for and handle invalid kqids being returned.
* Invalid kqids may be tested for using qid_valid().
*/
static inline struct kqid make_kqid(struct user_namespace *from,
enum quota_type type, qid_t qid)
{
struct kqid kqid;
kqid.type = type;
switch (type) {
case USRQUOTA:
kqid.uid = make_kuid(from, qid);
break;
case GRPQUOTA:
kqid.gid = make_kgid(from, qid);
break;
case PRJQUOTA:
kqid.projid = make_kprojid(from, qid);
break;
default:
BUG();
}
return kqid;
}
/**
* make_kqid_invalid - Explicitly make an invalid kqid
* @type: The type of quota identifier
*
* Returns an invalid kqid with the specified type.
*/
static inline struct kqid make_kqid_invalid(enum quota_type type)
{
struct kqid kqid;
kqid.type = type;
switch (type) {
case USRQUOTA:
kqid.uid = INVALID_UID;
break;
case GRPQUOTA:
kqid.gid = INVALID_GID;
break;
case PRJQUOTA:
kqid.projid = INVALID_PROJID;
break;
default:
BUG();
}
return kqid;
}
/**
* make_kqid_uid - Make a kqid from a kuid
* @uid: The kuid to make the quota identifier from
*/
static inline struct kqid make_kqid_uid(kuid_t uid)
{
struct kqid kqid;
kqid.type = USRQUOTA;
kqid.uid = uid;
return kqid;
}
/**
* make_kqid_gid - Make a kqid from a kgid
* @gid: The kgid to make the quota identifier from
*/
static inline struct kqid make_kqid_gid(kgid_t gid)
{
struct kqid kqid;
kqid.type = GRPQUOTA;
kqid.gid = gid;
return kqid;
}
/**
* make_kqid_projid - Make a kqid from a projid
* @projid: The kprojid to make the quota identifier from
*/
static inline struct kqid make_kqid_projid(kprojid_t projid)
{
struct kqid kqid;
kqid.type = PRJQUOTA;
kqid.projid = projid;
return kqid;
}
/**
* qid_has_mapping - Report if a qid maps into a user namespace.
* @ns: The user namespace to see if a value maps into.
* @qid: The kernel internal quota identifier to test.
*/
static inline bool qid_has_mapping(struct user_namespace *ns, struct kqid qid)
{
return from_kqid(ns, qid) != (qid_t) -1;
}
extern spinlock_t dq_data_lock;
/* Maximal numbers of writes for quota operation (insert/delete/update)
* (over VFS all formats) */
#define DQUOT_INIT_ALLOC max(V1_INIT_ALLOC, V2_INIT_ALLOC)
#define DQUOT_INIT_REWRITE max(V1_INIT_REWRITE, V2_INIT_REWRITE)
#define DQUOT_DEL_ALLOC max(V1_DEL_ALLOC, V2_DEL_ALLOC)
#define DQUOT_DEL_REWRITE max(V1_DEL_REWRITE, V2_DEL_REWRITE)
/*
* Data for one user/group kept in memory
*/
struct mem_dqblk {
qsize_t dqb_bhardlimit; /* absolute limit on disk blks alloc */
qsize_t dqb_bsoftlimit; /* preferred limit on disk blks */
qsize_t dqb_curspace; /* current used space */
qsize_t dqb_rsvspace; /* current reserved space for delalloc*/
qsize_t dqb_ihardlimit; /* absolute limit on allocated inodes */
qsize_t dqb_isoftlimit; /* preferred inode limit */
qsize_t dqb_curinodes; /* current # allocated inodes */
time64_t dqb_btime; /* time limit for excessive disk use */
time64_t dqb_itime; /* time limit for excessive inode use */
};
/*
* Data for one quotafile kept in memory
*/
struct quota_format_type;
struct mem_dqinfo {
struct quota_format_type *dqi_format;
int dqi_fmt_id; /* Id of the dqi_format - used when turning
* quotas on after remount RW */
struct list_head dqi_dirty_list; /* List of dirty dquots [dq_list_lock] */
unsigned long dqi_flags; /* DFQ_ flags [dq_data_lock] */
unsigned int dqi_bgrace; /* Space grace time [dq_data_lock] */
unsigned int dqi_igrace; /* Inode grace time [dq_data_lock] */
qsize_t dqi_max_spc_limit; /* Maximum space limit [static] */
qsize_t dqi_max_ino_limit; /* Maximum inode limit [static] */
void *dqi_priv;
};
struct super_block;
/* Mask for flags passed to userspace */
#define DQF_GETINFO_MASK (DQF_ROOT_SQUASH | DQF_SYS_FILE)
/* Mask for flags modifiable from userspace */
#define DQF_SETINFO_MASK DQF_ROOT_SQUASH
enum {
DQF_INFO_DIRTY_B = DQF_PRIVATE,
};
#define DQF_INFO_DIRTY (1 << DQF_INFO_DIRTY_B) /* Is info dirty? */
extern void mark_info_dirty(struct super_block *sb, int type);
static inline int info_dirty(struct mem_dqinfo *info)
{
return test_bit(DQF_INFO_DIRTY_B, &info->dqi_flags);
}
enum {
DQST_LOOKUPS,
DQST_DROPS,
DQST_READS,
DQST_WRITES,
DQST_CACHE_HITS,
DQST_ALLOC_DQUOTS,
DQST_FREE_DQUOTS,
DQST_SYNCS,
_DQST_DQSTAT_LAST
};
struct dqstats {
unsigned long stat[_DQST_DQSTAT_LAST];
struct percpu_counter counter[_DQST_DQSTAT_LAST];
};
extern struct dqstats dqstats;
static inline void dqstats_inc(unsigned int type)
{
percpu_counter_inc(&dqstats.counter[type]);
}
static inline void dqstats_dec(unsigned int type)
{
percpu_counter_dec(&dqstats.counter[type]);
}
#define DQ_MOD_B 0 /* dquot modified since read */
#define DQ_BLKS_B 1 /* uid/gid has been warned about blk limit */
#define DQ_INODES_B 2 /* uid/gid has been warned about inode limit */
#define DQ_FAKE_B 3 /* no limits only usage */
#define DQ_READ_B 4 /* dquot was read into memory */
#define DQ_ACTIVE_B 5 /* dquot is active (dquot_release not called) */
#define DQ_RELEASING_B 6 /* dquot is in releasing_dquots list waiting
* to be cleaned up */
#define DQ_LASTSET_B 7 /* Following 6 bits (see QIF_) are reserved\
* for the mask of entries set via SETQUOTA\
* quotactl. They are set under dq_data_lock\
* and the quota format handling dquot can\
* clear them when it sees fit. */
struct dquot {
struct hlist_node dq_hash; /* Hash list in memory [dq_list_lock] */
struct list_head dq_inuse; /* List of all quotas [dq_list_lock] */
struct list_head dq_free; /* Free list element [dq_list_lock] */
struct list_head dq_dirty; /* List of dirty dquots [dq_list_lock] */
struct mutex dq_lock; /* dquot IO lock */
spinlock_t dq_dqb_lock; /* Lock protecting dq_dqb changes */
atomic_t dq_count; /* Use count */
struct super_block *dq_sb; /* superblock this applies to */
struct kqid dq_id; /* ID this applies to (uid, gid, projid) */
loff_t dq_off; /* Offset of dquot on disk [dq_lock, stable once set] */
unsigned long dq_flags; /* See DQ_* */
struct mem_dqblk dq_dqb; /* Diskquota usage [dq_dqb_lock] */
};
/* Operations which must be implemented by each quota format */
struct quota_format_ops {
int (*check_quota_file)(struct super_block *sb, int type); /* Detect whether file is in our format */
int (*read_file_info)(struct super_block *sb, int type); /* Read main info about file - called on quotaon() */
int (*write_file_info)(struct super_block *sb, int type); /* Write main info about file */
int (*free_file_info)(struct super_block *sb, int type); /* Called on quotaoff() */
int (*read_dqblk)(struct dquot *dquot); /* Read structure for one user */
int (*commit_dqblk)(struct dquot *dquot); /* Write structure for one user */
int (*release_dqblk)(struct dquot *dquot); /* Called when last reference to dquot is being dropped */
int (*get_next_id)(struct super_block *sb, struct kqid *qid); /* Get next ID with existing structure in the quota file */
};
/* Operations working with dquots */
struct dquot_operations {
int (*write_dquot) (struct dquot *); /* Ordinary dquot write */
struct dquot *(*alloc_dquot)(struct super_block *, int); /* Allocate memory for new dquot */
void (*destroy_dquot)(struct dquot *); /* Free memory for dquot */
int (*acquire_dquot) (struct dquot *); /* Quota is going to be created on disk */
int (*release_dquot) (struct dquot *); /* Quota is going to be deleted from disk */
int (*mark_dirty) (struct dquot *); /* Dquot is marked dirty */
int (*write_info) (struct super_block *, int); /* Write of quota "superblock" */
/* get reserved quota for delayed alloc, value returned is managed by
* quota code only */
qsize_t *(*get_reserved_space) (struct inode *);
int (*get_projid) (struct inode *, kprojid_t *);/* Get project ID */
/* Get number of inodes that were charged for a given inode */
int (*get_inode_usage) (struct inode *, qsize_t *);
/* Get next ID with active quota structure */
int (*get_next_id) (struct super_block *sb, struct kqid *qid);
};
struct path;
/* Structure for communicating via ->get_dqblk() & ->set_dqblk() */
struct qc_dqblk {
int d_fieldmask; /* mask of fields to change in ->set_dqblk() */
u64 d_spc_hardlimit; /* absolute limit on used space */
u64 d_spc_softlimit; /* preferred limit on used space */
u64 d_ino_hardlimit; /* maximum # allocated inodes */
u64 d_ino_softlimit; /* preferred inode limit */
u64 d_space; /* Space owned by the user */
u64 d_ino_count; /* # inodes owned by the user */
s64 d_ino_timer; /* zero if within inode limits */
/* if not, we refuse service */
s64 d_spc_timer; /* similar to above; for space */
int d_ino_warns; /* # warnings issued wrt num inodes */
int d_spc_warns; /* # warnings issued wrt used space */
u64 d_rt_spc_hardlimit; /* absolute limit on realtime space */
u64 d_rt_spc_softlimit; /* preferred limit on RT space */
u64 d_rt_space; /* realtime space owned */
s64 d_rt_spc_timer; /* similar to above; for RT space */
int d_rt_spc_warns; /* # warnings issued wrt RT space */
};
/*
* Field specifiers for ->set_dqblk() in struct qc_dqblk and also for
* ->set_info() in struct qc_info
*/
#define QC_INO_SOFT (1<<0)
#define QC_INO_HARD (1<<1)
#define QC_SPC_SOFT (1<<2)
#define QC_SPC_HARD (1<<3)
#define QC_RT_SPC_SOFT (1<<4)
#define QC_RT_SPC_HARD (1<<5)
#define QC_LIMIT_MASK (QC_INO_SOFT | QC_INO_HARD | QC_SPC_SOFT | QC_SPC_HARD | \
QC_RT_SPC_SOFT | QC_RT_SPC_HARD)
#define QC_SPC_TIMER (1<<6)
#define QC_INO_TIMER (1<<7)
#define QC_RT_SPC_TIMER (1<<8)
#define QC_TIMER_MASK (QC_SPC_TIMER | QC_INO_TIMER | QC_RT_SPC_TIMER)
#define QC_SPC_WARNS (1<<9)
#define QC_INO_WARNS (1<<10)
#define QC_RT_SPC_WARNS (1<<11)
#define QC_WARNS_MASK (QC_SPC_WARNS | QC_INO_WARNS | QC_RT_SPC_WARNS)
#define QC_SPACE (1<<12)
#define QC_INO_COUNT (1<<13)
#define QC_RT_SPACE (1<<14)
#define QC_ACCT_MASK (QC_SPACE | QC_INO_COUNT | QC_RT_SPACE)
#define QC_FLAGS (1<<15)
#define QCI_SYSFILE (1 << 0) /* Quota file is hidden from userspace */
#define QCI_ROOT_SQUASH (1 << 1) /* Root squash turned on */
#define QCI_ACCT_ENABLED (1 << 2) /* Quota accounting enabled */
#define QCI_LIMITS_ENFORCED (1 << 3) /* Quota limits enforced */
/* Structures for communicating via ->get_state */
struct qc_type_state {
unsigned int flags; /* Flags QCI_* */
unsigned int spc_timelimit; /* Time after which space softlimit is
* enforced */
unsigned int ino_timelimit; /* Ditto for inode softlimit */
unsigned int rt_spc_timelimit; /* Ditto for real-time space */
unsigned int spc_warnlimit; /* Limit for number of space warnings */
unsigned int ino_warnlimit; /* Ditto for inodes */
unsigned int rt_spc_warnlimit; /* Ditto for real-time space */
unsigned long long ino; /* Inode number of quota file */
blkcnt_t blocks; /* Number of 512-byte blocks in the file */
blkcnt_t nextents; /* Number of extents in the file */
};
struct qc_state {
unsigned int s_incoredqs; /* Number of dquots in core */
struct qc_type_state s_state[MAXQUOTAS]; /* Per quota type information */
};
/* Structure for communicating via ->set_info */
struct qc_info {
int i_fieldmask; /* mask of fields to change in ->set_info() */
unsigned int i_flags; /* Flags QCI_* */
unsigned int i_spc_timelimit; /* Time after which space softlimit is
* enforced */
unsigned int i_ino_timelimit; /* Ditto for inode softlimit */
unsigned int i_rt_spc_timelimit;/* Ditto for real-time space */
unsigned int i_spc_warnlimit; /* Limit for number of space warnings */
unsigned int i_ino_warnlimit; /* Limit for number of inode warnings */
unsigned int i_rt_spc_warnlimit; /* Ditto for real-time space */
};
/* Operations handling requests from userspace */
struct quotactl_ops {
int (*quota_on)(struct super_block *, int, int, const struct path *);
int (*quota_off)(struct super_block *, int);
int (*quota_enable)(struct super_block *, unsigned int);
int (*quota_disable)(struct super_block *, unsigned int);
int (*quota_sync)(struct super_block *, int);
int (*set_info)(struct super_block *, int, struct qc_info *);
int (*get_dqblk)(struct super_block *, struct kqid, struct qc_dqblk *);
int (*get_nextdqblk)(struct super_block *, struct kqid *,
struct qc_dqblk *);
int (*set_dqblk)(struct super_block *, struct kqid, struct qc_dqblk *);
int (*get_state)(struct super_block *, struct qc_state *);
int (*rm_xquota)(struct super_block *, unsigned int);
};
struct quota_format_type {
int qf_fmt_id; /* Quota format id */
const struct quota_format_ops *qf_ops; /* Operations of format */
struct module *qf_owner; /* Module implementing quota format */
struct quota_format_type *qf_next;
};
/**
* Quota state flags - they come in three flavors - for users, groups and projects.
*
* Actual typed flags layout:
* USRQUOTA GRPQUOTA PRJQUOTA
* DQUOT_USAGE_ENABLED 0x0001 0x0002 0x0004
* DQUOT_LIMITS_ENABLED 0x0008 0x0010 0x0020
* DQUOT_SUSPENDED 0x0040 0x0080 0x0100
*
* Following bits are used for non-typed flags:
* DQUOT_QUOTA_SYS_FILE 0x0200
* DQUOT_NEGATIVE_USAGE 0x0400
* DQUOT_NOLIST_DIRTY 0x0800
*/
enum {
_DQUOT_USAGE_ENABLED = 0, /* Track disk usage for users */
_DQUOT_LIMITS_ENABLED, /* Enforce quota limits for users */
_DQUOT_SUSPENDED, /* User diskquotas are off, but
* we have necessary info in
* memory to turn them on */
_DQUOT_STATE_FLAGS
};
#define DQUOT_USAGE_ENABLED (1 << _DQUOT_USAGE_ENABLED * MAXQUOTAS)
#define DQUOT_LIMITS_ENABLED (1 << _DQUOT_LIMITS_ENABLED * MAXQUOTAS)
#define DQUOT_SUSPENDED (1 << _DQUOT_SUSPENDED * MAXQUOTAS)
#define DQUOT_STATE_FLAGS (DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED | \
DQUOT_SUSPENDED)
/* Other quota flags */
#define DQUOT_STATE_LAST (_DQUOT_STATE_FLAGS * MAXQUOTAS)
#define DQUOT_QUOTA_SYS_FILE (1 << DQUOT_STATE_LAST)
/* Quota file is a special
* system file and user cannot
* touch it. Filesystem is
* responsible for setting
* S_NOQUOTA, S_NOATIME flags
*/
#define DQUOT_NEGATIVE_USAGE (1 << (DQUOT_STATE_LAST + 1))
/* Allow negative quota usage */
/* Do not track dirty dquots in a list */
#define DQUOT_NOLIST_DIRTY (1 << (DQUOT_STATE_LAST + 2))
static inline unsigned int dquot_state_flag(unsigned int flags, int type)
{
return flags << type;
}
static inline unsigned int dquot_generic_flag(unsigned int flags, int type)
{
return (flags >> type) & DQUOT_STATE_FLAGS;
}
/* Bitmap of quota types where flag is set in flags */
static __always_inline unsigned dquot_state_types(unsigned flags, unsigned flag)
{
BUILD_BUG_ON_NOT_POWER_OF_2(flag);
return (flags / flag) & ((1 << MAXQUOTAS) - 1);
}
#ifdef CONFIG_QUOTA_NETLINK_INTERFACE
extern void quota_send_warning(struct kqid qid, dev_t dev,
const char warntype);
#else
static inline void quota_send_warning(struct kqid qid, dev_t dev,
const char warntype)
{
return;
}
#endif /* CONFIG_QUOTA_NETLINK_INTERFACE */
struct quota_info {
unsigned int flags; /* Flags for diskquotas on this device */
struct rw_semaphore dqio_sem; /* Lock quota file while I/O in progress */
struct inode *files[MAXQUOTAS]; /* inodes of quotafiles */
struct mem_dqinfo info[MAXQUOTAS]; /* Information for each quota type */
const struct quota_format_ops *ops[MAXQUOTAS]; /* Operations for each type */
};
int register_quota_format(struct quota_format_type *fmt);
void unregister_quota_format(struct quota_format_type *fmt);
struct quota_module_name {
int qm_fmt_id;
char *qm_mod_name;
};
#define INIT_QUOTA_MODULE_NAMES {\
{QFMT_VFS_OLD, "quota_v1"},\
{QFMT_VFS_V0, "quota_v2"},\
{QFMT_VFS_V1, "quota_v2"},\
{0, NULL}}
#endif /* _QUOTA_ */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCATTERLIST_H
#define _LINUX_SCATTERLIST_H
#include <linux/string.h>
#include <linux/types.h>
#include <linux/bug.h>
#include <linux/mm.h>
#include <asm/io.h>
struct scatterlist {
unsigned long page_link;
unsigned int offset;
unsigned int length;
dma_addr_t dma_address;
#ifdef CONFIG_NEED_SG_DMA_LENGTH
unsigned int dma_length;
#endif
#ifdef CONFIG_NEED_SG_DMA_FLAGS
unsigned int dma_flags;
#endif
};
/*
* These macros should be used after a dma_map_sg call has been done
* to get bus addresses of each of the SG entries and their lengths.
* You should only work with the number of sg entries dma_map_sg
* returns, or alternatively stop on the first sg_dma_len(sg) which
* is 0.
*/
#define sg_dma_address(sg) ((sg)->dma_address)
#ifdef CONFIG_NEED_SG_DMA_LENGTH
#define sg_dma_len(sg) ((sg)->dma_length)
#else
#define sg_dma_len(sg) ((sg)->length)
#endif
struct sg_table {
struct scatterlist *sgl; /* the list */
unsigned int nents; /* number of mapped entries */
unsigned int orig_nents; /* original size of list */
};
struct sg_append_table {
struct sg_table sgt; /* The scatter list table */
struct scatterlist *prv; /* last populated sge in the table */
unsigned int total_nents; /* Total entries in the table */
};
/*
* Notes on SG table design.
*
* We use the unsigned long page_link field in the scatterlist struct to place
* the page pointer AND encode information about the sg table as well. The two
* lower bits are reserved for this information.
*
* If bit 0 is set, then the page_link contains a pointer to the next sg
* table list. Otherwise the next entry is at sg + 1.
*
* If bit 1 is set, then this sg entry is the last element in a list.
*
* See sg_next().
*
*/
#define SG_CHAIN 0x01UL
#define SG_END 0x02UL
/*
* We overload the LSB of the page pointer to indicate whether it's
* a valid sg entry, or whether it points to the start of a new scatterlist.
* Those low bits are there for everyone! (thanks mason :-)
*/
#define SG_PAGE_LINK_MASK (SG_CHAIN | SG_END)
static inline unsigned int __sg_flags(struct scatterlist *sg)
{
return sg->page_link & SG_PAGE_LINK_MASK;
}
static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg)
{
return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);
}
static inline bool sg_is_chain(struct scatterlist *sg)
{
return __sg_flags(sg) & SG_CHAIN;
}
static inline bool sg_is_last(struct scatterlist *sg)
{
return __sg_flags(sg) & SG_END;
}
/**
* sg_assign_page - Assign a given page to an SG entry
* @sg: SG entry
* @page: The page
*
* Description:
* Assign page to sg entry. Also see sg_set_page(), the most commonly used
* variant.
*
**/
static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
{
unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);
/*
* In order for the low bit stealing approach to work, pages
* must be aligned at a 32-bit boundary as a minimum.
*/
BUG_ON((unsigned long)page & SG_PAGE_LINK_MASK);
#ifdef CONFIG_DEBUG_SG
BUG_ON(sg_is_chain(sg));
#endif
sg->page_link = page_link | (unsigned long) page;
}
/**
* sg_set_page - Set sg entry to point at given page
* @sg: SG entry
* @page: The page
* @len: Length of data
* @offset: Offset into page
*
* Description:
* Use this function to set an sg entry pointing at a page, never assign
* the page directly. We encode sg table information in the lower bits
* of the page pointer. See sg_page() for looking up the page belonging
* to an sg entry.
*
**/
static inline void sg_set_page(struct scatterlist *sg, struct page *page,
unsigned int len, unsigned int offset)
{
sg_assign_page(sg, page);
sg->offset = offset;
sg->length = len;
}
/**
* sg_set_folio - Set sg entry to point at given folio
* @sg: SG entry
* @folio: The folio
* @len: Length of data
* @offset: Offset into folio
*
* Description:
* Use this function to set an sg entry pointing at a folio, never assign
* the folio directly. We encode sg table information in the lower bits
* of the folio pointer. See sg_page() for looking up the page belonging
* to an sg entry.
*
**/
static inline void sg_set_folio(struct scatterlist *sg, struct folio *folio,
size_t len, size_t offset)
{
WARN_ON_ONCE(len > UINT_MAX);
WARN_ON_ONCE(offset > UINT_MAX);
sg_assign_page(sg, &folio->page);
sg->offset = offset;
sg->length = len;
}
static inline struct page *sg_page(struct scatterlist *sg)
{
#ifdef CONFIG_DEBUG_SG
BUG_ON(sg_is_chain(sg));
#endif
return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);
}
/**
* sg_set_buf - Set sg entry to point at given data
* @sg: SG entry
* @buf: Data
* @buflen: Data length
*
**/
static inline void sg_set_buf(struct scatterlist *sg, const void *buf,
unsigned int buflen)
{
#ifdef CONFIG_DEBUG_SG
BUG_ON(!virt_addr_valid(buf));
#endif
sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
}
/*
* Loop over each sg element, following the pointer to a new list if necessary
*/
#define for_each_sg(sglist, sg, nr, __i) \
for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
/*
* Loop over each sg element in the given sg_table object.
*/
#define for_each_sgtable_sg(sgt, sg, i) \
for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
/*
* Loop over each sg element in the given *DMA mapped* sg_table object.
* Please use sg_dma_address(sg) and sg_dma_len(sg) to extract DMA addresses
* of the each element.
*/
#define for_each_sgtable_dma_sg(sgt, sg, i) \
for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
static inline void __sg_chain(struct scatterlist *chain_sg,
struct scatterlist *sgl)
{
/*
* offset and length are unused for chain entry. Clear them.
*/
chain_sg->offset = 0;
chain_sg->length = 0;
/*
* Set lowest bit to indicate a link pointer, and make sure to clear
* the termination bit if it happens to be set.
*/
chain_sg->page_link = ((unsigned long) sgl | SG_CHAIN) & ~SG_END;
}
/**
* sg_chain - Chain two sglists together
* @prv: First scatterlist
* @prv_nents: Number of entries in prv
* @sgl: Second scatterlist
*
* Description:
* Links @prv@ and @sgl@ together, to form a longer scatterlist.
*
**/
static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents,
struct scatterlist *sgl)
{
__sg_chain(&prv[prv_nents - 1], sgl);
}
/**
* sg_mark_end - Mark the end of the scatterlist
* @sg: SG entryScatterlist
*
* Description:
* Marks the passed in sg entry as the termination point for the sg
* table. A call to sg_next() on this entry will return NULL.
*
**/
static inline void sg_mark_end(struct scatterlist *sg)
{
/*
* Set termination bit, clear potential chain bit
*/
sg->page_link |= SG_END;
sg->page_link &= ~SG_CHAIN;
}
/**
* sg_unmark_end - Undo setting the end of the scatterlist
* @sg: SG entryScatterlist
*
* Description:
* Removes the termination marker from the given entry of the scatterlist.
*
**/
static inline void sg_unmark_end(struct scatterlist *sg)
{
sg->page_link &= ~SG_END;
}
/*
* One 64-bit architectures there is a 4-byte padding in struct scatterlist
* (assuming also CONFIG_NEED_SG_DMA_LENGTH is set). Use this padding for DMA
* flags bits to indicate when a specific dma address is a bus address or the
* buffer may have been bounced via SWIOTLB.
*/
#ifdef CONFIG_NEED_SG_DMA_FLAGS
#define SG_DMA_BUS_ADDRESS (1 << 0)
#define SG_DMA_SWIOTLB (1 << 1)
/**
* sg_dma_is_bus_address - Return whether a given segment was marked
* as a bus address
* @sg: SG entry
*
* Description:
* Returns true if sg_dma_mark_bus_address() has been called on
* this segment.
**/
static inline bool sg_dma_is_bus_address(struct scatterlist *sg)
{
return sg->dma_flags & SG_DMA_BUS_ADDRESS;
}
/**
* sg_dma_mark_bus_address - Mark the scatterlist entry as a bus address
* @sg: SG entry
*
* Description:
* Marks the passed in sg entry to indicate that the dma_address is
* a bus address and doesn't need to be unmapped. This should only be
* used by dma_map_sg() implementations to mark bus addresses
* so they can be properly cleaned up in dma_unmap_sg().
**/
static inline void sg_dma_mark_bus_address(struct scatterlist *sg)
{
sg->dma_flags |= SG_DMA_BUS_ADDRESS;
}
/**
* sg_unmark_bus_address - Unmark the scatterlist entry as a bus address
* @sg: SG entry
*
* Description:
* Clears the bus address mark.
**/
static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)
{
sg->dma_flags &= ~SG_DMA_BUS_ADDRESS;
}
/**
* sg_dma_is_swiotlb - Return whether the scatterlist was marked for SWIOTLB
* bouncing
* @sg: SG entry
*
* Description:
* Returns true if the scatterlist was marked for SWIOTLB bouncing. Not all
* elements may have been bounced, so the caller would have to check
* individual SG entries with is_swiotlb_buffer().
*/
static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)
{
return sg->dma_flags & SG_DMA_SWIOTLB;
}
/**
* sg_dma_mark_swiotlb - Mark the scatterlist for SWIOTLB bouncing
* @sg: SG entry
*
* Description:
* Marks a a scatterlist for SWIOTLB bounce. Not all SG entries may be
* bounced.
*/
static inline void sg_dma_mark_swiotlb(struct scatterlist *sg)
{
sg->dma_flags |= SG_DMA_SWIOTLB;
}
#else
static inline bool sg_dma_is_bus_address(struct scatterlist *sg)
{
return false;
}
static inline void sg_dma_mark_bus_address(struct scatterlist *sg)
{
}
static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)
{
}
static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)
{
return false;
}
static inline void sg_dma_mark_swiotlb(struct scatterlist *sg)
{
}
#endif /* CONFIG_NEED_SG_DMA_FLAGS */
/**
* sg_phys - Return physical address of an sg entry
* @sg: SG entry
*
* Description:
* This calls page_to_phys() on the page in this sg entry, and adds the
* sg offset. The caller must know that it is legal to call page_to_phys()
* on the sg page.
*
**/
static inline dma_addr_t sg_phys(struct scatterlist *sg)
{
return page_to_phys(sg_page(sg)) + sg->offset;
}
/**
* sg_virt - Return virtual address of an sg entry
* @sg: SG entry
*
* Description:
* This calls page_address() on the page in this sg entry, and adds the
* sg offset. The caller must know that the sg page has a valid virtual
* mapping.
*
**/
static inline void *sg_virt(struct scatterlist *sg)
{
return page_address(sg_page(sg)) + sg->offset;
}
/**
* sg_init_marker - Initialize markers in sg table
* @sgl: The SG table
* @nents: Number of entries in table
*
**/
static inline void sg_init_marker(struct scatterlist *sgl,
unsigned int nents)
{
sg_mark_end(&sgl[nents - 1]);
}
int sg_nents(struct scatterlist *sg);
int sg_nents_for_len(struct scatterlist *sg, u64 len);
struct scatterlist *sg_next(struct scatterlist *);
struct scatterlist *sg_last(struct scatterlist *s, unsigned int);
void sg_init_table(struct scatterlist *, unsigned int);
void sg_init_one(struct scatterlist *, const void *, unsigned int);
int sg_split(struct scatterlist *in, const int in_mapped_nents,
const off_t skip, const int nb_splits,
const size_t *split_sizes,
struct scatterlist **out, int *out_mapped_nents,
gfp_t gfp_mask);
typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
void __sg_free_table(struct sg_table *, unsigned int, unsigned int,
sg_free_fn *, unsigned int);
void sg_free_table(struct sg_table *);
void sg_free_append_table(struct sg_append_table *sgt);
int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
struct scatterlist *, unsigned int, gfp_t, sg_alloc_fn *);
int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
int sg_alloc_append_table_from_pages(struct sg_append_table *sgt,
struct page **pages, unsigned int n_pages,
unsigned int offset, unsigned long size,
unsigned int max_segment,
unsigned int left_pages, gfp_t gfp_mask);
int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages,
unsigned int n_pages, unsigned int offset,
unsigned long size,
unsigned int max_segment, gfp_t gfp_mask);
/**
* sg_alloc_table_from_pages - Allocate and initialize an sg table from
* an array of pages
* @sgt: The sg table header to use
* @pages: Pointer to an array of page pointers
* @n_pages: Number of pages in the pages array
* @offset: Offset from start of the first page to the start of a buffer
* @size: Number of valid bytes in the buffer (after offset)
* @gfp_mask: GFP allocation mask
*
* Description:
* Allocate and initialize an sg table from a list of pages. Contiguous
* ranges of the pages are squashed into a single scatterlist node. A user
* may provide an offset at a start and a size of valid data in a buffer
* specified by the page array. The returned sg table is released by
* sg_free_table.
*
* Returns:
* 0 on success, negative error on failure
*/
static inline int sg_alloc_table_from_pages(struct sg_table *sgt,
struct page **pages,
unsigned int n_pages,
unsigned int offset,
unsigned long size, gfp_t gfp_mask)
{
return sg_alloc_table_from_pages_segment(sgt, pages, n_pages, offset,
size, UINT_MAX, gfp_mask);
}
#ifdef CONFIG_SGL_ALLOC
struct scatterlist *sgl_alloc_order(unsigned long long length,
unsigned int order, bool chainable,
gfp_t gfp, unsigned int *nent_p);
struct scatterlist *sgl_alloc(unsigned long long length, gfp_t gfp,
unsigned int *nent_p);
void sgl_free_n_order(struct scatterlist *sgl, int nents, int order);
void sgl_free_order(struct scatterlist *sgl, int order);
void sgl_free(struct scatterlist *sgl);
#endif /* CONFIG_SGL_ALLOC */
size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
size_t buflen, off_t skip, bool to_buffer);
size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
const void *buf, size_t buflen);
size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
void *buf, size_t buflen);
size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
const void *buf, size_t buflen, off_t skip);
size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
void *buf, size_t buflen, off_t skip);
size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
size_t buflen, off_t skip);
/*
* Maximum number of entries that will be allocated in one piece, if
* a list larger than this is required then chaining will be utilized.
*/
#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist))
/*
* The maximum number of SG segments that we will put inside a
* scatterlist (unless chaining is used). Should ideally fit inside a
* single page, to avoid a higher order allocation. We could define this
* to SG_MAX_SINGLE_ALLOC to pack correctly at the highest order. The
* minimum value is 32
*/
#define SG_CHUNK_SIZE 128
/*
* Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit
* is totally arbitrary, a setting of 2048 will get you at least 8mb ios.
*/
#ifdef CONFIG_ARCH_NO_SG_CHAIN
#define SG_MAX_SEGMENTS SG_CHUNK_SIZE
#else
#define SG_MAX_SEGMENTS 2048
#endif
#ifdef CONFIG_SG_POOL
void sg_free_table_chained(struct sg_table *table,
unsigned nents_first_chunk);
int sg_alloc_table_chained(struct sg_table *table, int nents,
struct scatterlist *first_chunk,
unsigned nents_first_chunk);
#endif
/*
* sg page iterator
*
* Iterates over sg entries page-by-page. On each successful iteration, you
* can call sg_page_iter_page(@piter) to get the current page.
* @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
* the page's page offset within the sg. The iteration will stop either when a
* maximum number of sg entries was reached or a terminating sg
* (sg_last(sg) == true) was reached.
*/
struct sg_page_iter {
struct scatterlist *sg; /* sg holding the page */
unsigned int sg_pgoffset; /* page offset within the sg */
/* these are internal states, keep away */
unsigned int __nents; /* remaining sg entries */
int __pg_advance; /* nr pages to advance at the
* next step */
};
/*
* sg page iterator for DMA addresses
*
* This is the same as sg_page_iter however you can call
* sg_page_iter_dma_address(@dma_iter) to get the page's DMA
* address. sg_page_iter_page() cannot be called on this iterator.
*/
struct sg_dma_page_iter {
struct sg_page_iter base;
};
bool __sg_page_iter_next(struct sg_page_iter *piter);
bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter);
void __sg_page_iter_start(struct sg_page_iter *piter,
struct scatterlist *sglist, unsigned int nents,
unsigned long pgoffset);
/**
* sg_page_iter_page - get the current page held by the page iterator
* @piter: page iterator holding the page
*/
static inline struct page *sg_page_iter_page(struct sg_page_iter *piter)
{
return nth_page(sg_page(piter->sg), piter->sg_pgoffset);
}
/**
* sg_page_iter_dma_address - get the dma address of the current page held by
* the page iterator.
* @dma_iter: page iterator holding the page
*/
static inline dma_addr_t
sg_page_iter_dma_address(struct sg_dma_page_iter *dma_iter)
{
return sg_dma_address(dma_iter->base.sg) +
(dma_iter->base.sg_pgoffset << PAGE_SHIFT);
}
/**
* for_each_sg_page - iterate over the pages of the given sg list
* @sglist: sglist to iterate over
* @piter: page iterator to hold current page, sg, sg_pgoffset
* @nents: maximum number of sg entries to iterate over
* @pgoffset: starting page offset (in pages)
*
* Callers may use sg_page_iter_page() to get each page pointer.
* In each loop it operates on PAGE_SIZE unit.
*/
#define for_each_sg_page(sglist, piter, nents, pgoffset) \
for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \
__sg_page_iter_next(piter);)
/**
* for_each_sg_dma_page - iterate over the pages of the given sg list
* @sglist: sglist to iterate over
* @dma_iter: DMA page iterator to hold current page
* @dma_nents: maximum number of sg entries to iterate over, this is the value
* returned from dma_map_sg
* @pgoffset: starting page offset (in pages)
*
* Callers may use sg_page_iter_dma_address() to get each page's DMA address.
* In each loop it operates on PAGE_SIZE unit.
*/
#define for_each_sg_dma_page(sglist, dma_iter, dma_nents, pgoffset) \
for (__sg_page_iter_start(&(dma_iter)->base, sglist, dma_nents, \
pgoffset); \
__sg_page_iter_dma_next(dma_iter);)
/**
* for_each_sgtable_page - iterate over all pages in the sg_table object
* @sgt: sg_table object to iterate over
* @piter: page iterator to hold current page
* @pgoffset: starting page offset (in pages)
*
* Iterates over the all memory pages in the buffer described by
* a scatterlist stored in the given sg_table object.
* See also for_each_sg_page(). In each loop it operates on PAGE_SIZE unit.
*/
#define for_each_sgtable_page(sgt, piter, pgoffset) \
for_each_sg_page((sgt)->sgl, piter, (sgt)->orig_nents, pgoffset)
/**
* for_each_sgtable_dma_page - iterate over the DMA mapped sg_table object
* @sgt: sg_table object to iterate over
* @dma_iter: DMA page iterator to hold current page
* @pgoffset: starting page offset (in pages)
*
* Iterates over the all DMA mapped pages in the buffer described by
* a scatterlist stored in the given sg_table object.
* See also for_each_sg_dma_page(). In each loop it operates on PAGE_SIZE
* unit.
*/
#define for_each_sgtable_dma_page(sgt, dma_iter, pgoffset) \
for_each_sg_dma_page((sgt)->sgl, dma_iter, (sgt)->nents, pgoffset)
/*
* Mapping sg iterator
*
* Iterates over sg entries mapping page-by-page. On each successful
* iteration, @miter->page points to the mapped page and
* @miter->length bytes of data can be accessed at @miter->addr. As
* long as an iteration is enclosed between start and stop, the user
* is free to choose control structure and when to stop.
*
* @miter->consumed is set to @miter->length on each iteration. It
* can be adjusted if the user can't consume all the bytes in one go.
* Also, a stopped iteration can be resumed by calling next on it.
* This is useful when iteration needs to release all resources and
* continue later (e.g. at the next interrupt).
*/
#define SG_MITER_ATOMIC (1 << 0) /* use kmap_atomic */
#define SG_MITER_TO_SG (1 << 1) /* flush back to phys on unmap */
#define SG_MITER_FROM_SG (1 << 2) /* nop */
struct sg_mapping_iter {
/* the following three fields can be accessed directly */
struct page *page; /* currently mapped page */
void *addr; /* pointer to the mapped area */
size_t length; /* length of the mapped area */
size_t consumed; /* number of consumed bytes */
struct sg_page_iter piter; /* page iterator */
/* these are internal states, keep away */
unsigned int __offset; /* offset within page */
unsigned int __remaining; /* remaining bytes on page */
unsigned int __flags;
};
void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
unsigned int nents, unsigned int flags);
bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset);
bool sg_miter_next(struct sg_mapping_iter *miter);
void sg_miter_stop(struct sg_mapping_iter *miter);
#endif /* _LINUX_SCATTERLIST_H */
// SPDX-License-Identifier: GPL-2.0
/* dvb-usb-remote.c is part of the DVB USB library.
*
* Copyright (C) 2004-6 Patrick Boettcher (patrick.boettcher@posteo.de)
* see dvb-usb-init.c for copyright information.
*
* This file contains functions for initializing the input-device and for handling remote-control-queries.
*/
#include "dvb-usb-common.h"
#include <linux/usb/input.h>
static unsigned int
legacy_dvb_usb_get_keymap_index(const struct input_keymap_entry *ke,
struct rc_map_table *keymap,
unsigned int keymap_size)
{
unsigned int index;
unsigned int scancode;
if (ke->flags & INPUT_KEYMAP_BY_INDEX) {
index = ke->index;
} else {
if (input_scancode_to_scalar(ke, &scancode))
return keymap_size;
/* See if we can match the raw key code. */
for (index = 0; index < keymap_size; index++) if (keymap[index].scancode == scancode)
break;
/* See if there is an unused hole in the map */
if (index >= keymap_size) { for (index = 0; index < keymap_size; index++) { if (keymap[index].keycode == KEY_RESERVED ||
keymap[index].keycode == KEY_UNKNOWN) {
break;
}
}
}
}
return index;
}
static int legacy_dvb_usb_getkeycode(struct input_dev *dev,
struct input_keymap_entry *ke)
{
struct dvb_usb_device *d = input_get_drvdata(dev);
struct rc_map_table *keymap = d->props.rc.legacy.rc_map_table;
unsigned int keymap_size = d->props.rc.legacy.rc_map_size;
unsigned int index;
index = legacy_dvb_usb_get_keymap_index(ke, keymap, keymap_size);
if (index >= keymap_size)
return -EINVAL;
ke->keycode = keymap[index].keycode;
if (ke->keycode == KEY_UNKNOWN)
ke->keycode = KEY_RESERVED;
ke->len = sizeof(keymap[index].scancode);
memcpy(&ke->scancode, &keymap[index].scancode, ke->len);
ke->index = index;
return 0;
}
static int legacy_dvb_usb_setkeycode(struct input_dev *dev,
const struct input_keymap_entry *ke,
unsigned int *old_keycode)
{
struct dvb_usb_device *d = input_get_drvdata(dev);
struct rc_map_table *keymap = d->props.rc.legacy.rc_map_table;
unsigned int keymap_size = d->props.rc.legacy.rc_map_size;
unsigned int index;
index = legacy_dvb_usb_get_keymap_index(ke, keymap, keymap_size);
/*
* FIXME: Currently, it is not possible to increase the size of
* scancode table. For it to happen, one possibility
* would be to allocate a table with key_map_size + 1,
* copying data, appending the new key on it, and freeing
* the old one - or maybe just allocating some spare space
*/
if (index >= keymap_size)
return -EINVAL;
*old_keycode = keymap[index].keycode;
keymap->keycode = ke->keycode;
__set_bit(ke->keycode, dev->keybit);
if (*old_keycode != KEY_RESERVED) {
__clear_bit(*old_keycode, dev->keybit); for (index = 0; index < keymap_size; index++) { if (keymap[index].keycode == *old_keycode) { __set_bit(*old_keycode, dev->keybit);
break;
}
}
}
return 0;
}
/* Remote-control poll function - called every dib->rc_query_interval ms to see
* whether the remote control has received anything.
*
* TODO: Fix the repeat rate of the input device.
*/
static void legacy_dvb_usb_read_remote_control(struct work_struct *work)
{
struct dvb_usb_device *d =
container_of(work, struct dvb_usb_device, rc_query_work.work);
u32 event;
int state;
/* TODO: need a lock here. We can simply skip checking for the remote control
if we're busy. */
/* when the parameter has been set to 1 via sysfs while the driver was running */
if (dvb_usb_disable_rc_polling)
return;
if (d->props.rc.legacy.rc_query(d,&event,&state)) {
err("error while querying for an remote control event.");
goto schedule;
}
switch (state) {
case REMOTE_NO_KEY_PRESSED:
break;
case REMOTE_KEY_PRESSED:
deb_rc("key pressed\n");
d->last_event = event;
input_event(d->input_dev, EV_KEY, event, 1);
input_sync(d->input_dev);
input_event(d->input_dev, EV_KEY, d->last_event, 0);
input_sync(d->input_dev);
break;
case REMOTE_KEY_REPEAT:
deb_rc("key repeated\n");
input_event(d->input_dev, EV_KEY, event, 1);
input_sync(d->input_dev);
input_event(d->input_dev, EV_KEY, d->last_event, 0);
input_sync(d->input_dev);
break;
default:
break;
}
/* improved repeat handling ???
switch (state) {
case REMOTE_NO_KEY_PRESSED:
deb_rc("NO KEY PRESSED\n");
if (d->last_state != REMOTE_NO_KEY_PRESSED) {
deb_rc("releasing event %d\n",d->last_event);
input_event(d->rc_input_dev, EV_KEY, d->last_event, 0);
input_sync(d->rc_input_dev);
}
d->last_state = REMOTE_NO_KEY_PRESSED;
d->last_event = 0;
break;
case REMOTE_KEY_PRESSED:
deb_rc("KEY PRESSED\n");
deb_rc("pressing event %d\n",event);
input_event(d->rc_input_dev, EV_KEY, event, 1);
input_sync(d->rc_input_dev);
d->last_event = event;
d->last_state = REMOTE_KEY_PRESSED;
break;
case REMOTE_KEY_REPEAT:
deb_rc("KEY_REPEAT\n");
if (d->last_state != REMOTE_NO_KEY_PRESSED) {
deb_rc("repeating event %d\n",d->last_event);
input_event(d->rc_input_dev, EV_KEY, d->last_event, 2);
input_sync(d->rc_input_dev);
d->last_state = REMOTE_KEY_REPEAT;
}
default:
break;
}
*/
schedule:
schedule_delayed_work(&d->rc_query_work,msecs_to_jiffies(d->props.rc.legacy.rc_interval));
}
static int legacy_dvb_usb_remote_init(struct dvb_usb_device *d)
{
int i, err, rc_interval;
struct input_dev *input_dev;
input_dev = input_allocate_device();
if (!input_dev)
return -ENOMEM;
input_dev->evbit[0] = BIT_MASK(EV_KEY);
input_dev->name = "IR-receiver inside an USB DVB receiver";
input_dev->phys = d->rc_phys;
usb_to_input_id(d->udev, &input_dev->id);
input_dev->dev.parent = &d->udev->dev;
d->input_dev = input_dev;
d->rc_dev = NULL;
input_dev->getkeycode = legacy_dvb_usb_getkeycode;
input_dev->setkeycode = legacy_dvb_usb_setkeycode;
/* set the bits for the keys */
deb_rc("key map size: %d\n", d->props.rc.legacy.rc_map_size);
for (i = 0; i < d->props.rc.legacy.rc_map_size; i++) {
deb_rc("setting bit for event %d item %d\n",
d->props.rc.legacy.rc_map_table[i].keycode, i);
set_bit(d->props.rc.legacy.rc_map_table[i].keycode, input_dev->keybit);
}
/* setting these two values to non-zero, we have to manage key repeats */
input_dev->rep[REP_PERIOD] = d->props.rc.legacy.rc_interval;
input_dev->rep[REP_DELAY] = d->props.rc.legacy.rc_interval + 150;
input_set_drvdata(input_dev, d);
err = input_register_device(input_dev);
if (err)
input_free_device(input_dev);
rc_interval = d->props.rc.legacy.rc_interval;
INIT_DELAYED_WORK(&d->rc_query_work, legacy_dvb_usb_read_remote_control);
info("schedule remote query interval to %d msecs.", rc_interval);
schedule_delayed_work(&d->rc_query_work,
msecs_to_jiffies(rc_interval));
d->state |= DVB_USB_STATE_REMOTE;
return err;
}
/* Remote-control poll function - called every dib->rc_query_interval ms to see
* whether the remote control has received anything.
*
* TODO: Fix the repeat rate of the input device.
*/
static void dvb_usb_read_remote_control(struct work_struct *work)
{
struct dvb_usb_device *d =
container_of(work, struct dvb_usb_device, rc_query_work.work);
int err;
/* TODO: need a lock here. We can simply skip checking for the remote control
if we're busy. */
/* when the parameter has been set to 1 via sysfs while the
* driver was running, or when bulk mode is enabled after IR init
*/
if (dvb_usb_disable_rc_polling || d->props.rc.core.bulk_mode)
return;
err = d->props.rc.core.rc_query(d);
if (err)
err("error %d while querying for an remote control event.", err);
schedule_delayed_work(&d->rc_query_work,
msecs_to_jiffies(d->props.rc.core.rc_interval));
}
static int rc_core_dvb_usb_remote_init(struct dvb_usb_device *d)
{
int err, rc_interval;
struct rc_dev *dev;
dev = rc_allocate_device(d->props.rc.core.driver_type);
if (!dev)
return -ENOMEM;
dev->driver_name = d->props.rc.core.module_name;
dev->map_name = d->props.rc.core.rc_codes;
dev->change_protocol = d->props.rc.core.change_protocol;
dev->allowed_protocols = d->props.rc.core.allowed_protos;
usb_to_input_id(d->udev, &dev->input_id);
dev->device_name = d->desc->name;
dev->input_phys = d->rc_phys;
dev->dev.parent = &d->udev->dev;
dev->priv = d;
dev->scancode_mask = d->props.rc.core.scancode_mask;
err = rc_register_device(dev);
if (err < 0) {
rc_free_device(dev);
return err;
}
d->input_dev = NULL;
d->rc_dev = dev;
if (!d->props.rc.core.rc_query || d->props.rc.core.bulk_mode)
return 0;
/* Polling mode - initialize a work queue for handling it */
INIT_DELAYED_WORK(&d->rc_query_work, dvb_usb_read_remote_control);
rc_interval = d->props.rc.core.rc_interval;
info("schedule remote query interval to %d msecs.", rc_interval);
schedule_delayed_work(&d->rc_query_work,
msecs_to_jiffies(rc_interval));
return 0;
}
int dvb_usb_remote_init(struct dvb_usb_device *d)
{
int err;
if (dvb_usb_disable_rc_polling)
return 0;
if (d->props.rc.legacy.rc_map_table && d->props.rc.legacy.rc_query)
d->props.rc.mode = DVB_RC_LEGACY;
else if (d->props.rc.core.rc_codes)
d->props.rc.mode = DVB_RC_CORE;
else
return 0;
usb_make_path(d->udev, d->rc_phys, sizeof(d->rc_phys));
strlcat(d->rc_phys, "/ir0", sizeof(d->rc_phys));
/* Start the remote-control polling. */
if (d->props.rc.legacy.rc_interval < 40)
d->props.rc.legacy.rc_interval = 100; /* default */
if (d->props.rc.mode == DVB_RC_LEGACY)
err = legacy_dvb_usb_remote_init(d);
else
err = rc_core_dvb_usb_remote_init(d);
if (err)
return err;
d->state |= DVB_USB_STATE_REMOTE;
return 0;
}
int dvb_usb_remote_exit(struct dvb_usb_device *d)
{
if (d->state & DVB_USB_STATE_REMOTE) { cancel_delayed_work_sync(&d->rc_query_work);
if (d->props.rc.mode == DVB_RC_LEGACY)
input_unregister_device(d->input_dev);
else
rc_unregister_device(d->rc_dev);
}
d->state &= ~DVB_USB_STATE_REMOTE;
return 0;
}
#define DVB_USB_RC_NEC_EMPTY 0x00
#define DVB_USB_RC_NEC_KEY_PRESSED 0x01
#define DVB_USB_RC_NEC_KEY_REPEATED 0x02
int dvb_usb_nec_rc_key_to_event(struct dvb_usb_device *d,
u8 keybuf[5], u32 *event, int *state)
{
int i;
struct rc_map_table *keymap = d->props.rc.legacy.rc_map_table;
*event = 0;
*state = REMOTE_NO_KEY_PRESSED;
switch (keybuf[0]) {
case DVB_USB_RC_NEC_EMPTY:
break;
case DVB_USB_RC_NEC_KEY_PRESSED:
if ((u8) ~keybuf[1] != keybuf[2] ||
(u8) ~keybuf[3] != keybuf[4]) {
deb_err("remote control checksum failed.\n");
break;
}
/* See if we can match the raw key code. */
for (i = 0; i < d->props.rc.legacy.rc_map_size; i++)
if (rc5_custom(&keymap[i]) == keybuf[1] &&
rc5_data(&keymap[i]) == keybuf[3]) {
*event = keymap[i].keycode;
*state = REMOTE_KEY_PRESSED;
return 0;
}
deb_err("key mapping failed - no appropriate key found in keymapping\n");
break;
case DVB_USB_RC_NEC_KEY_REPEATED:
*state = REMOTE_KEY_REPEAT;
break;
default:
deb_err("unknown type of remote status: %d\n",keybuf[0]);
break;
}
return 0;
}
EXPORT_SYMBOL(dvb_usb_nec_rc_key_to_event);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_USB_TYPEC_H
#define __LINUX_USB_TYPEC_H
#include <linux/types.h>
/* USB Type-C Specification releases */
#define USB_TYPEC_REV_1_0 0x100 /* 1.0 */
#define USB_TYPEC_REV_1_1 0x110 /* 1.1 */
#define USB_TYPEC_REV_1_2 0x120 /* 1.2 */
#define USB_TYPEC_REV_1_3 0x130 /* 1.3 */
#define USB_TYPEC_REV_1_4 0x140 /* 1.4 */
#define USB_TYPEC_REV_2_0 0x200 /* 2.0 */
struct typec_partner;
struct typec_cable;
struct typec_plug;
struct typec_port;
struct typec_altmode_ops;
struct typec_cable_ops;
struct fwnode_handle;
struct device;
struct usb_power_delivery;
struct usb_power_delivery_desc;
enum typec_port_type {
TYPEC_PORT_SRC,
TYPEC_PORT_SNK,
TYPEC_PORT_DRP,
};
enum typec_port_data {
TYPEC_PORT_DFP,
TYPEC_PORT_UFP,
TYPEC_PORT_DRD,
};
enum typec_plug_type {
USB_PLUG_NONE,
USB_PLUG_TYPE_A,
USB_PLUG_TYPE_B,
USB_PLUG_TYPE_C,
USB_PLUG_CAPTIVE,
};
enum typec_data_role {
TYPEC_DEVICE,
TYPEC_HOST,
};
enum typec_role {
TYPEC_SINK,
TYPEC_SOURCE,
};
static inline int is_sink(enum typec_role role)
{
return role == TYPEC_SINK;
}
static inline int is_source(enum typec_role role)
{
return role == TYPEC_SOURCE;
}
enum typec_pwr_opmode {
TYPEC_PWR_MODE_USB,
TYPEC_PWR_MODE_1_5A,
TYPEC_PWR_MODE_3_0A,
TYPEC_PWR_MODE_PD,
};
enum typec_accessory {
TYPEC_ACCESSORY_NONE,
TYPEC_ACCESSORY_AUDIO,
TYPEC_ACCESSORY_DEBUG,
};
#define TYPEC_MAX_ACCESSORY 3
enum typec_orientation {
TYPEC_ORIENTATION_NONE,
TYPEC_ORIENTATION_NORMAL,
TYPEC_ORIENTATION_REVERSE,
};
/*
* struct enter_usb_data - Enter_USB Message details
* @eudo: Enter_USB Data Object
* @active_link_training: Active Cable Plug Link Training
*
* @active_link_training is a flag that should be set with uni-directional SBRX
* communication, and left 0 with passive cables and with bi-directional SBRX
* communication.
*/
struct enter_usb_data {
u32 eudo;
unsigned char active_link_training:1;
};
/*
* struct usb_pd_identity - USB Power Delivery identity data
* @id_header: ID Header VDO
* @cert_stat: Cert Stat VDO
* @product: Product VDO
* @vdo: Product Type Specific VDOs
*
* USB power delivery Discover Identity command response data.
*
* REVISIT: This is USB Power Delivery specific information, so this structure
* probable belongs to USB Power Delivery header file once we have them.
*/
struct usb_pd_identity {
u32 id_header;
u32 cert_stat;
u32 product;
u32 vdo[3];
};
int typec_partner_set_identity(struct typec_partner *partner);
int typec_cable_set_identity(struct typec_cable *cable);
/*
* struct typec_altmode_desc - USB Type-C Alternate Mode Descriptor
* @svid: Standard or Vendor ID
* @mode: Index of the Mode
* @vdo: VDO returned by Discover Modes USB PD command
* @roles: Only for ports. DRP if the mode is available in both roles
*
* Description of an Alternate Mode which a connector, cable plug or partner
* supports.
*/
struct typec_altmode_desc {
u16 svid;
u8 mode;
u32 vdo;
/* Only used with ports */
enum typec_port_data roles;
};
void typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision);
int typec_partner_set_num_altmodes(struct typec_partner *partner, int num_altmodes);
struct typec_altmode
*typec_partner_register_altmode(struct typec_partner *partner,
const struct typec_altmode_desc *desc);
int typec_plug_set_num_altmodes(struct typec_plug *plug, int num_altmodes);
struct typec_altmode
*typec_plug_register_altmode(struct typec_plug *plug,
const struct typec_altmode_desc *desc);
struct typec_altmode
*typec_port_register_altmode(struct typec_port *port,
const struct typec_altmode_desc *desc);
void typec_port_register_altmodes(struct typec_port *port,
const struct typec_altmode_ops *ops, void *drvdata,
struct typec_altmode **altmodes, size_t n);
void typec_port_register_cable_ops(struct typec_altmode **altmodes, int max_altmodes,
const struct typec_cable_ops *ops);
void typec_unregister_altmode(struct typec_altmode *altmode);
struct typec_port *typec_altmode2port(struct typec_altmode *alt);
void typec_altmode_update_active(struct typec_altmode *alt, bool active);
enum typec_plug_index {
TYPEC_PLUG_SOP_P,
TYPEC_PLUG_SOP_PP,
};
/*
* struct typec_plug_desc - USB Type-C Cable Plug Descriptor
* @index: SOP Prime for the plug connected to DFP and SOP Double Prime for the
* plug connected to UFP
*
* Represents USB Type-C Cable Plug.
*/
struct typec_plug_desc {
enum typec_plug_index index;
};
/*
* struct typec_cable_desc - USB Type-C Cable Descriptor
* @type: The plug type from USB PD Cable VDO
* @active: Is the cable active or passive
* @identity: Result of Discover Identity command
* @pd_revision: USB Power Delivery Specification revision if supported
*
* Represents USB Type-C Cable attached to USB Type-C port.
*/
struct typec_cable_desc {
enum typec_plug_type type;
unsigned int active:1;
struct usb_pd_identity *identity;
u16 pd_revision; /* 0300H = "3.0" */
};
/*
* struct typec_partner_desc - USB Type-C Partner Descriptor
* @usb_pd: USB Power Delivery support
* @accessory: Audio, Debug or none.
* @identity: Discover Identity command data
* @pd_revision: USB Power Delivery Specification Revision if supported
* @attach: Notification about attached USB device
* @deattach: Notification about removed USB device
*
* Details about a partner that is attached to USB Type-C port. If @identity
* member exists when partner is registered, a directory named "identity" is
* created to sysfs for the partner device.
*
* @pd_revision is based on the setting of the "Specification Revision" field
* in the message header on the initial "Source Capabilities" message received
* from the partner, or a "Request" message received from the partner, depending
* on whether our port is a Sink or a Source.
*/
struct typec_partner_desc {
unsigned int usb_pd:1;
enum typec_accessory accessory;
struct usb_pd_identity *identity;
u16 pd_revision; /* 0300H = "3.0" */
void (*attach)(struct typec_partner *partner, struct device *dev);
void (*deattach)(struct typec_partner *partner, struct device *dev);
};
/**
* struct typec_operations - USB Type-C Port Operations
* @try_role: Set data role preference for DRP port
* @dr_set: Set Data Role
* @pr_set: Set Power Role
* @vconn_set: Source VCONN
* @port_type_set: Set port type
* @pd_get: Get available USB Power Delivery Capabilities.
* @pd_set: Set USB Power Delivery Capabilities.
*/
struct typec_operations {
int (*try_role)(struct typec_port *port, int role);
int (*dr_set)(struct typec_port *port, enum typec_data_role role);
int (*pr_set)(struct typec_port *port, enum typec_role role);
int (*vconn_set)(struct typec_port *port, enum typec_role role);
int (*port_type_set)(struct typec_port *port,
enum typec_port_type type);
struct usb_power_delivery **(*pd_get)(struct typec_port *port);
int (*pd_set)(struct typec_port *port, struct usb_power_delivery *pd);
};
enum usb_pd_svdm_ver {
SVDM_VER_1_0 = 0,
SVDM_VER_2_0 = 1,
SVDM_VER_MAX = SVDM_VER_2_0,
};
/*
* struct typec_capability - USB Type-C Port Capabilities
* @type: Supported power role of the port
* @data: Supported data role of the port
* @revision: USB Type-C Specification release. Binary coded decimal
* @pd_revision: USB Power Delivery Specification revision if supported
* @svdm_version: USB PD Structured VDM version if supported
* @prefer_role: Initial role preference (DRP ports).
* @accessory: Supported Accessory Modes
* @fwnode: Optional fwnode of the port
* @driver_data: Private pointer for driver specific info
* @pd: Optional USB Power Delivery Support
* @ops: Port operations vector
*
* Static capabilities of a single USB Type-C port.
*/
struct typec_capability {
enum typec_port_type type;
enum typec_port_data data;
u16 revision; /* 0120H = "1.2" */
u16 pd_revision; /* 0300H = "3.0" */
enum usb_pd_svdm_ver svdm_version;
int prefer_role;
enum typec_accessory accessory[TYPEC_MAX_ACCESSORY];
unsigned int orientation_aware:1;
struct fwnode_handle *fwnode;
void *driver_data;
struct usb_power_delivery *pd;
const struct typec_operations *ops;
};
/* Specific to try_role(). Indicates the user want's to clear the preference. */
#define TYPEC_NO_PREFERRED_ROLE (-1)
struct typec_port *typec_register_port(struct device *parent,
const struct typec_capability *cap);
void typec_unregister_port(struct typec_port *port);
struct typec_partner *typec_register_partner(struct typec_port *port,
struct typec_partner_desc *desc);
void typec_unregister_partner(struct typec_partner *partner);
struct typec_cable *typec_register_cable(struct typec_port *port,
struct typec_cable_desc *desc);
void typec_unregister_cable(struct typec_cable *cable);
struct typec_cable *typec_cable_get(struct typec_port *port);
void typec_cable_put(struct typec_cable *cable);
int typec_cable_is_active(struct typec_cable *cable);
struct typec_plug *typec_register_plug(struct typec_cable *cable,
struct typec_plug_desc *desc);
void typec_unregister_plug(struct typec_plug *plug);
void typec_set_data_role(struct typec_port *port, enum typec_data_role role);
void typec_set_pwr_role(struct typec_port *port, enum typec_role role);
void typec_set_vconn_role(struct typec_port *port, enum typec_role role);
void typec_set_pwr_opmode(struct typec_port *port, enum typec_pwr_opmode mode);
int typec_set_orientation(struct typec_port *port,
enum typec_orientation orientation);
enum typec_orientation typec_get_orientation(struct typec_port *port);
int typec_set_mode(struct typec_port *port, int mode);
void *typec_get_drvdata(struct typec_port *port);
int typec_get_fw_cap(struct typec_capability *cap,
struct fwnode_handle *fwnode);
int typec_find_pwr_opmode(const char *name);
int typec_find_orientation(const char *name);
int typec_find_port_power_role(const char *name);
int typec_find_power_role(const char *name);
int typec_find_port_data_role(const char *name);
void typec_partner_set_svdm_version(struct typec_partner *partner,
enum usb_pd_svdm_ver svdm_version);
int typec_get_negotiated_svdm_version(struct typec_port *port);
int typec_get_cable_svdm_version(struct typec_port *port);
void typec_cable_set_svdm_version(struct typec_cable *cable, enum usb_pd_svdm_ver svdm_version);
struct usb_power_delivery *typec_partner_usb_power_delivery_register(struct typec_partner *partner,
struct usb_power_delivery_desc *desc);
int typec_port_set_usb_power_delivery(struct typec_port *port, struct usb_power_delivery *pd);
int typec_partner_set_usb_power_delivery(struct typec_partner *partner,
struct usb_power_delivery *pd);
/**
* struct typec_connector - Representation of Type-C port for external drivers
* @attach: notification about device removal
* @deattach: notification about device removal
*
* Drivers that control the USB and other ports (DisplayPorts, etc.), that are
* connected to the Type-C connectors, can use these callbacks to inform the
* Type-C connector class about connections and disconnections. That information
* can then be used by the typec-port drivers to power on or off parts that are
* needed or not needed - as an example, in USB mode if USB2 device is
* enumerated, USB3 components (retimers, phys, and what have you) do not need
* to be powered on.
*
* The attached (enumerated) devices will be liked with the typec-partner device.
*/
struct typec_connector {
void (*attach)(struct typec_connector *con, struct device *dev);
void (*deattach)(struct typec_connector *con, struct device *dev);
};
static inline void typec_attach(struct typec_connector *con, struct device *dev)
{
if (con && con->attach) con->attach(con, dev);
}
static inline void typec_deattach(struct typec_connector *con, struct device *dev)
{
if (con && con->deattach) con->deattach(con, dev);
}
#endif /* __LINUX_USB_TYPEC_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* This file provides wrappers with sanitizer instrumentation for atomic bit
* operations.
*
* To use this functionality, an arch's bitops.h file needs to define each of
* the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
* arch___set_bit(), etc.).
*/
#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
#define _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
#include <linux/instrumented.h>
/**
* set_bit - Atomically set a bit in memory
* @nr: the bit to set
* @addr: the address to start counting from
*
* This is a relaxed atomic operation (no implied memory barriers).
*
* Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity.
*/
static __always_inline void set_bit(long nr, volatile unsigned long *addr)
{
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_set_bit(nr, addr);
}
/**
* clear_bit - Clears a bit in memory
* @nr: Bit to clear
* @addr: Address to start counting from
*
* This is a relaxed atomic operation (no implied memory barriers).
*/
static __always_inline void clear_bit(long nr, volatile unsigned long *addr)
{
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); arch_clear_bit(nr, addr);
}
/**
* change_bit - Toggle a bit in memory
* @nr: Bit to change
* @addr: Address to start counting from
*
* This is a relaxed atomic operation (no implied memory barriers).
*
* Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity.
*/
static __always_inline void change_bit(long nr, volatile unsigned long *addr)
{
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_change_bit(nr, addr);
}
/**
* test_and_set_bit - Set a bit and return its old value
* @nr: Bit to set
* @addr: Address to count from
*
* This is an atomic fully-ordered operation (implied full memory barrier).
*/
static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
{
kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_set_bit(nr, addr);
}
/**
* test_and_clear_bit - Clear a bit and return its old value
* @nr: Bit to clear
* @addr: Address to count from
*
* This is an atomic fully-ordered operation (implied full memory barrier).
*/
static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
{
kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_clear_bit(nr, addr);
}
/**
* test_and_change_bit - Change a bit and return its old value
* @nr: Bit to change
* @addr: Address to count from
*
* This is an atomic fully-ordered operation (implied full memory barrier).
*/
static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
{
kcsan_mb();
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_change_bit(nr, addr);
}
#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_LIST_H
#define _LINUX_LIST_H
#include <linux/container_of.h>
#include <linux/types.h>
#include <linux/stddef.h>
#include <linux/poison.h>
#include <linux/const.h>
#include <asm/barrier.h>
/*
* Circular doubly linked list implementation.
*
* Some of the internal functions ("__xxx") are useful when
* manipulating whole lists rather than single entries, as
* sometimes we already know the next/prev entries and we can
* generate better code by using them directly rather than
* using the generic single-entry routines.
*/
#define LIST_HEAD_INIT(name) { &(name), &(name) }
#define LIST_HEAD(name) \
struct list_head name = LIST_HEAD_INIT(name)
/**
* INIT_LIST_HEAD - Initialize a list_head structure
* @list: list_head structure to be initialized.
*
* Initializes the list_head to point to itself. If it is a list header,
* the result is an empty list.
*/
static inline void INIT_LIST_HEAD(struct list_head *list)
{
WRITE_ONCE(list->next, list);
WRITE_ONCE(list->prev, list);
}
#ifdef CONFIG_LIST_HARDENED
#ifdef CONFIG_DEBUG_LIST
# define __list_valid_slowpath
#else
# define __list_valid_slowpath __cold __preserve_most
#endif
/*
* Performs the full set of list corruption checks before __list_add().
* On list corruption reports a warning, and returns false.
*/
extern bool __list_valid_slowpath __list_add_valid_or_report(struct list_head *new,
struct list_head *prev,
struct list_head *next);
/*
* Performs list corruption checks before __list_add(). Returns false if a
* corruption is detected, true otherwise.
*
* With CONFIG_LIST_HARDENED only, performs minimal list integrity checking
* inline to catch non-faulting corruptions, and only if a corruption is
* detected calls the reporting function __list_add_valid_or_report().
*/
static __always_inline bool __list_add_valid(struct list_head *new,
struct list_head *prev,
struct list_head *next)
{
bool ret = true;
if (!IS_ENABLED(CONFIG_DEBUG_LIST)) {
/*
* With the hardening version, elide checking if next and prev
* are NULL, since the immediate dereference of them below would
* result in a fault if NULL.
*
* With the reduced set of checks, we can afford to inline the
* checks, which also gives the compiler a chance to elide some
* of them completely if they can be proven at compile-time. If
* one of the pre-conditions does not hold, the slow-path will
* show a report which pre-condition failed.
*/
if (likely(next->prev == prev && prev->next == next && new != prev && new != next))
return true;
ret = false;
}
ret &= __list_add_valid_or_report(new, prev, next);
return ret;
}
/*
* Performs the full set of list corruption checks before __list_del_entry().
* On list corruption reports a warning, and returns false.
*/
extern bool __list_valid_slowpath __list_del_entry_valid_or_report(struct list_head *entry);
/*
* Performs list corruption checks before __list_del_entry(). Returns false if a
* corruption is detected, true otherwise.
*
* With CONFIG_LIST_HARDENED only, performs minimal list integrity checking
* inline to catch non-faulting corruptions, and only if a corruption is
* detected calls the reporting function __list_del_entry_valid_or_report().
*/
static __always_inline bool __list_del_entry_valid(struct list_head *entry)
{
bool ret = true;
if (!IS_ENABLED(CONFIG_DEBUG_LIST)) {
struct list_head *prev = entry->prev;
struct list_head *next = entry->next;
/*
* With the hardening version, elide checking if next and prev
* are NULL, LIST_POISON1 or LIST_POISON2, since the immediate
* dereference of them below would result in a fault.
*/
if (likely(prev->next == entry && next->prev == entry))
return true;
ret = false;
}
ret &= __list_del_entry_valid_or_report(entry);
return ret;
}
#else
static inline bool __list_add_valid(struct list_head *new,
struct list_head *prev,
struct list_head *next)
{
return true;
}
static inline bool __list_del_entry_valid(struct list_head *entry)
{
return true;
}
#endif
/*
* Insert a new entry between two known consecutive entries.
*
* This is only for internal list manipulation where we know
* the prev/next entries already!
*/
static inline void __list_add(struct list_head *new,
struct list_head *prev,
struct list_head *next)
{
if (!__list_add_valid(new, prev, next))
return;
next->prev = new;
new->next = next;
new->prev = prev;
WRITE_ONCE(prev->next, new);
}
/**
* list_add - add a new entry
* @new: new entry to be added
* @head: list head to add it after
*
* Insert a new entry after the specified head.
* This is good for implementing stacks.
*/
static inline void list_add(struct list_head *new, struct list_head *head)
{
__list_add(new, head, head->next);
}
/**
* list_add_tail - add a new entry
* @new: new entry to be added
* @head: list head to add it before
*
* Insert a new entry before the specified head.
* This is useful for implementing queues.
*/
static inline void list_add_tail(struct list_head *new, struct list_head *head)
{
__list_add(new, head->prev, head);
}
/*
* Delete a list entry by making the prev/next entries
* point to each other.
*
* This is only for internal list manipulation where we know
* the prev/next entries already!
*/
static inline void __list_del(struct list_head * prev, struct list_head * next)
{
next->prev = prev;
WRITE_ONCE(prev->next, next);
}
/*
* Delete a list entry and clear the 'prev' pointer.
*
* This is a special-purpose list clearing method used in the networking code
* for lists allocated as per-cpu, where we don't want to incur the extra
* WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this
* needs to check the node 'prev' pointer instead of calling list_empty().
*/
static inline void __list_del_clearprev(struct list_head *entry)
{
__list_del(entry->prev, entry->next);
entry->prev = NULL;
}
static inline void __list_del_entry(struct list_head *entry)
{
if (!__list_del_entry_valid(entry))
return;
__list_del(entry->prev, entry->next);
}
/**
* list_del - deletes entry from list.
* @entry: the element to delete from the list.
* Note: list_empty() on entry does not return true after this, the entry is
* in an undefined state.
*/
static inline void list_del(struct list_head *entry)
{
__list_del_entry(entry); entry->next = LIST_POISON1; entry->prev = LIST_POISON2;
}
/**
* list_replace - replace old entry by new one
* @old : the element to be replaced
* @new : the new element to insert
*
* If @old was empty, it will be overwritten.
*/
static inline void list_replace(struct list_head *old,
struct list_head *new)
{
new->next = old->next;
new->next->prev = new;
new->prev = old->prev;
new->prev->next = new;
}
/**
* list_replace_init - replace old entry by new one and initialize the old one
* @old : the element to be replaced
* @new : the new element to insert
*
* If @old was empty, it will be overwritten.
*/
static inline void list_replace_init(struct list_head *old,
struct list_head *new)
{
list_replace(old, new);
INIT_LIST_HEAD(old);
}
/**
* list_swap - replace entry1 with entry2 and re-add entry1 at entry2's position
* @entry1: the location to place entry2
* @entry2: the location to place entry1
*/
static inline void list_swap(struct list_head *entry1,
struct list_head *entry2)
{
struct list_head *pos = entry2->prev;
list_del(entry2);
list_replace(entry1, entry2);
if (pos == entry1)
pos = entry2;
list_add(entry1, pos);
}
/**
* list_del_init - deletes entry from list and reinitialize it.
* @entry: the element to delete from the list.
*/
static inline void list_del_init(struct list_head *entry)
{
__list_del_entry(entry); INIT_LIST_HEAD(entry);
}
/**
* list_move - delete from one list and add as another's head
* @list: the entry to move
* @head: the head that will precede our entry
*/
static inline void list_move(struct list_head *list, struct list_head *head)
{
__list_del_entry(list); list_add(list, head);
}
/**
* list_move_tail - delete from one list and add as another's tail
* @list: the entry to move
* @head: the head that will follow our entry
*/
static inline void list_move_tail(struct list_head *list,
struct list_head *head)
{
__list_del_entry(list); list_add_tail(list, head);
}
/**
* list_bulk_move_tail - move a subsection of a list to its tail
* @head: the head that will follow our entry
* @first: first entry to move
* @last: last entry to move, can be the same as first
*
* Move all entries between @first and including @last before @head.
* All three entries must belong to the same linked list.
*/
static inline void list_bulk_move_tail(struct list_head *head,
struct list_head *first,
struct list_head *last)
{
first->prev->next = last->next;
last->next->prev = first->prev;
head->prev->next = first;
first->prev = head->prev;
last->next = head;
head->prev = last;
}
/**
* list_is_first -- tests whether @list is the first entry in list @head
* @list: the entry to test
* @head: the head of the list
*/
static inline int list_is_first(const struct list_head *list, const struct list_head *head)
{
return list->prev == head;
}
/**
* list_is_last - tests whether @list is the last entry in list @head
* @list: the entry to test
* @head: the head of the list
*/
static inline int list_is_last(const struct list_head *list, const struct list_head *head)
{
return list->next == head;
}
/**
* list_is_head - tests whether @list is the list @head
* @list: the entry to test
* @head: the head of the list
*/
static inline int list_is_head(const struct list_head *list, const struct list_head *head)
{
return list == head;
}
/**
* list_empty - tests whether a list is empty
* @head: the list to test.
*/
static inline int list_empty(const struct list_head *head)
{
return READ_ONCE(head->next) == head;
}
/**
* list_del_init_careful - deletes entry from list and reinitialize it.
* @entry: the element to delete from the list.
*
* This is the same as list_del_init(), except designed to be used
* together with list_empty_careful() in a way to guarantee ordering
* of other memory operations.
*
* Any memory operations done before a list_del_init_careful() are
* guaranteed to be visible after a list_empty_careful() test.
*/
static inline void list_del_init_careful(struct list_head *entry)
{
__list_del_entry(entry); WRITE_ONCE(entry->prev, entry);
smp_store_release(&entry->next, entry);
}
/**
* list_empty_careful - tests whether a list is empty and not being modified
* @head: the list to test
*
* Description:
* tests whether a list is empty _and_ checks that no other CPU might be
* in the process of modifying either member (next or prev)
*
* NOTE: using list_empty_careful() without synchronization
* can only be safe if the only activity that can happen
* to the list entry is list_del_init(). Eg. it cannot be used
* if another CPU could re-list_add() it.
*/
static inline int list_empty_careful(const struct list_head *head)
{
struct list_head *next = smp_load_acquire(&head->next);
return list_is_head(next, head) && (next == READ_ONCE(head->prev));
}
/**
* list_rotate_left - rotate the list to the left
* @head: the head of the list
*/
static inline void list_rotate_left(struct list_head *head)
{
struct list_head *first;
if (!list_empty(head)) {
first = head->next;
list_move_tail(first, head);
}
}
/**
* list_rotate_to_front() - Rotate list to specific item.
* @list: The desired new front of the list.
* @head: The head of the list.
*
* Rotates list so that @list becomes the new front of the list.
*/
static inline void list_rotate_to_front(struct list_head *list,
struct list_head *head)
{
/*
* Deletes the list head from the list denoted by @head and
* places it as the tail of @list, this effectively rotates the
* list so that @list is at the front.
*/
list_move_tail(head, list);
}
/**
* list_is_singular - tests whether a list has just one entry.
* @head: the list to test.
*/
static inline int list_is_singular(const struct list_head *head)
{
return !list_empty(head) && (head->next == head->prev);
}
static inline void __list_cut_position(struct list_head *list,
struct list_head *head, struct list_head *entry)
{
struct list_head *new_first = entry->next;
list->next = head->next;
list->next->prev = list;
list->prev = entry;
entry->next = list;
head->next = new_first;
new_first->prev = head;
}
/**
* list_cut_position - cut a list into two
* @list: a new list to add all removed entries
* @head: a list with entries
* @entry: an entry within head, could be the head itself
* and if so we won't cut the list
*
* This helper moves the initial part of @head, up to and
* including @entry, from @head to @list. You should
* pass on @entry an element you know is on @head. @list
* should be an empty list or a list you do not care about
* losing its data.
*
*/
static inline void list_cut_position(struct list_head *list,
struct list_head *head, struct list_head *entry)
{
if (list_empty(head))
return;
if (list_is_singular(head) && !list_is_head(entry, head) && (entry != head->next))
return;
if (list_is_head(entry, head))
INIT_LIST_HEAD(list);
else
__list_cut_position(list, head, entry);
}
/**
* list_cut_before - cut a list into two, before given entry
* @list: a new list to add all removed entries
* @head: a list with entries
* @entry: an entry within head, could be the head itself
*
* This helper moves the initial part of @head, up to but
* excluding @entry, from @head to @list. You should pass
* in @entry an element you know is on @head. @list should
* be an empty list or a list you do not care about losing
* its data.
* If @entry == @head, all entries on @head are moved to
* @list.
*/
static inline void list_cut_before(struct list_head *list,
struct list_head *head,
struct list_head *entry)
{
if (head->next == entry) {
INIT_LIST_HEAD(list);
return;
}
list->next = head->next;
list->next->prev = list;
list->prev = entry->prev;
list->prev->next = list;
head->next = entry;
entry->prev = head;
}
static inline void __list_splice(const struct list_head *list,
struct list_head *prev,
struct list_head *next)
{
struct list_head *first = list->next;
struct list_head *last = list->prev;
first->prev = prev;
prev->next = first;
last->next = next;
next->prev = last;
}
/**
* list_splice - join two lists, this is designed for stacks
* @list: the new list to add.
* @head: the place to add it in the first list.
*/
static inline void list_splice(const struct list_head *list,
struct list_head *head)
{
if (!list_empty(list))
__list_splice(list, head, head->next);
}
/**
* list_splice_tail - join two lists, each list being a queue
* @list: the new list to add.
* @head: the place to add it in the first list.
*/
static inline void list_splice_tail(struct list_head *list,
struct list_head *head)
{
if (!list_empty(list))
__list_splice(list, head->prev, head);
}
/**
* list_splice_init - join two lists and reinitialise the emptied list.
* @list: the new list to add.
* @head: the place to add it in the first list.
*
* The list at @list is reinitialised
*/
static inline void list_splice_init(struct list_head *list,
struct list_head *head)
{
if (!list_empty(list)) {
__list_splice(list, head, head->next);
INIT_LIST_HEAD(list);
}
}
/**
* list_splice_tail_init - join two lists and reinitialise the emptied list
* @list: the new list to add.
* @head: the place to add it in the first list.
*
* Each of the lists is a queue.
* The list at @list is reinitialised
*/
static inline void list_splice_tail_init(struct list_head *list,
struct list_head *head)
{
if (!list_empty(list)) {
__list_splice(list, head->prev, head);
INIT_LIST_HEAD(list);
}
}
/**
* list_entry - get the struct for this entry
* @ptr: the &struct list_head pointer.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*/
#define list_entry(ptr, type, member) \
container_of(ptr, type, member)
/**
* list_first_entry - get the first element from a list
* @ptr: the list head to take the element from.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* Note, that list is expected to be not empty.
*/
#define list_first_entry(ptr, type, member) \
list_entry((ptr)->next, type, member)
/**
* list_last_entry - get the last element from a list
* @ptr: the list head to take the element from.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* Note, that list is expected to be not empty.
*/
#define list_last_entry(ptr, type, member) \
list_entry((ptr)->prev, type, member)
/**
* list_first_entry_or_null - get the first element from a list
* @ptr: the list head to take the element from.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* Note that if the list is empty, it returns NULL.
*/
#define list_first_entry_or_null(ptr, type, member) ({ \
struct list_head *head__ = (ptr); \
struct list_head *pos__ = READ_ONCE(head__->next); \
pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
})
/**
* list_next_entry - get the next element in list
* @pos: the type * to cursor
* @member: the name of the list_head within the struct.
*/
#define list_next_entry(pos, member) \
list_entry((pos)->member.next, typeof(*(pos)), member)
/**
* list_next_entry_circular - get the next element in list
* @pos: the type * to cursor.
* @head: the list head to take the element from.
* @member: the name of the list_head within the struct.
*
* Wraparound if pos is the last element (return the first element).
* Note, that list is expected to be not empty.
*/
#define list_next_entry_circular(pos, head, member) \
(list_is_last(&(pos)->member, head) ? \
list_first_entry(head, typeof(*(pos)), member) : list_next_entry(pos, member))
/**
* list_prev_entry - get the prev element in list
* @pos: the type * to cursor
* @member: the name of the list_head within the struct.
*/
#define list_prev_entry(pos, member) \
list_entry((pos)->member.prev, typeof(*(pos)), member)
/**
* list_prev_entry_circular - get the prev element in list
* @pos: the type * to cursor.
* @head: the list head to take the element from.
* @member: the name of the list_head within the struct.
*
* Wraparound if pos is the first element (return the last element).
* Note, that list is expected to be not empty.
*/
#define list_prev_entry_circular(pos, head, member) \
(list_is_first(&(pos)->member, head) ? \
list_last_entry(head, typeof(*(pos)), member) : list_prev_entry(pos, member))
/**
* list_for_each - iterate over a list
* @pos: the &struct list_head to use as a loop cursor.
* @head: the head for your list.
*/
#define list_for_each(pos, head) \
for (pos = (head)->next; !list_is_head(pos, (head)); pos = pos->next)
/**
* list_for_each_reverse - iterate backwards over a list
* @pos: the &struct list_head to use as a loop cursor.
* @head: the head for your list.
*/
#define list_for_each_reverse(pos, head) \
for (pos = (head)->prev; pos != (head); pos = pos->prev)
/**
* list_for_each_rcu - Iterate over a list in an RCU-safe fashion
* @pos: the &struct list_head to use as a loop cursor.
* @head: the head for your list.
*/
#define list_for_each_rcu(pos, head) \
for (pos = rcu_dereference((head)->next); \
!list_is_head(pos, (head)); \
pos = rcu_dereference(pos->next))
/**
* list_for_each_continue - continue iteration over a list
* @pos: the &struct list_head to use as a loop cursor.
* @head: the head for your list.
*
* Continue to iterate over a list, continuing after the current position.
*/
#define list_for_each_continue(pos, head) \
for (pos = pos->next; !list_is_head(pos, (head)); pos = pos->next)
/**
* list_for_each_prev - iterate over a list backwards
* @pos: the &struct list_head to use as a loop cursor.
* @head: the head for your list.
*/
#define list_for_each_prev(pos, head) \
for (pos = (head)->prev; !list_is_head(pos, (head)); pos = pos->prev)
/**
* list_for_each_safe - iterate over a list safe against removal of list entry
* @pos: the &struct list_head to use as a loop cursor.
* @n: another &struct list_head to use as temporary storage
* @head: the head for your list.
*/
#define list_for_each_safe(pos, n, head) \
for (pos = (head)->next, n = pos->next; \
!list_is_head(pos, (head)); \
pos = n, n = pos->next)
/**
* list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry
* @pos: the &struct list_head to use as a loop cursor.
* @n: another &struct list_head to use as temporary storage
* @head: the head for your list.
*/
#define list_for_each_prev_safe(pos, n, head) \
for (pos = (head)->prev, n = pos->prev; \
!list_is_head(pos, (head)); \
pos = n, n = pos->prev)
/**
* list_count_nodes - count nodes in the list
* @head: the head for your list.
*/
static inline size_t list_count_nodes(struct list_head *head)
{
struct list_head *pos;
size_t count = 0;
list_for_each(pos, head)
count++;
return count;
}
/**
* list_entry_is_head - test if the entry points to the head of the list
* @pos: the type * to cursor
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*/
#define list_entry_is_head(pos, head, member) \
list_is_head(&pos->member, (head))
/**
* list_for_each_entry - iterate over list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*/
#define list_for_each_entry(pos, head, member) \
for (pos = list_first_entry(head, typeof(*pos), member); \
!list_entry_is_head(pos, head, member); \
pos = list_next_entry(pos, member))
/**
* list_for_each_entry_reverse - iterate backwards over list of given type.
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*/
#define list_for_each_entry_reverse(pos, head, member) \
for (pos = list_last_entry(head, typeof(*pos), member); \
!list_entry_is_head(pos, head, member); \
pos = list_prev_entry(pos, member))
/**
* list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue()
* @pos: the type * to use as a start point
* @head: the head of the list
* @member: the name of the list_head within the struct.
*
* Prepares a pos entry for use as a start point in list_for_each_entry_continue().
*/
#define list_prepare_entry(pos, head, member) \
((pos) ? : list_entry(head, typeof(*pos), member))
/**
* list_for_each_entry_continue - continue iteration over list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Continue to iterate over list of given type, continuing after
* the current position.
*/
#define list_for_each_entry_continue(pos, head, member) \
for (pos = list_next_entry(pos, member); \
!list_entry_is_head(pos, head, member); \
pos = list_next_entry(pos, member))
/**
* list_for_each_entry_continue_reverse - iterate backwards from the given point
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Start to iterate over list of given type backwards, continuing after
* the current position.
*/
#define list_for_each_entry_continue_reverse(pos, head, member) \
for (pos = list_prev_entry(pos, member); \
!list_entry_is_head(pos, head, member); \
pos = list_prev_entry(pos, member))
/**
* list_for_each_entry_from - iterate over list of given type from the current point
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Iterate over list of given type, continuing from current position.
*/
#define list_for_each_entry_from(pos, head, member) \
for (; !list_entry_is_head(pos, head, member); \
pos = list_next_entry(pos, member))
/**
* list_for_each_entry_from_reverse - iterate backwards over list of given type
* from the current point
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Iterate backwards over list of given type, continuing from current position.
*/
#define list_for_each_entry_from_reverse(pos, head, member) \
for (; !list_entry_is_head(pos, head, member); \
pos = list_prev_entry(pos, member))
/**
* list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
* @pos: the type * to use as a loop cursor.
* @n: another type * to use as temporary storage
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*/
#define list_for_each_entry_safe(pos, n, head, member) \
for (pos = list_first_entry(head, typeof(*pos), member), \
n = list_next_entry(pos, member); \
!list_entry_is_head(pos, head, member); \
pos = n, n = list_next_entry(n, member))
/**
* list_for_each_entry_safe_continue - continue list iteration safe against removal
* @pos: the type * to use as a loop cursor.
* @n: another type * to use as temporary storage
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Iterate over list of given type, continuing after current point,
* safe against removal of list entry.
*/
#define list_for_each_entry_safe_continue(pos, n, head, member) \
for (pos = list_next_entry(pos, member), \
n = list_next_entry(pos, member); \
!list_entry_is_head(pos, head, member); \
pos = n, n = list_next_entry(n, member))
/**
* list_for_each_entry_safe_from - iterate over list from current point safe against removal
* @pos: the type * to use as a loop cursor.
* @n: another type * to use as temporary storage
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Iterate over list of given type from current point, safe against
* removal of list entry.
*/
#define list_for_each_entry_safe_from(pos, n, head, member) \
for (n = list_next_entry(pos, member); \
!list_entry_is_head(pos, head, member); \
pos = n, n = list_next_entry(n, member))
/**
* list_for_each_entry_safe_reverse - iterate backwards over list safe against removal
* @pos: the type * to use as a loop cursor.
* @n: another type * to use as temporary storage
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Iterate backwards over list of given type, safe against removal
* of list entry.
*/
#define list_for_each_entry_safe_reverse(pos, n, head, member) \
for (pos = list_last_entry(head, typeof(*pos), member), \
n = list_prev_entry(pos, member); \
!list_entry_is_head(pos, head, member); \
pos = n, n = list_prev_entry(n, member))
/**
* list_safe_reset_next - reset a stale list_for_each_entry_safe loop
* @pos: the loop cursor used in the list_for_each_entry_safe loop
* @n: temporary storage used in list_for_each_entry_safe
* @member: the name of the list_head within the struct.
*
* list_safe_reset_next is not safe to use in general if the list may be
* modified concurrently (eg. the lock is dropped in the loop body). An
* exception to this is if the cursor element (pos) is pinned in the list,
* and list_safe_reset_next is called after re-taking the lock and before
* completing the current iteration of the loop body.
*/
#define list_safe_reset_next(pos, n, member) \
n = list_next_entry(pos, member)
/*
* Double linked lists with a single pointer list head.
* Mostly useful for hash tables where the two pointer list head is
* too wasteful.
* You lose the ability to access the tail in O(1).
*/
#define HLIST_HEAD_INIT { .first = NULL }
#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL }
#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
static inline void INIT_HLIST_NODE(struct hlist_node *h)
{
h->next = NULL;
h->pprev = NULL;
}
/**
* hlist_unhashed - Has node been removed from list and reinitialized?
* @h: Node to be checked
*
* Not that not all removal functions will leave a node in unhashed
* state. For example, hlist_nulls_del_init_rcu() does leave the
* node in unhashed state, but hlist_nulls_del() does not.
*/
static inline int hlist_unhashed(const struct hlist_node *h)
{
return !h->pprev;
}
/**
* hlist_unhashed_lockless - Version of hlist_unhashed for lockless use
* @h: Node to be checked
*
* This variant of hlist_unhashed() must be used in lockless contexts
* to avoid potential load-tearing. The READ_ONCE() is paired with the
* various WRITE_ONCE() in hlist helpers that are defined below.
*/
static inline int hlist_unhashed_lockless(const struct hlist_node *h)
{
return !READ_ONCE(h->pprev);
}
/**
* hlist_empty - Is the specified hlist_head structure an empty hlist?
* @h: Structure to check.
*/
static inline int hlist_empty(const struct hlist_head *h)
{
return !READ_ONCE(h->first);
}
static inline void __hlist_del(struct hlist_node *n)
{
struct hlist_node *next = n->next;
struct hlist_node **pprev = n->pprev;
WRITE_ONCE(*pprev, next);
if (next)
WRITE_ONCE(next->pprev, pprev);
}
/**
* hlist_del - Delete the specified hlist_node from its list
* @n: Node to delete.
*
* Note that this function leaves the node in hashed state. Use
* hlist_del_init() or similar instead to unhash @n.
*/
static inline void hlist_del(struct hlist_node *n)
{
__hlist_del(n); n->next = LIST_POISON1;
n->pprev = LIST_POISON2;
}
/**
* hlist_del_init - Delete the specified hlist_node from its list and initialize
* @n: Node to delete.
*
* Note that this function leaves the node in unhashed state.
*/
static inline void hlist_del_init(struct hlist_node *n)
{
if (!hlist_unhashed(n)) { __hlist_del(n); INIT_HLIST_NODE(n);
}
}
/**
* hlist_add_head - add a new entry at the beginning of the hlist
* @n: new entry to be added
* @h: hlist head to add it after
*
* Insert a new entry after the specified head.
* This is good for implementing stacks.
*/
static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
{
struct hlist_node *first = h->first;
WRITE_ONCE(n->next, first);
if (first)
WRITE_ONCE(first->pprev, &n->next); WRITE_ONCE(h->first, n); WRITE_ONCE(n->pprev, &h->first);
}
/**
* hlist_add_before - add a new entry before the one specified
* @n: new entry to be added
* @next: hlist node to add it before, which must be non-NULL
*/
static inline void hlist_add_before(struct hlist_node *n,
struct hlist_node *next)
{
WRITE_ONCE(n->pprev, next->pprev);
WRITE_ONCE(n->next, next);
WRITE_ONCE(next->pprev, &n->next);
WRITE_ONCE(*(n->pprev), n);
}
/**
* hlist_add_behind - add a new entry after the one specified
* @n: new entry to be added
* @prev: hlist node to add it after, which must be non-NULL
*/
static inline void hlist_add_behind(struct hlist_node *n,
struct hlist_node *prev)
{
WRITE_ONCE(n->next, prev->next);
WRITE_ONCE(prev->next, n);
WRITE_ONCE(n->pprev, &prev->next);
if (n->next)
WRITE_ONCE(n->next->pprev, &n->next);
}
/**
* hlist_add_fake - create a fake hlist consisting of a single headless node
* @n: Node to make a fake list out of
*
* This makes @n appear to be its own predecessor on a headless hlist.
* The point of this is to allow things like hlist_del() to work correctly
* in cases where there is no list.
*/
static inline void hlist_add_fake(struct hlist_node *n)
{
n->pprev = &n->next;
}
/**
* hlist_fake: Is this node a fake hlist?
* @h: Node to check for being a self-referential fake hlist.
*/
static inline bool hlist_fake(struct hlist_node *h)
{
return h->pprev == &h->next;
}
/**
* hlist_is_singular_node - is node the only element of the specified hlist?
* @n: Node to check for singularity.
* @h: Header for potentially singular list.
*
* Check whether the node is the only node of the head without
* accessing head, thus avoiding unnecessary cache misses.
*/
static inline bool
hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h)
{
return !n->next && n->pprev == &h->first;
}
/**
* hlist_move_list - Move an hlist
* @old: hlist_head for old list.
* @new: hlist_head for new list.
*
* Move a list from one list head to another. Fixup the pprev
* reference of the first entry if it exists.
*/
static inline void hlist_move_list(struct hlist_head *old,
struct hlist_head *new)
{
new->first = old->first;
if (new->first)
new->first->pprev = &new->first;
old->first = NULL;
}
/**
* hlist_splice_init() - move all entries from one list to another
* @from: hlist_head from which entries will be moved
* @last: last entry on the @from list
* @to: hlist_head to which entries will be moved
*
* @to can be empty, @from must contain at least @last.
*/
static inline void hlist_splice_init(struct hlist_head *from,
struct hlist_node *last,
struct hlist_head *to)
{
if (to->first) to->first->pprev = &last->next; last->next = to->first;
to->first = from->first;
from->first->pprev = &to->first;
from->first = NULL;
}
#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
#define hlist_for_each(pos, head) \
for (pos = (head)->first; pos ; pos = pos->next)
#define hlist_for_each_safe(pos, n, head) \
for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
pos = n)
#define hlist_entry_safe(ptr, type, member) \
({ typeof(ptr) ____ptr = (ptr); \
____ptr ? hlist_entry(____ptr, type, member) : NULL; \
})
/**
* hlist_for_each_entry - iterate over list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry(pos, head, member) \
for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\
pos; \
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
/**
* hlist_for_each_entry_continue - iterate over a hlist continuing after current point
* @pos: the type * to use as a loop cursor.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry_continue(pos, member) \
for (pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member);\
pos; \
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
/**
* hlist_for_each_entry_from - iterate over a hlist continuing from current point
* @pos: the type * to use as a loop cursor.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry_from(pos, member) \
for (; pos; \
pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member))
/**
* hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
* @pos: the type * to use as a loop cursor.
* @n: a &struct hlist_node to use as temporary storage
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry_safe(pos, n, head, member) \
for (pos = hlist_entry_safe((head)->first, typeof(*pos), member);\
pos && ({ n = pos->member.next; 1; }); \
pos = hlist_entry_safe(n, typeof(*pos), member))
/**
* hlist_count_nodes - count nodes in the hlist
* @head: the head for your hlist.
*/
static inline size_t hlist_count_nodes(struct hlist_head *head)
{
struct hlist_node *pos;
size_t count = 0;
hlist_for_each(pos, head)
count++;
return count;
}
#endif
// SPDX-License-Identifier: GPL-2.0+
/*
* User-space Probes (UProbes)
*
* Copyright (C) IBM Corporation, 2008-2012
* Authors:
* Srikar Dronamraju
* Jim Keniston
* Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
*/
#include <linux/kernel.h>
#include <linux/highmem.h>
#include <linux/pagemap.h> /* read_mapping_page */
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/export.h>
#include <linux/rmap.h> /* anon_vma_prepare */
#include <linux/mmu_notifier.h> /* set_pte_at_notify */
#include <linux/swap.h> /* folio_free_swap */
#include <linux/ptrace.h> /* user_enable_single_step */
#include <linux/kdebug.h> /* notifier mechanism */
#include <linux/percpu-rwsem.h>
#include <linux/task_work.h>
#include <linux/shmem_fs.h>
#include <linux/khugepaged.h>
#include <linux/uprobes.h>
#define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
#define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
static struct rb_root uprobes_tree = RB_ROOT;
/*
* allows us to skip the uprobe_mmap if there are no uprobe events active
* at this time. Probably a fine grained per inode count is better?
*/
#define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
#define UPROBES_HASH_SZ 13
/* serialize uprobe->pending_list */
static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
#define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
/* Have a copy of original instruction */
#define UPROBE_COPY_INSN 0
struct uprobe {
struct rb_node rb_node; /* node in the rb tree */
refcount_t ref;
struct rw_semaphore register_rwsem;
struct rw_semaphore consumer_rwsem;
struct list_head pending_list;
struct uprobe_consumer *consumers;
struct inode *inode; /* Also hold a ref to inode */
loff_t offset;
loff_t ref_ctr_offset;
unsigned long flags;
/*
* The generic code assumes that it has two members of unknown type
* owned by the arch-specific code:
*
* insn - copy_insn() saves the original instruction here for
* arch_uprobe_analyze_insn().
*
* ixol - potentially modified instruction to execute out of
* line, copied to xol_area by xol_get_insn_slot().
*/
struct arch_uprobe arch;
};
struct delayed_uprobe {
struct list_head list;
struct uprobe *uprobe;
struct mm_struct *mm;
};
static DEFINE_MUTEX(delayed_uprobe_lock);
static LIST_HEAD(delayed_uprobe_list);
/*
* Execute out of line area: anonymous executable mapping installed
* by the probed task to execute the copy of the original instruction
* mangled by set_swbp().
*
* On a breakpoint hit, thread contests for a slot. It frees the
* slot after singlestep. Currently a fixed number of slots are
* allocated.
*/
struct xol_area {
wait_queue_head_t wq; /* if all slots are busy */
atomic_t slot_count; /* number of in-use slots */
unsigned long *bitmap; /* 0 = free slot */
struct vm_special_mapping xol_mapping;
struct page *pages[2];
/*
* We keep the vma's vm_start rather than a pointer to the vma
* itself. The probed process or a naughty kernel module could make
* the vma go away, and we must handle that reasonably gracefully.
*/
unsigned long vaddr; /* Page(s) of instruction slots */
};
/*
* valid_vma: Verify if the specified vma is an executable vma
* Relax restrictions while unregistering: vm_flags might have
* changed after breakpoint was inserted.
* - is_register: indicates if we are in register context.
* - Return 1 if the specified virtual address is in an
* executable vma.
*/
static bool valid_vma(struct vm_area_struct *vma, bool is_register)
{
vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
if (is_register)
flags |= VM_WRITE;
return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
}
static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
{
return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
}
static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
{
return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
}
/**
* __replace_page - replace page in vma by new page.
* based on replace_page in mm/ksm.c
*
* @vma: vma that holds the pte pointing to page
* @addr: address the old @page is mapped at
* @old_page: the page we are replacing by new_page
* @new_page: the modified page we replace page by
*
* If @new_page is NULL, only unmap @old_page.
*
* Returns 0 on success, negative error code otherwise.
*/
static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
struct page *old_page, struct page *new_page)
{
struct folio *old_folio = page_folio(old_page);
struct folio *new_folio;
struct mm_struct *mm = vma->vm_mm;
DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0);
int err;
struct mmu_notifier_range range;
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, addr,
addr + PAGE_SIZE);
if (new_page) {
new_folio = page_folio(new_page);
err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL);
if (err)
return err;
}
/* For folio_free_swap() below */
folio_lock(old_folio);
mmu_notifier_invalidate_range_start(&range);
err = -EAGAIN;
if (!page_vma_mapped_walk(&pvmw))
goto unlock;
VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
if (new_page) {
folio_get(new_folio);
folio_add_new_anon_rmap(new_folio, vma, addr);
folio_add_lru_vma(new_folio, vma);
} else
/* no new page, just dec_mm_counter for old_page */
dec_mm_counter(mm, MM_ANONPAGES);
if (!folio_test_anon(old_folio)) {
dec_mm_counter(mm, mm_counter_file(old_folio));
inc_mm_counter(mm, MM_ANONPAGES);
}
flush_cache_page(vma, addr, pte_pfn(ptep_get(pvmw.pte)));
ptep_clear_flush(vma, addr, pvmw.pte);
if (new_page)
set_pte_at_notify(mm, addr, pvmw.pte,
mk_pte(new_page, vma->vm_page_prot));
folio_remove_rmap_pte(old_folio, old_page, vma);
if (!folio_mapped(old_folio))
folio_free_swap(old_folio);
page_vma_mapped_walk_done(&pvmw);
folio_put(old_folio);
err = 0;
unlock:
mmu_notifier_invalidate_range_end(&range);
folio_unlock(old_folio);
return err;
}
/**
* is_swbp_insn - check if instruction is breakpoint instruction.
* @insn: instruction to be checked.
* Default implementation of is_swbp_insn
* Returns true if @insn is a breakpoint instruction.
*/
bool __weak is_swbp_insn(uprobe_opcode_t *insn)
{
return *insn == UPROBE_SWBP_INSN;
}
/**
* is_trap_insn - check if instruction is breakpoint instruction.
* @insn: instruction to be checked.
* Default implementation of is_trap_insn
* Returns true if @insn is a breakpoint instruction.
*
* This function is needed for the case where an architecture has multiple
* trap instructions (like powerpc).
*/
bool __weak is_trap_insn(uprobe_opcode_t *insn)
{
return is_swbp_insn(insn);
}
static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
{
void *kaddr = kmap_atomic(page);
memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
kunmap_atomic(kaddr);
}
static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
{
void *kaddr = kmap_atomic(page);
memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
kunmap_atomic(kaddr);
}
static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
{
uprobe_opcode_t old_opcode;
bool is_swbp;
/*
* Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
* We do not check if it is any other 'trap variant' which could
* be conditional trap instruction such as the one powerpc supports.
*
* The logic is that we do not care if the underlying instruction
* is a trap variant; uprobes always wins over any other (gdb)
* breakpoint.
*/
copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
is_swbp = is_swbp_insn(&old_opcode);
if (is_swbp_insn(new_opcode)) {
if (is_swbp) /* register: already installed? */
return 0;
} else {
if (!is_swbp) /* unregister: was it changed by us? */
return 0;
}
return 1;
}
static struct delayed_uprobe *
delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
{
struct delayed_uprobe *du;
list_for_each_entry(du, &delayed_uprobe_list, list)
if (du->uprobe == uprobe && du->mm == mm)
return du;
return NULL;
}
static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
{
struct delayed_uprobe *du;
if (delayed_uprobe_check(uprobe, mm))
return 0;
du = kzalloc(sizeof(*du), GFP_KERNEL);
if (!du)
return -ENOMEM;
du->uprobe = uprobe;
du->mm = mm;
list_add(&du->list, &delayed_uprobe_list);
return 0;
}
static void delayed_uprobe_delete(struct delayed_uprobe *du)
{
if (WARN_ON(!du))
return;
list_del(&du->list);
kfree(du);
}
static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
{
struct list_head *pos, *q;
struct delayed_uprobe *du;
if (!uprobe && !mm)
return;
list_for_each_safe(pos, q, &delayed_uprobe_list) {
du = list_entry(pos, struct delayed_uprobe, list);
if (uprobe && du->uprobe != uprobe)
continue;
if (mm && du->mm != mm)
continue;
delayed_uprobe_delete(du);
}
}
static bool valid_ref_ctr_vma(struct uprobe *uprobe,
struct vm_area_struct *vma)
{
unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
return uprobe->ref_ctr_offset &&
vma->vm_file &&
file_inode(vma->vm_file) == uprobe->inode &&
(vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
vma->vm_start <= vaddr &&
vma->vm_end > vaddr;
}
static struct vm_area_struct *
find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
{
VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *tmp;
for_each_vma(vmi, tmp)
if (valid_ref_ctr_vma(uprobe, tmp))
return tmp;
return NULL;
}
static int
__update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
{
void *kaddr;
struct page *page;
int ret;
short *ptr;
if (!vaddr || !d)
return -EINVAL;
ret = get_user_pages_remote(mm, vaddr, 1,
FOLL_WRITE, &page, NULL);
if (unlikely(ret <= 0)) {
/*
* We are asking for 1 page. If get_user_pages_remote() fails,
* it may return 0, in that case we have to return error.
*/
return ret == 0 ? -EBUSY : ret;
}
kaddr = kmap_atomic(page);
ptr = kaddr + (vaddr & ~PAGE_MASK);
if (unlikely(*ptr + d < 0)) {
pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
"curr val: %d, delta: %d\n", vaddr, *ptr, d);
ret = -EINVAL;
goto out;
}
*ptr += d;
ret = 0;
out:
kunmap_atomic(kaddr);
put_page(page);
return ret;
}
static void update_ref_ctr_warn(struct uprobe *uprobe,
struct mm_struct *mm, short d)
{
pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
"0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
(unsigned long long) uprobe->offset,
(unsigned long long) uprobe->ref_ctr_offset, mm);
}
static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
short d)
{
struct vm_area_struct *rc_vma;
unsigned long rc_vaddr;
int ret = 0;
rc_vma = find_ref_ctr_vma(uprobe, mm);
if (rc_vma) {
rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
ret = __update_ref_ctr(mm, rc_vaddr, d);
if (ret)
update_ref_ctr_warn(uprobe, mm, d);
if (d > 0)
return ret;
}
mutex_lock(&delayed_uprobe_lock);
if (d > 0)
ret = delayed_uprobe_add(uprobe, mm);
else
delayed_uprobe_remove(uprobe, mm);
mutex_unlock(&delayed_uprobe_lock);
return ret;
}
/*
* NOTE:
* Expect the breakpoint instruction to be the smallest size instruction for
* the architecture. If an arch has variable length instruction and the
* breakpoint instruction is not of the smallest length instruction
* supported by that architecture then we need to modify is_trap_at_addr and
* uprobe_write_opcode accordingly. This would never be a problem for archs
* that have fixed length instructions.
*
* uprobe_write_opcode - write the opcode at a given virtual address.
* @auprobe: arch specific probepoint information.
* @mm: the probed process address space.
* @vaddr: the virtual address to store the opcode.
* @opcode: opcode to be written at @vaddr.
*
* Called with mm->mmap_lock held for write.
* Return 0 (success) or a negative errno.
*/
int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
unsigned long vaddr, uprobe_opcode_t opcode)
{
struct uprobe *uprobe;
struct page *old_page, *new_page;
struct vm_area_struct *vma;
int ret, is_register, ref_ctr_updated = 0;
bool orig_page_huge = false;
unsigned int gup_flags = FOLL_FORCE;
is_register = is_swbp_insn(&opcode);
uprobe = container_of(auprobe, struct uprobe, arch);
retry:
if (is_register)
gup_flags |= FOLL_SPLIT_PMD;
/* Read the page with vaddr into memory */
old_page = get_user_page_vma_remote(mm, vaddr, gup_flags, &vma);
if (IS_ERR(old_page))
return PTR_ERR(old_page);
ret = verify_opcode(old_page, vaddr, &opcode);
if (ret <= 0)
goto put_old;
if (WARN(!is_register && PageCompound(old_page),
"uprobe unregister should never work on compound page\n")) {
ret = -EINVAL;
goto put_old;
}
/* We are going to replace instruction, update ref_ctr. */
if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
if (ret)
goto put_old;
ref_ctr_updated = 1;
}
ret = 0;
if (!is_register && !PageAnon(old_page))
goto put_old;
ret = anon_vma_prepare(vma);
if (ret)
goto put_old;
ret = -ENOMEM;
new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
if (!new_page)
goto put_old;
__SetPageUptodate(new_page);
copy_highpage(new_page, old_page);
copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
if (!is_register) {
struct page *orig_page;
pgoff_t index;
VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
index);
if (orig_page) {
if (PageUptodate(orig_page) &&
pages_identical(new_page, orig_page)) {
/* let go new_page */
put_page(new_page);
new_page = NULL;
if (PageCompound(orig_page))
orig_page_huge = true;
}
put_page(orig_page);
}
}
ret = __replace_page(vma, vaddr & PAGE_MASK, old_page, new_page);
if (new_page)
put_page(new_page);
put_old:
put_page(old_page);
if (unlikely(ret == -EAGAIN))
goto retry;
/* Revert back reference counter if instruction update failed. */
if (ret && is_register && ref_ctr_updated)
update_ref_ctr(uprobe, mm, -1);
/* try collapse pmd for compound page */
if (!ret && orig_page_huge)
collapse_pte_mapped_thp(mm, vaddr, false);
return ret;
}
/**
* set_swbp - store breakpoint at a given address.
* @auprobe: arch specific probepoint information.
* @mm: the probed process address space.
* @vaddr: the virtual address to insert the opcode.
*
* For mm @mm, store the breakpoint instruction at @vaddr.
* Return 0 (success) or a negative errno.
*/
int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
{
return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
}
/**
* set_orig_insn - Restore the original instruction.
* @mm: the probed process address space.
* @auprobe: arch specific probepoint information.
* @vaddr: the virtual address to insert the opcode.
*
* For mm @mm, restore the original opcode (opcode) at @vaddr.
* Return 0 (success) or a negative errno.
*/
int __weak
set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
{
return uprobe_write_opcode(auprobe, mm, vaddr,
*(uprobe_opcode_t *)&auprobe->insn);
}
static struct uprobe *get_uprobe(struct uprobe *uprobe)
{
refcount_inc(&uprobe->ref);
return uprobe;
}
static void put_uprobe(struct uprobe *uprobe)
{
if (refcount_dec_and_test(&uprobe->ref)) {
/*
* If application munmap(exec_vma) before uprobe_unregister()
* gets called, we don't get a chance to remove uprobe from
* delayed_uprobe_list from remove_breakpoint(). Do it here.
*/
mutex_lock(&delayed_uprobe_lock);
delayed_uprobe_remove(uprobe, NULL);
mutex_unlock(&delayed_uprobe_lock);
kfree(uprobe);
}
}
static __always_inline
int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
const struct uprobe *r)
{
if (l_inode < r->inode)
return -1;
if (l_inode > r->inode)
return 1;
if (l_offset < r->offset)
return -1;
if (l_offset > r->offset)
return 1;
return 0;
}
#define __node_2_uprobe(node) \
rb_entry((node), struct uprobe, rb_node)
struct __uprobe_key {
struct inode *inode;
loff_t offset;
};
static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b)
{
const struct __uprobe_key *a = key;
return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b));
}
static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b)
{
struct uprobe *u = __node_2_uprobe(a);
return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b));
}
static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
{
struct __uprobe_key key = {
.inode = inode,
.offset = offset,
};
struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
if (node)
return get_uprobe(__node_2_uprobe(node));
return NULL;
}
/*
* Find a uprobe corresponding to a given inode:offset
* Acquires uprobes_treelock
*/
static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
{
struct uprobe *uprobe;
spin_lock(&uprobes_treelock);
uprobe = __find_uprobe(inode, offset);
spin_unlock(&uprobes_treelock);
return uprobe;
}
static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
{
struct rb_node *node;
node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
if (node)
return get_uprobe(__node_2_uprobe(node));
/* get access + creation ref */
refcount_set(&uprobe->ref, 2);
return NULL;
}
/*
* Acquire uprobes_treelock.
* Matching uprobe already exists in rbtree;
* increment (access refcount) and return the matching uprobe.
*
* No matching uprobe; insert the uprobe in rb_tree;
* get a double refcount (access + creation) and return NULL.
*/
static struct uprobe *insert_uprobe(struct uprobe *uprobe)
{
struct uprobe *u;
spin_lock(&uprobes_treelock);
u = __insert_uprobe(uprobe);
spin_unlock(&uprobes_treelock);
return u;
}
static void
ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
{
pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
"ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
(unsigned long long) cur_uprobe->ref_ctr_offset,
(unsigned long long) uprobe->ref_ctr_offset);
}
static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
loff_t ref_ctr_offset)
{
struct uprobe *uprobe, *cur_uprobe;
uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
if (!uprobe)
return NULL;
uprobe->inode = inode;
uprobe->offset = offset;
uprobe->ref_ctr_offset = ref_ctr_offset;
init_rwsem(&uprobe->register_rwsem);
init_rwsem(&uprobe->consumer_rwsem);
/* add to uprobes_tree, sorted on inode:offset */
cur_uprobe = insert_uprobe(uprobe);
/* a uprobe exists for this inode:offset combination */
if (cur_uprobe) {
if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
ref_ctr_mismatch_warn(cur_uprobe, uprobe);
put_uprobe(cur_uprobe);
kfree(uprobe);
return ERR_PTR(-EINVAL);
}
kfree(uprobe);
uprobe = cur_uprobe;
}
return uprobe;
}
static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
{
down_write(&uprobe->consumer_rwsem);
uc->next = uprobe->consumers;
uprobe->consumers = uc;
up_write(&uprobe->consumer_rwsem);
}
/*
* For uprobe @uprobe, delete the consumer @uc.
* Return true if the @uc is deleted successfully
* or return false.
*/
static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
{
struct uprobe_consumer **con;
bool ret = false;
down_write(&uprobe->consumer_rwsem);
for (con = &uprobe->consumers; *con; con = &(*con)->next) {
if (*con == uc) {
*con = uc->next;
ret = true;
break;
}
}
up_write(&uprobe->consumer_rwsem);
return ret;
}
static int __copy_insn(struct address_space *mapping, struct file *filp,
void *insn, int nbytes, loff_t offset)
{
struct page *page;
/*
* Ensure that the page that has the original instruction is populated
* and in page-cache. If ->read_folio == NULL it must be shmem_mapping(),
* see uprobe_register().
*/
if (mapping->a_ops->read_folio)
page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
else
page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
if (IS_ERR(page))
return PTR_ERR(page);
copy_from_page(page, offset, insn, nbytes);
put_page(page);
return 0;
}
static int copy_insn(struct uprobe *uprobe, struct file *filp)
{
struct address_space *mapping = uprobe->inode->i_mapping;
loff_t offs = uprobe->offset;
void *insn = &uprobe->arch.insn;
int size = sizeof(uprobe->arch.insn);
int len, err = -EIO;
/* Copy only available bytes, -EIO if nothing was read */
do {
if (offs >= i_size_read(uprobe->inode))
break;
len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
err = __copy_insn(mapping, filp, insn, len, offs);
if (err)
break;
insn += len;
offs += len;
size -= len;
} while (size);
return err;
}
static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
struct mm_struct *mm, unsigned long vaddr)
{
int ret = 0;
if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
return ret;
/* TODO: move this into _register, until then we abuse this sem. */
down_write(&uprobe->consumer_rwsem);
if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
goto out;
ret = copy_insn(uprobe, file);
if (ret)
goto out;
ret = -ENOTSUPP;
if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
goto out;
ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
if (ret)
goto out;
smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
set_bit(UPROBE_COPY_INSN, &uprobe->flags);
out:
up_write(&uprobe->consumer_rwsem);
return ret;
}
static inline bool consumer_filter(struct uprobe_consumer *uc,
enum uprobe_filter_ctx ctx, struct mm_struct *mm)
{
return !uc->filter || uc->filter(uc, ctx, mm);
}
static bool filter_chain(struct uprobe *uprobe,
enum uprobe_filter_ctx ctx, struct mm_struct *mm)
{
struct uprobe_consumer *uc;
bool ret = false;
down_read(&uprobe->consumer_rwsem);
for (uc = uprobe->consumers; uc; uc = uc->next) {
ret = consumer_filter(uc, ctx, mm);
if (ret)
break;
}
up_read(&uprobe->consumer_rwsem);
return ret;
}
static int
install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long vaddr)
{
bool first_uprobe;
int ret;
ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
if (ret)
return ret;
/*
* set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
* the task can hit this breakpoint right after __replace_page().
*/
first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
if (first_uprobe)
set_bit(MMF_HAS_UPROBES, &mm->flags);
ret = set_swbp(&uprobe->arch, mm, vaddr);
if (!ret)
clear_bit(MMF_RECALC_UPROBES, &mm->flags);
else if (first_uprobe)
clear_bit(MMF_HAS_UPROBES, &mm->flags);
return ret;
}
static int
remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
{
set_bit(MMF_RECALC_UPROBES, &mm->flags);
return set_orig_insn(&uprobe->arch, mm, vaddr);
}
static inline bool uprobe_is_active(struct uprobe *uprobe)
{
return !RB_EMPTY_NODE(&uprobe->rb_node);
}
/*
* There could be threads that have already hit the breakpoint. They
* will recheck the current insn and restart if find_uprobe() fails.
* See find_active_uprobe().
*/
static void delete_uprobe(struct uprobe *uprobe)
{
if (WARN_ON(!uprobe_is_active(uprobe)))
return;
spin_lock(&uprobes_treelock);
rb_erase(&uprobe->rb_node, &uprobes_tree);
spin_unlock(&uprobes_treelock);
RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
put_uprobe(uprobe);
}
struct map_info {
struct map_info *next;
struct mm_struct *mm;
unsigned long vaddr;
};
static inline struct map_info *free_map_info(struct map_info *info)
{
struct map_info *next = info->next;
kfree(info);
return next;
}
static struct map_info *
build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
{
unsigned long pgoff = offset >> PAGE_SHIFT;
struct vm_area_struct *vma;
struct map_info *curr = NULL;
struct map_info *prev = NULL;
struct map_info *info;
int more = 0;
again:
i_mmap_lock_read(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
if (!valid_vma(vma, is_register))
continue;
if (!prev && !more) {
/*
* Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
* reclaim. This is optimistic, no harm done if it fails.
*/
prev = kmalloc(sizeof(struct map_info),
GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
if (prev)
prev->next = NULL;
}
if (!prev) {
more++;
continue;
}
if (!mmget_not_zero(vma->vm_mm))
continue;
info = prev;
prev = prev->next;
info->next = curr;
curr = info;
info->mm = vma->vm_mm;
info->vaddr = offset_to_vaddr(vma, offset);
}
i_mmap_unlock_read(mapping);
if (!more)
goto out;
prev = curr;
while (curr) {
mmput(curr->mm);
curr = curr->next;
}
do {
info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
if (!info) {
curr = ERR_PTR(-ENOMEM);
goto out;
}
info->next = prev;
prev = info;
} while (--more);
goto again;
out:
while (prev)
prev = free_map_info(prev);
return curr;
}
static int
register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
{
bool is_register = !!new;
struct map_info *info;
int err = 0;
percpu_down_write(&dup_mmap_sem);
info = build_map_info(uprobe->inode->i_mapping,
uprobe->offset, is_register);
if (IS_ERR(info)) {
err = PTR_ERR(info);
goto out;
}
while (info) {
struct mm_struct *mm = info->mm;
struct vm_area_struct *vma;
if (err && is_register)
goto free;
mmap_write_lock(mm);
vma = find_vma(mm, info->vaddr);
if (!vma || !valid_vma(vma, is_register) ||
file_inode(vma->vm_file) != uprobe->inode)
goto unlock;
if (vma->vm_start > info->vaddr ||
vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
goto unlock;
if (is_register) {
/* consult only the "caller", new consumer. */
if (consumer_filter(new,
UPROBE_FILTER_REGISTER, mm))
err = install_breakpoint(uprobe, mm, vma, info->vaddr);
} else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
if (!filter_chain(uprobe,
UPROBE_FILTER_UNREGISTER, mm))
err |= remove_breakpoint(uprobe, mm, info->vaddr);
}
unlock:
mmap_write_unlock(mm);
free:
mmput(mm);
info = free_map_info(info);
}
out:
percpu_up_write(&dup_mmap_sem);
return err;
}
static void
__uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
{
int err;
if (WARN_ON(!consumer_del(uprobe, uc)))
return;
err = register_for_each_vma(uprobe, NULL);
/* TODO : cant unregister? schedule a worker thread */
if (!uprobe->consumers && !err)
delete_uprobe(uprobe);
}
/*
* uprobe_unregister - unregister an already registered probe.
* @inode: the file in which the probe has to be removed.
* @offset: offset from the start of the file.
* @uc: identify which probe if multiple probes are colocated.
*/
void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
{
struct uprobe *uprobe;
uprobe = find_uprobe(inode, offset);
if (WARN_ON(!uprobe))
return;
down_write(&uprobe->register_rwsem);
__uprobe_unregister(uprobe, uc);
up_write(&uprobe->register_rwsem);
put_uprobe(uprobe);
}
EXPORT_SYMBOL_GPL(uprobe_unregister);
/*
* __uprobe_register - register a probe
* @inode: the file in which the probe has to be placed.
* @offset: offset from the start of the file.
* @uc: information on howto handle the probe..
*
* Apart from the access refcount, __uprobe_register() takes a creation
* refcount (thro alloc_uprobe) if and only if this @uprobe is getting
* inserted into the rbtree (i.e first consumer for a @inode:@offset
* tuple). Creation refcount stops uprobe_unregister from freeing the
* @uprobe even before the register operation is complete. Creation
* refcount is released when the last @uc for the @uprobe
* unregisters. Caller of __uprobe_register() is required to keep @inode
* (and the containing mount) referenced.
*
* Return errno if it cannot successully install probes
* else return 0 (success)
*/
static int __uprobe_register(struct inode *inode, loff_t offset,
loff_t ref_ctr_offset, struct uprobe_consumer *uc)
{
struct uprobe *uprobe;
int ret;
/* Uprobe must have at least one set consumer */
if (!uc->handler && !uc->ret_handler)
return -EINVAL;
/* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
if (!inode->i_mapping->a_ops->read_folio &&
!shmem_mapping(inode->i_mapping))
return -EIO;
/* Racy, just to catch the obvious mistakes */
if (offset > i_size_read(inode))
return -EINVAL;
/*
* This ensures that copy_from_page(), copy_to_page() and
* __update_ref_ctr() can't cross page boundary.
*/
if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
return -EINVAL;
if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
return -EINVAL;
retry:
uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
if (!uprobe)
return -ENOMEM;
if (IS_ERR(uprobe))
return PTR_ERR(uprobe);
/*
* We can race with uprobe_unregister()->delete_uprobe().
* Check uprobe_is_active() and retry if it is false.
*/
down_write(&uprobe->register_rwsem);
ret = -EAGAIN;
if (likely(uprobe_is_active(uprobe))) {
consumer_add(uprobe, uc);
ret = register_for_each_vma(uprobe, uc);
if (ret)
__uprobe_unregister(uprobe, uc);
}
up_write(&uprobe->register_rwsem);
put_uprobe(uprobe);
if (unlikely(ret == -EAGAIN))
goto retry;
return ret;
}
int uprobe_register(struct inode *inode, loff_t offset,
struct uprobe_consumer *uc)
{
return __uprobe_register(inode, offset, 0, uc);
}
EXPORT_SYMBOL_GPL(uprobe_register);
int uprobe_register_refctr(struct inode *inode, loff_t offset,
loff_t ref_ctr_offset, struct uprobe_consumer *uc)
{
return __uprobe_register(inode, offset, ref_ctr_offset, uc);
}
EXPORT_SYMBOL_GPL(uprobe_register_refctr);
/*
* uprobe_apply - unregister an already registered probe.
* @inode: the file in which the probe has to be removed.
* @offset: offset from the start of the file.
* @uc: consumer which wants to add more or remove some breakpoints
* @add: add or remove the breakpoints
*/
int uprobe_apply(struct inode *inode, loff_t offset,
struct uprobe_consumer *uc, bool add)
{
struct uprobe *uprobe;
struct uprobe_consumer *con;
int ret = -ENOENT;
uprobe = find_uprobe(inode, offset);
if (WARN_ON(!uprobe))
return ret;
down_write(&uprobe->register_rwsem);
for (con = uprobe->consumers; con && con != uc ; con = con->next)
;
if (con)
ret = register_for_each_vma(uprobe, add ? uc : NULL);
up_write(&uprobe->register_rwsem);
put_uprobe(uprobe);
return ret;
}
static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
{
VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
int err = 0;
mmap_read_lock(mm);
for_each_vma(vmi, vma) {
unsigned long vaddr;
loff_t offset;
if (!valid_vma(vma, false) ||
file_inode(vma->vm_file) != uprobe->inode)
continue;
offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
if (uprobe->offset < offset ||
uprobe->offset >= offset + vma->vm_end - vma->vm_start)
continue;
vaddr = offset_to_vaddr(vma, uprobe->offset);
err |= remove_breakpoint(uprobe, mm, vaddr);
}
mmap_read_unlock(mm);
return err;
}
static struct rb_node *
find_node_in_range(struct inode *inode, loff_t min, loff_t max)
{
struct rb_node *n = uprobes_tree.rb_node;
while (n) {
struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
if (inode < u->inode) {
n = n->rb_left;
} else if (inode > u->inode) {
n = n->rb_right;
} else {
if (max < u->offset)
n = n->rb_left;
else if (min > u->offset)
n = n->rb_right;
else
break;
}
}
return n;
}
/*
* For a given range in vma, build a list of probes that need to be inserted.
*/
static void build_probe_list(struct inode *inode,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct list_head *head)
{
loff_t min, max;
struct rb_node *n, *t;
struct uprobe *u;
INIT_LIST_HEAD(head);
min = vaddr_to_offset(vma, start);
max = min + (end - start) - 1;
spin_lock(&uprobes_treelock);
n = find_node_in_range(inode, min, max);
if (n) {
for (t = n; t; t = rb_prev(t)) {
u = rb_entry(t, struct uprobe, rb_node);
if (u->inode != inode || u->offset < min)
break;
list_add(&u->pending_list, head);
get_uprobe(u);
}
for (t = n; (t = rb_next(t)); ) {
u = rb_entry(t, struct uprobe, rb_node);
if (u->inode != inode || u->offset > max)
break;
list_add(&u->pending_list, head);
get_uprobe(u);
}
}
spin_unlock(&uprobes_treelock);
}
/* @vma contains reference counter, not the probed instruction. */
static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
{
struct list_head *pos, *q;
struct delayed_uprobe *du;
unsigned long vaddr;
int ret = 0, err = 0;
mutex_lock(&delayed_uprobe_lock);
list_for_each_safe(pos, q, &delayed_uprobe_list) {
du = list_entry(pos, struct delayed_uprobe, list);
if (du->mm != vma->vm_mm ||
!valid_ref_ctr_vma(du->uprobe, vma))
continue;
vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
if (ret) {
update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
if (!err)
err = ret;
}
delayed_uprobe_delete(du);
}
mutex_unlock(&delayed_uprobe_lock);
return err;
}
/*
* Called from mmap_region/vma_merge with mm->mmap_lock acquired.
*
* Currently we ignore all errors and always return 0, the callers
* can't handle the failure anyway.
*/
int uprobe_mmap(struct vm_area_struct *vma)
{
struct list_head tmp_list;
struct uprobe *uprobe, *u;
struct inode *inode;
if (no_uprobe_events())
return 0;
if (vma->vm_file &&
(vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
delayed_ref_ctr_inc(vma);
if (!valid_vma(vma, true))
return 0;
inode = file_inode(vma->vm_file);
if (!inode)
return 0;
mutex_lock(uprobes_mmap_hash(inode));
build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
/*
* We can race with uprobe_unregister(), this uprobe can be already
* removed. But in this case filter_chain() must return false, all
* consumers have gone away.
*/
list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
if (!fatal_signal_pending(current) &&
filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
}
put_uprobe(uprobe);
}
mutex_unlock(uprobes_mmap_hash(inode));
return 0;
}
static bool
vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
loff_t min, max;
struct inode *inode;
struct rb_node *n;
inode = file_inode(vma->vm_file);
min = vaddr_to_offset(vma, start);
max = min + (end - start) - 1;
spin_lock(&uprobes_treelock);
n = find_node_in_range(inode, min, max);
spin_unlock(&uprobes_treelock);
return !!n;
}
/*
* Called in context of a munmap of a vma.
*/
void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
if (no_uprobe_events() || !valid_vma(vma, false))
return;
if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
return;
if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
return;
if (vma_has_uprobes(vma, start, end))
set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
}
/* Slot allocation for XOL */
static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
{
struct vm_area_struct *vma;
int ret;
if (mmap_write_lock_killable(mm))
return -EINTR;
if (mm->uprobes_state.xol_area) {
ret = -EALREADY;
goto fail;
}
if (!area->vaddr) {
/* Try to map as high as possible, this is only a hint. */
area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
PAGE_SIZE, 0, 0);
if (IS_ERR_VALUE(area->vaddr)) {
ret = area->vaddr;
goto fail;
}
}
vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
&area->xol_mapping);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto fail;
}
ret = 0;
/* pairs with get_xol_area() */
smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
fail:
mmap_write_unlock(mm);
return ret;
}
static struct xol_area *__create_xol_area(unsigned long vaddr)
{
struct mm_struct *mm = current->mm;
uprobe_opcode_t insn = UPROBE_SWBP_INSN;
struct xol_area *area;
area = kmalloc(sizeof(*area), GFP_KERNEL);
if (unlikely(!area))
goto out;
area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
GFP_KERNEL);
if (!area->bitmap)
goto free_area;
area->xol_mapping.name = "[uprobes]";
area->xol_mapping.fault = NULL;
area->xol_mapping.pages = area->pages;
area->pages[0] = alloc_page(GFP_HIGHUSER);
if (!area->pages[0])
goto free_bitmap;
area->pages[1] = NULL;
area->vaddr = vaddr;
init_waitqueue_head(&area->wq);
/* Reserve the 1st slot for get_trampoline_vaddr() */
set_bit(0, area->bitmap);
atomic_set(&area->slot_count, 1);
arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
if (!xol_add_vma(mm, area))
return area;
__free_page(area->pages[0]);
free_bitmap:
kfree(area->bitmap);
free_area:
kfree(area);
out:
return NULL;
}
/*
* get_xol_area - Allocate process's xol_area if necessary.
* This area will be used for storing instructions for execution out of line.
*
* Returns the allocated area or NULL.
*/
static struct xol_area *get_xol_area(void)
{
struct mm_struct *mm = current->mm;
struct xol_area *area;
if (!mm->uprobes_state.xol_area)
__create_xol_area(0);
/* Pairs with xol_add_vma() smp_store_release() */
area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
return area;
}
/*
* uprobe_clear_state - Free the area allocated for slots.
*/
void uprobe_clear_state(struct mm_struct *mm)
{
struct xol_area *area = mm->uprobes_state.xol_area;
mutex_lock(&delayed_uprobe_lock);
delayed_uprobe_remove(NULL, mm);
mutex_unlock(&delayed_uprobe_lock);
if (!area)
return;
put_page(area->pages[0]);
kfree(area->bitmap);
kfree(area);
}
void uprobe_start_dup_mmap(void)
{
percpu_down_read(&dup_mmap_sem);
}
void uprobe_end_dup_mmap(void)
{
percpu_up_read(&dup_mmap_sem);
}
void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
{
if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
set_bit(MMF_HAS_UPROBES, &newmm->flags);
/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
set_bit(MMF_RECALC_UPROBES, &newmm->flags);
}
}
/*
* - search for a free slot.
*/
static unsigned long xol_take_insn_slot(struct xol_area *area)
{
unsigned long slot_addr;
int slot_nr;
do {
slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
if (slot_nr < UINSNS_PER_PAGE) {
if (!test_and_set_bit(slot_nr, area->bitmap))
break;
slot_nr = UINSNS_PER_PAGE;
continue;
}
wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
} while (slot_nr >= UINSNS_PER_PAGE);
slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
atomic_inc(&area->slot_count);
return slot_addr;
}
/*
* xol_get_insn_slot - allocate a slot for xol.
* Returns the allocated slot address or 0.
*/
static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
{
struct xol_area *area;
unsigned long xol_vaddr;
area = get_xol_area();
if (!area)
return 0;
xol_vaddr = xol_take_insn_slot(area);
if (unlikely(!xol_vaddr))
return 0;
arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
&uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
return xol_vaddr;
}
/*
* xol_free_insn_slot - If slot was earlier allocated by
* @xol_get_insn_slot(), make the slot available for
* subsequent requests.
*/
static void xol_free_insn_slot(struct task_struct *tsk)
{
struct xol_area *area;
unsigned long vma_end;
unsigned long slot_addr;
if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
return;
slot_addr = tsk->utask->xol_vaddr;
if (unlikely(!slot_addr))
return;
area = tsk->mm->uprobes_state.xol_area;
vma_end = area->vaddr + PAGE_SIZE;
if (area->vaddr <= slot_addr && slot_addr < vma_end) {
unsigned long offset;
int slot_nr;
offset = slot_addr - area->vaddr;
slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
if (slot_nr >= UINSNS_PER_PAGE)
return;
clear_bit(slot_nr, area->bitmap);
atomic_dec(&area->slot_count);
smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
if (waitqueue_active(&area->wq))
wake_up(&area->wq);
tsk->utask->xol_vaddr = 0;
}
}
void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
void *src, unsigned long len)
{
/* Initialize the slot */
copy_to_page(page, vaddr, src, len);
/*
* We probably need flush_icache_user_page() but it needs vma.
* This should work on most of architectures by default. If
* architecture needs to do something different it can define
* its own version of the function.
*/
flush_dcache_page(page);
}
/**
* uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
* @regs: Reflects the saved state of the task after it has hit a breakpoint
* instruction.
* Return the address of the breakpoint instruction.
*/
unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
{
return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
}
unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
if (unlikely(utask && utask->active_uprobe))
return utask->vaddr;
return instruction_pointer(regs);
}
static struct return_instance *free_ret_instance(struct return_instance *ri)
{
struct return_instance *next = ri->next;
put_uprobe(ri->uprobe);
kfree(ri);
return next;
}
/*
* Called with no locks held.
* Called in context of an exiting or an exec-ing thread.
*/
void uprobe_free_utask(struct task_struct *t)
{
struct uprobe_task *utask = t->utask;
struct return_instance *ri;
if (!utask)
return;
if (utask->active_uprobe)
put_uprobe(utask->active_uprobe);
ri = utask->return_instances;
while (ri)
ri = free_ret_instance(ri);
xol_free_insn_slot(t);
kfree(utask);
t->utask = NULL;
}
/*
* Allocate a uprobe_task object for the task if necessary.
* Called when the thread hits a breakpoint.
*
* Returns:
* - pointer to new uprobe_task on success
* - NULL otherwise
*/
static struct uprobe_task *get_utask(void)
{
if (!current->utask)
current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
return current->utask;
}
static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
{
struct uprobe_task *n_utask;
struct return_instance **p, *o, *n;
n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
if (!n_utask)
return -ENOMEM;
t->utask = n_utask;
p = &n_utask->return_instances;
for (o = o_utask->return_instances; o; o = o->next) {
n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
if (!n)
return -ENOMEM;
*n = *o;
get_uprobe(n->uprobe);
n->next = NULL;
*p = n;
p = &n->next;
n_utask->depth++;
}
return 0;
}
static void uprobe_warn(struct task_struct *t, const char *msg)
{
pr_warn("uprobe: %s:%d failed to %s\n",
current->comm, current->pid, msg);
}
static void dup_xol_work(struct callback_head *work)
{
if (current->flags & PF_EXITING)
return;
if (!__create_xol_area(current->utask->dup_xol_addr) &&
!fatal_signal_pending(current))
uprobe_warn(current, "dup xol area");
}
/*
* Called in context of a new clone/fork from copy_process.
*/
void uprobe_copy_process(struct task_struct *t, unsigned long flags)
{
struct uprobe_task *utask = current->utask;
struct mm_struct *mm = current->mm;
struct xol_area *area;
t->utask = NULL;
if (!utask || !utask->return_instances)
return;
if (mm == t->mm && !(flags & CLONE_VFORK))
return;
if (dup_utask(t, utask))
return uprobe_warn(t, "dup ret instances");
/* The task can fork() after dup_xol_work() fails */
area = mm->uprobes_state.xol_area;
if (!area)
return uprobe_warn(t, "dup xol area");
if (mm == t->mm)
return;
t->utask->dup_xol_addr = area->vaddr;
init_task_work(&t->utask->dup_xol_work, dup_xol_work);
task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
}
/*
* Current area->vaddr notion assume the trampoline address is always
* equal area->vaddr.
*
* Returns -1 in case the xol_area is not allocated.
*/
static unsigned long get_trampoline_vaddr(void)
{
struct xol_area *area;
unsigned long trampoline_vaddr = -1;
/* Pairs with xol_add_vma() smp_store_release() */
area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
if (area)
trampoline_vaddr = area->vaddr;
return trampoline_vaddr;
}
static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
struct pt_regs *regs)
{
struct return_instance *ri = utask->return_instances;
enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
ri = free_ret_instance(ri);
utask->depth--;
}
utask->return_instances = ri;
}
static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
{
struct return_instance *ri;
struct uprobe_task *utask;
unsigned long orig_ret_vaddr, trampoline_vaddr;
bool chained;
if (!get_xol_area())
return;
utask = get_utask();
if (!utask)
return;
if (utask->depth >= MAX_URETPROBE_DEPTH) {
printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
" nestedness limit pid/tgid=%d/%d\n",
current->pid, current->tgid);
return;
}
ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
if (!ri)
return;
trampoline_vaddr = get_trampoline_vaddr();
orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
if (orig_ret_vaddr == -1)
goto fail;
/* drop the entries invalidated by longjmp() */
chained = (orig_ret_vaddr == trampoline_vaddr);
cleanup_return_instances(utask, chained, regs);
/*
* We don't want to keep trampoline address in stack, rather keep the
* original return address of first caller thru all the consequent
* instances. This also makes breakpoint unwrapping easier.
*/
if (chained) {
if (!utask->return_instances) {
/*
* This situation is not possible. Likely we have an
* attack from user-space.
*/
uprobe_warn(current, "handle tail call");
goto fail;
}
orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
}
ri->uprobe = get_uprobe(uprobe);
ri->func = instruction_pointer(regs);
ri->stack = user_stack_pointer(regs);
ri->orig_ret_vaddr = orig_ret_vaddr;
ri->chained = chained;
utask->depth++;
ri->next = utask->return_instances;
utask->return_instances = ri;
return;
fail:
kfree(ri);
}
/* Prepare to single-step probed instruction out of line. */
static int
pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
{
struct uprobe_task *utask;
unsigned long xol_vaddr;
int err;
utask = get_utask();
if (!utask)
return -ENOMEM;
xol_vaddr = xol_get_insn_slot(uprobe);
if (!xol_vaddr)
return -ENOMEM;
utask->xol_vaddr = xol_vaddr;
utask->vaddr = bp_vaddr;
err = arch_uprobe_pre_xol(&uprobe->arch, regs);
if (unlikely(err)) {
xol_free_insn_slot(current);
return err;
}
utask->active_uprobe = uprobe;
utask->state = UTASK_SSTEP;
return 0;
}
/*
* If we are singlestepping, then ensure this thread is not connected to
* non-fatal signals until completion of singlestep. When xol insn itself
* triggers the signal, restart the original insn even if the task is
* already SIGKILL'ed (since coredump should report the correct ip). This
* is even more important if the task has a handler for SIGSEGV/etc, The
* _same_ instruction should be repeated again after return from the signal
* handler, and SSTEP can never finish in this case.
*/
bool uprobe_deny_signal(void)
{
struct task_struct *t = current;
struct uprobe_task *utask = t->utask;
if (likely(!utask || !utask->active_uprobe)) return false; WARN_ON_ONCE(utask->state != UTASK_SSTEP); if (task_sigpending(t)) { spin_lock_irq(&t->sighand->siglock);
clear_tsk_thread_flag(t, TIF_SIGPENDING);
spin_unlock_irq(&t->sighand->siglock);
if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { utask->state = UTASK_SSTEP_TRAPPED;
set_tsk_thread_flag(t, TIF_UPROBE);
}
}
return true;
}
static void mmf_recalc_uprobes(struct mm_struct *mm)
{
VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
for_each_vma(vmi, vma) {
if (!valid_vma(vma, false))
continue;
/*
* This is not strictly accurate, we can race with
* uprobe_unregister() and see the already removed
* uprobe if delete_uprobe() was not yet called.
* Or this uprobe can be filtered out.
*/
if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
return;
}
clear_bit(MMF_HAS_UPROBES, &mm->flags);
}
static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
{
struct page *page;
uprobe_opcode_t opcode;
int result;
if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
return -EINVAL;
pagefault_disable();
result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
pagefault_enable();
if (likely(result == 0))
goto out;
/*
* The NULL 'tsk' here ensures that any faults that occur here
* will not be accounted to the task. 'mm' *is* current->mm,
* but we treat this as a 'remote' access since it is
* essentially a kernel access to the memory.
*/
result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page, NULL);
if (result < 0)
return result;
copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
put_page(page);
out:
/* This needs to return true for any variant of the trap insn */
return is_trap_insn(&opcode);
}
static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
{
struct mm_struct *mm = current->mm;
struct uprobe *uprobe = NULL;
struct vm_area_struct *vma;
mmap_read_lock(mm);
vma = vma_lookup(mm, bp_vaddr);
if (vma) {
if (valid_vma(vma, false)) {
struct inode *inode = file_inode(vma->vm_file);
loff_t offset = vaddr_to_offset(vma, bp_vaddr);
uprobe = find_uprobe(inode, offset);
}
if (!uprobe)
*is_swbp = is_trap_at_addr(mm, bp_vaddr);
} else {
*is_swbp = -EFAULT;
}
if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
mmf_recalc_uprobes(mm);
mmap_read_unlock(mm);
return uprobe;
}
static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
{
struct uprobe_consumer *uc;
int remove = UPROBE_HANDLER_REMOVE;
bool need_prep = false; /* prepare return uprobe, when needed */
down_read(&uprobe->register_rwsem);
for (uc = uprobe->consumers; uc; uc = uc->next) {
int rc = 0;
if (uc->handler) {
rc = uc->handler(uc, regs);
WARN(rc & ~UPROBE_HANDLER_MASK,
"bad rc=0x%x from %ps()\n", rc, uc->handler);
}
if (uc->ret_handler)
need_prep = true;
remove &= rc;
}
if (need_prep && !remove)
prepare_uretprobe(uprobe, regs); /* put bp at return */
if (remove && uprobe->consumers) {
WARN_ON(!uprobe_is_active(uprobe));
unapply_uprobe(uprobe, current->mm);
}
up_read(&uprobe->register_rwsem);
}
static void
handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
{
struct uprobe *uprobe = ri->uprobe;
struct uprobe_consumer *uc;
down_read(&uprobe->register_rwsem);
for (uc = uprobe->consumers; uc; uc = uc->next) {
if (uc->ret_handler)
uc->ret_handler(uc, ri->func, regs);
}
up_read(&uprobe->register_rwsem);
}
static struct return_instance *find_next_ret_chain(struct return_instance *ri)
{
bool chained;
do {
chained = ri->chained;
ri = ri->next; /* can't be NULL if chained */
} while (chained);
return ri;
}
static void handle_trampoline(struct pt_regs *regs)
{
struct uprobe_task *utask;
struct return_instance *ri, *next;
bool valid;
utask = current->utask;
if (!utask)
goto sigill;
ri = utask->return_instances;
if (!ri)
goto sigill;
do {
/*
* We should throw out the frames invalidated by longjmp().
* If this chain is valid, then the next one should be alive
* or NULL; the latter case means that nobody but ri->func
* could hit this trampoline on return. TODO: sigaltstack().
*/
next = find_next_ret_chain(ri);
valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
instruction_pointer_set(regs, ri->orig_ret_vaddr);
do {
if (valid)
handle_uretprobe_chain(ri, regs);
ri = free_ret_instance(ri);
utask->depth--;
} while (ri != next);
} while (!valid);
utask->return_instances = ri;
return;
sigill:
uprobe_warn(current, "handle uretprobe, sending SIGILL.");
force_sig(SIGILL);
}
bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
{
return false;
}
bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
struct pt_regs *regs)
{
return true;
}
/*
* Run handler and ask thread to singlestep.
* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
*/
static void handle_swbp(struct pt_regs *regs)
{
struct uprobe *uprobe;
unsigned long bp_vaddr;
int is_swbp;
bp_vaddr = uprobe_get_swbp_addr(regs);
if (bp_vaddr == get_trampoline_vaddr())
return handle_trampoline(regs);
uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
if (!uprobe) {
if (is_swbp > 0) {
/* No matching uprobe; signal SIGTRAP. */
force_sig(SIGTRAP);
} else {
/*
* Either we raced with uprobe_unregister() or we can't
* access this memory. The latter is only possible if
* another thread plays with our ->mm. In both cases
* we can simply restart. If this vma was unmapped we
* can pretend this insn was not executed yet and get
* the (correct) SIGSEGV after restart.
*/
instruction_pointer_set(regs, bp_vaddr);
}
return;
}
/* change it in advance for ->handler() and restart */
instruction_pointer_set(regs, bp_vaddr);
/*
* TODO: move copy_insn/etc into _register and remove this hack.
* After we hit the bp, _unregister + _register can install the
* new and not-yet-analyzed uprobe at the same address, restart.
*/
if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
goto out;
/*
* Pairs with the smp_wmb() in prepare_uprobe().
*
* Guarantees that if we see the UPROBE_COPY_INSN bit set, then
* we must also see the stores to &uprobe->arch performed by the
* prepare_uprobe() call.
*/
smp_rmb();
/* Tracing handlers use ->utask to communicate with fetch methods */
if (!get_utask())
goto out;
if (arch_uprobe_ignore(&uprobe->arch, regs))
goto out;
handler_chain(uprobe, regs);
if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
goto out;
if (!pre_ssout(uprobe, regs, bp_vaddr))
return;
/* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
out:
put_uprobe(uprobe);
}
/*
* Perform required fix-ups and disable singlestep.
* Allow pending signals to take effect.
*/
static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
{
struct uprobe *uprobe;
int err = 0;
uprobe = utask->active_uprobe;
if (utask->state == UTASK_SSTEP_ACK)
err = arch_uprobe_post_xol(&uprobe->arch, regs);
else if (utask->state == UTASK_SSTEP_TRAPPED)
arch_uprobe_abort_xol(&uprobe->arch, regs);
else
WARN_ON_ONCE(1);
put_uprobe(uprobe);
utask->active_uprobe = NULL;
utask->state = UTASK_RUNNING;
xol_free_insn_slot(current);
spin_lock_irq(¤t->sighand->siglock);
recalc_sigpending(); /* see uprobe_deny_signal() */
spin_unlock_irq(¤t->sighand->siglock);
if (unlikely(err)) {
uprobe_warn(current, "execute the probed insn, sending SIGILL.");
force_sig(SIGILL);
}
}
/*
* On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
* allows the thread to return from interrupt. After that handle_swbp()
* sets utask->active_uprobe.
*
* On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
* and allows the thread to return from interrupt.
*
* While returning to userspace, thread notices the TIF_UPROBE flag and calls
* uprobe_notify_resume().
*/
void uprobe_notify_resume(struct pt_regs *regs)
{
struct uprobe_task *utask;
clear_thread_flag(TIF_UPROBE);
utask = current->utask;
if (utask && utask->active_uprobe)
handle_singlestep(utask, regs);
else
handle_swbp(regs);
}
/*
* uprobe_pre_sstep_notifier gets called from interrupt context as part of
* notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
*/
int uprobe_pre_sstep_notifier(struct pt_regs *regs)
{
if (!current->mm)
return 0;
if (!test_bit(MMF_HAS_UPROBES, ¤t->mm->flags) &&
(!current->utask || !current->utask->return_instances))
return 0;
set_thread_flag(TIF_UPROBE);
return 1;
}
/*
* uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
* mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
*/
int uprobe_post_sstep_notifier(struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
if (!current->mm || !utask || !utask->active_uprobe)
/* task is currently not uprobed */
return 0;
utask->state = UTASK_SSTEP_ACK;
set_thread_flag(TIF_UPROBE);
return 1;
}
static struct notifier_block uprobe_exception_nb = {
.notifier_call = arch_uprobe_exception_notify,
.priority = INT_MAX-1, /* notified after kprobes, kgdb */
};
void __init uprobes_init(void)
{
int i;
for (i = 0; i < UPROBES_HASH_SZ; i++)
mutex_init(&uprobes_mmap_mutex[i]);
BUG_ON(register_die_notifier(&uprobe_exception_nb));
}
// SPDX-License-Identifier: GPL-2.0+
/*
* Restartable sequences system call
*
* Copyright (C) 2015, Google, Inc.,
* Paul Turner <pjt@google.com> and Andrew Hunter <ahh@google.com>
* Copyright (C) 2015-2018, EfficiOS Inc.,
* Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
*/
#include <linux/sched.h>
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/rseq.h>
#include <linux/types.h>
#include <asm/ptrace.h>
#define CREATE_TRACE_POINTS
#include <trace/events/rseq.h>
/* The original rseq structure size (including padding) is 32 bytes. */
#define ORIG_RSEQ_SIZE 32
#define RSEQ_CS_NO_RESTART_FLAGS (RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT | \
RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL | \
RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE)
/*
*
* Restartable sequences are a lightweight interface that allows
* user-level code to be executed atomically relative to scheduler
* preemption and signal delivery. Typically used for implementing
* per-cpu operations.
*
* It allows user-space to perform update operations on per-cpu data
* without requiring heavy-weight atomic operations.
*
* Detailed algorithm of rseq user-space assembly sequences:
*
* init(rseq_cs)
* cpu = TLS->rseq::cpu_id_start
* [1] TLS->rseq::rseq_cs = rseq_cs
* [start_ip] ----------------------------
* [2] if (cpu != TLS->rseq::cpu_id)
* goto abort_ip;
* [3] <last_instruction_in_cs>
* [post_commit_ip] ----------------------------
*
* The address of jump target abort_ip must be outside the critical
* region, i.e.:
*
* [abort_ip] < [start_ip] || [abort_ip] >= [post_commit_ip]
*
* Steps [2]-[3] (inclusive) need to be a sequence of instructions in
* userspace that can handle being interrupted between any of those
* instructions, and then resumed to the abort_ip.
*
* 1. Userspace stores the address of the struct rseq_cs assembly
* block descriptor into the rseq_cs field of the registered
* struct rseq TLS area. This update is performed through a single
* store within the inline assembly instruction sequence.
* [start_ip]
*
* 2. Userspace tests to check whether the current cpu_id field match
* the cpu number loaded before start_ip, branching to abort_ip
* in case of a mismatch.
*
* If the sequence is preempted or interrupted by a signal
* at or after start_ip and before post_commit_ip, then the kernel
* clears TLS->__rseq_abi::rseq_cs, and sets the user-space return
* ip to abort_ip before returning to user-space, so the preempted
* execution resumes at abort_ip.
*
* 3. Userspace critical section final instruction before
* post_commit_ip is the commit. The critical section is
* self-terminating.
* [post_commit_ip]
*
* 4. <success>
*
* On failure at [2], or if interrupted by preempt or signal delivery
* between [1] and [3]:
*
* [abort_ip]
* F1. <failure>
*/
static int rseq_update_cpu_node_id(struct task_struct *t)
{
struct rseq __user *rseq = t->rseq;
u32 cpu_id = raw_smp_processor_id();
u32 node_id = cpu_to_node(cpu_id);
u32 mm_cid = task_mm_cid(t);
WARN_ON_ONCE((int) mm_cid < 0); if (!user_write_access_begin(rseq, t->rseq_len))
goto efault;
unsafe_put_user(cpu_id, &rseq->cpu_id_start, efault_end);
unsafe_put_user(cpu_id, &rseq->cpu_id, efault_end); unsafe_put_user(node_id, &rseq->node_id, efault_end); unsafe_put_user(mm_cid, &rseq->mm_cid, efault_end);
/*
* Additional feature fields added after ORIG_RSEQ_SIZE
* need to be conditionally updated only if
* t->rseq_len != ORIG_RSEQ_SIZE.
*/
user_write_access_end(); trace_rseq_update(t);
return 0;
efault_end:
user_write_access_end();
efault:
return -EFAULT;
}
static int rseq_reset_rseq_cpu_node_id(struct task_struct *t)
{
u32 cpu_id_start = 0, cpu_id = RSEQ_CPU_ID_UNINITIALIZED, node_id = 0,
mm_cid = 0;
/*
* Reset cpu_id_start to its initial state (0).
*/
if (put_user(cpu_id_start, &t->rseq->cpu_id_start))
return -EFAULT;
/*
* Reset cpu_id to RSEQ_CPU_ID_UNINITIALIZED, so any user coming
* in after unregistration can figure out that rseq needs to be
* registered again.
*/
if (put_user(cpu_id, &t->rseq->cpu_id))
return -EFAULT;
/*
* Reset node_id to its initial state (0).
*/
if (put_user(node_id, &t->rseq->node_id))
return -EFAULT;
/*
* Reset mm_cid to its initial state (0).
*/
if (put_user(mm_cid, &t->rseq->mm_cid))
return -EFAULT;
/*
* Additional feature fields added after ORIG_RSEQ_SIZE
* need to be conditionally reset only if
* t->rseq_len != ORIG_RSEQ_SIZE.
*/
return 0;
}
static int rseq_get_rseq_cs(struct task_struct *t, struct rseq_cs *rseq_cs)
{
struct rseq_cs __user *urseq_cs;
u64 ptr;
u32 __user *usig;
u32 sig;
int ret;
#ifdef CONFIG_64BIT
if (get_user(ptr, &t->rseq->rseq_cs))
return -EFAULT;
#else
if (copy_from_user(&ptr, &t->rseq->rseq_cs, sizeof(ptr)))
return -EFAULT;
#endif
if (!ptr) { memset(rseq_cs, 0, sizeof(*rseq_cs));
return 0;
}
if (ptr >= TASK_SIZE)
return -EINVAL;
urseq_cs = (struct rseq_cs __user *)(unsigned long)ptr;
if (copy_from_user(rseq_cs, urseq_cs, sizeof(*rseq_cs)))
return -EFAULT;
if (rseq_cs->start_ip >= TASK_SIZE || rseq_cs->start_ip + rseq_cs->post_commit_offset >= TASK_SIZE || rseq_cs->abort_ip >= TASK_SIZE || rseq_cs->version > 0)
return -EINVAL;
/* Check for overflow. */
if (rseq_cs->start_ip + rseq_cs->post_commit_offset < rseq_cs->start_ip)
return -EINVAL;
/* Ensure that abort_ip is not in the critical section. */
if (rseq_cs->abort_ip - rseq_cs->start_ip < rseq_cs->post_commit_offset)
return -EINVAL;
usig = (u32 __user *)(unsigned long)(rseq_cs->abort_ip - sizeof(u32));
ret = get_user(sig, usig);
if (ret)
return ret;
if (current->rseq_sig != sig) { printk_ratelimited(KERN_WARNING
"Possible attack attempt. Unexpected rseq signature 0x%x, expecting 0x%x (pid=%d, addr=%p).\n",
sig, current->rseq_sig, current->pid, usig);
return -EINVAL;
}
return 0;
}
static bool rseq_warn_flags(const char *str, u32 flags)
{
u32 test_flags;
if (!flags)
return false;
test_flags = flags & RSEQ_CS_NO_RESTART_FLAGS;
if (test_flags)
pr_warn_once("Deprecated flags (%u) in %s ABI structure", test_flags, str);
test_flags = flags & ~RSEQ_CS_NO_RESTART_FLAGS;
if (test_flags)
pr_warn_once("Unknown flags (%u) in %s ABI structure", test_flags, str);
return true;
}
static int rseq_need_restart(struct task_struct *t, u32 cs_flags)
{
u32 flags, event_mask;
int ret;
if (rseq_warn_flags("rseq_cs", cs_flags))
return -EINVAL;
/* Get thread flags. */
ret = get_user(flags, &t->rseq->flags);
if (ret)
return ret;
if (rseq_warn_flags("rseq", flags))
return -EINVAL;
/*
* Load and clear event mask atomically with respect to
* scheduler preemption.
*/
preempt_disable();
event_mask = t->rseq_event_mask;
t->rseq_event_mask = 0;
preempt_enable();
return !!event_mask;
}
static int clear_rseq_cs(struct task_struct *t)
{
/*
* The rseq_cs field is set to NULL on preemption or signal
* delivery on top of rseq assembly block, as well as on top
* of code outside of the rseq assembly block. This performs
* a lazy clear of the rseq_cs field.
*
* Set rseq_cs to NULL.
*/
#ifdef CONFIG_64BIT
return put_user(0UL, &t->rseq->rseq_cs);
#else
if (clear_user(&t->rseq->rseq_cs, sizeof(t->rseq->rseq_cs)))
return -EFAULT;
return 0;
#endif
}
/*
* Unsigned comparison will be true when ip >= start_ip, and when
* ip < start_ip + post_commit_offset.
*/
static bool in_rseq_cs(unsigned long ip, struct rseq_cs *rseq_cs)
{
return ip - rseq_cs->start_ip < rseq_cs->post_commit_offset;
}
static int rseq_ip_fixup(struct pt_regs *regs)
{
unsigned long ip = instruction_pointer(regs);
struct task_struct *t = current;
struct rseq_cs rseq_cs;
int ret;
ret = rseq_get_rseq_cs(t, &rseq_cs); if (ret)
return ret;
/*
* Handle potentially not being within a critical section.
* If not nested over a rseq critical section, restart is useless.
* Clear the rseq_cs pointer and return.
*/
if (!in_rseq_cs(ip, &rseq_cs)) return clear_rseq_cs(t); ret = rseq_need_restart(t, rseq_cs.flags); if (ret <= 0)
return ret;
ret = clear_rseq_cs(t);
if (ret)
return ret;
trace_rseq_ip_fixup(ip, rseq_cs.start_ip, rseq_cs.post_commit_offset, rseq_cs.abort_ip); instruction_pointer_set(regs, (unsigned long)rseq_cs.abort_ip);
return 0;
}
/*
* This resume handler must always be executed between any of:
* - preemption,
* - signal delivery,
* and return to user-space.
*
* This is how we can ensure that the entire rseq critical section
* will issue the commit instruction only if executed atomically with
* respect to other threads scheduled on the same CPU, and with respect
* to signal handlers.
*/
void __rseq_handle_notify_resume(struct ksignal *ksig, struct pt_regs *regs)
{
struct task_struct *t = current;
int ret, sig;
if (unlikely(t->flags & PF_EXITING))
return;
/*
* regs is NULL if and only if the caller is in a syscall path. Skip
* fixup and leave rseq_cs as is so that rseq_sycall() will detect and
* kill a misbehaving userspace on debug kernels.
*/
if (regs) { ret = rseq_ip_fixup(regs); if (unlikely(ret < 0))
goto error;
}
if (unlikely(rseq_update_cpu_node_id(t)))
goto error;
return;
error:
sig = ksig ? ksig->sig : 0; force_sigsegv(sig);
}
#ifdef CONFIG_DEBUG_RSEQ
/*
* Terminate the process if a syscall is issued within a restartable
* sequence.
*/
void rseq_syscall(struct pt_regs *regs)
{
unsigned long ip = instruction_pointer(regs);
struct task_struct *t = current;
struct rseq_cs rseq_cs;
if (!t->rseq)
return;
if (rseq_get_rseq_cs(t, &rseq_cs) || in_rseq_cs(ip, &rseq_cs))
force_sig(SIGSEGV);
}
#endif
/*
* sys_rseq - setup restartable sequences for caller thread.
*/
SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len,
int, flags, u32, sig)
{
int ret;
if (flags & RSEQ_FLAG_UNREGISTER) {
if (flags & ~RSEQ_FLAG_UNREGISTER)
return -EINVAL;
/* Unregister rseq for current thread. */
if (current->rseq != rseq || !current->rseq)
return -EINVAL;
if (rseq_len != current->rseq_len)
return -EINVAL;
if (current->rseq_sig != sig)
return -EPERM;
ret = rseq_reset_rseq_cpu_node_id(current);
if (ret)
return ret;
current->rseq = NULL;
current->rseq_sig = 0;
current->rseq_len = 0;
return 0;
}
if (unlikely(flags))
return -EINVAL;
if (current->rseq) {
/*
* If rseq is already registered, check whether
* the provided address differs from the prior
* one.
*/
if (current->rseq != rseq || rseq_len != current->rseq_len)
return -EINVAL;
if (current->rseq_sig != sig)
return -EPERM;
/* Already registered. */
return -EBUSY;
}
/*
* If there was no rseq previously registered, ensure the provided rseq
* is properly aligned, as communcated to user-space through the ELF
* auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq
* size, the required alignment is the original struct rseq alignment.
*
* In order to be valid, rseq_len is either the original rseq size, or
* large enough to contain all supported fields, as communicated to
* user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE.
*/
if (rseq_len < ORIG_RSEQ_SIZE ||
(rseq_len == ORIG_RSEQ_SIZE && !IS_ALIGNED((unsigned long)rseq, ORIG_RSEQ_SIZE)) ||
(rseq_len != ORIG_RSEQ_SIZE && (!IS_ALIGNED((unsigned long)rseq, __alignof__(*rseq)) ||
rseq_len < offsetof(struct rseq, end))))
return -EINVAL;
if (!access_ok(rseq, rseq_len))
return -EFAULT;
current->rseq = rseq;
current->rseq_len = rseq_len;
current->rseq_sig = sig;
/*
* If rseq was previously inactive, and has just been
* registered, ensure the cpu_id_start and cpu_id fields
* are updated before returning to user-space.
*/
rseq_set_notify_resume(current);
return 0;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __KERNEL_PRINTK__
#define __KERNEL_PRINTK__
#include <linux/stdarg.h>
#include <linux/init.h>
#include <linux/kern_levels.h>
#include <linux/linkage.h>
#include <linux/ratelimit_types.h>
#include <linux/once_lite.h>
extern const char linux_banner[];
extern const char linux_proc_banner[];
extern int oops_in_progress; /* If set, an oops, panic(), BUG() or die() is in progress */
#define PRINTK_MAX_SINGLE_HEADER_LEN 2
static inline int printk_get_level(const char *buffer)
{
if (buffer[0] == KERN_SOH_ASCII && buffer[1]) { switch (buffer[1]) {
case '0' ... '7':
case 'c': /* KERN_CONT */
return buffer[1];
}
}
return 0;
}
static inline const char *printk_skip_level(const char *buffer)
{
if (printk_get_level(buffer))
return buffer + 2;
return buffer;
}
static inline const char *printk_skip_headers(const char *buffer)
{
while (printk_get_level(buffer))
buffer = printk_skip_level(buffer);
return buffer;
}
/* printk's without a loglevel use this.. */
#define MESSAGE_LOGLEVEL_DEFAULT CONFIG_MESSAGE_LOGLEVEL_DEFAULT
/* We show everything that is MORE important than this.. */
#define CONSOLE_LOGLEVEL_SILENT 0 /* Mum's the word */
#define CONSOLE_LOGLEVEL_MIN 1 /* Minimum loglevel we let people use */
#define CONSOLE_LOGLEVEL_DEBUG 10 /* issue debug messages */
#define CONSOLE_LOGLEVEL_MOTORMOUTH 15 /* You can't shut this one up */
/*
* Default used to be hard-coded at 7, quiet used to be hardcoded at 4,
* we're now allowing both to be set from kernel config.
*/
#define CONSOLE_LOGLEVEL_DEFAULT CONFIG_CONSOLE_LOGLEVEL_DEFAULT
#define CONSOLE_LOGLEVEL_QUIET CONFIG_CONSOLE_LOGLEVEL_QUIET
extern int console_printk[];
#define console_loglevel (console_printk[0])
#define default_message_loglevel (console_printk[1])
#define minimum_console_loglevel (console_printk[2])
#define default_console_loglevel (console_printk[3])
extern void console_verbose(void);
/* strlen("ratelimit") + 1 */
#define DEVKMSG_STR_MAX_SIZE 10
extern char devkmsg_log_str[];
struct ctl_table;
extern int suppress_printk;
struct va_format {
const char *fmt;
va_list *va;
};
/*
* FW_BUG
* Add this to a message where you are sure the firmware is buggy or behaves
* really stupid or out of spec. Be aware that the responsible BIOS developer
* should be able to fix this issue or at least get a concrete idea of the
* problem by reading your message without the need of looking at the kernel
* code.
*
* Use it for definite and high priority BIOS bugs.
*
* FW_WARN
* Use it for not that clear (e.g. could the kernel messed up things already?)
* and medium priority BIOS bugs.
*
* FW_INFO
* Use this one if you want to tell the user or vendor about something
* suspicious, but generally harmless related to the firmware.
*
* Use it for information or very low priority BIOS bugs.
*/
#define FW_BUG "[Firmware Bug]: "
#define FW_WARN "[Firmware Warn]: "
#define FW_INFO "[Firmware Info]: "
/*
* HW_ERR
* Add this to a message for hardware errors, so that user can report
* it to hardware vendor instead of LKML or software vendor.
*/
#define HW_ERR "[Hardware Error]: "
/*
* DEPRECATED
* Add this to a message whenever you want to warn user space about the use
* of a deprecated aspect of an API so they can stop using it
*/
#define DEPRECATED "[Deprecated]: "
/*
* Dummy printk for disabled debugging statements to use whilst maintaining
* gcc's format checking.
*/
#define no_printk(fmt, ...) \
({ \
if (0) \
printk(fmt, ##__VA_ARGS__); \
0; \
})
#ifdef CONFIG_EARLY_PRINTK
extern asmlinkage __printf(1, 2)
void early_printk(const char *fmt, ...);
#else
static inline __printf(1, 2) __cold
void early_printk(const char *s, ...) { }
#endif
struct dev_printk_info;
#ifdef CONFIG_PRINTK
asmlinkage __printf(4, 0)
int vprintk_emit(int facility, int level,
const struct dev_printk_info *dev_info,
const char *fmt, va_list args);
asmlinkage __printf(1, 0)
int vprintk(const char *fmt, va_list args);
asmlinkage __printf(1, 2) __cold
int _printk(const char *fmt, ...);
/*
* Special printk facility for scheduler/timekeeping use only, _DO_NOT_USE_ !
*/
__printf(1, 2) __cold int _printk_deferred(const char *fmt, ...);
extern void __printk_safe_enter(void);
extern void __printk_safe_exit(void);
/*
* The printk_deferred_enter/exit macros are available only as a hack for
* some code paths that need to defer all printk console printing. Interrupts
* must be disabled for the deferred duration.
*/
#define printk_deferred_enter __printk_safe_enter
#define printk_deferred_exit __printk_safe_exit
/*
* Please don't use printk_ratelimit(), because it shares ratelimiting state
* with all other unrelated printk_ratelimit() callsites. Instead use
* printk_ratelimited() or plain old __ratelimit().
*/
extern int __printk_ratelimit(const char *func);
#define printk_ratelimit() __printk_ratelimit(__func__)
extern bool printk_timed_ratelimit(unsigned long *caller_jiffies,
unsigned int interval_msec);
extern int printk_delay_msec;
extern int dmesg_restrict;
extern void wake_up_klogd(void);
char *log_buf_addr_get(void);
u32 log_buf_len_get(void);
void log_buf_vmcoreinfo_setup(void);
void __init setup_log_buf(int early);
__printf(1, 2) void dump_stack_set_arch_desc(const char *fmt, ...);
void dump_stack_print_info(const char *log_lvl);
void show_regs_print_info(const char *log_lvl);
extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold;
extern asmlinkage void dump_stack(void) __cold;
void printk_trigger_flush(void);
#else
static inline __printf(1, 0)
int vprintk(const char *s, va_list args)
{
return 0;
}
static inline __printf(1, 2) __cold
int _printk(const char *s, ...)
{
return 0;
}
static inline __printf(1, 2) __cold
int _printk_deferred(const char *s, ...)
{
return 0;
}
static inline void printk_deferred_enter(void)
{
}
static inline void printk_deferred_exit(void)
{
}
static inline int printk_ratelimit(void)
{
return 0;
}
static inline bool printk_timed_ratelimit(unsigned long *caller_jiffies,
unsigned int interval_msec)
{
return false;
}
static inline void wake_up_klogd(void)
{
}
static inline char *log_buf_addr_get(void)
{
return NULL;
}
static inline u32 log_buf_len_get(void)
{
return 0;
}
static inline void log_buf_vmcoreinfo_setup(void)
{
}
static inline void setup_log_buf(int early)
{
}
static inline __printf(1, 2) void dump_stack_set_arch_desc(const char *fmt, ...)
{
}
static inline void dump_stack_print_info(const char *log_lvl)
{
}
static inline void show_regs_print_info(const char *log_lvl)
{
}
static inline void dump_stack_lvl(const char *log_lvl)
{
}
static inline void dump_stack(void)
{
}
static inline void printk_trigger_flush(void)
{
}
#endif
bool this_cpu_in_panic(void);
#ifdef CONFIG_SMP
extern int __printk_cpu_sync_try_get(void);
extern void __printk_cpu_sync_wait(void);
extern void __printk_cpu_sync_put(void);
#else
#define __printk_cpu_sync_try_get() true
#define __printk_cpu_sync_wait()
#define __printk_cpu_sync_put()
#endif /* CONFIG_SMP */
/**
* printk_cpu_sync_get_irqsave() - Disable interrupts and acquire the printk
* cpu-reentrant spinning lock.
* @flags: Stack-allocated storage for saving local interrupt state,
* to be passed to printk_cpu_sync_put_irqrestore().
*
* If the lock is owned by another CPU, spin until it becomes available.
* Interrupts are restored while spinning.
*
* CAUTION: This function must be used carefully. It does not behave like a
* typical lock. Here are important things to watch out for...
*
* * This function is reentrant on the same CPU. Therefore the calling
* code must not assume exclusive access to data if code accessing the
* data can run reentrant or within NMI context on the same CPU.
*
* * If there exists usage of this function from NMI context, it becomes
* unsafe to perform any type of locking or spinning to wait for other
* CPUs after calling this function from any context. This includes
* using spinlocks or any other busy-waiting synchronization methods.
*/
#define printk_cpu_sync_get_irqsave(flags) \
for (;;) { \
local_irq_save(flags); \
if (__printk_cpu_sync_try_get()) \
break; \
local_irq_restore(flags); \
__printk_cpu_sync_wait(); \
}
/**
* printk_cpu_sync_put_irqrestore() - Release the printk cpu-reentrant spinning
* lock and restore interrupts.
* @flags: Caller's saved interrupt state, from printk_cpu_sync_get_irqsave().
*/
#define printk_cpu_sync_put_irqrestore(flags) \
do { \
__printk_cpu_sync_put(); \
local_irq_restore(flags); \
} while (0)
extern int kptr_restrict;
/**
* pr_fmt - used by the pr_*() macros to generate the printk format string
* @fmt: format string passed from a pr_*() macro
*
* This macro can be used to generate a unified format string for pr_*()
* macros. A common use is to prefix all pr_*() messages in a file with a common
* string. For example, defining this at the top of a source file:
*
* #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
*
* would prefix all pr_info, pr_emerg... messages in the file with the module
* name.
*/
#ifndef pr_fmt
#define pr_fmt(fmt) fmt
#endif
struct module;
#ifdef CONFIG_PRINTK_INDEX
struct pi_entry {
const char *fmt;
const char *func;
const char *file;
unsigned int line;
/*
* While printk and pr_* have the level stored in the string at compile
* time, some subsystems dynamically add it at runtime through the
* format string. For these dynamic cases, we allow the subsystem to
* tell us the level at compile time.
*
* NULL indicates that the level, if any, is stored in fmt.
*/
const char *level;
/*
* The format string used by various subsystem specific printk()
* wrappers to prefix the message.
*
* Note that the static prefix defined by the pr_fmt() macro is stored
* directly in the message format (@fmt), not here.
*/
const char *subsys_fmt_prefix;
} __packed;
#define __printk_index_emit(_fmt, _level, _subsys_fmt_prefix) \
do { \
if (__builtin_constant_p(_fmt) && __builtin_constant_p(_level)) { \
/*
* We check __builtin_constant_p multiple times here
* for the same input because GCC will produce an error
* if we try to assign a static variable to fmt if it
* is not a constant, even with the outer if statement.
*/ \
static const struct pi_entry _entry \
__used = { \
.fmt = __builtin_constant_p(_fmt) ? (_fmt) : NULL, \
.func = __func__, \
.file = __FILE__, \
.line = __LINE__, \
.level = __builtin_constant_p(_level) ? (_level) : NULL, \
.subsys_fmt_prefix = _subsys_fmt_prefix,\
}; \
static const struct pi_entry *_entry_ptr \
__used __section(".printk_index") = &_entry; \
} \
} while (0)
#else /* !CONFIG_PRINTK_INDEX */
#define __printk_index_emit(...) do {} while (0)
#endif /* CONFIG_PRINTK_INDEX */
/*
* Some subsystems have their own custom printk that applies a va_format to a
* generic format, for example, to include a device number or other metadata
* alongside the format supplied by the caller.
*
* In order to store these in the way they would be emitted by the printk
* infrastructure, the subsystem provides us with the start, fixed string, and
* any subsequent text in the format string.
*
* We take a variable argument list as pr_fmt/dev_fmt/etc are sometimes passed
* as multiple arguments (eg: `"%s: ", "blah"`), and we must only take the
* first one.
*
* subsys_fmt_prefix must be known at compile time, or compilation will fail
* (since this is a mistake). If fmt or level is not known at compile time, no
* index entry will be made (since this can legitimately happen).
*/
#define printk_index_subsys_emit(subsys_fmt_prefix, level, fmt, ...) \
__printk_index_emit(fmt, level, subsys_fmt_prefix)
#define printk_index_wrap(_p_func, _fmt, ...) \
({ \
__printk_index_emit(_fmt, NULL, NULL); \
_p_func(_fmt, ##__VA_ARGS__); \
})
/**
* printk - print a kernel message
* @fmt: format string
*
* This is printk(). It can be called from any context. We want it to work.
*
* If printk indexing is enabled, _printk() is called from printk_index_wrap.
* Otherwise, printk is simply #defined to _printk.
*
* We try to grab the console_lock. If we succeed, it's easy - we log the
* output and call the console drivers. If we fail to get the semaphore, we
* place the output into the log buffer and return. The current holder of
* the console_sem will notice the new output in console_unlock(); and will
* send it to the consoles before releasing the lock.
*
* One effect of this deferred printing is that code which calls printk() and
* then changes console_loglevel may break. This is because console_loglevel
* is inspected when the actual printing occurs.
*
* See also:
* printf(3)
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
#define printk(fmt, ...) printk_index_wrap(_printk, fmt, ##__VA_ARGS__)
#define printk_deferred(fmt, ...) \
printk_index_wrap(_printk_deferred, fmt, ##__VA_ARGS__)
/**
* pr_emerg - Print an emergency-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_EMERG loglevel. It uses pr_fmt() to
* generate the format string.
*/
#define pr_emerg(fmt, ...) \
printk(KERN_EMERG pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_alert - Print an alert-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_ALERT loglevel. It uses pr_fmt() to
* generate the format string.
*/
#define pr_alert(fmt, ...) \
printk(KERN_ALERT pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_crit - Print a critical-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_CRIT loglevel. It uses pr_fmt() to
* generate the format string.
*/
#define pr_crit(fmt, ...) \
printk(KERN_CRIT pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_err - Print an error-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_ERR loglevel. It uses pr_fmt() to
* generate the format string.
*/
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_warn - Print a warning-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_WARNING loglevel. It uses pr_fmt()
* to generate the format string.
*/
#define pr_warn(fmt, ...) \
printk(KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_notice - Print a notice-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_NOTICE loglevel. It uses pr_fmt() to
* generate the format string.
*/
#define pr_notice(fmt, ...) \
printk(KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_info - Print an info-level message
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_INFO loglevel. It uses pr_fmt() to
* generate the format string.
*/
#define pr_info(fmt, ...) \
printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__)
/**
* pr_cont - Continues a previous log message in the same line.
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_CONT loglevel. It should only be
* used when continuing a log message with no newline ('\n') enclosed. Otherwise
* it defaults back to KERN_DEFAULT loglevel.
*/
#define pr_cont(fmt, ...) \
printk(KERN_CONT fmt, ##__VA_ARGS__)
/**
* pr_devel - Print a debug-level message conditionally
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to a printk with KERN_DEBUG loglevel if DEBUG is
* defined. Otherwise it does nothing.
*
* It uses pr_fmt() to generate the format string.
*/
#ifdef DEBUG
#define pr_devel(fmt, ...) \
printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#else
#define pr_devel(fmt, ...) \
no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#endif
/* If you are writing a driver, please use dev_dbg instead */
#if defined(CONFIG_DYNAMIC_DEBUG) || \
(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
#include <linux/dynamic_debug.h>
/**
* pr_debug - Print a debug-level message conditionally
* @fmt: format string
* @...: arguments for the format string
*
* This macro expands to dynamic_pr_debug() if CONFIG_DYNAMIC_DEBUG is
* set. Otherwise, if DEBUG is defined, it's equivalent to a printk with
* KERN_DEBUG loglevel. If DEBUG is not defined it does nothing.
*
* It uses pr_fmt() to generate the format string (dynamic_pr_debug() uses
* pr_fmt() internally).
*/
#define pr_debug(fmt, ...) \
dynamic_pr_debug(fmt, ##__VA_ARGS__)
#elif defined(DEBUG)
#define pr_debug(fmt, ...) \
printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#else
#define pr_debug(fmt, ...) \
no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#endif
/*
* Print a one-time message (analogous to WARN_ONCE() et al):
*/
#ifdef CONFIG_PRINTK
#define printk_once(fmt, ...) \
DO_ONCE_LITE(printk, fmt, ##__VA_ARGS__)
#define printk_deferred_once(fmt, ...) \
DO_ONCE_LITE(printk_deferred, fmt, ##__VA_ARGS__)
#else
#define printk_once(fmt, ...) \
no_printk(fmt, ##__VA_ARGS__)
#define printk_deferred_once(fmt, ...) \
no_printk(fmt, ##__VA_ARGS__)
#endif
#define pr_emerg_once(fmt, ...) \
printk_once(KERN_EMERG pr_fmt(fmt), ##__VA_ARGS__)
#define pr_alert_once(fmt, ...) \
printk_once(KERN_ALERT pr_fmt(fmt), ##__VA_ARGS__)
#define pr_crit_once(fmt, ...) \
printk_once(KERN_CRIT pr_fmt(fmt), ##__VA_ARGS__)
#define pr_err_once(fmt, ...) \
printk_once(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
#define pr_warn_once(fmt, ...) \
printk_once(KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
#define pr_notice_once(fmt, ...) \
printk_once(KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__)
#define pr_info_once(fmt, ...) \
printk_once(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__)
/* no pr_cont_once, don't do that... */
#if defined(DEBUG)
#define pr_devel_once(fmt, ...) \
printk_once(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#else
#define pr_devel_once(fmt, ...) \
no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#endif
/* If you are writing a driver, please use dev_dbg instead */
#if defined(DEBUG)
#define pr_debug_once(fmt, ...) \
printk_once(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#else
#define pr_debug_once(fmt, ...) \
no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#endif
/*
* ratelimited messages with local ratelimit_state,
* no local ratelimit_state used in the !PRINTK case
*/
#ifdef CONFIG_PRINTK
#define printk_ratelimited(fmt, ...) \
({ \
static DEFINE_RATELIMIT_STATE(_rs, \
DEFAULT_RATELIMIT_INTERVAL, \
DEFAULT_RATELIMIT_BURST); \
\
if (__ratelimit(&_rs)) \
printk(fmt, ##__VA_ARGS__); \
})
#else
#define printk_ratelimited(fmt, ...) \
no_printk(fmt, ##__VA_ARGS__)
#endif
#define pr_emerg_ratelimited(fmt, ...) \
printk_ratelimited(KERN_EMERG pr_fmt(fmt), ##__VA_ARGS__)
#define pr_alert_ratelimited(fmt, ...) \
printk_ratelimited(KERN_ALERT pr_fmt(fmt), ##__VA_ARGS__)
#define pr_crit_ratelimited(fmt, ...) \
printk_ratelimited(KERN_CRIT pr_fmt(fmt), ##__VA_ARGS__)
#define pr_err_ratelimited(fmt, ...) \
printk_ratelimited(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
#define pr_warn_ratelimited(fmt, ...) \
printk_ratelimited(KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
#define pr_notice_ratelimited(fmt, ...) \
printk_ratelimited(KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__)
#define pr_info_ratelimited(fmt, ...) \
printk_ratelimited(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__)
/* no pr_cont_ratelimited, don't do that... */
#if defined(DEBUG)
#define pr_devel_ratelimited(fmt, ...) \
printk_ratelimited(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#else
#define pr_devel_ratelimited(fmt, ...) \
no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#endif
/* If you are writing a driver, please use dev_dbg instead */
#if defined(CONFIG_DYNAMIC_DEBUG) || \
(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
/* descriptor check is first to prevent flooding with "callbacks suppressed" */
#define pr_debug_ratelimited(fmt, ...) \
do { \
static DEFINE_RATELIMIT_STATE(_rs, \
DEFAULT_RATELIMIT_INTERVAL, \
DEFAULT_RATELIMIT_BURST); \
DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, pr_fmt(fmt)); \
if (DYNAMIC_DEBUG_BRANCH(descriptor) && \
__ratelimit(&_rs)) \
__dynamic_pr_debug(&descriptor, pr_fmt(fmt), ##__VA_ARGS__); \
} while (0)
#elif defined(DEBUG)
#define pr_debug_ratelimited(fmt, ...) \
printk_ratelimited(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#else
#define pr_debug_ratelimited(fmt, ...) \
no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__)
#endif
extern const struct file_operations kmsg_fops;
enum {
DUMP_PREFIX_NONE,
DUMP_PREFIX_ADDRESS,
DUMP_PREFIX_OFFSET
};
extern int hex_dump_to_buffer(const void *buf, size_t len, int rowsize,
int groupsize, char *linebuf, size_t linebuflen,
bool ascii);
#ifdef CONFIG_PRINTK
extern void print_hex_dump(const char *level, const char *prefix_str,
int prefix_type, int rowsize, int groupsize,
const void *buf, size_t len, bool ascii);
#else
static inline void print_hex_dump(const char *level, const char *prefix_str,
int prefix_type, int rowsize, int groupsize,
const void *buf, size_t len, bool ascii)
{
}
static inline void print_hex_dump_bytes(const char *prefix_str, int prefix_type,
const void *buf, size_t len)
{
}
#endif
#if defined(CONFIG_DYNAMIC_DEBUG) || \
(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
#define print_hex_dump_debug(prefix_str, prefix_type, rowsize, \
groupsize, buf, len, ascii) \
dynamic_hex_dump(prefix_str, prefix_type, rowsize, \
groupsize, buf, len, ascii)
#elif defined(DEBUG)
#define print_hex_dump_debug(prefix_str, prefix_type, rowsize, \
groupsize, buf, len, ascii) \
print_hex_dump(KERN_DEBUG, prefix_str, prefix_type, rowsize, \
groupsize, buf, len, ascii)
#else
static inline void print_hex_dump_debug(const char *prefix_str, int prefix_type,
int rowsize, int groupsize,
const void *buf, size_t len, bool ascii)
{
}
#endif
/**
* print_hex_dump_bytes - shorthand form of print_hex_dump() with default params
* @prefix_str: string to prefix each line with;
* caller supplies trailing spaces for alignment if desired
* @prefix_type: controls whether prefix of an offset, address, or none
* is printed (%DUMP_PREFIX_OFFSET, %DUMP_PREFIX_ADDRESS, %DUMP_PREFIX_NONE)
* @buf: data blob to dump
* @len: number of bytes in the @buf
*
* Calls print_hex_dump(), with log level of KERN_DEBUG,
* rowsize of 16, groupsize of 1, and ASCII output included.
*/
#define print_hex_dump_bytes(prefix_str, prefix_type, buf, len) \
print_hex_dump_debug(prefix_str, prefix_type, 16, 1, buf, len, true)
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* of.c The helpers for hcd device tree support
*
* Copyright (C) 2016 Freescale Semiconductor, Inc.
* Author: Peter Chen <peter.chen@freescale.com>
* Copyright (C) 2017 Johan Hovold <johan@kernel.org>
*/
#include <linux/of.h>
#include <linux/of_graph.h>
#include <linux/usb/of.h>
/**
* usb_of_get_device_node() - get a USB device node
* @hub: hub to which device is connected
* @port1: one-based index of port
*
* Look up the node of a USB device given its parent hub device and one-based
* port number.
*
* Return: A pointer to the node with incremented refcount if found, or
* %NULL otherwise.
*/
struct device_node *usb_of_get_device_node(struct usb_device *hub, int port1)
{
struct device_node *node;
u32 reg; for_each_child_of_node(hub->dev.of_node, node) { if (of_property_read_u32(node, "reg", ®))
continue;
if (reg == port1)
return node;
}
return NULL;
}
EXPORT_SYMBOL_GPL(usb_of_get_device_node);
/**
* usb_of_has_combined_node() - determine whether a device has a combined node
* @udev: USB device
*
* Determine whether a USB device has a so called combined node which is
* shared with its sole interface. This is the case if and only if the device
* has a node and its descriptors report the following:
*
* 1) bDeviceClass is 0 or 9, and
* 2) bNumConfigurations is 1, and
* 3) bNumInterfaces is 1.
*
* Return: True iff the device has a device node and its descriptors match the
* criteria for a combined node.
*/
bool usb_of_has_combined_node(struct usb_device *udev)
{
struct usb_device_descriptor *ddesc = &udev->descriptor;
struct usb_config_descriptor *cdesc;
if (!udev->dev.of_node)
return false;
switch (ddesc->bDeviceClass) {
case USB_CLASS_PER_INTERFACE:
case USB_CLASS_HUB:
if (ddesc->bNumConfigurations == 1) {
cdesc = &udev->config->desc;
if (cdesc->bNumInterfaces == 1)
return true;
}
}
return false;
}
EXPORT_SYMBOL_GPL(usb_of_has_combined_node);
static bool usb_of_has_devices_or_graph(const struct usb_device *hub)
{
const struct device_node *np = hub->dev.of_node;
struct device_node *child;
if (of_graph_is_present(np))
return true;
for_each_child_of_node(np, child)
if (of_property_present(child, "reg"))
return true;
return false;
}
/**
* usb_of_get_connect_type() - get a USB hub's port connect_type
* @hub: hub to which port is for @port1
* @port1: one-based index of port
*
* Get the connect_type of @port1 based on the device node for @hub. If the
* port is described in the OF graph, the connect_type is "hotplug". If the
* @hub has a child device has with a 'reg' property equal to @port1 the
* connect_type is "hard-wired". If there isn't an OF graph or child node at
* all then the connect_type is "unknown". Otherwise, the port is considered
* "unused" because it isn't described at all.
*
* Return: A connect_type for @port1 based on the device node for @hub.
*/
enum usb_port_connect_type usb_of_get_connect_type(struct usb_device *hub, int port1)
{
struct device_node *np, *child, *ep, *remote_np;
enum usb_port_connect_type connect_type;
/* Only set connect_type if binding has ports/hardwired devices. */
if (!usb_of_has_devices_or_graph(hub))
return USB_PORT_CONNECT_TYPE_UNKNOWN;
/* Assume port is unused if there's a graph or a child node. */
connect_type = USB_PORT_NOT_USED;
np = hub->dev.of_node;
/*
* Hotplug ports are connected to an available remote node, e.g.
* usb-a-connector compatible node, in the OF graph.
*/
if (of_graph_is_present(np)) {
ep = of_graph_get_endpoint_by_regs(np, port1, -1);
if (ep) {
remote_np = of_graph_get_remote_port_parent(ep);
of_node_put(ep);
if (of_device_is_available(remote_np))
connect_type = USB_PORT_CONNECT_TYPE_HOT_PLUG;
of_node_put(remote_np);
}
}
/*
* Hard-wired ports are child nodes with a reg property corresponding
* to the port number, i.e. a usb device.
*/
child = usb_of_get_device_node(hub, port1);
if (of_device_is_available(child))
connect_type = USB_PORT_CONNECT_TYPE_HARD_WIRED;
of_node_put(child);
return connect_type;
}
EXPORT_SYMBOL_GPL(usb_of_get_connect_type);
/**
* usb_of_get_interface_node() - get a USB interface node
* @udev: USB device of interface
* @config: configuration value
* @ifnum: interface number
*
* Look up the node of a USB interface given its USB device, configuration
* value and interface number.
*
* Return: A pointer to the node with incremented refcount if found, or
* %NULL otherwise.
*/
struct device_node *
usb_of_get_interface_node(struct usb_device *udev, u8 config, u8 ifnum)
{
struct device_node *node;
u32 reg[2];
for_each_child_of_node(udev->dev.of_node, node) {
if (of_property_read_u32_array(node, "reg", reg, 2))
continue;
if (reg[0] == ifnum && reg[1] == config)
return node;
}
return NULL;
}
EXPORT_SYMBOL_GPL(usb_of_get_interface_node);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* ALSA sequencer System services Client
* Copyright (c) 1998-1999 by Frank van de Pol <fvdpol@coil.demon.nl>
*/
#include <linux/init.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <sound/core.h>
#include "seq_system.h"
#include "seq_timer.h"
#include "seq_queue.h"
/* internal client that provide system services, access to timer etc. */
/*
* Port "Timer"
* - send tempo /start/stop etc. events to this port to manipulate the
* queue's timer. The queue address is specified in
* data.queue.queue.
* - this port supports subscription. The received timer events are
* broadcasted to all subscribed clients. The modified tempo
* value is stored on data.queue.value.
* The modifier client/port is not send.
*
* Port "Announce"
* - does not receive message
* - supports supscription. For each client or port attaching to or
* detaching from the system an announcement is send to the subscribed
* clients.
*
* Idea: the subscription mechanism might also work handy for distributing
* synchronisation and timing information. In this case we would ideally have
* a list of subscribers for each type of sync (time, tick), for each timing
* queue.
*
* NOTE: the queue to be started, stopped, etc. must be specified
* in data.queue.addr.queue field. queue is used only for
* scheduling, and no longer referred as affected queue.
* They are used only for timer broadcast (see above).
* -- iwai
*/
/* client id of our system client */
static int sysclient = -1;
/* port id numbers for this client */
static int announce_port = -1;
/* fill standard header data, source port & channel are filled in */
static int setheader(struct snd_seq_event * ev, int client, int port)
{
if (announce_port < 0)
return -ENODEV;
memset(ev, 0, sizeof(struct snd_seq_event));
ev->flags &= ~SNDRV_SEQ_EVENT_LENGTH_MASK;
ev->flags |= SNDRV_SEQ_EVENT_LENGTH_FIXED;
ev->source.client = sysclient;
ev->source.port = announce_port;
ev->dest.client = SNDRV_SEQ_ADDRESS_SUBSCRIBERS;
/* fill data */
/*ev->data.addr.queue = SNDRV_SEQ_ADDRESS_UNKNOWN;*/
ev->data.addr.client = client;
ev->data.addr.port = port;
return 0;
}
/* entry points for broadcasting system events */
void snd_seq_system_broadcast(int client, int port, int type)
{
struct snd_seq_event ev; if (setheader(&ev, client, port) < 0)
return;
ev.type = type;
snd_seq_kernel_client_dispatch(sysclient, &ev, 0, 0);
}
EXPORT_SYMBOL_GPL(snd_seq_system_broadcast);
/* entry points for broadcasting system events */
int snd_seq_system_notify(int client, int port, struct snd_seq_event *ev)
{
ev->flags = SNDRV_SEQ_EVENT_LENGTH_FIXED;
ev->source.client = sysclient;
ev->source.port = announce_port;
ev->dest.client = client;
ev->dest.port = port;
return snd_seq_kernel_client_dispatch(sysclient, ev, 0, 0);
}
/* call-back handler for timer events */
static int event_input_timer(struct snd_seq_event * ev, int direct, void *private_data, int atomic, int hop)
{
return snd_seq_control_queue(ev, atomic, hop);
}
/* register our internal client */
int __init snd_seq_system_client_init(void)
{
struct snd_seq_port_callback pcallbacks;
struct snd_seq_port_info *port;
int err;
port = kzalloc(sizeof(*port), GFP_KERNEL);
if (!port)
return -ENOMEM;
memset(&pcallbacks, 0, sizeof(pcallbacks));
pcallbacks.owner = THIS_MODULE;
pcallbacks.event_input = event_input_timer;
/* register client */
sysclient = snd_seq_create_kernel_client(NULL, 0, "System");
if (sysclient < 0) {
kfree(port);
return sysclient;
}
/* register timer */
strcpy(port->name, "Timer");
port->capability = SNDRV_SEQ_PORT_CAP_WRITE; /* accept queue control */
port->capability |= SNDRV_SEQ_PORT_CAP_READ|SNDRV_SEQ_PORT_CAP_SUBS_READ; /* for broadcast */
port->kernel = &pcallbacks;
port->type = 0;
port->flags = SNDRV_SEQ_PORT_FLG_GIVEN_PORT;
port->addr.client = sysclient;
port->addr.port = SNDRV_SEQ_PORT_SYSTEM_TIMER;
err = snd_seq_kernel_client_ctl(sysclient, SNDRV_SEQ_IOCTL_CREATE_PORT,
port);
if (err < 0)
goto error_port;
/* register announcement port */
strcpy(port->name, "Announce");
port->capability = SNDRV_SEQ_PORT_CAP_READ|SNDRV_SEQ_PORT_CAP_SUBS_READ; /* for broadcast only */
port->kernel = NULL;
port->type = 0;
port->flags = SNDRV_SEQ_PORT_FLG_GIVEN_PORT;
port->addr.client = sysclient;
port->addr.port = SNDRV_SEQ_PORT_SYSTEM_ANNOUNCE;
err = snd_seq_kernel_client_ctl(sysclient, SNDRV_SEQ_IOCTL_CREATE_PORT,
port);
if (err < 0)
goto error_port;
announce_port = port->addr.port;
kfree(port);
return 0;
error_port:
snd_seq_system_client_done();
kfree(port);
return err;
}
/* unregister our internal client */
void snd_seq_system_client_done(void)
{
int oldsysclient = sysclient;
if (oldsysclient >= 0) {
sysclient = -1;
announce_port = -1;
snd_seq_delete_kernel_client(oldsysclient);
}
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TTY_H
#define _LINUX_TTY_H
#include <linux/fs.h>
#include <linux/major.h>
#include <linux/termios.h>
#include <linux/workqueue.h>
#include <linux/tty_driver.h>
#include <linux/tty_ldisc.h>
#include <linux/tty_port.h>
#include <linux/mutex.h>
#include <linux/tty_flags.h>
#include <uapi/linux/tty.h>
#include <linux/rwsem.h>
#include <linux/llist.h>
/*
* (Note: the *_driver.minor_start values 1, 64, 128, 192 are
* hardcoded at present.)
*/
#define NR_UNIX98_PTY_DEFAULT 4096 /* Default maximum for Unix98 ptys */
#define NR_UNIX98_PTY_RESERVE 1024 /* Default reserve for main devpts */
#define NR_UNIX98_PTY_MAX (1 << MINORBITS) /* Absolute limit */
/*
* This character is the same as _POSIX_VDISABLE: it cannot be used as
* a c_cc[] character, but indicates that a particular special character
* isn't in use (eg VINTR has no character etc)
*/
#define __DISABLED_CHAR '\0'
#define INTR_CHAR(tty) ((tty)->termios.c_cc[VINTR])
#define QUIT_CHAR(tty) ((tty)->termios.c_cc[VQUIT])
#define ERASE_CHAR(tty) ((tty)->termios.c_cc[VERASE])
#define KILL_CHAR(tty) ((tty)->termios.c_cc[VKILL])
#define EOF_CHAR(tty) ((tty)->termios.c_cc[VEOF])
#define TIME_CHAR(tty) ((tty)->termios.c_cc[VTIME])
#define MIN_CHAR(tty) ((tty)->termios.c_cc[VMIN])
#define SWTC_CHAR(tty) ((tty)->termios.c_cc[VSWTC])
#define START_CHAR(tty) ((tty)->termios.c_cc[VSTART])
#define STOP_CHAR(tty) ((tty)->termios.c_cc[VSTOP])
#define SUSP_CHAR(tty) ((tty)->termios.c_cc[VSUSP])
#define EOL_CHAR(tty) ((tty)->termios.c_cc[VEOL])
#define REPRINT_CHAR(tty) ((tty)->termios.c_cc[VREPRINT])
#define DISCARD_CHAR(tty) ((tty)->termios.c_cc[VDISCARD])
#define WERASE_CHAR(tty) ((tty)->termios.c_cc[VWERASE])
#define LNEXT_CHAR(tty) ((tty)->termios.c_cc[VLNEXT])
#define EOL2_CHAR(tty) ((tty)->termios.c_cc[VEOL2])
#define _I_FLAG(tty, f) ((tty)->termios.c_iflag & (f))
#define _O_FLAG(tty, f) ((tty)->termios.c_oflag & (f))
#define _C_FLAG(tty, f) ((tty)->termios.c_cflag & (f))
#define _L_FLAG(tty, f) ((tty)->termios.c_lflag & (f))
#define I_IGNBRK(tty) _I_FLAG((tty), IGNBRK)
#define I_BRKINT(tty) _I_FLAG((tty), BRKINT)
#define I_IGNPAR(tty) _I_FLAG((tty), IGNPAR)
#define I_PARMRK(tty) _I_FLAG((tty), PARMRK)
#define I_INPCK(tty) _I_FLAG((tty), INPCK)
#define I_ISTRIP(tty) _I_FLAG((tty), ISTRIP)
#define I_INLCR(tty) _I_FLAG((tty), INLCR)
#define I_IGNCR(tty) _I_FLAG((tty), IGNCR)
#define I_ICRNL(tty) _I_FLAG((tty), ICRNL)
#define I_IUCLC(tty) _I_FLAG((tty), IUCLC)
#define I_IXON(tty) _I_FLAG((tty), IXON)
#define I_IXANY(tty) _I_FLAG((tty), IXANY)
#define I_IXOFF(tty) _I_FLAG((tty), IXOFF)
#define I_IMAXBEL(tty) _I_FLAG((tty), IMAXBEL)
#define I_IUTF8(tty) _I_FLAG((tty), IUTF8)
#define O_OPOST(tty) _O_FLAG((tty), OPOST)
#define O_OLCUC(tty) _O_FLAG((tty), OLCUC)
#define O_ONLCR(tty) _O_FLAG((tty), ONLCR)
#define O_OCRNL(tty) _O_FLAG((tty), OCRNL)
#define O_ONOCR(tty) _O_FLAG((tty), ONOCR)
#define O_ONLRET(tty) _O_FLAG((tty), ONLRET)
#define O_OFILL(tty) _O_FLAG((tty), OFILL)
#define O_OFDEL(tty) _O_FLAG((tty), OFDEL)
#define O_NLDLY(tty) _O_FLAG((tty), NLDLY)
#define O_CRDLY(tty) _O_FLAG((tty), CRDLY)
#define O_TABDLY(tty) _O_FLAG((tty), TABDLY)
#define O_BSDLY(tty) _O_FLAG((tty), BSDLY)
#define O_VTDLY(tty) _O_FLAG((tty), VTDLY)
#define O_FFDLY(tty) _O_FLAG((tty), FFDLY)
#define C_BAUD(tty) _C_FLAG((tty), CBAUD)
#define C_CSIZE(tty) _C_FLAG((tty), CSIZE)
#define C_CSTOPB(tty) _C_FLAG((tty), CSTOPB)
#define C_CREAD(tty) _C_FLAG((tty), CREAD)
#define C_PARENB(tty) _C_FLAG((tty), PARENB)
#define C_PARODD(tty) _C_FLAG((tty), PARODD)
#define C_HUPCL(tty) _C_FLAG((tty), HUPCL)
#define C_CLOCAL(tty) _C_FLAG((tty), CLOCAL)
#define C_CIBAUD(tty) _C_FLAG((tty), CIBAUD)
#define C_CRTSCTS(tty) _C_FLAG((tty), CRTSCTS)
#define C_CMSPAR(tty) _C_FLAG((tty), CMSPAR)
#define L_ISIG(tty) _L_FLAG((tty), ISIG)
#define L_ICANON(tty) _L_FLAG((tty), ICANON)
#define L_XCASE(tty) _L_FLAG((tty), XCASE)
#define L_ECHO(tty) _L_FLAG((tty), ECHO)
#define L_ECHOE(tty) _L_FLAG((tty), ECHOE)
#define L_ECHOK(tty) _L_FLAG((tty), ECHOK)
#define L_ECHONL(tty) _L_FLAG((tty), ECHONL)
#define L_NOFLSH(tty) _L_FLAG((tty), NOFLSH)
#define L_TOSTOP(tty) _L_FLAG((tty), TOSTOP)
#define L_ECHOCTL(tty) _L_FLAG((tty), ECHOCTL)
#define L_ECHOPRT(tty) _L_FLAG((tty), ECHOPRT)
#define L_ECHOKE(tty) _L_FLAG((tty), ECHOKE)
#define L_FLUSHO(tty) _L_FLAG((tty), FLUSHO)
#define L_PENDIN(tty) _L_FLAG((tty), PENDIN)
#define L_IEXTEN(tty) _L_FLAG((tty), IEXTEN)
#define L_EXTPROC(tty) _L_FLAG((tty), EXTPROC)
struct device;
struct signal_struct;
struct tty_operations;
/**
* struct tty_struct - state associated with a tty while open
*
* @kref: reference counting by tty_kref_get() and tty_kref_put(), reaching zero
* frees the structure
* @dev: class device or %NULL (e.g. ptys, serdev)
* @driver: &struct tty_driver operating this tty
* @ops: &struct tty_operations of @driver for this tty (open, close, etc.)
* @index: index of this tty (e.g. to construct @name like tty12)
* @ldisc_sem: protects line discipline changes (@ldisc) -- lock tty not pty
* @ldisc: the current line discipline for this tty (n_tty by default)
* @atomic_write_lock: protects against concurrent writers, i.e. locks
* @write_cnt, @write_buf and similar
* @legacy_mutex: leftover from history (BKL -> BTM -> @legacy_mutex),
* protecting several operations on this tty
* @throttle_mutex: protects against concurrent tty_throttle_safe() and
* tty_unthrottle_safe() (but not tty_unthrottle())
* @termios_rwsem: protects @termios and @termios_locked
* @winsize_mutex: protects @winsize
* @termios: termios for the current tty, copied from/to @driver.termios
* @termios_locked: locked termios (by %TIOCGLCKTRMIOS and %TIOCSLCKTRMIOS
* ioctls)
* @name: name of the tty constructed by tty_line_name() (e.g. ttyS3)
* @flags: bitwise OR of %TTY_THROTTLED, %TTY_IO_ERROR, ...
* @count: count of open processes, reaching zero cancels all the work for
* this tty and drops a @kref too (but does not free this tty)
* @winsize: size of the terminal "window" (cf. @winsize_mutex)
* @flow: flow settings grouped together, see also @flow.unused
* @flow.lock: lock for @flow members
* @flow.stopped: tty stopped/started by stop_tty()/start_tty()
* @flow.tco_stopped: tty stopped/started by %TCOOFF/%TCOON ioctls (it has
* precedence over @flow.stopped)
* @flow.unused: alignment for Alpha, so that no members other than @flow.* are
* modified by the same 64b word store. The @flow's __aligned is
* there for the very same reason.
* @ctrl: control settings grouped together, see also @ctrl.unused
* @ctrl.lock: lock for @ctrl members
* @ctrl.pgrp: process group of this tty (setpgrp(2))
* @ctrl.session: session of this tty (setsid(2)). Writes are protected by both
* @ctrl.lock and @legacy_mutex, readers must use at least one of
* them.
* @ctrl.pktstatus: packet mode status (bitwise OR of %TIOCPKT_ constants)
* @ctrl.packet: packet mode enabled
* @ctrl.unused: alignment for Alpha, see @flow.unused for explanation
* @hw_stopped: not controlled by the tty layer, under @driver's control for CTS
* handling
* @receive_room: bytes permitted to feed to @ldisc without any being lost
* @flow_change: controls behavior of throttling, see tty_throttle_safe() and
* tty_unthrottle_safe()
* @link: link to another pty (master -> slave and vice versa)
* @fasync: state for %O_ASYNC (for %SIGIO); managed by fasync_helper()
* @write_wait: concurrent writers are waiting in this queue until they are
* allowed to write
* @read_wait: readers wait for data in this queue
* @hangup_work: normally a work to perform a hangup (do_tty_hangup()); while
* freeing the tty, (re)used to release_one_tty()
* @disc_data: pointer to @ldisc's private data (e.g. to &struct n_tty_data)
* @driver_data: pointer to @driver's private data (e.g. &struct uart_state)
* @files_lock: protects @tty_files list
* @tty_files: list of (re)openers of this tty (i.e. linked &struct
* tty_file_private)
* @closing: when set during close, n_tty processes only START & STOP chars
* @write_buf: temporary buffer used during tty_write() to copy user data to
* @write_cnt: count of bytes written in tty_write() to @write_buf
* @SAK_work: if the tty has a pending do_SAK, it is queued here
* @port: persistent storage for this device (i.e. &struct tty_port)
*
* All of the state associated with a tty while the tty is open. Persistent
* storage for tty devices is referenced here as @port and is documented in
* &struct tty_port.
*/
struct tty_struct {
struct kref kref;
int index;
struct device *dev;
struct tty_driver *driver;
struct tty_port *port;
const struct tty_operations *ops;
struct tty_ldisc *ldisc;
struct ld_semaphore ldisc_sem;
struct mutex atomic_write_lock;
struct mutex legacy_mutex;
struct mutex throttle_mutex;
struct rw_semaphore termios_rwsem;
struct mutex winsize_mutex;
struct ktermios termios, termios_locked;
char name[64];
unsigned long flags;
int count;
unsigned int receive_room;
struct winsize winsize;
struct {
spinlock_t lock;
bool stopped;
bool tco_stopped;
unsigned long unused[0];
} __aligned(sizeof(unsigned long)) flow;
struct {
struct pid *pgrp;
struct pid *session;
spinlock_t lock;
unsigned char pktstatus;
bool packet;
unsigned long unused[0];
} __aligned(sizeof(unsigned long)) ctrl;
bool hw_stopped;
bool closing;
int flow_change;
struct tty_struct *link;
struct fasync_struct *fasync;
wait_queue_head_t write_wait;
wait_queue_head_t read_wait;
struct work_struct hangup_work;
void *disc_data;
void *driver_data;
spinlock_t files_lock;
int write_cnt;
u8 *write_buf;
struct list_head tty_files;
#define N_TTY_BUF_SIZE 4096
struct work_struct SAK_work;
} __randomize_layout;
/* Each of a tty's open files has private_data pointing to tty_file_private */
struct tty_file_private {
struct tty_struct *tty;
struct file *file;
struct list_head list;
};
/**
* DOC: TTY Struct Flags
*
* These bits are used in the :c:member:`tty_struct.flags` field.
*
* So that interrupts won't be able to mess up the queues,
* copy_to_cooked must be atomic with respect to itself, as must
* tty->write. Thus, you must use the inline functions set_bit() and
* clear_bit() to make things atomic.
*
* TTY_THROTTLED
* Driver input is throttled. The ldisc should call
* :c:member:`tty_driver.unthrottle()` in order to resume reception when
* it is ready to process more data (at threshold min).
*
* TTY_IO_ERROR
* If set, causes all subsequent userspace read/write calls on the tty to
* fail, returning -%EIO. (May be no ldisc too.)
*
* TTY_OTHER_CLOSED
* Device is a pty and the other side has closed.
*
* TTY_EXCLUSIVE
* Exclusive open mode (a single opener).
*
* TTY_DO_WRITE_WAKEUP
* If set, causes the driver to call the
* :c:member:`tty_ldisc_ops.write_wakeup()` method in order to resume
* transmission when it can accept more data to transmit.
*
* TTY_LDISC_OPEN
* Indicates that a line discipline is open. For debugging purposes only.
*
* TTY_PTY_LOCK
* A flag private to pty code to implement %TIOCSPTLCK/%TIOCGPTLCK logic.
*
* TTY_NO_WRITE_SPLIT
* Prevent driver from splitting up writes into smaller chunks (preserve
* write boundaries to driver).
*
* TTY_HUPPED
* The TTY was hung up. This is set post :c:member:`tty_driver.hangup()`.
*
* TTY_HUPPING
* The TTY is in the process of hanging up to abort potential readers.
*
* TTY_LDISC_CHANGING
* Line discipline for this TTY is being changed. I/O should not block
* when this is set. Use tty_io_nonblock() to check.
*
* TTY_LDISC_HALTED
* Line discipline for this TTY was stopped. No work should be queued to
* this ldisc.
*/
#define TTY_THROTTLED 0
#define TTY_IO_ERROR 1
#define TTY_OTHER_CLOSED 2
#define TTY_EXCLUSIVE 3
#define TTY_DO_WRITE_WAKEUP 5
#define TTY_LDISC_OPEN 11
#define TTY_PTY_LOCK 16
#define TTY_NO_WRITE_SPLIT 17
#define TTY_HUPPED 18
#define TTY_HUPPING 19
#define TTY_LDISC_CHANGING 20
#define TTY_LDISC_HALTED 22
static inline bool tty_io_nonblock(struct tty_struct *tty, struct file *file)
{
return file->f_flags & O_NONBLOCK ||
test_bit(TTY_LDISC_CHANGING, &tty->flags);
}
static inline bool tty_io_error(struct tty_struct *tty)
{
return test_bit(TTY_IO_ERROR, &tty->flags);
}
static inline bool tty_throttled(struct tty_struct *tty)
{
return test_bit(TTY_THROTTLED, &tty->flags);
}
#ifdef CONFIG_TTY
void tty_kref_put(struct tty_struct *tty);
struct pid *tty_get_pgrp(struct tty_struct *tty);
void tty_vhangup_self(void);
void disassociate_ctty(int priv);
dev_t tty_devnum(struct tty_struct *tty);
void proc_clear_tty(struct task_struct *p);
struct tty_struct *get_current_tty(void);
/* tty_io.c */
int __init tty_init(void);
const char *tty_name(const struct tty_struct *tty);
struct tty_struct *tty_kopen_exclusive(dev_t device);
struct tty_struct *tty_kopen_shared(dev_t device);
void tty_kclose(struct tty_struct *tty);
int tty_dev_name_to_number(const char *name, dev_t *number);
#else
static inline void tty_kref_put(struct tty_struct *tty)
{ }
static inline struct pid *tty_get_pgrp(struct tty_struct *tty)
{ return NULL; }
static inline void tty_vhangup_self(void)
{ }
static inline void disassociate_ctty(int priv)
{ }
static inline dev_t tty_devnum(struct tty_struct *tty)
{ return 0; }
static inline void proc_clear_tty(struct task_struct *p)
{ }
static inline struct tty_struct *get_current_tty(void)
{ return NULL; }
/* tty_io.c */
static inline int __init tty_init(void)
{ return 0; }
static inline const char *tty_name(const struct tty_struct *tty)
{ return "(none)"; }
static inline struct tty_struct *tty_kopen_exclusive(dev_t device)
{ return ERR_PTR(-ENODEV); }
static inline void tty_kclose(struct tty_struct *tty)
{ }
static inline int tty_dev_name_to_number(const char *name, dev_t *number)
{ return -ENOTSUPP; }
#endif
extern struct ktermios tty_std_termios;
int vcs_init(void);
extern const struct class tty_class;
/**
* tty_kref_get - get a tty reference
* @tty: tty device
*
* Returns: a new reference to a tty object
*
* Locking: The caller must hold sufficient locks/counts to ensure that their
* existing reference cannot go away.
*/
static inline struct tty_struct *tty_kref_get(struct tty_struct *tty)
{
if (tty)
kref_get(&tty->kref);
return tty;
}
const char *tty_driver_name(const struct tty_struct *tty);
void tty_wait_until_sent(struct tty_struct *tty, long timeout);
void stop_tty(struct tty_struct *tty);
void start_tty(struct tty_struct *tty);
void tty_write_message(struct tty_struct *tty, char *msg);
int tty_send_xchar(struct tty_struct *tty, u8 ch);
int tty_put_char(struct tty_struct *tty, u8 c);
unsigned int tty_chars_in_buffer(struct tty_struct *tty);
unsigned int tty_write_room(struct tty_struct *tty);
void tty_driver_flush_buffer(struct tty_struct *tty);
void tty_unthrottle(struct tty_struct *tty);
bool tty_throttle_safe(struct tty_struct *tty);
bool tty_unthrottle_safe(struct tty_struct *tty);
int tty_do_resize(struct tty_struct *tty, struct winsize *ws);
int tty_get_icount(struct tty_struct *tty,
struct serial_icounter_struct *icount);
int tty_get_tiocm(struct tty_struct *tty);
int is_current_pgrp_orphaned(void);
void tty_hangup(struct tty_struct *tty);
void tty_vhangup(struct tty_struct *tty);
int tty_hung_up_p(struct file *filp);
void do_SAK(struct tty_struct *tty);
void __do_SAK(struct tty_struct *tty);
void no_tty(void);
speed_t tty_termios_baud_rate(const struct ktermios *termios);
void tty_termios_encode_baud_rate(struct ktermios *termios, speed_t ibaud,
speed_t obaud);
void tty_encode_baud_rate(struct tty_struct *tty, speed_t ibaud,
speed_t obaud);
/**
* tty_get_baud_rate - get tty bit rates
* @tty: tty to query
*
* Returns: the baud rate as an integer for this terminal
*
* Locking: The termios lock must be held by the caller.
*/
static inline speed_t tty_get_baud_rate(const struct tty_struct *tty)
{
return tty_termios_baud_rate(&tty->termios);
}
unsigned char tty_get_char_size(unsigned int cflag);
unsigned char tty_get_frame_size(unsigned int cflag);
void tty_termios_copy_hw(struct ktermios *new, const struct ktermios *old);
bool tty_termios_hw_change(const struct ktermios *a, const struct ktermios *b);
int tty_set_termios(struct tty_struct *tty, struct ktermios *kt);
void tty_wakeup(struct tty_struct *tty);
int tty_mode_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg);
int tty_perform_flush(struct tty_struct *tty, unsigned long arg);
struct tty_struct *tty_init_dev(struct tty_driver *driver, int idx);
void tty_release_struct(struct tty_struct *tty, int idx);
void tty_init_termios(struct tty_struct *tty);
void tty_save_termios(struct tty_struct *tty);
int tty_standard_install(struct tty_driver *driver,
struct tty_struct *tty);
extern struct mutex tty_mutex;
/* n_tty.c */
void n_tty_inherit_ops(struct tty_ldisc_ops *ops);
#ifdef CONFIG_TTY
void __init n_tty_init(void);
#else
static inline void n_tty_init(void) { }
#endif
/* tty_audit.c */
#ifdef CONFIG_AUDIT
void tty_audit_exit(void);
void tty_audit_fork(struct signal_struct *sig);
int tty_audit_push(void);
#else
static inline void tty_audit_exit(void)
{
}
static inline void tty_audit_fork(struct signal_struct *sig)
{
}
static inline int tty_audit_push(void)
{
return 0;
}
#endif
/* tty_ioctl.c */
int n_tty_ioctl_helper(struct tty_struct *tty, unsigned int cmd,
unsigned long arg);
/* vt.c */
int vt_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg);
long vt_compat_ioctl(struct tty_struct *tty, unsigned int cmd,
unsigned long arg);
/* tty_mutex.c */
/* functions for preparation of BKL removal */
void tty_lock(struct tty_struct *tty);
int tty_lock_interruptible(struct tty_struct *tty);
void tty_unlock(struct tty_struct *tty);
void tty_lock_slave(struct tty_struct *tty);
void tty_unlock_slave(struct tty_struct *tty);
void tty_set_lock_subclass(struct tty_struct *tty);
#endif
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Definitions for the 'struct ptr_ring' datastructure.
*
* Author:
* Michael S. Tsirkin <mst@redhat.com>
*
* Copyright (C) 2016 Red Hat, Inc.
*
* This is a limited-size FIFO maintaining pointers in FIFO order, with
* one CPU producing entries and another consuming entries from a FIFO.
*
* This implementation tries to minimize cache-contention when there is a
* single producer and a single consumer CPU.
*/
#ifndef _LINUX_PTR_RING_H
#define _LINUX_PTR_RING_H 1
#ifdef __KERNEL__
#include <linux/spinlock.h>
#include <linux/cache.h>
#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <asm/errno.h>
#endif
struct ptr_ring {
int producer ____cacheline_aligned_in_smp;
spinlock_t producer_lock;
int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */
int consumer_tail; /* next entry to invalidate */
spinlock_t consumer_lock;
/* Shared consumer/producer data */
/* Read-only by both the producer and the consumer */
int size ____cacheline_aligned_in_smp; /* max entries in queue */
int batch; /* number of entries to consume in a batch */
void **queue;
};
/* Note: callers invoking this in a loop must use a compiler barrier,
* for example cpu_relax().
*
* NB: this is unlike __ptr_ring_empty in that callers must hold producer_lock:
* see e.g. ptr_ring_full.
*/
static inline bool __ptr_ring_full(struct ptr_ring *r)
{
return r->queue[r->producer];
}
static inline bool ptr_ring_full(struct ptr_ring *r)
{
bool ret;
spin_lock(&r->producer_lock);
ret = __ptr_ring_full(r);
spin_unlock(&r->producer_lock);
return ret;
}
static inline bool ptr_ring_full_irq(struct ptr_ring *r)
{
bool ret;
spin_lock_irq(&r->producer_lock);
ret = __ptr_ring_full(r);
spin_unlock_irq(&r->producer_lock);
return ret;
}
static inline bool ptr_ring_full_any(struct ptr_ring *r)
{
unsigned long flags;
bool ret;
spin_lock_irqsave(&r->producer_lock, flags);
ret = __ptr_ring_full(r);
spin_unlock_irqrestore(&r->producer_lock, flags);
return ret;
}
static inline bool ptr_ring_full_bh(struct ptr_ring *r)
{
bool ret;
spin_lock_bh(&r->producer_lock);
ret = __ptr_ring_full(r);
spin_unlock_bh(&r->producer_lock);
return ret;
}
/* Note: callers invoking this in a loop must use a compiler barrier,
* for example cpu_relax(). Callers must hold producer_lock.
* Callers are responsible for making sure pointer that is being queued
* points to a valid data.
*/
static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)
{
if (unlikely(!r->size) || r->queue[r->producer])
return -ENOSPC;
/* Make sure the pointer we are storing points to a valid data. */
/* Pairs with the dependency ordering in __ptr_ring_consume. */
smp_wmb();
WRITE_ONCE(r->queue[r->producer++], ptr);
if (unlikely(r->producer >= r->size))
r->producer = 0;
return 0;
}
/*
* Note: resize (below) nests producer lock within consumer lock, so if you
* consume in interrupt or BH context, you must disable interrupts/BH when
* calling this.
*/
static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr)
{
int ret;
spin_lock(&r->producer_lock);
ret = __ptr_ring_produce(r, ptr);
spin_unlock(&r->producer_lock);
return ret;
}
static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr)
{
int ret;
spin_lock_irq(&r->producer_lock);
ret = __ptr_ring_produce(r, ptr);
spin_unlock_irq(&r->producer_lock);
return ret;
}
static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr)
{
unsigned long flags;
int ret;
spin_lock_irqsave(&r->producer_lock, flags);
ret = __ptr_ring_produce(r, ptr);
spin_unlock_irqrestore(&r->producer_lock, flags);
return ret;
}
static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr)
{
int ret;
spin_lock_bh(&r->producer_lock);
ret = __ptr_ring_produce(r, ptr);
spin_unlock_bh(&r->producer_lock);
return ret;
}
static inline void *__ptr_ring_peek(struct ptr_ring *r)
{
if (likely(r->size))
return READ_ONCE(r->queue[r->consumer_head]);
return NULL;
}
/*
* Test ring empty status without taking any locks.
*
* NB: This is only safe to call if ring is never resized.
*
* However, if some other CPU consumes ring entries at the same time, the value
* returned is not guaranteed to be correct.
*
* In this case - to avoid incorrectly detecting the ring
* as empty - the CPU consuming the ring entries is responsible
* for either consuming all ring entries until the ring is empty,
* or synchronizing with some other CPU and causing it to
* re-test __ptr_ring_empty and/or consume the ring enteries
* after the synchronization point.
*
* Note: callers invoking this in a loop must use a compiler barrier,
* for example cpu_relax().
*/
static inline bool __ptr_ring_empty(struct ptr_ring *r)
{
if (likely(r->size))
return !r->queue[READ_ONCE(r->consumer_head)];
return true;
}
static inline bool ptr_ring_empty(struct ptr_ring *r)
{
bool ret;
spin_lock(&r->consumer_lock);
ret = __ptr_ring_empty(r);
spin_unlock(&r->consumer_lock);
return ret;
}
static inline bool ptr_ring_empty_irq(struct ptr_ring *r)
{
bool ret;
spin_lock_irq(&r->consumer_lock);
ret = __ptr_ring_empty(r);
spin_unlock_irq(&r->consumer_lock);
return ret;
}
static inline bool ptr_ring_empty_any(struct ptr_ring *r)
{
unsigned long flags;
bool ret;
spin_lock_irqsave(&r->consumer_lock, flags);
ret = __ptr_ring_empty(r);
spin_unlock_irqrestore(&r->consumer_lock, flags);
return ret;
}
static inline bool ptr_ring_empty_bh(struct ptr_ring *r)
{
bool ret;
spin_lock_bh(&r->consumer_lock);
ret = __ptr_ring_empty(r);
spin_unlock_bh(&r->consumer_lock);
return ret;
}
/* Must only be called after __ptr_ring_peek returned !NULL */
static inline void __ptr_ring_discard_one(struct ptr_ring *r)
{
/* Fundamentally, what we want to do is update consumer
* index and zero out the entry so producer can reuse it.
* Doing it naively at each consume would be as simple as:
* consumer = r->consumer;
* r->queue[consumer++] = NULL;
* if (unlikely(consumer >= r->size))
* consumer = 0;
* r->consumer = consumer;
* but that is suboptimal when the ring is full as producer is writing
* out new entries in the same cache line. Defer these updates until a
* batch of entries has been consumed.
*/
/* Note: we must keep consumer_head valid at all times for __ptr_ring_empty
* to work correctly.
*/
int consumer_head = r->consumer_head;
int head = consumer_head++;
/* Once we have processed enough entries invalidate them in
* the ring all at once so producer can reuse their space in the ring.
* We also do this when we reach end of the ring - not mandatory
* but helps keep the implementation simple.
*/
if (unlikely(consumer_head - r->consumer_tail >= r->batch ||
consumer_head >= r->size)) {
/* Zero out entries in the reverse order: this way we touch the
* cache line that producer might currently be reading the last;
* producer won't make progress and touch other cache lines
* besides the first one until we write out all entries.
*/
while (likely(head >= r->consumer_tail)) r->queue[head--] = NULL; r->consumer_tail = consumer_head;
}
if (unlikely(consumer_head >= r->size)) {
consumer_head = 0;
r->consumer_tail = 0;
}
/* matching READ_ONCE in __ptr_ring_empty for lockless tests */
WRITE_ONCE(r->consumer_head, consumer_head);
}
static inline void *__ptr_ring_consume(struct ptr_ring *r)
{
void *ptr;
/* The READ_ONCE in __ptr_ring_peek guarantees that anyone
* accessing data through the pointer is up to date. Pairs
* with smp_wmb in __ptr_ring_produce.
*/
ptr = __ptr_ring_peek(r);
if (ptr)
__ptr_ring_discard_one(r);
return ptr;
}
static inline int __ptr_ring_consume_batched(struct ptr_ring *r,
void **array, int n)
{
void *ptr;
int i;
for (i = 0; i < n; i++) {
ptr = __ptr_ring_consume(r);
if (!ptr)
break;
array[i] = ptr;
}
return i;
}
/*
* Note: resize (below) nests producer lock within consumer lock, so if you
* call this in interrupt or BH context, you must disable interrupts/BH when
* producing.
*/
static inline void *ptr_ring_consume(struct ptr_ring *r)
{
void *ptr;
spin_lock(&r->consumer_lock);
ptr = __ptr_ring_consume(r);
spin_unlock(&r->consumer_lock);
return ptr;
}
static inline void *ptr_ring_consume_irq(struct ptr_ring *r)
{
void *ptr;
spin_lock_irq(&r->consumer_lock);
ptr = __ptr_ring_consume(r);
spin_unlock_irq(&r->consumer_lock);
return ptr;
}
static inline void *ptr_ring_consume_any(struct ptr_ring *r)
{
unsigned long flags;
void *ptr;
spin_lock_irqsave(&r->consumer_lock, flags);
ptr = __ptr_ring_consume(r);
spin_unlock_irqrestore(&r->consumer_lock, flags);
return ptr;
}
static inline void *ptr_ring_consume_bh(struct ptr_ring *r)
{
void *ptr;
spin_lock_bh(&r->consumer_lock);
ptr = __ptr_ring_consume(r);
spin_unlock_bh(&r->consumer_lock);
return ptr;
}
static inline int ptr_ring_consume_batched(struct ptr_ring *r,
void **array, int n)
{
int ret;
spin_lock(&r->consumer_lock);
ret = __ptr_ring_consume_batched(r, array, n);
spin_unlock(&r->consumer_lock);
return ret;
}
static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r,
void **array, int n)
{
int ret;
spin_lock_irq(&r->consumer_lock);
ret = __ptr_ring_consume_batched(r, array, n);
spin_unlock_irq(&r->consumer_lock);
return ret;
}
static inline int ptr_ring_consume_batched_any(struct ptr_ring *r,
void **array, int n)
{
unsigned long flags;
int ret;
spin_lock_irqsave(&r->consumer_lock, flags);
ret = __ptr_ring_consume_batched(r, array, n);
spin_unlock_irqrestore(&r->consumer_lock, flags);
return ret;
}
static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r,
void **array, int n)
{
int ret;
spin_lock_bh(&r->consumer_lock);
ret = __ptr_ring_consume_batched(r, array, n);
spin_unlock_bh(&r->consumer_lock);
return ret;
}
/* Cast to structure type and call a function without discarding from FIFO.
* Function must return a value.
* Callers must take consumer_lock.
*/
#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r)))
#define PTR_RING_PEEK_CALL(r, f) ({ \
typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
\
spin_lock(&(r)->consumer_lock); \
__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
spin_unlock(&(r)->consumer_lock); \
__PTR_RING_PEEK_CALL_v; \
})
#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \
typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
\
spin_lock_irq(&(r)->consumer_lock); \
__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
spin_unlock_irq(&(r)->consumer_lock); \
__PTR_RING_PEEK_CALL_v; \
})
#define PTR_RING_PEEK_CALL_BH(r, f) ({ \
typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
\
spin_lock_bh(&(r)->consumer_lock); \
__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
spin_unlock_bh(&(r)->consumer_lock); \
__PTR_RING_PEEK_CALL_v; \
})
#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \
typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \
unsigned long __PTR_RING_PEEK_CALL_f;\
\
spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \
__PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \
spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \
__PTR_RING_PEEK_CALL_v; \
})
/* Not all gfp_t flags (besides GFP_KERNEL) are allowed. See
* documentation for vmalloc for which of them are legal.
*/
static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp)
{
if (size > KMALLOC_MAX_SIZE / sizeof(void *))
return NULL;
return kvmalloc_array(size, sizeof(void *), gfp | __GFP_ZERO);
}
static inline void __ptr_ring_set_size(struct ptr_ring *r, int size)
{
r->size = size;
r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue));
/* We need to set batch at least to 1 to make logic
* in __ptr_ring_discard_one work correctly.
* Batching too much (because ring is small) would cause a lot of
* burstiness. Needs tuning, for now disable batching.
*/
if (r->batch > r->size / 2 || !r->batch)
r->batch = 1;
}
static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp)
{
r->queue = __ptr_ring_init_queue_alloc(size, gfp);
if (!r->queue)
return -ENOMEM;
__ptr_ring_set_size(r, size);
r->producer = r->consumer_head = r->consumer_tail = 0;
spin_lock_init(&r->producer_lock);
spin_lock_init(&r->consumer_lock);
return 0;
}
/*
* Return entries into ring. Destroy entries that don't fit.
*
* Note: this is expected to be a rare slow path operation.
*
* Note: producer lock is nested within consumer lock, so if you
* resize you must make sure all uses nest correctly.
* In particular if you consume ring in interrupt or BH context, you must
* disable interrupts/BH when doing so.
*/
static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n,
void (*destroy)(void *))
{
unsigned long flags;
int head;
spin_lock_irqsave(&r->consumer_lock, flags);
spin_lock(&r->producer_lock);
if (!r->size)
goto done;
/*
* Clean out buffered entries (for simplicity). This way following code
* can test entries for NULL and if not assume they are valid.
*/
head = r->consumer_head - 1;
while (likely(head >= r->consumer_tail))
r->queue[head--] = NULL;
r->consumer_tail = r->consumer_head;
/*
* Go over entries in batch, start moving head back and copy entries.
* Stop when we run into previously unconsumed entries.
*/
while (n) {
head = r->consumer_head - 1;
if (head < 0)
head = r->size - 1;
if (r->queue[head]) {
/* This batch entry will have to be destroyed. */
goto done;
}
r->queue[head] = batch[--n];
r->consumer_tail = head;
/* matching READ_ONCE in __ptr_ring_empty for lockless tests */
WRITE_ONCE(r->consumer_head, head);
}
done:
/* Destroy all entries left in the batch. */
while (n)
destroy(batch[--n]);
spin_unlock(&r->producer_lock);
spin_unlock_irqrestore(&r->consumer_lock, flags);
}
static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue,
int size, gfp_t gfp,
void (*destroy)(void *))
{
int producer = 0;
void **old;
void *ptr;
while ((ptr = __ptr_ring_consume(r)))
if (producer < size)
queue[producer++] = ptr;
else if (destroy)
destroy(ptr);
if (producer >= size)
producer = 0;
__ptr_ring_set_size(r, size);
r->producer = producer;
r->consumer_head = 0;
r->consumer_tail = 0;
old = r->queue;
r->queue = queue;
return old;
}
/*
* Note: producer lock is nested within consumer lock, so if you
* resize you must make sure all uses nest correctly.
* In particular if you consume ring in interrupt or BH context, you must
* disable interrupts/BH when doing so.
*/
static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp,
void (*destroy)(void *))
{
unsigned long flags;
void **queue = __ptr_ring_init_queue_alloc(size, gfp);
void **old;
if (!queue)
return -ENOMEM;
spin_lock_irqsave(&(r)->consumer_lock, flags);
spin_lock(&(r)->producer_lock);
old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy);
spin_unlock(&(r)->producer_lock);
spin_unlock_irqrestore(&(r)->consumer_lock, flags);
kvfree(old);
return 0;
}
/*
* Note: producer lock is nested within consumer lock, so if you
* resize you must make sure all uses nest correctly.
* In particular if you consume ring in interrupt or BH context, you must
* disable interrupts/BH when doing so.
*/
static inline int ptr_ring_resize_multiple(struct ptr_ring **rings,
unsigned int nrings,
int size,
gfp_t gfp, void (*destroy)(void *))
{
unsigned long flags;
void ***queues;
int i;
queues = kmalloc_array(nrings, sizeof(*queues), gfp);
if (!queues)
goto noqueues;
for (i = 0; i < nrings; ++i) {
queues[i] = __ptr_ring_init_queue_alloc(size, gfp);
if (!queues[i])
goto nomem;
}
for (i = 0; i < nrings; ++i) {
spin_lock_irqsave(&(rings[i])->consumer_lock, flags);
spin_lock(&(rings[i])->producer_lock);
queues[i] = __ptr_ring_swap_queue(rings[i], queues[i],
size, gfp, destroy);
spin_unlock(&(rings[i])->producer_lock);
spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags);
}
for (i = 0; i < nrings; ++i)
kvfree(queues[i]);
kfree(queues);
return 0;
nomem:
while (--i >= 0)
kvfree(queues[i]);
kfree(queues);
noqueues:
return -ENOMEM;
}
static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *))
{
void *ptr;
if (destroy)
while ((ptr = ptr_ring_consume(r)))
destroy(ptr);
kvfree(r->queue);
}
#endif /* _LINUX_PTR_RING_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Released under the GPLv2 only.
*/
#include <linux/module.h>
#include <linux/string.h>
#include <linux/bitops.h>
#include <linux/slab.h>
#include <linux/log2.h>
#include <linux/kmsan.h>
#include <linux/usb.h>
#include <linux/wait.h>
#include <linux/usb/hcd.h>
#include <linux/scatterlist.h>
#define to_urb(d) container_of(d, struct urb, kref)
static void urb_destroy(struct kref *kref)
{
struct urb *urb = to_urb(kref);
if (urb->transfer_flags & URB_FREE_BUFFER)
kfree(urb->transfer_buffer); kfree(urb);
}
/**
* usb_init_urb - initializes a urb so that it can be used by a USB driver
* @urb: pointer to the urb to initialize
*
* Initializes a urb so that the USB subsystem can use it properly.
*
* If a urb is created with a call to usb_alloc_urb() it is not
* necessary to call this function. Only use this if you allocate the
* space for a struct urb on your own. If you call this function, be
* careful when freeing the memory for your urb that it is no longer in
* use by the USB core.
*
* Only use this function if you _really_ understand what you are doing.
*/
void usb_init_urb(struct urb *urb)
{
if (urb) { memset(urb, 0, sizeof(*urb));
kref_init(&urb->kref);
INIT_LIST_HEAD(&urb->urb_list);
INIT_LIST_HEAD(&urb->anchor_list);
}
}
EXPORT_SYMBOL_GPL(usb_init_urb);
/**
* usb_alloc_urb - creates a new urb for a USB driver to use
* @iso_packets: number of iso packets for this urb
* @mem_flags: the type of memory to allocate, see kmalloc() for a list of
* valid options for this.
*
* Creates an urb for the USB driver to use, initializes a few internal
* structures, increments the usage counter, and returns a pointer to it.
*
* If the driver want to use this urb for interrupt, control, or bulk
* endpoints, pass '0' as the number of iso packets.
*
* The driver must call usb_free_urb() when it is finished with the urb.
*
* Return: A pointer to the new urb, or %NULL if no memory is available.
*/
struct urb *usb_alloc_urb(int iso_packets, gfp_t mem_flags)
{
struct urb *urb;
urb = kmalloc(struct_size(urb, iso_frame_desc, iso_packets),
mem_flags);
if (!urb)
return NULL;
usb_init_urb(urb); return urb;
}
EXPORT_SYMBOL_GPL(usb_alloc_urb);
/**
* usb_free_urb - frees the memory used by a urb when all users of it are finished
* @urb: pointer to the urb to free, may be NULL
*
* Must be called when a user of a urb is finished with it. When the last user
* of the urb calls this function, the memory of the urb is freed.
*
* Note: The transfer buffer associated with the urb is not freed unless the
* URB_FREE_BUFFER transfer flag is set.
*/
void usb_free_urb(struct urb *urb)
{
if (urb) kref_put(&urb->kref, urb_destroy);
}
EXPORT_SYMBOL_GPL(usb_free_urb);
/**
* usb_get_urb - increments the reference count of the urb
* @urb: pointer to the urb to modify, may be NULL
*
* This must be called whenever a urb is transferred from a device driver to a
* host controller driver. This allows proper reference counting to happen
* for urbs.
*
* Return: A pointer to the urb with the incremented reference counter.
*/
struct urb *usb_get_urb(struct urb *urb)
{
if (urb) kref_get(&urb->kref); return urb;
}
EXPORT_SYMBOL_GPL(usb_get_urb);
/**
* usb_anchor_urb - anchors an URB while it is processed
* @urb: pointer to the urb to anchor
* @anchor: pointer to the anchor
*
* This can be called to have access to URBs which are to be executed
* without bothering to track them
*/
void usb_anchor_urb(struct urb *urb, struct usb_anchor *anchor)
{
unsigned long flags;
spin_lock_irqsave(&anchor->lock, flags); usb_get_urb(urb); list_add_tail(&urb->anchor_list, &anchor->urb_list); urb->anchor = anchor;
if (unlikely(anchor->poisoned))
atomic_inc(&urb->reject); spin_unlock_irqrestore(&anchor->lock, flags);
}
EXPORT_SYMBOL_GPL(usb_anchor_urb);
static int usb_anchor_check_wakeup(struct usb_anchor *anchor)
{
return atomic_read(&anchor->suspend_wakeups) == 0 && list_empty(&anchor->urb_list);
}
/* Callers must hold anchor->lock */
static void __usb_unanchor_urb(struct urb *urb, struct usb_anchor *anchor)
{
urb->anchor = NULL; list_del(&urb->anchor_list); usb_put_urb(urb); if (usb_anchor_check_wakeup(anchor)) wake_up(&anchor->wait);
}
/**
* usb_unanchor_urb - unanchors an URB
* @urb: pointer to the urb to anchor
*
* Call this to stop the system keeping track of this URB
*/
void usb_unanchor_urb(struct urb *urb)
{
unsigned long flags;
struct usb_anchor *anchor;
if (!urb)
return;
anchor = urb->anchor;
if (!anchor)
return;
spin_lock_irqsave(&anchor->lock, flags);
/*
* At this point, we could be competing with another thread which
* has the same intention. To protect the urb from being unanchored
* twice, only the winner of the race gets the job.
*/
if (likely(anchor == urb->anchor))
__usb_unanchor_urb(urb, anchor); spin_unlock_irqrestore(&anchor->lock, flags);
}
EXPORT_SYMBOL_GPL(usb_unanchor_urb);
/*-------------------------------------------------------------------*/
static const int pipetypes[4] = {
PIPE_CONTROL, PIPE_ISOCHRONOUS, PIPE_BULK, PIPE_INTERRUPT
};
/**
* usb_pipe_type_check - sanity check of a specific pipe for a usb device
* @dev: struct usb_device to be checked
* @pipe: pipe to check
*
* This performs a light-weight sanity check for the endpoint in the
* given usb device. It returns 0 if the pipe is valid for the specific usb
* device, otherwise a negative error code.
*/
int usb_pipe_type_check(struct usb_device *dev, unsigned int pipe)
{
const struct usb_host_endpoint *ep;
ep = usb_pipe_endpoint(dev, pipe);
if (!ep)
return -EINVAL;
if (usb_pipetype(pipe) != pipetypes[usb_endpoint_type(&ep->desc)])
return -EINVAL;
return 0;
}
EXPORT_SYMBOL_GPL(usb_pipe_type_check);
/**
* usb_urb_ep_type_check - sanity check of endpoint in the given urb
* @urb: urb to be checked
*
* This performs a light-weight sanity check for the endpoint in the
* given urb. It returns 0 if the urb contains a valid endpoint, otherwise
* a negative error code.
*/
int usb_urb_ep_type_check(const struct urb *urb)
{
return usb_pipe_type_check(urb->dev, urb->pipe);
}
EXPORT_SYMBOL_GPL(usb_urb_ep_type_check);
/**
* usb_submit_urb - issue an asynchronous transfer request for an endpoint
* @urb: pointer to the urb describing the request
* @mem_flags: the type of memory to allocate, see kmalloc() for a list
* of valid options for this.
*
* This submits a transfer request, and transfers control of the URB
* describing that request to the USB subsystem. Request completion will
* be indicated later, asynchronously, by calling the completion handler.
* The three types of completion are success, error, and unlink
* (a software-induced fault, also called "request cancellation").
*
* URBs may be submitted in interrupt context.
*
* The caller must have correctly initialized the URB before submitting
* it. Functions such as usb_fill_bulk_urb() and usb_fill_control_urb() are
* available to ensure that most fields are correctly initialized, for
* the particular kind of transfer, although they will not initialize
* any transfer flags.
*
* If the submission is successful, the complete() callback from the URB
* will be called exactly once, when the USB core and Host Controller Driver
* (HCD) are finished with the URB. When the completion function is called,
* control of the URB is returned to the device driver which issued the
* request. The completion handler may then immediately free or reuse that
* URB.
*
* With few exceptions, USB device drivers should never access URB fields
* provided by usbcore or the HCD until its complete() is called.
* The exceptions relate to periodic transfer scheduling. For both
* interrupt and isochronous urbs, as part of successful URB submission
* urb->interval is modified to reflect the actual transfer period used
* (normally some power of two units). And for isochronous urbs,
* urb->start_frame is modified to reflect when the URB's transfers were
* scheduled to start.
*
* Not all isochronous transfer scheduling policies will work, but most
* host controller drivers should easily handle ISO queues going from now
* until 10-200 msec into the future. Drivers should try to keep at
* least one or two msec of data in the queue; many controllers require
* that new transfers start at least 1 msec in the future when they are
* added. If the driver is unable to keep up and the queue empties out,
* the behavior for new submissions is governed by the URB_ISO_ASAP flag.
* If the flag is set, or if the queue is idle, then the URB is always
* assigned to the first available (and not yet expired) slot in the
* endpoint's schedule. If the flag is not set and the queue is active
* then the URB is always assigned to the next slot in the schedule
* following the end of the endpoint's previous URB, even if that slot is
* in the past. When a packet is assigned in this way to a slot that has
* already expired, the packet is not transmitted and the corresponding
* usb_iso_packet_descriptor's status field will return -EXDEV. If this
* would happen to all the packets in the URB, submission fails with a
* -EXDEV error code.
*
* For control endpoints, the synchronous usb_control_msg() call is
* often used (in non-interrupt context) instead of this call.
* That is often used through convenience wrappers, for the requests
* that are standardized in the USB 2.0 specification. For bulk
* endpoints, a synchronous usb_bulk_msg() call is available.
*
* Return:
* 0 on successful submissions. A negative error number otherwise.
*
* Request Queuing:
*
* URBs may be submitted to endpoints before previous ones complete, to
* minimize the impact of interrupt latencies and system overhead on data
* throughput. With that queuing policy, an endpoint's queue would never
* be empty. This is required for continuous isochronous data streams,
* and may also be required for some kinds of interrupt transfers. Such
* queuing also maximizes bandwidth utilization by letting USB controllers
* start work on later requests before driver software has finished the
* completion processing for earlier (successful) requests.
*
* As of Linux 2.6, all USB endpoint transfer queues support depths greater
* than one. This was previously a HCD-specific behavior, except for ISO
* transfers. Non-isochronous endpoint queues are inactive during cleanup
* after faults (transfer errors or cancellation).
*
* Reserved Bandwidth Transfers:
*
* Periodic transfers (interrupt or isochronous) are performed repeatedly,
* using the interval specified in the urb. Submitting the first urb to
* the endpoint reserves the bandwidth necessary to make those transfers.
* If the USB subsystem can't allocate sufficient bandwidth to perform
* the periodic request, submitting such a periodic request should fail.
*
* For devices under xHCI, the bandwidth is reserved at configuration time, or
* when the alt setting is selected. If there is not enough bus bandwidth, the
* configuration/alt setting request will fail. Therefore, submissions to
* periodic endpoints on devices under xHCI should never fail due to bandwidth
* constraints.
*
* Device drivers must explicitly request that repetition, by ensuring that
* some URB is always on the endpoint's queue (except possibly for short
* periods during completion callbacks). When there is no longer an urb
* queued, the endpoint's bandwidth reservation is canceled. This means
* drivers can use their completion handlers to ensure they keep bandwidth
* they need, by reinitializing and resubmitting the just-completed urb
* until the driver longer needs that periodic bandwidth.
*
* Memory Flags:
*
* The general rules for how to decide which mem_flags to use
* are the same as for kmalloc. There are four
* different possible values; GFP_KERNEL, GFP_NOFS, GFP_NOIO and
* GFP_ATOMIC.
*
* GFP_NOFS is not ever used, as it has not been implemented yet.
*
* GFP_ATOMIC is used when
* (a) you are inside a completion handler, an interrupt, bottom half,
* tasklet or timer, or
* (b) you are holding a spinlock or rwlock (does not apply to
* semaphores), or
* (c) current->state != TASK_RUNNING, this is the case only after
* you've changed it.
*
* GFP_NOIO is used in the block io path and error handling of storage
* devices.
*
* All other situations use GFP_KERNEL.
*
* Some more specific rules for mem_flags can be inferred, such as
* (1) start_xmit, timeout, and receive methods of network drivers must
* use GFP_ATOMIC (they are called with a spinlock held);
* (2) queuecommand methods of scsi drivers must use GFP_ATOMIC (also
* called with a spinlock held);
* (3) If you use a kernel thread with a network driver you must use
* GFP_NOIO, unless (b) or (c) apply;
* (4) after you have done a down() you can use GFP_KERNEL, unless (b) or (c)
* apply or your are in a storage driver's block io path;
* (5) USB probe and disconnect can use GFP_KERNEL unless (b) or (c) apply; and
* (6) changing firmware on a running storage or net device uses
* GFP_NOIO, unless b) or c) apply
*
*/
int usb_submit_urb(struct urb *urb, gfp_t mem_flags)
{
int xfertype, max;
struct usb_device *dev;
struct usb_host_endpoint *ep;
int is_out;
unsigned int allowed;
if (!urb || !urb->complete)
return -EINVAL;
if (urb->hcpriv) { WARN_ONCE(1, "URB %pK submitted while active\n", urb);
return -EBUSY;
}
dev = urb->dev; if ((!dev) || (dev->state < USB_STATE_UNAUTHENTICATED))
return -ENODEV;
/* For now, get the endpoint from the pipe. Eventually drivers
* will be required to set urb->ep directly and we will eliminate
* urb->pipe.
*/
ep = usb_pipe_endpoint(dev, urb->pipe);
if (!ep)
return -ENOENT;
urb->ep = ep;
urb->status = -EINPROGRESS;
urb->actual_length = 0;
/* Lots of sanity checks, so HCDs can rely on clean data
* and don't need to duplicate tests
*/
xfertype = usb_endpoint_type(&ep->desc);
if (xfertype == USB_ENDPOINT_XFER_CONTROL) {
struct usb_ctrlrequest *setup =
(struct usb_ctrlrequest *) urb->setup_packet;
if (!setup)
return -ENOEXEC;
is_out = !(setup->bRequestType & USB_DIR_IN) || !setup->wLength; dev_WARN_ONCE(&dev->dev, (usb_pipeout(urb->pipe) != is_out),
"BOGUS control dir, pipe %x doesn't match bRequestType %x\n",
urb->pipe, setup->bRequestType);
if (le16_to_cpu(setup->wLength) != urb->transfer_buffer_length) { dev_dbg(&dev->dev, "BOGUS control len %d doesn't match transfer length %d\n",
le16_to_cpu(setup->wLength),
urb->transfer_buffer_length);
return -EBADR;
}
} else {
is_out = usb_endpoint_dir_out(&ep->desc);
}
/* Clear the internal flags and cache the direction for later use */
urb->transfer_flags &= ~(URB_DIR_MASK | URB_DMA_MAP_SINGLE |
URB_DMA_MAP_PAGE | URB_DMA_MAP_SG | URB_MAP_LOCAL |
URB_SETUP_MAP_SINGLE | URB_SETUP_MAP_LOCAL |
URB_DMA_SG_COMBINED);
urb->transfer_flags |= (is_out ? URB_DIR_OUT : URB_DIR_IN);
kmsan_handle_urb(urb, is_out);
if (xfertype != USB_ENDPOINT_XFER_CONTROL &&
dev->state < USB_STATE_CONFIGURED)
return -ENODEV;
max = usb_endpoint_maxp(&ep->desc);
if (max <= 0) {
dev_dbg(&dev->dev,
"bogus endpoint ep%d%s in %s (bad maxpacket %d)\n",
usb_endpoint_num(&ep->desc), is_out ? "out" : "in",
__func__, max);
return -EMSGSIZE;
}
/* periodic transfers limit size per frame/uframe,
* but drivers only control those sizes for ISO.
* while we're checking, initialize return status.
*/
if (xfertype == USB_ENDPOINT_XFER_ISOC) {
int n, len;
/* SuperSpeed isoc endpoints have up to 16 bursts of up to
* 3 packets each
*/
if (dev->speed >= USB_SPEED_SUPER) {
int burst = 1 + ep->ss_ep_comp.bMaxBurst;
int mult = USB_SS_MULT(ep->ss_ep_comp.bmAttributes);
max *= burst;
max *= mult;
}
if (dev->speed == USB_SPEED_SUPER_PLUS &&
USB_SS_SSP_ISOC_COMP(ep->ss_ep_comp.bmAttributes)) {
struct usb_ssp_isoc_ep_comp_descriptor *isoc_ep_comp;
isoc_ep_comp = &ep->ssp_isoc_ep_comp;
max = le32_to_cpu(isoc_ep_comp->dwBytesPerInterval);
}
/* "high bandwidth" mode, 1-3 packets/uframe? */
if (dev->speed == USB_SPEED_HIGH) max *= usb_endpoint_maxp_mult(&ep->desc); if (urb->number_of_packets <= 0)
return -EINVAL;
for (n = 0; n < urb->number_of_packets; n++) {
len = urb->iso_frame_desc[n].length; if (len < 0 || len > max)
return -EMSGSIZE;
urb->iso_frame_desc[n].status = -EXDEV;
urb->iso_frame_desc[n].actual_length = 0;
}
} else if (urb->num_sgs && !urb->dev->bus->no_sg_constraint) {
struct scatterlist *sg;
int i;
for_each_sg(urb->sg, sg, urb->num_sgs - 1, i) if (sg->length % max)
return -EINVAL;
}
/* the I/O buffer must be mapped/unmapped, except when length=0 */
if (urb->transfer_buffer_length > INT_MAX)
return -EMSGSIZE;
/*
* stuff that drivers shouldn't do, but which shouldn't
* cause problems in HCDs if they get it wrong.
*/
/* Check that the pipe's type matches the endpoint's type */
if (usb_pipe_type_check(urb->dev, urb->pipe)) dev_WARN(&dev->dev, "BOGUS urb xfer, pipe %x != type %x\n",
usb_pipetype(urb->pipe), pipetypes[xfertype]);
/* Check against a simple/standard policy */
allowed = (URB_NO_TRANSFER_DMA_MAP | URB_NO_INTERRUPT | URB_DIR_MASK |
URB_FREE_BUFFER);
switch (xfertype) {
case USB_ENDPOINT_XFER_BULK:
case USB_ENDPOINT_XFER_INT:
if (is_out)
allowed |= URB_ZERO_PACKET;
fallthrough;
default: /* all non-iso endpoints */
if (!is_out)
allowed |= URB_SHORT_NOT_OK;
break;
case USB_ENDPOINT_XFER_ISOC:
allowed |= URB_ISO_ASAP;
break;
}
allowed &= urb->transfer_flags;
/* warn if submitter gave bogus flags */
if (allowed != urb->transfer_flags) dev_WARN(&dev->dev, "BOGUS urb flags, %x --> %x\n",
urb->transfer_flags, allowed);
/*
* Force periodic transfer intervals to be legal values that are
* a power of two (so HCDs don't need to).
*
* FIXME want bus->{intr,iso}_sched_horizon values here. Each HC
* supports different values... this uses EHCI/UHCI defaults (and
* EHCI can use smaller non-default values).
*/
switch (xfertype) {
case USB_ENDPOINT_XFER_ISOC:
case USB_ENDPOINT_XFER_INT:
/* too small? */
if (urb->interval <= 0)
return -EINVAL;
/* too big? */
switch (dev->speed) {
case USB_SPEED_SUPER_PLUS:
case USB_SPEED_SUPER: /* units are 125us */
/* Handle up to 2^(16-1) microframes */
if (urb->interval > (1 << 15))
return -EINVAL;
max = 1 << 15;
break;
case USB_SPEED_HIGH: /* units are microframes */
/* NOTE usb handles 2^15 */
if (urb->interval > (1024 * 8)) urb->interval = 1024 * 8;
max = 1024 * 8;
break;
case USB_SPEED_FULL: /* units are frames/msec */
case USB_SPEED_LOW:
if (xfertype == USB_ENDPOINT_XFER_INT) { if (urb->interval > 255)
return -EINVAL;
/* NOTE ohci only handles up to 32 */
max = 128;
} else {
if (urb->interval > 1024) urb->interval = 1024;
/* NOTE usb and ohci handle up to 2^15 */
max = 1024;
}
break;
default:
return -EINVAL;
}
/* Round down to a power of 2, no more than max */
urb->interval = min(max, 1 << ilog2(urb->interval));
}
return usb_hcd_submit_urb(urb, mem_flags);
}
EXPORT_SYMBOL_GPL(usb_submit_urb);
/*-------------------------------------------------------------------*/
/**
* usb_unlink_urb - abort/cancel a transfer request for an endpoint
* @urb: pointer to urb describing a previously submitted request,
* may be NULL
*
* This routine cancels an in-progress request. URBs complete only once
* per submission, and may be canceled only once per submission.
* Successful cancellation means termination of @urb will be expedited
* and the completion handler will be called with a status code
* indicating that the request has been canceled (rather than any other
* code).
*
* Drivers should not call this routine or related routines, such as
* usb_kill_urb() or usb_unlink_anchored_urbs(), after their disconnect
* method has returned. The disconnect function should synchronize with
* a driver's I/O routines to insure that all URB-related activity has
* completed before it returns.
*
* This request is asynchronous, however the HCD might call the ->complete()
* callback during unlink. Therefore when drivers call usb_unlink_urb(), they
* must not hold any locks that may be taken by the completion function.
* Success is indicated by returning -EINPROGRESS, at which time the URB will
* probably not yet have been given back to the device driver. When it is
* eventually called, the completion function will see @urb->status ==
* -ECONNRESET.
* Failure is indicated by usb_unlink_urb() returning any other value.
* Unlinking will fail when @urb is not currently "linked" (i.e., it was
* never submitted, or it was unlinked before, or the hardware is already
* finished with it), even if the completion handler has not yet run.
*
* The URB must not be deallocated while this routine is running. In
* particular, when a driver calls this routine, it must insure that the
* completion handler cannot deallocate the URB.
*
* Return: -EINPROGRESS on success. See description for other values on
* failure.
*
* Unlinking and Endpoint Queues:
*
* [The behaviors and guarantees described below do not apply to virtual
* root hubs but only to endpoint queues for physical USB devices.]
*
* Host Controller Drivers (HCDs) place all the URBs for a particular
* endpoint in a queue. Normally the queue advances as the controller
* hardware processes each request. But when an URB terminates with an
* error its queue generally stops (see below), at least until that URB's
* completion routine returns. It is guaranteed that a stopped queue
* will not restart until all its unlinked URBs have been fully retired,
* with their completion routines run, even if that's not until some time
* after the original completion handler returns. The same behavior and
* guarantee apply when an URB terminates because it was unlinked.
*
* Bulk and interrupt endpoint queues are guaranteed to stop whenever an
* URB terminates with any sort of error, including -ECONNRESET, -ENOENT,
* and -EREMOTEIO. Control endpoint queues behave the same way except
* that they are not guaranteed to stop for -EREMOTEIO errors. Queues
* for isochronous endpoints are treated differently, because they must
* advance at fixed rates. Such queues do not stop when an URB
* encounters an error or is unlinked. An unlinked isochronous URB may
* leave a gap in the stream of packets; it is undefined whether such
* gaps can be filled in.
*
* Note that early termination of an URB because a short packet was
* received will generate a -EREMOTEIO error if and only if the
* URB_SHORT_NOT_OK flag is set. By setting this flag, USB device
* drivers can build deep queues for large or complex bulk transfers
* and clean them up reliably after any sort of aborted transfer by
* unlinking all pending URBs at the first fault.
*
* When a control URB terminates with an error other than -EREMOTEIO, it
* is quite likely that the status stage of the transfer will not take
* place.
*/
int usb_unlink_urb(struct urb *urb)
{
if (!urb)
return -EINVAL;
if (!urb->dev)
return -ENODEV;
if (!urb->ep)
return -EIDRM;
return usb_hcd_unlink_urb(urb, -ECONNRESET);
}
EXPORT_SYMBOL_GPL(usb_unlink_urb);
/**
* usb_kill_urb - cancel a transfer request and wait for it to finish
* @urb: pointer to URB describing a previously submitted request,
* may be NULL
*
* This routine cancels an in-progress request. It is guaranteed that
* upon return all completion handlers will have finished and the URB
* will be totally idle and available for reuse. These features make
* this an ideal way to stop I/O in a disconnect() callback or close()
* function. If the request has not already finished or been unlinked
* the completion handler will see urb->status == -ENOENT.
*
* While the routine is running, attempts to resubmit the URB will fail
* with error -EPERM. Thus even if the URB's completion handler always
* tries to resubmit, it will not succeed and the URB will become idle.
*
* The URB must not be deallocated while this routine is running. In
* particular, when a driver calls this routine, it must insure that the
* completion handler cannot deallocate the URB.
*
* This routine may not be used in an interrupt context (such as a bottom
* half or a completion handler), or when holding a spinlock, or in other
* situations where the caller can't schedule().
*
* This routine should not be called by a driver after its disconnect
* method has returned.
*/
void usb_kill_urb(struct urb *urb)
{
might_sleep(); if (!(urb && urb->dev && urb->ep))
return;
atomic_inc(&urb->reject);
/*
* Order the write of urb->reject above before the read
* of urb->use_count below. Pairs with the barriers in
* __usb_hcd_giveback_urb() and usb_hcd_submit_urb().
*/
smp_mb__after_atomic();
usb_hcd_unlink_urb(urb, -ENOENT);
wait_event(usb_kill_urb_queue, atomic_read(&urb->use_count) == 0); atomic_dec(&urb->reject);
}
EXPORT_SYMBOL_GPL(usb_kill_urb);
/**
* usb_poison_urb - reliably kill a transfer and prevent further use of an URB
* @urb: pointer to URB describing a previously submitted request,
* may be NULL
*
* This routine cancels an in-progress request. It is guaranteed that
* upon return all completion handlers will have finished and the URB
* will be totally idle and cannot be reused. These features make
* this an ideal way to stop I/O in a disconnect() callback.
* If the request has not already finished or been unlinked
* the completion handler will see urb->status == -ENOENT.
*
* After and while the routine runs, attempts to resubmit the URB will fail
* with error -EPERM. Thus even if the URB's completion handler always
* tries to resubmit, it will not succeed and the URB will become idle.
*
* The URB must not be deallocated while this routine is running. In
* particular, when a driver calls this routine, it must insure that the
* completion handler cannot deallocate the URB.
*
* This routine may not be used in an interrupt context (such as a bottom
* half or a completion handler), or when holding a spinlock, or in other
* situations where the caller can't schedule().
*
* This routine should not be called by a driver after its disconnect
* method has returned.
*/
void usb_poison_urb(struct urb *urb)
{
might_sleep(); if (!urb)
return;
atomic_inc(&urb->reject);
/*
* Order the write of urb->reject above before the read
* of urb->use_count below. Pairs with the barriers in
* __usb_hcd_giveback_urb() and usb_hcd_submit_urb().
*/
smp_mb__after_atomic();
if (!urb->dev || !urb->ep)
return;
usb_hcd_unlink_urb(urb, -ENOENT); wait_event(usb_kill_urb_queue, atomic_read(&urb->use_count) == 0);
}
EXPORT_SYMBOL_GPL(usb_poison_urb);
void usb_unpoison_urb(struct urb *urb)
{
if (!urb)
return;
atomic_dec(&urb->reject);
}
EXPORT_SYMBOL_GPL(usb_unpoison_urb);
/**
* usb_block_urb - reliably prevent further use of an URB
* @urb: pointer to URB to be blocked, may be NULL
*
* After the routine has run, attempts to resubmit the URB will fail
* with error -EPERM. Thus even if the URB's completion handler always
* tries to resubmit, it will not succeed and the URB will become idle.
*
* The URB must not be deallocated while this routine is running. In
* particular, when a driver calls this routine, it must insure that the
* completion handler cannot deallocate the URB.
*/
void usb_block_urb(struct urb *urb)
{
if (!urb)
return;
atomic_inc(&urb->reject);
}
EXPORT_SYMBOL_GPL(usb_block_urb);
/**
* usb_kill_anchored_urbs - kill all URBs associated with an anchor
* @anchor: anchor the requests are bound to
*
* This kills all outstanding URBs starting from the back of the queue,
* with guarantee that no completer callbacks will take place from the
* anchor after this function returns.
*
* This routine should not be called by a driver after its disconnect
* method has returned.
*/
void usb_kill_anchored_urbs(struct usb_anchor *anchor)
{
struct urb *victim;
int surely_empty;
do {
spin_lock_irq(&anchor->lock);
while (!list_empty(&anchor->urb_list)) {
victim = list_entry(anchor->urb_list.prev,
struct urb, anchor_list);
/* make sure the URB isn't freed before we kill it */
usb_get_urb(victim); spin_unlock_irq(&anchor->lock);
/* this will unanchor the URB */
usb_kill_urb(victim); usb_put_urb(victim); spin_lock_irq(&anchor->lock);
}
surely_empty = usb_anchor_check_wakeup(anchor); spin_unlock_irq(&anchor->lock);
cpu_relax();
} while (!surely_empty);
}
EXPORT_SYMBOL_GPL(usb_kill_anchored_urbs);
/**
* usb_poison_anchored_urbs - cease all traffic from an anchor
* @anchor: anchor the requests are bound to
*
* this allows all outstanding URBs to be poisoned starting
* from the back of the queue. Newly added URBs will also be
* poisoned
*
* This routine should not be called by a driver after its disconnect
* method has returned.
*/
void usb_poison_anchored_urbs(struct usb_anchor *anchor)
{
struct urb *victim;
int surely_empty;
do {
spin_lock_irq(&anchor->lock);
anchor->poisoned = 1;
while (!list_empty(&anchor->urb_list)) {
victim = list_entry(anchor->urb_list.prev,
struct urb, anchor_list);
/* make sure the URB isn't freed before we kill it */
usb_get_urb(victim); spin_unlock_irq(&anchor->lock);
/* this will unanchor the URB */
usb_poison_urb(victim);
usb_put_urb(victim); spin_lock_irq(&anchor->lock);
}
surely_empty = usb_anchor_check_wakeup(anchor); spin_unlock_irq(&anchor->lock);
cpu_relax();
} while (!surely_empty);
}
EXPORT_SYMBOL_GPL(usb_poison_anchored_urbs);
/**
* usb_unpoison_anchored_urbs - let an anchor be used successfully again
* @anchor: anchor the requests are bound to
*
* Reverses the effect of usb_poison_anchored_urbs
* the anchor can be used normally after it returns
*/
void usb_unpoison_anchored_urbs(struct usb_anchor *anchor)
{
unsigned long flags;
struct urb *lazarus;
spin_lock_irqsave(&anchor->lock, flags);
list_for_each_entry(lazarus, &anchor->urb_list, anchor_list) {
usb_unpoison_urb(lazarus);
}
anchor->poisoned = 0;
spin_unlock_irqrestore(&anchor->lock, flags);
}
EXPORT_SYMBOL_GPL(usb_unpoison_anchored_urbs);
/**
* usb_unlink_anchored_urbs - asynchronously cancel transfer requests en masse
* @anchor: anchor the requests are bound to
*
* this allows all outstanding URBs to be unlinked starting
* from the back of the queue. This function is asynchronous.
* The unlinking is just triggered. It may happen after this
* function has returned.
*
* This routine should not be called by a driver after its disconnect
* method has returned.
*/
void usb_unlink_anchored_urbs(struct usb_anchor *anchor)
{
struct urb *victim;
while ((victim = usb_get_from_anchor(anchor)) != NULL) {
usb_unlink_urb(victim);
usb_put_urb(victim);
}
}
EXPORT_SYMBOL_GPL(usb_unlink_anchored_urbs);
/**
* usb_anchor_suspend_wakeups
* @anchor: the anchor you want to suspend wakeups on
*
* Call this to stop the last urb being unanchored from waking up any
* usb_wait_anchor_empty_timeout waiters. This is used in the hcd urb give-
* back path to delay waking up until after the completion handler has run.
*/
void usb_anchor_suspend_wakeups(struct usb_anchor *anchor)
{
if (anchor)
atomic_inc(&anchor->suspend_wakeups);
}
EXPORT_SYMBOL_GPL(usb_anchor_suspend_wakeups);
/**
* usb_anchor_resume_wakeups
* @anchor: the anchor you want to resume wakeups on
*
* Allow usb_wait_anchor_empty_timeout waiters to be woken up again, and
* wake up any current waiters if the anchor is empty.
*/
void usb_anchor_resume_wakeups(struct usb_anchor *anchor)
{
if (!anchor)
return;
atomic_dec(&anchor->suspend_wakeups);
if (usb_anchor_check_wakeup(anchor))
wake_up(&anchor->wait);
}
EXPORT_SYMBOL_GPL(usb_anchor_resume_wakeups);
/**
* usb_wait_anchor_empty_timeout - wait for an anchor to be unused
* @anchor: the anchor you want to become unused
* @timeout: how long you are willing to wait in milliseconds
*
* Call this is you want to be sure all an anchor's
* URBs have finished
*
* Return: Non-zero if the anchor became unused. Zero on timeout.
*/
int usb_wait_anchor_empty_timeout(struct usb_anchor *anchor,
unsigned int timeout)
{
return wait_event_timeout(anchor->wait,
usb_anchor_check_wakeup(anchor),
msecs_to_jiffies(timeout));
}
EXPORT_SYMBOL_GPL(usb_wait_anchor_empty_timeout);
/**
* usb_get_from_anchor - get an anchor's oldest urb
* @anchor: the anchor whose urb you want
*
* This will take the oldest urb from an anchor,
* unanchor and return it
*
* Return: The oldest urb from @anchor, or %NULL if @anchor has no
* urbs associated with it.
*/
struct urb *usb_get_from_anchor(struct usb_anchor *anchor)
{
struct urb *victim;
unsigned long flags;
spin_lock_irqsave(&anchor->lock, flags);
if (!list_empty(&anchor->urb_list)) {
victim = list_entry(anchor->urb_list.next, struct urb,
anchor_list);
usb_get_urb(victim); __usb_unanchor_urb(victim, anchor);
} else {
victim = NULL;
}
spin_unlock_irqrestore(&anchor->lock, flags);
return victim;
}
EXPORT_SYMBOL_GPL(usb_get_from_anchor);
/**
* usb_scuttle_anchored_urbs - unanchor all an anchor's urbs
* @anchor: the anchor whose urbs you want to unanchor
*
* use this to get rid of all an anchor's urbs
*/
void usb_scuttle_anchored_urbs(struct usb_anchor *anchor)
{
struct urb *victim;
unsigned long flags;
int surely_empty;
do {
spin_lock_irqsave(&anchor->lock, flags);
while (!list_empty(&anchor->urb_list)) {
victim = list_entry(anchor->urb_list.prev,
struct urb, anchor_list);
__usb_unanchor_urb(victim, anchor);
}
surely_empty = usb_anchor_check_wakeup(anchor); spin_unlock_irqrestore(&anchor->lock, flags);
cpu_relax();
} while (!surely_empty);
}
EXPORT_SYMBOL_GPL(usb_scuttle_anchored_urbs);
/**
* usb_anchor_empty - is an anchor empty
* @anchor: the anchor you want to query
*
* Return: 1 if the anchor has no urbs associated with it.
*/
int usb_anchor_empty(struct usb_anchor *anchor)
{
return list_empty(&anchor->urb_list);
}
EXPORT_SYMBOL_GPL(usb_anchor_empty);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Video capture interface for Linux version 2
*
* A generic video device interface for the LINUX operating system
* using a set of device structures/vectors for low level operations.
*
* Authors: Alan Cox, <alan@lxorguk.ukuu.org.uk> (version 1)
* Mauro Carvalho Chehab <mchehab@kernel.org> (version 2)
*
* Fixes: 20000516 Claudio Matsuoka <claudio@conectiva.com>
* - Added procfs support
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/debugfs.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <media/v4l2-common.h>
#include <media/v4l2-device.h>
#include <media/v4l2-ioctl.h>
#include <media/v4l2-event.h>
#define VIDEO_NUM_DEVICES 256
#define VIDEO_NAME "video4linux"
#define dprintk(fmt, arg...) do { \
printk(KERN_DEBUG pr_fmt("%s: " fmt), \
__func__, ##arg); \
} while (0)
/*
* sysfs stuff
*/
static ssize_t index_show(struct device *cd,
struct device_attribute *attr, char *buf)
{
struct video_device *vdev = to_video_device(cd);
return sprintf(buf, "%i\n", vdev->index);
}
static DEVICE_ATTR_RO(index);
static ssize_t dev_debug_show(struct device *cd,
struct device_attribute *attr, char *buf)
{
struct video_device *vdev = to_video_device(cd);
return sprintf(buf, "%i\n", vdev->dev_debug);
}
static ssize_t dev_debug_store(struct device *cd, struct device_attribute *attr,
const char *buf, size_t len)
{
struct video_device *vdev = to_video_device(cd);
int res = 0;
u16 value;
res = kstrtou16(buf, 0, &value);
if (res)
return res;
vdev->dev_debug = value;
return len;
}
static DEVICE_ATTR_RW(dev_debug);
static ssize_t name_show(struct device *cd,
struct device_attribute *attr, char *buf)
{
struct video_device *vdev = to_video_device(cd);
return sprintf(buf, "%.*s\n", (int)sizeof(vdev->name), vdev->name);
}
static DEVICE_ATTR_RO(name);
static struct attribute *video_device_attrs[] = {
&dev_attr_name.attr,
&dev_attr_dev_debug.attr,
&dev_attr_index.attr,
NULL,
};
ATTRIBUTE_GROUPS(video_device);
/*
* Active devices
*/
static struct video_device *video_devices[VIDEO_NUM_DEVICES];
static DEFINE_MUTEX(videodev_lock);
static DECLARE_BITMAP(devnode_nums[VFL_TYPE_MAX], VIDEO_NUM_DEVICES);
/* Device node utility functions */
/* Note: these utility functions all assume that vfl_type is in the range
[0, VFL_TYPE_MAX-1]. */
#ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES
/* Return the bitmap corresponding to vfl_type. */
static inline unsigned long *devnode_bits(enum vfl_devnode_type vfl_type)
{
/* Any types not assigned to fixed minor ranges must be mapped to
one single bitmap for the purposes of finding a free node number
since all those unassigned types use the same minor range. */
int idx = (vfl_type > VFL_TYPE_RADIO) ? VFL_TYPE_MAX - 1 : vfl_type;
return devnode_nums[idx];
}
#else
/* Return the bitmap corresponding to vfl_type. */
static inline unsigned long *devnode_bits(enum vfl_devnode_type vfl_type)
{
return devnode_nums[vfl_type];
}
#endif
/* Mark device node number vdev->num as used */
static inline void devnode_set(struct video_device *vdev)
{
set_bit(vdev->num, devnode_bits(vdev->vfl_type));
}
/* Mark device node number vdev->num as unused */
static inline void devnode_clear(struct video_device *vdev)
{
clear_bit(vdev->num, devnode_bits(vdev->vfl_type));
}
/* Try to find a free device node number in the range [from, to> */
static inline int devnode_find(struct video_device *vdev, int from, int to)
{
return find_next_zero_bit(devnode_bits(vdev->vfl_type), to, from);
}
struct video_device *video_device_alloc(void)
{
return kzalloc(sizeof(struct video_device), GFP_KERNEL);
}
EXPORT_SYMBOL(video_device_alloc);
void video_device_release(struct video_device *vdev)
{
kfree(vdev);
}
EXPORT_SYMBOL(video_device_release);
void video_device_release_empty(struct video_device *vdev)
{
/* Do nothing */
/* Only valid when the video_device struct is a static. */
}
EXPORT_SYMBOL(video_device_release_empty);
static inline void video_get(struct video_device *vdev)
{
get_device(&vdev->dev);
}
static inline void video_put(struct video_device *vdev)
{
put_device(&vdev->dev);
}
/* Called when the last user of the video device exits. */
static void v4l2_device_release(struct device *cd)
{
struct video_device *vdev = to_video_device(cd);
struct v4l2_device *v4l2_dev = vdev->v4l2_dev;
mutex_lock(&videodev_lock);
if (WARN_ON(video_devices[vdev->minor] != vdev)) {
/* should not happen */
mutex_unlock(&videodev_lock);
return;
}
/* Free up this device for reuse */
video_devices[vdev->minor] = NULL;
/* Delete the cdev on this minor as well */
cdev_del(vdev->cdev);
/* Just in case some driver tries to access this from
the release() callback. */
vdev->cdev = NULL;
/* Mark device node number as free */
devnode_clear(vdev);
mutex_unlock(&videodev_lock);
#if defined(CONFIG_MEDIA_CONTROLLER)
if (v4l2_dev->mdev && vdev->vfl_dir != VFL_DIR_M2M) {
/* Remove interfaces and interface links */
media_devnode_remove(vdev->intf_devnode);
if (vdev->entity.function != MEDIA_ENT_F_UNKNOWN)
media_device_unregister_entity(&vdev->entity);
}
#endif
/* Do not call v4l2_device_put if there is no release callback set.
* Drivers that have no v4l2_device release callback might free the
* v4l2_dev instance in the video_device release callback below, so we
* must perform this check here.
*
* TODO: In the long run all drivers that use v4l2_device should use the
* v4l2_device release callback. This check will then be unnecessary.
*/
if (v4l2_dev->release == NULL)
v4l2_dev = NULL;
/* Release video_device and perform other
cleanups as needed. */
vdev->release(vdev);
/* Decrease v4l2_device refcount */
if (v4l2_dev)
v4l2_device_put(v4l2_dev);
}
static struct class video_class = {
.name = VIDEO_NAME,
.dev_groups = video_device_groups,
};
struct video_device *video_devdata(struct file *file)
{
return video_devices[iminor(file_inode(file))];
}
EXPORT_SYMBOL(video_devdata);
/* Priority handling */
static inline bool prio_is_valid(enum v4l2_priority prio)
{
return prio == V4L2_PRIORITY_BACKGROUND ||
prio == V4L2_PRIORITY_INTERACTIVE ||
prio == V4L2_PRIORITY_RECORD;
}
void v4l2_prio_init(struct v4l2_prio_state *global)
{
memset(global, 0, sizeof(*global));
}
EXPORT_SYMBOL(v4l2_prio_init);
int v4l2_prio_change(struct v4l2_prio_state *global, enum v4l2_priority *local,
enum v4l2_priority new)
{
if (!prio_is_valid(new))
return -EINVAL;
if (*local == new)
return 0;
atomic_inc(&global->prios[new]);
if (prio_is_valid(*local))
atomic_dec(&global->prios[*local]);
*local = new;
return 0;
}
EXPORT_SYMBOL(v4l2_prio_change);
void v4l2_prio_open(struct v4l2_prio_state *global, enum v4l2_priority *local)
{
v4l2_prio_change(global, local, V4L2_PRIORITY_DEFAULT);
}
EXPORT_SYMBOL(v4l2_prio_open);
void v4l2_prio_close(struct v4l2_prio_state *global, enum v4l2_priority local)
{
if (prio_is_valid(local))
atomic_dec(&global->prios[local]);
}
EXPORT_SYMBOL(v4l2_prio_close);
enum v4l2_priority v4l2_prio_max(struct v4l2_prio_state *global)
{
if (atomic_read(&global->prios[V4L2_PRIORITY_RECORD]) > 0)
return V4L2_PRIORITY_RECORD;
if (atomic_read(&global->prios[V4L2_PRIORITY_INTERACTIVE]) > 0)
return V4L2_PRIORITY_INTERACTIVE;
if (atomic_read(&global->prios[V4L2_PRIORITY_BACKGROUND]) > 0)
return V4L2_PRIORITY_BACKGROUND;
return V4L2_PRIORITY_UNSET;
}
EXPORT_SYMBOL(v4l2_prio_max);
int v4l2_prio_check(struct v4l2_prio_state *global, enum v4l2_priority local)
{
return (local < v4l2_prio_max(global)) ? -EBUSY : 0;
}
EXPORT_SYMBOL(v4l2_prio_check);
static ssize_t v4l2_read(struct file *filp, char __user *buf,
size_t sz, loff_t *off)
{
struct video_device *vdev = video_devdata(filp);
int ret = -ENODEV;
if (!vdev->fops->read)
return -EINVAL;
if (video_is_registered(vdev))
ret = vdev->fops->read(filp, buf, sz, off);
if ((vdev->dev_debug & V4L2_DEV_DEBUG_FOP) &&
(vdev->dev_debug & V4L2_DEV_DEBUG_STREAMING))
dprintk("%s: read: %zd (%d)\n",
video_device_node_name(vdev), sz, ret);
return ret;
}
static ssize_t v4l2_write(struct file *filp, const char __user *buf,
size_t sz, loff_t *off)
{
struct video_device *vdev = video_devdata(filp);
int ret = -ENODEV;
if (!vdev->fops->write)
return -EINVAL;
if (video_is_registered(vdev))
ret = vdev->fops->write(filp, buf, sz, off);
if ((vdev->dev_debug & V4L2_DEV_DEBUG_FOP) &&
(vdev->dev_debug & V4L2_DEV_DEBUG_STREAMING))
dprintk("%s: write: %zd (%d)\n",
video_device_node_name(vdev), sz, ret);
return ret;
}
static __poll_t v4l2_poll(struct file *filp, struct poll_table_struct *poll)
{
struct video_device *vdev = video_devdata(filp);
__poll_t res = EPOLLERR | EPOLLHUP | EPOLLPRI;
if (video_is_registered(vdev)) {
if (!vdev->fops->poll)
res = DEFAULT_POLLMASK;
else
res = vdev->fops->poll(filp, poll);
}
if (vdev->dev_debug & V4L2_DEV_DEBUG_POLL)
dprintk("%s: poll: %08x %08x\n",
video_device_node_name(vdev), res,
poll_requested_events(poll));
return res;
}
static long v4l2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
struct video_device *vdev = video_devdata(filp);
int ret = -ENODEV;
if (vdev->fops->unlocked_ioctl) {
if (video_is_registered(vdev))
ret = vdev->fops->unlocked_ioctl(filp, cmd, arg);
} else
ret = -ENOTTY;
return ret;
}
#ifdef CONFIG_MMU
#define v4l2_get_unmapped_area NULL
#else
static unsigned long v4l2_get_unmapped_area(struct file *filp,
unsigned long addr, unsigned long len, unsigned long pgoff,
unsigned long flags)
{
struct video_device *vdev = video_devdata(filp);
int ret;
if (!vdev->fops->get_unmapped_area)
return -ENOSYS;
if (!video_is_registered(vdev))
return -ENODEV;
ret = vdev->fops->get_unmapped_area(filp, addr, len, pgoff, flags);
if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
dprintk("%s: get_unmapped_area (%d)\n",
video_device_node_name(vdev), ret);
return ret;
}
#endif
static int v4l2_mmap(struct file *filp, struct vm_area_struct *vm)
{
struct video_device *vdev = video_devdata(filp);
int ret = -ENODEV;
if (!vdev->fops->mmap)
return -ENODEV;
if (video_is_registered(vdev))
ret = vdev->fops->mmap(filp, vm);
if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
dprintk("%s: mmap (%d)\n",
video_device_node_name(vdev), ret);
return ret;
}
/* Override for the open function */
static int v4l2_open(struct inode *inode, struct file *filp)
{
struct video_device *vdev;
int ret = 0;
/* Check if the video device is available */
mutex_lock(&videodev_lock);
vdev = video_devdata(filp);
/* return ENODEV if the video device has already been removed. */
if (vdev == NULL || !video_is_registered(vdev)) {
mutex_unlock(&videodev_lock);
return -ENODEV;
}
/* and increase the device refcount */
video_get(vdev);
mutex_unlock(&videodev_lock);
if (vdev->fops->open) {
if (video_is_registered(vdev))
ret = vdev->fops->open(filp);
else
ret = -ENODEV;
}
if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
dprintk("%s: open (%d)\n",
video_device_node_name(vdev), ret);
/* decrease the refcount in case of an error */
if (ret)
video_put(vdev);
return ret;
}
/* Override for the release function */
static int v4l2_release(struct inode *inode, struct file *filp)
{
struct video_device *vdev = video_devdata(filp);
int ret = 0;
/*
* We need to serialize the release() with queueing new requests.
* The release() may trigger the cancellation of a streaming
* operation, and that should not be mixed with queueing a new
* request at the same time.
*/
if (vdev->fops->release) {
if (v4l2_device_supports_requests(vdev->v4l2_dev)) {
mutex_lock(&vdev->v4l2_dev->mdev->req_queue_mutex);
ret = vdev->fops->release(filp);
mutex_unlock(&vdev->v4l2_dev->mdev->req_queue_mutex);
} else {
ret = vdev->fops->release(filp);
}
}
if (vdev->dev_debug & V4L2_DEV_DEBUG_FOP)
dprintk("%s: release\n",
video_device_node_name(vdev));
/* decrease the refcount unconditionally since the release()
return value is ignored. */
video_put(vdev);
return ret;
}
static const struct file_operations v4l2_fops = {
.owner = THIS_MODULE,
.read = v4l2_read,
.write = v4l2_write,
.open = v4l2_open,
.get_unmapped_area = v4l2_get_unmapped_area,
.mmap = v4l2_mmap,
.unlocked_ioctl = v4l2_ioctl,
#ifdef CONFIG_COMPAT
.compat_ioctl = v4l2_compat_ioctl32,
#endif
.release = v4l2_release,
.poll = v4l2_poll,
.llseek = no_llseek,
};
/**
* get_index - assign stream index number based on v4l2_dev
* @vdev: video_device to assign index number to, vdev->v4l2_dev should be assigned
*
* Note that when this is called the new device has not yet been registered
* in the video_device array, but it was able to obtain a minor number.
*
* This means that we can always obtain a free stream index number since
* the worst case scenario is that there are VIDEO_NUM_DEVICES - 1 slots in
* use of the video_device array.
*
* Returns a free index number.
*/
static int get_index(struct video_device *vdev)
{
/* This can be static since this function is called with the global
videodev_lock held. */
static DECLARE_BITMAP(used, VIDEO_NUM_DEVICES);
int i;
bitmap_zero(used, VIDEO_NUM_DEVICES);
for (i = 0; i < VIDEO_NUM_DEVICES; i++) {
if (video_devices[i] != NULL &&
video_devices[i]->v4l2_dev == vdev->v4l2_dev) {
__set_bit(video_devices[i]->index, used);
}
}
return find_first_zero_bit(used, VIDEO_NUM_DEVICES);
}
#define SET_VALID_IOCTL(ops, cmd, op) \
do { if ((ops)->op) __set_bit(_IOC_NR(cmd), valid_ioctls); } while (0)
/* This determines which ioctls are actually implemented in the driver.
It's a one-time thing which simplifies video_ioctl2 as it can just do
a bit test.
Note that drivers can override this by setting bits to 1 in
vdev->valid_ioctls. If an ioctl is marked as 1 when this function is
called, then that ioctl will actually be marked as unimplemented.
It does that by first setting up the local valid_ioctls bitmap, and
at the end do a:
vdev->valid_ioctls = valid_ioctls & ~(vdev->valid_ioctls)
*/
static void determine_valid_ioctls(struct video_device *vdev)
{
const u32 vid_caps = V4L2_CAP_VIDEO_CAPTURE |
V4L2_CAP_VIDEO_CAPTURE_MPLANE |
V4L2_CAP_VIDEO_OUTPUT |
V4L2_CAP_VIDEO_OUTPUT_MPLANE |
V4L2_CAP_VIDEO_M2M | V4L2_CAP_VIDEO_M2M_MPLANE;
const u32 meta_caps = V4L2_CAP_META_CAPTURE |
V4L2_CAP_META_OUTPUT;
DECLARE_BITMAP(valid_ioctls, BASE_VIDIOC_PRIVATE);
const struct v4l2_ioctl_ops *ops = vdev->ioctl_ops;
bool is_vid = vdev->vfl_type == VFL_TYPE_VIDEO &&
(vdev->device_caps & vid_caps);
bool is_vbi = vdev->vfl_type == VFL_TYPE_VBI;
bool is_radio = vdev->vfl_type == VFL_TYPE_RADIO;
bool is_sdr = vdev->vfl_type == VFL_TYPE_SDR;
bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH;
bool is_meta = vdev->vfl_type == VFL_TYPE_VIDEO &&
(vdev->device_caps & meta_caps);
bool is_rx = vdev->vfl_dir != VFL_DIR_TX;
bool is_tx = vdev->vfl_dir != VFL_DIR_RX;
bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC;
bool has_streaming = vdev->device_caps & V4L2_CAP_STREAMING;
bitmap_zero(valid_ioctls, BASE_VIDIOC_PRIVATE);
/* vfl_type and vfl_dir independent ioctls */
SET_VALID_IOCTL(ops, VIDIOC_QUERYCAP, vidioc_querycap);
__set_bit(_IOC_NR(VIDIOC_G_PRIORITY), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_S_PRIORITY), valid_ioctls);
/* Note: the control handler can also be passed through the filehandle,
and that can't be tested here. If the bit for these control ioctls
is set, then the ioctl is valid. But if it is 0, then it can still
be valid if the filehandle passed the control handler. */
if (vdev->ctrl_handler || ops->vidioc_queryctrl)
__set_bit(_IOC_NR(VIDIOC_QUERYCTRL), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_query_ext_ctrl)
__set_bit(_IOC_NR(VIDIOC_QUERY_EXT_CTRL), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_g_ctrl || ops->vidioc_g_ext_ctrls)
__set_bit(_IOC_NR(VIDIOC_G_CTRL), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_s_ctrl || ops->vidioc_s_ext_ctrls)
__set_bit(_IOC_NR(VIDIOC_S_CTRL), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_g_ext_ctrls)
__set_bit(_IOC_NR(VIDIOC_G_EXT_CTRLS), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_s_ext_ctrls)
__set_bit(_IOC_NR(VIDIOC_S_EXT_CTRLS), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_try_ext_ctrls)
__set_bit(_IOC_NR(VIDIOC_TRY_EXT_CTRLS), valid_ioctls);
if (vdev->ctrl_handler || ops->vidioc_querymenu)
__set_bit(_IOC_NR(VIDIOC_QUERYMENU), valid_ioctls);
if (!is_tch) {
SET_VALID_IOCTL(ops, VIDIOC_G_FREQUENCY, vidioc_g_frequency);
SET_VALID_IOCTL(ops, VIDIOC_S_FREQUENCY, vidioc_s_frequency);
}
SET_VALID_IOCTL(ops, VIDIOC_LOG_STATUS, vidioc_log_status);
#ifdef CONFIG_VIDEO_ADV_DEBUG
__set_bit(_IOC_NR(VIDIOC_DBG_G_CHIP_INFO), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_DBG_G_REGISTER), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_DBG_S_REGISTER), valid_ioctls);
#endif
/* yes, really vidioc_subscribe_event */
SET_VALID_IOCTL(ops, VIDIOC_DQEVENT, vidioc_subscribe_event);
SET_VALID_IOCTL(ops, VIDIOC_SUBSCRIBE_EVENT, vidioc_subscribe_event);
SET_VALID_IOCTL(ops, VIDIOC_UNSUBSCRIBE_EVENT, vidioc_unsubscribe_event);
if (ops->vidioc_enum_freq_bands || ops->vidioc_g_tuner || ops->vidioc_g_modulator)
__set_bit(_IOC_NR(VIDIOC_ENUM_FREQ_BANDS), valid_ioctls);
if (is_vid) {
/* video specific ioctls */
if ((is_rx && (ops->vidioc_enum_fmt_vid_cap ||
ops->vidioc_enum_fmt_vid_overlay)) ||
(is_tx && ops->vidioc_enum_fmt_vid_out))
__set_bit(_IOC_NR(VIDIOC_ENUM_FMT), valid_ioctls);
if ((is_rx && (ops->vidioc_g_fmt_vid_cap ||
ops->vidioc_g_fmt_vid_cap_mplane ||
ops->vidioc_g_fmt_vid_overlay)) ||
(is_tx && (ops->vidioc_g_fmt_vid_out ||
ops->vidioc_g_fmt_vid_out_mplane ||
ops->vidioc_g_fmt_vid_out_overlay)))
__set_bit(_IOC_NR(VIDIOC_G_FMT), valid_ioctls);
if ((is_rx && (ops->vidioc_s_fmt_vid_cap ||
ops->vidioc_s_fmt_vid_cap_mplane ||
ops->vidioc_s_fmt_vid_overlay)) ||
(is_tx && (ops->vidioc_s_fmt_vid_out ||
ops->vidioc_s_fmt_vid_out_mplane ||
ops->vidioc_s_fmt_vid_out_overlay)))
__set_bit(_IOC_NR(VIDIOC_S_FMT), valid_ioctls);
if ((is_rx && (ops->vidioc_try_fmt_vid_cap ||
ops->vidioc_try_fmt_vid_cap_mplane ||
ops->vidioc_try_fmt_vid_overlay)) ||
(is_tx && (ops->vidioc_try_fmt_vid_out ||
ops->vidioc_try_fmt_vid_out_mplane ||
ops->vidioc_try_fmt_vid_out_overlay)))
__set_bit(_IOC_NR(VIDIOC_TRY_FMT), valid_ioctls);
SET_VALID_IOCTL(ops, VIDIOC_OVERLAY, vidioc_overlay);
SET_VALID_IOCTL(ops, VIDIOC_G_FBUF, vidioc_g_fbuf);
SET_VALID_IOCTL(ops, VIDIOC_S_FBUF, vidioc_s_fbuf);
SET_VALID_IOCTL(ops, VIDIOC_G_JPEGCOMP, vidioc_g_jpegcomp);
SET_VALID_IOCTL(ops, VIDIOC_S_JPEGCOMP, vidioc_s_jpegcomp);
SET_VALID_IOCTL(ops, VIDIOC_G_ENC_INDEX, vidioc_g_enc_index);
SET_VALID_IOCTL(ops, VIDIOC_ENCODER_CMD, vidioc_encoder_cmd);
SET_VALID_IOCTL(ops, VIDIOC_TRY_ENCODER_CMD, vidioc_try_encoder_cmd);
SET_VALID_IOCTL(ops, VIDIOC_DECODER_CMD, vidioc_decoder_cmd);
SET_VALID_IOCTL(ops, VIDIOC_TRY_DECODER_CMD, vidioc_try_decoder_cmd);
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FRAMESIZES, vidioc_enum_framesizes);
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FRAMEINTERVALS, vidioc_enum_frameintervals);
if (ops->vidioc_g_selection &&
!test_bit(_IOC_NR(VIDIOC_G_SELECTION), vdev->valid_ioctls)) {
__set_bit(_IOC_NR(VIDIOC_G_CROP), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_CROPCAP), valid_ioctls);
}
if (ops->vidioc_s_selection &&
!test_bit(_IOC_NR(VIDIOC_S_SELECTION), vdev->valid_ioctls))
__set_bit(_IOC_NR(VIDIOC_S_CROP), valid_ioctls);
SET_VALID_IOCTL(ops, VIDIOC_G_SELECTION, vidioc_g_selection);
SET_VALID_IOCTL(ops, VIDIOC_S_SELECTION, vidioc_s_selection);
}
if (is_meta && is_rx) {
/* metadata capture specific ioctls */
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_meta_cap);
SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_meta_cap);
SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_cap);
SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_meta_cap);
} else if (is_meta && is_tx) {
/* metadata output specific ioctls */
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_meta_out);
SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_meta_out);
SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_out);
SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_meta_out);
}
if (is_vbi) {
/* vbi specific ioctls */
if ((is_rx && (ops->vidioc_g_fmt_vbi_cap ||
ops->vidioc_g_fmt_sliced_vbi_cap)) ||
(is_tx && (ops->vidioc_g_fmt_vbi_out ||
ops->vidioc_g_fmt_sliced_vbi_out)))
__set_bit(_IOC_NR(VIDIOC_G_FMT), valid_ioctls);
if ((is_rx && (ops->vidioc_s_fmt_vbi_cap ||
ops->vidioc_s_fmt_sliced_vbi_cap)) ||
(is_tx && (ops->vidioc_s_fmt_vbi_out ||
ops->vidioc_s_fmt_sliced_vbi_out)))
__set_bit(_IOC_NR(VIDIOC_S_FMT), valid_ioctls);
if ((is_rx && (ops->vidioc_try_fmt_vbi_cap ||
ops->vidioc_try_fmt_sliced_vbi_cap)) ||
(is_tx && (ops->vidioc_try_fmt_vbi_out ||
ops->vidioc_try_fmt_sliced_vbi_out)))
__set_bit(_IOC_NR(VIDIOC_TRY_FMT), valid_ioctls);
SET_VALID_IOCTL(ops, VIDIOC_G_SLICED_VBI_CAP, vidioc_g_sliced_vbi_cap);
} else if (is_tch) {
/* touch specific ioctls */
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_vid_cap);
SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_vid_cap);
SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_vid_cap);
SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_vid_cap);
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FRAMESIZES, vidioc_enum_framesizes);
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FRAMEINTERVALS, vidioc_enum_frameintervals);
SET_VALID_IOCTL(ops, VIDIOC_ENUMINPUT, vidioc_enum_input);
SET_VALID_IOCTL(ops, VIDIOC_G_INPUT, vidioc_g_input);
SET_VALID_IOCTL(ops, VIDIOC_S_INPUT, vidioc_s_input);
SET_VALID_IOCTL(ops, VIDIOC_G_PARM, vidioc_g_parm);
SET_VALID_IOCTL(ops, VIDIOC_S_PARM, vidioc_s_parm);
} else if (is_sdr && is_rx) {
/* SDR receiver specific ioctls */
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_sdr_cap);
SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_sdr_cap);
SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_sdr_cap);
SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_sdr_cap);
} else if (is_sdr && is_tx) {
/* SDR transmitter specific ioctls */
SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_sdr_out);
SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_sdr_out);
SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_sdr_out);
SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_sdr_out);
}
if (has_streaming) {
/* ioctls valid for streaming I/O */
SET_VALID_IOCTL(ops, VIDIOC_REQBUFS, vidioc_reqbufs);
SET_VALID_IOCTL(ops, VIDIOC_QUERYBUF, vidioc_querybuf);
SET_VALID_IOCTL(ops, VIDIOC_QBUF, vidioc_qbuf);
SET_VALID_IOCTL(ops, VIDIOC_EXPBUF, vidioc_expbuf);
SET_VALID_IOCTL(ops, VIDIOC_DQBUF, vidioc_dqbuf);
SET_VALID_IOCTL(ops, VIDIOC_CREATE_BUFS, vidioc_create_bufs);
SET_VALID_IOCTL(ops, VIDIOC_PREPARE_BUF, vidioc_prepare_buf);
SET_VALID_IOCTL(ops, VIDIOC_STREAMON, vidioc_streamon);
SET_VALID_IOCTL(ops, VIDIOC_STREAMOFF, vidioc_streamoff);
}
if (is_vid || is_vbi || is_meta) {
/* ioctls valid for video, vbi and metadata */
if (ops->vidioc_s_std)
__set_bit(_IOC_NR(VIDIOC_ENUMSTD), valid_ioctls);
SET_VALID_IOCTL(ops, VIDIOC_S_STD, vidioc_s_std);
SET_VALID_IOCTL(ops, VIDIOC_G_STD, vidioc_g_std);
if (is_rx) {
SET_VALID_IOCTL(ops, VIDIOC_QUERYSTD, vidioc_querystd);
if (is_io_mc) {
__set_bit(_IOC_NR(VIDIOC_ENUMINPUT), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_G_INPUT), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_S_INPUT), valid_ioctls);
} else {
SET_VALID_IOCTL(ops, VIDIOC_ENUMINPUT, vidioc_enum_input);
SET_VALID_IOCTL(ops, VIDIOC_G_INPUT, vidioc_g_input);
SET_VALID_IOCTL(ops, VIDIOC_S_INPUT, vidioc_s_input);
}
SET_VALID_IOCTL(ops, VIDIOC_ENUMAUDIO, vidioc_enumaudio);
SET_VALID_IOCTL(ops, VIDIOC_G_AUDIO, vidioc_g_audio);
SET_VALID_IOCTL(ops, VIDIOC_S_AUDIO, vidioc_s_audio);
SET_VALID_IOCTL(ops, VIDIOC_QUERY_DV_TIMINGS, vidioc_query_dv_timings);
SET_VALID_IOCTL(ops, VIDIOC_S_EDID, vidioc_s_edid);
}
if (is_tx) {
if (is_io_mc) {
__set_bit(_IOC_NR(VIDIOC_ENUMOUTPUT), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_G_OUTPUT), valid_ioctls);
__set_bit(_IOC_NR(VIDIOC_S_OUTPUT), valid_ioctls);
} else {
SET_VALID_IOCTL(ops, VIDIOC_ENUMOUTPUT, vidioc_enum_output);
SET_VALID_IOCTL(ops, VIDIOC_G_OUTPUT, vidioc_g_output);
SET_VALID_IOCTL(ops, VIDIOC_S_OUTPUT, vidioc_s_output);
}
SET_VALID_IOCTL(ops, VIDIOC_ENUMAUDOUT, vidioc_enumaudout);
SET_VALID_IOCTL(ops, VIDIOC_G_AUDOUT, vidioc_g_audout);
SET_VALID_IOCTL(ops, VIDIOC_S_AUDOUT, vidioc_s_audout);
}
if (ops->vidioc_g_parm || ops->vidioc_g_std)
__set_bit(_IOC_NR(VIDIOC_G_PARM), valid_ioctls);
SET_VALID_IOCTL(ops, VIDIOC_S_PARM, vidioc_s_parm);
SET_VALID_IOCTL(ops, VIDIOC_S_DV_TIMINGS, vidioc_s_dv_timings);
SET_VALID_IOCTL(ops, VIDIOC_G_DV_TIMINGS, vidioc_g_dv_timings);
SET_VALID_IOCTL(ops, VIDIOC_ENUM_DV_TIMINGS, vidioc_enum_dv_timings);
SET_VALID_IOCTL(ops, VIDIOC_DV_TIMINGS_CAP, vidioc_dv_timings_cap);
SET_VALID_IOCTL(ops, VIDIOC_G_EDID, vidioc_g_edid);
}
if (is_tx && (is_radio || is_sdr)) {
/* radio transmitter only ioctls */
SET_VALID_IOCTL(ops, VIDIOC_G_MODULATOR, vidioc_g_modulator);
SET_VALID_IOCTL(ops, VIDIOC_S_MODULATOR, vidioc_s_modulator);
}
if (is_rx && !is_tch) {
/* receiver only ioctls */
SET_VALID_IOCTL(ops, VIDIOC_G_TUNER, vidioc_g_tuner);
SET_VALID_IOCTL(ops, VIDIOC_S_TUNER, vidioc_s_tuner);
SET_VALID_IOCTL(ops, VIDIOC_S_HW_FREQ_SEEK, vidioc_s_hw_freq_seek);
}
bitmap_andnot(vdev->valid_ioctls, valid_ioctls, vdev->valid_ioctls,
BASE_VIDIOC_PRIVATE);
}
static int video_register_media_controller(struct video_device *vdev)
{
#if defined(CONFIG_MEDIA_CONTROLLER)
u32 intf_type;
int ret;
/* Memory-to-memory devices are more complex and use
* their own function to register its mc entities.
*/
if (!vdev->v4l2_dev->mdev || vdev->vfl_dir == VFL_DIR_M2M)
return 0;
vdev->entity.obj_type = MEDIA_ENTITY_TYPE_VIDEO_DEVICE;
vdev->entity.function = MEDIA_ENT_F_UNKNOWN;
switch (vdev->vfl_type) {
case VFL_TYPE_VIDEO:
intf_type = MEDIA_INTF_T_V4L_VIDEO;
vdev->entity.function = MEDIA_ENT_F_IO_V4L;
break;
case VFL_TYPE_VBI:
intf_type = MEDIA_INTF_T_V4L_VBI;
vdev->entity.function = MEDIA_ENT_F_IO_VBI;
break;
case VFL_TYPE_SDR:
intf_type = MEDIA_INTF_T_V4L_SWRADIO;
vdev->entity.function = MEDIA_ENT_F_IO_SWRADIO;
break;
case VFL_TYPE_TOUCH:
intf_type = MEDIA_INTF_T_V4L_TOUCH;
vdev->entity.function = MEDIA_ENT_F_IO_V4L;
break;
case VFL_TYPE_RADIO:
intf_type = MEDIA_INTF_T_V4L_RADIO;
/*
* Radio doesn't have an entity at the V4L2 side to represent
* radio input or output. Instead, the audio input/output goes
* via either physical wires or ALSA.
*/
break;
case VFL_TYPE_SUBDEV:
intf_type = MEDIA_INTF_T_V4L_SUBDEV;
/* Entity will be created via v4l2_device_register_subdev() */
break;
default:
return 0;
}
if (vdev->entity.function != MEDIA_ENT_F_UNKNOWN) {
vdev->entity.name = vdev->name;
/* Needed just for backward compatibility with legacy MC API */
vdev->entity.info.dev.major = VIDEO_MAJOR;
vdev->entity.info.dev.minor = vdev->minor;
ret = media_device_register_entity(vdev->v4l2_dev->mdev,
&vdev->entity);
if (ret < 0) {
pr_warn("%s: media_device_register_entity failed\n",
__func__);
return ret;
}
}
vdev->intf_devnode = media_devnode_create(vdev->v4l2_dev->mdev,
intf_type,
0, VIDEO_MAJOR,
vdev->minor);
if (!vdev->intf_devnode) {
media_device_unregister_entity(&vdev->entity);
return -ENOMEM;
}
if (vdev->entity.function != MEDIA_ENT_F_UNKNOWN) {
struct media_link *link;
link = media_create_intf_link(&vdev->entity,
&vdev->intf_devnode->intf,
MEDIA_LNK_FL_ENABLED |
MEDIA_LNK_FL_IMMUTABLE);
if (!link) {
media_devnode_remove(vdev->intf_devnode);
media_device_unregister_entity(&vdev->entity);
return -ENOMEM;
}
}
/* FIXME: how to create the other interface links? */
#endif
return 0;
}
int __video_register_device(struct video_device *vdev,
enum vfl_devnode_type type,
int nr, int warn_if_nr_in_use,
struct module *owner)
{
int i = 0;
int ret;
int minor_offset = 0;
int minor_cnt = VIDEO_NUM_DEVICES;
const char *name_base;
/* A minor value of -1 marks this video device as never
having been registered */
vdev->minor = -1;
/* the release callback MUST be present */
if (WARN_ON(!vdev->release))
return -EINVAL;
/* the v4l2_dev pointer MUST be present */
if (WARN_ON(!vdev->v4l2_dev))
return -EINVAL;
/* the device_caps field MUST be set for all but subdevs */
if (WARN_ON(type != VFL_TYPE_SUBDEV && !vdev->device_caps))
return -EINVAL;
/* v4l2_fh support */
spin_lock_init(&vdev->fh_lock);
INIT_LIST_HEAD(&vdev->fh_list);
/* Part 1: check device type */
switch (type) {
case VFL_TYPE_VIDEO:
name_base = "video";
break;
case VFL_TYPE_VBI:
name_base = "vbi";
break;
case VFL_TYPE_RADIO:
name_base = "radio";
break;
case VFL_TYPE_SUBDEV:
name_base = "v4l-subdev";
break;
case VFL_TYPE_SDR:
/* Use device name 'swradio' because 'sdr' was already taken. */
name_base = "swradio";
break;
case VFL_TYPE_TOUCH:
name_base = "v4l-touch";
break;
default:
pr_err("%s called with unknown type: %d\n",
__func__, type);
return -EINVAL;
}
vdev->vfl_type = type;
vdev->cdev = NULL;
if (vdev->dev_parent == NULL)
vdev->dev_parent = vdev->v4l2_dev->dev;
if (vdev->ctrl_handler == NULL)
vdev->ctrl_handler = vdev->v4l2_dev->ctrl_handler;
/* If the prio state pointer is NULL, then use the v4l2_device
prio state. */
if (vdev->prio == NULL)
vdev->prio = &vdev->v4l2_dev->prio;
/* Part 2: find a free minor, device node number and device index. */
#ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES
/* Keep the ranges for the first four types for historical
* reasons.
* Newer devices (not yet in place) should use the range
* of 128-191 and just pick the first free minor there
* (new style). */
switch (type) {
case VFL_TYPE_VIDEO:
minor_offset = 0;
minor_cnt = 64;
break;
case VFL_TYPE_RADIO:
minor_offset = 64;
minor_cnt = 64;
break;
case VFL_TYPE_VBI:
minor_offset = 224;
minor_cnt = 32;
break;
default:
minor_offset = 128;
minor_cnt = 64;
break;
}
#endif
/* Pick a device node number */
mutex_lock(&videodev_lock);
nr = devnode_find(vdev, nr == -1 ? 0 : nr, minor_cnt);
if (nr == minor_cnt)
nr = devnode_find(vdev, 0, minor_cnt);
if (nr == minor_cnt) {
pr_err("could not get a free device node number\n");
mutex_unlock(&videodev_lock);
return -ENFILE;
}
#ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES
/* 1-on-1 mapping of device node number to minor number */
i = nr;
#else
/* The device node number and minor numbers are independent, so
we just find the first free minor number. */
for (i = 0; i < VIDEO_NUM_DEVICES; i++)
if (video_devices[i] == NULL)
break;
if (i == VIDEO_NUM_DEVICES) {
mutex_unlock(&videodev_lock);
pr_err("could not get a free minor\n");
return -ENFILE;
}
#endif
vdev->minor = i + minor_offset;
vdev->num = nr;
/* Should not happen since we thought this minor was free */
if (WARN_ON(video_devices[vdev->minor])) {
mutex_unlock(&videodev_lock);
pr_err("video_device not empty!\n");
return -ENFILE;
}
devnode_set(vdev);
vdev->index = get_index(vdev);
video_devices[vdev->minor] = vdev;
mutex_unlock(&videodev_lock);
if (vdev->ioctl_ops)
determine_valid_ioctls(vdev);
/* Part 3: Initialize the character device */
vdev->cdev = cdev_alloc();
if (vdev->cdev == NULL) {
ret = -ENOMEM;
goto cleanup;
}
vdev->cdev->ops = &v4l2_fops;
vdev->cdev->owner = owner;
ret = cdev_add(vdev->cdev, MKDEV(VIDEO_MAJOR, vdev->minor), 1);
if (ret < 0) {
pr_err("%s: cdev_add failed\n", __func__);
kfree(vdev->cdev);
vdev->cdev = NULL;
goto cleanup;
}
/* Part 4: register the device with sysfs */
vdev->dev.class = &video_class;
vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor);
vdev->dev.parent = vdev->dev_parent;
dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num);
ret = device_register(&vdev->dev);
if (ret < 0) {
pr_err("%s: device_register failed\n", __func__);
goto cleanup;
}
/* Register the release callback that will be called when the last
reference to the device goes away. */
vdev->dev.release = v4l2_device_release;
if (nr != -1 && nr != vdev->num && warn_if_nr_in_use)
pr_warn("%s: requested %s%d, got %s\n", __func__,
name_base, nr, video_device_node_name(vdev));
/* Increase v4l2_device refcount */
v4l2_device_get(vdev->v4l2_dev);
/* Part 5: Register the entity. */
ret = video_register_media_controller(vdev);
/* Part 6: Activate this minor. The char device can now be used. */
set_bit(V4L2_FL_REGISTERED, &vdev->flags);
return 0;
cleanup:
mutex_lock(&videodev_lock);
if (vdev->cdev)
cdev_del(vdev->cdev);
video_devices[vdev->minor] = NULL;
devnode_clear(vdev);
mutex_unlock(&videodev_lock);
/* Mark this video device as never having been registered. */
vdev->minor = -1;
return ret;
}
EXPORT_SYMBOL(__video_register_device);
/**
* video_unregister_device - unregister a video4linux device
* @vdev: the device to unregister
*
* This unregisters the passed device. Future open calls will
* be met with errors.
*/
void video_unregister_device(struct video_device *vdev)
{
/* Check if vdev was ever registered at all */
if (!vdev || !video_is_registered(vdev))
return;
mutex_lock(&videodev_lock);
/* This must be in a critical section to prevent a race with v4l2_open.
* Once this bit has been cleared video_get may never be called again.
*/
clear_bit(V4L2_FL_REGISTERED, &vdev->flags);
mutex_unlock(&videodev_lock);
if (test_bit(V4L2_FL_USES_V4L2_FH, &vdev->flags))
v4l2_event_wake_all(vdev); device_unregister(&vdev->dev);
}
EXPORT_SYMBOL(video_unregister_device);
#if defined(CONFIG_MEDIA_CONTROLLER)
__must_check int video_device_pipeline_start(struct video_device *vdev,
struct media_pipeline *pipe)
{
struct media_entity *entity = &vdev->entity;
if (entity->num_pads != 1)
return -ENODEV;
return media_pipeline_start(&entity->pads[0], pipe);
}
EXPORT_SYMBOL_GPL(video_device_pipeline_start);
__must_check int __video_device_pipeline_start(struct video_device *vdev,
struct media_pipeline *pipe)
{
struct media_entity *entity = &vdev->entity;
if (entity->num_pads != 1)
return -ENODEV;
return __media_pipeline_start(&entity->pads[0], pipe);
}
EXPORT_SYMBOL_GPL(__video_device_pipeline_start);
void video_device_pipeline_stop(struct video_device *vdev)
{
struct media_entity *entity = &vdev->entity;
if (WARN_ON(entity->num_pads != 1))
return;
return media_pipeline_stop(&entity->pads[0]);
}
EXPORT_SYMBOL_GPL(video_device_pipeline_stop);
void __video_device_pipeline_stop(struct video_device *vdev)
{
struct media_entity *entity = &vdev->entity;
if (WARN_ON(entity->num_pads != 1))
return;
return __media_pipeline_stop(&entity->pads[0]);
}
EXPORT_SYMBOL_GPL(__video_device_pipeline_stop);
__must_check int video_device_pipeline_alloc_start(struct video_device *vdev)
{
struct media_entity *entity = &vdev->entity;
if (entity->num_pads != 1)
return -ENODEV;
return media_pipeline_alloc_start(&entity->pads[0]);
}
EXPORT_SYMBOL_GPL(video_device_pipeline_alloc_start);
struct media_pipeline *video_device_pipeline(struct video_device *vdev)
{
struct media_entity *entity = &vdev->entity;
if (WARN_ON(entity->num_pads != 1))
return NULL;
return media_pad_pipeline(&entity->pads[0]);
}
EXPORT_SYMBOL_GPL(video_device_pipeline);
#endif /* CONFIG_MEDIA_CONTROLLER */
/*
* Initialise video for linux
*/
static int __init videodev_init(void)
{
dev_t dev = MKDEV(VIDEO_MAJOR, 0);
int ret;
pr_info("Linux video capture interface: v2.00\n");
ret = register_chrdev_region(dev, VIDEO_NUM_DEVICES, VIDEO_NAME);
if (ret < 0) {
pr_warn("videodev: unable to get major %d\n",
VIDEO_MAJOR);
return ret;
}
ret = class_register(&video_class);
if (ret < 0) {
unregister_chrdev_region(dev, VIDEO_NUM_DEVICES);
pr_warn("video_dev: class_register failed\n");
return -EIO;
}
return 0;
}
static void __exit videodev_exit(void)
{
dev_t dev = MKDEV(VIDEO_MAJOR, 0);
class_unregister(&video_class);
unregister_chrdev_region(dev, VIDEO_NUM_DEVICES);
}
subsys_initcall(videodev_init);
module_exit(videodev_exit)
MODULE_AUTHOR("Alan Cox, Mauro Carvalho Chehab <mchehab@kernel.org>, Bill Dirks, Justin Schoeman, Gerd Knorr");
MODULE_DESCRIPTION("Video4Linux2 core driver");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CHARDEV_MAJOR(VIDEO_MAJOR);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef LINUX_MM_INLINE_H
#define LINUX_MM_INLINE_H
#include <linux/atomic.h>
#include <linux/huge_mm.h>
#include <linux/mm_types.h>
#include <linux/swap.h>
#include <linux/string.h>
#include <linux/userfaultfd_k.h>
#include <linux/swapops.h>
/**
* folio_is_file_lru - Should the folio be on a file LRU or anon LRU?
* @folio: The folio to test.
*
* We would like to get this info without a page flag, but the state
* needs to survive until the folio is last deleted from the LRU, which
* could be as far down as __page_cache_release.
*
* Return: An integer (not a boolean!) used to sort a folio onto the
* right LRU list and to account folios correctly.
* 1 if @folio is a regular filesystem backed page cache folio
* or a lazily freed anonymous folio (e.g. via MADV_FREE).
* 0 if @folio is a normal anonymous folio, a tmpfs folio or otherwise
* ram or swap backed folio.
*/
static inline int folio_is_file_lru(struct folio *folio)
{
return !folio_test_swapbacked(folio);
}
static inline int page_is_file_lru(struct page *page)
{
return folio_is_file_lru(page_folio(page));
}
static __always_inline void __update_lru_size(struct lruvec *lruvec,
enum lru_list lru, enum zone_type zid,
long nr_pages)
{
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
lockdep_assert_held(&lruvec->lru_lock); WARN_ON_ONCE(nr_pages != (int)nr_pages); __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
__mod_zone_page_state(&pgdat->node_zones[zid],
NR_ZONE_LRU_BASE + lru, nr_pages);
}
static __always_inline void update_lru_size(struct lruvec *lruvec,
enum lru_list lru, enum zone_type zid,
long nr_pages)
{
__update_lru_size(lruvec, lru, zid, nr_pages);
#ifdef CONFIG_MEMCG
mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
#endif
}
/**
* __folio_clear_lru_flags - Clear page lru flags before releasing a page.
* @folio: The folio that was on lru and now has a zero reference.
*/
static __always_inline void __folio_clear_lru_flags(struct folio *folio)
{
VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio); __folio_clear_lru(folio);
/* this shouldn't happen, so leave the flags to bad_page() */
if (folio_test_active(folio) && folio_test_unevictable(folio))
return;
__folio_clear_active(folio);
__folio_clear_unevictable(folio);
}
/**
* folio_lru_list - Which LRU list should a folio be on?
* @folio: The folio to test.
*
* Return: The LRU list a folio should be on, as an index
* into the array of LRU lists.
*/
static __always_inline enum lru_list folio_lru_list(struct folio *folio)
{
enum lru_list lru;
VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio); if (folio_test_unevictable(folio))
return LRU_UNEVICTABLE;
lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;
if (folio_test_active(folio))
lru += LRU_ACTIVE;
return lru;
}
#ifdef CONFIG_LRU_GEN
#ifdef CONFIG_LRU_GEN_ENABLED
static inline bool lru_gen_enabled(void)
{
DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]);
return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]);
}
#else
static inline bool lru_gen_enabled(void)
{
DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]);
return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]);
}
#endif
static inline bool lru_gen_in_fault(void)
{
return current->in_lru_fault;
}
static inline int lru_gen_from_seq(unsigned long seq)
{
return seq % MAX_NR_GENS;
}
static inline int lru_hist_from_seq(unsigned long seq)
{
return seq % NR_HIST_GENS;
}
static inline int lru_tier_from_refs(int refs)
{
VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));
/* see the comment in folio_lru_refs() */
return order_base_2(refs + 1);
}
static inline int folio_lru_refs(struct folio *folio)
{
unsigned long flags = READ_ONCE(folio->flags);
bool workingset = flags & BIT(PG_workingset);
/*
* Return the number of accesses beyond PG_referenced, i.e., N-1 if the
* total number of accesses is N>1, since N=0,1 both map to the first
* tier. lru_tier_from_refs() will account for this off-by-one. Also see
* the comment on MAX_NR_TIERS.
*/
return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset;
}
static inline int folio_lru_gen(struct folio *folio)
{
unsigned long flags = READ_ONCE(folio->flags);
return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
}
static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
{
unsigned long max_seq = lruvec->lrugen.max_seq;
VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
/* see the comment on MIN_NR_GENS */
return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
}
static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
int old_gen, int new_gen)
{
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
int delta = folio_nr_pages(folio);
enum lru_list lru = type * LRU_INACTIVE_FILE;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
if (old_gen >= 0)
WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
lrugen->nr_pages[old_gen][type][zone] - delta);
if (new_gen >= 0)
WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
lrugen->nr_pages[new_gen][type][zone] + delta);
/* addition */
if (old_gen < 0) {
if (lru_gen_is_active(lruvec, new_gen))
lru += LRU_ACTIVE;
__update_lru_size(lruvec, lru, zone, delta);
return;
}
/* deletion */
if (new_gen < 0) {
if (lru_gen_is_active(lruvec, old_gen))
lru += LRU_ACTIVE;
__update_lru_size(lruvec, lru, zone, -delta);
return;
}
/* promotion */
if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
__update_lru_size(lruvec, lru, zone, -delta);
__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
}
/* demotion requires isolation, e.g., lru_deactivate_fn() */
VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
}
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
unsigned long seq;
unsigned long flags;
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
if (folio_test_unevictable(folio) || !lrugen->enabled)
return false;
/*
* There are four common cases for this page:
* 1. If it's hot, i.e., freshly faulted in, add it to the youngest
* generation, and it's protected over the rest below.
* 2. If it can't be evicted immediately, i.e., a dirty page pending
* writeback, add it to the second youngest generation.
* 3. If it should be evicted first, e.g., cold and clean from
* folio_rotate_reclaimable(), add it to the oldest generation.
* 4. Everything else falls between 2 & 3 above and is added to the
* second oldest generation if it's considered inactive, or the
* oldest generation otherwise. See lru_gen_is_active().
*/
if (folio_test_active(folio))
seq = lrugen->max_seq;
else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) ||
(folio_test_reclaim(folio) &&
(folio_test_dirty(folio) || folio_test_writeback(folio))))
seq = lrugen->max_seq - 1;
else if (reclaiming || lrugen->min_seq[type] + MIN_NR_GENS >= lrugen->max_seq)
seq = lrugen->min_seq[type];
else
seq = lrugen->min_seq[type] + 1;
gen = lru_gen_from_seq(seq);
flags = (gen + 1UL) << LRU_GEN_PGOFF;
/* see the comment on MIN_NR_GENS about PG_active */
set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);
lru_gen_update_size(lruvec, folio, -1, gen);
/* for folio_rotate_reclaimable() */
if (reclaiming)
list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
else
list_add(&folio->lru, &lrugen->folios[gen][type][zone]);
return true;
}
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
unsigned long flags;
int gen = folio_lru_gen(folio);
if (gen < 0)
return false;
VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
/* for folio_migrate_flags() */
flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);
gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
lru_gen_update_size(lruvec, folio, gen, -1);
list_del(&folio->lru);
return true;
}
#else /* !CONFIG_LRU_GEN */
static inline bool lru_gen_enabled(void)
{
return false;
}
static inline bool lru_gen_in_fault(void)
{
return false;
}
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
}
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
return false;
}
#endif /* CONFIG_LRU_GEN */
static __always_inline
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
{
enum lru_list lru = folio_lru_list(folio);
if (lru_gen_add_folio(lruvec, folio, false))
return;
update_lru_size(lruvec, lru, folio_zonenum(folio),
folio_nr_pages(folio));
if (lru != LRU_UNEVICTABLE)
list_add(&folio->lru, &lruvec->lists[lru]);
}
static __always_inline
void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
{
enum lru_list lru = folio_lru_list(folio);
if (lru_gen_add_folio(lruvec, folio, true))
return;
update_lru_size(lruvec, lru, folio_zonenum(folio),
folio_nr_pages(folio));
/* This is not expected to be used on LRU_UNEVICTABLE */
list_add_tail(&folio->lru, &lruvec->lists[lru]);
}
static __always_inline
void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
{
enum lru_list lru = folio_lru_list(folio);
if (lru_gen_del_folio(lruvec, folio, false))
return;
if (lru != LRU_UNEVICTABLE)
list_del(&folio->lru); update_lru_size(lruvec, lru, folio_zonenum(folio), -folio_nr_pages(folio));
}
#ifdef CONFIG_ANON_VMA_NAME
/* mmap_lock should be read-locked */
static inline void anon_vma_name_get(struct anon_vma_name *anon_name)
{
if (anon_name)
kref_get(&anon_name->kref);
}
static inline void anon_vma_name_put(struct anon_vma_name *anon_name)
{
if (anon_name)
kref_put(&anon_name->kref, anon_vma_name_free);
}
static inline
struct anon_vma_name *anon_vma_name_reuse(struct anon_vma_name *anon_name)
{
/* Prevent anon_name refcount saturation early on */
if (kref_read(&anon_name->kref) < REFCOUNT_MAX) {
anon_vma_name_get(anon_name);
return anon_name;
}
return anon_vma_name_alloc(anon_name->name);
}
static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
struct vm_area_struct *new_vma)
{
struct anon_vma_name *anon_name = anon_vma_name(orig_vma);
if (anon_name)
new_vma->anon_name = anon_vma_name_reuse(anon_name);
}
static inline void free_anon_vma_name(struct vm_area_struct *vma)
{
/*
* Not using anon_vma_name because it generates a warning if mmap_lock
* is not held, which might be the case here.
*/
anon_vma_name_put(vma->anon_name);
}
static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
struct anon_vma_name *anon_name2)
{
if (anon_name1 == anon_name2)
return true;
return anon_name1 && anon_name2 &&
!strcmp(anon_name1->name, anon_name2->name);
}
#else /* CONFIG_ANON_VMA_NAME */
static inline void anon_vma_name_get(struct anon_vma_name *anon_name) {}
static inline void anon_vma_name_put(struct anon_vma_name *anon_name) {}
static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
struct vm_area_struct *new_vma) {}
static inline void free_anon_vma_name(struct vm_area_struct *vma) {}
static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
struct anon_vma_name *anon_name2)
{
return true;
}
#endif /* CONFIG_ANON_VMA_NAME */
static inline void init_tlb_flush_pending(struct mm_struct *mm)
{
atomic_set(&mm->tlb_flush_pending, 0);
}
static inline void inc_tlb_flush_pending(struct mm_struct *mm)
{
atomic_inc(&mm->tlb_flush_pending);
/*
* The only time this value is relevant is when there are indeed pages
* to flush. And we'll only flush pages after changing them, which
* requires the PTL.
*
* So the ordering here is:
*
* atomic_inc(&mm->tlb_flush_pending);
* spin_lock(&ptl);
* ...
* set_pte_at();
* spin_unlock(&ptl);
*
* spin_lock(&ptl)
* mm_tlb_flush_pending();
* ....
* spin_unlock(&ptl);
*
* flush_tlb_range();
* atomic_dec(&mm->tlb_flush_pending);
*
* Where the increment if constrained by the PTL unlock, it thus
* ensures that the increment is visible if the PTE modification is
* visible. After all, if there is no PTE modification, nobody cares
* about TLB flushes either.
*
* This very much relies on users (mm_tlb_flush_pending() and
* mm_tlb_flush_nested()) only caring about _specific_ PTEs (and
* therefore specific PTLs), because with SPLIT_PTE_PTLOCKS and RCpc
* locks (PPC) the unlock of one doesn't order against the lock of
* another PTL.
*
* The decrement is ordered by the flush_tlb_range(), such that
* mm_tlb_flush_pending() will not return false unless all flushes have
* completed.
*/
}
static inline void dec_tlb_flush_pending(struct mm_struct *mm)
{
/*
* See inc_tlb_flush_pending().
*
* This cannot be smp_mb__before_atomic() because smp_mb() simply does
* not order against TLB invalidate completion, which is what we need.
*
* Therefore we must rely on tlb_flush_*() to guarantee order.
*/
atomic_dec(&mm->tlb_flush_pending);
}
static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
{
/*
* Must be called after having acquired the PTL; orders against that
* PTLs release and therefore ensures that if we observe the modified
* PTE we must also observe the increment from inc_tlb_flush_pending().
*
* That is, it only guarantees to return true if there is a flush
* pending for _this_ PTL.
*/
return atomic_read(&mm->tlb_flush_pending);
}
static inline bool mm_tlb_flush_nested(struct mm_struct *mm)
{
/*
* Similar to mm_tlb_flush_pending(), we must have acquired the PTL
* for which there is a TLB flush pending in order to guarantee
* we've seen both that PTE modification and the increment.
*
* (no requirement on actually still holding the PTL, that is irrelevant)
*/
return atomic_read(&mm->tlb_flush_pending) > 1;
}
#ifdef CONFIG_MMU
/*
* Computes the pte marker to copy from the given source entry into dst_vma.
* If no marker should be copied, returns 0.
* The caller should insert a new pte created with make_pte_marker().
*/
static inline pte_marker copy_pte_marker(
swp_entry_t entry, struct vm_area_struct *dst_vma)
{
pte_marker srcm = pte_marker_get(entry);
/* Always copy error entries. */
pte_marker dstm = srcm & PTE_MARKER_POISONED;
/* Only copy PTE markers if UFFD register matches. */
if ((srcm & PTE_MARKER_UFFD_WP) && userfaultfd_wp(dst_vma))
dstm |= PTE_MARKER_UFFD_WP;
return dstm;
}
#endif
/*
* If this pte is wr-protected by uffd-wp in any form, arm the special pte to
* replace a none pte. NOTE! This should only be called when *pte is already
* cleared so we will never accidentally replace something valuable. Meanwhile
* none pte also means we are not demoting the pte so tlb flushed is not needed.
* E.g., when pte cleared the caller should have taken care of the tlb flush.
*
* Must be called with pgtable lock held so that no thread will see the none
* pte, and if they see it, they'll fault and serialize at the pgtable lock.
*
* This function is a no-op if PTE_MARKER_UFFD_WP is not enabled.
*/
static inline void
pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
pte_t *pte, pte_t pteval)
{
#ifdef CONFIG_PTE_MARKER_UFFD_WP
bool arm_uffd_pte = false;
/* The current status of the pte should be "cleared" before calling */
WARN_ON_ONCE(!pte_none(ptep_get(pte)));
/*
* NOTE: userfaultfd_wp_unpopulated() doesn't need this whole
* thing, because when zapping either it means it's dropping the
* page, or in TTU where the present pte will be quickly replaced
* with a swap pte. There's no way of leaking the bit.
*/
if (vma_is_anonymous(vma) || !userfaultfd_wp(vma))
return;
/* A uffd-wp wr-protected normal pte */
if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval)))
arm_uffd_pte = true;
/*
* A uffd-wp wr-protected swap pte. Note: this should even cover an
* existing pte marker with uffd-wp bit set.
*/
if (unlikely(pte_swp_uffd_wp_any(pteval)))
arm_uffd_pte = true;
if (unlikely(arm_uffd_pte))
set_pte_at(vma->vm_mm, addr, pte,
make_pte_marker(PTE_MARKER_UFFD_WP));
#endif
}
static inline bool vma_has_recency(struct vm_area_struct *vma)
{
if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
return false;
if (vma->vm_file && (vma->vm_file->f_mode & FMODE_NOREUSE))
return false;
return true;
}
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* Infrastructure for migratable timers
*
* Copyright(C) 2022 linutronix GmbH
*/
#include <linux/cpuhotplug.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/timerqueue.h>
#include <trace/events/ipi.h>
#include "timer_migration.h"
#include "tick-internal.h"
#define CREATE_TRACE_POINTS
#include <trace/events/timer_migration.h>
/*
* The timer migration mechanism is built on a hierarchy of groups. The
* lowest level group contains CPUs, the next level groups of CPU groups
* and so forth. The CPU groups are kept per node so for the normal case
* lock contention won't happen across nodes. Depending on the number of
* CPUs per node even the next level might be kept as groups of CPU groups
* per node and only the levels above cross the node topology.
*
* Example topology for a two node system with 24 CPUs each.
*
* LVL 2 [GRP2:0]
* GRP1:0 = GRP1:M
*
* LVL 1 [GRP1:0] [GRP1:1]
* GRP0:0 - GRP0:2 GRP0:3 - GRP0:5
*
* LVL 0 [GRP0:0] [GRP0:1] [GRP0:2] [GRP0:3] [GRP0:4] [GRP0:5]
* CPUS 0-7 8-15 16-23 24-31 32-39 40-47
*
* The groups hold a timer queue of events sorted by expiry time. These
* queues are updated when CPUs go in idle. When they come out of idle
* ignore flag of events is set.
*
* Each group has a designated migrator CPU/group as long as a CPU/group is
* active in the group. This designated role is necessary to avoid that all
* active CPUs in a group try to migrate expired timers from other CPUs,
* which would result in massive lock bouncing.
*
* When a CPU is awake, it checks in it's own timer tick the group
* hierarchy up to the point where it is assigned the migrator role or if
* no CPU is active, it also checks the groups where no migrator is set
* (TMIGR_NONE).
*
* If it finds expired timers in one of the group queues it pulls them over
* from the idle CPU and runs the timer function. After that it updates the
* group and the parent groups if required.
*
* CPUs which go idle arm their CPU local timer hardware for the next local
* (pinned) timer event. If the next migratable timer expires after the
* next local timer or the CPU has no migratable timer pending then the
* CPU does not queue an event in the LVL0 group. If the next migratable
* timer expires before the next local timer then the CPU queues that timer
* in the LVL0 group. In both cases the CPU marks itself idle in the LVL0
* group.
*
* When CPU comes out of idle and when a group has at least a single active
* child, the ignore flag of the tmigr_event is set. This indicates, that
* the event is ignored even if it is still enqueued in the parent groups
* timer queue. It will be removed when touching the timer queue the next
* time. This spares locking in active path as the lock protects (after
* setup) only event information. For more information about locking,
* please read the section "Locking rules".
*
* If the CPU is the migrator of the group then it delegates that role to
* the next active CPU in the group or sets migrator to TMIGR_NONE when
* there is no active CPU in the group. This delegation needs to be
* propagated up the hierarchy so hand over from other leaves can happen at
* all hierarchy levels w/o doing a search.
*
* When the last CPU in the system goes idle, then it drops all migrator
* duties up to the top level of the hierarchy (LVL2 in the example). It
* then has to make sure, that it arms it's own local hardware timer for
* the earliest event in the system.
*
*
* Lifetime rules:
* ---------------
*
* The groups are built up at init time or when CPUs come online. They are
* not destroyed when a group becomes empty due to offlining. The group
* just won't participate in the hierarchy management anymore. Destroying
* groups would result in interesting race conditions which would just make
* the whole mechanism slow and complex.
*
*
* Locking rules:
* --------------
*
* For setting up new groups and handling events it's required to lock both
* child and parent group. The lock ordering is always bottom up. This also
* includes the per CPU locks in struct tmigr_cpu. For updating the migrator and
* active CPU/group information atomic_try_cmpxchg() is used instead and only
* the per CPU tmigr_cpu->lock is held.
*
* During the setup of groups tmigr_level_list is required. It is protected by
* @tmigr_mutex.
*
* When @timer_base->lock as well as tmigr related locks are required, the lock
* ordering is: first @timer_base->lock, afterwards tmigr related locks.
*
*
* Protection of the tmigr group state information:
* ------------------------------------------------
*
* The state information with the list of active children and migrator needs to
* be protected by a sequence counter. It prevents a race when updates in child
* groups are propagated in changed order. The state update is performed
* lockless and group wise. The following scenario describes what happens
* without updating the sequence counter:
*
* Therefore, let's take three groups and four CPUs (CPU2 and CPU3 as well
* as GRP0:1 will not change during the scenario):
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:0, GRP0:1
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = CPU0 migrator = CPU2
* active = CPU0 active = CPU2
* / \ / \
* CPUs 0 1 2 3
* active idle active idle
*
*
* 1. CPU0 goes idle. As the update is performed group wise, in the first step
* only GRP0:0 is updated. The update of GRP1:0 is pending as CPU0 has to
* walk the hierarchy.
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:0, GRP0:1
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* --> migrator = TMIGR_NONE migrator = CPU2
* --> active = active = CPU2
* / \ / \
* CPUs 0 1 2 3
* --> idle idle active idle
*
* 2. While CPU0 goes idle and continues to update the state, CPU1 comes out of
* idle. CPU1 updates GRP0:0. The update for GRP1:0 is pending as CPU1 also
* has to walk the hierarchy. Both CPUs (CPU0 and CPU1) now walk the
* hierarchy to perform the needed update from their point of view. The
* currently visible state looks the following:
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:0, GRP0:1
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* --> migrator = CPU1 migrator = CPU2
* --> active = CPU1 active = CPU2
* / \ / \
* CPUs 0 1 2 3
* idle --> active active idle
*
* 3. Here is the race condition: CPU1 managed to propagate its changes (from
* step 2) through the hierarchy to GRP1:0 before CPU0 (step 1) did. The
* active members of GRP1:0 remain unchanged after the update since it is
* still valid from CPU1 current point of view:
*
* LVL 1 [GRP1:0]
* --> migrator = GRP0:1
* --> active = GRP0:0, GRP0:1
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = CPU1 migrator = CPU2
* active = CPU1 active = CPU2
* / \ / \
* CPUs 0 1 2 3
* idle active active idle
*
* 4. Now CPU0 finally propagates its changes (from step 1) to GRP1:0.
*
* LVL 1 [GRP1:0]
* --> migrator = GRP0:1
* --> active = GRP0:1
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = CPU1 migrator = CPU2
* active = CPU1 active = CPU2
* / \ / \
* CPUs 0 1 2 3
* idle active active idle
*
*
* The race of CPU0 vs. CPU1 led to an inconsistent state in GRP1:0. CPU1 is
* active and is correctly listed as active in GRP0:0. However GRP1:0 does not
* have GRP0:0 listed as active, which is wrong. The sequence counter has been
* added to avoid inconsistent states during updates. The state is updated
* atomically only if all members, including the sequence counter, match the
* expected value (compare-and-exchange).
*
* Looking back at the previous example with the addition of the sequence
* counter: The update as performed by CPU0 in step 4 will fail. CPU1 changed
* the sequence number during the update in step 3 so the expected old value (as
* seen by CPU0 before starting the walk) does not match.
*
* Prevent race between new event and last CPU going inactive
* ----------------------------------------------------------
*
* When the last CPU is going idle and there is a concurrent update of a new
* first global timer of an idle CPU, the group and child states have to be read
* while holding the lock in tmigr_update_events(). The following scenario shows
* what happens, when this is not done.
*
* 1. Only CPU2 is active:
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:1
* next_expiry = KTIME_MAX
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = CPU2
* active = active = CPU2
* next_expiry = KTIME_MAX next_expiry = KTIME_MAX
* / \ / \
* CPUs 0 1 2 3
* idle idle active idle
*
* 2. Now CPU 2 goes idle (and has no global timer, that has to be handled) and
* propagates that to GRP0:1:
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:1
* next_expiry = KTIME_MAX
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE --> migrator = TMIGR_NONE
* active = --> active =
* next_expiry = KTIME_MAX next_expiry = KTIME_MAX
* / \ / \
* CPUs 0 1 2 3
* idle idle --> idle idle
*
* 3. Now the idle state is propagated up to GRP1:0. As this is now the last
* child going idle in top level group, the expiry of the next group event
* has to be handed back to make sure no event is lost. As there is no event
* enqueued, KTIME_MAX is handed back to CPU2.
*
* LVL 1 [GRP1:0]
* --> migrator = TMIGR_NONE
* --> active =
* next_expiry = KTIME_MAX
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = TMIGR_NONE
* active = active =
* next_expiry = KTIME_MAX next_expiry = KTIME_MAX
* / \ / \
* CPUs 0 1 2 3
* idle idle --> idle idle
*
* 4. CPU 0 has a new timer queued from idle and it expires at TIMER0. CPU0
* propagates that to GRP0:0:
*
* LVL 1 [GRP1:0]
* migrator = TMIGR_NONE
* active =
* next_expiry = KTIME_MAX
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = TMIGR_NONE
* active = active =
* --> next_expiry = TIMER0 next_expiry = KTIME_MAX
* / \ / \
* CPUs 0 1 2 3
* idle idle idle idle
*
* 5. GRP0:0 is not active, so the new timer has to be propagated to
* GRP1:0. Therefore the GRP1:0 state has to be read. When the stalled value
* (from step 2) is read, the timer is enqueued into GRP1:0, but nothing is
* handed back to CPU0, as it seems that there is still an active child in
* top level group.
*
* LVL 1 [GRP1:0]
* migrator = TMIGR_NONE
* active =
* --> next_expiry = TIMER0
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = TMIGR_NONE
* active = active =
* next_expiry = TIMER0 next_expiry = KTIME_MAX
* / \ / \
* CPUs 0 1 2 3
* idle idle idle idle
*
* This is prevented by reading the state when holding the lock (when a new
* timer has to be propagated from idle path)::
*
* CPU2 (tmigr_inactive_up()) CPU0 (tmigr_new_timer_up())
* -------------------------- ---------------------------
* // step 3:
* cmpxchg(&GRP1:0->state);
* tmigr_update_events() {
* spin_lock(&GRP1:0->lock);
* // ... update events ...
* // hand back first expiry when GRP1:0 is idle
* spin_unlock(&GRP1:0->lock);
* // ^^^ release state modification
* }
* tmigr_update_events() {
* spin_lock(&GRP1:0->lock)
* // ^^^ acquire state modification
* group_state = atomic_read(&GRP1:0->state)
* // .... update events ...
* // hand back first expiry when GRP1:0 is idle
* spin_unlock(&GRP1:0->lock) <3>
* // ^^^ makes state visible for other
* // callers of tmigr_new_timer_up()
* }
*
* When CPU0 grabs the lock directly after cmpxchg, the first timer is reported
* back to CPU0 and also later on to CPU2. So no timer is missed. A concurrent
* update of the group state from active path is no problem, as the upcoming CPU
* will take care of the group events.
*
* Required event and timerqueue update after a remote expiry:
* -----------------------------------------------------------
*
* After expiring timers of a remote CPU, a walk through the hierarchy and
* update of events and timerqueues is required. It is obviously needed if there
* is a 'new' global timer but also if there is no new global timer but the
* remote CPU is still idle.
*
* 1. CPU0 and CPU1 are idle and have both a global timer expiring at the same
* time. So both have an event enqueued in the timerqueue of GRP0:0. CPU3 is
* also idle and has no global timer pending. CPU2 is the only active CPU and
* thus also the migrator:
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:1
* --> timerqueue = evt-GRP0:0
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = CPU2
* active = active = CPU2
* groupevt.ignore = false groupevt.ignore = true
* groupevt.cpu = CPU0 groupevt.cpu =
* timerqueue = evt-CPU0, timerqueue =
* evt-CPU1
* / \ / \
* CPUs 0 1 2 3
* idle idle active idle
*
* 2. CPU2 starts to expire remote timers. It starts with LVL0 group
* GRP0:1. There is no event queued in the timerqueue, so CPU2 continues with
* the parent of GRP0:1: GRP1:0. In GRP1:0 it dequeues the first event. It
* looks at tmigr_event::cpu struct member and expires the pending timer(s)
* of CPU0.
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:1
* --> timerqueue =
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = CPU2
* active = active = CPU2
* groupevt.ignore = false groupevt.ignore = true
* --> groupevt.cpu = CPU0 groupevt.cpu =
* timerqueue = evt-CPU0, timerqueue =
* evt-CPU1
* / \ / \
* CPUs 0 1 2 3
* idle idle active idle
*
* 3. Some work has to be done after expiring the timers of CPU0. If we stop
* here, then CPU1's pending global timer(s) will not expire in time and the
* timerqueue of GRP0:0 has still an event for CPU0 enqueued which has just
* been processed. So it is required to walk the hierarchy from CPU0's point
* of view and update it accordingly. CPU0's event will be removed from the
* timerqueue because it has no pending timer. If CPU0 would have a timer
* pending then it has to expire after CPU1's first timer because all timers
* from this period were just expired. Either way CPU1's event will be first
* in GRP0:0's timerqueue and therefore set in the CPU field of the group
* event which is then enqueued in GRP1:0's timerqueue as GRP0:0 is still not
* active:
*
* LVL 1 [GRP1:0]
* migrator = GRP0:1
* active = GRP0:1
* --> timerqueue = evt-GRP0:0
* / \
* LVL 0 [GRP0:0] [GRP0:1]
* migrator = TMIGR_NONE migrator = CPU2
* active = active = CPU2
* groupevt.ignore = false groupevt.ignore = true
* --> groupevt.cpu = CPU1 groupevt.cpu =
* --> timerqueue = evt-CPU1 timerqueue =
* / \ / \
* CPUs 0 1 2 3
* idle idle active idle
*
* Now CPU2 (migrator) will continue step 2 at GRP1:0 and will expire the
* timer(s) of CPU1.
*
* The hierarchy walk in step 3 can be skipped if the migrator notices that a
* CPU of GRP0:0 is active again. The CPU will mark GRP0:0 active and take care
* of the group as migrator and any needed updates within the hierarchy.
*/
static DEFINE_MUTEX(tmigr_mutex);
static struct list_head *tmigr_level_list __read_mostly;
static unsigned int tmigr_hierarchy_levels __read_mostly;
static unsigned int tmigr_crossnode_level __read_mostly;
static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu);
#define TMIGR_NONE 0xFF
#define BIT_CNT 8
static inline bool tmigr_is_not_available(struct tmigr_cpu *tmc)
{
return !(tmc->tmgroup && tmc->online);
}
/*
* Returns true, when @childmask corresponds to the group migrator or when the
* group is not active - so no migrator is set.
*/
static bool tmigr_check_migrator(struct tmigr_group *group, u8 childmask)
{
union tmigr_state s;
s.state = atomic_read(&group->migr_state); if ((s.migrator == childmask) || (s.migrator == TMIGR_NONE))
return true;
return false;
}
static bool tmigr_check_migrator_and_lonely(struct tmigr_group *group, u8 childmask)
{
bool lonely, migrator = false;
unsigned long active;
union tmigr_state s;
s.state = atomic_read(&group->migr_state);
if ((s.migrator == childmask) || (s.migrator == TMIGR_NONE))
migrator = true;
active = s.active;
lonely = bitmap_weight(&active, BIT_CNT) <= 1;
return (migrator && lonely);
}
static bool tmigr_check_lonely(struct tmigr_group *group)
{
unsigned long active;
union tmigr_state s;
s.state = atomic_read(&group->migr_state);
active = s.active;
return bitmap_weight(&active, BIT_CNT) <= 1;
}
typedef bool (*up_f)(struct tmigr_group *, struct tmigr_group *, void *);
static void __walk_groups(up_f up, void *data,
struct tmigr_cpu *tmc)
{
struct tmigr_group *child = NULL, *group = tmc->tmgroup;
do {
WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels); if (up(group, child, data))
break;
child = group;
group = group->parent;
} while (group);
}
static void walk_groups(up_f up, void *data, struct tmigr_cpu *tmc)
{
lockdep_assert_held(&tmc->lock);
__walk_groups(up, data, tmc);
}
/**
* struct tmigr_walk - data required for walking the hierarchy
* @nextexp: Next CPU event expiry information which is handed into
* the timer migration code by the timer code
* (get_next_timer_interrupt())
* @firstexp: Contains the first event expiry information when last
* active CPU of hierarchy is on the way to idle to make
* sure CPU will be back in time.
* @evt: Pointer to tmigr_event which needs to be queued (of idle
* child group)
* @childmask: childmask of child group
* @remote: Is set, when the new timer path is executed in
* tmigr_handle_remote_cpu()
*/
struct tmigr_walk {
u64 nextexp;
u64 firstexp;
struct tmigr_event *evt;
u8 childmask;
bool remote;
};
/**
* struct tmigr_remote_data - data required for remote expiry hierarchy walk
* @basej: timer base in jiffies
* @now: timer base monotonic
* @firstexp: returns expiry of the first timer in the idle timer
* migration hierarchy to make sure the timer is handled in
* time; it is stored in the per CPU tmigr_cpu struct of
* CPU which expires remote timers
* @childmask: childmask of child group
* @check: is set if there is the need to handle remote timers;
* required in tmigr_requires_handle_remote() only
* @tmc_active: this flag indicates, whether the CPU which triggers
* the hierarchy walk is !idle in the timer migration
* hierarchy. When the CPU is idle and the whole hierarchy is
* idle, only the first event of the top level has to be
* considered.
*/
struct tmigr_remote_data {
unsigned long basej;
u64 now;
u64 firstexp;
u8 childmask;
bool check;
bool tmc_active;
};
/*
* Returns the next event of the timerqueue @group->events
*
* Removes timers with ignore flag and update next_expiry of the group. Values
* of the group event are updated in tmigr_update_events() only.
*/
static struct tmigr_event *tmigr_next_groupevt(struct tmigr_group *group)
{
struct timerqueue_node *node = NULL;
struct tmigr_event *evt = NULL;
lockdep_assert_held(&group->lock);
WRITE_ONCE(group->next_expiry, KTIME_MAX);
while ((node = timerqueue_getnext(&group->events))) {
evt = container_of(node, struct tmigr_event, nextevt);
if (!evt->ignore) {
WRITE_ONCE(group->next_expiry, evt->nextevt.expires);
return evt;
}
/*
* Remove next timers with ignore flag, because the group lock
* is held anyway
*/
if (!timerqueue_del(&group->events, node))
break;
}
return NULL;
}
/*
* Return the next event (with the expiry equal or before @now)
*
* Event, which is returned, is also removed from the queue.
*/
static struct tmigr_event *tmigr_next_expired_groupevt(struct tmigr_group *group,
u64 now)
{
struct tmigr_event *evt = tmigr_next_groupevt(group);
if (!evt || now < evt->nextevt.expires)
return NULL;
/*
* The event is ready to expire. Remove it and update next group event.
*/
timerqueue_del(&group->events, &evt->nextevt);
tmigr_next_groupevt(group);
return evt;
}
static u64 tmigr_next_groupevt_expires(struct tmigr_group *group)
{
struct tmigr_event *evt;
evt = tmigr_next_groupevt(group);
if (!evt)
return KTIME_MAX;
else
return evt->nextevt.expires;
}
static bool tmigr_active_up(struct tmigr_group *group,
struct tmigr_group *child,
void *ptr)
{
union tmigr_state curstate, newstate;
struct tmigr_walk *data = ptr;
bool walk_done;
u8 childmask;
childmask = data->childmask;
/*
* No memory barrier is required here in contrast to
* tmigr_inactive_up(), as the group state change does not depend on the
* child state.
*/
curstate.state = atomic_read(&group->migr_state);
do {
newstate = curstate;
walk_done = true;
if (newstate.migrator == TMIGR_NONE) {
newstate.migrator = childmask;
/* Changes need to be propagated */
walk_done = false;
}
newstate.active |= childmask;
newstate.seq++;
} while (!atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state));
if ((walk_done == false) && group->parent)
data->childmask = group->childmask;
/*
* The group is active (again). The group event might be still queued
* into the parent group's timerqueue but can now be handled by the
* migrator of this group. Therefore the ignore flag for the group event
* is updated to reflect this.
*
* The update of the ignore flag in the active path is done lockless. In
* worst case the migrator of the parent group observes the change too
* late and expires remotely all events belonging to this group. The
* lock is held while updating the ignore flag in idle path. So this
* state change will not be lost.
*/
group->groupevt.ignore = true;
trace_tmigr_group_set_cpu_active(group, newstate, childmask);
return walk_done;
}
static void __tmigr_cpu_activate(struct tmigr_cpu *tmc)
{
struct tmigr_walk data;
data.childmask = tmc->childmask;
trace_tmigr_cpu_active(tmc);
tmc->cpuevt.ignore = true;
WRITE_ONCE(tmc->wakeup, KTIME_MAX);
walk_groups(&tmigr_active_up, &data, tmc);
}
/**
* tmigr_cpu_activate() - set this CPU active in timer migration hierarchy
*
* Call site timer_clear_idle() is called with interrupts disabled.
*/
void tmigr_cpu_activate(void)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
if (tmigr_is_not_available(tmc))
return;
if (WARN_ON_ONCE(!tmc->idle))
return;
raw_spin_lock(&tmc->lock);
tmc->idle = false;
__tmigr_cpu_activate(tmc);
raw_spin_unlock(&tmc->lock);
}
/*
* Returns true, if there is nothing to be propagated to the next level
*
* @data->firstexp is set to expiry of first gobal event of the (top level of
* the) hierarchy, but only when hierarchy is completely idle.
*
* The child and group states need to be read under the lock, to prevent a race
* against a concurrent tmigr_inactive_up() run when the last CPU goes idle. See
* also section "Prevent race between new event and last CPU going inactive" in
* the documentation at the top.
*
* This is the only place where the group event expiry value is set.
*/
static
bool tmigr_update_events(struct tmigr_group *group, struct tmigr_group *child,
struct tmigr_walk *data)
{
struct tmigr_event *evt, *first_childevt;
union tmigr_state childstate, groupstate;
bool remote = data->remote;
bool walk_done = false;
u64 nextexp;
if (child) {
raw_spin_lock(&child->lock);
raw_spin_lock_nested(&group->lock, SINGLE_DEPTH_NESTING);
childstate.state = atomic_read(&child->migr_state);
groupstate.state = atomic_read(&group->migr_state);
if (childstate.active) {
walk_done = true;
goto unlock;
}
first_childevt = tmigr_next_groupevt(child);
nextexp = child->next_expiry;
evt = &child->groupevt;
evt->ignore = (nextexp == KTIME_MAX) ? true : false;
} else {
nextexp = data->nextexp;
first_childevt = evt = data->evt;
/*
* Walking the hierarchy is required in any case when a
* remote expiry was done before. This ensures to not lose
* already queued events in non active groups (see section
* "Required event and timerqueue update after a remote
* expiry" in the documentation at the top).
*
* The two call sites which are executed without a remote expiry
* before, are not prevented from propagating changes through
* the hierarchy by the return:
* - When entering this path by tmigr_new_timer(), @evt->ignore
* is never set.
* - tmigr_inactive_up() takes care of the propagation by
* itself and ignores the return value. But an immediate
* return is possible if there is a parent, sparing group
* locking at this level, because the upper walking call to
* the parent will take care about removing this event from
* within the group and update next_expiry accordingly.
*
* However if there is no parent, ie: the hierarchy has only a
* single level so @group is the top level group, make sure the
* first event information of the group is updated properly and
* also handled properly, so skip this fast return path.
*/
if (evt->ignore && !remote && group->parent)
return true;
raw_spin_lock(&group->lock);
childstate.state = 0;
groupstate.state = atomic_read(&group->migr_state);
}
/*
* If the child event is already queued in the group, remove it from the
* queue when the expiry time changed only or when it could be ignored.
*/
if (timerqueue_node_queued(&evt->nextevt)) {
if ((evt->nextevt.expires == nextexp) && !evt->ignore) {
/* Make sure not to miss a new CPU event with the same expiry */
evt->cpu = first_childevt->cpu;
goto check_toplvl;
}
if (!timerqueue_del(&group->events, &evt->nextevt))
WRITE_ONCE(group->next_expiry, KTIME_MAX);
}
if (evt->ignore) {
/*
* When the next child event could be ignored (nextexp is
* KTIME_MAX) and there was no remote timer handling before or
* the group is already active, there is no need to walk the
* hierarchy even if there is a parent group.
*
* The other way round: even if the event could be ignored, but
* if a remote timer handling was executed before and the group
* is not active, walking the hierarchy is required to not miss
* an enqueued timer in the non active group. The enqueued timer
* of the group needs to be propagated to a higher level to
* ensure it is handled.
*/
if (!remote || groupstate.active)
walk_done = true;
} else {
evt->nextevt.expires = nextexp;
evt->cpu = first_childevt->cpu;
if (timerqueue_add(&group->events, &evt->nextevt))
WRITE_ONCE(group->next_expiry, nextexp);
}
check_toplvl:
if (!group->parent && (groupstate.migrator == TMIGR_NONE)) {
walk_done = true;
/*
* Nothing to do when update was done during remote timer
* handling. First timer in top level group which needs to be
* handled when top level group is not active, is calculated
* directly in tmigr_handle_remote_up().
*/
if (remote)
goto unlock;
/*
* The top level group is idle and it has to be ensured the
* global timers are handled in time. (This could be optimized
* by keeping track of the last global scheduled event and only
* arming it on the CPU if the new event is earlier. Not sure if
* its worth the complexity.)
*/
data->firstexp = tmigr_next_groupevt_expires(group);
}
trace_tmigr_update_events(child, group, childstate, groupstate,
nextexp);
unlock:
raw_spin_unlock(&group->lock);
if (child)
raw_spin_unlock(&child->lock);
return walk_done;
}
static bool tmigr_new_timer_up(struct tmigr_group *group,
struct tmigr_group *child,
void *ptr)
{
struct tmigr_walk *data = ptr;
return tmigr_update_events(group, child, data);
}
/*
* Returns the expiry of the next timer that needs to be handled. KTIME_MAX is
* returned, if an active CPU will handle all the timer migration hierarchy
* timers.
*/
static u64 tmigr_new_timer(struct tmigr_cpu *tmc, u64 nextexp)
{
struct tmigr_walk data = { .nextexp = nextexp,
.firstexp = KTIME_MAX,
.evt = &tmc->cpuevt };
lockdep_assert_held(&tmc->lock);
if (tmc->remote)
return KTIME_MAX;
trace_tmigr_cpu_new_timer(tmc);
tmc->cpuevt.ignore = false;
data.remote = false;
walk_groups(&tmigr_new_timer_up, &data, tmc);
/* If there is a new first global event, make sure it is handled */
return data.firstexp;
}
static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now,
unsigned long jif)
{
struct timer_events tevt;
struct tmigr_walk data;
struct tmigr_cpu *tmc;
tmc = per_cpu_ptr(&tmigr_cpu, cpu);
raw_spin_lock_irq(&tmc->lock);
/*
* If the remote CPU is offline then the timers have been migrated to
* another CPU.
*
* If tmigr_cpu::remote is set, at the moment another CPU already
* expires the timers of the remote CPU.
*
* If tmigr_event::ignore is set, then the CPU returns from idle and
* takes care of its timers.
*
* If the next event expires in the future, then the event has been
* updated and there are no timers to expire right now. The CPU which
* updated the event takes care when hierarchy is completely
* idle. Otherwise the migrator does it as the event is enqueued.
*/
if (!tmc->online || tmc->remote || tmc->cpuevt.ignore ||
now < tmc->cpuevt.nextevt.expires) {
raw_spin_unlock_irq(&tmc->lock);
return;
}
trace_tmigr_handle_remote_cpu(tmc);
tmc->remote = true;
WRITE_ONCE(tmc->wakeup, KTIME_MAX);
/* Drop the lock to allow the remote CPU to exit idle */
raw_spin_unlock_irq(&tmc->lock);
if (cpu != smp_processor_id())
timer_expire_remote(cpu);
/*
* Lock ordering needs to be preserved - timer_base locks before tmigr
* related locks (see section "Locking rules" in the documentation at
* the top). During fetching the next timer interrupt, also tmc->lock
* needs to be held. Otherwise there is a possible race window against
* the CPU itself when it comes out of idle, updates the first timer in
* the hierarchy and goes back to idle.
*
* timer base locks are dropped as fast as possible: After checking
* whether the remote CPU went offline in the meantime and after
* fetching the next remote timer interrupt. Dropping the locks as fast
* as possible keeps the locking region small and prevents holding
* several (unnecessary) locks during walking the hierarchy for updating
* the timerqueue and group events.
*/
local_irq_disable();
timer_lock_remote_bases(cpu);
raw_spin_lock(&tmc->lock);
/*
* When the CPU went offline in the meantime, no hierarchy walk has to
* be done for updating the queued events, because the walk was
* already done during marking the CPU offline in the hierarchy.
*
* When the CPU is no longer idle, the CPU takes care of the timers and
* also of the timers in the hierarchy.
*
* (See also section "Required event and timerqueue update after a
* remote expiry" in the documentation at the top)
*/
if (!tmc->online || !tmc->idle) {
timer_unlock_remote_bases(cpu);
goto unlock;
}
/* next event of CPU */
fetch_next_timer_interrupt_remote(jif, now, &tevt, cpu);
timer_unlock_remote_bases(cpu);
data.nextexp = tevt.global;
data.firstexp = KTIME_MAX;
data.evt = &tmc->cpuevt;
data.remote = true;
/*
* The update is done even when there is no 'new' global timer pending
* on the remote CPU (see section "Required event and timerqueue update
* after a remote expiry" in the documentation at the top)
*/
walk_groups(&tmigr_new_timer_up, &data, tmc);
unlock:
tmc->remote = false;
raw_spin_unlock_irq(&tmc->lock);
}
static bool tmigr_handle_remote_up(struct tmigr_group *group,
struct tmigr_group *child,
void *ptr)
{
struct tmigr_remote_data *data = ptr;
struct tmigr_event *evt;
unsigned long jif;
u8 childmask;
u64 now;
jif = data->basej;
now = data->now;
childmask = data->childmask;
trace_tmigr_handle_remote(group);
again:
/*
* Handle the group only if @childmask is the migrator or if the
* group has no migrator. Otherwise the group is active and is
* handled by its own migrator.
*/
if (!tmigr_check_migrator(group, childmask))
return true;
raw_spin_lock_irq(&group->lock);
evt = tmigr_next_expired_groupevt(group, now);
if (evt) {
unsigned int remote_cpu = evt->cpu;
raw_spin_unlock_irq(&group->lock);
tmigr_handle_remote_cpu(remote_cpu, now, jif);
/* check if there is another event, that needs to be handled */
goto again;
}
/*
* Update of childmask for the next level and keep track of the expiry
* of the first event that needs to be handled (group->next_expiry was
* updated by tmigr_next_expired_groupevt(), next was set by
* tmigr_handle_remote_cpu()).
*/
data->childmask = group->childmask;
data->firstexp = group->next_expiry;
raw_spin_unlock_irq(&group->lock);
return false;
}
/**
* tmigr_handle_remote() - Handle global timers of remote idle CPUs
*
* Called from the timer soft interrupt with interrupts enabled.
*/
void tmigr_handle_remote(void)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
struct tmigr_remote_data data;
if (tmigr_is_not_available(tmc))
return;
data.childmask = tmc->childmask;
data.firstexp = KTIME_MAX;
/*
* NOTE: This is a doubled check because the migrator test will be done
* in tmigr_handle_remote_up() anyway. Keep this check to speed up the
* return when nothing has to be done.
*/
if (!tmigr_check_migrator(tmc->tmgroup, tmc->childmask)) {
/*
* If this CPU was an idle migrator, make sure to clear its wakeup
* value so it won't chase timers that have already expired elsewhere.
* This avoids endless requeue from tmigr_new_timer().
*/
if (READ_ONCE(tmc->wakeup) == KTIME_MAX)
return;
}
data.now = get_jiffies_update(&data.basej);
/*
* Update @tmc->wakeup only at the end and do not reset @tmc->wakeup to
* KTIME_MAX. Even if tmc->lock is not held during the whole remote
* handling, tmc->wakeup is fine to be stale as it is called in
* interrupt context and tick_nohz_next_event() is executed in interrupt
* exit path only after processing the last pending interrupt.
*/
__walk_groups(&tmigr_handle_remote_up, &data, tmc);
raw_spin_lock_irq(&tmc->lock);
WRITE_ONCE(tmc->wakeup, data.firstexp);
raw_spin_unlock_irq(&tmc->lock);
}
static bool tmigr_requires_handle_remote_up(struct tmigr_group *group,
struct tmigr_group *child,
void *ptr)
{
struct tmigr_remote_data *data = ptr;
u8 childmask;
childmask = data->childmask;
/*
* Handle the group only if the child is the migrator or if the group
* has no migrator. Otherwise the group is active and is handled by its
* own migrator.
*/
if (!tmigr_check_migrator(group, childmask))
return true;
/*
* When there is a parent group and the CPU which triggered the
* hierarchy walk is not active, proceed the walk to reach the top level
* group before reading the next_expiry value.
*/
if (group->parent && !data->tmc_active)
goto out;
/*
* The lock is required on 32bit architectures to read the variable
* consistently with a concurrent writer. On 64bit the lock is not
* required because the read operation is not split and so it is always
* consistent.
*/
if (IS_ENABLED(CONFIG_64BIT)) {
data->firstexp = READ_ONCE(group->next_expiry);
if (data->now >= data->firstexp) {
data->check = true;
return true;
}
} else {
raw_spin_lock(&group->lock);
data->firstexp = group->next_expiry;
if (data->now >= group->next_expiry) {
data->check = true;
raw_spin_unlock(&group->lock);
return true;
}
raw_spin_unlock(&group->lock);
}
out:
/* Update of childmask for the next level */
data->childmask = group->childmask;
return false;
}
/**
* tmigr_requires_handle_remote() - Check the need of remote timer handling
*
* Must be called with interrupts disabled.
*/
bool tmigr_requires_handle_remote(void)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
struct tmigr_remote_data data;
unsigned long jif;
bool ret = false;
if (tmigr_is_not_available(tmc))
return ret;
data.now = get_jiffies_update(&jif); data.childmask = tmc->childmask;
data.firstexp = KTIME_MAX;
data.tmc_active = !tmc->idle;
data.check = false;
/*
* If the CPU is active, walk the hierarchy to check whether a remote
* expiry is required.
*
* Check is done lockless as interrupts are disabled and @tmc->idle is
* set only by the local CPU.
*/
if (!tmc->idle) {
__walk_groups(&tmigr_requires_handle_remote_up, &data, tmc); return data.check;
}
/*
* When the CPU is idle, compare @tmc->wakeup with @data.now. The lock
* is required on 32bit architectures to read the variable consistently
* with a concurrent writer. On 64bit the lock is not required because
* the read operation is not split and so it is always consistent.
*/
if (IS_ENABLED(CONFIG_64BIT)) {
if (data.now >= READ_ONCE(tmc->wakeup))
return true;
} else {
raw_spin_lock(&tmc->lock);
if (data.now >= tmc->wakeup)
ret = true;
raw_spin_unlock(&tmc->lock);
}
return ret;
}
/**
* tmigr_cpu_new_timer() - enqueue next global timer into hierarchy (idle tmc)
* @nextexp: Next expiry of global timer (or KTIME_MAX if not)
*
* The CPU is already deactivated in the timer migration
* hierarchy. tick_nohz_get_sleep_length() calls tick_nohz_next_event()
* and thereby the timer idle path is executed once more. @tmc->wakeup
* holds the first timer, when the timer migration hierarchy is
* completely idle.
*
* Returns the first timer that needs to be handled by this CPU or KTIME_MAX if
* nothing needs to be done.
*/
u64 tmigr_cpu_new_timer(u64 nextexp)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
u64 ret;
if (tmigr_is_not_available(tmc))
return nextexp;
raw_spin_lock(&tmc->lock);
ret = READ_ONCE(tmc->wakeup);
if (nextexp != KTIME_MAX) {
if (nextexp != tmc->cpuevt.nextevt.expires ||
tmc->cpuevt.ignore) {
ret = tmigr_new_timer(tmc, nextexp);
}
}
/*
* Make sure the reevaluation of timers in idle path will not miss an
* event.
*/
WRITE_ONCE(tmc->wakeup, ret);
trace_tmigr_cpu_new_timer_idle(tmc, nextexp);
raw_spin_unlock(&tmc->lock);
return ret;
}
static bool tmigr_inactive_up(struct tmigr_group *group,
struct tmigr_group *child,
void *ptr)
{
union tmigr_state curstate, newstate, childstate;
struct tmigr_walk *data = ptr;
bool walk_done;
u8 childmask;
childmask = data->childmask;
childstate.state = 0;
/*
* The memory barrier is paired with the cmpxchg() in tmigr_active_up()
* to make sure the updates of child and group states are ordered. The
* ordering is mandatory, as the group state change depends on the child
* state.
*/
curstate.state = atomic_read_acquire(&group->migr_state);
for (;;) {
if (child)
childstate.state = atomic_read(&child->migr_state);
newstate = curstate;
walk_done = true;
/* Reset active bit when the child is no longer active */
if (!childstate.active)
newstate.active &= ~childmask;
if (newstate.migrator == childmask) {
/*
* Find a new migrator for the group, because the child
* group is idle!
*/
if (!childstate.active) {
unsigned long new_migr_bit, active = newstate.active;
new_migr_bit = find_first_bit(&active, BIT_CNT);
if (new_migr_bit != BIT_CNT) {
newstate.migrator = BIT(new_migr_bit);
} else {
newstate.migrator = TMIGR_NONE;
/* Changes need to be propagated */
walk_done = false;
}
}
}
newstate.seq++;
WARN_ON_ONCE((newstate.migrator != TMIGR_NONE) && !(newstate.active));
if (atomic_try_cmpxchg(&group->migr_state, &curstate.state,
newstate.state))
break;
/*
* The memory barrier is paired with the cmpxchg() in
* tmigr_active_up() to make sure the updates of child and group
* states are ordered. It is required only when the above
* try_cmpxchg() fails.
*/
smp_mb__after_atomic();
}
data->remote = false;
/* Event Handling */
tmigr_update_events(group, child, data);
if (group->parent && (walk_done == false))
data->childmask = group->childmask;
/*
* data->firstexp was set by tmigr_update_events() and contains the
* expiry of the first global event which needs to be handled. It
* differs from KTIME_MAX if:
* - group is the top level group and
* - group is idle (which means CPU was the last active CPU in the
* hierarchy) and
* - there is a pending event in the hierarchy
*/
WARN_ON_ONCE(data->firstexp != KTIME_MAX && group->parent);
trace_tmigr_group_set_cpu_inactive(group, newstate, childmask);
return walk_done;
}
static u64 __tmigr_cpu_deactivate(struct tmigr_cpu *tmc, u64 nextexp)
{
struct tmigr_walk data = { .nextexp = nextexp,
.firstexp = KTIME_MAX,
.evt = &tmc->cpuevt,
.childmask = tmc->childmask };
/*
* If nextexp is KTIME_MAX, the CPU event will be ignored because the
* local timer expires before the global timer, no global timer is set
* or CPU goes offline.
*/
if (nextexp != KTIME_MAX)
tmc->cpuevt.ignore = false;
walk_groups(&tmigr_inactive_up, &data, tmc);
return data.firstexp;
}
/**
* tmigr_cpu_deactivate() - Put current CPU into inactive state
* @nextexp: The next global timer expiry of the current CPU
*
* Must be called with interrupts disabled.
*
* Return: the next event expiry of the current CPU or the next event expiry
* from the hierarchy if this CPU is the top level migrator or the hierarchy is
* completely idle.
*/
u64 tmigr_cpu_deactivate(u64 nextexp)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
u64 ret;
if (tmigr_is_not_available(tmc))
return nextexp;
raw_spin_lock(&tmc->lock);
ret = __tmigr_cpu_deactivate(tmc, nextexp);
tmc->idle = true;
/*
* Make sure the reevaluation of timers in idle path will not miss an
* event.
*/
WRITE_ONCE(tmc->wakeup, ret);
trace_tmigr_cpu_idle(tmc, nextexp);
raw_spin_unlock(&tmc->lock);
return ret;
}
/**
* tmigr_quick_check() - Quick forecast of next tmigr event when CPU wants to
* go idle
* @nextevt: The next global timer expiry of the current CPU
*
* Return:
* * KTIME_MAX - when it is probable that nothing has to be done (not
* the only one in the level 0 group; and if it is the
* only one in level 0 group, but there are more than a
* single group active on the way to top level)
* * nextevt - when CPU is offline and has to handle timer on his own
* or when on the way to top in every group only a single
* child is active but @nextevt is before the lowest
* next_expiry encountered while walking up to top level.
* * next_expiry - value of lowest expiry encountered while walking groups
* if only a single child is active on each and @nextevt
* is after this lowest expiry.
*/
u64 tmigr_quick_check(u64 nextevt)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
struct tmigr_group *group = tmc->tmgroup;
if (tmigr_is_not_available(tmc))
return nextevt;
if (WARN_ON_ONCE(tmc->idle))
return nextevt;
if (!tmigr_check_migrator_and_lonely(tmc->tmgroup, tmc->childmask))
return KTIME_MAX;
do {
if (!tmigr_check_lonely(group)) {
return KTIME_MAX;
} else {
/*
* Since current CPU is active, events may not be sorted
* from bottom to the top because the CPU's event is ignored
* up to the top and its sibling's events not propagated upwards.
* Thus keep track of the lowest observed expiry.
*/
nextevt = min_t(u64, nextevt, READ_ONCE(group->next_expiry));
if (!group->parent)
return nextevt;
}
group = group->parent;
} while (group);
return KTIME_MAX;
}
static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,
int node)
{
union tmigr_state s;
raw_spin_lock_init(&group->lock);
group->level = lvl;
group->numa_node = lvl < tmigr_crossnode_level ? node : NUMA_NO_NODE;
group->num_children = 0;
s.migrator = TMIGR_NONE;
s.active = 0;
s.seq = 0;
atomic_set(&group->migr_state, s.state);
timerqueue_init_head(&group->events);
timerqueue_init(&group->groupevt.nextevt);
group->groupevt.nextevt.expires = KTIME_MAX;
WRITE_ONCE(group->next_expiry, KTIME_MAX);
group->groupevt.ignore = true;
}
static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node,
unsigned int lvl)
{
struct tmigr_group *tmp, *group = NULL;
lockdep_assert_held(&tmigr_mutex);
/* Try to attach to an existing group first */
list_for_each_entry(tmp, &tmigr_level_list[lvl], list) {
/*
* If @lvl is below the cross NUMA node level, check whether
* this group belongs to the same NUMA node.
*/
if (lvl < tmigr_crossnode_level && tmp->numa_node != node)
continue;
/* Capacity left? */
if (tmp->num_children >= TMIGR_CHILDREN_PER_GROUP)
continue;
/*
* TODO: A possible further improvement: Make sure that all CPU
* siblings end up in the same group of the lowest level of the
* hierarchy. Rely on the topology sibling mask would be a
* reasonable solution.
*/
group = tmp;
break;
}
if (group)
return group;
/* Allocate and set up a new group */
group = kzalloc_node(sizeof(*group), GFP_KERNEL, node);
if (!group)
return ERR_PTR(-ENOMEM);
tmigr_init_group(group, lvl, node);
/* Setup successful. Add it to the hierarchy */
list_add(&group->list, &tmigr_level_list[lvl]);
trace_tmigr_group_set(group);
return group;
}
static void tmigr_connect_child_parent(struct tmigr_group *child,
struct tmigr_group *parent)
{
union tmigr_state childstate;
raw_spin_lock_irq(&child->lock);
raw_spin_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING);
child->parent = parent;
child->childmask = BIT(parent->num_children++);
raw_spin_unlock(&parent->lock);
raw_spin_unlock_irq(&child->lock);
trace_tmigr_connect_child_parent(child);
/*
* To prevent inconsistent states, active children need to be active in
* the new parent as well. Inactive children are already marked inactive
* in the parent group:
*
* * When new groups were created by tmigr_setup_groups() starting from
* the lowest level (and not higher then one level below the current
* top level), then they are not active. They will be set active when
* the new online CPU comes active.
*
* * But if a new group above the current top level is required, it is
* mandatory to propagate the active state of the already existing
* child to the new parent. So tmigr_connect_child_parent() is
* executed with the formerly top level group (child) and the newly
* created group (parent).
*/
childstate.state = atomic_read(&child->migr_state);
if (childstate.migrator != TMIGR_NONE) {
struct tmigr_walk data;
data.childmask = child->childmask;
/*
* There is only one new level per time. When connecting the
* child and the parent and set the child active when the parent
* is inactive, the parent needs to be the uppermost
* level. Otherwise there went something wrong!
*/
WARN_ON(!tmigr_active_up(parent, child, &data) && parent->parent);
}
}
static int tmigr_setup_groups(unsigned int cpu, unsigned int node)
{
struct tmigr_group *group, *child, **stack;
int top = 0, err = 0, i = 0;
struct list_head *lvllist;
stack = kcalloc(tmigr_hierarchy_levels, sizeof(*stack), GFP_KERNEL);
if (!stack)
return -ENOMEM;
do {
group = tmigr_get_group(cpu, node, i);
if (IS_ERR(group)) {
err = PTR_ERR(group);
break;
}
top = i;
stack[i++] = group;
/*
* When booting only less CPUs of a system than CPUs are
* available, not all calculated hierarchy levels are required.
*
* The loop is aborted as soon as the highest level, which might
* be different from tmigr_hierarchy_levels, contains only a
* single group.
*/
if (group->parent || i == tmigr_hierarchy_levels ||
(list_empty(&tmigr_level_list[i]) &&
list_is_singular(&tmigr_level_list[i - 1])))
break;
} while (i < tmigr_hierarchy_levels);
do {
group = stack[--i];
if (err < 0) {
list_del(&group->list);
kfree(group);
continue;
}
WARN_ON_ONCE(i != group->level);
/*
* Update tmc -> group / child -> group connection
*/
if (i == 0) {
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
raw_spin_lock_irq(&group->lock);
tmc->tmgroup = group;
tmc->childmask = BIT(group->num_children++);
raw_spin_unlock_irq(&group->lock);
trace_tmigr_connect_cpu_parent(tmc);
/* There are no children that need to be connected */
continue;
} else {
child = stack[i - 1];
tmigr_connect_child_parent(child, group);
}
/* check if uppermost level was newly created */
if (top != i)
continue;
WARN_ON_ONCE(top == 0);
lvllist = &tmigr_level_list[top];
if (group->num_children == 1 && list_is_singular(lvllist)) {
lvllist = &tmigr_level_list[top - 1];
list_for_each_entry(child, lvllist, list) {
if (child->parent)
continue;
tmigr_connect_child_parent(child, group);
}
}
} while (i > 0);
kfree(stack);
return err;
}
static int tmigr_add_cpu(unsigned int cpu)
{
int node = cpu_to_node(cpu);
int ret;
mutex_lock(&tmigr_mutex);
ret = tmigr_setup_groups(cpu, node);
mutex_unlock(&tmigr_mutex);
return ret;
}
static int tmigr_cpu_online(unsigned int cpu)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
int ret;
/* First online attempt? Initialize CPU data */
if (!tmc->tmgroup) {
raw_spin_lock_init(&tmc->lock);
ret = tmigr_add_cpu(cpu);
if (ret < 0)
return ret;
if (tmc->childmask == 0)
return -EINVAL;
timerqueue_init(&tmc->cpuevt.nextevt);
tmc->cpuevt.nextevt.expires = KTIME_MAX;
tmc->cpuevt.ignore = true;
tmc->cpuevt.cpu = cpu;
tmc->remote = false;
WRITE_ONCE(tmc->wakeup, KTIME_MAX);
}
raw_spin_lock_irq(&tmc->lock);
trace_tmigr_cpu_online(tmc);
tmc->idle = timer_base_is_idle();
if (!tmc->idle)
__tmigr_cpu_activate(tmc);
tmc->online = true;
raw_spin_unlock_irq(&tmc->lock);
return 0;
}
/*
* tmigr_trigger_active() - trigger a CPU to become active again
*
* This function is executed on a CPU which is part of cpu_online_mask, when the
* last active CPU in the hierarchy is offlining. With this, it is ensured that
* the other CPU is active and takes over the migrator duty.
*/
static long tmigr_trigger_active(void *unused)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
WARN_ON_ONCE(!tmc->online || tmc->idle);
return 0;
}
static int tmigr_cpu_offline(unsigned int cpu)
{
struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
int migrator;
u64 firstexp;
raw_spin_lock_irq(&tmc->lock);
tmc->online = false;
WRITE_ONCE(tmc->wakeup, KTIME_MAX);
/*
* CPU has to handle the local events on his own, when on the way to
* offline; Therefore nextevt value is set to KTIME_MAX
*/
firstexp = __tmigr_cpu_deactivate(tmc, KTIME_MAX);
trace_tmigr_cpu_offline(tmc);
raw_spin_unlock_irq(&tmc->lock);
if (firstexp != KTIME_MAX) {
migrator = cpumask_any_but(cpu_online_mask, cpu);
work_on_cpu(migrator, tmigr_trigger_active, NULL);
}
return 0;
}
static int __init tmigr_init(void)
{
unsigned int cpulvl, nodelvl, cpus_per_node, i;
unsigned int nnodes = num_possible_nodes();
unsigned int ncpus = num_possible_cpus();
int ret = -ENOMEM;
BUILD_BUG_ON_NOT_POWER_OF_2(TMIGR_CHILDREN_PER_GROUP);
/* Nothing to do if running on UP */
if (ncpus == 1)
return 0;
/*
* Calculate the required hierarchy levels. Unfortunately there is no
* reliable information available, unless all possible CPUs have been
* brought up and all NUMA nodes are populated.
*
* Estimate the number of levels with the number of possible nodes and
* the number of possible CPUs. Assume CPUs are spread evenly across
* nodes. We cannot rely on cpumask_of_node() because it only works for
* online CPUs.
*/
cpus_per_node = DIV_ROUND_UP(ncpus, nnodes);
/* Calc the hierarchy levels required to hold the CPUs of a node */
cpulvl = DIV_ROUND_UP(order_base_2(cpus_per_node),
ilog2(TMIGR_CHILDREN_PER_GROUP));
/* Calculate the extra levels to connect all nodes */
nodelvl = DIV_ROUND_UP(order_base_2(nnodes),
ilog2(TMIGR_CHILDREN_PER_GROUP));
tmigr_hierarchy_levels = cpulvl + nodelvl;
/*
* If a NUMA node spawns more than one CPU level group then the next
* level(s) of the hierarchy contains groups which handle all CPU groups
* of the same NUMA node. The level above goes across NUMA nodes. Store
* this information for the setup code to decide in which level node
* matching is no longer required.
*/
tmigr_crossnode_level = cpulvl;
tmigr_level_list = kcalloc(tmigr_hierarchy_levels, sizeof(struct list_head), GFP_KERNEL);
if (!tmigr_level_list)
goto err;
for (i = 0; i < tmigr_hierarchy_levels; i++)
INIT_LIST_HEAD(&tmigr_level_list[i]);
pr_info("Timer migration: %d hierarchy levels; %d children per group;"
" %d crossnode level\n",
tmigr_hierarchy_levels, TMIGR_CHILDREN_PER_GROUP,
tmigr_crossnode_level);
ret = cpuhp_setup_state(CPUHP_AP_TMIGR_ONLINE, "tmigr:online",
tmigr_cpu_online, tmigr_cpu_offline);
if (ret)
goto err;
return 0;
err:
pr_err("Timer migration setup failed\n");
return ret;
}
late_initcall(tmigr_init);
// SPDX-License-Identifier: GPL-2.0-only
/*
* mm/percpu.c - percpu memory allocator
*
* Copyright (C) 2009 SUSE Linux Products GmbH
* Copyright (C) 2009 Tejun Heo <tj@kernel.org>
*
* Copyright (C) 2017 Facebook Inc.
* Copyright (C) 2017 Dennis Zhou <dennis@kernel.org>
*
* The percpu allocator handles both static and dynamic areas. Percpu
* areas are allocated in chunks which are divided into units. There is
* a 1-to-1 mapping for units to possible cpus. These units are grouped
* based on NUMA properties of the machine.
*
* c0 c1 c2
* ------------------- ------------------- ------------
* | u0 | u1 | u2 | u3 | | u0 | u1 | u2 | u3 | | u0 | u1 | u
* ------------------- ...... ------------------- .... ------------
*
* Allocation is done by offsets into a unit's address space. Ie., an
* area of 512 bytes at 6k in c1 occupies 512 bytes at 6k in c1:u0,
* c1:u1, c1:u2, etc. On NUMA machines, the mapping may be non-linear
* and even sparse. Access is handled by configuring percpu base
* registers according to the cpu to unit mappings and offsetting the
* base address using pcpu_unit_size.
*
* There is special consideration for the first chunk which must handle
* the static percpu variables in the kernel image as allocation services
* are not online yet. In short, the first chunk is structured like so:
*
* <Static | [Reserved] | Dynamic>
*
* The static data is copied from the original section managed by the
* linker. The reserved section, if non-zero, primarily manages static
* percpu variables from kernel modules. Finally, the dynamic section
* takes care of normal allocations.
*
* The allocator organizes chunks into lists according to free size and
* memcg-awareness. To make a percpu allocation memcg-aware the __GFP_ACCOUNT
* flag should be passed. All memcg-aware allocations are sharing one set
* of chunks and all unaccounted allocations and allocations performed
* by processes belonging to the root memory cgroup are using the second set.
*
* The allocator tries to allocate from the fullest chunk first. Each chunk
* is managed by a bitmap with metadata blocks. The allocation map is updated
* on every allocation and free to reflect the current state while the boundary
* map is only updated on allocation. Each metadata block contains
* information to help mitigate the need to iterate over large portions
* of the bitmap. The reverse mapping from page to chunk is stored in
* the page's index. Lastly, units are lazily backed and grow in unison.
*
* There is a unique conversion that goes on here between bytes and bits.
* Each bit represents a fragment of size PCPU_MIN_ALLOC_SIZE. The chunk
* tracks the number of pages it is responsible for in nr_pages. Helper
* functions are used to convert from between the bytes, bits, and blocks.
* All hints are managed in bits unless explicitly stated.
*
* To use this allocator, arch code should do the following:
*
* - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
* regular address to percpu pointer and back if they need to be
* different from the default
*
* - use pcpu_setup_first_chunk() during percpu area initialization to
* setup the first chunk containing the kernel static percpu area
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/bitmap.h>
#include <linux/cpumask.h>
#include <linux/memblock.h>
#include <linux/err.h>
#include <linux/list.h>
#include <linux/log2.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/percpu.h>
#include <linux/pfn.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
#include <linux/kmemleak.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/memcontrol.h>
#include <asm/cacheflush.h>
#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/io.h>
#define CREATE_TRACE_POINTS
#include <trace/events/percpu.h>
#include "percpu-internal.h"
/*
* The slots are sorted by the size of the biggest continuous free area.
* 1-31 bytes share the same slot.
*/
#define PCPU_SLOT_BASE_SHIFT 5
/* chunks in slots below this are subject to being sidelined on failed alloc */
#define PCPU_SLOT_FAIL_THRESHOLD 3
#define PCPU_EMPTY_POP_PAGES_LOW 2
#define PCPU_EMPTY_POP_PAGES_HIGH 4
#ifdef CONFIG_SMP
/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
#ifndef __addr_to_pcpu_ptr
#define __addr_to_pcpu_ptr(addr) \
(void __percpu *)((unsigned long)(addr) - \
(unsigned long)pcpu_base_addr + \
(unsigned long)__per_cpu_start)
#endif
#ifndef __pcpu_ptr_to_addr
#define __pcpu_ptr_to_addr(ptr) \
(void __force *)((unsigned long)(ptr) + \
(unsigned long)pcpu_base_addr - \
(unsigned long)__per_cpu_start)
#endif
#else /* CONFIG_SMP */
/* on UP, it's always identity mapped */
#define __addr_to_pcpu_ptr(addr) (void __percpu *)(addr)
#define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr)
#endif /* CONFIG_SMP */
static int pcpu_unit_pages __ro_after_init;
static int pcpu_unit_size __ro_after_init;
static int pcpu_nr_units __ro_after_init;
static int pcpu_atom_size __ro_after_init;
int pcpu_nr_slots __ro_after_init;
static int pcpu_free_slot __ro_after_init;
int pcpu_sidelined_slot __ro_after_init;
int pcpu_to_depopulate_slot __ro_after_init;
static size_t pcpu_chunk_struct_size __ro_after_init;
/* cpus with the lowest and highest unit addresses */
static unsigned int pcpu_low_unit_cpu __ro_after_init;
static unsigned int pcpu_high_unit_cpu __ro_after_init;
/* the address of the first chunk which starts with the kernel static area */
void *pcpu_base_addr __ro_after_init;
static const int *pcpu_unit_map __ro_after_init; /* cpu -> unit */
const unsigned long *pcpu_unit_offsets __ro_after_init; /* cpu -> unit offset */
/* group information, used for vm allocation */
static int pcpu_nr_groups __ro_after_init;
static const unsigned long *pcpu_group_offsets __ro_after_init;
static const size_t *pcpu_group_sizes __ro_after_init;
/*
* The first chunk which always exists. Note that unlike other
* chunks, this one can be allocated and mapped in several different
* ways and thus often doesn't live in the vmalloc area.
*/
struct pcpu_chunk *pcpu_first_chunk __ro_after_init;
/*
* Optional reserved chunk. This chunk reserves part of the first
* chunk and serves it for reserved allocations. When the reserved
* region doesn't exist, the following variable is NULL.
*/
struct pcpu_chunk *pcpu_reserved_chunk __ro_after_init;
DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */
static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop, map ext */
struct list_head *pcpu_chunk_lists __ro_after_init; /* chunk list slots */
/*
* The number of empty populated pages, protected by pcpu_lock.
* The reserved chunk doesn't contribute to the count.
*/
int pcpu_nr_empty_pop_pages;
/*
* The number of populated pages in use by the allocator, protected by
* pcpu_lock. This number is kept per a unit per chunk (i.e. when a page gets
* allocated/deallocated, it is allocated/deallocated in all units of a chunk
* and increments/decrements this count by 1).
*/
static unsigned long pcpu_nr_populated;
/*
* Balance work is used to populate or destroy chunks asynchronously. We
* try to keep the number of populated free pages between
* PCPU_EMPTY_POP_PAGES_LOW and HIGH for atomic allocations and at most one
* empty chunk.
*/
static void pcpu_balance_workfn(struct work_struct *work);
static DECLARE_WORK(pcpu_balance_work, pcpu_balance_workfn);
static bool pcpu_async_enabled __read_mostly;
static bool pcpu_atomic_alloc_failed;
static void pcpu_schedule_balance_work(void)
{
if (pcpu_async_enabled) schedule_work(&pcpu_balance_work);
}
/**
* pcpu_addr_in_chunk - check if the address is served from this chunk
* @chunk: chunk of interest
* @addr: percpu address
*
* RETURNS:
* True if the address is served from this chunk.
*/
static bool pcpu_addr_in_chunk(struct pcpu_chunk *chunk, void *addr)
{
void *start_addr, *end_addr;
if (!chunk)
return false;
start_addr = chunk->base_addr + chunk->start_offset; end_addr = chunk->base_addr + chunk->nr_pages * PAGE_SIZE -
chunk->end_offset;
return addr >= start_addr && addr < end_addr;
}
static int __pcpu_size_to_slot(int size)
{
int highbit = fls(size); /* size is in bytes */ return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1);
}
static int pcpu_size_to_slot(int size)
{
if (size == pcpu_unit_size)
return pcpu_free_slot; return __pcpu_size_to_slot(size);
}
static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
{
const struct pcpu_block_md *chunk_md = &chunk->chunk_md;
if (chunk->free_bytes < PCPU_MIN_ALLOC_SIZE || chunk_md->contig_hint == 0) return 0; return pcpu_size_to_slot(chunk_md->contig_hint * PCPU_MIN_ALLOC_SIZE);
}
/* set the pointer to a chunk in a page struct */
static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
{
page->index = (unsigned long)pcpu;
}
/* obtain pointer to a chunk from a page struct */
static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
{
return (struct pcpu_chunk *)page->index;
}
static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx)
{
return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
}
static unsigned long pcpu_unit_page_offset(unsigned int cpu, int page_idx)
{
return pcpu_unit_offsets[cpu] + (page_idx << PAGE_SHIFT);
}
static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
unsigned int cpu, int page_idx)
{
return (unsigned long)chunk->base_addr +
pcpu_unit_page_offset(cpu, page_idx);
}
/*
* The following are helper functions to help access bitmaps and convert
* between bitmap offsets to address offsets.
*/
static unsigned long *pcpu_index_alloc_map(struct pcpu_chunk *chunk, int index)
{
return chunk->alloc_map +
(index * PCPU_BITMAP_BLOCK_BITS / BITS_PER_LONG);
}
static unsigned long pcpu_off_to_block_index(int off)
{
return off / PCPU_BITMAP_BLOCK_BITS;
}
static unsigned long pcpu_off_to_block_off(int off)
{
return off & (PCPU_BITMAP_BLOCK_BITS - 1);
}
static unsigned long pcpu_block_off_to_off(int index, int off)
{
return index * PCPU_BITMAP_BLOCK_BITS + off;
}
/**
* pcpu_check_block_hint - check against the contig hint
* @block: block of interest
* @bits: size of allocation
* @align: alignment of area (max PAGE_SIZE)
*
* Check to see if the allocation can fit in the block's contig hint.
* Note, a chunk uses the same hints as a block so this can also check against
* the chunk's contig hint.
*/
static bool pcpu_check_block_hint(struct pcpu_block_md *block, int bits,
size_t align)
{
int bit_off = ALIGN(block->contig_hint_start, align) -
block->contig_hint_start;
return bit_off + bits <= block->contig_hint;
}
/*
* pcpu_next_hint - determine which hint to use
* @block: block of interest
* @alloc_bits: size of allocation
*
* This determines if we should scan based on the scan_hint or first_free.
* In general, we want to scan from first_free to fulfill allocations by
* first fit. However, if we know a scan_hint at position scan_hint_start
* cannot fulfill an allocation, we can begin scanning from there knowing
* the contig_hint will be our fallback.
*/
static int pcpu_next_hint(struct pcpu_block_md *block, int alloc_bits)
{
/*
* The three conditions below determine if we can skip past the
* scan_hint. First, does the scan hint exist. Second, is the
* contig_hint after the scan_hint (possibly not true iff
* contig_hint == scan_hint). Third, is the allocation request
* larger than the scan_hint.
*/
if (block->scan_hint &&
block->contig_hint_start > block->scan_hint_start &&
alloc_bits > block->scan_hint)
return block->scan_hint_start + block->scan_hint;
return block->first_free;
}
/**
* pcpu_next_md_free_region - finds the next hint free area
* @chunk: chunk of interest
* @bit_off: chunk offset
* @bits: size of free area
*
* Helper function for pcpu_for_each_md_free_region. It checks
* block->contig_hint and performs aggregation across blocks to find the
* next hint. It modifies bit_off and bits in-place to be consumed in the
* loop.
*/
static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,
int *bits)
{
int i = pcpu_off_to_block_index(*bit_off);
int block_off = pcpu_off_to_block_off(*bit_off);
struct pcpu_block_md *block;
*bits = 0;
for (block = chunk->md_blocks + i; i < pcpu_chunk_nr_blocks(chunk);
block++, i++) {
/* handles contig area across blocks */
if (*bits) { *bits += block->left_free;
if (block->left_free == PCPU_BITMAP_BLOCK_BITS)
continue;
return;
}
/*
* This checks three things. First is there a contig_hint to
* check. Second, have we checked this hint before by
* comparing the block_off. Third, is this the same as the
* right contig hint. In the last case, it spills over into
* the next block and should be handled by the contig area
* across blocks code.
*/
*bits = block->contig_hint; if (*bits && block->contig_hint_start >= block_off && *bits + block->contig_hint_start < PCPU_BITMAP_BLOCK_BITS) { *bit_off = pcpu_block_off_to_off(i,
block->contig_hint_start);
return;
}
/* reset to satisfy the second predicate above */
block_off = 0;
*bits = block->right_free; *bit_off = (i + 1) * PCPU_BITMAP_BLOCK_BITS - block->right_free;
}
}
/**
* pcpu_next_fit_region - finds fit areas for a given allocation request
* @chunk: chunk of interest
* @alloc_bits: size of allocation
* @align: alignment of area (max PAGE_SIZE)
* @bit_off: chunk offset
* @bits: size of free area
*
* Finds the next free region that is viable for use with a given size and
* alignment. This only returns if there is a valid area to be used for this
* allocation. block->first_free is returned if the allocation request fits
* within the block to see if the request can be fulfilled prior to the contig
* hint.
*/
static void pcpu_next_fit_region(struct pcpu_chunk *chunk, int alloc_bits,
int align, int *bit_off, int *bits)
{
int i = pcpu_off_to_block_index(*bit_off);
int block_off = pcpu_off_to_block_off(*bit_off);
struct pcpu_block_md *block;
*bits = 0;
for (block = chunk->md_blocks + i; i < pcpu_chunk_nr_blocks(chunk);
block++, i++) {
/* handles contig area across blocks */
if (*bits) {
*bits += block->left_free;
if (*bits >= alloc_bits)
return;
if (block->left_free == PCPU_BITMAP_BLOCK_BITS)
continue;
}
/* check block->contig_hint */
*bits = ALIGN(block->contig_hint_start, align) -
block->contig_hint_start;
/*
* This uses the block offset to determine if this has been
* checked in the prior iteration.
*/
if (block->contig_hint &&
block->contig_hint_start >= block_off &&
block->contig_hint >= *bits + alloc_bits) {
int start = pcpu_next_hint(block, alloc_bits);
*bits += alloc_bits + block->contig_hint_start -
start;
*bit_off = pcpu_block_off_to_off(i, start);
return;
}
/* reset to satisfy the second predicate above */
block_off = 0;
*bit_off = ALIGN(PCPU_BITMAP_BLOCK_BITS - block->right_free,
align);
*bits = PCPU_BITMAP_BLOCK_BITS - *bit_off;
*bit_off = pcpu_block_off_to_off(i, *bit_off);
if (*bits >= alloc_bits)
return;
}
/* no valid offsets were found - fail condition */
*bit_off = pcpu_chunk_map_bits(chunk);
}
/*
* Metadata free area iterators. These perform aggregation of free areas
* based on the metadata blocks and return the offset @bit_off and size in
* bits of the free area @bits. pcpu_for_each_fit_region only returns when
* a fit is found for the allocation request.
*/
#define pcpu_for_each_md_free_region(chunk, bit_off, bits) \
for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits)); \
(bit_off) < pcpu_chunk_map_bits((chunk)); \
(bit_off) += (bits) + 1, \
pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
#define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) \
for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
&(bits)); \
(bit_off) < pcpu_chunk_map_bits((chunk)); \
(bit_off) += (bits), \
pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
&(bits)))
/**
* pcpu_mem_zalloc - allocate memory
* @size: bytes to allocate
* @gfp: allocation flags
*
* Allocate @size bytes. If @size is smaller than PAGE_SIZE,
* kzalloc() is used; otherwise, the equivalent of vzalloc() is used.
* This is to facilitate passing through whitelisted flags. The
* returned memory is always zeroed.
*
* RETURNS:
* Pointer to the allocated area on success, NULL on failure.
*/
static void *pcpu_mem_zalloc(size_t size, gfp_t gfp)
{
if (WARN_ON_ONCE(!slab_is_available()))
return NULL;
if (size <= PAGE_SIZE)
return kzalloc(size, gfp);
else
return __vmalloc(size, gfp | __GFP_ZERO);
}
/**
* pcpu_mem_free - free memory
* @ptr: memory to free
*
* Free @ptr. @ptr should have been allocated using pcpu_mem_zalloc().
*/
static void pcpu_mem_free(void *ptr)
{
kvfree(ptr);
}
static void __pcpu_chunk_move(struct pcpu_chunk *chunk, int slot,
bool move_front)
{
if (chunk != pcpu_reserved_chunk) {
if (move_front)
list_move(&chunk->list, &pcpu_chunk_lists[slot]);
else
list_move_tail(&chunk->list, &pcpu_chunk_lists[slot]);
}
}
static void pcpu_chunk_move(struct pcpu_chunk *chunk, int slot)
{
__pcpu_chunk_move(chunk, slot, true);
}
/**
* pcpu_chunk_relocate - put chunk in the appropriate chunk slot
* @chunk: chunk of interest
* @oslot: the previous slot it was on
*
* This function is called after an allocation or free changed @chunk.
* New slot according to the changed state is determined and @chunk is
* moved to the slot. Note that the reserved chunk is never put on
* chunk slots.
*
* CONTEXT:
* pcpu_lock.
*/
static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
{
int nslot = pcpu_chunk_slot(chunk);
/* leave isolated chunks in-place */
if (chunk->isolated)
return;
if (oslot != nslot) __pcpu_chunk_move(chunk, nslot, oslot < nslot);
}
static void pcpu_isolate_chunk(struct pcpu_chunk *chunk)
{
lockdep_assert_held(&pcpu_lock); if (!chunk->isolated) { chunk->isolated = true;
pcpu_nr_empty_pop_pages -= chunk->nr_empty_pop_pages;
}
list_move(&chunk->list, &pcpu_chunk_lists[pcpu_to_depopulate_slot]);
}
static void pcpu_reintegrate_chunk(struct pcpu_chunk *chunk)
{
lockdep_assert_held(&pcpu_lock);
if (chunk->isolated) {
chunk->isolated = false;
pcpu_nr_empty_pop_pages += chunk->nr_empty_pop_pages;
pcpu_chunk_relocate(chunk, -1);
}
}
/*
* pcpu_update_empty_pages - update empty page counters
* @chunk: chunk of interest
* @nr: nr of empty pages
*
* This is used to keep track of the empty pages now based on the premise
* a md_block covers a page. The hint update functions recognize if a block
* is made full or broken to calculate deltas for keeping track of free pages.
*/
static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
{
chunk->nr_empty_pop_pages += nr; if (chunk != pcpu_reserved_chunk && !chunk->isolated) pcpu_nr_empty_pop_pages += nr;
}
/*
* pcpu_region_overlap - determines if two regions overlap
* @a: start of first region, inclusive
* @b: end of first region, exclusive
* @x: start of second region, inclusive
* @y: end of second region, exclusive
*
* This is used to determine if the hint region [a, b) overlaps with the
* allocated region [x, y).
*/
static inline bool pcpu_region_overlap(int a, int b, int x, int y)
{
return (a < y) && (x < b);
}
/**
* pcpu_block_update - updates a block given a free area
* @block: block of interest
* @start: start offset in block
* @end: end offset in block
*
* Updates a block given a known free area. The region [start, end) is
* expected to be the entirety of the free area within a block. Chooses
* the best starting offset if the contig hints are equal.
*/
static void pcpu_block_update(struct pcpu_block_md *block, int start, int end)
{
int contig = end - start;
block->first_free = min(block->first_free, start);
if (start == 0)
block->left_free = contig; if (end == block->nr_bits) block->right_free = contig; if (contig > block->contig_hint) {
/* promote the old contig_hint to be the new scan_hint */
if (start > block->contig_hint_start) { if (block->contig_hint > block->scan_hint) { block->scan_hint_start =
block->contig_hint_start;
block->scan_hint = block->contig_hint;
} else if (start < block->scan_hint_start) {
/*
* The old contig_hint == scan_hint. But, the
* new contig is larger so hold the invariant
* scan_hint_start < contig_hint_start.
*/
block->scan_hint = 0;
}
} else {
block->scan_hint = 0;
}
block->contig_hint_start = start;
block->contig_hint = contig;
} else if (contig == block->contig_hint) { if (block->contig_hint_start &&
(!start ||
__ffs(start) > __ffs(block->contig_hint_start))) {
/* start has a better alignment so use it */
block->contig_hint_start = start;
if (start < block->scan_hint_start &&
block->contig_hint > block->scan_hint) block->scan_hint = 0; } else if (start > block->scan_hint_start || block->contig_hint > block->scan_hint) {
/*
* Knowing contig == contig_hint, update the scan_hint
* if it is farther than or larger than the current
* scan_hint.
*/
block->scan_hint_start = start;
block->scan_hint = contig;
}
} else {
/*
* The region is smaller than the contig_hint. So only update
* the scan_hint if it is larger than or equal and farther than
* the current scan_hint.
*/
if ((start < block->contig_hint_start && (contig > block->scan_hint ||
(contig == block->scan_hint &&
start > block->scan_hint_start)))) { block->scan_hint_start = start;
block->scan_hint = contig;
}
}
}
/*
* pcpu_block_update_scan - update a block given a free area from a scan
* @chunk: chunk of interest
* @bit_off: chunk offset
* @bits: size of free area
*
* Finding the final allocation spot first goes through pcpu_find_block_fit()
* to find a block that can hold the allocation and then pcpu_alloc_area()
* where a scan is used. When allocations require specific alignments,
* we can inadvertently create holes which will not be seen in the alloc
* or free paths.
*
* This takes a given free area hole and updates a block as it may change the
* scan_hint. We need to scan backwards to ensure we don't miss free bits
* from alignment.
*/
static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,
int bits)
{
int s_off = pcpu_off_to_block_off(bit_off);
int e_off = s_off + bits;
int s_index, l_bit;
struct pcpu_block_md *block;
if (e_off > PCPU_BITMAP_BLOCK_BITS)
return;
s_index = pcpu_off_to_block_index(bit_off);
block = chunk->md_blocks + s_index;
/* scan backwards in case of alignment skipping free bits */
l_bit = find_last_bit(pcpu_index_alloc_map(chunk, s_index), s_off);
s_off = (s_off == l_bit) ? 0 : l_bit + 1;
pcpu_block_update(block, s_off, e_off);
}
/**
* pcpu_chunk_refresh_hint - updates metadata about a chunk
* @chunk: chunk of interest
* @full_scan: if we should scan from the beginning
*
* Iterates over the metadata blocks to find the largest contig area.
* A full scan can be avoided on the allocation path as this is triggered
* if we broke the contig_hint. In doing so, the scan_hint will be before
* the contig_hint or after if the scan_hint == contig_hint. This cannot
* be prevented on freeing as we want to find the largest area possibly
* spanning blocks.
*/
static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan)
{
struct pcpu_block_md *chunk_md = &chunk->chunk_md;
int bit_off, bits;
/* promote scan_hint to contig_hint */
if (!full_scan && chunk_md->scan_hint) { bit_off = chunk_md->scan_hint_start + chunk_md->scan_hint;
chunk_md->contig_hint_start = chunk_md->scan_hint_start;
chunk_md->contig_hint = chunk_md->scan_hint;
chunk_md->scan_hint = 0;
} else {
bit_off = chunk_md->first_free; chunk_md->contig_hint = 0;
}
bits = 0;
pcpu_for_each_md_free_region(chunk, bit_off, bits) pcpu_block_update(chunk_md, bit_off, bit_off + bits);
}
/**
* pcpu_block_refresh_hint
* @chunk: chunk of interest
* @index: index of the metadata block
*
* Scans over the block beginning at first_free and updates the block
* metadata accordingly.
*/
static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index)
{
struct pcpu_block_md *block = chunk->md_blocks + index;
unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index);
unsigned int start, end; /* region start, region end */
/* promote scan_hint to contig_hint */
if (block->scan_hint) {
start = block->scan_hint_start + block->scan_hint;
block->contig_hint_start = block->scan_hint_start;
block->contig_hint = block->scan_hint;
block->scan_hint = 0;
} else {
start = block->first_free;
block->contig_hint = 0;
}
block->right_free = 0;
/* iterate over free areas and update the contig hints */
for_each_clear_bitrange_from(start, end, alloc_map, PCPU_BITMAP_BLOCK_BITS)
pcpu_block_update(block, start, end);
}
/**
* pcpu_block_update_hint_alloc - update hint on allocation path
* @chunk: chunk of interest
* @bit_off: chunk offset
* @bits: size of request
*
* Updates metadata for the allocation path. The metadata only has to be
* refreshed by a full scan iff the chunk's contig hint is broken. Block level
* scans are required if the block's contig hint is broken.
*/
static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
int bits)
{
struct pcpu_block_md *chunk_md = &chunk->chunk_md;
int nr_empty_pages = 0;
struct pcpu_block_md *s_block, *e_block, *block;
int s_index, e_index; /* block indexes of the freed allocation */
int s_off, e_off; /* block offsets of the freed allocation */
/*
* Calculate per block offsets.
* The calculation uses an inclusive range, but the resulting offsets
* are [start, end). e_index always points to the last block in the
* range.
*/
s_index = pcpu_off_to_block_index(bit_off);
e_index = pcpu_off_to_block_index(bit_off + bits - 1);
s_off = pcpu_off_to_block_off(bit_off);
e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;
s_block = chunk->md_blocks + s_index;
e_block = chunk->md_blocks + e_index;
/*
* Update s_block.
*/
if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
nr_empty_pages++;
/*
* block->first_free must be updated if the allocation takes its place.
* If the allocation breaks the contig_hint, a scan is required to
* restore this hint.
*/
if (s_off == s_block->first_free)
s_block->first_free = find_next_zero_bit(
pcpu_index_alloc_map(chunk, s_index),
PCPU_BITMAP_BLOCK_BITS,
s_off + bits);
if (pcpu_region_overlap(s_block->scan_hint_start,
s_block->scan_hint_start + s_block->scan_hint,
s_off,
s_off + bits))
s_block->scan_hint = 0;
if (pcpu_region_overlap(s_block->contig_hint_start,
s_block->contig_hint_start +
s_block->contig_hint,
s_off,
s_off + bits)) {
/* block contig hint is broken - scan to fix it */
if (!s_off)
s_block->left_free = 0;
pcpu_block_refresh_hint(chunk, s_index);
} else {
/* update left and right contig manually */
s_block->left_free = min(s_block->left_free, s_off);
if (s_index == e_index)
s_block->right_free = min_t(int, s_block->right_free,
PCPU_BITMAP_BLOCK_BITS - e_off);
else
s_block->right_free = 0;
}
/*
* Update e_block.
*/
if (s_index != e_index) {
if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
nr_empty_pages++;
/*
* When the allocation is across blocks, the end is along
* the left part of the e_block.
*/
e_block->first_free = find_next_zero_bit(
pcpu_index_alloc_map(chunk, e_index),
PCPU_BITMAP_BLOCK_BITS, e_off);
if (e_off == PCPU_BITMAP_BLOCK_BITS) {
/* reset the block */
e_block++;
} else {
if (e_off > e_block->scan_hint_start)
e_block->scan_hint = 0;
e_block->left_free = 0;
if (e_off > e_block->contig_hint_start) {
/* contig hint is broken - scan to fix it */
pcpu_block_refresh_hint(chunk, e_index);
} else {
e_block->right_free =
min_t(int, e_block->right_free,
PCPU_BITMAP_BLOCK_BITS - e_off);
}
}
/* update in-between md_blocks */
nr_empty_pages += (e_index - s_index - 1);
for (block = s_block + 1; block < e_block; block++) {
block->scan_hint = 0;
block->contig_hint = 0;
block->left_free = 0;
block->right_free = 0;
}
}
/*
* If the allocation is not atomic, some blocks may not be
* populated with pages, while we account it here. The number
* of pages will be added back with pcpu_chunk_populated()
* when populating pages.
*/
if (nr_empty_pages)
pcpu_update_empty_pages(chunk, -nr_empty_pages);
if (pcpu_region_overlap(chunk_md->scan_hint_start,
chunk_md->scan_hint_start +
chunk_md->scan_hint,
bit_off,
bit_off + bits))
chunk_md->scan_hint = 0;
/*
* The only time a full chunk scan is required is if the chunk
* contig hint is broken. Otherwise, it means a smaller space
* was used and therefore the chunk contig hint is still correct.
*/
if (pcpu_region_overlap(chunk_md->contig_hint_start,
chunk_md->contig_hint_start +
chunk_md->contig_hint,
bit_off,
bit_off + bits))
pcpu_chunk_refresh_hint(chunk, false);
}
/**
* pcpu_block_update_hint_free - updates the block hints on the free path
* @chunk: chunk of interest
* @bit_off: chunk offset
* @bits: size of request
*
* Updates metadata for the allocation path. This avoids a blind block
* refresh by making use of the block contig hints. If this fails, it scans
* forward and backward to determine the extent of the free area. This is
* capped at the boundary of blocks.
*
* A chunk update is triggered if a page becomes free, a block becomes free,
* or the free spans across blocks. This tradeoff is to minimize iterating
* over the block metadata to update chunk_md->contig_hint.
* chunk_md->contig_hint may be off by up to a page, but it will never be more
* than the available space. If the contig hint is contained in one block, it
* will be accurate.
*/
static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
int bits)
{
int nr_empty_pages = 0;
struct pcpu_block_md *s_block, *e_block, *block;
int s_index, e_index; /* block indexes of the freed allocation */
int s_off, e_off; /* block offsets of the freed allocation */
int start, end; /* start and end of the whole free area */
/*
* Calculate per block offsets.
* The calculation uses an inclusive range, but the resulting offsets
* are [start, end). e_index always points to the last block in the
* range.
*/
s_index = pcpu_off_to_block_index(bit_off);
e_index = pcpu_off_to_block_index(bit_off + bits - 1);
s_off = pcpu_off_to_block_off(bit_off);
e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;
s_block = chunk->md_blocks + s_index;
e_block = chunk->md_blocks + e_index;
/*
* Check if the freed area aligns with the block->contig_hint.
* If it does, then the scan to find the beginning/end of the
* larger free area can be avoided.
*
* start and end refer to beginning and end of the free area
* within each their respective blocks. This is not necessarily
* the entire free area as it may span blocks past the beginning
* or end of the block.
*/
start = s_off;
if (s_off == s_block->contig_hint + s_block->contig_hint_start) {
start = s_block->contig_hint_start;
} else {
/*
* Scan backwards to find the extent of the free area.
* find_last_bit returns the starting bit, so if the start bit
* is returned, that means there was no last bit and the
* remainder of the chunk is free.
*/
int l_bit = find_last_bit(pcpu_index_alloc_map(chunk, s_index),
start);
start = (start == l_bit) ? 0 : l_bit + 1;
}
end = e_off;
if (e_off == e_block->contig_hint_start) end = e_block->contig_hint_start + e_block->contig_hint;
else
end = find_next_bit(pcpu_index_alloc_map(chunk, e_index),
PCPU_BITMAP_BLOCK_BITS, end);
/* update s_block */
e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS; if (!start && e_off == PCPU_BITMAP_BLOCK_BITS)
nr_empty_pages++;
pcpu_block_update(s_block, start, e_off);
/* freeing in the same block */
if (s_index != e_index) {
/* update e_block */
if (end == PCPU_BITMAP_BLOCK_BITS) nr_empty_pages++; pcpu_block_update(e_block, 0, end);
/* reset md_blocks in the middle */
nr_empty_pages += (e_index - s_index - 1);
for (block = s_block + 1; block < e_block; block++) {
block->first_free = 0;
block->scan_hint = 0;
block->contig_hint_start = 0;
block->contig_hint = PCPU_BITMAP_BLOCK_BITS;
block->left_free = PCPU_BITMAP_BLOCK_BITS;
block->right_free = PCPU_BITMAP_BLOCK_BITS;
}
}
if (nr_empty_pages) pcpu_update_empty_pages(chunk, nr_empty_pages);
/*
* Refresh chunk metadata when the free makes a block free or spans
* across blocks. The contig_hint may be off by up to a page, but if
* the contig_hint is contained in a block, it will be accurate with
* the else condition below.
*/
if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index) pcpu_chunk_refresh_hint(chunk, true);
else
pcpu_block_update(&chunk->chunk_md,
pcpu_block_off_to_off(s_index, start),
end);
}
/**
* pcpu_is_populated - determines if the region is populated
* @chunk: chunk of interest
* @bit_off: chunk offset
* @bits: size of area
* @next_off: return value for the next offset to start searching
*
* For atomic allocations, check if the backing pages are populated.
*
* RETURNS:
* Bool if the backing pages are populated.
* next_index is to skip over unpopulated blocks in pcpu_find_block_fit.
*/
static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,
int *next_off)
{
unsigned int start, end;
start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);
end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);
start = find_next_zero_bit(chunk->populated, end, start);
if (start >= end)
return true;
end = find_next_bit(chunk->populated, end, start + 1);
*next_off = end * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
return false;
}
/**
* pcpu_find_block_fit - finds the block index to start searching
* @chunk: chunk of interest
* @alloc_bits: size of request in allocation units
* @align: alignment of area (max PAGE_SIZE bytes)
* @pop_only: use populated regions only
*
* Given a chunk and an allocation spec, find the offset to begin searching
* for a free region. This iterates over the bitmap metadata blocks to
* find an offset that will be guaranteed to fit the requirements. It is
* not quite first fit as if the allocation does not fit in the contig hint
* of a block or chunk, it is skipped. This errs on the side of caution
* to prevent excess iteration. Poor alignment can cause the allocator to
* skip over blocks and chunks that have valid free areas.
*
* RETURNS:
* The offset in the bitmap to begin searching.
* -1 if no offset is found.
*/
static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits,
size_t align, bool pop_only)
{
struct pcpu_block_md *chunk_md = &chunk->chunk_md;
int bit_off, bits, next_off;
/*
* This is an optimization to prevent scanning by assuming if the
* allocation cannot fit in the global hint, there is memory pressure
* and creating a new chunk would happen soon.
*/
if (!pcpu_check_block_hint(chunk_md, alloc_bits, align))
return -1;
bit_off = pcpu_next_hint(chunk_md, alloc_bits);
bits = 0;
pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {
if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,
&next_off))
break;
bit_off = next_off;
bits = 0;
}
if (bit_off == pcpu_chunk_map_bits(chunk))
return -1;
return bit_off;
}
/*
* pcpu_find_zero_area - modified from bitmap_find_next_zero_area_off()
* @map: the address to base the search on
* @size: the bitmap size in bits
* @start: the bitnumber to start searching at
* @nr: the number of zeroed bits we're looking for
* @align_mask: alignment mask for zero area
* @largest_off: offset of the largest area skipped
* @largest_bits: size of the largest area skipped
*
* The @align_mask should be one less than a power of 2.
*
* This is a modified version of bitmap_find_next_zero_area_off() to remember
* the largest area that was skipped. This is imperfect, but in general is
* good enough. The largest remembered region is the largest failed region
* seen. This does not include anything we possibly skipped due to alignment.
* pcpu_block_update_scan() does scan backwards to try and recover what was
* lost to alignment. While this can cause scanning to miss earlier possible
* free areas, smaller allocations will eventually fill those holes.
*/
static unsigned long pcpu_find_zero_area(unsigned long *map,
unsigned long size,
unsigned long start,
unsigned long nr,
unsigned long align_mask,
unsigned long *largest_off,
unsigned long *largest_bits)
{
unsigned long index, end, i, area_off, area_bits;
again:
index = find_next_zero_bit(map, size, start);
/* Align allocation */
index = __ALIGN_MASK(index, align_mask);
area_off = index;
end = index + nr;
if (end > size)
return end;
i = find_next_bit(map, end, index);
if (i < end) {
area_bits = i - area_off;
/* remember largest unused area with best alignment */
if (area_bits > *largest_bits ||
(area_bits == *largest_bits && *largest_off &&
(!area_off || __ffs(area_off) > __ffs(*largest_off)))) {
*largest_off = area_off;
*largest_bits = area_bits;
}
start = i + 1;
goto again;
}
return index;
}
/**
* pcpu_alloc_area - allocates an area from a pcpu_chunk
* @chunk: chunk of interest
* @alloc_bits: size of request in allocation units
* @align: alignment of area (max PAGE_SIZE)
* @start: bit_off to start searching
*
* This function takes in a @start offset to begin searching to fit an
* allocation of @alloc_bits with alignment @align. It needs to scan
* the allocation map because if it fits within the block's contig hint,
* @start will be block->first_free. This is an attempt to fill the
* allocation prior to breaking the contig hint. The allocation and
* boundary maps are updated accordingly if it confirms a valid
* free area.
*
* RETURNS:
* Allocated addr offset in @chunk on success.
* -1 if no matching area is found.
*/
static int pcpu_alloc_area(struct pcpu_chunk *chunk, int alloc_bits,
size_t align, int start)
{
struct pcpu_block_md *chunk_md = &chunk->chunk_md;
size_t align_mask = (align) ? (align - 1) : 0;
unsigned long area_off = 0, area_bits = 0;
int bit_off, end, oslot;
lockdep_assert_held(&pcpu_lock);
oslot = pcpu_chunk_slot(chunk);
/*
* Search to find a fit.
*/
end = min_t(int, start + alloc_bits + PCPU_BITMAP_BLOCK_BITS,
pcpu_chunk_map_bits(chunk));
bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits,
align_mask, &area_off, &area_bits);
if (bit_off >= end)
return -1;
if (area_bits)
pcpu_block_update_scan(chunk, area_off, area_bits);
/* update alloc map */
bitmap_set(chunk->alloc_map, bit_off, alloc_bits);
/* update boundary map */
set_bit(bit_off, chunk->bound_map);
bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);
set_bit(bit_off + alloc_bits, chunk->bound_map);
chunk->free_bytes -= alloc_bits * PCPU_MIN_ALLOC_SIZE;
/* update first free bit */
if (bit_off == chunk_md->first_free)
chunk_md->first_free = find_next_zero_bit(
chunk->alloc_map,
pcpu_chunk_map_bits(chunk),
bit_off + alloc_bits);
pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);
pcpu_chunk_relocate(chunk, oslot);
return bit_off * PCPU_MIN_ALLOC_SIZE;
}
/**
* pcpu_free_area - frees the corresponding offset
* @chunk: chunk of interest
* @off: addr offset into chunk
*
* This function determines the size of an allocation to free using
* the boundary bitmap and clears the allocation map.
*
* RETURNS:
* Number of freed bytes.
*/
static int pcpu_free_area(struct pcpu_chunk *chunk, int off)
{
struct pcpu_block_md *chunk_md = &chunk->chunk_md;
int bit_off, bits, end, oslot, freed;
lockdep_assert_held(&pcpu_lock);
pcpu_stats_area_dealloc(chunk);
oslot = pcpu_chunk_slot(chunk);
bit_off = off / PCPU_MIN_ALLOC_SIZE;
/* find end index */
end = find_next_bit(chunk->bound_map, pcpu_chunk_map_bits(chunk),
bit_off + 1);
bits = end - bit_off;
bitmap_clear(chunk->alloc_map, bit_off, bits);
freed = bits * PCPU_MIN_ALLOC_SIZE;
/* update metadata */
chunk->free_bytes += freed;
/* update first free bit */
chunk_md->first_free = min(chunk_md->first_free, bit_off);
pcpu_block_update_hint_free(chunk, bit_off, bits); pcpu_chunk_relocate(chunk, oslot);
return freed;
}
static void pcpu_init_md_block(struct pcpu_block_md *block, int nr_bits)
{
block->scan_hint = 0;
block->contig_hint = nr_bits;
block->left_free = nr_bits;
block->right_free = nr_bits;
block->first_free = 0;
block->nr_bits = nr_bits;
}
static void pcpu_init_md_blocks(struct pcpu_chunk *chunk)
{
struct pcpu_block_md *md_block;
/* init the chunk's block */
pcpu_init_md_block(&chunk->chunk_md, pcpu_chunk_map_bits(chunk));
for (md_block = chunk->md_blocks;
md_block != chunk->md_blocks + pcpu_chunk_nr_blocks(chunk);
md_block++)
pcpu_init_md_block(md_block, PCPU_BITMAP_BLOCK_BITS);
}
/**
* pcpu_alloc_first_chunk - creates chunks that serve the first chunk
* @tmp_addr: the start of the region served
* @map_size: size of the region served
*
* This is responsible for creating the chunks that serve the first chunk. The
* base_addr is page aligned down of @tmp_addr while the region end is page
* aligned up. Offsets are kept track of to determine the region served. All
* this is done to appease the bitmap allocator in avoiding partial blocks.
*
* RETURNS:
* Chunk serving the region at @tmp_addr of @map_size.
*/
static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
int map_size)
{
struct pcpu_chunk *chunk;
unsigned long aligned_addr;
int start_offset, offset_bits, region_size, region_bits;
size_t alloc_size;
/* region calculations */
aligned_addr = tmp_addr & PAGE_MASK;
start_offset = tmp_addr - aligned_addr;
region_size = ALIGN(start_offset + map_size, PAGE_SIZE);
/* allocate chunk */
alloc_size = struct_size(chunk, populated,
BITS_TO_LONGS(region_size >> PAGE_SHIFT));
chunk = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!chunk)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
INIT_LIST_HEAD(&chunk->list);
chunk->base_addr = (void *)aligned_addr;
chunk->start_offset = start_offset;
chunk->end_offset = region_size - chunk->start_offset - map_size;
chunk->nr_pages = region_size >> PAGE_SHIFT;
region_bits = pcpu_chunk_map_bits(chunk);
alloc_size = BITS_TO_LONGS(region_bits) * sizeof(chunk->alloc_map[0]);
chunk->alloc_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!chunk->alloc_map)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
alloc_size =
BITS_TO_LONGS(region_bits + 1) * sizeof(chunk->bound_map[0]);
chunk->bound_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!chunk->bound_map)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
alloc_size = pcpu_chunk_nr_blocks(chunk) * sizeof(chunk->md_blocks[0]);
chunk->md_blocks = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!chunk->md_blocks)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
#ifdef CONFIG_MEMCG_KMEM
/* first chunk is free to use */
chunk->obj_cgroups = NULL;
#endif
pcpu_init_md_blocks(chunk);
/* manage populated page bitmap */
chunk->immutable = true;
bitmap_fill(chunk->populated, chunk->nr_pages);
chunk->nr_populated = chunk->nr_pages;
chunk->nr_empty_pop_pages = chunk->nr_pages;
chunk->free_bytes = map_size;
if (chunk->start_offset) {
/* hide the beginning of the bitmap */
offset_bits = chunk->start_offset / PCPU_MIN_ALLOC_SIZE;
bitmap_set(chunk->alloc_map, 0, offset_bits);
set_bit(0, chunk->bound_map);
set_bit(offset_bits, chunk->bound_map);
chunk->chunk_md.first_free = offset_bits;
pcpu_block_update_hint_alloc(chunk, 0, offset_bits);
}
if (chunk->end_offset) {
/* hide the end of the bitmap */
offset_bits = chunk->end_offset / PCPU_MIN_ALLOC_SIZE;
bitmap_set(chunk->alloc_map,
pcpu_chunk_map_bits(chunk) - offset_bits,
offset_bits);
set_bit((start_offset + map_size) / PCPU_MIN_ALLOC_SIZE,
chunk->bound_map);
set_bit(region_bits, chunk->bound_map);
pcpu_block_update_hint_alloc(chunk, pcpu_chunk_map_bits(chunk)
- offset_bits, offset_bits);
}
return chunk;
}
static struct pcpu_chunk *pcpu_alloc_chunk(gfp_t gfp)
{
struct pcpu_chunk *chunk;
int region_bits;
chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size, gfp);
if (!chunk)
return NULL;
INIT_LIST_HEAD(&chunk->list);
chunk->nr_pages = pcpu_unit_pages;
region_bits = pcpu_chunk_map_bits(chunk);
chunk->alloc_map = pcpu_mem_zalloc(BITS_TO_LONGS(region_bits) *
sizeof(chunk->alloc_map[0]), gfp);
if (!chunk->alloc_map)
goto alloc_map_fail;
chunk->bound_map = pcpu_mem_zalloc(BITS_TO_LONGS(region_bits + 1) *
sizeof(chunk->bound_map[0]), gfp);
if (!chunk->bound_map)
goto bound_map_fail;
chunk->md_blocks = pcpu_mem_zalloc(pcpu_chunk_nr_blocks(chunk) *
sizeof(chunk->md_blocks[0]), gfp);
if (!chunk->md_blocks)
goto md_blocks_fail;
#ifdef CONFIG_MEMCG_KMEM
if (!mem_cgroup_kmem_disabled()) {
chunk->obj_cgroups =
pcpu_mem_zalloc(pcpu_chunk_map_bits(chunk) *
sizeof(struct obj_cgroup *), gfp);
if (!chunk->obj_cgroups)
goto objcg_fail;
}
#endif
pcpu_init_md_blocks(chunk);
/* init metadata */
chunk->free_bytes = chunk->nr_pages * PAGE_SIZE;
return chunk;
#ifdef CONFIG_MEMCG_KMEM
objcg_fail:
pcpu_mem_free(chunk->md_blocks);
#endif
md_blocks_fail:
pcpu_mem_free(chunk->bound_map);
bound_map_fail:
pcpu_mem_free(chunk->alloc_map);
alloc_map_fail:
pcpu_mem_free(chunk);
return NULL;
}
static void pcpu_free_chunk(struct pcpu_chunk *chunk)
{
if (!chunk)
return;
#ifdef CONFIG_MEMCG_KMEM
pcpu_mem_free(chunk->obj_cgroups);
#endif
pcpu_mem_free(chunk->md_blocks);
pcpu_mem_free(chunk->bound_map);
pcpu_mem_free(chunk->alloc_map);
pcpu_mem_free(chunk);
}
/**
* pcpu_chunk_populated - post-population bookkeeping
* @chunk: pcpu_chunk which got populated
* @page_start: the start page
* @page_end: the end page
*
* Pages in [@page_start,@page_end) have been populated to @chunk. Update
* the bookkeeping information accordingly. Must be called after each
* successful population.
*/
static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,
int page_end)
{
int nr = page_end - page_start;
lockdep_assert_held(&pcpu_lock);
bitmap_set(chunk->populated, page_start, nr);
chunk->nr_populated += nr;
pcpu_nr_populated += nr;
pcpu_update_empty_pages(chunk, nr);
}
/**
* pcpu_chunk_depopulated - post-depopulation bookkeeping
* @chunk: pcpu_chunk which got depopulated
* @page_start: the start page
* @page_end: the end page
*
* Pages in [@page_start,@page_end) have been depopulated from @chunk.
* Update the bookkeeping information accordingly. Must be called after
* each successful depopulation.
*/
static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,
int page_start, int page_end)
{
int nr = page_end - page_start;
lockdep_assert_held(&pcpu_lock);
bitmap_clear(chunk->populated, page_start, nr);
chunk->nr_populated -= nr;
pcpu_nr_populated -= nr;
pcpu_update_empty_pages(chunk, -nr);
}
/*
* Chunk management implementation.
*
* To allow different implementations, chunk alloc/free and
* [de]population are implemented in a separate file which is pulled
* into this file and compiled together. The following functions
* should be implemented.
*
* pcpu_populate_chunk - populate the specified range of a chunk
* pcpu_depopulate_chunk - depopulate the specified range of a chunk
* pcpu_post_unmap_tlb_flush - flush tlb for the specified range of a chunk
* pcpu_create_chunk - create a new chunk
* pcpu_destroy_chunk - destroy a chunk, always preceded by full depop
* pcpu_addr_to_page - translate address to physical address
* pcpu_verify_alloc_info - check alloc_info is acceptable during init
*/
static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
int page_start, int page_end, gfp_t gfp);
static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
int page_start, int page_end);
static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
int page_start, int page_end);
static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp);
static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
static struct page *pcpu_addr_to_page(void *addr);
static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
#ifdef CONFIG_NEED_PER_CPU_KM
#include "percpu-km.c"
#else
#include "percpu-vm.c"
#endif
/**
* pcpu_chunk_addr_search - determine chunk containing specified address
* @addr: address for which the chunk needs to be determined.
*
* This is an internal function that handles all but static allocations.
* Static percpu address values should never be passed into the allocator.
*
* RETURNS:
* The address of the found chunk.
*/
static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
{
/* is it in the dynamic region (first chunk)? */
if (pcpu_addr_in_chunk(pcpu_first_chunk, addr))
return pcpu_first_chunk;
/* is it in the reserved region? */
if (pcpu_addr_in_chunk(pcpu_reserved_chunk, addr))
return pcpu_reserved_chunk;
/*
* The address is relative to unit0 which might be unused and
* thus unmapped. Offset the address to the unit space of the
* current processor before looking it up in the vmalloc
* space. Note that any possible cpu id can be used here, so
* there's no need to worry about preemption or cpu hotplug.
*/
addr += pcpu_unit_offsets[raw_smp_processor_id()]; return pcpu_get_page_chunk(pcpu_addr_to_page(addr));
}
#ifdef CONFIG_MEMCG_KMEM
static bool pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp,
struct obj_cgroup **objcgp)
{
struct obj_cgroup *objcg;
if (!memcg_kmem_online() || !(gfp & __GFP_ACCOUNT))
return true;
objcg = current_obj_cgroup();
if (!objcg)
return true;
if (obj_cgroup_charge(objcg, gfp, pcpu_obj_full_size(size)))
return false;
*objcgp = objcg;
return true;
}
static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg,
struct pcpu_chunk *chunk, int off,
size_t size)
{
if (!objcg)
return;
if (likely(chunk && chunk->obj_cgroups)) {
obj_cgroup_get(objcg);
chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = objcg;
rcu_read_lock();
mod_memcg_state(obj_cgroup_memcg(objcg), MEMCG_PERCPU_B,
pcpu_obj_full_size(size));
rcu_read_unlock();
} else {
obj_cgroup_uncharge(objcg, pcpu_obj_full_size(size));
}
}
static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
{
struct obj_cgroup *objcg;
if (unlikely(!chunk->obj_cgroups))
return;
objcg = chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT];
if (!objcg)
return;
chunk->obj_cgroups[off >> PCPU_MIN_ALLOC_SHIFT] = NULL;
obj_cgroup_uncharge(objcg, pcpu_obj_full_size(size)); rcu_read_lock(); mod_memcg_state(obj_cgroup_memcg(objcg), MEMCG_PERCPU_B, -pcpu_obj_full_size(size)); rcu_read_unlock(); obj_cgroup_put(objcg);
}
#else /* CONFIG_MEMCG_KMEM */
static bool
pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp, struct obj_cgroup **objcgp)
{
return true;
}
static void pcpu_memcg_post_alloc_hook(struct obj_cgroup *objcg,
struct pcpu_chunk *chunk, int off,
size_t size)
{
}
static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
{
}
#endif /* CONFIG_MEMCG_KMEM */
/**
* pcpu_alloc - the percpu allocator
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
* @reserved: allocate from the reserved chunk if available
* @gfp: allocation flags
*
* Allocate percpu area of @size bytes aligned at @align. If @gfp doesn't
* contain %GFP_KERNEL, the allocation is atomic. If @gfp has __GFP_NOWARN
* then no warning will be triggered on invalid or failed allocation
* requests.
*
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
gfp_t gfp)
{
gfp_t pcpu_gfp;
bool is_atomic;
bool do_warn;
struct obj_cgroup *objcg = NULL;
static int warn_limit = 10;
struct pcpu_chunk *chunk, *next;
const char *err;
int slot, off, cpu, ret;
unsigned long flags;
void __percpu *ptr;
size_t bits, bit_align;
gfp = current_gfp_context(gfp);
/* whitelisted flags that can be passed to the backing allocators */
pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
is_atomic = (gfp & GFP_KERNEL) != GFP_KERNEL;
do_warn = !(gfp & __GFP_NOWARN);
/*
* There is now a minimum allocation size of PCPU_MIN_ALLOC_SIZE,
* therefore alignment must be a minimum of that many bytes.
* An allocation may have internal fragmentation from rounding up
* of up to PCPU_MIN_ALLOC_SIZE - 1 bytes.
*/
if (unlikely(align < PCPU_MIN_ALLOC_SIZE))
align = PCPU_MIN_ALLOC_SIZE;
size = ALIGN(size, PCPU_MIN_ALLOC_SIZE);
bits = size >> PCPU_MIN_ALLOC_SHIFT;
bit_align = align >> PCPU_MIN_ALLOC_SHIFT;
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE ||
!is_power_of_2(align))) {
WARN(do_warn, "illegal size (%zu) or align (%zu) for percpu allocation\n",
size, align);
return NULL;
}
if (unlikely(!pcpu_memcg_pre_alloc_hook(size, gfp, &objcg)))
return NULL;
if (!is_atomic) {
/*
* pcpu_balance_workfn() allocates memory under this mutex,
* and it may wait for memory reclaim. Allow current task
* to become OOM victim, in case of memory pressure.
*/
if (gfp & __GFP_NOFAIL) {
mutex_lock(&pcpu_alloc_mutex);
} else if (mutex_lock_killable(&pcpu_alloc_mutex)) {
pcpu_memcg_post_alloc_hook(objcg, NULL, 0, size);
return NULL;
}
}
spin_lock_irqsave(&pcpu_lock, flags);
/* serve reserved allocations from the reserved chunk if available */
if (reserved && pcpu_reserved_chunk) {
chunk = pcpu_reserved_chunk;
off = pcpu_find_block_fit(chunk, bits, bit_align, is_atomic);
if (off < 0) {
err = "alloc from reserved chunk failed";
goto fail_unlock;
}
off = pcpu_alloc_area(chunk, bits, bit_align, off);
if (off >= 0)
goto area_found;
err = "alloc from reserved chunk failed";
goto fail_unlock;
}
restart:
/* search through normal chunks */
for (slot = pcpu_size_to_slot(size); slot <= pcpu_free_slot; slot++) {
list_for_each_entry_safe(chunk, next, &pcpu_chunk_lists[slot],
list) {
off = pcpu_find_block_fit(chunk, bits, bit_align,
is_atomic);
if (off < 0) {
if (slot < PCPU_SLOT_FAIL_THRESHOLD)
pcpu_chunk_move(chunk, 0);
continue;
}
off = pcpu_alloc_area(chunk, bits, bit_align, off);
if (off >= 0) {
pcpu_reintegrate_chunk(chunk);
goto area_found;
}
}
}
spin_unlock_irqrestore(&pcpu_lock, flags);
if (is_atomic) {
err = "atomic alloc failed, no space left";
goto fail;
}
/* No space left. Create a new chunk. */
if (list_empty(&pcpu_chunk_lists[pcpu_free_slot])) {
chunk = pcpu_create_chunk(pcpu_gfp);
if (!chunk) {
err = "failed to allocate new chunk";
goto fail;
}
spin_lock_irqsave(&pcpu_lock, flags);
pcpu_chunk_relocate(chunk, -1);
} else {
spin_lock_irqsave(&pcpu_lock, flags);
}
goto restart;
area_found:
pcpu_stats_area_alloc(chunk, size);
spin_unlock_irqrestore(&pcpu_lock, flags);
/* populate if not all pages are already there */
if (!is_atomic) {
unsigned int page_end, rs, re;
rs = PFN_DOWN(off);
page_end = PFN_UP(off + size);
for_each_clear_bitrange_from(rs, re, chunk->populated, page_end) {
WARN_ON(chunk->immutable);
ret = pcpu_populate_chunk(chunk, rs, re, pcpu_gfp);
spin_lock_irqsave(&pcpu_lock, flags);
if (ret) {
pcpu_free_area(chunk, off);
err = "failed to populate";
goto fail_unlock;
}
pcpu_chunk_populated(chunk, rs, re);
spin_unlock_irqrestore(&pcpu_lock, flags);
}
mutex_unlock(&pcpu_alloc_mutex);
}
if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW)
pcpu_schedule_balance_work();
/* clear the areas and return address relative to base address */
for_each_possible_cpu(cpu)
memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
ptr = __addr_to_pcpu_ptr(chunk->base_addr + off);
kmemleak_alloc_percpu(ptr, size, gfp);
trace_percpu_alloc_percpu(_RET_IP_, reserved, is_atomic, size, align,
chunk->base_addr, off, ptr,
pcpu_obj_full_size(size), gfp);
pcpu_memcg_post_alloc_hook(objcg, chunk, off, size);
return ptr;
fail_unlock:
spin_unlock_irqrestore(&pcpu_lock, flags);
fail:
trace_percpu_alloc_percpu_fail(reserved, is_atomic, size, align);
if (do_warn && warn_limit) {
pr_warn("allocation failed, size=%zu align=%zu atomic=%d, %s\n",
size, align, is_atomic, err);
if (!is_atomic)
dump_stack();
if (!--warn_limit)
pr_info("limit reached, disable warning\n");
}
if (is_atomic) {
/* see the flag handling in pcpu_balance_workfn() */
pcpu_atomic_alloc_failed = true;
pcpu_schedule_balance_work();
} else {
mutex_unlock(&pcpu_alloc_mutex);
}
pcpu_memcg_post_alloc_hook(objcg, NULL, 0, size);
return NULL;
}
/**
* __alloc_percpu_gfp - allocate dynamic percpu area
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
* @gfp: allocation flags
*
* Allocate zero-filled percpu area of @size bytes aligned at @align. If
* @gfp doesn't contain %GFP_KERNEL, the allocation doesn't block and can
* be called from any context but is a lot more likely to fail. If @gfp
* has __GFP_NOWARN then no warning will be triggered on invalid or failed
* allocation requests.
*
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
void __percpu *__alloc_percpu_gfp(size_t size, size_t align, gfp_t gfp)
{
return pcpu_alloc(size, align, false, gfp);
}
EXPORT_SYMBOL_GPL(__alloc_percpu_gfp);
/**
* __alloc_percpu - allocate dynamic percpu area
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
*
* Equivalent to __alloc_percpu_gfp(size, align, %GFP_KERNEL).
*/
void __percpu *__alloc_percpu(size_t size, size_t align)
{
return pcpu_alloc(size, align, false, GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(__alloc_percpu);
/**
* __alloc_reserved_percpu - allocate reserved percpu area
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
*
* Allocate zero-filled percpu area of @size bytes aligned at @align
* from reserved percpu area if arch has set it up; otherwise,
* allocation is served from the same dynamic area. Might sleep.
* Might trigger writeouts.
*
* CONTEXT:
* Does GFP_KERNEL allocation.
*
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
{
return pcpu_alloc(size, align, true, GFP_KERNEL);
}
/**
* pcpu_balance_free - manage the amount of free chunks
* @empty_only: free chunks only if there are no populated pages
*
* If empty_only is %false, reclaim all fully free chunks regardless of the
* number of populated pages. Otherwise, only reclaim chunks that have no
* populated pages.
*
* CONTEXT:
* pcpu_lock (can be dropped temporarily)
*/
static void pcpu_balance_free(bool empty_only)
{
LIST_HEAD(to_free);
struct list_head *free_head = &pcpu_chunk_lists[pcpu_free_slot];
struct pcpu_chunk *chunk, *next;
lockdep_assert_held(&pcpu_lock);
/*
* There's no reason to keep around multiple unused chunks and VM
* areas can be scarce. Destroy all free chunks except for one.
*/
list_for_each_entry_safe(chunk, next, free_head, list) {
WARN_ON(chunk->immutable);
/* spare the first one */
if (chunk == list_first_entry(free_head, struct pcpu_chunk, list))
continue;
if (!empty_only || chunk->nr_empty_pop_pages == 0)
list_move(&chunk->list, &to_free);
}
if (list_empty(&to_free))
return;
spin_unlock_irq(&pcpu_lock);
list_for_each_entry_safe(chunk, next, &to_free, list) {
unsigned int rs, re;
for_each_set_bitrange(rs, re, chunk->populated, chunk->nr_pages) {
pcpu_depopulate_chunk(chunk, rs, re);
spin_lock_irq(&pcpu_lock);
pcpu_chunk_depopulated(chunk, rs, re);
spin_unlock_irq(&pcpu_lock);
}
pcpu_destroy_chunk(chunk);
cond_resched();
}
spin_lock_irq(&pcpu_lock);
}
/**
* pcpu_balance_populated - manage the amount of populated pages
*
* Maintain a certain amount of populated pages to satisfy atomic allocations.
* It is possible that this is called when physical memory is scarce causing
* OOM killer to be triggered. We should avoid doing so until an actual
* allocation causes the failure as it is possible that requests can be
* serviced from already backed regions.
*
* CONTEXT:
* pcpu_lock (can be dropped temporarily)
*/
static void pcpu_balance_populated(void)
{
/* gfp flags passed to underlying allocators */
const gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
struct pcpu_chunk *chunk;
int slot, nr_to_pop, ret;
lockdep_assert_held(&pcpu_lock);
/*
* Ensure there are certain number of free populated pages for
* atomic allocs. Fill up from the most packed so that atomic
* allocs don't increase fragmentation. If atomic allocation
* failed previously, always populate the maximum amount. This
* should prevent atomic allocs larger than PAGE_SIZE from keeping
* failing indefinitely; however, large atomic allocs are not
* something we support properly and can be highly unreliable and
* inefficient.
*/
retry_pop:
if (pcpu_atomic_alloc_failed) {
nr_to_pop = PCPU_EMPTY_POP_PAGES_HIGH;
/* best effort anyway, don't worry about synchronization */
pcpu_atomic_alloc_failed = false;
} else {
nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -
pcpu_nr_empty_pop_pages,
0, PCPU_EMPTY_POP_PAGES_HIGH);
}
for (slot = pcpu_size_to_slot(PAGE_SIZE); slot <= pcpu_free_slot; slot++) {
unsigned int nr_unpop = 0, rs, re;
if (!nr_to_pop)
break;
list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) {
nr_unpop = chunk->nr_pages - chunk->nr_populated;
if (nr_unpop)
break;
}
if (!nr_unpop)
continue;
/* @chunk can't go away while pcpu_alloc_mutex is held */
for_each_clear_bitrange(rs, re, chunk->populated, chunk->nr_pages) {
int nr = min_t(int, re - rs, nr_to_pop);
spin_unlock_irq(&pcpu_lock);
ret = pcpu_populate_chunk(chunk, rs, rs + nr, gfp);
cond_resched();
spin_lock_irq(&pcpu_lock);
if (!ret) {
nr_to_pop -= nr;
pcpu_chunk_populated(chunk, rs, rs + nr);
} else {
nr_to_pop = 0;
}
if (!nr_to_pop)
break;
}
}
if (nr_to_pop) {
/* ran out of chunks to populate, create a new one and retry */
spin_unlock_irq(&pcpu_lock);
chunk = pcpu_create_chunk(gfp);
cond_resched();
spin_lock_irq(&pcpu_lock);
if (chunk) {
pcpu_chunk_relocate(chunk, -1);
goto retry_pop;
}
}
}
/**
* pcpu_reclaim_populated - scan over to_depopulate chunks and free empty pages
*
* Scan over chunks in the depopulate list and try to release unused populated
* pages back to the system. Depopulated chunks are sidelined to prevent
* repopulating these pages unless required. Fully free chunks are reintegrated
* and freed accordingly (1 is kept around). If we drop below the empty
* populated pages threshold, reintegrate the chunk if it has empty free pages.
* Each chunk is scanned in the reverse order to keep populated pages close to
* the beginning of the chunk.
*
* CONTEXT:
* pcpu_lock (can be dropped temporarily)
*
*/
static void pcpu_reclaim_populated(void)
{
struct pcpu_chunk *chunk;
struct pcpu_block_md *block;
int freed_page_start, freed_page_end;
int i, end;
bool reintegrate;
lockdep_assert_held(&pcpu_lock);
/*
* Once a chunk is isolated to the to_depopulate list, the chunk is no
* longer discoverable to allocations whom may populate pages. The only
* other accessor is the free path which only returns area back to the
* allocator not touching the populated bitmap.
*/
while ((chunk = list_first_entry_or_null(
&pcpu_chunk_lists[pcpu_to_depopulate_slot],
struct pcpu_chunk, list))) {
WARN_ON(chunk->immutable);
/*
* Scan chunk's pages in the reverse order to keep populated
* pages close to the beginning of the chunk.
*/
freed_page_start = chunk->nr_pages;
freed_page_end = 0;
reintegrate = false;
for (i = chunk->nr_pages - 1, end = -1; i >= 0; i--) {
/* no more work to do */
if (chunk->nr_empty_pop_pages == 0)
break;
/* reintegrate chunk to prevent atomic alloc failures */
if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_HIGH) {
reintegrate = true;
break;
}
/*
* If the page is empty and populated, start or
* extend the (i, end) range. If i == 0, decrease
* i and perform the depopulation to cover the last
* (first) page in the chunk.
*/
block = chunk->md_blocks + i;
if (block->contig_hint == PCPU_BITMAP_BLOCK_BITS &&
test_bit(i, chunk->populated)) {
if (end == -1)
end = i;
if (i > 0)
continue;
i--;
}
/* depopulate if there is an active range */
if (end == -1)
continue;
spin_unlock_irq(&pcpu_lock);
pcpu_depopulate_chunk(chunk, i + 1, end + 1);
cond_resched();
spin_lock_irq(&pcpu_lock);
pcpu_chunk_depopulated(chunk, i + 1, end + 1);
freed_page_start = min(freed_page_start, i + 1);
freed_page_end = max(freed_page_end, end + 1);
/* reset the range and continue */
end = -1;
}
/* batch tlb flush per chunk to amortize cost */
if (freed_page_start < freed_page_end) {
spin_unlock_irq(&pcpu_lock);
pcpu_post_unmap_tlb_flush(chunk,
freed_page_start,
freed_page_end);
cond_resched();
spin_lock_irq(&pcpu_lock);
}
if (reintegrate || chunk->free_bytes == pcpu_unit_size)
pcpu_reintegrate_chunk(chunk);
else
list_move_tail(&chunk->list,
&pcpu_chunk_lists[pcpu_sidelined_slot]);
}
}
/**
* pcpu_balance_workfn - manage the amount of free chunks and populated pages
* @work: unused
*
* For each chunk type, manage the number of fully free chunks and the number of
* populated pages. An important thing to consider is when pages are freed and
* how they contribute to the global counts.
*/
static void pcpu_balance_workfn(struct work_struct *work)
{
/*
* pcpu_balance_free() is called twice because the first time we may
* trim pages in the active pcpu_nr_empty_pop_pages which may cause us
* to grow other chunks. This then gives pcpu_reclaim_populated() time
* to move fully free chunks to the active list to be freed if
* appropriate.
*/
mutex_lock(&pcpu_alloc_mutex);
spin_lock_irq(&pcpu_lock);
pcpu_balance_free(false);
pcpu_reclaim_populated();
pcpu_balance_populated();
pcpu_balance_free(true);
spin_unlock_irq(&pcpu_lock);
mutex_unlock(&pcpu_alloc_mutex);
}
/**
* pcpu_alloc_size - the size of the dynamic percpu area
* @ptr: pointer to the dynamic percpu area
*
* Returns the size of the @ptr allocation. This is undefined for statically
* defined percpu variables as there is no corresponding chunk->bound_map.
*
* RETURNS:
* The size of the dynamic percpu area.
*
* CONTEXT:
* Can be called from atomic context.
*/
size_t pcpu_alloc_size(void __percpu *ptr)
{
struct pcpu_chunk *chunk;
unsigned long bit_off, end;
void *addr;
if (!ptr)
return 0;
addr = __pcpu_ptr_to_addr(ptr);
/* No pcpu_lock here: ptr has not been freed, so chunk is still alive */
chunk = pcpu_chunk_addr_search(addr);
bit_off = (addr - chunk->base_addr) / PCPU_MIN_ALLOC_SIZE;
end = find_next_bit(chunk->bound_map, pcpu_chunk_map_bits(chunk),
bit_off + 1);
return (end - bit_off) * PCPU_MIN_ALLOC_SIZE;
}
/**
* free_percpu - free percpu area
* @ptr: pointer to area to free
*
* Free percpu area @ptr.
*
* CONTEXT:
* Can be called from atomic context.
*/
void free_percpu(void __percpu *ptr)
{
void *addr;
struct pcpu_chunk *chunk;
unsigned long flags;
int size, off;
bool need_balance = false;
if (!ptr)
return;
kmemleak_free_percpu(ptr);
addr = __pcpu_ptr_to_addr(ptr);
chunk = pcpu_chunk_addr_search(addr);
off = addr - chunk->base_addr;
spin_lock_irqsave(&pcpu_lock, flags);
size = pcpu_free_area(chunk, off);
pcpu_memcg_free_hook(chunk, off, size);
/*
* If there are more than one fully free chunks, wake up grim reaper.
* If the chunk is isolated, it may be in the process of being
* reclaimed. Let reclaim manage cleaning up of that chunk.
*/
if (!chunk->isolated && chunk->free_bytes == pcpu_unit_size) {
struct pcpu_chunk *pos;
list_for_each_entry(pos, &pcpu_chunk_lists[pcpu_free_slot], list) if (pos != chunk) {
need_balance = true;
break;
}
} else if (pcpu_should_reclaim_chunk(chunk)) { pcpu_isolate_chunk(chunk);
need_balance = true;
}
trace_percpu_free_percpu(chunk->base_addr, off, ptr); spin_unlock_irqrestore(&pcpu_lock, flags);
if (need_balance)
pcpu_schedule_balance_work();
}
EXPORT_SYMBOL_GPL(free_percpu);
bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr)
{
#ifdef CONFIG_SMP
const size_t static_size = __per_cpu_end - __per_cpu_start;
void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
unsigned int cpu;
for_each_possible_cpu(cpu) { void *start = per_cpu_ptr(base, cpu);
void *va = (void *)addr;
if (va >= start && va < start + static_size) { if (can_addr) { *can_addr = (unsigned long) (va - start);
*can_addr += (unsigned long)
per_cpu_ptr(base, get_boot_cpu_id());
}
return true;
}
}
#endif
/* on UP, can't distinguish from other static vars, always false */
return false;
}
/**
* is_kernel_percpu_address - test whether address is from static percpu area
* @addr: address to test
*
* Test whether @addr belongs to in-kernel static percpu area. Module
* static percpu areas are not considered. For those, use
* is_module_percpu_address().
*
* RETURNS:
* %true if @addr is from in-kernel static percpu area, %false otherwise.
*/
bool is_kernel_percpu_address(unsigned long addr)
{
return __is_kernel_percpu_address(addr, NULL);
}
/**
* per_cpu_ptr_to_phys - convert translated percpu address to physical address
* @addr: the address to be converted to physical address
*
* Given @addr which is dereferenceable address obtained via one of
* percpu access macros, this function translates it into its physical
* address. The caller is responsible for ensuring @addr stays valid
* until this function finishes.
*
* percpu allocator has special setup for the first chunk, which currently
* supports either embedding in linear address space or vmalloc mapping,
* and, from the second one, the backing allocator (currently either vm or
* km) provides translation.
*
* The addr can be translated simply without checking if it falls into the
* first chunk. But the current code reflects better how percpu allocator
* actually works, and the verification can discover both bugs in percpu
* allocator itself and per_cpu_ptr_to_phys() callers. So we keep current
* code.
*
* RETURNS:
* The physical address for @addr.
*/
phys_addr_t per_cpu_ptr_to_phys(void *addr)
{
void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
bool in_first_chunk = false;
unsigned long first_low, first_high;
unsigned int cpu;
/*
* The following test on unit_low/high isn't strictly
* necessary but will speed up lookups of addresses which
* aren't in the first chunk.
*
* The address check is against full chunk sizes. pcpu_base_addr
* points to the beginning of the first chunk including the
* static region. Assumes good intent as the first chunk may
* not be full (ie. < pcpu_unit_pages in size).
*/
first_low = (unsigned long)pcpu_base_addr +
pcpu_unit_page_offset(pcpu_low_unit_cpu, 0);
first_high = (unsigned long)pcpu_base_addr +
pcpu_unit_page_offset(pcpu_high_unit_cpu, pcpu_unit_pages);
if ((unsigned long)addr >= first_low &&
(unsigned long)addr < first_high) {
for_each_possible_cpu(cpu) {
void *start = per_cpu_ptr(base, cpu);
if (addr >= start && addr < start + pcpu_unit_size) {
in_first_chunk = true;
break;
}
}
}
if (in_first_chunk) {
if (!is_vmalloc_addr(addr))
return __pa(addr);
else
return page_to_phys(vmalloc_to_page(addr)) +
offset_in_page(addr);
} else
return page_to_phys(pcpu_addr_to_page(addr)) +
offset_in_page(addr);
}
/**
* pcpu_alloc_alloc_info - allocate percpu allocation info
* @nr_groups: the number of groups
* @nr_units: the number of units
*
* Allocate ai which is large enough for @nr_groups groups containing
* @nr_units units. The returned ai's groups[0].cpu_map points to the
* cpu_map array which is long enough for @nr_units and filled with
* NR_CPUS. It's the caller's responsibility to initialize cpu_map
* pointer of other groups.
*
* RETURNS:
* Pointer to the allocated pcpu_alloc_info on success, NULL on
* failure.
*/
struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
int nr_units)
{
struct pcpu_alloc_info *ai;
size_t base_size, ai_size;
void *ptr;
int unit;
base_size = ALIGN(struct_size(ai, groups, nr_groups),
__alignof__(ai->groups[0].cpu_map[0]));
ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]);
ptr = memblock_alloc(PFN_ALIGN(ai_size), PAGE_SIZE);
if (!ptr)
return NULL;
ai = ptr;
ptr += base_size;
ai->groups[0].cpu_map = ptr;
for (unit = 0; unit < nr_units; unit++)
ai->groups[0].cpu_map[unit] = NR_CPUS;
ai->nr_groups = nr_groups;
ai->__ai_size = PFN_ALIGN(ai_size);
return ai;
}
/**
* pcpu_free_alloc_info - free percpu allocation info
* @ai: pcpu_alloc_info to free
*
* Free @ai which was allocated by pcpu_alloc_alloc_info().
*/
void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai)
{
memblock_free(ai, ai->__ai_size);
}
/**
* pcpu_dump_alloc_info - print out information about pcpu_alloc_info
* @lvl: loglevel
* @ai: allocation info to dump
*
* Print out information about @ai using loglevel @lvl.
*/
static void pcpu_dump_alloc_info(const char *lvl,
const struct pcpu_alloc_info *ai)
{
int group_width = 1, cpu_width = 1, width;
char empty_str[] = "--------";
int alloc = 0, alloc_end = 0;
int group, v;
int upa, apl; /* units per alloc, allocs per line */
v = ai->nr_groups;
while (v /= 10)
group_width++;
v = num_possible_cpus();
while (v /= 10)
cpu_width++;
empty_str[min_t(int, cpu_width, sizeof(empty_str) - 1)] = '\0';
upa = ai->alloc_size / ai->unit_size;
width = upa * (cpu_width + 1) + group_width + 3;
apl = rounddown_pow_of_two(max(60 / width, 1));
printk("%spcpu-alloc: s%zu r%zu d%zu u%zu alloc=%zu*%zu",
lvl, ai->static_size, ai->reserved_size, ai->dyn_size,
ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size);
for (group = 0; group < ai->nr_groups; group++) {
const struct pcpu_group_info *gi = &ai->groups[group];
int unit = 0, unit_end = 0;
BUG_ON(gi->nr_units % upa);
for (alloc_end += gi->nr_units / upa;
alloc < alloc_end; alloc++) {
if (!(alloc % apl)) {
pr_cont("\n");
printk("%spcpu-alloc: ", lvl);
}
pr_cont("[%0*d] ", group_width, group);
for (unit_end += upa; unit < unit_end; unit++)
if (gi->cpu_map[unit] != NR_CPUS)
pr_cont("%0*d ",
cpu_width, gi->cpu_map[unit]);
else
pr_cont("%s ", empty_str);
}
}
pr_cont("\n");
}
/**
* pcpu_setup_first_chunk - initialize the first percpu chunk
* @ai: pcpu_alloc_info describing how to percpu area is shaped
* @base_addr: mapped address
*
* Initialize the first percpu chunk which contains the kernel static
* percpu area. This function is to be called from arch percpu area
* setup path.
*
* @ai contains all information necessary to initialize the first
* chunk and prime the dynamic percpu allocator.
*
* @ai->static_size is the size of static percpu area.
*
* @ai->reserved_size, if non-zero, specifies the amount of bytes to
* reserve after the static area in the first chunk. This reserves
* the first chunk such that it's available only through reserved
* percpu allocation. This is primarily used to serve module percpu
* static areas on architectures where the addressing model has
* limited offset range for symbol relocations to guarantee module
* percpu symbols fall inside the relocatable range.
*
* @ai->dyn_size determines the number of bytes available for dynamic
* allocation in the first chunk. The area between @ai->static_size +
* @ai->reserved_size + @ai->dyn_size and @ai->unit_size is unused.
*
* @ai->unit_size specifies unit size and must be aligned to PAGE_SIZE
* and equal to or larger than @ai->static_size + @ai->reserved_size +
* @ai->dyn_size.
*
* @ai->atom_size is the allocation atom size and used as alignment
* for vm areas.
*
* @ai->alloc_size is the allocation size and always multiple of
* @ai->atom_size. This is larger than @ai->atom_size if
* @ai->unit_size is larger than @ai->atom_size.
*
* @ai->nr_groups and @ai->groups describe virtual memory layout of
* percpu areas. Units which should be colocated are put into the
* same group. Dynamic VM areas will be allocated according to these
* groupings. If @ai->nr_groups is zero, a single group containing
* all units is assumed.
*
* The caller should have mapped the first chunk at @base_addr and
* copied static data to each unit.
*
* The first chunk will always contain a static and a dynamic region.
* However, the static region is not managed by any chunk. If the first
* chunk also contains a reserved region, it is served by two chunks -
* one for the reserved region and one for the dynamic region. They
* share the same vm, but use offset regions in the area allocation map.
* The chunk serving the dynamic region is circulated in the chunk slots
* and available for dynamic allocation like any other chunk.
*/
void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
void *base_addr)
{
size_t size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
size_t static_size, dyn_size;
unsigned long *group_offsets;
size_t *group_sizes;
unsigned long *unit_off;
unsigned int cpu;
int *unit_map;
int group, unit, i;
unsigned long tmp_addr;
size_t alloc_size;
#define PCPU_SETUP_BUG_ON(cond) do { \
if (unlikely(cond)) { \
pr_emerg("failed to initialize, %s\n", #cond); \
pr_emerg("cpu_possible_mask=%*pb\n", \
cpumask_pr_args(cpu_possible_mask)); \
pcpu_dump_alloc_info(KERN_EMERG, ai); \
BUG(); \
} \
} while (0)
/* sanity checks */
PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);
#ifdef CONFIG_SMP
PCPU_SETUP_BUG_ON(!ai->static_size);
PCPU_SETUP_BUG_ON(offset_in_page(__per_cpu_start));
#endif
PCPU_SETUP_BUG_ON(!base_addr);
PCPU_SETUP_BUG_ON(offset_in_page(base_addr));
PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
PCPU_SETUP_BUG_ON(offset_in_page(ai->unit_size));
PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);
PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->unit_size, PCPU_BITMAP_BLOCK_SIZE));
PCPU_SETUP_BUG_ON(ai->dyn_size < PERCPU_DYNAMIC_EARLY_SIZE);
PCPU_SETUP_BUG_ON(!IS_ALIGNED(ai->reserved_size, PCPU_MIN_ALLOC_SIZE));
PCPU_SETUP_BUG_ON(!(IS_ALIGNED(PCPU_BITMAP_BLOCK_SIZE, PAGE_SIZE) ||
IS_ALIGNED(PAGE_SIZE, PCPU_BITMAP_BLOCK_SIZE)));
PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0);
/* process group information and build config tables accordingly */
alloc_size = ai->nr_groups * sizeof(group_offsets[0]);
group_offsets = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!group_offsets)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
alloc_size = ai->nr_groups * sizeof(group_sizes[0]);
group_sizes = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!group_sizes)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
alloc_size = nr_cpu_ids * sizeof(unit_map[0]);
unit_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!unit_map)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
alloc_size = nr_cpu_ids * sizeof(unit_off[0]);
unit_off = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
if (!unit_off)
panic("%s: Failed to allocate %zu bytes\n", __func__,
alloc_size);
for (cpu = 0; cpu < nr_cpu_ids; cpu++)
unit_map[cpu] = UINT_MAX;
pcpu_low_unit_cpu = NR_CPUS;
pcpu_high_unit_cpu = NR_CPUS;
for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) {
const struct pcpu_group_info *gi = &ai->groups[group];
group_offsets[group] = gi->base_offset;
group_sizes[group] = gi->nr_units * ai->unit_size;
for (i = 0; i < gi->nr_units; i++) {
cpu = gi->cpu_map[i];
if (cpu == NR_CPUS)
continue;
PCPU_SETUP_BUG_ON(cpu >= nr_cpu_ids);
PCPU_SETUP_BUG_ON(!cpu_possible(cpu));
PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX);
unit_map[cpu] = unit + i;
unit_off[cpu] = gi->base_offset + i * ai->unit_size;
/* determine low/high unit_cpu */
if (pcpu_low_unit_cpu == NR_CPUS ||
unit_off[cpu] < unit_off[pcpu_low_unit_cpu])
pcpu_low_unit_cpu = cpu;
if (pcpu_high_unit_cpu == NR_CPUS ||
unit_off[cpu] > unit_off[pcpu_high_unit_cpu])
pcpu_high_unit_cpu = cpu;
}
}
pcpu_nr_units = unit;
for_each_possible_cpu(cpu)
PCPU_SETUP_BUG_ON(unit_map[cpu] == UINT_MAX);
/* we're done parsing the input, undefine BUG macro and dump config */
#undef PCPU_SETUP_BUG_ON
pcpu_dump_alloc_info(KERN_DEBUG, ai);
pcpu_nr_groups = ai->nr_groups;
pcpu_group_offsets = group_offsets;
pcpu_group_sizes = group_sizes;
pcpu_unit_map = unit_map;
pcpu_unit_offsets = unit_off;
/* determine basic parameters */
pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT;
pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
pcpu_atom_size = ai->atom_size;
pcpu_chunk_struct_size = struct_size((struct pcpu_chunk *)0, populated,
BITS_TO_LONGS(pcpu_unit_pages));
pcpu_stats_save_ai(ai);
/*
* Allocate chunk slots. The slots after the active slots are:
* sidelined_slot - isolated, depopulated chunks
* free_slot - fully free chunks
* to_depopulate_slot - isolated, chunks to depopulate
*/
pcpu_sidelined_slot = __pcpu_size_to_slot(pcpu_unit_size) + 1;
pcpu_free_slot = pcpu_sidelined_slot + 1;
pcpu_to_depopulate_slot = pcpu_free_slot + 1;
pcpu_nr_slots = pcpu_to_depopulate_slot + 1;
pcpu_chunk_lists = memblock_alloc(pcpu_nr_slots *
sizeof(pcpu_chunk_lists[0]),
SMP_CACHE_BYTES);
if (!pcpu_chunk_lists)
panic("%s: Failed to allocate %zu bytes\n", __func__,
pcpu_nr_slots * sizeof(pcpu_chunk_lists[0]));
for (i = 0; i < pcpu_nr_slots; i++)
INIT_LIST_HEAD(&pcpu_chunk_lists[i]);
/*
* The end of the static region needs to be aligned with the
* minimum allocation size as this offsets the reserved and
* dynamic region. The first chunk ends page aligned by
* expanding the dynamic region, therefore the dynamic region
* can be shrunk to compensate while still staying above the
* configured sizes.
*/
static_size = ALIGN(ai->static_size, PCPU_MIN_ALLOC_SIZE);
dyn_size = ai->dyn_size - (static_size - ai->static_size);
/*
* Initialize first chunk:
* This chunk is broken up into 3 parts:
* < static | [reserved] | dynamic >
* - static - there is no backing chunk because these allocations can
* never be freed.
* - reserved (pcpu_reserved_chunk) - exists primarily to serve
* allocations from module load.
* - dynamic (pcpu_first_chunk) - serves the dynamic part of the first
* chunk.
*/
tmp_addr = (unsigned long)base_addr + static_size;
if (ai->reserved_size)
pcpu_reserved_chunk = pcpu_alloc_first_chunk(tmp_addr,
ai->reserved_size);
tmp_addr = (unsigned long)base_addr + static_size + ai->reserved_size;
pcpu_first_chunk = pcpu_alloc_first_chunk(tmp_addr, dyn_size);
pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;
pcpu_chunk_relocate(pcpu_first_chunk, -1);
/* include all regions of the first chunk */
pcpu_nr_populated += PFN_DOWN(size_sum);
pcpu_stats_chunk_alloc();
trace_percpu_create_chunk(base_addr);
/* we're done */
pcpu_base_addr = base_addr;
}
#ifdef CONFIG_SMP
const char * const pcpu_fc_names[PCPU_FC_NR] __initconst = {
[PCPU_FC_AUTO] = "auto",
[PCPU_FC_EMBED] = "embed",
[PCPU_FC_PAGE] = "page",
};
enum pcpu_fc pcpu_chosen_fc __initdata = PCPU_FC_AUTO;
static int __init percpu_alloc_setup(char *str)
{
if (!str)
return -EINVAL;
if (0)
/* nada */;
#ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
else if (!strcmp(str, "embed"))
pcpu_chosen_fc = PCPU_FC_EMBED;
#endif
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
else if (!strcmp(str, "page"))
pcpu_chosen_fc = PCPU_FC_PAGE;
#endif
else
pr_warn("unknown allocator %s specified\n", str);
return 0;
}
early_param("percpu_alloc", percpu_alloc_setup);
/*
* pcpu_embed_first_chunk() is used by the generic percpu setup.
* Build it if needed by the arch config or the generic setup is going
* to be used.
*/
#if defined(CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK) || \
!defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
#define BUILD_EMBED_FIRST_CHUNK
#endif
/* build pcpu_page_first_chunk() iff needed by the arch config */
#if defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK)
#define BUILD_PAGE_FIRST_CHUNK
#endif
/* pcpu_build_alloc_info() is used by both embed and page first chunk */
#if defined(BUILD_EMBED_FIRST_CHUNK) || defined(BUILD_PAGE_FIRST_CHUNK)
/**
* pcpu_build_alloc_info - build alloc_info considering distances between CPUs
* @reserved_size: the size of reserved percpu area in bytes
* @dyn_size: minimum free size for dynamic allocation in bytes
* @atom_size: allocation atom size
* @cpu_distance_fn: callback to determine distance between cpus, optional
*
* This function determines grouping of units, their mappings to cpus
* and other parameters considering needed percpu size, allocation
* atom size and distances between CPUs.
*
* Groups are always multiples of atom size and CPUs which are of
* LOCAL_DISTANCE both ways are grouped together and share space for
* units in the same group. The returned configuration is guaranteed
* to have CPUs on different nodes on different groups and >=75% usage
* of allocated virtual address space.
*
* RETURNS:
* On success, pointer to the new allocation_info is returned. On
* failure, ERR_PTR value is returned.
*/
static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info(
size_t reserved_size, size_t dyn_size,
size_t atom_size,
pcpu_fc_cpu_distance_fn_t cpu_distance_fn)
{
static int group_map[NR_CPUS] __initdata;
static int group_cnt[NR_CPUS] __initdata;
static struct cpumask mask __initdata;
const size_t static_size = __per_cpu_end - __per_cpu_start;
int nr_groups = 1, nr_units = 0;
size_t size_sum, min_unit_size, alloc_size;
int upa, max_upa, best_upa; /* units_per_alloc */
int last_allocs, group, unit;
unsigned int cpu, tcpu;
struct pcpu_alloc_info *ai;
unsigned int *cpu_map;
/* this function may be called multiple times */
memset(group_map, 0, sizeof(group_map));
memset(group_cnt, 0, sizeof(group_cnt));
cpumask_clear(&mask);
/* calculate size_sum and ensure dyn_size is enough for early alloc */
size_sum = PFN_ALIGN(static_size + reserved_size +
max_t(size_t, dyn_size, PERCPU_DYNAMIC_EARLY_SIZE));
dyn_size = size_sum - static_size - reserved_size;
/*
* Determine min_unit_size, alloc_size and max_upa such that
* alloc_size is multiple of atom_size and is the smallest
* which can accommodate 4k aligned segments which are equal to
* or larger than min_unit_size.
*/
min_unit_size = max_t(size_t, size_sum, PCPU_MIN_UNIT_SIZE);
/* determine the maximum # of units that can fit in an allocation */
alloc_size = roundup(min_unit_size, atom_size);
upa = alloc_size / min_unit_size;
while (alloc_size % upa || (offset_in_page(alloc_size / upa)))
upa--;
max_upa = upa;
cpumask_copy(&mask, cpu_possible_mask);
/* group cpus according to their proximity */
for (group = 0; !cpumask_empty(&mask); group++) {
/* pop the group's first cpu */
cpu = cpumask_first(&mask);
group_map[cpu] = group;
group_cnt[group]++;
cpumask_clear_cpu(cpu, &mask);
for_each_cpu(tcpu, &mask) {
if (!cpu_distance_fn ||
(cpu_distance_fn(cpu, tcpu) == LOCAL_DISTANCE &&
cpu_distance_fn(tcpu, cpu) == LOCAL_DISTANCE)) {
group_map[tcpu] = group;
group_cnt[group]++;
cpumask_clear_cpu(tcpu, &mask);
}
}
}
nr_groups = group;
/*
* Wasted space is caused by a ratio imbalance of upa to group_cnt.
* Expand the unit_size until we use >= 75% of the units allocated.
* Related to atom_size, which could be much larger than the unit_size.
*/
last_allocs = INT_MAX;
best_upa = 0;
for (upa = max_upa; upa; upa--) {
int allocs = 0, wasted = 0;
if (alloc_size % upa || (offset_in_page(alloc_size / upa)))
continue;
for (group = 0; group < nr_groups; group++) {
int this_allocs = DIV_ROUND_UP(group_cnt[group], upa);
allocs += this_allocs;
wasted += this_allocs * upa - group_cnt[group];
}
/*
* Don't accept if wastage is over 1/3. The
* greater-than comparison ensures upa==1 always
* passes the following check.
*/
if (wasted > num_possible_cpus() / 3)
continue;
/* and then don't consume more memory */
if (allocs > last_allocs)
break;
last_allocs = allocs;
best_upa = upa;
}
BUG_ON(!best_upa);
upa = best_upa;
/* allocate and fill alloc_info */
for (group = 0; group < nr_groups; group++)
nr_units += roundup(group_cnt[group], upa);
ai = pcpu_alloc_alloc_info(nr_groups, nr_units);
if (!ai)
return ERR_PTR(-ENOMEM);
cpu_map = ai->groups[0].cpu_map;
for (group = 0; group < nr_groups; group++) {
ai->groups[group].cpu_map = cpu_map;
cpu_map += roundup(group_cnt[group], upa);
}
ai->static_size = static_size;
ai->reserved_size = reserved_size;
ai->dyn_size = dyn_size;
ai->unit_size = alloc_size / upa;
ai->atom_size = atom_size;
ai->alloc_size = alloc_size;
for (group = 0, unit = 0; group < nr_groups; group++) {
struct pcpu_group_info *gi = &ai->groups[group];
/*
* Initialize base_offset as if all groups are located
* back-to-back. The caller should update this to
* reflect actual allocation.
*/
gi->base_offset = unit * ai->unit_size;
for_each_possible_cpu(cpu)
if (group_map[cpu] == group)
gi->cpu_map[gi->nr_units++] = cpu;
gi->nr_units = roundup(gi->nr_units, upa);
unit += gi->nr_units;
}
BUG_ON(unit != nr_units);
return ai;
}
static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align,
pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)
{
const unsigned long goal = __pa(MAX_DMA_ADDRESS);
#ifdef CONFIG_NUMA
int node = NUMA_NO_NODE;
void *ptr;
if (cpu_to_nd_fn)
node = cpu_to_nd_fn(cpu);
if (node == NUMA_NO_NODE || !node_online(node) || !NODE_DATA(node)) {
ptr = memblock_alloc_from(size, align, goal);
pr_info("cpu %d has no node %d or node-local memory\n",
cpu, node);
pr_debug("per cpu data for cpu%d %zu bytes at 0x%llx\n",
cpu, size, (u64)__pa(ptr));
} else {
ptr = memblock_alloc_try_nid(size, align, goal,
MEMBLOCK_ALLOC_ACCESSIBLE,
node);
pr_debug("per cpu data for cpu%d %zu bytes on node%d at 0x%llx\n",
cpu, size, node, (u64)__pa(ptr));
}
return ptr;
#else
return memblock_alloc_from(size, align, goal);
#endif
}
static void __init pcpu_fc_free(void *ptr, size_t size)
{
memblock_free(ptr, size);
}
#endif /* BUILD_EMBED_FIRST_CHUNK || BUILD_PAGE_FIRST_CHUNK */
#if defined(BUILD_EMBED_FIRST_CHUNK)
/**
* pcpu_embed_first_chunk - embed the first percpu chunk into bootmem
* @reserved_size: the size of reserved percpu area in bytes
* @dyn_size: minimum free size for dynamic allocation in bytes
* @atom_size: allocation atom size
* @cpu_distance_fn: callback to determine distance between cpus, optional
* @cpu_to_nd_fn: callback to convert cpu to it's node, optional
*
* This is a helper to ease setting up embedded first percpu chunk and
* can be called where pcpu_setup_first_chunk() is expected.
*
* If this function is used to setup the first chunk, it is allocated
* by calling pcpu_fc_alloc and used as-is without being mapped into
* vmalloc area. Allocations are always whole multiples of @atom_size
* aligned to @atom_size.
*
* This enables the first chunk to piggy back on the linear physical
* mapping which often uses larger page size. Please note that this
* can result in very sparse cpu->unit mapping on NUMA machines thus
* requiring large vmalloc address space. Don't use this allocator if
* vmalloc space is not orders of magnitude larger than distances
* between node memory addresses (ie. 32bit NUMA machines).
*
* @dyn_size specifies the minimum dynamic area size.
*
* If the needed size is smaller than the minimum or specified unit
* size, the leftover is returned using pcpu_fc_free.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
size_t atom_size,
pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)
{
void *base = (void *)ULONG_MAX;
void **areas = NULL;
struct pcpu_alloc_info *ai;
size_t size_sum, areas_size;
unsigned long max_distance;
int group, i, highest_group, rc = 0;
ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size,
cpu_distance_fn);
if (IS_ERR(ai))
return PTR_ERR(ai);
size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *));
areas = memblock_alloc(areas_size, SMP_CACHE_BYTES);
if (!areas) {
rc = -ENOMEM;
goto out_free;
}
/* allocate, copy and determine base address & max_distance */
highest_group = 0;
for (group = 0; group < ai->nr_groups; group++) {
struct pcpu_group_info *gi = &ai->groups[group];
unsigned int cpu = NR_CPUS;
void *ptr;
for (i = 0; i < gi->nr_units && cpu == NR_CPUS; i++)
cpu = gi->cpu_map[i];
BUG_ON(cpu == NR_CPUS);
/* allocate space for the whole group */
ptr = pcpu_fc_alloc(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn);
if (!ptr) {
rc = -ENOMEM;
goto out_free_areas;
}
/* kmemleak tracks the percpu allocations separately */
kmemleak_ignore_phys(__pa(ptr));
areas[group] = ptr;
base = min(ptr, base);
if (ptr > areas[highest_group])
highest_group = group;
}
max_distance = areas[highest_group] - base;
max_distance += ai->unit_size * ai->groups[highest_group].nr_units;
/* warn if maximum distance is further than 75% of vmalloc space */
if (max_distance > VMALLOC_TOTAL * 3 / 4) {
pr_warn("max_distance=0x%lx too large for vmalloc space 0x%lx\n",
max_distance, VMALLOC_TOTAL);
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
/* and fail if we have fallback */
rc = -EINVAL;
goto out_free_areas;
#endif
}
/*
* Copy data and free unused parts. This should happen after all
* allocations are complete; otherwise, we may end up with
* overlapping groups.
*/
for (group = 0; group < ai->nr_groups; group++) {
struct pcpu_group_info *gi = &ai->groups[group];
void *ptr = areas[group];
for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {
if (gi->cpu_map[i] == NR_CPUS) {
/* unused unit, free whole */
pcpu_fc_free(ptr, ai->unit_size);
continue;
}
/* copy and return the unused part */
memcpy(ptr, __per_cpu_load, ai->static_size);
pcpu_fc_free(ptr + size_sum, ai->unit_size - size_sum);
}
}
/* base address is now known, determine group base offsets */
for (group = 0; group < ai->nr_groups; group++) {
ai->groups[group].base_offset = areas[group] - base;
}
pr_info("Embedded %zu pages/cpu s%zu r%zu d%zu u%zu\n",
PFN_DOWN(size_sum), ai->static_size, ai->reserved_size,
ai->dyn_size, ai->unit_size);
pcpu_setup_first_chunk(ai, base);
goto out_free;
out_free_areas:
for (group = 0; group < ai->nr_groups; group++)
if (areas[group])
pcpu_fc_free(areas[group],
ai->groups[group].nr_units * ai->unit_size);
out_free:
pcpu_free_alloc_info(ai);
if (areas)
memblock_free(areas, areas_size);
return rc;
}
#endif /* BUILD_EMBED_FIRST_CHUNK */
#ifdef BUILD_PAGE_FIRST_CHUNK
#include <asm/pgalloc.h>
#ifndef P4D_TABLE_SIZE
#define P4D_TABLE_SIZE PAGE_SIZE
#endif
#ifndef PUD_TABLE_SIZE
#define PUD_TABLE_SIZE PAGE_SIZE
#endif
#ifndef PMD_TABLE_SIZE
#define PMD_TABLE_SIZE PAGE_SIZE
#endif
#ifndef PTE_TABLE_SIZE
#define PTE_TABLE_SIZE PAGE_SIZE
#endif
void __init __weak pcpu_populate_pte(unsigned long addr)
{
pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
if (pgd_none(*pgd)) {
p4d = memblock_alloc(P4D_TABLE_SIZE, P4D_TABLE_SIZE);
if (!p4d)
goto err_alloc;
pgd_populate(&init_mm, pgd, p4d);
}
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d)) {
pud = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE);
if (!pud)
goto err_alloc;
p4d_populate(&init_mm, p4d, pud);
}
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) {
pmd = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE);
if (!pmd)
goto err_alloc;
pud_populate(&init_mm, pud, pmd);
}
pmd = pmd_offset(pud, addr);
if (!pmd_present(*pmd)) {
pte_t *new;
new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE);
if (!new)
goto err_alloc;
pmd_populate_kernel(&init_mm, pmd, new);
}
return;
err_alloc:
panic("%s: Failed to allocate memory\n", __func__);
}
/**
* pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages
* @reserved_size: the size of reserved percpu area in bytes
* @cpu_to_nd_fn: callback to convert cpu to it's node, optional
*
* This is a helper to ease setting up page-remapped first percpu
* chunk and can be called where pcpu_setup_first_chunk() is expected.
*
* This is the basic allocator. Static percpu area is allocated
* page-by-page into vmalloc area.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)
{
static struct vm_struct vm;
struct pcpu_alloc_info *ai;
char psize_str[16];
int unit_pages;
size_t pages_size;
struct page **pages;
int unit, i, j, rc = 0;
int upa;
int nr_g0_units;
snprintf(psize_str, sizeof(psize_str), "%luK", PAGE_SIZE >> 10);
ai = pcpu_build_alloc_info(reserved_size, 0, PAGE_SIZE, NULL);
if (IS_ERR(ai))
return PTR_ERR(ai);
BUG_ON(ai->nr_groups != 1);
upa = ai->alloc_size/ai->unit_size;
nr_g0_units = roundup(num_possible_cpus(), upa);
if (WARN_ON(ai->groups[0].nr_units != nr_g0_units)) {
pcpu_free_alloc_info(ai);
return -EINVAL;
}
unit_pages = ai->unit_size >> PAGE_SHIFT;
/* unaligned allocations can't be freed, round up to page size */
pages_size = PFN_ALIGN(unit_pages * num_possible_cpus() *
sizeof(pages[0]));
pages = memblock_alloc(pages_size, SMP_CACHE_BYTES);
if (!pages)
panic("%s: Failed to allocate %zu bytes\n", __func__,
pages_size);
/* allocate pages */
j = 0;
for (unit = 0; unit < num_possible_cpus(); unit++) {
unsigned int cpu = ai->groups[0].cpu_map[unit];
for (i = 0; i < unit_pages; i++) {
void *ptr;
ptr = pcpu_fc_alloc(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn);
if (!ptr) {
pr_warn("failed to allocate %s page for cpu%u\n",
psize_str, cpu);
goto enomem;
}
/* kmemleak tracks the percpu allocations separately */
kmemleak_ignore_phys(__pa(ptr));
pages[j++] = virt_to_page(ptr);
}
}
/* allocate vm area, map the pages and copy static data */
vm.flags = VM_ALLOC;
vm.size = num_possible_cpus() * ai->unit_size;
vm_area_register_early(&vm, PAGE_SIZE);
for (unit = 0; unit < num_possible_cpus(); unit++) {
unsigned long unit_addr =
(unsigned long)vm.addr + unit * ai->unit_size;
for (i = 0; i < unit_pages; i++)
pcpu_populate_pte(unit_addr + (i << PAGE_SHIFT));
/* pte already populated, the following shouldn't fail */
rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages],
unit_pages);
if (rc < 0)
panic("failed to map percpu area, err=%d\n", rc);
flush_cache_vmap_early(unit_addr, unit_addr + ai->unit_size);
/* copy static data */
memcpy((void *)unit_addr, __per_cpu_load, ai->static_size);
}
/* we're ready, commit */
pr_info("%d %s pages/cpu s%zu r%zu d%zu\n",
unit_pages, psize_str, ai->static_size,
ai->reserved_size, ai->dyn_size);
pcpu_setup_first_chunk(ai, vm.addr);
goto out_free_ar;
enomem:
while (--j >= 0)
pcpu_fc_free(page_address(pages[j]), PAGE_SIZE);
rc = -ENOMEM;
out_free_ar:
memblock_free(pages, pages_size);
pcpu_free_alloc_info(ai);
return rc;
}
#endif /* BUILD_PAGE_FIRST_CHUNK */
#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
/*
* Generic SMP percpu area setup.
*
* The embedding helper is used because its behavior closely resembles
* the original non-dynamic generic percpu area setup. This is
* important because many archs have addressing restrictions and might
* fail if the percpu area is located far away from the previous
* location. As an added bonus, in non-NUMA cases, embedding is
* generally a good idea TLB-wise because percpu area can piggy back
* on the physical linear memory mapping which uses large page
* mappings on applicable archs.
*/
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(__per_cpu_offset);
void __init setup_per_cpu_areas(void)
{
unsigned long delta;
unsigned int cpu;
int rc;
/*
* Always reserve area for module percpu variables. That's
* what the legacy allocator did.
*/
rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE,
PAGE_SIZE, NULL, NULL);
if (rc < 0)
panic("Failed to initialize percpu areas.");
delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
for_each_possible_cpu(cpu)
__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
}
#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
#else /* CONFIG_SMP */
/*
* UP percpu area setup.
*
* UP always uses km-based percpu allocator with identity mapping.
* Static percpu variables are indistinguishable from the usual static
* variables and don't require any special preparation.
*/
void __init setup_per_cpu_areas(void)
{
const size_t unit_size =
roundup_pow_of_two(max_t(size_t, PCPU_MIN_UNIT_SIZE,
PERCPU_DYNAMIC_RESERVE));
struct pcpu_alloc_info *ai;
void *fc;
ai = pcpu_alloc_alloc_info(1, 1);
fc = memblock_alloc_from(unit_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
if (!ai || !fc)
panic("Failed to allocate memory for percpu areas.");
/* kmemleak tracks the percpu allocations separately */
kmemleak_ignore_phys(__pa(fc));
ai->dyn_size = unit_size;
ai->unit_size = unit_size;
ai->atom_size = unit_size;
ai->alloc_size = unit_size;
ai->groups[0].nr_units = 1;
ai->groups[0].cpu_map[0] = 0;
pcpu_setup_first_chunk(ai, fc);
pcpu_free_alloc_info(ai);
}
#endif /* CONFIG_SMP */
/*
* pcpu_nr_pages - calculate total number of populated backing pages
*
* This reflects the number of pages populated to back chunks. Metadata is
* excluded in the number exposed in meminfo as the number of backing pages
* scales with the number of cpus and can quickly outweigh the memory used for
* metadata. It also keeps this calculation nice and simple.
*
* RETURNS:
* Total number of populated backing pages in use by the allocator.
*/
unsigned long pcpu_nr_pages(void)
{
return pcpu_nr_populated * pcpu_nr_units;
}
/*
* Percpu allocator is initialized early during boot when neither slab or
* workqueue is available. Plug async management until everything is up
* and running.
*/
static int __init percpu_enable_async(void)
{
pcpu_async_enabled = true;
return 0;
}
subsys_initcall(percpu_enable_async);
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra
*
* Provides a framework for enqueueing and running callbacks from hardirq
* context. The enqueueing is NMI-safe.
*/
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/irq_work.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/irqflags.h>
#include <linux/sched.h>
#include <linux/tick.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/smp.h>
#include <linux/smpboot.h>
#include <asm/processor.h>
#include <linux/kasan.h>
#include <trace/events/ipi.h>
static DEFINE_PER_CPU(struct llist_head, raised_list);
static DEFINE_PER_CPU(struct llist_head, lazy_list);
static DEFINE_PER_CPU(struct task_struct *, irq_workd);
static void wake_irq_workd(void)
{
struct task_struct *tsk = __this_cpu_read(irq_workd);
if (!llist_empty(this_cpu_ptr(&lazy_list)) && tsk)
wake_up_process(tsk);
}
#ifdef CONFIG_SMP
static void irq_work_wake(struct irq_work *entry)
{
wake_irq_workd();
}
static DEFINE_PER_CPU(struct irq_work, irq_work_wakeup) =
IRQ_WORK_INIT_HARD(irq_work_wake);
#endif
static int irq_workd_should_run(unsigned int cpu)
{
return !llist_empty(this_cpu_ptr(&lazy_list));
}
/*
* Claim the entry so that no one else will poke at it.
*/
static bool irq_work_claim(struct irq_work *work)
{
int oflags;
oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags);
/*
* If the work is already pending, no need to raise the IPI.
* The pairing smp_mb() in irq_work_single() makes sure
* everything we did before is visible.
*/
if (oflags & IRQ_WORK_PENDING)
return false;
return true;
}
void __weak arch_irq_work_raise(void)
{
/*
* Lame architectures will get the timer tick callback
*/
}
static __always_inline void irq_work_raise(struct irq_work *work)
{
if (trace_ipi_send_cpu_enabled() && arch_irq_work_has_interrupt()) trace_ipi_send_cpu(smp_processor_id(), _RET_IP_, work->func); arch_irq_work_raise();
}
/* Enqueue on current CPU, work must already be claimed and preempt disabled */
static void __irq_work_queue_local(struct irq_work *work)
{
struct llist_head *list;
bool rt_lazy_work = false;
bool lazy_work = false;
int work_flags;
work_flags = atomic_read(&work->node.a_flags);
if (work_flags & IRQ_WORK_LAZY)
lazy_work = true;
else if (IS_ENABLED(CONFIG_PREEMPT_RT) &&
!(work_flags & IRQ_WORK_HARD_IRQ))
rt_lazy_work = true;
if (lazy_work || rt_lazy_work)
list = this_cpu_ptr(&lazy_list);
else
list = this_cpu_ptr(&raised_list);
if (!llist_add(&work->node.llist, list))
return;
/* If the work is "lazy", handle it from next tick if any */
if (!lazy_work || tick_nohz_tick_stopped()) irq_work_raise(work);
}
/* Enqueue the irq work @work on the current CPU */
bool irq_work_queue(struct irq_work *work)
{
/* Only queue if not already pending */
if (!irq_work_claim(work))
return false;
/* Queue the entry and raise the IPI if needed. */
preempt_disable();
__irq_work_queue_local(work);
preempt_enable();
return true;
}
EXPORT_SYMBOL_GPL(irq_work_queue);
/*
* Enqueue the irq_work @work on @cpu unless it's already pending
* somewhere.
*
* Can be re-enqueued while the callback is still in progress.
*/
bool irq_work_queue_on(struct irq_work *work, int cpu)
{
#ifndef CONFIG_SMP
return irq_work_queue(work);
#else /* CONFIG_SMP: */
/* All work should have been flushed before going offline */
WARN_ON_ONCE(cpu_is_offline(cpu));
/* Only queue if not already pending */
if (!irq_work_claim(work))
return false;
kasan_record_aux_stack_noalloc(work);
preempt_disable();
if (cpu != smp_processor_id()) {
/* Arch remote IPI send/receive backend aren't NMI safe */
WARN_ON_ONCE(in_nmi());
/*
* On PREEMPT_RT the items which are not marked as
* IRQ_WORK_HARD_IRQ are added to the lazy list and a HARD work
* item is used on the remote CPU to wake the thread.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT) &&
!(atomic_read(&work->node.a_flags) & IRQ_WORK_HARD_IRQ)) {
if (!llist_add(&work->node.llist, &per_cpu(lazy_list, cpu)))
goto out;
work = &per_cpu(irq_work_wakeup, cpu);
if (!irq_work_claim(work))
goto out;
}
__smp_call_single_queue(cpu, &work->node.llist);
} else {
__irq_work_queue_local(work);
}
out:
preempt_enable();
return true;
#endif /* CONFIG_SMP */
}
bool irq_work_needs_cpu(void)
{
struct llist_head *raised, *lazy;
raised = this_cpu_ptr(&raised_list);
lazy = this_cpu_ptr(&lazy_list);
if (llist_empty(raised) || arch_irq_work_has_interrupt())
if (llist_empty(lazy))
return false;
/* All work should have been flushed before going offline */
WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
return true;
}
void irq_work_single(void *arg)
{
struct irq_work *work = arg;
int flags;
/*
* Clear the PENDING bit, after this point the @work can be re-used.
* The PENDING bit acts as a lock, and we own it, so we can clear it
* without atomic ops.
*/
flags = atomic_read(&work->node.a_flags);
flags &= ~IRQ_WORK_PENDING;
atomic_set(&work->node.a_flags, flags);
/*
* See irq_work_claim().
*/
smp_mb();
lockdep_irq_work_enter(flags);
work->func(work);
lockdep_irq_work_exit(flags);
/*
* Clear the BUSY bit, if set, and return to the free state if no-one
* else claimed it meanwhile.
*/
(void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY);
if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) ||
!arch_irq_work_has_interrupt())
rcuwait_wake_up(&work->irqwait);
}
static void irq_work_run_list(struct llist_head *list)
{
struct irq_work *work, *tmp;
struct llist_node *llnode;
/*
* On PREEMPT_RT IRQ-work which is not marked as HARD will be processed
* in a per-CPU thread in preemptible context. Only the items which are
* marked as IRQ_WORK_HARD_IRQ will be processed in hardirq context.
*/
BUG_ON(!irqs_disabled() && !IS_ENABLED(CONFIG_PREEMPT_RT)); if (llist_empty(list))
return;
llnode = llist_del_all(list); llist_for_each_entry_safe(work, tmp, llnode, node.llist)
irq_work_single(work);
}
/*
* hotplug calls this through:
* hotplug_cfd() -> flush_smp_call_function_queue()
*/
void irq_work_run(void)
{
irq_work_run_list(this_cpu_ptr(&raised_list));
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
irq_work_run_list(this_cpu_ptr(&lazy_list));
else
wake_irq_workd();
}
EXPORT_SYMBOL_GPL(irq_work_run);
void irq_work_tick(void)
{
struct llist_head *raised = this_cpu_ptr(&raised_list); if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) irq_work_run_list(raised);
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
irq_work_run_list(this_cpu_ptr(&lazy_list));
else
wake_irq_workd();
}
/*
* Synchronize against the irq_work @entry, ensures the entry is not
* currently in use.
*/
void irq_work_sync(struct irq_work *work)
{
lockdep_assert_irqs_enabled();
might_sleep();
if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) ||
!arch_irq_work_has_interrupt()) {
rcuwait_wait_event(&work->irqwait, !irq_work_is_busy(work),
TASK_UNINTERRUPTIBLE);
return;
}
while (irq_work_is_busy(work))
cpu_relax();
}
EXPORT_SYMBOL_GPL(irq_work_sync);
static void run_irq_workd(unsigned int cpu)
{
irq_work_run_list(this_cpu_ptr(&lazy_list));
}
static void irq_workd_setup(unsigned int cpu)
{
sched_set_fifo_low(current);
}
static struct smp_hotplug_thread irqwork_threads = {
.store = &irq_workd,
.setup = irq_workd_setup,
.thread_should_run = irq_workd_should_run,
.thread_fn = run_irq_workd,
.thread_comm = "irq_work/%u",
};
static __init int irq_work_init_threads(void)
{
if (IS_ENABLED(CONFIG_PREEMPT_RT))
BUG_ON(smpboot_register_percpu_thread(&irqwork_threads));
return 0;
}
early_initcall(irq_work_init_threads);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_DELAY_H
#define _LINUX_DELAY_H
/*
* Copyright (C) 1993 Linus Torvalds
*
* Delay routines, using a pre-computed "loops_per_jiffy" value.
*
* Please note that ndelay(), udelay() and mdelay() may return early for
* several reasons:
* 1. computed loops_per_jiffy too low (due to the time taken to
* execute the timer interrupt.)
* 2. cache behaviour affecting the time it takes to execute the
* loop function.
* 3. CPU clock rate changes.
*
* Please see this thread:
* https://lists.openwall.net/linux-kernel/2011/01/09/56
*/
#include <linux/math.h>
#include <linux/sched.h>
extern unsigned long loops_per_jiffy;
#include <asm/delay.h>
/*
* Using udelay() for intervals greater than a few milliseconds can
* risk overflow for high loops_per_jiffy (high bogomips) machines. The
* mdelay() provides a wrapper to prevent this. For delays greater
* than MAX_UDELAY_MS milliseconds, the wrapper is used. Architecture
* specific values can be defined in asm-???/delay.h as an override.
* The 2nd mdelay() definition ensures GCC will optimize away the
* while loop for the common cases where n <= MAX_UDELAY_MS -- Paul G.
*/
#ifndef MAX_UDELAY_MS
#define MAX_UDELAY_MS 5
#endif
#ifndef mdelay
#define mdelay(n) (\
(__builtin_constant_p(n) && (n)<=MAX_UDELAY_MS) ? udelay((n)*1000) : \
({unsigned long __ms=(n); while (__ms--) udelay(1000);}))
#endif
#ifndef ndelay
static inline void ndelay(unsigned long x)
{
udelay(DIV_ROUND_UP(x, 1000));
}
#define ndelay(x) ndelay(x)
#endif
extern unsigned long lpj_fine;
void calibrate_delay(void);
unsigned long calibrate_delay_is_known(void);
void __attribute__((weak)) calibration_delay_done(void);
void msleep(unsigned int msecs);
unsigned long msleep_interruptible(unsigned int msecs);
void usleep_range_state(unsigned long min, unsigned long max,
unsigned int state);
static inline void usleep_range(unsigned long min, unsigned long max)
{
usleep_range_state(min, max, TASK_UNINTERRUPTIBLE);
}
static inline void usleep_idle_range(unsigned long min, unsigned long max)
{
usleep_range_state(min, max, TASK_IDLE);
}
static inline void ssleep(unsigned int seconds)
{
msleep(seconds * 1000);
}
/* see Documentation/timers/timers-howto.rst for the thresholds */
static inline void fsleep(unsigned long usecs)
{
if (usecs <= 10)
udelay(usecs);
else if (usecs <= 20000)
usleep_range(usecs, 2 * usecs);
else
msleep(DIV_ROUND_UP(usecs, 1000));
}
#endif /* defined(_LINUX_DELAY_H) */
/*
* Copyright (c) 2006-2008 Intel Corporation
* Copyright (c) 2007 Dave Airlie <airlied@linux.ie>
*
* DRM core CRTC related functions
*
* Permission to use, copy, modify, distribute, and sell this software and its
* documentation for any purpose is hereby granted without fee, provided that
* the above copyright notice appear in all copies and that both that copyright
* notice and this permission notice appear in supporting documentation, and
* that the name of the copyright holders not be used in advertising or
* publicity pertaining to distribution of the software without specific,
* written prior permission. The copyright holders make no representations
* about the suitability of this software for any purpose. It is provided "as
* is" without express or implied warranty.
*
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
* EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
* DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
* TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
* OF THIS SOFTWARE.
*
* Authors:
* Keith Packard
* Eric Anholt <eric@anholt.net>
* Dave Airlie <airlied@linux.ie>
* Jesse Barnes <jesse.barnes@intel.com>
*/
#include <linux/export.h>
#include <linux/moduleparam.h>
#include <drm/drm_bridge.h>
#include <drm/drm_client.h>
#include <drm/drm_crtc.h>
#include <drm/drm_edid.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_modeset_helper_vtables.h>
#include <drm/drm_print.h>
#include <drm/drm_probe_helper.h>
#include <drm/drm_sysfs.h>
#include "drm_crtc_helper_internal.h"
/**
* DOC: output probing helper overview
*
* This library provides some helper code for output probing. It provides an
* implementation of the core &drm_connector_funcs.fill_modes interface with
* drm_helper_probe_single_connector_modes().
*
* It also provides support for polling connectors with a work item and for
* generic hotplug interrupt handling where the driver doesn't or cannot keep
* track of a per-connector hpd interrupt.
*
* This helper library can be used independently of the modeset helper library.
* Drivers can also overwrite different parts e.g. use their own hotplug
* handling code to avoid probing unrelated outputs.
*
* The probe helpers share the function table structures with other display
* helper libraries. See &struct drm_connector_helper_funcs for the details.
*/
static bool drm_kms_helper_poll = true;
module_param_named(poll, drm_kms_helper_poll, bool, 0600);
static enum drm_mode_status
drm_mode_validate_flag(const struct drm_display_mode *mode,
int flags)
{
if ((mode->flags & DRM_MODE_FLAG_INTERLACE) &&
!(flags & DRM_MODE_FLAG_INTERLACE))
return MODE_NO_INTERLACE;
if ((mode->flags & DRM_MODE_FLAG_DBLSCAN) &&
!(flags & DRM_MODE_FLAG_DBLSCAN))
return MODE_NO_DBLESCAN;
if ((mode->flags & DRM_MODE_FLAG_3D_MASK) &&
!(flags & DRM_MODE_FLAG_3D_MASK))
return MODE_NO_STEREO;
return MODE_OK;
}
static int
drm_mode_validate_pipeline(struct drm_display_mode *mode,
struct drm_connector *connector,
struct drm_modeset_acquire_ctx *ctx,
enum drm_mode_status *status)
{
struct drm_device *dev = connector->dev;
struct drm_encoder *encoder;
int ret;
/* Step 1: Validate against connector */
ret = drm_connector_mode_valid(connector, mode, ctx, status);
if (ret || *status != MODE_OK)
return ret;
/* Step 2: Validate against encoders and crtcs */
drm_connector_for_each_possible_encoder(connector, encoder) {
struct drm_bridge *bridge;
struct drm_crtc *crtc;
*status = drm_encoder_mode_valid(encoder, mode);
if (*status != MODE_OK) {
/* No point in continuing for crtc check as this encoder
* will not accept the mode anyway. If all encoders
* reject the mode then, at exit, ret will not be
* MODE_OK. */
continue;
}
bridge = drm_bridge_chain_get_first_bridge(encoder);
*status = drm_bridge_chain_mode_valid(bridge,
&connector->display_info,
mode);
if (*status != MODE_OK) {
/* There is also no point in continuing for crtc check
* here. */
continue;
}
drm_for_each_crtc(crtc, dev) {
if (!drm_encoder_crtc_ok(encoder, crtc))
continue;
*status = drm_crtc_mode_valid(crtc, mode);
if (*status == MODE_OK) {
/* If we get to this point there is at least
* one combination of encoder+crtc that works
* for this mode. Lets return now. */
return 0;
}
}
}
return 0;
}
static int drm_helper_probe_add_cmdline_mode(struct drm_connector *connector)
{
struct drm_cmdline_mode *cmdline_mode;
struct drm_display_mode *mode;
cmdline_mode = &connector->cmdline_mode;
if (!cmdline_mode->specified)
return 0;
/* Only add a GTF mode if we find no matching probed modes */
list_for_each_entry(mode, &connector->probed_modes, head) {
if (mode->hdisplay != cmdline_mode->xres ||
mode->vdisplay != cmdline_mode->yres)
continue;
if (cmdline_mode->refresh_specified) {
/* The probed mode's vrefresh is set until later */
if (drm_mode_vrefresh(mode) != cmdline_mode->refresh)
continue;
}
/* Mark the matching mode as being preferred by the user */
mode->type |= DRM_MODE_TYPE_USERDEF;
return 0;
}
mode = drm_mode_create_from_cmdline_mode(connector->dev,
cmdline_mode);
if (mode == NULL)
return 0;
drm_mode_probed_add(connector, mode);
return 1;
}
enum drm_mode_status drm_crtc_mode_valid(struct drm_crtc *crtc,
const struct drm_display_mode *mode)
{
const struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
if (!crtc_funcs || !crtc_funcs->mode_valid)
return MODE_OK;
return crtc_funcs->mode_valid(crtc, mode);
}
enum drm_mode_status drm_encoder_mode_valid(struct drm_encoder *encoder,
const struct drm_display_mode *mode)
{
const struct drm_encoder_helper_funcs *encoder_funcs =
encoder->helper_private;
if (!encoder_funcs || !encoder_funcs->mode_valid)
return MODE_OK;
return encoder_funcs->mode_valid(encoder, mode);
}
int
drm_connector_mode_valid(struct drm_connector *connector,
struct drm_display_mode *mode,
struct drm_modeset_acquire_ctx *ctx,
enum drm_mode_status *status)
{
const struct drm_connector_helper_funcs *connector_funcs =
connector->helper_private;
int ret = 0;
if (!connector_funcs)
*status = MODE_OK;
else if (connector_funcs->mode_valid_ctx)
ret = connector_funcs->mode_valid_ctx(connector, mode, ctx,
status);
else if (connector_funcs->mode_valid)
*status = connector_funcs->mode_valid(connector, mode);
else
*status = MODE_OK;
return ret;
}
static void drm_kms_helper_disable_hpd(struct drm_device *dev)
{
struct drm_connector *connector;
struct drm_connector_list_iter conn_iter;
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) { const struct drm_connector_helper_funcs *funcs =
connector->helper_private;
if (funcs && funcs->disable_hpd) funcs->disable_hpd(connector);
}
drm_connector_list_iter_end(&conn_iter);
}
static bool drm_kms_helper_enable_hpd(struct drm_device *dev)
{
bool poll = false;
struct drm_connector *connector;
struct drm_connector_list_iter conn_iter;
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) {
const struct drm_connector_helper_funcs *funcs =
connector->helper_private;
if (funcs && funcs->enable_hpd)
funcs->enable_hpd(connector);
if (connector->polled & (DRM_CONNECTOR_POLL_CONNECT |
DRM_CONNECTOR_POLL_DISCONNECT))
poll = true;
}
drm_connector_list_iter_end(&conn_iter);
return poll;
}
#define DRM_OUTPUT_POLL_PERIOD (10*HZ)
static void reschedule_output_poll_work(struct drm_device *dev)
{
unsigned long delay = DRM_OUTPUT_POLL_PERIOD;
if (dev->mode_config.delayed_event)
/*
* FIXME:
*
* Use short (1s) delay to handle the initial delayed event.
* This delay should not be needed, but Optimus/nouveau will
* fail in a mysterious way if the delayed event is handled as
* soon as possible like it is done in
* drm_helper_probe_single_connector_modes() in case the poll
* was enabled before.
*/
delay = HZ;
schedule_delayed_work(&dev->mode_config.output_poll_work, delay);
}
/**
* drm_kms_helper_poll_enable - re-enable output polling.
* @dev: drm_device
*
* This function re-enables the output polling work, after it has been
* temporarily disabled using drm_kms_helper_poll_disable(), for example over
* suspend/resume.
*
* Drivers can call this helper from their device resume implementation. It is
* not an error to call this even when output polling isn't enabled.
*
* If device polling was never initialized before, this call will trigger a
* warning and return.
*
* Note that calls to enable and disable polling must be strictly ordered, which
* is automatically the case when they're only call from suspend/resume
* callbacks.
*/
void drm_kms_helper_poll_enable(struct drm_device *dev)
{
if (drm_WARN_ON_ONCE(dev, !dev->mode_config.poll_enabled) ||
!drm_kms_helper_poll || dev->mode_config.poll_running)
return;
if (drm_kms_helper_enable_hpd(dev) ||
dev->mode_config.delayed_event)
reschedule_output_poll_work(dev);
dev->mode_config.poll_running = true;
}
EXPORT_SYMBOL(drm_kms_helper_poll_enable);
/**
* drm_kms_helper_poll_reschedule - reschedule the output polling work
* @dev: drm_device
*
* This function reschedules the output polling work, after polling for a
* connector has been enabled.
*
* Drivers must call this helper after enabling polling for a connector by
* setting %DRM_CONNECTOR_POLL_CONNECT / %DRM_CONNECTOR_POLL_DISCONNECT flags
* in drm_connector::polled. Note that after disabling polling by clearing these
* flags for a connector will stop the output polling work automatically if
* the polling is disabled for all other connectors as well.
*
* The function can be called only after polling has been enabled by calling
* drm_kms_helper_poll_init() / drm_kms_helper_poll_enable().
*/
void drm_kms_helper_poll_reschedule(struct drm_device *dev)
{
if (dev->mode_config.poll_running)
reschedule_output_poll_work(dev);
}
EXPORT_SYMBOL(drm_kms_helper_poll_reschedule);
static enum drm_connector_status
drm_helper_probe_detect_ctx(struct drm_connector *connector, bool force)
{
const struct drm_connector_helper_funcs *funcs = connector->helper_private;
struct drm_modeset_acquire_ctx ctx;
int ret;
drm_modeset_acquire_init(&ctx, 0);
retry:
ret = drm_modeset_lock(&connector->dev->mode_config.connection_mutex, &ctx);
if (!ret) {
if (funcs->detect_ctx)
ret = funcs->detect_ctx(connector, &ctx, force);
else if (connector->funcs->detect)
ret = connector->funcs->detect(connector, force);
else
ret = connector_status_connected;
}
if (ret == -EDEADLK) {
drm_modeset_backoff(&ctx);
goto retry;
}
if (WARN_ON(ret < 0))
ret = connector_status_unknown;
if (ret != connector->status)
connector->epoch_counter += 1;
drm_modeset_drop_locks(&ctx);
drm_modeset_acquire_fini(&ctx);
return ret;
}
/**
* drm_helper_probe_detect - probe connector status
* @connector: connector to probe
* @ctx: acquire_ctx, or NULL to let this function handle locking.
* @force: Whether destructive probe operations should be performed.
*
* This function calls the detect callbacks of the connector.
* This function returns &drm_connector_status, or
* if @ctx is set, it might also return -EDEADLK.
*/
int
drm_helper_probe_detect(struct drm_connector *connector,
struct drm_modeset_acquire_ctx *ctx,
bool force)
{
const struct drm_connector_helper_funcs *funcs = connector->helper_private;
struct drm_device *dev = connector->dev;
int ret;
if (!ctx)
return drm_helper_probe_detect_ctx(connector, force);
ret = drm_modeset_lock(&dev->mode_config.connection_mutex, ctx);
if (ret)
return ret;
if (funcs->detect_ctx)
ret = funcs->detect_ctx(connector, ctx, force);
else if (connector->funcs->detect)
ret = connector->funcs->detect(connector, force);
else
ret = connector_status_connected;
if (ret != connector->status)
connector->epoch_counter += 1;
return ret;
}
EXPORT_SYMBOL(drm_helper_probe_detect);
static int drm_helper_probe_get_modes(struct drm_connector *connector)
{
const struct drm_connector_helper_funcs *connector_funcs =
connector->helper_private;
int count;
count = connector_funcs->get_modes(connector);
/* The .get_modes() callback should not return negative values. */
if (count < 0) {
drm_err(connector->dev, ".get_modes() returned %pe\n",
ERR_PTR(count));
count = 0;
}
/*
* Fallback for when DDC probe failed in drm_get_edid() and thus skipped
* override/firmware EDID.
*/
if (count == 0 && connector->status == connector_status_connected)
count = drm_edid_override_connector_update(connector);
return count;
}
static int __drm_helper_update_and_validate(struct drm_connector *connector,
uint32_t maxX, uint32_t maxY,
struct drm_modeset_acquire_ctx *ctx)
{
struct drm_device *dev = connector->dev;
struct drm_display_mode *mode;
int mode_flags = 0;
int ret;
drm_connector_list_update(connector);
if (connector->interlace_allowed)
mode_flags |= DRM_MODE_FLAG_INTERLACE;
if (connector->doublescan_allowed)
mode_flags |= DRM_MODE_FLAG_DBLSCAN;
if (connector->stereo_allowed)
mode_flags |= DRM_MODE_FLAG_3D_MASK;
list_for_each_entry(mode, &connector->modes, head) {
if (mode->status != MODE_OK)
continue;
mode->status = drm_mode_validate_driver(dev, mode);
if (mode->status != MODE_OK)
continue;
mode->status = drm_mode_validate_size(mode, maxX, maxY);
if (mode->status != MODE_OK)
continue;
mode->status = drm_mode_validate_flag(mode, mode_flags);
if (mode->status != MODE_OK)
continue;
ret = drm_mode_validate_pipeline(mode, connector, ctx,
&mode->status);
if (ret) {
drm_dbg_kms(dev,
"drm_mode_validate_pipeline failed: %d\n",
ret);
if (drm_WARN_ON_ONCE(dev, ret != -EDEADLK))
mode->status = MODE_ERROR;
else
return -EDEADLK;
}
if (mode->status != MODE_OK)
continue;
mode->status = drm_mode_validate_ycbcr420(mode, connector);
}
return 0;
}
/**
* drm_helper_probe_single_connector_modes - get complete set of display modes
* @connector: connector to probe
* @maxX: max width for modes
* @maxY: max height for modes
*
* Based on the helper callbacks implemented by @connector in struct
* &drm_connector_helper_funcs try to detect all valid modes. Modes will first
* be added to the connector's probed_modes list, then culled (based on validity
* and the @maxX, @maxY parameters) and put into the normal modes list.
*
* Intended to be used as a generic implementation of the
* &drm_connector_funcs.fill_modes() vfunc for drivers that use the CRTC helpers
* for output mode filtering and detection.
*
* The basic procedure is as follows
*
* 1. All modes currently on the connector's modes list are marked as stale
*
* 2. New modes are added to the connector's probed_modes list with
* drm_mode_probed_add(). New modes start their life with status as OK.
* Modes are added from a single source using the following priority order.
*
* - &drm_connector_helper_funcs.get_modes vfunc
* - if the connector status is connector_status_connected, standard
* VESA DMT modes up to 1024x768 are automatically added
* (drm_add_modes_noedid())
*
* Finally modes specified via the kernel command line (video=...) are
* added in addition to what the earlier probes produced
* (drm_helper_probe_add_cmdline_mode()). These modes are generated
* using the VESA GTF/CVT formulas.
*
* 3. Modes are moved from the probed_modes list to the modes list. Potential
* duplicates are merged together (see drm_connector_list_update()).
* After this step the probed_modes list will be empty again.
*
* 4. Any non-stale mode on the modes list then undergoes validation
*
* - drm_mode_validate_basic() performs basic sanity checks
* - drm_mode_validate_size() filters out modes larger than @maxX and @maxY
* (if specified)
* - drm_mode_validate_flag() checks the modes against basic connector
* capabilities (interlace_allowed,doublescan_allowed,stereo_allowed)
* - the optional &drm_connector_helper_funcs.mode_valid or
* &drm_connector_helper_funcs.mode_valid_ctx helpers can perform driver
* and/or sink specific checks
* - the optional &drm_crtc_helper_funcs.mode_valid,
* &drm_bridge_funcs.mode_valid and &drm_encoder_helper_funcs.mode_valid
* helpers can perform driver and/or source specific checks which are also
* enforced by the modeset/atomic helpers
*
* 5. Any mode whose status is not OK is pruned from the connector's modes list,
* accompanied by a debug message indicating the reason for the mode's
* rejection (see drm_mode_prune_invalid()).
*
* Returns:
* The number of modes found on @connector.
*/
int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
uint32_t maxX, uint32_t maxY)
{
struct drm_device *dev = connector->dev;
struct drm_display_mode *mode;
int count = 0, ret;
enum drm_connector_status old_status;
struct drm_modeset_acquire_ctx ctx;
WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
drm_modeset_acquire_init(&ctx, 0);
DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id,
connector->name);
retry:
ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx);
if (ret == -EDEADLK) {
drm_modeset_backoff(&ctx);
goto retry;
} else
WARN_ON(ret < 0);
/* set all old modes to the stale state */
list_for_each_entry(mode, &connector->modes, head)
mode->status = MODE_STALE;
old_status = connector->status;
if (connector->force) {
if (connector->force == DRM_FORCE_ON ||
connector->force == DRM_FORCE_ON_DIGITAL)
connector->status = connector_status_connected;
else
connector->status = connector_status_disconnected;
if (connector->funcs->force)
connector->funcs->force(connector);
} else {
ret = drm_helper_probe_detect(connector, &ctx, true);
if (ret == -EDEADLK) {
drm_modeset_backoff(&ctx);
goto retry;
} else if (WARN(ret < 0, "Invalid return value %i for connector detection\n", ret))
ret = connector_status_unknown;
connector->status = ret;
}
/*
* Normally either the driver's hpd code or the poll loop should
* pick up any changes and fire the hotplug event. But if
* userspace sneaks in a probe, we might miss a change. Hence
* check here, and if anything changed start the hotplug code.
*/
if (old_status != connector->status) {
DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
connector->base.id,
connector->name,
drm_get_connector_status_name(old_status),
drm_get_connector_status_name(connector->status));
/*
* The hotplug event code might call into the fb
* helpers, and so expects that we do not hold any
* locks. Fire up the poll struct instead, it will
* disable itself again.
*/
dev->mode_config.delayed_event = true;
if (dev->mode_config.poll_enabled)
mod_delayed_work(system_wq,
&dev->mode_config.output_poll_work,
0);
}
/*
* Re-enable polling in case the global poll config changed but polling
* is still initialized.
*/
if (dev->mode_config.poll_enabled)
drm_kms_helper_poll_enable(dev);
if (connector->status == connector_status_disconnected) {
DRM_DEBUG_KMS("[CONNECTOR:%d:%s] disconnected\n",
connector->base.id, connector->name);
drm_connector_update_edid_property(connector, NULL);
drm_mode_prune_invalid(dev, &connector->modes, false);
goto exit;
}
count = drm_helper_probe_get_modes(connector);
if (count == 0 && (connector->status == connector_status_connected ||
connector->status == connector_status_unknown)) {
count = drm_add_modes_noedid(connector, 1024, 768);
/*
* Section 4.2.2.6 (EDID Corruption Detection) of the DP 1.4a
* Link CTS specifies that 640x480 (the official "failsafe"
* mode) needs to be the default if there's no EDID.
*/
if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort)
drm_set_preferred_mode(connector, 640, 480);
}
count += drm_helper_probe_add_cmdline_mode(connector);
if (count != 0) {
ret = __drm_helper_update_and_validate(connector, maxX, maxY, &ctx);
if (ret == -EDEADLK) {
drm_modeset_backoff(&ctx);
goto retry;
}
}
drm_mode_prune_invalid(dev, &connector->modes, true);
/*
* Displayport spec section 5.2.1.2 ("Video Timing Format") says that
* all detachable sinks shall support 640x480 @60Hz as a fail safe
* mode. If all modes were pruned, perhaps because they need more
* lanes or a higher pixel clock than available, at least try to add
* in 640x480.
*/
if (list_empty(&connector->modes) &&
connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
count = drm_add_modes_noedid(connector, 640, 480);
ret = __drm_helper_update_and_validate(connector, maxX, maxY, &ctx);
if (ret == -EDEADLK) {
drm_modeset_backoff(&ctx);
goto retry;
}
drm_mode_prune_invalid(dev, &connector->modes, true);
}
exit:
drm_modeset_drop_locks(&ctx);
drm_modeset_acquire_fini(&ctx);
if (list_empty(&connector->modes))
return 0;
drm_mode_sort(&connector->modes);
DRM_DEBUG_KMS("[CONNECTOR:%d:%s] probed modes :\n", connector->base.id,
connector->name);
list_for_each_entry(mode, &connector->modes, head) {
drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
drm_mode_debug_printmodeline(mode);
}
return count;
}
EXPORT_SYMBOL(drm_helper_probe_single_connector_modes);
/**
* drm_kms_helper_hotplug_event - fire off KMS hotplug events
* @dev: drm_device whose connector state changed
*
* This function fires off the uevent for userspace and also calls the
* output_poll_changed function, which is most commonly used to inform the fbdev
* emulation code and allow it to update the fbcon output configuration.
*
* Drivers should call this from their hotplug handling code when a change is
* detected. Note that this function does not do any output detection of its
* own, like drm_helper_hpd_irq_event() does - this is assumed to be done by the
* driver already.
*
* This function must be called from process context with no mode
* setting locks held.
*
* If only a single connector has changed, consider calling
* drm_kms_helper_connector_hotplug_event() instead.
*/
void drm_kms_helper_hotplug_event(struct drm_device *dev)
{
/* send a uevent + call fbdev */
drm_sysfs_hotplug_event(dev);
if (dev->mode_config.funcs->output_poll_changed)
dev->mode_config.funcs->output_poll_changed(dev);
drm_client_dev_hotplug(dev);
}
EXPORT_SYMBOL(drm_kms_helper_hotplug_event);
/**
* drm_kms_helper_connector_hotplug_event - fire off a KMS connector hotplug event
* @connector: drm_connector which has changed
*
* This is the same as drm_kms_helper_hotplug_event(), except it fires a more
* fine-grained uevent for a single connector.
*/
void drm_kms_helper_connector_hotplug_event(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
/* send a uevent + call fbdev */
drm_sysfs_connector_hotplug_event(connector);
if (dev->mode_config.funcs->output_poll_changed)
dev->mode_config.funcs->output_poll_changed(dev);
drm_client_dev_hotplug(dev);
}
EXPORT_SYMBOL(drm_kms_helper_connector_hotplug_event);
static void output_poll_execute(struct work_struct *work)
{
struct delayed_work *delayed_work = to_delayed_work(work);
struct drm_device *dev = container_of(delayed_work, struct drm_device, mode_config.output_poll_work);
struct drm_connector *connector;
struct drm_connector_list_iter conn_iter;
enum drm_connector_status old_status;
bool repoll = false, changed;
u64 old_epoch_counter;
if (!dev->mode_config.poll_enabled)
return;
/* Pick up any changes detected by the probe functions. */
changed = dev->mode_config.delayed_event;
dev->mode_config.delayed_event = false;
if (!drm_kms_helper_poll) {
if (dev->mode_config.poll_running) {
drm_kms_helper_disable_hpd(dev);
dev->mode_config.poll_running = false;
}
goto out;
}
if (!mutex_trylock(&dev->mode_config.mutex)) {
repoll = true;
goto out;
}
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) {
/* Ignore forced connectors. */
if (connector->force)
continue;
/* Ignore HPD capable connectors and connectors where we don't
* want any hotplug detection at all for polling. */
if (!connector->polled || connector->polled == DRM_CONNECTOR_POLL_HPD)
continue;
old_status = connector->status;
/* if we are connected and don't want to poll for disconnect
skip it */
if (old_status == connector_status_connected &&
!(connector->polled & DRM_CONNECTOR_POLL_DISCONNECT))
continue;
repoll = true;
old_epoch_counter = connector->epoch_counter;
connector->status = drm_helper_probe_detect(connector, NULL, false);
if (old_epoch_counter != connector->epoch_counter) {
const char *old, *new;
/*
* The poll work sets force=false when calling detect so
* that drivers can avoid to do disruptive tests (e.g.
* when load detect cycles could cause flickering on
* other, running displays). This bears the risk that we
* flip-flop between unknown here in the poll work and
* the real state when userspace forces a full detect
* call after receiving a hotplug event due to this
* change.
*
* Hence clamp an unknown detect status to the old
* value.
*/
if (connector->status == connector_status_unknown) {
connector->status = old_status;
continue;
}
old = drm_get_connector_status_name(old_status);
new = drm_get_connector_status_name(connector->status);
DRM_DEBUG_KMS("[CONNECTOR:%d:%s] "
"status updated from %s to %s\n",
connector->base.id,
connector->name,
old, new);
DRM_DEBUG_KMS("[CONNECTOR:%d:%s] epoch counter %llu -> %llu\n",
connector->base.id, connector->name,
old_epoch_counter, connector->epoch_counter);
changed = true;
}
}
drm_connector_list_iter_end(&conn_iter);
mutex_unlock(&dev->mode_config.mutex);
out:
if (changed)
drm_kms_helper_hotplug_event(dev);
if (repoll)
schedule_delayed_work(delayed_work, DRM_OUTPUT_POLL_PERIOD);
}
/**
* drm_kms_helper_is_poll_worker - is %current task an output poll worker?
*
* Determine if %current task is an output poll worker. This can be used
* to select distinct code paths for output polling versus other contexts.
*
* One use case is to avoid a deadlock between the output poll worker and
* the autosuspend worker wherein the latter waits for polling to finish
* upon calling drm_kms_helper_poll_disable(), while the former waits for
* runtime suspend to finish upon calling pm_runtime_get_sync() in a
* connector ->detect hook.
*/
bool drm_kms_helper_is_poll_worker(void)
{
struct work_struct *work = current_work();
return work && work->func == output_poll_execute;
}
EXPORT_SYMBOL(drm_kms_helper_is_poll_worker);
/**
* drm_kms_helper_poll_disable - disable output polling
* @dev: drm_device
*
* This function disables the output polling work.
*
* Drivers can call this helper from their device suspend implementation. It is
* not an error to call this even when output polling isn't enabled or already
* disabled. Polling is re-enabled by calling drm_kms_helper_poll_enable().
*
* If however, the polling was never initialized, this call will trigger a
* warning and return
*
* Note that calls to enable and disable polling must be strictly ordered, which
* is automatically the case when they're only call from suspend/resume
* callbacks.
*/
void drm_kms_helper_poll_disable(struct drm_device *dev)
{
if (drm_WARN_ON(dev, !dev->mode_config.poll_enabled))
return;
if (dev->mode_config.poll_running) drm_kms_helper_disable_hpd(dev); cancel_delayed_work_sync(&dev->mode_config.output_poll_work);
dev->mode_config.poll_running = false;
}
EXPORT_SYMBOL(drm_kms_helper_poll_disable);
/**
* drm_kms_helper_poll_init - initialize and enable output polling
* @dev: drm_device
*
* This function initializes and then also enables output polling support for
* @dev. Drivers which do not have reliable hotplug support in hardware can use
* this helper infrastructure to regularly poll such connectors for changes in
* their connection state.
*
* Drivers can control which connectors are polled by setting the
* DRM_CONNECTOR_POLL_CONNECT and DRM_CONNECTOR_POLL_DISCONNECT flags. On
* connectors where probing live outputs can result in visual distortion drivers
* should not set the DRM_CONNECTOR_POLL_DISCONNECT flag to avoid this.
* Connectors which have no flag or only DRM_CONNECTOR_POLL_HPD set are
* completely ignored by the polling logic.
*
* Note that a connector can be both polled and probed from the hotplug handler,
* in case the hotplug interrupt is known to be unreliable.
*/
void drm_kms_helper_poll_init(struct drm_device *dev)
{
INIT_DELAYED_WORK(&dev->mode_config.output_poll_work, output_poll_execute);
dev->mode_config.poll_enabled = true;
drm_kms_helper_poll_enable(dev);
}
EXPORT_SYMBOL(drm_kms_helper_poll_init);
/**
* drm_kms_helper_poll_fini - disable output polling and clean it up
* @dev: drm_device
*/
void drm_kms_helper_poll_fini(struct drm_device *dev)
{
if (!dev->mode_config.poll_enabled)
return;
drm_kms_helper_poll_disable(dev);
dev->mode_config.poll_enabled = false;
}
EXPORT_SYMBOL(drm_kms_helper_poll_fini);
static bool check_connector_changed(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
enum drm_connector_status old_status;
u64 old_epoch_counter;
/* Only handle HPD capable connectors. */
drm_WARN_ON(dev, !(connector->polled & DRM_CONNECTOR_POLL_HPD));
drm_WARN_ON(dev, !mutex_is_locked(&dev->mode_config.mutex));
old_status = connector->status;
old_epoch_counter = connector->epoch_counter;
connector->status = drm_helper_probe_detect(connector, NULL, false);
if (old_epoch_counter == connector->epoch_counter) {
drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Same epoch counter %llu\n",
connector->base.id,
connector->name,
connector->epoch_counter);
return false;
}
drm_dbg_kms(dev, "[CONNECTOR:%d:%s] status updated from %s to %s\n",
connector->base.id,
connector->name,
drm_get_connector_status_name(old_status),
drm_get_connector_status_name(connector->status));
drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Changed epoch counter %llu => %llu\n",
connector->base.id,
connector->name,
old_epoch_counter,
connector->epoch_counter);
return true;
}
/**
* drm_connector_helper_hpd_irq_event - hotplug processing
* @connector: drm_connector
*
* Drivers can use this helper function to run a detect cycle on a connector
* which has the DRM_CONNECTOR_POLL_HPD flag set in its &polled member.
*
* This helper function is useful for drivers which can track hotplug
* interrupts for a single connector. Drivers that want to send a
* hotplug event for all connectors or can't track hotplug interrupts
* per connector need to use drm_helper_hpd_irq_event().
*
* This function must be called from process context with no mode
* setting locks held.
*
* Note that a connector can be both polled and probed from the hotplug
* handler, in case the hotplug interrupt is known to be unreliable.
*
* Returns:
* A boolean indicating whether the connector status changed or not
*/
bool drm_connector_helper_hpd_irq_event(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
bool changed;
mutex_lock(&dev->mode_config.mutex);
changed = check_connector_changed(connector);
mutex_unlock(&dev->mode_config.mutex);
if (changed) {
drm_kms_helper_connector_hotplug_event(connector);
drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Sent hotplug event\n",
connector->base.id,
connector->name);
}
return changed;
}
EXPORT_SYMBOL(drm_connector_helper_hpd_irq_event);
/**
* drm_helper_hpd_irq_event - hotplug processing
* @dev: drm_device
*
* Drivers can use this helper function to run a detect cycle on all connectors
* which have the DRM_CONNECTOR_POLL_HPD flag set in their &polled member. All
* other connectors are ignored, which is useful to avoid reprobing fixed
* panels.
*
* This helper function is useful for drivers which can't or don't track hotplug
* interrupts for each connector.
*
* Drivers which support hotplug interrupts for each connector individually and
* which have a more fine-grained detect logic can use
* drm_connector_helper_hpd_irq_event(). Alternatively, they should bypass this
* code and directly call drm_kms_helper_hotplug_event() in case the connector
* state changed.
*
* This function must be called from process context with no mode
* setting locks held.
*
* Note that a connector can be both polled and probed from the hotplug handler,
* in case the hotplug interrupt is known to be unreliable.
*
* Returns:
* A boolean indicating whether the connector status changed or not
*/
bool drm_helper_hpd_irq_event(struct drm_device *dev)
{
struct drm_connector *connector, *first_changed_connector = NULL;
struct drm_connector_list_iter conn_iter;
int changed = 0;
if (!dev->mode_config.poll_enabled)
return false;
mutex_lock(&dev->mode_config.mutex);
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) {
/* Only handle HPD capable connectors. */
if (!(connector->polled & DRM_CONNECTOR_POLL_HPD))
continue;
if (check_connector_changed(connector)) {
if (!first_changed_connector) {
drm_connector_get(connector);
first_changed_connector = connector;
}
changed++;
}
}
drm_connector_list_iter_end(&conn_iter);
mutex_unlock(&dev->mode_config.mutex);
if (changed == 1)
drm_kms_helper_connector_hotplug_event(first_changed_connector);
else if (changed > 0)
drm_kms_helper_hotplug_event(dev);
if (first_changed_connector)
drm_connector_put(first_changed_connector);
return changed;
}
EXPORT_SYMBOL(drm_helper_hpd_irq_event);
/**
* drm_crtc_helper_mode_valid_fixed - Validates a display mode
* @crtc: the crtc
* @mode: the mode to validate
* @fixed_mode: the display hardware's mode
*
* Returns:
* MODE_OK on success, or another mode-status code otherwise.
*/
enum drm_mode_status drm_crtc_helper_mode_valid_fixed(struct drm_crtc *crtc,
const struct drm_display_mode *mode,
const struct drm_display_mode *fixed_mode)
{
if (mode->hdisplay != fixed_mode->hdisplay && mode->vdisplay != fixed_mode->vdisplay)
return MODE_ONE_SIZE;
else if (mode->hdisplay != fixed_mode->hdisplay)
return MODE_ONE_WIDTH;
else if (mode->vdisplay != fixed_mode->vdisplay)
return MODE_ONE_HEIGHT;
return MODE_OK;
}
EXPORT_SYMBOL(drm_crtc_helper_mode_valid_fixed);
/**
* drm_connector_helper_get_modes_fixed - Duplicates a display mode for a connector
* @connector: the connector
* @fixed_mode: the display hardware's mode
*
* This function duplicates a display modes for a connector. Drivers for hardware
* that only supports a single fixed mode can use this function in their connector's
* get_modes helper.
*
* Returns:
* The number of created modes.
*/
int drm_connector_helper_get_modes_fixed(struct drm_connector *connector,
const struct drm_display_mode *fixed_mode)
{
struct drm_device *dev = connector->dev;
struct drm_display_mode *mode;
mode = drm_mode_duplicate(dev, fixed_mode);
if (!mode) {
drm_err(dev, "Failed to duplicate mode " DRM_MODE_FMT "\n",
DRM_MODE_ARG(fixed_mode));
return 0;
}
if (mode->name[0] == '\0')
drm_mode_set_name(mode);
mode->type |= DRM_MODE_TYPE_PREFERRED;
drm_mode_probed_add(connector, mode);
if (mode->width_mm)
connector->display_info.width_mm = mode->width_mm;
if (mode->height_mm)
connector->display_info.height_mm = mode->height_mm;
return 1;
}
EXPORT_SYMBOL(drm_connector_helper_get_modes_fixed);
/**
* drm_connector_helper_get_modes - Read EDID and update connector.
* @connector: The connector
*
* Read the EDID using drm_edid_read() (which requires that connector->ddc is
* set), and update the connector using the EDID.
*
* This can be used as the "default" connector helper .get_modes() hook if the
* driver does not need any special processing. This is sets the example what
* custom .get_modes() hooks should do regarding EDID read and connector update.
*
* Returns: Number of modes.
*/
int drm_connector_helper_get_modes(struct drm_connector *connector)
{
const struct drm_edid *drm_edid;
int count;
drm_edid = drm_edid_read(connector);
/*
* Unconditionally update the connector. If the EDID was read
* successfully, fill in the connector information derived from the
* EDID. Otherwise, if the EDID is NULL, clear the connector
* information.
*/
drm_edid_connector_update(connector, drm_edid);
count = drm_edid_connector_add_modes(connector);
drm_edid_free(drm_edid);
return count;
}
EXPORT_SYMBOL(drm_connector_helper_get_modes);
/**
* drm_connector_helper_tv_get_modes - Fills the modes availables to a TV connector
* @connector: The connector
*
* Fills the available modes for a TV connector based on the supported
* TV modes, and the default mode expressed by the kernel command line.
*
* This can be used as the default TV connector helper .get_modes() hook
* if the driver does not need any special processing.
*
* Returns:
* The number of modes added to the connector.
*/
int drm_connector_helper_tv_get_modes(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
struct drm_property *tv_mode_property =
dev->mode_config.tv_mode_property;
struct drm_cmdline_mode *cmdline = &connector->cmdline_mode;
unsigned int ntsc_modes = BIT(DRM_MODE_TV_MODE_NTSC) |
BIT(DRM_MODE_TV_MODE_NTSC_443) |
BIT(DRM_MODE_TV_MODE_NTSC_J) |
BIT(DRM_MODE_TV_MODE_PAL_M);
unsigned int pal_modes = BIT(DRM_MODE_TV_MODE_PAL) |
BIT(DRM_MODE_TV_MODE_PAL_N) |
BIT(DRM_MODE_TV_MODE_SECAM);
unsigned int tv_modes[2] = { UINT_MAX, UINT_MAX };
unsigned int i, supported_tv_modes = 0;
if (!tv_mode_property)
return 0;
for (i = 0; i < tv_mode_property->num_values; i++)
supported_tv_modes |= BIT(tv_mode_property->values[i]);
if ((supported_tv_modes & ntsc_modes) &&
(supported_tv_modes & pal_modes)) {
uint64_t default_mode;
if (drm_object_property_get_default_value(&connector->base,
tv_mode_property,
&default_mode))
return 0;
if (cmdline->tv_mode_specified)
default_mode = cmdline->tv_mode;
if (BIT(default_mode) & ntsc_modes) {
tv_modes[0] = DRM_MODE_TV_MODE_NTSC;
tv_modes[1] = DRM_MODE_TV_MODE_PAL;
} else {
tv_modes[0] = DRM_MODE_TV_MODE_PAL;
tv_modes[1] = DRM_MODE_TV_MODE_NTSC;
}
} else if (supported_tv_modes & ntsc_modes) {
tv_modes[0] = DRM_MODE_TV_MODE_NTSC;
} else if (supported_tv_modes & pal_modes) {
tv_modes[0] = DRM_MODE_TV_MODE_PAL;
} else {
return 0;
}
for (i = 0; i < ARRAY_SIZE(tv_modes); i++) {
struct drm_display_mode *mode;
if (tv_modes[i] == DRM_MODE_TV_MODE_NTSC)
mode = drm_mode_analog_ntsc_480i(dev);
else if (tv_modes[i] == DRM_MODE_TV_MODE_PAL)
mode = drm_mode_analog_pal_576i(dev);
else
break;
if (!mode)
return i;
if (!i)
mode->type |= DRM_MODE_TYPE_PREFERRED;
drm_mode_probed_add(connector, mode);
}
return i;
}
EXPORT_SYMBOL(drm_connector_helper_tv_get_modes);
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2001 Momchil Velikov
* Portions Copyright (C) 2001 Christoph Hellwig
* Copyright (C) 2006 Nick Piggin
* Copyright (C) 2012 Konstantin Khlebnikov
*/
#ifndef _LINUX_RADIX_TREE_H
#define _LINUX_RADIX_TREE_H
#include <linux/bitops.h>
#include <linux/gfp_types.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/math.h>
#include <linux/percpu.h>
#include <linux/preempt.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/xarray.h>
#include <linux/local_lock.h>
/* Keep unconverted code working */
#define radix_tree_root xarray
#define radix_tree_node xa_node
struct radix_tree_preload {
local_lock_t lock;
unsigned nr;
/* nodes->parent points to next preallocated node */
struct radix_tree_node *nodes;
};
DECLARE_PER_CPU(struct radix_tree_preload, radix_tree_preloads);
/*
* The bottom two bits of the slot determine how the remaining bits in the
* slot are interpreted:
*
* 00 - data pointer
* 10 - internal entry
* x1 - value entry
*
* The internal entry may be a pointer to the next level in the tree, a
* sibling entry, or an indicator that the entry in this slot has been moved
* to another location in the tree and the lookup should be restarted. While
* NULL fits the 'data pointer' pattern, it means that there is no entry in
* the tree for this index (no matter what level of the tree it is found at).
* This means that storing a NULL entry in the tree is the same as deleting
* the entry from the tree.
*/
#define RADIX_TREE_ENTRY_MASK 3UL
#define RADIX_TREE_INTERNAL_NODE 2UL
static inline bool radix_tree_is_internal_node(void *ptr)
{
return ((unsigned long)ptr & RADIX_TREE_ENTRY_MASK) ==
RADIX_TREE_INTERNAL_NODE;
}
/*** radix-tree API starts here ***/
#define RADIX_TREE_MAP_SHIFT XA_CHUNK_SHIFT
#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
#define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1)
#define RADIX_TREE_MAX_TAGS XA_MAX_MARKS
#define RADIX_TREE_TAG_LONGS XA_MARK_LONGS
#define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long))
#define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \
RADIX_TREE_MAP_SHIFT))
/* The IDR tag is stored in the low bits of xa_flags */
#define ROOT_IS_IDR ((__force gfp_t)4)
/* The top bits of xa_flags are used to store the root tags */
#define ROOT_TAG_SHIFT (__GFP_BITS_SHIFT)
#define RADIX_TREE_INIT(name, mask) XARRAY_INIT(name, mask)
#define RADIX_TREE(name, mask) \
struct radix_tree_root name = RADIX_TREE_INIT(name, mask)
#define INIT_RADIX_TREE(root, mask) xa_init_flags(root, mask)
static inline bool radix_tree_empty(const struct radix_tree_root *root)
{
return root->xa_head == NULL;
}
/**
* struct radix_tree_iter - radix tree iterator state
*
* @index: index of current slot
* @next_index: one beyond the last index for this chunk
* @tags: bit-mask for tag-iterating
* @node: node that contains current slot
*
* This radix tree iterator works in terms of "chunks" of slots. A chunk is a
* subinterval of slots contained within one radix tree leaf node. It is
* described by a pointer to its first slot and a struct radix_tree_iter
* which holds the chunk's position in the tree and its size. For tagged
* iteration radix_tree_iter also holds the slots' bit-mask for one chosen
* radix tree tag.
*/
struct radix_tree_iter {
unsigned long index;
unsigned long next_index;
unsigned long tags;
struct radix_tree_node *node;
};
/**
* Radix-tree synchronization
*
* The radix-tree API requires that users provide all synchronisation (with
* specific exceptions, noted below).
*
* Synchronization of access to the data items being stored in the tree, and
* management of their lifetimes must be completely managed by API users.
*
* For API usage, in general,
* - any function _modifying_ the tree or tags (inserting or deleting
* items, setting or clearing tags) must exclude other modifications, and
* exclude any functions reading the tree.
* - any function _reading_ the tree or tags (looking up items or tags,
* gang lookups) must exclude modifications to the tree, but may occur
* concurrently with other readers.
*
* The notable exceptions to this rule are the following functions:
* __radix_tree_lookup
* radix_tree_lookup
* radix_tree_lookup_slot
* radix_tree_tag_get
* radix_tree_gang_lookup
* radix_tree_gang_lookup_tag
* radix_tree_gang_lookup_tag_slot
* radix_tree_tagged
*
* The first 7 functions are able to be called locklessly, using RCU. The
* caller must ensure calls to these functions are made within rcu_read_lock()
* regions. Other readers (lock-free or otherwise) and modifications may be
* running concurrently.
*
* It is still required that the caller manage the synchronization and lifetimes
* of the items. So if RCU lock-free lookups are used, typically this would mean
* that the items have their own locks, or are amenable to lock-free access; and
* that the items are freed by RCU (or only freed after having been deleted from
* the radix tree *and* a synchronize_rcu() grace period).
*
* (Note, rcu_assign_pointer and rcu_dereference are not needed to control
* access to data items when inserting into or looking up from the radix tree)
*
* Note that the value returned by radix_tree_tag_get() may not be relied upon
* if only the RCU read lock is held. Functions to set/clear tags and to
* delete nodes running concurrently with it may affect its result such that
* two consecutive reads in the same locked section may return different
* values. If reliability is required, modification functions must also be
* excluded from concurrency.
*
* radix_tree_tagged is able to be called without locking or RCU.
*/
/**
* radix_tree_deref_slot - dereference a slot
* @slot: slot pointer, returned by radix_tree_lookup_slot
*
* For use with radix_tree_lookup_slot(). Caller must hold tree at least read
* locked across slot lookup and dereference. Not required if write lock is
* held (ie. items cannot be concurrently inserted).
*
* radix_tree_deref_retry must be used to confirm validity of the pointer if
* only the read lock is held.
*
* Return: entry stored in that slot.
*/
static inline void *radix_tree_deref_slot(void __rcu **slot)
{
return rcu_dereference(*slot);
}
/**
* radix_tree_deref_slot_protected - dereference a slot with tree lock held
* @slot: slot pointer, returned by radix_tree_lookup_slot
*
* Similar to radix_tree_deref_slot. The caller does not hold the RCU read
* lock but it must hold the tree lock to prevent parallel updates.
*
* Return: entry stored in that slot.
*/
static inline void *radix_tree_deref_slot_protected(void __rcu **slot,
spinlock_t *treelock)
{
return rcu_dereference_protected(*slot, lockdep_is_held(treelock));
}
/**
* radix_tree_deref_retry - check radix_tree_deref_slot
* @arg: pointer returned by radix_tree_deref_slot
* Returns: 0 if retry is not required, otherwise retry is required
*
* radix_tree_deref_retry must be used with radix_tree_deref_slot.
*/
static inline int radix_tree_deref_retry(void *arg)
{
return unlikely(radix_tree_is_internal_node(arg));
}
/**
* radix_tree_exception - radix_tree_deref_slot returned either exception?
* @arg: value returned by radix_tree_deref_slot
* Returns: 0 if well-aligned pointer, non-0 if either kind of exception.
*/
static inline int radix_tree_exception(void *arg)
{
return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK);
}
int radix_tree_insert(struct radix_tree_root *, unsigned long index,
void *);
void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index,
struct radix_tree_node **nodep, void __rcu ***slotp);
void *radix_tree_lookup(const struct radix_tree_root *, unsigned long);
void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *,
unsigned long index);
void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *,
void __rcu **slot, void *entry);
void radix_tree_iter_replace(struct radix_tree_root *,
const struct radix_tree_iter *, void __rcu **slot, void *entry);
void radix_tree_replace_slot(struct radix_tree_root *,
void __rcu **slot, void *entry);
void radix_tree_iter_delete(struct radix_tree_root *,
struct radix_tree_iter *iter, void __rcu **slot);
void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *);
void *radix_tree_delete(struct radix_tree_root *, unsigned long);
unsigned int radix_tree_gang_lookup(const struct radix_tree_root *,
void **results, unsigned long first_index,
unsigned int max_items);
int radix_tree_preload(gfp_t gfp_mask);
int radix_tree_maybe_preload(gfp_t gfp_mask);
void radix_tree_init(void);
void *radix_tree_tag_set(struct radix_tree_root *,
unsigned long index, unsigned int tag);
void *radix_tree_tag_clear(struct radix_tree_root *,
unsigned long index, unsigned int tag);
int radix_tree_tag_get(const struct radix_tree_root *,
unsigned long index, unsigned int tag);
void radix_tree_iter_tag_clear(struct radix_tree_root *,
const struct radix_tree_iter *iter, unsigned int tag);
unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *,
void **results, unsigned long first_index,
unsigned int max_items, unsigned int tag);
unsigned int radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *,
void __rcu ***results, unsigned long first_index,
unsigned int max_items, unsigned int tag);
int radix_tree_tagged(const struct radix_tree_root *, unsigned int tag);
static inline void radix_tree_preload_end(void)
{
local_unlock(&radix_tree_preloads.lock);
}
void __rcu **idr_get_free(struct radix_tree_root *root,
struct radix_tree_iter *iter, gfp_t gfp,
unsigned long max);
enum {
RADIX_TREE_ITER_TAG_MASK = 0x0f, /* tag index in lower nybble */
RADIX_TREE_ITER_TAGGED = 0x10, /* lookup tagged slots */
RADIX_TREE_ITER_CONTIG = 0x20, /* stop at first hole */
};
/**
* radix_tree_iter_init - initialize radix tree iterator
*
* @iter: pointer to iterator state
* @start: iteration starting index
* Returns: NULL
*/
static __always_inline void __rcu **
radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start)
{
/*
* Leave iter->tags uninitialized. radix_tree_next_chunk() will fill it
* in the case of a successful tagged chunk lookup. If the lookup was
* unsuccessful or non-tagged then nobody cares about ->tags.
*
* Set index to zero to bypass next_index overflow protection.
* See the comment in radix_tree_next_chunk() for details.
*/
iter->index = 0;
iter->next_index = start;
return NULL;
}
/**
* radix_tree_next_chunk - find next chunk of slots for iteration
*
* @root: radix tree root
* @iter: iterator state
* @flags: RADIX_TREE_ITER_* flags and tag index
* Returns: pointer to chunk first slot, or NULL if there no more left
*
* This function looks up the next chunk in the radix tree starting from
* @iter->next_index. It returns a pointer to the chunk's first slot.
* Also it fills @iter with data about chunk: position in the tree (index),
* its end (next_index), and constructs a bit mask for tagged iterating (tags).
*/
void __rcu **radix_tree_next_chunk(const struct radix_tree_root *,
struct radix_tree_iter *iter, unsigned flags);
/**
* radix_tree_iter_lookup - look up an index in the radix tree
* @root: radix tree root
* @iter: iterator state
* @index: key to look up
*
* If @index is present in the radix tree, this function returns the slot
* containing it and updates @iter to describe the entry. If @index is not
* present, it returns NULL.
*/
static inline void __rcu **
radix_tree_iter_lookup(const struct radix_tree_root *root,
struct radix_tree_iter *iter, unsigned long index)
{
radix_tree_iter_init(iter, index);
return radix_tree_next_chunk(root, iter, RADIX_TREE_ITER_CONTIG);
}
/**
* radix_tree_iter_retry - retry this chunk of the iteration
* @iter: iterator state
*
* If we iterate over a tree protected only by the RCU lock, a race
* against deletion or creation may result in seeing a slot for which
* radix_tree_deref_retry() returns true. If so, call this function
* and continue the iteration.
*/
static inline __must_check
void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter)
{
iter->next_index = iter->index;
iter->tags = 0;
return NULL;
}
static inline unsigned long
__radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots)
{
return iter->index + slots;
}
/**
* radix_tree_iter_resume - resume iterating when the chunk may be invalid
* @slot: pointer to current slot
* @iter: iterator state
* Returns: New slot pointer
*
* If the iterator needs to release then reacquire a lock, the chunk may
* have been invalidated by an insertion or deletion. Call this function
* before releasing the lock to continue the iteration from the next index.
*/
void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot,
struct radix_tree_iter *iter);
/**
* radix_tree_chunk_size - get current chunk size
*
* @iter: pointer to radix tree iterator
* Returns: current chunk size
*/
static __always_inline long
radix_tree_chunk_size(struct radix_tree_iter *iter)
{
return iter->next_index - iter->index;
}
/**
* radix_tree_next_slot - find next slot in chunk
*
* @slot: pointer to current slot
* @iter: pointer to iterator state
* @flags: RADIX_TREE_ITER_*, should be constant
* Returns: pointer to next slot, or NULL if there no more left
*
* This function updates @iter->index in the case of a successful lookup.
* For tagged lookup it also eats @iter->tags.
*
* There are several cases where 'slot' can be passed in as NULL to this
* function. These cases result from the use of radix_tree_iter_resume() or
* radix_tree_iter_retry(). In these cases we don't end up dereferencing
* 'slot' because either:
* a) we are doing tagged iteration and iter->tags has been set to 0, or
* b) we are doing non-tagged iteration, and iter->index and iter->next_index
* have been set up so that radix_tree_chunk_size() returns 1 or 0.
*/
static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
struct radix_tree_iter *iter, unsigned flags)
{
if (flags & RADIX_TREE_ITER_TAGGED) {
iter->tags >>= 1;
if (unlikely(!iter->tags))
return NULL;
if (likely(iter->tags & 1ul)) {
iter->index = __radix_tree_iter_add(iter, 1);
slot++;
goto found;
}
if (!(flags & RADIX_TREE_ITER_CONTIG)) {
unsigned offset = __ffs(iter->tags);
iter->tags >>= offset++;
iter->index = __radix_tree_iter_add(iter, offset);
slot += offset;
goto found;
}
} else {
long count = radix_tree_chunk_size(iter); while (--count > 0) { slot++;
iter->index = __radix_tree_iter_add(iter, 1);
if (likely(*slot))
goto found;
if (flags & RADIX_TREE_ITER_CONTIG) {
/* forbid switching to the next chunk */
iter->next_index = 0;
break;
}
}
}
return NULL;
found:
return slot;
}
/**
* radix_tree_for_each_slot - iterate over non-empty slots
*
* @slot: the void** variable for pointer to slot
* @root: the struct radix_tree_root pointer
* @iter: the struct radix_tree_iter pointer
* @start: iteration starting index
*
* @slot points to radix tree slot, @iter->index contains its index.
*/
#define radix_tree_for_each_slot(slot, root, iter, start) \
for (slot = radix_tree_iter_init(iter, start) ; \
slot || (slot = radix_tree_next_chunk(root, iter, 0)) ; \
slot = radix_tree_next_slot(slot, iter, 0))
/**
* radix_tree_for_each_tagged - iterate over tagged slots
*
* @slot: the void** variable for pointer to slot
* @root: the struct radix_tree_root pointer
* @iter: the struct radix_tree_iter pointer
* @start: iteration starting index
* @tag: tag index
*
* @slot points to radix tree slot, @iter->index contains its index.
*/
#define radix_tree_for_each_tagged(slot, root, iter, start, tag) \
for (slot = radix_tree_iter_init(iter, start) ; \
slot || (slot = radix_tree_next_chunk(root, iter, \
RADIX_TREE_ITER_TAGGED | tag)) ; \
slot = radix_tree_next_slot(slot, iter, \
RADIX_TREE_ITER_TAGGED | tag))
#endif /* _LINUX_RADIX_TREE_H */
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef LLIST_H
#define LLIST_H
/*
* Lock-less NULL terminated single linked list
*
* Cases where locking is not needed:
* If there are multiple producers and multiple consumers, llist_add can be
* used in producers and llist_del_all can be used in consumers simultaneously
* without locking. Also a single consumer can use llist_del_first while
* multiple producers simultaneously use llist_add, without any locking.
*
* Cases where locking is needed:
* If we have multiple consumers with llist_del_first used in one consumer, and
* llist_del_first or llist_del_all used in other consumers, then a lock is
* needed. This is because llist_del_first depends on list->first->next not
* changing, but without lock protection, there's no way to be sure about that
* if a preemption happens in the middle of the delete operation and on being
* preempted back, the list->first is the same as before causing the cmpxchg in
* llist_del_first to succeed. For example, while a llist_del_first operation
* is in progress in one consumer, then a llist_del_first, llist_add,
* llist_add (or llist_del_all, llist_add, llist_add) sequence in another
* consumer may cause violations.
*
* This can be summarized as follows:
*
* | add | del_first | del_all
* add | - | - | -
* del_first | | L | L
* del_all | | | -
*
* Where, a particular row's operation can happen concurrently with a column's
* operation, with "-" being no lock needed, while "L" being lock is needed.
*
* The list entries deleted via llist_del_all can be traversed with
* traversing function such as llist_for_each etc. But the list
* entries can not be traversed safely before deleted from the list.
* The order of deleted entries is from the newest to the oldest added
* one. If you want to traverse from the oldest to the newest, you
* must reverse the order by yourself before traversing.
*
* The basic atomic operation of this list is cmpxchg on long. On
* architectures that don't have NMI-safe cmpxchg implementation, the
* list can NOT be used in NMI handlers. So code that uses the list in
* an NMI handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
*
* Copyright 2010,2011 Intel Corp.
* Author: Huang Ying <ying.huang@intel.com>
*/
#include <linux/atomic.h>
#include <linux/container_of.h>
#include <linux/stddef.h>
#include <linux/types.h>
struct llist_head {
struct llist_node *first;
};
struct llist_node {
struct llist_node *next;
};
#define LLIST_HEAD_INIT(name) { NULL }
#define LLIST_HEAD(name) struct llist_head name = LLIST_HEAD_INIT(name)
/**
* init_llist_head - initialize lock-less list head
* @head: the head for your lock-less list
*/
static inline void init_llist_head(struct llist_head *list)
{
list->first = NULL;
}
/**
* init_llist_node - initialize lock-less list node
* @node: the node to be initialised
*
* In cases where there is a need to test if a node is on
* a list or not, this initialises the node to clearly
* not be on any list.
*/
static inline void init_llist_node(struct llist_node *node)
{
node->next = node;
}
/**
* llist_on_list - test if a lock-list list node is on a list
* @node: the node to test
*
* When a node is on a list the ->next pointer will be NULL or
* some other node. It can never point to itself. We use that
* in init_llist_node() to record that a node is not on any list,
* and here to test whether it is on any list.
*/
static inline bool llist_on_list(const struct llist_node *node)
{
return node->next != node;
}
/**
* llist_entry - get the struct of this entry
* @ptr: the &struct llist_node pointer.
* @type: the type of the struct this is embedded in.
* @member: the name of the llist_node within the struct.
*/
#define llist_entry(ptr, type, member) \
container_of(ptr, type, member)
/**
* member_address_is_nonnull - check whether the member address is not NULL
* @ptr: the object pointer (struct type * that contains the llist_node)
* @member: the name of the llist_node within the struct.
*
* This macro is conceptually the same as
* &ptr->member != NULL
* but it works around the fact that compilers can decide that taking a member
* address is never a NULL pointer.
*
* Real objects that start at a high address and have a member at NULL are
* unlikely to exist, but such pointers may be returned e.g. by the
* container_of() macro.
*/
#define member_address_is_nonnull(ptr, member) \
((uintptr_t)(ptr) + offsetof(typeof(*(ptr)), member) != 0)
/**
* llist_for_each - iterate over some deleted entries of a lock-less list
* @pos: the &struct llist_node to use as a loop cursor
* @node: the first entry of deleted list entries
*
* In general, some entries of the lock-less list can be traversed
* safely only after being deleted from list, so start with an entry
* instead of list head.
*
* If being used on entries deleted from lock-less list directly, the
* traverse order is from the newest to the oldest added entry. If
* you want to traverse from the oldest to the newest, you must
* reverse the order by yourself before traversing.
*/
#define llist_for_each(pos, node) \
for ((pos) = (node); pos; (pos) = (pos)->next)
/**
* llist_for_each_safe - iterate over some deleted entries of a lock-less list
* safe against removal of list entry
* @pos: the &struct llist_node to use as a loop cursor
* @n: another &struct llist_node to use as temporary storage
* @node: the first entry of deleted list entries
*
* In general, some entries of the lock-less list can be traversed
* safely only after being deleted from list, so start with an entry
* instead of list head.
*
* If being used on entries deleted from lock-less list directly, the
* traverse order is from the newest to the oldest added entry. If
* you want to traverse from the oldest to the newest, you must
* reverse the order by yourself before traversing.
*/
#define llist_for_each_safe(pos, n, node) \
for ((pos) = (node); (pos) && ((n) = (pos)->next, true); (pos) = (n))
/**
* llist_for_each_entry - iterate over some deleted entries of lock-less list of given type
* @pos: the type * to use as a loop cursor.
* @node: the fist entry of deleted list entries.
* @member: the name of the llist_node with the struct.
*
* In general, some entries of the lock-less list can be traversed
* safely only after being removed from list, so start with an entry
* instead of list head.
*
* If being used on entries deleted from lock-less list directly, the
* traverse order is from the newest to the oldest added entry. If
* you want to traverse from the oldest to the newest, you must
* reverse the order by yourself before traversing.
*/
#define llist_for_each_entry(pos, node, member) \
for ((pos) = llist_entry((node), typeof(*(pos)), member); \
member_address_is_nonnull(pos, member); \
(pos) = llist_entry((pos)->member.next, typeof(*(pos)), member))
/**
* llist_for_each_entry_safe - iterate over some deleted entries of lock-less list of given type
* safe against removal of list entry
* @pos: the type * to use as a loop cursor.
* @n: another type * to use as temporary storage
* @node: the first entry of deleted list entries.
* @member: the name of the llist_node with the struct.
*
* In general, some entries of the lock-less list can be traversed
* safely only after being removed from list, so start with an entry
* instead of list head.
*
* If being used on entries deleted from lock-less list directly, the
* traverse order is from the newest to the oldest added entry. If
* you want to traverse from the oldest to the newest, you must
* reverse the order by yourself before traversing.
*/
#define llist_for_each_entry_safe(pos, n, node, member) \
for (pos = llist_entry((node), typeof(*pos), member); \
member_address_is_nonnull(pos, member) && \
(n = llist_entry(pos->member.next, typeof(*n), member), true); \
pos = n)
/**
* llist_empty - tests whether a lock-less list is empty
* @head: the list to test
*
* Not guaranteed to be accurate or up to date. Just a quick way to
* test whether the list is empty without deleting something from the
* list.
*/
static inline bool llist_empty(const struct llist_head *head)
{
return READ_ONCE(head->first) == NULL;
}
static inline struct llist_node *llist_next(struct llist_node *node)
{
return node->next;
}
extern bool llist_add_batch(struct llist_node *new_first,
struct llist_node *new_last,
struct llist_head *head);
static inline bool __llist_add_batch(struct llist_node *new_first,
struct llist_node *new_last,
struct llist_head *head)
{
new_last->next = head->first;
head->first = new_first;
return new_last->next == NULL;
}
/**
* llist_add - add a new entry
* @new: new entry to be added
* @head: the head for your lock-less list
*
* Returns true if the list was empty prior to adding this entry.
*/
static inline bool llist_add(struct llist_node *new, struct llist_head *head)
{
return llist_add_batch(new, new, head);
}
static inline bool __llist_add(struct llist_node *new, struct llist_head *head)
{
return __llist_add_batch(new, new, head);
}
/**
* llist_del_all - delete all entries from lock-less list
* @head: the head of lock-less list to delete all entries
*
* If list is empty, return NULL, otherwise, delete all entries and
* return the pointer to the first entry. The order of entries
* deleted is from the newest to the oldest added one.
*/
static inline struct llist_node *llist_del_all(struct llist_head *head)
{
return xchg(&head->first, NULL);
}
static inline struct llist_node *__llist_del_all(struct llist_head *head)
{
struct llist_node *first = head->first;
head->first = NULL;
return first;
}
extern struct llist_node *llist_del_first(struct llist_head *head);
/**
* llist_del_first_init - delete first entry from lock-list and mark is as being off-list
* @head: the head of lock-less list to delete from.
*
* This behave the same as llist_del_first() except that llist_init_node() is called
* on the returned node so that llist_on_list() will report false for the node.
*/
static inline struct llist_node *llist_del_first_init(struct llist_head *head)
{
struct llist_node *n = llist_del_first(head);
if (n)
init_llist_node(n);
return n;
}
extern bool llist_del_first_this(struct llist_head *head,
struct llist_node *this);
struct llist_node *llist_reverse_order(struct llist_node *head);
#endif /* LLIST_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/fs/file_table.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
*/
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/security.h>
#include <linux/cred.h>
#include <linux/eventpoll.h>
#include <linux/rcupdate.h>
#include <linux/mount.h>
#include <linux/capability.h>
#include <linux/cdev.h>
#include <linux/fsnotify.h>
#include <linux/sysctl.h>
#include <linux/percpu_counter.h>
#include <linux/percpu.h>
#include <linux/task_work.h>
#include <linux/swap.h>
#include <linux/kmemleak.h>
#include <linux/atomic.h>
#include "internal.h"
/* sysctl tunables... */
static struct files_stat_struct files_stat = {
.max_files = NR_FILE
};
/* SLAB cache for file structures */
static struct kmem_cache *filp_cachep __ro_after_init;
static struct percpu_counter nr_files __cacheline_aligned_in_smp;
/* Container for backing file with optional user path */
struct backing_file {
struct file file;
struct path user_path;
};
static inline struct backing_file *backing_file(struct file *f)
{
return container_of(f, struct backing_file, file);
}
struct path *backing_file_user_path(struct file *f)
{
return &backing_file(f)->user_path;
}
EXPORT_SYMBOL_GPL(backing_file_user_path);
static inline void file_free(struct file *f)
{
security_file_free(f);
if (likely(!(f->f_mode & FMODE_NOACCOUNT)))
percpu_counter_dec(&nr_files); put_cred(f->f_cred); if (unlikely(f->f_mode & FMODE_BACKING)) { path_put(backing_file_user_path(f));
kfree(backing_file(f));
} else {
kmem_cache_free(filp_cachep, f);
}
}
/*
* Return the total number of open files in the system
*/
static long get_nr_files(void)
{
return percpu_counter_read_positive(&nr_files);
}
/*
* Return the maximum number of open files in the system
*/
unsigned long get_max_files(void)
{
return files_stat.max_files;
}
EXPORT_SYMBOL_GPL(get_max_files);
#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
/*
* Handle nr_files sysctl
*/
static int proc_nr_files(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
files_stat.nr_files = get_nr_files();
return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
}
static struct ctl_table fs_stat_sysctls[] = {
{
.procname = "file-nr",
.data = &files_stat,
.maxlen = sizeof(files_stat),
.mode = 0444,
.proc_handler = proc_nr_files,
},
{
.procname = "file-max",
.data = &files_stat.max_files,
.maxlen = sizeof(files_stat.max_files),
.mode = 0644,
.proc_handler = proc_doulongvec_minmax,
.extra1 = SYSCTL_LONG_ZERO,
.extra2 = SYSCTL_LONG_MAX,
},
{
.procname = "nr_open",
.data = &sysctl_nr_open,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = &sysctl_nr_open_min,
.extra2 = &sysctl_nr_open_max,
},
};
static int __init init_fs_stat_sysctls(void)
{
register_sysctl_init("fs", fs_stat_sysctls);
if (IS_ENABLED(CONFIG_BINFMT_MISC)) {
struct ctl_table_header *hdr;
hdr = register_sysctl_mount_point("fs/binfmt_misc");
kmemleak_not_leak(hdr);
}
return 0;
}
fs_initcall(init_fs_stat_sysctls);
#endif
static int init_file(struct file *f, int flags, const struct cred *cred)
{
int error;
f->f_cred = get_cred(cred);
error = security_file_alloc(f);
if (unlikely(error)) {
put_cred(f->f_cred);
return error;
}
rwlock_init(&f->f_owner.lock);
spin_lock_init(&f->f_lock);
mutex_init(&f->f_pos_lock);
f->f_flags = flags;
f->f_mode = OPEN_FMODE(flags);
/* f->f_version: 0 */
/*
* We're SLAB_TYPESAFE_BY_RCU so initialize f_count last. While
* fget-rcu pattern users need to be able to handle spurious
* refcount bumps we should reinitialize the reused file first.
*/
atomic_long_set(&f->f_count, 1);
return 0;
}
/* Find an unused file structure and return a pointer to it.
* Returns an error pointer if some error happend e.g. we over file
* structures limit, run out of memory or operation is not permitted.
*
* Be very careful using this. You are responsible for
* getting write access to any mount that you might assign
* to this filp, if it is opened for write. If this is not
* done, you will imbalance int the mount's writer count
* and a warning at __fput() time.
*/
struct file *alloc_empty_file(int flags, const struct cred *cred)
{
static long old_max;
struct file *f;
int error;
/*
* Privileged users can go above max_files
*/
if (get_nr_files() >= files_stat.max_files && !capable(CAP_SYS_ADMIN)) {
/*
* percpu_counters are inaccurate. Do an expensive check before
* we go and fail.
*/
if (percpu_counter_sum_positive(&nr_files) >= files_stat.max_files)
goto over;
}
f = kmem_cache_zalloc(filp_cachep, GFP_KERNEL);
if (unlikely(!f))
return ERR_PTR(-ENOMEM);
error = init_file(f, flags, cred);
if (unlikely(error)) {
kmem_cache_free(filp_cachep, f);
return ERR_PTR(error);
}
percpu_counter_inc(&nr_files); return f;
over:
/* Ran out of filps - report that */
if (get_nr_files() > old_max) { pr_info("VFS: file-max limit %lu reached\n", get_max_files());
old_max = get_nr_files();
}
return ERR_PTR(-ENFILE);
}
/*
* Variant of alloc_empty_file() that doesn't check and modify nr_files.
*
* This is only for kernel internal use, and the allocate file must not be
* installed into file tables or such.
*/
struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred)
{
struct file *f;
int error;
f = kmem_cache_zalloc(filp_cachep, GFP_KERNEL);
if (unlikely(!f))
return ERR_PTR(-ENOMEM);
error = init_file(f, flags, cred);
if (unlikely(error)) {
kmem_cache_free(filp_cachep, f);
return ERR_PTR(error);
}
f->f_mode |= FMODE_NOACCOUNT;
return f;
}
/*
* Variant of alloc_empty_file() that allocates a backing_file container
* and doesn't check and modify nr_files.
*
* This is only for kernel internal use, and the allocate file must not be
* installed into file tables or such.
*/
struct file *alloc_empty_backing_file(int flags, const struct cred *cred)
{
struct backing_file *ff;
int error;
ff = kzalloc(sizeof(struct backing_file), GFP_KERNEL);
if (unlikely(!ff))
return ERR_PTR(-ENOMEM);
error = init_file(&ff->file, flags, cred);
if (unlikely(error)) {
kfree(ff);
return ERR_PTR(error);
}
ff->file.f_mode |= FMODE_BACKING | FMODE_NOACCOUNT;
return &ff->file;
}
/**
* file_init_path - initialize a 'struct file' based on path
*
* @file: the file to set up
* @path: the (dentry, vfsmount) pair for the new file
* @fop: the 'struct file_operations' for the new file
*/
static void file_init_path(struct file *file, const struct path *path,
const struct file_operations *fop)
{
file->f_path = *path;
file->f_inode = path->dentry->d_inode;
file->f_mapping = path->dentry->d_inode->i_mapping;
file->f_wb_err = filemap_sample_wb_err(file->f_mapping);
file->f_sb_err = file_sample_sb_err(file);
if (fop->llseek)
file->f_mode |= FMODE_LSEEK;
if ((file->f_mode & FMODE_READ) &&
likely(fop->read || fop->read_iter))
file->f_mode |= FMODE_CAN_READ;
if ((file->f_mode & FMODE_WRITE) &&
likely(fop->write || fop->write_iter))
file->f_mode |= FMODE_CAN_WRITE;
file->f_iocb_flags = iocb_flags(file);
file->f_mode |= FMODE_OPENED;
file->f_op = fop;
if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ)
i_readcount_inc(path->dentry->d_inode);
}
/**
* alloc_file - allocate and initialize a 'struct file'
*
* @path: the (dentry, vfsmount) pair for the new file
* @flags: O_... flags with which the new file will be opened
* @fop: the 'struct file_operations' for the new file
*/
static struct file *alloc_file(const struct path *path, int flags,
const struct file_operations *fop)
{
struct file *file;
file = alloc_empty_file(flags, current_cred());
if (!IS_ERR(file))
file_init_path(file, path, fop);
return file;
}
static inline int alloc_path_pseudo(const char *name, struct inode *inode,
struct vfsmount *mnt, struct path *path)
{
struct qstr this = QSTR_INIT(name, strlen(name));
path->dentry = d_alloc_pseudo(mnt->mnt_sb, &this);
if (!path->dentry)
return -ENOMEM;
path->mnt = mntget(mnt);
d_instantiate(path->dentry, inode);
return 0;
}
struct file *alloc_file_pseudo(struct inode *inode, struct vfsmount *mnt,
const char *name, int flags,
const struct file_operations *fops)
{
int ret;
struct path path;
struct file *file;
ret = alloc_path_pseudo(name, inode, mnt, &path);
if (ret)
return ERR_PTR(ret);
file = alloc_file(&path, flags, fops);
if (IS_ERR(file)) {
ihold(inode);
path_put(&path);
}
return file;
}
EXPORT_SYMBOL(alloc_file_pseudo);
struct file *alloc_file_pseudo_noaccount(struct inode *inode,
struct vfsmount *mnt, const char *name,
int flags,
const struct file_operations *fops)
{
int ret;
struct path path;
struct file *file;
ret = alloc_path_pseudo(name, inode, mnt, &path);
if (ret)
return ERR_PTR(ret);
file = alloc_empty_file_noaccount(flags, current_cred());
if (IS_ERR(file)) {
ihold(inode);
path_put(&path);
return file;
}
file_init_path(file, &path, fops);
return file;
}
EXPORT_SYMBOL_GPL(alloc_file_pseudo_noaccount);
struct file *alloc_file_clone(struct file *base, int flags,
const struct file_operations *fops)
{
struct file *f = alloc_file(&base->f_path, flags, fops);
if (!IS_ERR(f)) {
path_get(&f->f_path);
f->f_mapping = base->f_mapping;
}
return f;
}
/* the real guts of fput() - releasing the last reference to file
*/
static void __fput(struct file *file)
{
struct dentry *dentry = file->f_path.dentry;
struct vfsmount *mnt = file->f_path.mnt;
struct inode *inode = file->f_inode;
fmode_t mode = file->f_mode;
if (unlikely(!(file->f_mode & FMODE_OPENED)))
goto out;
might_sleep(); fsnotify_close(file);
/*
* The function eventpoll_release() should be the first called
* in the file cleanup chain.
*/
eventpoll_release(file); locks_remove_file(file);
security_file_release(file);
if (unlikely(file->f_flags & FASYNC)) {
if (file->f_op->fasync) file->f_op->fasync(-1, file, 0);
}
if (file->f_op->release) file->f_op->release(inode, file); if (unlikely(S_ISCHR(inode->i_mode) && inode->i_cdev != NULL &&
!(mode & FMODE_PATH))) {
cdev_put(inode->i_cdev);
}
fops_put(file->f_op); put_pid(file->f_owner.pid); put_file_access(file); dput(dentry);
if (unlikely(mode & FMODE_NEED_UNMOUNT))
dissolve_on_fput(mnt); mntput(mnt);
out:
file_free(file);
}
static LLIST_HEAD(delayed_fput_list);
static void delayed_fput(struct work_struct *unused)
{
struct llist_node *node = llist_del_all(&delayed_fput_list);
struct file *f, *t;
llist_for_each_entry_safe(f, t, node, f_llist)
__fput(f);
}
static void ____fput(struct callback_head *work)
{
__fput(container_of(work, struct file, f_task_work));
}
/*
* If kernel thread really needs to have the final fput() it has done
* to complete, call this. The only user right now is the boot - we
* *do* need to make sure our writes to binaries on initramfs has
* not left us with opened struct file waiting for __fput() - execve()
* won't work without that. Please, don't add more callers without
* very good reasons; in particular, never call that with locks
* held and never call that from a thread that might need to do
* some work on any kind of umount.
*/
void flush_delayed_fput(void)
{
delayed_fput(NULL);
}
EXPORT_SYMBOL_GPL(flush_delayed_fput);
static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput);
void fput(struct file *file)
{
if (atomic_long_dec_and_test(&file->f_count)) { struct task_struct *task = current;
if (unlikely(!(file->f_mode & (FMODE_BACKING | FMODE_OPENED)))) {
file_free(file);
return;
}
if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) { init_task_work(&file->f_task_work, ____fput);
if (!task_work_add(task, &file->f_task_work, TWA_RESUME))
return;
/*
* After this task has run exit_task_work(),
* task_work_add() will fail. Fall through to delayed
* fput to avoid leaking *file.
*/
}
if (llist_add(&file->f_llist, &delayed_fput_list)) schedule_delayed_work(&delayed_fput_work, 1);
}
}
/*
* synchronous analog of fput(); for kernel threads that might be needed
* in some umount() (and thus can't use flush_delayed_fput() without
* risking deadlocks), need to wait for completion of __fput() and know
* for this specific struct file it won't involve anything that would
* need them. Use only if you really need it - at the very least,
* don't blindly convert fput() by kernel thread to that.
*/
void __fput_sync(struct file *file)
{
if (atomic_long_dec_and_test(&file->f_count)) __fput(file);
}
EXPORT_SYMBOL(fput);
EXPORT_SYMBOL(__fput_sync);
void __init files_init(void)
{
filp_cachep = kmem_cache_create("filp", sizeof(struct file), 0,
SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN |
SLAB_PANIC | SLAB_ACCOUNT, NULL);
percpu_counter_init(&nr_files, 0, GFP_KERNEL);
}
/*
* One file with associated inode and dcache is very roughly 1K. Per default
* do not use more than 10% of our memory for files.
*/
void __init files_maxfiles_init(void)
{
unsigned long n;
unsigned long nr_pages = totalram_pages();
unsigned long memreserve = (nr_pages - nr_free_pages()) * 3/2;
memreserve = min(memreserve, nr_pages - 1);
n = ((nr_pages - memreserve) * (PAGE_SIZE / 1024)) / 10;
files_stat.max_files = max_t(unsigned long, n, NR_FILE);
}
// SPDX-License-Identifier: GPL-2.0
/*
* attribute_container.c - implementation of a simple container for classes
*
* Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com>
*
* The basic idea here is to enable a device to be attached to an
* aritrary numer of classes without having to allocate storage for them.
* Instead, the contained classes select the devices they need to attach
* to via a matching function.
*/
#include <linux/attribute_container.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include "base.h"
/* This is a private structure used to tie the classdev and the
* container .. it should never be visible outside this file */
struct internal_container {
struct klist_node node;
struct attribute_container *cont;
struct device classdev;
};
static void internal_container_klist_get(struct klist_node *n)
{
struct internal_container *ic =
container_of(n, struct internal_container, node);
get_device(&ic->classdev);
}
static void internal_container_klist_put(struct klist_node *n)
{
struct internal_container *ic =
container_of(n, struct internal_container, node);
put_device(&ic->classdev);
}
/**
* attribute_container_classdev_to_container - given a classdev, return the container
*
* @classdev: the class device created by attribute_container_add_device.
*
* Returns the container associated with this classdev.
*/
struct attribute_container *
attribute_container_classdev_to_container(struct device *classdev)
{
struct internal_container *ic =
container_of(classdev, struct internal_container, classdev);
return ic->cont;
}
EXPORT_SYMBOL_GPL(attribute_container_classdev_to_container);
static LIST_HEAD(attribute_container_list);
static DEFINE_MUTEX(attribute_container_mutex);
/**
* attribute_container_register - register an attribute container
*
* @cont: The container to register. This must be allocated by the
* callee and should also be zeroed by it.
*/
int
attribute_container_register(struct attribute_container *cont)
{
INIT_LIST_HEAD(&cont->node);
klist_init(&cont->containers, internal_container_klist_get,
internal_container_klist_put);
mutex_lock(&attribute_container_mutex);
list_add_tail(&cont->node, &attribute_container_list);
mutex_unlock(&attribute_container_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(attribute_container_register);
/**
* attribute_container_unregister - remove a container registration
*
* @cont: previously registered container to remove
*/
int
attribute_container_unregister(struct attribute_container *cont)
{
int retval = -EBUSY;
mutex_lock(&attribute_container_mutex);
spin_lock(&cont->containers.k_lock);
if (!list_empty(&cont->containers.k_list))
goto out;
retval = 0;
list_del(&cont->node);
out:
spin_unlock(&cont->containers.k_lock);
mutex_unlock(&attribute_container_mutex);
return retval;
}
EXPORT_SYMBOL_GPL(attribute_container_unregister);
/* private function used as class release */
static void attribute_container_release(struct device *classdev)
{
struct internal_container *ic
= container_of(classdev, struct internal_container, classdev);
struct device *dev = classdev->parent;
kfree(ic);
put_device(dev);
}
/**
* attribute_container_add_device - see if any container is interested in dev
*
* @dev: device to add attributes to
* @fn: function to trigger addition of class device.
*
* This function allocates storage for the class device(s) to be
* attached to dev (one for each matching attribute_container). If no
* fn is provided, the code will simply register the class device via
* device_add. If a function is provided, it is expected to add
* the class device at the appropriate time. One of the things that
* might be necessary is to allocate and initialise the classdev and
* then add it a later time. To do this, call this routine for
* allocation and initialisation and then use
* attribute_container_device_trigger() to call device_add() on
* it. Note: after this, the class device contains a reference to dev
* which is not relinquished until the release of the classdev.
*/
void
attribute_container_add_device(struct device *dev,
int (*fn)(struct attribute_container *,
struct device *,
struct device *))
{
struct attribute_container *cont;
mutex_lock(&attribute_container_mutex);
list_for_each_entry(cont, &attribute_container_list, node) {
struct internal_container *ic;
if (attribute_container_no_classdevs(cont))
continue;
if (!cont->match(cont, dev))
continue;
ic = kzalloc(sizeof(*ic), GFP_KERNEL);
if (!ic) {
dev_err(dev, "failed to allocate class container\n");
continue;
}
ic->cont = cont;
device_initialize(&ic->classdev);
ic->classdev.parent = get_device(dev);
ic->classdev.class = cont->class;
cont->class->dev_release = attribute_container_release;
dev_set_name(&ic->classdev, "%s", dev_name(dev));
if (fn)
fn(cont, dev, &ic->classdev);
else
attribute_container_add_class_device(&ic->classdev);
klist_add_tail(&ic->node, &cont->containers);
}
mutex_unlock(&attribute_container_mutex);
}
/* FIXME: can't break out of this unless klist_iter_exit is also
* called before doing the break
*/
#define klist_for_each_entry(pos, head, member, iter) \
for (klist_iter_init(head, iter); (pos = ({ \
struct klist_node *n = klist_next(iter); \
n ? container_of(n, typeof(*pos), member) : \
({ klist_iter_exit(iter) ; NULL; }); \
})) != NULL;)
/**
* attribute_container_remove_device - make device eligible for removal.
*
* @dev: The generic device
* @fn: A function to call to remove the device
*
* This routine triggers device removal. If fn is NULL, then it is
* simply done via device_unregister (note that if something
* still has a reference to the classdev, then the memory occupied
* will not be freed until the classdev is released). If you want a
* two phase release: remove from visibility and then delete the
* device, then you should use this routine with a fn that calls
* device_del() and then use attribute_container_device_trigger()
* to do the final put on the classdev.
*/
void
attribute_container_remove_device(struct device *dev,
void (*fn)(struct attribute_container *,
struct device *,
struct device *))
{
struct attribute_container *cont;
mutex_lock(&attribute_container_mutex); list_for_each_entry(cont, &attribute_container_list, node) {
struct internal_container *ic;
struct klist_iter iter;
if (attribute_container_no_classdevs(cont))
continue;
if (!cont->match(cont, dev))
continue;
klist_for_each_entry(ic, &cont->containers, node, &iter) { if (dev != ic->classdev.parent)
continue;
klist_del(&ic->node);
if (fn)
fn(cont, dev, &ic->classdev);
else {
attribute_container_remove_attrs(&ic->classdev);
device_unregister(&ic->classdev);
}
}
}
mutex_unlock(&attribute_container_mutex);
}
static int
do_attribute_container_device_trigger_safe(struct device *dev,
struct attribute_container *cont,
int (*fn)(struct attribute_container *,
struct device *, struct device *),
int (*undo)(struct attribute_container *,
struct device *, struct device *))
{
int ret;
struct internal_container *ic, *failed;
struct klist_iter iter;
if (attribute_container_no_classdevs(cont))
return fn(cont, dev, NULL);
klist_for_each_entry(ic, &cont->containers, node, &iter) {
if (dev == ic->classdev.parent) {
ret = fn(cont, dev, &ic->classdev);
if (ret) {
failed = ic;
klist_iter_exit(&iter);
goto fail;
}
}
}
return 0;
fail:
if (!undo)
return ret;
/* Attempt to undo the work partially done. */
klist_for_each_entry(ic, &cont->containers, node, &iter) {
if (ic == failed) {
klist_iter_exit(&iter);
break;
}
if (dev == ic->classdev.parent)
undo(cont, dev, &ic->classdev);
}
return ret;
}
/**
* attribute_container_device_trigger_safe - execute a trigger for each
* matching classdev or fail all of them.
*
* @dev: The generic device to run the trigger for
* @fn: the function to execute for each classdev.
* @undo: A function to undo the work previously done in case of error
*
* This function is a safe version of
* attribute_container_device_trigger. It stops on the first error and
* undo the partial work that has been done, on previous classdev. It
* is guaranteed that either they all succeeded, or none of them
* succeeded.
*/
int
attribute_container_device_trigger_safe(struct device *dev,
int (*fn)(struct attribute_container *,
struct device *,
struct device *),
int (*undo)(struct attribute_container *,
struct device *,
struct device *))
{
struct attribute_container *cont, *failed = NULL;
int ret = 0;
mutex_lock(&attribute_container_mutex);
list_for_each_entry(cont, &attribute_container_list, node) {
if (!cont->match(cont, dev))
continue;
ret = do_attribute_container_device_trigger_safe(dev, cont,
fn, undo);
if (ret) {
failed = cont;
break;
}
}
if (ret && !WARN_ON(!undo)) {
list_for_each_entry(cont, &attribute_container_list, node) {
if (failed == cont)
break;
if (!cont->match(cont, dev))
continue;
do_attribute_container_device_trigger_safe(dev, cont,
undo, NULL);
}
}
mutex_unlock(&attribute_container_mutex);
return ret;
}
/**
* attribute_container_device_trigger - execute a trigger for each matching classdev
*
* @dev: The generic device to run the trigger for
* @fn: the function to execute for each classdev.
*
* This function is for executing a trigger when you need to know both
* the container and the classdev. If you only care about the
* container, then use attribute_container_trigger() instead.
*/
void
attribute_container_device_trigger(struct device *dev,
int (*fn)(struct attribute_container *,
struct device *,
struct device *))
{
struct attribute_container *cont;
mutex_lock(&attribute_container_mutex);
list_for_each_entry(cont, &attribute_container_list, node) {
struct internal_container *ic;
struct klist_iter iter;
if (!cont->match(cont, dev))
continue;
if (attribute_container_no_classdevs(cont)) { fn(cont, dev, NULL); continue;
}
klist_for_each_entry(ic, &cont->containers, node, &iter) { if (dev == ic->classdev.parent) fn(cont, dev, &ic->classdev);
}
}
mutex_unlock(&attribute_container_mutex);
}
/**
* attribute_container_trigger - trigger a function for each matching container
*
* @dev: The generic device to activate the trigger for
* @fn: the function to trigger
*
* This routine triggers a function that only needs to know the
* matching containers (not the classdev) associated with a device.
* It is more lightweight than attribute_container_device_trigger, so
* should be used in preference unless the triggering function
* actually needs to know the classdev.
*/
void
attribute_container_trigger(struct device *dev,
int (*fn)(struct attribute_container *,
struct device *))
{
struct attribute_container *cont;
mutex_lock(&attribute_container_mutex);
list_for_each_entry(cont, &attribute_container_list, node) {
if (cont->match(cont, dev))
fn(cont, dev);
}
mutex_unlock(&attribute_container_mutex);
}
/**
* attribute_container_add_attrs - add attributes
*
* @classdev: The class device
*
* This simply creates all the class device sysfs files from the
* attributes listed in the container
*/
int
attribute_container_add_attrs(struct device *classdev)
{
struct attribute_container *cont =
attribute_container_classdev_to_container(classdev);
struct device_attribute **attrs = cont->attrs;
int i, error;
BUG_ON(attrs && cont->grp);
if (!attrs && !cont->grp)
return 0;
if (cont->grp)
return sysfs_create_group(&classdev->kobj, cont->grp);
for (i = 0; attrs[i]; i++) {
sysfs_attr_init(&attrs[i]->attr);
error = device_create_file(classdev, attrs[i]);
if (error)
return error;
}
return 0;
}
/**
* attribute_container_add_class_device - same function as device_add
*
* @classdev: the class device to add
*
* This performs essentially the same function as device_add except for
* attribute containers, namely add the classdev to the system and then
* create the attribute files
*/
int
attribute_container_add_class_device(struct device *classdev)
{
int error = device_add(classdev);
if (error)
return error;
return attribute_container_add_attrs(classdev);
}
/**
* attribute_container_add_class_device_adapter - simple adapter for triggers
*
* @cont: the container to register.
* @dev: the generic device to activate the trigger for
* @classdev: the class device to add
*
* This function is identical to attribute_container_add_class_device except
* that it is designed to be called from the triggers
*/
int
attribute_container_add_class_device_adapter(struct attribute_container *cont,
struct device *dev,
struct device *classdev)
{
return attribute_container_add_class_device(classdev);
}
/**
* attribute_container_remove_attrs - remove any attribute files
*
* @classdev: The class device to remove the files from
*
*/
void
attribute_container_remove_attrs(struct device *classdev)
{
struct attribute_container *cont =
attribute_container_classdev_to_container(classdev);
struct device_attribute **attrs = cont->attrs;
int i;
if (!attrs && !cont->grp)
return;
if (cont->grp) {
sysfs_remove_group(&classdev->kobj, cont->grp);
return ;
}
for (i = 0; attrs[i]; i++)
device_remove_file(classdev, attrs[i]);
}
/**
* attribute_container_class_device_del - equivalent of class_device_del
*
* @classdev: the class device
*
* This function simply removes all the attribute files and then calls
* device_del.
*/
void
attribute_container_class_device_del(struct device *classdev)
{
attribute_container_remove_attrs(classdev);
device_del(classdev);
}
/**
* attribute_container_find_class_device - find the corresponding class_device
*
* @cont: the container
* @dev: the generic device
*
* Looks up the device in the container's list of class devices and returns
* the corresponding class_device.
*/
struct device *
attribute_container_find_class_device(struct attribute_container *cont,
struct device *dev)
{
struct device *cdev = NULL;
struct internal_container *ic;
struct klist_iter iter;
klist_for_each_entry(ic, &cont->containers, node, &iter) {
if (ic->classdev.parent == dev) {
cdev = &ic->classdev;
/* FIXME: must exit iterator then break */
klist_iter_exit(&iter);
break;
}
}
return cdev;
}
EXPORT_SYMBOL_GPL(attribute_container_find_class_device);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Derived from arch/ppc/mm/extable.c and arch/i386/mm/extable.c.
*
* Copyright (C) 2004 Paul Mackerras, IBM Corp.
*/
#include <linux/bsearch.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sort.h>
#include <linux/uaccess.h>
#include <linux/extable.h>
#ifndef ARCH_HAS_RELATIVE_EXTABLE
#define ex_to_insn(x) ((x)->insn)
#else
static inline unsigned long ex_to_insn(const struct exception_table_entry *x)
{
return (unsigned long)&x->insn + x->insn;
}
#endif
#ifndef ARCH_HAS_RELATIVE_EXTABLE
#define swap_ex NULL
#else
static void swap_ex(void *a, void *b, int size)
{
struct exception_table_entry *x = a, *y = b, tmp;
int delta = b - a;
tmp = *x;
x->insn = y->insn + delta;
y->insn = tmp.insn - delta;
#ifdef swap_ex_entry_fixup
swap_ex_entry_fixup(x, y, tmp, delta);
#else
x->fixup = y->fixup + delta;
y->fixup = tmp.fixup - delta;
#endif
}
#endif /* ARCH_HAS_RELATIVE_EXTABLE */
/*
* The exception table needs to be sorted so that the binary
* search that we use to find entries in it works properly.
* This is used both for the kernel exception table and for
* the exception tables of modules that get loaded.
*/
static int cmp_ex_sort(const void *a, const void *b)
{
const struct exception_table_entry *x = a, *y = b;
/* avoid overflow */
if (ex_to_insn(x) > ex_to_insn(y))
return 1;
if (ex_to_insn(x) < ex_to_insn(y))
return -1;
return 0;
}
void sort_extable(struct exception_table_entry *start,
struct exception_table_entry *finish)
{
sort(start, finish - start, sizeof(struct exception_table_entry),
cmp_ex_sort, swap_ex);
}
#ifdef CONFIG_MODULES
/*
* If the exception table is sorted, any referring to the module init
* will be at the beginning or the end.
*/
void trim_init_extable(struct module *m)
{
/*trim the beginning*/
while (m->num_exentries &&
within_module_init(ex_to_insn(&m->extable[0]), m)) {
m->extable++;
m->num_exentries--;
}
/*trim the end*/
while (m->num_exentries &&
within_module_init(ex_to_insn(&m->extable[m->num_exentries - 1]),
m))
m->num_exentries--;
}
#endif /* CONFIG_MODULES */
static int cmp_ex_search(const void *key, const void *elt)
{
const struct exception_table_entry *_elt = elt;
unsigned long _key = *(unsigned long *)key;
/* avoid overflow */
if (_key > ex_to_insn(_elt))
return 1;
if (_key < ex_to_insn(_elt))
return -1;
return 0;
}
/*
* Search one exception table for an entry corresponding to the
* given instruction address, and return the address of the entry,
* or NULL if none is found.
* We use a binary search, and thus we assume that the table is
* already sorted.
*/
const struct exception_table_entry *
search_extable(const struct exception_table_entry *base,
const size_t num,
unsigned long value)
{
return bsearch(&value, base, num,
sizeof(struct exception_table_entry), cmp_ex_search);
}
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1992, 1998-2004 Linus Torvalds, Ingo Molnar
*
* This file contains spurious interrupt handling.
*/
#include <linux/jiffies.h>
#include <linux/irq.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/moduleparam.h>
#include <linux/timer.h>
#include "internals.h"
static int irqfixup __read_mostly;
#define POLL_SPURIOUS_IRQ_INTERVAL (HZ/10)
static void poll_spurious_irqs(struct timer_list *unused);
static DEFINE_TIMER(poll_spurious_irq_timer, poll_spurious_irqs);
static int irq_poll_cpu;
static atomic_t irq_poll_active;
/*
* We wait here for a poller to finish.
*
* If the poll runs on this CPU, then we yell loudly and return
* false. That will leave the interrupt line disabled in the worst
* case, but it should never happen.
*
* We wait until the poller is done and then recheck disabled and
* action (about to be disabled). Only if it's still active, we return
* true and let the handler run.
*/
bool irq_wait_for_poll(struct irq_desc *desc)
__must_hold(&desc->lock)
{
if (WARN_ONCE(irq_poll_cpu == smp_processor_id(),
"irq poll in progress on cpu %d for irq %d\n",
smp_processor_id(), desc->irq_data.irq))
return false;
#ifdef CONFIG_SMP
do {
raw_spin_unlock(&desc->lock);
while (irqd_irq_inprogress(&desc->irq_data))
cpu_relax();
raw_spin_lock(&desc->lock);
} while (irqd_irq_inprogress(&desc->irq_data));
/* Might have been disabled in meantime */
return !irqd_irq_disabled(&desc->irq_data) && desc->action;
#else
return false;
#endif
}
/*
* Recovery handler for misrouted interrupts.
*/
static int try_one_irq(struct irq_desc *desc, bool force)
{
irqreturn_t ret = IRQ_NONE;
struct irqaction *action;
raw_spin_lock(&desc->lock);
/*
* PER_CPU, nested thread interrupts and interrupts explicitly
* marked polled are excluded from polling.
*/
if (irq_settings_is_per_cpu(desc) ||
irq_settings_is_nested_thread(desc) ||
irq_settings_is_polled(desc))
goto out;
/*
* Do not poll disabled interrupts unless the spurious
* disabled poller asks explicitly.
*/
if (irqd_irq_disabled(&desc->irq_data) && !force)
goto out;
/*
* All handlers must agree on IRQF_SHARED, so we test just the
* first.
*/
action = desc->action;
if (!action || !(action->flags & IRQF_SHARED) ||
(action->flags & __IRQF_TIMER))
goto out;
/* Already running on another processor */
if (irqd_irq_inprogress(&desc->irq_data)) {
/*
* Already running: If it is shared get the other
* CPU to go looking for our mystery interrupt too
*/
desc->istate |= IRQS_PENDING;
goto out;
}
/* Mark it poll in progress */
desc->istate |= IRQS_POLL_INPROGRESS;
do {
if (handle_irq_event(desc) == IRQ_HANDLED)
ret = IRQ_HANDLED;
/* Make sure that there is still a valid action */
action = desc->action;
} while ((desc->istate & IRQS_PENDING) && action);
desc->istate &= ~IRQS_POLL_INPROGRESS;
out:
raw_spin_unlock(&desc->lock);
return ret == IRQ_HANDLED;
}
static int misrouted_irq(int irq)
{
struct irq_desc *desc;
int i, ok = 0;
if (atomic_inc_return(&irq_poll_active) != 1)
goto out;
irq_poll_cpu = smp_processor_id(); for_each_irq_desc(i, desc) { if (!i)
continue;
if (i == irq) /* Already tried */
continue;
if (try_one_irq(desc, false))
ok = 1;
}
out:
atomic_dec(&irq_poll_active);
/* So the caller can adjust the irq error counts */
return ok;
}
static void poll_spurious_irqs(struct timer_list *unused)
{
struct irq_desc *desc;
int i;
if (atomic_inc_return(&irq_poll_active) != 1)
goto out;
irq_poll_cpu = smp_processor_id();
for_each_irq_desc(i, desc) {
unsigned int state;
if (!i)
continue;
/* Racy but it doesn't matter */
state = desc->istate;
barrier();
if (!(state & IRQS_SPURIOUS_DISABLED))
continue;
local_irq_disable();
try_one_irq(desc, true);
local_irq_enable();
}
out:
atomic_dec(&irq_poll_active);
mod_timer(&poll_spurious_irq_timer,
jiffies + POLL_SPURIOUS_IRQ_INTERVAL);
}
static inline int bad_action_ret(irqreturn_t action_ret)
{
unsigned int r = action_ret;
if (likely(r <= (IRQ_HANDLED | IRQ_WAKE_THREAD)))
return 0;
return 1;
}
/*
* If 99,900 of the previous 100,000 interrupts have not been handled
* then assume that the IRQ is stuck in some manner. Drop a diagnostic
* and try to turn the IRQ off.
*
* (The other 100-of-100,000 interrupts may have been a correctly
* functioning device sharing an IRQ with the failing one)
*/
static void __report_bad_irq(struct irq_desc *desc, irqreturn_t action_ret)
{
unsigned int irq = irq_desc_get_irq(desc);
struct irqaction *action;
unsigned long flags;
if (bad_action_ret(action_ret)) {
printk(KERN_ERR "irq event %d: bogus return value %x\n",
irq, action_ret);
} else {
printk(KERN_ERR "irq %d: nobody cared (try booting with "
"the \"irqpoll\" option)\n", irq);
}
dump_stack();
printk(KERN_ERR "handlers:\n");
/*
* We need to take desc->lock here. note_interrupt() is called
* w/o desc->lock held, but IRQ_PROGRESS set. We might race
* with something else removing an action. It's ok to take
* desc->lock here. See synchronize_irq().
*/
raw_spin_lock_irqsave(&desc->lock, flags);
for_each_action_of_desc(desc, action) {
printk(KERN_ERR "[<%p>] %ps", action->handler, action->handler);
if (action->thread_fn)
printk(KERN_CONT " threaded [<%p>] %ps",
action->thread_fn, action->thread_fn);
printk(KERN_CONT "\n");
}
raw_spin_unlock_irqrestore(&desc->lock, flags);
}
static void report_bad_irq(struct irq_desc *desc, irqreturn_t action_ret)
{
static int count = 100;
if (count > 0) { count--;
__report_bad_irq(desc, action_ret);
}
}
static inline int
try_misrouted_irq(unsigned int irq, struct irq_desc *desc,
irqreturn_t action_ret)
{
struct irqaction *action;
if (!irqfixup)
return 0;
/* We didn't actually handle the IRQ - see if it was misrouted? */
if (action_ret == IRQ_NONE)
return 1;
/*
* But for 'irqfixup == 2' we also do it for handled interrupts if
* they are marked as IRQF_IRQPOLL (or for irq zero, which is the
* traditional PC timer interrupt.. Legacy)
*/
if (irqfixup < 2)
return 0;
if (!irq)
return 1;
/*
* Since we don't get the descriptor lock, "action" can
* change under us. We don't really care, but we don't
* want to follow a NULL pointer. So tell the compiler to
* just load it once by using a barrier.
*/
action = desc->action;
barrier();
return action && (action->flags & IRQF_IRQPOLL);
}
#define SPURIOUS_DEFERRED 0x80000000
void note_interrupt(struct irq_desc *desc, irqreturn_t action_ret)
{
unsigned int irq;
if (desc->istate & IRQS_POLL_INPROGRESS || irq_settings_is_polled(desc))
return;
if (bad_action_ret(action_ret)) { report_bad_irq(desc, action_ret);
return;
}
/*
* We cannot call note_interrupt from the threaded handler
* because we need to look at the compound of all handlers
* (primary and threaded). Aside of that in the threaded
* shared case we have no serialization against an incoming
* hardware interrupt while we are dealing with a threaded
* result.
*
* So in case a thread is woken, we just note the fact and
* defer the analysis to the next hardware interrupt.
*
* The threaded handlers store whether they successfully
* handled an interrupt and we check whether that number
* changed versus the last invocation.
*
* We could handle all interrupts with the delayed by one
* mechanism, but for the non forced threaded case we'd just
* add pointless overhead to the straight hardirq interrupts
* for the sake of a few lines less code.
*/
if (action_ret & IRQ_WAKE_THREAD) {
/*
* There is a thread woken. Check whether one of the
* shared primary handlers returned IRQ_HANDLED. If
* not we defer the spurious detection to the next
* interrupt.
*/
if (action_ret == IRQ_WAKE_THREAD) {
int handled;
/*
* We use bit 31 of thread_handled_last to
* denote the deferred spurious detection
* active. No locking necessary as
* thread_handled_last is only accessed here
* and we have the guarantee that hard
* interrupts are not reentrant.
*/
if (!(desc->threads_handled_last & SPURIOUS_DEFERRED)) { desc->threads_handled_last |= SPURIOUS_DEFERRED;
return;
}
/*
* Check whether one of the threaded handlers
* returned IRQ_HANDLED since the last
* interrupt happened.
*
* For simplicity we just set bit 31, as it is
* set in threads_handled_last as well. So we
* avoid extra masking. And we really do not
* care about the high bits of the handled
* count. We just care about the count being
* different than the one we saw before.
*/
handled = atomic_read(&desc->threads_handled);
handled |= SPURIOUS_DEFERRED;
if (handled != desc->threads_handled_last) {
action_ret = IRQ_HANDLED;
/*
* Note: We keep the SPURIOUS_DEFERRED
* bit set. We are handling the
* previous invocation right now.
* Keep it for the current one, so the
* next hardware interrupt will
* account for it.
*/
desc->threads_handled_last = handled;
} else {
/*
* None of the threaded handlers felt
* responsible for the last interrupt
*
* We keep the SPURIOUS_DEFERRED bit
* set in threads_handled_last as we
* need to account for the current
* interrupt as well.
*/
action_ret = IRQ_NONE;
}
} else {
/*
* One of the primary handlers returned
* IRQ_HANDLED. So we don't care about the
* threaded handlers on the same line. Clear
* the deferred detection bit.
*
* In theory we could/should check whether the
* deferred bit is set and take the result of
* the previous run into account here as
* well. But it's really not worth the
* trouble. If every other interrupt is
* handled we never trigger the spurious
* detector. And if this is just the one out
* of 100k unhandled ones which is handled
* then we merily delay the spurious detection
* by one hard interrupt. Not a real problem.
*/
desc->threads_handled_last &= ~SPURIOUS_DEFERRED;
}
}
if (unlikely(action_ret == IRQ_NONE)) {
/*
* If we are seeing only the odd spurious IRQ caused by
* bus asynchronicity then don't eventually trigger an error,
* otherwise the counter becomes a doomsday timer for otherwise
* working systems
*/
if (time_after(jiffies, desc->last_unhandled + HZ/10))
desc->irqs_unhandled = 1;
else
desc->irqs_unhandled++;
desc->last_unhandled = jiffies;
}
irq = irq_desc_get_irq(desc); if (unlikely(try_misrouted_irq(irq, desc, action_ret))) { int ok = misrouted_irq(irq);
if (action_ret == IRQ_NONE)
desc->irqs_unhandled -= ok;
}
if (likely(!desc->irqs_unhandled))
return;
/* Now getting into unhandled irq detection */
desc->irq_count++;
if (likely(desc->irq_count < 100000))
return;
desc->irq_count = 0;
if (unlikely(desc->irqs_unhandled > 99900)) {
/*
* The interrupt is stuck
*/
__report_bad_irq(desc, action_ret);
/*
* Now kill the IRQ
*/
printk(KERN_EMERG "Disabling IRQ #%d\n", irq);
desc->istate |= IRQS_SPURIOUS_DISABLED;
desc->depth++;
irq_disable(desc);
mod_timer(&poll_spurious_irq_timer,
jiffies + POLL_SPURIOUS_IRQ_INTERVAL);
}
desc->irqs_unhandled = 0;
}
bool noirqdebug __read_mostly;
int noirqdebug_setup(char *str)
{
noirqdebug = 1;
printk(KERN_INFO "IRQ lockup detection disabled\n");
return 1;
}
__setup("noirqdebug", noirqdebug_setup);
module_param(noirqdebug, bool, 0644);
MODULE_PARM_DESC(noirqdebug, "Disable irq lockup detection when true");
static int __init irqfixup_setup(char *str)
{
if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
pr_warn("irqfixup boot option not supported with PREEMPT_RT\n");
return 1;
}
irqfixup = 1;
printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n");
printk(KERN_WARNING "This may impact system performance.\n");
return 1;
}
__setup("irqfixup", irqfixup_setup);
module_param(irqfixup, int, 0644);
static int __init irqpoll_setup(char *str)
{
if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
pr_warn("irqpoll boot option not supported with PREEMPT_RT\n");
return 1;
}
irqfixup = 2;
printk(KERN_WARNING "Misrouted IRQ fixup and polling support "
"enabled\n");
printk(KERN_WARNING "This may significantly impact system "
"performance\n");
return 1;
}
__setup("irqpoll", irqpoll_setup);
// SPDX-License-Identifier: GPL-2.0
/*
* Lockless hierarchical page accounting & limiting
*
* Copyright (C) 2014 Red Hat, Inc., Johannes Weiner
*/
#include <linux/page_counter.h>
#include <linux/atomic.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/sched.h>
#include <linux/bug.h>
#include <asm/page.h>
static void propagate_protected_usage(struct page_counter *c,
unsigned long usage)
{
unsigned long protected, old_protected;
long delta;
if (!c->parent)
return;
protected = min(usage, READ_ONCE(c->min));
old_protected = atomic_long_read(&c->min_usage);
if (protected != old_protected) {
old_protected = atomic_long_xchg(&c->min_usage, protected);
delta = protected - old_protected;
if (delta)
atomic_long_add(delta, &c->parent->children_min_usage);
}
protected = min(usage, READ_ONCE(c->low));
old_protected = atomic_long_read(&c->low_usage);
if (protected != old_protected) {
old_protected = atomic_long_xchg(&c->low_usage, protected);
delta = protected - old_protected;
if (delta)
atomic_long_add(delta, &c->parent->children_low_usage);
}
}
/**
* page_counter_cancel - take pages out of the local counter
* @counter: counter
* @nr_pages: number of pages to cancel
*/
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages)
{
long new;
new = atomic_long_sub_return(nr_pages, &counter->usage);
/* More uncharges than charges? */
if (WARN_ONCE(new < 0, "page_counter underflow: %ld nr_pages=%lu\n",
new, nr_pages)) {
new = 0;
atomic_long_set(&counter->usage, new);
}
propagate_protected_usage(counter, new);
}
/**
* page_counter_charge - hierarchically charge pages
* @counter: counter
* @nr_pages: number of pages to charge
*
* NOTE: This does not consider any configured counter limits.
*/
void page_counter_charge(struct page_counter *counter, unsigned long nr_pages)
{
struct page_counter *c;
for (c = counter; c; c = c->parent) {
long new;
new = atomic_long_add_return(nr_pages, &c->usage); propagate_protected_usage(c, new);
/*
* This is indeed racy, but we can live with some
* inaccuracy in the watermark.
*/
if (new > READ_ONCE(c->watermark)) WRITE_ONCE(c->watermark, new);
}
}
/**
* page_counter_try_charge - try to hierarchically charge pages
* @counter: counter
* @nr_pages: number of pages to charge
* @fail: points first counter to hit its limit, if any
*
* Returns %true on success, or %false and @fail if the counter or one
* of its ancestors has hit its configured limit.
*/
bool page_counter_try_charge(struct page_counter *counter,
unsigned long nr_pages,
struct page_counter **fail)
{
struct page_counter *c;
for (c = counter; c; c = c->parent) {
long new;
/*
* Charge speculatively to avoid an expensive CAS. If
* a bigger charge fails, it might falsely lock out a
* racing smaller charge and send it into reclaim
* early, but the error is limited to the difference
* between the two sizes, which is less than 2M/4M in
* case of a THP locking out a regular page charge.
*
* The atomic_long_add_return() implies a full memory
* barrier between incrementing the count and reading
* the limit. When racing with page_counter_set_max(),
* we either see the new limit or the setter sees the
* counter has changed and retries.
*/
new = atomic_long_add_return(nr_pages, &c->usage);
if (new > c->max) {
atomic_long_sub(nr_pages, &c->usage);
/*
* This is racy, but we can live with some
* inaccuracy in the failcnt which is only used
* to report stats.
*/
data_race(c->failcnt++);
*fail = c;
goto failed;
}
propagate_protected_usage(c, new);
/*
* Just like with failcnt, we can live with some
* inaccuracy in the watermark.
*/
if (new > READ_ONCE(c->watermark)) WRITE_ONCE(c->watermark, new);
}
return true;
failed:
for (c = counter; c != *fail; c = c->parent)
page_counter_cancel(c, nr_pages);
return false;
}
/**
* page_counter_uncharge - hierarchically uncharge pages
* @counter: counter
* @nr_pages: number of pages to uncharge
*/
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages)
{
struct page_counter *c;
for (c = counter; c; c = c->parent) page_counter_cancel(c, nr_pages);
}
/**
* page_counter_set_max - set the maximum number of pages allowed
* @counter: counter
* @nr_pages: limit to set
*
* Returns 0 on success, -EBUSY if the current number of pages on the
* counter already exceeds the specified limit.
*
* The caller must serialize invocations on the same counter.
*/
int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages)
{
for (;;) {
unsigned long old;
long usage;
/*
* Update the limit while making sure that it's not
* below the concurrently-changing counter value.
*
* The xchg implies two full memory barriers before
* and after, so the read-swap-read is ordered and
* ensures coherency with page_counter_try_charge():
* that function modifies the count before checking
* the limit, so if it sees the old limit, we see the
* modified counter and retry.
*/
usage = page_counter_read(counter);
if (usage > nr_pages)
return -EBUSY;
old = xchg(&counter->max, nr_pages);
if (page_counter_read(counter) <= usage || nr_pages >= old)
return 0;
counter->max = old;
cond_resched();
}
}
/**
* page_counter_set_min - set the amount of protected memory
* @counter: counter
* @nr_pages: value to set
*
* The caller must serialize invocations on the same counter.
*/
void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages)
{
struct page_counter *c;
WRITE_ONCE(counter->min, nr_pages);
for (c = counter; c; c = c->parent)
propagate_protected_usage(c, atomic_long_read(&c->usage));
}
/**
* page_counter_set_low - set the amount of protected memory
* @counter: counter
* @nr_pages: value to set
*
* The caller must serialize invocations on the same counter.
*/
void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages)
{
struct page_counter *c;
WRITE_ONCE(counter->low, nr_pages);
for (c = counter; c; c = c->parent)
propagate_protected_usage(c, atomic_long_read(&c->usage));
}
/**
* page_counter_memparse - memparse() for page counter limits
* @buf: string to parse
* @max: string meaning maximum possible value
* @nr_pages: returns the result in number of pages
*
* Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be
* limited to %PAGE_COUNTER_MAX.
*/
int page_counter_memparse(const char *buf, const char *max,
unsigned long *nr_pages)
{
char *end;
u64 bytes;
if (!strcmp(buf, max)) {
*nr_pages = PAGE_COUNTER_MAX;
return 0;
}
bytes = memparse(buf, &end);
if (*end != '\0')
return -EINVAL;
*nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX);
return 0;
}
/*
* Copyright (c) 2006-2008 Intel Corporation
* Copyright (c) 2007 Dave Airlie <airlied@linux.ie>
* Copyright (c) 2008 Red Hat Inc.
*
* DRM core CRTC related functions
*
* Permission to use, copy, modify, distribute, and sell this software and its
* documentation for any purpose is hereby granted without fee, provided that
* the above copyright notice appear in all copies and that both that copyright
* notice and this permission notice appear in supporting documentation, and
* that the name of the copyright holders not be used in advertising or
* publicity pertaining to distribution of the software without specific,
* written prior permission. The copyright holders make no representations
* about the suitability of this software for any purpose. It is provided "as
* is" without express or implied warranty.
*
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
* EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
* DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
* TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
* OF THIS SOFTWARE.
*
* Authors:
* Keith Packard
* Eric Anholt <eric@anholt.net>
* Dave Airlie <airlied@linux.ie>
* Jesse Barnes <jesse.barnes@intel.com>
*/
#include <linux/ctype.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/dma-fence.h>
#include <linux/uaccess.h>
#include <drm/drm_blend.h>
#include <drm/drm_crtc.h>
#include <drm/drm_edid.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_framebuffer.h>
#include <drm/drm_managed.h>
#include <drm/drm_modeset_lock.h>
#include <drm/drm_atomic.h>
#include <drm/drm_auth.h>
#include <drm/drm_debugfs_crc.h>
#include <drm/drm_drv.h>
#include <drm/drm_print.h>
#include <drm/drm_file.h>
#include "drm_crtc_internal.h"
#include "drm_internal.h"
/**
* DOC: overview
*
* A CRTC represents the overall display pipeline. It receives pixel data from
* &drm_plane and blends them together. The &drm_display_mode is also attached
* to the CRTC, specifying display timings. On the output side the data is fed
* to one or more &drm_encoder, which are then each connected to one
* &drm_connector.
*
* To create a CRTC, a KMS driver allocates and zeroes an instance of
* &struct drm_crtc (possibly as part of a larger structure) and registers it
* with a call to drm_crtc_init_with_planes().
*
* The CRTC is also the entry point for legacy modeset operations (see
* &drm_crtc_funcs.set_config), legacy plane operations (see
* &drm_crtc_funcs.page_flip and &drm_crtc_funcs.cursor_set2), and other legacy
* operations like &drm_crtc_funcs.gamma_set. For atomic drivers all these
* features are controlled through &drm_property and
* &drm_mode_config_funcs.atomic_check.
*/
/**
* drm_crtc_from_index - find the registered CRTC at an index
* @dev: DRM device
* @idx: index of registered CRTC to find for
*
* Given a CRTC index, return the registered CRTC from DRM device's
* list of CRTCs with matching index. This is the inverse of drm_crtc_index().
* It's useful in the vblank callbacks (like &drm_driver.enable_vblank or
* &drm_driver.disable_vblank), since that still deals with indices instead
* of pointers to &struct drm_crtc."
*/
struct drm_crtc *drm_crtc_from_index(struct drm_device *dev, int idx)
{
struct drm_crtc *crtc;
drm_for_each_crtc(crtc, dev)
if (idx == crtc->index)
return crtc;
return NULL;
}
EXPORT_SYMBOL(drm_crtc_from_index);
int drm_crtc_force_disable(struct drm_crtc *crtc)
{
struct drm_mode_set set = {
.crtc = crtc,
};
WARN_ON(drm_drv_uses_atomic_modeset(crtc->dev));
return drm_mode_set_config_internal(&set);
}
int drm_crtc_register_all(struct drm_device *dev)
{
struct drm_crtc *crtc;
int ret = 0;
drm_for_each_crtc(crtc, dev) {
drm_debugfs_crtc_add(crtc);
if (crtc->funcs->late_register)
ret = crtc->funcs->late_register(crtc);
if (ret)
return ret;
}
return 0;
}
void drm_crtc_unregister_all(struct drm_device *dev)
{
struct drm_crtc *crtc;
drm_for_each_crtc(crtc, dev) { if (crtc->funcs->early_unregister) crtc->funcs->early_unregister(crtc); drm_debugfs_crtc_remove(crtc);
}
}
static int drm_crtc_crc_init(struct drm_crtc *crtc)
{
#ifdef CONFIG_DEBUG_FS
spin_lock_init(&crtc->crc.lock);
init_waitqueue_head(&crtc->crc.wq);
crtc->crc.source = kstrdup("auto", GFP_KERNEL);
if (!crtc->crc.source)
return -ENOMEM;
#endif
return 0;
}
static void drm_crtc_crc_fini(struct drm_crtc *crtc)
{
#ifdef CONFIG_DEBUG_FS
kfree(crtc->crc.source);
#endif
}
static const struct dma_fence_ops drm_crtc_fence_ops;
static struct drm_crtc *fence_to_crtc(struct dma_fence *fence)
{
BUG_ON(fence->ops != &drm_crtc_fence_ops);
return container_of(fence->lock, struct drm_crtc, fence_lock);
}
static const char *drm_crtc_fence_get_driver_name(struct dma_fence *fence)
{
struct drm_crtc *crtc = fence_to_crtc(fence);
return crtc->dev->driver->name;
}
static const char *drm_crtc_fence_get_timeline_name(struct dma_fence *fence)
{
struct drm_crtc *crtc = fence_to_crtc(fence);
return crtc->timeline_name;
}
static const struct dma_fence_ops drm_crtc_fence_ops = {
.get_driver_name = drm_crtc_fence_get_driver_name,
.get_timeline_name = drm_crtc_fence_get_timeline_name,
};
struct dma_fence *drm_crtc_create_fence(struct drm_crtc *crtc)
{
struct dma_fence *fence;
fence = kzalloc(sizeof(*fence), GFP_KERNEL);
if (!fence)
return NULL;
dma_fence_init(fence, &drm_crtc_fence_ops, &crtc->fence_lock,
crtc->fence_context, ++crtc->fence_seqno);
return fence;
}
/**
* DOC: standard CRTC properties
*
* DRM CRTCs have a few standardized properties:
*
* ACTIVE:
* Atomic property for setting the power state of the CRTC. When set to 1
* the CRTC will actively display content. When set to 0 the CRTC will be
* powered off. There is no expectation that user-space will reset CRTC
* resources like the mode and planes when setting ACTIVE to 0.
*
* User-space can rely on an ACTIVE change to 1 to never fail an atomic
* test as long as no other property has changed. If a change to ACTIVE
* fails an atomic test, this is a driver bug. For this reason setting
* ACTIVE to 0 must not release internal resources (like reserved memory
* bandwidth or clock generators).
*
* Note that the legacy DPMS property on connectors is internally routed
* to control this property for atomic drivers.
* MODE_ID:
* Atomic property for setting the CRTC display timings. The value is the
* ID of a blob containing the DRM mode info. To disable the CRTC,
* user-space must set this property to 0.
*
* Setting MODE_ID to 0 will release reserved resources for the CRTC.
* SCALING_FILTER:
* Atomic property for setting the scaling filter for CRTC scaler
*
* The value of this property can be one of the following:
*
* Default:
* Driver's default scaling filter
* Nearest Neighbor:
* Nearest Neighbor scaling filter
*/
__printf(6, 0)
static int __drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
struct drm_plane *primary,
struct drm_plane *cursor,
const struct drm_crtc_funcs *funcs,
const char *name, va_list ap)
{
struct drm_mode_config *config = &dev->mode_config;
int ret;
WARN_ON(primary && primary->type != DRM_PLANE_TYPE_PRIMARY);
WARN_ON(cursor && cursor->type != DRM_PLANE_TYPE_CURSOR);
/* crtc index is used with 32bit bitmasks */
if (WARN_ON(config->num_crtc >= 32))
return -EINVAL;
WARN_ON(drm_drv_uses_atomic_modeset(dev) &&
(!funcs->atomic_destroy_state ||
!funcs->atomic_duplicate_state));
crtc->dev = dev;
crtc->funcs = funcs;
INIT_LIST_HEAD(&crtc->commit_list);
spin_lock_init(&crtc->commit_lock);
drm_modeset_lock_init(&crtc->mutex);
ret = drm_mode_object_add(dev, &crtc->base, DRM_MODE_OBJECT_CRTC);
if (ret)
return ret;
if (name) {
crtc->name = kvasprintf(GFP_KERNEL, name, ap);
} else {
crtc->name = kasprintf(GFP_KERNEL, "crtc-%d", config->num_crtc);
}
if (!crtc->name) {
drm_mode_object_unregister(dev, &crtc->base);
return -ENOMEM;
}
crtc->fence_context = dma_fence_context_alloc(1);
spin_lock_init(&crtc->fence_lock);
snprintf(crtc->timeline_name, sizeof(crtc->timeline_name),
"CRTC:%d-%s", crtc->base.id, crtc->name);
crtc->base.properties = &crtc->properties;
list_add_tail(&crtc->head, &config->crtc_list);
crtc->index = config->num_crtc++;
crtc->primary = primary;
crtc->cursor = cursor;
if (primary && !primary->possible_crtcs)
primary->possible_crtcs = drm_crtc_mask(crtc);
if (cursor && !cursor->possible_crtcs)
cursor->possible_crtcs = drm_crtc_mask(crtc);
ret = drm_crtc_crc_init(crtc);
if (ret) {
drm_mode_object_unregister(dev, &crtc->base);
return ret;
}
if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
drm_object_attach_property(&crtc->base, config->prop_active, 0);
drm_object_attach_property(&crtc->base, config->prop_mode_id, 0);
drm_object_attach_property(&crtc->base,
config->prop_out_fence_ptr, 0);
drm_object_attach_property(&crtc->base,
config->prop_vrr_enabled, 0);
}
return 0;
}
/**
* drm_crtc_init_with_planes - Initialise a new CRTC object with
* specified primary and cursor planes.
* @dev: DRM device
* @crtc: CRTC object to init
* @primary: Primary plane for CRTC
* @cursor: Cursor plane for CRTC
* @funcs: callbacks for the new CRTC
* @name: printf style format string for the CRTC name, or NULL for default name
*
* Inits a new object created as base part of a driver crtc object. Drivers
* should use this function instead of drm_crtc_init(), which is only provided
* for backwards compatibility with drivers which do not yet support universal
* planes). For really simple hardware which has only 1 plane look at
* drm_simple_display_pipe_init() instead.
* The &drm_crtc_funcs.destroy hook should call drm_crtc_cleanup() and kfree()
* the crtc structure. The crtc structure should not be allocated with
* devm_kzalloc().
*
* The @primary and @cursor planes are only relevant for legacy uAPI, see
* &drm_crtc.primary and &drm_crtc.cursor.
*
* Note: consider using drmm_crtc_alloc_with_planes() or
* drmm_crtc_init_with_planes() instead of drm_crtc_init_with_planes()
* to let the DRM managed resource infrastructure take care of cleanup
* and deallocation.
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
struct drm_plane *primary,
struct drm_plane *cursor,
const struct drm_crtc_funcs *funcs,
const char *name, ...)
{
va_list ap;
int ret;
WARN_ON(!funcs->destroy);
va_start(ap, name);
ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
name, ap);
va_end(ap);
return ret;
}
EXPORT_SYMBOL(drm_crtc_init_with_planes);
static void drmm_crtc_init_with_planes_cleanup(struct drm_device *dev,
void *ptr)
{
struct drm_crtc *crtc = ptr;
drm_crtc_cleanup(crtc);
}
__printf(6, 0)
static int __drmm_crtc_init_with_planes(struct drm_device *dev,
struct drm_crtc *crtc,
struct drm_plane *primary,
struct drm_plane *cursor,
const struct drm_crtc_funcs *funcs,
const char *name,
va_list args)
{
int ret;
drm_WARN_ON(dev, funcs && funcs->destroy);
ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
name, args);
if (ret)
return ret;
ret = drmm_add_action_or_reset(dev, drmm_crtc_init_with_planes_cleanup,
crtc);
if (ret)
return ret;
return 0;
}
/**
* drmm_crtc_init_with_planes - Initialise a new CRTC object with
* specified primary and cursor planes.
* @dev: DRM device
* @crtc: CRTC object to init
* @primary: Primary plane for CRTC
* @cursor: Cursor plane for CRTC
* @funcs: callbacks for the new CRTC
* @name: printf style format string for the CRTC name, or NULL for default name
*
* Inits a new object created as base part of a driver crtc object. Drivers
* should use this function instead of drm_crtc_init(), which is only provided
* for backwards compatibility with drivers which do not yet support universal
* planes). For really simple hardware which has only 1 plane look at
* drm_simple_display_pipe_init() instead.
*
* Cleanup is automatically handled through registering
* drmm_crtc_cleanup() with drmm_add_action(). The crtc structure should
* be allocated with drmm_kzalloc().
*
* The @drm_crtc_funcs.destroy hook must be NULL.
*
* The @primary and @cursor planes are only relevant for legacy uAPI, see
* &drm_crtc.primary and &drm_crtc.cursor.
*
* Returns:
* Zero on success, error code on failure.
*/
int drmm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
struct drm_plane *primary,
struct drm_plane *cursor,
const struct drm_crtc_funcs *funcs,
const char *name, ...)
{
va_list ap;
int ret;
va_start(ap, name);
ret = __drmm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
name, ap);
va_end(ap);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(drmm_crtc_init_with_planes);
void *__drmm_crtc_alloc_with_planes(struct drm_device *dev,
size_t size, size_t offset,
struct drm_plane *primary,
struct drm_plane *cursor,
const struct drm_crtc_funcs *funcs,
const char *name, ...)
{
void *container;
struct drm_crtc *crtc;
va_list ap;
int ret;
if (WARN_ON(!funcs || funcs->destroy))
return ERR_PTR(-EINVAL);
container = drmm_kzalloc(dev, size, GFP_KERNEL);
if (!container)
return ERR_PTR(-ENOMEM);
crtc = container + offset;
va_start(ap, name);
ret = __drmm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
name, ap);
va_end(ap);
if (ret)
return ERR_PTR(ret);
return container;
}
EXPORT_SYMBOL(__drmm_crtc_alloc_with_planes);
/**
* drm_crtc_cleanup - Clean up the core crtc usage
* @crtc: CRTC to cleanup
*
* This function cleans up @crtc and removes it from the DRM mode setting
* core. Note that the function does *not* free the crtc structure itself,
* this is the responsibility of the caller.
*/
void drm_crtc_cleanup(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
/* Note that the crtc_list is considered to be static; should we
* remove the drm_crtc at runtime we would have to decrement all
* the indices on the drm_crtc after us in the crtc_list.
*/
drm_crtc_crc_fini(crtc);
kfree(crtc->gamma_store);
crtc->gamma_store = NULL;
drm_modeset_lock_fini(&crtc->mutex);
drm_mode_object_unregister(dev, &crtc->base);
list_del(&crtc->head);
dev->mode_config.num_crtc--;
WARN_ON(crtc->state && !crtc->funcs->atomic_destroy_state);
if (crtc->state && crtc->funcs->atomic_destroy_state)
crtc->funcs->atomic_destroy_state(crtc, crtc->state);
kfree(crtc->name);
memset(crtc, 0, sizeof(*crtc));
}
EXPORT_SYMBOL(drm_crtc_cleanup);
/**
* drm_mode_getcrtc - get CRTC configuration
* @dev: drm device for the ioctl
* @data: data pointer for the ioctl
* @file_priv: drm file for the ioctl call
*
* Construct a CRTC configuration structure to return to the user.
*
* Called by the user via ioctl.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_getcrtc(struct drm_device *dev,
void *data, struct drm_file *file_priv)
{
struct drm_mode_crtc *crtc_resp = data;
struct drm_crtc *crtc;
struct drm_plane *plane;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
crtc = drm_crtc_find(dev, file_priv, crtc_resp->crtc_id);
if (!crtc)
return -ENOENT;
plane = crtc->primary;
crtc_resp->gamma_size = crtc->gamma_size;
drm_modeset_lock(&plane->mutex, NULL);
if (plane->state && plane->state->fb)
crtc_resp->fb_id = plane->state->fb->base.id;
else if (!plane->state && plane->fb)
crtc_resp->fb_id = plane->fb->base.id;
else
crtc_resp->fb_id = 0;
if (plane->state) {
crtc_resp->x = plane->state->src_x >> 16;
crtc_resp->y = plane->state->src_y >> 16;
}
drm_modeset_unlock(&plane->mutex);
drm_modeset_lock(&crtc->mutex, NULL);
if (crtc->state) {
if (crtc->state->enable) {
drm_mode_convert_to_umode(&crtc_resp->mode, &crtc->state->mode);
crtc_resp->mode_valid = 1;
} else {
crtc_resp->mode_valid = 0;
}
} else {
crtc_resp->x = crtc->x;
crtc_resp->y = crtc->y;
if (crtc->enabled) {
drm_mode_convert_to_umode(&crtc_resp->mode, &crtc->mode);
crtc_resp->mode_valid = 1;
} else {
crtc_resp->mode_valid = 0;
}
}
if (!file_priv->aspect_ratio_allowed)
crtc_resp->mode.flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;
drm_modeset_unlock(&crtc->mutex);
return 0;
}
static int __drm_mode_set_config_internal(struct drm_mode_set *set,
struct drm_modeset_acquire_ctx *ctx)
{
struct drm_crtc *crtc = set->crtc;
struct drm_framebuffer *fb;
struct drm_crtc *tmp;
int ret;
WARN_ON(drm_drv_uses_atomic_modeset(crtc->dev));
/*
* NOTE: ->set_config can also disable other crtcs (if we steal all
* connectors from it), hence we need to refcount the fbs across all
* crtcs. Atomic modeset will have saner semantics ...
*/
drm_for_each_crtc(tmp, crtc->dev) {
struct drm_plane *plane = tmp->primary;
plane->old_fb = plane->fb;
}
fb = set->fb;
ret = crtc->funcs->set_config(set, ctx);
if (ret == 0) {
struct drm_plane *plane = crtc->primary;
plane->crtc = fb ? crtc : NULL;
plane->fb = fb;
}
drm_for_each_crtc(tmp, crtc->dev) {
struct drm_plane *plane = tmp->primary;
if (plane->fb)
drm_framebuffer_get(plane->fb);
if (plane->old_fb)
drm_framebuffer_put(plane->old_fb);
plane->old_fb = NULL;
}
return ret;
}
/**
* drm_mode_set_config_internal - helper to call &drm_mode_config_funcs.set_config
* @set: modeset config to set
*
* This is a little helper to wrap internal calls to the
* &drm_mode_config_funcs.set_config driver interface. The only thing it adds is
* correct refcounting dance.
*
* This should only be used by non-atomic legacy drivers.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_set_config_internal(struct drm_mode_set *set)
{
WARN_ON(drm_drv_uses_atomic_modeset(set->crtc->dev));
return __drm_mode_set_config_internal(set, NULL);
}
EXPORT_SYMBOL(drm_mode_set_config_internal);
/**
* drm_crtc_check_viewport - Checks that a framebuffer is big enough for the
* CRTC viewport
* @crtc: CRTC that framebuffer will be displayed on
* @x: x panning
* @y: y panning
* @mode: mode that framebuffer will be displayed under
* @fb: framebuffer to check size of
*/
int drm_crtc_check_viewport(const struct drm_crtc *crtc,
int x, int y,
const struct drm_display_mode *mode,
const struct drm_framebuffer *fb)
{
int hdisplay, vdisplay;
drm_mode_get_hv_timing(mode, &hdisplay, &vdisplay);
if (crtc->state &&
drm_rotation_90_or_270(crtc->primary->state->rotation))
swap(hdisplay, vdisplay);
return drm_framebuffer_check_src_coords(x << 16, y << 16,
hdisplay << 16, vdisplay << 16,
fb);
}
EXPORT_SYMBOL(drm_crtc_check_viewport);
/**
* drm_mode_setcrtc - set CRTC configuration
* @dev: drm device for the ioctl
* @data: data pointer for the ioctl
* @file_priv: drm file for the ioctl call
*
* Build a new CRTC configuration based on user request.
*
* Called by the user via ioctl.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_setcrtc(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_mode_config *config = &dev->mode_config;
struct drm_mode_crtc *crtc_req = data;
struct drm_crtc *crtc;
struct drm_plane *plane;
struct drm_connector **connector_set = NULL, *connector;
struct drm_framebuffer *fb = NULL;
struct drm_display_mode *mode = NULL;
struct drm_mode_set set;
uint32_t __user *set_connectors_ptr;
struct drm_modeset_acquire_ctx ctx;
int ret, i, num_connectors = 0;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
/*
* Universal plane src offsets are only 16.16, prevent havoc for
* drivers using universal plane code internally.
*/
if (crtc_req->x & 0xffff0000 || crtc_req->y & 0xffff0000)
return -ERANGE;
crtc = drm_crtc_find(dev, file_priv, crtc_req->crtc_id);
if (!crtc) {
DRM_DEBUG_KMS("Unknown CRTC ID %d\n", crtc_req->crtc_id);
return -ENOENT;
}
DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
plane = crtc->primary;
/* allow disabling with the primary plane leased */
if (crtc_req->mode_valid && !drm_lease_held(file_priv, plane->base.id))
return -EACCES;
DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx,
DRM_MODESET_ACQUIRE_INTERRUPTIBLE, ret);
if (crtc_req->mode_valid) {
/* If we have a mode we need a framebuffer. */
/* If we pass -1, set the mode with the currently bound fb */
if (crtc_req->fb_id == -1) {
struct drm_framebuffer *old_fb;
if (plane->state)
old_fb = plane->state->fb;
else
old_fb = plane->fb;
if (!old_fb) {
DRM_DEBUG_KMS("CRTC doesn't have current FB\n");
ret = -EINVAL;
goto out;
}
fb = old_fb;
/* Make refcounting symmetric with the lookup path. */
drm_framebuffer_get(fb);
} else {
fb = drm_framebuffer_lookup(dev, file_priv, crtc_req->fb_id);
if (!fb) {
DRM_DEBUG_KMS("Unknown FB ID%d\n",
crtc_req->fb_id);
ret = -ENOENT;
goto out;
}
}
mode = drm_mode_create(dev);
if (!mode) {
ret = -ENOMEM;
goto out;
}
if (!file_priv->aspect_ratio_allowed &&
(crtc_req->mode.flags & DRM_MODE_FLAG_PIC_AR_MASK) != DRM_MODE_FLAG_PIC_AR_NONE) {
DRM_DEBUG_KMS("Unexpected aspect-ratio flag bits\n");
ret = -EINVAL;
goto out;
}
ret = drm_mode_convert_umode(dev, mode, &crtc_req->mode);
if (ret) {
DRM_DEBUG_KMS("Invalid mode (ret=%d, status=%s)\n",
ret, drm_get_mode_status_name(mode->status));
drm_mode_debug_printmodeline(mode);
goto out;
}
/*
* Check whether the primary plane supports the fb pixel format.
* Drivers not implementing the universal planes API use a
* default formats list provided by the DRM core which doesn't
* match real hardware capabilities. Skip the check in that
* case.
*/
if (!plane->format_default) {
ret = drm_plane_check_pixel_format(plane,
fb->format->format,
fb->modifier);
if (ret) {
DRM_DEBUG_KMS("Invalid pixel format %p4cc, modifier 0x%llx\n",
&fb->format->format,
fb->modifier);
goto out;
}
}
ret = drm_crtc_check_viewport(crtc, crtc_req->x, crtc_req->y,
mode, fb);
if (ret)
goto out;
}
if (crtc_req->count_connectors == 0 && mode) {
DRM_DEBUG_KMS("Count connectors is 0 but mode set\n");
ret = -EINVAL;
goto out;
}
if (crtc_req->count_connectors > 0 && (!mode || !fb)) {
DRM_DEBUG_KMS("Count connectors is %d but no mode or fb set\n",
crtc_req->count_connectors);
ret = -EINVAL;
goto out;
}
if (crtc_req->count_connectors > 0) {
u32 out_id;
/* Avoid unbounded kernel memory allocation */
if (crtc_req->count_connectors > config->num_connector) {
ret = -EINVAL;
goto out;
}
connector_set = kmalloc_array(crtc_req->count_connectors,
sizeof(struct drm_connector *),
GFP_KERNEL);
if (!connector_set) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < crtc_req->count_connectors; i++) {
connector_set[i] = NULL;
set_connectors_ptr = (uint32_t __user *)(unsigned long)crtc_req->set_connectors_ptr;
if (get_user(out_id, &set_connectors_ptr[i])) {
ret = -EFAULT;
goto out;
}
connector = drm_connector_lookup(dev, file_priv, out_id);
if (!connector) {
DRM_DEBUG_KMS("Connector id %d unknown\n",
out_id);
ret = -ENOENT;
goto out;
}
DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
connector->base.id,
connector->name);
connector_set[i] = connector;
num_connectors++;
}
}
set.crtc = crtc;
set.x = crtc_req->x;
set.y = crtc_req->y;
set.mode = mode;
set.connectors = connector_set;
set.num_connectors = num_connectors;
set.fb = fb;
if (drm_drv_uses_atomic_modeset(dev))
ret = crtc->funcs->set_config(&set, &ctx);
else
ret = __drm_mode_set_config_internal(&set, &ctx);
out:
if (fb)
drm_framebuffer_put(fb);
if (connector_set) {
for (i = 0; i < num_connectors; i++) {
if (connector_set[i])
drm_connector_put(connector_set[i]);
}
}
kfree(connector_set);
drm_mode_destroy(dev, mode);
/* In case we need to retry... */
connector_set = NULL;
fb = NULL;
mode = NULL;
num_connectors = 0;
DRM_MODESET_LOCK_ALL_END(dev, ctx, ret);
return ret;
}
int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
struct drm_property *property,
uint64_t value)
{
int ret = -EINVAL;
struct drm_crtc *crtc = obj_to_crtc(obj);
if (crtc->funcs->set_property)
ret = crtc->funcs->set_property(crtc, property, value);
if (!ret)
drm_object_property_set_value(obj, property, value);
return ret;
}
/**
* drm_crtc_create_scaling_filter_property - create a new scaling filter
* property
*
* @crtc: drm CRTC
* @supported_filters: bitmask of supported scaling filters, must include
* BIT(DRM_SCALING_FILTER_DEFAULT).
*
* This function lets driver to enable the scaling filter property on a given
* CRTC.
*
* RETURNS:
* Zero for success or -errno
*/
int drm_crtc_create_scaling_filter_property(struct drm_crtc *crtc,
unsigned int supported_filters)
{
struct drm_property *prop =
drm_create_scaling_filter_prop(crtc->dev, supported_filters);
if (IS_ERR(prop))
return PTR_ERR(prop);
drm_object_attach_property(&crtc->base, prop,
DRM_SCALING_FILTER_DEFAULT);
crtc->scaling_filter_property = prop;
return 0;
}
EXPORT_SYMBOL(drm_crtc_create_scaling_filter_property);
// SPDX-License-Identifier: GPL-2.0
/*
* device_cgroup.c - device cgroup subsystem
*
* Copyright 2007 IBM Corp
*/
#include <linux/bpf-cgroup.h>
#include <linux/device_cgroup.h>
#include <linux/cgroup.h>
#include <linux/ctype.h>
#include <linux/list.h>
#include <linux/uaccess.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/rcupdate.h>
#include <linux/mutex.h>
#ifdef CONFIG_CGROUP_DEVICE
static DEFINE_MUTEX(devcgroup_mutex);
enum devcg_behavior {
DEVCG_DEFAULT_NONE,
DEVCG_DEFAULT_ALLOW,
DEVCG_DEFAULT_DENY,
};
/*
* exception list locking rules:
* hold devcgroup_mutex for update/read.
* hold rcu_read_lock() for read.
*/
struct dev_exception_item {
u32 major, minor;
short type;
short access;
struct list_head list;
struct rcu_head rcu;
};
struct dev_cgroup {
struct cgroup_subsys_state css;
struct list_head exceptions;
enum devcg_behavior behavior;
};
static inline struct dev_cgroup *css_to_devcgroup(struct cgroup_subsys_state *s)
{
return s ? container_of(s, struct dev_cgroup, css) : NULL;
}
static inline struct dev_cgroup *task_devcgroup(struct task_struct *task)
{
return css_to_devcgroup(task_css(task, devices_cgrp_id));
}
/*
* called under devcgroup_mutex
*/
static int dev_exceptions_copy(struct list_head *dest, struct list_head *orig)
{
struct dev_exception_item *ex, *tmp, *new;
lockdep_assert_held(&devcgroup_mutex);
list_for_each_entry(ex, orig, list) {
new = kmemdup(ex, sizeof(*ex), GFP_KERNEL);
if (!new)
goto free_and_exit;
list_add_tail(&new->list, dest);
}
return 0;
free_and_exit:
list_for_each_entry_safe(ex, tmp, dest, list) {
list_del(&ex->list);
kfree(ex);
}
return -ENOMEM;
}
static void dev_exceptions_move(struct list_head *dest, struct list_head *orig)
{
struct dev_exception_item *ex, *tmp;
lockdep_assert_held(&devcgroup_mutex);
list_for_each_entry_safe(ex, tmp, orig, list) {
list_move_tail(&ex->list, dest);
}
}
/*
* called under devcgroup_mutex
*/
static int dev_exception_add(struct dev_cgroup *dev_cgroup,
struct dev_exception_item *ex)
{
struct dev_exception_item *excopy, *walk;
lockdep_assert_held(&devcgroup_mutex);
excopy = kmemdup(ex, sizeof(*ex), GFP_KERNEL);
if (!excopy)
return -ENOMEM;
list_for_each_entry(walk, &dev_cgroup->exceptions, list) {
if (walk->type != ex->type)
continue;
if (walk->major != ex->major)
continue;
if (walk->minor != ex->minor)
continue;
walk->access |= ex->access;
kfree(excopy);
excopy = NULL;
}
if (excopy != NULL)
list_add_tail_rcu(&excopy->list, &dev_cgroup->exceptions);
return 0;
}
/*
* called under devcgroup_mutex
*/
static void dev_exception_rm(struct dev_cgroup *dev_cgroup,
struct dev_exception_item *ex)
{
struct dev_exception_item *walk, *tmp;
lockdep_assert_held(&devcgroup_mutex);
list_for_each_entry_safe(walk, tmp, &dev_cgroup->exceptions, list) {
if (walk->type != ex->type)
continue;
if (walk->major != ex->major)
continue;
if (walk->minor != ex->minor)
continue;
walk->access &= ~ex->access;
if (!walk->access) {
list_del_rcu(&walk->list);
kfree_rcu(walk, rcu);
}
}
}
static void __dev_exception_clean(struct dev_cgroup *dev_cgroup)
{
struct dev_exception_item *ex, *tmp;
list_for_each_entry_safe(ex, tmp, &dev_cgroup->exceptions, list) {
list_del_rcu(&ex->list);
kfree_rcu(ex, rcu);
}
}
/**
* dev_exception_clean - frees all entries of the exception list
* @dev_cgroup: dev_cgroup with the exception list to be cleaned
*
* called under devcgroup_mutex
*/
static void dev_exception_clean(struct dev_cgroup *dev_cgroup)
{
lockdep_assert_held(&devcgroup_mutex);
__dev_exception_clean(dev_cgroup);
}
static inline bool is_devcg_online(const struct dev_cgroup *devcg)
{
return (devcg->behavior != DEVCG_DEFAULT_NONE);
}
/**
* devcgroup_online - initializes devcgroup's behavior and exceptions based on
* parent's
* @css: css getting online
* returns 0 in case of success, error code otherwise
*/
static int devcgroup_online(struct cgroup_subsys_state *css)
{
struct dev_cgroup *dev_cgroup = css_to_devcgroup(css);
struct dev_cgroup *parent_dev_cgroup = css_to_devcgroup(css->parent);
int ret = 0;
mutex_lock(&devcgroup_mutex);
if (parent_dev_cgroup == NULL)
dev_cgroup->behavior = DEVCG_DEFAULT_ALLOW;
else {
ret = dev_exceptions_copy(&dev_cgroup->exceptions,
&parent_dev_cgroup->exceptions);
if (!ret)
dev_cgroup->behavior = parent_dev_cgroup->behavior;
}
mutex_unlock(&devcgroup_mutex);
return ret;
}
static void devcgroup_offline(struct cgroup_subsys_state *css)
{
struct dev_cgroup *dev_cgroup = css_to_devcgroup(css);
mutex_lock(&devcgroup_mutex);
dev_cgroup->behavior = DEVCG_DEFAULT_NONE;
mutex_unlock(&devcgroup_mutex);
}
/*
* called from kernel/cgroup/cgroup.c with cgroup_lock() held.
*/
static struct cgroup_subsys_state *
devcgroup_css_alloc(struct cgroup_subsys_state *parent_css)
{
struct dev_cgroup *dev_cgroup;
dev_cgroup = kzalloc(sizeof(*dev_cgroup), GFP_KERNEL);
if (!dev_cgroup)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&dev_cgroup->exceptions);
dev_cgroup->behavior = DEVCG_DEFAULT_NONE;
return &dev_cgroup->css;
}
static void devcgroup_css_free(struct cgroup_subsys_state *css)
{
struct dev_cgroup *dev_cgroup = css_to_devcgroup(css);
__dev_exception_clean(dev_cgroup);
kfree(dev_cgroup);
}
#define DEVCG_ALLOW 1
#define DEVCG_DENY 2
#define DEVCG_LIST 3
#define MAJMINLEN 13
#define ACCLEN 4
static void set_access(char *acc, short access)
{
int idx = 0;
memset(acc, 0, ACCLEN);
if (access & DEVCG_ACC_READ)
acc[idx++] = 'r';
if (access & DEVCG_ACC_WRITE)
acc[idx++] = 'w';
if (access & DEVCG_ACC_MKNOD)
acc[idx++] = 'm';
}
static char type_to_char(short type)
{
if (type == DEVCG_DEV_ALL)
return 'a';
if (type == DEVCG_DEV_CHAR)
return 'c';
if (type == DEVCG_DEV_BLOCK)
return 'b';
return 'X';
}
static void set_majmin(char *str, unsigned m)
{
if (m == ~0)
strcpy(str, "*");
else
sprintf(str, "%u", m);
}
static int devcgroup_seq_show(struct seq_file *m, void *v)
{
struct dev_cgroup *devcgroup = css_to_devcgroup(seq_css(m));
struct dev_exception_item *ex;
char maj[MAJMINLEN], min[MAJMINLEN], acc[ACCLEN];
rcu_read_lock();
/*
* To preserve the compatibility:
* - Only show the "all devices" when the default policy is to allow
* - List the exceptions in case the default policy is to deny
* This way, the file remains as a "whitelist of devices"
*/
if (devcgroup->behavior == DEVCG_DEFAULT_ALLOW) {
set_access(acc, DEVCG_ACC_MASK);
set_majmin(maj, ~0);
set_majmin(min, ~0);
seq_printf(m, "%c %s:%s %s\n", type_to_char(DEVCG_DEV_ALL),
maj, min, acc);
} else {
list_for_each_entry_rcu(ex, &devcgroup->exceptions, list) {
set_access(acc, ex->access);
set_majmin(maj, ex->major);
set_majmin(min, ex->minor);
seq_printf(m, "%c %s:%s %s\n", type_to_char(ex->type),
maj, min, acc);
}
}
rcu_read_unlock();
return 0;
}
/**
* match_exception - iterates the exception list trying to find a complete match
* @exceptions: list of exceptions
* @type: device type (DEVCG_DEV_BLOCK or DEVCG_DEV_CHAR)
* @major: device file major number, ~0 to match all
* @minor: device file minor number, ~0 to match all
* @access: permission mask (DEVCG_ACC_READ, DEVCG_ACC_WRITE, DEVCG_ACC_MKNOD)
*
* It is considered a complete match if an exception is found that will
* contain the entire range of provided parameters.
*
* Return: true in case it matches an exception completely
*/
static bool match_exception(struct list_head *exceptions, short type,
u32 major, u32 minor, short access)
{
struct dev_exception_item *ex;
list_for_each_entry_rcu(ex, exceptions, list) {
if ((type & DEVCG_DEV_BLOCK) && !(ex->type & DEVCG_DEV_BLOCK))
continue;
if ((type & DEVCG_DEV_CHAR) && !(ex->type & DEVCG_DEV_CHAR))
continue;
if (ex->major != ~0 && ex->major != major)
continue;
if (ex->minor != ~0 && ex->minor != minor)
continue;
/* provided access cannot have more than the exception rule */
if (access & (~ex->access))
continue;
return true;
}
return false;
}
/**
* match_exception_partial - iterates the exception list trying to find a partial match
* @exceptions: list of exceptions
* @type: device type (DEVCG_DEV_BLOCK or DEVCG_DEV_CHAR)
* @major: device file major number, ~0 to match all
* @minor: device file minor number, ~0 to match all
* @access: permission mask (DEVCG_ACC_READ, DEVCG_ACC_WRITE, DEVCG_ACC_MKNOD)
*
* It is considered a partial match if an exception's range is found to
* contain *any* of the devices specified by provided parameters. This is
* used to make sure no extra access is being granted that is forbidden by
* any of the exception list.
*
* Return: true in case the provided range mat matches an exception completely
*/
static bool match_exception_partial(struct list_head *exceptions, short type,
u32 major, u32 minor, short access)
{
struct dev_exception_item *ex;
list_for_each_entry_rcu(ex, exceptions, list,
lockdep_is_held(&devcgroup_mutex)) {
if ((type & DEVCG_DEV_BLOCK) && !(ex->type & DEVCG_DEV_BLOCK))
continue;
if ((type & DEVCG_DEV_CHAR) && !(ex->type & DEVCG_DEV_CHAR))
continue;
/*
* We must be sure that both the exception and the provided
* range aren't masking all devices
*/
if (ex->major != ~0 && major != ~0 && ex->major != major)
continue;
if (ex->minor != ~0 && minor != ~0 && ex->minor != minor)
continue;
/*
* In order to make sure the provided range isn't matching
* an exception, all its access bits shouldn't match the
* exception's access bits
*/
if (!(access & ex->access))
continue;
return true;
}
return false;
}
/**
* verify_new_ex - verifies if a new exception is allowed by parent cgroup's permissions
* @dev_cgroup: dev cgroup to be tested against
* @refex: new exception
* @behavior: behavior of the exception's dev_cgroup
*
* This is used to make sure a child cgroup won't have more privileges
* than its parent
*/
static bool verify_new_ex(struct dev_cgroup *dev_cgroup,
struct dev_exception_item *refex,
enum devcg_behavior behavior)
{
bool match = false;
RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&
!lockdep_is_held(&devcgroup_mutex),
"device_cgroup:verify_new_ex called without proper synchronization");
if (dev_cgroup->behavior == DEVCG_DEFAULT_ALLOW) {
if (behavior == DEVCG_DEFAULT_ALLOW) {
/*
* new exception in the child doesn't matter, only
* adding extra restrictions
*/
return true;
} else {
/*
* new exception in the child will add more devices
* that can be accessed, so it can't match any of
* parent's exceptions, even slightly
*/
match = match_exception_partial(&dev_cgroup->exceptions,
refex->type,
refex->major,
refex->minor,
refex->access);
if (match)
return false;
return true;
}
} else {
/*
* Only behavior == DEVCG_DEFAULT_DENY allowed here, therefore
* the new exception will add access to more devices and must
* be contained completely in an parent's exception to be
* allowed
*/
match = match_exception(&dev_cgroup->exceptions, refex->type,
refex->major, refex->minor,
refex->access);
if (match)
/* parent has an exception that matches the proposed */
return true;
else
return false;
}
return false;
}
/*
* parent_has_perm:
* when adding a new allow rule to a device exception list, the rule
* must be allowed in the parent device
*/
static int parent_has_perm(struct dev_cgroup *childcg,
struct dev_exception_item *ex)
{
struct dev_cgroup *parent = css_to_devcgroup(childcg->css.parent);
if (!parent)
return 1;
return verify_new_ex(parent, ex, childcg->behavior);
}
/**
* parent_allows_removal - verify if it's ok to remove an exception
* @childcg: child cgroup from where the exception will be removed
* @ex: exception being removed
*
* When removing an exception in cgroups with default ALLOW policy, it must
* be checked if removing it will give the child cgroup more access than the
* parent.
*
* Return: true if it's ok to remove exception, false otherwise
*/
static bool parent_allows_removal(struct dev_cgroup *childcg,
struct dev_exception_item *ex)
{
struct dev_cgroup *parent = css_to_devcgroup(childcg->css.parent);
if (!parent)
return true;
/* It's always allowed to remove access to devices */
if (childcg->behavior == DEVCG_DEFAULT_DENY)
return true;
/*
* Make sure you're not removing part or a whole exception existing in
* the parent cgroup
*/
return !match_exception_partial(&parent->exceptions, ex->type,
ex->major, ex->minor, ex->access);
}
/**
* may_allow_all - checks if it's possible to change the behavior to
* allow based on parent's rules.
* @parent: device cgroup's parent
* returns: != 0 in case it's allowed, 0 otherwise
*/
static inline int may_allow_all(struct dev_cgroup *parent)
{
if (!parent)
return 1;
return parent->behavior == DEVCG_DEFAULT_ALLOW;
}
/**
* revalidate_active_exceptions - walks through the active exception list and
* revalidates the exceptions based on parent's
* behavior and exceptions. The exceptions that
* are no longer valid will be removed.
* Called with devcgroup_mutex held.
* @devcg: cgroup which exceptions will be checked
*
* This is one of the three key functions for hierarchy implementation.
* This function is responsible for re-evaluating all the cgroup's active
* exceptions due to a parent's exception change.
* Refer to Documentation/admin-guide/cgroup-v1/devices.rst for more details.
*/
static void revalidate_active_exceptions(struct dev_cgroup *devcg)
{
struct dev_exception_item *ex;
struct list_head *this, *tmp;
list_for_each_safe(this, tmp, &devcg->exceptions) {
ex = container_of(this, struct dev_exception_item, list);
if (!parent_has_perm(devcg, ex))
dev_exception_rm(devcg, ex);
}
}
/**
* propagate_exception - propagates a new exception to the children
* @devcg_root: device cgroup that added a new exception
* @ex: new exception to be propagated
*
* returns: 0 in case of success, != 0 in case of error
*/
static int propagate_exception(struct dev_cgroup *devcg_root,
struct dev_exception_item *ex)
{
struct cgroup_subsys_state *pos;
int rc = 0;
rcu_read_lock();
css_for_each_descendant_pre(pos, &devcg_root->css) {
struct dev_cgroup *devcg = css_to_devcgroup(pos);
/*
* Because devcgroup_mutex is held, no devcg will become
* online or offline during the tree walk (see on/offline
* methods), and online ones are safe to access outside RCU
* read lock without bumping refcnt.
*/
if (pos == &devcg_root->css || !is_devcg_online(devcg))
continue;
rcu_read_unlock();
/*
* in case both root's behavior and devcg is allow, a new
* restriction means adding to the exception list
*/
if (devcg_root->behavior == DEVCG_DEFAULT_ALLOW &&
devcg->behavior == DEVCG_DEFAULT_ALLOW) {
rc = dev_exception_add(devcg, ex);
if (rc)
return rc;
} else {
/*
* in the other possible cases:
* root's behavior: allow, devcg's: deny
* root's behavior: deny, devcg's: deny
* the exception will be removed
*/
dev_exception_rm(devcg, ex);
}
revalidate_active_exceptions(devcg);
rcu_read_lock();
}
rcu_read_unlock();
return rc;
}
/*
* Modify the exception list using allow/deny rules.
* CAP_SYS_ADMIN is needed for this. It's at least separate from CAP_MKNOD
* so we can give a container CAP_MKNOD to let it create devices but not
* modify the exception list.
* It seems likely we'll want to add a CAP_CONTAINER capability to allow
* us to also grant CAP_SYS_ADMIN to containers without giving away the
* device exception list controls, but for now we'll stick with CAP_SYS_ADMIN
*
* Taking rules away is always allowed (given CAP_SYS_ADMIN). Granting
* new access is only allowed if you're in the top-level cgroup, or your
* parent cgroup has the access you're asking for.
*/
static int devcgroup_update_access(struct dev_cgroup *devcgroup,
int filetype, char *buffer)
{
const char *b;
char temp[12]; /* 11 + 1 characters needed for a u32 */
int count, rc = 0;
struct dev_exception_item ex;
struct dev_cgroup *parent = css_to_devcgroup(devcgroup->css.parent);
struct dev_cgroup tmp_devcgrp;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
memset(&ex, 0, sizeof(ex));
memset(&tmp_devcgrp, 0, sizeof(tmp_devcgrp));
b = buffer;
switch (*b) {
case 'a':
switch (filetype) {
case DEVCG_ALLOW:
if (css_has_online_children(&devcgroup->css))
return -EINVAL;
if (!may_allow_all(parent))
return -EPERM;
if (!parent) {
devcgroup->behavior = DEVCG_DEFAULT_ALLOW;
dev_exception_clean(devcgroup);
break;
}
INIT_LIST_HEAD(&tmp_devcgrp.exceptions);
rc = dev_exceptions_copy(&tmp_devcgrp.exceptions,
&devcgroup->exceptions);
if (rc)
return rc;
dev_exception_clean(devcgroup);
rc = dev_exceptions_copy(&devcgroup->exceptions,
&parent->exceptions);
if (rc) {
dev_exceptions_move(&devcgroup->exceptions,
&tmp_devcgrp.exceptions);
return rc;
}
devcgroup->behavior = DEVCG_DEFAULT_ALLOW;
dev_exception_clean(&tmp_devcgrp);
break;
case DEVCG_DENY:
if (css_has_online_children(&devcgroup->css))
return -EINVAL;
dev_exception_clean(devcgroup);
devcgroup->behavior = DEVCG_DEFAULT_DENY;
break;
default:
return -EINVAL;
}
return 0;
case 'b':
ex.type = DEVCG_DEV_BLOCK;
break;
case 'c':
ex.type = DEVCG_DEV_CHAR;
break;
default:
return -EINVAL;
}
b++;
if (!isspace(*b))
return -EINVAL;
b++;
if (*b == '*') {
ex.major = ~0;
b++;
} else if (isdigit(*b)) {
memset(temp, 0, sizeof(temp));
for (count = 0; count < sizeof(temp) - 1; count++) {
temp[count] = *b;
b++;
if (!isdigit(*b))
break;
}
rc = kstrtou32(temp, 10, &ex.major);
if (rc)
return -EINVAL;
} else {
return -EINVAL;
}
if (*b != ':')
return -EINVAL;
b++;
/* read minor */
if (*b == '*') {
ex.minor = ~0;
b++;
} else if (isdigit(*b)) {
memset(temp, 0, sizeof(temp));
for (count = 0; count < sizeof(temp) - 1; count++) {
temp[count] = *b;
b++;
if (!isdigit(*b))
break;
}
rc = kstrtou32(temp, 10, &ex.minor);
if (rc)
return -EINVAL;
} else {
return -EINVAL;
}
if (!isspace(*b))
return -EINVAL;
for (b++, count = 0; count < 3; count++, b++) {
switch (*b) {
case 'r':
ex.access |= DEVCG_ACC_READ;
break;
case 'w':
ex.access |= DEVCG_ACC_WRITE;
break;
case 'm':
ex.access |= DEVCG_ACC_MKNOD;
break;
case '\n':
case '\0':
count = 3;
break;
default:
return -EINVAL;
}
}
switch (filetype) {
case DEVCG_ALLOW:
/*
* If the default policy is to allow by default, try to remove
* an matching exception instead. And be silent about it: we
* don't want to break compatibility
*/
if (devcgroup->behavior == DEVCG_DEFAULT_ALLOW) {
/* Check if the parent allows removing it first */
if (!parent_allows_removal(devcgroup, &ex))
return -EPERM;
dev_exception_rm(devcgroup, &ex);
break;
}
if (!parent_has_perm(devcgroup, &ex))
return -EPERM;
rc = dev_exception_add(devcgroup, &ex);
break;
case DEVCG_DENY:
/*
* If the default policy is to deny by default, try to remove
* an matching exception instead. And be silent about it: we
* don't want to break compatibility
*/
if (devcgroup->behavior == DEVCG_DEFAULT_DENY)
dev_exception_rm(devcgroup, &ex);
else
rc = dev_exception_add(devcgroup, &ex);
if (rc)
break;
/* we only propagate new restrictions */
rc = propagate_exception(devcgroup, &ex);
break;
default:
rc = -EINVAL;
}
return rc;
}
static ssize_t devcgroup_access_write(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
int retval;
mutex_lock(&devcgroup_mutex);
retval = devcgroup_update_access(css_to_devcgroup(of_css(of)),
of_cft(of)->private, strstrip(buf));
mutex_unlock(&devcgroup_mutex);
return retval ?: nbytes;
}
static struct cftype dev_cgroup_files[] = {
{
.name = "allow",
.write = devcgroup_access_write,
.private = DEVCG_ALLOW,
},
{
.name = "deny",
.write = devcgroup_access_write,
.private = DEVCG_DENY,
},
{
.name = "list",
.seq_show = devcgroup_seq_show,
.private = DEVCG_LIST,
},
{ } /* terminate */
};
struct cgroup_subsys devices_cgrp_subsys = {
.css_alloc = devcgroup_css_alloc,
.css_free = devcgroup_css_free,
.css_online = devcgroup_online,
.css_offline = devcgroup_offline,
.legacy_cftypes = dev_cgroup_files,
};
/**
* devcgroup_legacy_check_permission - checks if an inode operation is permitted
* @type: device type
* @major: device major number
* @minor: device minor number
* @access: combination of DEVCG_ACC_WRITE, DEVCG_ACC_READ and DEVCG_ACC_MKNOD
*
* returns 0 on success, -EPERM case the operation is not permitted
*/
static int devcgroup_legacy_check_permission(short type, u32 major, u32 minor,
short access)
{
struct dev_cgroup *dev_cgroup;
bool rc;
rcu_read_lock(); dev_cgroup = task_devcgroup(current);
if (dev_cgroup->behavior == DEVCG_DEFAULT_ALLOW)
/* Can't match any of the exceptions, even partially */
rc = !match_exception_partial(&dev_cgroup->exceptions,
type, major, minor, access);
else
/* Need to match completely one exception to be allowed */
rc = match_exception(&dev_cgroup->exceptions, type, major,
minor, access);
rcu_read_unlock();
if (!rc)
return -EPERM;
return 0;
}
#endif /* CONFIG_CGROUP_DEVICE */
#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
int devcgroup_check_permission(short type, u32 major, u32 minor, short access)
{
int rc = BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access);
if (rc)
return rc;
#ifdef CONFIG_CGROUP_DEVICE
return devcgroup_legacy_check_permission(type, major, minor, access);
#else /* CONFIG_CGROUP_DEVICE */
return 0;
#endif /* CONFIG_CGROUP_DEVICE */
}
EXPORT_SYMBOL(devcgroup_check_permission);
#endif /* defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF) */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_GENERIC_NETLINK_H
#define __NET_GENERIC_NETLINK_H
#include <linux/genetlink.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#define GENLMSG_DEFAULT_SIZE (NLMSG_DEFAULT_SIZE - GENL_HDRLEN)
/* Binding to multicast group requires %CAP_NET_ADMIN */
#define GENL_MCAST_CAP_NET_ADMIN BIT(0)
/* Binding to multicast group requires %CAP_SYS_ADMIN */
#define GENL_MCAST_CAP_SYS_ADMIN BIT(1)
/**
* struct genl_multicast_group - generic netlink multicast group
* @name: name of the multicast group, names are per-family
* @flags: GENL_MCAST_* flags
*/
struct genl_multicast_group {
char name[GENL_NAMSIZ];
u8 flags;
};
struct genl_split_ops;
struct genl_info;
/**
* struct genl_family - generic netlink family
* @hdrsize: length of user specific header in bytes
* @name: name of family
* @version: protocol version
* @maxattr: maximum number of attributes supported
* @policy: netlink policy
* @netnsok: set to true if the family can handle network
* namespaces and should be presented in all of them
* @parallel_ops: operations can be called in parallel and aren't
* synchronized by the core genetlink code
* @pre_doit: called before an operation's doit callback, it may
* do additional, common, filtering and return an error
* @post_doit: called after an operation's doit callback, it may
* undo operations done by pre_doit, for example release locks
* @bind: called when family multicast group is added to a netlink socket
* @unbind: called when family multicast group is removed from a netlink socket
* @module: pointer to the owning module (set to THIS_MODULE)
* @mcgrps: multicast groups used by this family
* @n_mcgrps: number of multicast groups
* @resv_start_op: first operation for which reserved fields of the header
* can be validated and policies are required (see below);
* new families should leave this field at zero
* @ops: the operations supported by this family
* @n_ops: number of operations supported by this family
* @small_ops: the small-struct operations supported by this family
* @n_small_ops: number of small-struct operations supported by this family
* @split_ops: the split do/dump form of operation definition
* @n_split_ops: number of entries in @split_ops, not that with split do/dump
* ops the number of entries is not the same as number of commands
* @sock_priv_size: the size of per-socket private memory
* @sock_priv_init: the per-socket private memory initializer
* @sock_priv_destroy: the per-socket private memory destructor
*
* Attribute policies (the combination of @policy and @maxattr fields)
* can be attached at the family level or at the operation level.
* If both are present the per-operation policy takes precedence.
* For operations before @resv_start_op lack of policy means that the core
* will perform no attribute parsing or validation. For newer operations
* if policy is not provided core will reject all TLV attributes.
*/
struct genl_family {
unsigned int hdrsize;
char name[GENL_NAMSIZ];
unsigned int version;
unsigned int maxattr;
u8 netnsok:1;
u8 parallel_ops:1;
u8 n_ops;
u8 n_small_ops;
u8 n_split_ops;
u8 n_mcgrps;
u8 resv_start_op;
const struct nla_policy *policy;
int (*pre_doit)(const struct genl_split_ops *ops,
struct sk_buff *skb,
struct genl_info *info);
void (*post_doit)(const struct genl_split_ops *ops,
struct sk_buff *skb,
struct genl_info *info);
int (*bind)(int mcgrp);
void (*unbind)(int mcgrp);
const struct genl_ops * ops;
const struct genl_small_ops *small_ops;
const struct genl_split_ops *split_ops;
const struct genl_multicast_group *mcgrps;
struct module *module;
size_t sock_priv_size;
void (*sock_priv_init)(void *priv);
void (*sock_priv_destroy)(void *priv);
/* private: internal use only */
/* protocol family identifier */
int id;
/* starting number of multicast group IDs in this family */
unsigned int mcgrp_offset;
/* list of per-socket privs */
struct xarray *sock_privs;
};
/**
* struct genl_info - receiving information
* @snd_seq: sending sequence number
* @snd_portid: netlink portid of sender
* @family: generic netlink family
* @nlhdr: netlink message header
* @genlhdr: generic netlink message header
* @attrs: netlink attributes
* @_net: network namespace
* @user_ptr: user pointers
* @extack: extended ACK report struct
*/
struct genl_info {
u32 snd_seq;
u32 snd_portid;
const struct genl_family *family;
const struct nlmsghdr * nlhdr;
struct genlmsghdr * genlhdr;
struct nlattr ** attrs;
possible_net_t _net;
void * user_ptr[2];
struct netlink_ext_ack *extack;
};
static inline struct net *genl_info_net(const struct genl_info *info)
{
return read_pnet(&info->_net);
}
static inline void genl_info_net_set(struct genl_info *info, struct net *net)
{
write_pnet(&info->_net, net);
}
static inline void *genl_info_userhdr(const struct genl_info *info)
{
return (u8 *)info->genlhdr + GENL_HDRLEN;
}
#define GENL_SET_ERR_MSG(info, msg) NL_SET_ERR_MSG((info)->extack, msg)
#define GENL_SET_ERR_MSG_FMT(info, msg, args...) \
NL_SET_ERR_MSG_FMT((info)->extack, msg, ##args)
/* Report that a root attribute is missing */
#define GENL_REQ_ATTR_CHECK(info, attr) ({ \
const struct genl_info *__info = (info); \
\
NL_REQ_ATTR_CHECK(__info->extack, NULL, __info->attrs, (attr)); \
})
enum genl_validate_flags {
GENL_DONT_VALIDATE_STRICT = BIT(0),
GENL_DONT_VALIDATE_DUMP = BIT(1),
GENL_DONT_VALIDATE_DUMP_STRICT = BIT(2),
};
/**
* struct genl_small_ops - generic netlink operations (small version)
* @cmd: command identifier
* @internal_flags: flags used by the family
* @flags: GENL_* flags (%GENL_ADMIN_PERM or %GENL_UNS_ADMIN_PERM)
* @validate: validation flags from enum genl_validate_flags
* @doit: standard command callback
* @dumpit: callback for dumpers
*
* This is a cut-down version of struct genl_ops for users who don't need
* most of the ancillary infra and want to save space.
*/
struct genl_small_ops {
int (*doit)(struct sk_buff *skb, struct genl_info *info);
int (*dumpit)(struct sk_buff *skb, struct netlink_callback *cb);
u8 cmd;
u8 internal_flags;
u8 flags;
u8 validate;
};
/**
* struct genl_ops - generic netlink operations
* @cmd: command identifier
* @internal_flags: flags used by the family
* @flags: GENL_* flags (%GENL_ADMIN_PERM or %GENL_UNS_ADMIN_PERM)
* @maxattr: maximum number of attributes supported
* @policy: netlink policy (takes precedence over family policy)
* @validate: validation flags from enum genl_validate_flags
* @doit: standard command callback
* @start: start callback for dumps
* @dumpit: callback for dumpers
* @done: completion callback for dumps
*/
struct genl_ops {
int (*doit)(struct sk_buff *skb,
struct genl_info *info);
int (*start)(struct netlink_callback *cb);
int (*dumpit)(struct sk_buff *skb,
struct netlink_callback *cb);
int (*done)(struct netlink_callback *cb);
const struct nla_policy *policy;
unsigned int maxattr;
u8 cmd;
u8 internal_flags;
u8 flags;
u8 validate;
};
/**
* struct genl_split_ops - generic netlink operations (do/dump split version)
* @cmd: command identifier
* @internal_flags: flags used by the family
* @flags: GENL_* flags (%GENL_ADMIN_PERM or %GENL_UNS_ADMIN_PERM)
* @validate: validation flags from enum genl_validate_flags
* @policy: netlink policy (takes precedence over family policy)
* @maxattr: maximum number of attributes supported
*
* Do callbacks:
* @pre_doit: called before an operation's @doit callback, it may
* do additional, common, filtering and return an error
* @doit: standard command callback
* @post_doit: called after an operation's @doit callback, it may
* undo operations done by pre_doit, for example release locks
*
* Dump callbacks:
* @start: start callback for dumps
* @dumpit: callback for dumpers
* @done: completion callback for dumps
*
* Do callbacks can be used if %GENL_CMD_CAP_DO is set in @flags.
* Dump callbacks can be used if %GENL_CMD_CAP_DUMP is set in @flags.
* Exactly one of those flags must be set.
*/
struct genl_split_ops {
union {
struct {
int (*pre_doit)(const struct genl_split_ops *ops,
struct sk_buff *skb,
struct genl_info *info);
int (*doit)(struct sk_buff *skb,
struct genl_info *info);
void (*post_doit)(const struct genl_split_ops *ops,
struct sk_buff *skb,
struct genl_info *info);
};
struct {
int (*start)(struct netlink_callback *cb);
int (*dumpit)(struct sk_buff *skb,
struct netlink_callback *cb);
int (*done)(struct netlink_callback *cb);
};
};
const struct nla_policy *policy;
unsigned int maxattr;
u8 cmd;
u8 internal_flags;
u8 flags;
u8 validate;
};
/**
* struct genl_dumpit_info - info that is available during dumpit op call
* @op: generic netlink ops - for internal genl code usage
* @attrs: netlink attributes
* @info: struct genl_info describing the request
*/
struct genl_dumpit_info {
struct genl_split_ops op;
struct genl_info info;
};
static inline const struct genl_dumpit_info *
genl_dumpit_info(struct netlink_callback *cb)
{
return cb->data;
}
static inline const struct genl_info *
genl_info_dump(struct netlink_callback *cb)
{
return &genl_dumpit_info(cb)->info;
}
/**
* genl_info_init_ntf() - initialize genl_info for notifications
* @info: genl_info struct to set up
* @family: pointer to the genetlink family
* @cmd: command to be used in the notification
*
* Initialize a locally declared struct genl_info to pass to various APIs.
* Intended to be used when creating notifications.
*/
static inline void
genl_info_init_ntf(struct genl_info *info, const struct genl_family *family,
u8 cmd)
{
struct genlmsghdr *hdr = (void *) &info->user_ptr[0];
memset(info, 0, sizeof(*info));
info->family = family;
info->genlhdr = hdr;
hdr->cmd = cmd;
}
static inline bool genl_info_is_ntf(const struct genl_info *info)
{
return !info->nlhdr;
}
void *__genl_sk_priv_get(struct genl_family *family, struct sock *sk);
void *genl_sk_priv_get(struct genl_family *family, struct sock *sk);
int genl_register_family(struct genl_family *family);
int genl_unregister_family(const struct genl_family *family);
void genl_notify(const struct genl_family *family, struct sk_buff *skb,
struct genl_info *info, u32 group, gfp_t flags);
void *genlmsg_put(struct sk_buff *skb, u32 portid, u32 seq,
const struct genl_family *family, int flags, u8 cmd);
static inline void *
__genlmsg_iput(struct sk_buff *skb, const struct genl_info *info, int flags)
{
return genlmsg_put(skb, info->snd_portid, info->snd_seq, info->family,
flags, info->genlhdr->cmd);
}
/**
* genlmsg_iput - start genetlink message based on genl_info
* @skb: skb in which message header will be placed
* @info: genl_info as provided to do/dump handlers
*
* Convenience wrapper which starts a genetlink message based on
* information in user request. @info should be either the struct passed
* by genetlink core to do/dump handlers (when constructing replies to
* such requests) or a struct initialized by genl_info_init_ntf()
* when constructing notifications.
*
* Returns pointer to new genetlink header.
*/
static inline void *
genlmsg_iput(struct sk_buff *skb, const struct genl_info *info)
{
return __genlmsg_iput(skb, info, 0);
}
/**
* genlmsg_nlhdr - Obtain netlink header from user specified header
* @user_hdr: user header as returned from genlmsg_put()
*
* Returns pointer to netlink header.
*/
static inline struct nlmsghdr *genlmsg_nlhdr(void *user_hdr)
{
return (struct nlmsghdr *)((char *)user_hdr -
GENL_HDRLEN -
NLMSG_HDRLEN);
}
/**
* genlmsg_parse_deprecated - parse attributes of a genetlink message
* @nlh: netlink message header
* @family: genetlink message family
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*/
static inline int genlmsg_parse_deprecated(const struct nlmsghdr *nlh,
const struct genl_family *family,
struct nlattr *tb[], int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nlmsg_parse(nlh, family->hdrsize + GENL_HDRLEN, tb, maxtype,
policy, NL_VALIDATE_LIBERAL, extack);
}
/**
* genlmsg_parse - parse attributes of a genetlink message
* @nlh: netlink message header
* @family: genetlink message family
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @extack: extended ACK report struct
*/
static inline int genlmsg_parse(const struct nlmsghdr *nlh,
const struct genl_family *family,
struct nlattr *tb[], int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack)
{
return __nlmsg_parse(nlh, family->hdrsize + GENL_HDRLEN, tb, maxtype,
policy, NL_VALIDATE_STRICT, extack);
}
/**
* genl_dump_check_consistent - check if sequence is consistent and advertise if not
* @cb: netlink callback structure that stores the sequence number
* @user_hdr: user header as returned from genlmsg_put()
*
* Cf. nl_dump_check_consistent(), this just provides a wrapper to make it
* simpler to use with generic netlink.
*/
static inline void genl_dump_check_consistent(struct netlink_callback *cb,
void *user_hdr)
{
nl_dump_check_consistent(cb, genlmsg_nlhdr(user_hdr));
}
/**
* genlmsg_put_reply - Add generic netlink header to a reply message
* @skb: socket buffer holding the message
* @info: receiver info
* @family: generic netlink family
* @flags: netlink message flags
* @cmd: generic netlink command
*
* Returns pointer to user specific header
*/
static inline void *genlmsg_put_reply(struct sk_buff *skb,
struct genl_info *info,
const struct genl_family *family,
int flags, u8 cmd)
{
return genlmsg_put(skb, info->snd_portid, info->snd_seq, family,
flags, cmd);
}
/**
* genlmsg_end - Finalize a generic netlink message
* @skb: socket buffer the message is stored in
* @hdr: user specific header
*/
static inline void genlmsg_end(struct sk_buff *skb, void *hdr)
{
nlmsg_end(skb, hdr - GENL_HDRLEN - NLMSG_HDRLEN);
}
/**
* genlmsg_cancel - Cancel construction of a generic netlink message
* @skb: socket buffer the message is stored in
* @hdr: generic netlink message header
*/
static inline void genlmsg_cancel(struct sk_buff *skb, void *hdr)
{
if (hdr)
nlmsg_cancel(skb, hdr - GENL_HDRLEN - NLMSG_HDRLEN);
}
/**
* genlmsg_multicast_netns_filtered - multicast a netlink message
* to a specific netns with filter
* function
* @family: the generic netlink family
* @net: the net namespace
* @skb: netlink message as socket buffer
* @portid: own netlink portid to avoid sending to yourself
* @group: offset of multicast group in groups array
* @flags: allocation flags
* @filter: filter function
* @filter_data: filter function private data
*
* Return: 0 on success, negative error code for failure.
*/
static inline int
genlmsg_multicast_netns_filtered(const struct genl_family *family,
struct net *net, struct sk_buff *skb,
u32 portid, unsigned int group, gfp_t flags,
netlink_filter_fn filter,
void *filter_data)
{
if (WARN_ON_ONCE(group >= family->n_mcgrps))
return -EINVAL;
group = family->mcgrp_offset + group; return nlmsg_multicast_filtered(net->genl_sock, skb, portid, group,
flags, filter, filter_data);
}
/**
* genlmsg_multicast_netns - multicast a netlink message to a specific netns
* @family: the generic netlink family
* @net: the net namespace
* @skb: netlink message as socket buffer
* @portid: own netlink portid to avoid sending to yourself
* @group: offset of multicast group in groups array
* @flags: allocation flags
*/
static inline int genlmsg_multicast_netns(const struct genl_family *family,
struct net *net, struct sk_buff *skb,
u32 portid, unsigned int group, gfp_t flags)
{
return genlmsg_multicast_netns_filtered(family, net, skb, portid,
group, flags, NULL, NULL);
}
/**
* genlmsg_multicast - multicast a netlink message to the default netns
* @family: the generic netlink family
* @skb: netlink message as socket buffer
* @portid: own netlink portid to avoid sending to yourself
* @group: offset of multicast group in groups array
* @flags: allocation flags
*/
static inline int genlmsg_multicast(const struct genl_family *family,
struct sk_buff *skb, u32 portid,
unsigned int group, gfp_t flags)
{
return genlmsg_multicast_netns(family, &init_net, skb,
portid, group, flags);
}
/**
* genlmsg_multicast_allns - multicast a netlink message to all net namespaces
* @family: the generic netlink family
* @skb: netlink message as socket buffer
* @portid: own netlink portid to avoid sending to yourself
* @group: offset of multicast group in groups array
* @flags: allocation flags
*
* This function must hold the RTNL or rcu_read_lock().
*/
int genlmsg_multicast_allns(const struct genl_family *family,
struct sk_buff *skb, u32 portid,
unsigned int group, gfp_t flags);
/**
* genlmsg_unicast - unicast a netlink message
* @net: network namespace to look up @portid in
* @skb: netlink message as socket buffer
* @portid: netlink portid of the destination socket
*/
static inline int genlmsg_unicast(struct net *net, struct sk_buff *skb, u32 portid)
{
return nlmsg_unicast(net->genl_sock, skb, portid);
}
/**
* genlmsg_reply - reply to a request
* @skb: netlink message to be sent back
* @info: receiver information
*/
static inline int genlmsg_reply(struct sk_buff *skb, struct genl_info *info)
{
return genlmsg_unicast(genl_info_net(info), skb, info->snd_portid);
}
/**
* genlmsg_data - head of message payload
* @gnlh: genetlink message header
*/
static inline void *genlmsg_data(const struct genlmsghdr *gnlh)
{
return ((unsigned char *) gnlh + GENL_HDRLEN);
}
/**
* genlmsg_len - length of message payload
* @gnlh: genetlink message header
*/
static inline int genlmsg_len(const struct genlmsghdr *gnlh)
{
struct nlmsghdr *nlh = (struct nlmsghdr *)((unsigned char *)gnlh -
NLMSG_HDRLEN);
return (nlh->nlmsg_len - GENL_HDRLEN - NLMSG_HDRLEN);
}
/**
* genlmsg_msg_size - length of genetlink message not including padding
* @payload: length of message payload
*/
static inline int genlmsg_msg_size(int payload)
{
return GENL_HDRLEN + payload;
}
/**
* genlmsg_total_size - length of genetlink message including padding
* @payload: length of message payload
*/
static inline int genlmsg_total_size(int payload)
{
return NLMSG_ALIGN(genlmsg_msg_size(payload));
}
/**
* genlmsg_new - Allocate a new generic netlink message
* @payload: size of the message payload
* @flags: the type of memory to allocate.
*/
static inline struct sk_buff *genlmsg_new(size_t payload, gfp_t flags)
{
return nlmsg_new(genlmsg_total_size(payload), flags);
}
/**
* genl_set_err - report error to genetlink broadcast listeners
* @family: the generic netlink family
* @net: the network namespace to report the error to
* @portid: the PORTID of a process that we want to skip (if any)
* @group: the broadcast group that will notice the error
* (this is the offset of the multicast group in the groups array)
* @code: error code, must be negative (as usual in kernelspace)
*
* This function returns the number of broadcast listeners that have set the
* NETLINK_RECV_NO_ENOBUFS socket option.
*/
static inline int genl_set_err(const struct genl_family *family,
struct net *net, u32 portid,
u32 group, int code)
{
if (WARN_ON_ONCE(group >= family->n_mcgrps))
return -EINVAL;
group = family->mcgrp_offset + group;
return netlink_set_err(net->genl_sock, portid, group, code);
}
static inline int genl_has_listeners(const struct genl_family *family,
struct net *net, unsigned int group)
{
if (WARN_ON_ONCE(group >= family->n_mcgrps))
return -EINVAL;
group = family->mcgrp_offset + group;
return netlink_has_listeners(net->genl_sock, group);
}
#endif /* __NET_GENERIC_NETLINK_H */
/*
* kernel/cpuset.c
*
* Processor and Memory placement constraints for sets of tasks.
*
* Copyright (C) 2003 BULL SA.
* Copyright (C) 2004-2007 Silicon Graphics, Inc.
* Copyright (C) 2006 Google, Inc
*
* Portions derived from Patrick Mochel's sysfs code.
* sysfs is Copyright (c) 2001-3 Patrick Mochel
*
* 2003-10-10 Written by Simon Derr.
* 2003-10-22 Updates by Stephen Hemminger.
* 2004 May-July Rework by Paul Jackson.
* 2006 Rework by Paul Menage to use generic cgroups
* 2008 Rework of the scheduler domains and CPU hotplug handling
* by Max Krasnyansky
*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file COPYING in the main directory of the Linux
* distribution for more details.
*/
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/cpuset.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/mempolicy.h>
#include <linux/mm.h>
#include <linux/memory.h>
#include <linux/export.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/sched/deadline.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/security.h>
#include <linux/spinlock.h>
#include <linux/oom.h>
#include <linux/sched/isolation.h>
#include <linux/cgroup.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
/*
* There could be abnormal cpuset configurations for cpu or memory
* node binding, add this key to provide a quick low-cost judgment
* of the situation.
*/
DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key);
/* See "Frequency meter" comments, below. */
struct fmeter {
int cnt; /* unprocessed events count */
int val; /* most recent output value */
time64_t time; /* clock (secs) when val computed */
spinlock_t lock; /* guards read or write of above */
};
/*
* Invalid partition error code
*/
enum prs_errcode {
PERR_NONE = 0,
PERR_INVCPUS,
PERR_INVPARENT,
PERR_NOTPART,
PERR_NOTEXCL,
PERR_NOCPUS,
PERR_HOTPLUG,
PERR_CPUSEMPTY,
PERR_HKEEPING,
};
static const char * const perr_strings[] = {
[PERR_INVCPUS] = "Invalid cpu list in cpuset.cpus.exclusive",
[PERR_INVPARENT] = "Parent is an invalid partition root",
[PERR_NOTPART] = "Parent is not a partition root",
[PERR_NOTEXCL] = "Cpu list in cpuset.cpus not exclusive",
[PERR_NOCPUS] = "Parent unable to distribute cpu downstream",
[PERR_HOTPLUG] = "No cpu available due to hotplug",
[PERR_CPUSEMPTY] = "cpuset.cpus is empty",
[PERR_HKEEPING] = "partition config conflicts with housekeeping setup",
};
struct cpuset {
struct cgroup_subsys_state css;
unsigned long flags; /* "unsigned long" so bitops work */
/*
* On default hierarchy:
*
* The user-configured masks can only be changed by writing to
* cpuset.cpus and cpuset.mems, and won't be limited by the
* parent masks.
*
* The effective masks is the real masks that apply to the tasks
* in the cpuset. They may be changed if the configured masks are
* changed or hotplug happens.
*
* effective_mask == configured_mask & parent's effective_mask,
* and if it ends up empty, it will inherit the parent's mask.
*
*
* On legacy hierarchy:
*
* The user-configured masks are always the same with effective masks.
*/
/* user-configured CPUs and Memory Nodes allow to tasks */
cpumask_var_t cpus_allowed;
nodemask_t mems_allowed;
/* effective CPUs and Memory Nodes allow to tasks */
cpumask_var_t effective_cpus;
nodemask_t effective_mems;
/*
* Exclusive CPUs dedicated to current cgroup (default hierarchy only)
*
* This exclusive CPUs must be a subset of cpus_allowed. A parent
* cgroup can only grant exclusive CPUs to one of its children.
*
* When the cgroup becomes a valid partition root, effective_xcpus
* defaults to cpus_allowed if not set. The effective_cpus of a valid
* partition root comes solely from its effective_xcpus and some of the
* effective_xcpus may be distributed to sub-partitions below & hence
* excluded from its effective_cpus.
*/
cpumask_var_t effective_xcpus;
/*
* Exclusive CPUs as requested by the user (default hierarchy only)
*/
cpumask_var_t exclusive_cpus;
/*
* This is old Memory Nodes tasks took on.
*
* - top_cpuset.old_mems_allowed is initialized to mems_allowed.
* - A new cpuset's old_mems_allowed is initialized when some
* task is moved into it.
* - old_mems_allowed is used in cpuset_migrate_mm() when we change
* cpuset.mems_allowed and have tasks' nodemask updated, and
* then old_mems_allowed is updated to mems_allowed.
*/
nodemask_t old_mems_allowed;
struct fmeter fmeter; /* memory_pressure filter */
/*
* Tasks are being attached to this cpuset. Used to prevent
* zeroing cpus/mems_allowed between ->can_attach() and ->attach().
*/
int attach_in_progress;
/* partition number for rebuild_sched_domains() */
int pn;
/* for custom sched domain */
int relax_domain_level;
/* number of valid sub-partitions */
int nr_subparts;
/* partition root state */
int partition_root_state;
/*
* Default hierarchy only:
* use_parent_ecpus - set if using parent's effective_cpus
* child_ecpus_count - # of children with use_parent_ecpus set
*/
int use_parent_ecpus;
int child_ecpus_count;
/*
* number of SCHED_DEADLINE tasks attached to this cpuset, so that we
* know when to rebuild associated root domain bandwidth information.
*/
int nr_deadline_tasks;
int nr_migrate_dl_tasks;
u64 sum_migrate_dl_bw;
/* Invalid partition error code, not lock protected */
enum prs_errcode prs_err;
/* Handle for cpuset.cpus.partition */
struct cgroup_file partition_file;
/* Remote partition silbling list anchored at remote_children */
struct list_head remote_sibling;
};
/*
* Exclusive CPUs distributed out to sub-partitions of top_cpuset
*/
static cpumask_var_t subpartitions_cpus;
/*
* Exclusive CPUs in isolated partitions
*/
static cpumask_var_t isolated_cpus;
/* List of remote partition root children */
static struct list_head remote_children;
/*
* Partition root states:
*
* 0 - member (not a partition root)
* 1 - partition root
* 2 - partition root without load balancing (isolated)
* -1 - invalid partition root
* -2 - invalid isolated partition root
*/
#define PRS_MEMBER 0
#define PRS_ROOT 1
#define PRS_ISOLATED 2
#define PRS_INVALID_ROOT -1
#define PRS_INVALID_ISOLATED -2
static inline bool is_prs_invalid(int prs_state)
{
return prs_state < 0;
}
/*
* Temporary cpumasks for working with partitions that are passed among
* functions to avoid memory allocation in inner functions.
*/
struct tmpmasks {
cpumask_var_t addmask, delmask; /* For partition root */
cpumask_var_t new_cpus; /* For update_cpumasks_hier() */
};
static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
{
return css ? container_of(css, struct cpuset, css) : NULL;
}
/* Retrieve the cpuset for a task */
static inline struct cpuset *task_cs(struct task_struct *task)
{
return css_cs(task_css(task, cpuset_cgrp_id));
}
static inline struct cpuset *parent_cs(struct cpuset *cs)
{
return css_cs(cs->css.parent);
}
void inc_dl_tasks_cs(struct task_struct *p)
{
struct cpuset *cs = task_cs(p);
cs->nr_deadline_tasks++;
}
void dec_dl_tasks_cs(struct task_struct *p)
{
struct cpuset *cs = task_cs(p);
cs->nr_deadline_tasks--;
}
/* bits in struct cpuset flags field */
typedef enum {
CS_ONLINE,
CS_CPU_EXCLUSIVE,
CS_MEM_EXCLUSIVE,
CS_MEM_HARDWALL,
CS_MEMORY_MIGRATE,
CS_SCHED_LOAD_BALANCE,
CS_SPREAD_PAGE,
CS_SPREAD_SLAB,
} cpuset_flagbits_t;
/* convenient tests for these bits */
static inline bool is_cpuset_online(struct cpuset *cs)
{
return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css);
}
static inline int is_cpu_exclusive(const struct cpuset *cs)
{
return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
}
static inline int is_mem_exclusive(const struct cpuset *cs)
{
return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
}
static inline int is_mem_hardwall(const struct cpuset *cs)
{
return test_bit(CS_MEM_HARDWALL, &cs->flags);
}
static inline int is_sched_load_balance(const struct cpuset *cs)
{
return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
}
static inline int is_memory_migrate(const struct cpuset *cs)
{
return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
}
static inline int is_spread_page(const struct cpuset *cs)
{
return test_bit(CS_SPREAD_PAGE, &cs->flags);
}
static inline int is_spread_slab(const struct cpuset *cs)
{
return test_bit(CS_SPREAD_SLAB, &cs->flags);
}
static inline int is_partition_valid(const struct cpuset *cs)
{
return cs->partition_root_state > 0;
}
static inline int is_partition_invalid(const struct cpuset *cs)
{
return cs->partition_root_state < 0;
}
/*
* Callers should hold callback_lock to modify partition_root_state.
*/
static inline void make_partition_invalid(struct cpuset *cs)
{
if (cs->partition_root_state > 0)
cs->partition_root_state = -cs->partition_root_state;
}
/*
* Send notification event of whenever partition_root_state changes.
*/
static inline void notify_partition_change(struct cpuset *cs, int old_prs)
{
if (old_prs == cs->partition_root_state)
return;
cgroup_file_notify(&cs->partition_file);
/* Reset prs_err if not invalid */
if (is_partition_valid(cs))
WRITE_ONCE(cs->prs_err, PERR_NONE);
}
static struct cpuset top_cpuset = {
.flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
(1 << CS_MEM_EXCLUSIVE)),
.partition_root_state = PRS_ROOT,
.remote_sibling = LIST_HEAD_INIT(top_cpuset.remote_sibling),
};
/**
* cpuset_for_each_child - traverse online children of a cpuset
* @child_cs: loop cursor pointing to the current child
* @pos_css: used for iteration
* @parent_cs: target cpuset to walk children of
*
* Walk @child_cs through the online children of @parent_cs. Must be used
* with RCU read locked.
*/
#define cpuset_for_each_child(child_cs, pos_css, parent_cs) \
css_for_each_child((pos_css), &(parent_cs)->css) \
if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
/**
* cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
* @des_cs: loop cursor pointing to the current descendant
* @pos_css: used for iteration
* @root_cs: target cpuset to walk ancestor of
*
* Walk @des_cs through the online descendants of @root_cs. Must be used
* with RCU read locked. The caller may modify @pos_css by calling
* css_rightmost_descendant() to skip subtree. @root_cs is included in the
* iteration and the first node to be visited.
*/
#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \
css_for_each_descendant_pre((pos_css), &(root_cs)->css) \
if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
/*
* There are two global locks guarding cpuset structures - cpuset_mutex and
* callback_lock. We also require taking task_lock() when dereferencing a
* task's cpuset pointer. See "The task_lock() exception", at the end of this
* comment. The cpuset code uses only cpuset_mutex. Other kernel subsystems
* can use cpuset_lock()/cpuset_unlock() to prevent change to cpuset
* structures. Note that cpuset_mutex needs to be a mutex as it is used in
* paths that rely on priority inheritance (e.g. scheduler - on RT) for
* correctness.
*
* A task must hold both locks to modify cpusets. If a task holds
* cpuset_mutex, it blocks others, ensuring that it is the only task able to
* also acquire callback_lock and be able to modify cpusets. It can perform
* various checks on the cpuset structure first, knowing nothing will change.
* It can also allocate memory while just holding cpuset_mutex. While it is
* performing these checks, various callback routines can briefly acquire
* callback_lock to query cpusets. Once it is ready to make the changes, it
* takes callback_lock, blocking everyone else.
*
* Calls to the kernel memory allocator can not be made while holding
* callback_lock, as that would risk double tripping on callback_lock
* from one of the callbacks into the cpuset code from within
* __alloc_pages().
*
* If a task is only holding callback_lock, then it has read-only
* access to cpusets.
*
* Now, the task_struct fields mems_allowed and mempolicy may be changed
* by other task, we use alloc_lock in the task_struct fields to protect
* them.
*
* The cpuset_common_file_read() handlers only hold callback_lock across
* small pieces of code, such as when reading out possibly multi-word
* cpumasks and nodemasks.
*
* Accessing a task's cpuset should be done in accordance with the
* guidelines for accessing subsystem state in kernel/cgroup.c
*/
static DEFINE_MUTEX(cpuset_mutex);
void cpuset_lock(void)
{
mutex_lock(&cpuset_mutex);
}
void cpuset_unlock(void)
{
mutex_unlock(&cpuset_mutex);
}
static DEFINE_SPINLOCK(callback_lock);
static struct workqueue_struct *cpuset_migrate_mm_wq;
/*
* CPU / memory hotplug is handled asynchronously.
*/
static void cpuset_hotplug_workfn(struct work_struct *work);
static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);
static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
static inline void check_insane_mems_config(nodemask_t *nodes)
{
if (!cpusets_insane_config() &&
movable_only_nodes(nodes)) {
static_branch_enable(&cpusets_insane_config_key);
pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n"
"Cpuset allocations might fail even with a lot of memory available.\n",
nodemask_pr_args(nodes));
}
}
/*
* Cgroup v2 behavior is used on the "cpus" and "mems" control files when
* on default hierarchy or when the cpuset_v2_mode flag is set by mounting
* the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option.
* With v2 behavior, "cpus" and "mems" are always what the users have
* requested and won't be changed by hotplug events. Only the effective
* cpus or mems will be affected.
*/
static inline bool is_in_v2_mode(void)
{
return cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
(cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE);
}
/**
* partition_is_populated - check if partition has tasks
* @cs: partition root to be checked
* @excluded_child: a child cpuset to be excluded in task checking
* Return: true if there are tasks, false otherwise
*
* It is assumed that @cs is a valid partition root. @excluded_child should
* be non-NULL when this cpuset is going to become a partition itself.
*/
static inline bool partition_is_populated(struct cpuset *cs,
struct cpuset *excluded_child)
{
struct cgroup_subsys_state *css;
struct cpuset *child;
if (cs->css.cgroup->nr_populated_csets)
return true;
if (!excluded_child && !cs->nr_subparts)
return cgroup_is_populated(cs->css.cgroup);
rcu_read_lock();
cpuset_for_each_child(child, css, cs) {
if (child == excluded_child)
continue;
if (is_partition_valid(child))
continue;
if (cgroup_is_populated(child->css.cgroup)) {
rcu_read_unlock();
return true;
}
}
rcu_read_unlock();
return false;
}
/*
* Return in pmask the portion of a task's cpusets's cpus_allowed that
* are online and are capable of running the task. If none are found,
* walk up the cpuset hierarchy until we find one that does have some
* appropriate cpus.
*
* One way or another, we guarantee to return some non-empty subset
* of cpu_online_mask.
*
* Call with callback_lock or cpuset_mutex held.
*/
static void guarantee_online_cpus(struct task_struct *tsk,
struct cpumask *pmask)
{
const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
struct cpuset *cs;
if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_online_mask)))
cpumask_copy(pmask, cpu_online_mask);
rcu_read_lock();
cs = task_cs(tsk);
while (!cpumask_intersects(cs->effective_cpus, pmask)) {
cs = parent_cs(cs);
if (unlikely(!cs)) {
/*
* The top cpuset doesn't have any online cpu as a
* consequence of a race between cpuset_hotplug_work
* and cpu hotplug notifier. But we know the top
* cpuset's effective_cpus is on its way to be
* identical to cpu_online_mask.
*/
goto out_unlock;
}
}
cpumask_and(pmask, pmask, cs->effective_cpus);
out_unlock:
rcu_read_unlock();
}
/*
* Return in *pmask the portion of a cpusets's mems_allowed that
* are online, with memory. If none are online with memory, walk
* up the cpuset hierarchy until we find one that does have some
* online mems. The top cpuset always has some mems online.
*
* One way or another, we guarantee to return some non-empty subset
* of node_states[N_MEMORY].
*
* Call with callback_lock or cpuset_mutex held.
*/
static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
{
while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
cs = parent_cs(cs);
nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
}
/*
* update task's spread flag if cpuset's page/slab spread flag is set
*
* Call with callback_lock or cpuset_mutex held. The check can be skipped
* if on default hierarchy.
*/
static void cpuset_update_task_spread_flags(struct cpuset *cs,
struct task_struct *tsk)
{
if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys))
return;
if (is_spread_page(cs))
task_set_spread_page(tsk);
else
task_clear_spread_page(tsk);
if (is_spread_slab(cs))
task_set_spread_slab(tsk);
else
task_clear_spread_slab(tsk);
}
/*
* is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
*
* One cpuset is a subset of another if all its allowed CPUs and
* Memory Nodes are a subset of the other, and its exclusive flags
* are only set if the other's are set. Call holding cpuset_mutex.
*/
static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
{
return cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
nodes_subset(p->mems_allowed, q->mems_allowed) &&
is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
is_mem_exclusive(p) <= is_mem_exclusive(q);
}
/**
* alloc_cpumasks - allocate three cpumasks for cpuset
* @cs: the cpuset that have cpumasks to be allocated.
* @tmp: the tmpmasks structure pointer
* Return: 0 if successful, -ENOMEM otherwise.
*
* Only one of the two input arguments should be non-NULL.
*/
static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
{
cpumask_var_t *pmask1, *pmask2, *pmask3, *pmask4;
if (cs) {
pmask1 = &cs->cpus_allowed;
pmask2 = &cs->effective_cpus;
pmask3 = &cs->effective_xcpus;
pmask4 = &cs->exclusive_cpus;
} else {
pmask1 = &tmp->new_cpus;
pmask2 = &tmp->addmask;
pmask3 = &tmp->delmask;
pmask4 = NULL;
}
if (!zalloc_cpumask_var(pmask1, GFP_KERNEL))
return -ENOMEM;
if (!zalloc_cpumask_var(pmask2, GFP_KERNEL))
goto free_one;
if (!zalloc_cpumask_var(pmask3, GFP_KERNEL))
goto free_two;
if (pmask4 && !zalloc_cpumask_var(pmask4, GFP_KERNEL))
goto free_three;
return 0;
free_three:
free_cpumask_var(*pmask3);
free_two:
free_cpumask_var(*pmask2);
free_one:
free_cpumask_var(*pmask1);
return -ENOMEM;
}
/**
* free_cpumasks - free cpumasks in a tmpmasks structure
* @cs: the cpuset that have cpumasks to be free.
* @tmp: the tmpmasks structure pointer
*/
static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
{
if (cs) {
free_cpumask_var(cs->cpus_allowed);
free_cpumask_var(cs->effective_cpus);
free_cpumask_var(cs->effective_xcpus);
free_cpumask_var(cs->exclusive_cpus);
}
if (tmp) {
free_cpumask_var(tmp->new_cpus);
free_cpumask_var(tmp->addmask);
free_cpumask_var(tmp->delmask);
}
}
/**
* alloc_trial_cpuset - allocate a trial cpuset
* @cs: the cpuset that the trial cpuset duplicates
*/
static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
{
struct cpuset *trial;
trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
if (!trial)
return NULL;
if (alloc_cpumasks(trial, NULL)) {
kfree(trial);
return NULL;
}
cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
cpumask_copy(trial->effective_cpus, cs->effective_cpus);
cpumask_copy(trial->effective_xcpus, cs->effective_xcpus);
cpumask_copy(trial->exclusive_cpus, cs->exclusive_cpus);
return trial;
}
/**
* free_cpuset - free the cpuset
* @cs: the cpuset to be freed
*/
static inline void free_cpuset(struct cpuset *cs)
{
free_cpumasks(cs, NULL);
kfree(cs);
}
static inline struct cpumask *fetch_xcpus(struct cpuset *cs)
{
return !cpumask_empty(cs->exclusive_cpus) ? cs->exclusive_cpus :
cpumask_empty(cs->effective_xcpus) ? cs->cpus_allowed
: cs->effective_xcpus;
}
/*
* cpusets_are_exclusive() - check if two cpusets are exclusive
*
* Return true if exclusive, false if not
*/
static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2)
{
struct cpumask *xcpus1 = fetch_xcpus(cs1);
struct cpumask *xcpus2 = fetch_xcpus(cs2);
if (cpumask_intersects(xcpus1, xcpus2))
return false;
return true;
}
/*
* validate_change_legacy() - Validate conditions specific to legacy (v1)
* behavior.
*/
static int validate_change_legacy(struct cpuset *cur, struct cpuset *trial)
{
struct cgroup_subsys_state *css;
struct cpuset *c, *par;
int ret;
WARN_ON_ONCE(!rcu_read_lock_held());
/* Each of our child cpusets must be a subset of us */
ret = -EBUSY;
cpuset_for_each_child(c, css, cur)
if (!is_cpuset_subset(c, trial))
goto out;
/* On legacy hierarchy, we must be a subset of our parent cpuset. */
ret = -EACCES;
par = parent_cs(cur);
if (par && !is_cpuset_subset(trial, par))
goto out;
ret = 0;
out:
return ret;
}
/*
* validate_change() - Used to validate that any proposed cpuset change
* follows the structural rules for cpusets.
*
* If we replaced the flag and mask values of the current cpuset
* (cur) with those values in the trial cpuset (trial), would
* our various subset and exclusive rules still be valid? Presumes
* cpuset_mutex held.
*
* 'cur' is the address of an actual, in-use cpuset. Operations
* such as list traversal that depend on the actual address of the
* cpuset in the list must use cur below, not trial.
*
* 'trial' is the address of bulk structure copy of cur, with
* perhaps one or more of the fields cpus_allowed, mems_allowed,
* or flags changed to new, trial values.
*
* Return 0 if valid, -errno if not.
*/
static int validate_change(struct cpuset *cur, struct cpuset *trial)
{
struct cgroup_subsys_state *css;
struct cpuset *c, *par;
int ret = 0;
rcu_read_lock();
if (!is_in_v2_mode())
ret = validate_change_legacy(cur, trial);
if (ret)
goto out;
/* Remaining checks don't apply to root cpuset */
if (cur == &top_cpuset)
goto out;
par = parent_cs(cur);
/*
* Cpusets with tasks - existing or newly being attached - can't
* be changed to have empty cpus_allowed or mems_allowed.
*/
ret = -ENOSPC;
if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) {
if (!cpumask_empty(cur->cpus_allowed) &&
cpumask_empty(trial->cpus_allowed))
goto out;
if (!nodes_empty(cur->mems_allowed) &&
nodes_empty(trial->mems_allowed))
goto out;
}
/*
* We can't shrink if we won't have enough room for SCHED_DEADLINE
* tasks.
*/
ret = -EBUSY;
if (is_cpu_exclusive(cur) &&
!cpuset_cpumask_can_shrink(cur->cpus_allowed,
trial->cpus_allowed))
goto out;
/*
* If either I or some sibling (!= me) is exclusive, we can't
* overlap
*/
ret = -EINVAL;
cpuset_for_each_child(c, css, par) {
if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
c != cur) {
if (!cpusets_are_exclusive(trial, c))
goto out;
}
if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
c != cur &&
nodes_intersects(trial->mems_allowed, c->mems_allowed))
goto out;
}
ret = 0;
out:
rcu_read_unlock();
return ret;
}
#ifdef CONFIG_SMP
/*
* Helper routine for generate_sched_domains().
* Do cpusets a, b have overlapping effective cpus_allowed masks?
*/
static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
{
return cpumask_intersects(a->effective_cpus, b->effective_cpus);
}
static void
update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
{
if (dattr->relax_domain_level < c->relax_domain_level)
dattr->relax_domain_level = c->relax_domain_level;
return;
}
static void update_domain_attr_tree(struct sched_domain_attr *dattr,
struct cpuset *root_cs)
{
struct cpuset *cp;
struct cgroup_subsys_state *pos_css;
rcu_read_lock();
cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
/* skip the whole subtree if @cp doesn't have any CPU */
if (cpumask_empty(cp->cpus_allowed)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
if (is_sched_load_balance(cp))
update_domain_attr(dattr, cp);
}
rcu_read_unlock();
}
/* Must be called with cpuset_mutex held. */
static inline int nr_cpusets(void)
{
/* jump label reference count + the top-level cpuset */
return static_key_count(&cpusets_enabled_key.key) + 1;
}
/*
* generate_sched_domains()
*
* This function builds a partial partition of the systems CPUs
* A 'partial partition' is a set of non-overlapping subsets whose
* union is a subset of that set.
* The output of this function needs to be passed to kernel/sched/core.c
* partition_sched_domains() routine, which will rebuild the scheduler's
* load balancing domains (sched domains) as specified by that partial
* partition.
*
* See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
* for a background explanation of this.
*
* Does not return errors, on the theory that the callers of this
* routine would rather not worry about failures to rebuild sched
* domains when operating in the severe memory shortage situations
* that could cause allocation failures below.
*
* Must be called with cpuset_mutex held.
*
* The three key local variables below are:
* cp - cpuset pointer, used (together with pos_css) to perform a
* top-down scan of all cpusets. For our purposes, rebuilding
* the schedulers sched domains, we can ignore !is_sched_load_
* balance cpusets.
* csa - (for CpuSet Array) Array of pointers to all the cpusets
* that need to be load balanced, for convenient iterative
* access by the subsequent code that finds the best partition,
* i.e the set of domains (subsets) of CPUs such that the
* cpus_allowed of every cpuset marked is_sched_load_balance
* is a subset of one of these domains, while there are as
* many such domains as possible, each as small as possible.
* doms - Conversion of 'csa' to an array of cpumasks, for passing to
* the kernel/sched/core.c routine partition_sched_domains() in a
* convenient format, that can be easily compared to the prior
* value to determine what partition elements (sched domains)
* were changed (added or removed.)
*
* Finding the best partition (set of domains):
* The triple nested loops below over i, j, k scan over the
* load balanced cpusets (using the array of cpuset pointers in
* csa[]) looking for pairs of cpusets that have overlapping
* cpus_allowed, but which don't have the same 'pn' partition
* number and gives them in the same partition number. It keeps
* looping on the 'restart' label until it can no longer find
* any such pairs.
*
* The union of the cpus_allowed masks from the set of
* all cpusets having the same 'pn' value then form the one
* element of the partition (one sched domain) to be passed to
* partition_sched_domains().
*/
static int generate_sched_domains(cpumask_var_t **domains,
struct sched_domain_attr **attributes)
{
struct cpuset *cp; /* top-down scan of cpusets */
struct cpuset **csa; /* array of all cpuset ptrs */
int csn; /* how many cpuset ptrs in csa so far */
int i, j, k; /* indices for partition finding loops */
cpumask_var_t *doms; /* resulting partition; i.e. sched domains */
struct sched_domain_attr *dattr; /* attributes for custom domains */
int ndoms = 0; /* number of sched domains in result */
int nslot; /* next empty doms[] struct cpumask slot */
struct cgroup_subsys_state *pos_css;
bool root_load_balance = is_sched_load_balance(&top_cpuset);
doms = NULL;
dattr = NULL;
csa = NULL;
/* Special case for the 99% of systems with one, full, sched domain */
if (root_load_balance && !top_cpuset.nr_subparts) {
ndoms = 1;
doms = alloc_sched_domains(ndoms);
if (!doms)
goto done;
dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
if (dattr) {
*dattr = SD_ATTR_INIT;
update_domain_attr_tree(dattr, &top_cpuset);
}
cpumask_and(doms[0], top_cpuset.effective_cpus,
housekeeping_cpumask(HK_TYPE_DOMAIN));
goto done;
}
csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL);
if (!csa)
goto done;
csn = 0;
rcu_read_lock();
if (root_load_balance)
csa[csn++] = &top_cpuset;
cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
if (cp == &top_cpuset)
continue;
/*
* Continue traversing beyond @cp iff @cp has some CPUs and
* isn't load balancing. The former is obvious. The
* latter: All child cpusets contain a subset of the
* parent's cpus, so just skip them, and then we call
* update_domain_attr_tree() to calc relax_domain_level of
* the corresponding sched domain.
*
* If root is load-balancing, we can skip @cp if it
* is a subset of the root's effective_cpus.
*/
if (!cpumask_empty(cp->cpus_allowed) &&
!(is_sched_load_balance(cp) &&
cpumask_intersects(cp->cpus_allowed,
housekeeping_cpumask(HK_TYPE_DOMAIN))))
continue;
if (root_load_balance &&
cpumask_subset(cp->cpus_allowed, top_cpuset.effective_cpus))
continue;
if (is_sched_load_balance(cp) &&
!cpumask_empty(cp->effective_cpus))
csa[csn++] = cp;
/* skip @cp's subtree if not a partition root */
if (!is_partition_valid(cp))
pos_css = css_rightmost_descendant(pos_css);
}
rcu_read_unlock();
for (i = 0; i < csn; i++)
csa[i]->pn = i;
ndoms = csn;
restart:
/* Find the best partition (set of sched domains) */
for (i = 0; i < csn; i++) {
struct cpuset *a = csa[i];
int apn = a->pn;
for (j = 0; j < csn; j++) {
struct cpuset *b = csa[j];
int bpn = b->pn;
if (apn != bpn && cpusets_overlap(a, b)) {
for (k = 0; k < csn; k++) {
struct cpuset *c = csa[k];
if (c->pn == bpn)
c->pn = apn;
}
ndoms--; /* one less element */
goto restart;
}
}
}
/*
* Now we know how many domains to create.
* Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
*/
doms = alloc_sched_domains(ndoms);
if (!doms)
goto done;
/*
* The rest of the code, including the scheduler, can deal with
* dattr==NULL case. No need to abort if alloc fails.
*/
dattr = kmalloc_array(ndoms, sizeof(struct sched_domain_attr),
GFP_KERNEL);
for (nslot = 0, i = 0; i < csn; i++) {
struct cpuset *a = csa[i];
struct cpumask *dp;
int apn = a->pn;
if (apn < 0) {
/* Skip completed partitions */
continue;
}
dp = doms[nslot];
if (nslot == ndoms) {
static int warnings = 10;
if (warnings) {
pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n",
nslot, ndoms, csn, i, apn);
warnings--;
}
continue;
}
cpumask_clear(dp);
if (dattr)
*(dattr + nslot) = SD_ATTR_INIT;
for (j = i; j < csn; j++) {
struct cpuset *b = csa[j];
if (apn == b->pn) {
cpumask_or(dp, dp, b->effective_cpus);
cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN));
if (dattr)
update_domain_attr_tree(dattr + nslot, b);
/* Done with this partition */
b->pn = -1;
}
}
nslot++;
}
BUG_ON(nslot != ndoms);
done:
kfree(csa);
/*
* Fallback to the default domain if kmalloc() failed.
* See comments in partition_sched_domains().
*/
if (doms == NULL)
ndoms = 1;
*domains = doms;
*attributes = dattr;
return ndoms;
}
static void dl_update_tasks_root_domain(struct cpuset *cs)
{
struct css_task_iter it;
struct task_struct *task;
if (cs->nr_deadline_tasks == 0)
return;
css_task_iter_start(&cs->css, 0, &it);
while ((task = css_task_iter_next(&it)))
dl_add_task_root_domain(task);
css_task_iter_end(&it);
}
static void dl_rebuild_rd_accounting(void)
{
struct cpuset *cs = NULL;
struct cgroup_subsys_state *pos_css;
lockdep_assert_held(&cpuset_mutex);
lockdep_assert_cpus_held();
lockdep_assert_held(&sched_domains_mutex);
rcu_read_lock();
/*
* Clear default root domain DL accounting, it will be computed again
* if a task belongs to it.
*/
dl_clear_root_domain(&def_root_domain);
cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
if (cpumask_empty(cs->effective_cpus)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
css_get(&cs->css);
rcu_read_unlock();
dl_update_tasks_root_domain(cs);
rcu_read_lock();
css_put(&cs->css);
}
rcu_read_unlock();
}
static void
partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
struct sched_domain_attr *dattr_new)
{
mutex_lock(&sched_domains_mutex);
partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
dl_rebuild_rd_accounting();
mutex_unlock(&sched_domains_mutex);
}
/*
* Rebuild scheduler domains.
*
* If the flag 'sched_load_balance' of any cpuset with non-empty
* 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
* which has that flag enabled, or if any cpuset with a non-empty
* 'cpus' is removed, then call this routine to rebuild the
* scheduler's dynamic sched domains.
*
* Call with cpuset_mutex held. Takes cpus_read_lock().
*/
static void rebuild_sched_domains_locked(void)
{
struct cgroup_subsys_state *pos_css;
struct sched_domain_attr *attr;
cpumask_var_t *doms;
struct cpuset *cs;
int ndoms;
lockdep_assert_cpus_held();
lockdep_assert_held(&cpuset_mutex);
/*
* If we have raced with CPU hotplug, return early to avoid
* passing doms with offlined cpu to partition_sched_domains().
* Anyways, cpuset_hotplug_workfn() will rebuild sched domains.
*
* With no CPUs in any subpartitions, top_cpuset's effective CPUs
* should be the same as the active CPUs, so checking only top_cpuset
* is enough to detect racing CPU offlines.
*/
if (cpumask_empty(subpartitions_cpus) &&
!cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask))
return;
/*
* With subpartition CPUs, however, the effective CPUs of a partition
* root should be only a subset of the active CPUs. Since a CPU in any
* partition root could be offlined, all must be checked.
*/
if (top_cpuset.nr_subparts) {
rcu_read_lock();
cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
if (!is_partition_valid(cs)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
if (!cpumask_subset(cs->effective_cpus,
cpu_active_mask)) {
rcu_read_unlock();
return;
}
}
rcu_read_unlock();
}
/* Generate domain masks and attrs */
ndoms = generate_sched_domains(&doms, &attr);
/* Have scheduler rebuild the domains */
partition_and_rebuild_sched_domains(ndoms, doms, attr);
}
#else /* !CONFIG_SMP */
static void rebuild_sched_domains_locked(void)
{
}
#endif /* CONFIG_SMP */
void rebuild_sched_domains(void)
{
cpus_read_lock();
mutex_lock(&cpuset_mutex);
rebuild_sched_domains_locked();
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
}
/**
* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
* @new_cpus: the temp variable for the new effective_cpus mask
*
* Iterate through each task of @cs updating its cpus_allowed to the
* effective cpuset's. As this function is called with cpuset_mutex held,
* cpuset membership stays stable. For top_cpuset, task_cpu_possible_mask()
* is used instead of effective_cpus to make sure all offline CPUs are also
* included as hotplug code won't update cpumasks for tasks in top_cpuset.
*/
static void update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
{
struct css_task_iter it;
struct task_struct *task;
bool top_cs = cs == &top_cpuset;
css_task_iter_start(&cs->css, 0, &it);
while ((task = css_task_iter_next(&it))) {
const struct cpumask *possible_mask = task_cpu_possible_mask(task);
if (top_cs) {
/*
* Percpu kthreads in top_cpuset are ignored
*/
if (kthread_is_per_cpu(task))
continue;
cpumask_andnot(new_cpus, possible_mask, subpartitions_cpus);
} else {
cpumask_and(new_cpus, possible_mask, cs->effective_cpus);
}
set_cpus_allowed_ptr(task, new_cpus);
}
css_task_iter_end(&it);
}
/**
* compute_effective_cpumask - Compute the effective cpumask of the cpuset
* @new_cpus: the temp variable for the new effective_cpus mask
* @cs: the cpuset the need to recompute the new effective_cpus mask
* @parent: the parent cpuset
*
* The result is valid only if the given cpuset isn't a partition root.
*/
static void compute_effective_cpumask(struct cpumask *new_cpus,
struct cpuset *cs, struct cpuset *parent)
{
cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
}
/*
* Commands for update_parent_effective_cpumask
*/
enum partition_cmd {
partcmd_enable, /* Enable partition root */
partcmd_enablei, /* Enable isolated partition root */
partcmd_disable, /* Disable partition root */
partcmd_update, /* Update parent's effective_cpus */
partcmd_invalidate, /* Make partition invalid */
};
static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
int turning_on);
static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
struct tmpmasks *tmp);
/*
* Update partition exclusive flag
*
* Return: 0 if successful, an error code otherwise
*/
static int update_partition_exclusive(struct cpuset *cs, int new_prs)
{
bool exclusive = (new_prs > 0);
if (exclusive && !is_cpu_exclusive(cs)) {
if (update_flag(CS_CPU_EXCLUSIVE, cs, 1))
return PERR_NOTEXCL;
} else if (!exclusive && is_cpu_exclusive(cs)) {
/* Turning off CS_CPU_EXCLUSIVE will not return error */
update_flag(CS_CPU_EXCLUSIVE, cs, 0);
}
return 0;
}
/*
* Update partition load balance flag and/or rebuild sched domain
*
* Changing load balance flag will automatically call
* rebuild_sched_domains_locked().
* This function is for cgroup v2 only.
*/
static void update_partition_sd_lb(struct cpuset *cs, int old_prs)
{
int new_prs = cs->partition_root_state;
bool rebuild_domains = (new_prs > 0) || (old_prs > 0);
bool new_lb;
/*
* If cs is not a valid partition root, the load balance state
* will follow its parent.
*/
if (new_prs > 0) {
new_lb = (new_prs != PRS_ISOLATED);
} else {
new_lb = is_sched_load_balance(parent_cs(cs));
}
if (new_lb != !!is_sched_load_balance(cs)) {
rebuild_domains = true;
if (new_lb)
set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
else
clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
}
if (rebuild_domains)
rebuild_sched_domains_locked();
}
/*
* tasks_nocpu_error - Return true if tasks will have no effective_cpus
*/
static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs,
struct cpumask *xcpus)
{
/*
* A populated partition (cs or parent) can't have empty effective_cpus
*/
return (cpumask_subset(parent->effective_cpus, xcpus) &&
partition_is_populated(parent, cs)) ||
(!cpumask_intersects(xcpus, cpu_active_mask) &&
partition_is_populated(cs, NULL));
}
static void reset_partition_data(struct cpuset *cs)
{
struct cpuset *parent = parent_cs(cs);
if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys))
return;
lockdep_assert_held(&callback_lock);
cs->nr_subparts = 0;
if (cpumask_empty(cs->exclusive_cpus)) {
cpumask_clear(cs->effective_xcpus);
if (is_cpu_exclusive(cs))
clear_bit(CS_CPU_EXCLUSIVE, &cs->flags);
}
if (!cpumask_and(cs->effective_cpus,
parent->effective_cpus, cs->cpus_allowed)) {
cs->use_parent_ecpus = true;
parent->child_ecpus_count++;
cpumask_copy(cs->effective_cpus, parent->effective_cpus);
}
}
/*
* partition_xcpus_newstate - Exclusive CPUs state change
* @old_prs: old partition_root_state
* @new_prs: new partition_root_state
* @xcpus: exclusive CPUs with state change
*/
static void partition_xcpus_newstate(int old_prs, int new_prs, struct cpumask *xcpus)
{
WARN_ON_ONCE(old_prs == new_prs);
if (new_prs == PRS_ISOLATED)
cpumask_or(isolated_cpus, isolated_cpus, xcpus);
else
cpumask_andnot(isolated_cpus, isolated_cpus, xcpus);
}
/*
* partition_xcpus_add - Add new exclusive CPUs to partition
* @new_prs: new partition_root_state
* @parent: parent cpuset
* @xcpus: exclusive CPUs to be added
* Return: true if isolated_cpus modified, false otherwise
*
* Remote partition if parent == NULL
*/
static bool partition_xcpus_add(int new_prs, struct cpuset *parent,
struct cpumask *xcpus)
{
bool isolcpus_updated;
WARN_ON_ONCE(new_prs < 0);
lockdep_assert_held(&callback_lock);
if (!parent)
parent = &top_cpuset;
if (parent == &top_cpuset)
cpumask_or(subpartitions_cpus, subpartitions_cpus, xcpus);
isolcpus_updated = (new_prs != parent->partition_root_state);
if (isolcpus_updated)
partition_xcpus_newstate(parent->partition_root_state, new_prs,
xcpus);
cpumask_andnot(parent->effective_cpus, parent->effective_cpus, xcpus);
return isolcpus_updated;
}
/*
* partition_xcpus_del - Remove exclusive CPUs from partition
* @old_prs: old partition_root_state
* @parent: parent cpuset
* @xcpus: exclusive CPUs to be removed
* Return: true if isolated_cpus modified, false otherwise
*
* Remote partition if parent == NULL
*/
static bool partition_xcpus_del(int old_prs, struct cpuset *parent,
struct cpumask *xcpus)
{
bool isolcpus_updated;
WARN_ON_ONCE(old_prs < 0);
lockdep_assert_held(&callback_lock);
if (!parent)
parent = &top_cpuset;
if (parent == &top_cpuset)
cpumask_andnot(subpartitions_cpus, subpartitions_cpus, xcpus);
isolcpus_updated = (old_prs != parent->partition_root_state);
if (isolcpus_updated)
partition_xcpus_newstate(old_prs, parent->partition_root_state,
xcpus);
cpumask_and(xcpus, xcpus, cpu_active_mask);
cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus);
return isolcpus_updated;
}
static void update_unbound_workqueue_cpumask(bool isolcpus_updated)
{
int ret;
lockdep_assert_cpus_held();
if (!isolcpus_updated)
return;
ret = workqueue_unbound_exclude_cpumask(isolated_cpus);
WARN_ON_ONCE(ret < 0);
}
/**
* cpuset_cpu_is_isolated - Check if the given CPU is isolated
* @cpu: the CPU number to be checked
* Return: true if CPU is used in an isolated partition, false otherwise
*/
bool cpuset_cpu_is_isolated(int cpu)
{
return cpumask_test_cpu(cpu, isolated_cpus);
}
EXPORT_SYMBOL_GPL(cpuset_cpu_is_isolated);
/*
* compute_effective_exclusive_cpumask - compute effective exclusive CPUs
* @cs: cpuset
* @xcpus: effective exclusive CPUs value to be set
* Return: true if xcpus is not empty, false otherwise.
*
* Starting with exclusive_cpus (cpus_allowed if exclusive_cpus is not set),
* it must be a subset of cpus_allowed and parent's effective_xcpus.
*/
static bool compute_effective_exclusive_cpumask(struct cpuset *cs,
struct cpumask *xcpus)
{
struct cpuset *parent = parent_cs(cs);
if (!xcpus)
xcpus = cs->effective_xcpus;
if (!cpumask_empty(cs->exclusive_cpus))
cpumask_and(xcpus, cs->exclusive_cpus, cs->cpus_allowed);
else
cpumask_copy(xcpus, cs->cpus_allowed);
return cpumask_and(xcpus, xcpus, parent->effective_xcpus);
}
static inline bool is_remote_partition(struct cpuset *cs)
{
return !list_empty(&cs->remote_sibling);
}
static inline bool is_local_partition(struct cpuset *cs)
{
return is_partition_valid(cs) && !is_remote_partition(cs);
}
/*
* remote_partition_enable - Enable current cpuset as a remote partition root
* @cs: the cpuset to update
* @new_prs: new partition_root_state
* @tmp: temparary masks
* Return: 1 if successful, 0 if error
*
* Enable the current cpuset to become a remote partition root taking CPUs
* directly from the top cpuset. cpuset_mutex must be held by the caller.
*/
static int remote_partition_enable(struct cpuset *cs, int new_prs,
struct tmpmasks *tmp)
{
bool isolcpus_updated;
/*
* The user must have sysadmin privilege.
*/
if (!capable(CAP_SYS_ADMIN))
return 0;
/*
* The requested exclusive_cpus must not be allocated to other
* partitions and it can't use up all the root's effective_cpus.
*
* Note that if there is any local partition root above it or
* remote partition root underneath it, its exclusive_cpus must
* have overlapped with subpartitions_cpus.
*/
compute_effective_exclusive_cpumask(cs, tmp->new_cpus);
if (cpumask_empty(tmp->new_cpus) ||
cpumask_intersects(tmp->new_cpus, subpartitions_cpus) ||
cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus))
return 0;
spin_lock_irq(&callback_lock);
isolcpus_updated = partition_xcpus_add(new_prs, NULL, tmp->new_cpus);
list_add(&cs->remote_sibling, &remote_children);
if (cs->use_parent_ecpus) {
struct cpuset *parent = parent_cs(cs);
cs->use_parent_ecpus = false;
parent->child_ecpus_count--;
}
spin_unlock_irq(&callback_lock);
update_unbound_workqueue_cpumask(isolcpus_updated);
/*
* Proprogate changes in top_cpuset's effective_cpus down the hierarchy.
*/
update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
update_sibling_cpumasks(&top_cpuset, NULL, tmp);
return 1;
}
/*
* remote_partition_disable - Remove current cpuset from remote partition list
* @cs: the cpuset to update
* @tmp: temparary masks
*
* The effective_cpus is also updated.
*
* cpuset_mutex must be held by the caller.
*/
static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp)
{
bool isolcpus_updated;
compute_effective_exclusive_cpumask(cs, tmp->new_cpus);
WARN_ON_ONCE(!is_remote_partition(cs));
WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, subpartitions_cpus));
spin_lock_irq(&callback_lock);
list_del_init(&cs->remote_sibling);
isolcpus_updated = partition_xcpus_del(cs->partition_root_state,
NULL, tmp->new_cpus);
cs->partition_root_state = -cs->partition_root_state;
if (!cs->prs_err)
cs->prs_err = PERR_INVCPUS;
reset_partition_data(cs);
spin_unlock_irq(&callback_lock);
update_unbound_workqueue_cpumask(isolcpus_updated);
/*
* Proprogate changes in top_cpuset's effective_cpus down the hierarchy.
*/
update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
update_sibling_cpumasks(&top_cpuset, NULL, tmp);
}
/*
* remote_cpus_update - cpus_exclusive change of remote partition
* @cs: the cpuset to be updated
* @newmask: the new effective_xcpus mask
* @tmp: temparary masks
*
* top_cpuset and subpartitions_cpus will be updated or partition can be
* invalidated.
*/
static void remote_cpus_update(struct cpuset *cs, struct cpumask *newmask,
struct tmpmasks *tmp)
{
bool adding, deleting;
int prs = cs->partition_root_state;
int isolcpus_updated = 0;
if (WARN_ON_ONCE(!is_remote_partition(cs)))
return;
WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus));
if (cpumask_empty(newmask))
goto invalidate;
adding = cpumask_andnot(tmp->addmask, newmask, cs->effective_xcpus);
deleting = cpumask_andnot(tmp->delmask, cs->effective_xcpus, newmask);
/*
* Additions of remote CPUs is only allowed if those CPUs are
* not allocated to other partitions and there are effective_cpus
* left in the top cpuset.
*/
if (adding && (!capable(CAP_SYS_ADMIN) ||
cpumask_intersects(tmp->addmask, subpartitions_cpus) ||
cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)))
goto invalidate;
spin_lock_irq(&callback_lock);
if (adding)
isolcpus_updated += partition_xcpus_add(prs, NULL, tmp->addmask);
if (deleting)
isolcpus_updated += partition_xcpus_del(prs, NULL, tmp->delmask);
spin_unlock_irq(&callback_lock);
update_unbound_workqueue_cpumask(isolcpus_updated);
/*
* Proprogate changes in top_cpuset's effective_cpus down the hierarchy.
*/
update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
update_sibling_cpumasks(&top_cpuset, NULL, tmp);
return;
invalidate:
remote_partition_disable(cs, tmp);
}
/*
* remote_partition_check - check if a child remote partition needs update
* @cs: the cpuset to be updated
* @newmask: the new effective_xcpus mask
* @delmask: temporary mask for deletion (not in tmp)
* @tmp: temparary masks
*
* This should be called before the given cs has updated its cpus_allowed
* and/or effective_xcpus.
*/
static void remote_partition_check(struct cpuset *cs, struct cpumask *newmask,
struct cpumask *delmask, struct tmpmasks *tmp)
{
struct cpuset *child, *next;
int disable_cnt = 0;
/*
* Compute the effective exclusive CPUs that will be deleted.
*/
if (!cpumask_andnot(delmask, cs->effective_xcpus, newmask) ||
!cpumask_intersects(delmask, subpartitions_cpus))
return; /* No deletion of exclusive CPUs in partitions */
/*
* Searching the remote children list to look for those that will
* be impacted by the deletion of exclusive CPUs.
*
* Since a cpuset must be removed from the remote children list
* before it can go offline and holding cpuset_mutex will prevent
* any change in cpuset status. RCU read lock isn't needed.
*/
lockdep_assert_held(&cpuset_mutex);
list_for_each_entry_safe(child, next, &remote_children, remote_sibling)
if (cpumask_intersects(child->effective_cpus, delmask)) {
remote_partition_disable(child, tmp);
disable_cnt++;
}
if (disable_cnt)
rebuild_sched_domains_locked();
}
/*
* prstate_housekeeping_conflict - check for partition & housekeeping conflicts
* @prstate: partition root state to be checked
* @new_cpus: cpu mask
* Return: true if there is conflict, false otherwise
*
* CPUs outside of housekeeping_cpumask(HK_TYPE_DOMAIN) can only be used in
* an isolated partition.
*/
static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus)
{
const struct cpumask *hk_domain = housekeeping_cpumask(HK_TYPE_DOMAIN);
bool all_in_hk = cpumask_subset(new_cpus, hk_domain);
if (!all_in_hk && (prstate != PRS_ISOLATED))
return true;
return false;
}
/**
* update_parent_effective_cpumask - update effective_cpus mask of parent cpuset
* @cs: The cpuset that requests change in partition root state
* @cmd: Partition root state change command
* @newmask: Optional new cpumask for partcmd_update
* @tmp: Temporary addmask and delmask
* Return: 0 or a partition root state error code
*
* For partcmd_enable*, the cpuset is being transformed from a non-partition
* root to a partition root. The effective_xcpus (cpus_allowed if
* effective_xcpus not set) mask of the given cpuset will be taken away from
* parent's effective_cpus. The function will return 0 if all the CPUs listed
* in effective_xcpus can be granted or an error code will be returned.
*
* For partcmd_disable, the cpuset is being transformed from a partition
* root back to a non-partition root. Any CPUs in effective_xcpus will be
* given back to parent's effective_cpus. 0 will always be returned.
*
* For partcmd_update, if the optional newmask is specified, the cpu list is
* to be changed from effective_xcpus to newmask. Otherwise, effective_xcpus is
* assumed to remain the same. The cpuset should either be a valid or invalid
* partition root. The partition root state may change from valid to invalid
* or vice versa. An error code will be returned if transitioning from
* invalid to valid violates the exclusivity rule.
*
* For partcmd_invalidate, the current partition will be made invalid.
*
* The partcmd_enable* and partcmd_disable commands are used by
* update_prstate(). An error code may be returned and the caller will check
* for error.
*
* The partcmd_update command is used by update_cpumasks_hier() with newmask
* NULL and update_cpumask() with newmask set. The partcmd_invalidate is used
* by update_cpumask() with NULL newmask. In both cases, the callers won't
* check for error and so partition_root_state and prs_error will be updated
* directly.
*/
static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
struct cpumask *newmask,
struct tmpmasks *tmp)
{
struct cpuset *parent = parent_cs(cs);
int adding; /* Adding cpus to parent's effective_cpus */
int deleting; /* Deleting cpus from parent's effective_cpus */
int old_prs, new_prs;
int part_error = PERR_NONE; /* Partition error? */
int subparts_delta = 0;
struct cpumask *xcpus; /* cs effective_xcpus */
int isolcpus_updated = 0;
bool nocpu;
lockdep_assert_held(&cpuset_mutex);
/*
* new_prs will only be changed for the partcmd_update and
* partcmd_invalidate commands.
*/
adding = deleting = false;
old_prs = new_prs = cs->partition_root_state;
xcpus = !cpumask_empty(cs->exclusive_cpus)
? cs->effective_xcpus : cs->cpus_allowed;
if (cmd == partcmd_invalidate) {
if (is_prs_invalid(old_prs))
return 0;
/*
* Make the current partition invalid.
*/
if (is_partition_valid(parent))
adding = cpumask_and(tmp->addmask,
xcpus, parent->effective_xcpus);
if (old_prs > 0) {
new_prs = -old_prs;
subparts_delta--;
}
goto write_error;
}
/*
* The parent must be a partition root.
* The new cpumask, if present, or the current cpus_allowed must
* not be empty.
*/
if (!is_partition_valid(parent)) {
return is_partition_invalid(parent)
? PERR_INVPARENT : PERR_NOTPART;
}
if (!newmask && cpumask_empty(cs->cpus_allowed))
return PERR_CPUSEMPTY;
nocpu = tasks_nocpu_error(parent, cs, xcpus);
if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) {
/*
* Enabling partition root is not allowed if its
* effective_xcpus is empty or doesn't overlap with
* parent's effective_xcpus.
*/
if (cpumask_empty(xcpus) ||
!cpumask_intersects(xcpus, parent->effective_xcpus))
return PERR_INVCPUS;
if (prstate_housekeeping_conflict(new_prs, xcpus))
return PERR_HKEEPING;
/*
* A parent can be left with no CPU as long as there is no
* task directly associated with the parent partition.
*/
if (nocpu)
return PERR_NOCPUS;
cpumask_copy(tmp->delmask, xcpus);
deleting = true;
subparts_delta++;
new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED;
} else if (cmd == partcmd_disable) {
/*
* May need to add cpus to parent's effective_cpus for
* valid partition root.
*/
adding = !is_prs_invalid(old_prs) &&
cpumask_and(tmp->addmask, xcpus, parent->effective_xcpus);
if (adding)
subparts_delta--;
new_prs = PRS_MEMBER;
} else if (newmask) {
/*
* Empty cpumask is not allowed
*/
if (cpumask_empty(newmask)) {
part_error = PERR_CPUSEMPTY;
goto write_error;
}
/*
* partcmd_update with newmask:
*
* Compute add/delete mask to/from effective_cpus
*
* For valid partition:
* addmask = exclusive_cpus & ~newmask
* & parent->effective_xcpus
* delmask = newmask & ~exclusive_cpus
* & parent->effective_xcpus
*
* For invalid partition:
* delmask = newmask & parent->effective_xcpus
*/
if (is_prs_invalid(old_prs)) {
adding = false;
deleting = cpumask_and(tmp->delmask,
newmask, parent->effective_xcpus);
} else {
cpumask_andnot(tmp->addmask, xcpus, newmask);
adding = cpumask_and(tmp->addmask, tmp->addmask,
parent->effective_xcpus);
cpumask_andnot(tmp->delmask, newmask, xcpus);
deleting = cpumask_and(tmp->delmask, tmp->delmask,
parent->effective_xcpus);
}
/*
* Make partition invalid if parent's effective_cpus could
* become empty and there are tasks in the parent.
*/
if (nocpu && (!adding ||
!cpumask_intersects(tmp->addmask, cpu_active_mask))) {
part_error = PERR_NOCPUS;
deleting = false;
adding = cpumask_and(tmp->addmask,
xcpus, parent->effective_xcpus);
}
} else {
/*
* partcmd_update w/o newmask
*
* delmask = effective_xcpus & parent->effective_cpus
*
* This can be called from:
* 1) update_cpumasks_hier()
* 2) cpuset_hotplug_update_tasks()
*
* Check to see if it can be transitioned from valid to
* invalid partition or vice versa.
*
* A partition error happens when parent has tasks and all
* its effective CPUs will have to be distributed out.
*/
WARN_ON_ONCE(!is_partition_valid(parent));
if (nocpu) {
part_error = PERR_NOCPUS;
if (is_partition_valid(cs))
adding = cpumask_and(tmp->addmask,
xcpus, parent->effective_xcpus);
} else if (is_partition_invalid(cs) &&
cpumask_subset(xcpus, parent->effective_xcpus)) {
struct cgroup_subsys_state *css;
struct cpuset *child;
bool exclusive = true;
/*
* Convert invalid partition to valid has to
* pass the cpu exclusivity test.
*/
rcu_read_lock();
cpuset_for_each_child(child, css, parent) {
if (child == cs)
continue;
if (!cpusets_are_exclusive(cs, child)) {
exclusive = false;
break;
}
}
rcu_read_unlock();
if (exclusive)
deleting = cpumask_and(tmp->delmask,
xcpus, parent->effective_cpus);
else
part_error = PERR_NOTEXCL;
}
}
write_error:
if (part_error)
WRITE_ONCE(cs->prs_err, part_error);
if (cmd == partcmd_update) {
/*
* Check for possible transition between valid and invalid
* partition root.
*/
switch (cs->partition_root_state) {
case PRS_ROOT:
case PRS_ISOLATED:
if (part_error) {
new_prs = -old_prs;
subparts_delta--;
}
break;
case PRS_INVALID_ROOT:
case PRS_INVALID_ISOLATED:
if (!part_error) {
new_prs = -old_prs;
subparts_delta++;
}
break;
}
}
if (!adding && !deleting && (new_prs == old_prs))
return 0;
/*
* Transitioning between invalid to valid or vice versa may require
* changing CS_CPU_EXCLUSIVE. In the case of partcmd_update,
* validate_change() has already been successfully called and
* CPU lists in cs haven't been updated yet. So defer it to later.
*/
if ((old_prs != new_prs) && (cmd != partcmd_update)) {
int err = update_partition_exclusive(cs, new_prs);
if (err)
return err;
}
/*
* Change the parent's effective_cpus & effective_xcpus (top cpuset
* only).
*
* Newly added CPUs will be removed from effective_cpus and
* newly deleted ones will be added back to effective_cpus.
*/
spin_lock_irq(&callback_lock);
if (old_prs != new_prs) {
cs->partition_root_state = new_prs;
if (new_prs <= 0)
cs->nr_subparts = 0;
}
/*
* Adding to parent's effective_cpus means deletion CPUs from cs
* and vice versa.
*/
if (adding)
isolcpus_updated += partition_xcpus_del(old_prs, parent,
tmp->addmask);
if (deleting)
isolcpus_updated += partition_xcpus_add(new_prs, parent,
tmp->delmask);
if (is_partition_valid(parent)) {
parent->nr_subparts += subparts_delta;
WARN_ON_ONCE(parent->nr_subparts < 0);
}
spin_unlock_irq(&callback_lock);
update_unbound_workqueue_cpumask(isolcpus_updated);
if ((old_prs != new_prs) && (cmd == partcmd_update))
update_partition_exclusive(cs, new_prs);
if (adding || deleting) {
update_tasks_cpumask(parent, tmp->addmask);
update_sibling_cpumasks(parent, cs, tmp);
}
/*
* For partcmd_update without newmask, it is being called from
* cpuset_hotplug_workfn() where cpus_read_lock() wasn't taken.
* Update the load balance flag and scheduling domain if
* cpus_read_trylock() is successful.
*/
if ((cmd == partcmd_update) && !newmask && cpus_read_trylock()) {
update_partition_sd_lb(cs, old_prs);
cpus_read_unlock();
}
notify_partition_change(cs, old_prs);
return 0;
}
/**
* compute_partition_effective_cpumask - compute effective_cpus for partition
* @cs: partition root cpuset
* @new_ecpus: previously computed effective_cpus to be updated
*
* Compute the effective_cpus of a partition root by scanning effective_xcpus
* of child partition roots and excluding their effective_xcpus.
*
* This has the side effect of invalidating valid child partition roots,
* if necessary. Since it is called from either cpuset_hotplug_update_tasks()
* or update_cpumasks_hier() where parent and children are modified
* successively, we don't need to call update_parent_effective_cpumask()
* and the child's effective_cpus will be updated in later iterations.
*
* Note that rcu_read_lock() is assumed to be held.
*/
static void compute_partition_effective_cpumask(struct cpuset *cs,
struct cpumask *new_ecpus)
{
struct cgroup_subsys_state *css;
struct cpuset *child;
bool populated = partition_is_populated(cs, NULL);
/*
* Check child partition roots to see if they should be
* invalidated when
* 1) child effective_xcpus not a subset of new
* excluisve_cpus
* 2) All the effective_cpus will be used up and cp
* has tasks
*/
compute_effective_exclusive_cpumask(cs, new_ecpus);
cpumask_and(new_ecpus, new_ecpus, cpu_active_mask);
rcu_read_lock();
cpuset_for_each_child(child, css, cs) {
if (!is_partition_valid(child))
continue;
child->prs_err = 0;
if (!cpumask_subset(child->effective_xcpus,
cs->effective_xcpus))
child->prs_err = PERR_INVCPUS;
else if (populated &&
cpumask_subset(new_ecpus, child->effective_xcpus))
child->prs_err = PERR_NOCPUS;
if (child->prs_err) {
int old_prs = child->partition_root_state;
/*
* Invalidate child partition
*/
spin_lock_irq(&callback_lock);
make_partition_invalid(child);
cs->nr_subparts--;
child->nr_subparts = 0;
spin_unlock_irq(&callback_lock);
notify_partition_change(child, old_prs);
continue;
}
cpumask_andnot(new_ecpus, new_ecpus,
child->effective_xcpus);
}
rcu_read_unlock();
}
/*
* update_cpumasks_hier() flags
*/
#define HIER_CHECKALL 0x01 /* Check all cpusets with no skipping */
#define HIER_NO_SD_REBUILD 0x02 /* Don't rebuild sched domains */
/*
* update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
* @cs: the cpuset to consider
* @tmp: temp variables for calculating effective_cpus & partition setup
* @force: don't skip any descendant cpusets if set
*
* When configured cpumask is changed, the effective cpumasks of this cpuset
* and all its descendants need to be updated.
*
* On legacy hierarchy, effective_cpus will be the same with cpu_allowed.
*
* Called with cpuset_mutex held
*/
static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
int flags)
{
struct cpuset *cp;
struct cgroup_subsys_state *pos_css;
bool need_rebuild_sched_domains = false;
int old_prs, new_prs;
rcu_read_lock();
cpuset_for_each_descendant_pre(cp, pos_css, cs) {
struct cpuset *parent = parent_cs(cp);
bool remote = is_remote_partition(cp);
bool update_parent = false;
/*
* Skip descendent remote partition that acquires CPUs
* directly from top cpuset unless it is cs.
*/
if (remote && (cp != cs)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
/*
* Update effective_xcpus if exclusive_cpus set.
* The case when exclusive_cpus isn't set is handled later.
*/
if (!cpumask_empty(cp->exclusive_cpus) && (cp != cs)) {
spin_lock_irq(&callback_lock);
compute_effective_exclusive_cpumask(cp, NULL);
spin_unlock_irq(&callback_lock);
}
old_prs = new_prs = cp->partition_root_state;
if (remote || (is_partition_valid(parent) &&
is_partition_valid(cp)))
compute_partition_effective_cpumask(cp, tmp->new_cpus);
else
compute_effective_cpumask(tmp->new_cpus, cp, parent);
/*
* A partition with no effective_cpus is allowed as long as
* there is no task associated with it. Call
* update_parent_effective_cpumask() to check it.
*/
if (is_partition_valid(cp) && cpumask_empty(tmp->new_cpus)) {
update_parent = true;
goto update_parent_effective;
}
/*
* If it becomes empty, inherit the effective mask of the
* parent, which is guaranteed to have some CPUs unless
* it is a partition root that has explicitly distributed
* out all its CPUs.
*/
if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) {
cpumask_copy(tmp->new_cpus, parent->effective_cpus);
if (!cp->use_parent_ecpus) {
cp->use_parent_ecpus = true;
parent->child_ecpus_count++;
}
} else if (cp->use_parent_ecpus) {
cp->use_parent_ecpus = false;
WARN_ON_ONCE(!parent->child_ecpus_count);
parent->child_ecpus_count--;
}
if (remote)
goto get_css;
/*
* Skip the whole subtree if
* 1) the cpumask remains the same,
* 2) has no partition root state,
* 3) HIER_CHECKALL flag not set, and
* 4) for v2 load balance state same as its parent.
*/
if (!cp->partition_root_state && !(flags & HIER_CHECKALL) &&
cpumask_equal(tmp->new_cpus, cp->effective_cpus) &&
(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
(is_sched_load_balance(parent) == is_sched_load_balance(cp)))) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
update_parent_effective:
/*
* update_parent_effective_cpumask() should have been called
* for cs already in update_cpumask(). We should also call
* update_tasks_cpumask() again for tasks in the parent
* cpuset if the parent's effective_cpus changes.
*/
if ((cp != cs) && old_prs) {
switch (parent->partition_root_state) {
case PRS_ROOT:
case PRS_ISOLATED:
update_parent = true;
break;
default:
/*
* When parent is not a partition root or is
* invalid, child partition roots become
* invalid too.
*/
if (is_partition_valid(cp))
new_prs = -cp->partition_root_state;
WRITE_ONCE(cp->prs_err,
is_partition_invalid(parent)
? PERR_INVPARENT : PERR_NOTPART);
break;
}
}
get_css:
if (!css_tryget_online(&cp->css))
continue;
rcu_read_unlock();
if (update_parent) {
update_parent_effective_cpumask(cp, partcmd_update, NULL, tmp);
/*
* The cpuset partition_root_state may become
* invalid. Capture it.
*/
new_prs = cp->partition_root_state;
}
spin_lock_irq(&callback_lock);
cpumask_copy(cp->effective_cpus, tmp->new_cpus);
cp->partition_root_state = new_prs;
/*
* Make sure effective_xcpus is properly set for a valid
* partition root.
*/
if ((new_prs > 0) && cpumask_empty(cp->exclusive_cpus))
cpumask_and(cp->effective_xcpus,
cp->cpus_allowed, parent->effective_xcpus);
else if (new_prs < 0)
reset_partition_data(cp);
spin_unlock_irq(&callback_lock);
notify_partition_change(cp, old_prs);
WARN_ON(!is_in_v2_mode() &&
!cpumask_equal(cp->cpus_allowed, cp->effective_cpus));
update_tasks_cpumask(cp, cp->effective_cpus);
/*
* On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE
* from parent if current cpuset isn't a valid partition root
* and their load balance states differ.
*/
if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
!is_partition_valid(cp) &&
(is_sched_load_balance(parent) != is_sched_load_balance(cp))) {
if (is_sched_load_balance(parent))
set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
else
clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
}
/*
* On legacy hierarchy, if the effective cpumask of any non-
* empty cpuset is changed, we need to rebuild sched domains.
* On default hierarchy, the cpuset needs to be a partition
* root as well.
*/
if (!cpumask_empty(cp->cpus_allowed) &&
is_sched_load_balance(cp) &&
(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
is_partition_valid(cp)))
need_rebuild_sched_domains = true;
rcu_read_lock();
css_put(&cp->css);
}
rcu_read_unlock();
if (need_rebuild_sched_domains && !(flags & HIER_NO_SD_REBUILD))
rebuild_sched_domains_locked();
}
/**
* update_sibling_cpumasks - Update siblings cpumasks
* @parent: Parent cpuset
* @cs: Current cpuset
* @tmp: Temp variables
*/
static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
struct tmpmasks *tmp)
{
struct cpuset *sibling;
struct cgroup_subsys_state *pos_css;
lockdep_assert_held(&cpuset_mutex);
/*
* Check all its siblings and call update_cpumasks_hier()
* if their effective_cpus will need to be changed.
*
* With the addition of effective_xcpus which is a subset of
* cpus_allowed. It is possible a change in parent's effective_cpus
* due to a change in a child partition's effective_xcpus will impact
* its siblings even if they do not inherit parent's effective_cpus
* directly.
*
* The update_cpumasks_hier() function may sleep. So we have to
* release the RCU read lock before calling it. HIER_NO_SD_REBUILD
* flag is used to suppress rebuild of sched domains as the callers
* will take care of that.
*/
rcu_read_lock();
cpuset_for_each_child(sibling, pos_css, parent) {
if (sibling == cs)
continue;
if (!sibling->use_parent_ecpus &&
!is_partition_valid(sibling)) {
compute_effective_cpumask(tmp->new_cpus, sibling,
parent);
if (cpumask_equal(tmp->new_cpus, sibling->effective_cpus))
continue;
}
if (!css_tryget_online(&sibling->css))
continue;
rcu_read_unlock();
update_cpumasks_hier(sibling, tmp, HIER_NO_SD_REBUILD);
rcu_read_lock();
css_put(&sibling->css);
}
rcu_read_unlock();
}
/**
* update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
* @cs: the cpuset to consider
* @trialcs: trial cpuset
* @buf: buffer of cpu numbers written to this cpuset
*/
static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
const char *buf)
{
int retval;
struct tmpmasks tmp;
struct cpuset *parent = parent_cs(cs);
bool invalidate = false;
int hier_flags = 0;
int old_prs = cs->partition_root_state;
/* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
if (cs == &top_cpuset)
return -EACCES;
/*
* An empty cpus_allowed is ok only if the cpuset has no tasks.
* Since cpulist_parse() fails on an empty mask, we special case
* that parsing. The validate_change() call ensures that cpusets
* with tasks have cpus.
*/
if (!*buf) {
cpumask_clear(trialcs->cpus_allowed);
cpumask_clear(trialcs->effective_xcpus);
} else {
retval = cpulist_parse(buf, trialcs->cpus_allowed);
if (retval < 0)
return retval;
if (!cpumask_subset(trialcs->cpus_allowed,
top_cpuset.cpus_allowed))
return -EINVAL;
/*
* When exclusive_cpus isn't explicitly set, it is constrainted
* by cpus_allowed and parent's effective_xcpus. Otherwise,
* trialcs->effective_xcpus is used as a temporary cpumask
* for checking validity of the partition root.
*/
if (!cpumask_empty(trialcs->exclusive_cpus) || is_partition_valid(cs))
compute_effective_exclusive_cpumask(trialcs, NULL);
}
/* Nothing to do if the cpus didn't change */
if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
return 0;
if (alloc_cpumasks(NULL, &tmp))
return -ENOMEM;
if (old_prs) {
if (is_partition_valid(cs) &&
cpumask_empty(trialcs->effective_xcpus)) {
invalidate = true;
cs->prs_err = PERR_INVCPUS;
} else if (prstate_housekeeping_conflict(old_prs, trialcs->effective_xcpus)) {
invalidate = true;
cs->prs_err = PERR_HKEEPING;
} else if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) {
invalidate = true;
cs->prs_err = PERR_NOCPUS;
}
}
/*
* Check all the descendants in update_cpumasks_hier() if
* effective_xcpus is to be changed.
*/
if (!cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus))
hier_flags = HIER_CHECKALL;
retval = validate_change(cs, trialcs);
if ((retval == -EINVAL) && cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) {
struct cgroup_subsys_state *css;
struct cpuset *cp;
/*
* The -EINVAL error code indicates that partition sibling
* CPU exclusivity rule has been violated. We still allow
* the cpumask change to proceed while invalidating the
* partition. However, any conflicting sibling partitions
* have to be marked as invalid too.
*/
invalidate = true;
rcu_read_lock();
cpuset_for_each_child(cp, css, parent) {
struct cpumask *xcpus = fetch_xcpus(trialcs);
if (is_partition_valid(cp) &&
cpumask_intersects(xcpus, cp->effective_xcpus)) {
rcu_read_unlock();
update_parent_effective_cpumask(cp, partcmd_invalidate, NULL, &tmp);
rcu_read_lock();
}
}
rcu_read_unlock();
retval = 0;
}
if (retval < 0)
goto out_free;
if (is_partition_valid(cs) ||
(is_partition_invalid(cs) && !invalidate)) {
struct cpumask *xcpus = trialcs->effective_xcpus;
if (cpumask_empty(xcpus) && is_partition_invalid(cs))
xcpus = trialcs->cpus_allowed;
/*
* Call remote_cpus_update() to handle valid remote partition
*/
if (is_remote_partition(cs))
remote_cpus_update(cs, xcpus, &tmp);
else if (invalidate)
update_parent_effective_cpumask(cs, partcmd_invalidate,
NULL, &tmp);
else
update_parent_effective_cpumask(cs, partcmd_update,
xcpus, &tmp);
} else if (!cpumask_empty(cs->exclusive_cpus)) {
/*
* Use trialcs->effective_cpus as a temp cpumask
*/
remote_partition_check(cs, trialcs->effective_xcpus,
trialcs->effective_cpus, &tmp);
}
spin_lock_irq(&callback_lock);
cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus);
if ((old_prs > 0) && !is_partition_valid(cs))
reset_partition_data(cs);
spin_unlock_irq(&callback_lock);
/* effective_cpus/effective_xcpus will be updated here */
update_cpumasks_hier(cs, &tmp, hier_flags);
/* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */
if (cs->partition_root_state)
update_partition_sd_lb(cs, old_prs);
out_free:
free_cpumasks(NULL, &tmp);
return retval;
}
/**
* update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset
* @cs: the cpuset to consider
* @trialcs: trial cpuset
* @buf: buffer of cpu numbers written to this cpuset
*
* The tasks' cpumask will be updated if cs is a valid partition root.
*/
static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs,
const char *buf)
{
int retval;
struct tmpmasks tmp;
struct cpuset *parent = parent_cs(cs);
bool invalidate = false;
int hier_flags = 0;
int old_prs = cs->partition_root_state;
if (!*buf) {
cpumask_clear(trialcs->exclusive_cpus);
cpumask_clear(trialcs->effective_xcpus);
} else {
retval = cpulist_parse(buf, trialcs->exclusive_cpus);
if (retval < 0)
return retval;
if (!is_cpu_exclusive(cs))
set_bit(CS_CPU_EXCLUSIVE, &trialcs->flags);
}
/* Nothing to do if the CPUs didn't change */
if (cpumask_equal(cs->exclusive_cpus, trialcs->exclusive_cpus))
return 0;
if (*buf)
compute_effective_exclusive_cpumask(trialcs, NULL);
/*
* Check all the descendants in update_cpumasks_hier() if
* effective_xcpus is to be changed.
*/
if (!cpumask_equal(cs->effective_xcpus, trialcs->effective_xcpus))
hier_flags = HIER_CHECKALL;
retval = validate_change(cs, trialcs);
if (retval)
return retval;
if (alloc_cpumasks(NULL, &tmp))
return -ENOMEM;
if (old_prs) {
if (cpumask_empty(trialcs->effective_xcpus)) {
invalidate = true;
cs->prs_err = PERR_INVCPUS;
} else if (prstate_housekeeping_conflict(old_prs, trialcs->effective_xcpus)) {
invalidate = true;
cs->prs_err = PERR_HKEEPING;
} else if (tasks_nocpu_error(parent, cs, trialcs->effective_xcpus)) {
invalidate = true;
cs->prs_err = PERR_NOCPUS;
}
if (is_remote_partition(cs)) {
if (invalidate)
remote_partition_disable(cs, &tmp);
else
remote_cpus_update(cs, trialcs->effective_xcpus,
&tmp);
} else if (invalidate) {
update_parent_effective_cpumask(cs, partcmd_invalidate,
NULL, &tmp);
} else {
update_parent_effective_cpumask(cs, partcmd_update,
trialcs->effective_xcpus, &tmp);
}
} else if (!cpumask_empty(trialcs->exclusive_cpus)) {
/*
* Use trialcs->effective_cpus as a temp cpumask
*/
remote_partition_check(cs, trialcs->effective_xcpus,
trialcs->effective_cpus, &tmp);
}
spin_lock_irq(&callback_lock);
cpumask_copy(cs->exclusive_cpus, trialcs->exclusive_cpus);
cpumask_copy(cs->effective_xcpus, trialcs->effective_xcpus);
if ((old_prs > 0) && !is_partition_valid(cs))
reset_partition_data(cs);
spin_unlock_irq(&callback_lock);
/*
* Call update_cpumasks_hier() to update effective_cpus/effective_xcpus
* of the subtree when it is a valid partition root or effective_xcpus
* is updated.
*/
if (is_partition_valid(cs) || hier_flags)
update_cpumasks_hier(cs, &tmp, hier_flags);
/* Update CS_SCHED_LOAD_BALANCE and/or sched_domains, if necessary */
if (cs->partition_root_state)
update_partition_sd_lb(cs, old_prs);
free_cpumasks(NULL, &tmp);
return 0;
}
/*
* Migrate memory region from one set of nodes to another. This is
* performed asynchronously as it can be called from process migration path
* holding locks involved in process management. All mm migrations are
* performed in the queued order and can be waited for by flushing
* cpuset_migrate_mm_wq.
*/
struct cpuset_migrate_mm_work {
struct work_struct work;
struct mm_struct *mm;
nodemask_t from;
nodemask_t to;
};
static void cpuset_migrate_mm_workfn(struct work_struct *work)
{
struct cpuset_migrate_mm_work *mwork =
container_of(work, struct cpuset_migrate_mm_work, work);
/* on a wq worker, no need to worry about %current's mems_allowed */
do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL);
mmput(mwork->mm);
kfree(mwork);
}
static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
const nodemask_t *to)
{
struct cpuset_migrate_mm_work *mwork;
if (nodes_equal(*from, *to)) {
mmput(mm);
return;
}
mwork = kzalloc(sizeof(*mwork), GFP_KERNEL);
if (mwork) {
mwork->mm = mm;
mwork->from = *from;
mwork->to = *to;
INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn);
queue_work(cpuset_migrate_mm_wq, &mwork->work);
} else {
mmput(mm);
}
}
static void cpuset_post_attach(void)
{
flush_workqueue(cpuset_migrate_mm_wq);
}
/*
* cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
* @tsk: the task to change
* @newmems: new nodes that the task will be set
*
* We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
* and rebind an eventual tasks' mempolicy. If the task is allocating in
* parallel, it might temporarily see an empty intersection, which results in
* a seqlock check and retry before OOM or allocation failure.
*/
static void cpuset_change_task_nodemask(struct task_struct *tsk,
nodemask_t *newmems)
{
task_lock(tsk);
local_irq_disable();
write_seqcount_begin(&tsk->mems_allowed_seq);
nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
mpol_rebind_task(tsk, newmems);
tsk->mems_allowed = *newmems;
write_seqcount_end(&tsk->mems_allowed_seq);
local_irq_enable();
task_unlock(tsk);
}
static void *cpuset_being_rebound;
/**
* update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
* @cs: the cpuset in which each task's mems_allowed mask needs to be changed
*
* Iterate through each task of @cs updating its mems_allowed to the
* effective cpuset's. As this function is called with cpuset_mutex held,
* cpuset membership stays stable.
*/
static void update_tasks_nodemask(struct cpuset *cs)
{
static nodemask_t newmems; /* protected by cpuset_mutex */
struct css_task_iter it;
struct task_struct *task;
cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
guarantee_online_mems(cs, &newmems);
/*
* The mpol_rebind_mm() call takes mmap_lock, which we couldn't
* take while holding tasklist_lock. Forks can happen - the
* mpol_dup() cpuset_being_rebound check will catch such forks,
* and rebind their vma mempolicies too. Because we still hold
* the global cpuset_mutex, we know that no other rebind effort
* will be contending for the global variable cpuset_being_rebound.
* It's ok if we rebind the same mm twice; mpol_rebind_mm()
* is idempotent. Also migrate pages in each mm to new nodes.
*/
css_task_iter_start(&cs->css, 0, &it);
while ((task = css_task_iter_next(&it))) {
struct mm_struct *mm;
bool migrate;
cpuset_change_task_nodemask(task, &newmems);
mm = get_task_mm(task);
if (!mm)
continue;
migrate = is_memory_migrate(cs);
mpol_rebind_mm(mm, &cs->mems_allowed);
if (migrate)
cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
else
mmput(mm);
}
css_task_iter_end(&it);
/*
* All the tasks' nodemasks have been updated, update
* cs->old_mems_allowed.
*/
cs->old_mems_allowed = newmems;
/* We're done rebinding vmas to this cpuset's new mems_allowed. */
cpuset_being_rebound = NULL;
}
/*
* update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
* @cs: the cpuset to consider
* @new_mems: a temp variable for calculating new effective_mems
*
* When configured nodemask is changed, the effective nodemasks of this cpuset
* and all its descendants need to be updated.
*
* On legacy hierarchy, effective_mems will be the same with mems_allowed.
*
* Called with cpuset_mutex held
*/
static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
{
struct cpuset *cp;
struct cgroup_subsys_state *pos_css;
rcu_read_lock();
cpuset_for_each_descendant_pre(cp, pos_css, cs) {
struct cpuset *parent = parent_cs(cp);
nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems);
/*
* If it becomes empty, inherit the effective mask of the
* parent, which is guaranteed to have some MEMs.
*/
if (is_in_v2_mode() && nodes_empty(*new_mems))
*new_mems = parent->effective_mems;
/* Skip the whole subtree if the nodemask remains the same. */
if (nodes_equal(*new_mems, cp->effective_mems)) {
pos_css = css_rightmost_descendant(pos_css);
continue;
}
if (!css_tryget_online(&cp->css))
continue;
rcu_read_unlock();
spin_lock_irq(&callback_lock);
cp->effective_mems = *new_mems;
spin_unlock_irq(&callback_lock);
WARN_ON(!is_in_v2_mode() &&
!nodes_equal(cp->mems_allowed, cp->effective_mems));
update_tasks_nodemask(cp);
rcu_read_lock();
css_put(&cp->css);
}
rcu_read_unlock();
}
/*
* Handle user request to change the 'mems' memory placement
* of a cpuset. Needs to validate the request, update the
* cpusets mems_allowed, and for each task in the cpuset,
* update mems_allowed and rebind task's mempolicy and any vma
* mempolicies and if the cpuset is marked 'memory_migrate',
* migrate the tasks pages to the new memory.
*
* Call with cpuset_mutex held. May take callback_lock during call.
* Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
* lock each such tasks mm->mmap_lock, scan its vma's and rebind
* their mempolicies to the cpusets new mems_allowed.
*/
static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
const char *buf)
{
int retval;
/*
* top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
* it's read-only
*/
if (cs == &top_cpuset) {
retval = -EACCES;
goto done;
}
/*
* An empty mems_allowed is ok iff there are no tasks in the cpuset.
* Since nodelist_parse() fails on an empty mask, we special case
* that parsing. The validate_change() call ensures that cpusets
* with tasks have memory.
*/
if (!*buf) {
nodes_clear(trialcs->mems_allowed);
} else {
retval = nodelist_parse(buf, trialcs->mems_allowed);
if (retval < 0)
goto done;
if (!nodes_subset(trialcs->mems_allowed,
top_cpuset.mems_allowed)) {
retval = -EINVAL;
goto done;
}
}
if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
retval = 0; /* Too easy - nothing to do */
goto done;
}
retval = validate_change(cs, trialcs);
if (retval < 0)
goto done;
check_insane_mems_config(&trialcs->mems_allowed);
spin_lock_irq(&callback_lock);
cs->mems_allowed = trialcs->mems_allowed;
spin_unlock_irq(&callback_lock);
/* use trialcs->mems_allowed as a temp variable */
update_nodemasks_hier(cs, &trialcs->mems_allowed);
done:
return retval;
}
bool current_cpuset_is_being_rebound(void)
{
bool ret;
rcu_read_lock();
ret = task_cs(current) == cpuset_being_rebound;
rcu_read_unlock();
return ret;
}
static int update_relax_domain_level(struct cpuset *cs, s64 val)
{
#ifdef CONFIG_SMP
if (val < -1 || val >= sched_domain_level_max)
return -EINVAL;
#endif
if (val != cs->relax_domain_level) {
cs->relax_domain_level = val;
if (!cpumask_empty(cs->cpus_allowed) &&
is_sched_load_balance(cs))
rebuild_sched_domains_locked();
}
return 0;
}
/**
* update_tasks_flags - update the spread flags of tasks in the cpuset.
* @cs: the cpuset in which each task's spread flags needs to be changed
*
* Iterate through each task of @cs updating its spread flags. As this
* function is called with cpuset_mutex held, cpuset membership stays
* stable.
*/
static void update_tasks_flags(struct cpuset *cs)
{
struct css_task_iter it;
struct task_struct *task;
css_task_iter_start(&cs->css, 0, &it);
while ((task = css_task_iter_next(&it)))
cpuset_update_task_spread_flags(cs, task);
css_task_iter_end(&it);
}
/*
* update_flag - read a 0 or a 1 in a file and update associated flag
* bit: the bit to update (see cpuset_flagbits_t)
* cs: the cpuset to update
* turning_on: whether the flag is being set or cleared
*
* Call with cpuset_mutex held.
*/
static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
int turning_on)
{
struct cpuset *trialcs;
int balance_flag_changed;
int spread_flag_changed;
int err;
trialcs = alloc_trial_cpuset(cs);
if (!trialcs)
return -ENOMEM;
if (turning_on)
set_bit(bit, &trialcs->flags);
else
clear_bit(bit, &trialcs->flags);
err = validate_change(cs, trialcs);
if (err < 0)
goto out;
balance_flag_changed = (is_sched_load_balance(cs) !=
is_sched_load_balance(trialcs));
spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
|| (is_spread_page(cs) != is_spread_page(trialcs)));
spin_lock_irq(&callback_lock);
cs->flags = trialcs->flags;
spin_unlock_irq(&callback_lock);
if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
rebuild_sched_domains_locked();
if (spread_flag_changed)
update_tasks_flags(cs);
out:
free_cpuset(trialcs);
return err;
}
/**
* update_prstate - update partition_root_state
* @cs: the cpuset to update
* @new_prs: new partition root state
* Return: 0 if successful, != 0 if error
*
* Call with cpuset_mutex held.
*/
static int update_prstate(struct cpuset *cs, int new_prs)
{
int err = PERR_NONE, old_prs = cs->partition_root_state;
struct cpuset *parent = parent_cs(cs);
struct tmpmasks tmpmask;
bool new_xcpus_state = false;
if (old_prs == new_prs)
return 0;
/*
* Treat a previously invalid partition root as if it is a "member".
*/
if (new_prs && is_prs_invalid(old_prs))
old_prs = PRS_MEMBER;
if (alloc_cpumasks(NULL, &tmpmask))
return -ENOMEM;
/*
* Setup effective_xcpus if not properly set yet, it will be cleared
* later if partition becomes invalid.
*/
if ((new_prs > 0) && cpumask_empty(cs->exclusive_cpus)) {
spin_lock_irq(&callback_lock);
cpumask_and(cs->effective_xcpus,
cs->cpus_allowed, parent->effective_xcpus);
spin_unlock_irq(&callback_lock);
}
err = update_partition_exclusive(cs, new_prs);
if (err)
goto out;
if (!old_prs) {
enum partition_cmd cmd = (new_prs == PRS_ROOT)
? partcmd_enable : partcmd_enablei;
/*
* cpus_allowed cannot be empty.
*/
if (cpumask_empty(cs->cpus_allowed)) {
err = PERR_CPUSEMPTY;
goto out;
}
err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask);
/*
* If an attempt to become local partition root fails,
* try to become a remote partition root instead.
*/
if (err && remote_partition_enable(cs, new_prs, &tmpmask))
err = 0;
} else if (old_prs && new_prs) {
/*
* A change in load balance state only, no change in cpumasks.
*/
new_xcpus_state = true;
} else {
/*
* Switching back to member is always allowed even if it
* disables child partitions.
*/
if (is_remote_partition(cs))
remote_partition_disable(cs, &tmpmask);
else
update_parent_effective_cpumask(cs, partcmd_disable,
NULL, &tmpmask);
/*
* Invalidation of child partitions will be done in
* update_cpumasks_hier().
*/
}
out:
/*
* Make partition invalid & disable CS_CPU_EXCLUSIVE if an error
* happens.
*/
if (err) {
new_prs = -new_prs;
update_partition_exclusive(cs, new_prs);
}
spin_lock_irq(&callback_lock);
cs->partition_root_state = new_prs;
WRITE_ONCE(cs->prs_err, err);
if (!is_partition_valid(cs))
reset_partition_data(cs);
else if (new_xcpus_state)
partition_xcpus_newstate(old_prs, new_prs, cs->effective_xcpus);
spin_unlock_irq(&callback_lock);
update_unbound_workqueue_cpumask(new_xcpus_state);
/* Force update if switching back to member */
update_cpumasks_hier(cs, &tmpmask, !new_prs ? HIER_CHECKALL : 0);
/* Update sched domains and load balance flag */
update_partition_sd_lb(cs, old_prs);
notify_partition_change(cs, old_prs);
free_cpumasks(NULL, &tmpmask);
return 0;
}
/*
* Frequency meter - How fast is some event occurring?
*
* These routines manage a digitally filtered, constant time based,
* event frequency meter. There are four routines:
* fmeter_init() - initialize a frequency meter.
* fmeter_markevent() - called each time the event happens.
* fmeter_getrate() - returns the recent rate of such events.
* fmeter_update() - internal routine used to update fmeter.
*
* A common data structure is passed to each of these routines,
* which is used to keep track of the state required to manage the
* frequency meter and its digital filter.
*
* The filter works on the number of events marked per unit time.
* The filter is single-pole low-pass recursive (IIR). The time unit
* is 1 second. Arithmetic is done using 32-bit integers scaled to
* simulate 3 decimal digits of precision (multiplied by 1000).
*
* With an FM_COEF of 933, and a time base of 1 second, the filter
* has a half-life of 10 seconds, meaning that if the events quit
* happening, then the rate returned from the fmeter_getrate()
* will be cut in half each 10 seconds, until it converges to zero.
*
* It is not worth doing a real infinitely recursive filter. If more
* than FM_MAXTICKS ticks have elapsed since the last filter event,
* just compute FM_MAXTICKS ticks worth, by which point the level
* will be stable.
*
* Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
* arithmetic overflow in the fmeter_update() routine.
*
* Given the simple 32 bit integer arithmetic used, this meter works
* best for reporting rates between one per millisecond (msec) and
* one per 32 (approx) seconds. At constant rates faster than one
* per msec it maxes out at values just under 1,000,000. At constant
* rates between one per msec, and one per second it will stabilize
* to a value N*1000, where N is the rate of events per second.
* At constant rates between one per second and one per 32 seconds,
* it will be choppy, moving up on the seconds that have an event,
* and then decaying until the next event. At rates slower than
* about one in 32 seconds, it decays all the way back to zero between
* each event.
*/
#define FM_COEF 933 /* coefficient for half-life of 10 secs */
#define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */
#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
#define FM_SCALE 1000 /* faux fixed point scale */
/* Initialize a frequency meter */
static void fmeter_init(struct fmeter *fmp)
{
fmp->cnt = 0;
fmp->val = 0;
fmp->time = 0;
spin_lock_init(&fmp->lock);
}
/* Internal meter update - process cnt events and update value */
static void fmeter_update(struct fmeter *fmp)
{
time64_t now;
u32 ticks;
now = ktime_get_seconds();
ticks = now - fmp->time;
if (ticks == 0)
return;
ticks = min(FM_MAXTICKS, ticks);
while (ticks-- > 0)
fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
fmp->time = now;
fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
fmp->cnt = 0;
}
/* Process any previous ticks, then bump cnt by one (times scale). */
static void fmeter_markevent(struct fmeter *fmp)
{
spin_lock(&fmp->lock);
fmeter_update(fmp);
fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
spin_unlock(&fmp->lock);
}
/* Process any previous ticks, then return current value. */
static int fmeter_getrate(struct fmeter *fmp)
{
int val;
spin_lock(&fmp->lock);
fmeter_update(fmp);
val = fmp->val;
spin_unlock(&fmp->lock);
return val;
}
static struct cpuset *cpuset_attach_old_cs;
/*
* Check to see if a cpuset can accept a new task
* For v1, cpus_allowed and mems_allowed can't be empty.
* For v2, effective_cpus can't be empty.
* Note that in v1, effective_cpus = cpus_allowed.
*/
static int cpuset_can_attach_check(struct cpuset *cs)
{
if (cpumask_empty(cs->effective_cpus) ||
(!is_in_v2_mode() && nodes_empty(cs->mems_allowed)))
return -ENOSPC;
return 0;
}
static void reset_migrate_dl_data(struct cpuset *cs)
{
cs->nr_migrate_dl_tasks = 0;
cs->sum_migrate_dl_bw = 0;
}
/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
static int cpuset_can_attach(struct cgroup_taskset *tset)
{
struct cgroup_subsys_state *css;
struct cpuset *cs, *oldcs;
struct task_struct *task;
bool cpus_updated, mems_updated;
int ret;
/* used later by cpuset_attach() */
cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
oldcs = cpuset_attach_old_cs;
cs = css_cs(css);
mutex_lock(&cpuset_mutex);
/* Check to see if task is allowed in the cpuset */
ret = cpuset_can_attach_check(cs);
if (ret)
goto out_unlock;
cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus);
mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);
cgroup_taskset_for_each(task, css, tset) {
ret = task_can_attach(task);
if (ret)
goto out_unlock;
/*
* Skip rights over task check in v2 when nothing changes,
* migration permission derives from hierarchy ownership in
* cgroup_procs_write_permission()).
*/
if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
(cpus_updated || mems_updated)) {
ret = security_task_setscheduler(task);
if (ret)
goto out_unlock;
}
if (dl_task(task)) {
cs->nr_migrate_dl_tasks++;
cs->sum_migrate_dl_bw += task->dl.dl_bw;
}
}
if (!cs->nr_migrate_dl_tasks)
goto out_success;
if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) {
int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
if (unlikely(cpu >= nr_cpu_ids)) {
reset_migrate_dl_data(cs);
ret = -EINVAL;
goto out_unlock;
}
ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
if (ret) {
reset_migrate_dl_data(cs);
goto out_unlock;
}
}
out_success:
/*
* Mark attach is in progress. This makes validate_change() fail
* changes which zero cpus/mems_allowed.
*/
cs->attach_in_progress++;
out_unlock:
mutex_unlock(&cpuset_mutex);
return ret;
}
static void cpuset_cancel_attach(struct cgroup_taskset *tset)
{
struct cgroup_subsys_state *css;
struct cpuset *cs;
cgroup_taskset_first(tset, &css);
cs = css_cs(css);
mutex_lock(&cpuset_mutex);
cs->attach_in_progress--;
if (!cs->attach_in_progress)
wake_up(&cpuset_attach_wq);
if (cs->nr_migrate_dl_tasks) {
int cpu = cpumask_any(cs->effective_cpus);
dl_bw_free(cpu, cs->sum_migrate_dl_bw);
reset_migrate_dl_data(cs);
}
mutex_unlock(&cpuset_mutex);
}
/*
* Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach_task()
* but we can't allocate it dynamically there. Define it global and
* allocate from cpuset_init().
*/
static cpumask_var_t cpus_attach;
static nodemask_t cpuset_attach_nodemask_to;
static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
{
lockdep_assert_held(&cpuset_mutex);
if (cs != &top_cpuset)
guarantee_online_cpus(task, cpus_attach);
else
cpumask_andnot(cpus_attach, task_cpu_possible_mask(task),
subpartitions_cpus);
/*
* can_attach beforehand should guarantee that this doesn't
* fail. TODO: have a better way to handle failure here
*/
WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));
cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
cpuset_update_task_spread_flags(cs, task);
}
static void cpuset_attach(struct cgroup_taskset *tset)
{
struct task_struct *task;
struct task_struct *leader;
struct cgroup_subsys_state *css;
struct cpuset *cs;
struct cpuset *oldcs = cpuset_attach_old_cs;
bool cpus_updated, mems_updated;
cgroup_taskset_first(tset, &css);
cs = css_cs(css);
lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */
mutex_lock(&cpuset_mutex);
cpus_updated = !cpumask_equal(cs->effective_cpus,
oldcs->effective_cpus);
mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);
/*
* In the default hierarchy, enabling cpuset in the child cgroups
* will trigger a number of cpuset_attach() calls with no change
* in effective cpus and mems. In that case, we can optimize out
* by skipping the task iteration and update.
*/
if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
!cpus_updated && !mems_updated) {
cpuset_attach_nodemask_to = cs->effective_mems;
goto out;
}
guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
cgroup_taskset_for_each(task, css, tset)
cpuset_attach_task(cs, task);
/*
* Change mm for all threadgroup leaders. This is expensive and may
* sleep and should be moved outside migration path proper. Skip it
* if there is no change in effective_mems and CS_MEMORY_MIGRATE is
* not set.
*/
cpuset_attach_nodemask_to = cs->effective_mems;
if (!is_memory_migrate(cs) && !mems_updated)
goto out;
cgroup_taskset_for_each_leader(leader, css, tset) {
struct mm_struct *mm = get_task_mm(leader);
if (mm) {
mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
/*
* old_mems_allowed is the same with mems_allowed
* here, except if this task is being moved
* automatically due to hotplug. In that case
* @mems_allowed has been updated and is empty, so
* @old_mems_allowed is the right nodesets that we
* migrate mm from.
*/
if (is_memory_migrate(cs))
cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
&cpuset_attach_nodemask_to);
else
mmput(mm);
}
}
out:
cs->old_mems_allowed = cpuset_attach_nodemask_to;
if (cs->nr_migrate_dl_tasks) {
cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks;
oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks;
reset_migrate_dl_data(cs);
}
cs->attach_in_progress--;
if (!cs->attach_in_progress)
wake_up(&cpuset_attach_wq);
mutex_unlock(&cpuset_mutex);
}
/* The various types of files and directories in a cpuset file system */
typedef enum {
FILE_MEMORY_MIGRATE,
FILE_CPULIST,
FILE_MEMLIST,
FILE_EFFECTIVE_CPULIST,
FILE_EFFECTIVE_MEMLIST,
FILE_SUBPARTS_CPULIST,
FILE_EXCLUSIVE_CPULIST,
FILE_EFFECTIVE_XCPULIST,
FILE_ISOLATED_CPULIST,
FILE_CPU_EXCLUSIVE,
FILE_MEM_EXCLUSIVE,
FILE_MEM_HARDWALL,
FILE_SCHED_LOAD_BALANCE,
FILE_PARTITION_ROOT,
FILE_SCHED_RELAX_DOMAIN_LEVEL,
FILE_MEMORY_PRESSURE_ENABLED,
FILE_MEMORY_PRESSURE,
FILE_SPREAD_PAGE,
FILE_SPREAD_SLAB,
} cpuset_filetype_t;
static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
u64 val)
{
struct cpuset *cs = css_cs(css);
cpuset_filetype_t type = cft->private;
int retval = 0;
cpus_read_lock();
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs)) {
retval = -ENODEV;
goto out_unlock;
}
switch (type) {
case FILE_CPU_EXCLUSIVE:
retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
break;
case FILE_MEM_EXCLUSIVE:
retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
break;
case FILE_MEM_HARDWALL:
retval = update_flag(CS_MEM_HARDWALL, cs, val);
break;
case FILE_SCHED_LOAD_BALANCE:
retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
break;
case FILE_MEMORY_MIGRATE:
retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
break;
case FILE_MEMORY_PRESSURE_ENABLED:
cpuset_memory_pressure_enabled = !!val;
break;
case FILE_SPREAD_PAGE:
retval = update_flag(CS_SPREAD_PAGE, cs, val);
break;
case FILE_SPREAD_SLAB:
retval = update_flag(CS_SPREAD_SLAB, cs, val);
break;
default:
retval = -EINVAL;
break;
}
out_unlock:
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
return retval;
}
static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
s64 val)
{
struct cpuset *cs = css_cs(css);
cpuset_filetype_t type = cft->private;
int retval = -ENODEV;
cpus_read_lock();
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs))
goto out_unlock;
switch (type) {
case FILE_SCHED_RELAX_DOMAIN_LEVEL:
retval = update_relax_domain_level(cs, val);
break;
default:
retval = -EINVAL;
break;
}
out_unlock:
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
return retval;
}
/*
* Common handling for a write to a "cpus" or "mems" file.
*/
static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
struct cpuset *cs = css_cs(of_css(of));
struct cpuset *trialcs;
int retval = -ENODEV;
buf = strstrip(buf);
/*
* CPU or memory hotunplug may leave @cs w/o any execution
* resources, in which case the hotplug code asynchronously updates
* configuration and transfers all tasks to the nearest ancestor
* which can execute.
*
* As writes to "cpus" or "mems" may restore @cs's execution
* resources, wait for the previously scheduled operations before
* proceeding, so that we don't end up keep removing tasks added
* after execution capability is restored.
*
* cpuset_hotplug_work calls back into cgroup core via
* cgroup_transfer_tasks() and waiting for it from a cgroupfs
* operation like this one can lead to a deadlock through kernfs
* active_ref protection. Let's break the protection. Losing the
* protection is okay as we check whether @cs is online after
* grabbing cpuset_mutex anyway. This only happens on the legacy
* hierarchies.
*/
css_get(&cs->css);
kernfs_break_active_protection(of->kn);
flush_work(&cpuset_hotplug_work);
cpus_read_lock();
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs))
goto out_unlock;
trialcs = alloc_trial_cpuset(cs);
if (!trialcs) {
retval = -ENOMEM;
goto out_unlock;
}
switch (of_cft(of)->private) {
case FILE_CPULIST:
retval = update_cpumask(cs, trialcs, buf);
break;
case FILE_EXCLUSIVE_CPULIST:
retval = update_exclusive_cpumask(cs, trialcs, buf);
break;
case FILE_MEMLIST:
retval = update_nodemask(cs, trialcs, buf);
break;
default:
retval = -EINVAL;
break;
}
free_cpuset(trialcs);
out_unlock:
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
kernfs_unbreak_active_protection(of->kn);
css_put(&cs->css);
flush_workqueue(cpuset_migrate_mm_wq);
return retval ?: nbytes;
}
/*
* These ascii lists should be read in a single call, by using a user
* buffer large enough to hold the entire map. If read in smaller
* chunks, there is no guarantee of atomicity. Since the display format
* used, list of ranges of sequential numbers, is variable length,
* and since these maps can change value dynamically, one could read
* gibberish by doing partial reads while a list was changing.
*/
static int cpuset_common_seq_show(struct seq_file *sf, void *v)
{
struct cpuset *cs = css_cs(seq_css(sf));
cpuset_filetype_t type = seq_cft(sf)->private;
int ret = 0;
spin_lock_irq(&callback_lock);
switch (type) {
case FILE_CPULIST:
seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed));
break;
case FILE_MEMLIST:
seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed));
break;
case FILE_EFFECTIVE_CPULIST:
seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus));
break;
case FILE_EFFECTIVE_MEMLIST:
seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems));
break;
case FILE_EXCLUSIVE_CPULIST:
seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->exclusive_cpus));
break;
case FILE_EFFECTIVE_XCPULIST:
seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_xcpus));
break;
case FILE_SUBPARTS_CPULIST:
seq_printf(sf, "%*pbl\n", cpumask_pr_args(subpartitions_cpus));
break;
case FILE_ISOLATED_CPULIST:
seq_printf(sf, "%*pbl\n", cpumask_pr_args(isolated_cpus));
break;
default:
ret = -EINVAL;
}
spin_unlock_irq(&callback_lock);
return ret;
}
static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
{
struct cpuset *cs = css_cs(css);
cpuset_filetype_t type = cft->private;
switch (type) {
case FILE_CPU_EXCLUSIVE:
return is_cpu_exclusive(cs);
case FILE_MEM_EXCLUSIVE:
return is_mem_exclusive(cs);
case FILE_MEM_HARDWALL:
return is_mem_hardwall(cs);
case FILE_SCHED_LOAD_BALANCE:
return is_sched_load_balance(cs);
case FILE_MEMORY_MIGRATE:
return is_memory_migrate(cs);
case FILE_MEMORY_PRESSURE_ENABLED:
return cpuset_memory_pressure_enabled;
case FILE_MEMORY_PRESSURE:
return fmeter_getrate(&cs->fmeter);
case FILE_SPREAD_PAGE:
return is_spread_page(cs);
case FILE_SPREAD_SLAB:
return is_spread_slab(cs);
default:
BUG();
}
/* Unreachable but makes gcc happy */
return 0;
}
static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
{
struct cpuset *cs = css_cs(css);
cpuset_filetype_t type = cft->private;
switch (type) {
case FILE_SCHED_RELAX_DOMAIN_LEVEL:
return cs->relax_domain_level;
default:
BUG();
}
/* Unreachable but makes gcc happy */
return 0;
}
static int sched_partition_show(struct seq_file *seq, void *v)
{
struct cpuset *cs = css_cs(seq_css(seq));
const char *err, *type = NULL;
switch (cs->partition_root_state) {
case PRS_ROOT:
seq_puts(seq, "root\n");
break;
case PRS_ISOLATED:
seq_puts(seq, "isolated\n");
break;
case PRS_MEMBER:
seq_puts(seq, "member\n");
break;
case PRS_INVALID_ROOT:
type = "root";
fallthrough;
case PRS_INVALID_ISOLATED:
if (!type)
type = "isolated";
err = perr_strings[READ_ONCE(cs->prs_err)];
if (err)
seq_printf(seq, "%s invalid (%s)\n", type, err);
else
seq_printf(seq, "%s invalid\n", type);
break;
}
return 0;
}
static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf,
size_t nbytes, loff_t off)
{
struct cpuset *cs = css_cs(of_css(of));
int val;
int retval = -ENODEV;
buf = strstrip(buf);
/*
* Convert "root" to ENABLED, and convert "member" to DISABLED.
*/
if (!strcmp(buf, "root"))
val = PRS_ROOT;
else if (!strcmp(buf, "member"))
val = PRS_MEMBER;
else if (!strcmp(buf, "isolated"))
val = PRS_ISOLATED;
else
return -EINVAL;
css_get(&cs->css);
cpus_read_lock();
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs))
goto out_unlock;
retval = update_prstate(cs, val);
out_unlock:
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
css_put(&cs->css);
return retval ?: nbytes;
}
/*
* for the common functions, 'private' gives the type of file
*/
static struct cftype legacy_files[] = {
{
.name = "cpus",
.seq_show = cpuset_common_seq_show,
.write = cpuset_write_resmask,
.max_write_len = (100U + 6 * NR_CPUS),
.private = FILE_CPULIST,
},
{
.name = "mems",
.seq_show = cpuset_common_seq_show,
.write = cpuset_write_resmask,
.max_write_len = (100U + 6 * MAX_NUMNODES),
.private = FILE_MEMLIST,
},
{
.name = "effective_cpus",
.seq_show = cpuset_common_seq_show,
.private = FILE_EFFECTIVE_CPULIST,
},
{
.name = "effective_mems",
.seq_show = cpuset_common_seq_show,
.private = FILE_EFFECTIVE_MEMLIST,
},
{
.name = "cpu_exclusive",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_CPU_EXCLUSIVE,
},
{
.name = "mem_exclusive",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_MEM_EXCLUSIVE,
},
{
.name = "mem_hardwall",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_MEM_HARDWALL,
},
{
.name = "sched_load_balance",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_SCHED_LOAD_BALANCE,
},
{
.name = "sched_relax_domain_level",
.read_s64 = cpuset_read_s64,
.write_s64 = cpuset_write_s64,
.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
},
{
.name = "memory_migrate",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_MEMORY_MIGRATE,
},
{
.name = "memory_pressure",
.read_u64 = cpuset_read_u64,
.private = FILE_MEMORY_PRESSURE,
},
{
.name = "memory_spread_page",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_SPREAD_PAGE,
},
{
/* obsolete, may be removed in the future */
.name = "memory_spread_slab",
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_SPREAD_SLAB,
},
{
.name = "memory_pressure_enabled",
.flags = CFTYPE_ONLY_ON_ROOT,
.read_u64 = cpuset_read_u64,
.write_u64 = cpuset_write_u64,
.private = FILE_MEMORY_PRESSURE_ENABLED,
},
{ } /* terminate */
};
/*
* This is currently a minimal set for the default hierarchy. It can be
* expanded later on by migrating more features and control files from v1.
*/
static struct cftype dfl_files[] = {
{
.name = "cpus",
.seq_show = cpuset_common_seq_show,
.write = cpuset_write_resmask,
.max_write_len = (100U + 6 * NR_CPUS),
.private = FILE_CPULIST,
.flags = CFTYPE_NOT_ON_ROOT,
},
{
.name = "mems",
.seq_show = cpuset_common_seq_show,
.write = cpuset_write_resmask,
.max_write_len = (100U + 6 * MAX_NUMNODES),
.private = FILE_MEMLIST,
.flags = CFTYPE_NOT_ON_ROOT,
},
{
.name = "cpus.effective",
.seq_show = cpuset_common_seq_show,
.private = FILE_EFFECTIVE_CPULIST,
},
{
.name = "mems.effective",
.seq_show = cpuset_common_seq_show,
.private = FILE_EFFECTIVE_MEMLIST,
},
{
.name = "cpus.partition",
.seq_show = sched_partition_show,
.write = sched_partition_write,
.private = FILE_PARTITION_ROOT,
.flags = CFTYPE_NOT_ON_ROOT,
.file_offset = offsetof(struct cpuset, partition_file),
},
{
.name = "cpus.exclusive",
.seq_show = cpuset_common_seq_show,
.write = cpuset_write_resmask,
.max_write_len = (100U + 6 * NR_CPUS),
.private = FILE_EXCLUSIVE_CPULIST,
.flags = CFTYPE_NOT_ON_ROOT,
},
{
.name = "cpus.exclusive.effective",
.seq_show = cpuset_common_seq_show,
.private = FILE_EFFECTIVE_XCPULIST,
.flags = CFTYPE_NOT_ON_ROOT,
},
{
.name = "cpus.subpartitions",
.seq_show = cpuset_common_seq_show,
.private = FILE_SUBPARTS_CPULIST,
.flags = CFTYPE_ONLY_ON_ROOT | CFTYPE_DEBUG,
},
{
.name = "cpus.isolated",
.seq_show = cpuset_common_seq_show,
.private = FILE_ISOLATED_CPULIST,
.flags = CFTYPE_ONLY_ON_ROOT,
},
{ } /* terminate */
};
/**
* cpuset_css_alloc - Allocate a cpuset css
* @parent_css: Parent css of the control group that the new cpuset will be
* part of
* Return: cpuset css on success, -ENOMEM on failure.
*
* Allocate and initialize a new cpuset css, for non-NULL @parent_css, return
* top cpuset css otherwise.
*/
static struct cgroup_subsys_state *
cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
{
struct cpuset *cs;
if (!parent_css)
return &top_cpuset.css;
cs = kzalloc(sizeof(*cs), GFP_KERNEL);
if (!cs)
return ERR_PTR(-ENOMEM);
if (alloc_cpumasks(cs, NULL)) {
kfree(cs);
return ERR_PTR(-ENOMEM);
}
__set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
nodes_clear(cs->mems_allowed);
nodes_clear(cs->effective_mems);
fmeter_init(&cs->fmeter);
cs->relax_domain_level = -1;
INIT_LIST_HEAD(&cs->remote_sibling);
/* Set CS_MEMORY_MIGRATE for default hierarchy */
if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys))
__set_bit(CS_MEMORY_MIGRATE, &cs->flags);
return &cs->css;
}
static int cpuset_css_online(struct cgroup_subsys_state *css)
{
struct cpuset *cs = css_cs(css);
struct cpuset *parent = parent_cs(cs);
struct cpuset *tmp_cs;
struct cgroup_subsys_state *pos_css;
if (!parent)
return 0;
cpus_read_lock();
mutex_lock(&cpuset_mutex);
set_bit(CS_ONLINE, &cs->flags);
if (is_spread_page(parent))
set_bit(CS_SPREAD_PAGE, &cs->flags);
if (is_spread_slab(parent))
set_bit(CS_SPREAD_SLAB, &cs->flags);
cpuset_inc();
spin_lock_irq(&callback_lock);
if (is_in_v2_mode()) {
cpumask_copy(cs->effective_cpus, parent->effective_cpus);
cs->effective_mems = parent->effective_mems;
cs->use_parent_ecpus = true;
parent->child_ecpus_count++;
/*
* Clear CS_SCHED_LOAD_BALANCE if parent is isolated
*/
if (!is_sched_load_balance(parent))
clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
}
/*
* For v2, clear CS_SCHED_LOAD_BALANCE if parent is isolated
*/
if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
!is_sched_load_balance(parent))
clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
spin_unlock_irq(&callback_lock);
if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
goto out_unlock;
/*
* Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
* set. This flag handling is implemented in cgroup core for
* historical reasons - the flag may be specified during mount.
*
* Currently, if any sibling cpusets have exclusive cpus or mem, we
* refuse to clone the configuration - thereby refusing the task to
* be entered, and as a result refusing the sys_unshare() or
* clone() which initiated it. If this becomes a problem for some
* users who wish to allow that scenario, then this could be
* changed to grant parent->cpus_allowed-sibling_cpus_exclusive
* (and likewise for mems) to the new cgroup.
*/
rcu_read_lock();
cpuset_for_each_child(tmp_cs, pos_css, parent) {
if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
rcu_read_unlock();
goto out_unlock;
}
}
rcu_read_unlock();
spin_lock_irq(&callback_lock);
cs->mems_allowed = parent->mems_allowed;
cs->effective_mems = parent->mems_allowed;
cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
spin_unlock_irq(&callback_lock);
out_unlock:
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
return 0;
}
/*
* If the cpuset being removed has its flag 'sched_load_balance'
* enabled, then simulate turning sched_load_balance off, which
* will call rebuild_sched_domains_locked(). That is not needed
* in the default hierarchy where only changes in partition
* will cause repartitioning.
*
* If the cpuset has the 'sched.partition' flag enabled, simulate
* turning 'sched.partition" off.
*/
static void cpuset_css_offline(struct cgroup_subsys_state *css)
{
struct cpuset *cs = css_cs(css);
cpus_read_lock();
mutex_lock(&cpuset_mutex);
if (is_partition_valid(cs))
update_prstate(cs, 0);
if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
is_sched_load_balance(cs))
update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
if (cs->use_parent_ecpus) {
struct cpuset *parent = parent_cs(cs);
cs->use_parent_ecpus = false;
parent->child_ecpus_count--;
}
cpuset_dec();
clear_bit(CS_ONLINE, &cs->flags);
mutex_unlock(&cpuset_mutex);
cpus_read_unlock();
}
static void cpuset_css_free(struct cgroup_subsys_state *css)
{
struct cpuset *cs = css_cs(css);
free_cpuset(cs);
}
static void cpuset_bind(struct cgroup_subsys_state *root_css)
{
mutex_lock(&cpuset_mutex);
spin_lock_irq(&callback_lock);
if (is_in_v2_mode()) {
cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
cpumask_copy(top_cpuset.effective_xcpus, cpu_possible_mask);
top_cpuset.mems_allowed = node_possible_map;
} else {
cpumask_copy(top_cpuset.cpus_allowed,
top_cpuset.effective_cpus);
top_cpuset.mems_allowed = top_cpuset.effective_mems;
}
spin_unlock_irq(&callback_lock);
mutex_unlock(&cpuset_mutex);
}
/*
* In case the child is cloned into a cpuset different from its parent,
* additional checks are done to see if the move is allowed.
*/
static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
{
struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
bool same_cs;
int ret;
rcu_read_lock();
same_cs = (cs == task_cs(current));
rcu_read_unlock();
if (same_cs)
return 0;
lockdep_assert_held(&cgroup_mutex);
mutex_lock(&cpuset_mutex);
/* Check to see if task is allowed in the cpuset */
ret = cpuset_can_attach_check(cs);
if (ret)
goto out_unlock;
ret = task_can_attach(task);
if (ret)
goto out_unlock;
ret = security_task_setscheduler(task);
if (ret)
goto out_unlock;
/*
* Mark attach is in progress. This makes validate_change() fail
* changes which zero cpus/mems_allowed.
*/
cs->attach_in_progress++;
out_unlock:
mutex_unlock(&cpuset_mutex);
return ret;
}
static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
{
struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
bool same_cs;
rcu_read_lock();
same_cs = (cs == task_cs(current));
rcu_read_unlock();
if (same_cs)
return;
mutex_lock(&cpuset_mutex);
cs->attach_in_progress--;
if (!cs->attach_in_progress)
wake_up(&cpuset_attach_wq);
mutex_unlock(&cpuset_mutex);
}
/*
* Make sure the new task conform to the current state of its parent,
* which could have been changed by cpuset just after it inherits the
* state from the parent and before it sits on the cgroup's task list.
*/
static void cpuset_fork(struct task_struct *task)
{
struct cpuset *cs;
bool same_cs;
rcu_read_lock();
cs = task_cs(task);
same_cs = (cs == task_cs(current));
rcu_read_unlock();
if (same_cs) {
if (cs == &top_cpuset)
return;
set_cpus_allowed_ptr(task, current->cpus_ptr);
task->mems_allowed = current->mems_allowed;
return;
}
/* CLONE_INTO_CGROUP */
mutex_lock(&cpuset_mutex);
guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
cpuset_attach_task(cs, task);
cs->attach_in_progress--;
if (!cs->attach_in_progress)
wake_up(&cpuset_attach_wq);
mutex_unlock(&cpuset_mutex);
}
struct cgroup_subsys cpuset_cgrp_subsys = {
.css_alloc = cpuset_css_alloc,
.css_online = cpuset_css_online,
.css_offline = cpuset_css_offline,
.css_free = cpuset_css_free,
.can_attach = cpuset_can_attach,
.cancel_attach = cpuset_cancel_attach,
.attach = cpuset_attach,
.post_attach = cpuset_post_attach,
.bind = cpuset_bind,
.can_fork = cpuset_can_fork,
.cancel_fork = cpuset_cancel_fork,
.fork = cpuset_fork,
.legacy_cftypes = legacy_files,
.dfl_cftypes = dfl_files,
.early_init = true,
.threaded = true,
};
/**
* cpuset_init - initialize cpusets at system boot
*
* Description: Initialize top_cpuset
**/
int __init cpuset_init(void)
{
BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_xcpus, GFP_KERNEL));
BUG_ON(!alloc_cpumask_var(&top_cpuset.exclusive_cpus, GFP_KERNEL));
BUG_ON(!zalloc_cpumask_var(&subpartitions_cpus, GFP_KERNEL));
BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL));
cpumask_setall(top_cpuset.cpus_allowed);
nodes_setall(top_cpuset.mems_allowed);
cpumask_setall(top_cpuset.effective_cpus);
cpumask_setall(top_cpuset.effective_xcpus);
cpumask_setall(top_cpuset.exclusive_cpus);
nodes_setall(top_cpuset.effective_mems);
fmeter_init(&top_cpuset.fmeter);
set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
top_cpuset.relax_domain_level = -1;
INIT_LIST_HEAD(&remote_children);
BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL));
return 0;
}
/*
* If CPU and/or memory hotplug handlers, below, unplug any CPUs
* or memory nodes, we need to walk over the cpuset hierarchy,
* removing that CPU or node from all cpusets. If this removes the
* last CPU or node from a cpuset, then move the tasks in the empty
* cpuset to its next-highest non-empty parent.
*/
static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
{
struct cpuset *parent;
/*
* Find its next-highest non-empty parent, (top cpuset
* has online cpus, so can't be empty).
*/
parent = parent_cs(cs);
while (cpumask_empty(parent->cpus_allowed) ||
nodes_empty(parent->mems_allowed))
parent = parent_cs(parent);
if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
pr_err("cpuset: failed to transfer tasks out of empty cpuset ");
pr_cont_cgroup_name(cs->css.cgroup);
pr_cont("\n");
}
}
static void
hotplug_update_tasks_legacy(struct cpuset *cs,
struct cpumask *new_cpus, nodemask_t *new_mems,
bool cpus_updated, bool mems_updated)
{
bool is_empty;
spin_lock_irq(&callback_lock);
cpumask_copy(cs->cpus_allowed, new_cpus);
cpumask_copy(cs->effective_cpus, new_cpus);
cs->mems_allowed = *new_mems;
cs->effective_mems = *new_mems;
spin_unlock_irq(&callback_lock);
/*
* Don't call update_tasks_cpumask() if the cpuset becomes empty,
* as the tasks will be migrated to an ancestor.
*/
if (cpus_updated && !cpumask_empty(cs->cpus_allowed))
update_tasks_cpumask(cs, new_cpus);
if (mems_updated && !nodes_empty(cs->mems_allowed))
update_tasks_nodemask(cs);
is_empty = cpumask_empty(cs->cpus_allowed) ||
nodes_empty(cs->mems_allowed);
/*
* Move tasks to the nearest ancestor with execution resources,
* This is full cgroup operation which will also call back into
* cpuset. Should be done outside any lock.
*/
if (is_empty) {
mutex_unlock(&cpuset_mutex);
remove_tasks_in_empty_cpuset(cs);
mutex_lock(&cpuset_mutex);
}
}
static void
hotplug_update_tasks(struct cpuset *cs,
struct cpumask *new_cpus, nodemask_t *new_mems,
bool cpus_updated, bool mems_updated)
{
/* A partition root is allowed to have empty effective cpus */
if (cpumask_empty(new_cpus) && !is_partition_valid(cs))
cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus);
if (nodes_empty(*new_mems))
*new_mems = parent_cs(cs)->effective_mems;
spin_lock_irq(&callback_lock);
cpumask_copy(cs->effective_cpus, new_cpus);
cs->effective_mems = *new_mems;
spin_unlock_irq(&callback_lock);
if (cpus_updated)
update_tasks_cpumask(cs, new_cpus);
if (mems_updated)
update_tasks_nodemask(cs);
}
static bool force_rebuild;
void cpuset_force_rebuild(void)
{
force_rebuild = true;
}
/*
* Attempt to acquire a cpus_read_lock while a hotplug operation may be in
* progress.
* Return: true if successful, false otherwise
*
* To avoid circular lock dependency between cpuset_mutex and cpus_read_lock,
* cpus_read_trylock() is used here to acquire the lock.
*/
static bool cpuset_hotplug_cpus_read_trylock(void)
{
int retries = 0;
while (!cpus_read_trylock()) {
/*
* CPU hotplug still in progress. Retry 5 times
* with a 10ms wait before bailing out.
*/
if (++retries > 5)
return false;
msleep(10);
}
return true;
}
/**
* cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
* @cs: cpuset in interest
* @tmp: the tmpmasks structure pointer
*
* Compare @cs's cpu and mem masks against top_cpuset and if some have gone
* offline, update @cs accordingly. If @cs ends up with no CPU or memory,
* all its tasks are moved to the nearest ancestor with both resources.
*/
static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
{
static cpumask_t new_cpus;
static nodemask_t new_mems;
bool cpus_updated;
bool mems_updated;
bool remote;
int partcmd = -1;
struct cpuset *parent;
retry:
wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
mutex_lock(&cpuset_mutex);
/*
* We have raced with task attaching. We wait until attaching
* is finished, so we won't attach a task to an empty cpuset.
*/
if (cs->attach_in_progress) {
mutex_unlock(&cpuset_mutex);
goto retry;
}
parent = parent_cs(cs);
compute_effective_cpumask(&new_cpus, cs, parent);
nodes_and(new_mems, cs->mems_allowed, parent->effective_mems);
if (!tmp || !cs->partition_root_state)
goto update_tasks;
/*
* Compute effective_cpus for valid partition root, may invalidate
* child partition roots if necessary.
*/
remote = is_remote_partition(cs);
if (remote || (is_partition_valid(cs) && is_partition_valid(parent)))
compute_partition_effective_cpumask(cs, &new_cpus);
if (remote && cpumask_empty(&new_cpus) &&
partition_is_populated(cs, NULL) &&
cpuset_hotplug_cpus_read_trylock()) {
remote_partition_disable(cs, tmp);
compute_effective_cpumask(&new_cpus, cs, parent);
remote = false;
cpuset_force_rebuild();
cpus_read_unlock();
}
/*
* Force the partition to become invalid if either one of
* the following conditions hold:
* 1) empty effective cpus but not valid empty partition.
* 2) parent is invalid or doesn't grant any cpus to child
* partitions.
*/
if (is_local_partition(cs) && (!is_partition_valid(parent) ||
tasks_nocpu_error(parent, cs, &new_cpus)))
partcmd = partcmd_invalidate;
/*
* On the other hand, an invalid partition root may be transitioned
* back to a regular one.
*/
else if (is_partition_valid(parent) && is_partition_invalid(cs))
partcmd = partcmd_update;
/*
* cpus_read_lock needs to be held before calling
* update_parent_effective_cpumask(). To avoid circular lock
* dependency between cpuset_mutex and cpus_read_lock,
* cpus_read_trylock() is used here to acquire the lock.
*/
if (partcmd >= 0) {
if (!cpuset_hotplug_cpus_read_trylock())
goto update_tasks;
update_parent_effective_cpumask(cs, partcmd, NULL, tmp);
cpus_read_unlock();
if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) {
compute_partition_effective_cpumask(cs, &new_cpus);
cpuset_force_rebuild();
}
}
update_tasks:
cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus);
mems_updated = !nodes_equal(new_mems, cs->effective_mems);
if (!cpus_updated && !mems_updated)
goto unlock; /* Hotplug doesn't affect this cpuset */
if (mems_updated)
check_insane_mems_config(&new_mems);
if (is_in_v2_mode())
hotplug_update_tasks(cs, &new_cpus, &new_mems,
cpus_updated, mems_updated);
else
hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems,
cpus_updated, mems_updated);
unlock:
mutex_unlock(&cpuset_mutex);
}
/**
* cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
* @work: unused
*
* This function is called after either CPU or memory configuration has
* changed and updates cpuset accordingly. The top_cpuset is always
* synchronized to cpu_active_mask and N_MEMORY, which is necessary in
* order to make cpusets transparent (of no affect) on systems that are
* actively using CPU hotplug but making no active use of cpusets.
*
* Non-root cpusets are only affected by offlining. If any CPUs or memory
* nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
* all descendants.
*
* Note that CPU offlining during suspend is ignored. We don't modify
* cpusets across suspend/resume cycles at all.
*/
static void cpuset_hotplug_workfn(struct work_struct *work)
{
static cpumask_t new_cpus;
static nodemask_t new_mems;
bool cpus_updated, mems_updated;
bool on_dfl = is_in_v2_mode();
struct tmpmasks tmp, *ptmp = NULL;
if (on_dfl && !alloc_cpumasks(NULL, &tmp))
ptmp = &tmp;
mutex_lock(&cpuset_mutex);
/* fetch the available cpus/mems and find out which changed how */
cpumask_copy(&new_cpus, cpu_active_mask);
new_mems = node_states[N_MEMORY];
/*
* If subpartitions_cpus is populated, it is likely that the check
* below will produce a false positive on cpus_updated when the cpu
* list isn't changed. It is extra work, but it is better to be safe.
*/
cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus) ||
!cpumask_empty(subpartitions_cpus);
mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);
/*
* In the rare case that hotplug removes all the cpus in
* subpartitions_cpus, we assumed that cpus are updated.
*/
if (!cpus_updated && top_cpuset.nr_subparts)
cpus_updated = true;
/* For v1, synchronize cpus_allowed to cpu_active_mask */
if (cpus_updated) {
spin_lock_irq(&callback_lock);
if (!on_dfl)
cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
/*
* Make sure that CPUs allocated to child partitions
* do not show up in effective_cpus. If no CPU is left,
* we clear the subpartitions_cpus & let the child partitions
* fight for the CPUs again.
*/
if (!cpumask_empty(subpartitions_cpus)) {
if (cpumask_subset(&new_cpus, subpartitions_cpus)) {
top_cpuset.nr_subparts = 0;
cpumask_clear(subpartitions_cpus);
} else {
cpumask_andnot(&new_cpus, &new_cpus,
subpartitions_cpus);
}
}
cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
spin_unlock_irq(&callback_lock);
/* we don't mess with cpumasks of tasks in top_cpuset */
}
/* synchronize mems_allowed to N_MEMORY */
if (mems_updated) {
spin_lock_irq(&callback_lock);
if (!on_dfl)
top_cpuset.mems_allowed = new_mems;
top_cpuset.effective_mems = new_mems;
spin_unlock_irq(&callback_lock);
update_tasks_nodemask(&top_cpuset);
}
mutex_unlock(&cpuset_mutex);
/* if cpus or mems changed, we need to propagate to descendants */
if (cpus_updated || mems_updated) {
struct cpuset *cs;
struct cgroup_subsys_state *pos_css;
rcu_read_lock();
cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
if (cs == &top_cpuset || !css_tryget_online(&cs->css))
continue;
rcu_read_unlock();
cpuset_hotplug_update_tasks(cs, ptmp);
rcu_read_lock();
css_put(&cs->css);
}
rcu_read_unlock();
}
/* rebuild sched domains if cpus_allowed has changed */
if (cpus_updated || force_rebuild) {
force_rebuild = false;
rebuild_sched_domains();
}
free_cpumasks(NULL, ptmp);
}
void cpuset_update_active_cpus(void)
{
/*
* We're inside cpu hotplug critical region which usually nests
* inside cgroup synchronization. Bounce actual hotplug processing
* to a work item to avoid reverse locking order.
*/
schedule_work(&cpuset_hotplug_work);
}
void cpuset_wait_for_hotplug(void)
{
flush_work(&cpuset_hotplug_work);
}
/*
* Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
* Call this routine anytime after node_states[N_MEMORY] changes.
* See cpuset_update_active_cpus() for CPU hotplug handling.
*/
static int cpuset_track_online_nodes(struct notifier_block *self,
unsigned long action, void *arg)
{
schedule_work(&cpuset_hotplug_work);
return NOTIFY_OK;
}
/**
* cpuset_init_smp - initialize cpus_allowed
*
* Description: Finish top cpuset after cpu, node maps are initialized
*/
void __init cpuset_init_smp(void)
{
/*
* cpus_allowd/mems_allowed set to v2 values in the initial
* cpuset_bind() call will be reset to v1 values in another
* cpuset_bind() call when v1 cpuset is mounted.
*/
top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
top_cpuset.effective_mems = node_states[N_MEMORY];
hotplug_memory_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);
cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
BUG_ON(!cpuset_migrate_mm_wq);
}
/**
* cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
* @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
* @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
*
* Description: Returns the cpumask_var_t cpus_allowed of the cpuset
* attached to the specified @tsk. Guaranteed to return some non-empty
* subset of cpu_online_mask, even if this means going outside the
* tasks cpuset, except when the task is in the top cpuset.
**/
void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
{
unsigned long flags;
struct cpuset *cs;
spin_lock_irqsave(&callback_lock, flags);
rcu_read_lock();
cs = task_cs(tsk);
if (cs != &top_cpuset)
guarantee_online_cpus(tsk, pmask);
/*
* Tasks in the top cpuset won't get update to their cpumasks
* when a hotplug online/offline event happens. So we include all
* offline cpus in the allowed cpu list.
*/
if ((cs == &top_cpuset) || cpumask_empty(pmask)) {
const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
/*
* We first exclude cpus allocated to partitions. If there is no
* allowable online cpu left, we fall back to all possible cpus.
*/
cpumask_andnot(pmask, possible_mask, subpartitions_cpus);
if (!cpumask_intersects(pmask, cpu_online_mask))
cpumask_copy(pmask, possible_mask);
}
rcu_read_unlock();
spin_unlock_irqrestore(&callback_lock, flags);
}
/**
* cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
* @tsk: pointer to task_struct with which the scheduler is struggling
*
* Description: In the case that the scheduler cannot find an allowed cpu in
* tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
* mode however, this value is the same as task_cs(tsk)->effective_cpus,
* which will not contain a sane cpumask during cases such as cpu hotplugging.
* This is the absolute last resort for the scheduler and it is only used if
* _every_ other avenue has been traveled.
*
* Returns true if the affinity of @tsk was changed, false otherwise.
**/
bool cpuset_cpus_allowed_fallback(struct task_struct *tsk)
{
const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
const struct cpumask *cs_mask;
bool changed = false;
rcu_read_lock();
cs_mask = task_cs(tsk)->cpus_allowed;
if (is_in_v2_mode() && cpumask_subset(cs_mask, possible_mask)) {
do_set_cpus_allowed(tsk, cs_mask);
changed = true;
}
rcu_read_unlock();
/*
* We own tsk->cpus_allowed, nobody can change it under us.
*
* But we used cs && cs->cpus_allowed lockless and thus can
* race with cgroup_attach_task() or update_cpumask() and get
* the wrong tsk->cpus_allowed. However, both cases imply the
* subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
* which takes task_rq_lock().
*
* If we are called after it dropped the lock we must see all
* changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
* set any mask even if it is not right from task_cs() pov,
* the pending set_cpus_allowed_ptr() will fix things.
*
* select_fallback_rq() will fix things ups and set cpu_possible_mask
* if required.
*/
return changed;
}
void __init cpuset_init_current_mems_allowed(void)
{
nodes_setall(current->mems_allowed);
}
/**
* cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
* @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
*
* Description: Returns the nodemask_t mems_allowed of the cpuset
* attached to the specified @tsk. Guaranteed to return some non-empty
* subset of node_states[N_MEMORY], even if this means going outside the
* tasks cpuset.
**/
nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
{
nodemask_t mask;
unsigned long flags;
spin_lock_irqsave(&callback_lock, flags);
rcu_read_lock();
guarantee_online_mems(task_cs(tsk), &mask);
rcu_read_unlock();
spin_unlock_irqrestore(&callback_lock, flags);
return mask;
}
/**
* cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed
* @nodemask: the nodemask to be checked
*
* Are any of the nodes in the nodemask allowed in current->mems_allowed?
*/
int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
{
return nodes_intersects(*nodemask, current->mems_allowed);
}
/*
* nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
* mem_hardwall ancestor to the specified cpuset. Call holding
* callback_lock. If no ancestor is mem_exclusive or mem_hardwall
* (an unusual configuration), then returns the root cpuset.
*/
static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
{
while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
cs = parent_cs(cs);
return cs;
}
/*
* cpuset_node_allowed - Can we allocate on a memory node?
* @node: is this an allowed node?
* @gfp_mask: memory allocation flags
*
* If we're in interrupt, yes, we can always allocate. If @node is set in
* current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this
* node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
* yes. If current has access to memory reserves as an oom victim, yes.
* Otherwise, no.
*
* GFP_USER allocations are marked with the __GFP_HARDWALL bit,
* and do not allow allocations outside the current tasks cpuset
* unless the task has been OOM killed.
* GFP_KERNEL allocations are not so marked, so can escape to the
* nearest enclosing hardwalled ancestor cpuset.
*
* Scanning up parent cpusets requires callback_lock. The
* __alloc_pages() routine only calls here with __GFP_HARDWALL bit
* _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
* current tasks mems_allowed came up empty on the first pass over
* the zonelist. So only GFP_KERNEL allocations, if all nodes in the
* cpuset are short of memory, might require taking the callback_lock.
*
* The first call here from mm/page_alloc:get_page_from_freelist()
* has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
* so no allocation on a node outside the cpuset is allowed (unless
* in interrupt, of course).
*
* The second pass through get_page_from_freelist() doesn't even call
* here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
* variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
* in alloc_flags. That logic and the checks below have the combined
* affect that:
* in_interrupt - any node ok (current task context irrelevant)
* GFP_ATOMIC - any node ok
* tsk_is_oom_victim - any node ok
* GFP_KERNEL - any node in enclosing hardwalled cpuset ok
* GFP_USER - only nodes in current tasks mems allowed ok.
*/
bool cpuset_node_allowed(int node, gfp_t gfp_mask)
{
struct cpuset *cs; /* current cpuset ancestors */
bool allowed; /* is allocation in zone z allowed? */
unsigned long flags;
if (in_interrupt()) return true; if (node_isset(node, current->mems_allowed))
return true;
/*
* Allow tasks that have access to memory reserves because they have
* been OOM killed to get memory anywhere.
*/
if (unlikely(tsk_is_oom_victim(current)))
return true;
if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
return false;
if (current->flags & PF_EXITING) /* Let dying task have memory */
return true;
/* Not hardwall and node outside mems_allowed: scan up cpusets */
spin_lock_irqsave(&callback_lock, flags); rcu_read_lock(); cs = nearest_hardwall_ancestor(task_cs(current)); allowed = node_isset(node, cs->mems_allowed); rcu_read_unlock();
spin_unlock_irqrestore(&callback_lock, flags);
return allowed;
}
/**
* cpuset_spread_node() - On which node to begin search for a page
* @rotor: round robin rotor
*
* If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
* tasks in a cpuset with is_spread_page or is_spread_slab set),
* and if the memory allocation used cpuset_mem_spread_node()
* to determine on which node to start looking, as it will for
* certain page cache or slab cache pages such as used for file
* system buffers and inode caches, then instead of starting on the
* local node to look for a free page, rather spread the starting
* node around the tasks mems_allowed nodes.
*
* We don't have to worry about the returned node being offline
* because "it can't happen", and even if it did, it would be ok.
*
* The routines calling guarantee_online_mems() are careful to
* only set nodes in task->mems_allowed that are online. So it
* should not be possible for the following code to return an
* offline node. But if it did, that would be ok, as this routine
* is not returning the node where the allocation must be, only
* the node where the search should start. The zonelist passed to
* __alloc_pages() will include all nodes. If the slab allocator
* is passed an offline node, it will fall back to the local node.
* See kmem_cache_alloc_node().
*/
static int cpuset_spread_node(int *rotor)
{
return *rotor = next_node_in(*rotor, current->mems_allowed);
}
/**
* cpuset_mem_spread_node() - On which node to begin search for a file page
*/
int cpuset_mem_spread_node(void)
{
if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
current->cpuset_mem_spread_rotor =
node_random(¤t->mems_allowed);
return cpuset_spread_node(¤t->cpuset_mem_spread_rotor);
}
/**
* cpuset_slab_spread_node() - On which node to begin search for a slab page
*/
int cpuset_slab_spread_node(void)
{
if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
current->cpuset_slab_spread_rotor =
node_random(¤t->mems_allowed);
return cpuset_spread_node(¤t->cpuset_slab_spread_rotor);
}
EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
/**
* cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
* @tsk1: pointer to task_struct of some task.
* @tsk2: pointer to task_struct of some other task.
*
* Description: Return true if @tsk1's mems_allowed intersects the
* mems_allowed of @tsk2. Used by the OOM killer to determine if
* one of the task's memory usage might impact the memory available
* to the other.
**/
int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
const struct task_struct *tsk2)
{
return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
}
/**
* cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
*
* Description: Prints current's name, cpuset name, and cached copy of its
* mems_allowed to the kernel log.
*/
void cpuset_print_current_mems_allowed(void)
{
struct cgroup *cgrp;
rcu_read_lock();
cgrp = task_cs(current)->css.cgroup;
pr_cont(",cpuset=");
pr_cont_cgroup_name(cgrp);
pr_cont(",mems_allowed=%*pbl",
nodemask_pr_args(¤t->mems_allowed));
rcu_read_unlock();
}
/*
* Collection of memory_pressure is suppressed unless
* this flag is enabled by writing "1" to the special
* cpuset file 'memory_pressure_enabled' in the root cpuset.
*/
int cpuset_memory_pressure_enabled __read_mostly;
/*
* __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
*
* Keep a running average of the rate of synchronous (direct)
* page reclaim efforts initiated by tasks in each cpuset.
*
* This represents the rate at which some task in the cpuset
* ran low on memory on all nodes it was allowed to use, and
* had to enter the kernels page reclaim code in an effort to
* create more free memory by tossing clean pages or swapping
* or writing dirty pages.
*
* Display to user space in the per-cpuset read-only file
* "memory_pressure". Value displayed is an integer
* representing the recent rate of entry into the synchronous
* (direct) page reclaim by any task attached to the cpuset.
*/
void __cpuset_memory_pressure_bump(void)
{
rcu_read_lock();
fmeter_markevent(&task_cs(current)->fmeter);
rcu_read_unlock();
}
#ifdef CONFIG_PROC_PID_CPUSET
/*
* proc_cpuset_show()
* - Print tasks cpuset path into seq_file.
* - Used for /proc/<pid>/cpuset.
* - No need to task_lock(tsk) on this tsk->cpuset reference, as it
* doesn't really matter if tsk->cpuset changes after we read it,
* and we take cpuset_mutex, keeping cpuset_attach() from changing it
* anyway.
*/
int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *tsk)
{
char *buf;
struct cgroup_subsys_state *css;
int retval;
retval = -ENOMEM;
buf = kmalloc(PATH_MAX, GFP_KERNEL);
if (!buf)
goto out;
css = task_get_css(tsk, cpuset_cgrp_id);
retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX,
current->nsproxy->cgroup_ns);
css_put(css);
if (retval == -E2BIG)
retval = -ENAMETOOLONG;
if (retval < 0)
goto out_free;
seq_puts(m, buf);
seq_putc(m, '\n');
retval = 0;
out_free:
kfree(buf);
out:
return retval;
}
#endif /* CONFIG_PROC_PID_CPUSET */
/* Display task mems_allowed in /proc/<pid>/status file. */
void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
{
seq_printf(m, "Mems_allowed:\t%*pb\n",
nodemask_pr_args(&task->mems_allowed));
seq_printf(m, "Mems_allowed_list:\t%*pbl\n",
nodemask_pr_args(&task->mems_allowed));
}
// SPDX-License-Identifier: GPL-2.0-or-later
// SPI init/core code
//
// Copyright (C) 2005 David Brownell
// Copyright (C) 2008 Secret Lab Technologies Ltd.
#include <linux/acpi.h>
#include <linux/cache.h>
#include <linux/clk/clk-conf.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/export.h>
#include <linux/gpio/consumer.h>
#include <linux/highmem.h>
#include <linux/idr.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/mod_devicetable.h>
#include <linux/mutex.h>
#include <linux/of_device.h>
#include <linux/of_irq.h>
#include <linux/percpu.h>
#include <linux/platform_data/x86/apple.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/property.h>
#include <linux/ptp_clock_kernel.h>
#include <linux/sched/rt.h>
#include <linux/slab.h>
#include <linux/spi/spi.h>
#include <linux/spi/spi-mem.h>
#include <uapi/linux/sched/types.h>
#define CREATE_TRACE_POINTS
#include <trace/events/spi.h>
EXPORT_TRACEPOINT_SYMBOL(spi_transfer_start);
EXPORT_TRACEPOINT_SYMBOL(spi_transfer_stop);
#include "internals.h"
static DEFINE_IDR(spi_master_idr);
static void spidev_release(struct device *dev)
{
struct spi_device *spi = to_spi_device(dev);
spi_controller_put(spi->controller); kfree(spi->driver_override);
free_percpu(spi->pcpu_statistics);
kfree(spi);
}
static ssize_t
modalias_show(struct device *dev, struct device_attribute *a, char *buf)
{
const struct spi_device *spi = to_spi_device(dev);
int len;
len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1);
if (len != -ENODEV)
return len;
return sysfs_emit(buf, "%s%s\n", SPI_MODULE_PREFIX, spi->modalias);
}
static DEVICE_ATTR_RO(modalias);
static ssize_t driver_override_store(struct device *dev,
struct device_attribute *a,
const char *buf, size_t count)
{
struct spi_device *spi = to_spi_device(dev);
int ret;
ret = driver_set_override(dev, &spi->driver_override, buf, count);
if (ret)
return ret;
return count;
}
static ssize_t driver_override_show(struct device *dev,
struct device_attribute *a, char *buf)
{
const struct spi_device *spi = to_spi_device(dev);
ssize_t len;
device_lock(dev);
len = sysfs_emit(buf, "%s\n", spi->driver_override ? : "");
device_unlock(dev);
return len;
}
static DEVICE_ATTR_RW(driver_override);
static struct spi_statistics __percpu *spi_alloc_pcpu_stats(struct device *dev)
{
struct spi_statistics __percpu *pcpu_stats;
if (dev)
pcpu_stats = devm_alloc_percpu(dev, struct spi_statistics);
else
pcpu_stats = alloc_percpu_gfp(struct spi_statistics, GFP_KERNEL);
if (pcpu_stats) {
int cpu;
for_each_possible_cpu(cpu) {
struct spi_statistics *stat;
stat = per_cpu_ptr(pcpu_stats, cpu);
u64_stats_init(&stat->syncp);
}
}
return pcpu_stats;
}
static ssize_t spi_emit_pcpu_stats(struct spi_statistics __percpu *stat,
char *buf, size_t offset)
{
u64 val = 0;
int i;
for_each_possible_cpu(i) {
const struct spi_statistics *pcpu_stats;
u64_stats_t *field;
unsigned int start;
u64 inc;
pcpu_stats = per_cpu_ptr(stat, i);
field = (void *)pcpu_stats + offset;
do {
start = u64_stats_fetch_begin(&pcpu_stats->syncp);
inc = u64_stats_read(field);
} while (u64_stats_fetch_retry(&pcpu_stats->syncp, start));
val += inc;
}
return sysfs_emit(buf, "%llu\n", val);
}
#define SPI_STATISTICS_ATTRS(field, file) \
static ssize_t spi_controller_##field##_show(struct device *dev, \
struct device_attribute *attr, \
char *buf) \
{ \
struct spi_controller *ctlr = container_of(dev, \
struct spi_controller, dev); \
return spi_statistics_##field##_show(ctlr->pcpu_statistics, buf); \
} \
static struct device_attribute dev_attr_spi_controller_##field = { \
.attr = { .name = file, .mode = 0444 }, \
.show = spi_controller_##field##_show, \
}; \
static ssize_t spi_device_##field##_show(struct device *dev, \
struct device_attribute *attr, \
char *buf) \
{ \
struct spi_device *spi = to_spi_device(dev); \
return spi_statistics_##field##_show(spi->pcpu_statistics, buf); \
} \
static struct device_attribute dev_attr_spi_device_##field = { \
.attr = { .name = file, .mode = 0444 }, \
.show = spi_device_##field##_show, \
}
#define SPI_STATISTICS_SHOW_NAME(name, file, field) \
static ssize_t spi_statistics_##name##_show(struct spi_statistics __percpu *stat, \
char *buf) \
{ \
return spi_emit_pcpu_stats(stat, buf, \
offsetof(struct spi_statistics, field)); \
} \
SPI_STATISTICS_ATTRS(name, file)
#define SPI_STATISTICS_SHOW(field) \
SPI_STATISTICS_SHOW_NAME(field, __stringify(field), \
field)
SPI_STATISTICS_SHOW(messages);
SPI_STATISTICS_SHOW(transfers);
SPI_STATISTICS_SHOW(errors);
SPI_STATISTICS_SHOW(timedout);
SPI_STATISTICS_SHOW(spi_sync);
SPI_STATISTICS_SHOW(spi_sync_immediate);
SPI_STATISTICS_SHOW(spi_async);
SPI_STATISTICS_SHOW(bytes);
SPI_STATISTICS_SHOW(bytes_rx);
SPI_STATISTICS_SHOW(bytes_tx);
#define SPI_STATISTICS_TRANSFER_BYTES_HISTO(index, number) \
SPI_STATISTICS_SHOW_NAME(transfer_bytes_histo##index, \
"transfer_bytes_histo_" number, \
transfer_bytes_histo[index])
SPI_STATISTICS_TRANSFER_BYTES_HISTO(0, "0-1");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(1, "2-3");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(2, "4-7");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(3, "8-15");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(4, "16-31");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(5, "32-63");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(6, "64-127");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(7, "128-255");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(8, "256-511");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(9, "512-1023");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(10, "1024-2047");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(11, "2048-4095");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(12, "4096-8191");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(13, "8192-16383");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(14, "16384-32767");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(15, "32768-65535");
SPI_STATISTICS_TRANSFER_BYTES_HISTO(16, "65536+");
SPI_STATISTICS_SHOW(transfers_split_maxsize);
static struct attribute *spi_dev_attrs[] = {
&dev_attr_modalias.attr,
&dev_attr_driver_override.attr,
NULL,
};
static const struct attribute_group spi_dev_group = {
.attrs = spi_dev_attrs,
};
static struct attribute *spi_device_statistics_attrs[] = {
&dev_attr_spi_device_messages.attr,
&dev_attr_spi_device_transfers.attr,
&dev_attr_spi_device_errors.attr,
&dev_attr_spi_device_timedout.attr,
&dev_attr_spi_device_spi_sync.attr,
&dev_attr_spi_device_spi_sync_immediate.attr,
&dev_attr_spi_device_spi_async.attr,
&dev_attr_spi_device_bytes.attr,
&dev_attr_spi_device_bytes_rx.attr,
&dev_attr_spi_device_bytes_tx.attr,
&dev_attr_spi_device_transfer_bytes_histo0.attr,
&dev_attr_spi_device_transfer_bytes_histo1.attr,
&dev_attr_spi_device_transfer_bytes_histo2.attr,
&dev_attr_spi_device_transfer_bytes_histo3.attr,
&dev_attr_spi_device_transfer_bytes_histo4.attr,
&dev_attr_spi_device_transfer_bytes_histo5.attr,
&dev_attr_spi_device_transfer_bytes_histo6.attr,
&dev_attr_spi_device_transfer_bytes_histo7.attr,
&dev_attr_spi_device_transfer_bytes_histo8.attr,
&dev_attr_spi_device_transfer_bytes_histo9.attr,
&dev_attr_spi_device_transfer_bytes_histo10.attr,
&dev_attr_spi_device_transfer_bytes_histo11.attr,
&dev_attr_spi_device_transfer_bytes_histo12.attr,
&dev_attr_spi_device_transfer_bytes_histo13.attr,
&dev_attr_spi_device_transfer_bytes_histo14.attr,
&dev_attr_spi_device_transfer_bytes_histo15.attr,
&dev_attr_spi_device_transfer_bytes_histo16.attr,
&dev_attr_spi_device_transfers_split_maxsize.attr,
NULL,
};
static const struct attribute_group spi_device_statistics_group = {
.name = "statistics",
.attrs = spi_device_statistics_attrs,
};
static const struct attribute_group *spi_dev_groups[] = {
&spi_dev_group,
&spi_device_statistics_group,
NULL,
};
static struct attribute *spi_controller_statistics_attrs[] = {
&dev_attr_spi_controller_messages.attr,
&dev_attr_spi_controller_transfers.attr,
&dev_attr_spi_controller_errors.attr,
&dev_attr_spi_controller_timedout.attr,
&dev_attr_spi_controller_spi_sync.attr,
&dev_attr_spi_controller_spi_sync_immediate.attr,
&dev_attr_spi_controller_spi_async.attr,
&dev_attr_spi_controller_bytes.attr,
&dev_attr_spi_controller_bytes_rx.attr,
&dev_attr_spi_controller_bytes_tx.attr,
&dev_attr_spi_controller_transfer_bytes_histo0.attr,
&dev_attr_spi_controller_transfer_bytes_histo1.attr,
&dev_attr_spi_controller_transfer_bytes_histo2.attr,
&dev_attr_spi_controller_transfer_bytes_histo3.attr,
&dev_attr_spi_controller_transfer_bytes_histo4.attr,
&dev_attr_spi_controller_transfer_bytes_histo5.attr,
&dev_attr_spi_controller_transfer_bytes_histo6.attr,
&dev_attr_spi_controller_transfer_bytes_histo7.attr,
&dev_attr_spi_controller_transfer_bytes_histo8.attr,
&dev_attr_spi_controller_transfer_bytes_histo9.attr,
&dev_attr_spi_controller_transfer_bytes_histo10.attr,
&dev_attr_spi_controller_transfer_bytes_histo11.attr,
&dev_attr_spi_controller_transfer_bytes_histo12.attr,
&dev_attr_spi_controller_transfer_bytes_histo13.attr,
&dev_attr_spi_controller_transfer_bytes_histo14.attr,
&dev_attr_spi_controller_transfer_bytes_histo15.attr,
&dev_attr_spi_controller_transfer_bytes_histo16.attr,
&dev_attr_spi_controller_transfers_split_maxsize.attr,
NULL,
};
static const struct attribute_group spi_controller_statistics_group = {
.name = "statistics",
.attrs = spi_controller_statistics_attrs,
};
static const struct attribute_group *spi_master_groups[] = {
&spi_controller_statistics_group,
NULL,
};
static void spi_statistics_add_transfer_stats(struct spi_statistics __percpu *pcpu_stats,
struct spi_transfer *xfer,
struct spi_controller *ctlr)
{
int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1;
struct spi_statistics *stats;
if (l2len < 0)
l2len = 0;
get_cpu();
stats = this_cpu_ptr(pcpu_stats);
u64_stats_update_begin(&stats->syncp);
u64_stats_inc(&stats->transfers);
u64_stats_inc(&stats->transfer_bytes_histo[l2len]);
u64_stats_add(&stats->bytes, xfer->len);
if ((xfer->tx_buf) &&
(xfer->tx_buf != ctlr->dummy_tx))
u64_stats_add(&stats->bytes_tx, xfer->len);
if ((xfer->rx_buf) &&
(xfer->rx_buf != ctlr->dummy_rx))
u64_stats_add(&stats->bytes_rx, xfer->len);
u64_stats_update_end(&stats->syncp);
put_cpu();
}
/*
* modalias support makes "modprobe $MODALIAS" new-style hotplug work,
* and the sysfs version makes coldplug work too.
*/
static const struct spi_device_id *spi_match_id(const struct spi_device_id *id, const char *name)
{
while (id->name[0]) {
if (!strcmp(name, id->name))
return id;
id++;
}
return NULL;
}
const struct spi_device_id *spi_get_device_id(const struct spi_device *sdev)
{
const struct spi_driver *sdrv = to_spi_driver(sdev->dev.driver);
return spi_match_id(sdrv->id_table, sdev->modalias);
}
EXPORT_SYMBOL_GPL(spi_get_device_id);
const void *spi_get_device_match_data(const struct spi_device *sdev)
{
const void *match;
match = device_get_match_data(&sdev->dev);
if (match)
return match;
return (const void *)spi_get_device_id(sdev)->driver_data;
}
EXPORT_SYMBOL_GPL(spi_get_device_match_data);
static int spi_match_device(struct device *dev, struct device_driver *drv)
{
const struct spi_device *spi = to_spi_device(dev);
const struct spi_driver *sdrv = to_spi_driver(drv);
/* Check override first, and if set, only use the named driver */
if (spi->driver_override)
return strcmp(spi->driver_override, drv->name) == 0;
/* Attempt an OF style match */
if (of_driver_match_device(dev, drv))
return 1;
/* Then try ACPI */
if (acpi_driver_match_device(dev, drv))
return 1;
if (sdrv->id_table)
return !!spi_match_id(sdrv->id_table, spi->modalias);
return strcmp(spi->modalias, drv->name) == 0;
}
static int spi_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct spi_device *spi = to_spi_device(dev);
int rc;
rc = acpi_device_uevent_modalias(dev, env); if (rc != -ENODEV)
return rc;
return add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);
}
static int spi_probe(struct device *dev)
{
const struct spi_driver *sdrv = to_spi_driver(dev->driver);
struct spi_device *spi = to_spi_device(dev);
int ret;
ret = of_clk_set_defaults(dev->of_node, false);
if (ret)
return ret;
if (dev->of_node) {
spi->irq = of_irq_get(dev->of_node, 0);
if (spi->irq == -EPROBE_DEFER)
return -EPROBE_DEFER;
if (spi->irq < 0)
spi->irq = 0;
}
ret = dev_pm_domain_attach(dev, true);
if (ret)
return ret;
if (sdrv->probe) {
ret = sdrv->probe(spi);
if (ret)
dev_pm_domain_detach(dev, true);
}
return ret;
}
static void spi_remove(struct device *dev)
{
const struct spi_driver *sdrv = to_spi_driver(dev->driver); if (sdrv->remove) sdrv->remove(to_spi_device(dev)); dev_pm_domain_detach(dev, true);
}
static void spi_shutdown(struct device *dev)
{
if (dev->driver) {
const struct spi_driver *sdrv = to_spi_driver(dev->driver);
if (sdrv->shutdown)
sdrv->shutdown(to_spi_device(dev));
}
}
const struct bus_type spi_bus_type = {
.name = "spi",
.dev_groups = spi_dev_groups,
.match = spi_match_device,
.uevent = spi_uevent,
.probe = spi_probe,
.remove = spi_remove,
.shutdown = spi_shutdown,
};
EXPORT_SYMBOL_GPL(spi_bus_type);
/**
* __spi_register_driver - register a SPI driver
* @owner: owner module of the driver to register
* @sdrv: the driver to register
* Context: can sleep
*
* Return: zero on success, else a negative error code.
*/
int __spi_register_driver(struct module *owner, struct spi_driver *sdrv)
{
sdrv->driver.owner = owner;
sdrv->driver.bus = &spi_bus_type;
/*
* For Really Good Reasons we use spi: modaliases not of:
* modaliases for DT so module autoloading won't work if we
* don't have a spi_device_id as well as a compatible string.
*/
if (sdrv->driver.of_match_table) {
const struct of_device_id *of_id;
for (of_id = sdrv->driver.of_match_table; of_id->compatible[0];
of_id++) {
const char *of_name;
/* Strip off any vendor prefix */
of_name = strnchr(of_id->compatible,
sizeof(of_id->compatible), ',');
if (of_name)
of_name++;
else
of_name = of_id->compatible;
if (sdrv->id_table) {
const struct spi_device_id *spi_id;
spi_id = spi_match_id(sdrv->id_table, of_name);
if (spi_id)
continue;
} else {
if (strcmp(sdrv->driver.name, of_name) == 0)
continue;
}
pr_warn("SPI driver %s has no spi_device_id for %s\n",
sdrv->driver.name, of_id->compatible);
}
}
return driver_register(&sdrv->driver);
}
EXPORT_SYMBOL_GPL(__spi_register_driver);
/*-------------------------------------------------------------------------*/
/*
* SPI devices should normally not be created by SPI device drivers; that
* would make them board-specific. Similarly with SPI controller drivers.
* Device registration normally goes into like arch/.../mach.../board-YYY.c
* with other readonly (flashable) information about mainboard devices.
*/
struct boardinfo {
struct list_head list;
struct spi_board_info board_info;
};
static LIST_HEAD(board_list);
static LIST_HEAD(spi_controller_list);
/*
* Used to protect add/del operation for board_info list and
* spi_controller list, and their matching process also used
* to protect object of type struct idr.
*/
static DEFINE_MUTEX(board_lock);
/**
* spi_alloc_device - Allocate a new SPI device
* @ctlr: Controller to which device is connected
* Context: can sleep
*
* Allows a driver to allocate and initialize a spi_device without
* registering it immediately. This allows a driver to directly
* fill the spi_device with device parameters before calling
* spi_add_device() on it.
*
* Caller is responsible to call spi_add_device() on the returned
* spi_device structure to add it to the SPI controller. If the caller
* needs to discard the spi_device without adding it, then it should
* call spi_dev_put() on it.
*
* Return: a pointer to the new device, or NULL.
*/
struct spi_device *spi_alloc_device(struct spi_controller *ctlr)
{
struct spi_device *spi;
if (!spi_controller_get(ctlr))
return NULL;
spi = kzalloc(sizeof(*spi), GFP_KERNEL);
if (!spi) {
spi_controller_put(ctlr);
return NULL;
}
spi->pcpu_statistics = spi_alloc_pcpu_stats(NULL);
if (!spi->pcpu_statistics) {
kfree(spi);
spi_controller_put(ctlr);
return NULL;
}
spi->controller = ctlr;
spi->dev.parent = &ctlr->dev;
spi->dev.bus = &spi_bus_type;
spi->dev.release = spidev_release;
spi->mode = ctlr->buswidth_override_bits;
device_initialize(&spi->dev);
return spi;
}
EXPORT_SYMBOL_GPL(spi_alloc_device);
static void spi_dev_set_name(struct spi_device *spi)
{
struct acpi_device *adev = ACPI_COMPANION(&spi->dev);
if (adev) {
dev_set_name(&spi->dev, "spi-%s", acpi_dev_name(adev));
return;
}
dev_set_name(&spi->dev, "%s.%u", dev_name(&spi->controller->dev),
spi_get_chipselect(spi, 0));
}
/*
* Zero(0) is a valid physical CS value and can be located at any
* logical CS in the spi->chip_select[]. If all the physical CS
* are initialized to 0 then It would be difficult to differentiate
* between a valid physical CS 0 & an unused logical CS whose physical
* CS can be 0. As a solution to this issue initialize all the CS to -1.
* Now all the unused logical CS will have -1 physical CS value & can be
* ignored while performing physical CS validity checks.
*/
#define SPI_INVALID_CS ((s8)-1)
static inline bool is_valid_cs(s8 chip_select)
{
return chip_select != SPI_INVALID_CS;
}
static inline int spi_dev_check_cs(struct device *dev,
struct spi_device *spi, u8 idx,
struct spi_device *new_spi, u8 new_idx)
{
u8 cs, cs_new;
u8 idx_new;
cs = spi_get_chipselect(spi, idx);
for (idx_new = new_idx; idx_new < SPI_CS_CNT_MAX; idx_new++) {
cs_new = spi_get_chipselect(new_spi, idx_new);
if (is_valid_cs(cs) && is_valid_cs(cs_new) && cs == cs_new) {
dev_err(dev, "chipselect %u already in use\n", cs_new);
return -EBUSY;
}
}
return 0;
}
static int spi_dev_check(struct device *dev, void *data)
{
struct spi_device *spi = to_spi_device(dev);
struct spi_device *new_spi = data;
int status, idx;
if (spi->controller == new_spi->controller) {
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
status = spi_dev_check_cs(dev, spi, idx, new_spi, 0);
if (status)
return status;
}
}
return 0;
}
static void spi_cleanup(struct spi_device *spi)
{
if (spi->controller->cleanup)
spi->controller->cleanup(spi);
}
static int __spi_add_device(struct spi_device *spi)
{
struct spi_controller *ctlr = spi->controller;
struct device *dev = ctlr->dev.parent;
int status, idx;
u8 cs;
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
/* Chipselects are numbered 0..max; validate. */
cs = spi_get_chipselect(spi, idx);
if (is_valid_cs(cs) && cs >= ctlr->num_chipselect) {
dev_err(dev, "cs%d >= max %d\n", spi_get_chipselect(spi, idx),
ctlr->num_chipselect);
return -EINVAL;
}
}
/*
* Make sure that multiple logical CS doesn't map to the same physical CS.
* For example, spi->chip_select[0] != spi->chip_select[1] and so on.
*/
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
status = spi_dev_check_cs(dev, spi, idx, spi, idx + 1);
if (status)
return status;
}
/* Set the bus ID string */
spi_dev_set_name(spi);
/*
* We need to make sure there's no other device with this
* chipselect **BEFORE** we call setup(), else we'll trash
* its configuration.
*/
status = bus_for_each_dev(&spi_bus_type, NULL, spi, spi_dev_check);
if (status)
return status;
/* Controller may unregister concurrently */
if (IS_ENABLED(CONFIG_SPI_DYNAMIC) &&
!device_is_registered(&ctlr->dev)) {
return -ENODEV;
}
if (ctlr->cs_gpiods) {
u8 cs;
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
cs = spi_get_chipselect(spi, idx);
if (is_valid_cs(cs))
spi_set_csgpiod(spi, idx, ctlr->cs_gpiods[cs]);
}
}
/*
* Drivers may modify this initial i/o setup, but will
* normally rely on the device being setup. Devices
* using SPI_CS_HIGH can't coexist well otherwise...
*/
status = spi_setup(spi);
if (status < 0) {
dev_err(dev, "can't setup %s, status %d\n",
dev_name(&spi->dev), status);
return status;
}
/* Device may be bound to an active driver when this returns */
status = device_add(&spi->dev);
if (status < 0) {
dev_err(dev, "can't add %s, status %d\n",
dev_name(&spi->dev), status);
spi_cleanup(spi);
} else {
dev_dbg(dev, "registered child %s\n", dev_name(&spi->dev));
}
return status;
}
/**
* spi_add_device - Add spi_device allocated with spi_alloc_device
* @spi: spi_device to register
*
* Companion function to spi_alloc_device. Devices allocated with
* spi_alloc_device can be added onto the SPI bus with this function.
*
* Return: 0 on success; negative errno on failure
*/
int spi_add_device(struct spi_device *spi)
{
struct spi_controller *ctlr = spi->controller;
int status;
/* Set the bus ID string */
spi_dev_set_name(spi);
mutex_lock(&ctlr->add_lock);
status = __spi_add_device(spi);
mutex_unlock(&ctlr->add_lock);
return status;
}
EXPORT_SYMBOL_GPL(spi_add_device);
static void spi_set_all_cs_unused(struct spi_device *spi)
{
u8 idx;
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++)
spi_set_chipselect(spi, idx, SPI_INVALID_CS);
}
/**
* spi_new_device - instantiate one new SPI device
* @ctlr: Controller to which device is connected
* @chip: Describes the SPI device
* Context: can sleep
*
* On typical mainboards, this is purely internal; and it's not needed
* after board init creates the hard-wired devices. Some development
* platforms may not be able to use spi_register_board_info though, and
* this is exported so that for example a USB or parport based adapter
* driver could add devices (which it would learn about out-of-band).
*
* Return: the new device, or NULL.
*/
struct spi_device *spi_new_device(struct spi_controller *ctlr,
struct spi_board_info *chip)
{
struct spi_device *proxy;
int status;
/*
* NOTE: caller did any chip->bus_num checks necessary.
*
* Also, unless we change the return value convention to use
* error-or-pointer (not NULL-or-pointer), troubleshootability
* suggests syslogged diagnostics are best here (ugh).
*/
proxy = spi_alloc_device(ctlr);
if (!proxy)
return NULL;
WARN_ON(strlen(chip->modalias) >= sizeof(proxy->modalias));
/* Use provided chip-select for proxy device */
spi_set_all_cs_unused(proxy);
spi_set_chipselect(proxy, 0, chip->chip_select);
proxy->max_speed_hz = chip->max_speed_hz;
proxy->mode = chip->mode;
proxy->irq = chip->irq;
strscpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias));
proxy->dev.platform_data = (void *) chip->platform_data;
proxy->controller_data = chip->controller_data;
proxy->controller_state = NULL;
/*
* spi->chip_select[i] gives the corresponding physical CS for logical CS i
* logical CS number is represented by setting the ith bit in spi->cs_index_mask
* So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and
* spi->chip_select[0] will give the physical CS.
* By default spi->chip_select[0] will hold the physical CS number so, set
* spi->cs_index_mask as 0x01.
*/
proxy->cs_index_mask = 0x01;
if (chip->swnode) {
status = device_add_software_node(&proxy->dev, chip->swnode);
if (status) {
dev_err(&ctlr->dev, "failed to add software node to '%s': %d\n",
chip->modalias, status);
goto err_dev_put;
}
}
status = spi_add_device(proxy);
if (status < 0)
goto err_dev_put;
return proxy;
err_dev_put:
device_remove_software_node(&proxy->dev);
spi_dev_put(proxy);
return NULL;
}
EXPORT_SYMBOL_GPL(spi_new_device);
/**
* spi_unregister_device - unregister a single SPI device
* @spi: spi_device to unregister
*
* Start making the passed SPI device vanish. Normally this would be handled
* by spi_unregister_controller().
*/
void spi_unregister_device(struct spi_device *spi)
{
if (!spi)
return;
if (spi->dev.of_node) { of_node_clear_flag(spi->dev.of_node, OF_POPULATED);
of_node_put(spi->dev.of_node);
}
if (ACPI_COMPANION(&spi->dev)) acpi_device_clear_enumerated(ACPI_COMPANION(&spi->dev)); device_remove_software_node(&spi->dev);
device_del(&spi->dev);
spi_cleanup(spi); put_device(&spi->dev);
}
EXPORT_SYMBOL_GPL(spi_unregister_device);
static void spi_match_controller_to_boardinfo(struct spi_controller *ctlr,
struct spi_board_info *bi)
{
struct spi_device *dev;
if (ctlr->bus_num != bi->bus_num)
return;
dev = spi_new_device(ctlr, bi);
if (!dev)
dev_err(ctlr->dev.parent, "can't create new device for %s\n",
bi->modalias);
}
/**
* spi_register_board_info - register SPI devices for a given board
* @info: array of chip descriptors
* @n: how many descriptors are provided
* Context: can sleep
*
* Board-specific early init code calls this (probably during arch_initcall)
* with segments of the SPI device table. Any device nodes are created later,
* after the relevant parent SPI controller (bus_num) is defined. We keep
* this table of devices forever, so that reloading a controller driver will
* not make Linux forget about these hard-wired devices.
*
* Other code can also call this, e.g. a particular add-on board might provide
* SPI devices through its expansion connector, so code initializing that board
* would naturally declare its SPI devices.
*
* The board info passed can safely be __initdata ... but be careful of
* any embedded pointers (platform_data, etc), they're copied as-is.
*
* Return: zero on success, else a negative error code.
*/
int spi_register_board_info(struct spi_board_info const *info, unsigned n)
{
struct boardinfo *bi;
int i;
if (!n)
return 0;
bi = kcalloc(n, sizeof(*bi), GFP_KERNEL);
if (!bi)
return -ENOMEM;
for (i = 0; i < n; i++, bi++, info++) {
struct spi_controller *ctlr;
memcpy(&bi->board_info, info, sizeof(*info));
mutex_lock(&board_lock);
list_add_tail(&bi->list, &board_list);
list_for_each_entry(ctlr, &spi_controller_list, list)
spi_match_controller_to_boardinfo(ctlr,
&bi->board_info);
mutex_unlock(&board_lock);
}
return 0;
}
/*-------------------------------------------------------------------------*/
/* Core methods for SPI resource management */
/**
* spi_res_alloc - allocate a spi resource that is life-cycle managed
* during the processing of a spi_message while using
* spi_transfer_one
* @spi: the SPI device for which we allocate memory
* @release: the release code to execute for this resource
* @size: size to alloc and return
* @gfp: GFP allocation flags
*
* Return: the pointer to the allocated data
*
* This may get enhanced in the future to allocate from a memory pool
* of the @spi_device or @spi_controller to avoid repeated allocations.
*/
static void *spi_res_alloc(struct spi_device *spi, spi_res_release_t release,
size_t size, gfp_t gfp)
{
struct spi_res *sres;
sres = kzalloc(sizeof(*sres) + size, gfp);
if (!sres)
return NULL;
INIT_LIST_HEAD(&sres->entry);
sres->release = release;
return sres->data;
}
/**
* spi_res_free - free an SPI resource
* @res: pointer to the custom data of a resource
*/
static void spi_res_free(void *res)
{
struct spi_res *sres = container_of(res, struct spi_res, data);
if (!res)
return;
WARN_ON(!list_empty(&sres->entry));
kfree(sres);
}
/**
* spi_res_add - add a spi_res to the spi_message
* @message: the SPI message
* @res: the spi_resource
*/
static void spi_res_add(struct spi_message *message, void *res)
{
struct spi_res *sres = container_of(res, struct spi_res, data);
WARN_ON(!list_empty(&sres->entry));
list_add_tail(&sres->entry, &message->resources);
}
/**
* spi_res_release - release all SPI resources for this message
* @ctlr: the @spi_controller
* @message: the @spi_message
*/
static void spi_res_release(struct spi_controller *ctlr, struct spi_message *message)
{
struct spi_res *res, *tmp;
list_for_each_entry_safe_reverse(res, tmp, &message->resources, entry) {
if (res->release)
res->release(ctlr, message, res->data);
list_del(&res->entry);
kfree(res);
}
}
/*-------------------------------------------------------------------------*/
static inline bool spi_is_last_cs(struct spi_device *spi)
{
u8 idx;
bool last = false;
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
if (spi->cs_index_mask & BIT(idx)) {
if (spi->controller->last_cs[idx] == spi_get_chipselect(spi, idx))
last = true;
}
}
return last;
}
static void spi_set_cs(struct spi_device *spi, bool enable, bool force)
{
bool activate = enable;
u8 idx;
/*
* Avoid calling into the driver (or doing delays) if the chip select
* isn't actually changing from the last time this was called.
*/
if (!force && ((enable && spi->controller->last_cs_index_mask == spi->cs_index_mask &&
spi_is_last_cs(spi)) ||
(!enable && spi->controller->last_cs_index_mask == spi->cs_index_mask &&
!spi_is_last_cs(spi))) &&
(spi->controller->last_cs_mode_high == (spi->mode & SPI_CS_HIGH)))
return;
trace_spi_set_cs(spi, activate);
spi->controller->last_cs_index_mask = spi->cs_index_mask;
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++)
spi->controller->last_cs[idx] = enable ? spi_get_chipselect(spi, 0) : SPI_INVALID_CS;
spi->controller->last_cs_mode_high = spi->mode & SPI_CS_HIGH;
if (spi->mode & SPI_CS_HIGH)
enable = !enable;
/*
* Handle chip select delays for GPIO based CS or controllers without
* programmable chip select timing.
*/
if ((spi_is_csgpiod(spi) || !spi->controller->set_cs_timing) && !activate)
spi_delay_exec(&spi->cs_hold, NULL);
if (spi_is_csgpiod(spi)) {
if (!(spi->mode & SPI_NO_CS)) {
/*
* Historically ACPI has no means of the GPIO polarity and
* thus the SPISerialBus() resource defines it on the per-chip
* basis. In order to avoid a chain of negations, the GPIO
* polarity is considered being Active High. Even for the cases
* when _DSD() is involved (in the updated versions of ACPI)
* the GPIO CS polarity must be defined Active High to avoid
* ambiguity. That's why we use enable, that takes SPI_CS_HIGH
* into account.
*/
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) {
if ((spi->cs_index_mask & BIT(idx)) && spi_get_csgpiod(spi, idx)) {
if (has_acpi_companion(&spi->dev))
gpiod_set_value_cansleep(spi_get_csgpiod(spi, idx),
!enable);
else
/* Polarity handled by GPIO library */
gpiod_set_value_cansleep(spi_get_csgpiod(spi, idx),
activate);
if (activate)
spi_delay_exec(&spi->cs_setup, NULL);
else
spi_delay_exec(&spi->cs_inactive, NULL);
}
}
}
/* Some SPI masters need both GPIO CS & slave_select */
if ((spi->controller->flags & SPI_CONTROLLER_GPIO_SS) &&
spi->controller->set_cs)
spi->controller->set_cs(spi, !enable);
} else if (spi->controller->set_cs) {
spi->controller->set_cs(spi, !enable);
}
if (spi_is_csgpiod(spi) || !spi->controller->set_cs_timing) {
if (activate)
spi_delay_exec(&spi->cs_setup, NULL);
else
spi_delay_exec(&spi->cs_inactive, NULL);
}
}
#ifdef CONFIG_HAS_DMA
static int spi_map_buf_attrs(struct spi_controller *ctlr, struct device *dev,
struct sg_table *sgt, void *buf, size_t len,
enum dma_data_direction dir, unsigned long attrs)
{
const bool vmalloced_buf = is_vmalloc_addr(buf);
unsigned int max_seg_size = dma_get_max_seg_size(dev);
#ifdef CONFIG_HIGHMEM
const bool kmap_buf = ((unsigned long)buf >= PKMAP_BASE &&
(unsigned long)buf < (PKMAP_BASE +
(LAST_PKMAP * PAGE_SIZE)));
#else
const bool kmap_buf = false;
#endif
int desc_len;
int sgs;
struct page *vm_page;
struct scatterlist *sg;
void *sg_buf;
size_t min;
int i, ret;
if (vmalloced_buf || kmap_buf) {
desc_len = min_t(unsigned long, max_seg_size, PAGE_SIZE);
sgs = DIV_ROUND_UP(len + offset_in_page(buf), desc_len);
} else if (virt_addr_valid(buf)) {
desc_len = min_t(size_t, max_seg_size, ctlr->max_dma_len);
sgs = DIV_ROUND_UP(len, desc_len);
} else {
return -EINVAL;
}
ret = sg_alloc_table(sgt, sgs, GFP_KERNEL);
if (ret != 0)
return ret;
sg = &sgt->sgl[0];
for (i = 0; i < sgs; i++) {
if (vmalloced_buf || kmap_buf) {
/*
* Next scatterlist entry size is the minimum between
* the desc_len and the remaining buffer length that
* fits in a page.
*/
min = min_t(size_t, desc_len,
min_t(size_t, len,
PAGE_SIZE - offset_in_page(buf)));
if (vmalloced_buf)
vm_page = vmalloc_to_page(buf);
else
vm_page = kmap_to_page(buf);
if (!vm_page) {
sg_free_table(sgt);
return -ENOMEM;
}
sg_set_page(sg, vm_page,
min, offset_in_page(buf));
} else {
min = min_t(size_t, len, desc_len);
sg_buf = buf;
sg_set_buf(sg, sg_buf, min);
}
buf += min;
len -= min;
sg = sg_next(sg);
}
ret = dma_map_sgtable(dev, sgt, dir, attrs);
if (ret < 0) {
sg_free_table(sgt);
return ret;
}
return 0;
}
int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
struct sg_table *sgt, void *buf, size_t len,
enum dma_data_direction dir)
{
return spi_map_buf_attrs(ctlr, dev, sgt, buf, len, dir, 0);
}
static void spi_unmap_buf_attrs(struct spi_controller *ctlr,
struct device *dev, struct sg_table *sgt,
enum dma_data_direction dir,
unsigned long attrs)
{
if (sgt->orig_nents) {
dma_unmap_sgtable(dev, sgt, dir, attrs);
sg_free_table(sgt);
sgt->orig_nents = 0;
sgt->nents = 0;
}
}
void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
struct sg_table *sgt, enum dma_data_direction dir)
{
spi_unmap_buf_attrs(ctlr, dev, sgt, dir, 0);
}
static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
{
struct device *tx_dev, *rx_dev;
struct spi_transfer *xfer;
int ret;
if (!ctlr->can_dma)
return 0;
if (ctlr->dma_tx)
tx_dev = ctlr->dma_tx->device->dev;
else if (ctlr->dma_map_dev)
tx_dev = ctlr->dma_map_dev;
else
tx_dev = ctlr->dev.parent;
if (ctlr->dma_rx)
rx_dev = ctlr->dma_rx->device->dev;
else if (ctlr->dma_map_dev)
rx_dev = ctlr->dma_map_dev;
else
rx_dev = ctlr->dev.parent;
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
/* The sync is done before each transfer. */
unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
if (!ctlr->can_dma(ctlr, msg->spi, xfer))
continue;
if (xfer->tx_buf != NULL) {
ret = spi_map_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,
(void *)xfer->tx_buf,
xfer->len, DMA_TO_DEVICE,
attrs);
if (ret != 0)
return ret;
}
if (xfer->rx_buf != NULL) {
ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,
xfer->rx_buf, xfer->len,
DMA_FROM_DEVICE, attrs);
if (ret != 0) {
spi_unmap_buf_attrs(ctlr, tx_dev,
&xfer->tx_sg, DMA_TO_DEVICE,
attrs);
return ret;
}
}
}
ctlr->cur_rx_dma_dev = rx_dev;
ctlr->cur_tx_dma_dev = tx_dev;
ctlr->cur_msg_mapped = true;
return 0;
}
static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg)
{
struct device *rx_dev = ctlr->cur_rx_dma_dev;
struct device *tx_dev = ctlr->cur_tx_dma_dev;
struct spi_transfer *xfer;
if (!ctlr->cur_msg_mapped || !ctlr->can_dma)
return 0;
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
/* The sync has already been done after each transfer. */
unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
if (!ctlr->can_dma(ctlr, msg->spi, xfer))
continue;
spi_unmap_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,
DMA_FROM_DEVICE, attrs);
spi_unmap_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,
DMA_TO_DEVICE, attrs);
}
ctlr->cur_msg_mapped = false;
return 0;
}
static void spi_dma_sync_for_device(struct spi_controller *ctlr,
struct spi_transfer *xfer)
{
struct device *rx_dev = ctlr->cur_rx_dma_dev;
struct device *tx_dev = ctlr->cur_tx_dma_dev;
if (!ctlr->cur_msg_mapped)
return;
if (xfer->tx_sg.orig_nents)
dma_sync_sgtable_for_device(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
if (xfer->rx_sg.orig_nents)
dma_sync_sgtable_for_device(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
}
static void spi_dma_sync_for_cpu(struct spi_controller *ctlr,
struct spi_transfer *xfer)
{
struct device *rx_dev = ctlr->cur_rx_dma_dev;
struct device *tx_dev = ctlr->cur_tx_dma_dev;
if (!ctlr->cur_msg_mapped)
return;
if (xfer->rx_sg.orig_nents)
dma_sync_sgtable_for_cpu(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
if (xfer->tx_sg.orig_nents)
dma_sync_sgtable_for_cpu(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
}
#else /* !CONFIG_HAS_DMA */
static inline int __spi_map_msg(struct spi_controller *ctlr,
struct spi_message *msg)
{
return 0;
}
static inline int __spi_unmap_msg(struct spi_controller *ctlr,
struct spi_message *msg)
{
return 0;
}
static void spi_dma_sync_for_device(struct spi_controller *ctrl,
struct spi_transfer *xfer)
{
}
static void spi_dma_sync_for_cpu(struct spi_controller *ctrl,
struct spi_transfer *xfer)
{
}
#endif /* !CONFIG_HAS_DMA */
static inline int spi_unmap_msg(struct spi_controller *ctlr,
struct spi_message *msg)
{
struct spi_transfer *xfer;
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
/*
* Restore the original value of tx_buf or rx_buf if they are
* NULL.
*/
if (xfer->tx_buf == ctlr->dummy_tx)
xfer->tx_buf = NULL;
if (xfer->rx_buf == ctlr->dummy_rx)
xfer->rx_buf = NULL;
}
return __spi_unmap_msg(ctlr, msg);
}
static int spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
{
struct spi_transfer *xfer;
void *tmp;
unsigned int max_tx, max_rx;
if ((ctlr->flags & (SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX))
&& !(msg->spi->mode & SPI_3WIRE)) {
max_tx = 0;
max_rx = 0;
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
if ((ctlr->flags & SPI_CONTROLLER_MUST_TX) &&
!xfer->tx_buf)
max_tx = max(xfer->len, max_tx);
if ((ctlr->flags & SPI_CONTROLLER_MUST_RX) &&
!xfer->rx_buf)
max_rx = max(xfer->len, max_rx);
}
if (max_tx) {
tmp = krealloc(ctlr->dummy_tx, max_tx,
GFP_KERNEL | GFP_DMA | __GFP_ZERO);
if (!tmp)
return -ENOMEM;
ctlr->dummy_tx = tmp;
}
if (max_rx) {
tmp = krealloc(ctlr->dummy_rx, max_rx,
GFP_KERNEL | GFP_DMA);
if (!tmp)
return -ENOMEM;
ctlr->dummy_rx = tmp;
}
if (max_tx || max_rx) {
list_for_each_entry(xfer, &msg->transfers,
transfer_list) {
if (!xfer->len)
continue;
if (!xfer->tx_buf)
xfer->tx_buf = ctlr->dummy_tx;
if (!xfer->rx_buf)
xfer->rx_buf = ctlr->dummy_rx;
}
}
}
return __spi_map_msg(ctlr, msg);
}
static int spi_transfer_wait(struct spi_controller *ctlr,
struct spi_message *msg,
struct spi_transfer *xfer)
{
struct spi_statistics __percpu *statm = ctlr->pcpu_statistics;
struct spi_statistics __percpu *stats = msg->spi->pcpu_statistics;
u32 speed_hz = xfer->speed_hz;
unsigned long long ms;
if (spi_controller_is_slave(ctlr)) {
if (wait_for_completion_interruptible(&ctlr->xfer_completion)) {
dev_dbg(&msg->spi->dev, "SPI transfer interrupted\n");
return -EINTR;
}
} else {
if (!speed_hz)
speed_hz = 100000;
/*
* For each byte we wait for 8 cycles of the SPI clock.
* Since speed is defined in Hz and we want milliseconds,
* use respective multiplier, but before the division,
* otherwise we may get 0 for short transfers.
*/
ms = 8LL * MSEC_PER_SEC * xfer->len;
do_div(ms, speed_hz);
/*
* Increase it twice and add 200 ms tolerance, use
* predefined maximum in case of overflow.
*/
ms += ms + 200;
if (ms > UINT_MAX)
ms = UINT_MAX;
ms = wait_for_completion_timeout(&ctlr->xfer_completion,
msecs_to_jiffies(ms));
if (ms == 0) {
SPI_STATISTICS_INCREMENT_FIELD(statm, timedout);
SPI_STATISTICS_INCREMENT_FIELD(stats, timedout);
dev_err(&msg->spi->dev,
"SPI transfer timed out\n");
return -ETIMEDOUT;
}
if (xfer->error & SPI_TRANS_FAIL_IO)
return -EIO;
}
return 0;
}
static void _spi_transfer_delay_ns(u32 ns)
{
if (!ns)
return;
if (ns <= NSEC_PER_USEC) {
ndelay(ns);
} else {
u32 us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
if (us <= 10)
udelay(us);
else
usleep_range(us, us + DIV_ROUND_UP(us, 10));
}
}
int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer)
{
u32 delay = _delay->value;
u32 unit = _delay->unit;
u32 hz;
if (!delay)
return 0;
switch (unit) {
case SPI_DELAY_UNIT_USECS:
delay *= NSEC_PER_USEC;
break;
case SPI_DELAY_UNIT_NSECS:
/* Nothing to do here */
break;
case SPI_DELAY_UNIT_SCK:
/* Clock cycles need to be obtained from spi_transfer */
if (!xfer)
return -EINVAL;
/*
* If there is unknown effective speed, approximate it
* by underestimating with half of the requested Hz.
*/
hz = xfer->effective_speed_hz ?: xfer->speed_hz / 2;
if (!hz)
return -EINVAL;
/* Convert delay to nanoseconds */
delay *= DIV_ROUND_UP(NSEC_PER_SEC, hz);
break;
default:
return -EINVAL;
}
return delay;
}
EXPORT_SYMBOL_GPL(spi_delay_to_ns);
int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer)
{
int delay;
might_sleep();
if (!_delay)
return -EINVAL;
delay = spi_delay_to_ns(_delay, xfer);
if (delay < 0)
return delay;
_spi_transfer_delay_ns(delay);
return 0;
}
EXPORT_SYMBOL_GPL(spi_delay_exec);
static void _spi_transfer_cs_change_delay(struct spi_message *msg,
struct spi_transfer *xfer)
{
u32 default_delay_ns = 10 * NSEC_PER_USEC;
u32 delay = xfer->cs_change_delay.value;
u32 unit = xfer->cs_change_delay.unit;
int ret;
/* Return early on "fast" mode - for everything but USECS */
if (!delay) {
if (unit == SPI_DELAY_UNIT_USECS)
_spi_transfer_delay_ns(default_delay_ns);
return;
}
ret = spi_delay_exec(&xfer->cs_change_delay, xfer);
if (ret) {
dev_err_once(&msg->spi->dev,
"Use of unsupported delay unit %i, using default of %luus\n",
unit, default_delay_ns / NSEC_PER_USEC);
_spi_transfer_delay_ns(default_delay_ns);
}
}
void spi_transfer_cs_change_delay_exec(struct spi_message *msg,
struct spi_transfer *xfer)
{
_spi_transfer_cs_change_delay(msg, xfer);
}
EXPORT_SYMBOL_GPL(spi_transfer_cs_change_delay_exec);
/*
* spi_transfer_one_message - Default implementation of transfer_one_message()
*
* This is a standard implementation of transfer_one_message() for
* drivers which implement a transfer_one() operation. It provides
* standard handling of delays and chip select management.
*/
static int spi_transfer_one_message(struct spi_controller *ctlr,
struct spi_message *msg)
{
struct spi_transfer *xfer;
bool keep_cs = false;
int ret = 0;
struct spi_statistics __percpu *statm = ctlr->pcpu_statistics;
struct spi_statistics __percpu *stats = msg->spi->pcpu_statistics;
xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list);
spi_set_cs(msg->spi, !xfer->cs_off, false);
SPI_STATISTICS_INCREMENT_FIELD(statm, messages);
SPI_STATISTICS_INCREMENT_FIELD(stats, messages);
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
trace_spi_transfer_start(msg, xfer);
spi_statistics_add_transfer_stats(statm, xfer, ctlr);
spi_statistics_add_transfer_stats(stats, xfer, ctlr);
if (!ctlr->ptp_sts_supported) {
xfer->ptp_sts_word_pre = 0;
ptp_read_system_prets(xfer->ptp_sts);
}
if ((xfer->tx_buf || xfer->rx_buf) && xfer->len) {
reinit_completion(&ctlr->xfer_completion);
fallback_pio:
spi_dma_sync_for_device(ctlr, xfer);
ret = ctlr->transfer_one(ctlr, msg->spi, xfer);
if (ret < 0) {
spi_dma_sync_for_cpu(ctlr, xfer);
if (ctlr->cur_msg_mapped &&
(xfer->error & SPI_TRANS_FAIL_NO_START)) {
__spi_unmap_msg(ctlr, msg);
ctlr->fallback = true;
xfer->error &= ~SPI_TRANS_FAIL_NO_START;
goto fallback_pio;
}
SPI_STATISTICS_INCREMENT_FIELD(statm,
errors);
SPI_STATISTICS_INCREMENT_FIELD(stats,
errors);
dev_err(&msg->spi->dev,
"SPI transfer failed: %d\n", ret);
goto out;
}
if (ret > 0) {
ret = spi_transfer_wait(ctlr, msg, xfer);
if (ret < 0)
msg->status = ret;
}
spi_dma_sync_for_cpu(ctlr, xfer);
} else {
if (xfer->len)
dev_err(&msg->spi->dev,
"Bufferless transfer has length %u\n",
xfer->len);
}
if (!ctlr->ptp_sts_supported) {
ptp_read_system_postts(xfer->ptp_sts);
xfer->ptp_sts_word_post = xfer->len;
}
trace_spi_transfer_stop(msg, xfer);
if (msg->status != -EINPROGRESS)
goto out;
spi_transfer_delay_exec(xfer);
if (xfer->cs_change) {
if (list_is_last(&xfer->transfer_list,
&msg->transfers)) {
keep_cs = true;
} else {
if (!xfer->cs_off)
spi_set_cs(msg->spi, false, false);
_spi_transfer_cs_change_delay(msg, xfer);
if (!list_next_entry(xfer, transfer_list)->cs_off)
spi_set_cs(msg->spi, true, false);
}
} else if (!list_is_last(&xfer->transfer_list, &msg->transfers) &&
xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) {
spi_set_cs(msg->spi, xfer->cs_off, false);
}
msg->actual_length += xfer->len;
}
out:
if (ret != 0 || !keep_cs)
spi_set_cs(msg->spi, false, false);
if (msg->status == -EINPROGRESS)
msg->status = ret;
if (msg->status && ctlr->handle_err)
ctlr->handle_err(ctlr, msg);
spi_finalize_current_message(ctlr);
return ret;
}
/**
* spi_finalize_current_transfer - report completion of a transfer
* @ctlr: the controller reporting completion
*
* Called by SPI drivers using the core transfer_one_message()
* implementation to notify it that the current interrupt driven
* transfer has finished and the next one may be scheduled.
*/
void spi_finalize_current_transfer(struct spi_controller *ctlr)
{
complete(&ctlr->xfer_completion);
}
EXPORT_SYMBOL_GPL(spi_finalize_current_transfer);
static void spi_idle_runtime_pm(struct spi_controller *ctlr)
{
if (ctlr->auto_runtime_pm) {
pm_runtime_mark_last_busy(ctlr->dev.parent);
pm_runtime_put_autosuspend(ctlr->dev.parent);
}
}
static int __spi_pump_transfer_message(struct spi_controller *ctlr,
struct spi_message *msg, bool was_busy)
{
struct spi_transfer *xfer;
int ret;
if (!was_busy && ctlr->auto_runtime_pm) {
ret = pm_runtime_get_sync(ctlr->dev.parent);
if (ret < 0) {
pm_runtime_put_noidle(ctlr->dev.parent);
dev_err(&ctlr->dev, "Failed to power device: %d\n",
ret);
msg->status = ret;
spi_finalize_current_message(ctlr);
return ret;
}
}
if (!was_busy)
trace_spi_controller_busy(ctlr);
if (!was_busy && ctlr->prepare_transfer_hardware) {
ret = ctlr->prepare_transfer_hardware(ctlr);
if (ret) {
dev_err(&ctlr->dev,
"failed to prepare transfer hardware: %d\n",
ret);
if (ctlr->auto_runtime_pm)
pm_runtime_put(ctlr->dev.parent);
msg->status = ret;
spi_finalize_current_message(ctlr);
return ret;
}
}
trace_spi_message_start(msg);
if (ctlr->prepare_message) {
ret = ctlr->prepare_message(ctlr, msg);
if (ret) {
dev_err(&ctlr->dev, "failed to prepare message: %d\n",
ret);
msg->status = ret;
spi_finalize_current_message(ctlr);
return ret;
}
msg->prepared = true;
}
ret = spi_map_msg(ctlr, msg);
if (ret) {
msg->status = ret;
spi_finalize_current_message(ctlr);
return ret;
}
if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) {
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
xfer->ptp_sts_word_pre = 0;
ptp_read_system_prets(xfer->ptp_sts);
}
}
/*
* Drivers implementation of transfer_one_message() must arrange for
* spi_finalize_current_message() to get called. Most drivers will do
* this in the calling context, but some don't. For those cases, a
* completion is used to guarantee that this function does not return
* until spi_finalize_current_message() is done accessing
* ctlr->cur_msg.
* Use of the following two flags enable to opportunistically skip the
* use of the completion since its use involves expensive spin locks.
* In case of a race with the context that calls
* spi_finalize_current_message() the completion will always be used,
* due to strict ordering of these flags using barriers.
*/
WRITE_ONCE(ctlr->cur_msg_incomplete, true);
WRITE_ONCE(ctlr->cur_msg_need_completion, false);
reinit_completion(&ctlr->cur_msg_completion);
smp_wmb(); /* Make these available to spi_finalize_current_message() */
ret = ctlr->transfer_one_message(ctlr, msg);
if (ret) {
dev_err(&ctlr->dev,
"failed to transfer one message from queue\n");
return ret;
}
WRITE_ONCE(ctlr->cur_msg_need_completion, true);
smp_mb(); /* See spi_finalize_current_message()... */
if (READ_ONCE(ctlr->cur_msg_incomplete))
wait_for_completion(&ctlr->cur_msg_completion);
return 0;
}
/**
* __spi_pump_messages - function which processes SPI message queue
* @ctlr: controller to process queue for
* @in_kthread: true if we are in the context of the message pump thread
*
* This function checks if there is any SPI message in the queue that
* needs processing and if so call out to the driver to initialize hardware
* and transfer each message.
*
* Note that it is called both from the kthread itself and also from
* inside spi_sync(); the queue extraction handling at the top of the
* function should deal with this safely.
*/
static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread)
{
struct spi_message *msg;
bool was_busy = false;
unsigned long flags;
int ret;
/* Take the I/O mutex */
mutex_lock(&ctlr->io_mutex);
/* Lock queue */
spin_lock_irqsave(&ctlr->queue_lock, flags);
/* Make sure we are not already running a message */
if (ctlr->cur_msg)
goto out_unlock;
/* Check if the queue is idle */
if (list_empty(&ctlr->queue) || !ctlr->running) {
if (!ctlr->busy)
goto out_unlock;
/* Defer any non-atomic teardown to the thread */
if (!in_kthread) {
if (!ctlr->dummy_rx && !ctlr->dummy_tx &&
!ctlr->unprepare_transfer_hardware) {
spi_idle_runtime_pm(ctlr);
ctlr->busy = false;
ctlr->queue_empty = true;
trace_spi_controller_idle(ctlr);
} else {
kthread_queue_work(ctlr->kworker,
&ctlr->pump_messages);
}
goto out_unlock;
}
ctlr->busy = false;
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
kfree(ctlr->dummy_rx);
ctlr->dummy_rx = NULL;
kfree(ctlr->dummy_tx);
ctlr->dummy_tx = NULL;
if (ctlr->unprepare_transfer_hardware &&
ctlr->unprepare_transfer_hardware(ctlr))
dev_err(&ctlr->dev,
"failed to unprepare transfer hardware\n");
spi_idle_runtime_pm(ctlr);
trace_spi_controller_idle(ctlr);
spin_lock_irqsave(&ctlr->queue_lock, flags);
ctlr->queue_empty = true;
goto out_unlock;
}
/* Extract head of queue */
msg = list_first_entry(&ctlr->queue, struct spi_message, queue);
ctlr->cur_msg = msg;
list_del_init(&msg->queue);
if (ctlr->busy)
was_busy = true;
else
ctlr->busy = true;
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ret = __spi_pump_transfer_message(ctlr, msg, was_busy);
kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);
ctlr->cur_msg = NULL;
ctlr->fallback = false;
mutex_unlock(&ctlr->io_mutex);
/* Prod the scheduler in case transfer_one() was busy waiting */
if (!ret)
cond_resched();
return;
out_unlock:
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
mutex_unlock(&ctlr->io_mutex);
}
/**
* spi_pump_messages - kthread work function which processes spi message queue
* @work: pointer to kthread work struct contained in the controller struct
*/
static void spi_pump_messages(struct kthread_work *work)
{
struct spi_controller *ctlr =
container_of(work, struct spi_controller, pump_messages);
__spi_pump_messages(ctlr, true);
}
/**
* spi_take_timestamp_pre - helper to collect the beginning of the TX timestamp
* @ctlr: Pointer to the spi_controller structure of the driver
* @xfer: Pointer to the transfer being timestamped
* @progress: How many words (not bytes) have been transferred so far
* @irqs_off: If true, will disable IRQs and preemption for the duration of the
* transfer, for less jitter in time measurement. Only compatible
* with PIO drivers. If true, must follow up with
* spi_take_timestamp_post or otherwise system will crash.
* WARNING: for fully predictable results, the CPU frequency must
* also be under control (governor).
*
* This is a helper for drivers to collect the beginning of the TX timestamp
* for the requested byte from the SPI transfer. The frequency with which this
* function must be called (once per word, once for the whole transfer, once
* per batch of words etc) is arbitrary as long as the @tx buffer offset is
* greater than or equal to the requested byte at the time of the call. The
* timestamp is only taken once, at the first such call. It is assumed that
* the driver advances its @tx buffer pointer monotonically.
*/
void spi_take_timestamp_pre(struct spi_controller *ctlr,
struct spi_transfer *xfer,
size_t progress, bool irqs_off)
{
if (!xfer->ptp_sts)
return;
if (xfer->timestamped)
return;
if (progress > xfer->ptp_sts_word_pre)
return;
/* Capture the resolution of the timestamp */
xfer->ptp_sts_word_pre = progress;
if (irqs_off) {
local_irq_save(ctlr->irq_flags);
preempt_disable();
}
ptp_read_system_prets(xfer->ptp_sts);
}
EXPORT_SYMBOL_GPL(spi_take_timestamp_pre);
/**
* spi_take_timestamp_post - helper to collect the end of the TX timestamp
* @ctlr: Pointer to the spi_controller structure of the driver
* @xfer: Pointer to the transfer being timestamped
* @progress: How many words (not bytes) have been transferred so far
* @irqs_off: If true, will re-enable IRQs and preemption for the local CPU.
*
* This is a helper for drivers to collect the end of the TX timestamp for
* the requested byte from the SPI transfer. Can be called with an arbitrary
* frequency: only the first call where @tx exceeds or is equal to the
* requested word will be timestamped.
*/
void spi_take_timestamp_post(struct spi_controller *ctlr,
struct spi_transfer *xfer,
size_t progress, bool irqs_off)
{
if (!xfer->ptp_sts)
return;
if (xfer->timestamped)
return;
if (progress < xfer->ptp_sts_word_post)
return;
ptp_read_system_postts(xfer->ptp_sts);
if (irqs_off) {
local_irq_restore(ctlr->irq_flags);
preempt_enable();
}
/* Capture the resolution of the timestamp */
xfer->ptp_sts_word_post = progress;
xfer->timestamped = 1;
}
EXPORT_SYMBOL_GPL(spi_take_timestamp_post);
/**
* spi_set_thread_rt - set the controller to pump at realtime priority
* @ctlr: controller to boost priority of
*
* This can be called because the controller requested realtime priority
* (by setting the ->rt value before calling spi_register_controller()) or
* because a device on the bus said that its transfers needed realtime
* priority.
*
* NOTE: at the moment if any device on a bus says it needs realtime then
* the thread will be at realtime priority for all transfers on that
* controller. If this eventually becomes a problem we may see if we can
* find a way to boost the priority only temporarily during relevant
* transfers.
*/
static void spi_set_thread_rt(struct spi_controller *ctlr)
{
dev_info(&ctlr->dev,
"will run message pump with realtime priority\n");
sched_set_fifo(ctlr->kworker->task);
}
static int spi_init_queue(struct spi_controller *ctlr)
{
ctlr->running = false;
ctlr->busy = false;
ctlr->queue_empty = true;
ctlr->kworker = kthread_create_worker(0, dev_name(&ctlr->dev));
if (IS_ERR(ctlr->kworker)) {
dev_err(&ctlr->dev, "failed to create message pump kworker\n");
return PTR_ERR(ctlr->kworker);
}
kthread_init_work(&ctlr->pump_messages, spi_pump_messages);
/*
* Controller config will indicate if this controller should run the
* message pump with high (realtime) priority to reduce the transfer
* latency on the bus by minimising the delay between a transfer
* request and the scheduling of the message pump thread. Without this
* setting the message pump thread will remain at default priority.
*/
if (ctlr->rt)
spi_set_thread_rt(ctlr);
return 0;
}
/**
* spi_get_next_queued_message() - called by driver to check for queued
* messages
* @ctlr: the controller to check for queued messages
*
* If there are more messages in the queue, the next message is returned from
* this call.
*
* Return: the next message in the queue, else NULL if the queue is empty.
*/
struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr)
{
struct spi_message *next;
unsigned long flags;
/* Get a pointer to the next message, if any */
spin_lock_irqsave(&ctlr->queue_lock, flags);
next = list_first_entry_or_null(&ctlr->queue, struct spi_message,
queue);
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
return next;
}
EXPORT_SYMBOL_GPL(spi_get_next_queued_message);
/*
* __spi_unoptimize_message - shared implementation of spi_unoptimize_message()
* and spi_maybe_unoptimize_message()
* @msg: the message to unoptimize
*
* Peripheral drivers should use spi_unoptimize_message() and callers inside
* core should use spi_maybe_unoptimize_message() rather than calling this
* function directly.
*
* It is not valid to call this on a message that is not currently optimized.
*/
static void __spi_unoptimize_message(struct spi_message *msg)
{
struct spi_controller *ctlr = msg->spi->controller;
if (ctlr->unoptimize_message)
ctlr->unoptimize_message(msg);
spi_res_release(ctlr, msg);
msg->optimized = false;
msg->opt_state = NULL;
}
/*
* spi_maybe_unoptimize_message - unoptimize msg not managed by a peripheral
* @msg: the message to unoptimize
*
* This function is used to unoptimize a message if and only if it was
* optimized by the core (via spi_maybe_optimize_message()).
*/
static void spi_maybe_unoptimize_message(struct spi_message *msg)
{
if (!msg->pre_optimized && msg->optimized)
__spi_unoptimize_message(msg);
}
/**
* spi_finalize_current_message() - the current message is complete
* @ctlr: the controller to return the message to
*
* Called by the driver to notify the core that the message in the front of the
* queue is complete and can be removed from the queue.
*/
void spi_finalize_current_message(struct spi_controller *ctlr)
{
struct spi_transfer *xfer;
struct spi_message *mesg;
int ret;
mesg = ctlr->cur_msg;
if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) {
list_for_each_entry(xfer, &mesg->transfers, transfer_list) {
ptp_read_system_postts(xfer->ptp_sts);
xfer->ptp_sts_word_post = xfer->len;
}
}
if (unlikely(ctlr->ptp_sts_supported))
list_for_each_entry(xfer, &mesg->transfers, transfer_list)
WARN_ON_ONCE(xfer->ptp_sts && !xfer->timestamped);
spi_unmap_msg(ctlr, mesg);
if (mesg->prepared && ctlr->unprepare_message) {
ret = ctlr->unprepare_message(ctlr, mesg);
if (ret) {
dev_err(&ctlr->dev, "failed to unprepare message: %d\n",
ret);
}
}
mesg->prepared = false;
spi_maybe_unoptimize_message(mesg);
WRITE_ONCE(ctlr->cur_msg_incomplete, false);
smp_mb(); /* See __spi_pump_transfer_message()... */
if (READ_ONCE(ctlr->cur_msg_need_completion))
complete(&ctlr->cur_msg_completion);
trace_spi_message_done(mesg);
mesg->state = NULL;
if (mesg->complete)
mesg->complete(mesg->context);
}
EXPORT_SYMBOL_GPL(spi_finalize_current_message);
static int spi_start_queue(struct spi_controller *ctlr)
{
unsigned long flags;
spin_lock_irqsave(&ctlr->queue_lock, flags);
if (ctlr->running || ctlr->busy) {
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
return -EBUSY;
}
ctlr->running = true;
ctlr->cur_msg = NULL;
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);
return 0;
}
static int spi_stop_queue(struct spi_controller *ctlr)
{
unsigned long flags;
unsigned limit = 500;
int ret = 0;
spin_lock_irqsave(&ctlr->queue_lock, flags);
/*
* This is a bit lame, but is optimized for the common execution path.
* A wait_queue on the ctlr->busy could be used, but then the common
* execution path (pump_messages) would be required to call wake_up or
* friends on every SPI message. Do this instead.
*/
while ((!list_empty(&ctlr->queue) || ctlr->busy) && limit--) { spin_unlock_irqrestore(&ctlr->queue_lock, flags);
usleep_range(10000, 11000);
spin_lock_irqsave(&ctlr->queue_lock, flags);
}
if (!list_empty(&ctlr->queue) || ctlr->busy)
ret = -EBUSY;
else
ctlr->running = false; spin_unlock_irqrestore(&ctlr->queue_lock, flags);
return ret;
}
static int spi_destroy_queue(struct spi_controller *ctlr)
{
int ret;
ret = spi_stop_queue(ctlr);
/*
* kthread_flush_worker will block until all work is done.
* If the reason that stop_queue timed out is that the work will never
* finish, then it does no good to call flush/stop thread, so
* return anyway.
*/
if (ret) {
dev_err(&ctlr->dev, "problem destroying queue\n");
return ret;
}
kthread_destroy_worker(ctlr->kworker);
return 0;
}
static int __spi_queued_transfer(struct spi_device *spi,
struct spi_message *msg,
bool need_pump)
{
struct spi_controller *ctlr = spi->controller;
unsigned long flags;
spin_lock_irqsave(&ctlr->queue_lock, flags);
if (!ctlr->running) {
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
return -ESHUTDOWN;
}
msg->actual_length = 0;
msg->status = -EINPROGRESS;
list_add_tail(&msg->queue, &ctlr->queue);
ctlr->queue_empty = false;
if (!ctlr->busy && need_pump)
kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);
spin_unlock_irqrestore(&ctlr->queue_lock, flags);
return 0;
}
/**
* spi_queued_transfer - transfer function for queued transfers
* @spi: SPI device which is requesting transfer
* @msg: SPI message which is to handled is queued to driver queue
*
* Return: zero on success, else a negative error code.
*/
static int spi_queued_transfer(struct spi_device *spi, struct spi_message *msg)
{
return __spi_queued_transfer(spi, msg, true);
}
static int spi_controller_initialize_queue(struct spi_controller *ctlr)
{
int ret;
ctlr->transfer = spi_queued_transfer;
if (!ctlr->transfer_one_message)
ctlr->transfer_one_message = spi_transfer_one_message;
/* Initialize and start queue */
ret = spi_init_queue(ctlr);
if (ret) {
dev_err(&ctlr->dev, "problem initializing queue\n");
goto err_init_queue;
}
ctlr->queued = true;
ret = spi_start_queue(ctlr);
if (ret) {
dev_err(&ctlr->dev, "problem starting queue\n");
goto err_start_queue;
}
return 0;
err_start_queue:
spi_destroy_queue(ctlr);
err_init_queue:
return ret;
}
/**
* spi_flush_queue - Send all pending messages in the queue from the callers'
* context
* @ctlr: controller to process queue for
*
* This should be used when one wants to ensure all pending messages have been
* sent before doing something. Is used by the spi-mem code to make sure SPI
* memory operations do not preempt regular SPI transfers that have been queued
* before the spi-mem operation.
*/
void spi_flush_queue(struct spi_controller *ctlr)
{
if (ctlr->transfer == spi_queued_transfer)
__spi_pump_messages(ctlr, false);
}
/*-------------------------------------------------------------------------*/
#if defined(CONFIG_OF)
static void of_spi_parse_dt_cs_delay(struct device_node *nc,
struct spi_delay *delay, const char *prop)
{
u32 value;
if (!of_property_read_u32(nc, prop, &value)) {
if (value > U16_MAX) {
delay->value = DIV_ROUND_UP(value, 1000);
delay->unit = SPI_DELAY_UNIT_USECS;
} else {
delay->value = value;
delay->unit = SPI_DELAY_UNIT_NSECS;
}
}
}
static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi,
struct device_node *nc)
{
u32 value, cs[SPI_CS_CNT_MAX];
int rc, idx;
/* Mode (clock phase/polarity/etc.) */
if (of_property_read_bool(nc, "spi-cpha"))
spi->mode |= SPI_CPHA;
if (of_property_read_bool(nc, "spi-cpol"))
spi->mode |= SPI_CPOL;
if (of_property_read_bool(nc, "spi-3wire"))
spi->mode |= SPI_3WIRE;
if (of_property_read_bool(nc, "spi-lsb-first"))
spi->mode |= SPI_LSB_FIRST;
if (of_property_read_bool(nc, "spi-cs-high"))
spi->mode |= SPI_CS_HIGH;
/* Device DUAL/QUAD mode */
if (!of_property_read_u32(nc, "spi-tx-bus-width", &value)) {
switch (value) {
case 0:
spi->mode |= SPI_NO_TX;
break;
case 1:
break;
case 2:
spi->mode |= SPI_TX_DUAL;
break;
case 4:
spi->mode |= SPI_TX_QUAD;
break;
case 8:
spi->mode |= SPI_TX_OCTAL;
break;
default:
dev_warn(&ctlr->dev,
"spi-tx-bus-width %d not supported\n",
value);
break;
}
}
if (!of_property_read_u32(nc, "spi-rx-bus-width", &value)) {
switch (value) {
case 0:
spi->mode |= SPI_NO_RX;
break;
case 1:
break;
case 2:
spi->mode |= SPI_RX_DUAL;
break;
case 4:
spi->mode |= SPI_RX_QUAD;
break;
case 8:
spi->mode |= SPI_RX_OCTAL;
break;
default:
dev_warn(&ctlr->dev,
"spi-rx-bus-width %d not supported\n",
value);
break;
}
}
if (spi_controller_is_slave(ctlr)) {
if (!of_node_name_eq(nc, "slave")) {
dev_err(&ctlr->dev, "%pOF is not called 'slave'\n",
nc);
return -EINVAL;
}
return 0;
}
if (ctlr->num_chipselect > SPI_CS_CNT_MAX) {
dev_err(&ctlr->dev, "No. of CS is more than max. no. of supported CS\n");
return -EINVAL;
}
spi_set_all_cs_unused(spi);
/* Device address */
rc = of_property_read_variable_u32_array(nc, "reg", &cs[0], 1,
SPI_CS_CNT_MAX);
if (rc < 0) {
dev_err(&ctlr->dev, "%pOF has no valid 'reg' property (%d)\n",
nc, rc);
return rc;
}
if (rc > ctlr->num_chipselect) {
dev_err(&ctlr->dev, "%pOF has number of CS > ctlr->num_chipselect (%d)\n",
nc, rc);
return rc;
}
if ((of_property_read_bool(nc, "parallel-memories")) &&
(!(ctlr->flags & SPI_CONTROLLER_MULTI_CS))) {
dev_err(&ctlr->dev, "SPI controller doesn't support multi CS\n");
return -EINVAL;
}
for (idx = 0; idx < rc; idx++)
spi_set_chipselect(spi, idx, cs[idx]);
/*
* By default spi->chip_select[0] will hold the physical CS number,
* so set bit 0 in spi->cs_index_mask.
*/
spi->cs_index_mask = BIT(0);
/* Device speed */
if (!of_property_read_u32(nc, "spi-max-frequency", &value))
spi->max_speed_hz = value;
/* Device CS delays */
of_spi_parse_dt_cs_delay(nc, &spi->cs_setup, "spi-cs-setup-delay-ns");
of_spi_parse_dt_cs_delay(nc, &spi->cs_hold, "spi-cs-hold-delay-ns");
of_spi_parse_dt_cs_delay(nc, &spi->cs_inactive, "spi-cs-inactive-delay-ns");
return 0;
}
static struct spi_device *
of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
{
struct spi_device *spi;
int rc;
/* Alloc an spi_device */
spi = spi_alloc_device(ctlr);
if (!spi) {
dev_err(&ctlr->dev, "spi_device alloc error for %pOF\n", nc);
rc = -ENOMEM;
goto err_out;
}
/* Select device driver */
rc = of_alias_from_compatible(nc, spi->modalias,
sizeof(spi->modalias));
if (rc < 0) {
dev_err(&ctlr->dev, "cannot find modalias for %pOF\n", nc);
goto err_out;
}
rc = of_spi_parse_dt(ctlr, spi, nc);
if (rc)
goto err_out;
/* Store a pointer to the node in the device structure */
of_node_get(nc);
device_set_node(&spi->dev, of_fwnode_handle(nc));
/* Register the new device */
rc = spi_add_device(spi);
if (rc) {
dev_err(&ctlr->dev, "spi_device register error %pOF\n", nc);
goto err_of_node_put;
}
return spi;
err_of_node_put:
of_node_put(nc);
err_out:
spi_dev_put(spi);
return ERR_PTR(rc);
}
/**
* of_register_spi_devices() - Register child devices onto the SPI bus
* @ctlr: Pointer to spi_controller device
*
* Registers an spi_device for each child node of controller node which
* represents a valid SPI slave.
*/
static void of_register_spi_devices(struct spi_controller *ctlr)
{
struct spi_device *spi;
struct device_node *nc;
for_each_available_child_of_node(ctlr->dev.of_node, nc) {
if (of_node_test_and_set_flag(nc, OF_POPULATED))
continue;
spi = of_register_spi_device(ctlr, nc);
if (IS_ERR(spi)) {
dev_warn(&ctlr->dev,
"Failed to create SPI device for %pOF\n", nc);
of_node_clear_flag(nc, OF_POPULATED);
}
}
}
#else
static void of_register_spi_devices(struct spi_controller *ctlr) { }
#endif
/**
* spi_new_ancillary_device() - Register ancillary SPI device
* @spi: Pointer to the main SPI device registering the ancillary device
* @chip_select: Chip Select of the ancillary device
*
* Register an ancillary SPI device; for example some chips have a chip-select
* for normal device usage and another one for setup/firmware upload.
*
* This may only be called from main SPI device's probe routine.
*
* Return: 0 on success; negative errno on failure
*/
struct spi_device *spi_new_ancillary_device(struct spi_device *spi,
u8 chip_select)
{
struct spi_controller *ctlr = spi->controller;
struct spi_device *ancillary;
int rc = 0;
/* Alloc an spi_device */
ancillary = spi_alloc_device(ctlr);
if (!ancillary) {
rc = -ENOMEM;
goto err_out;
}
strscpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias));
/* Use provided chip-select for ancillary device */
spi_set_all_cs_unused(ancillary);
spi_set_chipselect(ancillary, 0, chip_select);
/* Take over SPI mode/speed from SPI main device */
ancillary->max_speed_hz = spi->max_speed_hz;
ancillary->mode = spi->mode;
/*
* By default spi->chip_select[0] will hold the physical CS number,
* so set bit 0 in spi->cs_index_mask.
*/
ancillary->cs_index_mask = BIT(0);
WARN_ON(!mutex_is_locked(&ctlr->add_lock));
/* Register the new device */
rc = __spi_add_device(ancillary);
if (rc) {
dev_err(&spi->dev, "failed to register ancillary device\n");
goto err_out;
}
return ancillary;
err_out:
spi_dev_put(ancillary);
return ERR_PTR(rc);
}
EXPORT_SYMBOL_GPL(spi_new_ancillary_device);
#ifdef CONFIG_ACPI
struct acpi_spi_lookup {
struct spi_controller *ctlr;
u32 max_speed_hz;
u32 mode;
int irq;
u8 bits_per_word;
u8 chip_select;
int n;
int index;
};
static int acpi_spi_count(struct acpi_resource *ares, void *data)
{
struct acpi_resource_spi_serialbus *sb;
int *count = data;
if (ares->type != ACPI_RESOURCE_TYPE_SERIAL_BUS)
return 1;
sb = &ares->data.spi_serial_bus;
if (sb->type != ACPI_RESOURCE_SERIAL_TYPE_SPI)
return 1;
*count = *count + 1;
return 1;
}
/**
* acpi_spi_count_resources - Count the number of SpiSerialBus resources
* @adev: ACPI device
*
* Return: the number of SpiSerialBus resources in the ACPI-device's
* resource-list; or a negative error code.
*/
int acpi_spi_count_resources(struct acpi_device *adev)
{
LIST_HEAD(r);
int count = 0;
int ret;
ret = acpi_dev_get_resources(adev, &r, acpi_spi_count, &count);
if (ret < 0)
return ret;
acpi_dev_free_resource_list(&r);
return count;
}
EXPORT_SYMBOL_GPL(acpi_spi_count_resources);
static void acpi_spi_parse_apple_properties(struct acpi_device *dev,
struct acpi_spi_lookup *lookup)
{
const union acpi_object *obj;
if (!x86_apple_machine)
return;
if (!acpi_dev_get_property(dev, "spiSclkPeriod", ACPI_TYPE_BUFFER, &obj)
&& obj->buffer.length >= 4)
lookup->max_speed_hz = NSEC_PER_SEC / *(u32 *)obj->buffer.pointer;
if (!acpi_dev_get_property(dev, "spiWordSize", ACPI_TYPE_BUFFER, &obj)
&& obj->buffer.length == 8)
lookup->bits_per_word = *(u64 *)obj->buffer.pointer;
if (!acpi_dev_get_property(dev, "spiBitOrder", ACPI_TYPE_BUFFER, &obj)
&& obj->buffer.length == 8 && !*(u64 *)obj->buffer.pointer)
lookup->mode |= SPI_LSB_FIRST;
if (!acpi_dev_get_property(dev, "spiSPO", ACPI_TYPE_BUFFER, &obj)
&& obj->buffer.length == 8 && *(u64 *)obj->buffer.pointer)
lookup->mode |= SPI_CPOL;
if (!acpi_dev_get_property(dev, "spiSPH", ACPI_TYPE_BUFFER, &obj)
&& obj->buffer.length == 8 && *(u64 *)obj->buffer.pointer)
lookup->mode |= SPI_CPHA;
}
static int acpi_spi_add_resource(struct acpi_resource *ares, void *data)
{
struct acpi_spi_lookup *lookup = data;
struct spi_controller *ctlr = lookup->ctlr;
if (ares->type == ACPI_RESOURCE_TYPE_SERIAL_BUS) {
struct acpi_resource_spi_serialbus *sb;
acpi_handle parent_handle;
acpi_status status;
sb = &ares->data.spi_serial_bus;
if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_SPI) {
if (lookup->index != -1 && lookup->n++ != lookup->index)
return 1;
status = acpi_get_handle(NULL,
sb->resource_source.string_ptr,
&parent_handle);
if (ACPI_FAILURE(status))
return -ENODEV;
if (ctlr) {
if (ACPI_HANDLE(ctlr->dev.parent) != parent_handle)
return -ENODEV;
} else {
struct acpi_device *adev;
adev = acpi_fetch_acpi_dev(parent_handle);
if (!adev)
return -ENODEV;
ctlr = acpi_spi_find_controller_by_adev(adev);
if (!ctlr)
return -EPROBE_DEFER;
lookup->ctlr = ctlr;
}
/*
* ACPI DeviceSelection numbering is handled by the
* host controller driver in Windows and can vary
* from driver to driver. In Linux we always expect
* 0 .. max - 1 so we need to ask the driver to
* translate between the two schemes.
*/
if (ctlr->fw_translate_cs) {
int cs = ctlr->fw_translate_cs(ctlr,
sb->device_selection);
if (cs < 0)
return cs;
lookup->chip_select = cs;
} else {
lookup->chip_select = sb->device_selection;
}
lookup->max_speed_hz = sb->connection_speed;
lookup->bits_per_word = sb->data_bit_length;
if (sb->clock_phase == ACPI_SPI_SECOND_PHASE)
lookup->mode |= SPI_CPHA;
if (sb->clock_polarity == ACPI_SPI_START_HIGH)
lookup->mode |= SPI_CPOL;
if (sb->device_polarity == ACPI_SPI_ACTIVE_HIGH)
lookup->mode |= SPI_CS_HIGH;
}
} else if (lookup->irq < 0) {
struct resource r;
if (acpi_dev_resource_interrupt(ares, 0, &r))
lookup->irq = r.start;
}
/* Always tell the ACPI core to skip this resource */
return 1;
}
/**
* acpi_spi_device_alloc - Allocate a spi device, and fill it in with ACPI information
* @ctlr: controller to which the spi device belongs
* @adev: ACPI Device for the spi device
* @index: Index of the spi resource inside the ACPI Node
*
* This should be used to allocate a new SPI device from and ACPI Device node.
* The caller is responsible for calling spi_add_device to register the SPI device.
*
* If ctlr is set to NULL, the Controller for the SPI device will be looked up
* using the resource.
* If index is set to -1, index is not used.
* Note: If index is -1, ctlr must be set.
*
* Return: a pointer to the new device, or ERR_PTR on error.
*/
struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr,
struct acpi_device *adev,
int index)
{
acpi_handle parent_handle = NULL;
struct list_head resource_list;
struct acpi_spi_lookup lookup = {};
struct spi_device *spi;
int ret;
if (!ctlr && index == -1)
return ERR_PTR(-EINVAL);
lookup.ctlr = ctlr;
lookup.irq = -1;
lookup.index = index;
lookup.n = 0;
INIT_LIST_HEAD(&resource_list);
ret = acpi_dev_get_resources(adev, &resource_list,
acpi_spi_add_resource, &lookup);
acpi_dev_free_resource_list(&resource_list);
if (ret < 0)
/* Found SPI in _CRS but it points to another controller */
return ERR_PTR(ret);
if (!lookup.max_speed_hz &&
ACPI_SUCCESS(acpi_get_parent(adev->handle, &parent_handle)) &&
ACPI_HANDLE(lookup.ctlr->dev.parent) == parent_handle) {
/* Apple does not use _CRS but nested devices for SPI slaves */
acpi_spi_parse_apple_properties(adev, &lookup);
}
if (!lookup.max_speed_hz)
return ERR_PTR(-ENODEV);
spi = spi_alloc_device(lookup.ctlr);
if (!spi) {
dev_err(&lookup.ctlr->dev, "failed to allocate SPI device for %s\n",
dev_name(&adev->dev));
return ERR_PTR(-ENOMEM);
}
spi_set_all_cs_unused(spi);
spi_set_chipselect(spi, 0, lookup.chip_select);
ACPI_COMPANION_SET(&spi->dev, adev);
spi->max_speed_hz = lookup.max_speed_hz;
spi->mode |= lookup.mode;
spi->irq = lookup.irq;
spi->bits_per_word = lookup.bits_per_word;
/*
* By default spi->chip_select[0] will hold the physical CS number,
* so set bit 0 in spi->cs_index_mask.
*/
spi->cs_index_mask = BIT(0);
return spi;
}
EXPORT_SYMBOL_GPL(acpi_spi_device_alloc);
static acpi_status acpi_register_spi_device(struct spi_controller *ctlr,
struct acpi_device *adev)
{
struct spi_device *spi;
if (acpi_bus_get_status(adev) || !adev->status.present ||
acpi_device_enumerated(adev))
return AE_OK;
spi = acpi_spi_device_alloc(ctlr, adev, -1);
if (IS_ERR(spi)) {
if (PTR_ERR(spi) == -ENOMEM)
return AE_NO_MEMORY;
else
return AE_OK;
}
acpi_set_modalias(adev, acpi_device_hid(adev), spi->modalias,
sizeof(spi->modalias));
if (spi->irq < 0)
spi->irq = acpi_dev_gpio_irq_get(adev, 0);
acpi_device_set_enumerated(adev);
adev->power.flags.ignore_parent = true;
if (spi_add_device(spi)) {
adev->power.flags.ignore_parent = false;
dev_err(&ctlr->dev, "failed to add SPI device %s from ACPI\n",
dev_name(&adev->dev));
spi_dev_put(spi);
}
return AE_OK;
}
static acpi_status acpi_spi_add_device(acpi_handle handle, u32 level,
void *data, void **return_value)
{
struct acpi_device *adev = acpi_fetch_acpi_dev(handle);
struct spi_controller *ctlr = data;
if (!adev)
return AE_OK;
return acpi_register_spi_device(ctlr, adev);
}
#define SPI_ACPI_ENUMERATE_MAX_DEPTH 32
static void acpi_register_spi_devices(struct spi_controller *ctlr)
{
acpi_status status;
acpi_handle handle;
handle = ACPI_HANDLE(ctlr->dev.parent);
if (!handle)
return;
status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
SPI_ACPI_ENUMERATE_MAX_DEPTH,
acpi_spi_add_device, NULL, ctlr, NULL);
if (ACPI_FAILURE(status))
dev_warn(&ctlr->dev, "failed to enumerate SPI slaves\n");
}
#else
static inline void acpi_register_spi_devices(struct spi_controller *ctlr) {}
#endif /* CONFIG_ACPI */
static void spi_controller_release(struct device *dev)
{
struct spi_controller *ctlr;
ctlr = container_of(dev, struct spi_controller, dev);
kfree(ctlr);
}
static struct class spi_master_class = {
.name = "spi_master",
.dev_release = spi_controller_release,
.dev_groups = spi_master_groups,
};
#ifdef CONFIG_SPI_SLAVE
/**
* spi_slave_abort - abort the ongoing transfer request on an SPI slave
* controller
* @spi: device used for the current transfer
*/
int spi_slave_abort(struct spi_device *spi)
{
struct spi_controller *ctlr = spi->controller;
if (spi_controller_is_slave(ctlr) && ctlr->slave_abort)
return ctlr->slave_abort(ctlr);
return -ENOTSUPP;
}
EXPORT_SYMBOL_GPL(spi_slave_abort);
int spi_target_abort(struct spi_device *spi)
{
struct spi_controller *ctlr = spi->controller;
if (spi_controller_is_target(ctlr) && ctlr->target_abort)
return ctlr->target_abort(ctlr);
return -ENOTSUPP;
}
EXPORT_SYMBOL_GPL(spi_target_abort);
static ssize_t slave_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct spi_controller *ctlr = container_of(dev, struct spi_controller,
dev);
struct device *child;
child = device_find_any_child(&ctlr->dev);
return sysfs_emit(buf, "%s\n", child ? to_spi_device(child)->modalias : NULL);
}
static ssize_t slave_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct spi_controller *ctlr = container_of(dev, struct spi_controller,
dev);
struct spi_device *spi;
struct device *child;
char name[32];
int rc;
rc = sscanf(buf, "%31s", name);
if (rc != 1 || !name[0])
return -EINVAL;
child = device_find_any_child(&ctlr->dev);
if (child) {
/* Remove registered slave */
device_unregister(child);
put_device(child);
}
if (strcmp(name, "(null)")) {
/* Register new slave */
spi = spi_alloc_device(ctlr);
if (!spi)
return -ENOMEM;
strscpy(spi->modalias, name, sizeof(spi->modalias));
rc = spi_add_device(spi);
if (rc) {
spi_dev_put(spi);
return rc;
}
}
return count;
}
static DEVICE_ATTR_RW(slave);
static struct attribute *spi_slave_attrs[] = {
&dev_attr_slave.attr,
NULL,
};
static const struct attribute_group spi_slave_group = {
.attrs = spi_slave_attrs,
};
static const struct attribute_group *spi_slave_groups[] = {
&spi_controller_statistics_group,
&spi_slave_group,
NULL,
};
static struct class spi_slave_class = {
.name = "spi_slave",
.dev_release = spi_controller_release,
.dev_groups = spi_slave_groups,
};
#else
extern struct class spi_slave_class; /* dummy */
#endif
/**
* __spi_alloc_controller - allocate an SPI master or slave controller
* @dev: the controller, possibly using the platform_bus
* @size: how much zeroed driver-private data to allocate; the pointer to this
* memory is in the driver_data field of the returned device, accessible
* with spi_controller_get_devdata(); the memory is cacheline aligned;
* drivers granting DMA access to portions of their private data need to
* round up @size using ALIGN(size, dma_get_cache_alignment()).
* @slave: flag indicating whether to allocate an SPI master (false) or SPI
* slave (true) controller
* Context: can sleep
*
* This call is used only by SPI controller drivers, which are the
* only ones directly touching chip registers. It's how they allocate
* an spi_controller structure, prior to calling spi_register_controller().
*
* This must be called from context that can sleep.
*
* The caller is responsible for assigning the bus number and initializing the
* controller's methods before calling spi_register_controller(); and (after
* errors adding the device) calling spi_controller_put() to prevent a memory
* leak.
*
* Return: the SPI controller structure on success, else NULL.
*/
struct spi_controller *__spi_alloc_controller(struct device *dev,
unsigned int size, bool slave)
{
struct spi_controller *ctlr;
size_t ctlr_size = ALIGN(sizeof(*ctlr), dma_get_cache_alignment());
if (!dev)
return NULL;
ctlr = kzalloc(size + ctlr_size, GFP_KERNEL);
if (!ctlr)
return NULL;
device_initialize(&ctlr->dev);
INIT_LIST_HEAD(&ctlr->queue);
spin_lock_init(&ctlr->queue_lock);
spin_lock_init(&ctlr->bus_lock_spinlock);
mutex_init(&ctlr->bus_lock_mutex);
mutex_init(&ctlr->io_mutex);
mutex_init(&ctlr->add_lock);
ctlr->bus_num = -1;
ctlr->num_chipselect = 1;
ctlr->slave = slave;
if (IS_ENABLED(CONFIG_SPI_SLAVE) && slave)
ctlr->dev.class = &spi_slave_class;
else
ctlr->dev.class = &spi_master_class;
ctlr->dev.parent = dev;
pm_suspend_ignore_children(&ctlr->dev, true);
spi_controller_set_devdata(ctlr, (void *)ctlr + ctlr_size);
return ctlr;
}
EXPORT_SYMBOL_GPL(__spi_alloc_controller);
static void devm_spi_release_controller(struct device *dev, void *ctlr)
{
spi_controller_put(*(struct spi_controller **)ctlr);
}
/**
* __devm_spi_alloc_controller - resource-managed __spi_alloc_controller()
* @dev: physical device of SPI controller
* @size: how much zeroed driver-private data to allocate
* @slave: whether to allocate an SPI master (false) or SPI slave (true)
* Context: can sleep
*
* Allocate an SPI controller and automatically release a reference on it
* when @dev is unbound from its driver. Drivers are thus relieved from
* having to call spi_controller_put().
*
* The arguments to this function are identical to __spi_alloc_controller().
*
* Return: the SPI controller structure on success, else NULL.
*/
struct spi_controller *__devm_spi_alloc_controller(struct device *dev,
unsigned int size,
bool slave)
{
struct spi_controller **ptr, *ctlr;
ptr = devres_alloc(devm_spi_release_controller, sizeof(*ptr),
GFP_KERNEL);
if (!ptr)
return NULL;
ctlr = __spi_alloc_controller(dev, size, slave);
if (ctlr) {
ctlr->devm_allocated = true;
*ptr = ctlr;
devres_add(dev, ptr);
} else {
devres_free(ptr);
}
return ctlr;
}
EXPORT_SYMBOL_GPL(__devm_spi_alloc_controller);
/**
* spi_get_gpio_descs() - grab chip select GPIOs for the master
* @ctlr: The SPI master to grab GPIO descriptors for
*/
static int spi_get_gpio_descs(struct spi_controller *ctlr)
{
int nb, i;
struct gpio_desc **cs;
struct device *dev = &ctlr->dev;
unsigned long native_cs_mask = 0;
unsigned int num_cs_gpios = 0;
nb = gpiod_count(dev, "cs");
if (nb < 0) {
/* No GPIOs at all is fine, else return the error */
if (nb == -ENOENT)
return 0;
return nb;
}
ctlr->num_chipselect = max_t(int, nb, ctlr->num_chipselect);
cs = devm_kcalloc(dev, ctlr->num_chipselect, sizeof(*cs),
GFP_KERNEL);
if (!cs)
return -ENOMEM;
ctlr->cs_gpiods = cs;
for (i = 0; i < nb; i++) {
/*
* Most chipselects are active low, the inverted
* semantics are handled by special quirks in gpiolib,
* so initializing them GPIOD_OUT_LOW here means
* "unasserted", in most cases this will drive the physical
* line high.
*/
cs[i] = devm_gpiod_get_index_optional(dev, "cs", i,
GPIOD_OUT_LOW);
if (IS_ERR(cs[i]))
return PTR_ERR(cs[i]);
if (cs[i]) {
/*
* If we find a CS GPIO, name it after the device and
* chip select line.
*/
char *gpioname;
gpioname = devm_kasprintf(dev, GFP_KERNEL, "%s CS%d",
dev_name(dev), i);
if (!gpioname)
return -ENOMEM;
gpiod_set_consumer_name(cs[i], gpioname);
num_cs_gpios++;
continue;
}
if (ctlr->max_native_cs && i >= ctlr->max_native_cs) {
dev_err(dev, "Invalid native chip select %d\n", i);
return -EINVAL;
}
native_cs_mask |= BIT(i);
}
ctlr->unused_native_cs = ffs(~native_cs_mask) - 1;
if ((ctlr->flags & SPI_CONTROLLER_GPIO_SS) && num_cs_gpios &&
ctlr->max_native_cs && ctlr->unused_native_cs >= ctlr->max_native_cs) {
dev_err(dev, "No unused native chip select available\n");
return -EINVAL;
}
return 0;
}
static int spi_controller_check_ops(struct spi_controller *ctlr)
{
/*
* The controller may implement only the high-level SPI-memory like
* operations if it does not support regular SPI transfers, and this is
* valid use case.
* If ->mem_ops or ->mem_ops->exec_op is NULL, we request that at least
* one of the ->transfer_xxx() method be implemented.
*/
if (!ctlr->mem_ops || !ctlr->mem_ops->exec_op) {
if (!ctlr->transfer && !ctlr->transfer_one &&
!ctlr->transfer_one_message) {
return -EINVAL;
}
}
return 0;
}
/* Allocate dynamic bus number using Linux idr */
static int spi_controller_id_alloc(struct spi_controller *ctlr, int start, int end)
{
int id;
mutex_lock(&board_lock);
id = idr_alloc(&spi_master_idr, ctlr, start, end, GFP_KERNEL);
mutex_unlock(&board_lock);
if (WARN(id < 0, "couldn't get idr"))
return id == -ENOSPC ? -EBUSY : id;
ctlr->bus_num = id;
return 0;
}
/**
* spi_register_controller - register SPI master or slave controller
* @ctlr: initialized master, originally from spi_alloc_master() or
* spi_alloc_slave()
* Context: can sleep
*
* SPI controllers connect to their drivers using some non-SPI bus,
* such as the platform bus. The final stage of probe() in that code
* includes calling spi_register_controller() to hook up to this SPI bus glue.
*
* SPI controllers use board specific (often SOC specific) bus numbers,
* and board-specific addressing for SPI devices combines those numbers
* with chip select numbers. Since SPI does not directly support dynamic
* device identification, boards need configuration tables telling which
* chip is at which address.
*
* This must be called from context that can sleep. It returns zero on
* success, else a negative error code (dropping the controller's refcount).
* After a successful return, the caller is responsible for calling
* spi_unregister_controller().
*
* Return: zero on success, else a negative error code.
*/
int spi_register_controller(struct spi_controller *ctlr)
{
struct device *dev = ctlr->dev.parent;
struct boardinfo *bi;
int first_dynamic;
int status;
int idx;
if (!dev)
return -ENODEV;
/*
* Make sure all necessary hooks are implemented before registering
* the SPI controller.
*/
status = spi_controller_check_ops(ctlr);
if (status)
return status;
if (ctlr->bus_num < 0)
ctlr->bus_num = of_alias_get_id(ctlr->dev.of_node, "spi");
if (ctlr->bus_num >= 0) {
/* Devices with a fixed bus num must check-in with the num */
status = spi_controller_id_alloc(ctlr, ctlr->bus_num, ctlr->bus_num + 1);
if (status)
return status;
}
if (ctlr->bus_num < 0) {
first_dynamic = of_alias_get_highest_id("spi");
if (first_dynamic < 0)
first_dynamic = 0;
else
first_dynamic++;
status = spi_controller_id_alloc(ctlr, first_dynamic, 0);
if (status)
return status;
}
ctlr->bus_lock_flag = 0;
init_completion(&ctlr->xfer_completion);
init_completion(&ctlr->cur_msg_completion);
if (!ctlr->max_dma_len)
ctlr->max_dma_len = INT_MAX;
/*
* Register the device, then userspace will see it.
* Registration fails if the bus ID is in use.
*/
dev_set_name(&ctlr->dev, "spi%u", ctlr->bus_num);
if (!spi_controller_is_slave(ctlr) && ctlr->use_gpio_descriptors) {
status = spi_get_gpio_descs(ctlr);
if (status)
goto free_bus_id;
/*
* A controller using GPIO descriptors always
* supports SPI_CS_HIGH if need be.
*/
ctlr->mode_bits |= SPI_CS_HIGH;
}
/*
* Even if it's just one always-selected device, there must
* be at least one chipselect.
*/
if (!ctlr->num_chipselect) {
status = -EINVAL;
goto free_bus_id;
}
/* Setting last_cs to SPI_INVALID_CS means no chip selected */
for (idx = 0; idx < SPI_CS_CNT_MAX; idx++)
ctlr->last_cs[idx] = SPI_INVALID_CS;
status = device_add(&ctlr->dev);
if (status < 0)
goto free_bus_id;
dev_dbg(dev, "registered %s %s\n",
spi_controller_is_slave(ctlr) ? "slave" : "master",
dev_name(&ctlr->dev));
/*
* If we're using a queued driver, start the queue. Note that we don't
* need the queueing logic if the driver is only supporting high-level
* memory operations.
*/
if (ctlr->transfer) {
dev_info(dev, "controller is unqueued, this is deprecated\n");
} else if (ctlr->transfer_one || ctlr->transfer_one_message) {
status = spi_controller_initialize_queue(ctlr);
if (status) {
device_del(&ctlr->dev);
goto free_bus_id;
}
}
/* Add statistics */
ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev);
if (!ctlr->pcpu_statistics) {
dev_err(dev, "Error allocating per-cpu statistics\n");
status = -ENOMEM;
goto destroy_queue;
}
mutex_lock(&board_lock);
list_add_tail(&ctlr->list, &spi_controller_list);
list_for_each_entry(bi, &board_list, list)
spi_match_controller_to_boardinfo(ctlr, &bi->board_info);
mutex_unlock(&board_lock);
/* Register devices from the device tree and ACPI */
of_register_spi_devices(ctlr);
acpi_register_spi_devices(ctlr);
return status;
destroy_queue:
spi_destroy_queue(ctlr);
free_bus_id:
mutex_lock(&board_lock);
idr_remove(&spi_master_idr, ctlr->bus_num);
mutex_unlock(&board_lock);
return status;
}
EXPORT_SYMBOL_GPL(spi_register_controller);
static void devm_spi_unregister(struct device *dev, void *res)
{
spi_unregister_controller(*(struct spi_controller **)res);
}
/**
* devm_spi_register_controller - register managed SPI master or slave
* controller
* @dev: device managing SPI controller
* @ctlr: initialized controller, originally from spi_alloc_master() or
* spi_alloc_slave()
* Context: can sleep
*
* Register a SPI device as with spi_register_controller() which will
* automatically be unregistered and freed.
*
* Return: zero on success, else a negative error code.
*/
int devm_spi_register_controller(struct device *dev,
struct spi_controller *ctlr)
{
struct spi_controller **ptr;
int ret;
ptr = devres_alloc(devm_spi_unregister, sizeof(*ptr), GFP_KERNEL);
if (!ptr)
return -ENOMEM;
ret = spi_register_controller(ctlr);
if (!ret) {
*ptr = ctlr;
devres_add(dev, ptr);
} else {
devres_free(ptr);
}
return ret;
}
EXPORT_SYMBOL_GPL(devm_spi_register_controller);
static int __unregister(struct device *dev, void *null)
{
spi_unregister_device(to_spi_device(dev));
return 0;
}
/**
* spi_unregister_controller - unregister SPI master or slave controller
* @ctlr: the controller being unregistered
* Context: can sleep
*
* This call is used only by SPI controller drivers, which are the
* only ones directly touching chip registers.
*
* This must be called from context that can sleep.
*
* Note that this function also drops a reference to the controller.
*/
void spi_unregister_controller(struct spi_controller *ctlr)
{
struct spi_controller *found;
int id = ctlr->bus_num;
/* Prevent addition of new devices, unregister existing ones */
if (IS_ENABLED(CONFIG_SPI_DYNAMIC))
mutex_lock(&ctlr->add_lock);
device_for_each_child(&ctlr->dev, NULL, __unregister);
/* First make sure that this controller was ever added */
mutex_lock(&board_lock);
found = idr_find(&spi_master_idr, id);
mutex_unlock(&board_lock);
if (ctlr->queued) {
if (spi_destroy_queue(ctlr))
dev_err(&ctlr->dev, "queue remove failed\n");
}
mutex_lock(&board_lock); list_del(&ctlr->list);
mutex_unlock(&board_lock);
device_del(&ctlr->dev);
/* Free bus id */
mutex_lock(&board_lock);
if (found == ctlr)
idr_remove(&spi_master_idr, id); mutex_unlock(&board_lock);
if (IS_ENABLED(CONFIG_SPI_DYNAMIC))
mutex_unlock(&ctlr->add_lock);
/*
* Release the last reference on the controller if its driver
* has not yet been converted to devm_spi_alloc_master/slave().
*/
if (!ctlr->devm_allocated)
put_device(&ctlr->dev);
}
EXPORT_SYMBOL_GPL(spi_unregister_controller);
static inline int __spi_check_suspended(const struct spi_controller *ctlr)
{
return ctlr->flags & SPI_CONTROLLER_SUSPENDED ? -ESHUTDOWN : 0;
}
static inline void __spi_mark_suspended(struct spi_controller *ctlr)
{
mutex_lock(&ctlr->bus_lock_mutex);
ctlr->flags |= SPI_CONTROLLER_SUSPENDED;
mutex_unlock(&ctlr->bus_lock_mutex);
}
static inline void __spi_mark_resumed(struct spi_controller *ctlr)
{
mutex_lock(&ctlr->bus_lock_mutex);
ctlr->flags &= ~SPI_CONTROLLER_SUSPENDED;
mutex_unlock(&ctlr->bus_lock_mutex);
}
int spi_controller_suspend(struct spi_controller *ctlr)
{
int ret = 0;
/* Basically no-ops for non-queued controllers */
if (ctlr->queued) {
ret = spi_stop_queue(ctlr);
if (ret)
dev_err(&ctlr->dev, "queue stop failed\n");
}
__spi_mark_suspended(ctlr);
return ret;
}
EXPORT_SYMBOL_GPL(spi_controller_suspend);
int spi_controller_resume(struct spi_controller *ctlr)
{
int ret = 0;
__spi_mark_resumed(ctlr);
if (ctlr->queued) {
ret = spi_start_queue(ctlr);
if (ret)
dev_err(&ctlr->dev, "queue restart failed\n");
}
return ret;
}
EXPORT_SYMBOL_GPL(spi_controller_resume);
/*-------------------------------------------------------------------------*/
/* Core methods for spi_message alterations */
static void __spi_replace_transfers_release(struct spi_controller *ctlr,
struct spi_message *msg,
void *res)
{
struct spi_replaced_transfers *rxfer = res;
size_t i;
/* Call extra callback if requested */
if (rxfer->release)
rxfer->release(ctlr, msg, res);
/* Insert replaced transfers back into the message */
list_splice(&rxfer->replaced_transfers, rxfer->replaced_after);
/* Remove the formerly inserted entries */
for (i = 0; i < rxfer->inserted; i++)
list_del(&rxfer->inserted_transfers[i].transfer_list);
}
/**
* spi_replace_transfers - replace transfers with several transfers
* and register change with spi_message.resources
* @msg: the spi_message we work upon
* @xfer_first: the first spi_transfer we want to replace
* @remove: number of transfers to remove
* @insert: the number of transfers we want to insert instead
* @release: extra release code necessary in some circumstances
* @extradatasize: extra data to allocate (with alignment guarantees
* of struct @spi_transfer)
* @gfp: gfp flags
*
* Returns: pointer to @spi_replaced_transfers,
* PTR_ERR(...) in case of errors.
*/
static struct spi_replaced_transfers *spi_replace_transfers(
struct spi_message *msg,
struct spi_transfer *xfer_first,
size_t remove,
size_t insert,
spi_replaced_release_t release,
size_t extradatasize,
gfp_t gfp)
{
struct spi_replaced_transfers *rxfer;
struct spi_transfer *xfer;
size_t i;
/* Allocate the structure using spi_res */
rxfer = spi_res_alloc(msg->spi, __spi_replace_transfers_release,
struct_size(rxfer, inserted_transfers, insert)
+ extradatasize,
gfp);
if (!rxfer)
return ERR_PTR(-ENOMEM);
/* The release code to invoke before running the generic release */
rxfer->release = release;
/* Assign extradata */
if (extradatasize)
rxfer->extradata =
&rxfer->inserted_transfers[insert];
/* Init the replaced_transfers list */
INIT_LIST_HEAD(&rxfer->replaced_transfers);
/*
* Assign the list_entry after which we should reinsert
* the @replaced_transfers - it may be spi_message.messages!
*/
rxfer->replaced_after = xfer_first->transfer_list.prev;
/* Remove the requested number of transfers */
for (i = 0; i < remove; i++) {
/*
* If the entry after replaced_after it is msg->transfers
* then we have been requested to remove more transfers
* than are in the list.
*/
if (rxfer->replaced_after->next == &msg->transfers) {
dev_err(&msg->spi->dev,
"requested to remove more spi_transfers than are available\n");
/* Insert replaced transfers back into the message */
list_splice(&rxfer->replaced_transfers,
rxfer->replaced_after);
/* Free the spi_replace_transfer structure... */
spi_res_free(rxfer);
/* ...and return with an error */
return ERR_PTR(-EINVAL);
}
/*
* Remove the entry after replaced_after from list of
* transfers and add it to list of replaced_transfers.
*/
list_move_tail(rxfer->replaced_after->next,
&rxfer->replaced_transfers);
}
/*
* Create copy of the given xfer with identical settings
* based on the first transfer to get removed.
*/
for (i = 0; i < insert; i++) {
/* We need to run in reverse order */
xfer = &rxfer->inserted_transfers[insert - 1 - i];
/* Copy all spi_transfer data */
memcpy(xfer, xfer_first, sizeof(*xfer));
/* Add to list */
list_add(&xfer->transfer_list, rxfer->replaced_after);
/* Clear cs_change and delay for all but the last */
if (i) {
xfer->cs_change = false;
xfer->delay.value = 0;
}
}
/* Set up inserted... */
rxfer->inserted = insert;
/* ...and register it with spi_res/spi_message */
spi_res_add(msg, rxfer);
return rxfer;
}
static int __spi_split_transfer_maxsize(struct spi_controller *ctlr,
struct spi_message *msg,
struct spi_transfer **xferp,
size_t maxsize)
{
struct spi_transfer *xfer = *xferp, *xfers;
struct spi_replaced_transfers *srt;
size_t offset;
size_t count, i;
/* Calculate how many we have to replace */
count = DIV_ROUND_UP(xfer->len, maxsize);
/* Create replacement */
srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, GFP_KERNEL);
if (IS_ERR(srt))
return PTR_ERR(srt);
xfers = srt->inserted_transfers;
/*
* Now handle each of those newly inserted spi_transfers.
* Note that the replacements spi_transfers all are preset
* to the same values as *xferp, so tx_buf, rx_buf and len
* are all identical (as well as most others)
* so we just have to fix up len and the pointers.
*
* This also includes support for the depreciated
* spi_message.is_dma_mapped interface.
*/
/*
* The first transfer just needs the length modified, so we
* run it outside the loop.
*/
xfers[0].len = min_t(size_t, maxsize, xfer[0].len);
/* All the others need rx_buf/tx_buf also set */
for (i = 1, offset = maxsize; i < count; offset += maxsize, i++) {
/* Update rx_buf, tx_buf and DMA */
if (xfers[i].rx_buf)
xfers[i].rx_buf += offset;
if (xfers[i].rx_dma)
xfers[i].rx_dma += offset;
if (xfers[i].tx_buf)
xfers[i].tx_buf += offset;
if (xfers[i].tx_dma)
xfers[i].tx_dma += offset;
/* Update length */
xfers[i].len = min(maxsize, xfers[i].len - offset);
}
/*
* We set up xferp to the last entry we have inserted,
* so that we skip those already split transfers.
*/
*xferp = &xfers[count - 1];
/* Increment statistics counters */
SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics,
transfers_split_maxsize);
SPI_STATISTICS_INCREMENT_FIELD(msg->spi->pcpu_statistics,
transfers_split_maxsize);
return 0;
}
/**
* spi_split_transfers_maxsize - split spi transfers into multiple transfers
* when an individual transfer exceeds a
* certain size
* @ctlr: the @spi_controller for this transfer
* @msg: the @spi_message to transform
* @maxsize: the maximum when to apply this
*
* This function allocates resources that are automatically freed during the
* spi message unoptimize phase so this function should only be called from
* optimize_message callbacks.
*
* Return: status of transformation
*/
int spi_split_transfers_maxsize(struct spi_controller *ctlr,
struct spi_message *msg,
size_t maxsize)
{
struct spi_transfer *xfer;
int ret;
/*
* Iterate over the transfer_list,
* but note that xfer is advanced to the last transfer inserted
* to avoid checking sizes again unnecessarily (also xfer does
* potentially belong to a different list by the time the
* replacement has happened).
*/
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
if (xfer->len > maxsize) {
ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,
maxsize);
if (ret)
return ret;
}
}
return 0;
}
EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize);
/**
* spi_split_transfers_maxwords - split SPI transfers into multiple transfers
* when an individual transfer exceeds a
* certain number of SPI words
* @ctlr: the @spi_controller for this transfer
* @msg: the @spi_message to transform
* @maxwords: the number of words to limit each transfer to
*
* This function allocates resources that are automatically freed during the
* spi message unoptimize phase so this function should only be called from
* optimize_message callbacks.
*
* Return: status of transformation
*/
int spi_split_transfers_maxwords(struct spi_controller *ctlr,
struct spi_message *msg,
size_t maxwords)
{
struct spi_transfer *xfer;
/*
* Iterate over the transfer_list,
* but note that xfer is advanced to the last transfer inserted
* to avoid checking sizes again unnecessarily (also xfer does
* potentially belong to a different list by the time the
* replacement has happened).
*/
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
size_t maxsize;
int ret;
maxsize = maxwords * roundup_pow_of_two(BITS_TO_BYTES(xfer->bits_per_word));
if (xfer->len > maxsize) {
ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,
maxsize);
if (ret)
return ret;
}
}
return 0;
}
EXPORT_SYMBOL_GPL(spi_split_transfers_maxwords);
/*-------------------------------------------------------------------------*/
/*
* Core methods for SPI controller protocol drivers. Some of the
* other core methods are currently defined as inline functions.
*/
static int __spi_validate_bits_per_word(struct spi_controller *ctlr,
u8 bits_per_word)
{
if (ctlr->bits_per_word_mask) {
/* Only 32 bits fit in the mask */
if (bits_per_word > 32)
return -EINVAL;
if (!(ctlr->bits_per_word_mask & SPI_BPW_MASK(bits_per_word)))
return -EINVAL;
}
return 0;
}
/**
* spi_set_cs_timing - configure CS setup, hold, and inactive delays
* @spi: the device that requires specific CS timing configuration
*
* Return: zero on success, else a negative error code.
*/
static int spi_set_cs_timing(struct spi_device *spi)
{
struct device *parent = spi->controller->dev.parent;
int status = 0;
if (spi->controller->set_cs_timing && !spi_get_csgpiod(spi, 0)) {
if (spi->controller->auto_runtime_pm) {
status = pm_runtime_get_sync(parent);
if (status < 0) {
pm_runtime_put_noidle(parent);
dev_err(&spi->controller->dev, "Failed to power device: %d\n",
status);
return status;
}
status = spi->controller->set_cs_timing(spi);
pm_runtime_mark_last_busy(parent);
pm_runtime_put_autosuspend(parent);
} else {
status = spi->controller->set_cs_timing(spi);
}
}
return status;
}
/**
* spi_setup - setup SPI mode and clock rate
* @spi: the device whose settings are being modified
* Context: can sleep, and no requests are queued to the device
*
* SPI protocol drivers may need to update the transfer mode if the
* device doesn't work with its default. They may likewise need
* to update clock rates or word sizes from initial values. This function
* changes those settings, and must be called from a context that can sleep.
* Except for SPI_CS_HIGH, which takes effect immediately, the changes take
* effect the next time the device is selected and data is transferred to
* or from it. When this function returns, the SPI device is deselected.
*
* Note that this call will fail if the protocol driver specifies an option
* that the underlying controller or its driver does not support. For
* example, not all hardware supports wire transfers using nine bit words,
* LSB-first wire encoding, or active-high chipselects.
*
* Return: zero on success, else a negative error code.
*/
int spi_setup(struct spi_device *spi)
{
unsigned bad_bits, ugly_bits;
int status = 0;
/*
* Check mode to prevent that any two of DUAL, QUAD and NO_MOSI/MISO
* are set at the same time.
*/
if ((hweight_long(spi->mode &
(SPI_TX_DUAL | SPI_TX_QUAD | SPI_NO_TX)) > 1) ||
(hweight_long(spi->mode &
(SPI_RX_DUAL | SPI_RX_QUAD | SPI_NO_RX)) > 1)) {
dev_err(&spi->dev,
"setup: can not select any two of dual, quad and no-rx/tx at the same time\n");
return -EINVAL;
}
/* If it is SPI_3WIRE mode, DUAL and QUAD should be forbidden */
if ((spi->mode & SPI_3WIRE) && (spi->mode &
(SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL |
SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL)))
return -EINVAL;
/*
* Help drivers fail *cleanly* when they need options
* that aren't supported with their current controller.
* SPI_CS_WORD has a fallback software implementation,
* so it is ignored here.
*/
bad_bits = spi->mode & ~(spi->controller->mode_bits | SPI_CS_WORD |
SPI_NO_TX | SPI_NO_RX);
ugly_bits = bad_bits &
(SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL |
SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL);
if (ugly_bits) {
dev_warn(&spi->dev,
"setup: ignoring unsupported mode bits %x\n",
ugly_bits);
spi->mode &= ~ugly_bits;
bad_bits &= ~ugly_bits;
}
if (bad_bits) {
dev_err(&spi->dev, "setup: unsupported mode bits %x\n",
bad_bits);
return -EINVAL;
}
if (!spi->bits_per_word) {
spi->bits_per_word = 8;
} else {
/*
* Some controllers may not support the default 8 bits-per-word
* so only perform the check when this is explicitly provided.
*/
status = __spi_validate_bits_per_word(spi->controller,
spi->bits_per_word);
if (status)
return status;
}
if (spi->controller->max_speed_hz &&
(!spi->max_speed_hz ||
spi->max_speed_hz > spi->controller->max_speed_hz))
spi->max_speed_hz = spi->controller->max_speed_hz;
mutex_lock(&spi->controller->io_mutex);
if (spi->controller->setup) {
status = spi->controller->setup(spi);
if (status) {
mutex_unlock(&spi->controller->io_mutex);
dev_err(&spi->controller->dev, "Failed to setup device: %d\n",
status);
return status;
}
}
status = spi_set_cs_timing(spi);
if (status) {
mutex_unlock(&spi->controller->io_mutex);
return status;
}
if (spi->controller->auto_runtime_pm && spi->controller->set_cs) {
status = pm_runtime_resume_and_get(spi->controller->dev.parent);
if (status < 0) {
mutex_unlock(&spi->controller->io_mutex);
dev_err(&spi->controller->dev, "Failed to power device: %d\n",
status);
return status;
}
/*
* We do not want to return positive value from pm_runtime_get,
* there are many instances of devices calling spi_setup() and
* checking for a non-zero return value instead of a negative
* return value.
*/
status = 0;
spi_set_cs(spi, false, true);
pm_runtime_mark_last_busy(spi->controller->dev.parent);
pm_runtime_put_autosuspend(spi->controller->dev.parent);
} else {
spi_set_cs(spi, false, true);
}
mutex_unlock(&spi->controller->io_mutex);
if (spi->rt && !spi->controller->rt) {
spi->controller->rt = true;
spi_set_thread_rt(spi->controller);
}
trace_spi_setup(spi, status);
dev_dbg(&spi->dev, "setup mode %lu, %s%s%s%s%u bits/w, %u Hz max --> %d\n",
spi->mode & SPI_MODE_X_MASK,
(spi->mode & SPI_CS_HIGH) ? "cs_high, " : "",
(spi->mode & SPI_LSB_FIRST) ? "lsb, " : "",
(spi->mode & SPI_3WIRE) ? "3wire, " : "",
(spi->mode & SPI_LOOP) ? "loopback, " : "",
spi->bits_per_word, spi->max_speed_hz,
status);
return status;
}
EXPORT_SYMBOL_GPL(spi_setup);
static int _spi_xfer_word_delay_update(struct spi_transfer *xfer,
struct spi_device *spi)
{
int delay1, delay2;
delay1 = spi_delay_to_ns(&xfer->word_delay, xfer);
if (delay1 < 0)
return delay1;
delay2 = spi_delay_to_ns(&spi->word_delay, xfer);
if (delay2 < 0)
return delay2;
if (delay1 < delay2)
memcpy(&xfer->word_delay, &spi->word_delay,
sizeof(xfer->word_delay));
return 0;
}
static int __spi_validate(struct spi_device *spi, struct spi_message *message)
{
struct spi_controller *ctlr = spi->controller;
struct spi_transfer *xfer;
int w_size;
if (list_empty(&message->transfers))
return -EINVAL;
message->spi = spi;
/*
* Half-duplex links include original MicroWire, and ones with
* only one data pin like SPI_3WIRE (switches direction) or where
* either MOSI or MISO is missing. They can also be caused by
* software limitations.
*/
if ((ctlr->flags & SPI_CONTROLLER_HALF_DUPLEX) ||
(spi->mode & SPI_3WIRE)) {
unsigned flags = ctlr->flags;
list_for_each_entry(xfer, &message->transfers, transfer_list) {
if (xfer->rx_buf && xfer->tx_buf)
return -EINVAL;
if ((flags & SPI_CONTROLLER_NO_TX) && xfer->tx_buf)
return -EINVAL;
if ((flags & SPI_CONTROLLER_NO_RX) && xfer->rx_buf)
return -EINVAL;
}
}
/*
* Set transfer bits_per_word and max speed as spi device default if
* it is not set for this transfer.
* Set transfer tx_nbits and rx_nbits as single transfer default
* (SPI_NBITS_SINGLE) if it is not set for this transfer.
* Ensure transfer word_delay is at least as long as that required by
* device itself.
*/
message->frame_length = 0;
list_for_each_entry(xfer, &message->transfers, transfer_list) {
xfer->effective_speed_hz = 0;
message->frame_length += xfer->len;
if (!xfer->bits_per_word)
xfer->bits_per_word = spi->bits_per_word;
if (!xfer->speed_hz)
xfer->speed_hz = spi->max_speed_hz;
if (ctlr->max_speed_hz && xfer->speed_hz > ctlr->max_speed_hz)
xfer->speed_hz = ctlr->max_speed_hz;
if (__spi_validate_bits_per_word(ctlr, xfer->bits_per_word))
return -EINVAL;
/*
* SPI transfer length should be multiple of SPI word size
* where SPI word size should be power-of-two multiple.
*/
if (xfer->bits_per_word <= 8)
w_size = 1;
else if (xfer->bits_per_word <= 16)
w_size = 2;
else
w_size = 4;
/* No partial transfers accepted */
if (xfer->len % w_size)
return -EINVAL;
if (xfer->speed_hz && ctlr->min_speed_hz &&
xfer->speed_hz < ctlr->min_speed_hz)
return -EINVAL;
if (xfer->tx_buf && !xfer->tx_nbits)
xfer->tx_nbits = SPI_NBITS_SINGLE;
if (xfer->rx_buf && !xfer->rx_nbits)
xfer->rx_nbits = SPI_NBITS_SINGLE;
/*
* Check transfer tx/rx_nbits:
* 1. check the value matches one of single, dual and quad
* 2. check tx/rx_nbits match the mode in spi_device
*/
if (xfer->tx_buf) {
if (spi->mode & SPI_NO_TX)
return -EINVAL;
if (xfer->tx_nbits != SPI_NBITS_SINGLE &&
xfer->tx_nbits != SPI_NBITS_DUAL &&
xfer->tx_nbits != SPI_NBITS_QUAD)
return -EINVAL;
if ((xfer->tx_nbits == SPI_NBITS_DUAL) &&
!(spi->mode & (SPI_TX_DUAL | SPI_TX_QUAD)))
return -EINVAL;
if ((xfer->tx_nbits == SPI_NBITS_QUAD) &&
!(spi->mode & SPI_TX_QUAD))
return -EINVAL;
}
/* Check transfer rx_nbits */
if (xfer->rx_buf) {
if (spi->mode & SPI_NO_RX)
return -EINVAL;
if (xfer->rx_nbits != SPI_NBITS_SINGLE &&
xfer->rx_nbits != SPI_NBITS_DUAL &&
xfer->rx_nbits != SPI_NBITS_QUAD)
return -EINVAL;
if ((xfer->rx_nbits == SPI_NBITS_DUAL) &&
!(spi->mode & (SPI_RX_DUAL | SPI_RX_QUAD)))
return -EINVAL;
if ((xfer->rx_nbits == SPI_NBITS_QUAD) &&
!(spi->mode & SPI_RX_QUAD))
return -EINVAL;
}
if (_spi_xfer_word_delay_update(xfer, spi))
return -EINVAL;
}
message->status = -EINPROGRESS;
return 0;
}
/*
* spi_split_transfers - generic handling of transfer splitting
* @msg: the message to split
*
* Under certain conditions, a SPI controller may not support arbitrary
* transfer sizes or other features required by a peripheral. This function
* will split the transfers in the message into smaller transfers that are
* supported by the controller.
*
* Controllers with special requirements not covered here can also split
* transfers in the optimize_message() callback.
*
* Context: can sleep
* Return: zero on success, else a negative error code
*/
static int spi_split_transfers(struct spi_message *msg)
{
struct spi_controller *ctlr = msg->spi->controller;
struct spi_transfer *xfer;
int ret;
/*
* If an SPI controller does not support toggling the CS line on each
* transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO
* for the CS line, we can emulate the CS-per-word hardware function by
* splitting transfers into one-word transfers and ensuring that
* cs_change is set for each transfer.
*/
if ((msg->spi->mode & SPI_CS_WORD) &&
(!(ctlr->mode_bits & SPI_CS_WORD) || spi_is_csgpiod(msg->spi))) {
ret = spi_split_transfers_maxwords(ctlr, msg, 1);
if (ret)
return ret;
list_for_each_entry(xfer, &msg->transfers, transfer_list) {
/* Don't change cs_change on the last entry in the list */
if (list_is_last(&xfer->transfer_list, &msg->transfers))
break;
xfer->cs_change = 1;
}
} else {
ret = spi_split_transfers_maxsize(ctlr, msg,
spi_max_transfer_size(msg->spi));
if (ret)
return ret;
}
return 0;
}
/*
* __spi_optimize_message - shared implementation for spi_optimize_message()
* and spi_maybe_optimize_message()
* @spi: the device that will be used for the message
* @msg: the message to optimize
*
* Peripheral drivers will call spi_optimize_message() and the spi core will
* call spi_maybe_optimize_message() instead of calling this directly.
*
* It is not valid to call this on a message that has already been optimized.
*
* Return: zero on success, else a negative error code
*/
static int __spi_optimize_message(struct spi_device *spi,
struct spi_message *msg)
{
struct spi_controller *ctlr = spi->controller;
int ret;
ret = __spi_validate(spi, msg);
if (ret)
return ret;
ret = spi_split_transfers(msg);
if (ret)
return ret;
if (ctlr->optimize_message) {
ret = ctlr->optimize_message(msg);
if (ret) {
spi_res_release(ctlr, msg);
return ret;
}
}
msg->optimized = true;
return 0;
}
/*
* spi_maybe_optimize_message - optimize message if it isn't already pre-optimized
* @spi: the device that will be used for the message
* @msg: the message to optimize
* Return: zero on success, else a negative error code
*/
static int spi_maybe_optimize_message(struct spi_device *spi,
struct spi_message *msg)
{
if (msg->pre_optimized)
return 0;
return __spi_optimize_message(spi, msg);
}
/**
* spi_optimize_message - do any one-time validation and setup for a SPI message
* @spi: the device that will be used for the message
* @msg: the message to optimize
*
* Peripheral drivers that reuse the same message repeatedly may call this to
* perform as much message prep as possible once, rather than repeating it each
* time a message transfer is performed to improve throughput and reduce CPU
* usage.
*
* Once a message has been optimized, it cannot be modified with the exception
* of updating the contents of any xfer->tx_buf (the pointer can't be changed,
* only the data in the memory it points to).
*
* Calls to this function must be balanced with calls to spi_unoptimize_message()
* to avoid leaking resources.
*
* Context: can sleep
* Return: zero on success, else a negative error code
*/
int spi_optimize_message(struct spi_device *spi, struct spi_message *msg)
{
int ret;
ret = __spi_optimize_message(spi, msg);
if (ret)
return ret;
/*
* This flag indicates that the peripheral driver called spi_optimize_message()
* and therefore we shouldn't unoptimize message automatically when finalizing
* the message but rather wait until spi_unoptimize_message() is called
* by the peripheral driver.
*/
msg->pre_optimized = true;
return 0;
}
EXPORT_SYMBOL_GPL(spi_optimize_message);
/**
* spi_unoptimize_message - releases any resources allocated by spi_optimize_message()
* @msg: the message to unoptimize
*
* Calls to this function must be balanced with calls to spi_optimize_message().
*
* Context: can sleep
*/
void spi_unoptimize_message(struct spi_message *msg)
{
__spi_unoptimize_message(msg);
msg->pre_optimized = false;
}
EXPORT_SYMBOL_GPL(spi_unoptimize_message);
static int __spi_async(struct spi_device *spi, struct spi_message *message)
{
struct spi_controller *ctlr = spi->controller;
struct spi_transfer *xfer;
/*
* Some controllers do not support doing regular SPI transfers. Return
* ENOTSUPP when this is the case.
*/
if (!ctlr->transfer)
return -ENOTSUPP;
SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_async);
SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_async);
trace_spi_message_submit(message);
if (!ctlr->ptp_sts_supported) {
list_for_each_entry(xfer, &message->transfers, transfer_list) {
xfer->ptp_sts_word_pre = 0;
ptp_read_system_prets(xfer->ptp_sts);
}
}
return ctlr->transfer(spi, message);
}
/**
* spi_async - asynchronous SPI transfer
* @spi: device with which data will be exchanged
* @message: describes the data transfers, including completion callback
* Context: any (IRQs may be blocked, etc)
*
* This call may be used in_irq and other contexts which can't sleep,
* as well as from task contexts which can sleep.
*
* The completion callback is invoked in a context which can't sleep.
* Before that invocation, the value of message->status is undefined.
* When the callback is issued, message->status holds either zero (to
* indicate complete success) or a negative error code. After that
* callback returns, the driver which issued the transfer request may
* deallocate the associated memory; it's no longer in use by any SPI
* core or controller driver code.
*
* Note that although all messages to a spi_device are handled in
* FIFO order, messages may go to different devices in other orders.
* Some device might be higher priority, or have various "hard" access
* time requirements, for example.
*
* On detection of any fault during the transfer, processing of
* the entire message is aborted, and the device is deselected.
* Until returning from the associated message completion callback,
* no other spi_message queued to that device will be processed.
* (This rule applies equally to all the synchronous transfer calls,
* which are wrappers around this core asynchronous primitive.)
*
* Return: zero on success, else a negative error code.
*/
int spi_async(struct spi_device *spi, struct spi_message *message)
{
struct spi_controller *ctlr = spi->controller;
int ret;
unsigned long flags;
ret = spi_maybe_optimize_message(spi, message);
if (ret)
return ret;
spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
if (ctlr->bus_lock_flag)
ret = -EBUSY;
else
ret = __spi_async(spi, message);
spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
spi_maybe_unoptimize_message(message);
return ret;
}
EXPORT_SYMBOL_GPL(spi_async);
static void __spi_transfer_message_noqueue(struct spi_controller *ctlr, struct spi_message *msg)
{
bool was_busy;
int ret;
mutex_lock(&ctlr->io_mutex);
was_busy = ctlr->busy;
ctlr->cur_msg = msg;
ret = __spi_pump_transfer_message(ctlr, msg, was_busy);
if (ret)
dev_err(&ctlr->dev, "noqueue transfer failed\n");
ctlr->cur_msg = NULL;
ctlr->fallback = false;
if (!was_busy) {
kfree(ctlr->dummy_rx);
ctlr->dummy_rx = NULL;
kfree(ctlr->dummy_tx);
ctlr->dummy_tx = NULL;
if (ctlr->unprepare_transfer_hardware &&
ctlr->unprepare_transfer_hardware(ctlr))
dev_err(&ctlr->dev,
"failed to unprepare transfer hardware\n");
spi_idle_runtime_pm(ctlr);
}
mutex_unlock(&ctlr->io_mutex);
}
/*-------------------------------------------------------------------------*/
/*
* Utility methods for SPI protocol drivers, layered on
* top of the core. Some other utility methods are defined as
* inline functions.
*/
static void spi_complete(void *arg)
{
complete(arg);
}
static int __spi_sync(struct spi_device *spi, struct spi_message *message)
{
DECLARE_COMPLETION_ONSTACK(done);
unsigned long flags;
int status;
struct spi_controller *ctlr = spi->controller;
if (__spi_check_suspended(ctlr)) {
dev_warn_once(&spi->dev, "Attempted to sync while suspend\n");
return -ESHUTDOWN;
}
status = spi_maybe_optimize_message(spi, message);
if (status)
return status;
SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync);
SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync);
/*
* Checking queue_empty here only guarantees async/sync message
* ordering when coming from the same context. It does not need to
* guard against reentrancy from a different context. The io_mutex
* will catch those cases.
*/
if (READ_ONCE(ctlr->queue_empty) && !ctlr->must_async) {
message->actual_length = 0;
message->status = -EINPROGRESS;
trace_spi_message_submit(message);
SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync_immediate);
SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync_immediate);
__spi_transfer_message_noqueue(ctlr, message);
return message->status;
}
/*
* There are messages in the async queue that could have originated
* from the same context, so we need to preserve ordering.
* Therefor we send the message to the async queue and wait until they
* are completed.
*/
message->complete = spi_complete;
message->context = &done;
spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
status = __spi_async(spi, message);
spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
if (status == 0) {
wait_for_completion(&done);
status = message->status;
}
message->complete = NULL;
message->context = NULL;
return status;
}
/**
* spi_sync - blocking/synchronous SPI data transfers
* @spi: device with which data will be exchanged
* @message: describes the data transfers
* Context: can sleep
*
* This call may only be used from a context that may sleep. The sleep
* is non-interruptible, and has no timeout. Low-overhead controller
* drivers may DMA directly into and out of the message buffers.
*
* Note that the SPI device's chip select is active during the message,
* and then is normally disabled between messages. Drivers for some
* frequently-used devices may want to minimize costs of selecting a chip,
* by leaving it selected in anticipation that the next message will go
* to the same chip. (That may increase power usage.)
*
* Also, the caller is guaranteeing that the memory associated with the
* message will not be freed before this call returns.
*
* Return: zero on success, else a negative error code.
*/
int spi_sync(struct spi_device *spi, struct spi_message *message)
{
int ret;
mutex_lock(&spi->controller->bus_lock_mutex);
ret = __spi_sync(spi, message);
mutex_unlock(&spi->controller->bus_lock_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(spi_sync);
/**
* spi_sync_locked - version of spi_sync with exclusive bus usage
* @spi: device with which data will be exchanged
* @message: describes the data transfers
* Context: can sleep
*
* This call may only be used from a context that may sleep. The sleep
* is non-interruptible, and has no timeout. Low-overhead controller
* drivers may DMA directly into and out of the message buffers.
*
* This call should be used by drivers that require exclusive access to the
* SPI bus. It has to be preceded by a spi_bus_lock call. The SPI bus must
* be released by a spi_bus_unlock call when the exclusive access is over.
*
* Return: zero on success, else a negative error code.
*/
int spi_sync_locked(struct spi_device *spi, struct spi_message *message)
{
return __spi_sync(spi, message);
}
EXPORT_SYMBOL_GPL(spi_sync_locked);
/**
* spi_bus_lock - obtain a lock for exclusive SPI bus usage
* @ctlr: SPI bus master that should be locked for exclusive bus access
* Context: can sleep
*
* This call may only be used from a context that may sleep. The sleep
* is non-interruptible, and has no timeout.
*
* This call should be used by drivers that require exclusive access to the
* SPI bus. The SPI bus must be released by a spi_bus_unlock call when the
* exclusive access is over. Data transfer must be done by spi_sync_locked
* and spi_async_locked calls when the SPI bus lock is held.
*
* Return: always zero.
*/
int spi_bus_lock(struct spi_controller *ctlr)
{
unsigned long flags;
mutex_lock(&ctlr->bus_lock_mutex);
spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
ctlr->bus_lock_flag = 1;
spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
/* Mutex remains locked until spi_bus_unlock() is called */
return 0;
}
EXPORT_SYMBOL_GPL(spi_bus_lock);
/**
* spi_bus_unlock - release the lock for exclusive SPI bus usage
* @ctlr: SPI bus master that was locked for exclusive bus access
* Context: can sleep
*
* This call may only be used from a context that may sleep. The sleep
* is non-interruptible, and has no timeout.
*
* This call releases an SPI bus lock previously obtained by an spi_bus_lock
* call.
*
* Return: always zero.
*/
int spi_bus_unlock(struct spi_controller *ctlr)
{
ctlr->bus_lock_flag = 0;
mutex_unlock(&ctlr->bus_lock_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(spi_bus_unlock);
/* Portable code must never pass more than 32 bytes */
#define SPI_BUFSIZ max(32, SMP_CACHE_BYTES)
static u8 *buf;
/**
* spi_write_then_read - SPI synchronous write followed by read
* @spi: device with which data will be exchanged
* @txbuf: data to be written (need not be DMA-safe)
* @n_tx: size of txbuf, in bytes
* @rxbuf: buffer into which data will be read (need not be DMA-safe)
* @n_rx: size of rxbuf, in bytes
* Context: can sleep
*
* This performs a half duplex MicroWire style transaction with the
* device, sending txbuf and then reading rxbuf. The return value
* is zero for success, else a negative errno status code.
* This call may only be used from a context that may sleep.
*
* Parameters to this routine are always copied using a small buffer.
* Performance-sensitive or bulk transfer code should instead use
* spi_{async,sync}() calls with DMA-safe buffers.
*
* Return: zero on success, else a negative error code.
*/
int spi_write_then_read(struct spi_device *spi,
const void *txbuf, unsigned n_tx,
void *rxbuf, unsigned n_rx)
{
static DEFINE_MUTEX(lock);
int status;
struct spi_message message;
struct spi_transfer x[2];
u8 *local_buf;
/*
* Use preallocated DMA-safe buffer if we can. We can't avoid
* copying here, (as a pure convenience thing), but we can
* keep heap costs out of the hot path unless someone else is
* using the pre-allocated buffer or the transfer is too large.
*/
if ((n_tx + n_rx) > SPI_BUFSIZ || !mutex_trylock(&lock)) {
local_buf = kmalloc(max((unsigned)SPI_BUFSIZ, n_tx + n_rx),
GFP_KERNEL | GFP_DMA);
if (!local_buf)
return -ENOMEM;
} else {
local_buf = buf;
}
spi_message_init(&message);
memset(x, 0, sizeof(x));
if (n_tx) {
x[0].len = n_tx;
spi_message_add_tail(&x[0], &message);
}
if (n_rx) {
x[1].len = n_rx;
spi_message_add_tail(&x[1], &message);
}
memcpy(local_buf, txbuf, n_tx);
x[0].tx_buf = local_buf;
x[1].rx_buf = local_buf + n_tx;
/* Do the I/O */
status = spi_sync(spi, &message);
if (status == 0)
memcpy(rxbuf, x[1].rx_buf, n_rx);
if (x[0].tx_buf == buf)
mutex_unlock(&lock);
else
kfree(local_buf);
return status;
}
EXPORT_SYMBOL_GPL(spi_write_then_read);
/*-------------------------------------------------------------------------*/
#if IS_ENABLED(CONFIG_OF_DYNAMIC)
/* Must call put_device() when done with returned spi_device device */
static struct spi_device *of_find_spi_device_by_node(struct device_node *node)
{
struct device *dev = bus_find_device_by_of_node(&spi_bus_type, node);
return dev ? to_spi_device(dev) : NULL;
}
/* The spi controllers are not using spi_bus, so we find it with another way */
static struct spi_controller *of_find_spi_controller_by_node(struct device_node *node)
{
struct device *dev;
dev = class_find_device_by_of_node(&spi_master_class, node);
if (!dev && IS_ENABLED(CONFIG_SPI_SLAVE))
dev = class_find_device_by_of_node(&spi_slave_class, node);
if (!dev)
return NULL;
/* Reference got in class_find_device */
return container_of(dev, struct spi_controller, dev);
}
static int of_spi_notify(struct notifier_block *nb, unsigned long action,
void *arg)
{
struct of_reconfig_data *rd = arg;
struct spi_controller *ctlr;
struct spi_device *spi;
switch (of_reconfig_get_state_change(action, arg)) {
case OF_RECONFIG_CHANGE_ADD:
ctlr = of_find_spi_controller_by_node(rd->dn->parent);
if (ctlr == NULL)
return NOTIFY_OK; /* Not for us */
if (of_node_test_and_set_flag(rd->dn, OF_POPULATED)) {
put_device(&ctlr->dev);
return NOTIFY_OK;
}
/*
* Clear the flag before adding the device so that fw_devlink
* doesn't skip adding consumers to this device.
*/
rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE;
spi = of_register_spi_device(ctlr, rd->dn);
put_device(&ctlr->dev);
if (IS_ERR(spi)) {
pr_err("%s: failed to create for '%pOF'\n",
__func__, rd->dn);
of_node_clear_flag(rd->dn, OF_POPULATED);
return notifier_from_errno(PTR_ERR(spi));
}
break;
case OF_RECONFIG_CHANGE_REMOVE:
/* Already depopulated? */
if (!of_node_check_flag(rd->dn, OF_POPULATED))
return NOTIFY_OK;
/* Find our device by node */
spi = of_find_spi_device_by_node(rd->dn);
if (spi == NULL)
return NOTIFY_OK; /* No? not meant for us */
/* Unregister takes one ref away */
spi_unregister_device(spi);
/* And put the reference of the find */
put_device(&spi->dev);
break;
}
return NOTIFY_OK;
}
static struct notifier_block spi_of_notifier = {
.notifier_call = of_spi_notify,
};
#else /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
extern struct notifier_block spi_of_notifier;
#endif /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
#if IS_ENABLED(CONFIG_ACPI)
static int spi_acpi_controller_match(struct device *dev, const void *data)
{
return ACPI_COMPANION(dev->parent) == data;
}
struct spi_controller *acpi_spi_find_controller_by_adev(struct acpi_device *adev)
{
struct device *dev;
dev = class_find_device(&spi_master_class, NULL, adev,
spi_acpi_controller_match);
if (!dev && IS_ENABLED(CONFIG_SPI_SLAVE))
dev = class_find_device(&spi_slave_class, NULL, adev,
spi_acpi_controller_match);
if (!dev)
return NULL;
return container_of(dev, struct spi_controller, dev);
}
EXPORT_SYMBOL_GPL(acpi_spi_find_controller_by_adev);
static struct spi_device *acpi_spi_find_device_by_adev(struct acpi_device *adev)
{
struct device *dev;
dev = bus_find_device_by_acpi_dev(&spi_bus_type, adev);
return to_spi_device(dev);
}
static int acpi_spi_notify(struct notifier_block *nb, unsigned long value,
void *arg)
{
struct acpi_device *adev = arg;
struct spi_controller *ctlr;
struct spi_device *spi;
switch (value) {
case ACPI_RECONFIG_DEVICE_ADD:
ctlr = acpi_spi_find_controller_by_adev(acpi_dev_parent(adev));
if (!ctlr)
break;
acpi_register_spi_device(ctlr, adev);
put_device(&ctlr->dev);
break;
case ACPI_RECONFIG_DEVICE_REMOVE:
if (!acpi_device_enumerated(adev))
break;
spi = acpi_spi_find_device_by_adev(adev);
if (!spi)
break;
spi_unregister_device(spi);
put_device(&spi->dev);
break;
}
return NOTIFY_OK;
}
static struct notifier_block spi_acpi_notifier = {
.notifier_call = acpi_spi_notify,
};
#else
extern struct notifier_block spi_acpi_notifier;
#endif
static int __init spi_init(void)
{
int status;
buf = kmalloc(SPI_BUFSIZ, GFP_KERNEL);
if (!buf) {
status = -ENOMEM;
goto err0;
}
status = bus_register(&spi_bus_type);
if (status < 0)
goto err1;
status = class_register(&spi_master_class);
if (status < 0)
goto err2;
if (IS_ENABLED(CONFIG_SPI_SLAVE)) {
status = class_register(&spi_slave_class);
if (status < 0)
goto err3;
}
if (IS_ENABLED(CONFIG_OF_DYNAMIC))
WARN_ON(of_reconfig_notifier_register(&spi_of_notifier));
if (IS_ENABLED(CONFIG_ACPI))
WARN_ON(acpi_reconfig_notifier_register(&spi_acpi_notifier));
return 0;
err3:
class_unregister(&spi_master_class);
err2:
bus_unregister(&spi_bus_type);
err1:
kfree(buf);
buf = NULL;
err0:
return status;
}
/*
* A board_info is normally registered in arch_initcall(),
* but even essential drivers wait till later.
*
* REVISIT only boardinfo really needs static linking. The rest (device and
* driver registration) _could_ be dynamically linked (modular) ... Costs
* include needing to have boardinfo data structures be much more public.
*/
postcore_initcall(spi_init);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Digital Audio (PCM) abstract layer
* Copyright (c) by Jaroslav Kysela <perex@perex.cz>
*/
#include <linux/io.h>
#include <linux/time.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/moduleparam.h>
#include <linux/vmalloc.h>
#include <linux/export.h>
#include <sound/core.h>
#include <sound/pcm.h>
#include <sound/info.h>
#include <sound/initval.h>
#include "pcm_local.h"
static int preallocate_dma = 1;
module_param(preallocate_dma, int, 0444);
MODULE_PARM_DESC(preallocate_dma, "Preallocate DMA memory when the PCM devices are initialized.");
static int maximum_substreams = 4;
module_param(maximum_substreams, int, 0444);
MODULE_PARM_DESC(maximum_substreams, "Maximum substreams with preallocated DMA memory.");
static const size_t snd_minimum_buffer = 16384;
static unsigned long max_alloc_per_card = 32UL * 1024UL * 1024UL;
module_param(max_alloc_per_card, ulong, 0644);
MODULE_PARM_DESC(max_alloc_per_card, "Max total allocation bytes per card.");
static void __update_allocated_size(struct snd_card *card, ssize_t bytes)
{
card->total_pcm_alloc_bytes += bytes;
}
static void update_allocated_size(struct snd_card *card, ssize_t bytes)
{
guard(mutex)(&card->memory_mutex);
__update_allocated_size(card, bytes);
}
static void decrease_allocated_size(struct snd_card *card, size_t bytes)
{
guard(mutex)(&card->memory_mutex);
WARN_ON(card->total_pcm_alloc_bytes < bytes);
__update_allocated_size(card, -(ssize_t)bytes);
}
static int do_alloc_pages(struct snd_card *card, int type, struct device *dev,
int str, size_t size, struct snd_dma_buffer *dmab)
{
enum dma_data_direction dir;
int err;
/* check and reserve the requested size */
scoped_guard(mutex, &card->memory_mutex) {
if (max_alloc_per_card &&
card->total_pcm_alloc_bytes + size > max_alloc_per_card)
return -ENOMEM;
__update_allocated_size(card, size);
}
if (str == SNDRV_PCM_STREAM_PLAYBACK)
dir = DMA_TO_DEVICE;
else
dir = DMA_FROM_DEVICE;
err = snd_dma_alloc_dir_pages(type, dev, dir, size, dmab);
if (!err) {
/* the actual allocation size might be bigger than requested,
* and we need to correct the account
*/
if (dmab->bytes != size)
update_allocated_size(card, dmab->bytes - size);
} else {
/* take back on allocation failure */
decrease_allocated_size(card, size);
}
return err;
}
static void do_free_pages(struct snd_card *card, struct snd_dma_buffer *dmab)
{
if (!dmab->area)
return;
decrease_allocated_size(card, dmab->bytes);
snd_dma_free_pages(dmab);
dmab->area = NULL;
}
/*
* try to allocate as the large pages as possible.
* stores the resultant memory size in *res_size.
*
* the minimum size is snd_minimum_buffer. it should be power of 2.
*/
static int preallocate_pcm_pages(struct snd_pcm_substream *substream,
size_t size, bool no_fallback)
{
struct snd_dma_buffer *dmab = &substream->dma_buffer;
struct snd_card *card = substream->pcm->card;
size_t orig_size = size;
int err;
do {
err = do_alloc_pages(card, dmab->dev.type, dmab->dev.dev,
substream->stream, size, dmab);
if (err != -ENOMEM)
return err;
if (no_fallback)
break;
size >>= 1;
} while (size >= snd_minimum_buffer);
dmab->bytes = 0; /* tell error */
pr_warn("ALSA pcmC%dD%d%c,%d:%s: cannot preallocate for size %zu\n",
substream->pcm->card->number, substream->pcm->device,
substream->stream ? 'c' : 'p', substream->number,
substream->pcm->name, orig_size);
return -ENOMEM;
}
/**
* snd_pcm_lib_preallocate_free - release the preallocated buffer of the specified substream.
* @substream: the pcm substream instance
*
* Releases the pre-allocated buffer of the given substream.
*/
void snd_pcm_lib_preallocate_free(struct snd_pcm_substream *substream)
{
do_free_pages(substream->pcm->card, &substream->dma_buffer);
}
/**
* snd_pcm_lib_preallocate_free_for_all - release all pre-allocated buffers on the pcm
* @pcm: the pcm instance
*
* Releases all the pre-allocated buffers on the given pcm.
*/
void snd_pcm_lib_preallocate_free_for_all(struct snd_pcm *pcm)
{
struct snd_pcm_substream *substream;
int stream;
for_each_pcm_substream(pcm, stream, substream) snd_pcm_lib_preallocate_free(substream);
}
EXPORT_SYMBOL(snd_pcm_lib_preallocate_free_for_all);
#ifdef CONFIG_SND_VERBOSE_PROCFS
/*
* read callback for prealloc proc file
*
* prints the current allocated size in kB.
*/
static void snd_pcm_lib_preallocate_proc_read(struct snd_info_entry *entry,
struct snd_info_buffer *buffer)
{
struct snd_pcm_substream *substream = entry->private_data;
snd_iprintf(buffer, "%lu\n", (unsigned long) substream->dma_buffer.bytes / 1024);
}
/*
* read callback for prealloc_max proc file
*
* prints the maximum allowed size in kB.
*/
static void snd_pcm_lib_preallocate_max_proc_read(struct snd_info_entry *entry,
struct snd_info_buffer *buffer)
{
struct snd_pcm_substream *substream = entry->private_data;
snd_iprintf(buffer, "%lu\n", (unsigned long) substream->dma_max / 1024);
}
/*
* write callback for prealloc proc file
*
* accepts the preallocation size in kB.
*/
static void snd_pcm_lib_preallocate_proc_write(struct snd_info_entry *entry,
struct snd_info_buffer *buffer)
{
struct snd_pcm_substream *substream = entry->private_data;
struct snd_card *card = substream->pcm->card;
char line[64], str[64];
size_t size;
struct snd_dma_buffer new_dmab;
guard(mutex)(&substream->pcm->open_mutex);
if (substream->runtime) {
buffer->error = -EBUSY;
return;
}
if (!snd_info_get_line(buffer, line, sizeof(line))) {
snd_info_get_str(str, line, sizeof(str));
size = simple_strtoul(str, NULL, 10) * 1024;
if ((size != 0 && size < 8192) || size > substream->dma_max) {
buffer->error = -EINVAL;
return;
}
if (substream->dma_buffer.bytes == size)
return;
memset(&new_dmab, 0, sizeof(new_dmab));
new_dmab.dev = substream->dma_buffer.dev;
if (size > 0) {
if (do_alloc_pages(card,
substream->dma_buffer.dev.type,
substream->dma_buffer.dev.dev,
substream->stream,
size, &new_dmab) < 0) {
buffer->error = -ENOMEM;
pr_debug("ALSA pcmC%dD%d%c,%d:%s: cannot preallocate for size %zu\n",
substream->pcm->card->number, substream->pcm->device,
substream->stream ? 'c' : 'p', substream->number,
substream->pcm->name, size);
return;
}
substream->buffer_bytes_max = size;
} else {
substream->buffer_bytes_max = UINT_MAX;
}
if (substream->dma_buffer.area)
do_free_pages(card, &substream->dma_buffer);
substream->dma_buffer = new_dmab;
} else {
buffer->error = -EINVAL;
}
}
static inline void preallocate_info_init(struct snd_pcm_substream *substream)
{
struct snd_info_entry *entry;
entry = snd_info_create_card_entry(substream->pcm->card, "prealloc",
substream->proc_root);
if (entry) {
snd_info_set_text_ops(entry, substream,
snd_pcm_lib_preallocate_proc_read);
entry->c.text.write = snd_pcm_lib_preallocate_proc_write;
entry->mode |= 0200;
}
entry = snd_info_create_card_entry(substream->pcm->card, "prealloc_max",
substream->proc_root);
if (entry)
snd_info_set_text_ops(entry, substream,
snd_pcm_lib_preallocate_max_proc_read);
}
#else /* !CONFIG_SND_VERBOSE_PROCFS */
static inline void preallocate_info_init(struct snd_pcm_substream *substream)
{
}
#endif /* CONFIG_SND_VERBOSE_PROCFS */
/*
* pre-allocate the buffer and create a proc file for the substream
*/
static int preallocate_pages(struct snd_pcm_substream *substream,
int type, struct device *data,
size_t size, size_t max, bool managed)
{
int err;
if (snd_BUG_ON(substream->dma_buffer.dev.type))
return -EINVAL;
substream->dma_buffer.dev.type = type;
substream->dma_buffer.dev.dev = data;
if (size > 0) {
if (!max) {
/* no fallback, only also inform -ENOMEM */
err = preallocate_pcm_pages(substream, size, true);
if (err < 0)
return err;
} else if (preallocate_dma &&
substream->number < maximum_substreams) {
err = preallocate_pcm_pages(substream, size, false);
if (err < 0 && err != -ENOMEM)
return err;
}
}
if (substream->dma_buffer.bytes > 0)
substream->buffer_bytes_max = substream->dma_buffer.bytes;
substream->dma_max = max;
if (max > 0)
preallocate_info_init(substream);
if (managed)
substream->managed_buffer_alloc = 1;
return 0;
}
static int preallocate_pages_for_all(struct snd_pcm *pcm, int type,
void *data, size_t size, size_t max,
bool managed)
{
struct snd_pcm_substream *substream;
int stream, err;
for_each_pcm_substream(pcm, stream, substream) {
err = preallocate_pages(substream, type, data, size, max, managed);
if (err < 0)
return err;
}
return 0;
}
/**
* snd_pcm_lib_preallocate_pages - pre-allocation for the given DMA type
* @substream: the pcm substream instance
* @type: DMA type (SNDRV_DMA_TYPE_*)
* @data: DMA type dependent data
* @size: the requested pre-allocation size in bytes
* @max: the max. allowed pre-allocation size
*
* Do pre-allocation for the given DMA buffer type.
*/
void snd_pcm_lib_preallocate_pages(struct snd_pcm_substream *substream,
int type, struct device *data,
size_t size, size_t max)
{
preallocate_pages(substream, type, data, size, max, false);
}
EXPORT_SYMBOL(snd_pcm_lib_preallocate_pages);
/**
* snd_pcm_lib_preallocate_pages_for_all - pre-allocation for continuous memory type (all substreams)
* @pcm: the pcm instance
* @type: DMA type (SNDRV_DMA_TYPE_*)
* @data: DMA type dependent data
* @size: the requested pre-allocation size in bytes
* @max: the max. allowed pre-allocation size
*
* Do pre-allocation to all substreams of the given pcm for the
* specified DMA type.
*/
void snd_pcm_lib_preallocate_pages_for_all(struct snd_pcm *pcm,
int type, void *data,
size_t size, size_t max)
{
preallocate_pages_for_all(pcm, type, data, size, max, false);
}
EXPORT_SYMBOL(snd_pcm_lib_preallocate_pages_for_all);
/**
* snd_pcm_set_managed_buffer - set up buffer management for a substream
* @substream: the pcm substream instance
* @type: DMA type (SNDRV_DMA_TYPE_*)
* @data: DMA type dependent data
* @size: the requested pre-allocation size in bytes
* @max: the max. allowed pre-allocation size
*
* Do pre-allocation for the given DMA buffer type, and set the managed
* buffer allocation mode to the given substream.
* In this mode, PCM core will allocate a buffer automatically before PCM
* hw_params ops call, and release the buffer after PCM hw_free ops call
* as well, so that the driver doesn't need to invoke the allocation and
* the release explicitly in its callback.
* When a buffer is actually allocated before the PCM hw_params call, it
* turns on the runtime buffer_changed flag for drivers changing their h/w
* parameters accordingly.
*
* When @size is non-zero and @max is zero, this tries to allocate for only
* the exact buffer size without fallback, and may return -ENOMEM.
* Otherwise, the function tries to allocate smaller chunks if the allocation
* fails. This is the behavior of snd_pcm_set_fixed_buffer().
*
* When both @size and @max are zero, the function only sets up the buffer
* for later dynamic allocations. It's used typically for buffers with
* SNDRV_DMA_TYPE_VMALLOC type.
*
* Upon successful buffer allocation and setup, the function returns 0.
*
* Return: zero if successful, or a negative error code
*/
int snd_pcm_set_managed_buffer(struct snd_pcm_substream *substream, int type,
struct device *data, size_t size, size_t max)
{
return preallocate_pages(substream, type, data, size, max, true);
}
EXPORT_SYMBOL(snd_pcm_set_managed_buffer);
/**
* snd_pcm_set_managed_buffer_all - set up buffer management for all substreams
* for all substreams
* @pcm: the pcm instance
* @type: DMA type (SNDRV_DMA_TYPE_*)
* @data: DMA type dependent data
* @size: the requested pre-allocation size in bytes
* @max: the max. allowed pre-allocation size
*
* Do pre-allocation to all substreams of the given pcm for the specified DMA
* type and size, and set the managed_buffer_alloc flag to each substream.
*
* Return: zero if successful, or a negative error code
*/
int snd_pcm_set_managed_buffer_all(struct snd_pcm *pcm, int type,
struct device *data,
size_t size, size_t max)
{
return preallocate_pages_for_all(pcm, type, data, size, max, true);
}
EXPORT_SYMBOL(snd_pcm_set_managed_buffer_all);
/**
* snd_pcm_lib_malloc_pages - allocate the DMA buffer
* @substream: the substream to allocate the DMA buffer to
* @size: the requested buffer size in bytes
*
* Allocates the DMA buffer on the BUS type given earlier to
* snd_pcm_lib_preallocate_xxx_pages().
*
* Return: 1 if the buffer is changed, 0 if not changed, or a negative
* code on failure.
*/
int snd_pcm_lib_malloc_pages(struct snd_pcm_substream *substream, size_t size)
{
struct snd_card *card;
struct snd_pcm_runtime *runtime;
struct snd_dma_buffer *dmab = NULL;
if (PCM_RUNTIME_CHECK(substream))
return -EINVAL;
if (snd_BUG_ON(substream->dma_buffer.dev.type ==
SNDRV_DMA_TYPE_UNKNOWN))
return -EINVAL;
runtime = substream->runtime;
card = substream->pcm->card;
if (runtime->dma_buffer_p) {
/* perphaps, we might free the large DMA memory region
to save some space here, but the actual solution
costs us less time */
if (runtime->dma_buffer_p->bytes >= size) {
runtime->dma_bytes = size;
return 0; /* ok, do not change */
}
snd_pcm_lib_free_pages(substream);
}
if (substream->dma_buffer.area != NULL &&
substream->dma_buffer.bytes >= size) {
dmab = &substream->dma_buffer; /* use the pre-allocated buffer */
} else {
/* dma_max=0 means the fixed size preallocation */
if (substream->dma_buffer.area && !substream->dma_max)
return -ENOMEM;
dmab = kzalloc(sizeof(*dmab), GFP_KERNEL);
if (! dmab)
return -ENOMEM;
dmab->dev = substream->dma_buffer.dev;
if (do_alloc_pages(card,
substream->dma_buffer.dev.type,
substream->dma_buffer.dev.dev,
substream->stream,
size, dmab) < 0) {
kfree(dmab);
pr_debug("ALSA pcmC%dD%d%c,%d:%s: cannot preallocate for size %zu\n",
substream->pcm->card->number, substream->pcm->device,
substream->stream ? 'c' : 'p', substream->number,
substream->pcm->name, size);
return -ENOMEM;
}
}
snd_pcm_set_runtime_buffer(substream, dmab);
runtime->dma_bytes = size;
return 1; /* area was changed */
}
EXPORT_SYMBOL(snd_pcm_lib_malloc_pages);
/**
* snd_pcm_lib_free_pages - release the allocated DMA buffer.
* @substream: the substream to release the DMA buffer
*
* Releases the DMA buffer allocated via snd_pcm_lib_malloc_pages().
*
* Return: Zero if successful, or a negative error code on failure.
*/
int snd_pcm_lib_free_pages(struct snd_pcm_substream *substream)
{
struct snd_pcm_runtime *runtime;
if (PCM_RUNTIME_CHECK(substream))
return -EINVAL;
runtime = substream->runtime;
if (runtime->dma_area == NULL)
return 0;
if (runtime->dma_buffer_p != &substream->dma_buffer) {
struct snd_card *card = substream->pcm->card;
/* it's a newly allocated buffer. release it now. */
do_free_pages(card, runtime->dma_buffer_p);
kfree(runtime->dma_buffer_p);
}
snd_pcm_set_runtime_buffer(substream, NULL);
return 0;
}
EXPORT_SYMBOL(snd_pcm_lib_free_pages);
int _snd_pcm_lib_alloc_vmalloc_buffer(struct snd_pcm_substream *substream,
size_t size, gfp_t gfp_flags)
{
struct snd_pcm_runtime *runtime;
if (PCM_RUNTIME_CHECK(substream))
return -EINVAL;
runtime = substream->runtime;
if (runtime->dma_area) {
if (runtime->dma_bytes >= size)
return 0; /* already large enough */
vfree(runtime->dma_area);
}
runtime->dma_area = __vmalloc(size, gfp_flags);
if (!runtime->dma_area)
return -ENOMEM;
runtime->dma_bytes = size;
return 1;
}
EXPORT_SYMBOL(_snd_pcm_lib_alloc_vmalloc_buffer);
/**
* snd_pcm_lib_free_vmalloc_buffer - free vmalloc buffer
* @substream: the substream with a buffer allocated by
* snd_pcm_lib_alloc_vmalloc_buffer()
*
* Return: Zero if successful, or a negative error code on failure.
*/
int snd_pcm_lib_free_vmalloc_buffer(struct snd_pcm_substream *substream)
{
struct snd_pcm_runtime *runtime;
if (PCM_RUNTIME_CHECK(substream))
return -EINVAL;
runtime = substream->runtime;
vfree(runtime->dma_area);
runtime->dma_area = NULL;
return 0;
}
EXPORT_SYMBOL(snd_pcm_lib_free_vmalloc_buffer);
/**
* snd_pcm_lib_get_vmalloc_page - map vmalloc buffer offset to page struct
* @substream: the substream with a buffer allocated by
* snd_pcm_lib_alloc_vmalloc_buffer()
* @offset: offset in the buffer
*
* This function is to be used as the page callback in the PCM ops.
*
* Return: The page struct, or %NULL on failure.
*/
struct page *snd_pcm_lib_get_vmalloc_page(struct snd_pcm_substream *substream,
unsigned long offset)
{
return vmalloc_to_page(substream->runtime->dma_area + offset);
}
EXPORT_SYMBOL(snd_pcm_lib_get_vmalloc_page);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* dvb_frontend.c: DVB frontend tuning interface/thread
*
* Copyright (C) 1999-2001 Ralph Metzler
* Marcus Metzler
* Holger Waechtler
* for convergence integrated media GmbH
*
* Copyright (C) 2004 Andrew de Quincey (tuning thread cleanup)
*/
/* Enables DVBv3 compatibility bits at the headers */
#define __DVB_CORE__
#define pr_fmt(fmt) "dvb_frontend: " fmt
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/wait.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/semaphore.h>
#include <linux/module.h>
#include <linux/nospec.h>
#include <linux/list.h>
#include <linux/freezer.h>
#include <linux/jiffies.h>
#include <linux/kthread.h>
#include <linux/ktime.h>
#include <linux/compat.h>
#include <asm/processor.h>
#include <media/dvb_frontend.h>
#include <media/dvbdev.h>
#include <linux/dvb/version.h>
static int dvb_frontend_debug;
static int dvb_shutdown_timeout;
static int dvb_force_auto_inversion;
static int dvb_override_tune_delay;
static int dvb_powerdown_on_sleep = 1;
static int dvb_mfe_wait_time = 5;
module_param_named(frontend_debug, dvb_frontend_debug, int, 0644);
MODULE_PARM_DESC(frontend_debug, "Turn on/off frontend core debugging (default:off).");
module_param(dvb_shutdown_timeout, int, 0644);
MODULE_PARM_DESC(dvb_shutdown_timeout, "wait <shutdown_timeout> seconds after close() before suspending hardware");
module_param(dvb_force_auto_inversion, int, 0644);
MODULE_PARM_DESC(dvb_force_auto_inversion, "0: normal (default), 1: INVERSION_AUTO forced always");
module_param(dvb_override_tune_delay, int, 0644);
MODULE_PARM_DESC(dvb_override_tune_delay, "0: normal (default), >0 => delay in milliseconds to wait for lock after a tune attempt");
module_param(dvb_powerdown_on_sleep, int, 0644);
MODULE_PARM_DESC(dvb_powerdown_on_sleep, "0: do not power down, 1: turn LNB voltage off on sleep (default)");
module_param(dvb_mfe_wait_time, int, 0644);
MODULE_PARM_DESC(dvb_mfe_wait_time, "Wait up to <mfe_wait_time> seconds on open() for multi-frontend to become available (default:5 seconds)");
#define dprintk(fmt, arg...) \
printk(KERN_DEBUG pr_fmt("%s: " fmt), __func__, ##arg)
#define FESTATE_IDLE 1
#define FESTATE_RETUNE 2
#define FESTATE_TUNING_FAST 4
#define FESTATE_TUNING_SLOW 8
#define FESTATE_TUNED 16
#define FESTATE_ZIGZAG_FAST 32
#define FESTATE_ZIGZAG_SLOW 64
#define FESTATE_DISEQC 128
#define FESTATE_ERROR 256
#define FESTATE_WAITFORLOCK (FESTATE_TUNING_FAST | FESTATE_TUNING_SLOW | FESTATE_ZIGZAG_FAST | FESTATE_ZIGZAG_SLOW | FESTATE_DISEQC)
#define FESTATE_SEARCHING_FAST (FESTATE_TUNING_FAST | FESTATE_ZIGZAG_FAST)
#define FESTATE_SEARCHING_SLOW (FESTATE_TUNING_SLOW | FESTATE_ZIGZAG_SLOW)
#define FESTATE_LOSTLOCK (FESTATE_ZIGZAG_FAST | FESTATE_ZIGZAG_SLOW)
/*
* FESTATE_IDLE. No tuning parameters have been supplied and the loop is idling.
* FESTATE_RETUNE. Parameters have been supplied, but we have not yet performed the first tune.
* FESTATE_TUNING_FAST. Tuning parameters have been supplied and fast zigzag scan is in progress.
* FESTATE_TUNING_SLOW. Tuning parameters have been supplied. Fast zigzag failed, so we're trying again, but slower.
* FESTATE_TUNED. The frontend has successfully locked on.
* FESTATE_ZIGZAG_FAST. The lock has been lost, and a fast zigzag has been initiated to try and regain it.
* FESTATE_ZIGZAG_SLOW. The lock has been lost. Fast zigzag has been failed, so we're trying again, but slower.
* FESTATE_DISEQC. A DISEQC command has just been issued.
* FESTATE_WAITFORLOCK. When we're waiting for a lock.
* FESTATE_SEARCHING_FAST. When we're searching for a signal using a fast zigzag scan.
* FESTATE_SEARCHING_SLOW. When we're searching for a signal using a slow zigzag scan.
* FESTATE_LOSTLOCK. When the lock has been lost, and we're searching it again.
*/
static DEFINE_MUTEX(frontend_mutex);
struct dvb_frontend_private {
/* thread/frontend values */
struct dvb_device *dvbdev;
struct dvb_frontend_parameters parameters_out;
struct dvb_fe_events events;
struct semaphore sem;
struct list_head list_head;
wait_queue_head_t wait_queue;
struct task_struct *thread;
unsigned long release_jiffies;
unsigned int wakeup;
enum fe_status status;
unsigned long tune_mode_flags;
unsigned int delay;
unsigned int reinitialise;
int tone;
int voltage;
/* swzigzag values */
unsigned int state;
unsigned int bending;
int lnb_drift;
unsigned int inversion;
unsigned int auto_step;
unsigned int auto_sub_step;
unsigned int started_auto_step;
unsigned int min_delay;
unsigned int max_drift;
unsigned int step_size;
int quality;
unsigned int check_wrapped;
enum dvbfe_search algo_status;
#if defined(CONFIG_MEDIA_CONTROLLER_DVB)
struct media_pipeline pipe;
#endif
};
static void dvb_frontend_invoke_release(struct dvb_frontend *fe,
void (*release)(struct dvb_frontend *fe));
static void __dvb_frontend_free(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
if (fepriv)
dvb_device_put(fepriv->dvbdev); dvb_frontend_invoke_release(fe, fe->ops.release); kfree(fepriv);
}
static void dvb_frontend_free(struct kref *ref)
{
struct dvb_frontend *fe =
container_of(ref, struct dvb_frontend, refcount);
__dvb_frontend_free(fe);
}
static void dvb_frontend_put(struct dvb_frontend *fe)
{
/* call detach before dropping the reference count */
if (fe->ops.detach) fe->ops.detach(fe);
/*
* Check if the frontend was registered, as otherwise
* kref was not initialized yet.
*/
if (fe->frontend_priv) kref_put(&fe->refcount, dvb_frontend_free);
else
__dvb_frontend_free(fe);
}
static void dvb_frontend_get(struct dvb_frontend *fe)
{
kref_get(&fe->refcount);
}
static void dvb_frontend_wakeup(struct dvb_frontend *fe);
static int dtv_get_frontend(struct dvb_frontend *fe,
struct dtv_frontend_properties *c,
struct dvb_frontend_parameters *p_out);
static int
dtv_property_legacy_params_sync(struct dvb_frontend *fe,
const struct dtv_frontend_properties *c,
struct dvb_frontend_parameters *p);
static bool has_get_frontend(struct dvb_frontend *fe)
{
return fe->ops.get_frontend;
}
/*
* Due to DVBv3 API calls, a delivery system should be mapped into one of
* the 4 DVBv3 delivery systems (FE_QPSK, FE_QAM, FE_OFDM or FE_ATSC),
* otherwise, a DVBv3 call will fail.
*/
enum dvbv3_emulation_type {
DVBV3_UNKNOWN,
DVBV3_QPSK,
DVBV3_QAM,
DVBV3_OFDM,
DVBV3_ATSC,
};
static enum dvbv3_emulation_type dvbv3_type(u32 delivery_system)
{
switch (delivery_system) {
case SYS_DVBC_ANNEX_A:
case SYS_DVBC_ANNEX_C:
return DVBV3_QAM;
case SYS_DVBS:
case SYS_DVBS2:
case SYS_TURBO:
case SYS_ISDBS:
case SYS_DSS:
return DVBV3_QPSK;
case SYS_DVBT:
case SYS_DVBT2:
case SYS_ISDBT:
case SYS_DTMB:
return DVBV3_OFDM;
case SYS_ATSC:
case SYS_ATSCMH:
case SYS_DVBC_ANNEX_B:
return DVBV3_ATSC;
case SYS_UNDEFINED:
case SYS_ISDBC:
case SYS_DVBH:
case SYS_DAB:
default:
/*
* Doesn't know how to emulate those types and/or
* there's no frontend driver from this type yet
* with some emulation code, so, we're not sure yet how
* to handle them, or they're not compatible with a DVBv3 call.
*/
return DVBV3_UNKNOWN;
}
}
static void dvb_frontend_add_event(struct dvb_frontend *fe,
enum fe_status status)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
struct dvb_fe_events *events = &fepriv->events;
struct dvb_frontend_event *e;
int wp;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
if ((status & FE_HAS_LOCK) && has_get_frontend(fe))
dtv_get_frontend(fe, c, &fepriv->parameters_out);
mutex_lock(&events->mtx);
wp = (events->eventw + 1) % MAX_EVENT;
if (wp == events->eventr) {
events->overflow = 1;
events->eventr = (events->eventr + 1) % MAX_EVENT;
}
e = &events->events[events->eventw];
e->status = status;
e->parameters = fepriv->parameters_out;
events->eventw = wp;
mutex_unlock(&events->mtx);
wake_up_interruptible(&events->wait_queue);
}
static int dvb_frontend_test_event(struct dvb_frontend_private *fepriv,
struct dvb_fe_events *events)
{
int ret;
up(&fepriv->sem);
ret = events->eventw != events->eventr;
down(&fepriv->sem);
return ret;
}
static int dvb_frontend_get_event(struct dvb_frontend *fe,
struct dvb_frontend_event *event, int flags)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dvb_fe_events *events = &fepriv->events;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
if (events->overflow) {
events->overflow = 0;
return -EOVERFLOW;
}
if (events->eventw == events->eventr) {
struct wait_queue_entry wait;
int ret = 0;
if (flags & O_NONBLOCK)
return -EWOULDBLOCK;
init_waitqueue_entry(&wait, current);
add_wait_queue(&events->wait_queue, &wait);
while (!dvb_frontend_test_event(fepriv, events)) {
wait_woken(&wait, TASK_INTERRUPTIBLE, 0);
if (signal_pending(current)) {
ret = -ERESTARTSYS;
break;
}
}
remove_wait_queue(&events->wait_queue, &wait);
if (ret < 0)
return ret;
}
mutex_lock(&events->mtx);
*event = events->events[events->eventr];
events->eventr = (events->eventr + 1) % MAX_EVENT;
mutex_unlock(&events->mtx);
return 0;
}
static void dvb_frontend_clear_events(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dvb_fe_events *events = &fepriv->events;
mutex_lock(&events->mtx);
events->eventr = events->eventw;
mutex_unlock(&events->mtx);
}
static void dvb_frontend_init(struct dvb_frontend *fe)
{
dev_dbg(fe->dvb->device,
"%s: initialising adapter %i frontend %i (%s)...\n",
__func__, fe->dvb->num, fe->id, fe->ops.info.name);
if (fe->ops.init)
fe->ops.init(fe);
if (fe->ops.tuner_ops.init) {
if (fe->ops.i2c_gate_ctrl)
fe->ops.i2c_gate_ctrl(fe, 1);
fe->ops.tuner_ops.init(fe);
if (fe->ops.i2c_gate_ctrl)
fe->ops.i2c_gate_ctrl(fe, 0);
}
}
void dvb_frontend_reinitialise(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
fepriv->reinitialise = 1;
dvb_frontend_wakeup(fe);
}
EXPORT_SYMBOL(dvb_frontend_reinitialise);
static void dvb_frontend_swzigzag_update_delay(struct dvb_frontend_private *fepriv, int locked)
{
int q2;
struct dvb_frontend *fe = fepriv->dvbdev->priv;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
if (locked)
(fepriv->quality) = (fepriv->quality * 220 + 36 * 256) / 256;
else
(fepriv->quality) = (fepriv->quality * 220 + 0) / 256;
q2 = fepriv->quality - 128;
q2 *= q2;
fepriv->delay = fepriv->min_delay + q2 * HZ / (128 * 128);
}
/**
* dvb_frontend_swzigzag_autotune - Performs automatic twiddling of frontend
* parameters.
*
* @fe: The frontend concerned.
* @check_wrapped: Checks if an iteration has completed.
* DO NOT SET ON THE FIRST ATTEMPT.
*
* return: Number of complete iterations that have been performed.
*/
static int dvb_frontend_swzigzag_autotune(struct dvb_frontend *fe, int check_wrapped)
{
int autoinversion;
int ready = 0;
int fe_set_err = 0;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dtv_frontend_properties *c = &fe->dtv_property_cache, tmp;
int original_inversion = c->inversion;
u32 original_frequency = c->frequency;
/* are we using autoinversion? */
autoinversion = ((!(fe->ops.info.caps & FE_CAN_INVERSION_AUTO)) &&
(c->inversion == INVERSION_AUTO));
/* setup parameters correctly */
while (!ready) {
/* calculate the lnb_drift */
fepriv->lnb_drift = fepriv->auto_step * fepriv->step_size;
/* wrap the auto_step if we've exceeded the maximum drift */
if (fepriv->lnb_drift > fepriv->max_drift) {
fepriv->auto_step = 0;
fepriv->auto_sub_step = 0;
fepriv->lnb_drift = 0;
}
/* perform inversion and +/- zigzag */
switch (fepriv->auto_sub_step) {
case 0:
/* try with the current inversion and current drift setting */
ready = 1;
break;
case 1:
if (!autoinversion) break;
fepriv->inversion = (fepriv->inversion == INVERSION_OFF) ? INVERSION_ON : INVERSION_OFF;
ready = 1;
break;
case 2:
if (fepriv->lnb_drift == 0) break;
fepriv->lnb_drift = -fepriv->lnb_drift;
ready = 1;
break;
case 3:
if (fepriv->lnb_drift == 0) break;
if (!autoinversion) break;
fepriv->inversion = (fepriv->inversion == INVERSION_OFF) ? INVERSION_ON : INVERSION_OFF;
fepriv->lnb_drift = -fepriv->lnb_drift;
ready = 1;
break;
default:
fepriv->auto_step++;
fepriv->auto_sub_step = -1; /* it'll be incremented to 0 in a moment */
break;
}
if (!ready) fepriv->auto_sub_step++;
}
/* if this attempt would hit where we started, indicate a complete
* iteration has occurred */
if ((fepriv->auto_step == fepriv->started_auto_step) &&
(fepriv->auto_sub_step == 0) && check_wrapped) {
return 1;
}
dev_dbg(fe->dvb->device,
"%s: drift:%i inversion:%i auto_step:%i auto_sub_step:%i started_auto_step:%i\n",
__func__, fepriv->lnb_drift, fepriv->inversion,
fepriv->auto_step, fepriv->auto_sub_step,
fepriv->started_auto_step);
/* set the frontend itself */
c->frequency += fepriv->lnb_drift;
if (autoinversion)
c->inversion = fepriv->inversion;
tmp = *c;
if (fe->ops.set_frontend)
fe_set_err = fe->ops.set_frontend(fe);
*c = tmp;
if (fe_set_err < 0) {
fepriv->state = FESTATE_ERROR;
return fe_set_err;
}
c->frequency = original_frequency;
c->inversion = original_inversion;
fepriv->auto_sub_step++;
return 0;
}
static void dvb_frontend_swzigzag(struct dvb_frontend *fe)
{
enum fe_status s = FE_NONE;
int retval = 0;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dtv_frontend_properties *c = &fe->dtv_property_cache, tmp;
if (fepriv->max_drift)
dev_warn_once(fe->dvb->device,
"Frontend requested software zigzag, but didn't set the frequency step size\n");
/* if we've got no parameters, just keep idling */
if (fepriv->state & FESTATE_IDLE) {
fepriv->delay = 3 * HZ;
fepriv->quality = 0;
return;
}
/* in SCAN mode, we just set the frontend when asked and leave it alone */
if (fepriv->tune_mode_flags & FE_TUNE_MODE_ONESHOT) {
if (fepriv->state & FESTATE_RETUNE) {
tmp = *c;
if (fe->ops.set_frontend)
retval = fe->ops.set_frontend(fe);
*c = tmp;
if (retval < 0)
fepriv->state = FESTATE_ERROR;
else
fepriv->state = FESTATE_TUNED;
}
fepriv->delay = 3 * HZ;
fepriv->quality = 0;
return;
}
/* get the frontend status */
if (fepriv->state & FESTATE_RETUNE) {
s = 0;
} else {
if (fe->ops.read_status)
fe->ops.read_status(fe, &s);
if (s != fepriv->status) {
dvb_frontend_add_event(fe, s);
fepriv->status = s;
}
}
/* if we're not tuned, and we have a lock, move to the TUNED state */
if ((fepriv->state & FESTATE_WAITFORLOCK) && (s & FE_HAS_LOCK)) {
dvb_frontend_swzigzag_update_delay(fepriv, s & FE_HAS_LOCK);
fepriv->state = FESTATE_TUNED;
/* if we're tuned, then we have determined the correct inversion */
if ((!(fe->ops.info.caps & FE_CAN_INVERSION_AUTO)) &&
(c->inversion == INVERSION_AUTO)) {
c->inversion = fepriv->inversion;
}
return;
}
/* if we are tuned already, check we're still locked */
if (fepriv->state & FESTATE_TUNED) {
dvb_frontend_swzigzag_update_delay(fepriv, s & FE_HAS_LOCK);
/* we're tuned, and the lock is still good... */
if (s & FE_HAS_LOCK) {
return;
} else { /* if we _WERE_ tuned, but now don't have a lock */
fepriv->state = FESTATE_ZIGZAG_FAST;
fepriv->started_auto_step = fepriv->auto_step;
fepriv->check_wrapped = 0;
}
}
/* don't actually do anything if we're in the LOSTLOCK state,
* the frontend is set to FE_CAN_RECOVER, and the max_drift is 0 */
if ((fepriv->state & FESTATE_LOSTLOCK) &&
(fe->ops.info.caps & FE_CAN_RECOVER) && (fepriv->max_drift == 0)) {
dvb_frontend_swzigzag_update_delay(fepriv, s & FE_HAS_LOCK);
return;
}
/* don't do anything if we're in the DISEQC state, since this
* might be someone with a motorized dish controlled by DISEQC.
* If its actually a re-tune, there will be a SET_FRONTEND soon enough. */
if (fepriv->state & FESTATE_DISEQC) {
dvb_frontend_swzigzag_update_delay(fepriv, s & FE_HAS_LOCK);
return;
}
/* if we're in the RETUNE state, set everything up for a brand
* new scan, keeping the current inversion setting, as the next
* tune is _very_ likely to require the same */
if (fepriv->state & FESTATE_RETUNE) {
fepriv->lnb_drift = 0;
fepriv->auto_step = 0;
fepriv->auto_sub_step = 0;
fepriv->started_auto_step = 0;
fepriv->check_wrapped = 0;
}
/* fast zigzag. */
if ((fepriv->state & FESTATE_SEARCHING_FAST) || (fepriv->state & FESTATE_RETUNE)) {
fepriv->delay = fepriv->min_delay;
/* perform a tune */
retval = dvb_frontend_swzigzag_autotune(fe,
fepriv->check_wrapped);
if (retval < 0) {
return;
} else if (retval) {
/* OK, if we've run out of trials at the fast speed.
* Drop back to slow for the _next_ attempt */
fepriv->state = FESTATE_SEARCHING_SLOW;
fepriv->started_auto_step = fepriv->auto_step;
return;
}
fepriv->check_wrapped = 1;
/* if we've just re-tuned, enter the ZIGZAG_FAST state.
* This ensures we cannot return from an
* FE_SET_FRONTEND ioctl before the first frontend tune
* occurs */
if (fepriv->state & FESTATE_RETUNE) {
fepriv->state = FESTATE_TUNING_FAST;
}
}
/* slow zigzag */
if (fepriv->state & FESTATE_SEARCHING_SLOW) {
dvb_frontend_swzigzag_update_delay(fepriv, s & FE_HAS_LOCK);
/* Note: don't bother checking for wrapping; we stay in this
* state until we get a lock */
dvb_frontend_swzigzag_autotune(fe, 0);
}
}
static int dvb_frontend_is_exiting(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
if (fe->exit != DVB_FE_NO_EXIT)
return 1;
if (fepriv->dvbdev->writers == 1)
if (time_after_eq(jiffies, fepriv->release_jiffies +
dvb_shutdown_timeout * HZ))
return 1;
return 0;
}
static int dvb_frontend_should_wakeup(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
if (fepriv->wakeup) {
fepriv->wakeup = 0;
return 1;
}
return dvb_frontend_is_exiting(fe);
}
static void dvb_frontend_wakeup(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
fepriv->wakeup = 1;
wake_up_interruptible(&fepriv->wait_queue);
}
static int dvb_frontend_thread(void *data)
{
struct dvb_frontend *fe = data;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
enum fe_status s = FE_NONE;
enum dvbfe_algo algo;
bool re_tune = false;
bool semheld = false;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
fepriv->check_wrapped = 0;
fepriv->quality = 0;
fepriv->delay = 3 * HZ;
fepriv->status = 0;
fepriv->wakeup = 0;
fepriv->reinitialise = 0;
dvb_frontend_init(fe);
set_freezable();
while (1) {
up(&fepriv->sem); /* is locked when we enter the thread... */
wait_event_freezable_timeout(fepriv->wait_queue,
dvb_frontend_should_wakeup(fe) ||
kthread_should_stop(),
fepriv->delay);
if (kthread_should_stop() || dvb_frontend_is_exiting(fe)) {
/* got signal or quitting */
if (!down_interruptible(&fepriv->sem))
semheld = true;
fe->exit = DVB_FE_NORMAL_EXIT;
break;
}
if (down_interruptible(&fepriv->sem))
break;
if (fepriv->reinitialise) {
dvb_frontend_init(fe);
if (fe->ops.set_tone && fepriv->tone != -1)
fe->ops.set_tone(fe, fepriv->tone);
if (fe->ops.set_voltage && fepriv->voltage != -1)
fe->ops.set_voltage(fe, fepriv->voltage);
fepriv->reinitialise = 0;
}
/* do an iteration of the tuning loop */
if (fe->ops.get_frontend_algo) {
algo = fe->ops.get_frontend_algo(fe);
switch (algo) {
case DVBFE_ALGO_HW:
dev_dbg(fe->dvb->device, "%s: Frontend ALGO = DVBFE_ALGO_HW\n", __func__);
if (fepriv->state & FESTATE_RETUNE) {
dev_dbg(fe->dvb->device, "%s: Retune requested, FESTATE_RETUNE\n", __func__);
re_tune = true;
fepriv->state = FESTATE_TUNED;
} else {
re_tune = false;
}
if (fe->ops.tune)
fe->ops.tune(fe, re_tune, fepriv->tune_mode_flags, &fepriv->delay, &s);
if (s != fepriv->status && !(fepriv->tune_mode_flags & FE_TUNE_MODE_ONESHOT)) {
dev_dbg(fe->dvb->device, "%s: state changed, adding current state\n", __func__);
dvb_frontend_add_event(fe, s);
fepriv->status = s;
}
break;
case DVBFE_ALGO_SW:
dev_dbg(fe->dvb->device, "%s: Frontend ALGO = DVBFE_ALGO_SW\n", __func__);
dvb_frontend_swzigzag(fe);
break;
case DVBFE_ALGO_CUSTOM:
dev_dbg(fe->dvb->device, "%s: Frontend ALGO = DVBFE_ALGO_CUSTOM, state=%d\n", __func__, fepriv->state);
if (fepriv->state & FESTATE_RETUNE) {
dev_dbg(fe->dvb->device, "%s: Retune requested, FESTAT_RETUNE\n", __func__);
fepriv->state = FESTATE_TUNED;
}
/* Case where we are going to search for a carrier
* User asked us to retune again for some reason, possibly
* requesting a search with a new set of parameters
*/
if (fepriv->algo_status & DVBFE_ALGO_SEARCH_AGAIN) {
if (fe->ops.search) {
fepriv->algo_status = fe->ops.search(fe);
/* We did do a search as was requested, the flags are
* now unset as well and has the flags wrt to search.
*/
} else {
fepriv->algo_status &= ~DVBFE_ALGO_SEARCH_AGAIN;
}
}
/* Track the carrier if the search was successful */
if (fepriv->algo_status != DVBFE_ALGO_SEARCH_SUCCESS) {
fepriv->algo_status |= DVBFE_ALGO_SEARCH_AGAIN;
fepriv->delay = HZ / 2;
}
dtv_property_legacy_params_sync(fe, c, &fepriv->parameters_out);
fe->ops.read_status(fe, &s);
if (s != fepriv->status) {
dvb_frontend_add_event(fe, s); /* update event list */
fepriv->status = s;
if (!(s & FE_HAS_LOCK)) {
fepriv->delay = HZ / 10;
fepriv->algo_status |= DVBFE_ALGO_SEARCH_AGAIN;
} else {
fepriv->delay = 60 * HZ;
}
}
break;
default:
dev_dbg(fe->dvb->device, "%s: UNDEFINED ALGO !\n", __func__);
break;
}
} else {
dvb_frontend_swzigzag(fe);
}
}
if (dvb_powerdown_on_sleep) {
if (fe->ops.set_voltage)
fe->ops.set_voltage(fe, SEC_VOLTAGE_OFF);
if (fe->ops.tuner_ops.sleep) {
if (fe->ops.i2c_gate_ctrl)
fe->ops.i2c_gate_ctrl(fe, 1);
fe->ops.tuner_ops.sleep(fe);
if (fe->ops.i2c_gate_ctrl)
fe->ops.i2c_gate_ctrl(fe, 0);
}
if (fe->ops.sleep)
fe->ops.sleep(fe);
}
fepriv->thread = NULL;
if (kthread_should_stop())
fe->exit = DVB_FE_DEVICE_REMOVED;
else
fe->exit = DVB_FE_NO_EXIT;
mb();
if (semheld)
up(&fepriv->sem);
dvb_frontend_wakeup(fe);
return 0;
}
static void dvb_frontend_stop(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv; dev_dbg(fe->dvb->device, "%s:\n", __func__); if (fe->exit != DVB_FE_DEVICE_REMOVED) fe->exit = DVB_FE_NORMAL_EXIT; mb();
if (!fepriv->thread)
return;
kthread_stop(fepriv->thread);
sema_init(&fepriv->sem, 1);
fepriv->state = FESTATE_IDLE;
/* paranoia check in case a signal arrived */
if (fepriv->thread)
dev_warn(fe->dvb->device,
"dvb_frontend_stop: warning: thread %p won't exit\n",
fepriv->thread);
}
/*
* Sleep for the amount of time given by add_usec parameter
*
* This needs to be as precise as possible, as it affects the detection of
* the dish tone command at the satellite subsystem. The precision is improved
* by using a scheduled msleep followed by udelay for the remainder.
*/
void dvb_frontend_sleep_until(ktime_t *waketime, u32 add_usec)
{
s32 delta;
*waketime = ktime_add_us(*waketime, add_usec);
delta = ktime_us_delta(ktime_get_boottime(), *waketime);
if (delta > 2500) {
msleep((delta - 1500) / 1000);
delta = ktime_us_delta(ktime_get_boottime(), *waketime);
}
if (delta > 0)
udelay(delta);
}
EXPORT_SYMBOL(dvb_frontend_sleep_until);
static int dvb_frontend_start(struct dvb_frontend *fe)
{
int ret;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct task_struct *fe_thread;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
if (fepriv->thread) {
if (fe->exit == DVB_FE_NO_EXIT)
return 0;
else
dvb_frontend_stop(fe);
}
if (signal_pending(current))
return -EINTR;
if (down_interruptible(&fepriv->sem))
return -EINTR;
fepriv->state = FESTATE_IDLE;
fe->exit = DVB_FE_NO_EXIT;
fepriv->thread = NULL;
mb();
fe_thread = kthread_run(dvb_frontend_thread, fe,
"kdvb-ad-%i-fe-%i", fe->dvb->num, fe->id);
if (IS_ERR(fe_thread)) {
ret = PTR_ERR(fe_thread);
dev_warn(fe->dvb->device,
"dvb_frontend_start: failed to start kthread (%d)\n",
ret);
up(&fepriv->sem);
return ret;
}
fepriv->thread = fe_thread;
return 0;
}
static void dvb_frontend_get_frequency_limits(struct dvb_frontend *fe,
u32 *freq_min, u32 *freq_max,
u32 *tolerance)
{
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
u32 tuner_min = fe->ops.tuner_ops.info.frequency_min_hz;
u32 tuner_max = fe->ops.tuner_ops.info.frequency_max_hz;
u32 frontend_min = fe->ops.info.frequency_min_hz;
u32 frontend_max = fe->ops.info.frequency_max_hz;
*freq_min = max(frontend_min, tuner_min);
if (frontend_max == 0)
*freq_max = tuner_max;
else if (tuner_max == 0)
*freq_max = frontend_max;
else
*freq_max = min(frontend_max, tuner_max);
if (*freq_min == 0 || *freq_max == 0)
dev_warn(fe->dvb->device,
"DVB: adapter %i frontend %u frequency limits undefined - fix the driver\n",
fe->dvb->num, fe->id);
dev_dbg(fe->dvb->device, "frequency interval: tuner: %u...%u, frontend: %u...%u",
tuner_min, tuner_max, frontend_min, frontend_max);
/* If the standard is for satellite, convert frequencies to kHz */
switch (c->delivery_system) {
case SYS_DSS:
case SYS_DVBS:
case SYS_DVBS2:
case SYS_TURBO:
case SYS_ISDBS:
*freq_min /= kHz;
*freq_max /= kHz;
if (tolerance)
*tolerance = fe->ops.info.frequency_tolerance_hz / kHz;
break;
default:
if (tolerance)
*tolerance = fe->ops.info.frequency_tolerance_hz;
break;
}
}
static u32 dvb_frontend_get_stepsize(struct dvb_frontend *fe)
{
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
u32 fe_step = fe->ops.info.frequency_stepsize_hz;
u32 tuner_step = fe->ops.tuner_ops.info.frequency_step_hz;
u32 step = max(fe_step, tuner_step);
switch (c->delivery_system) {
case SYS_DSS:
case SYS_DVBS:
case SYS_DVBS2:
case SYS_TURBO:
case SYS_ISDBS:
step /= kHz;
break;
default:
break;
}
return step;
}
static int dvb_frontend_check_parameters(struct dvb_frontend *fe)
{
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
u32 freq_min;
u32 freq_max;
/* range check: frequency */
dvb_frontend_get_frequency_limits(fe, &freq_min, &freq_max, NULL);
if ((freq_min && c->frequency < freq_min) ||
(freq_max && c->frequency > freq_max)) {
dev_warn(fe->dvb->device, "DVB: adapter %i frontend %i frequency %u out of range (%u..%u)\n",
fe->dvb->num, fe->id, c->frequency,
freq_min, freq_max);
return -EINVAL;
}
/* range check: symbol rate */
switch (c->delivery_system) {
case SYS_DSS:
case SYS_DVBS:
case SYS_DVBS2:
case SYS_TURBO:
case SYS_DVBC_ANNEX_A:
case SYS_DVBC_ANNEX_C:
if ((fe->ops.info.symbol_rate_min &&
c->symbol_rate < fe->ops.info.symbol_rate_min) ||
(fe->ops.info.symbol_rate_max &&
c->symbol_rate > fe->ops.info.symbol_rate_max)) {
dev_warn(fe->dvb->device, "DVB: adapter %i frontend %i symbol rate %u out of range (%u..%u)\n",
fe->dvb->num, fe->id, c->symbol_rate,
fe->ops.info.symbol_rate_min,
fe->ops.info.symbol_rate_max);
return -EINVAL;
}
break;
default:
break;
}
return 0;
}
static int dvb_frontend_clear_cache(struct dvb_frontend *fe)
{
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
int i;
u32 delsys;
delsys = c->delivery_system;
memset(c, 0, offsetof(struct dtv_frontend_properties, strength));
c->delivery_system = delsys;
dev_dbg(fe->dvb->device, "%s: Clearing cache for delivery system %d\n",
__func__, c->delivery_system);
c->transmission_mode = TRANSMISSION_MODE_AUTO;
c->bandwidth_hz = 0; /* AUTO */
c->guard_interval = GUARD_INTERVAL_AUTO;
c->hierarchy = HIERARCHY_AUTO;
c->symbol_rate = 0;
c->code_rate_HP = FEC_AUTO;
c->code_rate_LP = FEC_AUTO;
c->fec_inner = FEC_AUTO;
c->rolloff = ROLLOFF_AUTO;
c->voltage = SEC_VOLTAGE_OFF;
c->sectone = SEC_TONE_OFF;
c->pilot = PILOT_AUTO;
c->isdbt_partial_reception = 0;
c->isdbt_sb_mode = 0;
c->isdbt_sb_subchannel = 0;
c->isdbt_sb_segment_idx = 0;
c->isdbt_sb_segment_count = 0;
c->isdbt_layer_enabled = 7; /* All layers (A,B,C) */
for (i = 0; i < 3; i++) {
c->layer[i].fec = FEC_AUTO;
c->layer[i].modulation = QAM_AUTO;
c->layer[i].interleaving = 0;
c->layer[i].segment_count = 0;
}
c->stream_id = NO_STREAM_ID_FILTER;
c->scrambling_sequence_index = 0;/* default sequence */
switch (c->delivery_system) {
case SYS_DSS:
c->modulation = QPSK;
c->rolloff = ROLLOFF_20;
break;
case SYS_DVBS:
case SYS_DVBS2:
case SYS_TURBO:
c->modulation = QPSK; /* implied for DVB-S in legacy API */
c->rolloff = ROLLOFF_35;/* implied for DVB-S */
break;
case SYS_ATSC:
c->modulation = VSB_8;
break;
case SYS_ISDBS:
c->symbol_rate = 28860000;
c->rolloff = ROLLOFF_35;
c->bandwidth_hz = c->symbol_rate / 100 * 135;
break;
default:
c->modulation = QAM_AUTO;
break;
}
c->lna = LNA_AUTO;
return 0;
}
#define _DTV_CMD(n) \
[n] = #n
static char *dtv_cmds[DTV_MAX_COMMAND + 1] = {
_DTV_CMD(DTV_TUNE),
_DTV_CMD(DTV_CLEAR),
/* Set */
_DTV_CMD(DTV_FREQUENCY),
_DTV_CMD(DTV_BANDWIDTH_HZ),
_DTV_CMD(DTV_MODULATION),
_DTV_CMD(DTV_INVERSION),
_DTV_CMD(DTV_DISEQC_MASTER),
_DTV_CMD(DTV_SYMBOL_RATE),
_DTV_CMD(DTV_INNER_FEC),
_DTV_CMD(DTV_VOLTAGE),
_DTV_CMD(DTV_TONE),
_DTV_CMD(DTV_PILOT),
_DTV_CMD(DTV_ROLLOFF),
_DTV_CMD(DTV_DELIVERY_SYSTEM),
_DTV_CMD(DTV_HIERARCHY),
_DTV_CMD(DTV_CODE_RATE_HP),
_DTV_CMD(DTV_CODE_RATE_LP),
_DTV_CMD(DTV_GUARD_INTERVAL),
_DTV_CMD(DTV_TRANSMISSION_MODE),
_DTV_CMD(DTV_INTERLEAVING),
_DTV_CMD(DTV_ISDBT_PARTIAL_RECEPTION),
_DTV_CMD(DTV_ISDBT_SOUND_BROADCASTING),
_DTV_CMD(DTV_ISDBT_SB_SUBCHANNEL_ID),
_DTV_CMD(DTV_ISDBT_SB_SEGMENT_IDX),
_DTV_CMD(DTV_ISDBT_SB_SEGMENT_COUNT),
_DTV_CMD(DTV_ISDBT_LAYER_ENABLED),
_DTV_CMD(DTV_ISDBT_LAYERA_FEC),
_DTV_CMD(DTV_ISDBT_LAYERA_MODULATION),
_DTV_CMD(DTV_ISDBT_LAYERA_SEGMENT_COUNT),
_DTV_CMD(DTV_ISDBT_LAYERA_TIME_INTERLEAVING),
_DTV_CMD(DTV_ISDBT_LAYERB_FEC),
_DTV_CMD(DTV_ISDBT_LAYERB_MODULATION),
_DTV_CMD(DTV_ISDBT_LAYERB_SEGMENT_COUNT),
_DTV_CMD(DTV_ISDBT_LAYERB_TIME_INTERLEAVING),
_DTV_CMD(DTV_ISDBT_LAYERC_FEC),
_DTV_CMD(DTV_ISDBT_LAYERC_MODULATION),
_DTV_CMD(DTV_ISDBT_LAYERC_SEGMENT_COUNT),
_DTV_CMD(DTV_ISDBT_LAYERC_TIME_INTERLEAVING),
_DTV_CMD(DTV_STREAM_ID),
_DTV_CMD(DTV_DVBT2_PLP_ID_LEGACY),
_DTV_CMD(DTV_SCRAMBLING_SEQUENCE_INDEX),
_DTV_CMD(DTV_LNA),
/* Get */
_DTV_CMD(DTV_DISEQC_SLAVE_REPLY),
_DTV_CMD(DTV_API_VERSION),
_DTV_CMD(DTV_ENUM_DELSYS),
_DTV_CMD(DTV_ATSCMH_PARADE_ID),
_DTV_CMD(DTV_ATSCMH_RS_FRAME_ENSEMBLE),
_DTV_CMD(DTV_ATSCMH_FIC_VER),
_DTV_CMD(DTV_ATSCMH_NOG),
_DTV_CMD(DTV_ATSCMH_TNOG),
_DTV_CMD(DTV_ATSCMH_SGN),
_DTV_CMD(DTV_ATSCMH_PRC),
_DTV_CMD(DTV_ATSCMH_RS_FRAME_MODE),
_DTV_CMD(DTV_ATSCMH_RS_CODE_MODE_PRI),
_DTV_CMD(DTV_ATSCMH_RS_CODE_MODE_SEC),
_DTV_CMD(DTV_ATSCMH_SCCC_BLOCK_MODE),
_DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_A),
_DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_B),
_DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_C),
_DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_D),
/* Statistics API */
_DTV_CMD(DTV_STAT_SIGNAL_STRENGTH),
_DTV_CMD(DTV_STAT_CNR),
_DTV_CMD(DTV_STAT_PRE_ERROR_BIT_COUNT),
_DTV_CMD(DTV_STAT_PRE_TOTAL_BIT_COUNT),
_DTV_CMD(DTV_STAT_POST_ERROR_BIT_COUNT),
_DTV_CMD(DTV_STAT_POST_TOTAL_BIT_COUNT),
_DTV_CMD(DTV_STAT_ERROR_BLOCK_COUNT),
_DTV_CMD(DTV_STAT_TOTAL_BLOCK_COUNT),
};
static char *dtv_cmd_name(u32 cmd)
{
cmd = array_index_nospec(cmd, DTV_MAX_COMMAND);
return dtv_cmds[cmd];
}
/* Synchronise the legacy tuning parameters into the cache, so that demodulator
* drivers can use a single set_frontend tuning function, regardless of whether
* it's being used for the legacy or new API, reducing code and complexity.
*/
static int dtv_property_cache_sync(struct dvb_frontend *fe,
struct dtv_frontend_properties *c,
const struct dvb_frontend_parameters *p)
{
c->frequency = p->frequency;
c->inversion = p->inversion;
switch (dvbv3_type(c->delivery_system)) {
case DVBV3_QPSK:
dev_dbg(fe->dvb->device, "%s: Preparing QPSK req\n", __func__);
c->symbol_rate = p->u.qpsk.symbol_rate;
c->fec_inner = p->u.qpsk.fec_inner;
break;
case DVBV3_QAM:
dev_dbg(fe->dvb->device, "%s: Preparing QAM req\n", __func__);
c->symbol_rate = p->u.qam.symbol_rate;
c->fec_inner = p->u.qam.fec_inner;
c->modulation = p->u.qam.modulation;
break;
case DVBV3_OFDM:
dev_dbg(fe->dvb->device, "%s: Preparing OFDM req\n", __func__);
switch (p->u.ofdm.bandwidth) {
case BANDWIDTH_10_MHZ:
c->bandwidth_hz = 10000000;
break;
case BANDWIDTH_8_MHZ:
c->bandwidth_hz = 8000000;
break;
case BANDWIDTH_7_MHZ:
c->bandwidth_hz = 7000000;
break;
case BANDWIDTH_6_MHZ:
c->bandwidth_hz = 6000000;
break;
case BANDWIDTH_5_MHZ:
c->bandwidth_hz = 5000000;
break;
case BANDWIDTH_1_712_MHZ:
c->bandwidth_hz = 1712000;
break;
case BANDWIDTH_AUTO:
c->bandwidth_hz = 0;
}
c->code_rate_HP = p->u.ofdm.code_rate_HP;
c->code_rate_LP = p->u.ofdm.code_rate_LP;
c->modulation = p->u.ofdm.constellation;
c->transmission_mode = p->u.ofdm.transmission_mode;
c->guard_interval = p->u.ofdm.guard_interval;
c->hierarchy = p->u.ofdm.hierarchy_information;
break;
case DVBV3_ATSC:
dev_dbg(fe->dvb->device, "%s: Preparing ATSC req\n", __func__);
c->modulation = p->u.vsb.modulation;
if (c->delivery_system == SYS_ATSCMH)
break;
if ((c->modulation == VSB_8) || (c->modulation == VSB_16))
c->delivery_system = SYS_ATSC;
else
c->delivery_system = SYS_DVBC_ANNEX_B;
break;
case DVBV3_UNKNOWN:
dev_err(fe->dvb->device,
"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
__func__, c->delivery_system);
return -EINVAL;
}
return 0;
}
/* Ensure the cached values are set correctly in the frontend
* legacy tuning structures, for the advanced tuning API.
*/
static int
dtv_property_legacy_params_sync(struct dvb_frontend *fe,
const struct dtv_frontend_properties *c,
struct dvb_frontend_parameters *p)
{
p->frequency = c->frequency;
p->inversion = c->inversion;
switch (dvbv3_type(c->delivery_system)) {
case DVBV3_UNKNOWN:
dev_err(fe->dvb->device,
"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
__func__, c->delivery_system);
return -EINVAL;
case DVBV3_QPSK:
dev_dbg(fe->dvb->device, "%s: Preparing QPSK req\n", __func__);
p->u.qpsk.symbol_rate = c->symbol_rate;
p->u.qpsk.fec_inner = c->fec_inner;
break;
case DVBV3_QAM:
dev_dbg(fe->dvb->device, "%s: Preparing QAM req\n", __func__);
p->u.qam.symbol_rate = c->symbol_rate;
p->u.qam.fec_inner = c->fec_inner;
p->u.qam.modulation = c->modulation;
break;
case DVBV3_OFDM:
dev_dbg(fe->dvb->device, "%s: Preparing OFDM req\n", __func__);
switch (c->bandwidth_hz) {
case 10000000:
p->u.ofdm.bandwidth = BANDWIDTH_10_MHZ;
break;
case 8000000:
p->u.ofdm.bandwidth = BANDWIDTH_8_MHZ;
break;
case 7000000:
p->u.ofdm.bandwidth = BANDWIDTH_7_MHZ;
break;
case 6000000:
p->u.ofdm.bandwidth = BANDWIDTH_6_MHZ;
break;
case 5000000:
p->u.ofdm.bandwidth = BANDWIDTH_5_MHZ;
break;
case 1712000:
p->u.ofdm.bandwidth = BANDWIDTH_1_712_MHZ;
break;
case 0:
default:
p->u.ofdm.bandwidth = BANDWIDTH_AUTO;
}
p->u.ofdm.code_rate_HP = c->code_rate_HP;
p->u.ofdm.code_rate_LP = c->code_rate_LP;
p->u.ofdm.constellation = c->modulation;
p->u.ofdm.transmission_mode = c->transmission_mode;
p->u.ofdm.guard_interval = c->guard_interval;
p->u.ofdm.hierarchy_information = c->hierarchy;
break;
case DVBV3_ATSC:
dev_dbg(fe->dvb->device, "%s: Preparing VSB req\n", __func__);
p->u.vsb.modulation = c->modulation;
break;
}
return 0;
}
/**
* dtv_get_frontend - calls a callback for retrieving DTV parameters
* @fe: struct dvb_frontend pointer
* @c: struct dtv_frontend_properties pointer (DVBv5 cache)
* @p_out: struct dvb_frontend_parameters pointer (DVBv3 FE struct)
*
* This routine calls either the DVBv3 or DVBv5 get_frontend call.
* If c is not null, it will update the DVBv5 cache struct pointed by it.
* If p_out is not null, it will update the DVBv3 params pointed by it.
*/
static int dtv_get_frontend(struct dvb_frontend *fe,
struct dtv_frontend_properties *c,
struct dvb_frontend_parameters *p_out)
{
int r;
if (fe->ops.get_frontend) {
r = fe->ops.get_frontend(fe, c);
if (unlikely(r < 0))
return r;
if (p_out)
dtv_property_legacy_params_sync(fe, c, p_out);
return 0;
}
/* As everything is in cache, get_frontend fops are always supported */
return 0;
}
static int dvb_frontend_handle_ioctl(struct file *file,
unsigned int cmd, void *parg);
static int dtv_property_process_get(struct dvb_frontend *fe,
const struct dtv_frontend_properties *c,
struct dtv_property *tvp,
struct file *file)
{
int ncaps;
unsigned int len = 1;
switch (tvp->cmd) {
case DTV_ENUM_DELSYS:
ncaps = 0;
while (ncaps < MAX_DELSYS && fe->ops.delsys[ncaps]) {
tvp->u.buffer.data[ncaps] = fe->ops.delsys[ncaps];
ncaps++;
}
tvp->u.buffer.len = ncaps;
len = ncaps;
break;
case DTV_FREQUENCY:
tvp->u.data = c->frequency;
break;
case DTV_MODULATION:
tvp->u.data = c->modulation;
break;
case DTV_BANDWIDTH_HZ:
tvp->u.data = c->bandwidth_hz;
break;
case DTV_INVERSION:
tvp->u.data = c->inversion;
break;
case DTV_SYMBOL_RATE:
tvp->u.data = c->symbol_rate;
break;
case DTV_INNER_FEC:
tvp->u.data = c->fec_inner;
break;
case DTV_PILOT:
tvp->u.data = c->pilot;
break;
case DTV_ROLLOFF:
tvp->u.data = c->rolloff;
break;
case DTV_DELIVERY_SYSTEM:
tvp->u.data = c->delivery_system;
break;
case DTV_VOLTAGE:
tvp->u.data = c->voltage;
break;
case DTV_TONE:
tvp->u.data = c->sectone;
break;
case DTV_API_VERSION:
tvp->u.data = (DVB_API_VERSION << 8) | DVB_API_VERSION_MINOR;
break;
case DTV_CODE_RATE_HP:
tvp->u.data = c->code_rate_HP;
break;
case DTV_CODE_RATE_LP:
tvp->u.data = c->code_rate_LP;
break;
case DTV_GUARD_INTERVAL:
tvp->u.data = c->guard_interval;
break;
case DTV_TRANSMISSION_MODE:
tvp->u.data = c->transmission_mode;
break;
case DTV_HIERARCHY:
tvp->u.data = c->hierarchy;
break;
case DTV_INTERLEAVING:
tvp->u.data = c->interleaving;
break;
/* ISDB-T Support here */
case DTV_ISDBT_PARTIAL_RECEPTION:
tvp->u.data = c->isdbt_partial_reception;
break;
case DTV_ISDBT_SOUND_BROADCASTING:
tvp->u.data = c->isdbt_sb_mode;
break;
case DTV_ISDBT_SB_SUBCHANNEL_ID:
tvp->u.data = c->isdbt_sb_subchannel;
break;
case DTV_ISDBT_SB_SEGMENT_IDX:
tvp->u.data = c->isdbt_sb_segment_idx;
break;
case DTV_ISDBT_SB_SEGMENT_COUNT:
tvp->u.data = c->isdbt_sb_segment_count;
break;
case DTV_ISDBT_LAYER_ENABLED:
tvp->u.data = c->isdbt_layer_enabled;
break;
case DTV_ISDBT_LAYERA_FEC:
tvp->u.data = c->layer[0].fec;
break;
case DTV_ISDBT_LAYERA_MODULATION:
tvp->u.data = c->layer[0].modulation;
break;
case DTV_ISDBT_LAYERA_SEGMENT_COUNT:
tvp->u.data = c->layer[0].segment_count;
break;
case DTV_ISDBT_LAYERA_TIME_INTERLEAVING:
tvp->u.data = c->layer[0].interleaving;
break;
case DTV_ISDBT_LAYERB_FEC:
tvp->u.data = c->layer[1].fec;
break;
case DTV_ISDBT_LAYERB_MODULATION:
tvp->u.data = c->layer[1].modulation;
break;
case DTV_ISDBT_LAYERB_SEGMENT_COUNT:
tvp->u.data = c->layer[1].segment_count;
break;
case DTV_ISDBT_LAYERB_TIME_INTERLEAVING:
tvp->u.data = c->layer[1].interleaving;
break;
case DTV_ISDBT_LAYERC_FEC:
tvp->u.data = c->layer[2].fec;
break;
case DTV_ISDBT_LAYERC_MODULATION:
tvp->u.data = c->layer[2].modulation;
break;
case DTV_ISDBT_LAYERC_SEGMENT_COUNT:
tvp->u.data = c->layer[2].segment_count;
break;
case DTV_ISDBT_LAYERC_TIME_INTERLEAVING:
tvp->u.data = c->layer[2].interleaving;
break;
/* Multistream support */
case DTV_STREAM_ID:
case DTV_DVBT2_PLP_ID_LEGACY:
tvp->u.data = c->stream_id;
break;
/* Physical layer scrambling support */
case DTV_SCRAMBLING_SEQUENCE_INDEX:
tvp->u.data = c->scrambling_sequence_index;
break;
/* ATSC-MH */
case DTV_ATSCMH_FIC_VER:
tvp->u.data = fe->dtv_property_cache.atscmh_fic_ver;
break;
case DTV_ATSCMH_PARADE_ID:
tvp->u.data = fe->dtv_property_cache.atscmh_parade_id;
break;
case DTV_ATSCMH_NOG:
tvp->u.data = fe->dtv_property_cache.atscmh_nog;
break;
case DTV_ATSCMH_TNOG:
tvp->u.data = fe->dtv_property_cache.atscmh_tnog;
break;
case DTV_ATSCMH_SGN:
tvp->u.data = fe->dtv_property_cache.atscmh_sgn;
break;
case DTV_ATSCMH_PRC:
tvp->u.data = fe->dtv_property_cache.atscmh_prc;
break;
case DTV_ATSCMH_RS_FRAME_MODE:
tvp->u.data = fe->dtv_property_cache.atscmh_rs_frame_mode;
break;
case DTV_ATSCMH_RS_FRAME_ENSEMBLE:
tvp->u.data = fe->dtv_property_cache.atscmh_rs_frame_ensemble;
break;
case DTV_ATSCMH_RS_CODE_MODE_PRI:
tvp->u.data = fe->dtv_property_cache.atscmh_rs_code_mode_pri;
break;
case DTV_ATSCMH_RS_CODE_MODE_SEC:
tvp->u.data = fe->dtv_property_cache.atscmh_rs_code_mode_sec;
break;
case DTV_ATSCMH_SCCC_BLOCK_MODE:
tvp->u.data = fe->dtv_property_cache.atscmh_sccc_block_mode;
break;
case DTV_ATSCMH_SCCC_CODE_MODE_A:
tvp->u.data = fe->dtv_property_cache.atscmh_sccc_code_mode_a;
break;
case DTV_ATSCMH_SCCC_CODE_MODE_B:
tvp->u.data = fe->dtv_property_cache.atscmh_sccc_code_mode_b;
break;
case DTV_ATSCMH_SCCC_CODE_MODE_C:
tvp->u.data = fe->dtv_property_cache.atscmh_sccc_code_mode_c;
break;
case DTV_ATSCMH_SCCC_CODE_MODE_D:
tvp->u.data = fe->dtv_property_cache.atscmh_sccc_code_mode_d;
break;
case DTV_LNA:
tvp->u.data = c->lna;
break;
/* Fill quality measures */
case DTV_STAT_SIGNAL_STRENGTH:
tvp->u.st = c->strength;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_CNR:
tvp->u.st = c->cnr;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_PRE_ERROR_BIT_COUNT:
tvp->u.st = c->pre_bit_error;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_PRE_TOTAL_BIT_COUNT:
tvp->u.st = c->pre_bit_count;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_POST_ERROR_BIT_COUNT:
tvp->u.st = c->post_bit_error;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_POST_TOTAL_BIT_COUNT:
tvp->u.st = c->post_bit_count;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_ERROR_BLOCK_COUNT:
tvp->u.st = c->block_error;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
case DTV_STAT_TOTAL_BLOCK_COUNT:
tvp->u.st = c->block_count;
if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32))
tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32);
len = tvp->u.buffer.len;
break;
default:
dev_dbg(fe->dvb->device,
"%s: FE property %d doesn't exist\n",
__func__, tvp->cmd);
return -EINVAL;
}
if (len < 1)
len = 1;
dev_dbg(fe->dvb->device,
"%s: GET cmd 0x%08x (%s) len %d: %*ph\n",
__func__, tvp->cmd, dtv_cmd_name(tvp->cmd),
tvp->u.buffer.len, tvp->u.buffer.len, tvp->u.buffer.data);
return 0;
}
static int dtv_set_frontend(struct dvb_frontend *fe);
static bool is_dvbv3_delsys(u32 delsys)
{
return (delsys == SYS_DVBT) || (delsys == SYS_DVBC_ANNEX_A) ||
(delsys == SYS_DVBS) || (delsys == SYS_ATSC);
}
/**
* emulate_delivery_system - emulate a DVBv5 delivery system with a DVBv3 type
* @fe: struct frontend;
* @delsys: DVBv5 type that will be used for emulation
*
* Provides emulation for delivery systems that are compatible with the old
* DVBv3 call. Among its usages, it provices support for ISDB-T, and allows
* using a DVB-S2 only frontend just like it were a DVB-S, if the frontend
* parameters are compatible with DVB-S spec.
*/
static int emulate_delivery_system(struct dvb_frontend *fe, u32 delsys)
{
int i;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
c->delivery_system = delsys;
/*
* If the call is for ISDB-T, put it into full-seg, auto mode, TV
*/
if (c->delivery_system == SYS_ISDBT) {
dev_dbg(fe->dvb->device,
"%s: Using defaults for SYS_ISDBT\n",
__func__);
if (!c->bandwidth_hz)
c->bandwidth_hz = 6000000;
c->isdbt_partial_reception = 0;
c->isdbt_sb_mode = 0;
c->isdbt_sb_subchannel = 0;
c->isdbt_sb_segment_idx = 0;
c->isdbt_sb_segment_count = 0;
c->isdbt_layer_enabled = 7;
for (i = 0; i < 3; i++) {
c->layer[i].fec = FEC_AUTO;
c->layer[i].modulation = QAM_AUTO;
c->layer[i].interleaving = 0;
c->layer[i].segment_count = 0;
}
}
dev_dbg(fe->dvb->device, "%s: change delivery system on cache to %d\n",
__func__, c->delivery_system);
return 0;
}
/**
* dvbv5_set_delivery_system - Sets the delivery system for a DVBv5 API call
* @fe: frontend struct
* @desired_system: delivery system requested by the user
*
* A DVBv5 call know what's the desired system it wants. So, set it.
*
* There are, however, a few known issues with early DVBv5 applications that
* are also handled by this logic:
*
* 1) Some early apps use SYS_UNDEFINED as the desired delivery system.
* This is an API violation, but, as we don't want to break userspace,
* convert it to the first supported delivery system.
* 2) Some apps might be using a DVBv5 call in a wrong way, passing, for
* example, SYS_DVBT instead of SYS_ISDBT. This is because early usage of
* ISDB-T provided backward compat with DVB-T.
*/
static int dvbv5_set_delivery_system(struct dvb_frontend *fe,
u32 desired_system)
{
int ncaps;
u32 delsys = SYS_UNDEFINED;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
enum dvbv3_emulation_type type;
/*
* It was reported that some old DVBv5 applications were
* filling delivery_system with SYS_UNDEFINED. If this happens,
* assume that the application wants to use the first supported
* delivery system.
*/
if (desired_system == SYS_UNDEFINED)
desired_system = fe->ops.delsys[0];
/*
* This is a DVBv5 call. So, it likely knows the supported
* delivery systems. So, check if the desired delivery system is
* supported
*/
ncaps = 0;
while (ncaps < MAX_DELSYS && fe->ops.delsys[ncaps]) {
if (fe->ops.delsys[ncaps] == desired_system) {
c->delivery_system = desired_system;
dev_dbg(fe->dvb->device,
"%s: Changing delivery system to %d\n",
__func__, desired_system);
return 0;
}
ncaps++;
}
/*
* The requested delivery system isn't supported. Maybe userspace
* is requesting a DVBv3 compatible delivery system.
*
* The emulation only works if the desired system is one of the
* delivery systems supported by DVBv3 API
*/
if (!is_dvbv3_delsys(desired_system)) {
dev_dbg(fe->dvb->device,
"%s: Delivery system %d not supported.\n",
__func__, desired_system);
return -EINVAL;
}
type = dvbv3_type(desired_system);
/*
* Get the last non-DVBv3 delivery system that has the same type
* of the desired system
*/
ncaps = 0;
while (ncaps < MAX_DELSYS && fe->ops.delsys[ncaps]) {
if (dvbv3_type(fe->ops.delsys[ncaps]) == type)
delsys = fe->ops.delsys[ncaps];
ncaps++;
}
/* There's nothing compatible with the desired delivery system */
if (delsys == SYS_UNDEFINED) {
dev_dbg(fe->dvb->device,
"%s: Delivery system %d not supported on emulation mode.\n",
__func__, desired_system);
return -EINVAL;
}
dev_dbg(fe->dvb->device,
"%s: Using delivery system %d emulated as if it were %d\n",
__func__, delsys, desired_system);
return emulate_delivery_system(fe, desired_system);
}
/**
* dvbv3_set_delivery_system - Sets the delivery system for a DVBv3 API call
* @fe: frontend struct
*
* A DVBv3 call doesn't know what's the desired system it wants. It also
* doesn't allow to switch between different types. Due to that, userspace
* should use DVBv5 instead.
* However, in order to avoid breaking userspace API, limited backward
* compatibility support is provided.
*
* There are some delivery systems that are incompatible with DVBv3 calls.
*
* This routine should work fine for frontends that support just one delivery
* system.
*
* For frontends that support multiple frontends:
* 1) It defaults to use the first supported delivery system. There's an
* userspace application that allows changing it at runtime;
*
* 2) If the current delivery system is not compatible with DVBv3, it gets
* the first one that it is compatible.
*
* NOTE: in order for this to work with applications like Kaffeine that
* uses a DVBv5 call for DVB-S2 and a DVBv3 call to go back to
* DVB-S, drivers that support both DVB-S and DVB-S2 should have the
* SYS_DVBS entry before the SYS_DVBS2, otherwise it won't switch back
* to DVB-S.
*/
static int dvbv3_set_delivery_system(struct dvb_frontend *fe)
{
int ncaps;
u32 delsys = SYS_UNDEFINED;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
/* If not set yet, defaults to the first supported delivery system */
if (c->delivery_system == SYS_UNDEFINED)
c->delivery_system = fe->ops.delsys[0];
/*
* Trivial case: just use the current one, if it already a DVBv3
* delivery system
*/
if (is_dvbv3_delsys(c->delivery_system)) {
dev_dbg(fe->dvb->device,
"%s: Using delivery system to %d\n",
__func__, c->delivery_system);
return 0;
}
/*
* Seek for the first delivery system that it is compatible with a
* DVBv3 standard
*/
ncaps = 0;
while (ncaps < MAX_DELSYS && fe->ops.delsys[ncaps]) {
if (dvbv3_type(fe->ops.delsys[ncaps]) != DVBV3_UNKNOWN) {
delsys = fe->ops.delsys[ncaps];
break;
}
ncaps++;
}
if (delsys == SYS_UNDEFINED) {
dev_dbg(fe->dvb->device,
"%s: Couldn't find a delivery system that works with FE_SET_FRONTEND\n",
__func__);
return -EINVAL;
}
return emulate_delivery_system(fe, delsys);
}
static void prepare_tuning_algo_parameters(struct dvb_frontend *fe)
{
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dvb_frontend_tune_settings fetunesettings = { 0 };
/* get frontend-specific tuning settings */
if (fe->ops.get_tune_settings && (fe->ops.get_tune_settings(fe, &fetunesettings) == 0)) {
fepriv->min_delay = (fetunesettings.min_delay_ms * HZ) / 1000;
fepriv->max_drift = fetunesettings.max_drift;
fepriv->step_size = fetunesettings.step_size;
} else {
/* default values */
switch (c->delivery_system) {
case SYS_DSS:
case SYS_DVBS:
case SYS_DVBS2:
case SYS_ISDBS:
case SYS_TURBO:
case SYS_DVBC_ANNEX_A:
case SYS_DVBC_ANNEX_C:
fepriv->min_delay = HZ / 20;
fepriv->step_size = c->symbol_rate / 16000;
fepriv->max_drift = c->symbol_rate / 2000;
break;
case SYS_DVBT:
case SYS_DVBT2:
case SYS_ISDBT:
case SYS_DTMB:
fepriv->min_delay = HZ / 20;
fepriv->step_size = dvb_frontend_get_stepsize(fe) * 2;
fepriv->max_drift = fepriv->step_size + 1;
break;
default:
/*
* FIXME: This sounds wrong! if freqency_stepsize is
* defined by the frontend, why not use it???
*/
fepriv->min_delay = HZ / 20;
fepriv->step_size = 0; /* no zigzag */
fepriv->max_drift = 0;
break;
}
}
if (dvb_override_tune_delay > 0)
fepriv->min_delay = (dvb_override_tune_delay * HZ) / 1000;
}
/**
* dtv_property_process_set - Sets a single DTV property
* @fe: Pointer to &struct dvb_frontend
* @file: Pointer to &struct file
* @cmd: Digital TV command
* @data: An unsigned 32-bits number
*
* This routine assigns the property
* value to the corresponding member of
* &struct dtv_frontend_properties
*
* Returns:
* Zero on success, negative errno on failure.
*/
static int dtv_property_process_set(struct dvb_frontend *fe,
struct file *file,
u32 cmd, u32 data)
{
int r = 0;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
/** Dump DTV command name and value*/
if (!cmd || cmd > DTV_MAX_COMMAND)
dev_warn(fe->dvb->device, "%s: SET cmd 0x%08x undefined\n",
__func__, cmd);
else
dev_dbg(fe->dvb->device,
"%s: SET cmd 0x%08x (%s) to 0x%08x\n",
__func__, cmd, dtv_cmd_name(cmd), data);
switch (cmd) {
case DTV_CLEAR:
/*
* Reset a cache of data specific to the frontend here. This does
* not effect hardware.
*/
dvb_frontend_clear_cache(fe);
break;
case DTV_TUNE:
/*
* Use the cached Digital TV properties to tune the
* frontend
*/
dev_dbg(fe->dvb->device,
"%s: Setting the frontend from property cache\n",
__func__);
r = dtv_set_frontend(fe);
break;
case DTV_FREQUENCY:
c->frequency = data;
break;
case DTV_MODULATION:
c->modulation = data;
break;
case DTV_BANDWIDTH_HZ:
c->bandwidth_hz = data;
break;
case DTV_INVERSION:
c->inversion = data;
break;
case DTV_SYMBOL_RATE:
c->symbol_rate = data;
break;
case DTV_INNER_FEC:
c->fec_inner = data;
break;
case DTV_PILOT:
c->pilot = data;
break;
case DTV_ROLLOFF:
c->rolloff = data;
break;
case DTV_DELIVERY_SYSTEM:
r = dvbv5_set_delivery_system(fe, data);
break;
case DTV_VOLTAGE:
c->voltage = data;
r = dvb_frontend_handle_ioctl(file, FE_SET_VOLTAGE,
(void *)c->voltage);
break;
case DTV_TONE:
c->sectone = data;
r = dvb_frontend_handle_ioctl(file, FE_SET_TONE,
(void *)c->sectone);
break;
case DTV_CODE_RATE_HP:
c->code_rate_HP = data;
break;
case DTV_CODE_RATE_LP:
c->code_rate_LP = data;
break;
case DTV_GUARD_INTERVAL:
c->guard_interval = data;
break;
case DTV_TRANSMISSION_MODE:
c->transmission_mode = data;
break;
case DTV_HIERARCHY:
c->hierarchy = data;
break;
case DTV_INTERLEAVING:
c->interleaving = data;
break;
/* ISDB-T Support here */
case DTV_ISDBT_PARTIAL_RECEPTION:
c->isdbt_partial_reception = data;
break;
case DTV_ISDBT_SOUND_BROADCASTING:
c->isdbt_sb_mode = data;
break;
case DTV_ISDBT_SB_SUBCHANNEL_ID:
c->isdbt_sb_subchannel = data;
break;
case DTV_ISDBT_SB_SEGMENT_IDX:
c->isdbt_sb_segment_idx = data;
break;
case DTV_ISDBT_SB_SEGMENT_COUNT:
c->isdbt_sb_segment_count = data;
break;
case DTV_ISDBT_LAYER_ENABLED:
c->isdbt_layer_enabled = data;
break;
case DTV_ISDBT_LAYERA_FEC:
c->layer[0].fec = data;
break;
case DTV_ISDBT_LAYERA_MODULATION:
c->layer[0].modulation = data;
break;
case DTV_ISDBT_LAYERA_SEGMENT_COUNT:
c->layer[0].segment_count = data;
break;
case DTV_ISDBT_LAYERA_TIME_INTERLEAVING:
c->layer[0].interleaving = data;
break;
case DTV_ISDBT_LAYERB_FEC:
c->layer[1].fec = data;
break;
case DTV_ISDBT_LAYERB_MODULATION:
c->layer[1].modulation = data;
break;
case DTV_ISDBT_LAYERB_SEGMENT_COUNT:
c->layer[1].segment_count = data;
break;
case DTV_ISDBT_LAYERB_TIME_INTERLEAVING:
c->layer[1].interleaving = data;
break;
case DTV_ISDBT_LAYERC_FEC:
c->layer[2].fec = data;
break;
case DTV_ISDBT_LAYERC_MODULATION:
c->layer[2].modulation = data;
break;
case DTV_ISDBT_LAYERC_SEGMENT_COUNT:
c->layer[2].segment_count = data;
break;
case DTV_ISDBT_LAYERC_TIME_INTERLEAVING:
c->layer[2].interleaving = data;
break;
/* Multistream support */
case DTV_STREAM_ID:
case DTV_DVBT2_PLP_ID_LEGACY:
c->stream_id = data;
break;
/* Physical layer scrambling support */
case DTV_SCRAMBLING_SEQUENCE_INDEX:
c->scrambling_sequence_index = data;
break;
/* ATSC-MH */
case DTV_ATSCMH_PARADE_ID:
fe->dtv_property_cache.atscmh_parade_id = data;
break;
case DTV_ATSCMH_RS_FRAME_ENSEMBLE:
fe->dtv_property_cache.atscmh_rs_frame_ensemble = data;
break;
case DTV_LNA:
c->lna = data;
if (fe->ops.set_lna)
r = fe->ops.set_lna(fe);
if (r < 0)
c->lna = LNA_AUTO;
break;
default:
return -EINVAL;
}
return r;
}
static int dvb_frontend_do_ioctl(struct file *file, unsigned int cmd,
void *parg)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
int err;
dev_dbg(fe->dvb->device, "%s: (%d)\n", __func__, _IOC_NR(cmd));
if (down_interruptible(&fepriv->sem))
return -ERESTARTSYS;
if (fe->exit != DVB_FE_NO_EXIT) {
up(&fepriv->sem);
return -ENODEV;
}
/*
* If the frontend is opened in read-only mode, only the ioctls
* that don't interfere with the tune logic should be accepted.
* That allows an external application to monitor the DVB QoS and
* statistics parameters.
*
* That matches all _IOR() ioctls, except for two special cases:
* - FE_GET_EVENT is part of the tuning logic on a DVB application;
* - FE_DISEQC_RECV_SLAVE_REPLY is part of DiSEqC 2.0
* setup
* So, those two ioctls should also return -EPERM, as otherwise
* reading from them would interfere with a DVB tune application
*/
if ((file->f_flags & O_ACCMODE) == O_RDONLY
&& (_IOC_DIR(cmd) != _IOC_READ
|| cmd == FE_GET_EVENT
|| cmd == FE_DISEQC_RECV_SLAVE_REPLY)) {
up(&fepriv->sem);
return -EPERM;
}
err = dvb_frontend_handle_ioctl(file, cmd, parg);
up(&fepriv->sem);
return err;
}
static long dvb_frontend_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct dvb_device *dvbdev = file->private_data;
if (!dvbdev)
return -ENODEV;
return dvb_usercopy(file, cmd, arg, dvb_frontend_do_ioctl);
}
#ifdef CONFIG_COMPAT
struct compat_dtv_property {
__u32 cmd;
__u32 reserved[3];
union {
__u32 data;
struct dtv_fe_stats st;
struct {
__u8 data[32];
__u32 len;
__u32 reserved1[3];
compat_uptr_t reserved2;
} buffer;
} u;
int result;
} __attribute__ ((packed));
struct compat_dtv_properties {
__u32 num;
compat_uptr_t props;
};
#define COMPAT_FE_SET_PROPERTY _IOW('o', 82, struct compat_dtv_properties)
#define COMPAT_FE_GET_PROPERTY _IOR('o', 83, struct compat_dtv_properties)
static int dvb_frontend_handle_compat_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
int i, err = 0;
if (cmd == COMPAT_FE_SET_PROPERTY) {
struct compat_dtv_properties prop, *tvps = NULL;
struct compat_dtv_property *tvp = NULL;
if (copy_from_user(&prop, compat_ptr(arg), sizeof(prop)))
return -EFAULT;
tvps = ∝
/*
* Put an arbitrary limit on the number of messages that can
* be sent at once
*/
if (!tvps->num || (tvps->num > DTV_IOCTL_MAX_MSGS))
return -EINVAL;
tvp = memdup_array_user(compat_ptr(tvps->props),
tvps->num, sizeof(*tvp));
if (IS_ERR(tvp))
return PTR_ERR(tvp);
for (i = 0; i < tvps->num; i++) {
err = dtv_property_process_set(fe, file,
(tvp + i)->cmd,
(tvp + i)->u.data);
if (err < 0) {
kfree(tvp);
return err;
}
}
kfree(tvp);
} else if (cmd == COMPAT_FE_GET_PROPERTY) {
struct compat_dtv_properties prop, *tvps = NULL;
struct compat_dtv_property *tvp = NULL;
struct dtv_frontend_properties getp = fe->dtv_property_cache;
if (copy_from_user(&prop, compat_ptr(arg), sizeof(prop)))
return -EFAULT;
tvps = ∝
/*
* Put an arbitrary limit on the number of messages that can
* be sent at once
*/
if (!tvps->num || (tvps->num > DTV_IOCTL_MAX_MSGS))
return -EINVAL;
tvp = memdup_array_user(compat_ptr(tvps->props),
tvps->num, sizeof(*tvp));
if (IS_ERR(tvp))
return PTR_ERR(tvp);
/*
* Let's use our own copy of property cache, in order to
* avoid mangling with DTV zigzag logic, as drivers might
* return crap, if they don't check if the data is available
* before updating the properties cache.
*/
if (fepriv->state != FESTATE_IDLE) {
err = dtv_get_frontend(fe, &getp, NULL);
if (err < 0) {
kfree(tvp);
return err;
}
}
for (i = 0; i < tvps->num; i++) {
err = dtv_property_process_get(
fe, &getp, (struct dtv_property *)(tvp + i), file);
if (err < 0) {
kfree(tvp);
return err;
}
}
if (copy_to_user((void __user *)compat_ptr(tvps->props), tvp,
tvps->num * sizeof(struct compat_dtv_property))) {
kfree(tvp);
return -EFAULT;
}
kfree(tvp);
}
return err;
}
static long dvb_frontend_compat_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
int err;
if (cmd == COMPAT_FE_SET_PROPERTY || cmd == COMPAT_FE_GET_PROPERTY) {
if (down_interruptible(&fepriv->sem))
return -ERESTARTSYS;
err = dvb_frontend_handle_compat_ioctl(file, cmd, arg);
up(&fepriv->sem);
return err;
}
return dvb_frontend_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
}
#endif
static int dtv_set_frontend(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
u32 rolloff = 0;
if (dvb_frontend_check_parameters(fe) < 0)
return -EINVAL;
/*
* Initialize output parameters to match the values given by
* the user. FE_SET_FRONTEND triggers an initial frontend event
* with status = 0, which copies output parameters to userspace.
*/
dtv_property_legacy_params_sync(fe, c, &fepriv->parameters_out);
/*
* Be sure that the bandwidth will be filled for all
* non-satellite systems, as tuners need to know what
* low pass/Nyquist half filter should be applied, in
* order to avoid inter-channel noise.
*
* ISDB-T and DVB-T/T2 already sets bandwidth.
* ATSC and DVB-C don't set, so, the core should fill it.
*
* On DVB-C Annex A and C, the bandwidth is a function of
* the roll-off and symbol rate. Annex B defines different
* roll-off factors depending on the modulation. Fortunately,
* Annex B is only used with 6MHz, so there's no need to
* calculate it.
*
* While not officially supported, a side effect of handling it at
* the cache level is that a program could retrieve the bandwidth
* via DTV_BANDWIDTH_HZ, which may be useful for test programs.
*/
switch (c->delivery_system) {
case SYS_ATSC:
case SYS_DVBC_ANNEX_B:
c->bandwidth_hz = 6000000;
break;
case SYS_DVBC_ANNEX_A:
rolloff = 115;
break;
case SYS_DVBC_ANNEX_C:
rolloff = 113;
break;
case SYS_DSS:
rolloff = 120;
break;
case SYS_DVBS:
case SYS_TURBO:
case SYS_ISDBS:
rolloff = 135;
break;
case SYS_DVBS2:
switch (c->rolloff) {
case ROLLOFF_20:
rolloff = 120;
break;
case ROLLOFF_25:
rolloff = 125;
break;
default:
case ROLLOFF_35:
rolloff = 135;
}
break;
default:
break;
}
if (rolloff)
c->bandwidth_hz = mult_frac(c->symbol_rate, rolloff, 100);
/* force auto frequency inversion if requested */
if (dvb_force_auto_inversion)
c->inversion = INVERSION_AUTO;
/*
* without hierarchical coding code_rate_LP is irrelevant,
* so we tolerate the otherwise invalid FEC_NONE setting
*/
if (c->hierarchy == HIERARCHY_NONE && c->code_rate_LP == FEC_NONE)
c->code_rate_LP = FEC_AUTO;
prepare_tuning_algo_parameters(fe);
fepriv->state = FESTATE_RETUNE;
/* Request the search algorithm to search */
fepriv->algo_status |= DVBFE_ALGO_SEARCH_AGAIN;
dvb_frontend_clear_events(fe);
dvb_frontend_add_event(fe, 0);
dvb_frontend_wakeup(fe);
fepriv->status = 0;
return 0;
}
static int dvb_get_property(struct dvb_frontend *fe, struct file *file,
struct dtv_properties *tvps)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dtv_property *tvp = NULL;
struct dtv_frontend_properties getp;
int i, err;
memcpy(&getp, &fe->dtv_property_cache, sizeof(getp));
dev_dbg(fe->dvb->device, "%s: properties.num = %d\n",
__func__, tvps->num);
dev_dbg(fe->dvb->device, "%s: properties.props = %p\n",
__func__, tvps->props);
/*
* Put an arbitrary limit on the number of messages that can
* be sent at once
*/
if (!tvps->num || tvps->num > DTV_IOCTL_MAX_MSGS)
return -EINVAL;
tvp = memdup_array_user((void __user *)tvps->props,
tvps->num, sizeof(*tvp));
if (IS_ERR(tvp))
return PTR_ERR(tvp);
/*
* Let's use our own copy of property cache, in order to
* avoid mangling with DTV zigzag logic, as drivers might
* return crap, if they don't check if the data is available
* before updating the properties cache.
*/
if (fepriv->state != FESTATE_IDLE) {
err = dtv_get_frontend(fe, &getp, NULL);
if (err < 0)
goto out;
}
for (i = 0; i < tvps->num; i++) {
err = dtv_property_process_get(fe, &getp,
tvp + i, file);
if (err < 0)
goto out;
}
if (copy_to_user((void __user *)tvps->props, tvp,
tvps->num * sizeof(struct dtv_property))) {
err = -EFAULT;
goto out;
}
err = 0;
out:
kfree(tvp);
return err;
}
static int dvb_get_frontend(struct dvb_frontend *fe,
struct dvb_frontend_parameters *p_out)
{
struct dtv_frontend_properties getp;
/*
* Let's use our own copy of property cache, in order to
* avoid mangling with DTV zigzag logic, as drivers might
* return crap, if they don't check if the data is available
* before updating the properties cache.
*/
memcpy(&getp, &fe->dtv_property_cache, sizeof(getp));
return dtv_get_frontend(fe, &getp, p_out);
}
static int dvb_frontend_handle_ioctl(struct file *file,
unsigned int cmd, void *parg)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dtv_frontend_properties *c = &fe->dtv_property_cache;
int i, err = -ENOTSUPP;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
switch (cmd) {
case FE_SET_PROPERTY: {
struct dtv_properties *tvps = parg;
struct dtv_property *tvp = NULL;
dev_dbg(fe->dvb->device, "%s: properties.num = %d\n",
__func__, tvps->num);
dev_dbg(fe->dvb->device, "%s: properties.props = %p\n",
__func__, tvps->props);
/*
* Put an arbitrary limit on the number of messages that can
* be sent at once
*/
if (!tvps->num || (tvps->num > DTV_IOCTL_MAX_MSGS))
return -EINVAL;
tvp = memdup_array_user((void __user *)tvps->props,
tvps->num, sizeof(*tvp));
if (IS_ERR(tvp))
return PTR_ERR(tvp);
for (i = 0; i < tvps->num; i++) {
err = dtv_property_process_set(fe, file,
(tvp + i)->cmd,
(tvp + i)->u.data);
if (err < 0) {
kfree(tvp);
return err;
}
}
kfree(tvp);
err = 0;
break;
}
case FE_GET_PROPERTY:
err = dvb_get_property(fe, file, parg);
break;
case FE_GET_INFO: {
struct dvb_frontend_info *info = parg;
memset(info, 0, sizeof(*info));
strscpy(info->name, fe->ops.info.name, sizeof(info->name));
info->symbol_rate_min = fe->ops.info.symbol_rate_min;
info->symbol_rate_max = fe->ops.info.symbol_rate_max;
info->symbol_rate_tolerance = fe->ops.info.symbol_rate_tolerance;
info->caps = fe->ops.info.caps;
info->frequency_stepsize = dvb_frontend_get_stepsize(fe);
dvb_frontend_get_frequency_limits(fe, &info->frequency_min,
&info->frequency_max,
&info->frequency_tolerance);
/*
* Associate the 4 delivery systems supported by DVBv3
* API with their DVBv5 counterpart. For the other standards,
* use the closest type, assuming that it would hopefully
* work with a DVBv3 application.
* It should be noticed that, on multi-frontend devices with
* different types (terrestrial and cable, for example),
* a pure DVBv3 application won't be able to use all delivery
* systems. Yet, changing the DVBv5 cache to the other delivery
* system should be enough for making it work.
*/
switch (dvbv3_type(c->delivery_system)) {
case DVBV3_QPSK:
info->type = FE_QPSK;
break;
case DVBV3_ATSC:
info->type = FE_ATSC;
break;
case DVBV3_QAM:
info->type = FE_QAM;
break;
case DVBV3_OFDM:
info->type = FE_OFDM;
break;
default:
dev_err(fe->dvb->device,
"%s: doesn't know how to handle a DVBv3 call to delivery system %i\n",
__func__, c->delivery_system);
info->type = FE_OFDM;
}
dev_dbg(fe->dvb->device, "%s: current delivery system on cache: %d, V3 type: %d\n",
__func__, c->delivery_system, info->type);
/* Set CAN_INVERSION_AUTO bit on in other than oneshot mode */
if (!(fepriv->tune_mode_flags & FE_TUNE_MODE_ONESHOT))
info->caps |= FE_CAN_INVERSION_AUTO;
err = 0;
break;
}
case FE_READ_STATUS: {
enum fe_status *status = parg;
/* if retune was requested but hasn't occurred yet, prevent
* that user get signal state from previous tuning */
if (fepriv->state == FESTATE_RETUNE ||
fepriv->state == FESTATE_ERROR) {
err = 0;
*status = 0;
break;
}
if (fe->ops.read_status)
err = fe->ops.read_status(fe, status);
break;
}
case FE_DISEQC_RESET_OVERLOAD:
if (fe->ops.diseqc_reset_overload) {
err = fe->ops.diseqc_reset_overload(fe);
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
}
break;
case FE_DISEQC_SEND_MASTER_CMD:
if (fe->ops.diseqc_send_master_cmd) {
struct dvb_diseqc_master_cmd *cmd = parg;
if (cmd->msg_len > sizeof(cmd->msg)) {
err = -EINVAL;
break;
}
err = fe->ops.diseqc_send_master_cmd(fe, cmd);
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
}
break;
case FE_DISEQC_SEND_BURST:
if (fe->ops.diseqc_send_burst) {
err = fe->ops.diseqc_send_burst(fe, (long)parg);
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
}
break;
case FE_SET_TONE:
if (fe->ops.set_tone) {
fepriv->tone = (long)parg;
err = fe->ops.set_tone(fe, fepriv->tone);
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
}
break;
case FE_SET_VOLTAGE:
if (fe->ops.set_voltage) {
fepriv->voltage = (long)parg;
err = fe->ops.set_voltage(fe, fepriv->voltage);
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
}
break;
case FE_DISEQC_RECV_SLAVE_REPLY:
if (fe->ops.diseqc_recv_slave_reply)
err = fe->ops.diseqc_recv_slave_reply(fe, parg);
break;
case FE_ENABLE_HIGH_LNB_VOLTAGE:
if (fe->ops.enable_high_lnb_voltage)
err = fe->ops.enable_high_lnb_voltage(fe, (long)parg);
break;
case FE_SET_FRONTEND_TUNE_MODE:
fepriv->tune_mode_flags = (unsigned long)parg;
err = 0;
break;
/* DEPRECATED dish control ioctls */
case FE_DISHNETWORK_SEND_LEGACY_CMD:
if (fe->ops.dishnetwork_send_legacy_command) {
err = fe->ops.dishnetwork_send_legacy_command(fe,
(unsigned long)parg);
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
} else if (fe->ops.set_voltage) {
/*
* NOTE: This is a fallback condition. Some frontends
* (stv0299 for instance) take longer than 8msec to
* respond to a set_voltage command. Those switches
* need custom routines to switch properly. For all
* other frontends, the following should work ok.
* Dish network legacy switches (as used by Dish500)
* are controlled by sending 9-bit command words
* spaced 8msec apart.
* the actual command word is switch/port dependent
* so it is up to the userspace application to send
* the right command.
* The command must always start with a '0' after
* initialization, so parg is 8 bits and does not
* include the initialization or start bit
*/
unsigned long swcmd = ((unsigned long)parg) << 1;
ktime_t nexttime;
ktime_t tv[10];
int i;
u8 last = 1;
if (dvb_frontend_debug)
dprintk("switch command: 0x%04lx\n",
swcmd);
nexttime = ktime_get_boottime();
if (dvb_frontend_debug)
tv[0] = nexttime;
/* before sending a command, initialize by sending
* a 32ms 18V to the switch
*/
fe->ops.set_voltage(fe, SEC_VOLTAGE_18);
dvb_frontend_sleep_until(&nexttime, 32000);
for (i = 0; i < 9; i++) {
if (dvb_frontend_debug)
tv[i + 1] = ktime_get_boottime();
if ((swcmd & 0x01) != last) {
/* set voltage to (last ? 13V : 18V) */
fe->ops.set_voltage(fe, (last) ? SEC_VOLTAGE_13 : SEC_VOLTAGE_18);
last = (last) ? 0 : 1;
}
swcmd = swcmd >> 1;
if (i != 8)
dvb_frontend_sleep_until(&nexttime, 8000);
}
if (dvb_frontend_debug) {
dprintk("(adapter %d): switch delay (should be 32k followed by all 8k)\n",
fe->dvb->num);
for (i = 1; i < 10; i++)
pr_info("%d: %d\n", i,
(int)ktime_us_delta(tv[i], tv[i - 1]));
}
err = 0;
fepriv->state = FESTATE_DISEQC;
fepriv->status = 0;
}
break;
/* DEPRECATED statistics ioctls */
case FE_READ_BER:
if (fe->ops.read_ber) {
if (fepriv->thread)
err = fe->ops.read_ber(fe, parg);
else
err = -EAGAIN;
}
break;
case FE_READ_SIGNAL_STRENGTH:
if (fe->ops.read_signal_strength) {
if (fepriv->thread)
err = fe->ops.read_signal_strength(fe, parg);
else
err = -EAGAIN;
}
break;
case FE_READ_SNR:
if (fe->ops.read_snr) {
if (fepriv->thread)
err = fe->ops.read_snr(fe, parg);
else
err = -EAGAIN;
}
break;
case FE_READ_UNCORRECTED_BLOCKS:
if (fe->ops.read_ucblocks) {
if (fepriv->thread)
err = fe->ops.read_ucblocks(fe, parg);
else
err = -EAGAIN;
}
break;
/* DEPRECATED DVBv3 ioctls */
case FE_SET_FRONTEND:
err = dvbv3_set_delivery_system(fe);
if (err)
break;
err = dtv_property_cache_sync(fe, c, parg);
if (err)
break;
err = dtv_set_frontend(fe);
break;
case FE_GET_EVENT:
err = dvb_frontend_get_event(fe, parg, file->f_flags);
break;
case FE_GET_FRONTEND:
err = dvb_get_frontend(fe, parg);
break;
default:
return -ENOTSUPP;
} /* switch */
return err;
}
static __poll_t dvb_frontend_poll(struct file *file, struct poll_table_struct *wait)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
dev_dbg_ratelimited(fe->dvb->device, "%s:\n", __func__);
poll_wait(file, &fepriv->events.wait_queue, wait);
if (fepriv->events.eventw != fepriv->events.eventr)
return (EPOLLIN | EPOLLRDNORM | EPOLLPRI);
return 0;
}
static int dvb_frontend_open(struct inode *inode, struct file *file)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
struct dvb_adapter *adapter = fe->dvb;
int ret;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
if (fe->exit == DVB_FE_DEVICE_REMOVED)
return -ENODEV;
if (adapter->mfe_shared == 2) {
mutex_lock(&adapter->mfe_lock);
if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
if (adapter->mfe_dvbdev &&
!adapter->mfe_dvbdev->writers) {
mutex_unlock(&adapter->mfe_lock);
return -EBUSY;
}
adapter->mfe_dvbdev = dvbdev;
}
} else if (adapter->mfe_shared) {
mutex_lock(&adapter->mfe_lock);
if (!adapter->mfe_dvbdev)
adapter->mfe_dvbdev = dvbdev;
else if (adapter->mfe_dvbdev != dvbdev) {
struct dvb_device
*mfedev = adapter->mfe_dvbdev;
struct dvb_frontend
*mfe = mfedev->priv;
struct dvb_frontend_private
*mfepriv = mfe->frontend_priv;
int mferetry = (dvb_mfe_wait_time << 1);
mutex_unlock(&adapter->mfe_lock);
while (mferetry-- && (mfedev->users != -1 ||
mfepriv->thread)) {
if (msleep_interruptible(500)) {
if (signal_pending(current))
return -EINTR;
}
}
mutex_lock(&adapter->mfe_lock);
if (adapter->mfe_dvbdev != dvbdev) {
mfedev = adapter->mfe_dvbdev;
mfe = mfedev->priv;
mfepriv = mfe->frontend_priv;
if (mfedev->users != -1 ||
mfepriv->thread) {
mutex_unlock(&adapter->mfe_lock);
return -EBUSY;
}
adapter->mfe_dvbdev = dvbdev;
}
}
}
if (dvbdev->users == -1 && fe->ops.ts_bus_ctrl) {
if ((ret = fe->ops.ts_bus_ctrl(fe, 1)) < 0)
goto err0;
/* If we took control of the bus, we need to force
reinitialization. This is because many ts_bus_ctrl()
functions strobe the RESET pin on the demod, and if the
frontend thread already exists then the dvb_init() routine
won't get called (which is what usually does initial
register configuration). */
fepriv->reinitialise = 1;
}
if ((ret = dvb_generic_open(inode, file)) < 0)
goto err1;
if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
/* normal tune mode when opened R/W */
fepriv->tune_mode_flags &= ~FE_TUNE_MODE_ONESHOT;
fepriv->tone = -1;
fepriv->voltage = -1;
#ifdef CONFIG_MEDIA_CONTROLLER_DVB
mutex_lock(&fe->dvb->mdev_lock);
if (fe->dvb->mdev) {
mutex_lock(&fe->dvb->mdev->graph_mutex);
if (fe->dvb->mdev->enable_source)
ret = fe->dvb->mdev->enable_source(
dvbdev->entity,
&fepriv->pipe);
mutex_unlock(&fe->dvb->mdev->graph_mutex);
if (ret) {
mutex_unlock(&fe->dvb->mdev_lock);
dev_err(fe->dvb->device,
"Tuner is busy. Error %d\n", ret);
goto err2;
}
}
mutex_unlock(&fe->dvb->mdev_lock);
#endif
ret = dvb_frontend_start(fe);
if (ret)
goto err3;
/* empty event queue */
fepriv->events.eventr = fepriv->events.eventw = 0;
}
dvb_frontend_get(fe);
if (adapter->mfe_shared)
mutex_unlock(&adapter->mfe_lock);
return ret;
err3:
#ifdef CONFIG_MEDIA_CONTROLLER_DVB
mutex_lock(&fe->dvb->mdev_lock);
if (fe->dvb->mdev) {
mutex_lock(&fe->dvb->mdev->graph_mutex);
if (fe->dvb->mdev->disable_source)
fe->dvb->mdev->disable_source(dvbdev->entity);
mutex_unlock(&fe->dvb->mdev->graph_mutex);
}
mutex_unlock(&fe->dvb->mdev_lock);
err2:
#endif
dvb_generic_release(inode, file);
err1:
if (dvbdev->users == -1 && fe->ops.ts_bus_ctrl)
fe->ops.ts_bus_ctrl(fe, 0);
err0:
if (adapter->mfe_shared)
mutex_unlock(&adapter->mfe_lock);
return ret;
}
static int dvb_frontend_release(struct inode *inode, struct file *file)
{
struct dvb_device *dvbdev = file->private_data;
struct dvb_frontend *fe = dvbdev->priv;
struct dvb_frontend_private *fepriv = fe->frontend_priv;
int ret;
dev_dbg(fe->dvb->device, "%s:\n", __func__);
if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
fepriv->release_jiffies = jiffies;
mb();
}
ret = dvb_generic_release(inode, file);
if (dvbdev->users == -1) {
wake_up(&fepriv->wait_queue);
#ifdef CONFIG_MEDIA_CONTROLLER_DVB
mutex_lock(&fe->dvb->mdev_lock);
if (fe->dvb->mdev) {
mutex_lock(&fe->dvb->mdev->graph_mutex);
if (fe->dvb->mdev->disable_source)
fe->dvb->mdev->disable_source(dvbdev->entity);
mutex_unlock(&fe->dvb->mdev->graph_mutex);
}
mutex_unlock(&fe->dvb->mdev_lock);
#endif
if (fe->exit != DVB_FE_NO_EXIT)
wake_up(&dvbdev->wait_queue);
if (fe->ops.ts_bus_ctrl)
fe->ops.ts_bus_ctrl(fe, 0);
}
dvb_frontend_put(fe);
return ret;
}
static const struct file_operations dvb_frontend_fops = {
.owner = THIS_MODULE,
.unlocked_ioctl = dvb_frontend_ioctl,
#ifdef CONFIG_COMPAT
.compat_ioctl = dvb_frontend_compat_ioctl,
#endif
.poll = dvb_frontend_poll,
.open = dvb_frontend_open,
.release = dvb_frontend_release,
.llseek = noop_llseek,
};
int dvb_frontend_suspend(struct dvb_frontend *fe)
{
int ret = 0;
dev_dbg(fe->dvb->device, "%s: adap=%d fe=%d\n", __func__, fe->dvb->num,
fe->id);
if (fe->ops.tuner_ops.suspend)
ret = fe->ops.tuner_ops.suspend(fe);
else if (fe->ops.tuner_ops.sleep)
ret = fe->ops.tuner_ops.sleep(fe);
if (fe->ops.suspend)
ret = fe->ops.suspend(fe);
else if (fe->ops.sleep)
ret = fe->ops.sleep(fe);
return ret;
}
EXPORT_SYMBOL(dvb_frontend_suspend);
int dvb_frontend_resume(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv;
int ret = 0;
dev_dbg(fe->dvb->device, "%s: adap=%d fe=%d\n", __func__, fe->dvb->num,
fe->id);
fe->exit = DVB_FE_DEVICE_RESUME;
if (fe->ops.resume)
ret = fe->ops.resume(fe);
else if (fe->ops.init)
ret = fe->ops.init(fe);
if (fe->ops.tuner_ops.resume)
ret = fe->ops.tuner_ops.resume(fe);
else if (fe->ops.tuner_ops.init)
ret = fe->ops.tuner_ops.init(fe);
if (fe->ops.set_tone && fepriv->tone != -1)
fe->ops.set_tone(fe, fepriv->tone);
if (fe->ops.set_voltage && fepriv->voltage != -1)
fe->ops.set_voltage(fe, fepriv->voltage);
fe->exit = DVB_FE_NO_EXIT;
fepriv->state = FESTATE_RETUNE;
dvb_frontend_wakeup(fe);
return ret;
}
EXPORT_SYMBOL(dvb_frontend_resume);
int dvb_register_frontend(struct dvb_adapter *dvb,
struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv;
const struct dvb_device dvbdev_template = {
.users = ~0,
.writers = 1,
.readers = (~0) - 1,
.fops = &dvb_frontend_fops,
#if defined(CONFIG_MEDIA_CONTROLLER_DVB)
.name = fe->ops.info.name,
#endif
};
int ret;
dev_dbg(dvb->device, "%s:\n", __func__);
if (mutex_lock_interruptible(&frontend_mutex))
return -ERESTARTSYS;
fe->frontend_priv = kzalloc(sizeof(struct dvb_frontend_private), GFP_KERNEL);
if (!fe->frontend_priv) {
mutex_unlock(&frontend_mutex);
return -ENOMEM;
}
fepriv = fe->frontend_priv;
kref_init(&fe->refcount);
/*
* After initialization, there need to be two references: one
* for dvb_unregister_frontend(), and another one for
* dvb_frontend_detach().
*/
dvb_frontend_get(fe);
sema_init(&fepriv->sem, 1);
init_waitqueue_head(&fepriv->wait_queue);
init_waitqueue_head(&fepriv->events.wait_queue);
mutex_init(&fepriv->events.mtx);
fe->dvb = dvb;
fepriv->inversion = INVERSION_OFF;
dev_info(fe->dvb->device,
"DVB: registering adapter %i frontend %i (%s)...\n",
fe->dvb->num, fe->id, fe->ops.info.name);
ret = dvb_register_device(fe->dvb, &fepriv->dvbdev, &dvbdev_template,
fe, DVB_DEVICE_FRONTEND, 0);
if (ret) {
dvb_frontend_put(fe);
mutex_unlock(&frontend_mutex);
return ret;
}
/*
* Initialize the cache to the proper values according with the
* first supported delivery system (ops->delsys[0])
*/
fe->dtv_property_cache.delivery_system = fe->ops.delsys[0];
dvb_frontend_clear_cache(fe);
mutex_unlock(&frontend_mutex);
return 0;
}
EXPORT_SYMBOL(dvb_register_frontend);
int dvb_unregister_frontend(struct dvb_frontend *fe)
{
struct dvb_frontend_private *fepriv = fe->frontend_priv; dev_dbg(fe->dvb->device, "%s:\n", __func__); mutex_lock(&frontend_mutex);
dvb_frontend_stop(fe);
dvb_remove_device(fepriv->dvbdev);
/* fe is invalid now */
mutex_unlock(&frontend_mutex);
dvb_frontend_put(fe);
return 0;
}
EXPORT_SYMBOL(dvb_unregister_frontend);
static void dvb_frontend_invoke_release(struct dvb_frontend *fe,
void (*release)(struct dvb_frontend *fe))
{
if (release) {
release(fe);
#ifdef CONFIG_MEDIA_ATTACH
dvb_detach(release);
#endif
}
}
void dvb_frontend_detach(struct dvb_frontend *fe)
{
dvb_frontend_invoke_release(fe, fe->ops.release_sec); dvb_frontend_invoke_release(fe, fe->ops.tuner_ops.release); dvb_frontend_invoke_release(fe, fe->ops.analog_ops.release); dvb_frontend_put(fe);
}
EXPORT_SYMBOL(dvb_frontend_detach);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk).
*
* (C) SGI 2006, Christoph Lameter
* Cleaned up and restructured to ease the addition of alternative
* implementations of SLAB allocators.
* (C) Linux Foundation 2008-2013
* Unified interface for all slab allocators
*/
#ifndef _LINUX_SLAB_H
#define _LINUX_SLAB_H
#include <linux/cache.h>
#include <linux/gfp.h>
#include <linux/overflow.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <linux/percpu-refcount.h>
#include <linux/cleanup.h>
#include <linux/hash.h>
enum _slab_flag_bits {
_SLAB_CONSISTENCY_CHECKS,
_SLAB_RED_ZONE,
_SLAB_POISON,
_SLAB_KMALLOC,
_SLAB_HWCACHE_ALIGN,
_SLAB_CACHE_DMA,
_SLAB_CACHE_DMA32,
_SLAB_STORE_USER,
_SLAB_PANIC,
_SLAB_TYPESAFE_BY_RCU,
_SLAB_TRACE,
#ifdef CONFIG_DEBUG_OBJECTS
_SLAB_DEBUG_OBJECTS,
#endif
_SLAB_NOLEAKTRACE,
_SLAB_NO_MERGE,
#ifdef CONFIG_FAILSLAB
_SLAB_FAILSLAB,
#endif
#ifdef CONFIG_MEMCG_KMEM
_SLAB_ACCOUNT,
#endif
#ifdef CONFIG_KASAN_GENERIC
_SLAB_KASAN,
#endif
_SLAB_NO_USER_FLAGS,
#ifdef CONFIG_KFENCE
_SLAB_SKIP_KFENCE,
#endif
#ifndef CONFIG_SLUB_TINY
_SLAB_RECLAIM_ACCOUNT,
#endif
_SLAB_OBJECT_POISON,
_SLAB_CMPXCHG_DOUBLE,
_SLAB_FLAGS_LAST_BIT
};
#define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr)))
#define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U))
/*
* Flags to pass to kmem_cache_create().
* The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op
*/
/* DEBUG: Perform (expensive) checks on alloc/free */
#define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS)
/* DEBUG: Red zone objs in a cache */
#define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE)
/* DEBUG: Poison objects */
#define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON)
/* Indicate a kmalloc slab */
#define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC)
/* Align objs on cache lines */
#define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN)
/* Use GFP_DMA memory */
#define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA)
/* Use GFP_DMA32 memory */
#define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32)
/* DEBUG: Store the last owner for bug hunting */
#define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER)
/* Panic if kmem_cache_create() fails */
#define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC)
/*
* SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS!
*
* This delays freeing the SLAB page by a grace period, it does _NOT_
* delay object freeing. This means that if you do kmem_cache_free()
* that memory location is free to be reused at any time. Thus it may
* be possible to see another object there in the same RCU grace period.
*
* This feature only ensures the memory location backing the object
* stays valid, the trick to using this is relying on an independent
* object validation pass. Something like:
*
* begin:
* rcu_read_lock();
* obj = lockless_lookup(key);
* if (obj) {
* if (!try_get_ref(obj)) // might fail for free objects
* rcu_read_unlock();
* goto begin;
*
* if (obj->key != key) { // not the object we expected
* put_ref(obj);
* rcu_read_unlock();
* goto begin;
* }
* }
* rcu_read_unlock();
*
* This is useful if we need to approach a kernel structure obliquely,
* from its address obtained without the usual locking. We can lock
* the structure to stabilize it and check it's still at the given address,
* only if we can be sure that the memory has not been meanwhile reused
* for some other kind of object (which our subsystem's lock might corrupt).
*
* rcu_read_lock before reading the address, then rcu_read_unlock after
* taking the spinlock within the structure expected at that address.
*
* Note that it is not possible to acquire a lock within a structure
* allocated with SLAB_TYPESAFE_BY_RCU without first acquiring a reference
* as described above. The reason is that SLAB_TYPESAFE_BY_RCU pages
* are not zeroed before being given to the slab, which means that any
* locks must be initialized after each and every kmem_struct_alloc().
* Alternatively, make the ctor passed to kmem_cache_create() initialize
* the locks at page-allocation time, as is done in __i915_request_ctor(),
* sighand_ctor(), and anon_vma_ctor(). Such a ctor permits readers
* to safely acquire those ctor-initialized locks under rcu_read_lock()
* protection.
*
* Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU.
*/
/* Defer freeing slabs to RCU */
#define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU)
/* Trace allocations and frees */
#define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE)
/* Flag to prevent checks on free */
#ifdef CONFIG_DEBUG_OBJECTS
# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS)
#else
# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED
#endif
/* Avoid kmemleak tracing */
#define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE)
/*
* Prevent merging with compatible kmem caches. This flag should be used
* cautiously. Valid use cases:
*
* - caches created for self-tests (e.g. kunit)
* - general caches created and used by a subsystem, only when a
* (subsystem-specific) debug option is enabled
* - performance critical caches, should be very rare and consulted with slab
* maintainers, and not used together with CONFIG_SLUB_TINY
*/
#define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE)
/* Fault injection mark */
#ifdef CONFIG_FAILSLAB
# define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB)
#else
# define SLAB_FAILSLAB __SLAB_FLAG_UNUSED
#endif
/* Account to memcg */
#ifdef CONFIG_MEMCG_KMEM
# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT)
#else
# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED
#endif
#ifdef CONFIG_KASAN_GENERIC
#define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN)
#else
#define SLAB_KASAN __SLAB_FLAG_UNUSED
#endif
/*
* Ignore user specified debugging flags.
* Intended for caches created for self-tests so they have only flags
* specified in the code and other flags are ignored.
*/
#define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS)
#ifdef CONFIG_KFENCE
#define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE)
#else
#define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED
#endif
/* The following flags affect the page allocator grouping pages by mobility */
/* Objects are reclaimable */
#ifndef CONFIG_SLUB_TINY
#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT)
#else
#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED
#endif
#define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
/*
* ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
*
* Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
*
* ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can.
* Both make kfree a no-op.
*/
#define ZERO_SIZE_PTR ((void *)16)
#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \
(unsigned long)ZERO_SIZE_PTR)
#include <linux/kasan.h>
struct list_lru;
struct mem_cgroup;
/*
* struct kmem_cache related prototypes
*/
bool slab_is_available(void);
struct kmem_cache *kmem_cache_create(const char *name, unsigned int size,
unsigned int align, slab_flags_t flags,
void (*ctor)(void *));
struct kmem_cache *kmem_cache_create_usercopy(const char *name,
unsigned int size, unsigned int align,
slab_flags_t flags,
unsigned int useroffset, unsigned int usersize,
void (*ctor)(void *));
void kmem_cache_destroy(struct kmem_cache *s);
int kmem_cache_shrink(struct kmem_cache *s);
/*
* Please use this macro to create slab caches. Simply specify the
* name of the structure and maybe some flags that are listed above.
*
* The alignment of the struct determines object alignment. If you
* f.e. add ____cacheline_aligned_in_smp to the struct declaration
* then the objects will be properly aligned in SMP configurations.
*/
#define KMEM_CACHE(__struct, __flags) \
kmem_cache_create(#__struct, sizeof(struct __struct), \
__alignof__(struct __struct), (__flags), NULL)
/*
* To whitelist a single field for copying to/from usercopy, use this
* macro instead for KMEM_CACHE() above.
*/
#define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \
kmem_cache_create_usercopy(#__struct, \
sizeof(struct __struct), \
__alignof__(struct __struct), (__flags), \
offsetof(struct __struct, __field), \
sizeof_field(struct __struct, __field), NULL)
/*
* Common kmalloc functions provided by all allocators
*/
void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2);
void kfree(const void *objp);
void kfree_sensitive(const void *objp);
size_t __ksize(const void *objp);
DEFINE_FREE(kfree, void *, if (_T) kfree(_T))
/**
* ksize - Report actual allocation size of associated object
*
* @objp: Pointer returned from a prior kmalloc()-family allocation.
*
* This should not be used for writing beyond the originally requested
* allocation size. Either use krealloc() or round up the allocation size
* with kmalloc_size_roundup() prior to allocation. If this is used to
* access beyond the originally requested allocation size, UBSAN_BOUNDS
* and/or FORTIFY_SOURCE may trip, since they only know about the
* originally allocated size via the __alloc_size attribute.
*/
size_t ksize(const void *objp);
#ifdef CONFIG_PRINTK
bool kmem_dump_obj(void *object);
#else
static inline bool kmem_dump_obj(void *object) { return false; }
#endif
/*
* Some archs want to perform DMA into kmalloc caches and need a guaranteed
* alignment larger than the alignment of a 64-bit integer.
* Setting ARCH_DMA_MINALIGN in arch headers allows that.
*/
#ifdef ARCH_HAS_DMA_MINALIGN
#if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN)
#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
#endif
#endif
#ifndef ARCH_KMALLOC_MINALIGN
#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
#elif ARCH_KMALLOC_MINALIGN > 8
#define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
#endif
/*
* Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment.
* Intended for arches that get misalignment faults even for 64 bit integer
* aligned buffers.
*/
#ifndef ARCH_SLAB_MINALIGN
#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
#endif
/*
* Arches can define this function if they want to decide the minimum slab
* alignment at runtime. The value returned by the function must be a power
* of two and >= ARCH_SLAB_MINALIGN.
*/
#ifndef arch_slab_minalign
static inline unsigned int arch_slab_minalign(void)
{
return ARCH_SLAB_MINALIGN;
}
#endif
/*
* kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN.
* kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN
* and ARCH_SLAB_MINALIGN, but here we only assume the former alignment.
*/
#define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN)
#define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN)
#define __assume_page_alignment __assume_aligned(PAGE_SIZE)
/*
* Kmalloc array related definitions
*/
/*
* SLUB directly allocates requests fitting in to an order-1 page
* (PAGE_SIZE*2). Larger requests are passed to the page allocator.
*/
#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
#define KMALLOC_SHIFT_MAX (MAX_PAGE_ORDER + PAGE_SHIFT)
#ifndef KMALLOC_SHIFT_LOW
#define KMALLOC_SHIFT_LOW 3
#endif
/* Maximum allocatable size */
#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
/* Maximum size for which we actually use a slab cache */
#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
/* Maximum order allocatable via the slab allocator */
#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
/*
* Kmalloc subsystem.
*/
#ifndef KMALLOC_MIN_SIZE
#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
#endif
/*
* This restriction comes from byte sized index implementation.
* Page size is normally 2^12 bytes and, in this case, if we want to use
* byte sized index which can represent 2^8 entries, the size of the object
* should be equal or greater to 2^12 / 2^8 = 2^4 = 16.
* If minimum size of kmalloc is less than 16, we use it as minimum object
* size and give up to use byte sized index.
*/
#define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \
(KMALLOC_MIN_SIZE) : 16)
#ifdef CONFIG_RANDOM_KMALLOC_CACHES
#define RANDOM_KMALLOC_CACHES_NR 15 // # of cache copies
#else
#define RANDOM_KMALLOC_CACHES_NR 0
#endif
/*
* Whenever changing this, take care of that kmalloc_type() and
* create_kmalloc_caches() still work as intended.
*
* KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
* is for accounted but unreclaimable and non-dma objects. All the other
* kmem caches can have both accounted and unaccounted objects.
*/
enum kmalloc_cache_type {
KMALLOC_NORMAL = 0,
#ifndef CONFIG_ZONE_DMA
KMALLOC_DMA = KMALLOC_NORMAL,
#endif
#ifndef CONFIG_MEMCG_KMEM
KMALLOC_CGROUP = KMALLOC_NORMAL,
#endif
KMALLOC_RANDOM_START = KMALLOC_NORMAL,
KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
#ifdef CONFIG_SLUB_TINY
KMALLOC_RECLAIM = KMALLOC_NORMAL,
#else
KMALLOC_RECLAIM,
#endif
#ifdef CONFIG_ZONE_DMA
KMALLOC_DMA,
#endif
#ifdef CONFIG_MEMCG_KMEM
KMALLOC_CGROUP,
#endif
NR_KMALLOC_TYPES
};
extern struct kmem_cache *
kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1];
/*
* Define gfp bits that should not be set for KMALLOC_NORMAL.
*/
#define KMALLOC_NOT_NORMAL_BITS \
(__GFP_RECLAIMABLE | \
(IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \
(IS_ENABLED(CONFIG_MEMCG_KMEM) ? __GFP_ACCOUNT : 0))
extern unsigned long random_kmalloc_seed;
static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, unsigned long caller)
{
/*
* The most common case is KMALLOC_NORMAL, so test for it
* with a single branch for all the relevant flags.
*/
if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
#ifdef CONFIG_RANDOM_KMALLOC_CACHES
/* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed,
ilog2(RANDOM_KMALLOC_CACHES_NR + 1));
#else
return KMALLOC_NORMAL;
#endif
/*
* At least one of the flags has to be set. Their priorities in
* decreasing order are:
* 1) __GFP_DMA
* 2) __GFP_RECLAIMABLE
* 3) __GFP_ACCOUNT
*/
if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA))
return KMALLOC_DMA;
if (!IS_ENABLED(CONFIG_MEMCG_KMEM) || (flags & __GFP_RECLAIMABLE))
return KMALLOC_RECLAIM;
else
return KMALLOC_CGROUP;
}
/*
* Figure out which kmalloc slab an allocation of a certain size
* belongs to.
* 0 = zero alloc
* 1 = 65 .. 96 bytes
* 2 = 129 .. 192 bytes
* n = 2^(n-1)+1 .. 2^n
*
* Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
* typical usage is via kmalloc_index() and therefore evaluated at compile-time.
* Callers where !size_is_constant should only be test modules, where runtime
* overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab().
*/
static __always_inline unsigned int __kmalloc_index(size_t size,
bool size_is_constant)
{
if (!size)
return 0;
if (size <= KMALLOC_MIN_SIZE)
return KMALLOC_SHIFT_LOW;
if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
return 1;
if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
return 2;
if (size <= 8) return 3;
if (size <= 16) return 4;
if (size <= 32) return 5;
if (size <= 64) return 6;
if (size <= 128) return 7;
if (size <= 256) return 8;
if (size <= 512) return 9;
if (size <= 1024) return 10;
if (size <= 2 * 1024) return 11;
if (size <= 4 * 1024) return 12;
if (size <= 8 * 1024) return 13;
if (size <= 16 * 1024) return 14;
if (size <= 32 * 1024) return 15;
if (size <= 64 * 1024) return 16;
if (size <= 128 * 1024) return 17;
if (size <= 256 * 1024) return 18;
if (size <= 512 * 1024) return 19;
if (size <= 1024 * 1024) return 20;
if (size <= 2 * 1024 * 1024) return 21;
if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
else
BUG();
/* Will never be reached. Needed because the compiler may complain */
return -1;
}
static_assert(PAGE_SHIFT <= 20);
#define kmalloc_index(s) __kmalloc_index(s, true)
void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1);
/**
* kmem_cache_alloc - Allocate an object
* @cachep: The cache to allocate from.
* @flags: See kmalloc().
*
* Allocate an object from this cache.
* See kmem_cache_zalloc() for a shortcut of adding __GFP_ZERO to flags.
*
* Return: pointer to the new object or %NULL in case of error
*/
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) __assume_slab_alignment __malloc;
void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags) __assume_slab_alignment __malloc;
void kmem_cache_free(struct kmem_cache *s, void *objp);
/*
* Bulk allocation and freeing operations. These are accelerated in an
* allocator specific way to avoid taking locks repeatedly or building
* metadata structures unnecessarily.
*
* Note that interrupts must be enabled when calling these functions.
*/
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
static __always_inline void kfree_bulk(size_t size, void **p)
{
kmem_cache_free_bulk(NULL, size, p);
}
void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment
__alloc_size(1);
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment
__malloc;
void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)
__assume_kmalloc_alignment __alloc_size(3);
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
int node, size_t size) __assume_kmalloc_alignment
__alloc_size(4);
void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment
__alloc_size(1);
void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment
__alloc_size(1);
/**
* kmalloc - allocate kernel memory
* @size: how many bytes of memory are required.
* @flags: describe the allocation context
*
* kmalloc is the normal method of allocating memory
* for objects smaller than page size in the kernel.
*
* The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
* bytes. For @size of power of two bytes, the alignment is also guaranteed
* to be at least to the size.
*
* The @flags argument may be one of the GFP flags defined at
* include/linux/gfp_types.h and described at
* :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
*
* The recommended usage of the @flags is described at
* :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
*
* Below is a brief outline of the most useful GFP flags
*
* %GFP_KERNEL
* Allocate normal kernel ram. May sleep.
*
* %GFP_NOWAIT
* Allocation will not sleep.
*
* %GFP_ATOMIC
* Allocation will not sleep. May use emergency pools.
*
* Also it is possible to set different flags by OR'ing
* in one or more of the following additional @flags:
*
* %__GFP_ZERO
* Zero the allocated memory before returning. Also see kzalloc().
*
* %__GFP_HIGH
* This allocation has high priority and may use emergency pools.
*
* %__GFP_NOFAIL
* Indicate that this allocation is in no way allowed to fail
* (think twice before using).
*
* %__GFP_NORETRY
* If memory is not immediately available,
* then give up at once.
*
* %__GFP_NOWARN
* If allocation fails, don't issue any warnings.
*
* %__GFP_RETRY_MAYFAIL
* Try really hard to succeed the allocation but fail
* eventually.
*/
static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
{
if (__builtin_constant_p(size) && size) {
unsigned int index;
if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large(size, flags);
index = kmalloc_index(size);
return kmalloc_trace(
kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
flags, size);
}
return __kmalloc(size, flags);
}
static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
{
if (__builtin_constant_p(size) && size) {
unsigned int index;
if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large_node(size, flags, node);
index = kmalloc_index(size);
return kmalloc_node_trace(
kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
flags, node, size);
}
return __kmalloc_node(size, flags, node);
}
/**
* kmalloc_array - allocate memory for an array.
* @n: number of elements.
* @size: element size.
* @flags: the type of memory to allocate (see kmalloc).
*/
static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
if (__builtin_constant_p(n) && __builtin_constant_p(size))
return kmalloc(bytes, flags); return __kmalloc(bytes, flags);
}
/**
* krealloc_array - reallocate memory for an array.
* @p: pointer to the memory chunk to reallocate
* @new_n: new number of elements to alloc
* @new_size: new size of a single member of the array
* @flags: the type of memory to allocate (see kmalloc)
*/
static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p,
size_t new_n,
size_t new_size,
gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
return NULL;
return krealloc(p, bytes, flags);
}
/**
* kcalloc - allocate memory for an array. The memory is set to zero.
* @n: number of elements.
* @size: element size.
* @flags: the type of memory to allocate (see kmalloc).
*/
static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flags)
{
return kmalloc_array(n, size, flags | __GFP_ZERO);
}
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
unsigned long caller) __alloc_size(1);
#define kmalloc_node_track_caller(size, flags, node) \
__kmalloc_node_track_caller(size, flags, node, \
_RET_IP_)
/*
* kmalloc_track_caller is a special version of kmalloc that records the
* calling function of the routine calling it for slab leak tracking instead
* of just the calling function (confusing, eh?).
* It's useful when the call to kmalloc comes from a widely-used standard
* allocator where we care about the real place the memory allocation
* request comes from.
*/
#define kmalloc_track_caller(size, flags) \
__kmalloc_node_track_caller(size, flags, \
NUMA_NO_NODE, _RET_IP_)
static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags,
int node)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
if (__builtin_constant_p(n) && __builtin_constant_p(size))
return kmalloc_node(bytes, flags, node);
return __kmalloc_node(bytes, flags, node);
}
static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node)
{
return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);
}
/*
* Shortcuts
*/
static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags)
{
return kmem_cache_alloc(k, flags | __GFP_ZERO);
}
/**
* kzalloc - allocate memory. The memory is set to zero.
* @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc).
*/
static inline __alloc_size(1) void *kzalloc(size_t size, gfp_t flags)
{
return kmalloc(size, flags | __GFP_ZERO);
}
/**
* kzalloc_node - allocate zeroed memory from a particular memory node.
* @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc).
* @node: memory node from which to allocate
*/
static inline __alloc_size(1) void *kzalloc_node(size_t size, gfp_t flags, int node)
{
return kmalloc_node(size, flags | __GFP_ZERO, node);
}
extern void *kvmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1);
static inline __alloc_size(1) void *kvmalloc(size_t size, gfp_t flags)
{
return kvmalloc_node(size, flags, NUMA_NO_NODE);
}
static inline __alloc_size(1) void *kvzalloc_node(size_t size, gfp_t flags, int node)
{
return kvmalloc_node(size, flags | __GFP_ZERO, node);
}
static inline __alloc_size(1) void *kvzalloc(size_t size, gfp_t flags)
{
return kvmalloc(size, flags | __GFP_ZERO);
}
static inline __alloc_size(1, 2) void *kvmalloc_array(size_t n, size_t size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
return kvmalloc(bytes, flags);
}
static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags)
{
return kvmalloc_array(n, size, flags | __GFP_ZERO);
}
extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
__realloc_size(3);
extern void kvfree(const void *addr);
DEFINE_FREE(kvfree, void *, if (_T) kvfree(_T))
extern void kvfree_sensitive(const void *addr, size_t len);
unsigned int kmem_cache_size(struct kmem_cache *s);
/**
* kmalloc_size_roundup - Report allocation bucket size for the given size
*
* @size: Number of bytes to round up from.
*
* This returns the number of bytes that would be available in a kmalloc()
* allocation of @size bytes. For example, a 126 byte request would be
* rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly
* for the general-purpose kmalloc()-based allocations, and is not for the
* pre-sized kmem_cache_alloc()-based allocations.)
*
* Use this to kmalloc() the full bucket size ahead of time instead of using
* ksize() to query the size after an allocation.
*/
size_t kmalloc_size_roundup(size_t size);
void __init kmem_cache_init_late(void);
#endif /* _LINUX_SLAB_H */
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Dynamic loading of modules into the kernel.
*
* Rewritten by Richard Henderson <rth@tamu.edu> Dec 1996
* Rewritten again by Rusty Russell, 2002
*/
#ifndef _LINUX_MODULE_H
#define _LINUX_MODULE_H
#include <linux/list.h>
#include <linux/stat.h>
#include <linux/buildid.h>
#include <linux/compiler.h>
#include <linux/cache.h>
#include <linux/kmod.h>
#include <linux/init.h>
#include <linux/elf.h>
#include <linux/stringify.h>
#include <linux/kobject.h>
#include <linux/moduleparam.h>
#include <linux/jump_label.h>
#include <linux/export.h>
#include <linux/rbtree_latch.h>
#include <linux/error-injection.h>
#include <linux/tracepoint-defs.h>
#include <linux/srcu.h>
#include <linux/static_call_types.h>
#include <linux/dynamic_debug.h>
#include <linux/percpu.h>
#include <asm/module.h>
#define MODULE_NAME_LEN MAX_PARAM_PREFIX_LEN
struct modversion_info {
unsigned long crc;
char name[MODULE_NAME_LEN];
};
struct module;
struct exception_table_entry;
struct module_kobject {
struct kobject kobj;
struct module *mod;
struct kobject *drivers_dir;
struct module_param_attrs *mp;
struct completion *kobj_completion;
} __randomize_layout;
struct module_attribute {
struct attribute attr;
ssize_t (*show)(struct module_attribute *, struct module_kobject *,
char *);
ssize_t (*store)(struct module_attribute *, struct module_kobject *,
const char *, size_t count);
void (*setup)(struct module *, const char *);
int (*test)(struct module *);
void (*free)(struct module *);
};
struct module_version_attribute {
struct module_attribute mattr;
const char *module_name;
const char *version;
};
extern ssize_t __modver_version_show(struct module_attribute *,
struct module_kobject *, char *);
extern struct module_attribute module_uevent;
/* These are either module local, or the kernel's dummy ones. */
extern int init_module(void);
extern void cleanup_module(void);
#ifndef MODULE
/**
* module_init() - driver initialization entry point
* @x: function to be run at kernel boot time or module insertion
*
* module_init() will either be called during do_initcalls() (if
* builtin) or at module insertion time (if a module). There can only
* be one per module.
*/
#define module_init(x) __initcall(x);
/**
* module_exit() - driver exit entry point
* @x: function to be run when driver is removed
*
* module_exit() will wrap the driver clean-up code
* with cleanup_module() when used with rmmod when
* the driver is a module. If the driver is statically
* compiled into the kernel, module_exit() has no effect.
* There can only be one per module.
*/
#define module_exit(x) __exitcall(x);
#else /* MODULE */
/*
* In most cases loadable modules do not need custom
* initcall levels. There are still some valid cases where
* a driver may be needed early if built in, and does not
* matter when built as a loadable module. Like bus
* snooping debug drivers.
*/
#define early_initcall(fn) module_init(fn)
#define core_initcall(fn) module_init(fn)
#define core_initcall_sync(fn) module_init(fn)
#define postcore_initcall(fn) module_init(fn)
#define postcore_initcall_sync(fn) module_init(fn)
#define arch_initcall(fn) module_init(fn)
#define subsys_initcall(fn) module_init(fn)
#define subsys_initcall_sync(fn) module_init(fn)
#define fs_initcall(fn) module_init(fn)
#define fs_initcall_sync(fn) module_init(fn)
#define rootfs_initcall(fn) module_init(fn)
#define device_initcall(fn) module_init(fn)
#define device_initcall_sync(fn) module_init(fn)
#define late_initcall(fn) module_init(fn)
#define late_initcall_sync(fn) module_init(fn)
#define console_initcall(fn) module_init(fn)
/* Each module must use one module_init(). */
#define module_init(initfn) \
static inline initcall_t __maybe_unused __inittest(void) \
{ return initfn; } \
int init_module(void) __copy(initfn) \
__attribute__((alias(#initfn))); \
___ADDRESSABLE(init_module, __initdata);
/* This is only required if you want to be unloadable. */
#define module_exit(exitfn) \
static inline exitcall_t __maybe_unused __exittest(void) \
{ return exitfn; } \
void cleanup_module(void) __copy(exitfn) \
__attribute__((alias(#exitfn))); \
___ADDRESSABLE(cleanup_module, __exitdata);
#endif
/* This means "can be init if no module support, otherwise module load
may call it." */
#ifdef CONFIG_MODULES
#define __init_or_module
#define __initdata_or_module
#define __initconst_or_module
#define __INIT_OR_MODULE .text
#define __INITDATA_OR_MODULE .data
#define __INITRODATA_OR_MODULE .section ".rodata","a",%progbits
#else
#define __init_or_module __init
#define __initdata_or_module __initdata
#define __initconst_or_module __initconst
#define __INIT_OR_MODULE __INIT
#define __INITDATA_OR_MODULE __INITDATA
#define __INITRODATA_OR_MODULE __INITRODATA
#endif /*CONFIG_MODULES*/
/* Generic info of form tag = "info" */
#define MODULE_INFO(tag, info) __MODULE_INFO(tag, tag, info)
/* For userspace: you can also call me... */
#define MODULE_ALIAS(_alias) MODULE_INFO(alias, _alias)
/* Soft module dependencies. See man modprobe.d for details.
* Example: MODULE_SOFTDEP("pre: module-foo module-bar post: module-baz")
*/
#define MODULE_SOFTDEP(_softdep) MODULE_INFO(softdep, _softdep)
/*
* MODULE_FILE is used for generating modules.builtin
* So, make it no-op when this is being built as a module
*/
#ifdef MODULE
#define MODULE_FILE
#else
#define MODULE_FILE MODULE_INFO(file, KBUILD_MODFILE);
#endif
/*
* The following license idents are currently accepted as indicating free
* software modules
*
* "GPL" [GNU Public License v2]
* "GPL v2" [GNU Public License v2]
* "GPL and additional rights" [GNU Public License v2 rights and more]
* "Dual BSD/GPL" [GNU Public License v2
* or BSD license choice]
* "Dual MIT/GPL" [GNU Public License v2
* or MIT license choice]
* "Dual MPL/GPL" [GNU Public License v2
* or Mozilla license choice]
*
* The following other idents are available
*
* "Proprietary" [Non free products]
*
* Both "GPL v2" and "GPL" (the latter also in dual licensed strings) are
* merely stating that the module is licensed under the GPL v2, but are not
* telling whether "GPL v2 only" or "GPL v2 or later". The reason why there
* are two variants is a historic and failed attempt to convey more
* information in the MODULE_LICENSE string. For module loading the
* "only/or later" distinction is completely irrelevant and does neither
* replace the proper license identifiers in the corresponding source file
* nor amends them in any way. The sole purpose is to make the
* 'Proprietary' flagging work and to refuse to bind symbols which are
* exported with EXPORT_SYMBOL_GPL when a non free module is loaded.
*
* In the same way "BSD" is not a clear license information. It merely
* states, that the module is licensed under one of the compatible BSD
* license variants. The detailed and correct license information is again
* to be found in the corresponding source files.
*
* There are dual licensed components, but when running with Linux it is the
* GPL that is relevant so this is a non issue. Similarly LGPL linked with GPL
* is a GPL combined work.
*
* This exists for several reasons
* 1. So modinfo can show license info for users wanting to vet their setup
* is free
* 2. So the community can ignore bug reports including proprietary modules
* 3. So vendors can do likewise based on their own policies
*/
#define MODULE_LICENSE(_license) MODULE_FILE MODULE_INFO(license, _license)
/*
* Author(s), use "Name <email>" or just "Name", for multiple
* authors use multiple MODULE_AUTHOR() statements/lines.
*/
#define MODULE_AUTHOR(_author) MODULE_INFO(author, _author)
/* What your module does. */
#define MODULE_DESCRIPTION(_description) MODULE_INFO(description, _description)
#ifdef MODULE
/* Creates an alias so file2alias.c can find device table. */
#define MODULE_DEVICE_TABLE(type, name) \
extern typeof(name) __mod_##type##__##name##_device_table \
__attribute__ ((unused, alias(__stringify(name))))
#else /* !MODULE */
#define MODULE_DEVICE_TABLE(type, name)
#endif
/* Version of form [<epoch>:]<version>[-<extra-version>].
* Or for CVS/RCS ID version, everything but the number is stripped.
* <epoch>: A (small) unsigned integer which allows you to start versions
* anew. If not mentioned, it's zero. eg. "2:1.0" is after
* "1:2.0".
* <version>: The <version> may contain only alphanumerics and the
* character `.'. Ordered by numeric sort for numeric parts,
* ascii sort for ascii parts (as per RPM or DEB algorithm).
* <extraversion>: Like <version>, but inserted for local
* customizations, eg "rh3" or "rusty1".
* Using this automatically adds a checksum of the .c files and the
* local headers in "srcversion".
*/
#if defined(MODULE) || !defined(CONFIG_SYSFS)
#define MODULE_VERSION(_version) MODULE_INFO(version, _version)
#else
#define MODULE_VERSION(_version) \
MODULE_INFO(version, _version); \
static struct module_version_attribute __modver_attr \
__used __section("__modver") \
__aligned(__alignof__(struct module_version_attribute)) \
= { \
.mattr = { \
.attr = { \
.name = "version", \
.mode = S_IRUGO, \
}, \
.show = __modver_version_show, \
}, \
.module_name = KBUILD_MODNAME, \
.version = _version, \
}
#endif
/* Optional firmware file (or files) needed by the module
* format is simply firmware file name. Multiple firmware
* files require multiple MODULE_FIRMWARE() specifiers */
#define MODULE_FIRMWARE(_firmware) MODULE_INFO(firmware, _firmware)
#define MODULE_IMPORT_NS(ns) MODULE_INFO(import_ns, __stringify(ns))
struct notifier_block;
#ifdef CONFIG_MODULES
extern int modules_disabled; /* for sysctl */
/* Get/put a kernel symbol (calls must be symmetric) */
void *__symbol_get(const char *symbol);
void *__symbol_get_gpl(const char *symbol);
#define symbol_get(x) ((typeof(&x))(__symbol_get(__stringify(x))))
/* modules using other modules: kdb wants to see this. */
struct module_use {
struct list_head source_list;
struct list_head target_list;
struct module *source, *target;
};
enum module_state {
MODULE_STATE_LIVE, /* Normal state. */
MODULE_STATE_COMING, /* Full formed, running module_init. */
MODULE_STATE_GOING, /* Going away. */
MODULE_STATE_UNFORMED, /* Still setting it up. */
};
struct mod_tree_node {
struct module *mod;
struct latch_tree_node node;
};
enum mod_mem_type {
MOD_TEXT = 0,
MOD_DATA,
MOD_RODATA,
MOD_RO_AFTER_INIT,
MOD_INIT_TEXT,
MOD_INIT_DATA,
MOD_INIT_RODATA,
MOD_MEM_NUM_TYPES,
MOD_INVALID = -1,
};
#define mod_mem_type_is_init(type) \
((type) == MOD_INIT_TEXT || \
(type) == MOD_INIT_DATA || \
(type) == MOD_INIT_RODATA)
#define mod_mem_type_is_core(type) (!mod_mem_type_is_init(type))
#define mod_mem_type_is_text(type) \
((type) == MOD_TEXT || \
(type) == MOD_INIT_TEXT)
#define mod_mem_type_is_data(type) (!mod_mem_type_is_text(type))
#define mod_mem_type_is_core_data(type) \
(mod_mem_type_is_core(type) && \
mod_mem_type_is_data(type))
#define for_each_mod_mem_type(type) \
for (enum mod_mem_type (type) = 0; \
(type) < MOD_MEM_NUM_TYPES; (type)++)
#define for_class_mod_mem_type(type, class) \
for_each_mod_mem_type(type) \
if (mod_mem_type_is_##class(type))
struct module_memory {
void *base;
unsigned int size;
#ifdef CONFIG_MODULES_TREE_LOOKUP
struct mod_tree_node mtn;
#endif
};
#ifdef CONFIG_MODULES_TREE_LOOKUP
/* Only touch one cacheline for common rbtree-for-core-layout case. */
#define __module_memory_align ____cacheline_aligned
#else
#define __module_memory_align
#endif
struct mod_kallsyms {
Elf_Sym *symtab;
unsigned int num_symtab;
char *strtab;
char *typetab;
};
#ifdef CONFIG_LIVEPATCH
/**
* struct klp_modinfo - ELF information preserved from the livepatch module
*
* @hdr: ELF header
* @sechdrs: Section header table
* @secstrings: String table for the section headers
* @symndx: The symbol table section index
*/
struct klp_modinfo {
Elf_Ehdr hdr;
Elf_Shdr *sechdrs;
char *secstrings;
unsigned int symndx;
};
#endif
struct module {
enum module_state state;
/* Member of list of modules */
struct list_head list;
/* Unique handle for this module */
char name[MODULE_NAME_LEN];
#ifdef CONFIG_STACKTRACE_BUILD_ID
/* Module build ID */
unsigned char build_id[BUILD_ID_SIZE_MAX];
#endif
/* Sysfs stuff. */
struct module_kobject mkobj;
struct module_attribute *modinfo_attrs;
const char *version;
const char *srcversion;
struct kobject *holders_dir;
/* Exported symbols */
const struct kernel_symbol *syms;
const s32 *crcs;
unsigned int num_syms;
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
s32 *kcfi_traps;
s32 *kcfi_traps_end;
#endif
/* Kernel parameters. */
#ifdef CONFIG_SYSFS
struct mutex param_lock;
#endif
struct kernel_param *kp;
unsigned int num_kp;
/* GPL-only exported symbols. */
unsigned int num_gpl_syms;
const struct kernel_symbol *gpl_syms;
const s32 *gpl_crcs;
bool using_gplonly_symbols;
#ifdef CONFIG_MODULE_SIG
/* Signature was verified. */
bool sig_ok;
#endif
bool async_probe_requested;
/* Exception table */
unsigned int num_exentries;
struct exception_table_entry *extable;
/* Startup function. */
int (*init)(void);
struct module_memory mem[MOD_MEM_NUM_TYPES] __module_memory_align;
/* Arch-specific module values */
struct mod_arch_specific arch;
unsigned long taints; /* same bits as kernel:taint_flags */
#ifdef CONFIG_GENERIC_BUG
/* Support for BUG */
unsigned num_bugs;
struct list_head bug_list;
struct bug_entry *bug_table;
#endif
#ifdef CONFIG_KALLSYMS
/* Protected by RCU and/or module_mutex: use rcu_dereference() */
struct mod_kallsyms __rcu *kallsyms;
struct mod_kallsyms core_kallsyms;
/* Section attributes */
struct module_sect_attrs *sect_attrs;
/* Notes attributes */
struct module_notes_attrs *notes_attrs;
#endif
/* The command line arguments (may be mangled). People like
keeping pointers to this stuff */
char *args;
#ifdef CONFIG_SMP
/* Per-cpu data. */
void __percpu *percpu;
unsigned int percpu_size;
#endif
void *noinstr_text_start;
unsigned int noinstr_text_size;
#ifdef CONFIG_TRACEPOINTS
unsigned int num_tracepoints;
tracepoint_ptr_t *tracepoints_ptrs;
#endif
#ifdef CONFIG_TREE_SRCU
unsigned int num_srcu_structs;
struct srcu_struct **srcu_struct_ptrs;
#endif
#ifdef CONFIG_BPF_EVENTS
unsigned int num_bpf_raw_events;
struct bpf_raw_event_map *bpf_raw_events;
#endif
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
unsigned int btf_data_size;
void *btf_data;
#endif
#ifdef CONFIG_JUMP_LABEL
struct jump_entry *jump_entries;
unsigned int num_jump_entries;
#endif
#ifdef CONFIG_TRACING
unsigned int num_trace_bprintk_fmt;
const char **trace_bprintk_fmt_start;
#endif
#ifdef CONFIG_EVENT_TRACING
struct trace_event_call **trace_events;
unsigned int num_trace_events;
struct trace_eval_map **trace_evals;
unsigned int num_trace_evals;
#endif
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
unsigned int num_ftrace_callsites;
unsigned long *ftrace_callsites;
#endif
#ifdef CONFIG_KPROBES
void *kprobes_text_start;
unsigned int kprobes_text_size;
unsigned long *kprobe_blacklist;
unsigned int num_kprobe_blacklist;
#endif
#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
int num_static_call_sites;
struct static_call_site *static_call_sites;
#endif
#if IS_ENABLED(CONFIG_KUNIT)
int num_kunit_init_suites;
struct kunit_suite **kunit_init_suites;
int num_kunit_suites;
struct kunit_suite **kunit_suites;
#endif
#ifdef CONFIG_LIVEPATCH
bool klp; /* Is this a livepatch module? */
bool klp_alive;
/* ELF information */
struct klp_modinfo *klp_info;
#endif
#ifdef CONFIG_PRINTK_INDEX
unsigned int printk_index_size;
struct pi_entry **printk_index_start;
#endif
#ifdef CONFIG_MODULE_UNLOAD
/* What modules depend on me? */
struct list_head source_list;
/* What modules do I depend on? */
struct list_head target_list;
/* Destruction function. */
void (*exit)(void);
atomic_t refcnt;
#endif
#ifdef CONFIG_CONSTRUCTORS
/* Constructor functions. */
ctor_fn_t *ctors;
unsigned int num_ctors;
#endif
#ifdef CONFIG_FUNCTION_ERROR_INJECTION
struct error_injection_entry *ei_funcs;
unsigned int num_ei_funcs;
#endif
#ifdef CONFIG_DYNAMIC_DEBUG_CORE
struct _ddebug_info dyndbg_info;
#endif
} ____cacheline_aligned __randomize_layout;
#ifndef MODULE_ARCH_INIT
#define MODULE_ARCH_INIT {}
#endif
#ifndef HAVE_ARCH_KALLSYMS_SYMBOL_VALUE
static inline unsigned long kallsyms_symbol_value(const Elf_Sym *sym)
{
return sym->st_value;
}
#endif
/* FIXME: It'd be nice to isolate modules during init, too, so they
aren't used before they (may) fail. But presently too much code
(IDE & SCSI) require entry into the module during init.*/
static inline bool module_is_live(struct module *mod)
{
return mod->state != MODULE_STATE_GOING;
}
struct module *__module_text_address(unsigned long addr);
struct module *__module_address(unsigned long addr);
bool is_module_address(unsigned long addr);
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr);
bool is_module_percpu_address(unsigned long addr);
bool is_module_text_address(unsigned long addr);
static inline bool within_module_mem_type(unsigned long addr,
const struct module *mod,
enum mod_mem_type type)
{
unsigned long base, size;
base = (unsigned long)mod->mem[type].base;
size = mod->mem[type].size;
return addr - base < size;
}
static inline bool within_module_core(unsigned long addr,
const struct module *mod)
{
for_class_mod_mem_type(type, core) { if (within_module_mem_type(addr, mod, type))
return true;
}
return false;
}
static inline bool within_module_init(unsigned long addr,
const struct module *mod)
{
for_class_mod_mem_type(type, init) { if (within_module_mem_type(addr, mod, type))
return true;
}
return false;
}
static inline bool within_module(unsigned long addr, const struct module *mod)
{
return within_module_init(addr, mod) || within_module_core(addr, mod);
}
/* Search for module by name: must be in a RCU-sched critical section. */
struct module *find_module(const char *name);
extern void __noreturn __module_put_and_kthread_exit(struct module *mod,
long code);
#define module_put_and_kthread_exit(code) __module_put_and_kthread_exit(THIS_MODULE, code)
#ifdef CONFIG_MODULE_UNLOAD
int module_refcount(struct module *mod);
void __symbol_put(const char *symbol);
#define symbol_put(x) __symbol_put(__stringify(x))
void symbol_put_addr(void *addr);
/* Sometimes we know we already have a refcount, and it's easier not
to handle the error case (which only happens with rmmod --wait). */
extern void __module_get(struct module *module);
/**
* try_module_get() - take module refcount unless module is being removed
* @module: the module we should check for
*
* Only try to get a module reference count if the module is not being removed.
* This call will fail if the module is in the process of being removed.
*
* Care must also be taken to ensure the module exists and is alive prior to
* usage of this call. This can be gauranteed through two means:
*
* 1) Direct protection: you know an earlier caller must have increased the
* module reference through __module_get(). This can typically be achieved
* by having another entity other than the module itself increment the
* module reference count.
*
* 2) Implied protection: there is an implied protection against module
* removal. An example of this is the implied protection used by kernfs /
* sysfs. The sysfs store / read file operations are guaranteed to exist
* through the use of kernfs's active reference (see kernfs_active()) and a
* sysfs / kernfs file removal cannot happen unless the same file is not
* active. Therefore, if a sysfs file is being read or written to the module
* which created it must still exist. It is therefore safe to use
* try_module_get() on module sysfs store / read ops.
*
* One of the real values to try_module_get() is the module_is_live() check
* which ensures that the caller of try_module_get() can yield to userspace
* module removal requests and gracefully fail if the module is on its way out.
*
* Returns true if the reference count was successfully incremented.
*/
extern bool try_module_get(struct module *module);
/**
* module_put() - release a reference count to a module
* @module: the module we should release a reference count for
*
* If you successfully bump a reference count to a module with try_module_get(),
* when you are finished you must call module_put() to release that reference
* count.
*/
extern void module_put(struct module *module);
#else /*!CONFIG_MODULE_UNLOAD*/
static inline bool try_module_get(struct module *module)
{
return !module || module_is_live(module);
}
static inline void module_put(struct module *module)
{
}
static inline void __module_get(struct module *module)
{
}
#define symbol_put(x) do { } while (0)
#define symbol_put_addr(p) do { } while (0)
#endif /* CONFIG_MODULE_UNLOAD */
/* This is a #define so the string doesn't get put in every .o file */
#define module_name(mod) \
({ \
struct module *__mod = (mod); \
__mod ? __mod->name : "kernel"; \
})
/* Dereference module function descriptor */
void *dereference_module_function_descriptor(struct module *mod, void *ptr);
int register_module_notifier(struct notifier_block *nb);
int unregister_module_notifier(struct notifier_block *nb);
extern void print_modules(void);
static inline bool module_requested_async_probing(struct module *module)
{
return module && module->async_probe_requested;
}
static inline bool is_livepatch_module(struct module *mod)
{
#ifdef CONFIG_LIVEPATCH
return mod->klp;
#else
return false;
#endif
}
void set_module_sig_enforced(void);
#else /* !CONFIG_MODULES... */
static inline struct module *__module_address(unsigned long addr)
{
return NULL;
}
static inline struct module *__module_text_address(unsigned long addr)
{
return NULL;
}
static inline bool is_module_address(unsigned long addr)
{
return false;
}
static inline bool is_module_percpu_address(unsigned long addr)
{
return false;
}
static inline bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
{
return false;
}
static inline bool is_module_text_address(unsigned long addr)
{
return false;
}
static inline bool within_module_core(unsigned long addr,
const struct module *mod)
{
return false;
}
static inline bool within_module_init(unsigned long addr,
const struct module *mod)
{
return false;
}
static inline bool within_module(unsigned long addr, const struct module *mod)
{
return false;
}
/* Get/put a kernel symbol (calls should be symmetric) */
#define symbol_get(x) ({ extern typeof(x) x __attribute__((weak,visibility("hidden"))); &(x); })
#define symbol_put(x) do { } while (0)
#define symbol_put_addr(x) do { } while (0)
static inline void __module_get(struct module *module)
{
}
static inline bool try_module_get(struct module *module)
{
return true;
}
static inline void module_put(struct module *module)
{
}
#define module_name(mod) "kernel"
static inline int register_module_notifier(struct notifier_block *nb)
{
/* no events will happen anyway, so this can always succeed */
return 0;
}
static inline int unregister_module_notifier(struct notifier_block *nb)
{
return 0;
}
#define module_put_and_kthread_exit(code) kthread_exit(code)
static inline void print_modules(void)
{
}
static inline bool module_requested_async_probing(struct module *module)
{
return false;
}
static inline void set_module_sig_enforced(void)
{
}
/* Dereference module function descriptor */
static inline
void *dereference_module_function_descriptor(struct module *mod, void *ptr)
{
return ptr;
}
#endif /* CONFIG_MODULES */
#ifdef CONFIG_SYSFS
extern struct kset *module_kset;
extern const struct kobj_type module_ktype;
#endif /* CONFIG_SYSFS */
#define symbol_request(x) try_then_request_module(symbol_get(x), "symbol:" #x)
/* BELOW HERE ALL THESE ARE OBSOLETE AND WILL VANISH */
#define __MODULE_STRING(x) __stringify(x)
#ifdef CONFIG_GENERIC_BUG
void module_bug_finalize(const Elf_Ehdr *, const Elf_Shdr *,
struct module *);
void module_bug_cleanup(struct module *);
#else /* !CONFIG_GENERIC_BUG */
static inline void module_bug_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs,
struct module *mod)
{
}
static inline void module_bug_cleanup(struct module *mod) {}
#endif /* CONFIG_GENERIC_BUG */
#ifdef CONFIG_MITIGATION_RETPOLINE
extern bool retpoline_module_ok(bool has_retpoline);
#else
static inline bool retpoline_module_ok(bool has_retpoline)
{
return true;
}
#endif
#ifdef CONFIG_MODULE_SIG
bool is_module_sig_enforced(void);
static inline bool module_sig_ok(struct module *module)
{
return module->sig_ok;
}
#else /* !CONFIG_MODULE_SIG */
static inline bool is_module_sig_enforced(void)
{
return false;
}
static inline bool module_sig_ok(struct module *module)
{
return true;
}
#endif /* CONFIG_MODULE_SIG */
#if defined(CONFIG_MODULES) && defined(CONFIG_KALLSYMS)
int module_kallsyms_on_each_symbol(const char *modname,
int (*fn)(void *, const char *, unsigned long),
void *data);
/* For kallsyms to ask for address resolution. namebuf should be at
* least KSYM_NAME_LEN long: a pointer to namebuf is returned if
* found, otherwise NULL.
*/
const char *module_address_lookup(unsigned long addr,
unsigned long *symbolsize,
unsigned long *offset,
char **modname, const unsigned char **modbuildid,
char *namebuf);
int lookup_module_symbol_name(unsigned long addr, char *symname);
int lookup_module_symbol_attrs(unsigned long addr,
unsigned long *size,
unsigned long *offset,
char *modname,
char *name);
/* Returns 0 and fills in value, defined and namebuf, or -ERANGE if
* symnum out of range.
*/
int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
char *name, char *module_name, int *exported);
/* Look for this name: can be of form module:name. */
unsigned long module_kallsyms_lookup_name(const char *name);
unsigned long find_kallsyms_symbol_value(struct module *mod, const char *name);
#else /* CONFIG_MODULES && CONFIG_KALLSYMS */
static inline int module_kallsyms_on_each_symbol(const char *modname,
int (*fn)(void *, const char *, unsigned long),
void *data)
{
return -EOPNOTSUPP;
}
/* For kallsyms to ask for address resolution. NULL means not found. */
static inline const char *module_address_lookup(unsigned long addr,
unsigned long *symbolsize,
unsigned long *offset,
char **modname,
const unsigned char **modbuildid,
char *namebuf)
{
return NULL;
}
static inline int lookup_module_symbol_name(unsigned long addr, char *symname)
{
return -ERANGE;
}
static inline int module_get_kallsym(unsigned int symnum, unsigned long *value,
char *type, char *name,
char *module_name, int *exported)
{
return -ERANGE;
}
static inline unsigned long module_kallsyms_lookup_name(const char *name)
{
return 0;
}
static inline unsigned long find_kallsyms_symbol_value(struct module *mod,
const char *name)
{
return 0;
}
#endif /* CONFIG_MODULES && CONFIG_KALLSYMS */
#endif /* _LINUX_MODULE_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Componentized device handling.
*/
#include <linux/component.h>
#include <linux/device.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/debugfs.h>
/**
* DOC: overview
*
* The component helper allows drivers to collect a pile of sub-devices,
* including their bound drivers, into an aggregate driver. Various subsystems
* already provide functions to get hold of such components, e.g.
* of_clk_get_by_name(). The component helper can be used when such a
* subsystem-specific way to find a device is not available: The component
* helper fills the niche of aggregate drivers for specific hardware, where
* further standardization into a subsystem would not be practical. The common
* example is when a logical device (e.g. a DRM display driver) is spread around
* the SoC on various components (scanout engines, blending blocks, transcoders
* for various outputs and so on).
*
* The component helper also doesn't solve runtime dependencies, e.g. for system
* suspend and resume operations. See also :ref:`device links<device_link>`.
*
* Components are registered using component_add() and unregistered with
* component_del(), usually from the driver's probe and disconnect functions.
*
* Aggregate drivers first assemble a component match list of what they need
* using component_match_add(). This is then registered as an aggregate driver
* using component_master_add_with_match(), and unregistered using
* component_master_del().
*/
struct component;
struct component_match_array {
void *data;
int (*compare)(struct device *, void *);
int (*compare_typed)(struct device *, int, void *);
void (*release)(struct device *, void *);
struct component *component;
bool duplicate;
};
struct component_match {
size_t alloc;
size_t num;
struct component_match_array *compare;
};
struct aggregate_device {
struct list_head node;
bool bound;
const struct component_master_ops *ops;
struct device *parent;
struct component_match *match;
};
struct component {
struct list_head node;
struct aggregate_device *adev;
bool bound;
const struct component_ops *ops;
int subcomponent;
struct device *dev;
};
static DEFINE_MUTEX(component_mutex);
static LIST_HEAD(component_list);
static LIST_HEAD(aggregate_devices);
#ifdef CONFIG_DEBUG_FS
static struct dentry *component_debugfs_dir;
static int component_devices_show(struct seq_file *s, void *data)
{
struct aggregate_device *m = s->private;
struct component_match *match = m->match;
size_t i;
mutex_lock(&component_mutex);
seq_printf(s, "%-40s %20s\n", "aggregate_device name", "status");
seq_puts(s, "-------------------------------------------------------------\n");
seq_printf(s, "%-40s %20s\n\n",
dev_name(m->parent), m->bound ? "bound" : "not bound");
seq_printf(s, "%-40s %20s\n", "device name", "status");
seq_puts(s, "-------------------------------------------------------------\n");
for (i = 0; i < match->num; i++) {
struct component *component = match->compare[i].component;
seq_printf(s, "%-40s %20s\n",
component ? dev_name(component->dev) : "(unknown)",
component ? (component->bound ? "bound" : "not bound") : "not registered");
}
mutex_unlock(&component_mutex);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(component_devices);
static int __init component_debug_init(void)
{
component_debugfs_dir = debugfs_create_dir("device_component", NULL);
return 0;
}
core_initcall(component_debug_init);
static void component_debugfs_add(struct aggregate_device *m)
{
debugfs_create_file(dev_name(m->parent), 0444, component_debugfs_dir, m,
&component_devices_fops);
}
static void component_debugfs_del(struct aggregate_device *m)
{
debugfs_lookup_and_remove(dev_name(m->parent), component_debugfs_dir);
}
#else
static void component_debugfs_add(struct aggregate_device *m)
{ }
static void component_debugfs_del(struct aggregate_device *m)
{ }
#endif
static struct aggregate_device *__aggregate_find(struct device *parent,
const struct component_master_ops *ops)
{
struct aggregate_device *m;
list_for_each_entry(m, &aggregate_devices, node)
if (m->parent == parent && (!ops || m->ops == ops))
return m;
return NULL;
}
static struct component *find_component(struct aggregate_device *adev,
struct component_match_array *mc)
{
struct component *c;
list_for_each_entry(c, &component_list, node) {
if (c->adev && c->adev != adev)
continue;
if (mc->compare && mc->compare(c->dev, mc->data))
return c;
if (mc->compare_typed &&
mc->compare_typed(c->dev, c->subcomponent, mc->data))
return c;
}
return NULL;
}
static int find_components(struct aggregate_device *adev)
{
struct component_match *match = adev->match;
size_t i;
int ret = 0;
/*
* Scan the array of match functions and attach
* any components which are found to this adev.
*/
for (i = 0; i < match->num; i++) {
struct component_match_array *mc = &match->compare[i];
struct component *c;
dev_dbg(adev->parent, "Looking for component %zu\n", i);
if (match->compare[i].component)
continue;
c = find_component(adev, mc);
if (!c) {
ret = -ENXIO;
break;
}
dev_dbg(adev->parent, "found component %s, duplicate %u\n",
dev_name(c->dev), !!c->adev);
/* Attach this component to the adev */
match->compare[i].duplicate = !!c->adev;
match->compare[i].component = c;
c->adev = adev;
}
return ret;
}
/* Detach component from associated aggregate_device */
static void remove_component(struct aggregate_device *adev, struct component *c)
{
size_t i;
/* Detach the component from this adev. */
for (i = 0; i < adev->match->num; i++) if (adev->match->compare[i].component == c) adev->match->compare[i].component = NULL;
}
/*
* Try to bring up an aggregate device. If component is NULL, we're interested
* in this aggregate device, otherwise it's a component which must be present
* to try and bring up the aggregate device.
*
* Returns 1 for successful bringup, 0 if not ready, or -ve errno.
*/
static int try_to_bring_up_aggregate_device(struct aggregate_device *adev,
struct component *component)
{
int ret;
dev_dbg(adev->parent, "trying to bring up adev\n");
if (find_components(adev)) {
dev_dbg(adev->parent, "master has incomplete components\n");
return 0;
}
if (component && component->adev != adev) {
dev_dbg(adev->parent, "master is not for this component (%s)\n",
dev_name(component->dev));
return 0;
}
if (!devres_open_group(adev->parent, adev, GFP_KERNEL))
return -ENOMEM;
/* Found all components */
ret = adev->ops->bind(adev->parent);
if (ret < 0) {
devres_release_group(adev->parent, NULL);
if (ret != -EPROBE_DEFER)
dev_info(adev->parent, "adev bind failed: %d\n", ret);
return ret;
}
devres_close_group(adev->parent, NULL);
adev->bound = true;
return 1;
}
static int try_to_bring_up_masters(struct component *component)
{
struct aggregate_device *adev;
int ret = 0;
list_for_each_entry(adev, &aggregate_devices, node) {
if (!adev->bound) {
ret = try_to_bring_up_aggregate_device(adev, component);
if (ret != 0)
break;
}
}
return ret;
}
static void take_down_aggregate_device(struct aggregate_device *adev)
{
if (adev->bound) { adev->ops->unbind(adev->parent);
devres_release_group(adev->parent, adev);
adev->bound = false;
}
}
/**
* component_compare_of - A common component compare function for of_node
* @dev: component device
* @data: @compare_data from component_match_add_release()
*
* A common compare function when compare_data is device of_node. e.g.
* component_match_add_release(masterdev, &match, component_release_of,
* component_compare_of, component_dev_of_node)
*/
int component_compare_of(struct device *dev, void *data)
{
return device_match_of_node(dev, data);
}
EXPORT_SYMBOL_GPL(component_compare_of);
/**
* component_release_of - A common component release function for of_node
* @dev: component device
* @data: @compare_data from component_match_add_release()
*
* About the example, Please see component_compare_of().
*/
void component_release_of(struct device *dev, void *data)
{
of_node_put(data);
}
EXPORT_SYMBOL_GPL(component_release_of);
/**
* component_compare_dev - A common component compare function for dev
* @dev: component device
* @data: @compare_data from component_match_add_release()
*
* A common compare function when compare_data is struce device. e.g.
* component_match_add(masterdev, &match, component_compare_dev, component_dev)
*/
int component_compare_dev(struct device *dev, void *data)
{
return dev == data;
}
EXPORT_SYMBOL_GPL(component_compare_dev);
/**
* component_compare_dev_name - A common component compare function for device name
* @dev: component device
* @data: @compare_data from component_match_add_release()
*
* A common compare function when compare_data is device name string. e.g.
* component_match_add(masterdev, &match, component_compare_dev_name,
* "component_dev_name")
*/
int component_compare_dev_name(struct device *dev, void *data)
{
return device_match_name(dev, data);
}
EXPORT_SYMBOL_GPL(component_compare_dev_name);
static void devm_component_match_release(struct device *parent, void *res)
{
struct component_match *match = res;
unsigned int i;
for (i = 0; i < match->num; i++) {
struct component_match_array *mc = &match->compare[i];
if (mc->release)
mc->release(parent, mc->data);
}
kfree(match->compare);
}
static int component_match_realloc(struct component_match *match, size_t num)
{
struct component_match_array *new;
if (match->alloc == num)
return 0;
new = kmalloc_array(num, sizeof(*new), GFP_KERNEL);
if (!new)
return -ENOMEM;
if (match->compare) {
memcpy(new, match->compare, sizeof(*new) *
min(match->num, num));
kfree(match->compare);
}
match->compare = new;
match->alloc = num;
return 0;
}
static void __component_match_add(struct device *parent,
struct component_match **matchptr,
void (*release)(struct device *, void *),
int (*compare)(struct device *, void *),
int (*compare_typed)(struct device *, int, void *),
void *compare_data)
{
struct component_match *match = *matchptr;
if (IS_ERR(match))
return;
if (!match) {
match = devres_alloc(devm_component_match_release,
sizeof(*match), GFP_KERNEL);
if (!match) {
*matchptr = ERR_PTR(-ENOMEM);
return;
}
devres_add(parent, match);
*matchptr = match;
}
if (match->num == match->alloc) {
size_t new_size = match->alloc + 16;
int ret;
ret = component_match_realloc(match, new_size);
if (ret) {
*matchptr = ERR_PTR(ret);
return;
}
}
match->compare[match->num].compare = compare;
match->compare[match->num].compare_typed = compare_typed;
match->compare[match->num].release = release;
match->compare[match->num].data = compare_data;
match->compare[match->num].component = NULL;
match->num++;
}
/**
* component_match_add_release - add a component match entry with release callback
* @parent: parent device of the aggregate driver
* @matchptr: pointer to the list of component matches
* @release: release function for @compare_data
* @compare: compare function to match against all components
* @compare_data: opaque pointer passed to the @compare function
*
* Adds a new component match to the list stored in @matchptr, which the
* aggregate driver needs to function. The list of component matches pointed to
* by @matchptr must be initialized to NULL before adding the first match. This
* only matches against components added with component_add().
*
* The allocated match list in @matchptr is automatically released using devm
* actions, where upon @release will be called to free any references held by
* @compare_data, e.g. when @compare_data is a &device_node that must be
* released with of_node_put().
*
* See also component_match_add() and component_match_add_typed().
*/
void component_match_add_release(struct device *parent,
struct component_match **matchptr,
void (*release)(struct device *, void *),
int (*compare)(struct device *, void *), void *compare_data)
{
__component_match_add(parent, matchptr, release, compare, NULL,
compare_data);
}
EXPORT_SYMBOL(component_match_add_release);
/**
* component_match_add_typed - add a component match entry for a typed component
* @parent: parent device of the aggregate driver
* @matchptr: pointer to the list of component matches
* @compare_typed: compare function to match against all typed components
* @compare_data: opaque pointer passed to the @compare function
*
* Adds a new component match to the list stored in @matchptr, which the
* aggregate driver needs to function. The list of component matches pointed to
* by @matchptr must be initialized to NULL before adding the first match. This
* only matches against components added with component_add_typed().
*
* The allocated match list in @matchptr is automatically released using devm
* actions.
*
* See also component_match_add_release() and component_match_add_typed().
*/
void component_match_add_typed(struct device *parent,
struct component_match **matchptr,
int (*compare_typed)(struct device *, int, void *), void *compare_data)
{
__component_match_add(parent, matchptr, NULL, NULL, compare_typed,
compare_data);
}
EXPORT_SYMBOL(component_match_add_typed);
static void free_aggregate_device(struct aggregate_device *adev)
{
struct component_match *match = adev->match;
int i;
component_debugfs_del(adev);
list_del(&adev->node);
if (match) {
for (i = 0; i < match->num; i++) {
struct component *c = match->compare[i].component;
if (c)
c->adev = NULL;
}
}
kfree(adev);
}
/**
* component_master_add_with_match - register an aggregate driver
* @parent: parent device of the aggregate driver
* @ops: callbacks for the aggregate driver
* @match: component match list for the aggregate driver
*
* Registers a new aggregate driver consisting of the components added to @match
* by calling one of the component_match_add() functions. Once all components in
* @match are available, it will be assembled by calling
* &component_master_ops.bind from @ops. Must be unregistered by calling
* component_master_del().
*/
int component_master_add_with_match(struct device *parent,
const struct component_master_ops *ops,
struct component_match *match)
{
struct aggregate_device *adev;
int ret;
/* Reallocate the match array for its true size */
ret = component_match_realloc(match, match->num);
if (ret)
return ret;
adev = kzalloc(sizeof(*adev), GFP_KERNEL);
if (!adev)
return -ENOMEM;
adev->parent = parent;
adev->ops = ops;
adev->match = match;
component_debugfs_add(adev);
/* Add to the list of available aggregate devices. */
mutex_lock(&component_mutex);
list_add(&adev->node, &aggregate_devices);
ret = try_to_bring_up_aggregate_device(adev, NULL);
if (ret < 0)
free_aggregate_device(adev);
mutex_unlock(&component_mutex);
return ret < 0 ? ret : 0;
}
EXPORT_SYMBOL_GPL(component_master_add_with_match);
/**
* component_master_del - unregister an aggregate driver
* @parent: parent device of the aggregate driver
* @ops: callbacks for the aggregate driver
*
* Unregisters an aggregate driver registered with
* component_master_add_with_match(). If necessary the aggregate driver is first
* disassembled by calling &component_master_ops.unbind from @ops.
*/
void component_master_del(struct device *parent,
const struct component_master_ops *ops)
{
struct aggregate_device *adev;
mutex_lock(&component_mutex);
adev = __aggregate_find(parent, ops);
if (adev) {
take_down_aggregate_device(adev);
free_aggregate_device(adev);
}
mutex_unlock(&component_mutex);
}
EXPORT_SYMBOL_GPL(component_master_del);
static void component_unbind(struct component *component,
struct aggregate_device *adev, void *data)
{
WARN_ON(!component->bound);
if (component->ops && component->ops->unbind)
component->ops->unbind(component->dev, adev->parent, data);
component->bound = false;
/* Release all resources claimed in the binding of this component */
devres_release_group(component->dev, component);
}
/**
* component_unbind_all - unbind all components of an aggregate driver
* @parent: parent device of the aggregate driver
* @data: opaque pointer, passed to all components
*
* Unbinds all components of the aggregate device by passing @data to their
* &component_ops.unbind functions. Should be called from
* &component_master_ops.unbind.
*/
void component_unbind_all(struct device *parent, void *data)
{
struct aggregate_device *adev;
struct component *c;
size_t i;
WARN_ON(!mutex_is_locked(&component_mutex));
adev = __aggregate_find(parent, NULL);
if (!adev)
return;
/* Unbind components in reverse order */
for (i = adev->match->num; i--; )
if (!adev->match->compare[i].duplicate) {
c = adev->match->compare[i].component;
component_unbind(c, adev, data);
}
}
EXPORT_SYMBOL_GPL(component_unbind_all);
static int component_bind(struct component *component, struct aggregate_device *adev,
void *data)
{
int ret;
/*
* Each component initialises inside its own devres group.
* This allows us to roll-back a failed component without
* affecting anything else.
*/
if (!devres_open_group(adev->parent, NULL, GFP_KERNEL))
return -ENOMEM;
/*
* Also open a group for the device itself: this allows us
* to release the resources claimed against the sub-device
* at the appropriate moment.
*/
if (!devres_open_group(component->dev, component, GFP_KERNEL)) {
devres_release_group(adev->parent, NULL);
return -ENOMEM;
}
dev_dbg(adev->parent, "binding %s (ops %ps)\n",
dev_name(component->dev), component->ops);
ret = component->ops->bind(component->dev, adev->parent, data);
if (!ret) {
component->bound = true;
/*
* Close the component device's group so that resources
* allocated in the binding are encapsulated for removal
* at unbind. Remove the group on the DRM device as we
* can clean those resources up independently.
*/
devres_close_group(component->dev, NULL);
devres_remove_group(adev->parent, NULL);
dev_info(adev->parent, "bound %s (ops %ps)\n",
dev_name(component->dev), component->ops);
} else {
devres_release_group(component->dev, NULL);
devres_release_group(adev->parent, NULL);
if (ret != -EPROBE_DEFER)
dev_err(adev->parent, "failed to bind %s (ops %ps): %d\n",
dev_name(component->dev), component->ops, ret);
}
return ret;
}
/**
* component_bind_all - bind all components of an aggregate driver
* @parent: parent device of the aggregate driver
* @data: opaque pointer, passed to all components
*
* Binds all components of the aggregate @dev by passing @data to their
* &component_ops.bind functions. Should be called from
* &component_master_ops.bind.
*/
int component_bind_all(struct device *parent, void *data)
{
struct aggregate_device *adev;
struct component *c;
size_t i;
int ret = 0;
WARN_ON(!mutex_is_locked(&component_mutex));
adev = __aggregate_find(parent, NULL);
if (!adev)
return -EINVAL;
/* Bind components in match order */
for (i = 0; i < adev->match->num; i++)
if (!adev->match->compare[i].duplicate) {
c = adev->match->compare[i].component;
ret = component_bind(c, adev, data);
if (ret)
break;
}
if (ret != 0) {
for (; i > 0; i--)
if (!adev->match->compare[i - 1].duplicate) {
c = adev->match->compare[i - 1].component;
component_unbind(c, adev, data);
}
}
return ret;
}
EXPORT_SYMBOL_GPL(component_bind_all);
static int __component_add(struct device *dev, const struct component_ops *ops,
int subcomponent)
{
struct component *component;
int ret;
component = kzalloc(sizeof(*component), GFP_KERNEL);
if (!component)
return -ENOMEM;
component->ops = ops;
component->dev = dev;
component->subcomponent = subcomponent;
dev_dbg(dev, "adding component (ops %ps)\n", ops);
mutex_lock(&component_mutex);
list_add_tail(&component->node, &component_list);
ret = try_to_bring_up_masters(component);
if (ret < 0) {
if (component->adev)
remove_component(component->adev, component);
list_del(&component->node);
kfree(component);
}
mutex_unlock(&component_mutex);
return ret < 0 ? ret : 0;
}
/**
* component_add_typed - register a component
* @dev: component device
* @ops: component callbacks
* @subcomponent: nonzero identifier for subcomponents
*
* Register a new component for @dev. Functions in @ops will be call when the
* aggregate driver is ready to bind the overall driver by calling
* component_bind_all(). See also &struct component_ops.
*
* @subcomponent must be nonzero and is used to differentiate between multiple
* components registered on the same device @dev. These components are match
* using component_match_add_typed().
*
* The component needs to be unregistered at driver unload/disconnect by
* calling component_del().
*
* See also component_add().
*/
int component_add_typed(struct device *dev, const struct component_ops *ops,
int subcomponent)
{
if (WARN_ON(subcomponent == 0))
return -EINVAL;
return __component_add(dev, ops, subcomponent);
}
EXPORT_SYMBOL_GPL(component_add_typed);
/**
* component_add - register a component
* @dev: component device
* @ops: component callbacks
*
* Register a new component for @dev. Functions in @ops will be called when the
* aggregate driver is ready to bind the overall driver by calling
* component_bind_all(). See also &struct component_ops.
*
* The component needs to be unregistered at driver unload/disconnect by
* calling component_del().
*
* See also component_add_typed() for a variant that allows multiple different
* components on the same device.
*/
int component_add(struct device *dev, const struct component_ops *ops)
{
return __component_add(dev, ops, 0);
}
EXPORT_SYMBOL_GPL(component_add);
/**
* component_del - unregister a component
* @dev: component device
* @ops: component callbacks
*
* Unregister a component added with component_add(). If the component is bound
* into an aggregate driver, this will force the entire aggregate driver, including
* all its components, to be unbound.
*/
void component_del(struct device *dev, const struct component_ops *ops)
{
struct component *c, *component = NULL;
mutex_lock(&component_mutex); list_for_each_entry(c, &component_list, node) if (c->dev == dev && c->ops == ops) { list_del(&c->node);
component = c;
break;
}
if (component && component->adev) { take_down_aggregate_device(component->adev); remove_component(component->adev, component);
}
mutex_unlock(&component_mutex);
WARN_ON(!component);
kfree(component);
}
EXPORT_SYMBOL_GPL(component_del);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_USB_H
#define __LINUX_USB_H
#include <linux/mod_devicetable.h>
#include <linux/usb/ch9.h>
#define USB_MAJOR 180
#define USB_DEVICE_MAJOR 189
#ifdef __KERNEL__
#include <linux/errno.h> /* for -ENODEV */
#include <linux/delay.h> /* for mdelay() */
#include <linux/interrupt.h> /* for in_interrupt() */
#include <linux/list.h> /* for struct list_head */
#include <linux/kref.h> /* for struct kref */
#include <linux/device.h> /* for struct device */
#include <linux/fs.h> /* for struct file_operations */
#include <linux/completion.h> /* for struct completion */
#include <linux/sched.h> /* for current && schedule_timeout */
#include <linux/mutex.h> /* for struct mutex */
#include <linux/pm_runtime.h> /* for runtime PM */
struct usb_device;
struct usb_driver;
/*-------------------------------------------------------------------------*/
/*
* Host-side wrappers for standard USB descriptors ... these are parsed
* from the data provided by devices. Parsing turns them from a flat
* sequence of descriptors into a hierarchy:
*
* - devices have one (usually) or more configs;
* - configs have one (often) or more interfaces;
* - interfaces have one (usually) or more settings;
* - each interface setting has zero or (usually) more endpoints.
* - a SuperSpeed endpoint has a companion descriptor
*
* And there might be other descriptors mixed in with those.
*
* Devices may also have class-specific or vendor-specific descriptors.
*/
struct ep_device;
/**
* struct usb_host_endpoint - host-side endpoint descriptor and queue
* @desc: descriptor for this endpoint, wMaxPacketSize in native byteorder
* @ss_ep_comp: SuperSpeed companion descriptor for this endpoint
* @ssp_isoc_ep_comp: SuperSpeedPlus isoc companion descriptor for this endpoint
* @urb_list: urbs queued to this endpoint; maintained by usbcore
* @hcpriv: for use by HCD; typically holds hardware dma queue head (QH)
* with one or more transfer descriptors (TDs) per urb
* @ep_dev: ep_device for sysfs info
* @extra: descriptors following this endpoint in the configuration
* @extralen: how many bytes of "extra" are valid
* @enabled: URBs may be submitted to this endpoint
* @streams: number of USB-3 streams allocated on the endpoint
*
* USB requests are always queued to a given endpoint, identified by a
* descriptor within an active interface in a given USB configuration.
*/
struct usb_host_endpoint {
struct usb_endpoint_descriptor desc;
struct usb_ss_ep_comp_descriptor ss_ep_comp;
struct usb_ssp_isoc_ep_comp_descriptor ssp_isoc_ep_comp;
struct list_head urb_list;
void *hcpriv;
struct ep_device *ep_dev; /* For sysfs info */
unsigned char *extra; /* Extra descriptors */
int extralen;
int enabled;
int streams;
};
/* host-side wrapper for one interface setting's parsed descriptors */
struct usb_host_interface {
struct usb_interface_descriptor desc;
int extralen;
unsigned char *extra; /* Extra descriptors */
/* array of desc.bNumEndpoints endpoints associated with this
* interface setting. these will be in no particular order.
*/
struct usb_host_endpoint *endpoint;
char *string; /* iInterface string, if present */
};
enum usb_interface_condition {
USB_INTERFACE_UNBOUND = 0,
USB_INTERFACE_BINDING,
USB_INTERFACE_BOUND,
USB_INTERFACE_UNBINDING,
};
int __must_check
usb_find_common_endpoints(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_in,
struct usb_endpoint_descriptor **bulk_out,
struct usb_endpoint_descriptor **int_in,
struct usb_endpoint_descriptor **int_out);
int __must_check
usb_find_common_endpoints_reverse(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_in,
struct usb_endpoint_descriptor **bulk_out,
struct usb_endpoint_descriptor **int_in,
struct usb_endpoint_descriptor **int_out);
static inline int __must_check
usb_find_bulk_in_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_in)
{
return usb_find_common_endpoints(alt, bulk_in, NULL, NULL, NULL);
}
static inline int __must_check
usb_find_bulk_out_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_out)
{
return usb_find_common_endpoints(alt, NULL, bulk_out, NULL, NULL);
}
static inline int __must_check
usb_find_int_in_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **int_in)
{
return usb_find_common_endpoints(alt, NULL, NULL, int_in, NULL);
}
static inline int __must_check
usb_find_int_out_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **int_out)
{
return usb_find_common_endpoints(alt, NULL, NULL, NULL, int_out);
}
static inline int __must_check
usb_find_last_bulk_in_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_in)
{
return usb_find_common_endpoints_reverse(alt, bulk_in, NULL, NULL, NULL);
}
static inline int __must_check
usb_find_last_bulk_out_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_out)
{
return usb_find_common_endpoints_reverse(alt, NULL, bulk_out, NULL, NULL);
}
static inline int __must_check
usb_find_last_int_in_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **int_in)
{
return usb_find_common_endpoints_reverse(alt, NULL, NULL, int_in, NULL);
}
static inline int __must_check
usb_find_last_int_out_endpoint(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **int_out)
{
return usb_find_common_endpoints_reverse(alt, NULL, NULL, NULL, int_out);
}
enum usb_wireless_status {
USB_WIRELESS_STATUS_NA = 0,
USB_WIRELESS_STATUS_DISCONNECTED,
USB_WIRELESS_STATUS_CONNECTED,
};
/**
* struct usb_interface - what usb device drivers talk to
* @altsetting: array of interface structures, one for each alternate
* setting that may be selected. Each one includes a set of
* endpoint configurations. They will be in no particular order.
* @cur_altsetting: the current altsetting.
* @num_altsetting: number of altsettings defined.
* @intf_assoc: interface association descriptor
* @minor: the minor number assigned to this interface, if this
* interface is bound to a driver that uses the USB major number.
* If this interface does not use the USB major, this field should
* be unused. The driver should set this value in the probe()
* function of the driver, after it has been assigned a minor
* number from the USB core by calling usb_register_dev().
* @condition: binding state of the interface: not bound, binding
* (in probe()), bound to a driver, or unbinding (in disconnect())
* @sysfs_files_created: sysfs attributes exist
* @ep_devs_created: endpoint child pseudo-devices exist
* @unregistering: flag set when the interface is being unregistered
* @needs_remote_wakeup: flag set when the driver requires remote-wakeup
* capability during autosuspend.
* @needs_altsetting0: flag set when a set-interface request for altsetting 0
* has been deferred.
* @needs_binding: flag set when the driver should be re-probed or unbound
* following a reset or suspend operation it doesn't support.
* @authorized: This allows to (de)authorize individual interfaces instead
* a whole device in contrast to the device authorization.
* @wireless_status: if the USB device uses a receiver/emitter combo, whether
* the emitter is connected.
* @wireless_status_work: Used for scheduling wireless status changes
* from atomic context.
* @dev: driver model's view of this device
* @usb_dev: if an interface is bound to the USB major, this will point
* to the sysfs representation for that device.
* @reset_ws: Used for scheduling resets from atomic context.
* @resetting_device: USB core reset the device, so use alt setting 0 as
* current; needs bandwidth alloc after reset.
*
* USB device drivers attach to interfaces on a physical device. Each
* interface encapsulates a single high level function, such as feeding
* an audio stream to a speaker or reporting a change in a volume control.
* Many USB devices only have one interface. The protocol used to talk to
* an interface's endpoints can be defined in a usb "class" specification,
* or by a product's vendor. The (default) control endpoint is part of
* every interface, but is never listed among the interface's descriptors.
*
* The driver that is bound to the interface can use standard driver model
* calls such as dev_get_drvdata() on the dev member of this structure.
*
* Each interface may have alternate settings. The initial configuration
* of a device sets altsetting 0, but the device driver can change
* that setting using usb_set_interface(). Alternate settings are often
* used to control the use of periodic endpoints, such as by having
* different endpoints use different amounts of reserved USB bandwidth.
* All standards-conformant USB devices that use isochronous endpoints
* will use them in non-default settings.
*
* The USB specification says that alternate setting numbers must run from
* 0 to one less than the total number of alternate settings. But some
* devices manage to mess this up, and the structures aren't necessarily
* stored in numerical order anyhow. Use usb_altnum_to_altsetting() to
* look up an alternate setting in the altsetting array based on its number.
*/
struct usb_interface {
/* array of alternate settings for this interface,
* stored in no particular order */
struct usb_host_interface *altsetting;
struct usb_host_interface *cur_altsetting; /* the currently
* active alternate setting */
unsigned num_altsetting; /* number of alternate settings */
/* If there is an interface association descriptor then it will list
* the associated interfaces */
struct usb_interface_assoc_descriptor *intf_assoc;
int minor; /* minor number this interface is
* bound to */
enum usb_interface_condition condition; /* state of binding */
unsigned sysfs_files_created:1; /* the sysfs attributes exist */
unsigned ep_devs_created:1; /* endpoint "devices" exist */
unsigned unregistering:1; /* unregistration is in progress */
unsigned needs_remote_wakeup:1; /* driver requires remote wakeup */
unsigned needs_altsetting0:1; /* switch to altsetting 0 is pending */
unsigned needs_binding:1; /* needs delayed unbind/rebind */
unsigned resetting_device:1; /* true: bandwidth alloc after reset */
unsigned authorized:1; /* used for interface authorization */
enum usb_wireless_status wireless_status;
struct work_struct wireless_status_work;
struct device dev; /* interface specific device info */
struct device *usb_dev;
struct work_struct reset_ws; /* for resets in atomic context */
};
#define to_usb_interface(__dev) container_of_const(__dev, struct usb_interface, dev)
static inline void *usb_get_intfdata(struct usb_interface *intf)
{
return dev_get_drvdata(&intf->dev);
}
/**
* usb_set_intfdata() - associate driver-specific data with an interface
* @intf: USB interface
* @data: driver data
*
* Drivers can use this function in their probe() callbacks to associate
* driver-specific data with an interface.
*
* Note that there is generally no need to clear the driver-data pointer even
* if some drivers do so for historical or implementation-specific reasons.
*/
static inline void usb_set_intfdata(struct usb_interface *intf, void *data)
{
dev_set_drvdata(&intf->dev, data);
}
struct usb_interface *usb_get_intf(struct usb_interface *intf);
void usb_put_intf(struct usb_interface *intf);
/* Hard limit */
#define USB_MAXENDPOINTS 30
/* this maximum is arbitrary */
#define USB_MAXINTERFACES 32
#define USB_MAXIADS (USB_MAXINTERFACES/2)
bool usb_check_bulk_endpoints(
const struct usb_interface *intf, const u8 *ep_addrs);
bool usb_check_int_endpoints(
const struct usb_interface *intf, const u8 *ep_addrs);
/*
* USB Resume Timer: Every Host controller driver should drive the resume
* signalling on the bus for the amount of time defined by this macro.
*
* That way we will have a 'stable' behavior among all HCDs supported by Linux.
*
* Note that the USB Specification states we should drive resume for *at least*
* 20 ms, but it doesn't give an upper bound. This creates two possible
* situations which we want to avoid:
*
* (a) sometimes an msleep(20) might expire slightly before 20 ms, which causes
* us to fail USB Electrical Tests, thus failing Certification
*
* (b) Some (many) devices actually need more than 20 ms of resume signalling,
* and while we can argue that's against the USB Specification, we don't have
* control over which devices a certification laboratory will be using for
* certification. If CertLab uses a device which was tested against Windows and
* that happens to have relaxed resume signalling rules, we might fall into
* situations where we fail interoperability and electrical tests.
*
* In order to avoid both conditions, we're using a 40 ms resume timeout, which
* should cope with both LPJ calibration errors and devices not following every
* detail of the USB Specification.
*/
#define USB_RESUME_TIMEOUT 40 /* ms */
/**
* struct usb_interface_cache - long-term representation of a device interface
* @num_altsetting: number of altsettings defined.
* @ref: reference counter.
* @altsetting: variable-length array of interface structures, one for
* each alternate setting that may be selected. Each one includes a
* set of endpoint configurations. They will be in no particular order.
*
* These structures persist for the lifetime of a usb_device, unlike
* struct usb_interface (which persists only as long as its configuration
* is installed). The altsetting arrays can be accessed through these
* structures at any time, permitting comparison of configurations and
* providing support for the /sys/kernel/debug/usb/devices pseudo-file.
*/
struct usb_interface_cache {
unsigned num_altsetting; /* number of alternate settings */
struct kref ref; /* reference counter */
/* variable-length array of alternate settings for this interface,
* stored in no particular order */
struct usb_host_interface altsetting[];
};
#define ref_to_usb_interface_cache(r) \
container_of(r, struct usb_interface_cache, ref)
#define altsetting_to_usb_interface_cache(a) \
container_of(a, struct usb_interface_cache, altsetting[0])
/**
* struct usb_host_config - representation of a device's configuration
* @desc: the device's configuration descriptor.
* @string: pointer to the cached version of the iConfiguration string, if
* present for this configuration.
* @intf_assoc: list of any interface association descriptors in this config
* @interface: array of pointers to usb_interface structures, one for each
* interface in the configuration. The number of interfaces is stored
* in desc.bNumInterfaces. These pointers are valid only while the
* configuration is active.
* @intf_cache: array of pointers to usb_interface_cache structures, one
* for each interface in the configuration. These structures exist
* for the entire life of the device.
* @extra: pointer to buffer containing all extra descriptors associated
* with this configuration (those preceding the first interface
* descriptor).
* @extralen: length of the extra descriptors buffer.
*
* USB devices may have multiple configurations, but only one can be active
* at any time. Each encapsulates a different operational environment;
* for example, a dual-speed device would have separate configurations for
* full-speed and high-speed operation. The number of configurations
* available is stored in the device descriptor as bNumConfigurations.
*
* A configuration can contain multiple interfaces. Each corresponds to
* a different function of the USB device, and all are available whenever
* the configuration is active. The USB standard says that interfaces
* are supposed to be numbered from 0 to desc.bNumInterfaces-1, but a lot
* of devices get this wrong. In addition, the interface array is not
* guaranteed to be sorted in numerical order. Use usb_ifnum_to_if() to
* look up an interface entry based on its number.
*
* Device drivers should not attempt to activate configurations. The choice
* of which configuration to install is a policy decision based on such
* considerations as available power, functionality provided, and the user's
* desires (expressed through userspace tools). However, drivers can call
* usb_reset_configuration() to reinitialize the current configuration and
* all its interfaces.
*/
struct usb_host_config {
struct usb_config_descriptor desc;
char *string; /* iConfiguration string, if present */
/* List of any Interface Association Descriptors in this
* configuration. */
struct usb_interface_assoc_descriptor *intf_assoc[USB_MAXIADS];
/* the interfaces associated with this configuration,
* stored in no particular order */
struct usb_interface *interface[USB_MAXINTERFACES];
/* Interface information available even when this is not the
* active configuration */
struct usb_interface_cache *intf_cache[USB_MAXINTERFACES];
unsigned char *extra; /* Extra descriptors */
int extralen;
};
/* USB2.0 and USB3.0 device BOS descriptor set */
struct usb_host_bos {
struct usb_bos_descriptor *desc;
struct usb_ext_cap_descriptor *ext_cap;
struct usb_ss_cap_descriptor *ss_cap;
struct usb_ssp_cap_descriptor *ssp_cap;
struct usb_ss_container_id_descriptor *ss_id;
struct usb_ptm_cap_descriptor *ptm_cap;
};
int __usb_get_extra_descriptor(char *buffer, unsigned size,
unsigned char type, void **ptr, size_t min);
#define usb_get_extra_descriptor(ifpoint, type, ptr) \
__usb_get_extra_descriptor((ifpoint)->extra, \
(ifpoint)->extralen, \
type, (void **)ptr, sizeof(**(ptr)))
/* ----------------------------------------------------------------------- */
/*
* Allocated per bus (tree of devices) we have:
*/
struct usb_bus {
struct device *controller; /* host side hardware */
struct device *sysdev; /* as seen from firmware or bus */
int busnum; /* Bus number (in order of reg) */
const char *bus_name; /* stable id (PCI slot_name etc) */
u8 uses_pio_for_control; /*
* Does the host controller use PIO
* for control transfers?
*/
u8 otg_port; /* 0, or number of OTG/HNP port */
unsigned is_b_host:1; /* true during some HNP roleswitches */
unsigned b_hnp_enable:1; /* OTG: did A-Host enable HNP? */
unsigned no_stop_on_short:1; /*
* Quirk: some controllers don't stop
* the ep queue on a short transfer
* with the URB_SHORT_NOT_OK flag set.
*/
unsigned no_sg_constraint:1; /* no sg constraint */
unsigned sg_tablesize; /* 0 or largest number of sg list entries */
int devnum_next; /* Next open device number in
* round-robin allocation */
struct mutex devnum_next_mutex; /* devnum_next mutex */
DECLARE_BITMAP(devmap, 128); /* USB device number allocation bitmap */
struct usb_device *root_hub; /* Root hub */
struct usb_bus *hs_companion; /* Companion EHCI bus, if any */
int bandwidth_allocated; /* on this bus: how much of the time
* reserved for periodic (intr/iso)
* requests is used, on average?
* Units: microseconds/frame.
* Limits: Full/low speed reserve 90%,
* while high speed reserves 80%.
*/
int bandwidth_int_reqs; /* number of Interrupt requests */
int bandwidth_isoc_reqs; /* number of Isoc. requests */
unsigned resuming_ports; /* bit array: resuming root-hub ports */
#if defined(CONFIG_USB_MON) || defined(CONFIG_USB_MON_MODULE)
struct mon_bus *mon_bus; /* non-null when associated */
int monitored; /* non-zero when monitored */
#endif
};
struct usb_dev_state;
/* ----------------------------------------------------------------------- */
struct usb_tt;
enum usb_port_connect_type {
USB_PORT_CONNECT_TYPE_UNKNOWN = 0,
USB_PORT_CONNECT_TYPE_HOT_PLUG,
USB_PORT_CONNECT_TYPE_HARD_WIRED,
USB_PORT_NOT_USED,
};
/*
* USB port quirks.
*/
/* For the given port, prefer the old (faster) enumeration scheme. */
#define USB_PORT_QUIRK_OLD_SCHEME BIT(0)
/* Decrease TRSTRCY to 10ms during device enumeration. */
#define USB_PORT_QUIRK_FAST_ENUM BIT(1)
/*
* USB 2.0 Link Power Management (LPM) parameters.
*/
struct usb2_lpm_parameters {
/* Best effort service latency indicate how long the host will drive
* resume on an exit from L1.
*/
unsigned int besl;
/* Timeout value in microseconds for the L1 inactivity (LPM) timer.
* When the timer counts to zero, the parent hub will initiate a LPM
* transition to L1.
*/
int timeout;
};
/*
* USB 3.0 Link Power Management (LPM) parameters.
*
* PEL and SEL are USB 3.0 Link PM latencies for device-initiated LPM exit.
* MEL is the USB 3.0 Link PM latency for host-initiated LPM exit.
* All three are stored in nanoseconds.
*/
struct usb3_lpm_parameters {
/*
* Maximum exit latency (MEL) for the host to send a packet to the
* device (either a Ping for isoc endpoints, or a data packet for
* interrupt endpoints), the hubs to decode the packet, and for all hubs
* in the path to transition the links to U0.
*/
unsigned int mel;
/*
* Maximum exit latency for a device-initiated LPM transition to bring
* all links into U0. Abbreviated as "PEL" in section 9.4.12 of the USB
* 3.0 spec, with no explanation of what "P" stands for. "Path"?
*/
unsigned int pel;
/*
* The System Exit Latency (SEL) includes PEL, and three other
* latencies. After a device initiates a U0 transition, it will take
* some time from when the device sends the ERDY to when it will finally
* receive the data packet. Basically, SEL should be the worse-case
* latency from when a device starts initiating a U0 transition to when
* it will get data.
*/
unsigned int sel;
/*
* The idle timeout value that is currently programmed into the parent
* hub for this device. When the timer counts to zero, the parent hub
* will initiate an LPM transition to either U1 or U2.
*/
int timeout;
};
/**
* struct usb_device - kernel's representation of a USB device
* @devnum: device number; address on a USB bus
* @devpath: device ID string for use in messages (e.g., /port/...)
* @route: tree topology hex string for use with xHCI
* @state: device state: configured, not attached, etc.
* @speed: device speed: high/full/low (or error)
* @rx_lanes: number of rx lanes in use, USB 3.2 adds dual-lane support
* @tx_lanes: number of tx lanes in use, USB 3.2 adds dual-lane support
* @ssp_rate: SuperSpeed Plus phy signaling rate and lane count
* @tt: Transaction Translator info; used with low/full speed dev, highspeed hub
* @ttport: device port on that tt hub
* @toggle: one bit for each endpoint, with ([0] = IN, [1] = OUT) endpoints
* @parent: our hub, unless we're the root
* @bus: bus we're part of
* @ep0: endpoint 0 data (default control pipe)
* @dev: generic device interface
* @descriptor: USB device descriptor
* @bos: USB device BOS descriptor set
* @config: all of the device's configs
* @actconfig: the active configuration
* @ep_in: array of IN endpoints
* @ep_out: array of OUT endpoints
* @rawdescriptors: raw descriptors for each config
* @bus_mA: Current available from the bus
* @portnum: parent port number (origin 1)
* @level: number of USB hub ancestors
* @devaddr: device address, XHCI: assigned by HW, others: same as devnum
* @can_submit: URBs may be submitted
* @persist_enabled: USB_PERSIST enabled for this device
* @reset_in_progress: the device is being reset
* @have_langid: whether string_langid is valid
* @authorized: policy has said we can use it;
* (user space) policy determines if we authorize this device to be
* used or not. By default, wired USB devices are authorized.
* WUSB devices are not, until we authorize them from user space.
* FIXME -- complete doc
* @authenticated: Crypto authentication passed
* @lpm_capable: device supports LPM
* @lpm_devinit_allow: Allow USB3 device initiated LPM, exit latency is in range
* @usb2_hw_lpm_capable: device can perform USB2 hardware LPM
* @usb2_hw_lpm_besl_capable: device can perform USB2 hardware BESL LPM
* @usb2_hw_lpm_enabled: USB2 hardware LPM is enabled
* @usb2_hw_lpm_allowed: Userspace allows USB 2.0 LPM to be enabled
* @usb3_lpm_u1_enabled: USB3 hardware U1 LPM enabled
* @usb3_lpm_u2_enabled: USB3 hardware U2 LPM enabled
* @string_langid: language ID for strings
* @product: iProduct string, if present (static)
* @manufacturer: iManufacturer string, if present (static)
* @serial: iSerialNumber string, if present (static)
* @filelist: usbfs files that are open to this device
* @maxchild: number of ports if hub
* @quirks: quirks of the whole device
* @urbnum: number of URBs submitted for the whole device
* @active_duration: total time device is not suspended
* @connect_time: time device was first connected
* @do_remote_wakeup: remote wakeup should be enabled
* @reset_resume: needs reset instead of resume
* @port_is_suspended: the upstream port is suspended (L2 or U3)
* @slot_id: Slot ID assigned by xHCI
* @l1_params: best effor service latency for USB2 L1 LPM state, and L1 timeout.
* @u1_params: exit latencies for USB3 U1 LPM state, and hub-initiated timeout.
* @u2_params: exit latencies for USB3 U2 LPM state, and hub-initiated timeout.
* @lpm_disable_count: Ref count used by usb_disable_lpm() and usb_enable_lpm()
* to keep track of the number of functions that require USB 3.0 Link Power
* Management to be disabled for this usb_device. This count should only
* be manipulated by those functions, with the bandwidth_mutex is held.
* @hub_delay: cached value consisting of:
* parent->hub_delay + wHubDelay + tTPTransmissionDelay (40ns)
* Will be used as wValue for SetIsochDelay requests.
* @use_generic_driver: ask driver core to reprobe using the generic driver.
*
* Notes:
* Usbcore drivers should not set usbdev->state directly. Instead use
* usb_set_device_state().
*/
struct usb_device {
int devnum;
char devpath[16];
u32 route;
enum usb_device_state state;
enum usb_device_speed speed;
unsigned int rx_lanes;
unsigned int tx_lanes;
enum usb_ssp_rate ssp_rate;
struct usb_tt *tt;
int ttport;
unsigned int toggle[2];
struct usb_device *parent;
struct usb_bus *bus;
struct usb_host_endpoint ep0;
struct device dev;
struct usb_device_descriptor descriptor;
struct usb_host_bos *bos;
struct usb_host_config *config;
struct usb_host_config *actconfig;
struct usb_host_endpoint *ep_in[16];
struct usb_host_endpoint *ep_out[16];
char **rawdescriptors;
unsigned short bus_mA;
u8 portnum;
u8 level;
u8 devaddr;
unsigned can_submit:1;
unsigned persist_enabled:1;
unsigned reset_in_progress:1;
unsigned have_langid:1;
unsigned authorized:1;
unsigned authenticated:1;
unsigned lpm_capable:1;
unsigned lpm_devinit_allow:1;
unsigned usb2_hw_lpm_capable:1;
unsigned usb2_hw_lpm_besl_capable:1;
unsigned usb2_hw_lpm_enabled:1;
unsigned usb2_hw_lpm_allowed:1;
unsigned usb3_lpm_u1_enabled:1;
unsigned usb3_lpm_u2_enabled:1;
int string_langid;
/* static strings from the device */
char *product;
char *manufacturer;
char *serial;
struct list_head filelist;
int maxchild;
u32 quirks;
atomic_t urbnum;
unsigned long active_duration;
unsigned long connect_time;
unsigned do_remote_wakeup:1;
unsigned reset_resume:1;
unsigned port_is_suspended:1;
int slot_id;
struct usb2_lpm_parameters l1_params;
struct usb3_lpm_parameters u1_params;
struct usb3_lpm_parameters u2_params;
unsigned lpm_disable_count;
u16 hub_delay;
unsigned use_generic_driver:1;
};
#define to_usb_device(__dev) container_of_const(__dev, struct usb_device, dev)
static inline struct usb_device *__intf_to_usbdev(struct usb_interface *intf)
{
return to_usb_device(intf->dev.parent);
}
static inline const struct usb_device *__intf_to_usbdev_const(const struct usb_interface *intf)
{
return to_usb_device((const struct device *)intf->dev.parent);
}
#define interface_to_usbdev(intf) \
_Generic((intf), \
const struct usb_interface *: __intf_to_usbdev_const, \
struct usb_interface *: __intf_to_usbdev)(intf)
extern struct usb_device *usb_get_dev(struct usb_device *dev);
extern void usb_put_dev(struct usb_device *dev);
extern struct usb_device *usb_hub_find_child(struct usb_device *hdev,
int port1);
/**
* usb_hub_for_each_child - iterate over all child devices on the hub
* @hdev: USB device belonging to the usb hub
* @port1: portnum associated with child device
* @child: child device pointer
*/
#define usb_hub_for_each_child(hdev, port1, child) \
for (port1 = 1, child = usb_hub_find_child(hdev, port1); \
port1 <= hdev->maxchild; \
child = usb_hub_find_child(hdev, ++port1)) \
if (!child) continue; else
/* USB device locking */
#define usb_lock_device(udev) device_lock(&(udev)->dev)
#define usb_unlock_device(udev) device_unlock(&(udev)->dev)
#define usb_lock_device_interruptible(udev) device_lock_interruptible(&(udev)->dev)
#define usb_trylock_device(udev) device_trylock(&(udev)->dev)
extern int usb_lock_device_for_reset(struct usb_device *udev,
const struct usb_interface *iface);
/* USB port reset for device reinitialization */
extern int usb_reset_device(struct usb_device *dev);
extern void usb_queue_reset_device(struct usb_interface *dev);
extern struct device *usb_intf_get_dma_device(struct usb_interface *intf);
#ifdef CONFIG_ACPI
extern int usb_acpi_set_power_state(struct usb_device *hdev, int index,
bool enable);
extern bool usb_acpi_power_manageable(struct usb_device *hdev, int index);
extern int usb_acpi_port_lpm_incapable(struct usb_device *hdev, int index);
#else
static inline int usb_acpi_set_power_state(struct usb_device *hdev, int index,
bool enable) { return 0; }
static inline bool usb_acpi_power_manageable(struct usb_device *hdev, int index)
{ return true; }
static inline int usb_acpi_port_lpm_incapable(struct usb_device *hdev, int index)
{ return 0; }
#endif
/* USB autosuspend and autoresume */
#ifdef CONFIG_PM
extern void usb_enable_autosuspend(struct usb_device *udev);
extern void usb_disable_autosuspend(struct usb_device *udev);
extern int usb_autopm_get_interface(struct usb_interface *intf);
extern void usb_autopm_put_interface(struct usb_interface *intf);
extern int usb_autopm_get_interface_async(struct usb_interface *intf);
extern void usb_autopm_put_interface_async(struct usb_interface *intf);
extern void usb_autopm_get_interface_no_resume(struct usb_interface *intf);
extern void usb_autopm_put_interface_no_suspend(struct usb_interface *intf);
static inline void usb_mark_last_busy(struct usb_device *udev)
{
pm_runtime_mark_last_busy(&udev->dev);
}
#else
static inline int usb_enable_autosuspend(struct usb_device *udev)
{ return 0; }
static inline int usb_disable_autosuspend(struct usb_device *udev)
{ return 0; }
static inline int usb_autopm_get_interface(struct usb_interface *intf)
{ return 0; }
static inline int usb_autopm_get_interface_async(struct usb_interface *intf)
{ return 0; }
static inline void usb_autopm_put_interface(struct usb_interface *intf)
{ }
static inline void usb_autopm_put_interface_async(struct usb_interface *intf)
{ }
static inline void usb_autopm_get_interface_no_resume(
struct usb_interface *intf)
{ }
static inline void usb_autopm_put_interface_no_suspend(
struct usb_interface *intf)
{ }
static inline void usb_mark_last_busy(struct usb_device *udev)
{ }
#endif
extern int usb_disable_lpm(struct usb_device *udev);
extern void usb_enable_lpm(struct usb_device *udev);
/* Same as above, but these functions lock/unlock the bandwidth_mutex. */
extern int usb_unlocked_disable_lpm(struct usb_device *udev);
extern void usb_unlocked_enable_lpm(struct usb_device *udev);
extern int usb_disable_ltm(struct usb_device *udev);
extern void usb_enable_ltm(struct usb_device *udev);
static inline bool usb_device_supports_ltm(struct usb_device *udev)
{
if (udev->speed < USB_SPEED_SUPER || !udev->bos || !udev->bos->ss_cap)
return false;
return udev->bos->ss_cap->bmAttributes & USB_LTM_SUPPORT;
}
static inline bool usb_device_no_sg_constraint(struct usb_device *udev)
{
return udev && udev->bus && udev->bus->no_sg_constraint;
}
/*-------------------------------------------------------------------------*/
/* for drivers using iso endpoints */
extern int usb_get_current_frame_number(struct usb_device *usb_dev);
/* Sets up a group of bulk endpoints to support multiple stream IDs. */
extern int usb_alloc_streams(struct usb_interface *interface,
struct usb_host_endpoint **eps, unsigned int num_eps,
unsigned int num_streams, gfp_t mem_flags);
/* Reverts a group of bulk endpoints back to not using stream IDs. */
extern int usb_free_streams(struct usb_interface *interface,
struct usb_host_endpoint **eps, unsigned int num_eps,
gfp_t mem_flags);
/* used these for multi-interface device registration */
extern int usb_driver_claim_interface(struct usb_driver *driver,
struct usb_interface *iface, void *data);
/**
* usb_interface_claimed - returns true iff an interface is claimed
* @iface: the interface being checked
*
* Return: %true (nonzero) iff the interface is claimed, else %false
* (zero).
*
* Note:
* Callers must own the driver model's usb bus readlock. So driver
* probe() entries don't need extra locking, but other call contexts
* may need to explicitly claim that lock.
*
*/
static inline int usb_interface_claimed(struct usb_interface *iface)
{
return (iface->dev.driver != NULL);
}
extern void usb_driver_release_interface(struct usb_driver *driver,
struct usb_interface *iface);
int usb_set_wireless_status(struct usb_interface *iface,
enum usb_wireless_status status);
const struct usb_device_id *usb_match_id(struct usb_interface *interface,
const struct usb_device_id *id);
extern int usb_match_one_id(struct usb_interface *interface,
const struct usb_device_id *id);
extern int usb_for_each_dev(void *data, int (*fn)(struct usb_device *, void *));
extern struct usb_interface *usb_find_interface(struct usb_driver *drv,
int minor);
extern struct usb_interface *usb_ifnum_to_if(const struct usb_device *dev,
unsigned ifnum);
extern struct usb_host_interface *usb_altnum_to_altsetting(
const struct usb_interface *intf, unsigned int altnum);
extern struct usb_host_interface *usb_find_alt_setting(
struct usb_host_config *config,
unsigned int iface_num,
unsigned int alt_num);
/* port claiming functions */
int usb_hub_claim_port(struct usb_device *hdev, unsigned port1,
struct usb_dev_state *owner);
int usb_hub_release_port(struct usb_device *hdev, unsigned port1,
struct usb_dev_state *owner);
/**
* usb_make_path - returns stable device path in the usb tree
* @dev: the device whose path is being constructed
* @buf: where to put the string
* @size: how big is "buf"?
*
* Return: Length of the string (> 0) or negative if size was too small.
*
* Note:
* This identifier is intended to be "stable", reflecting physical paths in
* hardware such as physical bus addresses for host controllers or ports on
* USB hubs. That makes it stay the same until systems are physically
* reconfigured, by re-cabling a tree of USB devices or by moving USB host
* controllers. Adding and removing devices, including virtual root hubs
* in host controller driver modules, does not change these path identifiers;
* neither does rebooting or re-enumerating. These are more useful identifiers
* than changeable ("unstable") ones like bus numbers or device addresses.
*
* With a partial exception for devices connected to USB 2.0 root hubs, these
* identifiers are also predictable. So long as the device tree isn't changed,
* plugging any USB device into a given hub port always gives it the same path.
* Because of the use of "companion" controllers, devices connected to ports on
* USB 2.0 root hubs (EHCI host controllers) will get one path ID if they are
* high speed, and a different one if they are full or low speed.
*/
static inline int usb_make_path(struct usb_device *dev, char *buf, size_t size)
{
int actual;
actual = snprintf(buf, size, "usb-%s-%s", dev->bus->bus_name,
dev->devpath);
return (actual >= (int)size) ? -1 : actual;
}
/*-------------------------------------------------------------------------*/
#define USB_DEVICE_ID_MATCH_DEVICE \
(USB_DEVICE_ID_MATCH_VENDOR | USB_DEVICE_ID_MATCH_PRODUCT)
#define USB_DEVICE_ID_MATCH_DEV_RANGE \
(USB_DEVICE_ID_MATCH_DEV_LO | USB_DEVICE_ID_MATCH_DEV_HI)
#define USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION \
(USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_DEV_RANGE)
#define USB_DEVICE_ID_MATCH_DEV_INFO \
(USB_DEVICE_ID_MATCH_DEV_CLASS | \
USB_DEVICE_ID_MATCH_DEV_SUBCLASS | \
USB_DEVICE_ID_MATCH_DEV_PROTOCOL)
#define USB_DEVICE_ID_MATCH_INT_INFO \
(USB_DEVICE_ID_MATCH_INT_CLASS | \
USB_DEVICE_ID_MATCH_INT_SUBCLASS | \
USB_DEVICE_ID_MATCH_INT_PROTOCOL)
/**
* USB_DEVICE - macro used to describe a specific usb device
* @vend: the 16 bit USB Vendor ID
* @prod: the 16 bit USB Product ID
*
* This macro is used to create a struct usb_device_id that matches a
* specific device.
*/
#define USB_DEVICE(vend, prod) \
.match_flags = USB_DEVICE_ID_MATCH_DEVICE, \
.idVendor = (vend), \
.idProduct = (prod)
/**
* USB_DEVICE_VER - describe a specific usb device with a version range
* @vend: the 16 bit USB Vendor ID
* @prod: the 16 bit USB Product ID
* @lo: the bcdDevice_lo value
* @hi: the bcdDevice_hi value
*
* This macro is used to create a struct usb_device_id that matches a
* specific device, with a version range.
*/
#define USB_DEVICE_VER(vend, prod, lo, hi) \
.match_flags = USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION, \
.idVendor = (vend), \
.idProduct = (prod), \
.bcdDevice_lo = (lo), \
.bcdDevice_hi = (hi)
/**
* USB_DEVICE_INTERFACE_CLASS - describe a usb device with a specific interface class
* @vend: the 16 bit USB Vendor ID
* @prod: the 16 bit USB Product ID
* @cl: bInterfaceClass value
*
* This macro is used to create a struct usb_device_id that matches a
* specific interface class of devices.
*/
#define USB_DEVICE_INTERFACE_CLASS(vend, prod, cl) \
.match_flags = USB_DEVICE_ID_MATCH_DEVICE | \
USB_DEVICE_ID_MATCH_INT_CLASS, \
.idVendor = (vend), \
.idProduct = (prod), \
.bInterfaceClass = (cl)
/**
* USB_DEVICE_INTERFACE_PROTOCOL - describe a usb device with a specific interface protocol
* @vend: the 16 bit USB Vendor ID
* @prod: the 16 bit USB Product ID
* @pr: bInterfaceProtocol value
*
* This macro is used to create a struct usb_device_id that matches a
* specific interface protocol of devices.
*/
#define USB_DEVICE_INTERFACE_PROTOCOL(vend, prod, pr) \
.match_flags = USB_DEVICE_ID_MATCH_DEVICE | \
USB_DEVICE_ID_MATCH_INT_PROTOCOL, \
.idVendor = (vend), \
.idProduct = (prod), \
.bInterfaceProtocol = (pr)
/**
* USB_DEVICE_INTERFACE_NUMBER - describe a usb device with a specific interface number
* @vend: the 16 bit USB Vendor ID
* @prod: the 16 bit USB Product ID
* @num: bInterfaceNumber value
*
* This macro is used to create a struct usb_device_id that matches a
* specific interface number of devices.
*/
#define USB_DEVICE_INTERFACE_NUMBER(vend, prod, num) \
.match_flags = USB_DEVICE_ID_MATCH_DEVICE | \
USB_DEVICE_ID_MATCH_INT_NUMBER, \
.idVendor = (vend), \
.idProduct = (prod), \
.bInterfaceNumber = (num)
/**
* USB_DEVICE_INFO - macro used to describe a class of usb devices
* @cl: bDeviceClass value
* @sc: bDeviceSubClass value
* @pr: bDeviceProtocol value
*
* This macro is used to create a struct usb_device_id that matches a
* specific class of devices.
*/
#define USB_DEVICE_INFO(cl, sc, pr) \
.match_flags = USB_DEVICE_ID_MATCH_DEV_INFO, \
.bDeviceClass = (cl), \
.bDeviceSubClass = (sc), \
.bDeviceProtocol = (pr)
/**
* USB_INTERFACE_INFO - macro used to describe a class of usb interfaces
* @cl: bInterfaceClass value
* @sc: bInterfaceSubClass value
* @pr: bInterfaceProtocol value
*
* This macro is used to create a struct usb_device_id that matches a
* specific class of interfaces.
*/
#define USB_INTERFACE_INFO(cl, sc, pr) \
.match_flags = USB_DEVICE_ID_MATCH_INT_INFO, \
.bInterfaceClass = (cl), \
.bInterfaceSubClass = (sc), \
.bInterfaceProtocol = (pr)
/**
* USB_DEVICE_AND_INTERFACE_INFO - describe a specific usb device with a class of usb interfaces
* @vend: the 16 bit USB Vendor ID
* @prod: the 16 bit USB Product ID
* @cl: bInterfaceClass value
* @sc: bInterfaceSubClass value
* @pr: bInterfaceProtocol value
*
* This macro is used to create a struct usb_device_id that matches a
* specific device with a specific class of interfaces.
*
* This is especially useful when explicitly matching devices that have
* vendor specific bDeviceClass values, but standards-compliant interfaces.
*/
#define USB_DEVICE_AND_INTERFACE_INFO(vend, prod, cl, sc, pr) \
.match_flags = USB_DEVICE_ID_MATCH_INT_INFO \
| USB_DEVICE_ID_MATCH_DEVICE, \
.idVendor = (vend), \
.idProduct = (prod), \
.bInterfaceClass = (cl), \
.bInterfaceSubClass = (sc), \
.bInterfaceProtocol = (pr)
/**
* USB_VENDOR_AND_INTERFACE_INFO - describe a specific usb vendor with a class of usb interfaces
* @vend: the 16 bit USB Vendor ID
* @cl: bInterfaceClass value
* @sc: bInterfaceSubClass value
* @pr: bInterfaceProtocol value
*
* This macro is used to create a struct usb_device_id that matches a
* specific vendor with a specific class of interfaces.
*
* This is especially useful when explicitly matching devices that have
* vendor specific bDeviceClass values, but standards-compliant interfaces.
*/
#define USB_VENDOR_AND_INTERFACE_INFO(vend, cl, sc, pr) \
.match_flags = USB_DEVICE_ID_MATCH_INT_INFO \
| USB_DEVICE_ID_MATCH_VENDOR, \
.idVendor = (vend), \
.bInterfaceClass = (cl), \
.bInterfaceSubClass = (sc), \
.bInterfaceProtocol = (pr)
/* ----------------------------------------------------------------------- */
/* Stuff for dynamic usb ids */
struct usb_dynids {
spinlock_t lock;
struct list_head list;
};
struct usb_dynid {
struct list_head node;
struct usb_device_id id;
};
extern ssize_t usb_store_new_id(struct usb_dynids *dynids,
const struct usb_device_id *id_table,
struct device_driver *driver,
const char *buf, size_t count);
extern ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf);
/**
* struct usb_driver - identifies USB interface driver to usbcore
* @name: The driver name should be unique among USB drivers,
* and should normally be the same as the module name.
* @probe: Called to see if the driver is willing to manage a particular
* interface on a device. If it is, probe returns zero and uses
* usb_set_intfdata() to associate driver-specific data with the
* interface. It may also use usb_set_interface() to specify the
* appropriate altsetting. If unwilling to manage the interface,
* return -ENODEV, if genuine IO errors occurred, an appropriate
* negative errno value.
* @disconnect: Called when the interface is no longer accessible, usually
* because its device has been (or is being) disconnected or the
* driver module is being unloaded.
* @unlocked_ioctl: Used for drivers that want to talk to userspace through
* the "usbfs" filesystem. This lets devices provide ways to
* expose information to user space regardless of where they
* do (or don't) show up otherwise in the filesystem.
* @suspend: Called when the device is going to be suspended by the
* system either from system sleep or runtime suspend context. The
* return value will be ignored in system sleep context, so do NOT
* try to continue using the device if suspend fails in this case.
* Instead, let the resume or reset-resume routine recover from
* the failure.
* @resume: Called when the device is being resumed by the system.
* @reset_resume: Called when the suspended device has been reset instead
* of being resumed.
* @pre_reset: Called by usb_reset_device() when the device is about to be
* reset. This routine must not return until the driver has no active
* URBs for the device, and no more URBs may be submitted until the
* post_reset method is called.
* @post_reset: Called by usb_reset_device() after the device
* has been reset
* @id_table: USB drivers use ID table to support hotplugging.
* Export this with MODULE_DEVICE_TABLE(usb,...). This must be set
* or your driver's probe function will never get called.
* @dev_groups: Attributes attached to the device that will be created once it
* is bound to the driver.
* @dynids: used internally to hold the list of dynamically added device
* ids for this driver.
* @driver: The driver-model core driver structure.
* @no_dynamic_id: if set to 1, the USB core will not allow dynamic ids to be
* added to this driver by preventing the sysfs file from being created.
* @supports_autosuspend: if set to 0, the USB core will not allow autosuspend
* for interfaces bound to this driver.
* @soft_unbind: if set to 1, the USB core will not kill URBs and disable
* endpoints before calling the driver's disconnect method.
* @disable_hub_initiated_lpm: if set to 1, the USB core will not allow hubs
* to initiate lower power link state transitions when an idle timeout
* occurs. Device-initiated USB 3.0 link PM will still be allowed.
*
* USB interface drivers must provide a name, probe() and disconnect()
* methods, and an id_table. Other driver fields are optional.
*
* The id_table is used in hotplugging. It holds a set of descriptors,
* and specialized data may be associated with each entry. That table
* is used by both user and kernel mode hotplugging support.
*
* The probe() and disconnect() methods are called in a context where
* they can sleep, but they should avoid abusing the privilege. Most
* work to connect to a device should be done when the device is opened,
* and undone at the last close. The disconnect code needs to address
* concurrency issues with respect to open() and close() methods, as
* well as forcing all pending I/O requests to complete (by unlinking
* them as necessary, and blocking until the unlinks complete).
*/
struct usb_driver {
const char *name;
int (*probe) (struct usb_interface *intf,
const struct usb_device_id *id);
void (*disconnect) (struct usb_interface *intf);
int (*unlocked_ioctl) (struct usb_interface *intf, unsigned int code,
void *buf);
int (*suspend) (struct usb_interface *intf, pm_message_t message);
int (*resume) (struct usb_interface *intf);
int (*reset_resume)(struct usb_interface *intf);
int (*pre_reset)(struct usb_interface *intf);
int (*post_reset)(struct usb_interface *intf);
const struct usb_device_id *id_table;
const struct attribute_group **dev_groups;
struct usb_dynids dynids;
struct device_driver driver;
unsigned int no_dynamic_id:1;
unsigned int supports_autosuspend:1;
unsigned int disable_hub_initiated_lpm:1;
unsigned int soft_unbind:1;
};
#define to_usb_driver(d) container_of(d, struct usb_driver, driver)
/**
* struct usb_device_driver - identifies USB device driver to usbcore
* @name: The driver name should be unique among USB drivers,
* and should normally be the same as the module name.
* @match: If set, used for better device/driver matching.
* @probe: Called to see if the driver is willing to manage a particular
* device. If it is, probe returns zero and uses dev_set_drvdata()
* to associate driver-specific data with the device. If unwilling
* to manage the device, return a negative errno value.
* @disconnect: Called when the device is no longer accessible, usually
* because it has been (or is being) disconnected or the driver's
* module is being unloaded.
* @suspend: Called when the device is going to be suspended by the system.
* @resume: Called when the device is being resumed by the system.
* @choose_configuration: If non-NULL, called instead of the default
* usb_choose_configuration(). If this returns an error then we'll go
* on to call the normal usb_choose_configuration().
* @dev_groups: Attributes attached to the device that will be created once it
* is bound to the driver.
* @driver: The driver-model core driver structure.
* @id_table: used with @match() to select better matching driver at
* probe() time.
* @supports_autosuspend: if set to 0, the USB core will not allow autosuspend
* for devices bound to this driver.
* @generic_subclass: if set to 1, the generic USB driver's probe, disconnect,
* resume and suspend functions will be called in addition to the driver's
* own, so this part of the setup does not need to be replicated.
*
* USB drivers must provide all the fields listed above except driver,
* match, and id_table.
*/
struct usb_device_driver {
const char *name;
bool (*match) (struct usb_device *udev);
int (*probe) (struct usb_device *udev);
void (*disconnect) (struct usb_device *udev);
int (*suspend) (struct usb_device *udev, pm_message_t message);
int (*resume) (struct usb_device *udev, pm_message_t message);
int (*choose_configuration) (struct usb_device *udev);
const struct attribute_group **dev_groups;
struct device_driver driver;
const struct usb_device_id *id_table;
unsigned int supports_autosuspend:1;
unsigned int generic_subclass:1;
};
#define to_usb_device_driver(d) container_of(d, struct usb_device_driver, \
driver)
/**
* struct usb_class_driver - identifies a USB driver that wants to use the USB major number
* @name: the usb class device name for this driver. Will show up in sysfs.
* @devnode: Callback to provide a naming hint for a possible
* device node to create.
* @fops: pointer to the struct file_operations of this driver.
* @minor_base: the start of the minor range for this driver.
*
* This structure is used for the usb_register_dev() and
* usb_deregister_dev() functions, to consolidate a number of the
* parameters used for them.
*/
struct usb_class_driver {
char *name;
char *(*devnode)(const struct device *dev, umode_t *mode);
const struct file_operations *fops;
int minor_base;
};
/*
* use these in module_init()/module_exit()
* and don't forget MODULE_DEVICE_TABLE(usb, ...)
*/
extern int usb_register_driver(struct usb_driver *, struct module *,
const char *);
/* use a define to avoid include chaining to get THIS_MODULE & friends */
#define usb_register(driver) \
usb_register_driver(driver, THIS_MODULE, KBUILD_MODNAME)
extern void usb_deregister(struct usb_driver *);
/**
* module_usb_driver() - Helper macro for registering a USB driver
* @__usb_driver: usb_driver struct
*
* Helper macro for USB drivers which do not do anything special in module
* init/exit. This eliminates a lot of boilerplate. Each module may only
* use this macro once, and calling it replaces module_init() and module_exit()
*/
#define module_usb_driver(__usb_driver) \
module_driver(__usb_driver, usb_register, \
usb_deregister)
extern int usb_register_device_driver(struct usb_device_driver *,
struct module *);
extern void usb_deregister_device_driver(struct usb_device_driver *);
extern int usb_register_dev(struct usb_interface *intf,
struct usb_class_driver *class_driver);
extern void usb_deregister_dev(struct usb_interface *intf,
struct usb_class_driver *class_driver);
extern int usb_disabled(void);
/* ----------------------------------------------------------------------- */
/*
* URB support, for asynchronous request completions
*/
/*
* urb->transfer_flags:
*
* Note: URB_DIR_IN/OUT is automatically set in usb_submit_urb().
*/
#define URB_SHORT_NOT_OK 0x0001 /* report short reads as errors */
#define URB_ISO_ASAP 0x0002 /* iso-only; use the first unexpired
* slot in the schedule */
#define URB_NO_TRANSFER_DMA_MAP 0x0004 /* urb->transfer_dma valid on submit */
#define URB_ZERO_PACKET 0x0040 /* Finish bulk OUT with short packet */
#define URB_NO_INTERRUPT 0x0080 /* HINT: no non-error interrupt
* needed */
#define URB_FREE_BUFFER 0x0100 /* Free transfer buffer with the URB */
/* The following flags are used internally by usbcore and HCDs */
#define URB_DIR_IN 0x0200 /* Transfer from device to host */
#define URB_DIR_OUT 0
#define URB_DIR_MASK URB_DIR_IN
#define URB_DMA_MAP_SINGLE 0x00010000 /* Non-scatter-gather mapping */
#define URB_DMA_MAP_PAGE 0x00020000 /* HCD-unsupported S-G */
#define URB_DMA_MAP_SG 0x00040000 /* HCD-supported S-G */
#define URB_MAP_LOCAL 0x00080000 /* HCD-local-memory mapping */
#define URB_SETUP_MAP_SINGLE 0x00100000 /* Setup packet DMA mapped */
#define URB_SETUP_MAP_LOCAL 0x00200000 /* HCD-local setup packet */
#define URB_DMA_SG_COMBINED 0x00400000 /* S-G entries were combined */
#define URB_ALIGNED_TEMP_BUFFER 0x00800000 /* Temp buffer was alloc'd */
struct usb_iso_packet_descriptor {
unsigned int offset;
unsigned int length; /* expected length */
unsigned int actual_length;
int status;
};
struct urb;
struct usb_anchor {
struct list_head urb_list;
wait_queue_head_t wait;
spinlock_t lock;
atomic_t suspend_wakeups;
unsigned int poisoned:1;
};
static inline void init_usb_anchor(struct usb_anchor *anchor)
{
memset(anchor, 0, sizeof(*anchor));
INIT_LIST_HEAD(&anchor->urb_list);
init_waitqueue_head(&anchor->wait);
spin_lock_init(&anchor->lock);
}
typedef void (*usb_complete_t)(struct urb *);
/**
* struct urb - USB Request Block
* @urb_list: For use by current owner of the URB.
* @anchor_list: membership in the list of an anchor
* @anchor: to anchor URBs to a common mooring
* @ep: Points to the endpoint's data structure. Will eventually
* replace @pipe.
* @pipe: Holds endpoint number, direction, type, and more.
* Create these values with the eight macros available;
* usb_{snd,rcv}TYPEpipe(dev,endpoint), where the TYPE is "ctrl"
* (control), "bulk", "int" (interrupt), or "iso" (isochronous).
* For example usb_sndbulkpipe() or usb_rcvintpipe(). Endpoint
* numbers range from zero to fifteen. Note that "in" endpoint two
* is a different endpoint (and pipe) from "out" endpoint two.
* The current configuration controls the existence, type, and
* maximum packet size of any given endpoint.
* @stream_id: the endpoint's stream ID for bulk streams
* @dev: Identifies the USB device to perform the request.
* @status: This is read in non-iso completion functions to get the
* status of the particular request. ISO requests only use it
* to tell whether the URB was unlinked; detailed status for
* each frame is in the fields of the iso_frame-desc.
* @transfer_flags: A variety of flags may be used to affect how URB
* submission, unlinking, or operation are handled. Different
* kinds of URB can use different flags.
* @transfer_buffer: This identifies the buffer to (or from) which the I/O
* request will be performed unless URB_NO_TRANSFER_DMA_MAP is set
* (however, do not leave garbage in transfer_buffer even then).
* This buffer must be suitable for DMA; allocate it with
* kmalloc() or equivalent. For transfers to "in" endpoints, contents
* of this buffer will be modified. This buffer is used for the data
* stage of control transfers.
* @transfer_dma: When transfer_flags includes URB_NO_TRANSFER_DMA_MAP,
* the device driver is saying that it provided this DMA address,
* which the host controller driver should use in preference to the
* transfer_buffer.
* @sg: scatter gather buffer list, the buffer size of each element in
* the list (except the last) must be divisible by the endpoint's
* max packet size if no_sg_constraint isn't set in 'struct usb_bus'
* @num_mapped_sgs: (internal) number of mapped sg entries
* @num_sgs: number of entries in the sg list
* @transfer_buffer_length: How big is transfer_buffer. The transfer may
* be broken up into chunks according to the current maximum packet
* size for the endpoint, which is a function of the configuration
* and is encoded in the pipe. When the length is zero, neither
* transfer_buffer nor transfer_dma is used.
* @actual_length: This is read in non-iso completion functions, and
* it tells how many bytes (out of transfer_buffer_length) were
* transferred. It will normally be the same as requested, unless
* either an error was reported or a short read was performed.
* The URB_SHORT_NOT_OK transfer flag may be used to make such
* short reads be reported as errors.
* @setup_packet: Only used for control transfers, this points to eight bytes
* of setup data. Control transfers always start by sending this data
* to the device. Then transfer_buffer is read or written, if needed.
* @setup_dma: DMA pointer for the setup packet. The caller must not use
* this field; setup_packet must point to a valid buffer.
* @start_frame: Returns the initial frame for isochronous transfers.
* @number_of_packets: Lists the number of ISO transfer buffers.
* @interval: Specifies the polling interval for interrupt or isochronous
* transfers. The units are frames (milliseconds) for full and low
* speed devices, and microframes (1/8 millisecond) for highspeed
* and SuperSpeed devices.
* @error_count: Returns the number of ISO transfers that reported errors.
* @context: For use in completion functions. This normally points to
* request-specific driver context.
* @complete: Completion handler. This URB is passed as the parameter to the
* completion function. The completion function may then do what
* it likes with the URB, including resubmitting or freeing it.
* @iso_frame_desc: Used to provide arrays of ISO transfer buffers and to
* collect the transfer status for each buffer.
*
* This structure identifies USB transfer requests. URBs must be allocated by
* calling usb_alloc_urb() and freed with a call to usb_free_urb().
* Initialization may be done using various usb_fill_*_urb() functions. URBs
* are submitted using usb_submit_urb(), and pending requests may be canceled
* using usb_unlink_urb() or usb_kill_urb().
*
* Data Transfer Buffers:
*
* Normally drivers provide I/O buffers allocated with kmalloc() or otherwise
* taken from the general page pool. That is provided by transfer_buffer
* (control requests also use setup_packet), and host controller drivers
* perform a dma mapping (and unmapping) for each buffer transferred. Those
* mapping operations can be expensive on some platforms (perhaps using a dma
* bounce buffer or talking to an IOMMU),
* although they're cheap on commodity x86 and ppc hardware.
*
* Alternatively, drivers may pass the URB_NO_TRANSFER_DMA_MAP transfer flag,
* which tells the host controller driver that no such mapping is needed for
* the transfer_buffer since
* the device driver is DMA-aware. For example, a device driver might
* allocate a DMA buffer with usb_alloc_coherent() or call usb_buffer_map().
* When this transfer flag is provided, host controller drivers will
* attempt to use the dma address found in the transfer_dma
* field rather than determining a dma address themselves.
*
* Note that transfer_buffer must still be set if the controller
* does not support DMA (as indicated by hcd_uses_dma()) and when talking
* to root hub. If you have to transfer between highmem zone and the device
* on such controller, create a bounce buffer or bail out with an error.
* If transfer_buffer cannot be set (is in highmem) and the controller is DMA
* capable, assign NULL to it, so that usbmon knows not to use the value.
* The setup_packet must always be set, so it cannot be located in highmem.
*
* Initialization:
*
* All URBs submitted must initialize the dev, pipe, transfer_flags (may be
* zero), and complete fields. All URBs must also initialize
* transfer_buffer and transfer_buffer_length. They may provide the
* URB_SHORT_NOT_OK transfer flag, indicating that short reads are
* to be treated as errors; that flag is invalid for write requests.
*
* Bulk URBs may
* use the URB_ZERO_PACKET transfer flag, indicating that bulk OUT transfers
* should always terminate with a short packet, even if it means adding an
* extra zero length packet.
*
* Control URBs must provide a valid pointer in the setup_packet field.
* Unlike the transfer_buffer, the setup_packet may not be mapped for DMA
* beforehand.
*
* Interrupt URBs must provide an interval, saying how often (in milliseconds
* or, for highspeed devices, 125 microsecond units)
* to poll for transfers. After the URB has been submitted, the interval
* field reflects how the transfer was actually scheduled.
* The polling interval may be more frequent than requested.
* For example, some controllers have a maximum interval of 32 milliseconds,
* while others support intervals of up to 1024 milliseconds.
* Isochronous URBs also have transfer intervals. (Note that for isochronous
* endpoints, as well as high speed interrupt endpoints, the encoding of
* the transfer interval in the endpoint descriptor is logarithmic.
* Device drivers must convert that value to linear units themselves.)
*
* If an isochronous endpoint queue isn't already running, the host
* controller will schedule a new URB to start as soon as bandwidth
* utilization allows. If the queue is running then a new URB will be
* scheduled to start in the first transfer slot following the end of the
* preceding URB, if that slot has not already expired. If the slot has
* expired (which can happen when IRQ delivery is delayed for a long time),
* the scheduling behavior depends on the URB_ISO_ASAP flag. If the flag
* is clear then the URB will be scheduled to start in the expired slot,
* implying that some of its packets will not be transferred; if the flag
* is set then the URB will be scheduled in the first unexpired slot,
* breaking the queue's synchronization. Upon URB completion, the
* start_frame field will be set to the (micro)frame number in which the
* transfer was scheduled. Ranges for frame counter values are HC-specific
* and can go from as low as 256 to as high as 65536 frames.
*
* Isochronous URBs have a different data transfer model, in part because
* the quality of service is only "best effort". Callers provide specially
* allocated URBs, with number_of_packets worth of iso_frame_desc structures
* at the end. Each such packet is an individual ISO transfer. Isochronous
* URBs are normally queued, submitted by drivers to arrange that
* transfers are at least double buffered, and then explicitly resubmitted
* in completion handlers, so
* that data (such as audio or video) streams at as constant a rate as the
* host controller scheduler can support.
*
* Completion Callbacks:
*
* The completion callback is made in_interrupt(), and one of the first
* things that a completion handler should do is check the status field.
* The status field is provided for all URBs. It is used to report
* unlinked URBs, and status for all non-ISO transfers. It should not
* be examined before the URB is returned to the completion handler.
*
* The context field is normally used to link URBs back to the relevant
* driver or request state.
*
* When the completion callback is invoked for non-isochronous URBs, the
* actual_length field tells how many bytes were transferred. This field
* is updated even when the URB terminated with an error or was unlinked.
*
* ISO transfer status is reported in the status and actual_length fields
* of the iso_frame_desc array, and the number of errors is reported in
* error_count. Completion callbacks for ISO transfers will normally
* (re)submit URBs to ensure a constant transfer rate.
*
* Note that even fields marked "public" should not be touched by the driver
* when the urb is owned by the hcd, that is, since the call to
* usb_submit_urb() till the entry into the completion routine.
*/
struct urb {
/* private: usb core and host controller only fields in the urb */
struct kref kref; /* reference count of the URB */
int unlinked; /* unlink error code */
void *hcpriv; /* private data for host controller */
atomic_t use_count; /* concurrent submissions counter */
atomic_t reject; /* submissions will fail */
/* public: documented fields in the urb that can be used by drivers */
struct list_head urb_list; /* list head for use by the urb's
* current owner */
struct list_head anchor_list; /* the URB may be anchored */
struct usb_anchor *anchor;
struct usb_device *dev; /* (in) pointer to associated device */
struct usb_host_endpoint *ep; /* (internal) pointer to endpoint */
unsigned int pipe; /* (in) pipe information */
unsigned int stream_id; /* (in) stream ID */
int status; /* (return) non-ISO status */
unsigned int transfer_flags; /* (in) URB_SHORT_NOT_OK | ...*/
void *transfer_buffer; /* (in) associated data buffer */
dma_addr_t transfer_dma; /* (in) dma addr for transfer_buffer */
struct scatterlist *sg; /* (in) scatter gather buffer list */
int num_mapped_sgs; /* (internal) mapped sg entries */
int num_sgs; /* (in) number of entries in the sg list */
u32 transfer_buffer_length; /* (in) data buffer length */
u32 actual_length; /* (return) actual transfer length */
unsigned char *setup_packet; /* (in) setup packet (control only) */
dma_addr_t setup_dma; /* (in) dma addr for setup_packet */
int start_frame; /* (modify) start frame (ISO) */
int number_of_packets; /* (in) number of ISO packets */
int interval; /* (modify) transfer interval
* (INT/ISO) */
int error_count; /* (return) number of ISO errors */
void *context; /* (in) context for completion */
usb_complete_t complete; /* (in) completion routine */
struct usb_iso_packet_descriptor iso_frame_desc[];
/* (in) ISO ONLY */
};
/* ----------------------------------------------------------------------- */
/**
* usb_fill_control_urb - initializes a control urb
* @urb: pointer to the urb to initialize.
* @dev: pointer to the struct usb_device for this urb.
* @pipe: the endpoint pipe
* @setup_packet: pointer to the setup_packet buffer. The buffer must be
* suitable for DMA.
* @transfer_buffer: pointer to the transfer buffer. The buffer must be
* suitable for DMA.
* @buffer_length: length of the transfer buffer
* @complete_fn: pointer to the usb_complete_t function
* @context: what to set the urb context to.
*
* Initializes a control urb with the proper information needed to submit
* it to a device.
*
* The transfer buffer and the setup_packet buffer will most likely be filled
* or read via DMA. The simplest way to get a buffer that can be DMAed to is
* allocating it via kmalloc() or equivalent, even for very small buffers.
* If the buffers are embedded in a bigger structure, there is a risk that
* the buffer itself, the previous fields and/or the next fields are corrupted
* due to cache incoherencies; or slowed down if they are evicted from the
* cache. For more information, check &struct urb.
*
*/
static inline void usb_fill_control_urb(struct urb *urb,
struct usb_device *dev,
unsigned int pipe,
unsigned char *setup_packet,
void *transfer_buffer,
int buffer_length,
usb_complete_t complete_fn,
void *context)
{
urb->dev = dev;
urb->pipe = pipe;
urb->setup_packet = setup_packet;
urb->transfer_buffer = transfer_buffer;
urb->transfer_buffer_length = buffer_length;
urb->complete = complete_fn;
urb->context = context;
}
/**
* usb_fill_bulk_urb - macro to help initialize a bulk urb
* @urb: pointer to the urb to initialize.
* @dev: pointer to the struct usb_device for this urb.
* @pipe: the endpoint pipe
* @transfer_buffer: pointer to the transfer buffer. The buffer must be
* suitable for DMA.
* @buffer_length: length of the transfer buffer
* @complete_fn: pointer to the usb_complete_t function
* @context: what to set the urb context to.
*
* Initializes a bulk urb with the proper information needed to submit it
* to a device.
*
* Refer to usb_fill_control_urb() for a description of the requirements for
* transfer_buffer.
*/
static inline void usb_fill_bulk_urb(struct urb *urb,
struct usb_device *dev,
unsigned int pipe,
void *transfer_buffer,
int buffer_length,
usb_complete_t complete_fn,
void *context)
{
urb->dev = dev;
urb->pipe = pipe;
urb->transfer_buffer = transfer_buffer;
urb->transfer_buffer_length = buffer_length;
urb->complete = complete_fn;
urb->context = context;
}
/**
* usb_fill_int_urb - macro to help initialize a interrupt urb
* @urb: pointer to the urb to initialize.
* @dev: pointer to the struct usb_device for this urb.
* @pipe: the endpoint pipe
* @transfer_buffer: pointer to the transfer buffer. The buffer must be
* suitable for DMA.
* @buffer_length: length of the transfer buffer
* @complete_fn: pointer to the usb_complete_t function
* @context: what to set the urb context to.
* @interval: what to set the urb interval to, encoded like
* the endpoint descriptor's bInterval value.
*
* Initializes a interrupt urb with the proper information needed to submit
* it to a device.
*
* Refer to usb_fill_control_urb() for a description of the requirements for
* transfer_buffer.
*
* Note that High Speed and SuperSpeed(+) interrupt endpoints use a logarithmic
* encoding of the endpoint interval, and express polling intervals in
* microframes (eight per millisecond) rather than in frames (one per
* millisecond).
*/
static inline void usb_fill_int_urb(struct urb *urb,
struct usb_device *dev,
unsigned int pipe,
void *transfer_buffer,
int buffer_length,
usb_complete_t complete_fn,
void *context,
int interval)
{
urb->dev = dev;
urb->pipe = pipe;
urb->transfer_buffer = transfer_buffer;
urb->transfer_buffer_length = buffer_length;
urb->complete = complete_fn;
urb->context = context;
if (dev->speed == USB_SPEED_HIGH || dev->speed >= USB_SPEED_SUPER) {
/* make sure interval is within allowed range */
interval = clamp(interval, 1, 16); urb->interval = 1 << (interval - 1);
} else {
urb->interval = interval;
}
urb->start_frame = -1;
}
extern void usb_init_urb(struct urb *urb);
extern struct urb *usb_alloc_urb(int iso_packets, gfp_t mem_flags);
extern void usb_free_urb(struct urb *urb);
#define usb_put_urb usb_free_urb
extern struct urb *usb_get_urb(struct urb *urb);
extern int usb_submit_urb(struct urb *urb, gfp_t mem_flags);
extern int usb_unlink_urb(struct urb *urb);
extern void usb_kill_urb(struct urb *urb);
extern void usb_poison_urb(struct urb *urb);
extern void usb_unpoison_urb(struct urb *urb);
extern void usb_block_urb(struct urb *urb);
extern void usb_kill_anchored_urbs(struct usb_anchor *anchor);
extern void usb_poison_anchored_urbs(struct usb_anchor *anchor);
extern void usb_unpoison_anchored_urbs(struct usb_anchor *anchor);
extern void usb_unlink_anchored_urbs(struct usb_anchor *anchor);
extern void usb_anchor_suspend_wakeups(struct usb_anchor *anchor);
extern void usb_anchor_resume_wakeups(struct usb_anchor *anchor);
extern void usb_anchor_urb(struct urb *urb, struct usb_anchor *anchor);
extern void usb_unanchor_urb(struct urb *urb);
extern int usb_wait_anchor_empty_timeout(struct usb_anchor *anchor,
unsigned int timeout);
extern struct urb *usb_get_from_anchor(struct usb_anchor *anchor);
extern void usb_scuttle_anchored_urbs(struct usb_anchor *anchor);
extern int usb_anchor_empty(struct usb_anchor *anchor);
#define usb_unblock_urb usb_unpoison_urb
/**
* usb_urb_dir_in - check if an URB describes an IN transfer
* @urb: URB to be checked
*
* Return: 1 if @urb describes an IN transfer (device-to-host),
* otherwise 0.
*/
static inline int usb_urb_dir_in(struct urb *urb)
{
return (urb->transfer_flags & URB_DIR_MASK) == URB_DIR_IN;
}
/**
* usb_urb_dir_out - check if an URB describes an OUT transfer
* @urb: URB to be checked
*
* Return: 1 if @urb describes an OUT transfer (host-to-device),
* otherwise 0.
*/
static inline int usb_urb_dir_out(struct urb *urb)
{
return (urb->transfer_flags & URB_DIR_MASK) == URB_DIR_OUT;
}
int usb_pipe_type_check(struct usb_device *dev, unsigned int pipe);
int usb_urb_ep_type_check(const struct urb *urb);
void *usb_alloc_coherent(struct usb_device *dev, size_t size,
gfp_t mem_flags, dma_addr_t *dma);
void usb_free_coherent(struct usb_device *dev, size_t size,
void *addr, dma_addr_t dma);
/*-------------------------------------------------------------------*
* SYNCHRONOUS CALL SUPPORT *
*-------------------------------------------------------------------*/
extern int usb_control_msg(struct usb_device *dev, unsigned int pipe,
__u8 request, __u8 requesttype, __u16 value, __u16 index,
void *data, __u16 size, int timeout);
extern int usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length, int timeout);
extern int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length,
int timeout);
/* wrappers around usb_control_msg() for the most common standard requests */
int usb_control_msg_send(struct usb_device *dev, __u8 endpoint, __u8 request,
__u8 requesttype, __u16 value, __u16 index,
const void *data, __u16 size, int timeout,
gfp_t memflags);
int usb_control_msg_recv(struct usb_device *dev, __u8 endpoint, __u8 request,
__u8 requesttype, __u16 value, __u16 index,
void *data, __u16 size, int timeout,
gfp_t memflags);
extern int usb_get_descriptor(struct usb_device *dev, unsigned char desctype,
unsigned char descindex, void *buf, int size);
extern int usb_get_status(struct usb_device *dev,
int recip, int type, int target, void *data);
static inline int usb_get_std_status(struct usb_device *dev,
int recip, int target, void *data)
{
return usb_get_status(dev, recip, USB_STATUS_TYPE_STANDARD, target,
data);
}
static inline int usb_get_ptm_status(struct usb_device *dev, void *data)
{
return usb_get_status(dev, USB_RECIP_DEVICE, USB_STATUS_TYPE_PTM,
0, data);
}
extern int usb_string(struct usb_device *dev, int index,
char *buf, size_t size);
extern char *usb_cache_string(struct usb_device *udev, int index);
/* wrappers that also update important state inside usbcore */
extern int usb_clear_halt(struct usb_device *dev, int pipe);
extern int usb_reset_configuration(struct usb_device *dev);
extern int usb_set_interface(struct usb_device *dev, int ifnum, int alternate);
extern void usb_reset_endpoint(struct usb_device *dev, unsigned int epaddr);
/* this request isn't really synchronous, but it belongs with the others */
extern int usb_driver_set_configuration(struct usb_device *udev, int config);
/* choose and set configuration for device */
extern int usb_choose_configuration(struct usb_device *udev);
extern int usb_set_configuration(struct usb_device *dev, int configuration);
/*
* timeouts, in milliseconds, used for sending/receiving control messages
* they typically complete within a few frames (msec) after they're issued
* USB identifies 5 second timeouts, maybe more in a few cases, and a few
* slow devices (like some MGE Ellipse UPSes) actually push that limit.
*/
#define USB_CTRL_GET_TIMEOUT 5000
#define USB_CTRL_SET_TIMEOUT 5000
/**
* struct usb_sg_request - support for scatter/gather I/O
* @status: zero indicates success, else negative errno
* @bytes: counts bytes transferred.
*
* These requests are initialized using usb_sg_init(), and then are used
* as request handles passed to usb_sg_wait() or usb_sg_cancel(). Most
* members of the request object aren't for driver access.
*
* The status and bytecount values are valid only after usb_sg_wait()
* returns. If the status is zero, then the bytecount matches the total
* from the request.
*
* After an error completion, drivers may need to clear a halt condition
* on the endpoint.
*/
struct usb_sg_request {
int status;
size_t bytes;
/* private:
* members below are private to usbcore,
* and are not provided for driver access!
*/
spinlock_t lock;
struct usb_device *dev;
int pipe;
int entries;
struct urb **urbs;
int count;
struct completion complete;
};
int usb_sg_init(
struct usb_sg_request *io,
struct usb_device *dev,
unsigned pipe,
unsigned period,
struct scatterlist *sg,
int nents,
size_t length,
gfp_t mem_flags
);
void usb_sg_cancel(struct usb_sg_request *io);
void usb_sg_wait(struct usb_sg_request *io);
/* ----------------------------------------------------------------------- */
/*
* For various legacy reasons, Linux has a small cookie that's paired with
* a struct usb_device to identify an endpoint queue. Queue characteristics
* are defined by the endpoint's descriptor. This cookie is called a "pipe",
* an unsigned int encoded as:
*
* - direction: bit 7 (0 = Host-to-Device [Out],
* 1 = Device-to-Host [In] ...
* like endpoint bEndpointAddress)
* - device address: bits 8-14 ... bit positions known to uhci-hcd
* - endpoint: bits 15-18 ... bit positions known to uhci-hcd
* - pipe type: bits 30-31 (00 = isochronous, 01 = interrupt,
* 10 = control, 11 = bulk)
*
* Given the device address and endpoint descriptor, pipes are redundant.
*/
/* NOTE: these are not the standard USB_ENDPOINT_XFER_* values!! */
/* (yet ... they're the values used by usbfs) */
#define PIPE_ISOCHRONOUS 0
#define PIPE_INTERRUPT 1
#define PIPE_CONTROL 2
#define PIPE_BULK 3
#define usb_pipein(pipe) ((pipe) & USB_DIR_IN)
#define usb_pipeout(pipe) (!usb_pipein(pipe))
#define usb_pipedevice(pipe) (((pipe) >> 8) & 0x7f)
#define usb_pipeendpoint(pipe) (((pipe) >> 15) & 0xf)
#define usb_pipetype(pipe) (((pipe) >> 30) & 3)
#define usb_pipeisoc(pipe) (usb_pipetype((pipe)) == PIPE_ISOCHRONOUS)
#define usb_pipeint(pipe) (usb_pipetype((pipe)) == PIPE_INTERRUPT)
#define usb_pipecontrol(pipe) (usb_pipetype((pipe)) == PIPE_CONTROL)
#define usb_pipebulk(pipe) (usb_pipetype((pipe)) == PIPE_BULK)
static inline unsigned int __create_pipe(struct usb_device *dev,
unsigned int endpoint)
{
return (dev->devnum << 8) | (endpoint << 15);
}
/* Create various pipes... */
#define usb_sndctrlpipe(dev, endpoint) \
((PIPE_CONTROL << 30) | __create_pipe(dev, endpoint))
#define usb_rcvctrlpipe(dev, endpoint) \
((PIPE_CONTROL << 30) | __create_pipe(dev, endpoint) | USB_DIR_IN)
#define usb_sndisocpipe(dev, endpoint) \
((PIPE_ISOCHRONOUS << 30) | __create_pipe(dev, endpoint))
#define usb_rcvisocpipe(dev, endpoint) \
((PIPE_ISOCHRONOUS << 30) | __create_pipe(dev, endpoint) | USB_DIR_IN)
#define usb_sndbulkpipe(dev, endpoint) \
((PIPE_BULK << 30) | __create_pipe(dev, endpoint))
#define usb_rcvbulkpipe(dev, endpoint) \
((PIPE_BULK << 30) | __create_pipe(dev, endpoint) | USB_DIR_IN)
#define usb_sndintpipe(dev, endpoint) \
((PIPE_INTERRUPT << 30) | __create_pipe(dev, endpoint))
#define usb_rcvintpipe(dev, endpoint) \
((PIPE_INTERRUPT << 30) | __create_pipe(dev, endpoint) | USB_DIR_IN)
static inline struct usb_host_endpoint *
usb_pipe_endpoint(struct usb_device *dev, unsigned int pipe)
{
struct usb_host_endpoint **eps;
eps = usb_pipein(pipe) ? dev->ep_in : dev->ep_out; return eps[usb_pipeendpoint(pipe)];
}
static inline u16 usb_maxpacket(struct usb_device *udev, int pipe)
{
struct usb_host_endpoint *ep = usb_pipe_endpoint(udev, pipe);
if (!ep)
return 0;
/* NOTE: only 0x07ff bits are for packet size... */
return usb_endpoint_maxp(&ep->desc);
}
/* translate USB error codes to codes user space understands */
static inline int usb_translate_errors(int error_code)
{
switch (error_code) {
case 0:
case -ENOMEM:
case -ENODEV:
case -EOPNOTSUPP:
return error_code;
default:
return -EIO;
}
}
/* Events from the usb core */
#define USB_DEVICE_ADD 0x0001
#define USB_DEVICE_REMOVE 0x0002
#define USB_BUS_ADD 0x0003
#define USB_BUS_REMOVE 0x0004
extern void usb_register_notify(struct notifier_block *nb);
extern void usb_unregister_notify(struct notifier_block *nb);
/* debugfs stuff */
extern struct dentry *usb_debug_root;
/* LED triggers */
enum usb_led_event {
USB_LED_EVENT_HOST = 0,
USB_LED_EVENT_GADGET = 1,
};
#ifdef CONFIG_USB_LED_TRIG
extern void usb_led_activity(enum usb_led_event ev);
#else
static inline void usb_led_activity(enum usb_led_event ev) {}
#endif
#endif /* __KERNEL__ */
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* Link physical devices with ACPI devices support
*
* Copyright (c) 2005 David Shaohua Li <shaohua.li@intel.com>
* Copyright (c) 2005 Intel Corp.
*/
#define pr_fmt(fmt) "ACPI: " fmt
#include <linux/acpi_iort.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/rwsem.h>
#include <linux/acpi.h>
#include <linux/dma-mapping.h>
#include <linux/pci.h>
#include <linux/pci-acpi.h>
#include <linux/platform_device.h>
#include "internal.h"
static LIST_HEAD(bus_type_list);
static DECLARE_RWSEM(bus_type_sem);
#define PHYSICAL_NODE_STRING "physical_node"
#define PHYSICAL_NODE_NAME_SIZE (sizeof(PHYSICAL_NODE_STRING) + 10)
int register_acpi_bus_type(struct acpi_bus_type *type)
{
if (acpi_disabled)
return -ENODEV;
if (type && type->match && type->find_companion) {
down_write(&bus_type_sem);
list_add_tail(&type->list, &bus_type_list);
up_write(&bus_type_sem);
pr_info("bus type %s registered\n", type->name);
return 0;
}
return -ENODEV;
}
EXPORT_SYMBOL_GPL(register_acpi_bus_type);
int unregister_acpi_bus_type(struct acpi_bus_type *type)
{
if (acpi_disabled)
return 0;
if (type) {
down_write(&bus_type_sem);
list_del_init(&type->list);
up_write(&bus_type_sem);
pr_info("bus type %s unregistered\n", type->name);
return 0;
}
return -ENODEV;
}
EXPORT_SYMBOL_GPL(unregister_acpi_bus_type);
static struct acpi_bus_type *acpi_get_bus_type(struct device *dev)
{
struct acpi_bus_type *tmp, *ret = NULL;
down_read(&bus_type_sem); list_for_each_entry(tmp, &bus_type_list, list) { if (tmp->match(dev)) {
ret = tmp;
break;
}
}
up_read(&bus_type_sem);
return ret;
}
#define FIND_CHILD_MIN_SCORE 1
#define FIND_CHILD_MID_SCORE 2
#define FIND_CHILD_MAX_SCORE 3
static int match_any(struct acpi_device *adev, void *not_used)
{
return 1;
}
static bool acpi_dev_has_children(struct acpi_device *adev)
{
return acpi_dev_for_each_child(adev, match_any, NULL) > 0;
}
static int find_child_checks(struct acpi_device *adev, bool check_children)
{
unsigned long long sta;
acpi_status status;
if (check_children && !acpi_dev_has_children(adev))
return -ENODEV;
status = acpi_evaluate_integer(adev->handle, "_STA", NULL, &sta);
if (status == AE_NOT_FOUND) {
/*
* Special case: backlight device objects without _STA are
* preferred to other objects with the same _ADR value, because
* it is more likely that they are actually useful.
*/
if (adev->pnp.type.backlight)
return FIND_CHILD_MID_SCORE;
return FIND_CHILD_MIN_SCORE;
}
if (ACPI_FAILURE(status) || !(sta & ACPI_STA_DEVICE_ENABLED))
return -ENODEV;
/*
* If the device has a _HID returning a valid ACPI/PNP device ID, it is
* better to make it look less attractive here, so that the other device
* with the same _ADR value (that may not have a valid device ID) can be
* matched going forward. [This means a second spec violation in a row,
* so whatever we do here is best effort anyway.]
*/
if (adev->pnp.type.platform_id)
return FIND_CHILD_MIN_SCORE;
return FIND_CHILD_MAX_SCORE;
}
struct find_child_walk_data {
struct acpi_device *adev;
u64 address;
int score;
bool check_sta;
bool check_children;
};
static int check_one_child(struct acpi_device *adev, void *data)
{
struct find_child_walk_data *wd = data;
int score;
if (!adev->pnp.type.bus_address || acpi_device_adr(adev) != wd->address)
return 0;
if (!wd->adev) {
/*
* This is the first matching object, so save it. If it is not
* necessary to look for any other matching objects, stop the
* search.
*/
wd->adev = adev;
return !(wd->check_sta || wd->check_children);
}
/*
* There is more than one matching device object with the same _ADR
* value. That really is unexpected, so we are kind of beyond the scope
* of the spec here. We have to choose which one to return, though.
*
* First, get the score for the previously found object and terminate
* the walk if it is maximum.
*/
if (!wd->score) {
score = find_child_checks(wd->adev, wd->check_children);
if (score == FIND_CHILD_MAX_SCORE)
return 1;
wd->score = score;
}
/*
* Second, if the object that has just been found has a better score,
* replace the previously found one with it and terminate the walk if
* the new score is maximum.
*/
score = find_child_checks(adev, wd->check_children);
if (score > wd->score) {
wd->adev = adev;
if (score == FIND_CHILD_MAX_SCORE)
return 1;
wd->score = score;
}
/* Continue, because there may be better matches. */
return 0;
}
static struct acpi_device *acpi_find_child(struct acpi_device *parent,
u64 address, bool check_children,
bool check_sta)
{
struct find_child_walk_data wd = {
.address = address,
.check_children = check_children,
.check_sta = check_sta,
.adev = NULL,
.score = 0,
};
if (parent)
acpi_dev_for_each_child(parent, check_one_child, &wd); return wd.adev;
}
struct acpi_device *acpi_find_child_device(struct acpi_device *parent,
u64 address, bool check_children)
{
return acpi_find_child(parent, address, check_children, true);
}
EXPORT_SYMBOL_GPL(acpi_find_child_device);
struct acpi_device *acpi_find_child_by_adr(struct acpi_device *adev,
acpi_bus_address adr)
{
return acpi_find_child(adev, adr, false, false);
}
EXPORT_SYMBOL_GPL(acpi_find_child_by_adr);
static void acpi_physnode_link_name(char *buf, unsigned int node_id)
{
if (node_id > 0)
snprintf(buf, PHYSICAL_NODE_NAME_SIZE,
PHYSICAL_NODE_STRING "%u", node_id);
else
strcpy(buf, PHYSICAL_NODE_STRING);
}
int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
{
struct acpi_device_physical_node *physical_node, *pn;
char physical_node_name[PHYSICAL_NODE_NAME_SIZE];
struct list_head *physnode_list;
unsigned int node_id;
int retval = -EINVAL;
if (has_acpi_companion(dev)) {
if (acpi_dev) { dev_warn(dev, "ACPI companion already set\n");
return -EINVAL;
} else {
acpi_dev = ACPI_COMPANION(dev);
}
}
if (!acpi_dev)
return -EINVAL;
acpi_dev_get(acpi_dev);
get_device(dev);
physical_node = kzalloc(sizeof(*physical_node), GFP_KERNEL);
if (!physical_node) {
retval = -ENOMEM;
goto err;
}
mutex_lock(&acpi_dev->physical_node_lock);
/*
* Keep the list sorted by node_id so that the IDs of removed nodes can
* be recycled easily.
*/
physnode_list = &acpi_dev->physical_node_list;
node_id = 0;
list_for_each_entry(pn, &acpi_dev->physical_node_list, node) {
/* Sanity check. */
if (pn->dev == dev) { mutex_unlock(&acpi_dev->physical_node_lock);
dev_warn(dev, "Already associated with ACPI node\n");
kfree(physical_node);
if (ACPI_COMPANION(dev) != acpi_dev)
goto err;
put_device(dev);
acpi_dev_put(acpi_dev);
return 0;
}
if (pn->node_id == node_id) {
physnode_list = &pn->node;
node_id++;
}
}
physical_node->node_id = node_id;
physical_node->dev = dev;
list_add(&physical_node->node, physnode_list); acpi_dev->physical_node_count++;
if (!has_acpi_companion(dev))
ACPI_COMPANION_SET(dev, acpi_dev); acpi_physnode_link_name(physical_node_name, node_id);
retval = sysfs_create_link(&acpi_dev->dev.kobj, &dev->kobj,
physical_node_name);
if (retval)
dev_err(&acpi_dev->dev, "Failed to create link %s (%d)\n",
physical_node_name, retval);
retval = sysfs_create_link(&dev->kobj, &acpi_dev->dev.kobj,
"firmware_node");
if (retval)
dev_err(dev, "Failed to create link firmware_node (%d)\n",
retval);
mutex_unlock(&acpi_dev->physical_node_lock);
if (acpi_dev->wakeup.flags.valid)
device_set_wakeup_capable(dev, true);
return 0;
err:
ACPI_COMPANION_SET(dev, NULL);
put_device(dev);
acpi_dev_put(acpi_dev);
return retval;
}
EXPORT_SYMBOL_GPL(acpi_bind_one);
int acpi_unbind_one(struct device *dev)
{
struct acpi_device *acpi_dev = ACPI_COMPANION(dev);
struct acpi_device_physical_node *entry;
if (!acpi_dev)
return 0;
mutex_lock(&acpi_dev->physical_node_lock);
list_for_each_entry(entry, &acpi_dev->physical_node_list, node)
if (entry->dev == dev) {
char physnode_name[PHYSICAL_NODE_NAME_SIZE];
list_del(&entry->node);
acpi_dev->physical_node_count--;
acpi_physnode_link_name(physnode_name, entry->node_id);
sysfs_remove_link(&acpi_dev->dev.kobj, physnode_name);
sysfs_remove_link(&dev->kobj, "firmware_node");
ACPI_COMPANION_SET(dev, NULL);
/* Drop references taken by acpi_bind_one(). */
put_device(dev);
acpi_dev_put(acpi_dev);
kfree(entry);
break;
}
mutex_unlock(&acpi_dev->physical_node_lock);
return 0;
}
EXPORT_SYMBOL_GPL(acpi_unbind_one);
void acpi_device_notify(struct device *dev)
{
struct acpi_device *adev;
int ret;
ret = acpi_bind_one(dev, NULL);
if (ret) {
struct acpi_bus_type *type = acpi_get_bus_type(dev);
if (!type)
goto err;
adev = type->find_companion(dev);
if (!adev) {
dev_dbg(dev, "ACPI companion not found\n");
goto err;
}
ret = acpi_bind_one(dev, adev);
if (ret)
goto err;
if (type->setup) {
type->setup(dev);
goto done;
}
} else {
adev = ACPI_COMPANION(dev); if (dev_is_pci(dev)) { pci_acpi_setup(dev, adev);
goto done;
} else if (dev_is_platform(dev)) {
acpi_configure_pmsi_domain(dev);
}
}
if (adev->handler && adev->handler->bind) adev->handler->bind(dev);
done:
acpi_handle_debug(ACPI_HANDLE(dev), "Bound to device %s\n",
dev_name(dev));
return;
err:
dev_dbg(dev, "No ACPI support\n");
}
void acpi_device_notify_remove(struct device *dev)
{
struct acpi_device *adev = ACPI_COMPANION(dev);
if (!adev)
return;
if (dev_is_pci(dev)) pci_acpi_cleanup(dev, adev); else if (adev->handler && adev->handler->unbind) adev->handler->unbind(dev); acpi_unbind_one(dev);
}
// SPDX-License-Identifier: GPL-2.0
/*
* FPU signal frame handling routines.
*/
#include <linux/compat.h>
#include <linux/cpu.h>
#include <linux/pagemap.h>
#include <asm/fpu/signal.h>
#include <asm/fpu/regset.h>
#include <asm/fpu/xstate.h>
#include <asm/sigframe.h>
#include <asm/trapnr.h>
#include <asm/trace/fpu.h>
#include "context.h"
#include "internal.h"
#include "legacy.h"
#include "xstate.h"
/*
* Check for the presence of extended state information in the
* user fpstate pointer in the sigcontext.
*/
static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
struct _fpx_sw_bytes *fx_sw)
{
int min_xstate_size = sizeof(struct fxregs_state) +
sizeof(struct xstate_header);
void __user *fpstate = fxbuf;
unsigned int magic2;
if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
return false;
/* Check for the first magic field and other error scenarios. */
if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
fx_sw->xstate_size < min_xstate_size ||
fx_sw->xstate_size > current->thread.fpu.fpstate->user_size ||
fx_sw->xstate_size > fx_sw->extended_size)
goto setfx;
/*
* Check for the presence of second magic word at the end of memory
* layout. This detects the case where the user just copied the legacy
* fpstate layout with out copying the extended state information
* in the memory layout.
*/
if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
return false;
if (likely(magic2 == FP_XSTATE_MAGIC2))
return true;
setfx:
trace_x86_fpu_xstate_check_failed(¤t->thread.fpu);
/* Set the parameters for fx only state */
fx_sw->magic1 = 0;
fx_sw->xstate_size = sizeof(struct fxregs_state);
fx_sw->xfeatures = XFEATURE_MASK_FPSSE;
return true;
}
/*
* Signal frame handlers.
*/
static inline bool save_fsave_header(struct task_struct *tsk, void __user *buf)
{
if (use_fxsr()) {
struct xregs_state *xsave = &tsk->thread.fpu.fpstate->regs.xsave;
struct user_i387_ia32_struct env;
struct _fpstate_32 __user *fp = buf;
fpregs_lock();
if (!test_thread_flag(TIF_NEED_FPU_LOAD))
fxsave(&tsk->thread.fpu.fpstate->regs.fxsave);
fpregs_unlock();
convert_from_fxsr(&env, tsk);
if (__copy_to_user(buf, &env, sizeof(env)) ||
__put_user(xsave->i387.swd, &fp->status) ||
__put_user(X86_FXSR_MAGIC, &fp->magic))
return false;
} else {
struct fregs_state __user *fp = buf;
u32 swd;
if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
return false;
}
return true;
}
/*
* Prepare the SW reserved portion of the fxsave memory layout, indicating
* the presence of the extended state information in the memory layout
* pointed to by the fpstate pointer in the sigcontext.
* This is saved when ever the FP and extended state context is
* saved on the user stack during the signal handler delivery to the user.
*/
static inline void save_sw_bytes(struct _fpx_sw_bytes *sw_bytes, bool ia32_frame,
struct fpstate *fpstate)
{
sw_bytes->magic1 = FP_XSTATE_MAGIC1;
sw_bytes->extended_size = fpstate->user_size + FP_XSTATE_MAGIC2_SIZE;
sw_bytes->xfeatures = fpstate->user_xfeatures;
sw_bytes->xstate_size = fpstate->user_size;
if (ia32_frame)
sw_bytes->extended_size += sizeof(struct fregs_state);
}
static inline bool save_xstate_epilog(void __user *buf, int ia32_frame,
struct fpstate *fpstate)
{
struct xregs_state __user *x = buf;
struct _fpx_sw_bytes sw_bytes = {};
u32 xfeatures;
int err;
/* Setup the bytes not touched by the [f]xsave and reserved for SW. */
save_sw_bytes(&sw_bytes, ia32_frame, fpstate);
err = __copy_to_user(&x->i387.sw_reserved, &sw_bytes, sizeof(sw_bytes)); if (!use_xsave()) return !err; err |= __put_user(FP_XSTATE_MAGIC2,
(__u32 __user *)(buf + fpstate->user_size));
/*
* Read the xfeatures which we copied (directly from the cpu or
* from the state in task struct) to the user buffers.
*/
err |= __get_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
/*
* For legacy compatible, we always set FP/SSE bits in the bit
* vector while saving the state to the user context. This will
* enable us capturing any changes(during sigreturn) to
* the FP/SSE bits by the legacy applications which don't touch
* xfeatures in the xsave header.
*
* xsave aware apps can change the xfeatures in the xsave
* header as well as change any contents in the memory layout.
* xrestore as part of sigreturn will capture all the changes.
*/
xfeatures |= XFEATURE_MASK_FPSSE;
err |= __put_user(xfeatures, (__u32 __user *)&x->header.xfeatures);
return !err;
}
static inline int copy_fpregs_to_sigframe(struct xregs_state __user *buf)
{
if (use_xsave())
return xsave_to_user_sigframe(buf);
if (use_fxsr())
return fxsave_to_user_sigframe((struct fxregs_state __user *) buf);
else
return fnsave_to_user_sigframe((struct fregs_state __user *) buf);
}
/*
* Save the fpu, extended register state to the user signal frame.
*
* 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
* state is copied.
* 'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
*
* buf == buf_fx for 64-bit frames and 32-bit fsave frame.
* buf != buf_fx for 32-bit frames with fxstate.
*
* Save it directly to the user frame with disabled page fault handler. If
* that faults, try to clear the frame which handles the page fault.
*
* If this is a 32-bit frame with fxstate, put a fsave header before
* the aligned state at 'buf_fx'.
*
* For [f]xsave state, update the SW reserved fields in the [f]xsave frame
* indicating the absence/presence of the extended state to the user.
*/
bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
{
struct task_struct *tsk = current;
struct fpstate *fpstate = tsk->thread.fpu.fpstate;
bool ia32_fxstate = (buf != buf_fx);
int ret;
ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
IS_ENABLED(CONFIG_IA32_EMULATION));
if (!static_cpu_has(X86_FEATURE_FPU)) {
struct user_i387_ia32_struct fp;
fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
.left = sizeof(fp)});
return !copy_to_user(buf, &fp, sizeof(fp));
}
if (!access_ok(buf, size)) return false; if (use_xsave()) {
struct xregs_state __user *xbuf = buf_fx;
/*
* Clear the xsave header first, so that reserved fields are
* initialized to zero.
*/
if (__clear_user(&xbuf->header, sizeof(xbuf->header)))
return false;
}
retry:
/*
* Load the FPU registers if they are not valid for the current task.
* With a valid FPU state we can attempt to save the state directly to
* userland's stack frame which will likely succeed. If it does not,
* resolve the fault in the user memory and try again.
*/
fpregs_lock();
if (test_thread_flag(TIF_NEED_FPU_LOAD))
fpregs_restore_userregs(); pagefault_disable(); ret = copy_fpregs_to_sigframe(buf_fx); pagefault_enable();
fpregs_unlock();
if (ret) {
if (!__clear_user(buf_fx, fpstate->user_size))
goto retry;
return false;
}
/* Save the fsave header for the 32-bit frames. */
if ((ia32_fxstate || !use_fxsr()) && !save_fsave_header(tsk, buf))
return false;
if (use_fxsr() && !save_xstate_epilog(buf_fx, ia32_fxstate, fpstate))
return false;
return true;
}
static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures,
u64 xrestore, bool fx_only)
{
if (use_xsave()) {
u64 init_bv = ufeatures & ~xrestore;
int ret;
if (likely(!fx_only))
ret = xrstor_from_user_sigframe(buf, xrestore);
else
ret = fxrstor_from_user_sigframe(buf);
if (!ret && unlikely(init_bv))
os_xrstor(&init_fpstate, init_bv);
return ret;
} else if (use_fxsr()) {
return fxrstor_from_user_sigframe(buf);
} else {
return frstor_from_user_sigframe(buf);
}
}
/*
* Attempt to restore the FPU registers directly from user memory.
* Pagefaults are handled and any errors returned are fatal.
*/
static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only)
{
struct fpu *fpu = ¤t->thread.fpu;
int ret;
/* Restore enabled features only. */
xrestore &= fpu->fpstate->user_xfeatures;
retry:
fpregs_lock();
/* Ensure that XFD is up to date */
xfd_update_state(fpu->fpstate);
pagefault_disable();
ret = __restore_fpregs_from_user(buf, fpu->fpstate->user_xfeatures,
xrestore, fx_only);
pagefault_enable();
if (unlikely(ret)) {
/*
* The above did an FPU restore operation, restricted to
* the user portion of the registers, and failed, but the
* microcode might have modified the FPU registers
* nevertheless.
*
* If the FPU registers do not belong to current, then
* invalidate the FPU register state otherwise the task
* might preempt current and return to user space with
* corrupted FPU registers.
*/
if (test_thread_flag(TIF_NEED_FPU_LOAD))
__cpu_invalidate_fpregs_state();
fpregs_unlock();
/* Try to handle #PF, but anything else is fatal. */
if (ret != X86_TRAP_PF)
return false;
if (!fault_in_readable(buf, fpu->fpstate->user_size))
goto retry;
return false;
}
/*
* Restore supervisor states: previous context switch etc has done
* XSAVES and saved the supervisor states in the kernel buffer from
* which they can be restored now.
*
* It would be optimal to handle this with a single XRSTORS, but
* this does not work because the rest of the FPU registers have
* been restored from a user buffer directly.
*/
if (test_thread_flag(TIF_NEED_FPU_LOAD) && xfeatures_mask_supervisor())
os_xrstor_supervisor(fpu->fpstate);
fpregs_mark_activate();
fpregs_unlock();
return true;
}
static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx,
bool ia32_fxstate)
{
struct task_struct *tsk = current;
struct fpu *fpu = &tsk->thread.fpu;
struct user_i387_ia32_struct env;
bool success, fx_only = false;
union fpregs_state *fpregs;
u64 user_xfeatures = 0;
if (use_xsave()) {
struct _fpx_sw_bytes fx_sw_user;
if (!check_xstate_in_sigframe(buf_fx, &fx_sw_user))
return false;
fx_only = !fx_sw_user.magic1;
user_xfeatures = fx_sw_user.xfeatures;
} else {
user_xfeatures = XFEATURE_MASK_FPSSE;
}
if (likely(!ia32_fxstate)) {
/* Restore the FPU registers directly from user memory. */
return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only);
}
/*
* Copy the legacy state because the FP portion of the FX frame has
* to be ignored for histerical raisins. The legacy state is folded
* in once the larger state has been copied.
*/
if (__copy_from_user(&env, buf, sizeof(env)))
return false;
/*
* By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
* not modified on context switch and that the xstate is considered
* to be loaded again on return to userland (overriding last_cpu avoids
* the optimisation).
*/
fpregs_lock();
if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
/*
* If supervisor states are available then save the
* hardware state in current's fpstate so that the
* supervisor state is preserved. Save the full state for
* simplicity. There is no point in optimizing this by only
* saving the supervisor states and then shuffle them to
* the right place in memory. It's ia32 mode. Shrug.
*/
if (xfeatures_mask_supervisor())
os_xsave(fpu->fpstate);
set_thread_flag(TIF_NEED_FPU_LOAD);
}
__fpu_invalidate_fpregs_state(fpu);
__cpu_invalidate_fpregs_state();
fpregs_unlock();
fpregs = &fpu->fpstate->regs;
if (use_xsave() && !fx_only) {
if (copy_sigframe_from_user_to_xstate(tsk, buf_fx))
return false;
} else {
if (__copy_from_user(&fpregs->fxsave, buf_fx,
sizeof(fpregs->fxsave)))
return false;
if (IS_ENABLED(CONFIG_X86_64)) {
/* Reject invalid MXCSR values. */
if (fpregs->fxsave.mxcsr & ~mxcsr_feature_mask)
return false;
} else {
/* Mask invalid bits out for historical reasons (broken hardware). */
fpregs->fxsave.mxcsr &= mxcsr_feature_mask;
}
/* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
if (use_xsave())
fpregs->xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
}
/* Fold the legacy FP storage */
convert_to_fxsr(&fpregs->fxsave, &env);
fpregs_lock();
if (use_xsave()) {
/*
* Remove all UABI feature bits not set in user_xfeatures
* from the memory xstate header which makes the full
* restore below bring them into init state. This works for
* fx_only mode as well because that has only FP and SSE
* set in user_xfeatures.
*
* Preserve supervisor states!
*/
u64 mask = user_xfeatures | xfeatures_mask_supervisor();
fpregs->xsave.header.xfeatures &= mask;
success = !os_xrstor_safe(fpu->fpstate,
fpu_kernel_cfg.max_features);
} else {
success = !fxrstor_safe(&fpregs->fxsave);
}
if (likely(success))
fpregs_mark_activate();
fpregs_unlock();
return success;
}
static inline unsigned int xstate_sigframe_size(struct fpstate *fpstate)
{
unsigned int size = fpstate->user_size;
return use_xsave() ? size + FP_XSTATE_MAGIC2_SIZE : size;
}
/*
* Restore FPU state from a sigframe:
*/
bool fpu__restore_sig(void __user *buf, int ia32_frame)
{
struct fpu *fpu = ¤t->thread.fpu;
void __user *buf_fx = buf;
bool ia32_fxstate = false;
bool success = false;
unsigned int size;
if (unlikely(!buf)) {
fpu__clear_user_states(fpu);
return true;
}
size = xstate_sigframe_size(fpu->fpstate);
ia32_frame &= (IS_ENABLED(CONFIG_X86_32) ||
IS_ENABLED(CONFIG_IA32_EMULATION));
/*
* Only FXSR enabled systems need the FX state quirk.
* FRSTOR does not need it and can use the fast path.
*/
if (ia32_frame && use_fxsr()) {
buf_fx = buf + sizeof(struct fregs_state);
size += sizeof(struct fregs_state);
ia32_fxstate = true;
}
if (!access_ok(buf, size))
goto out;
if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
success = !fpregs_soft_set(current, NULL, 0,
sizeof(struct user_i387_ia32_struct),
NULL, buf);
} else {
success = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
}
out:
if (unlikely(!success))
fpu__clear_user_states(fpu);
return success;
}
unsigned long
fpu__alloc_mathframe(unsigned long sp, int ia32_frame,
unsigned long *buf_fx, unsigned long *size)
{
unsigned long frame_size = xstate_sigframe_size(current->thread.fpu.fpstate);
*buf_fx = sp = round_down(sp - frame_size, 64);
if (ia32_frame && use_fxsr()) {
frame_size += sizeof(struct fregs_state);
sp -= sizeof(struct fregs_state);
}
*size = frame_size;
return sp;
}
unsigned long __init fpu__get_fpstate_size(void)
{
unsigned long ret = fpu_user_cfg.max_size;
if (use_xsave())
ret += FP_XSTATE_MAGIC2_SIZE;
/*
* This space is needed on (most) 32-bit kernels, or when a 32-bit
* app is running on a 64-bit kernel. To keep things simple, just
* assume the worst case and always include space for 'freg_state',
* even for 64-bit apps on 64-bit kernels. This wastes a bit of
* space, but keeps the code simple.
*/
if ((IS_ENABLED(CONFIG_IA32_EMULATION) ||
IS_ENABLED(CONFIG_X86_32)) && use_fxsr())
ret += sizeof(struct fregs_state);
return ret;
}
// SPDX-License-Identifier: GPL-2.0
/*
* When connected to the machine, the Thrustmaster wheels appear as
* a «generic» hid gamepad called "Thrustmaster FFB Wheel".
*
* When in this mode not every functionality of the wheel, like the force feedback,
* are available. To enable all functionalities of a Thrustmaster wheel we have to send
* to it a specific USB CONTROL request with a code different for each wheel.
*
* This driver tries to understand which model of Thrustmaster wheel the generic
* "Thrustmaster FFB Wheel" really is and then sends the appropriate control code.
*
* Copyright (c) 2020-2021 Dario Pagani <dario.pagani.146+linuxk@gmail.com>
* Copyright (c) 2020-2021 Kim Kuparinen <kimi.h.kuparinen@gmail.com>
*/
#include <linux/hid.h>
#include <linux/usb.h>
#include <linux/input.h>
#include <linux/slab.h>
#include <linux/module.h>
/*
* These interrupts are used to prevent a nasty crash when initializing the
* T300RS. Used in thrustmaster_interrupts().
*/
static const u8 setup_0[] = { 0x42, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
static const u8 setup_1[] = { 0x0a, 0x04, 0x90, 0x03, 0x00, 0x00, 0x00, 0x00 };
static const u8 setup_2[] = { 0x0a, 0x04, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00 };
static const u8 setup_3[] = { 0x0a, 0x04, 0x12, 0x10, 0x00, 0x00, 0x00, 0x00 };
static const u8 setup_4[] = { 0x0a, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00 };
static const u8 *const setup_arr[] = { setup_0, setup_1, setup_2, setup_3, setup_4 };
static const unsigned int setup_arr_sizes[] = {
ARRAY_SIZE(setup_0),
ARRAY_SIZE(setup_1),
ARRAY_SIZE(setup_2),
ARRAY_SIZE(setup_3),
ARRAY_SIZE(setup_4)
};
/*
* This struct contains for each type of
* Thrustmaster wheel
*
* Note: The values are stored in the CPU
* endianness, the USB protocols always use
* little endian; the macro cpu_to_le[BIT]()
* must be used when preparing USB packets
* and vice-versa
*/
struct tm_wheel_info {
uint16_t wheel_type;
/*
* See when the USB control out packet is prepared...
* @TODO The TMX seems to require multiple control codes to switch.
*/
uint16_t switch_value;
char const *const wheel_name;
};
/*
* Known wheels.
* Note: TMX does not work as it requires 2 control packets
*/
static const struct tm_wheel_info tm_wheels_infos[] = {
{0x0306, 0x0006, "Thrustmaster T150RS"},
{0x0200, 0x0005, "Thrustmaster T300RS (Missing Attachment)"},
{0x0206, 0x0005, "Thrustmaster T300RS"},
{0x0209, 0x0005, "Thrustmaster T300RS (Open Wheel Attachment)"},
{0x020a, 0x0005, "Thrustmaster T300RS (Sparco R383 Mod)"},
{0x0204, 0x0005, "Thrustmaster T300 Ferrari Alcantara Edition"},
{0x0002, 0x0002, "Thrustmaster T500RS"}
//{0x0407, 0x0001, "Thrustmaster TMX"}
};
static const uint8_t tm_wheels_infos_length = 7;
/*
* This structs contains (in little endian) the response data
* of the wheel to the request 73
*
* A sufficient research to understand what each field does is not
* beign conducted yet. The position and meaning of fields are a
* just a very optimistic guess based on instinct....
*/
struct __packed tm_wheel_response
{
/*
* Seems to be the type of packet
* - 0x0049 if is data.a (15 bytes)
* - 0x0047 if is data.b (7 bytes)
*/
uint16_t type;
union {
struct __packed {
uint16_t field0;
uint16_t field1;
/*
* Seems to be the model code of the wheel
* Read table thrustmaster_wheels to values
*/
uint16_t model;
uint16_t field2;
uint16_t field3;
uint16_t field4;
uint16_t field5;
} a;
struct __packed {
uint16_t field0;
uint16_t field1;
uint16_t model;
} b;
} data;
};
struct tm_wheel {
struct usb_device *usb_dev;
struct urb *urb;
struct usb_ctrlrequest *model_request;
struct tm_wheel_response *response;
struct usb_ctrlrequest *change_request;
};
/* The control packet to send to wheel */
static const struct usb_ctrlrequest model_request = {
.bRequestType = 0xc1,
.bRequest = 73,
.wValue = 0,
.wIndex = 0,
.wLength = cpu_to_le16(0x0010)
};
static const struct usb_ctrlrequest change_request = {
.bRequestType = 0x41,
.bRequest = 83,
.wValue = 0, // Will be filled by the driver
.wIndex = 0,
.wLength = 0
};
/*
* On some setups initializing the T300RS crashes the kernel,
* these interrupts fix that particular issue. So far they haven't caused any
* adverse effects in other wheels.
*/
static void thrustmaster_interrupts(struct hid_device *hdev)
{
int ret, trans, i, b_ep;
u8 *send_buf = kmalloc(256, GFP_KERNEL);
struct usb_host_endpoint *ep;
struct device *dev = &hdev->dev;
struct usb_interface *usbif = to_usb_interface(dev->parent);
struct usb_device *usbdev = interface_to_usbdev(usbif);
if (!send_buf) {
hid_err(hdev, "failed allocating send buffer\n");
return;
}
if (usbif->cur_altsetting->desc.bNumEndpoints < 2) {
kfree(send_buf);
hid_err(hdev, "Wrong number of endpoints?\n");
return;
}
ep = &usbif->cur_altsetting->endpoint[1];
b_ep = ep->desc.bEndpointAddress;
for (i = 0; i < ARRAY_SIZE(setup_arr); ++i) {
memcpy(send_buf, setup_arr[i], setup_arr_sizes[i]);
ret = usb_interrupt_msg(usbdev,
usb_sndintpipe(usbdev, b_ep),
send_buf,
setup_arr_sizes[i],
&trans,
USB_CTRL_SET_TIMEOUT);
if (ret) {
hid_err(hdev, "setup data couldn't be sent\n");
kfree(send_buf);
return;
}
}
kfree(send_buf);
}
static void thrustmaster_change_handler(struct urb *urb)
{
struct hid_device *hdev = urb->context;
// The wheel seems to kill himself before answering the host and therefore is violating the USB protocol...
if (urb->status == 0 || urb->status == -EPROTO || urb->status == -EPIPE)
hid_info(hdev, "Success?! The wheel should have been initialized!\n");
else
hid_warn(hdev, "URB to change wheel mode seems to have failed with error %d\n", urb->status);
}
/*
* Called by the USB subsystem when the wheel responses to our request
* to get [what it seems to be] the wheel's model.
*
* If the model id is recognized then we send an opportune USB CONTROL REQUEST
* to switch the wheel to its full capabilities
*/
static void thrustmaster_model_handler(struct urb *urb)
{
struct hid_device *hdev = urb->context;
struct tm_wheel *tm_wheel = hid_get_drvdata(hdev);
uint16_t model = 0;
int i, ret;
const struct tm_wheel_info *twi = NULL;
if (urb->status) {
hid_err(hdev, "URB to get model id failed with error %d\n", urb->status);
return;
}
if (tm_wheel->response->type == cpu_to_le16(0x49)) model = le16_to_cpu(tm_wheel->response->data.a.model); else if (tm_wheel->response->type == cpu_to_le16(0x47)) model = le16_to_cpu(tm_wheel->response->data.b.model);
else {
hid_err(hdev, "Unknown packet type 0x%x, unable to proceed further with wheel init\n", tm_wheel->response->type);
return;
}
for (i = 0; i < tm_wheels_infos_length && !twi; i++) if (tm_wheels_infos[i].wheel_type == model) twi = tm_wheels_infos + i;
if (twi)
hid_info(hdev, "Wheel with model id 0x%x is a %s\n", model, twi->wheel_name);
else {
hid_err(hdev, "Unknown wheel's model id 0x%x, unable to proceed further with wheel init\n", model);
return;
}
tm_wheel->change_request->wValue = cpu_to_le16(twi->switch_value);
usb_fill_control_urb(
tm_wheel->urb,
tm_wheel->usb_dev,
usb_sndctrlpipe(tm_wheel->usb_dev, 0),
(char *)tm_wheel->change_request,
NULL, 0, // We do not expect any response from the wheel
thrustmaster_change_handler,
hdev
);
ret = usb_submit_urb(tm_wheel->urb, GFP_ATOMIC);
if (ret)
hid_err(hdev, "Error %d while submitting the change URB. I am unable to initialize this wheel...\n", ret);
}
static void thrustmaster_remove(struct hid_device *hdev)
{
struct tm_wheel *tm_wheel = hid_get_drvdata(hdev);
usb_kill_urb(tm_wheel->urb);
kfree(tm_wheel->change_request);
kfree(tm_wheel->response);
kfree(tm_wheel->model_request);
usb_free_urb(tm_wheel->urb);
kfree(tm_wheel);
hid_hw_stop(hdev);
}
/*
* Function called by HID when a hid Thrustmaster FFB wheel is connected to the host.
* This function starts the hid dev, tries to allocate the tm_wheel data structure and
* finally send an USB CONTROL REQUEST to the wheel to get [what it seems to be] its
* model type.
*/
static int thrustmaster_probe(struct hid_device *hdev, const struct hid_device_id *id)
{
int ret = 0;
struct tm_wheel *tm_wheel = NULL;
if (!hid_is_usb(hdev))
return -EINVAL;
ret = hid_parse(hdev);
if (ret) {
hid_err(hdev, "parse failed with error %d\n", ret);
goto error0;
}
ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT & ~HID_CONNECT_FF);
if (ret) {
hid_err(hdev, "hw start failed with error %d\n", ret);
goto error0;
}
// Now we allocate the tm_wheel
tm_wheel = kzalloc(sizeof(struct tm_wheel), GFP_KERNEL);
if (!tm_wheel) {
ret = -ENOMEM;
goto error1;
}
tm_wheel->urb = usb_alloc_urb(0, GFP_ATOMIC);
if (!tm_wheel->urb) {
ret = -ENOMEM;
goto error2;
}
tm_wheel->model_request = kmemdup(&model_request,
sizeof(struct usb_ctrlrequest),
GFP_KERNEL);
if (!tm_wheel->model_request) {
ret = -ENOMEM;
goto error3;
}
tm_wheel->response = kzalloc(sizeof(struct tm_wheel_response), GFP_KERNEL);
if (!tm_wheel->response) {
ret = -ENOMEM;
goto error4;
}
tm_wheel->change_request = kmemdup(&change_request,
sizeof(struct usb_ctrlrequest),
GFP_KERNEL);
if (!tm_wheel->change_request) {
ret = -ENOMEM;
goto error5;
}
tm_wheel->usb_dev = interface_to_usbdev(to_usb_interface(hdev->dev.parent));
hid_set_drvdata(hdev, tm_wheel);
thrustmaster_interrupts(hdev);
usb_fill_control_urb(
tm_wheel->urb,
tm_wheel->usb_dev,
usb_rcvctrlpipe(tm_wheel->usb_dev, 0),
(char *)tm_wheel->model_request,
tm_wheel->response,
sizeof(struct tm_wheel_response),
thrustmaster_model_handler,
hdev
);
ret = usb_submit_urb(tm_wheel->urb, GFP_ATOMIC);
if (ret) {
hid_err(hdev, "Error %d while submitting the URB. I am unable to initialize this wheel...\n", ret);
goto error6;
}
return ret;
error6: kfree(tm_wheel->change_request);
error5: kfree(tm_wheel->response);
error4: kfree(tm_wheel->model_request);
error3: usb_free_urb(tm_wheel->urb);
error2: kfree(tm_wheel);
error1: hid_hw_stop(hdev);
error0:
return ret;
}
static const struct hid_device_id thrustmaster_devices[] = {
{ HID_USB_DEVICE(0x044f, 0xb65d)},
{}
};
MODULE_DEVICE_TABLE(hid, thrustmaster_devices);
static struct hid_driver thrustmaster_driver = {
.name = "hid-thrustmaster",
.id_table = thrustmaster_devices,
.probe = thrustmaster_probe,
.remove = thrustmaster_remove,
};
module_hid_driver(thrustmaster_driver);
MODULE_AUTHOR("Dario Pagani <dario.pagani.146+linuxk@gmail.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Driver to initialize some steering wheel joysticks from Thrustmaster");
// SPDX-License-Identifier: GPL-2.0-only
/*
* klist.c - Routines for manipulating klists.
*
* Copyright (C) 2005 Patrick Mochel
*
* This klist interface provides a couple of structures that wrap around
* struct list_head to provide explicit list "head" (struct klist) and list
* "node" (struct klist_node) objects. For struct klist, a spinlock is
* included that protects access to the actual list itself. struct
* klist_node provides a pointer to the klist that owns it and a kref
* reference count that indicates the number of current users of that node
* in the list.
*
* The entire point is to provide an interface for iterating over a list
* that is safe and allows for modification of the list during the
* iteration (e.g. insertion and removal), including modification of the
* current node on the list.
*
* It works using a 3rd object type - struct klist_iter - that is declared
* and initialized before an iteration. klist_next() is used to acquire the
* next element in the list. It returns NULL if there are no more items.
* Internally, that routine takes the klist's lock, decrements the
* reference count of the previous klist_node and increments the count of
* the next klist_node. It then drops the lock and returns.
*
* There are primitives for adding and removing nodes to/from a klist.
* When deleting, klist_del() will simply decrement the reference count.
* Only when the count goes to 0 is the node removed from the list.
* klist_remove() will try to delete the node from the list and block until
* it is actually removed. This is useful for objects (like devices) that
* have been removed from the system and must be freed (but must wait until
* all accessors have finished).
*/
#include <linux/klist.h>
#include <linux/export.h>
#include <linux/sched.h>
/*
* Use the lowest bit of n_klist to mark deleted nodes and exclude
* dead ones from iteration.
*/
#define KNODE_DEAD 1LU
#define KNODE_KLIST_MASK ~KNODE_DEAD
static struct klist *knode_klist(struct klist_node *knode)
{
return (struct klist *)
((unsigned long)knode->n_klist & KNODE_KLIST_MASK);
}
static bool knode_dead(struct klist_node *knode)
{
return (unsigned long)knode->n_klist & KNODE_DEAD;
}
static void knode_set_klist(struct klist_node *knode, struct klist *klist)
{
knode->n_klist = klist;
/* no knode deserves to start its life dead */
WARN_ON(knode_dead(knode));
}
static void knode_kill(struct klist_node *knode)
{
/* and no knode should die twice ever either, see we're very humane */
WARN_ON(knode_dead(knode)); *(unsigned long *)&knode->n_klist |= KNODE_DEAD;
}
/**
* klist_init - Initialize a klist structure.
* @k: The klist we're initializing.
* @get: The get function for the embedding object (NULL if none)
* @put: The put function for the embedding object (NULL if none)
*
* Initialises the klist structure. If the klist_node structures are
* going to be embedded in refcounted objects (necessary for safe
* deletion) then the get/put arguments are used to initialise
* functions that take and release references on the embedding
* objects.
*/
void klist_init(struct klist *k, void (*get)(struct klist_node *),
void (*put)(struct klist_node *))
{
INIT_LIST_HEAD(&k->k_list);
spin_lock_init(&k->k_lock);
k->get = get;
k->put = put;
}
EXPORT_SYMBOL_GPL(klist_init);
static void add_head(struct klist *k, struct klist_node *n)
{
spin_lock(&k->k_lock);
list_add(&n->n_node, &k->k_list);
spin_unlock(&k->k_lock);
}
static void add_tail(struct klist *k, struct klist_node *n)
{
spin_lock(&k->k_lock);
list_add_tail(&n->n_node, &k->k_list); spin_unlock(&k->k_lock);
}
static void klist_node_init(struct klist *k, struct klist_node *n)
{
INIT_LIST_HEAD(&n->n_node);
kref_init(&n->n_ref);
knode_set_klist(n, k); if (k->get) k->get(n);
}
/**
* klist_add_head - Initialize a klist_node and add it to front.
* @n: node we're adding.
* @k: klist it's going on.
*/
void klist_add_head(struct klist_node *n, struct klist *k)
{
klist_node_init(k, n);
add_head(k, n);
}
EXPORT_SYMBOL_GPL(klist_add_head);
/**
* klist_add_tail - Initialize a klist_node and add it to back.
* @n: node we're adding.
* @k: klist it's going on.
*/
void klist_add_tail(struct klist_node *n, struct klist *k)
{
klist_node_init(k, n); add_tail(k, n);
}
EXPORT_SYMBOL_GPL(klist_add_tail);
/**
* klist_add_behind - Init a klist_node and add it after an existing node
* @n: node we're adding.
* @pos: node to put @n after
*/
void klist_add_behind(struct klist_node *n, struct klist_node *pos)
{
struct klist *k = knode_klist(pos);
klist_node_init(k, n);
spin_lock(&k->k_lock);
list_add(&n->n_node, &pos->n_node);
spin_unlock(&k->k_lock);
}
EXPORT_SYMBOL_GPL(klist_add_behind);
/**
* klist_add_before - Init a klist_node and add it before an existing node
* @n: node we're adding.
* @pos: node to put @n after
*/
void klist_add_before(struct klist_node *n, struct klist_node *pos)
{
struct klist *k = knode_klist(pos);
klist_node_init(k, n);
spin_lock(&k->k_lock);
list_add_tail(&n->n_node, &pos->n_node);
spin_unlock(&k->k_lock);
}
EXPORT_SYMBOL_GPL(klist_add_before);
struct klist_waiter {
struct list_head list;
struct klist_node *node;
struct task_struct *process;
int woken;
};
static DEFINE_SPINLOCK(klist_remove_lock);
static LIST_HEAD(klist_remove_waiters);
static void klist_release(struct kref *kref)
{
struct klist_waiter *waiter, *tmp;
struct klist_node *n = container_of(kref, struct klist_node, n_ref); WARN_ON(!knode_dead(n)); list_del(&n->n_node);
spin_lock(&klist_remove_lock);
list_for_each_entry_safe(waiter, tmp, &klist_remove_waiters, list) { if (waiter->node != n)
continue;
list_del(&waiter->list);
waiter->woken = 1;
mb();
wake_up_process(waiter->process);
}
spin_unlock(&klist_remove_lock);
knode_set_klist(n, NULL);
}
static int klist_dec_and_del(struct klist_node *n)
{
return kref_put(&n->n_ref, klist_release);
}
static void klist_put(struct klist_node *n, bool kill)
{
struct klist *k = knode_klist(n);
void (*put)(struct klist_node *) = k->put;
spin_lock(&k->k_lock);
if (kill)
knode_kill(n); if (!klist_dec_and_del(n))
put = NULL;
spin_unlock(&k->k_lock);
if (put)
put(n);
}
/**
* klist_del - Decrement the reference count of node and try to remove.
* @n: node we're deleting.
*/
void klist_del(struct klist_node *n)
{
klist_put(n, true);
}
EXPORT_SYMBOL_GPL(klist_del);
/**
* klist_remove - Decrement the refcount of node and wait for it to go away.
* @n: node we're removing.
*/
void klist_remove(struct klist_node *n)
{
struct klist_waiter waiter;
waiter.node = n;
waiter.process = current;
waiter.woken = 0;
spin_lock(&klist_remove_lock);
list_add(&waiter.list, &klist_remove_waiters); spin_unlock(&klist_remove_lock);
klist_del(n);
for (;;) {
set_current_state(TASK_UNINTERRUPTIBLE);
if (waiter.woken)
break;
schedule();
}
__set_current_state(TASK_RUNNING);
}
EXPORT_SYMBOL_GPL(klist_remove);
/**
* klist_node_attached - Say whether a node is bound to a list or not.
* @n: Node that we're testing.
*/
int klist_node_attached(struct klist_node *n)
{
return (n->n_klist != NULL);
}
EXPORT_SYMBOL_GPL(klist_node_attached);
/**
* klist_iter_init_node - Initialize a klist_iter structure.
* @k: klist we're iterating.
* @i: klist_iter we're filling.
* @n: node to start with.
*
* Similar to klist_iter_init(), but starts the action off with @n,
* instead of with the list head.
*/
void klist_iter_init_node(struct klist *k, struct klist_iter *i,
struct klist_node *n)
{
i->i_klist = k;
i->i_cur = NULL;
if (n && kref_get_unless_zero(&n->n_ref)) i->i_cur = n;}
EXPORT_SYMBOL_GPL(klist_iter_init_node);
/**
* klist_iter_init - Iniitalize a klist_iter structure.
* @k: klist we're iterating.
* @i: klist_iter structure we're filling.
*
* Similar to klist_iter_init_node(), but start with the list head.
*/
void klist_iter_init(struct klist *k, struct klist_iter *i)
{
klist_iter_init_node(k, i, NULL);
}
EXPORT_SYMBOL_GPL(klist_iter_init);
/**
* klist_iter_exit - Finish a list iteration.
* @i: Iterator structure.
*
* Must be called when done iterating over list, as it decrements the
* refcount of the current node. Necessary in case iteration exited before
* the end of the list was reached, and always good form.
*/
void klist_iter_exit(struct klist_iter *i)
{
if (i->i_cur) { klist_put(i->i_cur, false);
i->i_cur = NULL;
}
}
EXPORT_SYMBOL_GPL(klist_iter_exit);
static struct klist_node *to_klist_node(struct list_head *n)
{
return container_of(n, struct klist_node, n_node);
}
/**
* klist_prev - Ante up prev node in list.
* @i: Iterator structure.
*
* First grab list lock. Decrement the reference count of the previous
* node, if there was one. Grab the prev node, increment its reference
* count, drop the lock, and return that prev node.
*/
struct klist_node *klist_prev(struct klist_iter *i)
{
void (*put)(struct klist_node *) = i->i_klist->put;
struct klist_node *last = i->i_cur;
struct klist_node *prev;
unsigned long flags;
spin_lock_irqsave(&i->i_klist->k_lock, flags);
if (last) {
prev = to_klist_node(last->n_node.prev); if (!klist_dec_and_del(last))
put = NULL;
} else
prev = to_klist_node(i->i_klist->k_list.prev); i->i_cur = NULL;
while (prev != to_klist_node(&i->i_klist->k_list)) {
if (likely(!knode_dead(prev))) { kref_get(&prev->n_ref); i->i_cur = prev;
break;
}
prev = to_klist_node(prev->n_node.prev);
}
spin_unlock_irqrestore(&i->i_klist->k_lock, flags); if (put && last) put(last); return i->i_cur;
}
EXPORT_SYMBOL_GPL(klist_prev);
/**
* klist_next - Ante up next node in list.
* @i: Iterator structure.
*
* First grab list lock. Decrement the reference count of the previous
* node, if there was one. Grab the next node, increment its reference
* count, drop the lock, and return that next node.
*/
struct klist_node *klist_next(struct klist_iter *i)
{
void (*put)(struct klist_node *) = i->i_klist->put;
struct klist_node *last = i->i_cur;
struct klist_node *next;
unsigned long flags;
spin_lock_irqsave(&i->i_klist->k_lock, flags);
if (last) {
next = to_klist_node(last->n_node.next); if (!klist_dec_and_del(last))
put = NULL;
} else
next = to_klist_node(i->i_klist->k_list.next); i->i_cur = NULL;
while (next != to_klist_node(&i->i_klist->k_list)) {
if (likely(!knode_dead(next))) { kref_get(&next->n_ref); i->i_cur = next;
break;
}
next = to_klist_node(next->n_node.next);
}
spin_unlock_irqrestore(&i->i_klist->k_lock, flags); if (put && last) put(last); return i->i_cur;
}
EXPORT_SYMBOL_GPL(klist_next);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PTRACE_H
#define _ASM_X86_PTRACE_H
#include <asm/segment.h>
#include <asm/page_types.h>
#include <uapi/asm/ptrace.h>
#ifndef __ASSEMBLY__
#ifdef __i386__
struct pt_regs {
/*
* NB: 32-bit x86 CPUs are inconsistent as what happens in the
* following cases (where %seg represents a segment register):
*
* - pushl %seg: some do a 16-bit write and leave the high
* bits alone
* - movl %seg, [mem]: some do a 16-bit write despite the movl
* - IDT entry: some (e.g. 486) will leave the high bits of CS
* and (if applicable) SS undefined.
*
* Fortunately, x86-32 doesn't read the high bits on POP or IRET,
* so we can just treat all of the segment registers as 16-bit
* values.
*/
unsigned long bx;
unsigned long cx;
unsigned long dx;
unsigned long si;
unsigned long di;
unsigned long bp;
unsigned long ax;
unsigned short ds;
unsigned short __dsh;
unsigned short es;
unsigned short __esh;
unsigned short fs;
unsigned short __fsh;
/*
* On interrupt, gs and __gsh store the vector number. They never
* store gs any more.
*/
unsigned short gs;
unsigned short __gsh;
/* On interrupt, this is the error code. */
unsigned long orig_ax;
unsigned long ip;
unsigned short cs;
unsigned short __csh;
unsigned long flags;
unsigned long sp;
unsigned short ss;
unsigned short __ssh;
};
#else /* __i386__ */
struct fred_cs {
/* CS selector */
u64 cs : 16,
/* Stack level at event time */
sl : 2,
/* IBT in WAIT_FOR_ENDBRANCH state */
wfe : 1,
: 45;
};
struct fred_ss {
/* SS selector */
u64 ss : 16,
/* STI state */
sti : 1,
/* Set if syscall, sysenter or INT n */
swevent : 1,
/* Event is NMI type */
nmi : 1,
: 13,
/* Event vector */
vector : 8,
: 8,
/* Event type */
type : 4,
: 4,
/* Event was incident to enclave execution */
enclave : 1,
/* CPU was in long mode */
lm : 1,
/*
* Nested exception during FRED delivery, not set
* for #DF.
*/
nested : 1,
: 1,
/*
* The length of the instruction causing the event.
* Only set for INTO, INT1, INT3, INT n, SYSCALL
* and SYSENTER. 0 otherwise.
*/
insnlen : 4;
};
struct pt_regs {
/*
* C ABI says these regs are callee-preserved. They aren't saved on
* kernel entry unless syscall needs a complete, fully filled
* "struct pt_regs".
*/
unsigned long r15;
unsigned long r14;
unsigned long r13;
unsigned long r12;
unsigned long bp;
unsigned long bx;
/* These regs are callee-clobbered. Always saved on kernel entry. */
unsigned long r11;
unsigned long r10;
unsigned long r9;
unsigned long r8;
unsigned long ax;
unsigned long cx;
unsigned long dx;
unsigned long si;
unsigned long di;
/*
* orig_ax is used on entry for:
* - the syscall number (syscall, sysenter, int80)
* - error_code stored by the CPU on traps and exceptions
* - the interrupt number for device interrupts
*
* A FRED stack frame starts here:
* 1) It _always_ includes an error code;
*
* 2) The return frame for ERET[US] starts here, but
* the content of orig_ax is ignored.
*/
unsigned long orig_ax;
/* The IRETQ return frame starts here */
unsigned long ip;
union {
/* CS selector */
u16 cs;
/* The extended 64-bit data slot containing CS */
u64 csx;
/* The FRED CS extension */
struct fred_cs fred_cs;
};
unsigned long flags;
unsigned long sp;
union {
/* SS selector */
u16 ss;
/* The extended 64-bit data slot containing SS */
u64 ssx;
/* The FRED SS extension */
struct fred_ss fred_ss;
};
/*
* Top of stack on IDT systems, while FRED systems have extra fields
* defined above for storing exception related information, e.g. CR2 or
* DR6.
*/
};
#endif /* !__i386__ */
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt_types.h>
#endif
#include <asm/proto.h>
struct cpuinfo_x86;
struct task_struct;
extern unsigned long profile_pc(struct pt_regs *regs);
extern unsigned long
convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs);
extern void send_sigtrap(struct pt_regs *regs, int error_code, int si_code);
static inline unsigned long regs_return_value(struct pt_regs *regs)
{
return regs->ax;
}
static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
{
regs->ax = rc;
}
/*
* user_mode(regs) determines whether a register set came from user
* mode. On x86_32, this is true if V8086 mode was enabled OR if the
* register set was from protected mode with RPL-3 CS value. This
* tricky test checks that with one comparison.
*
* On x86_64, vm86 mode is mercifully nonexistent, and we don't need
* the extra check.
*/
static __always_inline int user_mode(struct pt_regs *regs)
{
#ifdef CONFIG_X86_32
return ((regs->cs & SEGMENT_RPL_MASK) | (regs->flags & X86_VM_MASK)) >= USER_RPL;
#else
return !!(regs->cs & 3);
#endif
}
static __always_inline int v8086_mode(struct pt_regs *regs)
{
#ifdef CONFIG_X86_32
return (regs->flags & X86_VM_MASK);
#else
return 0; /* No V86 mode support in long mode */
#endif
}
static inline bool user_64bit_mode(struct pt_regs *regs)
{
#ifdef CONFIG_X86_64
#ifndef CONFIG_PARAVIRT_XXL
/*
* On non-paravirt systems, this is the only long mode CPL 3
* selector. We do not allow long mode selectors in the LDT.
*/
return regs->cs == __USER_CS;
#else
/* Headers are too twisted for this to go in paravirt.h. */
return regs->cs == __USER_CS || regs->cs == pv_info.extra_user_64bit_cs;
#endif
#else /* !CONFIG_X86_64 */
return false;
#endif
}
/*
* Determine whether the register set came from any context that is running in
* 64-bit mode.
*/
static inline bool any_64bit_mode(struct pt_regs *regs)
{
#ifdef CONFIG_X86_64
return !user_mode(regs) || user_64bit_mode(regs);
#else
return false;
#endif
}
#ifdef CONFIG_X86_64
#define current_user_stack_pointer() current_pt_regs()->sp
#define compat_user_stack_pointer() current_pt_regs()->sp
static __always_inline bool ip_within_syscall_gap(struct pt_regs *regs)
{
bool ret = (regs->ip >= (unsigned long)entry_SYSCALL_64 &&
regs->ip < (unsigned long)entry_SYSCALL_64_safe_stack);
ret = ret || (regs->ip >= (unsigned long)entry_SYSRETQ_unsafe_stack &&
regs->ip < (unsigned long)entry_SYSRETQ_end);
#ifdef CONFIG_IA32_EMULATION
ret = ret || (regs->ip >= (unsigned long)entry_SYSCALL_compat &&
regs->ip < (unsigned long)entry_SYSCALL_compat_safe_stack);
ret = ret || (regs->ip >= (unsigned long)entry_SYSRETL_compat_unsafe_stack &&
regs->ip < (unsigned long)entry_SYSRETL_compat_end);
#endif
return ret;
}
#endif
static inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
{
return regs->sp;
}
static inline unsigned long instruction_pointer(struct pt_regs *regs)
{
return regs->ip;
}
static inline void instruction_pointer_set(struct pt_regs *regs,
unsigned long val)
{
regs->ip = val;
}
static inline unsigned long frame_pointer(struct pt_regs *regs)
{
return regs->bp;
}
static inline unsigned long user_stack_pointer(struct pt_regs *regs)
{
return regs->sp;
}
static inline void user_stack_pointer_set(struct pt_regs *regs,
unsigned long val)
{
regs->sp = val;
}
static __always_inline bool regs_irqs_disabled(struct pt_regs *regs)
{
return !(regs->flags & X86_EFLAGS_IF);
}
/* Query offset/name of register from its name/offset */
extern int regs_query_register_offset(const char *name);
extern const char *regs_query_register_name(unsigned int offset);
#define MAX_REG_OFFSET (offsetof(struct pt_regs, ss))
/**
* regs_get_register() - get register value from its offset
* @regs: pt_regs from which register value is gotten.
* @offset: offset number of the register.
*
* regs_get_register returns the value of a register. The @offset is the
* offset of the register in struct pt_regs address which specified by @regs.
* If @offset is bigger than MAX_REG_OFFSET, this returns 0.
*/
static inline unsigned long regs_get_register(struct pt_regs *regs,
unsigned int offset)
{
if (unlikely(offset > MAX_REG_OFFSET))
return 0;
#ifdef CONFIG_X86_32
/* The selector fields are 16-bit. */
if (offset == offsetof(struct pt_regs, cs) ||
offset == offsetof(struct pt_regs, ss) ||
offset == offsetof(struct pt_regs, ds) ||
offset == offsetof(struct pt_regs, es) ||
offset == offsetof(struct pt_regs, fs) ||
offset == offsetof(struct pt_regs, gs)) {
return *(u16 *)((unsigned long)regs + offset);
}
#endif
return *(unsigned long *)((unsigned long)regs + offset);
}
/**
* regs_within_kernel_stack() - check the address in the stack
* @regs: pt_regs which contains kernel stack pointer.
* @addr: address which is checked.
*
* regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
* If @addr is within the kernel stack, it returns true. If not, returns false.
*/
static inline int regs_within_kernel_stack(struct pt_regs *regs,
unsigned long addr)
{
return ((addr & ~(THREAD_SIZE - 1)) == (regs->sp & ~(THREAD_SIZE - 1)));
}
/**
* regs_get_kernel_stack_nth_addr() - get the address of the Nth entry on stack
* @regs: pt_regs which contains kernel stack pointer.
* @n: stack entry number.
*
* regs_get_kernel_stack_nth() returns the address of the @n th entry of the
* kernel stack which is specified by @regs. If the @n th entry is NOT in
* the kernel stack, this returns NULL.
*/
static inline unsigned long *regs_get_kernel_stack_nth_addr(struct pt_regs *regs, unsigned int n)
{
unsigned long *addr = (unsigned long *)regs->sp;
addr += n;
if (regs_within_kernel_stack(regs, (unsigned long)addr))
return addr;
else
return NULL;
}
/* To avoid include hell, we can't include uaccess.h */
extern long copy_from_kernel_nofault(void *dst, const void *src, size_t size);
/**
* regs_get_kernel_stack_nth() - get Nth entry of the stack
* @regs: pt_regs which contains kernel stack pointer.
* @n: stack entry number.
*
* regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
* is specified by @regs. If the @n th entry is NOT in the kernel stack
* this returns 0.
*/
static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs,
unsigned int n)
{
unsigned long *addr;
unsigned long val;
long ret;
addr = regs_get_kernel_stack_nth_addr(regs, n);
if (addr) {
ret = copy_from_kernel_nofault(&val, addr, sizeof(val));
if (!ret)
return val;
}
return 0;
}
/**
* regs_get_kernel_argument() - get Nth function argument in kernel
* @regs: pt_regs of that context
* @n: function argument number (start from 0)
*
* regs_get_argument() returns @n th argument of the function call.
* Note that this chooses most probably assignment, in some case
* it can be incorrect.
* This is expected to be called from kprobes or ftrace with regs
* where the top of stack is the return address.
*/
static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,
unsigned int n)
{
static const unsigned int argument_offs[] = {
#ifdef __i386__
offsetof(struct pt_regs, ax),
offsetof(struct pt_regs, dx),
offsetof(struct pt_regs, cx),
#define NR_REG_ARGUMENTS 3
#else
offsetof(struct pt_regs, di),
offsetof(struct pt_regs, si),
offsetof(struct pt_regs, dx),
offsetof(struct pt_regs, cx),
offsetof(struct pt_regs, r8),
offsetof(struct pt_regs, r9),
#define NR_REG_ARGUMENTS 6
#endif
};
if (n >= NR_REG_ARGUMENTS) {
n -= NR_REG_ARGUMENTS - 1;
return regs_get_kernel_stack_nth(regs, n);
} else
return regs_get_register(regs, argument_offs[n]);
}
#define arch_has_single_step() (1)
#ifdef CONFIG_X86_DEBUGCTLMSR
#define arch_has_block_step() (1)
#else
#define arch_has_block_step() (boot_cpu_data.x86 >= 6)
#endif
#define ARCH_HAS_USER_SINGLE_STEP_REPORT
struct user_desc;
extern int do_get_thread_area(struct task_struct *p, int idx,
struct user_desc __user *info);
extern int do_set_thread_area(struct task_struct *p, int idx,
struct user_desc __user *info, int can_allocate);
#ifdef CONFIG_X86_64
# define do_set_thread_area_64(p, s, t) do_arch_prctl_64(p, s, t)
#else
# define do_set_thread_area_64(p, s, t) (0)
#endif
#endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_PTRACE_H */
#ifndef _LINUX_HASH_H
#define _LINUX_HASH_H
/* Fast hashing routine for ints, longs and pointers.
(C) 2002 Nadia Yvette Chambers, IBM */
#include <asm/types.h>
#include <linux/compiler.h>
/*
* The "GOLDEN_RATIO_PRIME" is used in ifs/btrfs/brtfs_inode.h and
* fs/inode.c. It's not actually prime any more (the previous primes
* were actively bad for hashing), but the name remains.
*/
#if BITS_PER_LONG == 32
#define GOLDEN_RATIO_PRIME GOLDEN_RATIO_32
#define hash_long(val, bits) hash_32(val, bits)
#elif BITS_PER_LONG == 64
#define hash_long(val, bits) hash_64(val, bits)
#define GOLDEN_RATIO_PRIME GOLDEN_RATIO_64
#else
#error Wordsize not 32 or 64
#endif
/*
* This hash multiplies the input by a large odd number and takes the
* high bits. Since multiplication propagates changes to the most
* significant end only, it is essential that the high bits of the
* product be used for the hash value.
*
* Chuck Lever verified the effectiveness of this technique:
* http://www.citi.umich.edu/techreports/reports/citi-tr-00-1.pdf
*
* Although a random odd number will do, it turns out that the golden
* ratio phi = (sqrt(5)-1)/2, or its negative, has particularly nice
* properties. (See Knuth vol 3, section 6.4, exercise 9.)
*
* These are the negative, (1 - phi) = phi**2 = (3 - sqrt(5))/2,
* which is very slightly easier to multiply by and makes no
* difference to the hash distribution.
*/
#define GOLDEN_RATIO_32 0x61C88647
#define GOLDEN_RATIO_64 0x61C8864680B583EBull
#ifdef CONFIG_HAVE_ARCH_HASH
/* This header may use the GOLDEN_RATIO_xx constants */
#include <asm/hash.h>
#endif
/*
* The _generic versions exist only so lib/test_hash.c can compare
* the arch-optimized versions with the generic.
*
* Note that if you change these, any <asm/hash.h> that aren't updated
* to match need to have their HAVE_ARCH_* define values updated so the
* self-test will not false-positive.
*/
#ifndef HAVE_ARCH__HASH_32
#define __hash_32 __hash_32_generic
#endif
static inline u32 __hash_32_generic(u32 val)
{
return val * GOLDEN_RATIO_32;
}
static inline u32 hash_32(u32 val, unsigned int bits)
{
/* High bits are more random, so use them. */
return __hash_32(val) >> (32 - bits);
}
#ifndef HAVE_ARCH_HASH_64
#define hash_64 hash_64_generic
#endif
static __always_inline u32 hash_64_generic(u64 val, unsigned int bits)
{
#if BITS_PER_LONG == 64
/* 64x64-bit multiply is efficient on all 64-bit processors */
return val * GOLDEN_RATIO_64 >> (64 - bits);
#else
/* Hash 64 bits using only 32x32-bit multiply. */
return hash_32((u32)val ^ __hash_32(val >> 32), bits);
#endif
}
static inline u32 hash_ptr(const void *ptr, unsigned int bits)
{
return hash_long((unsigned long)ptr, bits);
}
/* This really should be called fold32_ptr; it does no hashing to speak of. */
static inline u32 hash32_ptr(const void *ptr)
{
unsigned long val = (unsigned long)ptr;
#if BITS_PER_LONG == 64
val ^= (val >> 32);
#endif
return (u32)val;
}
#endif /* _LINUX_HASH_H */
// SPDX-License-Identifier: GPL-2.0
/*
* fs/sysfs/group.c - Operations for adding/removing multiple files at once.
*
* Copyright (c) 2003 Patrick Mochel
* Copyright (c) 2003 Open Source Development Lab
* Copyright (c) 2013 Greg Kroah-Hartman
* Copyright (c) 2013 The Linux Foundation
*/
#include <linux/kobject.h>
#include <linux/module.h>
#include <linux/dcache.h>
#include <linux/namei.h>
#include <linux/err.h>
#include <linux/fs.h>
#include "sysfs.h"
static void remove_files(struct kernfs_node *parent,
const struct attribute_group *grp)
{
struct attribute *const *attr;
struct bin_attribute *const *bin_attr;
if (grp->attrs) for (attr = grp->attrs; *attr; attr++) kernfs_remove_by_name(parent, (*attr)->name); if (grp->bin_attrs) for (bin_attr = grp->bin_attrs; *bin_attr; bin_attr++) kernfs_remove_by_name(parent, (*bin_attr)->attr.name);
}
static umode_t __first_visible(const struct attribute_group *grp, struct kobject *kobj)
{
if (grp->attrs && grp->attrs[0] && grp->is_visible)
return grp->is_visible(kobj, grp->attrs[0], 0);
if (grp->bin_attrs && grp->bin_attrs[0] && grp->is_bin_visible) return grp->is_bin_visible(kobj, grp->bin_attrs[0], 0);
return 0;
}
static int create_files(struct kernfs_node *parent, struct kobject *kobj,
kuid_t uid, kgid_t gid,
const struct attribute_group *grp, int update)
{
struct attribute *const *attr;
struct bin_attribute *const *bin_attr;
int error = 0, i;
if (grp->attrs) {
for (i = 0, attr = grp->attrs; *attr && !error; i++, attr++) { umode_t mode = (*attr)->mode;
/*
* In update mode, we're changing the permissions or
* visibility. Do this by first removing then
* re-adding (if required) the file.
*/
if (update)
kernfs_remove_by_name(parent, (*attr)->name); if (grp->is_visible) { mode = grp->is_visible(kobj, *attr, i);
mode &= ~SYSFS_GROUP_INVISIBLE;
if (!mode)
continue;
}
WARN(mode & ~(SYSFS_PREALLOC | 0664),
"Attribute %s: Invalid permissions 0%o\n",
(*attr)->name, mode);
mode &= SYSFS_PREALLOC | 0664;
error = sysfs_add_file_mode_ns(parent, *attr, mode, uid,
gid, NULL);
if (unlikely(error))
break;
}
if (error) {
remove_files(parent, grp);
goto exit;
}
}
if (grp->bin_attrs) { for (i = 0, bin_attr = grp->bin_attrs; *bin_attr; i++, bin_attr++) { umode_t mode = (*bin_attr)->attr.mode;
if (update)
kernfs_remove_by_name(parent,
(*bin_attr)->attr.name);
if (grp->is_bin_visible) { mode = grp->is_bin_visible(kobj, *bin_attr, i);
mode &= ~SYSFS_GROUP_INVISIBLE;
if (!mode)
continue;
}
WARN(mode & ~(SYSFS_PREALLOC | 0664),
"Attribute %s: Invalid permissions 0%o\n",
(*bin_attr)->attr.name, mode);
mode &= SYSFS_PREALLOC | 0664;
error = sysfs_add_bin_file_mode_ns(parent, *bin_attr,
mode, uid, gid,
NULL);
if (error)
break;
}
if (error) remove_files(parent, grp);
}
exit:
return error;
}
static int internal_create_group(struct kobject *kobj, int update,
const struct attribute_group *grp)
{
struct kernfs_node *kn;
kuid_t uid;
kgid_t gid;
int error;
if (WARN_ON(!kobj || (!update && !kobj->sd)))
return -EINVAL;
/* Updates may happen before the object has been instantiated */
if (unlikely(update && !kobj->sd))
return -EINVAL;
if (!grp->attrs && !grp->bin_attrs) { pr_debug("sysfs: (bin_)attrs not set by subsystem for group: %s/%s, skipping\n",
kobj->name, grp->name ?: "");
return 0;
}
kobject_get_ownership(kobj, &uid, &gid);
if (grp->name) {
umode_t mode = __first_visible(grp, kobj); if (mode & SYSFS_GROUP_INVISIBLE)
mode = 0;
else
mode = S_IRWXU | S_IRUGO | S_IXUGO;
if (update) { kn = kernfs_find_and_get(kobj->sd, grp->name);
if (!kn) {
pr_debug("attr grp %s/%s not created yet\n",
kobj->name, grp->name);
/* may have been invisible prior to this update */
update = 0;
} else if (!mode) {
sysfs_remove_group(kobj, grp);
kernfs_put(kn);
return 0;
}
}
if (!update) {
if (!mode)
return 0;
kn = kernfs_create_dir_ns(kobj->sd, grp->name, mode,
uid, gid, kobj, NULL);
if (IS_ERR(kn)) {
if (PTR_ERR(kn) == -EEXIST) sysfs_warn_dup(kobj->sd, grp->name); return PTR_ERR(kn);
}
}
} else {
kn = kobj->sd;
}
kernfs_get(kn); error = create_files(kn, kobj, uid, gid, grp, update);
if (error) {
if (grp->name)
kernfs_remove(kn);
}
kernfs_put(kn); if (grp->name && update) kernfs_put(kn);
return error;
}
/**
* sysfs_create_group - given a directory kobject, create an attribute group
* @kobj: The kobject to create the group on
* @grp: The attribute group to create
*
* This function creates a group for the first time. It will explicitly
* warn and error if any of the attribute files being created already exist.
*
* Returns 0 on success or error code on failure.
*/
int sysfs_create_group(struct kobject *kobj,
const struct attribute_group *grp)
{
return internal_create_group(kobj, 0, grp);
}
EXPORT_SYMBOL_GPL(sysfs_create_group);
static int internal_create_groups(struct kobject *kobj, int update,
const struct attribute_group **groups)
{
int error = 0;
int i;
if (!groups) return 0; for (i = 0; groups[i]; i++) { error = internal_create_group(kobj, update, groups[i]);
if (error) {
while (--i >= 0) sysfs_remove_group(kobj, groups[i]);
break;
}
}
return error;
}
/**
* sysfs_create_groups - given a directory kobject, create a bunch of attribute groups
* @kobj: The kobject to create the group on
* @groups: The attribute groups to create, NULL terminated
*
* This function creates a bunch of attribute groups. If an error occurs when
* creating a group, all previously created groups will be removed, unwinding
* everything back to the original state when this function was called.
* It will explicitly warn and error if any of the attribute files being
* created already exist.
*
* Returns 0 on success or error code from sysfs_create_group on failure.
*/
int sysfs_create_groups(struct kobject *kobj,
const struct attribute_group **groups)
{
return internal_create_groups(kobj, 0, groups);
}
EXPORT_SYMBOL_GPL(sysfs_create_groups);
/**
* sysfs_update_groups - given a directory kobject, create a bunch of attribute groups
* @kobj: The kobject to update the group on
* @groups: The attribute groups to update, NULL terminated
*
* This function update a bunch of attribute groups. If an error occurs when
* updating a group, all previously updated groups will be removed together
* with already existing (not updated) attributes.
*
* Returns 0 on success or error code from sysfs_update_group on failure.
*/
int sysfs_update_groups(struct kobject *kobj,
const struct attribute_group **groups)
{
return internal_create_groups(kobj, 1, groups);
}
EXPORT_SYMBOL_GPL(sysfs_update_groups);
/**
* sysfs_update_group - given a directory kobject, update an attribute group
* @kobj: The kobject to update the group on
* @grp: The attribute group to update
*
* This function updates an attribute group. Unlike
* sysfs_create_group(), it will explicitly not warn or error if any
* of the attribute files being created already exist. Furthermore,
* if the visibility of the files has changed through the is_visible()
* callback, it will update the permissions and add or remove the
* relevant files. Changing a group's name (subdirectory name under
* kobj's directory in sysfs) is not allowed.
*
* The primary use for this function is to call it after making a change
* that affects group visibility.
*
* Returns 0 on success or error code on failure.
*/
int sysfs_update_group(struct kobject *kobj,
const struct attribute_group *grp)
{
return internal_create_group(kobj, 1, grp);
}
EXPORT_SYMBOL_GPL(sysfs_update_group);
/**
* sysfs_remove_group: remove a group from a kobject
* @kobj: kobject to remove the group from
* @grp: group to remove
*
* This function removes a group of attributes from a kobject. The attributes
* previously have to have been created for this group, otherwise it will fail.
*/
void sysfs_remove_group(struct kobject *kobj,
const struct attribute_group *grp)
{
struct kernfs_node *parent = kobj->sd;
struct kernfs_node *kn;
if (grp->name) {
kn = kernfs_find_and_get(parent, grp->name);
if (!kn) {
pr_debug("sysfs group '%s' not found for kobject '%s'\n",
grp->name, kobject_name(kobj));
return;
}
} else {
kn = parent;
kernfs_get(kn);
}
remove_files(kn, grp);
if (grp->name)
kernfs_remove(kn); kernfs_put(kn);
}
EXPORT_SYMBOL_GPL(sysfs_remove_group);
/**
* sysfs_remove_groups - remove a list of groups
*
* @kobj: The kobject for the groups to be removed from
* @groups: NULL terminated list of groups to be removed
*
* If groups is not NULL, remove the specified groups from the kobject.
*/
void sysfs_remove_groups(struct kobject *kobj,
const struct attribute_group **groups)
{
int i;
if (!groups)
return;
for (i = 0; groups[i]; i++) sysfs_remove_group(kobj, groups[i]);
}
EXPORT_SYMBOL_GPL(sysfs_remove_groups);
/**
* sysfs_merge_group - merge files into a pre-existing named attribute group.
* @kobj: The kobject containing the group.
* @grp: The files to create and the attribute group they belong to.
*
* This function returns an error if the group doesn't exist, the .name field is
* NULL or any of the files already exist in that group, in which case none of
* the new files are created.
*/
int sysfs_merge_group(struct kobject *kobj,
const struct attribute_group *grp)
{
struct kernfs_node *parent;
kuid_t uid;
kgid_t gid;
int error = 0;
struct attribute *const *attr;
int i;
parent = kernfs_find_and_get(kobj->sd, grp->name);
if (!parent)
return -ENOENT;
kobject_get_ownership(kobj, &uid, &gid); for ((i = 0, attr = grp->attrs); *attr && !error; (++i, ++attr)) error = sysfs_add_file_mode_ns(parent, *attr, (*attr)->mode,
uid, gid, NULL);
if (error) { while (--i >= 0) kernfs_remove_by_name(parent, (*--attr)->name);
}
kernfs_put(parent); return error;
}
EXPORT_SYMBOL_GPL(sysfs_merge_group);
/**
* sysfs_unmerge_group - remove files from a pre-existing named attribute group.
* @kobj: The kobject containing the group.
* @grp: The files to remove and the attribute group they belong to.
*/
void sysfs_unmerge_group(struct kobject *kobj,
const struct attribute_group *grp)
{
struct kernfs_node *parent;
struct attribute *const *attr;
parent = kernfs_find_and_get(kobj->sd, grp->name);
if (parent) {
for (attr = grp->attrs; *attr; ++attr) kernfs_remove_by_name(parent, (*attr)->name); kernfs_put(parent);
}
}
EXPORT_SYMBOL_GPL(sysfs_unmerge_group);
/**
* sysfs_add_link_to_group - add a symlink to an attribute group.
* @kobj: The kobject containing the group.
* @group_name: The name of the group.
* @target: The target kobject of the symlink to create.
* @link_name: The name of the symlink to create.
*/
int sysfs_add_link_to_group(struct kobject *kobj, const char *group_name,
struct kobject *target, const char *link_name)
{
struct kernfs_node *parent;
int error = 0;
parent = kernfs_find_and_get(kobj->sd, group_name);
if (!parent)
return -ENOENT;
error = sysfs_create_link_sd(parent, target, link_name);
kernfs_put(parent);
return error;
}
EXPORT_SYMBOL_GPL(sysfs_add_link_to_group);
/**
* sysfs_remove_link_from_group - remove a symlink from an attribute group.
* @kobj: The kobject containing the group.
* @group_name: The name of the group.
* @link_name: The name of the symlink to remove.
*/
void sysfs_remove_link_from_group(struct kobject *kobj, const char *group_name,
const char *link_name)
{
struct kernfs_node *parent;
parent = kernfs_find_and_get(kobj->sd, group_name);
if (parent) {
kernfs_remove_by_name(parent, link_name);
kernfs_put(parent);
}
}
EXPORT_SYMBOL_GPL(sysfs_remove_link_from_group);
/**
* compat_only_sysfs_link_entry_to_kobj - add a symlink to a kobject pointing
* to a group or an attribute
* @kobj: The kobject containing the group.
* @target_kobj: The target kobject.
* @target_name: The name of the target group or attribute.
* @symlink_name: The name of the symlink file (target_name will be
* considered if symlink_name is NULL).
*/
int compat_only_sysfs_link_entry_to_kobj(struct kobject *kobj,
struct kobject *target_kobj,
const char *target_name,
const char *symlink_name)
{
struct kernfs_node *target;
struct kernfs_node *entry;
struct kernfs_node *link;
/*
* We don't own @target_kobj and it may be removed at any time.
* Synchronize using sysfs_symlink_target_lock. See sysfs_remove_dir()
* for details.
*/
spin_lock(&sysfs_symlink_target_lock);
target = target_kobj->sd;
if (target)
kernfs_get(target);
spin_unlock(&sysfs_symlink_target_lock);
if (!target)
return -ENOENT;
entry = kernfs_find_and_get(target, target_name);
if (!entry) {
kernfs_put(target);
return -ENOENT;
}
if (!symlink_name)
symlink_name = target_name;
link = kernfs_create_link(kobj->sd, symlink_name, entry);
if (PTR_ERR(link) == -EEXIST)
sysfs_warn_dup(kobj->sd, symlink_name);
kernfs_put(entry);
kernfs_put(target);
return PTR_ERR_OR_ZERO(link);
}
EXPORT_SYMBOL_GPL(compat_only_sysfs_link_entry_to_kobj);
static int sysfs_group_attrs_change_owner(struct kernfs_node *grp_kn,
const struct attribute_group *grp,
struct iattr *newattrs)
{
struct kernfs_node *kn;
int error;
if (grp->attrs) {
struct attribute *const *attr;
for (attr = grp->attrs; *attr; attr++) {
kn = kernfs_find_and_get(grp_kn, (*attr)->name);
if (!kn)
return -ENOENT;
error = kernfs_setattr(kn, newattrs);
kernfs_put(kn);
if (error)
return error;
}
}
if (grp->bin_attrs) {
struct bin_attribute *const *bin_attr;
for (bin_attr = grp->bin_attrs; *bin_attr; bin_attr++) {
kn = kernfs_find_and_get(grp_kn, (*bin_attr)->attr.name);
if (!kn)
return -ENOENT;
error = kernfs_setattr(kn, newattrs);
kernfs_put(kn);
if (error)
return error;
}
}
return 0;
}
/**
* sysfs_group_change_owner - change owner of an attribute group.
* @kobj: The kobject containing the group.
* @grp: The attribute group.
* @kuid: new owner's kuid
* @kgid: new owner's kgid
*
* Returns 0 on success or error code on failure.
*/
int sysfs_group_change_owner(struct kobject *kobj,
const struct attribute_group *grp, kuid_t kuid,
kgid_t kgid)
{
struct kernfs_node *grp_kn;
int error;
struct iattr newattrs = {
.ia_valid = ATTR_UID | ATTR_GID,
.ia_uid = kuid,
.ia_gid = kgid,
};
if (!kobj->state_in_sysfs)
return -EINVAL;
if (grp->name) {
grp_kn = kernfs_find_and_get(kobj->sd, grp->name);
} else {
kernfs_get(kobj->sd);
grp_kn = kobj->sd;
}
if (!grp_kn)
return -ENOENT;
error = kernfs_setattr(grp_kn, &newattrs);
if (!error)
error = sysfs_group_attrs_change_owner(grp_kn, grp, &newattrs);
kernfs_put(grp_kn);
return error;
}
EXPORT_SYMBOL_GPL(sysfs_group_change_owner);
/**
* sysfs_groups_change_owner - change owner of a set of attribute groups.
* @kobj: The kobject containing the groups.
* @groups: The attribute groups.
* @kuid: new owner's kuid
* @kgid: new owner's kgid
*
* Returns 0 on success or error code on failure.
*/
int sysfs_groups_change_owner(struct kobject *kobj,
const struct attribute_group **groups,
kuid_t kuid, kgid_t kgid)
{
int error = 0, i;
if (!kobj->state_in_sysfs)
return -EINVAL;
if (!groups)
return 0;
for (i = 0; groups[i]; i++) {
error = sysfs_group_change_owner(kobj, groups[i], kuid, kgid);
if (error)
break;
}
return error;
}
EXPORT_SYMBOL_GPL(sysfs_groups_change_owner);
/*
* Parallel-port resource manager code.
*
* Authors: David Campbell <campbell@tirian.che.curtin.edu.au>
* Tim Waugh <tim@cyberelk.demon.co.uk>
* Jose Renau <renau@acm.org>
* Philip Blundell <philb@gnu.org>
* Andrea Arcangeli
*
* based on work by Grant Guenther <grant@torque.net>
* and Philip Blundell
*
* Any part of this program may be used in documents licensed under
* the GNU Free Documentation License, Version 1.1 or any later version
* published by the Free Software Foundation.
*/
#undef PARPORT_DEBUG_SHARING /* undef for production */
#include <linux/module.h>
#include <linux/string.h>
#include <linux/threads.h>
#include <linux/parport.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/kmod.h>
#include <linux/device.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <asm/irq.h>
#undef PARPORT_PARANOID
#define PARPORT_DEFAULT_TIMESLICE (HZ/5)
unsigned long parport_default_timeslice = PARPORT_DEFAULT_TIMESLICE;
int parport_default_spintime = DEFAULT_SPIN_TIME;
static LIST_HEAD(portlist);
static DEFINE_SPINLOCK(parportlist_lock);
/* list of all allocated ports, sorted by ->number */
static LIST_HEAD(all_ports);
static DEFINE_SPINLOCK(full_list_lock);
static LIST_HEAD(drivers);
static DEFINE_MUTEX(registration_lock);
/* What you can do to a port that's gone away.. */
static void dead_write_lines(struct parport *p, unsigned char b){}
static unsigned char dead_read_lines(struct parport *p) { return 0; }
static unsigned char dead_frob_lines(struct parport *p, unsigned char b,
unsigned char c) { return 0; }
static void dead_onearg(struct parport *p){}
static void dead_initstate(struct pardevice *d, struct parport_state *s) { }
static void dead_state(struct parport *p, struct parport_state *s) { }
static size_t dead_write(struct parport *p, const void *b, size_t l, int f)
{ return 0; }
static size_t dead_read(struct parport *p, void *b, size_t l, int f)
{ return 0; }
static struct parport_operations dead_ops = {
.write_data = dead_write_lines, /* data */
.read_data = dead_read_lines,
.write_control = dead_write_lines, /* control */
.read_control = dead_read_lines,
.frob_control = dead_frob_lines,
.read_status = dead_read_lines, /* status */
.enable_irq = dead_onearg, /* enable_irq */
.disable_irq = dead_onearg, /* disable_irq */
.data_forward = dead_onearg, /* data_forward */
.data_reverse = dead_onearg, /* data_reverse */
.init_state = dead_initstate, /* init_state */
.save_state = dead_state,
.restore_state = dead_state,
.epp_write_data = dead_write, /* epp */
.epp_read_data = dead_read,
.epp_write_addr = dead_write,
.epp_read_addr = dead_read,
.ecp_write_data = dead_write, /* ecp */
.ecp_read_data = dead_read,
.ecp_write_addr = dead_write,
.compat_write_data = dead_write, /* compat */
.nibble_read_data = dead_read, /* nibble */
.byte_read_data = dead_read, /* byte */
.owner = NULL,
};
static struct device_type parport_device_type = {
.name = "parport",
};
static int is_parport(struct device *dev)
{
return dev->type == &parport_device_type;
}
static int parport_probe(struct device *dev)
{
struct parport_driver *drv;
if (is_parport(dev))
return -ENODEV;
drv = to_parport_driver(dev->driver);
if (!drv->probe) {
/* if driver has not defined a custom probe */
struct pardevice *par_dev = to_pardevice(dev);
if (strcmp(par_dev->name, drv->name))
return -ENODEV;
return 0;
}
/* if driver defined its own probe */
return drv->probe(to_pardevice(dev));
}
static struct bus_type parport_bus_type = {
.name = "parport",
.probe = parport_probe,
};
int parport_bus_init(void)
{
return bus_register(&parport_bus_type);
}
void parport_bus_exit(void)
{
bus_unregister(&parport_bus_type);
}
/*
* iterates through all the drivers registered with the bus and sends the port
* details to the match_port callback of the driver, so that the driver can
* know about the new port that just registered with the bus and decide if it
* wants to use this new port.
*/
static int driver_check(struct device_driver *dev_drv, void *_port)
{
struct parport *port = _port;
struct parport_driver *drv = to_parport_driver(dev_drv);
if (drv->match_port)
drv->match_port(port);
return 0;
}
/* Call attach(port) for each registered driver. */
static void attach_driver_chain(struct parport *port)
{
/* caller has exclusive registration_lock */
struct parport_driver *drv;
list_for_each_entry(drv, &drivers, list)
drv->attach(port);
/*
* call the driver_check function of the drivers registered in
* new device model
*/
bus_for_each_drv(&parport_bus_type, NULL, port, driver_check);
}
static int driver_detach(struct device_driver *_drv, void *_port)
{
struct parport *port = _port;
struct parport_driver *drv = to_parport_driver(_drv);
if (drv->detach)
drv->detach(port);
return 0;
}
/* Call detach(port) for each registered driver. */
static void detach_driver_chain(struct parport *port)
{
struct parport_driver *drv;
/* caller has exclusive registration_lock */
list_for_each_entry(drv, &drivers, list)
drv->detach(port);
/*
* call the detach function of the drivers registered in
* new device model
*/
bus_for_each_drv(&parport_bus_type, NULL, port, driver_detach);
}
/* Ask kmod for some lowlevel drivers. */
static void get_lowlevel_driver(void)
{
/*
* There is no actual module called this: you should set
* up an alias for modutils.
*/
request_module("parport_lowlevel");
}
/*
* iterates through all the devices connected to the bus and sends the device
* details to the match_port callback of the driver, so that the driver can
* know what are all the ports that are connected to the bus and choose the
* port to which it wants to register its device.
*/
static int port_check(struct device *dev, void *dev_drv)
{
struct parport_driver *drv = dev_drv;
/* only send ports, do not send other devices connected to bus */
if (is_parport(dev))
drv->match_port(to_parport_dev(dev));
return 0;
}
/*
* Iterates through all the devices connected to the bus and return 1
* if the device is a parallel port.
*/
static int port_detect(struct device *dev, void *dev_drv)
{
if (is_parport(dev))
return 1;
return 0;
}
/**
* __parport_register_driver - register a parallel port device driver
* @drv: structure describing the driver
* @owner: owner module of drv
* @mod_name: module name string
*
* This can be called by a parallel port device driver in order
* to receive notifications about ports being found in the
* system, as well as ports no longer available.
*
* If devmodel is true then the new device model is used
* for registration.
*
* The @drv structure is allocated by the caller and must not be
* deallocated until after calling parport_unregister_driver().
*
* If using the non device model:
* The driver's attach() function may block. The port that
* attach() is given will be valid for the duration of the
* callback, but if the driver wants to take a copy of the
* pointer it must call parport_get_port() to do so. Calling
* parport_register_device() on that port will do this for you.
*
* The driver's detach() function may block. The port that
* detach() is given will be valid for the duration of the
* callback, but if the driver wants to take a copy of the
* pointer it must call parport_get_port() to do so.
*
*
* Returns 0 on success. The non device model will always succeeds.
* but the new device model can fail and will return the error code.
**/
int __parport_register_driver(struct parport_driver *drv, struct module *owner,
const char *mod_name)
{
/* using device model */
int ret;
/* initialize common driver fields */
drv->driver.name = drv->name;
drv->driver.bus = &parport_bus_type;
drv->driver.owner = owner;
drv->driver.mod_name = mod_name;
ret = driver_register(&drv->driver);
if (ret)
return ret;
/*
* check if bus has any parallel port registered, if
* none is found then load the lowlevel driver.
*/
ret = bus_for_each_dev(&parport_bus_type, NULL, NULL,
port_detect);
if (!ret)
get_lowlevel_driver();
mutex_lock(®istration_lock);
if (drv->match_port)
bus_for_each_dev(&parport_bus_type, NULL, drv,
port_check);
mutex_unlock(®istration_lock);
return 0;
}
EXPORT_SYMBOL(__parport_register_driver);
static int port_detach(struct device *dev, void *_drv)
{
struct parport_driver *drv = _drv;
if (is_parport(dev) && drv->detach)
drv->detach(to_parport_dev(dev));
return 0;
}
/**
* parport_unregister_driver - deregister a parallel port device driver
* @drv: structure describing the driver that was given to
* parport_register_driver()
*
* This should be called by a parallel port device driver that
* has registered itself using parport_register_driver() when it
* is about to be unloaded.
*
* When it returns, the driver's attach() routine will no longer
* be called, and for each port that attach() was called for, the
* detach() routine will have been called.
*
* All the driver's attach() and detach() calls are guaranteed to have
* finished by the time this function returns.
**/
void parport_unregister_driver(struct parport_driver *drv)
{
mutex_lock(®istration_lock);
bus_for_each_dev(&parport_bus_type, NULL, drv, port_detach);
driver_unregister(&drv->driver);
mutex_unlock(®istration_lock);
}
EXPORT_SYMBOL(parport_unregister_driver);
static void free_port(struct device *dev)
{
int d;
struct parport *port = to_parport_dev(dev);
spin_lock(&full_list_lock);
list_del(&port->full_list);
spin_unlock(&full_list_lock);
for (d = 0; d < 5; d++) {
kfree(port->probe_info[d].class_name);
kfree(port->probe_info[d].mfr);
kfree(port->probe_info[d].model);
kfree(port->probe_info[d].cmdset);
kfree(port->probe_info[d].description);
}
kfree(port);
}
/**
* parport_get_port - increment a port's reference count
* @port: the port
*
* This ensures that a struct parport pointer remains valid
* until the matching parport_put_port() call.
**/
struct parport *parport_get_port(struct parport *port)
{
struct device *dev = get_device(&port->bus_dev);
return to_parport_dev(dev);
}
EXPORT_SYMBOL(parport_get_port);
void parport_del_port(struct parport *port)
{
device_unregister(&port->bus_dev);
}
EXPORT_SYMBOL(parport_del_port);
/**
* parport_put_port - decrement a port's reference count
* @port: the port
*
* This should be called once for each call to parport_get_port(),
* once the port is no longer needed. When the reference count reaches
* zero (port is no longer used), free_port is called.
**/
void parport_put_port(struct parport *port)
{
put_device(&port->bus_dev);
}
EXPORT_SYMBOL(parport_put_port);
/**
* parport_register_port - register a parallel port
* @base: base I/O address
* @irq: IRQ line
* @dma: DMA channel
* @ops: pointer to the port driver's port operations structure
*
* When a parallel port (lowlevel) driver finds a port that
* should be made available to parallel port device drivers, it
* should call parport_register_port(). The @base, @irq, and
* @dma parameters are for the convenience of port drivers, and
* for ports where they aren't meaningful needn't be set to
* anything special. They can be altered afterwards by adjusting
* the relevant members of the parport structure that is returned
* and represents the port. They should not be tampered with
* after calling parport_announce_port, however.
*
* If there are parallel port device drivers in the system that
* have registered themselves using parport_register_driver(),
* they are not told about the port at this time; that is done by
* parport_announce_port().
*
* The @ops structure is allocated by the caller, and must not be
* deallocated before calling parport_remove_port().
*
* If there is no memory to allocate a new parport structure,
* this function will return %NULL.
**/
struct parport *parport_register_port(unsigned long base, int irq, int dma,
struct parport_operations *ops)
{
struct list_head *l;
struct parport *tmp;
int num;
int device;
int ret;
tmp = kzalloc(sizeof(struct parport), GFP_KERNEL);
if (!tmp)
return NULL;
/* Init our structure */
tmp->base = base;
tmp->irq = irq;
tmp->dma = dma;
tmp->muxport = tmp->daisy = tmp->muxsel = -1;
INIT_LIST_HEAD(&tmp->list);
tmp->ops = ops;
tmp->physport = tmp;
rwlock_init(&tmp->cad_lock);
spin_lock_init(&tmp->waitlist_lock);
spin_lock_init(&tmp->pardevice_lock);
tmp->ieee1284.mode = IEEE1284_MODE_COMPAT;
tmp->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
sema_init(&tmp->ieee1284.irq, 0);
tmp->spintime = parport_default_spintime;
atomic_set(&tmp->ref_count, 1);
/* Search for the lowest free parport number. */
spin_lock(&full_list_lock);
num = 0;
list_for_each(l, &all_ports) {
struct parport *p = list_entry(l, struct parport, full_list);
if (p->number != num++)
break;
}
tmp->portnum = tmp->number = num;
list_add_tail(&tmp->full_list, l);
spin_unlock(&full_list_lock);
/*
* Now that the portnum is known finish doing the Init.
*/
dev_set_name(&tmp->bus_dev, "parport%d", tmp->portnum);
tmp->bus_dev.bus = &parport_bus_type;
tmp->bus_dev.release = free_port;
tmp->bus_dev.type = &parport_device_type;
tmp->name = dev_name(&tmp->bus_dev);
for (device = 0; device < 5; device++)
/* assume the worst */
tmp->probe_info[device].class = PARPORT_CLASS_LEGACY;
ret = device_register(&tmp->bus_dev);
if (ret) {
put_device(&tmp->bus_dev);
return NULL;
}
return tmp;
}
EXPORT_SYMBOL(parport_register_port);
/**
* parport_announce_port - tell device drivers about a parallel port
* @port: parallel port to announce
*
* After a port driver has registered a parallel port with
* parport_register_port, and performed any necessary
* initialisation or adjustments, it should call
* parport_announce_port() in order to notify all device drivers
* that have called parport_register_driver(). Their attach()
* functions will be called, with @port as the parameter.
**/
void parport_announce_port(struct parport *port)
{
int i;
#ifdef CONFIG_PARPORT_1284
/* Analyse the IEEE1284.3 topology of the port. */
parport_daisy_init(port);
#endif
if (!port->dev)
pr_warn("%s: fix this legacy no-device port driver!\n",
port->name);
parport_proc_register(port);
mutex_lock(®istration_lock);
spin_lock_irq(&parportlist_lock);
list_add_tail(&port->list, &portlist);
for (i = 1; i < 3; i++) {
struct parport *slave = port->slaves[i-1];
if (slave)
list_add_tail(&slave->list, &portlist);
}
spin_unlock_irq(&parportlist_lock);
/* Let drivers know that new port(s) has arrived. */
attach_driver_chain(port);
for (i = 1; i < 3; i++) {
struct parport *slave = port->slaves[i-1];
if (slave)
attach_driver_chain(slave);
}
mutex_unlock(®istration_lock);
}
EXPORT_SYMBOL(parport_announce_port);
/**
* parport_remove_port - deregister a parallel port
* @port: parallel port to deregister
*
* When a parallel port driver is forcibly unloaded, or a
* parallel port becomes inaccessible, the port driver must call
* this function in order to deal with device drivers that still
* want to use it.
*
* The parport structure associated with the port has its
* operations structure replaced with one containing 'null'
* operations that return errors or just don't do anything.
*
* Any drivers that have registered themselves using
* parport_register_driver() are notified that the port is no
* longer accessible by having their detach() routines called
* with @port as the parameter.
**/
void parport_remove_port(struct parport *port)
{
int i;
mutex_lock(®istration_lock);
/* Spread the word. */
detach_driver_chain(port);
#ifdef CONFIG_PARPORT_1284
/* Forget the IEEE1284.3 topology of the port. */
parport_daisy_fini(port);
for (i = 1; i < 3; i++) {
struct parport *slave = port->slaves[i-1];
if (!slave)
continue;
detach_driver_chain(slave);
parport_daisy_fini(slave);
}
#endif
port->ops = &dead_ops;
spin_lock(&parportlist_lock);
list_del_init(&port->list);
for (i = 1; i < 3; i++) {
struct parport *slave = port->slaves[i-1];
if (slave)
list_del_init(&slave->list);
}
spin_unlock(&parportlist_lock);
mutex_unlock(®istration_lock);
parport_proc_unregister(port);
for (i = 1; i < 3; i++) {
struct parport *slave = port->slaves[i-1];
if (slave)
parport_put_port(slave);
}
}
EXPORT_SYMBOL(parport_remove_port);
static void free_pardevice(struct device *dev)
{
struct pardevice *par_dev = to_pardevice(dev);
kfree_const(par_dev->name);
kfree(par_dev);
}
/**
* parport_register_dev_model - register a device on a parallel port
* @port: port to which the device is attached
* @name: a name to refer to the device
* @par_dev_cb: struct containing callbacks
* @id: device number to be given to the device
*
* This function, called by parallel port device drivers,
* declares that a device is connected to a port, and tells the
* system all it needs to know.
*
* The struct pardev_cb contains pointer to callbacks. preemption
* callback function, @preempt, is called when this device driver
* has claimed access to the port but another device driver wants
* to use it. It is given, @private, as its parameter, and should
* return zero if it is willing for the system to release the port
* to another driver on its behalf. If it wants to keep control of
* the port it should return non-zero, and no action will be taken.
* It is good manners for the driver to try to release the port at
* the earliest opportunity after its preemption callback rejects a
* preemption attempt. Note that if a preemption callback is happy
* for preemption to go ahead, there is no need to release the
* port; it is done automatically. This function may not block, as
* it may be called from interrupt context. If the device driver
* does not support preemption, @preempt can be %NULL.
*
* The wake-up ("kick") callback function, @wakeup, is called when
* the port is available to be claimed for exclusive access; that
* is, parport_claim() is guaranteed to succeed when called from
* inside the wake-up callback function. If the driver wants to
* claim the port it should do so; otherwise, it need not take
* any action. This function may not block, as it may be called
* from interrupt context. If the device driver does not want to
* be explicitly invited to claim the port in this way, @wakeup can
* be %NULL.
*
* The interrupt handler, @irq_func, is called when an interrupt
* arrives from the parallel port. Note that if a device driver
* wants to use interrupts it should use parport_enable_irq(),
* and can also check the irq member of the parport structure
* representing the port.
*
* The parallel port (lowlevel) driver is the one that has called
* request_irq() and whose interrupt handler is called first.
* This handler does whatever needs to be done to the hardware to
* acknowledge the interrupt (for PC-style ports there is nothing
* special to be done). It then tells the IEEE 1284 code about
* the interrupt, which may involve reacting to an IEEE 1284
* event depending on the current IEEE 1284 phase. After this,
* it calls @irq_func. Needless to say, @irq_func will be called
* from interrupt context, and may not block.
*
* The %PARPORT_DEV_EXCL flag is for preventing port sharing, and
* so should only be used when sharing the port with other device
* drivers is impossible and would lead to incorrect behaviour.
* Use it sparingly! Normally, @flags will be zero.
*
* This function returns a pointer to a structure that represents
* the device on the port, or %NULL if there is not enough memory
* to allocate space for that structure.
**/
struct pardevice *
parport_register_dev_model(struct parport *port, const char *name,
const struct pardev_cb *par_dev_cb, int id)
{
struct pardevice *par_dev;
const char *devname;
int ret;
if (port->physport->flags & PARPORT_FLAG_EXCL) {
/* An exclusive device is registered. */
pr_err("%s: no more devices allowed\n", port->name);
return NULL;
}
if (par_dev_cb->flags & PARPORT_DEV_LURK) {
if (!par_dev_cb->preempt || !par_dev_cb->wakeup) {
pr_info("%s: refused to register lurking device (%s) without callbacks\n",
port->name, name);
return NULL;
}
}
if (par_dev_cb->flags & PARPORT_DEV_EXCL) {
if (port->physport->devices) {
/*
* If a device is already registered and this new
* device wants exclusive access, then no need to
* continue as we can not grant exclusive access to
* this device.
*/
pr_err("%s: cannot grant exclusive access for device %s\n",
port->name, name);
return NULL;
}
}
if (!try_module_get(port->ops->owner))
return NULL;
parport_get_port(port);
par_dev = kzalloc(sizeof(*par_dev), GFP_KERNEL);
if (!par_dev)
goto err_put_port;
par_dev->state = kzalloc(sizeof(*par_dev->state), GFP_KERNEL);
if (!par_dev->state)
goto err_put_par_dev;
devname = kstrdup_const(name, GFP_KERNEL);
if (!devname)
goto err_free_par_dev;
par_dev->name = devname;
par_dev->port = port;
par_dev->daisy = -1;
par_dev->preempt = par_dev_cb->preempt;
par_dev->wakeup = par_dev_cb->wakeup;
par_dev->private = par_dev_cb->private;
par_dev->flags = par_dev_cb->flags;
par_dev->irq_func = par_dev_cb->irq_func;
par_dev->waiting = 0;
par_dev->timeout = 5 * HZ;
par_dev->dev.parent = &port->bus_dev;
par_dev->dev.bus = &parport_bus_type;
ret = dev_set_name(&par_dev->dev, "%s.%d", devname, id);
if (ret)
goto err_free_devname;
par_dev->dev.release = free_pardevice;
par_dev->devmodel = true;
ret = device_register(&par_dev->dev);
if (ret) {
kfree(par_dev->state);
put_device(&par_dev->dev);
goto err_put_port;
}
/* Chain this onto the list */
par_dev->prev = NULL;
/*
* This function must not run from an irq handler so we don' t need
* to clear irq on the local CPU. -arca
*/
spin_lock(&port->physport->pardevice_lock);
if (par_dev_cb->flags & PARPORT_DEV_EXCL) {
if (port->physport->devices) {
spin_unlock(&port->physport->pardevice_lock);
pr_debug("%s: cannot grant exclusive access for device %s\n",
port->name, name);
kfree(par_dev->state);
device_unregister(&par_dev->dev);
goto err_put_port;
}
port->flags |= PARPORT_FLAG_EXCL;
}
par_dev->next = port->physport->devices;
wmb(); /*
* Make sure that tmp->next is written before it's
* added to the list; see comments marked 'no locking
* required'
*/
if (port->physport->devices)
port->physport->devices->prev = par_dev;
port->physport->devices = par_dev;
spin_unlock(&port->physport->pardevice_lock);
init_waitqueue_head(&par_dev->wait_q);
par_dev->timeslice = parport_default_timeslice;
par_dev->waitnext = NULL;
par_dev->waitprev = NULL;
/*
* This has to be run as last thing since init_state may need other
* pardevice fields. -arca
*/
port->ops->init_state(par_dev, par_dev->state);
if (!test_and_set_bit(PARPORT_DEVPROC_REGISTERED, &port->devflags)) {
port->proc_device = par_dev;
parport_device_proc_register(par_dev);
}
return par_dev;
err_free_devname:
kfree_const(devname);
err_free_par_dev:
kfree(par_dev->state);
err_put_par_dev:
if (!par_dev->devmodel)
kfree(par_dev);
err_put_port:
parport_put_port(port);
module_put(port->ops->owner);
return NULL;
}
EXPORT_SYMBOL(parport_register_dev_model);
/**
* parport_unregister_device - deregister a device on a parallel port
* @dev: pointer to structure representing device
*
* This undoes the effect of parport_register_device().
**/
void parport_unregister_device(struct pardevice *dev)
{
struct parport *port;
#ifdef PARPORT_PARANOID
if (!dev) {
pr_err("%s: passed NULL\n", __func__);
return;
}
#endif
port = dev->port->physport;
if (port->proc_device == dev) {
port->proc_device = NULL;
clear_bit(PARPORT_DEVPROC_REGISTERED, &port->devflags);
parport_device_proc_unregister(dev);
}
if (port->cad == dev) {
printk(KERN_DEBUG "%s: %s forgot to release port\n",
port->name, dev->name);
parport_release(dev);
}
spin_lock(&port->pardevice_lock);
if (dev->next)
dev->next->prev = dev->prev;
if (dev->prev)
dev->prev->next = dev->next;
else
port->devices = dev->next;
if (dev->flags & PARPORT_DEV_EXCL)
port->flags &= ~PARPORT_FLAG_EXCL;
spin_unlock(&port->pardevice_lock);
/*
* Make sure we haven't left any pointers around in the wait
* list.
*/
spin_lock_irq(&port->waitlist_lock);
if (dev->waitprev || dev->waitnext || port->waithead == dev) {
if (dev->waitprev)
dev->waitprev->waitnext = dev->waitnext;
else
port->waithead = dev->waitnext;
if (dev->waitnext)
dev->waitnext->waitprev = dev->waitprev;
else
port->waittail = dev->waitprev;
}
spin_unlock_irq(&port->waitlist_lock);
kfree(dev->state);
device_unregister(&dev->dev);
module_put(port->ops->owner);
parport_put_port(port);
}
EXPORT_SYMBOL(parport_unregister_device);
/**
* parport_find_number - find a parallel port by number
* @number: parallel port number
*
* This returns the parallel port with the specified number, or
* %NULL if there is none.
*
* There is an implicit parport_get_port() done already; to throw
* away the reference to the port that parport_find_number()
* gives you, use parport_put_port().
*/
struct parport *parport_find_number(int number)
{
struct parport *port, *result = NULL;
if (list_empty(&portlist))
get_lowlevel_driver();
spin_lock(&parportlist_lock);
list_for_each_entry(port, &portlist, list) {
if (port->number == number) {
result = parport_get_port(port);
break;
}
}
spin_unlock(&parportlist_lock);
return result;
}
EXPORT_SYMBOL(parport_find_number);
/**
* parport_find_base - find a parallel port by base address
* @base: base I/O address
*
* This returns the parallel port with the specified base
* address, or %NULL if there is none.
*
* There is an implicit parport_get_port() done already; to throw
* away the reference to the port that parport_find_base()
* gives you, use parport_put_port().
*/
struct parport *parport_find_base(unsigned long base)
{
struct parport *port, *result = NULL;
if (list_empty(&portlist))
get_lowlevel_driver();
spin_lock(&parportlist_lock);
list_for_each_entry(port, &portlist, list) {
if (port->base == base) {
result = parport_get_port(port);
break;
}
}
spin_unlock(&parportlist_lock);
return result;
}
EXPORT_SYMBOL(parport_find_base);
/**
* parport_claim - claim access to a parallel port device
* @dev: pointer to structure representing a device on the port
*
* This function will not block and so can be used from interrupt
* context. If parport_claim() succeeds in claiming access to
* the port it returns zero and the port is available to use. It
* may fail (returning non-zero) if the port is in use by another
* driver and that driver is not willing to relinquish control of
* the port.
**/
int parport_claim(struct pardevice *dev)
{
struct pardevice *oldcad;
struct parport *port = dev->port->physport;
unsigned long flags;
if (port->cad == dev) {
pr_info("%s: %s already owner\n", dev->port->name, dev->name);
return 0;
}
/* Preempt any current device */
write_lock_irqsave(&port->cad_lock, flags);
oldcad = port->cad;
if (oldcad) {
if (oldcad->preempt) {
if (oldcad->preempt(oldcad->private))
goto blocked;
port->ops->save_state(port, dev->state);
} else
goto blocked;
if (port->cad != oldcad) {
/*
* I think we'll actually deadlock rather than
* get here, but just in case..
*/
pr_warn("%s: %s released port when preempted!\n",
port->name, oldcad->name);
if (port->cad)
goto blocked;
}
}
/* Can't fail from now on, so mark ourselves as no longer waiting. */
if (dev->waiting & 1) {
dev->waiting = 0;
/* Take ourselves out of the wait list again. */
spin_lock_irq(&port->waitlist_lock);
if (dev->waitprev)
dev->waitprev->waitnext = dev->waitnext;
else
port->waithead = dev->waitnext;
if (dev->waitnext)
dev->waitnext->waitprev = dev->waitprev;
else
port->waittail = dev->waitprev;
spin_unlock_irq(&port->waitlist_lock);
dev->waitprev = dev->waitnext = NULL;
}
/* Now we do the change of devices */
port->cad = dev;
#ifdef CONFIG_PARPORT_1284
/* If it's a mux port, select it. */
if (dev->port->muxport >= 0) {
/* FIXME */
port->muxsel = dev->port->muxport;
}
/* If it's a daisy chain device, select it. */
if (dev->daisy >= 0) {
/* This could be lazier. */
if (!parport_daisy_select(port, dev->daisy,
IEEE1284_MODE_COMPAT))
port->daisy = dev->daisy;
}
#endif /* IEEE1284.3 support */
/* Restore control registers */
port->ops->restore_state(port, dev->state);
write_unlock_irqrestore(&port->cad_lock, flags);
dev->time = jiffies;
return 0;
blocked:
/*
* If this is the first time we tried to claim the port, register an
* interest. This is only allowed for devices sleeping in
* parport_claim_or_block(), or those with a wakeup function.
*/
/* The cad_lock is still held for writing here */
if (dev->waiting & 2 || dev->wakeup) {
spin_lock(&port->waitlist_lock);
if (test_and_set_bit(0, &dev->waiting) == 0) {
/* First add ourselves to the end of the wait list. */
dev->waitnext = NULL;
dev->waitprev = port->waittail;
if (port->waittail) {
port->waittail->waitnext = dev;
port->waittail = dev;
} else
port->waithead = port->waittail = dev;
}
spin_unlock(&port->waitlist_lock);
}
write_unlock_irqrestore(&port->cad_lock, flags);
return -EAGAIN;
}
EXPORT_SYMBOL(parport_claim);
/**
* parport_claim_or_block - claim access to a parallel port device
* @dev: pointer to structure representing a device on the port
*
* This behaves like parport_claim(), but will block if necessary
* to wait for the port to be free. A return value of 1
* indicates that it slept; 0 means that it succeeded without
* needing to sleep. A negative error code indicates failure.
**/
int parport_claim_or_block(struct pardevice *dev)
{
int r;
/*
* Signal to parport_claim() that we can wait even without a
* wakeup function.
*/
dev->waiting = 2;
/* Try to claim the port. If this fails, we need to sleep. */
r = parport_claim(dev);
if (r == -EAGAIN) {
#ifdef PARPORT_DEBUG_SHARING
printk(KERN_DEBUG "%s: parport_claim() returned -EAGAIN\n",
dev->name);
#endif
/*
* FIXME!!! Use the proper locking for dev->waiting,
* and make this use the "wait_event_interruptible()"
* interfaces. The cli/sti that used to be here
* did nothing.
*
* See also parport_release()
*/
/*
* If dev->waiting is clear now, an interrupt
* gave us the port and we would deadlock if we slept.
*/
if (dev->waiting) {
wait_event_interruptible(dev->wait_q,
!dev->waiting);
if (signal_pending(current))
return -EINTR;
r = 1;
} else {
r = 0;
#ifdef PARPORT_DEBUG_SHARING
printk(KERN_DEBUG "%s: didn't sleep in parport_claim_or_block()\n",
dev->name);
#endif
}
#ifdef PARPORT_DEBUG_SHARING
if (dev->port->physport->cad != dev)
printk(KERN_DEBUG "%s: exiting parport_claim_or_block but %s owns port!\n",
dev->name, dev->port->physport->cad ?
dev->port->physport->cad->name : "nobody");
#endif
}
dev->waiting = 0;
return r;
}
EXPORT_SYMBOL(parport_claim_or_block);
/**
* parport_release - give up access to a parallel port device
* @dev: pointer to structure representing parallel port device
*
* This function cannot fail, but it should not be called without
* the port claimed. Similarly, if the port is already claimed
* you should not try claiming it again.
**/
void parport_release(struct pardevice *dev)
{
struct parport *port = dev->port->physport;
struct pardevice *pd;
unsigned long flags;
/* Make sure that dev is the current device */
write_lock_irqsave(&port->cad_lock, flags);
if (port->cad != dev) {
write_unlock_irqrestore(&port->cad_lock, flags);
pr_warn("%s: %s tried to release parport when not owner\n",
port->name, dev->name);
return;
}
#ifdef CONFIG_PARPORT_1284
/* If this is on a mux port, deselect it. */
if (dev->port->muxport >= 0) {
/* FIXME */
port->muxsel = -1;
}
/* If this is a daisy device, deselect it. */
if (dev->daisy >= 0) {
parport_daisy_deselect_all(port);
port->daisy = -1;
}
#endif
port->cad = NULL;
write_unlock_irqrestore(&port->cad_lock, flags);
/* Save control registers */
port->ops->save_state(port, dev->state);
/*
* If anybody is waiting, find out who's been there longest and
* then wake them up. (Note: no locking required)
*/
/* !!! LOCKING IS NEEDED HERE */
for (pd = port->waithead; pd; pd = pd->waitnext) {
if (pd->waiting & 2) { /* sleeping in claim_or_block */
parport_claim(pd);
if (waitqueue_active(&pd->wait_q))
wake_up_interruptible(&pd->wait_q);
return;
} else if (pd->wakeup) {
pd->wakeup(pd->private);
if (dev->port->cad) /* racy but no matter */
return;
} else {
pr_err("%s: don't know how to wake %s\n",
port->name, pd->name);
}
}
/*
* Nobody was waiting, so walk the list to see if anyone is
* interested in being woken up. (Note: no locking required)
*/
/* !!! LOCKING IS NEEDED HERE */
for (pd = port->devices; !port->cad && pd; pd = pd->next) {
if (pd->wakeup && pd != dev)
pd->wakeup(pd->private);
}
}
EXPORT_SYMBOL(parport_release);
irqreturn_t parport_irq_handler(int irq, void *dev_id)
{
struct parport *port = dev_id;
parport_generic_irq(port);
return IRQ_HANDLED;
}
EXPORT_SYMBOL(parport_irq_handler);
MODULE_LICENSE("GPL");
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
/*
* 'tty_io.c' gives an orthogonal feeling to tty's, be they consoles
* or rs-channels. It also implements echoing, cooked mode etc.
*
* Kill-line thanks to John T Kohl, who also corrected VMIN = VTIME = 0.
*
* Modified by Theodore Ts'o, 9/14/92, to dynamically allocate the
* tty_struct and tty_queue structures. Previously there was an array
* of 256 tty_struct's which was statically allocated, and the
* tty_queue structures were allocated at boot time. Both are now
* dynamically allocated only when the tty is open.
*
* Also restructured routines so that there is more of a separation
* between the high-level tty routines (tty_io.c and tty_ioctl.c) and
* the low-level tty routines (serial.c, pty.c, console.c). This
* makes for cleaner and more compact code. -TYT, 9/17/92
*
* Modified by Fred N. van Kempen, 01/29/93, to add line disciplines
* which can be dynamically activated and de-activated by the line
* discipline handling modules (like SLIP).
*
* NOTE: pay no attention to the line discipline code (yet); its
* interface is still subject to change in this version...
* -- TYT, 1/31/92
*
* Added functionality to the OPOST tty handling. No delays, but all
* other bits should be there.
* -- Nick Holloway <alfie@dcs.warwick.ac.uk>, 27th May 1993.
*
* Rewrote canonical mode and added more termios flags.
* -- julian@uhunix.uhcc.hawaii.edu (J. Cowley), 13Jan94
*
* Reorganized FASYNC support so mouse code can share it.
* -- ctm@ardi.com, 9Sep95
*
* New TIOCLINUX variants added.
* -- mj@k332.feld.cvut.cz, 19-Nov-95
*
* Restrict vt switching via ioctl()
* -- grif@cs.ucr.edu, 5-Dec-95
*
* Move console and virtual terminal code to more appropriate files,
* implement CONFIG_VT and generalize console device interface.
* -- Marko Kohtala <Marko.Kohtala@hut.fi>, March 97
*
* Rewrote tty_init_dev and tty_release_dev to eliminate races.
* -- Bill Hawes <whawes@star.net>, June 97
*
* Added devfs support.
* -- C. Scott Ananian <cananian@alumni.princeton.edu>, 13-Jan-1998
*
* Added support for a Unix98-style ptmx device.
* -- C. Scott Ananian <cananian@alumni.princeton.edu>, 14-Jan-1998
*
* Reduced memory usage for older ARM systems
* -- Russell King <rmk@arm.linux.org.uk>
*
* Move do_SAK() into process context. Less stack use in devfs functions.
* alloc_tty_struct() always uses kmalloc()
* -- Andrew Morton <andrewm@uow.edu.eu> 17Mar01
*/
#include <linux/types.h>
#include <linux/major.h>
#include <linux/errno.h>
#include <linux/signal.h>
#include <linux/fcntl.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/interrupt.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/tty_flip.h>
#include <linux/devpts_fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/console.h>
#include <linux/timer.h>
#include <linux/ctype.h>
#include <linux/kd.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/ppp-ioctl.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/wait.h>
#include <linux/bitops.h>
#include <linux/delay.h>
#include <linux/seq_file.h>
#include <linux/serial.h>
#include <linux/ratelimit.h>
#include <linux/compat.h>
#include <linux/uaccess.h>
#include <linux/termios_internal.h>
#include <linux/fs.h>
#include <linux/kbd_kern.h>
#include <linux/vt_kern.h>
#include <linux/selection.h>
#include <linux/kmod.h>
#include <linux/nsproxy.h>
#include "tty.h"
#undef TTY_DEBUG_HANGUP
#ifdef TTY_DEBUG_HANGUP
# define tty_debug_hangup(tty, f, args...) tty_debug(tty, f, ##args)
#else
# define tty_debug_hangup(tty, f, args...) do { } while (0)
#endif
#define TTY_PARANOIA_CHECK 1
#define CHECK_TTY_COUNT 1
struct ktermios tty_std_termios = { /* for the benefit of tty drivers */
.c_iflag = ICRNL | IXON,
.c_oflag = OPOST | ONLCR,
.c_cflag = B38400 | CS8 | CREAD | HUPCL,
.c_lflag = ISIG | ICANON | ECHO | ECHOE | ECHOK |
ECHOCTL | ECHOKE | IEXTEN,
.c_cc = INIT_C_CC,
.c_ispeed = 38400,
.c_ospeed = 38400,
/* .c_line = N_TTY, */
};
EXPORT_SYMBOL(tty_std_termios);
/* This list gets poked at by procfs and various bits of boot up code. This
* could do with some rationalisation such as pulling the tty proc function
* into this file.
*/
LIST_HEAD(tty_drivers); /* linked list of tty drivers */
/* Mutex to protect creating and releasing a tty */
DEFINE_MUTEX(tty_mutex);
static ssize_t tty_read(struct kiocb *, struct iov_iter *);
static ssize_t tty_write(struct kiocb *, struct iov_iter *);
static __poll_t tty_poll(struct file *, poll_table *);
static int tty_open(struct inode *, struct file *);
#ifdef CONFIG_COMPAT
static long tty_compat_ioctl(struct file *file, unsigned int cmd,
unsigned long arg);
#else
#define tty_compat_ioctl NULL
#endif
static int __tty_fasync(int fd, struct file *filp, int on);
static int tty_fasync(int fd, struct file *filp, int on);
static void release_tty(struct tty_struct *tty, int idx);
/**
* free_tty_struct - free a disused tty
* @tty: tty struct to free
*
* Free the write buffers, tty queue and tty memory itself.
*
* Locking: none. Must be called after tty is definitely unused
*/
static void free_tty_struct(struct tty_struct *tty)
{
tty_ldisc_deinit(tty);
put_device(tty->dev);
kvfree(tty->write_buf);
kfree(tty);
}
static inline struct tty_struct *file_tty(struct file *file)
{
return ((struct tty_file_private *)file->private_data)->tty;
}
int tty_alloc_file(struct file *file)
{
struct tty_file_private *priv;
priv = kmalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
file->private_data = priv;
return 0;
}
/* Associate a new file with the tty structure */
void tty_add_file(struct tty_struct *tty, struct file *file)
{
struct tty_file_private *priv = file->private_data;
priv->tty = tty;
priv->file = file;
spin_lock(&tty->files_lock);
list_add(&priv->list, &tty->tty_files);
spin_unlock(&tty->files_lock);
}
/**
* tty_free_file - free file->private_data
* @file: to free private_data of
*
* This shall be used only for fail path handling when tty_add_file was not
* called yet.
*/
void tty_free_file(struct file *file)
{
struct tty_file_private *priv = file->private_data;
file->private_data = NULL;
kfree(priv);
}
/* Delete file from its tty */
static void tty_del_file(struct file *file)
{
struct tty_file_private *priv = file->private_data;
struct tty_struct *tty = priv->tty;
spin_lock(&tty->files_lock);
list_del(&priv->list);
spin_unlock(&tty->files_lock);
tty_free_file(file);
}
/**
* tty_name - return tty naming
* @tty: tty structure
*
* Convert a tty structure into a name. The name reflects the kernel naming
* policy and if udev is in use may not reflect user space
*
* Locking: none
*/
const char *tty_name(const struct tty_struct *tty)
{
if (!tty) /* Hmm. NULL pointer. That's fun. */
return "NULL tty";
return tty->name;
}
EXPORT_SYMBOL(tty_name);
const char *tty_driver_name(const struct tty_struct *tty)
{
if (!tty || !tty->driver)
return "";
return tty->driver->name;
}
static int tty_paranoia_check(struct tty_struct *tty, struct inode *inode,
const char *routine)
{
#ifdef TTY_PARANOIA_CHECK
if (!tty) {
pr_warn("(%d:%d): %s: NULL tty\n",
imajor(inode), iminor(inode), routine);
return 1;
}
#endif
return 0;
}
/* Caller must hold tty_lock */
static void check_tty_count(struct tty_struct *tty, const char *routine)
{
#ifdef CHECK_TTY_COUNT
struct list_head *p;
int count = 0, kopen_count = 0;
spin_lock(&tty->files_lock);
list_for_each(p, &tty->tty_files) {
count++;
}
spin_unlock(&tty->files_lock);
if (tty->driver->type == TTY_DRIVER_TYPE_PTY &&
tty->driver->subtype == PTY_TYPE_SLAVE &&
tty->link && tty->link->count)
count++;
if (tty_port_kopened(tty->port))
kopen_count++;
if (tty->count != (count + kopen_count)) {
tty_warn(tty, "%s: tty->count(%d) != (#fd's(%d) + #kopen's(%d))\n",
routine, tty->count, count, kopen_count);
}
#endif
}
/**
* get_tty_driver - find device of a tty
* @device: device identifier
* @index: returns the index of the tty
*
* This routine returns a tty driver structure, given a device number and also
* passes back the index number.
*
* Locking: caller must hold tty_mutex
*/
static struct tty_driver *get_tty_driver(dev_t device, int *index)
{
struct tty_driver *p;
list_for_each_entry(p, &tty_drivers, tty_drivers) {
dev_t base = MKDEV(p->major, p->minor_start);
if (device < base || device >= base + p->num)
continue;
*index = device - base;
return tty_driver_kref_get(p);
}
return NULL;
}
/**
* tty_dev_name_to_number - return dev_t for device name
* @name: user space name of device under /dev
* @number: pointer to dev_t that this function will populate
*
* This function converts device names like ttyS0 or ttyUSB1 into dev_t like
* (4, 64) or (188, 1). If no corresponding driver is registered then the
* function returns -%ENODEV.
*
* Locking: this acquires tty_mutex to protect the tty_drivers list from
* being modified while we are traversing it, and makes sure to
* release it before exiting.
*/
int tty_dev_name_to_number(const char *name, dev_t *number)
{
struct tty_driver *p;
int ret;
int index, prefix_length = 0;
const char *str;
for (str = name; *str && !isdigit(*str); str++)
;
if (!*str)
return -EINVAL;
ret = kstrtoint(str, 10, &index);
if (ret)
return ret;
prefix_length = str - name;
mutex_lock(&tty_mutex);
list_for_each_entry(p, &tty_drivers, tty_drivers)
if (prefix_length == strlen(p->name) && strncmp(name,
p->name, prefix_length) == 0) {
if (index < p->num) {
*number = MKDEV(p->major, p->minor_start + index);
goto out;
}
}
/* if here then driver wasn't found */
ret = -ENODEV;
out:
mutex_unlock(&tty_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(tty_dev_name_to_number);
#ifdef CONFIG_CONSOLE_POLL
/**
* tty_find_polling_driver - find device of a polled tty
* @name: name string to match
* @line: pointer to resulting tty line nr
*
* This routine returns a tty driver structure, given a name and the condition
* that the tty driver is capable of polled operation.
*/
struct tty_driver *tty_find_polling_driver(char *name, int *line)
{
struct tty_driver *p, *res = NULL;
int tty_line = 0;
int len;
char *str, *stp;
for (str = name; *str; str++)
if ((*str >= '0' && *str <= '9') || *str == ',')
break;
if (!*str)
return NULL;
len = str - name;
tty_line = simple_strtoul(str, &str, 10);
mutex_lock(&tty_mutex);
/* Search through the tty devices to look for a match */
list_for_each_entry(p, &tty_drivers, tty_drivers) {
if (!len || strncmp(name, p->name, len) != 0)
continue;
stp = str;
if (*stp == ',')
stp++;
if (*stp == '\0')
stp = NULL;
if (tty_line >= 0 && tty_line < p->num && p->ops &&
p->ops->poll_init && !p->ops->poll_init(p, tty_line, stp)) {
res = tty_driver_kref_get(p);
*line = tty_line;
break;
}
}
mutex_unlock(&tty_mutex);
return res;
}
EXPORT_SYMBOL_GPL(tty_find_polling_driver);
#endif
static ssize_t hung_up_tty_read(struct kiocb *iocb, struct iov_iter *to)
{
return 0;
}
static ssize_t hung_up_tty_write(struct kiocb *iocb, struct iov_iter *from)
{
return -EIO;
}
/* No kernel lock held - none needed ;) */
static __poll_t hung_up_tty_poll(struct file *filp, poll_table *wait)
{
return EPOLLIN | EPOLLOUT | EPOLLERR | EPOLLHUP | EPOLLRDNORM | EPOLLWRNORM;
}
static long hung_up_tty_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
return cmd == TIOCSPGRP ? -ENOTTY : -EIO;
}
static long hung_up_tty_compat_ioctl(struct file *file,
unsigned int cmd, unsigned long arg)
{
return cmd == TIOCSPGRP ? -ENOTTY : -EIO;
}
static int hung_up_tty_fasync(int fd, struct file *file, int on)
{
return -ENOTTY;
}
static void tty_show_fdinfo(struct seq_file *m, struct file *file)
{
struct tty_struct *tty = file_tty(file);
if (tty && tty->ops && tty->ops->show_fdinfo)
tty->ops->show_fdinfo(tty, m);
}
static const struct file_operations tty_fops = {
.llseek = no_llseek,
.read_iter = tty_read,
.write_iter = tty_write,
.splice_read = copy_splice_read,
.splice_write = iter_file_splice_write,
.poll = tty_poll,
.unlocked_ioctl = tty_ioctl,
.compat_ioctl = tty_compat_ioctl,
.open = tty_open,
.release = tty_release,
.fasync = tty_fasync,
.show_fdinfo = tty_show_fdinfo,
};
static const struct file_operations console_fops = {
.llseek = no_llseek,
.read_iter = tty_read,
.write_iter = redirected_tty_write,
.splice_read = copy_splice_read,
.splice_write = iter_file_splice_write,
.poll = tty_poll,
.unlocked_ioctl = tty_ioctl,
.compat_ioctl = tty_compat_ioctl,
.open = tty_open,
.release = tty_release,
.fasync = tty_fasync,
};
static const struct file_operations hung_up_tty_fops = {
.llseek = no_llseek,
.read_iter = hung_up_tty_read,
.write_iter = hung_up_tty_write,
.poll = hung_up_tty_poll,
.unlocked_ioctl = hung_up_tty_ioctl,
.compat_ioctl = hung_up_tty_compat_ioctl,
.release = tty_release,
.fasync = hung_up_tty_fasync,
};
static DEFINE_SPINLOCK(redirect_lock);
static struct file *redirect;
/**
* tty_wakeup - request more data
* @tty: terminal
*
* Internal and external helper for wakeups of tty. This function informs the
* line discipline if present that the driver is ready to receive more output
* data.
*/
void tty_wakeup(struct tty_struct *tty)
{
struct tty_ldisc *ld;
if (test_bit(TTY_DO_WRITE_WAKEUP, &tty->flags)) {
ld = tty_ldisc_ref(tty);
if (ld) {
if (ld->ops->write_wakeup)
ld->ops->write_wakeup(tty);
tty_ldisc_deref(ld);
}
}
wake_up_interruptible_poll(&tty->write_wait, EPOLLOUT);
}
EXPORT_SYMBOL_GPL(tty_wakeup);
/**
* tty_release_redirect - Release a redirect on a pty if present
* @tty: tty device
*
* This is available to the pty code so if the master closes, if the slave is a
* redirect it can release the redirect.
*/
static struct file *tty_release_redirect(struct tty_struct *tty)
{
struct file *f = NULL;
spin_lock(&redirect_lock);
if (redirect && file_tty(redirect) == tty) {
f = redirect;
redirect = NULL;
}
spin_unlock(&redirect_lock);
return f;
}
/**
* __tty_hangup - actual handler for hangup events
* @tty: tty device
* @exit_session: if non-zero, signal all foreground group processes
*
* This can be called by a "kworker" kernel thread. That is process synchronous
* but doesn't hold any locks, so we need to make sure we have the appropriate
* locks for what we're doing.
*
* The hangup event clears any pending redirections onto the hung up device. It
* ensures future writes will error and it does the needed line discipline
* hangup and signal delivery. The tty object itself remains intact.
*
* Locking:
* * BTM
*
* * redirect lock for undoing redirection
* * file list lock for manipulating list of ttys
* * tty_ldiscs_lock from called functions
* * termios_rwsem resetting termios data
* * tasklist_lock to walk task list for hangup event
*
* * ->siglock to protect ->signal/->sighand
*
*/
static void __tty_hangup(struct tty_struct *tty, int exit_session)
{
struct file *cons_filp = NULL;
struct file *filp, *f;
struct tty_file_private *priv;
int closecount = 0, n;
int refs;
if (!tty)
return;
f = tty_release_redirect(tty);
tty_lock(tty);
if (test_bit(TTY_HUPPED, &tty->flags)) {
tty_unlock(tty);
return;
}
/*
* Some console devices aren't actually hung up for technical and
* historical reasons, which can lead to indefinite interruptible
* sleep in n_tty_read(). The following explicitly tells
* n_tty_read() to abort readers.
*/
set_bit(TTY_HUPPING, &tty->flags);
/* inuse_filps is protected by the single tty lock,
* this really needs to change if we want to flush the
* workqueue with the lock held.
*/
check_tty_count(tty, "tty_hangup");
spin_lock(&tty->files_lock);
/* This breaks for file handles being sent over AF_UNIX sockets ? */
list_for_each_entry(priv, &tty->tty_files, list) {
filp = priv->file;
if (filp->f_op->write_iter == redirected_tty_write)
cons_filp = filp;
if (filp->f_op->write_iter != tty_write)
continue;
closecount++;
__tty_fasync(-1, filp, 0); /* can't block */
filp->f_op = &hung_up_tty_fops;
}
spin_unlock(&tty->files_lock);
refs = tty_signal_session_leader(tty, exit_session);
/* Account for the p->signal references we killed */
while (refs--)
tty_kref_put(tty);
tty_ldisc_hangup(tty, cons_filp != NULL);
spin_lock_irq(&tty->ctrl.lock);
clear_bit(TTY_THROTTLED, &tty->flags);
clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags);
put_pid(tty->ctrl.session);
put_pid(tty->ctrl.pgrp);
tty->ctrl.session = NULL;
tty->ctrl.pgrp = NULL;
tty->ctrl.pktstatus = 0;
spin_unlock_irq(&tty->ctrl.lock);
/*
* If one of the devices matches a console pointer, we
* cannot just call hangup() because that will cause
* tty->count and state->count to go out of sync.
* So we just call close() the right number of times.
*/
if (cons_filp) {
if (tty->ops->close)
for (n = 0; n < closecount; n++)
tty->ops->close(tty, cons_filp);
} else if (tty->ops->hangup)
tty->ops->hangup(tty);
/*
* We don't want to have driver/ldisc interactions beyond the ones
* we did here. The driver layer expects no calls after ->hangup()
* from the ldisc side, which is now guaranteed.
*/
set_bit(TTY_HUPPED, &tty->flags);
clear_bit(TTY_HUPPING, &tty->flags);
tty_unlock(tty);
if (f)
fput(f);
}
static void do_tty_hangup(struct work_struct *work)
{
struct tty_struct *tty =
container_of(work, struct tty_struct, hangup_work);
__tty_hangup(tty, 0);
}
/**
* tty_hangup - trigger a hangup event
* @tty: tty to hangup
*
* A carrier loss (virtual or otherwise) has occurred on @tty. Schedule a
* hangup sequence to run after this event.
*/
void tty_hangup(struct tty_struct *tty)
{
tty_debug_hangup(tty, "hangup\n");
schedule_work(&tty->hangup_work);
}
EXPORT_SYMBOL(tty_hangup);
/**
* tty_vhangup - process vhangup
* @tty: tty to hangup
*
* The user has asked via system call for the terminal to be hung up. We do
* this synchronously so that when the syscall returns the process is complete.
* That guarantee is necessary for security reasons.
*/
void tty_vhangup(struct tty_struct *tty)
{
tty_debug_hangup(tty, "vhangup\n");
__tty_hangup(tty, 0);
}
EXPORT_SYMBOL(tty_vhangup);
/**
* tty_vhangup_self - process vhangup for own ctty
*
* Perform a vhangup on the current controlling tty
*/
void tty_vhangup_self(void)
{
struct tty_struct *tty;
tty = get_current_tty();
if (tty) {
tty_vhangup(tty);
tty_kref_put(tty);
}
}
/**
* tty_vhangup_session - hangup session leader exit
* @tty: tty to hangup
*
* The session leader is exiting and hanging up its controlling terminal.
* Every process in the foreground process group is signalled %SIGHUP.
*
* We do this synchronously so that when the syscall returns the process is
* complete. That guarantee is necessary for security reasons.
*/
void tty_vhangup_session(struct tty_struct *tty)
{
tty_debug_hangup(tty, "session hangup\n");
__tty_hangup(tty, 1);
}
/**
* tty_hung_up_p - was tty hung up
* @filp: file pointer of tty
*
* Return: true if the tty has been subject to a vhangup or a carrier loss
*/
int tty_hung_up_p(struct file *filp)
{
return (filp && filp->f_op == &hung_up_tty_fops);
}
EXPORT_SYMBOL(tty_hung_up_p);
void __stop_tty(struct tty_struct *tty)
{
if (tty->flow.stopped)
return;
tty->flow.stopped = true;
if (tty->ops->stop)
tty->ops->stop(tty);
}
/**
* stop_tty - propagate flow control
* @tty: tty to stop
*
* Perform flow control to the driver. May be called on an already stopped
* device and will not re-call the &tty_driver->stop() method.
*
* This functionality is used by both the line disciplines for halting incoming
* flow and by the driver. It may therefore be called from any context, may be
* under the tty %atomic_write_lock but not always.
*
* Locking:
* flow.lock
*/
void stop_tty(struct tty_struct *tty)
{
unsigned long flags;
spin_lock_irqsave(&tty->flow.lock, flags);
__stop_tty(tty);
spin_unlock_irqrestore(&tty->flow.lock, flags);
}
EXPORT_SYMBOL(stop_tty);
void __start_tty(struct tty_struct *tty)
{
if (!tty->flow.stopped || tty->flow.tco_stopped)
return;
tty->flow.stopped = false;
if (tty->ops->start)
tty->ops->start(tty);
tty_wakeup(tty);
}
/**
* start_tty - propagate flow control
* @tty: tty to start
*
* Start a tty that has been stopped if at all possible. If @tty was previously
* stopped and is now being started, the &tty_driver->start() method is invoked
* and the line discipline woken.
*
* Locking:
* flow.lock
*/
void start_tty(struct tty_struct *tty)
{
unsigned long flags;
spin_lock_irqsave(&tty->flow.lock, flags);
__start_tty(tty);
spin_unlock_irqrestore(&tty->flow.lock, flags);
}
EXPORT_SYMBOL(start_tty);
static void tty_update_time(struct tty_struct *tty, bool mtime)
{
time64_t sec = ktime_get_real_seconds();
struct tty_file_private *priv;
spin_lock(&tty->files_lock);
list_for_each_entry(priv, &tty->tty_files, list) {
struct inode *inode = file_inode(priv->file);
struct timespec64 time = mtime ? inode_get_mtime(inode) : inode_get_atime(inode);
/*
* We only care if the two values differ in anything other than the
* lower three bits (i.e every 8 seconds). If so, then we can update
* the time of the tty device, otherwise it could be construded as a
* security leak to let userspace know the exact timing of the tty.
*/
if ((sec ^ time.tv_sec) & ~7) {
if (mtime)
inode_set_mtime(inode, sec, 0);
else
inode_set_atime(inode, sec, 0);
}
}
spin_unlock(&tty->files_lock);
}
/*
* Iterate on the ldisc ->read() function until we've gotten all
* the data the ldisc has for us.
*
* The "cookie" is something that the ldisc read function can fill
* in to let us know that there is more data to be had.
*
* We promise to continue to call the ldisc until it stops returning
* data or clears the cookie. The cookie may be something that the
* ldisc maintains state for and needs to free.
*/
static ssize_t iterate_tty_read(struct tty_ldisc *ld, struct tty_struct *tty,
struct file *file, struct iov_iter *to)
{
void *cookie = NULL;
unsigned long offset = 0;
ssize_t retval = 0;
size_t copied, count = iov_iter_count(to);
u8 kernel_buf[64];
do {
ssize_t size = min(count, sizeof(kernel_buf));
size = ld->ops->read(tty, file, kernel_buf, size, &cookie, offset);
if (!size)
break;
if (size < 0) {
/* Did we have an earlier error (ie -EFAULT)? */
if (retval)
break;
retval = size;
/*
* -EOVERFLOW means we didn't have enough space
* for a whole packet, and we shouldn't return
* a partial result.
*/
if (retval == -EOVERFLOW)
offset = 0;
break;
}
copied = copy_to_iter(kernel_buf, size, to);
offset += copied;
count -= copied;
/*
* If the user copy failed, we still need to do another ->read()
* call if we had a cookie to let the ldisc clear up.
*
* But make sure size is zeroed.
*/
if (unlikely(copied != size)) {
count = 0;
retval = -EFAULT;
}
} while (cookie);
/* We always clear tty buffer in case they contained passwords */
memzero_explicit(kernel_buf, sizeof(kernel_buf));
return offset ? offset : retval;
}
/**
* tty_read - read method for tty device files
* @iocb: kernel I/O control block
* @to: destination for the data read
*
* Perform the read system call function on this terminal device. Checks
* for hung up devices before calling the line discipline method.
*
* Locking:
* Locks the line discipline internally while needed. Multiple read calls
* may be outstanding in parallel.
*/
static ssize_t tty_read(struct kiocb *iocb, struct iov_iter *to)
{
struct file *file = iocb->ki_filp;
struct inode *inode = file_inode(file);
struct tty_struct *tty = file_tty(file);
struct tty_ldisc *ld;
ssize_t ret;
if (tty_paranoia_check(tty, inode, "tty_read"))
return -EIO;
if (!tty || tty_io_error(tty))
return -EIO;
/* We want to wait for the line discipline to sort out in this
* situation.
*/
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return hung_up_tty_read(iocb, to);
ret = -EIO;
if (ld->ops->read)
ret = iterate_tty_read(ld, tty, file, to);
tty_ldisc_deref(ld);
if (ret > 0)
tty_update_time(tty, false);
return ret;
}
void tty_write_unlock(struct tty_struct *tty)
{
mutex_unlock(&tty->atomic_write_lock);
wake_up_interruptible_poll(&tty->write_wait, EPOLLOUT);
}
int tty_write_lock(struct tty_struct *tty, bool ndelay)
{
if (!mutex_trylock(&tty->atomic_write_lock)) {
if (ndelay)
return -EAGAIN;
if (mutex_lock_interruptible(&tty->atomic_write_lock))
return -ERESTARTSYS;
}
return 0;
}
/*
* Split writes up in sane blocksizes to avoid
* denial-of-service type attacks
*/
static ssize_t iterate_tty_write(struct tty_ldisc *ld, struct tty_struct *tty,
struct file *file, struct iov_iter *from)
{
size_t chunk, count = iov_iter_count(from);
ssize_t ret, written = 0;
ret = tty_write_lock(tty, file->f_flags & O_NDELAY);
if (ret < 0)
return ret;
/*
* We chunk up writes into a temporary buffer. This
* simplifies low-level drivers immensely, since they
* don't have locking issues and user mode accesses.
*
* But if TTY_NO_WRITE_SPLIT is set, we should use a
* big chunk-size..
*
* The default chunk-size is 2kB, because the NTTY
* layer has problems with bigger chunks. It will
* claim to be able to handle more characters than
* it actually does.
*/
chunk = 2048;
if (test_bit(TTY_NO_WRITE_SPLIT, &tty->flags))
chunk = 65536;
if (count < chunk)
chunk = count;
/* write_buf/write_cnt is protected by the atomic_write_lock mutex */
if (tty->write_cnt < chunk) {
u8 *buf_chunk;
if (chunk < 1024)
chunk = 1024;
buf_chunk = kvmalloc(chunk, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
if (!buf_chunk) {
ret = -ENOMEM;
goto out;
}
kvfree(tty->write_buf);
tty->write_cnt = chunk;
tty->write_buf = buf_chunk;
}
/* Do the write .. */
for (;;) {
size_t size = min(chunk, count);
ret = -EFAULT;
if (copy_from_iter(tty->write_buf, size, from) != size)
break;
ret = ld->ops->write(tty, file, tty->write_buf, size);
if (ret <= 0)
break;
written += ret;
if (ret > size)
break;
/* FIXME! Have Al check this! */
if (ret != size)
iov_iter_revert(from, size-ret);
count -= ret;
if (!count)
break;
ret = -ERESTARTSYS;
if (signal_pending(current))
break;
cond_resched();
}
if (written) {
tty_update_time(tty, true);
ret = written;
}
out:
tty_write_unlock(tty);
return ret;
}
#ifdef CONFIG_PRINT_QUOTA_WARNING
/**
* tty_write_message - write a message to a certain tty, not just the console.
* @tty: the destination tty_struct
* @msg: the message to write
*
* This is used for messages that need to be redirected to a specific tty. We
* don't put it into the syslog queue right now maybe in the future if really
* needed.
*
* We must still hold the BTM and test the CLOSING flag for the moment.
*
* This function is DEPRECATED, do not use in new code.
*/
void tty_write_message(struct tty_struct *tty, char *msg)
{
if (tty) {
mutex_lock(&tty->atomic_write_lock);
tty_lock(tty);
if (tty->ops->write && tty->count > 0)
tty->ops->write(tty, msg, strlen(msg));
tty_unlock(tty);
tty_write_unlock(tty);
}
}
#endif
static ssize_t file_tty_write(struct file *file, struct kiocb *iocb, struct iov_iter *from)
{
struct tty_struct *tty = file_tty(file);
struct tty_ldisc *ld;
ssize_t ret;
if (tty_paranoia_check(tty, file_inode(file), "tty_write"))
return -EIO;
if (!tty || !tty->ops->write || tty_io_error(tty))
return -EIO;
/* Short term debug to catch buggy drivers */
if (tty->ops->write_room == NULL)
tty_err(tty, "missing write_room method\n");
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return hung_up_tty_write(iocb, from);
if (!ld->ops->write)
ret = -EIO;
else
ret = iterate_tty_write(ld, tty, file, from);
tty_ldisc_deref(ld);
return ret;
}
/**
* tty_write - write method for tty device file
* @iocb: kernel I/O control block
* @from: iov_iter with data to write
*
* Write data to a tty device via the line discipline.
*
* Locking:
* Locks the line discipline as required
* Writes to the tty driver are serialized by the atomic_write_lock
* and are then processed in chunks to the device. The line
* discipline write method will not be invoked in parallel for
* each device.
*/
static ssize_t tty_write(struct kiocb *iocb, struct iov_iter *from)
{
return file_tty_write(iocb->ki_filp, iocb, from);
}
ssize_t redirected_tty_write(struct kiocb *iocb, struct iov_iter *iter)
{
struct file *p = NULL;
spin_lock(&redirect_lock);
if (redirect)
p = get_file(redirect);
spin_unlock(&redirect_lock);
/*
* We know the redirected tty is just another tty, we can
* call file_tty_write() directly with that file pointer.
*/
if (p) {
ssize_t res;
res = file_tty_write(p, iocb, iter);
fput(p);
return res;
}
return tty_write(iocb, iter);
}
/**
* tty_send_xchar - send priority character
* @tty: the tty to send to
* @ch: xchar to send
*
* Send a high priority character to the tty even if stopped.
*
* Locking: none for xchar method, write ordering for write method.
*/
int tty_send_xchar(struct tty_struct *tty, u8 ch)
{
bool was_stopped = tty->flow.stopped;
if (tty->ops->send_xchar) {
down_read(&tty->termios_rwsem);
tty->ops->send_xchar(tty, ch);
up_read(&tty->termios_rwsem);
return 0;
}
if (tty_write_lock(tty, false) < 0)
return -ERESTARTSYS;
down_read(&tty->termios_rwsem);
if (was_stopped)
start_tty(tty);
tty->ops->write(tty, &ch, 1);
if (was_stopped)
stop_tty(tty);
up_read(&tty->termios_rwsem);
tty_write_unlock(tty);
return 0;
}
/**
* pty_line_name - generate name for a pty
* @driver: the tty driver in use
* @index: the minor number
* @p: output buffer of at least 6 bytes
*
* Generate a name from a @driver reference and write it to the output buffer
* @p.
*
* Locking: None
*/
static void pty_line_name(struct tty_driver *driver, int index, char *p)
{
static const char ptychar[] = "pqrstuvwxyzabcde";
int i = index + driver->name_base;
/* ->name is initialized to "ttyp", but "tty" is expected */
sprintf(p, "%s%c%x",
driver->subtype == PTY_TYPE_SLAVE ? "tty" : driver->name,
ptychar[i >> 4 & 0xf], i & 0xf);
}
/**
* tty_line_name - generate name for a tty
* @driver: the tty driver in use
* @index: the minor number
* @p: output buffer of at least 7 bytes
*
* Generate a name from a @driver reference and write it to the output buffer
* @p.
*
* Locking: None
*/
static ssize_t tty_line_name(struct tty_driver *driver, int index, char *p)
{
if (driver->flags & TTY_DRIVER_UNNUMBERED_NODE)
return sprintf(p, "%s", driver->name);
else
return sprintf(p, "%s%d", driver->name,
index + driver->name_base);
}
/**
* tty_driver_lookup_tty() - find an existing tty, if any
* @driver: the driver for the tty
* @file: file object
* @idx: the minor number
*
* Return: the tty, if found. If not found, return %NULL or ERR_PTR() if the
* driver lookup() method returns an error.
*
* Locking: tty_mutex must be held. If the tty is found, bump the tty kref.
*/
static struct tty_struct *tty_driver_lookup_tty(struct tty_driver *driver,
struct file *file, int idx)
{
struct tty_struct *tty;
if (driver->ops->lookup) {
if (!file)
tty = ERR_PTR(-EIO);
else
tty = driver->ops->lookup(driver, file, idx);
} else {
if (idx >= driver->num)
return ERR_PTR(-EINVAL);
tty = driver->ttys[idx];
}
if (!IS_ERR(tty))
tty_kref_get(tty);
return tty;
}
/**
* tty_init_termios - helper for termios setup
* @tty: the tty to set up
*
* Initialise the termios structure for this tty. This runs under the
* %tty_mutex currently so we can be relaxed about ordering.
*/
void tty_init_termios(struct tty_struct *tty)
{
struct ktermios *tp;
int idx = tty->index;
if (tty->driver->flags & TTY_DRIVER_RESET_TERMIOS)
tty->termios = tty->driver->init_termios;
else {
/* Check for lazy saved data */
tp = tty->driver->termios[idx];
if (tp != NULL) {
tty->termios = *tp;
tty->termios.c_line = tty->driver->init_termios.c_line;
} else
tty->termios = tty->driver->init_termios;
}
/* Compatibility until drivers always set this */
tty->termios.c_ispeed = tty_termios_input_baud_rate(&tty->termios);
tty->termios.c_ospeed = tty_termios_baud_rate(&tty->termios);
}
EXPORT_SYMBOL_GPL(tty_init_termios);
/**
* tty_standard_install - usual tty->ops->install
* @driver: the driver for the tty
* @tty: the tty
*
* If the @driver overrides @tty->ops->install, it still can call this function
* to perform the standard install operations.
*/
int tty_standard_install(struct tty_driver *driver, struct tty_struct *tty)
{
tty_init_termios(tty);
tty_driver_kref_get(driver);
tty->count++;
driver->ttys[tty->index] = tty;
return 0;
}
EXPORT_SYMBOL_GPL(tty_standard_install);
/**
* tty_driver_install_tty() - install a tty entry in the driver
* @driver: the driver for the tty
* @tty: the tty
*
* Install a tty object into the driver tables. The @tty->index field will be
* set by the time this is called. This method is responsible for ensuring any
* need additional structures are allocated and configured.
*
* Locking: tty_mutex for now
*/
static int tty_driver_install_tty(struct tty_driver *driver,
struct tty_struct *tty)
{
return driver->ops->install ? driver->ops->install(driver, tty) :
tty_standard_install(driver, tty);
}
/**
* tty_driver_remove_tty() - remove a tty from the driver tables
* @driver: the driver for the tty
* @tty: tty to remove
*
* Remove a tty object from the driver tables. The tty->index field will be set
* by the time this is called.
*
* Locking: tty_mutex for now
*/
static void tty_driver_remove_tty(struct tty_driver *driver, struct tty_struct *tty)
{
if (driver->ops->remove)
driver->ops->remove(driver, tty);
else
driver->ttys[tty->index] = NULL;
}
/**
* tty_reopen() - fast re-open of an open tty
* @tty: the tty to open
*
* Re-opens on master ptys are not allowed and return -%EIO.
*
* Locking: Caller must hold tty_lock
* Return: 0 on success, -errno on error.
*/
static int tty_reopen(struct tty_struct *tty)
{
struct tty_driver *driver = tty->driver;
struct tty_ldisc *ld;
int retval = 0;
if (driver->type == TTY_DRIVER_TYPE_PTY &&
driver->subtype == PTY_TYPE_MASTER)
return -EIO;
if (!tty->count)
return -EAGAIN;
if (test_bit(TTY_EXCLUSIVE, &tty->flags) && !capable(CAP_SYS_ADMIN))
return -EBUSY;
ld = tty_ldisc_ref_wait(tty);
if (ld) {
tty_ldisc_deref(ld);
} else {
retval = tty_ldisc_lock(tty, 5 * HZ);
if (retval)
return retval;
if (!tty->ldisc)
retval = tty_ldisc_reinit(tty, tty->termios.c_line);
tty_ldisc_unlock(tty);
}
if (retval == 0)
tty->count++;
return retval;
}
/**
* tty_init_dev - initialise a tty device
* @driver: tty driver we are opening a device on
* @idx: device index
*
* Prepare a tty device. This may not be a "new" clean device but could also be
* an active device. The pty drivers require special handling because of this.
*
* Locking:
* The function is called under the tty_mutex, which protects us from the
* tty struct or driver itself going away.
*
* On exit the tty device has the line discipline attached and a reference
* count of 1. If a pair was created for pty/tty use and the other was a pty
* master then it too has a reference count of 1.
*
* WSH 06/09/97: Rewritten to remove races and properly clean up after a failed
* open. The new code protects the open with a mutex, so it's really quite
* straightforward. The mutex locking can probably be relaxed for the (most
* common) case of reopening a tty.
*
* Return: new tty structure
*/
struct tty_struct *tty_init_dev(struct tty_driver *driver, int idx)
{
struct tty_struct *tty;
int retval;
/*
* First time open is complex, especially for PTY devices.
* This code guarantees that either everything succeeds and the
* TTY is ready for operation, or else the table slots are vacated
* and the allocated memory released. (Except that the termios
* may be retained.)
*/
if (!try_module_get(driver->owner))
return ERR_PTR(-ENODEV);
tty = alloc_tty_struct(driver, idx);
if (!tty) {
retval = -ENOMEM;
goto err_module_put;
}
tty_lock(tty);
retval = tty_driver_install_tty(driver, tty);
if (retval < 0)
goto err_free_tty;
if (!tty->port)
tty->port = driver->ports[idx];
if (WARN_RATELIMIT(!tty->port,
"%s: %s driver does not set tty->port. This would crash the kernel. Fix the driver!\n",
__func__, tty->driver->name)) {
retval = -EINVAL;
goto err_release_lock;
}
retval = tty_ldisc_lock(tty, 5 * HZ);
if (retval)
goto err_release_lock;
tty->port->itty = tty;
/*
* Structures all installed ... call the ldisc open routines.
* If we fail here just call release_tty to clean up. No need
* to decrement the use counts, as release_tty doesn't care.
*/
retval = tty_ldisc_setup(tty, tty->link);
if (retval)
goto err_release_tty;
tty_ldisc_unlock(tty);
/* Return the tty locked so that it cannot vanish under the caller */
return tty;
err_free_tty:
tty_unlock(tty);
free_tty_struct(tty);
err_module_put:
module_put(driver->owner);
return ERR_PTR(retval);
/* call the tty release_tty routine to clean out this slot */
err_release_tty:
tty_ldisc_unlock(tty);
tty_info_ratelimited(tty, "ldisc open failed (%d), clearing slot %d\n",
retval, idx);
err_release_lock:
tty_unlock(tty);
release_tty(tty, idx);
return ERR_PTR(retval);
}
/**
* tty_save_termios() - save tty termios data in driver table
* @tty: tty whose termios data to save
*
* Locking: Caller guarantees serialisation with tty_init_termios().
*/
void tty_save_termios(struct tty_struct *tty)
{
struct ktermios *tp;
int idx = tty->index;
/* If the port is going to reset then it has no termios to save */
if (tty->driver->flags & TTY_DRIVER_RESET_TERMIOS)
return;
/* Stash the termios data */
tp = tty->driver->termios[idx];
if (tp == NULL) {
tp = kmalloc(sizeof(*tp), GFP_KERNEL);
if (tp == NULL)
return;
tty->driver->termios[idx] = tp;
}
*tp = tty->termios;
}
EXPORT_SYMBOL_GPL(tty_save_termios);
/**
* tty_flush_works - flush all works of a tty/pty pair
* @tty: tty device to flush works for (or either end of a pty pair)
*
* Sync flush all works belonging to @tty (and the 'other' tty).
*/
static void tty_flush_works(struct tty_struct *tty)
{
flush_work(&tty->SAK_work);
flush_work(&tty->hangup_work);
if (tty->link) {
flush_work(&tty->link->SAK_work);
flush_work(&tty->link->hangup_work);
}
}
/**
* release_one_tty - release tty structure memory
* @work: work of tty we are obliterating
*
* Releases memory associated with a tty structure, and clears out the
* driver table slots. This function is called when a device is no longer
* in use. It also gets called when setup of a device fails.
*
* Locking:
* takes the file list lock internally when working on the list of ttys
* that the driver keeps.
*
* This method gets called from a work queue so that the driver private
* cleanup ops can sleep (needed for USB at least)
*/
static void release_one_tty(struct work_struct *work)
{
struct tty_struct *tty =
container_of(work, struct tty_struct, hangup_work);
struct tty_driver *driver = tty->driver;
struct module *owner = driver->owner;
if (tty->ops->cleanup)
tty->ops->cleanup(tty);
tty_driver_kref_put(driver);
module_put(owner);
spin_lock(&tty->files_lock);
list_del_init(&tty->tty_files);
spin_unlock(&tty->files_lock);
put_pid(tty->ctrl.pgrp);
put_pid(tty->ctrl.session);
free_tty_struct(tty);
}
static void queue_release_one_tty(struct kref *kref)
{
struct tty_struct *tty = container_of(kref, struct tty_struct, kref);
/* The hangup queue is now free so we can reuse it rather than
* waste a chunk of memory for each port.
*/
INIT_WORK(&tty->hangup_work, release_one_tty);
schedule_work(&tty->hangup_work);
}
/**
* tty_kref_put - release a tty kref
* @tty: tty device
*
* Release a reference to the @tty device and if need be let the kref layer
* destruct the object for us.
*/
void tty_kref_put(struct tty_struct *tty)
{
if (tty)
kref_put(&tty->kref, queue_release_one_tty);
}
EXPORT_SYMBOL(tty_kref_put);
/**
* release_tty - release tty structure memory
* @tty: tty device release
* @idx: index of the tty device release
*
* Release both @tty and a possible linked partner (think pty pair),
* and decrement the refcount of the backing module.
*
* Locking:
* tty_mutex
* takes the file list lock internally when working on the list of ttys
* that the driver keeps.
*/
static void release_tty(struct tty_struct *tty, int idx)
{
/* This should always be true but check for the moment */
WARN_ON(tty->index != idx);
WARN_ON(!mutex_is_locked(&tty_mutex));
if (tty->ops->shutdown)
tty->ops->shutdown(tty);
tty_save_termios(tty);
tty_driver_remove_tty(tty->driver, tty);
if (tty->port)
tty->port->itty = NULL;
if (tty->link)
tty->link->port->itty = NULL;
if (tty->port)
tty_buffer_cancel_work(tty->port);
if (tty->link)
tty_buffer_cancel_work(tty->link->port);
tty_kref_put(tty->link);
tty_kref_put(tty);
}
/**
* tty_release_checks - check a tty before real release
* @tty: tty to check
* @idx: index of the tty
*
* Performs some paranoid checking before true release of the @tty. This is a
* no-op unless %TTY_PARANOIA_CHECK is defined.
*/
static int tty_release_checks(struct tty_struct *tty, int idx)
{
#ifdef TTY_PARANOIA_CHECK
if (idx < 0 || idx >= tty->driver->num) {
tty_debug(tty, "bad idx %d\n", idx);
return -1;
}
/* not much to check for devpts */
if (tty->driver->flags & TTY_DRIVER_DEVPTS_MEM)
return 0;
if (tty != tty->driver->ttys[idx]) {
tty_debug(tty, "bad driver table[%d] = %p\n",
idx, tty->driver->ttys[idx]);
return -1;
}
if (tty->driver->other) {
struct tty_struct *o_tty = tty->link;
if (o_tty != tty->driver->other->ttys[idx]) {
tty_debug(tty, "bad other table[%d] = %p\n",
idx, tty->driver->other->ttys[idx]);
return -1;
}
if (o_tty->link != tty) {
tty_debug(tty, "bad link = %p\n", o_tty->link);
return -1;
}
}
#endif
return 0;
}
/**
* tty_kclose - closes tty opened by tty_kopen
* @tty: tty device
*
* Performs the final steps to release and free a tty device. It is the same as
* tty_release_struct() except that it also resets %TTY_PORT_KOPENED flag on
* @tty->port.
*/
void tty_kclose(struct tty_struct *tty)
{
/*
* Ask the line discipline code to release its structures
*/
tty_ldisc_release(tty);
/* Wait for pending work before tty destruction commences */
tty_flush_works(tty);
tty_debug_hangup(tty, "freeing structure\n");
/*
* The release_tty function takes care of the details of clearing
* the slots and preserving the termios structure.
*/
mutex_lock(&tty_mutex);
tty_port_set_kopened(tty->port, 0);
release_tty(tty, tty->index);
mutex_unlock(&tty_mutex);
}
EXPORT_SYMBOL_GPL(tty_kclose);
/**
* tty_release_struct - release a tty struct
* @tty: tty device
* @idx: index of the tty
*
* Performs the final steps to release and free a tty device. It is roughly the
* reverse of tty_init_dev().
*/
void tty_release_struct(struct tty_struct *tty, int idx)
{
/*
* Ask the line discipline code to release its structures
*/
tty_ldisc_release(tty);
/* Wait for pending work before tty destruction commmences */
tty_flush_works(tty);
tty_debug_hangup(tty, "freeing structure\n");
/*
* The release_tty function takes care of the details of clearing
* the slots and preserving the termios structure.
*/
mutex_lock(&tty_mutex);
release_tty(tty, idx);
mutex_unlock(&tty_mutex);
}
EXPORT_SYMBOL_GPL(tty_release_struct);
/**
* tty_release - vfs callback for close
* @inode: inode of tty
* @filp: file pointer for handle to tty
*
* Called the last time each file handle is closed that references this tty.
* There may however be several such references.
*
* Locking:
* Takes BKL. See tty_release_dev().
*
* Even releasing the tty structures is a tricky business. We have to be very
* careful that the structures are all released at the same time, as interrupts
* might otherwise get the wrong pointers.
*
* WSH 09/09/97: rewritten to avoid some nasty race conditions that could
* lead to double frees or releasing memory still in use.
*/
int tty_release(struct inode *inode, struct file *filp)
{
struct tty_struct *tty = file_tty(filp);
struct tty_struct *o_tty = NULL;
int do_sleep, final;
int idx;
long timeout = 0;
int once = 1;
if (tty_paranoia_check(tty, inode, __func__))
return 0;
tty_lock(tty);
check_tty_count(tty, __func__);
__tty_fasync(-1, filp, 0);
idx = tty->index;
if (tty->driver->type == TTY_DRIVER_TYPE_PTY &&
tty->driver->subtype == PTY_TYPE_MASTER)
o_tty = tty->link;
if (tty_release_checks(tty, idx)) {
tty_unlock(tty);
return 0;
}
tty_debug_hangup(tty, "releasing (count=%d)\n", tty->count);
if (tty->ops->close)
tty->ops->close(tty, filp);
/* If tty is pty master, lock the slave pty (stable lock order) */
tty_lock_slave(o_tty);
/*
* Sanity check: if tty->count is going to zero, there shouldn't be
* any waiters on tty->read_wait or tty->write_wait. We test the
* wait queues and kick everyone out _before_ actually starting to
* close. This ensures that we won't block while releasing the tty
* structure.
*
* The test for the o_tty closing is necessary, since the master and
* slave sides may close in any order. If the slave side closes out
* first, its count will be one, since the master side holds an open.
* Thus this test wouldn't be triggered at the time the slave closed,
* so we do it now.
*/
while (1) {
do_sleep = 0;
if (tty->count <= 1) {
if (waitqueue_active(&tty->read_wait)) {
wake_up_poll(&tty->read_wait, EPOLLIN);
do_sleep++;
}
if (waitqueue_active(&tty->write_wait)) {
wake_up_poll(&tty->write_wait, EPOLLOUT);
do_sleep++;
}
}
if (o_tty && o_tty->count <= 1) {
if (waitqueue_active(&o_tty->read_wait)) {
wake_up_poll(&o_tty->read_wait, EPOLLIN);
do_sleep++;
}
if (waitqueue_active(&o_tty->write_wait)) {
wake_up_poll(&o_tty->write_wait, EPOLLOUT);
do_sleep++;
}
}
if (!do_sleep)
break;
if (once) {
once = 0;
tty_warn(tty, "read/write wait queue active!\n");
}
schedule_timeout_killable(timeout);
if (timeout < 120 * HZ)
timeout = 2 * timeout + 1;
else
timeout = MAX_SCHEDULE_TIMEOUT;
}
if (o_tty) {
if (--o_tty->count < 0) {
tty_warn(tty, "bad slave count (%d)\n", o_tty->count);
o_tty->count = 0;
}
}
if (--tty->count < 0) {
tty_warn(tty, "bad tty->count (%d)\n", tty->count);
tty->count = 0;
}
/*
* We've decremented tty->count, so we need to remove this file
* descriptor off the tty->tty_files list; this serves two
* purposes:
* - check_tty_count sees the correct number of file descriptors
* associated with this tty.
* - do_tty_hangup no longer sees this file descriptor as
* something that needs to be handled for hangups.
*/
tty_del_file(filp);
/*
* Perform some housekeeping before deciding whether to return.
*
* If _either_ side is closing, make sure there aren't any
* processes that still think tty or o_tty is their controlling
* tty.
*/
if (!tty->count) {
read_lock(&tasklist_lock);
session_clear_tty(tty->ctrl.session);
if (o_tty)
session_clear_tty(o_tty->ctrl.session);
read_unlock(&tasklist_lock);
}
/* check whether both sides are closing ... */
final = !tty->count && !(o_tty && o_tty->count);
tty_unlock_slave(o_tty);
tty_unlock(tty);
/* At this point, the tty->count == 0 should ensure a dead tty
* cannot be re-opened by a racing opener.
*/
if (!final)
return 0;
tty_debug_hangup(tty, "final close\n");
tty_release_struct(tty, idx);
return 0;
}
/**
* tty_open_current_tty - get locked tty of current task
* @device: device number
* @filp: file pointer to tty
* @return: locked tty of the current task iff @device is /dev/tty
*
* Performs a re-open of the current task's controlling tty.
*
* We cannot return driver and index like for the other nodes because devpts
* will not work then. It expects inodes to be from devpts FS.
*/
static struct tty_struct *tty_open_current_tty(dev_t device, struct file *filp)
{
struct tty_struct *tty;
int retval;
if (device != MKDEV(TTYAUX_MAJOR, 0))
return NULL;
tty = get_current_tty();
if (!tty)
return ERR_PTR(-ENXIO);
filp->f_flags |= O_NONBLOCK; /* Don't let /dev/tty block */
/* noctty = 1; */
tty_lock(tty);
tty_kref_put(tty); /* safe to drop the kref now */
retval = tty_reopen(tty);
if (retval < 0) {
tty_unlock(tty);
tty = ERR_PTR(retval);
}
return tty;
}
/**
* tty_lookup_driver - lookup a tty driver for a given device file
* @device: device number
* @filp: file pointer to tty
* @index: index for the device in the @return driver
*
* If returned value is not erroneous, the caller is responsible to decrement
* the refcount by tty_driver_kref_put().
*
* Locking: %tty_mutex protects get_tty_driver()
*
* Return: driver for this inode (with increased refcount)
*/
static struct tty_driver *tty_lookup_driver(dev_t device, struct file *filp,
int *index)
{
struct tty_driver *driver = NULL;
switch (device) {
#ifdef CONFIG_VT
case MKDEV(TTY_MAJOR, 0): {
extern struct tty_driver *console_driver;
driver = tty_driver_kref_get(console_driver);
*index = fg_console;
break;
}
#endif
case MKDEV(TTYAUX_MAJOR, 1): {
struct tty_driver *console_driver = console_device(index);
if (console_driver) {
driver = tty_driver_kref_get(console_driver);
if (driver && filp) {
/* Don't let /dev/console block */
filp->f_flags |= O_NONBLOCK;
break;
}
}
if (driver)
tty_driver_kref_put(driver);
return ERR_PTR(-ENODEV);
}
default:
driver = get_tty_driver(device, index);
if (!driver)
return ERR_PTR(-ENODEV);
break;
}
return driver;
}
static struct tty_struct *tty_kopen(dev_t device, int shared)
{
struct tty_struct *tty;
struct tty_driver *driver;
int index = -1;
mutex_lock(&tty_mutex);
driver = tty_lookup_driver(device, NULL, &index);
if (IS_ERR(driver)) {
mutex_unlock(&tty_mutex);
return ERR_CAST(driver);
}
/* check whether we're reopening an existing tty */
tty = tty_driver_lookup_tty(driver, NULL, index);
if (IS_ERR(tty) || shared)
goto out;
if (tty) {
/* drop kref from tty_driver_lookup_tty() */
tty_kref_put(tty);
tty = ERR_PTR(-EBUSY);
} else { /* tty_init_dev returns tty with the tty_lock held */
tty = tty_init_dev(driver, index);
if (IS_ERR(tty))
goto out;
tty_port_set_kopened(tty->port, 1);
}
out:
mutex_unlock(&tty_mutex);
tty_driver_kref_put(driver);
return tty;
}
/**
* tty_kopen_exclusive - open a tty device for kernel
* @device: dev_t of device to open
*
* Opens tty exclusively for kernel. Performs the driver lookup, makes sure
* it's not already opened and performs the first-time tty initialization.
*
* Claims the global %tty_mutex to serialize:
* * concurrent first-time tty initialization
* * concurrent tty driver removal w/ lookup
* * concurrent tty removal from driver table
*
* Return: the locked initialized &tty_struct
*/
struct tty_struct *tty_kopen_exclusive(dev_t device)
{
return tty_kopen(device, 0);
}
EXPORT_SYMBOL_GPL(tty_kopen_exclusive);
/**
* tty_kopen_shared - open a tty device for shared in-kernel use
* @device: dev_t of device to open
*
* Opens an already existing tty for in-kernel use. Compared to
* tty_kopen_exclusive() above it doesn't ensure to be the only user.
*
* Locking: identical to tty_kopen() above.
*/
struct tty_struct *tty_kopen_shared(dev_t device)
{
return tty_kopen(device, 1);
}
EXPORT_SYMBOL_GPL(tty_kopen_shared);
/**
* tty_open_by_driver - open a tty device
* @device: dev_t of device to open
* @filp: file pointer to tty
*
* Performs the driver lookup, checks for a reopen, or otherwise performs the
* first-time tty initialization.
*
*
* Claims the global tty_mutex to serialize:
* * concurrent first-time tty initialization
* * concurrent tty driver removal w/ lookup
* * concurrent tty removal from driver table
*
* Return: the locked initialized or re-opened &tty_struct
*/
static struct tty_struct *tty_open_by_driver(dev_t device,
struct file *filp)
{
struct tty_struct *tty;
struct tty_driver *driver = NULL;
int index = -1;
int retval;
mutex_lock(&tty_mutex);
driver = tty_lookup_driver(device, filp, &index);
if (IS_ERR(driver)) {
mutex_unlock(&tty_mutex);
return ERR_CAST(driver);
}
/* check whether we're reopening an existing tty */
tty = tty_driver_lookup_tty(driver, filp, index);
if (IS_ERR(tty)) {
mutex_unlock(&tty_mutex);
goto out;
}
if (tty) {
if (tty_port_kopened(tty->port)) {
tty_kref_put(tty);
mutex_unlock(&tty_mutex);
tty = ERR_PTR(-EBUSY);
goto out;
}
mutex_unlock(&tty_mutex);
retval = tty_lock_interruptible(tty);
tty_kref_put(tty); /* drop kref from tty_driver_lookup_tty() */
if (retval) {
if (retval == -EINTR)
retval = -ERESTARTSYS;
tty = ERR_PTR(retval);
goto out;
}
retval = tty_reopen(tty);
if (retval < 0) {
tty_unlock(tty);
tty = ERR_PTR(retval);
}
} else { /* Returns with the tty_lock held for now */
tty = tty_init_dev(driver, index);
mutex_unlock(&tty_mutex);
}
out:
tty_driver_kref_put(driver);
return tty;
}
/**
* tty_open - open a tty device
* @inode: inode of device file
* @filp: file pointer to tty
*
* tty_open() and tty_release() keep up the tty count that contains the number
* of opens done on a tty. We cannot use the inode-count, as different inodes
* might point to the same tty.
*
* Open-counting is needed for pty masters, as well as for keeping track of
* serial lines: DTR is dropped when the last close happens.
* (This is not done solely through tty->count, now. - Ted 1/27/92)
*
* The termios state of a pty is reset on the first open so that settings don't
* persist across reuse.
*
* Locking:
* * %tty_mutex protects tty, tty_lookup_driver() and tty_init_dev().
* * @tty->count should protect the rest.
* * ->siglock protects ->signal/->sighand
*
* Note: the tty_unlock/lock cases without a ref are only safe due to %tty_mutex
*/
static int tty_open(struct inode *inode, struct file *filp)
{
struct tty_struct *tty;
int noctty, retval;
dev_t device = inode->i_rdev;
unsigned saved_flags = filp->f_flags;
nonseekable_open(inode, filp);
retry_open:
retval = tty_alloc_file(filp);
if (retval)
return -ENOMEM;
tty = tty_open_current_tty(device, filp);
if (!tty)
tty = tty_open_by_driver(device, filp);
if (IS_ERR(tty)) {
tty_free_file(filp);
retval = PTR_ERR(tty);
if (retval != -EAGAIN || signal_pending(current))
return retval;
schedule();
goto retry_open;
}
tty_add_file(tty, filp);
check_tty_count(tty, __func__);
tty_debug_hangup(tty, "opening (count=%d)\n", tty->count);
if (tty->ops->open)
retval = tty->ops->open(tty, filp);
else
retval = -ENODEV;
filp->f_flags = saved_flags;
if (retval) {
tty_debug_hangup(tty, "open error %d, releasing\n", retval);
tty_unlock(tty); /* need to call tty_release without BTM */
tty_release(inode, filp);
if (retval != -ERESTARTSYS)
return retval;
if (signal_pending(current))
return retval;
schedule();
/*
* Need to reset f_op in case a hangup happened.
*/
if (tty_hung_up_p(filp))
filp->f_op = &tty_fops;
goto retry_open;
}
clear_bit(TTY_HUPPED, &tty->flags);
noctty = (filp->f_flags & O_NOCTTY) ||
(IS_ENABLED(CONFIG_VT) && device == MKDEV(TTY_MAJOR, 0)) ||
device == MKDEV(TTYAUX_MAJOR, 1) ||
(tty->driver->type == TTY_DRIVER_TYPE_PTY &&
tty->driver->subtype == PTY_TYPE_MASTER);
if (!noctty)
tty_open_proc_set_tty(filp, tty);
tty_unlock(tty);
return 0;
}
/**
* tty_poll - check tty status
* @filp: file being polled
* @wait: poll wait structures to update
*
* Call the line discipline polling method to obtain the poll status of the
* device.
*
* Locking: locks called line discipline but ldisc poll method may be
* re-entered freely by other callers.
*/
static __poll_t tty_poll(struct file *filp, poll_table *wait)
{
struct tty_struct *tty = file_tty(filp);
struct tty_ldisc *ld;
__poll_t ret = 0;
if (tty_paranoia_check(tty, file_inode(filp), "tty_poll"))
return 0;
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return hung_up_tty_poll(filp, wait);
if (ld->ops->poll)
ret = ld->ops->poll(tty, filp, wait);
tty_ldisc_deref(ld);
return ret;
}
static int __tty_fasync(int fd, struct file *filp, int on)
{
struct tty_struct *tty = file_tty(filp);
unsigned long flags;
int retval = 0;
if (tty_paranoia_check(tty, file_inode(filp), "tty_fasync"))
goto out;
retval = fasync_helper(fd, filp, on, &tty->fasync);
if (retval <= 0)
goto out;
if (on) {
enum pid_type type;
struct pid *pid;
spin_lock_irqsave(&tty->ctrl.lock, flags);
if (tty->ctrl.pgrp) {
pid = tty->ctrl.pgrp;
type = PIDTYPE_PGID;
} else {
pid = task_pid(current);
type = PIDTYPE_TGID;
}
get_pid(pid);
spin_unlock_irqrestore(&tty->ctrl.lock, flags);
__f_setown(filp, pid, type, 0);
put_pid(pid);
retval = 0;
}
out:
return retval;
}
static int tty_fasync(int fd, struct file *filp, int on)
{
struct tty_struct *tty = file_tty(filp);
int retval = -ENOTTY;
tty_lock(tty);
if (!tty_hung_up_p(filp))
retval = __tty_fasync(fd, filp, on);
tty_unlock(tty);
return retval;
}
static bool tty_legacy_tiocsti __read_mostly = IS_ENABLED(CONFIG_LEGACY_TIOCSTI);
/**
* tiocsti - fake input character
* @tty: tty to fake input into
* @p: pointer to character
*
* Fake input to a tty device. Does the necessary locking and input management.
*
* FIXME: does not honour flow control ??
*
* Locking:
* * Called functions take tty_ldiscs_lock
* * current->signal->tty check is safe without locks
*/
static int tiocsti(struct tty_struct *tty, u8 __user *p)
{
struct tty_ldisc *ld;
u8 ch;
if (!tty_legacy_tiocsti && !capable(CAP_SYS_ADMIN))
return -EIO;
if ((current->signal->tty != tty) && !capable(CAP_SYS_ADMIN))
return -EPERM;
if (get_user(ch, p))
return -EFAULT;
tty_audit_tiocsti(tty, ch);
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return -EIO;
tty_buffer_lock_exclusive(tty->port);
if (ld->ops->receive_buf)
ld->ops->receive_buf(tty, &ch, NULL, 1);
tty_buffer_unlock_exclusive(tty->port);
tty_ldisc_deref(ld);
return 0;
}
/**
* tiocgwinsz - implement window query ioctl
* @tty: tty
* @arg: user buffer for result
*
* Copies the kernel idea of the window size into the user buffer.
*
* Locking: @tty->winsize_mutex is taken to ensure the winsize data is
* consistent.
*/
static int tiocgwinsz(struct tty_struct *tty, struct winsize __user *arg)
{
int err;
mutex_lock(&tty->winsize_mutex);
err = copy_to_user(arg, &tty->winsize, sizeof(*arg));
mutex_unlock(&tty->winsize_mutex);
return err ? -EFAULT : 0;
}
/**
* tty_do_resize - resize event
* @tty: tty being resized
* @ws: new dimensions
*
* Update the termios variables and send the necessary signals to peform a
* terminal resize correctly.
*/
int tty_do_resize(struct tty_struct *tty, struct winsize *ws)
{
struct pid *pgrp;
/* Lock the tty */
mutex_lock(&tty->winsize_mutex);
if (!memcmp(ws, &tty->winsize, sizeof(*ws)))
goto done;
/* Signal the foreground process group */
pgrp = tty_get_pgrp(tty);
if (pgrp)
kill_pgrp(pgrp, SIGWINCH, 1);
put_pid(pgrp);
tty->winsize = *ws;
done:
mutex_unlock(&tty->winsize_mutex);
return 0;
}
EXPORT_SYMBOL(tty_do_resize);
/**
* tiocswinsz - implement window size set ioctl
* @tty: tty side of tty
* @arg: user buffer for result
*
* Copies the user idea of the window size to the kernel. Traditionally this is
* just advisory information but for the Linux console it actually has driver
* level meaning and triggers a VC resize.
*
* Locking:
* Driver dependent. The default do_resize method takes the tty termios
* mutex and ctrl.lock. The console takes its own lock then calls into the
* default method.
*/
static int tiocswinsz(struct tty_struct *tty, struct winsize __user *arg)
{
struct winsize tmp_ws;
if (copy_from_user(&tmp_ws, arg, sizeof(*arg)))
return -EFAULT;
if (tty->ops->resize)
return tty->ops->resize(tty, &tmp_ws);
else
return tty_do_resize(tty, &tmp_ws);
}
/**
* tioccons - allow admin to move logical console
* @file: the file to become console
*
* Allow the administrator to move the redirected console device.
*
* Locking: uses redirect_lock to guard the redirect information
*/
static int tioccons(struct file *file)
{
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (file->f_op->write_iter == redirected_tty_write) {
struct file *f;
spin_lock(&redirect_lock);
f = redirect;
redirect = NULL;
spin_unlock(&redirect_lock);
if (f)
fput(f);
return 0;
}
if (file->f_op->write_iter != tty_write)
return -ENOTTY;
if (!(file->f_mode & FMODE_WRITE))
return -EBADF;
if (!(file->f_mode & FMODE_CAN_WRITE))
return -EINVAL;
spin_lock(&redirect_lock);
if (redirect) {
spin_unlock(&redirect_lock);
return -EBUSY;
}
redirect = get_file(file);
spin_unlock(&redirect_lock);
return 0;
}
/**
* tiocsetd - set line discipline
* @tty: tty device
* @p: pointer to user data
*
* Set the line discipline according to user request.
*
* Locking: see tty_set_ldisc(), this function is just a helper
*/
static int tiocsetd(struct tty_struct *tty, int __user *p)
{
int disc;
int ret;
if (get_user(disc, p))
return -EFAULT;
ret = tty_set_ldisc(tty, disc);
return ret;
}
/**
* tiocgetd - get line discipline
* @tty: tty device
* @p: pointer to user data
*
* Retrieves the line discipline id directly from the ldisc.
*
* Locking: waits for ldisc reference (in case the line discipline is changing
* or the @tty is being hungup)
*/
static int tiocgetd(struct tty_struct *tty, int __user *p)
{
struct tty_ldisc *ld;
int ret;
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return -EIO;
ret = put_user(ld->ops->num, p);
tty_ldisc_deref(ld);
return ret;
}
/**
* send_break - performed time break
* @tty: device to break on
* @duration: timeout in mS
*
* Perform a timed break on hardware that lacks its own driver level timed
* break functionality.
*
* Locking:
* @tty->atomic_write_lock serializes
*/
static int send_break(struct tty_struct *tty, unsigned int duration)
{
int retval;
if (tty->ops->break_ctl == NULL)
return 0;
if (tty->driver->flags & TTY_DRIVER_HARDWARE_BREAK)
return tty->ops->break_ctl(tty, duration);
/* Do the work ourselves */
if (tty_write_lock(tty, false) < 0)
return -EINTR;
retval = tty->ops->break_ctl(tty, -1);
if (!retval) {
msleep_interruptible(duration);
retval = tty->ops->break_ctl(tty, 0);
} else if (retval == -EOPNOTSUPP) {
/* some drivers can tell only dynamically */
retval = 0;
}
tty_write_unlock(tty);
if (signal_pending(current))
retval = -EINTR;
return retval;
}
/**
* tty_get_tiocm - get tiocm status register
* @tty: tty device
*
* Obtain the modem status bits from the tty driver if the feature
* is supported.
*/
int tty_get_tiocm(struct tty_struct *tty)
{
int retval = -ENOTTY;
if (tty->ops->tiocmget)
retval = tty->ops->tiocmget(tty);
return retval;
}
EXPORT_SYMBOL_GPL(tty_get_tiocm);
/**
* tty_tiocmget - get modem status
* @tty: tty device
* @p: pointer to result
*
* Obtain the modem status bits from the tty driver if the feature is
* supported. Return -%ENOTTY if it is not available.
*
* Locking: none (up to the driver)
*/
static int tty_tiocmget(struct tty_struct *tty, int __user *p)
{
int retval;
retval = tty_get_tiocm(tty);
if (retval >= 0)
retval = put_user(retval, p);
return retval;
}
/**
* tty_tiocmset - set modem status
* @tty: tty device
* @cmd: command - clear bits, set bits or set all
* @p: pointer to desired bits
*
* Set the modem status bits from the tty driver if the feature
* is supported. Return -%ENOTTY if it is not available.
*
* Locking: none (up to the driver)
*/
static int tty_tiocmset(struct tty_struct *tty, unsigned int cmd,
unsigned __user *p)
{
int retval;
unsigned int set, clear, val;
if (tty->ops->tiocmset == NULL)
return -ENOTTY;
retval = get_user(val, p);
if (retval)
return retval;
set = clear = 0;
switch (cmd) {
case TIOCMBIS:
set = val;
break;
case TIOCMBIC:
clear = val;
break;
case TIOCMSET:
set = val;
clear = ~val;
break;
}
set &= TIOCM_DTR|TIOCM_RTS|TIOCM_OUT1|TIOCM_OUT2|TIOCM_LOOP;
clear &= TIOCM_DTR|TIOCM_RTS|TIOCM_OUT1|TIOCM_OUT2|TIOCM_LOOP;
return tty->ops->tiocmset(tty, set, clear);
}
/**
* tty_get_icount - get tty statistics
* @tty: tty device
* @icount: output parameter
*
* Gets a copy of the @tty's icount statistics.
*
* Locking: none (up to the driver)
*/
int tty_get_icount(struct tty_struct *tty,
struct serial_icounter_struct *icount)
{
memset(icount, 0, sizeof(*icount));
if (tty->ops->get_icount)
return tty->ops->get_icount(tty, icount);
else
return -ENOTTY;
}
EXPORT_SYMBOL_GPL(tty_get_icount);
static int tty_tiocgicount(struct tty_struct *tty, void __user *arg)
{
struct serial_icounter_struct icount;
int retval;
retval = tty_get_icount(tty, &icount);
if (retval != 0)
return retval;
if (copy_to_user(arg, &icount, sizeof(icount)))
return -EFAULT;
return 0;
}
static int tty_set_serial(struct tty_struct *tty, struct serial_struct *ss)
{
char comm[TASK_COMM_LEN];
int flags;
flags = ss->flags & ASYNC_DEPRECATED;
if (flags)
pr_warn_ratelimited("%s: '%s' is using deprecated serial flags (with no effect): %.8x\n",
__func__, get_task_comm(comm, current), flags);
if (!tty->ops->set_serial)
return -ENOTTY;
return tty->ops->set_serial(tty, ss);
}
static int tty_tiocsserial(struct tty_struct *tty, struct serial_struct __user *ss)
{
struct serial_struct v;
if (copy_from_user(&v, ss, sizeof(*ss)))
return -EFAULT;
return tty_set_serial(tty, &v);
}
static int tty_tiocgserial(struct tty_struct *tty, struct serial_struct __user *ss)
{
struct serial_struct v;
int err;
memset(&v, 0, sizeof(v));
if (!tty->ops->get_serial)
return -ENOTTY;
err = tty->ops->get_serial(tty, &v);
if (!err && copy_to_user(ss, &v, sizeof(v)))
err = -EFAULT;
return err;
}
/*
* if pty, return the slave side (real_tty)
* otherwise, return self
*/
static struct tty_struct *tty_pair_get_tty(struct tty_struct *tty)
{
if (tty->driver->type == TTY_DRIVER_TYPE_PTY &&
tty->driver->subtype == PTY_TYPE_MASTER)
tty = tty->link;
return tty;
}
/*
* Split this up, as gcc can choke on it otherwise..
*/
long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
struct tty_struct *tty = file_tty(file);
struct tty_struct *real_tty;
void __user *p = (void __user *)arg;
int retval;
struct tty_ldisc *ld;
if (tty_paranoia_check(tty, file_inode(file), "tty_ioctl"))
return -EINVAL;
real_tty = tty_pair_get_tty(tty);
/*
* Factor out some common prep work
*/
switch (cmd) {
case TIOCSETD:
case TIOCSBRK:
case TIOCCBRK:
case TCSBRK:
case TCSBRKP:
retval = tty_check_change(tty);
if (retval)
return retval;
if (cmd != TIOCCBRK) {
tty_wait_until_sent(tty, 0);
if (signal_pending(current))
return -EINTR;
}
break;
}
/*
* Now do the stuff.
*/
switch (cmd) {
case TIOCSTI:
return tiocsti(tty, p);
case TIOCGWINSZ:
return tiocgwinsz(real_tty, p);
case TIOCSWINSZ:
return tiocswinsz(real_tty, p);
case TIOCCONS:
return real_tty != tty ? -EINVAL : tioccons(file);
case TIOCEXCL:
set_bit(TTY_EXCLUSIVE, &tty->flags);
return 0;
case TIOCNXCL:
clear_bit(TTY_EXCLUSIVE, &tty->flags);
return 0;
case TIOCGEXCL:
{
int excl = test_bit(TTY_EXCLUSIVE, &tty->flags);
return put_user(excl, (int __user *)p);
}
case TIOCGETD:
return tiocgetd(tty, p);
case TIOCSETD:
return tiocsetd(tty, p);
case TIOCVHANGUP:
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
tty_vhangup(tty);
return 0;
case TIOCGDEV:
{
unsigned int ret = new_encode_dev(tty_devnum(real_tty));
return put_user(ret, (unsigned int __user *)p);
}
/*
* Break handling
*/
case TIOCSBRK: /* Turn break on, unconditionally */
if (tty->ops->break_ctl)
return tty->ops->break_ctl(tty, -1);
return 0;
case TIOCCBRK: /* Turn break off, unconditionally */
if (tty->ops->break_ctl)
return tty->ops->break_ctl(tty, 0);
return 0;
case TCSBRK: /* SVID version: non-zero arg --> no break */
/* non-zero arg means wait for all output data
* to be sent (performed above) but don't send break.
* This is used by the tcdrain() termios function.
*/
if (!arg)
return send_break(tty, 250);
return 0;
case TCSBRKP: /* support for POSIX tcsendbreak() */
return send_break(tty, arg ? arg*100 : 250);
case TIOCMGET:
return tty_tiocmget(tty, p);
case TIOCMSET:
case TIOCMBIC:
case TIOCMBIS:
return tty_tiocmset(tty, cmd, p);
case TIOCGICOUNT:
return tty_tiocgicount(tty, p);
case TCFLSH:
switch (arg) {
case TCIFLUSH:
case TCIOFLUSH:
/* flush tty buffer and allow ldisc to process ioctl */
tty_buffer_flush(tty, NULL);
break;
}
break;
case TIOCSSERIAL:
return tty_tiocsserial(tty, p);
case TIOCGSERIAL:
return tty_tiocgserial(tty, p);
case TIOCGPTPEER:
/* Special because the struct file is needed */
return ptm_open_peer(file, tty, (int)arg);
default:
retval = tty_jobctrl_ioctl(tty, real_tty, file, cmd, arg);
if (retval != -ENOIOCTLCMD)
return retval;
}
if (tty->ops->ioctl) {
retval = tty->ops->ioctl(tty, cmd, arg);
if (retval != -ENOIOCTLCMD)
return retval;
}
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return hung_up_tty_ioctl(file, cmd, arg);
retval = -EINVAL;
if (ld->ops->ioctl) {
retval = ld->ops->ioctl(tty, cmd, arg);
if (retval == -ENOIOCTLCMD)
retval = -ENOTTY;
}
tty_ldisc_deref(ld);
return retval;
}
#ifdef CONFIG_COMPAT
struct serial_struct32 {
compat_int_t type;
compat_int_t line;
compat_uint_t port;
compat_int_t irq;
compat_int_t flags;
compat_int_t xmit_fifo_size;
compat_int_t custom_divisor;
compat_int_t baud_base;
unsigned short close_delay;
char io_type;
char reserved_char;
compat_int_t hub6;
unsigned short closing_wait; /* time to wait before closing */
unsigned short closing_wait2; /* no longer used... */
compat_uint_t iomem_base;
unsigned short iomem_reg_shift;
unsigned int port_high;
/* compat_ulong_t iomap_base FIXME */
compat_int_t reserved;
};
static int compat_tty_tiocsserial(struct tty_struct *tty,
struct serial_struct32 __user *ss)
{
struct serial_struct32 v32;
struct serial_struct v;
if (copy_from_user(&v32, ss, sizeof(*ss)))
return -EFAULT;
memcpy(&v, &v32, offsetof(struct serial_struct32, iomem_base));
v.iomem_base = compat_ptr(v32.iomem_base);
v.iomem_reg_shift = v32.iomem_reg_shift;
v.port_high = v32.port_high;
v.iomap_base = 0;
return tty_set_serial(tty, &v);
}
static int compat_tty_tiocgserial(struct tty_struct *tty,
struct serial_struct32 __user *ss)
{
struct serial_struct32 v32;
struct serial_struct v;
int err;
memset(&v, 0, sizeof(v));
memset(&v32, 0, sizeof(v32));
if (!tty->ops->get_serial)
return -ENOTTY;
err = tty->ops->get_serial(tty, &v);
if (!err) {
memcpy(&v32, &v, offsetof(struct serial_struct32, iomem_base));
v32.iomem_base = (unsigned long)v.iomem_base >> 32 ?
0xfffffff : ptr_to_compat(v.iomem_base);
v32.iomem_reg_shift = v.iomem_reg_shift;
v32.port_high = v.port_high;
if (copy_to_user(ss, &v32, sizeof(v32)))
err = -EFAULT;
}
return err;
}
static long tty_compat_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct tty_struct *tty = file_tty(file);
struct tty_ldisc *ld;
int retval = -ENOIOCTLCMD;
switch (cmd) {
case TIOCOUTQ:
case TIOCSTI:
case TIOCGWINSZ:
case TIOCSWINSZ:
case TIOCGEXCL:
case TIOCGETD:
case TIOCSETD:
case TIOCGDEV:
case TIOCMGET:
case TIOCMSET:
case TIOCMBIC:
case TIOCMBIS:
case TIOCGICOUNT:
case TIOCGPGRP:
case TIOCSPGRP:
case TIOCGSID:
case TIOCSERGETLSR:
case TIOCGRS485:
case TIOCSRS485:
#ifdef TIOCGETP
case TIOCGETP:
case TIOCSETP:
case TIOCSETN:
#endif
#ifdef TIOCGETC
case TIOCGETC:
case TIOCSETC:
#endif
#ifdef TIOCGLTC
case TIOCGLTC:
case TIOCSLTC:
#endif
case TCSETSF:
case TCSETSW:
case TCSETS:
case TCGETS:
#ifdef TCGETS2
case TCGETS2:
case TCSETSF2:
case TCSETSW2:
case TCSETS2:
#endif
case TCGETA:
case TCSETAF:
case TCSETAW:
case TCSETA:
case TIOCGLCKTRMIOS:
case TIOCSLCKTRMIOS:
#ifdef TCGETX
case TCGETX:
case TCSETX:
case TCSETXW:
case TCSETXF:
#endif
case TIOCGSOFTCAR:
case TIOCSSOFTCAR:
case PPPIOCGCHAN:
case PPPIOCGUNIT:
return tty_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
case TIOCCONS:
case TIOCEXCL:
case TIOCNXCL:
case TIOCVHANGUP:
case TIOCSBRK:
case TIOCCBRK:
case TCSBRK:
case TCSBRKP:
case TCFLSH:
case TIOCGPTPEER:
case TIOCNOTTY:
case TIOCSCTTY:
case TCXONC:
case TIOCMIWAIT:
case TIOCSERCONFIG:
return tty_ioctl(file, cmd, arg);
}
if (tty_paranoia_check(tty, file_inode(file), "tty_ioctl"))
return -EINVAL;
switch (cmd) {
case TIOCSSERIAL:
return compat_tty_tiocsserial(tty, compat_ptr(arg));
case TIOCGSERIAL:
return compat_tty_tiocgserial(tty, compat_ptr(arg));
}
if (tty->ops->compat_ioctl) {
retval = tty->ops->compat_ioctl(tty, cmd, arg);
if (retval != -ENOIOCTLCMD)
return retval;
}
ld = tty_ldisc_ref_wait(tty);
if (!ld)
return hung_up_tty_compat_ioctl(file, cmd, arg);
if (ld->ops->compat_ioctl)
retval = ld->ops->compat_ioctl(tty, cmd, arg);
if (retval == -ENOIOCTLCMD && ld->ops->ioctl)
retval = ld->ops->ioctl(tty, (unsigned long)compat_ptr(cmd),
arg);
tty_ldisc_deref(ld);
return retval;
}
#endif
static int this_tty(const void *t, struct file *file, unsigned fd)
{
if (likely(file->f_op->read_iter != tty_read))
return 0;
return file_tty(file) != t ? 0 : fd + 1;
}
/*
* This implements the "Secure Attention Key" --- the idea is to
* prevent trojan horses by killing all processes associated with this
* tty when the user hits the "Secure Attention Key". Required for
* super-paranoid applications --- see the Orange Book for more details.
*
* This code could be nicer; ideally it should send a HUP, wait a few
* seconds, then send a INT, and then a KILL signal. But you then
* have to coordinate with the init process, since all processes associated
* with the current tty must be dead before the new getty is allowed
* to spawn.
*
* Now, if it would be correct ;-/ The current code has a nasty hole -
* it doesn't catch files in flight. We may send the descriptor to ourselves
* via AF_UNIX socket, close it and later fetch from socket. FIXME.
*
* Nasty bug: do_SAK is being called in interrupt context. This can
* deadlock. We punt it up to process context. AKPM - 16Mar2001
*/
void __do_SAK(struct tty_struct *tty)
{
struct task_struct *g, *p;
struct pid *session;
int i;
unsigned long flags;
spin_lock_irqsave(&tty->ctrl.lock, flags);
session = get_pid(tty->ctrl.session);
spin_unlock_irqrestore(&tty->ctrl.lock, flags);
tty_ldisc_flush(tty);
tty_driver_flush_buffer(tty);
read_lock(&tasklist_lock);
/* Kill the entire session */
do_each_pid_task(session, PIDTYPE_SID, p) {
tty_notice(tty, "SAK: killed process %d (%s): by session\n",
task_pid_nr(p), p->comm);
group_send_sig_info(SIGKILL, SEND_SIG_PRIV, p, PIDTYPE_SID);
} while_each_pid_task(session, PIDTYPE_SID, p);
/* Now kill any processes that happen to have the tty open */
for_each_process_thread(g, p) {
if (p->signal->tty == tty) {
tty_notice(tty, "SAK: killed process %d (%s): by controlling tty\n",
task_pid_nr(p), p->comm);
group_send_sig_info(SIGKILL, SEND_SIG_PRIV, p,
PIDTYPE_SID);
continue;
}
task_lock(p);
i = iterate_fd(p->files, 0, this_tty, tty);
if (i != 0) {
tty_notice(tty, "SAK: killed process %d (%s): by fd#%d\n",
task_pid_nr(p), p->comm, i - 1);
group_send_sig_info(SIGKILL, SEND_SIG_PRIV, p,
PIDTYPE_SID);
}
task_unlock(p);
}
read_unlock(&tasklist_lock);
put_pid(session);
}
static void do_SAK_work(struct work_struct *work)
{
struct tty_struct *tty =
container_of(work, struct tty_struct, SAK_work);
__do_SAK(tty);
}
/*
* The tq handling here is a little racy - tty->SAK_work may already be queued.
* Fortunately we don't need to worry, because if ->SAK_work is already queued,
* the values which we write to it will be identical to the values which it
* already has. --akpm
*/
void do_SAK(struct tty_struct *tty)
{
if (!tty)
return;
schedule_work(&tty->SAK_work);
}
EXPORT_SYMBOL(do_SAK);
/* Must put_device() after it's unused! */
static struct device *tty_get_device(struct tty_struct *tty)
{
dev_t devt = tty_devnum(tty);
return class_find_device_by_devt(&tty_class, devt);
}
/**
* alloc_tty_struct - allocate a new tty
* @driver: driver which will handle the returned tty
* @idx: minor of the tty
*
* This subroutine allocates and initializes a tty structure.
*
* Locking: none - @tty in question is not exposed at this point
*/
struct tty_struct *alloc_tty_struct(struct tty_driver *driver, int idx)
{
struct tty_struct *tty;
tty = kzalloc(sizeof(*tty), GFP_KERNEL_ACCOUNT);
if (!tty)
return NULL;
kref_init(&tty->kref);
if (tty_ldisc_init(tty)) {
kfree(tty);
return NULL;
}
tty->ctrl.session = NULL;
tty->ctrl.pgrp = NULL;
mutex_init(&tty->legacy_mutex);
mutex_init(&tty->throttle_mutex);
init_rwsem(&tty->termios_rwsem);
mutex_init(&tty->winsize_mutex);
init_ldsem(&tty->ldisc_sem);
init_waitqueue_head(&tty->write_wait);
init_waitqueue_head(&tty->read_wait);
INIT_WORK(&tty->hangup_work, do_tty_hangup);
mutex_init(&tty->atomic_write_lock);
spin_lock_init(&tty->ctrl.lock);
spin_lock_init(&tty->flow.lock);
spin_lock_init(&tty->files_lock);
INIT_LIST_HEAD(&tty->tty_files);
INIT_WORK(&tty->SAK_work, do_SAK_work);
tty->driver = driver;
tty->ops = driver->ops;
tty->index = idx;
tty_line_name(driver, idx, tty->name);
tty->dev = tty_get_device(tty);
return tty;
}
/**
* tty_put_char - write one character to a tty
* @tty: tty
* @ch: character to write
*
* Write one byte to the @tty using the provided @tty->ops->put_char() method
* if present.
*
* Note: the specific put_char operation in the driver layer may go
* away soon. Don't call it directly, use this method
*
* Return: the number of characters successfully output.
*/
int tty_put_char(struct tty_struct *tty, u8 ch)
{
if (tty->ops->put_char)
return tty->ops->put_char(tty, ch);
return tty->ops->write(tty, &ch, 1);
}
EXPORT_SYMBOL_GPL(tty_put_char);
static int tty_cdev_add(struct tty_driver *driver, dev_t dev,
unsigned int index, unsigned int count)
{
int err;
/* init here, since reused cdevs cause crashes */
driver->cdevs[index] = cdev_alloc();
if (!driver->cdevs[index])
return -ENOMEM;
driver->cdevs[index]->ops = &tty_fops;
driver->cdevs[index]->owner = driver->owner;
err = cdev_add(driver->cdevs[index], dev, count);
if (err)
kobject_put(&driver->cdevs[index]->kobj);
return err;
}
/**
* tty_register_device - register a tty device
* @driver: the tty driver that describes the tty device
* @index: the index in the tty driver for this tty device
* @device: a struct device that is associated with this tty device.
* This field is optional, if there is no known struct device
* for this tty device it can be set to NULL safely.
*
* This call is required to be made to register an individual tty device
* if the tty driver's flags have the %TTY_DRIVER_DYNAMIC_DEV bit set. If
* that bit is not set, this function should not be called by a tty
* driver.
*
* Locking: ??
*
* Return: A pointer to the struct device for this tty device (or
* ERR_PTR(-EFOO) on error).
*/
struct device *tty_register_device(struct tty_driver *driver, unsigned index,
struct device *device)
{
return tty_register_device_attr(driver, index, device, NULL, NULL);
}
EXPORT_SYMBOL(tty_register_device);
static void tty_device_create_release(struct device *dev)
{
dev_dbg(dev, "releasing...\n"); kfree(dev);
}
/**
* tty_register_device_attr - register a tty device
* @driver: the tty driver that describes the tty device
* @index: the index in the tty driver for this tty device
* @device: a struct device that is associated with this tty device.
* This field is optional, if there is no known struct device
* for this tty device it can be set to %NULL safely.
* @drvdata: Driver data to be set to device.
* @attr_grp: Attribute group to be set on device.
*
* This call is required to be made to register an individual tty device if the
* tty driver's flags have the %TTY_DRIVER_DYNAMIC_DEV bit set. If that bit is
* not set, this function should not be called by a tty driver.
*
* Locking: ??
*
* Return: A pointer to the struct device for this tty device (or
* ERR_PTR(-EFOO) on error).
*/
struct device *tty_register_device_attr(struct tty_driver *driver,
unsigned index, struct device *device,
void *drvdata,
const struct attribute_group **attr_grp)
{
char name[64];
dev_t devt = MKDEV(driver->major, driver->minor_start) + index;
struct ktermios *tp;
struct device *dev;
int retval;
if (index >= driver->num) {
pr_err("%s: Attempt to register invalid tty line number (%d)\n",
driver->name, index);
return ERR_PTR(-EINVAL);
}
if (driver->type == TTY_DRIVER_TYPE_PTY)
pty_line_name(driver, index, name);
else
tty_line_name(driver, index, name);
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return ERR_PTR(-ENOMEM);
dev->devt = devt;
dev->class = &tty_class;
dev->parent = device;
dev->release = tty_device_create_release;
dev_set_name(dev, "%s", name);
dev->groups = attr_grp;
dev_set_drvdata(dev, drvdata);
dev_set_uevent_suppress(dev, 1);
retval = device_register(dev);
if (retval)
goto err_put;
if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) {
/*
* Free any saved termios data so that the termios state is
* reset when reusing a minor number.
*/
tp = driver->termios[index];
if (tp) {
driver->termios[index] = NULL;
kfree(tp);
}
retval = tty_cdev_add(driver, devt, index, 1);
if (retval)
goto err_del;
}
dev_set_uevent_suppress(dev, 0);
kobject_uevent(&dev->kobj, KOBJ_ADD);
return dev;
err_del:
device_del(dev);
err_put:
put_device(dev);
return ERR_PTR(retval);
}
EXPORT_SYMBOL_GPL(tty_register_device_attr);
/**
* tty_unregister_device - unregister a tty device
* @driver: the tty driver that describes the tty device
* @index: the index in the tty driver for this tty device
*
* If a tty device is registered with a call to tty_register_device() then
* this function must be called when the tty device is gone.
*
* Locking: ??
*/
void tty_unregister_device(struct tty_driver *driver, unsigned index)
{
device_destroy(&tty_class, MKDEV(driver->major, driver->minor_start) + index);
if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) {
cdev_del(driver->cdevs[index]);
driver->cdevs[index] = NULL;
}
}
EXPORT_SYMBOL(tty_unregister_device);
/**
* __tty_alloc_driver - allocate tty driver
* @lines: count of lines this driver can handle at most
* @owner: module which is responsible for this driver
* @flags: some of %TTY_DRIVER_ flags, will be set in driver->flags
*
* This should not be called directly, some of the provided macros should be
* used instead. Use IS_ERR() and friends on @retval.
*/
struct tty_driver *__tty_alloc_driver(unsigned int lines, struct module *owner,
unsigned long flags)
{
struct tty_driver *driver;
unsigned int cdevs = 1;
int err;
if (!lines || (flags & TTY_DRIVER_UNNUMBERED_NODE && lines > 1))
return ERR_PTR(-EINVAL);
driver = kzalloc(sizeof(*driver), GFP_KERNEL);
if (!driver)
return ERR_PTR(-ENOMEM);
kref_init(&driver->kref);
driver->num = lines;
driver->owner = owner;
driver->flags = flags;
if (!(flags & TTY_DRIVER_DEVPTS_MEM)) {
driver->ttys = kcalloc(lines, sizeof(*driver->ttys),
GFP_KERNEL);
driver->termios = kcalloc(lines, sizeof(*driver->termios),
GFP_KERNEL);
if (!driver->ttys || !driver->termios) {
err = -ENOMEM;
goto err_free_all;
}
}
if (!(flags & TTY_DRIVER_DYNAMIC_ALLOC)) {
driver->ports = kcalloc(lines, sizeof(*driver->ports),
GFP_KERNEL);
if (!driver->ports) {
err = -ENOMEM;
goto err_free_all;
}
cdevs = lines;
}
driver->cdevs = kcalloc(cdevs, sizeof(*driver->cdevs), GFP_KERNEL);
if (!driver->cdevs) {
err = -ENOMEM;
goto err_free_all;
}
return driver;
err_free_all:
kfree(driver->ports);
kfree(driver->ttys);
kfree(driver->termios);
kfree(driver->cdevs);
kfree(driver);
return ERR_PTR(err);
}
EXPORT_SYMBOL(__tty_alloc_driver);
static void destruct_tty_driver(struct kref *kref)
{
struct tty_driver *driver = container_of(kref, struct tty_driver, kref);
int i;
struct ktermios *tp;
if (driver->flags & TTY_DRIVER_INSTALLED) {
for (i = 0; i < driver->num; i++) {
tp = driver->termios[i];
if (tp) {
driver->termios[i] = NULL;
kfree(tp);
}
if (!(driver->flags & TTY_DRIVER_DYNAMIC_DEV))
tty_unregister_device(driver, i);
}
proc_tty_unregister_driver(driver);
if (driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)
cdev_del(driver->cdevs[0]);
}
kfree(driver->cdevs);
kfree(driver->ports);
kfree(driver->termios);
kfree(driver->ttys);
kfree(driver);
}
/**
* tty_driver_kref_put - drop a reference to a tty driver
* @driver: driver of which to drop the reference
*
* The final put will destroy and free up the driver.
*/
void tty_driver_kref_put(struct tty_driver *driver)
{
kref_put(&driver->kref, destruct_tty_driver);
}
EXPORT_SYMBOL(tty_driver_kref_put);
/**
* tty_register_driver - register a tty driver
* @driver: driver to register
*
* Called by a tty driver to register itself.
*/
int tty_register_driver(struct tty_driver *driver)
{
int error;
int i;
dev_t dev;
struct device *d;
if (!driver->major) {
error = alloc_chrdev_region(&dev, driver->minor_start,
driver->num, driver->name);
if (!error) {
driver->major = MAJOR(dev);
driver->minor_start = MINOR(dev);
}
} else {
dev = MKDEV(driver->major, driver->minor_start);
error = register_chrdev_region(dev, driver->num, driver->name);
}
if (error < 0)
goto err;
if (driver->flags & TTY_DRIVER_DYNAMIC_ALLOC) {
error = tty_cdev_add(driver, dev, 0, driver->num);
if (error)
goto err_unreg_char;
}
mutex_lock(&tty_mutex);
list_add(&driver->tty_drivers, &tty_drivers);
mutex_unlock(&tty_mutex);
if (!(driver->flags & TTY_DRIVER_DYNAMIC_DEV)) {
for (i = 0; i < driver->num; i++) {
d = tty_register_device(driver, i, NULL);
if (IS_ERR(d)) {
error = PTR_ERR(d);
goto err_unreg_devs;
}
}
}
proc_tty_register_driver(driver);
driver->flags |= TTY_DRIVER_INSTALLED;
return 0;
err_unreg_devs:
for (i--; i >= 0; i--)
tty_unregister_device(driver, i);
mutex_lock(&tty_mutex);
list_del(&driver->tty_drivers);
mutex_unlock(&tty_mutex);
err_unreg_char:
unregister_chrdev_region(dev, driver->num);
err:
return error;
}
EXPORT_SYMBOL(tty_register_driver);
/**
* tty_unregister_driver - unregister a tty driver
* @driver: driver to unregister
*
* Called by a tty driver to unregister itself.
*/
void tty_unregister_driver(struct tty_driver *driver)
{
unregister_chrdev_region(MKDEV(driver->major, driver->minor_start),
driver->num);
mutex_lock(&tty_mutex);
list_del(&driver->tty_drivers);
mutex_unlock(&tty_mutex);
}
EXPORT_SYMBOL(tty_unregister_driver);
dev_t tty_devnum(struct tty_struct *tty)
{
return MKDEV(tty->driver->major, tty->driver->minor_start) + tty->index;
}
EXPORT_SYMBOL(tty_devnum);
void tty_default_fops(struct file_operations *fops)
{
*fops = tty_fops;
}
static char *tty_devnode(const struct device *dev, umode_t *mode)
{
if (!mode)
return NULL;
if (dev->devt == MKDEV(TTYAUX_MAJOR, 0) ||
dev->devt == MKDEV(TTYAUX_MAJOR, 2))
*mode = 0666;
return NULL;
}
const struct class tty_class = {
.name = "tty",
.devnode = tty_devnode,
};
static int __init tty_class_init(void)
{
return class_register(&tty_class);
}
postcore_initcall(tty_class_init);
/* 3/2004 jmc: why do these devices exist? */
static struct cdev tty_cdev, console_cdev;
static ssize_t show_cons_active(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct console *cs[16];
int i = 0;
struct console *c;
ssize_t count = 0;
/*
* Hold the console_list_lock to guarantee that no consoles are
* unregistered until all console processing is complete.
* This also allows safe traversal of the console list and
* race-free reading of @flags.
*/
console_list_lock();
for_each_console(c) {
if (!c->device)
continue;
if (!c->write)
continue;
if ((c->flags & CON_ENABLED) == 0)
continue;
cs[i++] = c;
if (i >= ARRAY_SIZE(cs))
break;
}
/*
* Take console_lock to serialize device() callback with
* other console operations. For example, fg_console is
* modified under console_lock when switching vt.
*/
console_lock();
while (i--) {
int index = cs[i]->index;
struct tty_driver *drv = cs[i]->device(cs[i], &index);
/* don't resolve tty0 as some programs depend on it */
if (drv && (cs[i]->index > 0 || drv->major != TTY_MAJOR))
count += tty_line_name(drv, index, buf + count);
else
count += sprintf(buf + count, "%s%d",
cs[i]->name, cs[i]->index);
count += sprintf(buf + count, "%c", i ? ' ':'\n');
}
console_unlock();
console_list_unlock();
return count;
}
static DEVICE_ATTR(active, S_IRUGO, show_cons_active, NULL);
static struct attribute *cons_dev_attrs[] = {
&dev_attr_active.attr,
NULL
};
ATTRIBUTE_GROUPS(cons_dev);
static struct device *consdev;
void console_sysfs_notify(void)
{
if (consdev)
sysfs_notify(&consdev->kobj, NULL, "active");
}
static struct ctl_table tty_table[] = {
{
.procname = "legacy_tiocsti",
.data = &tty_legacy_tiocsti,
.maxlen = sizeof(tty_legacy_tiocsti),
.mode = 0644,
.proc_handler = proc_dobool,
},
{
.procname = "ldisc_autoload",
.data = &tty_ldisc_autoload,
.maxlen = sizeof(tty_ldisc_autoload),
.mode = 0644,
.proc_handler = proc_dointvec,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
};
/*
* Ok, now we can initialize the rest of the tty devices and can count
* on memory allocations, interrupts etc..
*/
int __init tty_init(void)
{
register_sysctl_init("dev/tty", tty_table);
cdev_init(&tty_cdev, &tty_fops);
if (cdev_add(&tty_cdev, MKDEV(TTYAUX_MAJOR, 0), 1) ||
register_chrdev_region(MKDEV(TTYAUX_MAJOR, 0), 1, "/dev/tty") < 0)
panic("Couldn't register /dev/tty driver\n");
device_create(&tty_class, NULL, MKDEV(TTYAUX_MAJOR, 0), NULL, "tty");
cdev_init(&console_cdev, &console_fops);
if (cdev_add(&console_cdev, MKDEV(TTYAUX_MAJOR, 1), 1) ||
register_chrdev_region(MKDEV(TTYAUX_MAJOR, 1), 1, "/dev/console") < 0)
panic("Couldn't register /dev/console driver\n");
consdev = device_create_with_groups(&tty_class, NULL,
MKDEV(TTYAUX_MAJOR, 1), NULL,
cons_dev_groups, "console");
if (IS_ERR(consdev))
consdev = NULL;
#ifdef CONFIG_VT
vty_init(&console_fops);
#endif
return 0;
}
/*
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
*
* Pentium III FXSR, SSE support
* Gareth Hughes <gareth@valinux.com>, May 2000
*/
/*
* Handle hardware traps and faults.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/context_tracking.h>
#include <linux/interrupt.h>
#include <linux/kallsyms.h>
#include <linux/kmsan.h>
#include <linux/spinlock.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/kdebug.h>
#include <linux/kgdb.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/ptrace.h>
#include <linux/uprobes.h>
#include <linux/string.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/kexec.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/timer.h>
#include <linux/init.h>
#include <linux/bug.h>
#include <linux/nmi.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/io.h>
#include <linux/hardirq.h>
#include <linux/atomic.h>
#include <linux/iommu.h>
#include <asm/stacktrace.h>
#include <asm/processor.h>
#include <asm/debugreg.h>
#include <asm/realmode.h>
#include <asm/text-patching.h>
#include <asm/ftrace.h>
#include <asm/traps.h>
#include <asm/desc.h>
#include <asm/fred.h>
#include <asm/fpu/api.h>
#include <asm/cpu.h>
#include <asm/cpu_entry_area.h>
#include <asm/mce.h>
#include <asm/fixmap.h>
#include <asm/mach_traps.h>
#include <asm/alternative.h>
#include <asm/fpu/xstate.h>
#include <asm/vm86.h>
#include <asm/umip.h>
#include <asm/insn.h>
#include <asm/insn-eval.h>
#include <asm/vdso.h>
#include <asm/tdx.h>
#include <asm/cfi.h>
#ifdef CONFIG_X86_64
#include <asm/x86_init.h>
#else
#include <asm/processor-flags.h>
#include <asm/setup.h>
#endif
#include <asm/proto.h>
DECLARE_BITMAP(system_vectors, NR_VECTORS);
__always_inline int is_valid_bugaddr(unsigned long addr)
{
if (addr < TASK_SIZE_MAX)
return 0;
/*
* We got #UD, if the text isn't readable we'd have gotten
* a different exception.
*/
return *(unsigned short *)addr == INSN_UD2;
}
static nokprobe_inline int
do_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str,
struct pt_regs *regs, long error_code)
{
if (v8086_mode(regs)) {
/*
* Traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
* On nmi (interrupt 2), do_trap should not be called.
*/
if (trapnr < X86_TRAP_UD) {
if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,
error_code, trapnr))
return 0;
}
} else if (!user_mode(regs)) {
if (fixup_exception(regs, trapnr, error_code, 0))
return 0;
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = trapnr;
die(str, regs, error_code);
} else {
if (fixup_vdso_exception(regs, trapnr, error_code, 0))
return 0;
}
/*
* We want error_code and trap_nr set for userspace faults and
* kernelspace faults which result in die(), but not
* kernelspace faults which are fixed up. die() gives the
* process no chance to handle the signal and notice the
* kernel fault information, so that won't result in polluting
* the information about previously queued, but not yet
* delivered, faults. See also exc_general_protection below.
*/
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = trapnr;
return -1;
}
static void show_signal(struct task_struct *tsk, int signr,
const char *type, const char *desc,
struct pt_regs *regs, long error_code)
{
if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
printk_ratelimit()) { pr_info("%s[%d] %s%s ip:%lx sp:%lx error:%lx",
tsk->comm, task_pid_nr(tsk), type, desc,
regs->ip, regs->sp, error_code);
print_vma_addr(KERN_CONT " in ", regs->ip);
pr_cont("\n");
}
}
static void
do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
long error_code, int sicode, void __user *addr)
{
struct task_struct *tsk = current;
if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))
return;
show_signal(tsk, signr, "trap ", str, regs, error_code);
if (!sicode)
force_sig(signr);
else
force_sig_fault(signr, sicode, addr);
}
NOKPROBE_SYMBOL(do_trap);
static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
unsigned long trapnr, int signr, int sicode, void __user *addr)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
NOTIFY_STOP) {
cond_local_irq_enable(regs);
do_trap(trapnr, signr, str, regs, error_code, sicode, addr);
cond_local_irq_disable(regs);
}
}
/*
* Posix requires to provide the address of the faulting instruction for
* SIGILL (#UD) and SIGFPE (#DE) in the si_addr member of siginfo_t.
*
* This address is usually regs->ip, but when an uprobe moved the code out
* of line then regs->ip points to the XOL code which would confuse
* anything which analyzes the fault address vs. the unmodified binary. If
* a trap happened in XOL code then uprobe maps regs->ip back to the
* original instruction address.
*/
static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs)
{
return (void __user *)uprobe_get_trap_addr(regs);
}
DEFINE_IDTENTRY(exc_divide_error)
{
do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,
FPE_INTDIV, error_get_trap_addr(regs));
}
DEFINE_IDTENTRY(exc_overflow)
{
do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);
}
#ifdef CONFIG_X86_F00F_BUG
void handle_invalid_op(struct pt_regs *regs)
#else
static inline void handle_invalid_op(struct pt_regs *regs)
#endif
{
do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,
ILL_ILLOPN, error_get_trap_addr(regs));
}
static noinstr bool handle_bug(struct pt_regs *regs)
{
bool handled = false;
/*
* Normally @regs are unpoisoned by irqentry_enter(), but handle_bug()
* is a rare case that uses @regs without passing them to
* irqentry_enter().
*/
kmsan_unpoison_entry_regs(regs);
if (!is_valid_bugaddr(regs->ip))
return handled;
/*
* All lies, just get the WARN/BUG out.
*/
instrumentation_begin();
/*
* Since we're emulating a CALL with exceptions, restore the interrupt
* state to what it was at the exception site.
*/
if (regs->flags & X86_EFLAGS_IF)
raw_local_irq_enable();
if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN ||
handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {
regs->ip += LEN_UD2;
handled = true;
}
if (regs->flags & X86_EFLAGS_IF)
raw_local_irq_disable();
instrumentation_end();
return handled;
}
DEFINE_IDTENTRY_RAW(exc_invalid_op)
{
irqentry_state_t state;
/*
* We use UD2 as a short encoding for 'CALL __WARN', as such
* handle it before exception entry to avoid recursive WARN
* in case exception entry is the one triggering WARNs.
*/
if (!user_mode(regs) && handle_bug(regs))
return;
state = irqentry_enter(regs);
instrumentation_begin();
handle_invalid_op(regs);
instrumentation_end();
irqentry_exit(regs, state);
}
DEFINE_IDTENTRY(exc_coproc_segment_overrun)
{
do_error_trap(regs, 0, "coprocessor segment overrun",
X86_TRAP_OLD_MF, SIGFPE, 0, NULL);
}
DEFINE_IDTENTRY_ERRORCODE(exc_invalid_tss)
{
do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV,
0, NULL);
}
DEFINE_IDTENTRY_ERRORCODE(exc_segment_not_present)
{
do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP,
SIGBUS, 0, NULL);
}
DEFINE_IDTENTRY_ERRORCODE(exc_stack_segment)
{
do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS,
0, NULL);
}
DEFINE_IDTENTRY_ERRORCODE(exc_alignment_check)
{
char *str = "alignment check";
if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)
return;
if (!user_mode(regs))
die("Split lock detected\n", regs, error_code);
local_irq_enable();
if (handle_user_split_lock(regs, error_code))
goto out;
do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs,
error_code, BUS_ADRALN, NULL);
out:
local_irq_disable();
}
#ifdef CONFIG_VMAP_STACK
__visible void __noreturn handle_stack_overflow(struct pt_regs *regs,
unsigned long fault_address,
struct stack_info *info)
{
const char *name = stack_type_name(info->type);
printk(KERN_EMERG "BUG: %s stack guard page was hit at %p (stack is %p..%p)\n",
name, (void *)fault_address, info->begin, info->end);
die("stack guard page", regs, 0);
/* Be absolutely certain we don't return. */
panic("%s stack guard hit", name);
}
#endif
/*
* Runs on an IST stack for x86_64 and on a special task stack for x86_32.
*
* On x86_64, this is more or less a normal kernel entry. Notwithstanding the
* SDM's warnings about double faults being unrecoverable, returning works as
* expected. Presumably what the SDM actually means is that the CPU may get
* the register state wrong on entry, so returning could be a bad idea.
*
* Various CPU engineers have promised that double faults due to an IRET fault
* while the stack is read-only are, in fact, recoverable.
*
* On x86_32, this is entered through a task gate, and regs are synthesized
* from the TSS. Returning is, in principle, okay, but changes to regs will
* be lost. If, for some reason, we need to return to a context with modified
* regs, the shim code could be adjusted to synchronize the registers.
*
* The 32bit #DF shim provides CR2 already as an argument. On 64bit it needs
* to be read before doing anything else.
*/
DEFINE_IDTENTRY_DF(exc_double_fault)
{
static const char str[] = "double fault";
struct task_struct *tsk = current;
#ifdef CONFIG_VMAP_STACK
unsigned long address = read_cr2();
struct stack_info info;
#endif
#ifdef CONFIG_X86_ESPFIX64
extern unsigned char native_irq_return_iret[];
/*
* If IRET takes a non-IST fault on the espfix64 stack, then we
* end up promoting it to a doublefault. In that case, take
* advantage of the fact that we're not using the normal (TSS.sp0)
* stack right now. We can write a fake #GP(0) frame at TSS.sp0
* and then modify our own IRET frame so that, when we return,
* we land directly at the #GP(0) vector with the stack already
* set up according to its expectations.
*
* The net result is that our #GP handler will think that we
* entered from usermode with the bad user context.
*
* No need for nmi_enter() here because we don't use RCU.
*/
if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&
regs->cs == __KERNEL_CS &&
regs->ip == (unsigned long)native_irq_return_iret)
{
struct pt_regs *gpregs = (struct pt_regs *)this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1;
unsigned long *p = (unsigned long *)regs->sp;
/*
* regs->sp points to the failing IRET frame on the
* ESPFIX64 stack. Copy it to the entry stack. This fills
* in gpregs->ss through gpregs->ip.
*
*/
gpregs->ip = p[0];
gpregs->cs = p[1];
gpregs->flags = p[2];
gpregs->sp = p[3];
gpregs->ss = p[4];
gpregs->orig_ax = 0; /* Missing (lost) #GP error code */
/*
* Adjust our frame so that we return straight to the #GP
* vector with the expected RSP value. This is safe because
* we won't enable interrupts or schedule before we invoke
* general_protection, so nothing will clobber the stack
* frame we just set up.
*
* We will enter general_protection with kernel GSBASE,
* which is what the stub expects, given that the faulting
* RIP will be the IRET instruction.
*/
regs->ip = (unsigned long)asm_exc_general_protection;
regs->sp = (unsigned long)&gpregs->orig_ax;
return;
}
#endif
irqentry_nmi_enter(regs);
instrumentation_begin();
notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_DF;
#ifdef CONFIG_VMAP_STACK
/*
* If we overflow the stack into a guard page, the CPU will fail
* to deliver #PF and will send #DF instead. Similarly, if we
* take any non-IST exception while too close to the bottom of
* the stack, the processor will get a page fault while
* delivering the exception and will generate a double fault.
*
* According to the SDM (footnote in 6.15 under "Interrupt 14 -
* Page-Fault Exception (#PF):
*
* Processors update CR2 whenever a page fault is detected. If a
* second page fault occurs while an earlier page fault is being
* delivered, the faulting linear address of the second fault will
* overwrite the contents of CR2 (replacing the previous
* address). These updates to CR2 occur even if the page fault
* results in a double fault or occurs during the delivery of a
* double fault.
*
* The logic below has a small possibility of incorrectly diagnosing
* some errors as stack overflows. For example, if the IDT or GDT
* gets corrupted such that #GP delivery fails due to a bad descriptor
* causing #GP and we hit this condition while CR2 coincidentally
* points to the stack guard page, we'll think we overflowed the
* stack. Given that we're going to panic one way or another
* if this happens, this isn't necessarily worth fixing.
*
* If necessary, we could improve the test by only diagnosing
* a stack overflow if the saved RSP points within 47 bytes of
* the bottom of the stack: if RSP == tsk_stack + 48 and we
* take an exception, the stack is already aligned and there
* will be enough room SS, RSP, RFLAGS, CS, RIP, and a
* possible error code, so a stack overflow would *not* double
* fault. With any less space left, exception delivery could
* fail, and, as a practical matter, we've overflowed the
* stack even if the actual trigger for the double fault was
* something else.
*/
if (get_stack_guard_info((void *)address, &info))
handle_stack_overflow(regs, address, &info);
#endif
pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code);
die("double fault", regs, error_code);
panic("Machine halted.");
instrumentation_end();
}
DEFINE_IDTENTRY(exc_bounds)
{
if (notify_die(DIE_TRAP, "bounds", regs, 0,
X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
return;
cond_local_irq_enable(regs);
if (!user_mode(regs))
die("bounds", regs, 0);
do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);
cond_local_irq_disable(regs);
}
enum kernel_gp_hint {
GP_NO_HINT,
GP_NON_CANONICAL,
GP_CANONICAL
};
/*
* When an uncaught #GP occurs, try to determine the memory address accessed by
* the instruction and return that address to the caller. Also, try to figure
* out whether any part of the access to that address was non-canonical.
*/
static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,
unsigned long *addr)
{
u8 insn_buf[MAX_INSN_SIZE];
struct insn insn;
int ret;
if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,
MAX_INSN_SIZE))
return GP_NO_HINT;
ret = insn_decode_kernel(&insn, insn_buf);
if (ret < 0)
return GP_NO_HINT;
*addr = (unsigned long)insn_get_addr_ref(&insn, regs);
if (*addr == -1UL)
return GP_NO_HINT;
#ifdef CONFIG_X86_64
/*
* Check that:
* - the operand is not in the kernel half
* - the last byte of the operand is not in the user canonical half
*/
if (*addr < ~__VIRTUAL_MASK &&
*addr + insn.opnd_bytes - 1 > __VIRTUAL_MASK)
return GP_NON_CANONICAL;
#endif
return GP_CANONICAL;
}
#define GPFSTR "general protection fault"
static bool fixup_iopl_exception(struct pt_regs *regs)
{
struct thread_struct *t = ¤t->thread;
unsigned char byte;
unsigned long ip;
if (!IS_ENABLED(CONFIG_X86_IOPL_IOPERM) || t->iopl_emul != 3)
return false; if (insn_get_effective_ip(regs, &ip))
return false;
if (get_user(byte, (const char __user *)ip))
return false;
if (byte != 0xfa && byte != 0xfb)
return false;
if (!t->iopl_warn && printk_ratelimit()) { pr_err("%s[%d] attempts to use CLI/STI, pretending it's a NOP, ip:%lx",
current->comm, task_pid_nr(current), ip);
print_vma_addr(KERN_CONT " in ", ip);
pr_cont("\n");
t->iopl_warn = 1;
}
regs->ip += 1;
return true;
}
/*
* The unprivileged ENQCMD instruction generates #GPs if the
* IA32_PASID MSR has not been populated. If possible, populate
* the MSR from a PASID previously allocated to the mm.
*/
static bool try_fixup_enqcmd_gp(void)
{
#ifdef CONFIG_ARCH_HAS_CPU_PASID
u32 pasid;
/*
* MSR_IA32_PASID is managed using XSAVE. Directly
* writing to the MSR is only possible when fpregs
* are valid and the fpstate is not. This is
* guaranteed when handling a userspace exception
* in *before* interrupts are re-enabled.
*/
lockdep_assert_irqs_disabled();
/*
* Hardware without ENQCMD will not generate
* #GPs that can be fixed up here.
*/
if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
return false;
/*
* If the mm has not been allocated a
* PASID, the #GP can not be fixed up.
*/
if (!mm_valid_pasid(current->mm))
return false;
pasid = mm_get_enqcmd_pasid(current->mm);
/*
* Did this thread already have its PASID activated?
* If so, the #GP must be from something else.
*/
if (current->pasid_activated)
return false;
wrmsrl(MSR_IA32_PASID, pasid | MSR_IA32_PASID_VALID);
current->pasid_activated = 1;
return true;
#else
return false;
#endif
}
static bool gp_try_fixup_and_notify(struct pt_regs *regs, int trapnr,
unsigned long error_code, const char *str,
unsigned long address)
{
if (fixup_exception(regs, trapnr, error_code, address))
return true;
current->thread.error_code = error_code;
current->thread.trap_nr = trapnr;
/*
* To be potentially processing a kprobe fault and to trust the result
* from kprobe_running(), we have to be non-preemptible.
*/
if (!preemptible() && kprobe_running() &&
kprobe_fault_handler(regs, trapnr))
return true;
return notify_die(DIE_GPF, str, regs, error_code, trapnr, SIGSEGV) == NOTIFY_STOP;
}
static void gp_user_force_sig_segv(struct pt_regs *regs, int trapnr,
unsigned long error_code, const char *str)
{
current->thread.error_code = error_code;
current->thread.trap_nr = trapnr;
show_signal(current, SIGSEGV, "", str, regs, error_code); force_sig(SIGSEGV);
}
DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
{
char desc[sizeof(GPFSTR) + 50 + 2*sizeof(unsigned long) + 1] = GPFSTR;
enum kernel_gp_hint hint = GP_NO_HINT;
unsigned long gp_addr;
if (user_mode(regs) && try_fixup_enqcmd_gp())
return;
cond_local_irq_enable(regs);
if (static_cpu_has(X86_FEATURE_UMIP)) {
if (user_mode(regs) && fixup_umip_exception(regs))
goto exit;
}
if (v8086_mode(regs)) {
local_irq_enable();
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
local_irq_disable();
return;
}
if (user_mode(regs)) {
if (fixup_iopl_exception(regs))
goto exit;
if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0))
goto exit;
gp_user_force_sig_segv(regs, X86_TRAP_GP, error_code, desc);
goto exit;
}
if (gp_try_fixup_and_notify(regs, X86_TRAP_GP, error_code, desc, 0))
goto exit;
if (error_code)
snprintf(desc, sizeof(desc), "segment-related " GPFSTR);
else
hint = get_kernel_gp_address(regs, &gp_addr);
if (hint != GP_NO_HINT)
snprintf(desc, sizeof(desc), GPFSTR ", %s 0x%lx",
(hint == GP_NON_CANONICAL) ? "probably for non-canonical address"
: "maybe for address",
gp_addr);
/*
* KASAN is interested only in the non-canonical case, clear it
* otherwise.
*/
if (hint != GP_NON_CANONICAL)
gp_addr = 0;
die_addr(desc, regs, error_code, gp_addr);
exit:
cond_local_irq_disable(regs);
}
static bool do_int3(struct pt_regs *regs)
{
int res;
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,
SIGTRAP) == NOTIFY_STOP)
return true;
#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
#ifdef CONFIG_KPROBES
if (kprobe_int3_handler(regs))
return true;
#endif
res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP);
return res == NOTIFY_STOP;
}
NOKPROBE_SYMBOL(do_int3);
static void do_int3_user(struct pt_regs *regs)
{
if (do_int3(regs))
return;
cond_local_irq_enable(regs);
do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);
cond_local_irq_disable(regs);
}
DEFINE_IDTENTRY_RAW(exc_int3)
{
/*
* poke_int3_handler() is completely self contained code; it does (and
* must) *NOT* call out to anything, lest it hits upon yet another
* INT3.
*/
if (poke_int3_handler(regs))
return;
/*
* irqentry_enter_from_user_mode() uses static_branch_{,un}likely()
* and therefore can trigger INT3, hence poke_int3_handler() must
* be done before. If the entry came from kernel mode, then use
* nmi_enter() because the INT3 could have been hit in any context
* including NMI.
*/
if (user_mode(regs)) {
irqentry_enter_from_user_mode(regs);
instrumentation_begin();
do_int3_user(regs);
instrumentation_end();
irqentry_exit_to_user_mode(regs);
} else {
irqentry_state_t irq_state = irqentry_nmi_enter(regs);
instrumentation_begin();
if (!do_int3(regs))
die("int3", regs, 0);
instrumentation_end();
irqentry_nmi_exit(regs, irq_state);
}
}
#ifdef CONFIG_X86_64
/*
* Help handler running on a per-cpu (IST or entry trampoline) stack
* to switch to the normal thread stack if the interrupted code was in
* user mode. The actual stack switch is done in entry_64.S
*/
asmlinkage __visible noinstr struct pt_regs *sync_regs(struct pt_regs *eregs)
{
struct pt_regs *regs = (struct pt_regs *)current_top_of_stack() - 1;
if (regs != eregs)
*regs = *eregs;
return regs;
}
#ifdef CONFIG_AMD_MEM_ENCRYPT
asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)
{
unsigned long sp, *stack;
struct stack_info info;
struct pt_regs *regs_ret;
/*
* In the SYSCALL entry path the RSP value comes from user-space - don't
* trust it and switch to the current kernel stack
*/
if (ip_within_syscall_gap(regs)) {
sp = current_top_of_stack();
goto sync;
}
/*
* From here on the RSP value is trusted. Now check whether entry
* happened from a safe stack. Not safe are the entry or unknown stacks,
* use the fall-back stack instead in this case.
*/
sp = regs->sp;
stack = (unsigned long *)sp;
if (!get_stack_info_noinstr(stack, current, &info) || info.type == STACK_TYPE_ENTRY ||
info.type > STACK_TYPE_EXCEPTION_LAST)
sp = __this_cpu_ist_top_va(VC2);
sync:
/*
* Found a safe stack - switch to it as if the entry didn't happen via
* IST stack. The code below only copies pt_regs, the real switch happens
* in assembly code.
*/
sp = ALIGN_DOWN(sp, 8) - sizeof(*regs_ret);
regs_ret = (struct pt_regs *)sp;
*regs_ret = *regs;
return regs_ret;
}
#endif
asmlinkage __visible noinstr struct pt_regs *fixup_bad_iret(struct pt_regs *bad_regs)
{
struct pt_regs tmp, *new_stack;
/*
* This is called from entry_64.S early in handling a fault
* caused by a bad iret to user mode. To handle the fault
* correctly, we want to move our stack frame to where it would
* be had we entered directly on the entry stack (rather than
* just below the IRET frame) and we want to pretend that the
* exception came from the IRET target.
*/
new_stack = (struct pt_regs *)__this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1;
/* Copy the IRET target to the temporary storage. */
__memcpy(&tmp.ip, (void *)bad_regs->sp, 5*8);
/* Copy the remainder of the stack from the current stack. */
__memcpy(&tmp, bad_regs, offsetof(struct pt_regs, ip));
/* Update the entry stack */
__memcpy(new_stack, &tmp, sizeof(tmp));
BUG_ON(!user_mode(new_stack));
return new_stack;
}
#endif
static bool is_sysenter_singlestep(struct pt_regs *regs)
{
/*
* We don't try for precision here. If we're anywhere in the region of
* code that can be single-stepped in the SYSENTER entry path, then
* assume that this is a useless single-step trap due to SYSENTER
* being invoked with TF set. (We don't know in advance exactly
* which instructions will be hit because BTF could plausibly
* be set.)
*/
#ifdef CONFIG_X86_32
return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <
(unsigned long)__end_SYSENTER_singlestep_region -
(unsigned long)__begin_SYSENTER_singlestep_region;
#elif defined(CONFIG_IA32_EMULATION)
return (regs->ip - (unsigned long)entry_SYSENTER_compat) <
(unsigned long)__end_entry_SYSENTER_compat -
(unsigned long)entry_SYSENTER_compat;
#else
return false;
#endif
}
static __always_inline unsigned long debug_read_clear_dr6(void)
{
unsigned long dr6;
/*
* The Intel SDM says:
*
* Certain debug exceptions may clear bits 0-3. The remaining
* contents of the DR6 register are never cleared by the
* processor. To avoid confusion in identifying debug
* exceptions, debug handlers should clear the register before
* returning to the interrupted task.
*
* Keep it simple: clear DR6 immediately.
*/
get_debugreg(dr6, 6);
set_debugreg(DR6_RESERVED, 6);
dr6 ^= DR6_RESERVED; /* Flip to positive polarity */
return dr6;
}
/*
* Our handling of the processor debug registers is non-trivial.
* We do not clear them on entry and exit from the kernel. Therefore
* it is possible to get a watchpoint trap here from inside the kernel.
* However, the code in ./ptrace.c has ensured that the user can
* only set watchpoints on userspace addresses. Therefore the in-kernel
* watchpoint trap can only occur in code which is reading/writing
* from user space. Such code must not hold kernel locks (since it
* can equally take a page fault), therefore it is safe to call
* force_sig_info even though that claims and releases locks.
*
* Code in ./signal.c ensures that the debug control register
* is restored before we deliver any signal, and therefore that
* user code runs with the correct debug control register even though
* we clear it here.
*
* Being careful here means that we don't have to be as careful in a
* lot of more complicated places (task switching can be a bit lazy
* about restoring all the debug state, and ptrace doesn't have to
* find every occurrence of the TF bit that could be saved away even
* by user code)
*
* May run on IST stack.
*/
static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)
{
/*
* Notifiers will clear bits in @dr6 to indicate the event has been
* consumed - hw_breakpoint_handler(), single_stop_cont().
*
* Notifiers will set bits in @virtual_dr6 to indicate the desire
* for signals - ptrace_triggered(), kgdb_hw_overflow_handler().
*/
if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)
return true;
return false;
}
static noinstr void exc_debug_kernel(struct pt_regs *regs, unsigned long dr6)
{
/*
* Disable breakpoints during exception handling; recursive exceptions
* are exceedingly 'fun'.
*
* Since this function is NOKPROBE, and that also applies to
* HW_BREAKPOINT_X, we can't hit a breakpoint before this (XXX except a
* HW_BREAKPOINT_W on our stack)
*
* Entry text is excluded for HW_BP_X and cpu_entry_area, which
* includes the entry stack is excluded for everything.
*
* For FRED, nested #DB should just work fine. But when a watchpoint or
* breakpoint is set in the code path which is executed by #DB handler,
* it results in an endless recursion and stack overflow. Thus we stay
* with the IDT approach, i.e., save DR7 and disable #DB.
*/
unsigned long dr7 = local_db_save();
irqentry_state_t irq_state = irqentry_nmi_enter(regs);
instrumentation_begin();
/*
* If something gets miswired and we end up here for a user mode
* #DB, we will malfunction.
*/
WARN_ON_ONCE(user_mode(regs));
if (test_thread_flag(TIF_BLOCKSTEP)) {
/*
* The SDM says "The processor clears the BTF flag when it
* generates a debug exception." but PTRACE_BLOCKSTEP requested
* it for userspace, but we just took a kernel #DB, so re-set
* BTF.
*/
unsigned long debugctl;
rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
debugctl |= DEBUGCTLMSR_BTF;
wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
}
/*
* Catch SYSENTER with TF set and clear DR_STEP. If this hit a
* watchpoint at the same time then that will still be handled.
*/
if (!cpu_feature_enabled(X86_FEATURE_FRED) &&
(dr6 & DR_STEP) && is_sysenter_singlestep(regs))
dr6 &= ~DR_STEP;
/*
* The kernel doesn't use INT1
*/
if (!dr6)
goto out;
if (notify_debug(regs, &dr6))
goto out;
/*
* The kernel doesn't use TF single-step outside of:
*
* - Kprobes, consumed through kprobe_debug_handler()
* - KGDB, consumed through notify_debug()
*
* So if we get here with DR_STEP set, something is wonky.
*
* A known way to trigger this is through QEMU's GDB stub,
* which leaks #DB into the guest and causes IST recursion.
*/
if (WARN_ON_ONCE(dr6 & DR_STEP))
regs->flags &= ~X86_EFLAGS_TF;
out:
instrumentation_end();
irqentry_nmi_exit(regs, irq_state);
local_db_restore(dr7);
}
static noinstr void exc_debug_user(struct pt_regs *regs, unsigned long dr6)
{
bool icebp;
/*
* If something gets miswired and we end up here for a kernel mode
* #DB, we will malfunction.
*/
WARN_ON_ONCE(!user_mode(regs));
/*
* NB: We can't easily clear DR7 here because
* irqentry_exit_to_usermode() can invoke ptrace, schedule, access
* user memory, etc. This means that a recursive #DB is possible. If
* this happens, that #DB will hit exc_debug_kernel() and clear DR7.
* Since we're not on the IST stack right now, everything will be
* fine.
*/
irqentry_enter_from_user_mode(regs);
instrumentation_begin();
/*
* Start the virtual/ptrace DR6 value with just the DR_STEP mask
* of the real DR6. ptrace_triggered() will set the DR_TRAPn bits.
*
* Userspace expects DR_STEP to be visible in ptrace_get_debugreg(6)
* even if it is not the result of PTRACE_SINGLESTEP.
*/
current->thread.virtual_dr6 = (dr6 & DR_STEP);
/*
* The SDM says "The processor clears the BTF flag when it
* generates a debug exception." Clear TIF_BLOCKSTEP to keep
* TIF_BLOCKSTEP in sync with the hardware BTF flag.
*/
clear_thread_flag(TIF_BLOCKSTEP);
/*
* If dr6 has no reason to give us about the origin of this trap,
* then it's very likely the result of an icebp/int01 trap.
* User wants a sigtrap for that.
*/
icebp = !dr6;
if (notify_debug(regs, &dr6))
goto out;
/* It's safe to allow irq's after DR6 has been saved */
local_irq_enable();
if (v8086_mode(regs)) {
handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);
goto out_irq;
}
/* #DB for bus lock can only be triggered from userspace. */
if (dr6 & DR_BUS_LOCK)
handle_bus_lock(regs);
/* Add the virtual_dr6 bits for signals. */
dr6 |= current->thread.virtual_dr6;
if (dr6 & (DR_STEP | DR_TRAP_BITS) || icebp)
send_sigtrap(regs, 0, get_si_code(dr6));
out_irq:
local_irq_disable();
out:
instrumentation_end();
irqentry_exit_to_user_mode(regs);
}
#ifdef CONFIG_X86_64
/* IST stack entry */
DEFINE_IDTENTRY_DEBUG(exc_debug)
{
exc_debug_kernel(regs, debug_read_clear_dr6());
}
/* User entry, runs on regular task stack */
DEFINE_IDTENTRY_DEBUG_USER(exc_debug)
{
exc_debug_user(regs, debug_read_clear_dr6());
}
#ifdef CONFIG_X86_FRED
/*
* When occurred on different ring level, i.e., from user or kernel
* context, #DB needs to be handled on different stack: User #DB on
* current task stack, while kernel #DB on a dedicated stack.
*
* This is exactly how FRED event delivery invokes an exception
* handler: ring 3 event on level 0 stack, i.e., current task stack;
* ring 0 event on the #DB dedicated stack specified in the
* IA32_FRED_STKLVLS MSR. So unlike IDT, the FRED debug exception
* entry stub doesn't do stack switch.
*/
DEFINE_FREDENTRY_DEBUG(exc_debug)
{
/*
* FRED #DB stores DR6 on the stack in the format which
* debug_read_clear_dr6() returns for the IDT entry points.
*/
unsigned long dr6 = fred_event_data(regs);
if (user_mode(regs))
exc_debug_user(regs, dr6);
else
exc_debug_kernel(regs, dr6);
}
#endif /* CONFIG_X86_FRED */
#else
/* 32 bit does not have separate entry points. */
DEFINE_IDTENTRY_RAW(exc_debug)
{
unsigned long dr6 = debug_read_clear_dr6();
if (user_mode(regs))
exc_debug_user(regs, dr6);
else
exc_debug_kernel(regs, dr6);
}
#endif
/*
* Note that we play around with the 'TS' bit in an attempt to get
* the correct behaviour even in the presence of the asynchronous
* IRQ13 behaviour
*/
static void math_error(struct pt_regs *regs, int trapnr)
{
struct task_struct *task = current;
struct fpu *fpu = &task->thread.fpu;
int si_code;
char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
"simd exception";
cond_local_irq_enable(regs);
if (!user_mode(regs)) {
if (fixup_exception(regs, trapnr, 0, 0))
goto exit;
task->thread.error_code = 0;
task->thread.trap_nr = trapnr;
if (notify_die(DIE_TRAP, str, regs, 0, trapnr,
SIGFPE) != NOTIFY_STOP)
die(str, regs, 0);
goto exit;
}
/*
* Synchronize the FPU register state to the memory register state
* if necessary. This allows the exception handler to inspect it.
*/
fpu_sync_fpstate(fpu);
task->thread.trap_nr = trapnr;
task->thread.error_code = 0;
si_code = fpu__exception_code(fpu, trapnr);
/* Retry when we get spurious exceptions: */
if (!si_code)
goto exit;
if (fixup_vdso_exception(regs, trapnr, 0, 0))
goto exit;
force_sig_fault(SIGFPE, si_code,
(void __user *)uprobe_get_trap_addr(regs));
exit:
cond_local_irq_disable(regs);
}
DEFINE_IDTENTRY(exc_coprocessor_error)
{
math_error(regs, X86_TRAP_MF);
}
DEFINE_IDTENTRY(exc_simd_coprocessor_error)
{
if (IS_ENABLED(CONFIG_X86_INVD_BUG)) {
/* AMD 486 bug: INVD in CPL 0 raises #XF instead of #GP */
if (!static_cpu_has(X86_FEATURE_XMM)) {
__exc_general_protection(regs, 0);
return;
}
}
math_error(regs, X86_TRAP_XF);
}
DEFINE_IDTENTRY(exc_spurious_interrupt_bug)
{
/*
* This addresses a Pentium Pro Erratum:
*
* PROBLEM: If the APIC subsystem is configured in mixed mode with
* Virtual Wire mode implemented through the local APIC, an
* interrupt vector of 0Fh (Intel reserved encoding) may be
* generated by the local APIC (Int 15). This vector may be
* generated upon receipt of a spurious interrupt (an interrupt
* which is removed before the system receives the INTA sequence)
* instead of the programmed 8259 spurious interrupt vector.
*
* IMPLICATION: The spurious interrupt vector programmed in the
* 8259 is normally handled by an operating system's spurious
* interrupt handler. However, a vector of 0Fh is unknown to some
* operating systems, which would crash if this erratum occurred.
*
* In theory this could be limited to 32bit, but the handler is not
* hurting and who knows which other CPUs suffer from this.
*/
}
static bool handle_xfd_event(struct pt_regs *regs)
{
u64 xfd_err;
int err;
if (!IS_ENABLED(CONFIG_X86_64) || !cpu_feature_enabled(X86_FEATURE_XFD))
return false;
rdmsrl(MSR_IA32_XFD_ERR, xfd_err);
if (!xfd_err)
return false;
wrmsrl(MSR_IA32_XFD_ERR, 0);
/* Die if that happens in kernel space */
if (WARN_ON(!user_mode(regs)))
return false;
local_irq_enable();
err = xfd_enable_feature(xfd_err);
switch (err) {
case -EPERM:
force_sig_fault(SIGILL, ILL_ILLOPC, error_get_trap_addr(regs));
break;
case -EFAULT:
force_sig(SIGSEGV);
break;
}
local_irq_disable();
return true;
}
DEFINE_IDTENTRY(exc_device_not_available)
{
unsigned long cr0 = read_cr0();
if (handle_xfd_event(regs))
return;
#ifdef CONFIG_MATH_EMULATION
if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) {
struct math_emu_info info = { };
cond_local_irq_enable(regs);
info.regs = regs;
math_emulate(&info);
cond_local_irq_disable(regs);
return;
}
#endif
/* This should not happen. */
if (WARN(cr0 & X86_CR0_TS, "CR0.TS was set")) {
/* Try to fix it up and carry on. */
write_cr0(cr0 & ~X86_CR0_TS);
} else {
/*
* Something terrible happened, and we're better off trying
* to kill the task than getting stuck in a never-ending
* loop of #NM faults.
*/
die("unexpected #NM exception", regs, 0);
}
}
#ifdef CONFIG_INTEL_TDX_GUEST
#define VE_FAULT_STR "VE fault"
static void ve_raise_fault(struct pt_regs *regs, long error_code,
unsigned long address)
{
if (user_mode(regs)) {
gp_user_force_sig_segv(regs, X86_TRAP_VE, error_code, VE_FAULT_STR);
return;
}
if (gp_try_fixup_and_notify(regs, X86_TRAP_VE, error_code,
VE_FAULT_STR, address)) {
return;
}
die_addr(VE_FAULT_STR, regs, error_code, address);
}
/*
* Virtualization Exceptions (#VE) are delivered to TDX guests due to
* specific guest actions which may happen in either user space or the
* kernel:
*
* * Specific instructions (WBINVD, for example)
* * Specific MSR accesses
* * Specific CPUID leaf accesses
* * Access to specific guest physical addresses
*
* In the settings that Linux will run in, virtualization exceptions are
* never generated on accesses to normal, TD-private memory that has been
* accepted (by BIOS or with tdx_enc_status_changed()).
*
* Syscall entry code has a critical window where the kernel stack is not
* yet set up. Any exception in this window leads to hard to debug issues
* and can be exploited for privilege escalation. Exceptions in the NMI
* entry code also cause issues. Returning from the exception handler with
* IRET will re-enable NMIs and nested NMI will corrupt the NMI stack.
*
* For these reasons, the kernel avoids #VEs during the syscall gap and
* the NMI entry code. Entry code paths do not access TD-shared memory,
* MMIO regions, use #VE triggering MSRs, instructions, or CPUID leaves
* that might generate #VE. VMM can remove memory from TD at any point,
* but access to unaccepted (or missing) private memory leads to VM
* termination, not to #VE.
*
* Similarly to page faults and breakpoints, #VEs are allowed in NMI
* handlers once the kernel is ready to deal with nested NMIs.
*
* During #VE delivery, all interrupts, including NMIs, are blocked until
* TDGETVEINFO is called. It prevents #VE nesting until the kernel reads
* the VE info.
*
* If a guest kernel action which would normally cause a #VE occurs in
* the interrupt-disabled region before TDGETVEINFO, a #DF (fault
* exception) is delivered to the guest which will result in an oops.
*
* The entry code has been audited carefully for following these expectations.
* Changes in the entry code have to be audited for correctness vs. this
* aspect. Similarly to #PF, #VE in these places will expose kernel to
* privilege escalation or may lead to random crashes.
*/
DEFINE_IDTENTRY(exc_virtualization_exception)
{
struct ve_info ve;
/*
* NMIs/Machine-checks/Interrupts will be in a disabled state
* till TDGETVEINFO TDCALL is executed. This ensures that VE
* info cannot be overwritten by a nested #VE.
*/
tdx_get_ve_info(&ve);
cond_local_irq_enable(regs);
/*
* If tdx_handle_virt_exception() could not process
* it successfully, treat it as #GP(0) and handle it.
*/
if (!tdx_handle_virt_exception(regs, &ve))
ve_raise_fault(regs, 0, ve.gla);
cond_local_irq_disable(regs);
}
#endif
#ifdef CONFIG_X86_32
DEFINE_IDTENTRY_SW(iret_error)
{
local_irq_enable();
if (notify_die(DIE_TRAP, "iret exception", regs, 0,
X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {
do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0,
ILL_BADSTK, (void __user *)NULL);
}
local_irq_disable();
}
#endif
/* Do not enable FRED by default yet. */
static bool enable_fred __ro_after_init = false;
#ifdef CONFIG_X86_FRED
static int __init fred_setup(char *str)
{
if (!str)
return -EINVAL;
if (!cpu_feature_enabled(X86_FEATURE_FRED))
return 0;
if (!strcmp(str, "on"))
enable_fred = true;
else if (!strcmp(str, "off"))
enable_fred = false;
else
pr_warn("invalid FRED option: 'fred=%s'\n", str);
return 0;
}
early_param("fred", fred_setup);
#endif
void __init trap_init(void)
{
if (cpu_feature_enabled(X86_FEATURE_FRED) && !enable_fred)
setup_clear_cpu_cap(X86_FEATURE_FRED);
/* Init cpu_entry_area before IST entries are set up */
setup_cpu_entry_areas();
/* Init GHCB memory pages when running as an SEV-ES guest */
sev_es_init_vc_handling();
/* Initialize TSS before setting up traps so ISTs work */
cpu_init_exception_handling();
/* Setup traps as cpu_init() might #GP */
if (!cpu_feature_enabled(X86_FEATURE_FRED))
idt_setup_traps();
cpu_init();
}
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* net/core/dev_addr_lists.c - Functions for handling net device lists
* Copyright (c) 2010 Jiri Pirko <jpirko@redhat.com>
*
* This file contains functions for working with unicast, multicast and device
* addresses lists.
*/
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/export.h>
#include <linux/list.h>
#include "dev.h"
/*
* General list handling functions
*/
static int __hw_addr_insert(struct netdev_hw_addr_list *list,
struct netdev_hw_addr *new, int addr_len)
{
struct rb_node **ins_point = &list->tree.rb_node, *parent = NULL;
struct netdev_hw_addr *ha;
while (*ins_point) {
int diff;
ha = rb_entry(*ins_point, struct netdev_hw_addr, node);
diff = memcmp(new->addr, ha->addr, addr_len);
if (diff == 0)
diff = memcmp(&new->type, &ha->type, sizeof(new->type));
parent = *ins_point;
if (diff < 0)
ins_point = &parent->rb_left;
else if (diff > 0)
ins_point = &parent->rb_right;
else
return -EEXIST;
}
rb_link_node_rcu(&new->node, parent, ins_point);
rb_insert_color(&new->node, &list->tree);
return 0;
}
static struct netdev_hw_addr*
__hw_addr_create(const unsigned char *addr, int addr_len,
unsigned char addr_type, bool global, bool sync)
{
struct netdev_hw_addr *ha;
int alloc_size;
alloc_size = sizeof(*ha);
if (alloc_size < L1_CACHE_BYTES)
alloc_size = L1_CACHE_BYTES;
ha = kmalloc(alloc_size, GFP_ATOMIC);
if (!ha)
return NULL;
memcpy(ha->addr, addr, addr_len);
ha->type = addr_type;
ha->refcount = 1;
ha->global_use = global;
ha->synced = sync ? 1 : 0;
ha->sync_cnt = 0;
return ha;
}
static int __hw_addr_add_ex(struct netdev_hw_addr_list *list,
const unsigned char *addr, int addr_len,
unsigned char addr_type, bool global, bool sync,
int sync_count, bool exclusive)
{
struct rb_node **ins_point = &list->tree.rb_node, *parent = NULL;
struct netdev_hw_addr *ha;
if (addr_len > MAX_ADDR_LEN)
return -EINVAL;
while (*ins_point) {
int diff;
ha = rb_entry(*ins_point, struct netdev_hw_addr, node);
diff = memcmp(addr, ha->addr, addr_len);
if (diff == 0)
diff = memcmp(&addr_type, &ha->type, sizeof(addr_type));
parent = *ins_point;
if (diff < 0) {
ins_point = &parent->rb_left;
} else if (diff > 0) {
ins_point = &parent->rb_right;
} else {
if (exclusive)
return -EEXIST;
if (global) {
/* check if addr is already used as global */
if (ha->global_use)
return 0;
else
ha->global_use = true;
}
if (sync) {
if (ha->synced && sync_count)
return -EEXIST;
else
ha->synced++;
}
ha->refcount++;
return 0;
}
}
ha = __hw_addr_create(addr, addr_len, addr_type, global, sync);
if (!ha)
return -ENOMEM;
rb_link_node(&ha->node, parent, ins_point);
rb_insert_color(&ha->node, &list->tree);
list_add_tail_rcu(&ha->list, &list->list);
list->count++;
return 0;
}
static int __hw_addr_add(struct netdev_hw_addr_list *list,
const unsigned char *addr, int addr_len,
unsigned char addr_type)
{
return __hw_addr_add_ex(list, addr, addr_len, addr_type, false, false,
0, false);
}
static int __hw_addr_del_entry(struct netdev_hw_addr_list *list,
struct netdev_hw_addr *ha, bool global,
bool sync)
{
if (global && !ha->global_use)
return -ENOENT;
if (sync && !ha->synced)
return -ENOENT;
if (global)
ha->global_use = false;
if (sync)
ha->synced--; if (--ha->refcount) return 0; rb_erase(&ha->node, &list->tree); list_del_rcu(&ha->list); kfree_rcu(ha, rcu_head); list->count--;
return 0;
}
static struct netdev_hw_addr *__hw_addr_lookup(struct netdev_hw_addr_list *list,
const unsigned char *addr, int addr_len,
unsigned char addr_type)
{
struct rb_node *node;
node = list->tree.rb_node;
while (node) { struct netdev_hw_addr *ha = rb_entry(node, struct netdev_hw_addr, node); int diff = memcmp(addr, ha->addr, addr_len); if (diff == 0 && addr_type) diff = memcmp(&addr_type, &ha->type, sizeof(addr_type)); if (diff < 0) node = node->rb_left; else if (diff > 0) node = node->rb_right;
else
return ha;
}
return NULL;
}
static int __hw_addr_del_ex(struct netdev_hw_addr_list *list,
const unsigned char *addr, int addr_len,
unsigned char addr_type, bool global, bool sync)
{
struct netdev_hw_addr *ha = __hw_addr_lookup(list, addr, addr_len, addr_type);
if (!ha)
return -ENOENT;
return __hw_addr_del_entry(list, ha, global, sync);
}
static int __hw_addr_del(struct netdev_hw_addr_list *list,
const unsigned char *addr, int addr_len,
unsigned char addr_type)
{
return __hw_addr_del_ex(list, addr, addr_len, addr_type, false, false);
}
static int __hw_addr_sync_one(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr *ha,
int addr_len)
{
int err;
err = __hw_addr_add_ex(to_list, ha->addr, addr_len, ha->type,
false, true, ha->sync_cnt, false);
if (err && err != -EEXIST)
return err;
if (!err) {
ha->sync_cnt++;
ha->refcount++;
}
return 0;
}
static void __hw_addr_unsync_one(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr_list *from_list,
struct netdev_hw_addr *ha,
int addr_len)
{
int err;
err = __hw_addr_del_ex(to_list, ha->addr, addr_len, ha->type,
false, true);
if (err)
return;
ha->sync_cnt--;
/* address on from list is not marked synced */
__hw_addr_del_entry(from_list, ha, false, false);
}
static int __hw_addr_sync_multiple(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr_list *from_list,
int addr_len)
{
int err = 0;
struct netdev_hw_addr *ha, *tmp;
list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
if (ha->sync_cnt == ha->refcount) {
__hw_addr_unsync_one(to_list, from_list, ha, addr_len);
} else {
err = __hw_addr_sync_one(to_list, ha, addr_len);
if (err)
break;
}
}
return err;
}
/* This function only works where there is a strict 1-1 relationship
* between source and destionation of they synch. If you ever need to
* sync addresses to more then 1 destination, you need to use
* __hw_addr_sync_multiple().
*/
int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr_list *from_list,
int addr_len)
{
int err = 0;
struct netdev_hw_addr *ha, *tmp;
list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
if (!ha->sync_cnt) {
err = __hw_addr_sync_one(to_list, ha, addr_len);
if (err)
break;
} else if (ha->refcount == 1)
__hw_addr_unsync_one(to_list, from_list, ha, addr_len);
}
return err;
}
EXPORT_SYMBOL(__hw_addr_sync);
void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr_list *from_list,
int addr_len)
{
struct netdev_hw_addr *ha, *tmp;
list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
if (ha->sync_cnt)
__hw_addr_unsync_one(to_list, from_list, ha, addr_len);
}
}
EXPORT_SYMBOL(__hw_addr_unsync);
/**
* __hw_addr_sync_dev - Synchonize device's multicast list
* @list: address list to syncronize
* @dev: device to sync
* @sync: function to call if address should be added
* @unsync: function to call if address should be removed
*
* This function is intended to be called from the ndo_set_rx_mode
* function of devices that require explicit address add/remove
* notifications. The unsync function may be NULL in which case
* the addresses requiring removal will simply be removed without
* any notification to the device.
**/
int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*sync)(struct net_device *, const unsigned char *),
int (*unsync)(struct net_device *,
const unsigned char *))
{
struct netdev_hw_addr *ha, *tmp;
int err;
/* first go through and flush out any stale entries */
list_for_each_entry_safe(ha, tmp, &list->list, list) {
if (!ha->sync_cnt || ha->refcount != 1)
continue;
/* if unsync is defined and fails defer unsyncing address */
if (unsync && unsync(dev, ha->addr))
continue;
ha->sync_cnt--;
__hw_addr_del_entry(list, ha, false, false);
}
/* go through and sync new entries to the list */
list_for_each_entry_safe(ha, tmp, &list->list, list) {
if (ha->sync_cnt)
continue;
err = sync(dev, ha->addr);
if (err)
return err;
ha->sync_cnt++;
ha->refcount++;
}
return 0;
}
EXPORT_SYMBOL(__hw_addr_sync_dev);
/**
* __hw_addr_ref_sync_dev - Synchronize device's multicast address list taking
* into account references
* @list: address list to synchronize
* @dev: device to sync
* @sync: function to call if address or reference on it should be added
* @unsync: function to call if address or some reference on it should removed
*
* This function is intended to be called from the ndo_set_rx_mode
* function of devices that require explicit address or references on it
* add/remove notifications. The unsync function may be NULL in which case
* the addresses or references on it requiring removal will simply be
* removed without any notification to the device. That is responsibility of
* the driver to identify and distribute address or references on it between
* internal address tables.
**/
int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*sync)(struct net_device *,
const unsigned char *, int),
int (*unsync)(struct net_device *,
const unsigned char *, int))
{
struct netdev_hw_addr *ha, *tmp;
int err, ref_cnt;
/* first go through and flush out any unsynced/stale entries */
list_for_each_entry_safe(ha, tmp, &list->list, list) {
/* sync if address is not used */
if ((ha->sync_cnt << 1) <= ha->refcount)
continue;
/* if fails defer unsyncing address */
ref_cnt = ha->refcount - ha->sync_cnt;
if (unsync && unsync(dev, ha->addr, ref_cnt))
continue;
ha->refcount = (ref_cnt << 1) + 1;
ha->sync_cnt = ref_cnt;
__hw_addr_del_entry(list, ha, false, false);
}
/* go through and sync updated/new entries to the list */
list_for_each_entry_safe(ha, tmp, &list->list, list) {
/* sync if address added or reused */
if ((ha->sync_cnt << 1) >= ha->refcount)
continue;
ref_cnt = ha->refcount - ha->sync_cnt;
err = sync(dev, ha->addr, ref_cnt);
if (err)
return err;
ha->refcount = ref_cnt << 1;
ha->sync_cnt = ref_cnt;
}
return 0;
}
EXPORT_SYMBOL(__hw_addr_ref_sync_dev);
/**
* __hw_addr_ref_unsync_dev - Remove synchronized addresses and references on
* it from device
* @list: address list to remove synchronized addresses (references on it) from
* @dev: device to sync
* @unsync: function to call if address and references on it should be removed
*
* Remove all addresses that were added to the device by
* __hw_addr_ref_sync_dev(). This function is intended to be called from the
* ndo_stop or ndo_open functions on devices that require explicit address (or
* references on it) add/remove notifications. If the unsync function pointer
* is NULL then this function can be used to just reset the sync_cnt for the
* addresses in the list.
**/
void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*unsync)(struct net_device *,
const unsigned char *, int))
{
struct netdev_hw_addr *ha, *tmp;
list_for_each_entry_safe(ha, tmp, &list->list, list) {
if (!ha->sync_cnt)
continue;
/* if fails defer unsyncing address */
if (unsync && unsync(dev, ha->addr, ha->sync_cnt))
continue;
ha->refcount -= ha->sync_cnt - 1;
ha->sync_cnt = 0;
__hw_addr_del_entry(list, ha, false, false);
}
}
EXPORT_SYMBOL(__hw_addr_ref_unsync_dev);
/**
* __hw_addr_unsync_dev - Remove synchronized addresses from device
* @list: address list to remove synchronized addresses from
* @dev: device to sync
* @unsync: function to call if address should be removed
*
* Remove all addresses that were added to the device by __hw_addr_sync_dev().
* This function is intended to be called from the ndo_stop or ndo_open
* functions on devices that require explicit address add/remove
* notifications. If the unsync function pointer is NULL then this function
* can be used to just reset the sync_cnt for the addresses in the list.
**/
void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*unsync)(struct net_device *,
const unsigned char *))
{
struct netdev_hw_addr *ha, *tmp;
list_for_each_entry_safe(ha, tmp, &list->list, list) {
if (!ha->sync_cnt)
continue;
/* if unsync is defined and fails defer unsyncing address */
if (unsync && unsync(dev, ha->addr))
continue;
ha->sync_cnt--;
__hw_addr_del_entry(list, ha, false, false);
}
}
EXPORT_SYMBOL(__hw_addr_unsync_dev);
static void __hw_addr_flush(struct netdev_hw_addr_list *list)
{
struct netdev_hw_addr *ha, *tmp;
list->tree = RB_ROOT; list_for_each_entry_safe(ha, tmp, &list->list, list) { list_del_rcu(&ha->list); kfree_rcu(ha, rcu_head);
}
list->count = 0;
}
void __hw_addr_init(struct netdev_hw_addr_list *list)
{
INIT_LIST_HEAD(&list->list);
list->count = 0;
list->tree = RB_ROOT;
}
EXPORT_SYMBOL(__hw_addr_init);
/*
* Device addresses handling functions
*/
/* Check that netdev->dev_addr is not written to directly as this would
* break the rbtree layout. All changes should go thru dev_addr_set() and co.
* Remove this check in mid-2024.
*/
void dev_addr_check(struct net_device *dev)
{
if (!memcmp(dev->dev_addr, dev->dev_addr_shadow, MAX_ADDR_LEN))
return;
netdev_warn(dev, "Current addr: %*ph\n", MAX_ADDR_LEN, dev->dev_addr);
netdev_warn(dev, "Expected addr: %*ph\n",
MAX_ADDR_LEN, dev->dev_addr_shadow);
netdev_WARN(dev, "Incorrect netdev->dev_addr\n");
}
/**
* dev_addr_flush - Flush device address list
* @dev: device
*
* Flush device address list and reset ->dev_addr.
*
* The caller must hold the rtnl_mutex.
*/
void dev_addr_flush(struct net_device *dev)
{
/* rtnl_mutex must be held here */
dev_addr_check(dev);
__hw_addr_flush(&dev->dev_addrs);
dev->dev_addr = NULL;
}
/**
* dev_addr_init - Init device address list
* @dev: device
*
* Init device address list and create the first element,
* used by ->dev_addr.
*
* The caller must hold the rtnl_mutex.
*/
int dev_addr_init(struct net_device *dev)
{
unsigned char addr[MAX_ADDR_LEN];
struct netdev_hw_addr *ha;
int err;
/* rtnl_mutex must be held here */
__hw_addr_init(&dev->dev_addrs);
memset(addr, 0, sizeof(addr));
err = __hw_addr_add(&dev->dev_addrs, addr, sizeof(addr),
NETDEV_HW_ADDR_T_LAN);
if (!err) {
/*
* Get the first (previously created) address from the list
* and set dev_addr pointer to this location.
*/
ha = list_first_entry(&dev->dev_addrs.list,
struct netdev_hw_addr, list);
dev->dev_addr = ha->addr;
}
return err;
}
void dev_addr_mod(struct net_device *dev, unsigned int offset,
const void *addr, size_t len)
{
struct netdev_hw_addr *ha;
dev_addr_check(dev);
ha = container_of(dev->dev_addr, struct netdev_hw_addr, addr[0]);
rb_erase(&ha->node, &dev->dev_addrs.tree);
memcpy(&ha->addr[offset], addr, len);
memcpy(&dev->dev_addr_shadow[offset], addr, len);
WARN_ON(__hw_addr_insert(&dev->dev_addrs, ha, dev->addr_len));
}
EXPORT_SYMBOL(dev_addr_mod);
/**
* dev_addr_add - Add a device address
* @dev: device
* @addr: address to add
* @addr_type: address type
*
* Add a device address to the device or increase the reference count if
* it already exists.
*
* The caller must hold the rtnl_mutex.
*/
int dev_addr_add(struct net_device *dev, const unsigned char *addr,
unsigned char addr_type)
{
int err;
ASSERT_RTNL();
err = dev_pre_changeaddr_notify(dev, addr, NULL);
if (err)
return err;
err = __hw_addr_add(&dev->dev_addrs, addr, dev->addr_len, addr_type);
if (!err)
call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
return err;
}
EXPORT_SYMBOL(dev_addr_add);
/**
* dev_addr_del - Release a device address.
* @dev: device
* @addr: address to delete
* @addr_type: address type
*
* Release reference to a device address and remove it from the device
* if the reference count drops to zero.
*
* The caller must hold the rtnl_mutex.
*/
int dev_addr_del(struct net_device *dev, const unsigned char *addr,
unsigned char addr_type)
{
int err;
struct netdev_hw_addr *ha;
ASSERT_RTNL();
/*
* We can not remove the first address from the list because
* dev->dev_addr points to that.
*/
ha = list_first_entry(&dev->dev_addrs.list,
struct netdev_hw_addr, list);
if (!memcmp(ha->addr, addr, dev->addr_len) &&
ha->type == addr_type && ha->refcount == 1)
return -ENOENT;
err = __hw_addr_del(&dev->dev_addrs, addr, dev->addr_len,
addr_type);
if (!err)
call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
return err;
}
EXPORT_SYMBOL(dev_addr_del);
/*
* Unicast list handling functions
*/
/**
* dev_uc_add_excl - Add a global secondary unicast address
* @dev: device
* @addr: address to add
*/
int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr)
{
int err;
netif_addr_lock_bh(dev);
err = __hw_addr_add_ex(&dev->uc, addr, dev->addr_len,
NETDEV_HW_ADDR_T_UNICAST, true, false,
0, true);
if (!err)
__dev_set_rx_mode(dev);
netif_addr_unlock_bh(dev);
return err;
}
EXPORT_SYMBOL(dev_uc_add_excl);
/**
* dev_uc_add - Add a secondary unicast address
* @dev: device
* @addr: address to add
*
* Add a secondary unicast address to the device or increase
* the reference count if it already exists.
*/
int dev_uc_add(struct net_device *dev, const unsigned char *addr)
{
int err;
netif_addr_lock_bh(dev);
err = __hw_addr_add(&dev->uc, addr, dev->addr_len,
NETDEV_HW_ADDR_T_UNICAST);
if (!err)
__dev_set_rx_mode(dev);
netif_addr_unlock_bh(dev);
return err;
}
EXPORT_SYMBOL(dev_uc_add);
/**
* dev_uc_del - Release secondary unicast address.
* @dev: device
* @addr: address to delete
*
* Release reference to a secondary unicast address and remove it
* from the device if the reference count drops to zero.
*/
int dev_uc_del(struct net_device *dev, const unsigned char *addr)
{
int err;
netif_addr_lock_bh(dev);
err = __hw_addr_del(&dev->uc, addr, dev->addr_len,
NETDEV_HW_ADDR_T_UNICAST);
if (!err)
__dev_set_rx_mode(dev);
netif_addr_unlock_bh(dev);
return err;
}
EXPORT_SYMBOL(dev_uc_del);
/**
* dev_uc_sync - Synchronize device's unicast list to another device
* @to: destination device
* @from: source device
*
* Add newly added addresses to the destination device and release
* addresses that have no users left. The source device must be
* locked by netif_addr_lock_bh.
*
* This function is intended to be called from the dev->set_rx_mode
* function of layered software devices. This function assumes that
* addresses will only ever be synced to the @to devices and no other.
*/
int dev_uc_sync(struct net_device *to, struct net_device *from)
{
int err = 0;
if (to->addr_len != from->addr_len)
return -EINVAL;
netif_addr_lock(to);
err = __hw_addr_sync(&to->uc, &from->uc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
netif_addr_unlock(to);
return err;
}
EXPORT_SYMBOL(dev_uc_sync);
/**
* dev_uc_sync_multiple - Synchronize device's unicast list to another
* device, but allow for multiple calls to sync to multiple devices.
* @to: destination device
* @from: source device
*
* Add newly added addresses to the destination device and release
* addresses that have been deleted from the source. The source device
* must be locked by netif_addr_lock_bh.
*
* This function is intended to be called from the dev->set_rx_mode
* function of layered software devices. It allows for a single source
* device to be synced to multiple destination devices.
*/
int dev_uc_sync_multiple(struct net_device *to, struct net_device *from)
{
int err = 0;
if (to->addr_len != from->addr_len)
return -EINVAL;
netif_addr_lock(to);
err = __hw_addr_sync_multiple(&to->uc, &from->uc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
netif_addr_unlock(to);
return err;
}
EXPORT_SYMBOL(dev_uc_sync_multiple);
/**
* dev_uc_unsync - Remove synchronized addresses from the destination device
* @to: destination device
* @from: source device
*
* Remove all addresses that were added to the destination device by
* dev_uc_sync(). This function is intended to be called from the
* dev->stop function of layered software devices.
*/
void dev_uc_unsync(struct net_device *to, struct net_device *from)
{
if (to->addr_len != from->addr_len)
return;
/* netif_addr_lock_bh() uses lockdep subclass 0, this is okay for two
* reasons:
* 1) This is always called without any addr_list_lock, so as the
* outermost one here, it must be 0.
* 2) This is called by some callers after unlinking the upper device,
* so the dev->lower_level becomes 1 again.
* Therefore, the subclass for 'from' is 0, for 'to' is either 1 or
* larger.
*/
netif_addr_lock_bh(from);
netif_addr_lock(to);
__hw_addr_unsync(&to->uc, &from->uc, to->addr_len);
__dev_set_rx_mode(to);
netif_addr_unlock(to);
netif_addr_unlock_bh(from);
}
EXPORT_SYMBOL(dev_uc_unsync);
/**
* dev_uc_flush - Flush unicast addresses
* @dev: device
*
* Flush unicast addresses.
*/
void dev_uc_flush(struct net_device *dev)
{
netif_addr_lock_bh(dev);
__hw_addr_flush(&dev->uc);
netif_addr_unlock_bh(dev);
}
EXPORT_SYMBOL(dev_uc_flush);
/**
* dev_uc_init - Init unicast address list
* @dev: device
*
* Init unicast address list.
*/
void dev_uc_init(struct net_device *dev)
{
__hw_addr_init(&dev->uc);
}
EXPORT_SYMBOL(dev_uc_init);
/*
* Multicast list handling functions
*/
/**
* dev_mc_add_excl - Add a global secondary multicast address
* @dev: device
* @addr: address to add
*/
int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr)
{
int err;
netif_addr_lock_bh(dev);
err = __hw_addr_add_ex(&dev->mc, addr, dev->addr_len,
NETDEV_HW_ADDR_T_MULTICAST, true, false,
0, true);
if (!err)
__dev_set_rx_mode(dev);
netif_addr_unlock_bh(dev);
return err;
}
EXPORT_SYMBOL(dev_mc_add_excl);
static int __dev_mc_add(struct net_device *dev, const unsigned char *addr,
bool global)
{
int err;
netif_addr_lock_bh(dev);
err = __hw_addr_add_ex(&dev->mc, addr, dev->addr_len,
NETDEV_HW_ADDR_T_MULTICAST, global, false,
0, false);
if (!err)
__dev_set_rx_mode(dev);
netif_addr_unlock_bh(dev);
return err;
}
/**
* dev_mc_add - Add a multicast address
* @dev: device
* @addr: address to add
*
* Add a multicast address to the device or increase
* the reference count if it already exists.
*/
int dev_mc_add(struct net_device *dev, const unsigned char *addr)
{
return __dev_mc_add(dev, addr, false);
}
EXPORT_SYMBOL(dev_mc_add);
/**
* dev_mc_add_global - Add a global multicast address
* @dev: device
* @addr: address to add
*
* Add a global multicast address to the device.
*/
int dev_mc_add_global(struct net_device *dev, const unsigned char *addr)
{
return __dev_mc_add(dev, addr, true);
}
EXPORT_SYMBOL(dev_mc_add_global);
static int __dev_mc_del(struct net_device *dev, const unsigned char *addr,
bool global)
{
int err;
netif_addr_lock_bh(dev);
err = __hw_addr_del_ex(&dev->mc, addr, dev->addr_len,
NETDEV_HW_ADDR_T_MULTICAST, global, false);
if (!err)
__dev_set_rx_mode(dev); netif_addr_unlock_bh(dev);
return err;
}
/**
* dev_mc_del - Delete a multicast address.
* @dev: device
* @addr: address to delete
*
* Release reference to a multicast address and remove it
* from the device if the reference count drops to zero.
*/
int dev_mc_del(struct net_device *dev, const unsigned char *addr)
{
return __dev_mc_del(dev, addr, false);
}
EXPORT_SYMBOL(dev_mc_del);
/**
* dev_mc_del_global - Delete a global multicast address.
* @dev: device
* @addr: address to delete
*
* Release reference to a multicast address and remove it
* from the device if the reference count drops to zero.
*/
int dev_mc_del_global(struct net_device *dev, const unsigned char *addr)
{
return __dev_mc_del(dev, addr, true);
}
EXPORT_SYMBOL(dev_mc_del_global);
/**
* dev_mc_sync - Synchronize device's multicast list to another device
* @to: destination device
* @from: source device
*
* Add newly added addresses to the destination device and release
* addresses that have no users left. The source device must be
* locked by netif_addr_lock_bh.
*
* This function is intended to be called from the ndo_set_rx_mode
* function of layered software devices.
*/
int dev_mc_sync(struct net_device *to, struct net_device *from)
{
int err = 0;
if (to->addr_len != from->addr_len)
return -EINVAL;
netif_addr_lock(to);
err = __hw_addr_sync(&to->mc, &from->mc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
netif_addr_unlock(to);
return err;
}
EXPORT_SYMBOL(dev_mc_sync);
/**
* dev_mc_sync_multiple - Synchronize device's multicast list to another
* device, but allow for multiple calls to sync to multiple devices.
* @to: destination device
* @from: source device
*
* Add newly added addresses to the destination device and release
* addresses that have no users left. The source device must be
* locked by netif_addr_lock_bh.
*
* This function is intended to be called from the ndo_set_rx_mode
* function of layered software devices. It allows for a single
* source device to be synced to multiple destination devices.
*/
int dev_mc_sync_multiple(struct net_device *to, struct net_device *from)
{
int err = 0;
if (to->addr_len != from->addr_len)
return -EINVAL;
netif_addr_lock(to);
err = __hw_addr_sync_multiple(&to->mc, &from->mc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
netif_addr_unlock(to);
return err;
}
EXPORT_SYMBOL(dev_mc_sync_multiple);
/**
* dev_mc_unsync - Remove synchronized addresses from the destination device
* @to: destination device
* @from: source device
*
* Remove all addresses that were added to the destination device by
* dev_mc_sync(). This function is intended to be called from the
* dev->stop function of layered software devices.
*/
void dev_mc_unsync(struct net_device *to, struct net_device *from)
{
if (to->addr_len != from->addr_len)
return;
/* See the above comments inside dev_uc_unsync(). */
netif_addr_lock_bh(from);
netif_addr_lock(to);
__hw_addr_unsync(&to->mc, &from->mc, to->addr_len);
__dev_set_rx_mode(to);
netif_addr_unlock(to);
netif_addr_unlock_bh(from);
}
EXPORT_SYMBOL(dev_mc_unsync);
/**
* dev_mc_flush - Flush multicast addresses
* @dev: device
*
* Flush multicast addresses.
*/
void dev_mc_flush(struct net_device *dev)
{
netif_addr_lock_bh(dev);
__hw_addr_flush(&dev->mc);
netif_addr_unlock_bh(dev);
}
EXPORT_SYMBOL(dev_mc_flush);
/**
* dev_mc_init - Init multicast address list
* @dev: device
*
* Init multicast address list.
*/
void dev_mc_init(struct net_device *dev)
{
__hw_addr_init(&dev->mc);
}
EXPORT_SYMBOL(dev_mc_init);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_RCULIST_BL_H
#define _LINUX_RCULIST_BL_H
/*
* RCU-protected bl list version. See include/linux/list_bl.h.
*/
#include <linux/list_bl.h>
#include <linux/rcupdate.h>
static inline void hlist_bl_set_first_rcu(struct hlist_bl_head *h,
struct hlist_bl_node *n)
{
LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) !=
LIST_BL_LOCKMASK);
rcu_assign_pointer(h->first,
(struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK));
}
static inline struct hlist_bl_node *hlist_bl_first_rcu(struct hlist_bl_head *h)
{
return (struct hlist_bl_node *)
((unsigned long)rcu_dereference_check(h->first, hlist_bl_is_locked(h)) & ~LIST_BL_LOCKMASK);
}
/**
* hlist_bl_del_rcu - deletes entry from hash list without re-initialization
* @n: the element to delete from the hash list.
*
* Note: hlist_bl_unhashed() on entry does not return true after this,
* the entry is in an undefined state. It is useful for RCU based
* lockfree traversal.
*
* In particular, it means that we can not poison the forward
* pointers that may still be used for walking the hash list.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_bl_add_head_rcu()
* or hlist_bl_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_bl_for_each_entry().
*/
static inline void hlist_bl_del_rcu(struct hlist_bl_node *n)
{
__hlist_bl_del(n);
n->pprev = LIST_POISON2;
}
/**
* hlist_bl_add_head_rcu
* @n: the element to add to the hash list.
* @h: the list to add to.
*
* Description:
* Adds the specified element to the specified hlist_bl,
* while permitting racing traversals.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_bl_add_head_rcu()
* or hlist_bl_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_bl_for_each_entry_rcu(), used to prevent memory-consistency
* problems on Alpha CPUs. Regardless of the type of CPU, the
* list-traversal primitive must be guarded by rcu_read_lock().
*/
static inline void hlist_bl_add_head_rcu(struct hlist_bl_node *n,
struct hlist_bl_head *h)
{
struct hlist_bl_node *first;
/* don't need hlist_bl_first_rcu because we're under lock */
first = hlist_bl_first(h);
n->next = first;
if (first)
first->pprev = &n->next; n->pprev = &h->first;
/* need _rcu because we can have concurrent lock free readers */
hlist_bl_set_first_rcu(h, n);
}
/**
* hlist_bl_for_each_entry_rcu - iterate over rcu list of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct hlist_bl_node to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_bl_node within the struct.
*
*/
#define hlist_bl_for_each_entry_rcu(tpos, pos, head, member) \
for (pos = hlist_bl_first_rcu(head); \
pos && \
({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1; }); \
pos = rcu_dereference_raw(pos->next))
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_FS_H
#define _LINUX_FS_H
#include <linux/linkage.h>
#include <linux/wait_bit.h>
#include <linux/kdev_t.h>
#include <linux/dcache.h>
#include <linux/path.h>
#include <linux/stat.h>
#include <linux/cache.h>
#include <linux/list.h>
#include <linux/list_lru.h>
#include <linux/llist.h>
#include <linux/radix-tree.h>
#include <linux/xarray.h>
#include <linux/rbtree.h>
#include <linux/init.h>
#include <linux/pid.h>
#include <linux/bug.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/mm_types.h>
#include <linux/capability.h>
#include <linux/semaphore.h>
#include <linux/fcntl.h>
#include <linux/rculist_bl.h>
#include <linux/atomic.h>
#include <linux/shrinker.h>
#include <linux/migrate_mode.h>
#include <linux/uidgid.h>
#include <linux/lockdep.h>
#include <linux/percpu-rwsem.h>
#include <linux/workqueue.h>
#include <linux/delayed_call.h>
#include <linux/uuid.h>
#include <linux/errseq.h>
#include <linux/ioprio.h>
#include <linux/fs_types.h>
#include <linux/build_bug.h>
#include <linux/stddef.h>
#include <linux/mount.h>
#include <linux/cred.h>
#include <linux/mnt_idmapping.h>
#include <linux/slab.h>
#include <linux/maple_tree.h>
#include <linux/rw_hint.h>
#include <asm/byteorder.h>
#include <uapi/linux/fs.h>
struct backing_dev_info;
struct bdi_writeback;
struct bio;
struct io_comp_batch;
struct export_operations;
struct fiemap_extent_info;
struct hd_geometry;
struct iovec;
struct kiocb;
struct kobject;
struct pipe_inode_info;
struct poll_table_struct;
struct kstatfs;
struct vm_area_struct;
struct vfsmount;
struct cred;
struct swap_info_struct;
struct seq_file;
struct workqueue_struct;
struct iov_iter;
struct fscrypt_inode_info;
struct fscrypt_operations;
struct fsverity_info;
struct fsverity_operations;
struct fs_context;
struct fs_parameter_spec;
struct fileattr;
struct iomap_ops;
extern void __init inode_init(void);
extern void __init inode_init_early(void);
extern void __init files_init(void);
extern void __init files_maxfiles_init(void);
extern unsigned long get_max_files(void);
extern unsigned int sysctl_nr_open;
typedef __kernel_rwf_t rwf_t;
struct buffer_head;
typedef int (get_block_t)(struct inode *inode, sector_t iblock,
struct buffer_head *bh_result, int create);
typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
ssize_t bytes, void *private);
#define MAY_EXEC 0x00000001
#define MAY_WRITE 0x00000002
#define MAY_READ 0x00000004
#define MAY_APPEND 0x00000008
#define MAY_ACCESS 0x00000010
#define MAY_OPEN 0x00000020
#define MAY_CHDIR 0x00000040
/* called from RCU mode, don't block */
#define MAY_NOT_BLOCK 0x00000080
/*
* flags in file.f_mode. Note that FMODE_READ and FMODE_WRITE must correspond
* to O_WRONLY and O_RDWR via the strange trick in do_dentry_open()
*/
/* file is open for reading */
#define FMODE_READ ((__force fmode_t)0x1)
/* file is open for writing */
#define FMODE_WRITE ((__force fmode_t)0x2)
/* file is seekable */
#define FMODE_LSEEK ((__force fmode_t)0x4)
/* file can be accessed using pread */
#define FMODE_PREAD ((__force fmode_t)0x8)
/* file can be accessed using pwrite */
#define FMODE_PWRITE ((__force fmode_t)0x10)
/* File is opened for execution with sys_execve / sys_uselib */
#define FMODE_EXEC ((__force fmode_t)0x20)
/* File writes are restricted (block device specific) */
#define FMODE_WRITE_RESTRICTED ((__force fmode_t)0x40)
/* 32bit hashes as llseek() offset (for directories) */
#define FMODE_32BITHASH ((__force fmode_t)0x200)
/* 64bit hashes as llseek() offset (for directories) */
#define FMODE_64BITHASH ((__force fmode_t)0x400)
/*
* Don't update ctime and mtime.
*
* Currently a special hack for the XFS open_by_handle ioctl, but we'll
* hopefully graduate it to a proper O_CMTIME flag supported by open(2) soon.
*/
#define FMODE_NOCMTIME ((__force fmode_t)0x800)
/* Expect random access pattern */
#define FMODE_RANDOM ((__force fmode_t)0x1000)
/* File is huge (eg. /dev/mem): treat loff_t as unsigned */
#define FMODE_UNSIGNED_OFFSET ((__force fmode_t)0x2000)
/* File is opened with O_PATH; almost nothing can be done with it */
#define FMODE_PATH ((__force fmode_t)0x4000)
/* File needs atomic accesses to f_pos */
#define FMODE_ATOMIC_POS ((__force fmode_t)0x8000)
/* Write access to underlying fs */
#define FMODE_WRITER ((__force fmode_t)0x10000)
/* Has read method(s) */
#define FMODE_CAN_READ ((__force fmode_t)0x20000)
/* Has write method(s) */
#define FMODE_CAN_WRITE ((__force fmode_t)0x40000)
#define FMODE_OPENED ((__force fmode_t)0x80000)
#define FMODE_CREATED ((__force fmode_t)0x100000)
/* File is stream-like */
#define FMODE_STREAM ((__force fmode_t)0x200000)
/* File supports DIRECT IO */
#define FMODE_CAN_ODIRECT ((__force fmode_t)0x400000)
#define FMODE_NOREUSE ((__force fmode_t)0x800000)
/* File supports non-exclusive O_DIRECT writes from multiple threads */
#define FMODE_DIO_PARALLEL_WRITE ((__force fmode_t)0x1000000)
/* File is embedded in backing_file object */
#define FMODE_BACKING ((__force fmode_t)0x2000000)
/* File was opened by fanotify and shouldn't generate fanotify events */
#define FMODE_NONOTIFY ((__force fmode_t)0x4000000)
/* File is capable of returning -EAGAIN if I/O will block */
#define FMODE_NOWAIT ((__force fmode_t)0x8000000)
/* File represents mount that needs unmounting */
#define FMODE_NEED_UNMOUNT ((__force fmode_t)0x10000000)
/* File does not contribute to nr_files count */
#define FMODE_NOACCOUNT ((__force fmode_t)0x20000000)
/* File supports async buffered reads */
#define FMODE_BUF_RASYNC ((__force fmode_t)0x40000000)
/* File supports async nowait buffered writes */
#define FMODE_BUF_WASYNC ((__force fmode_t)0x80000000)
/*
* Attribute flags. These should be or-ed together to figure out what
* has been changed!
*/
#define ATTR_MODE (1 << 0)
#define ATTR_UID (1 << 1)
#define ATTR_GID (1 << 2)
#define ATTR_SIZE (1 << 3)
#define ATTR_ATIME (1 << 4)
#define ATTR_MTIME (1 << 5)
#define ATTR_CTIME (1 << 6)
#define ATTR_ATIME_SET (1 << 7)
#define ATTR_MTIME_SET (1 << 8)
#define ATTR_FORCE (1 << 9) /* Not a change, but a change it */
#define ATTR_KILL_SUID (1 << 11)
#define ATTR_KILL_SGID (1 << 12)
#define ATTR_FILE (1 << 13)
#define ATTR_KILL_PRIV (1 << 14)
#define ATTR_OPEN (1 << 15) /* Truncating from open(O_TRUNC) */
#define ATTR_TIMES_SET (1 << 16)
#define ATTR_TOUCH (1 << 17)
/*
* Whiteout is represented by a char device. The following constants define the
* mode and device number to use.
*/
#define WHITEOUT_MODE 0
#define WHITEOUT_DEV 0
/*
* This is the Inode Attributes structure, used for notify_change(). It
* uses the above definitions as flags, to know which values have changed.
* Also, in this manner, a Filesystem can look at only the values it cares
* about. Basically, these are the attributes that the VFS layer can
* request to change from the FS layer.
*
* Derek Atkins <warlord@MIT.EDU> 94-10-20
*/
struct iattr {
unsigned int ia_valid;
umode_t ia_mode;
/*
* The two anonymous unions wrap structures with the same member.
*
* Filesystems raising FS_ALLOW_IDMAP need to use ia_vfs{g,u}id which
* are a dedicated type requiring the filesystem to use the dedicated
* helpers. Other filesystem can continue to use ia_{g,u}id until they
* have been ported.
*
* They always contain the same value. In other words FS_ALLOW_IDMAP
* pass down the same value on idmapped mounts as they would on regular
* mounts.
*/
union {
kuid_t ia_uid;
vfsuid_t ia_vfsuid;
};
union {
kgid_t ia_gid;
vfsgid_t ia_vfsgid;
};
loff_t ia_size;
struct timespec64 ia_atime;
struct timespec64 ia_mtime;
struct timespec64 ia_ctime;
/*
* Not an attribute, but an auxiliary info for filesystems wanting to
* implement an ftruncate() like method. NOTE: filesystem should
* check for (ia_valid & ATTR_FILE), and not for (ia_file != NULL).
*/
struct file *ia_file;
};
/*
* Includes for diskquotas.
*/
#include <linux/quota.h>
/*
* Maximum number of layers of fs stack. Needs to be limited to
* prevent kernel stack overflow
*/
#define FILESYSTEM_MAX_STACK_DEPTH 2
/**
* enum positive_aop_returns - aop return codes with specific semantics
*
* @AOP_WRITEPAGE_ACTIVATE: Informs the caller that page writeback has
* completed, that the page is still locked, and
* should be considered active. The VM uses this hint
* to return the page to the active list -- it won't
* be a candidate for writeback again in the near
* future. Other callers must be careful to unlock
* the page if they get this return. Returned by
* writepage();
*
* @AOP_TRUNCATED_PAGE: The AOP method that was handed a locked page has
* unlocked it and the page might have been truncated.
* The caller should back up to acquiring a new page and
* trying again. The aop will be taking reasonable
* precautions not to livelock. If the caller held a page
* reference, it should drop it before retrying. Returned
* by read_folio().
*
* address_space_operation functions return these large constants to indicate
* special semantics to the caller. These are much larger than the bytes in a
* page to allow for functions that return the number of bytes operated on in a
* given page.
*/
enum positive_aop_returns {
AOP_WRITEPAGE_ACTIVATE = 0x80000,
AOP_TRUNCATED_PAGE = 0x80001,
};
/*
* oh the beauties of C type declarations.
*/
struct page;
struct address_space;
struct writeback_control;
struct readahead_control;
/* Match RWF_* bits to IOCB bits */
#define IOCB_HIPRI (__force int) RWF_HIPRI
#define IOCB_DSYNC (__force int) RWF_DSYNC
#define IOCB_SYNC (__force int) RWF_SYNC
#define IOCB_NOWAIT (__force int) RWF_NOWAIT
#define IOCB_APPEND (__force int) RWF_APPEND
/* non-RWF related bits - start at 16 */
#define IOCB_EVENTFD (1 << 16)
#define IOCB_DIRECT (1 << 17)
#define IOCB_WRITE (1 << 18)
/* iocb->ki_waitq is valid */
#define IOCB_WAITQ (1 << 19)
#define IOCB_NOIO (1 << 20)
/* can use bio alloc cache */
#define IOCB_ALLOC_CACHE (1 << 21)
/*
* IOCB_DIO_CALLER_COMP can be set by the iocb owner, to indicate that the
* iocb completion can be passed back to the owner for execution from a safe
* context rather than needing to be punted through a workqueue. If this
* flag is set, the bio completion handling may set iocb->dio_complete to a
* handler function and iocb->private to context information for that handler.
* The issuer should call the handler with that context information from task
* context to complete the processing of the iocb. Note that while this
* provides a task context for the dio_complete() callback, it should only be
* used on the completion side for non-IO generating completions. It's fine to
* call blocking functions from this callback, but they should not wait for
* unrelated IO (like cache flushing, new IO generation, etc).
*/
#define IOCB_DIO_CALLER_COMP (1 << 22)
/* kiocb is a read or write operation submitted by fs/aio.c. */
#define IOCB_AIO_RW (1 << 23)
/* for use in trace events */
#define TRACE_IOCB_STRINGS \
{ IOCB_HIPRI, "HIPRI" }, \
{ IOCB_DSYNC, "DSYNC" }, \
{ IOCB_SYNC, "SYNC" }, \
{ IOCB_NOWAIT, "NOWAIT" }, \
{ IOCB_APPEND, "APPEND" }, \
{ IOCB_EVENTFD, "EVENTFD"}, \
{ IOCB_DIRECT, "DIRECT" }, \
{ IOCB_WRITE, "WRITE" }, \
{ IOCB_WAITQ, "WAITQ" }, \
{ IOCB_NOIO, "NOIO" }, \
{ IOCB_ALLOC_CACHE, "ALLOC_CACHE" }, \
{ IOCB_DIO_CALLER_COMP, "CALLER_COMP" }
struct kiocb {
struct file *ki_filp;
loff_t ki_pos;
void (*ki_complete)(struct kiocb *iocb, long ret);
void *private;
int ki_flags;
u16 ki_ioprio; /* See linux/ioprio.h */
union {
/*
* Only used for async buffered reads, where it denotes the
* page waitqueue associated with completing the read. Valid
* IFF IOCB_WAITQ is set.
*/
struct wait_page_queue *ki_waitq;
/*
* Can be used for O_DIRECT IO, where the completion handling
* is punted back to the issuer of the IO. May only be set
* if IOCB_DIO_CALLER_COMP is set by the issuer, and the issuer
* must then check for presence of this handler when ki_complete
* is invoked. The data passed in to this handler must be
* assigned to ->private when dio_complete is assigned.
*/
ssize_t (*dio_complete)(void *data);
};
};
static inline bool is_sync_kiocb(struct kiocb *kiocb)
{
return kiocb->ki_complete == NULL;
}
struct address_space_operations {
int (*writepage)(struct page *page, struct writeback_control *wbc);
int (*read_folio)(struct file *, struct folio *);
/* Write back some dirty pages from this mapping. */
int (*writepages)(struct address_space *, struct writeback_control *);
/* Mark a folio dirty. Return true if this dirtied it */
bool (*dirty_folio)(struct address_space *, struct folio *);
void (*readahead)(struct readahead_control *);
int (*write_begin)(struct file *, struct address_space *mapping,
loff_t pos, unsigned len,
struct page **pagep, void **fsdata);
int (*write_end)(struct file *, struct address_space *mapping,
loff_t pos, unsigned len, unsigned copied,
struct page *page, void *fsdata);
/* Unfortunately this kludge is needed for FIBMAP. Don't use it */
sector_t (*bmap)(struct address_space *, sector_t);
void (*invalidate_folio) (struct folio *, size_t offset, size_t len);
bool (*release_folio)(struct folio *, gfp_t);
void (*free_folio)(struct folio *folio);
ssize_t (*direct_IO)(struct kiocb *, struct iov_iter *iter);
/*
* migrate the contents of a folio to the specified target. If
* migrate_mode is MIGRATE_ASYNC, it must not block.
*/
int (*migrate_folio)(struct address_space *, struct folio *dst,
struct folio *src, enum migrate_mode);
int (*launder_folio)(struct folio *);
bool (*is_partially_uptodate) (struct folio *, size_t from,
size_t count);
void (*is_dirty_writeback) (struct folio *, bool *dirty, bool *wb);
int (*error_remove_folio)(struct address_space *, struct folio *);
/* swapfile support */
int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
sector_t *span);
void (*swap_deactivate)(struct file *file);
int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
};
extern const struct address_space_operations empty_aops;
/**
* struct address_space - Contents of a cacheable, mappable object.
* @host: Owner, either the inode or the block_device.
* @i_pages: Cached pages.
* @invalidate_lock: Guards coherency between page cache contents and
* file offset->disk block mappings in the filesystem during invalidates.
* It is also used to block modification of page cache contents through
* memory mappings.
* @gfp_mask: Memory allocation flags to use for allocating pages.
* @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings.
* @nr_thps: Number of THPs in the pagecache (non-shmem only).
* @i_mmap: Tree of private and shared mappings.
* @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
* @nrpages: Number of page entries, protected by the i_pages lock.
* @writeback_index: Writeback starts here.
* @a_ops: Methods.
* @flags: Error bits and flags (AS_*).
* @wb_err: The most recent error which has occurred.
* @i_private_lock: For use by the owner of the address_space.
* @i_private_list: For use by the owner of the address_space.
* @i_private_data: For use by the owner of the address_space.
*/
struct address_space {
struct inode *host;
struct xarray i_pages;
struct rw_semaphore invalidate_lock;
gfp_t gfp_mask;
atomic_t i_mmap_writable;
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
/* number of thp, only for non-shmem files */
atomic_t nr_thps;
#endif
struct rb_root_cached i_mmap;
unsigned long nrpages;
pgoff_t writeback_index;
const struct address_space_operations *a_ops;
unsigned long flags;
errseq_t wb_err;
spinlock_t i_private_lock;
struct list_head i_private_list;
struct rw_semaphore i_mmap_rwsem;
void * i_private_data;
} __attribute__((aligned(sizeof(long)))) __randomize_layout;
/*
* On most architectures that alignment is already the case; but
* must be enforced here for CRIS, to let the least significant bit
* of struct page's "mapping" pointer be used for PAGE_MAPPING_ANON.
*/
/* XArray tags, for tagging dirty and writeback pages in the pagecache. */
#define PAGECACHE_TAG_DIRTY XA_MARK_0
#define PAGECACHE_TAG_WRITEBACK XA_MARK_1
#define PAGECACHE_TAG_TOWRITE XA_MARK_2
/*
* Returns true if any of the pages in the mapping are marked with the tag.
*/
static inline bool mapping_tagged(struct address_space *mapping, xa_mark_t tag)
{
return xa_marked(&mapping->i_pages, tag);
}
static inline void i_mmap_lock_write(struct address_space *mapping)
{
down_write(&mapping->i_mmap_rwsem);
}
static inline int i_mmap_trylock_write(struct address_space *mapping)
{
return down_write_trylock(&mapping->i_mmap_rwsem);
}
static inline void i_mmap_unlock_write(struct address_space *mapping)
{
up_write(&mapping->i_mmap_rwsem);
}
static inline int i_mmap_trylock_read(struct address_space *mapping)
{
return down_read_trylock(&mapping->i_mmap_rwsem);
}
static inline void i_mmap_lock_read(struct address_space *mapping)
{
down_read(&mapping->i_mmap_rwsem);
}
static inline void i_mmap_unlock_read(struct address_space *mapping)
{
up_read(&mapping->i_mmap_rwsem);
}
static inline void i_mmap_assert_locked(struct address_space *mapping)
{
lockdep_assert_held(&mapping->i_mmap_rwsem);
}
static inline void i_mmap_assert_write_locked(struct address_space *mapping)
{
lockdep_assert_held_write(&mapping->i_mmap_rwsem);
}
/*
* Might pages of this file be mapped into userspace?
*/
static inline int mapping_mapped(struct address_space *mapping)
{
return !RB_EMPTY_ROOT(&mapping->i_mmap.rb_root);
}
/*
* Might pages of this file have been modified in userspace?
* Note that i_mmap_writable counts all VM_SHARED, VM_MAYWRITE vmas: do_mmap
* marks vma as VM_SHARED if it is shared, and the file was opened for
* writing i.e. vma may be mprotected writable even if now readonly.
*
* If i_mmap_writable is negative, no new writable mappings are allowed. You
* can only deny writable mappings, if none exists right now.
*/
static inline int mapping_writably_mapped(struct address_space *mapping)
{
return atomic_read(&mapping->i_mmap_writable) > 0;
}
static inline int mapping_map_writable(struct address_space *mapping)
{
return atomic_inc_unless_negative(&mapping->i_mmap_writable) ?
0 : -EPERM;
}
static inline void mapping_unmap_writable(struct address_space *mapping)
{
atomic_dec(&mapping->i_mmap_writable);
}
static inline int mapping_deny_writable(struct address_space *mapping)
{
return atomic_dec_unless_positive(&mapping->i_mmap_writable) ?
0 : -EBUSY;
}
static inline void mapping_allow_writable(struct address_space *mapping)
{
atomic_inc(&mapping->i_mmap_writable);
}
/*
* Use sequence counter to get consistent i_size on 32-bit processors.
*/
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
#include <linux/seqlock.h>
#define __NEED_I_SIZE_ORDERED
#define i_size_ordered_init(inode) seqcount_init(&inode->i_size_seqcount)
#else
#define i_size_ordered_init(inode) do { } while (0)
#endif
struct posix_acl;
#define ACL_NOT_CACHED ((void *)(-1))
/*
* ACL_DONT_CACHE is for stacked filesystems, that rely on underlying fs to
* cache the ACL. This also means that ->get_inode_acl() can be called in RCU
* mode with the LOOKUP_RCU flag.
*/
#define ACL_DONT_CACHE ((void *)(-3))
static inline struct posix_acl *
uncached_acl_sentinel(struct task_struct *task)
{
return (void *)task + 1;
}
static inline bool
is_uncached_acl(struct posix_acl *acl)
{
return (long)acl & 1;
}
#define IOP_FASTPERM 0x0001
#define IOP_LOOKUP 0x0002
#define IOP_NOFOLLOW 0x0004
#define IOP_XATTR 0x0008
#define IOP_DEFAULT_READLINK 0x0010
struct fsnotify_mark_connector;
/*
* Keep mostly read-only and often accessed (especially for
* the RCU path lookup and 'stat' data) fields at the beginning
* of the 'struct inode'
*/
struct inode {
umode_t i_mode;
unsigned short i_opflags;
kuid_t i_uid;
kgid_t i_gid;
unsigned int i_flags;
#ifdef CONFIG_FS_POSIX_ACL
struct posix_acl *i_acl;
struct posix_acl *i_default_acl;
#endif
const struct inode_operations *i_op;
struct super_block *i_sb;
struct address_space *i_mapping;
#ifdef CONFIG_SECURITY
void *i_security;
#endif
/* Stat data, not accessed from path walking */
unsigned long i_ino;
/*
* Filesystems may only read i_nlink directly. They shall use the
* following functions for modification:
*
* (set|clear|inc|drop)_nlink
* inode_(inc|dec)_link_count
*/
union {
const unsigned int i_nlink;
unsigned int __i_nlink;
};
dev_t i_rdev;
loff_t i_size;
struct timespec64 __i_atime;
struct timespec64 __i_mtime;
struct timespec64 __i_ctime; /* use inode_*_ctime accessors! */
spinlock_t i_lock; /* i_blocks, i_bytes, maybe i_size */
unsigned short i_bytes;
u8 i_blkbits;
enum rw_hint i_write_hint;
blkcnt_t i_blocks;
#ifdef __NEED_I_SIZE_ORDERED
seqcount_t i_size_seqcount;
#endif
/* Misc */
unsigned long i_state;
struct rw_semaphore i_rwsem;
unsigned long dirtied_when; /* jiffies of first dirtying */
unsigned long dirtied_time_when;
struct hlist_node i_hash;
struct list_head i_io_list; /* backing dev IO list */
#ifdef CONFIG_CGROUP_WRITEBACK
struct bdi_writeback *i_wb; /* the associated cgroup wb */
/* foreign inode detection, see wbc_detach_inode() */
int i_wb_frn_winner;
u16 i_wb_frn_avg_time;
u16 i_wb_frn_history;
#endif
struct list_head i_lru; /* inode LRU list */
struct list_head i_sb_list;
struct list_head i_wb_list; /* backing dev writeback list */
union {
struct hlist_head i_dentry;
struct rcu_head i_rcu;
};
atomic64_t i_version;
atomic64_t i_sequence; /* see futex */
atomic_t i_count;
atomic_t i_dio_count;
atomic_t i_writecount;
#if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
atomic_t i_readcount; /* struct files open RO */
#endif
union {
const struct file_operations *i_fop; /* former ->i_op->default_file_ops */
void (*free_inode)(struct inode *);
};
struct file_lock_context *i_flctx;
struct address_space i_data;
struct list_head i_devices;
union {
struct pipe_inode_info *i_pipe;
struct cdev *i_cdev;
char *i_link;
unsigned i_dir_seq;
};
__u32 i_generation;
#ifdef CONFIG_FSNOTIFY
__u32 i_fsnotify_mask; /* all events this inode cares about */
struct fsnotify_mark_connector __rcu *i_fsnotify_marks;
#endif
#ifdef CONFIG_FS_ENCRYPTION
struct fscrypt_inode_info *i_crypt_info;
#endif
#ifdef CONFIG_FS_VERITY
struct fsverity_info *i_verity_info;
#endif
void *i_private; /* fs or device private pointer */
} __randomize_layout;
struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode);
static inline unsigned int i_blocksize(const struct inode *node)
{
return (1 << node->i_blkbits);
}
static inline int inode_unhashed(struct inode *inode)
{
return hlist_unhashed(&inode->i_hash);
}
/*
* __mark_inode_dirty expects inodes to be hashed. Since we don't
* want special inodes in the fileset inode space, we make them
* appear hashed, but do not put on any lists. hlist_del()
* will work fine and require no locking.
*/
static inline void inode_fake_hash(struct inode *inode)
{
hlist_add_fake(&inode->i_hash);
}
/*
* inode->i_mutex nesting subclasses for the lock validator:
*
* 0: the object of the current VFS operation
* 1: parent
* 2: child/target
* 3: xattr
* 4: second non-directory
* 5: second parent (when locking independent directories in rename)
*
* I_MUTEX_NONDIR2 is for certain operations (such as rename) which lock two
* non-directories at once.
*
* The locking order between these classes is
* parent[2] -> child -> grandchild -> normal -> xattr -> second non-directory
*/
enum inode_i_mutex_lock_class
{
I_MUTEX_NORMAL,
I_MUTEX_PARENT,
I_MUTEX_CHILD,
I_MUTEX_XATTR,
I_MUTEX_NONDIR2,
I_MUTEX_PARENT2,
};
static inline void inode_lock(struct inode *inode)
{
down_write(&inode->i_rwsem);
}
static inline void inode_unlock(struct inode *inode)
{
up_write(&inode->i_rwsem);
}
static inline void inode_lock_shared(struct inode *inode)
{
down_read(&inode->i_rwsem);
}
static inline void inode_unlock_shared(struct inode *inode)
{
up_read(&inode->i_rwsem);
}
static inline int inode_trylock(struct inode *inode)
{
return down_write_trylock(&inode->i_rwsem);
}
static inline int inode_trylock_shared(struct inode *inode)
{
return down_read_trylock(&inode->i_rwsem);
}
static inline int inode_is_locked(struct inode *inode)
{
return rwsem_is_locked(&inode->i_rwsem);
}
static inline void inode_lock_nested(struct inode *inode, unsigned subclass)
{
down_write_nested(&inode->i_rwsem, subclass);
}
static inline void inode_lock_shared_nested(struct inode *inode, unsigned subclass)
{
down_read_nested(&inode->i_rwsem, subclass);
}
static inline void filemap_invalidate_lock(struct address_space *mapping)
{
down_write(&mapping->invalidate_lock);
}
static inline void filemap_invalidate_unlock(struct address_space *mapping)
{
up_write(&mapping->invalidate_lock);
}
static inline void filemap_invalidate_lock_shared(struct address_space *mapping)
{
down_read(&mapping->invalidate_lock);
}
static inline int filemap_invalidate_trylock_shared(
struct address_space *mapping)
{
return down_read_trylock(&mapping->invalidate_lock);
}
static inline void filemap_invalidate_unlock_shared(
struct address_space *mapping)
{
up_read(&mapping->invalidate_lock);
}
void lock_two_nondirectories(struct inode *, struct inode*);
void unlock_two_nondirectories(struct inode *, struct inode*);
void filemap_invalidate_lock_two(struct address_space *mapping1,
struct address_space *mapping2);
void filemap_invalidate_unlock_two(struct address_space *mapping1,
struct address_space *mapping2);
/*
* NOTE: in a 32bit arch with a preemptable kernel and
* an UP compile the i_size_read/write must be atomic
* with respect to the local cpu (unlike with preempt disabled),
* but they don't need to be atomic with respect to other cpus like in
* true SMP (so they need either to either locally disable irq around
* the read or for example on x86 they can be still implemented as a
* cmpxchg8b without the need of the lock prefix). For SMP compiles
* and 64bit archs it makes no difference if preempt is enabled or not.
*/
static inline loff_t i_size_read(const struct inode *inode)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
loff_t i_size;
unsigned int seq;
do {
seq = read_seqcount_begin(&inode->i_size_seqcount);
i_size = inode->i_size;
} while (read_seqcount_retry(&inode->i_size_seqcount, seq));
return i_size;
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
loff_t i_size;
preempt_disable();
i_size = inode->i_size;
preempt_enable();
return i_size;
#else
/* Pairs with smp_store_release() in i_size_write() */
return smp_load_acquire(&inode->i_size);
#endif
}
/*
* NOTE: unlike i_size_read(), i_size_write() does need locking around it
* (normally i_mutex), otherwise on 32bit/SMP an update of i_size_seqcount
* can be lost, resulting in subsequent i_size_read() calls spinning forever.
*/
static inline void i_size_write(struct inode *inode, loff_t i_size)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
preempt_disable();
write_seqcount_begin(&inode->i_size_seqcount);
inode->i_size = i_size;
write_seqcount_end(&inode->i_size_seqcount);
preempt_enable();
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
preempt_disable();
inode->i_size = i_size;
preempt_enable();
#else
/*
* Pairs with smp_load_acquire() in i_size_read() to ensure
* changes related to inode size (such as page contents) are
* visible before we see the changed inode size.
*/
smp_store_release(&inode->i_size, i_size);
#endif
}
static inline unsigned iminor(const struct inode *inode)
{
return MINOR(inode->i_rdev);
}
static inline unsigned imajor(const struct inode *inode)
{
return MAJOR(inode->i_rdev);
}
struct fown_struct {
rwlock_t lock; /* protects pid, uid, euid fields */
struct pid *pid; /* pid or -pgrp where SIGIO should be sent */
enum pid_type pid_type; /* Kind of process group SIGIO should be sent to */
kuid_t uid, euid; /* uid/euid of process setting the owner */
int signum; /* posix.1b rt signal to be delivered on IO */
};
/**
* struct file_ra_state - Track a file's readahead state.
* @start: Where the most recent readahead started.
* @size: Number of pages read in the most recent readahead.
* @async_size: Numer of pages that were/are not needed immediately
* and so were/are genuinely "ahead". Start next readahead when
* the first of these pages is accessed.
* @ra_pages: Maximum size of a readahead request, copied from the bdi.
* @mmap_miss: How many mmap accesses missed in the page cache.
* @prev_pos: The last byte in the most recent read request.
*
* When this structure is passed to ->readahead(), the "most recent"
* readahead means the current readahead.
*/
struct file_ra_state {
pgoff_t start;
unsigned int size;
unsigned int async_size;
unsigned int ra_pages;
unsigned int mmap_miss;
loff_t prev_pos;
};
/*
* Check if @index falls in the readahead windows.
*/
static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index)
{
return (index >= ra->start &&
index < ra->start + ra->size);
}
/*
* f_{lock,count,pos_lock} members can be highly contended and share
* the same cacheline. f_{lock,mode} are very frequently used together
* and so share the same cacheline as well. The read-mostly
* f_{path,inode,op} are kept on a separate cacheline.
*/
struct file {
union {
/* fput() uses task work when closing and freeing file (default). */
struct callback_head f_task_work;
/* fput() must use workqueue (most kernel threads). */
struct llist_node f_llist;
unsigned int f_iocb_flags;
};
/*
* Protects f_ep, f_flags.
* Must not be taken from IRQ context.
*/
spinlock_t f_lock;
fmode_t f_mode;
atomic_long_t f_count;
struct mutex f_pos_lock;
loff_t f_pos;
unsigned int f_flags;
struct fown_struct f_owner;
const struct cred *f_cred;
struct file_ra_state f_ra;
struct path f_path;
struct inode *f_inode; /* cached value */
const struct file_operations *f_op;
u64 f_version;
#ifdef CONFIG_SECURITY
void *f_security;
#endif
/* needed for tty driver, and maybe others */
void *private_data;
#ifdef CONFIG_EPOLL
/* Used by fs/eventpoll.c to link all the hooks to this file */
struct hlist_head *f_ep;
#endif /* #ifdef CONFIG_EPOLL */
struct address_space *f_mapping;
errseq_t f_wb_err;
errseq_t f_sb_err; /* for syncfs */
} __randomize_layout
__attribute__((aligned(4))); /* lest something weird decides that 2 is OK */
struct file_handle {
__u32 handle_bytes;
int handle_type;
/* file identifier */
unsigned char f_handle[];
};
static inline struct file *get_file(struct file *f)
{
atomic_long_inc(&f->f_count);
return f;
}
struct file *get_file_rcu(struct file __rcu **f);
struct file *get_file_active(struct file **f);
#define file_count(x) atomic_long_read(&(x)->f_count)
#define MAX_NON_LFS ((1UL<<31) - 1)
/* Page cache limit. The filesystems should put that into their s_maxbytes
limits, otherwise bad things can happen in VM. */
#if BITS_PER_LONG==32
#define MAX_LFS_FILESIZE ((loff_t)ULONG_MAX << PAGE_SHIFT)
#elif BITS_PER_LONG==64
#define MAX_LFS_FILESIZE ((loff_t)LLONG_MAX)
#endif
/* legacy typedef, should eventually be removed */
typedef void *fl_owner_t;
struct file_lock;
struct file_lease;
/* The following constant reflects the upper bound of the file/locking space */
#ifndef OFFSET_MAX
#define OFFSET_MAX type_max(loff_t)
#define OFFT_OFFSET_MAX type_max(off_t)
#endif
extern void send_sigio(struct fown_struct *fown, int fd, int band);
static inline struct inode *file_inode(const struct file *f)
{
return f->f_inode;
}
/*
* file_dentry() is a relic from the days that overlayfs was using files with a
* "fake" path, meaning, f_path on overlayfs and f_inode on underlying fs.
* In those days, file_dentry() was needed to get the underlying fs dentry that
* matches f_inode.
* Files with "fake" path should not exist nowadays, so use an assertion to make
* sure that file_dentry() was not papering over filesystem bugs.
*/
static inline struct dentry *file_dentry(const struct file *file)
{
struct dentry *dentry = file->f_path.dentry; WARN_ON_ONCE(d_inode(dentry) != file_inode(file));
return dentry;
}
struct fasync_struct {
rwlock_t fa_lock;
int magic;
int fa_fd;
struct fasync_struct *fa_next; /* singly linked list */
struct file *fa_file;
struct rcu_head fa_rcu;
};
#define FASYNC_MAGIC 0x4601
/* SMP safe fasync helpers: */
extern int fasync_helper(int, struct file *, int, struct fasync_struct **);
extern struct fasync_struct *fasync_insert_entry(int, struct file *, struct fasync_struct **, struct fasync_struct *);
extern int fasync_remove_entry(struct file *, struct fasync_struct **);
extern struct fasync_struct *fasync_alloc(void);
extern void fasync_free(struct fasync_struct *);
/* can be called from interrupts */
extern void kill_fasync(struct fasync_struct **, int, int);
extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force);
extern int f_setown(struct file *filp, int who, int force);
extern void f_delown(struct file *filp);
extern pid_t f_getown(struct file *filp);
extern int send_sigurg(struct fown_struct *fown);
/*
* sb->s_flags. Note that these mirror the equivalent MS_* flags where
* represented in both.
*/
#define SB_RDONLY BIT(0) /* Mount read-only */
#define SB_NOSUID BIT(1) /* Ignore suid and sgid bits */
#define SB_NODEV BIT(2) /* Disallow access to device special files */
#define SB_NOEXEC BIT(3) /* Disallow program execution */
#define SB_SYNCHRONOUS BIT(4) /* Writes are synced at once */
#define SB_MANDLOCK BIT(6) /* Allow mandatory locks on an FS */
#define SB_DIRSYNC BIT(7) /* Directory modifications are synchronous */
#define SB_NOATIME BIT(10) /* Do not update access times. */
#define SB_NODIRATIME BIT(11) /* Do not update directory access times */
#define SB_SILENT BIT(15)
#define SB_POSIXACL BIT(16) /* Supports POSIX ACLs */
#define SB_INLINECRYPT BIT(17) /* Use blk-crypto for encrypted files */
#define SB_KERNMOUNT BIT(22) /* this is a kern_mount call */
#define SB_I_VERSION BIT(23) /* Update inode I_version field */
#define SB_LAZYTIME BIT(25) /* Update the on-disk [acm]times lazily */
/* These sb flags are internal to the kernel */
#define SB_DEAD BIT(21)
#define SB_DYING BIT(24)
#define SB_SUBMOUNT BIT(26)
#define SB_FORCE BIT(27)
#define SB_NOSEC BIT(28)
#define SB_BORN BIT(29)
#define SB_ACTIVE BIT(30)
#define SB_NOUSER BIT(31)
/* These flags relate to encoding and casefolding */
#define SB_ENC_STRICT_MODE_FL (1 << 0)
#define sb_has_strict_encoding(sb) \
(sb->s_encoding_flags & SB_ENC_STRICT_MODE_FL)
/*
* Umount options
*/
#define MNT_FORCE 0x00000001 /* Attempt to forcibily umount */
#define MNT_DETACH 0x00000002 /* Just detach from the tree */
#define MNT_EXPIRE 0x00000004 /* Mark for expiry */
#define UMOUNT_NOFOLLOW 0x00000008 /* Don't follow symlink on umount */
#define UMOUNT_UNUSED 0x80000000 /* Flag guaranteed to be unused */
/* sb->s_iflags */
#define SB_I_CGROUPWB 0x00000001 /* cgroup-aware writeback enabled */
#define SB_I_NOEXEC 0x00000002 /* Ignore executables on this fs */
#define SB_I_NODEV 0x00000004 /* Ignore devices on this fs */
#define SB_I_STABLE_WRITES 0x00000008 /* don't modify blks until WB is done */
/* sb->s_iflags to limit user namespace mounts */
#define SB_I_USERNS_VISIBLE 0x00000010 /* fstype already mounted */
#define SB_I_IMA_UNVERIFIABLE_SIGNATURE 0x00000020
#define SB_I_UNTRUSTED_MOUNTER 0x00000040
#define SB_I_EVM_UNSUPPORTED 0x00000080
#define SB_I_SKIP_SYNC 0x00000100 /* Skip superblock at global sync */
#define SB_I_PERSB_BDI 0x00000200 /* has a per-sb bdi */
#define SB_I_TS_EXPIRY_WARNED 0x00000400 /* warned about timestamp range expiry */
#define SB_I_RETIRED 0x00000800 /* superblock shouldn't be reused */
#define SB_I_NOUMASK 0x00001000 /* VFS does not apply umask */
/* Possible states of 'frozen' field */
enum {
SB_UNFROZEN = 0, /* FS is unfrozen */
SB_FREEZE_WRITE = 1, /* Writes, dir ops, ioctls frozen */
SB_FREEZE_PAGEFAULT = 2, /* Page faults stopped as well */
SB_FREEZE_FS = 3, /* For internal FS use (e.g. to stop
* internal threads if needed) */
SB_FREEZE_COMPLETE = 4, /* ->freeze_fs finished successfully */
};
#define SB_FREEZE_LEVELS (SB_FREEZE_COMPLETE - 1)
struct sb_writers {
unsigned short frozen; /* Is sb frozen? */
int freeze_kcount; /* How many kernel freeze requests? */
int freeze_ucount; /* How many userspace freeze requests? */
struct percpu_rw_semaphore rw_sem[SB_FREEZE_LEVELS];
};
struct super_block {
struct list_head s_list; /* Keep this first */
dev_t s_dev; /* search index; _not_ kdev_t */
unsigned char s_blocksize_bits;
unsigned long s_blocksize;
loff_t s_maxbytes; /* Max file size */
struct file_system_type *s_type;
const struct super_operations *s_op;
const struct dquot_operations *dq_op;
const struct quotactl_ops *s_qcop;
const struct export_operations *s_export_op;
unsigned long s_flags;
unsigned long s_iflags; /* internal SB_I_* flags */
unsigned long s_magic;
struct dentry *s_root;
struct rw_semaphore s_umount;
int s_count;
atomic_t s_active;
#ifdef CONFIG_SECURITY
void *s_security;
#endif
const struct xattr_handler * const *s_xattr;
#ifdef CONFIG_FS_ENCRYPTION
const struct fscrypt_operations *s_cop;
struct fscrypt_keyring *s_master_keys; /* master crypto keys in use */
#endif
#ifdef CONFIG_FS_VERITY
const struct fsverity_operations *s_vop;
#endif
#if IS_ENABLED(CONFIG_UNICODE)
struct unicode_map *s_encoding;
__u16 s_encoding_flags;
#endif
struct hlist_bl_head s_roots; /* alternate root dentries for NFS */
struct list_head s_mounts; /* list of mounts; _not_ for fs use */
struct block_device *s_bdev; /* can go away once we use an accessor for @s_bdev_file */
struct file *s_bdev_file;
struct backing_dev_info *s_bdi;
struct mtd_info *s_mtd;
struct hlist_node s_instances;
unsigned int s_quota_types; /* Bitmask of supported quota types */
struct quota_info s_dquot; /* Diskquota specific options */
struct sb_writers s_writers;
/*
* Keep s_fs_info, s_time_gran, s_fsnotify_mask, and
* s_fsnotify_marks together for cache efficiency. They are frequently
* accessed and rarely modified.
*/
void *s_fs_info; /* Filesystem private info */
/* Granularity of c/m/atime in ns (cannot be worse than a second) */
u32 s_time_gran;
/* Time limits for c/m/atime in seconds */
time64_t s_time_min;
time64_t s_time_max;
#ifdef CONFIG_FSNOTIFY
__u32 s_fsnotify_mask;
struct fsnotify_mark_connector __rcu *s_fsnotify_marks;
#endif
/*
* q: why are s_id and s_sysfs_name not the same? both are human
* readable strings that identify the filesystem
* a: s_id is allowed to change at runtime; it's used in log messages,
* and we want to when a device starts out as single device (s_id is dev
* name) but then a device is hot added and we have to switch to
* identifying it by UUID
* but s_sysfs_name is a handle for programmatic access, and can't
* change at runtime
*/
char s_id[32]; /* Informational name */
uuid_t s_uuid; /* UUID */
u8 s_uuid_len; /* Default 16, possibly smaller for weird filesystems */
/* if set, fs shows up under sysfs at /sys/fs/$FSTYP/s_sysfs_name */
char s_sysfs_name[UUID_STRING_LEN + 1];
unsigned int s_max_links;
/*
* The next field is for VFS *only*. No filesystems have any business
* even looking at it. You had been warned.
*/
struct mutex s_vfs_rename_mutex; /* Kludge */
/*
* Filesystem subtype. If non-empty the filesystem type field
* in /proc/mounts will be "type.subtype"
*/
const char *s_subtype;
const struct dentry_operations *s_d_op; /* default d_op for dentries */
struct shrinker *s_shrink; /* per-sb shrinker handle */
/* Number of inodes with nlink == 0 but still referenced */
atomic_long_t s_remove_count;
/*
* Number of inode/mount/sb objects that are being watched, note that
* inodes objects are currently double-accounted.
*/
atomic_long_t s_fsnotify_connectors;
/* Read-only state of the superblock is being changed */
int s_readonly_remount;
/* per-sb errseq_t for reporting writeback errors via syncfs */
errseq_t s_wb_err;
/* AIO completions deferred from interrupt context */
struct workqueue_struct *s_dio_done_wq;
struct hlist_head s_pins;
/*
* Owning user namespace and default context in which to
* interpret filesystem uids, gids, quotas, device nodes,
* xattrs and security labels.
*/
struct user_namespace *s_user_ns;
/*
* The list_lru structure is essentially just a pointer to a table
* of per-node lru lists, each of which has its own spinlock.
* There is no need to put them into separate cachelines.
*/
struct list_lru s_dentry_lru;
struct list_lru s_inode_lru;
struct rcu_head rcu;
struct work_struct destroy_work;
struct mutex s_sync_lock; /* sync serialisation lock */
/*
* Indicates how deep in a filesystem stack this SB is
*/
int s_stack_depth;
/* s_inode_list_lock protects s_inodes */
spinlock_t s_inode_list_lock ____cacheline_aligned_in_smp;
struct list_head s_inodes; /* all inodes */
spinlock_t s_inode_wblist_lock;
struct list_head s_inodes_wb; /* writeback inodes */
} __randomize_layout;
static inline struct user_namespace *i_user_ns(const struct inode *inode)
{
return inode->i_sb->s_user_ns;
}
/* Helper functions so that in most cases filesystems will
* not need to deal directly with kuid_t and kgid_t and can
* instead deal with the raw numeric values that are stored
* in the filesystem.
*/
static inline uid_t i_uid_read(const struct inode *inode)
{
return from_kuid(i_user_ns(inode), inode->i_uid);
}
static inline gid_t i_gid_read(const struct inode *inode)
{
return from_kgid(i_user_ns(inode), inode->i_gid);
}
static inline void i_uid_write(struct inode *inode, uid_t uid)
{
inode->i_uid = make_kuid(i_user_ns(inode), uid);
}
static inline void i_gid_write(struct inode *inode, gid_t gid)
{
inode->i_gid = make_kgid(i_user_ns(inode), gid);
}
/**
* i_uid_into_vfsuid - map an inode's i_uid down according to an idmapping
* @idmap: idmap of the mount the inode was found from
* @inode: inode to map
*
* Return: whe inode's i_uid mapped down according to @idmap.
* If the inode's i_uid has no mapping INVALID_VFSUID is returned.
*/
static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap,
const struct inode *inode)
{
return make_vfsuid(idmap, i_user_ns(inode), inode->i_uid);
}
/**
* i_uid_needs_update - check whether inode's i_uid needs to be updated
* @idmap: idmap of the mount the inode was found from
* @attr: the new attributes of @inode
* @inode: the inode to update
*
* Check whether the $inode's i_uid field needs to be updated taking idmapped
* mounts into account if the filesystem supports it.
*
* Return: true if @inode's i_uid field needs to be updated, false if not.
*/
static inline bool i_uid_needs_update(struct mnt_idmap *idmap,
const struct iattr *attr,
const struct inode *inode)
{
return ((attr->ia_valid & ATTR_UID) && !vfsuid_eq(attr->ia_vfsuid,
i_uid_into_vfsuid(idmap, inode)));
}
/**
* i_uid_update - update @inode's i_uid field
* @idmap: idmap of the mount the inode was found from
* @attr: the new attributes of @inode
* @inode: the inode to update
*
* Safely update @inode's i_uid field translating the vfsuid of any idmapped
* mount into the filesystem kuid.
*/
static inline void i_uid_update(struct mnt_idmap *idmap,
const struct iattr *attr,
struct inode *inode)
{
if (attr->ia_valid & ATTR_UID)
inode->i_uid = from_vfsuid(idmap, i_user_ns(inode),
attr->ia_vfsuid);
}
/**
* i_gid_into_vfsgid - map an inode's i_gid down according to an idmapping
* @idmap: idmap of the mount the inode was found from
* @inode: inode to map
*
* Return: the inode's i_gid mapped down according to @idmap.
* If the inode's i_gid has no mapping INVALID_VFSGID is returned.
*/
static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap,
const struct inode *inode)
{
return make_vfsgid(idmap, i_user_ns(inode), inode->i_gid);
}
/**
* i_gid_needs_update - check whether inode's i_gid needs to be updated
* @idmap: idmap of the mount the inode was found from
* @attr: the new attributes of @inode
* @inode: the inode to update
*
* Check whether the $inode's i_gid field needs to be updated taking idmapped
* mounts into account if the filesystem supports it.
*
* Return: true if @inode's i_gid field needs to be updated, false if not.
*/
static inline bool i_gid_needs_update(struct mnt_idmap *idmap,
const struct iattr *attr,
const struct inode *inode)
{
return ((attr->ia_valid & ATTR_GID) && !vfsgid_eq(attr->ia_vfsgid,
i_gid_into_vfsgid(idmap, inode)));
}
/**
* i_gid_update - update @inode's i_gid field
* @idmap: idmap of the mount the inode was found from
* @attr: the new attributes of @inode
* @inode: the inode to update
*
* Safely update @inode's i_gid field translating the vfsgid of any idmapped
* mount into the filesystem kgid.
*/
static inline void i_gid_update(struct mnt_idmap *idmap,
const struct iattr *attr,
struct inode *inode)
{
if (attr->ia_valid & ATTR_GID) inode->i_gid = from_vfsgid(idmap, i_user_ns(inode),
attr->ia_vfsgid);
}
/**
* inode_fsuid_set - initialize inode's i_uid field with callers fsuid
* @inode: inode to initialize
* @idmap: idmap of the mount the inode was found from
*
* Initialize the i_uid field of @inode. If the inode was found/created via
* an idmapped mount map the caller's fsuid according to @idmap.
*/
static inline void inode_fsuid_set(struct inode *inode,
struct mnt_idmap *idmap)
{
inode->i_uid = mapped_fsuid(idmap, i_user_ns(inode));
}
/**
* inode_fsgid_set - initialize inode's i_gid field with callers fsgid
* @inode: inode to initialize
* @idmap: idmap of the mount the inode was found from
*
* Initialize the i_gid field of @inode. If the inode was found/created via
* an idmapped mount map the caller's fsgid according to @idmap.
*/
static inline void inode_fsgid_set(struct inode *inode,
struct mnt_idmap *idmap)
{
inode->i_gid = mapped_fsgid(idmap, i_user_ns(inode));
}
/**
* fsuidgid_has_mapping() - check whether caller's fsuid/fsgid is mapped
* @sb: the superblock we want a mapping in
* @idmap: idmap of the relevant mount
*
* Check whether the caller's fsuid and fsgid have a valid mapping in the
* s_user_ns of the superblock @sb. If the caller is on an idmapped mount map
* the caller's fsuid and fsgid according to the @idmap first.
*
* Return: true if fsuid and fsgid is mapped, false if not.
*/
static inline bool fsuidgid_has_mapping(struct super_block *sb,
struct mnt_idmap *idmap)
{
struct user_namespace *fs_userns = sb->s_user_ns;
kuid_t kuid;
kgid_t kgid;
kuid = mapped_fsuid(idmap, fs_userns);
if (!uid_valid(kuid))
return false;
kgid = mapped_fsgid(idmap, fs_userns);
if (!gid_valid(kgid))
return false;
return kuid_has_mapping(fs_userns, kuid) &&
kgid_has_mapping(fs_userns, kgid);
}
struct timespec64 current_time(struct inode *inode);
struct timespec64 inode_set_ctime_current(struct inode *inode);
static inline time64_t inode_get_atime_sec(const struct inode *inode)
{
return inode->__i_atime.tv_sec;
}
static inline long inode_get_atime_nsec(const struct inode *inode)
{
return inode->__i_atime.tv_nsec;
}
static inline struct timespec64 inode_get_atime(const struct inode *inode)
{
return inode->__i_atime;
}
static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode,
struct timespec64 ts)
{
inode->__i_atime = ts;
return ts;
}
static inline struct timespec64 inode_set_atime(struct inode *inode,
time64_t sec, long nsec)
{
struct timespec64 ts = { .tv_sec = sec,
.tv_nsec = nsec };
return inode_set_atime_to_ts(inode, ts);
}
static inline time64_t inode_get_mtime_sec(const struct inode *inode)
{
return inode->__i_mtime.tv_sec;
}
static inline long inode_get_mtime_nsec(const struct inode *inode)
{
return inode->__i_mtime.tv_nsec;
}
static inline struct timespec64 inode_get_mtime(const struct inode *inode)
{
return inode->__i_mtime;
}
static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode,
struct timespec64 ts)
{
inode->__i_mtime = ts;
return ts;
}
static inline struct timespec64 inode_set_mtime(struct inode *inode,
time64_t sec, long nsec)
{
struct timespec64 ts = { .tv_sec = sec,
.tv_nsec = nsec };
return inode_set_mtime_to_ts(inode, ts);
}
static inline time64_t inode_get_ctime_sec(const struct inode *inode)
{
return inode->__i_ctime.tv_sec;
}
static inline long inode_get_ctime_nsec(const struct inode *inode)
{
return inode->__i_ctime.tv_nsec;
}
static inline struct timespec64 inode_get_ctime(const struct inode *inode)
{
return inode->__i_ctime;
}
static inline struct timespec64 inode_set_ctime_to_ts(struct inode *inode,
struct timespec64 ts)
{
inode->__i_ctime = ts;
return ts;
}
/**
* inode_set_ctime - set the ctime in the inode
* @inode: inode in which to set the ctime
* @sec: tv_sec value to set
* @nsec: tv_nsec value to set
*
* Set the ctime in @inode to { @sec, @nsec }
*/
static inline struct timespec64 inode_set_ctime(struct inode *inode,
time64_t sec, long nsec)
{
struct timespec64 ts = { .tv_sec = sec,
.tv_nsec = nsec };
return inode_set_ctime_to_ts(inode, ts);
}
struct timespec64 simple_inode_init_ts(struct inode *inode);
/*
* Snapshotting support.
*/
/*
* These are internal functions, please use sb_start_{write,pagefault,intwrite}
* instead.
*/
static inline void __sb_end_write(struct super_block *sb, int level)
{
percpu_up_read(sb->s_writers.rw_sem + level-1);
}
static inline void __sb_start_write(struct super_block *sb, int level)
{
percpu_down_read(sb->s_writers.rw_sem + level - 1);
}
static inline bool __sb_start_write_trylock(struct super_block *sb, int level)
{
return percpu_down_read_trylock(sb->s_writers.rw_sem + level - 1);
}
#define __sb_writers_acquired(sb, lev) \
percpu_rwsem_acquire(&(sb)->s_writers.rw_sem[(lev)-1], 1, _THIS_IP_)
#define __sb_writers_release(sb, lev) \
percpu_rwsem_release(&(sb)->s_writers.rw_sem[(lev)-1], 1, _THIS_IP_)
/**
* __sb_write_started - check if sb freeze level is held
* @sb: the super we write to
* @level: the freeze level
*
* * > 0 - sb freeze level is held
* * 0 - sb freeze level is not held
* * < 0 - !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN
*/
static inline int __sb_write_started(const struct super_block *sb, int level)
{
return lockdep_is_held_type(sb->s_writers.rw_sem + level - 1, 1);
}
/**
* sb_write_started - check if SB_FREEZE_WRITE is held
* @sb: the super we write to
*
* May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
*/
static inline bool sb_write_started(const struct super_block *sb)
{
return __sb_write_started(sb, SB_FREEZE_WRITE);
}
/**
* sb_write_not_started - check if SB_FREEZE_WRITE is not held
* @sb: the super we write to
*
* May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
*/
static inline bool sb_write_not_started(const struct super_block *sb)
{
return __sb_write_started(sb, SB_FREEZE_WRITE) <= 0;
}
/**
* file_write_started - check if SB_FREEZE_WRITE is held
* @file: the file we write to
*
* May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
* May be false positive with !S_ISREG, because file_start_write() has
* no effect on !S_ISREG.
*/
static inline bool file_write_started(const struct file *file)
{
if (!S_ISREG(file_inode(file)->i_mode))
return true;
return sb_write_started(file_inode(file)->i_sb);
}
/**
* file_write_not_started - check if SB_FREEZE_WRITE is not held
* @file: the file we write to
*
* May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
* May be false positive with !S_ISREG, because file_start_write() has
* no effect on !S_ISREG.
*/
static inline bool file_write_not_started(const struct file *file)
{
if (!S_ISREG(file_inode(file)->i_mode))
return true;
return sb_write_not_started(file_inode(file)->i_sb);
}
/**
* sb_end_write - drop write access to a superblock
* @sb: the super we wrote to
*
* Decrement number of writers to the filesystem. Wake up possible waiters
* wanting to freeze the filesystem.
*/
static inline void sb_end_write(struct super_block *sb)
{
__sb_end_write(sb, SB_FREEZE_WRITE);
}
/**
* sb_end_pagefault - drop write access to a superblock from a page fault
* @sb: the super we wrote to
*
* Decrement number of processes handling write page fault to the filesystem.
* Wake up possible waiters wanting to freeze the filesystem.
*/
static inline void sb_end_pagefault(struct super_block *sb)
{
__sb_end_write(sb, SB_FREEZE_PAGEFAULT);
}
/**
* sb_end_intwrite - drop write access to a superblock for internal fs purposes
* @sb: the super we wrote to
*
* Decrement fs-internal number of writers to the filesystem. Wake up possible
* waiters wanting to freeze the filesystem.
*/
static inline void sb_end_intwrite(struct super_block *sb)
{
__sb_end_write(sb, SB_FREEZE_FS);
}
/**
* sb_start_write - get write access to a superblock
* @sb: the super we write to
*
* When a process wants to write data or metadata to a file system (i.e. dirty
* a page or an inode), it should embed the operation in a sb_start_write() -
* sb_end_write() pair to get exclusion against file system freezing. This
* function increments number of writers preventing freezing. If the file
* system is already frozen, the function waits until the file system is
* thawed.
*
* Since freeze protection behaves as a lock, users have to preserve
* ordering of freeze protection and other filesystem locks. Generally,
* freeze protection should be the outermost lock. In particular, we have:
*
* sb_start_write
* -> i_mutex (write path, truncate, directory ops, ...)
* -> s_umount (freeze_super, thaw_super)
*/
static inline void sb_start_write(struct super_block *sb)
{
__sb_start_write(sb, SB_FREEZE_WRITE);
}
static inline bool sb_start_write_trylock(struct super_block *sb)
{
return __sb_start_write_trylock(sb, SB_FREEZE_WRITE);
}
/**
* sb_start_pagefault - get write access to a superblock from a page fault
* @sb: the super we write to
*
* When a process starts handling write page fault, it should embed the
* operation into sb_start_pagefault() - sb_end_pagefault() pair to get
* exclusion against file system freezing. This is needed since the page fault
* is going to dirty a page. This function increments number of running page
* faults preventing freezing. If the file system is already frozen, the
* function waits until the file system is thawed.
*
* Since page fault freeze protection behaves as a lock, users have to preserve
* ordering of freeze protection and other filesystem locks. It is advised to
* put sb_start_pagefault() close to mmap_lock in lock ordering. Page fault
* handling code implies lock dependency:
*
* mmap_lock
* -> sb_start_pagefault
*/
static inline void sb_start_pagefault(struct super_block *sb)
{
__sb_start_write(sb, SB_FREEZE_PAGEFAULT);
}
/**
* sb_start_intwrite - get write access to a superblock for internal fs purposes
* @sb: the super we write to
*
* This is the third level of protection against filesystem freezing. It is
* free for use by a filesystem. The only requirement is that it must rank
* below sb_start_pagefault.
*
* For example filesystem can call sb_start_intwrite() when starting a
* transaction which somewhat eases handling of freezing for internal sources
* of filesystem changes (internal fs threads, discarding preallocation on file
* close, etc.).
*/
static inline void sb_start_intwrite(struct super_block *sb)
{
__sb_start_write(sb, SB_FREEZE_FS);
}
static inline bool sb_start_intwrite_trylock(struct super_block *sb)
{
return __sb_start_write_trylock(sb, SB_FREEZE_FS);
}
bool inode_owner_or_capable(struct mnt_idmap *idmap,
const struct inode *inode);
/*
* VFS helper functions..
*/
int vfs_create(struct mnt_idmap *, struct inode *,
struct dentry *, umode_t, bool);
int vfs_mkdir(struct mnt_idmap *, struct inode *,
struct dentry *, umode_t);
int vfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *,
umode_t, dev_t);
int vfs_symlink(struct mnt_idmap *, struct inode *,
struct dentry *, const char *);
int vfs_link(struct dentry *, struct mnt_idmap *, struct inode *,
struct dentry *, struct inode **);
int vfs_rmdir(struct mnt_idmap *, struct inode *, struct dentry *);
int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *,
struct inode **);
/**
* struct renamedata - contains all information required for renaming
* @old_mnt_idmap: idmap of the old mount the inode was found from
* @old_dir: parent of source
* @old_dentry: source
* @new_mnt_idmap: idmap of the new mount the inode was found from
* @new_dir: parent of destination
* @new_dentry: destination
* @delegated_inode: returns an inode needing a delegation break
* @flags: rename flags
*/
struct renamedata {
struct mnt_idmap *old_mnt_idmap;
struct inode *old_dir;
struct dentry *old_dentry;
struct mnt_idmap *new_mnt_idmap;
struct inode *new_dir;
struct dentry *new_dentry;
struct inode **delegated_inode;
unsigned int flags;
} __randomize_layout;
int vfs_rename(struct renamedata *);
static inline int vfs_whiteout(struct mnt_idmap *idmap,
struct inode *dir, struct dentry *dentry)
{
return vfs_mknod(idmap, dir, dentry, S_IFCHR | WHITEOUT_MODE,
WHITEOUT_DEV);
}
struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
const struct path *parentpath,
umode_t mode, int open_flag,
const struct cred *cred);
struct file *kernel_file_open(const struct path *path, int flags,
struct inode *inode, const struct cred *cred);
int vfs_mkobj(struct dentry *, umode_t,
int (*f)(struct dentry *, umode_t, void *),
void *);
int vfs_fchown(struct file *file, uid_t user, gid_t group);
int vfs_fchmod(struct file *file, umode_t mode);
int vfs_utimes(const struct path *path, struct timespec64 *times);
extern long vfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
#ifdef CONFIG_COMPAT
extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
unsigned long arg);
#else
#define compat_ptr_ioctl NULL
#endif
/*
* VFS file helper functions.
*/
void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
const struct inode *dir, umode_t mode);
extern bool may_open_dev(const struct path *path);
umode_t mode_strip_sgid(struct mnt_idmap *idmap,
const struct inode *dir, umode_t mode);
/*
* This is the "filldir" function type, used by readdir() to let
* the kernel specify what kind of dirent layout it wants to have.
* This allows the kernel to read directories into kernel space or
* to have different dirent layouts depending on the binary type.
* Return 'true' to keep going and 'false' if there are no more entries.
*/
struct dir_context;
typedef bool (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
unsigned);
struct dir_context {
filldir_t actor;
loff_t pos;
};
/*
* These flags let !MMU mmap() govern direct device mapping vs immediate
* copying more easily for MAP_PRIVATE, especially for ROM filesystems.
*
* NOMMU_MAP_COPY: Copy can be mapped (MAP_PRIVATE)
* NOMMU_MAP_DIRECT: Can be mapped directly (MAP_SHARED)
* NOMMU_MAP_READ: Can be mapped for reading
* NOMMU_MAP_WRITE: Can be mapped for writing
* NOMMU_MAP_EXEC: Can be mapped for execution
*/
#define NOMMU_MAP_COPY 0x00000001
#define NOMMU_MAP_DIRECT 0x00000008
#define NOMMU_MAP_READ VM_MAYREAD
#define NOMMU_MAP_WRITE VM_MAYWRITE
#define NOMMU_MAP_EXEC VM_MAYEXEC
#define NOMMU_VMFLAGS \
(NOMMU_MAP_READ | NOMMU_MAP_WRITE | NOMMU_MAP_EXEC)
/*
* These flags control the behavior of the remap_file_range function pointer.
* If it is called with len == 0 that means "remap to end of source file".
* See Documentation/filesystems/vfs.rst for more details about this call.
*
* REMAP_FILE_DEDUP: only remap if contents identical (i.e. deduplicate)
* REMAP_FILE_CAN_SHORTEN: caller can handle a shortened request
*/
#define REMAP_FILE_DEDUP (1 << 0)
#define REMAP_FILE_CAN_SHORTEN (1 << 1)
/*
* These flags signal that the caller is ok with altering various aspects of
* the behavior of the remap operation. The changes must be made by the
* implementation; the vfs remap helper functions can take advantage of them.
* Flags in this category exist to preserve the quirky behavior of the hoisted
* btrfs clone/dedupe ioctls.
*/
#define REMAP_FILE_ADVISORY (REMAP_FILE_CAN_SHORTEN)
/*
* These flags control the behavior of vfs_copy_file_range().
* They are not available to the user via syscall.
*
* COPY_FILE_SPLICE: call splice direct instead of fs clone/copy ops
*/
#define COPY_FILE_SPLICE (1 << 0)
struct iov_iter;
struct io_uring_cmd;
struct offset_ctx;
struct file_operations {
struct module *owner;
loff_t (*llseek) (struct file *, loff_t, int);
ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
int (*iopoll)(struct kiocb *kiocb, struct io_comp_batch *,
unsigned int flags);
int (*iterate_shared) (struct file *, struct dir_context *);
__poll_t (*poll) (struct file *, struct poll_table_struct *);
long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
int (*mmap) (struct file *, struct vm_area_struct *);
unsigned long mmap_supported_flags;
int (*open) (struct inode *, struct file *);
int (*flush) (struct file *, fl_owner_t id);
int (*release) (struct inode *, struct file *);
int (*fsync) (struct file *, loff_t, loff_t, int datasync);
int (*fasync) (int, struct file *, int);
int (*lock) (struct file *, int, struct file_lock *);
unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
int (*check_flags)(int);
int (*flock) (struct file *, int, struct file_lock *);
ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int);
ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int);
void (*splice_eof)(struct file *file);
int (*setlease)(struct file *, int, struct file_lease **, void **);
long (*fallocate)(struct file *file, int mode, loff_t offset,
loff_t len);
void (*show_fdinfo)(struct seq_file *m, struct file *f);
#ifndef CONFIG_MMU
unsigned (*mmap_capabilities)(struct file *);
#endif
ssize_t (*copy_file_range)(struct file *, loff_t, struct file *,
loff_t, size_t, unsigned int);
loff_t (*remap_file_range)(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t len, unsigned int remap_flags);
int (*fadvise)(struct file *, loff_t, loff_t, int);
int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
unsigned int poll_flags);
} __randomize_layout;
/* Wrap a directory iterator that needs exclusive inode access */
int wrap_directory_iterator(struct file *, struct dir_context *,
int (*) (struct file *, struct dir_context *));
#define WRAP_DIR_ITER(x) \
static int shared_##x(struct file *file , struct dir_context *ctx) \
{ return wrap_directory_iterator(file, ctx, x); }
struct inode_operations {
struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *);
int (*permission) (struct mnt_idmap *, struct inode *, int);
struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);
int (*readlink) (struct dentry *, char __user *,int);
int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,
umode_t, bool);
int (*link) (struct dentry *,struct inode *,struct dentry *);
int (*unlink) (struct inode *,struct dentry *);
int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,
const char *);
int (*mkdir) (struct mnt_idmap *, struct inode *,struct dentry *,
umode_t);
int (*rmdir) (struct inode *,struct dentry *);
int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,
umode_t,dev_t);
int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
struct inode *, struct dentry *, unsigned int);
int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
int (*getattr) (struct mnt_idmap *, const struct path *,
struct kstat *, u32, unsigned int);
ssize_t (*listxattr) (struct dentry *, char *, size_t);
int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
u64 len);
int (*update_time)(struct inode *, int);
int (*atomic_open)(struct inode *, struct dentry *,
struct file *, unsigned open_flag,
umode_t create_mode);
int (*tmpfile) (struct mnt_idmap *, struct inode *,
struct file *, umode_t);
struct posix_acl *(*get_acl)(struct mnt_idmap *, struct dentry *,
int);
int (*set_acl)(struct mnt_idmap *, struct dentry *,
struct posix_acl *, int);
int (*fileattr_set)(struct mnt_idmap *idmap,
struct dentry *dentry, struct fileattr *fa);
int (*fileattr_get)(struct dentry *dentry, struct fileattr *fa);
struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
} ____cacheline_aligned;
static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio,
struct iov_iter *iter)
{
return file->f_op->read_iter(kio, iter);
}
static inline ssize_t call_write_iter(struct file *file, struct kiocb *kio,
struct iov_iter *iter)
{
return file->f_op->write_iter(kio, iter);
}
static inline int call_mmap(struct file *file, struct vm_area_struct *vma)
{
return file->f_op->mmap(file, vma);
}
extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *);
extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *);
extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *,
loff_t, size_t, unsigned int);
int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t *len, unsigned int remap_flags,
const struct iomap_ops *dax_read_ops);
int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t *count, unsigned int remap_flags);
extern loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t len, unsigned int remap_flags);
extern int vfs_dedupe_file_range(struct file *file,
struct file_dedupe_range *same);
extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
struct file *dst_file, loff_t dst_pos,
loff_t len, unsigned int remap_flags);
/**
* enum freeze_holder - holder of the freeze
* @FREEZE_HOLDER_KERNEL: kernel wants to freeze or thaw filesystem
* @FREEZE_HOLDER_USERSPACE: userspace wants to freeze or thaw filesystem
* @FREEZE_MAY_NEST: whether nesting freeze and thaw requests is allowed
*
* Indicate who the owner of the freeze or thaw request is and whether
* the freeze needs to be exclusive or can nest.
* Without @FREEZE_MAY_NEST, multiple freeze and thaw requests from the
* same holder aren't allowed. It is however allowed to hold a single
* @FREEZE_HOLDER_USERSPACE and a single @FREEZE_HOLDER_KERNEL freeze at
* the same time. This is relied upon by some filesystems during online
* repair or similar.
*/
enum freeze_holder {
FREEZE_HOLDER_KERNEL = (1U << 0),
FREEZE_HOLDER_USERSPACE = (1U << 1),
FREEZE_MAY_NEST = (1U << 2),
};
struct super_operations {
struct inode *(*alloc_inode)(struct super_block *sb);
void (*destroy_inode)(struct inode *);
void (*free_inode)(struct inode *);
void (*dirty_inode) (struct inode *, int flags);
int (*write_inode) (struct inode *, struct writeback_control *wbc);
int (*drop_inode) (struct inode *);
void (*evict_inode) (struct inode *);
void (*put_super) (struct super_block *);
int (*sync_fs)(struct super_block *sb, int wait);
int (*freeze_super) (struct super_block *, enum freeze_holder who);
int (*freeze_fs) (struct super_block *);
int (*thaw_super) (struct super_block *, enum freeze_holder who);
int (*unfreeze_fs) (struct super_block *);
int (*statfs) (struct dentry *, struct kstatfs *);
int (*remount_fs) (struct super_block *, int *, char *);
void (*umount_begin) (struct super_block *);
int (*show_options)(struct seq_file *, struct dentry *);
int (*show_devname)(struct seq_file *, struct dentry *);
int (*show_path)(struct seq_file *, struct dentry *);
int (*show_stats)(struct seq_file *, struct dentry *);
#ifdef CONFIG_QUOTA
ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t);
ssize_t (*quota_write)(struct super_block *, int, const char *, size_t, loff_t);
struct dquot __rcu **(*get_dquots)(struct inode *);
#endif
long (*nr_cached_objects)(struct super_block *,
struct shrink_control *);
long (*free_cached_objects)(struct super_block *,
struct shrink_control *);
void (*shutdown)(struct super_block *sb);
};
/*
* Inode flags - they have no relation to superblock flags now
*/
#define S_SYNC (1 << 0) /* Writes are synced at once */
#define S_NOATIME (1 << 1) /* Do not update access times */
#define S_APPEND (1 << 2) /* Append-only file */
#define S_IMMUTABLE (1 << 3) /* Immutable file */
#define S_DEAD (1 << 4) /* removed, but still open directory */
#define S_NOQUOTA (1 << 5) /* Inode is not counted to quota */
#define S_DIRSYNC (1 << 6) /* Directory modifications are synchronous */
#define S_NOCMTIME (1 << 7) /* Do not update file c/mtime */
#define S_SWAPFILE (1 << 8) /* Do not truncate: swapon got its bmaps */
#define S_PRIVATE (1 << 9) /* Inode is fs-internal */
#define S_IMA (1 << 10) /* Inode has an associated IMA struct */
#define S_AUTOMOUNT (1 << 11) /* Automount/referral quasi-directory */
#define S_NOSEC (1 << 12) /* no suid or xattr security attributes */
#ifdef CONFIG_FS_DAX
#define S_DAX (1 << 13) /* Direct Access, avoiding the page cache */
#else
#define S_DAX 0 /* Make all the DAX code disappear */
#endif
#define S_ENCRYPTED (1 << 14) /* Encrypted file (using fs/crypto/) */
#define S_CASEFOLD (1 << 15) /* Casefolded file */
#define S_VERITY (1 << 16) /* Verity file (using fs/verity/) */
#define S_KERNEL_FILE (1 << 17) /* File is in use by the kernel (eg. fs/cachefiles) */
/*
* Note that nosuid etc flags are inode-specific: setting some file-system
* flags just means all the inodes inherit those flags by default. It might be
* possible to override it selectively if you really wanted to with some
* ioctl() that is not currently implemented.
*
* Exception: SB_RDONLY is always applied to the entire file system.
*
* Unfortunately, it is possible to change a filesystems flags with it mounted
* with files in use. This means that all of the inodes will not have their
* i_flags updated. Hence, i_flags no longer inherit the superblock mount
* flags, so these have to be checked separately. -- rmk@arm.uk.linux.org
*/
#define __IS_FLG(inode, flg) ((inode)->i_sb->s_flags & (flg))
static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags & SB_RDONLY; }
#define IS_RDONLY(inode) sb_rdonly((inode)->i_sb)
#define IS_SYNC(inode) (__IS_FLG(inode, SB_SYNCHRONOUS) || \
((inode)->i_flags & S_SYNC))
#define IS_DIRSYNC(inode) (__IS_FLG(inode, SB_SYNCHRONOUS|SB_DIRSYNC) || \
((inode)->i_flags & (S_SYNC|S_DIRSYNC)))
#define IS_MANDLOCK(inode) __IS_FLG(inode, SB_MANDLOCK)
#define IS_NOATIME(inode) __IS_FLG(inode, SB_RDONLY|SB_NOATIME)
#define IS_I_VERSION(inode) __IS_FLG(inode, SB_I_VERSION)
#define IS_NOQUOTA(inode) ((inode)->i_flags & S_NOQUOTA)
#define IS_APPEND(inode) ((inode)->i_flags & S_APPEND)
#define IS_IMMUTABLE(inode) ((inode)->i_flags & S_IMMUTABLE)
#ifdef CONFIG_FS_POSIX_ACL
#define IS_POSIXACL(inode) __IS_FLG(inode, SB_POSIXACL)
#else
#define IS_POSIXACL(inode) 0
#endif
#define IS_DEADDIR(inode) ((inode)->i_flags & S_DEAD)
#define IS_NOCMTIME(inode) ((inode)->i_flags & S_NOCMTIME)
#define IS_SWAPFILE(inode) ((inode)->i_flags & S_SWAPFILE)
#define IS_PRIVATE(inode) ((inode)->i_flags & S_PRIVATE)
#define IS_IMA(inode) ((inode)->i_flags & S_IMA)
#define IS_AUTOMOUNT(inode) ((inode)->i_flags & S_AUTOMOUNT)
#define IS_NOSEC(inode) ((inode)->i_flags & S_NOSEC)
#define IS_DAX(inode) ((inode)->i_flags & S_DAX)
#define IS_ENCRYPTED(inode) ((inode)->i_flags & S_ENCRYPTED)
#define IS_CASEFOLDED(inode) ((inode)->i_flags & S_CASEFOLD)
#define IS_VERITY(inode) ((inode)->i_flags & S_VERITY)
#define IS_WHITEOUT(inode) (S_ISCHR(inode->i_mode) && \
(inode)->i_rdev == WHITEOUT_DEV)
static inline bool HAS_UNMAPPED_ID(struct mnt_idmap *idmap,
struct inode *inode)
{
return !vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) || !vfsgid_valid(i_gid_into_vfsgid(idmap, inode));
}
static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
{
*kiocb = (struct kiocb) {
.ki_filp = filp,
.ki_flags = filp->f_iocb_flags,
.ki_ioprio = get_current_ioprio(),
};
}
static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src,
struct file *filp)
{
*kiocb = (struct kiocb) {
.ki_filp = filp,
.ki_flags = kiocb_src->ki_flags,
.ki_ioprio = kiocb_src->ki_ioprio,
.ki_pos = kiocb_src->ki_pos,
};
}
/*
* Inode state bits. Protected by inode->i_lock
*
* Four bits determine the dirty state of the inode: I_DIRTY_SYNC,
* I_DIRTY_DATASYNC, I_DIRTY_PAGES, and I_DIRTY_TIME.
*
* Four bits define the lifetime of an inode. Initially, inodes are I_NEW,
* until that flag is cleared. I_WILL_FREE, I_FREEING and I_CLEAR are set at
* various stages of removing an inode.
*
* Two bits are used for locking and completion notification, I_NEW and I_SYNC.
*
* I_DIRTY_SYNC Inode is dirty, but doesn't have to be written on
* fdatasync() (unless I_DIRTY_DATASYNC is also set).
* Timestamp updates are the usual cause.
* I_DIRTY_DATASYNC Data-related inode changes pending. We keep track of
* these changes separately from I_DIRTY_SYNC so that we
* don't have to write inode on fdatasync() when only
* e.g. the timestamps have changed.
* I_DIRTY_PAGES Inode has dirty pages. Inode itself may be clean.
* I_DIRTY_TIME The inode itself has dirty timestamps, and the
* lazytime mount option is enabled. We keep track of this
* separately from I_DIRTY_SYNC in order to implement
* lazytime. This gets cleared if I_DIRTY_INODE
* (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. But
* I_DIRTY_TIME can still be set if I_DIRTY_SYNC is already
* in place because writeback might already be in progress
* and we don't want to lose the time update
* I_NEW Serves as both a mutex and completion notification.
* New inodes set I_NEW. If two processes both create
* the same inode, one of them will release its inode and
* wait for I_NEW to be released before returning.
* Inodes in I_WILL_FREE, I_FREEING or I_CLEAR state can
* also cause waiting on I_NEW, without I_NEW actually
* being set. find_inode() uses this to prevent returning
* nearly-dead inodes.
* I_WILL_FREE Must be set when calling write_inode_now() if i_count
* is zero. I_FREEING must be set when I_WILL_FREE is
* cleared.
* I_FREEING Set when inode is about to be freed but still has dirty
* pages or buffers attached or the inode itself is still
* dirty.
* I_CLEAR Added by clear_inode(). In this state the inode is
* clean and can be destroyed. Inode keeps I_FREEING.
*
* Inodes that are I_WILL_FREE, I_FREEING or I_CLEAR are
* prohibited for many purposes. iget() must wait for
* the inode to be completely released, then create it
* anew. Other functions will just ignore such inodes,
* if appropriate. I_NEW is used for waiting.
*
* I_SYNC Writeback of inode is running. The bit is set during
* data writeback, and cleared with a wakeup on the bit
* address once it is done. The bit is also used to pin
* the inode in memory for flusher thread.
*
* I_REFERENCED Marks the inode as recently references on the LRU list.
*
* I_DIO_WAKEUP Never set. Only used as a key for wait_on_bit().
*
* I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to
* synchronize competing switching instances and to tell
* wb stat updates to grab the i_pages lock. See
* inode_switch_wbs_work_fn() for details.
*
* I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper
* and work dirs among overlayfs mounts.
*
* I_CREATING New object's inode in the middle of setting up.
*
* I_DONTCACHE Evict inode as soon as it is not used anymore.
*
* I_SYNC_QUEUED Inode is queued in b_io or b_more_io writeback lists.
* Used to detect that mark_inode_dirty() should not move
* inode between dirty lists.
*
* I_PINNING_FSCACHE_WB Inode is pinning an fscache object for writeback.
*
* Q: What is the difference between I_WILL_FREE and I_FREEING?
*/
#define I_DIRTY_SYNC (1 << 0)
#define I_DIRTY_DATASYNC (1 << 1)
#define I_DIRTY_PAGES (1 << 2)
#define __I_NEW 3
#define I_NEW (1 << __I_NEW)
#define I_WILL_FREE (1 << 4)
#define I_FREEING (1 << 5)
#define I_CLEAR (1 << 6)
#define __I_SYNC 7
#define I_SYNC (1 << __I_SYNC)
#define I_REFERENCED (1 << 8)
#define __I_DIO_WAKEUP 9
#define I_DIO_WAKEUP (1 << __I_DIO_WAKEUP)
#define I_LINKABLE (1 << 10)
#define I_DIRTY_TIME (1 << 11)
#define I_WB_SWITCH (1 << 13)
#define I_OVL_INUSE (1 << 14)
#define I_CREATING (1 << 15)
#define I_DONTCACHE (1 << 16)
#define I_SYNC_QUEUED (1 << 17)
#define I_PINNING_NETFS_WB (1 << 18)
#define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
#define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES)
#define I_DIRTY_ALL (I_DIRTY | I_DIRTY_TIME)
extern void __mark_inode_dirty(struct inode *, int);
static inline void mark_inode_dirty(struct inode *inode)
{
__mark_inode_dirty(inode, I_DIRTY);
}
static inline void mark_inode_dirty_sync(struct inode *inode)
{
__mark_inode_dirty(inode, I_DIRTY_SYNC);
}
/*
* Returns true if the given inode itself only has dirty timestamps (its pages
* may still be dirty) and isn't currently being allocated or freed.
* Filesystems should call this if when writing an inode when lazytime is
* enabled, they want to opportunistically write the timestamps of other inodes
* located very nearby on-disk, e.g. in the same inode block. This returns true
* if the given inode is in need of such an opportunistic update. Requires
* i_lock, or at least later re-checking under i_lock.
*/
static inline bool inode_is_dirtytime_only(struct inode *inode)
{
return (inode->i_state & (I_DIRTY_TIME | I_NEW |
I_FREEING | I_WILL_FREE)) == I_DIRTY_TIME;
}
extern void inc_nlink(struct inode *inode);
extern void drop_nlink(struct inode *inode);
extern void clear_nlink(struct inode *inode);
extern void set_nlink(struct inode *inode, unsigned int nlink);
static inline void inode_inc_link_count(struct inode *inode)
{
inc_nlink(inode);
mark_inode_dirty(inode);
}
static inline void inode_dec_link_count(struct inode *inode)
{
drop_nlink(inode);
mark_inode_dirty(inode);
}
enum file_time_flags {
S_ATIME = 1,
S_MTIME = 2,
S_CTIME = 4,
S_VERSION = 8,
};
extern bool atime_needs_update(const struct path *, struct inode *);
extern void touch_atime(const struct path *);
int inode_update_time(struct inode *inode, int flags);
static inline void file_accessed(struct file *file)
{
if (!(file->f_flags & O_NOATIME))
touch_atime(&file->f_path);
}
extern int file_modified(struct file *file);
int kiocb_modified(struct kiocb *iocb);
int sync_inode_metadata(struct inode *inode, int wait);
struct file_system_type {
const char *name;
int fs_flags;
#define FS_REQUIRES_DEV 1
#define FS_BINARY_MOUNTDATA 2
#define FS_HAS_SUBTYPE 4
#define FS_USERNS_MOUNT 8 /* Can be mounted by userns root */
#define FS_DISALLOW_NOTIFY_PERM 16 /* Disable fanotify permission events */
#define FS_ALLOW_IDMAP 32 /* FS has been updated to handle vfs idmappings. */
#define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */
int (*init_fs_context)(struct fs_context *);
const struct fs_parameter_spec *parameters;
struct dentry *(*mount) (struct file_system_type *, int,
const char *, void *);
void (*kill_sb) (struct super_block *);
struct module *owner;
struct file_system_type * next;
struct hlist_head fs_supers;
struct lock_class_key s_lock_key;
struct lock_class_key s_umount_key;
struct lock_class_key s_vfs_rename_key;
struct lock_class_key s_writers_key[SB_FREEZE_LEVELS];
struct lock_class_key i_lock_key;
struct lock_class_key i_mutex_key;
struct lock_class_key invalidate_lock_key;
struct lock_class_key i_mutex_dir_key;
};
#define MODULE_ALIAS_FS(NAME) MODULE_ALIAS("fs-" NAME)
extern struct dentry *mount_bdev(struct file_system_type *fs_type,
int flags, const char *dev_name, void *data,
int (*fill_super)(struct super_block *, void *, int));
extern struct dentry *mount_single(struct file_system_type *fs_type,
int flags, void *data,
int (*fill_super)(struct super_block *, void *, int));
extern struct dentry *mount_nodev(struct file_system_type *fs_type,
int flags, void *data,
int (*fill_super)(struct super_block *, void *, int));
extern struct dentry *mount_subtree(struct vfsmount *mnt, const char *path);
void retire_super(struct super_block *sb);
void generic_shutdown_super(struct super_block *sb);
void kill_block_super(struct super_block *sb);
void kill_anon_super(struct super_block *sb);
void kill_litter_super(struct super_block *sb);
void deactivate_super(struct super_block *sb);
void deactivate_locked_super(struct super_block *sb);
int set_anon_super(struct super_block *s, void *data);
int set_anon_super_fc(struct super_block *s, struct fs_context *fc);
int get_anon_bdev(dev_t *);
void free_anon_bdev(dev_t);
struct super_block *sget_fc(struct fs_context *fc,
int (*test)(struct super_block *, struct fs_context *),
int (*set)(struct super_block *, struct fs_context *));
struct super_block *sget(struct file_system_type *type,
int (*test)(struct super_block *,void *),
int (*set)(struct super_block *,void *),
int flags, void *data);
struct super_block *sget_dev(struct fs_context *fc, dev_t dev);
/* Alas, no aliases. Too much hassle with bringing module.h everywhere */
#define fops_get(fops) \
(((fops) && try_module_get((fops)->owner) ? (fops) : NULL))
#define fops_put(fops) \
do { if (fops) module_put((fops)->owner); } while(0)
/*
* This one is to be used *ONLY* from ->open() instances.
* fops must be non-NULL, pinned down *and* module dependencies
* should be sufficient to pin the caller down as well.
*/
#define replace_fops(f, fops) \
do { \
struct file *__file = (f); \
fops_put(__file->f_op); \
BUG_ON(!(__file->f_op = (fops))); \
} while(0)
extern int register_filesystem(struct file_system_type *);
extern int unregister_filesystem(struct file_system_type *);
extern int vfs_statfs(const struct path *, struct kstatfs *);
extern int user_statfs(const char __user *, struct kstatfs *);
extern int fd_statfs(int, struct kstatfs *);
int freeze_super(struct super_block *super, enum freeze_holder who);
int thaw_super(struct super_block *super, enum freeze_holder who);
extern __printf(2, 3)
int super_setup_bdi_name(struct super_block *sb, char *fmt, ...);
extern int super_setup_bdi(struct super_block *sb);
static inline void super_set_uuid(struct super_block *sb, const u8 *uuid, unsigned len)
{
if (WARN_ON(len > sizeof(sb->s_uuid)))
len = sizeof(sb->s_uuid);
sb->s_uuid_len = len;
memcpy(&sb->s_uuid, uuid, len);
}
/* set sb sysfs name based on sb->s_bdev */
static inline void super_set_sysfs_name_bdev(struct super_block *sb)
{
snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pg", sb->s_bdev);
}
/* set sb sysfs name based on sb->s_uuid */
static inline void super_set_sysfs_name_uuid(struct super_block *sb)
{
WARN_ON(sb->s_uuid_len != sizeof(sb->s_uuid));
snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pU", sb->s_uuid.b);
}
/* set sb sysfs name based on sb->s_id */
static inline void super_set_sysfs_name_id(struct super_block *sb)
{
strscpy(sb->s_sysfs_name, sb->s_id, sizeof(sb->s_sysfs_name));
}
/* try to use something standard before you use this */
__printf(2, 3)
static inline void super_set_sysfs_name_generic(struct super_block *sb, const char *fmt, ...)
{
va_list args;
va_start(args, fmt);
vsnprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), fmt, args);
va_end(args);
}
extern int current_umask(void);
extern void ihold(struct inode * inode);
extern void iput(struct inode *);
int inode_update_timestamps(struct inode *inode, int flags);
int generic_update_time(struct inode *, int);
/* /sys/fs */
extern struct kobject *fs_kobj;
#define MAX_RW_COUNT (INT_MAX & PAGE_MASK)
/* fs/open.c */
struct audit_names;
struct filename {
const char *name; /* pointer to actual string */
const __user char *uptr; /* original userland pointer */
atomic_t refcnt;
struct audit_names *aname;
const char iname[];
};
static_assert(offsetof(struct filename, iname) % sizeof(long) == 0);
static inline struct mnt_idmap *file_mnt_idmap(const struct file *file)
{
return mnt_idmap(file->f_path.mnt);
}
/**
* is_idmapped_mnt - check whether a mount is mapped
* @mnt: the mount to check
*
* If @mnt has an non @nop_mnt_idmap attached to it then @mnt is mapped.
*
* Return: true if mount is mapped, false if not.
*/
static inline bool is_idmapped_mnt(const struct vfsmount *mnt)
{
return mnt_idmap(mnt) != &nop_mnt_idmap;
}
extern long vfs_truncate(const struct path *, loff_t);
int do_truncate(struct mnt_idmap *, struct dentry *, loff_t start,
unsigned int time_attrs, struct file *filp);
extern int vfs_fallocate(struct file *file, int mode, loff_t offset,
loff_t len);
extern long do_sys_open(int dfd, const char __user *filename, int flags,
umode_t mode);
extern struct file *file_open_name(struct filename *, int, umode_t);
extern struct file *filp_open(const char *, int, umode_t);
extern struct file *file_open_root(const struct path *,
const char *, int, umode_t);
static inline struct file *file_open_root_mnt(struct vfsmount *mnt,
const char *name, int flags, umode_t mode)
{
return file_open_root(&(struct path){.mnt = mnt, .dentry = mnt->mnt_root},
name, flags, mode);
}
struct file *dentry_open(const struct path *path, int flags,
const struct cred *creds);
struct file *dentry_create(const struct path *path, int flags, umode_t mode,
const struct cred *cred);
struct path *backing_file_user_path(struct file *f);
/*
* When mmapping a file on a stackable filesystem (e.g., overlayfs), the file
* stored in ->vm_file is a backing file whose f_inode is on the underlying
* filesystem. When the mapped file path and inode number are displayed to
* user (e.g. via /proc/<pid>/maps), these helpers should be used to get the
* path and inode number to display to the user, which is the path of the fd
* that user has requested to map and the inode number that would be returned
* by fstat() on that same fd.
*/
/* Get the path to display in /proc/<pid>/maps */
static inline const struct path *file_user_path(struct file *f)
{
if (unlikely(f->f_mode & FMODE_BACKING))
return backing_file_user_path(f);
return &f->f_path;
}
/* Get the inode whose inode number to display in /proc/<pid>/maps */
static inline const struct inode *file_user_inode(struct file *f)
{
if (unlikely(f->f_mode & FMODE_BACKING))
return d_inode(backing_file_user_path(f)->dentry);
return file_inode(f);
}
static inline struct file *file_clone_open(struct file *file)
{
return dentry_open(&file->f_path, file->f_flags, file->f_cred);
}
extern int filp_close(struct file *, fl_owner_t id);
extern struct filename *getname_flags(const char __user *, int, int *);
extern struct filename *getname_uflags(const char __user *, int);
extern struct filename *getname(const char __user *);
extern struct filename *getname_kernel(const char *);
extern void putname(struct filename *name);
extern int finish_open(struct file *file, struct dentry *dentry,
int (*open)(struct inode *, struct file *));
extern int finish_no_open(struct file *file, struct dentry *dentry);
/* Helper for the simple case when original dentry is used */
static inline int finish_open_simple(struct file *file, int error)
{
if (error)
return error;
return finish_open(file, file->f_path.dentry, NULL);
}
/* fs/dcache.c */
extern void __init vfs_caches_init_early(void);
extern void __init vfs_caches_init(void);
extern struct kmem_cache *names_cachep;
#define __getname() kmem_cache_alloc(names_cachep, GFP_KERNEL)
#define __putname(name) kmem_cache_free(names_cachep, (void *)(name))
extern struct super_block *blockdev_superblock;
static inline bool sb_is_blkdev_sb(struct super_block *sb)
{
return IS_ENABLED(CONFIG_BLOCK) && sb == blockdev_superblock;
}
void emergency_thaw_all(void);
extern int sync_filesystem(struct super_block *);
extern const struct file_operations def_blk_fops;
extern const struct file_operations def_chr_fops;
/* fs/char_dev.c */
#define CHRDEV_MAJOR_MAX 512
/* Marks the bottom of the first segment of free char majors */
#define CHRDEV_MAJOR_DYN_END 234
/* Marks the top and bottom of the second segment of free char majors */
#define CHRDEV_MAJOR_DYN_EXT_START 511
#define CHRDEV_MAJOR_DYN_EXT_END 384
extern int alloc_chrdev_region(dev_t *, unsigned, unsigned, const char *);
extern int register_chrdev_region(dev_t, unsigned, const char *);
extern int __register_chrdev(unsigned int major, unsigned int baseminor,
unsigned int count, const char *name,
const struct file_operations *fops);
extern void __unregister_chrdev(unsigned int major, unsigned int baseminor,
unsigned int count, const char *name);
extern void unregister_chrdev_region(dev_t, unsigned);
extern void chrdev_show(struct seq_file *,off_t);
static inline int register_chrdev(unsigned int major, const char *name,
const struct file_operations *fops)
{
return __register_chrdev(major, 0, 256, name, fops);
}
static inline void unregister_chrdev(unsigned int major, const char *name)
{
__unregister_chrdev(major, 0, 256, name);
}
extern void init_special_inode(struct inode *, umode_t, dev_t);
/* Invalid inode operations -- fs/bad_inode.c */
extern void make_bad_inode(struct inode *);
extern bool is_bad_inode(struct inode *);
extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
loff_t lend);
extern int __must_check file_check_and_advance_wb_err(struct file *file);
extern int __must_check file_write_and_wait_range(struct file *file,
loff_t start, loff_t end);
static inline int file_write_and_wait(struct file *file)
{
return file_write_and_wait_range(file, 0, LLONG_MAX);
}
extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
int datasync);
extern int vfs_fsync(struct file *file, int datasync);
extern int sync_file_range(struct file *file, loff_t offset, loff_t nbytes,
unsigned int flags);
static inline bool iocb_is_dsync(const struct kiocb *iocb)
{
return (iocb->ki_flags & IOCB_DSYNC) ||
IS_SYNC(iocb->ki_filp->f_mapping->host);
}
/*
* Sync the bytes written if this was a synchronous write. Expect ki_pos
* to already be updated for the write, and will return either the amount
* of bytes passed in, or an error if syncing the file failed.
*/
static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count)
{
if (iocb_is_dsync(iocb)) {
int ret = vfs_fsync_range(iocb->ki_filp,
iocb->ki_pos - count, iocb->ki_pos - 1,
(iocb->ki_flags & IOCB_SYNC) ? 0 : 1);
if (ret)
return ret;
}
return count;
}
extern void emergency_sync(void);
extern void emergency_remount(void);
#ifdef CONFIG_BLOCK
extern int bmap(struct inode *inode, sector_t *block);
#else
static inline int bmap(struct inode *inode, sector_t *block)
{
return -EINVAL;
}
#endif
int notify_change(struct mnt_idmap *, struct dentry *,
struct iattr *, struct inode **);
int inode_permission(struct mnt_idmap *, struct inode *, int);
int generic_permission(struct mnt_idmap *, struct inode *, int);
static inline int file_permission(struct file *file, int mask)
{
return inode_permission(file_mnt_idmap(file),
file_inode(file), mask);
}
static inline int path_permission(const struct path *path, int mask)
{
return inode_permission(mnt_idmap(path->mnt),
d_inode(path->dentry), mask);
}
int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
struct inode *inode);
static inline bool execute_ok(struct inode *inode)
{
return (inode->i_mode & S_IXUGO) || S_ISDIR(inode->i_mode);
}
static inline bool inode_wrong_type(const struct inode *inode, umode_t mode)
{
return (inode->i_mode ^ mode) & S_IFMT;
}
/**
* file_start_write - get write access to a superblock for regular file io
* @file: the file we want to write to
*
* This is a variant of sb_start_write() which is a noop on non-regualr file.
* Should be matched with a call to file_end_write().
*/
static inline void file_start_write(struct file *file)
{
if (!S_ISREG(file_inode(file)->i_mode))
return;
sb_start_write(file_inode(file)->i_sb);
}
static inline bool file_start_write_trylock(struct file *file)
{
if (!S_ISREG(file_inode(file)->i_mode))
return true;
return sb_start_write_trylock(file_inode(file)->i_sb);
}
/**
* file_end_write - drop write access to a superblock of a regular file
* @file: the file we wrote to
*
* Should be matched with a call to file_start_write().
*/
static inline void file_end_write(struct file *file)
{
if (!S_ISREG(file_inode(file)->i_mode))
return;
sb_end_write(file_inode(file)->i_sb);
}
/**
* kiocb_start_write - get write access to a superblock for async file io
* @iocb: the io context we want to submit the write with
*
* This is a variant of sb_start_write() for async io submission.
* Should be matched with a call to kiocb_end_write().
*/
static inline void kiocb_start_write(struct kiocb *iocb)
{
struct inode *inode = file_inode(iocb->ki_filp);
sb_start_write(inode->i_sb);
/*
* Fool lockdep by telling it the lock got released so that it
* doesn't complain about the held lock when we return to userspace.
*/
__sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);
}
/**
* kiocb_end_write - drop write access to a superblock after async file io
* @iocb: the io context we sumbitted the write with
*
* Should be matched with a call to kiocb_start_write().
*/
static inline void kiocb_end_write(struct kiocb *iocb)
{
struct inode *inode = file_inode(iocb->ki_filp);
/*
* Tell lockdep we inherited freeze protection from submission thread.
*/
__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
sb_end_write(inode->i_sb);
}
/*
* This is used for regular files where some users -- especially the
* currently executed binary in a process, previously handled via
* VM_DENYWRITE -- cannot handle concurrent write (and maybe mmap
* read-write shared) accesses.
*
* get_write_access() gets write permission for a file.
* put_write_access() releases this write permission.
* deny_write_access() denies write access to a file.
* allow_write_access() re-enables write access to a file.
*
* The i_writecount field of an inode can have the following values:
* 0: no write access, no denied write access
* < 0: (-i_writecount) users that denied write access to the file.
* > 0: (i_writecount) users that have write access to the file.
*
* Normally we operate on that counter with atomic_{inc,dec} and it's safe
* except for the cases where we don't hold i_writecount yet. Then we need to
* use {get,deny}_write_access() - these functions check the sign and refuse
* to do the change if sign is wrong.
*/
static inline int get_write_access(struct inode *inode)
{
return atomic_inc_unless_negative(&inode->i_writecount) ? 0 : -ETXTBSY;
}
static inline int deny_write_access(struct file *file)
{
struct inode *inode = file_inode(file);
return atomic_dec_unless_positive(&inode->i_writecount) ? 0 : -ETXTBSY;
}
static inline void put_write_access(struct inode * inode)
{
atomic_dec(&inode->i_writecount);
}
static inline void allow_write_access(struct file *file)
{
if (file)
atomic_inc(&file_inode(file)->i_writecount);
}
static inline bool inode_is_open_for_write(const struct inode *inode)
{
return atomic_read(&inode->i_writecount) > 0;
}
#if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
static inline void i_readcount_dec(struct inode *inode)
{
BUG_ON(atomic_dec_return(&inode->i_readcount) < 0);
}
static inline void i_readcount_inc(struct inode *inode)
{
atomic_inc(&inode->i_readcount);
}
#else
static inline void i_readcount_dec(struct inode *inode)
{
return;
}
static inline void i_readcount_inc(struct inode *inode)
{
return;
}
#endif
extern int do_pipe_flags(int *, int);
extern ssize_t kernel_read(struct file *, void *, size_t, loff_t *);
ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos);
extern ssize_t kernel_write(struct file *, const void *, size_t, loff_t *);
extern ssize_t __kernel_write(struct file *, const void *, size_t, loff_t *);
extern struct file * open_exec(const char *);
/* fs/dcache.c -- generic fs support functions */
extern bool is_subdir(struct dentry *, struct dentry *);
extern bool path_is_under(const struct path *, const struct path *);
extern char *file_path(struct file *, char *, int);
/**
* is_dot_dotdot - returns true only if @name is "." or ".."
* @name: file name to check
* @len: length of file name, in bytes
*/
static inline bool is_dot_dotdot(const char *name, size_t len)
{
return len && unlikely(name[0] == '.') &&
(len == 1 || (len == 2 && name[1] == '.'));
}
#include <linux/err.h>
/* needed for stackable file system support */
extern loff_t default_llseek(struct file *file, loff_t offset, int whence);
extern loff_t vfs_llseek(struct file *file, loff_t offset, int whence);
extern int inode_init_always(struct super_block *, struct inode *);
extern void inode_init_once(struct inode *);
extern void address_space_init_once(struct address_space *mapping);
extern struct inode * igrab(struct inode *);
extern ino_t iunique(struct super_block *, ino_t);
extern int inode_needs_sync(struct inode *inode);
extern int generic_delete_inode(struct inode *inode);
static inline int generic_drop_inode(struct inode *inode)
{
return !inode->i_nlink || inode_unhashed(inode);
}
extern void d_mark_dontcache(struct inode *inode);
extern struct inode *ilookup5_nowait(struct super_block *sb,
unsigned long hashval, int (*test)(struct inode *, void *),
void *data);
extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
int (*test)(struct inode *, void *), void *data);
extern struct inode *ilookup(struct super_block *sb, unsigned long ino);
extern struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
int (*test)(struct inode *, void *),
int (*set)(struct inode *, void *),
void *data);
extern struct inode * iget5_locked(struct super_block *, unsigned long, int (*test)(struct inode *, void *), int (*set)(struct inode *, void *), void *);
extern struct inode * iget_locked(struct super_block *, unsigned long);
extern struct inode *find_inode_nowait(struct super_block *,
unsigned long,
int (*match)(struct inode *,
unsigned long, void *),
void *data);
extern struct inode *find_inode_rcu(struct super_block *, unsigned long,
int (*)(struct inode *, void *), void *);
extern struct inode *find_inode_by_ino_rcu(struct super_block *, unsigned long);
extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struct inode *, void *), void *);
extern int insert_inode_locked(struct inode *);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern void lockdep_annotate_inode_mutex_key(struct inode *inode);
#else
static inline void lockdep_annotate_inode_mutex_key(struct inode *inode) { };
#endif
extern void unlock_new_inode(struct inode *);
extern void discard_new_inode(struct inode *);
extern unsigned int get_next_ino(void);
extern void evict_inodes(struct super_block *sb);
void dump_mapping(const struct address_space *);
/*
* Userspace may rely on the inode number being non-zero. For example, glibc
* simply ignores files with zero i_ino in unlink() and other places.
*
* As an additional complication, if userspace was compiled with
* _FILE_OFFSET_BITS=32 on a 64-bit kernel we'll only end up reading out the
* lower 32 bits, so we need to check that those aren't zero explicitly. With
* _FILE_OFFSET_BITS=64, this may cause some harmless false-negatives, but
* better safe than sorry.
*/
static inline bool is_zero_ino(ino_t ino)
{
return (u32)ino == 0;
}
extern void __iget(struct inode * inode);
extern void iget_failed(struct inode *);
extern void clear_inode(struct inode *);
extern void __destroy_inode(struct inode *);
extern struct inode *new_inode_pseudo(struct super_block *sb);
extern struct inode *new_inode(struct super_block *sb);
extern void free_inode_nonrcu(struct inode *inode);
extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *);
extern int file_remove_privs_flags(struct file *file, unsigned int flags);
extern int file_remove_privs(struct file *);
int setattr_should_drop_sgid(struct mnt_idmap *idmap,
const struct inode *inode);
/*
* This must be used for allocating filesystems specific inodes to set
* up the inode reclaim context correctly.
*/
static inline void *
alloc_inode_sb(struct super_block *sb, struct kmem_cache *cache, gfp_t gfp)
{
return kmem_cache_alloc_lru(cache, &sb->s_inode_lru, gfp);
}
extern void __insert_inode_hash(struct inode *, unsigned long hashval);
static inline void insert_inode_hash(struct inode *inode)
{
__insert_inode_hash(inode, inode->i_ino);
}
extern void __remove_inode_hash(struct inode *);
static inline void remove_inode_hash(struct inode *inode)
{
if (!inode_unhashed(inode) && !hlist_fake(&inode->i_hash)) __remove_inode_hash(inode);
}
extern void inode_sb_list_add(struct inode *inode);
extern void inode_add_lru(struct inode *inode);
extern int sb_set_blocksize(struct super_block *, int);
extern int sb_min_blocksize(struct super_block *, int);
extern int generic_file_mmap(struct file *, struct vm_area_struct *);
extern int generic_file_readonly_mmap(struct file *, struct vm_area_struct *);
extern ssize_t generic_write_checks(struct kiocb *, struct iov_iter *);
int generic_write_checks_count(struct kiocb *iocb, loff_t *count);
extern int generic_write_check_limits(struct file *file, loff_t pos,
loff_t *count);
extern int generic_file_rw_checks(struct file *file_in, struct file *file_out);
ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *to,
ssize_t already_read);
extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *);
extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *);
extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *);
extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *);
ssize_t generic_perform_write(struct kiocb *, struct iov_iter *);
ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter,
ssize_t direct_written, ssize_t buffered_written);
ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
rwf_t flags);
ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
rwf_t flags);
ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
struct iov_iter *iter);
ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
struct iov_iter *iter);
/* fs/splice.c */
ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe,
size_t len, unsigned int flags);
ssize_t copy_splice_read(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe,
size_t len, unsigned int flags);
extern ssize_t iter_file_splice_write(struct pipe_inode_info *,
struct file *, loff_t *, size_t, unsigned int);
extern void
file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
extern loff_t noop_llseek(struct file *file, loff_t offset, int whence);
#define no_llseek NULL
extern loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize);
extern loff_t generic_file_llseek(struct file *file, loff_t offset, int whence);
extern loff_t generic_file_llseek_size(struct file *file, loff_t offset,
int whence, loff_t maxsize, loff_t eof);
extern loff_t fixed_size_llseek(struct file *file, loff_t offset,
int whence, loff_t size);
extern loff_t no_seek_end_llseek_size(struct file *, loff_t, int, loff_t);
extern loff_t no_seek_end_llseek(struct file *, loff_t, int);
int rw_verify_area(int, struct file *, const loff_t *, size_t);
extern int generic_file_open(struct inode * inode, struct file * filp);
extern int nonseekable_open(struct inode * inode, struct file * filp);
extern int stream_open(struct inode * inode, struct file * filp);
#ifdef CONFIG_BLOCK
typedef void (dio_submit_t)(struct bio *bio, struct inode *inode,
loff_t file_offset);
enum {
/* need locking between buffered and direct access */
DIO_LOCKING = 0x01,
/* filesystem does not support filling holes */
DIO_SKIP_HOLES = 0x02,
};
ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
struct block_device *bdev, struct iov_iter *iter,
get_block_t get_block,
dio_iodone_t end_io,
int flags);
static inline ssize_t blockdev_direct_IO(struct kiocb *iocb,
struct inode *inode,
struct iov_iter *iter,
get_block_t get_block)
{
return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
get_block, NULL, DIO_LOCKING | DIO_SKIP_HOLES);
}
#endif
void inode_dio_wait(struct inode *inode);
/**
* inode_dio_begin - signal start of a direct I/O requests
* @inode: inode the direct I/O happens on
*
* This is called once we've finished processing a direct I/O request,
* and is used to wake up callers waiting for direct I/O to be quiesced.
*/
static inline void inode_dio_begin(struct inode *inode)
{
atomic_inc(&inode->i_dio_count);
}
/**
* inode_dio_end - signal finish of a direct I/O requests
* @inode: inode the direct I/O happens on
*
* This is called once we've finished processing a direct I/O request,
* and is used to wake up callers waiting for direct I/O to be quiesced.
*/
static inline void inode_dio_end(struct inode *inode)
{
if (atomic_dec_and_test(&inode->i_dio_count))
wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
}
extern void inode_set_flags(struct inode *inode, unsigned int flags,
unsigned int mask);
extern const struct file_operations generic_ro_fops;
#define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
extern int readlink_copy(char __user *, int, const char *);
extern int page_readlink(struct dentry *, char __user *, int);
extern const char *page_get_link(struct dentry *, struct inode *,
struct delayed_call *);
extern void page_put_link(void *);
extern int page_symlink(struct inode *inode, const char *symname, int len);
extern const struct inode_operations page_symlink_inode_operations;
extern void kfree_link(void *);
void generic_fillattr(struct mnt_idmap *, u32, struct inode *, struct kstat *);
void generic_fill_statx_attr(struct inode *inode, struct kstat *stat);
extern int vfs_getattr_nosec(const struct path *, struct kstat *, u32, unsigned int);
extern int vfs_getattr(const struct path *, struct kstat *, u32, unsigned int);
void __inode_add_bytes(struct inode *inode, loff_t bytes);
void inode_add_bytes(struct inode *inode, loff_t bytes);
void __inode_sub_bytes(struct inode *inode, loff_t bytes);
void inode_sub_bytes(struct inode *inode, loff_t bytes);
static inline loff_t __inode_get_bytes(struct inode *inode)
{
return (((loff_t)inode->i_blocks) << 9) + inode->i_bytes;
}
loff_t inode_get_bytes(struct inode *inode);
void inode_set_bytes(struct inode *inode, loff_t bytes);
const char *simple_get_link(struct dentry *, struct inode *,
struct delayed_call *);
extern const struct inode_operations simple_symlink_inode_operations;
extern int iterate_dir(struct file *, struct dir_context *);
int vfs_fstatat(int dfd, const char __user *filename, struct kstat *stat,
int flags);
int vfs_fstat(int fd, struct kstat *stat);
static inline int vfs_stat(const char __user *filename, struct kstat *stat)
{
return vfs_fstatat(AT_FDCWD, filename, stat, 0);
}
static inline int vfs_lstat(const char __user *name, struct kstat *stat)
{
return vfs_fstatat(AT_FDCWD, name, stat, AT_SYMLINK_NOFOLLOW);
}
extern const char *vfs_get_link(struct dentry *, struct delayed_call *);
extern int vfs_readlink(struct dentry *, char __user *, int);
extern struct file_system_type *get_filesystem(struct file_system_type *fs);
extern void put_filesystem(struct file_system_type *fs);
extern struct file_system_type *get_fs_type(const char *name);
extern void drop_super(struct super_block *sb);
extern void drop_super_exclusive(struct super_block *sb);
extern void iterate_supers(void (*)(struct super_block *, void *), void *);
extern void iterate_supers_type(struct file_system_type *,
void (*)(struct super_block *, void *), void *);
extern int dcache_dir_open(struct inode *, struct file *);
extern int dcache_dir_close(struct inode *, struct file *);
extern loff_t dcache_dir_lseek(struct file *, loff_t, int);
extern int dcache_readdir(struct file *, struct dir_context *);
extern int simple_setattr(struct mnt_idmap *, struct dentry *,
struct iattr *);
extern int simple_getattr(struct mnt_idmap *, const struct path *,
struct kstat *, u32, unsigned int);
extern int simple_statfs(struct dentry *, struct kstatfs *);
extern int simple_open(struct inode *inode, struct file *file);
extern int simple_link(struct dentry *, struct inode *, struct dentry *);
extern int simple_unlink(struct inode *, struct dentry *);
extern int simple_rmdir(struct inode *, struct dentry *);
void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry);
extern int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry);
extern int simple_rename(struct mnt_idmap *, struct inode *,
struct dentry *, struct inode *, struct dentry *,
unsigned int);
extern void simple_recursive_removal(struct dentry *,
void (*callback)(struct dentry *));
extern int noop_fsync(struct file *, loff_t, loff_t, int);
extern ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
extern int simple_empty(struct dentry *);
extern int simple_write_begin(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len,
struct page **pagep, void **fsdata);
extern const struct address_space_operations ram_aops;
extern int always_delete_dentry(const struct dentry *);
extern struct inode *alloc_anon_inode(struct super_block *);
extern int simple_nosetlease(struct file *, int, struct file_lease **, void **);
extern const struct dentry_operations simple_dentry_operations;
extern struct dentry *simple_lookup(struct inode *, struct dentry *, unsigned int flags);
extern ssize_t generic_read_dir(struct file *, char __user *, size_t, loff_t *);
extern const struct file_operations simple_dir_operations;
extern const struct inode_operations simple_dir_inode_operations;
extern void make_empty_dir_inode(struct inode *inode);
extern bool is_empty_dir_inode(struct inode *inode);
struct tree_descr { const char *name; const struct file_operations *ops; int mode; };
struct dentry *d_alloc_name(struct dentry *, const char *);
extern int simple_fill_super(struct super_block *, unsigned long,
const struct tree_descr *);
extern int simple_pin_fs(struct file_system_type *, struct vfsmount **mount, int *count);
extern void simple_release_fs(struct vfsmount **mount, int *count);
extern ssize_t simple_read_from_buffer(void __user *to, size_t count,
loff_t *ppos, const void *from, size_t available);
extern ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos,
const void __user *from, size_t count);
struct offset_ctx {
struct maple_tree mt;
unsigned long next_offset;
};
void simple_offset_init(struct offset_ctx *octx);
int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry);
void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry);
int simple_offset_empty(struct dentry *dentry);
int simple_offset_rename_exchange(struct inode *old_dir,
struct dentry *old_dentry,
struct inode *new_dir,
struct dentry *new_dentry);
void simple_offset_destroy(struct offset_ctx *octx);
extern const struct file_operations simple_offset_dir_operations;
extern int __generic_file_fsync(struct file *, loff_t, loff_t, int);
extern int generic_file_fsync(struct file *, loff_t, loff_t, int);
extern int generic_check_addressable(unsigned, u64);
extern void generic_set_sb_d_ops(struct super_block *sb);
static inline bool sb_has_encoding(const struct super_block *sb)
{
#if IS_ENABLED(CONFIG_UNICODE)
return !!sb->s_encoding;
#else
return false;
#endif
}
int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
unsigned int ia_valid);
int setattr_prepare(struct mnt_idmap *, struct dentry *, struct iattr *);
extern int inode_newsize_ok(const struct inode *, loff_t offset);
void setattr_copy(struct mnt_idmap *, struct inode *inode,
const struct iattr *attr);
extern int file_update_time(struct file *file);
static inline bool vma_is_dax(const struct vm_area_struct *vma)
{
return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host);
}
static inline bool vma_is_fsdax(struct vm_area_struct *vma)
{
struct inode *inode;
if (!IS_ENABLED(CONFIG_FS_DAX) || !vma->vm_file)
return false;
if (!vma_is_dax(vma))
return false;
inode = file_inode(vma->vm_file);
if (S_ISCHR(inode->i_mode))
return false; /* device-dax */
return true;
}
static inline int iocb_flags(struct file *file)
{
int res = 0;
if (file->f_flags & O_APPEND)
res |= IOCB_APPEND;
if (file->f_flags & O_DIRECT)
res |= IOCB_DIRECT; if (file->f_flags & O_DSYNC) res |= IOCB_DSYNC; if (file->f_flags & __O_SYNC) res |= IOCB_SYNC;
return res;
}
static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
{
int kiocb_flags = 0;
/* make sure there's no overlap between RWF and private IOCB flags */
BUILD_BUG_ON((__force int) RWF_SUPPORTED & IOCB_EVENTFD);
if (!flags)
return 0;
if (unlikely(flags & ~RWF_SUPPORTED))
return -EOPNOTSUPP;
if (unlikely((flags & RWF_APPEND) && (flags & RWF_NOAPPEND)))
return -EINVAL;
if (flags & RWF_NOWAIT) {
if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
return -EOPNOTSUPP;
kiocb_flags |= IOCB_NOIO;
}
kiocb_flags |= (__force int) (flags & RWF_SUPPORTED);
if (flags & RWF_SYNC)
kiocb_flags |= IOCB_DSYNC;
if ((flags & RWF_NOAPPEND) && (ki->ki_flags & IOCB_APPEND)) {
if (IS_APPEND(file_inode(ki->ki_filp)))
return -EPERM;
ki->ki_flags &= ~IOCB_APPEND;
}
ki->ki_flags |= kiocb_flags;
return 0;
}
static inline ino_t parent_ino(struct dentry *dentry)
{
ino_t res;
/*
* Don't strictly need d_lock here? If the parent ino could change
* then surely we'd have a deeper race in the caller?
*/
spin_lock(&dentry->d_lock);
res = dentry->d_parent->d_inode->i_ino;
spin_unlock(&dentry->d_lock);
return res;
}
/* Transaction based IO helpers */
/*
* An argresp is stored in an allocated page and holds the
* size of the argument or response, along with its content
*/
struct simple_transaction_argresp {
ssize_t size;
char data[];
};
#define SIMPLE_TRANSACTION_LIMIT (PAGE_SIZE - sizeof(struct simple_transaction_argresp))
char *simple_transaction_get(struct file *file, const char __user *buf,
size_t size);
ssize_t simple_transaction_read(struct file *file, char __user *buf,
size_t size, loff_t *pos);
int simple_transaction_release(struct inode *inode, struct file *file);
void simple_transaction_set(struct file *file, size_t n);
/*
* simple attribute files
*
* These attributes behave similar to those in sysfs:
*
* Writing to an attribute immediately sets a value, an open file can be
* written to multiple times.
*
* Reading from an attribute creates a buffer from the value that might get
* read with multiple read calls. When the attribute has been read
* completely, no further read calls are possible until the file is opened
* again.
*
* All attributes contain a text representation of a numeric value
* that are accessed with the get() and set() functions.
*/
#define DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, __is_signed) \
static int __fops ## _open(struct inode *inode, struct file *file) \
{ \
__simple_attr_check_format(__fmt, 0ull); \
return simple_attr_open(inode, file, __get, __set, __fmt); \
} \
static const struct file_operations __fops = { \
.owner = THIS_MODULE, \
.open = __fops ## _open, \
.release = simple_attr_release, \
.read = simple_attr_read, \
.write = (__is_signed) ? simple_attr_write_signed : simple_attr_write, \
.llseek = generic_file_llseek, \
}
#define DEFINE_SIMPLE_ATTRIBUTE(__fops, __get, __set, __fmt) \
DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, false)
#define DEFINE_SIMPLE_ATTRIBUTE_SIGNED(__fops, __get, __set, __fmt) \
DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, true)
static inline __printf(1, 2)
void __simple_attr_check_format(const char *fmt, ...)
{
/* don't do anything, just let the compiler check the arguments; */
}
int simple_attr_open(struct inode *inode, struct file *file,
int (*get)(void *, u64 *), int (*set)(void *, u64),
const char *fmt);
int simple_attr_release(struct inode *inode, struct file *file);
ssize_t simple_attr_read(struct file *file, char __user *buf,
size_t len, loff_t *ppos);
ssize_t simple_attr_write(struct file *file, const char __user *buf,
size_t len, loff_t *ppos);
ssize_t simple_attr_write_signed(struct file *file, const char __user *buf,
size_t len, loff_t *ppos);
struct ctl_table;
int __init list_bdev_fs_names(char *buf, size_t size);
#define __FMODE_EXEC ((__force int) FMODE_EXEC)
#define __FMODE_NONOTIFY ((__force int) FMODE_NONOTIFY)
#define ACC_MODE(x) ("\004\002\006\006"[(x)&O_ACCMODE])
#define OPEN_FMODE(flag) ((__force fmode_t)(((flag + 1) & O_ACCMODE) | \
(flag & __FMODE_NONOTIFY)))
static inline bool is_sxid(umode_t mode)
{
return mode & (S_ISUID | S_ISGID);
}
static inline int check_sticky(struct mnt_idmap *idmap,
struct inode *dir, struct inode *inode)
{
if (!(dir->i_mode & S_ISVTX))
return 0;
return __check_sticky(idmap, dir, inode);
}
static inline void inode_has_no_xattr(struct inode *inode)
{
if (!is_sxid(inode->i_mode) && (inode->i_sb->s_flags & SB_NOSEC)) inode->i_flags |= S_NOSEC;
}
static inline bool is_root_inode(struct inode *inode)
{
return inode == inode->i_sb->s_root->d_inode;
}
static inline bool dir_emit(struct dir_context *ctx,
const char *name, int namelen,
u64 ino, unsigned type)
{
return ctx->actor(ctx, name, namelen, ctx->pos, ino, type);
}
static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx)
{
return ctx->actor(ctx, ".", 1, ctx->pos,
file->f_path.dentry->d_inode->i_ino, DT_DIR);
}
static inline bool dir_emit_dotdot(struct file *file, struct dir_context *ctx)
{
return ctx->actor(ctx, "..", 2, ctx->pos,
parent_ino(file->f_path.dentry), DT_DIR);
}
static inline bool dir_emit_dots(struct file *file, struct dir_context *ctx)
{
if (ctx->pos == 0) {
if (!dir_emit_dot(file, ctx))
return false;
ctx->pos = 1;
}
if (ctx->pos == 1) {
if (!dir_emit_dotdot(file, ctx))
return false;
ctx->pos = 2;
}
return true;
}
static inline bool dir_relax(struct inode *inode)
{
inode_unlock(inode);
inode_lock(inode);
return !IS_DEADDIR(inode);
}
static inline bool dir_relax_shared(struct inode *inode)
{
inode_unlock_shared(inode);
inode_lock_shared(inode);
return !IS_DEADDIR(inode);
}
extern bool path_noexec(const struct path *path);
extern void inode_nohighmem(struct inode *inode);
/* mm/fadvise.c */
extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len,
int advice);
extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
int advice);
#endif /* _LINUX_FS_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_ERR_H
#define _LINUX_ERR_H
#include <linux/compiler.h>
#include <linux/types.h>
#include <asm/errno.h>
/*
* Kernel pointers have redundant information, so we can use a
* scheme where we can return either an error code or a normal
* pointer with the same return value.
*
* This should be a per-architecture thing, to allow different
* error and pointer decisions.
*/
#define MAX_ERRNO 4095
#ifndef __ASSEMBLY__
/**
* IS_ERR_VALUE - Detect an error pointer.
* @x: The pointer to check.
*
* Like IS_ERR(), but does not generate a compiler warning if result is unused.
*/
#define IS_ERR_VALUE(x) unlikely((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO)
/**
* ERR_PTR - Create an error pointer.
* @error: A negative error code.
*
* Encodes @error into a pointer value. Users should consider the result
* opaque and not assume anything about how the error is encoded.
*
* Return: A pointer with @error encoded within its value.
*/
static inline void * __must_check ERR_PTR(long error)
{
return (void *) error;
}
/**
* PTR_ERR - Extract the error code from an error pointer.
* @ptr: An error pointer.
* Return: The error code within @ptr.
*/
static inline long __must_check PTR_ERR(__force const void *ptr)
{
return (long) ptr;
}
/**
* IS_ERR - Detect an error pointer.
* @ptr: The pointer to check.
* Return: true if @ptr is an error pointer, false otherwise.
*/
static inline bool __must_check IS_ERR(__force const void *ptr)
{
return IS_ERR_VALUE((unsigned long)ptr);
}
/**
* IS_ERR_OR_NULL - Detect an error pointer or a null pointer.
* @ptr: The pointer to check.
*
* Like IS_ERR(), but also returns true for a null pointer.
*/
static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr)
{
return unlikely(!ptr) || IS_ERR_VALUE((unsigned long)ptr);
}
/**
* ERR_CAST - Explicitly cast an error-valued pointer to another pointer type
* @ptr: The pointer to cast.
*
* Explicitly cast an error-valued pointer to another pointer type in such a
* way as to make it clear that's what's going on.
*/
static inline void * __must_check ERR_CAST(__force const void *ptr)
{
/* cast away the const */
return (void *) ptr;
}
/**
* PTR_ERR_OR_ZERO - Extract the error code from a pointer if it has one.
* @ptr: A potential error pointer.
*
* Convenience function that can be used inside a function that returns
* an error code to propagate errors received as error pointers.
* For example, ``return PTR_ERR_OR_ZERO(ptr);`` replaces:
*
* .. code-block:: c
*
* if (IS_ERR(ptr))
* return PTR_ERR(ptr);
* else
* return 0;
*
* Return: The error code within @ptr if it is an error pointer; 0 otherwise.
*/
static inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr)
{
if (IS_ERR(ptr))
return PTR_ERR(ptr);
else
return 0;
}
#endif
#endif /* _LINUX_ERR_H */
// SPDX-License-Identifier: GPL-2.0+
/*
* Base port operations for 8250/16550-type serial ports
*
* Based on drivers/char/serial.c, by Linus Torvalds, Theodore Ts'o.
* Split from 8250_core.c, Copyright (C) 2001 Russell King.
*
* A note about mapbase / membase
*
* mapbase is the physical address of the IO port.
* membase is an 'ioremapped' cookie.
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/ioport.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/console.h>
#include <linux/gpio/consumer.h>
#include <linux/sysrq.h>
#include <linux/delay.h>
#include <linux/platform_device.h>
#include <linux/tty.h>
#include <linux/ratelimit.h>
#include <linux/tty_flip.h>
#include <linux/serial.h>
#include <linux/serial_8250.h>
#include <linux/nmi.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/pm_runtime.h>
#include <linux/ktime.h>
#include <asm/io.h>
#include <asm/irq.h>
#include "8250.h"
/*
* Debugging.
*/
#if 0
#define DEBUG_AUTOCONF(fmt...) printk(fmt)
#else
#define DEBUG_AUTOCONF(fmt...) do { } while (0)
#endif
/*
* Here we define the default xmit fifo size used for each type of UART.
*/
static const struct serial8250_config uart_config[] = {
[PORT_UNKNOWN] = {
.name = "unknown",
.fifo_size = 1,
.tx_loadsz = 1,
},
[PORT_8250] = {
.name = "8250",
.fifo_size = 1,
.tx_loadsz = 1,
},
[PORT_16450] = {
.name = "16450",
.fifo_size = 1,
.tx_loadsz = 1,
},
[PORT_16550] = {
.name = "16550",
.fifo_size = 1,
.tx_loadsz = 1,
},
[PORT_16550A] = {
.name = "16550A",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 4, 8, 14},
.flags = UART_CAP_FIFO,
},
[PORT_CIRRUS] = {
.name = "Cirrus",
.fifo_size = 1,
.tx_loadsz = 1,
},
[PORT_16650] = {
.name = "ST16650",
.fifo_size = 1,
.tx_loadsz = 1,
.flags = UART_CAP_FIFO | UART_CAP_EFR | UART_CAP_SLEEP,
},
[PORT_16650V2] = {
.name = "ST16650V2",
.fifo_size = 32,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01 |
UART_FCR_T_TRIG_00,
.rxtrig_bytes = {8, 16, 24, 28},
.flags = UART_CAP_FIFO | UART_CAP_EFR | UART_CAP_SLEEP,
},
[PORT_16750] = {
.name = "TI16750",
.fifo_size = 64,
.tx_loadsz = 64,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10 |
UART_FCR7_64BYTE,
.rxtrig_bytes = {1, 16, 32, 56},
.flags = UART_CAP_FIFO | UART_CAP_SLEEP | UART_CAP_AFE,
},
[PORT_STARTECH] = {
.name = "Startech",
.fifo_size = 1,
.tx_loadsz = 1,
},
[PORT_16C950] = {
.name = "16C950/954",
.fifo_size = 128,
.tx_loadsz = 128,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01,
.rxtrig_bytes = {16, 32, 112, 120},
/* UART_CAP_EFR breaks billionon CF bluetooth card. */
.flags = UART_CAP_FIFO | UART_CAP_SLEEP,
},
[PORT_16654] = {
.name = "ST16654",
.fifo_size = 64,
.tx_loadsz = 32,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01 |
UART_FCR_T_TRIG_10,
.rxtrig_bytes = {8, 16, 56, 60},
.flags = UART_CAP_FIFO | UART_CAP_EFR | UART_CAP_SLEEP,
},
[PORT_16850] = {
.name = "XR16850",
.fifo_size = 128,
.tx_loadsz = 128,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.flags = UART_CAP_FIFO | UART_CAP_EFR | UART_CAP_SLEEP,
},
[PORT_RSA] = {
.name = "RSA",
.fifo_size = 2048,
.tx_loadsz = 2048,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_11,
.flags = UART_CAP_FIFO,
},
[PORT_NS16550A] = {
.name = "NS16550A",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.flags = UART_CAP_FIFO | UART_NATSEMI,
},
[PORT_XSCALE] = {
.name = "XScale",
.fifo_size = 32,
.tx_loadsz = 32,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.flags = UART_CAP_FIFO | UART_CAP_UUE | UART_CAP_RTOIE,
},
[PORT_OCTEON] = {
.name = "OCTEON",
.fifo_size = 64,
.tx_loadsz = 64,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.flags = UART_CAP_FIFO,
},
[PORT_U6_16550A] = {
.name = "U6_16550A",
.fifo_size = 64,
.tx_loadsz = 64,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.flags = UART_CAP_FIFO | UART_CAP_AFE,
},
[PORT_TEGRA] = {
.name = "Tegra",
.fifo_size = 32,
.tx_loadsz = 8,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01 |
UART_FCR_T_TRIG_01,
.rxtrig_bytes = {1, 4, 8, 14},
.flags = UART_CAP_FIFO | UART_CAP_RTOIE,
},
[PORT_XR17D15X] = {
.name = "XR17D15X",
.fifo_size = 64,
.tx_loadsz = 64,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.flags = UART_CAP_FIFO | UART_CAP_AFE | UART_CAP_EFR |
UART_CAP_SLEEP,
},
[PORT_XR17V35X] = {
.name = "XR17V35X",
.fifo_size = 256,
.tx_loadsz = 256,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_11 |
UART_FCR_T_TRIG_11,
.flags = UART_CAP_FIFO | UART_CAP_AFE | UART_CAP_EFR |
UART_CAP_SLEEP,
},
[PORT_LPC3220] = {
.name = "LPC3220",
.fifo_size = 64,
.tx_loadsz = 32,
.fcr = UART_FCR_DMA_SELECT | UART_FCR_ENABLE_FIFO |
UART_FCR_R_TRIG_00 | UART_FCR_T_TRIG_00,
.flags = UART_CAP_FIFO,
},
[PORT_BRCM_TRUMANAGE] = {
.name = "TruManage",
.fifo_size = 1,
.tx_loadsz = 1024,
.flags = UART_CAP_HFIFO,
},
[PORT_8250_CIR] = {
.name = "CIR port"
},
[PORT_ALTR_16550_F32] = {
.name = "Altera 16550 FIFO32",
.fifo_size = 32,
.tx_loadsz = 32,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 8, 16, 30},
.flags = UART_CAP_FIFO | UART_CAP_AFE,
},
[PORT_ALTR_16550_F64] = {
.name = "Altera 16550 FIFO64",
.fifo_size = 64,
.tx_loadsz = 64,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 16, 32, 62},
.flags = UART_CAP_FIFO | UART_CAP_AFE,
},
[PORT_ALTR_16550_F128] = {
.name = "Altera 16550 FIFO128",
.fifo_size = 128,
.tx_loadsz = 128,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 32, 64, 126},
.flags = UART_CAP_FIFO | UART_CAP_AFE,
},
/*
* tx_loadsz is set to 63-bytes instead of 64-bytes to implement
* workaround of errata A-008006 which states that tx_loadsz should
* be configured less than Maximum supported fifo bytes.
*/
[PORT_16550A_FSL64] = {
.name = "16550A_FSL64",
.fifo_size = 64,
.tx_loadsz = 63,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10 |
UART_FCR7_64BYTE,
.flags = UART_CAP_FIFO | UART_CAP_NOTEMT,
},
[PORT_RT2880] = {
.name = "Palmchip BK-3103",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 4, 8, 14},
.flags = UART_CAP_FIFO,
},
[PORT_DA830] = {
.name = "TI DA8xx/66AK2x",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_DMA_SELECT | UART_FCR_ENABLE_FIFO |
UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 4, 8, 14},
.flags = UART_CAP_FIFO | UART_CAP_AFE,
},
[PORT_MTK_BTIF] = {
.name = "MediaTek BTIF",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO |
UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT,
.flags = UART_CAP_FIFO,
},
[PORT_NPCM] = {
.name = "Nuvoton 16550",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10 |
UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT,
.rxtrig_bytes = {1, 4, 8, 14},
.flags = UART_CAP_FIFO,
},
[PORT_SUNIX] = {
.name = "Sunix",
.fifo_size = 128,
.tx_loadsz = 128,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10,
.rxtrig_bytes = {1, 32, 64, 112},
.flags = UART_CAP_FIFO | UART_CAP_SLEEP,
},
[PORT_ASPEED_VUART] = {
.name = "ASPEED VUART",
.fifo_size = 16,
.tx_loadsz = 16,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_00,
.rxtrig_bytes = {1, 4, 8, 14},
.flags = UART_CAP_FIFO,
},
[PORT_MCHP16550A] = {
.name = "MCHP16550A",
.fifo_size = 256,
.tx_loadsz = 256,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01,
.rxtrig_bytes = {2, 66, 130, 194},
.flags = UART_CAP_FIFO,
},
[PORT_BCM7271] = {
.name = "Broadcom BCM7271 UART",
.fifo_size = 32,
.tx_loadsz = 32,
.fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01,
.rxtrig_bytes = {1, 8, 16, 30},
.flags = UART_CAP_FIFO | UART_CAP_AFE,
},
};
/* Uart divisor latch read */
static u32 default_serial_dl_read(struct uart_8250_port *up)
{
/* Assign these in pieces to truncate any bits above 7. */
unsigned char dll = serial_in(up, UART_DLL);
unsigned char dlm = serial_in(up, UART_DLM);
return dll | dlm << 8;
}
/* Uart divisor latch write */
static void default_serial_dl_write(struct uart_8250_port *up, u32 value)
{
serial_out(up, UART_DLL, value & 0xff);
serial_out(up, UART_DLM, value >> 8 & 0xff);
}
static unsigned int hub6_serial_in(struct uart_port *p, int offset)
{
offset = offset << p->regshift;
outb(p->hub6 - 1 + offset, p->iobase);
return inb(p->iobase + 1);
}
static void hub6_serial_out(struct uart_port *p, int offset, int value)
{
offset = offset << p->regshift;
outb(p->hub6 - 1 + offset, p->iobase);
outb(value, p->iobase + 1);
}
static unsigned int mem_serial_in(struct uart_port *p, int offset)
{
offset = offset << p->regshift;
return readb(p->membase + offset);
}
static void mem_serial_out(struct uart_port *p, int offset, int value)
{
offset = offset << p->regshift;
writeb(value, p->membase + offset);
}
static void mem16_serial_out(struct uart_port *p, int offset, int value)
{
offset = offset << p->regshift;
writew(value, p->membase + offset);
}
static unsigned int mem16_serial_in(struct uart_port *p, int offset)
{
offset = offset << p->regshift;
return readw(p->membase + offset);
}
static void mem32_serial_out(struct uart_port *p, int offset, int value)
{
offset = offset << p->regshift;
writel(value, p->membase + offset);
}
static unsigned int mem32_serial_in(struct uart_port *p, int offset)
{
offset = offset << p->regshift;
return readl(p->membase + offset);
}
static void mem32be_serial_out(struct uart_port *p, int offset, int value)
{
offset = offset << p->regshift;
iowrite32be(value, p->membase + offset);
}
static unsigned int mem32be_serial_in(struct uart_port *p, int offset)
{
offset = offset << p->regshift;
return ioread32be(p->membase + offset);
}
static unsigned int io_serial_in(struct uart_port *p, int offset)
{
offset = offset << p->regshift;
return inb(p->iobase + offset);
}
static void io_serial_out(struct uart_port *p, int offset, int value)
{
offset = offset << p->regshift;
outb(value, p->iobase + offset);
}
static int serial8250_default_handle_irq(struct uart_port *port);
static void set_io_from_upio(struct uart_port *p)
{
struct uart_8250_port *up = up_to_u8250p(p);
up->dl_read = default_serial_dl_read;
up->dl_write = default_serial_dl_write;
switch (p->iotype) {
case UPIO_HUB6:
p->serial_in = hub6_serial_in;
p->serial_out = hub6_serial_out;
break;
case UPIO_MEM:
p->serial_in = mem_serial_in;
p->serial_out = mem_serial_out;
break;
case UPIO_MEM16:
p->serial_in = mem16_serial_in;
p->serial_out = mem16_serial_out;
break;
case UPIO_MEM32:
p->serial_in = mem32_serial_in;
p->serial_out = mem32_serial_out;
break;
case UPIO_MEM32BE:
p->serial_in = mem32be_serial_in;
p->serial_out = mem32be_serial_out;
break;
default:
p->serial_in = io_serial_in;
p->serial_out = io_serial_out;
break;
}
/* Remember loaded iotype */
up->cur_iotype = p->iotype;
p->handle_irq = serial8250_default_handle_irq;
}
static void
serial_port_out_sync(struct uart_port *p, int offset, int value)
{
switch (p->iotype) {
case UPIO_MEM:
case UPIO_MEM16:
case UPIO_MEM32:
case UPIO_MEM32BE:
case UPIO_AU:
p->serial_out(p, offset, value);
p->serial_in(p, UART_LCR); /* safe, no side-effects */
break;
default:
p->serial_out(p, offset, value);
}
}
/*
* FIFO support.
*/
static void serial8250_clear_fifos(struct uart_8250_port *p)
{
if (p->capabilities & UART_CAP_FIFO) {
serial_out(p, UART_FCR, UART_FCR_ENABLE_FIFO);
serial_out(p, UART_FCR, UART_FCR_ENABLE_FIFO |
UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
serial_out(p, UART_FCR, 0);
}
}
static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t);
static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t);
void serial8250_clear_and_reinit_fifos(struct uart_8250_port *p)
{
serial8250_clear_fifos(p);
serial_out(p, UART_FCR, p->fcr);
}
EXPORT_SYMBOL_GPL(serial8250_clear_and_reinit_fifos);
void serial8250_rpm_get(struct uart_8250_port *p)
{
if (!(p->capabilities & UART_CAP_RPM))
return;
pm_runtime_get_sync(p->port.dev);
}
EXPORT_SYMBOL_GPL(serial8250_rpm_get);
void serial8250_rpm_put(struct uart_8250_port *p)
{
if (!(p->capabilities & UART_CAP_RPM))
return;
pm_runtime_mark_last_busy(p->port.dev);
pm_runtime_put_autosuspend(p->port.dev);
}
EXPORT_SYMBOL_GPL(serial8250_rpm_put);
/**
* serial8250_em485_init() - put uart_8250_port into rs485 emulating
* @p: uart_8250_port port instance
*
* The function is used to start rs485 software emulating on the
* &struct uart_8250_port* @p. Namely, RTS is switched before/after
* transmission. The function is idempotent, so it is safe to call it
* multiple times.
*
* The caller MUST enable interrupt on empty shift register before
* calling serial8250_em485_init(). This interrupt is not a part of
* 8250 standard, but implementation defined.
*
* The function is supposed to be called from .rs485_config callback
* or from any other callback protected with p->port.lock spinlock.
*
* See also serial8250_em485_destroy()
*
* Return 0 - success, -errno - otherwise
*/
static int serial8250_em485_init(struct uart_8250_port *p)
{
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&p->port.lock);
if (p->em485)
goto deassert_rts;
p->em485 = kmalloc(sizeof(struct uart_8250_em485), GFP_ATOMIC);
if (!p->em485)
return -ENOMEM;
hrtimer_init(&p->em485->stop_tx_timer, CLOCK_MONOTONIC,
HRTIMER_MODE_REL);
hrtimer_init(&p->em485->start_tx_timer, CLOCK_MONOTONIC,
HRTIMER_MODE_REL);
p->em485->stop_tx_timer.function = &serial8250_em485_handle_stop_tx;
p->em485->start_tx_timer.function = &serial8250_em485_handle_start_tx;
p->em485->port = p;
p->em485->active_timer = NULL;
p->em485->tx_stopped = true;
deassert_rts:
if (p->em485->tx_stopped)
p->rs485_stop_tx(p);
return 0;
}
/**
* serial8250_em485_destroy() - put uart_8250_port into normal state
* @p: uart_8250_port port instance
*
* The function is used to stop rs485 software emulating on the
* &struct uart_8250_port* @p. The function is idempotent, so it is safe to
* call it multiple times.
*
* The function is supposed to be called from .rs485_config callback
* or from any other callback protected with p->port.lock spinlock.
*
* See also serial8250_em485_init()
*/
void serial8250_em485_destroy(struct uart_8250_port *p)
{
if (!p->em485)
return;
hrtimer_cancel(&p->em485->start_tx_timer);
hrtimer_cancel(&p->em485->stop_tx_timer);
kfree(p->em485);
p->em485 = NULL;
}
EXPORT_SYMBOL_GPL(serial8250_em485_destroy);
struct serial_rs485 serial8250_em485_supported = {
.flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND | SER_RS485_RTS_AFTER_SEND |
SER_RS485_TERMINATE_BUS | SER_RS485_RX_DURING_TX,
.delay_rts_before_send = 1,
.delay_rts_after_send = 1,
};
EXPORT_SYMBOL_GPL(serial8250_em485_supported);
/**
* serial8250_em485_config() - generic ->rs485_config() callback
* @port: uart port
* @termios: termios structure
* @rs485: rs485 settings
*
* Generic callback usable by 8250 uart drivers to activate rs485 settings
* if the uart is incapable of driving RTS as a Transmit Enable signal in
* hardware, relying on software emulation instead.
*/
int serial8250_em485_config(struct uart_port *port, struct ktermios *termios,
struct serial_rs485 *rs485)
{
struct uart_8250_port *up = up_to_u8250p(port);
/* pick sane settings if the user hasn't */
if (!!(rs485->flags & SER_RS485_RTS_ON_SEND) ==
!!(rs485->flags & SER_RS485_RTS_AFTER_SEND)) {
rs485->flags |= SER_RS485_RTS_ON_SEND;
rs485->flags &= ~SER_RS485_RTS_AFTER_SEND;
}
/*
* Both serial8250_em485_init() and serial8250_em485_destroy()
* are idempotent.
*/
if (rs485->flags & SER_RS485_ENABLED)
return serial8250_em485_init(up);
serial8250_em485_destroy(up);
return 0;
}
EXPORT_SYMBOL_GPL(serial8250_em485_config);
/*
* These two wrappers ensure that enable_runtime_pm_tx() can be called more than
* once and disable_runtime_pm_tx() will still disable RPM because the fifo is
* empty and the HW can idle again.
*/
void serial8250_rpm_get_tx(struct uart_8250_port *p)
{
unsigned char rpm_active;
if (!(p->capabilities & UART_CAP_RPM))
return;
rpm_active = xchg(&p->rpm_tx_active, 1);
if (rpm_active)
return;
pm_runtime_get_sync(p->port.dev);
}
EXPORT_SYMBOL_GPL(serial8250_rpm_get_tx);
void serial8250_rpm_put_tx(struct uart_8250_port *p)
{
unsigned char rpm_active;
if (!(p->capabilities & UART_CAP_RPM))
return;
rpm_active = xchg(&p->rpm_tx_active, 0);
if (!rpm_active)
return;
pm_runtime_mark_last_busy(p->port.dev);
pm_runtime_put_autosuspend(p->port.dev);
}
EXPORT_SYMBOL_GPL(serial8250_rpm_put_tx);
/*
* IER sleep support. UARTs which have EFRs need the "extended
* capability" bit enabled. Note that on XR16C850s, we need to
* reset LCR to write to IER.
*/
static void serial8250_set_sleep(struct uart_8250_port *p, int sleep)
{
unsigned char lcr = 0, efr = 0;
serial8250_rpm_get(p);
if (p->capabilities & UART_CAP_SLEEP) {
/* Synchronize UART_IER access against the console. */
uart_port_lock_irq(&p->port);
if (p->capabilities & UART_CAP_EFR) {
lcr = serial_in(p, UART_LCR);
efr = serial_in(p, UART_EFR);
serial_out(p, UART_LCR, UART_LCR_CONF_MODE_B);
serial_out(p, UART_EFR, UART_EFR_ECB);
serial_out(p, UART_LCR, 0);
}
serial_out(p, UART_IER, sleep ? UART_IERX_SLEEP : 0);
if (p->capabilities & UART_CAP_EFR) {
serial_out(p, UART_LCR, UART_LCR_CONF_MODE_B);
serial_out(p, UART_EFR, efr);
serial_out(p, UART_LCR, lcr);
}
uart_port_unlock_irq(&p->port);
}
serial8250_rpm_put(p);
}
static void serial8250_clear_IER(struct uart_8250_port *up)
{
if (up->capabilities & UART_CAP_UUE) serial_out(up, UART_IER, UART_IER_UUE);
else
serial_out(up, UART_IER, 0);
}
#ifdef CONFIG_SERIAL_8250_RSA
/*
* Attempts to turn on the RSA FIFO. Returns zero on failure.
* We set the port uart clock rate if we succeed.
*/
static int __enable_rsa(struct uart_8250_port *up)
{
unsigned char mode;
int result;
mode = serial_in(up, UART_RSA_MSR);
result = mode & UART_RSA_MSR_FIFO;
if (!result) {
serial_out(up, UART_RSA_MSR, mode | UART_RSA_MSR_FIFO);
mode = serial_in(up, UART_RSA_MSR);
result = mode & UART_RSA_MSR_FIFO;
}
if (result)
up->port.uartclk = SERIAL_RSA_BAUD_BASE * 16;
return result;
}
static void enable_rsa(struct uart_8250_port *up)
{
if (up->port.type == PORT_RSA) {
if (up->port.uartclk != SERIAL_RSA_BAUD_BASE * 16) {
uart_port_lock_irq(&up->port);
__enable_rsa(up);
uart_port_unlock_irq(&up->port);
}
if (up->port.uartclk == SERIAL_RSA_BAUD_BASE * 16)
serial_out(up, UART_RSA_FRR, 0);
}
}
/*
* Attempts to turn off the RSA FIFO. Returns zero on failure.
* It is unknown why interrupts were disabled in here. However,
* the caller is expected to preserve this behaviour by grabbing
* the spinlock before calling this function.
*/
static void disable_rsa(struct uart_8250_port *up)
{
unsigned char mode;
int result;
if (up->port.type == PORT_RSA &&
up->port.uartclk == SERIAL_RSA_BAUD_BASE * 16) {
uart_port_lock_irq(&up->port);
mode = serial_in(up, UART_RSA_MSR);
result = !(mode & UART_RSA_MSR_FIFO);
if (!result) {
serial_out(up, UART_RSA_MSR, mode & ~UART_RSA_MSR_FIFO);
mode = serial_in(up, UART_RSA_MSR);
result = !(mode & UART_RSA_MSR_FIFO);
}
if (result)
up->port.uartclk = SERIAL_RSA_BAUD_BASE_LO * 16;
uart_port_unlock_irq(&up->port);
}
}
#endif /* CONFIG_SERIAL_8250_RSA */
/*
* This is a quickie test to see how big the FIFO is.
* It doesn't work at all the time, more's the pity.
*/
static int size_fifo(struct uart_8250_port *up)
{
unsigned char old_fcr, old_mcr, old_lcr;
u32 old_dl;
int count;
old_lcr = serial_in(up, UART_LCR);
serial_out(up, UART_LCR, 0);
old_fcr = serial_in(up, UART_FCR);
old_mcr = serial8250_in_MCR(up);
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO |
UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
serial8250_out_MCR(up, UART_MCR_LOOP);
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);
old_dl = serial_dl_read(up);
serial_dl_write(up, 0x0001);
serial_out(up, UART_LCR, UART_LCR_WLEN8);
for (count = 0; count < 256; count++)
serial_out(up, UART_TX, count);
mdelay(20);/* FIXME - schedule_timeout */
for (count = 0; (serial_in(up, UART_LSR) & UART_LSR_DR) &&
(count < 256); count++)
serial_in(up, UART_RX);
serial_out(up, UART_FCR, old_fcr);
serial8250_out_MCR(up, old_mcr);
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);
serial_dl_write(up, old_dl);
serial_out(up, UART_LCR, old_lcr);
return count;
}
/*
* Read UART ID using the divisor method - set DLL and DLM to zero
* and the revision will be in DLL and device type in DLM. We
* preserve the device state across this.
*/
static unsigned int autoconfig_read_divisor_id(struct uart_8250_port *p)
{
unsigned char old_lcr;
unsigned int id, old_dl;
old_lcr = serial_in(p, UART_LCR);
serial_out(p, UART_LCR, UART_LCR_CONF_MODE_A);
old_dl = serial_dl_read(p);
serial_dl_write(p, 0);
id = serial_dl_read(p);
serial_dl_write(p, old_dl);
serial_out(p, UART_LCR, old_lcr);
return id;
}
/*
* This is a helper routine to autodetect StarTech/Exar/Oxsemi UART's.
* When this function is called we know it is at least a StarTech
* 16650 V2, but it might be one of several StarTech UARTs, or one of
* its clones. (We treat the broken original StarTech 16650 V1 as a
* 16550, and why not? Startech doesn't seem to even acknowledge its
* existence.)
*
* What evil have men's minds wrought...
*/
static void autoconfig_has_efr(struct uart_8250_port *up)
{
unsigned int id1, id2, id3, rev;
/*
* Everything with an EFR has SLEEP
*/
up->capabilities |= UART_CAP_EFR | UART_CAP_SLEEP;
/*
* First we check to see if it's an Oxford Semiconductor UART.
*
* If we have to do this here because some non-National
* Semiconductor clone chips lock up if you try writing to the
* LSR register (which serial_icr_read does)
*/
/*
* Check for Oxford Semiconductor 16C950.
*
* EFR [4] must be set else this test fails.
*
* This shouldn't be necessary, but Mike Hudson (Exoray@isys.ca)
* claims that it's needed for 952 dual UART's (which are not
* recommended for new designs).
*/
up->acr = 0;
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
serial_out(up, UART_EFR, UART_EFR_ECB);
serial_out(up, UART_LCR, 0x00);
id1 = serial_icr_read(up, UART_ID1);
id2 = serial_icr_read(up, UART_ID2);
id3 = serial_icr_read(up, UART_ID3);
rev = serial_icr_read(up, UART_REV);
DEBUG_AUTOCONF("950id=%02x:%02x:%02x:%02x ", id1, id2, id3, rev);
if (id1 == 0x16 && id2 == 0xC9 &&
(id3 == 0x50 || id3 == 0x52 || id3 == 0x54)) {
up->port.type = PORT_16C950;
/*
* Enable work around for the Oxford Semiconductor 952 rev B
* chip which causes it to seriously miscalculate baud rates
* when DLL is 0.
*/
if (id3 == 0x52 && rev == 0x01)
up->bugs |= UART_BUG_QUOT;
return;
}
/*
* We check for a XR16C850 by setting DLL and DLM to 0, and then
* reading back DLL and DLM. The chip type depends on the DLM
* value read back:
* 0x10 - XR16C850 and the DLL contains the chip revision.
* 0x12 - XR16C2850.
* 0x14 - XR16C854.
*/
id1 = autoconfig_read_divisor_id(up);
DEBUG_AUTOCONF("850id=%04x ", id1);
id2 = id1 >> 8;
if (id2 == 0x10 || id2 == 0x12 || id2 == 0x14) {
up->port.type = PORT_16850;
return;
}
/*
* It wasn't an XR16C850.
*
* We distinguish between the '654 and the '650 by counting
* how many bytes are in the FIFO. I'm using this for now,
* since that's the technique that was sent to me in the
* serial driver update, but I'm not convinced this works.
* I've had problems doing this in the past. -TYT
*/
if (size_fifo(up) == 64)
up->port.type = PORT_16654;
else
up->port.type = PORT_16650V2;
}
/*
* We detected a chip without a FIFO. Only two fall into
* this category - the original 8250 and the 16450. The
* 16450 has a scratch register (accessible with LCR=0)
*/
static void autoconfig_8250(struct uart_8250_port *up)
{
unsigned char scratch, status1, status2;
up->port.type = PORT_8250;
scratch = serial_in(up, UART_SCR);
serial_out(up, UART_SCR, 0xa5);
status1 = serial_in(up, UART_SCR);
serial_out(up, UART_SCR, 0x5a);
status2 = serial_in(up, UART_SCR);
serial_out(up, UART_SCR, scratch);
if (status1 == 0xa5 && status2 == 0x5a)
up->port.type = PORT_16450;
}
static int broken_efr(struct uart_8250_port *up)
{
/*
* Exar ST16C2550 "A2" devices incorrectly detect as
* having an EFR, and report an ID of 0x0201. See
* http://linux.derkeiler.com/Mailing-Lists/Kernel/2004-11/4812.html
*/
if (autoconfig_read_divisor_id(up) == 0x0201 && size_fifo(up) == 16)
return 1;
return 0;
}
/*
* We know that the chip has FIFOs. Does it have an EFR? The
* EFR is located in the same register position as the IIR and
* we know the top two bits of the IIR are currently set. The
* EFR should contain zero. Try to read the EFR.
*/
static void autoconfig_16550a(struct uart_8250_port *up)
{
unsigned char status1, status2;
unsigned int iersave;
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&up->port.lock);
up->port.type = PORT_16550A;
up->capabilities |= UART_CAP_FIFO;
if (!IS_ENABLED(CONFIG_SERIAL_8250_16550A_VARIANTS) &&
!(up->port.flags & UPF_FULL_PROBE))
return;
/*
* Check for presence of the EFR when DLAB is set.
* Only ST16C650V1 UARTs pass this test.
*/
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);
if (serial_in(up, UART_EFR) == 0) {
serial_out(up, UART_EFR, 0xA8);
if (serial_in(up, UART_EFR) != 0) {
DEBUG_AUTOCONF("EFRv1 ");
up->port.type = PORT_16650;
up->capabilities |= UART_CAP_EFR | UART_CAP_SLEEP;
} else {
serial_out(up, UART_LCR, 0);
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO |
UART_FCR7_64BYTE);
status1 = serial_in(up, UART_IIR) & UART_IIR_FIFO_ENABLED_16750;
serial_out(up, UART_FCR, 0);
serial_out(up, UART_LCR, 0);
if (status1 == UART_IIR_FIFO_ENABLED_16750)
up->port.type = PORT_16550A_FSL64;
else
DEBUG_AUTOCONF("Motorola 8xxx DUART ");
}
serial_out(up, UART_EFR, 0);
return;
}
/*
* Maybe it requires 0xbf to be written to the LCR.
* (other ST16C650V2 UARTs, TI16C752A, etc)
*/
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
if (serial_in(up, UART_EFR) == 0 && !broken_efr(up)) {
DEBUG_AUTOCONF("EFRv2 ");
autoconfig_has_efr(up);
return;
}
/*
* Check for a National Semiconductor SuperIO chip.
* Attempt to switch to bank 2, read the value of the LOOP bit
* from EXCR1. Switch back to bank 0, change it in MCR. Then
* switch back to bank 2, read it from EXCR1 again and check
* it's changed. If so, set baud_base in EXCR2 to 921600. -- dwmw2
*/
serial_out(up, UART_LCR, 0);
status1 = serial8250_in_MCR(up);
serial_out(up, UART_LCR, 0xE0);
status2 = serial_in(up, 0x02); /* EXCR1 */
if (!((status2 ^ status1) & UART_MCR_LOOP)) {
serial_out(up, UART_LCR, 0);
serial8250_out_MCR(up, status1 ^ UART_MCR_LOOP);
serial_out(up, UART_LCR, 0xE0);
status2 = serial_in(up, 0x02); /* EXCR1 */
serial_out(up, UART_LCR, 0);
serial8250_out_MCR(up, status1);
if ((status2 ^ status1) & UART_MCR_LOOP) {
unsigned short quot;
serial_out(up, UART_LCR, 0xE0);
quot = serial_dl_read(up);
quot <<= 3;
if (ns16550a_goto_highspeed(up))
serial_dl_write(up, quot);
serial_out(up, UART_LCR, 0);
up->port.uartclk = 921600*16;
up->port.type = PORT_NS16550A;
up->capabilities |= UART_NATSEMI;
return;
}
}
/*
* No EFR. Try to detect a TI16750, which only sets bit 5 of
* the IIR when 64 byte FIFO mode is enabled when DLAB is set.
* Try setting it with and without DLAB set. Cheap clones
* set bit 5 without DLAB set.
*/
serial_out(up, UART_LCR, 0);
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO | UART_FCR7_64BYTE);
status1 = serial_in(up, UART_IIR) & UART_IIR_FIFO_ENABLED_16750;
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO);
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO | UART_FCR7_64BYTE);
status2 = serial_in(up, UART_IIR) & UART_IIR_FIFO_ENABLED_16750;
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO);
serial_out(up, UART_LCR, 0);
DEBUG_AUTOCONF("iir1=%d iir2=%d ", status1, status2);
if (status1 == UART_IIR_FIFO_ENABLED_16550A &&
status2 == UART_IIR_FIFO_ENABLED_16750) {
up->port.type = PORT_16750;
up->capabilities |= UART_CAP_AFE | UART_CAP_SLEEP;
return;
}
/*
* Try writing and reading the UART_IER_UUE bit (b6).
* If it works, this is probably one of the Xscale platform's
* internal UARTs.
* We're going to explicitly set the UUE bit to 0 before
* trying to write and read a 1 just to make sure it's not
* already a 1 and maybe locked there before we even start.
*/
iersave = serial_in(up, UART_IER);
serial_out(up, UART_IER, iersave & ~UART_IER_UUE);
if (!(serial_in(up, UART_IER) & UART_IER_UUE)) {
/*
* OK it's in a known zero state, try writing and reading
* without disturbing the current state of the other bits.
*/
serial_out(up, UART_IER, iersave | UART_IER_UUE);
if (serial_in(up, UART_IER) & UART_IER_UUE) {
/*
* It's an Xscale.
* We'll leave the UART_IER_UUE bit set to 1 (enabled).
*/
DEBUG_AUTOCONF("Xscale ");
up->port.type = PORT_XSCALE;
up->capabilities |= UART_CAP_UUE | UART_CAP_RTOIE;
return;
}
} else {
/*
* If we got here we couldn't force the IER_UUE bit to 0.
* Log it and continue.
*/
DEBUG_AUTOCONF("Couldn't force IER_UUE to 0 ");
}
serial_out(up, UART_IER, iersave);
/*
* We distinguish between 16550A and U6 16550A by counting
* how many bytes are in the FIFO.
*/
if (up->port.type == PORT_16550A && size_fifo(up) == 64) {
up->port.type = PORT_U6_16550A;
up->capabilities |= UART_CAP_AFE;
}
}
/*
* This routine is called by rs_init() to initialize a specific serial
* port. It determines what type of UART chip this serial port is
* using: 8250, 16450, 16550, 16550A. The important question is
* whether or not this UART is a 16550A or not, since this will
* determine whether or not we can use its FIFO features or not.
*/
static void autoconfig(struct uart_8250_port *up)
{
unsigned char status1, scratch, scratch2, scratch3;
unsigned char save_lcr, save_mcr;
struct uart_port *port = &up->port;
unsigned long flags;
unsigned int old_capabilities;
if (!port->iobase && !port->mapbase && !port->membase)
return;
DEBUG_AUTOCONF("%s: autoconf (0x%04lx, 0x%p): ",
port->name, port->iobase, port->membase);
/*
* We really do need global IRQs disabled here - we're going to
* be frobbing the chips IRQ enable register to see if it exists.
*
* Synchronize UART_IER access against the console.
*/
uart_port_lock_irqsave(port, &flags);
up->capabilities = 0;
up->bugs = 0;
if (!(port->flags & UPF_BUGGY_UART)) {
/*
* Do a simple existence test first; if we fail this,
* there's no point trying anything else.
*
* 0x80 is used as a nonsense port to prevent against
* false positives due to ISA bus float. The
* assumption is that 0x80 is a non-existent port;
* which should be safe since include/asm/io.h also
* makes this assumption.
*
* Note: this is safe as long as MCR bit 4 is clear
* and the device is in "PC" mode.
*/
scratch = serial_in(up, UART_IER);
serial_out(up, UART_IER, 0);
#ifdef __i386__
outb(0xff, 0x080);
#endif
/*
* Mask out IER[7:4] bits for test as some UARTs (e.g. TL
* 16C754B) allow only to modify them if an EFR bit is set.
*/
scratch2 = serial_in(up, UART_IER) & UART_IER_ALL_INTR;
serial_out(up, UART_IER, UART_IER_ALL_INTR);
#ifdef __i386__
outb(0, 0x080);
#endif
scratch3 = serial_in(up, UART_IER) & UART_IER_ALL_INTR;
serial_out(up, UART_IER, scratch);
if (scratch2 != 0 || scratch3 != UART_IER_ALL_INTR) {
/*
* We failed; there's nothing here
*/
uart_port_unlock_irqrestore(port, flags);
DEBUG_AUTOCONF("IER test failed (%02x, %02x) ",
scratch2, scratch3);
goto out;
}
}
save_mcr = serial8250_in_MCR(up);
save_lcr = serial_in(up, UART_LCR);
/*
* Check to see if a UART is really there. Certain broken
* internal modems based on the Rockwell chipset fail this
* test, because they apparently don't implement the loopback
* test mode. So this test is skipped on the COM 1 through
* COM 4 ports. This *should* be safe, since no board
* manufacturer would be stupid enough to design a board
* that conflicts with COM 1-4 --- we hope!
*/
if (!(port->flags & UPF_SKIP_TEST)) {
serial8250_out_MCR(up, UART_MCR_LOOP | UART_MCR_OUT2 | UART_MCR_RTS);
status1 = serial_in(up, UART_MSR) & UART_MSR_STATUS_BITS;
serial8250_out_MCR(up, save_mcr);
if (status1 != (UART_MSR_DCD | UART_MSR_CTS)) {
uart_port_unlock_irqrestore(port, flags);
DEBUG_AUTOCONF("LOOP test failed (%02x) ",
status1);
goto out;
}
}
/*
* We're pretty sure there's a port here. Lets find out what
* type of port it is. The IIR top two bits allows us to find
* out if it's 8250 or 16450, 16550, 16550A or later. This
* determines what we test for next.
*
* We also initialise the EFR (if any) to zero for later. The
* EFR occupies the same register location as the FCR and IIR.
*/
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
serial_out(up, UART_EFR, 0);
serial_out(up, UART_LCR, 0);
serial_out(up, UART_FCR, UART_FCR_ENABLE_FIFO);
switch (serial_in(up, UART_IIR) & UART_IIR_FIFO_ENABLED) {
case UART_IIR_FIFO_ENABLED_8250:
autoconfig_8250(up);
break;
case UART_IIR_FIFO_ENABLED_16550:
port->type = PORT_16550;
break;
case UART_IIR_FIFO_ENABLED_16550A:
autoconfig_16550a(up);
break;
default:
port->type = PORT_UNKNOWN;
break;
}
#ifdef CONFIG_SERIAL_8250_RSA
/*
* Only probe for RSA ports if we got the region.
*/
if (port->type == PORT_16550A && up->probe & UART_PROBE_RSA &&
__enable_rsa(up))
port->type = PORT_RSA;
#endif
serial_out(up, UART_LCR, save_lcr);
port->fifosize = uart_config[up->port.type].fifo_size;
old_capabilities = up->capabilities;
up->capabilities = uart_config[port->type].flags;
up->tx_loadsz = uart_config[port->type].tx_loadsz;
if (port->type == PORT_UNKNOWN)
goto out_unlock;
/*
* Reset the UART.
*/
#ifdef CONFIG_SERIAL_8250_RSA
if (port->type == PORT_RSA)
serial_out(up, UART_RSA_FRR, 0);
#endif
serial8250_out_MCR(up, save_mcr);
serial8250_clear_fifos(up);
serial_in(up, UART_RX);
serial8250_clear_IER(up);
out_unlock:
uart_port_unlock_irqrestore(port, flags);
/*
* Check if the device is a Fintek F81216A
*/
if (port->type == PORT_16550A && port->iotype == UPIO_PORT)
fintek_8250_probe(up);
if (up->capabilities != old_capabilities) {
dev_warn(port->dev, "detected caps %08x should be %08x\n",
old_capabilities, up->capabilities);
}
out:
DEBUG_AUTOCONF("iir=%d ", scratch);
DEBUG_AUTOCONF("type=%s\n", uart_config[port->type].name);
}
static void autoconfig_irq(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
unsigned char save_mcr, save_ier;
unsigned char save_ICP = 0;
unsigned int ICP = 0;
unsigned long irqs;
int irq;
if (port->flags & UPF_FOURPORT) {
ICP = (port->iobase & 0xfe0) | 0x1f;
save_ICP = inb_p(ICP);
outb_p(0x80, ICP);
inb_p(ICP);
}
/* forget possible initially masked and pending IRQ */
probe_irq_off(probe_irq_on());
save_mcr = serial8250_in_MCR(up);
/* Synchronize UART_IER access against the console. */
uart_port_lock_irq(port);
save_ier = serial_in(up, UART_IER);
uart_port_unlock_irq(port);
serial8250_out_MCR(up, UART_MCR_OUT1 | UART_MCR_OUT2);
irqs = probe_irq_on();
serial8250_out_MCR(up, 0);
udelay(10);
if (port->flags & UPF_FOURPORT) {
serial8250_out_MCR(up, UART_MCR_DTR | UART_MCR_RTS);
} else {
serial8250_out_MCR(up,
UART_MCR_DTR | UART_MCR_RTS | UART_MCR_OUT2);
}
/* Synchronize UART_IER access against the console. */
uart_port_lock_irq(port);
serial_out(up, UART_IER, UART_IER_ALL_INTR);
uart_port_unlock_irq(port);
serial_in(up, UART_LSR);
serial_in(up, UART_RX);
serial_in(up, UART_IIR);
serial_in(up, UART_MSR);
serial_out(up, UART_TX, 0xFF);
udelay(20);
irq = probe_irq_off(irqs);
serial8250_out_MCR(up, save_mcr);
/* Synchronize UART_IER access against the console. */
uart_port_lock_irq(port);
serial_out(up, UART_IER, save_ier);
uart_port_unlock_irq(port);
if (port->flags & UPF_FOURPORT)
outb_p(save_ICP, ICP);
port->irq = (irq > 0) ? irq : 0;
}
static void serial8250_stop_rx(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&port->lock);
serial8250_rpm_get(up);
up->ier &= ~(UART_IER_RLSI | UART_IER_RDI);
up->port.read_status_mask &= ~UART_LSR_DR;
serial_port_out(port, UART_IER, up->ier);
serial8250_rpm_put(up);
}
/**
* serial8250_em485_stop_tx() - generic ->rs485_stop_tx() callback
* @p: uart 8250 port
*
* Generic callback usable by 8250 uart drivers to stop rs485 transmission.
*/
void serial8250_em485_stop_tx(struct uart_8250_port *p)
{
unsigned char mcr = serial8250_in_MCR(p);
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&p->port.lock);
if (p->port.rs485.flags & SER_RS485_RTS_AFTER_SEND)
mcr |= UART_MCR_RTS;
else
mcr &= ~UART_MCR_RTS;
serial8250_out_MCR(p, mcr);
/*
* Empty the RX FIFO, we are not interested in anything
* received during the half-duplex transmission.
* Enable previously disabled RX interrupts.
*/
if (!(p->port.rs485.flags & SER_RS485_RX_DURING_TX)) {
serial8250_clear_and_reinit_fifos(p);
p->ier |= UART_IER_RLSI | UART_IER_RDI;
serial_port_out(&p->port, UART_IER, p->ier);
}
}
EXPORT_SYMBOL_GPL(serial8250_em485_stop_tx);
static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t)
{
struct uart_8250_em485 *em485 = container_of(t, struct uart_8250_em485,
stop_tx_timer);
struct uart_8250_port *p = em485->port;
unsigned long flags;
serial8250_rpm_get(p);
uart_port_lock_irqsave(&p->port, &flags);
if (em485->active_timer == &em485->stop_tx_timer) {
p->rs485_stop_tx(p);
em485->active_timer = NULL;
em485->tx_stopped = true;
}
uart_port_unlock_irqrestore(&p->port, flags);
serial8250_rpm_put(p);
return HRTIMER_NORESTART;
}
static void start_hrtimer_ms(struct hrtimer *hrt, unsigned long msec)
{
hrtimer_start(hrt, ms_to_ktime(msec), HRTIMER_MODE_REL);
}
static void __stop_tx_rs485(struct uart_8250_port *p, u64 stop_delay)
{
struct uart_8250_em485 *em485 = p->em485;
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&p->port.lock);
stop_delay += (u64)p->port.rs485.delay_rts_after_send * NSEC_PER_MSEC;
/*
* rs485_stop_tx() is going to set RTS according to config
* AND flush RX FIFO if required.
*/
if (stop_delay > 0) {
em485->active_timer = &em485->stop_tx_timer;
hrtimer_start(&em485->stop_tx_timer, ns_to_ktime(stop_delay), HRTIMER_MODE_REL);
} else {
p->rs485_stop_tx(p);
em485->active_timer = NULL;
em485->tx_stopped = true;
}
}
static inline void __stop_tx(struct uart_8250_port *p)
{
struct uart_8250_em485 *em485 = p->em485;
if (em485) {
u16 lsr = serial_lsr_in(p);
u64 stop_delay = 0;
if (!(lsr & UART_LSR_THRE))
return;
/*
* To provide required timing and allow FIFO transfer,
* __stop_tx_rs485() must be called only when both FIFO and
* shift register are empty. The device driver should either
* enable interrupt on TEMT or set UART_CAP_NOTEMT that will
* enlarge stop_tx_timer by the tx time of one frame to cover
* for emptying of the shift register.
*/
if (!(lsr & UART_LSR_TEMT)) {
if (!(p->capabilities & UART_CAP_NOTEMT))
return;
/*
* RTS might get deasserted too early with the normal
* frame timing formula. It seems to suggest THRE might
* get asserted already during tx of the stop bit
* rather than after it is fully sent.
* Roughly estimate 1 extra bit here with / 7.
*/
stop_delay = p->port.frame_time + DIV_ROUND_UP(p->port.frame_time, 7);
}
__stop_tx_rs485(p, stop_delay);
}
if (serial8250_clear_THRI(p))
serial8250_rpm_put_tx(p);
}
static void serial8250_stop_tx(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
serial8250_rpm_get(up);
__stop_tx(up);
/*
* We really want to stop the transmitter from sending.
*/
if (port->type == PORT_16C950) {
up->acr |= UART_ACR_TXDIS;
serial_icr_write(up, UART_ACR, up->acr);
}
serial8250_rpm_put(up);
}
static inline void __start_tx(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
if (up->dma && !up->dma->tx_dma(up))
return;
if (serial8250_set_THRI(up)) {
if (up->bugs & UART_BUG_TXEN) {
u16 lsr = serial_lsr_in(up);
if (lsr & UART_LSR_THRE)
serial8250_tx_chars(up);
}
}
/*
* Re-enable the transmitter if we disabled it.
*/
if (port->type == PORT_16C950 && up->acr & UART_ACR_TXDIS) {
up->acr &= ~UART_ACR_TXDIS;
serial_icr_write(up, UART_ACR, up->acr);
}
}
/**
* serial8250_em485_start_tx() - generic ->rs485_start_tx() callback
* @up: uart 8250 port
*
* Generic callback usable by 8250 uart drivers to start rs485 transmission.
* Assumes that setting the RTS bit in the MCR register means RTS is high.
* (Some chips use inverse semantics.) Further assumes that reception is
* stoppable by disabling the UART_IER_RDI interrupt. (Some chips set the
* UART_LSR_DR bit even when UART_IER_RDI is disabled, foiling this approach.)
*/
void serial8250_em485_start_tx(struct uart_8250_port *up)
{
unsigned char mcr = serial8250_in_MCR(up);
if (!(up->port.rs485.flags & SER_RS485_RX_DURING_TX))
serial8250_stop_rx(&up->port);
if (up->port.rs485.flags & SER_RS485_RTS_ON_SEND)
mcr |= UART_MCR_RTS;
else
mcr &= ~UART_MCR_RTS;
serial8250_out_MCR(up, mcr);
}
EXPORT_SYMBOL_GPL(serial8250_em485_start_tx);
/* Returns false, if start_tx_timer was setup to defer TX start */
static bool start_tx_rs485(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
struct uart_8250_em485 *em485 = up->em485;
/*
* While serial8250_em485_handle_stop_tx() is a noop if
* em485->active_timer != &em485->stop_tx_timer, it might happen that
* the timer is still armed and triggers only after the current bunch of
* chars is send and em485->active_timer == &em485->stop_tx_timer again.
* So cancel the timer. There is still a theoretical race condition if
* the timer is already running and only comes around to check for
* em485->active_timer when &em485->stop_tx_timer is armed again.
*/
if (em485->active_timer == &em485->stop_tx_timer)
hrtimer_try_to_cancel(&em485->stop_tx_timer);
em485->active_timer = NULL;
if (em485->tx_stopped) {
em485->tx_stopped = false;
up->rs485_start_tx(up);
if (up->port.rs485.delay_rts_before_send > 0) {
em485->active_timer = &em485->start_tx_timer;
start_hrtimer_ms(&em485->start_tx_timer,
up->port.rs485.delay_rts_before_send);
return false;
}
}
return true;
}
static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t)
{
struct uart_8250_em485 *em485 = container_of(t, struct uart_8250_em485,
start_tx_timer);
struct uart_8250_port *p = em485->port;
unsigned long flags;
uart_port_lock_irqsave(&p->port, &flags);
if (em485->active_timer == &em485->start_tx_timer) {
__start_tx(&p->port);
em485->active_timer = NULL;
}
uart_port_unlock_irqrestore(&p->port, flags);
return HRTIMER_NORESTART;
}
static void serial8250_start_tx(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
struct uart_8250_em485 *em485 = up->em485;
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&port->lock);
if (!port->x_char && uart_circ_empty(&port->state->xmit))
return;
serial8250_rpm_get_tx(up);
if (em485) {
if ((em485->active_timer == &em485->start_tx_timer) ||
!start_tx_rs485(port))
return;
}
__start_tx(port);
}
static void serial8250_throttle(struct uart_port *port)
{
port->throttle(port);
}
static void serial8250_unthrottle(struct uart_port *port)
{
port->unthrottle(port);
}
static void serial8250_disable_ms(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&port->lock);
/* no MSR capabilities */
if (up->bugs & UART_BUG_NOMSR)
return;
mctrl_gpio_disable_ms(up->gpios);
up->ier &= ~UART_IER_MSI;
serial_port_out(port, UART_IER, up->ier);
}
static void serial8250_enable_ms(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
/* Port locked to synchronize UART_IER access against the console. */
lockdep_assert_held_once(&port->lock);
/* no MSR capabilities */
if (up->bugs & UART_BUG_NOMSR)
return;
mctrl_gpio_enable_ms(up->gpios);
up->ier |= UART_IER_MSI;
serial8250_rpm_get(up);
serial_port_out(port, UART_IER, up->ier);
serial8250_rpm_put(up);
}
void serial8250_read_char(struct uart_8250_port *up, u16 lsr)
{
struct uart_port *port = &up->port;
u8 ch, flag = TTY_NORMAL;
if (likely(lsr & UART_LSR_DR))
ch = serial_in(up, UART_RX);
else
/*
* Intel 82571 has a Serial Over Lan device that will
* set UART_LSR_BI without setting UART_LSR_DR when
* it receives a break. To avoid reading from the
* receive buffer without UART_LSR_DR bit set, we
* just force the read character to be 0
*/
ch = 0;
port->icount.rx++;
lsr |= up->lsr_saved_flags;
up->lsr_saved_flags = 0;
if (unlikely(lsr & UART_LSR_BRK_ERROR_BITS)) {
if (lsr & UART_LSR_BI) {
lsr &= ~(UART_LSR_FE | UART_LSR_PE);
port->icount.brk++;
/*
* We do the SysRQ and SAK checking
* here because otherwise the break
* may get masked by ignore_status_mask
* or read_status_mask.
*/
if (uart_handle_break(port))
return;
} else if (lsr & UART_LSR_PE)
port->icount.parity++;
else if (lsr & UART_LSR_FE)
port->icount.frame++;
if (lsr & UART_LSR_OE)
port->icount.overrun++;
/*
* Mask off conditions which should be ignored.
*/
lsr &= port->read_status_mask;
if (lsr & UART_LSR_BI) {
dev_dbg(port->dev, "handling break\n");
flag = TTY_BREAK;
} else if (lsr & UART_LSR_PE)
flag = TTY_PARITY;
else if (lsr & UART_LSR_FE)
flag = TTY_FRAME;
}
if (uart_prepare_sysrq_char(port, ch))
return;
uart_insert_char(port, lsr, UART_LSR_OE, ch, flag);
}
EXPORT_SYMBOL_GPL(serial8250_read_char);
/*
* serial8250_rx_chars - Read characters. The first LSR value must be passed in.
*
* Returns LSR bits. The caller should rely only on non-Rx related LSR bits
* (such as THRE) because the LSR value might come from an already consumed
* character.
*/
u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr)
{
struct uart_port *port = &up->port;
int max_count = 256;
do {
serial8250_read_char(up, lsr);
if (--max_count == 0)
break;
lsr = serial_in(up, UART_LSR);
} while (lsr & (UART_LSR_DR | UART_LSR_BI));
tty_flip_buffer_push(&port->state->port);
return lsr;
}
EXPORT_SYMBOL_GPL(serial8250_rx_chars);
void serial8250_tx_chars(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
struct circ_buf *xmit = &port->state->xmit;
int count;
if (port->x_char) {
uart_xchar_out(port, UART_TX);
return;
}
if (uart_tx_stopped(port)) {
serial8250_stop_tx(port);
return;
}
if (uart_circ_empty(xmit)) {
__stop_tx(up);
return;
}
count = up->tx_loadsz;
do {
serial_out(up, UART_TX, xmit->buf[xmit->tail]);
if (up->bugs & UART_BUG_TXRACE) {
/*
* The Aspeed BMC virtual UARTs have a bug where data
* may get stuck in the BMC's Tx FIFO from bursts of
* writes on the APB interface.
*
* Delay back-to-back writes by a read cycle to avoid
* stalling the VUART. Read a register that won't have
* side-effects and discard the result.
*/
serial_in(up, UART_SCR);
}
uart_xmit_advance(port, 1);
if (uart_circ_empty(xmit))
break;
if ((up->capabilities & UART_CAP_HFIFO) &&
!uart_lsr_tx_empty(serial_in(up, UART_LSR)))
break;
/* The BCM2835 MINI UART THRE bit is really a not-full bit. */
if ((up->capabilities & UART_CAP_MINI) &&
!(serial_in(up, UART_LSR) & UART_LSR_THRE))
break;
} while (--count > 0);
if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
uart_write_wakeup(port);
/*
* With RPM enabled, we have to wait until the FIFO is empty before the
* HW can go idle. So we get here once again with empty FIFO and disable
* the interrupt and RPM in __stop_tx()
*/
if (uart_circ_empty(xmit) && !(up->capabilities & UART_CAP_RPM))
__stop_tx(up);
}
EXPORT_SYMBOL_GPL(serial8250_tx_chars);
/* Caller holds uart port lock */
unsigned int serial8250_modem_status(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
unsigned int status = serial_in(up, UART_MSR);
status |= up->msr_saved_flags;
up->msr_saved_flags = 0;
if (status & UART_MSR_ANY_DELTA && up->ier & UART_IER_MSI &&
port->state != NULL) {
if (status & UART_MSR_TERI)
port->icount.rng++;
if (status & UART_MSR_DDSR)
port->icount.dsr++;
if (status & UART_MSR_DDCD)
uart_handle_dcd_change(port, status & UART_MSR_DCD);
if (status & UART_MSR_DCTS)
uart_handle_cts_change(port, status & UART_MSR_CTS);
wake_up_interruptible(&port->state->port.delta_msr_wait);
}
return status;
}
EXPORT_SYMBOL_GPL(serial8250_modem_status);
static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)
{
switch (iir & 0x3f) {
case UART_IIR_THRI:
/*
* Postpone DMA or not decision to IIR_RDI or IIR_RX_TIMEOUT
* because it's impossible to do an informed decision about
* that with IIR_THRI.
*
* This also fixes one known DMA Rx corruption issue where
* DR is asserted but DMA Rx only gets a corrupted zero byte
* (too early DR?).
*/
return false;
case UART_IIR_RDI:
if (!up->dma->rx_running)
break;
fallthrough;
case UART_IIR_RLSI:
case UART_IIR_RX_TIMEOUT:
serial8250_rx_dma_flush(up);
return true;
}
return up->dma->rx_dma(up);
}
/*
* This handles the interrupt from one port.
*/
int serial8250_handle_irq(struct uart_port *port, unsigned int iir)
{
struct uart_8250_port *up = up_to_u8250p(port);
struct tty_port *tport = &port->state->port;
bool skip_rx = false;
unsigned long flags;
u16 status;
if (iir & UART_IIR_NO_INT)
return 0;
uart_port_lock_irqsave(port, &flags);
status = serial_lsr_in(up);
/*
* If port is stopped and there are no error conditions in the
* FIFO, then don't drain the FIFO, as this may lead to TTY buffer
* overflow. Not servicing, RX FIFO would trigger auto HW flow
* control when FIFO occupancy reaches preset threshold, thus
* halting RX. This only works when auto HW flow control is
* available.
*/
if (!(status & (UART_LSR_FIFOE | UART_LSR_BRK_ERROR_BITS)) &&
(port->status & (UPSTAT_AUTOCTS | UPSTAT_AUTORTS)) &&
!(port->read_status_mask & UART_LSR_DR))
skip_rx = true;
if (status & (UART_LSR_DR | UART_LSR_BI) && !skip_rx) {
struct irq_data *d;
d = irq_get_irq_data(port->irq);
if (d && irqd_is_wakeup_set(d))
pm_wakeup_event(tport->tty->dev, 0);
if (!up->dma || handle_rx_dma(up, iir))
status = serial8250_rx_chars(up, status);
}
serial8250_modem_status(up);
if ((status & UART_LSR_THRE) && (up->ier & UART_IER_THRI)) {
if (!up->dma || up->dma->tx_err)
serial8250_tx_chars(up);
else if (!up->dma->tx_running)
__stop_tx(up);
}
uart_unlock_and_check_sysrq_irqrestore(port, flags);
return 1;
}
EXPORT_SYMBOL_GPL(serial8250_handle_irq);
static int serial8250_default_handle_irq(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned int iir;
int ret;
serial8250_rpm_get(up);
iir = serial_port_in(port, UART_IIR);
ret = serial8250_handle_irq(port, iir);
serial8250_rpm_put(up);
return ret;
}
/*
* Newer 16550 compatible parts such as the SC16C650 & Altera 16550 Soft IP
* have a programmable TX threshold that triggers the THRE interrupt in
* the IIR register. In this case, the THRE interrupt indicates the FIFO
* has space available. Load it up with tx_loadsz bytes.
*/
static int serial8250_tx_threshold_handle_irq(struct uart_port *port)
{
unsigned long flags;
unsigned int iir = serial_port_in(port, UART_IIR);
/* TX Threshold IRQ triggered so load up FIFO */
if ((iir & UART_IIR_ID) == UART_IIR_THRI) {
struct uart_8250_port *up = up_to_u8250p(port);
uart_port_lock_irqsave(port, &flags);
serial8250_tx_chars(up);
uart_port_unlock_irqrestore(port, flags);
}
iir = serial_port_in(port, UART_IIR);
return serial8250_handle_irq(port, iir);
}
static unsigned int serial8250_tx_empty(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned int result = 0;
unsigned long flags;
serial8250_rpm_get(up);
uart_port_lock_irqsave(port, &flags);
if (!serial8250_tx_dma_running(up) && uart_lsr_tx_empty(serial_lsr_in(up)))
result = TIOCSER_TEMT;
uart_port_unlock_irqrestore(port, flags);
serial8250_rpm_put(up);
return result;
}
unsigned int serial8250_do_get_mctrl(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned int status;
unsigned int val;
serial8250_rpm_get(up);
status = serial8250_modem_status(up);
serial8250_rpm_put(up);
val = serial8250_MSR_to_TIOCM(status);
if (up->gpios)
return mctrl_gpio_get(up->gpios, &val);
return val;
}
EXPORT_SYMBOL_GPL(serial8250_do_get_mctrl);
static unsigned int serial8250_get_mctrl(struct uart_port *port)
{
if (port->get_mctrl)
return port->get_mctrl(port);
return serial8250_do_get_mctrl(port);
}
void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned char mcr;
mcr = serial8250_TIOCM_to_MCR(mctrl);
mcr |= up->mcr;
serial8250_out_MCR(up, mcr);
}
EXPORT_SYMBOL_GPL(serial8250_do_set_mctrl);
static void serial8250_set_mctrl(struct uart_port *port, unsigned int mctrl)
{
if (port->rs485.flags & SER_RS485_ENABLED)
return;
if (port->set_mctrl)
port->set_mctrl(port, mctrl);
else
serial8250_do_set_mctrl(port, mctrl);
}
static void serial8250_break_ctl(struct uart_port *port, int break_state)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned long flags;
serial8250_rpm_get(up);
uart_port_lock_irqsave(port, &flags);
if (break_state == -1)
up->lcr |= UART_LCR_SBC;
else
up->lcr &= ~UART_LCR_SBC;
serial_port_out(port, UART_LCR, up->lcr);
uart_port_unlock_irqrestore(port, flags);
serial8250_rpm_put(up);
}
static void wait_for_lsr(struct uart_8250_port *up, int bits)
{
unsigned int status, tmout = 10000;
/* Wait up to 10ms for the character(s) to be sent. */
for (;;) {
status = serial_lsr_in(up); if ((status & bits) == bits)
break;
if (--tmout == 0)
break;
udelay(1);
touch_nmi_watchdog();
}
}
/*
* Wait for transmitter & holding register to empty
*/
static void wait_for_xmitr(struct uart_8250_port *up, int bits)
{
unsigned int tmout;
wait_for_lsr(up, bits);
/* Wait up to 1s for flow control if necessary */
if (up->port.flags & UPF_CONS_FLOW) {
for (tmout = 1000000; tmout; tmout--) {
unsigned int msr = serial_in(up, UART_MSR);
up->msr_saved_flags |= msr & MSR_SAVE_FLAGS;
if (msr & UART_MSR_CTS)
break;
udelay(1);
touch_nmi_watchdog();
}
}
}
#ifdef CONFIG_CONSOLE_POLL
/*
* Console polling routines for writing and reading from the uart while
* in an interrupt or debug context.
*/
static int serial8250_get_poll_char(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
int status;
u16 lsr;
serial8250_rpm_get(up);
lsr = serial_port_in(port, UART_LSR);
if (!(lsr & UART_LSR_DR)) {
status = NO_POLL_CHAR;
goto out;
}
status = serial_port_in(port, UART_RX);
out:
serial8250_rpm_put(up);
return status;
}
static void serial8250_put_poll_char(struct uart_port *port,
unsigned char c)
{
unsigned int ier;
struct uart_8250_port *up = up_to_u8250p(port);
/*
* Normally the port is locked to synchronize UART_IER access
* against the console. However, this function is only used by
* KDB/KGDB, where it may not be possible to acquire the port
* lock because all other CPUs are quiesced. The quiescence
* should allow safe lockless usage here.
*/
serial8250_rpm_get(up);
/*
* First save the IER then disable the interrupts
*/
ier = serial_port_in(port, UART_IER);
serial8250_clear_IER(up);
wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
/*
* Send the character out.
*/
serial_port_out(port, UART_TX, c);
/*
* Finally, wait for transmitter to become empty
* and restore the IER
*/
wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
serial_port_out(port, UART_IER, ier);
serial8250_rpm_put(up);
}
#endif /* CONFIG_CONSOLE_POLL */
int serial8250_do_startup(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned long flags;
unsigned char iir;
int retval;
u16 lsr;
if (!port->fifosize)
port->fifosize = uart_config[port->type].fifo_size;
if (!up->tx_loadsz)
up->tx_loadsz = uart_config[port->type].tx_loadsz;
if (!up->capabilities)
up->capabilities = uart_config[port->type].flags;
up->mcr = 0;
if (port->iotype != up->cur_iotype)
set_io_from_upio(port);
serial8250_rpm_get(up);
if (port->type == PORT_16C950) {
/*
* Wake up and initialize UART
*
* Synchronize UART_IER access against the console.
*/
uart_port_lock_irqsave(port, &flags);
up->acr = 0;
serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B);
serial_port_out(port, UART_EFR, UART_EFR_ECB);
serial_port_out(port, UART_IER, 0);
serial_port_out(port, UART_LCR, 0);
serial_icr_write(up, UART_CSR, 0); /* Reset the UART */
serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B);
serial_port_out(port, UART_EFR, UART_EFR_ECB);
serial_port_out(port, UART_LCR, 0);
uart_port_unlock_irqrestore(port, flags);
}
if (port->type == PORT_DA830) {
/*
* Reset the port
*
* Synchronize UART_IER access against the console.
*/
uart_port_lock_irqsave(port, &flags);
serial_port_out(port, UART_IER, 0);
serial_port_out(port, UART_DA830_PWREMU_MGMT, 0);
uart_port_unlock_irqrestore(port, flags);
mdelay(10);
/* Enable Tx, Rx and free run mode */
serial_port_out(port, UART_DA830_PWREMU_MGMT,
UART_DA830_PWREMU_MGMT_UTRST |
UART_DA830_PWREMU_MGMT_URRST |
UART_DA830_PWREMU_MGMT_FREE);
}
#ifdef CONFIG_SERIAL_8250_RSA
/*
* If this is an RSA port, see if we can kick it up to the
* higher speed clock.
*/
enable_rsa(up);
#endif
/*
* Clear the FIFO buffers and disable them.
* (they will be reenabled in set_termios())
*/
serial8250_clear_fifos(up);
/*
* Clear the interrupt registers.
*/
serial_port_in(port, UART_LSR);
serial_port_in(port, UART_RX);
serial_port_in(port, UART_IIR);
serial_port_in(port, UART_MSR);
/*
* At this point, there's no way the LSR could still be 0xff;
* if it is, then bail out, because there's likely no UART
* here.
*/
if (!(port->flags & UPF_BUGGY_UART) &&
(serial_port_in(port, UART_LSR) == 0xff)) {
dev_info_ratelimited(port->dev, "LSR safety check engaged!\n");
retval = -ENODEV;
goto out;
}
/*
* For a XR16C850, we need to set the trigger levels
*/
if (port->type == PORT_16850) {
unsigned char fctr;
serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
fctr = serial_in(up, UART_FCTR) & ~(UART_FCTR_RX|UART_FCTR_TX);
serial_port_out(port, UART_FCTR,
fctr | UART_FCTR_TRGD | UART_FCTR_RX);
serial_port_out(port, UART_TRG, UART_TRG_96);
serial_port_out(port, UART_FCTR,
fctr | UART_FCTR_TRGD | UART_FCTR_TX);
serial_port_out(port, UART_TRG, UART_TRG_96);
serial_port_out(port, UART_LCR, 0);
}
/*
* For the Altera 16550 variants, set TX threshold trigger level.
*/
if (((port->type == PORT_ALTR_16550_F32) ||
(port->type == PORT_ALTR_16550_F64) ||
(port->type == PORT_ALTR_16550_F128)) && (port->fifosize > 1)) {
/* Bounds checking of TX threshold (valid 0 to fifosize-2) */
if ((up->tx_loadsz < 2) || (up->tx_loadsz > port->fifosize)) {
dev_err(port->dev, "TX FIFO Threshold errors, skipping\n");
} else {
serial_port_out(port, UART_ALTR_AFR,
UART_ALTR_EN_TXFIFO_LW);
serial_port_out(port, UART_ALTR_TX_LOW,
port->fifosize - up->tx_loadsz);
port->handle_irq = serial8250_tx_threshold_handle_irq;
}
}
/* Check if we need to have shared IRQs */
if (port->irq && (up->port.flags & UPF_SHARE_IRQ))
up->port.irqflags |= IRQF_SHARED;
retval = up->ops->setup_irq(up);
if (retval)
goto out;
if (port->irq && !(up->port.flags & UPF_NO_THRE_TEST)) {
unsigned char iir1;
if (port->irqflags & IRQF_SHARED)
disable_irq_nosync(port->irq);
/*
* Test for UARTs that do not reassert THRE when the
* transmitter is idle and the interrupt has already
* been cleared. Real 16550s should always reassert
* this interrupt whenever the transmitter is idle and
* the interrupt is enabled. Delays are necessary to
* allow register changes to become visible.
*
* Synchronize UART_IER access against the console.
*/
uart_port_lock_irqsave(port, &flags);
wait_for_xmitr(up, UART_LSR_THRE);
serial_port_out_sync(port, UART_IER, UART_IER_THRI);
udelay(1); /* allow THRE to set */
iir1 = serial_port_in(port, UART_IIR);
serial_port_out(port, UART_IER, 0);
serial_port_out_sync(port, UART_IER, UART_IER_THRI);
udelay(1); /* allow a working UART time to re-assert THRE */
iir = serial_port_in(port, UART_IIR);
serial_port_out(port, UART_IER, 0);
uart_port_unlock_irqrestore(port, flags);
if (port->irqflags & IRQF_SHARED)
enable_irq(port->irq);
/*
* If the interrupt is not reasserted, or we otherwise
* don't trust the iir, setup a timer to kick the UART
* on a regular basis.
*/
if ((!(iir1 & UART_IIR_NO_INT) && (iir & UART_IIR_NO_INT)) ||
up->port.flags & UPF_BUG_THRE) {
up->bugs |= UART_BUG_THRE;
}
}
up->ops->setup_timer(up);
/*
* Now, initialize the UART
*/
serial_port_out(port, UART_LCR, UART_LCR_WLEN8);
uart_port_lock_irqsave(port, &flags);
if (up->port.flags & UPF_FOURPORT) {
if (!up->port.irq)
up->port.mctrl |= TIOCM_OUT1;
} else
/*
* Most PC uarts need OUT2 raised to enable interrupts.
*/
if (port->irq)
up->port.mctrl |= TIOCM_OUT2;
serial8250_set_mctrl(port, port->mctrl);
/*
* Serial over Lan (SoL) hack:
* Intel 8257x Gigabit ethernet chips have a 16550 emulation, to be
* used for Serial Over Lan. Those chips take a longer time than a
* normal serial device to signalize that a transmission data was
* queued. Due to that, the above test generally fails. One solution
* would be to delay the reading of iir. However, this is not
* reliable, since the timeout is variable. So, let's just don't
* test if we receive TX irq. This way, we'll never enable
* UART_BUG_TXEN.
*/
if (up->port.quirks & UPQ_NO_TXEN_TEST)
goto dont_test_tx_en;
/*
* Do a quick test to see if we receive an interrupt when we enable
* the TX irq.
*/
serial_port_out(port, UART_IER, UART_IER_THRI);
lsr = serial_port_in(port, UART_LSR);
iir = serial_port_in(port, UART_IIR);
serial_port_out(port, UART_IER, 0);
if (lsr & UART_LSR_TEMT && iir & UART_IIR_NO_INT) {
if (!(up->bugs & UART_BUG_TXEN)) {
up->bugs |= UART_BUG_TXEN;
dev_dbg(port->dev, "enabling bad tx status workarounds\n");
}
} else {
up->bugs &= ~UART_BUG_TXEN;
}
dont_test_tx_en:
uart_port_unlock_irqrestore(port, flags);
/*
* Clear the interrupt registers again for luck, and clear the
* saved flags to avoid getting false values from polling
* routines or the previous session.
*/
serial_port_in(port, UART_LSR);
serial_port_in(port, UART_RX);
serial_port_in(port, UART_IIR);
serial_port_in(port, UART_MSR);
up->lsr_saved_flags = 0;
up->msr_saved_flags = 0;
/*
* Request DMA channels for both RX and TX.
*/
if (up->dma) {
const char *msg = NULL;
if (uart_console(port))
msg = "forbid DMA for kernel console";
else if (serial8250_request_dma(up))
msg = "failed to request DMA";
if (msg) {
dev_warn_ratelimited(port->dev, "%s\n", msg);
up->dma = NULL;
}
}
/*
* Set the IER shadow for rx interrupts but defer actual interrupt
* enable until after the FIFOs are enabled; otherwise, an already-
* active sender can swamp the interrupt handler with "too much work".
*/
up->ier = UART_IER_RLSI | UART_IER_RDI;
if (port->flags & UPF_FOURPORT) {
unsigned int icp;
/*
* Enable interrupts on the AST Fourport board
*/
icp = (port->iobase & 0xfe0) | 0x01f;
outb_p(0x80, icp);
inb_p(icp);
}
retval = 0;
out:
serial8250_rpm_put(up);
return retval;
}
EXPORT_SYMBOL_GPL(serial8250_do_startup);
static int serial8250_startup(struct uart_port *port)
{
if (port->startup)
return port->startup(port);
return serial8250_do_startup(port);
}
void serial8250_do_shutdown(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned long flags;
serial8250_rpm_get(up);
/*
* Disable interrupts from this port
*
* Synchronize UART_IER access against the console.
*/
uart_port_lock_irqsave(port, &flags);
up->ier = 0;
serial_port_out(port, UART_IER, 0);
uart_port_unlock_irqrestore(port, flags);
synchronize_irq(port->irq);
if (up->dma)
serial8250_release_dma(up);
uart_port_lock_irqsave(port, &flags);
if (port->flags & UPF_FOURPORT) {
/* reset interrupts on the AST Fourport board */
inb((port->iobase & 0xfe0) | 0x1f);
port->mctrl |= TIOCM_OUT1;
} else
port->mctrl &= ~TIOCM_OUT2;
serial8250_set_mctrl(port, port->mctrl);
uart_port_unlock_irqrestore(port, flags);
/*
* Disable break condition and FIFOs
*/
serial_port_out(port, UART_LCR,
serial_port_in(port, UART_LCR) & ~UART_LCR_SBC);
serial8250_clear_fifos(up);
#ifdef CONFIG_SERIAL_8250_RSA
/*
* Reset the RSA board back to 115kbps compat mode.
*/
disable_rsa(up);
#endif
/*
* Read data port to reset things, and then unlink from
* the IRQ chain.
*/
serial_port_in(port, UART_RX);
serial8250_rpm_put(up);
up->ops->release_irq(up);
}
EXPORT_SYMBOL_GPL(serial8250_do_shutdown);
static void serial8250_shutdown(struct uart_port *port)
{
if (port->shutdown)
port->shutdown(port);
else
serial8250_do_shutdown(port);
}
static unsigned int serial8250_do_get_divisor(struct uart_port *port,
unsigned int baud,
unsigned int *frac)
{
upf_t magic_multiplier = port->flags & UPF_MAGIC_MULTIPLIER;
struct uart_8250_port *up = up_to_u8250p(port);
unsigned int quot;
/*
* Handle magic divisors for baud rates above baud_base on SMSC
* Super I/O chips. We clamp custom rates from clk/6 and clk/12
* up to clk/4 (0x8001) and clk/8 (0x8002) respectively. These
* magic divisors actually reprogram the baud rate generator's
* reference clock derived from chips's 14.318MHz clock input.
*
* Documentation claims that with these magic divisors the base
* frequencies of 7.3728MHz and 3.6864MHz are used respectively
* for the extra baud rates of 460800bps and 230400bps rather
* than the usual base frequency of 1.8462MHz. However empirical
* evidence contradicts that.
*
* Instead bit 7 of the DLM register (bit 15 of the divisor) is
* effectively used as a clock prescaler selection bit for the
* base frequency of 7.3728MHz, always used. If set to 0, then
* the base frequency is divided by 4 for use by the Baud Rate
* Generator, for the usual arrangement where the value of 1 of
* the divisor produces the baud rate of 115200bps. Conversely,
* if set to 1 and high-speed operation has been enabled with the
* Serial Port Mode Register in the Device Configuration Space,
* then the base frequency is supplied directly to the Baud Rate
* Generator, so for the divisor values of 0x8001, 0x8002, 0x8003,
* 0x8004, etc. the respective baud rates produced are 460800bps,
* 230400bps, 153600bps, 115200bps, etc.
*
* In all cases only low 15 bits of the divisor are used to divide
* the baud base and therefore 32767 is the maximum divisor value
* possible, even though documentation says that the programmable
* Baud Rate Generator is capable of dividing the internal PLL
* clock by any divisor from 1 to 65535.
*/
if (magic_multiplier && baud >= port->uartclk / 6)
quot = 0x8001;
else if (magic_multiplier && baud >= port->uartclk / 12)
quot = 0x8002;
else
quot = uart_get_divisor(port, baud);
/*
* Oxford Semi 952 rev B workaround
*/
if (up->bugs & UART_BUG_QUOT && (quot & 0xff) == 0)
quot++;
return quot;
}
static unsigned int serial8250_get_divisor(struct uart_port *port,
unsigned int baud,
unsigned int *frac)
{
if (port->get_divisor)
return port->get_divisor(port, baud, frac);
return serial8250_do_get_divisor(port, baud, frac);
}
static unsigned char serial8250_compute_lcr(struct uart_8250_port *up,
tcflag_t c_cflag)
{
unsigned char cval;
cval = UART_LCR_WLEN(tty_get_char_size(c_cflag));
if (c_cflag & CSTOPB)
cval |= UART_LCR_STOP;
if (c_cflag & PARENB)
cval |= UART_LCR_PARITY;
if (!(c_cflag & PARODD))
cval |= UART_LCR_EPAR;
if (c_cflag & CMSPAR)
cval |= UART_LCR_SPAR;
return cval;
}
void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud,
unsigned int quot, unsigned int quot_frac)
{
struct uart_8250_port *up = up_to_u8250p(port);
/* Workaround to enable 115200 baud on OMAP1510 internal ports */
if (is_omap1510_8250(up)) {
if (baud == 115200) {
quot = 1;
serial_port_out(port, UART_OMAP_OSC_12M_SEL, 1);
} else
serial_port_out(port, UART_OMAP_OSC_12M_SEL, 0);
}
/*
* For NatSemi, switch to bank 2 not bank 1, to avoid resetting EXCR2,
* otherwise just set DLAB
*/
if (up->capabilities & UART_NATSEMI)
serial_port_out(port, UART_LCR, 0xe0);
else
serial_port_out(port, UART_LCR, up->lcr | UART_LCR_DLAB);
serial_dl_write(up, quot);
}
EXPORT_SYMBOL_GPL(serial8250_do_set_divisor);
static void serial8250_set_divisor(struct uart_port *port, unsigned int baud,
unsigned int quot, unsigned int quot_frac)
{
if (port->set_divisor)
port->set_divisor(port, baud, quot, quot_frac);
else
serial8250_do_set_divisor(port, baud, quot, quot_frac);
}
static unsigned int serial8250_get_baud_rate(struct uart_port *port,
struct ktermios *termios,
const struct ktermios *old)
{
unsigned int tolerance = port->uartclk / 100;
unsigned int min;
unsigned int max;
/*
* Handle magic divisors for baud rates above baud_base on SMSC
* Super I/O chips. Enable custom rates of clk/4 and clk/8, but
* disable divisor values beyond 32767, which are unavailable.
*/
if (port->flags & UPF_MAGIC_MULTIPLIER) {
min = port->uartclk / 16 / UART_DIV_MAX >> 1;
max = (port->uartclk + tolerance) / 4;
} else {
min = port->uartclk / 16 / UART_DIV_MAX;
max = (port->uartclk + tolerance) / 16;
}
/*
* Ask the core to calculate the divisor for us.
* Allow 1% tolerance at the upper limit so uart clks marginally
* slower than nominal still match standard baud rates without
* causing transmission errors.
*/
return uart_get_baud_rate(port, termios, old, min, max);
}
/*
* Note in order to avoid the tty port mutex deadlock don't use the next method
* within the uart port callbacks. Primarily it's supposed to be utilized to
* handle a sudden reference clock rate change.
*/
void serial8250_update_uartclk(struct uart_port *port, unsigned int uartclk)
{
struct tty_port *tport = &port->state->port;
struct tty_struct *tty;
tty = tty_port_tty_get(tport);
if (!tty) {
mutex_lock(&tport->mutex);
port->uartclk = uartclk;
mutex_unlock(&tport->mutex);
return;
}
down_write(&tty->termios_rwsem);
mutex_lock(&tport->mutex);
if (port->uartclk == uartclk)
goto out_unlock;
port->uartclk = uartclk;
if (!tty_port_initialized(tport))
goto out_unlock;
serial8250_do_set_termios(port, &tty->termios, NULL);
out_unlock:
mutex_unlock(&tport->mutex);
up_write(&tty->termios_rwsem);
tty_kref_put(tty);
}
EXPORT_SYMBOL_GPL(serial8250_update_uartclk);
void
serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios,
const struct ktermios *old)
{
struct uart_8250_port *up = up_to_u8250p(port);
unsigned char cval;
unsigned long flags;
unsigned int baud, quot, frac = 0;
if (up->capabilities & UART_CAP_MINI) {
termios->c_cflag &= ~(CSTOPB | PARENB | PARODD | CMSPAR);
if ((termios->c_cflag & CSIZE) == CS5 ||
(termios->c_cflag & CSIZE) == CS6)
termios->c_cflag = (termios->c_cflag & ~CSIZE) | CS7;
}
cval = serial8250_compute_lcr(up, termios->c_cflag);
baud = serial8250_get_baud_rate(port, termios, old);
quot = serial8250_get_divisor(port, baud, &frac);
/*
* Ok, we're now changing the port state. Do it with
* interrupts disabled.
*
* Synchronize UART_IER access against the console.
*/
serial8250_rpm_get(up);
uart_port_lock_irqsave(port, &flags);
up->lcr = cval; /* Save computed LCR */
if (up->capabilities & UART_CAP_FIFO && port->fifosize > 1) {
if (baud < 2400 && !up->dma) {
up->fcr &= ~UART_FCR_TRIGGER_MASK;
up->fcr |= UART_FCR_TRIGGER_1;
}
}
/*
* MCR-based auto flow control. When AFE is enabled, RTS will be
* deasserted when the receive FIFO contains more characters than
* the trigger, or the MCR RTS bit is cleared.
*/
if (up->capabilities & UART_CAP_AFE) {
up->mcr &= ~UART_MCR_AFE;
if (termios->c_cflag & CRTSCTS)
up->mcr |= UART_MCR_AFE;
}
/*
* Update the per-port timeout.
*/
uart_update_timeout(port, termios->c_cflag, baud);
port->read_status_mask = UART_LSR_OE | UART_LSR_THRE | UART_LSR_DR;
if (termios->c_iflag & INPCK)
port->read_status_mask |= UART_LSR_FE | UART_LSR_PE;
if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
port->read_status_mask |= UART_LSR_BI;
/*
* Characters to ignore
*/
port->ignore_status_mask = 0;
if (termios->c_iflag & IGNPAR)
port->ignore_status_mask |= UART_LSR_PE | UART_LSR_FE;
if (termios->c_iflag & IGNBRK) {
port->ignore_status_mask |= UART_LSR_BI;
/*
* If we're ignoring parity and break indicators,
* ignore overruns too (for real raw support).
*/
if (termios->c_iflag & IGNPAR)
port->ignore_status_mask |= UART_LSR_OE;
}
/*
* ignore all characters if CREAD is not set
*/
if ((termios->c_cflag & CREAD) == 0)
port->ignore_status_mask |= UART_LSR_DR;
/*
* CTS flow control flag and modem status interrupts
*/
up->ier &= ~UART_IER_MSI;
if (!(up->bugs & UART_BUG_NOMSR) &&
UART_ENABLE_MS(&up->port, termios->c_cflag))
up->ier |= UART_IER_MSI;
if (up->capabilities & UART_CAP_UUE)
up->ier |= UART_IER_UUE;
if (up->capabilities & UART_CAP_RTOIE)
up->ier |= UART_IER_RTOIE;
serial_port_out(port, UART_IER, up->ier);
if (up->capabilities & UART_CAP_EFR) {
unsigned char efr = 0;
/*
* TI16C752/Startech hardware flow control. FIXME:
* - TI16C752 requires control thresholds to be set.
* - UART_MCR_RTS is ineffective if auto-RTS mode is enabled.
*/
if (termios->c_cflag & CRTSCTS)
efr |= UART_EFR_CTS;
serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B);
if (port->flags & UPF_EXAR_EFR)
serial_port_out(port, UART_XR_EFR, efr);
else
serial_port_out(port, UART_EFR, efr);
}
serial8250_set_divisor(port, baud, quot, frac);
/*
* LCR DLAB must be set to enable 64-byte FIFO mode. If the FCR
* is written without DLAB set, this mode will be disabled.
*/
if (port->type == PORT_16750)
serial_port_out(port, UART_FCR, up->fcr);
serial_port_out(port, UART_LCR, up->lcr); /* reset DLAB */
if (port->type != PORT_16750) {
/* emulated UARTs (Lucent Venus 167x) need two steps */
if (up->fcr & UART_FCR_ENABLE_FIFO)
serial_port_out(port, UART_FCR, UART_FCR_ENABLE_FIFO);
serial_port_out(port, UART_FCR, up->fcr); /* set fcr */
}
serial8250_set_mctrl(port, port->mctrl);
uart_port_unlock_irqrestore(port, flags);
serial8250_rpm_put(up);
/* Don't rewrite B0 */
if (tty_termios_baud_rate(termios))
tty_termios_encode_baud_rate(termios, baud, baud);
}
EXPORT_SYMBOL(serial8250_do_set_termios);
static void
serial8250_set_termios(struct uart_port *port, struct ktermios *termios,
const struct ktermios *old)
{
if (port->set_termios)
port->set_termios(port, termios, old);
else
serial8250_do_set_termios(port, termios, old);
}
void serial8250_do_set_ldisc(struct uart_port *port, struct ktermios *termios)
{
if (termios->c_line == N_PPS) {
port->flags |= UPF_HARDPPS_CD;
uart_port_lock_irq(port);
serial8250_enable_ms(port);
uart_port_unlock_irq(port);
} else {
port->flags &= ~UPF_HARDPPS_CD;
if (!UART_ENABLE_MS(port, termios->c_cflag)) {
uart_port_lock_irq(port);
serial8250_disable_ms(port);
uart_port_unlock_irq(port);
}
}
}
EXPORT_SYMBOL_GPL(serial8250_do_set_ldisc);
static void
serial8250_set_ldisc(struct uart_port *port, struct ktermios *termios)
{
if (port->set_ldisc)
port->set_ldisc(port, termios);
else
serial8250_do_set_ldisc(port, termios);
}
void serial8250_do_pm(struct uart_port *port, unsigned int state,
unsigned int oldstate)
{
struct uart_8250_port *p = up_to_u8250p(port);
serial8250_set_sleep(p, state != 0);
}
EXPORT_SYMBOL(serial8250_do_pm);
static void
serial8250_pm(struct uart_port *port, unsigned int state,
unsigned int oldstate)
{
if (port->pm)
port->pm(port, state, oldstate);
else
serial8250_do_pm(port, state, oldstate);
}
static unsigned int serial8250_port_size(struct uart_8250_port *pt)
{
if (pt->port.mapsize)
return pt->port.mapsize;
if (is_omap1_8250(pt))
return 0x16 << pt->port.regshift;
return 8 << pt->port.regshift;
}
/*
* Resource handling.
*/
static int serial8250_request_std_resource(struct uart_8250_port *up)
{
unsigned int size = serial8250_port_size(up);
struct uart_port *port = &up->port;
int ret = 0;
switch (port->iotype) {
case UPIO_AU:
case UPIO_TSI:
case UPIO_MEM32:
case UPIO_MEM32BE:
case UPIO_MEM16:
case UPIO_MEM:
if (!port->mapbase) {
ret = -EINVAL;
break;
}
if (!request_mem_region(port->mapbase, size, "serial")) {
ret = -EBUSY;
break;
}
if (port->flags & UPF_IOREMAP) {
port->membase = ioremap(port->mapbase, size);
if (!port->membase) {
release_mem_region(port->mapbase, size);
ret = -ENOMEM;
}
}
break;
case UPIO_HUB6:
case UPIO_PORT:
if (!request_region(port->iobase, size, "serial"))
ret = -EBUSY;
break;
}
return ret;
}
static void serial8250_release_std_resource(struct uart_8250_port *up)
{
unsigned int size = serial8250_port_size(up);
struct uart_port *port = &up->port;
switch (port->iotype) {
case UPIO_AU:
case UPIO_TSI:
case UPIO_MEM32:
case UPIO_MEM32BE:
case UPIO_MEM16:
case UPIO_MEM:
if (!port->mapbase)
break;
if (port->flags & UPF_IOREMAP) {
iounmap(port->membase);
port->membase = NULL;
}
release_mem_region(port->mapbase, size);
break;
case UPIO_HUB6:
case UPIO_PORT:
release_region(port->iobase, size);
break;
}
}
static void serial8250_release_port(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
serial8250_release_std_resource(up);
}
static int serial8250_request_port(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
return serial8250_request_std_resource(up);
}
static int fcr_get_rxtrig_bytes(struct uart_8250_port *up)
{
const struct serial8250_config *conf_type = &uart_config[up->port.type];
unsigned char bytes;
bytes = conf_type->rxtrig_bytes[UART_FCR_R_TRIG_BITS(up->fcr)];
return bytes ? bytes : -EOPNOTSUPP;
}
static int bytes_to_fcr_rxtrig(struct uart_8250_port *up, unsigned char bytes)
{
const struct serial8250_config *conf_type = &uart_config[up->port.type];
int i;
if (!conf_type->rxtrig_bytes[UART_FCR_R_TRIG_BITS(UART_FCR_R_TRIG_00)])
return -EOPNOTSUPP;
for (i = 1; i < UART_FCR_R_TRIG_MAX_STATE; i++) {
if (bytes < conf_type->rxtrig_bytes[i])
/* Use the nearest lower value */
return (--i) << UART_FCR_R_TRIG_SHIFT;
}
return UART_FCR_R_TRIG_11;
}
static int do_get_rxtrig(struct tty_port *port)
{
struct uart_state *state = container_of(port, struct uart_state, port);
struct uart_port *uport = state->uart_port;
struct uart_8250_port *up = up_to_u8250p(uport);
if (!(up->capabilities & UART_CAP_FIFO) || uport->fifosize <= 1)
return -EINVAL;
return fcr_get_rxtrig_bytes(up);
}
static int do_serial8250_get_rxtrig(struct tty_port *port)
{
int rxtrig_bytes;
mutex_lock(&port->mutex);
rxtrig_bytes = do_get_rxtrig(port);
mutex_unlock(&port->mutex);
return rxtrig_bytes;
}
static ssize_t rx_trig_bytes_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct tty_port *port = dev_get_drvdata(dev);
int rxtrig_bytes;
rxtrig_bytes = do_serial8250_get_rxtrig(port);
if (rxtrig_bytes < 0)
return rxtrig_bytes;
return sysfs_emit(buf, "%d\n", rxtrig_bytes);
}
static int do_set_rxtrig(struct tty_port *port, unsigned char bytes)
{
struct uart_state *state = container_of(port, struct uart_state, port);
struct uart_port *uport = state->uart_port;
struct uart_8250_port *up = up_to_u8250p(uport);
int rxtrig;
if (!(up->capabilities & UART_CAP_FIFO) || uport->fifosize <= 1)
return -EINVAL;
rxtrig = bytes_to_fcr_rxtrig(up, bytes);
if (rxtrig < 0)
return rxtrig;
serial8250_clear_fifos(up);
up->fcr &= ~UART_FCR_TRIGGER_MASK;
up->fcr |= (unsigned char)rxtrig;
serial_out(up, UART_FCR, up->fcr);
return 0;
}
static int do_serial8250_set_rxtrig(struct tty_port *port, unsigned char bytes)
{
int ret;
mutex_lock(&port->mutex);
ret = do_set_rxtrig(port, bytes);
mutex_unlock(&port->mutex);
return ret;
}
static ssize_t rx_trig_bytes_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
struct tty_port *port = dev_get_drvdata(dev);
unsigned char bytes;
int ret;
if (!count)
return -EINVAL;
ret = kstrtou8(buf, 10, &bytes);
if (ret < 0)
return ret;
ret = do_serial8250_set_rxtrig(port, bytes);
if (ret < 0)
return ret;
return count;
}
static DEVICE_ATTR_RW(rx_trig_bytes);
static struct attribute *serial8250_dev_attrs[] = {
&dev_attr_rx_trig_bytes.attr,
NULL
};
static struct attribute_group serial8250_dev_attr_group = {
.attrs = serial8250_dev_attrs,
};
static void register_dev_spec_attr_grp(struct uart_8250_port *up)
{
const struct serial8250_config *conf_type = &uart_config[up->port.type];
if (conf_type->rxtrig_bytes[0])
up->port.attr_group = &serial8250_dev_attr_group;
}
static void serial8250_config_port(struct uart_port *port, int flags)
{
struct uart_8250_port *up = up_to_u8250p(port);
int ret;
/*
* Find the region that we can probe for. This in turn
* tells us whether we can probe for the type of port.
*/
ret = serial8250_request_std_resource(up);
if (ret < 0)
return;
if (port->iotype != up->cur_iotype)
set_io_from_upio(port);
if (flags & UART_CONFIG_TYPE)
autoconfig(up);
/* HW bugs may trigger IRQ while IIR == NO_INT */
if (port->type == PORT_TEGRA)
up->bugs |= UART_BUG_NOMSR;
if (port->type != PORT_UNKNOWN && flags & UART_CONFIG_IRQ)
autoconfig_irq(up);
if (port->type == PORT_UNKNOWN)
serial8250_release_std_resource(up);
register_dev_spec_attr_grp(up);
up->fcr = uart_config[up->port.type].fcr;
}
static int
serial8250_verify_port(struct uart_port *port, struct serial_struct *ser)
{
if (ser->irq >= nr_irqs || ser->irq < 0 ||
ser->baud_base < 9600 || ser->type < PORT_UNKNOWN ||
ser->type >= ARRAY_SIZE(uart_config) || ser->type == PORT_CIRRUS ||
ser->type == PORT_STARTECH)
return -EINVAL;
return 0;
}
static const char *serial8250_type(struct uart_port *port)
{
int type = port->type;
if (type >= ARRAY_SIZE(uart_config))
type = 0;
return uart_config[type].name;
}
static const struct uart_ops serial8250_pops = {
.tx_empty = serial8250_tx_empty,
.set_mctrl = serial8250_set_mctrl,
.get_mctrl = serial8250_get_mctrl,
.stop_tx = serial8250_stop_tx,
.start_tx = serial8250_start_tx,
.throttle = serial8250_throttle,
.unthrottle = serial8250_unthrottle,
.stop_rx = serial8250_stop_rx,
.enable_ms = serial8250_enable_ms,
.break_ctl = serial8250_break_ctl,
.startup = serial8250_startup,
.shutdown = serial8250_shutdown,
.set_termios = serial8250_set_termios,
.set_ldisc = serial8250_set_ldisc,
.pm = serial8250_pm,
.type = serial8250_type,
.release_port = serial8250_release_port,
.request_port = serial8250_request_port,
.config_port = serial8250_config_port,
.verify_port = serial8250_verify_port,
#ifdef CONFIG_CONSOLE_POLL
.poll_get_char = serial8250_get_poll_char,
.poll_put_char = serial8250_put_poll_char,
#endif
};
void serial8250_init_port(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
spin_lock_init(&port->lock);
port->ctrl_id = 0;
port->pm = NULL;
port->ops = &serial8250_pops;
port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE);
up->cur_iotype = 0xFF;
}
EXPORT_SYMBOL_GPL(serial8250_init_port);
void serial8250_set_defaults(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
if (up->port.flags & UPF_FIXED_TYPE) {
unsigned int type = up->port.type;
if (!up->port.fifosize)
up->port.fifosize = uart_config[type].fifo_size;
if (!up->tx_loadsz)
up->tx_loadsz = uart_config[type].tx_loadsz;
if (!up->capabilities)
up->capabilities = uart_config[type].flags;
}
set_io_from_upio(port);
/* default dma handlers */
if (up->dma) {
if (!up->dma->tx_dma)
up->dma->tx_dma = serial8250_tx_dma;
if (!up->dma->rx_dma)
up->dma->rx_dma = serial8250_rx_dma;
}
}
EXPORT_SYMBOL_GPL(serial8250_set_defaults);
#ifdef CONFIG_SERIAL_8250_CONSOLE
static void serial8250_console_putchar(struct uart_port *port, unsigned char ch)
{
struct uart_8250_port *up = up_to_u8250p(port);
wait_for_xmitr(up, UART_LSR_THRE);
serial_port_out(port, UART_TX, ch);
}
/*
* Restore serial console when h/w power-off detected
*/
static void serial8250_console_restore(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
struct ktermios termios;
unsigned int baud, quot, frac = 0;
termios.c_cflag = port->cons->cflag;
termios.c_ispeed = port->cons->ispeed;
termios.c_ospeed = port->cons->ospeed;
if (port->state->port.tty && termios.c_cflag == 0) { termios.c_cflag = port->state->port.tty->termios.c_cflag;
termios.c_ispeed = port->state->port.tty->termios.c_ispeed;
termios.c_ospeed = port->state->port.tty->termios.c_ospeed;
}
baud = serial8250_get_baud_rate(port, &termios, NULL);
quot = serial8250_get_divisor(port, baud, &frac);
serial8250_set_divisor(port, baud, quot, frac); serial_port_out(port, UART_LCR, up->lcr); serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS);
}
/*
* Print a string to the serial port using the device FIFO
*
* It sends fifosize bytes and then waits for the fifo
* to get empty.
*/
static void serial8250_console_fifo_write(struct uart_8250_port *up,
const char *s, unsigned int count)
{
int i;
const char *end = s + count;
unsigned int fifosize = up->tx_loadsz;
bool cr_sent = false;
while (s != end) { wait_for_lsr(up, UART_LSR_THRE); for (i = 0; i < fifosize && s != end; ++i) { if (*s == '\n' && !cr_sent) { serial_out(up, UART_TX, '\r');
cr_sent = true;
} else {
serial_out(up, UART_TX, *s++);
cr_sent = false;
}
}
}
}
/*
* Print a string to the serial port trying not to disturb
* any possible real use of the port...
*
* The console_lock must be held when we get here.
*
* Doing runtime PM is really a bad idea for the kernel console.
* Thus, we assume the function is called when device is powered up.
*/
void serial8250_console_write(struct uart_8250_port *up, const char *s,
unsigned int count)
{
struct uart_8250_em485 *em485 = up->em485;
struct uart_port *port = &up->port;
unsigned long flags;
unsigned int ier, use_fifo;
int locked = 1;
touch_nmi_watchdog();
if (oops_in_progress)
locked = uart_port_trylock_irqsave(port, &flags);
else
uart_port_lock_irqsave(port, &flags);
/*
* First save the IER then disable the interrupts
*/
ier = serial_port_in(port, UART_IER);
serial8250_clear_IER(up);
/* check scratch reg to see if port powered off during system sleep */
if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) { serial8250_console_restore(up);
up->canary = 0;
}
if (em485) { if (em485->tx_stopped) up->rs485_start_tx(up); mdelay(port->rs485.delay_rts_before_send);
}
use_fifo = (up->capabilities & UART_CAP_FIFO) &&
/*
* BCM283x requires to check the fifo
* after each byte.
*/
!(up->capabilities & UART_CAP_MINI) &&
/*
* tx_loadsz contains the transmit fifo size
*/
up->tx_loadsz > 1 && (up->fcr & UART_FCR_ENABLE_FIFO) && port->state && test_bit(TTY_PORT_INITIALIZED, &port->state->port.iflags) &&
/*
* After we put a data in the fifo, the controller will send
* it regardless of the CTS state. Therefore, only use fifo
* if we don't use control flow.
*/
!(up->port.flags & UPF_CONS_FLOW);
if (likely(use_fifo))
serial8250_console_fifo_write(up, s, count);
else
uart_console_write(port, s, count, serial8250_console_putchar);
/*
* Finally, wait for transmitter to become empty
* and restore the IER
*/
wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
if (em485) {
mdelay(port->rs485.delay_rts_after_send); if (em485->tx_stopped) up->rs485_stop_tx(up);
}
serial_port_out(port, UART_IER, ier);
/*
* The receive handling will happen properly because the
* receive ready bit will still be set; it is not cleared
* on read. However, modem control will not, we must
* call it if we have saved something in the saved flags
* while processing with interrupts off.
*/
if (up->msr_saved_flags)
serial8250_modem_status(up); if (locked) uart_port_unlock_irqrestore(port, flags);}
static unsigned int probe_baud(struct uart_port *port)
{
unsigned char lcr, dll, dlm;
unsigned int quot;
lcr = serial_port_in(port, UART_LCR);
serial_port_out(port, UART_LCR, lcr | UART_LCR_DLAB);
dll = serial_port_in(port, UART_DLL);
dlm = serial_port_in(port, UART_DLM);
serial_port_out(port, UART_LCR, lcr);
quot = (dlm << 8) | dll;
return (port->uartclk / 16) / quot;
}
int serial8250_console_setup(struct uart_port *port, char *options, bool probe)
{
int baud = 9600;
int bits = 8;
int parity = 'n';
int flow = 'n';
int ret;
if (!port->iobase && !port->membase)
return -ENODEV;
if (options)
uart_parse_options(options, &baud, &parity, &bits, &flow);
else if (probe)
baud = probe_baud(port);
ret = uart_set_options(port, port->cons, baud, parity, bits, flow);
if (ret)
return ret;
if (port->dev)
pm_runtime_get_sync(port->dev);
return 0;
}
int serial8250_console_exit(struct uart_port *port)
{
if (port->dev)
pm_runtime_put_sync(port->dev);
return 0;
}
#endif /* CONFIG_SERIAL_8250_CONSOLE */
MODULE_LICENSE("GPL");
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SWAP_H
#define _LINUX_SWAP_H
#include <linux/spinlock.h>
#include <linux/linkage.h>
#include <linux/mmzone.h>
#include <linux/list.h>
#include <linux/memcontrol.h>
#include <linux/sched.h>
#include <linux/node.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/atomic.h>
#include <linux/page-flags.h>
#include <uapi/linux/mempolicy.h>
#include <asm/page.h>
struct notifier_block;
struct bio;
struct pagevec;
#define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */
#define SWAP_FLAG_PRIO_MASK 0x7fff
#define SWAP_FLAG_PRIO_SHIFT 0
#define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */
#define SWAP_FLAG_DISCARD_ONCE 0x20000 /* discard swap area at swapon-time */
#define SWAP_FLAG_DISCARD_PAGES 0x40000 /* discard page-clusters after use */
#define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \
SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \
SWAP_FLAG_DISCARD_PAGES)
#define SWAP_BATCH 64
static inline int current_is_kswapd(void)
{
return current->flags & PF_KSWAPD;
}
/*
* MAX_SWAPFILES defines the maximum number of swaptypes: things which can
* be swapped to. The swap type and the offset into that swap type are
* encoded into pte's and into pgoff_t's in the swapcache. Using five bits
* for the type means that the maximum number of swapcache pages is 27 bits
* on 32-bit-pgoff_t architectures. And that assumes that the architecture packs
* the type/offset into the pte as 5/27 as well.
*/
#define MAX_SWAPFILES_SHIFT 5
/*
* Use some of the swap files numbers for other purposes. This
* is a convenient way to hook into the VM to trigger special
* actions on faults.
*/
/*
* PTE markers are used to persist information onto PTEs that otherwise
* should be a none pte. As its name "PTE" hints, it should only be
* applied to the leaves of pgtables.
*/
#define SWP_PTE_MARKER_NUM 1
#define SWP_PTE_MARKER (MAX_SWAPFILES + SWP_HWPOISON_NUM + \
SWP_MIGRATION_NUM + SWP_DEVICE_NUM)
/*
* Unaddressable device memory support. See include/linux/hmm.h and
* Documentation/mm/hmm.rst. Short description is we need struct pages for
* device memory that is unaddressable (inaccessible) by CPU, so that we can
* migrate part of a process memory to device memory.
*
* When a page is migrated from CPU to device, we set the CPU page table entry
* to a special SWP_DEVICE_{READ|WRITE} entry.
*
* When a page is mapped by the device for exclusive access we set the CPU page
* table entries to special SWP_DEVICE_EXCLUSIVE_* entries.
*/
#ifdef CONFIG_DEVICE_PRIVATE
#define SWP_DEVICE_NUM 4
#define SWP_DEVICE_WRITE (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM)
#define SWP_DEVICE_READ (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+1)
#define SWP_DEVICE_EXCLUSIVE_WRITE (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+2)
#define SWP_DEVICE_EXCLUSIVE_READ (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+3)
#else
#define SWP_DEVICE_NUM 0
#endif
/*
* Page migration support.
*
* SWP_MIGRATION_READ_EXCLUSIVE is only applicable to anonymous pages and
* indicates that the referenced (part of) an anonymous page is exclusive to
* a single process. For SWP_MIGRATION_WRITE, that information is implicit:
* (part of) an anonymous page that are mapped writable are exclusive to a
* single process.
*/
#ifdef CONFIG_MIGRATION
#define SWP_MIGRATION_NUM 3
#define SWP_MIGRATION_READ (MAX_SWAPFILES + SWP_HWPOISON_NUM)
#define SWP_MIGRATION_READ_EXCLUSIVE (MAX_SWAPFILES + SWP_HWPOISON_NUM + 1)
#define SWP_MIGRATION_WRITE (MAX_SWAPFILES + SWP_HWPOISON_NUM + 2)
#else
#define SWP_MIGRATION_NUM 0
#endif
/*
* Handling of hardware poisoned pages with memory corruption.
*/
#ifdef CONFIG_MEMORY_FAILURE
#define SWP_HWPOISON_NUM 1
#define SWP_HWPOISON MAX_SWAPFILES
#else
#define SWP_HWPOISON_NUM 0
#endif
#define MAX_SWAPFILES \
((1 << MAX_SWAPFILES_SHIFT) - SWP_DEVICE_NUM - \
SWP_MIGRATION_NUM - SWP_HWPOISON_NUM - \
SWP_PTE_MARKER_NUM)
/*
* Magic header for a swap area. The first part of the union is
* what the swap magic looks like for the old (limited to 128MB)
* swap area format, the second part of the union adds - in the
* old reserved area - some extra information. Note that the first
* kilobyte is reserved for boot loader or disk label stuff...
*
* Having the magic at the end of the PAGE_SIZE makes detecting swap
* areas somewhat tricky on machines that support multiple page sizes.
* For 2.5 we'll probably want to move the magic to just beyond the
* bootbits...
*/
union swap_header {
struct {
char reserved[PAGE_SIZE - 10];
char magic[10]; /* SWAP-SPACE or SWAPSPACE2 */
} magic;
struct {
char bootbits[1024]; /* Space for disklabel etc. */
__u32 version;
__u32 last_page;
__u32 nr_badpages;
unsigned char sws_uuid[16];
unsigned char sws_volume[16];
__u32 padding[117];
__u32 badpages[1];
} info;
};
/*
* current->reclaim_state points to one of these when a task is running
* memory reclaim
*/
struct reclaim_state {
/* pages reclaimed outside of LRU-based reclaim */
unsigned long reclaimed;
#ifdef CONFIG_LRU_GEN
/* per-thread mm walk data */
struct lru_gen_mm_walk *mm_walk;
#endif
};
/*
* mm_account_reclaimed_pages(): account reclaimed pages outside of LRU-based
* reclaim
* @pages: number of pages reclaimed
*
* If the current process is undergoing a reclaim operation, increment the
* number of reclaimed pages by @pages.
*/
static inline void mm_account_reclaimed_pages(unsigned long pages)
{
if (current->reclaim_state)
current->reclaim_state->reclaimed += pages;
}
#ifdef __KERNEL__
struct address_space;
struct sysinfo;
struct writeback_control;
struct zone;
/*
* A swap extent maps a range of a swapfile's PAGE_SIZE pages onto a range of
* disk blocks. A rbtree of swap extents maps the entire swapfile (Where the
* term `swapfile' refers to either a blockdevice or an IS_REG file). Apart
* from setup, they're handled identically.
*
* We always assume that blocks are of size PAGE_SIZE.
*/
struct swap_extent {
struct rb_node rb_node;
pgoff_t start_page;
pgoff_t nr_pages;
sector_t start_block;
};
/*
* Max bad pages in the new format..
*/
#define MAX_SWAP_BADPAGES \
((offsetof(union swap_header, magic.magic) - \
offsetof(union swap_header, info.badpages)) / sizeof(int))
enum {
SWP_USED = (1 << 0), /* is slot in swap_info[] used? */
SWP_WRITEOK = (1 << 1), /* ok to write to this swap? */
SWP_DISCARDABLE = (1 << 2), /* blkdev support discard */
SWP_DISCARDING = (1 << 3), /* now discarding a free cluster */
SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */
SWP_CONTINUED = (1 << 5), /* swap_map has count continuation */
SWP_BLKDEV = (1 << 6), /* its a block device */
SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */
SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */
SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */
SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */
/* add others here before... */
SWP_SCANNING = (1 << 14), /* refcount in scan_swap_map */
};
#define SWAP_CLUSTER_MAX 32UL
#define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX
/* Bit flag in swap_map */
#define SWAP_HAS_CACHE 0x40 /* Flag page is cached, in first swap_map */
#define COUNT_CONTINUED 0x80 /* Flag swap_map continuation for full count */
/* Special value in first swap_map */
#define SWAP_MAP_MAX 0x3e /* Max count */
#define SWAP_MAP_BAD 0x3f /* Note page is bad */
#define SWAP_MAP_SHMEM 0xbf /* Owned by shmem/tmpfs */
/* Special value in each swap_map continuation */
#define SWAP_CONT_MAX 0x7f /* Max count */
/*
* We use this to track usage of a cluster. A cluster is a block of swap disk
* space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All
* free clusters are organized into a list. We fetch an entry from the list to
* get a free cluster.
*
* The data field stores next cluster if the cluster is free or cluster usage
* counter otherwise. The flags field determines if a cluster is free. This is
* protected by swap_info_struct.lock.
*/
struct swap_cluster_info {
spinlock_t lock; /*
* Protect swap_cluster_info fields
* and swap_info_struct->swap_map
* elements correspond to the swap
* cluster
*/
unsigned int data:24;
unsigned int flags:8;
};
#define CLUSTER_FLAG_FREE 1 /* This cluster is free */
#define CLUSTER_FLAG_NEXT_NULL 2 /* This cluster has no next cluster */
#define CLUSTER_FLAG_HUGE 4 /* This cluster is backing a transparent huge page */
/*
* We assign a cluster to each CPU, so each CPU can allocate swap entry from
* its own cluster and swapout sequentially. The purpose is to optimize swapout
* throughput.
*/
struct percpu_cluster {
struct swap_cluster_info index; /* Current cluster index */
unsigned int next; /* Likely next allocation offset */
};
struct swap_cluster_list {
struct swap_cluster_info head;
struct swap_cluster_info tail;
};
/*
* The in-memory structure used to track swap areas.
*/
struct swap_info_struct {
struct percpu_ref users; /* indicate and keep swap device valid. */
unsigned long flags; /* SWP_USED etc: see above */
signed short prio; /* swap priority of this type */
struct plist_node list; /* entry in swap_active_head */
signed char type; /* strange name for an index */
unsigned int max; /* extent of the swap_map */
unsigned char *swap_map; /* vmalloc'ed array of usage counts */
struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
struct swap_cluster_list free_clusters; /* free clusters list */
unsigned int lowest_bit; /* index of first free in swap_map */
unsigned int highest_bit; /* index of last free in swap_map */
unsigned int pages; /* total of usable pages of swap */
unsigned int inuse_pages; /* number of those currently in use */
unsigned int cluster_next; /* likely index for next allocation */
unsigned int cluster_nr; /* countdown to next cluster search */
unsigned int __percpu *cluster_next_cpu; /*percpu index for next allocation */
struct percpu_cluster __percpu *percpu_cluster; /* per cpu's swap location */
struct rb_root swap_extent_root;/* root of the swap extent rbtree */
struct file *bdev_file; /* open handle of the bdev */
struct block_device *bdev; /* swap device or bdev of swap file */
struct file *swap_file; /* seldom referenced */
unsigned int old_block_size; /* seldom referenced */
struct completion comp; /* seldom referenced */
spinlock_t lock; /*
* protect map scan related fields like
* swap_map, lowest_bit, highest_bit,
* inuse_pages, cluster_next,
* cluster_nr, lowest_alloc,
* highest_alloc, free/discard cluster
* list. other fields are only changed
* at swapon/swapoff, so are protected
* by swap_lock. changing flags need
* hold this lock and swap_lock. If
* both locks need hold, hold swap_lock
* first.
*/
spinlock_t cont_lock; /*
* protect swap count continuation page
* list.
*/
struct work_struct discard_work; /* discard worker */
struct swap_cluster_list discard_clusters; /* discard clusters list */
struct plist_node avail_lists[]; /*
* entries in swap_avail_heads, one
* entry per node.
* Must be last as the number of the
* array is nr_node_ids, which is not
* a fixed value so have to allocate
* dynamically.
* And it has to be an array so that
* plist_for_each_* can work.
*/
};
static inline swp_entry_t page_swap_entry(struct page *page)
{
struct folio *folio = page_folio(page);
swp_entry_t entry = folio->swap;
entry.val += folio_page_idx(folio, page);
return entry;
}
/* linux/mm/workingset.c */
bool workingset_test_recent(void *shadow, bool file, bool *workingset);
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
void workingset_refault(struct folio *folio, void *shadow);
void workingset_activation(struct folio *folio);
/* linux/mm/page_alloc.c */
extern unsigned long totalreserve_pages;
/* Definition of global_zone_page_state not available yet */
#define nr_free_pages() global_zone_page_state(NR_FREE_PAGES)
/* linux/mm/swap.c */
void lru_note_cost(struct lruvec *lruvec, bool file,
unsigned int nr_io, unsigned int nr_rotated);
void lru_note_cost_refault(struct folio *);
void folio_add_lru(struct folio *);
void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
void mark_page_accessed(struct page *);
void folio_mark_accessed(struct folio *);
extern atomic_t lru_disable_count;
static inline bool lru_cache_disabled(void)
{
return atomic_read(&lru_disable_count);
}
static inline void lru_cache_enable(void)
{
atomic_dec(&lru_disable_count);
}
extern void lru_cache_disable(void);
extern void lru_add_drain(void);
extern void lru_add_drain_cpu(int cpu);
extern void lru_add_drain_cpu_zone(struct zone *zone);
extern void lru_add_drain_all(void);
void folio_deactivate(struct folio *folio);
void folio_mark_lazyfree(struct folio *folio);
extern void swap_setup(void);
/* linux/mm/vmscan.c */
extern unsigned long zone_reclaimable_pages(struct zone *zone);
extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
gfp_t gfp_mask, nodemask_t *mask);
#define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
#define MEMCG_RECLAIM_PROACTIVE (1 << 2)
extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
unsigned long nr_pages,
gfp_t gfp_mask,
unsigned int reclaim_options);
extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem,
gfp_t gfp_mask, bool noswap,
pg_data_t *pgdat,
unsigned long *nr_scanned);
extern unsigned long shrink_all_memory(unsigned long nr_pages);
extern int vm_swappiness;
long remove_mapping(struct address_space *mapping, struct folio *folio);
#ifdef CONFIG_NUMA
extern int node_reclaim_mode;
extern int sysctl_min_unmapped_ratio;
extern int sysctl_min_slab_ratio;
#else
#define node_reclaim_mode 0
#endif
static inline bool node_reclaim_enabled(void)
{
/* Is any node_reclaim_mode bit set? */
return node_reclaim_mode & (RECLAIM_ZONE|RECLAIM_WRITE|RECLAIM_UNMAP);
}
void check_move_unevictable_folios(struct folio_batch *fbatch);
extern void __meminit kswapd_run(int nid);
extern void __meminit kswapd_stop(int nid);
#ifdef CONFIG_SWAP
int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
unsigned long nr_pages, sector_t start_block);
int generic_swapfile_activate(struct swap_info_struct *, struct file *,
sector_t *);
static inline unsigned long total_swapcache_pages(void)
{
return global_node_page_state(NR_SWAPCACHE);
}
void free_swap_cache(struct folio *folio);
void free_page_and_swap_cache(struct page *);
void free_pages_and_swap_cache(struct encoded_page **, int);
/* linux/mm/swapfile.c */
extern atomic_long_t nr_swap_pages;
extern long total_swap_pages;
extern atomic_t nr_rotate_swap;
extern bool has_usable_swap(void);
/* Swap 50% full? Release swapcache more aggressively.. */
static inline bool vm_swap_full(void)
{
return atomic_long_read(&nr_swap_pages) * 2 < total_swap_pages;
}
static inline long get_nr_swap_pages(void)
{
return atomic_long_read(&nr_swap_pages);
}
extern void si_swapinfo(struct sysinfo *);
swp_entry_t folio_alloc_swap(struct folio *folio);
bool folio_free_swap(struct folio *folio);
void put_swap_folio(struct folio *folio, swp_entry_t entry);
extern swp_entry_t get_swap_page_of_type(int);
extern int get_swap_pages(int n, swp_entry_t swp_entries[], int entry_size);
extern int add_swap_count_continuation(swp_entry_t, gfp_t);
extern void swap_shmem_alloc(swp_entry_t);
extern int swap_duplicate(swp_entry_t);
extern int swapcache_prepare(swp_entry_t);
extern void swap_free(swp_entry_t);
extern void swapcache_free_entries(swp_entry_t *entries, int n);
extern int free_swap_and_cache(swp_entry_t);
int swap_type_of(dev_t device, sector_t offset);
int find_first_swap(dev_t *device);
extern unsigned int count_swap_pages(int, int);
extern sector_t swapdev_block(int, pgoff_t);
extern int __swap_count(swp_entry_t entry);
extern int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry);
extern int swp_swapcount(swp_entry_t entry);
struct swap_info_struct *swp_swap_info(swp_entry_t entry);
struct backing_dev_info;
extern int init_swap_address_space(unsigned int type, unsigned long nr_pages);
extern void exit_swap_address_space(unsigned int type);
extern struct swap_info_struct *get_swap_device(swp_entry_t entry);
sector_t swap_folio_sector(struct folio *folio);
static inline void put_swap_device(struct swap_info_struct *si)
{
percpu_ref_put(&si->users);
}
#else /* CONFIG_SWAP */
static inline struct swap_info_struct *swp_swap_info(swp_entry_t entry)
{
return NULL;
}
static inline struct swap_info_struct *get_swap_device(swp_entry_t entry)
{
return NULL;
}
static inline void put_swap_device(struct swap_info_struct *si)
{
}
#define get_nr_swap_pages() 0L
#define total_swap_pages 0L
#define total_swapcache_pages() 0UL
#define vm_swap_full() 0
#define si_swapinfo(val) \
do { (val)->freeswap = (val)->totalswap = 0; } while (0)
/* only sparc can not include linux/pagemap.h in this file
* so leave put_page and release_pages undeclared... */
#define free_page_and_swap_cache(page) \
put_page(page)
#define free_pages_and_swap_cache(pages, nr) \
release_pages((pages), (nr));
/* used to sanity check ptes in zap_pte_range when CONFIG_SWAP=0 */
#define free_swap_and_cache(e) is_pfn_swap_entry(e)
static inline void free_swap_cache(struct folio *folio)
{
}
static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask)
{
return 0;
}
static inline void swap_shmem_alloc(swp_entry_t swp)
{
}
static inline int swap_duplicate(swp_entry_t swp)
{
return 0;
}
static inline int swapcache_prepare(swp_entry_t swp)
{
return 0;
}
static inline void swap_free(swp_entry_t swp)
{
}
static inline void put_swap_folio(struct folio *folio, swp_entry_t swp)
{
}
static inline int __swap_count(swp_entry_t entry)
{
return 0;
}
static inline int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
{
return 0;
}
static inline int swp_swapcount(swp_entry_t entry)
{
return 0;
}
static inline swp_entry_t folio_alloc_swap(struct folio *folio)
{
swp_entry_t entry;
entry.val = 0;
return entry;
}
static inline bool folio_free_swap(struct folio *folio)
{
return false;
}
static inline int add_swap_extent(struct swap_info_struct *sis,
unsigned long start_page,
unsigned long nr_pages, sector_t start_block)
{
return -EINVAL;
}
#endif /* CONFIG_SWAP */
#ifdef CONFIG_THP_SWAP
extern int split_swap_cluster(swp_entry_t entry);
#else
static inline int split_swap_cluster(swp_entry_t entry)
{
return 0;
}
#endif
#ifdef CONFIG_MEMCG
static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
{
/* Cgroup2 doesn't have per-cgroup swappiness */
if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
return READ_ONCE(vm_swappiness);
/* root ? */
if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
return READ_ONCE(vm_swappiness);
return READ_ONCE(memcg->swappiness);
}
#else
static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
{
return READ_ONCE(vm_swappiness);
}
#endif
#if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp);
static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
{
if (mem_cgroup_disabled())
return;
__folio_throttle_swaprate(folio, gfp);
}
#else
static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
{
}
#endif
#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);
int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry);
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
swp_entry_t entry)
{
if (mem_cgroup_disabled())
return 0;
return __mem_cgroup_try_charge_swap(folio, entry);
}
extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
{
if (mem_cgroup_disabled())
return;
__mem_cgroup_uncharge_swap(entry, nr_pages);
}
extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
extern bool mem_cgroup_swap_full(struct folio *folio);
#else
static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
{
}
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
swp_entry_t entry)
{
return 0;
}
static inline void mem_cgroup_uncharge_swap(swp_entry_t entry,
unsigned int nr_pages)
{
}
static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
{
return get_nr_swap_pages();
}
static inline bool mem_cgroup_swap_full(struct folio *folio)
{
return vm_swap_full();
}
#endif
#endif /* __KERNEL__*/
#endif /* _LINUX_SWAP_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H
#include <asm/processor-flags.h>
/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
struct io_bitmap;
struct vm86;
#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
#include <uapi/asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeatures.h>
#include <asm/cpuid.h>
#include <asm/page.h>
#include <asm/pgtable_types.h>
#include <asm/percpu.h>
#include <asm/desc_defs.h>
#include <asm/nops.h>
#include <asm/special_insns.h>
#include <asm/fpu/types.h>
#include <asm/unwind_hints.h>
#include <asm/vmxfeatures.h>
#include <asm/vdso/processor.h>
#include <asm/shstk.h>
#include <linux/personality.h>
#include <linux/cache.h>
#include <linux/threads.h>
#include <linux/math64.h>
#include <linux/err.h>
#include <linux/irqflags.h>
#include <linux/mem_encrypt.h>
/*
* We handle most unaligned accesses in hardware. On the other hand
* unaligned DMA can be quite expensive on some Nehalem processors.
*
* Based on this we disable the IP header alignment in network drivers.
*/
#define NET_IP_ALIGN 0
#define HBP_NUM 4
/*
* These alignment constraints are for performance in the vSMP case,
* but in the task_struct case we must also meet hardware imposed
* alignment requirements of the FPU state:
*/
#ifdef CONFIG_X86_VSMP
# define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT)
#else
# define ARCH_MIN_TASKALIGN __alignof__(union fpregs_state)
# define ARCH_MIN_MMSTRUCT_ALIGN 0
#endif
enum tlb_infos {
ENTRIES,
NR_INFO
};
extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_1g[NR_INFO];
/*
* CPU type and hardware bug flags. Kept separately for each CPU.
*/
struct cpuinfo_topology {
// Real APIC ID read from the local APIC
u32 apicid;
// The initial APIC ID provided by CPUID
u32 initial_apicid;
// Physical package ID
u32 pkg_id;
// Physical die ID on AMD, Relative on Intel
u32 die_id;
// Compute unit ID - AMD specific
u32 cu_id;
// Core ID relative to the package
u32 core_id;
// Logical ID mappings
u32 logical_pkg_id;
u32 logical_die_id;
// AMD Node ID and Nodes per Package info
u32 amd_node_id;
// Cache level topology IDs
u32 llc_id;
u32 l2c_id;
};
struct cpuinfo_x86 {
__u8 x86; /* CPU family */
__u8 x86_vendor; /* CPU vendor */
__u8 x86_model;
__u8 x86_stepping;
#ifdef CONFIG_X86_64
/* Number of 4K pages in DTLB/ITLB combined(in pages): */
int x86_tlbsize;
#endif
#ifdef CONFIG_X86_VMX_FEATURE_NAMES
__u32 vmx_capability[NVMXINTS];
#endif
__u8 x86_virt_bits;
__u8 x86_phys_bits;
/* Max extended CPUID function supported: */
__u32 extended_cpuid_level;
/* Maximum supported CPUID level, -1=no CPUID: */
int cpuid_level;
/*
* Align to size of unsigned long because the x86_capability array
* is passed to bitops which require the alignment. Use unnamed
* union to enforce the array is aligned to size of unsigned long.
*/
union {
__u32 x86_capability[NCAPINTS + NBUGINTS];
unsigned long x86_capability_alignment;
};
char x86_vendor_id[16];
char x86_model_id[64];
struct cpuinfo_topology topo;
/* in KB - valid for CPUS which support this call: */
unsigned int x86_cache_size;
int x86_cache_alignment; /* In bytes */
/* Cache QoS architectural values, valid only on the BSP: */
int x86_cache_max_rmid; /* max index */
int x86_cache_occ_scale; /* scale to bytes */
int x86_cache_mbm_width_offset;
int x86_power;
unsigned long loops_per_jiffy;
/* protected processor identification number */
u64 ppin;
u16 x86_clflush_size;
/* number of cores as seen by the OS: */
u16 booted_cores;
/* Index into per_cpu list: */
u16 cpu_index;
/* Is SMT active on this core? */
bool smt_active;
u32 microcode;
/* Address space bits used by the cache internally */
u8 x86_cache_bits;
unsigned initialized : 1;
} __randomize_layout;
#define X86_VENDOR_INTEL 0
#define X86_VENDOR_CYRIX 1
#define X86_VENDOR_AMD 2
#define X86_VENDOR_UMC 3
#define X86_VENDOR_CENTAUR 5
#define X86_VENDOR_TRANSMETA 7
#define X86_VENDOR_NSC 8
#define X86_VENDOR_HYGON 9
#define X86_VENDOR_ZHAOXIN 10
#define X86_VENDOR_VORTEX 11
#define X86_VENDOR_NUM 12
#define X86_VENDOR_UNKNOWN 0xff
/*
* capabilities of CPUs
*/
extern struct cpuinfo_x86 boot_cpu_data;
extern struct cpuinfo_x86 new_cpu_data;
extern __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
extern __u32 cpu_caps_set[NCAPINTS + NBUGINTS];
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
#define cpu_data(cpu) per_cpu(cpu_info, cpu)
extern const struct seq_operations cpuinfo_op;
#define cache_line_size() (boot_cpu_data.x86_cache_alignment)
extern void cpu_detect(struct cpuinfo_x86 *c);
static inline unsigned long long l1tf_pfn_limit(void)
{
return BIT_ULL(boot_cpu_data.x86_cache_bits - 1 - PAGE_SHIFT);
}
extern void early_cpu_init(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
extern void print_cpu_info(struct cpuinfo_x86 *);
void print_cpu_msr(struct cpuinfo_x86 *);
/*
* Friendlier CR3 helpers.
*/
static inline unsigned long read_cr3_pa(void)
{
return __read_cr3() & CR3_ADDR_MASK;
}
static inline unsigned long native_read_cr3_pa(void)
{
return __native_read_cr3() & CR3_ADDR_MASK;
}
static inline void load_cr3(pgd_t *pgdir)
{
write_cr3(__sme_pa(pgdir));
}
/*
* Note that while the legacy 'TSS' name comes from 'Task State Segment',
* on modern x86 CPUs the TSS also holds information important to 64-bit mode,
* unrelated to the task-switch mechanism:
*/
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
unsigned short back_link, __blh;
unsigned long sp0;
unsigned short ss0, __ss0h;
unsigned long sp1;
/*
* We don't use ring 1, so ss1 is a convenient scratch space in
* the same cacheline as sp0. We use ss1 to cache the value in
* MSR_IA32_SYSENTER_CS. When we context switch
* MSR_IA32_SYSENTER_CS, we first check if the new value being
* written matches ss1, and, if it's not, then we wrmsr the new
* value and update ss1.
*
* The only reason we context switch MSR_IA32_SYSENTER_CS is
* that we set it to zero in vm86 tasks to avoid corrupting the
* stack if we were to go through the sysenter path from vm86
* mode.
*/
unsigned short ss1; /* MSR_IA32_SYSENTER_CS */
unsigned short __ss1h;
unsigned long sp2;
unsigned short ss2, __ss2h;
unsigned long __cr3;
unsigned long ip;
unsigned long flags;
unsigned long ax;
unsigned long cx;
unsigned long dx;
unsigned long bx;
unsigned long sp;
unsigned long bp;
unsigned long si;
unsigned long di;
unsigned short es, __esh;
unsigned short cs, __csh;
unsigned short ss, __ssh;
unsigned short ds, __dsh;
unsigned short fs, __fsh;
unsigned short gs, __gsh;
unsigned short ldt, __ldth;
unsigned short trace;
unsigned short io_bitmap_base;
} __attribute__((packed));
#else
struct x86_hw_tss {
u32 reserved1;
u64 sp0;
u64 sp1;
/*
* Since Linux does not use ring 2, the 'sp2' slot is unused by
* hardware. entry_SYSCALL_64 uses it as scratch space to stash
* the user RSP value.
*/
u64 sp2;
u64 reserved2;
u64 ist[7];
u32 reserved3;
u32 reserved4;
u16 reserved5;
u16 io_bitmap_base;
} __attribute__((packed));
#endif
/*
* IO-bitmap sizes:
*/
#define IO_BITMAP_BITS 65536
#define IO_BITMAP_BYTES (IO_BITMAP_BITS / BITS_PER_BYTE)
#define IO_BITMAP_LONGS (IO_BITMAP_BYTES / sizeof(long))
#define IO_BITMAP_OFFSET_VALID_MAP \
(offsetof(struct tss_struct, io_bitmap.bitmap) - \
offsetof(struct tss_struct, x86_tss))
#define IO_BITMAP_OFFSET_VALID_ALL \
(offsetof(struct tss_struct, io_bitmap.mapall) - \
offsetof(struct tss_struct, x86_tss))
#ifdef CONFIG_X86_IOPL_IOPERM
/*
* sizeof(unsigned long) coming from an extra "long" at the end of the
* iobitmap. The limit is inclusive, i.e. the last valid byte.
*/
# define __KERNEL_TSS_LIMIT \
(IO_BITMAP_OFFSET_VALID_ALL + IO_BITMAP_BYTES + \
sizeof(unsigned long) - 1)
#else
# define __KERNEL_TSS_LIMIT \
(offsetof(struct tss_struct, x86_tss) + sizeof(struct x86_hw_tss) - 1)
#endif
/* Base offset outside of TSS_LIMIT so unpriviledged IO causes #GP */
#define IO_BITMAP_OFFSET_INVALID (__KERNEL_TSS_LIMIT + 1)
struct entry_stack {
char stack[PAGE_SIZE];
};
struct entry_stack_page {
struct entry_stack stack;
} __aligned(PAGE_SIZE);
/*
* All IO bitmap related data stored in the TSS:
*/
struct x86_io_bitmap {
/* The sequence number of the last active bitmap. */
u64 prev_sequence;
/*
* Store the dirty size of the last io bitmap offender. The next
* one will have to do the cleanup as the switch out to a non io
* bitmap user will just set x86_tss.io_bitmap_base to a value
* outside of the TSS limit. So for sane tasks there is no need to
* actually touch the io_bitmap at all.
*/
unsigned int prev_max;
/*
* The extra 1 is there because the CPU will access an
* additional byte beyond the end of the IO permission
* bitmap. The extra byte must be all 1 bits, and must
* be within the limit.
*/
unsigned long bitmap[IO_BITMAP_LONGS + 1];
/*
* Special I/O bitmap to emulate IOPL(3). All bytes zero,
* except the additional byte at the end.
*/
unsigned long mapall[IO_BITMAP_LONGS + 1];
};
struct tss_struct {
/*
* The fixed hardware portion. This must not cross a page boundary
* at risk of violating the SDM's advice and potentially triggering
* errata.
*/
struct x86_hw_tss x86_tss;
struct x86_io_bitmap io_bitmap;
} __aligned(PAGE_SIZE);
DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);
/* Per CPU interrupt stacks */
struct irq_stack {
char stack[IRQ_STACK_SIZE];
} __aligned(IRQ_STACK_SIZE);
#ifdef CONFIG_X86_64
struct fixed_percpu_data {
/*
* GCC hardcodes the stack canary as %gs:40. Since the
* irq_stack is the object at %gs:0, we reserve the bottom
* 48 bytes of the irq stack for the canary.
*
* Once we are willing to require -mstack-protector-guard-symbol=
* support for x86_64 stackprotector, we can get rid of this.
*/
char gs_base[40];
unsigned long stack_canary;
};
DECLARE_PER_CPU_FIRST(struct fixed_percpu_data, fixed_percpu_data) __visible;
DECLARE_INIT_PER_CPU(fixed_percpu_data);
static inline unsigned long cpu_kernelmode_gs_base(int cpu)
{
return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
}
extern asmlinkage void entry_SYSCALL32_ignore(void);
/* Save actual FS/GS selectors and bases to current->thread */
void current_save_fsgs(void);
#else /* X86_64 */
#ifdef CONFIG_STACKPROTECTOR
DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
#endif
#endif /* !X86_64 */
struct perf_event;
struct thread_struct {
/* Cached TLS descriptors: */
struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES];
#ifdef CONFIG_X86_32
unsigned long sp0;
#endif
unsigned long sp;
#ifdef CONFIG_X86_32
unsigned long sysenter_cs;
#else
unsigned short es;
unsigned short ds;
unsigned short fsindex;
unsigned short gsindex;
#endif
#ifdef CONFIG_X86_64
unsigned long fsbase;
unsigned long gsbase;
#else
/*
* XXX: this could presumably be unsigned short. Alternatively,
* 32-bit kernels could be taught to use fsindex instead.
*/
unsigned long fs;
unsigned long gs;
#endif
/* Save middle states of ptrace breakpoints */
struct perf_event *ptrace_bps[HBP_NUM];
/* Debug status used for traps, single steps, etc... */
unsigned long virtual_dr6;
/* Keep track of the exact dr7 value set by the user */
unsigned long ptrace_dr7;
/* Fault info: */
unsigned long cr2;
unsigned long trap_nr;
unsigned long error_code;
#ifdef CONFIG_VM86
/* Virtual 86 mode info */
struct vm86 *vm86;
#endif
/* IO permissions: */
struct io_bitmap *io_bitmap;
/*
* IOPL. Privilege level dependent I/O permission which is
* emulated via the I/O bitmap to prevent user space from disabling
* interrupts.
*/
unsigned long iopl_emul;
unsigned int iopl_warn:1;
/*
* Protection Keys Register for Userspace. Loaded immediately on
* context switch. Store it in thread_struct to avoid a lookup in
* the tasks's FPU xstate buffer. This value is only valid when a
* task is scheduled out. For 'current' the authoritative source of
* PKRU is the hardware itself.
*/
u32 pkru;
#ifdef CONFIG_X86_USER_SHADOW_STACK
unsigned long features;
unsigned long features_locked;
struct thread_shstk shstk;
#endif
/* Floating point and extended processor state */
struct fpu fpu;
/*
* WARNING: 'fpu' is dynamically-sized. It *MUST* be at
* the end.
*/
};
extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size);
static inline void arch_thread_struct_whitelist(unsigned long *offset,
unsigned long *size)
{
fpu_thread_struct_whitelist(offset, size);
}
static inline void
native_load_sp0(unsigned long sp0)
{
this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
}
static __always_inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
asm volatile("swapgs" ::: "memory");
#endif
}
static __always_inline unsigned long current_top_of_stack(void)
{
/*
* We can't read directly from tss.sp0: sp0 on x86_32 is special in
* and around vm86 mode and sp0 on x86_64 is special because of the
* entry trampoline.
*/
if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
return this_cpu_read_const(const_pcpu_hot.top_of_stack);
return this_cpu_read_stable(pcpu_hot.top_of_stack);
}
static __always_inline bool on_thread_stack(void)
{
return (unsigned long)(current_top_of_stack() -
current_stack_pointer) < THREAD_SIZE;
}
#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
static inline void load_sp0(unsigned long sp0)
{
native_load_sp0(sp0);
}
#endif /* CONFIG_PARAVIRT_XXL */
unsigned long __get_wchan(struct task_struct *p);
extern void select_idle_routine(void);
extern void amd_e400_c1e_apic_setup(void);
extern unsigned long boot_option_idle_override;
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
IDLE_POLL};
extern void enable_sep_cpu(void);
/* Defined in head.S */
extern struct desc_ptr early_gdt_descr;
extern void switch_gdt_and_percpu_base(int);
extern void load_direct_gdt(int);
extern void load_fixmap_gdt(int);
extern void cpu_init(void);
extern void cpu_init_exception_handling(void);
extern void cr4_init(void);
extern void set_task_blockstep(struct task_struct *task, bool on);
/* Boot loader type from the setup header: */
extern int bootloader_type;
extern int bootloader_version;
extern char ignore_fpu_irq;
#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#ifdef CONFIG_X86_32
# define BASE_PREFETCH ""
# define ARCH_HAS_PREFETCH
#else
# define BASE_PREFETCH "prefetcht0 %P1"
#endif
/*
* Prefetch instructions for Pentium III (+) and AMD Athlon (+)
*
* It's not worth to care about 3dnow prefetches for the K6
* because they are microcoded there and very slow.
*/
static inline void prefetch(const void *x)
{
alternative_input(BASE_PREFETCH, "prefetchnta %P1",
X86_FEATURE_XMM,
"m" (*(const char *)x));
}
/*
* 3dnow prefetch to get an exclusive cache line.
* Useful for spinlocks to avoid one state transition in the
* cache coherency protocol:
*/
static __always_inline void prefetchw(const void *x)
{
alternative_input(BASE_PREFETCH, "prefetchw %P1",
X86_FEATURE_3DNOWPREFETCH,
"m" (*(const char *)x));
}
#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
TOP_OF_KERNEL_STACK_PADDING)
#define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))
#define task_pt_regs(task) \
({ \
unsigned long __ptr = (unsigned long)task_stack_page(task); \
__ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING; \
((struct pt_regs *)__ptr) - 1; \
})
#ifdef CONFIG_X86_32
#define INIT_THREAD { \
.sp0 = TOP_OF_INIT_STACK, \
.sysenter_cs = __KERNEL_CS, \
}
#define KSTK_ESP(task) (task_pt_regs(task)->sp)
#else
extern unsigned long __end_init_task[];
#define INIT_THREAD { \
.sp = (unsigned long)&__end_init_task - \
TOP_OF_KERNEL_STACK_PADDING - \
sizeof(struct pt_regs), \
}
extern unsigned long KSTK_ESP(struct task_struct *task);
#endif /* CONFIG_X86_64 */
extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
unsigned long new_sp);
/*
* This decides where the kernel will search for a free chunk of vm
* space during mmap's.
*/
#define __TASK_UNMAPPED_BASE(task_size) (PAGE_ALIGN(task_size / 3))
#define TASK_UNMAPPED_BASE __TASK_UNMAPPED_BASE(TASK_SIZE_LOW)
#define KSTK_EIP(task) (task_pt_regs(task)->ip)
/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr) get_tsc_mode((adr))
#define SET_TSC_CTL(val) set_tsc_mode((val))
extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);
DECLARE_PER_CPU(u64, msr_misc_features_shadow);
static inline u32 per_cpu_llc_id(unsigned int cpu)
{
return per_cpu(cpu_info.topo.llc_id, cpu);
}
static inline u32 per_cpu_l2c_id(unsigned int cpu)
{
return per_cpu(cpu_info.topo.l2c_id, cpu);
}
#ifdef CONFIG_CPU_SUP_AMD
extern u32 amd_get_highest_perf(void);
extern void amd_clear_divider(void);
extern void amd_check_microcode(void);
#else
static inline u32 amd_get_highest_perf(void) { return 0; }
static inline void amd_clear_divider(void) { }
static inline void amd_check_microcode(void) { }
#endif
extern unsigned long arch_align_stack(unsigned long sp);
void free_init_pages(const char *what, unsigned long begin, unsigned long end);
extern void free_kernel_image_pages(const char *what, void *begin, void *end);
void default_idle(void);
#ifdef CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif
void __noreturn stop_this_cpu(void *dummy);
void microcode_check(struct cpuinfo_x86 *prev_info);
void store_cpu_caps(struct cpuinfo_x86 *info);
enum l1tf_mitigations {
L1TF_MITIGATION_OFF,
L1TF_MITIGATION_FLUSH_NOWARN,
L1TF_MITIGATION_FLUSH,
L1TF_MITIGATION_FLUSH_NOSMT,
L1TF_MITIGATION_FULL,
L1TF_MITIGATION_FULL_FORCE
};
extern enum l1tf_mitigations l1tf_mitigation;
enum mds_mitigations {
MDS_MITIGATION_OFF,
MDS_MITIGATION_FULL,
MDS_MITIGATION_VMWERV,
};
extern bool gds_ucode_mitigated(void);
/*
* Make previous memory operations globally visible before
* a WRMSR.
*
* MFENCE makes writes visible, but only affects load/store
* instructions. WRMSR is unfortunately not a load/store
* instruction and is unaffected by MFENCE. The LFENCE ensures
* that the WRMSR is not reordered.
*
* Most WRMSRs are full serializing instructions themselves and
* do not require this barrier. This is only required for the
* IA32_TSC_DEADLINE and X2APIC MSRs.
*/
static inline void weak_wrmsr_fence(void)
{
alternative("mfence; lfence", "", ALT_NOT(X86_FEATURE_APIC_MSRS_FENCE));
}
#endif /* _ASM_X86_PROCESSOR_H */
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/compiler.h>
#include <linux/export.h>
#include <linux/err.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/sched/task_stack.h>
#include <linux/security.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mman.h>
#include <linux/hugetlb.h>
#include <linux/vmalloc.h>
#include <linux/userfaultfd_k.h>
#include <linux/elf.h>
#include <linux/elf-randomize.h>
#include <linux/personality.h>
#include <linux/random.h>
#include <linux/processor.h>
#include <linux/sizes.h>
#include <linux/compat.h>
#include <linux/uaccess.h>
#include "internal.h"
#include "swap.h"
/**
* kfree_const - conditionally free memory
* @x: pointer to the memory
*
* Function calls kfree only if @x is not in .rodata section.
*/
void kfree_const(const void *x)
{
if (!is_kernel_rodata((unsigned long)x)) kfree(x);
}
EXPORT_SYMBOL(kfree_const);
/**
* kstrdup - allocate space for and copy an existing string
* @s: the string to duplicate
* @gfp: the GFP mask used in the kmalloc() call when allocating memory
*
* Return: newly allocated copy of @s or %NULL in case of error
*/
noinline
char *kstrdup(const char *s, gfp_t gfp)
{
size_t len;
char *buf;
if (!s)
return NULL;
len = strlen(s) + 1;
buf = kmalloc_track_caller(len, gfp);
if (buf)
memcpy(buf, s, len);
return buf;
}
EXPORT_SYMBOL(kstrdup);
/**
* kstrdup_const - conditionally duplicate an existing const string
* @s: the string to duplicate
* @gfp: the GFP mask used in the kmalloc() call when allocating memory
*
* Note: Strings allocated by kstrdup_const should be freed by kfree_const and
* must not be passed to krealloc().
*
* Return: source string if it is in .rodata section otherwise
* fallback to kstrdup.
*/
const char *kstrdup_const(const char *s, gfp_t gfp)
{
if (is_kernel_rodata((unsigned long)s))
return s;
return kstrdup(s, gfp);
}
EXPORT_SYMBOL(kstrdup_const);
/**
* kstrndup - allocate space for and copy an existing string
* @s: the string to duplicate
* @max: read at most @max chars from @s
* @gfp: the GFP mask used in the kmalloc() call when allocating memory
*
* Note: Use kmemdup_nul() instead if the size is known exactly.
*
* Return: newly allocated copy of @s or %NULL in case of error
*/
char *kstrndup(const char *s, size_t max, gfp_t gfp)
{
size_t len;
char *buf;
if (!s)
return NULL;
len = strnlen(s, max);
buf = kmalloc_track_caller(len+1, gfp);
if (buf) {
memcpy(buf, s, len);
buf[len] = '\0';
}
return buf;
}
EXPORT_SYMBOL(kstrndup);
/**
* kmemdup - duplicate region of memory
*
* @src: memory region to duplicate
* @len: memory region length
* @gfp: GFP mask to use
*
* Return: newly allocated copy of @src or %NULL in case of error,
* result is physically contiguous. Use kfree() to free.
*/
void *kmemdup(const void *src, size_t len, gfp_t gfp)
{
void *p;
p = kmalloc_track_caller(len, gfp);
if (p)
memcpy(p, src, len); return p;
}
EXPORT_SYMBOL(kmemdup);
/**
* kmemdup_array - duplicate a given array.
*
* @src: array to duplicate.
* @element_size: size of each element of array.
* @count: number of elements to duplicate from array.
* @gfp: GFP mask to use.
*
* Return: duplicated array of @src or %NULL in case of error,
* result is physically contiguous. Use kfree() to free.
*/
void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp)
{
return kmemdup(src, size_mul(element_size, count), gfp);
}
EXPORT_SYMBOL(kmemdup_array);
/**
* kvmemdup - duplicate region of memory
*
* @src: memory region to duplicate
* @len: memory region length
* @gfp: GFP mask to use
*
* Return: newly allocated copy of @src or %NULL in case of error,
* result may be not physically contiguous. Use kvfree() to free.
*/
void *kvmemdup(const void *src, size_t len, gfp_t gfp)
{
void *p;
p = kvmalloc(len, gfp);
if (p)
memcpy(p, src, len);
return p;
}
EXPORT_SYMBOL(kvmemdup);
/**
* kmemdup_nul - Create a NUL-terminated string from unterminated data
* @s: The data to stringify
* @len: The size of the data
* @gfp: the GFP mask used in the kmalloc() call when allocating memory
*
* Return: newly allocated copy of @s with NUL-termination or %NULL in
* case of error
*/
char *kmemdup_nul(const char *s, size_t len, gfp_t gfp)
{
char *buf;
if (!s)
return NULL;
buf = kmalloc_track_caller(len + 1, gfp);
if (buf) {
memcpy(buf, s, len);
buf[len] = '\0';
}
return buf;
}
EXPORT_SYMBOL(kmemdup_nul);
/**
* memdup_user - duplicate memory region from user space
*
* @src: source address in user space
* @len: number of bytes to copy
*
* Return: an ERR_PTR() on failure. Result is physically
* contiguous, to be freed by kfree().
*/
void *memdup_user(const void __user *src, size_t len)
{
void *p;
p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);
if (!p)
return ERR_PTR(-ENOMEM);
if (copy_from_user(p, src, len)) { kfree(p); return ERR_PTR(-EFAULT);
}
return p;
}
EXPORT_SYMBOL(memdup_user);
/**
* vmemdup_user - duplicate memory region from user space
*
* @src: source address in user space
* @len: number of bytes to copy
*
* Return: an ERR_PTR() on failure. Result may be not
* physically contiguous. Use kvfree() to free.
*/
void *vmemdup_user(const void __user *src, size_t len)
{
void *p;
p = kvmalloc(len, GFP_USER);
if (!p)
return ERR_PTR(-ENOMEM);
if (copy_from_user(p, src, len)) {
kvfree(p);
return ERR_PTR(-EFAULT);
}
return p;
}
EXPORT_SYMBOL(vmemdup_user);
/**
* strndup_user - duplicate an existing string from user space
* @s: The string to duplicate
* @n: Maximum number of bytes to copy, including the trailing NUL.
*
* Return: newly allocated copy of @s or an ERR_PTR() in case of error
*/
char *strndup_user(const char __user *s, long n)
{
char *p;
long length;
length = strnlen_user(s, n);
if (!length)
return ERR_PTR(-EFAULT);
if (length > n)
return ERR_PTR(-EINVAL);
p = memdup_user(s, length);
if (IS_ERR(p))
return p;
p[length - 1] = '\0';
return p;
}
EXPORT_SYMBOL(strndup_user);
/**
* memdup_user_nul - duplicate memory region from user space and NUL-terminate
*
* @src: source address in user space
* @len: number of bytes to copy
*
* Return: an ERR_PTR() on failure.
*/
void *memdup_user_nul(const void __user *src, size_t len)
{
char *p;
/*
* Always use GFP_KERNEL, since copy_from_user() can sleep and
* cause pagefault, which makes it pointless to use GFP_NOFS
* or GFP_ATOMIC.
*/
p = kmalloc_track_caller(len + 1, GFP_KERNEL);
if (!p)
return ERR_PTR(-ENOMEM);
if (copy_from_user(p, src, len)) {
kfree(p);
return ERR_PTR(-EFAULT);
}
p[len] = '\0';
return p;
}
EXPORT_SYMBOL(memdup_user_nul);
/* Check if the vma is being used as a stack by this task */
int vma_is_stack_for_current(struct vm_area_struct *vma)
{
struct task_struct * __maybe_unused t = current;
return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
}
/*
* Change backing file, only valid to use during initial VMA setup.
*/
void vma_set_file(struct vm_area_struct *vma, struct file *file)
{
/* Changing an anonymous vma with this is illegal */
get_file(file);
swap(vma->vm_file, file);
fput(file);
}
EXPORT_SYMBOL(vma_set_file);
#ifndef STACK_RND_MASK
#define STACK_RND_MASK (0x7ff >> (PAGE_SHIFT - 12)) /* 8MB of VA */
#endif
unsigned long randomize_stack_top(unsigned long stack_top)
{
unsigned long random_variable = 0;
if (current->flags & PF_RANDOMIZE) {
random_variable = get_random_long();
random_variable &= STACK_RND_MASK;
random_variable <<= PAGE_SHIFT;
}
#ifdef CONFIG_STACK_GROWSUP
return PAGE_ALIGN(stack_top) + random_variable;
#else
return PAGE_ALIGN(stack_top) - random_variable;
#endif
}
/**
* randomize_page - Generate a random, page aligned address
* @start: The smallest acceptable address the caller will take.
* @range: The size of the area, starting at @start, within which the
* random address must fall.
*
* If @start + @range would overflow, @range is capped.
*
* NOTE: Historical use of randomize_range, which this replaces, presumed that
* @start was already page aligned. We now align it regardless.
*
* Return: A page aligned address within [start, start + range). On error,
* @start is returned.
*/
unsigned long randomize_page(unsigned long start, unsigned long range)
{
if (!PAGE_ALIGNED(start)) {
range -= PAGE_ALIGN(start) - start;
start = PAGE_ALIGN(start);
}
if (start > ULONG_MAX - range)
range = ULONG_MAX - start;
range >>= PAGE_SHIFT;
if (range == 0)
return start;
return start + (get_random_long() % range << PAGE_SHIFT);
}
#ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
unsigned long __weak arch_randomize_brk(struct mm_struct *mm)
{
/* Is the current task 32bit ? */
if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task())
return randomize_page(mm->brk, SZ_32M);
return randomize_page(mm->brk, SZ_1G);
}
unsigned long arch_mmap_rnd(void)
{
unsigned long rnd;
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
if (is_compat_task())
rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1);
else
#endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */
rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1);
return rnd << PAGE_SHIFT;
}
static int mmap_is_legacy(struct rlimit *rlim_stack)
{
if (current->personality & ADDR_COMPAT_LAYOUT)
return 1;
/* On parisc the stack always grows up - so a unlimited stack should
* not be an indicator to use the legacy memory layout. */
if (rlim_stack->rlim_cur == RLIM_INFINITY &&
!IS_ENABLED(CONFIG_STACK_GROWSUP))
return 1;
return sysctl_legacy_va_layout;
}
/*
* Leave enough space between the mmap area and the stack to honour ulimit in
* the face of randomisation.
*/
#define MIN_GAP (SZ_128M)
#define MAX_GAP (STACK_TOP / 6 * 5)
static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack)
{
#ifdef CONFIG_STACK_GROWSUP
/*
* For an upwards growing stack the calculation is much simpler.
* Memory for the maximum stack size is reserved at the top of the
* task. mmap_base starts directly below the stack and grows
* downwards.
*/
return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd);
#else
unsigned long gap = rlim_stack->rlim_cur;
unsigned long pad = stack_guard_gap;
/* Account for stack randomization if necessary */
if (current->flags & PF_RANDOMIZE)
pad += (STACK_RND_MASK << PAGE_SHIFT);
/* Values close to RLIM_INFINITY can overflow. */
if (gap + pad > gap)
gap += pad;
if (gap < MIN_GAP)
gap = MIN_GAP;
else if (gap > MAX_GAP)
gap = MAX_GAP;
return PAGE_ALIGN(STACK_TOP - gap - rnd);
#endif
}
void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
{
unsigned long random_factor = 0UL;
if (current->flags & PF_RANDOMIZE)
random_factor = arch_mmap_rnd();
if (mmap_is_legacy(rlim_stack)) {
mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
mm->get_unmapped_area = arch_get_unmapped_area;
} else {
mm->mmap_base = mmap_base(random_factor, rlim_stack);
mm->get_unmapped_area = arch_get_unmapped_area_topdown;
}
}
#elif defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
{
mm->mmap_base = TASK_UNMAPPED_BASE;
mm->get_unmapped_area = arch_get_unmapped_area;
}
#endif
/**
* __account_locked_vm - account locked pages to an mm's locked_vm
* @mm: mm to account against
* @pages: number of pages to account
* @inc: %true if @pages should be considered positive, %false if not
* @task: task used to check RLIMIT_MEMLOCK
* @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
*
* Assumes @task and @mm are valid (i.e. at least one reference on each), and
* that mmap_lock is held as writer.
*
* Return:
* * 0 on success
* * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
*/
int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
struct task_struct *task, bool bypass_rlim)
{
unsigned long locked_vm, limit;
int ret = 0;
mmap_assert_write_locked(mm);
locked_vm = mm->locked_vm;
if (inc) {
if (!bypass_rlim) {
limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if (locked_vm + pages > limit)
ret = -ENOMEM;
}
if (!ret)
mm->locked_vm = locked_vm + pages;
} else {
WARN_ON_ONCE(pages > locked_vm);
mm->locked_vm = locked_vm - pages;
}
pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,
(void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,
locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
ret ? " - exceeded" : "");
return ret;
}
EXPORT_SYMBOL_GPL(__account_locked_vm);
/**
* account_locked_vm - account locked pages to an mm's locked_vm
* @mm: mm to account against, may be NULL
* @pages: number of pages to account
* @inc: %true if @pages should be considered positive, %false if not
*
* Assumes a non-NULL @mm is valid (i.e. at least one reference on it).
*
* Return:
* * 0 on success, or if mm is NULL
* * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
*/
int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc)
{
int ret;
if (pages == 0 || !mm)
return 0;
mmap_write_lock(mm);
ret = __account_locked_vm(mm, pages, inc, current,
capable(CAP_IPC_LOCK));
mmap_write_unlock(mm);
return ret;
}
EXPORT_SYMBOL_GPL(account_locked_vm);
unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flag, unsigned long pgoff)
{
unsigned long ret;
struct mm_struct *mm = current->mm;
unsigned long populate;
LIST_HEAD(uf);
ret = security_mmap_file(file, prot, flag);
if (!ret) {
if (mmap_write_lock_killable(mm))
return -EINTR;
ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate,
&uf);
mmap_write_unlock(mm);
userfaultfd_unmap_complete(mm, &uf);
if (populate)
mm_populate(ret, populate);
}
return ret;
}
unsigned long vm_mmap(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flag, unsigned long offset)
{
if (unlikely(offset + PAGE_ALIGN(len) < offset))
return -EINVAL;
if (unlikely(offset_in_page(offset)))
return -EINVAL;
return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
}
EXPORT_SYMBOL(vm_mmap);
/**
* kvmalloc_node - attempt to allocate physically contiguous memory, but upon
* failure, fall back to non-contiguous (vmalloc) allocation.
* @size: size of the request.
* @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
* @node: numa node to allocate from
*
* Uses kmalloc to get the memory but if the allocation fails then falls back
* to the vmalloc allocator. Use kvfree for freeing the memory.
*
* GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier.
* __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
* preferable to the vmalloc fallback, due to visible performance drawbacks.
*
* Return: pointer to the allocated memory of %NULL in case of failure
*/
void *kvmalloc_node(size_t size, gfp_t flags, int node)
{
gfp_t kmalloc_flags = flags;
void *ret;
/*
* We want to attempt a large physically contiguous block first because
* it is less likely to fragment multiple larger blocks and therefore
* contribute to a long term fragmentation less than vmalloc fallback.
* However make sure that larger requests are not too disruptive - no
* OOM killer and no allocation failure warnings as we have a fallback.
*/
if (size > PAGE_SIZE) { kmalloc_flags |= __GFP_NOWARN; if (!(kmalloc_flags & __GFP_RETRY_MAYFAIL)) kmalloc_flags |= __GFP_NORETRY;
/* nofail semantic is implemented by the vmalloc fallback */
kmalloc_flags &= ~__GFP_NOFAIL;
}
ret = kmalloc_node(size, kmalloc_flags, node);
/*
* It doesn't really make sense to fallback to vmalloc for sub page
* requests
*/
if (ret || size <= PAGE_SIZE)
return ret;
/* non-sleeping allocations are not supported by vmalloc */
if (!gfpflags_allow_blocking(flags))
return NULL;
/* Don't even allow crazy sizes */
if (unlikely(size > INT_MAX)) { WARN_ON_ONCE(!(flags & __GFP_NOWARN));
return NULL;
}
/*
* kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
* since the callers already cannot assume anything
* about the resulting pointer, and cannot play
* protection games.
*/
return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
node, __builtin_return_address(0));
}
EXPORT_SYMBOL(kvmalloc_node);
/**
* kvfree() - Free memory.
* @addr: Pointer to allocated memory.
*
* kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
* It is slightly more efficient to use kfree() or vfree() if you are certain
* that you know which one to use.
*
* Context: Either preemptible task context or not-NMI interrupt.
*/
void kvfree(const void *addr)
{
if (is_vmalloc_addr(addr)) vfree(addr);
else
kfree(addr);
}
EXPORT_SYMBOL(kvfree);
/**
* kvfree_sensitive - Free a data object containing sensitive information.
* @addr: address of the data object to be freed.
* @len: length of the data object.
*
* Use the special memzero_explicit() function to clear the content of a
* kvmalloc'ed object containing sensitive data to make sure that the
* compiler won't optimize out the data clearing.
*/
void kvfree_sensitive(const void *addr, size_t len)
{
if (likely(!ZERO_OR_NULL_PTR(addr))) {
memzero_explicit((void *)addr, len);
kvfree(addr);
}
}
EXPORT_SYMBOL(kvfree_sensitive);
void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
{
void *newp;
if (oldsize >= newsize)
return (void *)p;
newp = kvmalloc(newsize, flags);
if (!newp)
return NULL;
memcpy(newp, p, oldsize);
kvfree(p);
return newp;
}
EXPORT_SYMBOL(kvrealloc);
/**
* __vmalloc_array - allocate memory for a virtually contiguous array.
* @n: number of elements.
* @size: element size.
* @flags: the type of memory to allocate (see kmalloc).
*/
void *__vmalloc_array(size_t n, size_t size, gfp_t flags)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
return __vmalloc(bytes, flags);
}
EXPORT_SYMBOL(__vmalloc_array);
/**
* vmalloc_array - allocate memory for a virtually contiguous array.
* @n: number of elements.
* @size: element size.
*/
void *vmalloc_array(size_t n, size_t size)
{
return __vmalloc_array(n, size, GFP_KERNEL);
}
EXPORT_SYMBOL(vmalloc_array);
/**
* __vcalloc - allocate and zero memory for a virtually contiguous array.
* @n: number of elements.
* @size: element size.
* @flags: the type of memory to allocate (see kmalloc).
*/
void *__vcalloc(size_t n, size_t size, gfp_t flags)
{
return __vmalloc_array(n, size, flags | __GFP_ZERO);
}
EXPORT_SYMBOL(__vcalloc);
/**
* vcalloc - allocate and zero memory for a virtually contiguous array.
* @n: number of elements.
* @size: element size.
*/
void *vcalloc(size_t n, size_t size)
{
return __vmalloc_array(n, size, GFP_KERNEL | __GFP_ZERO);
}
EXPORT_SYMBOL(vcalloc);
struct anon_vma *folio_anon_vma(struct folio *folio)
{
unsigned long mapping = (unsigned long)folio->mapping;
if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
return NULL;
return (void *)(mapping - PAGE_MAPPING_ANON);
}
/**
* folio_mapping - Find the mapping where this folio is stored.
* @folio: The folio.
*
* For folios which are in the page cache, return the mapping that this
* page belongs to. Folios in the swap cache return the swap mapping
* this page is stored in (which is different from the mapping for the
* swap file or swap device where the data is stored).
*
* You can call this for folios which aren't in the swap cache or page
* cache and it will return NULL.
*/
struct address_space *folio_mapping(struct folio *folio)
{
struct address_space *mapping;
/* This happens if someone calls flush_dcache_page on slab page */
if (unlikely(folio_test_slab(folio)))
return NULL;
if (unlikely(folio_test_swapcache(folio))) return swap_address_space(folio->swap); mapping = folio->mapping; if ((unsigned long)mapping & PAGE_MAPPING_FLAGS)
return NULL;
return mapping;
}
EXPORT_SYMBOL(folio_mapping);
/**
* folio_copy - Copy the contents of one folio to another.
* @dst: Folio to copy to.
* @src: Folio to copy from.
*
* The bytes in the folio represented by @src are copied to @dst.
* Assumes the caller has validated that @dst is at least as large as @src.
* Can be called in atomic context for order-0 folios, but if the folio is
* larger, it may sleep.
*/
void folio_copy(struct folio *dst, struct folio *src)
{
long i = 0;
long nr = folio_nr_pages(src);
for (;;) {
copy_highpage(folio_page(dst, i), folio_page(src, i));
if (++i == nr)
break;
cond_resched();
}
}
EXPORT_SYMBOL(folio_copy);
int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
int sysctl_overcommit_ratio __read_mostly = 50;
unsigned long sysctl_overcommit_kbytes __read_mostly;
int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
int overcommit_ratio_handler(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
int ret;
ret = proc_dointvec(table, write, buffer, lenp, ppos);
if (ret == 0 && write)
sysctl_overcommit_kbytes = 0;
return ret;
}
static void sync_overcommit_as(struct work_struct *dummy)
{
percpu_counter_sync(&vm_committed_as);
}
int overcommit_policy_handler(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
struct ctl_table t;
int new_policy = -1;
int ret;
/*
* The deviation of sync_overcommit_as could be big with loose policy
* like OVERCOMMIT_ALWAYS/OVERCOMMIT_GUESS. When changing policy to
* strict OVERCOMMIT_NEVER, we need to reduce the deviation to comply
* with the strict "NEVER", and to avoid possible race condition (even
* though user usually won't too frequently do the switching to policy
* OVERCOMMIT_NEVER), the switch is done in the following order:
* 1. changing the batch
* 2. sync percpu count on each CPU
* 3. switch the policy
*/
if (write) {
t = *table;
t.data = &new_policy;
ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
if (ret || new_policy == -1)
return ret;
mm_compute_batch(new_policy);
if (new_policy == OVERCOMMIT_NEVER)
schedule_on_each_cpu(sync_overcommit_as);
sysctl_overcommit_memory = new_policy;
} else {
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
}
return ret;
}
int overcommit_kbytes_handler(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
int ret;
ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
if (ret == 0 && write)
sysctl_overcommit_ratio = 0;
return ret;
}
/*
* Committed memory limit enforced when OVERCOMMIT_NEVER policy is used
*/
unsigned long vm_commit_limit(void)
{
unsigned long allowed;
if (sysctl_overcommit_kbytes) allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10);
else
allowed = ((totalram_pages() - hugetlb_total_pages())
* sysctl_overcommit_ratio / 100);
allowed += total_swap_pages;
return allowed;
}
/*
* Make sure vm_committed_as in one cacheline and not cacheline shared with
* other variables. It can be updated by several CPUs frequently.
*/
struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
/*
* The global memory commitment made in the system can be a metric
* that can be used to drive ballooning decisions when Linux is hosted
* as a guest. On Hyper-V, the host implements a policy engine for dynamically
* balancing memory across competing virtual machines that are hosted.
* Several metrics drive this policy engine including the guest reported
* memory commitment.
*
* The time cost of this is very low for small platforms, and for big
* platform like a 2S/36C/72T Skylake server, in worst case where
* vm_committed_as's spinlock is under severe contention, the time cost
* could be about 30~40 microseconds.
*/
unsigned long vm_memory_committed(void)
{
return percpu_counter_sum_positive(&vm_committed_as);
}
EXPORT_SYMBOL_GPL(vm_memory_committed);
/*
* Check that a process has enough memory to allocate a new virtual
* mapping. 0 means there is enough memory for the allocation to
* succeed and -ENOMEM implies there is not.
*
* We currently support three overcommit policies, which are set via the
* vm.overcommit_memory sysctl. See Documentation/mm/overcommit-accounting.rst
*
* Strict overcommit modes added 2002 Feb 26 by Alan Cox.
* Additional code 2002 Jul 20 by Robert Love.
*
* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
*
* Note this is a helper function intended to be used by LSMs which
* wish to use this logic.
*/
int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
{
long allowed;
unsigned long bytes_failed;
vm_acct_memory(pages);
/*
* Sometimes we want to use more memory than we have
*/
if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
return 0; if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { if (pages > totalram_pages() + total_swap_pages)
goto error;
return 0;
}
allowed = vm_commit_limit();
/*
* Reserve some for root
*/
if (!cap_sys_admin) allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
/*
* Don't let a single process grow so big a user can't recover
*/
if (mm) {
long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
allowed -= min_t(long, mm->total_vm / 32, reserve);
}
if (percpu_counter_read_positive(&vm_committed_as) < allowed)
return 0;
error:
bytes_failed = pages << PAGE_SHIFT;
pr_warn_ratelimited("%s: pid: %d, comm: %s, bytes: %lu not enough memory for the allocation\n",
__func__, current->pid, current->comm, bytes_failed);
vm_unacct_memory(pages);
return -ENOMEM;
}
/**
* get_cmdline() - copy the cmdline value to a buffer.
* @task: the task whose cmdline value to copy.
* @buffer: the buffer to copy to.
* @buflen: the length of the buffer. Larger cmdline values are truncated
* to this length.
*
* Return: the size of the cmdline field copied. Note that the copy does
* not guarantee an ending NULL byte.
*/
int get_cmdline(struct task_struct *task, char *buffer, int buflen)
{
int res = 0;
unsigned int len;
struct mm_struct *mm = get_task_mm(task);
unsigned long arg_start, arg_end, env_start, env_end;
if (!mm)
goto out;
if (!mm->arg_end)
goto out_mm; /* Shh! No looking before we're done */
spin_lock(&mm->arg_lock);
arg_start = mm->arg_start;
arg_end = mm->arg_end;
env_start = mm->env_start;
env_end = mm->env_end;
spin_unlock(&mm->arg_lock);
len = arg_end - arg_start;
if (len > buflen)
len = buflen;
res = access_process_vm(task, arg_start, buffer, len, FOLL_FORCE);
/*
* If the nul at the end of args has been overwritten, then
* assume application is using setproctitle(3).
*/
if (res > 0 && buffer[res-1] != '\0' && len < buflen) {
len = strnlen(buffer, res);
if (len < res) {
res = len;
} else {
len = env_end - env_start;
if (len > buflen - res)
len = buflen - res;
res += access_process_vm(task, env_start,
buffer+res, len,
FOLL_FORCE);
res = strnlen(buffer, res);
}
}
out_mm:
mmput(mm);
out:
return res;
}
int __weak memcmp_pages(struct page *page1, struct page *page2)
{
char *addr1, *addr2;
int ret;
addr1 = kmap_local_page(page1);
addr2 = kmap_local_page(page2);
ret = memcmp(addr1, addr2, PAGE_SIZE);
kunmap_local(addr2);
kunmap_local(addr1);
return ret;
}
#ifdef CONFIG_PRINTK
/**
* mem_dump_obj - Print available provenance information
* @object: object for which to find provenance information.
*
* This function uses pr_cont(), so that the caller is expected to have
* printed out whatever preamble is appropriate. The provenance information
* depends on the type of object and on how much debugging is enabled.
* For example, for a slab-cache object, the slab name is printed, and,
* if available, the return address and stack trace from the allocation
* and last free path of that object.
*/
void mem_dump_obj(void *object)
{
const char *type;
if (kmem_dump_obj(object))
return;
if (vmalloc_dump_obj(object))
return;
if (is_vmalloc_addr(object))
type = "vmalloc memory";
else if (virt_addr_valid(object))
type = "non-slab/vmalloc memory";
else if (object == NULL)
type = "NULL pointer";
else if (object == ZERO_SIZE_PTR)
type = "zero-size pointer";
else
type = "non-paged memory";
pr_cont(" %s\n", type);
}
EXPORT_SYMBOL_GPL(mem_dump_obj);
#endif
/*
* A driver might set a page logically offline -- PageOffline() -- and
* turn the page inaccessible in the hypervisor; after that, access to page
* content can be fatal.
*
* Some special PFN walkers -- i.e., /proc/kcore -- read content of random
* pages after checking PageOffline(); however, these PFN walkers can race
* with drivers that set PageOffline().
*
* page_offline_freeze()/page_offline_thaw() allows for a subsystem to
* synchronize with such drivers, achieving that a page cannot be set
* PageOffline() while frozen.
*
* page_offline_begin()/page_offline_end() is used by drivers that care about
* such races when setting a page PageOffline().
*/
static DECLARE_RWSEM(page_offline_rwsem);
void page_offline_freeze(void)
{
down_read(&page_offline_rwsem);
}
void page_offline_thaw(void)
{
up_read(&page_offline_rwsem);
}
void page_offline_begin(void)
{
down_write(&page_offline_rwsem);
}
EXPORT_SYMBOL(page_offline_begin);
void page_offline_end(void)
{
up_write(&page_offline_rwsem);
}
EXPORT_SYMBOL(page_offline_end);
#ifndef flush_dcache_folio
void flush_dcache_folio(struct folio *folio)
{
long i, nr = folio_nr_pages(folio);
for (i = 0; i < nr; i++)
flush_dcache_page(folio_page(folio, i));
}
EXPORT_SYMBOL(flush_dcache_folio);
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __X86_KERNEL_FPU_CONTEXT_H
#define __X86_KERNEL_FPU_CONTEXT_H
#include <asm/fpu/xstate.h>
#include <asm/trace/fpu.h>
/* Functions related to FPU context tracking */
/*
* The in-register FPU state for an FPU context on a CPU is assumed to be
* valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx
* matches the FPU.
*
* If the FPU register state is valid, the kernel can skip restoring the
* FPU state from memory.
*
* Any code that clobbers the FPU registers or updates the in-memory
* FPU state for a task MUST let the rest of the kernel know that the
* FPU registers are no longer valid for this task.
*
* Invalidate a resource you control: CPU if using the CPU for something else
* (with preemption disabled), FPU for the current task, or a task that
* is prevented from running by the current task.
*/
static inline void __cpu_invalidate_fpregs_state(void)
{
__this_cpu_write(fpu_fpregs_owner_ctx, NULL);
}
static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
{
fpu->last_cpu = -1;
}
static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu)
{
return fpu == this_cpu_read(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
}
static inline void fpregs_deactivate(struct fpu *fpu)
{
__this_cpu_write(fpu_fpregs_owner_ctx, NULL);
trace_x86_fpu_regs_deactivated(fpu);
}
static inline void fpregs_activate(struct fpu *fpu)
{
__this_cpu_write(fpu_fpregs_owner_ctx, fpu);
trace_x86_fpu_regs_activated(fpu);
}
/* Internal helper for switch_fpu_return() and signal frame setup */
static inline void fpregs_restore_userregs(void)
{
struct fpu *fpu = ¤t->thread.fpu;
int cpu = smp_processor_id();
if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_USER_WORKER)))
return;
if (!fpregs_state_valid(fpu, cpu)) {
/*
* This restores _all_ xstate which has not been
* established yet.
*
* If PKRU is enabled, then the PKRU value is already
* correct because it was either set in switch_to() or in
* flush_thread(). So it is excluded because it might be
* not up to date in current->thread.fpu.xsave state.
*
* XFD state is handled in restore_fpregs_from_fpstate().
*/
restore_fpregs_from_fpstate(fpu->fpstate, XFEATURE_MASK_FPSTATE); fpregs_activate(fpu); fpu->last_cpu = cpu;
}
clear_thread_flag(TIF_NEED_FPU_LOAD);
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_SECTIONS_H_
#define _ASM_GENERIC_SECTIONS_H_
/* References to section boundaries */
#include <linux/compiler.h>
#include <linux/types.h>
/*
* Usage guidelines:
* _text, _data: architecture specific, don't use them in arch-independent code
* [_stext, _etext]: contains .text.* sections, may also contain .rodata.*
* and/or .init.* sections
* [_sdata, _edata]: contains .data.* sections, may also contain .rodata.*
* and/or .init.* sections.
* [__start_rodata, __end_rodata]: contains .rodata.* sections
* [__start_ro_after_init, __end_ro_after_init]:
* contains .data..ro_after_init section
* [__init_begin, __init_end]: contains .init.* sections, but .init.text.*
* may be out of this range on some architectures.
* [_sinittext, _einittext]: contains .init.text.* sections
* [__bss_start, __bss_stop]: contains BSS sections
*
* Following global variables are optional and may be unavailable on some
* architectures and/or kernel configurations.
* _text, _data
* __kprobes_text_start, __kprobes_text_end
* __entry_text_start, __entry_text_end
* __ctors_start, __ctors_end
* __irqentry_text_start, __irqentry_text_end
* __softirqentry_text_start, __softirqentry_text_end
* __start_opd, __end_opd
*/
extern char _text[], _stext[], _etext[];
extern char _data[], _sdata[], _edata[];
extern char __bss_start[], __bss_stop[];
extern char __init_begin[], __init_end[];
extern char _sinittext[], _einittext[];
extern char __start_ro_after_init[], __end_ro_after_init[];
extern char _end[];
extern char __per_cpu_load[], __per_cpu_start[], __per_cpu_end[];
extern char __kprobes_text_start[], __kprobes_text_end[];
extern char __entry_text_start[], __entry_text_end[];
extern char __start_rodata[], __end_rodata[];
extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __softirqentry_text_start[], __softirqentry_text_end[];
extern char __start_once[], __end_once[];
/* Start and end of .ctors section - used for constructor calls. */
extern char __ctors_start[], __ctors_end[];
/* Start and end of .opd section - used for function descriptors. */
extern char __start_opd[], __end_opd[];
/* Start and end of instrumentation protected text section */
extern char __noinstr_text_start[], __noinstr_text_end[];
extern __visible const void __nosave_begin, __nosave_end;
/* Function descriptor handling (if any). Override in asm/sections.h */
#ifdef CONFIG_HAVE_FUNCTION_DESCRIPTORS
void *dereference_function_descriptor(void *ptr);
void *dereference_kernel_function_descriptor(void *ptr);
#else
#define dereference_function_descriptor(p) ((void *)(p))
#define dereference_kernel_function_descriptor(p) ((void *)(p))
/* An address is simply the address of the function. */
typedef struct {
unsigned long addr;
} func_desc_t;
#endif
static inline bool have_function_descriptors(void)
{
return IS_ENABLED(CONFIG_HAVE_FUNCTION_DESCRIPTORS);
}
/**
* memory_contains - checks if an object is contained within a memory region
* @begin: virtual address of the beginning of the memory region
* @end: virtual address of the end of the memory region
* @virt: virtual address of the memory object
* @size: size of the memory object
*
* Returns: true if the object specified by @virt and @size is entirely
* contained within the memory region defined by @begin and @end, false
* otherwise.
*/
static inline bool memory_contains(void *begin, void *end, void *virt,
size_t size)
{
return virt >= begin && virt + size <= end;
}
/**
* memory_intersects - checks if the region occupied by an object intersects
* with another memory region
* @begin: virtual address of the beginning of the memory region
* @end: virtual address of the end of the memory region
* @virt: virtual address of the memory object
* @size: size of the memory object
*
* Returns: true if an object's memory region, specified by @virt and @size,
* intersects with the region specified by @begin and @end, false otherwise.
*/
static inline bool memory_intersects(void *begin, void *end, void *virt,
size_t size)
{
void *vend = virt + size;
if (virt < end && vend > begin)
return true;
return false;
}
/**
* init_section_contains - checks if an object is contained within the init
* section
* @virt: virtual address of the memory object
* @size: size of the memory object
*
* Returns: true if the object specified by @virt and @size is entirely
* contained within the init section, false otherwise.
*/
static inline bool init_section_contains(void *virt, size_t size)
{
return memory_contains(__init_begin, __init_end, virt, size);
}
/**
* init_section_intersects - checks if the region occupied by an object
* intersects with the init section
* @virt: virtual address of the memory object
* @size: size of the memory object
*
* Returns: true if an object's memory region, specified by @virt and @size,
* intersects with the init section, false otherwise.
*/
static inline bool init_section_intersects(void *virt, size_t size)
{
return memory_intersects(__init_begin, __init_end, virt, size);
}
/**
* is_kernel_core_data - checks if the pointer address is located in the
* .data or .bss section
*
* @addr: address to check
*
* Returns: true if the address is located in .data or .bss, false otherwise.
* Note: On some archs it may return true for core RODATA, and false
* for others. But will always be true for core RW data.
*/
static inline bool is_kernel_core_data(unsigned long addr)
{
if (addr >= (unsigned long)_sdata && addr < (unsigned long)_edata)
return true;
if (addr >= (unsigned long)__bss_start &&
addr < (unsigned long)__bss_stop)
return true;
return false;
}
/**
* is_kernel_rodata - checks if the pointer address is located in the
* .rodata section
*
* @addr: address to check
*
* Returns: true if the address is located in .rodata, false otherwise.
*/
static inline bool is_kernel_rodata(unsigned long addr)
{
return addr >= (unsigned long)__start_rodata && addr < (unsigned long)__end_rodata;
}
/**
* is_kernel_inittext - checks if the pointer address is located in the
* .init.text section
*
* @addr: address to check
*
* Returns: true if the address is located in .init.text, false otherwise.
*/
static inline bool is_kernel_inittext(unsigned long addr)
{
return addr >= (unsigned long)_sinittext && addr < (unsigned long)_einittext;
}
/**
* __is_kernel_text - checks if the pointer address is located in the
* .text section
*
* @addr: address to check
*
* Returns: true if the address is located in .text, false otherwise.
* Note: an internal helper, only check the range of _stext to _etext.
*/
static inline bool __is_kernel_text(unsigned long addr)
{
return addr >= (unsigned long)_stext &&
addr < (unsigned long)_etext;
}
/**
* __is_kernel - checks if the pointer address is located in the kernel range
*
* @addr: address to check
*
* Returns: true if the address is located in the kernel range, false otherwise.
* Note: an internal helper, check the range of _stext to _end,
* and range from __init_begin to __init_end, which can be outside
* of the _stext to _end range.
*/
static inline bool __is_kernel(unsigned long addr)
{
return ((addr >= (unsigned long)_stext &&
addr < (unsigned long)_end) ||
(addr >= (unsigned long)__init_begin &&
addr < (unsigned long)__init_end));
}
#endif /* _ASM_GENERIC_SECTIONS_H_ */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/fs/open.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/fsnotify.h>
#include <linux/module.h>
#include <linux/tty.h>
#include <linux/namei.h>
#include <linux/backing-dev.h>
#include <linux/capability.h>
#include <linux/securebits.h>
#include <linux/security.h>
#include <linux/mount.h>
#include <linux/fcntl.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/fs.h>
#include <linux/personality.h>
#include <linux/pagemap.h>
#include <linux/syscalls.h>
#include <linux/rcupdate.h>
#include <linux/audit.h>
#include <linux/falloc.h>
#include <linux/fs_struct.h>
#include <linux/dnotify.h>
#include <linux/compat.h>
#include <linux/mnt_idmapping.h>
#include <linux/filelock.h>
#include "internal.h"
int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry,
loff_t length, unsigned int time_attrs, struct file *filp)
{
int ret;
struct iattr newattrs;
/* Not pretty: "inode->i_size" shouldn't really be signed. But it is. */
if (length < 0)
return -EINVAL;
newattrs.ia_size = length;
newattrs.ia_valid = ATTR_SIZE | time_attrs;
if (filp) {
newattrs.ia_file = filp;
newattrs.ia_valid |= ATTR_FILE;
}
/* Remove suid, sgid, and file capabilities on truncate too */
ret = dentry_needs_remove_privs(idmap, dentry);
if (ret < 0)
return ret;
if (ret) newattrs.ia_valid |= ret | ATTR_FORCE; inode_lock(dentry->d_inode);
/* Note any delegations or leases have already been broken: */
ret = notify_change(idmap, dentry, &newattrs, NULL);
inode_unlock(dentry->d_inode);
return ret;
}
long vfs_truncate(const struct path *path, loff_t length)
{
struct mnt_idmap *idmap;
struct inode *inode;
long error;
inode = path->dentry->d_inode;
/* For directories it's -EISDIR, for other non-regulars - -EINVAL */
if (S_ISDIR(inode->i_mode))
return -EISDIR;
if (!S_ISREG(inode->i_mode))
return -EINVAL;
error = mnt_want_write(path->mnt);
if (error)
goto out;
idmap = mnt_idmap(path->mnt);
error = inode_permission(idmap, inode, MAY_WRITE);
if (error)
goto mnt_drop_write_and_out;
error = -EPERM;
if (IS_APPEND(inode))
goto mnt_drop_write_and_out;
error = get_write_access(inode);
if (error)
goto mnt_drop_write_and_out;
/*
* Make sure that there are no leases. get_write_access() protects
* against the truncate racing with a lease-granting setlease().
*/
error = break_lease(inode, O_WRONLY);
if (error)
goto put_write_and_out;
error = security_path_truncate(path);
if (!error)
error = do_truncate(idmap, path->dentry, length, 0, NULL);
put_write_and_out:
put_write_access(inode);
mnt_drop_write_and_out:
mnt_drop_write(path->mnt);
out:
return error;
}
EXPORT_SYMBOL_GPL(vfs_truncate);
long do_sys_truncate(const char __user *pathname, loff_t length)
{
unsigned int lookup_flags = LOOKUP_FOLLOW;
struct path path;
int error;
if (length < 0) /* sorry, but loff_t says... */
return -EINVAL;
retry:
error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path);
if (!error) {
error = vfs_truncate(&path, length);
path_put(&path);
}
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
return error;
}
SYSCALL_DEFINE2(truncate, const char __user *, path, long, length)
{
return do_sys_truncate(path, length);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(truncate, const char __user *, path, compat_off_t, length)
{
return do_sys_truncate(path, length);
}
#endif
long do_ftruncate(struct file *file, loff_t length, int small)
{
struct inode *inode;
struct dentry *dentry;
int error;
/* explicitly opened as large or we are on 64-bit box */
if (file->f_flags & O_LARGEFILE)
small = 0;
dentry = file->f_path.dentry;
inode = dentry->d_inode;
if (!S_ISREG(inode->i_mode) || !(file->f_mode & FMODE_WRITE))
return -EINVAL;
/* Cannot ftruncate over 2^31 bytes without large file support */
if (small && length > MAX_NON_LFS)
return -EINVAL;
/* Check IS_APPEND on real upper inode */
if (IS_APPEND(file_inode(file)))
return -EPERM;
sb_start_write(inode->i_sb);
error = security_file_truncate(file);
if (!error)
error = do_truncate(file_mnt_idmap(file), dentry, length,
ATTR_MTIME | ATTR_CTIME, file);
sb_end_write(inode->i_sb);
return error;
}
long do_sys_ftruncate(unsigned int fd, loff_t length, int small)
{
struct fd f;
int error;
if (length < 0)
return -EINVAL;
f = fdget(fd);
if (!f.file)
return -EBADF;
error = do_ftruncate(f.file, length, small);
fdput(f);
return error;
}
SYSCALL_DEFINE2(ftruncate, unsigned int, fd, unsigned long, length)
{
return do_sys_ftruncate(fd, length, 1);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(ftruncate, unsigned int, fd, compat_ulong_t, length)
{
return do_sys_ftruncate(fd, length, 1);
}
#endif
/* LFS versions of truncate are only needed on 32 bit machines */
#if BITS_PER_LONG == 32
SYSCALL_DEFINE2(truncate64, const char __user *, path, loff_t, length)
{
return do_sys_truncate(path, length);
}
SYSCALL_DEFINE2(ftruncate64, unsigned int, fd, loff_t, length)
{
return do_sys_ftruncate(fd, length, 0);
}
#endif /* BITS_PER_LONG == 32 */
#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_TRUNCATE64)
COMPAT_SYSCALL_DEFINE3(truncate64, const char __user *, pathname,
compat_arg_u64_dual(length))
{
return ksys_truncate(pathname, compat_arg_u64_glue(length));
}
#endif
#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_FTRUNCATE64)
COMPAT_SYSCALL_DEFINE3(ftruncate64, unsigned int, fd,
compat_arg_u64_dual(length))
{
return ksys_ftruncate(fd, compat_arg_u64_glue(length));
}
#endif
int vfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
{
struct inode *inode = file_inode(file);
long ret;
if (offset < 0 || len <= 0)
return -EINVAL;
/* Return error if mode is not supported */
if (mode & ~FALLOC_FL_SUPPORTED_MASK)
return -EOPNOTSUPP;
/* Punch hole and zero range are mutually exclusive */
if ((mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) ==
(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))
return -EOPNOTSUPP;
/* Punch hole must have keep size set */
if ((mode & FALLOC_FL_PUNCH_HOLE) &&
!(mode & FALLOC_FL_KEEP_SIZE))
return -EOPNOTSUPP;
/* Collapse range should only be used exclusively. */
if ((mode & FALLOC_FL_COLLAPSE_RANGE) && (mode & ~FALLOC_FL_COLLAPSE_RANGE))
return -EINVAL;
/* Insert range should only be used exclusively. */
if ((mode & FALLOC_FL_INSERT_RANGE) && (mode & ~FALLOC_FL_INSERT_RANGE))
return -EINVAL;
/* Unshare range should only be used with allocate mode. */
if ((mode & FALLOC_FL_UNSHARE_RANGE) && (mode & ~(FALLOC_FL_UNSHARE_RANGE | FALLOC_FL_KEEP_SIZE)))
return -EINVAL;
if (!(file->f_mode & FMODE_WRITE))
return -EBADF;
/*
* We can only allow pure fallocate on append only files
*/
if ((mode & ~FALLOC_FL_KEEP_SIZE) && IS_APPEND(inode))
return -EPERM;
if (IS_IMMUTABLE(inode))
return -EPERM;
/*
* We cannot allow any fallocate operation on an active swapfile
*/
if (IS_SWAPFILE(inode))
return -ETXTBSY;
/*
* Revalidate the write permissions, in case security policy has
* changed since the files were opened.
*/
ret = security_file_permission(file, MAY_WRITE);
if (ret)
return ret;
ret = fsnotify_file_area_perm(file, MAY_WRITE, &offset, len);
if (ret)
return ret;
if (S_ISFIFO(inode->i_mode))
return -ESPIPE;
if (S_ISDIR(inode->i_mode))
return -EISDIR;
if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
return -ENODEV;
/* Check for wrap through zero too */
if (((offset + len) > inode->i_sb->s_maxbytes) || ((offset + len) < 0))
return -EFBIG;
if (!file->f_op->fallocate)
return -EOPNOTSUPP;
file_start_write(file); ret = file->f_op->fallocate(file, mode, offset, len);
/*
* Create inotify and fanotify events.
*
* To keep the logic simple always create events if fallocate succeeds.
* This implies that events are even created if the file size remains
* unchanged, e.g. when using flag FALLOC_FL_KEEP_SIZE.
*/
if (ret == 0)
fsnotify_modify(file); file_end_write(file); return ret;
}
EXPORT_SYMBOL_GPL(vfs_fallocate);
int ksys_fallocate(int fd, int mode, loff_t offset, loff_t len)
{
struct fd f = fdget(fd);
int error = -EBADF;
if (f.file) {
error = vfs_fallocate(f.file, mode, offset, len);
fdput(f);
}
return error;
}
SYSCALL_DEFINE4(fallocate, int, fd, int, mode, loff_t, offset, loff_t, len)
{
return ksys_fallocate(fd, mode, offset, len);
}
#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_FALLOCATE)
COMPAT_SYSCALL_DEFINE6(fallocate, int, fd, int, mode, compat_arg_u64_dual(offset),
compat_arg_u64_dual(len))
{
return ksys_fallocate(fd, mode, compat_arg_u64_glue(offset),
compat_arg_u64_glue(len));
}
#endif
/*
* access() needs to use the real uid/gid, not the effective uid/gid.
* We do this by temporarily clearing all FS-related capabilities and
* switching the fsuid/fsgid around to the real ones.
*
* Creating new credentials is expensive, so we try to skip doing it,
* which we can if the result would match what we already got.
*/
static bool access_need_override_creds(int flags)
{
const struct cred *cred;
if (flags & AT_EACCESS)
return false;
cred = current_cred();
if (!uid_eq(cred->fsuid, cred->uid) ||
!gid_eq(cred->fsgid, cred->gid))
return true;
if (!issecure(SECURE_NO_SETUID_FIXUP)) {
kuid_t root_uid = make_kuid(cred->user_ns, 0);
if (!uid_eq(cred->uid, root_uid)) {
if (!cap_isclear(cred->cap_effective))
return true;
} else {
if (!cap_isidentical(cred->cap_effective,
cred->cap_permitted))
return true;
}
}
return false;
}
static const struct cred *access_override_creds(void)
{
const struct cred *old_cred;
struct cred *override_cred;
override_cred = prepare_creds();
if (!override_cred)
return NULL;
/*
* XXX access_need_override_creds performs checks in hopes of skipping
* this work. Make sure it stays in sync if making any changes in this
* routine.
*/
override_cred->fsuid = override_cred->uid;
override_cred->fsgid = override_cred->gid;
if (!issecure(SECURE_NO_SETUID_FIXUP)) {
/* Clear the capabilities if we switch to a non-root user */
kuid_t root_uid = make_kuid(override_cred->user_ns, 0);
if (!uid_eq(override_cred->uid, root_uid))
cap_clear(override_cred->cap_effective);
else
override_cred->cap_effective =
override_cred->cap_permitted;
}
/*
* The new set of credentials can *only* be used in
* task-synchronous circumstances, and does not need
* RCU freeing, unless somebody then takes a separate
* reference to it.
*
* NOTE! This is _only_ true because this credential
* is used purely for override_creds() that installs
* it as the subjective cred. Other threads will be
* accessing ->real_cred, not the subjective cred.
*
* If somebody _does_ make a copy of this (using the
* 'get_current_cred()' function), that will clear the
* non_rcu field, because now that other user may be
* expecting RCU freeing. But normal thread-synchronous
* cred accesses will keep things non-racy to avoid RCU
* freeing.
*/
override_cred->non_rcu = 1;
old_cred = override_creds(override_cred);
/* override_cred() gets its own ref */
put_cred(override_cred);
return old_cred;
}
static long do_faccessat(int dfd, const char __user *filename, int mode, int flags)
{
struct path path;
struct inode *inode;
int res;
unsigned int lookup_flags = LOOKUP_FOLLOW;
const struct cred *old_cred = NULL;
if (mode & ~S_IRWXO) /* where's F_OK, X_OK, W_OK, R_OK? */
return -EINVAL;
if (flags & ~(AT_EACCESS | AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH))
return -EINVAL;
if (flags & AT_SYMLINK_NOFOLLOW)
lookup_flags &= ~LOOKUP_FOLLOW;
if (flags & AT_EMPTY_PATH)
lookup_flags |= LOOKUP_EMPTY;
if (access_need_override_creds(flags)) {
old_cred = access_override_creds();
if (!old_cred)
return -ENOMEM;
}
retry:
res = user_path_at(dfd, filename, lookup_flags, &path);
if (res)
goto out;
inode = d_backing_inode(path.dentry);
if ((mode & MAY_EXEC) && S_ISREG(inode->i_mode)) {
/*
* MAY_EXEC on regular files is denied if the fs is mounted
* with the "noexec" flag.
*/
res = -EACCES;
if (path_noexec(&path))
goto out_path_release;
}
res = inode_permission(mnt_idmap(path.mnt), inode, mode | MAY_ACCESS);
/* SuS v2 requires we report a read only fs too */
if (res || !(mode & S_IWOTH) || special_file(inode->i_mode))
goto out_path_release;
/*
* This is a rare case where using __mnt_is_readonly()
* is OK without a mnt_want/drop_write() pair. Since
* no actual write to the fs is performed here, we do
* not need to telegraph to that to anyone.
*
* By doing this, we accept that this access is
* inherently racy and know that the fs may change
* state before we even see this result.
*/
if (__mnt_is_readonly(path.mnt))
res = -EROFS;
out_path_release:
path_put(&path);
if (retry_estale(res, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out:
if (old_cred)
revert_creds(old_cred);
return res;
}
SYSCALL_DEFINE3(faccessat, int, dfd, const char __user *, filename, int, mode)
{
return do_faccessat(dfd, filename, mode, 0);
}
SYSCALL_DEFINE4(faccessat2, int, dfd, const char __user *, filename, int, mode,
int, flags)
{
return do_faccessat(dfd, filename, mode, flags);
}
SYSCALL_DEFINE2(access, const char __user *, filename, int, mode)
{
return do_faccessat(AT_FDCWD, filename, mode, 0);
}
SYSCALL_DEFINE1(chdir, const char __user *, filename)
{
struct path path;
int error;
unsigned int lookup_flags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
retry:
error = user_path_at(AT_FDCWD, filename, lookup_flags, &path);
if (error)
goto out;
error = path_permission(&path, MAY_EXEC | MAY_CHDIR);
if (error)
goto dput_and_out;
set_fs_pwd(current->fs, &path);
dput_and_out:
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out:
return error;
}
SYSCALL_DEFINE1(fchdir, unsigned int, fd)
{
struct fd f = fdget_raw(fd);
int error;
error = -EBADF;
if (!f.file)
goto out;
error = -ENOTDIR;
if (!d_can_lookup(f.file->f_path.dentry))
goto out_putf;
error = file_permission(f.file, MAY_EXEC | MAY_CHDIR);
if (!error)
set_fs_pwd(current->fs, &f.file->f_path);
out_putf:
fdput(f);
out:
return error;
}
SYSCALL_DEFINE1(chroot, const char __user *, filename)
{
struct path path;
int error;
unsigned int lookup_flags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
retry:
error = user_path_at(AT_FDCWD, filename, lookup_flags, &path);
if (error)
goto out;
error = path_permission(&path, MAY_EXEC | MAY_CHDIR);
if (error)
goto dput_and_out;
error = -EPERM;
if (!ns_capable(current_user_ns(), CAP_SYS_CHROOT))
goto dput_and_out;
error = security_path_chroot(&path);
if (error)
goto dput_and_out;
set_fs_root(current->fs, &path);
error = 0;
dput_and_out:
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out:
return error;
}
int chmod_common(const struct path *path, umode_t mode)
{
struct inode *inode = path->dentry->d_inode;
struct inode *delegated_inode = NULL;
struct iattr newattrs;
int error;
error = mnt_want_write(path->mnt);
if (error)
return error;
retry_deleg:
inode_lock(inode);
error = security_path_chmod(path, mode);
if (error)
goto out_unlock;
newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO);
newattrs.ia_valid = ATTR_MODE | ATTR_CTIME;
error = notify_change(mnt_idmap(path->mnt), path->dentry,
&newattrs, &delegated_inode);
out_unlock:
inode_unlock(inode);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
mnt_drop_write(path->mnt);
return error;
}
int vfs_fchmod(struct file *file, umode_t mode)
{
audit_file(file);
return chmod_common(&file->f_path, mode);
}
SYSCALL_DEFINE2(fchmod, unsigned int, fd, umode_t, mode)
{
struct fd f = fdget(fd);
int err = -EBADF;
if (f.file) {
err = vfs_fchmod(f.file, mode);
fdput(f);
}
return err;
}
static int do_fchmodat(int dfd, const char __user *filename, umode_t mode,
unsigned int flags)
{
struct path path;
int error;
unsigned int lookup_flags;
if (unlikely(flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)))
return -EINVAL;
lookup_flags = (flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW;
if (flags & AT_EMPTY_PATH)
lookup_flags |= LOOKUP_EMPTY;
retry:
error = user_path_at(dfd, filename, lookup_flags, &path);
if (!error) {
error = chmod_common(&path, mode);
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
}
return error;
}
SYSCALL_DEFINE4(fchmodat2, int, dfd, const char __user *, filename,
umode_t, mode, unsigned int, flags)
{
return do_fchmodat(dfd, filename, mode, flags);
}
SYSCALL_DEFINE3(fchmodat, int, dfd, const char __user *, filename,
umode_t, mode)
{
return do_fchmodat(dfd, filename, mode, 0);
}
SYSCALL_DEFINE2(chmod, const char __user *, filename, umode_t, mode)
{
return do_fchmodat(AT_FDCWD, filename, mode, 0);
}
/*
* Check whether @kuid is valid and if so generate and set vfsuid_t in
* ia_vfsuid.
*
* Return: true if @kuid is valid, false if not.
*/
static inline bool setattr_vfsuid(struct iattr *attr, kuid_t kuid)
{
if (!uid_valid(kuid))
return false;
attr->ia_valid |= ATTR_UID;
attr->ia_vfsuid = VFSUIDT_INIT(kuid);
return true;
}
/*
* Check whether @kgid is valid and if so generate and set vfsgid_t in
* ia_vfsgid.
*
* Return: true if @kgid is valid, false if not.
*/
static inline bool setattr_vfsgid(struct iattr *attr, kgid_t kgid)
{
if (!gid_valid(kgid))
return false;
attr->ia_valid |= ATTR_GID;
attr->ia_vfsgid = VFSGIDT_INIT(kgid);
return true;
}
int chown_common(const struct path *path, uid_t user, gid_t group)
{
struct mnt_idmap *idmap;
struct user_namespace *fs_userns;
struct inode *inode = path->dentry->d_inode;
struct inode *delegated_inode = NULL;
int error;
struct iattr newattrs;
kuid_t uid;
kgid_t gid;
uid = make_kuid(current_user_ns(), user);
gid = make_kgid(current_user_ns(), group);
idmap = mnt_idmap(path->mnt);
fs_userns = i_user_ns(inode);
retry_deleg:
newattrs.ia_vfsuid = INVALID_VFSUID;
newattrs.ia_vfsgid = INVALID_VFSGID;
newattrs.ia_valid = ATTR_CTIME;
if ((user != (uid_t)-1) && !setattr_vfsuid(&newattrs, uid))
return -EINVAL;
if ((group != (gid_t)-1) && !setattr_vfsgid(&newattrs, gid))
return -EINVAL;
inode_lock(inode);
if (!S_ISDIR(inode->i_mode))
newattrs.ia_valid |= ATTR_KILL_SUID | ATTR_KILL_PRIV |
setattr_should_drop_sgid(idmap, inode);
/* Continue to send actual fs values, not the mount values. */
error = security_path_chown(
path,
from_vfsuid(idmap, fs_userns, newattrs.ia_vfsuid),
from_vfsgid(idmap, fs_userns, newattrs.ia_vfsgid));
if (!error)
error = notify_change(idmap, path->dentry, &newattrs,
&delegated_inode);
inode_unlock(inode);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
return error;
}
int do_fchownat(int dfd, const char __user *filename, uid_t user, gid_t group,
int flag)
{
struct path path;
int error = -EINVAL;
int lookup_flags;
if ((flag & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0)
goto out;
lookup_flags = (flag & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW;
if (flag & AT_EMPTY_PATH)
lookup_flags |= LOOKUP_EMPTY;
retry:
error = user_path_at(dfd, filename, lookup_flags, &path);
if (error)
goto out;
error = mnt_want_write(path.mnt);
if (error)
goto out_release;
error = chown_common(&path, user, group);
mnt_drop_write(path.mnt);
out_release:
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out:
return error;
}
SYSCALL_DEFINE5(fchownat, int, dfd, const char __user *, filename, uid_t, user,
gid_t, group, int, flag)
{
return do_fchownat(dfd, filename, user, group, flag);
}
SYSCALL_DEFINE3(chown, const char __user *, filename, uid_t, user, gid_t, group)
{
return do_fchownat(AT_FDCWD, filename, user, group, 0);
}
SYSCALL_DEFINE3(lchown, const char __user *, filename, uid_t, user, gid_t, group)
{
return do_fchownat(AT_FDCWD, filename, user, group,
AT_SYMLINK_NOFOLLOW);
}
int vfs_fchown(struct file *file, uid_t user, gid_t group)
{
int error;
error = mnt_want_write_file(file);
if (error)
return error;
audit_file(file);
error = chown_common(&file->f_path, user, group);
mnt_drop_write_file(file);
return error;
}
int ksys_fchown(unsigned int fd, uid_t user, gid_t group)
{
struct fd f = fdget(fd);
int error = -EBADF;
if (f.file) {
error = vfs_fchown(f.file, user, group);
fdput(f);
}
return error;
}
SYSCALL_DEFINE3(fchown, unsigned int, fd, uid_t, user, gid_t, group)
{
return ksys_fchown(fd, user, group);
}
static inline int file_get_write_access(struct file *f)
{
int error;
error = get_write_access(f->f_inode);
if (unlikely(error))
return error;
error = mnt_get_write_access(f->f_path.mnt);
if (unlikely(error))
goto cleanup_inode;
if (unlikely(f->f_mode & FMODE_BACKING)) { error = mnt_get_write_access(backing_file_user_path(f)->mnt);
if (unlikely(error))
goto cleanup_mnt;
}
return 0;
cleanup_mnt:
mnt_put_write_access(f->f_path.mnt);
cleanup_inode:
put_write_access(f->f_inode);
return error;
}
static int do_dentry_open(struct file *f,
struct inode *inode,
int (*open)(struct inode *, struct file *))
{
static const struct file_operations empty_fops = {};
int error;
path_get(&f->f_path);
f->f_inode = inode;
f->f_mapping = inode->i_mapping;
f->f_wb_err = filemap_sample_wb_err(f->f_mapping);
f->f_sb_err = file_sample_sb_err(f);
if (unlikely(f->f_flags & O_PATH)) {
f->f_mode = FMODE_PATH | FMODE_OPENED;
f->f_op = &empty_fops;
return 0;
}
if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) { i_readcount_inc(inode); } else if (f->f_mode & FMODE_WRITE && !special_file(inode->i_mode)) { error = file_get_write_access(f);
if (unlikely(error))
goto cleanup_file;
f->f_mode |= FMODE_WRITER;
}
/* POSIX.1-2008/SUSv4 Section XSI 2.9.7 */
if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)) f->f_mode |= FMODE_ATOMIC_POS; f->f_op = fops_get(inode->i_fop); if (WARN_ON(!f->f_op)) {
error = -ENODEV;
goto cleanup_all;
}
error = security_file_open(f);
if (error)
goto cleanup_all;
error = break_lease(file_inode(f), f->f_flags);
if (error)
goto cleanup_all;
/* normally all 3 are set; ->open() can clear them if needed */
f->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
if (!open)
open = f->f_op->open;
if (open) {
error = open(inode, f);
if (error)
goto cleanup_all;
}
f->f_mode |= FMODE_OPENED;
if ((f->f_mode & FMODE_READ) &&
likely(f->f_op->read || f->f_op->read_iter)) f->f_mode |= FMODE_CAN_READ; if ((f->f_mode & FMODE_WRITE) && likely(f->f_op->write || f->f_op->write_iter)) f->f_mode |= FMODE_CAN_WRITE; if ((f->f_mode & FMODE_LSEEK) && !f->f_op->llseek) f->f_mode &= ~FMODE_LSEEK; if (f->f_mapping->a_ops && f->f_mapping->a_ops->direct_IO) f->f_mode |= FMODE_CAN_ODIRECT; f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC); f->f_iocb_flags = iocb_flags(f);
file_ra_state_init(&f->f_ra, f->f_mapping->host->i_mapping);
if ((f->f_flags & O_DIRECT) && !(f->f_mode & FMODE_CAN_ODIRECT))
return -EINVAL;
/*
* XXX: Huge page cache doesn't support writing yet. Drop all page
* cache for this file before processing writes.
*/
if (f->f_mode & FMODE_WRITE) {
/*
* Paired with smp_mb() in collapse_file() to ensure nr_thps
* is up to date and the update to i_writecount by
* get_write_access() is visible. Ensures subsequent insertion
* of THPs into the page cache will fail.
*/
smp_mb();
if (filemap_nr_thps(inode->i_mapping)) {
struct address_space *mapping = inode->i_mapping;
filemap_invalidate_lock(inode->i_mapping);
/*
* unmap_mapping_range just need to be called once
* here, because the private pages is not need to be
* unmapped mapping (e.g. data segment of dynamic
* shared libraries here).
*/
unmap_mapping_range(mapping, 0, 0, 0);
truncate_inode_pages(mapping, 0);
filemap_invalidate_unlock(inode->i_mapping);
}
}
/*
* Once we return a file with FMODE_OPENED, __fput() will call
* fsnotify_close(), so we need fsnotify_open() here for symmetry.
*/
fsnotify_open(f);
return 0;
cleanup_all:
if (WARN_ON_ONCE(error > 0))
error = -EINVAL;
fops_put(f->f_op); put_file_access(f);
cleanup_file:
path_put(&f->f_path);
f->f_path.mnt = NULL;
f->f_path.dentry = NULL;
f->f_inode = NULL;
return error;
}
/**
* finish_open - finish opening a file
* @file: file pointer
* @dentry: pointer to dentry
* @open: open callback
*
* This can be used to finish opening a file passed to i_op->atomic_open().
*
* If the open callback is set to NULL, then the standard f_op->open()
* filesystem callback is substituted.
*
* NB: the dentry reference is _not_ consumed. If, for example, the dentry is
* the return value of d_splice_alias(), then the caller needs to perform dput()
* on it after finish_open().
*
* Returns zero on success or -errno if the open failed.
*/
int finish_open(struct file *file, struct dentry *dentry,
int (*open)(struct inode *, struct file *))
{
BUG_ON(file->f_mode & FMODE_OPENED); /* once it's opened, it's opened */
file->f_path.dentry = dentry;
return do_dentry_open(file, d_backing_inode(dentry), open);
}
EXPORT_SYMBOL(finish_open);
/**
* finish_no_open - finish ->atomic_open() without opening the file
*
* @file: file pointer
* @dentry: dentry or NULL (as returned from ->lookup())
*
* This can be used to set the result of a successful lookup in ->atomic_open().
*
* NB: unlike finish_open() this function does consume the dentry reference and
* the caller need not dput() it.
*
* Returns "0" which must be the return value of ->atomic_open() after having
* called this function.
*/
int finish_no_open(struct file *file, struct dentry *dentry)
{
file->f_path.dentry = dentry;
return 0;
}
EXPORT_SYMBOL(finish_no_open);
char *file_path(struct file *filp, char *buf, int buflen)
{
return d_path(&filp->f_path, buf, buflen);
}
EXPORT_SYMBOL(file_path);
/**
* vfs_open - open the file at the given path
* @path: path to open
* @file: newly allocated file with f_flag initialized
*/
int vfs_open(const struct path *path, struct file *file)
{
file->f_path = *path;
return do_dentry_open(file, d_backing_inode(path->dentry), NULL);
}
struct file *dentry_open(const struct path *path, int flags,
const struct cred *cred)
{
int error;
struct file *f;
/* We must always pass in a valid mount pointer. */
BUG_ON(!path->mnt);
f = alloc_empty_file(flags, cred);
if (!IS_ERR(f)) {
error = vfs_open(path, f);
if (error) {
fput(f);
f = ERR_PTR(error);
}
}
return f;
}
EXPORT_SYMBOL(dentry_open);
/**
* dentry_create - Create and open a file
* @path: path to create
* @flags: O_ flags
* @mode: mode bits for new file
* @cred: credentials to use
*
* Caller must hold the parent directory's lock, and have prepared
* a negative dentry, placed in @path->dentry, for the new file.
*
* Caller sets @path->mnt to the vfsmount of the filesystem where
* the new file is to be created. The parent directory and the
* negative dentry must reside on the same filesystem instance.
*
* On success, returns a "struct file *". Otherwise a ERR_PTR
* is returned.
*/
struct file *dentry_create(const struct path *path, int flags, umode_t mode,
const struct cred *cred)
{
struct file *f;
int error;
f = alloc_empty_file(flags, cred);
if (IS_ERR(f))
return f;
error = vfs_create(mnt_idmap(path->mnt),
d_inode(path->dentry->d_parent),
path->dentry, mode, true);
if (!error)
error = vfs_open(path, f);
if (unlikely(error)) {
fput(f);
return ERR_PTR(error);
}
return f;
}
EXPORT_SYMBOL(dentry_create);
/**
* kernel_file_open - open a file for kernel internal use
* @path: path of the file to open
* @flags: open flags
* @inode: the inode
* @cred: credentials for open
*
* Open a file for use by in-kernel consumers. The file is not accounted
* against nr_files and must not be installed into the file descriptor
* table.
*
* Return: Opened file on success, an error pointer on failure.
*/
struct file *kernel_file_open(const struct path *path, int flags,
struct inode *inode, const struct cred *cred)
{
struct file *f;
int error;
f = alloc_empty_file_noaccount(flags, cred);
if (IS_ERR(f))
return f;
f->f_path = *path;
error = do_dentry_open(f, inode, NULL);
if (error) {
fput(f);
f = ERR_PTR(error);
}
return f;
}
EXPORT_SYMBOL_GPL(kernel_file_open);
#define WILL_CREATE(flags) (flags & (O_CREAT | __O_TMPFILE))
#define O_PATH_FLAGS (O_DIRECTORY | O_NOFOLLOW | O_PATH | O_CLOEXEC)
inline struct open_how build_open_how(int flags, umode_t mode)
{
struct open_how how = {
.flags = flags & VALID_OPEN_FLAGS,
.mode = mode & S_IALLUGO,
};
/* O_PATH beats everything else. */
if (how.flags & O_PATH) how.flags &= O_PATH_FLAGS;
/* Modes should only be set for create-like flags. */
if (!WILL_CREATE(how.flags))
how.mode = 0; return how;
}
inline int build_open_flags(const struct open_how *how, struct open_flags *op)
{
u64 flags = how->flags;
u64 strip = __FMODE_NONOTIFY | O_CLOEXEC;
int lookup_flags = 0;
int acc_mode = ACC_MODE(flags);
BUILD_BUG_ON_MSG(upper_32_bits(VALID_OPEN_FLAGS),
"struct open_flags doesn't yet handle flags > 32 bits");
/*
* Strip flags that either shouldn't be set by userspace like
* FMODE_NONOTIFY or that aren't relevant in determining struct
* open_flags like O_CLOEXEC.
*/
flags &= ~strip;
/*
* Older syscalls implicitly clear all of the invalid flags or argument
* values before calling build_open_flags(), but openat2(2) checks all
* of its arguments.
*/
if (flags & ~VALID_OPEN_FLAGS)
return -EINVAL;
if (how->resolve & ~VALID_RESOLVE_FLAGS)
return -EINVAL;
/* Scoping flags are mutually exclusive. */
if ((how->resolve & RESOLVE_BENEATH) && (how->resolve & RESOLVE_IN_ROOT))
return -EINVAL;
/* Deal with the mode. */
if (WILL_CREATE(flags)) { if (how->mode & ~S_IALLUGO)
return -EINVAL;
op->mode = how->mode | S_IFREG;
} else {
if (how->mode != 0)
return -EINVAL;
op->mode = 0;
}
/*
* Block bugs where O_DIRECTORY | O_CREAT created regular files.
* Note, that blocking O_DIRECTORY | O_CREAT here also protects
* O_TMPFILE below which requires O_DIRECTORY being raised.
*/
if ((flags & (O_DIRECTORY | O_CREAT)) == (O_DIRECTORY | O_CREAT))
return -EINVAL;
/* Now handle the creative implementation of O_TMPFILE. */
if (flags & __O_TMPFILE) {
/*
* In order to ensure programs get explicit errors when trying
* to use O_TMPFILE on old kernels we enforce that O_DIRECTORY
* is raised alongside __O_TMPFILE.
*/
if (!(flags & O_DIRECTORY))
return -EINVAL;
if (!(acc_mode & MAY_WRITE))
return -EINVAL;
}
if (flags & O_PATH) {
/* O_PATH only permits certain other flags to be set. */
if (flags & ~O_PATH_FLAGS)
return -EINVAL;
acc_mode = 0;
}
/*
* O_SYNC is implemented as __O_SYNC|O_DSYNC. As many places only
* check for O_DSYNC if the need any syncing at all we enforce it's
* always set instead of having to deal with possibly weird behaviour
* for malicious applications setting only __O_SYNC.
*/
if (flags & __O_SYNC) flags |= O_DSYNC;
op->open_flag = flags;
/* O_TRUNC implies we need access checks for write permissions */
if (flags & O_TRUNC)
acc_mode |= MAY_WRITE;
/* Allow the LSM permission hook to distinguish append
access from general write access. */
if (flags & O_APPEND) acc_mode |= MAY_APPEND; op->acc_mode = acc_mode; op->intent = flags & O_PATH ? 0 : LOOKUP_OPEN;
if (flags & O_CREAT) {
op->intent |= LOOKUP_CREATE; if (flags & O_EXCL) { op->intent |= LOOKUP_EXCL;
flags |= O_NOFOLLOW;
}
}
if (flags & O_DIRECTORY)
lookup_flags |= LOOKUP_DIRECTORY;
if (!(flags & O_NOFOLLOW))
lookup_flags |= LOOKUP_FOLLOW; if (how->resolve & RESOLVE_NO_XDEV) lookup_flags |= LOOKUP_NO_XDEV; if (how->resolve & RESOLVE_NO_MAGICLINKS) lookup_flags |= LOOKUP_NO_MAGICLINKS; if (how->resolve & RESOLVE_NO_SYMLINKS) lookup_flags |= LOOKUP_NO_SYMLINKS; if (how->resolve & RESOLVE_BENEATH) lookup_flags |= LOOKUP_BENEATH; if (how->resolve & RESOLVE_IN_ROOT) lookup_flags |= LOOKUP_IN_ROOT; if (how->resolve & RESOLVE_CACHED) {
/* Don't bother even trying for create/truncate/tmpfile open */
if (flags & (O_TRUNC | O_CREAT | __O_TMPFILE))
return -EAGAIN;
lookup_flags |= LOOKUP_CACHED;
}
op->lookup_flags = lookup_flags; return 0;
}
/**
* file_open_name - open file and return file pointer
*
* @name: struct filename containing path to open
* @flags: open flags as per the open(2) second argument
* @mode: mode for the new file if O_CREAT is set, else ignored
*
* This is the helper to open a file from kernelspace if you really
* have to. But in generally you should not do this, so please move
* along, nothing to see here..
*/
struct file *file_open_name(struct filename *name, int flags, umode_t mode)
{
struct open_flags op;
struct open_how how = build_open_how(flags, mode);
int err = build_open_flags(&how, &op);
if (err)
return ERR_PTR(err);
return do_filp_open(AT_FDCWD, name, &op);
}
/**
* filp_open - open file and return file pointer
*
* @filename: path to open
* @flags: open flags as per the open(2) second argument
* @mode: mode for the new file if O_CREAT is set, else ignored
*
* This is the helper to open a file from kernelspace if you really
* have to. But in generally you should not do this, so please move
* along, nothing to see here..
*/
struct file *filp_open(const char *filename, int flags, umode_t mode)
{
struct filename *name = getname_kernel(filename);
struct file *file = ERR_CAST(name);
if (!IS_ERR(name)) {
file = file_open_name(name, flags, mode);
putname(name);
}
return file;
}
EXPORT_SYMBOL(filp_open);
struct file *file_open_root(const struct path *root,
const char *filename, int flags, umode_t mode)
{
struct open_flags op;
struct open_how how = build_open_how(flags, mode);
int err = build_open_flags(&how, &op);
if (err)
return ERR_PTR(err);
return do_file_open_root(root, filename, &op);
}
EXPORT_SYMBOL(file_open_root);
static long do_sys_openat2(int dfd, const char __user *filename,
struct open_how *how)
{
struct open_flags op;
int fd = build_open_flags(how, &op);
struct filename *tmp;
if (fd)
return fd; tmp = getname(filename);
if (IS_ERR(tmp))
return PTR_ERR(tmp); fd = get_unused_fd_flags(how->flags);
if (fd >= 0) {
struct file *f = do_filp_open(dfd, tmp, &op);
if (IS_ERR(f)) {
put_unused_fd(fd);
fd = PTR_ERR(f);
} else {
fd_install(fd, f);
}
}
putname(tmp);
return fd;
}
long do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode)
{
struct open_how how = build_open_how(flags, mode);
return do_sys_openat2(dfd, filename, &how);
}
SYSCALL_DEFINE3(open, const char __user *, filename, int, flags, umode_t, mode)
{
if (force_o_largefile())
flags |= O_LARGEFILE;
return do_sys_open(AT_FDCWD, filename, flags, mode);
}
SYSCALL_DEFINE4(openat, int, dfd, const char __user *, filename, int, flags,
umode_t, mode)
{
if (force_o_largefile())
flags |= O_LARGEFILE;
return do_sys_open(dfd, filename, flags, mode);
}
SYSCALL_DEFINE4(openat2, int, dfd, const char __user *, filename,
struct open_how __user *, how, size_t, usize)
{
int err;
struct open_how tmp;
BUILD_BUG_ON(sizeof(struct open_how) < OPEN_HOW_SIZE_VER0);
BUILD_BUG_ON(sizeof(struct open_how) != OPEN_HOW_SIZE_LATEST);
if (unlikely(usize < OPEN_HOW_SIZE_VER0))
return -EINVAL;
err = copy_struct_from_user(&tmp, sizeof(tmp), how, usize);
if (err)
return err;
audit_openat2_how(&tmp);
/* O_LARGEFILE is only allowed for non-O_PATH. */
if (!(tmp.flags & O_PATH) && force_o_largefile())
tmp.flags |= O_LARGEFILE;
return do_sys_openat2(dfd, filename, &tmp);
}
#ifdef CONFIG_COMPAT
/*
* Exactly like sys_open(), except that it doesn't set the
* O_LARGEFILE flag.
*/
COMPAT_SYSCALL_DEFINE3(open, const char __user *, filename, int, flags, umode_t, mode)
{
return do_sys_open(AT_FDCWD, filename, flags, mode);
}
/*
* Exactly like sys_openat(), except that it doesn't set the
* O_LARGEFILE flag.
*/
COMPAT_SYSCALL_DEFINE4(openat, int, dfd, const char __user *, filename, int, flags, umode_t, mode)
{
return do_sys_open(dfd, filename, flags, mode);
}
#endif
#ifndef __alpha__
/*
* For backward compatibility? Maybe this should be moved
* into arch/i386 instead?
*/
SYSCALL_DEFINE2(creat, const char __user *, pathname, umode_t, mode)
{
int flags = O_CREAT | O_WRONLY | O_TRUNC;
if (force_o_largefile())
flags |= O_LARGEFILE;
return do_sys_open(AT_FDCWD, pathname, flags, mode);
}
#endif
/*
* "id" is the POSIX thread ID. We use the
* files pointer for this..
*/
static int filp_flush(struct file *filp, fl_owner_t id)
{
int retval = 0;
if (CHECK_DATA_CORRUPTION(file_count(filp) == 0,
"VFS: Close: file count is 0 (f_op=%ps)",
filp->f_op)) {
return 0;
}
if (filp->f_op->flush) retval = filp->f_op->flush(filp, id); if (likely(!(filp->f_mode & FMODE_PATH))) { dnotify_flush(filp, id);
locks_remove_posix(filp, id);
}
return retval;
}
int filp_close(struct file *filp, fl_owner_t id)
{
int retval;
retval = filp_flush(filp, id);
fput(filp);
return retval;
}
EXPORT_SYMBOL(filp_close);
/*
* Careful here! We test whether the file pointer is NULL before
* releasing the fd. This ensures that one clone task can't release
* an fd while another clone is opening it.
*/
SYSCALL_DEFINE1(close, unsigned int, fd)
{
int retval;
struct file *file;
file = file_close_fd(fd);
if (!file)
return -EBADF;
retval = filp_flush(file, current->files);
/*
* We're returning to user space. Don't bother
* with any delayed fput() cases.
*/
__fput_sync(file);
/* can't restart close syscall because file table entry was cleared */
if (unlikely(retval == -ERESTARTSYS ||
retval == -ERESTARTNOINTR ||
retval == -ERESTARTNOHAND ||
retval == -ERESTART_RESTARTBLOCK))
retval = -EINTR;
return retval;
}
/**
* sys_close_range() - Close all file descriptors in a given range.
*
* @fd: starting file descriptor to close
* @max_fd: last file descriptor to close
* @flags: reserved for future extensions
*
* This closes a range of file descriptors. All file descriptors
* from @fd up to and including @max_fd are closed.
* Currently, errors to close a given file descriptor are ignored.
*/
SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd,
unsigned int, flags)
{
return __close_range(fd, max_fd, flags);
}
/*
* This routine simulates a hangup on the tty, to arrange that users
* are given clean terminals at login time.
*/
SYSCALL_DEFINE0(vhangup)
{
if (capable(CAP_SYS_TTY_CONFIG)) {
tty_vhangup_self();
return 0;
}
return -EPERM;
}
/*
* Called when an inode is about to be open.
* We use this to disallow opening large files on 32bit systems if
* the caller didn't specify O_LARGEFILE. On 64bit systems we force
* on this flag in sys_open.
*/
int generic_file_open(struct inode * inode, struct file * filp)
{
if (!(filp->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS)
return -EOVERFLOW;
return 0;
}
EXPORT_SYMBOL(generic_file_open);
/*
* This is used by subsystems that don't want seekable
* file descriptors. The function is not supposed to ever fail, the only
* reason it returns an 'int' and not 'void' is so that it can be plugged
* directly into file_operations structure.
*/
int nonseekable_open(struct inode *inode, struct file *filp)
{
filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);
return 0;
}
EXPORT_SYMBOL(nonseekable_open);
/*
* stream_open is used by subsystems that want stream-like file descriptors.
* Such file descriptors are not seekable and don't have notion of position
* (file.f_pos is always 0 and ppos passed to .read()/.write() is always NULL).
* Contrary to file descriptors of other regular files, .read() and .write()
* can run simultaneously.
*
* stream_open never fails and is marked to return int so that it could be
* directly used as file_operations.open .
*/
int stream_open(struct inode *inode, struct file *filp)
{
filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE | FMODE_ATOMIC_POS);
filp->f_mode |= FMODE_STREAM;
return 0;
}
EXPORT_SYMBOL(stream_open);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_BACKING_DEV_DEFS_H
#define __LINUX_BACKING_DEV_DEFS_H
#include <linux/list.h>
#include <linux/radix-tree.h>
#include <linux/rbtree.h>
#include <linux/spinlock.h>
#include <linux/percpu_counter.h>
#include <linux/percpu-refcount.h>
#include <linux/flex_proportions.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/kref.h>
#include <linux/refcount.h>
struct page;
struct device;
struct dentry;
/*
* Bits in bdi_writeback.state
*/
enum wb_state {
WB_registered, /* bdi_register() was done */
WB_writeback_running, /* Writeback is in progress */
WB_has_dirty_io, /* Dirty inodes on ->b_{dirty|io|more_io} */
WB_start_all, /* nr_pages == 0 (all) work pending */
};
enum wb_stat_item {
WB_RECLAIMABLE,
WB_WRITEBACK,
WB_DIRTIED,
WB_WRITTEN,
NR_WB_STAT_ITEMS
};
#define WB_STAT_BATCH (8*(1+ilog2(nr_cpu_ids)))
/*
* why some writeback work was initiated
*/
enum wb_reason {
WB_REASON_BACKGROUND,
WB_REASON_VMSCAN,
WB_REASON_SYNC,
WB_REASON_PERIODIC,
WB_REASON_LAPTOP_TIMER,
WB_REASON_FS_FREE_SPACE,
/*
* There is no bdi forker thread any more and works are done
* by emergency worker, however, this is TPs userland visible
* and we'll be exposing exactly the same information,
* so it has a mismatch name.
*/
WB_REASON_FORKER_THREAD,
WB_REASON_FOREIGN_FLUSH,
WB_REASON_MAX,
};
struct wb_completion {
atomic_t cnt;
wait_queue_head_t *waitq;
};
#define __WB_COMPLETION_INIT(_waitq) \
(struct wb_completion){ .cnt = ATOMIC_INIT(1), .waitq = (_waitq) }
/*
* If one wants to wait for one or more wb_writeback_works, each work's
* ->done should be set to a wb_completion defined using the following
* macro. Once all work items are issued with wb_queue_work(), the caller
* can wait for the completion of all using wb_wait_for_completion(). Work
* items which are waited upon aren't freed automatically on completion.
*/
#define WB_COMPLETION_INIT(bdi) __WB_COMPLETION_INIT(&(bdi)->wb_waitq)
#define DEFINE_WB_COMPLETION(cmpl, bdi) \
struct wb_completion cmpl = WB_COMPLETION_INIT(bdi)
/*
* Each wb (bdi_writeback) can perform writeback operations, is measured
* and throttled, independently. Without cgroup writeback, each bdi
* (bdi_writeback) is served by its embedded bdi->wb.
*
* On the default hierarchy, blkcg implicitly enables memcg. This allows
* using memcg's page ownership for attributing writeback IOs, and every
* memcg - blkcg combination can be served by its own wb by assigning a
* dedicated wb to each memcg, which enables isolation across different
* cgroups and propagation of IO back pressure down from the IO layer upto
* the tasks which are generating the dirty pages to be written back.
*
* A cgroup wb is indexed on its bdi by the ID of the associated memcg,
* refcounted with the number of inodes attached to it, and pins the memcg
* and the corresponding blkcg. As the corresponding blkcg for a memcg may
* change as blkcg is disabled and enabled higher up in the hierarchy, a wb
* is tested for blkcg after lookup and removed from index on mismatch so
* that a new wb for the combination can be created.
*
* Each bdi_writeback that is not embedded into the backing_dev_info must hold
* a reference to the parent backing_dev_info. See cgwb_create() for details.
*/
struct bdi_writeback {
struct backing_dev_info *bdi; /* our parent bdi */
unsigned long state; /* Always use atomic bitops on this */
unsigned long last_old_flush; /* last old data flush */
struct list_head b_dirty; /* dirty inodes */
struct list_head b_io; /* parked for writeback */
struct list_head b_more_io; /* parked for more writeback */
struct list_head b_dirty_time; /* time stamps are dirty */
spinlock_t list_lock; /* protects the b_* lists */
atomic_t writeback_inodes; /* number of inodes under writeback */
struct percpu_counter stat[NR_WB_STAT_ITEMS];
unsigned long bw_time_stamp; /* last time write bw is updated */
unsigned long dirtied_stamp;
unsigned long written_stamp; /* pages written at bw_time_stamp */
unsigned long write_bandwidth; /* the estimated write bandwidth */
unsigned long avg_write_bandwidth; /* further smoothed write bw, > 0 */
/*
* The base dirty throttle rate, re-calculated on every 200ms.
* All the bdi tasks' dirty rate will be curbed under it.
* @dirty_ratelimit tracks the estimated @balanced_dirty_ratelimit
* in small steps and is much more smooth/stable than the latter.
*/
unsigned long dirty_ratelimit;
unsigned long balanced_dirty_ratelimit;
struct fprop_local_percpu completions;
int dirty_exceeded;
enum wb_reason start_all_reason;
spinlock_t work_lock; /* protects work_list & dwork scheduling */
struct list_head work_list;
struct delayed_work dwork; /* work item used for writeback */
struct delayed_work bw_dwork; /* work item used for bandwidth estimate */
struct list_head bdi_node; /* anchored at bdi->wb_list */
#ifdef CONFIG_CGROUP_WRITEBACK
struct percpu_ref refcnt; /* used only for !root wb's */
struct fprop_local_percpu memcg_completions;
struct cgroup_subsys_state *memcg_css; /* the associated memcg */
struct cgroup_subsys_state *blkcg_css; /* and blkcg */
struct list_head memcg_node; /* anchored at memcg->cgwb_list */
struct list_head blkcg_node; /* anchored at blkcg->cgwb_list */
struct list_head b_attached; /* attached inodes, protected by list_lock */
struct list_head offline_node; /* anchored at offline_cgwbs */
union {
struct work_struct release_work;
struct rcu_head rcu;
};
#endif
};
struct backing_dev_info {
u64 id;
struct rb_node rb_node; /* keyed by ->id */
struct list_head bdi_list;
unsigned long ra_pages; /* max readahead in PAGE_SIZE units */
unsigned long io_pages; /* max allowed IO size */
struct kref refcnt; /* Reference counter for the structure */
unsigned int capabilities; /* Device capabilities */
unsigned int min_ratio;
unsigned int max_ratio, max_prop_frac;
/*
* Sum of avg_write_bw of wbs with dirty inodes. > 0 if there are
* any dirty wbs, which is depended upon by bdi_has_dirty().
*/
atomic_long_t tot_write_bandwidth;
/*
* Jiffies when last process was dirty throttled on this bdi. Used by
* blk-wbt.
*/
unsigned long last_bdp_sleep;
struct bdi_writeback wb; /* the root writeback info for this bdi */
struct list_head wb_list; /* list of all wbs */
#ifdef CONFIG_CGROUP_WRITEBACK
struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */
struct mutex cgwb_release_mutex; /* protect shutdown of wb structs */
struct rw_semaphore wb_switch_rwsem; /* no cgwb switch while syncing */
#endif
wait_queue_head_t wb_waitq;
struct device *dev;
char dev_name[64];
struct device *owner;
struct timer_list laptop_mode_wb_timer;
#ifdef CONFIG_DEBUG_FS
struct dentry *debug_dir;
#endif
};
struct wb_lock_cookie {
bool locked;
unsigned long flags;
};
#ifdef CONFIG_CGROUP_WRITEBACK
/**
* wb_tryget - try to increment a wb's refcount
* @wb: bdi_writeback to get
*/
static inline bool wb_tryget(struct bdi_writeback *wb)
{
if (wb != &wb->bdi->wb) return percpu_ref_tryget(&wb->refcnt);
return true;
}
/**
* wb_get - increment a wb's refcount
* @wb: bdi_writeback to get
*/
static inline void wb_get(struct bdi_writeback *wb)
{
if (wb != &wb->bdi->wb)
percpu_ref_get(&wb->refcnt);
}
/**
* wb_put - decrement a wb's refcount
* @wb: bdi_writeback to put
* @nr: number of references to put
*/
static inline void wb_put_many(struct bdi_writeback *wb, unsigned long nr)
{
if (WARN_ON_ONCE(!wb->bdi)) {
/*
* A driver bug might cause a file to be removed before bdi was
* initialized.
*/
return;
}
if (wb != &wb->bdi->wb) percpu_ref_put_many(&wb->refcnt, nr);
}
/**
* wb_put - decrement a wb's refcount
* @wb: bdi_writeback to put
*/
static inline void wb_put(struct bdi_writeback *wb)
{
wb_put_many(wb, 1);
}
/**
* wb_dying - is a wb dying?
* @wb: bdi_writeback of interest
*
* Returns whether @wb is unlinked and being drained.
*/
static inline bool wb_dying(struct bdi_writeback *wb)
{
return percpu_ref_is_dying(&wb->refcnt);
}
#else /* CONFIG_CGROUP_WRITEBACK */
static inline bool wb_tryget(struct bdi_writeback *wb)
{
return true;
}
static inline void wb_get(struct bdi_writeback *wb)
{
}
static inline void wb_put(struct bdi_writeback *wb)
{
}
static inline void wb_put_many(struct bdi_writeback *wb, unsigned long nr)
{
}
static inline bool wb_dying(struct bdi_writeback *wb)
{
return false;
}
#endif /* CONFIG_CGROUP_WRITEBACK */
#endif /* __LINUX_BACKING_DEV_DEFS_H */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Routines having to do with the 'struct sk_buff' memory handlers.
*
* Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
* Florian La Roche <rzsfl@rz.uni-sb.de>
*
* Fixes:
* Alan Cox : Fixed the worst of the load
* balancer bugs.
* Dave Platt : Interrupt stacking fix.
* Richard Kooijman : Timestamp fixes.
* Alan Cox : Changed buffer format.
* Alan Cox : destructor hook for AF_UNIX etc.
* Linus Torvalds : Better skb_clone.
* Alan Cox : Added skb_copy.
* Alan Cox : Added all the changed routines Linus
* only put in the headers
* Ray VanTassle : Fixed --skb->lock in free
* Alan Cox : skb_copy copy arp field
* Andi Kleen : slabified it.
* Robert Olsson : Removed skb_head_pool
*
* NOTE:
* The __skb_ routines should be called with interrupts
* disabled, or you better be *real* sure that the operation is atomic
* with respect to whatever list is being frobbed (e.g. via lock_sock()
* or via disabling bottom half handlers, etc).
*/
/*
* The functions in this file will not compile correctly with gcc 2.4.x
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/slab.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/sctp.h>
#include <linux/netdevice.h>
#ifdef CONFIG_NET_CLS_ACT
#include <net/pkt_sched.h>
#endif
#include <linux/string.h>
#include <linux/skbuff.h>
#include <linux/splice.h>
#include <linux/cache.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
#include <linux/scatterlist.h>
#include <linux/errqueue.h>
#include <linux/prefetch.h>
#include <linux/bitfield.h>
#include <linux/if_vlan.h>
#include <linux/mpls.h>
#include <linux/kcov.h>
#include <linux/iov_iter.h>
#include <net/protocol.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/checksum.h>
#include <net/gso.h>
#include <net/hotdata.h>
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
#include <net/mpls.h>
#include <net/mptcp.h>
#include <net/mctp.h>
#include <net/page_pool/helpers.h>
#include <net/dropreason.h>
#include <linux/uaccess.h>
#include <trace/events/skb.h>
#include <linux/highmem.h>
#include <linux/capability.h>
#include <linux/user_namespace.h>
#include <linux/indirect_call_wrapper.h>
#include <linux/textsearch.h>
#include "dev.h"
#include "sock_destructor.h"
#ifdef CONFIG_SKB_EXTENSIONS
static struct kmem_cache *skbuff_ext_cache __ro_after_init;
#endif
#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)
/* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
* This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique
* size, and we can differentiate heads from skb_small_head_cache
* vs system slabs by looking at their size (skb_end_offset()).
*/
#define SKB_SMALL_HEAD_CACHE_SIZE \
(is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \
(SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \
SKB_SMALL_HEAD_SIZE)
#define SKB_SMALL_HEAD_HEADROOM \
SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
EXPORT_SYMBOL(sysctl_max_skb_frags);
/* kcm_write_msgs() relies on casting paged frags to bio_vec to use
* iov_iter_bvec(). These static asserts ensure the cast is valid is long as the
* netmem is a page.
*/
static_assert(offsetof(struct bio_vec, bv_page) ==
offsetof(skb_frag_t, netmem));
static_assert(sizeof_field(struct bio_vec, bv_page) ==
sizeof_field(skb_frag_t, netmem));
static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len));
static_assert(sizeof_field(struct bio_vec, bv_len) ==
sizeof_field(skb_frag_t, len));
static_assert(offsetof(struct bio_vec, bv_offset) ==
offsetof(skb_frag_t, offset));
static_assert(sizeof_field(struct bio_vec, bv_offset) ==
sizeof_field(skb_frag_t, offset));
#undef FN
#define FN(reason) [SKB_DROP_REASON_##reason] = #reason,
static const char * const drop_reasons[] = {
[SKB_CONSUMED] = "CONSUMED",
DEFINE_DROP_REASON(FN, FN)
};
static const struct drop_reason_list drop_reasons_core = {
.reasons = drop_reasons,
.n_reasons = ARRAY_SIZE(drop_reasons),
};
const struct drop_reason_list __rcu *
drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = {
[SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core),
};
EXPORT_SYMBOL(drop_reasons_by_subsys);
/**
* drop_reasons_register_subsys - register another drop reason subsystem
* @subsys: the subsystem to register, must not be the core
* @list: the list of drop reasons within the subsystem, must point to
* a statically initialized list
*/
void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,
const struct drop_reason_list *list)
{
if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
"invalid subsystem %d\n", subsys))
return;
/* must point to statically allocated memory, so INIT is OK */
RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list);
}
EXPORT_SYMBOL_GPL(drop_reasons_register_subsys);
/**
* drop_reasons_unregister_subsys - unregister a drop reason subsystem
* @subsys: the subsystem to remove, must not be the core
*
* Note: This will synchronize_rcu() to ensure no users when it returns.
*/
void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys)
{
if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
"invalid subsystem %d\n", subsys))
return;
RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL);
synchronize_rcu();
}
EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys);
/**
* skb_panic - private function for out-of-line support
* @skb: buffer
* @sz: size
* @addr: address
* @msg: skb_over_panic or skb_under_panic
*
* Out-of-line support for skb_put() and skb_push().
* Called via the wrapper skb_over_panic() or skb_under_panic().
* Keep out of line to prevent kernel bloat.
* __builtin_return_address is not used because it is not always reliable.
*/
static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
const char msg[])
{
pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
msg, addr, skb->len, sz, skb->head, skb->data,
(unsigned long)skb->tail, (unsigned long)skb->end,
skb->dev ? skb->dev->name : "<NULL>");
BUG();
}
static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
{
skb_panic(skb, sz, addr, __func__);
}
static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
{
skb_panic(skb, sz, addr, __func__);
}
#define NAPI_SKB_CACHE_SIZE 64
#define NAPI_SKB_CACHE_BULK 16
#define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2)
#if PAGE_SIZE == SZ_4K
#define NAPI_HAS_SMALL_PAGE_FRAG 1
#define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc)
/* specialized page frag allocator using a single order 0 page
* and slicing it into 1K sized fragment. Constrained to systems
* with a very limited amount of 1K fragments fitting a single
* page - to avoid excessive truesize underestimation
*/
struct page_frag_1k {
void *va;
u16 offset;
bool pfmemalloc;
};
static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp)
{
struct page *page;
int offset;
offset = nc->offset - SZ_1K;
if (likely(offset >= 0))
goto use_frag;
page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
if (!page)
return NULL;
nc->va = page_address(page);
nc->pfmemalloc = page_is_pfmemalloc(page);
offset = PAGE_SIZE - SZ_1K;
page_ref_add(page, offset / SZ_1K);
use_frag:
nc->offset = offset;
return nc->va + offset;
}
#else
/* the small page is actually unused in this build; add dummy helpers
* to please the compiler and avoid later preprocessor's conditionals
*/
#define NAPI_HAS_SMALL_PAGE_FRAG 0
#define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false
struct page_frag_1k {
};
static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask)
{
return NULL;
}
#endif
struct napi_alloc_cache {
struct page_frag_cache page;
struct page_frag_1k page_small;
unsigned int skb_count;
void *skb_cache[NAPI_SKB_CACHE_SIZE];
};
static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
/* Double check that napi_get_frags() allocates skbs with
* skb->head being backed by slab, not a page fragment.
* This is to make sure bug fixed in 3226b158e67c
* ("net: avoid 32 x truesize under-estimation for tiny skbs")
* does not accidentally come back.
*/
void napi_get_frags_check(struct napi_struct *napi)
{
struct sk_buff *skb;
local_bh_disable();
skb = napi_get_frags(napi);
WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag);
napi_free_frags(napi);
local_bh_enable();
}
void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
{
struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
fragsz = SKB_DATA_ALIGN(fragsz);
return __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC,
align_mask);
}
EXPORT_SYMBOL(__napi_alloc_frag_align);
void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
{
void *data;
fragsz = SKB_DATA_ALIGN(fragsz);
if (in_hardirq() || irqs_disabled()) {
struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
data = __page_frag_alloc_align(nc, fragsz, GFP_ATOMIC,
align_mask);
} else {
struct napi_alloc_cache *nc;
local_bh_disable();
nc = this_cpu_ptr(&napi_alloc_cache);
data = __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC,
align_mask);
local_bh_enable();
}
return data;
}
EXPORT_SYMBOL(__netdev_alloc_frag_align);
static struct sk_buff *napi_skb_cache_get(void)
{
struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
struct sk_buff *skb;
if (unlikely(!nc->skb_count)) {
nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC,
NAPI_SKB_CACHE_BULK,
nc->skb_cache);
if (unlikely(!nc->skb_count))
return NULL;
}
skb = nc->skb_cache[--nc->skb_count];
kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache));
return skb;
}
static inline void __finalize_skb_around(struct sk_buff *skb, void *data,
unsigned int size)
{
struct skb_shared_info *shinfo;
size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
/* Assumes caller memset cleared SKB */
skb->truesize = SKB_TRUESIZE(size);
refcount_set(&skb->users, 1);
skb->head = data;
skb->data = data;
skb_reset_tail_pointer(skb);
skb_set_end_offset(skb, size);
skb->mac_header = (typeof(skb->mac_header))~0U;
skb->transport_header = (typeof(skb->transport_header))~0U;
skb->alloc_cpu = raw_smp_processor_id();
/* make sure we initialize shinfo sequentially */
shinfo = skb_shinfo(skb);
memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
atomic_set(&shinfo->dataref, 1);
skb_set_kcov_handle(skb, kcov_common_handle());
}
static inline void *__slab_build_skb(struct sk_buff *skb, void *data,
unsigned int *size)
{
void *resized;
/* Must find the allocation size (and grow it to match). */
*size = ksize(data);
/* krealloc() will immediately return "data" when
* "ksize(data)" is requested: it is the existing upper
* bounds. As a result, GFP_ATOMIC will be ignored. Note
* that this "new" pointer needs to be passed back to the
* caller for use so the __alloc_size hinting will be
* tracked correctly.
*/
resized = krealloc(data, *size, GFP_ATOMIC);
WARN_ON_ONCE(resized != data);
return resized;
}
/* build_skb() variant which can operate on slab buffers.
* Note that this should be used sparingly as slab buffers
* cannot be combined efficiently by GRO!
*/
struct sk_buff *slab_build_skb(void *data)
{
struct sk_buff *skb;
unsigned int size;
skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
if (unlikely(!skb))
return NULL;
memset(skb, 0, offsetof(struct sk_buff, tail));
data = __slab_build_skb(skb, data, &size);
__finalize_skb_around(skb, data, size);
return skb;
}
EXPORT_SYMBOL(slab_build_skb);
/* Caller must provide SKB that is memset cleared */
static void __build_skb_around(struct sk_buff *skb, void *data,
unsigned int frag_size)
{
unsigned int size = frag_size;
/* frag_size == 0 is considered deprecated now. Callers
* using slab buffer should use slab_build_skb() instead.
*/
if (WARN_ONCE(size == 0, "Use slab_build_skb() instead")) data = __slab_build_skb(skb, data, &size); __finalize_skb_around(skb, data, size);
}
/**
* __build_skb - build a network buffer
* @data: data buffer provided by caller
* @frag_size: size of data (must not be 0)
*
* Allocate a new &sk_buff. Caller provides space holding head and
* skb_shared_info. @data must have been allocated from the page
* allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc()
* allocation is deprecated, and callers should use slab_build_skb()
* instead.)
* The return is the new skb buffer.
* On a failure the return is %NULL, and @data is not freed.
* Notes :
* Before IO, driver allocates only data buffer where NIC put incoming frame
* Driver should add room at head (NET_SKB_PAD) and
* MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
* After IO, driver calls build_skb(), to allocate sk_buff and populate it
* before giving packet to stack.
* RX rings only contains data buffers, not full skbs.
*/
struct sk_buff *__build_skb(void *data, unsigned int frag_size)
{
struct sk_buff *skb;
skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
if (unlikely(!skb))
return NULL;
memset(skb, 0, offsetof(struct sk_buff, tail));
__build_skb_around(skb, data, frag_size);
return skb;
}
/* build_skb() is wrapper over __build_skb(), that specifically
* takes care of skb->head and skb->pfmemalloc
*/
struct sk_buff *build_skb(void *data, unsigned int frag_size)
{
struct sk_buff *skb = __build_skb(data, frag_size);
if (likely(skb && frag_size)) {
skb->head_frag = 1;
skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
}
return skb;
}
EXPORT_SYMBOL(build_skb);
/**
* build_skb_around - build a network buffer around provided skb
* @skb: sk_buff provide by caller, must be memset cleared
* @data: data buffer provided by caller
* @frag_size: size of data
*/
struct sk_buff *build_skb_around(struct sk_buff *skb,
void *data, unsigned int frag_size)
{
if (unlikely(!skb))
return NULL;
__build_skb_around(skb, data, frag_size);
if (frag_size) {
skb->head_frag = 1;
skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
}
return skb;
}
EXPORT_SYMBOL(build_skb_around);
/**
* __napi_build_skb - build a network buffer
* @data: data buffer provided by caller
* @frag_size: size of data
*
* Version of __build_skb() that uses NAPI percpu caches to obtain
* skbuff_head instead of inplace allocation.
*
* Returns a new &sk_buff on success, %NULL on allocation failure.
*/
static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size)
{
struct sk_buff *skb;
skb = napi_skb_cache_get();
if (unlikely(!skb))
return NULL;
memset(skb, 0, offsetof(struct sk_buff, tail));
__build_skb_around(skb, data, frag_size);
return skb;
}
/**
* napi_build_skb - build a network buffer
* @data: data buffer provided by caller
* @frag_size: size of data
*
* Version of __napi_build_skb() that takes care of skb->head_frag
* and skb->pfmemalloc when the data is a page or page fragment.
*
* Returns a new &sk_buff on success, %NULL on allocation failure.
*/
struct sk_buff *napi_build_skb(void *data, unsigned int frag_size)
{
struct sk_buff *skb = __napi_build_skb(data, frag_size);
if (likely(skb) && frag_size) {
skb->head_frag = 1;
skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
}
return skb;
}
EXPORT_SYMBOL(napi_build_skb);
/*
* kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
* the caller if emergency pfmemalloc reserves are being used. If it is and
* the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
* may be used. Otherwise, the packet data may be discarded until enough
* memory is free
*/
static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
bool *pfmemalloc)
{
bool ret_pfmemalloc = false;
size_t obj_size;
void *obj;
obj_size = SKB_HEAD_ALIGN(*size);
if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
!(flags & KMALLOC_NOT_NORMAL_BITS)) { obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
node);
*size = SKB_SMALL_HEAD_CACHE_SIZE;
if (obj || !(gfp_pfmemalloc_allowed(flags)))
goto out;
/* Try again but now we are using pfmemalloc reserves */
ret_pfmemalloc = true;
obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node);
goto out;
}
obj_size = kmalloc_size_roundup(obj_size);
/* The following cast might truncate high-order bits of obj_size, this
* is harmless because kmalloc(obj_size >= 2^32) will fail anyway.
*/
*size = (unsigned int)obj_size;
/*
* Try a regular allocation, when that fails and we're not entitled
* to the reserves, fail.
*/
obj = kmalloc_node_track_caller(obj_size,
flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
node);
if (obj || !(gfp_pfmemalloc_allowed(flags)))
goto out;
/* Try again but now we are using pfmemalloc reserves */
ret_pfmemalloc = true;
obj = kmalloc_node_track_caller(obj_size, flags, node);
out:
if (pfmemalloc) *pfmemalloc = ret_pfmemalloc; return obj;
}
/* Allocate a new skbuff. We do this ourselves so we can fill in a few
* 'private' fields and also do memory statistics to find all the
* [BEEP] leaks.
*
*/
/**
* __alloc_skb - allocate a network buffer
* @size: size to allocate
* @gfp_mask: allocation mask
* @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
* instead of head cache and allocate a cloned (child) skb.
* If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
* allocations in case the data is required for writeback
* @node: numa node to allocate memory on
*
* Allocate a new &sk_buff. The returned buffer has no headroom and a
* tail room of at least size bytes. The object has a reference count
* of one. The return is the buffer. On a failure the return is %NULL.
*
* Buffers may only be allocated from interrupts using a @gfp_mask of
* %GFP_ATOMIC.
*/
struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
int flags, int node)
{
struct kmem_cache *cache;
struct sk_buff *skb;
bool pfmemalloc;
u8 *data;
cache = (flags & SKB_ALLOC_FCLONE)
? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;
if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) gfp_mask |= __GFP_MEMALLOC;
/* Get the HEAD */
if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && likely(node == NUMA_NO_NODE || node == numa_mem_id())) skb = napi_skb_cache_get();
else
skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); if (unlikely(!skb))
return NULL;
prefetchw(skb);
/* We do our best to align skb_shared_info on a separate cache
* line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
* aligned memory blocks, unless SLUB/SLAB debug is enabled.
* Both skb->head and skb_shared_info are cache line aligned.
*/
data = kmalloc_reserve(&size, gfp_mask, node, &pfmemalloc);
if (unlikely(!data))
goto nodata;
/* kmalloc_size_roundup() might give us more room than requested.
* Put skb_shared_info exactly at the end of allocated zone,
* to allow max possible filling before reallocation.
*/
prefetchw(data + SKB_WITH_OVERHEAD(size));
/*
* Only clear those fields we need to clear, not those that we will
* actually initialise below. Hence, don't put any more fields after
* the tail pointer in struct sk_buff!
*/
memset(skb, 0, offsetof(struct sk_buff, tail));
__build_skb_around(skb, data, size);
skb->pfmemalloc = pfmemalloc;
if (flags & SKB_ALLOC_FCLONE) {
struct sk_buff_fclones *fclones;
fclones = container_of(skb, struct sk_buff_fclones, skb1);
skb->fclone = SKB_FCLONE_ORIG;
refcount_set(&fclones->fclone_ref, 1);
}
return skb;
nodata:
kmem_cache_free(cache, skb);
return NULL;
}
EXPORT_SYMBOL(__alloc_skb);
/**
* __netdev_alloc_skb - allocate an skbuff for rx on a specific device
* @dev: network device to receive on
* @len: length to allocate
* @gfp_mask: get_free_pages mask, passed to alloc_skb
*
* Allocate a new &sk_buff and assign it a usage count of one. The
* buffer has NET_SKB_PAD headroom built in. Users should allocate
* the headroom they think they need without accounting for the
* built in space. The built in space is used for optimisations.
*
* %NULL is returned if there is no free memory.
*/
struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
gfp_t gfp_mask)
{
struct page_frag_cache *nc;
struct sk_buff *skb;
bool pfmemalloc;
void *data;
len += NET_SKB_PAD;
/* If requested length is either too small or too big,
* we use kmalloc() for skb->head allocation.
*/
if (len <= SKB_WITH_OVERHEAD(1024) ||
len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
(gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
if (!skb)
goto skb_fail;
goto skb_success;
}
len = SKB_HEAD_ALIGN(len);
if (sk_memalloc_socks())
gfp_mask |= __GFP_MEMALLOC; if (in_hardirq() || irqs_disabled()) { nc = this_cpu_ptr(&netdev_alloc_cache);
data = page_frag_alloc(nc, len, gfp_mask);
pfmemalloc = nc->pfmemalloc;
} else {
local_bh_disable();
nc = this_cpu_ptr(&napi_alloc_cache.page);
data = page_frag_alloc(nc, len, gfp_mask);
pfmemalloc = nc->pfmemalloc;
local_bh_enable();
}
if (unlikely(!data))
return NULL;
skb = __build_skb(data, len);
if (unlikely(!skb)) {
skb_free_frag(data);
return NULL;
}
if (pfmemalloc) skb->pfmemalloc = 1; skb->head_frag = 1;
skb_success:
skb_reserve(skb, NET_SKB_PAD); skb->dev = dev;
skb_fail:
return skb;
}
EXPORT_SYMBOL(__netdev_alloc_skb);
/**
* __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
* @napi: napi instance this buffer was allocated for
* @len: length to allocate
* @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
*
* Allocate a new sk_buff for use in NAPI receive. This buffer will
* attempt to allocate the head from a special reserved region used
* only for NAPI Rx allocation. By doing this we can save several
* CPU cycles by avoiding having to disable and re-enable IRQs.
*
* %NULL is returned if there is no free memory.
*/
struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
gfp_t gfp_mask)
{
struct napi_alloc_cache *nc;
struct sk_buff *skb;
bool pfmemalloc;
void *data;
DEBUG_NET_WARN_ON_ONCE(!in_softirq());
len += NET_SKB_PAD + NET_IP_ALIGN;
/* If requested length is either too small or too big,
* we use kmalloc() for skb->head allocation.
* When the small frag allocator is available, prefer it over kmalloc
* for small fragments
*/
if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) ||
len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
(gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
NUMA_NO_NODE);
if (!skb)
goto skb_fail;
goto skb_success;
}
nc = this_cpu_ptr(&napi_alloc_cache);
if (sk_memalloc_socks())
gfp_mask |= __GFP_MEMALLOC;
if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) {
/* we are artificially inflating the allocation size, but
* that is not as bad as it may look like, as:
* - 'len' less than GRO_MAX_HEAD makes little sense
* - On most systems, larger 'len' values lead to fragment
* size above 512 bytes
* - kmalloc would use the kmalloc-1k slab for such values
* - Builds with smaller GRO_MAX_HEAD will very likely do
* little networking, as that implies no WiFi and no
* tunnels support, and 32 bits arches.
*/
len = SZ_1K;
data = page_frag_alloc_1k(&nc->page_small, gfp_mask);
pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small);
} else {
len = SKB_HEAD_ALIGN(len);
data = page_frag_alloc(&nc->page, len, gfp_mask);
pfmemalloc = nc->page.pfmemalloc;
}
if (unlikely(!data))
return NULL;
skb = __napi_build_skb(data, len);
if (unlikely(!skb)) {
skb_free_frag(data);
return NULL;
}
if (pfmemalloc)
skb->pfmemalloc = 1;
skb->head_frag = 1;
skb_success:
skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
skb->dev = napi->dev;
skb_fail:
return skb;
}
EXPORT_SYMBOL(__napi_alloc_skb);
void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
int off, int size, unsigned int truesize)
{
DEBUG_NET_WARN_ON_ONCE(size > truesize);
skb_fill_netmem_desc(skb, i, netmem, off, size);
skb->len += size;
skb->data_len += size;
skb->truesize += truesize;
}
EXPORT_SYMBOL(skb_add_rx_frag_netmem);
void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
unsigned int truesize)
{
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
DEBUG_NET_WARN_ON_ONCE(size > truesize);
skb_frag_size_add(frag, size);
skb->len += size;
skb->data_len += size;
skb->truesize += truesize;
}
EXPORT_SYMBOL(skb_coalesce_rx_frag);
static void skb_drop_list(struct sk_buff **listp)
{
kfree_skb_list(*listp);
*listp = NULL;
}
static inline void skb_drop_fraglist(struct sk_buff *skb)
{
skb_drop_list(&skb_shinfo(skb)->frag_list);
}
static void skb_clone_fraglist(struct sk_buff *skb)
{
struct sk_buff *list;
skb_walk_frags(skb, list)
skb_get(list);
}
static bool is_pp_page(struct page *page)
{
return (page->pp_magic & ~0x3UL) == PP_SIGNATURE;
}
int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
unsigned int headroom)
{
#if IS_ENABLED(CONFIG_PAGE_POOL)
u32 size, truesize, len, max_head_size, off;
struct sk_buff *skb = *pskb, *nskb;
int err, i, head_off;
void *data;
/* XDP does not support fraglist so we need to linearize
* the skb.
*/
if (skb_has_frag_list(skb))
return -EOPNOTSUPP;
max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom);
if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
return -ENOMEM;
size = min_t(u32, skb->len, max_head_size);
truesize = SKB_HEAD_ALIGN(size) + headroom;
data = page_pool_dev_alloc_va(pool, &truesize);
if (!data)
return -ENOMEM;
nskb = napi_build_skb(data, truesize);
if (!nskb) {
page_pool_free_va(pool, data, true);
return -ENOMEM;
}
skb_reserve(nskb, headroom);
skb_copy_header(nskb, skb);
skb_mark_for_recycle(nskb);
err = skb_copy_bits(skb, 0, nskb->data, size);
if (err) {
consume_skb(nskb);
return err;
}
skb_put(nskb, size);
head_off = skb_headroom(nskb) - skb_headroom(skb);
skb_headers_offset_update(nskb, head_off);
off = size;
len = skb->len - off;
for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
struct page *page;
u32 page_off;
size = min_t(u32, len, PAGE_SIZE);
truesize = size;
page = page_pool_dev_alloc(pool, &page_off, &truesize);
if (!page) {
consume_skb(nskb);
return -ENOMEM;
}
skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
err = skb_copy_bits(skb, off, page_address(page) + page_off,
size);
if (err) {
consume_skb(nskb);
return err;
}
len -= size;
off += size;
}
consume_skb(skb);
*pskb = nskb;
return 0;
#else
return -EOPNOTSUPP;
#endif
}
EXPORT_SYMBOL(skb_pp_cow_data);
int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
struct bpf_prog *prog)
{
if (!prog->aux->xdp_has_frags)
return -EINVAL;
return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM);
}
EXPORT_SYMBOL(skb_cow_data_for_xdp);
#if IS_ENABLED(CONFIG_PAGE_POOL)
bool napi_pp_put_page(struct page *page, bool napi_safe)
{
bool allow_direct = false;
struct page_pool *pp;
page = compound_head(page);
/* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation
* in order to preserve any existing bits, such as bit 0 for the
* head page of compound page and bit 1 for pfmemalloc page, so
* mask those bits for freeing side when doing below checking,
* and page_is_pfmemalloc() is checked in __page_pool_put_page()
* to avoid recycling the pfmemalloc page.
*/
if (unlikely(!is_pp_page(page)))
return false;
pp = page->pp;
/* Allow direct recycle if we have reasons to believe that we are
* in the same context as the consumer would run, so there's
* no possible race.
* __page_pool_put_page() makes sure we're not in hardirq context
* and interrupts are enabled prior to accessing the cache.
*/
if (napi_safe || in_softirq()) {
const struct napi_struct *napi = READ_ONCE(pp->p.napi);
unsigned int cpuid = smp_processor_id();
allow_direct = napi && READ_ONCE(napi->list_owner) == cpuid;
allow_direct |= READ_ONCE(pp->cpuid) == cpuid;
}
/* Driver set this to memory recycling info. Reset it on recycle.
* This will *not* work for NIC using a split-page memory model.
* The page will be returned to the pool here regardless of the
* 'flipped' fragment being in use or not.
*/
page_pool_put_full_page(pp, page, allow_direct);
return true;
}
EXPORT_SYMBOL(napi_pp_put_page);
#endif
static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe)
{
if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
return false;
return napi_pp_put_page(virt_to_page(data), napi_safe);
}
/**
* skb_pp_frag_ref() - Increase fragment references of a page pool aware skb
* @skb: page pool aware skb
*
* Increase the fragment reference count (pp_ref_count) of a skb. This is
* intended to gain fragment references only for page pool aware skbs,
* i.e. when skb->pp_recycle is true, and not for fragments in a
* non-pp-recycling skb. It has a fallback to increase references on normal
* pages, as page pool aware skbs may also have normal page fragments.
*/
static int skb_pp_frag_ref(struct sk_buff *skb)
{
struct skb_shared_info *shinfo;
struct page *head_page;
int i;
if (!skb->pp_recycle)
return -EINVAL;
shinfo = skb_shinfo(skb);
for (i = 0; i < shinfo->nr_frags; i++) {
head_page = compound_head(skb_frag_page(&shinfo->frags[i]));
if (likely(is_pp_page(head_page)))
page_pool_ref_page(head_page);
else
page_ref_inc(head_page);
}
return 0;
}
static void skb_kfree_head(void *head, unsigned int end_offset)
{
if (end_offset == SKB_SMALL_HEAD_HEADROOM) kmem_cache_free(net_hotdata.skb_small_head_cache, head);
else
kfree(head);
}
static void skb_free_head(struct sk_buff *skb, bool napi_safe)
{
unsigned char *head = skb->head;
if (skb->head_frag) {
if (skb_pp_recycle(skb, head, napi_safe))
return;
skb_free_frag(head);
} else {
skb_kfree_head(head, skb_end_offset(skb));
}
}
static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason,
bool napi_safe)
{
struct skb_shared_info *shinfo = skb_shinfo(skb);
int i;
if (!skb_data_unref(skb, shinfo))
goto exit;
if (skb_zcopy(skb)) { bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; skb_zcopy_clear(skb, true);
if (skip_unref)
goto free_head;
}
for (i = 0; i < shinfo->nr_frags; i++) napi_frag_unref(&shinfo->frags[i], skb->pp_recycle, napi_safe);
free_head:
if (shinfo->frag_list) kfree_skb_list_reason(shinfo->frag_list, reason); skb_free_head(skb, napi_safe);
exit:
/* When we clone an SKB we copy the reycling bit. The pp_recycle
* bit is only set on the head though, so in order to avoid races
* while trying to recycle fragments on __skb_frag_unref() we need
* to make one SKB responsible for triggering the recycle path.
* So disable the recycling bit if an SKB is cloned and we have
* additional references to the fragmented part of the SKB.
* Eventually the last SKB will have the recycling bit set and it's
* dataref set to 0, which will trigger the recycling
*/
skb->pp_recycle = 0;
}
/*
* Free an skbuff by memory without cleaning the state.
*/
static void kfree_skbmem(struct sk_buff *skb)
{
struct sk_buff_fclones *fclones;
switch (skb->fclone) {
case SKB_FCLONE_UNAVAILABLE:
kmem_cache_free(net_hotdata.skbuff_cache, skb);
return;
case SKB_FCLONE_ORIG:
fclones = container_of(skb, struct sk_buff_fclones, skb1);
/* We usually free the clone (TX completion) before original skb
* This test would have no chance to be true for the clone,
* while here, branch prediction will be good.
*/
if (refcount_read(&fclones->fclone_ref) == 1)
goto fastpath;
break;
default: /* SKB_FCLONE_CLONE */
fclones = container_of(skb, struct sk_buff_fclones, skb2);
break;
}
if (!refcount_dec_and_test(&fclones->fclone_ref))
return;
fastpath:
kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
}
void skb_release_head_state(struct sk_buff *skb)
{
skb_dst_drop(skb); if (skb->destructor) { DEBUG_NET_WARN_ON_ONCE(in_hardirq()); skb->destructor(skb);
}
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
nf_conntrack_put(skb_nfct(skb));
#endif
skb_ext_put(skb);}
/* Free everything but the sk_buff shell. */
static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason,
bool napi_safe)
{
skb_release_head_state(skb);
if (likely(skb->head))
skb_release_data(skb, reason, napi_safe);
}
/**
* __kfree_skb - private function
* @skb: buffer
*
* Free an sk_buff. Release anything attached to the buffer.
* Clean the state. This is an internal helper function. Users should
* always call kfree_skb
*/
void __kfree_skb(struct sk_buff *skb)
{
skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED, false); kfree_skbmem(skb);
}
EXPORT_SYMBOL(__kfree_skb);
static __always_inline
bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
if (unlikely(!skb_unref(skb)))
return false;
DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET ||
u32_get_bits(reason,
SKB_DROP_REASON_SUBSYS_MASK) >=
SKB_DROP_REASON_SUBSYS_NUM);
if (reason == SKB_CONSUMED) trace_consume_skb(skb, __builtin_return_address(0));
else
trace_kfree_skb(skb, __builtin_return_address(0), reason);
return true;
}
/**
* kfree_skb_reason - free an sk_buff with special reason
* @skb: buffer to free
* @reason: reason why this skb is dropped
*
* Drop a reference to the buffer and free it if the usage count has
* hit zero. Meanwhile, pass the drop reason to 'kfree_skb'
* tracepoint.
*/
void __fix_address
kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
if (__kfree_skb_reason(skb, reason)) __kfree_skb(skb);
}
EXPORT_SYMBOL(kfree_skb_reason);
#define KFREE_SKB_BULK_SIZE 16
struct skb_free_array {
unsigned int skb_count;
void *skb_array[KFREE_SKB_BULK_SIZE];
};
static void kfree_skb_add_bulk(struct sk_buff *skb,
struct skb_free_array *sa,
enum skb_drop_reason reason)
{
/* if SKB is a clone, don't handle this case */
if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {
__kfree_skb(skb);
return;
}
skb_release_all(skb, reason, false);
sa->skb_array[sa->skb_count++] = skb;
if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
sa->skb_array);
sa->skb_count = 0;
}
}
void __fix_address
kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
{
struct skb_free_array sa;
sa.skb_count = 0;
while (segs) {
struct sk_buff *next = segs->next;
if (__kfree_skb_reason(segs, reason)) {
skb_poison_list(segs);
kfree_skb_add_bulk(segs, &sa, reason);
}
segs = next;
}
if (sa.skb_count)
kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
}
EXPORT_SYMBOL(kfree_skb_list_reason);
/* Dump skb information and contents.
*
* Must only be called from net_ratelimit()-ed paths.
*
* Dumps whole packets if full_pkt, only headers otherwise.
*/
void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
{
struct skb_shared_info *sh = skb_shinfo(skb);
struct net_device *dev = skb->dev;
struct sock *sk = skb->sk;
struct sk_buff *list_skb;
bool has_mac, has_trans;
int headroom, tailroom;
int i, len, seg_len;
if (full_pkt)
len = skb->len;
else
len = min_t(int, skb->len, MAX_HEADER + 128);
headroom = skb_headroom(skb);
tailroom = skb_tailroom(skb);
has_mac = skb_mac_header_was_set(skb);
has_trans = skb_transport_header_was_set(skb);
printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
"mac=(%d,%d) net=(%d,%d) trans=%d\n"
"shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
"csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
"hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
level, skb->len, headroom, skb_headlen(skb), tailroom,
has_mac ? skb->mac_header : -1,
has_mac ? skb_mac_header_len(skb) : -1,
skb->network_header,
has_trans ? skb_network_header_len(skb) : -1,
has_trans ? skb->transport_header : -1,
sh->tx_flags, sh->nr_frags,
sh->gso_size, sh->gso_type, sh->gso_segs,
skb->csum, skb->ip_summed, skb->csum_complete_sw,
skb->csum_valid, skb->csum_level,
skb->hash, skb->sw_hash, skb->l4_hash,
ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
if (dev)
printk("%sdev name=%s feat=%pNF\n",
level, dev->name, &dev->features);
if (sk)
printk("%ssk family=%hu type=%u proto=%u\n",
level, sk->sk_family, sk->sk_type, sk->sk_protocol);
if (full_pkt && headroom)
print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
16, 1, skb->head, headroom, false);
seg_len = min_t(int, skb_headlen(skb), len);
if (seg_len)
print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
16, 1, skb->data, seg_len, false);
len -= seg_len;
if (full_pkt && tailroom)
print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
16, 1, skb_tail_pointer(skb), tailroom, false);
for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
u32 p_off, p_len, copied;
struct page *p;
u8 *vaddr;
skb_frag_foreach_page(frag, skb_frag_off(frag),
skb_frag_size(frag), p, p_off, p_len,
copied) {
seg_len = min_t(int, p_len, len);
vaddr = kmap_atomic(p);
print_hex_dump(level, "skb frag: ",
DUMP_PREFIX_OFFSET,
16, 1, vaddr + p_off, seg_len, false);
kunmap_atomic(vaddr);
len -= seg_len;
if (!len)
break;
}
}
if (full_pkt && skb_has_frag_list(skb)) {
printk("skb fraglist:\n");
skb_walk_frags(skb, list_skb)
skb_dump(level, list_skb, true);
}
}
EXPORT_SYMBOL(skb_dump);
/**
* skb_tx_error - report an sk_buff xmit error
* @skb: buffer that triggered an error
*
* Report xmit error if a device callback is tracking this skb.
* skb must be freed afterwards.
*/
void skb_tx_error(struct sk_buff *skb)
{
if (skb) {
skb_zcopy_downgrade_managed(skb);
skb_zcopy_clear(skb, true);
}
}
EXPORT_SYMBOL(skb_tx_error);
#ifdef CONFIG_TRACEPOINTS
/**
* consume_skb - free an skbuff
* @skb: buffer to free
*
* Drop a ref to the buffer and free it if the usage count has hit zero
* Functions identically to kfree_skb, but kfree_skb assumes that the frame
* is being dropped after a failure and notes that
*/
void consume_skb(struct sk_buff *skb)
{
if (!skb_unref(skb))
return;
trace_consume_skb(skb, __builtin_return_address(0)); __kfree_skb(skb);
}
EXPORT_SYMBOL(consume_skb);
#endif
/**
* __consume_stateless_skb - free an skbuff, assuming it is stateless
* @skb: buffer to free
*
* Alike consume_skb(), but this variant assumes that this is the last
* skb reference and all the head states have been already dropped
*/
void __consume_stateless_skb(struct sk_buff *skb)
{
trace_consume_skb(skb, __builtin_return_address(0));
skb_release_data(skb, SKB_CONSUMED, false);
kfree_skbmem(skb);
}
static void napi_skb_cache_put(struct sk_buff *skb)
{
struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
u32 i;
if (!kasan_mempool_poison_object(skb))
return;
nc->skb_cache[nc->skb_count++] = skb;
if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
kasan_mempool_unpoison_object(nc->skb_cache[i],
kmem_cache_size(net_hotdata.skbuff_cache));
kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF,
nc->skb_cache + NAPI_SKB_CACHE_HALF);
nc->skb_count = NAPI_SKB_CACHE_HALF;
}
}
void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason)
{
skb_release_all(skb, reason, true);
napi_skb_cache_put(skb);
}
void napi_skb_free_stolen_head(struct sk_buff *skb)
{
if (unlikely(skb->slow_gro)) {
nf_reset_ct(skb);
skb_dst_drop(skb);
skb_ext_put(skb);
skb_orphan(skb);
skb->slow_gro = 0;
}
napi_skb_cache_put(skb);
}
void napi_consume_skb(struct sk_buff *skb, int budget)
{
/* Zero budget indicate non-NAPI context called us, like netpoll */
if (unlikely(!budget)) {
dev_consume_skb_any(skb);
return;
}
DEBUG_NET_WARN_ON_ONCE(!in_softirq());
if (!skb_unref(skb))
return;
/* if reaching here SKB is ready to free */
trace_consume_skb(skb, __builtin_return_address(0));
/* if SKB is a clone, don't handle this case */
if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
__kfree_skb(skb);
return;
}
skb_release_all(skb, SKB_CONSUMED, !!budget);
napi_skb_cache_put(skb);
}
EXPORT_SYMBOL(napi_consume_skb);
/* Make sure a field is contained by headers group */
#define CHECK_SKB_FIELD(field) \
BUILD_BUG_ON(offsetof(struct sk_buff, field) != \
offsetof(struct sk_buff, headers.field)); \
static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
{
new->tstamp = old->tstamp;
/* We do not copy old->sk */
new->dev = old->dev;
memcpy(new->cb, old->cb, sizeof(old->cb));
skb_dst_copy(new, old); __skb_ext_copy(new, old); __nf_copy(new, old, false);
/* Note : this field could be in the headers group.
* It is not yet because we do not want to have a 16 bit hole
*/
new->queue_mapping = old->queue_mapping;
memcpy(&new->headers, &old->headers, sizeof(new->headers));
CHECK_SKB_FIELD(protocol);
CHECK_SKB_FIELD(csum);
CHECK_SKB_FIELD(hash);
CHECK_SKB_FIELD(priority);
CHECK_SKB_FIELD(skb_iif);
CHECK_SKB_FIELD(vlan_proto);
CHECK_SKB_FIELD(vlan_tci);
CHECK_SKB_FIELD(transport_header);
CHECK_SKB_FIELD(network_header);
CHECK_SKB_FIELD(mac_header);
CHECK_SKB_FIELD(inner_protocol);
CHECK_SKB_FIELD(inner_transport_header);
CHECK_SKB_FIELD(inner_network_header);
CHECK_SKB_FIELD(inner_mac_header);
CHECK_SKB_FIELD(mark);
#ifdef CONFIG_NETWORK_SECMARK
CHECK_SKB_FIELD(secmark);
#endif
#ifdef CONFIG_NET_RX_BUSY_POLL
CHECK_SKB_FIELD(napi_id);
#endif
CHECK_SKB_FIELD(alloc_cpu);
#ifdef CONFIG_XPS
CHECK_SKB_FIELD(sender_cpu);
#endif
#ifdef CONFIG_NET_SCHED
CHECK_SKB_FIELD(tc_index);
#endif
}
/*
* You should not add any new code to this function. Add it to
* __copy_skb_header above instead.
*/
static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
{
#define C(x) n->x = skb->x
n->next = n->prev = NULL;
n->sk = NULL;
__copy_skb_header(n, skb);
C(len);
C(data_len);
C(mac_len);
n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
n->cloned = 1;
n->nohdr = 0;
n->peeked = 0;
C(pfmemalloc);
C(pp_recycle);
n->destructor = NULL;
C(tail);
C(end);
C(head);
C(head_frag);
C(data);
C(truesize);
refcount_set(&n->users, 1);
atomic_inc(&(skb_shinfo(skb)->dataref));
skb->cloned = 1;
return n;
#undef C
}
/**
* alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
* @first: first sk_buff of the msg
*/
struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
{
struct sk_buff *n;
n = alloc_skb(0, GFP_ATOMIC);
if (!n)
return NULL;
n->len = first->len;
n->data_len = first->len;
n->truesize = first->truesize;
skb_shinfo(n)->frag_list = first;
__copy_skb_header(n, first);
n->destructor = NULL;
return n;
}
EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
/**
* skb_morph - morph one skb into another
* @dst: the skb to receive the contents
* @src: the skb to supply the contents
*
* This is identical to skb_clone except that the target skb is
* supplied by the user.
*
* The target skb is returned upon exit.
*/
struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
{
skb_release_all(dst, SKB_CONSUMED, false);
return __skb_clone(dst, src);
}
EXPORT_SYMBOL_GPL(skb_morph);
int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
{
unsigned long max_pg, num_pg, new_pg, old_pg, rlim;
struct user_struct *user;
if (capable(CAP_IPC_LOCK) || !size)
return 0;
rlim = rlimit(RLIMIT_MEMLOCK);
if (rlim == RLIM_INFINITY)
return 0;
num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
max_pg = rlim >> PAGE_SHIFT;
user = mmp->user ? : current_user();
old_pg = atomic_long_read(&user->locked_vm);
do {
new_pg = old_pg + num_pg;
if (new_pg > max_pg)
return -ENOBUFS;
} while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg));
if (!mmp->user) {
mmp->user = get_uid(user);
mmp->num_pg = num_pg;
} else {
mmp->num_pg += num_pg;
}
return 0;
}
EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
void mm_unaccount_pinned_pages(struct mmpin *mmp)
{
if (mmp->user) {
atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
free_uid(mmp->user);
}
}
EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
{
struct ubuf_info_msgzc *uarg;
struct sk_buff *skb;
WARN_ON_ONCE(!in_task());
skb = sock_omalloc(sk, 0, GFP_KERNEL);
if (!skb)
return NULL;
BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
uarg = (void *)skb->cb;
uarg->mmp.user = NULL;
if (mm_account_pinned_pages(&uarg->mmp, size)) {
kfree_skb(skb);
return NULL;
}
uarg->ubuf.callback = msg_zerocopy_callback;
uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
uarg->len = 1;
uarg->bytelen = size;
uarg->zerocopy = 1;
uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
refcount_set(&uarg->ubuf.refcnt, 1);
sock_hold(sk);
return &uarg->ubuf;
}
static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
{
return container_of((void *)uarg, struct sk_buff, cb);
}
struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
struct ubuf_info *uarg)
{
if (uarg) {
struct ubuf_info_msgzc *uarg_zc;
const u32 byte_limit = 1 << 19; /* limit to a few TSO */
u32 bytelen, next;
/* there might be non MSG_ZEROCOPY users */
if (uarg->callback != msg_zerocopy_callback)
return NULL;
/* realloc only when socket is locked (TCP, UDP cork),
* so uarg->len and sk_zckey access is serialized
*/
if (!sock_owned_by_user(sk)) {
WARN_ON_ONCE(1);
return NULL;
}
uarg_zc = uarg_to_msgzc(uarg);
bytelen = uarg_zc->bytelen + size;
if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
/* TCP can create new skb to attach new uarg */
if (sk->sk_type == SOCK_STREAM)
goto new_alloc;
return NULL;
}
next = (u32)atomic_read(&sk->sk_zckey);
if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
if (mm_account_pinned_pages(&uarg_zc->mmp, size))
return NULL;
uarg_zc->len++;
uarg_zc->bytelen = bytelen;
atomic_set(&sk->sk_zckey, ++next);
/* no extra ref when appending to datagram (MSG_MORE) */
if (sk->sk_type == SOCK_STREAM)
net_zcopy_get(uarg);
return uarg;
}
}
new_alloc:
return msg_zerocopy_alloc(sk, size);
}
EXPORT_SYMBOL_GPL(msg_zerocopy_realloc);
static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
{
struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
u32 old_lo, old_hi;
u64 sum_len;
old_lo = serr->ee.ee_info;
old_hi = serr->ee.ee_data;
sum_len = old_hi - old_lo + 1ULL + len;
if (sum_len >= (1ULL << 32))
return false;
if (lo != old_hi + 1)
return false;
serr->ee.ee_data += len;
return true;
}
static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
{
struct sk_buff *tail, *skb = skb_from_uarg(uarg);
struct sock_exterr_skb *serr;
struct sock *sk = skb->sk;
struct sk_buff_head *q;
unsigned long flags;
bool is_zerocopy;
u32 lo, hi;
u16 len;
mm_unaccount_pinned_pages(&uarg->mmp);
/* if !len, there was only 1 call, and it was aborted
* so do not queue a completion notification
*/
if (!uarg->len || sock_flag(sk, SOCK_DEAD))
goto release;
len = uarg->len;
lo = uarg->id;
hi = uarg->id + len - 1;
is_zerocopy = uarg->zerocopy;
serr = SKB_EXT_ERR(skb);
memset(serr, 0, sizeof(*serr));
serr->ee.ee_errno = 0;
serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
serr->ee.ee_data = hi;
serr->ee.ee_info = lo;
if (!is_zerocopy)
serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
q = &sk->sk_error_queue;
spin_lock_irqsave(&q->lock, flags);
tail = skb_peek_tail(q);
if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
!skb_zerocopy_notify_extend(tail, lo, len)) {
__skb_queue_tail(q, skb);
skb = NULL;
}
spin_unlock_irqrestore(&q->lock, flags);
sk_error_report(sk);
release:
consume_skb(skb);
sock_put(sk);
}
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
bool success)
{
struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
uarg_zc->zerocopy = uarg_zc->zerocopy & success;
if (refcount_dec_and_test(&uarg->refcnt))
__msg_zerocopy_callback(uarg_zc);
}
EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{
struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
atomic_dec(&sk->sk_zckey);
uarg_to_msgzc(uarg)->len--;
if (have_uref)
msg_zerocopy_callback(NULL, uarg, true);
}
EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
struct msghdr *msg, int len,
struct ubuf_info *uarg)
{
struct ubuf_info *orig_uarg = skb_zcopy(skb);
int err, orig_len = skb->len;
/* An skb can only point to one uarg. This edge case happens when
* TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
*/
if (orig_uarg && uarg != orig_uarg)
return -EEXIST;
err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
struct sock *save_sk = skb->sk;
/* Streams do not free skb on error. Reset to prev state. */
iov_iter_revert(&msg->msg_iter, skb->len - orig_len);
skb->sk = sk;
___pskb_trim(skb, orig_len);
skb->sk = save_sk;
return err;
}
skb_zcopy_set(skb, uarg, NULL);
return skb->len - orig_len;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
void __skb_zcopy_downgrade_managed(struct sk_buff *skb)
{
int i;
skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
skb_frag_ref(skb, i);
}
EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed);
static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
gfp_t gfp_mask)
{
if (skb_zcopy(orig)) {
if (skb_zcopy(nskb)) {
/* !gfp_mask callers are verified to !skb_zcopy(nskb) */
if (!gfp_mask) {
WARN_ON_ONCE(1);
return -ENOMEM;
}
if (skb_uarg(nskb) == skb_uarg(orig))
return 0;
if (skb_copy_ubufs(nskb, GFP_ATOMIC))
return -EIO;
}
skb_zcopy_set(nskb, skb_uarg(orig), NULL);
}
return 0;
}
/**
* skb_copy_ubufs - copy userspace skb frags buffers to kernel
* @skb: the skb to modify
* @gfp_mask: allocation priority
*
* This must be called on skb with SKBFL_ZEROCOPY_ENABLE.
* It will copy all frags into kernel and drop the reference
* to userspace pages.
*
* If this function is called from an interrupt gfp_mask() must be
* %GFP_ATOMIC.
*
* Returns 0 on success or a negative error code on failure
* to allocate kernel memory to copy to.
*/
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
{
int num_frags = skb_shinfo(skb)->nr_frags;
struct page *page, *head = NULL;
int i, order, psize, new_frags;
u32 d_off;
if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
return -EINVAL;
if (!num_frags)
goto release;
/* We might have to allocate high order pages, so compute what minimum
* page order is needed.
*/
order = 0;
while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb))
order++;
psize = (PAGE_SIZE << order);
new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order);
for (i = 0; i < new_frags; i++) {
page = alloc_pages(gfp_mask | __GFP_COMP, order);
if (!page) {
while (head) {
struct page *next = (struct page *)page_private(head);
put_page(head);
head = next;
}
return -ENOMEM;
}
set_page_private(page, (unsigned long)head);
head = page;
}
page = head;
d_off = 0;
for (i = 0; i < num_frags; i++) {
skb_frag_t *f = &skb_shinfo(skb)->frags[i];
u32 p_off, p_len, copied;
struct page *p;
u8 *vaddr;
skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
p, p_off, p_len, copied) {
u32 copy, done = 0;
vaddr = kmap_atomic(p);
while (done < p_len) {
if (d_off == psize) {
d_off = 0;
page = (struct page *)page_private(page);
}
copy = min_t(u32, psize - d_off, p_len - done);
memcpy(page_address(page) + d_off,
vaddr + p_off + done, copy);
done += copy;
d_off += copy;
}
kunmap_atomic(vaddr);
}
}
/* skb frags release userspace buffers */
for (i = 0; i < num_frags; i++)
skb_frag_unref(skb, i);
/* skb frags point to kernel buffers */
for (i = 0; i < new_frags - 1; i++) {
__skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize);
head = (struct page *)page_private(head);
}
__skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0,
d_off);
skb_shinfo(skb)->nr_frags = new_frags;
release:
skb_zcopy_clear(skb, false);
return 0;
}
EXPORT_SYMBOL_GPL(skb_copy_ubufs);
/**
* skb_clone - duplicate an sk_buff
* @skb: buffer to clone
* @gfp_mask: allocation priority
*
* Duplicate an &sk_buff. The new one is not owned by a socket. Both
* copies share the same packet data but not structure. The new
* buffer has a reference count of 1. If the allocation fails the
* function returns %NULL otherwise the new buffer is returned.
*
* If this function is called from an interrupt gfp_mask() must be
* %GFP_ATOMIC.
*/
struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
{
struct sk_buff_fclones *fclones = container_of(skb,
struct sk_buff_fclones,
skb1);
struct sk_buff *n;
if (skb_orphan_frags(skb, gfp_mask))
return NULL;
if (skb->fclone == SKB_FCLONE_ORIG && refcount_read(&fclones->fclone_ref) == 1) { n = &fclones->skb2;
refcount_set(&fclones->fclone_ref, 2);
n->fclone = SKB_FCLONE_CLONE;
} else {
if (skb_pfmemalloc(skb)) gfp_mask |= __GFP_MEMALLOC; n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
if (!n)
return NULL;
n->fclone = SKB_FCLONE_UNAVAILABLE;
}
return __skb_clone(n, skb);
}
EXPORT_SYMBOL(skb_clone);
void skb_headers_offset_update(struct sk_buff *skb, int off)
{
/* Only adjust this if it actually is csum_start rather than csum */
if (skb->ip_summed == CHECKSUM_PARTIAL) skb->csum_start += off;
/* {transport,network,mac}_header and tail are relative to skb->head */
skb->transport_header += off;
skb->network_header += off;
if (skb_mac_header_was_set(skb))
skb->mac_header += off; skb->inner_transport_header += off;
skb->inner_network_header += off;
skb->inner_mac_header += off;
}
EXPORT_SYMBOL(skb_headers_offset_update);
void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
{
__copy_skb_header(new, old);
skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
}
EXPORT_SYMBOL(skb_copy_header);
static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
{
if (skb_pfmemalloc(skb))
return SKB_ALLOC_RX;
return 0;
}
/**
* skb_copy - create private copy of an sk_buff
* @skb: buffer to copy
* @gfp_mask: allocation priority
*
* Make a copy of both an &sk_buff and its data. This is used when the
* caller wishes to modify the data and needs a private copy of the
* data to alter. Returns %NULL on failure or the pointer to the buffer
* on success. The returned buffer has a reference count of 1.
*
* As by-product this function converts non-linear &sk_buff to linear
* one, so that &sk_buff becomes completely private and caller is allowed
* to modify all the data of returned buffer. This means that this
* function is not recommended for use in circumstances when only
* header is going to be modified. Use pskb_copy() instead.
*/
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
{
struct sk_buff *n;
unsigned int size;
int headerlen;
if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
return NULL;
headerlen = skb_headroom(skb);
size = skb_end_offset(skb) + skb->data_len;
n = __alloc_skb(size, gfp_mask,
skb_alloc_rx_flag(skb), NUMA_NO_NODE);
if (!n)
return NULL;
/* Set the data pointer */
skb_reserve(n, headerlen);
/* Set the tail pointer and length */
skb_put(n, skb->len);
BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
skb_copy_header(n, skb);
return n;
}
EXPORT_SYMBOL(skb_copy);
/**
* __pskb_copy_fclone - create copy of an sk_buff with private head.
* @skb: buffer to copy
* @headroom: headroom of new skb
* @gfp_mask: allocation priority
* @fclone: if true allocate the copy of the skb from the fclone
* cache instead of the head cache; it is recommended to set this
* to true for the cases where the copy will likely be cloned
*
* Make a copy of both an &sk_buff and part of its data, located
* in header. Fragmented data remain shared. This is used when
* the caller wishes to modify only header of &sk_buff and needs
* private copy of the header to alter. Returns %NULL on failure
* or the pointer to the buffer on success.
* The returned buffer has a reference count of 1.
*/
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
gfp_t gfp_mask, bool fclone)
{
unsigned int size = skb_headlen(skb) + headroom;
int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
if (!n)
goto out;
/* Set the data pointer */
skb_reserve(n, headroom);
/* Set the tail pointer and length */
skb_put(n, skb_headlen(skb));
/* Copy the bytes */
skb_copy_from_linear_data(skb, n->data, n->len);
n->truesize += skb->data_len;
n->data_len = skb->data_len;
n->len = skb->len;
if (skb_shinfo(skb)->nr_frags) {
int i;
if (skb_orphan_frags(skb, gfp_mask) ||
skb_zerocopy_clone(n, skb, gfp_mask)) {
kfree_skb(n);
n = NULL;
goto out;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
skb_frag_ref(skb, i);
}
skb_shinfo(n)->nr_frags = i;
}
if (skb_has_frag_list(skb)) {
skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
skb_clone_fraglist(n);
}
skb_copy_header(n, skb);
out:
return n;
}
EXPORT_SYMBOL(__pskb_copy_fclone);
/**
* pskb_expand_head - reallocate header of &sk_buff
* @skb: buffer to reallocate
* @nhead: room to add at head
* @ntail: room to add at tail
* @gfp_mask: allocation priority
*
* Expands (or creates identical copy, if @nhead and @ntail are zero)
* header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
* reference count of 1. Returns zero in the case of success or error,
* if expansion failed. In the last case, &sk_buff is not changed.
*
* All the pointers pointing into skb header may change and must be
* reloaded after call to this function.
*/
int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
gfp_t gfp_mask)
{
unsigned int osize = skb_end_offset(skb);
unsigned int size = osize + nhead + ntail;
long off;
u8 *data;
int i;
BUG_ON(nhead < 0); BUG_ON(skb_shared(skb)); skb_zcopy_downgrade_managed(skb); if (skb_pfmemalloc(skb)) gfp_mask |= __GFP_MEMALLOC; data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
if (!data)
goto nodata;
size = SKB_WITH_OVERHEAD(size);
/* Copy only real data... and, alas, header. This should be
* optimized for the cases when header is void.
*/
memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
memcpy((struct skb_shared_info *)(data + size),
skb_shinfo(skb),
offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
/*
* if shinfo is shared we must drop the old head gracefully, but if it
* is not we can just drop the old head and let the existing refcount
* be since all we did is relocate the values
*/
if (skb_cloned(skb)) { if (skb_orphan_frags(skb, gfp_mask))
goto nofrags;
if (skb_zcopy(skb)) refcount_inc(&skb_uarg(skb)->refcnt); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) skb_frag_ref(skb, i); if (skb_has_frag_list(skb)) skb_clone_fraglist(skb); skb_release_data(skb, SKB_CONSUMED, false);
} else {
skb_free_head(skb, false);
}
off = (data + nhead) - skb->head;
skb->head = data;
skb->head_frag = 0;
skb->data += off;
skb_set_end_offset(skb, size);
#ifdef NET_SKBUFF_DATA_USES_OFFSET
off = nhead;
#endif
skb->tail += off;
skb_headers_offset_update(skb, nhead);
skb->cloned = 0;
skb->hdr_len = 0;
skb->nohdr = 0;
atomic_set(&skb_shinfo(skb)->dataref, 1);
skb_metadata_clear(skb);
/* It is not generally safe to change skb->truesize.
* For the moment, we really care of rx path, or
* when skb is orphaned (not attached to a socket).
*/
if (!skb->sk || skb->destructor == sock_edemux) skb->truesize += size - osize; return 0;
nofrags:
skb_kfree_head(data, size);
nodata:
return -ENOMEM;
}
EXPORT_SYMBOL(pskb_expand_head);
/* Make private copy of skb with writable head and some headroom */
struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
{
struct sk_buff *skb2;
int delta = headroom - skb_headroom(skb);
if (delta <= 0)
skb2 = pskb_copy(skb, GFP_ATOMIC);
else {
skb2 = skb_clone(skb, GFP_ATOMIC);
if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
GFP_ATOMIC)) {
kfree_skb(skb2);
skb2 = NULL;
}
}
return skb2;
}
EXPORT_SYMBOL(skb_realloc_headroom);
/* Note: We plan to rework this in linux-6.4 */
int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
{
unsigned int saved_end_offset, saved_truesize;
struct skb_shared_info *shinfo;
int res;
saved_end_offset = skb_end_offset(skb);
saved_truesize = skb->truesize;
res = pskb_expand_head(skb, 0, 0, pri);
if (res)
return res;
skb->truesize = saved_truesize;
if (likely(skb_end_offset(skb) == saved_end_offset))
return 0;
/* We can not change skb->end if the original or new value
* is SKB_SMALL_HEAD_HEADROOM, as it might break skb_kfree_head().
*/
if (saved_end_offset == SKB_SMALL_HEAD_HEADROOM ||
skb_end_offset(skb) == SKB_SMALL_HEAD_HEADROOM) {
/* We think this path should not be taken.
* Add a temporary trace to warn us just in case.
*/
pr_err_once("__skb_unclone_keeptruesize() skb_end_offset() %u -> %u\n",
saved_end_offset, skb_end_offset(skb));
WARN_ON_ONCE(1);
return 0;
}
shinfo = skb_shinfo(skb);
/* We are about to change back skb->end,
* we need to move skb_shinfo() to its new location.
*/
memmove(skb->head + saved_end_offset,
shinfo,
offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
skb_set_end_offset(skb, saved_end_offset);
return 0;
}
/**
* skb_expand_head - reallocate header of &sk_buff
* @skb: buffer to reallocate
* @headroom: needed headroom
*
* Unlike skb_realloc_headroom, this one does not allocate a new skb
* if possible; copies skb->sk to new skb as needed
* and frees original skb in case of failures.
*
* It expect increased headroom and generates warning otherwise.
*/
struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom)
{
int delta = headroom - skb_headroom(skb);
int osize = skb_end_offset(skb);
struct sock *sk = skb->sk;
if (WARN_ONCE(delta <= 0,
"%s is expecting an increase in the headroom", __func__))
return skb;
delta = SKB_DATA_ALIGN(delta);
/* pskb_expand_head() might crash, if skb is shared. */
if (skb_shared(skb) || !is_skb_wmem(skb)) {
struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
if (unlikely(!nskb))
goto fail;
if (sk)
skb_set_owner_w(nskb, sk);
consume_skb(skb);
skb = nskb;
}
if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC))
goto fail;
if (sk && is_skb_wmem(skb)) {
delta = skb_end_offset(skb) - osize;
refcount_add(delta, &sk->sk_wmem_alloc);
skb->truesize += delta;
}
return skb;
fail:
kfree_skb(skb);
return NULL;
}
EXPORT_SYMBOL(skb_expand_head);
/**
* skb_copy_expand - copy and expand sk_buff
* @skb: buffer to copy
* @newheadroom: new free bytes at head
* @newtailroom: new free bytes at tail
* @gfp_mask: allocation priority
*
* Make a copy of both an &sk_buff and its data and while doing so
* allocate additional space.
*
* This is used when the caller wishes to modify the data and needs a
* private copy of the data to alter as well as more space for new fields.
* Returns %NULL on failure or the pointer to the buffer
* on success. The returned buffer has a reference count of 1.
*
* You must pass %GFP_ATOMIC as the allocation priority if this function
* is called from an interrupt.
*/
struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
int newheadroom, int newtailroom,
gfp_t gfp_mask)
{
/*
* Allocate the copy buffer
*/
int head_copy_len, head_copy_off;
struct sk_buff *n;
int oldheadroom;
if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
return NULL;
oldheadroom = skb_headroom(skb);
n = __alloc_skb(newheadroom + skb->len + newtailroom,
gfp_mask, skb_alloc_rx_flag(skb),
NUMA_NO_NODE);
if (!n)
return NULL;
skb_reserve(n, newheadroom);
/* Set the tail pointer and length */
skb_put(n, skb->len);
head_copy_len = oldheadroom;
head_copy_off = 0;
if (newheadroom <= head_copy_len)
head_copy_len = newheadroom;
else
head_copy_off = newheadroom - head_copy_len;
/* Copy the linear header and data. */
BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
skb->len + head_copy_len));
skb_copy_header(n, skb);
skb_headers_offset_update(n, newheadroom - oldheadroom);
return n;
}
EXPORT_SYMBOL(skb_copy_expand);
/**
* __skb_pad - zero pad the tail of an skb
* @skb: buffer to pad
* @pad: space to pad
* @free_on_error: free buffer on error
*
* Ensure that a buffer is followed by a padding area that is zero
* filled. Used by network drivers which may DMA or transfer data
* beyond the buffer end onto the wire.
*
* May return error in out of memory cases. The skb is freed on error
* if @free_on_error is true.
*/
int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
{
int err;
int ntail;
/* If the skbuff is non linear tailroom is always zero.. */
if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
memset(skb->data+skb->len, 0, pad);
return 0;
}
ntail = skb->data_len + pad - (skb->end - skb->tail);
if (likely(skb_cloned(skb) || ntail > 0)) {
err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
if (unlikely(err))
goto free_skb;
}
/* FIXME: The use of this function with non-linear skb's really needs
* to be audited.
*/
err = skb_linearize(skb);
if (unlikely(err))
goto free_skb;
memset(skb->data + skb->len, 0, pad);
return 0;
free_skb:
if (free_on_error)
kfree_skb(skb);
return err;
}
EXPORT_SYMBOL(__skb_pad);
/**
* pskb_put - add data to the tail of a potentially fragmented buffer
* @skb: start of the buffer to use
* @tail: tail fragment of the buffer to use
* @len: amount of data to add
*
* This function extends the used data area of the potentially
* fragmented buffer. @tail must be the last fragment of @skb -- or
* @skb itself. If this would exceed the total buffer size the kernel
* will panic. A pointer to the first byte of the extra data is
* returned.
*/
void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
{
if (tail != skb) {
skb->data_len += len;
skb->len += len;
}
return skb_put(tail, len);
}
EXPORT_SYMBOL_GPL(pskb_put);
/**
* skb_put - add data to a buffer
* @skb: buffer to use
* @len: amount of data to add
*
* This function extends the used data area of the buffer. If this would
* exceed the total buffer size the kernel will panic. A pointer to the
* first byte of the extra data is returned.
*/
void *skb_put(struct sk_buff *skb, unsigned int len)
{
void *tmp = skb_tail_pointer(skb); SKB_LINEAR_ASSERT(skb); skb->tail += len;
skb->len += len;
if (unlikely(skb->tail > skb->end))
skb_over_panic(skb, len, __builtin_return_address(0)); return tmp;
}
EXPORT_SYMBOL(skb_put);
/**
* skb_push - add data to the start of a buffer
* @skb: buffer to use
* @len: amount of data to add
*
* This function extends the used data area of the buffer at the buffer
* start. If this would exceed the total buffer headroom the kernel will
* panic. A pointer to the first byte of the extra data is returned.
*/
void *skb_push(struct sk_buff *skb, unsigned int len)
{
skb->data -= len;
skb->len += len;
if (unlikely(skb->data < skb->head))
skb_under_panic(skb, len, __builtin_return_address(0));
return skb->data;
}
EXPORT_SYMBOL(skb_push);
/**
* skb_pull - remove data from the start of a buffer
* @skb: buffer to use
* @len: amount of data to remove
*
* This function removes data from the start of a buffer, returning
* the memory to the headroom. A pointer to the next data in the buffer
* is returned. Once the data has been pulled future pushes will overwrite
* the old data.
*/
void *skb_pull(struct sk_buff *skb, unsigned int len)
{
return skb_pull_inline(skb, len);
}
EXPORT_SYMBOL(skb_pull);
/**
* skb_pull_data - remove data from the start of a buffer returning its
* original position.
* @skb: buffer to use
* @len: amount of data to remove
*
* This function removes data from the start of a buffer, returning
* the memory to the headroom. A pointer to the original data in the buffer
* is returned after checking if there is enough data to pull. Once the
* data has been pulled future pushes will overwrite the old data.
*/
void *skb_pull_data(struct sk_buff *skb, size_t len)
{
void *data = skb->data;
if (skb->len < len)
return NULL;
skb_pull(skb, len);
return data;
}
EXPORT_SYMBOL(skb_pull_data);
/**
* skb_trim - remove end from a buffer
* @skb: buffer to alter
* @len: new length
*
* Cut the length of a buffer down by removing data from the tail. If
* the buffer is already under the length specified it is not modified.
* The skb must be linear.
*/
void skb_trim(struct sk_buff *skb, unsigned int len)
{
if (skb->len > len) __skb_trim(skb, len);
}
EXPORT_SYMBOL(skb_trim);
/* Trims skb to length len. It can change skb pointers.
*/
int ___pskb_trim(struct sk_buff *skb, unsigned int len)
{
struct sk_buff **fragp;
struct sk_buff *frag;
int offset = skb_headlen(skb);
int nfrags = skb_shinfo(skb)->nr_frags;
int i;
int err;
if (skb_cloned(skb) &&
unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
return err;
i = 0;
if (offset >= len)
goto drop_pages;
for (; i < nfrags; i++) {
int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
if (end < len) {
offset = end;
continue;
}
skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
drop_pages:
skb_shinfo(skb)->nr_frags = i;
for (; i < nfrags; i++)
skb_frag_unref(skb, i);
if (skb_has_frag_list(skb))
skb_drop_fraglist(skb);
goto done;
}
for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
fragp = &frag->next) {
int end = offset + frag->len;
if (skb_shared(frag)) {
struct sk_buff *nfrag;
nfrag = skb_clone(frag, GFP_ATOMIC);
if (unlikely(!nfrag))
return -ENOMEM;
nfrag->next = frag->next;
consume_skb(frag);
frag = nfrag;
*fragp = frag;
}
if (end < len) {
offset = end;
continue;
}
if (end > len &&
unlikely((err = pskb_trim(frag, len - offset))))
return err;
if (frag->next)
skb_drop_list(&frag->next);
break;
}
done:
if (len > skb_headlen(skb)) {
skb->data_len -= skb->len - len;
skb->len = len;
} else {
skb->len = len;
skb->data_len = 0;
skb_set_tail_pointer(skb, len);
}
if (!skb->sk || skb->destructor == sock_edemux)
skb_condense(skb);
return 0;
}
EXPORT_SYMBOL(___pskb_trim);
/* Note : use pskb_trim_rcsum() instead of calling this directly
*/
int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
{
if (skb->ip_summed == CHECKSUM_COMPLETE) {
int delta = skb->len - len;
skb->csum = csum_block_sub(skb->csum,
skb_checksum(skb, len, delta, 0),
len);
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
if (offset + sizeof(__sum16) > hdlen)
return -EINVAL;
}
return __pskb_trim(skb, len);
}
EXPORT_SYMBOL(pskb_trim_rcsum_slow);
/**
* __pskb_pull_tail - advance tail of skb header
* @skb: buffer to reallocate
* @delta: number of bytes to advance tail
*
* The function makes a sense only on a fragmented &sk_buff,
* it expands header moving its tail forward and copying necessary
* data from fragmented part.
*
* &sk_buff MUST have reference count of 1.
*
* Returns %NULL (and &sk_buff does not change) if pull failed
* or value of new tail of skb in the case of success.
*
* All the pointers pointing into skb header may change and must be
* reloaded after call to this function.
*/
/* Moves tail of skb head forward, copying data from fragmented part,
* when it is necessary.
* 1. It may fail due to malloc failure.
* 2. It may change skb pointers.
*
* It is pretty complicated. Luckily, it is called only in exceptional cases.
*/
void *__pskb_pull_tail(struct sk_buff *skb, int delta)
{
/* If skb has not enough free space at tail, get new one
* plus 128 bytes for future expansions. If we have enough
* room at tail, reallocate without expansion only if skb is cloned.
*/
int i, k, eat = (skb->tail + delta) - skb->end;
if (eat > 0 || skb_cloned(skb)) {
if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
GFP_ATOMIC))
return NULL;
}
BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
skb_tail_pointer(skb), delta));
/* Optimization: no fragments, no reasons to preestimate
* size of pulled pages. Superb.
*/
if (!skb_has_frag_list(skb))
goto pull_pages;
/* Estimate size of pulled pages. */
eat = delta;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
if (size >= eat)
goto pull_pages;
eat -= size;
}
/* If we need update frag list, we are in troubles.
* Certainly, it is possible to add an offset to skb data,
* but taking into account that pulling is expected to
* be very rare operation, it is worth to fight against
* further bloating skb head and crucify ourselves here instead.
* Pure masohism, indeed. 8)8)
*/
if (eat) {
struct sk_buff *list = skb_shinfo(skb)->frag_list;
struct sk_buff *clone = NULL;
struct sk_buff *insp = NULL;
do {
if (list->len <= eat) {
/* Eaten as whole. */
eat -= list->len;
list = list->next;
insp = list;
} else {
/* Eaten partially. */
if (skb_is_gso(skb) && !list->head_frag &&
skb_headlen(list))
skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
if (skb_shared(list)) {
/* Sucks! We need to fork list. :-( */
clone = skb_clone(list, GFP_ATOMIC);
if (!clone)
return NULL;
insp = list->next;
list = clone;
} else {
/* This may be pulled without
* problems. */
insp = list;
}
if (!pskb_pull(list, eat)) {
kfree_skb(clone);
return NULL;
}
break;
}
} while (eat);
/* Free pulled out fragments. */
while ((list = skb_shinfo(skb)->frag_list) != insp) {
skb_shinfo(skb)->frag_list = list->next;
consume_skb(list);
}
/* And insert new clone at head. */
if (clone) {
clone->next = list;
skb_shinfo(skb)->frag_list = clone;
}
}
/* Success! Now we may commit changes to skb data. */
pull_pages:
eat = delta;
k = 0;
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
if (size <= eat) {
skb_frag_unref(skb, i);
eat -= size;
} else {
skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
*frag = skb_shinfo(skb)->frags[i];
if (eat) {
skb_frag_off_add(frag, eat);
skb_frag_size_sub(frag, eat);
if (!i)
goto end;
eat = 0;
}
k++;
}
}
skb_shinfo(skb)->nr_frags = k;
end:
skb->tail += delta;
skb->data_len -= delta;
if (!skb->data_len)
skb_zcopy_clear(skb, false);
return skb_tail_pointer(skb);
}
EXPORT_SYMBOL(__pskb_pull_tail);
/**
* skb_copy_bits - copy bits from skb to kernel buffer
* @skb: source skb
* @offset: offset in source
* @to: destination buffer
* @len: number of bytes to copy
*
* Copy the specified number of bytes from the source skb to the
* destination buffer.
*
* CAUTION ! :
* If its prototype is ever changed,
* check arch/{*}/net/{*}.S files,
* since it is called from BPF assembly code.
*/
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
{
int start = skb_headlen(skb);
struct sk_buff *frag_iter;
int i, copy;
if (offset > (int)skb->len - len)
goto fault;
/* Copy header. */
if ((copy = start - offset) > 0) {
if (copy > len)
copy = len;
skb_copy_from_linear_data_offset(skb, offset, to, copy);
if ((len -= copy) == 0)
return 0;
offset += copy;
to += copy;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
skb_frag_t *f = &skb_shinfo(skb)->frags[i];
WARN_ON(start > offset + len);
end = start + skb_frag_size(f);
if ((copy = end - offset) > 0) {
u32 p_off, p_len, copied;
struct page *p;
u8 *vaddr;
if (copy > len)
copy = len;
skb_frag_foreach_page(f,
skb_frag_off(f) + offset - start,
copy, p, p_off, p_len, copied) {
vaddr = kmap_atomic(p);
memcpy(to + copied, vaddr + p_off, p_len);
kunmap_atomic(vaddr);
}
if ((len -= copy) == 0)
return 0;
offset += copy;
to += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
if (skb_copy_bits(frag_iter, offset - start, to, copy))
goto fault;
if ((len -= copy) == 0)
return 0;
offset += copy;
to += copy;
}
start = end;
}
if (!len)
return 0;
fault:
return -EFAULT;
}
EXPORT_SYMBOL(skb_copy_bits);
/*
* Callback from splice_to_pipe(), if we need to release some pages
* at the end of the spd in case we error'ed out in filling the pipe.
*/
static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
put_page(spd->pages[i]);
}
static struct page *linear_to_page(struct page *page, unsigned int *len,
unsigned int *offset,
struct sock *sk)
{
struct page_frag *pfrag = sk_page_frag(sk);
if (!sk_page_frag_refill(sk, pfrag))
return NULL;
*len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
memcpy(page_address(pfrag->page) + pfrag->offset,
page_address(page) + *offset, *len);
*offset = pfrag->offset;
pfrag->offset += *len;
return pfrag->page;
}
static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
struct page *page,
unsigned int offset)
{
return spd->nr_pages &&
spd->pages[spd->nr_pages - 1] == page &&
(spd->partial[spd->nr_pages - 1].offset +
spd->partial[spd->nr_pages - 1].len == offset);
}
/*
* Fill page/offset/length into spd, if it can hold more pages.
*/
static bool spd_fill_page(struct splice_pipe_desc *spd,
struct pipe_inode_info *pipe, struct page *page,
unsigned int *len, unsigned int offset,
bool linear,
struct sock *sk)
{
if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
return true;
if (linear) {
page = linear_to_page(page, len, &offset, sk);
if (!page)
return true;
}
if (spd_can_coalesce(spd, page, offset)) {
spd->partial[spd->nr_pages - 1].len += *len;
return false;
}
get_page(page);
spd->pages[spd->nr_pages] = page;
spd->partial[spd->nr_pages].len = *len;
spd->partial[spd->nr_pages].offset = offset;
spd->nr_pages++;
return false;
}
static bool __splice_segment(struct page *page, unsigned int poff,
unsigned int plen, unsigned int *off,
unsigned int *len,
struct splice_pipe_desc *spd, bool linear,
struct sock *sk,
struct pipe_inode_info *pipe)
{
if (!*len)
return true;
/* skip this segment if already processed */
if (*off >= plen) {
*off -= plen;
return false;
}
/* ignore any bits we already processed */
poff += *off;
plen -= *off;
*off = 0;
do {
unsigned int flen = min(*len, plen);
if (spd_fill_page(spd, pipe, page, &flen, poff,
linear, sk))
return true;
poff += flen;
plen -= flen;
*len -= flen;
} while (*len && plen);
return false;
}
/*
* Map linear and fragment data from the skb to spd. It reports true if the
* pipe is full or if we already spliced the requested length.
*/
static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
unsigned int *offset, unsigned int *len,
struct splice_pipe_desc *spd, struct sock *sk)
{
int seg;
struct sk_buff *iter;
/* map the linear part :
* If skb->head_frag is set, this 'linear' part is backed by a
* fragment, and if the head is not shared with any clones then
* we can avoid a copy since we own the head portion of this page.
*/
if (__splice_segment(virt_to_page(skb->data),
(unsigned long) skb->data & (PAGE_SIZE - 1),
skb_headlen(skb),
offset, len, spd,
skb_head_is_locked(skb),
sk, pipe))
return true;
/*
* then map the fragments
*/
for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
if (__splice_segment(skb_frag_page(f),
skb_frag_off(f), skb_frag_size(f),
offset, len, spd, false, sk, pipe))
return true;
}
skb_walk_frags(skb, iter) {
if (*offset >= iter->len) {
*offset -= iter->len;
continue;
}
/* __skb_splice_bits() only fails if the output has no room
* left, so no point in going over the frag_list for the error
* case.
*/
if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
return true;
}
return false;
}
/*
* Map data from the skb to a pipe. Should handle both the linear part,
* the fragments, and the frag list.
*/
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
struct pipe_inode_info *pipe, unsigned int tlen,
unsigned int flags)
{
struct partial_page partial[MAX_SKB_FRAGS];
struct page *pages[MAX_SKB_FRAGS];
struct splice_pipe_desc spd = {
.pages = pages,
.partial = partial,
.nr_pages_max = MAX_SKB_FRAGS,
.ops = &nosteal_pipe_buf_ops,
.spd_release = sock_spd_release,
};
int ret = 0;
__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
if (spd.nr_pages)
ret = splice_to_pipe(pipe, &spd);
return ret;
}
EXPORT_SYMBOL_GPL(skb_splice_bits);
static int sendmsg_locked(struct sock *sk, struct msghdr *msg)
{
struct socket *sock = sk->sk_socket;
size_t size = msg_data_left(msg);
if (!sock)
return -EINVAL;
if (!sock->ops->sendmsg_locked)
return sock_no_sendmsg_locked(sk, msg, size);
return sock->ops->sendmsg_locked(sk, msg, size);
}
static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
{
struct socket *sock = sk->sk_socket;
if (!sock)
return -EINVAL;
return sock_sendmsg(sock, msg);
}
typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
int len, sendmsg_func sendmsg)
{
unsigned int orig_len = len;
struct sk_buff *head = skb;
unsigned short fragidx;
int slen, ret;
do_frag_list:
/* Deal with head data */
while (offset < skb_headlen(skb) && len) {
struct kvec kv;
struct msghdr msg;
slen = min_t(int, len, skb_headlen(skb) - offset);
kv.iov_base = skb->data + offset;
kv.iov_len = slen;
memset(&msg, 0, sizeof(msg));
msg.msg_flags = MSG_DONTWAIT;
iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
sendmsg_unlocked, sk, &msg);
if (ret <= 0)
goto error;
offset += ret;
len -= ret;
}
/* All the data was skb head? */
if (!len)
goto out;
/* Make offset relative to start of frags */
offset -= skb_headlen(skb);
/* Find where we are in frag list */
for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
if (offset < skb_frag_size(frag))
break;
offset -= skb_frag_size(frag);
}
for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
slen = min_t(size_t, len, skb_frag_size(frag) - offset);
while (slen) {
struct bio_vec bvec;
struct msghdr msg = {
.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT,
};
bvec_set_page(&bvec, skb_frag_page(frag), slen,
skb_frag_off(frag) + offset);
iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
slen);
ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
sendmsg_unlocked, sk, &msg);
if (ret <= 0)
goto error;
len -= ret;
offset += ret;
slen -= ret;
}
offset = 0;
}
if (len) {
/* Process any frag lists */
if (skb == head) {
if (skb_has_frag_list(skb)) {
skb = skb_shinfo(skb)->frag_list;
goto do_frag_list;
}
} else if (skb->next) {
skb = skb->next;
goto do_frag_list;
}
}
out:
return orig_len - len;
error:
return orig_len == len ? ret : orig_len - len;
}
/* Send skb data on a socket. Socket must be locked. */
int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
int len)
{
return __skb_send_sock(sk, skb, offset, len, sendmsg_locked);
}
EXPORT_SYMBOL_GPL(skb_send_sock_locked);
/* Send skb data on a socket. Socket must be unlocked. */
int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
{
return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked);
}
/**
* skb_store_bits - store bits from kernel buffer to skb
* @skb: destination buffer
* @offset: offset in destination
* @from: source buffer
* @len: number of bytes to copy
*
* Copy the specified number of bytes from the source buffer to the
* destination skb. This function handles all the messy bits of
* traversing fragment lists and such.
*/
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
{
int start = skb_headlen(skb);
struct sk_buff *frag_iter;
int i, copy;
if (offset > (int)skb->len - len)
goto fault;
if ((copy = start - offset) > 0) {
if (copy > len)
copy = len;
skb_copy_to_linear_data_offset(skb, offset, from, copy);
if ((len -= copy) == 0)
return 0;
offset += copy;
from += copy;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
int end;
WARN_ON(start > offset + len);
end = start + skb_frag_size(frag);
if ((copy = end - offset) > 0) {
u32 p_off, p_len, copied;
struct page *p;
u8 *vaddr;
if (copy > len)
copy = len;
skb_frag_foreach_page(frag,
skb_frag_off(frag) + offset - start,
copy, p, p_off, p_len, copied) {
vaddr = kmap_atomic(p);
memcpy(vaddr + p_off, from + copied, p_len);
kunmap_atomic(vaddr);
}
if ((len -= copy) == 0)
return 0;
offset += copy;
from += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
if (skb_store_bits(frag_iter, offset - start,
from, copy))
goto fault;
if ((len -= copy) == 0)
return 0;
offset += copy;
from += copy;
}
start = end;
}
if (!len)
return 0;
fault:
return -EFAULT;
}
EXPORT_SYMBOL(skb_store_bits);
/* Checksum skb data. */
__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
__wsum csum, const struct skb_checksum_ops *ops)
{
int start = skb_headlen(skb);
int i, copy = start - offset;
struct sk_buff *frag_iter;
int pos = 0;
/* Checksum header. */
if (copy > 0) {
if (copy > len)
copy = len;
csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
skb->data + offset, copy, csum);
if ((len -= copy) == 0)
return csum;
offset += copy;
pos = copy;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
WARN_ON(start > offset + len);
end = start + skb_frag_size(frag);
if ((copy = end - offset) > 0) {
u32 p_off, p_len, copied;
struct page *p;
__wsum csum2;
u8 *vaddr;
if (copy > len)
copy = len;
skb_frag_foreach_page(frag,
skb_frag_off(frag) + offset - start,
copy, p, p_off, p_len, copied) {
vaddr = kmap_atomic(p);
csum2 = INDIRECT_CALL_1(ops->update,
csum_partial_ext,
vaddr + p_off, p_len, 0);
kunmap_atomic(vaddr);
csum = INDIRECT_CALL_1(ops->combine,
csum_block_add_ext, csum,
csum2, pos, p_len);
pos += p_len;
}
if (!(len -= copy))
return csum;
offset += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
__wsum csum2;
if (copy > len)
copy = len;
csum2 = __skb_checksum(frag_iter, offset - start,
copy, 0, ops);
csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
csum, csum2, pos, copy);
if ((len -= copy) == 0)
return csum;
offset += copy;
pos += copy;
}
start = end;
}
BUG_ON(len);
return csum;
}
EXPORT_SYMBOL(__skb_checksum);
__wsum skb_checksum(const struct sk_buff *skb, int offset,
int len, __wsum csum)
{
const struct skb_checksum_ops ops = {
.update = csum_partial_ext,
.combine = csum_block_add_ext,
};
return __skb_checksum(skb, offset, len, csum, &ops);
}
EXPORT_SYMBOL(skb_checksum);
/* Both of above in one bottle. */
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
u8 *to, int len)
{
int start = skb_headlen(skb);
int i, copy = start - offset;
struct sk_buff *frag_iter;
int pos = 0;
__wsum csum = 0;
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
csum = csum_partial_copy_nocheck(skb->data + offset, to,
copy);
if ((len -= copy) == 0)
return csum;
offset += copy;
to += copy;
pos = copy;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
WARN_ON(start > offset + len);
end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
if ((copy = end - offset) > 0) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
u32 p_off, p_len, copied;
struct page *p;
__wsum csum2;
u8 *vaddr;
if (copy > len)
copy = len;
skb_frag_foreach_page(frag,
skb_frag_off(frag) + offset - start,
copy, p, p_off, p_len, copied) {
vaddr = kmap_atomic(p);
csum2 = csum_partial_copy_nocheck(vaddr + p_off,
to + copied,
p_len);
kunmap_atomic(vaddr);
csum = csum_block_add(csum, csum2, pos);
pos += p_len;
}
if (!(len -= copy))
return csum;
offset += copy;
to += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
__wsum csum2;
int end;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
if (copy > len)
copy = len;
csum2 = skb_copy_and_csum_bits(frag_iter,
offset - start,
to, copy);
csum = csum_block_add(csum, csum2, pos);
if ((len -= copy) == 0)
return csum;
offset += copy;
to += copy;
pos += copy;
}
start = end;
}
BUG_ON(len);
return csum;
}
EXPORT_SYMBOL(skb_copy_and_csum_bits);
__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
{
__sum16 sum;
sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
/* See comments in __skb_checksum_complete(). */
if (likely(!sum)) {
if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
!skb->csum_complete_sw)
netdev_rx_csum_fault(skb->dev, skb);
}
if (!skb_shared(skb))
skb->csum_valid = !sum;
return sum;
}
EXPORT_SYMBOL(__skb_checksum_complete_head);
/* This function assumes skb->csum already holds pseudo header's checksum,
* which has been changed from the hardware checksum, for example, by
* __skb_checksum_validate_complete(). And, the original skb->csum must
* have been validated unsuccessfully for CHECKSUM_COMPLETE case.
*
* It returns non-zero if the recomputed checksum is still invalid, otherwise
* zero. The new checksum is stored back into skb->csum unless the skb is
* shared.
*/
__sum16 __skb_checksum_complete(struct sk_buff *skb)
{
__wsum csum;
__sum16 sum;
csum = skb_checksum(skb, 0, skb->len, 0);
sum = csum_fold(csum_add(skb->csum, csum));
/* This check is inverted, because we already knew the hardware
* checksum is invalid before calling this function. So, if the
* re-computed checksum is valid instead, then we have a mismatch
* between the original skb->csum and skb_checksum(). This means either
* the original hardware checksum is incorrect or we screw up skb->csum
* when moving skb->data around.
*/
if (likely(!sum)) {
if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
!skb->csum_complete_sw)
netdev_rx_csum_fault(skb->dev, skb);
}
if (!skb_shared(skb)) {
/* Save full packet checksum */
skb->csum = csum;
skb->ip_summed = CHECKSUM_COMPLETE;
skb->csum_complete_sw = 1;
skb->csum_valid = !sum;
}
return sum;
}
EXPORT_SYMBOL(__skb_checksum_complete);
static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
{
net_warn_ratelimited(
"%s: attempt to compute crc32c without libcrc32c.ko\n",
__func__);
return 0;
}
static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
int offset, int len)
{
net_warn_ratelimited(
"%s: attempt to compute crc32c without libcrc32c.ko\n",
__func__);
return 0;
}
static const struct skb_checksum_ops default_crc32c_ops = {
.update = warn_crc32c_csum_update,
.combine = warn_crc32c_csum_combine,
};
const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
&default_crc32c_ops;
EXPORT_SYMBOL(crc32c_csum_stub);
/**
* skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
* @from: source buffer
*
* Calculates the amount of linear headroom needed in the 'to' skb passed
* into skb_zerocopy().
*/
unsigned int
skb_zerocopy_headlen(const struct sk_buff *from)
{
unsigned int hlen = 0;
if (!from->head_frag ||
skb_headlen(from) < L1_CACHE_BYTES ||
skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
hlen = skb_headlen(from);
if (!hlen)
hlen = from->len;
}
if (skb_has_frag_list(from))
hlen = from->len;
return hlen;
}
EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
/**
* skb_zerocopy - Zero copy skb to skb
* @to: destination buffer
* @from: source buffer
* @len: number of bytes to copy from source buffer
* @hlen: size of linear headroom in destination buffer
*
* Copies up to `len` bytes from `from` to `to` by creating references
* to the frags in the source buffer.
*
* The `hlen` as calculated by skb_zerocopy_headlen() specifies the
* headroom in the `to` buffer.
*
* Return value:
* 0: everything is OK
* -ENOMEM: couldn't orphan frags of @from due to lack of memory
* -EFAULT: skb_copy_bits() found some problem with skb geometry
*/
int
skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
{
int i, j = 0;
int plen = 0; /* length of skb->head fragment */
int ret;
struct page *page;
unsigned int offset;
BUG_ON(!from->head_frag && !hlen);
/* dont bother with small payloads */
if (len <= skb_tailroom(to))
return skb_copy_bits(from, 0, skb_put(to, len), len);
if (hlen) {
ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
if (unlikely(ret))
return ret;
len -= hlen;
} else {
plen = min_t(int, skb_headlen(from), len);
if (plen) {
page = virt_to_head_page(from->head);
offset = from->data - (unsigned char *)page_address(page);
__skb_fill_netmem_desc(to, 0, page_to_netmem(page),
offset, plen);
get_page(page);
j = 1;
len -= plen;
}
}
skb_len_add(to, len + plen);
if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
skb_tx_error(from);
return -ENOMEM;
}
skb_zerocopy_clone(to, from, GFP_ATOMIC);
for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
int size;
if (!len)
break;
skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
len);
skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
len -= size;
skb_frag_ref(to, j);
j++;
}
skb_shinfo(to)->nr_frags = j;
return 0;
}
EXPORT_SYMBOL_GPL(skb_zerocopy);
void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
{
__wsum csum;
long csstart;
if (skb->ip_summed == CHECKSUM_PARTIAL)
csstart = skb_checksum_start_offset(skb);
else
csstart = skb_headlen(skb);
BUG_ON(csstart > skb_headlen(skb));
skb_copy_from_linear_data(skb, to, csstart);
csum = 0;
if (csstart != skb->len)
csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
skb->len - csstart);
if (skb->ip_summed == CHECKSUM_PARTIAL) {
long csstuff = csstart + skb->csum_offset;
*((__sum16 *)(to + csstuff)) = csum_fold(csum);
}
}
EXPORT_SYMBOL(skb_copy_and_csum_dev);
/**
* skb_dequeue - remove from the head of the queue
* @list: list to dequeue from
*
* Remove the head of the list. The list lock is taken so the function
* may be used safely with other locking list functions. The head item is
* returned or %NULL if the list is empty.
*/
struct sk_buff *skb_dequeue(struct sk_buff_head *list)
{
unsigned long flags;
struct sk_buff *result;
spin_lock_irqsave(&list->lock, flags); result = __skb_dequeue(list); spin_unlock_irqrestore(&list->lock, flags);
return result;
}
EXPORT_SYMBOL(skb_dequeue);
/**
* skb_dequeue_tail - remove from the tail of the queue
* @list: list to dequeue from
*
* Remove the tail of the list. The list lock is taken so the function
* may be used safely with other locking list functions. The tail item is
* returned or %NULL if the list is empty.
*/
struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
{
unsigned long flags;
struct sk_buff *result;
spin_lock_irqsave(&list->lock, flags);
result = __skb_dequeue_tail(list);
spin_unlock_irqrestore(&list->lock, flags);
return result;
}
EXPORT_SYMBOL(skb_dequeue_tail);
/**
* skb_queue_purge_reason - empty a list
* @list: list to empty
* @reason: drop reason
*
* Delete all buffers on an &sk_buff list. Each buffer is removed from
* the list and one reference dropped. This function takes the list
* lock and is atomic with respect to other list locking functions.
*/
void skb_queue_purge_reason(struct sk_buff_head *list,
enum skb_drop_reason reason)
{
struct sk_buff_head tmp;
unsigned long flags;
if (skb_queue_empty_lockless(list))
return; __skb_queue_head_init(&tmp);
spin_lock_irqsave(&list->lock, flags);
skb_queue_splice_init(list, &tmp); spin_unlock_irqrestore(&list->lock, flags); __skb_queue_purge_reason(&tmp, reason);
}
EXPORT_SYMBOL(skb_queue_purge_reason);
/**
* skb_rbtree_purge - empty a skb rbtree
* @root: root of the rbtree to empty
* Return value: the sum of truesizes of all purged skbs.
*
* Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
* the list and one reference dropped. This function does not take
* any lock. Synchronization should be handled by the caller (e.g., TCP
* out-of-order queue is protected by the socket lock).
*/
unsigned int skb_rbtree_purge(struct rb_root *root)
{
struct rb_node *p = rb_first(root);
unsigned int sum = 0;
while (p) {
struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
p = rb_next(p);
rb_erase(&skb->rbnode, root);
sum += skb->truesize;
kfree_skb(skb);
}
return sum;
}
void skb_errqueue_purge(struct sk_buff_head *list)
{
struct sk_buff *skb, *next;
struct sk_buff_head kill;
unsigned long flags;
__skb_queue_head_init(&kill);
spin_lock_irqsave(&list->lock, flags);
skb_queue_walk_safe(list, skb, next) {
if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY ||
SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING)
continue;
__skb_unlink(skb, list);
__skb_queue_tail(&kill, skb);
}
spin_unlock_irqrestore(&list->lock, flags);
__skb_queue_purge(&kill);
}
EXPORT_SYMBOL(skb_errqueue_purge);
/**
* skb_queue_head - queue a buffer at the list head
* @list: list to use
* @newsk: buffer to queue
*
* Queue a buffer at the start of the list. This function takes the
* list lock and can be used safely with other locking &sk_buff functions
* safely.
*
* A buffer cannot be placed on two lists at the same time.
*/
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
{
unsigned long flags;
spin_lock_irqsave(&list->lock, flags);
__skb_queue_head(list, newsk);
spin_unlock_irqrestore(&list->lock, flags);
}
EXPORT_SYMBOL(skb_queue_head);
/**
* skb_queue_tail - queue a buffer at the list tail
* @list: list to use
* @newsk: buffer to queue
*
* Queue a buffer at the tail of the list. This function takes the
* list lock and can be used safely with other locking &sk_buff functions
* safely.
*
* A buffer cannot be placed on two lists at the same time.
*/
void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
{
unsigned long flags;
spin_lock_irqsave(&list->lock, flags);
__skb_queue_tail(list, newsk);
spin_unlock_irqrestore(&list->lock, flags);
}
EXPORT_SYMBOL(skb_queue_tail);
/**
* skb_unlink - remove a buffer from a list
* @skb: buffer to remove
* @list: list to use
*
* Remove a packet from a list. The list locks are taken and this
* function is atomic with respect to other list locked calls
*
* You must know what list the SKB is on.
*/
void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
{
unsigned long flags;
spin_lock_irqsave(&list->lock, flags);
__skb_unlink(skb, list);
spin_unlock_irqrestore(&list->lock, flags);
}
EXPORT_SYMBOL(skb_unlink);
/**
* skb_append - append a buffer
* @old: buffer to insert after
* @newsk: buffer to insert
* @list: list to use
*
* Place a packet after a given packet in a list. The list locks are taken
* and this function is atomic with respect to other list locked calls.
* A buffer cannot be placed on two lists at the same time.
*/
void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
{
unsigned long flags;
spin_lock_irqsave(&list->lock, flags);
__skb_queue_after(list, old, newsk);
spin_unlock_irqrestore(&list->lock, flags);
}
EXPORT_SYMBOL(skb_append);
static inline void skb_split_inside_header(struct sk_buff *skb,
struct sk_buff* skb1,
const u32 len, const int pos)
{
int i;
skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
pos - len);
/* And move data appendix as is. */
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
skb_shinfo(skb)->nr_frags = 0;
skb1->data_len = skb->data_len;
skb1->len += skb1->data_len;
skb->data_len = 0;
skb->len = len;
skb_set_tail_pointer(skb, len);
}
static inline void skb_split_no_header(struct sk_buff *skb,
struct sk_buff* skb1,
const u32 len, int pos)
{
int i, k = 0;
const int nfrags = skb_shinfo(skb)->nr_frags;
skb_shinfo(skb)->nr_frags = 0;
skb1->len = skb1->data_len = skb->len - len;
skb->len = len;
skb->data_len = len - pos;
for (i = 0; i < nfrags; i++) {
int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
if (pos + size > len) {
skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
if (pos < len) {
/* Split frag.
* We have two variants in this case:
* 1. Move all the frag to the second
* part, if it is possible. F.e.
* this approach is mandatory for TUX,
* where splitting is expensive.
* 2. Split is accurately. We make this.
*/
skb_frag_ref(skb, i);
skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
skb_shinfo(skb)->nr_frags++;
}
k++;
} else
skb_shinfo(skb)->nr_frags++;
pos += size;
}
skb_shinfo(skb1)->nr_frags = k;
}
/**
* skb_split - Split fragmented skb to two parts at length len.
* @skb: the buffer to split
* @skb1: the buffer to receive the second part
* @len: new length for skb
*/
void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
{
int pos = skb_headlen(skb);
const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
skb_zcopy_downgrade_managed(skb);
skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
skb_zerocopy_clone(skb1, skb, 0);
if (len < pos) /* Split line is inside header. */
skb_split_inside_header(skb, skb1, len, pos);
else /* Second chunk has no header, nothing to copy. */
skb_split_no_header(skb, skb1, len, pos);
}
EXPORT_SYMBOL(skb_split);
/* Shifting from/to a cloned skb is a no-go.
*
* Caller cannot keep skb_shinfo related pointers past calling here!
*/
static int skb_prepare_for_shift(struct sk_buff *skb)
{
return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
}
/**
* skb_shift - Shifts paged data partially from skb to another
* @tgt: buffer into which tail data gets added
* @skb: buffer from which the paged data comes from
* @shiftlen: shift up to this many bytes
*
* Attempts to shift up to shiftlen worth of bytes, which may be less than
* the length of the skb, from skb to tgt. Returns number bytes shifted.
* It's up to caller to free skb if everything was shifted.
*
* If @tgt runs out of frags, the whole operation is aborted.
*
* Skb cannot include anything else but paged data while tgt is allowed
* to have non-paged data as well.
*
* TODO: full sized shift could be optimized but that would need
* specialized skb free'er to handle frags without up-to-date nr_frags.
*/
int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
{
int from, to, merge, todo;
skb_frag_t *fragfrom, *fragto;
BUG_ON(shiftlen > skb->len);
if (skb_headlen(skb))
return 0;
if (skb_zcopy(tgt) || skb_zcopy(skb))
return 0;
todo = shiftlen;
from = 0;
to = skb_shinfo(tgt)->nr_frags;
fragfrom = &skb_shinfo(skb)->frags[from];
/* Actual merge is delayed until the point when we know we can
* commit all, so that we don't have to undo partial changes
*/
if (!to ||
!skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
skb_frag_off(fragfrom))) {
merge = -1;
} else {
merge = to - 1;
todo -= skb_frag_size(fragfrom);
if (todo < 0) {
if (skb_prepare_for_shift(skb) ||
skb_prepare_for_shift(tgt))
return 0;
/* All previous frag pointers might be stale! */
fragfrom = &skb_shinfo(skb)->frags[from];
fragto = &skb_shinfo(tgt)->frags[merge];
skb_frag_size_add(fragto, shiftlen);
skb_frag_size_sub(fragfrom, shiftlen);
skb_frag_off_add(fragfrom, shiftlen);
goto onlymerged;
}
from++;
}
/* Skip full, not-fitting skb to avoid expensive operations */
if ((shiftlen == skb->len) &&
(skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
return 0;
if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
return 0;
while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
if (to == MAX_SKB_FRAGS)
return 0;
fragfrom = &skb_shinfo(skb)->frags[from];
fragto = &skb_shinfo(tgt)->frags[to];
if (todo >= skb_frag_size(fragfrom)) {
*fragto = *fragfrom;
todo -= skb_frag_size(fragfrom);
from++;
to++;
} else {
__skb_frag_ref(fragfrom);
skb_frag_page_copy(fragto, fragfrom);
skb_frag_off_copy(fragto, fragfrom);
skb_frag_size_set(fragto, todo);
skb_frag_off_add(fragfrom, todo);
skb_frag_size_sub(fragfrom, todo);
todo = 0;
to++;
break;
}
}
/* Ready to "commit" this state change to tgt */
skb_shinfo(tgt)->nr_frags = to;
if (merge >= 0) {
fragfrom = &skb_shinfo(skb)->frags[0];
fragto = &skb_shinfo(tgt)->frags[merge];
skb_frag_size_add(fragto, skb_frag_size(fragfrom));
__skb_frag_unref(fragfrom, skb->pp_recycle);
}
/* Reposition in the original skb */
to = 0;
while (from < skb_shinfo(skb)->nr_frags)
skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
skb_shinfo(skb)->nr_frags = to;
BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
onlymerged:
/* Most likely the tgt won't ever need its checksum anymore, skb on
* the other hand might need it if it needs to be resent
*/
tgt->ip_summed = CHECKSUM_PARTIAL;
skb->ip_summed = CHECKSUM_PARTIAL;
skb_len_add(skb, -shiftlen);
skb_len_add(tgt, shiftlen);
return shiftlen;
}
/**
* skb_prepare_seq_read - Prepare a sequential read of skb data
* @skb: the buffer to read
* @from: lower offset of data to be read
* @to: upper offset of data to be read
* @st: state variable
*
* Initializes the specified state variable. Must be called before
* invoking skb_seq_read() for the first time.
*/
void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
unsigned int to, struct skb_seq_state *st)
{
st->lower_offset = from;
st->upper_offset = to;
st->root_skb = st->cur_skb = skb;
st->frag_idx = st->stepped_offset = 0;
st->frag_data = NULL;
st->frag_off = 0;
}
EXPORT_SYMBOL(skb_prepare_seq_read);
/**
* skb_seq_read - Sequentially read skb data
* @consumed: number of bytes consumed by the caller so far
* @data: destination pointer for data to be returned
* @st: state variable
*
* Reads a block of skb data at @consumed relative to the
* lower offset specified to skb_prepare_seq_read(). Assigns
* the head of the data block to @data and returns the length
* of the block or 0 if the end of the skb data or the upper
* offset has been reached.
*
* The caller is not required to consume all of the data
* returned, i.e. @consumed is typically set to the number
* of bytes already consumed and the next call to
* skb_seq_read() will return the remaining part of the block.
*
* Note 1: The size of each block of data returned can be arbitrary,
* this limitation is the cost for zerocopy sequential
* reads of potentially non linear data.
*
* Note 2: Fragment lists within fragments are not implemented
* at the moment, state->root_skb could be replaced with
* a stack for this purpose.
*/
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
struct skb_seq_state *st)
{
unsigned int block_limit, abs_offset = consumed + st->lower_offset;
skb_frag_t *frag;
if (unlikely(abs_offset >= st->upper_offset)) {
if (st->frag_data) {
kunmap_atomic(st->frag_data);
st->frag_data = NULL;
}
return 0;
}
next_skb:
block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
if (abs_offset < block_limit && !st->frag_data) {
*data = st->cur_skb->data + (abs_offset - st->stepped_offset);
return block_limit - abs_offset;
}
if (st->frag_idx == 0 && !st->frag_data)
st->stepped_offset += skb_headlen(st->cur_skb);
while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
unsigned int pg_idx, pg_off, pg_sz;
frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
pg_idx = 0;
pg_off = skb_frag_off(frag);
pg_sz = skb_frag_size(frag);
if (skb_frag_must_loop(skb_frag_page(frag))) {
pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
pg_off = offset_in_page(pg_off + st->frag_off);
pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
PAGE_SIZE - pg_off);
}
block_limit = pg_sz + st->stepped_offset;
if (abs_offset < block_limit) {
if (!st->frag_data)
st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
*data = (u8 *)st->frag_data + pg_off +
(abs_offset - st->stepped_offset);
return block_limit - abs_offset;
}
if (st->frag_data) {
kunmap_atomic(st->frag_data);
st->frag_data = NULL;
}
st->stepped_offset += pg_sz;
st->frag_off += pg_sz;
if (st->frag_off == skb_frag_size(frag)) {
st->frag_off = 0;
st->frag_idx++;
}
}
if (st->frag_data) {
kunmap_atomic(st->frag_data);
st->frag_data = NULL;
}
if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
st->frag_idx = 0;
goto next_skb;
} else if (st->cur_skb->next) {
st->cur_skb = st->cur_skb->next;
st->frag_idx = 0;
goto next_skb;
}
return 0;
}
EXPORT_SYMBOL(skb_seq_read);
/**
* skb_abort_seq_read - Abort a sequential read of skb data
* @st: state variable
*
* Must be called if skb_seq_read() was not called until it
* returned 0.
*/
void skb_abort_seq_read(struct skb_seq_state *st)
{
if (st->frag_data)
kunmap_atomic(st->frag_data);
}
EXPORT_SYMBOL(skb_abort_seq_read);
#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
struct ts_config *conf,
struct ts_state *state)
{
return skb_seq_read(offset, text, TS_SKB_CB(state));
}
static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
{
skb_abort_seq_read(TS_SKB_CB(state));
}
/**
* skb_find_text - Find a text pattern in skb data
* @skb: the buffer to look in
* @from: search offset
* @to: search limit
* @config: textsearch configuration
*
* Finds a pattern in the skb data according to the specified
* textsearch configuration. Use textsearch_next() to retrieve
* subsequent occurrences of the pattern. Returns the offset
* to the first occurrence or UINT_MAX if no match was found.
*/
unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
unsigned int to, struct ts_config *config)
{
unsigned int patlen = config->ops->get_pattern_len(config);
struct ts_state state;
unsigned int ret;
BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb));
config->get_next_block = skb_ts_get_next_block;
config->finish = skb_ts_finish;
skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
ret = textsearch_find(config, &state);
return (ret + patlen <= to - from ? ret : UINT_MAX);
}
EXPORT_SYMBOL(skb_find_text);
int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
int offset, size_t size, size_t max_frags)
{
int i = skb_shinfo(skb)->nr_frags;
if (skb_can_coalesce(skb, i, page, offset)) {
skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
} else if (i < max_frags) {
skb_zcopy_downgrade_managed(skb);
get_page(page);
skb_fill_page_desc_noacc(skb, i, page, offset, size);
} else {
return -EMSGSIZE;
}
return 0;
}
EXPORT_SYMBOL_GPL(skb_append_pagefrags);
/**
* skb_pull_rcsum - pull skb and update receive checksum
* @skb: buffer to update
* @len: length of data pulled
*
* This function performs an skb_pull on the packet and updates
* the CHECKSUM_COMPLETE checksum. It should be used on
* receive path processing instead of skb_pull unless you know
* that the checksum difference is zero (e.g., a valid IP header)
* or you are setting ip_summed to CHECKSUM_NONE.
*/
void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
{
unsigned char *data = skb->data;
BUG_ON(len > skb->len);
__skb_pull(skb, len);
skb_postpull_rcsum(skb, data, len);
return skb->data;
}
EXPORT_SYMBOL_GPL(skb_pull_rcsum);
static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
{
skb_frag_t head_frag;
struct page *page;
page = virt_to_head_page(frag_skb->head);
skb_frag_fill_page_desc(&head_frag, page, frag_skb->data -
(unsigned char *)page_address(page),
skb_headlen(frag_skb));
return head_frag;
}
struct sk_buff *skb_segment_list(struct sk_buff *skb,
netdev_features_t features,
unsigned int offset)
{
struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
unsigned int tnl_hlen = skb_tnl_header_len(skb);
unsigned int delta_truesize = 0;
unsigned int delta_len = 0;
struct sk_buff *tail = NULL;
struct sk_buff *nskb, *tmp;
int len_diff, err;
skb_push(skb, -skb_network_offset(skb) + offset);
/* Ensure the head is writeable before touching the shared info */
err = skb_unclone(skb, GFP_ATOMIC);
if (err)
goto err_linearize;
skb_shinfo(skb)->frag_list = NULL;
while (list_skb) {
nskb = list_skb;
list_skb = list_skb->next;
err = 0;
delta_truesize += nskb->truesize;
if (skb_shared(nskb)) {
tmp = skb_clone(nskb, GFP_ATOMIC);
if (tmp) {
consume_skb(nskb);
nskb = tmp;
err = skb_unclone(nskb, GFP_ATOMIC);
} else {
err = -ENOMEM;
}
}
if (!tail)
skb->next = nskb;
else
tail->next = nskb;
if (unlikely(err)) {
nskb->next = list_skb;
goto err_linearize;
}
tail = nskb;
delta_len += nskb->len;
skb_push(nskb, -skb_network_offset(nskb) + offset);
skb_release_head_state(nskb);
len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
__copy_skb_header(nskb, skb);
skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
nskb->transport_header += len_diff;
skb_copy_from_linear_data_offset(skb, -tnl_hlen,
nskb->data - tnl_hlen,
offset + tnl_hlen);
if (skb_needs_linearize(nskb, features) &&
__skb_linearize(nskb))
goto err_linearize;
}
skb->truesize = skb->truesize - delta_truesize;
skb->data_len = skb->data_len - delta_len;
skb->len = skb->len - delta_len;
skb_gso_reset(skb);
skb->prev = tail;
if (skb_needs_linearize(skb, features) &&
__skb_linearize(skb))
goto err_linearize;
skb_get(skb);
return skb;
err_linearize:
kfree_skb_list(skb->next);
skb->next = NULL;
return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL_GPL(skb_segment_list);
/**
* skb_segment - Perform protocol segmentation on skb.
* @head_skb: buffer to segment
* @features: features for the output path (see dev->features)
*
* This function performs segmentation on the given skb. It returns
* a pointer to the first in a list of new skbs for the segments.
* In case of error it returns ERR_PTR(err).
*/
struct sk_buff *skb_segment(struct sk_buff *head_skb,
netdev_features_t features)
{
struct sk_buff *segs = NULL;
struct sk_buff *tail = NULL;
struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
unsigned int mss = skb_shinfo(head_skb)->gso_size;
unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
unsigned int offset = doffset;
unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
unsigned int partial_segs = 0;
unsigned int headroom;
unsigned int len = head_skb->len;
struct sk_buff *frag_skb;
skb_frag_t *frag;
__be16 proto;
bool csum, sg;
int err = -ENOMEM;
int i = 0;
int nfrags, pos;
if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) {
struct sk_buff *check_skb;
for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
if (skb_headlen(check_skb) && !check_skb->head_frag) {
/* gso_size is untrusted, and we have a frag_list with
* a linear non head_frag item.
*
* If head_skb's headlen does not fit requested gso_size,
* it means that the frag_list members do NOT terminate
* on exact gso_size boundaries. Hence we cannot perform
* skb_frag_t page sharing. Therefore we must fallback to
* copying the frag_list skbs; we do so by disabling SG.
*/
features &= ~NETIF_F_SG;
break;
}
}
}
__skb_push(head_skb, doffset);
proto = skb_network_protocol(head_skb, NULL);
if (unlikely(!proto))
return ERR_PTR(-EINVAL);
sg = !!(features & NETIF_F_SG);
csum = !!can_checksum_protocol(features, proto);
if (sg && csum && (mss != GSO_BY_FRAGS)) {
if (!(features & NETIF_F_GSO_PARTIAL)) {
struct sk_buff *iter;
unsigned int frag_len;
if (!list_skb ||
!net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
goto normal;
/* If we get here then all the required
* GSO features except frag_list are supported.
* Try to split the SKB to multiple GSO SKBs
* with no frag_list.
* Currently we can do that only when the buffers don't
* have a linear part and all the buffers except
* the last are of the same length.
*/
frag_len = list_skb->len;
skb_walk_frags(head_skb, iter) {
if (frag_len != iter->len && iter->next)
goto normal;
if (skb_headlen(iter) && !iter->head_frag)
goto normal;
len -= iter->len;
}
if (len != frag_len)
goto normal;
}
/* GSO partial only requires that we trim off any excess that
* doesn't fit into an MSS sized block, so take care of that
* now.
* Cap len to not accidentally hit GSO_BY_FRAGS.
*/
partial_segs = min(len, GSO_BY_FRAGS - 1) / mss;
if (partial_segs > 1)
mss *= partial_segs;
else
partial_segs = 0;
}
normal:
headroom = skb_headroom(head_skb);
pos = skb_headlen(head_skb);
if (skb_orphan_frags(head_skb, GFP_ATOMIC))
return ERR_PTR(-ENOMEM);
nfrags = skb_shinfo(head_skb)->nr_frags;
frag = skb_shinfo(head_skb)->frags;
frag_skb = head_skb;
do {
struct sk_buff *nskb;
skb_frag_t *nskb_frag;
int hsize;
int size;
if (unlikely(mss == GSO_BY_FRAGS)) {
len = list_skb->len;
} else {
len = head_skb->len - offset;
if (len > mss)
len = mss;
}
hsize = skb_headlen(head_skb) - offset;
if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) &&
(skb_headlen(list_skb) == len || sg)) {
BUG_ON(skb_headlen(list_skb) > len);
nskb = skb_clone(list_skb, GFP_ATOMIC);
if (unlikely(!nskb))
goto err;
i = 0;
nfrags = skb_shinfo(list_skb)->nr_frags;
frag = skb_shinfo(list_skb)->frags;
frag_skb = list_skb;
pos += skb_headlen(list_skb);
while (pos < offset + len) {
BUG_ON(i >= nfrags);
size = skb_frag_size(frag);
if (pos + size > offset + len)
break;
i++;
pos += size;
frag++;
}
list_skb = list_skb->next;
if (unlikely(pskb_trim(nskb, len))) {
kfree_skb(nskb);
goto err;
}
hsize = skb_end_offset(nskb);
if (skb_cow_head(nskb, doffset + headroom)) {
kfree_skb(nskb);
goto err;
}
nskb->truesize += skb_end_offset(nskb) - hsize;
skb_release_head_state(nskb);
__skb_push(nskb, doffset);
} else {
if (hsize < 0)
hsize = 0;
if (hsize > len || !sg)
hsize = len;
nskb = __alloc_skb(hsize + doffset + headroom,
GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
NUMA_NO_NODE);
if (unlikely(!nskb))
goto err;
skb_reserve(nskb, headroom);
__skb_put(nskb, doffset);
}
if (segs)
tail->next = nskb;
else
segs = nskb;
tail = nskb;
__copy_skb_header(nskb, head_skb);
skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
skb_reset_mac_len(nskb);
skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
nskb->data - tnl_hlen,
doffset + tnl_hlen);
if (nskb->len == len + doffset)
goto perform_csum_check;
if (!sg) {
if (!csum) {
if (!nskb->remcsum_offload)
nskb->ip_summed = CHECKSUM_NONE;
SKB_GSO_CB(nskb)->csum =
skb_copy_and_csum_bits(head_skb, offset,
skb_put(nskb,
len),
len);
SKB_GSO_CB(nskb)->csum_start =
skb_headroom(nskb) + doffset;
} else {
if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
goto err;
}
continue;
}
nskb_frag = skb_shinfo(nskb)->frags;
skb_copy_from_linear_data_offset(head_skb, offset,
skb_put(nskb, hsize), hsize);
skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags &
SKBFL_SHARED_FRAG;
if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
goto err;
while (pos < offset + len) {
if (i >= nfrags) {
if (skb_orphan_frags(list_skb, GFP_ATOMIC) ||
skb_zerocopy_clone(nskb, list_skb,
GFP_ATOMIC))
goto err;
i = 0;
nfrags = skb_shinfo(list_skb)->nr_frags;
frag = skb_shinfo(list_skb)->frags;
frag_skb = list_skb;
if (!skb_headlen(list_skb)) {
BUG_ON(!nfrags);
} else {
BUG_ON(!list_skb->head_frag);
/* to make room for head_frag. */
i--;
frag--;
}
list_skb = list_skb->next;
}
if (unlikely(skb_shinfo(nskb)->nr_frags >=
MAX_SKB_FRAGS)) {
net_warn_ratelimited(
"skb_segment: too many frags: %u %u\n",
pos, mss);
err = -EINVAL;
goto err;
}
*nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
__skb_frag_ref(nskb_frag);
size = skb_frag_size(nskb_frag);
if (pos < offset) {
skb_frag_off_add(nskb_frag, offset - pos);
skb_frag_size_sub(nskb_frag, offset - pos);
}
skb_shinfo(nskb)->nr_frags++;
if (pos + size <= offset + len) {
i++;
frag++;
pos += size;
} else {
skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
goto skip_fraglist;
}
nskb_frag++;
}
skip_fraglist:
nskb->data_len = len - hsize;
nskb->len += nskb->data_len;
nskb->truesize += nskb->data_len;
perform_csum_check:
if (!csum) {
if (skb_has_shared_frag(nskb) &&
__skb_linearize(nskb))
goto err;
if (!nskb->remcsum_offload)
nskb->ip_summed = CHECKSUM_NONE;
SKB_GSO_CB(nskb)->csum =
skb_checksum(nskb, doffset,
nskb->len - doffset, 0);
SKB_GSO_CB(nskb)->csum_start =
skb_headroom(nskb) + doffset;
}
} while ((offset += len) < head_skb->len);
/* Some callers want to get the end of the list.
* Put it in segs->prev to avoid walking the list.
* (see validate_xmit_skb_list() for example)
*/
segs->prev = tail;
if (partial_segs) {
struct sk_buff *iter;
int type = skb_shinfo(head_skb)->gso_type;
unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
/* Update type to add partial and then remove dodgy if set */
type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
type &= ~SKB_GSO_DODGY;
/* Update GSO info and prepare to start updating headers on
* our way back down the stack of protocols.
*/
for (iter = segs; iter; iter = iter->next) {
skb_shinfo(iter)->gso_size = gso_size;
skb_shinfo(iter)->gso_segs = partial_segs;
skb_shinfo(iter)->gso_type = type;
SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
}
if (tail->len - doffset <= gso_size)
skb_shinfo(tail)->gso_size = 0;
else if (tail != segs)
skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
}
/* Following permits correct backpressure, for protocols
* using skb_set_owner_w().
* Idea is to tranfert ownership from head_skb to last segment.
*/
if (head_skb->destructor == sock_wfree) {
swap(tail->truesize, head_skb->truesize);
swap(tail->destructor, head_skb->destructor);
swap(tail->sk, head_skb->sk);
}
return segs;
err:
kfree_skb_list(segs);
return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(skb_segment);
#ifdef CONFIG_SKB_EXTENSIONS
#define SKB_EXT_ALIGN_VALUE 8
#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
static const u8 skb_ext_type_len[] = {
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
[SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
#endif
#ifdef CONFIG_XFRM
[SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
#endif
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
[TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
#endif
#if IS_ENABLED(CONFIG_MPTCP)
[SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
#endif
#if IS_ENABLED(CONFIG_MCTP_FLOWS)
[SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow),
#endif
};
static __always_inline unsigned int skb_ext_total_length(void)
{
unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext);
int i;
for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++)
l += skb_ext_type_len[i];
return l;
}
static void skb_extensions_init(void)
{
BUILD_BUG_ON(SKB_EXT_NUM >= 8);
#if !IS_ENABLED(CONFIG_KCOV_INSTRUMENT_ALL)
BUILD_BUG_ON(skb_ext_total_length() > 255);
#endif
skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
}
#else
static void skb_extensions_init(void) {}
#endif
/* The SKB kmem_cache slab is critical for network performance. Never
* merge/alias the slab with similar sized objects. This avoids fragmentation
* that hurts performance of kmem_cache_{alloc,free}_bulk APIs.
*/
#ifndef CONFIG_SLUB_TINY
#define FLAG_SKB_NO_MERGE SLAB_NO_MERGE
#else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */
#define FLAG_SKB_NO_MERGE 0
#endif
void __init skb_init(void)
{
net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
sizeof(struct sk_buff),
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC|
FLAG_SKB_NO_MERGE,
offsetof(struct sk_buff, cb),
sizeof_field(struct sk_buff, cb),
NULL);
net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
sizeof(struct sk_buff_fclones),
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
/* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes.
* struct skb_shared_info is located at the end of skb->head,
* and should not be copied to/from user.
*/
net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
SKB_SMALL_HEAD_CACHE_SIZE,
0,
SLAB_HWCACHE_ALIGN | SLAB_PANIC,
0,
SKB_SMALL_HEAD_HEADROOM,
NULL);
skb_extensions_init();
}
static int
__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
unsigned int recursion_level)
{
int start = skb_headlen(skb);
int i, copy = start - offset;
struct sk_buff *frag_iter;
int elt = 0;
if (unlikely(recursion_level >= 24))
return -EMSGSIZE;
if (copy > 0) {
if (copy > len)
copy = len;
sg_set_buf(sg, skb->data + offset, copy);
elt++;
if ((len -= copy) == 0)
return elt;
offset += copy;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
WARN_ON(start > offset + len);
end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
if ((copy = end - offset) > 0) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
if (unlikely(elt && sg_is_last(&sg[elt - 1])))
return -EMSGSIZE;
if (copy > len)
copy = len;
sg_set_page(&sg[elt], skb_frag_page(frag), copy,
skb_frag_off(frag) + offset - start);
elt++;
if (!(len -= copy))
return elt;
offset += copy;
}
start = end;
}
skb_walk_frags(skb, frag_iter) {
int end, ret;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
if ((copy = end - offset) > 0) {
if (unlikely(elt && sg_is_last(&sg[elt - 1])))
return -EMSGSIZE;
if (copy > len)
copy = len;
ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
copy, recursion_level + 1);
if (unlikely(ret < 0))
return ret;
elt += ret;
if ((len -= copy) == 0)
return elt;
offset += copy;
}
start = end;
}
BUG_ON(len);
return elt;
}
/**
* skb_to_sgvec - Fill a scatter-gather list from a socket buffer
* @skb: Socket buffer containing the buffers to be mapped
* @sg: The scatter-gather list to map into
* @offset: The offset into the buffer's contents to start mapping
* @len: Length of buffer space to be mapped
*
* Fill the specified scatter-gather list with mappings/pointers into a
* region of the buffer space attached to a socket buffer. Returns either
* the number of scatterlist items used, or -EMSGSIZE if the contents
* could not fit.
*/
int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
{
int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
if (nsg <= 0)
return nsg;
sg_mark_end(&sg[nsg - 1]);
return nsg;
}
EXPORT_SYMBOL_GPL(skb_to_sgvec);
/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
* sglist without mark the sg which contain last skb data as the end.
* So the caller can mannipulate sg list as will when padding new data after
* the first call without calling sg_unmark_end to expend sg list.
*
* Scenario to use skb_to_sgvec_nomark:
* 1. sg_init_table
* 2. skb_to_sgvec_nomark(payload1)
* 3. skb_to_sgvec_nomark(payload2)
*
* This is equivalent to:
* 1. sg_init_table
* 2. skb_to_sgvec(payload1)
* 3. sg_unmark_end
* 4. skb_to_sgvec(payload2)
*
* When mapping mutilple payload conditionally, skb_to_sgvec_nomark
* is more preferable.
*/
int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
int offset, int len)
{
return __skb_to_sgvec(skb, sg, offset, len, 0);
}
EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
/**
* skb_cow_data - Check that a socket buffer's data buffers are writable
* @skb: The socket buffer to check.
* @tailbits: Amount of trailing space to be added
* @trailer: Returned pointer to the skb where the @tailbits space begins
*
* Make sure that the data buffers attached to a socket buffer are
* writable. If they are not, private copies are made of the data buffers
* and the socket buffer is set to use these instead.
*
* If @tailbits is given, make sure that there is space to write @tailbits
* bytes of data beyond current end of socket buffer. @trailer will be
* set to point to the skb in which this space begins.
*
* The number of scatterlist elements required to completely map the
* COW'd and extended socket buffer will be returned.
*/
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
{
int copyflag;
int elt;
struct sk_buff *skb1, **skb_p;
/* If skb is cloned or its head is paged, reallocate
* head pulling out all the pages (pages are considered not writable
* at the moment even if they are anonymous).
*/
if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
!__pskb_pull_tail(skb, __skb_pagelen(skb)))
return -ENOMEM;
/* Easy case. Most of packets will go this way. */
if (!skb_has_frag_list(skb)) {
/* A little of trouble, not enough of space for trailer.
* This should not happen, when stack is tuned to generate
* good frames. OK, on miss we reallocate and reserve even more
* space, 128 bytes is fair. */
if (skb_tailroom(skb) < tailbits &&
pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
return -ENOMEM;
/* Voila! */
*trailer = skb;
return 1;
}
/* Misery. We are in troubles, going to mincer fragments... */
elt = 1;
skb_p = &skb_shinfo(skb)->frag_list;
copyflag = 0;
while ((skb1 = *skb_p) != NULL) {
int ntail = 0;
/* The fragment is partially pulled by someone,
* this can happen on input. Copy it and everything
* after it. */
if (skb_shared(skb1))
copyflag = 1;
/* If the skb is the last, worry about trailer. */
if (skb1->next == NULL && tailbits) {
if (skb_shinfo(skb1)->nr_frags ||
skb_has_frag_list(skb1) ||
skb_tailroom(skb1) < tailbits)
ntail = tailbits + 128;
}
if (copyflag ||
skb_cloned(skb1) ||
ntail ||
skb_shinfo(skb1)->nr_frags ||
skb_has_frag_list(skb1)) {
struct sk_buff *skb2;
/* Fuck, we are miserable poor guys... */
if (ntail == 0)
skb2 = skb_copy(skb1, GFP_ATOMIC);
else
skb2 = skb_copy_expand(skb1,
skb_headroom(skb1),
ntail,
GFP_ATOMIC);
if (unlikely(skb2 == NULL))
return -ENOMEM;
if (skb1->sk)
skb_set_owner_w(skb2, skb1->sk);
/* Looking around. Are we still alive?
* OK, link new skb, drop old one */
skb2->next = skb1->next;
*skb_p = skb2;
kfree_skb(skb1);
skb1 = skb2;
}
elt++;
*trailer = skb1;
skb_p = &skb1->next;
}
return elt;
}
EXPORT_SYMBOL_GPL(skb_cow_data);
static void sock_rmem_free(struct sk_buff *skb)
{
struct sock *sk = skb->sk;
atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
}
static void skb_set_err_queue(struct sk_buff *skb)
{
/* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
* So, it is safe to (mis)use it to mark skbs on the error queue.
*/
skb->pkt_type = PACKET_OUTGOING;
BUILD_BUG_ON(PACKET_OUTGOING == 0);
}
/*
* Note: We dont mem charge error packets (no sk_forward_alloc changes)
*/
int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
{
if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
(unsigned int)READ_ONCE(sk->sk_rcvbuf))
return -ENOMEM;
skb_orphan(skb);
skb->sk = sk;
skb->destructor = sock_rmem_free;
atomic_add(skb->truesize, &sk->sk_rmem_alloc);
skb_set_err_queue(skb);
/* before exiting rcu section, make sure dst is refcounted */
skb_dst_force(skb);
skb_queue_tail(&sk->sk_error_queue, skb);
if (!sock_flag(sk, SOCK_DEAD))
sk_error_report(sk);
return 0;
}
EXPORT_SYMBOL(sock_queue_err_skb);
static bool is_icmp_err_skb(const struct sk_buff *skb)
{
return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
}
struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
{
struct sk_buff_head *q = &sk->sk_error_queue;
struct sk_buff *skb, *skb_next = NULL;
bool icmp_next = false;
unsigned long flags;
if (skb_queue_empty_lockless(q))
return NULL;
spin_lock_irqsave(&q->lock, flags);
skb = __skb_dequeue(q);
if (skb && (skb_next = skb_peek(q))) {
icmp_next = is_icmp_err_skb(skb_next);
if (icmp_next)
sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
}
spin_unlock_irqrestore(&q->lock, flags);
if (is_icmp_err_skb(skb) && !icmp_next)
sk->sk_err = 0;
if (skb_next)
sk_error_report(sk);
return skb;
}
EXPORT_SYMBOL(sock_dequeue_err_skb);
/**
* skb_clone_sk - create clone of skb, and take reference to socket
* @skb: the skb to clone
*
* This function creates a clone of a buffer that holds a reference on
* sk_refcnt. Buffers created via this function are meant to be
* returned using sock_queue_err_skb, or free via kfree_skb.
*
* When passing buffers allocated with this function to sock_queue_err_skb
* it is necessary to wrap the call with sock_hold/sock_put in order to
* prevent the socket from being released prior to being enqueued on
* the sk_error_queue.
*/
struct sk_buff *skb_clone_sk(struct sk_buff *skb)
{
struct sock *sk = skb->sk;
struct sk_buff *clone;
if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
return NULL;
clone = skb_clone(skb, GFP_ATOMIC);
if (!clone) {
sock_put(sk);
return NULL;
}
clone->sk = sk;
clone->destructor = sock_efree;
return clone;
}
EXPORT_SYMBOL(skb_clone_sk);
static void __skb_complete_tx_timestamp(struct sk_buff *skb,
struct sock *sk,
int tstype,
bool opt_stats)
{
struct sock_exterr_skb *serr;
int err;
BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
serr = SKB_EXT_ERR(skb);
memset(serr, 0, sizeof(*serr));
serr->ee.ee_errno = ENOMSG;
serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
serr->ee.ee_info = tstype;
serr->opt_stats = opt_stats;
serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
serr->ee.ee_data = skb_shinfo(skb)->tskey;
if (sk_is_tcp(sk))
serr->ee.ee_data -= atomic_read(&sk->sk_tskey);
}
err = sock_queue_err_skb(sk, skb);
if (err)
kfree_skb(skb);
}
static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
{
bool ret;
if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly))
return true;
read_lock_bh(&sk->sk_callback_lock);
ret = sk->sk_socket && sk->sk_socket->file &&
file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
read_unlock_bh(&sk->sk_callback_lock);
return ret;
}
void skb_complete_tx_timestamp(struct sk_buff *skb,
struct skb_shared_hwtstamps *hwtstamps)
{
struct sock *sk = skb->sk;
if (!skb_may_tx_timestamp(sk, false))
goto err;
/* Take a reference to prevent skb_orphan() from freeing the socket,
* but only if the socket refcount is not zero.
*/
if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
*skb_hwtstamps(skb) = *hwtstamps;
__skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
sock_put(sk);
return;
}
err:
kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
void __skb_tstamp_tx(struct sk_buff *orig_skb,
const struct sk_buff *ack_skb,
struct skb_shared_hwtstamps *hwtstamps,
struct sock *sk, int tstype)
{
struct sk_buff *skb;
bool tsonly, opt_stats = false;
u32 tsflags;
if (!sk)
return;
tsflags = READ_ONCE(sk->sk_tsflags);
if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
return;
tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
if (!skb_may_tx_timestamp(sk, tsonly))
return;
if (tsonly) {
#ifdef CONFIG_INET
if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
sk_is_tcp(sk)) {
skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
ack_skb);
opt_stats = true;
} else
#endif
skb = alloc_skb(0, GFP_ATOMIC);
} else {
skb = skb_clone(orig_skb, GFP_ATOMIC);
if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
kfree_skb(skb);
return;
}
}
if (!skb)
return;
if (tsonly) {
skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
SKBTX_ANY_TSTAMP;
skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
}
if (hwtstamps)
*skb_hwtstamps(skb) = *hwtstamps;
else
__net_timestamp(skb);
__skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
}
EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
void skb_tstamp_tx(struct sk_buff *orig_skb,
struct skb_shared_hwtstamps *hwtstamps)
{
return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk,
SCM_TSTAMP_SND);
}
EXPORT_SYMBOL_GPL(skb_tstamp_tx);
#ifdef CONFIG_WIRELESS
void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
{
struct sock *sk = skb->sk;
struct sock_exterr_skb *serr;
int err = 1;
skb->wifi_acked_valid = 1;
skb->wifi_acked = acked;
serr = SKB_EXT_ERR(skb);
memset(serr, 0, sizeof(*serr));
serr->ee.ee_errno = ENOMSG;
serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
/* Take a reference to prevent skb_orphan() from freeing the socket,
* but only if the socket refcount is not zero.
*/
if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
err = sock_queue_err_skb(sk, skb);
sock_put(sk);
}
if (err)
kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
#endif /* CONFIG_WIRELESS */
/**
* skb_partial_csum_set - set up and verify partial csum values for packet
* @skb: the skb to set
* @start: the number of bytes after skb->data to start checksumming.
* @off: the offset from start to place the checksum.
*
* For untrusted partially-checksummed packets, we need to make sure the values
* for skb->csum_start and skb->csum_offset are valid so we don't oops.
*
* This function checks and sets those values and skb->ip_summed: if this
* returns false you should drop the packet.
*/
bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
{
u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
u32 csum_start = skb_headroom(skb) + (u32)start;
if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) {
net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
start, off, skb_headroom(skb), skb_headlen(skb));
return false;
}
skb->ip_summed = CHECKSUM_PARTIAL;
skb->csum_start = csum_start;
skb->csum_offset = off;
skb->transport_header = csum_start;
return true;
}
EXPORT_SYMBOL_GPL(skb_partial_csum_set);
static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
unsigned int max)
{
if (skb_headlen(skb) >= len)
return 0;
/* If we need to pullup then pullup to the max, so we
* won't need to do it again.
*/
if (max > skb->len)
max = skb->len;
if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
return -ENOMEM;
if (skb_headlen(skb) < len)
return -EPROTO;
return 0;
}
#define MAX_TCP_HDR_LEN (15 * 4)
static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
typeof(IPPROTO_IP) proto,
unsigned int off)
{
int err;
switch (proto) {
case IPPROTO_TCP:
err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
off + MAX_TCP_HDR_LEN);
if (!err && !skb_partial_csum_set(skb, off,
offsetof(struct tcphdr,
check)))
err = -EPROTO;
return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
case IPPROTO_UDP:
err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
off + sizeof(struct udphdr));
if (!err && !skb_partial_csum_set(skb, off,
offsetof(struct udphdr,
check)))
err = -EPROTO;
return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
}
return ERR_PTR(-EPROTO);
}
/* This value should be large enough to cover a tagged ethernet header plus
* maximally sized IP and TCP or UDP headers.
*/
#define MAX_IP_HDR_LEN 128
static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
{
unsigned int off;
bool fragment;
__sum16 *csum;
int err;
fragment = false;
err = skb_maybe_pull_tail(skb,
sizeof(struct iphdr),
MAX_IP_HDR_LEN);
if (err < 0)
goto out;
if (ip_is_fragment(ip_hdr(skb)))
fragment = true;
off = ip_hdrlen(skb);
err = -EPROTO;
if (fragment)
goto out;
csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
if (IS_ERR(csum))
return PTR_ERR(csum);
if (recalculate)
*csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
ip_hdr(skb)->daddr,
skb->len - off,
ip_hdr(skb)->protocol, 0);
err = 0;
out:
return err;
}
/* This value should be large enough to cover a tagged ethernet header plus
* an IPv6 header, all options, and a maximal TCP or UDP header.
*/
#define MAX_IPV6_HDR_LEN 256
#define OPT_HDR(type, skb, off) \
(type *)(skb_network_header(skb) + (off))
static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
{
int err;
u8 nexthdr;
unsigned int off;
unsigned int len;
bool fragment;
bool done;
__sum16 *csum;
fragment = false;
done = false;
off = sizeof(struct ipv6hdr);
err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
if (err < 0)
goto out;
nexthdr = ipv6_hdr(skb)->nexthdr;
len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
while (off <= len && !done) {
switch (nexthdr) {
case IPPROTO_DSTOPTS:
case IPPROTO_HOPOPTS:
case IPPROTO_ROUTING: {
struct ipv6_opt_hdr *hp;
err = skb_maybe_pull_tail(skb,
off +
sizeof(struct ipv6_opt_hdr),
MAX_IPV6_HDR_LEN);
if (err < 0)
goto out;
hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
nexthdr = hp->nexthdr;
off += ipv6_optlen(hp);
break;
}
case IPPROTO_AH: {
struct ip_auth_hdr *hp;
err = skb_maybe_pull_tail(skb,
off +
sizeof(struct ip_auth_hdr),
MAX_IPV6_HDR_LEN);
if (err < 0)
goto out;
hp = OPT_HDR(struct ip_auth_hdr, skb, off);
nexthdr = hp->nexthdr;
off += ipv6_authlen(hp);
break;
}
case IPPROTO_FRAGMENT: {
struct frag_hdr *hp;
err = skb_maybe_pull_tail(skb,
off +
sizeof(struct frag_hdr),
MAX_IPV6_HDR_LEN);
if (err < 0)
goto out;
hp = OPT_HDR(struct frag_hdr, skb, off);
if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
fragment = true;
nexthdr = hp->nexthdr;
off += sizeof(struct frag_hdr);
break;
}
default:
done = true;
break;
}
}
err = -EPROTO;
if (!done || fragment)
goto out;
csum = skb_checksum_setup_ip(skb, nexthdr, off);
if (IS_ERR(csum))
return PTR_ERR(csum);
if (recalculate)
*csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
&ipv6_hdr(skb)->daddr,
skb->len - off, nexthdr, 0);
err = 0;
out:
return err;
}
/**
* skb_checksum_setup - set up partial checksum offset
* @skb: the skb to set up
* @recalculate: if true the pseudo-header checksum will be recalculated
*/
int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
{
int err;
switch (skb->protocol) {
case htons(ETH_P_IP):
err = skb_checksum_setup_ipv4(skb, recalculate);
break;
case htons(ETH_P_IPV6):
err = skb_checksum_setup_ipv6(skb, recalculate);
break;
default:
err = -EPROTO;
break;
}
return err;
}
EXPORT_SYMBOL(skb_checksum_setup);
/**
* skb_checksum_maybe_trim - maybe trims the given skb
* @skb: the skb to check
* @transport_len: the data length beyond the network header
*
* Checks whether the given skb has data beyond the given transport length.
* If so, returns a cloned skb trimmed to this transport length.
* Otherwise returns the provided skb. Returns NULL in error cases
* (e.g. transport_len exceeds skb length or out-of-memory).
*
* Caller needs to set the skb transport header and free any returned skb if it
* differs from the provided skb.
*/
static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
unsigned int transport_len)
{
struct sk_buff *skb_chk;
unsigned int len = skb_transport_offset(skb) + transport_len;
int ret;
if (skb->len < len)
return NULL;
else if (skb->len == len)
return skb;
skb_chk = skb_clone(skb, GFP_ATOMIC);
if (!skb_chk)
return NULL;
ret = pskb_trim_rcsum(skb_chk, len);
if (ret) {
kfree_skb(skb_chk);
return NULL;
}
return skb_chk;
}
/**
* skb_checksum_trimmed - validate checksum of an skb
* @skb: the skb to check
* @transport_len: the data length beyond the network header
* @skb_chkf: checksum function to use
*
* Applies the given checksum function skb_chkf to the provided skb.
* Returns a checked and maybe trimmed skb. Returns NULL on error.
*
* If the skb has data beyond the given transport length, then a
* trimmed & cloned skb is checked and returned.
*
* Caller needs to set the skb transport header and free any returned skb if it
* differs from the provided skb.
*/
struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
unsigned int transport_len,
__sum16(*skb_chkf)(struct sk_buff *skb))
{
struct sk_buff *skb_chk;
unsigned int offset = skb_transport_offset(skb);
__sum16 ret;
skb_chk = skb_checksum_maybe_trim(skb, transport_len);
if (!skb_chk)
goto err;
if (!pskb_may_pull(skb_chk, offset))
goto err;
skb_pull_rcsum(skb_chk, offset);
ret = skb_chkf(skb_chk);
skb_push_rcsum(skb_chk, offset);
if (ret)
goto err;
return skb_chk;
err:
if (skb_chk && skb_chk != skb)
kfree_skb(skb_chk);
return NULL;
}
EXPORT_SYMBOL(skb_checksum_trimmed);
void __skb_warn_lro_forwarding(const struct sk_buff *skb)
{
net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
skb->dev->name);
}
EXPORT_SYMBOL(__skb_warn_lro_forwarding);
void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
{
if (head_stolen) {
skb_release_head_state(skb);
kmem_cache_free(net_hotdata.skbuff_cache, skb);
} else {
__kfree_skb(skb);
}
}
EXPORT_SYMBOL(kfree_skb_partial);
/**
* skb_try_coalesce - try to merge skb to prior one
* @to: prior buffer
* @from: buffer to add
* @fragstolen: pointer to boolean
* @delta_truesize: how much more was allocated than was requested
*/
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
bool *fragstolen, int *delta_truesize)
{
struct skb_shared_info *to_shinfo, *from_shinfo;
int i, delta, len = from->len;
*fragstolen = false;
if (skb_cloned(to))
return false;
/* In general, avoid mixing page_pool and non-page_pool allocated
* pages within the same SKB. In theory we could take full
* references if @from is cloned and !@to->pp_recycle but its
* tricky (due to potential race with the clone disappearing) and
* rare, so not worth dealing with.
*/
if (to->pp_recycle != from->pp_recycle)
return false;
if (len <= skb_tailroom(to)) {
if (len)
BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
*delta_truesize = 0;
return true;
}
to_shinfo = skb_shinfo(to);
from_shinfo = skb_shinfo(from);
if (to_shinfo->frag_list || from_shinfo->frag_list)
return false;
if (skb_zcopy(to) || skb_zcopy(from))
return false;
if (skb_headlen(from) != 0) {
struct page *page;
unsigned int offset;
if (to_shinfo->nr_frags +
from_shinfo->nr_frags >= MAX_SKB_FRAGS)
return false;
if (skb_head_is_locked(from))
return false;
delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
page = virt_to_head_page(from->head);
offset = from->data - (unsigned char *)page_address(page);
skb_fill_page_desc(to, to_shinfo->nr_frags,
page, offset, skb_headlen(from));
*fragstolen = true;
} else {
if (to_shinfo->nr_frags +
from_shinfo->nr_frags > MAX_SKB_FRAGS)
return false;
delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
}
WARN_ON_ONCE(delta < len);
memcpy(to_shinfo->frags + to_shinfo->nr_frags,
from_shinfo->frags,
from_shinfo->nr_frags * sizeof(skb_frag_t));
to_shinfo->nr_frags += from_shinfo->nr_frags;
if (!skb_cloned(from))
from_shinfo->nr_frags = 0;
/* if the skb is not cloned this does nothing
* since we set nr_frags to 0.
*/
if (skb_pp_frag_ref(from)) {
for (i = 0; i < from_shinfo->nr_frags; i++)
__skb_frag_ref(&from_shinfo->frags[i]);
}
to->truesize += delta;
to->len += len;
to->data_len += len;
*delta_truesize = delta;
return true;
}
EXPORT_SYMBOL(skb_try_coalesce);
/**
* skb_scrub_packet - scrub an skb
*
* @skb: buffer to clean
* @xnet: packet is crossing netns
*
* skb_scrub_packet can be used after encapsulating or decapsulting a packet
* into/from a tunnel. Some information have to be cleared during these
* operations.
* skb_scrub_packet can also be used to clean a skb before injecting it in
* another namespace (@xnet == true). We have to clear all information in the
* skb that could impact namespace isolation.
*/
void skb_scrub_packet(struct sk_buff *skb, bool xnet)
{
skb->pkt_type = PACKET_HOST;
skb->skb_iif = 0;
skb->ignore_df = 0;
skb_dst_drop(skb);
skb_ext_reset(skb);
nf_reset_ct(skb);
nf_reset_trace(skb);
#ifdef CONFIG_NET_SWITCHDEV
skb->offload_fwd_mark = 0;
skb->offload_l3_fwd_mark = 0;
#endif
if (!xnet)
return;
ipvs_reset(skb);
skb->mark = 0;
skb_clear_tstamp(skb);
}
EXPORT_SYMBOL_GPL(skb_scrub_packet);
static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
{
int mac_len, meta_len;
void *meta;
if (skb_cow(skb, skb_headroom(skb)) < 0) {
kfree_skb(skb);
return NULL;
}
mac_len = skb->data - skb_mac_header(skb);
if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
mac_len - VLAN_HLEN - ETH_TLEN);
}
meta_len = skb_metadata_len(skb);
if (meta_len) {
meta = skb_metadata_end(skb) - meta_len;
memmove(meta + VLAN_HLEN, meta, meta_len);
}
skb->mac_header += VLAN_HLEN;
return skb;
}
struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
{
struct vlan_hdr *vhdr;
u16 vlan_tci;
if (unlikely(skb_vlan_tag_present(skb))) {
/* vlan_tci is already set-up so leave this for another time */
return skb;
}
skb = skb_share_check(skb, GFP_ATOMIC);
if (unlikely(!skb))
goto err_free;
/* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */
if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
goto err_free;
vhdr = (struct vlan_hdr *)skb->data;
vlan_tci = ntohs(vhdr->h_vlan_TCI);
__vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
skb_pull_rcsum(skb, VLAN_HLEN);
vlan_set_encap_proto(skb, vhdr);
skb = skb_reorder_vlan_header(skb);
if (unlikely(!skb))
goto err_free;
skb_reset_network_header(skb);
if (!skb_transport_header_was_set(skb))
skb_reset_transport_header(skb);
skb_reset_mac_len(skb);
return skb;
err_free:
kfree_skb(skb);
return NULL;
}
EXPORT_SYMBOL(skb_vlan_untag);
int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len)
{
if (!pskb_may_pull(skb, write_len))
return -ENOMEM;
if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
return 0;
return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
}
EXPORT_SYMBOL(skb_ensure_writable);
int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev)
{
int needed_headroom = dev->needed_headroom;
int needed_tailroom = dev->needed_tailroom;
/* For tail taggers, we need to pad short frames ourselves, to ensure
* that the tail tag does not fail at its role of being at the end of
* the packet, once the conduit interface pads the frame. Account for
* that pad length here, and pad later.
*/
if (unlikely(needed_tailroom && skb->len < ETH_ZLEN))
needed_tailroom += ETH_ZLEN - skb->len;
/* skb_headroom() returns unsigned int... */
needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0);
needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0);
if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb)))
/* No reallocation needed, yay! */
return 0;
return pskb_expand_head(skb, needed_headroom, needed_tailroom,
GFP_ATOMIC);
}
EXPORT_SYMBOL(skb_ensure_writable_head_tail);
/* remove VLAN header from packet and update csum accordingly.
* expects a non skb_vlan_tag_present skb with a vlan tag payload
*/
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
{
int offset = skb->data - skb_mac_header(skb);
int err;
if (WARN_ONCE(offset,
"__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
offset)) {
return -EINVAL;
}
err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
if (unlikely(err))
return err;
skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
vlan_remove_tag(skb, vlan_tci);
skb->mac_header += VLAN_HLEN;
if (skb_network_offset(skb) < ETH_HLEN)
skb_set_network_header(skb, ETH_HLEN);
skb_reset_mac_len(skb);
return err;
}
EXPORT_SYMBOL(__skb_vlan_pop);
/* Pop a vlan tag either from hwaccel or from payload.
* Expects skb->data at mac header.
*/
int skb_vlan_pop(struct sk_buff *skb)
{
u16 vlan_tci;
__be16 vlan_proto;
int err;
if (likely(skb_vlan_tag_present(skb))) {
__vlan_hwaccel_clear_tag(skb);
} else {
if (unlikely(!eth_type_vlan(skb->protocol)))
return 0;
err = __skb_vlan_pop(skb, &vlan_tci);
if (err)
return err;
}
/* move next vlan tag to hw accel tag */
if (likely(!eth_type_vlan(skb->protocol)))
return 0;
vlan_proto = skb->protocol;
err = __skb_vlan_pop(skb, &vlan_tci);
if (unlikely(err))
return err;
__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
return 0;
}
EXPORT_SYMBOL(skb_vlan_pop);
/* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
* Expects skb->data at mac header.
*/
int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
{
if (skb_vlan_tag_present(skb)) {
int offset = skb->data - skb_mac_header(skb);
int err;
if (WARN_ONCE(offset,
"skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
offset)) {
return -EINVAL;
}
err = __vlan_insert_tag(skb, skb->vlan_proto,
skb_vlan_tag_get(skb));
if (err)
return err;
skb->protocol = skb->vlan_proto;
skb->mac_len += VLAN_HLEN;
skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
}
__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
return 0;
}
EXPORT_SYMBOL(skb_vlan_push);
/**
* skb_eth_pop() - Drop the Ethernet header at the head of a packet
*
* @skb: Socket buffer to modify
*
* Drop the Ethernet header of @skb.
*
* Expects that skb->data points to the mac header and that no VLAN tags are
* present.
*
* Returns 0 on success, -errno otherwise.
*/
int skb_eth_pop(struct sk_buff *skb)
{
if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
skb_network_offset(skb) < ETH_HLEN)
return -EPROTO;
skb_pull_rcsum(skb, ETH_HLEN);
skb_reset_mac_header(skb);
skb_reset_mac_len(skb);
return 0;
}
EXPORT_SYMBOL(skb_eth_pop);
/**
* skb_eth_push() - Add a new Ethernet header at the head of a packet
*
* @skb: Socket buffer to modify
* @dst: Destination MAC address of the new header
* @src: Source MAC address of the new header
*
* Prepend @skb with a new Ethernet header.
*
* Expects that skb->data points to the mac header, which must be empty.
*
* Returns 0 on success, -errno otherwise.
*/
int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
const unsigned char *src)
{
struct ethhdr *eth;
int err;
if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
return -EPROTO;
err = skb_cow_head(skb, sizeof(*eth));
if (err < 0)
return err;
skb_push(skb, sizeof(*eth));
skb_reset_mac_header(skb);
skb_reset_mac_len(skb);
eth = eth_hdr(skb);
ether_addr_copy(eth->h_dest, dst);
ether_addr_copy(eth->h_source, src);
eth->h_proto = skb->protocol;
skb_postpush_rcsum(skb, eth, sizeof(*eth));
return 0;
}
EXPORT_SYMBOL(skb_eth_push);
/* Update the ethertype of hdr and the skb csum value if required. */
static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
__be16 ethertype)
{
if (skb->ip_summed == CHECKSUM_COMPLETE) {
__be16 diff[] = { ~hdr->h_proto, ethertype };
skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
}
hdr->h_proto = ethertype;
}
/**
* skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
* the packet
*
* @skb: buffer
* @mpls_lse: MPLS label stack entry to push
* @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
* @mac_len: length of the MAC header
* @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
* ethernet
*
* Expects skb->data at mac header.
*
* Returns 0 on success, -errno otherwise.
*/
int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
int mac_len, bool ethernet)
{
struct mpls_shim_hdr *lse;
int err;
if (unlikely(!eth_p_mpls(mpls_proto)))
return -EINVAL;
/* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
if (skb->encapsulation)
return -EINVAL;
err = skb_cow_head(skb, MPLS_HLEN);
if (unlikely(err))
return err;
if (!skb->inner_protocol) {
skb_set_inner_network_header(skb, skb_network_offset(skb));
skb_set_inner_protocol(skb, skb->protocol);
}
skb_push(skb, MPLS_HLEN);
memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
mac_len);
skb_reset_mac_header(skb);
skb_set_network_header(skb, mac_len);
skb_reset_mac_len(skb);
lse = mpls_hdr(skb);
lse->label_stack_entry = mpls_lse;
skb_postpush_rcsum(skb, lse, MPLS_HLEN);
if (ethernet && mac_len >= ETH_HLEN)
skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
skb->protocol = mpls_proto;
return 0;
}
EXPORT_SYMBOL_GPL(skb_mpls_push);
/**
* skb_mpls_pop() - pop the outermost MPLS header
*
* @skb: buffer
* @next_proto: ethertype of header after popped MPLS header
* @mac_len: length of the MAC header
* @ethernet: flag to indicate if the packet is ethernet
*
* Expects skb->data at mac header.
*
* Returns 0 on success, -errno otherwise.
*/
int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
bool ethernet)
{
int err;
if (unlikely(!eth_p_mpls(skb->protocol)))
return 0;
err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
if (unlikely(err))
return err;
skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
mac_len);
__skb_pull(skb, MPLS_HLEN);
skb_reset_mac_header(skb);
skb_set_network_header(skb, mac_len);
if (ethernet && mac_len >= ETH_HLEN) {
struct ethhdr *hdr;
/* use mpls_hdr() to get ethertype to account for VLANs. */
hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
skb_mod_eth_type(skb, hdr, next_proto);
}
skb->protocol = next_proto;
return 0;
}
EXPORT_SYMBOL_GPL(skb_mpls_pop);
/**
* skb_mpls_update_lse() - modify outermost MPLS header and update csum
*
* @skb: buffer
* @mpls_lse: new MPLS label stack entry to update to
*
* Expects skb->data at mac header.
*
* Returns 0 on success, -errno otherwise.
*/
int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
{
int err;
if (unlikely(!eth_p_mpls(skb->protocol)))
return -EINVAL;
err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
if (unlikely(err))
return err;
if (skb->ip_summed == CHECKSUM_COMPLETE) {
__be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
}
mpls_hdr(skb)->label_stack_entry = mpls_lse;
return 0;
}
EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
/**
* skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
*
* @skb: buffer
*
* Expects skb->data at mac header.
*
* Returns 0 on success, -errno otherwise.
*/
int skb_mpls_dec_ttl(struct sk_buff *skb)
{
u32 lse;
u8 ttl;
if (unlikely(!eth_p_mpls(skb->protocol)))
return -EINVAL;
if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
return -ENOMEM;
lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
if (!--ttl)
return -EINVAL;
lse &= ~MPLS_LS_TTL_MASK;
lse |= ttl << MPLS_LS_TTL_SHIFT;
return skb_mpls_update_lse(skb, cpu_to_be32(lse));
}
EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
/**
* alloc_skb_with_frags - allocate skb with page frags
*
* @header_len: size of linear part
* @data_len: needed length in frags
* @order: max page order desired.
* @errcode: pointer to error code if any
* @gfp_mask: allocation mask
*
* This can be used to allocate a paged skb, given a maximal order for frags.
*/
struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
unsigned long data_len,
int order,
int *errcode,
gfp_t gfp_mask)
{
unsigned long chunk;
struct sk_buff *skb;
struct page *page;
int nr_frags = 0;
*errcode = -EMSGSIZE;
if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order)))
return NULL;
*errcode = -ENOBUFS;
skb = alloc_skb(header_len, gfp_mask);
if (!skb)
return NULL;
while (data_len) {
if (nr_frags == MAX_SKB_FRAGS - 1)
goto failure;
while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order))
order--;
if (order) {
page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
__GFP_COMP |
__GFP_NOWARN,
order);
if (!page) {
order--;
continue;
}
} else {
page = alloc_page(gfp_mask);
if (!page)
goto failure;
}
chunk = min_t(unsigned long, data_len,
PAGE_SIZE << order);
skb_fill_page_desc(skb, nr_frags, page, 0, chunk);
nr_frags++;
skb->truesize += (PAGE_SIZE << order);
data_len -= chunk;
}
return skb;
failure:
kfree_skb(skb);
return NULL;
}
EXPORT_SYMBOL(alloc_skb_with_frags);
/* carve out the first off bytes from skb when off < headlen */
static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
const int headlen, gfp_t gfp_mask)
{
int i;
unsigned int size = skb_end_offset(skb);
int new_hlen = headlen - off;
u8 *data;
if (skb_pfmemalloc(skb))
gfp_mask |= __GFP_MEMALLOC;
data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
if (!data)
return -ENOMEM;
size = SKB_WITH_OVERHEAD(size);
/* Copy real data, and all frags */
skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
skb->len -= off;
memcpy((struct skb_shared_info *)(data + size),
skb_shinfo(skb),
offsetof(struct skb_shared_info,
frags[skb_shinfo(skb)->nr_frags]));
if (skb_cloned(skb)) {
/* drop the old head gracefully */
if (skb_orphan_frags(skb, gfp_mask)) {
skb_kfree_head(data, size);
return -ENOMEM;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
skb_frag_ref(skb, i);
if (skb_has_frag_list(skb))
skb_clone_fraglist(skb);
skb_release_data(skb, SKB_CONSUMED, false);
} else {
/* we can reuse existing recount- all we did was
* relocate values
*/
skb_free_head(skb, false);
}
skb->head = data;
skb->data = data;
skb->head_frag = 0;
skb_set_end_offset(skb, size);
skb_set_tail_pointer(skb, skb_headlen(skb));
skb_headers_offset_update(skb, 0);
skb->cloned = 0;
skb->hdr_len = 0;
skb->nohdr = 0;
atomic_set(&skb_shinfo(skb)->dataref, 1);
return 0;
}
static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
/* carve out the first eat bytes from skb's frag_list. May recurse into
* pskb_carve()
*/
static int pskb_carve_frag_list(struct sk_buff *skb,
struct skb_shared_info *shinfo, int eat,
gfp_t gfp_mask)
{
struct sk_buff *list = shinfo->frag_list;
struct sk_buff *clone = NULL;
struct sk_buff *insp = NULL;
do {
if (!list) {
pr_err("Not enough bytes to eat. Want %d\n", eat);
return -EFAULT;
}
if (list->len <= eat) {
/* Eaten as whole. */
eat -= list->len;
list = list->next;
insp = list;
} else {
/* Eaten partially. */
if (skb_shared(list)) {
clone = skb_clone(list, gfp_mask);
if (!clone)
return -ENOMEM;
insp = list->next;
list = clone;
} else {
/* This may be pulled without problems. */
insp = list;
}
if (pskb_carve(list, eat, gfp_mask) < 0) {
kfree_skb(clone);
return -ENOMEM;
}
break;
}
} while (eat);
/* Free pulled out fragments. */
while ((list = shinfo->frag_list) != insp) {
shinfo->frag_list = list->next;
consume_skb(list);
}
/* And insert new clone at head. */
if (clone) {
clone->next = list;
shinfo->frag_list = clone;
}
return 0;
}
/* carve off first len bytes from skb. Split line (off) is in the
* non-linear part of skb
*/
static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
int pos, gfp_t gfp_mask)
{
int i, k = 0;
unsigned int size = skb_end_offset(skb);
u8 *data;
const int nfrags = skb_shinfo(skb)->nr_frags;
struct skb_shared_info *shinfo;
if (skb_pfmemalloc(skb))
gfp_mask |= __GFP_MEMALLOC;
data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
if (!data)
return -ENOMEM;
size = SKB_WITH_OVERHEAD(size);
memcpy((struct skb_shared_info *)(data + size),
skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
if (skb_orphan_frags(skb, gfp_mask)) {
skb_kfree_head(data, size);
return -ENOMEM;
}
shinfo = (struct skb_shared_info *)(data + size);
for (i = 0; i < nfrags; i++) {
int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
if (pos + fsize > off) {
shinfo->frags[k] = skb_shinfo(skb)->frags[i];
if (pos < off) {
/* Split frag.
* We have two variants in this case:
* 1. Move all the frag to the second
* part, if it is possible. F.e.
* this approach is mandatory for TUX,
* where splitting is expensive.
* 2. Split is accurately. We make this.
*/
skb_frag_off_add(&shinfo->frags[0], off - pos);
skb_frag_size_sub(&shinfo->frags[0], off - pos);
}
skb_frag_ref(skb, i);
k++;
}
pos += fsize;
}
shinfo->nr_frags = k;
if (skb_has_frag_list(skb))
skb_clone_fraglist(skb);
/* split line is in frag list */
if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
/* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
if (skb_has_frag_list(skb))
kfree_skb_list(skb_shinfo(skb)->frag_list);
skb_kfree_head(data, size);
return -ENOMEM;
}
skb_release_data(skb, SKB_CONSUMED, false);
skb->head = data;
skb->head_frag = 0;
skb->data = data;
skb_set_end_offset(skb, size);
skb_reset_tail_pointer(skb);
skb_headers_offset_update(skb, 0);
skb->cloned = 0;
skb->hdr_len = 0;
skb->nohdr = 0;
skb->len -= off;
skb->data_len = skb->len;
atomic_set(&skb_shinfo(skb)->dataref, 1);
return 0;
}
/* remove len bytes from the beginning of the skb */
static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
{
int headlen = skb_headlen(skb);
if (len < headlen)
return pskb_carve_inside_header(skb, len, headlen, gfp);
else
return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
}
/* Extract to_copy bytes starting at off from skb, and return this in
* a new skb
*/
struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
int to_copy, gfp_t gfp)
{
struct sk_buff *clone = skb_clone(skb, gfp);
if (!clone)
return NULL;
if (pskb_carve(clone, off, gfp) < 0 ||
pskb_trim(clone, to_copy)) {
kfree_skb(clone);
return NULL;
}
return clone;
}
EXPORT_SYMBOL(pskb_extract);
/**
* skb_condense - try to get rid of fragments/frag_list if possible
* @skb: buffer
*
* Can be used to save memory before skb is added to a busy queue.
* If packet has bytes in frags and enough tail room in skb->head,
* pull all of them, so that we can free the frags right now and adjust
* truesize.
* Notes:
* We do not reallocate skb->head thus can not fail.
* Caller must re-evaluate skb->truesize if needed.
*/
void skb_condense(struct sk_buff *skb)
{
if (skb->data_len) {
if (skb->data_len > skb->end - skb->tail ||
skb_cloned(skb))
return;
/* Nice, we can free page frag(s) right now */
__pskb_pull_tail(skb, skb->data_len);
}
/* At this point, skb->truesize might be over estimated,
* because skb had a fragment, and fragments do not tell
* their truesize.
* When we pulled its content into skb->head, fragment
* was freed, but __pskb_pull_tail() could not possibly
* adjust skb->truesize, not knowing the frag truesize.
*/
skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
}
EXPORT_SYMBOL(skb_condense);
#ifdef CONFIG_SKB_EXTENSIONS
static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
{
return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
}
/**
* __skb_ext_alloc - allocate a new skb extensions storage
*
* @flags: See kmalloc().
*
* Returns the newly allocated pointer. The pointer can later attached to a
* skb via __skb_ext_set().
* Note: caller must handle the skb_ext as an opaque data.
*/
struct skb_ext *__skb_ext_alloc(gfp_t flags)
{
struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
if (new) {
memset(new->offset, 0, sizeof(new->offset));
refcount_set(&new->refcnt, 1);
}
return new;
}
static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
unsigned int old_active)
{
struct skb_ext *new;
if (refcount_read(&old->refcnt) == 1)
return old;
new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
if (!new)
return NULL;
memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
refcount_set(&new->refcnt, 1);
#ifdef CONFIG_XFRM
if (old_active & (1 << SKB_EXT_SEC_PATH)) {
struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
unsigned int i;
for (i = 0; i < sp->len; i++)
xfrm_state_hold(sp->xvec[i]);
}
#endif
#ifdef CONFIG_MCTP_FLOWS
if (old_active & (1 << SKB_EXT_MCTP)) {
struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP);
if (flow->key)
refcount_inc(&flow->key->refs);
}
#endif
__skb_ext_put(old);
return new;
}
/**
* __skb_ext_set - attach the specified extension storage to this skb
* @skb: buffer
* @id: extension id
* @ext: extension storage previously allocated via __skb_ext_alloc()
*
* Existing extensions, if any, are cleared.
*
* Returns the pointer to the extension.
*/
void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
struct skb_ext *ext)
{
unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
skb_ext_put(skb);
newlen = newoff + skb_ext_type_len[id];
ext->chunks = newlen;
ext->offset[id] = newoff;
skb->extensions = ext;
skb->active_extensions = 1 << id;
return skb_ext_get_ptr(ext, id);
}
/**
* skb_ext_add - allocate space for given extension, COW if needed
* @skb: buffer
* @id: extension to allocate space for
*
* Allocates enough space for the given extension.
* If the extension is already present, a pointer to that extension
* is returned.
*
* If the skb was cloned, COW applies and the returned memory can be
* modified without changing the extension space of clones buffers.
*
* Returns pointer to the extension or NULL on allocation failure.
*/
void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
{
struct skb_ext *new, *old = NULL;
unsigned int newlen, newoff;
if (skb->active_extensions) {
old = skb->extensions;
new = skb_ext_maybe_cow(old, skb->active_extensions);
if (!new)
return NULL;
if (__skb_ext_exist(new, id))
goto set_active;
newoff = new->chunks;
} else {
newoff = SKB_EXT_CHUNKSIZEOF(*new);
new = __skb_ext_alloc(GFP_ATOMIC);
if (!new)
return NULL;
}
newlen = newoff + skb_ext_type_len[id];
new->chunks = newlen;
new->offset[id] = newoff;
set_active:
skb->slow_gro = 1;
skb->extensions = new;
skb->active_extensions |= 1 << id;
return skb_ext_get_ptr(new, id);
}
EXPORT_SYMBOL(skb_ext_add);
#ifdef CONFIG_XFRM
static void skb_ext_put_sp(struct sec_path *sp)
{
unsigned int i;
for (i = 0; i < sp->len; i++)
xfrm_state_put(sp->xvec[i]);
}
#endif
#ifdef CONFIG_MCTP_FLOWS
static void skb_ext_put_mctp(struct mctp_flow *flow)
{
if (flow->key)
mctp_key_unref(flow->key);
}
#endif
void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
{
struct skb_ext *ext = skb->extensions;
skb->active_extensions &= ~(1 << id);
if (skb->active_extensions == 0) {
skb->extensions = NULL;
__skb_ext_put(ext);
#ifdef CONFIG_XFRM
} else if (id == SKB_EXT_SEC_PATH &&
refcount_read(&ext->refcnt) == 1) {
struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
skb_ext_put_sp(sp);
sp->len = 0;
#endif
}
}
EXPORT_SYMBOL(__skb_ext_del);
void __skb_ext_put(struct skb_ext *ext)
{
/* If this is last clone, nothing can increment
* it after check passes. Avoids one atomic op.
*/
if (refcount_read(&ext->refcnt) == 1)
goto free_now;
if (!refcount_dec_and_test(&ext->refcnt))
return;
free_now:
#ifdef CONFIG_XFRM
if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
#endif
#ifdef CONFIG_MCTP_FLOWS
if (__skb_ext_exist(ext, SKB_EXT_MCTP))
skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
#endif
kmem_cache_free(skbuff_ext_cache, ext);
}
EXPORT_SYMBOL(__skb_ext_put);
#endif /* CONFIG_SKB_EXTENSIONS */
/**
* skb_attempt_defer_free - queue skb for remote freeing
* @skb: buffer
*
* Put @skb in a per-cpu list, using the cpu which
* allocated the skb/pages to reduce false sharing
* and memory zone spinlock contention.
*/
void skb_attempt_defer_free(struct sk_buff *skb)
{
int cpu = skb->alloc_cpu;
struct softnet_data *sd;
unsigned int defer_max;
bool kick;
if (WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
!cpu_online(cpu) ||
cpu == raw_smp_processor_id()) {
nodefer: __kfree_skb(skb);
return;
}
DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
DEBUG_NET_WARN_ON_ONCE(skb->destructor);
sd = &per_cpu(softnet_data, cpu);
defer_max = READ_ONCE(sysctl_skb_defer_max);
if (READ_ONCE(sd->defer_count) >= defer_max)
goto nodefer;
spin_lock_bh(&sd->defer_lock);
/* Send an IPI every time queue reaches half capacity. */
kick = sd->defer_count == (defer_max >> 1);
/* Paired with the READ_ONCE() few lines above */
WRITE_ONCE(sd->defer_count, sd->defer_count + 1);
skb->next = sd->defer_list;
/* Paired with READ_ONCE() in skb_defer_free_flush() */
WRITE_ONCE(sd->defer_list, skb);
spin_unlock_bh(&sd->defer_lock);
/* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
* if we are unlucky enough (this seems very unlikely).
*/
if (unlikely(kick) && !cmpxchg(&sd->defer_ipi_scheduled, 0, 1))
smp_call_function_single_async(cpu, &sd->defer_csd);
}
static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
size_t offset, size_t len)
{
const char *kaddr;
__wsum csum;
kaddr = kmap_local_page(page);
csum = csum_partial(kaddr + offset, len, 0);
kunmap_local(kaddr);
skb->csum = csum_block_add(skb->csum, csum, skb->len);
}
/**
* skb_splice_from_iter - Splice (or copy) pages to skbuff
* @skb: The buffer to add pages to
* @iter: Iterator representing the pages to be added
* @maxsize: Maximum amount of pages to be added
* @gfp: Allocation flags
*
* This is a common helper function for supporting MSG_SPLICE_PAGES. It
* extracts pages from an iterator and adds them to the socket buffer if
* possible, copying them to fragments if not possible (such as if they're slab
* pages).
*
* Returns the amount of data spliced/copied or -EMSGSIZE if there's
* insufficient space in the buffer to transfer anything.
*/
ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
ssize_t maxsize, gfp_t gfp)
{
size_t frag_limit = READ_ONCE(sysctl_max_skb_frags);
struct page *pages[8], **ppages = pages;
ssize_t spliced = 0, ret = 0;
unsigned int i;
while (iter->count > 0) {
ssize_t space, nr, len;
size_t off;
ret = -EMSGSIZE;
space = frag_limit - skb_shinfo(skb)->nr_frags;
if (space < 0)
break;
/* We might be able to coalesce without increasing nr_frags */
nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages));
len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off);
if (len <= 0) {
ret = len ?: -EIO;
break;
}
i = 0;
do {
struct page *page = pages[i++];
size_t part = min_t(size_t, PAGE_SIZE - off, len);
ret = -EIO;
if (WARN_ON_ONCE(!sendpage_ok(page)))
goto out;
ret = skb_append_pagefrags(skb, page, off, part,
frag_limit);
if (ret < 0) {
iov_iter_revert(iter, len);
goto out;
}
if (skb->ip_summed == CHECKSUM_NONE)
skb_splice_csum_page(skb, page, off, part);
off = 0;
spliced += part;
maxsize -= part;
len -= part;
} while (len > 0);
if (maxsize <= 0)
break;
}
out:
skb_len_add(skb, spliced);
return spliced ?: ret;
}
EXPORT_SYMBOL(skb_splice_from_iter);
static __always_inline
size_t memcpy_from_iter_csum(void *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
__wsum *csum = priv2;
__wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len);
*csum = csum_block_add(*csum, next, progress);
return 0;
}
static __always_inline
size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
__wsum next, *csum = priv2;
next = csum_and_copy_from_user(iter_from, to + progress, len);
*csum = csum_block_add(*csum, next, progress);
return next ? 0 : len;
}
bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
__wsum *csum, struct iov_iter *i)
{
size_t copied;
if (WARN_ON_ONCE(!i->data_source))
return false;
copied = iterate_and_advance2(i, bytes, addr, csum,
copy_from_user_iter_csum,
memcpy_from_iter_csum);
if (likely(copied == bytes))
return true;
iov_iter_revert(i, copied);
return false;
}
EXPORT_SYMBOL(csum_and_copy_from_iter_full);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H
#include <asm/rmwcc.h>
#include <asm/percpu.h>
#include <asm/current.h>
#include <linux/static_call_types.h>
/* We use the MSB mostly because its available */
#define PREEMPT_NEED_RESCHED 0x80000000
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
* that a decrement hitting 0 means we can and should reschedule.
*/
#define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED)
/*
* We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
* that think a non-zero value indicates we cannot preempt.
*/
static __always_inline int preempt_count(void)
{
return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED;
}
static __always_inline void preempt_count_set(int pc)
{
int old, new;
old = raw_cpu_read_4(pcpu_hot.preempt_count);
do {
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
} while (!raw_cpu_try_cmpxchg_4(pcpu_hot.preempt_count, &old, new));
}
/*
* must be macros to avoid header recursion hell
*/
#define init_task_preempt_count(p) do { } while (0)
#define init_idle_preempt_count(p, cpu) do { \
per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \
} while (0)
/*
* We fold the NEED_RESCHED bit into the preempt count such that
* preempt_enable() can decrement and test for needing to reschedule with a
* single instruction.
*
* We invert the actual bit, so that when the decrement hits 0 we know we both
* need to resched (the bit is cleared) and can resched (no preempt count).
*/
static __always_inline void set_preempt_need_resched(void)
{
raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED);
}
static __always_inline void clear_preempt_need_resched(void)
{
raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED);
}
static __always_inline bool test_preempt_need_resched(void)
{
return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED);
}
/*
* The various preempt_count add/sub methods
*/
static __always_inline void __preempt_count_add(int val)
{
raw_cpu_add_4(pcpu_hot.preempt_count, val);
}
static __always_inline void __preempt_count_sub(int val)
{
raw_cpu_add_4(pcpu_hot.preempt_count, -val);
}
/*
* Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
* a decrement which hits zero means we have no preempt_count and should
* reschedule.
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.preempt_count), e,
__percpu_arg([var]));
}
/*
* Returns true when we need to resched and can (barring IRQ state).
*/
static __always_inline bool should_resched(int preempt_offset)
{
return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset);
}
#ifdef CONFIG_PREEMPTION
extern asmlinkage void preempt_schedule(void);
extern asmlinkage void preempt_schedule_thunk(void);
#define preempt_schedule_dynamic_enabled preempt_schedule_thunk
#define preempt_schedule_dynamic_disabled NULL
extern asmlinkage void preempt_schedule_notrace(void);
extern asmlinkage void preempt_schedule_notrace_thunk(void);
#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace_thunk
#define preempt_schedule_notrace_dynamic_disabled NULL
#ifdef CONFIG_PREEMPT_DYNAMIC
DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
#define __preempt_schedule() \
do { \
__STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \
asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \
} while (0)
DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
#define __preempt_schedule_notrace() \
do { \
__STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \
asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \
} while (0)
#else /* PREEMPT_DYNAMIC */
#define __preempt_schedule() \
asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT);
#define __preempt_schedule_notrace() \
asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT);
#endif /* PREEMPT_DYNAMIC */
#endif /* PREEMPTION */
#endif /* __ASM_PREEMPT_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_SIGNAL_H
#define _LINUX_SCHED_SIGNAL_H
#include <linux/rculist.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/sched/jobctl.h>
#include <linux/sched/task.h>
#include <linux/cred.h>
#include <linux/refcount.h>
#include <linux/pid.h>
#include <linux/posix-timers.h>
#include <linux/mm_types.h>
#include <asm/ptrace.h>
/*
* Types defining task->signal and task->sighand and APIs using them:
*/
struct sighand_struct {
spinlock_t siglock;
refcount_t count;
wait_queue_head_t signalfd_wqh;
struct k_sigaction action[_NSIG];
};
/*
* Per-process accounting stats:
*/
struct pacct_struct {
int ac_flag;
long ac_exitcode;
unsigned long ac_mem;
u64 ac_utime, ac_stime;
unsigned long ac_minflt, ac_majflt;
};
struct cpu_itimer {
u64 expires;
u64 incr;
};
/*
* This is the atomic variant of task_cputime, which can be used for
* storing and updating task_cputime statistics without locking.
*/
struct task_cputime_atomic {
atomic64_t utime;
atomic64_t stime;
atomic64_t sum_exec_runtime;
};
#define INIT_CPUTIME_ATOMIC \
(struct task_cputime_atomic) { \
.utime = ATOMIC64_INIT(0), \
.stime = ATOMIC64_INIT(0), \
.sum_exec_runtime = ATOMIC64_INIT(0), \
}
/**
* struct thread_group_cputimer - thread group interval timer counts
* @cputime_atomic: atomic thread group interval timers.
*
* This structure contains the version of task_cputime, above, that is
* used for thread group CPU timer calculations.
*/
struct thread_group_cputimer {
struct task_cputime_atomic cputime_atomic;
};
struct multiprocess_signals {
sigset_t signal;
struct hlist_node node;
};
struct core_thread {
struct task_struct *task;
struct core_thread *next;
};
struct core_state {
atomic_t nr_threads;
struct core_thread dumper;
struct completion startup;
};
/*
* NOTE! "signal_struct" does not have its own
* locking, because a shared signal_struct always
* implies a shared sighand_struct, so locking
* sighand_struct is always a proper superset of
* the locking of signal_struct.
*/
struct signal_struct {
refcount_t sigcnt;
atomic_t live;
int nr_threads;
int quick_threads;
struct list_head thread_head;
wait_queue_head_t wait_chldexit; /* for wait4() */
/* current thread group signal load-balancing target: */
struct task_struct *curr_target;
/* shared signal handling: */
struct sigpending shared_pending;
/* For collecting multiprocess signals during fork */
struct hlist_head multiprocess;
/* thread group exit support */
int group_exit_code;
/* notify group_exec_task when notify_count is less or equal to 0 */
int notify_count;
struct task_struct *group_exec_task;
/* thread group stop support, overloads group_exit_code too */
int group_stop_count;
unsigned int flags; /* see SIGNAL_* flags below */
struct core_state *core_state; /* coredumping support */
/*
* PR_SET_CHILD_SUBREAPER marks a process, like a service
* manager, to re-parent orphan (double-forking) child processes
* to this process instead of 'init'. The service manager is
* able to receive SIGCHLD signals and is able to investigate
* the process until it calls wait(). All children of this
* process will inherit a flag if they should look for a
* child_subreaper process at exit.
*/
unsigned int is_child_subreaper:1;
unsigned int has_child_subreaper:1;
#ifdef CONFIG_POSIX_TIMERS
/* POSIX.1b Interval Timers */
unsigned int next_posix_timer_id;
struct list_head posix_timers;
/* ITIMER_REAL timer for the process */
struct hrtimer real_timer;
ktime_t it_real_incr;
/*
* ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
* CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
* values are defined to 0 and 1 respectively
*/
struct cpu_itimer it[2];
/*
* Thread group totals for process CPU timers.
* See thread_group_cputimer(), et al, for details.
*/
struct thread_group_cputimer cputimer;
#endif
/* Empty if CONFIG_POSIX_TIMERS=n */
struct posix_cputimers posix_cputimers;
/* PID/PID hash table linkage. */
struct pid *pids[PIDTYPE_MAX];
#ifdef CONFIG_NO_HZ_FULL
atomic_t tick_dep_mask;
#endif
struct pid *tty_old_pgrp;
/* boolean value for session group leader */
int leader;
struct tty_struct *tty; /* NULL if no tty */
#ifdef CONFIG_SCHED_AUTOGROUP
struct autogroup *autogroup;
#endif
/*
* Cumulative resource counters for dead threads in the group,
* and for reaped dead child processes forked by this group.
* Live threads maintain their own counters and add to these
* in __exit_signal, except for the group leader.
*/
seqlock_t stats_lock;
u64 utime, stime, cutime, cstime;
u64 gtime;
u64 cgtime;
struct prev_cputime prev_cputime;
unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
unsigned long inblock, oublock, cinblock, coublock;
unsigned long maxrss, cmaxrss;
struct task_io_accounting ioac;
/*
* Cumulative ns of schedule CPU time fo dead threads in the
* group, not including a zombie group leader, (This only differs
* from jiffies_to_ns(utime + stime) if sched_clock uses something
* other than jiffies.)
*/
unsigned long long sum_sched_runtime;
/*
* We don't bother to synchronize most readers of this at all,
* because there is no reader checking a limit that actually needs
* to get both rlim_cur and rlim_max atomically, and either one
* alone is a single word that can safely be read normally.
* getrlimit/setrlimit use task_lock(current->group_leader) to
* protect this instead of the siglock, because they really
* have no need to disable irqs.
*/
struct rlimit rlim[RLIM_NLIMITS];
#ifdef CONFIG_BSD_PROCESS_ACCT
struct pacct_struct pacct; /* per-process accounting information */
#endif
#ifdef CONFIG_TASKSTATS
struct taskstats *stats;
#endif
#ifdef CONFIG_AUDIT
unsigned audit_tty;
struct tty_audit_buf *tty_audit_buf;
#endif
/*
* Thread is the potential origin of an oom condition; kill first on
* oom
*/
bool oom_flag_origin;
short oom_score_adj; /* OOM kill score adjustment */
short oom_score_adj_min; /* OOM kill score adjustment min value.
* Only settable by CAP_SYS_RESOURCE. */
struct mm_struct *oom_mm; /* recorded mm when the thread group got
* killed by the oom killer */
struct mutex cred_guard_mutex; /* guard against foreign influences on
* credential calculations
* (notably. ptrace)
* Deprecated do not use in new code.
* Use exec_update_lock instead.
*/
struct rw_semaphore exec_update_lock; /* Held while task_struct is
* being updated during exec,
* and may have inconsistent
* permissions.
*/
} __randomize_layout;
/*
* Bits in flags field of signal_struct.
*/
#define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */
#define SIGNAL_STOP_CONTINUED 0x00000002 /* SIGCONT since WCONTINUED reap */
#define SIGNAL_GROUP_EXIT 0x00000004 /* group exit in progress */
/*
* Pending notifications to parent.
*/
#define SIGNAL_CLD_STOPPED 0x00000010
#define SIGNAL_CLD_CONTINUED 0x00000020
#define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
#define SIGNAL_UNKILLABLE 0x00000040 /* for init: ignore fatal signals */
#define SIGNAL_STOP_MASK (SIGNAL_CLD_MASK | SIGNAL_STOP_STOPPED | \
SIGNAL_STOP_CONTINUED)
static inline void signal_set_stop_flags(struct signal_struct *sig,
unsigned int flags)
{
WARN_ON(sig->flags & SIGNAL_GROUP_EXIT); sig->flags = (sig->flags & ~SIGNAL_STOP_MASK) | flags;
}
extern void flush_signals(struct task_struct *);
extern void ignore_signals(struct task_struct *);
extern void flush_signal_handlers(struct task_struct *, int force_default);
extern int dequeue_signal(struct task_struct *task, sigset_t *mask,
kernel_siginfo_t *info, enum pid_type *type);
static inline int kernel_dequeue_signal(void)
{
struct task_struct *task = current;
kernel_siginfo_t __info;
enum pid_type __type;
int ret;
spin_lock_irq(&task->sighand->siglock);
ret = dequeue_signal(task, &task->blocked, &__info, &__type);
spin_unlock_irq(&task->sighand->siglock);
return ret;
}
static inline void kernel_signal_stop(void)
{
spin_lock_irq(¤t->sighand->siglock);
if (current->jobctl & JOBCTL_STOP_DEQUEUED) {
current->jobctl |= JOBCTL_STOPPED;
set_special_state(TASK_STOPPED);
}
spin_unlock_irq(¤t->sighand->siglock);
schedule();
}
int force_sig_fault_to_task(int sig, int code, void __user *addr,
struct task_struct *t);
int force_sig_fault(int sig, int code, void __user *addr);
int send_sig_fault(int sig, int code, void __user *addr, struct task_struct *t);
int force_sig_mceerr(int code, void __user *, short);
int send_sig_mceerr(int code, void __user *, short, struct task_struct *);
int force_sig_bnderr(void __user *addr, void __user *lower, void __user *upper);
int force_sig_pkuerr(void __user *addr, u32 pkey);
int send_sig_perf(void __user *addr, u32 type, u64 sig_data);
int force_sig_ptrace_errno_trap(int errno, void __user *addr);
int force_sig_fault_trapno(int sig, int code, void __user *addr, int trapno);
int send_sig_fault_trapno(int sig, int code, void __user *addr, int trapno,
struct task_struct *t);
int force_sig_seccomp(int syscall, int reason, bool force_coredump);
extern int send_sig_info(int, struct kernel_siginfo *, struct task_struct *);
extern void force_sigsegv(int sig);
extern int force_sig_info(struct kernel_siginfo *);
extern int __kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp);
extern int kill_pid_info(int sig, struct kernel_siginfo *info, struct pid *pid);
extern int kill_pid_usb_asyncio(int sig, int errno, sigval_t addr, struct pid *,
const struct cred *);
extern int kill_pgrp(struct pid *pid, int sig, int priv);
extern int kill_pid(struct pid *pid, int sig, int priv);
extern __must_check bool do_notify_parent(struct task_struct *, int);
extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
extern void force_sig(int);
extern void force_fatal_sig(int);
extern void force_exit_sig(int);
extern int send_sig(int, struct task_struct *, int);
extern int zap_other_threads(struct task_struct *p);
extern struct sigqueue *sigqueue_alloc(void);
extern void sigqueue_free(struct sigqueue *);
extern int send_sigqueue(struct sigqueue *, struct pid *, enum pid_type);
extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
static inline void clear_notify_signal(void)
{
clear_thread_flag(TIF_NOTIFY_SIGNAL);
smp_mb__after_atomic();
}
/*
* Returns 'true' if kick_process() is needed to force a transition from
* user -> kernel to guarantee expedient run of TWA_SIGNAL based task_work.
*/
static inline bool __set_notify_signal(struct task_struct *task)
{
return !test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && !wake_up_state(task, TASK_INTERRUPTIBLE);
}
/*
* Called to break out of interruptible wait loops, and enter the
* exit_to_user_mode_loop().
*/
static inline void set_notify_signal(struct task_struct *task)
{
if (__set_notify_signal(task))
kick_process(task);
}
static inline int restart_syscall(void)
{
set_tsk_thread_flag(current, TIF_SIGPENDING);
return -ERESTARTNOINTR;
}
static inline int task_sigpending(struct task_struct *p)
{
return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
}
static inline int signal_pending(struct task_struct *p)
{
/*
* TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same
* behavior in terms of ensuring that we break out of wait loops
* so that notify signal callbacks can be processed.
*/
if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)))
return 1;
return task_sigpending(p);
}
static inline int __fatal_signal_pending(struct task_struct *p)
{
return unlikely(sigismember(&p->pending.signal, SIGKILL));
}
static inline int fatal_signal_pending(struct task_struct *p)
{
return task_sigpending(p) && __fatal_signal_pending(p);
}
static inline int signal_pending_state(unsigned int state, struct task_struct *p)
{
if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
return 0;
if (!signal_pending(p))
return 0;
return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
}
/*
* This should only be used in fault handlers to decide whether we
* should stop the current fault routine to handle the signals
* instead, especially with the case where we've got interrupted with
* a VM_FAULT_RETRY.
*/
static inline bool fault_signal_pending(vm_fault_t fault_flags,
struct pt_regs *regs)
{
return unlikely((fault_flags & VM_FAULT_RETRY) &&
(fatal_signal_pending(current) ||
(user_mode(regs) && signal_pending(current))));
}
/*
* Reevaluate whether the task has signals pending delivery.
* Wake the task if so.
* This is required every time the blocked sigset_t changes.
* callers must hold sighand->siglock.
*/
extern void recalc_sigpending(void);
extern void calculate_sigpending(void);
extern void signal_wake_up_state(struct task_struct *t, unsigned int state);
static inline void signal_wake_up(struct task_struct *t, bool fatal)
{
unsigned int state = 0;
if (fatal && !(t->jobctl & JOBCTL_PTRACE_FROZEN)) { t->jobctl &= ~(JOBCTL_STOPPED | JOBCTL_TRACED);
state = TASK_WAKEKILL | __TASK_TRACED;
}
signal_wake_up_state(t, state);
}
static inline void ptrace_signal_wake_up(struct task_struct *t, bool resume)
{
unsigned int state = 0;
if (resume) {
t->jobctl &= ~JOBCTL_TRACED;
state = __TASK_TRACED;
}
signal_wake_up_state(t, state);
}
void task_join_group_stop(struct task_struct *task);
#ifdef TIF_RESTORE_SIGMASK
/*
* Legacy restore_sigmask accessors. These are inefficient on
* SMP architectures because they require atomic operations.
*/
/**
* set_restore_sigmask() - make sure saved_sigmask processing gets done
*
* This sets TIF_RESTORE_SIGMASK and ensures that the arch signal code
* will run before returning to user mode, to process the flag. For
* all callers, TIF_SIGPENDING is already set or it's no harm to set
* it. TIF_RESTORE_SIGMASK need not be in the set of bits that the
* arch code will notice on return to user mode, in case those bits
* are scarce. We set TIF_SIGPENDING here to ensure that the arch
* signal code always gets run when TIF_RESTORE_SIGMASK is set.
*/
static inline void set_restore_sigmask(void)
{
set_thread_flag(TIF_RESTORE_SIGMASK);
}
static inline void clear_tsk_restore_sigmask(struct task_struct *task)
{
clear_tsk_thread_flag(task, TIF_RESTORE_SIGMASK);
}
static inline void clear_restore_sigmask(void)
{
clear_thread_flag(TIF_RESTORE_SIGMASK);
}
static inline bool test_tsk_restore_sigmask(struct task_struct *task)
{
return test_tsk_thread_flag(task, TIF_RESTORE_SIGMASK);
}
static inline bool test_restore_sigmask(void)
{
return test_thread_flag(TIF_RESTORE_SIGMASK);
}
static inline bool test_and_clear_restore_sigmask(void)
{
return test_and_clear_thread_flag(TIF_RESTORE_SIGMASK);
}
#else /* TIF_RESTORE_SIGMASK */
/* Higher-quality implementation, used if TIF_RESTORE_SIGMASK doesn't exist. */
static inline void set_restore_sigmask(void)
{
current->restore_sigmask = true;
}
static inline void clear_tsk_restore_sigmask(struct task_struct *task)
{
task->restore_sigmask = false;
}
static inline void clear_restore_sigmask(void)
{
current->restore_sigmask = false;
}
static inline bool test_restore_sigmask(void)
{
return current->restore_sigmask;
}
static inline bool test_tsk_restore_sigmask(struct task_struct *task)
{
return task->restore_sigmask;
}
static inline bool test_and_clear_restore_sigmask(void)
{
if (!current->restore_sigmask)
return false;
current->restore_sigmask = false;
return true;
}
#endif
static inline void restore_saved_sigmask(void)
{
if (test_and_clear_restore_sigmask()) __set_current_blocked(¤t->saved_sigmask);
}
extern int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize);
static inline void restore_saved_sigmask_unless(bool interrupted)
{
if (interrupted)
WARN_ON(!signal_pending(current));
else
restore_saved_sigmask();
}
static inline sigset_t *sigmask_to_save(void)
{
sigset_t *res = ¤t->blocked;
if (unlikely(test_restore_sigmask()))
res = ¤t->saved_sigmask;
return res;
}
static inline int kill_cad_pid(int sig, int priv)
{
return kill_pid(cad_pid, sig, priv);
}
/* These can be the second arg to send_sig_info/send_group_sig_info. */
#define SEND_SIG_NOINFO ((struct kernel_siginfo *) 0)
#define SEND_SIG_PRIV ((struct kernel_siginfo *) 1)
static inline int __on_sig_stack(unsigned long sp)
{
#ifdef CONFIG_STACK_GROWSUP
return sp >= current->sas_ss_sp &&
sp - current->sas_ss_sp < current->sas_ss_size;
#else
return sp > current->sas_ss_sp && sp - current->sas_ss_sp <= current->sas_ss_size;
#endif
}
/*
* True if we are on the alternate signal stack.
*/
static inline int on_sig_stack(unsigned long sp)
{
/*
* If the signal stack is SS_AUTODISARM then, by construction, we
* can't be on the signal stack unless user code deliberately set
* SS_AUTODISARM when we were already on it.
*
* This improves reliability: if user state gets corrupted such that
* the stack pointer points very close to the end of the signal stack,
* then this check will enable the signal to be handled anyway.
*/
if (current->sas_ss_flags & SS_AUTODISARM)
return 0;
return __on_sig_stack(sp);
}
static inline int sas_ss_flags(unsigned long sp)
{
if (!current->sas_ss_size)
return SS_DISABLE;
return on_sig_stack(sp) ? SS_ONSTACK : 0;
}
static inline void sas_ss_reset(struct task_struct *p)
{
p->sas_ss_sp = 0;
p->sas_ss_size = 0;
p->sas_ss_flags = SS_DISABLE;
}
static inline unsigned long sigsp(unsigned long sp, struct ksignal *ksig)
{
if (unlikely((ksig->ka.sa.sa_flags & SA_ONSTACK)) && ! sas_ss_flags(sp))
#ifdef CONFIG_STACK_GROWSUP
return current->sas_ss_sp;
#else
return current->sas_ss_sp + current->sas_ss_size;
#endif
return sp;
}
extern void __cleanup_sighand(struct sighand_struct *);
extern void flush_itimer_signals(void);
#define tasklist_empty() \
list_empty(&init_task.tasks)
#define next_task(p) \
list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
#define for_each_process(p) \
for (p = &init_task ; (p = next_task(p)) != &init_task ; )
extern bool current_is_single_threaded(void);
/*
* Without tasklist/siglock it is only rcu-safe if g can't exit/exec,
* otherwise next_thread(t) will never reach g after list_del_rcu(g).
*/
#define while_each_thread(g, t) \
while ((t = next_thread(t)) != g)
#define for_other_threads(p, t) \
for (t = p; (t = next_thread(t)) != p; )
#define __for_each_thread(signal, t) \
list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node, \
lockdep_is_held(&tasklist_lock))
#define for_each_thread(p, t) \
__for_each_thread((p)->signal, t)
/* Careful: this is a double loop, 'break' won't work as expected. */
#define for_each_process_thread(p, t) \
for_each_process(p) for_each_thread(p, t)
typedef int (*proc_visitor)(struct task_struct *p, void *data);
void walk_process_tree(struct task_struct *top, proc_visitor, void *);
static inline
struct pid *task_pid_type(struct task_struct *task, enum pid_type type)
{
struct pid *pid;
if (type == PIDTYPE_PID)
pid = task_pid(task);
else
pid = task->signal->pids[type];
return pid;
}
static inline struct pid *task_tgid(struct task_struct *task)
{
return task->signal->pids[PIDTYPE_TGID];
}
/*
* Without tasklist or RCU lock it is not safe to dereference
* the result of task_pgrp/task_session even if task == current,
* we can race with another thread doing sys_setsid/sys_setpgid.
*/
static inline struct pid *task_pgrp(struct task_struct *task)
{
return task->signal->pids[PIDTYPE_PGID];
}
static inline struct pid *task_session(struct task_struct *task)
{
return task->signal->pids[PIDTYPE_SID];
}
static inline int get_nr_threads(struct task_struct *task)
{
return task->signal->nr_threads;
}
static inline bool thread_group_leader(struct task_struct *p)
{
return p->exit_signal >= 0;
}
static inline
bool same_thread_group(struct task_struct *p1, struct task_struct *p2)
{
return p1->signal == p2->signal;
}
/*
* returns NULL if p is the last thread in the thread group
*/
static inline struct task_struct *__next_thread(struct task_struct *p)
{
return list_next_or_null_rcu(&p->signal->thread_head,
&p->thread_node,
struct task_struct,
thread_node);
}
static inline struct task_struct *next_thread(struct task_struct *p)
{
return __next_thread(p) ?: p->group_leader;
}
static inline int thread_group_empty(struct task_struct *p)
{
return thread_group_leader(p) && list_is_last(&p->thread_node, &p->signal->thread_head);
}
#define delay_group_leader(p) \
(thread_group_leader(p) && !thread_group_empty(p))
extern struct sighand_struct *__lock_task_sighand(struct task_struct *task,
unsigned long *flags);
static inline struct sighand_struct *lock_task_sighand(struct task_struct *task,
unsigned long *flags)
{
struct sighand_struct *ret;
ret = __lock_task_sighand(task, flags);
(void)__cond_lock(&task->sighand->siglock, ret);
return ret;
}
static inline void unlock_task_sighand(struct task_struct *task,
unsigned long *flags)
{
spin_unlock_irqrestore(&task->sighand->siglock, *flags);
}
#ifdef CONFIG_LOCKDEP
extern void lockdep_assert_task_sighand_held(struct task_struct *task);
#else
static inline void lockdep_assert_task_sighand_held(struct task_struct *task) { }
#endif
static inline unsigned long task_rlimit(const struct task_struct *task,
unsigned int limit)
{
return READ_ONCE(task->signal->rlim[limit].rlim_cur);
}
static inline unsigned long task_rlimit_max(const struct task_struct *task,
unsigned int limit)
{
return READ_ONCE(task->signal->rlim[limit].rlim_max);
}
static inline unsigned long rlimit(unsigned int limit)
{
return task_rlimit(current, limit);
}
static inline unsigned long rlimit_max(unsigned int limit)
{
return task_rlimit_max(current, limit);
}
#endif /* _LINUX_SCHED_SIGNAL_H */
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM irq
#if !defined(_TRACE_IRQ_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_IRQ_H
#include <linux/tracepoint.h>
struct irqaction;
struct softirq_action;
#define SOFTIRQ_NAME_LIST \
softirq_name(HI) \
softirq_name(TIMER) \
softirq_name(NET_TX) \
softirq_name(NET_RX) \
softirq_name(BLOCK) \
softirq_name(IRQ_POLL) \
softirq_name(TASKLET) \
softirq_name(SCHED) \
softirq_name(HRTIMER) \
softirq_name_end(RCU)
#undef softirq_name
#undef softirq_name_end
#define softirq_name(sirq) TRACE_DEFINE_ENUM(sirq##_SOFTIRQ);
#define softirq_name_end(sirq) TRACE_DEFINE_ENUM(sirq##_SOFTIRQ);
SOFTIRQ_NAME_LIST
#undef softirq_name
#undef softirq_name_end
#define softirq_name(sirq) { sirq##_SOFTIRQ, #sirq },
#define softirq_name_end(sirq) { sirq##_SOFTIRQ, #sirq }
#define show_softirq_name(val) \
__print_symbolic(val, SOFTIRQ_NAME_LIST)
/**
* irq_handler_entry - called immediately before the irq action handler
* @irq: irq number
* @action: pointer to struct irqaction
*
* The struct irqaction pointed to by @action contains various
* information about the handler, including the device name,
* @action->name, and the device id, @action->dev_id. When used in
* conjunction with the irq_handler_exit tracepoint, we can figure
* out irq handler latencies.
*/
TRACE_EVENT(irq_handler_entry,
TP_PROTO(int irq, struct irqaction *action),
TP_ARGS(irq, action),
TP_STRUCT__entry(
__field( int, irq )
__string( name, action->name )
),
TP_fast_assign(
__entry->irq = irq;
__assign_str(name, action->name);
),
TP_printk("irq=%d name=%s", __entry->irq, __get_str(name))
);
/**
* irq_handler_exit - called immediately after the irq action handler returns
* @irq: irq number
* @action: pointer to struct irqaction
* @ret: return value
*
* If the @ret value is set to IRQ_HANDLED, then we know that the corresponding
* @action->handler successfully handled this irq. Otherwise, the irq might be
* a shared irq line, or the irq was not handled successfully. Can be used in
* conjunction with the irq_handler_entry to understand irq handler latencies.
*/
TRACE_EVENT(irq_handler_exit,
TP_PROTO(int irq, struct irqaction *action, int ret),
TP_ARGS(irq, action, ret),
TP_STRUCT__entry(
__field( int, irq )
__field( int, ret )
),
TP_fast_assign(
__entry->irq = irq;
__entry->ret = ret;
),
TP_printk("irq=%d ret=%s",
__entry->irq, __entry->ret ? "handled" : "unhandled")
);
DECLARE_EVENT_CLASS(softirq,
TP_PROTO(unsigned int vec_nr),
TP_ARGS(vec_nr),
TP_STRUCT__entry(
__field( unsigned int, vec )
),
TP_fast_assign(
__entry->vec = vec_nr;
),
TP_printk("vec=%u [action=%s]", __entry->vec,
show_softirq_name(__entry->vec))
);
/**
* softirq_entry - called immediately before the softirq handler
* @vec_nr: softirq vector number
*
* When used in combination with the softirq_exit tracepoint
* we can determine the softirq handler routine.
*/
DEFINE_EVENT(softirq, softirq_entry,
TP_PROTO(unsigned int vec_nr),
TP_ARGS(vec_nr)
);
/**
* softirq_exit - called immediately after the softirq handler returns
* @vec_nr: softirq vector number
*
* When used in combination with the softirq_entry tracepoint
* we can determine the softirq handler routine.
*/
DEFINE_EVENT(softirq, softirq_exit,
TP_PROTO(unsigned int vec_nr),
TP_ARGS(vec_nr)
);
/**
* softirq_raise - called immediately when a softirq is raised
* @vec_nr: softirq vector number
*
* When used in combination with the softirq_entry tracepoint
* we can determine the softirq raise to run latency.
*/
DEFINE_EVENT(softirq, softirq_raise,
TP_PROTO(unsigned int vec_nr),
TP_ARGS(vec_nr)
);
DECLARE_EVENT_CLASS(tasklet,
TP_PROTO(struct tasklet_struct *t, void *func),
TP_ARGS(t, func),
TP_STRUCT__entry(
__field( void *, tasklet)
__field( void *, func)
),
TP_fast_assign(
__entry->tasklet = t;
__entry->func = func;
),
TP_printk("tasklet=%ps function=%ps", __entry->tasklet, __entry->func)
);
/**
* tasklet_entry - called immediately before the tasklet is run
* @t: tasklet pointer
* @func: tasklet callback or function being run
*
* Used to find individual tasklet execution time
*/
DEFINE_EVENT(tasklet, tasklet_entry,
TP_PROTO(struct tasklet_struct *t, void *func),
TP_ARGS(t, func)
);
/**
* tasklet_exit - called immediately after the tasklet is run
* @t: tasklet pointer
* @func: tasklet callback or function being run
*
* Used to find individual tasklet execution time
*/
DEFINE_EVENT(tasklet, tasklet_exit,
TP_PROTO(struct tasklet_struct *t, void *func),
TP_ARGS(t, func)
);
#endif /* _TRACE_IRQ_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
// SPDX-License-Identifier: GPL-2.0
/*
* BlueZ - Bluetooth protocol stack for Linux
*
* Copyright (C) 2021 Intel Corporation
* Copyright 2023 NXP
*/
#include <linux/property.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <net/bluetooth/mgmt.h>
#include "hci_request.h"
#include "hci_codec.h"
#include "hci_debugfs.h"
#include "smp.h"
#include "eir.h"
#include "msft.h"
#include "aosp.h"
#include "leds.h"
static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode,
struct sk_buff *skb)
{
bt_dev_dbg(hdev, "result 0x%2.2x", result);
if (hdev->req_status != HCI_REQ_PEND)
return;
hdev->req_result = result;
hdev->req_status = HCI_REQ_DONE;
/* Free the request command so it is not used as response */
kfree_skb(hdev->req_skb);
hdev->req_skb = NULL;
if (skb) {
struct sock *sk = hci_skb_sk(skb);
/* Drop sk reference if set */
if (sk)
sock_put(sk);
hdev->req_rsp = skb_get(skb);
}
wake_up_interruptible(&hdev->req_wait_q);
}
static struct sk_buff *hci_cmd_sync_alloc(struct hci_dev *hdev, u16 opcode,
u32 plen, const void *param,
struct sock *sk)
{
int len = HCI_COMMAND_HDR_SIZE + plen;
struct hci_command_hdr *hdr;
struct sk_buff *skb;
skb = bt_skb_alloc(len, GFP_ATOMIC);
if (!skb)
return NULL;
hdr = skb_put(skb, HCI_COMMAND_HDR_SIZE);
hdr->opcode = cpu_to_le16(opcode);
hdr->plen = plen;
if (plen)
skb_put_data(skb, param, plen);
bt_dev_dbg(hdev, "skb len %d", skb->len);
hci_skb_pkt_type(skb) = HCI_COMMAND_PKT;
hci_skb_opcode(skb) = opcode;
/* Grab a reference if command needs to be associated with a sock (e.g.
* likely mgmt socket that initiated the command).
*/
if (sk) {
hci_skb_sk(skb) = sk;
sock_hold(sk);
}
return skb;
}
static void hci_cmd_sync_add(struct hci_request *req, u16 opcode, u32 plen,
const void *param, u8 event, struct sock *sk)
{
struct hci_dev *hdev = req->hdev;
struct sk_buff *skb;
bt_dev_dbg(hdev, "opcode 0x%4.4x plen %d", opcode, plen);
/* If an error occurred during request building, there is no point in
* queueing the HCI command. We can simply return.
*/
if (req->err)
return;
skb = hci_cmd_sync_alloc(hdev, opcode, plen, param, sk);
if (!skb) {
bt_dev_err(hdev, "no memory for command (opcode 0x%4.4x)",
opcode);
req->err = -ENOMEM;
return;
}
if (skb_queue_empty(&req->cmd_q))
bt_cb(skb)->hci.req_flags |= HCI_REQ_START;
hci_skb_event(skb) = event;
skb_queue_tail(&req->cmd_q, skb);
}
static int hci_cmd_sync_run(struct hci_request *req)
{
struct hci_dev *hdev = req->hdev;
struct sk_buff *skb;
unsigned long flags;
bt_dev_dbg(hdev, "length %u", skb_queue_len(&req->cmd_q));
/* If an error occurred during request building, remove all HCI
* commands queued on the HCI request queue.
*/
if (req->err) {
skb_queue_purge(&req->cmd_q);
return req->err;
}
/* Do not allow empty requests */
if (skb_queue_empty(&req->cmd_q))
return -ENODATA;
skb = skb_peek_tail(&req->cmd_q);
bt_cb(skb)->hci.req_complete_skb = hci_cmd_sync_complete;
bt_cb(skb)->hci.req_flags |= HCI_REQ_SKB;
spin_lock_irqsave(&hdev->cmd_q.lock, flags);
skb_queue_splice_tail(&req->cmd_q, &hdev->cmd_q);
spin_unlock_irqrestore(&hdev->cmd_q.lock, flags);
queue_work(hdev->workqueue, &hdev->cmd_work);
return 0;
}
/* This function requires the caller holds hdev->req_lock. */
struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param, u8 event, u32 timeout,
struct sock *sk)
{
struct hci_request req;
struct sk_buff *skb;
int err = 0;
bt_dev_dbg(hdev, "Opcode 0x%4.4x", opcode);
hci_req_init(&req, hdev);
hci_cmd_sync_add(&req, opcode, plen, param, event, sk);
hdev->req_status = HCI_REQ_PEND;
err = hci_cmd_sync_run(&req);
if (err < 0)
return ERR_PTR(err);
err = wait_event_interruptible_timeout(hdev->req_wait_q,
hdev->req_status != HCI_REQ_PEND,
timeout);
if (err == -ERESTARTSYS)
return ERR_PTR(-EINTR);
switch (hdev->req_status) {
case HCI_REQ_DONE:
err = -bt_to_errno(hdev->req_result);
break;
case HCI_REQ_CANCELED:
err = -hdev->req_result;
break;
default:
err = -ETIMEDOUT;
break;
}
hdev->req_status = 0;
hdev->req_result = 0;
skb = hdev->req_rsp;
hdev->req_rsp = NULL;
bt_dev_dbg(hdev, "end: err %d", err);
if (err < 0) {
kfree_skb(skb);
return ERR_PTR(err);
}
return skb;
}
EXPORT_SYMBOL(__hci_cmd_sync_sk);
/* This function requires the caller holds hdev->req_lock. */
struct sk_buff *__hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param, u32 timeout)
{
return __hci_cmd_sync_sk(hdev, opcode, plen, param, 0, timeout, NULL);
}
EXPORT_SYMBOL(__hci_cmd_sync);
/* Send HCI command and wait for command complete event */
struct sk_buff *hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param, u32 timeout)
{
struct sk_buff *skb;
if (!test_bit(HCI_UP, &hdev->flags))
return ERR_PTR(-ENETDOWN);
bt_dev_dbg(hdev, "opcode 0x%4.4x plen %d", opcode, plen);
hci_req_sync_lock(hdev);
skb = __hci_cmd_sync(hdev, opcode, plen, param, timeout);
hci_req_sync_unlock(hdev);
return skb;
}
EXPORT_SYMBOL(hci_cmd_sync);
/* This function requires the caller holds hdev->req_lock. */
struct sk_buff *__hci_cmd_sync_ev(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param, u8 event, u32 timeout)
{
return __hci_cmd_sync_sk(hdev, opcode, plen, param, event, timeout,
NULL);
}
EXPORT_SYMBOL(__hci_cmd_sync_ev);
/* This function requires the caller holds hdev->req_lock. */
int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param, u8 event, u32 timeout,
struct sock *sk)
{
struct sk_buff *skb;
u8 status;
skb = __hci_cmd_sync_sk(hdev, opcode, plen, param, event, timeout, sk);
if (IS_ERR(skb)) {
if (!event)
bt_dev_err(hdev, "Opcode 0x%4.4x failed: %ld", opcode,
PTR_ERR(skb));
return PTR_ERR(skb);
}
/* If command return a status event skb will be set to NULL as there are
* no parameters, in case of failure IS_ERR(skb) would have be set to
* the actual error would be found with PTR_ERR(skb).
*/
if (!skb)
return 0;
status = skb->data[0];
kfree_skb(skb);
return status;
}
EXPORT_SYMBOL(__hci_cmd_sync_status_sk);
int __hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param, u32 timeout)
{
return __hci_cmd_sync_status_sk(hdev, opcode, plen, param, 0, timeout,
NULL);
}
EXPORT_SYMBOL(__hci_cmd_sync_status);
static void hci_cmd_sync_work(struct work_struct *work)
{
struct hci_dev *hdev = container_of(work, struct hci_dev, cmd_sync_work);
bt_dev_dbg(hdev, "");
/* Dequeue all entries and run them */
while (1) {
struct hci_cmd_sync_work_entry *entry;
mutex_lock(&hdev->cmd_sync_work_lock);
entry = list_first_entry_or_null(&hdev->cmd_sync_work_list,
struct hci_cmd_sync_work_entry,
list);
if (entry)
list_del(&entry->list);
mutex_unlock(&hdev->cmd_sync_work_lock);
if (!entry)
break;
bt_dev_dbg(hdev, "entry %p", entry);
if (entry->func) {
int err;
hci_req_sync_lock(hdev);
err = entry->func(hdev, entry->data);
if (entry->destroy)
entry->destroy(hdev, entry->data, err);
hci_req_sync_unlock(hdev);
}
kfree(entry);
}
}
static void hci_cmd_sync_cancel_work(struct work_struct *work)
{
struct hci_dev *hdev = container_of(work, struct hci_dev, cmd_sync_cancel_work);
cancel_delayed_work_sync(&hdev->cmd_timer);
cancel_delayed_work_sync(&hdev->ncmd_timer);
atomic_set(&hdev->cmd_cnt, 1);
wake_up_interruptible(&hdev->req_wait_q);
}
static int hci_scan_disable_sync(struct hci_dev *hdev);
static int scan_disable_sync(struct hci_dev *hdev, void *data)
{
return hci_scan_disable_sync(hdev);
}
static int hci_inquiry_sync(struct hci_dev *hdev, u8 length);
static int interleaved_inquiry_sync(struct hci_dev *hdev, void *data)
{
return hci_inquiry_sync(hdev, DISCOV_INTERLEAVED_INQUIRY_LEN);
}
static void le_scan_disable(struct work_struct *work)
{
struct hci_dev *hdev = container_of(work, struct hci_dev,
le_scan_disable.work);
int status;
bt_dev_dbg(hdev, "");
hci_dev_lock(hdev);
if (!hci_dev_test_flag(hdev, HCI_LE_SCAN))
goto _return;
status = hci_cmd_sync_queue(hdev, scan_disable_sync, NULL, NULL);
if (status) {
bt_dev_err(hdev, "failed to disable LE scan: %d", status);
goto _return;
}
hdev->discovery.scan_start = 0;
/* If we were running LE only scan, change discovery state. If
* we were running both LE and BR/EDR inquiry simultaneously,
* and BR/EDR inquiry is already finished, stop discovery,
* otherwise BR/EDR inquiry will stop discovery when finished.
* If we will resolve remote device name, do not change
* discovery state.
*/
if (hdev->discovery.type == DISCOV_TYPE_LE)
goto discov_stopped;
if (hdev->discovery.type != DISCOV_TYPE_INTERLEAVED)
goto _return;
if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks)) {
if (!test_bit(HCI_INQUIRY, &hdev->flags) &&
hdev->discovery.state != DISCOVERY_RESOLVING)
goto discov_stopped;
goto _return;
}
status = hci_cmd_sync_queue(hdev, interleaved_inquiry_sync, NULL, NULL);
if (status) {
bt_dev_err(hdev, "inquiry failed: status %d", status);
goto discov_stopped;
}
goto _return;
discov_stopped:
hci_discovery_set_state(hdev, DISCOVERY_STOPPED);
_return:
hci_dev_unlock(hdev);
}
static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val,
u8 filter_dup);
static int reenable_adv_sync(struct hci_dev *hdev, void *data)
{
bt_dev_dbg(hdev, "");
if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) &&
list_empty(&hdev->adv_instances))
return 0;
if (hdev->cur_adv_instance) {
return hci_schedule_adv_instance_sync(hdev,
hdev->cur_adv_instance,
true);
} else {
if (ext_adv_capable(hdev)) {
hci_start_ext_adv_sync(hdev, 0x00);
} else {
hci_update_adv_data_sync(hdev, 0x00);
hci_update_scan_rsp_data_sync(hdev, 0x00);
hci_enable_advertising_sync(hdev);
}
}
return 0;
}
static void reenable_adv(struct work_struct *work)
{
struct hci_dev *hdev = container_of(work, struct hci_dev,
reenable_adv_work);
int status;
bt_dev_dbg(hdev, "");
hci_dev_lock(hdev);
status = hci_cmd_sync_queue(hdev, reenable_adv_sync, NULL, NULL);
if (status)
bt_dev_err(hdev, "failed to reenable ADV: %d", status);
hci_dev_unlock(hdev);
}
static void cancel_adv_timeout(struct hci_dev *hdev)
{
if (hdev->adv_instance_timeout) {
hdev->adv_instance_timeout = 0;
cancel_delayed_work(&hdev->adv_instance_expire);
}
}
/* For a single instance:
* - force == true: The instance will be removed even when its remaining
* lifetime is not zero.
* - force == false: the instance will be deactivated but kept stored unless
* the remaining lifetime is zero.
*
* For instance == 0x00:
* - force == true: All instances will be removed regardless of their timeout
* setting.
* - force == false: Only instances that have a timeout will be removed.
*/
int hci_clear_adv_instance_sync(struct hci_dev *hdev, struct sock *sk,
u8 instance, bool force)
{
struct adv_info *adv_instance, *n, *next_instance = NULL;
int err;
u8 rem_inst;
/* Cancel any timeout concerning the removed instance(s). */
if (!instance || hdev->cur_adv_instance == instance)
cancel_adv_timeout(hdev);
/* Get the next instance to advertise BEFORE we remove
* the current one. This can be the same instance again
* if there is only one instance.
*/
if (instance && hdev->cur_adv_instance == instance)
next_instance = hci_get_next_instance(hdev, instance);
if (instance == 0x00) {
list_for_each_entry_safe(adv_instance, n, &hdev->adv_instances,
list) {
if (!(force || adv_instance->timeout))
continue;
rem_inst = adv_instance->instance;
err = hci_remove_adv_instance(hdev, rem_inst);
if (!err)
mgmt_advertising_removed(sk, hdev, rem_inst);
}
} else {
adv_instance = hci_find_adv_instance(hdev, instance);
if (force || (adv_instance && adv_instance->timeout &&
!adv_instance->remaining_time)) {
/* Don't advertise a removed instance. */
if (next_instance &&
next_instance->instance == instance)
next_instance = NULL;
err = hci_remove_adv_instance(hdev, instance);
if (!err)
mgmt_advertising_removed(sk, hdev, instance);
}
}
if (!hdev_is_powered(hdev) || hci_dev_test_flag(hdev, HCI_ADVERTISING))
return 0;
if (next_instance && !ext_adv_capable(hdev))
return hci_schedule_adv_instance_sync(hdev,
next_instance->instance,
false);
return 0;
}
static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data)
{
u8 instance = *(u8 *)data;
kfree(data);
hci_clear_adv_instance_sync(hdev, NULL, instance, false);
if (list_empty(&hdev->adv_instances))
return hci_disable_advertising_sync(hdev);
return 0;
}
static void adv_timeout_expire(struct work_struct *work)
{
u8 *inst_ptr;
struct hci_dev *hdev = container_of(work, struct hci_dev,
adv_instance_expire.work);
bt_dev_dbg(hdev, "");
hci_dev_lock(hdev);
hdev->adv_instance_timeout = 0;
if (hdev->cur_adv_instance == 0x00)
goto unlock;
inst_ptr = kmalloc(1, GFP_KERNEL);
if (!inst_ptr)
goto unlock;
*inst_ptr = hdev->cur_adv_instance;
hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, NULL);
unlock:
hci_dev_unlock(hdev);
}
void hci_cmd_sync_init(struct hci_dev *hdev)
{
INIT_WORK(&hdev->cmd_sync_work, hci_cmd_sync_work);
INIT_LIST_HEAD(&hdev->cmd_sync_work_list);
mutex_init(&hdev->cmd_sync_work_lock);
mutex_init(&hdev->unregister_lock);
INIT_WORK(&hdev->cmd_sync_cancel_work, hci_cmd_sync_cancel_work);
INIT_WORK(&hdev->reenable_adv_work, reenable_adv);
INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable);
INIT_DELAYED_WORK(&hdev->adv_instance_expire, adv_timeout_expire);
}
static void _hci_cmd_sync_cancel_entry(struct hci_dev *hdev,
struct hci_cmd_sync_work_entry *entry,
int err)
{
if (entry->destroy)
entry->destroy(hdev, entry->data, err);
list_del(&entry->list);
kfree(entry);
}
void hci_cmd_sync_clear(struct hci_dev *hdev)
{
struct hci_cmd_sync_work_entry *entry, *tmp;
cancel_work_sync(&hdev->cmd_sync_work);
cancel_work_sync(&hdev->reenable_adv_work);
mutex_lock(&hdev->cmd_sync_work_lock);
list_for_each_entry_safe(entry, tmp, &hdev->cmd_sync_work_list, list)
_hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); mutex_unlock(&hdev->cmd_sync_work_lock);
}
void hci_cmd_sync_cancel(struct hci_dev *hdev, int err)
{
bt_dev_dbg(hdev, "err 0x%2.2x", err);
if (hdev->req_status == HCI_REQ_PEND) {
hdev->req_result = err;
hdev->req_status = HCI_REQ_CANCELED;
queue_work(hdev->workqueue, &hdev->cmd_sync_cancel_work);
}
}
EXPORT_SYMBOL(hci_cmd_sync_cancel);
/* Cancel ongoing command request synchronously:
*
* - Set result and mark status to HCI_REQ_CANCELED
* - Wakeup command sync thread
*/
void hci_cmd_sync_cancel_sync(struct hci_dev *hdev, int err)
{
bt_dev_dbg(hdev, "err 0x%2.2x", err); if (hdev->req_status == HCI_REQ_PEND) {
/* req_result is __u32 so error must be positive to be properly
* propagated.
*/
hdev->req_result = err < 0 ? -err : err;
hdev->req_status = HCI_REQ_CANCELED;
wake_up_interruptible(&hdev->req_wait_q);
}
}
EXPORT_SYMBOL(hci_cmd_sync_cancel_sync);
/* Submit HCI command to be run in as cmd_sync_work:
*
* - hdev must _not_ be unregistered
*/
int hci_cmd_sync_submit(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
struct hci_cmd_sync_work_entry *entry;
int err = 0;
mutex_lock(&hdev->unregister_lock);
if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) {
err = -ENODEV;
goto unlock;
}
entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry) {
err = -ENOMEM;
goto unlock;
}
entry->func = func;
entry->data = data;
entry->destroy = destroy;
mutex_lock(&hdev->cmd_sync_work_lock);
list_add_tail(&entry->list, &hdev->cmd_sync_work_list);
mutex_unlock(&hdev->cmd_sync_work_lock);
queue_work(hdev->req_workqueue, &hdev->cmd_sync_work);
unlock:
mutex_unlock(&hdev->unregister_lock);
return err;
}
EXPORT_SYMBOL(hci_cmd_sync_submit);
/* Queue HCI command:
*
* - hdev must be running
*/
int hci_cmd_sync_queue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
/* Only queue command if hdev is running which means it had been opened
* and is either on init phase or is already up.
*/
if (!test_bit(HCI_RUNNING, &hdev->flags))
return -ENETDOWN;
return hci_cmd_sync_submit(hdev, func, data, destroy);
}
EXPORT_SYMBOL(hci_cmd_sync_queue);
static struct hci_cmd_sync_work_entry *
_hci_cmd_sync_lookup_entry(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
struct hci_cmd_sync_work_entry *entry, *tmp;
list_for_each_entry_safe(entry, tmp, &hdev->cmd_sync_work_list, list) {
if (func && entry->func != func)
continue;
if (data && entry->data != data)
continue;
if (destroy && entry->destroy != destroy)
continue;
return entry;
}
return NULL;
}
/* Queue HCI command entry once:
*
* - Lookup if an entry already exist and only if it doesn't creates a new entry
* and queue it.
*/
int hci_cmd_sync_queue_once(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
if (hci_cmd_sync_lookup_entry(hdev, func, data, destroy))
return 0;
return hci_cmd_sync_queue(hdev, func, data, destroy);
}
EXPORT_SYMBOL(hci_cmd_sync_queue_once);
/* Lookup HCI command entry:
*
* - Return first entry that matches by function callback or data or
* destroy callback.
*/
struct hci_cmd_sync_work_entry *
hci_cmd_sync_lookup_entry(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
struct hci_cmd_sync_work_entry *entry;
mutex_lock(&hdev->cmd_sync_work_lock);
entry = _hci_cmd_sync_lookup_entry(hdev, func, data, destroy);
mutex_unlock(&hdev->cmd_sync_work_lock);
return entry;
}
EXPORT_SYMBOL(hci_cmd_sync_lookup_entry);
/* Cancel HCI command entry */
void hci_cmd_sync_cancel_entry(struct hci_dev *hdev,
struct hci_cmd_sync_work_entry *entry)
{
mutex_lock(&hdev->cmd_sync_work_lock);
_hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED);
mutex_unlock(&hdev->cmd_sync_work_lock);
}
EXPORT_SYMBOL(hci_cmd_sync_cancel_entry);
/* Dequeue one HCI command entry:
*
* - Lookup and cancel first entry that matches.
*/
bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev,
hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
struct hci_cmd_sync_work_entry *entry;
entry = hci_cmd_sync_lookup_entry(hdev, func, data, destroy);
if (!entry)
return false;
hci_cmd_sync_cancel_entry(hdev, entry);
return true;
}
EXPORT_SYMBOL(hci_cmd_sync_dequeue_once);
/* Dequeue HCI command entry:
*
* - Lookup and cancel any entry that matches by function callback or data or
* destroy callback.
*/
bool hci_cmd_sync_dequeue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
void *data, hci_cmd_sync_work_destroy_t destroy)
{
struct hci_cmd_sync_work_entry *entry;
bool ret = false;
mutex_lock(&hdev->cmd_sync_work_lock);
while ((entry = _hci_cmd_sync_lookup_entry(hdev, func, data,
destroy))) {
_hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED);
ret = true;
}
mutex_unlock(&hdev->cmd_sync_work_lock);
return ret;
}
EXPORT_SYMBOL(hci_cmd_sync_dequeue);
int hci_update_eir_sync(struct hci_dev *hdev)
{
struct hci_cp_write_eir cp;
bt_dev_dbg(hdev, "");
if (!hdev_is_powered(hdev))
return 0;
if (!lmp_ext_inq_capable(hdev))
return 0;
if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED))
return 0;
if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE))
return 0;
memset(&cp, 0, sizeof(cp));
eir_create(hdev, cp.data);
if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0)
return 0;
memcpy(hdev->eir, cp.data, sizeof(cp.data));
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_EIR, sizeof(cp), &cp,
HCI_CMD_TIMEOUT);
}
static u8 get_service_classes(struct hci_dev *hdev)
{
struct bt_uuid *uuid;
u8 val = 0;
list_for_each_entry(uuid, &hdev->uuids, list)
val |= uuid->svc_hint;
return val;
}
int hci_update_class_sync(struct hci_dev *hdev)
{
u8 cod[3];
bt_dev_dbg(hdev, "");
if (!hdev_is_powered(hdev))
return 0;
if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
return 0;
if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE))
return 0;
cod[0] = hdev->minor_class;
cod[1] = hdev->major_class;
cod[2] = get_service_classes(hdev);
if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE))
cod[1] |= 0x20;
if (memcmp(cod, hdev->dev_class, 3) == 0)
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CLASS_OF_DEV,
sizeof(cod), cod, HCI_CMD_TIMEOUT);
}
static bool is_advertising_allowed(struct hci_dev *hdev, bool connectable)
{
/* If there is no connection we are OK to advertise. */
if (hci_conn_num(hdev, LE_LINK) == 0)
return true;
/* Check le_states if there is any connection in peripheral role. */
if (hdev->conn_hash.le_num_peripheral > 0) {
/* Peripheral connection state and non connectable mode
* bit 20.
*/
if (!connectable && !(hdev->le_states[2] & 0x10))
return false;
/* Peripheral connection state and connectable mode bit 38
* and scannable bit 21.
*/
if (connectable && (!(hdev->le_states[4] & 0x40) ||
!(hdev->le_states[2] & 0x20)))
return false;
}
/* Check le_states if there is any connection in central role. */
if (hci_conn_num(hdev, LE_LINK) != hdev->conn_hash.le_num_peripheral) {
/* Central connection state and non connectable mode bit 18. */
if (!connectable && !(hdev->le_states[2] & 0x02))
return false;
/* Central connection state and connectable mode bit 35 and
* scannable 19.
*/
if (connectable && (!(hdev->le_states[4] & 0x08) ||
!(hdev->le_states[2] & 0x08)))
return false;
}
return true;
}
static bool adv_use_rpa(struct hci_dev *hdev, uint32_t flags)
{
/* If privacy is not enabled don't use RPA */
if (!hci_dev_test_flag(hdev, HCI_PRIVACY))
return false;
/* If basic privacy mode is enabled use RPA */
if (!hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY))
return true;
/* If limited privacy mode is enabled don't use RPA if we're
* both discoverable and bondable.
*/
if ((flags & MGMT_ADV_FLAG_DISCOV) &&
hci_dev_test_flag(hdev, HCI_BONDABLE))
return false;
/* We're neither bondable nor discoverable in the limited
* privacy mode, therefore use RPA.
*/
return true;
}
static int hci_set_random_addr_sync(struct hci_dev *hdev, bdaddr_t *rpa)
{
/* If we're advertising or initiating an LE connection we can't
* go ahead and change the random address at this time. This is
* because the eventual initiator address used for the
* subsequently created connection will be undefined (some
* controllers use the new address and others the one we had
* when the operation started).
*
* In this kind of scenario skip the update and let the random
* address be updated at the next cycle.
*/
if (hci_dev_test_flag(hdev, HCI_LE_ADV) ||
hci_lookup_le_connect(hdev)) {
bt_dev_dbg(hdev, "Deferring random address update");
hci_dev_set_flag(hdev, HCI_RPA_EXPIRED);
return 0;
}
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RANDOM_ADDR,
6, rpa, HCI_CMD_TIMEOUT);
}
int hci_update_random_address_sync(struct hci_dev *hdev, bool require_privacy,
bool rpa, u8 *own_addr_type)
{
int err;
/* If privacy is enabled use a resolvable private address. If
* current RPA has expired or there is something else than
* the current RPA in use, then generate a new one.
*/
if (rpa) {
/* If Controller supports LL Privacy use own address type is
* 0x03
*/
if (use_ll_privacy(hdev))
*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
else
*own_addr_type = ADDR_LE_DEV_RANDOM;
/* Check if RPA is valid */
if (rpa_valid(hdev))
return 0;
err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
if (err < 0) {
bt_dev_err(hdev, "failed to generate new RPA");
return err;
}
err = hci_set_random_addr_sync(hdev, &hdev->rpa);
if (err)
return err;
return 0;
}
/* In case of required privacy without resolvable private address,
* use an non-resolvable private address. This is useful for active
* scanning and non-connectable advertising.
*/
if (require_privacy) {
bdaddr_t nrpa;
while (true) {
/* The non-resolvable private address is generated
* from random six bytes with the two most significant
* bits cleared.
*/
get_random_bytes(&nrpa, 6);
nrpa.b[5] &= 0x3f;
/* The non-resolvable private address shall not be
* equal to the public address.
*/
if (bacmp(&hdev->bdaddr, &nrpa))
break;
}
*own_addr_type = ADDR_LE_DEV_RANDOM;
return hci_set_random_addr_sync(hdev, &nrpa);
}
/* If forcing static address is in use or there is no public
* address use the static address as random address (but skip
* the HCI command if the current random address is already the
* static one.
*
* In case BR/EDR has been disabled on a dual-mode controller
* and a static address has been configured, then use that
* address instead of the public BR/EDR address.
*/
if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) ||
!bacmp(&hdev->bdaddr, BDADDR_ANY) ||
(!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) &&
bacmp(&hdev->static_addr, BDADDR_ANY))) {
*own_addr_type = ADDR_LE_DEV_RANDOM;
if (bacmp(&hdev->static_addr, &hdev->random_addr))
return hci_set_random_addr_sync(hdev,
&hdev->static_addr);
return 0;
}
/* Neither privacy nor static address is being used so use a
* public address.
*/
*own_addr_type = ADDR_LE_DEV_PUBLIC;
return 0;
}
static int hci_disable_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_ext_adv_enable *cp;
struct hci_cp_ext_adv_set *set;
u8 data[sizeof(*cp) + sizeof(*set) * 1];
u8 size;
/* If request specifies an instance that doesn't exist, fail */
if (instance > 0) {
struct adv_info *adv;
adv = hci_find_adv_instance(hdev, instance);
if (!adv)
return -EINVAL;
/* If not enabled there is nothing to do */
if (!adv->enabled)
return 0;
}
memset(data, 0, sizeof(data));
cp = (void *)data;
set = (void *)cp->data;
/* Instance 0x00 indicates all advertising instances will be disabled */
cp->num_of_sets = !!instance;
cp->enable = 0x00;
set->handle = instance;
size = sizeof(*cp) + sizeof(*set) * cp->num_of_sets;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_ENABLE,
size, data, HCI_CMD_TIMEOUT);
}
static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance,
bdaddr_t *random_addr)
{
struct hci_cp_le_set_adv_set_rand_addr cp;
int err;
if (!instance) {
/* Instance 0x00 doesn't have an adv_info, instead it uses
* hdev->random_addr to track its address so whenever it needs
* to be updated this also set the random address since
* hdev->random_addr is shared with scan state machine.
*/
err = hci_set_random_addr_sync(hdev, random_addr);
if (err)
return err;
}
memset(&cp, 0, sizeof(cp));
cp.handle = instance;
bacpy(&cp.bdaddr, random_addr);
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_SET_RAND_ADDR,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_ext_adv_params cp;
bool connectable;
u32 flags;
bdaddr_t random_addr;
u8 own_addr_type;
int err;
struct adv_info *adv;
bool secondary_adv;
if (instance > 0) {
adv = hci_find_adv_instance(hdev, instance);
if (!adv)
return -EINVAL;
} else {
adv = NULL;
}
/* Updating parameters of an active instance will return a
* Command Disallowed error, so we must first disable the
* instance if it is active.
*/
if (adv && !adv->pending) {
err = hci_disable_ext_adv_instance_sync(hdev, instance);
if (err)
return err;
}
flags = hci_adv_instance_flags(hdev, instance);
/* If the "connectable" instance flag was not set, then choose between
* ADV_IND and ADV_NONCONN_IND based on the global connectable setting.
*/
connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) ||
mgmt_get_connectable(hdev);
if (!is_advertising_allowed(hdev, connectable))
return -EPERM;
/* Set require_privacy to true only when non-connectable
* advertising is used. In that case it is fine to use a
* non-resolvable private address.
*/
err = hci_get_random_address(hdev, !connectable,
adv_use_rpa(hdev, flags), adv,
&own_addr_type, &random_addr);
if (err < 0)
return err;
memset(&cp, 0, sizeof(cp));
if (adv) {
hci_cpu_to_le24(adv->min_interval, cp.min_interval);
hci_cpu_to_le24(adv->max_interval, cp.max_interval);
cp.tx_power = adv->tx_power;
} else {
hci_cpu_to_le24(hdev->le_adv_min_interval, cp.min_interval);
hci_cpu_to_le24(hdev->le_adv_max_interval, cp.max_interval);
cp.tx_power = HCI_ADV_TX_POWER_NO_PREFERENCE;
}
secondary_adv = (flags & MGMT_ADV_FLAG_SEC_MASK);
if (connectable) {
if (secondary_adv)
cp.evt_properties = cpu_to_le16(LE_EXT_ADV_CONN_IND);
else
cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_IND);
} else if (hci_adv_instance_is_scannable(hdev, instance) ||
(flags & MGMT_ADV_PARAM_SCAN_RSP)) {
if (secondary_adv)
cp.evt_properties = cpu_to_le16(LE_EXT_ADV_SCAN_IND);
else
cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_SCAN_IND);
} else {
if (secondary_adv)
cp.evt_properties = cpu_to_le16(LE_EXT_ADV_NON_CONN_IND);
else
cp.evt_properties = cpu_to_le16(LE_LEGACY_NONCONN_IND);
}
/* If Own_Address_Type equals 0x02 or 0x03, the Peer_Address parameter
* contains the peer’s Identity Address and the Peer_Address_Type
* parameter contains the peer’s Identity Type (i.e., 0x00 or 0x01).
* These parameters are used to locate the corresponding local IRK in
* the resolving list; this IRK is used to generate their own address
* used in the advertisement.
*/
if (own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED)
hci_copy_identity_address(hdev, &cp.peer_addr,
&cp.peer_addr_type);
cp.own_addr_type = own_addr_type;
cp.channel_map = hdev->le_adv_channel_map;
cp.handle = instance;
if (flags & MGMT_ADV_FLAG_SEC_2M) {
cp.primary_phy = HCI_ADV_PHY_1M;
cp.secondary_phy = HCI_ADV_PHY_2M;
} else if (flags & MGMT_ADV_FLAG_SEC_CODED) {
cp.primary_phy = HCI_ADV_PHY_CODED;
cp.secondary_phy = HCI_ADV_PHY_CODED;
} else {
/* In all other cases use 1M */
cp.primary_phy = HCI_ADV_PHY_1M;
cp.secondary_phy = HCI_ADV_PHY_1M;
}
err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (err)
return err;
if ((own_addr_type == ADDR_LE_DEV_RANDOM ||
own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) &&
bacmp(&random_addr, BDADDR_ANY)) {
/* Check if random address need to be updated */
if (adv) {
if (!bacmp(&random_addr, &adv->random_addr))
return 0;
} else {
if (!bacmp(&random_addr, &hdev->random_addr))
return 0;
}
return hci_set_adv_set_random_addr_sync(hdev, instance,
&random_addr);
}
return 0;
}
static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance)
{
struct {
struct hci_cp_le_set_ext_scan_rsp_data cp;
u8 data[HCI_MAX_EXT_AD_LENGTH];
} pdu;
u8 len;
struct adv_info *adv = NULL;
int err;
memset(&pdu, 0, sizeof(pdu));
if (instance) {
adv = hci_find_adv_instance(hdev, instance);
if (!adv || !adv->scan_rsp_changed)
return 0;
}
len = eir_create_scan_rsp(hdev, instance, pdu.data);
pdu.cp.handle = instance;
pdu.cp.length = len;
pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE;
pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG;
err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA,
sizeof(pdu.cp) + len, &pdu.cp,
HCI_CMD_TIMEOUT);
if (err)
return err;
if (adv) {
adv->scan_rsp_changed = false;
} else {
memcpy(hdev->scan_rsp_data, pdu.data, len);
hdev->scan_rsp_data_len = len;
}
return 0;
}
static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_scan_rsp_data cp;
u8 len;
memset(&cp, 0, sizeof(cp));
len = eir_create_scan_rsp(hdev, instance, cp.data);
if (hdev->scan_rsp_data_len == len &&
!memcmp(cp.data, hdev->scan_rsp_data, len))
return 0;
memcpy(hdev->scan_rsp_data, cp.data, sizeof(cp.data));
hdev->scan_rsp_data_len = len;
cp.length = len;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_RSP_DATA,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_update_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance)
{
if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
return 0;
if (ext_adv_capable(hdev))
return hci_set_ext_scan_rsp_data_sync(hdev, instance);
return __hci_set_scan_rsp_data_sync(hdev, instance);
}
int hci_enable_ext_advertising_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_ext_adv_enable *cp;
struct hci_cp_ext_adv_set *set;
u8 data[sizeof(*cp) + sizeof(*set) * 1];
struct adv_info *adv;
if (instance > 0) {
adv = hci_find_adv_instance(hdev, instance);
if (!adv)
return -EINVAL;
/* If already enabled there is nothing to do */
if (adv->enabled)
return 0;
} else {
adv = NULL;
}
cp = (void *)data;
set = (void *)cp->data;
memset(cp, 0, sizeof(*cp));
cp->enable = 0x01;
cp->num_of_sets = 0x01;
memset(set, 0, sizeof(*set));
set->handle = instance;
/* Set duration per instance since controller is responsible for
* scheduling it.
*/
if (adv && adv->timeout) {
u16 duration = adv->timeout * MSEC_PER_SEC;
/* Time = N * 10 ms */
set->duration = cpu_to_le16(duration / 10);
}
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_ENABLE,
sizeof(*cp) +
sizeof(*set) * cp->num_of_sets,
data, HCI_CMD_TIMEOUT);
}
int hci_start_ext_adv_sync(struct hci_dev *hdev, u8 instance)
{
int err;
err = hci_setup_ext_adv_instance_sync(hdev, instance);
if (err)
return err;
err = hci_set_ext_scan_rsp_data_sync(hdev, instance);
if (err)
return err;
return hci_enable_ext_advertising_sync(hdev, instance);
}
int hci_disable_per_advertising_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_per_adv_enable cp;
struct adv_info *adv = NULL;
/* If periodic advertising already disabled there is nothing to do. */
adv = hci_find_adv_instance(hdev, instance);
if (!adv || !adv->periodic || !adv->enabled)
return 0;
memset(&cp, 0, sizeof(cp));
cp.enable = 0x00;
cp.handle = instance;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_ENABLE,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_set_per_adv_params_sync(struct hci_dev *hdev, u8 instance,
u16 min_interval, u16 max_interval)
{
struct hci_cp_le_set_per_adv_params cp;
memset(&cp, 0, sizeof(cp));
if (!min_interval)
min_interval = DISCOV_LE_PER_ADV_INT_MIN;
if (!max_interval)
max_interval = DISCOV_LE_PER_ADV_INT_MAX;
cp.handle = instance;
cp.min_interval = cpu_to_le16(min_interval);
cp.max_interval = cpu_to_le16(max_interval);
cp.periodic_properties = 0x0000;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_PARAMS,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance)
{
struct {
struct hci_cp_le_set_per_adv_data cp;
u8 data[HCI_MAX_PER_AD_LENGTH];
} pdu;
u8 len;
memset(&pdu, 0, sizeof(pdu));
if (instance) {
struct adv_info *adv = hci_find_adv_instance(hdev, instance);
if (!adv || !adv->periodic)
return 0;
}
len = eir_create_per_adv_data(hdev, instance, pdu.data);
pdu.cp.length = len;
pdu.cp.handle = instance;
pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_DATA,
sizeof(pdu.cp) + len, &pdu,
HCI_CMD_TIMEOUT);
}
static int hci_enable_per_advertising_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_per_adv_enable cp;
struct adv_info *adv = NULL;
/* If periodic advertising already enabled there is nothing to do. */
adv = hci_find_adv_instance(hdev, instance);
if (adv && adv->periodic && adv->enabled)
return 0;
memset(&cp, 0, sizeof(cp));
cp.enable = 0x01;
cp.handle = instance;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_ENABLE,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
/* Checks if periodic advertising data contains a Basic Announcement and if it
* does generates a Broadcast ID and add Broadcast Announcement.
*/
static int hci_adv_bcast_annoucement(struct hci_dev *hdev, struct adv_info *adv)
{
u8 bid[3];
u8 ad[4 + 3];
/* Skip if NULL adv as instance 0x00 is used for general purpose
* advertising so it cannot used for the likes of Broadcast Announcement
* as it can be overwritten at any point.
*/
if (!adv)
return 0;
/* Check if PA data doesn't contains a Basic Audio Announcement then
* there is nothing to do.
*/
if (!eir_get_service_data(adv->per_adv_data, adv->per_adv_data_len,
0x1851, NULL))
return 0;
/* Check if advertising data already has a Broadcast Announcement since
* the process may want to control the Broadcast ID directly and in that
* case the kernel shall no interfere.
*/
if (eir_get_service_data(adv->adv_data, adv->adv_data_len, 0x1852,
NULL))
return 0;
/* Generate Broadcast ID */
get_random_bytes(bid, sizeof(bid));
eir_append_service_data(ad, 0, 0x1852, bid, sizeof(bid));
hci_set_adv_instance_data(hdev, adv->instance, sizeof(ad), ad, 0, NULL);
return hci_update_adv_data_sync(hdev, adv->instance);
}
int hci_start_per_adv_sync(struct hci_dev *hdev, u8 instance, u8 data_len,
u8 *data, u32 flags, u16 min_interval,
u16 max_interval, u16 sync_interval)
{
struct adv_info *adv = NULL;
int err;
bool added = false;
hci_disable_per_advertising_sync(hdev, instance);
if (instance) {
adv = hci_find_adv_instance(hdev, instance);
/* Create an instance if that could not be found */
if (!adv) {
adv = hci_add_per_instance(hdev, instance, flags,
data_len, data,
sync_interval,
sync_interval);
if (IS_ERR(adv))
return PTR_ERR(adv);
adv->pending = false;
added = true;
}
}
/* Start advertising */
err = hci_start_ext_adv_sync(hdev, instance);
if (err < 0)
goto fail;
err = hci_adv_bcast_annoucement(hdev, adv);
if (err < 0)
goto fail;
err = hci_set_per_adv_params_sync(hdev, instance, min_interval,
max_interval);
if (err < 0)
goto fail;
err = hci_set_per_adv_data_sync(hdev, instance);
if (err < 0)
goto fail;
err = hci_enable_per_advertising_sync(hdev, instance);
if (err < 0)
goto fail;
return 0;
fail:
if (added)
hci_remove_adv_instance(hdev, instance);
return err;
}
static int hci_start_adv_sync(struct hci_dev *hdev, u8 instance)
{
int err;
if (ext_adv_capable(hdev))
return hci_start_ext_adv_sync(hdev, instance);
err = hci_update_adv_data_sync(hdev, instance);
if (err)
return err;
err = hci_update_scan_rsp_data_sync(hdev, instance);
if (err)
return err;
return hci_enable_advertising_sync(hdev);
}
int hci_enable_advertising_sync(struct hci_dev *hdev)
{
struct adv_info *adv_instance;
struct hci_cp_le_set_adv_param cp;
u8 own_addr_type, enable = 0x01;
bool connectable;
u16 adv_min_interval, adv_max_interval;
u32 flags;
u8 status;
if (ext_adv_capable(hdev))
return hci_enable_ext_advertising_sync(hdev,
hdev->cur_adv_instance);
flags = hci_adv_instance_flags(hdev, hdev->cur_adv_instance);
adv_instance = hci_find_adv_instance(hdev, hdev->cur_adv_instance);
/* If the "connectable" instance flag was not set, then choose between
* ADV_IND and ADV_NONCONN_IND based on the global connectable setting.
*/
connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) ||
mgmt_get_connectable(hdev);
if (!is_advertising_allowed(hdev, connectable))
return -EINVAL;
status = hci_disable_advertising_sync(hdev);
if (status)
return status;
/* Clear the HCI_LE_ADV bit temporarily so that the
* hci_update_random_address knows that it's safe to go ahead
* and write a new random address. The flag will be set back on
* as soon as the SET_ADV_ENABLE HCI command completes.
*/
hci_dev_clear_flag(hdev, HCI_LE_ADV);
/* Set require_privacy to true only when non-connectable
* advertising is used. In that case it is fine to use a
* non-resolvable private address.
*/
status = hci_update_random_address_sync(hdev, !connectable,
adv_use_rpa(hdev, flags),
&own_addr_type);
if (status)
return status;
memset(&cp, 0, sizeof(cp));
if (adv_instance) {
adv_min_interval = adv_instance->min_interval;
adv_max_interval = adv_instance->max_interval;
} else {
adv_min_interval = hdev->le_adv_min_interval;
adv_max_interval = hdev->le_adv_max_interval;
}
if (connectable) {
cp.type = LE_ADV_IND;
} else {
if (hci_adv_instance_is_scannable(hdev, hdev->cur_adv_instance))
cp.type = LE_ADV_SCAN_IND;
else
cp.type = LE_ADV_NONCONN_IND;
if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE) ||
hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) {
adv_min_interval = DISCOV_LE_FAST_ADV_INT_MIN;
adv_max_interval = DISCOV_LE_FAST_ADV_INT_MAX;
}
}
cp.min_interval = cpu_to_le16(adv_min_interval);
cp.max_interval = cpu_to_le16(adv_max_interval);
cp.own_address_type = own_addr_type;
cp.channel_map = hdev->le_adv_channel_map;
status = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_PARAM,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (status)
return status;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE,
sizeof(enable), &enable, HCI_CMD_TIMEOUT);
}
static int enable_advertising_sync(struct hci_dev *hdev, void *data)
{
return hci_enable_advertising_sync(hdev);
}
int hci_enable_advertising(struct hci_dev *hdev)
{
if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) &&
list_empty(&hdev->adv_instances))
return 0;
return hci_cmd_sync_queue(hdev, enable_advertising_sync, NULL, NULL);
}
int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance,
struct sock *sk)
{
int err;
if (!ext_adv_capable(hdev))
return 0;
err = hci_disable_ext_adv_instance_sync(hdev, instance);
if (err)
return err;
/* If request specifies an instance that doesn't exist, fail */
if (instance > 0 && !hci_find_adv_instance(hdev, instance))
return -EINVAL;
return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_REMOVE_ADV_SET,
sizeof(instance), &instance, 0,
HCI_CMD_TIMEOUT, sk);
}
static int remove_ext_adv_sync(struct hci_dev *hdev, void *data)
{
struct adv_info *adv = data;
u8 instance = 0;
if (adv)
instance = adv->instance;
return hci_remove_ext_adv_instance_sync(hdev, instance, NULL);
}
int hci_remove_ext_adv_instance(struct hci_dev *hdev, u8 instance)
{
struct adv_info *adv = NULL;
if (instance) {
adv = hci_find_adv_instance(hdev, instance);
if (!adv)
return -EINVAL;
}
return hci_cmd_sync_queue(hdev, remove_ext_adv_sync, adv, NULL);
}
int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason)
{
struct hci_cp_le_term_big cp;
memset(&cp, 0, sizeof(cp));
cp.handle = handle;
cp.reason = reason;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_TERM_BIG,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance)
{
struct {
struct hci_cp_le_set_ext_adv_data cp;
u8 data[HCI_MAX_EXT_AD_LENGTH];
} pdu;
u8 len;
struct adv_info *adv = NULL;
int err;
memset(&pdu, 0, sizeof(pdu));
if (instance) {
adv = hci_find_adv_instance(hdev, instance);
if (!adv || !adv->adv_data_changed)
return 0;
}
len = eir_create_adv_data(hdev, instance, pdu.data);
pdu.cp.length = len;
pdu.cp.handle = instance;
pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE;
pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG;
err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA,
sizeof(pdu.cp) + len, &pdu.cp,
HCI_CMD_TIMEOUT);
if (err)
return err;
/* Update data if the command succeed */
if (adv) {
adv->adv_data_changed = false;
} else {
memcpy(hdev->adv_data, pdu.data, len);
hdev->adv_data_len = len;
}
return 0;
}
static int hci_set_adv_data_sync(struct hci_dev *hdev, u8 instance)
{
struct hci_cp_le_set_adv_data cp;
u8 len;
memset(&cp, 0, sizeof(cp));
len = eir_create_adv_data(hdev, instance, cp.data);
/* There's nothing to do if the data hasn't changed */
if (hdev->adv_data_len == len &&
memcmp(cp.data, hdev->adv_data, len) == 0)
return 0;
memcpy(hdev->adv_data, cp.data, sizeof(cp.data));
hdev->adv_data_len = len;
cp.length = len;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_DATA,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance)
{
if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
return 0;
if (ext_adv_capable(hdev))
return hci_set_ext_adv_data_sync(hdev, instance);
return hci_set_adv_data_sync(hdev, instance);
}
int hci_schedule_adv_instance_sync(struct hci_dev *hdev, u8 instance,
bool force)
{
struct adv_info *adv = NULL;
u16 timeout;
if (hci_dev_test_flag(hdev, HCI_ADVERTISING) && !ext_adv_capable(hdev))
return -EPERM;
if (hdev->adv_instance_timeout)
return -EBUSY;
adv = hci_find_adv_instance(hdev, instance);
if (!adv)
return -ENOENT;
/* A zero timeout means unlimited advertising. As long as there is
* only one instance, duration should be ignored. We still set a timeout
* in case further instances are being added later on.
*
* If the remaining lifetime of the instance is more than the duration
* then the timeout corresponds to the duration, otherwise it will be
* reduced to the remaining instance lifetime.
*/
if (adv->timeout == 0 || adv->duration <= adv->remaining_time)
timeout = adv->duration;
else
timeout = adv->remaining_time;
/* The remaining time is being reduced unless the instance is being
* advertised without time limit.
*/
if (adv->timeout)
adv->remaining_time = adv->remaining_time - timeout;
/* Only use work for scheduling instances with legacy advertising */
if (!ext_adv_capable(hdev)) {
hdev->adv_instance_timeout = timeout;
queue_delayed_work(hdev->req_workqueue,
&hdev->adv_instance_expire,
msecs_to_jiffies(timeout * 1000));
}
/* If we're just re-scheduling the same instance again then do not
* execute any HCI commands. This happens when a single instance is
* being advertised.
*/
if (!force && hdev->cur_adv_instance == instance &&
hci_dev_test_flag(hdev, HCI_LE_ADV))
return 0;
hdev->cur_adv_instance = instance;
return hci_start_adv_sync(hdev, instance);
}
static int hci_clear_adv_sets_sync(struct hci_dev *hdev, struct sock *sk)
{
int err;
if (!ext_adv_capable(hdev))
return 0;
/* Disable instance 0x00 to disable all instances */
err = hci_disable_ext_adv_instance_sync(hdev, 0x00);
if (err)
return err;
return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_CLEAR_ADV_SETS,
0, NULL, 0, HCI_CMD_TIMEOUT, sk);
}
static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force)
{
struct adv_info *adv, *n;
int err = 0;
if (ext_adv_capable(hdev))
/* Remove all existing sets */
err = hci_clear_adv_sets_sync(hdev, sk);
if (ext_adv_capable(hdev))
return err;
/* This is safe as long as there is no command send while the lock is
* held.
*/
hci_dev_lock(hdev);
/* Cleanup non-ext instances */
list_for_each_entry_safe(adv, n, &hdev->adv_instances, list) {
u8 instance = adv->instance;
int err;
if (!(force || adv->timeout))
continue;
err = hci_remove_adv_instance(hdev, instance);
if (!err)
mgmt_advertising_removed(sk, hdev, instance);
}
hci_dev_unlock(hdev);
return 0;
}
static int hci_remove_adv_sync(struct hci_dev *hdev, u8 instance,
struct sock *sk)
{
int err = 0;
/* If we use extended advertising, instance has to be removed first. */
if (ext_adv_capable(hdev))
err = hci_remove_ext_adv_instance_sync(hdev, instance, sk);
if (ext_adv_capable(hdev))
return err;
/* This is safe as long as there is no command send while the lock is
* held.
*/
hci_dev_lock(hdev);
err = hci_remove_adv_instance(hdev, instance);
if (!err)
mgmt_advertising_removed(sk, hdev, instance);
hci_dev_unlock(hdev);
return err;
}
/* For a single instance:
* - force == true: The instance will be removed even when its remaining
* lifetime is not zero.
* - force == false: the instance will be deactivated but kept stored unless
* the remaining lifetime is zero.
*
* For instance == 0x00:
* - force == true: All instances will be removed regardless of their timeout
* setting.
* - force == false: Only instances that have a timeout will be removed.
*/
int hci_remove_advertising_sync(struct hci_dev *hdev, struct sock *sk,
u8 instance, bool force)
{
struct adv_info *next = NULL;
int err;
/* Cancel any timeout concerning the removed instance(s). */
if (!instance || hdev->cur_adv_instance == instance)
cancel_adv_timeout(hdev);
/* Get the next instance to advertise BEFORE we remove
* the current one. This can be the same instance again
* if there is only one instance.
*/
if (hdev->cur_adv_instance == instance)
next = hci_get_next_instance(hdev, instance);
if (!instance) {
err = hci_clear_adv_sync(hdev, sk, force);
if (err)
return err;
} else {
struct adv_info *adv = hci_find_adv_instance(hdev, instance);
if (force || (adv && adv->timeout && !adv->remaining_time)) {
/* Don't advertise a removed instance. */
if (next && next->instance == instance)
next = NULL;
err = hci_remove_adv_sync(hdev, instance, sk);
if (err)
return err;
}
}
if (!hdev_is_powered(hdev) || hci_dev_test_flag(hdev, HCI_ADVERTISING))
return 0;
if (next && !ext_adv_capable(hdev))
hci_schedule_adv_instance_sync(hdev, next->instance, false);
return 0;
}
int hci_read_rssi_sync(struct hci_dev *hdev, __le16 handle)
{
struct hci_cp_read_rssi cp;
cp.handle = handle;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_RSSI,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_read_clock_sync(struct hci_dev *hdev, struct hci_cp_read_clock *cp)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_CLOCK,
sizeof(*cp), cp, HCI_CMD_TIMEOUT);
}
int hci_read_tx_power_sync(struct hci_dev *hdev, __le16 handle, u8 type)
{
struct hci_cp_read_tx_power cp;
cp.handle = handle;
cp.type = type;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_TX_POWER,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_disable_advertising_sync(struct hci_dev *hdev)
{
u8 enable = 0x00;
int err = 0;
/* If controller is not advertising we are done. */
if (!hci_dev_test_flag(hdev, HCI_LE_ADV))
return 0;
if (ext_adv_capable(hdev))
err = hci_disable_ext_adv_instance_sync(hdev, 0x00);
if (ext_adv_capable(hdev))
return err;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE,
sizeof(enable), &enable, HCI_CMD_TIMEOUT);
}
static int hci_le_set_ext_scan_enable_sync(struct hci_dev *hdev, u8 val,
u8 filter_dup)
{
struct hci_cp_le_set_ext_scan_enable cp;
memset(&cp, 0, sizeof(cp));
cp.enable = val;
if (hci_dev_test_flag(hdev, HCI_MESH))
cp.filter_dup = LE_SCAN_FILTER_DUP_DISABLE;
else
cp.filter_dup = filter_dup;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_ENABLE,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val,
u8 filter_dup)
{
struct hci_cp_le_set_scan_enable cp;
if (use_ext_scan(hdev))
return hci_le_set_ext_scan_enable_sync(hdev, val, filter_dup);
memset(&cp, 0, sizeof(cp));
cp.enable = val;
if (val && hci_dev_test_flag(hdev, HCI_MESH))
cp.filter_dup = LE_SCAN_FILTER_DUP_DISABLE;
else
cp.filter_dup = filter_dup;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_ENABLE,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_le_set_addr_resolution_enable_sync(struct hci_dev *hdev, u8 val)
{
if (!use_ll_privacy(hdev))
return 0;
/* If controller is not/already resolving we are done. */
if (val == hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE,
sizeof(val), &val, HCI_CMD_TIMEOUT);
}
static int hci_scan_disable_sync(struct hci_dev *hdev)
{
int err;
/* If controller is not scanning we are done. */
if (!hci_dev_test_flag(hdev, HCI_LE_SCAN))
return 0;
if (hdev->scanning_paused) {
bt_dev_dbg(hdev, "Scanning is paused for suspend");
return 0;
}
err = hci_le_set_scan_enable_sync(hdev, LE_SCAN_DISABLE, 0x00);
if (err) {
bt_dev_err(hdev, "Unable to disable scanning: %d", err);
return err;
}
return err;
}
static bool scan_use_rpa(struct hci_dev *hdev)
{
return hci_dev_test_flag(hdev, HCI_PRIVACY);
}
static void hci_start_interleave_scan(struct hci_dev *hdev)
{
hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER;
queue_delayed_work(hdev->req_workqueue,
&hdev->interleave_scan, 0);
}
static bool is_interleave_scanning(struct hci_dev *hdev)
{
return hdev->interleave_scan_state != INTERLEAVE_SCAN_NONE;
}
static void cancel_interleave_scan(struct hci_dev *hdev)
{
bt_dev_dbg(hdev, "cancelling interleave scan");
cancel_delayed_work_sync(&hdev->interleave_scan);
hdev->interleave_scan_state = INTERLEAVE_SCAN_NONE;
}
/* Return true if interleave_scan wasn't started until exiting this function,
* otherwise, return false
*/
static bool hci_update_interleaved_scan_sync(struct hci_dev *hdev)
{
/* Do interleaved scan only if all of the following are true:
* - There is at least one ADV monitor
* - At least one pending LE connection or one device to be scanned for
* - Monitor offloading is not supported
* If so, we should alternate between allowlist scan and one without
* any filters to save power.
*/
bool use_interleaving = hci_is_adv_monitoring(hdev) &&
!(list_empty(&hdev->pend_le_conns) &&
list_empty(&hdev->pend_le_reports)) &&
hci_get_adv_monitor_offload_ext(hdev) ==
HCI_ADV_MONITOR_EXT_NONE;
bool is_interleaving = is_interleave_scanning(hdev);
if (use_interleaving && !is_interleaving) {
hci_start_interleave_scan(hdev);
bt_dev_dbg(hdev, "starting interleave scan");
return true;
}
if (!use_interleaving && is_interleaving)
cancel_interleave_scan(hdev);
return false;
}
/* Removes connection to resolve list if needed.*/
static int hci_le_del_resolve_list_sync(struct hci_dev *hdev,
bdaddr_t *bdaddr, u8 bdaddr_type)
{
struct hci_cp_le_del_from_resolv_list cp;
struct bdaddr_list_with_irk *entry;
if (!use_ll_privacy(hdev))
return 0;
/* Check if the IRK has been programmed */
entry = hci_bdaddr_list_lookup_with_irk(&hdev->le_resolv_list, bdaddr,
bdaddr_type);
if (!entry)
return 0;
cp.bdaddr_type = bdaddr_type;
bacpy(&cp.bdaddr, bdaddr);
return __hci_cmd_sync_status(hdev, HCI_OP_LE_DEL_FROM_RESOLV_LIST,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_le_del_accept_list_sync(struct hci_dev *hdev,
bdaddr_t *bdaddr, u8 bdaddr_type)
{
struct hci_cp_le_del_from_accept_list cp;
int err;
/* Check if device is on accept list before removing it */
if (!hci_bdaddr_list_lookup(&hdev->le_accept_list, bdaddr, bdaddr_type))
return 0;
cp.bdaddr_type = bdaddr_type;
bacpy(&cp.bdaddr, bdaddr);
/* Ignore errors when removing from resolving list as that is likely
* that the device was never added.
*/
hci_le_del_resolve_list_sync(hdev, &cp.bdaddr, cp.bdaddr_type);
err = __hci_cmd_sync_status(hdev, HCI_OP_LE_DEL_FROM_ACCEPT_LIST,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (err) {
bt_dev_err(hdev, "Unable to remove from allow list: %d", err);
return err;
}
bt_dev_dbg(hdev, "Remove %pMR (0x%x) from allow list", &cp.bdaddr,
cp.bdaddr_type);
return 0;
}
struct conn_params {
bdaddr_t addr;
u8 addr_type;
hci_conn_flags_t flags;
u8 privacy_mode;
};
/* Adds connection to resolve list if needed.
* Setting params to NULL programs local hdev->irk
*/
static int hci_le_add_resolve_list_sync(struct hci_dev *hdev,
struct conn_params *params)
{
struct hci_cp_le_add_to_resolv_list cp;
struct smp_irk *irk;
struct bdaddr_list_with_irk *entry;
struct hci_conn_params *p;
if (!use_ll_privacy(hdev))
return 0;
/* Attempt to program local identity address, type and irk if params is
* NULL.
*/
if (!params) {
if (!hci_dev_test_flag(hdev, HCI_PRIVACY))
return 0;
hci_copy_identity_address(hdev, &cp.bdaddr, &cp.bdaddr_type);
memcpy(cp.peer_irk, hdev->irk, 16);
goto done;
}
irk = hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type);
if (!irk)
return 0;
/* Check if the IK has _not_ been programmed yet. */
entry = hci_bdaddr_list_lookup_with_irk(&hdev->le_resolv_list,
¶ms->addr,
params->addr_type);
if (entry)
return 0;
cp.bdaddr_type = params->addr_type;
bacpy(&cp.bdaddr, ¶ms->addr);
memcpy(cp.peer_irk, irk->val, 16);
/* Default privacy mode is always Network */
params->privacy_mode = HCI_NETWORK_PRIVACY;
rcu_read_lock();
p = hci_pend_le_action_lookup(&hdev->pend_le_conns,
¶ms->addr, params->addr_type);
if (!p)
p = hci_pend_le_action_lookup(&hdev->pend_le_reports,
¶ms->addr, params->addr_type);
if (p)
WRITE_ONCE(p->privacy_mode, HCI_NETWORK_PRIVACY);
rcu_read_unlock();
done:
if (hci_dev_test_flag(hdev, HCI_PRIVACY))
memcpy(cp.local_irk, hdev->irk, 16);
else
memset(cp.local_irk, 0, 16);
return __hci_cmd_sync_status(hdev, HCI_OP_LE_ADD_TO_RESOLV_LIST,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
/* Set Device Privacy Mode. */
static int hci_le_set_privacy_mode_sync(struct hci_dev *hdev,
struct conn_params *params)
{
struct hci_cp_le_set_privacy_mode cp;
struct smp_irk *irk;
/* If device privacy mode has already been set there is nothing to do */
if (params->privacy_mode == HCI_DEVICE_PRIVACY)
return 0;
/* Check if HCI_CONN_FLAG_DEVICE_PRIVACY has been set as it also
* indicates that LL Privacy has been enabled and
* HCI_OP_LE_SET_PRIVACY_MODE is supported.
*/
if (!(params->flags & HCI_CONN_FLAG_DEVICE_PRIVACY))
return 0;
irk = hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type);
if (!irk)
return 0;
memset(&cp, 0, sizeof(cp));
cp.bdaddr_type = irk->addr_type;
bacpy(&cp.bdaddr, &irk->bdaddr);
cp.mode = HCI_DEVICE_PRIVACY;
/* Note: params->privacy_mode is not updated since it is a copy */
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PRIVACY_MODE,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
/* Adds connection to allow list if needed, if the device uses RPA (has IRK)
* this attempts to program the device in the resolving list as well and
* properly set the privacy mode.
*/
static int hci_le_add_accept_list_sync(struct hci_dev *hdev,
struct conn_params *params,
u8 *num_entries)
{
struct hci_cp_le_add_to_accept_list cp;
int err;
/* During suspend, only wakeable devices can be in acceptlist */
if (hdev->suspended &&
!(params->flags & HCI_CONN_FLAG_REMOTE_WAKEUP)) {
hci_le_del_accept_list_sync(hdev, ¶ms->addr,
params->addr_type);
return 0;
}
/* Select filter policy to accept all advertising */
if (*num_entries >= hdev->le_accept_list_size)
return -ENOSPC;
/* Accept list can not be used with RPAs */
if (!use_ll_privacy(hdev) &&
hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type))
return -EINVAL;
/* Attempt to program the device in the resolving list first to avoid
* having to rollback in case it fails since the resolving list is
* dynamic it can probably be smaller than the accept list.
*/
err = hci_le_add_resolve_list_sync(hdev, params);
if (err) {
bt_dev_err(hdev, "Unable to add to resolve list: %d", err);
return err;
}
/* Set Privacy Mode */
err = hci_le_set_privacy_mode_sync(hdev, params);
if (err) {
bt_dev_err(hdev, "Unable to set privacy mode: %d", err);
return err;
}
/* Check if already in accept list */
if (hci_bdaddr_list_lookup(&hdev->le_accept_list, ¶ms->addr,
params->addr_type))
return 0;
*num_entries += 1;
cp.bdaddr_type = params->addr_type;
bacpy(&cp.bdaddr, ¶ms->addr);
err = __hci_cmd_sync_status(hdev, HCI_OP_LE_ADD_TO_ACCEPT_LIST,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (err) {
bt_dev_err(hdev, "Unable to add to allow list: %d", err);
/* Rollback the device from the resolving list */
hci_le_del_resolve_list_sync(hdev, &cp.bdaddr, cp.bdaddr_type);
return err;
}
bt_dev_dbg(hdev, "Add %pMR (0x%x) to allow list", &cp.bdaddr,
cp.bdaddr_type);
return 0;
}
/* This function disables/pause all advertising instances */
static int hci_pause_advertising_sync(struct hci_dev *hdev)
{
int err;
int old_state;
/* If already been paused there is nothing to do. */
if (hdev->advertising_paused)
return 0;
bt_dev_dbg(hdev, "Pausing directed advertising");
/* Stop directed advertising */
old_state = hci_dev_test_flag(hdev, HCI_ADVERTISING);
if (old_state) {
/* When discoverable timeout triggers, then just make sure
* the limited discoverable flag is cleared. Even in the case
* of a timeout triggered from general discoverable, it is
* safe to unconditionally clear the flag.
*/
hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE);
hci_dev_clear_flag(hdev, HCI_DISCOVERABLE);
hdev->discov_timeout = 0;
}
bt_dev_dbg(hdev, "Pausing advertising instances");
/* Call to disable any advertisements active on the controller.
* This will succeed even if no advertisements are configured.
*/
err = hci_disable_advertising_sync(hdev);
if (err)
return err;
/* If we are using software rotation, pause the loop */
if (!ext_adv_capable(hdev))
cancel_adv_timeout(hdev);
hdev->advertising_paused = true;
hdev->advertising_old_state = old_state;
return 0;
}
/* This function enables all user advertising instances */
static int hci_resume_advertising_sync(struct hci_dev *hdev)
{
struct adv_info *adv, *tmp;
int err;
/* If advertising has not been paused there is nothing to do. */
if (!hdev->advertising_paused)
return 0;
/* Resume directed advertising */
hdev->advertising_paused = false;
if (hdev->advertising_old_state) {
hci_dev_set_flag(hdev, HCI_ADVERTISING);
hdev->advertising_old_state = 0;
}
bt_dev_dbg(hdev, "Resuming advertising instances");
if (ext_adv_capable(hdev)) {
/* Call for each tracked instance to be re-enabled */
list_for_each_entry_safe(adv, tmp, &hdev->adv_instances, list) {
err = hci_enable_ext_advertising_sync(hdev,
adv->instance);
if (!err)
continue;
/* If the instance cannot be resumed remove it */
hci_remove_ext_adv_instance_sync(hdev, adv->instance,
NULL);
}
} else {
/* Schedule for most recent instance to be restarted and begin
* the software rotation loop
*/
err = hci_schedule_adv_instance_sync(hdev,
hdev->cur_adv_instance,
true);
}
hdev->advertising_paused = false;
return err;
}
static int hci_pause_addr_resolution(struct hci_dev *hdev)
{
int err;
if (!use_ll_privacy(hdev))
return 0;
if (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION))
return 0;
/* Cannot disable addr resolution if scanning is enabled or
* when initiating an LE connection.
*/
if (hci_dev_test_flag(hdev, HCI_LE_SCAN) ||
hci_lookup_le_connect(hdev)) {
bt_dev_err(hdev, "Command not allowed when scan/LE connect");
return -EPERM;
}
/* Cannot disable addr resolution if advertising is enabled. */
err = hci_pause_advertising_sync(hdev);
if (err) {
bt_dev_err(hdev, "Pause advertising failed: %d", err);
return err;
}
err = hci_le_set_addr_resolution_enable_sync(hdev, 0x00);
if (err)
bt_dev_err(hdev, "Unable to disable Address Resolution: %d",
err);
/* Return if address resolution is disabled and RPA is not used. */
if (!err && scan_use_rpa(hdev))
return 0;
hci_resume_advertising_sync(hdev);
return err;
}
struct sk_buff *hci_read_local_oob_data_sync(struct hci_dev *hdev,
bool extended, struct sock *sk)
{
u16 opcode = extended ? HCI_OP_READ_LOCAL_OOB_EXT_DATA :
HCI_OP_READ_LOCAL_OOB_DATA;
return __hci_cmd_sync_sk(hdev, opcode, 0, NULL, 0, HCI_CMD_TIMEOUT, sk);
}
static struct conn_params *conn_params_copy(struct list_head *list, size_t *n)
{
struct hci_conn_params *params;
struct conn_params *p;
size_t i;
rcu_read_lock();
i = 0;
list_for_each_entry_rcu(params, list, action)
++i;
*n = i;
rcu_read_unlock();
p = kvcalloc(*n, sizeof(struct conn_params), GFP_KERNEL);
if (!p)
return NULL;
rcu_read_lock();
i = 0;
list_for_each_entry_rcu(params, list, action) {
/* Racing adds are handled in next scan update */
if (i >= *n)
break;
/* No hdev->lock, but: addr, addr_type are immutable.
* privacy_mode is only written by us or in
* hci_cc_le_set_privacy_mode that we wait for.
* We should be idempotent so MGMT updating flags
* while we are processing is OK.
*/
bacpy(&p[i].addr, ¶ms->addr);
p[i].addr_type = params->addr_type;
p[i].flags = READ_ONCE(params->flags);
p[i].privacy_mode = READ_ONCE(params->privacy_mode);
++i;
}
rcu_read_unlock();
*n = i;
return p;
}
/* Clear LE Accept List */
static int hci_le_clear_accept_list_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[26] & 0x80))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_CLEAR_ACCEPT_LIST, 0, NULL,
HCI_CMD_TIMEOUT);
}
/* Device must not be scanning when updating the accept list.
*
* Update is done using the following sequence:
*
* use_ll_privacy((Disable Advertising) -> Disable Resolving List) ->
* Remove Devices From Accept List ->
* (has IRK && use_ll_privacy(Remove Devices From Resolving List))->
* Add Devices to Accept List ->
* (has IRK && use_ll_privacy(Remove Devices From Resolving List)) ->
* use_ll_privacy(Enable Resolving List -> (Enable Advertising)) ->
* Enable Scanning
*
* In case of failure advertising shall be restored to its original state and
* return would disable accept list since either accept or resolving list could
* not be programmed.
*
*/
static u8 hci_update_accept_list_sync(struct hci_dev *hdev)
{
struct conn_params *params;
struct bdaddr_list *b, *t;
u8 num_entries = 0;
bool pend_conn, pend_report;
u8 filter_policy;
size_t i, n;
int err;
/* Pause advertising if resolving list can be used as controllers
* cannot accept resolving list modifications while advertising.
*/
if (use_ll_privacy(hdev)) {
err = hci_pause_advertising_sync(hdev);
if (err) {
bt_dev_err(hdev, "pause advertising failed: %d", err);
return 0x00;
}
}
/* Disable address resolution while reprogramming accept list since
* devices that do have an IRK will be programmed in the resolving list
* when LL Privacy is enabled.
*/
err = hci_le_set_addr_resolution_enable_sync(hdev, 0x00);
if (err) {
bt_dev_err(hdev, "Unable to disable LL privacy: %d", err);
goto done;
}
/* Force address filtering if PA Sync is in progress */
if (hci_dev_test_flag(hdev, HCI_PA_SYNC)) {
struct hci_cp_le_pa_create_sync *sent;
sent = hci_sent_cmd_data(hdev, HCI_OP_LE_PA_CREATE_SYNC);
if (sent) {
struct conn_params pa;
memset(&pa, 0, sizeof(pa));
bacpy(&pa.addr, &sent->addr);
pa.addr_type = sent->addr_type;
/* Clear first since there could be addresses left
* behind.
*/
hci_le_clear_accept_list_sync(hdev);
num_entries = 1;
err = hci_le_add_accept_list_sync(hdev, &pa,
&num_entries);
goto done;
}
}
/* Go through the current accept list programmed into the
* controller one by one and check if that address is connected or is
* still in the list of pending connections or list of devices to
* report. If not present in either list, then remove it from
* the controller.
*/
list_for_each_entry_safe(b, t, &hdev->le_accept_list, list) {
if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type))
continue;
/* Pointers not dereferenced, no locks needed */
pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns,
&b->bdaddr,
b->bdaddr_type);
pend_report = hci_pend_le_action_lookup(&hdev->pend_le_reports,
&b->bdaddr,
b->bdaddr_type);
/* If the device is not likely to connect or report,
* remove it from the acceptlist.
*/
if (!pend_conn && !pend_report) {
hci_le_del_accept_list_sync(hdev, &b->bdaddr,
b->bdaddr_type);
continue;
}
num_entries++;
}
/* Since all no longer valid accept list entries have been
* removed, walk through the list of pending connections
* and ensure that any new device gets programmed into
* the controller.
*
* If the list of the devices is larger than the list of
* available accept list entries in the controller, then
* just abort and return filer policy value to not use the
* accept list.
*
* The list and params may be mutated while we wait for events,
* so make a copy and iterate it.
*/
params = conn_params_copy(&hdev->pend_le_conns, &n);
if (!params) {
err = -ENOMEM;
goto done;
}
for (i = 0; i < n; ++i) {
err = hci_le_add_accept_list_sync(hdev, ¶ms[i],
&num_entries);
if (err) {
kvfree(params);
goto done;
}
}
kvfree(params);
/* After adding all new pending connections, walk through
* the list of pending reports and also add these to the
* accept list if there is still space. Abort if space runs out.
*/
params = conn_params_copy(&hdev->pend_le_reports, &n);
if (!params) {
err = -ENOMEM;
goto done;
}
for (i = 0; i < n; ++i) {
err = hci_le_add_accept_list_sync(hdev, ¶ms[i],
&num_entries);
if (err) {
kvfree(params);
goto done;
}
}
kvfree(params);
/* Use the allowlist unless the following conditions are all true:
* - We are not currently suspending
* - There are 1 or more ADV monitors registered and it's not offloaded
* - Interleaved scanning is not currently using the allowlist
*/
if (!idr_is_empty(&hdev->adv_monitors_idr) && !hdev->suspended &&
hci_get_adv_monitor_offload_ext(hdev) == HCI_ADV_MONITOR_EXT_NONE &&
hdev->interleave_scan_state != INTERLEAVE_SCAN_ALLOWLIST)
err = -EINVAL;
done:
filter_policy = err ? 0x00 : 0x01;
/* Enable address resolution when LL Privacy is enabled. */
err = hci_le_set_addr_resolution_enable_sync(hdev, 0x01);
if (err)
bt_dev_err(hdev, "Unable to enable LL privacy: %d", err);
/* Resume advertising if it was paused */
if (use_ll_privacy(hdev))
hci_resume_advertising_sync(hdev);
/* Select filter policy to use accept list */
return filter_policy;
}
static void hci_le_scan_phy_params(struct hci_cp_le_scan_phy_params *cp,
u8 type, u16 interval, u16 window)
{
cp->type = type;
cp->interval = cpu_to_le16(interval);
cp->window = cpu_to_le16(window);
}
static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type,
u16 interval, u16 window,
u8 own_addr_type, u8 filter_policy)
{
struct hci_cp_le_set_ext_scan_params *cp;
struct hci_cp_le_scan_phy_params *phy;
u8 data[sizeof(*cp) + sizeof(*phy) * 2];
u8 num_phy = 0x00;
cp = (void *)data;
phy = (void *)cp->data;
memset(data, 0, sizeof(data));
cp->own_addr_type = own_addr_type;
cp->filter_policy = filter_policy;
/* Check if PA Sync is in progress then select the PHY based on the
* hci_conn.iso_qos.
*/
if (hci_dev_test_flag(hdev, HCI_PA_SYNC)) {
struct hci_cp_le_add_to_accept_list *sent;
sent = hci_sent_cmd_data(hdev, HCI_OP_LE_ADD_TO_ACCEPT_LIST);
if (sent) {
struct hci_conn *conn;
conn = hci_conn_hash_lookup_ba(hdev, ISO_LINK,
&sent->bdaddr);
if (conn) {
struct bt_iso_qos *qos = &conn->iso_qos;
if (qos->bcast.in.phy & BT_ISO_PHY_1M ||
qos->bcast.in.phy & BT_ISO_PHY_2M) {
cp->scanning_phys |= LE_SCAN_PHY_1M;
hci_le_scan_phy_params(phy, type,
interval,
window);
num_phy++;
phy++;
}
if (qos->bcast.in.phy & BT_ISO_PHY_CODED) {
cp->scanning_phys |= LE_SCAN_PHY_CODED;
hci_le_scan_phy_params(phy, type,
interval * 3,
window * 3);
num_phy++;
phy++;
}
if (num_phy)
goto done;
}
}
}
if (scan_1m(hdev) || scan_2m(hdev)) {
cp->scanning_phys |= LE_SCAN_PHY_1M;
hci_le_scan_phy_params(phy, type, interval, window);
num_phy++;
phy++;
}
if (scan_coded(hdev)) {
cp->scanning_phys |= LE_SCAN_PHY_CODED;
hci_le_scan_phy_params(phy, type, interval * 3, window * 3);
num_phy++;
phy++;
}
done:
if (!num_phy)
return -EINVAL;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_PARAMS,
sizeof(*cp) + sizeof(*phy) * num_phy,
data, HCI_CMD_TIMEOUT);
}
static int hci_le_set_scan_param_sync(struct hci_dev *hdev, u8 type,
u16 interval, u16 window,
u8 own_addr_type, u8 filter_policy)
{
struct hci_cp_le_set_scan_param cp;
if (use_ext_scan(hdev))
return hci_le_set_ext_scan_param_sync(hdev, type, interval,
window, own_addr_type,
filter_policy);
memset(&cp, 0, sizeof(cp));
cp.type = type;
cp.interval = cpu_to_le16(interval);
cp.window = cpu_to_le16(window);
cp.own_address_type = own_addr_type;
cp.filter_policy = filter_policy;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_PARAM,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_start_scan_sync(struct hci_dev *hdev, u8 type, u16 interval,
u16 window, u8 own_addr_type, u8 filter_policy,
u8 filter_dup)
{
int err;
if (hdev->scanning_paused) {
bt_dev_dbg(hdev, "Scanning is paused for suspend");
return 0;
}
err = hci_le_set_scan_param_sync(hdev, type, interval, window,
own_addr_type, filter_policy);
if (err)
return err;
return hci_le_set_scan_enable_sync(hdev, LE_SCAN_ENABLE, filter_dup);
}
static int hci_passive_scan_sync(struct hci_dev *hdev)
{
u8 own_addr_type;
u8 filter_policy;
u16 window, interval;
u8 filter_dups = LE_SCAN_FILTER_DUP_ENABLE;
int err;
if (hdev->scanning_paused) {
bt_dev_dbg(hdev, "Scanning is paused for suspend");
return 0;
}
err = hci_scan_disable_sync(hdev);
if (err) {
bt_dev_err(hdev, "disable scanning failed: %d", err);
return err;
}
/* Set require_privacy to false since no SCAN_REQ are send
* during passive scanning. Not using an non-resolvable address
* here is important so that peer devices using direct
* advertising with our address will be correctly reported
* by the controller.
*/
if (hci_update_random_address_sync(hdev, false, scan_use_rpa(hdev),
&own_addr_type))
return 0;
if (hdev->enable_advmon_interleave_scan &&
hci_update_interleaved_scan_sync(hdev))
return 0;
bt_dev_dbg(hdev, "interleave state %d", hdev->interleave_scan_state);
/* Adding or removing entries from the accept list must
* happen before enabling scanning. The controller does
* not allow accept list modification while scanning.
*/
filter_policy = hci_update_accept_list_sync(hdev);
/* When the controller is using random resolvable addresses and
* with that having LE privacy enabled, then controllers with
* Extended Scanner Filter Policies support can now enable support
* for handling directed advertising.
*
* So instead of using filter polices 0x00 (no acceptlist)
* and 0x01 (acceptlist enabled) use the new filter policies
* 0x02 (no acceptlist) and 0x03 (acceptlist enabled).
*/
if (hci_dev_test_flag(hdev, HCI_PRIVACY) &&
(hdev->le_features[0] & HCI_LE_EXT_SCAN_POLICY))
filter_policy |= 0x02;
if (hdev->suspended) {
window = hdev->le_scan_window_suspend;
interval = hdev->le_scan_int_suspend;
} else if (hci_is_le_conn_scanning(hdev)) {
window = hdev->le_scan_window_connect;
interval = hdev->le_scan_int_connect;
} else if (hci_is_adv_monitoring(hdev)) {
window = hdev->le_scan_window_adv_monitor;
interval = hdev->le_scan_int_adv_monitor;
} else {
window = hdev->le_scan_window;
interval = hdev->le_scan_interval;
}
/* Disable all filtering for Mesh */
if (hci_dev_test_flag(hdev, HCI_MESH)) {
filter_policy = 0;
filter_dups = LE_SCAN_FILTER_DUP_DISABLE;
}
bt_dev_dbg(hdev, "LE passive scan with acceptlist = %d", filter_policy);
return hci_start_scan_sync(hdev, LE_SCAN_PASSIVE, interval, window,
own_addr_type, filter_policy, filter_dups);
}
/* This function controls the passive scanning based on hdev->pend_le_conns
* list. If there are pending LE connection we start the background scanning,
* otherwise we stop it in the following sequence:
*
* If there are devices to scan:
*
* Disable Scanning -> Update Accept List ->
* use_ll_privacy((Disable Advertising) -> Disable Resolving List ->
* Update Resolving List -> Enable Resolving List -> (Enable Advertising)) ->
* Enable Scanning
*
* Otherwise:
*
* Disable Scanning
*/
int hci_update_passive_scan_sync(struct hci_dev *hdev)
{
int err;
if (!test_bit(HCI_UP, &hdev->flags) ||
test_bit(HCI_INIT, &hdev->flags) ||
hci_dev_test_flag(hdev, HCI_SETUP) ||
hci_dev_test_flag(hdev, HCI_CONFIG) ||
hci_dev_test_flag(hdev, HCI_AUTO_OFF) ||
hci_dev_test_flag(hdev, HCI_UNREGISTER))
return 0;
/* No point in doing scanning if LE support hasn't been enabled */
if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
return 0;
/* If discovery is active don't interfere with it */
if (hdev->discovery.state != DISCOVERY_STOPPED)
return 0;
/* Reset RSSI and UUID filters when starting background scanning
* since these filters are meant for service discovery only.
*
* The Start Discovery and Start Service Discovery operations
* ensure to set proper values for RSSI threshold and UUID
* filter list. So it is safe to just reset them here.
*/
hci_discovery_filter_clear(hdev);
bt_dev_dbg(hdev, "ADV monitoring is %s",
hci_is_adv_monitoring(hdev) ? "on" : "off");
if (!hci_dev_test_flag(hdev, HCI_MESH) &&
list_empty(&hdev->pend_le_conns) &&
list_empty(&hdev->pend_le_reports) &&
!hci_is_adv_monitoring(hdev) &&
!hci_dev_test_flag(hdev, HCI_PA_SYNC)) {
/* If there is no pending LE connections or devices
* to be scanned for or no ADV monitors, we should stop the
* background scanning.
*/
bt_dev_dbg(hdev, "stopping background scanning");
err = hci_scan_disable_sync(hdev);
if (err)
bt_dev_err(hdev, "stop background scanning failed: %d",
err);
} else {
/* If there is at least one pending LE connection, we should
* keep the background scan running.
*/
/* If controller is connecting, we should not start scanning
* since some controllers are not able to scan and connect at
* the same time.
*/
if (hci_lookup_le_connect(hdev))
return 0;
bt_dev_dbg(hdev, "start background scanning");
err = hci_passive_scan_sync(hdev);
if (err)
bt_dev_err(hdev, "start background scanning failed: %d",
err);
}
return err;
}
static int update_scan_sync(struct hci_dev *hdev, void *data)
{
return hci_update_scan_sync(hdev);
}
int hci_update_scan(struct hci_dev *hdev)
{
return hci_cmd_sync_queue(hdev, update_scan_sync, NULL, NULL);
}
static int update_passive_scan_sync(struct hci_dev *hdev, void *data)
{
return hci_update_passive_scan_sync(hdev);
}
int hci_update_passive_scan(struct hci_dev *hdev)
{
/* Only queue if it would have any effect */
if (!test_bit(HCI_UP, &hdev->flags) ||
test_bit(HCI_INIT, &hdev->flags) ||
hci_dev_test_flag(hdev, HCI_SETUP) ||
hci_dev_test_flag(hdev, HCI_CONFIG) ||
hci_dev_test_flag(hdev, HCI_AUTO_OFF) ||
hci_dev_test_flag(hdev, HCI_UNREGISTER))
return 0;
return hci_cmd_sync_queue_once(hdev, update_passive_scan_sync, NULL,
NULL);
}
int hci_write_sc_support_sync(struct hci_dev *hdev, u8 val)
{
int err;
if (!bredr_sc_enabled(hdev) || lmp_host_sc_capable(hdev))
return 0;
err = __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SC_SUPPORT,
sizeof(val), &val, HCI_CMD_TIMEOUT);
if (!err) {
if (val) {
hdev->features[1][0] |= LMP_HOST_SC;
hci_dev_set_flag(hdev, HCI_SC_ENABLED);
} else {
hdev->features[1][0] &= ~LMP_HOST_SC;
hci_dev_clear_flag(hdev, HCI_SC_ENABLED);
}
}
return err;
}
int hci_write_ssp_mode_sync(struct hci_dev *hdev, u8 mode)
{
int err;
if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED) ||
lmp_host_ssp_capable(hdev))
return 0;
if (!mode && hci_dev_test_flag(hdev, HCI_USE_DEBUG_KEYS)) {
__hci_cmd_sync_status(hdev, HCI_OP_WRITE_SSP_DEBUG_MODE,
sizeof(mode), &mode, HCI_CMD_TIMEOUT);
}
err = __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SSP_MODE,
sizeof(mode), &mode, HCI_CMD_TIMEOUT);
if (err)
return err;
return hci_write_sc_support_sync(hdev, 0x01);
}
int hci_write_le_host_supported_sync(struct hci_dev *hdev, u8 le, u8 simul)
{
struct hci_cp_write_le_host_supported cp;
if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED) ||
!lmp_bredr_capable(hdev))
return 0;
/* Check first if we already have the right host state
* (host features set)
*/
if (le == lmp_host_le_capable(hdev) &&
simul == lmp_host_le_br_capable(hdev))
return 0;
memset(&cp, 0, sizeof(cp));
cp.le = le;
cp.simul = simul;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LE_HOST_SUPPORTED,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_powered_update_adv_sync(struct hci_dev *hdev)
{
struct adv_info *adv, *tmp;
int err;
if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
return 0;
/* If RPA Resolution has not been enable yet it means the
* resolving list is empty and we should attempt to program the
* local IRK in order to support using own_addr_type
* ADDR_LE_DEV_RANDOM_RESOLVED (0x03).
*/
if (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) {
hci_le_add_resolve_list_sync(hdev, NULL);
hci_le_set_addr_resolution_enable_sync(hdev, 0x01);
}
/* Make sure the controller has a good default for
* advertising data. This also applies to the case
* where BR/EDR was toggled during the AUTO_OFF phase.
*/
if (hci_dev_test_flag(hdev, HCI_ADVERTISING) ||
list_empty(&hdev->adv_instances)) {
if (ext_adv_capable(hdev)) {
err = hci_setup_ext_adv_instance_sync(hdev, 0x00);
if (!err)
hci_update_scan_rsp_data_sync(hdev, 0x00);
} else {
err = hci_update_adv_data_sync(hdev, 0x00);
if (!err)
hci_update_scan_rsp_data_sync(hdev, 0x00);
}
if (hci_dev_test_flag(hdev, HCI_ADVERTISING))
hci_enable_advertising_sync(hdev);
}
/* Call for each tracked instance to be scheduled */
list_for_each_entry_safe(adv, tmp, &hdev->adv_instances, list)
hci_schedule_adv_instance_sync(hdev, adv->instance, true);
return 0;
}
static int hci_write_auth_enable_sync(struct hci_dev *hdev)
{
u8 link_sec;
link_sec = hci_dev_test_flag(hdev, HCI_LINK_SECURITY);
if (link_sec == test_bit(HCI_AUTH, &hdev->flags))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_AUTH_ENABLE,
sizeof(link_sec), &link_sec,
HCI_CMD_TIMEOUT);
}
int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable)
{
struct hci_cp_write_page_scan_activity cp;
u8 type;
int err = 0;
if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
return 0;
if (hdev->hci_ver < BLUETOOTH_VER_1_2)
return 0;
memset(&cp, 0, sizeof(cp));
if (enable) {
type = PAGE_SCAN_TYPE_INTERLACED;
/* 160 msec page scan interval */
cp.interval = cpu_to_le16(0x0100);
} else {
type = hdev->def_page_scan_type;
cp.interval = cpu_to_le16(hdev->def_page_scan_int);
}
cp.window = cpu_to_le16(hdev->def_page_scan_window);
if (__cpu_to_le16(hdev->page_scan_interval) != cp.interval ||
__cpu_to_le16(hdev->page_scan_window) != cp.window) {
err = __hci_cmd_sync_status(hdev,
HCI_OP_WRITE_PAGE_SCAN_ACTIVITY,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (err)
return err;
}
if (hdev->page_scan_type != type)
err = __hci_cmd_sync_status(hdev,
HCI_OP_WRITE_PAGE_SCAN_TYPE,
sizeof(type), &type,
HCI_CMD_TIMEOUT);
return err;
}
static bool disconnected_accept_list_entries(struct hci_dev *hdev)
{
struct bdaddr_list *b;
list_for_each_entry(b, &hdev->accept_list, list) {
struct hci_conn *conn;
conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &b->bdaddr);
if (!conn)
return true;
if (conn->state != BT_CONNECTED && conn->state != BT_CONFIG)
return true;
}
return false;
}
static int hci_write_scan_enable_sync(struct hci_dev *hdev, u8 val)
{
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SCAN_ENABLE,
sizeof(val), &val,
HCI_CMD_TIMEOUT);
}
int hci_update_scan_sync(struct hci_dev *hdev)
{
u8 scan;
if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
return 0;
if (!hdev_is_powered(hdev))
return 0;
if (mgmt_powering_down(hdev))
return 0;
if (hdev->scanning_paused)
return 0;
if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) ||
disconnected_accept_list_entries(hdev))
scan = SCAN_PAGE;
else
scan = SCAN_DISABLED;
if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE))
scan |= SCAN_INQUIRY;
if (test_bit(HCI_PSCAN, &hdev->flags) == !!(scan & SCAN_PAGE) &&
test_bit(HCI_ISCAN, &hdev->flags) == !!(scan & SCAN_INQUIRY))
return 0;
return hci_write_scan_enable_sync(hdev, scan);
}
int hci_update_name_sync(struct hci_dev *hdev)
{
struct hci_cp_write_local_name cp;
memset(&cp, 0, sizeof(cp));
memcpy(cp.name, hdev->dev_name, sizeof(cp.name));
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LOCAL_NAME,
sizeof(cp), &cp,
HCI_CMD_TIMEOUT);
}
/* This function perform powered update HCI command sequence after the HCI init
* sequence which end up resetting all states, the sequence is as follows:
*
* HCI_SSP_ENABLED(Enable SSP)
* HCI_LE_ENABLED(Enable LE)
* HCI_LE_ENABLED(use_ll_privacy(Add local IRK to Resolving List) ->
* Update adv data)
* Enable Authentication
* lmp_bredr_capable(Set Fast Connectable -> Set Scan Type -> Set Class ->
* Set Name -> Set EIR)
* HCI_FORCE_STATIC_ADDR | BDADDR_ANY && !HCI_BREDR_ENABLED (Set Static Address)
*/
int hci_powered_update_sync(struct hci_dev *hdev)
{
int err;
/* Register the available SMP channels (BR/EDR and LE) only when
* successfully powering on the controller. This late
* registration is required so that LE SMP can clearly decide if
* the public address or static address is used.
*/
smp_register(hdev);
err = hci_write_ssp_mode_sync(hdev, 0x01);
if (err)
return err;
err = hci_write_le_host_supported_sync(hdev, 0x01, 0x00);
if (err)
return err;
err = hci_powered_update_adv_sync(hdev);
if (err)
return err;
err = hci_write_auth_enable_sync(hdev);
if (err)
return err;
if (lmp_bredr_capable(hdev)) {
if (hci_dev_test_flag(hdev, HCI_FAST_CONNECTABLE))
hci_write_fast_connectable_sync(hdev, true);
else
hci_write_fast_connectable_sync(hdev, false);
hci_update_scan_sync(hdev);
hci_update_class_sync(hdev);
hci_update_name_sync(hdev);
hci_update_eir_sync(hdev);
}
/* If forcing static address is in use or there is no public
* address use the static address as random address (but skip
* the HCI command if the current random address is already the
* static one.
*
* In case BR/EDR has been disabled on a dual-mode controller
* and a static address has been configured, then use that
* address instead of the public BR/EDR address.
*/
if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) ||
(!bacmp(&hdev->bdaddr, BDADDR_ANY) &&
!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))) {
if (bacmp(&hdev->static_addr, BDADDR_ANY))
return hci_set_random_addr_sync(hdev,
&hdev->static_addr);
}
return 0;
}
/**
* hci_dev_get_bd_addr_from_property - Get the Bluetooth Device Address
* (BD_ADDR) for a HCI device from
* a firmware node property.
* @hdev: The HCI device
*
* Search the firmware node for 'local-bd-address'.
*
* All-zero BD addresses are rejected, because those could be properties
* that exist in the firmware tables, but were not updated by the firmware. For
* example, the DTS could define 'local-bd-address', with zero BD addresses.
*/
static void hci_dev_get_bd_addr_from_property(struct hci_dev *hdev)
{
struct fwnode_handle *fwnode = dev_fwnode(hdev->dev.parent);
bdaddr_t ba;
int ret;
ret = fwnode_property_read_u8_array(fwnode, "local-bd-address",
(u8 *)&ba, sizeof(ba));
if (ret < 0 || !bacmp(&ba, BDADDR_ANY))
return;
if (test_bit(HCI_QUIRK_BDADDR_PROPERTY_BROKEN, &hdev->quirks))
baswap(&hdev->public_addr, &ba);
else
bacpy(&hdev->public_addr, &ba);
}
struct hci_init_stage {
int (*func)(struct hci_dev *hdev);
};
/* Run init stage NULL terminated function table */
static int hci_init_stage_sync(struct hci_dev *hdev,
const struct hci_init_stage *stage)
{
size_t i;
for (i = 0; stage[i].func; i++) {
int err;
err = stage[i].func(hdev);
if (err)
return err;
}
return 0;
}
/* Read Local Version */
static int hci_read_local_version_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_VERSION,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read BD Address */
static int hci_read_bd_addr_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_BD_ADDR,
0, NULL, HCI_CMD_TIMEOUT);
}
#define HCI_INIT(_func) \
{ \
.func = _func, \
}
static const struct hci_init_stage hci_init0[] = {
/* HCI_OP_READ_LOCAL_VERSION */
HCI_INIT(hci_read_local_version_sync),
/* HCI_OP_READ_BD_ADDR */
HCI_INIT(hci_read_bd_addr_sync),
{}
};
int hci_reset_sync(struct hci_dev *hdev)
{
int err;
set_bit(HCI_RESET, &hdev->flags);
err = __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL,
HCI_CMD_TIMEOUT);
if (err)
return err;
return 0;
}
static int hci_init0_sync(struct hci_dev *hdev)
{
int err;
bt_dev_dbg(hdev, "");
/* Reset */
if (!test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks)) {
err = hci_reset_sync(hdev);
if (err)
return err;
}
return hci_init_stage_sync(hdev, hci_init0);
}
static int hci_unconf_init_sync(struct hci_dev *hdev)
{
int err;
if (test_bit(HCI_QUIRK_RAW_DEVICE, &hdev->quirks))
return 0;
err = hci_init0_sync(hdev);
if (err < 0)
return err;
if (hci_dev_test_flag(hdev, HCI_SETUP))
hci_debugfs_create_basic(hdev);
return 0;
}
/* Read Local Supported Features. */
static int hci_read_local_features_sync(struct hci_dev *hdev)
{
/* Not all AMP controllers support this command */
if (hdev->dev_type == HCI_AMP && !(hdev->commands[14] & 0x20))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_FEATURES,
0, NULL, HCI_CMD_TIMEOUT);
}
/* BR Controller init stage 1 command sequence */
static const struct hci_init_stage br_init1[] = {
/* HCI_OP_READ_LOCAL_FEATURES */
HCI_INIT(hci_read_local_features_sync),
/* HCI_OP_READ_LOCAL_VERSION */
HCI_INIT(hci_read_local_version_sync),
/* HCI_OP_READ_BD_ADDR */
HCI_INIT(hci_read_bd_addr_sync),
{}
};
/* Read Local Commands */
static int hci_read_local_cmds_sync(struct hci_dev *hdev)
{
/* All Bluetooth 1.2 and later controllers should support the
* HCI command for reading the local supported commands.
*
* Unfortunately some controllers indicate Bluetooth 1.2 support,
* but do not have support for this command. If that is the case,
* the driver can quirk the behavior and skip reading the local
* supported commands.
*/
if (hdev->hci_ver > BLUETOOTH_VER_1_1 &&
!test_bit(HCI_QUIRK_BROKEN_LOCAL_COMMANDS, &hdev->quirks))
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_COMMANDS,
0, NULL, HCI_CMD_TIMEOUT);
return 0;
}
/* Read Local AMP Info */
static int hci_read_local_amp_info_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_AMP_INFO,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Data Blk size */
static int hci_read_data_block_size_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_DATA_BLOCK_SIZE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Flow Control Mode */
static int hci_read_flow_control_mode_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_FLOW_CONTROL_MODE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Location Data */
static int hci_read_location_data_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCATION_DATA,
0, NULL, HCI_CMD_TIMEOUT);
}
/* AMP Controller init stage 1 command sequence */
static const struct hci_init_stage amp_init1[] = {
/* HCI_OP_READ_LOCAL_VERSION */
HCI_INIT(hci_read_local_version_sync),
/* HCI_OP_READ_LOCAL_COMMANDS */
HCI_INIT(hci_read_local_cmds_sync),
/* HCI_OP_READ_LOCAL_AMP_INFO */
HCI_INIT(hci_read_local_amp_info_sync),
/* HCI_OP_READ_DATA_BLOCK_SIZE */
HCI_INIT(hci_read_data_block_size_sync),
/* HCI_OP_READ_FLOW_CONTROL_MODE */
HCI_INIT(hci_read_flow_control_mode_sync),
/* HCI_OP_READ_LOCATION_DATA */
HCI_INIT(hci_read_location_data_sync),
{}
};
static int hci_init1_sync(struct hci_dev *hdev)
{
int err;
bt_dev_dbg(hdev, "");
/* Reset */
if (!test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks)) {
err = hci_reset_sync(hdev);
if (err)
return err;
}
switch (hdev->dev_type) {
case HCI_PRIMARY:
hdev->flow_ctl_mode = HCI_FLOW_CTL_MODE_PACKET_BASED;
return hci_init_stage_sync(hdev, br_init1);
case HCI_AMP:
hdev->flow_ctl_mode = HCI_FLOW_CTL_MODE_BLOCK_BASED;
return hci_init_stage_sync(hdev, amp_init1);
default:
bt_dev_err(hdev, "Unknown device type %d", hdev->dev_type);
break;
}
return 0;
}
/* AMP Controller init stage 2 command sequence */
static const struct hci_init_stage amp_init2[] = {
/* HCI_OP_READ_LOCAL_FEATURES */
HCI_INIT(hci_read_local_features_sync),
{}
};
/* Read Buffer Size (ACL mtu, max pkt, etc.) */
static int hci_read_buffer_size_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_BUFFER_SIZE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Class of Device */
static int hci_read_dev_class_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_CLASS_OF_DEV,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Local Name */
static int hci_read_local_name_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_NAME,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Voice Setting */
static int hci_read_voice_setting_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_VOICE_SETTING,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Number of Supported IAC */
static int hci_read_num_supported_iac_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_NUM_SUPPORTED_IAC,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read Current IAC LAP */
static int hci_read_current_iac_lap_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_READ_CURRENT_IAC_LAP,
0, NULL, HCI_CMD_TIMEOUT);
}
static int hci_set_event_filter_sync(struct hci_dev *hdev, u8 flt_type,
u8 cond_type, bdaddr_t *bdaddr,
u8 auto_accept)
{
struct hci_cp_set_event_filter cp;
if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
return 0;
if (test_bit(HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL, &hdev->quirks))
return 0;
memset(&cp, 0, sizeof(cp));
cp.flt_type = flt_type;
if (flt_type != HCI_FLT_CLEAR_ALL) {
cp.cond_type = cond_type;
bacpy(&cp.addr_conn_flt.bdaddr, bdaddr);
cp.addr_conn_flt.auto_accept = auto_accept;
}
return __hci_cmd_sync_status(hdev, HCI_OP_SET_EVENT_FLT,
flt_type == HCI_FLT_CLEAR_ALL ?
sizeof(cp.flt_type) : sizeof(cp), &cp,
HCI_CMD_TIMEOUT);
}
static int hci_clear_event_filter_sync(struct hci_dev *hdev)
{
if (!hci_dev_test_flag(hdev, HCI_EVENT_FILTER_CONFIGURED))
return 0;
/* In theory the state machine should not reach here unless
* a hci_set_event_filter_sync() call succeeds, but we do
* the check both for parity and as a future reminder.
*/
if (test_bit(HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL, &hdev->quirks))
return 0;
return hci_set_event_filter_sync(hdev, HCI_FLT_CLEAR_ALL, 0x00,
BDADDR_ANY, 0x00);
}
/* Connection accept timeout ~20 secs */
static int hci_write_ca_timeout_sync(struct hci_dev *hdev)
{
__le16 param = cpu_to_le16(0x7d00);
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CA_TIMEOUT,
sizeof(param), ¶m, HCI_CMD_TIMEOUT);
}
/* BR Controller init stage 2 command sequence */
static const struct hci_init_stage br_init2[] = {
/* HCI_OP_READ_BUFFER_SIZE */
HCI_INIT(hci_read_buffer_size_sync),
/* HCI_OP_READ_CLASS_OF_DEV */
HCI_INIT(hci_read_dev_class_sync),
/* HCI_OP_READ_LOCAL_NAME */
HCI_INIT(hci_read_local_name_sync),
/* HCI_OP_READ_VOICE_SETTING */
HCI_INIT(hci_read_voice_setting_sync),
/* HCI_OP_READ_NUM_SUPPORTED_IAC */
HCI_INIT(hci_read_num_supported_iac_sync),
/* HCI_OP_READ_CURRENT_IAC_LAP */
HCI_INIT(hci_read_current_iac_lap_sync),
/* HCI_OP_SET_EVENT_FLT */
HCI_INIT(hci_clear_event_filter_sync),
/* HCI_OP_WRITE_CA_TIMEOUT */
HCI_INIT(hci_write_ca_timeout_sync),
{}
};
static int hci_write_ssp_mode_1_sync(struct hci_dev *hdev)
{
u8 mode = 0x01;
if (!lmp_ssp_capable(hdev) || !hci_dev_test_flag(hdev, HCI_SSP_ENABLED))
return 0;
/* When SSP is available, then the host features page
* should also be available as well. However some
* controllers list the max_page as 0 as long as SSP
* has not been enabled. To achieve proper debugging
* output, force the minimum max_page to 1 at least.
*/
hdev->max_page = 0x01;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SSP_MODE,
sizeof(mode), &mode, HCI_CMD_TIMEOUT);
}
static int hci_write_eir_sync(struct hci_dev *hdev)
{
struct hci_cp_write_eir cp;
if (!lmp_ssp_capable(hdev) || hci_dev_test_flag(hdev, HCI_SSP_ENABLED))
return 0;
memset(hdev->eir, 0, sizeof(hdev->eir));
memset(&cp, 0, sizeof(cp));
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_EIR, sizeof(cp), &cp,
HCI_CMD_TIMEOUT);
}
static int hci_write_inquiry_mode_sync(struct hci_dev *hdev)
{
u8 mode;
if (!lmp_inq_rssi_capable(hdev) &&
!test_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks))
return 0;
/* If Extended Inquiry Result events are supported, then
* they are clearly preferred over Inquiry Result with RSSI
* events.
*/
mode = lmp_ext_inq_capable(hdev) ? 0x02 : 0x01;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_INQUIRY_MODE,
sizeof(mode), &mode, HCI_CMD_TIMEOUT);
}
static int hci_read_inq_rsp_tx_power_sync(struct hci_dev *hdev)
{
if (!lmp_inq_tx_pwr_capable(hdev))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_INQ_RSP_TX_POWER,
0, NULL, HCI_CMD_TIMEOUT);
}
static int hci_read_local_ext_features_sync(struct hci_dev *hdev, u8 page)
{
struct hci_cp_read_local_ext_features cp;
if (!lmp_ext_feat_capable(hdev))
return 0;
memset(&cp, 0, sizeof(cp));
cp.page = page;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_EXT_FEATURES,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_read_local_ext_features_1_sync(struct hci_dev *hdev)
{
return hci_read_local_ext_features_sync(hdev, 0x01);
}
/* HCI Controller init stage 2 command sequence */
static const struct hci_init_stage hci_init2[] = {
/* HCI_OP_READ_LOCAL_COMMANDS */
HCI_INIT(hci_read_local_cmds_sync),
/* HCI_OP_WRITE_SSP_MODE */
HCI_INIT(hci_write_ssp_mode_1_sync),
/* HCI_OP_WRITE_EIR */
HCI_INIT(hci_write_eir_sync),
/* HCI_OP_WRITE_INQUIRY_MODE */
HCI_INIT(hci_write_inquiry_mode_sync),
/* HCI_OP_READ_INQ_RSP_TX_POWER */
HCI_INIT(hci_read_inq_rsp_tx_power_sync),
/* HCI_OP_READ_LOCAL_EXT_FEATURES */
HCI_INIT(hci_read_local_ext_features_1_sync),
/* HCI_OP_WRITE_AUTH_ENABLE */
HCI_INIT(hci_write_auth_enable_sync),
{}
};
/* Read LE Buffer Size */
static int hci_le_read_buffer_size_sync(struct hci_dev *hdev)
{
/* Use Read LE Buffer Size V2 if supported */
if (iso_capable(hdev) && hdev->commands[41] & 0x20)
return __hci_cmd_sync_status(hdev,
HCI_OP_LE_READ_BUFFER_SIZE_V2,
0, NULL, HCI_CMD_TIMEOUT);
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_BUFFER_SIZE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read LE Local Supported Features */
static int hci_le_read_local_features_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_LOCAL_FEATURES,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read LE Supported States */
static int hci_le_read_supported_states_sync(struct hci_dev *hdev)
{
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_SUPPORTED_STATES,
0, NULL, HCI_CMD_TIMEOUT);
}
/* LE Controller init stage 2 command sequence */
static const struct hci_init_stage le_init2[] = {
/* HCI_OP_LE_READ_LOCAL_FEATURES */
HCI_INIT(hci_le_read_local_features_sync),
/* HCI_OP_LE_READ_BUFFER_SIZE */
HCI_INIT(hci_le_read_buffer_size_sync),
/* HCI_OP_LE_READ_SUPPORTED_STATES */
HCI_INIT(hci_le_read_supported_states_sync),
{}
};
static int hci_init2_sync(struct hci_dev *hdev)
{
int err;
bt_dev_dbg(hdev, "");
if (hdev->dev_type == HCI_AMP)
return hci_init_stage_sync(hdev, amp_init2);
err = hci_init_stage_sync(hdev, hci_init2);
if (err)
return err;
if (lmp_bredr_capable(hdev)) {
err = hci_init_stage_sync(hdev, br_init2);
if (err)
return err;
} else {
hci_dev_clear_flag(hdev, HCI_BREDR_ENABLED);
}
if (lmp_le_capable(hdev)) {
err = hci_init_stage_sync(hdev, le_init2);
if (err)
return err;
/* LE-only controllers have LE implicitly enabled */
if (!lmp_bredr_capable(hdev))
hci_dev_set_flag(hdev, HCI_LE_ENABLED);
}
return 0;
}
static int hci_set_event_mask_sync(struct hci_dev *hdev)
{
/* The second byte is 0xff instead of 0x9f (two reserved bits
* disabled) since a Broadcom 1.2 dongle doesn't respond to the
* command otherwise.
*/
u8 events[8] = { 0xff, 0xff, 0xfb, 0xff, 0x00, 0x00, 0x00, 0x00 };
/* CSR 1.1 dongles does not accept any bitfield so don't try to set
* any event mask for pre 1.2 devices.
*/
if (hdev->hci_ver < BLUETOOTH_VER_1_2)
return 0;
if (lmp_bredr_capable(hdev)) {
events[4] |= 0x01; /* Flow Specification Complete */
/* Don't set Disconnect Complete and mode change when
* suspended as that would wakeup the host when disconnecting
* due to suspend.
*/
if (hdev->suspended) {
events[0] &= 0xef;
events[2] &= 0xf7;
}
} else {
/* Use a different default for LE-only devices */
memset(events, 0, sizeof(events));
events[1] |= 0x20; /* Command Complete */
events[1] |= 0x40; /* Command Status */
events[1] |= 0x80; /* Hardware Error */
/* If the controller supports the Disconnect command, enable
* the corresponding event. In addition enable packet flow
* control related events.
*/
if (hdev->commands[0] & 0x20) {
/* Don't set Disconnect Complete when suspended as that
* would wakeup the host when disconnecting due to
* suspend.
*/
if (!hdev->suspended)
events[0] |= 0x10; /* Disconnection Complete */
events[2] |= 0x04; /* Number of Completed Packets */
events[3] |= 0x02; /* Data Buffer Overflow */
}
/* If the controller supports the Read Remote Version
* Information command, enable the corresponding event.
*/
if (hdev->commands[2] & 0x80)
events[1] |= 0x08; /* Read Remote Version Information
* Complete
*/
if (hdev->le_features[0] & HCI_LE_ENCRYPTION) {
events[0] |= 0x80; /* Encryption Change */
events[5] |= 0x80; /* Encryption Key Refresh Complete */
}
}
if (lmp_inq_rssi_capable(hdev) ||
test_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks))
events[4] |= 0x02; /* Inquiry Result with RSSI */
if (lmp_ext_feat_capable(hdev))
events[4] |= 0x04; /* Read Remote Extended Features Complete */
if (lmp_esco_capable(hdev)) {
events[5] |= 0x08; /* Synchronous Connection Complete */
events[5] |= 0x10; /* Synchronous Connection Changed */
}
if (lmp_sniffsubr_capable(hdev))
events[5] |= 0x20; /* Sniff Subrating */
if (lmp_pause_enc_capable(hdev))
events[5] |= 0x80; /* Encryption Key Refresh Complete */
if (lmp_ext_inq_capable(hdev))
events[5] |= 0x40; /* Extended Inquiry Result */
if (lmp_no_flush_capable(hdev))
events[7] |= 0x01; /* Enhanced Flush Complete */
if (lmp_lsto_capable(hdev))
events[6] |= 0x80; /* Link Supervision Timeout Changed */
if (lmp_ssp_capable(hdev)) {
events[6] |= 0x01; /* IO Capability Request */
events[6] |= 0x02; /* IO Capability Response */
events[6] |= 0x04; /* User Confirmation Request */
events[6] |= 0x08; /* User Passkey Request */
events[6] |= 0x10; /* Remote OOB Data Request */
events[6] |= 0x20; /* Simple Pairing Complete */
events[7] |= 0x04; /* User Passkey Notification */
events[7] |= 0x08; /* Keypress Notification */
events[7] |= 0x10; /* Remote Host Supported
* Features Notification
*/
}
if (lmp_le_capable(hdev))
events[7] |= 0x20; /* LE Meta-Event */
return __hci_cmd_sync_status(hdev, HCI_OP_SET_EVENT_MASK,
sizeof(events), events, HCI_CMD_TIMEOUT);
}
static int hci_read_stored_link_key_sync(struct hci_dev *hdev)
{
struct hci_cp_read_stored_link_key cp;
if (!(hdev->commands[6] & 0x20) ||
test_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks))
return 0;
memset(&cp, 0, sizeof(cp));
bacpy(&cp.bdaddr, BDADDR_ANY);
cp.read_all = 0x01;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_STORED_LINK_KEY,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_setup_link_policy_sync(struct hci_dev *hdev)
{
struct hci_cp_write_def_link_policy cp;
u16 link_policy = 0;
if (!(hdev->commands[5] & 0x10))
return 0;
memset(&cp, 0, sizeof(cp));
if (lmp_rswitch_capable(hdev))
link_policy |= HCI_LP_RSWITCH;
if (lmp_hold_capable(hdev))
link_policy |= HCI_LP_HOLD;
if (lmp_sniff_capable(hdev))
link_policy |= HCI_LP_SNIFF;
if (lmp_park_capable(hdev))
link_policy |= HCI_LP_PARK;
cp.policy = cpu_to_le16(link_policy);
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_DEF_LINK_POLICY,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_read_page_scan_activity_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[8] & 0x01))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_PAGE_SCAN_ACTIVITY,
0, NULL, HCI_CMD_TIMEOUT);
}
static int hci_read_def_err_data_reporting_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[18] & 0x04) ||
!(hdev->features[0][6] & LMP_ERR_DATA_REPORTING) ||
test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_DEF_ERR_DATA_REPORTING,
0, NULL, HCI_CMD_TIMEOUT);
}
static int hci_read_page_scan_type_sync(struct hci_dev *hdev)
{
/* Some older Broadcom based Bluetooth 1.2 controllers do not
* support the Read Page Scan Type command. Check support for
* this command in the bit mask of supported commands.
*/
if (!(hdev->commands[13] & 0x01))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_PAGE_SCAN_TYPE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read features beyond page 1 if available */
static int hci_read_local_ext_features_all_sync(struct hci_dev *hdev)
{
u8 page;
int err;
if (!lmp_ext_feat_capable(hdev))
return 0;
for (page = 2; page < HCI_MAX_PAGES && page <= hdev->max_page;
page++) {
err = hci_read_local_ext_features_sync(hdev, page);
if (err)
return err;
}
return 0;
}
/* HCI Controller init stage 3 command sequence */
static const struct hci_init_stage hci_init3[] = {
/* HCI_OP_SET_EVENT_MASK */
HCI_INIT(hci_set_event_mask_sync),
/* HCI_OP_READ_STORED_LINK_KEY */
HCI_INIT(hci_read_stored_link_key_sync),
/* HCI_OP_WRITE_DEF_LINK_POLICY */
HCI_INIT(hci_setup_link_policy_sync),
/* HCI_OP_READ_PAGE_SCAN_ACTIVITY */
HCI_INIT(hci_read_page_scan_activity_sync),
/* HCI_OP_READ_DEF_ERR_DATA_REPORTING */
HCI_INIT(hci_read_def_err_data_reporting_sync),
/* HCI_OP_READ_PAGE_SCAN_TYPE */
HCI_INIT(hci_read_page_scan_type_sync),
/* HCI_OP_READ_LOCAL_EXT_FEATURES */
HCI_INIT(hci_read_local_ext_features_all_sync),
{}
};
static int hci_le_set_event_mask_sync(struct hci_dev *hdev)
{
u8 events[8];
if (!lmp_le_capable(hdev))
return 0;
memset(events, 0, sizeof(events));
if (hdev->le_features[0] & HCI_LE_ENCRYPTION)
events[0] |= 0x10; /* LE Long Term Key Request */
/* If controller supports the Connection Parameters Request
* Link Layer Procedure, enable the corresponding event.
*/
if (hdev->le_features[0] & HCI_LE_CONN_PARAM_REQ_PROC)
/* LE Remote Connection Parameter Request */
events[0] |= 0x20;
/* If the controller supports the Data Length Extension
* feature, enable the corresponding event.
*/
if (hdev->le_features[0] & HCI_LE_DATA_LEN_EXT)
events[0] |= 0x40; /* LE Data Length Change */
/* If the controller supports LL Privacy feature or LE Extended Adv,
* enable the corresponding event.
*/
if (use_enhanced_conn_complete(hdev))
events[1] |= 0x02; /* LE Enhanced Connection Complete */
/* If the controller supports Extended Scanner Filter
* Policies, enable the corresponding event.
*/
if (hdev->le_features[0] & HCI_LE_EXT_SCAN_POLICY)
events[1] |= 0x04; /* LE Direct Advertising Report */
/* If the controller supports Channel Selection Algorithm #2
* feature, enable the corresponding event.
*/
if (hdev->le_features[1] & HCI_LE_CHAN_SEL_ALG2)
events[2] |= 0x08; /* LE Channel Selection Algorithm */
/* If the controller supports the LE Set Scan Enable command,
* enable the corresponding advertising report event.
*/
if (hdev->commands[26] & 0x08)
events[0] |= 0x02; /* LE Advertising Report */
/* If the controller supports the LE Create Connection
* command, enable the corresponding event.
*/
if (hdev->commands[26] & 0x10)
events[0] |= 0x01; /* LE Connection Complete */
/* If the controller supports the LE Connection Update
* command, enable the corresponding event.
*/
if (hdev->commands[27] & 0x04)
events[0] |= 0x04; /* LE Connection Update Complete */
/* If the controller supports the LE Read Remote Used Features
* command, enable the corresponding event.
*/
if (hdev->commands[27] & 0x20)
/* LE Read Remote Used Features Complete */
events[0] |= 0x08;
/* If the controller supports the LE Read Local P-256
* Public Key command, enable the corresponding event.
*/
if (hdev->commands[34] & 0x02)
/* LE Read Local P-256 Public Key Complete */
events[0] |= 0x80;
/* If the controller supports the LE Generate DHKey
* command, enable the corresponding event.
*/
if (hdev->commands[34] & 0x04)
events[1] |= 0x01; /* LE Generate DHKey Complete */
/* If the controller supports the LE Set Default PHY or
* LE Set PHY commands, enable the corresponding event.
*/
if (hdev->commands[35] & (0x20 | 0x40))
events[1] |= 0x08; /* LE PHY Update Complete */
/* If the controller supports LE Set Extended Scan Parameters
* and LE Set Extended Scan Enable commands, enable the
* corresponding event.
*/
if (use_ext_scan(hdev))
events[1] |= 0x10; /* LE Extended Advertising Report */
/* If the controller supports the LE Extended Advertising
* command, enable the corresponding event.
*/
if (ext_adv_capable(hdev))
events[2] |= 0x02; /* LE Advertising Set Terminated */
if (cis_capable(hdev)) {
events[3] |= 0x01; /* LE CIS Established */
if (cis_peripheral_capable(hdev))
events[3] |= 0x02; /* LE CIS Request */
}
if (bis_capable(hdev)) {
events[1] |= 0x20; /* LE PA Report */
events[1] |= 0x40; /* LE PA Sync Established */
events[3] |= 0x04; /* LE Create BIG Complete */
events[3] |= 0x08; /* LE Terminate BIG Complete */
events[3] |= 0x10; /* LE BIG Sync Established */
events[3] |= 0x20; /* LE BIG Sync Loss */
events[4] |= 0x02; /* LE BIG Info Advertising Report */
}
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EVENT_MASK,
sizeof(events), events, HCI_CMD_TIMEOUT);
}
/* Read LE Advertising Channel TX Power */
static int hci_le_read_adv_tx_power_sync(struct hci_dev *hdev)
{
if ((hdev->commands[25] & 0x40) && !ext_adv_capable(hdev)) {
/* HCI TS spec forbids mixing of legacy and extended
* advertising commands wherein READ_ADV_TX_POWER is
* also included. So do not call it if extended adv
* is supported otherwise controller will return
* COMMAND_DISALLOWED for extended commands.
*/
return __hci_cmd_sync_status(hdev,
HCI_OP_LE_READ_ADV_TX_POWER,
0, NULL, HCI_CMD_TIMEOUT);
}
return 0;
}
/* Read LE Min/Max Tx Power*/
static int hci_le_read_tx_power_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[38] & 0x80) ||
test_bit(HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER, &hdev->quirks))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_TRANSMIT_POWER,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read LE Accept List Size */
static int hci_le_read_accept_list_size_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[26] & 0x40))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_ACCEPT_LIST_SIZE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Read LE Resolving List Size */
static int hci_le_read_resolv_list_size_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[34] & 0x40))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_RESOLV_LIST_SIZE,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Clear LE Resolving List */
static int hci_le_clear_resolv_list_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[34] & 0x20))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_CLEAR_RESOLV_LIST, 0, NULL,
HCI_CMD_TIMEOUT);
}
/* Set RPA timeout */
static int hci_le_set_rpa_timeout_sync(struct hci_dev *hdev)
{
__le16 timeout = cpu_to_le16(hdev->rpa_timeout);
if (!(hdev->commands[35] & 0x04) ||
test_bit(HCI_QUIRK_BROKEN_SET_RPA_TIMEOUT, &hdev->quirks))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RPA_TIMEOUT,
sizeof(timeout), &timeout,
HCI_CMD_TIMEOUT);
}
/* Read LE Maximum Data Length */
static int hci_le_read_max_data_len_sync(struct hci_dev *hdev)
{
if (!(hdev->le_features[0] & HCI_LE_DATA_LEN_EXT))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_MAX_DATA_LEN, 0, NULL,
HCI_CMD_TIMEOUT);
}
/* Read LE Suggested Default Data Length */
static int hci_le_read_def_data_len_sync(struct hci_dev *hdev)
{
if (!(hdev->le_features[0] & HCI_LE_DATA_LEN_EXT))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_DEF_DATA_LEN, 0, NULL,
HCI_CMD_TIMEOUT);
}
/* Read LE Number of Supported Advertising Sets */
static int hci_le_read_num_support_adv_sets_sync(struct hci_dev *hdev)
{
if (!ext_adv_capable(hdev))
return 0;
return __hci_cmd_sync_status(hdev,
HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Write LE Host Supported */
static int hci_set_le_support_sync(struct hci_dev *hdev)
{
struct hci_cp_write_le_host_supported cp;
/* LE-only devices do not support explicit enablement */
if (!lmp_bredr_capable(hdev))
return 0;
memset(&cp, 0, sizeof(cp));
if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) {
cp.le = 0x01;
cp.simul = 0x00;
}
if (cp.le == lmp_host_le_capable(hdev))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LE_HOST_SUPPORTED,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
/* LE Set Host Feature */
static int hci_le_set_host_feature_sync(struct hci_dev *hdev)
{
struct hci_cp_le_set_host_feature cp;
if (!cis_capable(hdev))
return 0;
memset(&cp, 0, sizeof(cp));
/* Connected Isochronous Channels (Host Support) */
cp.bit_number = 32;
cp.bit_value = 1;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_HOST_FEATURE,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
/* LE Controller init stage 3 command sequence */
static const struct hci_init_stage le_init3[] = {
/* HCI_OP_LE_SET_EVENT_MASK */
HCI_INIT(hci_le_set_event_mask_sync),
/* HCI_OP_LE_READ_ADV_TX_POWER */
HCI_INIT(hci_le_read_adv_tx_power_sync),
/* HCI_OP_LE_READ_TRANSMIT_POWER */
HCI_INIT(hci_le_read_tx_power_sync),
/* HCI_OP_LE_READ_ACCEPT_LIST_SIZE */
HCI_INIT(hci_le_read_accept_list_size_sync),
/* HCI_OP_LE_CLEAR_ACCEPT_LIST */
HCI_INIT(hci_le_clear_accept_list_sync),
/* HCI_OP_LE_READ_RESOLV_LIST_SIZE */
HCI_INIT(hci_le_read_resolv_list_size_sync),
/* HCI_OP_LE_CLEAR_RESOLV_LIST */
HCI_INIT(hci_le_clear_resolv_list_sync),
/* HCI_OP_LE_SET_RPA_TIMEOUT */
HCI_INIT(hci_le_set_rpa_timeout_sync),
/* HCI_OP_LE_READ_MAX_DATA_LEN */
HCI_INIT(hci_le_read_max_data_len_sync),
/* HCI_OP_LE_READ_DEF_DATA_LEN */
HCI_INIT(hci_le_read_def_data_len_sync),
/* HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS */
HCI_INIT(hci_le_read_num_support_adv_sets_sync),
/* HCI_OP_WRITE_LE_HOST_SUPPORTED */
HCI_INIT(hci_set_le_support_sync),
/* HCI_OP_LE_SET_HOST_FEATURE */
HCI_INIT(hci_le_set_host_feature_sync),
{}
};
static int hci_init3_sync(struct hci_dev *hdev)
{
int err;
bt_dev_dbg(hdev, "");
err = hci_init_stage_sync(hdev, hci_init3);
if (err)
return err;
if (lmp_le_capable(hdev))
return hci_init_stage_sync(hdev, le_init3);
return 0;
}
static int hci_delete_stored_link_key_sync(struct hci_dev *hdev)
{
struct hci_cp_delete_stored_link_key cp;
/* Some Broadcom based Bluetooth controllers do not support the
* Delete Stored Link Key command. They are clearly indicating its
* absence in the bit mask of supported commands.
*
* Check the supported commands and only if the command is marked
* as supported send it. If not supported assume that the controller
* does not have actual support for stored link keys which makes this
* command redundant anyway.
*
* Some controllers indicate that they support handling deleting
* stored link keys, but they don't. The quirk lets a driver
* just disable this command.
*/
if (!(hdev->commands[6] & 0x80) ||
test_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks))
return 0;
memset(&cp, 0, sizeof(cp));
bacpy(&cp.bdaddr, BDADDR_ANY);
cp.delete_all = 0x01;
return __hci_cmd_sync_status(hdev, HCI_OP_DELETE_STORED_LINK_KEY,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_set_event_mask_page_2_sync(struct hci_dev *hdev)
{
u8 events[8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
bool changed = false;
/* Set event mask page 2 if the HCI command for it is supported */
if (!(hdev->commands[22] & 0x04))
return 0;
/* If Connectionless Peripheral Broadcast central role is supported
* enable all necessary events for it.
*/
if (lmp_cpb_central_capable(hdev)) {
events[1] |= 0x40; /* Triggered Clock Capture */
events[1] |= 0x80; /* Synchronization Train Complete */
events[2] |= 0x08; /* Truncated Page Complete */
events[2] |= 0x20; /* CPB Channel Map Change */
changed = true;
}
/* If Connectionless Peripheral Broadcast peripheral role is supported
* enable all necessary events for it.
*/
if (lmp_cpb_peripheral_capable(hdev)) {
events[2] |= 0x01; /* Synchronization Train Received */
events[2] |= 0x02; /* CPB Receive */
events[2] |= 0x04; /* CPB Timeout */
events[2] |= 0x10; /* Peripheral Page Response Timeout */
changed = true;
}
/* Enable Authenticated Payload Timeout Expired event if supported */
if (lmp_ping_capable(hdev) || hdev->le_features[0] & HCI_LE_PING) {
events[2] |= 0x80;
changed = true;
}
/* Some Broadcom based controllers indicate support for Set Event
* Mask Page 2 command, but then actually do not support it. Since
* the default value is all bits set to zero, the command is only
* required if the event mask has to be changed. In case no change
* to the event mask is needed, skip this command.
*/
if (!changed)
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_SET_EVENT_MASK_PAGE_2,
sizeof(events), events, HCI_CMD_TIMEOUT);
}
/* Read local codec list if the HCI command is supported */
static int hci_read_local_codecs_sync(struct hci_dev *hdev)
{
if (hdev->commands[45] & 0x04)
hci_read_supported_codecs_v2(hdev);
else if (hdev->commands[29] & 0x20)
hci_read_supported_codecs(hdev);
return 0;
}
/* Read local pairing options if the HCI command is supported */
static int hci_read_local_pairing_opts_sync(struct hci_dev *hdev)
{
if (!(hdev->commands[41] & 0x08))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_PAIRING_OPTS,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Get MWS transport configuration if the HCI command is supported */
static int hci_get_mws_transport_config_sync(struct hci_dev *hdev)
{
if (!mws_transport_config_capable(hdev))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_GET_MWS_TRANSPORT_CONFIG,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Check for Synchronization Train support */
static int hci_read_sync_train_params_sync(struct hci_dev *hdev)
{
if (!lmp_sync_train_capable(hdev))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_READ_SYNC_TRAIN_PARAMS,
0, NULL, HCI_CMD_TIMEOUT);
}
/* Enable Secure Connections if supported and configured */
static int hci_write_sc_support_1_sync(struct hci_dev *hdev)
{
u8 support = 0x01;
if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED) ||
!bredr_sc_enabled(hdev))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SC_SUPPORT,
sizeof(support), &support,
HCI_CMD_TIMEOUT);
}
/* Set erroneous data reporting if supported to the wideband speech
* setting value
*/
static int hci_set_err_data_report_sync(struct hci_dev *hdev)
{
struct hci_cp_write_def_err_data_reporting cp;
bool enabled = hci_dev_test_flag(hdev, HCI_WIDEBAND_SPEECH_ENABLED);
if (!(hdev->commands[18] & 0x08) ||
!(hdev->features[0][6] & LMP_ERR_DATA_REPORTING) ||
test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks))
return 0;
if (enabled == hdev->err_data_reporting)
return 0;
memset(&cp, 0, sizeof(cp));
cp.err_data_reporting = enabled ? ERR_DATA_REPORTING_ENABLED :
ERR_DATA_REPORTING_DISABLED;
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_DEF_ERR_DATA_REPORTING,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static const struct hci_init_stage hci_init4[] = {
/* HCI_OP_DELETE_STORED_LINK_KEY */
HCI_INIT(hci_delete_stored_link_key_sync),
/* HCI_OP_SET_EVENT_MASK_PAGE_2 */
HCI_INIT(hci_set_event_mask_page_2_sync),
/* HCI_OP_READ_LOCAL_CODECS */
HCI_INIT(hci_read_local_codecs_sync),
/* HCI_OP_READ_LOCAL_PAIRING_OPTS */
HCI_INIT(hci_read_local_pairing_opts_sync),
/* HCI_OP_GET_MWS_TRANSPORT_CONFIG */
HCI_INIT(hci_get_mws_transport_config_sync),
/* HCI_OP_READ_SYNC_TRAIN_PARAMS */
HCI_INIT(hci_read_sync_train_params_sync),
/* HCI_OP_WRITE_SC_SUPPORT */
HCI_INIT(hci_write_sc_support_1_sync),
/* HCI_OP_WRITE_DEF_ERR_DATA_REPORTING */
HCI_INIT(hci_set_err_data_report_sync),
{}
};
/* Set Suggested Default Data Length to maximum if supported */
static int hci_le_set_write_def_data_len_sync(struct hci_dev *hdev)
{
struct hci_cp_le_write_def_data_len cp;
if (!(hdev->le_features[0] & HCI_LE_DATA_LEN_EXT))
return 0;
memset(&cp, 0, sizeof(cp));
cp.tx_len = cpu_to_le16(hdev->le_max_tx_len);
cp.tx_time = cpu_to_le16(hdev->le_max_tx_time);
return __hci_cmd_sync_status(hdev, HCI_OP_LE_WRITE_DEF_DATA_LEN,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
/* Set Default PHY parameters if command is supported, enables all supported
* PHYs according to the LE Features bits.
*/
static int hci_le_set_default_phy_sync(struct hci_dev *hdev)
{
struct hci_cp_le_set_default_phy cp;
if (!(hdev->commands[35] & 0x20)) {
/* If the command is not supported it means only 1M PHY is
* supported.
*/
hdev->le_tx_def_phys = HCI_LE_SET_PHY_1M;
hdev->le_rx_def_phys = HCI_LE_SET_PHY_1M;
return 0;
}
memset(&cp, 0, sizeof(cp));
cp.all_phys = 0x00;
cp.tx_phys = HCI_LE_SET_PHY_1M;
cp.rx_phys = HCI_LE_SET_PHY_1M;
/* Enables 2M PHY if supported */
if (le_2m_capable(hdev)) {
cp.tx_phys |= HCI_LE_SET_PHY_2M;
cp.rx_phys |= HCI_LE_SET_PHY_2M;
}
/* Enables Coded PHY if supported */
if (le_coded_capable(hdev)) {
cp.tx_phys |= HCI_LE_SET_PHY_CODED;
cp.rx_phys |= HCI_LE_SET_PHY_CODED;
}
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_DEFAULT_PHY,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static const struct hci_init_stage le_init4[] = {
/* HCI_OP_LE_WRITE_DEF_DATA_LEN */
HCI_INIT(hci_le_set_write_def_data_len_sync),
/* HCI_OP_LE_SET_DEFAULT_PHY */
HCI_INIT(hci_le_set_default_phy_sync),
{}
};
static int hci_init4_sync(struct hci_dev *hdev)
{
int err;
bt_dev_dbg(hdev, "");
err = hci_init_stage_sync(hdev, hci_init4);
if (err)
return err;
if (lmp_le_capable(hdev))
return hci_init_stage_sync(hdev, le_init4);
return 0;
}
static int hci_init_sync(struct hci_dev *hdev)
{
int err;
err = hci_init1_sync(hdev);
if (err < 0)
return err;
if (hci_dev_test_flag(hdev, HCI_SETUP))
hci_debugfs_create_basic(hdev);
err = hci_init2_sync(hdev);
if (err < 0)
return err;
/* HCI_PRIMARY covers both single-mode LE, BR/EDR and dual-mode
* BR/EDR/LE type controllers. AMP controllers only need the
* first two stages of init.
*/
if (hdev->dev_type != HCI_PRIMARY)
return 0;
err = hci_init3_sync(hdev);
if (err < 0)
return err;
err = hci_init4_sync(hdev);
if (err < 0)
return err;
/* This function is only called when the controller is actually in
* configured state. When the controller is marked as unconfigured,
* this initialization procedure is not run.
*
* It means that it is possible that a controller runs through its
* setup phase and then discovers missing settings. If that is the
* case, then this function will not be called. It then will only
* be called during the config phase.
*
* So only when in setup phase or config phase, create the debugfs
* entries and register the SMP channels.
*/
if (!hci_dev_test_flag(hdev, HCI_SETUP) &&
!hci_dev_test_flag(hdev, HCI_CONFIG))
return 0;
if (hci_dev_test_and_set_flag(hdev, HCI_DEBUGFS_CREATED))
return 0;
hci_debugfs_create_common(hdev);
if (lmp_bredr_capable(hdev))
hci_debugfs_create_bredr(hdev);
if (lmp_le_capable(hdev))
hci_debugfs_create_le(hdev);
return 0;
}
#define HCI_QUIRK_BROKEN(_quirk, _desc) { HCI_QUIRK_BROKEN_##_quirk, _desc }
static const struct {
unsigned long quirk;
const char *desc;
} hci_broken_table[] = {
HCI_QUIRK_BROKEN(LOCAL_COMMANDS,
"HCI Read Local Supported Commands not supported"),
HCI_QUIRK_BROKEN(STORED_LINK_KEY,
"HCI Delete Stored Link Key command is advertised, "
"but not supported."),
HCI_QUIRK_BROKEN(ERR_DATA_REPORTING,
"HCI Read Default Erroneous Data Reporting command is "
"advertised, but not supported."),
HCI_QUIRK_BROKEN(READ_TRANSMIT_POWER,
"HCI Read Transmit Power Level command is advertised, "
"but not supported."),
HCI_QUIRK_BROKEN(FILTER_CLEAR_ALL,
"HCI Set Event Filter command not supported."),
HCI_QUIRK_BROKEN(ENHANCED_SETUP_SYNC_CONN,
"HCI Enhanced Setup Synchronous Connection command is "
"advertised, but not supported."),
HCI_QUIRK_BROKEN(SET_RPA_TIMEOUT,
"HCI LE Set Random Private Address Timeout command is "
"advertised, but not supported."),
HCI_QUIRK_BROKEN(LE_CODED,
"HCI LE Coded PHY feature bit is set, "
"but its usage is not supported.")
};
/* This function handles hdev setup stage:
*
* Calls hdev->setup
* Setup address if HCI_QUIRK_USE_BDADDR_PROPERTY is set.
*/
static int hci_dev_setup_sync(struct hci_dev *hdev)
{
int ret = 0;
bool invalid_bdaddr;
size_t i;
if (!hci_dev_test_flag(hdev, HCI_SETUP) &&
!test_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks))
return 0;
bt_dev_dbg(hdev, "");
hci_sock_dev_event(hdev, HCI_DEV_SETUP);
if (hdev->setup)
ret = hdev->setup(hdev);
for (i = 0; i < ARRAY_SIZE(hci_broken_table); i++) {
if (test_bit(hci_broken_table[i].quirk, &hdev->quirks))
bt_dev_warn(hdev, "%s", hci_broken_table[i].desc);
}
/* The transport driver can set the quirk to mark the
* BD_ADDR invalid before creating the HCI device or in
* its setup callback.
*/
invalid_bdaddr = test_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks) ||
test_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks);
if (!ret) {
if (test_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks) &&
!bacmp(&hdev->public_addr, BDADDR_ANY))
hci_dev_get_bd_addr_from_property(hdev);
if (invalid_bdaddr && bacmp(&hdev->public_addr, BDADDR_ANY) &&
hdev->set_bdaddr) {
ret = hdev->set_bdaddr(hdev, &hdev->public_addr);
if (!ret)
invalid_bdaddr = false;
}
}
/* The transport driver can set these quirks before
* creating the HCI device or in its setup callback.
*
* For the invalid BD_ADDR quirk it is possible that
* it becomes a valid address if the bootloader does
* provide it (see above).
*
* In case any of them is set, the controller has to
* start up as unconfigured.
*/
if (test_bit(HCI_QUIRK_EXTERNAL_CONFIG, &hdev->quirks) ||
invalid_bdaddr)
hci_dev_set_flag(hdev, HCI_UNCONFIGURED);
/* For an unconfigured controller it is required to
* read at least the version information provided by
* the Read Local Version Information command.
*
* If the set_bdaddr driver callback is provided, then
* also the original Bluetooth public device address
* will be read using the Read BD Address command.
*/
if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED))
return hci_unconf_init_sync(hdev);
return ret;
}
/* This function handles hdev init stage:
*
* Calls hci_dev_setup_sync to perform setup stage
* Calls hci_init_sync to perform HCI command init sequence
*/
static int hci_dev_init_sync(struct hci_dev *hdev)
{
int ret;
bt_dev_dbg(hdev, "");
atomic_set(&hdev->cmd_cnt, 1);
set_bit(HCI_INIT, &hdev->flags);
ret = hci_dev_setup_sync(hdev);
if (hci_dev_test_flag(hdev, HCI_CONFIG)) {
/* If public address change is configured, ensure that
* the address gets programmed. If the driver does not
* support changing the public address, fail the power
* on procedure.
*/
if (bacmp(&hdev->public_addr, BDADDR_ANY) &&
hdev->set_bdaddr)
ret = hdev->set_bdaddr(hdev, &hdev->public_addr);
else
ret = -EADDRNOTAVAIL;
}
if (!ret) {
if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED) &&
!hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) {
ret = hci_init_sync(hdev);
if (!ret && hdev->post_init)
ret = hdev->post_init(hdev);
}
}
/* If the HCI Reset command is clearing all diagnostic settings,
* then they need to be reprogrammed after the init procedure
* completed.
*/
if (test_bit(HCI_QUIRK_NON_PERSISTENT_DIAG, &hdev->quirks) &&
!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
hci_dev_test_flag(hdev, HCI_VENDOR_DIAG) && hdev->set_diag)
ret = hdev->set_diag(hdev, true);
if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) {
msft_do_open(hdev);
aosp_do_open(hdev);
}
clear_bit(HCI_INIT, &hdev->flags);
return ret;
}
int hci_dev_open_sync(struct hci_dev *hdev)
{
int ret;
bt_dev_dbg(hdev, "");
if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) {
ret = -ENODEV;
goto done;
}
if (!hci_dev_test_flag(hdev, HCI_SETUP) &&
!hci_dev_test_flag(hdev, HCI_CONFIG)) {
/* Check for rfkill but allow the HCI setup stage to
* proceed (which in itself doesn't cause any RF activity).
*/
if (hci_dev_test_flag(hdev, HCI_RFKILLED)) {
ret = -ERFKILL;
goto done;
}
/* Check for valid public address or a configured static
* random address, but let the HCI setup proceed to
* be able to determine if there is a public address
* or not.
*
* In case of user channel usage, it is not important
* if a public address or static random address is
* available.
*
* This check is only valid for BR/EDR controllers
* since AMP controllers do not have an address.
*/
if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
hdev->dev_type == HCI_PRIMARY &&
!bacmp(&hdev->bdaddr, BDADDR_ANY) &&
!bacmp(&hdev->static_addr, BDADDR_ANY)) {
ret = -EADDRNOTAVAIL;
goto done;
}
}
if (test_bit(HCI_UP, &hdev->flags)) {
ret = -EALREADY;
goto done;
}
if (hdev->open(hdev)) {
ret = -EIO;
goto done;
}
hci_devcd_reset(hdev);
set_bit(HCI_RUNNING, &hdev->flags);
hci_sock_dev_event(hdev, HCI_DEV_OPEN);
ret = hci_dev_init_sync(hdev);
if (!ret) {
hci_dev_hold(hdev);
hci_dev_set_flag(hdev, HCI_RPA_EXPIRED);
hci_adv_instances_set_rpa_expired(hdev, true);
set_bit(HCI_UP, &hdev->flags);
hci_sock_dev_event(hdev, HCI_DEV_UP);
hci_leds_update_powered(hdev, true);
if (!hci_dev_test_flag(hdev, HCI_SETUP) &&
!hci_dev_test_flag(hdev, HCI_CONFIG) &&
!hci_dev_test_flag(hdev, HCI_UNCONFIGURED) &&
!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
hci_dev_test_flag(hdev, HCI_MGMT) &&
hdev->dev_type == HCI_PRIMARY) {
ret = hci_powered_update_sync(hdev);
mgmt_power_on(hdev, ret);
}
} else {
/* Init failed, cleanup */
flush_work(&hdev->tx_work);
/* Since hci_rx_work() is possible to awake new cmd_work
* it should be flushed first to avoid unexpected call of
* hci_cmd_work()
*/
flush_work(&hdev->rx_work);
flush_work(&hdev->cmd_work);
skb_queue_purge(&hdev->cmd_q);
skb_queue_purge(&hdev->rx_q);
if (hdev->flush)
hdev->flush(hdev);
if (hdev->sent_cmd) {
cancel_delayed_work_sync(&hdev->cmd_timer);
kfree_skb(hdev->sent_cmd);
hdev->sent_cmd = NULL;
}
if (hdev->req_skb) {
kfree_skb(hdev->req_skb);
hdev->req_skb = NULL;
}
clear_bit(HCI_RUNNING, &hdev->flags);
hci_sock_dev_event(hdev, HCI_DEV_CLOSE);
hdev->close(hdev);
hdev->flags &= BIT(HCI_RAW);
}
done:
return ret;
}
/* This function requires the caller holds hdev->lock */
static void hci_pend_le_actions_clear(struct hci_dev *hdev)
{
struct hci_conn_params *p;
list_for_each_entry(p, &hdev->le_conn_params, list) { hci_pend_le_list_del_init(p);
if (p->conn) {
hci_conn_drop(p->conn); hci_conn_put(p->conn);
p->conn = NULL;
}
}
BT_DBG("All LE pending actions cleared");
}
static int hci_dev_shutdown(struct hci_dev *hdev)
{
int err = 0;
/* Similar to how we first do setup and then set the exclusive access
* bit for userspace, we must first unset userchannel and then clean up.
* Otherwise, the kernel can't properly use the hci channel to clean up
* the controller (some shutdown routines require sending additional
* commands to the controller for example).
*/
bool was_userchannel =
hci_dev_test_and_clear_flag(hdev, HCI_USER_CHANNEL);
if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) &&
test_bit(HCI_UP, &hdev->flags)) {
/* Execute vendor specific shutdown routine */
if (hdev->shutdown) err = hdev->shutdown(hdev);
}
if (was_userchannel) hci_dev_set_flag(hdev, HCI_USER_CHANNEL);
return err;
}
int hci_dev_close_sync(struct hci_dev *hdev)
{
bool auto_off;
int err = 0;
bt_dev_dbg(hdev, ""); cancel_delayed_work(&hdev->power_off);
cancel_delayed_work(&hdev->ncmd_timer);
cancel_delayed_work(&hdev->le_scan_disable);
hci_request_cancel_all(hdev);
if (hdev->adv_instance_timeout) {
cancel_delayed_work_sync(&hdev->adv_instance_expire);
hdev->adv_instance_timeout = 0;
}
err = hci_dev_shutdown(hdev); if (!test_and_clear_bit(HCI_UP, &hdev->flags)) { cancel_delayed_work_sync(&hdev->cmd_timer);
return err;
}
hci_leds_update_powered(hdev, false);
/* Flush RX and TX works */
flush_work(&hdev->tx_work);
flush_work(&hdev->rx_work);
if (hdev->discov_timeout > 0) {
hdev->discov_timeout = 0;
hci_dev_clear_flag(hdev, HCI_DISCOVERABLE);
hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE);
}
if (hci_dev_test_and_clear_flag(hdev, HCI_SERVICE_CACHE)) cancel_delayed_work(&hdev->service_cache); if (hci_dev_test_flag(hdev, HCI_MGMT)) {
struct adv_info *adv_instance;
cancel_delayed_work_sync(&hdev->rpa_expired);
list_for_each_entry(adv_instance, &hdev->adv_instances, list)
cancel_delayed_work_sync(&adv_instance->rpa_expired_cb);
}
/* Avoid potential lockdep warnings from the *_flush() calls by
* ensuring the workqueue is empty up front.
*/
drain_workqueue(hdev->workqueue);
hci_dev_lock(hdev);
hci_discovery_set_state(hdev, DISCOVERY_STOPPED);
auto_off = hci_dev_test_and_clear_flag(hdev, HCI_AUTO_OFF);
if (!auto_off && hdev->dev_type == HCI_PRIMARY && !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && hci_dev_test_flag(hdev, HCI_MGMT)) __mgmt_power_off(hdev); hci_inquiry_cache_flush(hdev); hci_pend_le_actions_clear(hdev); hci_conn_hash_flush(hdev);
/* Prevent data races on hdev->smp_data or hdev->smp_bredr_data */
smp_unregister(hdev);
hci_dev_unlock(hdev);
hci_sock_dev_event(hdev, HCI_DEV_DOWN);
if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) {
aosp_do_close(hdev);
msft_do_close(hdev);
}
if (hdev->flush)
hdev->flush(hdev);
/* Reset device */
skb_queue_purge(&hdev->cmd_q);
atomic_set(&hdev->cmd_cnt, 1);
if (test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks) && !auto_off && !hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) { set_bit(HCI_INIT, &hdev->flags);
hci_reset_sync(hdev);
clear_bit(HCI_INIT, &hdev->flags);
}
/* flush cmd work */
flush_work(&hdev->cmd_work);
/* Drop queues */
skb_queue_purge(&hdev->rx_q);
skb_queue_purge(&hdev->cmd_q);
skb_queue_purge(&hdev->raw_q);
/* Drop last sent command */
if (hdev->sent_cmd) {
cancel_delayed_work_sync(&hdev->cmd_timer);
kfree_skb(hdev->sent_cmd);
hdev->sent_cmd = NULL;
}
/* Drop last request */
if (hdev->req_skb) { kfree_skb(hdev->req_skb);
hdev->req_skb = NULL;
}
clear_bit(HCI_RUNNING, &hdev->flags);
hci_sock_dev_event(hdev, HCI_DEV_CLOSE);
/* After this point our queues are empty and no tasks are scheduled. */
hdev->close(hdev);
/* Clear flags */
hdev->flags &= BIT(HCI_RAW);
hci_dev_clear_volatile_flags(hdev);
/* Controller radio is available but is currently powered down */
hdev->amp_status = AMP_STATUS_POWERED_DOWN;
memset(hdev->eir, 0, sizeof(hdev->eir));
memset(hdev->dev_class, 0, sizeof(hdev->dev_class));
bacpy(&hdev->random_addr, BDADDR_ANY);
hci_codec_list_clear(&hdev->local_codecs);
hci_dev_put(hdev);
return err;
}
/* This function perform power on HCI command sequence as follows:
*
* If controller is already up (HCI_UP) performs hci_powered_update_sync
* sequence otherwise run hci_dev_open_sync which will follow with
* hci_powered_update_sync after the init sequence is completed.
*/
static int hci_power_on_sync(struct hci_dev *hdev)
{
int err;
if (test_bit(HCI_UP, &hdev->flags) &&
hci_dev_test_flag(hdev, HCI_MGMT) &&
hci_dev_test_and_clear_flag(hdev, HCI_AUTO_OFF)) {
cancel_delayed_work(&hdev->power_off);
return hci_powered_update_sync(hdev);
}
err = hci_dev_open_sync(hdev);
if (err < 0)
return err;
/* During the HCI setup phase, a few error conditions are
* ignored and they need to be checked now. If they are still
* valid, it is important to return the device back off.
*/
if (hci_dev_test_flag(hdev, HCI_RFKILLED) ||
hci_dev_test_flag(hdev, HCI_UNCONFIGURED) ||
(hdev->dev_type == HCI_PRIMARY &&
!bacmp(&hdev->bdaddr, BDADDR_ANY) &&
!bacmp(&hdev->static_addr, BDADDR_ANY))) {
hci_dev_clear_flag(hdev, HCI_AUTO_OFF);
hci_dev_close_sync(hdev);
} else if (hci_dev_test_flag(hdev, HCI_AUTO_OFF)) {
queue_delayed_work(hdev->req_workqueue, &hdev->power_off,
HCI_AUTO_OFF_TIMEOUT);
}
if (hci_dev_test_and_clear_flag(hdev, HCI_SETUP)) {
/* For unconfigured devices, set the HCI_RAW flag
* so that userspace can easily identify them.
*/
if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED))
set_bit(HCI_RAW, &hdev->flags);
/* For fully configured devices, this will send
* the Index Added event. For unconfigured devices,
* it will send Unconfigued Index Added event.
*
* Devices with HCI_QUIRK_RAW_DEVICE are ignored
* and no event will be send.
*/
mgmt_index_added(hdev);
} else if (hci_dev_test_and_clear_flag(hdev, HCI_CONFIG)) {
/* When the controller is now configured, then it
* is important to clear the HCI_RAW flag.
*/
if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED))
clear_bit(HCI_RAW, &hdev->flags);
/* Powering on the controller with HCI_CONFIG set only
* happens with the transition from unconfigured to
* configured. This will send the Index Added event.
*/
mgmt_index_added(hdev);
}
return 0;
}
static int hci_remote_name_cancel_sync(struct hci_dev *hdev, bdaddr_t *addr)
{
struct hci_cp_remote_name_req_cancel cp;
memset(&cp, 0, sizeof(cp));
bacpy(&cp.bdaddr, addr);
return __hci_cmd_sync_status(hdev, HCI_OP_REMOTE_NAME_REQ_CANCEL,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_stop_discovery_sync(struct hci_dev *hdev)
{
struct discovery_state *d = &hdev->discovery;
struct inquiry_entry *e;
int err;
bt_dev_dbg(hdev, "state %u", hdev->discovery.state);
if (d->state == DISCOVERY_FINDING || d->state == DISCOVERY_STOPPING) {
if (test_bit(HCI_INQUIRY, &hdev->flags)) {
err = __hci_cmd_sync_status(hdev, HCI_OP_INQUIRY_CANCEL,
0, NULL, HCI_CMD_TIMEOUT);
if (err)
return err;
}
if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) {
cancel_delayed_work(&hdev->le_scan_disable);
err = hci_scan_disable_sync(hdev);
if (err)
return err;
}
} else {
err = hci_scan_disable_sync(hdev);
if (err)
return err;
}
/* Resume advertising if it was paused */
if (use_ll_privacy(hdev))
hci_resume_advertising_sync(hdev);
/* No further actions needed for LE-only discovery */
if (d->type == DISCOV_TYPE_LE)
return 0;
if (d->state == DISCOVERY_RESOLVING || d->state == DISCOVERY_STOPPING) {
e = hci_inquiry_cache_lookup_resolve(hdev, BDADDR_ANY,
NAME_PENDING);
if (!e)
return 0;
return hci_remote_name_cancel_sync(hdev, &e->data.bdaddr);
}
return 0;
}
static int hci_disconnect_phy_link_sync(struct hci_dev *hdev, u16 handle,
u8 reason)
{
struct hci_cp_disconn_phy_link cp;
memset(&cp, 0, sizeof(cp));
cp.phy_handle = HCI_PHY_HANDLE(handle);
cp.reason = reason;
return __hci_cmd_sync_status(hdev, HCI_OP_DISCONN_PHY_LINK,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_disconnect_sync(struct hci_dev *hdev, struct hci_conn *conn,
u8 reason)
{
struct hci_cp_disconnect cp;
if (conn->type == AMP_LINK)
return hci_disconnect_phy_link_sync(hdev, conn->handle, reason);
if (test_bit(HCI_CONN_BIG_CREATED, &conn->flags)) {
/* This is a BIS connection, hci_conn_del will
* do the necessary cleanup.
*/
hci_dev_lock(hdev);
hci_conn_failed(conn, reason);
hci_dev_unlock(hdev);
return 0;
}
memset(&cp, 0, sizeof(cp));
cp.handle = cpu_to_le16(conn->handle);
cp.reason = reason;
/* Wait for HCI_EV_DISCONN_COMPLETE, not HCI_EV_CMD_STATUS, when the
* reason is anything but HCI_ERROR_REMOTE_POWER_OFF. This reason is
* used when suspending or powering off, where we don't want to wait
* for the peer's response.
*/
if (reason != HCI_ERROR_REMOTE_POWER_OFF)
return __hci_cmd_sync_status_sk(hdev, HCI_OP_DISCONNECT,
sizeof(cp), &cp,
HCI_EV_DISCONN_COMPLETE,
HCI_CMD_TIMEOUT, NULL);
return __hci_cmd_sync_status(hdev, HCI_OP_DISCONNECT, sizeof(cp), &cp,
HCI_CMD_TIMEOUT);
}
static int hci_le_connect_cancel_sync(struct hci_dev *hdev,
struct hci_conn *conn, u8 reason)
{
/* Return reason if scanning since the connection shall probably be
* cleanup directly.
*/
if (test_bit(HCI_CONN_SCANNING, &conn->flags))
return reason;
if (conn->role == HCI_ROLE_SLAVE ||
test_and_set_bit(HCI_CONN_CANCEL, &conn->flags))
return 0;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_CREATE_CONN_CANCEL,
0, NULL, HCI_CMD_TIMEOUT);
}
static int hci_connect_cancel_sync(struct hci_dev *hdev, struct hci_conn *conn,
u8 reason)
{
if (conn->type == LE_LINK)
return hci_le_connect_cancel_sync(hdev, conn, reason);
if (conn->type == ISO_LINK) {
/* BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 4, Part E
* page 1857:
*
* If this command is issued for a CIS on the Central and the
* CIS is successfully terminated before being established,
* then an HCI_LE_CIS_Established event shall also be sent for
* this CIS with the Status Operation Cancelled by Host (0x44).
*/
if (test_bit(HCI_CONN_CREATE_CIS, &conn->flags))
return hci_disconnect_sync(hdev, conn, reason);
/* CIS with no Create CIS sent have nothing to cancel */
if (bacmp(&conn->dst, BDADDR_ANY))
return HCI_ERROR_LOCAL_HOST_TERM;
/* There is no way to cancel a BIS without terminating the BIG
* which is done later on connection cleanup.
*/
return 0;
}
if (hdev->hci_ver < BLUETOOTH_VER_1_2)
return 0;
/* Wait for HCI_EV_CONN_COMPLETE, not HCI_EV_CMD_STATUS, when the
* reason is anything but HCI_ERROR_REMOTE_POWER_OFF. This reason is
* used when suspending or powering off, where we don't want to wait
* for the peer's response.
*/
if (reason != HCI_ERROR_REMOTE_POWER_OFF)
return __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN_CANCEL,
6, &conn->dst,
HCI_EV_CONN_COMPLETE,
HCI_CMD_TIMEOUT, NULL);
return __hci_cmd_sync_status(hdev, HCI_OP_CREATE_CONN_CANCEL,
6, &conn->dst, HCI_CMD_TIMEOUT);
}
static int hci_reject_sco_sync(struct hci_dev *hdev, struct hci_conn *conn,
u8 reason)
{
struct hci_cp_reject_sync_conn_req cp;
memset(&cp, 0, sizeof(cp));
bacpy(&cp.bdaddr, &conn->dst);
cp.reason = reason;
/* SCO rejection has its own limited set of
* allowed error values (0x0D-0x0F).
*/
if (reason < 0x0d || reason > 0x0f)
cp.reason = HCI_ERROR_REJ_LIMITED_RESOURCES;
return __hci_cmd_sync_status(hdev, HCI_OP_REJECT_SYNC_CONN_REQ,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_le_reject_cis_sync(struct hci_dev *hdev, struct hci_conn *conn,
u8 reason)
{
struct hci_cp_le_reject_cis cp;
memset(&cp, 0, sizeof(cp));
cp.handle = cpu_to_le16(conn->handle);
cp.reason = reason;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_REJECT_CIS,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_reject_conn_sync(struct hci_dev *hdev, struct hci_conn *conn,
u8 reason)
{
struct hci_cp_reject_conn_req cp;
if (conn->type == ISO_LINK)
return hci_le_reject_cis_sync(hdev, conn, reason);
if (conn->type == SCO_LINK || conn->type == ESCO_LINK)
return hci_reject_sco_sync(hdev, conn, reason);
memset(&cp, 0, sizeof(cp));
bacpy(&cp.bdaddr, &conn->dst);
cp.reason = reason;
return __hci_cmd_sync_status(hdev, HCI_OP_REJECT_CONN_REQ,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_abort_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, u8 reason)
{
int err = 0;
u16 handle = conn->handle;
bool disconnect = false;
struct hci_conn *c;
switch (conn->state) {
case BT_CONNECTED:
case BT_CONFIG:
err = hci_disconnect_sync(hdev, conn, reason);
break;
case BT_CONNECT:
err = hci_connect_cancel_sync(hdev, conn, reason);
break;
case BT_CONNECT2:
err = hci_reject_conn_sync(hdev, conn, reason);
break;
case BT_OPEN:
case BT_BOUND:
break;
default:
disconnect = true;
break;
}
hci_dev_lock(hdev);
/* Check if the connection has been cleaned up concurrently */
c = hci_conn_hash_lookup_handle(hdev, handle);
if (!c || c != conn) {
err = 0;
goto unlock;
}
/* Cleanup hci_conn object if it cannot be cancelled as it
* likelly means the controller and host stack are out of sync
* or in case of LE it was still scanning so it can be cleanup
* safely.
*/
if (disconnect) {
conn->state = BT_CLOSED;
hci_disconn_cfm(conn, reason);
hci_conn_del(conn);
} else {
hci_conn_failed(conn, reason);
}
unlock:
hci_dev_unlock(hdev);
return err;
}
static int hci_disconnect_all_sync(struct hci_dev *hdev, u8 reason)
{
struct list_head *head = &hdev->conn_hash.list;
struct hci_conn *conn;
rcu_read_lock();
while ((conn = list_first_or_null_rcu(head, struct hci_conn, list))) {
/* Make sure the connection is not freed while unlocking */
conn = hci_conn_get(conn);
rcu_read_unlock();
/* Disregard possible errors since hci_conn_del shall have been
* called even in case of errors had occurred since it would
* then cause hci_conn_failed to be called which calls
* hci_conn_del internally.
*/
hci_abort_conn_sync(hdev, conn, reason);
hci_conn_put(conn);
rcu_read_lock();
}
rcu_read_unlock();
return 0;
}
/* This function perform power off HCI command sequence as follows:
*
* Clear Advertising
* Stop Discovery
* Disconnect all connections
* hci_dev_close_sync
*/
static int hci_power_off_sync(struct hci_dev *hdev)
{
int err;
/* If controller is already down there is nothing to do */
if (!test_bit(HCI_UP, &hdev->flags))
return 0;
hci_dev_set_flag(hdev, HCI_POWERING_DOWN);
if (test_bit(HCI_ISCAN, &hdev->flags) ||
test_bit(HCI_PSCAN, &hdev->flags)) {
err = hci_write_scan_enable_sync(hdev, 0x00);
if (err)
goto out;
}
err = hci_clear_adv_sync(hdev, NULL, false);
if (err)
goto out;
err = hci_stop_discovery_sync(hdev);
if (err)
goto out;
/* Terminated due to Power Off */
err = hci_disconnect_all_sync(hdev, HCI_ERROR_REMOTE_POWER_OFF);
if (err)
goto out;
err = hci_dev_close_sync(hdev);
out:
hci_dev_clear_flag(hdev, HCI_POWERING_DOWN);
return err;
}
int hci_set_powered_sync(struct hci_dev *hdev, u8 val)
{
if (val)
return hci_power_on_sync(hdev);
return hci_power_off_sync(hdev);
}
static int hci_write_iac_sync(struct hci_dev *hdev)
{
struct hci_cp_write_current_iac_lap cp;
if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE))
return 0;
memset(&cp, 0, sizeof(cp));
if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) {
/* Limited discoverable mode */
cp.num_iac = min_t(u8, hdev->num_iac, 2);
cp.iac_lap[0] = 0x00; /* LIAC */
cp.iac_lap[1] = 0x8b;
cp.iac_lap[2] = 0x9e;
cp.iac_lap[3] = 0x33; /* GIAC */
cp.iac_lap[4] = 0x8b;
cp.iac_lap[5] = 0x9e;
} else {
/* General discoverable mode */
cp.num_iac = 1;
cp.iac_lap[0] = 0x33; /* GIAC */
cp.iac_lap[1] = 0x8b;
cp.iac_lap[2] = 0x9e;
}
return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CURRENT_IAC_LAP,
(cp.num_iac * 3) + 1, &cp,
HCI_CMD_TIMEOUT);
}
int hci_update_discoverable_sync(struct hci_dev *hdev)
{
int err = 0;
if (hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) {
err = hci_write_iac_sync(hdev);
if (err)
return err;
err = hci_update_scan_sync(hdev);
if (err)
return err;
err = hci_update_class_sync(hdev);
if (err)
return err;
}
/* Advertising instances don't use the global discoverable setting, so
* only update AD if advertising was enabled using Set Advertising.
*/
if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) {
err = hci_update_adv_data_sync(hdev, 0x00);
if (err)
return err;
/* Discoverable mode affects the local advertising
* address in limited privacy mode.
*/
if (hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) {
if (ext_adv_capable(hdev))
err = hci_start_ext_adv_sync(hdev, 0x00);
else
err = hci_enable_advertising_sync(hdev);
}
}
return err;
}
static int update_discoverable_sync(struct hci_dev *hdev, void *data)
{
return hci_update_discoverable_sync(hdev);
}
int hci_update_discoverable(struct hci_dev *hdev)
{
/* Only queue if it would have any effect */
if (hdev_is_powered(hdev) &&
hci_dev_test_flag(hdev, HCI_ADVERTISING) &&
hci_dev_test_flag(hdev, HCI_DISCOVERABLE) &&
hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY))
return hci_cmd_sync_queue(hdev, update_discoverable_sync, NULL,
NULL);
return 0;
}
int hci_update_connectable_sync(struct hci_dev *hdev)
{
int err;
err = hci_update_scan_sync(hdev);
if (err)
return err;
/* If BR/EDR is not enabled and we disable advertising as a
* by-product of disabling connectable, we need to update the
* advertising flags.
*/
if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
err = hci_update_adv_data_sync(hdev, hdev->cur_adv_instance);
/* Update the advertising parameters if necessary */
if (hci_dev_test_flag(hdev, HCI_ADVERTISING) ||
!list_empty(&hdev->adv_instances)) {
if (ext_adv_capable(hdev))
err = hci_start_ext_adv_sync(hdev,
hdev->cur_adv_instance);
else
err = hci_enable_advertising_sync(hdev);
if (err)
return err;
}
return hci_update_passive_scan_sync(hdev);
}
static int hci_inquiry_sync(struct hci_dev *hdev, u8 length)
{
const u8 giac[3] = { 0x33, 0x8b, 0x9e };
const u8 liac[3] = { 0x00, 0x8b, 0x9e };
struct hci_cp_inquiry cp;
bt_dev_dbg(hdev, "");
if (test_bit(HCI_INQUIRY, &hdev->flags))
return 0;
hci_dev_lock(hdev);
hci_inquiry_cache_flush(hdev);
hci_dev_unlock(hdev);
memset(&cp, 0, sizeof(cp));
if (hdev->discovery.limited)
memcpy(&cp.lap, liac, sizeof(cp.lap));
else
memcpy(&cp.lap, giac, sizeof(cp.lap));
cp.length = length;
return __hci_cmd_sync_status(hdev, HCI_OP_INQUIRY,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
static int hci_active_scan_sync(struct hci_dev *hdev, uint16_t interval)
{
u8 own_addr_type;
/* Accept list is not used for discovery */
u8 filter_policy = 0x00;
/* Default is to enable duplicates filter */
u8 filter_dup = LE_SCAN_FILTER_DUP_ENABLE;
int err;
bt_dev_dbg(hdev, "");
/* If controller is scanning, it means the passive scanning is
* running. Thus, we should temporarily stop it in order to set the
* discovery scanning parameters.
*/
err = hci_scan_disable_sync(hdev);
if (err) {
bt_dev_err(hdev, "Unable to disable scanning: %d", err);
return err;
}
cancel_interleave_scan(hdev);
/* Pause address resolution for active scan and stop advertising if
* privacy is enabled.
*/
err = hci_pause_addr_resolution(hdev);
if (err)
goto failed;
/* All active scans will be done with either a resolvable private
* address (when privacy feature has been enabled) or non-resolvable
* private address.
*/
err = hci_update_random_address_sync(hdev, true, scan_use_rpa(hdev),
&own_addr_type);
if (err < 0)
own_addr_type = ADDR_LE_DEV_PUBLIC;
if (hci_is_adv_monitoring(hdev) ||
(test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) &&
hdev->discovery.result_filtering)) {
/* Duplicate filter should be disabled when some advertisement
* monitor is activated, otherwise AdvMon can only receive one
* advertisement for one peer(*) during active scanning, and
* might report loss to these peers.
*
* If controller does strict duplicate filtering and the
* discovery requires result filtering disables controller based
* filtering since that can cause reports that would match the
* host filter to not be reported.
*/
filter_dup = LE_SCAN_FILTER_DUP_DISABLE;
}
err = hci_start_scan_sync(hdev, LE_SCAN_ACTIVE, interval,
hdev->le_scan_window_discovery,
own_addr_type, filter_policy, filter_dup);
if (!err)
return err;
failed:
/* Resume advertising if it was paused */
if (use_ll_privacy(hdev))
hci_resume_advertising_sync(hdev);
/* Resume passive scanning */
hci_update_passive_scan_sync(hdev);
return err;
}
static int hci_start_interleaved_discovery_sync(struct hci_dev *hdev)
{
int err;
bt_dev_dbg(hdev, "");
err = hci_active_scan_sync(hdev, hdev->le_scan_int_discovery * 2);
if (err)
return err;
return hci_inquiry_sync(hdev, DISCOV_BREDR_INQUIRY_LEN);
}
int hci_start_discovery_sync(struct hci_dev *hdev)
{
unsigned long timeout;
int err;
bt_dev_dbg(hdev, "type %u", hdev->discovery.type);
switch (hdev->discovery.type) {
case DISCOV_TYPE_BREDR:
return hci_inquiry_sync(hdev, DISCOV_BREDR_INQUIRY_LEN);
case DISCOV_TYPE_INTERLEAVED:
/* When running simultaneous discovery, the LE scanning time
* should occupy the whole discovery time sine BR/EDR inquiry
* and LE scanning are scheduled by the controller.
*
* For interleaving discovery in comparison, BR/EDR inquiry
* and LE scanning are done sequentially with separate
* timeouts.
*/
if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY,
&hdev->quirks)) {
timeout = msecs_to_jiffies(DISCOV_LE_TIMEOUT);
/* During simultaneous discovery, we double LE scan
* interval. We must leave some time for the controller
* to do BR/EDR inquiry.
*/
err = hci_start_interleaved_discovery_sync(hdev);
break;
}
timeout = msecs_to_jiffies(hdev->discov_interleaved_timeout);
err = hci_active_scan_sync(hdev, hdev->le_scan_int_discovery);
break;
case DISCOV_TYPE_LE:
timeout = msecs_to_jiffies(DISCOV_LE_TIMEOUT);
err = hci_active_scan_sync(hdev, hdev->le_scan_int_discovery);
break;
default:
return -EINVAL;
}
if (err)
return err;
bt_dev_dbg(hdev, "timeout %u ms", jiffies_to_msecs(timeout));
queue_delayed_work(hdev->req_workqueue, &hdev->le_scan_disable,
timeout);
return 0;
}
static void hci_suspend_monitor_sync(struct hci_dev *hdev)
{
switch (hci_get_adv_monitor_offload_ext(hdev)) {
case HCI_ADV_MONITOR_EXT_MSFT:
msft_suspend_sync(hdev);
break;
default:
return;
}
}
/* This function disables discovery and mark it as paused */
static int hci_pause_discovery_sync(struct hci_dev *hdev)
{
int old_state = hdev->discovery.state;
int err;
/* If discovery already stopped/stopping/paused there nothing to do */
if (old_state == DISCOVERY_STOPPED || old_state == DISCOVERY_STOPPING ||
hdev->discovery_paused)
return 0;
hci_discovery_set_state(hdev, DISCOVERY_STOPPING);
err = hci_stop_discovery_sync(hdev);
if (err)
return err;
hdev->discovery_paused = true;
hdev->discovery_old_state = old_state;
hci_discovery_set_state(hdev, DISCOVERY_STOPPED);
return 0;
}
static int hci_update_event_filter_sync(struct hci_dev *hdev)
{
struct bdaddr_list_with_flags *b;
u8 scan = SCAN_DISABLED;
bool scanning = test_bit(HCI_PSCAN, &hdev->flags);
int err;
if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
return 0;
/* Some fake CSR controllers lock up after setting this type of
* filter, so avoid sending the request altogether.
*/
if (test_bit(HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL, &hdev->quirks))
return 0;
/* Always clear event filter when starting */
hci_clear_event_filter_sync(hdev);
list_for_each_entry(b, &hdev->accept_list, list) {
if (!(b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP))
continue;
bt_dev_dbg(hdev, "Adding event filters for %pMR", &b->bdaddr);
err = hci_set_event_filter_sync(hdev, HCI_FLT_CONN_SETUP,
HCI_CONN_SETUP_ALLOW_BDADDR,
&b->bdaddr,
HCI_CONN_SETUP_AUTO_ON);
if (err)
bt_dev_dbg(hdev, "Failed to set event filter for %pMR",
&b->bdaddr);
else
scan = SCAN_PAGE;
}
if (scan && !scanning)
hci_write_scan_enable_sync(hdev, scan);
else if (!scan && scanning)
hci_write_scan_enable_sync(hdev, scan);
return 0;
}
/* This function disables scan (BR and LE) and mark it as paused */
static int hci_pause_scan_sync(struct hci_dev *hdev)
{
if (hdev->scanning_paused)
return 0;
/* Disable page scan if enabled */
if (test_bit(HCI_PSCAN, &hdev->flags))
hci_write_scan_enable_sync(hdev, SCAN_DISABLED);
hci_scan_disable_sync(hdev);
hdev->scanning_paused = true;
return 0;
}
/* This function performs the HCI suspend procedures in the follow order:
*
* Pause discovery (active scanning/inquiry)
* Pause Directed Advertising/Advertising
* Pause Scanning (passive scanning in case discovery was not active)
* Disconnect all connections
* Set suspend_status to BT_SUSPEND_DISCONNECT if hdev cannot wakeup
* otherwise:
* Update event mask (only set events that are allowed to wake up the host)
* Update event filter (with devices marked with HCI_CONN_FLAG_REMOTE_WAKEUP)
* Update passive scanning (lower duty cycle)
* Set suspend_status to BT_SUSPEND_CONFIGURE_WAKE
*/
int hci_suspend_sync(struct hci_dev *hdev)
{
int err;
/* If marked as suspended there nothing to do */
if (hdev->suspended)
return 0;
/* Mark device as suspended */
hdev->suspended = true;
/* Pause discovery if not already stopped */
hci_pause_discovery_sync(hdev);
/* Pause other advertisements */
hci_pause_advertising_sync(hdev);
/* Suspend monitor filters */
hci_suspend_monitor_sync(hdev);
/* Prevent disconnects from causing scanning to be re-enabled */
hci_pause_scan_sync(hdev);
if (hci_conn_count(hdev)) {
/* Soft disconnect everything (power off) */
err = hci_disconnect_all_sync(hdev, HCI_ERROR_REMOTE_POWER_OFF);
if (err) {
/* Set state to BT_RUNNING so resume doesn't notify */
hdev->suspend_state = BT_RUNNING;
hci_resume_sync(hdev);
return err;
}
/* Update event mask so only the allowed event can wakeup the
* host.
*/
hci_set_event_mask_sync(hdev);
}
/* Only configure accept list if disconnect succeeded and wake
* isn't being prevented.
*/
if (!hdev->wakeup || !hdev->wakeup(hdev)) {
hdev->suspend_state = BT_SUSPEND_DISCONNECT;
return 0;
}
/* Unpause to take care of updating scanning params */
hdev->scanning_paused = false;
/* Enable event filter for paired devices */
hci_update_event_filter_sync(hdev);
/* Update LE passive scan if enabled */
hci_update_passive_scan_sync(hdev);
/* Pause scan changes again. */
hdev->scanning_paused = true;
hdev->suspend_state = BT_SUSPEND_CONFIGURE_WAKE;
return 0;
}
/* This function resumes discovery */
static int hci_resume_discovery_sync(struct hci_dev *hdev)
{
int err;
/* If discovery not paused there nothing to do */
if (!hdev->discovery_paused)
return 0;
hdev->discovery_paused = false;
hci_discovery_set_state(hdev, DISCOVERY_STARTING);
err = hci_start_discovery_sync(hdev);
hci_discovery_set_state(hdev, err ? DISCOVERY_STOPPED :
DISCOVERY_FINDING);
return err;
}
static void hci_resume_monitor_sync(struct hci_dev *hdev)
{
switch (hci_get_adv_monitor_offload_ext(hdev)) {
case HCI_ADV_MONITOR_EXT_MSFT:
msft_resume_sync(hdev);
break;
default:
return;
}
}
/* This function resume scan and reset paused flag */
static int hci_resume_scan_sync(struct hci_dev *hdev)
{
if (!hdev->scanning_paused)
return 0;
hdev->scanning_paused = false;
hci_update_scan_sync(hdev);
/* Reset passive scanning to normal */
hci_update_passive_scan_sync(hdev);
return 0;
}
/* This function performs the HCI suspend procedures in the follow order:
*
* Restore event mask
* Clear event filter
* Update passive scanning (normal duty cycle)
* Resume Directed Advertising/Advertising
* Resume discovery (active scanning/inquiry)
*/
int hci_resume_sync(struct hci_dev *hdev)
{
/* If not marked as suspended there nothing to do */
if (!hdev->suspended)
return 0;
hdev->suspended = false;
/* Restore event mask */
hci_set_event_mask_sync(hdev);
/* Clear any event filters and restore scan state */
hci_clear_event_filter_sync(hdev);
/* Resume scanning */
hci_resume_scan_sync(hdev);
/* Resume monitor filters */
hci_resume_monitor_sync(hdev);
/* Resume other advertisements */
hci_resume_advertising_sync(hdev);
/* Resume discovery */
hci_resume_discovery_sync(hdev);
return 0;
}
static bool conn_use_rpa(struct hci_conn *conn)
{
struct hci_dev *hdev = conn->hdev;
return hci_dev_test_flag(hdev, HCI_PRIVACY);
}
static int hci_le_ext_directed_advertising_sync(struct hci_dev *hdev,
struct hci_conn *conn)
{
struct hci_cp_le_set_ext_adv_params cp;
int err;
bdaddr_t random_addr;
u8 own_addr_type;
err = hci_update_random_address_sync(hdev, false, conn_use_rpa(conn),
&own_addr_type);
if (err)
return err;
/* Set require_privacy to false so that the remote device has a
* chance of identifying us.
*/
err = hci_get_random_address(hdev, false, conn_use_rpa(conn), NULL,
&own_addr_type, &random_addr);
if (err)
return err;
memset(&cp, 0, sizeof(cp));
cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_DIRECT_IND);
cp.channel_map = hdev->le_adv_channel_map;
cp.tx_power = HCI_TX_POWER_INVALID;
cp.primary_phy = HCI_ADV_PHY_1M;
cp.secondary_phy = HCI_ADV_PHY_1M;
cp.handle = 0x00; /* Use instance 0 for directed adv */
cp.own_addr_type = own_addr_type;
cp.peer_addr_type = conn->dst_type;
bacpy(&cp.peer_addr, &conn->dst);
/* As per Core Spec 5.2 Vol 2, PART E, Sec 7.8.53, for
* advertising_event_property LE_LEGACY_ADV_DIRECT_IND
* does not supports advertising data when the advertising set already
* contains some, the controller shall return erroc code 'Invalid
* HCI Command Parameters(0x12).
* So it is required to remove adv set for handle 0x00. since we use
* instance 0 for directed adv.
*/
err = hci_remove_ext_adv_instance_sync(hdev, cp.handle, NULL);
if (err)
return err;
err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (err)
return err;
/* Check if random address need to be updated */
if (own_addr_type == ADDR_LE_DEV_RANDOM &&
bacmp(&random_addr, BDADDR_ANY) &&
bacmp(&random_addr, &hdev->random_addr)) {
err = hci_set_adv_set_random_addr_sync(hdev, 0x00,
&random_addr);
if (err)
return err;
}
return hci_enable_ext_advertising_sync(hdev, 0x00);
}
static int hci_le_directed_advertising_sync(struct hci_dev *hdev,
struct hci_conn *conn)
{
struct hci_cp_le_set_adv_param cp;
u8 status;
u8 own_addr_type;
u8 enable;
if (ext_adv_capable(hdev))
return hci_le_ext_directed_advertising_sync(hdev, conn);
/* Clear the HCI_LE_ADV bit temporarily so that the
* hci_update_random_address knows that it's safe to go ahead
* and write a new random address. The flag will be set back on
* as soon as the SET_ADV_ENABLE HCI command completes.
*/
hci_dev_clear_flag(hdev, HCI_LE_ADV);
/* Set require_privacy to false so that the remote device has a
* chance of identifying us.
*/
status = hci_update_random_address_sync(hdev, false, conn_use_rpa(conn),
&own_addr_type);
if (status)
return status;
memset(&cp, 0, sizeof(cp));
/* Some controllers might reject command if intervals are not
* within range for undirected advertising.
* BCM20702A0 is known to be affected by this.
*/
cp.min_interval = cpu_to_le16(0x0020);
cp.max_interval = cpu_to_le16(0x0020);
cp.type = LE_ADV_DIRECT_IND;
cp.own_address_type = own_addr_type;
cp.direct_addr_type = conn->dst_type;
bacpy(&cp.direct_addr, &conn->dst);
cp.channel_map = hdev->le_adv_channel_map;
status = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_PARAM,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
if (status)
return status;
enable = 0x01;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE,
sizeof(enable), &enable, HCI_CMD_TIMEOUT);
}
static void set_ext_conn_params(struct hci_conn *conn,
struct hci_cp_le_ext_conn_param *p)
{
struct hci_dev *hdev = conn->hdev;
memset(p, 0, sizeof(*p));
p->scan_interval = cpu_to_le16(hdev->le_scan_int_connect);
p->scan_window = cpu_to_le16(hdev->le_scan_window_connect);
p->conn_interval_min = cpu_to_le16(conn->le_conn_min_interval);
p->conn_interval_max = cpu_to_le16(conn->le_conn_max_interval);
p->conn_latency = cpu_to_le16(conn->le_conn_latency);
p->supervision_timeout = cpu_to_le16(conn->le_supv_timeout);
p->min_ce_len = cpu_to_le16(0x0000);
p->max_ce_len = cpu_to_le16(0x0000);
}
static int hci_le_ext_create_conn_sync(struct hci_dev *hdev,
struct hci_conn *conn, u8 own_addr_type)
{
struct hci_cp_le_ext_create_conn *cp;
struct hci_cp_le_ext_conn_param *p;
u8 data[sizeof(*cp) + sizeof(*p) * 3];
u32 plen;
cp = (void *)data;
p = (void *)cp->data;
memset(cp, 0, sizeof(*cp));
bacpy(&cp->peer_addr, &conn->dst);
cp->peer_addr_type = conn->dst_type;
cp->own_addr_type = own_addr_type;
plen = sizeof(*cp);
if (scan_1m(hdev) && (conn->le_adv_phy == HCI_ADV_PHY_1M ||
conn->le_adv_sec_phy == HCI_ADV_PHY_1M)) {
cp->phys |= LE_SCAN_PHY_1M;
set_ext_conn_params(conn, p);
p++;
plen += sizeof(*p);
}
if (scan_2m(hdev) && (conn->le_adv_phy == HCI_ADV_PHY_2M ||
conn->le_adv_sec_phy == HCI_ADV_PHY_2M)) {
cp->phys |= LE_SCAN_PHY_2M;
set_ext_conn_params(conn, p);
p++;
plen += sizeof(*p);
}
if (scan_coded(hdev) && (conn->le_adv_phy == HCI_ADV_PHY_CODED ||
conn->le_adv_sec_phy == HCI_ADV_PHY_CODED)) {
cp->phys |= LE_SCAN_PHY_CODED;
set_ext_conn_params(conn, p);
plen += sizeof(*p);
}
return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_EXT_CREATE_CONN,
plen, data,
HCI_EV_LE_ENHANCED_CONN_COMPLETE,
conn->conn_timeout, NULL);
}
static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data)
{
struct hci_cp_le_create_conn cp;
struct hci_conn_params *params;
u8 own_addr_type;
int err;
struct hci_conn *conn = data;
if (!hci_conn_valid(hdev, conn))
return -ECANCELED;
bt_dev_dbg(hdev, "conn %p", conn);
clear_bit(HCI_CONN_SCANNING, &conn->flags);
conn->state = BT_CONNECT;
/* If requested to connect as peripheral use directed advertising */
if (conn->role == HCI_ROLE_SLAVE) {
/* If we're active scanning and simultaneous roles is not
* enabled simply reject the attempt.
*/
if (hci_dev_test_flag(hdev, HCI_LE_SCAN) &&
hdev->le_scan_type == LE_SCAN_ACTIVE &&
!hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES)) {
hci_conn_del(conn);
return -EBUSY;
}
/* Pause advertising while doing directed advertising. */
hci_pause_advertising_sync(hdev);
err = hci_le_directed_advertising_sync(hdev, conn);
goto done;
}
/* Disable advertising if simultaneous roles is not in use. */
if (!hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES))
hci_pause_advertising_sync(hdev);
params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type);
if (params) {
conn->le_conn_min_interval = params->conn_min_interval;
conn->le_conn_max_interval = params->conn_max_interval;
conn->le_conn_latency = params->conn_latency;
conn->le_supv_timeout = params->supervision_timeout;
} else {
conn->le_conn_min_interval = hdev->le_conn_min_interval;
conn->le_conn_max_interval = hdev->le_conn_max_interval;
conn->le_conn_latency = hdev->le_conn_latency;
conn->le_supv_timeout = hdev->le_supv_timeout;
}
/* If controller is scanning, we stop it since some controllers are
* not able to scan and connect at the same time. Also set the
* HCI_LE_SCAN_INTERRUPTED flag so that the command complete
* handler for scan disabling knows to set the correct discovery
* state.
*/
if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) {
hci_scan_disable_sync(hdev);
hci_dev_set_flag(hdev, HCI_LE_SCAN_INTERRUPTED);
}
/* Update random address, but set require_privacy to false so
* that we never connect with an non-resolvable address.
*/
err = hci_update_random_address_sync(hdev, false, conn_use_rpa(conn),
&own_addr_type);
if (err)
goto done;
if (use_ext_conn(hdev)) {
err = hci_le_ext_create_conn_sync(hdev, conn, own_addr_type);
goto done;
}
memset(&cp, 0, sizeof(cp));
cp.scan_interval = cpu_to_le16(hdev->le_scan_int_connect);
cp.scan_window = cpu_to_le16(hdev->le_scan_window_connect);
bacpy(&cp.peer_addr, &conn->dst);
cp.peer_addr_type = conn->dst_type;
cp.own_address_type = own_addr_type;
cp.conn_interval_min = cpu_to_le16(conn->le_conn_min_interval);
cp.conn_interval_max = cpu_to_le16(conn->le_conn_max_interval);
cp.conn_latency = cpu_to_le16(conn->le_conn_latency);
cp.supervision_timeout = cpu_to_le16(conn->le_supv_timeout);
cp.min_ce_len = cpu_to_le16(0x0000);
cp.max_ce_len = cpu_to_le16(0x0000);
/* BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 4, Part E page 2261:
*
* If this event is unmasked and the HCI_LE_Connection_Complete event
* is unmasked, only the HCI_LE_Enhanced_Connection_Complete event is
* sent when a new connection has been created.
*/
err = __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_CREATE_CONN,
sizeof(cp), &cp,
use_enhanced_conn_complete(hdev) ?
HCI_EV_LE_ENHANCED_CONN_COMPLETE :
HCI_EV_LE_CONN_COMPLETE,
conn->conn_timeout, NULL);
done:
if (err == -ETIMEDOUT)
hci_le_connect_cancel_sync(hdev, conn, 0x00);
/* Re-enable advertising after the connection attempt is finished. */
hci_resume_advertising_sync(hdev);
return err;
}
int hci_le_create_cis_sync(struct hci_dev *hdev)
{
struct {
struct hci_cp_le_create_cis cp;
struct hci_cis cis[0x1f];
} cmd;
struct hci_conn *conn;
u8 cig = BT_ISO_QOS_CIG_UNSET;
/* The spec allows only one pending LE Create CIS command at a time. If
* the command is pending now, don't do anything. We check for pending
* connections after each CIS Established event.
*
* BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 4, Part E
* page 2566:
*
* If the Host issues this command before all the
* HCI_LE_CIS_Established events from the previous use of the
* command have been generated, the Controller shall return the
* error code Command Disallowed (0x0C).
*
* BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 4, Part E
* page 2567:
*
* When the Controller receives the HCI_LE_Create_CIS command, the
* Controller sends the HCI_Command_Status event to the Host. An
* HCI_LE_CIS_Established event will be generated for each CIS when it
* is established or if it is disconnected or considered lost before
* being established; until all the events are generated, the command
* remains pending.
*/
memset(&cmd, 0, sizeof(cmd));
hci_dev_lock(hdev);
rcu_read_lock();
/* Wait until previous Create CIS has completed */
list_for_each_entry_rcu(conn, &hdev->conn_hash.list, list) {
if (test_bit(HCI_CONN_CREATE_CIS, &conn->flags))
goto done;
}
/* Find CIG with all CIS ready */
list_for_each_entry_rcu(conn, &hdev->conn_hash.list, list) {
struct hci_conn *link;
if (hci_conn_check_create_cis(conn))
continue;
cig = conn->iso_qos.ucast.cig;
list_for_each_entry_rcu(link, &hdev->conn_hash.list, list) {
if (hci_conn_check_create_cis(link) > 0 &&
link->iso_qos.ucast.cig == cig &&
link->state != BT_CONNECTED) {
cig = BT_ISO_QOS_CIG_UNSET;
break;
}
}
if (cig != BT_ISO_QOS_CIG_UNSET)
break;
}
if (cig == BT_ISO_QOS_CIG_UNSET)
goto done;
list_for_each_entry_rcu(conn, &hdev->conn_hash.list, list) {
struct hci_cis *cis = &cmd.cis[cmd.cp.num_cis];
if (hci_conn_check_create_cis(conn) ||
conn->iso_qos.ucast.cig != cig)
continue;
set_bit(HCI_CONN_CREATE_CIS, &conn->flags);
cis->acl_handle = cpu_to_le16(conn->parent->handle);
cis->cis_handle = cpu_to_le16(conn->handle);
cmd.cp.num_cis++;
if (cmd.cp.num_cis >= ARRAY_SIZE(cmd.cis))
break;
}
done:
rcu_read_unlock();
hci_dev_unlock(hdev);
if (!cmd.cp.num_cis)
return 0;
/* Wait for HCI_LE_CIS_Established */
return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_CREATE_CIS,
sizeof(cmd.cp) + sizeof(cmd.cis[0]) *
cmd.cp.num_cis, &cmd,
HCI_EVT_LE_CIS_ESTABLISHED,
conn->conn_timeout, NULL);
}
int hci_le_remove_cig_sync(struct hci_dev *hdev, u8 handle)
{
struct hci_cp_le_remove_cig cp;
memset(&cp, 0, sizeof(cp));
cp.cig_id = handle;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_REMOVE_CIG, sizeof(cp),
&cp, HCI_CMD_TIMEOUT);
}
int hci_le_big_terminate_sync(struct hci_dev *hdev, u8 handle)
{
struct hci_cp_le_big_term_sync cp;
memset(&cp, 0, sizeof(cp));
cp.handle = handle;
return __hci_cmd_sync_status(hdev, HCI_OP_LE_BIG_TERM_SYNC,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_le_pa_terminate_sync(struct hci_dev *hdev, u16 handle)
{
struct hci_cp_le_pa_term_sync cp;
memset(&cp, 0, sizeof(cp));
cp.handle = cpu_to_le16(handle);
return __hci_cmd_sync_status(hdev, HCI_OP_LE_PA_TERM_SYNC,
sizeof(cp), &cp, HCI_CMD_TIMEOUT);
}
int hci_get_random_address(struct hci_dev *hdev, bool require_privacy,
bool use_rpa, struct adv_info *adv_instance,
u8 *own_addr_type, bdaddr_t *rand_addr)
{
int err;
bacpy(rand_addr, BDADDR_ANY);
/* If privacy is enabled use a resolvable private address. If
* current RPA has expired then generate a new one.
*/
if (use_rpa) {
/* If Controller supports LL Privacy use own address type is
* 0x03
*/
if (use_ll_privacy(hdev))
*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
else
*own_addr_type = ADDR_LE_DEV_RANDOM;
if (adv_instance) {
if (adv_rpa_valid(adv_instance))
return 0;
} else {
if (rpa_valid(hdev))
return 0;
}
err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
if (err < 0) {
bt_dev_err(hdev, "failed to generate new RPA");
return err;
}
bacpy(rand_addr, &hdev->rpa);
return 0;
}
/* In case of required privacy without resolvable private address,
* use an non-resolvable private address. This is useful for
* non-connectable advertising.
*/
if (require_privacy) {
bdaddr_t nrpa;
while (true) {
/* The non-resolvable private address is generated
* from random six bytes with the two most significant
* bits cleared.
*/
get_random_bytes(&nrpa, 6);
nrpa.b[5] &= 0x3f;
/* The non-resolvable private address shall not be
* equal to the public address.
*/
if (bacmp(&hdev->bdaddr, &nrpa))
break;
}
*own_addr_type = ADDR_LE_DEV_RANDOM;
bacpy(rand_addr, &nrpa);
return 0;
}
/* No privacy so use a public address. */
*own_addr_type = ADDR_LE_DEV_PUBLIC;
return 0;
}
static int _update_adv_data_sync(struct hci_dev *hdev, void *data)
{
u8 instance = PTR_UINT(data);
return hci_update_adv_data_sync(hdev, instance);
}
int hci_update_adv_data(struct hci_dev *hdev, u8 instance)
{
return hci_cmd_sync_queue(hdev, _update_adv_data_sync,
UINT_PTR(instance), NULL);
}
static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data)
{
struct hci_conn *conn = data;
struct inquiry_entry *ie;
struct hci_cp_create_conn cp;
int err;
if (!hci_conn_valid(hdev, conn))
return -ECANCELED;
/* Many controllers disallow HCI Create Connection while it is doing
* HCI Inquiry. So we cancel the Inquiry first before issuing HCI Create
* Connection. This may cause the MGMT discovering state to become false
* without user space's request but it is okay since the MGMT Discovery
* APIs do not promise that discovery should be done forever. Instead,
* the user space monitors the status of MGMT discovering and it may
* request for discovery again when this flag becomes false.
*/
if (test_bit(HCI_INQUIRY, &hdev->flags)) {
err = __hci_cmd_sync_status(hdev, HCI_OP_INQUIRY_CANCEL, 0,
NULL, HCI_CMD_TIMEOUT);
if (err)
bt_dev_warn(hdev, "Failed to cancel inquiry %d", err);
}
conn->state = BT_CONNECT;
conn->out = true;
conn->role = HCI_ROLE_MASTER;
conn->attempt++;
conn->link_policy = hdev->link_policy;
memset(&cp, 0, sizeof(cp));
bacpy(&cp.bdaddr, &conn->dst);
cp.pscan_rep_mode = 0x02;
ie = hci_inquiry_cache_lookup(hdev, &conn->dst);
if (ie) {
if (inquiry_entry_age(ie) <= INQUIRY_ENTRY_AGE_MAX) {
cp.pscan_rep_mode = ie->data.pscan_rep_mode;
cp.pscan_mode = ie->data.pscan_mode;
cp.clock_offset = ie->data.clock_offset |
cpu_to_le16(0x8000);
}
memcpy(conn->dev_class, ie->data.dev_class, 3);
}
cp.pkt_type = cpu_to_le16(conn->pkt_type);
if (lmp_rswitch_capable(hdev) && !(hdev->link_mode & HCI_LM_MASTER))
cp.role_switch = 0x01;
else
cp.role_switch = 0x00;
return __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN,
sizeof(cp), &cp,
HCI_EV_CONN_COMPLETE,
conn->conn_timeout, NULL);
}
int hci_connect_acl_sync(struct hci_dev *hdev, struct hci_conn *conn)
{
return hci_cmd_sync_queue_once(hdev, hci_acl_create_conn_sync, conn,
NULL);
}
static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err)
{
struct hci_conn *conn = data;
bt_dev_dbg(hdev, "err %d", err);
if (err == -ECANCELED)
return;
hci_dev_lock(hdev);
if (!hci_conn_valid(hdev, conn))
goto done;
if (!err) {
hci_connect_le_scan_cleanup(conn, 0x00);
goto done;
}
/* Check if connection is still pending */
if (conn != hci_lookup_le_connect(hdev))
goto done;
/* Flush to make sure we send create conn cancel command if needed */
flush_delayed_work(&conn->le_conn_timeout);
hci_conn_failed(conn, bt_status(err));
done:
hci_dev_unlock(hdev);
}
int hci_connect_le_sync(struct hci_dev *hdev, struct hci_conn *conn)
{
return hci_cmd_sync_queue_once(hdev, hci_le_create_conn_sync, conn,
create_le_conn_complete);
}
int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn)
{
if (conn->state != BT_OPEN)
return -EINVAL;
switch (conn->type) {
case ACL_LINK:
return !hci_cmd_sync_dequeue_once(hdev,
hci_acl_create_conn_sync,
conn, NULL);
case LE_LINK:
return !hci_cmd_sync_dequeue_once(hdev, hci_le_create_conn_sync,
conn, create_le_conn_complete);
}
return -ENOENT;
}
// SPDX-License-Identifier: GPL-2.0
/*
* USB-ACPI glue code
*
* Copyright 2012 Red Hat <mjg@redhat.com>
*/
#include <linux/module.h>
#include <linux/usb.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/acpi.h>
#include <linux/pci.h>
#include <linux/usb/hcd.h>
#include "hub.h"
/**
* usb_acpi_power_manageable - check whether usb port has
* acpi power resource.
* @hdev: USB device belonging to the usb hub
* @index: port index based zero
*
* Return true if the port has acpi power resource and false if no.
*/
bool usb_acpi_power_manageable(struct usb_device *hdev, int index)
{
acpi_handle port_handle;
int port1 = index + 1;
port_handle = usb_get_hub_port_acpi_handle(hdev,
port1);
if (port_handle)
return acpi_bus_power_manageable(port_handle);
else
return false;
}
EXPORT_SYMBOL_GPL(usb_acpi_power_manageable);
#define UUID_USB_CONTROLLER_DSM "ce2ee385-00e6-48cb-9f05-2edb927c4899"
#define USB_DSM_DISABLE_U1_U2_FOR_PORT 5
/**
* usb_acpi_port_lpm_incapable - check if lpm should be disabled for a port.
* @hdev: USB device belonging to the usb hub
* @index: zero based port index
*
* Some USB3 ports may not support USB3 link power management U1/U2 states
* due to different retimer setup. ACPI provides _DSM method which returns 0x01
* if U1 and U2 states should be disabled. Evaluate _DSM with:
* Arg0: UUID = ce2ee385-00e6-48cb-9f05-2edb927c4899
* Arg1: Revision ID = 0
* Arg2: Function Index = 5
* Arg3: (empty)
*
* Return 1 if USB3 port is LPM incapable, negative on error, otherwise 0
*/
int usb_acpi_port_lpm_incapable(struct usb_device *hdev, int index)
{
union acpi_object *obj;
acpi_handle port_handle;
int port1 = index + 1;
guid_t guid;
int ret;
ret = guid_parse(UUID_USB_CONTROLLER_DSM, &guid);
if (ret)
return ret;
port_handle = usb_get_hub_port_acpi_handle(hdev, port1);
if (!port_handle) {
dev_dbg(&hdev->dev, "port-%d no acpi handle\n", port1);
return -ENODEV;
}
if (!acpi_check_dsm(port_handle, &guid, 0,
BIT(USB_DSM_DISABLE_U1_U2_FOR_PORT))) {
dev_dbg(&hdev->dev, "port-%d no _DSM function %d\n",
port1, USB_DSM_DISABLE_U1_U2_FOR_PORT);
return -ENODEV;
}
obj = acpi_evaluate_dsm_typed(port_handle, &guid, 0,
USB_DSM_DISABLE_U1_U2_FOR_PORT, NULL,
ACPI_TYPE_INTEGER);
if (!obj) {
dev_dbg(&hdev->dev, "evaluate port-%d _DSM failed\n", port1);
return -EINVAL;
}
if (obj->integer.value == 0x01)
ret = 1;
ACPI_FREE(obj);
return ret;
}
EXPORT_SYMBOL_GPL(usb_acpi_port_lpm_incapable);
/**
* usb_acpi_set_power_state - control usb port's power via acpi power
* resource
* @hdev: USB device belonging to the usb hub
* @index: port index based zero
* @enable: power state expected to be set
*
* Notice to use usb_acpi_power_manageable() to check whether the usb port
* has acpi power resource before invoking this function.
*
* Returns 0 on success, else negative errno.
*/
int usb_acpi_set_power_state(struct usb_device *hdev, int index, bool enable)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
struct usb_port *port_dev;
acpi_handle port_handle;
unsigned char state;
int port1 = index + 1;
int error = -EINVAL;
if (!hub)
return -ENODEV;
port_dev = hub->ports[port1 - 1];
port_handle = (acpi_handle) usb_get_hub_port_acpi_handle(hdev, port1);
if (!port_handle)
return error;
if (enable)
state = ACPI_STATE_D0;
else
state = ACPI_STATE_D3_COLD;
error = acpi_bus_set_power(port_handle, state);
if (!error)
dev_dbg(&port_dev->dev, "acpi: power was set to %d\n", enable);
else
dev_dbg(&port_dev->dev, "acpi: power failed to be set\n");
return error;
}
EXPORT_SYMBOL_GPL(usb_acpi_set_power_state);
/*
* Private to usb-acpi, all the core needs to know is that
* port_dev->location is non-zero when it has been set by the firmware.
*/
#define USB_ACPI_LOCATION_VALID (1 << 31)
static void
usb_acpi_get_connect_type(struct usb_port *port_dev, acpi_handle *handle)
{
enum usb_port_connect_type connect_type = USB_PORT_CONNECT_TYPE_UNKNOWN;
struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
union acpi_object *upc = NULL;
struct acpi_pld_info *pld = NULL;
acpi_status status;
/*
* According to 9.14 in ACPI Spec 6.2. _PLD indicates whether usb port
* is user visible and _UPC indicates whether it is connectable. If
* the port was visible and connectable, it could be freely connected
* and disconnected with USB devices. If no visible and connectable,
* a usb device is directly hard-wired to the port. If no visible and
* no connectable, the port would be not used.
*/
status = acpi_get_physical_device_location(handle, &pld);
if (ACPI_SUCCESS(status) && pld) port_dev->location = USB_ACPI_LOCATION_VALID |
pld->group_token << 8 | pld->group_position;
status = acpi_evaluate_object(handle, "_UPC", NULL, &buffer);
if (ACPI_FAILURE(status))
goto out;
upc = buffer.pointer; if (!upc || (upc->type != ACPI_TYPE_PACKAGE) || upc->package.count != 4)
goto out;
/* UPC states port is connectable */
if (upc->package.elements[0].integer.value) if (!pld)
; /* keep connect_type as unknown */
else if (pld->user_visible)
connect_type = USB_PORT_CONNECT_TYPE_HOT_PLUG;
else
connect_type = USB_PORT_CONNECT_TYPE_HARD_WIRED;
else
connect_type = USB_PORT_NOT_USED;
out:
port_dev->connect_type = connect_type;
kfree(upc);
ACPI_FREE(pld);
}
static struct acpi_device *
usb_acpi_get_companion_for_port(struct usb_port *port_dev)
{
struct usb_device *udev;
struct acpi_device *adev;
acpi_handle *parent_handle;
int port1;
/* Get the struct usb_device point of port's hub */
udev = to_usb_device(port_dev->dev.parent->parent);
/*
* The root hub ports' parent is the root hub. The non-root-hub
* ports' parent is the parent hub port which the hub is
* connected to.
*/
if (!udev->parent) {
adev = ACPI_COMPANION(&udev->dev);
port1 = usb_hcd_find_raw_port_number(bus_to_hcd(udev->bus),
port_dev->portnum);
} else {
parent_handle = usb_get_hub_port_acpi_handle(udev->parent,
udev->portnum);
if (!parent_handle)
return NULL;
adev = acpi_fetch_acpi_dev(parent_handle);
port1 = port_dev->portnum;
}
return acpi_find_child_by_adr(adev, port1);
}
static struct acpi_device *
usb_acpi_find_companion_for_port(struct usb_port *port_dev)
{
struct acpi_device *adev;
adev = usb_acpi_get_companion_for_port(port_dev);
if (!adev)
return NULL;
usb_acpi_get_connect_type(port_dev, adev->handle);
return adev;
}
static struct acpi_device *
usb_acpi_find_companion_for_device(struct usb_device *udev)
{
struct acpi_device *adev;
struct usb_port *port_dev;
struct usb_hub *hub;
if (!udev->parent) {
/*
* root hub is only child (_ADR=0) under its parent, the HC.
* sysdev pointer is the HC as seen from firmware.
*/
adev = ACPI_COMPANION(udev->bus->sysdev); return acpi_find_child_device(adev, 0, false);
}
hub = usb_hub_to_struct_hub(udev->parent);
if (!hub)
return NULL;
/*
* This is an embedded USB device connected to a port and such
* devices share port's ACPI companion.
*/
port_dev = hub->ports[udev->portnum - 1];
return usb_acpi_get_companion_for_port(port_dev);
}
static struct acpi_device *usb_acpi_find_companion(struct device *dev)
{
/*
* The USB hierarchy like following:
*
* Device (EHC1)
* Device (HUBN)
* Device (PR01)
* Device (PR11)
* Device (PR12)
* Device (FN12)
* Device (FN13)
* Device (PR13)
* ...
* where HUBN is root hub, and PRNN are USB ports and devices
* connected to them, and FNNN are individualk functions for
* connected composite USB devices. PRNN and FNNN may contain
* _CRS and other methods describing sideband resources for
* the connected device.
*
* On the kernel side both root hub and embedded USB devices are
* represented as instances of usb_device structure, and ports
* are represented as usb_port structures, so the whole process
* is split into 2 parts: finding companions for devices and
* finding companions for ports.
*
* Note that we do not handle individual functions of composite
* devices yet, for that we would need to assign companions to
* devices corresponding to USB interfaces.
*/
if (is_usb_device(dev)) return usb_acpi_find_companion_for_device(to_usb_device(dev)); else if (is_usb_port(dev)) return usb_acpi_find_companion_for_port(to_usb_port(dev));
return NULL;
}
static bool usb_acpi_bus_match(struct device *dev)
{
return is_usb_device(dev) || is_usb_port(dev);
}
static struct acpi_bus_type usb_acpi_bus = {
.name = "USB",
.match = usb_acpi_bus_match,
.find_companion = usb_acpi_find_companion,
};
int usb_acpi_register(void)
{
return register_acpi_bus_type(&usb_acpi_bus);
}
void usb_acpi_unregister(void)
{
unregister_acpi_bus_type(&usb_acpi_bus);
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* xsave/xrstor support.
*
* Author: Suresh Siddha <suresh.b.siddha@intel.com>
*/
#include <linux/bitops.h>
#include <linux/compat.h>
#include <linux/cpu.h>
#include <linux/mman.h>
#include <linux/nospec.h>
#include <linux/pkeys.h>
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <linux/vmalloc.h>
#include <asm/fpu/api.h>
#include <asm/fpu/regset.h>
#include <asm/fpu/signal.h>
#include <asm/fpu/xcr.h>
#include <asm/tlbflush.h>
#include <asm/prctl.h>
#include <asm/elf.h>
#include "context.h"
#include "internal.h"
#include "legacy.h"
#include "xstate.h"
#define for_each_extended_xfeature(bit, mask) \
(bit) = FIRST_EXTENDED_XFEATURE; \
for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask))
/*
* Although we spell it out in here, the Processor Trace
* xfeature is completely unused. We use other mechanisms
* to save/restore PT state in Linux.
*/
static const char *xfeature_names[] =
{
"x87 floating point registers",
"SSE registers",
"AVX registers",
"MPX bounds registers",
"MPX CSR",
"AVX-512 opmask",
"AVX-512 Hi256",
"AVX-512 ZMM_Hi256",
"Processor Trace (unused)",
"Protection Keys User registers",
"PASID state",
"Control-flow User registers",
"Control-flow Kernel registers (unused)",
"unknown xstate feature",
"unknown xstate feature",
"unknown xstate feature",
"unknown xstate feature",
"AMX Tile config",
"AMX Tile data",
"unknown xstate feature",
};
static unsigned short xsave_cpuid_features[] __initdata = {
[XFEATURE_FP] = X86_FEATURE_FPU,
[XFEATURE_SSE] = X86_FEATURE_XMM,
[XFEATURE_YMM] = X86_FEATURE_AVX,
[XFEATURE_BNDREGS] = X86_FEATURE_MPX,
[XFEATURE_BNDCSR] = X86_FEATURE_MPX,
[XFEATURE_OPMASK] = X86_FEATURE_AVX512F,
[XFEATURE_ZMM_Hi256] = X86_FEATURE_AVX512F,
[XFEATURE_Hi16_ZMM] = X86_FEATURE_AVX512F,
[XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT,
[XFEATURE_PKRU] = X86_FEATURE_OSPKE,
[XFEATURE_PASID] = X86_FEATURE_ENQCMD,
[XFEATURE_CET_USER] = X86_FEATURE_SHSTK,
[XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE,
[XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE,
};
static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init =
{ [ 0 ... XFEATURE_MAX - 1] = -1};
static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init =
{ [ 0 ... XFEATURE_MAX - 1] = -1};
static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init;
#define XSTATE_FLAG_SUPERVISOR BIT(0)
#define XSTATE_FLAG_ALIGNED64 BIT(1)
/*
* Return whether the system supports a given xfeature.
*
* Also return the name of the (most advanced) feature that the caller requested:
*/
int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
{
u64 xfeatures_missing = xfeatures_needed & ~fpu_kernel_cfg.max_features;
if (unlikely(feature_name)) {
long xfeature_idx, max_idx;
u64 xfeatures_print;
/*
* So we use FLS here to be able to print the most advanced
* feature that was requested but is missing. So if a driver
* asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
* missing AVX feature - this is the most informative message
* to users:
*/
if (xfeatures_missing)
xfeatures_print = xfeatures_missing;
else
xfeatures_print = xfeatures_needed;
xfeature_idx = fls64(xfeatures_print)-1;
max_idx = ARRAY_SIZE(xfeature_names)-1;
xfeature_idx = min(xfeature_idx, max_idx);
*feature_name = xfeature_names[xfeature_idx];
}
if (xfeatures_missing)
return 0;
return 1;
}
EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
static bool xfeature_is_aligned64(int xfeature_nr)
{
return xstate_flags[xfeature_nr] & XSTATE_FLAG_ALIGNED64;
}
static bool xfeature_is_supervisor(int xfeature_nr)
{
return xstate_flags[xfeature_nr] & XSTATE_FLAG_SUPERVISOR;
}
static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature)
{
unsigned int offs, i;
/*
* Non-compacted format and legacy features use the cached fixed
* offsets.
*/
if (!cpu_feature_enabled(X86_FEATURE_XCOMPACTED) ||
xfeature <= XFEATURE_SSE)
return xstate_offsets[xfeature];
/*
* Compacted format offsets depend on the actual content of the
* compacted xsave area which is determined by the xcomp_bv header
* field.
*/
offs = FXSAVE_SIZE + XSAVE_HDR_SIZE;
for_each_extended_xfeature(i, xcomp_bv) {
if (xfeature_is_aligned64(i))
offs = ALIGN(offs, 64);
if (i == xfeature)
break;
offs += xstate_sizes[i];
}
return offs;
}
/*
* Enable the extended processor state save/restore feature.
* Called once per CPU onlining.
*/
void fpu__init_cpu_xstate(void)
{
if (!boot_cpu_has(X86_FEATURE_XSAVE) || !fpu_kernel_cfg.max_features)
return;
cr4_set_bits(X86_CR4_OSXSAVE);
/*
* Must happen after CR4 setup and before xsetbv() to allow KVM
* lazy passthrough. Write independent of the dynamic state static
* key as that does not work on the boot CPU. This also ensures
* that any stale state is wiped out from XFD. Reset the per CPU
* xfd cache too.
*/
if (cpu_feature_enabled(X86_FEATURE_XFD))
xfd_set_state(init_fpstate.xfd);
/*
* XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
* managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user
* states can be set here.
*/
xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features);
/*
* MSR_IA32_XSS sets supervisor states managed by XSAVES.
*/
if (boot_cpu_has(X86_FEATURE_XSAVES)) {
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
xfeatures_mask_independent());
}
}
static bool xfeature_enabled(enum xfeature xfeature)
{
return fpu_kernel_cfg.max_features & BIT_ULL(xfeature);
}
/*
* Record the offsets and sizes of various xstates contained
* in the XSAVE state memory layout.
*/
static void __init setup_xstate_cache(void)
{
u32 eax, ebx, ecx, edx, i;
/* start at the beginning of the "extended state" */
unsigned int last_good_offset = offsetof(struct xregs_state,
extended_state_area);
/*
* The FP xstates and SSE xstates are legacy states. They are always
* in the fixed offsets in the xsave area in either compacted form
* or standard form.
*/
xstate_offsets[XFEATURE_FP] = 0;
xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state,
xmm_space);
xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP];
xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state,
xmm_space);
for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {
cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
xstate_sizes[i] = eax;
xstate_flags[i] = ecx;
/*
* If an xfeature is supervisor state, the offset in EBX is
* invalid, leave it to -1.
*/
if (xfeature_is_supervisor(i))
continue;
xstate_offsets[i] = ebx;
/*
* In our xstate size checks, we assume that the highest-numbered
* xstate feature has the highest offset in the buffer. Ensure
* it does.
*/
WARN_ONCE(last_good_offset > xstate_offsets[i],
"x86/fpu: misordered xstate at %d\n", last_good_offset);
last_good_offset = xstate_offsets[i];
}
}
static void __init print_xstate_feature(u64 xstate_mask)
{
const char *feature_name;
if (cpu_has_xfeatures(xstate_mask, &feature_name))
pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
}
/*
* Print out all the supported xstate features:
*/
static void __init print_xstate_features(void)
{
print_xstate_feature(XFEATURE_MASK_FP);
print_xstate_feature(XFEATURE_MASK_SSE);
print_xstate_feature(XFEATURE_MASK_YMM);
print_xstate_feature(XFEATURE_MASK_BNDREGS);
print_xstate_feature(XFEATURE_MASK_BNDCSR);
print_xstate_feature(XFEATURE_MASK_OPMASK);
print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
print_xstate_feature(XFEATURE_MASK_PKRU);
print_xstate_feature(XFEATURE_MASK_PASID);
print_xstate_feature(XFEATURE_MASK_CET_USER);
print_xstate_feature(XFEATURE_MASK_XTILE_CFG);
print_xstate_feature(XFEATURE_MASK_XTILE_DATA);
}
/*
* This check is important because it is easy to get XSTATE_*
* confused with XSTATE_BIT_*.
*/
#define CHECK_XFEATURE(nr) do { \
WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \
WARN_ON(nr >= XFEATURE_MAX); \
} while (0)
/*
* Print out xstate component offsets and sizes
*/
static void __init print_xstate_offset_size(void)
{
int i;
for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {
pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
i, xfeature_get_offset(fpu_kernel_cfg.max_features, i),
i, xstate_sizes[i]);
}
}
/*
* This function is called only during boot time when x86 caps are not set
* up and alternative can not be used yet.
*/
static __init void os_xrstor_booting(struct xregs_state *xstate)
{
u64 mask = fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSTATE;
u32 lmask = mask;
u32 hmask = mask >> 32;
int err;
if (cpu_feature_enabled(X86_FEATURE_XSAVES))
XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
else
XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
/*
* We should never fault when copying from a kernel buffer, and the FPU
* state we set at boot time should be valid.
*/
WARN_ON_FPU(err);
}
/*
* All supported features have either init state all zeros or are
* handled in setup_init_fpu() individually. This is an explicit
* feature list and does not use XFEATURE_MASK*SUPPORTED to catch
* newly added supported features at build time and make people
* actually look at the init state for the new feature.
*/
#define XFEATURES_INIT_FPSTATE_HANDLED \
(XFEATURE_MASK_FP | \
XFEATURE_MASK_SSE | \
XFEATURE_MASK_YMM | \
XFEATURE_MASK_OPMASK | \
XFEATURE_MASK_ZMM_Hi256 | \
XFEATURE_MASK_Hi16_ZMM | \
XFEATURE_MASK_PKRU | \
XFEATURE_MASK_BNDREGS | \
XFEATURE_MASK_BNDCSR | \
XFEATURE_MASK_PASID | \
XFEATURE_MASK_CET_USER | \
XFEATURE_MASK_XTILE)
/*
* setup the xstate image representing the init state
*/
static void __init setup_init_fpu_buf(void)
{
BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
XFEATURES_INIT_FPSTATE_HANDLED);
if (!boot_cpu_has(X86_FEATURE_XSAVE))
return;
print_xstate_features();
xstate_init_xcomp_bv(&init_fpstate.regs.xsave, init_fpstate.xfeatures);
/*
* Init all the features state with header.xfeatures being 0x0
*/
os_xrstor_booting(&init_fpstate.regs.xsave);
/*
* All components are now in init state. Read the state back so
* that init_fpstate contains all non-zero init state. This only
* works with XSAVE, but not with XSAVEOPT and XSAVEC/S because
* those use the init optimization which skips writing data for
* components in init state.
*
* XSAVE could be used, but that would require to reshuffle the
* data when XSAVEC/S is available because XSAVEC/S uses xstate
* compaction. But doing so is a pointless exercise because most
* components have an all zeros init state except for the legacy
* ones (FP and SSE). Those can be saved with FXSAVE into the
* legacy area. Adding new features requires to ensure that init
* state is all zeroes or if not to add the necessary handling
* here.
*/
fxsave(&init_fpstate.regs.fxsave);
}
int xfeature_size(int xfeature_nr)
{
u32 eax, ebx, ecx, edx;
CHECK_XFEATURE(xfeature_nr);
cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
return eax;
}
/* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
static int validate_user_xstate_header(const struct xstate_header *hdr,
struct fpstate *fpstate)
{
/* No unknown or supervisor features may be set */
if (hdr->xfeatures & ~fpstate->user_xfeatures)
return -EINVAL;
/* Userspace must use the uncompacted format */
if (hdr->xcomp_bv)
return -EINVAL;
/*
* If 'reserved' is shrunken to add a new field, make sure to validate
* that new field here!
*/
BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
/* No reserved bits may be set */
if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
return -EINVAL;
return 0;
}
static void __init __xstate_dump_leaves(void)
{
int i;
u32 eax, ebx, ecx, edx;
static int should_dump = 1;
if (!should_dump)
return;
should_dump = 0;
/*
* Dump out a few leaves past the ones that we support
* just in case there are some goodies up there
*/
for (i = 0; i < XFEATURE_MAX + 10; i++) {
cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
XSTATE_CPUID, i, eax, ebx, ecx, edx);
}
}
#define XSTATE_WARN_ON(x, fmt, ...) do { \
if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) { \
__xstate_dump_leaves(); \
} \
} while (0)
#define XCHECK_SZ(sz, nr, __struct) ({ \
if (WARN_ONCE(sz != sizeof(__struct), \
"[%s]: struct is %zu bytes, cpu state %d bytes\n", \
xfeature_names[nr], sizeof(__struct), sz)) { \
__xstate_dump_leaves(); \
} \
true; \
})
/**
* check_xtile_data_against_struct - Check tile data state size.
*
* Calculate the state size by multiplying the single tile size which is
* recorded in a C struct, and the number of tiles that the CPU informs.
* Compare the provided size with the calculation.
*
* @size: The tile data state size
*
* Returns: 0 on success, -EINVAL on mismatch.
*/
static int __init check_xtile_data_against_struct(int size)
{
u32 max_palid, palid, state_size;
u32 eax, ebx, ecx, edx;
u16 max_tile;
/*
* Check the maximum palette id:
* eax: the highest numbered palette subleaf.
*/
cpuid_count(TILE_CPUID, 0, &max_palid, &ebx, &ecx, &edx);
/*
* Cross-check each tile size and find the maximum number of
* supported tiles.
*/
for (palid = 1, max_tile = 0; palid <= max_palid; palid++) {
u16 tile_size, max;
/*
* Check the tile size info:
* eax[31:16]: bytes per title
* ebx[31:16]: the max names (or max number of tiles)
*/
cpuid_count(TILE_CPUID, palid, &eax, &ebx, &edx, &edx);
tile_size = eax >> 16;
max = ebx >> 16;
if (tile_size != sizeof(struct xtile_data)) {
pr_err("%s: struct is %zu bytes, cpu xtile %d bytes\n",
__stringify(XFEATURE_XTILE_DATA),
sizeof(struct xtile_data), tile_size);
__xstate_dump_leaves();
return -EINVAL;
}
if (max > max_tile)
max_tile = max;
}
state_size = sizeof(struct xtile_data) * max_tile;
if (size != state_size) {
pr_err("%s: calculated size is %u bytes, cpu state %d bytes\n",
__stringify(XFEATURE_XTILE_DATA), state_size, size);
__xstate_dump_leaves();
return -EINVAL;
}
return 0;
}
/*
* We have a C struct for each 'xstate'. We need to ensure
* that our software representation matches what the CPU
* tells us about the state's size.
*/
static bool __init check_xstate_against_struct(int nr)
{
/*
* Ask the CPU for the size of the state.
*/
int sz = xfeature_size(nr);
/*
* Match each CPU state with the corresponding software
* structure.
*/
switch (nr) {
case XFEATURE_YMM: return XCHECK_SZ(sz, nr, struct ymmh_struct);
case XFEATURE_BNDREGS: return XCHECK_SZ(sz, nr, struct mpx_bndreg_state);
case XFEATURE_BNDCSR: return XCHECK_SZ(sz, nr, struct mpx_bndcsr_state);
case XFEATURE_OPMASK: return XCHECK_SZ(sz, nr, struct avx_512_opmask_state);
case XFEATURE_ZMM_Hi256: return XCHECK_SZ(sz, nr, struct avx_512_zmm_uppers_state);
case XFEATURE_Hi16_ZMM: return XCHECK_SZ(sz, nr, struct avx_512_hi16_state);
case XFEATURE_PKRU: return XCHECK_SZ(sz, nr, struct pkru_state);
case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state);
case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg);
case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state);
case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true;
default:
XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
return false;
}
return true;
}
static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted)
{
unsigned int topmost = fls64(xfeatures) - 1;
unsigned int offset = xstate_offsets[topmost];
if (topmost <= XFEATURE_SSE)
return sizeof(struct xregs_state);
if (compacted)
offset = xfeature_get_offset(xfeatures, topmost);
return offset + xstate_sizes[topmost];
}
/*
* This essentially double-checks what the cpu told us about
* how large the XSAVE buffer needs to be. We are recalculating
* it to be safe.
*
* Independent XSAVE features allocate their own buffers and are not
* covered by these checks. Only the size of the buffer for task->fpu
* is checked here.
*/
static bool __init paranoid_xstate_size_valid(unsigned int kernel_size)
{
bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
bool xsaves = cpu_feature_enabled(X86_FEATURE_XSAVES);
unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
int i;
for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) {
if (!check_xstate_against_struct(i))
return false;
/*
* Supervisor state components can be managed only by
* XSAVES.
*/
if (!xsaves && xfeature_is_supervisor(i)) {
XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i);
return false;
}
}
size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted);
XSTATE_WARN_ON(size != kernel_size,
"size %u != kernel_size %u\n", size, kernel_size);
return size == kernel_size;
}
/*
* Get total size of enabled xstates in XCR0 | IA32_XSS.
*
* Note the SDM's wording here. "sub-function 0" only enumerates
* the size of the *user* states. If we use it to size a buffer
* that we use 'XSAVES' on, we could potentially overflow the
* buffer because 'XSAVES' saves system states too.
*
* This also takes compaction into account. So this works for
* XSAVEC as well.
*/
static unsigned int __init get_compacted_size(void)
{
unsigned int eax, ebx, ecx, edx;
/*
* - CPUID function 0DH, sub-function 1:
* EBX enumerates the size (in bytes) required by
* the XSAVES instruction for an XSAVE area
* containing all the state components
* corresponding to bits currently set in
* XCR0 | IA32_XSS.
*
* When XSAVES is not available but XSAVEC is (virt), then there
* are no supervisor states, but XSAVEC still uses compacted
* format.
*/
cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
return ebx;
}
/*
* Get the total size of the enabled xstates without the independent supervisor
* features.
*/
static unsigned int __init get_xsave_compacted_size(void)
{
u64 mask = xfeatures_mask_independent();
unsigned int size;
if (!mask)
return get_compacted_size();
/* Disable independent features. */
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
/*
* Ask the hardware what size is required of the buffer.
* This is the size required for the task->fpu buffer.
*/
size = get_compacted_size();
/* Re-enable independent features so XSAVES will work on them again. */
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
return size;
}
static unsigned int __init get_xsave_size_user(void)
{
unsigned int eax, ebx, ecx, edx;
/*
* - CPUID function 0DH, sub-function 0:
* EBX enumerates the size (in bytes) required by
* the XSAVE instruction for an XSAVE area
* containing all the *user* state components
* corresponding to bits currently set in XCR0.
*/
cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
return ebx;
}
static int __init init_xstate_size(void)
{
/* Recompute the context size for enabled features: */
unsigned int user_size, kernel_size, kernel_default_size;
bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
/* Uncompacted user space size */
user_size = get_xsave_size_user();
/*
* XSAVES kernel size includes supervisor states and uses compacted
* format. XSAVEC uses compacted format, but does not save
* supervisor states.
*
* XSAVE[OPT] do not support supervisor states so kernel and user
* size is identical.
*/
if (compacted)
kernel_size = get_xsave_compacted_size();
else
kernel_size = user_size;
kernel_default_size =
xstate_calculate_size(fpu_kernel_cfg.default_features, compacted);
if (!paranoid_xstate_size_valid(kernel_size))
return -EINVAL;
fpu_kernel_cfg.max_size = kernel_size;
fpu_user_cfg.max_size = user_size;
fpu_kernel_cfg.default_size = kernel_default_size;
fpu_user_cfg.default_size =
xstate_calculate_size(fpu_user_cfg.default_features, false);
return 0;
}
/*
* We enabled the XSAVE hardware, but something went wrong and
* we can not use it. Disable it.
*/
static void __init fpu__init_disable_system_xstate(unsigned int legacy_size)
{
fpu_kernel_cfg.max_features = 0;
cr4_clear_bits(X86_CR4_OSXSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
/* Restore the legacy size.*/
fpu_kernel_cfg.max_size = legacy_size;
fpu_kernel_cfg.default_size = legacy_size;
fpu_user_cfg.max_size = legacy_size;
fpu_user_cfg.default_size = legacy_size;
/*
* Prevent enabling the static branch which enables writes to the
* XFD MSR.
*/
init_fpstate.xfd = 0;
fpstate_reset(¤t->thread.fpu);
}
/*
* Enable and initialize the xsave feature.
* Called once per system bootup.
*/
void __init fpu__init_system_xstate(unsigned int legacy_size)
{
unsigned int eax, ebx, ecx, edx;
u64 xfeatures;
int err;
int i;
if (!boot_cpu_has(X86_FEATURE_FPU)) {
pr_info("x86/fpu: No FPU detected\n");
return;
}
if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
pr_info("x86/fpu: x87 FPU will use %s\n",
boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
return;
}
if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
WARN_ON_FPU(1);
return;
}
/*
* Find user xstates supported by the processor.
*/
cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
fpu_kernel_cfg.max_features = eax + ((u64)edx << 32);
/*
* Find supervisor xstates supported by the processor.
*/
cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
fpu_kernel_cfg.max_features |= ecx + ((u64)edx << 32);
if ((fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
/*
* This indicates that something really unexpected happened
* with the enumeration. Disable XSAVE and try to continue
* booting without it. This is too early to BUG().
*/
pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
fpu_kernel_cfg.max_features);
goto out_disable;
}
/*
* Clear XSAVE features that are disabled in the normal CPUID.
*/
for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
unsigned short cid = xsave_cpuid_features[i];
/* Careful: X86_FEATURE_FPU is 0! */
if ((i != XFEATURE_FP && !cid) || !boot_cpu_has(cid))
fpu_kernel_cfg.max_features &= ~BIT_ULL(i);
}
if (!cpu_feature_enabled(X86_FEATURE_XFD))
fpu_kernel_cfg.max_features &= ~XFEATURE_MASK_USER_DYNAMIC;
if (!cpu_feature_enabled(X86_FEATURE_XSAVES))
fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED;
else
fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED |
XFEATURE_MASK_SUPERVISOR_SUPPORTED;
fpu_user_cfg.max_features = fpu_kernel_cfg.max_features;
fpu_user_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED;
/* Clean out dynamic features from default */
fpu_kernel_cfg.default_features = fpu_kernel_cfg.max_features;
fpu_kernel_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC;
fpu_user_cfg.default_features = fpu_user_cfg.max_features;
fpu_user_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC;
/* Store it for paranoia check at the end */
xfeatures = fpu_kernel_cfg.max_features;
/*
* Initialize the default XFD state in initfp_state and enable the
* dynamic sizing mechanism if dynamic states are available. The
* static key cannot be enabled here because this runs before
* jump_label_init(). This is delayed to an initcall.
*/
init_fpstate.xfd = fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC;
/* Set up compaction feature bit */
if (cpu_feature_enabled(X86_FEATURE_XSAVEC) ||
cpu_feature_enabled(X86_FEATURE_XSAVES))
setup_force_cpu_cap(X86_FEATURE_XCOMPACTED);
/* Enable xstate instructions to be able to continue with initialization: */
fpu__init_cpu_xstate();
/* Cache size, offset and flags for initialization */
setup_xstate_cache();
err = init_xstate_size();
if (err)
goto out_disable;
/* Reset the state for the current task */
fpstate_reset(¤t->thread.fpu);
/*
* Update info used for ptrace frames; use standard-format size and no
* supervisor xstates:
*/
update_regset_xstate_info(fpu_user_cfg.max_size,
fpu_user_cfg.max_features);
/*
* init_fpstate excludes dynamic states as they are large but init
* state is zero.
*/
init_fpstate.size = fpu_kernel_cfg.default_size;
init_fpstate.xfeatures = fpu_kernel_cfg.default_features;
if (init_fpstate.size > sizeof(init_fpstate.regs)) {
pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d), disabling XSAVE\n",
sizeof(init_fpstate.regs), init_fpstate.size);
goto out_disable;
}
setup_init_fpu_buf();
/*
* Paranoia check whether something in the setup modified the
* xfeatures mask.
*/
if (xfeatures != fpu_kernel_cfg.max_features) {
pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n",
xfeatures, fpu_kernel_cfg.max_features);
goto out_disable;
}
/*
* CPU capabilities initialization runs before FPU init. So
* X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely
* functional, set the feature bit so depending code works.
*/
setup_force_cpu_cap(X86_FEATURE_OSXSAVE);
print_xstate_offset_size();
pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
fpu_kernel_cfg.max_features,
fpu_kernel_cfg.max_size,
boot_cpu_has(X86_FEATURE_XCOMPACTED) ? "compacted" : "standard");
return;
out_disable:
/* something went wrong, try to boot without any XSAVE support */
fpu__init_disable_system_xstate(legacy_size);
}
/*
* Restore minimal FPU state after suspend:
*/
void fpu__resume_cpu(void)
{
/*
* Restore XCR0 on xsave capable CPUs:
*/
if (cpu_feature_enabled(X86_FEATURE_XSAVE))
xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features);
/*
* Restore IA32_XSS. The same CPUID bit enumerates support
* of XSAVES and MSR_IA32_XSS.
*/
if (cpu_feature_enabled(X86_FEATURE_XSAVES)) {
wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
xfeatures_mask_independent());
}
if (fpu_state_size_dynamic())
wrmsrl(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd);
}
/*
* Given an xstate feature nr, calculate where in the xsave
* buffer the state is. Callers should ensure that the buffer
* is valid.
*/
static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
{
u64 xcomp_bv = xsave->header.xcomp_bv;
if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))
return NULL;
if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED)) {
if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr))))
return NULL;
}
return (void *)xsave + xfeature_get_offset(xcomp_bv, xfeature_nr);
}
/*
* Given the xsave area and a state inside, this function returns the
* address of the state.
*
* This is the API that is called to get xstate address in either
* standard format or compacted format of xsave area.
*
* Note that if there is no data for the field in the xsave buffer
* this will return NULL.
*
* Inputs:
* xstate: the thread's storage area for all FPU data
* xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
* XFEATURE_SSE, etc...)
* Output:
* address of the state in the xsave area, or NULL if the
* field is not present in the xsave buffer.
*/
void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
{
/*
* Do we even *have* xsave state?
*/
if (!boot_cpu_has(X86_FEATURE_XSAVE))
return NULL;
/*
* We should not ever be requesting features that we
* have not enabled.
*/
if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr)))
return NULL;
/*
* This assumes the last 'xsave*' instruction to
* have requested that 'xfeature_nr' be saved.
* If it did not, we might be seeing and old value
* of the field in the buffer.
*
* This can happen because the last 'xsave' did not
* request that this feature be saved (unlikely)
* or because the "init optimization" caused it
* to not be saved.
*/
if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
return NULL;
return __raw_xsave_addr(xsave, xfeature_nr);
}
#ifdef CONFIG_ARCH_HAS_PKEYS
/*
* This will go out and modify PKRU register to set the access
* rights for @pkey to @init_val.
*/
int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
unsigned long init_val)
{
u32 old_pkru, new_pkru_bits = 0;
int pkey_shift;
/*
* This check implies XSAVE support. OSPKE only gets
* set if we enable XSAVE and we enable PKU in XCR0.
*/
if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
return -EINVAL;
/*
* This code should only be called with valid 'pkey'
* values originating from in-kernel users. Complain
* if a bad value is observed.
*/
if (WARN_ON_ONCE(pkey >= arch_max_pkey()))
return -EINVAL;
/* Set the bits we need in PKRU: */
if (init_val & PKEY_DISABLE_ACCESS)
new_pkru_bits |= PKRU_AD_BIT;
if (init_val & PKEY_DISABLE_WRITE)
new_pkru_bits |= PKRU_WD_BIT;
/* Shift the bits in to the correct place in PKRU for pkey: */
pkey_shift = pkey * PKRU_BITS_PER_PKEY;
new_pkru_bits <<= pkey_shift;
/* Get old PKRU and mask off any old bits in place: */
old_pkru = read_pkru();
old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
/* Write old part along with new part: */
write_pkru(old_pkru | new_pkru_bits);
return 0;
}
#endif /* ! CONFIG_ARCH_HAS_PKEYS */
static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
void *init_xstate, unsigned int size)
{
membuf_write(to, from_xstate ? xstate : init_xstate, size);
}
/**
* __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
* @to: membuf descriptor
* @fpstate: The fpstate buffer from which to copy
* @xfeatures: The mask of xfeatures to save (XSAVE mode only)
* @pkru_val: The PKRU value to store in the PKRU component
* @copy_mode: The requested copy mode
*
* Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
* format, i.e. from the kernel internal hardware dependent storage format
* to the requested @mode. UABI XSTATE is always uncompacted!
*
* It supports partial copy but @to.pos always starts from zero.
*/
void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate,
u64 xfeatures, u32 pkru_val,
enum xstate_copy_mode copy_mode)
{
const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
struct xregs_state *xinit = &init_fpstate.regs.xsave;
struct xregs_state *xsave = &fpstate->regs.xsave;
struct xstate_header header;
unsigned int zerofrom;
u64 mask;
int i;
memset(&header, 0, sizeof(header));
header.xfeatures = xsave->header.xfeatures;
/* Mask out the feature bits depending on copy mode */
switch (copy_mode) {
case XSTATE_COPY_FP:
header.xfeatures &= XFEATURE_MASK_FP;
break;
case XSTATE_COPY_FX:
header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE;
break;
case XSTATE_COPY_XSAVE:
header.xfeatures &= fpstate->user_xfeatures & xfeatures;
break;
}
/* Copy FP state up to MXCSR */
copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
&xinit->i387, off_mxcsr);
/* Copy MXCSR when SSE or YMM are set in the feature mask */
copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
&to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
MXCSR_AND_FLAGS_SIZE);
/* Copy the remaining FP state */
copy_feature(header.xfeatures & XFEATURE_MASK_FP,
&to, &xsave->i387.st_space, &xinit->i387.st_space,
sizeof(xsave->i387.st_space));
/* Copy the SSE state - shared with YMM, but independently managed */
copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
&to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
sizeof(xsave->i387.xmm_space));
if (copy_mode != XSTATE_COPY_XSAVE)
goto out;
/* Zero the padding area */
membuf_zero(&to, sizeof(xsave->i387.padding));
/* Copy xsave->i387.sw_reserved */
membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
/* Copy the user space relevant state of @xsave->header */
membuf_write(&to, &header, sizeof(header));
zerofrom = offsetof(struct xregs_state, extended_state_area);
/*
* This 'mask' indicates which states to copy from fpstate.
* Those extended states that are not present in fpstate are
* either disabled or initialized:
*
* In non-compacted format, disabled features still occupy
* state space but there is no state to copy from in the
* compacted init_fpstate. The gap tracking will zero these
* states.
*
* The extended features have an all zeroes init state. Thus,
* remove them from 'mask' to zero those features in the user
* buffer instead of retrieving them from init_fpstate.
*/
mask = header.xfeatures;
for_each_extended_xfeature(i, mask) {
/*
* If there was a feature or alignment gap, zero the space
* in the destination buffer.
*/
if (zerofrom < xstate_offsets[i])
membuf_zero(&to, xstate_offsets[i] - zerofrom);
if (i == XFEATURE_PKRU) {
struct pkru_state pkru = {0};
/*
* PKRU is not necessarily up to date in the
* XSAVE buffer. Use the provided value.
*/
pkru.pkru = pkru_val;
membuf_write(&to, &pkru, sizeof(pkru));
} else {
membuf_write(&to,
__raw_xsave_addr(xsave, i),
xstate_sizes[i]);
}
/*
* Keep track of the last copied state in the non-compacted
* target buffer for gap zeroing.
*/
zerofrom = xstate_offsets[i] + xstate_sizes[i];
}
out:
if (to.left)
membuf_zero(&to, to.left);
}
/**
* copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
* @to: membuf descriptor
* @tsk: The task from which to copy the saved xstate
* @copy_mode: The requested copy mode
*
* Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
* format, i.e. from the kernel internal hardware dependent storage format
* to the requested @mode. UABI XSTATE is always uncompacted!
*
* It supports partial copy but @to.pos always starts from zero.
*/
void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,
enum xstate_copy_mode copy_mode)
{
__copy_xstate_to_uabi_buf(to, tsk->thread.fpu.fpstate,
tsk->thread.fpu.fpstate->user_xfeatures,
tsk->thread.pkru, copy_mode);
}
static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size,
const void *kbuf, const void __user *ubuf)
{
if (kbuf) {
memcpy(dst, kbuf + offset, size);
} else {
if (copy_from_user(dst, ubuf + offset, size))
return -EFAULT;
}
return 0;
}
/**
* copy_uabi_to_xstate - Copy a UABI format buffer to the kernel xstate
* @fpstate: The fpstate buffer to copy to
* @kbuf: The UABI format buffer, if it comes from the kernel
* @ubuf: The UABI format buffer, if it comes from userspace
* @pkru: The location to write the PKRU value to
*
* Converts from the UABI format into the kernel internal hardware
* dependent format.
*
* This function ultimately has three different callers with distinct PKRU
* behavior.
* 1. When called from sigreturn the PKRU register will be restored from
* @fpstate via an XRSTOR. Correctly copying the UABI format buffer to
* @fpstate is sufficient to cover this case, but the caller will also
* pass a pointer to the thread_struct's pkru field in @pkru and updating
* it is harmless.
* 2. When called from ptrace the PKRU register will be restored from the
* thread_struct's pkru field. A pointer to that is passed in @pkru.
* The kernel will restore it manually, so the XRSTOR behavior that resets
* the PKRU register to the hardware init value (0) if the corresponding
* xfeatures bit is not set is emulated here.
* 3. When called from KVM the PKRU register will be restored from the vcpu's
* pkru field. A pointer to that is passed in @pkru. KVM hasn't used
* XRSTOR and hasn't had the PKRU resetting behavior described above. To
* preserve that KVM behavior, it passes NULL for @pkru if the xfeatures
* bit is not set.
*/
static int copy_uabi_to_xstate(struct fpstate *fpstate, const void *kbuf,
const void __user *ubuf, u32 *pkru)
{
struct xregs_state *xsave = &fpstate->regs.xsave;
unsigned int offset, size;
struct xstate_header hdr;
u64 mask;
int i;
offset = offsetof(struct xregs_state, header);
if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf))
return -EFAULT;
if (validate_user_xstate_header(&hdr, fpstate))
return -EINVAL;
/* Validate MXCSR when any of the related features is in use */
mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM;
if (hdr.xfeatures & mask) {
u32 mxcsr[2];
offset = offsetof(struct fxregs_state, mxcsr);
if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf))
return -EFAULT;
/* Reserved bits in MXCSR must be zero. */
if (mxcsr[0] & ~mxcsr_feature_mask)
return -EINVAL;
/* SSE and YMM require MXCSR even when FP is not in use. */
if (!(hdr.xfeatures & XFEATURE_MASK_FP)) {
xsave->i387.mxcsr = mxcsr[0];
xsave->i387.mxcsr_mask = mxcsr[1];
}
}
for (i = 0; i < XFEATURE_MAX; i++) {
mask = BIT_ULL(i);
if (hdr.xfeatures & mask) {
void *dst = __raw_xsave_addr(xsave, i);
offset = xstate_offsets[i];
size = xstate_sizes[i];
if (copy_from_buffer(dst, offset, size, kbuf, ubuf))
return -EFAULT;
}
}
if (hdr.xfeatures & XFEATURE_MASK_PKRU) {
struct pkru_state *xpkru;
xpkru = __raw_xsave_addr(xsave, XFEATURE_PKRU);
*pkru = xpkru->pkru;
} else {
/*
* KVM may pass NULL here to indicate that it does not need
* PKRU updated.
*/
if (pkru)
*pkru = 0;
}
/*
* The state that came in from userspace was user-state only.
* Mask all the user states out of 'xfeatures':
*/
xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
/*
* Add back in the features that came in from userspace:
*/
xsave->header.xfeatures |= hdr.xfeatures;
return 0;
}
/*
* Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S]
* format and copy to the target thread. Used by ptrace and KVM.
*/
int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru)
{
return copy_uabi_to_xstate(fpstate, kbuf, NULL, pkru);
}
/*
* Convert from a sigreturn standard-format user-space buffer to kernel
* XSAVE[S] format and copy to the target thread. This is called from the
* sigreturn() and rt_sigreturn() system calls.
*/
int copy_sigframe_from_user_to_xstate(struct task_struct *tsk,
const void __user *ubuf)
{
return copy_uabi_to_xstate(tsk->thread.fpu.fpstate, NULL, ubuf, &tsk->thread.pkru);
}
static bool validate_independent_components(u64 mask)
{
u64 xchk;
if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES)))
return false;
xchk = ~xfeatures_mask_independent();
if (WARN_ON_ONCE(!mask || mask & xchk))
return false;
return true;
}
/**
* xsaves - Save selected components to a kernel xstate buffer
* @xstate: Pointer to the buffer
* @mask: Feature mask to select the components to save
*
* The @xstate buffer must be 64 byte aligned and correctly initialized as
* XSAVES does not write the full xstate header. Before first use the
* buffer should be zeroed otherwise a consecutive XRSTORS from that buffer
* can #GP.
*
* The feature mask must be a subset of the independent features.
*/
void xsaves(struct xregs_state *xstate, u64 mask)
{
int err;
if (!validate_independent_components(mask))
return;
XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err);
WARN_ON_ONCE(err);
}
/**
* xrstors - Restore selected components from a kernel xstate buffer
* @xstate: Pointer to the buffer
* @mask: Feature mask to select the components to restore
*
* The @xstate buffer must be 64 byte aligned and correctly initialized
* otherwise XRSTORS from that buffer can #GP.
*
* Proper usage is to restore the state which was saved with
* xsaves() into @xstate.
*
* The feature mask must be a subset of the independent features.
*/
void xrstors(struct xregs_state *xstate, u64 mask)
{
int err;
if (!validate_independent_components(mask))
return;
XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err);
WARN_ON_ONCE(err);
}
#if IS_ENABLED(CONFIG_KVM)
void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature)
{
void *addr = get_xsave_addr(&fps->regs.xsave, xfeature);
if (addr)
memset(addr, 0, xstate_sizes[xfeature]);
}
EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component);
#endif
#ifdef CONFIG_X86_64
#ifdef CONFIG_X86_DEBUG_FPU
/*
* Ensure that a subsequent XSAVE* or XRSTOR* instruction with RFBM=@mask
* can safely operate on the @fpstate buffer.
*/
static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor)
{
u64 xfd = __this_cpu_read(xfd_state);
if (fpstate->xfd == xfd)
return true;
/*
* The XFD MSR does not match fpstate->xfd. That's invalid when
* the passed in fpstate is current's fpstate.
*/
if (fpstate->xfd == current->thread.fpu.fpstate->xfd)
return false;
/*
* XRSTOR(S) from init_fpstate are always correct as it will just
* bring all components into init state and not read from the
* buffer. XSAVE(S) raises #PF after init.
*/
if (fpstate == &init_fpstate)
return rstor;
/*
* XSAVE(S): clone(), fpu_swap_kvm_fpu()
* XRSTORS(S): fpu_swap_kvm_fpu()
*/
/*
* No XSAVE/XRSTOR instructions (except XSAVE itself) touch
* the buffer area for XFD-disabled state components.
*/
mask &= ~xfd;
/*
* Remove features which are valid in fpstate. They
* have space allocated in fpstate.
*/
mask &= ~fpstate->xfeatures;
/*
* Any remaining state components in 'mask' might be written
* by XSAVE/XRSTOR. Fail validation it found.
*/
return !mask;
}
void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor)
{
WARN_ON_ONCE(!xstate_op_valid(fpstate, mask, rstor));
}
#endif /* CONFIG_X86_DEBUG_FPU */
static int __init xfd_update_static_branch(void)
{
/*
* If init_fpstate.xfd has bits set then dynamic features are
* available and the dynamic sizing must be enabled.
*/
if (init_fpstate.xfd)
static_branch_enable(&__fpu_state_size_dynamic);
return 0;
}
arch_initcall(xfd_update_static_branch)
void fpstate_free(struct fpu *fpu)
{
if (fpu->fpstate && fpu->fpstate != &fpu->__fpstate)
vfree(fpu->fpstate);
}
/**
* fpstate_realloc - Reallocate struct fpstate for the requested new features
*
* @xfeatures: A bitmap of xstate features which extend the enabled features
* of that task
* @ksize: The required size for the kernel buffer
* @usize: The required size for user space buffers
* @guest_fpu: Pointer to a guest FPU container. NULL for host allocations
*
* Note vs. vmalloc(): If the task with a vzalloc()-allocated buffer
* terminates quickly, vfree()-induced IPIs may be a concern, but tasks
* with large states are likely to live longer.
*
* Returns: 0 on success, -ENOMEM on allocation error.
*/
static int fpstate_realloc(u64 xfeatures, unsigned int ksize,
unsigned int usize, struct fpu_guest *guest_fpu)
{
struct fpu *fpu = ¤t->thread.fpu;
struct fpstate *curfps, *newfps = NULL;
unsigned int fpsize;
bool in_use;
fpsize = ksize + ALIGN(offsetof(struct fpstate, regs), 64);
newfps = vzalloc(fpsize);
if (!newfps)
return -ENOMEM;
newfps->size = ksize;
newfps->user_size = usize;
newfps->is_valloc = true;
/*
* When a guest FPU is supplied, use @guest_fpu->fpstate
* as reference independent whether it is in use or not.
*/
curfps = guest_fpu ? guest_fpu->fpstate : fpu->fpstate;
/* Determine whether @curfps is the active fpstate */
in_use = fpu->fpstate == curfps;
if (guest_fpu) {
newfps->is_guest = true;
newfps->is_confidential = curfps->is_confidential;
newfps->in_use = curfps->in_use;
guest_fpu->xfeatures |= xfeatures;
guest_fpu->uabi_size = usize;
}
fpregs_lock();
/*
* If @curfps is in use, ensure that the current state is in the
* registers before swapping fpstate as that might invalidate it
* due to layout changes.
*/
if (in_use && test_thread_flag(TIF_NEED_FPU_LOAD))
fpregs_restore_userregs();
newfps->xfeatures = curfps->xfeatures | xfeatures;
newfps->user_xfeatures = curfps->user_xfeatures | xfeatures;
newfps->xfd = curfps->xfd & ~xfeatures;
/* Do the final updates within the locked region */
xstate_init_xcomp_bv(&newfps->regs.xsave, newfps->xfeatures);
if (guest_fpu) {
guest_fpu->fpstate = newfps;
/* If curfps is active, update the FPU fpstate pointer */
if (in_use)
fpu->fpstate = newfps;
} else {
fpu->fpstate = newfps;
}
if (in_use)
xfd_update_state(fpu->fpstate);
fpregs_unlock();
/* Only free valloc'ed state */
if (curfps && curfps->is_valloc)
vfree(curfps);
return 0;
}
static int validate_sigaltstack(unsigned int usize)
{
struct task_struct *thread, *leader = current->group_leader;
unsigned long framesize = get_sigframe_size();
lockdep_assert_held(¤t->sighand->siglock);
/* get_sigframe_size() is based on fpu_user_cfg.max_size */
framesize -= fpu_user_cfg.max_size;
framesize += usize;
for_each_thread(leader, thread) {
if (thread->sas_ss_size && thread->sas_ss_size < framesize)
return -ENOSPC;
}
return 0;
}
static int __xstate_request_perm(u64 permitted, u64 requested, bool guest)
{
/*
* This deliberately does not exclude !XSAVES as we still might
* decide to optionally context switch XCR0 or talk the silicon
* vendors into extending XFD for the pre AMX states, especially
* AVX512.
*/
bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
struct fpu *fpu = ¤t->group_leader->thread.fpu;
struct fpu_state_perm *perm;
unsigned int ksize, usize;
u64 mask;
int ret = 0;
/* Check whether fully enabled */
if ((permitted & requested) == requested)
return 0;
/* Calculate the resulting kernel state size */
mask = permitted | requested;
/* Take supervisor states into account on the host */
if (!guest)
mask |= xfeatures_mask_supervisor();
ksize = xstate_calculate_size(mask, compacted);
/* Calculate the resulting user state size */
mask &= XFEATURE_MASK_USER_SUPPORTED;
usize = xstate_calculate_size(mask, false);
if (!guest) {
ret = validate_sigaltstack(usize);
if (ret)
return ret;
}
perm = guest ? &fpu->guest_perm : &fpu->perm;
/* Pairs with the READ_ONCE() in xstate_get_group_perm() */
WRITE_ONCE(perm->__state_perm, mask);
/* Protected by sighand lock */
perm->__state_size = ksize;
perm->__user_state_size = usize;
return ret;
}
/*
* Permissions array to map facilities with more than one component
*/
static const u64 xstate_prctl_req[XFEATURE_MAX] = {
[XFEATURE_XTILE_DATA] = XFEATURE_MASK_XTILE_DATA,
};
static int xstate_request_perm(unsigned long idx, bool guest)
{
u64 permitted, requested;
int ret;
if (idx >= XFEATURE_MAX)
return -EINVAL;
/*
* Look up the facility mask which can require more than
* one xstate component.
*/
idx = array_index_nospec(idx, ARRAY_SIZE(xstate_prctl_req));
requested = xstate_prctl_req[idx];
if (!requested)
return -EOPNOTSUPP;
if ((fpu_user_cfg.max_features & requested) != requested)
return -EOPNOTSUPP;
/* Lockless quick check */
permitted = xstate_get_group_perm(guest);
if ((permitted & requested) == requested)
return 0;
/* Protect against concurrent modifications */
spin_lock_irq(¤t->sighand->siglock);
permitted = xstate_get_group_perm(guest);
/* First vCPU allocation locks the permissions. */
if (guest && (permitted & FPU_GUEST_PERM_LOCKED))
ret = -EBUSY;
else
ret = __xstate_request_perm(permitted, requested, guest);
spin_unlock_irq(¤t->sighand->siglock);
return ret;
}
int __xfd_enable_feature(u64 xfd_err, struct fpu_guest *guest_fpu)
{
u64 xfd_event = xfd_err & XFEATURE_MASK_USER_DYNAMIC;
struct fpu_state_perm *perm;
unsigned int ksize, usize;
struct fpu *fpu;
if (!xfd_event) {
if (!guest_fpu)
pr_err_once("XFD: Invalid xfd error: %016llx\n", xfd_err);
return 0;
}
/* Protect against concurrent modifications */
spin_lock_irq(¤t->sighand->siglock);
/* If not permitted let it die */
if ((xstate_get_group_perm(!!guest_fpu) & xfd_event) != xfd_event) {
spin_unlock_irq(¤t->sighand->siglock);
return -EPERM;
}
fpu = ¤t->group_leader->thread.fpu;
perm = guest_fpu ? &fpu->guest_perm : &fpu->perm;
ksize = perm->__state_size;
usize = perm->__user_state_size;
/*
* The feature is permitted. State size is sufficient. Dropping
* the lock is safe here even if more features are added from
* another task, the retrieved buffer sizes are valid for the
* currently requested feature(s).
*/
spin_unlock_irq(¤t->sighand->siglock);
/*
* Try to allocate a new fpstate. If that fails there is no way
* out.
*/
if (fpstate_realloc(xfd_event, ksize, usize, guest_fpu))
return -EFAULT;
return 0;
}
int xfd_enable_feature(u64 xfd_err)
{
return __xfd_enable_feature(xfd_err, NULL);
}
#else /* CONFIG_X86_64 */
static inline int xstate_request_perm(unsigned long idx, bool guest)
{
return -EPERM;
}
#endif /* !CONFIG_X86_64 */
u64 xstate_get_guest_group_perm(void)
{
return xstate_get_group_perm(true);
}
EXPORT_SYMBOL_GPL(xstate_get_guest_group_perm);
/**
* fpu_xstate_prctl - xstate permission operations
* @option: A subfunction of arch_prctl()
* @arg2: option argument
* Return: 0 if successful; otherwise, an error code
*
* Option arguments:
*
* ARCH_GET_XCOMP_SUPP: Pointer to user space u64 to store the info
* ARCH_GET_XCOMP_PERM: Pointer to user space u64 to store the info
* ARCH_REQ_XCOMP_PERM: Facility number requested
*
* For facilities which require more than one XSTATE component, the request
* must be the highest state component number related to that facility,
* e.g. for AMX which requires XFEATURE_XTILE_CFG(17) and
* XFEATURE_XTILE_DATA(18) this would be XFEATURE_XTILE_DATA(18).
*/
long fpu_xstate_prctl(int option, unsigned long arg2)
{
u64 __user *uptr = (u64 __user *)arg2;
u64 permitted, supported;
unsigned long idx = arg2;
bool guest = false;
switch (option) {
case ARCH_GET_XCOMP_SUPP:
supported = fpu_user_cfg.max_features | fpu_user_cfg.legacy_features;
return put_user(supported, uptr);
case ARCH_GET_XCOMP_PERM:
/*
* Lockless snapshot as it can also change right after the
* dropping the lock.
*/
permitted = xstate_get_host_group_perm();
permitted &= XFEATURE_MASK_USER_SUPPORTED;
return put_user(permitted, uptr);
case ARCH_GET_XCOMP_GUEST_PERM:
permitted = xstate_get_guest_group_perm();
permitted &= XFEATURE_MASK_USER_SUPPORTED;
return put_user(permitted, uptr);
case ARCH_REQ_XCOMP_GUEST_PERM:
guest = true;
fallthrough;
case ARCH_REQ_XCOMP_PERM:
if (!IS_ENABLED(CONFIG_X86_64))
return -EOPNOTSUPP;
return xstate_request_perm(idx, guest);
default:
return -EINVAL;
}
}
#ifdef CONFIG_PROC_PID_ARCH_STATUS
/*
* Report the amount of time elapsed in millisecond since last AVX512
* use in the task.
*/
static void avx512_status(struct seq_file *m, struct task_struct *task)
{
unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
long delta;
if (!timestamp) {
/*
* Report -1 if no AVX512 usage
*/
delta = -1;
} else {
delta = (long)(jiffies - timestamp);
/*
* Cap to LONG_MAX if time difference > LONG_MAX
*/
if (delta < 0)
delta = LONG_MAX;
delta = jiffies_to_msecs(delta);
}
seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
seq_putc(m, '\n');
}
/*
* Report architecture specific information
*/
int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task)
{
/*
* Report AVX512 state if the processor and build option supported.
*/
if (cpu_feature_enabled(X86_FEATURE_AVX512F))
avx512_status(m, task);
return 0;
}
#endif /* CONFIG_PROC_PID_ARCH_STATUS */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/kernel/exit.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/mm.h>
#include <linux/sched/stat.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/capability.h>
#include <linux/completion.h>
#include <linux/personality.h>
#include <linux/tty.h>
#include <linux/iocontext.h>
#include <linux/key.h>
#include <linux/cpu.h>
#include <linux/acct.h>
#include <linux/tsacct_kern.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/freezer.h>
#include <linux/binfmts.h>
#include <linux/nsproxy.h>
#include <linux/pid_namespace.h>
#include <linux/ptrace.h>
#include <linux/profile.h>
#include <linux/mount.h>
#include <linux/proc_fs.h>
#include <linux/kthread.h>
#include <linux/mempolicy.h>
#include <linux/taskstats_kern.h>
#include <linux/delayacct.h>
#include <linux/cgroup.h>
#include <linux/syscalls.h>
#include <linux/signal.h>
#include <linux/posix-timers.h>
#include <linux/cn_proc.h>
#include <linux/mutex.h>
#include <linux/futex.h>
#include <linux/pipe_fs_i.h>
#include <linux/audit.h> /* for audit_free() */
#include <linux/resource.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/blkdev.h>
#include <linux/task_work.h>
#include <linux/fs_struct.h>
#include <linux/init_task.h>
#include <linux/perf_event.h>
#include <trace/events/sched.h>
#include <linux/hw_breakpoint.h>
#include <linux/oom.h>
#include <linux/writeback.h>
#include <linux/shm.h>
#include <linux/kcov.h>
#include <linux/kmsan.h>
#include <linux/random.h>
#include <linux/rcuwait.h>
#include <linux/compat.h>
#include <linux/io_uring.h>
#include <linux/kprobes.h>
#include <linux/rethook.h>
#include <linux/sysfs.h>
#include <linux/user_events.h>
#include <linux/uaccess.h>
#include <uapi/linux/wait.h>
#include <asm/unistd.h>
#include <asm/mmu_context.h>
#include "exit.h"
/*
* The default value should be high enough to not crash a system that randomly
* crashes its kernel from time to time, but low enough to at least not permit
* overflowing 32-bit refcounts or the ldsem writer count.
*/
static unsigned int oops_limit = 10000;
#ifdef CONFIG_SYSCTL
static struct ctl_table kern_exit_table[] = {
{
.procname = "oops_limit",
.data = &oops_limit,
.maxlen = sizeof(oops_limit),
.mode = 0644,
.proc_handler = proc_douintvec,
},
{ }
};
static __init int kernel_exit_sysctls_init(void)
{
register_sysctl_init("kernel", kern_exit_table);
return 0;
}
late_initcall(kernel_exit_sysctls_init);
#endif
static atomic_t oops_count = ATOMIC_INIT(0);
#ifdef CONFIG_SYSFS
static ssize_t oops_count_show(struct kobject *kobj, struct kobj_attribute *attr,
char *page)
{
return sysfs_emit(page, "%d\n", atomic_read(&oops_count));
}
static struct kobj_attribute oops_count_attr = __ATTR_RO(oops_count);
static __init int kernel_exit_sysfs_init(void)
{
sysfs_add_file_to_group(kernel_kobj, &oops_count_attr.attr, NULL);
return 0;
}
late_initcall(kernel_exit_sysfs_init);
#endif
static void __unhash_process(struct task_struct *p, bool group_dead)
{
nr_threads--;
detach_pid(p, PIDTYPE_PID);
if (group_dead) {
detach_pid(p, PIDTYPE_TGID);
detach_pid(p, PIDTYPE_PGID);
detach_pid(p, PIDTYPE_SID);
list_del_rcu(&p->tasks);
list_del_init(&p->sibling);
__this_cpu_dec(process_counts);
}
list_del_rcu(&p->thread_node);
}
/*
* This function expects the tasklist_lock write-locked.
*/
static void __exit_signal(struct task_struct *tsk)
{
struct signal_struct *sig = tsk->signal;
bool group_dead = thread_group_leader(tsk);
struct sighand_struct *sighand;
struct tty_struct *tty;
u64 utime, stime;
sighand = rcu_dereference_check(tsk->sighand,
lockdep_tasklist_lock_is_held());
spin_lock(&sighand->siglock);
#ifdef CONFIG_POSIX_TIMERS
posix_cpu_timers_exit(tsk);
if (group_dead)
posix_cpu_timers_exit_group(tsk);
#endif
if (group_dead) {
tty = sig->tty;
sig->tty = NULL;
} else {
/*
* If there is any task waiting for the group exit
* then notify it:
*/
if (sig->notify_count > 0 && !--sig->notify_count)
wake_up_process(sig->group_exec_task);
if (tsk == sig->curr_target)
sig->curr_target = next_thread(tsk);
}
add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
sizeof(unsigned long long));
/*
* Accumulate here the counters for all threads as they die. We could
* skip the group leader because it is the last user of signal_struct,
* but we want to avoid the race with thread_group_cputime() which can
* see the empty ->thread_head list.
*/
task_cputime(tsk, &utime, &stime);
write_seqlock(&sig->stats_lock);
sig->utime += utime;
sig->stime += stime;
sig->gtime += task_gtime(tsk);
sig->min_flt += tsk->min_flt;
sig->maj_flt += tsk->maj_flt;
sig->nvcsw += tsk->nvcsw;
sig->nivcsw += tsk->nivcsw;
sig->inblock += task_io_get_inblock(tsk);
sig->oublock += task_io_get_oublock(tsk);
task_io_accounting_add(&sig->ioac, &tsk->ioac);
sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
sig->nr_threads--;
__unhash_process(tsk, group_dead);
write_sequnlock(&sig->stats_lock);
/*
* Do this under ->siglock, we can race with another thread
* doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
*/
flush_sigqueue(&tsk->pending);
tsk->sighand = NULL;
spin_unlock(&sighand->siglock);
__cleanup_sighand(sighand);
clear_tsk_thread_flag(tsk, TIF_SIGPENDING);
if (group_dead) {
flush_sigqueue(&sig->shared_pending);
tty_kref_put(tty);
}
}
static void delayed_put_task_struct(struct rcu_head *rhp)
{
struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
kprobe_flush_task(tsk);
rethook_flush_task(tsk);
perf_event_delayed_put(tsk);
trace_sched_process_free(tsk);
put_task_struct(tsk);
}
void put_task_struct_rcu_user(struct task_struct *task)
{
if (refcount_dec_and_test(&task->rcu_users))
call_rcu(&task->rcu, delayed_put_task_struct);
}
void __weak release_thread(struct task_struct *dead_task)
{
}
void release_task(struct task_struct *p)
{
struct task_struct *leader;
struct pid *thread_pid;
int zap_leader;
repeat:
/* don't need to get the RCU readlock here - the process is dead and
* can't be modifying its own credentials. But shut RCU-lockdep up */
rcu_read_lock();
dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
rcu_read_unlock();
cgroup_release(p);
write_lock_irq(&tasklist_lock);
ptrace_release_task(p);
thread_pid = get_pid(p->thread_pid);
__exit_signal(p);
/*
* If we are the last non-leader member of the thread
* group, and the leader is zombie, then notify the
* group leader's parent process. (if it wants notification.)
*/
zap_leader = 0;
leader = p->group_leader;
if (leader != p && thread_group_empty(leader)
&& leader->exit_state == EXIT_ZOMBIE) {
/*
* If we were the last child thread and the leader has
* exited already, and the leader's parent ignores SIGCHLD,
* then we are the one who should release the leader.
*/
zap_leader = do_notify_parent(leader, leader->exit_signal);
if (zap_leader)
leader->exit_state = EXIT_DEAD;
}
write_unlock_irq(&tasklist_lock);
seccomp_filter_release(p);
proc_flush_pid(thread_pid);
put_pid(thread_pid);
release_thread(p);
put_task_struct_rcu_user(p);
p = leader;
if (unlikely(zap_leader))
goto repeat;
}
int rcuwait_wake_up(struct rcuwait *w)
{
int ret = 0;
struct task_struct *task;
rcu_read_lock();
/*
* Order condition vs @task, such that everything prior to the load
* of @task is visible. This is the condition as to why the user called
* rcuwait_wake() in the first place. Pairs with set_current_state()
* barrier (A) in rcuwait_wait_event().
*
* WAIT WAKE
* [S] tsk = current [S] cond = true
* MB (A) MB (B)
* [L] cond [L] tsk
*/
smp_mb(); /* (B) */
task = rcu_dereference(w->task);
if (task)
ret = wake_up_process(task);
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(rcuwait_wake_up);
/*
* Determine if a process group is "orphaned", according to the POSIX
* definition in 2.2.2.52. Orphaned process groups are not to be affected
* by terminal-generated stop signals. Newly orphaned process groups are
* to receive a SIGHUP and a SIGCONT.
*
* "I ask you, have you ever known what it is to be an orphan?"
*/
static int will_become_orphaned_pgrp(struct pid *pgrp,
struct task_struct *ignored_task)
{
struct task_struct *p;
do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
if ((p == ignored_task) ||
(p->exit_state && thread_group_empty(p)) ||
is_global_init(p->real_parent))
continue;
if (task_pgrp(p->real_parent) != pgrp &&
task_session(p->real_parent) == task_session(p))
return 0;
} while_each_pid_task(pgrp, PIDTYPE_PGID, p);
return 1;
}
int is_current_pgrp_orphaned(void)
{
int retval;
read_lock(&tasklist_lock);
retval = will_become_orphaned_pgrp(task_pgrp(current), NULL);
read_unlock(&tasklist_lock);
return retval;
}
static bool has_stopped_jobs(struct pid *pgrp)
{
struct task_struct *p;
do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
if (p->signal->flags & SIGNAL_STOP_STOPPED)
return true;
} while_each_pid_task(pgrp, PIDTYPE_PGID, p);
return false;
}
/*
* Check to see if any process groups have become orphaned as
* a result of our exiting, and if they have any stopped jobs,
* send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
*/
static void
kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent)
{
struct pid *pgrp = task_pgrp(tsk);
struct task_struct *ignored_task = tsk;
if (!parent)
/* exit: our father is in a different pgrp than
* we are and we were the only connection outside.
*/
parent = tsk->real_parent;
else
/* reparent: our child is in a different pgrp than
* we are, and it was the only connection outside.
*/
ignored_task = NULL;
if (task_pgrp(parent) != pgrp &&
task_session(parent) == task_session(tsk) &&
will_become_orphaned_pgrp(pgrp, ignored_task) &&
has_stopped_jobs(pgrp)) {
__kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp);
__kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp);
}
}
static void coredump_task_exit(struct task_struct *tsk)
{
struct core_state *core_state;
/*
* Serialize with any possible pending coredump.
* We must hold siglock around checking core_state
* and setting PF_POSTCOREDUMP. The core-inducing thread
* will increment ->nr_threads for each thread in the
* group without PF_POSTCOREDUMP set.
*/
spin_lock_irq(&tsk->sighand->siglock);
tsk->flags |= PF_POSTCOREDUMP;
core_state = tsk->signal->core_state;
spin_unlock_irq(&tsk->sighand->siglock);
/* The vhost_worker does not particpate in coredumps */
if (core_state &&
((tsk->flags & (PF_IO_WORKER | PF_USER_WORKER)) != PF_USER_WORKER)) {
struct core_thread self;
self.task = current;
if (self.task->flags & PF_SIGNALED)
self.next = xchg(&core_state->dumper.next, &self);
else
self.task = NULL;
/*
* Implies mb(), the result of xchg() must be visible
* to core_state->dumper.
*/
if (atomic_dec_and_test(&core_state->nr_threads))
complete(&core_state->startup);
for (;;) {
set_current_state(TASK_UNINTERRUPTIBLE|TASK_FREEZABLE);
if (!self.task) /* see coredump_finish() */
break;
schedule();
}
__set_current_state(TASK_RUNNING);
}
}
#ifdef CONFIG_MEMCG
/*
* A task is exiting. If it owned this mm, find a new owner for the mm.
*/
void mm_update_next_owner(struct mm_struct *mm)
{
struct task_struct *c, *g, *p = current;
retry:
/*
* If the exiting or execing task is not the owner, it's
* someone else's problem.
*/
if (mm->owner != p)
return;
/*
* The current owner is exiting/execing and there are no other
* candidates. Do not leave the mm pointing to a possibly
* freed task structure.
*/
if (atomic_read(&mm->mm_users) <= 1) {
WRITE_ONCE(mm->owner, NULL);
return;
}
read_lock(&tasklist_lock);
/*
* Search in the children
*/
list_for_each_entry(c, &p->children, sibling) {
if (c->mm == mm)
goto assign_new_owner;
}
/*
* Search in the siblings
*/
list_for_each_entry(c, &p->real_parent->children, sibling) {
if (c->mm == mm)
goto assign_new_owner;
}
/*
* Search through everything else, we should not get here often.
*/
for_each_process(g) {
if (g->flags & PF_KTHREAD)
continue;
for_each_thread(g, c) {
if (c->mm == mm)
goto assign_new_owner;
if (c->mm)
break;
}
}
read_unlock(&tasklist_lock);
/*
* We found no owner yet mm_users > 1: this implies that we are
* most likely racing with swapoff (try_to_unuse()) or /proc or
* ptrace or page migration (get_task_mm()). Mark owner as NULL.
*/
WRITE_ONCE(mm->owner, NULL);
return;
assign_new_owner:
BUG_ON(c == p);
get_task_struct(c);
/*
* The task_lock protects c->mm from changing.
* We always want mm->owner->mm == mm
*/
task_lock(c);
/*
* Delay read_unlock() till we have the task_lock()
* to ensure that c does not slip away underneath us
*/
read_unlock(&tasklist_lock);
if (c->mm != mm) {
task_unlock(c);
put_task_struct(c);
goto retry;
}
WRITE_ONCE(mm->owner, c);
lru_gen_migrate_mm(mm);
task_unlock(c);
put_task_struct(c);
}
#endif /* CONFIG_MEMCG */
/*
* Turn us into a lazy TLB process if we
* aren't already..
*/
static void exit_mm(void)
{
struct mm_struct *mm = current->mm;
exit_mm_release(current, mm);
if (!mm)
return;
mmap_read_lock(mm);
mmgrab_lazy_tlb(mm);
BUG_ON(mm != current->active_mm);
/* more a memory barrier than a real lock */
task_lock(current);
/*
* When a thread stops operating on an address space, the loop
* in membarrier_private_expedited() may not observe that
* tsk->mm, and the loop in membarrier_global_expedited() may
* not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED
* rq->membarrier_state, so those would not issue an IPI.
* Membarrier requires a memory barrier after accessing
* user-space memory, before clearing tsk->mm or the
* rq->membarrier_state.
*/
smp_mb__after_spinlock();
local_irq_disable();
current->mm = NULL;
membarrier_update_current_mm(NULL);
enter_lazy_tlb(mm, current);
local_irq_enable();
task_unlock(current);
mmap_read_unlock(mm);
mm_update_next_owner(mm);
mmput(mm);
if (test_thread_flag(TIF_MEMDIE))
exit_oom_victim();
}
static struct task_struct *find_alive_thread(struct task_struct *p)
{
struct task_struct *t;
for_each_thread(p, t) {
if (!(t->flags & PF_EXITING))
return t;
}
return NULL;
}
static struct task_struct *find_child_reaper(struct task_struct *father,
struct list_head *dead)
__releases(&tasklist_lock)
__acquires(&tasklist_lock)
{
struct pid_namespace *pid_ns = task_active_pid_ns(father);
struct task_struct *reaper = pid_ns->child_reaper;
struct task_struct *p, *n;
if (likely(reaper != father))
return reaper;
reaper = find_alive_thread(father);
if (reaper) {
pid_ns->child_reaper = reaper;
return reaper;
}
write_unlock_irq(&tasklist_lock);
list_for_each_entry_safe(p, n, dead, ptrace_entry) {
list_del_init(&p->ptrace_entry);
release_task(p);
}
zap_pid_ns_processes(pid_ns);
write_lock_irq(&tasklist_lock);
return father;
}
/*
* When we die, we re-parent all our children, and try to:
* 1. give them to another thread in our thread group, if such a member exists
* 2. give it to the first ancestor process which prctl'd itself as a
* child_subreaper for its children (like a service manager)
* 3. give it to the init process (PID 1) in our pid namespace
*/
static struct task_struct *find_new_reaper(struct task_struct *father,
struct task_struct *child_reaper)
{
struct task_struct *thread, *reaper;
thread = find_alive_thread(father);
if (thread)
return thread;
if (father->signal->has_child_subreaper) {
unsigned int ns_level = task_pid(father)->level;
/*
* Find the first ->is_child_subreaper ancestor in our pid_ns.
* We can't check reaper != child_reaper to ensure we do not
* cross the namespaces, the exiting parent could be injected
* by setns() + fork().
* We check pid->level, this is slightly more efficient than
* task_active_pid_ns(reaper) != task_active_pid_ns(father).
*/
for (reaper = father->real_parent;
task_pid(reaper)->level == ns_level;
reaper = reaper->real_parent) {
if (reaper == &init_task)
break;
if (!reaper->signal->is_child_subreaper)
continue;
thread = find_alive_thread(reaper);
if (thread)
return thread;
}
}
return child_reaper;
}
/*
* Any that need to be release_task'd are put on the @dead list.
*/
static void reparent_leader(struct task_struct *father, struct task_struct *p,
struct list_head *dead)
{
if (unlikely(p->exit_state == EXIT_DEAD))
return;
/* We don't want people slaying init. */
p->exit_signal = SIGCHLD;
/* If it has exited notify the new parent about this child's death. */
if (!p->ptrace &&
p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {
if (do_notify_parent(p, p->exit_signal)) {
p->exit_state = EXIT_DEAD;
list_add(&p->ptrace_entry, dead);
}
}
kill_orphaned_pgrp(p, father);
}
/*
* This does two things:
*
* A. Make init inherit all the child processes
* B. Check to see if any process groups have become orphaned
* as a result of our exiting, and if they have any stopped
* jobs, send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
*/
static void forget_original_parent(struct task_struct *father,
struct list_head *dead)
{
struct task_struct *p, *t, *reaper;
if (unlikely(!list_empty(&father->ptraced)))
exit_ptrace(father, dead);
/* Can drop and reacquire tasklist_lock */
reaper = find_child_reaper(father, dead);
if (list_empty(&father->children))
return;
reaper = find_new_reaper(father, reaper);
list_for_each_entry(p, &father->children, sibling) {
for_each_thread(p, t) {
RCU_INIT_POINTER(t->real_parent, reaper);
BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father));
if (likely(!t->ptrace))
t->parent = t->real_parent;
if (t->pdeath_signal)
group_send_sig_info(t->pdeath_signal,
SEND_SIG_NOINFO, t,
PIDTYPE_TGID);
}
/*
* If this is a threaded reparent there is no need to
* notify anyone anything has happened.
*/
if (!same_thread_group(reaper, father))
reparent_leader(father, p, dead);
}
list_splice_tail_init(&father->children, &reaper->children);
}
/*
* Send signals to all our closest relatives so that they know
* to properly mourn us..
*/
static void exit_notify(struct task_struct *tsk, int group_dead)
{
bool autoreap;
struct task_struct *p, *n;
LIST_HEAD(dead);
write_lock_irq(&tasklist_lock);
forget_original_parent(tsk, &dead);
if (group_dead)
kill_orphaned_pgrp(tsk->group_leader, NULL);
tsk->exit_state = EXIT_ZOMBIE;
/*
* sub-thread or delay_group_leader(), wake up the
* PIDFD_THREAD waiters.
*/
if (!thread_group_empty(tsk))
do_notify_pidfd(tsk);
if (unlikely(tsk->ptrace)) {
int sig = thread_group_leader(tsk) &&
thread_group_empty(tsk) &&
!ptrace_reparented(tsk) ?
tsk->exit_signal : SIGCHLD;
autoreap = do_notify_parent(tsk, sig);
} else if (thread_group_leader(tsk)) {
autoreap = thread_group_empty(tsk) &&
do_notify_parent(tsk, tsk->exit_signal);
} else {
autoreap = true;
}
if (autoreap) {
tsk->exit_state = EXIT_DEAD;
list_add(&tsk->ptrace_entry, &dead);
}
/* mt-exec, de_thread() is waiting for group leader */
if (unlikely(tsk->signal->notify_count < 0))
wake_up_process(tsk->signal->group_exec_task);
write_unlock_irq(&tasklist_lock);
list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
list_del_init(&p->ptrace_entry);
release_task(p);
}
}
#ifdef CONFIG_DEBUG_STACK_USAGE
static void check_stack_usage(void)
{
static DEFINE_SPINLOCK(low_water_lock);
static int lowest_to_date = THREAD_SIZE;
unsigned long free;
free = stack_not_used(current);
if (free >= lowest_to_date)
return;
spin_lock(&low_water_lock);
if (free < lowest_to_date) {
pr_info("%s (%d) used greatest stack depth: %lu bytes left\n",
current->comm, task_pid_nr(current), free);
lowest_to_date = free;
}
spin_unlock(&low_water_lock);
}
#else
static inline void check_stack_usage(void) {}
#endif
static void synchronize_group_exit(struct task_struct *tsk, long code)
{
struct sighand_struct *sighand = tsk->sighand;
struct signal_struct *signal = tsk->signal;
spin_lock_irq(&sighand->siglock);
signal->quick_threads--;
if ((signal->quick_threads == 0) &&
!(signal->flags & SIGNAL_GROUP_EXIT)) {
signal->flags = SIGNAL_GROUP_EXIT;
signal->group_exit_code = code;
signal->group_stop_count = 0;
}
spin_unlock_irq(&sighand->siglock);
}
void __noreturn do_exit(long code)
{
struct task_struct *tsk = current;
int group_dead;
WARN_ON(irqs_disabled());
synchronize_group_exit(tsk, code);
WARN_ON(tsk->plug);
kcov_task_exit(tsk);
kmsan_task_exit(tsk);
coredump_task_exit(tsk);
ptrace_event(PTRACE_EVENT_EXIT, code);
user_events_exit(tsk);
io_uring_files_cancel();
exit_signals(tsk); /* sets PF_EXITING */
acct_update_integrals(tsk);
group_dead = atomic_dec_and_test(&tsk->signal->live);
if (group_dead) {
/*
* If the last thread of global init has exited, panic
* immediately to get a useable coredump.
*/
if (unlikely(is_global_init(tsk)))
panic("Attempted to kill init! exitcode=0x%08x\n",
tsk->signal->group_exit_code ?: (int)code);
#ifdef CONFIG_POSIX_TIMERS
hrtimer_cancel(&tsk->signal->real_timer);
exit_itimers(tsk);
#endif
if (tsk->mm)
setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
}
acct_collect(code, group_dead);
if (group_dead)
tty_audit_exit();
audit_free(tsk);
tsk->exit_code = code;
taskstats_exit(tsk, group_dead);
exit_mm();
if (group_dead)
acct_process();
trace_sched_process_exit(tsk);
exit_sem(tsk);
exit_shm(tsk);
exit_files(tsk);
exit_fs(tsk);
if (group_dead)
disassociate_ctty(1);
exit_task_namespaces(tsk);
exit_task_work(tsk);
exit_thread(tsk);
/*
* Flush inherited counters to the parent - before the parent
* gets woken up by child-exit notifications.
*
* because of cgroup mode, must be called before cgroup_exit()
*/
perf_event_exit_task(tsk);
sched_autogroup_exit_task(tsk);
cgroup_exit(tsk);
/*
* FIXME: do that only when needed, using sched_exit tracepoint
*/
flush_ptrace_hw_breakpoint(tsk);
exit_tasks_rcu_start();
exit_notify(tsk, group_dead);
proc_exit_connector(tsk);
mpol_put_task_policy(tsk);
#ifdef CONFIG_FUTEX
if (unlikely(current->pi_state_cache))
kfree(current->pi_state_cache);
#endif
/*
* Make sure we are holding no locks:
*/
debug_check_no_locks_held();
if (tsk->io_context)
exit_io_context(tsk);
if (tsk->splice_pipe)
free_pipe_info(tsk->splice_pipe);
if (tsk->task_frag.page)
put_page(tsk->task_frag.page);
exit_task_stack_account(tsk);
check_stack_usage();
preempt_disable();
if (tsk->nr_dirtied)
__this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
exit_rcu();
exit_tasks_rcu_finish();
lockdep_free_task(tsk);
do_task_dead();
}
void __noreturn make_task_dead(int signr)
{
/*
* Take the task off the cpu after something catastrophic has
* happened.
*
* We can get here from a kernel oops, sometimes with preemption off.
* Start by checking for critical errors.
* Then fix up important state like USER_DS and preemption.
* Then do everything else.
*/
struct task_struct *tsk = current;
unsigned int limit;
if (unlikely(in_interrupt()))
panic("Aiee, killing interrupt handler!");
if (unlikely(!tsk->pid))
panic("Attempted to kill the idle task!");
if (unlikely(irqs_disabled())) {
pr_info("note: %s[%d] exited with irqs disabled\n",
current->comm, task_pid_nr(current));
local_irq_enable();
}
if (unlikely(in_atomic())) {
pr_info("note: %s[%d] exited with preempt_count %d\n",
current->comm, task_pid_nr(current),
preempt_count());
preempt_count_set(PREEMPT_ENABLED);
}
/*
* Every time the system oopses, if the oops happens while a reference
* to an object was held, the reference leaks.
* If the oops doesn't also leak memory, repeated oopsing can cause
* reference counters to wrap around (if they're not using refcount_t).
* This means that repeated oopsing can make unexploitable-looking bugs
* exploitable through repeated oopsing.
* To make sure this can't happen, place an upper bound on how often the
* kernel may oops without panic().
*/
limit = READ_ONCE(oops_limit);
if (atomic_inc_return(&oops_count) >= limit && limit)
panic("Oopsed too often (kernel.oops_limit is %d)", limit);
/*
* We're taking recursive faults here in make_task_dead. Safest is to just
* leave this task alone and wait for reboot.
*/
if (unlikely(tsk->flags & PF_EXITING)) {
pr_alert("Fixing recursive fault but reboot is needed!\n");
futex_exit_recursive(tsk);
tsk->exit_state = EXIT_DEAD;
refcount_inc(&tsk->rcu_users);
do_task_dead();
}
do_exit(signr);
}
SYSCALL_DEFINE1(exit, int, error_code)
{
do_exit((error_code&0xff)<<8);
}
/*
* Take down every thread in the group. This is called by fatal signals
* as well as by sys_exit_group (below).
*/
void __noreturn
do_group_exit(int exit_code)
{
struct signal_struct *sig = current->signal;
if (sig->flags & SIGNAL_GROUP_EXIT)
exit_code = sig->group_exit_code;
else if (sig->group_exec_task)
exit_code = 0;
else {
struct sighand_struct *const sighand = current->sighand;
spin_lock_irq(&sighand->siglock);
if (sig->flags & SIGNAL_GROUP_EXIT)
/* Another thread got here before we took the lock. */
exit_code = sig->group_exit_code;
else if (sig->group_exec_task)
exit_code = 0;
else {
sig->group_exit_code = exit_code;
sig->flags = SIGNAL_GROUP_EXIT;
zap_other_threads(current);
}
spin_unlock_irq(&sighand->siglock);
}
do_exit(exit_code);
/* NOTREACHED */
}
/*
* this kills every thread in the thread group. Note that any externally
* wait4()-ing process will get the correct exit code - even if this
* thread is not the thread group leader.
*/
SYSCALL_DEFINE1(exit_group, int, error_code)
{
do_group_exit((error_code & 0xff) << 8);
/* NOTREACHED */
return 0;
}
static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
{
return wo->wo_type == PIDTYPE_MAX ||
task_pid_type(p, wo->wo_type) == wo->wo_pid;
}
static int
eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p)
{
if (!eligible_pid(wo, p))
return 0;
/*
* Wait for all children (clone and not) if __WALL is set or
* if it is traced by us.
*/
if (ptrace || (wo->wo_flags & __WALL))
return 1;
/*
* Otherwise, wait for clone children *only* if __WCLONE is set;
* otherwise, wait for non-clone children *only*.
*
* Note: a "clone" child here is one that reports to its parent
* using a signal other than SIGCHLD, or a non-leader thread which
* we can only see if it is traced by us.
*/
if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
return 0;
return 1;
}
/*
* Handle sys_wait4 work for one task in state EXIT_ZOMBIE. We hold
* read_lock(&tasklist_lock) on entry. If we return zero, we still hold
* the lock and this task is uninteresting. If we return nonzero, we have
* released the lock and the system call should return.
*/
static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
{
int state, status;
pid_t pid = task_pid_vnr(p);
uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p));
struct waitid_info *infop;
if (!likely(wo->wo_flags & WEXITED))
return 0;
if (unlikely(wo->wo_flags & WNOWAIT)) {
status = (p->signal->flags & SIGNAL_GROUP_EXIT)
? p->signal->group_exit_code : p->exit_code;
get_task_struct(p);
read_unlock(&tasklist_lock);
sched_annotate_sleep();
if (wo->wo_rusage)
getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
put_task_struct(p);
goto out_info;
}
/*
* Move the task's state to DEAD/TRACE, only one thread can do this.
*/
state = (ptrace_reparented(p) && thread_group_leader(p)) ?
EXIT_TRACE : EXIT_DEAD;
if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE)
return 0;
/*
* We own this thread, nobody else can reap it.
*/
read_unlock(&tasklist_lock);
sched_annotate_sleep();
/*
* Check thread_group_leader() to exclude the traced sub-threads.
*/
if (state == EXIT_DEAD && thread_group_leader(p)) {
struct signal_struct *sig = p->signal;
struct signal_struct *psig = current->signal;
unsigned long maxrss;
u64 tgutime, tgstime;
/*
* The resource counters for the group leader are in its
* own task_struct. Those for dead threads in the group
* are in its signal_struct, as are those for the child
* processes it has previously reaped. All these
* accumulate in the parent's signal_struct c* fields.
*
* We don't bother to take a lock here to protect these
* p->signal fields because the whole thread group is dead
* and nobody can change them.
*
* psig->stats_lock also protects us from our sub-threads
* which can reap other children at the same time.
*
* We use thread_group_cputime_adjusted() to get times for
* the thread group, which consolidates times for all threads
* in the group including the group leader.
*/
thread_group_cputime_adjusted(p, &tgutime, &tgstime);
write_seqlock_irq(&psig->stats_lock);
psig->cutime += tgutime + sig->cutime;
psig->cstime += tgstime + sig->cstime;
psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime;
psig->cmin_flt +=
p->min_flt + sig->min_flt + sig->cmin_flt;
psig->cmaj_flt +=
p->maj_flt + sig->maj_flt + sig->cmaj_flt;
psig->cnvcsw +=
p->nvcsw + sig->nvcsw + sig->cnvcsw;
psig->cnivcsw +=
p->nivcsw + sig->nivcsw + sig->cnivcsw;
psig->cinblock +=
task_io_get_inblock(p) +
sig->inblock + sig->cinblock;
psig->coublock +=
task_io_get_oublock(p) +
sig->oublock + sig->coublock;
maxrss = max(sig->maxrss, sig->cmaxrss);
if (psig->cmaxrss < maxrss)
psig->cmaxrss = maxrss;
task_io_accounting_add(&psig->ioac, &p->ioac);
task_io_accounting_add(&psig->ioac, &sig->ioac);
write_sequnlock_irq(&psig->stats_lock);
}
if (wo->wo_rusage)
getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
status = (p->signal->flags & SIGNAL_GROUP_EXIT)
? p->signal->group_exit_code : p->exit_code;
wo->wo_stat = status;
if (state == EXIT_TRACE) {
write_lock_irq(&tasklist_lock);
/* We dropped tasklist, ptracer could die and untrace */
ptrace_unlink(p);
/* If parent wants a zombie, don't release it now */
state = EXIT_ZOMBIE;
if (do_notify_parent(p, p->exit_signal))
state = EXIT_DEAD;
p->exit_state = state;
write_unlock_irq(&tasklist_lock);
}
if (state == EXIT_DEAD)
release_task(p);
out_info:
infop = wo->wo_info;
if (infop) {
if ((status & 0x7f) == 0) {
infop->cause = CLD_EXITED;
infop->status = status >> 8;
} else {
infop->cause = (status & 0x80) ? CLD_DUMPED : CLD_KILLED;
infop->status = status & 0x7f;
}
infop->pid = pid;
infop->uid = uid;
}
return pid;
}
static int *task_stopped_code(struct task_struct *p, bool ptrace)
{
if (ptrace) {
if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING))
return &p->exit_code;
} else {
if (p->signal->flags & SIGNAL_STOP_STOPPED)
return &p->signal->group_exit_code;
}
return NULL;
}
/**
* wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED
* @wo: wait options
* @ptrace: is the wait for ptrace
* @p: task to wait for
*
* Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED.
*
* CONTEXT:
* read_lock(&tasklist_lock), which is released if return value is
* non-zero. Also, grabs and releases @p->sighand->siglock.
*
* RETURNS:
* 0 if wait condition didn't exist and search for other wait conditions
* should continue. Non-zero return, -errno on failure and @p's pid on
* success, implies that tasklist_lock is released and wait condition
* search should terminate.
*/
static int wait_task_stopped(struct wait_opts *wo,
int ptrace, struct task_struct *p)
{
struct waitid_info *infop;
int exit_code, *p_code, why;
uid_t uid = 0; /* unneeded, required by compiler */
pid_t pid;
/*
* Traditionally we see ptrace'd stopped tasks regardless of options.
*/
if (!ptrace && !(wo->wo_flags & WUNTRACED))
return 0;
if (!task_stopped_code(p, ptrace))
return 0;
exit_code = 0;
spin_lock_irq(&p->sighand->siglock);
p_code = task_stopped_code(p, ptrace);
if (unlikely(!p_code))
goto unlock_sig;
exit_code = *p_code;
if (!exit_code)
goto unlock_sig;
if (!unlikely(wo->wo_flags & WNOWAIT))
*p_code = 0;
uid = from_kuid_munged(current_user_ns(), task_uid(p));
unlock_sig:
spin_unlock_irq(&p->sighand->siglock);
if (!exit_code)
return 0;
/*
* Now we are pretty sure this task is interesting.
* Make sure it doesn't get reaped out from under us while we
* give up the lock and then examine it below. We don't want to
* keep holding onto the tasklist_lock while we call getrusage and
* possibly take page faults for user memory.
*/
get_task_struct(p);
pid = task_pid_vnr(p);
why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
read_unlock(&tasklist_lock);
sched_annotate_sleep();
if (wo->wo_rusage)
getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
put_task_struct(p);
if (likely(!(wo->wo_flags & WNOWAIT)))
wo->wo_stat = (exit_code << 8) | 0x7f;
infop = wo->wo_info;
if (infop) {
infop->cause = why;
infop->status = exit_code;
infop->pid = pid;
infop->uid = uid;
}
return pid;
}
/*
* Handle do_wait work for one task in a live, non-stopped state.
* read_lock(&tasklist_lock) on entry. If we return zero, we still hold
* the lock and this task is uninteresting. If we return nonzero, we have
* released the lock and the system call should return.
*/
static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
{
struct waitid_info *infop;
pid_t pid;
uid_t uid;
if (!unlikely(wo->wo_flags & WCONTINUED))
return 0;
if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
return 0;
spin_lock_irq(&p->sighand->siglock);
/* Re-check with the lock held. */
if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) {
spin_unlock_irq(&p->sighand->siglock);
return 0;
}
if (!unlikely(wo->wo_flags & WNOWAIT))
p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
uid = from_kuid_munged(current_user_ns(), task_uid(p));
spin_unlock_irq(&p->sighand->siglock);
pid = task_pid_vnr(p);
get_task_struct(p);
read_unlock(&tasklist_lock);
sched_annotate_sleep();
if (wo->wo_rusage)
getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
put_task_struct(p);
infop = wo->wo_info;
if (!infop) {
wo->wo_stat = 0xffff;
} else {
infop->cause = CLD_CONTINUED;
infop->pid = pid;
infop->uid = uid;
infop->status = SIGCONT;
}
return pid;
}
/*
* Consider @p for a wait by @parent.
*
* -ECHILD should be in ->notask_error before the first call.
* Returns nonzero for a final return, when we have unlocked tasklist_lock.
* Returns zero if the search for a child should continue;
* then ->notask_error is 0 if @p is an eligible child,
* or still -ECHILD.
*/
static int wait_consider_task(struct wait_opts *wo, int ptrace,
struct task_struct *p)
{
/*
* We can race with wait_task_zombie() from another thread.
* Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition
* can't confuse the checks below.
*/
int exit_state = READ_ONCE(p->exit_state);
int ret;
if (unlikely(exit_state == EXIT_DEAD))
return 0;
ret = eligible_child(wo, ptrace, p);
if (!ret)
return ret;
if (unlikely(exit_state == EXIT_TRACE)) {
/*
* ptrace == 0 means we are the natural parent. In this case
* we should clear notask_error, debugger will notify us.
*/
if (likely(!ptrace))
wo->notask_error = 0;
return 0;
}
if (likely(!ptrace) && unlikely(p->ptrace)) {
/*
* If it is traced by its real parent's group, just pretend
* the caller is ptrace_do_wait() and reap this child if it
* is zombie.
*
* This also hides group stop state from real parent; otherwise
* a single stop can be reported twice as group and ptrace stop.
* If a ptracer wants to distinguish these two events for its
* own children it should create a separate process which takes
* the role of real parent.
*/
if (!ptrace_reparented(p))
ptrace = 1;
}
/* slay zombie? */
if (exit_state == EXIT_ZOMBIE) {
/* we don't reap group leaders with subthreads */
if (!delay_group_leader(p)) {
/*
* A zombie ptracee is only visible to its ptracer.
* Notification and reaping will be cascaded to the
* real parent when the ptracer detaches.
*/
if (unlikely(ptrace) || likely(!p->ptrace))
return wait_task_zombie(wo, p);
}
/*
* Allow access to stopped/continued state via zombie by
* falling through. Clearing of notask_error is complex.
*
* When !@ptrace:
*
* If WEXITED is set, notask_error should naturally be
* cleared. If not, subset of WSTOPPED|WCONTINUED is set,
* so, if there are live subthreads, there are events to
* wait for. If all subthreads are dead, it's still safe
* to clear - this function will be called again in finite
* amount time once all the subthreads are released and
* will then return without clearing.
*
* When @ptrace:
*
* Stopped state is per-task and thus can't change once the
* target task dies. Only continued and exited can happen.
* Clear notask_error if WCONTINUED | WEXITED.
*/
if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED)))
wo->notask_error = 0;
} else {
/*
* @p is alive and it's gonna stop, continue or exit, so
* there always is something to wait for.
*/
wo->notask_error = 0;
}
/*
* Wait for stopped. Depending on @ptrace, different stopped state
* is used and the two don't interact with each other.
*/
ret = wait_task_stopped(wo, ptrace, p);
if (ret)
return ret;
/*
* Wait for continued. There's only one continued state and the
* ptracer can consume it which can confuse the real parent. Don't
* use WCONTINUED from ptracer. You don't need or want it.
*/
return wait_task_continued(wo, p);
}
/*
* Do the work of do_wait() for one thread in the group, @tsk.
*
* -ECHILD should be in ->notask_error before the first call.
* Returns nonzero for a final return, when we have unlocked tasklist_lock.
* Returns zero if the search for a child should continue; then
* ->notask_error is 0 if there were any eligible children,
* or still -ECHILD.
*/
static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
{
struct task_struct *p;
list_for_each_entry(p, &tsk->children, sibling) {
int ret = wait_consider_task(wo, 0, p);
if (ret)
return ret;
}
return 0;
}
static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
{
struct task_struct *p;
list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
int ret = wait_consider_task(wo, 1, p);
if (ret)
return ret;
}
return 0;
}
bool pid_child_should_wake(struct wait_opts *wo, struct task_struct *p)
{
if (!eligible_pid(wo, p))
return false;
if ((wo->wo_flags & __WNOTHREAD) && wo->child_wait.private != p->parent)
return false;
return true;
}
static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,
int sync, void *key)
{
struct wait_opts *wo = container_of(wait, struct wait_opts,
child_wait);
struct task_struct *p = key;
if (pid_child_should_wake(wo, p))
return default_wake_function(wait, mode, sync, key);
return 0;
}
void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
{
__wake_up_sync_key(&parent->signal->wait_chldexit,
TASK_INTERRUPTIBLE, p);
}
static bool is_effectively_child(struct wait_opts *wo, bool ptrace,
struct task_struct *target)
{
struct task_struct *parent =
!ptrace ? target->real_parent : target->parent;
return current == parent || (!(wo->wo_flags & __WNOTHREAD) &&
same_thread_group(current, parent));
}
/*
* Optimization for waiting on PIDTYPE_PID. No need to iterate through child
* and tracee lists to find the target task.
*/
static int do_wait_pid(struct wait_opts *wo)
{
bool ptrace;
struct task_struct *target;
int retval;
ptrace = false;
target = pid_task(wo->wo_pid, PIDTYPE_TGID);
if (target && is_effectively_child(wo, ptrace, target)) {
retval = wait_consider_task(wo, ptrace, target);
if (retval)
return retval;
}
ptrace = true;
target = pid_task(wo->wo_pid, PIDTYPE_PID);
if (target && target->ptrace &&
is_effectively_child(wo, ptrace, target)) {
retval = wait_consider_task(wo, ptrace, target);
if (retval)
return retval;
}
return 0;
}
long __do_wait(struct wait_opts *wo)
{
long retval;
/*
* If there is nothing that can match our criteria, just get out.
* We will clear ->notask_error to zero if we see any child that
* might later match our criteria, even if we are not able to reap
* it yet.
*/
wo->notask_error = -ECHILD;
if ((wo->wo_type < PIDTYPE_MAX) &&
(!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type)))
goto notask;
read_lock(&tasklist_lock);
if (wo->wo_type == PIDTYPE_PID) {
retval = do_wait_pid(wo);
if (retval)
return retval;
} else {
struct task_struct *tsk = current;
do {
retval = do_wait_thread(wo, tsk);
if (retval)
return retval;
retval = ptrace_do_wait(wo, tsk);
if (retval)
return retval;
if (wo->wo_flags & __WNOTHREAD)
break;
} while_each_thread(current, tsk);
}
read_unlock(&tasklist_lock);
notask:
retval = wo->notask_error;
if (!retval && !(wo->wo_flags & WNOHANG))
return -ERESTARTSYS;
return retval;
}
static long do_wait(struct wait_opts *wo)
{
int retval;
trace_sched_process_wait(wo->wo_pid);
init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
wo->child_wait.private = current;
add_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait);
do {
set_current_state(TASK_INTERRUPTIBLE);
retval = __do_wait(wo);
if (retval != -ERESTARTSYS)
break;
if (signal_pending(current))
break;
schedule();
} while (1);
__set_current_state(TASK_RUNNING);
remove_wait_queue(¤t->signal->wait_chldexit, &wo->child_wait);
return retval;
}
int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid,
struct waitid_info *infop, int options,
struct rusage *ru)
{
unsigned int f_flags = 0;
struct pid *pid = NULL;
enum pid_type type;
if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
__WNOTHREAD|__WCLONE|__WALL))
return -EINVAL;
if (!(options & (WEXITED|WSTOPPED|WCONTINUED)))
return -EINVAL;
switch (which) {
case P_ALL:
type = PIDTYPE_MAX;
break;
case P_PID:
type = PIDTYPE_PID;
if (upid <= 0)
return -EINVAL;
pid = find_get_pid(upid);
break;
case P_PGID:
type = PIDTYPE_PGID;
if (upid < 0)
return -EINVAL;
if (upid)
pid = find_get_pid(upid);
else
pid = get_task_pid(current, PIDTYPE_PGID);
break;
case P_PIDFD:
type = PIDTYPE_PID;
if (upid < 0)
return -EINVAL;
pid = pidfd_get_pid(upid, &f_flags);
if (IS_ERR(pid))
return PTR_ERR(pid);
break;
default:
return -EINVAL;
}
wo->wo_type = type;
wo->wo_pid = pid;
wo->wo_flags = options;
wo->wo_info = infop;
wo->wo_rusage = ru;
if (f_flags & O_NONBLOCK)
wo->wo_flags |= WNOHANG;
return 0;
}
static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
int options, struct rusage *ru)
{
struct wait_opts wo;
long ret;
ret = kernel_waitid_prepare(&wo, which, upid, infop, options, ru);
if (ret)
return ret;
ret = do_wait(&wo);
if (!ret && !(options & WNOHANG) && (wo.wo_flags & WNOHANG))
ret = -EAGAIN;
put_pid(wo.wo_pid);
return ret;
}
SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
infop, int, options, struct rusage __user *, ru)
{
struct rusage r;
struct waitid_info info = {.status = 0};
long err = kernel_waitid(which, upid, &info, options, ru ? &r : NULL);
int signo = 0;
if (err > 0) {
signo = SIGCHLD;
err = 0;
if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
return -EFAULT;
}
if (!infop)
return err;
if (!user_write_access_begin(infop, sizeof(*infop)))
return -EFAULT;
unsafe_put_user(signo, &infop->si_signo, Efault);
unsafe_put_user(0, &infop->si_errno, Efault);
unsafe_put_user(info.cause, &infop->si_code, Efault);
unsafe_put_user(info.pid, &infop->si_pid, Efault);
unsafe_put_user(info.uid, &infop->si_uid, Efault);
unsafe_put_user(info.status, &infop->si_status, Efault);
user_write_access_end();
return err;
Efault:
user_write_access_end();
return -EFAULT;
}
long kernel_wait4(pid_t upid, int __user *stat_addr, int options,
struct rusage *ru)
{
struct wait_opts wo;
struct pid *pid = NULL;
enum pid_type type;
long ret;
if (options & ~(WNOHANG|WUNTRACED|WCONTINUED|
__WNOTHREAD|__WCLONE|__WALL))
return -EINVAL;
/* -INT_MIN is not defined */
if (upid == INT_MIN)
return -ESRCH;
if (upid == -1)
type = PIDTYPE_MAX;
else if (upid < 0) {
type = PIDTYPE_PGID;
pid = find_get_pid(-upid);
} else if (upid == 0) {
type = PIDTYPE_PGID;
pid = get_task_pid(current, PIDTYPE_PGID);
} else /* upid > 0 */ {
type = PIDTYPE_PID;
pid = find_get_pid(upid);
}
wo.wo_type = type;
wo.wo_pid = pid;
wo.wo_flags = options | WEXITED;
wo.wo_info = NULL;
wo.wo_stat = 0;
wo.wo_rusage = ru;
ret = do_wait(&wo);
put_pid(pid);
if (ret > 0 && stat_addr && put_user(wo.wo_stat, stat_addr))
ret = -EFAULT;
return ret;
}
int kernel_wait(pid_t pid, int *stat)
{
struct wait_opts wo = {
.wo_type = PIDTYPE_PID,
.wo_pid = find_get_pid(pid),
.wo_flags = WEXITED,
};
int ret;
ret = do_wait(&wo);
if (ret > 0 && wo.wo_stat)
*stat = wo.wo_stat;
put_pid(wo.wo_pid);
return ret;
}
SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
int, options, struct rusage __user *, ru)
{
struct rusage r;
long err = kernel_wait4(upid, stat_addr, options, ru ? &r : NULL);
if (err > 0) {
if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
return -EFAULT;
}
return err;
}
#ifdef __ARCH_WANT_SYS_WAITPID
/*
* sys_waitpid() remains for compatibility. waitpid() should be
* implemented by calling sys_wait4() from libc.a.
*/
SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options)
{
return kernel_wait4(pid, stat_addr, options, NULL);
}
#endif
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(wait4,
compat_pid_t, pid,
compat_uint_t __user *, stat_addr,
int, options,
struct compat_rusage __user *, ru)
{
struct rusage r;
long err = kernel_wait4(pid, stat_addr, options, ru ? &r : NULL);
if (err > 0) {
if (ru && put_compat_rusage(&r, ru))
return -EFAULT;
}
return err;
}
COMPAT_SYSCALL_DEFINE5(waitid,
int, which, compat_pid_t, pid,
struct compat_siginfo __user *, infop, int, options,
struct compat_rusage __user *, uru)
{
struct rusage ru;
struct waitid_info info = {.status = 0};
long err = kernel_waitid(which, pid, &info, options, uru ? &ru : NULL);
int signo = 0;
if (err > 0) {
signo = SIGCHLD;
err = 0;
if (uru) {
/* kernel_waitid() overwrites everything in ru */
if (COMPAT_USE_64BIT_TIME)
err = copy_to_user(uru, &ru, sizeof(ru));
else
err = put_compat_rusage(&ru, uru);
if (err)
return -EFAULT;
}
}
if (!infop)
return err;
if (!user_write_access_begin(infop, sizeof(*infop)))
return -EFAULT;
unsafe_put_user(signo, &infop->si_signo, Efault);
unsafe_put_user(0, &infop->si_errno, Efault);
unsafe_put_user(info.cause, &infop->si_code, Efault);
unsafe_put_user(info.pid, &infop->si_pid, Efault);
unsafe_put_user(info.uid, &infop->si_uid, Efault);
unsafe_put_user(info.status, &infop->si_status, Efault);
user_write_access_end();
return err;
Efault:
user_write_access_end();
return -EFAULT;
}
#endif
/*
* This needs to be __function_aligned as GCC implicitly makes any
* implementation of abort() cold and drops alignment specified by
* -falign-functions=N.
*
* See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345#c11
*/
__weak __function_aligned void abort(void)
{
BUG();
/* if that doesn't kill us, halt */
panic("Oops failed to kill thread");
}
EXPORT_SYMBOL(abort);
// SPDX-License-Identifier: GPL-2.0-or-later
//
// core.c -- Voltage/Current Regulator framework.
//
// Copyright 2007, 2008 Wolfson Microelectronics PLC.
// Copyright 2008 SlimLogic Ltd.
//
// Author: Liam Girdwood <lrg@slimlogic.co.uk>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/async.h>
#include <linux/err.h>
#include <linux/mutex.h>
#include <linux/suspend.h>
#include <linux/delay.h>
#include <linux/gpio/consumer.h>
#include <linux/of.h>
#include <linux/reboot.h>
#include <linux/regmap.h>
#include <linux/regulator/of_regulator.h>
#include <linux/regulator/consumer.h>
#include <linux/regulator/coupler.h>
#include <linux/regulator/driver.h>
#include <linux/regulator/machine.h>
#include <linux/module.h>
#define CREATE_TRACE_POINTS
#include <trace/events/regulator.h>
#include "dummy.h"
#include "internal.h"
#include "regnl.h"
static DEFINE_WW_CLASS(regulator_ww_class);
static DEFINE_MUTEX(regulator_nesting_mutex);
static DEFINE_MUTEX(regulator_list_mutex);
static LIST_HEAD(regulator_map_list);
static LIST_HEAD(regulator_ena_gpio_list);
static LIST_HEAD(regulator_supply_alias_list);
static LIST_HEAD(regulator_coupler_list);
static bool has_full_constraints;
static struct dentry *debugfs_root;
/*
* struct regulator_map
*
* Used to provide symbolic supply names to devices.
*/
struct regulator_map {
struct list_head list;
const char *dev_name; /* The dev_name() for the consumer */
const char *supply;
struct regulator_dev *regulator;
};
/*
* struct regulator_enable_gpio
*
* Management for shared enable GPIO pin
*/
struct regulator_enable_gpio {
struct list_head list;
struct gpio_desc *gpiod;
u32 enable_count; /* a number of enabled shared GPIO */
u32 request_count; /* a number of requested shared GPIO */
};
/*
* struct regulator_supply_alias
*
* Used to map lookups for a supply onto an alternative device.
*/
struct regulator_supply_alias {
struct list_head list;
struct device *src_dev;
const char *src_supply;
struct device *alias_dev;
const char *alias_supply;
};
static int _regulator_is_enabled(struct regulator_dev *rdev);
static int _regulator_disable(struct regulator *regulator);
static int _regulator_get_error_flags(struct regulator_dev *rdev, unsigned int *flags);
static int _regulator_get_current_limit(struct regulator_dev *rdev);
static unsigned int _regulator_get_mode(struct regulator_dev *rdev);
static int _notifier_call_chain(struct regulator_dev *rdev,
unsigned long event, void *data);
static int _regulator_do_set_voltage(struct regulator_dev *rdev,
int min_uV, int max_uV);
static int regulator_balance_voltage(struct regulator_dev *rdev,
suspend_state_t state);
static struct regulator *create_regulator(struct regulator_dev *rdev,
struct device *dev,
const char *supply_name);
static void destroy_regulator(struct regulator *regulator);
static void _regulator_put(struct regulator *regulator);
const char *rdev_get_name(struct regulator_dev *rdev)
{
if (rdev->constraints && rdev->constraints->name)
return rdev->constraints->name;
else if (rdev->desc->name)
return rdev->desc->name;
else
return "";
}
EXPORT_SYMBOL_GPL(rdev_get_name);
static bool have_full_constraints(void)
{
return has_full_constraints || of_have_populated_dt();
}
static bool regulator_ops_is_valid(struct regulator_dev *rdev, int ops)
{
if (!rdev->constraints) {
rdev_err(rdev, "no constraints\n");
return false;
}
if (rdev->constraints->valid_ops_mask & ops)
return true;
return false;
}
/**
* regulator_lock_nested - lock a single regulator
* @rdev: regulator source
* @ww_ctx: w/w mutex acquire context
*
* This function can be called many times by one task on
* a single regulator and its mutex will be locked only
* once. If a task, which is calling this function is other
* than the one, which initially locked the mutex, it will
* wait on mutex.
*/
static inline int regulator_lock_nested(struct regulator_dev *rdev,
struct ww_acquire_ctx *ww_ctx)
{
bool lock = false;
int ret = 0;
mutex_lock(®ulator_nesting_mutex);
if (!ww_mutex_trylock(&rdev->mutex, ww_ctx)) {
if (rdev->mutex_owner == current)
rdev->ref_cnt++;
else
lock = true;
if (lock) {
mutex_unlock(®ulator_nesting_mutex);
ret = ww_mutex_lock(&rdev->mutex, ww_ctx);
mutex_lock(®ulator_nesting_mutex);
}
} else {
lock = true;
}
if (lock && ret != -EDEADLK) {
rdev->ref_cnt++;
rdev->mutex_owner = current;
}
mutex_unlock(®ulator_nesting_mutex);
return ret;
}
/**
* regulator_lock - lock a single regulator
* @rdev: regulator source
*
* This function can be called many times by one task on
* a single regulator and its mutex will be locked only
* once. If a task, which is calling this function is other
* than the one, which initially locked the mutex, it will
* wait on mutex.
*/
static void regulator_lock(struct regulator_dev *rdev)
{
regulator_lock_nested(rdev, NULL);
}
/**
* regulator_unlock - unlock a single regulator
* @rdev: regulator_source
*
* This function unlocks the mutex when the
* reference counter reaches 0.
*/
static void regulator_unlock(struct regulator_dev *rdev)
{
mutex_lock(®ulator_nesting_mutex);
if (--rdev->ref_cnt == 0) {
rdev->mutex_owner = NULL;
ww_mutex_unlock(&rdev->mutex);
}
WARN_ON_ONCE(rdev->ref_cnt < 0);
mutex_unlock(®ulator_nesting_mutex);
}
/**
* regulator_lock_two - lock two regulators
* @rdev1: first regulator
* @rdev2: second regulator
* @ww_ctx: w/w mutex acquire context
*
* Locks both rdevs using the regulator_ww_class.
*/
static void regulator_lock_two(struct regulator_dev *rdev1,
struct regulator_dev *rdev2,
struct ww_acquire_ctx *ww_ctx)
{
struct regulator_dev *held, *contended;
int ret;
ww_acquire_init(ww_ctx, ®ulator_ww_class);
/* Try to just grab both of them */
ret = regulator_lock_nested(rdev1, ww_ctx);
WARN_ON(ret);
ret = regulator_lock_nested(rdev2, ww_ctx);
if (ret != -EDEADLOCK) {
WARN_ON(ret);
goto exit;
}
held = rdev1;
contended = rdev2;
while (true) {
regulator_unlock(held);
ww_mutex_lock_slow(&contended->mutex, ww_ctx);
contended->ref_cnt++;
contended->mutex_owner = current;
swap(held, contended);
ret = regulator_lock_nested(contended, ww_ctx);
if (ret != -EDEADLOCK) {
WARN_ON(ret);
break;
}
}
exit:
ww_acquire_done(ww_ctx);
}
/**
* regulator_unlock_two - unlock two regulators
* @rdev1: first regulator
* @rdev2: second regulator
* @ww_ctx: w/w mutex acquire context
*
* The inverse of regulator_lock_two().
*/
static void regulator_unlock_two(struct regulator_dev *rdev1,
struct regulator_dev *rdev2,
struct ww_acquire_ctx *ww_ctx)
{
regulator_unlock(rdev2);
regulator_unlock(rdev1);
ww_acquire_fini(ww_ctx);
}
static bool regulator_supply_is_couple(struct regulator_dev *rdev)
{
struct regulator_dev *c_rdev;
int i;
for (i = 1; i < rdev->coupling_desc.n_coupled; i++) {
c_rdev = rdev->coupling_desc.coupled_rdevs[i];
if (rdev->supply->rdev == c_rdev)
return true;
}
return false;
}
static void regulator_unlock_recursive(struct regulator_dev *rdev,
unsigned int n_coupled)
{
struct regulator_dev *c_rdev, *supply_rdev;
int i, supply_n_coupled;
for (i = n_coupled; i > 0; i--) {
c_rdev = rdev->coupling_desc.coupled_rdevs[i - 1];
if (!c_rdev)
continue;
if (c_rdev->supply && !regulator_supply_is_couple(c_rdev)) {
supply_rdev = c_rdev->supply->rdev;
supply_n_coupled = supply_rdev->coupling_desc.n_coupled;
regulator_unlock_recursive(supply_rdev,
supply_n_coupled);
}
regulator_unlock(c_rdev);
}
}
static int regulator_lock_recursive(struct regulator_dev *rdev,
struct regulator_dev **new_contended_rdev,
struct regulator_dev **old_contended_rdev,
struct ww_acquire_ctx *ww_ctx)
{
struct regulator_dev *c_rdev;
int i, err;
for (i = 0; i < rdev->coupling_desc.n_coupled; i++) {
c_rdev = rdev->coupling_desc.coupled_rdevs[i];
if (!c_rdev)
continue;
if (c_rdev != *old_contended_rdev) {
err = regulator_lock_nested(c_rdev, ww_ctx);
if (err) {
if (err == -EDEADLK) {
*new_contended_rdev = c_rdev;
goto err_unlock;
}
/* shouldn't happen */
WARN_ON_ONCE(err != -EALREADY);
}
} else {
*old_contended_rdev = NULL;
}
if (c_rdev->supply && !regulator_supply_is_couple(c_rdev)) {
err = regulator_lock_recursive(c_rdev->supply->rdev,
new_contended_rdev,
old_contended_rdev,
ww_ctx);
if (err) {
regulator_unlock(c_rdev);
goto err_unlock;
}
}
}
return 0;
err_unlock:
regulator_unlock_recursive(rdev, i);
return err;
}
/**
* regulator_unlock_dependent - unlock regulator's suppliers and coupled
* regulators
* @rdev: regulator source
* @ww_ctx: w/w mutex acquire context
*
* Unlock all regulators related with rdev by coupling or supplying.
*/
static void regulator_unlock_dependent(struct regulator_dev *rdev,
struct ww_acquire_ctx *ww_ctx)
{
regulator_unlock_recursive(rdev, rdev->coupling_desc.n_coupled);
ww_acquire_fini(ww_ctx);
}
/**
* regulator_lock_dependent - lock regulator's suppliers and coupled regulators
* @rdev: regulator source
* @ww_ctx: w/w mutex acquire context
*
* This function as a wrapper on regulator_lock_recursive(), which locks
* all regulators related with rdev by coupling or supplying.
*/
static void regulator_lock_dependent(struct regulator_dev *rdev,
struct ww_acquire_ctx *ww_ctx)
{
struct regulator_dev *new_contended_rdev = NULL;
struct regulator_dev *old_contended_rdev = NULL;
int err;
mutex_lock(®ulator_list_mutex);
ww_acquire_init(ww_ctx, ®ulator_ww_class);
do {
if (new_contended_rdev) {
ww_mutex_lock_slow(&new_contended_rdev->mutex, ww_ctx);
old_contended_rdev = new_contended_rdev;
old_contended_rdev->ref_cnt++;
old_contended_rdev->mutex_owner = current;
}
err = regulator_lock_recursive(rdev,
&new_contended_rdev,
&old_contended_rdev,
ww_ctx);
if (old_contended_rdev)
regulator_unlock(old_contended_rdev);
} while (err == -EDEADLK);
ww_acquire_done(ww_ctx);
mutex_unlock(®ulator_list_mutex);
}
/**
* of_get_child_regulator - get a child regulator device node
* based on supply name
* @parent: Parent device node
* @prop_name: Combination regulator supply name and "-supply"
*
* Traverse all child nodes.
* Extract the child regulator device node corresponding to the supply name.
* returns the device node corresponding to the regulator if found, else
* returns NULL.
*/
static struct device_node *of_get_child_regulator(struct device_node *parent,
const char *prop_name)
{
struct device_node *regnode = NULL;
struct device_node *child = NULL;
for_each_child_of_node(parent, child) {
regnode = of_parse_phandle(child, prop_name, 0);
if (!regnode) {
regnode = of_get_child_regulator(child, prop_name);
if (regnode)
goto err_node_put;
} else {
goto err_node_put;
}
}
return NULL;
err_node_put:
of_node_put(child);
return regnode;
}
/**
* of_get_regulator - get a regulator device node based on supply name
* @dev: Device pointer for the consumer (of regulator) device
* @supply: regulator supply name
*
* Extract the regulator device node corresponding to the supply name.
* returns the device node corresponding to the regulator if found, else
* returns NULL.
*/
static struct device_node *of_get_regulator(struct device *dev, const char *supply)
{
struct device_node *regnode = NULL;
char prop_name[64]; /* 64 is max size of property name */
dev_dbg(dev, "Looking up %s-supply from device tree\n", supply);
snprintf(prop_name, 64, "%s-supply", supply);
regnode = of_parse_phandle(dev->of_node, prop_name, 0);
if (!regnode) {
regnode = of_get_child_regulator(dev->of_node, prop_name);
if (regnode)
return regnode;
dev_dbg(dev, "Looking up %s property in node %pOF failed\n",
prop_name, dev->of_node);
return NULL;
}
return regnode;
}
/* Platform voltage constraint check */
int regulator_check_voltage(struct regulator_dev *rdev,
int *min_uV, int *max_uV)
{
BUG_ON(*min_uV > *max_uV);
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
rdev_err(rdev, "voltage operation not allowed\n");
return -EPERM;
}
if (*max_uV > rdev->constraints->max_uV)
*max_uV = rdev->constraints->max_uV;
if (*min_uV < rdev->constraints->min_uV)
*min_uV = rdev->constraints->min_uV;
if (*min_uV > *max_uV) {
rdev_err(rdev, "unsupportable voltage range: %d-%duV\n",
*min_uV, *max_uV);
return -EINVAL;
}
return 0;
}
/* return 0 if the state is valid */
static int regulator_check_states(suspend_state_t state)
{
return (state > PM_SUSPEND_MAX || state == PM_SUSPEND_TO_IDLE);
}
/* Make sure we select a voltage that suits the needs of all
* regulator consumers
*/
int regulator_check_consumers(struct regulator_dev *rdev,
int *min_uV, int *max_uV,
suspend_state_t state)
{
struct regulator *regulator;
struct regulator_voltage *voltage;
list_for_each_entry(regulator, &rdev->consumer_list, list) {
voltage = ®ulator->voltage[state];
/*
* Assume consumers that didn't say anything are OK
* with anything in the constraint range.
*/
if (!voltage->min_uV && !voltage->max_uV)
continue;
if (*max_uV > voltage->max_uV)
*max_uV = voltage->max_uV;
if (*min_uV < voltage->min_uV)
*min_uV = voltage->min_uV;
}
if (*min_uV > *max_uV) {
rdev_err(rdev, "Restricting voltage, %u-%uuV\n",
*min_uV, *max_uV);
return -EINVAL;
}
return 0;
}
/* current constraint check */
static int regulator_check_current_limit(struct regulator_dev *rdev,
int *min_uA, int *max_uA)
{
BUG_ON(*min_uA > *max_uA);
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_CURRENT)) {
rdev_err(rdev, "current operation not allowed\n");
return -EPERM;
}
if (*max_uA > rdev->constraints->max_uA)
*max_uA = rdev->constraints->max_uA;
if (*min_uA < rdev->constraints->min_uA)
*min_uA = rdev->constraints->min_uA;
if (*min_uA > *max_uA) {
rdev_err(rdev, "unsupportable current range: %d-%duA\n",
*min_uA, *max_uA);
return -EINVAL;
}
return 0;
}
/* operating mode constraint check */
static int regulator_mode_constrain(struct regulator_dev *rdev,
unsigned int *mode)
{
switch (*mode) {
case REGULATOR_MODE_FAST:
case REGULATOR_MODE_NORMAL:
case REGULATOR_MODE_IDLE:
case REGULATOR_MODE_STANDBY:
break;
default:
rdev_err(rdev, "invalid mode %x specified\n", *mode);
return -EINVAL;
}
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_MODE)) {
rdev_err(rdev, "mode operation not allowed\n");
return -EPERM;
}
/* The modes are bitmasks, the most power hungry modes having
* the lowest values. If the requested mode isn't supported
* try higher modes.
*/
while (*mode) {
if (rdev->constraints->valid_modes_mask & *mode)
return 0;
*mode /= 2;
}
return -EINVAL;
}
static inline struct regulator_state *
regulator_get_suspend_state(struct regulator_dev *rdev, suspend_state_t state)
{
if (rdev->constraints == NULL)
return NULL;
switch (state) {
case PM_SUSPEND_STANDBY:
return &rdev->constraints->state_standby;
case PM_SUSPEND_MEM:
return &rdev->constraints->state_mem;
case PM_SUSPEND_MAX:
return &rdev->constraints->state_disk;
default:
return NULL;
}
}
static const struct regulator_state *
regulator_get_suspend_state_check(struct regulator_dev *rdev, suspend_state_t state)
{
const struct regulator_state *rstate;
rstate = regulator_get_suspend_state(rdev, state);
if (rstate == NULL)
return NULL;
/* If we have no suspend mode configuration don't set anything;
* only warn if the driver implements set_suspend_voltage or
* set_suspend_mode callback.
*/
if (rstate->enabled != ENABLE_IN_SUSPEND &&
rstate->enabled != DISABLE_IN_SUSPEND) {
if (rdev->desc->ops->set_suspend_voltage ||
rdev->desc->ops->set_suspend_mode)
rdev_warn(rdev, "No configuration\n");
return NULL;
}
return rstate;
}
static ssize_t microvolts_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
int uV;
regulator_lock(rdev);
uV = regulator_get_voltage_rdev(rdev);
regulator_unlock(rdev);
if (uV < 0)
return uV;
return sprintf(buf, "%d\n", uV);
}
static DEVICE_ATTR_RO(microvolts);
static ssize_t microamps_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return sprintf(buf, "%d\n", _regulator_get_current_limit(rdev));
}
static DEVICE_ATTR_RO(microamps);
static ssize_t name_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return sprintf(buf, "%s\n", rdev_get_name(rdev));
}
static DEVICE_ATTR_RO(name);
static const char *regulator_opmode_to_str(int mode)
{
switch (mode) {
case REGULATOR_MODE_FAST:
return "fast";
case REGULATOR_MODE_NORMAL:
return "normal";
case REGULATOR_MODE_IDLE:
return "idle";
case REGULATOR_MODE_STANDBY:
return "standby";
}
return "unknown";
}
static ssize_t regulator_print_opmode(char *buf, int mode)
{
return sprintf(buf, "%s\n", regulator_opmode_to_str(mode));
}
static ssize_t opmode_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_opmode(buf, _regulator_get_mode(rdev));
}
static DEVICE_ATTR_RO(opmode);
static ssize_t regulator_print_state(char *buf, int state)
{
if (state > 0)
return sprintf(buf, "enabled\n");
else if (state == 0)
return sprintf(buf, "disabled\n");
else
return sprintf(buf, "unknown\n");
}
static ssize_t state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
ssize_t ret;
regulator_lock(rdev);
ret = regulator_print_state(buf, _regulator_is_enabled(rdev));
regulator_unlock(rdev);
return ret;
}
static DEVICE_ATTR_RO(state);
static ssize_t status_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
int status;
char *label;
status = rdev->desc->ops->get_status(rdev);
if (status < 0)
return status;
switch (status) {
case REGULATOR_STATUS_OFF:
label = "off";
break;
case REGULATOR_STATUS_ON:
label = "on";
break;
case REGULATOR_STATUS_ERROR:
label = "error";
break;
case REGULATOR_STATUS_FAST:
label = "fast";
break;
case REGULATOR_STATUS_NORMAL:
label = "normal";
break;
case REGULATOR_STATUS_IDLE:
label = "idle";
break;
case REGULATOR_STATUS_STANDBY:
label = "standby";
break;
case REGULATOR_STATUS_BYPASS:
label = "bypass";
break;
case REGULATOR_STATUS_UNDEFINED:
label = "undefined";
break;
default:
return -ERANGE;
}
return sprintf(buf, "%s\n", label);
}
static DEVICE_ATTR_RO(status);
static ssize_t min_microamps_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
if (!rdev->constraints)
return sprintf(buf, "constraint not defined\n");
return sprintf(buf, "%d\n", rdev->constraints->min_uA);
}
static DEVICE_ATTR_RO(min_microamps);
static ssize_t max_microamps_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
if (!rdev->constraints)
return sprintf(buf, "constraint not defined\n");
return sprintf(buf, "%d\n", rdev->constraints->max_uA);
}
static DEVICE_ATTR_RO(max_microamps);
static ssize_t min_microvolts_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
if (!rdev->constraints)
return sprintf(buf, "constraint not defined\n");
return sprintf(buf, "%d\n", rdev->constraints->min_uV);
}
static DEVICE_ATTR_RO(min_microvolts);
static ssize_t max_microvolts_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
if (!rdev->constraints)
return sprintf(buf, "constraint not defined\n");
return sprintf(buf, "%d\n", rdev->constraints->max_uV);
}
static DEVICE_ATTR_RO(max_microvolts);
static ssize_t requested_microamps_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
struct regulator *regulator;
int uA = 0;
regulator_lock(rdev);
list_for_each_entry(regulator, &rdev->consumer_list, list) {
if (regulator->enable_count)
uA += regulator->uA_load;
}
regulator_unlock(rdev);
return sprintf(buf, "%d\n", uA);
}
static DEVICE_ATTR_RO(requested_microamps);
static ssize_t num_users_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return sprintf(buf, "%d\n", rdev->use_count);
}
static DEVICE_ATTR_RO(num_users);
static ssize_t type_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
switch (rdev->desc->type) {
case REGULATOR_VOLTAGE:
return sprintf(buf, "voltage\n");
case REGULATOR_CURRENT:
return sprintf(buf, "current\n");
}
return sprintf(buf, "unknown\n");
}
static DEVICE_ATTR_RO(type);
static ssize_t suspend_mem_microvolts_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return sprintf(buf, "%d\n", rdev->constraints->state_mem.uV);
}
static DEVICE_ATTR_RO(suspend_mem_microvolts);
static ssize_t suspend_disk_microvolts_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return sprintf(buf, "%d\n", rdev->constraints->state_disk.uV);
}
static DEVICE_ATTR_RO(suspend_disk_microvolts);
static ssize_t suspend_standby_microvolts_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return sprintf(buf, "%d\n", rdev->constraints->state_standby.uV);
}
static DEVICE_ATTR_RO(suspend_standby_microvolts);
static ssize_t suspend_mem_mode_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_opmode(buf,
rdev->constraints->state_mem.mode);
}
static DEVICE_ATTR_RO(suspend_mem_mode);
static ssize_t suspend_disk_mode_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_opmode(buf,
rdev->constraints->state_disk.mode);
}
static DEVICE_ATTR_RO(suspend_disk_mode);
static ssize_t suspend_standby_mode_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_opmode(buf,
rdev->constraints->state_standby.mode);
}
static DEVICE_ATTR_RO(suspend_standby_mode);
static ssize_t suspend_mem_state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_state(buf,
rdev->constraints->state_mem.enabled);
}
static DEVICE_ATTR_RO(suspend_mem_state);
static ssize_t suspend_disk_state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_state(buf,
rdev->constraints->state_disk.enabled);
}
static DEVICE_ATTR_RO(suspend_disk_state);
static ssize_t suspend_standby_state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
return regulator_print_state(buf,
rdev->constraints->state_standby.enabled);
}
static DEVICE_ATTR_RO(suspend_standby_state);
static ssize_t bypass_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
const char *report;
bool bypass;
int ret;
ret = rdev->desc->ops->get_bypass(rdev, &bypass);
if (ret != 0)
report = "unknown";
else if (bypass)
report = "enabled";
else
report = "disabled";
return sprintf(buf, "%s\n", report);
}
static DEVICE_ATTR_RO(bypass);
#define REGULATOR_ERROR_ATTR(name, bit) \
static ssize_t name##_show(struct device *dev, struct device_attribute *attr, \
char *buf) \
{ \
int ret; \
unsigned int flags; \
struct regulator_dev *rdev = dev_get_drvdata(dev); \
ret = _regulator_get_error_flags(rdev, &flags); \
if (ret) \
return ret; \
return sysfs_emit(buf, "%d\n", !!(flags & (bit))); \
} \
static DEVICE_ATTR_RO(name)
REGULATOR_ERROR_ATTR(under_voltage, REGULATOR_ERROR_UNDER_VOLTAGE);
REGULATOR_ERROR_ATTR(over_current, REGULATOR_ERROR_OVER_CURRENT);
REGULATOR_ERROR_ATTR(regulation_out, REGULATOR_ERROR_REGULATION_OUT);
REGULATOR_ERROR_ATTR(fail, REGULATOR_ERROR_FAIL);
REGULATOR_ERROR_ATTR(over_temp, REGULATOR_ERROR_OVER_TEMP);
REGULATOR_ERROR_ATTR(under_voltage_warn, REGULATOR_ERROR_UNDER_VOLTAGE_WARN);
REGULATOR_ERROR_ATTR(over_current_warn, REGULATOR_ERROR_OVER_CURRENT_WARN);
REGULATOR_ERROR_ATTR(over_voltage_warn, REGULATOR_ERROR_OVER_VOLTAGE_WARN);
REGULATOR_ERROR_ATTR(over_temp_warn, REGULATOR_ERROR_OVER_TEMP_WARN);
/* Calculate the new optimum regulator operating mode based on the new total
* consumer load. All locks held by caller
*/
static int drms_uA_update(struct regulator_dev *rdev)
{
struct regulator *sibling;
int current_uA = 0, output_uV, input_uV, err;
unsigned int mode;
/*
* first check to see if we can set modes at all, otherwise just
* tell the consumer everything is OK.
*/
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_DRMS)) {
rdev_dbg(rdev, "DRMS operation not allowed\n");
return 0;
}
if (!rdev->desc->ops->get_optimum_mode &&
!rdev->desc->ops->set_load)
return 0;
if (!rdev->desc->ops->set_mode &&
!rdev->desc->ops->set_load)
return -EINVAL;
/* calc total requested load */
list_for_each_entry(sibling, &rdev->consumer_list, list) {
if (sibling->enable_count)
current_uA += sibling->uA_load;
}
current_uA += rdev->constraints->system_load;
if (rdev->desc->ops->set_load) {
/* set the optimum mode for our new total regulator load */
err = rdev->desc->ops->set_load(rdev, current_uA);
if (err < 0)
rdev_err(rdev, "failed to set load %d: %pe\n",
current_uA, ERR_PTR(err));
} else {
/*
* Unfortunately in some cases the constraints->valid_ops has
* REGULATOR_CHANGE_DRMS but there are no valid modes listed.
* That's not really legit but we won't consider it a fatal
* error here. We'll treat it as if REGULATOR_CHANGE_DRMS
* wasn't set.
*/
if (!rdev->constraints->valid_modes_mask) {
rdev_dbg(rdev, "Can change modes; but no valid mode\n");
return 0;
}
/* get output voltage */
output_uV = regulator_get_voltage_rdev(rdev);
/*
* Don't return an error; if regulator driver cares about
* output_uV then it's up to the driver to validate.
*/
if (output_uV <= 0)
rdev_dbg(rdev, "invalid output voltage found\n");
/* get input voltage */
input_uV = 0;
if (rdev->supply)
input_uV = regulator_get_voltage_rdev(rdev->supply->rdev);
if (input_uV <= 0)
input_uV = rdev->constraints->input_uV;
/*
* Don't return an error; if regulator driver cares about
* input_uV then it's up to the driver to validate.
*/
if (input_uV <= 0)
rdev_dbg(rdev, "invalid input voltage found\n");
/* now get the optimum mode for our new total regulator load */
mode = rdev->desc->ops->get_optimum_mode(rdev, input_uV,
output_uV, current_uA);
/* check the new mode is allowed */
err = regulator_mode_constrain(rdev, &mode);
if (err < 0) {
rdev_err(rdev, "failed to get optimum mode @ %d uA %d -> %d uV: %pe\n",
current_uA, input_uV, output_uV, ERR_PTR(err));
return err;
}
err = rdev->desc->ops->set_mode(rdev, mode);
if (err < 0)
rdev_err(rdev, "failed to set optimum mode %x: %pe\n",
mode, ERR_PTR(err));
}
return err;
}
static int __suspend_set_state(struct regulator_dev *rdev,
const struct regulator_state *rstate)
{
int ret = 0;
if (rstate->enabled == ENABLE_IN_SUSPEND &&
rdev->desc->ops->set_suspend_enable)
ret = rdev->desc->ops->set_suspend_enable(rdev);
else if (rstate->enabled == DISABLE_IN_SUSPEND &&
rdev->desc->ops->set_suspend_disable)
ret = rdev->desc->ops->set_suspend_disable(rdev);
else /* OK if set_suspend_enable or set_suspend_disable is NULL */
ret = 0;
if (ret < 0) {
rdev_err(rdev, "failed to enabled/disable: %pe\n", ERR_PTR(ret));
return ret;
}
if (rdev->desc->ops->set_suspend_voltage && rstate->uV > 0) {
ret = rdev->desc->ops->set_suspend_voltage(rdev, rstate->uV);
if (ret < 0) {
rdev_err(rdev, "failed to set voltage: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->desc->ops->set_suspend_mode && rstate->mode > 0) {
ret = rdev->desc->ops->set_suspend_mode(rdev, rstate->mode);
if (ret < 0) {
rdev_err(rdev, "failed to set mode: %pe\n", ERR_PTR(ret));
return ret;
}
}
return ret;
}
static int suspend_set_initial_state(struct regulator_dev *rdev)
{
const struct regulator_state *rstate;
rstate = regulator_get_suspend_state_check(rdev,
rdev->constraints->initial_state);
if (!rstate)
return 0;
return __suspend_set_state(rdev, rstate);
}
#if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG)
static void print_constraints_debug(struct regulator_dev *rdev)
{
struct regulation_constraints *constraints = rdev->constraints;
char buf[160] = "";
size_t len = sizeof(buf) - 1;
int count = 0;
int ret;
if (constraints->min_uV && constraints->max_uV) {
if (constraints->min_uV == constraints->max_uV)
count += scnprintf(buf + count, len - count, "%d mV ",
constraints->min_uV / 1000);
else
count += scnprintf(buf + count, len - count,
"%d <--> %d mV ",
constraints->min_uV / 1000,
constraints->max_uV / 1000);
}
if (!constraints->min_uV ||
constraints->min_uV != constraints->max_uV) {
ret = regulator_get_voltage_rdev(rdev);
if (ret > 0)
count += scnprintf(buf + count, len - count,
"at %d mV ", ret / 1000);
}
if (constraints->uV_offset)
count += scnprintf(buf + count, len - count, "%dmV offset ",
constraints->uV_offset / 1000);
if (constraints->min_uA && constraints->max_uA) {
if (constraints->min_uA == constraints->max_uA)
count += scnprintf(buf + count, len - count, "%d mA ",
constraints->min_uA / 1000);
else
count += scnprintf(buf + count, len - count,
"%d <--> %d mA ",
constraints->min_uA / 1000,
constraints->max_uA / 1000);
}
if (!constraints->min_uA ||
constraints->min_uA != constraints->max_uA) {
ret = _regulator_get_current_limit(rdev);
if (ret > 0)
count += scnprintf(buf + count, len - count,
"at %d mA ", ret / 1000);
}
if (constraints->valid_modes_mask & REGULATOR_MODE_FAST)
count += scnprintf(buf + count, len - count, "fast ");
if (constraints->valid_modes_mask & REGULATOR_MODE_NORMAL)
count += scnprintf(buf + count, len - count, "normal ");
if (constraints->valid_modes_mask & REGULATOR_MODE_IDLE)
count += scnprintf(buf + count, len - count, "idle ");
if (constraints->valid_modes_mask & REGULATOR_MODE_STANDBY)
count += scnprintf(buf + count, len - count, "standby ");
if (!count)
count = scnprintf(buf, len, "no parameters");
else
--count;
count += scnprintf(buf + count, len - count, ", %s",
_regulator_is_enabled(rdev) ? "enabled" : "disabled");
rdev_dbg(rdev, "%s\n", buf);
}
#else /* !DEBUG && !CONFIG_DYNAMIC_DEBUG */
static inline void print_constraints_debug(struct regulator_dev *rdev) {}
#endif /* !DEBUG && !CONFIG_DYNAMIC_DEBUG */
static void print_constraints(struct regulator_dev *rdev)
{
struct regulation_constraints *constraints = rdev->constraints;
print_constraints_debug(rdev);
if ((constraints->min_uV != constraints->max_uV) &&
!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE))
rdev_warn(rdev,
"Voltage range but no REGULATOR_CHANGE_VOLTAGE\n");
}
static int machine_constraints_voltage(struct regulator_dev *rdev,
struct regulation_constraints *constraints)
{
const struct regulator_ops *ops = rdev->desc->ops;
int ret;
/* do we need to apply the constraint voltage */
if (rdev->constraints->apply_uV &&
rdev->constraints->min_uV && rdev->constraints->max_uV) {
int target_min, target_max;
int current_uV = regulator_get_voltage_rdev(rdev);
if (current_uV == -ENOTRECOVERABLE) {
/* This regulator can't be read and must be initialized */
rdev_info(rdev, "Setting %d-%duV\n",
rdev->constraints->min_uV,
rdev->constraints->max_uV);
_regulator_do_set_voltage(rdev,
rdev->constraints->min_uV,
rdev->constraints->max_uV);
current_uV = regulator_get_voltage_rdev(rdev);
}
if (current_uV < 0) {
if (current_uV != -EPROBE_DEFER)
rdev_err(rdev,
"failed to get the current voltage: %pe\n",
ERR_PTR(current_uV));
return current_uV;
}
/*
* If we're below the minimum voltage move up to the
* minimum voltage, if we're above the maximum voltage
* then move down to the maximum.
*/
target_min = current_uV;
target_max = current_uV;
if (current_uV < rdev->constraints->min_uV) {
target_min = rdev->constraints->min_uV;
target_max = rdev->constraints->min_uV;
}
if (current_uV > rdev->constraints->max_uV) {
target_min = rdev->constraints->max_uV;
target_max = rdev->constraints->max_uV;
}
if (target_min != current_uV || target_max != current_uV) {
rdev_info(rdev, "Bringing %duV into %d-%duV\n",
current_uV, target_min, target_max);
ret = _regulator_do_set_voltage(
rdev, target_min, target_max);
if (ret < 0) {
rdev_err(rdev,
"failed to apply %d-%duV constraint: %pe\n",
target_min, target_max, ERR_PTR(ret));
return ret;
}
}
}
/* constrain machine-level voltage specs to fit
* the actual range supported by this regulator.
*/
if (ops->list_voltage && rdev->desc->n_voltages) {
int count = rdev->desc->n_voltages;
int i;
int min_uV = INT_MAX;
int max_uV = INT_MIN;
int cmin = constraints->min_uV;
int cmax = constraints->max_uV;
/* it's safe to autoconfigure fixed-voltage supplies
* and the constraints are used by list_voltage.
*/
if (count == 1 && !cmin) {
cmin = 1;
cmax = INT_MAX;
constraints->min_uV = cmin;
constraints->max_uV = cmax;
}
/* voltage constraints are optional */
if ((cmin == 0) && (cmax == 0))
return 0;
/* else require explicit machine-level constraints */
if (cmin <= 0 || cmax <= 0 || cmax < cmin) {
rdev_err(rdev, "invalid voltage constraints\n");
return -EINVAL;
}
/* no need to loop voltages if range is continuous */
if (rdev->desc->continuous_voltage_range)
return 0;
/* initial: [cmin..cmax] valid, [min_uV..max_uV] not */
for (i = 0; i < count; i++) {
int value;
value = ops->list_voltage(rdev, i);
if (value <= 0)
continue;
/* maybe adjust [min_uV..max_uV] */
if (value >= cmin && value < min_uV)
min_uV = value;
if (value <= cmax && value > max_uV)
max_uV = value;
}
/* final: [min_uV..max_uV] valid iff constraints valid */
if (max_uV < min_uV) {
rdev_err(rdev,
"unsupportable voltage constraints %u-%uuV\n",
min_uV, max_uV);
return -EINVAL;
}
/* use regulator's subset of machine constraints */
if (constraints->min_uV < min_uV) {
rdev_dbg(rdev, "override min_uV, %d -> %d\n",
constraints->min_uV, min_uV);
constraints->min_uV = min_uV;
}
if (constraints->max_uV > max_uV) {
rdev_dbg(rdev, "override max_uV, %d -> %d\n",
constraints->max_uV, max_uV);
constraints->max_uV = max_uV;
}
}
return 0;
}
static int machine_constraints_current(struct regulator_dev *rdev,
struct regulation_constraints *constraints)
{
const struct regulator_ops *ops = rdev->desc->ops;
int ret;
if (!constraints->min_uA && !constraints->max_uA)
return 0;
if (constraints->min_uA > constraints->max_uA) {
rdev_err(rdev, "Invalid current constraints\n");
return -EINVAL;
}
if (!ops->set_current_limit || !ops->get_current_limit) {
rdev_warn(rdev, "Operation of current configuration missing\n");
return 0;
}
/* Set regulator current in constraints range */
ret = ops->set_current_limit(rdev, constraints->min_uA,
constraints->max_uA);
if (ret < 0) {
rdev_err(rdev, "Failed to set current constraint, %d\n", ret);
return ret;
}
return 0;
}
static int _regulator_do_enable(struct regulator_dev *rdev);
static int notif_set_limit(struct regulator_dev *rdev,
int (*set)(struct regulator_dev *, int, int, bool),
int limit, int severity)
{
bool enable;
if (limit == REGULATOR_NOTIF_LIMIT_DISABLE) {
enable = false;
limit = 0;
} else {
enable = true;
}
if (limit == REGULATOR_NOTIF_LIMIT_ENABLE)
limit = 0;
return set(rdev, limit, severity, enable);
}
static int handle_notify_limits(struct regulator_dev *rdev,
int (*set)(struct regulator_dev *, int, int, bool),
struct notification_limit *limits)
{
int ret = 0;
if (!set)
return -EOPNOTSUPP;
if (limits->prot)
ret = notif_set_limit(rdev, set, limits->prot,
REGULATOR_SEVERITY_PROT);
if (ret)
return ret;
if (limits->err)
ret = notif_set_limit(rdev, set, limits->err,
REGULATOR_SEVERITY_ERR);
if (ret)
return ret;
if (limits->warn)
ret = notif_set_limit(rdev, set, limits->warn,
REGULATOR_SEVERITY_WARN);
return ret;
}
/**
* set_machine_constraints - sets regulator constraints
* @rdev: regulator source
*
* Allows platform initialisation code to define and constrain
* regulator circuits e.g. valid voltage/current ranges, etc. NOTE:
* Constraints *must* be set by platform code in order for some
* regulator operations to proceed i.e. set_voltage, set_current_limit,
* set_mode.
*/
static int set_machine_constraints(struct regulator_dev *rdev)
{
int ret = 0;
const struct regulator_ops *ops = rdev->desc->ops;
ret = machine_constraints_voltage(rdev, rdev->constraints);
if (ret != 0)
return ret;
ret = machine_constraints_current(rdev, rdev->constraints);
if (ret != 0)
return ret;
if (rdev->constraints->ilim_uA && ops->set_input_current_limit) {
ret = ops->set_input_current_limit(rdev,
rdev->constraints->ilim_uA);
if (ret < 0) {
rdev_err(rdev, "failed to set input limit: %pe\n", ERR_PTR(ret));
return ret;
}
}
/* do we need to setup our suspend state */
if (rdev->constraints->initial_state) {
ret = suspend_set_initial_state(rdev);
if (ret < 0) {
rdev_err(rdev, "failed to set suspend state: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->constraints->initial_mode) {
if (!ops->set_mode) {
rdev_err(rdev, "no set_mode operation\n");
return -EINVAL;
}
ret = ops->set_mode(rdev, rdev->constraints->initial_mode);
if (ret < 0) {
rdev_err(rdev, "failed to set initial mode: %pe\n", ERR_PTR(ret));
return ret;
}
} else if (rdev->constraints->system_load) {
/*
* We'll only apply the initial system load if an
* initial mode wasn't specified.
*/
drms_uA_update(rdev);
}
if ((rdev->constraints->ramp_delay || rdev->constraints->ramp_disable)
&& ops->set_ramp_delay) {
ret = ops->set_ramp_delay(rdev, rdev->constraints->ramp_delay);
if (ret < 0) {
rdev_err(rdev, "failed to set ramp_delay: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->constraints->pull_down && ops->set_pull_down) {
ret = ops->set_pull_down(rdev);
if (ret < 0) {
rdev_err(rdev, "failed to set pull down: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->constraints->soft_start && ops->set_soft_start) {
ret = ops->set_soft_start(rdev);
if (ret < 0) {
rdev_err(rdev, "failed to set soft start: %pe\n", ERR_PTR(ret));
return ret;
}
}
/*
* Existing logic does not warn if over_current_protection is given as
* a constraint but driver does not support that. I think we should
* warn about this type of issues as it is possible someone changes
* PMIC on board to another type - and the another PMIC's driver does
* not support setting protection. Board composer may happily believe
* the DT limits are respected - especially if the new PMIC HW also
* supports protection but the driver does not. I won't change the logic
* without hearing more experienced opinion on this though.
*
* If warning is seen as a good idea then we can merge handling the
* over-curret protection and detection and get rid of this special
* handling.
*/
if (rdev->constraints->over_current_protection
&& ops->set_over_current_protection) {
int lim = rdev->constraints->over_curr_limits.prot;
ret = ops->set_over_current_protection(rdev, lim,
REGULATOR_SEVERITY_PROT,
true);
if (ret < 0) {
rdev_err(rdev, "failed to set over current protection: %pe\n",
ERR_PTR(ret));
return ret;
}
}
if (rdev->constraints->over_current_detection)
ret = handle_notify_limits(rdev,
ops->set_over_current_protection,
&rdev->constraints->over_curr_limits);
if (ret) {
if (ret != -EOPNOTSUPP) {
rdev_err(rdev, "failed to set over current limits: %pe\n",
ERR_PTR(ret));
return ret;
}
rdev_warn(rdev,
"IC does not support requested over-current limits\n");
}
if (rdev->constraints->over_voltage_detection)
ret = handle_notify_limits(rdev,
ops->set_over_voltage_protection,
&rdev->constraints->over_voltage_limits);
if (ret) {
if (ret != -EOPNOTSUPP) {
rdev_err(rdev, "failed to set over voltage limits %pe\n",
ERR_PTR(ret));
return ret;
}
rdev_warn(rdev,
"IC does not support requested over voltage limits\n");
}
if (rdev->constraints->under_voltage_detection)
ret = handle_notify_limits(rdev,
ops->set_under_voltage_protection,
&rdev->constraints->under_voltage_limits);
if (ret) {
if (ret != -EOPNOTSUPP) {
rdev_err(rdev, "failed to set under voltage limits %pe\n",
ERR_PTR(ret));
return ret;
}
rdev_warn(rdev,
"IC does not support requested under voltage limits\n");
}
if (rdev->constraints->over_temp_detection)
ret = handle_notify_limits(rdev,
ops->set_thermal_protection,
&rdev->constraints->temp_limits);
if (ret) {
if (ret != -EOPNOTSUPP) {
rdev_err(rdev, "failed to set temperature limits %pe\n",
ERR_PTR(ret));
return ret;
}
rdev_warn(rdev,
"IC does not support requested temperature limits\n");
}
if (rdev->constraints->active_discharge && ops->set_active_discharge) {
bool ad_state = (rdev->constraints->active_discharge ==
REGULATOR_ACTIVE_DISCHARGE_ENABLE) ? true : false;
ret = ops->set_active_discharge(rdev, ad_state);
if (ret < 0) {
rdev_err(rdev, "failed to set active discharge: %pe\n", ERR_PTR(ret));
return ret;
}
}
/*
* If there is no mechanism for controlling the regulator then
* flag it as always_on so we don't end up duplicating checks
* for this so much. Note that we could control the state of
* a supply to control the output on a regulator that has no
* direct control.
*/
if (!rdev->ena_pin && !ops->enable) {
if (rdev->supply_name && !rdev->supply)
return -EPROBE_DEFER;
if (rdev->supply)
rdev->constraints->always_on =
rdev->supply->rdev->constraints->always_on;
else
rdev->constraints->always_on = true;
}
/* If the constraints say the regulator should be on at this point
* and we have control then make sure it is enabled.
*/
if (rdev->constraints->always_on || rdev->constraints->boot_on) {
/* If we want to enable this regulator, make sure that we know
* the supplying regulator.
*/
if (rdev->supply_name && !rdev->supply)
return -EPROBE_DEFER;
/* If supplying regulator has already been enabled,
* it's not intended to have use_count increment
* when rdev is only boot-on.
*/
if (rdev->supply &&
(rdev->constraints->always_on ||
!regulator_is_enabled(rdev->supply))) {
ret = regulator_enable(rdev->supply);
if (ret < 0) {
_regulator_put(rdev->supply);
rdev->supply = NULL;
return ret;
}
}
ret = _regulator_do_enable(rdev);
if (ret < 0 && ret != -EINVAL) {
rdev_err(rdev, "failed to enable: %pe\n", ERR_PTR(ret));
return ret;
}
if (rdev->constraints->always_on)
rdev->use_count++;
} else if (rdev->desc->off_on_delay) {
rdev->last_off = ktime_get();
}
print_constraints(rdev);
return 0;
}
/**
* set_supply - set regulator supply regulator
* @rdev: regulator (locked)
* @supply_rdev: supply regulator (locked))
*
* Called by platform initialisation code to set the supply regulator for this
* regulator. This ensures that a regulators supply will also be enabled by the
* core if it's child is enabled.
*/
static int set_supply(struct regulator_dev *rdev,
struct regulator_dev *supply_rdev)
{
int err;
rdev_dbg(rdev, "supplied by %s\n", rdev_get_name(supply_rdev));
if (!try_module_get(supply_rdev->owner))
return -ENODEV;
rdev->supply = create_regulator(supply_rdev, &rdev->dev, "SUPPLY");
if (rdev->supply == NULL) {
module_put(supply_rdev->owner);
err = -ENOMEM;
return err;
}
supply_rdev->open_count++;
return 0;
}
/**
* set_consumer_device_supply - Bind a regulator to a symbolic supply
* @rdev: regulator source
* @consumer_dev_name: dev_name() string for device supply applies to
* @supply: symbolic name for supply
*
* Allows platform initialisation code to map physical regulator
* sources to symbolic names for supplies for use by devices. Devices
* should use these symbolic names to request regulators, avoiding the
* need to provide board-specific regulator names as platform data.
*/
static int set_consumer_device_supply(struct regulator_dev *rdev,
const char *consumer_dev_name,
const char *supply)
{
struct regulator_map *node, *new_node;
int has_dev;
if (supply == NULL)
return -EINVAL;
if (consumer_dev_name != NULL)
has_dev = 1;
else
has_dev = 0;
new_node = kzalloc(sizeof(struct regulator_map), GFP_KERNEL);
if (new_node == NULL)
return -ENOMEM;
new_node->regulator = rdev;
new_node->supply = supply;
if (has_dev) {
new_node->dev_name = kstrdup(consumer_dev_name, GFP_KERNEL);
if (new_node->dev_name == NULL) {
kfree(new_node);
return -ENOMEM;
}
}
mutex_lock(®ulator_list_mutex);
list_for_each_entry(node, ®ulator_map_list, list) {
if (node->dev_name && consumer_dev_name) {
if (strcmp(node->dev_name, consumer_dev_name) != 0)
continue;
} else if (node->dev_name || consumer_dev_name) {
continue;
}
if (strcmp(node->supply, supply) != 0)
continue;
pr_debug("%s: %s/%s is '%s' supply; fail %s/%s\n",
consumer_dev_name,
dev_name(&node->regulator->dev),
node->regulator->desc->name,
supply,
dev_name(&rdev->dev), rdev_get_name(rdev));
goto fail;
}
list_add(&new_node->list, ®ulator_map_list);
mutex_unlock(®ulator_list_mutex);
return 0;
fail:
mutex_unlock(®ulator_list_mutex);
kfree(new_node->dev_name);
kfree(new_node);
return -EBUSY;
}
static void unset_regulator_supplies(struct regulator_dev *rdev)
{
struct regulator_map *node, *n;
list_for_each_entry_safe(node, n, ®ulator_map_list, list) {
if (rdev == node->regulator) {
list_del(&node->list);
kfree(node->dev_name);
kfree(node);
}
}
}
#ifdef CONFIG_DEBUG_FS
static ssize_t constraint_flags_read_file(struct file *file,
char __user *user_buf,
size_t count, loff_t *ppos)
{
const struct regulator *regulator = file->private_data;
const struct regulation_constraints *c = regulator->rdev->constraints;
char *buf;
ssize_t ret;
if (!c)
return 0;
buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!buf)
return -ENOMEM;
ret = snprintf(buf, PAGE_SIZE,
"always_on: %u\n"
"boot_on: %u\n"
"apply_uV: %u\n"
"ramp_disable: %u\n"
"soft_start: %u\n"
"pull_down: %u\n"
"over_current_protection: %u\n",
c->always_on,
c->boot_on,
c->apply_uV,
c->ramp_disable,
c->soft_start,
c->pull_down,
c->over_current_protection);
ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
kfree(buf);
return ret;
}
#endif
static const struct file_operations constraint_flags_fops = {
#ifdef CONFIG_DEBUG_FS
.open = simple_open,
.read = constraint_flags_read_file,
.llseek = default_llseek,
#endif
};
#define REG_STR_SIZE 64
static struct regulator *create_regulator(struct regulator_dev *rdev,
struct device *dev,
const char *supply_name)
{
struct regulator *regulator;
int err = 0;
lockdep_assert_held_once(&rdev->mutex.base);
if (dev) {
char buf[REG_STR_SIZE];
int size;
size = snprintf(buf, REG_STR_SIZE, "%s-%s",
dev->kobj.name, supply_name);
if (size >= REG_STR_SIZE)
return NULL;
supply_name = kstrdup(buf, GFP_KERNEL);
if (supply_name == NULL)
return NULL;
} else {
supply_name = kstrdup_const(supply_name, GFP_KERNEL);
if (supply_name == NULL)
return NULL;
}
regulator = kzalloc(sizeof(*regulator), GFP_KERNEL);
if (regulator == NULL) {
kfree_const(supply_name);
return NULL;
}
regulator->rdev = rdev;
regulator->supply_name = supply_name;
list_add(®ulator->list, &rdev->consumer_list);
if (dev) {
regulator->dev = dev;
/* Add a link to the device sysfs entry */
err = sysfs_create_link_nowarn(&rdev->dev.kobj, &dev->kobj,
supply_name);
if (err) {
rdev_dbg(rdev, "could not add device link %s: %pe\n",
dev->kobj.name, ERR_PTR(err));
/* non-fatal */
}
}
if (err != -EEXIST)
regulator->debugfs = debugfs_create_dir(supply_name, rdev->debugfs);
if (IS_ERR(regulator->debugfs))
rdev_dbg(rdev, "Failed to create debugfs directory\n");
debugfs_create_u32("uA_load", 0444, regulator->debugfs,
®ulator->uA_load);
debugfs_create_u32("min_uV", 0444, regulator->debugfs,
®ulator->voltage[PM_SUSPEND_ON].min_uV);
debugfs_create_u32("max_uV", 0444, regulator->debugfs,
®ulator->voltage[PM_SUSPEND_ON].max_uV);
debugfs_create_file("constraint_flags", 0444, regulator->debugfs,
regulator, &constraint_flags_fops);
/*
* Check now if the regulator is an always on regulator - if
* it is then we don't need to do nearly so much work for
* enable/disable calls.
*/
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS) &&
_regulator_is_enabled(rdev))
regulator->always_on = true;
return regulator;
}
static int _regulator_get_enable_time(struct regulator_dev *rdev)
{
if (rdev->constraints && rdev->constraints->enable_time)
return rdev->constraints->enable_time;
if (rdev->desc->ops->enable_time)
return rdev->desc->ops->enable_time(rdev);
return rdev->desc->enable_time;
}
static struct regulator_supply_alias *regulator_find_supply_alias(
struct device *dev, const char *supply)
{
struct regulator_supply_alias *map;
list_for_each_entry(map, ®ulator_supply_alias_list, list) if (map->src_dev == dev && strcmp(map->src_supply, supply) == 0)
return map;
return NULL;
}
static void regulator_supply_alias(struct device **dev, const char **supply)
{
struct regulator_supply_alias *map;
map = regulator_find_supply_alias(*dev, *supply);
if (map) {
dev_dbg(*dev, "Mapping supply %s to %s,%s\n",
*supply, map->alias_supply,
dev_name(map->alias_dev));
*dev = map->alias_dev;
*supply = map->alias_supply;
}
}
static int regulator_match(struct device *dev, const void *data)
{
struct regulator_dev *r = dev_to_rdev(dev);
return strcmp(rdev_get_name(r), data) == 0;
}
static struct regulator_dev *regulator_lookup_by_name(const char *name)
{
struct device *dev;
dev = class_find_device(®ulator_class, NULL, name, regulator_match);
return dev ? dev_to_rdev(dev) : NULL;
}
/**
* regulator_dev_lookup - lookup a regulator device.
* @dev: device for regulator "consumer".
* @supply: Supply name or regulator ID.
*
* If successful, returns a struct regulator_dev that corresponds to the name
* @supply and with the embedded struct device refcount incremented by one.
* The refcount must be dropped by calling put_device().
* On failure one of the following ERR-PTR-encoded values is returned:
* -ENODEV if lookup fails permanently, -EPROBE_DEFER if lookup could succeed
* in the future.
*/
static struct regulator_dev *regulator_dev_lookup(struct device *dev,
const char *supply)
{
struct regulator_dev *r = NULL;
struct device_node *node;
struct regulator_map *map;
const char *devname = NULL;
regulator_supply_alias(&dev, &supply);
/* first do a dt based lookup */
if (dev && dev->of_node) {
node = of_get_regulator(dev, supply);
if (node) {
r = of_find_regulator_by_node(node);
of_node_put(node);
if (r)
return r;
/*
* We have a node, but there is no device.
* assume it has not registered yet.
*/
return ERR_PTR(-EPROBE_DEFER);
}
}
/* if not found, try doing it non-dt way */
if (dev)
devname = dev_name(dev);
mutex_lock(®ulator_list_mutex);
list_for_each_entry(map, ®ulator_map_list, list) {
/* If the mapping has a device set up it must match */
if (map->dev_name &&
(!devname || strcmp(map->dev_name, devname)))
continue;
if (strcmp(map->supply, supply) == 0 &&
get_device(&map->regulator->dev)) {
r = map->regulator;
break;
}
}
mutex_unlock(®ulator_list_mutex);
if (r)
return r;
r = regulator_lookup_by_name(supply);
if (r)
return r;
return ERR_PTR(-ENODEV);
}
static int regulator_resolve_supply(struct regulator_dev *rdev)
{
struct regulator_dev *r;
struct device *dev = rdev->dev.parent;
struct ww_acquire_ctx ww_ctx;
int ret = 0;
/* No supply to resolve? */
if (!rdev->supply_name)
return 0;
/* Supply already resolved? (fast-path without locking contention) */
if (rdev->supply)
return 0;
r = regulator_dev_lookup(dev, rdev->supply_name);
if (IS_ERR(r)) {
ret = PTR_ERR(r);
/* Did the lookup explicitly defer for us? */
if (ret == -EPROBE_DEFER)
goto out;
if (have_full_constraints()) {
r = dummy_regulator_rdev;
get_device(&r->dev);
} else {
dev_err(dev, "Failed to resolve %s-supply for %s\n",
rdev->supply_name, rdev->desc->name);
ret = -EPROBE_DEFER;
goto out;
}
}
if (r == rdev) {
dev_err(dev, "Supply for %s (%s) resolved to itself\n",
rdev->desc->name, rdev->supply_name);
if (!have_full_constraints()) {
ret = -EINVAL;
goto out;
}
r = dummy_regulator_rdev;
get_device(&r->dev);
}
/*
* If the supply's parent device is not the same as the
* regulator's parent device, then ensure the parent device
* is bound before we resolve the supply, in case the parent
* device get probe deferred and unregisters the supply.
*/
if (r->dev.parent && r->dev.parent != rdev->dev.parent) {
if (!device_is_bound(r->dev.parent)) {
put_device(&r->dev);
ret = -EPROBE_DEFER;
goto out;
}
}
/* Recursively resolve the supply of the supply */
ret = regulator_resolve_supply(r);
if (ret < 0) {
put_device(&r->dev);
goto out;
}
/*
* Recheck rdev->supply with rdev->mutex lock held to avoid a race
* between rdev->supply null check and setting rdev->supply in
* set_supply() from concurrent tasks.
*/
regulator_lock_two(rdev, r, &ww_ctx);
/* Supply just resolved by a concurrent task? */
if (rdev->supply) {
regulator_unlock_two(rdev, r, &ww_ctx);
put_device(&r->dev);
goto out;
}
ret = set_supply(rdev, r);
if (ret < 0) {
regulator_unlock_two(rdev, r, &ww_ctx);
put_device(&r->dev);
goto out;
}
regulator_unlock_two(rdev, r, &ww_ctx);
/*
* In set_machine_constraints() we may have turned this regulator on
* but we couldn't propagate to the supply if it hadn't been resolved
* yet. Do it now.
*/
if (rdev->use_count) {
ret = regulator_enable(rdev->supply);
if (ret < 0) {
_regulator_put(rdev->supply);
rdev->supply = NULL;
goto out;
}
}
out:
return ret;
}
/* Internal regulator request function */
struct regulator *_regulator_get(struct device *dev, const char *id,
enum regulator_get_type get_type)
{
struct regulator_dev *rdev;
struct regulator *regulator;
struct device_link *link;
int ret;
if (get_type >= MAX_GET_TYPE) {
dev_err(dev, "invalid type %d in %s\n", get_type, __func__);
return ERR_PTR(-EINVAL);
}
if (id == NULL) {
pr_err("get() with no identifier\n");
return ERR_PTR(-EINVAL);
}
rdev = regulator_dev_lookup(dev, id);
if (IS_ERR(rdev)) {
ret = PTR_ERR(rdev);
/*
* If regulator_dev_lookup() fails with error other
* than -ENODEV our job here is done, we simply return it.
*/
if (ret != -ENODEV)
return ERR_PTR(ret);
if (!have_full_constraints()) {
dev_warn(dev,
"incomplete constraints, dummy supplies not allowed\n");
return ERR_PTR(-ENODEV);
}
switch (get_type) {
case NORMAL_GET:
/*
* Assume that a regulator is physically present and
* enabled, even if it isn't hooked up, and just
* provide a dummy.
*/
dev_warn(dev, "supply %s not found, using dummy regulator\n", id);
rdev = dummy_regulator_rdev;
get_device(&rdev->dev);
break;
case EXCLUSIVE_GET:
dev_warn(dev,
"dummy supplies not allowed for exclusive requests\n");
fallthrough;
default:
return ERR_PTR(-ENODEV);
}
}
if (rdev->exclusive) {
regulator = ERR_PTR(-EPERM);
put_device(&rdev->dev);
return regulator;
}
if (get_type == EXCLUSIVE_GET && rdev->open_count) {
regulator = ERR_PTR(-EBUSY);
put_device(&rdev->dev);
return regulator;
}
mutex_lock(®ulator_list_mutex);
ret = (rdev->coupling_desc.n_resolved != rdev->coupling_desc.n_coupled);
mutex_unlock(®ulator_list_mutex);
if (ret != 0) {
regulator = ERR_PTR(-EPROBE_DEFER);
put_device(&rdev->dev);
return regulator;
}
ret = regulator_resolve_supply(rdev);
if (ret < 0) {
regulator = ERR_PTR(ret);
put_device(&rdev->dev);
return regulator;
}
if (!try_module_get(rdev->owner)) {
regulator = ERR_PTR(-EPROBE_DEFER);
put_device(&rdev->dev);
return regulator;
}
regulator_lock(rdev);
regulator = create_regulator(rdev, dev, id);
regulator_unlock(rdev);
if (regulator == NULL) {
regulator = ERR_PTR(-ENOMEM);
module_put(rdev->owner);
put_device(&rdev->dev);
return regulator;
}
rdev->open_count++;
if (get_type == EXCLUSIVE_GET) {
rdev->exclusive = 1;
ret = _regulator_is_enabled(rdev);
if (ret > 0) {
rdev->use_count = 1;
regulator->enable_count = 1;
/* Propagate the regulator state to its supply */
if (rdev->supply) {
ret = regulator_enable(rdev->supply);
if (ret < 0) {
destroy_regulator(regulator);
module_put(rdev->owner);
put_device(&rdev->dev);
return ERR_PTR(ret);
}
}
} else {
rdev->use_count = 0;
regulator->enable_count = 0;
}
}
link = device_link_add(dev, &rdev->dev, DL_FLAG_STATELESS);
if (!IS_ERR_OR_NULL(link))
regulator->device_link = true;
return regulator;
}
/**
* regulator_get - lookup and obtain a reference to a regulator.
* @dev: device for regulator "consumer"
* @id: Supply name or regulator ID.
*
* Returns a struct regulator corresponding to the regulator producer,
* or IS_ERR() condition containing errno.
*
* Use of supply names configured via set_consumer_device_supply() is
* strongly encouraged. It is recommended that the supply name used
* should match the name used for the supply and/or the relevant
* device pins in the datasheet.
*/
struct regulator *regulator_get(struct device *dev, const char *id)
{
return _regulator_get(dev, id, NORMAL_GET);
}
EXPORT_SYMBOL_GPL(regulator_get);
/**
* regulator_get_exclusive - obtain exclusive access to a regulator.
* @dev: device for regulator "consumer"
* @id: Supply name or regulator ID.
*
* Returns a struct regulator corresponding to the regulator producer,
* or IS_ERR() condition containing errno. Other consumers will be
* unable to obtain this regulator while this reference is held and the
* use count for the regulator will be initialised to reflect the current
* state of the regulator.
*
* This is intended for use by consumers which cannot tolerate shared
* use of the regulator such as those which need to force the
* regulator off for correct operation of the hardware they are
* controlling.
*
* Use of supply names configured via set_consumer_device_supply() is
* strongly encouraged. It is recommended that the supply name used
* should match the name used for the supply and/or the relevant
* device pins in the datasheet.
*/
struct regulator *regulator_get_exclusive(struct device *dev, const char *id)
{
return _regulator_get(dev, id, EXCLUSIVE_GET);
}
EXPORT_SYMBOL_GPL(regulator_get_exclusive);
/**
* regulator_get_optional - obtain optional access to a regulator.
* @dev: device for regulator "consumer"
* @id: Supply name or regulator ID.
*
* Returns a struct regulator corresponding to the regulator producer,
* or IS_ERR() condition containing errno.
*
* This is intended for use by consumers for devices which can have
* some supplies unconnected in normal use, such as some MMC devices.
* It can allow the regulator core to provide stub supplies for other
* supplies requested using normal regulator_get() calls without
* disrupting the operation of drivers that can handle absent
* supplies.
*
* Use of supply names configured via set_consumer_device_supply() is
* strongly encouraged. It is recommended that the supply name used
* should match the name used for the supply and/or the relevant
* device pins in the datasheet.
*/
struct regulator *regulator_get_optional(struct device *dev, const char *id)
{
return _regulator_get(dev, id, OPTIONAL_GET);
}
EXPORT_SYMBOL_GPL(regulator_get_optional);
static void destroy_regulator(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
debugfs_remove_recursive(regulator->debugfs);
if (regulator->dev) {
if (regulator->device_link)
device_link_remove(regulator->dev, &rdev->dev);
/* remove any sysfs entries */
sysfs_remove_link(&rdev->dev.kobj, regulator->supply_name);
}
regulator_lock(rdev);
list_del(®ulator->list);
rdev->open_count--;
rdev->exclusive = 0;
regulator_unlock(rdev);
kfree_const(regulator->supply_name);
kfree(regulator);
}
/* regulator_list_mutex lock held by regulator_put() */
static void _regulator_put(struct regulator *regulator)
{
struct regulator_dev *rdev;
if (IS_ERR_OR_NULL(regulator))
return;
lockdep_assert_held_once(®ulator_list_mutex);
/* Docs say you must disable before calling regulator_put() */
WARN_ON(regulator->enable_count);
rdev = regulator->rdev;
destroy_regulator(regulator);
module_put(rdev->owner);
put_device(&rdev->dev);
}
/**
* regulator_put - "free" the regulator source
* @regulator: regulator source
*
* Note: drivers must ensure that all regulator_enable calls made on this
* regulator source are balanced by regulator_disable calls prior to calling
* this function.
*/
void regulator_put(struct regulator *regulator)
{
mutex_lock(®ulator_list_mutex);
_regulator_put(regulator);
mutex_unlock(®ulator_list_mutex);
}
EXPORT_SYMBOL_GPL(regulator_put);
/**
* regulator_register_supply_alias - Provide device alias for supply lookup
*
* @dev: device that will be given as the regulator "consumer"
* @id: Supply name or regulator ID
* @alias_dev: device that should be used to lookup the supply
* @alias_id: Supply name or regulator ID that should be used to lookup the
* supply
*
* All lookups for id on dev will instead be conducted for alias_id on
* alias_dev.
*/
int regulator_register_supply_alias(struct device *dev, const char *id,
struct device *alias_dev,
const char *alias_id)
{
struct regulator_supply_alias *map;
map = regulator_find_supply_alias(dev, id);
if (map)
return -EEXIST;
map = kzalloc(sizeof(struct regulator_supply_alias), GFP_KERNEL);
if (!map)
return -ENOMEM;
map->src_dev = dev;
map->src_supply = id;
map->alias_dev = alias_dev;
map->alias_supply = alias_id;
list_add(&map->list, ®ulator_supply_alias_list);
pr_info("Adding alias for supply %s,%s -> %s,%s\n",
id, dev_name(dev), alias_id, dev_name(alias_dev));
return 0;
}
EXPORT_SYMBOL_GPL(regulator_register_supply_alias);
/**
* regulator_unregister_supply_alias - Remove device alias
*
* @dev: device that will be given as the regulator "consumer"
* @id: Supply name or regulator ID
*
* Remove a lookup alias if one exists for id on dev.
*/
void regulator_unregister_supply_alias(struct device *dev, const char *id)
{
struct regulator_supply_alias *map;
map = regulator_find_supply_alias(dev, id); if (map) { list_del(&map->list);
kfree(map);
}
}
EXPORT_SYMBOL_GPL(regulator_unregister_supply_alias);
/**
* regulator_bulk_register_supply_alias - register multiple aliases
*
* @dev: device that will be given as the regulator "consumer"
* @id: List of supply names or regulator IDs
* @alias_dev: device that should be used to lookup the supply
* @alias_id: List of supply names or regulator IDs that should be used to
* lookup the supply
* @num_id: Number of aliases to register
*
* @return 0 on success, an errno on failure.
*
* This helper function allows drivers to register several supply
* aliases in one operation. If any of the aliases cannot be
* registered any aliases that were registered will be removed
* before returning to the caller.
*/
int regulator_bulk_register_supply_alias(struct device *dev,
const char *const *id,
struct device *alias_dev,
const char *const *alias_id,
int num_id)
{
int i;
int ret;
for (i = 0; i < num_id; ++i) {
ret = regulator_register_supply_alias(dev, id[i], alias_dev,
alias_id[i]);
if (ret < 0)
goto err;
}
return 0;
err:
dev_err(dev,
"Failed to create supply alias %s,%s -> %s,%s\n",
id[i], dev_name(dev), alias_id[i], dev_name(alias_dev));
while (--i >= 0)
regulator_unregister_supply_alias(dev, id[i]);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_register_supply_alias);
/**
* regulator_bulk_unregister_supply_alias - unregister multiple aliases
*
* @dev: device that will be given as the regulator "consumer"
* @id: List of supply names or regulator IDs
* @num_id: Number of aliases to unregister
*
* This helper function allows drivers to unregister several supply
* aliases in one operation.
*/
void regulator_bulk_unregister_supply_alias(struct device *dev,
const char *const *id,
int num_id)
{
int i;
for (i = 0; i < num_id; ++i) regulator_unregister_supply_alias(dev, id[i]);
}
EXPORT_SYMBOL_GPL(regulator_bulk_unregister_supply_alias);
/* Manage enable GPIO list. Same GPIO pin can be shared among regulators */
static int regulator_ena_gpio_request(struct regulator_dev *rdev,
const struct regulator_config *config)
{
struct regulator_enable_gpio *pin, *new_pin;
struct gpio_desc *gpiod;
gpiod = config->ena_gpiod;
new_pin = kzalloc(sizeof(*new_pin), GFP_KERNEL);
mutex_lock(®ulator_list_mutex);
list_for_each_entry(pin, ®ulator_ena_gpio_list, list) {
if (pin->gpiod == gpiod) {
rdev_dbg(rdev, "GPIO is already used\n");
goto update_ena_gpio_to_rdev;
}
}
if (new_pin == NULL) {
mutex_unlock(®ulator_list_mutex);
return -ENOMEM;
}
pin = new_pin;
new_pin = NULL;
pin->gpiod = gpiod;
list_add(&pin->list, ®ulator_ena_gpio_list);
update_ena_gpio_to_rdev:
pin->request_count++;
rdev->ena_pin = pin;
mutex_unlock(®ulator_list_mutex);
kfree(new_pin);
return 0;
}
static void regulator_ena_gpio_free(struct regulator_dev *rdev)
{
struct regulator_enable_gpio *pin, *n;
if (!rdev->ena_pin)
return;
/* Free the GPIO only in case of no use */
list_for_each_entry_safe(pin, n, ®ulator_ena_gpio_list, list) {
if (pin != rdev->ena_pin)
continue;
if (--pin->request_count)
break;
gpiod_put(pin->gpiod);
list_del(&pin->list);
kfree(pin);
break;
}
rdev->ena_pin = NULL;
}
/**
* regulator_ena_gpio_ctrl - balance enable_count of each GPIO and actual GPIO pin control
* @rdev: regulator_dev structure
* @enable: enable GPIO at initial use?
*
* GPIO is enabled in case of initial use. (enable_count is 0)
* GPIO is disabled when it is not shared any more. (enable_count <= 1)
*/
static int regulator_ena_gpio_ctrl(struct regulator_dev *rdev, bool enable)
{
struct regulator_enable_gpio *pin = rdev->ena_pin;
if (!pin)
return -EINVAL;
if (enable) {
/* Enable GPIO at initial use */
if (pin->enable_count == 0)
gpiod_set_value_cansleep(pin->gpiod, 1);
pin->enable_count++;
} else {
if (pin->enable_count > 1) {
pin->enable_count--;
return 0;
}
/* Disable GPIO if not used */
if (pin->enable_count <= 1) {
gpiod_set_value_cansleep(pin->gpiod, 0);
pin->enable_count = 0;
}
}
return 0;
}
/**
* _regulator_delay_helper - a delay helper function
* @delay: time to delay in microseconds
*
* Delay for the requested amount of time as per the guidelines in:
*
* Documentation/timers/timers-howto.rst
*
* The assumption here is that these regulator operations will never used in
* atomic context and therefore sleeping functions can be used.
*/
static void _regulator_delay_helper(unsigned int delay)
{
unsigned int ms = delay / 1000;
unsigned int us = delay % 1000;
if (ms > 0) {
/*
* For small enough values, handle super-millisecond
* delays in the usleep_range() call below.
*/
if (ms < 20)
us += ms * 1000;
else
msleep(ms);
}
/*
* Give the scheduler some room to coalesce with any other
* wakeup sources. For delays shorter than 10 us, don't even
* bother setting up high-resolution timers and just busy-
* loop.
*/
if (us >= 10)
usleep_range(us, us + 100);
else
udelay(us);
}
/**
* _regulator_check_status_enabled
*
* A helper function to check if the regulator status can be interpreted
* as 'regulator is enabled'.
* @rdev: the regulator device to check
*
* Return:
* * 1 - if status shows regulator is in enabled state
* * 0 - if not enabled state
* * Error Value - as received from ops->get_status()
*/
static inline int _regulator_check_status_enabled(struct regulator_dev *rdev)
{
int ret = rdev->desc->ops->get_status(rdev);
if (ret < 0) {
rdev_info(rdev, "get_status returned error: %d\n", ret);
return ret;
}
switch (ret) {
case REGULATOR_STATUS_OFF:
case REGULATOR_STATUS_ERROR:
case REGULATOR_STATUS_UNDEFINED:
return 0;
default:
return 1;
}
}
static int _regulator_do_enable(struct regulator_dev *rdev)
{
int ret, delay;
/* Query before enabling in case configuration dependent. */
ret = _regulator_get_enable_time(rdev);
if (ret >= 0) {
delay = ret;
} else {
rdev_warn(rdev, "enable_time() failed: %pe\n", ERR_PTR(ret));
delay = 0;
}
trace_regulator_enable(rdev_get_name(rdev));
if (rdev->desc->off_on_delay) {
/* if needed, keep a distance of off_on_delay from last time
* this regulator was disabled.
*/
ktime_t end = ktime_add_us(rdev->last_off, rdev->desc->off_on_delay);
s64 remaining = ktime_us_delta(end, ktime_get_boottime());
if (remaining > 0)
_regulator_delay_helper(remaining);
}
if (rdev->ena_pin) {
if (!rdev->ena_gpio_state) {
ret = regulator_ena_gpio_ctrl(rdev, true);
if (ret < 0)
return ret;
rdev->ena_gpio_state = 1;
}
} else if (rdev->desc->ops->enable) {
ret = rdev->desc->ops->enable(rdev);
if (ret < 0)
return ret;
} else {
return -EINVAL;
}
/* Allow the regulator to ramp; it would be useful to extend
* this for bulk operations so that the regulators can ramp
* together.
*/
trace_regulator_enable_delay(rdev_get_name(rdev));
/* If poll_enabled_time is set, poll upto the delay calculated
* above, delaying poll_enabled_time uS to check if the regulator
* actually got enabled.
* If the regulator isn't enabled after our delay helper has expired,
* return -ETIMEDOUT.
*/
if (rdev->desc->poll_enabled_time) {
int time_remaining = delay;
while (time_remaining > 0) {
_regulator_delay_helper(rdev->desc->poll_enabled_time);
if (rdev->desc->ops->get_status) {
ret = _regulator_check_status_enabled(rdev);
if (ret < 0)
return ret;
else if (ret)
break;
} else if (rdev->desc->ops->is_enabled(rdev))
break;
time_remaining -= rdev->desc->poll_enabled_time;
}
if (time_remaining <= 0) {
rdev_err(rdev, "Enabled check timed out\n");
return -ETIMEDOUT;
}
} else {
_regulator_delay_helper(delay);
}
trace_regulator_enable_complete(rdev_get_name(rdev));
return 0;
}
/**
* _regulator_handle_consumer_enable - handle that a consumer enabled
* @regulator: regulator source
*
* Some things on a regulator consumer (like the contribution towards total
* load on the regulator) only have an effect when the consumer wants the
* regulator enabled. Explained in example with two consumers of the same
* regulator:
* consumer A: set_load(100); => total load = 0
* consumer A: regulator_enable(); => total load = 100
* consumer B: set_load(1000); => total load = 100
* consumer B: regulator_enable(); => total load = 1100
* consumer A: regulator_disable(); => total_load = 1000
*
* This function (together with _regulator_handle_consumer_disable) is
* responsible for keeping track of the refcount for a given regulator consumer
* and applying / unapplying these things.
*
* Returns 0 upon no error; -error upon error.
*/
static int _regulator_handle_consumer_enable(struct regulator *regulator)
{
int ret;
struct regulator_dev *rdev = regulator->rdev;
lockdep_assert_held_once(&rdev->mutex.base);
regulator->enable_count++;
if (regulator->uA_load && regulator->enable_count == 1) {
ret = drms_uA_update(rdev);
if (ret)
regulator->enable_count--;
return ret;
}
return 0;
}
/**
* _regulator_handle_consumer_disable - handle that a consumer disabled
* @regulator: regulator source
*
* The opposite of _regulator_handle_consumer_enable().
*
* Returns 0 upon no error; -error upon error.
*/
static int _regulator_handle_consumer_disable(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
lockdep_assert_held_once(&rdev->mutex.base);
if (!regulator->enable_count) {
rdev_err(rdev, "Underflow of regulator enable count\n");
return -EINVAL;
}
regulator->enable_count--;
if (regulator->uA_load && regulator->enable_count == 0)
return drms_uA_update(rdev);
return 0;
}
/* locks held by regulator_enable() */
static int _regulator_enable(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
int ret;
lockdep_assert_held_once(&rdev->mutex.base);
if (rdev->use_count == 0 && rdev->supply) {
ret = _regulator_enable(rdev->supply);
if (ret < 0)
return ret;
}
/* balance only if there are regulators coupled */
if (rdev->coupling_desc.n_coupled > 1) {
ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
if (ret < 0)
goto err_disable_supply;
}
ret = _regulator_handle_consumer_enable(regulator);
if (ret < 0)
goto err_disable_supply;
if (rdev->use_count == 0) {
/*
* The regulator may already be enabled if it's not switchable
* or was left on
*/
ret = _regulator_is_enabled(rdev);
if (ret == -EINVAL || ret == 0) {
if (!regulator_ops_is_valid(rdev,
REGULATOR_CHANGE_STATUS)) {
ret = -EPERM;
goto err_consumer_disable;
}
ret = _regulator_do_enable(rdev);
if (ret < 0)
goto err_consumer_disable;
_notifier_call_chain(rdev, REGULATOR_EVENT_ENABLE,
NULL);
} else if (ret < 0) {
rdev_err(rdev, "is_enabled() failed: %pe\n", ERR_PTR(ret));
goto err_consumer_disable;
}
/* Fallthrough on positive return values - already enabled */
}
if (regulator->enable_count == 1)
rdev->use_count++;
return 0;
err_consumer_disable:
_regulator_handle_consumer_disable(regulator);
err_disable_supply:
if (rdev->use_count == 0 && rdev->supply)
_regulator_disable(rdev->supply);
return ret;
}
/**
* regulator_enable - enable regulator output
* @regulator: regulator source
*
* Request that the regulator be enabled with the regulator output at
* the predefined voltage or current value. Calls to regulator_enable()
* must be balanced with calls to regulator_disable().
*
* NOTE: the output value can be set by other drivers, boot loader or may be
* hardwired in the regulator.
*/
int regulator_enable(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
struct ww_acquire_ctx ww_ctx;
int ret;
regulator_lock_dependent(rdev, &ww_ctx);
ret = _regulator_enable(regulator);
regulator_unlock_dependent(rdev, &ww_ctx);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_enable);
static int _regulator_do_disable(struct regulator_dev *rdev)
{
int ret;
trace_regulator_disable(rdev_get_name(rdev));
if (rdev->ena_pin) {
if (rdev->ena_gpio_state) {
ret = regulator_ena_gpio_ctrl(rdev, false);
if (ret < 0)
return ret;
rdev->ena_gpio_state = 0;
}
} else if (rdev->desc->ops->disable) {
ret = rdev->desc->ops->disable(rdev);
if (ret != 0)
return ret;
}
if (rdev->desc->off_on_delay)
rdev->last_off = ktime_get_boottime();
trace_regulator_disable_complete(rdev_get_name(rdev));
return 0;
}
/* locks held by regulator_disable() */
static int _regulator_disable(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
int ret = 0;
lockdep_assert_held_once(&rdev->mutex.base);
if (WARN(regulator->enable_count == 0,
"unbalanced disables for %s\n", rdev_get_name(rdev)))
return -EIO;
if (regulator->enable_count == 1) {
/* disabling last enable_count from this regulator */
/* are we the last user and permitted to disable ? */
if (rdev->use_count == 1 &&
(rdev->constraints && !rdev->constraints->always_on)) {
/* we are last user */
if (regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS)) {
ret = _notifier_call_chain(rdev,
REGULATOR_EVENT_PRE_DISABLE,
NULL);
if (ret & NOTIFY_STOP_MASK)
return -EINVAL;
ret = _regulator_do_disable(rdev);
if (ret < 0) {
rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret));
_notifier_call_chain(rdev,
REGULATOR_EVENT_ABORT_DISABLE,
NULL);
return ret;
}
_notifier_call_chain(rdev, REGULATOR_EVENT_DISABLE,
NULL);
}
rdev->use_count = 0;
} else if (rdev->use_count > 1) {
rdev->use_count--;
}
}
if (ret == 0)
ret = _regulator_handle_consumer_disable(regulator);
if (ret == 0 && rdev->coupling_desc.n_coupled > 1)
ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
if (ret == 0 && rdev->use_count == 0 && rdev->supply)
ret = _regulator_disable(rdev->supply);
return ret;
}
/**
* regulator_disable - disable regulator output
* @regulator: regulator source
*
* Disable the regulator output voltage or current. Calls to
* regulator_enable() must be balanced with calls to
* regulator_disable().
*
* NOTE: this will only disable the regulator output if no other consumer
* devices have it enabled, the regulator device supports disabling and
* machine constraints permit this operation.
*/
int regulator_disable(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
struct ww_acquire_ctx ww_ctx;
int ret;
regulator_lock_dependent(rdev, &ww_ctx);
ret = _regulator_disable(regulator);
regulator_unlock_dependent(rdev, &ww_ctx);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_disable);
/* locks held by regulator_force_disable() */
static int _regulator_force_disable(struct regulator_dev *rdev)
{
int ret = 0;
lockdep_assert_held_once(&rdev->mutex.base);
ret = _notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
REGULATOR_EVENT_PRE_DISABLE, NULL);
if (ret & NOTIFY_STOP_MASK)
return -EINVAL;
ret = _regulator_do_disable(rdev);
if (ret < 0) {
rdev_err(rdev, "failed to force disable: %pe\n", ERR_PTR(ret));
_notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
REGULATOR_EVENT_ABORT_DISABLE, NULL);
return ret;
}
_notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
REGULATOR_EVENT_DISABLE, NULL);
return 0;
}
/**
* regulator_force_disable - force disable regulator output
* @regulator: regulator source
*
* Forcibly disable the regulator output voltage or current.
* NOTE: this *will* disable the regulator output even if other consumer
* devices have it enabled. This should be used for situations when device
* damage will likely occur if the regulator is not disabled (e.g. over temp).
*/
int regulator_force_disable(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
struct ww_acquire_ctx ww_ctx;
int ret;
regulator_lock_dependent(rdev, &ww_ctx);
ret = _regulator_force_disable(regulator->rdev);
if (rdev->coupling_desc.n_coupled > 1)
regulator_balance_voltage(rdev, PM_SUSPEND_ON);
if (regulator->uA_load) {
regulator->uA_load = 0;
ret = drms_uA_update(rdev);
}
if (rdev->use_count != 0 && rdev->supply)
_regulator_disable(rdev->supply);
regulator_unlock_dependent(rdev, &ww_ctx);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_force_disable);
static void regulator_disable_work(struct work_struct *work)
{
struct regulator_dev *rdev = container_of(work, struct regulator_dev,
disable_work.work);
struct ww_acquire_ctx ww_ctx;
int count, i, ret;
struct regulator *regulator;
int total_count = 0;
regulator_lock_dependent(rdev, &ww_ctx);
/*
* Workqueue functions queue the new work instance while the previous
* work instance is being processed. Cancel the queued work instance
* as the work instance under processing does the job of the queued
* work instance.
*/
cancel_delayed_work(&rdev->disable_work);
list_for_each_entry(regulator, &rdev->consumer_list, list) {
count = regulator->deferred_disables;
if (!count)
continue;
total_count += count;
regulator->deferred_disables = 0;
for (i = 0; i < count; i++) {
ret = _regulator_disable(regulator);
if (ret != 0)
rdev_err(rdev, "Deferred disable failed: %pe\n",
ERR_PTR(ret));
}
}
WARN_ON(!total_count);
if (rdev->coupling_desc.n_coupled > 1)
regulator_balance_voltage(rdev, PM_SUSPEND_ON);
regulator_unlock_dependent(rdev, &ww_ctx);
}
/**
* regulator_disable_deferred - disable regulator output with delay
* @regulator: regulator source
* @ms: milliseconds until the regulator is disabled
*
* Execute regulator_disable() on the regulator after a delay. This
* is intended for use with devices that require some time to quiesce.
*
* NOTE: this will only disable the regulator output if no other consumer
* devices have it enabled, the regulator device supports disabling and
* machine constraints permit this operation.
*/
int regulator_disable_deferred(struct regulator *regulator, int ms)
{
struct regulator_dev *rdev = regulator->rdev;
if (!ms)
return regulator_disable(regulator);
regulator_lock(rdev);
regulator->deferred_disables++;
mod_delayed_work(system_power_efficient_wq, &rdev->disable_work,
msecs_to_jiffies(ms));
regulator_unlock(rdev);
return 0;
}
EXPORT_SYMBOL_GPL(regulator_disable_deferred);
static int _regulator_is_enabled(struct regulator_dev *rdev)
{
/* A GPIO control always takes precedence */
if (rdev->ena_pin)
return rdev->ena_gpio_state;
/* If we don't know then assume that the regulator is always on */
if (!rdev->desc->ops->is_enabled)
return 1;
return rdev->desc->ops->is_enabled(rdev);
}
static int _regulator_list_voltage(struct regulator_dev *rdev,
unsigned selector, int lock)
{
const struct regulator_ops *ops = rdev->desc->ops;
int ret;
if (rdev->desc->fixed_uV && rdev->desc->n_voltages == 1 && !selector)
return rdev->desc->fixed_uV;
if (ops->list_voltage) {
if (selector >= rdev->desc->n_voltages)
return -EINVAL;
if (selector < rdev->desc->linear_min_sel)
return 0;
if (lock)
regulator_lock(rdev);
ret = ops->list_voltage(rdev, selector);
if (lock)
regulator_unlock(rdev);
} else if (rdev->is_switch && rdev->supply) {
ret = _regulator_list_voltage(rdev->supply->rdev,
selector, lock);
} else {
return -EINVAL;
}
if (ret > 0) {
if (ret < rdev->constraints->min_uV)
ret = 0;
else if (ret > rdev->constraints->max_uV)
ret = 0;
}
return ret;
}
/**
* regulator_is_enabled - is the regulator output enabled
* @regulator: regulator source
*
* Returns positive if the regulator driver backing the source/client
* has requested that the device be enabled, zero if it hasn't, else a
* negative errno code.
*
* Note that the device backing this regulator handle can have multiple
* users, so it might be enabled even if regulator_enable() was never
* called for this particular source.
*/
int regulator_is_enabled(struct regulator *regulator)
{
int ret;
if (regulator->always_on)
return 1;
regulator_lock(regulator->rdev);
ret = _regulator_is_enabled(regulator->rdev);
regulator_unlock(regulator->rdev);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_is_enabled);
/**
* regulator_count_voltages - count regulator_list_voltage() selectors
* @regulator: regulator source
*
* Returns number of selectors, or negative errno. Selectors are
* numbered starting at zero, and typically correspond to bitfields
* in hardware registers.
*/
int regulator_count_voltages(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
if (rdev->desc->n_voltages)
return rdev->desc->n_voltages;
if (!rdev->is_switch || !rdev->supply)
return -EINVAL;
return regulator_count_voltages(rdev->supply);
}
EXPORT_SYMBOL_GPL(regulator_count_voltages);
/**
* regulator_list_voltage - enumerate supported voltages
* @regulator: regulator source
* @selector: identify voltage to list
* Context: can sleep
*
* Returns a voltage that can be passed to @regulator_set_voltage(),
* zero if this selector code can't be used on this system, or a
* negative errno.
*/
int regulator_list_voltage(struct regulator *regulator, unsigned selector)
{
return _regulator_list_voltage(regulator->rdev, selector, 1);
}
EXPORT_SYMBOL_GPL(regulator_list_voltage);
/**
* regulator_get_regmap - get the regulator's register map
* @regulator: regulator source
*
* Returns the register map for the given regulator, or an ERR_PTR value
* if the regulator doesn't use regmap.
*/
struct regmap *regulator_get_regmap(struct regulator *regulator)
{
struct regmap *map = regulator->rdev->regmap;
return map ? map : ERR_PTR(-EOPNOTSUPP);
}
/**
* regulator_get_hardware_vsel_register - get the HW voltage selector register
* @regulator: regulator source
* @vsel_reg: voltage selector register, output parameter
* @vsel_mask: mask for voltage selector bitfield, output parameter
*
* Returns the hardware register offset and bitmask used for setting the
* regulator voltage. This might be useful when configuring voltage-scaling
* hardware or firmware that can make I2C requests behind the kernel's back,
* for example.
*
* On success, the output parameters @vsel_reg and @vsel_mask are filled in
* and 0 is returned, otherwise a negative errno is returned.
*/
int regulator_get_hardware_vsel_register(struct regulator *regulator,
unsigned *vsel_reg,
unsigned *vsel_mask)
{
struct regulator_dev *rdev = regulator->rdev;
const struct regulator_ops *ops = rdev->desc->ops;
if (ops->set_voltage_sel != regulator_set_voltage_sel_regmap)
return -EOPNOTSUPP;
*vsel_reg = rdev->desc->vsel_reg;
*vsel_mask = rdev->desc->vsel_mask;
return 0;
}
EXPORT_SYMBOL_GPL(regulator_get_hardware_vsel_register);
/**
* regulator_list_hardware_vsel - get the HW-specific register value for a selector
* @regulator: regulator source
* @selector: identify voltage to list
*
* Converts the selector to a hardware-specific voltage selector that can be
* directly written to the regulator registers. The address of the voltage
* register can be determined by calling @regulator_get_hardware_vsel_register.
*
* On error a negative errno is returned.
*/
int regulator_list_hardware_vsel(struct regulator *regulator,
unsigned selector)
{
struct regulator_dev *rdev = regulator->rdev;
const struct regulator_ops *ops = rdev->desc->ops;
if (selector >= rdev->desc->n_voltages)
return -EINVAL;
if (selector < rdev->desc->linear_min_sel)
return 0;
if (ops->set_voltage_sel != regulator_set_voltage_sel_regmap)
return -EOPNOTSUPP;
return selector;
}
EXPORT_SYMBOL_GPL(regulator_list_hardware_vsel);
/**
* regulator_get_linear_step - return the voltage step size between VSEL values
* @regulator: regulator source
*
* Returns the voltage step size between VSEL values for linear
* regulators, or return 0 if the regulator isn't a linear regulator.
*/
unsigned int regulator_get_linear_step(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
return rdev->desc->uV_step;
}
EXPORT_SYMBOL_GPL(regulator_get_linear_step);
/**
* regulator_is_supported_voltage - check if a voltage range can be supported
*
* @regulator: Regulator to check.
* @min_uV: Minimum required voltage in uV.
* @max_uV: Maximum required voltage in uV.
*
* Returns a boolean.
*/
int regulator_is_supported_voltage(struct regulator *regulator,
int min_uV, int max_uV)
{
struct regulator_dev *rdev = regulator->rdev;
int i, voltages, ret;
/* If we can't change voltage check the current voltage */
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
ret = regulator_get_voltage(regulator);
if (ret >= 0)
return min_uV <= ret && ret <= max_uV;
else
return ret;
}
/* Any voltage within constrains range is fine? */
if (rdev->desc->continuous_voltage_range)
return min_uV >= rdev->constraints->min_uV &&
max_uV <= rdev->constraints->max_uV;
ret = regulator_count_voltages(regulator);
if (ret < 0)
return 0;
voltages = ret;
for (i = 0; i < voltages; i++) {
ret = regulator_list_voltage(regulator, i);
if (ret >= min_uV && ret <= max_uV)
return 1;
}
return 0;
}
EXPORT_SYMBOL_GPL(regulator_is_supported_voltage);
static int regulator_map_voltage(struct regulator_dev *rdev, int min_uV,
int max_uV)
{
const struct regulator_desc *desc = rdev->desc;
if (desc->ops->map_voltage)
return desc->ops->map_voltage(rdev, min_uV, max_uV);
if (desc->ops->list_voltage == regulator_list_voltage_linear)
return regulator_map_voltage_linear(rdev, min_uV, max_uV);
if (desc->ops->list_voltage == regulator_list_voltage_linear_range)
return regulator_map_voltage_linear_range(rdev, min_uV, max_uV);
if (desc->ops->list_voltage ==
regulator_list_voltage_pickable_linear_range)
return regulator_map_voltage_pickable_linear_range(rdev,
min_uV, max_uV);
return regulator_map_voltage_iterate(rdev, min_uV, max_uV);
}
static int _regulator_call_set_voltage(struct regulator_dev *rdev,
int min_uV, int max_uV,
unsigned *selector)
{
struct pre_voltage_change_data data;
int ret;
data.old_uV = regulator_get_voltage_rdev(rdev);
data.min_uV = min_uV;
data.max_uV = max_uV;
ret = _notifier_call_chain(rdev, REGULATOR_EVENT_PRE_VOLTAGE_CHANGE,
&data);
if (ret & NOTIFY_STOP_MASK)
return -EINVAL;
ret = rdev->desc->ops->set_voltage(rdev, min_uV, max_uV, selector);
if (ret >= 0)
return ret;
_notifier_call_chain(rdev, REGULATOR_EVENT_ABORT_VOLTAGE_CHANGE,
(void *)data.old_uV);
return ret;
}
static int _regulator_call_set_voltage_sel(struct regulator_dev *rdev,
int uV, unsigned selector)
{
struct pre_voltage_change_data data;
int ret;
data.old_uV = regulator_get_voltage_rdev(rdev);
data.min_uV = uV;
data.max_uV = uV;
ret = _notifier_call_chain(rdev, REGULATOR_EVENT_PRE_VOLTAGE_CHANGE,
&data);
if (ret & NOTIFY_STOP_MASK)
return -EINVAL;
ret = rdev->desc->ops->set_voltage_sel(rdev, selector);
if (ret >= 0)
return ret;
_notifier_call_chain(rdev, REGULATOR_EVENT_ABORT_VOLTAGE_CHANGE,
(void *)data.old_uV);
return ret;
}
static int _regulator_set_voltage_sel_step(struct regulator_dev *rdev,
int uV, int new_selector)
{
const struct regulator_ops *ops = rdev->desc->ops;
int diff, old_sel, curr_sel, ret;
/* Stepping is only needed if the regulator is enabled. */
if (!_regulator_is_enabled(rdev))
goto final_set;
if (!ops->get_voltage_sel)
return -EINVAL;
old_sel = ops->get_voltage_sel(rdev);
if (old_sel < 0)
return old_sel;
diff = new_selector - old_sel;
if (diff == 0)
return 0; /* No change needed. */
if (diff > 0) {
/* Stepping up. */
for (curr_sel = old_sel + rdev->desc->vsel_step;
curr_sel < new_selector;
curr_sel += rdev->desc->vsel_step) {
/*
* Call the callback directly instead of using
* _regulator_call_set_voltage_sel() as we don't
* want to notify anyone yet. Same in the branch
* below.
*/
ret = ops->set_voltage_sel(rdev, curr_sel);
if (ret)
goto try_revert;
}
} else {
/* Stepping down. */
for (curr_sel = old_sel - rdev->desc->vsel_step;
curr_sel > new_selector;
curr_sel -= rdev->desc->vsel_step) {
ret = ops->set_voltage_sel(rdev, curr_sel);
if (ret)
goto try_revert;
}
}
final_set:
/* The final selector will trigger the notifiers. */
return _regulator_call_set_voltage_sel(rdev, uV, new_selector);
try_revert:
/*
* At least try to return to the previous voltage if setting a new
* one failed.
*/
(void)ops->set_voltage_sel(rdev, old_sel);
return ret;
}
static int _regulator_set_voltage_time(struct regulator_dev *rdev,
int old_uV, int new_uV)
{
unsigned int ramp_delay = 0;
if (rdev->constraints->ramp_delay)
ramp_delay = rdev->constraints->ramp_delay;
else if (rdev->desc->ramp_delay)
ramp_delay = rdev->desc->ramp_delay;
else if (rdev->constraints->settling_time)
return rdev->constraints->settling_time;
else if (rdev->constraints->settling_time_up &&
(new_uV > old_uV))
return rdev->constraints->settling_time_up;
else if (rdev->constraints->settling_time_down &&
(new_uV < old_uV))
return rdev->constraints->settling_time_down;
if (ramp_delay == 0)
return 0;
return DIV_ROUND_UP(abs(new_uV - old_uV), ramp_delay);
}
static int _regulator_do_set_voltage(struct regulator_dev *rdev,
int min_uV, int max_uV)
{
int ret;
int delay = 0;
int best_val = 0;
unsigned int selector;
int old_selector = -1;
const struct regulator_ops *ops = rdev->desc->ops;
int old_uV = regulator_get_voltage_rdev(rdev);
trace_regulator_set_voltage(rdev_get_name(rdev), min_uV, max_uV);
min_uV += rdev->constraints->uV_offset;
max_uV += rdev->constraints->uV_offset;
/*
* If we can't obtain the old selector there is not enough
* info to call set_voltage_time_sel().
*/
if (_regulator_is_enabled(rdev) &&
ops->set_voltage_time_sel && ops->get_voltage_sel) {
old_selector = ops->get_voltage_sel(rdev);
if (old_selector < 0)
return old_selector;
}
if (ops->set_voltage) {
ret = _regulator_call_set_voltage(rdev, min_uV, max_uV,
&selector);
if (ret >= 0) {
if (ops->list_voltage)
best_val = ops->list_voltage(rdev,
selector);
else
best_val = regulator_get_voltage_rdev(rdev);
}
} else if (ops->set_voltage_sel) {
ret = regulator_map_voltage(rdev, min_uV, max_uV);
if (ret >= 0) {
best_val = ops->list_voltage(rdev, ret);
if (min_uV <= best_val && max_uV >= best_val) {
selector = ret;
if (old_selector == selector)
ret = 0;
else if (rdev->desc->vsel_step)
ret = _regulator_set_voltage_sel_step(
rdev, best_val, selector);
else
ret = _regulator_call_set_voltage_sel(
rdev, best_val, selector);
} else {
ret = -EINVAL;
}
}
} else {
ret = -EINVAL;
}
if (ret)
goto out;
if (ops->set_voltage_time_sel) {
/*
* Call set_voltage_time_sel if successfully obtained
* old_selector
*/
if (old_selector >= 0 && old_selector != selector)
delay = ops->set_voltage_time_sel(rdev, old_selector,
selector);
} else {
if (old_uV != best_val) {
if (ops->set_voltage_time)
delay = ops->set_voltage_time(rdev, old_uV,
best_val);
else
delay = _regulator_set_voltage_time(rdev,
old_uV,
best_val);
}
}
if (delay < 0) {
rdev_warn(rdev, "failed to get delay: %pe\n", ERR_PTR(delay));
delay = 0;
}
/* Insert any necessary delays */
_regulator_delay_helper(delay);
if (best_val >= 0) {
unsigned long data = best_val;
_notifier_call_chain(rdev, REGULATOR_EVENT_VOLTAGE_CHANGE,
(void *)data);
}
out:
trace_regulator_set_voltage_complete(rdev_get_name(rdev), best_val);
return ret;
}
static int _regulator_do_set_suspend_voltage(struct regulator_dev *rdev,
int min_uV, int max_uV, suspend_state_t state)
{
struct regulator_state *rstate;
int uV, sel;
rstate = regulator_get_suspend_state(rdev, state);
if (rstate == NULL)
return -EINVAL;
if (min_uV < rstate->min_uV)
min_uV = rstate->min_uV;
if (max_uV > rstate->max_uV)
max_uV = rstate->max_uV;
sel = regulator_map_voltage(rdev, min_uV, max_uV);
if (sel < 0)
return sel;
uV = rdev->desc->ops->list_voltage(rdev, sel);
if (uV >= min_uV && uV <= max_uV)
rstate->uV = uV;
return 0;
}
static int regulator_set_voltage_unlocked(struct regulator *regulator,
int min_uV, int max_uV,
suspend_state_t state)
{
struct regulator_dev *rdev = regulator->rdev;
struct regulator_voltage *voltage = ®ulator->voltage[state];
int ret = 0;
int old_min_uV, old_max_uV;
int current_uV;
/* If we're setting the same range as last time the change
* should be a noop (some cpufreq implementations use the same
* voltage for multiple frequencies, for example).
*/
if (voltage->min_uV == min_uV && voltage->max_uV == max_uV)
goto out;
/* If we're trying to set a range that overlaps the current voltage,
* return successfully even though the regulator does not support
* changing the voltage.
*/
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
current_uV = regulator_get_voltage_rdev(rdev);
if (min_uV <= current_uV && current_uV <= max_uV) {
voltage->min_uV = min_uV;
voltage->max_uV = max_uV;
goto out;
}
}
/* sanity check */
if (!rdev->desc->ops->set_voltage &&
!rdev->desc->ops->set_voltage_sel) {
ret = -EINVAL;
goto out;
}
/* constraints check */
ret = regulator_check_voltage(rdev, &min_uV, &max_uV);
if (ret < 0)
goto out;
/* restore original values in case of error */
old_min_uV = voltage->min_uV;
old_max_uV = voltage->max_uV;
voltage->min_uV = min_uV;
voltage->max_uV = max_uV;
/* for not coupled regulators this will just set the voltage */
ret = regulator_balance_voltage(rdev, state);
if (ret < 0) {
voltage->min_uV = old_min_uV;
voltage->max_uV = old_max_uV;
}
out:
return ret;
}
int regulator_set_voltage_rdev(struct regulator_dev *rdev, int min_uV,
int max_uV, suspend_state_t state)
{
int best_supply_uV = 0;
int supply_change_uV = 0;
int ret;
if (rdev->supply &&
regulator_ops_is_valid(rdev->supply->rdev,
REGULATOR_CHANGE_VOLTAGE) &&
(rdev->desc->min_dropout_uV || !(rdev->desc->ops->get_voltage ||
rdev->desc->ops->get_voltage_sel))) {
int current_supply_uV;
int selector;
selector = regulator_map_voltage(rdev, min_uV, max_uV);
if (selector < 0) {
ret = selector;
goto out;
}
best_supply_uV = _regulator_list_voltage(rdev, selector, 0);
if (best_supply_uV < 0) {
ret = best_supply_uV;
goto out;
}
best_supply_uV += rdev->desc->min_dropout_uV;
current_supply_uV = regulator_get_voltage_rdev(rdev->supply->rdev);
if (current_supply_uV < 0) {
ret = current_supply_uV;
goto out;
}
supply_change_uV = best_supply_uV - current_supply_uV;
}
if (supply_change_uV > 0) {
ret = regulator_set_voltage_unlocked(rdev->supply,
best_supply_uV, INT_MAX, state);
if (ret) {
dev_err(&rdev->dev, "Failed to increase supply voltage: %pe\n",
ERR_PTR(ret));
goto out;
}
}
if (state == PM_SUSPEND_ON)
ret = _regulator_do_set_voltage(rdev, min_uV, max_uV);
else
ret = _regulator_do_set_suspend_voltage(rdev, min_uV,
max_uV, state);
if (ret < 0)
goto out;
if (supply_change_uV < 0) {
ret = regulator_set_voltage_unlocked(rdev->supply,
best_supply_uV, INT_MAX, state);
if (ret)
dev_warn(&rdev->dev, "Failed to decrease supply voltage: %pe\n",
ERR_PTR(ret));
/* No need to fail here */
ret = 0;
}
out:
return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_voltage_rdev);
static int regulator_limit_voltage_step(struct regulator_dev *rdev,
int *current_uV, int *min_uV)
{
struct regulation_constraints *constraints = rdev->constraints;
/* Limit voltage change only if necessary */
if (!constraints->max_uV_step || !_regulator_is_enabled(rdev))
return 1;
if (*current_uV < 0) {
*current_uV = regulator_get_voltage_rdev(rdev);
if (*current_uV < 0)
return *current_uV;
}
if (abs(*current_uV - *min_uV) <= constraints->max_uV_step)
return 1;
/* Clamp target voltage within the given step */
if (*current_uV < *min_uV)
*min_uV = min(*current_uV + constraints->max_uV_step,
*min_uV);
else
*min_uV = max(*current_uV - constraints->max_uV_step,
*min_uV);
return 0;
}
static int regulator_get_optimal_voltage(struct regulator_dev *rdev,
int *current_uV,
int *min_uV, int *max_uV,
suspend_state_t state,
int n_coupled)
{
struct coupling_desc *c_desc = &rdev->coupling_desc;
struct regulator_dev **c_rdevs = c_desc->coupled_rdevs;
struct regulation_constraints *constraints = rdev->constraints;
int desired_min_uV = 0, desired_max_uV = INT_MAX;
int max_current_uV = 0, min_current_uV = INT_MAX;
int highest_min_uV = 0, target_uV, possible_uV;
int i, ret, max_spread;
bool done;
*current_uV = -1;
/*
* If there are no coupled regulators, simply set the voltage
* demanded by consumers.
*/
if (n_coupled == 1) {
/*
* If consumers don't provide any demands, set voltage
* to min_uV
*/
desired_min_uV = constraints->min_uV;
desired_max_uV = constraints->max_uV;
ret = regulator_check_consumers(rdev,
&desired_min_uV,
&desired_max_uV, state);
if (ret < 0)
return ret;
done = true;
goto finish;
}
/* Find highest min desired voltage */
for (i = 0; i < n_coupled; i++) {
int tmp_min = 0;
int tmp_max = INT_MAX;
lockdep_assert_held_once(&c_rdevs[i]->mutex.base);
ret = regulator_check_consumers(c_rdevs[i],
&tmp_min,
&tmp_max, state);
if (ret < 0)
return ret;
ret = regulator_check_voltage(c_rdevs[i], &tmp_min, &tmp_max);
if (ret < 0)
return ret;
highest_min_uV = max(highest_min_uV, tmp_min);
if (i == 0) {
desired_min_uV = tmp_min;
desired_max_uV = tmp_max;
}
}
max_spread = constraints->max_spread[0];
/*
* Let target_uV be equal to the desired one if possible.
* If not, set it to minimum voltage, allowed by other coupled
* regulators.
*/
target_uV = max(desired_min_uV, highest_min_uV - max_spread);
/*
* Find min and max voltages, which currently aren't violating
* max_spread.
*/
for (i = 1; i < n_coupled; i++) {
int tmp_act;
if (!_regulator_is_enabled(c_rdevs[i]))
continue;
tmp_act = regulator_get_voltage_rdev(c_rdevs[i]);
if (tmp_act < 0)
return tmp_act;
min_current_uV = min(tmp_act, min_current_uV);
max_current_uV = max(tmp_act, max_current_uV);
}
/* There aren't any other regulators enabled */
if (max_current_uV == 0) {
possible_uV = target_uV;
} else {
/*
* Correct target voltage, so as it currently isn't
* violating max_spread
*/
possible_uV = max(target_uV, max_current_uV - max_spread);
possible_uV = min(possible_uV, min_current_uV + max_spread);
}
if (possible_uV > desired_max_uV)
return -EINVAL;
done = (possible_uV == target_uV);
desired_min_uV = possible_uV;
finish:
/* Apply max_uV_step constraint if necessary */
if (state == PM_SUSPEND_ON) {
ret = regulator_limit_voltage_step(rdev, current_uV,
&desired_min_uV);
if (ret < 0)
return ret;
if (ret == 0)
done = false;
}
/* Set current_uV if wasn't done earlier in the code and if necessary */
if (n_coupled > 1 && *current_uV == -1) {
if (_regulator_is_enabled(rdev)) {
ret = regulator_get_voltage_rdev(rdev);
if (ret < 0)
return ret;
*current_uV = ret;
} else {
*current_uV = desired_min_uV;
}
}
*min_uV = desired_min_uV;
*max_uV = desired_max_uV;
return done;
}
int regulator_do_balance_voltage(struct regulator_dev *rdev,
suspend_state_t state, bool skip_coupled)
{
struct regulator_dev **c_rdevs;
struct regulator_dev *best_rdev;
struct coupling_desc *c_desc = &rdev->coupling_desc;
int i, ret, n_coupled, best_min_uV, best_max_uV, best_c_rdev;
unsigned int delta, best_delta;
unsigned long c_rdev_done = 0;
bool best_c_rdev_done;
c_rdevs = c_desc->coupled_rdevs;
n_coupled = skip_coupled ? 1 : c_desc->n_coupled;
/*
* Find the best possible voltage change on each loop. Leave the loop
* if there isn't any possible change.
*/
do {
best_c_rdev_done = false;
best_delta = 0;
best_min_uV = 0;
best_max_uV = 0;
best_c_rdev = 0;
best_rdev = NULL;
/*
* Find highest difference between optimal voltage
* and current voltage.
*/
for (i = 0; i < n_coupled; i++) {
/*
* optimal_uV is the best voltage that can be set for
* i-th regulator at the moment without violating
* max_spread constraint in order to balance
* the coupled voltages.
*/
int optimal_uV = 0, optimal_max_uV = 0, current_uV = 0;
if (test_bit(i, &c_rdev_done))
continue;
ret = regulator_get_optimal_voltage(c_rdevs[i],
¤t_uV,
&optimal_uV,
&optimal_max_uV,
state, n_coupled);
if (ret < 0)
goto out;
delta = abs(optimal_uV - current_uV);
if (delta && best_delta <= delta) {
best_c_rdev_done = ret;
best_delta = delta;
best_rdev = c_rdevs[i];
best_min_uV = optimal_uV;
best_max_uV = optimal_max_uV;
best_c_rdev = i;
}
}
/* Nothing to change, return successfully */
if (!best_rdev) {
ret = 0;
goto out;
}
ret = regulator_set_voltage_rdev(best_rdev, best_min_uV,
best_max_uV, state);
if (ret < 0)
goto out;
if (best_c_rdev_done)
set_bit(best_c_rdev, &c_rdev_done);
} while (n_coupled > 1);
out:
return ret;
}
static int regulator_balance_voltage(struct regulator_dev *rdev,
suspend_state_t state)
{
struct coupling_desc *c_desc = &rdev->coupling_desc;
struct regulator_coupler *coupler = c_desc->coupler;
bool skip_coupled = false;
/*
* If system is in a state other than PM_SUSPEND_ON, don't check
* other coupled regulators.
*/
if (state != PM_SUSPEND_ON)
skip_coupled = true;
if (c_desc->n_resolved < c_desc->n_coupled) {
rdev_err(rdev, "Not all coupled regulators registered\n");
return -EPERM;
}
/* Invoke custom balancer for customized couplers */
if (coupler && coupler->balance_voltage)
return coupler->balance_voltage(coupler, rdev, state);
return regulator_do_balance_voltage(rdev, state, skip_coupled);
}
/**
* regulator_set_voltage - set regulator output voltage
* @regulator: regulator source
* @min_uV: Minimum required voltage in uV
* @max_uV: Maximum acceptable voltage in uV
*
* Sets a voltage regulator to the desired output voltage. This can be set
* during any regulator state. IOW, regulator can be disabled or enabled.
*
* If the regulator is enabled then the voltage will change to the new value
* immediately otherwise if the regulator is disabled the regulator will
* output at the new voltage when enabled.
*
* NOTE: If the regulator is shared between several devices then the lowest
* request voltage that meets the system constraints will be used.
* Regulator system constraints must be set for this regulator before
* calling this function otherwise this call will fail.
*/
int regulator_set_voltage(struct regulator *regulator, int min_uV, int max_uV)
{
struct ww_acquire_ctx ww_ctx;
int ret;
regulator_lock_dependent(regulator->rdev, &ww_ctx);
ret = regulator_set_voltage_unlocked(regulator, min_uV, max_uV,
PM_SUSPEND_ON);
regulator_unlock_dependent(regulator->rdev, &ww_ctx);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_voltage);
static inline int regulator_suspend_toggle(struct regulator_dev *rdev,
suspend_state_t state, bool en)
{
struct regulator_state *rstate;
rstate = regulator_get_suspend_state(rdev, state);
if (rstate == NULL)
return -EINVAL;
if (!rstate->changeable)
return -EPERM;
rstate->enabled = (en) ? ENABLE_IN_SUSPEND : DISABLE_IN_SUSPEND;
return 0;
}
int regulator_suspend_enable(struct regulator_dev *rdev,
suspend_state_t state)
{
return regulator_suspend_toggle(rdev, state, true);
}
EXPORT_SYMBOL_GPL(regulator_suspend_enable);
int regulator_suspend_disable(struct regulator_dev *rdev,
suspend_state_t state)
{
struct regulator *regulator;
struct regulator_voltage *voltage;
/*
* if any consumer wants this regulator device keeping on in
* suspend states, don't set it as disabled.
*/
list_for_each_entry(regulator, &rdev->consumer_list, list) {
voltage = ®ulator->voltage[state];
if (voltage->min_uV || voltage->max_uV)
return 0;
}
return regulator_suspend_toggle(rdev, state, false);
}
EXPORT_SYMBOL_GPL(regulator_suspend_disable);
static int _regulator_set_suspend_voltage(struct regulator *regulator,
int min_uV, int max_uV,
suspend_state_t state)
{
struct regulator_dev *rdev = regulator->rdev;
struct regulator_state *rstate;
rstate = regulator_get_suspend_state(rdev, state);
if (rstate == NULL)
return -EINVAL;
if (rstate->min_uV == rstate->max_uV) {
rdev_err(rdev, "The suspend voltage can't be changed!\n");
return -EPERM;
}
return regulator_set_voltage_unlocked(regulator, min_uV, max_uV, state);
}
int regulator_set_suspend_voltage(struct regulator *regulator, int min_uV,
int max_uV, suspend_state_t state)
{
struct ww_acquire_ctx ww_ctx;
int ret;
/* PM_SUSPEND_ON is handled by regulator_set_voltage() */
if (regulator_check_states(state) || state == PM_SUSPEND_ON)
return -EINVAL;
regulator_lock_dependent(regulator->rdev, &ww_ctx);
ret = _regulator_set_suspend_voltage(regulator, min_uV,
max_uV, state);
regulator_unlock_dependent(regulator->rdev, &ww_ctx);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_suspend_voltage);
/**
* regulator_set_voltage_time - get raise/fall time
* @regulator: regulator source
* @old_uV: starting voltage in microvolts
* @new_uV: target voltage in microvolts
*
* Provided with the starting and ending voltage, this function attempts to
* calculate the time in microseconds required to rise or fall to this new
* voltage.
*/
int regulator_set_voltage_time(struct regulator *regulator,
int old_uV, int new_uV)
{
struct regulator_dev *rdev = regulator->rdev;
const struct regulator_ops *ops = rdev->desc->ops;
int old_sel = -1;
int new_sel = -1;
int voltage;
int i;
if (ops->set_voltage_time)
return ops->set_voltage_time(rdev, old_uV, new_uV);
else if (!ops->set_voltage_time_sel)
return _regulator_set_voltage_time(rdev, old_uV, new_uV);
/* Currently requires operations to do this */
if (!ops->list_voltage || !rdev->desc->n_voltages)
return -EINVAL;
for (i = 0; i < rdev->desc->n_voltages; i++) {
/* We only look for exact voltage matches here */
if (i < rdev->desc->linear_min_sel)
continue;
if (old_sel >= 0 && new_sel >= 0)
break;
voltage = regulator_list_voltage(regulator, i);
if (voltage < 0)
return -EINVAL;
if (voltage == 0)
continue;
if (voltage == old_uV)
old_sel = i;
if (voltage == new_uV)
new_sel = i;
}
if (old_sel < 0 || new_sel < 0)
return -EINVAL;
return ops->set_voltage_time_sel(rdev, old_sel, new_sel);
}
EXPORT_SYMBOL_GPL(regulator_set_voltage_time);
/**
* regulator_set_voltage_time_sel - get raise/fall time
* @rdev: regulator source device
* @old_selector: selector for starting voltage
* @new_selector: selector for target voltage
*
* Provided with the starting and target voltage selectors, this function
* returns time in microseconds required to rise or fall to this new voltage
*
* Drivers providing ramp_delay in regulation_constraints can use this as their
* set_voltage_time_sel() operation.
*/
int regulator_set_voltage_time_sel(struct regulator_dev *rdev,
unsigned int old_selector,
unsigned int new_selector)
{
int old_volt, new_volt;
/* sanity check */
if (!rdev->desc->ops->list_voltage)
return -EINVAL;
old_volt = rdev->desc->ops->list_voltage(rdev, old_selector);
new_volt = rdev->desc->ops->list_voltage(rdev, new_selector);
if (rdev->desc->ops->set_voltage_time)
return rdev->desc->ops->set_voltage_time(rdev, old_volt,
new_volt);
else
return _regulator_set_voltage_time(rdev, old_volt, new_volt);
}
EXPORT_SYMBOL_GPL(regulator_set_voltage_time_sel);
int regulator_sync_voltage_rdev(struct regulator_dev *rdev)
{
int ret;
regulator_lock(rdev);
if (!rdev->desc->ops->set_voltage &&
!rdev->desc->ops->set_voltage_sel) {
ret = -EINVAL;
goto out;
}
/* balance only, if regulator is coupled */
if (rdev->coupling_desc.n_coupled > 1)
ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
else
ret = -EOPNOTSUPP;
out:
regulator_unlock(rdev);
return ret;
}
/**
* regulator_sync_voltage - re-apply last regulator output voltage
* @regulator: regulator source
*
* Re-apply the last configured voltage. This is intended to be used
* where some external control source the consumer is cooperating with
* has caused the configured voltage to change.
*/
int regulator_sync_voltage(struct regulator *regulator)
{
struct regulator_dev *rdev = regulator->rdev;
struct regulator_voltage *voltage = ®ulator->voltage[PM_SUSPEND_ON];
int ret, min_uV, max_uV;
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE))
return 0;
regulator_lock(rdev);
if (!rdev->desc->ops->set_voltage &&
!rdev->desc->ops->set_voltage_sel) {
ret = -EINVAL;
goto out;
}
/* This is only going to work if we've had a voltage configured. */
if (!voltage->min_uV && !voltage->max_uV) {
ret = -EINVAL;
goto out;
}
min_uV = voltage->min_uV;
max_uV = voltage->max_uV;
/* This should be a paranoia check... */
ret = regulator_check_voltage(rdev, &min_uV, &max_uV);
if (ret < 0)
goto out;
ret = regulator_check_consumers(rdev, &min_uV, &max_uV, 0);
if (ret < 0)
goto out;
/* balance only, if regulator is coupled */
if (rdev->coupling_desc.n_coupled > 1)
ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
else
ret = _regulator_do_set_voltage(rdev, min_uV, max_uV);
out:
regulator_unlock(rdev);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_sync_voltage);
int regulator_get_voltage_rdev(struct regulator_dev *rdev)
{
int sel, ret;
bool bypassed;
if (rdev->desc->ops->get_bypass) {
ret = rdev->desc->ops->get_bypass(rdev, &bypassed);
if (ret < 0)
return ret;
if (bypassed) {
/* if bypassed the regulator must have a supply */
if (!rdev->supply) {
rdev_err(rdev,
"bypassed regulator has no supply!\n");
return -EPROBE_DEFER;
}
return regulator_get_voltage_rdev(rdev->supply->rdev);
}
}
if (rdev->desc->ops->get_voltage_sel) {
sel = rdev->desc->ops->get_voltage_sel(rdev);
if (sel < 0)
return sel;
ret = rdev->desc->ops->list_voltage(rdev, sel);
} else if (rdev->desc->ops->get_voltage) {
ret = rdev->desc->ops->get_voltage(rdev);
} else if (rdev->desc->ops->list_voltage) {
ret = rdev->desc->ops->list_voltage(rdev, 0);
} else if (rdev->desc->fixed_uV && (rdev->desc->n_voltages == 1)) {
ret = rdev->desc->fixed_uV;
} else if (rdev->supply) {
ret = regulator_get_voltage_rdev(rdev->supply->rdev);
} else if (rdev->supply_name) {
return -EPROBE_DEFER;
} else {
return -EINVAL;
}
if (ret < 0)
return ret;
return ret - rdev->constraints->uV_offset;
}
EXPORT_SYMBOL_GPL(regulator_get_voltage_rdev);
/**
* regulator_get_voltage - get regulator output voltage
* @regulator: regulator source
*
* This returns the current regulator voltage in uV.
*
* NOTE: If the regulator is disabled it will return the voltage value. This
* function should not be used to determine regulator state.
*/
int regulator_get_voltage(struct regulator *regulator)
{
struct ww_acquire_ctx ww_ctx;
int ret;
regulator_lock_dependent(regulator->rdev, &ww_ctx);
ret = regulator_get_voltage_rdev(regulator->rdev);
regulator_unlock_dependent(regulator->rdev, &ww_ctx);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_get_voltage);
/**
* regulator_set_current_limit - set regulator output current limit
* @regulator: regulator source
* @min_uA: Minimum supported current in uA
* @max_uA: Maximum supported current in uA
*
* Sets current sink to the desired output current. This can be set during
* any regulator state. IOW, regulator can be disabled or enabled.
*
* If the regulator is enabled then the current will change to the new value
* immediately otherwise if the regulator is disabled the regulator will
* output at the new current when enabled.
*
* NOTE: Regulator system constraints must be set for this regulator before
* calling this function otherwise this call will fail.
*/
int regulator_set_current_limit(struct regulator *regulator,
int min_uA, int max_uA)
{
struct regulator_dev *rdev = regulator->rdev;
int ret;
regulator_lock(rdev);
/* sanity check */
if (!rdev->desc->ops->set_current_limit) {
ret = -EINVAL;
goto out;
}
/* constraints check */
ret = regulator_check_current_limit(rdev, &min_uA, &max_uA);
if (ret < 0)
goto out;
ret = rdev->desc->ops->set_current_limit(rdev, min_uA, max_uA);
out:
regulator_unlock(rdev);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_current_limit);
static int _regulator_get_current_limit_unlocked(struct regulator_dev *rdev)
{
/* sanity check */
if (!rdev->desc->ops->get_current_limit)
return -EINVAL;
return rdev->desc->ops->get_current_limit(rdev);
}
static int _regulator_get_current_limit(struct regulator_dev *rdev)
{
int ret;
regulator_lock(rdev);
ret = _regulator_get_current_limit_unlocked(rdev);
regulator_unlock(rdev);
return ret;
}
/**
* regulator_get_current_limit - get regulator output current
* @regulator: regulator source
*
* This returns the current supplied by the specified current sink in uA.
*
* NOTE: If the regulator is disabled it will return the current value. This
* function should not be used to determine regulator state.
*/
int regulator_get_current_limit(struct regulator *regulator)
{
return _regulator_get_current_limit(regulator->rdev);
}
EXPORT_SYMBOL_GPL(regulator_get_current_limit);
/**
* regulator_set_mode - set regulator operating mode
* @regulator: regulator source
* @mode: operating mode - one of the REGULATOR_MODE constants
*
* Set regulator operating mode to increase regulator efficiency or improve
* regulation performance.
*
* NOTE: Regulator system constraints must be set for this regulator before
* calling this function otherwise this call will fail.
*/
int regulator_set_mode(struct regulator *regulator, unsigned int mode)
{
struct regulator_dev *rdev = regulator->rdev;
int ret;
int regulator_curr_mode;
regulator_lock(rdev);
/* sanity check */
if (!rdev->desc->ops->set_mode) {
ret = -EINVAL;
goto out;
}
/* return if the same mode is requested */
if (rdev->desc->ops->get_mode) {
regulator_curr_mode = rdev->desc->ops->get_mode(rdev);
if (regulator_curr_mode == mode) {
ret = 0;
goto out;
}
}
/* constraints check */
ret = regulator_mode_constrain(rdev, &mode);
if (ret < 0)
goto out;
ret = rdev->desc->ops->set_mode(rdev, mode);
out:
regulator_unlock(rdev);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_mode);
static unsigned int _regulator_get_mode_unlocked(struct regulator_dev *rdev)
{
/* sanity check */
if (!rdev->desc->ops->get_mode)
return -EINVAL;
return rdev->desc->ops->get_mode(rdev);
}
static unsigned int _regulator_get_mode(struct regulator_dev *rdev)
{
int ret;
regulator_lock(rdev);
ret = _regulator_get_mode_unlocked(rdev);
regulator_unlock(rdev);
return ret;
}
/**
* regulator_get_mode - get regulator operating mode
* @regulator: regulator source
*
* Get the current regulator operating mode.
*/
unsigned int regulator_get_mode(struct regulator *regulator)
{
return _regulator_get_mode(regulator->rdev);
}
EXPORT_SYMBOL_GPL(regulator_get_mode);
static int rdev_get_cached_err_flags(struct regulator_dev *rdev)
{
int ret = 0;
if (rdev->use_cached_err) {
spin_lock(&rdev->err_lock);
ret = rdev->cached_err;
spin_unlock(&rdev->err_lock);
}
return ret;
}
static int _regulator_get_error_flags(struct regulator_dev *rdev,
unsigned int *flags)
{
int cached_flags, ret = 0;
regulator_lock(rdev);
cached_flags = rdev_get_cached_err_flags(rdev);
if (rdev->desc->ops->get_error_flags)
ret = rdev->desc->ops->get_error_flags(rdev, flags);
else if (!rdev->use_cached_err)
ret = -EINVAL;
*flags |= cached_flags;
regulator_unlock(rdev);
return ret;
}
/**
* regulator_get_error_flags - get regulator error information
* @regulator: regulator source
* @flags: pointer to store error flags
*
* Get the current regulator error information.
*/
int regulator_get_error_flags(struct regulator *regulator,
unsigned int *flags)
{
return _regulator_get_error_flags(regulator->rdev, flags);
}
EXPORT_SYMBOL_GPL(regulator_get_error_flags);
/**
* regulator_set_load - set regulator load
* @regulator: regulator source
* @uA_load: load current
*
* Notifies the regulator core of a new device load. This is then used by
* DRMS (if enabled by constraints) to set the most efficient regulator
* operating mode for the new regulator loading.
*
* Consumer devices notify their supply regulator of the maximum power
* they will require (can be taken from device datasheet in the power
* consumption tables) when they change operational status and hence power
* state. Examples of operational state changes that can affect power
* consumption are :-
*
* o Device is opened / closed.
* o Device I/O is about to begin or has just finished.
* o Device is idling in between work.
*
* This information is also exported via sysfs to userspace.
*
* DRMS will sum the total requested load on the regulator and change
* to the most efficient operating mode if platform constraints allow.
*
* NOTE: when a regulator consumer requests to have a regulator
* disabled then any load that consumer requested no longer counts
* toward the total requested load. If the regulator is re-enabled
* then the previously requested load will start counting again.
*
* If a regulator is an always-on regulator then an individual consumer's
* load will still be removed if that consumer is fully disabled.
*
* On error a negative errno is returned.
*/
int regulator_set_load(struct regulator *regulator, int uA_load)
{
struct regulator_dev *rdev = regulator->rdev;
int old_uA_load;
int ret = 0;
regulator_lock(rdev);
old_uA_load = regulator->uA_load;
regulator->uA_load = uA_load;
if (regulator->enable_count && old_uA_load != uA_load) {
ret = drms_uA_update(rdev);
if (ret < 0)
regulator->uA_load = old_uA_load;
}
regulator_unlock(rdev);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_load);
/**
* regulator_allow_bypass - allow the regulator to go into bypass mode
*
* @regulator: Regulator to configure
* @enable: enable or disable bypass mode
*
* Allow the regulator to go into bypass mode if all other consumers
* for the regulator also enable bypass mode and the machine
* constraints allow this. Bypass mode means that the regulator is
* simply passing the input directly to the output with no regulation.
*/
int regulator_allow_bypass(struct regulator *regulator, bool enable)
{
struct regulator_dev *rdev = regulator->rdev;
const char *name = rdev_get_name(rdev);
int ret = 0;
if (!rdev->desc->ops->set_bypass)
return 0;
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_BYPASS))
return 0;
regulator_lock(rdev);
if (enable && !regulator->bypass) {
rdev->bypass_count++;
if (rdev->bypass_count == rdev->open_count) {
trace_regulator_bypass_enable(name);
ret = rdev->desc->ops->set_bypass(rdev, enable);
if (ret != 0)
rdev->bypass_count--;
else
trace_regulator_bypass_enable_complete(name);
}
} else if (!enable && regulator->bypass) {
rdev->bypass_count--;
if (rdev->bypass_count != rdev->open_count) {
trace_regulator_bypass_disable(name);
ret = rdev->desc->ops->set_bypass(rdev, enable);
if (ret != 0)
rdev->bypass_count++;
else
trace_regulator_bypass_disable_complete(name);
}
}
if (ret == 0)
regulator->bypass = enable;
regulator_unlock(rdev);
return ret;
}
EXPORT_SYMBOL_GPL(regulator_allow_bypass);
/**
* regulator_register_notifier - register regulator event notifier
* @regulator: regulator source
* @nb: notifier block
*
* Register notifier block to receive regulator events.
*/
int regulator_register_notifier(struct regulator *regulator,
struct notifier_block *nb)
{
return blocking_notifier_chain_register(®ulator->rdev->notifier,
nb);
}
EXPORT_SYMBOL_GPL(regulator_register_notifier);
/**
* regulator_unregister_notifier - unregister regulator event notifier
* @regulator: regulator source
* @nb: notifier block
*
* Unregister regulator event notifier block.
*/
int regulator_unregister_notifier(struct regulator *regulator,
struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(®ulator->rdev->notifier,
nb);
}
EXPORT_SYMBOL_GPL(regulator_unregister_notifier);
/* notify regulator consumers and downstream regulator consumers.
* Note mutex must be held by caller.
*/
static int _notifier_call_chain(struct regulator_dev *rdev,
unsigned long event, void *data)
{
/* call rdev chain first */
int ret = blocking_notifier_call_chain(&rdev->notifier, event, data);
if (IS_REACHABLE(CONFIG_REGULATOR_NETLINK_EVENTS)) {
struct device *parent = rdev->dev.parent;
const char *rname = rdev_get_name(rdev);
char name[32];
/* Avoid duplicate debugfs directory names */
if (parent && rname == rdev->desc->name) {
snprintf(name, sizeof(name), "%s-%s", dev_name(parent),
rname);
rname = name;
}
reg_generate_netlink_event(rname, event);
}
return ret;
}
int _regulator_bulk_get(struct device *dev, int num_consumers,
struct regulator_bulk_data *consumers, enum regulator_get_type get_type)
{
int i;
int ret;
for (i = 0; i < num_consumers; i++)
consumers[i].consumer = NULL;
for (i = 0; i < num_consumers; i++) {
consumers[i].consumer = _regulator_get(dev,
consumers[i].supply, get_type);
if (IS_ERR(consumers[i].consumer)) {
ret = dev_err_probe(dev, PTR_ERR(consumers[i].consumer),
"Failed to get supply '%s'",
consumers[i].supply);
consumers[i].consumer = NULL;
goto err;
}
if (consumers[i].init_load_uA > 0) {
ret = regulator_set_load(consumers[i].consumer,
consumers[i].init_load_uA);
if (ret) {
i++;
goto err;
}
}
}
return 0;
err:
while (--i >= 0)
regulator_put(consumers[i].consumer);
return ret;
}
/**
* regulator_bulk_get - get multiple regulator consumers
*
* @dev: Device to supply
* @num_consumers: Number of consumers to register
* @consumers: Configuration of consumers; clients are stored here.
*
* @return 0 on success, an errno on failure.
*
* This helper function allows drivers to get several regulator
* consumers in one operation. If any of the regulators cannot be
* acquired then any regulators that were allocated will be freed
* before returning to the caller.
*/
int regulator_bulk_get(struct device *dev, int num_consumers,
struct regulator_bulk_data *consumers)
{
return _regulator_bulk_get(dev, num_consumers, consumers, NORMAL_GET);
}
EXPORT_SYMBOL_GPL(regulator_bulk_get);
static void regulator_bulk_enable_async(void *data, async_cookie_t cookie)
{
struct regulator_bulk_data *bulk = data;
bulk->ret = regulator_enable(bulk->consumer);
}
/**
* regulator_bulk_enable - enable multiple regulator consumers
*
* @num_consumers: Number of consumers
* @consumers: Consumer data; clients are stored here.
* @return 0 on success, an errno on failure
*
* This convenience API allows consumers to enable multiple regulator
* clients in a single API call. If any consumers cannot be enabled
* then any others that were enabled will be disabled again prior to
* return.
*/
int regulator_bulk_enable(int num_consumers,
struct regulator_bulk_data *consumers)
{
ASYNC_DOMAIN_EXCLUSIVE(async_domain);
int i;
int ret = 0;
for (i = 0; i < num_consumers; i++) {
async_schedule_domain(regulator_bulk_enable_async,
&consumers[i], &async_domain);
}
async_synchronize_full_domain(&async_domain);
/* If any consumer failed we need to unwind any that succeeded */
for (i = 0; i < num_consumers; i++) {
if (consumers[i].ret != 0) {
ret = consumers[i].ret;
goto err;
}
}
return 0;
err:
for (i = 0; i < num_consumers; i++) {
if (consumers[i].ret < 0)
pr_err("Failed to enable %s: %pe\n", consumers[i].supply,
ERR_PTR(consumers[i].ret));
else
regulator_disable(consumers[i].consumer);
}
return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_enable);
/**
* regulator_bulk_disable - disable multiple regulator consumers
*
* @num_consumers: Number of consumers
* @consumers: Consumer data; clients are stored here.
* @return 0 on success, an errno on failure
*
* This convenience API allows consumers to disable multiple regulator
* clients in a single API call. If any consumers cannot be disabled
* then any others that were disabled will be enabled again prior to
* return.
*/
int regulator_bulk_disable(int num_consumers,
struct regulator_bulk_data *consumers)
{
int i;
int ret, r;
for (i = num_consumers - 1; i >= 0; --i) {
ret = regulator_disable(consumers[i].consumer);
if (ret != 0)
goto err;
}
return 0;
err:
pr_err("Failed to disable %s: %pe\n", consumers[i].supply, ERR_PTR(ret));
for (++i; i < num_consumers; ++i) {
r = regulator_enable(consumers[i].consumer);
if (r != 0)
pr_err("Failed to re-enable %s: %pe\n",
consumers[i].supply, ERR_PTR(r));
}
return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_disable);
/**
* regulator_bulk_force_disable - force disable multiple regulator consumers
*
* @num_consumers: Number of consumers
* @consumers: Consumer data; clients are stored here.
* @return 0 on success, an errno on failure
*
* This convenience API allows consumers to forcibly disable multiple regulator
* clients in a single API call.
* NOTE: This should be used for situations when device damage will
* likely occur if the regulators are not disabled (e.g. over temp).
* Although regulator_force_disable function call for some consumers can
* return error numbers, the function is called for all consumers.
*/
int regulator_bulk_force_disable(int num_consumers,
struct regulator_bulk_data *consumers)
{
int i;
int ret = 0;
for (i = 0; i < num_consumers; i++) {
consumers[i].ret =
regulator_force_disable(consumers[i].consumer);
/* Store first error for reporting */
if (consumers[i].ret && !ret)
ret = consumers[i].ret;
}
return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_force_disable);
/**
* regulator_bulk_free - free multiple regulator consumers
*
* @num_consumers: Number of consumers
* @consumers: Consumer data; clients are stored here.
*
* This convenience API allows consumers to free multiple regulator
* clients in a single API call.
*/
void regulator_bulk_free(int num_consumers,
struct regulator_bulk_data *consumers)
{
int i;
for (i = 0; i < num_consumers; i++) {
regulator_put(consumers[i].consumer);
consumers[i].consumer = NULL;
}
}
EXPORT_SYMBOL_GPL(regulator_bulk_free);
/**
* regulator_handle_critical - Handle events for system-critical regulators.
* @rdev: The regulator device.
* @event: The event being handled.
*
* This function handles critical events such as under-voltage, over-current,
* and unknown errors for regulators deemed system-critical. On detecting such
* events, it triggers a hardware protection shutdown with a defined timeout.
*/
static void regulator_handle_critical(struct regulator_dev *rdev,
unsigned long event)
{
const char *reason = NULL;
if (!rdev->constraints->system_critical)
return;
switch (event) {
case REGULATOR_EVENT_UNDER_VOLTAGE:
reason = "System critical regulator: voltage drop detected";
break;
case REGULATOR_EVENT_OVER_CURRENT:
reason = "System critical regulator: over-current detected";
break;
case REGULATOR_EVENT_FAIL:
reason = "System critical regulator: unknown error";
}
if (!reason)
return;
hw_protection_shutdown(reason,
rdev->constraints->uv_less_critical_window_ms);
}
/**
* regulator_notifier_call_chain - call regulator event notifier
* @rdev: regulator source
* @event: notifier block
* @data: callback-specific data.
*
* Called by regulator drivers to notify clients a regulator event has
* occurred.
*/
int regulator_notifier_call_chain(struct regulator_dev *rdev,
unsigned long event, void *data)
{
regulator_handle_critical(rdev, event);
_notifier_call_chain(rdev, event, data);
return NOTIFY_DONE;
}
EXPORT_SYMBOL_GPL(regulator_notifier_call_chain);
/**
* regulator_mode_to_status - convert a regulator mode into a status
*
* @mode: Mode to convert
*
* Convert a regulator mode into a status.
*/
int regulator_mode_to_status(unsigned int mode)
{
switch (mode) {
case REGULATOR_MODE_FAST:
return REGULATOR_STATUS_FAST;
case REGULATOR_MODE_NORMAL:
return REGULATOR_STATUS_NORMAL;
case REGULATOR_MODE_IDLE:
return REGULATOR_STATUS_IDLE;
case REGULATOR_MODE_STANDBY:
return REGULATOR_STATUS_STANDBY;
default:
return REGULATOR_STATUS_UNDEFINED;
}
}
EXPORT_SYMBOL_GPL(regulator_mode_to_status);
static struct attribute *regulator_dev_attrs[] = {
&dev_attr_name.attr,
&dev_attr_num_users.attr,
&dev_attr_type.attr,
&dev_attr_microvolts.attr,
&dev_attr_microamps.attr,
&dev_attr_opmode.attr,
&dev_attr_state.attr,
&dev_attr_status.attr,
&dev_attr_bypass.attr,
&dev_attr_requested_microamps.attr,
&dev_attr_min_microvolts.attr,
&dev_attr_max_microvolts.attr,
&dev_attr_min_microamps.attr,
&dev_attr_max_microamps.attr,
&dev_attr_under_voltage.attr,
&dev_attr_over_current.attr,
&dev_attr_regulation_out.attr,
&dev_attr_fail.attr,
&dev_attr_over_temp.attr,
&dev_attr_under_voltage_warn.attr,
&dev_attr_over_current_warn.attr,
&dev_attr_over_voltage_warn.attr,
&dev_attr_over_temp_warn.attr,
&dev_attr_suspend_standby_state.attr,
&dev_attr_suspend_mem_state.attr,
&dev_attr_suspend_disk_state.attr,
&dev_attr_suspend_standby_microvolts.attr,
&dev_attr_suspend_mem_microvolts.attr,
&dev_attr_suspend_disk_microvolts.attr,
&dev_attr_suspend_standby_mode.attr,
&dev_attr_suspend_mem_mode.attr,
&dev_attr_suspend_disk_mode.attr,
NULL
};
/*
* To avoid cluttering sysfs (and memory) with useless state, only
* create attributes that can be meaningfully displayed.
*/
static umode_t regulator_attr_is_visible(struct kobject *kobj,
struct attribute *attr, int idx)
{
struct device *dev = kobj_to_dev(kobj);
struct regulator_dev *rdev = dev_to_rdev(dev);
const struct regulator_ops *ops = rdev->desc->ops;
umode_t mode = attr->mode;
/* these three are always present */
if (attr == &dev_attr_name.attr ||
attr == &dev_attr_num_users.attr ||
attr == &dev_attr_type.attr)
return mode;
/* some attributes need specific methods to be displayed */
if (attr == &dev_attr_microvolts.attr) {
if ((ops->get_voltage && ops->get_voltage(rdev) >= 0) ||
(ops->get_voltage_sel && ops->get_voltage_sel(rdev) >= 0) ||
(ops->list_voltage && ops->list_voltage(rdev, 0) >= 0) ||
(rdev->desc->fixed_uV && rdev->desc->n_voltages == 1))
return mode;
return 0;
}
if (attr == &dev_attr_microamps.attr)
return ops->get_current_limit ? mode : 0;
if (attr == &dev_attr_opmode.attr)
return ops->get_mode ? mode : 0;
if (attr == &dev_attr_state.attr)
return (rdev->ena_pin || ops->is_enabled) ? mode : 0;
if (attr == &dev_attr_status.attr)
return ops->get_status ? mode : 0;
if (attr == &dev_attr_bypass.attr)
return ops->get_bypass ? mode : 0;
if (attr == &dev_attr_under_voltage.attr ||
attr == &dev_attr_over_current.attr ||
attr == &dev_attr_regulation_out.attr ||
attr == &dev_attr_fail.attr ||
attr == &dev_attr_over_temp.attr ||
attr == &dev_attr_under_voltage_warn.attr ||
attr == &dev_attr_over_current_warn.attr ||
attr == &dev_attr_over_voltage_warn.attr ||
attr == &dev_attr_over_temp_warn.attr)
return ops->get_error_flags ? mode : 0;
/* constraints need specific supporting methods */
if (attr == &dev_attr_min_microvolts.attr ||
attr == &dev_attr_max_microvolts.attr)
return (ops->set_voltage || ops->set_voltage_sel) ? mode : 0;
if (attr == &dev_attr_min_microamps.attr ||
attr == &dev_attr_max_microamps.attr)
return ops->set_current_limit ? mode : 0;
if (attr == &dev_attr_suspend_standby_state.attr ||
attr == &dev_attr_suspend_mem_state.attr ||
attr == &dev_attr_suspend_disk_state.attr)
return mode;
if (attr == &dev_attr_suspend_standby_microvolts.attr ||
attr == &dev_attr_suspend_mem_microvolts.attr ||
attr == &dev_attr_suspend_disk_microvolts.attr)
return ops->set_suspend_voltage ? mode : 0;
if (attr == &dev_attr_suspend_standby_mode.attr ||
attr == &dev_attr_suspend_mem_mode.attr ||
attr == &dev_attr_suspend_disk_mode.attr)
return ops->set_suspend_mode ? mode : 0;
return mode;
}
static const struct attribute_group regulator_dev_group = {
.attrs = regulator_dev_attrs,
.is_visible = regulator_attr_is_visible,
};
static const struct attribute_group *regulator_dev_groups[] = {
®ulator_dev_group,
NULL
};
static void regulator_dev_release(struct device *dev)
{
struct regulator_dev *rdev = dev_get_drvdata(dev);
debugfs_remove_recursive(rdev->debugfs);
kfree(rdev->constraints);
of_node_put(rdev->dev.of_node);
kfree(rdev);
}
static void rdev_init_debugfs(struct regulator_dev *rdev)
{
struct device *parent = rdev->dev.parent;
const char *rname = rdev_get_name(rdev);
char name[NAME_MAX];
/* Avoid duplicate debugfs directory names */
if (parent && rname == rdev->desc->name) {
snprintf(name, sizeof(name), "%s-%s", dev_name(parent),
rname);
rname = name;
}
rdev->debugfs = debugfs_create_dir(rname, debugfs_root);
if (IS_ERR(rdev->debugfs))
rdev_dbg(rdev, "Failed to create debugfs directory\n");
debugfs_create_u32("use_count", 0444, rdev->debugfs,
&rdev->use_count);
debugfs_create_u32("open_count", 0444, rdev->debugfs,
&rdev->open_count);
debugfs_create_u32("bypass_count", 0444, rdev->debugfs,
&rdev->bypass_count);
}
static int regulator_register_resolve_supply(struct device *dev, void *data)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
if (regulator_resolve_supply(rdev))
rdev_dbg(rdev, "unable to resolve supply\n");
return 0;
}
int regulator_coupler_register(struct regulator_coupler *coupler)
{
mutex_lock(®ulator_list_mutex);
list_add_tail(&coupler->list, ®ulator_coupler_list);
mutex_unlock(®ulator_list_mutex);
return 0;
}
static struct regulator_coupler *
regulator_find_coupler(struct regulator_dev *rdev)
{
struct regulator_coupler *coupler;
int err;
/*
* Note that regulators are appended to the list and the generic
* coupler is registered first, hence it will be attached at last
* if nobody cared.
*/
list_for_each_entry_reverse(coupler, ®ulator_coupler_list, list) {
err = coupler->attach_regulator(coupler, rdev);
if (!err) {
if (!coupler->balance_voltage &&
rdev->coupling_desc.n_coupled > 2)
goto err_unsupported;
return coupler;
}
if (err < 0)
return ERR_PTR(err);
if (err == 1)
continue;
break;
}
return ERR_PTR(-EINVAL);
err_unsupported:
if (coupler->detach_regulator)
coupler->detach_regulator(coupler, rdev);
rdev_err(rdev,
"Voltage balancing for multiple regulator couples is unimplemented\n");
return ERR_PTR(-EPERM);
}
static void regulator_resolve_coupling(struct regulator_dev *rdev)
{
struct regulator_coupler *coupler = rdev->coupling_desc.coupler;
struct coupling_desc *c_desc = &rdev->coupling_desc;
int n_coupled = c_desc->n_coupled;
struct regulator_dev *c_rdev;
int i;
for (i = 1; i < n_coupled; i++) {
/* already resolved */
if (c_desc->coupled_rdevs[i])
continue;
c_rdev = of_parse_coupled_regulator(rdev, i - 1);
if (!c_rdev)
continue;
if (c_rdev->coupling_desc.coupler != coupler) {
rdev_err(rdev, "coupler mismatch with %s\n",
rdev_get_name(c_rdev));
return;
}
c_desc->coupled_rdevs[i] = c_rdev;
c_desc->n_resolved++;
regulator_resolve_coupling(c_rdev);
}
}
static void regulator_remove_coupling(struct regulator_dev *rdev)
{
struct regulator_coupler *coupler = rdev->coupling_desc.coupler;
struct coupling_desc *__c_desc, *c_desc = &rdev->coupling_desc;
struct regulator_dev *__c_rdev, *c_rdev;
unsigned int __n_coupled, n_coupled;
int i, k;
int err;
n_coupled = c_desc->n_coupled;
for (i = 1; i < n_coupled; i++) {
c_rdev = c_desc->coupled_rdevs[i];
if (!c_rdev)
continue;
regulator_lock(c_rdev);
__c_desc = &c_rdev->coupling_desc;
__n_coupled = __c_desc->n_coupled;
for (k = 1; k < __n_coupled; k++) {
__c_rdev = __c_desc->coupled_rdevs[k];
if (__c_rdev == rdev) {
__c_desc->coupled_rdevs[k] = NULL;
__c_desc->n_resolved--;
break;
}
}
regulator_unlock(c_rdev);
c_desc->coupled_rdevs[i] = NULL;
c_desc->n_resolved--;
}
if (coupler && coupler->detach_regulator) {
err = coupler->detach_regulator(coupler, rdev);
if (err)
rdev_err(rdev, "failed to detach from coupler: %pe\n",
ERR_PTR(err));
}
kfree(rdev->coupling_desc.coupled_rdevs);
rdev->coupling_desc.coupled_rdevs = NULL;
}
static int regulator_init_coupling(struct regulator_dev *rdev)
{
struct regulator_dev **coupled;
int err, n_phandles;
if (!IS_ENABLED(CONFIG_OF))
n_phandles = 0;
else
n_phandles = of_get_n_coupled(rdev);
coupled = kcalloc(n_phandles + 1, sizeof(*coupled), GFP_KERNEL);
if (!coupled)
return -ENOMEM;
rdev->coupling_desc.coupled_rdevs = coupled;
/*
* Every regulator should always have coupling descriptor filled with
* at least pointer to itself.
*/
rdev->coupling_desc.coupled_rdevs[0] = rdev;
rdev->coupling_desc.n_coupled = n_phandles + 1;
rdev->coupling_desc.n_resolved++;
/* regulator isn't coupled */
if (n_phandles == 0)
return 0;
if (!of_check_coupling_data(rdev))
return -EPERM;
mutex_lock(®ulator_list_mutex);
rdev->coupling_desc.coupler = regulator_find_coupler(rdev);
mutex_unlock(®ulator_list_mutex);
if (IS_ERR(rdev->coupling_desc.coupler)) {
err = PTR_ERR(rdev->coupling_desc.coupler);
rdev_err(rdev, "failed to get coupler: %pe\n", ERR_PTR(err));
return err;
}
return 0;
}
static int generic_coupler_attach(struct regulator_coupler *coupler,
struct regulator_dev *rdev)
{
if (rdev->coupling_desc.n_coupled > 2) {
rdev_err(rdev,
"Voltage balancing for multiple regulator couples is unimplemented\n");
return -EPERM;
}
if (!rdev->constraints->always_on) {
rdev_err(rdev,
"Coupling of a non always-on regulator is unimplemented\n");
return -ENOTSUPP;
}
return 0;
}
static struct regulator_coupler generic_regulator_coupler = {
.attach_regulator = generic_coupler_attach,
};
/**
* regulator_register - register regulator
* @dev: the device that drive the regulator
* @regulator_desc: regulator to register
* @cfg: runtime configuration for regulator
*
* Called by regulator drivers to register a regulator.
* Returns a valid pointer to struct regulator_dev on success
* or an ERR_PTR() on error.
*/
struct regulator_dev *
regulator_register(struct device *dev,
const struct regulator_desc *regulator_desc,
const struct regulator_config *cfg)
{
const struct regulator_init_data *init_data;
struct regulator_config *config = NULL;
static atomic_t regulator_no = ATOMIC_INIT(-1);
struct regulator_dev *rdev;
bool dangling_cfg_gpiod = false;
bool dangling_of_gpiod = false;
int ret, i;
bool resolved_early = false;
if (cfg == NULL)
return ERR_PTR(-EINVAL);
if (cfg->ena_gpiod)
dangling_cfg_gpiod = true;
if (regulator_desc == NULL) {
ret = -EINVAL;
goto rinse;
}
WARN_ON(!dev || !cfg->dev);
if (regulator_desc->name == NULL || regulator_desc->ops == NULL) {
ret = -EINVAL;
goto rinse;
}
if (regulator_desc->type != REGULATOR_VOLTAGE &&
regulator_desc->type != REGULATOR_CURRENT) {
ret = -EINVAL;
goto rinse;
}
/* Only one of each should be implemented */
WARN_ON(regulator_desc->ops->get_voltage &&
regulator_desc->ops->get_voltage_sel);
WARN_ON(regulator_desc->ops->set_voltage &&
regulator_desc->ops->set_voltage_sel);
/* If we're using selectors we must implement list_voltage. */
if (regulator_desc->ops->get_voltage_sel &&
!regulator_desc->ops->list_voltage) {
ret = -EINVAL;
goto rinse;
}
if (regulator_desc->ops->set_voltage_sel &&
!regulator_desc->ops->list_voltage) {
ret = -EINVAL;
goto rinse;
}
rdev = kzalloc(sizeof(struct regulator_dev), GFP_KERNEL);
if (rdev == NULL) {
ret = -ENOMEM;
goto rinse;
}
device_initialize(&rdev->dev);
dev_set_drvdata(&rdev->dev, rdev);
rdev->dev.class = ®ulator_class;
spin_lock_init(&rdev->err_lock);
/*
* Duplicate the config so the driver could override it after
* parsing init data.
*/
config = kmemdup(cfg, sizeof(*cfg), GFP_KERNEL);
if (config == NULL) {
ret = -ENOMEM;
goto clean;
}
init_data = regulator_of_get_init_data(dev, regulator_desc, config,
&rdev->dev.of_node);
/*
* Sometimes not all resources are probed already so we need to take
* that into account. This happens most the time if the ena_gpiod comes
* from a gpio extender or something else.
*/
if (PTR_ERR(init_data) == -EPROBE_DEFER) {
ret = -EPROBE_DEFER;
goto clean;
}
/*
* We need to keep track of any GPIO descriptor coming from the
* device tree until we have handled it over to the core. If the
* config that was passed in to this function DOES NOT contain
* a descriptor, and the config after this call DOES contain
* a descriptor, we definitely got one from parsing the device
* tree.
*/
if (!cfg->ena_gpiod && config->ena_gpiod)
dangling_of_gpiod = true;
if (!init_data) {
init_data = config->init_data;
rdev->dev.of_node = of_node_get(config->of_node);
}
ww_mutex_init(&rdev->mutex, ®ulator_ww_class);
rdev->reg_data = config->driver_data;
rdev->owner = regulator_desc->owner;
rdev->desc = regulator_desc;
if (config->regmap)
rdev->regmap = config->regmap;
else if (dev_get_regmap(dev, NULL))
rdev->regmap = dev_get_regmap(dev, NULL);
else if (dev->parent)
rdev->regmap = dev_get_regmap(dev->parent, NULL);
INIT_LIST_HEAD(&rdev->consumer_list);
INIT_LIST_HEAD(&rdev->list);
BLOCKING_INIT_NOTIFIER_HEAD(&rdev->notifier);
INIT_DELAYED_WORK(&rdev->disable_work, regulator_disable_work);
if (init_data && init_data->supply_regulator)
rdev->supply_name = init_data->supply_regulator;
else if (regulator_desc->supply_name)
rdev->supply_name = regulator_desc->supply_name;
/* register with sysfs */
rdev->dev.parent = config->dev;
dev_set_name(&rdev->dev, "regulator.%lu",
(unsigned long) atomic_inc_return(®ulator_no));
/* set regulator constraints */
if (init_data)
rdev->constraints = kmemdup(&init_data->constraints,
sizeof(*rdev->constraints),
GFP_KERNEL);
else
rdev->constraints = kzalloc(sizeof(*rdev->constraints),
GFP_KERNEL);
if (!rdev->constraints) {
ret = -ENOMEM;
goto wash;
}
if ((rdev->supply_name && !rdev->supply) &&
(rdev->constraints->always_on ||
rdev->constraints->boot_on)) {
ret = regulator_resolve_supply(rdev);
if (ret)
rdev_dbg(rdev, "unable to resolve supply early: %pe\n",
ERR_PTR(ret));
resolved_early = true;
}
/* perform any regulator specific init */
if (init_data && init_data->regulator_init) {
ret = init_data->regulator_init(rdev->reg_data);
if (ret < 0)
goto wash;
}
if (config->ena_gpiod) {
ret = regulator_ena_gpio_request(rdev, config);
if (ret != 0) {
rdev_err(rdev, "Failed to request enable GPIO: %pe\n",
ERR_PTR(ret));
goto wash;
}
/* The regulator core took over the GPIO descriptor */
dangling_cfg_gpiod = false;
dangling_of_gpiod = false;
}
ret = set_machine_constraints(rdev);
if (ret == -EPROBE_DEFER && !resolved_early) {
/* Regulator might be in bypass mode and so needs its supply
* to set the constraints
*/
/* FIXME: this currently triggers a chicken-and-egg problem
* when creating -SUPPLY symlink in sysfs to a regulator
* that is just being created
*/
rdev_dbg(rdev, "will resolve supply early: %s\n",
rdev->supply_name);
ret = regulator_resolve_supply(rdev);
if (!ret)
ret = set_machine_constraints(rdev);
else
rdev_dbg(rdev, "unable to resolve supply early: %pe\n",
ERR_PTR(ret));
}
if (ret < 0)
goto wash;
ret = regulator_init_coupling(rdev);
if (ret < 0)
goto wash;
/* add consumers devices */
if (init_data) {
for (i = 0; i < init_data->num_consumer_supplies; i++) {
ret = set_consumer_device_supply(rdev,
init_data->consumer_supplies[i].dev_name,
init_data->consumer_supplies[i].supply);
if (ret < 0) {
dev_err(dev, "Failed to set supply %s\n",
init_data->consumer_supplies[i].supply);
goto unset_supplies;
}
}
}
if (!rdev->desc->ops->get_voltage &&
!rdev->desc->ops->list_voltage &&
!rdev->desc->fixed_uV)
rdev->is_switch = true;
ret = device_add(&rdev->dev);
if (ret != 0)
goto unset_supplies;
rdev_init_debugfs(rdev);
/* try to resolve regulators coupling since a new one was registered */
mutex_lock(®ulator_list_mutex);
regulator_resolve_coupling(rdev);
mutex_unlock(®ulator_list_mutex);
/* try to resolve regulators supply since a new one was registered */
class_for_each_device(®ulator_class, NULL, NULL,
regulator_register_resolve_supply);
kfree(config);
return rdev;
unset_supplies:
mutex_lock(®ulator_list_mutex);
unset_regulator_supplies(rdev);
regulator_remove_coupling(rdev);
mutex_unlock(®ulator_list_mutex);
wash:
regulator_put(rdev->supply);
kfree(rdev->coupling_desc.coupled_rdevs);
mutex_lock(®ulator_list_mutex);
regulator_ena_gpio_free(rdev);
mutex_unlock(®ulator_list_mutex);
clean:
if (dangling_of_gpiod)
gpiod_put(config->ena_gpiod);
kfree(config);
put_device(&rdev->dev);
rinse:
if (dangling_cfg_gpiod)
gpiod_put(cfg->ena_gpiod);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(regulator_register);
/**
* regulator_unregister - unregister regulator
* @rdev: regulator to unregister
*
* Called by regulator drivers to unregister a regulator.
*/
void regulator_unregister(struct regulator_dev *rdev)
{
if (rdev == NULL)
return;
if (rdev->supply) {
while (rdev->use_count--)
regulator_disable(rdev->supply);
regulator_put(rdev->supply);
}
flush_work(&rdev->disable_work.work);
mutex_lock(®ulator_list_mutex);
WARN_ON(rdev->open_count);
regulator_remove_coupling(rdev);
unset_regulator_supplies(rdev);
list_del(&rdev->list);
regulator_ena_gpio_free(rdev);
device_unregister(&rdev->dev);
mutex_unlock(®ulator_list_mutex);
}
EXPORT_SYMBOL_GPL(regulator_unregister);
#ifdef CONFIG_SUSPEND
/**
* regulator_suspend - prepare regulators for system wide suspend
* @dev: ``&struct device`` pointer that is passed to _regulator_suspend()
*
* Configure each regulator with it's suspend operating parameters for state.
*/
static int regulator_suspend(struct device *dev)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
suspend_state_t state = pm_suspend_target_state;
int ret;
const struct regulator_state *rstate;
rstate = regulator_get_suspend_state_check(rdev, state);
if (!rstate)
return 0;
regulator_lock(rdev);
ret = __suspend_set_state(rdev, rstate);
regulator_unlock(rdev);
return ret;
}
static int regulator_resume(struct device *dev)
{
suspend_state_t state = pm_suspend_target_state;
struct regulator_dev *rdev = dev_to_rdev(dev);
struct regulator_state *rstate;
int ret = 0;
rstate = regulator_get_suspend_state(rdev, state);
if (rstate == NULL)
return 0;
/* Avoid grabbing the lock if we don't need to */
if (!rdev->desc->ops->resume)
return 0;
regulator_lock(rdev);
if (rstate->enabled == ENABLE_IN_SUSPEND ||
rstate->enabled == DISABLE_IN_SUSPEND)
ret = rdev->desc->ops->resume(rdev);
regulator_unlock(rdev);
return ret;
}
#else /* !CONFIG_SUSPEND */
#define regulator_suspend NULL
#define regulator_resume NULL
#endif /* !CONFIG_SUSPEND */
#ifdef CONFIG_PM
static const struct dev_pm_ops __maybe_unused regulator_pm_ops = {
.suspend = regulator_suspend,
.resume = regulator_resume,
};
#endif
const struct class regulator_class = {
.name = "regulator",
.dev_release = regulator_dev_release,
.dev_groups = regulator_dev_groups,
#ifdef CONFIG_PM
.pm = ®ulator_pm_ops,
#endif
};
/**
* regulator_has_full_constraints - the system has fully specified constraints
*
* Calling this function will cause the regulator API to disable all
* regulators which have a zero use count and don't have an always_on
* constraint in a late_initcall.
*
* The intention is that this will become the default behaviour in a
* future kernel release so users are encouraged to use this facility
* now.
*/
void regulator_has_full_constraints(void)
{
has_full_constraints = 1;
}
EXPORT_SYMBOL_GPL(regulator_has_full_constraints);
/**
* rdev_get_drvdata - get rdev regulator driver data
* @rdev: regulator
*
* Get rdev regulator driver private data. This call can be used in the
* regulator driver context.
*/
void *rdev_get_drvdata(struct regulator_dev *rdev)
{
return rdev->reg_data;
}
EXPORT_SYMBOL_GPL(rdev_get_drvdata);
/**
* regulator_get_drvdata - get regulator driver data
* @regulator: regulator
*
* Get regulator driver private data. This call can be used in the consumer
* driver context when non API regulator specific functions need to be called.
*/
void *regulator_get_drvdata(struct regulator *regulator)
{
return regulator->rdev->reg_data;
}
EXPORT_SYMBOL_GPL(regulator_get_drvdata);
/**
* regulator_set_drvdata - set regulator driver data
* @regulator: regulator
* @data: data
*/
void regulator_set_drvdata(struct regulator *regulator, void *data)
{
regulator->rdev->reg_data = data;
}
EXPORT_SYMBOL_GPL(regulator_set_drvdata);
/**
* rdev_get_id - get regulator ID
* @rdev: regulator
*/
int rdev_get_id(struct regulator_dev *rdev)
{
return rdev->desc->id;
}
EXPORT_SYMBOL_GPL(rdev_get_id);
struct device *rdev_get_dev(struct regulator_dev *rdev)
{
return &rdev->dev;
}
EXPORT_SYMBOL_GPL(rdev_get_dev);
struct regmap *rdev_get_regmap(struct regulator_dev *rdev)
{
return rdev->regmap;
}
EXPORT_SYMBOL_GPL(rdev_get_regmap);
void *regulator_get_init_drvdata(struct regulator_init_data *reg_init_data)
{
return reg_init_data->driver_data;
}
EXPORT_SYMBOL_GPL(regulator_get_init_drvdata);
#ifdef CONFIG_DEBUG_FS
static int supply_map_show(struct seq_file *sf, void *data)
{
struct regulator_map *map;
list_for_each_entry(map, ®ulator_map_list, list) {
seq_printf(sf, "%s -> %s.%s\n",
rdev_get_name(map->regulator), map->dev_name,
map->supply);
}
return 0;
}
DEFINE_SHOW_ATTRIBUTE(supply_map);
struct summary_data {
struct seq_file *s;
struct regulator_dev *parent;
int level;
};
static void regulator_summary_show_subtree(struct seq_file *s,
struct regulator_dev *rdev,
int level);
static int regulator_summary_show_children(struct device *dev, void *data)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
struct summary_data *summary_data = data;
if (rdev->supply && rdev->supply->rdev == summary_data->parent)
regulator_summary_show_subtree(summary_data->s, rdev,
summary_data->level + 1);
return 0;
}
static void regulator_summary_show_subtree(struct seq_file *s,
struct regulator_dev *rdev,
int level)
{
struct regulation_constraints *c;
struct regulator *consumer;
struct summary_data summary_data;
unsigned int opmode;
if (!rdev)
return;
opmode = _regulator_get_mode_unlocked(rdev);
seq_printf(s, "%*s%-*s %3d %4d %6d %7s ",
level * 3 + 1, "",
30 - level * 3, rdev_get_name(rdev),
rdev->use_count, rdev->open_count, rdev->bypass_count,
regulator_opmode_to_str(opmode));
seq_printf(s, "%5dmV ", regulator_get_voltage_rdev(rdev) / 1000);
seq_printf(s, "%5dmA ",
_regulator_get_current_limit_unlocked(rdev) / 1000);
c = rdev->constraints;
if (c) {
switch (rdev->desc->type) {
case REGULATOR_VOLTAGE:
seq_printf(s, "%5dmV %5dmV ",
c->min_uV / 1000, c->max_uV / 1000);
break;
case REGULATOR_CURRENT:
seq_printf(s, "%5dmA %5dmA ",
c->min_uA / 1000, c->max_uA / 1000);
break;
}
}
seq_puts(s, "\n");
list_for_each_entry(consumer, &rdev->consumer_list, list) {
if (consumer->dev && consumer->dev->class == ®ulator_class)
continue;
seq_printf(s, "%*s%-*s ",
(level + 1) * 3 + 1, "",
30 - (level + 1) * 3,
consumer->supply_name ? consumer->supply_name :
consumer->dev ? dev_name(consumer->dev) : "deviceless");
switch (rdev->desc->type) {
case REGULATOR_VOLTAGE:
seq_printf(s, "%3d %33dmA%c%5dmV %5dmV",
consumer->enable_count,
consumer->uA_load / 1000,
consumer->uA_load && !consumer->enable_count ?
'*' : ' ',
consumer->voltage[PM_SUSPEND_ON].min_uV / 1000,
consumer->voltage[PM_SUSPEND_ON].max_uV / 1000);
break;
case REGULATOR_CURRENT:
break;
}
seq_puts(s, "\n");
}
summary_data.s = s;
summary_data.level = level;
summary_data.parent = rdev;
class_for_each_device(®ulator_class, NULL, &summary_data,
regulator_summary_show_children);
}
struct summary_lock_data {
struct ww_acquire_ctx *ww_ctx;
struct regulator_dev **new_contended_rdev;
struct regulator_dev **old_contended_rdev;
};
static int regulator_summary_lock_one(struct device *dev, void *data)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
struct summary_lock_data *lock_data = data;
int ret = 0;
if (rdev != *lock_data->old_contended_rdev) {
ret = regulator_lock_nested(rdev, lock_data->ww_ctx);
if (ret == -EDEADLK)
*lock_data->new_contended_rdev = rdev;
else
WARN_ON_ONCE(ret);
} else {
*lock_data->old_contended_rdev = NULL;
}
return ret;
}
static int regulator_summary_unlock_one(struct device *dev, void *data)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
struct summary_lock_data *lock_data = data;
if (lock_data) {
if (rdev == *lock_data->new_contended_rdev)
return -EDEADLK;
}
regulator_unlock(rdev);
return 0;
}
static int regulator_summary_lock_all(struct ww_acquire_ctx *ww_ctx,
struct regulator_dev **new_contended_rdev,
struct regulator_dev **old_contended_rdev)
{
struct summary_lock_data lock_data;
int ret;
lock_data.ww_ctx = ww_ctx;
lock_data.new_contended_rdev = new_contended_rdev;
lock_data.old_contended_rdev = old_contended_rdev;
ret = class_for_each_device(®ulator_class, NULL, &lock_data,
regulator_summary_lock_one);
if (ret)
class_for_each_device(®ulator_class, NULL, &lock_data,
regulator_summary_unlock_one);
return ret;
}
static void regulator_summary_lock(struct ww_acquire_ctx *ww_ctx)
{
struct regulator_dev *new_contended_rdev = NULL;
struct regulator_dev *old_contended_rdev = NULL;
int err;
mutex_lock(®ulator_list_mutex);
ww_acquire_init(ww_ctx, ®ulator_ww_class);
do {
if (new_contended_rdev) {
ww_mutex_lock_slow(&new_contended_rdev->mutex, ww_ctx);
old_contended_rdev = new_contended_rdev;
old_contended_rdev->ref_cnt++;
old_contended_rdev->mutex_owner = current;
}
err = regulator_summary_lock_all(ww_ctx,
&new_contended_rdev,
&old_contended_rdev);
if (old_contended_rdev)
regulator_unlock(old_contended_rdev);
} while (err == -EDEADLK);
ww_acquire_done(ww_ctx);
}
static void regulator_summary_unlock(struct ww_acquire_ctx *ww_ctx)
{
class_for_each_device(®ulator_class, NULL, NULL,
regulator_summary_unlock_one);
ww_acquire_fini(ww_ctx);
mutex_unlock(®ulator_list_mutex);
}
static int regulator_summary_show_roots(struct device *dev, void *data)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
struct seq_file *s = data;
if (!rdev->supply)
regulator_summary_show_subtree(s, rdev, 0);
return 0;
}
static int regulator_summary_show(struct seq_file *s, void *data)
{
struct ww_acquire_ctx ww_ctx;
seq_puts(s, " regulator use open bypass opmode voltage current min max\n");
seq_puts(s, "---------------------------------------------------------------------------------------\n");
regulator_summary_lock(&ww_ctx);
class_for_each_device(®ulator_class, NULL, s,
regulator_summary_show_roots);
regulator_summary_unlock(&ww_ctx);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(regulator_summary);
#endif /* CONFIG_DEBUG_FS */
static int __init regulator_init(void)
{
int ret;
ret = class_register(®ulator_class);
debugfs_root = debugfs_create_dir("regulator", NULL);
if (IS_ERR(debugfs_root))
pr_debug("regulator: Failed to create debugfs directory\n");
#ifdef CONFIG_DEBUG_FS
debugfs_create_file("supply_map", 0444, debugfs_root, NULL,
&supply_map_fops);
debugfs_create_file("regulator_summary", 0444, debugfs_root,
NULL, ®ulator_summary_fops);
#endif
regulator_dummy_init();
regulator_coupler_register(&generic_regulator_coupler);
return ret;
}
/* init early to allow our consumers to complete system booting */
core_initcall(regulator_init);
static int regulator_late_cleanup(struct device *dev, void *data)
{
struct regulator_dev *rdev = dev_to_rdev(dev);
struct regulation_constraints *c = rdev->constraints;
int ret;
if (c && c->always_on)
return 0;
if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS))
return 0;
regulator_lock(rdev);
if (rdev->use_count)
goto unlock;
/* If reading the status failed, assume that it's off. */
if (_regulator_is_enabled(rdev) <= 0)
goto unlock;
if (have_full_constraints()) {
/* We log since this may kill the system if it goes
* wrong.
*/
rdev_info(rdev, "disabling\n");
ret = _regulator_do_disable(rdev);
if (ret != 0)
rdev_err(rdev, "couldn't disable: %pe\n", ERR_PTR(ret));
} else {
/* The intention is that in future we will
* assume that full constraints are provided
* so warn even if we aren't going to do
* anything here.
*/
rdev_warn(rdev, "incomplete constraints, leaving on\n");
}
unlock:
regulator_unlock(rdev);
return 0;
}
static bool regulator_ignore_unused;
static int __init regulator_ignore_unused_setup(char *__unused)
{
regulator_ignore_unused = true;
return 1;
}
__setup("regulator_ignore_unused", regulator_ignore_unused_setup);
static void regulator_init_complete_work_function(struct work_struct *work)
{
/*
* Regulators may had failed to resolve their input supplies
* when were registered, either because the input supply was
* not registered yet or because its parent device was not
* bound yet. So attempt to resolve the input supplies for
* pending regulators before trying to disable unused ones.
*/
class_for_each_device(®ulator_class, NULL, NULL,
regulator_register_resolve_supply);
/*
* For debugging purposes, it may be useful to prevent unused
* regulators from being disabled.
*/
if (regulator_ignore_unused) {
pr_warn("regulator: Not disabling unused regulators\n");
return;
}
/* If we have a full configuration then disable any regulators
* we have permission to change the status for and which are
* not in use or always_on. This is effectively the default
* for DT and ACPI as they have full constraints.
*/
class_for_each_device(®ulator_class, NULL, NULL,
regulator_late_cleanup);
}
static DECLARE_DELAYED_WORK(regulator_init_complete_work,
regulator_init_complete_work_function);
static int __init regulator_init_complete(void)
{
/*
* Since DT doesn't provide an idiomatic mechanism for
* enabling full constraints and since it's much more natural
* with DT to provide them just assume that a DT enabled
* system has full constraints.
*/
if (of_have_populated_dt())
has_full_constraints = true;
/*
* We punt completion for an arbitrary amount of time since
* systems like distros will load many drivers from userspace
* so consumers might not always be ready yet, this is
* particularly an issue with laptops where this might bounce
* the display off then on. Ideally we'd get a notification
* from userspace when this happens but we don't so just wait
* a bit and hope we waited long enough. It'd be better if
* we'd only do this on systems that need it, and a kernel
* command line option might be useful.
*/
schedule_delayed_work(®ulator_init_complete_work,
msecs_to_jiffies(30000));
return 0;
}
late_initcall_sync(regulator_init_complete);
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/power/wakeup.c - System wakeup events framework
*
* Copyright (c) 2010 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
*/
#define pr_fmt(fmt) "PM: " fmt
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/capability.h>
#include <linux/export.h>
#include <linux/suspend.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/pm_wakeirq.h>
#include <trace/events/power.h>
#include "power.h"
#define list_for_each_entry_rcu_locked(pos, head, member) \
list_for_each_entry_rcu(pos, head, member, \
srcu_read_lock_held(&wakeup_srcu))
/*
* If set, the suspend/hibernate code will abort transitions to a sleep state
* if wakeup events are registered during or immediately before the transition.
*/
bool events_check_enabled __read_mostly;
/* First wakeup IRQ seen by the kernel in the last cycle. */
static unsigned int wakeup_irq[2] __read_mostly;
static DEFINE_RAW_SPINLOCK(wakeup_irq_lock);
/* If greater than 0 and the system is suspending, terminate the suspend. */
static atomic_t pm_abort_suspend __read_mostly;
/*
* Combined counters of registered wakeup events and wakeup events in progress.
* They need to be modified together atomically, so it's better to use one
* atomic variable to hold them both.
*/
static atomic_t combined_event_count = ATOMIC_INIT(0);
#define IN_PROGRESS_BITS (sizeof(int) * 4)
#define MAX_IN_PROGRESS ((1 << IN_PROGRESS_BITS) - 1)
static void split_counters(unsigned int *cnt, unsigned int *inpr)
{
unsigned int comb = atomic_read(&combined_event_count);
*cnt = (comb >> IN_PROGRESS_BITS);
*inpr = comb & MAX_IN_PROGRESS;
}
/* A preserved old value of the events counter. */
static unsigned int saved_count;
static DEFINE_RAW_SPINLOCK(events_lock);
static void pm_wakeup_timer_fn(struct timer_list *t);
static LIST_HEAD(wakeup_sources);
static DECLARE_WAIT_QUEUE_HEAD(wakeup_count_wait_queue);
DEFINE_STATIC_SRCU(wakeup_srcu);
static struct wakeup_source deleted_ws = {
.name = "deleted",
.lock = __SPIN_LOCK_UNLOCKED(deleted_ws.lock),
};
static DEFINE_IDA(wakeup_ida);
/**
* wakeup_source_create - Create a struct wakeup_source object.
* @name: Name of the new wakeup source.
*/
struct wakeup_source *wakeup_source_create(const char *name)
{
struct wakeup_source *ws;
const char *ws_name;
int id;
ws = kzalloc(sizeof(*ws), GFP_KERNEL);
if (!ws)
goto err_ws;
ws_name = kstrdup_const(name, GFP_KERNEL);
if (!ws_name)
goto err_name;
ws->name = ws_name;
id = ida_alloc(&wakeup_ida, GFP_KERNEL);
if (id < 0)
goto err_id;
ws->id = id;
return ws;
err_id:
kfree_const(ws->name);
err_name:
kfree(ws);
err_ws:
return NULL;
}
EXPORT_SYMBOL_GPL(wakeup_source_create);
/*
* Record wakeup_source statistics being deleted into a dummy wakeup_source.
*/
static void wakeup_source_record(struct wakeup_source *ws)
{
unsigned long flags;
spin_lock_irqsave(&deleted_ws.lock, flags);
if (ws->event_count) {
deleted_ws.total_time =
ktime_add(deleted_ws.total_time, ws->total_time);
deleted_ws.prevent_sleep_time =
ktime_add(deleted_ws.prevent_sleep_time,
ws->prevent_sleep_time);
deleted_ws.max_time =
ktime_compare(deleted_ws.max_time, ws->max_time) > 0 ?
deleted_ws.max_time : ws->max_time;
deleted_ws.event_count += ws->event_count;
deleted_ws.active_count += ws->active_count;
deleted_ws.relax_count += ws->relax_count;
deleted_ws.expire_count += ws->expire_count;
deleted_ws.wakeup_count += ws->wakeup_count;
}
spin_unlock_irqrestore(&deleted_ws.lock, flags);
}
static void wakeup_source_free(struct wakeup_source *ws)
{
ida_free(&wakeup_ida, ws->id);
kfree_const(ws->name);
kfree(ws);
}
/**
* wakeup_source_destroy - Destroy a struct wakeup_source object.
* @ws: Wakeup source to destroy.
*
* Use only for wakeup source objects created with wakeup_source_create().
*/
void wakeup_source_destroy(struct wakeup_source *ws)
{
if (!ws)
return;
__pm_relax(ws);
wakeup_source_record(ws);
wakeup_source_free(ws);
}
EXPORT_SYMBOL_GPL(wakeup_source_destroy);
/**
* wakeup_source_add - Add given object to the list of wakeup sources.
* @ws: Wakeup source object to add to the list.
*/
void wakeup_source_add(struct wakeup_source *ws)
{
unsigned long flags;
if (WARN_ON(!ws))
return;
spin_lock_init(&ws->lock);
timer_setup(&ws->timer, pm_wakeup_timer_fn, 0);
ws->active = false;
raw_spin_lock_irqsave(&events_lock, flags);
list_add_rcu(&ws->entry, &wakeup_sources);
raw_spin_unlock_irqrestore(&events_lock, flags);
}
EXPORT_SYMBOL_GPL(wakeup_source_add);
/**
* wakeup_source_remove - Remove given object from the wakeup sources list.
* @ws: Wakeup source object to remove from the list.
*/
void wakeup_source_remove(struct wakeup_source *ws)
{
unsigned long flags;
if (WARN_ON(!ws))
return;
raw_spin_lock_irqsave(&events_lock, flags); list_del_rcu(&ws->entry);
raw_spin_unlock_irqrestore(&events_lock, flags);
synchronize_srcu(&wakeup_srcu);
del_timer_sync(&ws->timer);
/*
* Clear timer.function to make wakeup_source_not_registered() treat
* this wakeup source as not registered.
*/
ws->timer.function = NULL;
}
EXPORT_SYMBOL_GPL(wakeup_source_remove);
/**
* wakeup_source_register - Create wakeup source and add it to the list.
* @dev: Device this wakeup source is associated with (or NULL if virtual).
* @name: Name of the wakeup source to register.
*/
struct wakeup_source *wakeup_source_register(struct device *dev,
const char *name)
{
struct wakeup_source *ws;
int ret;
ws = wakeup_source_create(name);
if (ws) {
if (!dev || device_is_registered(dev)) {
ret = wakeup_source_sysfs_add(dev, ws);
if (ret) {
wakeup_source_free(ws);
return NULL;
}
}
wakeup_source_add(ws);
}
return ws;
}
EXPORT_SYMBOL_GPL(wakeup_source_register);
/**
* wakeup_source_unregister - Remove wakeup source from the list and remove it.
* @ws: Wakeup source object to unregister.
*/
void wakeup_source_unregister(struct wakeup_source *ws)
{
if (ws) { wakeup_source_remove(ws);
if (ws->dev)
wakeup_source_sysfs_remove(ws); wakeup_source_destroy(ws);
}
}
EXPORT_SYMBOL_GPL(wakeup_source_unregister);
/**
* wakeup_sources_read_lock - Lock wakeup source list for read.
*
* Returns an index of srcu lock for struct wakeup_srcu.
* This index must be passed to the matching wakeup_sources_read_unlock().
*/
int wakeup_sources_read_lock(void)
{
return srcu_read_lock(&wakeup_srcu);
}
EXPORT_SYMBOL_GPL(wakeup_sources_read_lock);
/**
* wakeup_sources_read_unlock - Unlock wakeup source list.
* @idx: return value from corresponding wakeup_sources_read_lock()
*/
void wakeup_sources_read_unlock(int idx)
{
srcu_read_unlock(&wakeup_srcu, idx);
}
EXPORT_SYMBOL_GPL(wakeup_sources_read_unlock);
/**
* wakeup_sources_walk_start - Begin a walk on wakeup source list
*
* Returns first object of the list of wakeup sources.
*
* Note that to be safe, wakeup sources list needs to be locked by calling
* wakeup_source_read_lock() for this.
*/
struct wakeup_source *wakeup_sources_walk_start(void)
{
struct list_head *ws_head = &wakeup_sources;
return list_entry_rcu(ws_head->next, struct wakeup_source, entry);
}
EXPORT_SYMBOL_GPL(wakeup_sources_walk_start);
/**
* wakeup_sources_walk_next - Get next wakeup source from the list
* @ws: Previous wakeup source object
*
* Note that to be safe, wakeup sources list needs to be locked by calling
* wakeup_source_read_lock() for this.
*/
struct wakeup_source *wakeup_sources_walk_next(struct wakeup_source *ws)
{
struct list_head *ws_head = &wakeup_sources;
return list_next_or_null_rcu(ws_head, &ws->entry,
struct wakeup_source, entry);
}
EXPORT_SYMBOL_GPL(wakeup_sources_walk_next);
/**
* device_wakeup_attach - Attach a wakeup source object to a device object.
* @dev: Device to handle.
* @ws: Wakeup source object to attach to @dev.
*
* This causes @dev to be treated as a wakeup device.
*/
static int device_wakeup_attach(struct device *dev, struct wakeup_source *ws)
{
spin_lock_irq(&dev->power.lock);
if (dev->power.wakeup) {
spin_unlock_irq(&dev->power.lock);
return -EEXIST;
}
dev->power.wakeup = ws;
if (dev->power.wakeirq)
device_wakeup_attach_irq(dev, dev->power.wakeirq);
spin_unlock_irq(&dev->power.lock);
return 0;
}
/**
* device_wakeup_enable - Enable given device to be a wakeup source.
* @dev: Device to handle.
*
* Create a wakeup source object, register it and attach it to @dev.
*/
int device_wakeup_enable(struct device *dev)
{
struct wakeup_source *ws;
int ret;
if (!dev || !dev->power.can_wakeup)
return -EINVAL;
if (pm_suspend_target_state != PM_SUSPEND_ON)
dev_dbg(dev, "Suspicious %s() during system transition!\n", __func__);
ws = wakeup_source_register(dev, dev_name(dev));
if (!ws)
return -ENOMEM;
ret = device_wakeup_attach(dev, ws);
if (ret)
wakeup_source_unregister(ws);
return ret;
}
EXPORT_SYMBOL_GPL(device_wakeup_enable);
/**
* device_wakeup_attach_irq - Attach a wakeirq to a wakeup source
* @dev: Device to handle
* @wakeirq: Device specific wakeirq entry
*
* Attach a device wakeirq to the wakeup source so the device
* wake IRQ can be configured automatically for suspend and
* resume.
*
* Call under the device's power.lock lock.
*/
void device_wakeup_attach_irq(struct device *dev,
struct wake_irq *wakeirq)
{
struct wakeup_source *ws;
ws = dev->power.wakeup;
if (!ws)
return;
if (ws->wakeirq)
dev_err(dev, "Leftover wakeup IRQ found, overriding\n");
ws->wakeirq = wakeirq;
}
/**
* device_wakeup_detach_irq - Detach a wakeirq from a wakeup source
* @dev: Device to handle
*
* Removes a device wakeirq from the wakeup source.
*
* Call under the device's power.lock lock.
*/
void device_wakeup_detach_irq(struct device *dev)
{
struct wakeup_source *ws;
ws = dev->power.wakeup;
if (ws)
ws->wakeirq = NULL;
}
/**
* device_wakeup_arm_wake_irqs -
*
* Iterates over the list of device wakeirqs to arm them.
*/
void device_wakeup_arm_wake_irqs(void)
{
struct wakeup_source *ws;
int srcuidx;
srcuidx = srcu_read_lock(&wakeup_srcu);
list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry)
dev_pm_arm_wake_irq(ws->wakeirq);
srcu_read_unlock(&wakeup_srcu, srcuidx);
}
/**
* device_wakeup_disarm_wake_irqs -
*
* Iterates over the list of device wakeirqs to disarm them.
*/
void device_wakeup_disarm_wake_irqs(void)
{
struct wakeup_source *ws;
int srcuidx;
srcuidx = srcu_read_lock(&wakeup_srcu);
list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry)
dev_pm_disarm_wake_irq(ws->wakeirq);
srcu_read_unlock(&wakeup_srcu, srcuidx);
}
/**
* device_wakeup_detach - Detach a device's wakeup source object from it.
* @dev: Device to detach the wakeup source object from.
*
* After it returns, @dev will not be treated as a wakeup device any more.
*/
static struct wakeup_source *device_wakeup_detach(struct device *dev)
{
struct wakeup_source *ws;
spin_lock_irq(&dev->power.lock);
ws = dev->power.wakeup;
dev->power.wakeup = NULL;
spin_unlock_irq(&dev->power.lock);
return ws;
}
/**
* device_wakeup_disable - Do not regard a device as a wakeup source any more.
* @dev: Device to handle.
*
* Detach the @dev's wakeup source object from it, unregister this wakeup source
* object and destroy it.
*/
int device_wakeup_disable(struct device *dev)
{
struct wakeup_source *ws;
if (!dev || !dev->power.can_wakeup)
return -EINVAL;
ws = device_wakeup_detach(dev); wakeup_source_unregister(ws); return 0;
}
EXPORT_SYMBOL_GPL(device_wakeup_disable);
/**
* device_set_wakeup_capable - Set/reset device wakeup capability flag.
* @dev: Device to handle.
* @capable: Whether or not @dev is capable of waking up the system from sleep.
*
* If @capable is set, set the @dev's power.can_wakeup flag and add its
* wakeup-related attributes to sysfs. Otherwise, unset the @dev's
* power.can_wakeup flag and remove its wakeup-related attributes from sysfs.
*
* This function may sleep and it can't be called from any context where
* sleeping is not allowed.
*/
void device_set_wakeup_capable(struct device *dev, bool capable)
{
if (!!dev->power.can_wakeup == !!capable)
return;
dev->power.can_wakeup = capable; if (device_is_registered(dev) && !list_empty(&dev->power.entry)) { if (capable) { int ret = wakeup_sysfs_add(dev);
if (ret)
dev_info(dev, "Wakeup sysfs attributes not added\n");
} else {
wakeup_sysfs_remove(dev);
}
}
}
EXPORT_SYMBOL_GPL(device_set_wakeup_capable);
/**
* device_set_wakeup_enable - Enable or disable a device to wake up the system.
* @dev: Device to handle.
* @enable: enable/disable flag
*/
int device_set_wakeup_enable(struct device *dev, bool enable)
{
return enable ? device_wakeup_enable(dev) : device_wakeup_disable(dev);
}
EXPORT_SYMBOL_GPL(device_set_wakeup_enable);
/**
* wakeup_source_not_registered - validate the given wakeup source.
* @ws: Wakeup source to be validated.
*/
static bool wakeup_source_not_registered(struct wakeup_source *ws)
{
/*
* Use timer struct to check if the given source is initialized
* by wakeup_source_add.
*/
return ws->timer.function != pm_wakeup_timer_fn;
}
/*
* The functions below use the observation that each wakeup event starts a
* period in which the system should not be suspended. The moment this period
* will end depends on how the wakeup event is going to be processed after being
* detected and all of the possible cases can be divided into two distinct
* groups.
*
* First, a wakeup event may be detected by the same functional unit that will
* carry out the entire processing of it and possibly will pass it to user space
* for further processing. In that case the functional unit that has detected
* the event may later "close" the "no suspend" period associated with it
* directly as soon as it has been dealt with. The pair of pm_stay_awake() and
* pm_relax(), balanced with each other, is supposed to be used in such
* situations.
*
* Second, a wakeup event may be detected by one functional unit and processed
* by another one. In that case the unit that has detected it cannot really
* "close" the "no suspend" period associated with it, unless it knows in
* advance what's going to happen to the event during processing. This
* knowledge, however, may not be available to it, so it can simply specify time
* to wait before the system can be suspended and pass it as the second
* argument of pm_wakeup_event().
*
* It is valid to call pm_relax() after pm_wakeup_event(), in which case the
* "no suspend" period will be ended either by the pm_relax(), or by the timer
* function executed when the timer expires, whichever comes first.
*/
/**
* wakeup_source_activate - Mark given wakeup source as active.
* @ws: Wakeup source to handle.
*
* Update the @ws' statistics and, if @ws has just been activated, notify the PM
* core of the event by incrementing the counter of the wakeup events being
* processed.
*/
static void wakeup_source_activate(struct wakeup_source *ws)
{
unsigned int cec;
if (WARN_ONCE(wakeup_source_not_registered(ws),
"unregistered wakeup source\n"))
return;
ws->active = true;
ws->active_count++;
ws->last_time = ktime_get();
if (ws->autosleep_enabled)
ws->start_prevent_time = ws->last_time;
/* Increment the counter of events in progress. */
cec = atomic_inc_return(&combined_event_count);
trace_wakeup_source_activate(ws->name, cec);
}
/**
* wakeup_source_report_event - Report wakeup event using the given source.
* @ws: Wakeup source to report the event for.
* @hard: If set, abort suspends in progress and wake up from suspend-to-idle.
*/
static void wakeup_source_report_event(struct wakeup_source *ws, bool hard)
{
ws->event_count++;
/* This is racy, but the counter is approximate anyway. */
if (events_check_enabled)
ws->wakeup_count++;
if (!ws->active)
wakeup_source_activate(ws);
if (hard)
pm_system_wakeup();
}
/**
* __pm_stay_awake - Notify the PM core of a wakeup event.
* @ws: Wakeup source object associated with the source of the event.
*
* It is safe to call this function from interrupt context.
*/
void __pm_stay_awake(struct wakeup_source *ws)
{
unsigned long flags;
if (!ws)
return;
spin_lock_irqsave(&ws->lock, flags);
wakeup_source_report_event(ws, false);
del_timer(&ws->timer);
ws->timer_expires = 0;
spin_unlock_irqrestore(&ws->lock, flags);
}
EXPORT_SYMBOL_GPL(__pm_stay_awake);
/**
* pm_stay_awake - Notify the PM core that a wakeup event is being processed.
* @dev: Device the wakeup event is related to.
*
* Notify the PM core of a wakeup event (signaled by @dev) by calling
* __pm_stay_awake for the @dev's wakeup source object.
*
* Call this function after detecting of a wakeup event if pm_relax() is going
* to be called directly after processing the event (and possibly passing it to
* user space for further processing).
*/
void pm_stay_awake(struct device *dev)
{
unsigned long flags;
if (!dev)
return;
spin_lock_irqsave(&dev->power.lock, flags);
__pm_stay_awake(dev->power.wakeup);
spin_unlock_irqrestore(&dev->power.lock, flags);
}
EXPORT_SYMBOL_GPL(pm_stay_awake);
#ifdef CONFIG_PM_AUTOSLEEP
static void update_prevent_sleep_time(struct wakeup_source *ws, ktime_t now)
{
ktime_t delta = ktime_sub(now, ws->start_prevent_time);
ws->prevent_sleep_time = ktime_add(ws->prevent_sleep_time, delta);
}
#else
static inline void update_prevent_sleep_time(struct wakeup_source *ws,
ktime_t now) {}
#endif
/**
* wakeup_source_deactivate - Mark given wakeup source as inactive.
* @ws: Wakeup source to handle.
*
* Update the @ws' statistics and notify the PM core that the wakeup source has
* become inactive by decrementing the counter of wakeup events being processed
* and incrementing the counter of registered wakeup events.
*/
static void wakeup_source_deactivate(struct wakeup_source *ws)
{
unsigned int cnt, inpr, cec;
ktime_t duration;
ktime_t now;
ws->relax_count++;
/*
* __pm_relax() may be called directly or from a timer function.
* If it is called directly right after the timer function has been
* started, but before the timer function calls __pm_relax(), it is
* possible that __pm_stay_awake() will be called in the meantime and
* will set ws->active. Then, ws->active may be cleared immediately
* by the __pm_relax() called from the timer function, but in such a
* case ws->relax_count will be different from ws->active_count.
*/
if (ws->relax_count != ws->active_count) {
ws->relax_count--;
return;
}
ws->active = false;
now = ktime_get();
duration = ktime_sub(now, ws->last_time);
ws->total_time = ktime_add(ws->total_time, duration);
if (ktime_to_ns(duration) > ktime_to_ns(ws->max_time))
ws->max_time = duration;
ws->last_time = now;
del_timer(&ws->timer);
ws->timer_expires = 0;
if (ws->autosleep_enabled)
update_prevent_sleep_time(ws, now);
/*
* Increment the counter of registered wakeup events and decrement the
* counter of wakeup events in progress simultaneously.
*/
cec = atomic_add_return(MAX_IN_PROGRESS, &combined_event_count);
trace_wakeup_source_deactivate(ws->name, cec);
split_counters(&cnt, &inpr);
if (!inpr && waitqueue_active(&wakeup_count_wait_queue))
wake_up(&wakeup_count_wait_queue);
}
/**
* __pm_relax - Notify the PM core that processing of a wakeup event has ended.
* @ws: Wakeup source object associated with the source of the event.
*
* Call this function for wakeup events whose processing started with calling
* __pm_stay_awake().
*
* It is safe to call it from interrupt context.
*/
void __pm_relax(struct wakeup_source *ws)
{
unsigned long flags;
if (!ws)
return;
spin_lock_irqsave(&ws->lock, flags);
if (ws->active)
wakeup_source_deactivate(ws); spin_unlock_irqrestore(&ws->lock, flags);
}
EXPORT_SYMBOL_GPL(__pm_relax);
/**
* pm_relax - Notify the PM core that processing of a wakeup event has ended.
* @dev: Device that signaled the event.
*
* Execute __pm_relax() for the @dev's wakeup source object.
*/
void pm_relax(struct device *dev)
{
unsigned long flags;
if (!dev)
return;
spin_lock_irqsave(&dev->power.lock, flags);
__pm_relax(dev->power.wakeup);
spin_unlock_irqrestore(&dev->power.lock, flags);
}
EXPORT_SYMBOL_GPL(pm_relax);
/**
* pm_wakeup_timer_fn - Delayed finalization of a wakeup event.
* @t: timer list
*
* Call wakeup_source_deactivate() for the wakeup source whose address is stored
* in @data if it is currently active and its timer has not been canceled and
* the expiration time of the timer is not in future.
*/
static void pm_wakeup_timer_fn(struct timer_list *t)
{
struct wakeup_source *ws = from_timer(ws, t, timer);
unsigned long flags;
spin_lock_irqsave(&ws->lock, flags);
if (ws->active && ws->timer_expires
&& time_after_eq(jiffies, ws->timer_expires)) {
wakeup_source_deactivate(ws);
ws->expire_count++;
}
spin_unlock_irqrestore(&ws->lock, flags);
}
/**
* pm_wakeup_ws_event - Notify the PM core of a wakeup event.
* @ws: Wakeup source object associated with the event source.
* @msec: Anticipated event processing time (in milliseconds).
* @hard: If set, abort suspends in progress and wake up from suspend-to-idle.
*
* Notify the PM core of a wakeup event whose source is @ws that will take
* approximately @msec milliseconds to be processed by the kernel. If @ws is
* not active, activate it. If @msec is nonzero, set up the @ws' timer to
* execute pm_wakeup_timer_fn() in future.
*
* It is safe to call this function from interrupt context.
*/
void pm_wakeup_ws_event(struct wakeup_source *ws, unsigned int msec, bool hard)
{
unsigned long flags;
unsigned long expires;
if (!ws)
return;
spin_lock_irqsave(&ws->lock, flags);
wakeup_source_report_event(ws, hard);
if (!msec) {
wakeup_source_deactivate(ws);
goto unlock;
}
expires = jiffies + msecs_to_jiffies(msec);
if (!expires)
expires = 1;
if (!ws->timer_expires || time_after(expires, ws->timer_expires)) {
mod_timer(&ws->timer, expires);
ws->timer_expires = expires;
}
unlock:
spin_unlock_irqrestore(&ws->lock, flags);
}
EXPORT_SYMBOL_GPL(pm_wakeup_ws_event);
/**
* pm_wakeup_dev_event - Notify the PM core of a wakeup event.
* @dev: Device the wakeup event is related to.
* @msec: Anticipated event processing time (in milliseconds).
* @hard: If set, abort suspends in progress and wake up from suspend-to-idle.
*
* Call pm_wakeup_ws_event() for the @dev's wakeup source object.
*/
void pm_wakeup_dev_event(struct device *dev, unsigned int msec, bool hard)
{
unsigned long flags;
if (!dev)
return;
spin_lock_irqsave(&dev->power.lock, flags); pm_wakeup_ws_event(dev->power.wakeup, msec, hard); spin_unlock_irqrestore(&dev->power.lock, flags);
}
EXPORT_SYMBOL_GPL(pm_wakeup_dev_event);
void pm_print_active_wakeup_sources(void)
{
struct wakeup_source *ws;
int srcuidx, active = 0;
struct wakeup_source *last_activity_ws = NULL;
srcuidx = srcu_read_lock(&wakeup_srcu);
list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) {
if (ws->active) {
pm_pr_dbg("active wakeup source: %s\n", ws->name);
active = 1;
} else if (!active &&
(!last_activity_ws ||
ktime_to_ns(ws->last_time) >
ktime_to_ns(last_activity_ws->last_time))) {
last_activity_ws = ws;
}
}
if (!active && last_activity_ws)
pm_pr_dbg("last active wakeup source: %s\n",
last_activity_ws->name);
srcu_read_unlock(&wakeup_srcu, srcuidx);
}
EXPORT_SYMBOL_GPL(pm_print_active_wakeup_sources);
/**
* pm_wakeup_pending - Check if power transition in progress should be aborted.
*
* Compare the current number of registered wakeup events with its preserved
* value from the past and return true if new wakeup events have been registered
* since the old value was stored. Also return true if the current number of
* wakeup events being processed is different from zero.
*/
bool pm_wakeup_pending(void)
{
unsigned long flags;
bool ret = false;
raw_spin_lock_irqsave(&events_lock, flags);
if (events_check_enabled) {
unsigned int cnt, inpr;
split_counters(&cnt, &inpr);
ret = (cnt != saved_count || inpr > 0);
events_check_enabled = !ret;
}
raw_spin_unlock_irqrestore(&events_lock, flags);
if (ret) {
pm_pr_dbg("Wakeup pending, aborting suspend\n");
pm_print_active_wakeup_sources();
}
return ret || atomic_read(&pm_abort_suspend) > 0;
}
EXPORT_SYMBOL_GPL(pm_wakeup_pending);
void pm_system_wakeup(void)
{
atomic_inc(&pm_abort_suspend);
s2idle_wake();
}
EXPORT_SYMBOL_GPL(pm_system_wakeup);
void pm_system_cancel_wakeup(void)
{
atomic_dec_if_positive(&pm_abort_suspend);
}
void pm_wakeup_clear(unsigned int irq_number)
{
raw_spin_lock_irq(&wakeup_irq_lock);
if (irq_number && wakeup_irq[0] == irq_number)
wakeup_irq[0] = wakeup_irq[1];
else
wakeup_irq[0] = 0;
wakeup_irq[1] = 0;
raw_spin_unlock_irq(&wakeup_irq_lock);
if (!irq_number)
atomic_set(&pm_abort_suspend, 0);
}
void pm_system_irq_wakeup(unsigned int irq_number)
{
unsigned long flags;
raw_spin_lock_irqsave(&wakeup_irq_lock, flags);
if (wakeup_irq[0] == 0)
wakeup_irq[0] = irq_number;
else if (wakeup_irq[1] == 0)
wakeup_irq[1] = irq_number;
else
irq_number = 0;
pm_pr_dbg("Triggering wakeup from IRQ %d\n", irq_number);
raw_spin_unlock_irqrestore(&wakeup_irq_lock, flags);
if (irq_number)
pm_system_wakeup();
}
unsigned int pm_wakeup_irq(void)
{
return wakeup_irq[0];
}
/**
* pm_get_wakeup_count - Read the number of registered wakeup events.
* @count: Address to store the value at.
* @block: Whether or not to block.
*
* Store the number of registered wakeup events at the address in @count. If
* @block is set, block until the current number of wakeup events being
* processed is zero.
*
* Return 'false' if the current number of wakeup events being processed is
* nonzero. Otherwise return 'true'.
*/
bool pm_get_wakeup_count(unsigned int *count, bool block)
{
unsigned int cnt, inpr;
if (block) {
DEFINE_WAIT(wait);
for (;;) {
prepare_to_wait(&wakeup_count_wait_queue, &wait,
TASK_INTERRUPTIBLE);
split_counters(&cnt, &inpr);
if (inpr == 0 || signal_pending(current))
break;
pm_print_active_wakeup_sources();
schedule();
}
finish_wait(&wakeup_count_wait_queue, &wait);
}
split_counters(&cnt, &inpr);
*count = cnt;
return !inpr;
}
/**
* pm_save_wakeup_count - Save the current number of registered wakeup events.
* @count: Value to compare with the current number of registered wakeup events.
*
* If @count is equal to the current number of registered wakeup events and the
* current number of wakeup events being processed is zero, store @count as the
* old number of registered wakeup events for pm_check_wakeup_events(), enable
* wakeup events detection and return 'true'. Otherwise disable wakeup events
* detection and return 'false'.
*/
bool pm_save_wakeup_count(unsigned int count)
{
unsigned int cnt, inpr;
unsigned long flags;
events_check_enabled = false;
raw_spin_lock_irqsave(&events_lock, flags);
split_counters(&cnt, &inpr);
if (cnt == count && inpr == 0) {
saved_count = count;
events_check_enabled = true;
}
raw_spin_unlock_irqrestore(&events_lock, flags);
return events_check_enabled;
}
#ifdef CONFIG_PM_AUTOSLEEP
/**
* pm_wakep_autosleep_enabled - Modify autosleep_enabled for all wakeup sources.
* @set: Whether to set or to clear the autosleep_enabled flags.
*/
void pm_wakep_autosleep_enabled(bool set)
{
struct wakeup_source *ws;
ktime_t now = ktime_get();
int srcuidx;
srcuidx = srcu_read_lock(&wakeup_srcu);
list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) {
spin_lock_irq(&ws->lock);
if (ws->autosleep_enabled != set) {
ws->autosleep_enabled = set;
if (ws->active) {
if (set)
ws->start_prevent_time = now;
else
update_prevent_sleep_time(ws, now);
}
}
spin_unlock_irq(&ws->lock);
}
srcu_read_unlock(&wakeup_srcu, srcuidx);
}
#endif /* CONFIG_PM_AUTOSLEEP */
/**
* print_wakeup_source_stats - Print wakeup source statistics information.
* @m: seq_file to print the statistics into.
* @ws: Wakeup source object to print the statistics for.
*/
static int print_wakeup_source_stats(struct seq_file *m,
struct wakeup_source *ws)
{
unsigned long flags;
ktime_t total_time;
ktime_t max_time;
unsigned long active_count;
ktime_t active_time;
ktime_t prevent_sleep_time;
spin_lock_irqsave(&ws->lock, flags);
total_time = ws->total_time;
max_time = ws->max_time;
prevent_sleep_time = ws->prevent_sleep_time;
active_count = ws->active_count;
if (ws->active) {
ktime_t now = ktime_get();
active_time = ktime_sub(now, ws->last_time);
total_time = ktime_add(total_time, active_time);
if (active_time > max_time)
max_time = active_time;
if (ws->autosleep_enabled)
prevent_sleep_time = ktime_add(prevent_sleep_time,
ktime_sub(now, ws->start_prevent_time));
} else {
active_time = 0;
}
seq_printf(m, "%-12s\t%lu\t\t%lu\t\t%lu\t\t%lu\t\t%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n",
ws->name, active_count, ws->event_count,
ws->wakeup_count, ws->expire_count,
ktime_to_ms(active_time), ktime_to_ms(total_time),
ktime_to_ms(max_time), ktime_to_ms(ws->last_time),
ktime_to_ms(prevent_sleep_time));
spin_unlock_irqrestore(&ws->lock, flags);
return 0;
}
static void *wakeup_sources_stats_seq_start(struct seq_file *m,
loff_t *pos)
{
struct wakeup_source *ws;
loff_t n = *pos;
int *srcuidx = m->private;
if (n == 0) {
seq_puts(m, "name\t\tactive_count\tevent_count\twakeup_count\t"
"expire_count\tactive_since\ttotal_time\tmax_time\t"
"last_change\tprevent_suspend_time\n");
}
*srcuidx = srcu_read_lock(&wakeup_srcu);
list_for_each_entry_rcu_locked(ws, &wakeup_sources, entry) {
if (n-- <= 0)
return ws;
}
return NULL;
}
static void *wakeup_sources_stats_seq_next(struct seq_file *m,
void *v, loff_t *pos)
{
struct wakeup_source *ws = v;
struct wakeup_source *next_ws = NULL;
++(*pos);
list_for_each_entry_continue_rcu(ws, &wakeup_sources, entry) {
next_ws = ws;
break;
}
if (!next_ws)
print_wakeup_source_stats(m, &deleted_ws);
return next_ws;
}
static void wakeup_sources_stats_seq_stop(struct seq_file *m, void *v)
{
int *srcuidx = m->private;
srcu_read_unlock(&wakeup_srcu, *srcuidx);
}
/**
* wakeup_sources_stats_seq_show - Print wakeup sources statistics information.
* @m: seq_file to print the statistics into.
* @v: wakeup_source of each iteration
*/
static int wakeup_sources_stats_seq_show(struct seq_file *m, void *v)
{
struct wakeup_source *ws = v;
print_wakeup_source_stats(m, ws);
return 0;
}
static const struct seq_operations wakeup_sources_stats_seq_ops = {
.start = wakeup_sources_stats_seq_start,
.next = wakeup_sources_stats_seq_next,
.stop = wakeup_sources_stats_seq_stop,
.show = wakeup_sources_stats_seq_show,
};
static int wakeup_sources_stats_open(struct inode *inode, struct file *file)
{
return seq_open_private(file, &wakeup_sources_stats_seq_ops, sizeof(int));
}
static const struct file_operations wakeup_sources_stats_fops = {
.owner = THIS_MODULE,
.open = wakeup_sources_stats_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
static int __init wakeup_sources_debugfs_init(void)
{
debugfs_create_file("wakeup_sources", 0444, NULL, NULL,
&wakeup_sources_stats_fops);
return 0;
}
postcore_initcall(wakeup_sources_debugfs_init);
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
#include <linux/bug.h>
#include <linux/atomic.h>
#include <linux/errseq.h>
#include <linux/log2.h>
/*
* An errseq_t is a way of recording errors in one place, and allowing any
* number of "subscribers" to tell whether it has changed since a previous
* point where it was sampled.
*
* It's implemented as an unsigned 32-bit value. The low order bits are
* designated to hold an error code (between 0 and -MAX_ERRNO). The upper bits
* are used as a counter. This is done with atomics instead of locking so that
* these functions can be called from any context.
*
* The general idea is for consumers to sample an errseq_t value. That value
* can later be used to tell whether any new errors have occurred since that
* sampling was done.
*
* Note that there is a risk of collisions if new errors are being recorded
* frequently, since we have so few bits to use as a counter.
*
* To mitigate this, one bit is used as a flag to tell whether the value has
* been sampled since a new value was recorded. That allows us to avoid bumping
* the counter if no one has sampled it since the last time an error was
* recorded.
*
* A new errseq_t should always be zeroed out. A errseq_t value of all zeroes
* is the special (but common) case where there has never been an error. An all
* zero value thus serves as the "epoch" if one wishes to know whether there
* has ever been an error set since it was first initialized.
*/
/* The low bits are designated for error code (max of MAX_ERRNO) */
#define ERRSEQ_SHIFT ilog2(MAX_ERRNO + 1)
/* This bit is used as a flag to indicate whether the value has been seen */
#define ERRSEQ_SEEN (1 << ERRSEQ_SHIFT)
/* The lowest bit of the counter */
#define ERRSEQ_CTR_INC (1 << (ERRSEQ_SHIFT + 1))
/**
* errseq_set - set a errseq_t for later reporting
* @eseq: errseq_t field that should be set
* @err: error to set (must be between -1 and -MAX_ERRNO)
*
* This function sets the error in @eseq, and increments the sequence counter
* if the last sequence was sampled at some point in the past.
*
* Any error set will always overwrite an existing error.
*
* Return: The previous value, primarily for debugging purposes. The
* return value should not be used as a previously sampled value in later
* calls as it will not have the SEEN flag set.
*/
errseq_t errseq_set(errseq_t *eseq, int err)
{
errseq_t cur, old;
/* MAX_ERRNO must be able to serve as a mask */
BUILD_BUG_ON_NOT_POWER_OF_2(MAX_ERRNO + 1);
/*
* Ensure the error code actually fits where we want it to go. If it
* doesn't then just throw a warning and don't record anything. We
* also don't accept zero here as that would effectively clear a
* previous error.
*/
old = READ_ONCE(*eseq);
if (WARN(unlikely(err == 0 || (unsigned int)-err > MAX_ERRNO),
"err = %d\n", err))
return old;
for (;;) {
errseq_t new;
/* Clear out error bits and set new error */
new = (old & ~(MAX_ERRNO|ERRSEQ_SEEN)) | -err;
/* Only increment if someone has looked at it */
if (old & ERRSEQ_SEEN)
new += ERRSEQ_CTR_INC;
/* If there would be no change, then call it done */
if (new == old) {
cur = new;
break;
}
/* Try to swap the new value into place */
cur = cmpxchg(eseq, old, new);
/*
* Call it success if we did the swap or someone else beat us
* to it for the same value.
*/
if (likely(cur == old || cur == new))
break;
/* Raced with an update, try again */
old = cur;
}
return cur;
}
EXPORT_SYMBOL(errseq_set);
/**
* errseq_sample() - Grab current errseq_t value.
* @eseq: Pointer to errseq_t to be sampled.
*
* This function allows callers to initialise their errseq_t variable.
* If the error has been "seen", new callers will not see an old error.
* If there is an unseen error in @eseq, the caller of this function will
* see it the next time it checks for an error.
*
* Context: Any context.
* Return: The current errseq value.
*/
errseq_t errseq_sample(errseq_t *eseq)
{
errseq_t old = READ_ONCE(*eseq);
/* If nobody has seen this error yet, then we can be the first. */
if (!(old & ERRSEQ_SEEN))
old = 0;
return old;
}
EXPORT_SYMBOL(errseq_sample);
/**
* errseq_check() - Has an error occurred since a particular sample point?
* @eseq: Pointer to errseq_t value to be checked.
* @since: Previously-sampled errseq_t from which to check.
*
* Grab the value that eseq points to, and see if it has changed @since
* the given value was sampled. The @since value is not advanced, so there
* is no need to mark the value as seen.
*
* Return: The latest error set in the errseq_t or 0 if it hasn't changed.
*/
int errseq_check(errseq_t *eseq, errseq_t since)
{
errseq_t cur = READ_ONCE(*eseq);
if (likely(cur == since))
return 0;
return -(cur & MAX_ERRNO);
}
EXPORT_SYMBOL(errseq_check);
/**
* errseq_check_and_advance() - Check an errseq_t and advance to current value.
* @eseq: Pointer to value being checked and reported.
* @since: Pointer to previously-sampled errseq_t to check against and advance.
*
* Grab the eseq value, and see whether it matches the value that @since
* points to. If it does, then just return 0.
*
* If it doesn't, then the value has changed. Set the "seen" flag, and try to
* swap it into place as the new eseq value. Then, set that value as the new
* "since" value, and return whatever the error portion is set to.
*
* Note that no locking is provided here for concurrent updates to the "since"
* value. The caller must provide that if necessary. Because of this, callers
* may want to do a lockless errseq_check before taking the lock and calling
* this.
*
* Return: Negative errno if one has been stored, or 0 if no new error has
* occurred.
*/
int errseq_check_and_advance(errseq_t *eseq, errseq_t *since)
{
int err = 0;
errseq_t old, new;
/*
* Most callers will want to use the inline wrapper to check this,
* so that the common case of no error is handled without needing
* to take the lock that protects the "since" value.
*/
old = READ_ONCE(*eseq);
if (old != *since) {
/*
* Set the flag and try to swap it into place if it has
* changed.
*
* We don't care about the outcome of the swap here. If the
* swap doesn't occur, then it has either been updated by a
* writer who is altering the value in some way (updating
* counter or resetting the error), or another reader who is
* just setting the "seen" flag. Either outcome is OK, and we
* can advance "since" and return an error based on what we
* have.
*/
new = old | ERRSEQ_SEEN;
if (new != old)
cmpxchg(eseq, old, new);
*since = new;
err = -(new & MAX_ERRNO);
}
return err;
}
EXPORT_SYMBOL(errseq_check_and_advance);
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Media entity
*
* Copyright (C) 2010 Nokia Corporation
*
* Contacts: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
* Sakari Ailus <sakari.ailus@iki.fi>
*/
#ifndef _MEDIA_ENTITY_H
#define _MEDIA_ENTITY_H
#include <linux/bitmap.h>
#include <linux/bug.h>
#include <linux/container_of.h>
#include <linux/fwnode.h>
#include <linux/list.h>
#include <linux/media.h>
#include <linux/minmax.h>
#include <linux/types.h>
/* Enums used internally at the media controller to represent graphs */
/**
* enum media_gobj_type - type of a graph object
*
* @MEDIA_GRAPH_ENTITY: Identify a media entity
* @MEDIA_GRAPH_PAD: Identify a media pad
* @MEDIA_GRAPH_LINK: Identify a media link
* @MEDIA_GRAPH_INTF_DEVNODE: Identify a media Kernel API interface via
* a device node
*/
enum media_gobj_type {
MEDIA_GRAPH_ENTITY,
MEDIA_GRAPH_PAD,
MEDIA_GRAPH_LINK,
MEDIA_GRAPH_INTF_DEVNODE,
};
#define MEDIA_BITS_PER_TYPE 8
#define MEDIA_BITS_PER_ID (32 - MEDIA_BITS_PER_TYPE)
#define MEDIA_ID_MASK GENMASK_ULL(MEDIA_BITS_PER_ID - 1, 0)
/* Structs to represent the objects that belong to a media graph */
/**
* struct media_gobj - Define a graph object.
*
* @mdev: Pointer to the struct &media_device that owns the object
* @id: Non-zero object ID identifier. The ID should be unique
* inside a media_device, as it is composed by
* %MEDIA_BITS_PER_TYPE to store the type plus
* %MEDIA_BITS_PER_ID to store the ID
* @list: List entry stored in one of the per-type mdev object lists
*
* All objects on the media graph should have this struct embedded
*/
struct media_gobj {
struct media_device *mdev;
u32 id;
struct list_head list;
};
#define MEDIA_ENTITY_ENUM_MAX_DEPTH 16
/**
* struct media_entity_enum - An enumeration of media entities.
*
* @bmap: Bit map in which each bit represents one entity at struct
* media_entity->internal_idx.
* @idx_max: Number of bits in bmap
*/
struct media_entity_enum {
unsigned long *bmap;
int idx_max;
};
/**
* struct media_graph - Media graph traversal state
*
* @stack: Graph traversal stack; the stack contains information
* on the path the media entities to be walked and the
* links through which they were reached.
* @stack.entity: pointer to &struct media_entity at the graph.
* @stack.link: pointer to &struct list_head.
* @ent_enum: Visited entities
* @top: The top of the stack
*/
struct media_graph {
struct {
struct media_entity *entity;
struct list_head *link;
} stack[MEDIA_ENTITY_ENUM_MAX_DEPTH];
struct media_entity_enum ent_enum;
int top;
};
/**
* struct media_pipeline - Media pipeline related information
*
* @allocated: Media pipeline allocated and freed by the framework
* @mdev: The media device the pipeline is part of
* @pads: List of media_pipeline_pad
* @start_count: Media pipeline start - stop count
*/
struct media_pipeline {
bool allocated;
struct media_device *mdev;
struct list_head pads;
int start_count;
};
/**
* struct media_pipeline_pad - A pad part of a media pipeline
*
* @list: Entry in the media_pad pads list
* @pipe: The media_pipeline that the pad is part of
* @pad: The media pad
*
* This structure associate a pad with a media pipeline. Instances of
* media_pipeline_pad are created by media_pipeline_start() when it builds the
* pipeline, and stored in the &media_pad.pads list. media_pipeline_stop()
* removes the entries from the list and deletes them.
*/
struct media_pipeline_pad {
struct list_head list;
struct media_pipeline *pipe;
struct media_pad *pad;
};
/**
* struct media_pipeline_pad_iter - Iterator for media_pipeline_for_each_pad
*
* @cursor: The current element
*/
struct media_pipeline_pad_iter {
struct list_head *cursor;
};
/**
* struct media_pipeline_entity_iter - Iterator for media_pipeline_for_each_entity
*
* @ent_enum: The entity enumeration tracker
* @cursor: The current element
*/
struct media_pipeline_entity_iter {
struct media_entity_enum ent_enum;
struct list_head *cursor;
};
/**
* struct media_link - A link object part of a media graph.
*
* @graph_obj: Embedded structure containing the media object common data
* @list: Linked list associated with an entity or an interface that
* owns the link.
* @gobj0: Part of a union. Used to get the pointer for the first
* graph_object of the link.
* @source: Part of a union. Used only if the first object (gobj0) is
* a pad. In that case, it represents the source pad.
* @intf: Part of a union. Used only if the first object (gobj0) is
* an interface.
* @gobj1: Part of a union. Used to get the pointer for the second
* graph_object of the link.
* @sink: Part of a union. Used only if the second object (gobj1) is
* a pad. In that case, it represents the sink pad.
* @entity: Part of a union. Used only if the second object (gobj1) is
* an entity.
* @reverse: Pointer to the link for the reverse direction of a pad to pad
* link.
* @flags: Link flags, as defined in uapi/media.h (MEDIA_LNK_FL_*)
* @is_backlink: Indicate if the link is a backlink.
*/
struct media_link {
struct media_gobj graph_obj;
struct list_head list;
union {
struct media_gobj *gobj0;
struct media_pad *source;
struct media_interface *intf;
};
union {
struct media_gobj *gobj1;
struct media_pad *sink;
struct media_entity *entity;
};
struct media_link *reverse;
unsigned long flags;
bool is_backlink;
};
/**
* enum media_pad_signal_type - type of the signal inside a media pad
*
* @PAD_SIGNAL_DEFAULT:
* Default signal. Use this when all inputs or all outputs are
* uniquely identified by the pad number.
* @PAD_SIGNAL_ANALOG:
* The pad contains an analog signal. It can be Radio Frequency,
* Intermediate Frequency, a baseband signal or sub-carriers.
* Tuner inputs, IF-PLL demodulators, composite and s-video signals
* should use it.
* @PAD_SIGNAL_DV:
* Contains a digital video signal, with can be a bitstream of samples
* taken from an analog TV video source. On such case, it usually
* contains the VBI data on it.
* @PAD_SIGNAL_AUDIO:
* Contains an Intermediate Frequency analog signal from an audio
* sub-carrier or an audio bitstream. IF signals are provided by tuners
* and consumed by audio AM/FM decoders. Bitstream audio is provided by
* an audio decoder.
*/
enum media_pad_signal_type {
PAD_SIGNAL_DEFAULT = 0,
PAD_SIGNAL_ANALOG,
PAD_SIGNAL_DV,
PAD_SIGNAL_AUDIO,
};
/**
* struct media_pad - A media pad graph object.
*
* @graph_obj: Embedded structure containing the media object common data
* @entity: Entity this pad belongs to
* @index: Pad index in the entity pads array, numbered from 0 to n
* @num_links: Number of links connected to this pad
* @sig_type: Type of the signal inside a media pad
* @flags: Pad flags, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* (seek for ``MEDIA_PAD_FL_*``)
* @pipe: Pipeline this pad belongs to. Use media_entity_pipeline() to
* access this field.
*/
struct media_pad {
struct media_gobj graph_obj; /* must be first field in struct */
struct media_entity *entity;
u16 index;
u16 num_links;
enum media_pad_signal_type sig_type;
unsigned long flags;
/*
* The fields below are private, and should only be accessed via
* appropriate functions.
*/
struct media_pipeline *pipe;
};
/**
* struct media_entity_operations - Media entity operations
* @get_fwnode_pad: Return the pad number based on a fwnode endpoint or
* a negative value on error. This operation can be used
* to map a fwnode to a media pad number. Optional.
* @link_setup: Notify the entity of link changes. The operation can
* return an error, in which case link setup will be
* cancelled. Optional.
* @link_validate: Return whether a link is valid from the entity point of
* view. The media_pipeline_start() function
* validates all links by calling this operation. Optional.
* @has_pad_interdep: Return whether two pads of the entity are
* interdependent. If two pads are interdependent they are
* part of the same pipeline and enabling one of the pads
* means that the other pad will become "locked" and
* doesn't allow configuration changes. pad0 and pad1 are
* guaranteed to not both be sinks or sources. Never call
* the .has_pad_interdep() operation directly, always use
* media_entity_has_pad_interdep().
* Optional: If the operation isn't implemented all pads
* will be considered as interdependent.
*
* .. note::
*
* Those these callbacks are called with struct &media_device.graph_mutex
* mutex held.
*/
struct media_entity_operations {
int (*get_fwnode_pad)(struct media_entity *entity,
struct fwnode_endpoint *endpoint);
int (*link_setup)(struct media_entity *entity,
const struct media_pad *local,
const struct media_pad *remote, u32 flags);
int (*link_validate)(struct media_link *link);
bool (*has_pad_interdep)(struct media_entity *entity, unsigned int pad0,
unsigned int pad1);
};
/**
* enum media_entity_type - Media entity type
*
* @MEDIA_ENTITY_TYPE_BASE:
* The entity isn't embedded in another subsystem structure.
* @MEDIA_ENTITY_TYPE_VIDEO_DEVICE:
* The entity is embedded in a struct video_device instance.
* @MEDIA_ENTITY_TYPE_V4L2_SUBDEV:
* The entity is embedded in a struct v4l2_subdev instance.
*
* Media entity objects are often not instantiated directly, but the media
* entity structure is inherited by (through embedding) other subsystem-specific
* structures. The media entity type identifies the type of the subclass
* structure that implements a media entity instance.
*
* This allows runtime type identification of media entities and safe casting to
* the correct object type. For instance, a media entity structure instance
* embedded in a v4l2_subdev structure instance will have the type
* %MEDIA_ENTITY_TYPE_V4L2_SUBDEV and can safely be cast to a &v4l2_subdev
* structure using the container_of() macro.
*/
enum media_entity_type {
MEDIA_ENTITY_TYPE_BASE,
MEDIA_ENTITY_TYPE_VIDEO_DEVICE,
MEDIA_ENTITY_TYPE_V4L2_SUBDEV,
};
/**
* struct media_entity - A media entity graph object.
*
* @graph_obj: Embedded structure containing the media object common data.
* @name: Entity name.
* @obj_type: Type of the object that implements the media_entity.
* @function: Entity main function, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* (seek for ``MEDIA_ENT_F_*``)
* @flags: Entity flags, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* (seek for ``MEDIA_ENT_FL_*``)
* @num_pads: Number of sink and source pads.
* @num_links: Total number of links, forward and back, enabled and disabled.
* @num_backlinks: Number of backlinks
* @internal_idx: An unique internal entity specific number. The numbers are
* re-used if entities are unregistered or registered again.
* @pads: Pads array with the size defined by @num_pads.
* @links: List of data links.
* @ops: Entity operations.
* @use_count: Use count for the entity.
* @info: Union with devnode information. Kept just for backward
* compatibility.
* @info.dev: Contains device major and minor info.
* @info.dev.major: device node major, if the device is a devnode.
* @info.dev.minor: device node minor, if the device is a devnode.
*
* .. note::
*
* The @use_count reference count must never be negative, but is a signed
* integer on purpose: a simple ``WARN_ON(<0)`` check can be used to detect
* reference count bugs that would make it negative.
*/
struct media_entity {
struct media_gobj graph_obj; /* must be first field in struct */
const char *name;
enum media_entity_type obj_type;
u32 function;
unsigned long flags;
u16 num_pads;
u16 num_links;
u16 num_backlinks;
int internal_idx;
struct media_pad *pads;
struct list_head links;
const struct media_entity_operations *ops;
int use_count;
union {
struct {
u32 major;
u32 minor;
} dev;
} info;
};
/**
* media_entity_for_each_pad - Iterate on all pads in an entity
* @entity: The entity the pads belong to
* @iter: The iterator pad
*
* Iterate on all pads in a media entity.
*/
#define media_entity_for_each_pad(entity, iter) \
for (iter = (entity)->pads; \
iter < &(entity)->pads[(entity)->num_pads]; \
++iter)
/**
* struct media_interface - A media interface graph object.
*
* @graph_obj: embedded graph object
* @links: List of links pointing to graph entities
* @type: Type of the interface as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* (seek for ``MEDIA_INTF_T_*``)
* @flags: Interface flags as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* (seek for ``MEDIA_INTF_FL_*``)
*
* .. note::
*
* Currently, no flags for &media_interface is defined.
*/
struct media_interface {
struct media_gobj graph_obj;
struct list_head links;
u32 type;
u32 flags;
};
/**
* struct media_intf_devnode - A media interface via a device node.
*
* @intf: embedded interface object
* @major: Major number of a device node
* @minor: Minor number of a device node
*/
struct media_intf_devnode {
struct media_interface intf;
/* Should match the fields at media_v2_intf_devnode */
u32 major;
u32 minor;
};
/**
* media_entity_id() - return the media entity graph object id
*
* @entity: pointer to &media_entity
*/
static inline u32 media_entity_id(struct media_entity *entity)
{
return entity->graph_obj.id;
}
/**
* media_type() - return the media object type
*
* @gobj: Pointer to the struct &media_gobj graph object
*/
static inline enum media_gobj_type media_type(struct media_gobj *gobj)
{
return gobj->id >> MEDIA_BITS_PER_ID;
}
/**
* media_id() - return the media object ID
*
* @gobj: Pointer to the struct &media_gobj graph object
*/
static inline u32 media_id(struct media_gobj *gobj)
{
return gobj->id & MEDIA_ID_MASK;
}
/**
* media_gobj_gen_id() - encapsulates type and ID on at the object ID
*
* @type: object type as define at enum &media_gobj_type.
* @local_id: next ID, from struct &media_device.id.
*/
static inline u32 media_gobj_gen_id(enum media_gobj_type type, u64 local_id)
{
u32 id;
id = type << MEDIA_BITS_PER_ID;
id |= local_id & MEDIA_ID_MASK;
return id;
}
/**
* is_media_entity_v4l2_video_device() - Check if the entity is a video_device
* @entity: pointer to entity
*
* Return: %true if the entity is an instance of a video_device object and can
* safely be cast to a struct video_device using the container_of() macro, or
* %false otherwise.
*/
static inline bool is_media_entity_v4l2_video_device(struct media_entity *entity)
{
return entity && entity->obj_type == MEDIA_ENTITY_TYPE_VIDEO_DEVICE;
}
/**
* is_media_entity_v4l2_subdev() - Check if the entity is a v4l2_subdev
* @entity: pointer to entity
*
* Return: %true if the entity is an instance of a &v4l2_subdev object and can
* safely be cast to a struct &v4l2_subdev using the container_of() macro, or
* %false otherwise.
*/
static inline bool is_media_entity_v4l2_subdev(struct media_entity *entity)
{
return entity && entity->obj_type == MEDIA_ENTITY_TYPE_V4L2_SUBDEV;
}
/**
* media_entity_enum_init - Initialise an entity enumeration
*
* @ent_enum: Entity enumeration to be initialised
* @mdev: The related media device
*
* Return: zero on success or a negative error code.
*/
__must_check int media_entity_enum_init(struct media_entity_enum *ent_enum,
struct media_device *mdev);
/**
* media_entity_enum_cleanup - Release resources of an entity enumeration
*
* @ent_enum: Entity enumeration to be released
*/
void media_entity_enum_cleanup(struct media_entity_enum *ent_enum);
/**
* media_entity_enum_zero - Clear the entire enum
*
* @ent_enum: Entity enumeration to be cleared
*/
static inline void media_entity_enum_zero(struct media_entity_enum *ent_enum)
{
bitmap_zero(ent_enum->bmap, ent_enum->idx_max);
}
/**
* media_entity_enum_set - Mark a single entity in the enum
*
* @ent_enum: Entity enumeration
* @entity: Entity to be marked
*/
static inline void media_entity_enum_set(struct media_entity_enum *ent_enum,
struct media_entity *entity)
{
if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
return;
__set_bit(entity->internal_idx, ent_enum->bmap);
}
/**
* media_entity_enum_clear - Unmark a single entity in the enum
*
* @ent_enum: Entity enumeration
* @entity: Entity to be unmarked
*/
static inline void media_entity_enum_clear(struct media_entity_enum *ent_enum,
struct media_entity *entity)
{
if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
return;
__clear_bit(entity->internal_idx, ent_enum->bmap);
}
/**
* media_entity_enum_test - Test whether the entity is marked
*
* @ent_enum: Entity enumeration
* @entity: Entity to be tested
*
* Returns %true if the entity was marked.
*/
static inline bool media_entity_enum_test(struct media_entity_enum *ent_enum,
struct media_entity *entity)
{
if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
return true;
return test_bit(entity->internal_idx, ent_enum->bmap);
}
/**
* media_entity_enum_test_and_set - Test whether the entity is marked,
* and mark it
*
* @ent_enum: Entity enumeration
* @entity: Entity to be tested
*
* Returns %true if the entity was marked, and mark it before doing so.
*/
static inline bool
media_entity_enum_test_and_set(struct media_entity_enum *ent_enum,
struct media_entity *entity)
{
if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
return true;
return __test_and_set_bit(entity->internal_idx, ent_enum->bmap);
}
/**
* media_entity_enum_empty - Test whether the entire enum is empty
*
* @ent_enum: Entity enumeration
*
* Return: %true if the entity was empty.
*/
static inline bool media_entity_enum_empty(struct media_entity_enum *ent_enum)
{
return bitmap_empty(ent_enum->bmap, ent_enum->idx_max);
}
/**
* media_entity_enum_intersects - Test whether two enums intersect
*
* @ent_enum1: First entity enumeration
* @ent_enum2: Second entity enumeration
*
* Return: %true if entity enumerations @ent_enum1 and @ent_enum2 intersect,
* otherwise %false.
*/
static inline bool media_entity_enum_intersects(
struct media_entity_enum *ent_enum1,
struct media_entity_enum *ent_enum2)
{
WARN_ON(ent_enum1->idx_max != ent_enum2->idx_max);
return bitmap_intersects(ent_enum1->bmap, ent_enum2->bmap,
min(ent_enum1->idx_max, ent_enum2->idx_max));
}
/**
* gobj_to_entity - returns the struct &media_entity pointer from the
* @gobj contained on it.
*
* @gobj: Pointer to the struct &media_gobj graph object
*/
#define gobj_to_entity(gobj) \
container_of(gobj, struct media_entity, graph_obj)
/**
* gobj_to_pad - returns the struct &media_pad pointer from the
* @gobj contained on it.
*
* @gobj: Pointer to the struct &media_gobj graph object
*/
#define gobj_to_pad(gobj) \
container_of(gobj, struct media_pad, graph_obj)
/**
* gobj_to_link - returns the struct &media_link pointer from the
* @gobj contained on it.
*
* @gobj: Pointer to the struct &media_gobj graph object
*/
#define gobj_to_link(gobj) \
container_of(gobj, struct media_link, graph_obj)
/**
* gobj_to_intf - returns the struct &media_interface pointer from the
* @gobj contained on it.
*
* @gobj: Pointer to the struct &media_gobj graph object
*/
#define gobj_to_intf(gobj) \
container_of(gobj, struct media_interface, graph_obj)
/**
* intf_to_devnode - returns the struct media_intf_devnode pointer from the
* @intf contained on it.
*
* @intf: Pointer to struct &media_intf_devnode
*/
#define intf_to_devnode(intf) \
container_of(intf, struct media_intf_devnode, intf)
/**
* media_gobj_create - Initialize a graph object
*
* @mdev: Pointer to the &media_device that contains the object
* @type: Type of the object
* @gobj: Pointer to the struct &media_gobj graph object
*
* This routine initializes the embedded struct &media_gobj inside a
* media graph object. It is called automatically if ``media_*_create``
* function calls are used. However, if the object (entity, link, pad,
* interface) is embedded on some other object, this function should be
* called before registering the object at the media controller.
*/
void media_gobj_create(struct media_device *mdev,
enum media_gobj_type type,
struct media_gobj *gobj);
/**
* media_gobj_destroy - Stop using a graph object on a media device
*
* @gobj: Pointer to the struct &media_gobj graph object
*
* This should be called by all routines like media_device_unregister()
* that remove/destroy media graph objects.
*/
void media_gobj_destroy(struct media_gobj *gobj);
/**
* media_entity_pads_init() - Initialize the entity pads
*
* @entity: entity where the pads belong
* @num_pads: total number of sink and source pads
* @pads: Array of @num_pads pads.
*
* The pads array is managed by the entity driver and passed to
* media_entity_pads_init() where its pointer will be stored in the
* &media_entity structure.
*
* If no pads are needed, drivers could either directly fill
* &media_entity->num_pads with 0 and &media_entity->pads with %NULL or call
* this function that will do the same.
*
* As the number of pads is known in advance, the pads array is not allocated
* dynamically but is managed by the entity driver. Most drivers will embed the
* pads array in a driver-specific structure, avoiding dynamic allocation.
*
* Drivers must set the direction of every pad in the pads array before calling
* media_entity_pads_init(). The function will initialize the other pads fields.
*/
int media_entity_pads_init(struct media_entity *entity, u16 num_pads,
struct media_pad *pads);
/**
* media_entity_cleanup() - free resources associated with an entity
*
* @entity: entity where the pads belong
*
* This function must be called during the cleanup phase after unregistering
* the entity (currently, it does nothing).
*
* Calling media_entity_cleanup() on a media_entity whose memory has been
* zeroed but that has not been initialized with media_entity_pad_init() is
* valid and is a no-op.
*/
#if IS_ENABLED(CONFIG_MEDIA_CONTROLLER)
static inline void media_entity_cleanup(struct media_entity *entity) {}
#else
#define media_entity_cleanup(entity) do { } while (false)
#endif
/**
* media_get_pad_index() - retrieves a pad index from an entity
*
* @entity: entity where the pads belong
* @pad_type: the type of the pad, one of MEDIA_PAD_FL_* pad types
* @sig_type: type of signal of the pad to be search
*
* This helper function finds the first pad index inside an entity that
* satisfies both @is_sink and @sig_type conditions.
*
* Return:
*
* On success, return the pad number. If the pad was not found or the media
* entity is a NULL pointer, return -EINVAL.
*/
int media_get_pad_index(struct media_entity *entity, u32 pad_type,
enum media_pad_signal_type sig_type);
/**
* media_create_pad_link() - creates a link between two entities.
*
* @source: pointer to &media_entity of the source pad.
* @source_pad: number of the source pad in the pads array
* @sink: pointer to &media_entity of the sink pad.
* @sink_pad: number of the sink pad in the pads array.
* @flags: Link flags, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* ( seek for ``MEDIA_LNK_FL_*``)
*
* Valid values for flags:
*
* %MEDIA_LNK_FL_ENABLED
* Indicates that the link is enabled and can be used to transfer media data.
* When two or more links target a sink pad, only one of them can be
* enabled at a time.
*
* %MEDIA_LNK_FL_IMMUTABLE
* Indicates that the link enabled state can't be modified at runtime. If
* %MEDIA_LNK_FL_IMMUTABLE is set, then %MEDIA_LNK_FL_ENABLED must also be
* set, since an immutable link is always enabled.
*
* .. note::
*
* Before calling this function, media_entity_pads_init() and
* media_device_register_entity() should be called previously for both ends.
*/
__must_check int media_create_pad_link(struct media_entity *source,
u16 source_pad, struct media_entity *sink,
u16 sink_pad, u32 flags);
/**
* media_create_pad_links() - creates a link between two entities.
*
* @mdev: Pointer to the media_device that contains the object
* @source_function: Function of the source entities. Used only if @source is
* NULL.
* @source: pointer to &media_entity of the source pad. If NULL, it will use
* all entities that matches the @sink_function.
* @source_pad: number of the source pad in the pads array
* @sink_function: Function of the sink entities. Used only if @sink is NULL.
* @sink: pointer to &media_entity of the sink pad. If NULL, it will use
* all entities that matches the @sink_function.
* @sink_pad: number of the sink pad in the pads array.
* @flags: Link flags, as defined in include/uapi/linux/media.h.
* @allow_both_undefined: if %true, then both @source and @sink can be NULL.
* In such case, it will create a crossbar between all entities that
* matches @source_function to all entities that matches @sink_function.
* If %false, it will return 0 and won't create any link if both @source
* and @sink are NULL.
*
* Valid values for flags:
*
* A %MEDIA_LNK_FL_ENABLED flag indicates that the link is enabled and can be
* used to transfer media data. If multiple links are created and this
* flag is passed as an argument, only the first created link will have
* this flag.
*
* A %MEDIA_LNK_FL_IMMUTABLE flag indicates that the link enabled state can't
* be modified at runtime. If %MEDIA_LNK_FL_IMMUTABLE is set, then
* %MEDIA_LNK_FL_ENABLED must also be set since an immutable link is
* always enabled.
*
* It is common for some devices to have multiple source and/or sink entities
* of the same type that should be linked. While media_create_pad_link()
* creates link by link, this function is meant to allow 1:n, n:1 and even
* cross-bar (n:n) links.
*
* .. note::
*
* Before calling this function, media_entity_pads_init() and
* media_device_register_entity() should be called previously for the
* entities to be linked.
*/
int media_create_pad_links(const struct media_device *mdev,
const u32 source_function,
struct media_entity *source,
const u16 source_pad,
const u32 sink_function,
struct media_entity *sink,
const u16 sink_pad,
u32 flags,
const bool allow_both_undefined);
void __media_entity_remove_links(struct media_entity *entity);
/**
* media_entity_remove_links() - remove all links associated with an entity
*
* @entity: pointer to &media_entity
*
* .. note::
*
* This is called automatically when an entity is unregistered via
* media_device_register_entity().
*/
void media_entity_remove_links(struct media_entity *entity);
/**
* __media_entity_setup_link - Configure a media link without locking
* @link: The link being configured
* @flags: Link configuration flags
*
* The bulk of link setup is handled by the two entities connected through the
* link. This function notifies both entities of the link configuration change.
*
* If the link is immutable or if the current and new configuration are
* identical, return immediately.
*
* The user is expected to hold link->source->parent->mutex. If not,
* media_entity_setup_link() should be used instead.
*/
int __media_entity_setup_link(struct media_link *link, u32 flags);
/**
* media_entity_setup_link() - changes the link flags properties in runtime
*
* @link: pointer to &media_link
* @flags: the requested new link flags
*
* The only configurable property is the %MEDIA_LNK_FL_ENABLED link flag
* to enable/disable a link. Links marked with the
* %MEDIA_LNK_FL_IMMUTABLE link flag can not be enabled or disabled.
*
* When a link is enabled or disabled, the media framework calls the
* link_setup operation for the two entities at the source and sink of the
* link, in that order. If the second link_setup call fails, another
* link_setup call is made on the first entity to restore the original link
* flags.
*
* Media device drivers can be notified of link setup operations by setting the
* &media_device.link_notify pointer to a callback function. If provided, the
* notification callback will be called before enabling and after disabling
* links.
*
* Entity drivers must implement the link_setup operation if any of their links
* is non-immutable. The operation must either configure the hardware or store
* the configuration information to be applied later.
*
* Link configuration must not have any side effect on other links. If an
* enabled link at a sink pad prevents another link at the same pad from
* being enabled, the link_setup operation must return %-EBUSY and can't
* implicitly disable the first enabled link.
*
* .. note::
*
* The valid values of the flags for the link is the same as described
* on media_create_pad_link(), for pad to pad links or the same as described
* on media_create_intf_link(), for interface to entity links.
*/
int media_entity_setup_link(struct media_link *link, u32 flags);
/**
* media_entity_find_link - Find a link between two pads
* @source: Source pad
* @sink: Sink pad
*
* Return: returns a pointer to the link between the two entities. If no
* such link exists, return %NULL.
*/
struct media_link *media_entity_find_link(struct media_pad *source,
struct media_pad *sink);
/**
* media_pad_remote_pad_first - Find the first pad at the remote end of a link
* @pad: Pad at the local end of the link
*
* Search for a remote pad connected to the given pad by iterating over all
* links originating or terminating at that pad until an enabled link is found.
*
* Return: returns a pointer to the pad at the remote end of the first found
* enabled link, or %NULL if no enabled link has been found.
*/
struct media_pad *media_pad_remote_pad_first(const struct media_pad *pad);
/**
* media_pad_remote_pad_unique - Find a remote pad connected to a pad
* @pad: The pad
*
* Search for and return a remote pad connected to @pad through an enabled
* link. If multiple (or no) remote pads are found, an error is returned.
*
* The uniqueness constraint makes this helper function suitable for entities
* that support a single active source at a time on a given pad.
*
* Return: A pointer to the remote pad, or one of the following error pointers
* if an error occurs:
*
* * -ENOTUNIQ - Multiple links are enabled
* * -ENOLINK - No connected pad found
*/
struct media_pad *media_pad_remote_pad_unique(const struct media_pad *pad);
/**
* media_entity_remote_pad_unique - Find a remote pad connected to an entity
* @entity: The entity
* @type: The type of pad to find (MEDIA_PAD_FL_SINK or MEDIA_PAD_FL_SOURCE)
*
* Search for and return a remote pad of @type connected to @entity through an
* enabled link. If multiple (or no) remote pads match these criteria, an error
* is returned.
*
* The uniqueness constraint makes this helper function suitable for entities
* that support a single active source or sink at a time.
*
* Return: A pointer to the remote pad, or one of the following error pointers
* if an error occurs:
*
* * -ENOTUNIQ - Multiple links are enabled
* * -ENOLINK - No connected pad found
*/
struct media_pad *
media_entity_remote_pad_unique(const struct media_entity *entity,
unsigned int type);
/**
* media_entity_remote_source_pad_unique - Find a remote source pad connected to
* an entity
* @entity: The entity
*
* Search for and return a remote source pad connected to @entity through an
* enabled link. If multiple (or no) remote pads match these criteria, an error
* is returned.
*
* The uniqueness constraint makes this helper function suitable for entities
* that support a single active source at a time.
*
* Return: A pointer to the remote pad, or one of the following error pointers
* if an error occurs:
*
* * -ENOTUNIQ - Multiple links are enabled
* * -ENOLINK - No connected pad found
*/
static inline struct media_pad *
media_entity_remote_source_pad_unique(const struct media_entity *entity)
{
return media_entity_remote_pad_unique(entity, MEDIA_PAD_FL_SOURCE);
}
/**
* media_pad_is_streaming - Test if a pad is part of a streaming pipeline
* @pad: The pad
*
* Return: True if the pad is part of a pipeline started with the
* media_pipeline_start() function, false otherwise.
*/
static inline bool media_pad_is_streaming(const struct media_pad *pad)
{
return pad->pipe;
}
/**
* media_entity_is_streaming - Test if an entity is part of a streaming pipeline
* @entity: The entity
*
* Return: True if the entity is part of a pipeline started with the
* media_pipeline_start() function, false otherwise.
*/
static inline bool media_entity_is_streaming(const struct media_entity *entity)
{
struct media_pad *pad;
media_entity_for_each_pad(entity, pad) {
if (media_pad_is_streaming(pad))
return true;
}
return false;
}
/**
* media_entity_pipeline - Get the media pipeline an entity is part of
* @entity: The entity
*
* DEPRECATED: use media_pad_pipeline() instead.
*
* This function returns the media pipeline that an entity has been associated
* with when constructing the pipeline with media_pipeline_start(). The pointer
* remains valid until media_pipeline_stop() is called.
*
* In general, entities can be part of multiple pipelines, when carrying
* multiple streams (either on different pads, or on the same pad using
* multiplexed streams). This function is to be used only for entities that
* do not support multiple pipelines.
*
* Return: The media_pipeline the entity is part of, or NULL if the entity is
* not part of any pipeline.
*/
struct media_pipeline *media_entity_pipeline(struct media_entity *entity);
/**
* media_pad_pipeline - Get the media pipeline a pad is part of
* @pad: The pad
*
* This function returns the media pipeline that a pad has been associated
* with when constructing the pipeline with media_pipeline_start(). The pointer
* remains valid until media_pipeline_stop() is called.
*
* Return: The media_pipeline the pad is part of, or NULL if the pad is
* not part of any pipeline.
*/
struct media_pipeline *media_pad_pipeline(struct media_pad *pad);
/**
* media_entity_get_fwnode_pad - Get pad number from fwnode
*
* @entity: The entity
* @fwnode: Pointer to the fwnode_handle which should be used to find the pad
* @direction_flags: Expected direction of the pad, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* (seek for ``MEDIA_PAD_FL_*``)
*
* This function can be used to resolve the media pad number from
* a fwnode. This is useful for devices which use more complex
* mappings of media pads.
*
* If the entity does not implement the get_fwnode_pad() operation
* then this function searches the entity for the first pad that
* matches the @direction_flags.
*
* Return: returns the pad number on success or a negative error code.
*/
int media_entity_get_fwnode_pad(struct media_entity *entity,
const struct fwnode_handle *fwnode,
unsigned long direction_flags);
/**
* media_graph_walk_init - Allocate resources used by graph walk.
*
* @graph: Media graph structure that will be used to walk the graph
* @mdev: Pointer to the &media_device that contains the object
*
* This function is deprecated, use media_pipeline_for_each_pad() instead.
*
* The caller is required to hold the media_device graph_mutex during the graph
* walk until the graph state is released.
*
* Returns zero on success or a negative error code otherwise.
*/
__must_check int media_graph_walk_init(
struct media_graph *graph, struct media_device *mdev);
/**
* media_graph_walk_cleanup - Release resources used by graph walk.
*
* @graph: Media graph structure that will be used to walk the graph
*
* This function is deprecated, use media_pipeline_for_each_pad() instead.
*/
void media_graph_walk_cleanup(struct media_graph *graph);
/**
* media_graph_walk_start - Start walking the media graph at a
* given entity
*
* @graph: Media graph structure that will be used to walk the graph
* @entity: Starting entity
*
* This function is deprecated, use media_pipeline_for_each_pad() instead.
*
* Before using this function, media_graph_walk_init() must be
* used to allocate resources used for walking the graph. This
* function initializes the graph traversal structure to walk the
* entities graph starting at the given entity. The traversal
* structure must not be modified by the caller during graph
* traversal. After the graph walk, the resources must be released
* using media_graph_walk_cleanup().
*/
void media_graph_walk_start(struct media_graph *graph,
struct media_entity *entity);
/**
* media_graph_walk_next - Get the next entity in the graph
* @graph: Media graph structure
*
* This function is deprecated, use media_pipeline_for_each_pad() instead.
*
* Perform a depth-first traversal of the given media entities graph.
*
* The graph structure must have been previously initialized with a call to
* media_graph_walk_start().
*
* Return: returns the next entity in the graph or %NULL if the whole graph
* have been traversed.
*/
struct media_entity *media_graph_walk_next(struct media_graph *graph);
/**
* media_pipeline_start - Mark a pipeline as streaming
* @pad: Starting pad
* @pipe: Media pipeline to be assigned to all pads in the pipeline.
*
* Mark all pads connected to a given pad through enabled links, either
* directly or indirectly, as streaming. The given pipeline object is assigned
* to every pad in the pipeline and stored in the media_pad pipe field.
*
* Calls to this function can be nested, in which case the same number of
* media_pipeline_stop() calls will be required to stop streaming. The
* pipeline pointer must be identical for all nested calls to
* media_pipeline_start().
*/
__must_check int media_pipeline_start(struct media_pad *pad,
struct media_pipeline *pipe);
/**
* __media_pipeline_start - Mark a pipeline as streaming
*
* @pad: Starting pad
* @pipe: Media pipeline to be assigned to all pads in the pipeline.
*
* ..note:: This is the non-locking version of media_pipeline_start()
*/
__must_check int __media_pipeline_start(struct media_pad *pad,
struct media_pipeline *pipe);
/**
* media_pipeline_stop - Mark a pipeline as not streaming
* @pad: Starting pad
*
* Mark all pads connected to a given pad through enabled links, either
* directly or indirectly, as not streaming. The media_pad pipe field is
* reset to %NULL.
*
* If multiple calls to media_pipeline_start() have been made, the same
* number of calls to this function are required to mark the pipeline as not
* streaming.
*/
void media_pipeline_stop(struct media_pad *pad);
/**
* __media_pipeline_stop - Mark a pipeline as not streaming
*
* @pad: Starting pad
*
* .. note:: This is the non-locking version of media_pipeline_stop()
*/
void __media_pipeline_stop(struct media_pad *pad);
struct media_pad *
__media_pipeline_pad_iter_next(struct media_pipeline *pipe,
struct media_pipeline_pad_iter *iter,
struct media_pad *pad);
/**
* media_pipeline_for_each_pad - Iterate on all pads in a media pipeline
* @pipe: The pipeline
* @iter: The iterator (struct media_pipeline_pad_iter)
* @pad: The iterator pad
*
* Iterate on all pads in a media pipeline. This is only valid after the
* pipeline has been built with media_pipeline_start() and before it gets
* destroyed with media_pipeline_stop().
*/
#define media_pipeline_for_each_pad(pipe, iter, pad) \
for (pad = __media_pipeline_pad_iter_next((pipe), iter, NULL); \
pad != NULL; \
pad = __media_pipeline_pad_iter_next((pipe), iter, pad))
/**
* media_pipeline_entity_iter_init - Initialize a pipeline entity iterator
* @pipe: The pipeline
* @iter: The iterator
*
* This function must be called to initialize the iterator before using it in a
* media_pipeline_for_each_entity() loop. The iterator must be destroyed by a
* call to media_pipeline_entity_iter_cleanup after the loop (including in code
* paths that break from the loop).
*
* The same iterator can be used in multiple consecutive loops without being
* destroyed and reinitialized.
*
* Return: 0 on success or a negative error code otherwise.
*/
int media_pipeline_entity_iter_init(struct media_pipeline *pipe,
struct media_pipeline_entity_iter *iter);
/**
* media_pipeline_entity_iter_cleanup - Destroy a pipeline entity iterator
* @iter: The iterator
*
* This function must be called to destroy iterators initialized with
* media_pipeline_entity_iter_init().
*/
void media_pipeline_entity_iter_cleanup(struct media_pipeline_entity_iter *iter);
struct media_entity *
__media_pipeline_entity_iter_next(struct media_pipeline *pipe,
struct media_pipeline_entity_iter *iter,
struct media_entity *entity);
/**
* media_pipeline_for_each_entity - Iterate on all entities in a media pipeline
* @pipe: The pipeline
* @iter: The iterator (struct media_pipeline_entity_iter)
* @entity: The iterator entity
*
* Iterate on all entities in a media pipeline. This is only valid after the
* pipeline has been built with media_pipeline_start() and before it gets
* destroyed with media_pipeline_stop(). The iterator must be initialized with
* media_pipeline_entity_iter_init() before iteration, and destroyed with
* media_pipeline_entity_iter_cleanup() after (including in code paths that
* break from the loop).
*/
#define media_pipeline_for_each_entity(pipe, iter, entity) \
for (entity = __media_pipeline_entity_iter_next((pipe), iter, NULL); \
entity != NULL; \
entity = __media_pipeline_entity_iter_next((pipe), iter, entity))
/**
* media_pipeline_alloc_start - Mark a pipeline as streaming
* @pad: Starting pad
*
* media_pipeline_alloc_start() is similar to media_pipeline_start() but instead
* of working on a given pipeline the function will use an existing pipeline if
* the pad is already part of a pipeline, or allocate a new pipeline.
*
* Calls to media_pipeline_alloc_start() must be matched with
* media_pipeline_stop().
*/
__must_check int media_pipeline_alloc_start(struct media_pad *pad);
/**
* media_devnode_create() - creates and initializes a device node interface
*
* @mdev: pointer to struct &media_device
* @type: type of the interface, as given by
* :ref:`include/uapi/linux/media.h <media_header>`
* ( seek for ``MEDIA_INTF_T_*``) macros.
* @flags: Interface flags, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* ( seek for ``MEDIA_INTF_FL_*``)
* @major: Device node major number.
* @minor: Device node minor number.
*
* Return: if succeeded, returns a pointer to the newly allocated
* &media_intf_devnode pointer.
*
* .. note::
*
* Currently, no flags for &media_interface is defined.
*/
struct media_intf_devnode *
__must_check media_devnode_create(struct media_device *mdev,
u32 type, u32 flags,
u32 major, u32 minor);
/**
* media_devnode_remove() - removes a device node interface
*
* @devnode: pointer to &media_intf_devnode to be freed.
*
* When a device node interface is removed, all links to it are automatically
* removed.
*/
void media_devnode_remove(struct media_intf_devnode *devnode);
/**
* media_create_intf_link() - creates a link between an entity and an interface
*
* @entity: pointer to %media_entity
* @intf: pointer to %media_interface
* @flags: Link flags, as defined in
* :ref:`include/uapi/linux/media.h <media_header>`
* ( seek for ``MEDIA_LNK_FL_*``)
*
*
* Valid values for flags:
*
* %MEDIA_LNK_FL_ENABLED
* Indicates that the interface is connected to the entity hardware.
* That's the default value for interfaces. An interface may be disabled if
* the hardware is busy due to the usage of some other interface that it is
* currently controlling the hardware.
*
* A typical example is an hybrid TV device that handle only one type of
* stream on a given time. So, when the digital TV is streaming,
* the V4L2 interfaces won't be enabled, as such device is not able to
* also stream analog TV or radio.
*
* .. note::
*
* Before calling this function, media_devnode_create() should be called for
* the interface and media_device_register_entity() should be called for the
* interface that will be part of the link.
*/
struct media_link *
__must_check media_create_intf_link(struct media_entity *entity,
struct media_interface *intf,
u32 flags);
/**
* __media_remove_intf_link() - remove a single interface link
*
* @link: pointer to &media_link.
*
* .. note:: This is an unlocked version of media_remove_intf_link()
*/
void __media_remove_intf_link(struct media_link *link);
/**
* media_remove_intf_link() - remove a single interface link
*
* @link: pointer to &media_link.
*
* .. note:: Prefer to use this one, instead of __media_remove_intf_link()
*/
void media_remove_intf_link(struct media_link *link);
/**
* __media_remove_intf_links() - remove all links associated with an interface
*
* @intf: pointer to &media_interface
*
* .. note:: This is an unlocked version of media_remove_intf_links().
*/
void __media_remove_intf_links(struct media_interface *intf);
/**
* media_remove_intf_links() - remove all links associated with an interface
*
* @intf: pointer to &media_interface
*
* .. note::
*
* #) This is called automatically when an entity is unregistered via
* media_device_register_entity() and by media_devnode_remove().
*
* #) Prefer to use this one, instead of __media_remove_intf_links().
*/
void media_remove_intf_links(struct media_interface *intf);
/**
* media_entity_call - Calls a struct media_entity_operations operation on
* an entity
*
* @entity: entity where the @operation will be called
* @operation: type of the operation. Should be the name of a member of
* struct &media_entity_operations.
*
* This helper function will check if @operation is not %NULL. On such case,
* it will issue a call to @operation\(@entity, @args\).
*/
#define media_entity_call(entity, operation, args...) \
(((entity)->ops && (entity)->ops->operation) ? \
(entity)->ops->operation((entity) , ##args) : -ENOIOCTLCMD)
/**
* media_create_ancillary_link() - create an ancillary link between two
* instances of &media_entity
*
* @primary: pointer to the primary &media_entity
* @ancillary: pointer to the ancillary &media_entity
*
* Create an ancillary link between two entities, indicating that they
* represent two connected pieces of hardware that form a single logical unit.
* A typical example is a camera lens controller being linked to the sensor that
* it is supporting.
*
* The function sets both MEDIA_LNK_FL_ENABLED and MEDIA_LNK_FL_IMMUTABLE for
* the new link.
*/
struct media_link *
media_create_ancillary_link(struct media_entity *primary,
struct media_entity *ancillary);
/**
* __media_entity_next_link() - Iterate through a &media_entity's links
*
* @entity: pointer to the &media_entity
* @link: pointer to a &media_link to hold the iterated values
* @link_type: one of the MEDIA_LNK_FL_LINK_TYPE flags
*
* Return the next link against an entity matching a specific link type. This
* allows iteration through an entity's links whilst guaranteeing all of the
* returned links are of the given type.
*/
struct media_link *__media_entity_next_link(struct media_entity *entity,
struct media_link *link,
unsigned long link_type);
/**
* for_each_media_entity_data_link() - Iterate through an entity's data links
*
* @entity: pointer to the &media_entity
* @link: pointer to a &media_link to hold the iterated values
*
* Iterate over a &media_entity's data links
*/
#define for_each_media_entity_data_link(entity, link) \
for (link = __media_entity_next_link(entity, NULL, \
MEDIA_LNK_FL_DATA_LINK); \
link; \
link = __media_entity_next_link(entity, link, \
MEDIA_LNK_FL_DATA_LINK))
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* property.c - Unified device property interface.
*
* Copyright (C) 2014, Intel Corporation
* Authors: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
* Mika Westerberg <mika.westerberg@linux.intel.com>
*/
#include <linux/device.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/kconfig.h>
#include <linux/of.h>
#include <linux/property.h>
#include <linux/phy.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/types.h>
struct fwnode_handle *__dev_fwnode(struct device *dev)
{
return IS_ENABLED(CONFIG_OF) && dev->of_node ?
of_fwnode_handle(dev->of_node) : dev->fwnode;
}
EXPORT_SYMBOL_GPL(__dev_fwnode);
const struct fwnode_handle *__dev_fwnode_const(const struct device *dev)
{
return IS_ENABLED(CONFIG_OF) && dev->of_node ?
of_fwnode_handle(dev->of_node) : dev->fwnode;
}
EXPORT_SYMBOL_GPL(__dev_fwnode_const);
/**
* device_property_present - check if a property of a device is present
* @dev: Device whose property is being checked
* @propname: Name of the property
*
* Check if property @propname is present in the device firmware description.
*
* Return: true if property @propname is present. Otherwise, returns false.
*/
bool device_property_present(const struct device *dev, const char *propname)
{
return fwnode_property_present(dev_fwnode(dev), propname);
}
EXPORT_SYMBOL_GPL(device_property_present);
/**
* fwnode_property_present - check if a property of a firmware node is present
* @fwnode: Firmware node whose property to check
* @propname: Name of the property
*
* Return: true if property @propname is present. Otherwise, returns false.
*/
bool fwnode_property_present(const struct fwnode_handle *fwnode,
const char *propname)
{
bool ret;
if (IS_ERR_OR_NULL(fwnode))
return false;
ret = fwnode_call_bool_op(fwnode, property_present, propname);
if (ret)
return ret;
return fwnode_call_bool_op(fwnode->secondary, property_present, propname);
}
EXPORT_SYMBOL_GPL(fwnode_property_present);
/**
* device_property_read_u8_array - return a u8 array property of a device
* @dev: Device to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Function reads an array of u8 properties with @propname from the device
* firmware description and stores them to @val if found.
*
* It's recommended to call device_property_count_u8() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected.
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_read_u8_array(const struct device *dev, const char *propname,
u8 *val, size_t nval)
{
return fwnode_property_read_u8_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u8_array);
/**
* device_property_read_u16_array - return a u16 array property of a device
* @dev: Device to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Function reads an array of u16 properties with @propname from the device
* firmware description and stores them to @val if found.
*
* It's recommended to call device_property_count_u16() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected.
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_read_u16_array(const struct device *dev, const char *propname,
u16 *val, size_t nval)
{
return fwnode_property_read_u16_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u16_array);
/**
* device_property_read_u32_array - return a u32 array property of a device
* @dev: Device to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Function reads an array of u32 properties with @propname from the device
* firmware description and stores them to @val if found.
*
* It's recommended to call device_property_count_u32() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected.
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_read_u32_array(const struct device *dev, const char *propname,
u32 *val, size_t nval)
{
return fwnode_property_read_u32_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u32_array);
/**
* device_property_read_u64_array - return a u64 array property of a device
* @dev: Device to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Function reads an array of u64 properties with @propname from the device
* firmware description and stores them to @val if found.
*
* It's recommended to call device_property_count_u64() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected.
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_read_u64_array(const struct device *dev, const char *propname,
u64 *val, size_t nval)
{
return fwnode_property_read_u64_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u64_array);
/**
* device_property_read_string_array - return a string array property of device
* @dev: Device to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Function reads an array of string properties with @propname from the device
* firmware description and stores them to @val if found.
*
* It's recommended to call device_property_string_array_count() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values read on success if @val is non-NULL,
* number of values available on success if @val is NULL,
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO or %-EILSEQ if the property is not an array of strings,
* %-EOVERFLOW if the size of the property is not as expected.
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_read_string_array(const struct device *dev, const char *propname,
const char **val, size_t nval)
{
return fwnode_property_read_string_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_string_array);
/**
* device_property_read_string - return a string property of a device
* @dev: Device to get the property of
* @propname: Name of the property
* @val: The value is stored here
*
* Function reads property @propname from the device firmware description and
* stores the value into @val if found. The value is checked to be a string.
*
* Return: %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO or %-EILSEQ if the property type is not a string.
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_read_string(const struct device *dev, const char *propname,
const char **val)
{
return fwnode_property_read_string(dev_fwnode(dev), propname, val);
}
EXPORT_SYMBOL_GPL(device_property_read_string);
/**
* device_property_match_string - find a string in an array and return index
* @dev: Device to get the property of
* @propname: Name of the property holding the array
* @string: String to look for
*
* Find a given string in a string array and if it is found return the
* index back.
*
* Return: index, starting from %0, if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of strings,
* %-ENXIO if no suitable firmware interface is present.
*/
int device_property_match_string(const struct device *dev, const char *propname,
const char *string)
{
return fwnode_property_match_string(dev_fwnode(dev), propname, string);
}
EXPORT_SYMBOL_GPL(device_property_match_string);
static int fwnode_property_read_int_array(const struct fwnode_handle *fwnode,
const char *propname,
unsigned int elem_size, void *val,
size_t nval)
{
int ret;
if (IS_ERR_OR_NULL(fwnode))
return -EINVAL;
ret = fwnode_call_int_op(fwnode, property_read_int_array, propname,
elem_size, val, nval);
if (ret != -EINVAL)
return ret;
return fwnode_call_int_op(fwnode->secondary, property_read_int_array, propname,
elem_size, val, nval);
}
/**
* fwnode_property_read_u8_array - return a u8 array property of firmware node
* @fwnode: Firmware node to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Read an array of u8 properties with @propname from @fwnode and stores them to
* @val if found.
*
* It's recommended to call fwnode_property_count_u8() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_read_u8_array(const struct fwnode_handle *fwnode,
const char *propname, u8 *val, size_t nval)
{
return fwnode_property_read_int_array(fwnode, propname, sizeof(u8),
val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u8_array);
/**
* fwnode_property_read_u16_array - return a u16 array property of firmware node
* @fwnode: Firmware node to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Read an array of u16 properties with @propname from @fwnode and store them to
* @val if found.
*
* It's recommended to call fwnode_property_count_u16() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_read_u16_array(const struct fwnode_handle *fwnode,
const char *propname, u16 *val, size_t nval)
{
return fwnode_property_read_int_array(fwnode, propname, sizeof(u16),
val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u16_array);
/**
* fwnode_property_read_u32_array - return a u32 array property of firmware node
* @fwnode: Firmware node to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Read an array of u32 properties with @propname from @fwnode store them to
* @val if found.
*
* It's recommended to call fwnode_property_count_u32() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_read_u32_array(const struct fwnode_handle *fwnode,
const char *propname, u32 *val, size_t nval)
{
return fwnode_property_read_int_array(fwnode, propname, sizeof(u32),
val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u32_array);
/**
* fwnode_property_read_u64_array - return a u64 array property firmware node
* @fwnode: Firmware node to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Read an array of u64 properties with @propname from @fwnode and store them to
* @val if found.
*
* It's recommended to call fwnode_property_count_u64() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values if @val was %NULL,
* %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of numbers,
* %-EOVERFLOW if the size of the property is not as expected,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_read_u64_array(const struct fwnode_handle *fwnode,
const char *propname, u64 *val, size_t nval)
{
return fwnode_property_read_int_array(fwnode, propname, sizeof(u64),
val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u64_array);
/**
* fwnode_property_read_string_array - return string array property of a node
* @fwnode: Firmware node to get the property of
* @propname: Name of the property
* @val: The values are stored here or %NULL to return the number of values
* @nval: Size of the @val array
*
* Read an string list property @propname from the given firmware node and store
* them to @val if found.
*
* It's recommended to call fwnode_property_string_array_count() instead of calling
* this function with @val equals %NULL and @nval equals 0.
*
* Return: number of values read on success if @val is non-NULL,
* number of values available on success if @val is NULL,
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO or %-EILSEQ if the property is not an array of strings,
* %-EOVERFLOW if the size of the property is not as expected,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
const char *propname, const char **val,
size_t nval)
{
int ret;
if (IS_ERR_OR_NULL(fwnode))
return -EINVAL;
ret = fwnode_call_int_op(fwnode, property_read_string_array, propname,
val, nval);
if (ret != -EINVAL)
return ret;
return fwnode_call_int_op(fwnode->secondary, property_read_string_array, propname,
val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_string_array);
/**
* fwnode_property_read_string - return a string property of a firmware node
* @fwnode: Firmware node to get the property of
* @propname: Name of the property
* @val: The value is stored here
*
* Read property @propname from the given firmware node and store the value into
* @val if found. The value is checked to be a string.
*
* Return: %0 if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO or %-EILSEQ if the property is not a string,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_read_string(const struct fwnode_handle *fwnode,
const char *propname, const char **val)
{
int ret = fwnode_property_read_string_array(fwnode, propname, val, 1);
return ret < 0 ? ret : 0;
}
EXPORT_SYMBOL_GPL(fwnode_property_read_string);
/**
* fwnode_property_match_string - find a string in an array and return index
* @fwnode: Firmware node to get the property of
* @propname: Name of the property holding the array
* @string: String to look for
*
* Find a given string in a string array and if it is found return the
* index back.
*
* Return: index, starting from %0, if the property was found (success),
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO if the property is not an array of strings,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_match_string(const struct fwnode_handle *fwnode,
const char *propname, const char *string)
{
const char **values;
int nval, ret;
nval = fwnode_property_string_array_count(fwnode, propname);
if (nval < 0)
return nval;
if (nval == 0)
return -ENODATA;
values = kcalloc(nval, sizeof(*values), GFP_KERNEL);
if (!values)
return -ENOMEM;
ret = fwnode_property_read_string_array(fwnode, propname, values, nval);
if (ret < 0)
goto out_free;
ret = match_string(values, nval, string);
if (ret < 0)
ret = -ENODATA;
out_free:
kfree(values);
return ret;
}
EXPORT_SYMBOL_GPL(fwnode_property_match_string);
/**
* fwnode_property_match_property_string - find a property string value in an array and return index
* @fwnode: Firmware node to get the property of
* @propname: Name of the property holding the string value
* @array: String array to search in
* @n: Size of the @array
*
* Find a property string value in a given @array and if it is found return
* the index back.
*
* Return: index, starting from %0, if the string value was found in the @array (success),
* %-ENOENT when the string value was not found in the @array,
* %-EINVAL if given arguments are not valid,
* %-ENODATA if the property does not have a value,
* %-EPROTO or %-EILSEQ if the property is not a string,
* %-ENXIO if no suitable firmware interface is present.
*/
int fwnode_property_match_property_string(const struct fwnode_handle *fwnode,
const char *propname, const char * const *array, size_t n)
{
const char *string;
int ret;
ret = fwnode_property_read_string(fwnode, propname, &string);
if (ret)
return ret;
ret = match_string(array, n, string);
if (ret < 0)
ret = -ENOENT;
return ret;
}
EXPORT_SYMBOL_GPL(fwnode_property_match_property_string);
/**
* fwnode_property_get_reference_args() - Find a reference with arguments
* @fwnode: Firmware node where to look for the reference
* @prop: The name of the property
* @nargs_prop: The name of the property telling the number of
* arguments in the referred node. NULL if @nargs is known,
* otherwise @nargs is ignored. Only relevant on OF.
* @nargs: Number of arguments. Ignored if @nargs_prop is non-NULL.
* @index: Index of the reference, from zero onwards.
* @args: Result structure with reference and integer arguments.
* May be NULL.
*
* Obtain a reference based on a named property in an fwnode, with
* integer arguments.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* @args->fwnode pointer.
*
* Return: %0 on success
* %-ENOENT when the index is out of bounds, the index has an empty
* reference or the property was not found
* %-EINVAL on parse error
*/
int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode,
const char *prop, const char *nargs_prop,
unsigned int nargs, unsigned int index,
struct fwnode_reference_args *args)
{
int ret;
if (IS_ERR_OR_NULL(fwnode))
return -ENOENT;
ret = fwnode_call_int_op(fwnode, get_reference_args, prop, nargs_prop,
nargs, index, args);
if (ret == 0)
return ret;
if (IS_ERR_OR_NULL(fwnode->secondary))
return ret;
return fwnode_call_int_op(fwnode->secondary, get_reference_args, prop, nargs_prop,
nargs, index, args);
}
EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);
/**
* fwnode_find_reference - Find named reference to a fwnode_handle
* @fwnode: Firmware node where to look for the reference
* @name: The name of the reference
* @index: Index of the reference
*
* @index can be used when the named reference holds a table of references.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*
* Return: a pointer to the reference fwnode, when found. Otherwise,
* returns an error pointer.
*/
struct fwnode_handle *fwnode_find_reference(const struct fwnode_handle *fwnode,
const char *name,
unsigned int index)
{
struct fwnode_reference_args args;
int ret;
ret = fwnode_property_get_reference_args(fwnode, name, NULL, 0, index,
&args);
return ret ? ERR_PTR(ret) : args.fwnode;
}
EXPORT_SYMBOL_GPL(fwnode_find_reference);
/**
* fwnode_get_name - Return the name of a node
* @fwnode: The firmware node
*
* Return: a pointer to the node name, or %NULL.
*/
const char *fwnode_get_name(const struct fwnode_handle *fwnode)
{
return fwnode_call_ptr_op(fwnode, get_name);
}
EXPORT_SYMBOL_GPL(fwnode_get_name);
/**
* fwnode_get_name_prefix - Return the prefix of node for printing purposes
* @fwnode: The firmware node
*
* Return: the prefix of a node, intended to be printed right before the node.
* The prefix works also as a separator between the nodes.
*/
const char *fwnode_get_name_prefix(const struct fwnode_handle *fwnode)
{
return fwnode_call_ptr_op(fwnode, get_name_prefix);
}
/**
* fwnode_name_eq - Return true if node name is equal
* @fwnode: The firmware node
* @name: The name to which to compare the node name
*
* Compare the name provided as an argument to the name of the node, stopping
* the comparison at either NUL or '@' character, whichever comes first. This
* function is generally used for comparing node names while ignoring the
* possible unit address of the node.
*
* Return: true if the node name matches with the name provided in the @name
* argument, false otherwise.
*/
bool fwnode_name_eq(const struct fwnode_handle *fwnode, const char *name)
{
const char *node_name;
ptrdiff_t len;
node_name = fwnode_get_name(fwnode);
if (!node_name)
return false;
len = strchrnul(node_name, '@') - node_name;
return str_has_prefix(node_name, name) == len;
}
EXPORT_SYMBOL_GPL(fwnode_name_eq);
/**
* fwnode_get_parent - Return parent firwmare node
* @fwnode: Firmware whose parent is retrieved
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*
* Return: parent firmware node of the given node if possible or %NULL if no
* parent was available.
*/
struct fwnode_handle *fwnode_get_parent(const struct fwnode_handle *fwnode)
{
return fwnode_call_ptr_op(fwnode, get_parent);
}
EXPORT_SYMBOL_GPL(fwnode_get_parent);
/**
* fwnode_get_next_parent - Iterate to the node's parent
* @fwnode: Firmware whose parent is retrieved
*
* This is like fwnode_get_parent() except that it drops the refcount
* on the passed node, making it suitable for iterating through a
* node's parents.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer. Note that this function also puts a reference to @fwnode
* unconditionally.
*
* Return: parent firmware node of the given node if possible or %NULL if no
* parent was available.
*/
struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode)
{
struct fwnode_handle *parent = fwnode_get_parent(fwnode);
fwnode_handle_put(fwnode);
return parent;
}
EXPORT_SYMBOL_GPL(fwnode_get_next_parent);
/**
* fwnode_count_parents - Return the number of parents a node has
* @fwnode: The node the parents of which are to be counted
*
* Return: the number of parents a node has.
*/
unsigned int fwnode_count_parents(const struct fwnode_handle *fwnode)
{
struct fwnode_handle *parent;
unsigned int count = 0;
fwnode_for_each_parent_node(fwnode, parent)
count++;
return count;
}
EXPORT_SYMBOL_GPL(fwnode_count_parents);
/**
* fwnode_get_nth_parent - Return an nth parent of a node
* @fwnode: The node the parent of which is requested
* @depth: Distance of the parent from the node
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*
* Return: the nth parent of a node. If there is no parent at the requested
* @depth, %NULL is returned. If @depth is 0, the functionality is equivalent to
* fwnode_handle_get(). For @depth == 1, it is fwnode_get_parent() and so on.
*/
struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwnode,
unsigned int depth)
{
struct fwnode_handle *parent;
if (depth == 0)
return fwnode_handle_get(fwnode);
fwnode_for_each_parent_node(fwnode, parent) {
if (--depth == 0)
return parent;
}
return NULL;
}
EXPORT_SYMBOL_GPL(fwnode_get_nth_parent);
/**
* fwnode_get_next_child_node - Return the next child node handle for a node
* @fwnode: Firmware node to find the next child node for.
* @child: Handle to one of the node's child nodes or a %NULL handle.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer. Note that this function also puts a reference to @child
* unconditionally.
*/
struct fwnode_handle *
fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
struct fwnode_handle *child)
{
return fwnode_call_ptr_op(fwnode, get_next_child_node, child);
}
EXPORT_SYMBOL_GPL(fwnode_get_next_child_node);
/**
* fwnode_get_next_available_child_node - Return the next available child node handle for a node
* @fwnode: Firmware node to find the next child node for.
* @child: Handle to one of the node's child nodes or a %NULL handle.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer. Note that this function also puts a reference to @child
* unconditionally.
*/
struct fwnode_handle *
fwnode_get_next_available_child_node(const struct fwnode_handle *fwnode,
struct fwnode_handle *child)
{
struct fwnode_handle *next_child = child;
if (IS_ERR_OR_NULL(fwnode))
return NULL;
do {
next_child = fwnode_get_next_child_node(fwnode, next_child);
if (!next_child)
return NULL;
} while (!fwnode_device_is_available(next_child));
return next_child;
}
EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node);
/**
* device_get_next_child_node - Return the next child node handle for a device
* @dev: Device to find the next child node for.
* @child: Handle to one of the device's child nodes or a %NULL handle.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer. Note that this function also puts a reference to @child
* unconditionally.
*/
struct fwnode_handle *device_get_next_child_node(const struct device *dev,
struct fwnode_handle *child)
{
const struct fwnode_handle *fwnode = dev_fwnode(dev);
struct fwnode_handle *next;
if (IS_ERR_OR_NULL(fwnode))
return NULL;
/* Try to find a child in primary fwnode */
next = fwnode_get_next_child_node(fwnode, child);
if (next)
return next;
/* When no more children in primary, continue with secondary */
return fwnode_get_next_child_node(fwnode->secondary, child);
}
EXPORT_SYMBOL_GPL(device_get_next_child_node);
/**
* fwnode_get_named_child_node - Return first matching named child node handle
* @fwnode: Firmware node to find the named child node for.
* @childname: String to match child node name against.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*/
struct fwnode_handle *
fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
const char *childname)
{
return fwnode_call_ptr_op(fwnode, get_named_child_node, childname);
}
EXPORT_SYMBOL_GPL(fwnode_get_named_child_node);
/**
* device_get_named_child_node - Return first matching named child node handle
* @dev: Device to find the named child node for.
* @childname: String to match child node name against.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*/
struct fwnode_handle *device_get_named_child_node(const struct device *dev,
const char *childname)
{
return fwnode_get_named_child_node(dev_fwnode(dev), childname);
}
EXPORT_SYMBOL_GPL(device_get_named_child_node);
/**
* fwnode_handle_get - Obtain a reference to a device node
* @fwnode: Pointer to the device node to obtain the reference to.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*
* Return: the fwnode handle.
*/
struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *fwnode)
{
if (!fwnode_has_op(fwnode, get))
return fwnode;
return fwnode_call_ptr_op(fwnode, get);
}
EXPORT_SYMBOL_GPL(fwnode_handle_get);
/**
* fwnode_handle_put - Drop reference to a device node
* @fwnode: Pointer to the device node to drop the reference to.
*
* This has to be used when terminating device_for_each_child_node() iteration
* with break or return to prevent stale device node references from being left
* behind.
*/
void fwnode_handle_put(struct fwnode_handle *fwnode)
{
fwnode_call_void_op(fwnode, put);
}
EXPORT_SYMBOL_GPL(fwnode_handle_put);
/**
* fwnode_device_is_available - check if a device is available for use
* @fwnode: Pointer to the fwnode of the device.
*
* Return: true if device is available for use. Otherwise, returns false.
*
* For fwnode node types that don't implement the .device_is_available()
* operation, this function returns true.
*/
bool fwnode_device_is_available(const struct fwnode_handle *fwnode)
{
if (IS_ERR_OR_NULL(fwnode))
return false;
if (!fwnode_has_op(fwnode, device_is_available))
return true;
return fwnode_call_bool_op(fwnode, device_is_available);
}
EXPORT_SYMBOL_GPL(fwnode_device_is_available);
/**
* device_get_child_node_count - return the number of child nodes for device
* @dev: Device to cound the child nodes for
*
* Return: the number of child nodes for a given device.
*/
unsigned int device_get_child_node_count(const struct device *dev)
{
struct fwnode_handle *child;
unsigned int count = 0;
device_for_each_child_node(dev, child)
count++;
return count;
}
EXPORT_SYMBOL_GPL(device_get_child_node_count);
bool device_dma_supported(const struct device *dev)
{
return fwnode_call_bool_op(dev_fwnode(dev), device_dma_supported);
}
EXPORT_SYMBOL_GPL(device_dma_supported);
enum dev_dma_attr device_get_dma_attr(const struct device *dev)
{
if (!fwnode_has_op(dev_fwnode(dev), device_get_dma_attr))
return DEV_DMA_NOT_SUPPORTED;
return fwnode_call_int_op(dev_fwnode(dev), device_get_dma_attr);
}
EXPORT_SYMBOL_GPL(device_get_dma_attr);
/**
* fwnode_get_phy_mode - Get phy mode for given firmware node
* @fwnode: Pointer to the given node
*
* The function gets phy interface string from property 'phy-mode' or
* 'phy-connection-type', and return its index in phy_modes table, or errno in
* error case.
*/
int fwnode_get_phy_mode(const struct fwnode_handle *fwnode)
{
const char *pm;
int err, i;
err = fwnode_property_read_string(fwnode, "phy-mode", &pm);
if (err < 0)
err = fwnode_property_read_string(fwnode,
"phy-connection-type", &pm);
if (err < 0)
return err;
for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++)
if (!strcasecmp(pm, phy_modes(i)))
return i;
return -ENODEV;
}
EXPORT_SYMBOL_GPL(fwnode_get_phy_mode);
/**
* device_get_phy_mode - Get phy mode for given device
* @dev: Pointer to the given device
*
* The function gets phy interface string from property 'phy-mode' or
* 'phy-connection-type', and return its index in phy_modes table, or errno in
* error case.
*/
int device_get_phy_mode(struct device *dev)
{
return fwnode_get_phy_mode(dev_fwnode(dev));
}
EXPORT_SYMBOL_GPL(device_get_phy_mode);
/**
* fwnode_iomap - Maps the memory mapped IO for a given fwnode
* @fwnode: Pointer to the firmware node
* @index: Index of the IO range
*
* Return: a pointer to the mapped memory.
*/
void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index)
{
return fwnode_call_ptr_op(fwnode, iomap, index);
}
EXPORT_SYMBOL(fwnode_iomap);
/**
* fwnode_irq_get - Get IRQ directly from a fwnode
* @fwnode: Pointer to the firmware node
* @index: Zero-based index of the IRQ
*
* Return: Linux IRQ number on success. Negative errno on failure.
*/
int fwnode_irq_get(const struct fwnode_handle *fwnode, unsigned int index)
{
int ret;
ret = fwnode_call_int_op(fwnode, irq_get, index);
/* We treat mapping errors as invalid case */
if (ret == 0)
return -EINVAL;
return ret;
}
EXPORT_SYMBOL(fwnode_irq_get);
/**
* fwnode_irq_get_byname - Get IRQ from a fwnode using its name
* @fwnode: Pointer to the firmware node
* @name: IRQ name
*
* Description:
* Find a match to the string @name in the 'interrupt-names' string array
* in _DSD for ACPI, or of_node for Device Tree. Then get the Linux IRQ
* number of the IRQ resource corresponding to the index of the matched
* string.
*
* Return: Linux IRQ number on success, or negative errno otherwise.
*/
int fwnode_irq_get_byname(const struct fwnode_handle *fwnode, const char *name)
{
int index;
if (!name)
return -EINVAL;
index = fwnode_property_match_string(fwnode, "interrupt-names", name);
if (index < 0)
return index;
return fwnode_irq_get(fwnode, index);
}
EXPORT_SYMBOL(fwnode_irq_get_byname);
/**
* fwnode_graph_get_next_endpoint - Get next endpoint firmware node
* @fwnode: Pointer to the parent firmware node
* @prev: Previous endpoint node or %NULL to get the first
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer. Note that this function also puts a reference to @prev
* unconditionally.
*
* Return: an endpoint firmware node pointer or %NULL if no more endpoints
* are available.
*/
struct fwnode_handle *
fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode,
struct fwnode_handle *prev)
{
struct fwnode_handle *ep, *port_parent = NULL;
const struct fwnode_handle *parent;
/*
* If this function is in a loop and the previous iteration returned
* an endpoint from fwnode->secondary, then we need to use the secondary
* as parent rather than @fwnode.
*/
if (prev) {
port_parent = fwnode_graph_get_port_parent(prev);
parent = port_parent;
} else {
parent = fwnode;
}
if (IS_ERR_OR_NULL(parent))
return NULL;
ep = fwnode_call_ptr_op(parent, graph_get_next_endpoint, prev);
if (ep)
goto out_put_port_parent;
ep = fwnode_graph_get_next_endpoint(parent->secondary, NULL);
out_put_port_parent:
fwnode_handle_put(port_parent);
return ep;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_next_endpoint);
/**
* fwnode_graph_get_port_parent - Return the device fwnode of a port endpoint
* @endpoint: Endpoint firmware node of the port
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*
* Return: the firmware node of the device the @endpoint belongs to.
*/
struct fwnode_handle *
fwnode_graph_get_port_parent(const struct fwnode_handle *endpoint)
{
struct fwnode_handle *port, *parent;
port = fwnode_get_parent(endpoint);
parent = fwnode_call_ptr_op(port, graph_get_port_parent);
fwnode_handle_put(port);
return parent;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_port_parent);
/**
* fwnode_graph_get_remote_port_parent - Return fwnode of a remote device
* @fwnode: Endpoint firmware node pointing to the remote endpoint
*
* Extracts firmware node of a remote device the @fwnode points to.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*/
struct fwnode_handle *
fwnode_graph_get_remote_port_parent(const struct fwnode_handle *fwnode)
{
struct fwnode_handle *endpoint, *parent;
endpoint = fwnode_graph_get_remote_endpoint(fwnode);
parent = fwnode_graph_get_port_parent(endpoint);
fwnode_handle_put(endpoint);
return parent;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port_parent);
/**
* fwnode_graph_get_remote_port - Return fwnode of a remote port
* @fwnode: Endpoint firmware node pointing to the remote endpoint
*
* Extracts firmware node of a remote port the @fwnode points to.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*/
struct fwnode_handle *
fwnode_graph_get_remote_port(const struct fwnode_handle *fwnode)
{
return fwnode_get_next_parent(fwnode_graph_get_remote_endpoint(fwnode));
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port);
/**
* fwnode_graph_get_remote_endpoint - Return fwnode of a remote endpoint
* @fwnode: Endpoint firmware node pointing to the remote endpoint
*
* Extracts firmware node of a remote endpoint the @fwnode points to.
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*/
struct fwnode_handle *
fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode)
{
return fwnode_call_ptr_op(fwnode, graph_get_remote_endpoint);
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_endpoint);
static bool fwnode_graph_remote_available(struct fwnode_handle *ep)
{
struct fwnode_handle *dev_node;
bool available;
dev_node = fwnode_graph_get_remote_port_parent(ep);
available = fwnode_device_is_available(dev_node);
fwnode_handle_put(dev_node);
return available;
}
/**
* fwnode_graph_get_endpoint_by_id - get endpoint by port and endpoint numbers
* @fwnode: parent fwnode_handle containing the graph
* @port: identifier of the port node
* @endpoint: identifier of the endpoint node under the port node
* @flags: fwnode lookup flags
*
* The caller is responsible for calling fwnode_handle_put() on the returned
* fwnode pointer.
*
* Return: the fwnode handle of the local endpoint corresponding the port and
* endpoint IDs or %NULL if not found.
*
* If FWNODE_GRAPH_ENDPOINT_NEXT is passed in @flags and the specified endpoint
* has not been found, look for the closest endpoint ID greater than the
* specified one and return the endpoint that corresponds to it, if present.
*
* Does not return endpoints that belong to disabled devices or endpoints that
* are unconnected, unless FWNODE_GRAPH_DEVICE_DISABLED is passed in @flags.
*/
struct fwnode_handle *
fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode,
u32 port, u32 endpoint, unsigned long flags)
{
struct fwnode_handle *ep, *best_ep = NULL;
unsigned int best_ep_id = 0;
bool endpoint_next = flags & FWNODE_GRAPH_ENDPOINT_NEXT;
bool enabled_only = !(flags & FWNODE_GRAPH_DEVICE_DISABLED);
fwnode_graph_for_each_endpoint(fwnode, ep) {
struct fwnode_endpoint fwnode_ep = { 0 };
int ret;
if (enabled_only && !fwnode_graph_remote_available(ep))
continue;
ret = fwnode_graph_parse_endpoint(ep, &fwnode_ep);
if (ret < 0)
continue;
if (fwnode_ep.port != port)
continue;
if (fwnode_ep.id == endpoint)
return ep;
if (!endpoint_next)
continue;
/*
* If the endpoint that has just been found is not the first
* matching one and the ID of the one found previously is closer
* to the requested endpoint ID, skip it.
*/
if (fwnode_ep.id < endpoint ||
(best_ep && best_ep_id < fwnode_ep.id))
continue;
fwnode_handle_put(best_ep);
best_ep = fwnode_handle_get(ep);
best_ep_id = fwnode_ep.id;
}
return best_ep;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_by_id);
/**
* fwnode_graph_get_endpoint_count - Count endpoints on a device node
* @fwnode: The node related to a device
* @flags: fwnode lookup flags
* Count endpoints in a device node.
*
* If FWNODE_GRAPH_DEVICE_DISABLED flag is specified, also unconnected endpoints
* and endpoints connected to disabled devices are counted.
*/
unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode,
unsigned long flags)
{
struct fwnode_handle *ep;
unsigned int count = 0;
fwnode_graph_for_each_endpoint(fwnode, ep) {
if (flags & FWNODE_GRAPH_DEVICE_DISABLED ||
fwnode_graph_remote_available(ep))
count++;
}
return count;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_count);
/**
* fwnode_graph_parse_endpoint - parse common endpoint node properties
* @fwnode: pointer to endpoint fwnode_handle
* @endpoint: pointer to the fwnode endpoint data structure
*
* Parse @fwnode representing a graph endpoint node and store the
* information in @endpoint. The caller must hold a reference to
* @fwnode.
*/
int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
struct fwnode_endpoint *endpoint)
{
memset(endpoint, 0, sizeof(*endpoint));
return fwnode_call_int_op(fwnode, graph_parse_endpoint, endpoint);
}
EXPORT_SYMBOL(fwnode_graph_parse_endpoint);
const void *device_get_match_data(const struct device *dev)
{
return fwnode_call_ptr_op(dev_fwnode(dev), device_get_match_data, dev);
}
EXPORT_SYMBOL_GPL(device_get_match_data);
static unsigned int fwnode_graph_devcon_matches(const struct fwnode_handle *fwnode,
const char *con_id, void *data,
devcon_match_fn_t match,
void **matches,
unsigned int matches_len)
{
struct fwnode_handle *node;
struct fwnode_handle *ep;
unsigned int count = 0;
void *ret;
fwnode_graph_for_each_endpoint(fwnode, ep) {
if (matches && count >= matches_len) {
fwnode_handle_put(ep);
break;
}
node = fwnode_graph_get_remote_port_parent(ep);
if (!fwnode_device_is_available(node)) {
fwnode_handle_put(node);
continue;
}
ret = match(node, con_id, data);
fwnode_handle_put(node);
if (ret) {
if (matches)
matches[count] = ret;
count++;
}
}
return count;
}
static unsigned int fwnode_devcon_matches(const struct fwnode_handle *fwnode,
const char *con_id, void *data,
devcon_match_fn_t match,
void **matches,
unsigned int matches_len)
{
struct fwnode_handle *node;
unsigned int count = 0;
unsigned int i;
void *ret;
for (i = 0; ; i++) {
if (matches && count >= matches_len)
break;
node = fwnode_find_reference(fwnode, con_id, i);
if (IS_ERR(node))
break;
ret = match(node, NULL, data);
fwnode_handle_put(node);
if (ret) {
if (matches)
matches[count] = ret;
count++;
}
}
return count;
}
/**
* fwnode_connection_find_match - Find connection from a device node
* @fwnode: Device node with the connection
* @con_id: Identifier for the connection
* @data: Data for the match function
* @match: Function to check and convert the connection description
*
* Find a connection with unique identifier @con_id between @fwnode and another
* device node. @match will be used to convert the connection description to
* data the caller is expecting to be returned.
*/
void *fwnode_connection_find_match(const struct fwnode_handle *fwnode,
const char *con_id, void *data,
devcon_match_fn_t match)
{
unsigned int count;
void *ret;
if (!fwnode || !match)
return NULL;
count = fwnode_graph_devcon_matches(fwnode, con_id, data, match, &ret, 1);
if (count)
return ret;
count = fwnode_devcon_matches(fwnode, con_id, data, match, &ret, 1);
return count ? ret : NULL;
}
EXPORT_SYMBOL_GPL(fwnode_connection_find_match);
/**
* fwnode_connection_find_matches - Find connections from a device node
* @fwnode: Device node with the connection
* @con_id: Identifier for the connection
* @data: Data for the match function
* @match: Function to check and convert the connection description
* @matches: (Optional) array of pointers to fill with matches
* @matches_len: Length of @matches
*
* Find up to @matches_len connections with unique identifier @con_id between
* @fwnode and other device nodes. @match will be used to convert the
* connection description to data the caller is expecting to be returned
* through the @matches array.
*
* If @matches is %NULL @matches_len is ignored and the total number of resolved
* matches is returned.
*
* Return: Number of matches resolved, or negative errno.
*/
int fwnode_connection_find_matches(const struct fwnode_handle *fwnode,
const char *con_id, void *data,
devcon_match_fn_t match,
void **matches, unsigned int matches_len)
{
unsigned int count_graph;
unsigned int count_ref;
if (!fwnode || !match)
return -EINVAL;
count_graph = fwnode_graph_devcon_matches(fwnode, con_id, data, match,
matches, matches_len);
if (matches) {
matches += count_graph;
matches_len -= count_graph;
}
count_ref = fwnode_devcon_matches(fwnode, con_id, data, match,
matches, matches_len);
return count_graph + count_ref;
}
EXPORT_SYMBOL_GPL(fwnode_connection_find_matches);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2001 Momchil Velikov
* Portions Copyright (C) 2001 Christoph Hellwig
* Copyright (C) 2005 SGI, Christoph Lameter
* Copyright (C) 2006 Nick Piggin
* Copyright (C) 2012 Konstantin Khlebnikov
* Copyright (C) 2016 Intel, Matthew Wilcox
* Copyright (C) 2016 Intel, Ross Zwisler
*/
#include <linux/bitmap.h>
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/cpu.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/idr.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kmemleak.h>
#include <linux/percpu.h>
#include <linux/preempt.h> /* in_interrupt() */
#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/xarray.h>
#include "radix-tree.h"
/*
* Radix tree node cache.
*/
struct kmem_cache *radix_tree_node_cachep;
/*
* The radix tree is variable-height, so an insert operation not only has
* to build the branch to its corresponding item, it also has to build the
* branch to existing items if the size has to be increased (by
* radix_tree_extend).
*
* The worst case is a zero height tree with just a single item at index 0,
* and then inserting an item at index ULONG_MAX. This requires 2 new branches
* of RADIX_TREE_MAX_PATH size to be created, with only the root node shared.
* Hence:
*/
#define RADIX_TREE_PRELOAD_SIZE (RADIX_TREE_MAX_PATH * 2 - 1)
/*
* The IDR does not have to be as high as the radix tree since it uses
* signed integers, not unsigned longs.
*/
#define IDR_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(int) - 1)
#define IDR_MAX_PATH (DIV_ROUND_UP(IDR_INDEX_BITS, \
RADIX_TREE_MAP_SHIFT))
#define IDR_PRELOAD_SIZE (IDR_MAX_PATH * 2 - 1)
/*
* Per-cpu pool of preloaded nodes
*/
DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = {
.lock = INIT_LOCAL_LOCK(lock),
};
EXPORT_PER_CPU_SYMBOL_GPL(radix_tree_preloads);
static inline struct radix_tree_node *entry_to_node(void *ptr)
{
return (void *)((unsigned long)ptr & ~RADIX_TREE_INTERNAL_NODE);
}
static inline void *node_to_entry(void *ptr)
{
return (void *)((unsigned long)ptr | RADIX_TREE_INTERNAL_NODE);
}
#define RADIX_TREE_RETRY XA_RETRY_ENTRY
static inline unsigned long
get_slot_offset(const struct radix_tree_node *parent, void __rcu **slot)
{
return parent ? slot - parent->slots : 0;
}
static unsigned int radix_tree_descend(const struct radix_tree_node *parent,
struct radix_tree_node **nodep, unsigned long index)
{
unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK;
void __rcu **entry = rcu_dereference_raw(parent->slots[offset]);
*nodep = (void *)entry;
return offset;
}
static inline gfp_t root_gfp_mask(const struct radix_tree_root *root)
{
return root->xa_flags & (__GFP_BITS_MASK & ~GFP_ZONEMASK);
}
static inline void tag_set(struct radix_tree_node *node, unsigned int tag,
int offset)
{
__set_bit(offset, node->tags[tag]);
}
static inline void tag_clear(struct radix_tree_node *node, unsigned int tag,
int offset)
{
__clear_bit(offset, node->tags[tag]);
}
static inline int tag_get(const struct radix_tree_node *node, unsigned int tag,
int offset)
{
return test_bit(offset, node->tags[tag]);
}
static inline void root_tag_set(struct radix_tree_root *root, unsigned tag)
{
root->xa_flags |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT));
}
static inline void root_tag_clear(struct radix_tree_root *root, unsigned tag)
{
root->xa_flags &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT));
}
static inline void root_tag_clear_all(struct radix_tree_root *root)
{
root->xa_flags &= (__force gfp_t)((1 << ROOT_TAG_SHIFT) - 1);
}
static inline int root_tag_get(const struct radix_tree_root *root, unsigned tag)
{
return (__force int)root->xa_flags & (1 << (tag + ROOT_TAG_SHIFT));
}
static inline unsigned root_tags_get(const struct radix_tree_root *root)
{
return (__force unsigned)root->xa_flags >> ROOT_TAG_SHIFT;
}
static inline bool is_idr(const struct radix_tree_root *root)
{
return !!(root->xa_flags & ROOT_IS_IDR);
}
/*
* Returns 1 if any slot in the node has this tag set.
* Otherwise returns 0.
*/
static inline int any_tag_set(const struct radix_tree_node *node,
unsigned int tag)
{
unsigned idx;
for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) {
if (node->tags[tag][idx])
return 1;
}
return 0;
}
static inline void all_tag_set(struct radix_tree_node *node, unsigned int tag)
{
bitmap_fill(node->tags[tag], RADIX_TREE_MAP_SIZE);
}
/**
* radix_tree_find_next_bit - find the next set bit in a memory region
*
* @node: where to begin the search
* @tag: the tag index
* @offset: the bitnumber to start searching at
*
* Unrollable variant of find_next_bit() for constant size arrays.
* Tail bits starting from size to roundup(size, BITS_PER_LONG) must be zero.
* Returns next bit offset, or size if nothing found.
*/
static __always_inline unsigned long
radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag,
unsigned long offset)
{
const unsigned long *addr = node->tags[tag];
if (offset < RADIX_TREE_MAP_SIZE) {
unsigned long tmp;
addr += offset / BITS_PER_LONG;
tmp = *addr >> (offset % BITS_PER_LONG);
if (tmp)
return __ffs(tmp) + offset;
offset = (offset + BITS_PER_LONG) & ~(BITS_PER_LONG - 1);
while (offset < RADIX_TREE_MAP_SIZE) {
tmp = *++addr;
if (tmp)
return __ffs(tmp) + offset;
offset += BITS_PER_LONG;
}
}
return RADIX_TREE_MAP_SIZE;
}
static unsigned int iter_offset(const struct radix_tree_iter *iter)
{
return iter->index & RADIX_TREE_MAP_MASK;
}
/*
* The maximum index which can be stored in a radix tree
*/
static inline unsigned long shift_maxindex(unsigned int shift)
{
return (RADIX_TREE_MAP_SIZE << shift) - 1;
}
static inline unsigned long node_maxindex(const struct radix_tree_node *node)
{
return shift_maxindex(node->shift);
}
static unsigned long next_index(unsigned long index,
const struct radix_tree_node *node,
unsigned long offset)
{
return (index & ~node_maxindex(node)) + (offset << node->shift);
}
/*
* This assumes that the caller has performed appropriate preallocation, and
* that the caller has pinned this thread of control to the current CPU.
*/
static struct radix_tree_node *
radix_tree_node_alloc(gfp_t gfp_mask, struct radix_tree_node *parent,
struct radix_tree_root *root,
unsigned int shift, unsigned int offset,
unsigned int count, unsigned int nr_values)
{
struct radix_tree_node *ret = NULL;
/*
* Preload code isn't irq safe and it doesn't make sense to use
* preloading during an interrupt anyway as all the allocations have
* to be atomic. So just do normal allocation when in interrupt.
*/
if (!gfpflags_allow_blocking(gfp_mask) && !in_interrupt()) {
struct radix_tree_preload *rtp;
/*
* Even if the caller has preloaded, try to allocate from the
* cache first for the new node to get accounted to the memory
* cgroup.
*/
ret = kmem_cache_alloc(radix_tree_node_cachep,
gfp_mask | __GFP_NOWARN);
if (ret)
goto out;
/*
* Provided the caller has preloaded here, we will always
* succeed in getting a node here (and never reach
* kmem_cache_alloc)
*/
rtp = this_cpu_ptr(&radix_tree_preloads);
if (rtp->nr) {
ret = rtp->nodes;
rtp->nodes = ret->parent;
rtp->nr--;
}
/*
* Update the allocation stack trace as this is more useful
* for debugging.
*/
kmemleak_update_trace(ret);
goto out;
}
ret = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
out:
BUG_ON(radix_tree_is_internal_node(ret)); if (ret) { ret->shift = shift;
ret->offset = offset;
ret->count = count;
ret->nr_values = nr_values;
ret->parent = parent;
ret->array = root;
}
return ret;
}
void radix_tree_node_rcu_free(struct rcu_head *head)
{
struct radix_tree_node *node =
container_of(head, struct radix_tree_node, rcu_head);
/*
* Must only free zeroed nodes into the slab. We can be left with
* non-NULL entries by radix_tree_free_nodes, so clear the entries
* and tags here.
*/
memset(node->slots, 0, sizeof(node->slots));
memset(node->tags, 0, sizeof(node->tags));
INIT_LIST_HEAD(&node->private_list);
kmem_cache_free(radix_tree_node_cachep, node);
}
static inline void
radix_tree_node_free(struct radix_tree_node *node)
{
call_rcu(&node->rcu_head, radix_tree_node_rcu_free);
}
/*
* Load up this CPU's radix_tree_node buffer with sufficient objects to
* ensure that the addition of a single element in the tree cannot fail. On
* success, return zero, with preemption disabled. On error, return -ENOMEM
* with preemption not disabled.
*
* To make use of this facility, the radix tree must be initialised without
* __GFP_DIRECT_RECLAIM being passed to INIT_RADIX_TREE().
*/
static __must_check int __radix_tree_preload(gfp_t gfp_mask, unsigned nr)
{
struct radix_tree_preload *rtp;
struct radix_tree_node *node;
int ret = -ENOMEM;
/*
* Nodes preloaded by one cgroup can be used by another cgroup, so
* they should never be accounted to any particular memory cgroup.
*/
gfp_mask &= ~__GFP_ACCOUNT; local_lock(&radix_tree_preloads.lock);
rtp = this_cpu_ptr(&radix_tree_preloads);
while (rtp->nr < nr) { local_unlock(&radix_tree_preloads.lock); node = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
if (node == NULL)
goto out;
local_lock(&radix_tree_preloads.lock);
rtp = this_cpu_ptr(&radix_tree_preloads);
if (rtp->nr < nr) {
node->parent = rtp->nodes;
rtp->nodes = node;
rtp->nr++;
} else {
kmem_cache_free(radix_tree_node_cachep, node);
}
}
ret = 0;
out:
return ret;
}
/*
* Load up this CPU's radix_tree_node buffer with sufficient objects to
* ensure that the addition of a single element in the tree cannot fail. On
* success, return zero, with preemption disabled. On error, return -ENOMEM
* with preemption not disabled.
*
* To make use of this facility, the radix tree must be initialised without
* __GFP_DIRECT_RECLAIM being passed to INIT_RADIX_TREE().
*/
int radix_tree_preload(gfp_t gfp_mask)
{
/* Warn on non-sensical use... */
WARN_ON_ONCE(!gfpflags_allow_blocking(gfp_mask));
return __radix_tree_preload(gfp_mask, RADIX_TREE_PRELOAD_SIZE);
}
EXPORT_SYMBOL(radix_tree_preload);
/*
* The same as above function, except we don't guarantee preloading happens.
* We do it, if we decide it helps. On success, return zero with preemption
* disabled. On error, return -ENOMEM with preemption not disabled.
*/
int radix_tree_maybe_preload(gfp_t gfp_mask)
{
if (gfpflags_allow_blocking(gfp_mask))
return __radix_tree_preload(gfp_mask, RADIX_TREE_PRELOAD_SIZE);
/* Preloading doesn't help anything with this gfp mask, skip it */
local_lock(&radix_tree_preloads.lock);
return 0;
}
EXPORT_SYMBOL(radix_tree_maybe_preload);
static unsigned radix_tree_load_root(const struct radix_tree_root *root,
struct radix_tree_node **nodep, unsigned long *maxindex)
{
struct radix_tree_node *node = rcu_dereference_raw(root->xa_head);
*nodep = node;
if (likely(radix_tree_is_internal_node(node))) { node = entry_to_node(node); *maxindex = node_maxindex(node);
return node->shift + RADIX_TREE_MAP_SHIFT;
}
*maxindex = 0;
return 0;
}
/*
* Extend a radix tree so it can store key @index.
*/
static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp,
unsigned long index, unsigned int shift)
{
void *entry;
unsigned int maxshift;
int tag;
/* Figure out what the shift should be. */
maxshift = shift;
while (index > shift_maxindex(maxshift))
maxshift += RADIX_TREE_MAP_SHIFT;
entry = rcu_dereference_raw(root->xa_head);
if (!entry && (!is_idr(root) || root_tag_get(root, IDR_FREE)))
goto out;
do {
struct radix_tree_node *node = radix_tree_node_alloc(gfp, NULL,
root, shift, 0, 1, 0);
if (!node)
return -ENOMEM;
if (is_idr(root)) {
all_tag_set(node, IDR_FREE);
if (!root_tag_get(root, IDR_FREE)) {
tag_clear(node, IDR_FREE, 0);
root_tag_set(root, IDR_FREE);
}
} else {
/* Propagate the aggregated tag info to the new child */
for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
if (root_tag_get(root, tag))
tag_set(node, tag, 0);
}
}
BUG_ON(shift > BITS_PER_LONG);
if (radix_tree_is_internal_node(entry)) {
entry_to_node(entry)->parent = node;
} else if (xa_is_value(entry)) {
/* Moving a value entry root->xa_head to a node */
node->nr_values = 1;
}
/*
* entry was already in the radix tree, so we do not need
* rcu_assign_pointer here
*/
node->slots[0] = (void __rcu *)entry;
entry = node_to_entry(node);
rcu_assign_pointer(root->xa_head, entry);
shift += RADIX_TREE_MAP_SHIFT;
} while (shift <= maxshift);
out:
return maxshift + RADIX_TREE_MAP_SHIFT;
}
/**
* radix_tree_shrink - shrink radix tree to minimum height
* @root: radix tree root
*/
static inline bool radix_tree_shrink(struct radix_tree_root *root)
{
bool shrunk = false;
for (;;) {
struct radix_tree_node *node = rcu_dereference_raw(root->xa_head);
struct radix_tree_node *child;
if (!radix_tree_is_internal_node(node))
break;
node = entry_to_node(node);
/*
* The candidate node has more than one child, or its child
* is not at the leftmost slot, we cannot shrink.
*/
if (node->count != 1)
break;
child = rcu_dereference_raw(node->slots[0]);
if (!child)
break;
/*
* For an IDR, we must not shrink entry 0 into the root in
* case somebody calls idr_replace() with a pointer that
* appears to be an internal entry
*/
if (!node->shift && is_idr(root))
break;
if (radix_tree_is_internal_node(child)) entry_to_node(child)->parent = NULL;
/*
* We don't need rcu_assign_pointer(), since we are simply
* moving the node from one part of the tree to another: if it
* was safe to dereference the old pointer to it
* (node->slots[0]), it will be safe to dereference the new
* one (root->xa_head) as far as dependent read barriers go.
*/
root->xa_head = (void __rcu *)child; if (is_idr(root) && !tag_get(node, IDR_FREE, 0)) root_tag_clear(root, IDR_FREE);
/*
* We have a dilemma here. The node's slot[0] must not be
* NULLed in case there are concurrent lookups expecting to
* find the item. However if this was a bottom-level node,
* then it may be subject to the slot pointer being visible
* to callers dereferencing it. If item corresponding to
* slot[0] is subsequently deleted, these callers would expect
* their slot to become empty sooner or later.
*
* For example, lockless pagecache will look up a slot, deref
* the page pointer, and if the page has 0 refcount it means it
* was concurrently deleted from pagecache so try the deref
* again. Fortunately there is already a requirement for logic
* to retry the entire slot lookup -- the indirect pointer
* problem (replacing direct root node with an indirect pointer
* also results in a stale slot). So tag the slot as indirect
* to force callers to retry.
*/
node->count = 0;
if (!radix_tree_is_internal_node(child)) {
node->slots[0] = (void __rcu *)RADIX_TREE_RETRY;
}
WARN_ON_ONCE(!list_empty(&node->private_list)); radix_tree_node_free(node);
shrunk = true;
}
return shrunk;
}
static bool delete_node(struct radix_tree_root *root,
struct radix_tree_node *node)
{
bool deleted = false;
do {
struct radix_tree_node *parent;
if (node->count) { if (node_to_entry(node) ==
rcu_dereference_raw(root->xa_head))
deleted |= radix_tree_shrink(root);
return deleted;
}
parent = node->parent;
if (parent) {
parent->slots[node->offset] = NULL;
parent->count--;
} else {
/*
* Shouldn't the tags already have all been cleared
* by the caller?
*/
if (!is_idr(root)) root_tag_clear_all(root); root->xa_head = NULL;
}
WARN_ON_ONCE(!list_empty(&node->private_list)); radix_tree_node_free(node);
deleted = true;
node = parent;
} while (node);
return deleted;
}
/**
* __radix_tree_create - create a slot in a radix tree
* @root: radix tree root
* @index: index key
* @nodep: returns node
* @slotp: returns slot
*
* Create, if necessary, and return the node and slot for an item
* at position @index in the radix tree @root.
*
* Until there is more than one item in the tree, no nodes are
* allocated and @root->xa_head is used as a direct slot instead of
* pointing to a node, in which case *@nodep will be NULL.
*
* Returns -ENOMEM, or 0 for success.
*/
static int __radix_tree_create(struct radix_tree_root *root,
unsigned long index, struct radix_tree_node **nodep,
void __rcu ***slotp)
{
struct radix_tree_node *node = NULL, *child;
void __rcu **slot = (void __rcu **)&root->xa_head;
unsigned long maxindex;
unsigned int shift, offset = 0;
unsigned long max = index;
gfp_t gfp = root_gfp_mask(root);
shift = radix_tree_load_root(root, &child, &maxindex);
/* Make sure the tree is high enough. */
if (max > maxindex) {
int error = radix_tree_extend(root, gfp, max, shift);
if (error < 0)
return error;
shift = error;
child = rcu_dereference_raw(root->xa_head);
}
while (shift > 0) {
shift -= RADIX_TREE_MAP_SHIFT;
if (child == NULL) {
/* Have to add a child node. */
child = radix_tree_node_alloc(gfp, node, root, shift,
offset, 0, 0);
if (!child)
return -ENOMEM;
rcu_assign_pointer(*slot, node_to_entry(child));
if (node)
node->count++;
} else if (!radix_tree_is_internal_node(child))
break;
/* Go a level down */
node = entry_to_node(child);
offset = radix_tree_descend(node, &child, index);
slot = &node->slots[offset];
}
if (nodep)
*nodep = node;
if (slotp)
*slotp = slot;
return 0;
}
/*
* Free any nodes below this node. The tree is presumed to not need
* shrinking, and any user data in the tree is presumed to not need a
* destructor called on it. If we need to add a destructor, we can
* add that functionality later. Note that we may not clear tags or
* slots from the tree as an RCU walker may still have a pointer into
* this subtree. We could replace the entries with RADIX_TREE_RETRY,
* but we'll still have to clear those in rcu_free.
*/
static void radix_tree_free_nodes(struct radix_tree_node *node)
{
unsigned offset = 0;
struct radix_tree_node *child = entry_to_node(node);
for (;;) {
void *entry = rcu_dereference_raw(child->slots[offset]); if (xa_is_node(entry) && child->shift) { child = entry_to_node(entry);
offset = 0;
continue;
}
offset++; while (offset == RADIX_TREE_MAP_SIZE) {
struct radix_tree_node *old = child;
offset = child->offset + 1;
child = child->parent;
WARN_ON_ONCE(!list_empty(&old->private_list)); radix_tree_node_free(old);
if (old == entry_to_node(node))
return;
}
}
}
static inline int insert_entries(struct radix_tree_node *node,
void __rcu **slot, void *item)
{
if (*slot)
return -EEXIST;
rcu_assign_pointer(*slot, item);
if (node) {
node->count++;
if (xa_is_value(item))
node->nr_values++;
}
return 1;
}
/**
* radix_tree_insert - insert into a radix tree
* @root: radix tree root
* @index: index key
* @item: item to insert
*
* Insert an item into the radix tree at position @index.
*/
int radix_tree_insert(struct radix_tree_root *root, unsigned long index,
void *item)
{
struct radix_tree_node *node;
void __rcu **slot;
int error;
BUG_ON(radix_tree_is_internal_node(item));
error = __radix_tree_create(root, index, &node, &slot);
if (error)
return error;
error = insert_entries(node, slot, item);
if (error < 0)
return error;
if (node) {
unsigned offset = get_slot_offset(node, slot);
BUG_ON(tag_get(node, 0, offset));
BUG_ON(tag_get(node, 1, offset));
BUG_ON(tag_get(node, 2, offset));
} else {
BUG_ON(root_tags_get(root));
}
return 0;
}
EXPORT_SYMBOL(radix_tree_insert);
/**
* __radix_tree_lookup - lookup an item in a radix tree
* @root: radix tree root
* @index: index key
* @nodep: returns node
* @slotp: returns slot
*
* Lookup and return the item at position @index in the radix
* tree @root.
*
* Until there is more than one item in the tree, no nodes are
* allocated and @root->xa_head is used as a direct slot instead of
* pointing to a node, in which case *@nodep will be NULL.
*/
void *__radix_tree_lookup(const struct radix_tree_root *root,
unsigned long index, struct radix_tree_node **nodep,
void __rcu ***slotp)
{
struct radix_tree_node *node, *parent;
unsigned long maxindex;
void __rcu **slot;
restart:
parent = NULL;
slot = (void __rcu **)&root->xa_head; radix_tree_load_root(root, &node, &maxindex); if (index > maxindex)
return NULL;
while (radix_tree_is_internal_node(node)) {
unsigned offset;
parent = entry_to_node(node); offset = radix_tree_descend(parent, &node, index);
slot = parent->slots + offset;
if (node == RADIX_TREE_RETRY)
goto restart;
if (parent->shift == 0)
break;
}
if (nodep) *nodep = parent; if (slotp) *slotp = slot;
return node;
}
/**
* radix_tree_lookup_slot - lookup a slot in a radix tree
* @root: radix tree root
* @index: index key
*
* Returns: the slot corresponding to the position @index in the
* radix tree @root. This is useful for update-if-exists operations.
*
* This function can be called under rcu_read_lock iff the slot is not
* modified by radix_tree_replace_slot, otherwise it must be called
* exclusive from other writers. Any dereference of the slot must be done
* using radix_tree_deref_slot.
*/
void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *root,
unsigned long index)
{
void __rcu **slot;
if (!__radix_tree_lookup(root, index, NULL, &slot))
return NULL;
return slot;
}
EXPORT_SYMBOL(radix_tree_lookup_slot);
/**
* radix_tree_lookup - perform lookup operation on a radix tree
* @root: radix tree root
* @index: index key
*
* Lookup the item at the position @index in the radix tree @root.
*
* This function can be called under rcu_read_lock, however the caller
* must manage lifetimes of leaf nodes (eg. RCU may also be used to free
* them safely). No RCU barriers are required to access or modify the
* returned item, however.
*/
void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index)
{
return __radix_tree_lookup(root, index, NULL, NULL);
}
EXPORT_SYMBOL(radix_tree_lookup);
static void replace_slot(void __rcu **slot, void *item,
struct radix_tree_node *node, int count, int values)
{
if (node && (count || values)) { node->count += count;
node->nr_values += values;
}
rcu_assign_pointer(*slot, item);
}
static bool node_tag_get(const struct radix_tree_root *root,
const struct radix_tree_node *node,
unsigned int tag, unsigned int offset)
{
if (node)
return tag_get(node, tag, offset);
return root_tag_get(root, tag);
}
/*
* IDR users want to be able to store NULL in the tree, so if the slot isn't
* free, don't adjust the count, even if it's transitioning between NULL and
* non-NULL. For the IDA, we mark slots as being IDR_FREE while they still
* have empty bits, but it only stores NULL in slots when they're being
* deleted.
*/
static int calculate_count(struct radix_tree_root *root,
struct radix_tree_node *node, void __rcu **slot,
void *item, void *old)
{
if (is_idr(root)) {
unsigned offset = get_slot_offset(node, slot); bool free = node_tag_get(root, node, IDR_FREE, offset);
if (!free)
return 0;
if (!old)
return 1;
}
return !!item - !!old;
}
/**
* __radix_tree_replace - replace item in a slot
* @root: radix tree root
* @node: pointer to tree node
* @slot: pointer to slot in @node
* @item: new item to store in the slot.
*
* For use with __radix_tree_lookup(). Caller must hold tree write locked
* across slot lookup and replacement.
*/
void __radix_tree_replace(struct radix_tree_root *root,
struct radix_tree_node *node,
void __rcu **slot, void *item)
{
void *old = rcu_dereference_raw(*slot);
int values = !!xa_is_value(item) - !!xa_is_value(old);
int count = calculate_count(root, node, slot, item, old);
/*
* This function supports replacing value entries and
* deleting entries, but that needs accounting against the
* node unless the slot is root->xa_head.
*/
WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->xa_head) &&
(count || values));
replace_slot(slot, item, node, count, values);
if (!node)
return;
delete_node(root, node);
}
/**
* radix_tree_replace_slot - replace item in a slot
* @root: radix tree root
* @slot: pointer to slot
* @item: new item to store in the slot.
*
* For use with radix_tree_lookup_slot() and
* radix_tree_gang_lookup_tag_slot(). Caller must hold tree write locked
* across slot lookup and replacement.
*
* NOTE: This cannot be used to switch between non-entries (empty slots),
* regular entries, and value entries, as that requires accounting
* inside the radix tree node. When switching from one type of entry or
* deleting, use __radix_tree_lookup() and __radix_tree_replace() or
* radix_tree_iter_replace().
*/
void radix_tree_replace_slot(struct radix_tree_root *root,
void __rcu **slot, void *item)
{
__radix_tree_replace(root, NULL, slot, item);
}
EXPORT_SYMBOL(radix_tree_replace_slot);
/**
* radix_tree_iter_replace - replace item in a slot
* @root: radix tree root
* @iter: iterator state
* @slot: pointer to slot
* @item: new item to store in the slot.
*
* For use with radix_tree_for_each_slot().
* Caller must hold tree write locked.
*/
void radix_tree_iter_replace(struct radix_tree_root *root,
const struct radix_tree_iter *iter,
void __rcu **slot, void *item)
{
__radix_tree_replace(root, iter->node, slot, item);
}
static void node_tag_set(struct radix_tree_root *root,
struct radix_tree_node *node,
unsigned int tag, unsigned int offset)
{
while (node) {
if (tag_get(node, tag, offset))
return;
tag_set(node, tag, offset);
offset = node->offset;
node = node->parent;
}
if (!root_tag_get(root, tag)) root_tag_set(root, tag);
}
/**
* radix_tree_tag_set - set a tag on a radix tree node
* @root: radix tree root
* @index: index key
* @tag: tag index
*
* Set the search tag (which must be < RADIX_TREE_MAX_TAGS)
* corresponding to @index in the radix tree. From
* the root all the way down to the leaf node.
*
* Returns the address of the tagged item. Setting a tag on a not-present
* item is a bug.
*/
void *radix_tree_tag_set(struct radix_tree_root *root,
unsigned long index, unsigned int tag)
{
struct radix_tree_node *node, *parent;
unsigned long maxindex;
radix_tree_load_root(root, &node, &maxindex);
BUG_ON(index > maxindex);
while (radix_tree_is_internal_node(node)) {
unsigned offset;
parent = entry_to_node(node);
offset = radix_tree_descend(parent, &node, index);
BUG_ON(!node);
if (!tag_get(parent, tag, offset))
tag_set(parent, tag, offset);
}
/* set the root's tag bit */
if (!root_tag_get(root, tag))
root_tag_set(root, tag);
return node;
}
EXPORT_SYMBOL(radix_tree_tag_set);
static void node_tag_clear(struct radix_tree_root *root,
struct radix_tree_node *node,
unsigned int tag, unsigned int offset)
{
while (node) { if (!tag_get(node, tag, offset))
return;
tag_clear(node, tag, offset);
if (any_tag_set(node, tag))
return;
offset = node->offset;
node = node->parent;
}
/* clear the root's tag bit */
if (root_tag_get(root, tag)) root_tag_clear(root, tag);
}
/**
* radix_tree_tag_clear - clear a tag on a radix tree node
* @root: radix tree root
* @index: index key
* @tag: tag index
*
* Clear the search tag (which must be < RADIX_TREE_MAX_TAGS)
* corresponding to @index in the radix tree. If this causes
* the leaf node to have no tags set then clear the tag in the
* next-to-leaf node, etc.
*
* Returns the address of the tagged item on success, else NULL. ie:
* has the same return value and semantics as radix_tree_lookup().
*/
void *radix_tree_tag_clear(struct radix_tree_root *root,
unsigned long index, unsigned int tag)
{
struct radix_tree_node *node, *parent;
unsigned long maxindex;
int offset = 0;
radix_tree_load_root(root, &node, &maxindex);
if (index > maxindex)
return NULL;
parent = NULL;
while (radix_tree_is_internal_node(node)) {
parent = entry_to_node(node);
offset = radix_tree_descend(parent, &node, index);
}
if (node)
node_tag_clear(root, parent, tag, offset);
return node;
}
EXPORT_SYMBOL(radix_tree_tag_clear);
/**
* radix_tree_iter_tag_clear - clear a tag on the current iterator entry
* @root: radix tree root
* @iter: iterator state
* @tag: tag to clear
*/
void radix_tree_iter_tag_clear(struct radix_tree_root *root,
const struct radix_tree_iter *iter, unsigned int tag)
{
node_tag_clear(root, iter->node, tag, iter_offset(iter));
}
/**
* radix_tree_tag_get - get a tag on a radix tree node
* @root: radix tree root
* @index: index key
* @tag: tag index (< RADIX_TREE_MAX_TAGS)
*
* Return values:
*
* 0: tag not present or not set
* 1: tag set
*
* Note that the return value of this function may not be relied on, even if
* the RCU lock is held, unless tag modification and node deletion are excluded
* from concurrency.
*/
int radix_tree_tag_get(const struct radix_tree_root *root,
unsigned long index, unsigned int tag)
{
struct radix_tree_node *node, *parent;
unsigned long maxindex;
if (!root_tag_get(root, tag)) return 0; radix_tree_load_root(root, &node, &maxindex); if (index > maxindex)
return 0;
while (radix_tree_is_internal_node(node)) {
unsigned offset;
parent = entry_to_node(node); offset = radix_tree_descend(parent, &node, index);
if (!tag_get(parent, tag, offset))
return 0;
if (node == RADIX_TREE_RETRY)
break;
}
return 1;
}
EXPORT_SYMBOL(radix_tree_tag_get);
/* Construct iter->tags bit-mask from node->tags[tag] array */
static void set_iter_tags(struct radix_tree_iter *iter,
struct radix_tree_node *node, unsigned offset,
unsigned tag)
{
unsigned tag_long = offset / BITS_PER_LONG;
unsigned tag_bit = offset % BITS_PER_LONG;
if (!node) {
iter->tags = 1;
return;
}
iter->tags = node->tags[tag][tag_long] >> tag_bit;
/* This never happens if RADIX_TREE_TAG_LONGS == 1 */
if (tag_long < RADIX_TREE_TAG_LONGS - 1) {
/* Pick tags from next element */
if (tag_bit)
iter->tags |= node->tags[tag][tag_long + 1] <<
(BITS_PER_LONG - tag_bit);
/* Clip chunk size, here only BITS_PER_LONG tags */
iter->next_index = __radix_tree_iter_add(iter, BITS_PER_LONG);
}
}
void __rcu **radix_tree_iter_resume(void __rcu **slot,
struct radix_tree_iter *iter)
{
iter->index = __radix_tree_iter_add(iter, 1);
iter->next_index = iter->index;
iter->tags = 0;
return NULL;
}
EXPORT_SYMBOL(radix_tree_iter_resume);
/**
* radix_tree_next_chunk - find next chunk of slots for iteration
*
* @root: radix tree root
* @iter: iterator state
* @flags: RADIX_TREE_ITER_* flags and tag index
* Returns: pointer to chunk first slot, or NULL if iteration is over
*/
void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root,
struct radix_tree_iter *iter, unsigned flags)
{
unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
struct radix_tree_node *node, *child;
unsigned long index, offset, maxindex;
if ((flags & RADIX_TREE_ITER_TAGGED) && !root_tag_get(root, tag))
return NULL;
/*
* Catch next_index overflow after ~0UL. iter->index never overflows
* during iterating; it can be zero only at the beginning.
* And we cannot overflow iter->next_index in a single step,
* because RADIX_TREE_MAP_SHIFT < BITS_PER_LONG.
*
* This condition also used by radix_tree_next_slot() to stop
* contiguous iterating, and forbid switching to the next chunk.
*/
index = iter->next_index; if (!index && iter->index)
return NULL;
restart:
radix_tree_load_root(root, &child, &maxindex); if (index > maxindex)
return NULL;
if (!child)
return NULL;
if (!radix_tree_is_internal_node(child)) {
/* Single-slot tree */
iter->index = index;
iter->next_index = maxindex + 1;
iter->tags = 1;
iter->node = NULL;
return (void __rcu **)&root->xa_head;
}
do {
node = entry_to_node(child); offset = radix_tree_descend(node, &child, index); if ((flags & RADIX_TREE_ITER_TAGGED) ? !tag_get(node, tag, offset) : !child) {
/* Hole detected */
if (flags & RADIX_TREE_ITER_CONTIG)
return NULL;
if (flags & RADIX_TREE_ITER_TAGGED)
offset = radix_tree_find_next_bit(node, tag,
offset + 1);
else
while (++offset < RADIX_TREE_MAP_SIZE) { void *slot = rcu_dereference_raw(
node->slots[offset]);
if (slot)
break;
}
index &= ~node_maxindex(node); index += offset << node->shift;
/* Overflow after ~0UL */
if (!index)
return NULL;
if (offset == RADIX_TREE_MAP_SIZE)
goto restart;
child = rcu_dereference_raw(node->slots[offset]);
}
if (!child)
goto restart;
if (child == RADIX_TREE_RETRY)
break;
} while (node->shift && radix_tree_is_internal_node(child));
/* Update the iterator state */
iter->index = (index &~ node_maxindex(node)) | offset; iter->next_index = (index | node_maxindex(node)) + 1;
iter->node = node;
if (flags & RADIX_TREE_ITER_TAGGED)
set_iter_tags(iter, node, offset, tag); return node->slots + offset;
}
EXPORT_SYMBOL(radix_tree_next_chunk);
/**
* radix_tree_gang_lookup - perform multiple lookup on a radix tree
* @root: radix tree root
* @results: where the results of the lookup are placed
* @first_index: start the lookup from this key
* @max_items: place up to this many items at *results
*
* Performs an index-ascending scan of the tree for present items. Places
* them at *@results and returns the number of items which were placed at
* *@results.
*
* The implementation is naive.
*
* Like radix_tree_lookup, radix_tree_gang_lookup may be called under
* rcu_read_lock. In this case, rather than the returned results being
* an atomic snapshot of the tree at a single point in time, the
* semantics of an RCU protected gang lookup are as though multiple
* radix_tree_lookups have been issued in individual locks, and results
* stored in 'results'.
*/
unsigned int
radix_tree_gang_lookup(const struct radix_tree_root *root, void **results,
unsigned long first_index, unsigned int max_items)
{
struct radix_tree_iter iter;
void __rcu **slot;
unsigned int ret = 0;
if (unlikely(!max_items))
return 0;
radix_tree_for_each_slot(slot, root, &iter, first_index) {
results[ret] = rcu_dereference_raw(*slot);
if (!results[ret])
continue;
if (radix_tree_is_internal_node(results[ret])) {
slot = radix_tree_iter_retry(&iter);
continue;
}
if (++ret == max_items)
break;
}
return ret;
}
EXPORT_SYMBOL(radix_tree_gang_lookup);
/**
* radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
* based on a tag
* @root: radix tree root
* @results: where the results of the lookup are placed
* @first_index: start the lookup from this key
* @max_items: place up to this many items at *results
* @tag: the tag index (< RADIX_TREE_MAX_TAGS)
*
* Performs an index-ascending scan of the tree for present items which
* have the tag indexed by @tag set. Places the items at *@results and
* returns the number of items which were placed at *@results.
*/
unsigned int
radix_tree_gang_lookup_tag(const struct radix_tree_root *root, void **results,
unsigned long first_index, unsigned int max_items,
unsigned int tag)
{
struct radix_tree_iter iter;
void __rcu **slot;
unsigned int ret = 0;
if (unlikely(!max_items))
return 0;
radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
results[ret] = rcu_dereference_raw(*slot);
if (!results[ret])
continue;
if (radix_tree_is_internal_node(results[ret])) {
slot = radix_tree_iter_retry(&iter);
continue;
}
if (++ret == max_items)
break;
}
return ret;
}
EXPORT_SYMBOL(radix_tree_gang_lookup_tag);
/**
* radix_tree_gang_lookup_tag_slot - perform multiple slot lookup on a
* radix tree based on a tag
* @root: radix tree root
* @results: where the results of the lookup are placed
* @first_index: start the lookup from this key
* @max_items: place up to this many items at *results
* @tag: the tag index (< RADIX_TREE_MAX_TAGS)
*
* Performs an index-ascending scan of the tree for present items which
* have the tag indexed by @tag set. Places the slots at *@results and
* returns the number of slots which were placed at *@results.
*/
unsigned int
radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *root,
void __rcu ***results, unsigned long first_index,
unsigned int max_items, unsigned int tag)
{
struct radix_tree_iter iter;
void __rcu **slot;
unsigned int ret = 0;
if (unlikely(!max_items))
return 0;
radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
results[ret] = slot;
if (++ret == max_items)
break;
}
return ret;
}
EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot);
static bool __radix_tree_delete(struct radix_tree_root *root,
struct radix_tree_node *node, void __rcu **slot)
{
void *old = rcu_dereference_raw(*slot);
int values = xa_is_value(old) ? -1 : 0;
unsigned offset = get_slot_offset(node, slot);
int tag;
if (is_idr(root)) node_tag_set(root, node, IDR_FREE, offset);
else
for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
node_tag_clear(root, node, tag, offset); replace_slot(slot, NULL, node, -1, values); return node && delete_node(root, node);
}
/**
* radix_tree_iter_delete - delete the entry at this iterator position
* @root: radix tree root
* @iter: iterator state
* @slot: pointer to slot
*
* Delete the entry at the position currently pointed to by the iterator.
* This may result in the current node being freed; if it is, the iterator
* is advanced so that it will not reference the freed memory. This
* function may be called without any locking if there are no other threads
* which can access this tree.
*/
void radix_tree_iter_delete(struct radix_tree_root *root,
struct radix_tree_iter *iter, void __rcu **slot)
{
if (__radix_tree_delete(root, iter->node, slot))
iter->index = iter->next_index;
}
EXPORT_SYMBOL(radix_tree_iter_delete);
/**
* radix_tree_delete_item - delete an item from a radix tree
* @root: radix tree root
* @index: index key
* @item: expected item
*
* Remove @item at @index from the radix tree rooted at @root.
*
* Return: the deleted entry, or %NULL if it was not present
* or the entry at the given @index was not @item.
*/
void *radix_tree_delete_item(struct radix_tree_root *root,
unsigned long index, void *item)
{
struct radix_tree_node *node = NULL;
void __rcu **slot = NULL;
void *entry;
entry = __radix_tree_lookup(root, index, &node, &slot);
if (!slot)
return NULL;
if (!entry && (!is_idr(root) || node_tag_get(root, node, IDR_FREE, get_slot_offset(node, slot))))
return NULL;
if (item && entry != item)
return NULL;
__radix_tree_delete(root, node, slot); return entry;
}
EXPORT_SYMBOL(radix_tree_delete_item);
/**
* radix_tree_delete - delete an entry from a radix tree
* @root: radix tree root
* @index: index key
*
* Remove the entry at @index from the radix tree rooted at @root.
*
* Return: The deleted entry, or %NULL if it was not present.
*/
void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
{
return radix_tree_delete_item(root, index, NULL);
}
EXPORT_SYMBOL(radix_tree_delete);
/**
* radix_tree_tagged - test whether any items in the tree are tagged
* @root: radix tree root
* @tag: tag to test
*/
int radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag)
{
return root_tag_get(root, tag);
}
EXPORT_SYMBOL(radix_tree_tagged);
/**
* idr_preload - preload for idr_alloc()
* @gfp_mask: allocation mask to use for preloading
*
* Preallocate memory to use for the next call to idr_alloc(). This function
* returns with preemption disabled. It will be enabled by idr_preload_end().
*/
void idr_preload(gfp_t gfp_mask)
{
if (__radix_tree_preload(gfp_mask, IDR_PRELOAD_SIZE)) local_lock(&radix_tree_preloads.lock);
}
EXPORT_SYMBOL(idr_preload);
void __rcu **idr_get_free(struct radix_tree_root *root,
struct radix_tree_iter *iter, gfp_t gfp,
unsigned long max)
{
struct radix_tree_node *node = NULL, *child;
void __rcu **slot = (void __rcu **)&root->xa_head;
unsigned long maxindex, start = iter->next_index;
unsigned int shift, offset = 0;
grow:
shift = radix_tree_load_root(root, &child, &maxindex); if (!radix_tree_tagged(root, IDR_FREE)) start = max(start, maxindex + 1); if (start > max)
return ERR_PTR(-ENOSPC);
if (start > maxindex) { int error = radix_tree_extend(root, gfp, start, shift);
if (error < 0)
return ERR_PTR(error); shift = error;
child = rcu_dereference_raw(root->xa_head);
}
if (start == 0 && shift == 0) shift = RADIX_TREE_MAP_SHIFT; while (shift) { shift -= RADIX_TREE_MAP_SHIFT;
if (child == NULL) {
/* Have to add a child node. */
child = radix_tree_node_alloc(gfp, node, root, shift,
offset, 0, 0);
if (!child)
return ERR_PTR(-ENOMEM);
all_tag_set(child, IDR_FREE);
rcu_assign_pointer(*slot, node_to_entry(child));
if (node)
node->count++; } else if (!radix_tree_is_internal_node(child))
break;
node = entry_to_node(child); offset = radix_tree_descend(node, &child, start);
if (!tag_get(node, IDR_FREE, offset)) {
offset = radix_tree_find_next_bit(node, IDR_FREE, offset + 1); start = next_index(start, node, offset); if (start > max || start == 0)
return ERR_PTR(-ENOSPC);
while (offset == RADIX_TREE_MAP_SIZE) { offset = node->offset + 1;
node = node->parent;
if (!node)
goto grow;
shift = node->shift;
}
child = rcu_dereference_raw(node->slots[offset]);
}
slot = &node->slots[offset];
}
iter->index = start;
if (node)
iter->next_index = 1 + min(max, (start | node_maxindex(node)));
else
iter->next_index = 1;
iter->node = node;
set_iter_tags(iter, node, offset, IDR_FREE);
return slot;
}
/**
* idr_destroy - release all internal memory from an IDR
* @idr: idr handle
*
* After this function is called, the IDR is empty, and may be reused or
* the data structure containing it may be freed.
*
* A typical clean-up sequence for objects stored in an idr tree will use
* idr_for_each() to free all objects, if necessary, then idr_destroy() to
* free the memory used to keep track of those objects.
*/
void idr_destroy(struct idr *idr)
{
struct radix_tree_node *node = rcu_dereference_raw(idr->idr_rt.xa_head);
if (radix_tree_is_internal_node(node))
radix_tree_free_nodes(node); idr->idr_rt.xa_head = NULL;
root_tag_set(&idr->idr_rt, IDR_FREE);
}
EXPORT_SYMBOL(idr_destroy);
static void
radix_tree_node_ctor(void *arg)
{
struct radix_tree_node *node = arg;
memset(node, 0, sizeof(*node));
INIT_LIST_HEAD(&node->private_list);
}
static int radix_tree_cpu_dead(unsigned int cpu)
{
struct radix_tree_preload *rtp;
struct radix_tree_node *node;
/* Free per-cpu pool of preloaded nodes */
rtp = &per_cpu(radix_tree_preloads, cpu);
while (rtp->nr) {
node = rtp->nodes;
rtp->nodes = node->parent;
kmem_cache_free(radix_tree_node_cachep, node);
rtp->nr--;
}
return 0;
}
void __init radix_tree_init(void)
{
int ret;
BUILD_BUG_ON(RADIX_TREE_MAX_TAGS + __GFP_BITS_SHIFT > 32);
BUILD_BUG_ON(ROOT_IS_IDR & ~GFP_ZONEMASK);
BUILD_BUG_ON(XA_CHUNK_SIZE > 255);
radix_tree_node_cachep = kmem_cache_create("radix_tree_node",
sizeof(struct radix_tree_node), 0,
SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
radix_tree_node_ctor);
ret = cpuhp_setup_state_nocalls(CPUHP_RADIX_DEAD, "lib/radix:dead",
NULL, radix_tree_cpu_dead);
WARN_ON(ret < 0);
}
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/bitmap.h>
#include <linux/bug.h>
#include <linux/export.h>
#include <linux/idr.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/xarray.h>
/**
* idr_alloc_u32() - Allocate an ID.
* @idr: IDR handle.
* @ptr: Pointer to be associated with the new ID.
* @nextid: Pointer to an ID.
* @max: The maximum ID to allocate (inclusive).
* @gfp: Memory allocation flags.
*
* Allocates an unused ID in the range specified by @nextid and @max.
* Note that @max is inclusive whereas the @end parameter to idr_alloc()
* is exclusive. The new ID is assigned to @nextid before the pointer
* is inserted into the IDR, so if @nextid points into the object pointed
* to by @ptr, a concurrent lookup will not find an uninitialised ID.
*
* The caller should provide their own locking to ensure that two
* concurrent modifications to the IDR are not possible. Read-only
* accesses to the IDR may be done under the RCU read lock or may
* exclude simultaneous writers.
*
* Return: 0 if an ID was allocated, -ENOMEM if memory allocation failed,
* or -ENOSPC if no free IDs could be found. If an error occurred,
* @nextid is unchanged.
*/
int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid,
unsigned long max, gfp_t gfp)
{
struct radix_tree_iter iter;
void __rcu **slot;
unsigned int base = idr->idr_base;
unsigned int id = *nextid;
if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR)))
idr->idr_rt.xa_flags |= IDR_RT_MARKER;
id = (id < base) ? 0 : id - base; radix_tree_iter_init(&iter, id);
slot = idr_get_free(&idr->idr_rt, &iter, gfp, max - base);
if (IS_ERR(slot))
return PTR_ERR(slot); *nextid = iter.index + base;
/* there is a memory barrier inside radix_tree_iter_replace() */
radix_tree_iter_replace(&idr->idr_rt, &iter, slot, ptr);
radix_tree_iter_tag_clear(&idr->idr_rt, &iter, IDR_FREE);
return 0;
}
EXPORT_SYMBOL_GPL(idr_alloc_u32);
/**
* idr_alloc() - Allocate an ID.
* @idr: IDR handle.
* @ptr: Pointer to be associated with the new ID.
* @start: The minimum ID (inclusive).
* @end: The maximum ID (exclusive).
* @gfp: Memory allocation flags.
*
* Allocates an unused ID in the range specified by @start and @end. If
* @end is <= 0, it is treated as one larger than %INT_MAX. This allows
* callers to use @start + N as @end as long as N is within integer range.
*
* The caller should provide their own locking to ensure that two
* concurrent modifications to the IDR are not possible. Read-only
* accesses to the IDR may be done under the RCU read lock or may
* exclude simultaneous writers.
*
* Return: The newly allocated ID, -ENOMEM if memory allocation failed,
* or -ENOSPC if no free IDs could be found.
*/
int idr_alloc(struct idr *idr, void *ptr, int start, int end, gfp_t gfp)
{
u32 id = start;
int ret;
if (WARN_ON_ONCE(start < 0))
return -EINVAL;
ret = idr_alloc_u32(idr, ptr, &id, end > 0 ? end - 1 : INT_MAX, gfp);
if (ret)
return ret;
return id;
}
EXPORT_SYMBOL_GPL(idr_alloc);
/**
* idr_alloc_cyclic() - Allocate an ID cyclically.
* @idr: IDR handle.
* @ptr: Pointer to be associated with the new ID.
* @start: The minimum ID (inclusive).
* @end: The maximum ID (exclusive).
* @gfp: Memory allocation flags.
*
* Allocates an unused ID in the range specified by @start and @end. If
* @end is <= 0, it is treated as one larger than %INT_MAX. This allows
* callers to use @start + N as @end as long as N is within integer range.
* The search for an unused ID will start at the last ID allocated and will
* wrap around to @start if no free IDs are found before reaching @end.
*
* The caller should provide their own locking to ensure that two
* concurrent modifications to the IDR are not possible. Read-only
* accesses to the IDR may be done under the RCU read lock or may
* exclude simultaneous writers.
*
* Return: The newly allocated ID, -ENOMEM if memory allocation failed,
* or -ENOSPC if no free IDs could be found.
*/
int idr_alloc_cyclic(struct idr *idr, void *ptr, int start, int end, gfp_t gfp)
{
u32 id = idr->idr_next; int err, max = end > 0 ? end - 1 : INT_MAX; if ((int)id < start) id = start; err = idr_alloc_u32(idr, ptr, &id, max, gfp); if ((err == -ENOSPC) && (id > start)) { id = start;
err = idr_alloc_u32(idr, ptr, &id, max, gfp);
}
if (err)
return err;
idr->idr_next = id + 1; return id;
}
EXPORT_SYMBOL(idr_alloc_cyclic);
/**
* idr_remove() - Remove an ID from the IDR.
* @idr: IDR handle.
* @id: Pointer ID.
*
* Removes this ID from the IDR. If the ID was not previously in the IDR,
* this function returns %NULL.
*
* Since this function modifies the IDR, the caller should provide their
* own locking to ensure that concurrent modification of the same IDR is
* not possible.
*
* Return: The pointer formerly associated with this ID.
*/
void *idr_remove(struct idr *idr, unsigned long id)
{
return radix_tree_delete_item(&idr->idr_rt, id - idr->idr_base, NULL);
}
EXPORT_SYMBOL_GPL(idr_remove);
/**
* idr_find() - Return pointer for given ID.
* @idr: IDR handle.
* @id: Pointer ID.
*
* Looks up the pointer associated with this ID. A %NULL pointer may
* indicate that @id is not allocated or that the %NULL pointer was
* associated with this ID.
*
* This function can be called under rcu_read_lock(), given that the leaf
* pointers lifetimes are correctly managed.
*
* Return: The pointer associated with this ID.
*/
void *idr_find(const struct idr *idr, unsigned long id)
{
return radix_tree_lookup(&idr->idr_rt, id - idr->idr_base);
}
EXPORT_SYMBOL_GPL(idr_find);
/**
* idr_for_each() - Iterate through all stored pointers.
* @idr: IDR handle.
* @fn: Function to be called for each pointer.
* @data: Data passed to callback function.
*
* The callback function will be called for each entry in @idr, passing
* the ID, the entry and @data.
*
* If @fn returns anything other than %0, the iteration stops and that
* value is returned from this function.
*
* idr_for_each() can be called concurrently with idr_alloc() and
* idr_remove() if protected by RCU. Newly added entries may not be
* seen and deleted entries may be seen, but adding and removing entries
* will not cause other entries to be skipped, nor spurious ones to be seen.
*/
int idr_for_each(const struct idr *idr,
int (*fn)(int id, void *p, void *data), void *data)
{
struct radix_tree_iter iter;
void __rcu **slot;
int base = idr->idr_base;
radix_tree_for_each_slot(slot, &idr->idr_rt, &iter, 0) {
int ret;
unsigned long id = iter.index + base; if (WARN_ON_ONCE(id > INT_MAX))
break;
ret = fn(id, rcu_dereference_raw(*slot), data);
if (ret)
return ret;
}
return 0;
}
EXPORT_SYMBOL(idr_for_each);
/**
* idr_get_next_ul() - Find next populated entry.
* @idr: IDR handle.
* @nextid: Pointer to an ID.
*
* Returns the next populated entry in the tree with an ID greater than
* or equal to the value pointed to by @nextid. On exit, @nextid is updated
* to the ID of the found value. To use in a loop, the value pointed to by
* nextid must be incremented by the user.
*/
void *idr_get_next_ul(struct idr *idr, unsigned long *nextid)
{
struct radix_tree_iter iter;
void __rcu **slot;
void *entry = NULL;
unsigned long base = idr->idr_base;
unsigned long id = *nextid;
id = (id < base) ? 0 : id - base; radix_tree_for_each_slot(slot, &idr->idr_rt, &iter, id) { entry = rcu_dereference_raw(*slot);
if (!entry)
continue;
if (!xa_is_internal(entry))
break;
if (slot != &idr->idr_rt.xa_head && !xa_is_retry(entry))
break;
slot = radix_tree_iter_retry(&iter);
}
if (!slot)
return NULL;
*nextid = iter.index + base; return entry;
}
EXPORT_SYMBOL(idr_get_next_ul);
/**
* idr_get_next() - Find next populated entry.
* @idr: IDR handle.
* @nextid: Pointer to an ID.
*
* Returns the next populated entry in the tree with an ID greater than
* or equal to the value pointed to by @nextid. On exit, @nextid is updated
* to the ID of the found value. To use in a loop, the value pointed to by
* nextid must be incremented by the user.
*/
void *idr_get_next(struct idr *idr, int *nextid)
{
unsigned long id = *nextid;
void *entry = idr_get_next_ul(idr, &id);
if (WARN_ON_ONCE(id > INT_MAX))
return NULL;
*nextid = id; return entry;
}
EXPORT_SYMBOL(idr_get_next);
/**
* idr_replace() - replace pointer for given ID.
* @idr: IDR handle.
* @ptr: New pointer to associate with the ID.
* @id: ID to change.
*
* Replace the pointer registered with an ID and return the old value.
* This function can be called under the RCU read lock concurrently with
* idr_alloc() and idr_remove() (as long as the ID being removed is not
* the one being replaced!).
*
* Returns: the old value on success. %-ENOENT indicates that @id was not
* found. %-EINVAL indicates that @ptr was not valid.
*/
void *idr_replace(struct idr *idr, void *ptr, unsigned long id)
{
struct radix_tree_node *node;
void __rcu **slot = NULL;
void *entry;
id -= idr->idr_base;
entry = __radix_tree_lookup(&idr->idr_rt, id, &node, &slot);
if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) return ERR_PTR(-ENOENT); __radix_tree_replace(&idr->idr_rt, node, slot, ptr);
return entry;
}
EXPORT_SYMBOL(idr_replace);
/**
* DOC: IDA description
*
* The IDA is an ID allocator which does not provide the ability to
* associate an ID with a pointer. As such, it only needs to store one
* bit per ID, and so is more space efficient than an IDR. To use an IDA,
* define it using DEFINE_IDA() (or embed a &struct ida in a data structure,
* then initialise it using ida_init()). To allocate a new ID, call
* ida_alloc(), ida_alloc_min(), ida_alloc_max() or ida_alloc_range().
* To free an ID, call ida_free().
*
* ida_destroy() can be used to dispose of an IDA without needing to
* free the individual IDs in it. You can use ida_is_empty() to find
* out whether the IDA has any IDs currently allocated.
*
* The IDA handles its own locking. It is safe to call any of the IDA
* functions without synchronisation in your code.
*
* IDs are currently limited to the range [0-INT_MAX]. If this is an awkward
* limitation, it should be quite straightforward to raise the maximum.
*/
/*
* Developer's notes:
*
* The IDA uses the functionality provided by the XArray to store bitmaps in
* each entry. The XA_FREE_MARK is only cleared when all bits in the bitmap
* have been set.
*
* I considered telling the XArray that each slot is an order-10 node
* and indexing by bit number, but the XArray can't allow a single multi-index
* entry in the head, which would significantly increase memory consumption
* for the IDA. So instead we divide the index by the number of bits in the
* leaf bitmap before doing a radix tree lookup.
*
* As an optimisation, if there are only a few low bits set in any given
* leaf, instead of allocating a 128-byte bitmap, we store the bits
* as a value entry. Value entries never have the XA_FREE_MARK cleared
* because we can always convert them into a bitmap entry.
*
* It would be possible to optimise further; once we've run out of a
* single 128-byte bitmap, we currently switch to a 576-byte node, put
* the 128-byte bitmap in the first entry and then start allocating extra
* 128-byte entries. We could instead use the 512 bytes of the node's
* data as a bitmap before moving to that scheme. I do not believe this
* is a worthwhile optimisation; Rasmus Villemoes surveyed the current
* users of the IDA and almost none of them use more than 1024 entries.
* Those that do use more than the 8192 IDs that the 512 bytes would
* provide.
*
* The IDA always uses a lock to alloc/free. If we add a 'test_bit'
* equivalent, it will still need locking. Going to RCU lookup would require
* using RCU to free bitmaps, and that's not trivial without embedding an
* RCU head in the bitmap, which adds a 2-pointer overhead to each 128-byte
* bitmap, which is excessive.
*/
/**
* ida_alloc_range() - Allocate an unused ID.
* @ida: IDA handle.
* @min: Lowest ID to allocate.
* @max: Highest ID to allocate.
* @gfp: Memory allocation flags.
*
* Allocate an ID between @min and @max, inclusive. The allocated ID will
* not exceed %INT_MAX, even if @max is larger.
*
* Context: Any context. It is safe to call this function without
* locking in your code.
* Return: The allocated ID, or %-ENOMEM if memory could not be allocated,
* or %-ENOSPC if there are no free IDs.
*/
int ida_alloc_range(struct ida *ida, unsigned int min, unsigned int max,
gfp_t gfp)
{
XA_STATE(xas, &ida->xa, min / IDA_BITMAP_BITS);
unsigned bit = min % IDA_BITMAP_BITS;
unsigned long flags;
struct ida_bitmap *bitmap, *alloc = NULL;
if ((int)min < 0)
return -ENOSPC; if ((int)max < 0)
max = INT_MAX;
retry:
xas_lock_irqsave(&xas, flags);
next:
bitmap = xas_find_marked(&xas, max / IDA_BITMAP_BITS, XA_FREE_MARK);
if (xas.xa_index > min / IDA_BITMAP_BITS)
bit = 0;
if (xas.xa_index * IDA_BITMAP_BITS + bit > max)
goto nospc;
if (xa_is_value(bitmap)) { unsigned long tmp = xa_to_value(bitmap);
if (bit < BITS_PER_XA_VALUE) {
bit = find_next_zero_bit(&tmp, BITS_PER_XA_VALUE, bit); if (xas.xa_index * IDA_BITMAP_BITS + bit > max)
goto nospc;
if (bit < BITS_PER_XA_VALUE) { tmp |= 1UL << bit;
xas_store(&xas, xa_mk_value(tmp));
goto out;
}
}
bitmap = alloc;
if (!bitmap) bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT);
if (!bitmap)
goto alloc;
bitmap->bitmap[0] = tmp;
xas_store(&xas, bitmap);
if (xas_error(&xas)) { bitmap->bitmap[0] = 0;
goto out;
}
}
if (bitmap) { bit = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, bit);
if (xas.xa_index * IDA_BITMAP_BITS + bit > max)
goto nospc;
if (bit == IDA_BITMAP_BITS)
goto next;
__set_bit(bit, bitmap->bitmap);
if (bitmap_full(bitmap->bitmap, IDA_BITMAP_BITS))
xas_clear_mark(&xas, XA_FREE_MARK);
} else {
if (bit < BITS_PER_XA_VALUE) { bitmap = xa_mk_value(1UL << bit);
} else {
bitmap = alloc;
if (!bitmap) bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT);
if (!bitmap)
goto alloc;
__set_bit(bit, bitmap->bitmap);
}
xas_store(&xas, bitmap);
}
out:
xas_unlock_irqrestore(&xas, flags);
if (xas_nomem(&xas, gfp)) {
xas.xa_index = min / IDA_BITMAP_BITS;
bit = min % IDA_BITMAP_BITS;
goto retry;
}
if (bitmap != alloc) kfree(alloc); if (xas_error(&xas))
return xas_error(&xas);
return xas.xa_index * IDA_BITMAP_BITS + bit;
alloc:
xas_unlock_irqrestore(&xas, flags); alloc = kzalloc(sizeof(*bitmap), gfp);
if (!alloc)
return -ENOMEM;
xas_set(&xas, min / IDA_BITMAP_BITS);
bit = min % IDA_BITMAP_BITS;
goto retry;
nospc:
xas_unlock_irqrestore(&xas, flags);
kfree(alloc);
return -ENOSPC;
}
EXPORT_SYMBOL(ida_alloc_range);
/**
* ida_free() - Release an allocated ID.
* @ida: IDA handle.
* @id: Previously allocated ID.
*
* Context: Any context. It is safe to call this function without
* locking in your code.
*/
void ida_free(struct ida *ida, unsigned int id)
{
XA_STATE(xas, &ida->xa, id / IDA_BITMAP_BITS); unsigned bit = id % IDA_BITMAP_BITS;
struct ida_bitmap *bitmap;
unsigned long flags;
if ((int)id < 0)
return;
xas_lock_irqsave(&xas, flags);
bitmap = xas_load(&xas);
if (xa_is_value(bitmap)) {
unsigned long v = xa_to_value(bitmap); if (bit >= BITS_PER_XA_VALUE)
goto err;
if (!(v & (1UL << bit)))
goto err;
v &= ~(1UL << bit);
if (!v)
goto delete;
xas_store(&xas, xa_mk_value(v));
} else {
if (!bitmap || !test_bit(bit, bitmap->bitmap))
goto err;
__clear_bit(bit, bitmap->bitmap);
xas_set_mark(&xas, XA_FREE_MARK);
if (bitmap_empty(bitmap->bitmap, IDA_BITMAP_BITS)) {
kfree(bitmap);
delete:
xas_store(&xas, NULL);
}
}
xas_unlock_irqrestore(&xas, flags);
return;
err:
xas_unlock_irqrestore(&xas, flags); WARN(1, "ida_free called for id=%d which is not allocated.\n", id);
}
EXPORT_SYMBOL(ida_free);
/**
* ida_destroy() - Free all IDs.
* @ida: IDA handle.
*
* Calling this function frees all IDs and releases all resources used
* by an IDA. When this call returns, the IDA is empty and can be reused
* or freed. If the IDA is already empty, there is no need to call this
* function.
*
* Context: Any context. It is safe to call this function without
* locking in your code.
*/
void ida_destroy(struct ida *ida)
{
XA_STATE(xas, &ida->xa, 0);
struct ida_bitmap *bitmap;
unsigned long flags;
xas_lock_irqsave(&xas, flags);
xas_for_each(&xas, bitmap, ULONG_MAX) { if (!xa_is_value(bitmap)) kfree(bitmap); xas_store(&xas, NULL);
}
xas_unlock_irqrestore(&xas, flags);
}
EXPORT_SYMBOL(ida_destroy);
#ifndef __KERNEL__
extern void xa_dump_index(unsigned long index, unsigned int shift);
#define IDA_CHUNK_SHIFT ilog2(IDA_BITMAP_BITS)
static void ida_dump_entry(void *entry, unsigned long index)
{
unsigned long i;
if (!entry)
return;
if (xa_is_node(entry)) {
struct xa_node *node = xa_to_node(entry);
unsigned int shift = node->shift + IDA_CHUNK_SHIFT +
XA_CHUNK_SHIFT;
xa_dump_index(index * IDA_BITMAP_BITS, shift);
xa_dump_node(node);
for (i = 0; i < XA_CHUNK_SIZE; i++)
ida_dump_entry(node->slots[i],
index | (i << node->shift));
} else if (xa_is_value(entry)) {
xa_dump_index(index * IDA_BITMAP_BITS, ilog2(BITS_PER_LONG));
pr_cont("value: data %lx [%px]\n", xa_to_value(entry), entry);
} else {
struct ida_bitmap *bitmap = entry;
xa_dump_index(index * IDA_BITMAP_BITS, IDA_CHUNK_SHIFT);
pr_cont("bitmap: %p data", bitmap);
for (i = 0; i < IDA_BITMAP_LONGS; i++)
pr_cont(" %lx", bitmap->bitmap[i]);
pr_cont("\n");
}
}
static void ida_dump(struct ida *ida)
{
struct xarray *xa = &ida->xa;
pr_debug("ida: %p node %p free %d\n", ida, xa->xa_head,
xa->xa_flags >> ROOT_TAG_SHIFT);
ida_dump_entry(xa->xa_head, 0);
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_JIFFIES_H
#define _LINUX_JIFFIES_H
#include <linux/cache.h>
#include <linux/limits.h>
#include <linux/math64.h>
#include <linux/minmax.h>
#include <linux/types.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <vdso/jiffies.h>
#include <asm/param.h> /* for HZ */
#include <generated/timeconst.h>
/*
* The following defines establish the engineering parameters of the PLL
* model. The HZ variable establishes the timer interrupt frequency, 100 Hz
* for the SunOS kernel, 256 Hz for the Ultrix kernel and 1024 Hz for the
* OSF/1 kernel. The SHIFT_HZ define expresses the same value as the
* nearest power of two in order to avoid hardware multiply operations.
*/
#if HZ >= 12 && HZ < 24
# define SHIFT_HZ 4
#elif HZ >= 24 && HZ < 48
# define SHIFT_HZ 5
#elif HZ >= 48 && HZ < 96
# define SHIFT_HZ 6
#elif HZ >= 96 && HZ < 192
# define SHIFT_HZ 7
#elif HZ >= 192 && HZ < 384
# define SHIFT_HZ 8
#elif HZ >= 384 && HZ < 768
# define SHIFT_HZ 9
#elif HZ >= 768 && HZ < 1536
# define SHIFT_HZ 10
#elif HZ >= 1536 && HZ < 3072
# define SHIFT_HZ 11
#elif HZ >= 3072 && HZ < 6144
# define SHIFT_HZ 12
#elif HZ >= 6144 && HZ < 12288
# define SHIFT_HZ 13
#else
# error Invalid value of HZ.
#endif
/* Suppose we want to divide two numbers NOM and DEN: NOM/DEN, then we can
* improve accuracy by shifting LSH bits, hence calculating:
* (NOM << LSH) / DEN
* This however means trouble for large NOM, because (NOM << LSH) may no
* longer fit in 32 bits. The following way of calculating this gives us
* some slack, under the following conditions:
* - (NOM / DEN) fits in (32 - LSH) bits.
* - (NOM % DEN) fits in (32 - LSH) bits.
*/
#define SH_DIV(NOM,DEN,LSH) ( (((NOM) / (DEN)) << (LSH)) \
+ ((((NOM) % (DEN)) << (LSH)) + (DEN) / 2) / (DEN))
/* LATCH is used in the interval timer and ftape setup. */
#define LATCH ((CLOCK_TICK_RATE + HZ/2) / HZ) /* For divider */
extern int register_refined_jiffies(long clock_tick_rate);
/* TICK_USEC is the time between ticks in usec assuming SHIFTED_HZ */
#define TICK_USEC ((USEC_PER_SEC + HZ/2) / HZ)
/* USER_TICK_USEC is the time between ticks in usec assuming fake USER_HZ */
#define USER_TICK_USEC ((1000000UL + USER_HZ/2) / USER_HZ)
#ifndef __jiffy_arch_data
#define __jiffy_arch_data
#endif
/*
* The 64-bit value is not atomic on 32-bit systems - you MUST NOT read it
* without sampling the sequence number in jiffies_lock.
* get_jiffies_64() will do this for you as appropriate.
*
* jiffies and jiffies_64 are at the same address for little-endian systems
* and for 64-bit big-endian systems.
* On 32-bit big-endian systems, jiffies is the lower 32 bits of jiffies_64
* (i.e., at address @jiffies_64 + 4).
* See arch/ARCH/kernel/vmlinux.lds.S
*/
extern u64 __cacheline_aligned_in_smp jiffies_64;
extern unsigned long volatile __cacheline_aligned_in_smp __jiffy_arch_data jiffies;
#if (BITS_PER_LONG < 64)
u64 get_jiffies_64(void);
#else
/**
* get_jiffies_64 - read the 64-bit non-atomic jiffies_64 value
*
* When BITS_PER_LONG < 64, this uses sequence number sampling using
* jiffies_lock to protect the 64-bit read.
*
* Return: current 64-bit jiffies value
*/
static inline u64 get_jiffies_64(void)
{
return (u64)jiffies;
}
#endif
/**
* DOC: General information about time_* inlines
*
* These inlines deal with timer wrapping correctly. You are strongly encouraged
* to use them:
*
* #. Because people otherwise forget
* #. Because if the timer wrap changes in future you won't have to alter your
* driver code.
*/
/**
* time_after - returns true if the time a is after time b.
* @a: first comparable as unsigned long
* @b: second comparable as unsigned long
*
* Do this with "<0" and ">=0" to only test the sign of the result. A
* good compiler would generate better code (and a really good compiler
* wouldn't care). Gcc is currently neither.
*
* Return: %true is time a is after time b, otherwise %false.
*/
#define time_after(a,b) \
(typecheck(unsigned long, a) && \
typecheck(unsigned long, b) && \
((long)((b) - (a)) < 0))
/**
* time_before - returns true if the time a is before time b.
* @a: first comparable as unsigned long
* @b: second comparable as unsigned long
*
* Return: %true is time a is before time b, otherwise %false.
*/
#define time_before(a,b) time_after(b,a)
/**
* time_after_eq - returns true if the time a is after or the same as time b.
* @a: first comparable as unsigned long
* @b: second comparable as unsigned long
*
* Return: %true is time a is after or the same as time b, otherwise %false.
*/
#define time_after_eq(a,b) \
(typecheck(unsigned long, a) && \
typecheck(unsigned long, b) && \
((long)((a) - (b)) >= 0))
/**
* time_before_eq - returns true if the time a is before or the same as time b.
* @a: first comparable as unsigned long
* @b: second comparable as unsigned long
*
* Return: %true is time a is before or the same as time b, otherwise %false.
*/
#define time_before_eq(a,b) time_after_eq(b,a)
/**
* time_in_range - Calculate whether a is in the range of [b, c].
* @a: time to test
* @b: beginning of the range
* @c: end of the range
*
* Return: %true is time a is in the range [b, c], otherwise %false.
*/
#define time_in_range(a,b,c) \
(time_after_eq(a,b) && \
time_before_eq(a,c))
/**
* time_in_range_open - Calculate whether a is in the range of [b, c).
* @a: time to test
* @b: beginning of the range
* @c: end of the range
*
* Return: %true is time a is in the range [b, c), otherwise %false.
*/
#define time_in_range_open(a,b,c) \
(time_after_eq(a,b) && \
time_before(a,c))
/* Same as above, but does so with platform independent 64bit types.
* These must be used when utilizing jiffies_64 (i.e. return value of
* get_jiffies_64()). */
/**
* time_after64 - returns true if the time a is after time b.
* @a: first comparable as __u64
* @b: second comparable as __u64
*
* This must be used when utilizing jiffies_64 (i.e. return value of
* get_jiffies_64()).
*
* Return: %true is time a is after time b, otherwise %false.
*/
#define time_after64(a,b) \
(typecheck(__u64, a) && \
typecheck(__u64, b) && \
((__s64)((b) - (a)) < 0))
/**
* time_before64 - returns true if the time a is before time b.
* @a: first comparable as __u64
* @b: second comparable as __u64
*
* This must be used when utilizing jiffies_64 (i.e. return value of
* get_jiffies_64()).
*
* Return: %true is time a is before time b, otherwise %false.
*/
#define time_before64(a,b) time_after64(b,a)
/**
* time_after_eq64 - returns true if the time a is after or the same as time b.
* @a: first comparable as __u64
* @b: second comparable as __u64
*
* This must be used when utilizing jiffies_64 (i.e. return value of
* get_jiffies_64()).
*
* Return: %true is time a is after or the same as time b, otherwise %false.
*/
#define time_after_eq64(a,b) \
(typecheck(__u64, a) && \
typecheck(__u64, b) && \
((__s64)((a) - (b)) >= 0))
/**
* time_before_eq64 - returns true if the time a is before or the same as time b.
* @a: first comparable as __u64
* @b: second comparable as __u64
*
* This must be used when utilizing jiffies_64 (i.e. return value of
* get_jiffies_64()).
*
* Return: %true is time a is before or the same as time b, otherwise %false.
*/
#define time_before_eq64(a,b) time_after_eq64(b,a)
/**
* time_in_range64 - Calculate whether a is in the range of [b, c].
* @a: time to test
* @b: beginning of the range
* @c: end of the range
*
* Return: %true is time a is in the range [b, c], otherwise %false.
*/
#define time_in_range64(a, b, c) \
(time_after_eq64(a, b) && \
time_before_eq64(a, c))
/*
* These eight macros compare jiffies[_64] and 'a' for convenience.
*/
/**
* time_is_before_jiffies - return true if a is before jiffies
* @a: time (unsigned long) to compare to jiffies
*
* Return: %true is time a is before jiffies, otherwise %false.
*/
#define time_is_before_jiffies(a) time_after(jiffies, a)
/**
* time_is_before_jiffies64 - return true if a is before jiffies_64
* @a: time (__u64) to compare to jiffies_64
*
* Return: %true is time a is before jiffies_64, otherwise %false.
*/
#define time_is_before_jiffies64(a) time_after64(get_jiffies_64(), a)
/**
* time_is_after_jiffies - return true if a is after jiffies
* @a: time (unsigned long) to compare to jiffies
*
* Return: %true is time a is after jiffies, otherwise %false.
*/
#define time_is_after_jiffies(a) time_before(jiffies, a)
/**
* time_is_after_jiffies64 - return true if a is after jiffies_64
* @a: time (__u64) to compare to jiffies_64
*
* Return: %true is time a is after jiffies_64, otherwise %false.
*/
#define time_is_after_jiffies64(a) time_before64(get_jiffies_64(), a)
/**
* time_is_before_eq_jiffies - return true if a is before or equal to jiffies
* @a: time (unsigned long) to compare to jiffies
*
* Return: %true is time a is before or the same as jiffies, otherwise %false.
*/
#define time_is_before_eq_jiffies(a) time_after_eq(jiffies, a)
/**
* time_is_before_eq_jiffies64 - return true if a is before or equal to jiffies_64
* @a: time (__u64) to compare to jiffies_64
*
* Return: %true is time a is before or the same jiffies_64, otherwise %false.
*/
#define time_is_before_eq_jiffies64(a) time_after_eq64(get_jiffies_64(), a)
/**
* time_is_after_eq_jiffies - return true if a is after or equal to jiffies
* @a: time (unsigned long) to compare to jiffies
*
* Return: %true is time a is after or the same as jiffies, otherwise %false.
*/
#define time_is_after_eq_jiffies(a) time_before_eq(jiffies, a)
/**
* time_is_after_eq_jiffies64 - return true if a is after or equal to jiffies_64
* @a: time (__u64) to compare to jiffies_64
*
* Return: %true is time a is after or the same as jiffies_64, otherwise %false.
*/
#define time_is_after_eq_jiffies64(a) time_before_eq64(get_jiffies_64(), a)
/*
* Have the 32-bit jiffies value wrap 5 minutes after boot
* so jiffies wrap bugs show up earlier.
*/
#define INITIAL_JIFFIES ((unsigned long)(unsigned int) (-300*HZ))
/*
* Change timeval to jiffies, trying to avoid the
* most obvious overflows..
*
* And some not so obvious.
*
* Note that we don't want to return LONG_MAX, because
* for various timeout reasons we often end up having
* to wait "jiffies+1" in order to guarantee that we wait
* at _least_ "jiffies" - so "jiffies+1" had better still
* be positive.
*/
#define MAX_JIFFY_OFFSET ((LONG_MAX >> 1)-1)
extern unsigned long preset_lpj;
/*
* We want to do realistic conversions of time so we need to use the same
* values the update wall clock code uses as the jiffies size. This value
* is: TICK_NSEC (which is defined in timex.h). This
* is a constant and is in nanoseconds. We will use scaled math
* with a set of scales defined here as SEC_JIFFIE_SC, USEC_JIFFIE_SC and
* NSEC_JIFFIE_SC. Note that these defines contain nothing but
* constants and so are computed at compile time. SHIFT_HZ (computed in
* timex.h) adjusts the scaling for different HZ values.
* Scaled math??? What is that?
*
* Scaled math is a way to do integer math on values that would,
* otherwise, either overflow, underflow, or cause undesired div
* instructions to appear in the execution path. In short, we "scale"
* up the operands so they take more bits (more precision, less
* underflow), do the desired operation and then "scale" the result back
* by the same amount. If we do the scaling by shifting we avoid the
* costly mpy and the dastardly div instructions.
* Suppose, for example, we want to convert from seconds to jiffies
* where jiffies is defined in nanoseconds as NSEC_PER_JIFFIE. The
* simple math is: jiff = (sec * NSEC_PER_SEC) / NSEC_PER_JIFFIE; We
* observe that (NSEC_PER_SEC / NSEC_PER_JIFFIE) is a constant which we
* might calculate at compile time, however, the result will only have
* about 3-4 bits of precision (less for smaller values of HZ).
*
* So, we scale as follows:
* jiff = (sec) * (NSEC_PER_SEC / NSEC_PER_JIFFIE);
* jiff = ((sec) * ((NSEC_PER_SEC * SCALE)/ NSEC_PER_JIFFIE)) / SCALE;
* Then we make SCALE a power of two so:
* jiff = ((sec) * ((NSEC_PER_SEC << SCALE)/ NSEC_PER_JIFFIE)) >> SCALE;
* Now we define:
* #define SEC_CONV = ((NSEC_PER_SEC << SCALE)/ NSEC_PER_JIFFIE))
* jiff = (sec * SEC_CONV) >> SCALE;
*
* Often the math we use will expand beyond 32-bits so we tell C how to
* do this and pass the 64-bit result of the mpy through the ">> SCALE"
* which should take the result back to 32-bits. We want this expansion
* to capture as much precision as possible. At the same time we don't
* want to overflow so we pick the SCALE to avoid this. In this file,
* that means using a different scale for each range of HZ values (as
* defined in timex.h).
*
* For those who want to know, gcc will give a 64-bit result from a "*"
* operator if the result is a long long AND at least one of the
* operands is cast to long long (usually just prior to the "*" so as
* not to confuse it into thinking it really has a 64-bit operand,
* which, buy the way, it can do, but it takes more code and at least 2
* mpys).
* We also need to be aware that one second in nanoseconds is only a
* couple of bits away from overflowing a 32-bit word, so we MUST use
* 64-bits to get the full range time in nanoseconds.
*/
/*
* Here are the scales we will use. One for seconds, nanoseconds and
* microseconds.
*
* Within the limits of cpp we do a rough cut at the SEC_JIFFIE_SC and
* check if the sign bit is set. If not, we bump the shift count by 1.
* (Gets an extra bit of precision where we can use it.)
* We know it is set for HZ = 1024 and HZ = 100 not for 1000.
* Haven't tested others.
* Limits of cpp (for #if expressions) only long (no long long), but
* then we only need the most signicant bit.
*/
#define SEC_JIFFIE_SC (31 - SHIFT_HZ)
#if !((((NSEC_PER_SEC << 2) / TICK_NSEC) << (SEC_JIFFIE_SC - 2)) & 0x80000000)
#undef SEC_JIFFIE_SC
#define SEC_JIFFIE_SC (32 - SHIFT_HZ)
#endif
#define NSEC_JIFFIE_SC (SEC_JIFFIE_SC + 29)
#define SEC_CONVERSION ((unsigned long)((((u64)NSEC_PER_SEC << SEC_JIFFIE_SC) +\
TICK_NSEC -1) / (u64)TICK_NSEC))
#define NSEC_CONVERSION ((unsigned long)((((u64)1 << NSEC_JIFFIE_SC) +\
TICK_NSEC -1) / (u64)TICK_NSEC))
/*
* The maximum jiffie value is (MAX_INT >> 1). Here we translate that
* into seconds. The 64-bit case will overflow if we are not careful,
* so use the messy SH_DIV macro to do it. Still all constants.
*/
#if BITS_PER_LONG < 64
# define MAX_SEC_IN_JIFFIES \
(long)((u64)((u64)MAX_JIFFY_OFFSET * TICK_NSEC) / NSEC_PER_SEC)
#else /* take care of overflow on 64-bit machines */
# define MAX_SEC_IN_JIFFIES \
(SH_DIV((MAX_JIFFY_OFFSET >> SEC_JIFFIE_SC) * TICK_NSEC, NSEC_PER_SEC, 1) - 1)
#endif
/*
* Convert various time units to each other:
*/
extern unsigned int jiffies_to_msecs(const unsigned long j);
extern unsigned int jiffies_to_usecs(const unsigned long j);
/**
* jiffies_to_nsecs - Convert jiffies to nanoseconds
* @j: jiffies value
*
* Return: nanoseconds value
*/
static inline u64 jiffies_to_nsecs(const unsigned long j)
{
return (u64)jiffies_to_usecs(j) * NSEC_PER_USEC;
}
extern u64 jiffies64_to_nsecs(u64 j);
extern u64 jiffies64_to_msecs(u64 j);
extern unsigned long __msecs_to_jiffies(const unsigned int m);
#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ)
/*
* HZ is equal to or smaller than 1000, and 1000 is a nice round
* multiple of HZ, divide with the factor between them, but round
* upwards:
*/
static inline unsigned long _msecs_to_jiffies(const unsigned int m)
{
return (m + (MSEC_PER_SEC / HZ) - 1) / (MSEC_PER_SEC / HZ);
}
#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC)
/*
* HZ is larger than 1000, and HZ is a nice round multiple of 1000 -
* simply multiply with the factor between them.
*
* But first make sure the multiplication result cannot overflow:
*/
static inline unsigned long _msecs_to_jiffies(const unsigned int m)
{
if (m > jiffies_to_msecs(MAX_JIFFY_OFFSET))
return MAX_JIFFY_OFFSET;
return m * (HZ / MSEC_PER_SEC);
}
#else
/*
* Generic case - multiply, round and divide. But first check that if
* we are doing a net multiplication, that we wouldn't overflow:
*/
static inline unsigned long _msecs_to_jiffies(const unsigned int m)
{
if (HZ > MSEC_PER_SEC && m > jiffies_to_msecs(MAX_JIFFY_OFFSET))
return MAX_JIFFY_OFFSET;
return (MSEC_TO_HZ_MUL32 * m + MSEC_TO_HZ_ADJ32) >> MSEC_TO_HZ_SHR32;
}
#endif
/**
* msecs_to_jiffies: - convert milliseconds to jiffies
* @m: time in milliseconds
*
* conversion is done as follows:
*
* - negative values mean 'infinite timeout' (MAX_JIFFY_OFFSET)
*
* - 'too large' values [that would result in larger than
* MAX_JIFFY_OFFSET values] mean 'infinite timeout' too.
*
* - all other values are converted to jiffies by either multiplying
* the input value by a factor or dividing it with a factor and
* handling any 32-bit overflows.
* for the details see __msecs_to_jiffies()
*
* msecs_to_jiffies() checks for the passed in value being a constant
* via __builtin_constant_p() allowing gcc to eliminate most of the
* code. __msecs_to_jiffies() is called if the value passed does not
* allow constant folding and the actual conversion must be done at
* runtime.
* The HZ range specific helpers _msecs_to_jiffies() are called both
* directly here and from __msecs_to_jiffies() in the case where
* constant folding is not possible.
*
* Return: jiffies value
*/
static __always_inline unsigned long msecs_to_jiffies(const unsigned int m)
{
if (__builtin_constant_p(m)) {
if ((int)m < 0)
return MAX_JIFFY_OFFSET;
return _msecs_to_jiffies(m);
} else {
return __msecs_to_jiffies(m);
}
}
extern unsigned long __usecs_to_jiffies(const unsigned int u);
#if !(USEC_PER_SEC % HZ)
static inline unsigned long _usecs_to_jiffies(const unsigned int u)
{
return (u + (USEC_PER_SEC / HZ) - 1) / (USEC_PER_SEC / HZ);
}
#else
static inline unsigned long _usecs_to_jiffies(const unsigned int u)
{
return (USEC_TO_HZ_MUL32 * u + USEC_TO_HZ_ADJ32)
>> USEC_TO_HZ_SHR32;
}
#endif
/**
* usecs_to_jiffies: - convert microseconds to jiffies
* @u: time in microseconds
*
* conversion is done as follows:
*
* - 'too large' values [that would result in larger than
* MAX_JIFFY_OFFSET values] mean 'infinite timeout' too.
*
* - all other values are converted to jiffies by either multiplying
* the input value by a factor or dividing it with a factor and
* handling any 32-bit overflows as for msecs_to_jiffies.
*
* usecs_to_jiffies() checks for the passed in value being a constant
* via __builtin_constant_p() allowing gcc to eliminate most of the
* code. __usecs_to_jiffies() is called if the value passed does not
* allow constant folding and the actual conversion must be done at
* runtime.
* The HZ range specific helpers _usecs_to_jiffies() are called both
* directly here and from __msecs_to_jiffies() in the case where
* constant folding is not possible.
*
* Return: jiffies value
*/
static __always_inline unsigned long usecs_to_jiffies(const unsigned int u)
{
if (__builtin_constant_p(u)) {
if (u > jiffies_to_usecs(MAX_JIFFY_OFFSET))
return MAX_JIFFY_OFFSET;
return _usecs_to_jiffies(u);
} else {
return __usecs_to_jiffies(u);
}
}
extern unsigned long timespec64_to_jiffies(const struct timespec64 *value);
extern void jiffies_to_timespec64(const unsigned long jiffies,
struct timespec64 *value);
extern clock_t jiffies_to_clock_t(unsigned long x);
static inline clock_t jiffies_delta_to_clock_t(long delta)
{
return jiffies_to_clock_t(max(0L, delta));
}
static inline unsigned int jiffies_delta_to_msecs(long delta)
{
return jiffies_to_msecs(max(0L, delta));
}
extern unsigned long clock_t_to_jiffies(unsigned long x);
extern u64 jiffies_64_to_clock_t(u64 x);
extern u64 nsec_to_clock_t(u64 x);
extern u64 nsecs_to_jiffies64(u64 n);
extern unsigned long nsecs_to_jiffies(u64 n);
#define TIMESTAMP_SIZE 30
#endif
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef LINUX_RESUME_USER_MODE_H
#define LINUX_RESUME_USER_MODE_H
#include <linux/sched.h>
#include <linux/task_work.h>
#include <linux/memcontrol.h>
#include <linux/rseq.h>
#include <linux/blk-cgroup.h>
/**
* set_notify_resume - cause resume_user_mode_work() to be called
* @task: task that will call resume_user_mode_work()
*
* Calling this arranges that @task will call resume_user_mode_work()
* before returning to user mode. If it's already running in user mode,
* it will enter the kernel and call resume_user_mode_work() soon.
* If it's blocked, it will not be woken.
*/
static inline void set_notify_resume(struct task_struct *task)
{
if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_RESUME)) kick_process(task);
}
/**
* resume_user_mode_work - Perform work before returning to user mode
* @regs: user-mode registers of @current task
*
* This is called when %TIF_NOTIFY_RESUME has been set. Now we are
* about to return to user mode, and the user state in @regs can be
* inspected or adjusted. The caller in arch code has cleared
* %TIF_NOTIFY_RESUME before the call. If the flag gets set again
* asynchronously, this will be called again before we return to
* user mode.
*
* Called without locks.
*/
static inline void resume_user_mode_work(struct pt_regs *regs)
{
clear_thread_flag(TIF_NOTIFY_RESUME);
/*
* This barrier pairs with task_work_add()->set_notify_resume() after
* hlist_add_head(task->task_works);
*/
smp_mb__after_atomic();
if (unlikely(task_work_pending(current)))
task_work_run();
#ifdef CONFIG_KEYS_REQUEST_CACHE
if (unlikely(current->cached_requested_key)) {
key_put(current->cached_requested_key);
current->cached_requested_key = NULL;
}
#endif
mem_cgroup_handle_over_high(GFP_KERNEL);
blkcg_maybe_throttle_current();
rseq_handle_notify_resume(NULL, regs);
}
#endif /* LINUX_RESUME_USER_MODE_H */
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* kref.h - library routines for handling generic reference counted objects
*
* Copyright (C) 2004 Greg Kroah-Hartman <greg@kroah.com>
* Copyright (C) 2004 IBM Corp.
*
* based on kobject.h which was:
* Copyright (C) 2002-2003 Patrick Mochel <mochel@osdl.org>
* Copyright (C) 2002-2003 Open Source Development Labs
*/
#ifndef _KREF_H_
#define _KREF_H_
#include <linux/spinlock.h>
#include <linux/refcount.h>
struct kref {
refcount_t refcount;
};
#define KREF_INIT(n) { .refcount = REFCOUNT_INIT(n), }
/**
* kref_init - initialize object.
* @kref: object in question.
*/
static inline void kref_init(struct kref *kref)
{
refcount_set(&kref->refcount, 1);
}
static inline unsigned int kref_read(const struct kref *kref)
{
return refcount_read(&kref->refcount);
}
/**
* kref_get - increment refcount for object.
* @kref: object.
*/
static inline void kref_get(struct kref *kref)
{
refcount_inc(&kref->refcount);
}
/**
* kref_put - decrement refcount for object.
* @kref: object.
* @release: pointer to the function that will clean up the object when the
* last reference to the object is released.
* This pointer is required, and it is not acceptable to pass kfree
* in as this function.
*
* Decrement the refcount, and if 0, call release().
* Return 1 if the object was removed, otherwise return 0. Beware, if this
* function returns 0, you still can not count on the kref from remaining in
* memory. Only use the return value if you want to see if the kref is now
* gone, not present.
*/
static inline int kref_put(struct kref *kref, void (*release)(struct kref *kref))
{
if (refcount_dec_and_test(&kref->refcount)) { release(kref);
return 1;
}
return 0;
}
static inline int kref_put_mutex(struct kref *kref,
void (*release)(struct kref *kref),
struct mutex *lock)
{
if (refcount_dec_and_mutex_lock(&kref->refcount, lock)) {
release(kref);
return 1;
}
return 0;
}
static inline int kref_put_lock(struct kref *kref,
void (*release)(struct kref *kref),
spinlock_t *lock)
{
if (refcount_dec_and_lock(&kref->refcount, lock)) {
release(kref);
return 1;
}
return 0;
}
/**
* kref_get_unless_zero - Increment refcount for object unless it is zero.
* @kref: object.
*
* Return non-zero if the increment succeeded. Otherwise return 0.
*
* This function is intended to simplify locking around refcounting for
* objects that can be looked up from a lookup structure, and which are
* removed from that lookup structure in the object destructor.
* Operations on such objects require at least a read lock around
* lookup + kref_get, and a write lock around kref_put + remove from lookup
* structure. Furthermore, RCU implementations become extremely tricky.
* With a lookup followed by a kref_get_unless_zero *with return value check*
* locking in the kref_put path can be deferred to the actual removal from
* the lookup structure and RCU lookups become trivial.
*/
static inline int __must_check kref_get_unless_zero(struct kref *kref)
{
return refcount_inc_not_zero(&kref->refcount);
}
#endif /* _KREF_H_ */
// SPDX-License-Identifier: GPL-2.0
#include <linux/acpi.h>
#include <linux/bitmap.h>
#include <linux/cleanup.h>
#include <linux/compat.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/idr.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/pinctrl/consumer.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/srcu.h>
#include <linux/string.h>
#include <linux/gpio.h>
#include <linux/gpio/driver.h>
#include <linux/gpio/machine.h>
#include <uapi/linux/gpio.h>
#include "gpiolib-acpi.h"
#include "gpiolib-cdev.h"
#include "gpiolib-of.h"
#include "gpiolib-swnode.h"
#include "gpiolib-sysfs.h"
#include "gpiolib.h"
#define CREATE_TRACE_POINTS
#include <trace/events/gpio.h>
/* Implementation infrastructure for GPIO interfaces.
*
* The GPIO programming interface allows for inlining speed-critical
* get/set operations for common cases, so that access to SOC-integrated
* GPIOs can sometimes cost only an instruction or two per bit.
*/
/* Device and char device-related information */
static DEFINE_IDA(gpio_ida);
static dev_t gpio_devt;
#define GPIO_DEV_MAX 256 /* 256 GPIO chip devices supported */
static int gpio_bus_match(struct device *dev, struct device_driver *drv)
{
struct fwnode_handle *fwnode = dev_fwnode(dev);
/*
* Only match if the fwnode doesn't already have a proper struct device
* created for it.
*/
if (fwnode && fwnode->dev != dev)
return 0;
return 1;
}
static const struct bus_type gpio_bus_type = {
.name = "gpio",
.match = gpio_bus_match,
};
/*
* Number of GPIOs to use for the fast path in set array
*/
#define FASTPATH_NGPIO CONFIG_GPIOLIB_FASTPATH_LIMIT
static DEFINE_MUTEX(gpio_lookup_lock);
static LIST_HEAD(gpio_lookup_list);
static LIST_HEAD(gpio_devices);
/* Protects the GPIO device list against concurrent modifications. */
static DEFINE_MUTEX(gpio_devices_lock);
/* Ensures coherence during read-only accesses to the list of GPIO devices. */
DEFINE_STATIC_SRCU(gpio_devices_srcu);
static DEFINE_MUTEX(gpio_machine_hogs_mutex);
static LIST_HEAD(gpio_machine_hogs);
static void gpiochip_free_hogs(struct gpio_chip *gc);
static int gpiochip_add_irqchip(struct gpio_chip *gc,
struct lock_class_key *lock_key,
struct lock_class_key *request_key);
static void gpiochip_irqchip_remove(struct gpio_chip *gc);
static int gpiochip_irqchip_init_hw(struct gpio_chip *gc);
static int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gc);
static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc);
static bool gpiolib_initialized;
const char *gpiod_get_label(struct gpio_desc *desc)
{
unsigned long flags;
flags = READ_ONCE(desc->flags);
if (test_bit(FLAG_USED_AS_IRQ, &flags) &&
!test_bit(FLAG_REQUESTED, &flags))
return "interrupt";
return test_bit(FLAG_REQUESTED, &flags) ?
srcu_dereference(desc->label, &desc->srcu) : NULL;
}
static int desc_set_label(struct gpio_desc *desc, const char *label)
{
const char *new = NULL, *old;
if (label) {
new = kstrdup_const(label, GFP_KERNEL);
if (!new)
return -ENOMEM;
}
old = rcu_replace_pointer(desc->label, new, 1);
synchronize_srcu(&desc->srcu);
kfree_const(old);
return 0;
}
/**
* gpio_to_desc - Convert a GPIO number to its descriptor
* @gpio: global GPIO number
*
* Returns:
* The GPIO descriptor associated with the given GPIO, or %NULL if no GPIO
* with the given number exists in the system.
*/
struct gpio_desc *gpio_to_desc(unsigned gpio)
{
struct gpio_device *gdev;
scoped_guard(srcu, &gpio_devices_srcu) {
list_for_each_entry_srcu(gdev, &gpio_devices, list,
srcu_read_lock_held(&gpio_devices_srcu)) {
if (gdev->base <= gpio &&
gdev->base + gdev->ngpio > gpio)
return &gdev->descs[gpio - gdev->base];
}
}
if (!gpio_is_valid(gpio))
pr_warn("invalid GPIO %d\n", gpio);
return NULL;
}
EXPORT_SYMBOL_GPL(gpio_to_desc);
/* This function is deprecated and will be removed soon, don't use. */
struct gpio_desc *gpiochip_get_desc(struct gpio_chip *gc,
unsigned int hwnum)
{
return gpio_device_get_desc(gc->gpiodev, hwnum);
}
EXPORT_SYMBOL_GPL(gpiochip_get_desc);
/**
* gpio_device_get_desc() - get the GPIO descriptor corresponding to the given
* hardware number for this GPIO device
* @gdev: GPIO device to get the descriptor from
* @hwnum: hardware number of the GPIO for this chip
*
* Returns:
* A pointer to the GPIO descriptor or %EINVAL if no GPIO exists in the given
* chip for the specified hardware number or %ENODEV if the underlying chip
* already vanished.
*
* The reference count of struct gpio_device is *NOT* increased like when the
* GPIO is being requested for exclusive usage. It's up to the caller to make
* sure the GPIO device will stay alive together with the descriptor returned
* by this function.
*/
struct gpio_desc *
gpio_device_get_desc(struct gpio_device *gdev, unsigned int hwnum)
{
if (hwnum >= gdev->ngpio)
return ERR_PTR(-EINVAL);
return &gdev->descs[hwnum];
}
EXPORT_SYMBOL_GPL(gpio_device_get_desc);
/**
* desc_to_gpio - convert a GPIO descriptor to the integer namespace
* @desc: GPIO descriptor
*
* This should disappear in the future but is needed since we still
* use GPIO numbers for error messages and sysfs nodes.
*
* Returns:
* The global GPIO number for the GPIO specified by its descriptor.
*/
int desc_to_gpio(const struct gpio_desc *desc)
{
return desc->gdev->base + (desc - &desc->gdev->descs[0]);
}
EXPORT_SYMBOL_GPL(desc_to_gpio);
/**
* gpiod_to_chip - Return the GPIO chip to which a GPIO descriptor belongs
* @desc: descriptor to return the chip of
*
* *DEPRECATED*
* This function is unsafe and should not be used. Using the chip address
* without taking the SRCU read lock may result in dereferencing a dangling
* pointer.
*/
struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc)
{
if (!desc)
return NULL;
return gpio_device_get_chip(desc->gdev);
}
EXPORT_SYMBOL_GPL(gpiod_to_chip);
/**
* gpiod_to_gpio_device() - Return the GPIO device to which this descriptor
* belongs.
* @desc: Descriptor for which to return the GPIO device.
*
* This *DOES NOT* increase the reference count of the GPIO device as it's
* expected that the descriptor is requested and the users already holds a
* reference to the device.
*
* Returns:
* Address of the GPIO device owning this descriptor.
*/
struct gpio_device *gpiod_to_gpio_device(struct gpio_desc *desc)
{
if (!desc)
return NULL;
return desc->gdev;
}
EXPORT_SYMBOL_GPL(gpiod_to_gpio_device);
/**
* gpio_device_get_base() - Get the base GPIO number allocated by this device
* @gdev: GPIO device
*
* Returns:
* First GPIO number in the global GPIO numberspace for this device.
*/
int gpio_device_get_base(struct gpio_device *gdev)
{
return gdev->base;
}
EXPORT_SYMBOL_GPL(gpio_device_get_base);
/**
* gpio_device_get_label() - Get the label of this GPIO device
* @gdev: GPIO device
*
* Returns:
* Pointer to the string containing the GPIO device label. The string's
* lifetime is tied to that of the underlying GPIO device.
*/
const char *gpio_device_get_label(struct gpio_device *gdev)
{
return gdev->label;
}
EXPORT_SYMBOL(gpio_device_get_label);
/**
* gpio_device_get_chip() - Get the gpio_chip implementation of this GPIO device
* @gdev: GPIO device
*
* Returns:
* Address of the GPIO chip backing this device.
*
* *DEPRECATED*
* Until we can get rid of all non-driver users of struct gpio_chip, we must
* provide a way of retrieving the pointer to it from struct gpio_device. This
* is *NOT* safe as the GPIO API is considered to be hot-unpluggable and the
* chip can dissapear at any moment (unlike reference-counted struct
* gpio_device).
*
* Use at your own risk.
*/
struct gpio_chip *gpio_device_get_chip(struct gpio_device *gdev)
{
return rcu_dereference_check(gdev->chip, 1);
}
EXPORT_SYMBOL_GPL(gpio_device_get_chip);
/* dynamic allocation of GPIOs, e.g. on a hotplugged device */
static int gpiochip_find_base_unlocked(int ngpio)
{
struct gpio_device *gdev;
int base = GPIO_DYNAMIC_BASE;
list_for_each_entry_srcu(gdev, &gpio_devices, list,
lockdep_is_held(&gpio_devices_lock)) {
/* found a free space? */
if (gdev->base >= base + ngpio)
break;
/* nope, check the space right after the chip */
base = gdev->base + gdev->ngpio;
if (base < GPIO_DYNAMIC_BASE)
base = GPIO_DYNAMIC_BASE;
}
if (gpio_is_valid(base)) {
pr_debug("%s: found new base at %d\n", __func__, base);
return base;
} else {
pr_err("%s: cannot find free range\n", __func__);
return -ENOSPC;
}
}
/**
* gpiod_get_direction - return the current direction of a GPIO
* @desc: GPIO to get the direction of
*
* Returns 0 for output, 1 for input, or an error code in case of error.
*
* This function may sleep if gpiod_cansleep() is true.
*/
int gpiod_get_direction(struct gpio_desc *desc)
{
unsigned long flags;
unsigned int offset;
int ret;
/*
* We cannot use VALIDATE_DESC() as we must not return 0 for a NULL
* descriptor like we usually do.
*/
if (!desc || IS_ERR(desc))
return -EINVAL;
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
offset = gpio_chip_hwgpio(desc);
flags = READ_ONCE(desc->flags);
/*
* Open drain emulation using input mode may incorrectly report
* input here, fix that up.
*/
if (test_bit(FLAG_OPEN_DRAIN, &flags) &&
test_bit(FLAG_IS_OUT, &flags))
return 0;
if (!guard.gc->get_direction)
return -ENOTSUPP;
ret = guard.gc->get_direction(guard.gc, offset);
if (ret < 0)
return ret;
/* GPIOF_DIR_IN or other positive, otherwise GPIOF_DIR_OUT */
if (ret > 0)
ret = 1;
assign_bit(FLAG_IS_OUT, &flags, !ret);
WRITE_ONCE(desc->flags, flags);
return ret;
}
EXPORT_SYMBOL_GPL(gpiod_get_direction);
/*
* Add a new chip to the global chips list, keeping the list of chips sorted
* by range(means [base, base + ngpio - 1]) order.
*
* Return -EBUSY if the new chip overlaps with some other chip's integer
* space.
*/
static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev)
{
struct gpio_device *prev, *next;
lockdep_assert_held(&gpio_devices_lock);
if (list_empty(&gpio_devices)) {
/* initial entry in list */
list_add_tail_rcu(&gdev->list, &gpio_devices);
return 0;
}
next = list_first_entry(&gpio_devices, struct gpio_device, list);
if (gdev->base + gdev->ngpio <= next->base) {
/* add before first entry */
list_add_rcu(&gdev->list, &gpio_devices);
return 0;
}
prev = list_last_entry(&gpio_devices, struct gpio_device, list);
if (prev->base + prev->ngpio <= gdev->base) {
/* add behind last entry */
list_add_tail_rcu(&gdev->list, &gpio_devices);
return 0;
}
list_for_each_entry_safe(prev, next, &gpio_devices, list) {
/* at the end of the list */
if (&next->list == &gpio_devices)
break;
/* add between prev and next */
if (prev->base + prev->ngpio <= gdev->base
&& gdev->base + gdev->ngpio <= next->base) {
list_add_rcu(&gdev->list, &prev->list);
return 0;
}
}
synchronize_srcu(&gpio_devices_srcu);
return -EBUSY;
}
/*
* Convert a GPIO name to its descriptor
* Note that there is no guarantee that GPIO names are globally unique!
* Hence this function will return, if it exists, a reference to the first GPIO
* line found that matches the given name.
*/
static struct gpio_desc *gpio_name_to_desc(const char * const name)
{
struct gpio_device *gdev;
struct gpio_desc *desc;
struct gpio_chip *gc;
if (!name)
return NULL;
guard(srcu)(&gpio_devices_srcu);
list_for_each_entry_srcu(gdev, &gpio_devices, list,
srcu_read_lock_held(&gpio_devices_srcu)) {
guard(srcu)(&gdev->srcu);
gc = srcu_dereference(gdev->chip, &gdev->srcu);
if (!gc)
continue;
for_each_gpio_desc(gc, desc) {
if (desc->name && !strcmp(desc->name, name))
return desc;
}
}
return NULL;
}
/*
* Take the names from gc->names and assign them to their GPIO descriptors.
* Warn if a name is already used for a GPIO line on a different GPIO chip.
*
* Note that:
* 1. Non-unique names are still accepted,
* 2. Name collisions within the same GPIO chip are not reported.
*/
static int gpiochip_set_desc_names(struct gpio_chip *gc)
{
struct gpio_device *gdev = gc->gpiodev;
int i;
/* First check all names if they are unique */
for (i = 0; i != gc->ngpio; ++i) {
struct gpio_desc *gpio;
gpio = gpio_name_to_desc(gc->names[i]);
if (gpio)
dev_warn(&gdev->dev,
"Detected name collision for GPIO name '%s'\n",
gc->names[i]);
}
/* Then add all names to the GPIO descriptors */
for (i = 0; i != gc->ngpio; ++i)
gdev->descs[i].name = gc->names[i];
return 0;
}
/*
* gpiochip_set_names - Set GPIO line names using device properties
* @chip: GPIO chip whose lines should be named, if possible
*
* Looks for device property "gpio-line-names" and if it exists assigns
* GPIO line names for the chip. The memory allocated for the assigned
* names belong to the underlying firmware node and should not be released
* by the caller.
*/
static int gpiochip_set_names(struct gpio_chip *chip)
{
struct gpio_device *gdev = chip->gpiodev;
struct device *dev = &gdev->dev;
const char **names;
int ret, i;
int count;
count = device_property_string_array_count(dev, "gpio-line-names");
if (count < 0)
return 0;
/*
* When offset is set in the driver side we assume the driver internally
* is using more than one gpiochip per the same device. We have to stop
* setting friendly names if the specified ones with 'gpio-line-names'
* are less than the offset in the device itself. This means all the
* lines are not present for every single pin within all the internal
* gpiochips.
*/
if (count <= chip->offset) {
dev_warn(dev, "gpio-line-names too short (length %d), cannot map names for the gpiochip at offset %u\n",
count, chip->offset);
return 0;
}
names = kcalloc(count, sizeof(*names), GFP_KERNEL);
if (!names)
return -ENOMEM;
ret = device_property_read_string_array(dev, "gpio-line-names",
names, count);
if (ret < 0) {
dev_warn(dev, "failed to read GPIO line names\n");
kfree(names);
return ret;
}
/*
* When more that one gpiochip per device is used, 'count' can
* contain at most number gpiochips x chip->ngpio. We have to
* correctly distribute all defined lines taking into account
* chip->offset as starting point from where we will assign
* the names to pins from the 'names' array. Since property
* 'gpio-line-names' cannot contains gaps, we have to be sure
* we only assign those pins that really exists since chip->ngpio
* can be different of the chip->offset.
*/
count = (count > chip->offset) ? count - chip->offset : count;
if (count > chip->ngpio)
count = chip->ngpio;
for (i = 0; i < count; i++) {
/*
* Allow overriding "fixed" names provided by the GPIO
* provider. The "fixed" names are more often than not
* generic and less informative than the names given in
* device properties.
*/
if (names[chip->offset + i] && names[chip->offset + i][0])
gdev->descs[i].name = names[chip->offset + i];
}
kfree(names);
return 0;
}
static unsigned long *gpiochip_allocate_mask(struct gpio_chip *gc)
{
unsigned long *p;
p = bitmap_alloc(gc->ngpio, GFP_KERNEL);
if (!p)
return NULL;
/* Assume by default all GPIOs are valid */
bitmap_fill(p, gc->ngpio);
return p;
}
static void gpiochip_free_mask(unsigned long **p)
{
bitmap_free(*p);
*p = NULL;
}
static unsigned int gpiochip_count_reserved_ranges(struct gpio_chip *gc)
{
struct device *dev = &gc->gpiodev->dev;
int size;
/* Format is "start, count, ..." */
size = device_property_count_u32(dev, "gpio-reserved-ranges");
if (size > 0 && size % 2 == 0)
return size;
return 0;
}
static int gpiochip_apply_reserved_ranges(struct gpio_chip *gc)
{
struct device *dev = &gc->gpiodev->dev;
unsigned int size;
u32 *ranges;
int ret;
size = gpiochip_count_reserved_ranges(gc);
if (size == 0)
return 0;
ranges = kmalloc_array(size, sizeof(*ranges), GFP_KERNEL);
if (!ranges)
return -ENOMEM;
ret = device_property_read_u32_array(dev, "gpio-reserved-ranges",
ranges, size);
if (ret) {
kfree(ranges);
return ret;
}
while (size) {
u32 count = ranges[--size];
u32 start = ranges[--size];
if (start >= gc->ngpio || start + count > gc->ngpio)
continue;
bitmap_clear(gc->valid_mask, start, count);
}
kfree(ranges);
return 0;
}
static int gpiochip_init_valid_mask(struct gpio_chip *gc)
{
int ret;
if (!(gpiochip_count_reserved_ranges(gc) || gc->init_valid_mask))
return 0;
gc->valid_mask = gpiochip_allocate_mask(gc);
if (!gc->valid_mask)
return -ENOMEM;
ret = gpiochip_apply_reserved_ranges(gc);
if (ret)
return ret;
if (gc->init_valid_mask)
return gc->init_valid_mask(gc,
gc->valid_mask,
gc->ngpio);
return 0;
}
static void gpiochip_free_valid_mask(struct gpio_chip *gc)
{
gpiochip_free_mask(&gc->valid_mask);
}
static int gpiochip_add_pin_ranges(struct gpio_chip *gc)
{
/*
* Device Tree platforms are supposed to use "gpio-ranges"
* property. This check ensures that the ->add_pin_ranges()
* won't be called for them.
*/
if (device_property_present(&gc->gpiodev->dev, "gpio-ranges"))
return 0;
if (gc->add_pin_ranges)
return gc->add_pin_ranges(gc);
return 0;
}
bool gpiochip_line_is_valid(const struct gpio_chip *gc,
unsigned int offset)
{
/* No mask means all valid */
if (likely(!gc->valid_mask))
return true;
return test_bit(offset, gc->valid_mask);
}
EXPORT_SYMBOL_GPL(gpiochip_line_is_valid);
static void gpiodev_release(struct device *dev)
{
struct gpio_device *gdev = to_gpio_device(dev);
unsigned int i;
for (i = 0; i < gdev->ngpio; i++) cleanup_srcu_struct(&gdev->descs[i].srcu); ida_free(&gpio_ida, gdev->id);
kfree_const(gdev->label);
kfree(gdev->descs);
cleanup_srcu_struct(&gdev->srcu);
kfree(gdev);
}
static const struct device_type gpio_dev_type = {
.name = "gpio_chip",
.release = gpiodev_release,
};
#ifdef CONFIG_GPIO_CDEV
#define gcdev_register(gdev, devt) gpiolib_cdev_register((gdev), (devt))
#define gcdev_unregister(gdev) gpiolib_cdev_unregister((gdev))
#else
/*
* gpiolib_cdev_register() indirectly calls device_add(), which is still
* required even when cdev is not selected.
*/
#define gcdev_register(gdev, devt) device_add(&(gdev)->dev)
#define gcdev_unregister(gdev) device_del(&(gdev)->dev)
#endif
static int gpiochip_setup_dev(struct gpio_device *gdev)
{
struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev);
int ret;
device_initialize(&gdev->dev);
/*
* If fwnode doesn't belong to another device, it's safe to clear its
* initialized flag.
*/
if (fwnode && !fwnode->dev)
fwnode_dev_initialized(fwnode, false);
ret = gcdev_register(gdev, gpio_devt);
if (ret)
return ret;
ret = gpiochip_sysfs_register(gdev);
if (ret)
goto err_remove_device;
dev_dbg(&gdev->dev, "registered GPIOs %d to %d on %s\n", gdev->base,
gdev->base + gdev->ngpio - 1, gdev->label);
return 0;
err_remove_device:
gcdev_unregister(gdev);
return ret;
}
static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog)
{
struct gpio_desc *desc;
int rv;
desc = gpiochip_get_desc(gc, hog->chip_hwnum);
if (IS_ERR(desc)) {
chip_err(gc, "%s: unable to get GPIO desc: %ld\n", __func__,
PTR_ERR(desc));
return;
}
rv = gpiod_hog(desc, hog->line_name, hog->lflags, hog->dflags);
if (rv)
gpiod_err(desc, "%s: unable to hog GPIO line (%s:%u): %d\n",
__func__, gc->label, hog->chip_hwnum, rv);
}
static void machine_gpiochip_add(struct gpio_chip *gc)
{
struct gpiod_hog *hog;
mutex_lock(&gpio_machine_hogs_mutex);
list_for_each_entry(hog, &gpio_machine_hogs, list) {
if (!strcmp(gc->label, hog->chip_label))
gpiochip_machine_hog(gc, hog);
}
mutex_unlock(&gpio_machine_hogs_mutex);
}
static void gpiochip_setup_devs(void)
{
struct gpio_device *gdev;
int ret;
guard(srcu)(&gpio_devices_srcu);
list_for_each_entry_srcu(gdev, &gpio_devices, list,
srcu_read_lock_held(&gpio_devices_srcu)) {
ret = gpiochip_setup_dev(gdev);
if (ret)
dev_err(&gdev->dev,
"Failed to initialize gpio device (%d)\n", ret);
}
}
static void gpiochip_set_data(struct gpio_chip *gc, void *data)
{
gc->gpiodev->data = data;
}
/**
* gpiochip_get_data() - get per-subdriver data for the chip
* @gc: GPIO chip
*
* Returns:
* The per-subdriver data for the chip.
*/
void *gpiochip_get_data(struct gpio_chip *gc)
{
return gc->gpiodev->data;
}
EXPORT_SYMBOL_GPL(gpiochip_get_data);
int gpiochip_get_ngpios(struct gpio_chip *gc, struct device *dev)
{
u32 ngpios = gc->ngpio;
int ret;
if (ngpios == 0) {
ret = device_property_read_u32(dev, "ngpios", &ngpios);
if (ret == -ENODATA)
/*
* -ENODATA means that there is no property found and
* we want to issue the error message to the user.
* Besides that, we want to return different error code
* to state that supplied value is not valid.
*/
ngpios = 0;
else if (ret)
return ret;
gc->ngpio = ngpios;
}
if (gc->ngpio == 0) {
chip_err(gc, "tried to insert a GPIO chip with zero lines\n");
return -EINVAL;
}
if (gc->ngpio > FASTPATH_NGPIO)
chip_warn(gc, "line cnt %u is greater than fast path cnt %u\n",
gc->ngpio, FASTPATH_NGPIO);
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_get_ngpios);
int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data,
struct lock_class_key *lock_key,
struct lock_class_key *request_key)
{
struct gpio_device *gdev;
unsigned int desc_index;
int base = 0;
int ret = 0;
/*
* First: allocate and populate the internal stat container, and
* set up the struct device.
*/
gdev = kzalloc(sizeof(*gdev), GFP_KERNEL);
if (!gdev)
return -ENOMEM;
gdev->dev.type = &gpio_dev_type;
gdev->dev.bus = &gpio_bus_type;
gdev->dev.parent = gc->parent;
rcu_assign_pointer(gdev->chip, gc);
gc->gpiodev = gdev;
gpiochip_set_data(gc, data);
/*
* If the calling driver did not initialize firmware node,
* do it here using the parent device, if any.
*/
if (gc->fwnode)
device_set_node(&gdev->dev, gc->fwnode);
else if (gc->parent)
device_set_node(&gdev->dev, dev_fwnode(gc->parent));
gdev->id = ida_alloc(&gpio_ida, GFP_KERNEL);
if (gdev->id < 0) {
ret = gdev->id;
goto err_free_gdev;
}
ret = dev_set_name(&gdev->dev, GPIOCHIP_NAME "%d", gdev->id);
if (ret)
goto err_free_ida;
if (gc->parent && gc->parent->driver)
gdev->owner = gc->parent->driver->owner;
else if (gc->owner)
/* TODO: remove chip->owner */
gdev->owner = gc->owner;
else
gdev->owner = THIS_MODULE;
ret = gpiochip_get_ngpios(gc, &gdev->dev);
if (ret)
goto err_free_dev_name;
gdev->descs = kcalloc(gc->ngpio, sizeof(*gdev->descs), GFP_KERNEL);
if (!gdev->descs) {
ret = -ENOMEM;
goto err_free_dev_name;
}
gdev->label = kstrdup_const(gc->label ?: "unknown", GFP_KERNEL);
if (!gdev->label) {
ret = -ENOMEM;
goto err_free_descs;
}
gdev->ngpio = gc->ngpio;
gdev->can_sleep = gc->can_sleep;
scoped_guard(mutex, &gpio_devices_lock) {
/*
* TODO: this allocates a Linux GPIO number base in the global
* GPIO numberspace for this chip. In the long run we want to
* get *rid* of this numberspace and use only descriptors, but
* it may be a pipe dream. It will not happen before we get rid
* of the sysfs interface anyways.
*/
base = gc->base;
if (base < 0) {
base = gpiochip_find_base_unlocked(gc->ngpio);
if (base < 0) {
ret = base;
base = 0;
goto err_free_label;
}
/*
* TODO: it should not be necessary to reflect the
* assigned base outside of the GPIO subsystem. Go over
* drivers and see if anyone makes use of this, else
* drop this and assign a poison instead.
*/
gc->base = base;
} else {
dev_warn(&gdev->dev,
"Static allocation of GPIO base is deprecated, use dynamic allocation.\n");
}
gdev->base = base;
ret = gpiodev_add_to_list_unlocked(gdev);
if (ret) {
chip_err(gc, "GPIO integer space overlap, cannot add chip\n");
goto err_free_label;
}
}
for (desc_index = 0; desc_index < gc->ngpio; desc_index++)
gdev->descs[desc_index].gdev = gdev;
BLOCKING_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier);
BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier);
ret = init_srcu_struct(&gdev->srcu);
if (ret)
goto err_remove_from_list;
#ifdef CONFIG_PINCTRL
INIT_LIST_HEAD(&gdev->pin_ranges);
#endif
if (gc->names) {
ret = gpiochip_set_desc_names(gc);
if (ret)
goto err_cleanup_gdev_srcu;
}
ret = gpiochip_set_names(gc);
if (ret)
goto err_cleanup_gdev_srcu;
ret = gpiochip_init_valid_mask(gc);
if (ret)
goto err_cleanup_gdev_srcu;
for (desc_index = 0; desc_index < gc->ngpio; desc_index++) {
struct gpio_desc *desc = &gdev->descs[desc_index];
ret = init_srcu_struct(&desc->srcu);
if (ret)
goto err_cleanup_desc_srcu;
if (gc->get_direction && gpiochip_line_is_valid(gc, desc_index)) {
assign_bit(FLAG_IS_OUT,
&desc->flags, !gc->get_direction(gc, desc_index));
} else {
assign_bit(FLAG_IS_OUT,
&desc->flags, !gc->direction_input);
}
}
ret = of_gpiochip_add(gc);
if (ret)
goto err_cleanup_desc_srcu;
ret = gpiochip_add_pin_ranges(gc);
if (ret)
goto err_remove_of_chip;
acpi_gpiochip_add(gc);
machine_gpiochip_add(gc);
ret = gpiochip_irqchip_init_valid_mask(gc);
if (ret)
goto err_free_hogs;
ret = gpiochip_irqchip_init_hw(gc);
if (ret)
goto err_remove_irqchip_mask;
ret = gpiochip_add_irqchip(gc, lock_key, request_key);
if (ret)
goto err_remove_irqchip_mask;
/*
* By first adding the chardev, and then adding the device,
* we get a device node entry in sysfs under
* /sys/bus/gpio/devices/gpiochipN/dev that can be used for
* coldplug of device nodes and other udev business.
* We can do this only if gpiolib has been initialized.
* Otherwise, defer until later.
*/
if (gpiolib_initialized) {
ret = gpiochip_setup_dev(gdev);
if (ret)
goto err_remove_irqchip;
}
return 0;
err_remove_irqchip:
gpiochip_irqchip_remove(gc);
err_remove_irqchip_mask:
gpiochip_irqchip_free_valid_mask(gc);
err_free_hogs:
gpiochip_free_hogs(gc);
acpi_gpiochip_remove(gc);
gpiochip_remove_pin_ranges(gc);
err_remove_of_chip:
of_gpiochip_remove(gc);
err_cleanup_desc_srcu:
while (desc_index--)
cleanup_srcu_struct(&gdev->descs[desc_index].srcu);
gpiochip_free_valid_mask(gc);
err_cleanup_gdev_srcu:
cleanup_srcu_struct(&gdev->srcu);
err_remove_from_list:
scoped_guard(mutex, &gpio_devices_lock)
list_del_rcu(&gdev->list);
synchronize_srcu(&gpio_devices_srcu);
if (gdev->dev.release) {
/* release() has been registered by gpiochip_setup_dev() */
gpio_device_put(gdev);
goto err_print_message;
}
err_free_label:
kfree_const(gdev->label);
err_free_descs:
kfree(gdev->descs);
err_free_dev_name:
kfree(dev_name(&gdev->dev));
err_free_ida:
ida_free(&gpio_ida, gdev->id);
err_free_gdev:
kfree(gdev);
err_print_message:
/* failures here can mean systems won't boot... */
if (ret != -EPROBE_DEFER) {
pr_err("%s: GPIOs %d..%d (%s) failed to register, %d\n", __func__,
base, base + (int)gc->ngpio - 1,
gc->label ? : "generic", ret);
}
return ret;
}
EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key);
/**
* gpiochip_remove() - unregister a gpio_chip
* @gc: the chip to unregister
*
* A gpio_chip with any GPIOs still requested may not be removed.
*/
void gpiochip_remove(struct gpio_chip *gc)
{
struct gpio_device *gdev = gc->gpiodev;
/* FIXME: should the legacy sysfs handling be moved to gpio_device? */
gpiochip_sysfs_unregister(gdev);
gpiochip_free_hogs(gc);
scoped_guard(mutex, &gpio_devices_lock)
list_del_rcu(&gdev->list);
synchronize_srcu(&gpio_devices_srcu);
/* Numb the device, cancelling all outstanding operations */
rcu_assign_pointer(gdev->chip, NULL);
synchronize_srcu(&gdev->srcu);
gpiochip_irqchip_remove(gc);
acpi_gpiochip_remove(gc);
of_gpiochip_remove(gc);
gpiochip_remove_pin_ranges(gc);
gpiochip_free_valid_mask(gc);
/*
* We accept no more calls into the driver from this point, so
* NULL the driver data pointer.
*/
gpiochip_set_data(gc, NULL);
/*
* The gpiochip side puts its use of the device to rest here:
* if there are no userspace clients, the chardev and device will
* be removed, else it will be dangling until the last user is
* gone.
*/
gcdev_unregister(gdev);
gpio_device_put(gdev);
}
EXPORT_SYMBOL_GPL(gpiochip_remove);
/**
* gpio_device_find() - find a specific GPIO device
* @data: data to pass to match function
* @match: Callback function to check gpio_chip
*
* Returns:
* New reference to struct gpio_device.
*
* Similar to bus_find_device(). It returns a reference to a gpio_device as
* determined by a user supplied @match callback. The callback should return
* 0 if the device doesn't match and non-zero if it does. If the callback
* returns non-zero, this function will return to the caller and not iterate
* over any more gpio_devices.
*
* The callback takes the GPIO chip structure as argument. During the execution
* of the callback function the chip is protected from being freed. TODO: This
* actually has yet to be implemented.
*
* If the function returns non-NULL, the returned reference must be freed by
* the caller using gpio_device_put().
*/
struct gpio_device *gpio_device_find(const void *data,
int (*match)(struct gpio_chip *gc,
const void *data))
{
struct gpio_device *gdev;
struct gpio_chip *gc;
/*
* Not yet but in the future the spinlock below will become a mutex.
* Annotate this function before anyone tries to use it in interrupt
* context like it happened with gpiochip_find().
*/
might_sleep();
guard(srcu)(&gpio_devices_srcu);
list_for_each_entry_srcu(gdev, &gpio_devices, list,
srcu_read_lock_held(&gpio_devices_srcu)) {
if (!device_is_registered(&gdev->dev))
continue;
guard(srcu)(&gdev->srcu);
gc = srcu_dereference(gdev->chip, &gdev->srcu);
if (gc && match(gc, data))
return gpio_device_get(gdev);
}
return NULL;
}
EXPORT_SYMBOL_GPL(gpio_device_find);
static int gpio_chip_match_by_label(struct gpio_chip *gc, const void *label)
{
return gc->label && !strcmp(gc->label, label);
}
/**
* gpio_device_find_by_label() - wrapper around gpio_device_find() finding the
* GPIO device by its backing chip's label
* @label: Label to lookup
*
* Returns:
* Reference to the GPIO device or NULL. Reference must be released with
* gpio_device_put().
*/
struct gpio_device *gpio_device_find_by_label(const char *label)
{
return gpio_device_find((void *)label, gpio_chip_match_by_label);
}
EXPORT_SYMBOL_GPL(gpio_device_find_by_label);
static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, const void *fwnode)
{
return device_match_fwnode(&gc->gpiodev->dev, fwnode);
}
/**
* gpio_device_find_by_fwnode() - wrapper around gpio_device_find() finding
* the GPIO device by its fwnode
* @fwnode: Firmware node to lookup
*
* Returns:
* Reference to the GPIO device or NULL. Reference must be released with
* gpio_device_put().
*/
struct gpio_device *gpio_device_find_by_fwnode(const struct fwnode_handle *fwnode)
{
return gpio_device_find((void *)fwnode, gpio_chip_match_by_fwnode);
}
EXPORT_SYMBOL_GPL(gpio_device_find_by_fwnode);
/**
* gpio_device_get() - Increase the reference count of this GPIO device
* @gdev: GPIO device to increase the refcount for
*
* Returns:
* Pointer to @gdev.
*/
struct gpio_device *gpio_device_get(struct gpio_device *gdev)
{
return to_gpio_device(get_device(&gdev->dev));
}
EXPORT_SYMBOL_GPL(gpio_device_get);
/**
* gpio_device_put() - Decrease the reference count of this GPIO device and
* possibly free all resources associated with it.
* @gdev: GPIO device to decrease the reference count for
*/
void gpio_device_put(struct gpio_device *gdev)
{
put_device(&gdev->dev);
}
EXPORT_SYMBOL_GPL(gpio_device_put);
/**
* gpio_device_to_device() - Retrieve the address of the underlying struct
* device.
* @gdev: GPIO device for which to return the address.
*
* This does not increase the reference count of the GPIO device nor the
* underlying struct device.
*
* Returns:
* Address of struct device backing this GPIO device.
*/
struct device *gpio_device_to_device(struct gpio_device *gdev)
{
return &gdev->dev;
}
EXPORT_SYMBOL_GPL(gpio_device_to_device);
#ifdef CONFIG_GPIOLIB_IRQCHIP
/*
* The following is irqchip helper code for gpiochips.
*/
static int gpiochip_irqchip_init_hw(struct gpio_chip *gc)
{
struct gpio_irq_chip *girq = &gc->irq;
if (!girq->init_hw)
return 0;
return girq->init_hw(gc);
}
static int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gc)
{
struct gpio_irq_chip *girq = &gc->irq;
if (!girq->init_valid_mask)
return 0;
girq->valid_mask = gpiochip_allocate_mask(gc);
if (!girq->valid_mask)
return -ENOMEM;
girq->init_valid_mask(gc, girq->valid_mask, gc->ngpio);
return 0;
}
static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc)
{
gpiochip_free_mask(&gc->irq.valid_mask);
}
static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc,
unsigned int offset)
{
if (!gpiochip_line_is_valid(gc, offset))
return false;
/* No mask means all valid */
if (likely(!gc->irq.valid_mask))
return true;
return test_bit(offset, gc->irq.valid_mask);
}
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
/**
* gpiochip_set_hierarchical_irqchip() - connects a hierarchical irqchip
* to a gpiochip
* @gc: the gpiochip to set the irqchip hierarchical handler to
* @irqchip: the irqchip to handle this level of the hierarchy, the interrupt
* will then percolate up to the parent
*/
static void gpiochip_set_hierarchical_irqchip(struct gpio_chip *gc,
struct irq_chip *irqchip)
{
/* DT will deal with mapping each IRQ as we go along */
if (is_of_node(gc->irq.fwnode))
return;
/*
* This is for legacy and boardfile "irqchip" fwnodes: allocate
* irqs upfront instead of dynamically since we don't have the
* dynamic type of allocation that hardware description languages
* provide. Once all GPIO drivers using board files are gone from
* the kernel we can delete this code, but for a transitional period
* it is necessary to keep this around.
*/
if (is_fwnode_irqchip(gc->irq.fwnode)) {
int i;
int ret;
for (i = 0; i < gc->ngpio; i++) {
struct irq_fwspec fwspec;
unsigned int parent_hwirq;
unsigned int parent_type;
struct gpio_irq_chip *girq = &gc->irq;
/*
* We call the child to parent translation function
* only to check if the child IRQ is valid or not.
* Just pick the rising edge type here as that is what
* we likely need to support.
*/
ret = girq->child_to_parent_hwirq(gc, i,
IRQ_TYPE_EDGE_RISING,
&parent_hwirq,
&parent_type);
if (ret) {
chip_err(gc, "skip set-up on hwirq %d\n",
i);
continue;
}
fwspec.fwnode = gc->irq.fwnode;
/* This is the hwirq for the GPIO line side of things */
fwspec.param[0] = girq->child_offset_to_irq(gc, i);
/* Just pick something */
fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
fwspec.param_count = 2;
ret = irq_domain_alloc_irqs(gc->irq.domain, 1,
NUMA_NO_NODE, &fwspec);
if (ret < 0) {
chip_err(gc,
"can not allocate irq for GPIO line %d parent hwirq %d in hierarchy domain: %d\n",
i, parent_hwirq,
ret);
}
}
}
chip_err(gc, "%s unknown fwnode type proceed anyway\n", __func__);
return;
}
static int gpiochip_hierarchy_irq_domain_translate(struct irq_domain *d,
struct irq_fwspec *fwspec,
unsigned long *hwirq,
unsigned int *type)
{
/* We support standard DT translation */
if (is_of_node(fwspec->fwnode) && fwspec->param_count == 2) {
return irq_domain_translate_twocell(d, fwspec, hwirq, type);
}
/* This is for board files and others not using DT */
if (is_fwnode_irqchip(fwspec->fwnode)) {
int ret;
ret = irq_domain_translate_twocell(d, fwspec, hwirq, type);
if (ret)
return ret;
WARN_ON(*type == IRQ_TYPE_NONE);
return 0;
}
return -EINVAL;
}
static int gpiochip_hierarchy_irq_domain_alloc(struct irq_domain *d,
unsigned int irq,
unsigned int nr_irqs,
void *data)
{
struct gpio_chip *gc = d->host_data;
irq_hw_number_t hwirq;
unsigned int type = IRQ_TYPE_NONE;
struct irq_fwspec *fwspec = data;
union gpio_irq_fwspec gpio_parent_fwspec = {};
unsigned int parent_hwirq;
unsigned int parent_type;
struct gpio_irq_chip *girq = &gc->irq;
int ret;
/*
* The nr_irqs parameter is always one except for PCI multi-MSI
* so this should not happen.
*/
WARN_ON(nr_irqs != 1);
ret = gc->irq.child_irq_domain_ops.translate(d, fwspec, &hwirq, &type);
if (ret)
return ret;
chip_dbg(gc, "allocate IRQ %d, hwirq %lu\n", irq, hwirq);
ret = girq->child_to_parent_hwirq(gc, hwirq, type,
&parent_hwirq, &parent_type);
if (ret) {
chip_err(gc, "can't look up hwirq %lu\n", hwirq);
return ret;
}
chip_dbg(gc, "found parent hwirq %u\n", parent_hwirq);
/*
* We set handle_bad_irq because the .set_type() should
* always be invoked and set the right type of handler.
*/
irq_domain_set_info(d,
irq,
hwirq,
gc->irq.chip,
gc,
girq->handler,
NULL, NULL);
irq_set_probe(irq);
/* This parent only handles asserted level IRQs */
ret = girq->populate_parent_alloc_arg(gc, &gpio_parent_fwspec,
parent_hwirq, parent_type);
if (ret)
return ret;
chip_dbg(gc, "alloc_irqs_parent for %d parent hwirq %d\n",
irq, parent_hwirq);
irq_set_lockdep_class(irq, gc->irq.lock_key, gc->irq.request_key);
ret = irq_domain_alloc_irqs_parent(d, irq, 1, &gpio_parent_fwspec);
/*
* If the parent irqdomain is msi, the interrupts have already
* been allocated, so the EEXIST is good.
*/
if (irq_domain_is_msi(d->parent) && (ret == -EEXIST))
ret = 0;
if (ret)
chip_err(gc,
"failed to allocate parent hwirq %d for hwirq %lu\n",
parent_hwirq, hwirq);
return ret;
}
static unsigned int gpiochip_child_offset_to_irq_noop(struct gpio_chip *gc,
unsigned int offset)
{
return offset;
}
/**
* gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ
* @domain: The IRQ domain used by this IRQ chip
* @data: Outermost irq_data associated with the IRQ
* @reserve: If set, only reserve an interrupt vector instead of assigning one
*
* This function is a wrapper that calls gpiochip_lock_as_irq() and is to be
* used as the activate function for the &struct irq_domain_ops. The host_data
* for the IRQ domain must be the &struct gpio_chip.
*/
static int gpiochip_irq_domain_activate(struct irq_domain *domain,
struct irq_data *data, bool reserve)
{
struct gpio_chip *gc = domain->host_data;
unsigned int hwirq = irqd_to_hwirq(data);
return gpiochip_lock_as_irq(gc, hwirq);
}
/**
* gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ
* @domain: The IRQ domain used by this IRQ chip
* @data: Outermost irq_data associated with the IRQ
*
* This function is a wrapper that will call gpiochip_unlock_as_irq() and is to
* be used as the deactivate function for the &struct irq_domain_ops. The
* host_data for the IRQ domain must be the &struct gpio_chip.
*/
static void gpiochip_irq_domain_deactivate(struct irq_domain *domain,
struct irq_data *data)
{
struct gpio_chip *gc = domain->host_data;
unsigned int hwirq = irqd_to_hwirq(data);
return gpiochip_unlock_as_irq(gc, hwirq);
}
static void gpiochip_hierarchy_setup_domain_ops(struct irq_domain_ops *ops)
{
ops->activate = gpiochip_irq_domain_activate;
ops->deactivate = gpiochip_irq_domain_deactivate;
ops->alloc = gpiochip_hierarchy_irq_domain_alloc;
/*
* We only allow overriding the translate() and free() functions for
* hierarchical chips, and this should only be done if the user
* really need something other than 1:1 translation for translate()
* callback and free if user wants to free up any resources which
* were allocated during callbacks, for example populate_parent_alloc_arg.
*/
if (!ops->translate)
ops->translate = gpiochip_hierarchy_irq_domain_translate;
if (!ops->free)
ops->free = irq_domain_free_irqs_common;
}
static struct irq_domain *gpiochip_hierarchy_create_domain(struct gpio_chip *gc)
{
struct irq_domain *domain;
if (!gc->irq.child_to_parent_hwirq ||
!gc->irq.fwnode) {
chip_err(gc, "missing irqdomain vital data\n");
return ERR_PTR(-EINVAL);
}
if (!gc->irq.child_offset_to_irq)
gc->irq.child_offset_to_irq = gpiochip_child_offset_to_irq_noop;
if (!gc->irq.populate_parent_alloc_arg)
gc->irq.populate_parent_alloc_arg =
gpiochip_populate_parent_fwspec_twocell;
gpiochip_hierarchy_setup_domain_ops(&gc->irq.child_irq_domain_ops);
domain = irq_domain_create_hierarchy(
gc->irq.parent_domain,
0,
gc->ngpio,
gc->irq.fwnode,
&gc->irq.child_irq_domain_ops,
gc);
if (!domain)
return ERR_PTR(-ENOMEM);
gpiochip_set_hierarchical_irqchip(gc, gc->irq.chip);
return domain;
}
static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc)
{
return !!gc->irq.parent_domain;
}
int gpiochip_populate_parent_fwspec_twocell(struct gpio_chip *gc,
union gpio_irq_fwspec *gfwspec,
unsigned int parent_hwirq,
unsigned int parent_type)
{
struct irq_fwspec *fwspec = &gfwspec->fwspec;
fwspec->fwnode = gc->irq.parent_domain->fwnode;
fwspec->param_count = 2;
fwspec->param[0] = parent_hwirq;
fwspec->param[1] = parent_type;
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_populate_parent_fwspec_twocell);
int gpiochip_populate_parent_fwspec_fourcell(struct gpio_chip *gc,
union gpio_irq_fwspec *gfwspec,
unsigned int parent_hwirq,
unsigned int parent_type)
{
struct irq_fwspec *fwspec = &gfwspec->fwspec;
fwspec->fwnode = gc->irq.parent_domain->fwnode;
fwspec->param_count = 4;
fwspec->param[0] = 0;
fwspec->param[1] = parent_hwirq;
fwspec->param[2] = 0;
fwspec->param[3] = parent_type;
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_populate_parent_fwspec_fourcell);
#else
static struct irq_domain *gpiochip_hierarchy_create_domain(struct gpio_chip *gc)
{
return ERR_PTR(-EINVAL);
}
static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc)
{
return false;
}
#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
/**
* gpiochip_irq_map() - maps an IRQ into a GPIO irqchip
* @d: the irqdomain used by this irqchip
* @irq: the global irq number used by this GPIO irqchip irq
* @hwirq: the local IRQ/GPIO line offset on this gpiochip
*
* This function will set up the mapping for a certain IRQ line on a
* gpiochip by assigning the gpiochip as chip data, and using the irqchip
* stored inside the gpiochip.
*/
static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
irq_hw_number_t hwirq)
{
struct gpio_chip *gc = d->host_data;
int ret = 0;
if (!gpiochip_irqchip_irq_valid(gc, hwirq))
return -ENXIO;
irq_set_chip_data(irq, gc);
/*
* This lock class tells lockdep that GPIO irqs are in a different
* category than their parents, so it won't report false recursion.
*/
irq_set_lockdep_class(irq, gc->irq.lock_key, gc->irq.request_key);
irq_set_chip_and_handler(irq, gc->irq.chip, gc->irq.handler);
/* Chips that use nested thread handlers have them marked */
if (gc->irq.threaded)
irq_set_nested_thread(irq, 1);
irq_set_noprobe(irq);
if (gc->irq.num_parents == 1)
ret = irq_set_parent(irq, gc->irq.parents[0]);
else if (gc->irq.map)
ret = irq_set_parent(irq, gc->irq.map[hwirq]);
if (ret < 0)
return ret;
/*
* No set-up of the hardware will happen if IRQ_TYPE_NONE
* is passed as default type.
*/
if (gc->irq.default_type != IRQ_TYPE_NONE)
irq_set_irq_type(irq, gc->irq.default_type);
return 0;
}
static void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq)
{
struct gpio_chip *gc = d->host_data;
if (gc->irq.threaded)
irq_set_nested_thread(irq, 0);
irq_set_chip_and_handler(irq, NULL, NULL);
irq_set_chip_data(irq, NULL);
}
static const struct irq_domain_ops gpiochip_domain_ops = {
.map = gpiochip_irq_map,
.unmap = gpiochip_irq_unmap,
/* Virtually all GPIO irqchips are twocell:ed */
.xlate = irq_domain_xlate_twocell,
};
static struct irq_domain *gpiochip_simple_create_domain(struct gpio_chip *gc)
{
struct fwnode_handle *fwnode = dev_fwnode(&gc->gpiodev->dev);
struct irq_domain *domain;
domain = irq_domain_create_simple(fwnode, gc->ngpio, gc->irq.first,
&gpiochip_domain_ops, gc);
if (!domain)
return ERR_PTR(-EINVAL);
return domain;
}
static int gpiochip_to_irq(struct gpio_chip *gc, unsigned int offset)
{
struct irq_domain *domain = gc->irq.domain;
#ifdef CONFIG_GPIOLIB_IRQCHIP
/*
* Avoid race condition with other code, which tries to lookup
* an IRQ before the irqchip has been properly registered,
* i.e. while gpiochip is still being brought up.
*/
if (!gc->irq.initialized)
return -EPROBE_DEFER;
#endif
if (!gpiochip_irqchip_irq_valid(gc, offset))
return -ENXIO;
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
if (irq_domain_is_hierarchy(domain)) {
struct irq_fwspec spec;
spec.fwnode = domain->fwnode;
spec.param_count = 2;
spec.param[0] = gc->irq.child_offset_to_irq(gc, offset);
spec.param[1] = IRQ_TYPE_NONE;
return irq_create_fwspec_mapping(&spec);
}
#endif
return irq_create_mapping(domain, offset);
}
int gpiochip_irq_reqres(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
unsigned int hwirq = irqd_to_hwirq(d);
return gpiochip_reqres_irq(gc, hwirq);
}
EXPORT_SYMBOL(gpiochip_irq_reqres);
void gpiochip_irq_relres(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
unsigned int hwirq = irqd_to_hwirq(d);
gpiochip_relres_irq(gc, hwirq);
}
EXPORT_SYMBOL(gpiochip_irq_relres);
static void gpiochip_irq_mask(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
unsigned int hwirq = irqd_to_hwirq(d);
if (gc->irq.irq_mask)
gc->irq.irq_mask(d);
gpiochip_disable_irq(gc, hwirq);
}
static void gpiochip_irq_unmask(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
unsigned int hwirq = irqd_to_hwirq(d);
gpiochip_enable_irq(gc, hwirq);
if (gc->irq.irq_unmask)
gc->irq.irq_unmask(d);
}
static void gpiochip_irq_enable(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
unsigned int hwirq = irqd_to_hwirq(d);
gpiochip_enable_irq(gc, hwirq);
gc->irq.irq_enable(d);
}
static void gpiochip_irq_disable(struct irq_data *d)
{
struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
unsigned int hwirq = irqd_to_hwirq(d);
gc->irq.irq_disable(d);
gpiochip_disable_irq(gc, hwirq);
}
static void gpiochip_set_irq_hooks(struct gpio_chip *gc)
{
struct irq_chip *irqchip = gc->irq.chip;
if (irqchip->flags & IRQCHIP_IMMUTABLE)
return;
chip_warn(gc, "not an immutable chip, please consider fixing it!\n");
if (!irqchip->irq_request_resources &&
!irqchip->irq_release_resources) {
irqchip->irq_request_resources = gpiochip_irq_reqres;
irqchip->irq_release_resources = gpiochip_irq_relres;
}
if (WARN_ON(gc->irq.irq_enable))
return;
/* Check if the irqchip already has this hook... */
if (irqchip->irq_enable == gpiochip_irq_enable ||
irqchip->irq_mask == gpiochip_irq_mask) {
/*
* ...and if so, give a gentle warning that this is bad
* practice.
*/
chip_info(gc,
"detected irqchip that is shared with multiple gpiochips: please fix the driver.\n");
return;
}
if (irqchip->irq_disable) {
gc->irq.irq_disable = irqchip->irq_disable;
irqchip->irq_disable = gpiochip_irq_disable;
} else {
gc->irq.irq_mask = irqchip->irq_mask;
irqchip->irq_mask = gpiochip_irq_mask;
}
if (irqchip->irq_enable) {
gc->irq.irq_enable = irqchip->irq_enable;
irqchip->irq_enable = gpiochip_irq_enable;
} else {
gc->irq.irq_unmask = irqchip->irq_unmask;
irqchip->irq_unmask = gpiochip_irq_unmask;
}
}
static int gpiochip_irqchip_add_allocated_domain(struct gpio_chip *gc,
struct irq_domain *domain,
bool allocated_externally)
{
if (!domain)
return -EINVAL;
if (gc->to_irq)
chip_warn(gc, "to_irq is redefined in %s and you shouldn't rely on it\n", __func__);
gc->to_irq = gpiochip_to_irq;
gc->irq.domain = domain;
gc->irq.domain_is_allocated_externally = allocated_externally;
/*
* Using barrier() here to prevent compiler from reordering
* gc->irq.initialized before adding irqdomain.
*/
barrier();
gc->irq.initialized = true;
return 0;
}
/**
* gpiochip_add_irqchip() - adds an IRQ chip to a GPIO chip
* @gc: the GPIO chip to add the IRQ chip to
* @lock_key: lockdep class for IRQ lock
* @request_key: lockdep class for IRQ request
*/
static int gpiochip_add_irqchip(struct gpio_chip *gc,
struct lock_class_key *lock_key,
struct lock_class_key *request_key)
{
struct fwnode_handle *fwnode = dev_fwnode(&gc->gpiodev->dev);
struct irq_chip *irqchip = gc->irq.chip;
struct irq_domain *domain;
unsigned int type;
unsigned int i;
int ret;
if (!irqchip)
return 0;
if (gc->irq.parent_handler && gc->can_sleep) {
chip_err(gc, "you cannot have chained interrupts on a chip that may sleep\n");
return -EINVAL;
}
type = gc->irq.default_type;
/*
* Specifying a default trigger is a terrible idea if DT or ACPI is
* used to configure the interrupts, as you may end up with
* conflicting triggers. Tell the user, and reset to NONE.
*/
if (WARN(fwnode && type != IRQ_TYPE_NONE,
"%pfw: Ignoring %u default trigger\n", fwnode, type))
type = IRQ_TYPE_NONE;
gc->irq.default_type = type;
gc->irq.lock_key = lock_key;
gc->irq.request_key = request_key;
/* If a parent irqdomain is provided, let's build a hierarchy */
if (gpiochip_hierarchy_is_hierarchical(gc)) {
domain = gpiochip_hierarchy_create_domain(gc);
} else {
domain = gpiochip_simple_create_domain(gc);
}
if (IS_ERR(domain))
return PTR_ERR(domain);
if (gc->irq.parent_handler) {
for (i = 0; i < gc->irq.num_parents; i++) {
void *data;
if (gc->irq.per_parent_data)
data = gc->irq.parent_handler_data_array[i];
else
data = gc->irq.parent_handler_data ?: gc;
/*
* The parent IRQ chip is already using the chip_data
* for this IRQ chip, so our callbacks simply use the
* handler_data.
*/
irq_set_chained_handler_and_data(gc->irq.parents[i],
gc->irq.parent_handler,
data);
}
}
gpiochip_set_irq_hooks(gc);
ret = gpiochip_irqchip_add_allocated_domain(gc, domain, false);
if (ret)
return ret;
acpi_gpiochip_request_interrupts(gc);
return 0;
}
/**
* gpiochip_irqchip_remove() - removes an irqchip added to a gpiochip
* @gc: the gpiochip to remove the irqchip from
*
* This is called only from gpiochip_remove()
*/
static void gpiochip_irqchip_remove(struct gpio_chip *gc)
{
struct irq_chip *irqchip = gc->irq.chip;
unsigned int offset;
acpi_gpiochip_free_interrupts(gc);
if (irqchip && gc->irq.parent_handler) {
struct gpio_irq_chip *irq = &gc->irq;
unsigned int i;
for (i = 0; i < irq->num_parents; i++) irq_set_chained_handler_and_data(irq->parents[i],
NULL, NULL);
}
/* Remove all IRQ mappings and delete the domain */
if (!gc->irq.domain_is_allocated_externally && gc->irq.domain) {
unsigned int irq;
for (offset = 0; offset < gc->ngpio; offset++) { if (!gpiochip_irqchip_irq_valid(gc, offset))
continue;
irq = irq_find_mapping(gc->irq.domain, offset);
irq_dispose_mapping(irq);
}
irq_domain_remove(gc->irq.domain);
}
if (irqchip && !(irqchip->flags & IRQCHIP_IMMUTABLE)) { if (irqchip->irq_request_resources == gpiochip_irq_reqres) { irqchip->irq_request_resources = NULL;
irqchip->irq_release_resources = NULL;
}
if (irqchip->irq_enable == gpiochip_irq_enable) { irqchip->irq_enable = gc->irq.irq_enable;
irqchip->irq_disable = gc->irq.irq_disable;
}
}
gc->irq.irq_enable = NULL;
gc->irq.irq_disable = NULL;
gc->irq.chip = NULL;
gpiochip_irqchip_free_valid_mask(gc);
}
/**
* gpiochip_irqchip_add_domain() - adds an irqdomain to a gpiochip
* @gc: the gpiochip to add the irqchip to
* @domain: the irqdomain to add to the gpiochip
*
* This function adds an IRQ domain to the gpiochip.
*/
int gpiochip_irqchip_add_domain(struct gpio_chip *gc,
struct irq_domain *domain)
{
return gpiochip_irqchip_add_allocated_domain(gc, domain, true);
}
EXPORT_SYMBOL_GPL(gpiochip_irqchip_add_domain);
#else /* CONFIG_GPIOLIB_IRQCHIP */
static inline int gpiochip_add_irqchip(struct gpio_chip *gc,
struct lock_class_key *lock_key,
struct lock_class_key *request_key)
{
return 0;
}
static void gpiochip_irqchip_remove(struct gpio_chip *gc) {}
static inline int gpiochip_irqchip_init_hw(struct gpio_chip *gc)
{
return 0;
}
static inline int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gc)
{
return 0;
}
static inline void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc)
{ }
#endif /* CONFIG_GPIOLIB_IRQCHIP */
/**
* gpiochip_generic_request() - request the gpio function for a pin
* @gc: the gpiochip owning the GPIO
* @offset: the offset of the GPIO to request for GPIO function
*/
int gpiochip_generic_request(struct gpio_chip *gc, unsigned int offset)
{
#ifdef CONFIG_PINCTRL
if (list_empty(&gc->gpiodev->pin_ranges))
return 0;
#endif
return pinctrl_gpio_request(gc, offset);
}
EXPORT_SYMBOL_GPL(gpiochip_generic_request);
/**
* gpiochip_generic_free() - free the gpio function from a pin
* @gc: the gpiochip to request the gpio function for
* @offset: the offset of the GPIO to free from GPIO function
*/
void gpiochip_generic_free(struct gpio_chip *gc, unsigned int offset)
{
#ifdef CONFIG_PINCTRL
if (list_empty(&gc->gpiodev->pin_ranges))
return;
#endif
pinctrl_gpio_free(gc, offset);
}
EXPORT_SYMBOL_GPL(gpiochip_generic_free);
/**
* gpiochip_generic_config() - apply configuration for a pin
* @gc: the gpiochip owning the GPIO
* @offset: the offset of the GPIO to apply the configuration
* @config: the configuration to be applied
*/
int gpiochip_generic_config(struct gpio_chip *gc, unsigned int offset,
unsigned long config)
{
#ifdef CONFIG_PINCTRL
if (list_empty(&gc->gpiodev->pin_ranges))
return -ENOTSUPP;
#endif
return pinctrl_gpio_set_config(gc, offset, config);
}
EXPORT_SYMBOL_GPL(gpiochip_generic_config);
#ifdef CONFIG_PINCTRL
/**
* gpiochip_add_pingroup_range() - add a range for GPIO <-> pin mapping
* @gc: the gpiochip to add the range for
* @pctldev: the pin controller to map to
* @gpio_offset: the start offset in the current gpio_chip number space
* @pin_group: name of the pin group inside the pin controller
*
* Calling this function directly from a DeviceTree-supported
* pinctrl driver is DEPRECATED. Please see Section 2.1 of
* Documentation/devicetree/bindings/gpio/gpio.txt on how to
* bind pinctrl and gpio drivers via the "gpio-ranges" property.
*/
int gpiochip_add_pingroup_range(struct gpio_chip *gc,
struct pinctrl_dev *pctldev,
unsigned int gpio_offset, const char *pin_group)
{
struct gpio_pin_range *pin_range;
struct gpio_device *gdev = gc->gpiodev;
int ret;
pin_range = kzalloc(sizeof(*pin_range), GFP_KERNEL);
if (!pin_range) {
chip_err(gc, "failed to allocate pin ranges\n");
return -ENOMEM;
}
/* Use local offset as range ID */
pin_range->range.id = gpio_offset;
pin_range->range.gc = gc;
pin_range->range.name = gc->label;
pin_range->range.base = gdev->base + gpio_offset;
pin_range->pctldev = pctldev;
ret = pinctrl_get_group_pins(pctldev, pin_group,
&pin_range->range.pins,
&pin_range->range.npins);
if (ret < 0) {
kfree(pin_range);
return ret;
}
pinctrl_add_gpio_range(pctldev, &pin_range->range);
chip_dbg(gc, "created GPIO range %d->%d ==> %s PINGRP %s\n",
gpio_offset, gpio_offset + pin_range->range.npins - 1,
pinctrl_dev_get_devname(pctldev), pin_group);
list_add_tail(&pin_range->node, &gdev->pin_ranges);
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_add_pingroup_range);
/**
* gpiochip_add_pin_range() - add a range for GPIO <-> pin mapping
* @gc: the gpiochip to add the range for
* @pinctl_name: the dev_name() of the pin controller to map to
* @gpio_offset: the start offset in the current gpio_chip number space
* @pin_offset: the start offset in the pin controller number space
* @npins: the number of pins from the offset of each pin space (GPIO and
* pin controller) to accumulate in this range
*
* Returns:
* 0 on success, or a negative error-code on failure.
*
* Calling this function directly from a DeviceTree-supported
* pinctrl driver is DEPRECATED. Please see Section 2.1 of
* Documentation/devicetree/bindings/gpio/gpio.txt on how to
* bind pinctrl and gpio drivers via the "gpio-ranges" property.
*/
int gpiochip_add_pin_range(struct gpio_chip *gc, const char *pinctl_name,
unsigned int gpio_offset, unsigned int pin_offset,
unsigned int npins)
{
struct gpio_pin_range *pin_range;
struct gpio_device *gdev = gc->gpiodev;
int ret;
pin_range = kzalloc(sizeof(*pin_range), GFP_KERNEL);
if (!pin_range) {
chip_err(gc, "failed to allocate pin ranges\n");
return -ENOMEM;
}
/* Use local offset as range ID */
pin_range->range.id = gpio_offset;
pin_range->range.gc = gc;
pin_range->range.name = gc->label;
pin_range->range.base = gdev->base + gpio_offset;
pin_range->range.pin_base = pin_offset;
pin_range->range.npins = npins;
pin_range->pctldev = pinctrl_find_and_add_gpio_range(pinctl_name,
&pin_range->range);
if (IS_ERR(pin_range->pctldev)) {
ret = PTR_ERR(pin_range->pctldev);
chip_err(gc, "could not create pin range\n");
kfree(pin_range);
return ret;
}
chip_dbg(gc, "created GPIO range %d->%d ==> %s PIN %d->%d\n",
gpio_offset, gpio_offset + npins - 1,
pinctl_name,
pin_offset, pin_offset + npins - 1);
list_add_tail(&pin_range->node, &gdev->pin_ranges);
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_add_pin_range);
/**
* gpiochip_remove_pin_ranges() - remove all the GPIO <-> pin mappings
* @gc: the chip to remove all the mappings for
*/
void gpiochip_remove_pin_ranges(struct gpio_chip *gc)
{
struct gpio_pin_range *pin_range, *tmp;
struct gpio_device *gdev = gc->gpiodev;
list_for_each_entry_safe(pin_range, tmp, &gdev->pin_ranges, node) {
list_del(&pin_range->node);
pinctrl_remove_gpio_range(pin_range->pctldev,
&pin_range->range);
kfree(pin_range);
}
}
EXPORT_SYMBOL_GPL(gpiochip_remove_pin_ranges);
#endif /* CONFIG_PINCTRL */
/* These "optional" allocation calls help prevent drivers from stomping
* on each other, and help provide better diagnostics in debugfs.
* They're called even less than the "set direction" calls.
*/
static int gpiod_request_commit(struct gpio_desc *desc, const char *label)
{
unsigned int offset;
int ret;
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
if (test_and_set_bit(FLAG_REQUESTED, &desc->flags))
return -EBUSY;
/* NOTE: gpio_request() can be called in early boot,
* before IRQs are enabled, for non-sleeping (SOC) GPIOs.
*/
if (guard.gc->request) {
offset = gpio_chip_hwgpio(desc);
if (gpiochip_line_is_valid(guard.gc, offset))
ret = guard.gc->request(guard.gc, offset);
else
ret = -EINVAL;
if (ret)
goto out_clear_bit;
}
if (guard.gc->get_direction)
gpiod_get_direction(desc);
ret = desc_set_label(desc, label ? : "?");
if (ret)
goto out_clear_bit;
return 0;
out_clear_bit:
clear_bit(FLAG_REQUESTED, &desc->flags);
return ret;
}
/*
* This descriptor validation needs to be inserted verbatim into each
* function taking a descriptor, so we need to use a preprocessor
* macro to avoid endless duplication. If the desc is NULL it is an
* optional GPIO and calls should just bail out.
*/
static int validate_desc(const struct gpio_desc *desc, const char *func)
{
if (!desc)
return 0;
if (IS_ERR(desc)) {
pr_warn("%s: invalid GPIO (errorpointer)\n", func);
return PTR_ERR(desc);
}
return 1;
}
#define VALIDATE_DESC(desc) do { \
int __valid = validate_desc(desc, __func__); \
if (__valid <= 0) \
return __valid; \
} while (0)
#define VALIDATE_DESC_VOID(desc) do { \
int __valid = validate_desc(desc, __func__); \
if (__valid <= 0) \
return; \
} while (0)
int gpiod_request(struct gpio_desc *desc, const char *label)
{
int ret = -EPROBE_DEFER;
VALIDATE_DESC(desc);
if (try_module_get(desc->gdev->owner)) {
ret = gpiod_request_commit(desc, label);
if (ret)
module_put(desc->gdev->owner);
else
gpio_device_get(desc->gdev);
}
if (ret)
gpiod_dbg(desc, "%s: status %d\n", __func__, ret);
return ret;
}
static void gpiod_free_commit(struct gpio_desc *desc)
{
unsigned long flags;
might_sleep();
CLASS(gpio_chip_guard, guard)(desc);
flags = READ_ONCE(desc->flags);
if (guard.gc && test_bit(FLAG_REQUESTED, &flags)) {
if (guard.gc->free)
guard.gc->free(guard.gc, gpio_chip_hwgpio(desc));
clear_bit(FLAG_ACTIVE_LOW, &flags);
clear_bit(FLAG_REQUESTED, &flags);
clear_bit(FLAG_OPEN_DRAIN, &flags);
clear_bit(FLAG_OPEN_SOURCE, &flags);
clear_bit(FLAG_PULL_UP, &flags);
clear_bit(FLAG_PULL_DOWN, &flags);
clear_bit(FLAG_BIAS_DISABLE, &flags);
clear_bit(FLAG_EDGE_RISING, &flags);
clear_bit(FLAG_EDGE_FALLING, &flags);
clear_bit(FLAG_IS_HOGGED, &flags);
#ifdef CONFIG_OF_DYNAMIC
WRITE_ONCE(desc->hog, NULL);
#endif
desc_set_label(desc, NULL);
WRITE_ONCE(desc->flags, flags);
gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED);
}
}
void gpiod_free(struct gpio_desc *desc)
{
VALIDATE_DESC_VOID(desc);
gpiod_free_commit(desc);
module_put(desc->gdev->owner);
gpio_device_put(desc->gdev);
}
/**
* gpiochip_dup_line_label - Get a copy of the consumer label.
* @gc: GPIO chip controlling this line.
* @offset: Hardware offset of the line.
*
* Returns:
* Pointer to a copy of the consumer label if the line is requested or NULL
* if it's not. If a valid pointer was returned, it must be freed using
* kfree(). In case of a memory allocation error, the function returns %ENOMEM.
*
* Must not be called from atomic context.
*/
char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset)
{
struct gpio_desc *desc;
char *label;
desc = gpiochip_get_desc(gc, offset);
if (IS_ERR(desc))
return NULL;
if (!test_bit(FLAG_REQUESTED, &desc->flags))
return NULL;
guard(srcu)(&desc->srcu);
label = kstrdup(gpiod_get_label(desc), GFP_KERNEL);
if (!label)
return ERR_PTR(-ENOMEM);
return label;
}
EXPORT_SYMBOL_GPL(gpiochip_dup_line_label);
static inline const char *function_name_or_default(const char *con_id)
{
return con_id ?: "(default)";
}
/**
* gpiochip_request_own_desc - Allow GPIO chip to request its own descriptor
* @gc: GPIO chip
* @hwnum: hardware number of the GPIO for which to request the descriptor
* @label: label for the GPIO
* @lflags: lookup flags for this GPIO or 0 if default, this can be used to
* specify things like line inversion semantics with the machine flags
* such as GPIO_OUT_LOW
* @dflags: descriptor request flags for this GPIO or 0 if default, this
* can be used to specify consumer semantics such as open drain
*
* Function allows GPIO chip drivers to request and use their own GPIO
* descriptors via gpiolib API. Difference to gpiod_request() is that this
* function will not increase reference count of the GPIO chip module. This
* allows the GPIO chip module to be unloaded as needed (we assume that the
* GPIO chip driver handles freeing the GPIOs it has requested).
*
* Returns:
* A pointer to the GPIO descriptor, or an ERR_PTR()-encoded negative error
* code on failure.
*/
struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *gc,
unsigned int hwnum,
const char *label,
enum gpio_lookup_flags lflags,
enum gpiod_flags dflags)
{
struct gpio_desc *desc = gpiochip_get_desc(gc, hwnum);
const char *name = function_name_or_default(label);
int ret;
if (IS_ERR(desc)) {
chip_err(gc, "failed to get GPIO %s descriptor\n", name);
return desc;
}
ret = gpiod_request_commit(desc, label);
if (ret < 0)
return ERR_PTR(ret);
ret = gpiod_configure_flags(desc, label, lflags, dflags);
if (ret) {
gpiod_free_commit(desc);
chip_err(gc, "setup of own GPIO %s failed\n", name);
return ERR_PTR(ret);
}
return desc;
}
EXPORT_SYMBOL_GPL(gpiochip_request_own_desc);
/**
* gpiochip_free_own_desc - Free GPIO requested by the chip driver
* @desc: GPIO descriptor to free
*
* Function frees the given GPIO requested previously with
* gpiochip_request_own_desc().
*/
void gpiochip_free_own_desc(struct gpio_desc *desc)
{
if (desc)
gpiod_free_commit(desc);
}
EXPORT_SYMBOL_GPL(gpiochip_free_own_desc);
/*
* Drivers MUST set GPIO direction before making get/set calls. In
* some cases this is done in early boot, before IRQs are enabled.
*
* As a rule these aren't called more than once (except for drivers
* using the open-drain emulation idiom) so these are natural places
* to accumulate extra debugging checks. Note that we can't (yet)
* rely on gpio_request() having been called beforehand.
*/
static int gpio_do_set_config(struct gpio_chip *gc, unsigned int offset,
unsigned long config)
{
if (!gc->set_config)
return -ENOTSUPP;
return gc->set_config(gc, offset, config);
}
static int gpio_set_config_with_argument(struct gpio_desc *desc,
enum pin_config_param mode,
u32 argument)
{
unsigned long config;
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
config = pinconf_to_config_packed(mode, argument);
return gpio_do_set_config(guard.gc, gpio_chip_hwgpio(desc), config);
}
static int gpio_set_config_with_argument_optional(struct gpio_desc *desc,
enum pin_config_param mode,
u32 argument)
{
struct device *dev = &desc->gdev->dev;
int gpio = gpio_chip_hwgpio(desc);
int ret;
ret = gpio_set_config_with_argument(desc, mode, argument);
if (ret != -ENOTSUPP)
return ret;
switch (mode) {
case PIN_CONFIG_PERSIST_STATE:
dev_dbg(dev, "Persistence not supported for GPIO %d\n", gpio);
break;
default:
break;
}
return 0;
}
static int gpio_set_config(struct gpio_desc *desc, enum pin_config_param mode)
{
return gpio_set_config_with_argument(desc, mode, 0);
}
static int gpio_set_bias(struct gpio_desc *desc)
{
enum pin_config_param bias;
unsigned long flags;
unsigned int arg;
flags = READ_ONCE(desc->flags);
if (test_bit(FLAG_BIAS_DISABLE, &flags))
bias = PIN_CONFIG_BIAS_DISABLE;
else if (test_bit(FLAG_PULL_UP, &flags))
bias = PIN_CONFIG_BIAS_PULL_UP;
else if (test_bit(FLAG_PULL_DOWN, &flags))
bias = PIN_CONFIG_BIAS_PULL_DOWN;
else
return 0;
switch (bias) {
case PIN_CONFIG_BIAS_PULL_DOWN:
case PIN_CONFIG_BIAS_PULL_UP:
arg = 1;
break;
default:
arg = 0;
break;
}
return gpio_set_config_with_argument_optional(desc, bias, arg);
}
/**
* gpio_set_debounce_timeout() - Set debounce timeout
* @desc: GPIO descriptor to set the debounce timeout
* @debounce: Debounce timeout in microseconds
*
* The function calls the certain GPIO driver to set debounce timeout
* in the hardware.
*
* Returns 0 on success, or negative error code otherwise.
*/
int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce)
{
return gpio_set_config_with_argument_optional(desc,
PIN_CONFIG_INPUT_DEBOUNCE,
debounce);
}
/**
* gpiod_direction_input - set the GPIO direction to input
* @desc: GPIO to set to input
*
* Set the direction of the passed GPIO to input, such as gpiod_get_value() can
* be called safely on it.
*
* Return 0 in case of success, else an error code.
*/
int gpiod_direction_input(struct gpio_desc *desc)
{
int ret = 0;
VALIDATE_DESC(desc);
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
/*
* It is legal to have no .get() and .direction_input() specified if
* the chip is output-only, but you can't specify .direction_input()
* and not support the .get() operation, that doesn't make sense.
*/
if (!guard.gc->get && guard.gc->direction_input) {
gpiod_warn(desc,
"%s: missing get() but have direction_input()\n",
__func__);
return -EIO;
}
/*
* If we have a .direction_input() callback, things are simple,
* just call it. Else we are some input-only chip so try to check the
* direction (if .get_direction() is supported) else we silently
* assume we are in input mode after this.
*/
if (guard.gc->direction_input) {
ret = guard.gc->direction_input(guard.gc,
gpio_chip_hwgpio(desc));
} else if (guard.gc->get_direction &&
(guard.gc->get_direction(guard.gc,
gpio_chip_hwgpio(desc)) != 1)) {
gpiod_warn(desc,
"%s: missing direction_input() operation and line is output\n",
__func__);
return -EIO;
}
if (ret == 0) {
clear_bit(FLAG_IS_OUT, &desc->flags);
ret = gpio_set_bias(desc);
}
trace_gpio_direction(desc_to_gpio(desc), 1, ret);
return ret;
}
EXPORT_SYMBOL_GPL(gpiod_direction_input);
static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value)
{
int val = !!value, ret = 0;
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
/*
* It's OK not to specify .direction_output() if the gpiochip is
* output-only, but if there is then not even a .set() operation it
* is pretty tricky to drive the output line.
*/
if (!guard.gc->set && !guard.gc->direction_output) {
gpiod_warn(desc,
"%s: missing set() and direction_output() operations\n",
__func__);
return -EIO;
}
if (guard.gc->direction_output) {
ret = guard.gc->direction_output(guard.gc,
gpio_chip_hwgpio(desc), val);
} else {
/* Check that we are in output mode if we can */
if (guard.gc->get_direction &&
guard.gc->get_direction(guard.gc, gpio_chip_hwgpio(desc))) {
gpiod_warn(desc,
"%s: missing direction_output() operation\n",
__func__);
return -EIO;
}
/*
* If we can't actively set the direction, we are some
* output-only chip, so just drive the output as desired.
*/
guard.gc->set(guard.gc, gpio_chip_hwgpio(desc), val);
}
if (!ret)
set_bit(FLAG_IS_OUT, &desc->flags);
trace_gpio_value(desc_to_gpio(desc), 0, val);
trace_gpio_direction(desc_to_gpio(desc), 0, ret);
return ret;
}
/**
* gpiod_direction_output_raw - set the GPIO direction to output
* @desc: GPIO to set to output
* @value: initial output value of the GPIO
*
* Set the direction of the passed GPIO to output, such as gpiod_set_value() can
* be called safely on it. The initial value of the output must be specified
* as raw value on the physical line without regard for the ACTIVE_LOW status.
*
* Return 0 in case of success, else an error code.
*/
int gpiod_direction_output_raw(struct gpio_desc *desc, int value)
{
VALIDATE_DESC(desc);
return gpiod_direction_output_raw_commit(desc, value);
}
EXPORT_SYMBOL_GPL(gpiod_direction_output_raw);
/**
* gpiod_direction_output - set the GPIO direction to output
* @desc: GPIO to set to output
* @value: initial output value of the GPIO
*
* Set the direction of the passed GPIO to output, such as gpiod_set_value() can
* be called safely on it. The initial value of the output must be specified
* as the logical value of the GPIO, i.e. taking its ACTIVE_LOW status into
* account.
*
* Return 0 in case of success, else an error code.
*/
int gpiod_direction_output(struct gpio_desc *desc, int value)
{
unsigned long flags;
int ret;
VALIDATE_DESC(desc);
flags = READ_ONCE(desc->flags);
if (test_bit(FLAG_ACTIVE_LOW, &flags))
value = !value;
else
value = !!value;
/* GPIOs used for enabled IRQs shall not be set as output */
if (test_bit(FLAG_USED_AS_IRQ, &flags) &&
test_bit(FLAG_IRQ_IS_ENABLED, &flags)) {
gpiod_err(desc,
"%s: tried to set a GPIO tied to an IRQ as output\n",
__func__);
return -EIO;
}
if (test_bit(FLAG_OPEN_DRAIN, &flags)) {
/* First see if we can enable open drain in hardware */
ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_DRAIN);
if (!ret)
goto set_output_value;
/* Emulate open drain by not actively driving the line high */
if (value) {
ret = gpiod_direction_input(desc);
goto set_output_flag;
}
} else if (test_bit(FLAG_OPEN_SOURCE, &flags)) {
ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_SOURCE);
if (!ret)
goto set_output_value;
/* Emulate open source by not actively driving the line low */
if (!value) {
ret = gpiod_direction_input(desc);
goto set_output_flag;
}
} else {
gpio_set_config(desc, PIN_CONFIG_DRIVE_PUSH_PULL);
}
set_output_value:
ret = gpio_set_bias(desc);
if (ret)
return ret;
return gpiod_direction_output_raw_commit(desc, value);
set_output_flag:
/*
* When emulating open-source or open-drain functionalities by not
* actively driving the line (setting mode to input) we still need to
* set the IS_OUT flag or otherwise we won't be able to set the line
* value anymore.
*/
if (ret == 0)
set_bit(FLAG_IS_OUT, &desc->flags);
return ret;
}
EXPORT_SYMBOL_GPL(gpiod_direction_output);
/**
* gpiod_enable_hw_timestamp_ns - Enable hardware timestamp in nanoseconds.
*
* @desc: GPIO to enable.
* @flags: Flags related to GPIO edge.
*
* Return 0 in case of success, else negative error code.
*/
int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags)
{
int ret = 0;
VALIDATE_DESC(desc);
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
if (!guard.gc->en_hw_timestamp) {
gpiod_warn(desc, "%s: hw ts not supported\n", __func__);
return -ENOTSUPP;
}
ret = guard.gc->en_hw_timestamp(guard.gc,
gpio_chip_hwgpio(desc), flags);
if (ret)
gpiod_warn(desc, "%s: hw ts request failed\n", __func__);
return ret;
}
EXPORT_SYMBOL_GPL(gpiod_enable_hw_timestamp_ns);
/**
* gpiod_disable_hw_timestamp_ns - Disable hardware timestamp.
*
* @desc: GPIO to disable.
* @flags: Flags related to GPIO edge, same value as used during enable call.
*
* Return 0 in case of success, else negative error code.
*/
int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags)
{
int ret = 0;
VALIDATE_DESC(desc);
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
if (!guard.gc->dis_hw_timestamp) {
gpiod_warn(desc, "%s: hw ts not supported\n", __func__);
return -ENOTSUPP;
}
ret = guard.gc->dis_hw_timestamp(guard.gc, gpio_chip_hwgpio(desc),
flags);
if (ret)
gpiod_warn(desc, "%s: hw ts release failed\n", __func__);
return ret;
}
EXPORT_SYMBOL_GPL(gpiod_disable_hw_timestamp_ns);
/**
* gpiod_set_config - sets @config for a GPIO
* @desc: descriptor of the GPIO for which to set the configuration
* @config: Same packed config format as generic pinconf
*
* Returns:
* 0 on success, %-ENOTSUPP if the controller doesn't support setting the
* configuration.
*/
int gpiod_set_config(struct gpio_desc *desc, unsigned long config)
{
VALIDATE_DESC(desc);
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
return gpio_do_set_config(guard.gc, gpio_chip_hwgpio(desc), config);
}
EXPORT_SYMBOL_GPL(gpiod_set_config);
/**
* gpiod_set_debounce - sets @debounce time for a GPIO
* @desc: descriptor of the GPIO for which to set debounce time
* @debounce: debounce time in microseconds
*
* Returns:
* 0 on success, %-ENOTSUPP if the controller doesn't support setting the
* debounce time.
*/
int gpiod_set_debounce(struct gpio_desc *desc, unsigned int debounce)
{
unsigned long config;
config = pinconf_to_config_packed(PIN_CONFIG_INPUT_DEBOUNCE, debounce);
return gpiod_set_config(desc, config);
}
EXPORT_SYMBOL_GPL(gpiod_set_debounce);
/**
* gpiod_set_transitory - Lose or retain GPIO state on suspend or reset
* @desc: descriptor of the GPIO for which to configure persistence
* @transitory: True to lose state on suspend or reset, false for persistence
*
* Returns:
* 0 on success, otherwise a negative error code.
*/
int gpiod_set_transitory(struct gpio_desc *desc, bool transitory)
{
VALIDATE_DESC(desc);
/*
* Handle FLAG_TRANSITORY first, enabling queries to gpiolib for
* persistence state.
*/
assign_bit(FLAG_TRANSITORY, &desc->flags, transitory);
/* If the driver supports it, set the persistence state now */
return gpio_set_config_with_argument_optional(desc,
PIN_CONFIG_PERSIST_STATE,
!transitory);
}
/**
* gpiod_is_active_low - test whether a GPIO is active-low or not
* @desc: the gpio descriptor to test
*
* Returns 1 if the GPIO is active-low, 0 otherwise.
*/
int gpiod_is_active_low(const struct gpio_desc *desc)
{
VALIDATE_DESC(desc);
return test_bit(FLAG_ACTIVE_LOW, &desc->flags);
}
EXPORT_SYMBOL_GPL(gpiod_is_active_low);
/**
* gpiod_toggle_active_low - toggle whether a GPIO is active-low or not
* @desc: the gpio descriptor to change
*/
void gpiod_toggle_active_low(struct gpio_desc *desc)
{
VALIDATE_DESC_VOID(desc);
change_bit(FLAG_ACTIVE_LOW, &desc->flags);
}
EXPORT_SYMBOL_GPL(gpiod_toggle_active_low);
static int gpio_chip_get_value(struct gpio_chip *gc, const struct gpio_desc *desc)
{
return gc->get ? gc->get(gc, gpio_chip_hwgpio(desc)) : -EIO;
}
/* I/O calls are only valid after configuration completed; the relevant
* "is this a valid GPIO" error checks should already have been done.
*
* "Get" operations are often inlinable as reading a pin value register,
* and masking the relevant bit in that register.
*
* When "set" operations are inlinable, they involve writing that mask to
* one register to set a low value, or a different register to set it high.
* Otherwise locking is needed, so there may be little value to inlining.
*
*------------------------------------------------------------------------
*
* IMPORTANT!!! The hot paths -- get/set value -- assume that callers
* have requested the GPIO. That can include implicit requesting by
* a direction setting call. Marking a gpio as requested locks its chip
* in memory, guaranteeing that these table lookups need no more locking
* and that gpiochip_remove() will fail.
*
* REVISIT when debugging, consider adding some instrumentation to ensure
* that the GPIO was actually requested.
*/
static int gpiod_get_raw_value_commit(const struct gpio_desc *desc)
{
struct gpio_device *gdev;
struct gpio_chip *gc;
int value;
/* FIXME Unable to use gpio_chip_guard due to const desc. */
gdev = desc->gdev;
guard(srcu)(&gdev->srcu);
gc = srcu_dereference(gdev->chip, &gdev->srcu);
if (!gc)
return -ENODEV;
value = gpio_chip_get_value(gc, desc);
value = value < 0 ? value : !!value;
trace_gpio_value(desc_to_gpio(desc), 1, value);
return value;
}
static int gpio_chip_get_multiple(struct gpio_chip *gc,
unsigned long *mask, unsigned long *bits)
{
if (gc->get_multiple)
return gc->get_multiple(gc, mask, bits);
if (gc->get) {
int i, value;
for_each_set_bit(i, mask, gc->ngpio) {
value = gc->get(gc, i);
if (value < 0)
return value;
__assign_bit(i, bits, value);
}
return 0;
}
return -EIO;
}
/* The 'other' chip must be protected with its GPIO device's SRCU. */
static bool gpio_device_chip_cmp(struct gpio_device *gdev, struct gpio_chip *gc)
{
guard(srcu)(&gdev->srcu);
return gc == srcu_dereference(gdev->chip, &gdev->srcu);
}
int gpiod_get_array_value_complex(bool raw, bool can_sleep,
unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
int ret, i = 0;
/*
* Validate array_info against desc_array and its size.
* It should immediately follow desc_array if both
* have been obtained from the same gpiod_get_array() call.
*/
if (array_info && array_info->desc == desc_array &&
array_size <= array_info->size &&
(void *)array_info == desc_array + array_info->size) {
if (!can_sleep)
WARN_ON(array_info->chip->can_sleep);
ret = gpio_chip_get_multiple(array_info->chip,
array_info->get_mask,
value_bitmap);
if (ret)
return ret;
if (!raw && !bitmap_empty(array_info->invert_mask, array_size))
bitmap_xor(value_bitmap, value_bitmap,
array_info->invert_mask, array_size);
i = find_first_zero_bit(array_info->get_mask, array_size);
if (i == array_size)
return 0;
} else {
array_info = NULL;
}
while (i < array_size) {
DECLARE_BITMAP(fastpath_mask, FASTPATH_NGPIO);
DECLARE_BITMAP(fastpath_bits, FASTPATH_NGPIO);
unsigned long *mask, *bits;
int first, j;
CLASS(gpio_chip_guard, guard)(desc_array[i]);
if (!guard.gc)
return -ENODEV;
if (likely(guard.gc->ngpio <= FASTPATH_NGPIO)) {
mask = fastpath_mask;
bits = fastpath_bits;
} else {
gfp_t flags = can_sleep ? GFP_KERNEL : GFP_ATOMIC;
mask = bitmap_alloc(guard.gc->ngpio, flags);
if (!mask)
return -ENOMEM;
bits = bitmap_alloc(guard.gc->ngpio, flags);
if (!bits) {
bitmap_free(mask);
return -ENOMEM;
}
}
bitmap_zero(mask, guard.gc->ngpio);
if (!can_sleep)
WARN_ON(guard.gc->can_sleep);
/* collect all inputs belonging to the same chip */
first = i;
do {
const struct gpio_desc *desc = desc_array[i];
int hwgpio = gpio_chip_hwgpio(desc);
__set_bit(hwgpio, mask);
i++;
if (array_info)
i = find_next_zero_bit(array_info->get_mask,
array_size, i);
} while ((i < array_size) &&
gpio_device_chip_cmp(desc_array[i]->gdev, guard.gc));
ret = gpio_chip_get_multiple(guard.gc, mask, bits);
if (ret) {
if (mask != fastpath_mask)
bitmap_free(mask);
if (bits != fastpath_bits)
bitmap_free(bits);
return ret;
}
for (j = first; j < i; ) {
const struct gpio_desc *desc = desc_array[j];
int hwgpio = gpio_chip_hwgpio(desc);
int value = test_bit(hwgpio, bits);
if (!raw && test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
__assign_bit(j, value_bitmap, value);
trace_gpio_value(desc_to_gpio(desc), 1, value);
j++;
if (array_info)
j = find_next_zero_bit(array_info->get_mask, i,
j);
}
if (mask != fastpath_mask)
bitmap_free(mask);
if (bits != fastpath_bits)
bitmap_free(bits);
}
return 0;
}
/**
* gpiod_get_raw_value() - return a gpio's raw value
* @desc: gpio whose value will be returned
*
* Return the GPIO's raw value, i.e. the value of the physical line disregarding
* its ACTIVE_LOW status, or negative errno on failure.
*
* This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_raw_value(const struct gpio_desc *desc)
{
VALIDATE_DESC(desc);
/* Should be using gpiod_get_raw_value_cansleep() */
WARN_ON(desc->gdev->can_sleep);
return gpiod_get_raw_value_commit(desc);
}
EXPORT_SYMBOL_GPL(gpiod_get_raw_value);
/**
* gpiod_get_value() - return a gpio's value
* @desc: gpio whose value will be returned
*
* Return the GPIO's logical value, i.e. taking the ACTIVE_LOW status into
* account, or negative errno on failure.
*
* This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_value(const struct gpio_desc *desc)
{
int value;
VALIDATE_DESC(desc);
/* Should be using gpiod_get_value_cansleep() */
WARN_ON(desc->gdev->can_sleep);
value = gpiod_get_raw_value_commit(desc);
if (value < 0)
return value;
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
return value;
}
EXPORT_SYMBOL_GPL(gpiod_get_value);
/**
* gpiod_get_raw_array_value() - read raw values from an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be read
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap to store the read values
*
* Read the raw values of the GPIOs, i.e. the values of the physical lines
* without regard for their ACTIVE_LOW status. Return 0 in case of success,
* else an error code.
*
* This function can be called from contexts where we cannot sleep,
* and it will complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_raw_array_value(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
if (!desc_array)
return -EINVAL;
return gpiod_get_array_value_complex(true, false, array_size,
desc_array, array_info,
value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_get_raw_array_value);
/**
* gpiod_get_array_value() - read values from an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be read
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap to store the read values
*
* Read the logical values of the GPIOs, i.e. taking their ACTIVE_LOW status
* into account. Return 0 in case of success, else an error code.
*
* This function can be called from contexts where we cannot sleep,
* and it will complain if the GPIO chip functions potentially sleep.
*/
int gpiod_get_array_value(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
if (!desc_array)
return -EINVAL;
return gpiod_get_array_value_complex(false, false, array_size,
desc_array, array_info,
value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_get_array_value);
/*
* gpio_set_open_drain_value_commit() - Set the open drain gpio's value.
* @desc: gpio descriptor whose state need to be set.
* @value: Non-zero for setting it HIGH otherwise it will set to LOW.
*/
static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value)
{
int ret = 0, offset = gpio_chip_hwgpio(desc);
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return;
if (value) {
ret = guard.gc->direction_input(guard.gc, offset);
} else {
ret = guard.gc->direction_output(guard.gc, offset, 0);
if (!ret)
set_bit(FLAG_IS_OUT, &desc->flags);
}
trace_gpio_direction(desc_to_gpio(desc), value, ret);
if (ret < 0)
gpiod_err(desc,
"%s: Error in set_value for open drain err %d\n",
__func__, ret);
}
/*
* _gpio_set_open_source_value() - Set the open source gpio's value.
* @desc: gpio descriptor whose state need to be set.
* @value: Non-zero for setting it HIGH otherwise it will set to LOW.
*/
static void gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value)
{
int ret = 0, offset = gpio_chip_hwgpio(desc);
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return;
if (value) {
ret = guard.gc->direction_output(guard.gc, offset, 1);
if (!ret)
set_bit(FLAG_IS_OUT, &desc->flags);
} else {
ret = guard.gc->direction_input(guard.gc, offset);
}
trace_gpio_direction(desc_to_gpio(desc), !value, ret);
if (ret < 0)
gpiod_err(desc,
"%s: Error in set_value for open source err %d\n",
__func__, ret);
}
static void gpiod_set_raw_value_commit(struct gpio_desc *desc, bool value)
{
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return;
trace_gpio_value(desc_to_gpio(desc), 0, value);
guard.gc->set(guard.gc, gpio_chip_hwgpio(desc), value);
}
/*
* set multiple outputs on the same chip;
* use the chip's set_multiple function if available;
* otherwise set the outputs sequentially;
* @chip: the GPIO chip we operate on
* @mask: bit mask array; one bit per output; BITS_PER_LONG bits per word
* defines which outputs are to be changed
* @bits: bit value array; one bit per output; BITS_PER_LONG bits per word
* defines the values the outputs specified by mask are to be set to
*/
static void gpio_chip_set_multiple(struct gpio_chip *gc,
unsigned long *mask, unsigned long *bits)
{
if (gc->set_multiple) {
gc->set_multiple(gc, mask, bits);
} else {
unsigned int i;
/* set outputs if the corresponding mask bit is set */
for_each_set_bit(i, mask, gc->ngpio)
gc->set(gc, i, test_bit(i, bits));
}
}
int gpiod_set_array_value_complex(bool raw, bool can_sleep,
unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
int i = 0;
/*
* Validate array_info against desc_array and its size.
* It should immediately follow desc_array if both
* have been obtained from the same gpiod_get_array() call.
*/
if (array_info && array_info->desc == desc_array &&
array_size <= array_info->size &&
(void *)array_info == desc_array + array_info->size) {
if (!can_sleep)
WARN_ON(array_info->chip->can_sleep);
if (!raw && !bitmap_empty(array_info->invert_mask, array_size))
bitmap_xor(value_bitmap, value_bitmap,
array_info->invert_mask, array_size);
gpio_chip_set_multiple(array_info->chip, array_info->set_mask,
value_bitmap);
i = find_first_zero_bit(array_info->set_mask, array_size);
if (i == array_size)
return 0;
} else {
array_info = NULL;
}
while (i < array_size) {
DECLARE_BITMAP(fastpath_mask, FASTPATH_NGPIO);
DECLARE_BITMAP(fastpath_bits, FASTPATH_NGPIO);
unsigned long *mask, *bits;
int count = 0;
CLASS(gpio_chip_guard, guard)(desc_array[i]);
if (!guard.gc)
return -ENODEV;
if (likely(guard.gc->ngpio <= FASTPATH_NGPIO)) {
mask = fastpath_mask;
bits = fastpath_bits;
} else {
gfp_t flags = can_sleep ? GFP_KERNEL : GFP_ATOMIC;
mask = bitmap_alloc(guard.gc->ngpio, flags);
if (!mask)
return -ENOMEM;
bits = bitmap_alloc(guard.gc->ngpio, flags);
if (!bits) {
bitmap_free(mask);
return -ENOMEM;
}
}
bitmap_zero(mask, guard.gc->ngpio);
if (!can_sleep)
WARN_ON(guard.gc->can_sleep);
do {
struct gpio_desc *desc = desc_array[i];
int hwgpio = gpio_chip_hwgpio(desc);
int value = test_bit(i, value_bitmap);
/*
* Pins applicable for fast input but not for
* fast output processing may have been already
* inverted inside the fast path, skip them.
*/
if (!raw && !(array_info &&
test_bit(i, array_info->invert_mask)) &&
test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
trace_gpio_value(desc_to_gpio(desc), 0, value);
/*
* collect all normal outputs belonging to the same chip
* open drain and open source outputs are set individually
*/
if (test_bit(FLAG_OPEN_DRAIN, &desc->flags) && !raw) {
gpio_set_open_drain_value_commit(desc, value);
} else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags) && !raw) {
gpio_set_open_source_value_commit(desc, value);
} else {
__set_bit(hwgpio, mask);
__assign_bit(hwgpio, bits, value);
count++;
}
i++;
if (array_info)
i = find_next_zero_bit(array_info->set_mask,
array_size, i);
} while ((i < array_size) &&
gpio_device_chip_cmp(desc_array[i]->gdev, guard.gc));
/* push collected bits to outputs */
if (count != 0)
gpio_chip_set_multiple(guard.gc, mask, bits);
if (mask != fastpath_mask)
bitmap_free(mask);
if (bits != fastpath_bits)
bitmap_free(bits);
}
return 0;
}
/**
* gpiod_set_raw_value() - assign a gpio's raw value
* @desc: gpio whose value will be assigned
* @value: value to assign
*
* Set the raw value of the GPIO, i.e. the value of its physical line without
* regard for its ACTIVE_LOW status.
*
* This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
void gpiod_set_raw_value(struct gpio_desc *desc, int value)
{
VALIDATE_DESC_VOID(desc);
/* Should be using gpiod_set_raw_value_cansleep() */
WARN_ON(desc->gdev->can_sleep);
gpiod_set_raw_value_commit(desc, value);
}
EXPORT_SYMBOL_GPL(gpiod_set_raw_value);
/**
* gpiod_set_value_nocheck() - set a GPIO line value without checking
* @desc: the descriptor to set the value on
* @value: value to set
*
* This sets the value of a GPIO line backing a descriptor, applying
* different semantic quirks like active low and open drain/source
* handling.
*/
static void gpiod_set_value_nocheck(struct gpio_desc *desc, int value)
{
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
if (test_bit(FLAG_OPEN_DRAIN, &desc->flags))
gpio_set_open_drain_value_commit(desc, value);
else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags))
gpio_set_open_source_value_commit(desc, value);
else
gpiod_set_raw_value_commit(desc, value);
}
/**
* gpiod_set_value() - assign a gpio's value
* @desc: gpio whose value will be assigned
* @value: value to assign
*
* Set the logical value of the GPIO, i.e. taking its ACTIVE_LOW,
* OPEN_DRAIN and OPEN_SOURCE flags into account.
*
* This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
void gpiod_set_value(struct gpio_desc *desc, int value)
{
VALIDATE_DESC_VOID(desc);
/* Should be using gpiod_set_value_cansleep() */
WARN_ON(desc->gdev->can_sleep);
gpiod_set_value_nocheck(desc, value);
}
EXPORT_SYMBOL_GPL(gpiod_set_value);
/**
* gpiod_set_raw_array_value() - assign values to an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be assigned
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap of values to assign
*
* Set the raw values of the GPIOs, i.e. the values of the physical lines
* without regard for their ACTIVE_LOW status.
*
* This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_set_raw_array_value(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
if (!desc_array)
return -EINVAL;
return gpiod_set_array_value_complex(true, false, array_size,
desc_array, array_info, value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_set_raw_array_value);
/**
* gpiod_set_array_value() - assign values to an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be assigned
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap of values to assign
*
* Set the logical values of the GPIOs, i.e. taking their ACTIVE_LOW status
* into account.
*
* This function can be called from contexts where we cannot sleep, and will
* complain if the GPIO chip functions potentially sleep.
*/
int gpiod_set_array_value(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
if (!desc_array)
return -EINVAL;
return gpiod_set_array_value_complex(false, false, array_size,
desc_array, array_info,
value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_set_array_value);
/**
* gpiod_cansleep() - report whether gpio value access may sleep
* @desc: gpio to check
*
*/
int gpiod_cansleep(const struct gpio_desc *desc)
{
VALIDATE_DESC(desc);
return desc->gdev->can_sleep;
}
EXPORT_SYMBOL_GPL(gpiod_cansleep);
/**
* gpiod_set_consumer_name() - set the consumer name for the descriptor
* @desc: gpio to set the consumer name on
* @name: the new consumer name
*/
int gpiod_set_consumer_name(struct gpio_desc *desc, const char *name)
{
VALIDATE_DESC(desc);
return desc_set_label(desc, name);
}
EXPORT_SYMBOL_GPL(gpiod_set_consumer_name);
/**
* gpiod_to_irq() - return the IRQ corresponding to a GPIO
* @desc: gpio whose IRQ will be returned (already requested)
*
* Return the IRQ corresponding to the passed GPIO, or an error code in case of
* error.
*/
int gpiod_to_irq(const struct gpio_desc *desc)
{
struct gpio_device *gdev;
struct gpio_chip *gc;
int offset;
/*
* Cannot VALIDATE_DESC() here as gpiod_to_irq() consumer semantics
* requires this function to not return zero on an invalid descriptor
* but rather a negative error number.
*/
if (!desc || IS_ERR(desc))
return -EINVAL;
gdev = desc->gdev;
/* FIXME Cannot use gpio_chip_guard due to const desc. */
guard(srcu)(&gdev->srcu);
gc = srcu_dereference(gdev->chip, &gdev->srcu);
if (!gc)
return -ENODEV;
offset = gpio_chip_hwgpio(desc);
if (gc->to_irq) {
int retirq = gc->to_irq(gc, offset);
/* Zero means NO_IRQ */
if (!retirq)
return -ENXIO;
return retirq;
}
#ifdef CONFIG_GPIOLIB_IRQCHIP
if (gc->irq.chip) {
/*
* Avoid race condition with other code, which tries to lookup
* an IRQ before the irqchip has been properly registered,
* i.e. while gpiochip is still being brought up.
*/
return -EPROBE_DEFER;
}
#endif
return -ENXIO;
}
EXPORT_SYMBOL_GPL(gpiod_to_irq);
/**
* gpiochip_lock_as_irq() - lock a GPIO to be used as IRQ
* @gc: the chip the GPIO to lock belongs to
* @offset: the offset of the GPIO to lock as IRQ
*
* This is used directly by GPIO drivers that want to lock down
* a certain GPIO line to be used for IRQs.
*/
int gpiochip_lock_as_irq(struct gpio_chip *gc, unsigned int offset)
{
struct gpio_desc *desc;
desc = gpiochip_get_desc(gc, offset);
if (IS_ERR(desc))
return PTR_ERR(desc);
/*
* If it's fast: flush the direction setting if something changed
* behind our back
*/
if (!gc->can_sleep && gc->get_direction) {
int dir = gpiod_get_direction(desc);
if (dir < 0) {
chip_err(gc, "%s: cannot get GPIO direction\n",
__func__);
return dir;
}
}
/* To be valid for IRQ the line needs to be input or open drain */
if (test_bit(FLAG_IS_OUT, &desc->flags) &&
!test_bit(FLAG_OPEN_DRAIN, &desc->flags)) {
chip_err(gc,
"%s: tried to flag a GPIO set as output for IRQ\n",
__func__);
return -EIO;
}
set_bit(FLAG_USED_AS_IRQ, &desc->flags);
set_bit(FLAG_IRQ_IS_ENABLED, &desc->flags);
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_lock_as_irq);
/**
* gpiochip_unlock_as_irq() - unlock a GPIO used as IRQ
* @gc: the chip the GPIO to lock belongs to
* @offset: the offset of the GPIO to lock as IRQ
*
* This is used directly by GPIO drivers that want to indicate
* that a certain GPIO is no longer used exclusively for IRQ.
*/
void gpiochip_unlock_as_irq(struct gpio_chip *gc, unsigned int offset)
{
struct gpio_desc *desc;
desc = gpiochip_get_desc(gc, offset);
if (IS_ERR(desc))
return;
clear_bit(FLAG_USED_AS_IRQ, &desc->flags);
clear_bit(FLAG_IRQ_IS_ENABLED, &desc->flags);
}
EXPORT_SYMBOL_GPL(gpiochip_unlock_as_irq);
void gpiochip_disable_irq(struct gpio_chip *gc, unsigned int offset)
{
struct gpio_desc *desc = gpiochip_get_desc(gc, offset);
if (!IS_ERR(desc) &&
!WARN_ON(!test_bit(FLAG_USED_AS_IRQ, &desc->flags)))
clear_bit(FLAG_IRQ_IS_ENABLED, &desc->flags);
}
EXPORT_SYMBOL_GPL(gpiochip_disable_irq);
void gpiochip_enable_irq(struct gpio_chip *gc, unsigned int offset)
{
struct gpio_desc *desc = gpiochip_get_desc(gc, offset);
if (!IS_ERR(desc) &&
!WARN_ON(!test_bit(FLAG_USED_AS_IRQ, &desc->flags))) {
/*
* We must not be output when using IRQ UNLESS we are
* open drain.
*/
WARN_ON(test_bit(FLAG_IS_OUT, &desc->flags) &&
!test_bit(FLAG_OPEN_DRAIN, &desc->flags));
set_bit(FLAG_IRQ_IS_ENABLED, &desc->flags);
}
}
EXPORT_SYMBOL_GPL(gpiochip_enable_irq);
bool gpiochip_line_is_irq(struct gpio_chip *gc, unsigned int offset)
{
if (offset >= gc->ngpio)
return false;
return test_bit(FLAG_USED_AS_IRQ, &gc->gpiodev->descs[offset].flags);
}
EXPORT_SYMBOL_GPL(gpiochip_line_is_irq);
int gpiochip_reqres_irq(struct gpio_chip *gc, unsigned int offset)
{
int ret;
if (!try_module_get(gc->gpiodev->owner))
return -ENODEV;
ret = gpiochip_lock_as_irq(gc, offset);
if (ret) {
chip_err(gc, "unable to lock HW IRQ %u for IRQ\n", offset);
module_put(gc->gpiodev->owner);
return ret;
}
return 0;
}
EXPORT_SYMBOL_GPL(gpiochip_reqres_irq);
void gpiochip_relres_irq(struct gpio_chip *gc, unsigned int offset)
{
gpiochip_unlock_as_irq(gc, offset);
module_put(gc->gpiodev->owner);
}
EXPORT_SYMBOL_GPL(gpiochip_relres_irq);
bool gpiochip_line_is_open_drain(struct gpio_chip *gc, unsigned int offset)
{
if (offset >= gc->ngpio)
return false;
return test_bit(FLAG_OPEN_DRAIN, &gc->gpiodev->descs[offset].flags);
}
EXPORT_SYMBOL_GPL(gpiochip_line_is_open_drain);
bool gpiochip_line_is_open_source(struct gpio_chip *gc, unsigned int offset)
{
if (offset >= gc->ngpio)
return false;
return test_bit(FLAG_OPEN_SOURCE, &gc->gpiodev->descs[offset].flags);
}
EXPORT_SYMBOL_GPL(gpiochip_line_is_open_source);
bool gpiochip_line_is_persistent(struct gpio_chip *gc, unsigned int offset)
{
if (offset >= gc->ngpio)
return false;
return !test_bit(FLAG_TRANSITORY, &gc->gpiodev->descs[offset].flags);
}
EXPORT_SYMBOL_GPL(gpiochip_line_is_persistent);
/**
* gpiod_get_raw_value_cansleep() - return a gpio's raw value
* @desc: gpio whose value will be returned
*
* Return the GPIO's raw value, i.e. the value of the physical line disregarding
* its ACTIVE_LOW status, or negative errno on failure.
*
* This function is to be called from contexts that can sleep.
*/
int gpiod_get_raw_value_cansleep(const struct gpio_desc *desc)
{
might_sleep();
VALIDATE_DESC(desc);
return gpiod_get_raw_value_commit(desc);
}
EXPORT_SYMBOL_GPL(gpiod_get_raw_value_cansleep);
/**
* gpiod_get_value_cansleep() - return a gpio's value
* @desc: gpio whose value will be returned
*
* Return the GPIO's logical value, i.e. taking the ACTIVE_LOW status into
* account, or negative errno on failure.
*
* This function is to be called from contexts that can sleep.
*/
int gpiod_get_value_cansleep(const struct gpio_desc *desc)
{
int value;
might_sleep();
VALIDATE_DESC(desc);
value = gpiod_get_raw_value_commit(desc);
if (value < 0)
return value;
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
value = !value;
return value;
}
EXPORT_SYMBOL_GPL(gpiod_get_value_cansleep);
/**
* gpiod_get_raw_array_value_cansleep() - read raw values from an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be read
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap to store the read values
*
* Read the raw values of the GPIOs, i.e. the values of the physical lines
* without regard for their ACTIVE_LOW status. Return 0 in case of success,
* else an error code.
*
* This function is to be called from contexts that can sleep.
*/
int gpiod_get_raw_array_value_cansleep(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
might_sleep();
if (!desc_array)
return -EINVAL;
return gpiod_get_array_value_complex(true, true, array_size,
desc_array, array_info,
value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_get_raw_array_value_cansleep);
/**
* gpiod_get_array_value_cansleep() - read values from an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be read
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap to store the read values
*
* Read the logical values of the GPIOs, i.e. taking their ACTIVE_LOW status
* into account. Return 0 in case of success, else an error code.
*
* This function is to be called from contexts that can sleep.
*/
int gpiod_get_array_value_cansleep(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
might_sleep();
if (!desc_array)
return -EINVAL;
return gpiod_get_array_value_complex(false, true, array_size,
desc_array, array_info,
value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_get_array_value_cansleep);
/**
* gpiod_set_raw_value_cansleep() - assign a gpio's raw value
* @desc: gpio whose value will be assigned
* @value: value to assign
*
* Set the raw value of the GPIO, i.e. the value of its physical line without
* regard for its ACTIVE_LOW status.
*
* This function is to be called from contexts that can sleep.
*/
void gpiod_set_raw_value_cansleep(struct gpio_desc *desc, int value)
{
might_sleep();
VALIDATE_DESC_VOID(desc);
gpiod_set_raw_value_commit(desc, value);
}
EXPORT_SYMBOL_GPL(gpiod_set_raw_value_cansleep);
/**
* gpiod_set_value_cansleep() - assign a gpio's value
* @desc: gpio whose value will be assigned
* @value: value to assign
*
* Set the logical value of the GPIO, i.e. taking its ACTIVE_LOW status into
* account
*
* This function is to be called from contexts that can sleep.
*/
void gpiod_set_value_cansleep(struct gpio_desc *desc, int value)
{
might_sleep();
VALIDATE_DESC_VOID(desc);
gpiod_set_value_nocheck(desc, value);
}
EXPORT_SYMBOL_GPL(gpiod_set_value_cansleep);
/**
* gpiod_set_raw_array_value_cansleep() - assign values to an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be assigned
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap of values to assign
*
* Set the raw values of the GPIOs, i.e. the values of the physical lines
* without regard for their ACTIVE_LOW status.
*
* This function is to be called from contexts that can sleep.
*/
int gpiod_set_raw_array_value_cansleep(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
might_sleep();
if (!desc_array)
return -EINVAL;
return gpiod_set_array_value_complex(true, true, array_size, desc_array,
array_info, value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_set_raw_array_value_cansleep);
/**
* gpiod_add_lookup_tables() - register GPIO device consumers
* @tables: list of tables of consumers to register
* @n: number of tables in the list
*/
void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n)
{
unsigned int i;
mutex_lock(&gpio_lookup_lock);
for (i = 0; i < n; i++)
list_add_tail(&tables[i]->list, &gpio_lookup_list);
mutex_unlock(&gpio_lookup_lock);
}
/**
* gpiod_set_array_value_cansleep() - assign values to an array of GPIOs
* @array_size: number of elements in the descriptor array / value bitmap
* @desc_array: array of GPIO descriptors whose values will be assigned
* @array_info: information on applicability of fast bitmap processing path
* @value_bitmap: bitmap of values to assign
*
* Set the logical values of the GPIOs, i.e. taking their ACTIVE_LOW status
* into account.
*
* This function is to be called from contexts that can sleep.
*/
int gpiod_set_array_value_cansleep(unsigned int array_size,
struct gpio_desc **desc_array,
struct gpio_array *array_info,
unsigned long *value_bitmap)
{
might_sleep();
if (!desc_array)
return -EINVAL;
return gpiod_set_array_value_complex(false, true, array_size,
desc_array, array_info,
value_bitmap);
}
EXPORT_SYMBOL_GPL(gpiod_set_array_value_cansleep);
void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action)
{
blocking_notifier_call_chain(&desc->gdev->line_state_notifier,
action, desc);
}
/**
* gpiod_add_lookup_table() - register GPIO device consumers
* @table: table of consumers to register
*/
void gpiod_add_lookup_table(struct gpiod_lookup_table *table)
{
gpiod_add_lookup_tables(&table, 1);
}
EXPORT_SYMBOL_GPL(gpiod_add_lookup_table);
/**
* gpiod_remove_lookup_table() - unregister GPIO device consumers
* @table: table of consumers to unregister
*/
void gpiod_remove_lookup_table(struct gpiod_lookup_table *table)
{
/* Nothing to remove */
if (!table)
return;
mutex_lock(&gpio_lookup_lock);
list_del(&table->list);
mutex_unlock(&gpio_lookup_lock);
}
EXPORT_SYMBOL_GPL(gpiod_remove_lookup_table);
/**
* gpiod_add_hogs() - register a set of GPIO hogs from machine code
* @hogs: table of gpio hog entries with a zeroed sentinel at the end
*/
void gpiod_add_hogs(struct gpiod_hog *hogs)
{
struct gpiod_hog *hog;
mutex_lock(&gpio_machine_hogs_mutex);
for (hog = &hogs[0]; hog->chip_label; hog++) {
list_add_tail(&hog->list, &gpio_machine_hogs);
/*
* The chip may have been registered earlier, so check if it
* exists and, if so, try to hog the line now.
*/
struct gpio_device *gdev __free(gpio_device_put) =
gpio_device_find_by_label(hog->chip_label);
if (gdev)
gpiochip_machine_hog(gpio_device_get_chip(gdev), hog);
}
mutex_unlock(&gpio_machine_hogs_mutex);
}
EXPORT_SYMBOL_GPL(gpiod_add_hogs);
void gpiod_remove_hogs(struct gpiod_hog *hogs)
{
struct gpiod_hog *hog;
mutex_lock(&gpio_machine_hogs_mutex);
for (hog = &hogs[0]; hog->chip_label; hog++)
list_del(&hog->list);
mutex_unlock(&gpio_machine_hogs_mutex);
}
EXPORT_SYMBOL_GPL(gpiod_remove_hogs);
static struct gpiod_lookup_table *gpiod_find_lookup_table(struct device *dev)
{
const char *dev_id = dev ? dev_name(dev) : NULL;
struct gpiod_lookup_table *table;
list_for_each_entry(table, &gpio_lookup_list, list) {
if (table->dev_id && dev_id) {
/*
* Valid strings on both ends, must be identical to have
* a match
*/
if (!strcmp(table->dev_id, dev_id))
return table;
} else {
/*
* One of the pointers is NULL, so both must be to have
* a match
*/
if (dev_id == table->dev_id)
return table;
}
}
return NULL;
}
static struct gpio_desc *gpiod_find(struct device *dev, const char *con_id,
unsigned int idx, unsigned long *flags)
{
struct gpio_desc *desc = ERR_PTR(-ENOENT);
struct gpiod_lookup_table *table;
struct gpiod_lookup *p;
struct gpio_chip *gc;
guard(mutex)(&gpio_lookup_lock);
table = gpiod_find_lookup_table(dev);
if (!table)
return desc;
for (p = &table->table[0]; p->key; p++) {
/* idx must always match exactly */
if (p->idx != idx)
continue;
/* If the lookup entry has a con_id, require exact match */
if (p->con_id && (!con_id || strcmp(p->con_id, con_id)))
continue;
if (p->chip_hwnum == U16_MAX) {
desc = gpio_name_to_desc(p->key);
if (desc) {
*flags = p->flags;
return desc;
}
dev_warn(dev, "cannot find GPIO line %s, deferring\n",
p->key);
return ERR_PTR(-EPROBE_DEFER);
}
struct gpio_device *gdev __free(gpio_device_put) =
gpio_device_find_by_label(p->key);
if (!gdev) {
/*
* As the lookup table indicates a chip with
* p->key should exist, assume it may
* still appear later and let the interested
* consumer be probed again or let the Deferred
* Probe infrastructure handle the error.
*/
dev_warn(dev, "cannot find GPIO chip %s, deferring\n",
p->key);
return ERR_PTR(-EPROBE_DEFER);
}
gc = gpio_device_get_chip(gdev);
if (gc->ngpio <= p->chip_hwnum) {
dev_err(dev,
"requested GPIO %u (%u) is out of range [0..%u] for chip %s\n",
idx, p->chip_hwnum, gc->ngpio - 1,
gc->label);
return ERR_PTR(-EINVAL);
}
desc = gpio_device_get_desc(gdev, p->chip_hwnum);
*flags = p->flags;
return desc;
}
return desc;
}
static int platform_gpio_count(struct device *dev, const char *con_id)
{
struct gpiod_lookup_table *table;
struct gpiod_lookup *p;
unsigned int count = 0;
scoped_guard(mutex, &gpio_lookup_lock) {
table = gpiod_find_lookup_table(dev);
if (!table)
return -ENOENT;
for (p = &table->table[0]; p->key; p++) {
if ((con_id && p->con_id && !strcmp(con_id, p->con_id)) ||
(!con_id && !p->con_id))
count++;
}
}
if (!count)
return -ENOENT;
return count;
}
static struct gpio_desc *gpiod_find_by_fwnode(struct fwnode_handle *fwnode,
struct device *consumer,
const char *con_id,
unsigned int idx,
enum gpiod_flags *flags,
unsigned long *lookupflags)
{
const char *name = function_name_or_default(con_id);
struct gpio_desc *desc = ERR_PTR(-ENOENT);
if (is_of_node(fwnode)) {
dev_dbg(consumer, "using DT '%pfw' for '%s' GPIO lookup\n", fwnode, name);
desc = of_find_gpio(to_of_node(fwnode), con_id, idx, lookupflags);
} else if (is_acpi_node(fwnode)) {
dev_dbg(consumer, "using ACPI '%pfw' for '%s' GPIO lookup\n", fwnode, name);
desc = acpi_find_gpio(fwnode, con_id, idx, flags, lookupflags);
} else if (is_software_node(fwnode)) {
dev_dbg(consumer, "using swnode '%pfw' for '%s' GPIO lookup\n", fwnode, name);
desc = swnode_find_gpio(fwnode, con_id, idx, lookupflags);
}
return desc;
}
struct gpio_desc *gpiod_find_and_request(struct device *consumer,
struct fwnode_handle *fwnode,
const char *con_id,
unsigned int idx,
enum gpiod_flags flags,
const char *label,
bool platform_lookup_allowed)
{
unsigned long lookupflags = GPIO_LOOKUP_FLAGS_DEFAULT;
const char *name = function_name_or_default(con_id);
/*
* scoped_guard() is implemented as a for loop, meaning static
* analyzers will complain about these two not being initialized.
*/
struct gpio_desc *desc = NULL;
int ret = 0;
scoped_guard(srcu, &gpio_devices_srcu) {
desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx,
&flags, &lookupflags);
if (gpiod_not_found(desc) && platform_lookup_allowed) {
/*
* Either we are not using DT or ACPI, or their lookup
* did not return a result. In that case, use platform
* lookup as a fallback.
*/
dev_dbg(consumer,
"using lookup tables for GPIO lookup\n");
desc = gpiod_find(consumer, con_id, idx, &lookupflags);
}
if (IS_ERR(desc)) {
dev_dbg(consumer, "No GPIO consumer %s found\n", name);
return desc;
}
/*
* If a connection label was passed use that, else attempt to use
* the device name as label
*/
ret = gpiod_request(desc, label);
}
if (ret) {
if (!(ret == -EBUSY && flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE))
return ERR_PTR(ret);
/*
* This happens when there are several consumers for
* the same GPIO line: we just return here without
* further initialization. It is a bit of a hack.
* This is necessary to support fixed regulators.
*
* FIXME: Make this more sane and safe.
*/
dev_info(consumer, "nonexclusive access to GPIO for %s\n", name);
return desc;
}
ret = gpiod_configure_flags(desc, con_id, lookupflags, flags);
if (ret < 0) {
gpiod_put(desc);
dev_dbg(consumer, "setup of GPIO %s failed\n", name);
return ERR_PTR(ret);
}
gpiod_line_state_notify(desc, GPIOLINE_CHANGED_REQUESTED);
return desc;
}
/**
* fwnode_gpiod_get_index - obtain a GPIO from firmware node
* @fwnode: handle of the firmware node
* @con_id: function within the GPIO consumer
* @index: index of the GPIO to obtain for the consumer
* @flags: GPIO initialization flags
* @label: label to attach to the requested GPIO
*
* This function can be used for drivers that get their configuration
* from opaque firmware.
*
* The function properly finds the corresponding GPIO using whatever is the
* underlying firmware interface and then makes sure that the GPIO
* descriptor is requested before it is returned to the caller.
*
* Returns:
* On successful request the GPIO pin is configured in accordance with
* provided @flags.
*
* In case of error an ERR_PTR() is returned.
*/
struct gpio_desc *fwnode_gpiod_get_index(struct fwnode_handle *fwnode,
const char *con_id,
int index,
enum gpiod_flags flags,
const char *label)
{
return gpiod_find_and_request(NULL, fwnode, con_id, index, flags, label, false);
}
EXPORT_SYMBOL_GPL(fwnode_gpiod_get_index);
/**
* gpiod_count - return the number of GPIOs associated with a device / function
* or -ENOENT if no GPIO has been assigned to the requested function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
*/
int gpiod_count(struct device *dev, const char *con_id)
{
const struct fwnode_handle *fwnode = dev ? dev_fwnode(dev) : NULL;
int count = -ENOENT;
if (is_of_node(fwnode))
count = of_gpio_count(fwnode, con_id);
else if (is_acpi_node(fwnode))
count = acpi_gpio_count(fwnode, con_id);
else if (is_software_node(fwnode))
count = swnode_gpio_count(fwnode, con_id);
if (count < 0)
count = platform_gpio_count(dev, con_id);
return count;
}
EXPORT_SYMBOL_GPL(gpiod_count);
/**
* gpiod_get - obtain a GPIO for a given GPIO function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
* @flags: optional GPIO initialization flags
*
* Return the GPIO descriptor corresponding to the function con_id of device
* dev, -ENOENT if no GPIO has been assigned to the requested function, or
* another IS_ERR() code if an error occurred while trying to acquire the GPIO.
*/
struct gpio_desc *__must_check gpiod_get(struct device *dev, const char *con_id,
enum gpiod_flags flags)
{
return gpiod_get_index(dev, con_id, 0, flags);
}
EXPORT_SYMBOL_GPL(gpiod_get);
/**
* gpiod_get_optional - obtain an optional GPIO for a given GPIO function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
* @flags: optional GPIO initialization flags
*
* This is equivalent to gpiod_get(), except that when no GPIO was assigned to
* the requested function it will return NULL. This is convenient for drivers
* that need to handle optional GPIOs.
*/
struct gpio_desc *__must_check gpiod_get_optional(struct device *dev,
const char *con_id,
enum gpiod_flags flags)
{
return gpiod_get_index_optional(dev, con_id, 0, flags);
}
EXPORT_SYMBOL_GPL(gpiod_get_optional);
/**
* gpiod_configure_flags - helper function to configure a given GPIO
* @desc: gpio whose value will be assigned
* @con_id: function within the GPIO consumer
* @lflags: bitmask of gpio_lookup_flags GPIO_* values - returned from
* of_find_gpio() or of_get_gpio_hog()
* @dflags: gpiod_flags - optional GPIO initialization flags
*
* Return 0 on success, -ENOENT if no GPIO has been assigned to the
* requested function and/or index, or another IS_ERR() code if an error
* occurred while trying to acquire the GPIO.
*/
int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
unsigned long lflags, enum gpiod_flags dflags)
{
const char *name = function_name_or_default(con_id);
int ret;
if (lflags & GPIO_ACTIVE_LOW)
set_bit(FLAG_ACTIVE_LOW, &desc->flags);
if (lflags & GPIO_OPEN_DRAIN)
set_bit(FLAG_OPEN_DRAIN, &desc->flags);
else if (dflags & GPIOD_FLAGS_BIT_OPEN_DRAIN) {
/*
* This enforces open drain mode from the consumer side.
* This is necessary for some busses like I2C, but the lookup
* should *REALLY* have specified them as open drain in the
* first place, so print a little warning here.
*/
set_bit(FLAG_OPEN_DRAIN, &desc->flags);
gpiod_warn(desc,
"enforced open drain please flag it properly in DT/ACPI DSDT/board file\n");
}
if (lflags & GPIO_OPEN_SOURCE)
set_bit(FLAG_OPEN_SOURCE, &desc->flags);
if (((lflags & GPIO_PULL_UP) && (lflags & GPIO_PULL_DOWN)) ||
((lflags & GPIO_PULL_UP) && (lflags & GPIO_PULL_DISABLE)) ||
((lflags & GPIO_PULL_DOWN) && (lflags & GPIO_PULL_DISABLE))) {
gpiod_err(desc,
"multiple pull-up, pull-down or pull-disable enabled, invalid configuration\n");
return -EINVAL;
}
if (lflags & GPIO_PULL_UP)
set_bit(FLAG_PULL_UP, &desc->flags);
else if (lflags & GPIO_PULL_DOWN)
set_bit(FLAG_PULL_DOWN, &desc->flags);
else if (lflags & GPIO_PULL_DISABLE)
set_bit(FLAG_BIAS_DISABLE, &desc->flags);
ret = gpiod_set_transitory(desc, (lflags & GPIO_TRANSITORY));
if (ret < 0)
return ret;
/* No particular flag request, return here... */
if (!(dflags & GPIOD_FLAGS_BIT_DIR_SET)) {
gpiod_dbg(desc, "no flags found for GPIO %s\n", name);
return 0;
}
/* Process flags */
if (dflags & GPIOD_FLAGS_BIT_DIR_OUT)
ret = gpiod_direction_output(desc,
!!(dflags & GPIOD_FLAGS_BIT_DIR_VAL));
else
ret = gpiod_direction_input(desc);
return ret;
}
/**
* gpiod_get_index - obtain a GPIO from a multi-index GPIO function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
* @idx: index of the GPIO to obtain in the consumer
* @flags: optional GPIO initialization flags
*
* This variant of gpiod_get() allows to access GPIOs other than the first
* defined one for functions that define several GPIOs.
*
* Return a valid GPIO descriptor, -ENOENT if no GPIO has been assigned to the
* requested function and/or index, or another IS_ERR() code if an error
* occurred while trying to acquire the GPIO.
*/
struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
const char *con_id,
unsigned int idx,
enum gpiod_flags flags)
{
struct fwnode_handle *fwnode = dev ? dev_fwnode(dev) : NULL;
const char *devname = dev ? dev_name(dev) : "?";
const char *label = con_id ?: devname;
return gpiod_find_and_request(dev, fwnode, con_id, idx, flags, label, true);
}
EXPORT_SYMBOL_GPL(gpiod_get_index);
/**
* gpiod_get_index_optional - obtain an optional GPIO from a multi-index GPIO
* function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
* @index: index of the GPIO to obtain in the consumer
* @flags: optional GPIO initialization flags
*
* This is equivalent to gpiod_get_index(), except that when no GPIO with the
* specified index was assigned to the requested function it will return NULL.
* This is convenient for drivers that need to handle optional GPIOs.
*/
struct gpio_desc *__must_check gpiod_get_index_optional(struct device *dev,
const char *con_id,
unsigned int index,
enum gpiod_flags flags)
{
struct gpio_desc *desc;
desc = gpiod_get_index(dev, con_id, index, flags);
if (gpiod_not_found(desc))
return NULL;
return desc;
}
EXPORT_SYMBOL_GPL(gpiod_get_index_optional);
/**
* gpiod_hog - Hog the specified GPIO desc given the provided flags
* @desc: gpio whose value will be assigned
* @name: gpio line name
* @lflags: bitmask of gpio_lookup_flags GPIO_* values - returned from
* of_find_gpio() or of_get_gpio_hog()
* @dflags: gpiod_flags - optional GPIO initialization flags
*/
int gpiod_hog(struct gpio_desc *desc, const char *name,
unsigned long lflags, enum gpiod_flags dflags)
{
struct gpio_device *gdev = desc->gdev;
struct gpio_desc *local_desc;
int hwnum;
int ret;
CLASS(gpio_chip_guard, guard)(desc);
if (!guard.gc)
return -ENODEV;
if (test_and_set_bit(FLAG_IS_HOGGED, &desc->flags))
return 0;
hwnum = gpio_chip_hwgpio(desc);
local_desc = gpiochip_request_own_desc(guard.gc, hwnum, name,
lflags, dflags);
if (IS_ERR(local_desc)) {
clear_bit(FLAG_IS_HOGGED, &desc->flags);
ret = PTR_ERR(local_desc);
pr_err("requesting hog GPIO %s (chip %s, offset %d) failed, %d\n",
name, gdev->label, hwnum, ret);
return ret;
}
gpiod_dbg(desc, "hogged as %s%s\n",
(dflags & GPIOD_FLAGS_BIT_DIR_OUT) ? "output" : "input",
(dflags & GPIOD_FLAGS_BIT_DIR_OUT) ?
(dflags & GPIOD_FLAGS_BIT_DIR_VAL) ? "/high" : "/low" : "");
return 0;
}
/**
* gpiochip_free_hogs - Scan gpio-controller chip and release GPIO hog
* @gc: gpio chip to act on
*/
static void gpiochip_free_hogs(struct gpio_chip *gc)
{
struct gpio_desc *desc;
for_each_gpio_desc_with_flag(gc, desc, FLAG_IS_HOGGED) gpiochip_free_own_desc(desc);
}
/**
* gpiod_get_array - obtain multiple GPIOs from a multi-index GPIO function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
* @flags: optional GPIO initialization flags
*
* This function acquires all the GPIOs defined under a given function.
*
* Return a struct gpio_descs containing an array of descriptors, -ENOENT if
* no GPIO has been assigned to the requested function, or another IS_ERR()
* code if an error occurred while trying to acquire the GPIOs.
*/
struct gpio_descs *__must_check gpiod_get_array(struct device *dev,
const char *con_id,
enum gpiod_flags flags)
{
struct gpio_desc *desc;
struct gpio_descs *descs;
struct gpio_array *array_info = NULL;
struct gpio_chip *gc;
int count, bitmap_size;
size_t descs_size;
count = gpiod_count(dev, con_id);
if (count < 0)
return ERR_PTR(count);
descs_size = struct_size(descs, desc, count);
descs = kzalloc(descs_size, GFP_KERNEL);
if (!descs)
return ERR_PTR(-ENOMEM);
for (descs->ndescs = 0; descs->ndescs < count; descs->ndescs++) {
desc = gpiod_get_index(dev, con_id, descs->ndescs, flags);
if (IS_ERR(desc)) {
gpiod_put_array(descs);
return ERR_CAST(desc);
}
descs->desc[descs->ndescs] = desc;
gc = gpiod_to_chip(desc);
/*
* If pin hardware number of array member 0 is also 0, select
* its chip as a candidate for fast bitmap processing path.
*/
if (descs->ndescs == 0 && gpio_chip_hwgpio(desc) == 0) {
struct gpio_descs *array;
bitmap_size = BITS_TO_LONGS(gc->ngpio > count ?
gc->ngpio : count);
array = krealloc(descs, descs_size +
struct_size(array_info, invert_mask, 3 * bitmap_size),
GFP_KERNEL | __GFP_ZERO);
if (!array) {
gpiod_put_array(descs);
return ERR_PTR(-ENOMEM);
}
descs = array;
array_info = (void *)descs + descs_size;
array_info->get_mask = array_info->invert_mask +
bitmap_size;
array_info->set_mask = array_info->get_mask +
bitmap_size;
array_info->desc = descs->desc;
array_info->size = count;
array_info->chip = gc;
bitmap_set(array_info->get_mask, descs->ndescs,
count - descs->ndescs);
bitmap_set(array_info->set_mask, descs->ndescs,
count - descs->ndescs);
descs->info = array_info;
}
/* If there is no cache for fast bitmap processing path, continue */
if (!array_info)
continue;
/* Unmark array members which don't belong to the 'fast' chip */
if (array_info->chip != gc) {
__clear_bit(descs->ndescs, array_info->get_mask);
__clear_bit(descs->ndescs, array_info->set_mask);
}
/*
* Detect array members which belong to the 'fast' chip
* but their pins are not in hardware order.
*/
else if (gpio_chip_hwgpio(desc) != descs->ndescs) {
/*
* Don't use fast path if all array members processed so
* far belong to the same chip as this one but its pin
* hardware number is different from its array index.
*/
if (bitmap_full(array_info->get_mask, descs->ndescs)) {
array_info = NULL;
} else {
__clear_bit(descs->ndescs,
array_info->get_mask);
__clear_bit(descs->ndescs,
array_info->set_mask);
}
} else {
/* Exclude open drain or open source from fast output */
if (gpiochip_line_is_open_drain(gc, descs->ndescs) ||
gpiochip_line_is_open_source(gc, descs->ndescs))
__clear_bit(descs->ndescs,
array_info->set_mask);
/* Identify 'fast' pins which require invertion */
if (gpiod_is_active_low(desc))
__set_bit(descs->ndescs,
array_info->invert_mask);
}
}
if (array_info)
dev_dbg(dev,
"GPIO array info: chip=%s, size=%d, get_mask=%lx, set_mask=%lx, invert_mask=%lx\n",
array_info->chip->label, array_info->size,
*array_info->get_mask, *array_info->set_mask,
*array_info->invert_mask);
return descs;
}
EXPORT_SYMBOL_GPL(gpiod_get_array);
/**
* gpiod_get_array_optional - obtain multiple GPIOs from a multi-index GPIO
* function
* @dev: GPIO consumer, can be NULL for system-global GPIOs
* @con_id: function within the GPIO consumer
* @flags: optional GPIO initialization flags
*
* This is equivalent to gpiod_get_array(), except that when no GPIO was
* assigned to the requested function it will return NULL.
*/
struct gpio_descs *__must_check gpiod_get_array_optional(struct device *dev,
const char *con_id,
enum gpiod_flags flags)
{
struct gpio_descs *descs;
descs = gpiod_get_array(dev, con_id, flags);
if (gpiod_not_found(descs))
return NULL;
return descs;
}
EXPORT_SYMBOL_GPL(gpiod_get_array_optional);
/**
* gpiod_put - dispose of a GPIO descriptor
* @desc: GPIO descriptor to dispose of
*
* No descriptor can be used after gpiod_put() has been called on it.
*/
void gpiod_put(struct gpio_desc *desc)
{
if (desc)
gpiod_free(desc);
}
EXPORT_SYMBOL_GPL(gpiod_put);
/**
* gpiod_put_array - dispose of multiple GPIO descriptors
* @descs: struct gpio_descs containing an array of descriptors
*/
void gpiod_put_array(struct gpio_descs *descs)
{
unsigned int i;
for (i = 0; i < descs->ndescs; i++)
gpiod_put(descs->desc[i]);
kfree(descs);
}
EXPORT_SYMBOL_GPL(gpiod_put_array);
static int gpio_stub_drv_probe(struct device *dev)
{
/*
* The DT node of some GPIO chips have a "compatible" property, but
* never have a struct device added and probed by a driver to register
* the GPIO chip with gpiolib. In such cases, fw_devlink=on will cause
* the consumers of the GPIO chip to get probe deferred forever because
* they will be waiting for a device associated with the GPIO chip
* firmware node to get added and bound to a driver.
*
* To allow these consumers to probe, we associate the struct
* gpio_device of the GPIO chip with the firmware node and then simply
* bind it to this stub driver.
*/
return 0;
}
static struct device_driver gpio_stub_drv = {
.name = "gpio_stub_drv",
.bus = &gpio_bus_type,
.probe = gpio_stub_drv_probe,
};
static int __init gpiolib_dev_init(void)
{
int ret;
/* Register GPIO sysfs bus */
ret = bus_register(&gpio_bus_type);
if (ret < 0) {
pr_err("gpiolib: could not register GPIO bus type\n");
return ret;
}
ret = driver_register(&gpio_stub_drv);
if (ret < 0) {
pr_err("gpiolib: could not register GPIO stub driver\n");
bus_unregister(&gpio_bus_type);
return ret;
}
ret = alloc_chrdev_region(&gpio_devt, 0, GPIO_DEV_MAX, GPIOCHIP_NAME);
if (ret < 0) {
pr_err("gpiolib: failed to allocate char dev region\n");
driver_unregister(&gpio_stub_drv);
bus_unregister(&gpio_bus_type);
return ret;
}
gpiolib_initialized = true;
gpiochip_setup_devs();
#if IS_ENABLED(CONFIG_OF_DYNAMIC) && IS_ENABLED(CONFIG_OF_GPIO)
WARN_ON(of_reconfig_notifier_register(&gpio_of_notifier));
#endif /* CONFIG_OF_DYNAMIC && CONFIG_OF_GPIO */
return ret;
}
core_initcall(gpiolib_dev_init);
#ifdef CONFIG_DEBUG_FS
static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev)
{
bool active_low, is_irq, is_out;
unsigned int gpio = gdev->base;
struct gpio_desc *desc;
struct gpio_chip *gc;
int value;
guard(srcu)(&gdev->srcu);
gc = srcu_dereference(gdev->chip, &gdev->srcu);
if (!gc) {
seq_puts(s, "Underlying GPIO chip is gone\n");
return;
}
for_each_gpio_desc(gc, desc) {
guard(srcu)(&desc->srcu);
if (test_bit(FLAG_REQUESTED, &desc->flags)) {
gpiod_get_direction(desc);
is_out = test_bit(FLAG_IS_OUT, &desc->flags);
value = gpio_chip_get_value(gc, desc);
is_irq = test_bit(FLAG_USED_AS_IRQ, &desc->flags);
active_low = test_bit(FLAG_ACTIVE_LOW, &desc->flags);
seq_printf(s, " gpio-%-3d (%-20.20s|%-20.20s) %s %s %s%s\n",
gpio, desc->name ?: "", gpiod_get_label(desc),
is_out ? "out" : "in ",
value >= 0 ? (value ? "hi" : "lo") : "? ",
is_irq ? "IRQ " : "",
active_low ? "ACTIVE LOW" : "");
} else if (desc->name) {
seq_printf(s, " gpio-%-3d (%-20.20s)\n", gpio, desc->name);
}
gpio++;
}
}
struct gpiolib_seq_priv {
bool newline;
int idx;
};
static void *gpiolib_seq_start(struct seq_file *s, loff_t *pos)
{
struct gpiolib_seq_priv *priv;
struct gpio_device *gdev;
loff_t index = *pos;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return NULL;
s->private = priv;
priv->idx = srcu_read_lock(&gpio_devices_srcu);
list_for_each_entry_srcu(gdev, &gpio_devices, list,
srcu_read_lock_held(&gpio_devices_srcu)) {
if (index-- == 0)
return gdev;
}
return NULL;
}
static void *gpiolib_seq_next(struct seq_file *s, void *v, loff_t *pos)
{
struct gpiolib_seq_priv *priv = s->private;
struct gpio_device *gdev = v, *next;
next = list_entry_rcu(gdev->list.next, struct gpio_device, list);
gdev = &next->list == &gpio_devices ? NULL : next;
priv->newline = true;
++*pos;
return gdev;
}
static void gpiolib_seq_stop(struct seq_file *s, void *v)
{
struct gpiolib_seq_priv *priv = s->private;
srcu_read_unlock(&gpio_devices_srcu, priv->idx);
kfree(priv);
}
static int gpiolib_seq_show(struct seq_file *s, void *v)
{
struct gpiolib_seq_priv *priv = s->private;
struct gpio_device *gdev = v;
struct gpio_chip *gc;
struct device *parent;
guard(srcu)(&gdev->srcu);
gc = srcu_dereference(gdev->chip, &gdev->srcu);
if (!gc) {
seq_printf(s, "%s%s: (dangling chip)",
priv->newline ? "\n" : "",
dev_name(&gdev->dev));
return 0;
}
seq_printf(s, "%s%s: GPIOs %d-%d", priv->newline ? "\n" : "",
dev_name(&gdev->dev),
gdev->base, gdev->base + gdev->ngpio - 1);
parent = gc->parent;
if (parent)
seq_printf(s, ", parent: %s/%s",
parent->bus ? parent->bus->name : "no-bus",
dev_name(parent));
if (gc->label)
seq_printf(s, ", %s", gc->label);
if (gc->can_sleep)
seq_printf(s, ", can sleep");
seq_printf(s, ":\n");
if (gc->dbg_show)
gc->dbg_show(s, gc);
else
gpiolib_dbg_show(s, gdev);
return 0;
}
static const struct seq_operations gpiolib_sops = {
.start = gpiolib_seq_start,
.next = gpiolib_seq_next,
.stop = gpiolib_seq_stop,
.show = gpiolib_seq_show,
};
DEFINE_SEQ_ATTRIBUTE(gpiolib);
static int __init gpiolib_debugfs_init(void)
{
/* /sys/kernel/debug/gpio */
debugfs_create_file("gpio", 0444, NULL, NULL, &gpiolib_fops);
return 0;
}
subsys_initcall(gpiolib_debugfs_init);
#endif /* DEBUG_FS */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Sound core. This file is composed of two parts. sound_class
* which is common to both OSS and ALSA and OSS sound core which
* is used OSS or emulation of it.
*/
/*
* First, the common part.
*/
#include <linux/module.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/kdev_t.h>
#include <linux/major.h>
#include <sound/core.h>
#ifdef CONFIG_SOUND_OSS_CORE
static int __init init_oss_soundcore(void);
static void cleanup_oss_soundcore(void);
#else
static inline int init_oss_soundcore(void) { return 0; }
static inline void cleanup_oss_soundcore(void) { }
#endif
MODULE_DESCRIPTION("Core sound module");
MODULE_AUTHOR("Alan Cox");
MODULE_LICENSE("GPL");
static char *sound_devnode(const struct device *dev, umode_t *mode)
{
if (MAJOR(dev->devt) == SOUND_MAJOR)
return NULL;
return kasprintf(GFP_KERNEL, "snd/%s", dev_name(dev));
}
const struct class sound_class = {
.name = "sound",
.devnode = sound_devnode,
};
EXPORT_SYMBOL(sound_class);
static int __init init_soundcore(void)
{
int rc;
rc = init_oss_soundcore();
if (rc)
return rc;
rc = class_register(&sound_class);
if (rc) {
cleanup_oss_soundcore();
return rc;
}
return 0;
}
static void __exit cleanup_soundcore(void)
{
cleanup_oss_soundcore();
class_unregister(&sound_class);
}
subsys_initcall(init_soundcore);
module_exit(cleanup_soundcore);
#ifdef CONFIG_SOUND_OSS_CORE
/*
* OSS sound core handling. Breaks out sound functions to submodules
*
* Author: Alan Cox <alan@lxorguk.ukuu.org.uk>
*
* Fixes:
*
* --------------------
*
* Top level handler for the sound subsystem. Various devices can
* plug into this. The fact they don't all go via OSS doesn't mean
* they don't have to implement the OSS API. There is a lot of logic
* to keeping much of the OSS weight out of the code in a compatibility
* module, but it's up to the driver to rember to load it...
*
* The code provides a set of functions for registration of devices
* by type. This is done rather than providing a single call so that
* we can hide any future changes in the internals (eg when we go to
* 32bit dev_t) from the modules and their interface.
*
* Secondly we need to allocate the dsp, dsp16 and audio devices as
* one. Thus we misuse the chains a bit to simplify this.
*
* Thirdly to make it more fun and for 2.3.x and above we do all
* of this using fine grained locking.
*
* FIXME: we have to resolve modules and fine grained load/unload
* locking at some point in 2.3.x.
*/
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sound.h>
#include <linux/kmod.h>
#define SOUND_STEP 16
struct sound_unit
{
int unit_minor;
const struct file_operations *unit_fops;
struct sound_unit *next;
char name[32];
};
/*
* By default, OSS sound_core claims full legacy minor range (0-255)
* of SOUND_MAJOR to trap open attempts to any sound minor and
* requests modules using custom sound-slot/service-* module aliases.
* The only benefit of doing this is allowing use of custom module
* aliases instead of the standard char-major-* ones. This behavior
* prevents alternative OSS implementation and is scheduled to be
* removed.
*
* CONFIG_SOUND_OSS_CORE_PRECLAIM and soundcore.preclaim_oss kernel
* parameter are added to allow distros and developers to try and
* switch to alternative implementations without needing to rebuild
* the kernel in the meantime. If preclaim_oss is non-zero, the
* kernel will behave the same as before. All SOUND_MAJOR minors are
* preclaimed and the custom module aliases along with standard chrdev
* ones are emitted if a missing device is opened. If preclaim_oss is
* zero, sound_core only grabs what's actually in use and for missing
* devices only the standard chrdev aliases are requested.
*
* All these clutters are scheduled to be removed along with
* sound-slot/service-* module aliases.
*/
static int preclaim_oss = IS_ENABLED(CONFIG_SOUND_OSS_CORE_PRECLAIM);
module_param(preclaim_oss, int, 0444);
static int soundcore_open(struct inode *, struct file *);
static const struct file_operations soundcore_fops =
{
/* We must have an owner or the module locking fails */
.owner = THIS_MODULE,
.open = soundcore_open,
.llseek = noop_llseek,
};
/*
* Low level list operator. Scan the ordered list, find a hole and
* join into it. Called with the lock asserted
*/
static int __sound_insert_unit(struct sound_unit * s, struct sound_unit **list, const struct file_operations *fops, int index, int low, int top)
{
int n=low;
if (index < 0) { /* first free */
while (*list && (*list)->unit_minor<n)
list=&((*list)->next);
while(n<top)
{
/* Found a hole ? */
if(*list==NULL || (*list)->unit_minor>n)
break;
list=&((*list)->next);
n+=SOUND_STEP;
}
if(n>=top)
return -ENOENT;
} else {
n = low+(index*16);
while (*list) {
if ((*list)->unit_minor==n)
return -EBUSY;
if ((*list)->unit_minor>n)
break;
list=&((*list)->next);
}
}
/*
* Fill it in
*/
s->unit_minor=n;
s->unit_fops=fops;
/*
* Link it
*/
s->next=*list;
*list=s;
return n;
}
/*
* Remove a node from the chain. Called with the lock asserted
*/
static struct sound_unit *__sound_remove_unit(struct sound_unit **list, int unit)
{
while(*list)
{
struct sound_unit *p=*list;
if(p->unit_minor==unit)
{
*list=p->next;
return p;
}
list=&(p->next);
}
printk(KERN_ERR "Sound device %d went missing!\n", unit);
return NULL;
}
/*
* This lock guards the sound loader list.
*/
static DEFINE_SPINLOCK(sound_loader_lock);
/*
* Allocate the controlling structure and add it to the sound driver
* list. Acquires locks as needed
*/
static int sound_insert_unit(struct sound_unit **list, const struct file_operations *fops, int index, int low, int top, const char *name, umode_t mode, struct device *dev)
{
struct sound_unit *s = kmalloc(sizeof(*s), GFP_KERNEL);
int r;
if (!s)
return -ENOMEM;
spin_lock(&sound_loader_lock);
retry:
r = __sound_insert_unit(s, list, fops, index, low, top);
spin_unlock(&sound_loader_lock);
if (r < 0)
goto fail;
else if (r < SOUND_STEP)
sprintf(s->name, "sound/%s", name);
else
sprintf(s->name, "sound/%s%d", name, r / SOUND_STEP);
if (!preclaim_oss) {
/*
* Something else might have grabbed the minor. If
* first free slot is requested, rescan with @low set
* to the next unit; otherwise, -EBUSY.
*/
r = __register_chrdev(SOUND_MAJOR, s->unit_minor, 1, s->name,
&soundcore_fops);
if (r < 0) {
spin_lock(&sound_loader_lock);
__sound_remove_unit(list, s->unit_minor);
if (index < 0) {
low = s->unit_minor + SOUND_STEP;
goto retry;
}
spin_unlock(&sound_loader_lock);
r = -EBUSY;
goto fail;
}
}
device_create(&sound_class, dev, MKDEV(SOUND_MAJOR, s->unit_minor),
NULL, "%s", s->name+6);
return s->unit_minor;
fail:
kfree(s);
return r;
}
/*
* Remove a unit. Acquires locks as needed. The drivers MUST have
* completed the removal before their file operations become
* invalid.
*/
static void sound_remove_unit(struct sound_unit **list, int unit)
{
struct sound_unit *p;
spin_lock(&sound_loader_lock);
p = __sound_remove_unit(list, unit);
spin_unlock(&sound_loader_lock);
if (p) {
if (!preclaim_oss)
__unregister_chrdev(SOUND_MAJOR, p->unit_minor, 1,
p->name);
device_destroy(&sound_class, MKDEV(SOUND_MAJOR, p->unit_minor));
kfree(p);
}
}
/*
* Allocations
*
* 0 *16 Mixers
* 1 *8 Sequencers
* 2 *16 Midi
* 3 *16 DSP
* 4 *16 SunDSP
* 5 *16 DSP16
* 6 -- sndstat (obsolete)
* 7 *16 unused
* 8 -- alternate sequencer (see above)
* 9 *16 raw synthesizer access
* 10 *16 unused
* 11 *16 unused
* 12 *16 unused
* 13 *16 unused
* 14 *16 unused
* 15 *16 unused
*/
static struct sound_unit *chains[SOUND_STEP];
/**
* register_sound_special_device - register a special sound node
* @fops: File operations for the driver
* @unit: Unit number to allocate
* @dev: device pointer
*
* Allocate a special sound device by minor number from the sound
* subsystem.
*
* Return: The allocated number is returned on success. On failure,
* a negative error code is returned.
*/
int register_sound_special_device(const struct file_operations *fops, int unit,
struct device *dev)
{
const int chain = unit % SOUND_STEP;
int max_unit = 256;
const char *name;
char _name[16];
switch (chain) {
case 0:
name = "mixer";
break;
case 1:
name = "sequencer";
if (unit >= SOUND_STEP)
goto __unknown;
max_unit = unit + 1;
break;
case 2:
name = "midi";
break;
case 3:
name = "dsp";
break;
case 4:
name = "audio";
break;
case 5:
name = "dspW";
break;
case 8:
name = "sequencer2";
if (unit >= SOUND_STEP)
goto __unknown;
max_unit = unit + 1;
break;
case 9:
name = "dmmidi";
break;
case 10:
name = "dmfm";
break;
case 12:
name = "adsp";
break;
case 13:
name = "amidi";
break;
case 14:
name = "admmidi";
break;
default:
{
__unknown:
sprintf(_name, "unknown%d", chain);
if (unit >= SOUND_STEP)
strcat(_name, "-");
name = _name;
}
break;
}
return sound_insert_unit(&chains[chain], fops, -1, unit, max_unit,
name, 0600, dev);
}
EXPORT_SYMBOL(register_sound_special_device);
int register_sound_special(const struct file_operations *fops, int unit)
{
return register_sound_special_device(fops, unit, NULL);
}
EXPORT_SYMBOL(register_sound_special);
/**
* register_sound_mixer - register a mixer device
* @fops: File operations for the driver
* @dev: Unit number to allocate
*
* Allocate a mixer device. Unit is the number of the mixer requested.
* Pass -1 to request the next free mixer unit.
*
* Return: On success, the allocated number is returned. On failure,
* a negative error code is returned.
*/
int register_sound_mixer(const struct file_operations *fops, int dev)
{
return sound_insert_unit(&chains[0], fops, dev, 0, 128,
"mixer", 0600, NULL);
}
EXPORT_SYMBOL(register_sound_mixer);
/*
* DSP's are registered as a triple. Register only one and cheat
* in open - see below.
*/
/**
* register_sound_dsp - register a DSP device
* @fops: File operations for the driver
* @dev: Unit number to allocate
*
* Allocate a DSP device. Unit is the number of the DSP requested.
* Pass -1 to request the next free DSP unit.
*
* This function allocates both the audio and dsp device entries together
* and will always allocate them as a matching pair - eg dsp3/audio3
*
* Return: On success, the allocated number is returned. On failure,
* a negative error code is returned.
*/
int register_sound_dsp(const struct file_operations *fops, int dev)
{
return sound_insert_unit(&chains[3], fops, dev, 3, 131,
"dsp", 0600, NULL);
}
EXPORT_SYMBOL(register_sound_dsp);
/**
* unregister_sound_special - unregister a special sound device
* @unit: unit number to allocate
*
* Release a sound device that was allocated with
* register_sound_special(). The unit passed is the return value from
* the register function.
*/
void unregister_sound_special(int unit)
{
sound_remove_unit(&chains[unit % SOUND_STEP], unit);
}
EXPORT_SYMBOL(unregister_sound_special);
/**
* unregister_sound_mixer - unregister a mixer
* @unit: unit number to allocate
*
* Release a sound device that was allocated with register_sound_mixer().
* The unit passed is the return value from the register function.
*/
void unregister_sound_mixer(int unit)
{
sound_remove_unit(&chains[0], unit);
}
EXPORT_SYMBOL(unregister_sound_mixer);
/**
* unregister_sound_dsp - unregister a DSP device
* @unit: unit number to allocate
*
* Release a sound device that was allocated with register_sound_dsp().
* The unit passed is the return value from the register function.
*
* Both of the allocated units are released together automatically.
*/
void unregister_sound_dsp(int unit)
{
sound_remove_unit(&chains[3], unit);
}
EXPORT_SYMBOL(unregister_sound_dsp);
static struct sound_unit *__look_for_unit(int chain, int unit)
{
struct sound_unit *s;
s=chains[chain];
while(s && s->unit_minor <= unit)
{
if(s->unit_minor==unit)
return s;
s=s->next;
}
return NULL;
}
static int soundcore_open(struct inode *inode, struct file *file)
{
int chain;
int unit = iminor(inode);
struct sound_unit *s;
const struct file_operations *new_fops = NULL;
chain=unit&0x0F;
if(chain==4 || chain==5) /* dsp/audio/dsp16 */
{
unit&=0xF0;
unit|=3;
chain=3;
}
spin_lock(&sound_loader_lock);
s = __look_for_unit(chain, unit);
if (s)
new_fops = fops_get(s->unit_fops);
if (preclaim_oss && !new_fops) {
spin_unlock(&sound_loader_lock);
/*
* Please, don't change this order or code.
* For ALSA slot means soundcard and OSS emulation code
* comes as add-on modules which aren't depend on
* ALSA toplevel modules for soundcards, thus we need
* load them at first. [Jaroslav Kysela <perex@jcu.cz>]
*/
request_module("sound-slot-%i", unit>>4);
request_module("sound-service-%i-%i", unit>>4, chain);
/*
* sound-slot/service-* module aliases are scheduled
* for removal in favor of the standard char-major-*
* module aliases. For the time being, generate both
* the legacy and standard module aliases to ease
* transition.
*/
if (request_module("char-major-%d-%d", SOUND_MAJOR, unit) > 0)
request_module("char-major-%d", SOUND_MAJOR);
spin_lock(&sound_loader_lock);
s = __look_for_unit(chain, unit);
if (s)
new_fops = fops_get(s->unit_fops);
}
spin_unlock(&sound_loader_lock);
if (!new_fops)
return -ENODEV;
/*
* We rely upon the fact that we can't be unloaded while the
* subdriver is there.
*/
replace_fops(file, new_fops);
if (!file->f_op->open)
return -ENODEV;
return file->f_op->open(inode, file);
}
MODULE_ALIAS_CHARDEV_MAJOR(SOUND_MAJOR);
static void cleanup_oss_soundcore(void)
{
/* We have nothing to really do here - we know the lists must be
empty */
unregister_chrdev(SOUND_MAJOR, "sound");
}
static int __init init_oss_soundcore(void)
{
if (preclaim_oss &&
register_chrdev(SOUND_MAJOR, "sound", &soundcore_fops) < 0) {
printk(KERN_ERR "soundcore: sound device already in use.\n");
return -EBUSY;
}
return 0;
}
#endif /* CONFIG_SOUND_OSS_CORE */
// SPDX-License-Identifier: GPL-2.0 or MIT
/*
* Copyright 2018 Noralf Trønnes
*/
#include <linux/iosys-map.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <drm/drm_client.h>
#include <drm/drm_debugfs.h>
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_framebuffer.h>
#include <drm/drm_gem.h>
#include <drm/drm_mode.h>
#include <drm/drm_print.h>
#include "drm_crtc_internal.h"
#include "drm_internal.h"
/**
* DOC: overview
*
* This library provides support for clients running in the kernel like fbdev and bootsplash.
*
* GEM drivers which provide a GEM based dumb buffer with a virtual address are supported.
*/
static int drm_client_open(struct drm_client_dev *client)
{
struct drm_device *dev = client->dev;
struct drm_file *file;
file = drm_file_alloc(dev->primary);
if (IS_ERR(file))
return PTR_ERR(file);
mutex_lock(&dev->filelist_mutex);
list_add(&file->lhead, &dev->filelist_internal);
mutex_unlock(&dev->filelist_mutex);
client->file = file;
return 0;
}
static void drm_client_close(struct drm_client_dev *client)
{
struct drm_device *dev = client->dev;
mutex_lock(&dev->filelist_mutex);
list_del(&client->file->lhead);
mutex_unlock(&dev->filelist_mutex);
drm_file_free(client->file);
}
/**
* drm_client_init - Initialise a DRM client
* @dev: DRM device
* @client: DRM client
* @name: Client name
* @funcs: DRM client functions (optional)
*
* This initialises the client and opens a &drm_file.
* Use drm_client_register() to complete the process.
* The caller needs to hold a reference on @dev before calling this function.
* The client is freed when the &drm_device is unregistered. See drm_client_release().
*
* Returns:
* Zero on success or negative error code on failure.
*/
int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
const char *name, const struct drm_client_funcs *funcs)
{
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET) || !dev->driver->dumb_create)
return -EOPNOTSUPP;
client->dev = dev;
client->name = name;
client->funcs = funcs;
ret = drm_client_modeset_create(client);
if (ret)
return ret;
ret = drm_client_open(client);
if (ret)
goto err_free;
drm_dev_get(dev);
return 0;
err_free:
drm_client_modeset_free(client);
return ret;
}
EXPORT_SYMBOL(drm_client_init);
/**
* drm_client_register - Register client
* @client: DRM client
*
* Add the client to the &drm_device client list to activate its callbacks.
* @client must be initialized by a call to drm_client_init(). After
* drm_client_register() it is no longer permissible to call drm_client_release()
* directly (outside the unregister callback), instead cleanup will happen
* automatically on driver unload.
*
* Registering a client generates a hotplug event that allows the client
* to set up its display from pre-existing outputs. The client must have
* initialized its state to able to handle the hotplug event successfully.
*/
void drm_client_register(struct drm_client_dev *client)
{
struct drm_device *dev = client->dev;
int ret;
mutex_lock(&dev->clientlist_mutex);
list_add(&client->list, &dev->clientlist);
if (client->funcs && client->funcs->hotplug) {
/*
* Perform an initial hotplug event to pick up the
* display configuration for the client. This step
* has to be performed *after* registering the client
* in the list of clients, or a concurrent hotplug
* event might be lost; leaving the display off.
*
* Hold the clientlist_mutex as for a regular hotplug
* event.
*/
ret = client->funcs->hotplug(client);
if (ret)
drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
}
mutex_unlock(&dev->clientlist_mutex);
}
EXPORT_SYMBOL(drm_client_register);
/**
* drm_client_release - Release DRM client resources
* @client: DRM client
*
* Releases resources by closing the &drm_file that was opened by drm_client_init().
* It is called automatically if the &drm_client_funcs.unregister callback is _not_ set.
*
* This function should only be called from the unregister callback. An exception
* is fbdev which cannot free the buffer if userspace has open file descriptors.
*
* Note:
* Clients cannot initiate a release by themselves. This is done to keep the code simple.
* The driver has to be unloaded before the client can be unloaded.
*/
void drm_client_release(struct drm_client_dev *client)
{
struct drm_device *dev = client->dev;
drm_dbg_kms(dev, "%s\n", client->name);
drm_client_modeset_free(client);
drm_client_close(client);
drm_dev_put(dev);
}
EXPORT_SYMBOL(drm_client_release);
void drm_client_dev_unregister(struct drm_device *dev)
{
struct drm_client_dev *client, *tmp;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return;
mutex_lock(&dev->clientlist_mutex); list_for_each_entry_safe(client, tmp, &dev->clientlist, list) { list_del(&client->list); if (client->funcs && client->funcs->unregister) { client->funcs->unregister(client);
} else {
drm_client_release(client);
kfree(client);
}
}
mutex_unlock(&dev->clientlist_mutex);
}
/**
* drm_client_dev_hotplug - Send hotplug event to clients
* @dev: DRM device
*
* This function calls the &drm_client_funcs.hotplug callback on the attached clients.
*
* drm_kms_helper_hotplug_event() calls this function, so drivers that use it
* don't need to call this function themselves.
*/
void drm_client_dev_hotplug(struct drm_device *dev)
{
struct drm_client_dev *client;
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return;
if (!dev->mode_config.num_connector) {
drm_dbg_kms(dev, "No connectors found, will not send hotplug events!\n");
return;
}
mutex_lock(&dev->clientlist_mutex);
list_for_each_entry(client, &dev->clientlist, list) {
if (!client->funcs || !client->funcs->hotplug)
continue;
if (client->hotplug_failed)
continue;
ret = client->funcs->hotplug(client);
drm_dbg_kms(dev, "%s: ret=%d\n", client->name, ret);
if (ret)
client->hotplug_failed = true;
}
mutex_unlock(&dev->clientlist_mutex);
}
EXPORT_SYMBOL(drm_client_dev_hotplug);
void drm_client_dev_restore(struct drm_device *dev)
{
struct drm_client_dev *client;
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return;
mutex_lock(&dev->clientlist_mutex);
list_for_each_entry(client, &dev->clientlist, list) {
if (!client->funcs || !client->funcs->restore)
continue;
ret = client->funcs->restore(client);
drm_dbg_kms(dev, "%s: ret=%d\n", client->name, ret);
if (!ret) /* The first one to return zero gets the privilege to restore */
break;
}
mutex_unlock(&dev->clientlist_mutex);
}
static void drm_client_buffer_delete(struct drm_client_buffer *buffer)
{
if (buffer->gem) {
drm_gem_vunmap_unlocked(buffer->gem, &buffer->map);
drm_gem_object_put(buffer->gem);
}
kfree(buffer);
}
static struct drm_client_buffer *
drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height,
u32 format, u32 *handle)
{
const struct drm_format_info *info = drm_format_info(format);
struct drm_mode_create_dumb dumb_args = { };
struct drm_device *dev = client->dev;
struct drm_client_buffer *buffer;
struct drm_gem_object *obj;
int ret;
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
if (!buffer)
return ERR_PTR(-ENOMEM);
buffer->client = client;
dumb_args.width = width;
dumb_args.height = height;
dumb_args.bpp = drm_format_info_bpp(info, 0);
ret = drm_mode_create_dumb(dev, &dumb_args, client->file);
if (ret)
goto err_delete;
obj = drm_gem_object_lookup(client->file, dumb_args.handle);
if (!obj) {
ret = -ENOENT;
goto err_delete;
}
buffer->pitch = dumb_args.pitch;
buffer->gem = obj;
*handle = dumb_args.handle;
return buffer;
err_delete:
drm_client_buffer_delete(buffer);
return ERR_PTR(ret);
}
/**
* drm_client_buffer_vmap - Map DRM client buffer into address space
* @buffer: DRM client buffer
* @map_copy: Returns the mapped memory's address
*
* This function maps a client buffer into kernel address space. If the
* buffer is already mapped, it returns the existing mapping's address.
*
* Client buffer mappings are not ref'counted. Each call to
* drm_client_buffer_vmap() should be followed by a call to
* drm_client_buffer_vunmap(); or the client buffer should be mapped
* throughout its lifetime.
*
* The returned address is a copy of the internal value. In contrast to
* other vmap interfaces, you don't need it for the client's vunmap
* function. So you can modify it at will during blit and draw operations.
*
* Returns:
* 0 on success, or a negative errno code otherwise.
*/
int
drm_client_buffer_vmap(struct drm_client_buffer *buffer,
struct iosys_map *map_copy)
{
struct iosys_map *map = &buffer->map;
int ret;
/*
* FIXME: The dependency on GEM here isn't required, we could
* convert the driver handle to a dma-buf instead and use the
* backend-agnostic dma-buf vmap support instead. This would
* require that the handle2fd prime ioctl is reworked to pull the
* fd_install step out of the driver backend hooks, to make that
* final step optional for internal users.
*/
ret = drm_gem_vmap_unlocked(buffer->gem, map);
if (ret)
return ret;
*map_copy = *map;
return 0;
}
EXPORT_SYMBOL(drm_client_buffer_vmap);
/**
* drm_client_buffer_vunmap - Unmap DRM client buffer
* @buffer: DRM client buffer
*
* This function removes a client buffer's memory mapping. Calling this
* function is only required by clients that manage their buffer mappings
* by themselves.
*/
void drm_client_buffer_vunmap(struct drm_client_buffer *buffer)
{
struct iosys_map *map = &buffer->map;
drm_gem_vunmap_unlocked(buffer->gem, map);
}
EXPORT_SYMBOL(drm_client_buffer_vunmap);
static void drm_client_buffer_rmfb(struct drm_client_buffer *buffer)
{
int ret;
if (!buffer->fb)
return;
ret = drm_mode_rmfb(buffer->client->dev, buffer->fb->base.id, buffer->client->file);
if (ret)
drm_err(buffer->client->dev,
"Error removing FB:%u (%d)\n", buffer->fb->base.id, ret);
buffer->fb = NULL;
}
static int drm_client_buffer_addfb(struct drm_client_buffer *buffer,
u32 width, u32 height, u32 format,
u32 handle)
{
struct drm_client_dev *client = buffer->client;
struct drm_mode_fb_cmd2 fb_req = { };
int ret;
fb_req.width = width;
fb_req.height = height;
fb_req.pixel_format = format;
fb_req.handles[0] = handle;
fb_req.pitches[0] = buffer->pitch;
ret = drm_mode_addfb2(client->dev, &fb_req, client->file);
if (ret)
return ret;
buffer->fb = drm_framebuffer_lookup(client->dev, buffer->client->file, fb_req.fb_id);
if (WARN_ON(!buffer->fb))
return -ENOENT;
/* drop the reference we picked up in framebuffer lookup */
drm_framebuffer_put(buffer->fb);
strscpy(buffer->fb->comm, client->name, TASK_COMM_LEN);
return 0;
}
/**
* drm_client_framebuffer_create - Create a client framebuffer
* @client: DRM client
* @width: Framebuffer width
* @height: Framebuffer height
* @format: Buffer format
*
* This function creates a &drm_client_buffer which consists of a
* &drm_framebuffer backed by a dumb buffer.
* Call drm_client_framebuffer_delete() to free the buffer.
*
* Returns:
* Pointer to a client buffer or an error pointer on failure.
*/
struct drm_client_buffer *
drm_client_framebuffer_create(struct drm_client_dev *client, u32 width, u32 height, u32 format)
{
struct drm_client_buffer *buffer;
u32 handle;
int ret;
buffer = drm_client_buffer_create(client, width, height, format,
&handle);
if (IS_ERR(buffer))
return buffer;
ret = drm_client_buffer_addfb(buffer, width, height, format, handle);
/*
* The handle is only needed for creating the framebuffer, destroy it
* again to solve a circular dependency should anybody export the GEM
* object as DMA-buf. The framebuffer and our buffer structure are still
* holding references to the GEM object to prevent its destruction.
*/
drm_mode_destroy_dumb(client->dev, handle, client->file);
if (ret) {
drm_client_buffer_delete(buffer);
return ERR_PTR(ret);
}
return buffer;
}
EXPORT_SYMBOL(drm_client_framebuffer_create);
/**
* drm_client_framebuffer_delete - Delete a client framebuffer
* @buffer: DRM client buffer (can be NULL)
*/
void drm_client_framebuffer_delete(struct drm_client_buffer *buffer)
{
if (!buffer)
return;
drm_client_buffer_rmfb(buffer);
drm_client_buffer_delete(buffer);
}
EXPORT_SYMBOL(drm_client_framebuffer_delete);
/**
* drm_client_framebuffer_flush - Manually flush client framebuffer
* @buffer: DRM client buffer (can be NULL)
* @rect: Damage rectangle (if NULL flushes all)
*
* This calls &drm_framebuffer_funcs->dirty (if present) to flush buffer changes
* for drivers that need it.
*
* Returns:
* Zero on success or negative error code on failure.
*/
int drm_client_framebuffer_flush(struct drm_client_buffer *buffer, struct drm_rect *rect)
{
if (!buffer || !buffer->fb || !buffer->fb->funcs->dirty)
return 0;
if (rect) {
struct drm_clip_rect clip = {
.x1 = rect->x1,
.y1 = rect->y1,
.x2 = rect->x2,
.y2 = rect->y2,
};
return buffer->fb->funcs->dirty(buffer->fb, buffer->client->file,
0, 0, &clip, 1);
}
return buffer->fb->funcs->dirty(buffer->fb, buffer->client->file,
0, 0, NULL, 0);
}
EXPORT_SYMBOL(drm_client_framebuffer_flush);
#ifdef CONFIG_DEBUG_FS
static int drm_client_debugfs_internal_clients(struct seq_file *m, void *data)
{
struct drm_debugfs_entry *entry = m->private;
struct drm_device *dev = entry->dev;
struct drm_printer p = drm_seq_file_printer(m);
struct drm_client_dev *client;
mutex_lock(&dev->clientlist_mutex);
list_for_each_entry(client, &dev->clientlist, list)
drm_printf(&p, "%s\n", client->name);
mutex_unlock(&dev->clientlist_mutex);
return 0;
}
static const struct drm_debugfs_info drm_client_debugfs_list[] = {
{ "internal_clients", drm_client_debugfs_internal_clients, 0 },
};
void drm_client_debugfs_init(struct drm_device *dev)
{
drm_debugfs_add_files(dev, drm_client_debugfs_list,
ARRAY_SIZE(drm_client_debugfs_list));
}
#endif
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Force feedback support for Linux input subsystem
*
* Copyright (c) 2006 Anssi Hannula <anssi.hannula@gmail.com>
* Copyright (c) 2006 Dmitry Torokhov <dtor@mail.ru>
*/
/* #define DEBUG */
#include <linux/input.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/sched.h>
#include <linux/slab.h>
/*
* Check that the effect_id is a valid effect and whether the user
* is the owner
*/
static int check_effect_access(struct ff_device *ff, int effect_id,
struct file *file)
{
if (effect_id < 0 || effect_id >= ff->max_effects || !ff->effect_owners[effect_id])
return -EINVAL;
if (file && ff->effect_owners[effect_id] != file)
return -EACCES;
return 0;
}
/*
* Checks whether 2 effects can be combined together
*/
static inline int check_effects_compatible(struct ff_effect *e1,
struct ff_effect *e2)
{
return e1->type == e2->type &&
(e1->type != FF_PERIODIC ||
e1->u.periodic.waveform == e2->u.periodic.waveform);
}
/*
* Convert an effect into compatible one
*/
static int compat_effect(struct ff_device *ff, struct ff_effect *effect)
{
int magnitude;
switch (effect->type) {
case FF_RUMBLE:
if (!test_bit(FF_PERIODIC, ff->ffbit))
return -EINVAL;
/*
* calculate magnitude of sine wave as average of rumble's
* 2/3 of strong magnitude and 1/3 of weak magnitude
*/
magnitude = effect->u.rumble.strong_magnitude / 3 +
effect->u.rumble.weak_magnitude / 6;
effect->type = FF_PERIODIC;
effect->u.periodic.waveform = FF_SINE;
effect->u.periodic.period = 50;
effect->u.periodic.magnitude = magnitude;
effect->u.periodic.offset = 0;
effect->u.periodic.phase = 0;
effect->u.periodic.envelope.attack_length = 0;
effect->u.periodic.envelope.attack_level = 0;
effect->u.periodic.envelope.fade_length = 0;
effect->u.periodic.envelope.fade_level = 0;
return 0;
default:
/* Let driver handle conversion */
return 0;
}
}
/**
* input_ff_upload() - upload effect into force-feedback device
* @dev: input device
* @effect: effect to be uploaded
* @file: owner of the effect
*/
int input_ff_upload(struct input_dev *dev, struct ff_effect *effect,
struct file *file)
{
struct ff_device *ff = dev->ff;
struct ff_effect *old;
int ret = 0;
int id;
if (!test_bit(EV_FF, dev->evbit))
return -ENOSYS;
if (effect->type < FF_EFFECT_MIN || effect->type > FF_EFFECT_MAX || !test_bit(effect->type, dev->ffbit)) { dev_dbg(&dev->dev, "invalid or not supported effect type in upload\n");
return -EINVAL;
}
if (effect->type == FF_PERIODIC && (effect->u.periodic.waveform < FF_WAVEFORM_MIN ||
effect->u.periodic.waveform > FF_WAVEFORM_MAX ||
!test_bit(effect->u.periodic.waveform, dev->ffbit))) { dev_dbg(&dev->dev, "invalid or not supported wave form in upload\n");
return -EINVAL;
}
if (!test_bit(effect->type, ff->ffbit)) { ret = compat_effect(ff, effect);
if (ret)
return ret;
}
mutex_lock(&ff->mutex);
if (effect->id == -1) {
for (id = 0; id < ff->max_effects; id++) if (!ff->effect_owners[id])
break;
if (id >= ff->max_effects) {
ret = -ENOSPC;
goto out;
}
effect->id = id;
old = NULL;
} else {
id = effect->id; ret = check_effect_access(ff, id, file);
if (ret)
goto out;
old = &ff->effects[id]; if (!check_effects_compatible(effect, old)) {
ret = -EINVAL;
goto out;
}
}
ret = ff->upload(dev, effect, old);
if (ret)
goto out;
spin_lock_irq(&dev->event_lock);
ff->effects[id] = *effect;
ff->effect_owners[id] = file;
spin_unlock_irq(&dev->event_lock);
out:
mutex_unlock(&ff->mutex); return ret;
}
EXPORT_SYMBOL_GPL(input_ff_upload);
/*
* Erases the effect if the requester is also the effect owner. The mutex
* should already be locked before calling this function.
*/
static int erase_effect(struct input_dev *dev, int effect_id,
struct file *file)
{
struct ff_device *ff = dev->ff;
int error;
error = check_effect_access(ff, effect_id, file);
if (error)
return error;
spin_lock_irq(&dev->event_lock);
ff->playback(dev, effect_id, 0);
ff->effect_owners[effect_id] = NULL;
spin_unlock_irq(&dev->event_lock);
if (ff->erase) {
error = ff->erase(dev, effect_id);
if (error) {
spin_lock_irq(&dev->event_lock);
ff->effect_owners[effect_id] = file;
spin_unlock_irq(&dev->event_lock);
return error;
}
}
return 0;
}
/**
* input_ff_erase - erase a force-feedback effect from device
* @dev: input device to erase effect from
* @effect_id: id of the effect to be erased
* @file: purported owner of the request
*
* This function erases a force-feedback effect from specified device.
* The effect will only be erased if it was uploaded through the same
* file handle that is requesting erase.
*/
int input_ff_erase(struct input_dev *dev, int effect_id, struct file *file)
{
struct ff_device *ff = dev->ff;
int ret;
if (!test_bit(EV_FF, dev->evbit))
return -ENOSYS;
mutex_lock(&ff->mutex);
ret = erase_effect(dev, effect_id, file);
mutex_unlock(&ff->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(input_ff_erase);
/*
* input_ff_flush - erase all effects owned by a file handle
* @dev: input device to erase effect from
* @file: purported owner of the effects
*
* This function erases all force-feedback effects associated with
* the given owner from specified device. Note that @file may be %NULL,
* in which case all effects will be erased.
*/
int input_ff_flush(struct input_dev *dev, struct file *file)
{
struct ff_device *ff = dev->ff;
int i;
dev_dbg(&dev->dev, "flushing now\n"); mutex_lock(&ff->mutex);
for (i = 0; i < ff->max_effects; i++)
erase_effect(dev, i, file); mutex_unlock(&ff->mutex);
return 0;
}
EXPORT_SYMBOL_GPL(input_ff_flush);
/**
* input_ff_event() - generic handler for force-feedback events
* @dev: input device to send the effect to
* @type: event type (anything but EV_FF is ignored)
* @code: event code
* @value: event value
*/
int input_ff_event(struct input_dev *dev, unsigned int type,
unsigned int code, int value)
{
struct ff_device *ff = dev->ff;
if (type != EV_FF)
return 0;
switch (code) {
case FF_GAIN:
if (!test_bit(FF_GAIN, dev->ffbit) || value > 0xffffU)
break;
ff->set_gain(dev, value);
break;
case FF_AUTOCENTER:
if (!test_bit(FF_AUTOCENTER, dev->ffbit) || value > 0xffffU)
break;
ff->set_autocenter(dev, value);
break;
default:
if (check_effect_access(ff, code, NULL) == 0)
ff->playback(dev, code, value);
break;
}
return 0;
}
EXPORT_SYMBOL_GPL(input_ff_event);
/**
* input_ff_create() - create force-feedback device
* @dev: input device supporting force-feedback
* @max_effects: maximum number of effects supported by the device
*
* This function allocates all necessary memory for a force feedback
* portion of an input device and installs all default handlers.
* @dev->ffbit should be already set up before calling this function.
* Once ff device is created you need to setup its upload, erase,
* playback and other handlers before registering input device
*/
int input_ff_create(struct input_dev *dev, unsigned int max_effects)
{
struct ff_device *ff;
size_t ff_dev_size;
int i;
if (!max_effects) {
dev_err(&dev->dev, "cannot allocate device without any effects\n");
return -EINVAL;
}
if (max_effects > FF_MAX_EFFECTS) {
dev_err(&dev->dev, "cannot allocate more than FF_MAX_EFFECTS effects\n");
return -EINVAL;
}
ff_dev_size = sizeof(struct ff_device) +
max_effects * sizeof(struct file *);
if (ff_dev_size < max_effects) /* overflow */
return -EINVAL;
ff = kzalloc(ff_dev_size, GFP_KERNEL);
if (!ff)
return -ENOMEM;
ff->effects = kcalloc(max_effects, sizeof(struct ff_effect),
GFP_KERNEL);
if (!ff->effects) {
kfree(ff);
return -ENOMEM;
}
ff->max_effects = max_effects;
mutex_init(&ff->mutex);
dev->ff = ff;
dev->flush = input_ff_flush;
dev->event = input_ff_event;
__set_bit(EV_FF, dev->evbit);
/* Copy "true" bits into ff device bitmap */
for_each_set_bit(i, dev->ffbit, FF_CNT)
__set_bit(i, ff->ffbit);
/* we can emulate RUMBLE with periodic effects */
if (test_bit(FF_PERIODIC, ff->ffbit))
__set_bit(FF_RUMBLE, dev->ffbit);
return 0;
}
EXPORT_SYMBOL_GPL(input_ff_create);
/**
* input_ff_destroy() - frees force feedback portion of input device
* @dev: input device supporting force feedback
*
* This function is only needed in error path as input core will
* automatically free force feedback structures when device is
* destroyed.
*/
void input_ff_destroy(struct input_dev *dev)
{
struct ff_device *ff = dev->ff;
__clear_bit(EV_FF, dev->evbit);
if (ff) {
if (ff->destroy) ff->destroy(ff); kfree(ff->private);
kfree(ff->effects);
kfree(ff);
dev->ff = NULL;
}
}
EXPORT_SYMBOL_GPL(input_ff_destroy);
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
#ifndef _UAPI_LINUX_IOPRIO_H
#define _UAPI_LINUX_IOPRIO_H
#include <linux/stddef.h>
#include <linux/types.h>
/*
* Gives us 8 prio classes with 13-bits of data for each class
*/
#define IOPRIO_CLASS_SHIFT 13
#define IOPRIO_NR_CLASSES 8
#define IOPRIO_CLASS_MASK (IOPRIO_NR_CLASSES - 1)
#define IOPRIO_PRIO_MASK ((1UL << IOPRIO_CLASS_SHIFT) - 1)
#define IOPRIO_PRIO_CLASS(ioprio) \
(((ioprio) >> IOPRIO_CLASS_SHIFT) & IOPRIO_CLASS_MASK)
#define IOPRIO_PRIO_DATA(ioprio) ((ioprio) & IOPRIO_PRIO_MASK)
/*
* These are the io priority classes as implemented by the BFQ and mq-deadline
* schedulers. RT is the realtime class, it always gets premium service. For
* ATA disks supporting NCQ IO priority, RT class IOs will be processed using
* high priority NCQ commands. BE is the best-effort scheduling class, the
* default for any process. IDLE is the idle scheduling class, it is only
* served when no one else is using the disk.
*/
enum {
IOPRIO_CLASS_NONE = 0,
IOPRIO_CLASS_RT = 1,
IOPRIO_CLASS_BE = 2,
IOPRIO_CLASS_IDLE = 3,
/* Special class to indicate an invalid ioprio value */
IOPRIO_CLASS_INVALID = 7,
};
/*
* The RT and BE priority classes both support up to 8 priority levels that
* can be specified using the lower 3-bits of the priority data.
*/
#define IOPRIO_LEVEL_NR_BITS 3
#define IOPRIO_NR_LEVELS (1 << IOPRIO_LEVEL_NR_BITS)
#define IOPRIO_LEVEL_MASK (IOPRIO_NR_LEVELS - 1)
#define IOPRIO_PRIO_LEVEL(ioprio) ((ioprio) & IOPRIO_LEVEL_MASK)
#define IOPRIO_BE_NR IOPRIO_NR_LEVELS
/*
* Possible values for the "which" argument of the ioprio_get() and
* ioprio_set() system calls (see "man ioprio_set").
*/
enum {
IOPRIO_WHO_PROCESS = 1,
IOPRIO_WHO_PGRP,
IOPRIO_WHO_USER,
};
/*
* Fallback BE class priority level.
*/
#define IOPRIO_NORM 4
#define IOPRIO_BE_NORM IOPRIO_NORM
/*
* The 10 bits between the priority class and the priority level are used to
* optionally define I/O hints for any combination of I/O priority class and
* level. Depending on the kernel configuration, I/O scheduler being used and
* the target I/O device being used, hints can influence how I/Os are processed
* without affecting the I/O scheduling ordering defined by the I/O priority
* class and level.
*/
#define IOPRIO_HINT_SHIFT IOPRIO_LEVEL_NR_BITS
#define IOPRIO_HINT_NR_BITS 10
#define IOPRIO_NR_HINTS (1 << IOPRIO_HINT_NR_BITS)
#define IOPRIO_HINT_MASK (IOPRIO_NR_HINTS - 1)
#define IOPRIO_PRIO_HINT(ioprio) \
(((ioprio) >> IOPRIO_HINT_SHIFT) & IOPRIO_HINT_MASK)
/*
* I/O hints.
*/
enum {
/* No hint */
IOPRIO_HINT_NONE = 0,
/*
* Device command duration limits: indicate to the device a desired
* duration limit for the commands that will be used to process an I/O.
* These will currently only be effective for SCSI and ATA devices that
* support the command duration limits feature. If this feature is
* enabled, then the commands issued to the device to process an I/O with
* one of these hints set will have the duration limit index (dld field)
* set to the value of the hint.
*/
IOPRIO_HINT_DEV_DURATION_LIMIT_1 = 1,
IOPRIO_HINT_DEV_DURATION_LIMIT_2 = 2,
IOPRIO_HINT_DEV_DURATION_LIMIT_3 = 3,
IOPRIO_HINT_DEV_DURATION_LIMIT_4 = 4,
IOPRIO_HINT_DEV_DURATION_LIMIT_5 = 5,
IOPRIO_HINT_DEV_DURATION_LIMIT_6 = 6,
IOPRIO_HINT_DEV_DURATION_LIMIT_7 = 7,
};
#define IOPRIO_BAD_VALUE(val, max) ((val) < 0 || (val) >= (max))
/*
* Return an I/O priority value based on a class, a level and a hint.
*/
static __always_inline __u16 ioprio_value(int prioclass, int priolevel,
int priohint)
{
if (IOPRIO_BAD_VALUE(prioclass, IOPRIO_NR_CLASSES) ||
IOPRIO_BAD_VALUE(priolevel, IOPRIO_NR_LEVELS) ||
IOPRIO_BAD_VALUE(priohint, IOPRIO_NR_HINTS))
return IOPRIO_CLASS_INVALID << IOPRIO_CLASS_SHIFT;
return (prioclass << IOPRIO_CLASS_SHIFT) |
(priohint << IOPRIO_HINT_SHIFT) | priolevel;
}
#define IOPRIO_PRIO_VALUE(prioclass, priolevel) \
ioprio_value(prioclass, priolevel, IOPRIO_HINT_NONE)
#define IOPRIO_PRIO_VALUE_HINT(prioclass, priolevel, priohint) \
ioprio_value(prioclass, priolevel, priohint)
#endif /* _UAPI_LINUX_IOPRIO_H */
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* async.h: Asynchronous function calls for boot performance
*
* (C) Copyright 2009 Intel Corporation
* Author: Arjan van de Ven <arjan@linux.intel.com>
*/
#ifndef __ASYNC_H__
#define __ASYNC_H__
#include <linux/types.h>
#include <linux/list.h>
#include <linux/numa.h>
#include <linux/device.h>
typedef u64 async_cookie_t;
typedef void (*async_func_t) (void *data, async_cookie_t cookie);
struct async_domain {
struct list_head pending;
unsigned registered:1;
};
/*
* domain participates in global async_synchronize_full
*/
#define ASYNC_DOMAIN(_name) \
struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \
.registered = 1 }
/*
* domain is free to go out of scope as soon as all pending work is
* complete, this domain does not participate in async_synchronize_full
*/
#define ASYNC_DOMAIN_EXCLUSIVE(_name) \
struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \
.registered = 0 }
async_cookie_t async_schedule_node(async_func_t func, void *data,
int node);
async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
int node,
struct async_domain *domain);
/**
* async_schedule - schedule a function for asynchronous execution
* @func: function to execute asynchronously
* @data: data pointer to pass to the function
*
* Returns an async_cookie_t that may be used for checkpointing later.
* Note: This function may be called from atomic or non-atomic contexts.
*/
static inline async_cookie_t async_schedule(async_func_t func, void *data)
{
return async_schedule_node(func, data, NUMA_NO_NODE);
}
/**
* async_schedule_domain - schedule a function for asynchronous execution within a certain domain
* @func: function to execute asynchronously
* @data: data pointer to pass to the function
* @domain: the domain
*
* Returns an async_cookie_t that may be used for checkpointing later.
* @domain may be used in the async_synchronize_*_domain() functions to
* wait within a certain synchronization domain rather than globally.
* Note: This function may be called from atomic or non-atomic contexts.
*/
static inline async_cookie_t
async_schedule_domain(async_func_t func, void *data,
struct async_domain *domain)
{
return async_schedule_node_domain(func, data, NUMA_NO_NODE, domain);
}
/**
* async_schedule_dev - A device specific version of async_schedule
* @func: function to execute asynchronously
* @dev: device argument to be passed to function
*
* Returns an async_cookie_t that may be used for checkpointing later.
* @dev is used as both the argument for the function and to provide NUMA
* context for where to run the function. By doing this we can try to
* provide for the best possible outcome by operating on the device on the
* CPUs closest to the device.
* Note: This function may be called from atomic or non-atomic contexts.
*/
static inline async_cookie_t
async_schedule_dev(async_func_t func, struct device *dev)
{
return async_schedule_node(func, dev, dev_to_node(dev));
}
bool async_schedule_dev_nocall(async_func_t func, struct device *dev);
/**
* async_schedule_dev_domain - A device specific version of async_schedule_domain
* @func: function to execute asynchronously
* @dev: device argument to be passed to function
* @domain: the domain
*
* Returns an async_cookie_t that may be used for checkpointing later.
* @dev is used as both the argument for the function and to provide NUMA
* context for where to run the function. By doing this we can try to
* provide for the best possible outcome by operating on the device on the
* CPUs closest to the device.
* @domain may be used in the async_synchronize_*_domain() functions to
* wait within a certain synchronization domain rather than globally.
* Note: This function may be called from atomic or non-atomic contexts.
*/
static inline async_cookie_t
async_schedule_dev_domain(async_func_t func, struct device *dev,
struct async_domain *domain)
{
return async_schedule_node_domain(func, dev, dev_to_node(dev), domain);
}
extern void async_synchronize_full(void);
extern void async_synchronize_full_domain(struct async_domain *domain);
extern void async_synchronize_cookie(async_cookie_t cookie);
extern void async_synchronize_cookie_domain(async_cookie_t cookie,
struct async_domain *domain);
extern bool current_is_async(void);
extern void async_init(void);
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* class.c - basic device class management
*
* Copyright (c) 2002-3 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
* Copyright (c) 2003-2004 Greg Kroah-Hartman
* Copyright (c) 2003-2004 IBM Corp.
*/
#include <linux/device/class.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/kdev_t.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/mutex.h>
#include "base.h"
/* /sys/class */
static struct kset *class_kset;
#define to_class_attr(_attr) container_of(_attr, struct class_attribute, attr)
/**
* class_to_subsys - Turn a struct class into a struct subsys_private
*
* @class: pointer to the struct bus_type to look up
*
* The driver core internals need to work on the subsys_private structure, not
* the external struct class pointer. This function walks the list of
* registered classes in the system and finds the matching one and returns the
* internal struct subsys_private that relates to that class.
*
* Note, the reference count of the return value is INCREMENTED if it is not
* NULL. A call to subsys_put() must be done when finished with the pointer in
* order for it to be properly freed.
*/
struct subsys_private *class_to_subsys(const struct class *class)
{
struct subsys_private *sp = NULL;
struct kobject *kobj;
if (!class || !class_kset)
return NULL;
spin_lock(&class_kset->list_lock);
if (list_empty(&class_kset->list))
goto done;
list_for_each_entry(kobj, &class_kset->list, entry) { struct kset *kset = container_of(kobj, struct kset, kobj);
sp = container_of_const(kset, struct subsys_private, subsys);
if (sp->class == class)
goto done;
}
sp = NULL;
done:
sp = subsys_get(sp); spin_unlock(&class_kset->list_lock); return sp;
}
static ssize_t class_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct class_attribute *class_attr = to_class_attr(attr);
struct subsys_private *cp = to_subsys_private(kobj);
ssize_t ret = -EIO;
if (class_attr->show)
ret = class_attr->show(cp->class, class_attr, buf);
return ret;
}
static ssize_t class_attr_store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t count)
{
struct class_attribute *class_attr = to_class_attr(attr);
struct subsys_private *cp = to_subsys_private(kobj);
ssize_t ret = -EIO;
if (class_attr->store)
ret = class_attr->store(cp->class, class_attr, buf, count);
return ret;
}
static void class_release(struct kobject *kobj)
{
struct subsys_private *cp = to_subsys_private(kobj);
const struct class *class = cp->class;
pr_debug("class '%s': release.\n", class->name);
if (class->class_release)
class->class_release(class);
else
pr_debug("class '%s' does not have a release() function, "
"be careful\n", class->name);
lockdep_unregister_key(&cp->lock_key);
kfree(cp);
}
static const struct kobj_ns_type_operations *class_child_ns_type(const struct kobject *kobj)
{
const struct subsys_private *cp = to_subsys_private(kobj);
const struct class *class = cp->class;
return class->ns_type;
}
static const struct sysfs_ops class_sysfs_ops = {
.show = class_attr_show,
.store = class_attr_store,
};
static const struct kobj_type class_ktype = {
.sysfs_ops = &class_sysfs_ops,
.release = class_release,
.child_ns_type = class_child_ns_type,
};
int class_create_file_ns(const struct class *cls, const struct class_attribute *attr,
const void *ns)
{
struct subsys_private *sp = class_to_subsys(cls);
int error;
if (!sp)
return -EINVAL;
error = sysfs_create_file_ns(&sp->subsys.kobj, &attr->attr, ns);
subsys_put(sp);
return error;
}
EXPORT_SYMBOL_GPL(class_create_file_ns);
void class_remove_file_ns(const struct class *cls, const struct class_attribute *attr,
const void *ns)
{
struct subsys_private *sp = class_to_subsys(cls);
if (!sp)
return;
sysfs_remove_file_ns(&sp->subsys.kobj, &attr->attr, ns);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(class_remove_file_ns);
static struct device *klist_class_to_dev(struct klist_node *n)
{
struct device_private *p = to_device_private_class(n);
return p->device;
}
static void klist_class_dev_get(struct klist_node *n)
{
struct device *dev = klist_class_to_dev(n);
get_device(dev);
}
static void klist_class_dev_put(struct klist_node *n)
{
struct device *dev = klist_class_to_dev(n);
put_device(dev);
}
int class_register(const struct class *cls)
{
struct subsys_private *cp;
struct lock_class_key *key;
int error;
pr_debug("device class '%s': registering\n", cls->name);
cp = kzalloc(sizeof(*cp), GFP_KERNEL);
if (!cp)
return -ENOMEM;
klist_init(&cp->klist_devices, klist_class_dev_get, klist_class_dev_put);
INIT_LIST_HEAD(&cp->interfaces);
kset_init(&cp->glue_dirs);
key = &cp->lock_key;
lockdep_register_key(key);
__mutex_init(&cp->mutex, "subsys mutex", key);
error = kobject_set_name(&cp->subsys.kobj, "%s", cls->name);
if (error)
goto err_out;
cp->subsys.kobj.kset = class_kset;
cp->subsys.kobj.ktype = &class_ktype;
cp->class = cls;
error = kset_register(&cp->subsys);
if (error)
goto err_out;
error = sysfs_create_groups(&cp->subsys.kobj, cls->class_groups);
if (error) {
kobject_del(&cp->subsys.kobj);
kfree_const(cp->subsys.kobj.name);
goto err_out;
}
return 0;
err_out:
lockdep_unregister_key(key);
kfree(cp);
return error;
}
EXPORT_SYMBOL_GPL(class_register);
void class_unregister(const struct class *cls)
{
struct subsys_private *sp = class_to_subsys(cls);
if (!sp)
return;
pr_debug("device class '%s': unregistering\n", cls->name);
sysfs_remove_groups(&sp->subsys.kobj, cls->class_groups);
kset_unregister(&sp->subsys);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(class_unregister);
static void class_create_release(const struct class *cls)
{
pr_debug("%s called for %s\n", __func__, cls->name);
kfree(cls);
}
/**
* class_create - create a struct class structure
* @name: pointer to a string for the name of this class.
*
* This is used to create a struct class pointer that can then be used
* in calls to device_create().
*
* Returns &struct class pointer on success, or ERR_PTR() on error.
*
* Note, the pointer created here is to be destroyed when finished by
* making a call to class_destroy().
*/
struct class *class_create(const char *name)
{
struct class *cls;
int retval;
cls = kzalloc(sizeof(*cls), GFP_KERNEL);
if (!cls) {
retval = -ENOMEM;
goto error;
}
cls->name = name;
cls->class_release = class_create_release;
retval = class_register(cls);
if (retval)
goto error;
return cls;
error:
kfree(cls);
return ERR_PTR(retval);
}
EXPORT_SYMBOL_GPL(class_create);
/**
* class_destroy - destroys a struct class structure
* @cls: pointer to the struct class that is to be destroyed
*
* Note, the pointer to be destroyed must have been created with a call
* to class_create().
*/
void class_destroy(const struct class *cls)
{
if (IS_ERR_OR_NULL(cls))
return;
class_unregister(cls);
}
EXPORT_SYMBOL_GPL(class_destroy);
/**
* class_dev_iter_init - initialize class device iterator
* @iter: class iterator to initialize
* @class: the class we wanna iterate over
* @start: the device to start iterating from, if any
* @type: device_type of the devices to iterate over, NULL for all
*
* Initialize class iterator @iter such that it iterates over devices
* of @class. If @start is set, the list iteration will start there,
* otherwise if it is NULL, the iteration starts at the beginning of
* the list.
*/
void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class,
const struct device *start, const struct device_type *type)
{
struct subsys_private *sp = class_to_subsys(class);
struct klist_node *start_knode = NULL;
if (!sp)
return;
if (start) start_knode = &start->p->knode_class; klist_iter_init_node(&sp->klist_devices, &iter->ki, start_knode);
iter->type = type;
iter->sp = sp;
}
EXPORT_SYMBOL_GPL(class_dev_iter_init);
/**
* class_dev_iter_next - iterate to the next device
* @iter: class iterator to proceed
*
* Proceed @iter to the next device and return it. Returns NULL if
* iteration is complete.
*
* The returned device is referenced and won't be released till
* iterator is proceed to the next device or exited. The caller is
* free to do whatever it wants to do with the device including
* calling back into class code.
*/
struct device *class_dev_iter_next(struct class_dev_iter *iter)
{
struct klist_node *knode;
struct device *dev;
while (1) {
knode = klist_next(&iter->ki);
if (!knode)
return NULL;
dev = klist_class_to_dev(knode); if (!iter->type || iter->type == dev->type)
return dev;
}
}
EXPORT_SYMBOL_GPL(class_dev_iter_next);
/**
* class_dev_iter_exit - finish iteration
* @iter: class iterator to finish
*
* Finish an iteration. Always call this function after iteration is
* complete whether the iteration ran till the end or not.
*/
void class_dev_iter_exit(struct class_dev_iter *iter)
{
klist_iter_exit(&iter->ki); subsys_put(iter->sp);
}
EXPORT_SYMBOL_GPL(class_dev_iter_exit);
/**
* class_for_each_device - device iterator
* @class: the class we're iterating
* @start: the device to start with in the list, if any.
* @data: data for the callback
* @fn: function to be called for each device
*
* Iterate over @class's list of devices, and call @fn for each,
* passing it @data. If @start is set, the list iteration will start
* there, otherwise if it is NULL, the iteration starts at the
* beginning of the list.
*
* We check the return of @fn each time. If it returns anything
* other than 0, we break out and return that value.
*
* @fn is allowed to do anything including calling back into class
* code. There's no locking restriction.
*/
int class_for_each_device(const struct class *class, const struct device *start,
void *data, int (*fn)(struct device *, void *))
{
struct subsys_private *sp = class_to_subsys(class);
struct class_dev_iter iter;
struct device *dev;
int error = 0;
if (!class)
return -EINVAL;
if (!sp) {
WARN(1, "%s called for class '%s' before it was initialized",
__func__, class->name);
return -EINVAL;
}
class_dev_iter_init(&iter, class, start, NULL);
while ((dev = class_dev_iter_next(&iter))) {
error = fn(dev, data);
if (error)
break;
}
class_dev_iter_exit(&iter);
subsys_put(sp);
return error;
}
EXPORT_SYMBOL_GPL(class_for_each_device);
/**
* class_find_device - device iterator for locating a particular device
* @class: the class we're iterating
* @start: Device to begin with
* @data: data for the match function
* @match: function to check device
*
* This is similar to the class_for_each_dev() function above, but it
* returns a reference to a device that is 'found' for later use, as
* determined by the @match callback.
*
* The callback should return 0 if the device doesn't match and non-zero
* if it does. If the callback returns non-zero, this function will
* return to the caller and not iterate over any more devices.
*
* Note, you will need to drop the reference with put_device() after use.
*
* @match is allowed to do anything including calling back into class
* code. There's no locking restriction.
*/
struct device *class_find_device(const struct class *class, const struct device *start,
const void *data,
int (*match)(struct device *, const void *))
{
struct subsys_private *sp = class_to_subsys(class);
struct class_dev_iter iter;
struct device *dev;
if (!class)
return NULL;
if (!sp) { WARN(1, "%s called for class '%s' before it was initialized",
__func__, class->name);
return NULL;
}
class_dev_iter_init(&iter, class, start, NULL); while ((dev = class_dev_iter_next(&iter))) { if (match(dev, data)) { get_device(dev);
break;
}
}
class_dev_iter_exit(&iter); subsys_put(sp); return dev;
}
EXPORT_SYMBOL_GPL(class_find_device);
int class_interface_register(struct class_interface *class_intf)
{
struct subsys_private *sp;
const struct class *parent;
struct class_dev_iter iter;
struct device *dev;
if (!class_intf || !class_intf->class)
return -ENODEV;
parent = class_intf->class;
sp = class_to_subsys(parent);
if (!sp)
return -EINVAL;
/*
* Reference in sp is now incremented and will be dropped when
* the interface is removed in the call to class_interface_unregister()
*/
mutex_lock(&sp->mutex);
list_add_tail(&class_intf->node, &sp->interfaces);
if (class_intf->add_dev) {
class_dev_iter_init(&iter, parent, NULL, NULL);
while ((dev = class_dev_iter_next(&iter)))
class_intf->add_dev(dev);
class_dev_iter_exit(&iter);
}
mutex_unlock(&sp->mutex);
return 0;
}
EXPORT_SYMBOL_GPL(class_interface_register);
void class_interface_unregister(struct class_interface *class_intf)
{
struct subsys_private *sp;
const struct class *parent = class_intf->class;
struct class_dev_iter iter;
struct device *dev;
if (!parent)
return;
sp = class_to_subsys(parent);
if (!sp)
return;
mutex_lock(&sp->mutex);
list_del_init(&class_intf->node);
if (class_intf->remove_dev) {
class_dev_iter_init(&iter, parent, NULL, NULL);
while ((dev = class_dev_iter_next(&iter)))
class_intf->remove_dev(dev);
class_dev_iter_exit(&iter);
}
mutex_unlock(&sp->mutex);
/*
* Decrement the reference count twice, once for the class_to_subsys()
* call in the start of this function, and the second one from the
* reference increment in class_interface_register()
*/
subsys_put(sp);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(class_interface_unregister);
ssize_t show_class_attr_string(const struct class *class,
const struct class_attribute *attr, char *buf)
{
struct class_attribute_string *cs;
cs = container_of(attr, struct class_attribute_string, attr);
return sysfs_emit(buf, "%s\n", cs->str);
}
EXPORT_SYMBOL_GPL(show_class_attr_string);
struct class_compat {
struct kobject *kobj;
};
/**
* class_compat_register - register a compatibility class
* @name: the name of the class
*
* Compatibility class are meant as a temporary user-space compatibility
* workaround when converting a family of class devices to a bus devices.
*/
struct class_compat *class_compat_register(const char *name)
{
struct class_compat *cls;
cls = kmalloc(sizeof(struct class_compat), GFP_KERNEL);
if (!cls)
return NULL;
cls->kobj = kobject_create_and_add(name, &class_kset->kobj);
if (!cls->kobj) {
kfree(cls);
return NULL;
}
return cls;
}
EXPORT_SYMBOL_GPL(class_compat_register);
/**
* class_compat_unregister - unregister a compatibility class
* @cls: the class to unregister
*/
void class_compat_unregister(struct class_compat *cls)
{
kobject_put(cls->kobj);
kfree(cls);
}
EXPORT_SYMBOL_GPL(class_compat_unregister);
/**
* class_compat_create_link - create a compatibility class device link to
* a bus device
* @cls: the compatibility class
* @dev: the target bus device
* @device_link: an optional device to which a "device" link should be created
*/
int class_compat_create_link(struct class_compat *cls, struct device *dev,
struct device *device_link)
{
int error;
error = sysfs_create_link(cls->kobj, &dev->kobj, dev_name(dev));
if (error)
return error;
/*
* Optionally add a "device" link (typically to the parent), as a
* class device would have one and we want to provide as much
* backwards compatibility as possible.
*/
if (device_link) {
error = sysfs_create_link(&dev->kobj, &device_link->kobj,
"device");
if (error)
sysfs_remove_link(cls->kobj, dev_name(dev));
}
return error;
}
EXPORT_SYMBOL_GPL(class_compat_create_link);
/**
* class_compat_remove_link - remove a compatibility class device link to
* a bus device
* @cls: the compatibility class
* @dev: the target bus device
* @device_link: an optional device to which a "device" link was previously
* created
*/
void class_compat_remove_link(struct class_compat *cls, struct device *dev,
struct device *device_link)
{
if (device_link) sysfs_remove_link(&dev->kobj, "device"); sysfs_remove_link(cls->kobj, dev_name(dev));
}
EXPORT_SYMBOL_GPL(class_compat_remove_link);
/**
* class_is_registered - determine if at this moment in time, a class is
* registered in the driver core or not.
* @class: the class to check
*
* Returns a boolean to state if the class is registered in the driver core
* or not. Note that the value could switch right after this call is made,
* so only use this in places where you "know" it is safe to do so (usually
* to determine if the specific class has been registered yet or not).
*
* Be careful in using this.
*/
bool class_is_registered(const struct class *class)
{
struct subsys_private *sp = class_to_subsys(class);
bool is_initialized = false;
if (sp) {
is_initialized = true;
subsys_put(sp);
}
return is_initialized;
}
EXPORT_SYMBOL_GPL(class_is_registered);
int __init classes_init(void)
{
class_kset = kset_create_and_add("class", NULL, NULL);
if (!class_kset)
return -ENOMEM;
return 0;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _MM_PERCPU_INTERNAL_H
#define _MM_PERCPU_INTERNAL_H
#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/memcontrol.h>
/*
* pcpu_block_md is the metadata block struct.
* Each chunk's bitmap is split into a number of full blocks.
* All units are in terms of bits.
*
* The scan hint is the largest known contiguous area before the contig hint.
* It is not necessarily the actual largest contig hint though. There is an
* invariant that the scan_hint_start > contig_hint_start iff
* scan_hint == contig_hint. This is necessary because when scanning forward,
* we don't know if a new contig hint would be better than the current one.
*/
struct pcpu_block_md {
int scan_hint; /* scan hint for block */
int scan_hint_start; /* block relative starting
position of the scan hint */
int contig_hint; /* contig hint for block */
int contig_hint_start; /* block relative starting
position of the contig hint */
int left_free; /* size of free space along
the left side of the block */
int right_free; /* size of free space along
the right side of the block */
int first_free; /* block position of first free */
int nr_bits; /* total bits responsible for */
};
struct pcpu_chunk {
#ifdef CONFIG_PERCPU_STATS
int nr_alloc; /* # of allocations */
size_t max_alloc_size; /* largest allocation size */
#endif
struct list_head list; /* linked to pcpu_slot lists */
int free_bytes; /* free bytes in the chunk */
struct pcpu_block_md chunk_md;
unsigned long *bound_map; /* boundary map */
/*
* base_addr is the base address of this chunk.
* To reduce false sharing, current layout is optimized to make sure
* base_addr locate in the different cacheline with free_bytes and
* chunk_md.
*/
void *base_addr ____cacheline_aligned_in_smp;
unsigned long *alloc_map; /* allocation map */
struct pcpu_block_md *md_blocks; /* metadata blocks */
void *data; /* chunk data */
bool immutable; /* no [de]population allowed */
bool isolated; /* isolated from active chunk
slots */
int start_offset; /* the overlap with the previous
region to have a page aligned
base_addr */
int end_offset; /* additional area required to
have the region end page
aligned */
#ifdef CONFIG_MEMCG_KMEM
struct obj_cgroup **obj_cgroups; /* vector of object cgroups */
#endif
int nr_pages; /* # of pages served by this chunk */
int nr_populated; /* # of populated pages */
int nr_empty_pop_pages; /* # of empty populated pages */
unsigned long populated[]; /* populated bitmap */
};
extern spinlock_t pcpu_lock;
extern struct list_head *pcpu_chunk_lists;
extern int pcpu_nr_slots;
extern int pcpu_sidelined_slot;
extern int pcpu_to_depopulate_slot;
extern int pcpu_nr_empty_pop_pages;
extern struct pcpu_chunk *pcpu_first_chunk;
extern struct pcpu_chunk *pcpu_reserved_chunk;
/**
* pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks
* @chunk: chunk of interest
*
* This conversion is from the number of physical pages that the chunk
* serves to the number of bitmap blocks used.
*/
static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
{
return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
}
/**
* pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap
* @pages: number of physical pages
*
* This conversion is from physical pages to the number of bits
* required in the bitmap.
*/
static inline int pcpu_nr_pages_to_map_bits(int pages)
{
return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
}
/**
* pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap
* @chunk: chunk of interest
*
* This conversion is from the number of physical pages that the chunk
* serves to the number of bits in the bitmap.
*/
static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
{
return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
}
/**
* pcpu_obj_full_size - helper to calculate size of each accounted object
* @size: size of area to allocate in bytes
*
* For each accounted object there is an extra space which is used to store
* obj_cgroup membership if kmemcg is not disabled. Charge it too.
*/
static inline size_t pcpu_obj_full_size(size_t size)
{
size_t extra_size = 0;
#ifdef CONFIG_MEMCG_KMEM
if (!mem_cgroup_kmem_disabled()) extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *);
#endif
return size * num_possible_cpus() + extra_size;
}
#ifdef CONFIG_PERCPU_STATS
#include <linux/spinlock.h>
struct percpu_stats {
u64 nr_alloc; /* lifetime # of allocations */
u64 nr_dealloc; /* lifetime # of deallocations */
u64 nr_cur_alloc; /* current # of allocations */
u64 nr_max_alloc; /* max # of live allocations */
u32 nr_chunks; /* current # of live chunks */
u32 nr_max_chunks; /* max # of live chunks */
size_t min_alloc_size; /* min allocation size */
size_t max_alloc_size; /* max allocation size */
};
extern struct percpu_stats pcpu_stats;
extern struct pcpu_alloc_info pcpu_stats_ai;
/*
* For debug purposes. We don't care about the flexible array.
*/
static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
{
memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
/* initialize min_alloc_size to unit_size */
pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
}
/*
* pcpu_stats_area_alloc - increment area allocation stats
* @chunk: the location of the area being allocated
* @size: size of area to allocate in bytes
*
* CONTEXT:
* pcpu_lock.
*/
static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
{
lockdep_assert_held(&pcpu_lock);
pcpu_stats.nr_alloc++;
pcpu_stats.nr_cur_alloc++;
pcpu_stats.nr_max_alloc =
max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
pcpu_stats.min_alloc_size =
min(pcpu_stats.min_alloc_size, size);
pcpu_stats.max_alloc_size =
max(pcpu_stats.max_alloc_size, size);
chunk->nr_alloc++;
chunk->max_alloc_size = max(chunk->max_alloc_size, size);
}
/*
* pcpu_stats_area_dealloc - decrement allocation stats
* @chunk: the location of the area being deallocated
*
* CONTEXT:
* pcpu_lock.
*/
static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
{
lockdep_assert_held(&pcpu_lock);
pcpu_stats.nr_dealloc++;
pcpu_stats.nr_cur_alloc--;
chunk->nr_alloc--;
}
/*
* pcpu_stats_chunk_alloc - increment chunk stats
*/
static inline void pcpu_stats_chunk_alloc(void)
{
unsigned long flags;
spin_lock_irqsave(&pcpu_lock, flags);
pcpu_stats.nr_chunks++;
pcpu_stats.nr_max_chunks =
max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
spin_unlock_irqrestore(&pcpu_lock, flags);
}
/*
* pcpu_stats_chunk_dealloc - decrement chunk stats
*/
static inline void pcpu_stats_chunk_dealloc(void)
{
unsigned long flags;
spin_lock_irqsave(&pcpu_lock, flags);
pcpu_stats.nr_chunks--;
spin_unlock_irqrestore(&pcpu_lock, flags);
}
#else
static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
{
}
static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
{
}
static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
{
}
static inline void pcpu_stats_chunk_alloc(void)
{
}
static inline void pcpu_stats_chunk_dealloc(void)
{
}
#endif /* !CONFIG_PERCPU_STATS */
#endif
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Device management routines
* Copyright (c) by Jaroslav Kysela <perex@perex.cz>
*/
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/export.h>
#include <linux/errno.h>
#include <sound/core.h>
/**
* snd_device_new - create an ALSA device component
* @card: the card instance
* @type: the device type, SNDRV_DEV_XXX
* @device_data: the data pointer of this device
* @ops: the operator table
*
* Creates a new device component for the given data pointer.
* The device will be assigned to the card and managed together
* by the card.
*
* The data pointer plays a role as the identifier, too, so the
* pointer address must be unique and unchanged.
*
* Return: Zero if successful, or a negative error code on failure.
*/
int snd_device_new(struct snd_card *card, enum snd_device_type type,
void *device_data, const struct snd_device_ops *ops)
{
struct snd_device *dev;
struct list_head *p;
if (snd_BUG_ON(!card || !device_data || !ops))
return -ENXIO;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return -ENOMEM;
INIT_LIST_HEAD(&dev->list);
dev->card = card;
dev->type = type;
dev->state = SNDRV_DEV_BUILD;
dev->device_data = device_data;
dev->ops = ops;
/* insert the entry in an incrementally sorted list */
list_for_each_prev(p, &card->devices) {
struct snd_device *pdev = list_entry(p, struct snd_device, list);
if ((unsigned int)pdev->type <= (unsigned int)type)
break;
}
list_add(&dev->list, p);
return 0;
}
EXPORT_SYMBOL(snd_device_new);
static void __snd_device_disconnect(struct snd_device *dev)
{
if (dev->state == SNDRV_DEV_REGISTERED) { if (dev->ops->dev_disconnect && dev->ops->dev_disconnect(dev)) dev_err(dev->card->dev, "device disconnect failure\n"); dev->state = SNDRV_DEV_DISCONNECTED;
}
}
static void __snd_device_free(struct snd_device *dev)
{
/* unlink */
list_del(&dev->list); __snd_device_disconnect(dev); if (dev->ops->dev_free) { if (dev->ops->dev_free(dev)) dev_err(dev->card->dev, "device free failure\n");
}
kfree(dev);
}
static struct snd_device *look_for_dev(struct snd_card *card, void *device_data)
{
struct snd_device *dev;
list_for_each_entry(dev, &card->devices, list)
if (dev->device_data == device_data)
return dev;
return NULL;
}
/**
* snd_device_disconnect - disconnect the device
* @card: the card instance
* @device_data: the data pointer to disconnect
*
* Turns the device into the disconnection state, invoking
* dev_disconnect callback, if the device was already registered.
*
* Usually called from snd_card_disconnect().
*
* Return: Zero if successful, or a negative error code on failure or if the
* device not found.
*/
void snd_device_disconnect(struct snd_card *card, void *device_data)
{
struct snd_device *dev;
if (snd_BUG_ON(!card || !device_data))
return;
dev = look_for_dev(card, device_data);
if (dev)
__snd_device_disconnect(dev);
else
dev_dbg(card->dev, "device disconnect %p (from %pS), not found\n",
device_data, __builtin_return_address(0));
}
EXPORT_SYMBOL_GPL(snd_device_disconnect);
/**
* snd_device_free - release the device from the card
* @card: the card instance
* @device_data: the data pointer to release
*
* Removes the device from the list on the card and invokes the
* callbacks, dev_disconnect and dev_free, corresponding to the state.
* Then release the device.
*/
void snd_device_free(struct snd_card *card, void *device_data)
{
struct snd_device *dev;
if (snd_BUG_ON(!card || !device_data))
return;
dev = look_for_dev(card, device_data);
if (dev)
__snd_device_free(dev);
else
dev_dbg(card->dev, "device free %p (from %pS), not found\n",
device_data, __builtin_return_address(0));
}
EXPORT_SYMBOL(snd_device_free);
static int __snd_device_register(struct snd_device *dev)
{
if (dev->state == SNDRV_DEV_BUILD) {
if (dev->ops->dev_register) {
int err = dev->ops->dev_register(dev);
if (err < 0)
return err;
}
dev->state = SNDRV_DEV_REGISTERED;
}
return 0;
}
/**
* snd_device_register - register the device
* @card: the card instance
* @device_data: the data pointer to register
*
* Registers the device which was already created via
* snd_device_new(). Usually this is called from snd_card_register(),
* but it can be called later if any new devices are created after
* invocation of snd_card_register().
*
* Return: Zero if successful, or a negative error code on failure or if the
* device not found.
*/
int snd_device_register(struct snd_card *card, void *device_data)
{
struct snd_device *dev;
if (snd_BUG_ON(!card || !device_data))
return -ENXIO;
dev = look_for_dev(card, device_data);
if (dev)
return __snd_device_register(dev);
snd_BUG();
return -ENXIO;
}
EXPORT_SYMBOL(snd_device_register);
/*
* register all the devices on the card.
* called from init.c
*/
int snd_device_register_all(struct snd_card *card)
{
struct snd_device *dev;
int err;
if (snd_BUG_ON(!card))
return -ENXIO;
list_for_each_entry(dev, &card->devices, list) {
err = __snd_device_register(dev);
if (err < 0)
return err;
}
return 0;
}
/*
* disconnect all the devices on the card.
* called from init.c
*/
void snd_device_disconnect_all(struct snd_card *card)
{
struct snd_device *dev;
if (snd_BUG_ON(!card))
return;
list_for_each_entry_reverse(dev, &card->devices, list) __snd_device_disconnect(dev);
}
/*
* release all the devices on the card.
* called from init.c
*/
void snd_device_free_all(struct snd_card *card)
{
struct snd_device *dev, *next;
if (snd_BUG_ON(!card))
return;
list_for_each_entry_safe_reverse(dev, next, &card->devices, list) {
/* exception: free ctl and lowlevel stuff later */
if (dev->type == SNDRV_DEV_CONTROL ||
dev->type == SNDRV_DEV_LOWLEVEL)
continue;
__snd_device_free(dev);
}
/* free all */
list_for_each_entry_safe_reverse(dev, next, &card->devices, list) __snd_device_free(dev);
}
/**
* snd_device_get_state - Get the current state of the given device
* @card: the card instance
* @device_data: the data pointer to release
*
* Returns the current state of the given device object. For the valid
* device, either @SNDRV_DEV_BUILD, @SNDRV_DEV_REGISTERED or
* @SNDRV_DEV_DISCONNECTED is returned.
* Or for a non-existing device, -1 is returned as an error.
*
* Return: the current state, or -1 if not found
*/
int snd_device_get_state(struct snd_card *card, void *device_data)
{
struct snd_device *dev;
dev = look_for_dev(card, device_data);
if (dev)
return dev->state;
return -1;
}
EXPORT_SYMBOL_GPL(snd_device_get_state);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* fs/eventpoll.c (Efficient event retrieval implementation)
* Copyright (C) 2001,...,2009 Davide Libenzi
*
* Davide Libenzi <davidel@xmailserver.org>
*/
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/signal.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/spinlock.h>
#include <linux/syscalls.h>
#include <linux/rbtree.h>
#include <linux/wait.h>
#include <linux/eventpoll.h>
#include <linux/mount.h>
#include <linux/bitops.h>
#include <linux/mutex.h>
#include <linux/anon_inodes.h>
#include <linux/device.h>
#include <linux/uaccess.h>
#include <asm/io.h>
#include <asm/mman.h>
#include <linux/atomic.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/compat.h>
#include <linux/rculist.h>
#include <linux/capability.h>
#include <net/busy_poll.h>
/*
* LOCKING:
* There are three level of locking required by epoll :
*
* 1) epnested_mutex (mutex)
* 2) ep->mtx (mutex)
* 3) ep->lock (rwlock)
*
* The acquire order is the one listed above, from 1 to 3.
* We need a rwlock (ep->lock) because we manipulate objects
* from inside the poll callback, that might be triggered from
* a wake_up() that in turn might be called from IRQ context.
* So we can't sleep inside the poll callback and hence we need
* a spinlock. During the event transfer loop (from kernel to
* user space) we could end up sleeping due a copy_to_user(), so
* we need a lock that will allow us to sleep. This lock is a
* mutex (ep->mtx). It is acquired during the event transfer loop,
* during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().
* The epnested_mutex is acquired when inserting an epoll fd onto another
* epoll fd. We do this so that we walk the epoll tree and ensure that this
* insertion does not create a cycle of epoll file descriptors, which
* could lead to deadlock. We need a global mutex to prevent two
* simultaneous inserts (A into B and B into A) from racing and
* constructing a cycle without either insert observing that it is
* going to.
* It is necessary to acquire multiple "ep->mtx"es at once in the
* case when one epoll fd is added to another. In this case, we
* always acquire the locks in the order of nesting (i.e. after
* epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
* before e2->mtx). Since we disallow cycles of epoll file
* descriptors, this ensures that the mutexes are well-ordered. In
* order to communicate this nesting to lockdep, when walking a tree
* of epoll file descriptors, we use the current recursion depth as
* the lockdep subkey.
* It is possible to drop the "ep->mtx" and to use the global
* mutex "epnested_mutex" (together with "ep->lock") to have it working,
* but having "ep->mtx" will make the interface more scalable.
* Events that require holding "epnested_mutex" are very rare, while for
* normal operations the epoll private "ep->mtx" will guarantee
* a better scalability.
*/
/* Epoll private bits inside the event mask */
#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)
#define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT)
#define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \
EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE)
/* Maximum number of nesting allowed inside epoll sets */
#define EP_MAX_NESTS 4
#define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
#define EP_UNACTIVE_PTR ((void *) -1L)
#define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))
struct epoll_filefd {
struct file *file;
int fd;
} __packed;
/* Wait structure used by the poll hooks */
struct eppoll_entry {
/* List header used to link this structure to the "struct epitem" */
struct eppoll_entry *next;
/* The "base" pointer is set to the container "struct epitem" */
struct epitem *base;
/*
* Wait queue item that will be linked to the target file wait
* queue head.
*/
wait_queue_entry_t wait;
/* The wait queue head that linked the "wait" wait queue item */
wait_queue_head_t *whead;
};
/*
* Each file descriptor added to the eventpoll interface will
* have an entry of this type linked to the "rbr" RB tree.
* Avoid increasing the size of this struct, there can be many thousands
* of these on a server and we do not want this to take another cache line.
*/
struct epitem {
union {
/* RB tree node links this structure to the eventpoll RB tree */
struct rb_node rbn;
/* Used to free the struct epitem */
struct rcu_head rcu;
};
/* List header used to link this structure to the eventpoll ready list */
struct list_head rdllink;
/*
* Works together "struct eventpoll"->ovflist in keeping the
* single linked chain of items.
*/
struct epitem *next;
/* The file descriptor information this item refers to */
struct epoll_filefd ffd;
/*
* Protected by file->f_lock, true for to-be-released epitem already
* removed from the "struct file" items list; together with
* eventpoll->refcount orchestrates "struct eventpoll" disposal
*/
bool dying;
/* List containing poll wait queues */
struct eppoll_entry *pwqlist;
/* The "container" of this item */
struct eventpoll *ep;
/* List header used to link this item to the "struct file" items list */
struct hlist_node fllink;
/* wakeup_source used when EPOLLWAKEUP is set */
struct wakeup_source __rcu *ws;
/* The structure that describe the interested events and the source fd */
struct epoll_event event;
};
/*
* This structure is stored inside the "private_data" member of the file
* structure and represents the main data structure for the eventpoll
* interface.
*/
struct eventpoll {
/*
* This mutex is used to ensure that files are not removed
* while epoll is using them. This is held during the event
* collection loop, the file cleanup path, the epoll file exit
* code and the ctl operations.
*/
struct mutex mtx;
/* Wait queue used by sys_epoll_wait() */
wait_queue_head_t wq;
/* Wait queue used by file->poll() */
wait_queue_head_t poll_wait;
/* List of ready file descriptors */
struct list_head rdllist;
/* Lock which protects rdllist and ovflist */
rwlock_t lock;
/* RB tree root used to store monitored fd structs */
struct rb_root_cached rbr;
/*
* This is a single linked list that chains all the "struct epitem" that
* happened while transferring ready events to userspace w/out
* holding ->lock.
*/
struct epitem *ovflist;
/* wakeup_source used when ep_send_events or __ep_eventpoll_poll is running */
struct wakeup_source *ws;
/* The user that created the eventpoll descriptor */
struct user_struct *user;
struct file *file;
/* used to optimize loop detection check */
u64 gen;
struct hlist_head refs;
/*
* usage count, used together with epitem->dying to
* orchestrate the disposal of this struct
*/
refcount_t refcount;
#ifdef CONFIG_NET_RX_BUSY_POLL
/* used to track busy poll napi_id */
unsigned int napi_id;
/* busy poll timeout */
u32 busy_poll_usecs;
/* busy poll packet budget */
u16 busy_poll_budget;
bool prefer_busy_poll;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
/* tracks wakeup nests for lockdep validation */
u8 nests;
#endif
};
/* Wrapper struct used by poll queueing */
struct ep_pqueue {
poll_table pt;
struct epitem *epi;
};
/*
* Configuration options available inside /proc/sys/fs/epoll/
*/
/* Maximum number of epoll watched descriptors, per user */
static long max_user_watches __read_mostly;
/* Used for cycles detection */
static DEFINE_MUTEX(epnested_mutex);
static u64 loop_check_gen = 0;
/* Used to check for epoll file descriptor inclusion loops */
static struct eventpoll *inserting_into;
/* Slab cache used to allocate "struct epitem" */
static struct kmem_cache *epi_cache __ro_after_init;
/* Slab cache used to allocate "struct eppoll_entry" */
static struct kmem_cache *pwq_cache __ro_after_init;
/*
* List of files with newly added links, where we may need to limit the number
* of emanating paths. Protected by the epnested_mutex.
*/
struct epitems_head {
struct hlist_head epitems;
struct epitems_head *next;
};
static struct epitems_head *tfile_check_list = EP_UNACTIVE_PTR;
static struct kmem_cache *ephead_cache __ro_after_init;
static inline void free_ephead(struct epitems_head *head)
{
if (head)
kmem_cache_free(ephead_cache, head);
}
static void list_file(struct file *file)
{
struct epitems_head *head;
head = container_of(file->f_ep, struct epitems_head, epitems);
if (!head->next) {
head->next = tfile_check_list;
tfile_check_list = head;
}
}
static void unlist_file(struct epitems_head *head)
{
struct epitems_head *to_free = head;
struct hlist_node *p = rcu_dereference(hlist_first_rcu(&head->epitems));
if (p) {
struct epitem *epi= container_of(p, struct epitem, fllink);
spin_lock(&epi->ffd.file->f_lock);
if (!hlist_empty(&head->epitems))
to_free = NULL;
head->next = NULL;
spin_unlock(&epi->ffd.file->f_lock);
}
free_ephead(to_free);
}
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
static long long_zero;
static long long_max = LONG_MAX;
static struct ctl_table epoll_table[] = {
{
.procname = "max_user_watches",
.data = &max_user_watches,
.maxlen = sizeof(max_user_watches),
.mode = 0644,
.proc_handler = proc_doulongvec_minmax,
.extra1 = &long_zero,
.extra2 = &long_max,
},
};
static void __init epoll_sysctls_init(void)
{
register_sysctl("fs/epoll", epoll_table);
}
#else
#define epoll_sysctls_init() do { } while (0)
#endif /* CONFIG_SYSCTL */
static const struct file_operations eventpoll_fops;
static inline int is_file_epoll(struct file *f)
{
return f->f_op == &eventpoll_fops;
}
/* Setup the structure that is used as key for the RB tree */
static inline void ep_set_ffd(struct epoll_filefd *ffd,
struct file *file, int fd)
{
ffd->file = file;
ffd->fd = fd;
}
/* Compare RB tree keys */
static inline int ep_cmp_ffd(struct epoll_filefd *p1,
struct epoll_filefd *p2)
{
return (p1->file > p2->file ? +1:
(p1->file < p2->file ? -1 : p1->fd - p2->fd));
}
/* Tells us if the item is currently linked */
static inline int ep_is_linked(struct epitem *epi)
{
return !list_empty(&epi->rdllink);
}
static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p)
{
return container_of(p, struct eppoll_entry, wait);
}
/* Get the "struct epitem" from a wait queue pointer */
static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)
{
return container_of(p, struct eppoll_entry, wait)->base;
}
/**
* ep_events_available - Checks if ready events might be available.
*
* @ep: Pointer to the eventpoll context.
*
* Return: a value different than %zero if ready events are available,
* or %zero otherwise.
*/
static inline int ep_events_available(struct eventpoll *ep)
{
return !list_empty_careful(&ep->rdllist) ||
READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
}
#ifdef CONFIG_NET_RX_BUSY_POLL
/**
* busy_loop_ep_timeout - check if busy poll has timed out. The timeout value
* from the epoll instance ep is preferred, but if it is not set fallback to
* the system-wide global via busy_loop_timeout.
*
* @start_time: The start time used to compute the remaining time until timeout.
* @ep: Pointer to the eventpoll context.
*
* Return: true if the timeout has expired, false otherwise.
*/
static bool busy_loop_ep_timeout(unsigned long start_time,
struct eventpoll *ep)
{
unsigned long bp_usec = READ_ONCE(ep->busy_poll_usecs);
if (bp_usec) {
unsigned long end_time = start_time + bp_usec;
unsigned long now = busy_loop_current_time();
return time_after(now, end_time);
} else {
return busy_loop_timeout(start_time);
}
}
static bool ep_busy_loop_on(struct eventpoll *ep)
{
return !!ep->busy_poll_usecs || net_busy_loop_on();
}
static bool ep_busy_loop_end(void *p, unsigned long start_time)
{
struct eventpoll *ep = p;
return ep_events_available(ep) || busy_loop_ep_timeout(start_time, ep);
}
/*
* Busy poll if globally on and supporting sockets found && no events,
* busy loop will return if need_resched or ep_events_available.
*
* we must do our busy polling with irqs enabled
*/
static bool ep_busy_loop(struct eventpoll *ep, int nonblock)
{
unsigned int napi_id = READ_ONCE(ep->napi_id);
u16 budget = READ_ONCE(ep->busy_poll_budget);
bool prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll);
if (!budget)
budget = BUSY_POLL_BUDGET;
if (napi_id >= MIN_NAPI_ID && ep_busy_loop_on(ep)) {
napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end,
ep, prefer_busy_poll, budget);
if (ep_events_available(ep))
return true;
/*
* Busy poll timed out. Drop NAPI ID for now, we can add
* it back in when we have moved a socket with a valid NAPI
* ID onto the ready list.
*/
ep->napi_id = 0;
return false;
}
return false;
}
/*
* Set epoll busy poll NAPI ID from sk.
*/
static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
{
struct eventpoll *ep = epi->ep;
unsigned int napi_id;
struct socket *sock;
struct sock *sk;
if (!ep_busy_loop_on(ep))
return;
sock = sock_from_file(epi->ffd.file);
if (!sock)
return;
sk = sock->sk;
if (!sk)
return;
napi_id = READ_ONCE(sk->sk_napi_id);
/* Non-NAPI IDs can be rejected
* or
* Nothing to do if we already have this ID
*/
if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)
return;
/* record NAPI ID for use in next busy poll */
ep->napi_id = napi_id;
}
static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
struct eventpoll *ep = file->private_data;
void __user *uarg = (void __user *)arg;
struct epoll_params epoll_params;
switch (cmd) {
case EPIOCSPARAMS:
if (copy_from_user(&epoll_params, uarg, sizeof(epoll_params)))
return -EFAULT;
/* pad byte must be zero */
if (epoll_params.__pad)
return -EINVAL;
if (epoll_params.busy_poll_usecs > S32_MAX)
return -EINVAL;
if (epoll_params.prefer_busy_poll > 1)
return -EINVAL;
if (epoll_params.busy_poll_budget > NAPI_POLL_WEIGHT &&
!capable(CAP_NET_ADMIN))
return -EPERM;
WRITE_ONCE(ep->busy_poll_usecs, epoll_params.busy_poll_usecs);
WRITE_ONCE(ep->busy_poll_budget, epoll_params.busy_poll_budget);
WRITE_ONCE(ep->prefer_busy_poll, epoll_params.prefer_busy_poll);
return 0;
case EPIOCGPARAMS:
memset(&epoll_params, 0, sizeof(epoll_params));
epoll_params.busy_poll_usecs = READ_ONCE(ep->busy_poll_usecs);
epoll_params.busy_poll_budget = READ_ONCE(ep->busy_poll_budget);
epoll_params.prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll);
if (copy_to_user(uarg, &epoll_params, sizeof(epoll_params)))
return -EFAULT;
return 0;
default:
return -ENOIOCTLCMD;
}
}
#else
static inline bool ep_busy_loop(struct eventpoll *ep, int nonblock)
{
return false;
}
static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
{
}
static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_NET_RX_BUSY_POLL */
/*
* As described in commit 0ccf831cb lockdep: annotate epoll
* the use of wait queues used by epoll is done in a very controlled
* manner. Wake ups can nest inside each other, but are never done
* with the same locking. For example:
*
* dfd = socket(...);
* efd1 = epoll_create();
* efd2 = epoll_create();
* epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...);
* epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...);
*
* When a packet arrives to the device underneath "dfd", the net code will
* issue a wake_up() on its poll wake list. Epoll (efd1) has installed a
* callback wakeup entry on that queue, and the wake_up() performed by the
* "dfd" net code will end up in ep_poll_callback(). At this point epoll
* (efd1) notices that it may have some event ready, so it needs to wake up
* the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
* that ends up in another wake_up(), after having checked about the
* recursion constraints. That are, no more than EP_MAX_NESTS, to avoid
* stack blasting.
*
* When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle
* this special case of epoll.
*/
#ifdef CONFIG_DEBUG_LOCK_ALLOC
static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,
unsigned pollflags)
{
struct eventpoll *ep_src;
unsigned long flags;
u8 nests = 0;
/*
* To set the subclass or nesting level for spin_lock_irqsave_nested()
* it might be natural to create a per-cpu nest count. However, since
* we can recurse on ep->poll_wait.lock, and a non-raw spinlock can
* schedule() in the -rt kernel, the per-cpu variable are no longer
* protected. Thus, we are introducing a per eventpoll nest field.
* If we are not being call from ep_poll_callback(), epi is NULL and
* we are at the first level of nesting, 0. Otherwise, we are being
* called from ep_poll_callback() and if a previous wakeup source is
* not an epoll file itself, we are at depth 1 since the wakeup source
* is depth 0. If the wakeup source is a previous epoll file in the
* wakeup chain then we use its nests value and record ours as
* nests + 1. The previous epoll file nests value is stable since its
* already holding its own poll_wait.lock.
*/
if (epi) {
if ((is_file_epoll(epi->ffd.file))) { ep_src = epi->ffd.file->private_data;
nests = ep_src->nests;
} else {
nests = 1;
}
}
spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests);
ep->nests = nests + 1;
wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags);
ep->nests = 0;
spin_unlock_irqrestore(&ep->poll_wait.lock, flags);
}
#else
static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,
__poll_t pollflags)
{
wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags);
}
#endif
static void ep_remove_wait_queue(struct eppoll_entry *pwq)
{
wait_queue_head_t *whead;
rcu_read_lock();
/*
* If it is cleared by POLLFREE, it should be rcu-safe.
* If we read NULL we need a barrier paired with
* smp_store_release() in ep_poll_callback(), otherwise
* we rely on whead->lock.
*/
whead = smp_load_acquire(&pwq->whead);
if (whead)
remove_wait_queue(whead, &pwq->wait);
rcu_read_unlock();
}
/*
* This function unregisters poll callbacks from the associated file
* descriptor. Must be called with "mtx" held.
*/
static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
{
struct eppoll_entry **p = &epi->pwqlist;
struct eppoll_entry *pwq;
while ((pwq = *p) != NULL) {
*p = pwq->next;
ep_remove_wait_queue(pwq);
kmem_cache_free(pwq_cache, pwq);
}
}
/* call only when ep->mtx is held */
static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)
{
return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));
}
/* call only when ep->mtx is held */
static inline void ep_pm_stay_awake(struct epitem *epi)
{
struct wakeup_source *ws = ep_wakeup_source(epi);
if (ws)
__pm_stay_awake(ws);
}
static inline bool ep_has_wakeup_source(struct epitem *epi)
{
return rcu_access_pointer(epi->ws) ? true : false;
}
/* call when ep->mtx cannot be held (ep_poll_callback) */
static inline void ep_pm_stay_awake_rcu(struct epitem *epi)
{
struct wakeup_source *ws;
rcu_read_lock(); ws = rcu_dereference(epi->ws); if (ws) __pm_stay_awake(ws); rcu_read_unlock();
}
/*
* ep->mutex needs to be held because we could be hit by
* eventpoll_release_file() and epoll_ctl().
*/
static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist)
{
/*
* Steal the ready list, and re-init the original one to the
* empty list. Also, set ep->ovflist to NULL so that events
* happening while looping w/out locks, are not lost. We cannot
* have the poll callback to queue directly on ep->rdllist,
* because we want the "sproc" callback to be able to do it
* in a lockless way.
*/
lockdep_assert_irqs_enabled();
write_lock_irq(&ep->lock);
list_splice_init(&ep->rdllist, txlist);
WRITE_ONCE(ep->ovflist, NULL);
write_unlock_irq(&ep->lock);
}
static void ep_done_scan(struct eventpoll *ep,
struct list_head *txlist)
{
struct epitem *epi, *nepi;
write_lock_irq(&ep->lock);
/*
* During the time we spent inside the "sproc" callback, some
* other events might have been queued by the poll callback.
* We re-insert them inside the main ready-list here.
*/
for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;
nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
/*
* We need to check if the item is already in the list.
* During the "sproc" callback execution time, items are
* queued into ->ovflist but the "txlist" might already
* contain them, and the list_splice() below takes care of them.
*/
if (!ep_is_linked(epi)) {
/*
* ->ovflist is LIFO, so we have to reverse it in order
* to keep in FIFO.
*/
list_add(&epi->rdllink, &ep->rdllist);
ep_pm_stay_awake(epi);
}
}
/*
* We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
* releasing the lock, events will be queued in the normal way inside
* ep->rdllist.
*/
WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
/*
* Quickly re-inject items left on "txlist".
*/
list_splice(txlist, &ep->rdllist);
__pm_relax(ep->ws);
if (!list_empty(&ep->rdllist)) {
if (waitqueue_active(&ep->wq))
wake_up(&ep->wq);
}
write_unlock_irq(&ep->lock);
}
static void ep_get(struct eventpoll *ep)
{
refcount_inc(&ep->refcount);
}
/*
* Returns true if the event poll can be disposed
*/
static bool ep_refcount_dec_and_test(struct eventpoll *ep)
{
if (!refcount_dec_and_test(&ep->refcount))
return false;
WARN_ON_ONCE(!RB_EMPTY_ROOT(&ep->rbr.rb_root));
return true;
}
static void ep_free(struct eventpoll *ep)
{
mutex_destroy(&ep->mtx);
free_uid(ep->user);
wakeup_source_unregister(ep->ws);
kfree(ep);
}
/*
* Removes a "struct epitem" from the eventpoll RB tree and deallocates
* all the associated resources. Must be called with "mtx" held.
* If the dying flag is set, do the removal only if force is true.
* This prevents ep_clear_and_put() from dropping all the ep references
* while running concurrently with eventpoll_release_file().
* Returns true if the eventpoll can be disposed.
*/
static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force)
{
struct file *file = epi->ffd.file;
struct epitems_head *to_free;
struct hlist_head *head;
lockdep_assert_irqs_enabled();
/*
* Removes poll wait queue hooks.
*/
ep_unregister_pollwait(ep, epi);
/* Remove the current item from the list of epoll hooks */
spin_lock(&file->f_lock);
if (epi->dying && !force) {
spin_unlock(&file->f_lock);
return false;
}
to_free = NULL;
head = file->f_ep;
if (head->first == &epi->fllink && !epi->fllink.next) {
file->f_ep = NULL;
if (!is_file_epoll(file)) {
struct epitems_head *v;
v = container_of(head, struct epitems_head, epitems);
if (!smp_load_acquire(&v->next))
to_free = v;
}
}
hlist_del_rcu(&epi->fllink);
spin_unlock(&file->f_lock);
free_ephead(to_free);
rb_erase_cached(&epi->rbn, &ep->rbr);
write_lock_irq(&ep->lock);
if (ep_is_linked(epi))
list_del_init(&epi->rdllink);
write_unlock_irq(&ep->lock);
wakeup_source_unregister(ep_wakeup_source(epi));
/*
* At this point it is safe to free the eventpoll item. Use the union
* field epi->rcu, since we are trying to minimize the size of
* 'struct epitem'. The 'rbn' field is no longer in use. Protected by
* ep->mtx. The rcu read side, reverse_path_check_proc(), does not make
* use of the rbn field.
*/
kfree_rcu(epi, rcu);
percpu_counter_dec(&ep->user->epoll_watches);
return ep_refcount_dec_and_test(ep);
}
/*
* ep_remove variant for callers owing an additional reference to the ep
*/
static void ep_remove_safe(struct eventpoll *ep, struct epitem *epi)
{
WARN_ON_ONCE(__ep_remove(ep, epi, false));
}
static void ep_clear_and_put(struct eventpoll *ep)
{
struct rb_node *rbp, *next;
struct epitem *epi;
bool dispose;
/* We need to release all tasks waiting for these file */
if (waitqueue_active(&ep->poll_wait))
ep_poll_safewake(ep, NULL, 0);
mutex_lock(&ep->mtx);
/*
* Walks through the whole tree by unregistering poll callbacks.
*/
for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
epi = rb_entry(rbp, struct epitem, rbn);
ep_unregister_pollwait(ep, epi);
cond_resched();
}
/*
* Walks through the whole tree and try to free each "struct epitem".
* Note that ep_remove_safe() will not remove the epitem in case of a
* racing eventpoll_release_file(); the latter will do the removal.
* At this point we are sure no poll callbacks will be lingering around.
* Since we still own a reference to the eventpoll struct, the loop can't
* dispose it.
*/
for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) {
next = rb_next(rbp);
epi = rb_entry(rbp, struct epitem, rbn);
ep_remove_safe(ep, epi);
cond_resched();
}
dispose = ep_refcount_dec_and_test(ep);
mutex_unlock(&ep->mtx);
if (dispose)
ep_free(ep);
}
static long ep_eventpoll_ioctl(struct file *file, unsigned int cmd,
unsigned long arg)
{
int ret;
if (!is_file_epoll(file))
return -EINVAL;
switch (cmd) {
case EPIOCSPARAMS:
case EPIOCGPARAMS:
ret = ep_eventpoll_bp_ioctl(file, cmd, arg);
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
static int ep_eventpoll_release(struct inode *inode, struct file *file)
{
struct eventpoll *ep = file->private_data;
if (ep)
ep_clear_and_put(ep);
return 0;
}
static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth);
static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth)
{
struct eventpoll *ep = file->private_data;
LIST_HEAD(txlist);
struct epitem *epi, *tmp;
poll_table pt;
__poll_t res = 0;
init_poll_funcptr(&pt, NULL);
/* Insert inside our poll wait queue */
poll_wait(file, &ep->poll_wait, wait);
/*
* Proceed to find out if wanted events are really available inside
* the ready list.
*/
mutex_lock_nested(&ep->mtx, depth);
ep_start_scan(ep, &txlist);
list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {
if (ep_item_poll(epi, &pt, depth + 1)) {
res = EPOLLIN | EPOLLRDNORM;
break;
} else {
/*
* Item has been dropped into the ready list by the poll
* callback, but it's not actually ready, as far as
* caller requested events goes. We can remove it here.
*/
__pm_relax(ep_wakeup_source(epi));
list_del_init(&epi->rdllink);
}
}
ep_done_scan(ep, &txlist);
mutex_unlock(&ep->mtx);
return res;
}
/*
* The ffd.file pointer may be in the process of being torn down due to
* being closed, but we may not have finished eventpoll_release() yet.
*
* Normally, even with the atomic_long_inc_not_zero, the file may have
* been free'd and then gotten re-allocated to something else (since
* files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU).
*
* But for epoll, users hold the ep->mtx mutex, and as such any file in
* the process of being free'd will block in eventpoll_release_file()
* and thus the underlying file allocation will not be free'd, and the
* file re-use cannot happen.
*
* For the same reason we can avoid a rcu_read_lock() around the
* operation - 'ffd.file' cannot go away even if the refcount has
* reached zero (but we must still not call out to ->poll() functions
* etc).
*/
static struct file *epi_fget(const struct epitem *epi)
{
struct file *file;
file = epi->ffd.file;
if (!atomic_long_inc_not_zero(&file->f_count))
file = NULL;
return file;
}
/*
* Differs from ep_eventpoll_poll() in that internal callers already have
* the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
* is correctly annotated.
*/
static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,
int depth)
{
struct file *file = epi_fget(epi);
__poll_t res;
/*
* We could return EPOLLERR | EPOLLHUP or something, but let's
* treat this more as "file doesn't exist, poll didn't happen".
*/
if (!file)
return 0;
pt->_key = epi->event.events;
if (!is_file_epoll(file))
res = vfs_poll(file, pt);
else
res = __ep_eventpoll_poll(file, pt, depth);
fput(file);
return res & epi->event.events;
}
static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)
{
return __ep_eventpoll_poll(file, wait, 0);
}
#ifdef CONFIG_PROC_FS
static void ep_show_fdinfo(struct seq_file *m, struct file *f)
{
struct eventpoll *ep = f->private_data;
struct rb_node *rbp;
mutex_lock(&ep->mtx);
for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
struct epitem *epi = rb_entry(rbp, struct epitem, rbn);
struct inode *inode = file_inode(epi->ffd.file);
seq_printf(m, "tfd: %8d events: %8x data: %16llx "
" pos:%lli ino:%lx sdev:%x\n",
epi->ffd.fd, epi->event.events,
(long long)epi->event.data,
(long long)epi->ffd.file->f_pos,
inode->i_ino, inode->i_sb->s_dev);
if (seq_has_overflowed(m))
break;
}
mutex_unlock(&ep->mtx);
}
#endif
/* File callbacks that implement the eventpoll file behaviour */
static const struct file_operations eventpoll_fops = {
#ifdef CONFIG_PROC_FS
.show_fdinfo = ep_show_fdinfo,
#endif
.release = ep_eventpoll_release,
.poll = ep_eventpoll_poll,
.llseek = noop_llseek,
.unlocked_ioctl = ep_eventpoll_ioctl,
.compat_ioctl = compat_ptr_ioctl,
};
/*
* This is called from eventpoll_release() to unlink files from the eventpoll
* interface. We need to have this facility to cleanup correctly files that are
* closed without being removed from the eventpoll interface.
*/
void eventpoll_release_file(struct file *file)
{
struct eventpoll *ep;
struct epitem *epi;
bool dispose;
/*
* Use the 'dying' flag to prevent a concurrent ep_clear_and_put() from
* touching the epitems list before eventpoll_release_file() can access
* the ep->mtx.
*/
again:
spin_lock(&file->f_lock);
if (file->f_ep && file->f_ep->first) {
epi = hlist_entry(file->f_ep->first, struct epitem, fllink);
epi->dying = true;
spin_unlock(&file->f_lock);
/*
* ep access is safe as we still own a reference to the ep
* struct
*/
ep = epi->ep;
mutex_lock(&ep->mtx);
dispose = __ep_remove(ep, epi, true);
mutex_unlock(&ep->mtx);
if (dispose)
ep_free(ep);
goto again;
}
spin_unlock(&file->f_lock);
}
static int ep_alloc(struct eventpoll **pep)
{
struct eventpoll *ep;
ep = kzalloc(sizeof(*ep), GFP_KERNEL);
if (unlikely(!ep))
return -ENOMEM;
mutex_init(&ep->mtx);
rwlock_init(&ep->lock);
init_waitqueue_head(&ep->wq);
init_waitqueue_head(&ep->poll_wait);
INIT_LIST_HEAD(&ep->rdllist);
ep->rbr = RB_ROOT_CACHED;
ep->ovflist = EP_UNACTIVE_PTR;
ep->user = get_current_user();
refcount_set(&ep->refcount, 1);
*pep = ep;
return 0;
}
/*
* Search the file inside the eventpoll tree. The RB tree operations
* are protected by the "mtx" mutex, and ep_find() must be called with
* "mtx" held.
*/
static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
{
int kcmp;
struct rb_node *rbp;
struct epitem *epi, *epir = NULL;
struct epoll_filefd ffd;
ep_set_ffd(&ffd, file, fd);
for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {
epi = rb_entry(rbp, struct epitem, rbn);
kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
if (kcmp > 0)
rbp = rbp->rb_right;
else if (kcmp < 0)
rbp = rbp->rb_left;
else {
epir = epi;
break;
}
}
return epir;
}
#ifdef CONFIG_KCMP
static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
{
struct rb_node *rbp;
struct epitem *epi;
for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
epi = rb_entry(rbp, struct epitem, rbn);
if (epi->ffd.fd == tfd) {
if (toff == 0)
return epi;
else
toff--;
}
cond_resched();
}
return NULL;
}
struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
unsigned long toff)
{
struct file *file_raw;
struct eventpoll *ep;
struct epitem *epi;
if (!is_file_epoll(file))
return ERR_PTR(-EINVAL);
ep = file->private_data;
mutex_lock(&ep->mtx);
epi = ep_find_tfd(ep, tfd, toff);
if (epi)
file_raw = epi->ffd.file;
else
file_raw = ERR_PTR(-ENOENT);
mutex_unlock(&ep->mtx);
return file_raw;
}
#endif /* CONFIG_KCMP */
/*
* Adds a new entry to the tail of the list in a lockless way, i.e.
* multiple CPUs are allowed to call this function concurrently.
*
* Beware: it is necessary to prevent any other modifications of the
* existing list until all changes are completed, in other words
* concurrent list_add_tail_lockless() calls should be protected
* with a read lock, where write lock acts as a barrier which
* makes sure all list_add_tail_lockless() calls are fully
* completed.
*
* Also an element can be locklessly added to the list only in one
* direction i.e. either to the tail or to the head, otherwise
* concurrent access will corrupt the list.
*
* Return: %false if element has been already added to the list, %true
* otherwise.
*/
static inline bool list_add_tail_lockless(struct list_head *new,
struct list_head *head)
{
struct list_head *prev;
/*
* This is simple 'new->next = head' operation, but cmpxchg()
* is used in order to detect that same element has been just
* added to the list from another CPU: the winner observes
* new->next == new.
*/
if (!try_cmpxchg(&new->next, &new, head))
return false;
/*
* Initially ->next of a new element must be updated with the head
* (we are inserting to the tail) and only then pointers are atomically
* exchanged. XCHG guarantees memory ordering, thus ->next should be
* updated before pointers are actually swapped and pointers are
* swapped before prev->next is updated.
*/
prev = xchg(&head->prev, new);
/*
* It is safe to modify prev->next and new->prev, because a new element
* is added only to the tail and new->next is updated before XCHG.
*/
prev->next = new;
new->prev = prev;
return true;
}
/*
* Chains a new epi entry to the tail of the ep->ovflist in a lockless way,
* i.e. multiple CPUs are allowed to call this function concurrently.
*
* Return: %false if epi element has been already chained, %true otherwise.
*/
static inline bool chain_epi_lockless(struct epitem *epi)
{
struct eventpoll *ep = epi->ep;
/* Fast preliminary check */
if (epi->next != EP_UNACTIVE_PTR)
return false;
/* Check that the same epi has not been just chained from another CPU */
if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)
return false;
/* Atomically exchange tail */
epi->next = xchg(&ep->ovflist, epi);
return true;
}
/*
* This is the callback that is passed to the wait queue wakeup
* mechanism. It is called by the stored file descriptors when they
* have events to report.
*
* This callback takes a read lock in order not to contend with concurrent
* events from another file descriptor, thus all modifications to ->rdllist
* or ->ovflist are lockless. Read lock is paired with the write lock from
* ep_start/done_scan(), which stops all list modifications and guarantees
* that lists state is seen correctly.
*
* Another thing worth to mention is that ep_poll_callback() can be called
* concurrently for the same @epi from different CPUs if poll table was inited
* with several wait queues entries. Plural wakeup from different CPUs of a
* single wait queue is serialized by wq.lock, but the case when multiple wait
* queues are used should be detected accordingly. This is detected using
* cmpxchg() operation.
*/
static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
{
int pwake = 0;
struct epitem *epi = ep_item_from_wait(wait);
struct eventpoll *ep = epi->ep;
__poll_t pollflags = key_to_poll(key);
unsigned long flags;
int ewake = 0;
read_lock_irqsave(&ep->lock, flags);
ep_set_busy_poll_napi_id(epi);
/*
* If the event mask does not contain any poll(2) event, we consider the
* descriptor to be disabled. This condition is likely the effect of the
* EPOLLONESHOT bit that disables the descriptor when an event is received,
* until the next EPOLL_CTL_MOD will be issued.
*/
if (!(epi->event.events & ~EP_PRIVATE_BITS))
goto out_unlock;
/*
* Check the events coming with the callback. At this stage, not
* every device reports the events in the "key" parameter of the
* callback. We need to be able to handle both cases here, hence the
* test for "key" != NULL before the event match test.
*/
if (pollflags && !(pollflags & epi->event.events))
goto out_unlock;
/*
* If we are transferring events to userspace, we can hold no locks
* (because we're accessing user memory, and because of linux f_op->poll()
* semantics). All the events that happen during that period of time are
* chained in ep->ovflist and requeued later on.
*/
if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { if (chain_epi_lockless(epi))
ep_pm_stay_awake_rcu(epi);
} else if (!ep_is_linked(epi)) {
/* In the usual case, add event to ready list. */
if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))
ep_pm_stay_awake_rcu(epi);
}
/*
* Wake up ( if active ) both the eventpoll wait list and the ->poll()
* wait list.
*/
if (waitqueue_active(&ep->wq)) { if ((epi->event.events & EPOLLEXCLUSIVE) &&
!(pollflags & POLLFREE)) {
switch (pollflags & EPOLLINOUT_BITS) {
case EPOLLIN:
if (epi->event.events & EPOLLIN)
ewake = 1;
break;
case EPOLLOUT:
if (epi->event.events & EPOLLOUT)
ewake = 1;
break;
case 0:
ewake = 1;
break;
}
}
wake_up(&ep->wq);
}
if (waitqueue_active(&ep->poll_wait))
pwake++;
out_unlock:
read_unlock_irqrestore(&ep->lock, flags);
/* We have to call this outside the lock */
if (pwake)
ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE); if (!(epi->event.events & EPOLLEXCLUSIVE))
ewake = 1;
if (pollflags & POLLFREE) {
/*
* If we race with ep_remove_wait_queue() it can miss
* ->whead = NULL and do another remove_wait_queue() after
* us, so we can't use __remove_wait_queue().
*/
list_del_init(&wait->entry);
/*
* ->whead != NULL protects us from the race with
* ep_clear_and_put() or ep_remove(), ep_remove_wait_queue()
* takes whead->lock held by the caller. Once we nullify it,
* nothing protects ep/epi or even wait.
*/
smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);
}
return ewake;
}
/*
* This is the callback that is used to add our wait queue to the
* target file wakeup lists.
*/
static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
poll_table *pt)
{
struct ep_pqueue *epq = container_of(pt, struct ep_pqueue, pt);
struct epitem *epi = epq->epi;
struct eppoll_entry *pwq;
if (unlikely(!epi)) // an earlier allocation has failed
return;
pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL);
if (unlikely(!pwq)) {
epq->epi = NULL;
return;
}
init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
pwq->whead = whead;
pwq->base = epi;
if (epi->event.events & EPOLLEXCLUSIVE)
add_wait_queue_exclusive(whead, &pwq->wait);
else
add_wait_queue(whead, &pwq->wait);
pwq->next = epi->pwqlist;
epi->pwqlist = pwq;
}
static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
{
int kcmp;
struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL;
struct epitem *epic;
bool leftmost = true;
while (*p) {
parent = *p;
epic = rb_entry(parent, struct epitem, rbn);
kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);
if (kcmp > 0) {
p = &parent->rb_right;
leftmost = false;
} else
p = &parent->rb_left;
}
rb_link_node(&epi->rbn, parent, p);
rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);
}
#define PATH_ARR_SIZE 5
/*
* These are the number paths of length 1 to 5, that we are allowing to emanate
* from a single file of interest. For example, we allow 1000 paths of length
* 1, to emanate from each file of interest. This essentially represents the
* potential wakeup paths, which need to be limited in order to avoid massive
* uncontrolled wakeup storms. The common use case should be a single ep which
* is connected to n file sources. In this case each file source has 1 path
* of length 1. Thus, the numbers below should be more than sufficient. These
* path limits are enforced during an EPOLL_CTL_ADD operation, since a modify
* and delete can't add additional paths. Protected by the epnested_mutex.
*/
static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 };
static int path_count[PATH_ARR_SIZE];
static int path_count_inc(int nests)
{
/* Allow an arbitrary number of depth 1 paths */
if (nests == 0)
return 0;
if (++path_count[nests] > path_limits[nests])
return -1;
return 0;
}
static void path_count_init(void)
{
int i;
for (i = 0; i < PATH_ARR_SIZE; i++)
path_count[i] = 0;
}
static int reverse_path_check_proc(struct hlist_head *refs, int depth)
{
int error = 0;
struct epitem *epi;
if (depth > EP_MAX_NESTS) /* too deep nesting */
return -1;
/* CTL_DEL can remove links here, but that can't increase our count */
hlist_for_each_entry_rcu(epi, refs, fllink) {
struct hlist_head *refs = &epi->ep->refs;
if (hlist_empty(refs))
error = path_count_inc(depth);
else
error = reverse_path_check_proc(refs, depth + 1);
if (error != 0)
break;
}
return error;
}
/**
* reverse_path_check - The tfile_check_list is list of epitem_head, which have
* links that are proposed to be newly added. We need to
* make sure that those added links don't add too many
* paths such that we will spend all our time waking up
* eventpoll objects.
*
* Return: %zero if the proposed links don't create too many paths,
* %-1 otherwise.
*/
static int reverse_path_check(void)
{
struct epitems_head *p;
for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) {
int error;
path_count_init();
rcu_read_lock();
error = reverse_path_check_proc(&p->epitems, 0);
rcu_read_unlock();
if (error)
return error;
}
return 0;
}
static int ep_create_wakeup_source(struct epitem *epi)
{
struct name_snapshot n;
struct wakeup_source *ws;
if (!epi->ep->ws) {
epi->ep->ws = wakeup_source_register(NULL, "eventpoll");
if (!epi->ep->ws)
return -ENOMEM;
}
take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);
ws = wakeup_source_register(NULL, n.name.name);
release_dentry_name_snapshot(&n);
if (!ws)
return -ENOMEM;
rcu_assign_pointer(epi->ws, ws);
return 0;
}
/* rare code path, only used when EPOLL_CTL_MOD removes a wakeup source */
static noinline void ep_destroy_wakeup_source(struct epitem *epi)
{
struct wakeup_source *ws = ep_wakeup_source(epi);
RCU_INIT_POINTER(epi->ws, NULL);
/*
* wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is
* used internally by wakeup_source_remove, too (called by
* wakeup_source_unregister), so we cannot use call_rcu
*/
synchronize_rcu();
wakeup_source_unregister(ws);
}
static int attach_epitem(struct file *file, struct epitem *epi)
{
struct epitems_head *to_free = NULL;
struct hlist_head *head = NULL;
struct eventpoll *ep = NULL;
if (is_file_epoll(file))
ep = file->private_data;
if (ep) {
head = &ep->refs;
} else if (!READ_ONCE(file->f_ep)) {
allocate:
to_free = kmem_cache_zalloc(ephead_cache, GFP_KERNEL);
if (!to_free)
return -ENOMEM;
head = &to_free->epitems;
}
spin_lock(&file->f_lock);
if (!file->f_ep) {
if (unlikely(!head)) {
spin_unlock(&file->f_lock);
goto allocate;
}
file->f_ep = head;
to_free = NULL;
}
hlist_add_head_rcu(&epi->fllink, file->f_ep);
spin_unlock(&file->f_lock);
free_ephead(to_free);
return 0;
}
/*
* Must be called with "mtx" held.
*/
static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
struct file *tfile, int fd, int full_check)
{
int error, pwake = 0;
__poll_t revents;
struct epitem *epi;
struct ep_pqueue epq;
struct eventpoll *tep = NULL;
if (is_file_epoll(tfile))
tep = tfile->private_data;
lockdep_assert_irqs_enabled();
if (unlikely(percpu_counter_compare(&ep->user->epoll_watches,
max_user_watches) >= 0))
return -ENOSPC;
percpu_counter_inc(&ep->user->epoll_watches);
if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) {
percpu_counter_dec(&ep->user->epoll_watches);
return -ENOMEM;
}
/* Item initialization follow here ... */
INIT_LIST_HEAD(&epi->rdllink);
epi->ep = ep;
ep_set_ffd(&epi->ffd, tfile, fd);
epi->event = *event;
epi->next = EP_UNACTIVE_PTR;
if (tep)
mutex_lock_nested(&tep->mtx, 1);
/* Add the current item to the list of active epoll hook for this file */
if (unlikely(attach_epitem(tfile, epi) < 0)) {
if (tep)
mutex_unlock(&tep->mtx);
kmem_cache_free(epi_cache, epi);
percpu_counter_dec(&ep->user->epoll_watches);
return -ENOMEM;
}
if (full_check && !tep)
list_file(tfile);
/*
* Add the current item to the RB tree. All RB tree operations are
* protected by "mtx", and ep_insert() is called with "mtx" held.
*/
ep_rbtree_insert(ep, epi);
if (tep)
mutex_unlock(&tep->mtx);
/*
* ep_remove_safe() calls in the later error paths can't lead to
* ep_free() as the ep file itself still holds an ep reference.
*/
ep_get(ep);
/* now check if we've created too many backpaths */
if (unlikely(full_check && reverse_path_check())) {
ep_remove_safe(ep, epi);
return -EINVAL;
}
if (epi->event.events & EPOLLWAKEUP) {
error = ep_create_wakeup_source(epi);
if (error) {
ep_remove_safe(ep, epi);
return error;
}
}
/* Initialize the poll table using the queue callback */
epq.epi = epi;
init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
/*
* Attach the item to the poll hooks and get current event bits.
* We can safely use the file* here because its usage count has
* been increased by the caller of this function. Note that after
* this operation completes, the poll callback can start hitting
* the new item.
*/
revents = ep_item_poll(epi, &epq.pt, 1);
/*
* We have to check if something went wrong during the poll wait queue
* install process. Namely an allocation for a wait queue failed due
* high memory pressure.
*/
if (unlikely(!epq.epi)) {
ep_remove_safe(ep, epi);
return -ENOMEM;
}
/* We have to drop the new item inside our item list to keep track of it */
write_lock_irq(&ep->lock);
/* record NAPI ID of new item if present */
ep_set_busy_poll_napi_id(epi);
/* If the file is already "ready" we drop it inside the ready list */
if (revents && !ep_is_linked(epi)) {
list_add_tail(&epi->rdllink, &ep->rdllist);
ep_pm_stay_awake(epi);
/* Notify waiting tasks that events are available */
if (waitqueue_active(&ep->wq))
wake_up(&ep->wq);
if (waitqueue_active(&ep->poll_wait))
pwake++;
}
write_unlock_irq(&ep->lock);
/* We have to call this outside the lock */
if (pwake)
ep_poll_safewake(ep, NULL, 0);
return 0;
}
/*
* Modify the interest event mask by dropping an event if the new mask
* has a match in the current file status. Must be called with "mtx" held.
*/
static int ep_modify(struct eventpoll *ep, struct epitem *epi,
const struct epoll_event *event)
{
int pwake = 0;
poll_table pt;
lockdep_assert_irqs_enabled();
init_poll_funcptr(&pt, NULL);
/*
* Set the new event interest mask before calling f_op->poll();
* otherwise we might miss an event that happens between the
* f_op->poll() call and the new event set registering.
*/
epi->event.events = event->events; /* need barrier below */
epi->event.data = event->data; /* protected by mtx */
if (epi->event.events & EPOLLWAKEUP) {
if (!ep_has_wakeup_source(epi))
ep_create_wakeup_source(epi);
} else if (ep_has_wakeup_source(epi)) {
ep_destroy_wakeup_source(epi);
}
/*
* The following barrier has two effects:
*
* 1) Flush epi changes above to other CPUs. This ensures
* we do not miss events from ep_poll_callback if an
* event occurs immediately after we call f_op->poll().
* We need this because we did not take ep->lock while
* changing epi above (but ep_poll_callback does take
* ep->lock).
*
* 2) We also need to ensure we do not miss _past_ events
* when calling f_op->poll(). This barrier also
* pairs with the barrier in wq_has_sleeper (see
* comments for wq_has_sleeper).
*
* This barrier will now guarantee ep_poll_callback or f_op->poll
* (or both) will notice the readiness of an item.
*/
smp_mb();
/*
* Get current event bits. We can safely use the file* here because
* its usage count has been increased by the caller of this function.
* If the item is "hot" and it is not registered inside the ready
* list, push it inside.
*/
if (ep_item_poll(epi, &pt, 1)) {
write_lock_irq(&ep->lock);
if (!ep_is_linked(epi)) {
list_add_tail(&epi->rdllink, &ep->rdllist);
ep_pm_stay_awake(epi);
/* Notify waiting tasks that events are available */
if (waitqueue_active(&ep->wq))
wake_up(&ep->wq);
if (waitqueue_active(&ep->poll_wait))
pwake++;
}
write_unlock_irq(&ep->lock);
}
/* We have to call this outside the lock */
if (pwake)
ep_poll_safewake(ep, NULL, 0);
return 0;
}
static int ep_send_events(struct eventpoll *ep,
struct epoll_event __user *events, int maxevents)
{
struct epitem *epi, *tmp;
LIST_HEAD(txlist);
poll_table pt;
int res = 0;
/*
* Always short-circuit for fatal signals to allow threads to make a
* timely exit without the chance of finding more events available and
* fetching repeatedly.
*/
if (fatal_signal_pending(current))
return -EINTR;
init_poll_funcptr(&pt, NULL);
mutex_lock(&ep->mtx);
ep_start_scan(ep, &txlist);
/*
* We can loop without lock because we are passed a task private list.
* Items cannot vanish during the loop we are holding ep->mtx.
*/
list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {
struct wakeup_source *ws;
__poll_t revents;
if (res >= maxevents)
break;
/*
* Activate ep->ws before deactivating epi->ws to prevent
* triggering auto-suspend here (in case we reactive epi->ws
* below).
*
* This could be rearranged to delay the deactivation of epi->ws
* instead, but then epi->ws would temporarily be out of sync
* with ep_is_linked().
*/
ws = ep_wakeup_source(epi);
if (ws) {
if (ws->active)
__pm_stay_awake(ep->ws);
__pm_relax(ws);
}
list_del_init(&epi->rdllink);
/*
* If the event mask intersect the caller-requested one,
* deliver the event to userspace. Again, we are holding ep->mtx,
* so no operations coming from userspace can change the item.
*/
revents = ep_item_poll(epi, &pt, 1);
if (!revents)
continue;
events = epoll_put_uevent(revents, epi->event.data, events);
if (!events) {
list_add(&epi->rdllink, &txlist);
ep_pm_stay_awake(epi);
if (!res)
res = -EFAULT;
break;
}
res++;
if (epi->event.events & EPOLLONESHOT)
epi->event.events &= EP_PRIVATE_BITS;
else if (!(epi->event.events & EPOLLET)) {
/*
* If this file has been added with Level
* Trigger mode, we need to insert back inside
* the ready list, so that the next call to
* epoll_wait() will check again the events
* availability. At this point, no one can insert
* into ep->rdllist besides us. The epoll_ctl()
* callers are locked out by
* ep_send_events() holding "mtx" and the
* poll callback will queue them in ep->ovflist.
*/
list_add_tail(&epi->rdllink, &ep->rdllist);
ep_pm_stay_awake(epi);
}
}
ep_done_scan(ep, &txlist);
mutex_unlock(&ep->mtx);
return res;
}
static struct timespec64 *ep_timeout_to_timespec(struct timespec64 *to, long ms)
{
struct timespec64 now;
if (ms < 0)
return NULL;
if (!ms) {
to->tv_sec = 0;
to->tv_nsec = 0;
return to;
}
to->tv_sec = ms / MSEC_PER_SEC;
to->tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC);
ktime_get_ts64(&now);
*to = timespec64_add_safe(now, *to);
return to;
}
/*
* autoremove_wake_function, but remove even on failure to wake up, because we
* know that default_wake_function/ttwu will only fail if the thread is already
* woken, and in that case the ep_poll loop will remove the entry anyways, not
* try to reuse it.
*/
static int ep_autoremove_wake_function(struct wait_queue_entry *wq_entry,
unsigned int mode, int sync, void *key)
{
int ret = default_wake_function(wq_entry, mode, sync, key);
/*
* Pairs with list_empty_careful in ep_poll, and ensures future loop
* iterations see the cause of this wakeup.
*/
list_del_init_careful(&wq_entry->entry);
return ret;
}
/**
* ep_poll - Retrieves ready events, and delivers them to the caller-supplied
* event buffer.
*
* @ep: Pointer to the eventpoll context.
* @events: Pointer to the userspace buffer where the ready events should be
* stored.
* @maxevents: Size (in terms of number of events) of the caller event buffer.
* @timeout: Maximum timeout for the ready events fetch operation, in
* timespec. If the timeout is zero, the function will not block,
* while if the @timeout ptr is NULL, the function will block
* until at least one event has been retrieved (or an error
* occurred).
*
* Return: the number of ready events which have been fetched, or an
* error code, in case of error.
*/
static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
int maxevents, struct timespec64 *timeout)
{
int res, eavail, timed_out = 0;
u64 slack = 0;
wait_queue_entry_t wait;
ktime_t expires, *to = NULL;
lockdep_assert_irqs_enabled();
if (timeout && (timeout->tv_sec | timeout->tv_nsec)) {
slack = select_estimate_accuracy(timeout);
to = &expires;
*to = timespec64_to_ktime(*timeout);
} else if (timeout) {
/*
* Avoid the unnecessary trip to the wait queue loop, if the
* caller specified a non blocking operation.
*/
timed_out = 1;
}
/*
* This call is racy: We may or may not see events that are being added
* to the ready list under the lock (e.g., in IRQ callbacks). For cases
* with a non-zero timeout, this thread will check the ready list under
* lock and will add to the wait queue. For cases with a zero
* timeout, the user by definition should not care and will have to
* recheck again.
*/
eavail = ep_events_available(ep);
while (1) {
if (eavail) {
/*
* Try to transfer events to user space. In case we get
* 0 events and there's still timeout left over, we go
* trying again in search of more luck.
*/
res = ep_send_events(ep, events, maxevents);
if (res)
return res;
}
if (timed_out)
return 0;
eavail = ep_busy_loop(ep, timed_out);
if (eavail)
continue;
if (signal_pending(current))
return -EINTR;
/*
* Internally init_wait() uses autoremove_wake_function(),
* thus wait entry is removed from the wait queue on each
* wakeup. Why it is important? In case of several waiters
* each new wakeup will hit the next waiter, giving it the
* chance to harvest new event. Otherwise wakeup can be
* lost. This is also good performance-wise, because on
* normal wakeup path no need to call __remove_wait_queue()
* explicitly, thus ep->lock is not taken, which halts the
* event delivery.
*
* In fact, we now use an even more aggressive function that
* unconditionally removes, because we don't reuse the wait
* entry between loop iterations. This lets us also avoid the
* performance issue if a process is killed, causing all of its
* threads to wake up without being removed normally.
*/
init_wait(&wait);
wait.func = ep_autoremove_wake_function;
write_lock_irq(&ep->lock);
/*
* Barrierless variant, waitqueue_active() is called under
* the same lock on wakeup ep_poll_callback() side, so it
* is safe to avoid an explicit barrier.
*/
__set_current_state(TASK_INTERRUPTIBLE);
/*
* Do the final check under the lock. ep_start/done_scan()
* plays with two lists (->rdllist and ->ovflist) and there
* is always a race when both lists are empty for short
* period of time although events are pending, so lock is
* important.
*/
eavail = ep_events_available(ep);
if (!eavail)
__add_wait_queue_exclusive(&ep->wq, &wait);
write_unlock_irq(&ep->lock);
if (!eavail)
timed_out = !schedule_hrtimeout_range(to, slack,
HRTIMER_MODE_ABS);
__set_current_state(TASK_RUNNING);
/*
* We were woken up, thus go and try to harvest some events.
* If timed out and still on the wait queue, recheck eavail
* carefully under lock, below.
*/
eavail = 1;
if (!list_empty_careful(&wait.entry)) {
write_lock_irq(&ep->lock);
/*
* If the thread timed out and is not on the wait queue,
* it means that the thread was woken up after its
* timeout expired before it could reacquire the lock.
* Thus, when wait.entry is empty, it needs to harvest
* events.
*/
if (timed_out)
eavail = list_empty(&wait.entry);
__remove_wait_queue(&ep->wq, &wait);
write_unlock_irq(&ep->lock);
}
}
}
/**
* ep_loop_check_proc - verify that adding an epoll file inside another
* epoll structure does not violate the constraints, in
* terms of closed loops, or too deep chains (which can
* result in excessive stack usage).
*
* @ep: the &struct eventpoll to be currently checked.
* @depth: Current depth of the path being checked.
*
* Return: %zero if adding the epoll @file inside current epoll
* structure @ep does not violate the constraints, or %-1 otherwise.
*/
static int ep_loop_check_proc(struct eventpoll *ep, int depth)
{
int error = 0;
struct rb_node *rbp;
struct epitem *epi;
mutex_lock_nested(&ep->mtx, depth + 1);
ep->gen = loop_check_gen;
for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
epi = rb_entry(rbp, struct epitem, rbn);
if (unlikely(is_file_epoll(epi->ffd.file))) {
struct eventpoll *ep_tovisit;
ep_tovisit = epi->ffd.file->private_data;
if (ep_tovisit->gen == loop_check_gen)
continue;
if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS)
error = -1;
else
error = ep_loop_check_proc(ep_tovisit, depth + 1);
if (error != 0)
break;
} else {
/*
* If we've reached a file that is not associated with
* an ep, then we need to check if the newly added
* links are going to add too many wakeup paths. We do
* this by adding it to the tfile_check_list, if it's
* not already there, and calling reverse_path_check()
* during ep_insert().
*/
list_file(epi->ffd.file);
}
}
mutex_unlock(&ep->mtx);
return error;
}
/**
* ep_loop_check - Performs a check to verify that adding an epoll file (@to)
* into another epoll file (represented by @ep) does not create
* closed loops or too deep chains.
*
* @ep: Pointer to the epoll we are inserting into.
* @to: Pointer to the epoll to be inserted.
*
* Return: %zero if adding the epoll @to inside the epoll @from
* does not violate the constraints, or %-1 otherwise.
*/
static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to)
{
inserting_into = ep;
return ep_loop_check_proc(to, 0);
}
static void clear_tfile_check_list(void)
{
rcu_read_lock();
while (tfile_check_list != EP_UNACTIVE_PTR) {
struct epitems_head *head = tfile_check_list;
tfile_check_list = head->next;
unlist_file(head);
}
rcu_read_unlock();
}
/*
* Open an eventpoll file descriptor.
*/
static int do_epoll_create(int flags)
{
int error, fd;
struct eventpoll *ep = NULL;
struct file *file;
/* Check the EPOLL_* constant for consistency. */
BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);
if (flags & ~EPOLL_CLOEXEC)
return -EINVAL;
/*
* Create the internal data structure ("struct eventpoll").
*/
error = ep_alloc(&ep);
if (error < 0)
return error;
/*
* Creates all the items needed to setup an eventpoll file. That is,
* a file structure and a free file descriptor.
*/
fd = get_unused_fd_flags(O_RDWR | (flags & O_CLOEXEC));
if (fd < 0) {
error = fd;
goto out_free_ep;
}
file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
O_RDWR | (flags & O_CLOEXEC));
if (IS_ERR(file)) {
error = PTR_ERR(file);
goto out_free_fd;
}
#ifdef CONFIG_NET_RX_BUSY_POLL
ep->busy_poll_usecs = 0;
ep->busy_poll_budget = 0;
ep->prefer_busy_poll = false;
#endif
ep->file = file;
fd_install(fd, file);
return fd;
out_free_fd:
put_unused_fd(fd);
out_free_ep:
ep_clear_and_put(ep);
return error;
}
SYSCALL_DEFINE1(epoll_create1, int, flags)
{
return do_epoll_create(flags);
}
SYSCALL_DEFINE1(epoll_create, int, size)
{
if (size <= 0)
return -EINVAL;
return do_epoll_create(0);
}
#ifdef CONFIG_PM_SLEEP
static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev)
{
if ((epev->events & EPOLLWAKEUP) && !capable(CAP_BLOCK_SUSPEND))
epev->events &= ~EPOLLWAKEUP;
}
#else
static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev)
{
epev->events &= ~EPOLLWAKEUP;
}
#endif
static inline int epoll_mutex_lock(struct mutex *mutex, int depth,
bool nonblock)
{
if (!nonblock) {
mutex_lock_nested(mutex, depth);
return 0;
}
if (mutex_trylock(mutex))
return 0;
return -EAGAIN;
}
int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
bool nonblock)
{
int error;
int full_check = 0;
struct fd f, tf;
struct eventpoll *ep;
struct epitem *epi;
struct eventpoll *tep = NULL;
error = -EBADF;
f = fdget(epfd);
if (!f.file)
goto error_return;
/* Get the "struct file *" for the target file */
tf = fdget(fd);
if (!tf.file)
goto error_fput;
/* The target file descriptor must support poll */
error = -EPERM;
if (!file_can_poll(tf.file))
goto error_tgt_fput;
/* Check if EPOLLWAKEUP is allowed */
if (ep_op_has_event(op))
ep_take_care_of_epollwakeup(epds);
/*
* We have to check that the file structure underneath the file descriptor
* the user passed to us _is_ an eventpoll file. And also we do not permit
* adding an epoll file descriptor inside itself.
*/
error = -EINVAL;
if (f.file == tf.file || !is_file_epoll(f.file))
goto error_tgt_fput;
/*
* epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,
* so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.
* Also, we do not currently supported nested exclusive wakeups.
*/
if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) {
if (op == EPOLL_CTL_MOD)
goto error_tgt_fput;
if (op == EPOLL_CTL_ADD && (is_file_epoll(tf.file) ||
(epds->events & ~EPOLLEXCLUSIVE_OK_BITS)))
goto error_tgt_fput;
}
/*
* At this point it is safe to assume that the "private_data" contains
* our own data structure.
*/
ep = f.file->private_data;
/*
* When we insert an epoll file descriptor inside another epoll file
* descriptor, there is the chance of creating closed loops, which are
* better be handled here, than in more critical paths. While we are
* checking for loops we also determine the list of files reachable
* and hang them on the tfile_check_list, so we can check that we
* haven't created too many possible wakeup paths.
*
* We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when
* the epoll file descriptor is attaching directly to a wakeup source,
* unless the epoll file descriptor is nested. The purpose of taking the
* 'epnested_mutex' on add is to prevent complex toplogies such as loops and
* deep wakeup paths from forming in parallel through multiple
* EPOLL_CTL_ADD operations.
*/
error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
if (error)
goto error_tgt_fput;
if (op == EPOLL_CTL_ADD) {
if (READ_ONCE(f.file->f_ep) || ep->gen == loop_check_gen ||
is_file_epoll(tf.file)) {
mutex_unlock(&ep->mtx);
error = epoll_mutex_lock(&epnested_mutex, 0, nonblock);
if (error)
goto error_tgt_fput;
loop_check_gen++;
full_check = 1;
if (is_file_epoll(tf.file)) {
tep = tf.file->private_data;
error = -ELOOP;
if (ep_loop_check(ep, tep) != 0)
goto error_tgt_fput;
}
error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
if (error)
goto error_tgt_fput;
}
}
/*
* Try to lookup the file inside our RB tree. Since we grabbed "mtx"
* above, we can be sure to be able to use the item looked up by
* ep_find() till we release the mutex.
*/
epi = ep_find(ep, tf.file, fd);
error = -EINVAL;
switch (op) {
case EPOLL_CTL_ADD:
if (!epi) {
epds->events |= EPOLLERR | EPOLLHUP;
error = ep_insert(ep, epds, tf.file, fd, full_check);
} else
error = -EEXIST;
break;
case EPOLL_CTL_DEL:
if (epi) {
/*
* The eventpoll itself is still alive: the refcount
* can't go to zero here.
*/
ep_remove_safe(ep, epi);
error = 0;
} else {
error = -ENOENT;
}
break;
case EPOLL_CTL_MOD:
if (epi) {
if (!(epi->event.events & EPOLLEXCLUSIVE)) {
epds->events |= EPOLLERR | EPOLLHUP;
error = ep_modify(ep, epi, epds);
}
} else
error = -ENOENT;
break;
}
mutex_unlock(&ep->mtx);
error_tgt_fput:
if (full_check) {
clear_tfile_check_list();
loop_check_gen++;
mutex_unlock(&epnested_mutex);
}
fdput(tf);
error_fput:
fdput(f);
error_return:
return error;
}
/*
* The following function implements the controller interface for
* the eventpoll file that enables the insertion/removal/change of
* file descriptors inside the interest set.
*/
SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
struct epoll_event __user *, event)
{
struct epoll_event epds;
if (ep_op_has_event(op) &&
copy_from_user(&epds, event, sizeof(struct epoll_event)))
return -EFAULT;
return do_epoll_ctl(epfd, op, fd, &epds, false);
}
/*
* Implement the event wait interface for the eventpoll file. It is the kernel
* part of the user space epoll_wait(2).
*/
static int do_epoll_wait(int epfd, struct epoll_event __user *events,
int maxevents, struct timespec64 *to)
{
int error;
struct fd f;
struct eventpoll *ep;
/* The maximum number of event must be greater than zero */
if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)
return -EINVAL;
/* Verify that the area passed by the user is writeable */
if (!access_ok(events, maxevents * sizeof(struct epoll_event)))
return -EFAULT;
/* Get the "struct file *" for the eventpoll file */
f = fdget(epfd);
if (!f.file)
return -EBADF;
/*
* We have to check that the file structure underneath the fd
* the user passed to us _is_ an eventpoll file.
*/
error = -EINVAL;
if (!is_file_epoll(f.file))
goto error_fput;
/*
* At this point it is safe to assume that the "private_data" contains
* our own data structure.
*/
ep = f.file->private_data;
/* Time to fish for events ... */
error = ep_poll(ep, events, maxevents, to);
error_fput:
fdput(f);
return error;
}
SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
int, maxevents, int, timeout)
{
struct timespec64 to;
return do_epoll_wait(epfd, events, maxevents,
ep_timeout_to_timespec(&to, timeout));
}
/*
* Implement the event wait interface for the eventpoll file. It is the kernel
* part of the user space epoll_pwait(2).
*/
static int do_epoll_pwait(int epfd, struct epoll_event __user *events,
int maxevents, struct timespec64 *to,
const sigset_t __user *sigmask, size_t sigsetsize)
{
int error;
/*
* If the caller wants a certain signal mask to be set during the wait,
* we apply it here.
*/
error = set_user_sigmask(sigmask, sigsetsize);
if (error)
return error;
error = do_epoll_wait(epfd, events, maxevents, to);
restore_saved_sigmask_unless(error == -EINTR);
return error;
}
SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events,
int, maxevents, int, timeout, const sigset_t __user *, sigmask,
size_t, sigsetsize)
{
struct timespec64 to;
return do_epoll_pwait(epfd, events, maxevents,
ep_timeout_to_timespec(&to, timeout),
sigmask, sigsetsize);
}
SYSCALL_DEFINE6(epoll_pwait2, int, epfd, struct epoll_event __user *, events,
int, maxevents, const struct __kernel_timespec __user *, timeout,
const sigset_t __user *, sigmask, size_t, sigsetsize)
{
struct timespec64 ts, *to = NULL;
if (timeout) {
if (get_timespec64(&ts, timeout))
return -EFAULT;
to = &ts;
if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
return -EINVAL;
}
return do_epoll_pwait(epfd, events, maxevents, to,
sigmask, sigsetsize);
}
#ifdef CONFIG_COMPAT
static int do_compat_epoll_pwait(int epfd, struct epoll_event __user *events,
int maxevents, struct timespec64 *timeout,
const compat_sigset_t __user *sigmask,
compat_size_t sigsetsize)
{
long err;
/*
* If the caller wants a certain signal mask to be set during the wait,
* we apply it here.
*/
err = set_compat_user_sigmask(sigmask, sigsetsize);
if (err)
return err;
err = do_epoll_wait(epfd, events, maxevents, timeout);
restore_saved_sigmask_unless(err == -EINTR);
return err;
}
COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd,
struct epoll_event __user *, events,
int, maxevents, int, timeout,
const compat_sigset_t __user *, sigmask,
compat_size_t, sigsetsize)
{
struct timespec64 to;
return do_compat_epoll_pwait(epfd, events, maxevents,
ep_timeout_to_timespec(&to, timeout),
sigmask, sigsetsize);
}
COMPAT_SYSCALL_DEFINE6(epoll_pwait2, int, epfd,
struct epoll_event __user *, events,
int, maxevents,
const struct __kernel_timespec __user *, timeout,
const compat_sigset_t __user *, sigmask,
compat_size_t, sigsetsize)
{
struct timespec64 ts, *to = NULL;
if (timeout) {
if (get_timespec64(&ts, timeout))
return -EFAULT;
to = &ts;
if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
return -EINVAL;
}
return do_compat_epoll_pwait(epfd, events, maxevents, to,
sigmask, sigsetsize);
}
#endif
static int __init eventpoll_init(void)
{
struct sysinfo si;
si_meminfo(&si);
/*
* Allows top 4% of lomem to be allocated for epoll watches (per user).
*/
max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) /
EP_ITEM_COST;
BUG_ON(max_user_watches < 0);
/*
* We can have many thousands of epitems, so prevent this from
* using an extra cache line on 64-bit (and smaller) CPUs
*/
BUILD_BUG_ON(sizeof(void *) <= 8 && sizeof(struct epitem) > 128);
/* Allocates slab cache used to allocate "struct epitem" items */
epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem),
0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
/* Allocates slab cache used to allocate "struct eppoll_entry" */
pwq_cache = kmem_cache_create("eventpoll_pwq",
sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);
epoll_sysctls_init();
ephead_cache = kmem_cache_create("ep_head",
sizeof(struct epitems_head), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);
return 0;
}
fs_initcall(eventpoll_init);
// SPDX-License-Identifier: GPL-2.0
/*
* NTP state machine interfaces and logic.
*
* This code was mainly moved from kernel/timer.c and kernel/time.c
* Please see those files for relevant copyright info and historical
* changelogs.
*/
#include <linux/capability.h>
#include <linux/clocksource.h>
#include <linux/workqueue.h>
#include <linux/hrtimer.h>
#include <linux/jiffies.h>
#include <linux/math64.h>
#include <linux/timex.h>
#include <linux/time.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/rtc.h>
#include <linux/audit.h>
#include "ntp_internal.h"
#include "timekeeping_internal.h"
/*
* NTP timekeeping variables:
*
* Note: All of the NTP state is protected by the timekeeping locks.
*/
/* USER_HZ period (usecs): */
unsigned long tick_usec = USER_TICK_USEC;
/* SHIFTED_HZ period (nsecs): */
unsigned long tick_nsec;
static u64 tick_length;
static u64 tick_length_base;
#define SECS_PER_DAY 86400
#define MAX_TICKADJ 500LL /* usecs */
#define MAX_TICKADJ_SCALED \
(((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
#define MAX_TAI_OFFSET 100000
/*
* phase-lock loop variables
*/
/*
* clock synchronization status
*
* (TIME_ERROR prevents overwriting the CMOS clock)
*/
static int time_state = TIME_OK;
/* clock status bits: */
static int time_status = STA_UNSYNC;
/* time adjustment (nsecs): */
static s64 time_offset;
/* pll time constant: */
static long time_constant = 2;
/* maximum error (usecs): */
static long time_maxerror = NTP_PHASE_LIMIT;
/* estimated error (usecs): */
static long time_esterror = NTP_PHASE_LIMIT;
/* frequency offset (scaled nsecs/secs): */
static s64 time_freq;
/* time at last adjustment (secs): */
static time64_t time_reftime;
static long time_adjust;
/* constant (boot-param configurable) NTP tick adjustment (upscaled) */
static s64 ntp_tick_adj;
/* second value of the next pending leapsecond, or TIME64_MAX if no leap */
static time64_t ntp_next_leap_sec = TIME64_MAX;
#ifdef CONFIG_NTP_PPS
/*
* The following variables are used when a pulse-per-second (PPS) signal
* is available. They establish the engineering parameters of the clock
* discipline loop when controlled by the PPS signal.
*/
#define PPS_VALID 10 /* PPS signal watchdog max (s) */
#define PPS_POPCORN 4 /* popcorn spike threshold (shift) */
#define PPS_INTMIN 2 /* min freq interval (s) (shift) */
#define PPS_INTMAX 8 /* max freq interval (s) (shift) */
#define PPS_INTCOUNT 4 /* number of consecutive good intervals to
increase pps_shift or consecutive bad
intervals to decrease it */
#define PPS_MAXWANDER 100000 /* max PPS freq wander (ns/s) */
static int pps_valid; /* signal watchdog counter */
static long pps_tf[3]; /* phase median filter */
static long pps_jitter; /* current jitter (ns) */
static struct timespec64 pps_fbase; /* beginning of the last freq interval */
static int pps_shift; /* current interval duration (s) (shift) */
static int pps_intcnt; /* interval counter */
static s64 pps_freq; /* frequency offset (scaled ns/s) */
static long pps_stabil; /* current stability (scaled ns/s) */
/*
* PPS signal quality monitors
*/
static long pps_calcnt; /* calibration intervals */
static long pps_jitcnt; /* jitter limit exceeded */
static long pps_stbcnt; /* stability limit exceeded */
static long pps_errcnt; /* calibration errors */
/* PPS kernel consumer compensates the whole phase error immediately.
* Otherwise, reduce the offset by a fixed factor times the time constant.
*/
static inline s64 ntp_offset_chunk(s64 offset)
{
if (time_status & STA_PPSTIME && time_status & STA_PPSSIGNAL)
return offset;
else
return shift_right(offset, SHIFT_PLL + time_constant);
}
static inline void pps_reset_freq_interval(void)
{
/* the PPS calibration interval may end
surprisingly early */
pps_shift = PPS_INTMIN;
pps_intcnt = 0;
}
/**
* pps_clear - Clears the PPS state variables
*/
static inline void pps_clear(void)
{
pps_reset_freq_interval();
pps_tf[0] = 0;
pps_tf[1] = 0;
pps_tf[2] = 0;
pps_fbase.tv_sec = pps_fbase.tv_nsec = 0;
pps_freq = 0;
}
/* Decrease pps_valid to indicate that another second has passed since
* the last PPS signal. When it reaches 0, indicate that PPS signal is
* missing.
*/
static inline void pps_dec_valid(void)
{
if (pps_valid > 0)
pps_valid--;
else {
time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
STA_PPSWANDER | STA_PPSERROR);
pps_clear();
}
}
static inline void pps_set_freq(s64 freq)
{
pps_freq = freq;
}
static inline int is_error_status(int status)
{
return (status & (STA_UNSYNC|STA_CLOCKERR))
/* PPS signal lost when either PPS time or
* PPS frequency synchronization requested
*/
|| ((status & (STA_PPSFREQ|STA_PPSTIME))
&& !(status & STA_PPSSIGNAL))
/* PPS jitter exceeded when
* PPS time synchronization requested */
|| ((status & (STA_PPSTIME|STA_PPSJITTER))
== (STA_PPSTIME|STA_PPSJITTER))
/* PPS wander exceeded or calibration error when
* PPS frequency synchronization requested
*/
|| ((status & STA_PPSFREQ)
&& (status & (STA_PPSWANDER|STA_PPSERROR)));
}
static inline void pps_fill_timex(struct __kernel_timex *txc)
{
txc->ppsfreq = shift_right((pps_freq >> PPM_SCALE_INV_SHIFT) *
PPM_SCALE_INV, NTP_SCALE_SHIFT);
txc->jitter = pps_jitter;
if (!(time_status & STA_NANO))
txc->jitter = pps_jitter / NSEC_PER_USEC;
txc->shift = pps_shift;
txc->stabil = pps_stabil;
txc->jitcnt = pps_jitcnt;
txc->calcnt = pps_calcnt;
txc->errcnt = pps_errcnt;
txc->stbcnt = pps_stbcnt;
}
#else /* !CONFIG_NTP_PPS */
static inline s64 ntp_offset_chunk(s64 offset)
{
return shift_right(offset, SHIFT_PLL + time_constant);
}
static inline void pps_reset_freq_interval(void) {}
static inline void pps_clear(void) {}
static inline void pps_dec_valid(void) {}
static inline void pps_set_freq(s64 freq) {}
static inline int is_error_status(int status)
{
return status & (STA_UNSYNC|STA_CLOCKERR);
}
static inline void pps_fill_timex(struct __kernel_timex *txc)
{
/* PPS is not implemented, so these are zero */
txc->ppsfreq = 0;
txc->jitter = 0;
txc->shift = 0;
txc->stabil = 0;
txc->jitcnt = 0;
txc->calcnt = 0;
txc->errcnt = 0;
txc->stbcnt = 0;
}
#endif /* CONFIG_NTP_PPS */
/**
* ntp_synced - Returns 1 if the NTP status is not UNSYNC
*
*/
static inline int ntp_synced(void)
{
return !(time_status & STA_UNSYNC);
}
/*
* NTP methods:
*/
/*
* Update (tick_length, tick_length_base, tick_nsec), based
* on (tick_usec, ntp_tick_adj, time_freq):
*/
static void ntp_update_frequency(void)
{
u64 second_length;
u64 new_base;
second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ)
<< NTP_SCALE_SHIFT;
second_length += ntp_tick_adj;
second_length += time_freq;
tick_nsec = div_u64(second_length, HZ) >> NTP_SCALE_SHIFT;
new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
/*
* Don't wait for the next second_overflow, apply
* the change to the tick length immediately:
*/
tick_length += new_base - tick_length_base;
tick_length_base = new_base;
}
static inline s64 ntp_update_offset_fll(s64 offset64, long secs)
{
time_status &= ~STA_MODE;
if (secs < MINSEC)
return 0;
if (!(time_status & STA_FLL) && (secs <= MAXSEC))
return 0;
time_status |= STA_MODE;
return div64_long(offset64 << (NTP_SCALE_SHIFT - SHIFT_FLL), secs);
}
static void ntp_update_offset(long offset)
{
s64 freq_adj;
s64 offset64;
long secs;
if (!(time_status & STA_PLL))
return;
if (!(time_status & STA_NANO)) {
/* Make sure the multiplication below won't overflow */
offset = clamp(offset, -USEC_PER_SEC, USEC_PER_SEC);
offset *= NSEC_PER_USEC;
}
/*
* Scale the phase adjustment and
* clamp to the operating range.
*/
offset = clamp(offset, -MAXPHASE, MAXPHASE);
/*
* Select how the frequency is to be controlled
* and in which mode (PLL or FLL).
*/
secs = (long)(__ktime_get_real_seconds() - time_reftime);
if (unlikely(time_status & STA_FREQHOLD))
secs = 0;
time_reftime = __ktime_get_real_seconds();
offset64 = offset;
freq_adj = ntp_update_offset_fll(offset64, secs);
/*
* Clamp update interval to reduce PLL gain with low
* sampling rate (e.g. intermittent network connection)
* to avoid instability.
*/
if (unlikely(secs > 1 << (SHIFT_PLL + 1 + time_constant)))
secs = 1 << (SHIFT_PLL + 1 + time_constant);
freq_adj += (offset64 * secs) <<
(NTP_SCALE_SHIFT - 2 * (SHIFT_PLL + 2 + time_constant));
freq_adj = min(freq_adj + time_freq, MAXFREQ_SCALED);
time_freq = max(freq_adj, -MAXFREQ_SCALED);
time_offset = div_s64(offset64 << NTP_SCALE_SHIFT, NTP_INTERVAL_FREQ);
}
/**
* ntp_clear - Clears the NTP state variables
*/
void ntp_clear(void)
{
time_adjust = 0; /* stop active adjtime() */
time_status |= STA_UNSYNC;
time_maxerror = NTP_PHASE_LIMIT;
time_esterror = NTP_PHASE_LIMIT;
ntp_update_frequency();
tick_length = tick_length_base;
time_offset = 0;
ntp_next_leap_sec = TIME64_MAX;
/* Clear PPS state variables */
pps_clear();
}
u64 ntp_tick_length(void)
{
return tick_length;
}
/**
* ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
*
* Provides the time of the next leapsecond against CLOCK_REALTIME in
* a ktime_t format. Returns KTIME_MAX if no leapsecond is pending.
*/
ktime_t ntp_get_next_leap(void)
{
ktime_t ret;
if ((time_state == TIME_INS) && (time_status & STA_INS)) return ktime_set(ntp_next_leap_sec, 0);
ret = KTIME_MAX;
return ret;
}
/*
* this routine handles the overflow of the microsecond field
*
* The tricky bits of code to handle the accurate clock support
* were provided by Dave Mills (Mills@UDEL.EDU) of NTP fame.
* They were originally developed for SUN and DEC kernels.
* All the kudos should go to Dave for this stuff.
*
* Also handles leap second processing, and returns leap offset
*/
int second_overflow(time64_t secs)
{
s64 delta;
int leap = 0;
s32 rem;
/*
* Leap second processing. If in leap-insert state at the end of the
* day, the system clock is set back one second; if in leap-delete
* state, the system clock is set ahead one second.
*/
switch (time_state) {
case TIME_OK:
if (time_status & STA_INS) {
time_state = TIME_INS;
div_s64_rem(secs, SECS_PER_DAY, &rem);
ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
} else if (time_status & STA_DEL) {
time_state = TIME_DEL;
div_s64_rem(secs + 1, SECS_PER_DAY, &rem);
ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
}
break;
case TIME_INS:
if (!(time_status & STA_INS)) {
ntp_next_leap_sec = TIME64_MAX;
time_state = TIME_OK;
} else if (secs == ntp_next_leap_sec) {
leap = -1;
time_state = TIME_OOP;
printk(KERN_NOTICE
"Clock: inserting leap second 23:59:60 UTC\n");
}
break;
case TIME_DEL:
if (!(time_status & STA_DEL)) {
ntp_next_leap_sec = TIME64_MAX;
time_state = TIME_OK;
} else if (secs == ntp_next_leap_sec) {
leap = 1;
ntp_next_leap_sec = TIME64_MAX;
time_state = TIME_WAIT;
printk(KERN_NOTICE
"Clock: deleting leap second 23:59:59 UTC\n");
}
break;
case TIME_OOP:
ntp_next_leap_sec = TIME64_MAX;
time_state = TIME_WAIT;
break;
case TIME_WAIT:
if (!(time_status & (STA_INS | STA_DEL)))
time_state = TIME_OK;
break;
}
/* Bump the maxerror field */
time_maxerror += MAXFREQ / NSEC_PER_USEC;
if (time_maxerror > NTP_PHASE_LIMIT) {
time_maxerror = NTP_PHASE_LIMIT;
time_status |= STA_UNSYNC;
}
/* Compute the phase adjustment for the next second */
tick_length = tick_length_base;
delta = ntp_offset_chunk(time_offset);
time_offset -= delta;
tick_length += delta;
/* Check PPS signal */
pps_dec_valid();
if (!time_adjust)
goto out;
if (time_adjust > MAX_TICKADJ) {
time_adjust -= MAX_TICKADJ;
tick_length += MAX_TICKADJ_SCALED;
goto out;
}
if (time_adjust < -MAX_TICKADJ) {
time_adjust += MAX_TICKADJ;
tick_length -= MAX_TICKADJ_SCALED;
goto out;
}
tick_length += (s64)(time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
<< NTP_SCALE_SHIFT;
time_adjust = 0;
out:
return leap;
}
#if defined(CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC)
static void sync_hw_clock(struct work_struct *work);
static DECLARE_WORK(sync_work, sync_hw_clock);
static struct hrtimer sync_hrtimer;
#define SYNC_PERIOD_NS (11ULL * 60 * NSEC_PER_SEC)
static enum hrtimer_restart sync_timer_callback(struct hrtimer *timer)
{
queue_work(system_freezable_power_efficient_wq, &sync_work);
return HRTIMER_NORESTART;
}
static void sched_sync_hw_clock(unsigned long offset_nsec, bool retry)
{
ktime_t exp = ktime_set(ktime_get_real_seconds(), 0);
if (retry)
exp = ktime_add_ns(exp, 2ULL * NSEC_PER_SEC - offset_nsec);
else
exp = ktime_add_ns(exp, SYNC_PERIOD_NS - offset_nsec);
hrtimer_start(&sync_hrtimer, exp, HRTIMER_MODE_ABS);
}
/*
* Check whether @now is correct versus the required time to update the RTC
* and calculate the value which needs to be written to the RTC so that the
* next seconds increment of the RTC after the write is aligned with the next
* seconds increment of clock REALTIME.
*
* tsched t1 write(t2.tv_sec - 1sec)) t2 RTC increments seconds
*
* t2.tv_nsec == 0
* tsched = t2 - set_offset_nsec
* newval = t2 - NSEC_PER_SEC
*
* ==> neval = tsched + set_offset_nsec - NSEC_PER_SEC
*
* As the execution of this code is not guaranteed to happen exactly at
* tsched this allows it to happen within a fuzzy region:
*
* abs(now - tsched) < FUZZ
*
* If @now is not inside the allowed window the function returns false.
*/
static inline bool rtc_tv_nsec_ok(unsigned long set_offset_nsec,
struct timespec64 *to_set,
const struct timespec64 *now)
{
/* Allowed error in tv_nsec, arbitrarily set to 5 jiffies in ns. */
const unsigned long TIME_SET_NSEC_FUZZ = TICK_NSEC * 5;
struct timespec64 delay = {.tv_sec = -1,
.tv_nsec = set_offset_nsec};
*to_set = timespec64_add(*now, delay);
if (to_set->tv_nsec < TIME_SET_NSEC_FUZZ) {
to_set->tv_nsec = 0;
return true;
}
if (to_set->tv_nsec > NSEC_PER_SEC - TIME_SET_NSEC_FUZZ) {
to_set->tv_sec++;
to_set->tv_nsec = 0;
return true;
}
return false;
}
#ifdef CONFIG_GENERIC_CMOS_UPDATE
int __weak update_persistent_clock64(struct timespec64 now64)
{
return -ENODEV;
}
#else
static inline int update_persistent_clock64(struct timespec64 now64)
{
return -ENODEV;
}
#endif
#ifdef CONFIG_RTC_SYSTOHC
/* Save NTP synchronized time to the RTC */
static int update_rtc(struct timespec64 *to_set, unsigned long *offset_nsec)
{
struct rtc_device *rtc;
struct rtc_time tm;
int err = -ENODEV;
rtc = rtc_class_open(CONFIG_RTC_SYSTOHC_DEVICE);
if (!rtc)
return -ENODEV;
if (!rtc->ops || !rtc->ops->set_time)
goto out_close;
/* First call might not have the correct offset */
if (*offset_nsec == rtc->set_offset_nsec) {
rtc_time64_to_tm(to_set->tv_sec, &tm);
err = rtc_set_time(rtc, &tm);
} else {
/* Store the update offset and let the caller try again */
*offset_nsec = rtc->set_offset_nsec;
err = -EAGAIN;
}
out_close:
rtc_class_close(rtc);
return err;
}
#else
static inline int update_rtc(struct timespec64 *to_set, unsigned long *offset_nsec)
{
return -ENODEV;
}
#endif
/*
* If we have an externally synchronized Linux clock, then update RTC clock
* accordingly every ~11 minutes. Generally RTCs can only store second
* precision, but many RTCs will adjust the phase of their second tick to
* match the moment of update. This infrastructure arranges to call to the RTC
* set at the correct moment to phase synchronize the RTC second tick over
* with the kernel clock.
*/
static void sync_hw_clock(struct work_struct *work)
{
/*
* The default synchronization offset is 500ms for the deprecated
* update_persistent_clock64() under the assumption that it uses
* the infamous CMOS clock (MC146818).
*/
static unsigned long offset_nsec = NSEC_PER_SEC / 2;
struct timespec64 now, to_set;
int res = -EAGAIN;
/*
* Don't update if STA_UNSYNC is set and if ntp_notify_cmos_timer()
* managed to schedule the work between the timer firing and the
* work being able to rearm the timer. Wait for the timer to expire.
*/
if (!ntp_synced() || hrtimer_is_queued(&sync_hrtimer))
return;
ktime_get_real_ts64(&now);
/* If @now is not in the allowed window, try again */
if (!rtc_tv_nsec_ok(offset_nsec, &to_set, &now))
goto rearm;
/* Take timezone adjusted RTCs into account */
if (persistent_clock_is_local)
to_set.tv_sec -= (sys_tz.tz_minuteswest * 60);
/* Try the legacy RTC first. */
res = update_persistent_clock64(to_set);
if (res != -ENODEV)
goto rearm;
/* Try the RTC class */
res = update_rtc(&to_set, &offset_nsec);
if (res == -ENODEV)
return;
rearm:
sched_sync_hw_clock(offset_nsec, res != 0);
}
void ntp_notify_cmos_timer(void)
{
/*
* When the work is currently executed but has not yet the timer
* rearmed this queues the work immediately again. No big issue,
* just a pointless work scheduled.
*/
if (ntp_synced() && !hrtimer_is_queued(&sync_hrtimer))
queue_work(system_freezable_power_efficient_wq, &sync_work);
}
static void __init ntp_init_cmos_sync(void)
{
hrtimer_init(&sync_hrtimer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
sync_hrtimer.function = sync_timer_callback;
}
#else /* CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC) */
static inline void __init ntp_init_cmos_sync(void) { }
#endif /* !CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC) */
/*
* Propagate a new txc->status value into the NTP state:
*/
static inline void process_adj_status(const struct __kernel_timex *txc)
{
if ((time_status & STA_PLL) && !(txc->status & STA_PLL)) {
time_state = TIME_OK;
time_status = STA_UNSYNC;
ntp_next_leap_sec = TIME64_MAX;
/* restart PPS frequency calibration */
pps_reset_freq_interval();
}
/*
* If we turn on PLL adjustments then reset the
* reference time to current time.
*/
if (!(time_status & STA_PLL) && (txc->status & STA_PLL))
time_reftime = __ktime_get_real_seconds();
/* only set allowed bits */
time_status &= STA_RONLY;
time_status |= txc->status & ~STA_RONLY;
}
static inline void process_adjtimex_modes(const struct __kernel_timex *txc,
s32 *time_tai)
{
if (txc->modes & ADJ_STATUS)
process_adj_status(txc);
if (txc->modes & ADJ_NANO)
time_status |= STA_NANO;
if (txc->modes & ADJ_MICRO)
time_status &= ~STA_NANO;
if (txc->modes & ADJ_FREQUENCY) {
time_freq = txc->freq * PPM_SCALE;
time_freq = min(time_freq, MAXFREQ_SCALED);
time_freq = max(time_freq, -MAXFREQ_SCALED);
/* update pps_freq */
pps_set_freq(time_freq);
}
if (txc->modes & ADJ_MAXERROR)
time_maxerror = txc->maxerror;
if (txc->modes & ADJ_ESTERROR)
time_esterror = txc->esterror;
if (txc->modes & ADJ_TIMECONST) {
time_constant = txc->constant;
if (!(time_status & STA_NANO))
time_constant += 4;
time_constant = min(time_constant, (long)MAXTC);
time_constant = max(time_constant, 0l);
}
if (txc->modes & ADJ_TAI &&
txc->constant >= 0 && txc->constant <= MAX_TAI_OFFSET)
*time_tai = txc->constant;
if (txc->modes & ADJ_OFFSET)
ntp_update_offset(txc->offset);
if (txc->modes & ADJ_TICK)
tick_usec = txc->tick;
if (txc->modes & (ADJ_TICK|ADJ_FREQUENCY|ADJ_OFFSET))
ntp_update_frequency();
}
/*
* adjtimex mainly allows reading (and writing, if superuser) of
* kernel time-keeping variables. used by xntpd.
*/
int __do_adjtimex(struct __kernel_timex *txc, const struct timespec64 *ts,
s32 *time_tai, struct audit_ntp_data *ad)
{
int result;
if (txc->modes & ADJ_ADJTIME) {
long save_adjust = time_adjust;
if (!(txc->modes & ADJ_OFFSET_READONLY)) {
/* adjtime() is independent from ntp_adjtime() */
time_adjust = txc->offset;
ntp_update_frequency();
audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust);
audit_ntp_set_new(ad, AUDIT_NTP_ADJUST, time_adjust);
}
txc->offset = save_adjust;
} else {
/* If there are input parameters, then process them: */
if (txc->modes) {
audit_ntp_set_old(ad, AUDIT_NTP_OFFSET, time_offset);
audit_ntp_set_old(ad, AUDIT_NTP_FREQ, time_freq);
audit_ntp_set_old(ad, AUDIT_NTP_STATUS, time_status);
audit_ntp_set_old(ad, AUDIT_NTP_TAI, *time_tai);
audit_ntp_set_old(ad, AUDIT_NTP_TICK, tick_usec);
process_adjtimex_modes(txc, time_tai);
audit_ntp_set_new(ad, AUDIT_NTP_OFFSET, time_offset);
audit_ntp_set_new(ad, AUDIT_NTP_FREQ, time_freq);
audit_ntp_set_new(ad, AUDIT_NTP_STATUS, time_status);
audit_ntp_set_new(ad, AUDIT_NTP_TAI, *time_tai);
audit_ntp_set_new(ad, AUDIT_NTP_TICK, tick_usec);
}
txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ,
NTP_SCALE_SHIFT);
if (!(time_status & STA_NANO))
txc->offset = (u32)txc->offset / NSEC_PER_USEC;
}
result = time_state; /* mostly `TIME_OK' */
/* check for errors */
if (is_error_status(time_status))
result = TIME_ERROR;
txc->freq = shift_right((time_freq >> PPM_SCALE_INV_SHIFT) *
PPM_SCALE_INV, NTP_SCALE_SHIFT);
txc->maxerror = time_maxerror;
txc->esterror = time_esterror;
txc->status = time_status;
txc->constant = time_constant;
txc->precision = 1;
txc->tolerance = MAXFREQ_SCALED / PPM_SCALE;
txc->tick = tick_usec;
txc->tai = *time_tai;
/* fill PPS status fields */
pps_fill_timex(txc);
txc->time.tv_sec = ts->tv_sec;
txc->time.tv_usec = ts->tv_nsec;
if (!(time_status & STA_NANO))
txc->time.tv_usec = ts->tv_nsec / NSEC_PER_USEC;
/* Handle leapsec adjustments */
if (unlikely(ts->tv_sec >= ntp_next_leap_sec)) {
if ((time_state == TIME_INS) && (time_status & STA_INS)) {
result = TIME_OOP;
txc->tai++;
txc->time.tv_sec--;
}
if ((time_state == TIME_DEL) && (time_status & STA_DEL)) {
result = TIME_WAIT;
txc->tai--;
txc->time.tv_sec++;
}
if ((time_state == TIME_OOP) &&
(ts->tv_sec == ntp_next_leap_sec)) {
result = TIME_WAIT;
}
}
return result;
}
#ifdef CONFIG_NTP_PPS
/* actually struct pps_normtime is good old struct timespec, but it is
* semantically different (and it is the reason why it was invented):
* pps_normtime.nsec has a range of ( -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ]
* while timespec.tv_nsec has a range of [0, NSEC_PER_SEC) */
struct pps_normtime {
s64 sec; /* seconds */
long nsec; /* nanoseconds */
};
/* normalize the timestamp so that nsec is in the
( -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ] interval */
static inline struct pps_normtime pps_normalize_ts(struct timespec64 ts)
{
struct pps_normtime norm = {
.sec = ts.tv_sec,
.nsec = ts.tv_nsec
};
if (norm.nsec > (NSEC_PER_SEC >> 1)) {
norm.nsec -= NSEC_PER_SEC;
norm.sec++;
}
return norm;
}
/* get current phase correction and jitter */
static inline long pps_phase_filter_get(long *jitter)
{
*jitter = pps_tf[0] - pps_tf[1];
if (*jitter < 0)
*jitter = -*jitter;
/* TODO: test various filters */
return pps_tf[0];
}
/* add the sample to the phase filter */
static inline void pps_phase_filter_add(long err)
{
pps_tf[2] = pps_tf[1];
pps_tf[1] = pps_tf[0];
pps_tf[0] = err;
}
/* decrease frequency calibration interval length.
* It is halved after four consecutive unstable intervals.
*/
static inline void pps_dec_freq_interval(void)
{
if (--pps_intcnt <= -PPS_INTCOUNT) {
pps_intcnt = -PPS_INTCOUNT;
if (pps_shift > PPS_INTMIN) {
pps_shift--;
pps_intcnt = 0;
}
}
}
/* increase frequency calibration interval length.
* It is doubled after four consecutive stable intervals.
*/
static inline void pps_inc_freq_interval(void)
{
if (++pps_intcnt >= PPS_INTCOUNT) {
pps_intcnt = PPS_INTCOUNT;
if (pps_shift < PPS_INTMAX) {
pps_shift++;
pps_intcnt = 0;
}
}
}
/* update clock frequency based on MONOTONIC_RAW clock PPS signal
* timestamps
*
* At the end of the calibration interval the difference between the
* first and last MONOTONIC_RAW clock timestamps divided by the length
* of the interval becomes the frequency update. If the interval was
* too long, the data are discarded.
* Returns the difference between old and new frequency values.
*/
static long hardpps_update_freq(struct pps_normtime freq_norm)
{
long delta, delta_mod;
s64 ftemp;
/* check if the frequency interval was too long */
if (freq_norm.sec > (2 << pps_shift)) {
time_status |= STA_PPSERROR;
pps_errcnt++;
pps_dec_freq_interval();
printk_deferred(KERN_ERR
"hardpps: PPSERROR: interval too long - %lld s\n",
freq_norm.sec);
return 0;
}
/* here the raw frequency offset and wander (stability) is
* calculated. If the wander is less than the wander threshold
* the interval is increased; otherwise it is decreased.
*/
ftemp = div_s64(((s64)(-freq_norm.nsec)) << NTP_SCALE_SHIFT,
freq_norm.sec);
delta = shift_right(ftemp - pps_freq, NTP_SCALE_SHIFT);
pps_freq = ftemp;
if (delta > PPS_MAXWANDER || delta < -PPS_MAXWANDER) {
printk_deferred(KERN_WARNING
"hardpps: PPSWANDER: change=%ld\n", delta);
time_status |= STA_PPSWANDER;
pps_stbcnt++;
pps_dec_freq_interval();
} else { /* good sample */
pps_inc_freq_interval();
}
/* the stability metric is calculated as the average of recent
* frequency changes, but is used only for performance
* monitoring
*/
delta_mod = delta;
if (delta_mod < 0)
delta_mod = -delta_mod;
pps_stabil += (div_s64(((s64)delta_mod) <<
(NTP_SCALE_SHIFT - SHIFT_USEC),
NSEC_PER_USEC) - pps_stabil) >> PPS_INTMIN;
/* if enabled, the system clock frequency is updated */
if ((time_status & STA_PPSFREQ) != 0 &&
(time_status & STA_FREQHOLD) == 0) {
time_freq = pps_freq;
ntp_update_frequency();
}
return delta;
}
/* correct REALTIME clock phase error against PPS signal */
static void hardpps_update_phase(long error)
{
long correction = -error;
long jitter;
/* add the sample to the median filter */
pps_phase_filter_add(correction);
correction = pps_phase_filter_get(&jitter);
/* Nominal jitter is due to PPS signal noise. If it exceeds the
* threshold, the sample is discarded; otherwise, if so enabled,
* the time offset is updated.
*/
if (jitter > (pps_jitter << PPS_POPCORN)) {
printk_deferred(KERN_WARNING
"hardpps: PPSJITTER: jitter=%ld, limit=%ld\n",
jitter, (pps_jitter << PPS_POPCORN));
time_status |= STA_PPSJITTER;
pps_jitcnt++;
} else if (time_status & STA_PPSTIME) {
/* correct the time using the phase offset */
time_offset = div_s64(((s64)correction) << NTP_SCALE_SHIFT,
NTP_INTERVAL_FREQ);
/* cancel running adjtime() */
time_adjust = 0;
}
/* update jitter */
pps_jitter += (jitter - pps_jitter) >> PPS_INTMIN;
}
/*
* __hardpps() - discipline CPU clock oscillator to external PPS signal
*
* This routine is called at each PPS signal arrival in order to
* discipline the CPU clock oscillator to the PPS signal. It takes two
* parameters: REALTIME and MONOTONIC_RAW clock timestamps. The former
* is used to correct clock phase error and the latter is used to
* correct the frequency.
*
* This code is based on David Mills's reference nanokernel
* implementation. It was mostly rewritten but keeps the same idea.
*/
void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
{
struct pps_normtime pts_norm, freq_norm;
pts_norm = pps_normalize_ts(*phase_ts);
/* clear the error bits, they will be set again if needed */
time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
/* indicate signal presence */
time_status |= STA_PPSSIGNAL;
pps_valid = PPS_VALID;
/* when called for the first time,
* just start the frequency interval */
if (unlikely(pps_fbase.tv_sec == 0)) {
pps_fbase = *raw_ts;
return;
}
/* ok, now we have a base for frequency calculation */
freq_norm = pps_normalize_ts(timespec64_sub(*raw_ts, pps_fbase));
/* check that the signal is in the range
* [1s - MAXFREQ us, 1s + MAXFREQ us], otherwise reject it */
if ((freq_norm.sec == 0) ||
(freq_norm.nsec > MAXFREQ * freq_norm.sec) ||
(freq_norm.nsec < -MAXFREQ * freq_norm.sec)) {
time_status |= STA_PPSJITTER;
/* restart the frequency calibration interval */
pps_fbase = *raw_ts;
printk_deferred(KERN_ERR "hardpps: PPSJITTER: bad pulse\n");
return;
}
/* signal is ok */
/* check if the current frequency interval is finished */
if (freq_norm.sec >= (1 << pps_shift)) {
pps_calcnt++;
/* restart the frequency calibration interval */
pps_fbase = *raw_ts;
hardpps_update_freq(freq_norm);
}
hardpps_update_phase(pts_norm.nsec);
}
#endif /* CONFIG_NTP_PPS */
static int __init ntp_tick_adj_setup(char *str)
{
int rc = kstrtos64(str, 0, &ntp_tick_adj);
if (rc)
return rc;
ntp_tick_adj <<= NTP_SCALE_SHIFT;
return 1;
}
__setup("ntp_tick_adj=", ntp_tick_adj_setup);
void __init ntp_init(void)
{
ntp_clear();
ntp_init_cmos_sync();
}
// SPDX-License-Identifier: GPL-2.0
/*
* USB Serial Console driver
*
* Copyright (C) 2001 - 2002 Greg Kroah-Hartman (greg@kroah.com)
*
* Thanks to Randy Dunlap for the original version of this code.
*
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/tty.h>
#include <linux/console.h>
#include <linux/serial.h>
#include <linux/usb.h>
#include <linux/usb/serial.h>
struct usbcons_info {
int magic;
int break_flag;
struct usb_serial_port *port;
};
static struct usbcons_info usbcons_info;
static struct console usbcons;
/*
* ------------------------------------------------------------
* USB Serial console driver
*
* Much of the code here is copied from drivers/char/serial.c
* and implements a phony serial console in the same way that
* serial.c does so that in case some software queries it,
* it will get the same results.
*
* Things that are different from the way the serial port code
* does things, is that we call the lower level usb-serial
* driver code to initialize the device, and we set the initial
* console speeds based on the command line arguments.
* ------------------------------------------------------------
*/
static const struct tty_operations usb_console_fake_tty_ops = {
};
/*
* The parsing of the command line works exactly like the
* serial.c code, except that the specifier is "ttyUSB" instead
* of "ttyS".
*/
static int usb_console_setup(struct console *co, char *options)
{
struct usbcons_info *info = &usbcons_info;
int baud = 9600;
int bits = 8;
int parity = 'n';
int doflow = 0;
int cflag = CREAD | HUPCL | CLOCAL;
char *s;
struct usb_serial *serial;
struct usb_serial_port *port;
int retval;
struct tty_struct *tty = NULL;
struct ktermios dummy;
if (options) {
baud = simple_strtoul(options, NULL, 10);
s = options;
while (*s >= '0' && *s <= '9')
s++;
if (*s)
parity = *s++;
if (*s)
bits = *s++ - '0';
if (*s)
doflow = (*s++ == 'r');
}
/* Sane default */
if (baud == 0)
baud = 9600;
switch (bits) {
case 7:
cflag |= CS7;
break;
default:
case 8:
cflag |= CS8;
break;
}
switch (parity) {
case 'o': case 'O':
cflag |= PARODD;
break;
case 'e': case 'E':
cflag |= PARENB;
break;
}
if (doflow)
cflag |= CRTSCTS;
/*
* no need to check the index here: if the index is wrong, console
* code won't call us
*/
port = usb_serial_port_get_by_minor(co->index);
if (port == NULL) {
/* no device is connected yet, sorry :( */
pr_err("No USB device connected to ttyUSB%i\n", co->index);
return -ENODEV;
}
serial = port->serial;
retval = usb_autopm_get_interface(serial->interface);
if (retval)
goto error_get_interface;
tty_port_tty_set(&port->port, NULL);
info->port = port;
++port->port.count;
if (!tty_port_initialized(&port->port)) {
if (serial->type->set_termios) {
/*
* allocate a fake tty so the driver can initialize
* the termios structure, then later call set_termios to
* configure according to command line arguments
*/
tty = kzalloc(sizeof(*tty), GFP_KERNEL);
if (!tty) {
retval = -ENOMEM;
goto reset_open_count;
}
kref_init(&tty->kref);
tty->driver = usb_serial_tty_driver;
tty->index = co->index;
init_ldsem(&tty->ldisc_sem);
spin_lock_init(&tty->files_lock);
INIT_LIST_HEAD(&tty->tty_files);
kref_get(&tty->driver->kref);
__module_get(tty->driver->owner);
tty->ops = &usb_console_fake_tty_ops;
tty_init_termios(tty);
tty_port_tty_set(&port->port, tty);
}
/* only call the device specific open if this
* is the first time the port is opened */
retval = serial->type->open(NULL, port);
if (retval) {
dev_err(&port->dev, "could not open USB console port\n");
goto fail;
}
if (serial->type->set_termios) {
tty->termios.c_cflag = cflag;
tty_termios_encode_baud_rate(&tty->termios, baud, baud);
memset(&dummy, 0, sizeof(struct ktermios));
serial->type->set_termios(tty, port, &dummy);
tty_port_tty_set(&port->port, NULL);
tty_save_termios(tty);
tty_kref_put(tty);
}
tty_port_set_initialized(&port->port, true);
}
/* Now that any required fake tty operations are completed restore
* the tty port count */
--port->port.count;
/* The console is special in terms of closing the device so
* indicate this port is now acting as a system console. */
port->port.console = 1;
mutex_unlock(&serial->disc_mutex);
return retval;
fail:
tty_port_tty_set(&port->port, NULL);
tty_kref_put(tty);
reset_open_count:
port->port.count = 0;
info->port = NULL;
usb_autopm_put_interface(serial->interface);
error_get_interface:
mutex_unlock(&serial->disc_mutex);
usb_serial_put(serial);
return retval;
}
static void usb_console_write(struct console *co,
const char *buf, unsigned count)
{
static struct usbcons_info *info = &usbcons_info;
struct usb_serial_port *port = info->port;
struct usb_serial *serial;
int retval = -ENODEV;
if (!port || port->serial->dev->state == USB_STATE_NOTATTACHED)
return;
serial = port->serial;
if (count == 0)
return;
dev_dbg(&port->dev, "%s - %d byte(s)\n", __func__, count);
if (!port->port.console) {
dev_dbg(&port->dev, "%s - port not opened\n", __func__);
return;
}
while (count) {
unsigned int i;
unsigned int lf;
/* search for LF so we can insert CR if necessary */
for (i = 0, lf = 0 ; i < count ; i++) {
if (*(buf + i) == 10) {
lf = 1;
i++;
break;
}
}
/* pass on to the driver specific version of this function if
it is available */
retval = serial->type->write(NULL, port, buf, i);
dev_dbg(&port->dev, "%s - write: %d\n", __func__, retval);
if (lf) {
/* append CR after LF */
unsigned char cr = 13;
retval = serial->type->write(NULL, port, &cr, 1);
dev_dbg(&port->dev, "%s - write cr: %d\n",
__func__, retval);
}
buf += i;
count -= i;
}
}
static struct tty_driver *usb_console_device(struct console *co, int *index)
{
struct tty_driver **p = (struct tty_driver **)co->data;
if (!*p)
return NULL;
*index = co->index;
return *p;
}
static struct console usbcons = {
.name = "ttyUSB",
.write = usb_console_write,
.device = usb_console_device,
.setup = usb_console_setup,
.flags = CON_PRINTBUFFER,
.index = -1,
.data = &usb_serial_tty_driver,
};
void usb_serial_console_disconnect(struct usb_serial *serial)
{
if (serial->port[0] && serial->port[0] == usbcons_info.port) { usb_serial_console_exit();
usb_serial_put(serial);
}
}
void usb_serial_console_init(int minor)
{
if (minor == 0) {
/*
* Call register_console() if this is the first device plugged
* in. If we call it earlier, then the callback to
* console_setup() will fail, as there is not a device seen by
* the USB subsystem yet.
*/
/*
* Register console.
* NOTES:
* console_setup() is called (back) immediately (from
* register_console). console_write() is called immediately
* from register_console iff CON_PRINTBUFFER is set in flags.
*/
pr_debug("registering the USB serial console.\n");
register_console(&usbcons);
}
}
void usb_serial_console_exit(void)
{
if (usbcons_info.port) {
unregister_console(&usbcons);
usbcons_info.port->port.console = 0;
usbcons_info.port = NULL;
}
}
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/spinlock.h>
#include <linux/mm.h>
#include <linux/memremap.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/secretmem.h>
#include <linux/sched/signal.h>
#include <linux/rwsem.h>
#include <linux/hugetlb.h>
#include <linux/migrate.h>
#include <linux/mm_inline.h>
#include <linux/sched/mm.h>
#include <linux/shmem_fs.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
#include "internal.h"
struct follow_page_context {
struct dev_pagemap *pgmap;
unsigned int page_mask;
};
static inline void sanity_check_pinned_pages(struct page **pages,
unsigned long npages)
{
if (!IS_ENABLED(CONFIG_DEBUG_VM))
return;
/*
* We only pin anonymous pages if they are exclusive. Once pinned, we
* can no longer turn them possibly shared and PageAnonExclusive() will
* stick around until the page is freed.
*
* We'd like to verify that our pinned anonymous pages are still mapped
* exclusively. The issue with anon THP is that we don't know how
* they are/were mapped when pinning them. However, for anon
* THP we can assume that either the given page (PTE-mapped THP) or
* the head page (PMD-mapped THP) should be PageAnonExclusive(). If
* neither is the case, there is certainly something wrong.
*/
for (; npages; npages--, pages++) {
struct page *page = *pages;
struct folio *folio = page_folio(page);
if (is_zero_page(page) ||
!folio_test_anon(folio))
continue;
if (!folio_test_large(folio) || folio_test_hugetlb(folio))
VM_BUG_ON_PAGE(!PageAnonExclusive(&folio->page), page);
else
/* Either a PTE-mapped or a PMD-mapped THP. */
VM_BUG_ON_PAGE(!PageAnonExclusive(&folio->page) &&
!PageAnonExclusive(page), page);
}
}
/*
* Return the folio with ref appropriately incremented,
* or NULL if that failed.
*/
static inline struct folio *try_get_folio(struct page *page, int refs)
{
struct folio *folio;
retry:
folio = page_folio(page);
if (WARN_ON_ONCE(folio_ref_count(folio) < 0))
return NULL;
if (unlikely(!folio_ref_try_add_rcu(folio, refs)))
return NULL;
/*
* At this point we have a stable reference to the folio; but it
* could be that between calling page_folio() and the refcount
* increment, the folio was split, in which case we'd end up
* holding a reference on a folio that has nothing to do with the page
* we were given anymore.
* So now that the folio is stable, recheck that the page still
* belongs to this folio.
*/
if (unlikely(page_folio(page) != folio)) {
if (!put_devmap_managed_page_refs(&folio->page, refs))
folio_put_refs(folio, refs);
goto retry;
}
return folio;
}
/**
* try_grab_folio() - Attempt to get or pin a folio.
* @page: pointer to page to be grabbed
* @refs: the value to (effectively) add to the folio's refcount
* @flags: gup flags: these are the FOLL_* flag values.
*
* "grab" names in this file mean, "look at flags to decide whether to use
* FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount.
*
* Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
* same time. (That's true throughout the get_user_pages*() and
* pin_user_pages*() APIs.) Cases:
*
* FOLL_GET: folio's refcount will be incremented by @refs.
*
* FOLL_PIN on large folios: folio's refcount will be incremented by
* @refs, and its pincount will be incremented by @refs.
*
* FOLL_PIN on single-page folios: folio's refcount will be incremented by
* @refs * GUP_PIN_COUNTING_BIAS.
*
* Return: The folio containing @page (with refcount appropriately
* incremented) for success, or NULL upon failure. If neither FOLL_GET
* nor FOLL_PIN was set, that's considered failure, and furthermore,
* a likely bug in the caller, so a warning is also emitted.
*/
struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags)
{
struct folio *folio;
if (WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == 0))
return NULL;
if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)))
return NULL;
if (flags & FOLL_GET)
return try_get_folio(page, refs);
/* FOLL_PIN is set */
/*
* Don't take a pin on the zero page - it's not going anywhere
* and it is used in a *lot* of places.
*/
if (is_zero_page(page))
return page_folio(page);
folio = try_get_folio(page, refs);
if (!folio)
return NULL;
/*
* Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a
* right zone, so fail and let the caller fall back to the slow
* path.
*/
if (unlikely((flags & FOLL_LONGTERM) &&
!folio_is_longterm_pinnable(folio))) {
if (!put_devmap_managed_page_refs(&folio->page, refs))
folio_put_refs(folio, refs);
return NULL;
}
/*
* When pinning a large folio, use an exact count to track it.
*
* However, be sure to *also* increment the normal folio
* refcount field at least once, so that the folio really
* is pinned. That's why the refcount from the earlier
* try_get_folio() is left intact.
*/
if (folio_test_large(folio))
atomic_add(refs, &folio->_pincount);
else
folio_ref_add(folio,
refs * (GUP_PIN_COUNTING_BIAS - 1));
/*
* Adjust the pincount before re-checking the PTE for changes.
* This is essentially a smp_mb() and is paired with a memory
* barrier in folio_try_share_anon_rmap_*().
*/
smp_mb__after_atomic();
node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
return folio;
}
static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
{
if (flags & FOLL_PIN) {
if (is_zero_folio(folio))
return;
node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);
if (folio_test_large(folio))
atomic_sub(refs, &folio->_pincount);
else
refs *= GUP_PIN_COUNTING_BIAS;
}
if (!put_devmap_managed_page_refs(&folio->page, refs))
folio_put_refs(folio, refs);
}
/**
* try_grab_page() - elevate a page's refcount by a flag-dependent amount
* @page: pointer to page to be grabbed
* @flags: gup flags: these are the FOLL_* flag values.
*
* This might not do anything at all, depending on the flags argument.
*
* "grab" names in this file mean, "look at flags to decide whether to use
* FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
*
* Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
* time. Cases: please see the try_grab_folio() documentation, with
* "refs=1".
*
* Return: 0 for success, or if no action was required (if neither FOLL_PIN
* nor FOLL_GET was set, nothing is done). A negative error code for failure:
*
* -ENOMEM FOLL_GET or FOLL_PIN was set, but the page could not
* be grabbed.
*/
int __must_check try_grab_page(struct page *page, unsigned int flags)
{
struct folio *folio = page_folio(page);
if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
return -ENOMEM;
if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)))
return -EREMOTEIO;
if (flags & FOLL_GET)
folio_ref_inc(folio);
else if (flags & FOLL_PIN) {
/*
* Don't take a pin on the zero page - it's not going anywhere
* and it is used in a *lot* of places.
*/
if (is_zero_page(page))
return 0;
/*
* Similar to try_grab_folio(): be sure to *also*
* increment the normal page refcount field at least once,
* so that the page really is pinned.
*/
if (folio_test_large(folio)) {
folio_ref_add(folio, 1);
atomic_add(1, &folio->_pincount);
} else {
folio_ref_add(folio, GUP_PIN_COUNTING_BIAS);
}
node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, 1);
}
return 0;
}
/**
* unpin_user_page() - release a dma-pinned page
* @page: pointer to page to be released
*
* Pages that were pinned via pin_user_pages*() must be released via either
* unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
* that such pages can be separately tracked and uniquely handled. In
* particular, interactions with RDMA and filesystems need special handling.
*/
void unpin_user_page(struct page *page)
{
sanity_check_pinned_pages(&page, 1);
gup_put_folio(page_folio(page), 1, FOLL_PIN);
}
EXPORT_SYMBOL(unpin_user_page);
/**
* folio_add_pin - Try to get an additional pin on a pinned folio
* @folio: The folio to be pinned
*
* Get an additional pin on a folio we already have a pin on. Makes no change
* if the folio is a zero_page.
*/
void folio_add_pin(struct folio *folio)
{
if (is_zero_folio(folio))
return;
/*
* Similar to try_grab_folio(): be sure to *also* increment the normal
* page refcount field at least once, so that the page really is
* pinned.
*/
if (folio_test_large(folio)) {
WARN_ON_ONCE(atomic_read(&folio->_pincount) < 1);
folio_ref_inc(folio);
atomic_inc(&folio->_pincount);
} else {
WARN_ON_ONCE(folio_ref_count(folio) < GUP_PIN_COUNTING_BIAS);
folio_ref_add(folio, GUP_PIN_COUNTING_BIAS);
}
}
static inline struct folio *gup_folio_range_next(struct page *start,
unsigned long npages, unsigned long i, unsigned int *ntails)
{
struct page *next = nth_page(start, i);
struct folio *folio = page_folio(next);
unsigned int nr = 1;
if (folio_test_large(folio))
nr = min_t(unsigned int, npages - i,
folio_nr_pages(folio) - folio_page_idx(folio, next));
*ntails = nr;
return folio;
}
static inline struct folio *gup_folio_next(struct page **list,
unsigned long npages, unsigned long i, unsigned int *ntails)
{
struct folio *folio = page_folio(list[i]);
unsigned int nr;
for (nr = i + 1; nr < npages; nr++) {
if (page_folio(list[nr]) != folio)
break;
}
*ntails = nr - i;
return folio;
}
/**
* unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
* @pages: array of pages to be maybe marked dirty, and definitely released.
* @npages: number of pages in the @pages array.
* @make_dirty: whether to mark the pages dirty
*
* "gup-pinned page" refers to a page that has had one of the get_user_pages()
* variants called on that page.
*
* For each page in the @pages array, make that page (or its head page, if a
* compound page) dirty, if @make_dirty is true, and if the page was previously
* listed as clean. In any case, releases all pages using unpin_user_page(),
* possibly via unpin_user_pages(), for the non-dirty case.
*
* Please see the unpin_user_page() documentation for details.
*
* set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
* required, then the caller should a) verify that this is really correct,
* because _lock() is usually required, and b) hand code it:
* set_page_dirty_lock(), unpin_user_page().
*
*/
void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
bool make_dirty)
{
unsigned long i;
struct folio *folio;
unsigned int nr;
if (!make_dirty) {
unpin_user_pages(pages, npages);
return;
}
sanity_check_pinned_pages(pages, npages);
for (i = 0; i < npages; i += nr) {
folio = gup_folio_next(pages, npages, i, &nr);
/*
* Checking PageDirty at this point may race with
* clear_page_dirty_for_io(), but that's OK. Two key
* cases:
*
* 1) This code sees the page as already dirty, so it
* skips the call to set_page_dirty(). That could happen
* because clear_page_dirty_for_io() called
* page_mkclean(), followed by set_page_dirty().
* However, now the page is going to get written back,
* which meets the original intention of setting it
* dirty, so all is well: clear_page_dirty_for_io() goes
* on to call TestClearPageDirty(), and write the page
* back.
*
* 2) This code sees the page as clean, so it calls
* set_page_dirty(). The page stays dirty, despite being
* written back, so it gets written back again in the
* next writeback cycle. This is harmless.
*/
if (!folio_test_dirty(folio)) {
folio_lock(folio);
folio_mark_dirty(folio);
folio_unlock(folio);
}
gup_put_folio(folio, nr, FOLL_PIN);
}
}
EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
/**
* unpin_user_page_range_dirty_lock() - release and optionally dirty
* gup-pinned page range
*
* @page: the starting page of a range maybe marked dirty, and definitely released.
* @npages: number of consecutive pages to release.
* @make_dirty: whether to mark the pages dirty
*
* "gup-pinned page range" refers to a range of pages that has had one of the
* pin_user_pages() variants called on that page.
*
* For the page ranges defined by [page .. page+npages], make that range (or
* its head pages, if a compound page) dirty, if @make_dirty is true, and if the
* page range was previously listed as clean.
*
* set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
* required, then the caller should a) verify that this is really correct,
* because _lock() is usually required, and b) hand code it:
* set_page_dirty_lock(), unpin_user_page().
*
*/
void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
bool make_dirty)
{
unsigned long i;
struct folio *folio;
unsigned int nr;
for (i = 0; i < npages; i += nr) {
folio = gup_folio_range_next(page, npages, i, &nr);
if (make_dirty && !folio_test_dirty(folio)) {
folio_lock(folio);
folio_mark_dirty(folio);
folio_unlock(folio);
}
gup_put_folio(folio, nr, FOLL_PIN);
}
}
EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
static void unpin_user_pages_lockless(struct page **pages, unsigned long npages)
{
unsigned long i;
struct folio *folio;
unsigned int nr;
/*
* Don't perform any sanity checks because we might have raced with
* fork() and some anonymous pages might now actually be shared --
* which is why we're unpinning after all.
*/
for (i = 0; i < npages; i += nr) {
folio = gup_folio_next(pages, npages, i, &nr);
gup_put_folio(folio, nr, FOLL_PIN);
}
}
/**
* unpin_user_pages() - release an array of gup-pinned pages.
* @pages: array of pages to be marked dirty and released.
* @npages: number of pages in the @pages array.
*
* For each page in the @pages array, release the page using unpin_user_page().
*
* Please see the unpin_user_page() documentation for details.
*/
void unpin_user_pages(struct page **pages, unsigned long npages)
{
unsigned long i;
struct folio *folio;
unsigned int nr;
/*
* If this WARN_ON() fires, then the system *might* be leaking pages (by
* leaving them pinned), but probably not. More likely, gup/pup returned
* a hard -ERRNO error to the caller, who erroneously passed it here.
*/
if (WARN_ON(IS_ERR_VALUE(npages)))
return;
sanity_check_pinned_pages(pages, npages);
for (i = 0; i < npages; i += nr) {
folio = gup_folio_next(pages, npages, i, &nr);
gup_put_folio(folio, nr, FOLL_PIN);
}
}
EXPORT_SYMBOL(unpin_user_pages);
/*
* Set the MMF_HAS_PINNED if not set yet; after set it'll be there for the mm's
* lifecycle. Avoid setting the bit unless necessary, or it might cause write
* cache bouncing on large SMP machines for concurrent pinned gups.
*/
static inline void mm_set_has_pinned_flag(unsigned long *mm_flags)
{
if (!test_bit(MMF_HAS_PINNED, mm_flags))
set_bit(MMF_HAS_PINNED, mm_flags);
}
#ifdef CONFIG_MMU
static struct page *no_page_table(struct vm_area_struct *vma,
unsigned int flags)
{
/*
* When core dumping an enormous anonymous area that nobody
* has touched so far, we don't want to allocate unnecessary pages or
* page tables. Return error instead of NULL to skip handle_mm_fault,
* then get_dump_page() will return NULL to leave a hole in the dump.
* But we can only make this optimization where a hole would surely
* be zero-filled if handle_mm_fault() actually did handle it.
*/
if ((flags & FOLL_DUMP) &&
(vma_is_anonymous(vma) || !vma->vm_ops->fault))
return ERR_PTR(-EFAULT);
return NULL;
}
static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
pte_t *pte, unsigned int flags)
{
if (flags & FOLL_TOUCH) {
pte_t orig_entry = ptep_get(pte);
pte_t entry = orig_entry;
if (flags & FOLL_WRITE)
entry = pte_mkdirty(entry);
entry = pte_mkyoung(entry);
if (!pte_same(orig_entry, entry)) {
set_pte_at(vma->vm_mm, address, pte, entry);
update_mmu_cache(vma, address, pte);
}
}
/* Proper page table entry exists, but no corresponding struct page */
return -EEXIST;
}
/* FOLL_FORCE can write to even unwritable PTEs in COW mappings. */
static inline bool can_follow_write_pte(pte_t pte, struct page *page,
struct vm_area_struct *vma,
unsigned int flags)
{
/* If the pte is writable, we can write to the page. */
if (pte_write(pte))
return true;
/* Maybe FOLL_FORCE is set to override it? */
if (!(flags & FOLL_FORCE))
return false;
/* But FOLL_FORCE has no effect on shared mappings */
if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))
return false;
/* ... or read-only private ones */
if (!(vma->vm_flags & VM_MAYWRITE))
return false;
/* ... or already writable ones that just need to take a write fault */
if (vma->vm_flags & VM_WRITE)
return false;
/*
* See can_change_pte_writable(): we broke COW and could map the page
* writable if we have an exclusive anonymous page ...
*/
if (!page || !PageAnon(page) || !PageAnonExclusive(page))
return false;
/* ... and a write-fault isn't required for other reasons. */
if (vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte))
return false;
return !userfaultfd_pte_wp(vma, pte);
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags,
struct dev_pagemap **pgmap)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
spinlock_t *ptl;
pte_t *ptep, pte;
int ret;
/* FOLL_GET and FOLL_PIN are mutually exclusive. */
if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
(FOLL_PIN | FOLL_GET)))
return ERR_PTR(-EINVAL);
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
if (!ptep)
return no_page_table(vma, flags);
pte = ptep_get(ptep);
if (!pte_present(pte))
goto no_page;
if (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags))
goto no_page;
page = vm_normal_page(vma, address, pte);
/*
* We only care about anon pages in can_follow_write_pte() and don't
* have to worry about pte_devmap() because they are never anon.
*/
if ((flags & FOLL_WRITE) &&
!can_follow_write_pte(pte, page, vma, flags)) {
page = NULL;
goto out;
}
if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
/*
* Only return device mapping pages in the FOLL_GET or FOLL_PIN
* case since they are only valid while holding the pgmap
* reference.
*/
*pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
if (*pgmap)
page = pte_page(pte);
else
goto no_page;
} else if (unlikely(!page)) {
if (flags & FOLL_DUMP) {
/* Avoid special (like zero) pages in core dumps */
page = ERR_PTR(-EFAULT);
goto out;
}
if (is_zero_pfn(pte_pfn(pte))) {
page = pte_page(pte);
} else {
ret = follow_pfn_pte(vma, address, ptep, flags);
page = ERR_PTR(ret);
goto out;
}
}
if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) {
page = ERR_PTR(-EMLINK);
goto out;
}
VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
!PageAnonExclusive(page), page);
/* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
ret = try_grab_page(page, flags);
if (unlikely(ret)) {
page = ERR_PTR(ret);
goto out;
}
/*
* We need to make the page accessible if and only if we are going
* to access its content (the FOLL_PIN case). Please see
* Documentation/core-api/pin_user_pages.rst for details.
*/
if (flags & FOLL_PIN) {
ret = arch_make_page_accessible(page);
if (ret) {
unpin_user_page(page);
page = ERR_PTR(ret);
goto out;
}
}
if (flags & FOLL_TOUCH) {
if ((flags & FOLL_WRITE) &&
!pte_dirty(pte) && !PageDirty(page))
set_page_dirty(page);
/*
* pte_mkyoung() would be more correct here, but atomic care
* is needed to avoid losing the dirty bit: it is easier to use
* mark_page_accessed().
*/
mark_page_accessed(page);
}
out:
pte_unmap_unlock(ptep, ptl);
return page;
no_page:
pte_unmap_unlock(ptep, ptl);
if (!pte_none(pte))
return NULL;
return no_page_table(vma, flags);
}
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
unsigned long address, pud_t *pudp,
unsigned int flags,
struct follow_page_context *ctx)
{
pmd_t *pmd, pmdval;
spinlock_t *ptl;
struct page *page;
struct mm_struct *mm = vma->vm_mm;
pmd = pmd_offset(pudp, address);
pmdval = pmdp_get_lockless(pmd);
if (pmd_none(pmdval))
return no_page_table(vma, flags);
if (!pmd_present(pmdval))
return no_page_table(vma, flags);
if (pmd_devmap(pmdval)) {
ptl = pmd_lock(mm, pmd);
page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
spin_unlock(ptl);
if (page)
return page;
return no_page_table(vma, flags);
}
if (likely(!pmd_trans_huge(pmdval)))
return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags))
return no_page_table(vma, flags);
ptl = pmd_lock(mm, pmd);
if (unlikely(!pmd_present(*pmd))) {
spin_unlock(ptl);
return no_page_table(vma, flags);
}
if (unlikely(!pmd_trans_huge(*pmd))) {
spin_unlock(ptl);
return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
}
if (flags & FOLL_SPLIT_PMD) {
spin_unlock(ptl);
split_huge_pmd(vma, pmd, address);
/* If pmd was left empty, stuff a page table in there quickly */
return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) :
follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
}
page = follow_trans_huge_pmd(vma, address, pmd, flags);
spin_unlock(ptl);
ctx->page_mask = HPAGE_PMD_NR - 1;
return page;
}
static struct page *follow_pud_mask(struct vm_area_struct *vma,
unsigned long address, p4d_t *p4dp,
unsigned int flags,
struct follow_page_context *ctx)
{
pud_t *pud;
spinlock_t *ptl;
struct page *page;
struct mm_struct *mm = vma->vm_mm;
pud = pud_offset(p4dp, address);
if (pud_none(*pud))
return no_page_table(vma, flags);
if (pud_devmap(*pud)) {
ptl = pud_lock(mm, pud);
page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
spin_unlock(ptl);
if (page)
return page;
return no_page_table(vma, flags);
}
if (unlikely(pud_bad(*pud)))
return no_page_table(vma, flags);
return follow_pmd_mask(vma, address, pud, flags, ctx);
}
static struct page *follow_p4d_mask(struct vm_area_struct *vma,
unsigned long address, pgd_t *pgdp,
unsigned int flags,
struct follow_page_context *ctx)
{
p4d_t *p4d;
p4d = p4d_offset(pgdp, address);
if (p4d_none(*p4d))
return no_page_table(vma, flags);
BUILD_BUG_ON(p4d_huge(*p4d));
if (unlikely(p4d_bad(*p4d)))
return no_page_table(vma, flags);
return follow_pud_mask(vma, address, p4d, flags, ctx);
}
/**
* follow_page_mask - look up a page descriptor from a user-virtual address
* @vma: vm_area_struct mapping @address
* @address: virtual address to look up
* @flags: flags modifying lookup behaviour
* @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
* pointer to output page_mask
*
* @flags can have FOLL_ flags set, defined in <linux/mm.h>
*
* When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
* the device's dev_pagemap metadata to avoid repeating expensive lookups.
*
* When getting an anonymous page and the caller has to trigger unsharing
* of a shared anonymous page first, -EMLINK is returned. The caller should
* trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only
* relevant with FOLL_PIN and !FOLL_WRITE.
*
* On output, the @ctx->page_mask is set according to the size of the page.
*
* Return: the mapped (struct page *), %NULL if no mapping exists, or
* an error pointer if there is a mapping to something not represented
* by a page descriptor (see also vm_normal_page()).
*/
static struct page *follow_page_mask(struct vm_area_struct *vma,
unsigned long address, unsigned int flags,
struct follow_page_context *ctx)
{
pgd_t *pgd;
struct mm_struct *mm = vma->vm_mm;
ctx->page_mask = 0;
/*
* Call hugetlb_follow_page_mask for hugetlb vmas as it will use
* special hugetlb page table walking code. This eliminates the
* need to check for hugetlb entries in the general walking code.
*/
if (is_vm_hugetlb_page(vma))
return hugetlb_follow_page_mask(vma, address, flags,
&ctx->page_mask);
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
return no_page_table(vma, flags);
return follow_p4d_mask(vma, address, pgd, flags, ctx);
}
struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
unsigned int foll_flags)
{
struct follow_page_context ctx = { NULL };
struct page *page;
if (vma_is_secretmem(vma))
return NULL;
if (WARN_ON_ONCE(foll_flags & FOLL_PIN))
return NULL;
/*
* We never set FOLL_HONOR_NUMA_FAULT because callers don't expect
* to fail on PROT_NONE-mapped pages.
*/
page = follow_page_mask(vma, address, foll_flags, &ctx);
if (ctx.pgmap)
put_dev_pagemap(ctx.pgmap);
return page;
}
static int get_gate_page(struct mm_struct *mm, unsigned long address,
unsigned int gup_flags, struct vm_area_struct **vma,
struct page **page)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
pte_t entry;
int ret = -EFAULT;
/* user gate pages are read-only */
if (gup_flags & FOLL_WRITE)
return -EFAULT;
if (address > TASK_SIZE)
pgd = pgd_offset_k(address);
else
pgd = pgd_offset_gate(mm, address);
if (pgd_none(*pgd))
return -EFAULT;
p4d = p4d_offset(pgd, address);
if (p4d_none(*p4d))
return -EFAULT;
pud = pud_offset(p4d, address);
if (pud_none(*pud))
return -EFAULT;
pmd = pmd_offset(pud, address);
if (!pmd_present(*pmd))
return -EFAULT;
pte = pte_offset_map(pmd, address);
if (!pte)
return -EFAULT;
entry = ptep_get(pte);
if (pte_none(entry))
goto unmap;
*vma = get_gate_vma(mm);
if (!page)
goto out;
*page = vm_normal_page(*vma, address, entry);
if (!*page) {
if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(entry)))
goto unmap;
*page = pte_page(entry);
}
ret = try_grab_page(*page, gup_flags);
if (unlikely(ret))
goto unmap;
out:
ret = 0;
unmap:
pte_unmap(pte);
return ret;
}
/*
* mmap_lock must be held on entry. If @flags has FOLL_UNLOCKABLE but not
* FOLL_NOWAIT, the mmap_lock may be released. If it is, *@locked will be set
* to 0 and -EBUSY returned.
*/
static int faultin_page(struct vm_area_struct *vma,
unsigned long address, unsigned int *flags, bool unshare,
int *locked)
{
unsigned int fault_flags = 0;
vm_fault_t ret;
if (*flags & FOLL_NOFAULT)
return -EFAULT;
if (*flags & FOLL_WRITE)
fault_flags |= FAULT_FLAG_WRITE;
if (*flags & FOLL_REMOTE)
fault_flags |= FAULT_FLAG_REMOTE;
if (*flags & FOLL_UNLOCKABLE) {
fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
/*
* FAULT_FLAG_INTERRUPTIBLE is opt-in. GUP callers must set
* FOLL_INTERRUPTIBLE to enable FAULT_FLAG_INTERRUPTIBLE.
* That's because some callers may not be prepared to
* handle early exits caused by non-fatal signals.
*/
if (*flags & FOLL_INTERRUPTIBLE)
fault_flags |= FAULT_FLAG_INTERRUPTIBLE;
}
if (*flags & FOLL_NOWAIT)
fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
if (*flags & FOLL_TRIED) {
/*
* Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
* can co-exist
*/
fault_flags |= FAULT_FLAG_TRIED;
}
if (unshare) {
fault_flags |= FAULT_FLAG_UNSHARE;
/* FAULT_FLAG_WRITE and FAULT_FLAG_UNSHARE are incompatible */
VM_BUG_ON(fault_flags & FAULT_FLAG_WRITE);
}
ret = handle_mm_fault(vma, address, fault_flags, NULL);
if (ret & VM_FAULT_COMPLETED) {
/*
* With FAULT_FLAG_RETRY_NOWAIT we'll never release the
* mmap lock in the page fault handler. Sanity check this.
*/
WARN_ON_ONCE(fault_flags & FAULT_FLAG_RETRY_NOWAIT);
*locked = 0;
/*
* We should do the same as VM_FAULT_RETRY, but let's not
* return -EBUSY since that's not reflecting the reality of
* what has happened - we've just fully completed a page
* fault, with the mmap lock released. Use -EAGAIN to show
* that we want to take the mmap lock _again_.
*/
return -EAGAIN;
}
if (ret & VM_FAULT_ERROR) {
int err = vm_fault_to_errno(ret, *flags);
if (err)
return err;
BUG();
}
if (ret & VM_FAULT_RETRY) {
if (!(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
*locked = 0;
return -EBUSY;
}
return 0;
}
/*
* Writing to file-backed mappings which require folio dirty tracking using GUP
* is a fundamentally broken operation, as kernel write access to GUP mappings
* do not adhere to the semantics expected by a file system.
*
* Consider the following scenario:-
*
* 1. A folio is written to via GUP which write-faults the memory, notifying
* the file system and dirtying the folio.
* 2. Later, writeback is triggered, resulting in the folio being cleaned and
* the PTE being marked read-only.
* 3. The GUP caller writes to the folio, as it is mapped read/write via the
* direct mapping.
* 4. The GUP caller, now done with the page, unpins it and sets it dirty
* (though it does not have to).
*
* This results in both data being written to a folio without writenotify, and
* the folio being dirtied unexpectedly (if the caller decides to do so).
*/
static bool writable_file_mapping_allowed(struct vm_area_struct *vma,
unsigned long gup_flags)
{
/*
* If we aren't pinning then no problematic write can occur. A long term
* pin is the most egregious case so this is the case we disallow.
*/
if ((gup_flags & (FOLL_PIN | FOLL_LONGTERM)) !=
(FOLL_PIN | FOLL_LONGTERM))
return true;
/*
* If the VMA does not require dirty tracking then no problematic write
* can occur either.
*/
return !vma_needs_dirty_tracking(vma);
}
static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
{
vm_flags_t vm_flags = vma->vm_flags;
int write = (gup_flags & FOLL_WRITE);
int foreign = (gup_flags & FOLL_REMOTE);
bool vma_anon = vma_is_anonymous(vma);
if (vm_flags & (VM_IO | VM_PFNMAP))
return -EFAULT;
if ((gup_flags & FOLL_ANON) && !vma_anon)
return -EFAULT;
if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma))
return -EOPNOTSUPP;
if (vma_is_secretmem(vma))
return -EFAULT;
if (write) {
if (!vma_anon &&
!writable_file_mapping_allowed(vma, gup_flags))
return -EFAULT;
if (!(vm_flags & VM_WRITE) || (vm_flags & VM_SHADOW_STACK)) {
if (!(gup_flags & FOLL_FORCE))
return -EFAULT;
/* hugetlb does not support FOLL_FORCE|FOLL_WRITE. */
if (is_vm_hugetlb_page(vma))
return -EFAULT;
/*
* We used to let the write,force case do COW in a
* VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
* set a breakpoint in a read-only mapping of an
* executable, without corrupting the file (yet only
* when that file had been opened for writing!).
* Anon pages in shared mappings are surprising: now
* just reject it.
*/
if (!is_cow_mapping(vm_flags))
return -EFAULT;
}
} else if (!(vm_flags & VM_READ)) {
if (!(gup_flags & FOLL_FORCE))
return -EFAULT;
/*
* Is there actually any vma we can reach here which does not
* have VM_MAYREAD set?
*/
if (!(vm_flags & VM_MAYREAD))
return -EFAULT;
}
/*
* gups are always data accesses, not instruction
* fetches, so execute=false here
*/
if (!arch_vma_access_permitted(vma, write, false, foreign))
return -EFAULT;
return 0;
}
/*
* This is "vma_lookup()", but with a warning if we would have
* historically expanded the stack in the GUP code.
*/
static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm,
unsigned long addr)
{
#ifdef CONFIG_STACK_GROWSUP
return vma_lookup(mm, addr);
#else
static volatile unsigned long next_warn;
struct vm_area_struct *vma;
unsigned long now, next;
vma = find_vma(mm, addr);
if (!vma || (addr >= vma->vm_start))
return vma;
/* Only warn for half-way relevant accesses */
if (!(vma->vm_flags & VM_GROWSDOWN))
return NULL;
if (vma->vm_start - addr > 65536)
return NULL;
/* Let's not warn more than once an hour.. */
now = jiffies; next = next_warn;
if (next && time_before(now, next))
return NULL;
next_warn = now + 60*60*HZ;
/* Let people know things may have changed. */
pr_warn("GUP no longer grows the stack in %s (%d): %lx-%lx (%lx)\n",
current->comm, task_pid_nr(current),
vma->vm_start, vma->vm_end, addr);
dump_stack();
return NULL;
#endif
}
/**
* __get_user_pages() - pin user pages in memory
* @mm: mm_struct of target mm
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying pin behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long. Or NULL, if caller
* only intends to ensure the pages are faulted in.
* @locked: whether we're still with the mmap_lock held
*
* Returns either number of pages pinned (which may be less than the
* number requested), or an error. Details about the return value:
*
* -- If nr_pages is 0, returns 0.
* -- If nr_pages is >0, but no pages were pinned, returns -errno.
* -- If nr_pages is >0, and some pages were pinned, returns the number of
* pages pinned. Again, this may be less than nr_pages.
* -- 0 return value is possible when the fault would need to be retried.
*
* The caller is responsible for releasing returned @pages, via put_page().
*
* Must be called with mmap_lock held. It may be released. See below.
*
* __get_user_pages walks a process's page tables and takes a reference to
* each struct page that each user address corresponds to at a given
* instant. That is, it takes the page that would be accessed if a user
* thread accesses the given user virtual address at that instant.
*
* This does not guarantee that the page exists in the user mappings when
* __get_user_pages returns, and there may even be a completely different
* page there in some cases (eg. if mmapped pagecache has been invalidated
* and subsequently re-faulted). However it does guarantee that the page
* won't be freed completely. And mostly callers simply care that the page
* contains data that was valid *at some point in time*. Typically, an IO
* or similar operation cannot guarantee anything stronger anyway because
* locks can't be held over the syscall boundary.
*
* If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
* the page is written to, set_page_dirty (or set_page_dirty_lock, as
* appropriate) must be called after the page is finished with, and
* before put_page is called.
*
* If FOLL_UNLOCKABLE is set without FOLL_NOWAIT then the mmap_lock may
* be released. If this happens *@locked will be set to 0 on return.
*
* A caller using such a combination of @gup_flags must therefore hold the
* mmap_lock for reading only, and recognize when it's been released. Otherwise,
* it must be held for either reading or writing and will not be released.
*
* In most cases, get_user_pages or get_user_pages_fast should be used
* instead of __get_user_pages. __get_user_pages should be used only if
* you need some special @gup_flags.
*/
static long __get_user_pages(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
int *locked)
{
long ret = 0, i = 0;
struct vm_area_struct *vma = NULL;
struct follow_page_context ctx = { NULL };
if (!nr_pages)
return 0;
start = untagged_addr_remote(mm, start);
VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
do {
struct page *page;
unsigned int foll_flags = gup_flags;
unsigned int page_increm;
/* first iteration or cross vma bound */
if (!vma || start >= vma->vm_end) {
/*
* MADV_POPULATE_(READ|WRITE) wants to handle VMA
* lookups+error reporting differently.
*/
if (gup_flags & FOLL_MADV_POPULATE) {
vma = vma_lookup(mm, start);
if (!vma) {
ret = -ENOMEM;
goto out;
}
if (check_vma_flags(vma, gup_flags)) {
ret = -EINVAL;
goto out;
}
goto retry;
}
vma = gup_vma_lookup(mm, start);
if (!vma && in_gate_area(mm, start)) {
ret = get_gate_page(mm, start & PAGE_MASK,
gup_flags, &vma,
pages ? &page : NULL);
if (ret)
goto out;
ctx.page_mask = 0;
goto next_page;
}
if (!vma) {
ret = -EFAULT;
goto out;
}
ret = check_vma_flags(vma, gup_flags);
if (ret)
goto out;
}
retry:
/*
* If we have a pending SIGKILL, don't keep faulting pages and
* potentially allocating memory.
*/
if (fatal_signal_pending(current)) {
ret = -EINTR;
goto out;
}
cond_resched();
page = follow_page_mask(vma, start, foll_flags, &ctx);
if (!page || PTR_ERR(page) == -EMLINK) {
ret = faultin_page(vma, start, &foll_flags,
PTR_ERR(page) == -EMLINK, locked);
switch (ret) {
case 0:
goto retry;
case -EBUSY:
case -EAGAIN:
ret = 0;
fallthrough;
case -EFAULT:
case -ENOMEM:
case -EHWPOISON:
goto out;
}
BUG();
} else if (PTR_ERR(page) == -EEXIST) {
/*
* Proper page table entry exists, but no corresponding
* struct page. If the caller expects **pages to be
* filled in, bail out now, because that can't be done
* for this page.
*/
if (pages) {
ret = PTR_ERR(page);
goto out;
}
} else if (IS_ERR(page)) {
ret = PTR_ERR(page);
goto out;
}
next_page:
page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
if (page_increm > nr_pages)
page_increm = nr_pages;
if (pages) {
struct page *subpage;
unsigned int j;
/*
* This must be a large folio (and doesn't need to
* be the whole folio; it can be part of it), do
* the refcount work for all the subpages too.
*
* NOTE: here the page may not be the head page
* e.g. when start addr is not thp-size aligned.
* try_grab_folio() should have taken care of tail
* pages.
*/
if (page_increm > 1) {
struct folio *folio;
/*
* Since we already hold refcount on the
* large folio, this should never fail.
*/
folio = try_grab_folio(page, page_increm - 1,
foll_flags);
if (WARN_ON_ONCE(!folio)) {
/*
* Release the 1st page ref if the
* folio is problematic, fail hard.
*/
gup_put_folio(page_folio(page), 1,
foll_flags);
ret = -EFAULT;
goto out;
}
}
for (j = 0; j < page_increm; j++) {
subpage = nth_page(page, j);
pages[i + j] = subpage;
flush_anon_page(vma, subpage, start + j * PAGE_SIZE);
flush_dcache_page(subpage);
}
}
i += page_increm;
start += page_increm * PAGE_SIZE;
nr_pages -= page_increm;
} while (nr_pages);
out:
if (ctx.pgmap)
put_dev_pagemap(ctx.pgmap);
return i ? i : ret;
}
static bool vma_permits_fault(struct vm_area_struct *vma,
unsigned int fault_flags)
{
bool write = !!(fault_flags & FAULT_FLAG_WRITE);
bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
if (!(vm_flags & vma->vm_flags))
return false;
/*
* The architecture might have a hardware protection
* mechanism other than read/write that can deny access.
*
* gup always represents data access, not instruction
* fetches, so execute=false here:
*/
if (!arch_vma_access_permitted(vma, write, false, foreign))
return false;
return true;
}
/**
* fixup_user_fault() - manually resolve a user page fault
* @mm: mm_struct of target mm
* @address: user address
* @fault_flags:flags to pass down to handle_mm_fault()
* @unlocked: did we unlock the mmap_lock while retrying, maybe NULL if caller
* does not allow retry. If NULL, the caller must guarantee
* that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY.
*
* This is meant to be called in the specific scenario where for locking reasons
* we try to access user memory in atomic context (within a pagefault_disable()
* section), this returns -EFAULT, and we want to resolve the user fault before
* trying again.
*
* Typically this is meant to be used by the futex code.
*
* The main difference with get_user_pages() is that this function will
* unconditionally call handle_mm_fault() which will in turn perform all the
* necessary SW fixup of the dirty and young bits in the PTE, while
* get_user_pages() only guarantees to update these in the struct page.
*
* This is important for some architectures where those bits also gate the
* access permission to the page because they are maintained in software. On
* such architectures, gup() will not be enough to make a subsequent access
* succeed.
*
* This function will not return with an unlocked mmap_lock. So it has not the
* same semantics wrt the @mm->mmap_lock as does filemap_fault().
*/
int fixup_user_fault(struct mm_struct *mm,
unsigned long address, unsigned int fault_flags,
bool *unlocked)
{
struct vm_area_struct *vma;
vm_fault_t ret;
address = untagged_addr_remote(mm, address);
if (unlocked)
fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
retry:
vma = gup_vma_lookup(mm, address);
if (!vma)
return -EFAULT;
if (!vma_permits_fault(vma, fault_flags))
return -EFAULT;
if ((fault_flags & FAULT_FLAG_KILLABLE) &&
fatal_signal_pending(current))
return -EINTR;
ret = handle_mm_fault(vma, address, fault_flags, NULL);
if (ret & VM_FAULT_COMPLETED) {
/*
* NOTE: it's a pity that we need to retake the lock here
* to pair with the unlock() in the callers. Ideally we
* could tell the callers so they do not need to unlock.
*/
mmap_read_lock(mm);
*unlocked = true;
return 0;
}
if (ret & VM_FAULT_ERROR) {
int err = vm_fault_to_errno(ret, 0);
if (err)
return err;
BUG();
}
if (ret & VM_FAULT_RETRY) {
mmap_read_lock(mm);
*unlocked = true;
fault_flags |= FAULT_FLAG_TRIED;
goto retry;
}
return 0;
}
EXPORT_SYMBOL_GPL(fixup_user_fault);
/*
* GUP always responds to fatal signals. When FOLL_INTERRUPTIBLE is
* specified, it'll also respond to generic signals. The caller of GUP
* that has FOLL_INTERRUPTIBLE should take care of the GUP interruption.
*/
static bool gup_signal_pending(unsigned int flags)
{
if (fatal_signal_pending(current))
return true;
if (!(flags & FOLL_INTERRUPTIBLE))
return false;
return signal_pending(current);
}
/*
* Locking: (*locked == 1) means that the mmap_lock has already been acquired by
* the caller. This function may drop the mmap_lock. If it does so, then it will
* set (*locked = 0).
*
* (*locked == 0) means that the caller expects this function to acquire and
* drop the mmap_lock. Therefore, the value of *locked will still be zero when
* the function returns, even though it may have changed temporarily during
* function execution.
*
* Please note that this function, unlike __get_user_pages(), will not return 0
* for nr_pages > 0, unless FOLL_NOWAIT is used.
*/
static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
unsigned long start,
unsigned long nr_pages,
struct page **pages,
int *locked,
unsigned int flags)
{
long ret, pages_done;
bool must_unlock = false;
if (!nr_pages)
return 0;
/*
* The internal caller expects GUP to manage the lock internally and the
* lock must be released when this returns.
*/
if (!*locked) {
if (mmap_read_lock_killable(mm))
return -EAGAIN;
must_unlock = true;
*locked = 1;
}
else
mmap_assert_locked(mm);
if (flags & FOLL_PIN)
mm_set_has_pinned_flag(&mm->flags);
/*
* FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
* is to set FOLL_GET if the caller wants pages[] filled in (but has
* carelessly failed to specify FOLL_GET), so keep doing that, but only
* for FOLL_GET, not for the newer FOLL_PIN.
*
* FOLL_PIN always expects pages to be non-null, but no need to assert
* that here, as any failures will be obvious enough.
*/
if (pages && !(flags & FOLL_PIN))
flags |= FOLL_GET;
pages_done = 0;
for (;;) {
ret = __get_user_pages(mm, start, nr_pages, flags, pages,
locked);
if (!(flags & FOLL_UNLOCKABLE)) {
/* VM_FAULT_RETRY couldn't trigger, bypass */
pages_done = ret;
break;
}
/* VM_FAULT_RETRY or VM_FAULT_COMPLETED cannot return errors */
if (!*locked) {
BUG_ON(ret < 0);
BUG_ON(ret >= nr_pages);
}
if (ret > 0) {
nr_pages -= ret;
pages_done += ret;
if (!nr_pages)
break;
}
if (*locked) {
/*
* VM_FAULT_RETRY didn't trigger or it was a
* FOLL_NOWAIT.
*/
if (!pages_done)
pages_done = ret;
break;
}
/*
* VM_FAULT_RETRY triggered, so seek to the faulting offset.
* For the prefault case (!pages) we only update counts.
*/
if (likely(pages))
pages += ret;
start += ret << PAGE_SHIFT;
/* The lock was temporarily dropped, so we must unlock later */
must_unlock = true;
retry:
/*
* Repeat on the address that fired VM_FAULT_RETRY
* with both FAULT_FLAG_ALLOW_RETRY and
* FAULT_FLAG_TRIED. Note that GUP can be interrupted
* by fatal signals of even common signals, depending on
* the caller's request. So we need to check it before we
* start trying again otherwise it can loop forever.
*/
if (gup_signal_pending(flags)) {
if (!pages_done)
pages_done = -EINTR;
break;
}
ret = mmap_read_lock_killable(mm);
if (ret) {
BUG_ON(ret > 0);
if (!pages_done)
pages_done = ret;
break;
}
*locked = 1;
ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
pages, locked);
if (!*locked) {
/* Continue to retry until we succeeded */
BUG_ON(ret != 0);
goto retry;
}
if (ret != 1) {
BUG_ON(ret > 1);
if (!pages_done)
pages_done = ret;
break;
}
nr_pages--;
pages_done++;
if (!nr_pages)
break;
if (likely(pages))
pages++;
start += PAGE_SIZE;
}
if (must_unlock && *locked) {
/*
* We either temporarily dropped the lock, or the caller
* requested that we both acquire and drop the lock. Either way,
* we must now unlock, and notify the caller of that state.
*/
mmap_read_unlock(mm);
*locked = 0;
}
/*
* Failing to pin anything implies something has gone wrong (except when
* FOLL_NOWAIT is specified).
*/
if (WARN_ON_ONCE(pages_done == 0 && !(flags & FOLL_NOWAIT)))
return -EFAULT;
return pages_done;
}
/**
* populate_vma_page_range() - populate a range of pages in the vma.
* @vma: target vma
* @start: start address
* @end: end address
* @locked: whether the mmap_lock is still held
*
* This takes care of mlocking the pages too if VM_LOCKED is set.
*
* Return either number of pages pinned in the vma, or a negative error
* code on error.
*
* vma->vm_mm->mmap_lock must be held.
*
* If @locked is NULL, it may be held for read or write and will
* be unperturbed.
*
* If @locked is non-NULL, it must held for read only and may be
* released. If it's released, *@locked will be set to 0.
*/
long populate_vma_page_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end, int *locked)
{
struct mm_struct *mm = vma->vm_mm;
unsigned long nr_pages = (end - start) / PAGE_SIZE;
int local_locked = 1;
int gup_flags;
long ret;
VM_BUG_ON(!PAGE_ALIGNED(start));
VM_BUG_ON(!PAGE_ALIGNED(end));
VM_BUG_ON_VMA(start < vma->vm_start, vma);
VM_BUG_ON_VMA(end > vma->vm_end, vma);
mmap_assert_locked(mm);
/*
* Rightly or wrongly, the VM_LOCKONFAULT case has never used
* faultin_page() to break COW, so it has no work to do here.
*/
if (vma->vm_flags & VM_LOCKONFAULT)
return nr_pages;
/* ... similarly, we've never faulted in PROT_NONE pages */
if (!vma_is_accessible(vma))
return -EFAULT;
gup_flags = FOLL_TOUCH;
/*
* We want to touch writable mappings with a write fault in order
* to break COW, except for shared mappings because these don't COW
* and we would not want to dirty them for nothing.
*
* Otherwise, do a read fault, and use FOLL_FORCE in case it's not
* readable (ie write-only or executable).
*/
if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
gup_flags |= FOLL_WRITE;
else
gup_flags |= FOLL_FORCE;
if (locked)
gup_flags |= FOLL_UNLOCKABLE;
/*
* We made sure addr is within a VMA, so the following will
* not result in a stack expansion that recurses back here.
*/
ret = __get_user_pages(mm, start, nr_pages, gup_flags,
NULL, locked ? locked : &local_locked);
lru_add_drain();
return ret;
}
/*
* faultin_page_range() - populate (prefault) page tables inside the
* given range readable/writable
*
* This takes care of mlocking the pages, too, if VM_LOCKED is set.
*
* @mm: the mm to populate page tables in
* @start: start address
* @end: end address
* @write: whether to prefault readable or writable
* @locked: whether the mmap_lock is still held
*
* Returns either number of processed pages in the MM, or a negative error
* code on error (see __get_user_pages()). Note that this function reports
* errors related to VMAs, such as incompatible mappings, as expected by
* MADV_POPULATE_(READ|WRITE).
*
* The range must be page-aligned.
*
* mm->mmap_lock must be held. If it's released, *@locked will be set to 0.
*/
long faultin_page_range(struct mm_struct *mm, unsigned long start,
unsigned long end, bool write, int *locked)
{
unsigned long nr_pages = (end - start) / PAGE_SIZE;
int gup_flags;
long ret;
VM_BUG_ON(!PAGE_ALIGNED(start));
VM_BUG_ON(!PAGE_ALIGNED(end));
mmap_assert_locked(mm);
/*
* FOLL_TOUCH: Mark page accessed and thereby young; will also mark
* the page dirty with FOLL_WRITE -- which doesn't make a
* difference with !FOLL_FORCE, because the page is writable
* in the page table.
* FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit
* a poisoned page.
* !FOLL_FORCE: Require proper access permissions.
*/
gup_flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_UNLOCKABLE |
FOLL_MADV_POPULATE;
if (write)
gup_flags |= FOLL_WRITE;
ret = __get_user_pages_locked(mm, start, nr_pages, NULL, locked,
gup_flags);
lru_add_drain();
return ret;
}
/*
* __mm_populate - populate and/or mlock pages within a range of address space.
*
* This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
* flags. VMAs must be already marked with the desired vm_flags, and
* mmap_lock must not be held.
*/
int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
{
struct mm_struct *mm = current->mm;
unsigned long end, nstart, nend;
struct vm_area_struct *vma = NULL;
int locked = 0;
long ret = 0;
end = start + len;
for (nstart = start; nstart < end; nstart = nend) {
/*
* We want to fault in pages for [nstart; end) address range.
* Find first corresponding VMA.
*/
if (!locked) {
locked = 1;
mmap_read_lock(mm);
vma = find_vma_intersection(mm, nstart, end);
} else if (nstart >= vma->vm_end)
vma = find_vma_intersection(mm, vma->vm_end, end);
if (!vma)
break;
/*
* Set [nstart; nend) to intersection of desired address
* range with the first VMA. Also, skip undesirable VMA types.
*/
nend = min(end, vma->vm_end);
if (vma->vm_flags & (VM_IO | VM_PFNMAP))
continue;
if (nstart < vma->vm_start)
nstart = vma->vm_start;
/*
* Now fault in a range of pages. populate_vma_page_range()
* double checks the vma flags, so that it won't mlock pages
* if the vma was already munlocked.
*/
ret = populate_vma_page_range(vma, nstart, nend, &locked);
if (ret < 0) {
if (ignore_errors) {
ret = 0;
continue; /* continue at next VMA */
}
break;
}
nend = nstart + ret * PAGE_SIZE;
ret = 0;
}
if (locked)
mmap_read_unlock(mm);
return ret; /* 0 or negative error code */
}
#else /* CONFIG_MMU */
static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
unsigned long nr_pages, struct page **pages,
int *locked, unsigned int foll_flags)
{
struct vm_area_struct *vma;
bool must_unlock = false;
unsigned long vm_flags;
long i;
if (!nr_pages)
return 0;
/*
* The internal caller expects GUP to manage the lock internally and the
* lock must be released when this returns.
*/
if (!*locked) {
if (mmap_read_lock_killable(mm))
return -EAGAIN;
must_unlock = true;
*locked = 1;
}
/* calculate required read or write permissions.
* If FOLL_FORCE is set, we only require the "MAY" flags.
*/
vm_flags = (foll_flags & FOLL_WRITE) ?
(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
vm_flags &= (foll_flags & FOLL_FORCE) ?
(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
for (i = 0; i < nr_pages; i++) {
vma = find_vma(mm, start);
if (!vma)
break;
/* protect what we can, including chardevs */
if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
!(vm_flags & vma->vm_flags))
break;
if (pages) {
pages[i] = virt_to_page((void *)start);
if (pages[i])
get_page(pages[i]);
}
start = (start + PAGE_SIZE) & PAGE_MASK;
}
if (must_unlock && *locked) {
mmap_read_unlock(mm);
*locked = 0;
}
return i ? : -EFAULT;
}
#endif /* !CONFIG_MMU */
/**
* fault_in_writeable - fault in userspace address range for writing
* @uaddr: start of address range
* @size: size of address range
*
* Returns the number of bytes not faulted in (like copy_to_user() and
* copy_from_user()).
*/
size_t fault_in_writeable(char __user *uaddr, size_t size)
{
char __user *start = uaddr, *end;
if (unlikely(size == 0))
return 0;
if (!user_write_access_begin(uaddr, size))
return size;
if (!PAGE_ALIGNED(uaddr)) {
unsafe_put_user(0, uaddr, out);
uaddr = (char __user *)PAGE_ALIGN((unsigned long)uaddr);
}
end = (char __user *)PAGE_ALIGN((unsigned long)start + size);
if (unlikely(end < start))
end = NULL;
while (uaddr != end) {
unsafe_put_user(0, uaddr, out);
uaddr += PAGE_SIZE;
}
out:
user_write_access_end();
if (size > uaddr - start)
return size - (uaddr - start);
return 0;
}
EXPORT_SYMBOL(fault_in_writeable);
/**
* fault_in_subpage_writeable - fault in an address range for writing
* @uaddr: start of address range
* @size: size of address range
*
* Fault in a user address range for writing while checking for permissions at
* sub-page granularity (e.g. arm64 MTE). This function should be used when
* the caller cannot guarantee forward progress of a copy_to_user() loop.
*
* Returns the number of bytes not faulted in (like copy_to_user() and
* copy_from_user()).
*/
size_t fault_in_subpage_writeable(char __user *uaddr, size_t size)
{
size_t faulted_in;
/*
* Attempt faulting in at page granularity first for page table
* permission checking. The arch-specific probe_subpage_writeable()
* functions may not check for this.
*/
faulted_in = size - fault_in_writeable(uaddr, size);
if (faulted_in)
faulted_in -= probe_subpage_writeable(uaddr, faulted_in);
return size - faulted_in;
}
EXPORT_SYMBOL(fault_in_subpage_writeable);
/*
* fault_in_safe_writeable - fault in an address range for writing
* @uaddr: start of address range
* @size: length of address range
*
* Faults in an address range for writing. This is primarily useful when we
* already know that some or all of the pages in the address range aren't in
* memory.
*
* Unlike fault_in_writeable(), this function is non-destructive.
*
* Note that we don't pin or otherwise hold the pages referenced that we fault
* in. There's no guarantee that they'll stay in memory for any duration of
* time.
*
* Returns the number of bytes not faulted in, like copy_to_user() and
* copy_from_user().
*/
size_t fault_in_safe_writeable(const char __user *uaddr, size_t size)
{
unsigned long start = (unsigned long)uaddr, end;
struct mm_struct *mm = current->mm;
bool unlocked = false;
if (unlikely(size == 0))
return 0;
end = PAGE_ALIGN(start + size);
if (end < start)
end = 0;
mmap_read_lock(mm);
do {
if (fixup_user_fault(mm, start, FAULT_FLAG_WRITE, &unlocked))
break;
start = (start + PAGE_SIZE) & PAGE_MASK;
} while (start != end);
mmap_read_unlock(mm);
if (size > (unsigned long)uaddr - start)
return size - ((unsigned long)uaddr - start);
return 0;
}
EXPORT_SYMBOL(fault_in_safe_writeable);
/**
* fault_in_readable - fault in userspace address range for reading
* @uaddr: start of user address range
* @size: size of user address range
*
* Returns the number of bytes not faulted in (like copy_to_user() and
* copy_from_user()).
*/
size_t fault_in_readable(const char __user *uaddr, size_t size)
{
const char __user *start = uaddr, *end;
volatile char c;
if (unlikely(size == 0))
return 0; if (!user_read_access_begin(uaddr, size))
return size;
if (!PAGE_ALIGNED(uaddr)) {
unsafe_get_user(c, uaddr, out);
uaddr = (const char __user *)PAGE_ALIGN((unsigned long)uaddr);
}
end = (const char __user *)PAGE_ALIGN((unsigned long)start + size);
if (unlikely(end < start))
end = NULL;
while (uaddr != end) { unsafe_get_user(c, uaddr, out);
uaddr += PAGE_SIZE;
}
out:
user_read_access_end();
(void)c;
if (size > uaddr - start) return size - (uaddr - start);
return 0;
}
EXPORT_SYMBOL(fault_in_readable);
/**
* get_dump_page() - pin user page in memory while writing it to core dump
* @addr: user address
*
* Returns struct page pointer of user page pinned for dump,
* to be freed afterwards by put_page().
*
* Returns NULL on any kind of failure - a hole must then be inserted into
* the corefile, to preserve alignment with its headers; and also returns
* NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
* allowing a hole to be left in the corefile to save disk space.
*
* Called without mmap_lock (takes and releases the mmap_lock by itself).
*/
#ifdef CONFIG_ELF_CORE
struct page *get_dump_page(unsigned long addr)
{
struct page *page;
int locked = 0;
int ret;
ret = __get_user_pages_locked(current->mm, addr, 1, &page, &locked,
FOLL_FORCE | FOLL_DUMP | FOLL_GET);
return (ret == 1) ? page : NULL;
}
#endif /* CONFIG_ELF_CORE */
#ifdef CONFIG_MIGRATION
/*
* Returns the number of collected pages. Return value is always >= 0.
*/
static unsigned long collect_longterm_unpinnable_pages(
struct list_head *movable_page_list,
unsigned long nr_pages,
struct page **pages)
{
unsigned long i, collected = 0;
struct folio *prev_folio = NULL;
bool drain_allow = true;
for (i = 0; i < nr_pages; i++) {
struct folio *folio = page_folio(pages[i]);
if (folio == prev_folio)
continue;
prev_folio = folio;
if (folio_is_longterm_pinnable(folio))
continue;
collected++;
if (folio_is_device_coherent(folio))
continue;
if (folio_test_hugetlb(folio)) {
isolate_hugetlb(folio, movable_page_list);
continue;
}
if (!folio_test_lru(folio) && drain_allow) {
lru_add_drain_all();
drain_allow = false;
}
if (!folio_isolate_lru(folio))
continue;
list_add_tail(&folio->lru, movable_page_list);
node_stat_mod_folio(folio,
NR_ISOLATED_ANON + folio_is_file_lru(folio),
folio_nr_pages(folio));
}
return collected;
}
/*
* Unpins all pages and migrates device coherent pages and movable_page_list.
* Returns -EAGAIN if all pages were successfully migrated or -errno for failure
* (or partial success).
*/
static int migrate_longterm_unpinnable_pages(
struct list_head *movable_page_list,
unsigned long nr_pages,
struct page **pages)
{
int ret;
unsigned long i;
for (i = 0; i < nr_pages; i++) {
struct folio *folio = page_folio(pages[i]);
if (folio_is_device_coherent(folio)) {
/*
* Migration will fail if the page is pinned, so convert
* the pin on the source page to a normal reference.
*/
pages[i] = NULL;
folio_get(folio);
gup_put_folio(folio, 1, FOLL_PIN);
if (migrate_device_coherent_page(&folio->page)) {
ret = -EBUSY;
goto err;
}
continue;
}
/*
* We can't migrate pages with unexpected references, so drop
* the reference obtained by __get_user_pages_locked().
* Migrating pages have been added to movable_page_list after
* calling folio_isolate_lru() which takes a reference so the
* page won't be freed if it's migrating.
*/
unpin_user_page(pages[i]);
pages[i] = NULL;
}
if (!list_empty(movable_page_list)) {
struct migration_target_control mtc = {
.nid = NUMA_NO_NODE,
.gfp_mask = GFP_USER | __GFP_NOWARN,
};
if (migrate_pages(movable_page_list, alloc_migration_target,
NULL, (unsigned long)&mtc, MIGRATE_SYNC,
MR_LONGTERM_PIN, NULL)) {
ret = -ENOMEM;
goto err;
}
}
putback_movable_pages(movable_page_list);
return -EAGAIN;
err:
for (i = 0; i < nr_pages; i++)
if (pages[i])
unpin_user_page(pages[i]);
putback_movable_pages(movable_page_list);
return ret;
}
/*
* Check whether all pages are *allowed* to be pinned. Rather confusingly, all
* pages in the range are required to be pinned via FOLL_PIN, before calling
* this routine.
*
* If any pages in the range are not allowed to be pinned, then this routine
* will migrate those pages away, unpin all the pages in the range and return
* -EAGAIN. The caller should re-pin the entire range with FOLL_PIN and then
* call this routine again.
*
* If an error other than -EAGAIN occurs, this indicates a migration failure.
* The caller should give up, and propagate the error back up the call stack.
*
* If everything is OK and all pages in the range are allowed to be pinned, then
* this routine leaves all pages pinned and returns zero for success.
*/
static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages)
{
unsigned long collected;
LIST_HEAD(movable_page_list);
collected = collect_longterm_unpinnable_pages(&movable_page_list,
nr_pages, pages);
if (!collected)
return 0;
return migrate_longterm_unpinnable_pages(&movable_page_list, nr_pages,
pages);
}
#else
static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages)
{
return 0;
}
#endif /* CONFIG_MIGRATION */
/*
* __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
* allows us to process the FOLL_LONGTERM flag.
*/
static long __gup_longterm_locked(struct mm_struct *mm,
unsigned long start,
unsigned long nr_pages,
struct page **pages,
int *locked,
unsigned int gup_flags)
{
unsigned int flags;
long rc, nr_pinned_pages;
if (!(gup_flags & FOLL_LONGTERM))
return __get_user_pages_locked(mm, start, nr_pages, pages,
locked, gup_flags);
flags = memalloc_pin_save();
do {
nr_pinned_pages = __get_user_pages_locked(mm, start, nr_pages,
pages, locked,
gup_flags);
if (nr_pinned_pages <= 0) {
rc = nr_pinned_pages;
break;
}
/* FOLL_LONGTERM implies FOLL_PIN */
rc = check_and_migrate_movable_pages(nr_pinned_pages, pages);
} while (rc == -EAGAIN);
memalloc_pin_restore(flags);
return rc ? rc : nr_pinned_pages;
}
/*
* Check that the given flags are valid for the exported gup/pup interface, and
* update them with the required flags that the caller must have set.
*/
static bool is_valid_gup_args(struct page **pages, int *locked,
unsigned int *gup_flags_p, unsigned int to_set)
{
unsigned int gup_flags = *gup_flags_p;
/*
* These flags not allowed to be specified externally to the gup
* interfaces:
* - FOLL_TOUCH/FOLL_PIN/FOLL_TRIED/FOLL_FAST_ONLY are internal only
* - FOLL_REMOTE is internal only and used on follow_page()
* - FOLL_UNLOCKABLE is internal only and used if locked is !NULL
*/
if (WARN_ON_ONCE(gup_flags & INTERNAL_GUP_FLAGS))
return false;
gup_flags |= to_set;
if (locked) {
/* At the external interface locked must be set */
if (WARN_ON_ONCE(*locked != 1))
return false;
gup_flags |= FOLL_UNLOCKABLE;
}
/* FOLL_GET and FOLL_PIN are mutually exclusive. */
if (WARN_ON_ONCE((gup_flags & (FOLL_PIN | FOLL_GET)) ==
(FOLL_PIN | FOLL_GET)))
return false;
/* LONGTERM can only be specified when pinning */
if (WARN_ON_ONCE(!(gup_flags & FOLL_PIN) && (gup_flags & FOLL_LONGTERM)))
return false;
/* Pages input must be given if using GET/PIN */
if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages))
return false;
/* We want to allow the pgmap to be hot-unplugged at all times */
if (WARN_ON_ONCE((gup_flags & FOLL_LONGTERM) &&
(gup_flags & FOLL_PCI_P2PDMA)))
return false;
*gup_flags_p = gup_flags;
return true;
}
#ifdef CONFIG_MMU
/**
* get_user_pages_remote() - pin user pages in memory
* @mm: mm_struct of target mm
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying lookup behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long. Or NULL, if caller
* only intends to ensure the pages are faulted in.
* @locked: pointer to lock flag indicating whether lock is held and
* subsequently whether VM_FAULT_RETRY functionality can be
* utilised. Lock must initially be held.
*
* Returns either number of pages pinned (which may be less than the
* number requested), or an error. Details about the return value:
*
* -- If nr_pages is 0, returns 0.
* -- If nr_pages is >0, but no pages were pinned, returns -errno.
* -- If nr_pages is >0, and some pages were pinned, returns the number of
* pages pinned. Again, this may be less than nr_pages.
*
* The caller is responsible for releasing returned @pages, via put_page().
*
* Must be called with mmap_lock held for read or write.
*
* get_user_pages_remote walks a process's page tables and takes a reference
* to each struct page that each user address corresponds to at a given
* instant. That is, it takes the page that would be accessed if a user
* thread accesses the given user virtual address at that instant.
*
* This does not guarantee that the page exists in the user mappings when
* get_user_pages_remote returns, and there may even be a completely different
* page there in some cases (eg. if mmapped pagecache has been invalidated
* and subsequently re-faulted). However it does guarantee that the page
* won't be freed completely. And mostly callers simply care that the page
* contains data that was valid *at some point in time*. Typically, an IO
* or similar operation cannot guarantee anything stronger anyway because
* locks can't be held over the syscall boundary.
*
* If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
* is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
* be called after the page is finished with, and before put_page is called.
*
* get_user_pages_remote is typically used for fewer-copy IO operations,
* to get a handle on the memory by some means other than accesses
* via the user virtual addresses. The pages may be submitted for
* DMA to devices or accessed via their kernel linear mapping (via the
* kmap APIs). Care should be taken to use the correct cache flushing APIs.
*
* See also get_user_pages_fast, for performance critical applications.
*
* get_user_pages_remote should be phased out in favor of
* get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
* should use get_user_pages_remote because it cannot pass
* FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
*/
long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
int *locked)
{
int local_locked = 1;
if (!is_valid_gup_args(pages, locked, &gup_flags,
FOLL_TOUCH | FOLL_REMOTE))
return -EINVAL;
return __get_user_pages_locked(mm, start, nr_pages, pages,
locked ? locked : &local_locked,
gup_flags);
}
EXPORT_SYMBOL(get_user_pages_remote);
#else /* CONFIG_MMU */
long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
int *locked)
{
return 0;
}
#endif /* !CONFIG_MMU */
/**
* get_user_pages() - pin user pages in memory
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying lookup behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long. Or NULL, if caller
* only intends to ensure the pages are faulted in.
*
* This is the same as get_user_pages_remote(), just with a less-flexible
* calling convention where we assume that the mm being operated on belongs to
* the current task, and doesn't allow passing of a locked parameter. We also
* obviously don't pass FOLL_REMOTE in here.
*/
long get_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages)
{
int locked = 1;
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH))
return -EINVAL;
return __get_user_pages_locked(current->mm, start, nr_pages, pages,
&locked, gup_flags);
}
EXPORT_SYMBOL(get_user_pages);
/*
* get_user_pages_unlocked() is suitable to replace the form:
*
* mmap_read_lock(mm);
* get_user_pages(mm, ..., pages, NULL);
* mmap_read_unlock(mm);
*
* with:
*
* get_user_pages_unlocked(mm, ..., pages);
*
* It is functionally equivalent to get_user_pages_fast so
* get_user_pages_fast should be used instead if specific gup_flags
* (e.g. FOLL_FORCE) are not required.
*/
long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
struct page **pages, unsigned int gup_flags)
{
int locked = 0;
if (!is_valid_gup_args(pages, NULL, &gup_flags,
FOLL_TOUCH | FOLL_UNLOCKABLE))
return -EINVAL;
return __get_user_pages_locked(current->mm, start, nr_pages, pages,
&locked, gup_flags);
}
EXPORT_SYMBOL(get_user_pages_unlocked);
/*
* Fast GUP
*
* get_user_pages_fast attempts to pin user pages by walking the page
* tables directly and avoids taking locks. Thus the walker needs to be
* protected from page table pages being freed from under it, and should
* block any THP splits.
*
* One way to achieve this is to have the walker disable interrupts, and
* rely on IPIs from the TLB flushing code blocking before the page table
* pages are freed. This is unsuitable for architectures that do not need
* to broadcast an IPI when invalidating TLBs.
*
* Another way to achieve this is to batch up page table containing pages
* belonging to more than one mm_user, then rcu_sched a callback to free those
* pages. Disabling interrupts will allow the fast_gup walker to both block
* the rcu_sched callback, and an IPI that we broadcast for splitting THPs
* (which is a relatively rare event). The code below adopts this strategy.
*
* Before activating this code, please be aware that the following assumptions
* are currently made:
*
* *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
* free pages containing page tables or TLB flushing requires IPI broadcast.
*
* *) ptes can be read atomically by the architecture.
*
* *) access_ok is sufficient to validate userspace address ranges.
*
* The last two assumptions can be relaxed by the addition of helper functions.
*
* This code is based heavily on the PowerPC implementation by Nick Piggin.
*/
#ifdef CONFIG_HAVE_FAST_GUP
/*
* Used in the GUP-fast path to determine whether a pin is permitted for a
* specific folio.
*
* This call assumes the caller has pinned the folio, that the lowest page table
* level still points to this folio, and that interrupts have been disabled.
*
* Writing to pinned file-backed dirty tracked folios is inherently problematic
* (see comment describing the writable_file_mapping_allowed() function). We
* therefore try to avoid the most egregious case of a long-term mapping doing
* so.
*
* This function cannot be as thorough as that one as the VMA is not available
* in the fast path, so instead we whitelist known good cases and if in doubt,
* fall back to the slow path.
*/
static bool folio_fast_pin_allowed(struct folio *folio, unsigned int flags)
{
struct address_space *mapping;
unsigned long mapping_flags;
/*
* If we aren't pinning then no problematic write can occur. A long term
* pin is the most egregious case so this is the one we disallow.
*/
if ((flags & (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) !=
(FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE))
return true;
/* The folio is pinned, so we can safely access folio fields. */
if (WARN_ON_ONCE(folio_test_slab(folio)))
return false;
/* hugetlb mappings do not require dirty-tracking. */
if (folio_test_hugetlb(folio))
return true;
/*
* GUP-fast disables IRQs. When IRQS are disabled, RCU grace periods
* cannot proceed, which means no actions performed under RCU can
* proceed either.
*
* inodes and thus their mappings are freed under RCU, which means the
* mapping cannot be freed beneath us and thus we can safely dereference
* it.
*/
lockdep_assert_irqs_disabled();
/*
* However, there may be operations which _alter_ the mapping, so ensure
* we read it once and only once.
*/
mapping = READ_ONCE(folio->mapping);
/*
* The mapping may have been truncated, in any case we cannot determine
* if this mapping is safe - fall back to slow path to determine how to
* proceed.
*/
if (!mapping)
return false;
/* Anonymous folios pose no problem. */
mapping_flags = (unsigned long)mapping & PAGE_MAPPING_FLAGS;
if (mapping_flags)
return mapping_flags & PAGE_MAPPING_ANON;
/*
* At this point, we know the mapping is non-null and points to an
* address_space object. The only remaining whitelisted file system is
* shmem.
*/
return shmem_mapping(mapping);
}
static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
unsigned int flags,
struct page **pages)
{
while ((*nr) - nr_start) {
struct page *page = pages[--(*nr)];
ClearPageReferenced(page);
if (flags & FOLL_PIN)
unpin_user_page(page);
else
put_page(page);
}
}
#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
/*
* Fast-gup relies on pte change detection to avoid concurrent pgtable
* operations.
*
* To pin the page, fast-gup needs to do below in order:
* (1) pin the page (by prefetching pte), then (2) check pte not changed.
*
* For the rest of pgtable operations where pgtable updates can be racy
* with fast-gup, we need to do (1) clear pte, then (2) check whether page
* is pinned.
*
* Above will work for all pte-level operations, including THP split.
*
* For THP collapse, it's a bit more complicated because fast-gup may be
* walking a pgtable page that is being freed (pte is still valid but pmd
* can be cleared already). To avoid race in such condition, we need to
* also check pmd here to make sure pmd doesn't change (corresponds to
* pmdp_collapse_flush() in the THP collapse code path).
*/
static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
struct dev_pagemap *pgmap = NULL;
int nr_start = *nr, ret = 0;
pte_t *ptep, *ptem;
ptem = ptep = pte_offset_map(&pmd, addr);
if (!ptep)
return 0;
do {
pte_t pte = ptep_get_lockless(ptep);
struct page *page;
struct folio *folio;
/*
* Always fallback to ordinary GUP on PROT_NONE-mapped pages:
* pte_access_permitted() better should reject these pages
* either way: otherwise, GUP-fast might succeed in
* cases where ordinary GUP would fail due to VMA access
* permissions.
*/
if (pte_protnone(pte))
goto pte_unmap;
if (!pte_access_permitted(pte, flags & FOLL_WRITE))
goto pte_unmap;
if (pte_devmap(pte)) {
if (unlikely(flags & FOLL_LONGTERM))
goto pte_unmap;
pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
if (unlikely(!pgmap)) {
undo_dev_pagemap(nr, nr_start, flags, pages);
goto pte_unmap;
}
} else if (pte_special(pte))
goto pte_unmap;
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
page = pte_page(pte);
folio = try_grab_folio(page, 1, flags);
if (!folio)
goto pte_unmap;
if (unlikely(folio_is_secretmem(folio))) {
gup_put_folio(folio, 1, flags);
goto pte_unmap;
}
if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) ||
unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) {
gup_put_folio(folio, 1, flags);
goto pte_unmap;
}
if (!folio_fast_pin_allowed(folio, flags)) {
gup_put_folio(folio, 1, flags);
goto pte_unmap;
}
if (!pte_write(pte) && gup_must_unshare(NULL, flags, page)) {
gup_put_folio(folio, 1, flags);
goto pte_unmap;
}
/*
* We need to make the page accessible if and only if we are
* going to access its content (the FOLL_PIN case). Please
* see Documentation/core-api/pin_user_pages.rst for
* details.
*/
if (flags & FOLL_PIN) {
ret = arch_make_page_accessible(page);
if (ret) {
gup_put_folio(folio, 1, flags);
goto pte_unmap;
}
}
folio_set_referenced(folio);
pages[*nr] = page;
(*nr)++;
} while (ptep++, addr += PAGE_SIZE, addr != end);
ret = 1;
pte_unmap:
if (pgmap)
put_dev_pagemap(pgmap);
pte_unmap(ptem);
return ret;
}
#else
/*
* If we can't determine whether or not a pte is special, then fail immediately
* for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
* to be special.
*
* For a futex to be placed on a THP tail page, get_futex_key requires a
* get_user_pages_fast_only implementation that can pin pages. Thus it's still
* useful to have gup_huge_pmd even if we can't operate on ptes.
*/
static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
return 0;
}
#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
static int __gup_device_huge(unsigned long pfn, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
int nr_start = *nr;
struct dev_pagemap *pgmap = NULL;
do {
struct page *page = pfn_to_page(pfn);
pgmap = get_dev_pagemap(pfn, pgmap);
if (unlikely(!pgmap)) {
undo_dev_pagemap(nr, nr_start, flags, pages);
break;
}
if (!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)) {
undo_dev_pagemap(nr, nr_start, flags, pages);
break;
}
SetPageReferenced(page);
pages[*nr] = page;
if (unlikely(try_grab_page(page, flags))) {
undo_dev_pagemap(nr, nr_start, flags, pages);
break;
}
(*nr)++;
pfn++;
} while (addr += PAGE_SIZE, addr != end);
put_dev_pagemap(pgmap);
return addr == end;
}
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
unsigned long fault_pfn;
int nr_start = *nr;
fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
return 0;
if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
undo_dev_pagemap(nr, nr_start, flags, pages);
return 0;
}
return 1;
}
static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
unsigned long fault_pfn;
int nr_start = *nr;
fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
return 0;
if (unlikely(pud_val(orig) != pud_val(*pudp))) {
undo_dev_pagemap(nr, nr_start, flags, pages);
return 0;
}
return 1;
}
#else
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
BUILD_BUG();
return 0;
}
static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
BUILD_BUG();
return 0;
}
#endif
static int record_subpages(struct page *page, unsigned long addr,
unsigned long end, struct page **pages)
{
int nr;
for (nr = 0; addr != end; nr++, addr += PAGE_SIZE)
pages[nr] = nth_page(page, nr);
return nr;
}
#ifdef CONFIG_ARCH_HAS_HUGEPD
static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
unsigned long sz)
{
unsigned long __boundary = (addr + sz) & ~(sz-1);
return (__boundary - 1 < end - 1) ? __boundary : end;
}
static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
unsigned long pte_end;
struct page *page;
struct folio *folio;
pte_t pte;
int refs;
pte_end = (addr + sz) & ~(sz-1);
if (pte_end < end)
end = pte_end;
pte = huge_ptep_get(ptep);
if (!pte_access_permitted(pte, flags & FOLL_WRITE))
return 0;
/* hugepages are never "special" */
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT);
refs = record_subpages(page, addr, end, pages + *nr);
folio = try_grab_folio(page, refs, flags);
if (!folio)
return 0;
if (unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!folio_fast_pin_allowed(folio, flags)) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!pte_write(pte) && gup_must_unshare(NULL, flags, &folio->page)) {
gup_put_folio(folio, refs, flags);
return 0;
}
*nr += refs;
folio_set_referenced(folio);
return 1;
}
static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
unsigned int pdshift, unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
pte_t *ptep;
unsigned long sz = 1UL << hugepd_shift(hugepd);
unsigned long next;
ptep = hugepte_offset(hugepd, addr, pdshift);
do {
next = hugepte_addr_end(addr, end, sz);
if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
return 0;
} while (ptep++, addr = next, addr != end);
return 1;
}
#else
static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
unsigned int pdshift, unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
return 0;
}
#endif /* CONFIG_ARCH_HAS_HUGEPD */
static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
struct page *page;
struct folio *folio;
int refs;
if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
return 0;
if (pmd_devmap(orig)) {
if (unlikely(flags & FOLL_LONGTERM))
return 0;
return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
pages, nr);
}
page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT);
refs = record_subpages(page, addr, end, pages + *nr);
folio = try_grab_folio(page, refs, flags);
if (!folio)
return 0;
if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!folio_fast_pin_allowed(folio, flags)) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
gup_put_folio(folio, refs, flags);
return 0;
}
*nr += refs;
folio_set_referenced(folio);
return 1;
}
static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
struct page *page;
struct folio *folio;
int refs;
if (!pud_access_permitted(orig, flags & FOLL_WRITE))
return 0;
if (pud_devmap(orig)) {
if (unlikely(flags & FOLL_LONGTERM))
return 0;
return __gup_device_huge_pud(orig, pudp, addr, end, flags,
pages, nr);
}
page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT);
refs = record_subpages(page, addr, end, pages + *nr);
folio = try_grab_folio(page, refs, flags);
if (!folio)
return 0;
if (unlikely(pud_val(orig) != pud_val(*pudp))) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!folio_fast_pin_allowed(folio, flags)) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
gup_put_folio(folio, refs, flags);
return 0;
}
*nr += refs;
folio_set_referenced(folio);
return 1;
}
static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
unsigned long end, unsigned int flags,
struct page **pages, int *nr)
{
int refs;
struct page *page;
struct folio *folio;
if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
return 0;
BUILD_BUG_ON(pgd_devmap(orig));
page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT);
refs = record_subpages(page, addr, end, pages + *nr);
folio = try_grab_folio(page, refs, flags);
if (!folio)
return 0;
if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!pgd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
gup_put_folio(folio, refs, flags);
return 0;
}
if (!folio_fast_pin_allowed(folio, flags)) {
gup_put_folio(folio, refs, flags);
return 0;
}
*nr += refs;
folio_set_referenced(folio);
return 1;
}
static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pmd_t *pmdp;
pmdp = pmd_offset_lockless(pudp, pud, addr);
do {
pmd_t pmd = pmdp_get_lockless(pmdp);
next = pmd_addr_end(addr, end);
if (!pmd_present(pmd))
return 0;
if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
pmd_devmap(pmd))) {
/* See gup_pte_range() */
if (pmd_protnone(pmd))
return 0;
if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
pages, nr))
return 0;
} else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
/*
* architecture have different format for hugetlbfs
* pmd format and THP pmd format
*/
if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
PMD_SHIFT, next, flags, pages, nr))
return 0;
} else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
return 0;
} while (pmdp++, addr = next, addr != end);
return 1;
}
static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pud_t *pudp;
pudp = pud_offset_lockless(p4dp, p4d, addr);
do {
pud_t pud = READ_ONCE(*pudp);
next = pud_addr_end(addr, end);
if (unlikely(!pud_present(pud)))
return 0;
if (unlikely(pud_huge(pud) || pud_devmap(pud))) {
if (!gup_huge_pud(pud, pudp, addr, next, flags,
pages, nr))
return 0;
} else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
PUD_SHIFT, next, flags, pages, nr))
return 0;
} else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
return 0;
} while (pudp++, addr = next, addr != end);
return 1;
}
static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
p4d_t *p4dp;
p4dp = p4d_offset_lockless(pgdp, pgd, addr);
do {
p4d_t p4d = READ_ONCE(*p4dp);
next = p4d_addr_end(addr, end);
if (p4d_none(p4d))
return 0;
BUILD_BUG_ON(p4d_huge(p4d));
if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
P4D_SHIFT, next, flags, pages, nr))
return 0;
} else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
return 0;
} while (p4dp++, addr = next, addr != end);
return 1;
}
static void gup_pgd_range(unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pgd_t *pgdp;
pgdp = pgd_offset(current->mm, addr);
do {
pgd_t pgd = READ_ONCE(*pgdp);
next = pgd_addr_end(addr, end);
if (pgd_none(pgd))
return;
if (unlikely(pgd_huge(pgd))) {
if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
pages, nr))
return;
} else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
PGDIR_SHIFT, next, flags, pages, nr))
return;
} else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
return;
} while (pgdp++, addr = next, addr != end);
}
#else
static inline void gup_pgd_range(unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
}
#endif /* CONFIG_HAVE_FAST_GUP */
#ifndef gup_fast_permitted
/*
* Check if it's allowed to use get_user_pages_fast_only() for the range, or
* we need to fall back to the slow version:
*/
static bool gup_fast_permitted(unsigned long start, unsigned long end)
{
return true;
}
#endif
static unsigned long lockless_pages_from_mm(unsigned long start,
unsigned long end,
unsigned int gup_flags,
struct page **pages)
{
unsigned long flags;
int nr_pinned = 0;
unsigned seq;
if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) ||
!gup_fast_permitted(start, end))
return 0;
if (gup_flags & FOLL_PIN) {
seq = raw_read_seqcount(¤t->mm->write_protect_seq);
if (seq & 1)
return 0;
}
/*
* Disable interrupts. The nested form is used, in order to allow full,
* general purpose use of this routine.
*
* With interrupts disabled, we block page table pages from being freed
* from under us. See struct mmu_table_batch comments in
* include/asm-generic/tlb.h for more details.
*
* We do not adopt an rcu_read_lock() here as we also want to block IPIs
* that come from THPs splitting.
*/
local_irq_save(flags);
gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
local_irq_restore(flags);
/*
* When pinning pages for DMA there could be a concurrent write protect
* from fork() via copy_page_range(), in this case always fail fast GUP.
*/
if (gup_flags & FOLL_PIN) {
if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) {
unpin_user_pages_lockless(pages, nr_pinned);
return 0;
} else {
sanity_check_pinned_pages(pages, nr_pinned);
}
}
return nr_pinned;
}
static int internal_get_user_pages_fast(unsigned long start,
unsigned long nr_pages,
unsigned int gup_flags,
struct page **pages)
{
unsigned long len, end;
unsigned long nr_pinned;
int locked = 0;
int ret;
if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
FOLL_FORCE | FOLL_PIN | FOLL_GET |
FOLL_FAST_ONLY | FOLL_NOFAULT |
FOLL_PCI_P2PDMA | FOLL_HONOR_NUMA_FAULT)))
return -EINVAL;
if (gup_flags & FOLL_PIN)
mm_set_has_pinned_flag(¤t->mm->flags);
if (!(gup_flags & FOLL_FAST_ONLY))
might_lock_read(¤t->mm->mmap_lock);
start = untagged_addr(start) & PAGE_MASK;
len = nr_pages << PAGE_SHIFT;
if (check_add_overflow(start, len, &end))
return -EOVERFLOW;
if (end > TASK_SIZE_MAX)
return -EFAULT;
if (unlikely(!access_ok((void __user *)start, len)))
return -EFAULT;
nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
return nr_pinned;
/* Slow path: try to get the remaining pages with get_user_pages */
start += nr_pinned << PAGE_SHIFT;
pages += nr_pinned;
ret = __gup_longterm_locked(current->mm, start, nr_pages - nr_pinned,
pages, &locked,
gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE);
if (ret < 0) {
/*
* The caller has to unpin the pages we already pinned so
* returning -errno is not an option
*/
if (nr_pinned)
return nr_pinned;
return ret;
}
return ret + nr_pinned;
}
/**
* get_user_pages_fast_only() - pin user pages in memory
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying pin behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long.
*
* Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
* the regular GUP.
*
* If the architecture does not support this function, simply return with no
* pages pinned.
*
* Careful, careful! COW breaking can go either way, so a non-write
* access can get ambiguous page results. If you call this function without
* 'write' set, you'd better be sure that you're ok with that ambiguity.
*/
int get_user_pages_fast_only(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages)
{
/*
* Internally (within mm/gup.c), gup fast variants must set FOLL_GET,
* because gup fast is always a "pin with a +1 page refcount" request.
*
* FOLL_FAST_ONLY is required in order to match the API description of
* this routine: no fall back to regular ("slow") GUP.
*/
if (!is_valid_gup_args(pages, NULL, &gup_flags,
FOLL_GET | FOLL_FAST_ONLY))
return -EINVAL;
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
}
EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
/**
* get_user_pages_fast() - pin user pages in memory
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying pin behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long.
*
* Attempt to pin user pages in memory without taking mm->mmap_lock.
* If not successful, it will fall back to taking the lock and
* calling get_user_pages().
*
* Returns number of pages pinned. This may be fewer than the number requested.
* If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
* -errno.
*/
int get_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages)
{
/*
* The caller may or may not have explicitly set FOLL_GET; either way is
* OK. However, internally (within mm/gup.c), gup fast variants must set
* FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
* request.
*/
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET))
return -EINVAL;
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
}
EXPORT_SYMBOL_GPL(get_user_pages_fast);
/**
* pin_user_pages_fast() - pin user pages in memory without taking locks
*
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying pin behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long.
*
* Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
* get_user_pages_fast() for documentation on the function arguments, because
* the arguments here are identical.
*
* FOLL_PIN means that the pages must be released via unpin_user_page(). Please
* see Documentation/core-api/pin_user_pages.rst for further details.
*
* Note that if a zero_page is amongst the returned pages, it will not have
* pins in it and unpin_user_page() will not remove pins from it.
*/
int pin_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages)
{
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
return -EINVAL;
return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
}
EXPORT_SYMBOL_GPL(pin_user_pages_fast);
/**
* pin_user_pages_remote() - pin pages of a remote process
*
* @mm: mm_struct of target mm
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying lookup behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long.
* @locked: pointer to lock flag indicating whether lock is held and
* subsequently whether VM_FAULT_RETRY functionality can be
* utilised. Lock must initially be held.
*
* Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
* get_user_pages_remote() for documentation on the function arguments, because
* the arguments here are identical.
*
* FOLL_PIN means that the pages must be released via unpin_user_page(). Please
* see Documentation/core-api/pin_user_pages.rst for details.
*
* Note that if a zero_page is amongst the returned pages, it will not have
* pins in it and unpin_user_page*() will not remove pins from it.
*/
long pin_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
int *locked)
{
int local_locked = 1;
if (!is_valid_gup_args(pages, locked, &gup_flags,
FOLL_PIN | FOLL_TOUCH | FOLL_REMOTE))
return 0;
return __gup_longterm_locked(mm, start, nr_pages, pages,
locked ? locked : &local_locked,
gup_flags);
}
EXPORT_SYMBOL(pin_user_pages_remote);
/**
* pin_user_pages() - pin user pages in memory for use by other devices
*
* @start: starting user address
* @nr_pages: number of pages from start to pin
* @gup_flags: flags modifying lookup behaviour
* @pages: array that receives pointers to the pages pinned.
* Should be at least nr_pages long.
*
* Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
* FOLL_PIN is set.
*
* FOLL_PIN means that the pages must be released via unpin_user_page(). Please
* see Documentation/core-api/pin_user_pages.rst for details.
*
* Note that if a zero_page is amongst the returned pages, it will not have
* pins in it and unpin_user_page*() will not remove pins from it.
*/
long pin_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages)
{
int locked = 1;
if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
return 0;
return __gup_longterm_locked(current->mm, start, nr_pages,
pages, &locked, gup_flags);
}
EXPORT_SYMBOL(pin_user_pages);
/*
* pin_user_pages_unlocked() is the FOLL_PIN variant of
* get_user_pages_unlocked(). Behavior is the same, except that this one sets
* FOLL_PIN and rejects FOLL_GET.
*
* Note that if a zero_page is amongst the returned pages, it will not have
* pins in it and unpin_user_page*() will not remove pins from it.
*/
long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
struct page **pages, unsigned int gup_flags)
{
int locked = 0;
if (!is_valid_gup_args(pages, NULL, &gup_flags,
FOLL_PIN | FOLL_TOUCH | FOLL_UNLOCKABLE))
return 0;
return __gup_longterm_locked(current->mm, start, nr_pages, pages,
&locked, gup_flags);
}
EXPORT_SYMBOL(pin_user_pages_unlocked);
// SPDX-License-Identifier: GPL-2.0+
/*
* 2002-10-15 Posix Clocks & timers
* by George Anzinger george@mvista.com
* Copyright (C) 2002 2003 by MontaVista Software.
*
* 2004-06-01 Fix CLOCK_REALTIME clock/timer TIMER_ABSTIME bug.
* Copyright (C) 2004 Boris Hu
*
* These are all the functions necessary to implement POSIX clocks & timers
*/
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/mutex.h>
#include <linux/sched/task.h>
#include <linux/uaccess.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/hash.h>
#include <linux/posix-clock.h>
#include <linux/posix-timers.h>
#include <linux/syscalls.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
#include <linux/export.h>
#include <linux/hashtable.h>
#include <linux/compat.h>
#include <linux/nospec.h>
#include <linux/time_namespace.h>
#include "timekeeping.h"
#include "posix-timers.h"
static struct kmem_cache *posix_timers_cache;
/*
* Timers are managed in a hash table for lockless lookup. The hash key is
* constructed from current::signal and the timer ID and the timer is
* matched against current::signal and the timer ID when walking the hash
* bucket list.
*
* This allows checkpoint/restore to reconstruct the exact timer IDs for
* a process.
*/
static DEFINE_HASHTABLE(posix_timers_hashtable, 9);
static DEFINE_SPINLOCK(hash_lock);
static const struct k_clock * const posix_clocks[];
static const struct k_clock *clockid_to_kclock(const clockid_t id);
static const struct k_clock clock_realtime, clock_monotonic;
/* SIGEV_THREAD_ID cannot share a bit with the other SIGEV values. */
#if SIGEV_THREAD_ID != (SIGEV_THREAD_ID & \
~(SIGEV_SIGNAL | SIGEV_NONE | SIGEV_THREAD))
#error "SIGEV_THREAD_ID must not share bit with other SIGEV values!"
#endif
static struct k_itimer *__lock_timer(timer_t timer_id, unsigned long *flags);
#define lock_timer(tid, flags) \
({ struct k_itimer *__timr; \
__cond_lock(&__timr->it_lock, __timr = __lock_timer(tid, flags)); \
__timr; \
})
static int hash(struct signal_struct *sig, unsigned int nr)
{
return hash_32(hash32_ptr(sig) ^ nr, HASH_BITS(posix_timers_hashtable));
}
static struct k_itimer *__posix_timers_find(struct hlist_head *head,
struct signal_struct *sig,
timer_t id)
{
struct k_itimer *timer;
hlist_for_each_entry_rcu(timer, head, t_hash, lockdep_is_held(&hash_lock)) {
/* timer->it_signal can be set concurrently */
if ((READ_ONCE(timer->it_signal) == sig) && (timer->it_id == id))
return timer;
}
return NULL;
}
static struct k_itimer *posix_timer_by_id(timer_t id)
{
struct signal_struct *sig = current->signal;
struct hlist_head *head = &posix_timers_hashtable[hash(sig, id)];
return __posix_timers_find(head, sig, id);
}
static int posix_timer_add(struct k_itimer *timer)
{
struct signal_struct *sig = current->signal;
struct hlist_head *head;
unsigned int cnt, id;
/*
* FIXME: Replace this by a per signal struct xarray once there is
* a plan to handle the resulting CRIU regression gracefully.
*/
for (cnt = 0; cnt <= INT_MAX; cnt++) {
spin_lock(&hash_lock);
id = sig->next_posix_timer_id;
/* Write the next ID back. Clamp it to the positive space */
sig->next_posix_timer_id = (id + 1) & INT_MAX;
head = &posix_timers_hashtable[hash(sig, id)];
if (!__posix_timers_find(head, sig, id)) {
hlist_add_head_rcu(&timer->t_hash, head);
spin_unlock(&hash_lock);
return id;
}
spin_unlock(&hash_lock);
}
/* POSIX return code when no timer ID could be allocated */
return -EAGAIN;
}
static inline void unlock_timer(struct k_itimer *timr, unsigned long flags)
{
spin_unlock_irqrestore(&timr->it_lock, flags);
}
static int posix_get_realtime_timespec(clockid_t which_clock, struct timespec64 *tp)
{
ktime_get_real_ts64(tp);
return 0;
}
static ktime_t posix_get_realtime_ktime(clockid_t which_clock)
{
return ktime_get_real();
}
static int posix_clock_realtime_set(const clockid_t which_clock,
const struct timespec64 *tp)
{
return do_sys_settimeofday64(tp, NULL);
}
static int posix_clock_realtime_adj(const clockid_t which_clock,
struct __kernel_timex *t)
{
return do_adjtimex(t);
}
static int posix_get_monotonic_timespec(clockid_t which_clock, struct timespec64 *tp)
{
ktime_get_ts64(tp);
timens_add_monotonic(tp);
return 0;
}
static ktime_t posix_get_monotonic_ktime(clockid_t which_clock)
{
return ktime_get();
}
static int posix_get_monotonic_raw(clockid_t which_clock, struct timespec64 *tp)
{
ktime_get_raw_ts64(tp);
timens_add_monotonic(tp);
return 0;
}
static int posix_get_realtime_coarse(clockid_t which_clock, struct timespec64 *tp)
{
ktime_get_coarse_real_ts64(tp);
return 0;
}
static int posix_get_monotonic_coarse(clockid_t which_clock,
struct timespec64 *tp)
{
ktime_get_coarse_ts64(tp);
timens_add_monotonic(tp);
return 0;
}
static int posix_get_coarse_res(const clockid_t which_clock, struct timespec64 *tp)
{
*tp = ktime_to_timespec64(KTIME_LOW_RES);
return 0;
}
static int posix_get_boottime_timespec(const clockid_t which_clock, struct timespec64 *tp)
{
ktime_get_boottime_ts64(tp);
timens_add_boottime(tp);
return 0;
}
static ktime_t posix_get_boottime_ktime(const clockid_t which_clock)
{
return ktime_get_boottime();
}
static int posix_get_tai_timespec(clockid_t which_clock, struct timespec64 *tp)
{
ktime_get_clocktai_ts64(tp);
return 0;
}
static ktime_t posix_get_tai_ktime(clockid_t which_clock)
{
return ktime_get_clocktai();
}
static int posix_get_hrtimer_res(clockid_t which_clock, struct timespec64 *tp)
{
tp->tv_sec = 0;
tp->tv_nsec = hrtimer_resolution;
return 0;
}
static __init int init_posix_timers(void)
{
posix_timers_cache = kmem_cache_create("posix_timers_cache",
sizeof(struct k_itimer), 0,
SLAB_PANIC | SLAB_ACCOUNT, NULL);
return 0;
}
__initcall(init_posix_timers);
/*
* The siginfo si_overrun field and the return value of timer_getoverrun(2)
* are of type int. Clamp the overrun value to INT_MAX
*/
static inline int timer_overrun_to_int(struct k_itimer *timr, int baseval)
{
s64 sum = timr->it_overrun_last + (s64)baseval;
return sum > (s64)INT_MAX ? INT_MAX : (int)sum;
}
static void common_hrtimer_rearm(struct k_itimer *timr)
{
struct hrtimer *timer = &timr->it.real.timer;
timr->it_overrun += hrtimer_forward(timer, timer->base->get_time(),
timr->it_interval);
hrtimer_restart(timer);
}
/*
* This function is called from the signal delivery code if
* info->si_sys_private is not zero, which indicates that the timer has to
* be rearmed. Restart the timer and update info::si_overrun.
*/
void posixtimer_rearm(struct kernel_siginfo *info)
{
struct k_itimer *timr;
unsigned long flags;
timr = lock_timer(info->si_tid, &flags);
if (!timr)
return;
if (timr->it_interval && timr->it_requeue_pending == info->si_sys_private) {
timr->kclock->timer_rearm(timr);
timr->it_active = 1;
timr->it_overrun_last = timr->it_overrun;
timr->it_overrun = -1LL;
++timr->it_requeue_pending;
info->si_overrun = timer_overrun_to_int(timr, info->si_overrun);
}
unlock_timer(timr, flags);
}
int posix_timer_event(struct k_itimer *timr, int si_private)
{
enum pid_type type;
int ret;
/*
* FIXME: if ->sigq is queued we can race with
* dequeue_signal()->posixtimer_rearm().
*
* If dequeue_signal() sees the "right" value of
* si_sys_private it calls posixtimer_rearm().
* We re-queue ->sigq and drop ->it_lock().
* posixtimer_rearm() locks the timer
* and re-schedules it while ->sigq is pending.
* Not really bad, but not that we want.
*/
timr->sigq->info.si_sys_private = si_private;
type = !(timr->it_sigev_notify & SIGEV_THREAD_ID) ? PIDTYPE_TGID : PIDTYPE_PID;
ret = send_sigqueue(timr->sigq, timr->it_pid, type);
/* If we failed to send the signal the timer stops. */
return ret > 0;
}
/*
* This function gets called when a POSIX.1b interval timer expires from
* the HRTIMER interrupt (soft interrupt on RT kernels).
*
* Handles CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_BOOTTIME and CLOCK_TAI
* based timers.
*/
static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer)
{
enum hrtimer_restart ret = HRTIMER_NORESTART;
struct k_itimer *timr;
unsigned long flags;
int si_private = 0;
timr = container_of(timer, struct k_itimer, it.real.timer);
spin_lock_irqsave(&timr->it_lock, flags);
timr->it_active = 0;
if (timr->it_interval != 0)
si_private = ++timr->it_requeue_pending;
if (posix_timer_event(timr, si_private)) {
/*
* The signal was not queued due to SIG_IGN. As a
* consequence the timer is not going to be rearmed from
* the signal delivery path. But as a real signal handler
* can be installed later the timer must be rearmed here.
*/
if (timr->it_interval != 0) {
ktime_t now = hrtimer_cb_get_time(timer);
/*
* FIXME: What we really want, is to stop this
* timer completely and restart it in case the
* SIG_IGN is removed. This is a non trivial
* change to the signal handling code.
*
* For now let timers with an interval less than a
* jiffie expire every jiffie and recheck for a
* valid signal handler.
*
* This avoids interrupt starvation in case of a
* very small interval, which would expire the
* timer immediately again.
*
* Moving now ahead of time by one jiffie tricks
* hrtimer_forward() to expire the timer later,
* while it still maintains the overrun accuracy
* for the price of a slight inconsistency in the
* timer_gettime() case. This is at least better
* than a timer storm.
*
* Only required when high resolution timers are
* enabled as the periodic tick based timers are
* automatically aligned to the next tick.
*/
if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS)) {
ktime_t kj = TICK_NSEC;
if (timr->it_interval < kj)
now = ktime_add(now, kj);
}
timr->it_overrun += hrtimer_forward(timer, now, timr->it_interval);
ret = HRTIMER_RESTART;
++timr->it_requeue_pending;
timr->it_active = 1;
}
}
unlock_timer(timr, flags);
return ret;
}
static struct pid *good_sigevent(sigevent_t * event)
{
struct pid *pid = task_tgid(current);
struct task_struct *rtn;
switch (event->sigev_notify) {
case SIGEV_SIGNAL | SIGEV_THREAD_ID:
pid = find_vpid(event->sigev_notify_thread_id);
rtn = pid_task(pid, PIDTYPE_PID);
if (!rtn || !same_thread_group(rtn, current))
return NULL;
fallthrough;
case SIGEV_SIGNAL:
case SIGEV_THREAD:
if (event->sigev_signo <= 0 || event->sigev_signo > SIGRTMAX)
return NULL;
fallthrough;
case SIGEV_NONE:
return pid;
default:
return NULL;
}
}
static struct k_itimer * alloc_posix_timer(void)
{
struct k_itimer *tmr = kmem_cache_zalloc(posix_timers_cache, GFP_KERNEL);
if (!tmr)
return tmr;
if (unlikely(!(tmr->sigq = sigqueue_alloc()))) {
kmem_cache_free(posix_timers_cache, tmr);
return NULL;
}
clear_siginfo(&tmr->sigq->info);
return tmr;
}
static void k_itimer_rcu_free(struct rcu_head *head)
{
struct k_itimer *tmr = container_of(head, struct k_itimer, rcu);
kmem_cache_free(posix_timers_cache, tmr);
}
static void posix_timer_free(struct k_itimer *tmr)
{
put_pid(tmr->it_pid);
sigqueue_free(tmr->sigq);
call_rcu(&tmr->rcu, k_itimer_rcu_free);
}
static void posix_timer_unhash_and_free(struct k_itimer *tmr)
{
spin_lock(&hash_lock);
hlist_del_rcu(&tmr->t_hash);
spin_unlock(&hash_lock);
posix_timer_free(tmr);
}
static int common_timer_create(struct k_itimer *new_timer)
{
hrtimer_init(&new_timer->it.real.timer, new_timer->it_clock, 0);
return 0;
}
/* Create a POSIX.1b interval timer. */
static int do_timer_create(clockid_t which_clock, struct sigevent *event,
timer_t __user *created_timer_id)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct k_itimer *new_timer;
int error, new_timer_id;
if (!kc)
return -EINVAL;
if (!kc->timer_create)
return -EOPNOTSUPP;
new_timer = alloc_posix_timer();
if (unlikely(!new_timer))
return -EAGAIN;
spin_lock_init(&new_timer->it_lock);
/*
* Add the timer to the hash table. The timer is not yet valid
* because new_timer::it_signal is still NULL. The timer id is also
* not yet visible to user space.
*/
new_timer_id = posix_timer_add(new_timer);
if (new_timer_id < 0) {
posix_timer_free(new_timer);
return new_timer_id;
}
new_timer->it_id = (timer_t) new_timer_id;
new_timer->it_clock = which_clock;
new_timer->kclock = kc;
new_timer->it_overrun = -1LL;
if (event) {
rcu_read_lock();
new_timer->it_pid = get_pid(good_sigevent(event));
rcu_read_unlock();
if (!new_timer->it_pid) {
error = -EINVAL;
goto out;
}
new_timer->it_sigev_notify = event->sigev_notify;
new_timer->sigq->info.si_signo = event->sigev_signo;
new_timer->sigq->info.si_value = event->sigev_value;
} else {
new_timer->it_sigev_notify = SIGEV_SIGNAL;
new_timer->sigq->info.si_signo = SIGALRM;
memset(&new_timer->sigq->info.si_value, 0, sizeof(sigval_t));
new_timer->sigq->info.si_value.sival_int = new_timer->it_id;
new_timer->it_pid = get_pid(task_tgid(current));
}
new_timer->sigq->info.si_tid = new_timer->it_id;
new_timer->sigq->info.si_code = SI_TIMER;
if (copy_to_user(created_timer_id, &new_timer_id, sizeof (new_timer_id))) {
error = -EFAULT;
goto out;
}
/*
* After succesful copy out, the timer ID is visible to user space
* now but not yet valid because new_timer::signal is still NULL.
*
* Complete the initialization with the clock specific create
* callback.
*/
error = kc->timer_create(new_timer);
if (error)
goto out;
spin_lock_irq(¤t->sighand->siglock);
/* This makes the timer valid in the hash table */
WRITE_ONCE(new_timer->it_signal, current->signal);
list_add(&new_timer->list, ¤t->signal->posix_timers);
spin_unlock_irq(¤t->sighand->siglock);
/*
* After unlocking sighand::siglock @new_timer is subject to
* concurrent removal and cannot be touched anymore
*/
return 0;
out:
posix_timer_unhash_and_free(new_timer);
return error;
}
SYSCALL_DEFINE3(timer_create, const clockid_t, which_clock,
struct sigevent __user *, timer_event_spec,
timer_t __user *, created_timer_id)
{
if (timer_event_spec) {
sigevent_t event;
if (copy_from_user(&event, timer_event_spec, sizeof (event)))
return -EFAULT;
return do_timer_create(which_clock, &event, created_timer_id);
}
return do_timer_create(which_clock, NULL, created_timer_id);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(timer_create, clockid_t, which_clock,
struct compat_sigevent __user *, timer_event_spec,
timer_t __user *, created_timer_id)
{
if (timer_event_spec) {
sigevent_t event;
if (get_compat_sigevent(&event, timer_event_spec))
return -EFAULT;
return do_timer_create(which_clock, &event, created_timer_id);
}
return do_timer_create(which_clock, NULL, created_timer_id);
}
#endif
static struct k_itimer *__lock_timer(timer_t timer_id, unsigned long *flags)
{
struct k_itimer *timr;
/*
* timer_t could be any type >= int and we want to make sure any
* @timer_id outside positive int range fails lookup.
*/
if ((unsigned long long)timer_id > INT_MAX)
return NULL;
/*
* The hash lookup and the timers are RCU protected.
*
* Timers are added to the hash in invalid state where
* timr::it_signal == NULL. timer::it_signal is only set after the
* rest of the initialization succeeded.
*
* Timer destruction happens in steps:
* 1) Set timr::it_signal to NULL with timr::it_lock held
* 2) Release timr::it_lock
* 3) Remove from the hash under hash_lock
* 4) Call RCU for removal after the grace period
*
* Holding rcu_read_lock() accross the lookup ensures that
* the timer cannot be freed.
*
* The lookup validates locklessly that timr::it_signal ==
* current::it_signal and timr::it_id == @timer_id. timr::it_id
* can't change, but timr::it_signal becomes NULL during
* destruction.
*/
rcu_read_lock();
timr = posix_timer_by_id(timer_id);
if (timr) {
spin_lock_irqsave(&timr->it_lock, *flags);
/*
* Validate under timr::it_lock that timr::it_signal is
* still valid. Pairs with #1 above.
*/
if (timr->it_signal == current->signal) {
rcu_read_unlock();
return timr;
}
spin_unlock_irqrestore(&timr->it_lock, *flags);
}
rcu_read_unlock();
return NULL;
}
static ktime_t common_hrtimer_remaining(struct k_itimer *timr, ktime_t now)
{
struct hrtimer *timer = &timr->it.real.timer;
return __hrtimer_expires_remaining_adjusted(timer, now);
}
static s64 common_hrtimer_forward(struct k_itimer *timr, ktime_t now)
{
struct hrtimer *timer = &timr->it.real.timer;
return hrtimer_forward(timer, now, timr->it_interval);
}
/*
* Get the time remaining on a POSIX.1b interval timer.
*
* Two issues to handle here:
*
* 1) The timer has a requeue pending. The return value must appear as
* if the timer has been requeued right now.
*
* 2) The timer is a SIGEV_NONE timer. These timers are never enqueued
* into the hrtimer queue and therefore never expired. Emulate expiry
* here taking #1 into account.
*/
void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting)
{
const struct k_clock *kc = timr->kclock;
ktime_t now, remaining, iv;
bool sig_none;
sig_none = timr->it_sigev_notify == SIGEV_NONE;
iv = timr->it_interval;
/* interval timer ? */
if (iv) {
cur_setting->it_interval = ktime_to_timespec64(iv);
} else if (!timr->it_active) {
/*
* SIGEV_NONE oneshot timers are never queued and therefore
* timr->it_active is always false. The check below
* vs. remaining time will handle this case.
*
* For all other timers there is nothing to update here, so
* return.
*/
if (!sig_none)
return;
}
now = kc->clock_get_ktime(timr->it_clock);
/*
* If this is an interval timer and either has requeue pending or
* is a SIGEV_NONE timer move the expiry time forward by intervals,
* so expiry is > now.
*/
if (iv && (timr->it_requeue_pending & REQUEUE_PENDING || sig_none))
timr->it_overrun += kc->timer_forward(timr, now);
remaining = kc->timer_remaining(timr, now);
/*
* As @now is retrieved before a possible timer_forward() and
* cannot be reevaluated by the compiler @remaining is based on the
* same @now value. Therefore @remaining is consistent vs. @now.
*
* Consequently all interval timers, i.e. @iv > 0, cannot have a
* remaining time <= 0 because timer_forward() guarantees to move
* them forward so that the next timer expiry is > @now.
*/
if (remaining <= 0) {
/*
* A single shot SIGEV_NONE timer must return 0, when it is
* expired! Timers which have a real signal delivery mode
* must return a remaining time greater than 0 because the
* signal has not yet been delivered.
*/
if (!sig_none)
cur_setting->it_value.tv_nsec = 1;
} else {
cur_setting->it_value = ktime_to_timespec64(remaining);
}
}
static int do_timer_gettime(timer_t timer_id, struct itimerspec64 *setting)
{
const struct k_clock *kc;
struct k_itimer *timr;
unsigned long flags;
int ret = 0;
timr = lock_timer(timer_id, &flags);
if (!timr)
return -EINVAL;
memset(setting, 0, sizeof(*setting));
kc = timr->kclock;
if (WARN_ON_ONCE(!kc || !kc->timer_get))
ret = -EINVAL;
else
kc->timer_get(timr, setting);
unlock_timer(timr, flags);
return ret;
}
/* Get the time remaining on a POSIX.1b interval timer. */
SYSCALL_DEFINE2(timer_gettime, timer_t, timer_id,
struct __kernel_itimerspec __user *, setting)
{
struct itimerspec64 cur_setting;
int ret = do_timer_gettime(timer_id, &cur_setting);
if (!ret) {
if (put_itimerspec64(&cur_setting, setting))
ret = -EFAULT;
}
return ret;
}
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE2(timer_gettime32, timer_t, timer_id,
struct old_itimerspec32 __user *, setting)
{
struct itimerspec64 cur_setting;
int ret = do_timer_gettime(timer_id, &cur_setting);
if (!ret) {
if (put_old_itimerspec32(&cur_setting, setting))
ret = -EFAULT;
}
return ret;
}
#endif
/**
* sys_timer_getoverrun - Get the number of overruns of a POSIX.1b interval timer
* @timer_id: The timer ID which identifies the timer
*
* The "overrun count" of a timer is one plus the number of expiration
* intervals which have elapsed between the first expiry, which queues the
* signal and the actual signal delivery. On signal delivery the "overrun
* count" is calculated and cached, so it can be returned directly here.
*
* As this is relative to the last queued signal the returned overrun count
* is meaningless outside of the signal delivery path and even there it
* does not accurately reflect the current state when user space evaluates
* it.
*
* Returns:
* -EINVAL @timer_id is invalid
* 1..INT_MAX The number of overruns related to the last delivered signal
*/
SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id)
{
struct k_itimer *timr;
unsigned long flags;
int overrun;
timr = lock_timer(timer_id, &flags);
if (!timr)
return -EINVAL;
overrun = timer_overrun_to_int(timr, 0);
unlock_timer(timr, flags);
return overrun;
}
static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
bool absolute, bool sigev_none)
{
struct hrtimer *timer = &timr->it.real.timer;
enum hrtimer_mode mode;
mode = absolute ? HRTIMER_MODE_ABS : HRTIMER_MODE_REL;
/*
* Posix magic: Relative CLOCK_REALTIME timers are not affected by
* clock modifications, so they become CLOCK_MONOTONIC based under the
* hood. See hrtimer_init(). Update timr->kclock, so the generic
* functions which use timr->kclock->clock_get_*() work.
*
* Note: it_clock stays unmodified, because the next timer_set() might
* use ABSTIME, so it needs to switch back.
*/
if (timr->it_clock == CLOCK_REALTIME)
timr->kclock = absolute ? &clock_realtime : &clock_monotonic;
hrtimer_init(&timr->it.real.timer, timr->it_clock, mode);
timr->it.real.timer.function = posix_timer_fn;
if (!absolute)
expires = ktime_add_safe(expires, timer->base->get_time());
hrtimer_set_expires(timer, expires);
if (!sigev_none)
hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
}
static int common_hrtimer_try_to_cancel(struct k_itimer *timr)
{
return hrtimer_try_to_cancel(&timr->it.real.timer);
}
static void common_timer_wait_running(struct k_itimer *timer)
{
hrtimer_cancel_wait_running(&timer->it.real.timer);
}
/*
* On PREEMPT_RT this prevents priority inversion and a potential livelock
* against the ksoftirqd thread in case that ksoftirqd gets preempted while
* executing a hrtimer callback.
*
* See the comments in hrtimer_cancel_wait_running(). For PREEMPT_RT=n this
* just results in a cpu_relax().
*
* For POSIX CPU timers with CONFIG_POSIX_CPU_TIMERS_TASK_WORK=n this is
* just a cpu_relax(). With CONFIG_POSIX_CPU_TIMERS_TASK_WORK=y this
* prevents spinning on an eventually scheduled out task and a livelock
* when the task which tries to delete or disarm the timer has preempted
* the task which runs the expiry in task work context.
*/
static struct k_itimer *timer_wait_running(struct k_itimer *timer,
unsigned long *flags)
{
const struct k_clock *kc = READ_ONCE(timer->kclock);
timer_t timer_id = READ_ONCE(timer->it_id);
/* Prevent kfree(timer) after dropping the lock */
rcu_read_lock();
unlock_timer(timer, *flags);
/*
* kc->timer_wait_running() might drop RCU lock. So @timer
* cannot be touched anymore after the function returns!
*/
if (!WARN_ON_ONCE(!kc->timer_wait_running))
kc->timer_wait_running(timer);
rcu_read_unlock();
/* Relock the timer. It might be not longer hashed. */
return lock_timer(timer_id, flags);
}
/* Set a POSIX.1b interval timer. */
int common_timer_set(struct k_itimer *timr, int flags,
struct itimerspec64 *new_setting,
struct itimerspec64 *old_setting)
{
const struct k_clock *kc = timr->kclock;
bool sigev_none;
ktime_t expires;
if (old_setting)
common_timer_get(timr, old_setting);
/* Prevent rearming by clearing the interval */
timr->it_interval = 0;
/*
* Careful here. On SMP systems the timer expiry function could be
* active and spinning on timr->it_lock.
*/
if (kc->timer_try_to_cancel(timr) < 0)
return TIMER_RETRY;
timr->it_active = 0;
timr->it_requeue_pending = (timr->it_requeue_pending + 2) &
~REQUEUE_PENDING;
timr->it_overrun_last = 0;
/* Switch off the timer when it_value is zero */
if (!new_setting->it_value.tv_sec && !new_setting->it_value.tv_nsec)
return 0;
timr->it_interval = timespec64_to_ktime(new_setting->it_interval);
expires = timespec64_to_ktime(new_setting->it_value);
if (flags & TIMER_ABSTIME)
expires = timens_ktime_to_host(timr->it_clock, expires);
sigev_none = timr->it_sigev_notify == SIGEV_NONE;
kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none);
timr->it_active = !sigev_none;
return 0;
}
static int do_timer_settime(timer_t timer_id, int tmr_flags,
struct itimerspec64 *new_spec64,
struct itimerspec64 *old_spec64)
{
const struct k_clock *kc;
struct k_itimer *timr;
unsigned long flags;
int error = 0;
if (!timespec64_valid(&new_spec64->it_interval) ||
!timespec64_valid(&new_spec64->it_value))
return -EINVAL;
if (old_spec64)
memset(old_spec64, 0, sizeof(*old_spec64));
timr = lock_timer(timer_id, &flags);
retry:
if (!timr)
return -EINVAL;
kc = timr->kclock;
if (WARN_ON_ONCE(!kc || !kc->timer_set))
error = -EINVAL;
else
error = kc->timer_set(timr, tmr_flags, new_spec64, old_spec64);
if (error == TIMER_RETRY) {
// We already got the old time...
old_spec64 = NULL;
/* Unlocks and relocks the timer if it still exists */
timr = timer_wait_running(timr, &flags);
goto retry;
}
unlock_timer(timr, flags);
return error;
}
/* Set a POSIX.1b interval timer */
SYSCALL_DEFINE4(timer_settime, timer_t, timer_id, int, flags,
const struct __kernel_itimerspec __user *, new_setting,
struct __kernel_itimerspec __user *, old_setting)
{
struct itimerspec64 new_spec, old_spec, *rtn;
int error = 0;
if (!new_setting)
return -EINVAL;
if (get_itimerspec64(&new_spec, new_setting))
return -EFAULT;
rtn = old_setting ? &old_spec : NULL;
error = do_timer_settime(timer_id, flags, &new_spec, rtn);
if (!error && old_setting) {
if (put_itimerspec64(&old_spec, old_setting))
error = -EFAULT;
}
return error;
}
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE4(timer_settime32, timer_t, timer_id, int, flags,
struct old_itimerspec32 __user *, new,
struct old_itimerspec32 __user *, old)
{
struct itimerspec64 new_spec, old_spec;
struct itimerspec64 *rtn = old ? &old_spec : NULL;
int error = 0;
if (!new)
return -EINVAL;
if (get_old_itimerspec32(&new_spec, new))
return -EFAULT;
error = do_timer_settime(timer_id, flags, &new_spec, rtn);
if (!error && old) {
if (put_old_itimerspec32(&old_spec, old))
error = -EFAULT;
}
return error;
}
#endif
int common_timer_del(struct k_itimer *timer)
{
const struct k_clock *kc = timer->kclock;
timer->it_interval = 0;
if (kc->timer_try_to_cancel(timer) < 0)
return TIMER_RETRY;
timer->it_active = 0;
return 0;
}
static inline int timer_delete_hook(struct k_itimer *timer)
{
const struct k_clock *kc = timer->kclock;
if (WARN_ON_ONCE(!kc || !kc->timer_del))
return -EINVAL;
return kc->timer_del(timer);
}
/* Delete a POSIX.1b interval timer. */
SYSCALL_DEFINE1(timer_delete, timer_t, timer_id)
{
struct k_itimer *timer;
unsigned long flags;
timer = lock_timer(timer_id, &flags);
retry_delete:
if (!timer)
return -EINVAL;
if (unlikely(timer_delete_hook(timer) == TIMER_RETRY)) {
/* Unlocks and relocks the timer if it still exists */
timer = timer_wait_running(timer, &flags);
goto retry_delete;
}
spin_lock(¤t->sighand->siglock);
list_del(&timer->list);
spin_unlock(¤t->sighand->siglock);
/*
* A concurrent lookup could check timer::it_signal lockless. It
* will reevaluate with timer::it_lock held and observe the NULL.
*/
WRITE_ONCE(timer->it_signal, NULL);
unlock_timer(timer, flags);
posix_timer_unhash_and_free(timer);
return 0;
}
/*
* Delete a timer if it is armed, remove it from the hash and schedule it
* for RCU freeing.
*/
static void itimer_delete(struct k_itimer *timer)
{
unsigned long flags;
/*
* irqsave is required to make timer_wait_running() work.
*/
spin_lock_irqsave(&timer->it_lock, flags);
retry_delete:
/*
* Even if the timer is not longer accessible from other tasks
* it still might be armed and queued in the underlying timer
* mechanism. Worse, that timer mechanism might run the expiry
* function concurrently.
*/
if (timer_delete_hook(timer) == TIMER_RETRY) {
/*
* Timer is expired concurrently, prevent livelocks
* and pointless spinning on RT.
*
* timer_wait_running() drops timer::it_lock, which opens
* the possibility for another task to delete the timer.
*
* That's not possible here because this is invoked from
* do_exit() only for the last thread of the thread group.
* So no other task can access and delete that timer.
*/
if (WARN_ON_ONCE(timer_wait_running(timer, &flags) != timer))
return;
goto retry_delete;
}
list_del(&timer->list);
/*
* Setting timer::it_signal to NULL is technically not required
* here as nothing can access the timer anymore legitimately via
* the hash table. Set it to NULL nevertheless so that all deletion
* paths are consistent.
*/
WRITE_ONCE(timer->it_signal, NULL);
spin_unlock_irqrestore(&timer->it_lock, flags);
posix_timer_unhash_and_free(timer);
}
/*
* Invoked from do_exit() when the last thread of a thread group exits.
* At that point no other task can access the timers of the dying
* task anymore.
*/
void exit_itimers(struct task_struct *tsk)
{
struct list_head timers;
struct k_itimer *tmr;
if (list_empty(&tsk->signal->posix_timers))
return;
/* Protect against concurrent read via /proc/$PID/timers */
spin_lock_irq(&tsk->sighand->siglock);
list_replace_init(&tsk->signal->posix_timers, &timers);
spin_unlock_irq(&tsk->sighand->siglock);
/* The timers are not longer accessible via tsk::signal */
while (!list_empty(&timers)) {
tmr = list_first_entry(&timers, struct k_itimer, list);
itimer_delete(tmr);
}
}
SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock,
const struct __kernel_timespec __user *, tp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 new_tp;
if (!kc || !kc->clock_set)
return -EINVAL;
if (get_timespec64(&new_tp, tp))
return -EFAULT;
/*
* Permission checks have to be done inside the clock specific
* setter callback.
*/
return kc->clock_set(which_clock, &new_tp);
}
SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock,
struct __kernel_timespec __user *, tp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 kernel_tp;
int error;
if (!kc)
return -EINVAL;
error = kc->clock_get_timespec(which_clock, &kernel_tp);
if (!error && put_timespec64(&kernel_tp, tp))
error = -EFAULT;
return error;
}
int do_clock_adjtime(const clockid_t which_clock, struct __kernel_timex * ktx)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
if (!kc)
return -EINVAL;
if (!kc->clock_adj)
return -EOPNOTSUPP;
return kc->clock_adj(which_clock, ktx);
}
SYSCALL_DEFINE2(clock_adjtime, const clockid_t, which_clock,
struct __kernel_timex __user *, utx)
{
struct __kernel_timex ktx;
int err;
if (copy_from_user(&ktx, utx, sizeof(ktx)))
return -EFAULT;
err = do_clock_adjtime(which_clock, &ktx);
if (err >= 0 && copy_to_user(utx, &ktx, sizeof(ktx)))
return -EFAULT;
return err;
}
/**
* sys_clock_getres - Get the resolution of a clock
* @which_clock: The clock to get the resolution for
* @tp: Pointer to a a user space timespec64 for storage
*
* POSIX defines:
*
* "The clock_getres() function shall return the resolution of any
* clock. Clock resolutions are implementation-defined and cannot be set by
* a process. If the argument res is not NULL, the resolution of the
* specified clock shall be stored in the location pointed to by res. If
* res is NULL, the clock resolution is not returned. If the time argument
* of clock_settime() is not a multiple of res, then the value is truncated
* to a multiple of res."
*
* Due to the various hardware constraints the real resolution can vary
* wildly and even change during runtime when the underlying devices are
* replaced. The kernel also can use hardware devices with different
* resolutions for reading the time and for arming timers.
*
* The kernel therefore deviates from the POSIX spec in various aspects:
*
* 1) The resolution returned to user space
*
* For CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_BOOTTIME, CLOCK_TAI,
* CLOCK_REALTIME_ALARM, CLOCK_BOOTTIME_ALAREM and CLOCK_MONOTONIC_RAW
* the kernel differentiates only two cases:
*
* I) Low resolution mode:
*
* When high resolution timers are disabled at compile or runtime
* the resolution returned is nanoseconds per tick, which represents
* the precision at which timers expire.
*
* II) High resolution mode:
*
* When high resolution timers are enabled the resolution returned
* is always one nanosecond independent of the actual resolution of
* the underlying hardware devices.
*
* For CLOCK_*_ALARM the actual resolution depends on system
* state. When system is running the resolution is the same as the
* resolution of the other clocks. During suspend the actual
* resolution is the resolution of the underlying RTC device which
* might be way less precise than the clockevent device used during
* running state.
*
* For CLOCK_REALTIME_COARSE and CLOCK_MONOTONIC_COARSE the resolution
* returned is always nanoseconds per tick.
*
* For CLOCK_PROCESS_CPUTIME and CLOCK_THREAD_CPUTIME the resolution
* returned is always one nanosecond under the assumption that the
* underlying scheduler clock has a better resolution than nanoseconds
* per tick.
*
* For dynamic POSIX clocks (PTP devices) the resolution returned is
* always one nanosecond.
*
* 2) Affect on sys_clock_settime()
*
* The kernel does not truncate the time which is handed in to
* sys_clock_settime(). The kernel internal timekeeping is always using
* nanoseconds precision independent of the clocksource device which is
* used to read the time from. The resolution of that device only
* affects the presicion of the time returned by sys_clock_gettime().
*
* Returns:
* 0 Success. @tp contains the resolution
* -EINVAL @which_clock is not a valid clock ID
* -EFAULT Copying the resolution to @tp faulted
* -ENODEV Dynamic POSIX clock is not backed by a device
* -EOPNOTSUPP Dynamic POSIX clock does not support getres()
*/
SYSCALL_DEFINE2(clock_getres, const clockid_t, which_clock,
struct __kernel_timespec __user *, tp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 rtn_tp;
int error;
if (!kc)
return -EINVAL;
error = kc->clock_getres(which_clock, &rtn_tp);
if (!error && tp && put_timespec64(&rtn_tp, tp))
error = -EFAULT;
return error;
}
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE2(clock_settime32, clockid_t, which_clock,
struct old_timespec32 __user *, tp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 ts;
if (!kc || !kc->clock_set)
return -EINVAL;
if (get_old_timespec32(&ts, tp))
return -EFAULT;
return kc->clock_set(which_clock, &ts);
}
SYSCALL_DEFINE2(clock_gettime32, clockid_t, which_clock,
struct old_timespec32 __user *, tp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 ts;
int err;
if (!kc)
return -EINVAL;
err = kc->clock_get_timespec(which_clock, &ts);
if (!err && put_old_timespec32(&ts, tp))
err = -EFAULT;
return err;
}
SYSCALL_DEFINE2(clock_adjtime32, clockid_t, which_clock,
struct old_timex32 __user *, utp)
{
struct __kernel_timex ktx;
int err;
err = get_old_timex32(&ktx, utp);
if (err)
return err;
err = do_clock_adjtime(which_clock, &ktx);
if (err >= 0 && put_old_timex32(utp, &ktx))
return -EFAULT;
return err;
}
SYSCALL_DEFINE2(clock_getres_time32, clockid_t, which_clock,
struct old_timespec32 __user *, tp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 ts;
int err;
if (!kc)
return -EINVAL;
err = kc->clock_getres(which_clock, &ts);
if (!err && tp && put_old_timespec32(&ts, tp))
return -EFAULT;
return err;
}
#endif
/*
* sys_clock_nanosleep() for CLOCK_REALTIME and CLOCK_TAI
*/
static int common_nsleep(const clockid_t which_clock, int flags,
const struct timespec64 *rqtp)
{ ktime_t texp = timespec64_to_ktime(*rqtp); return hrtimer_nanosleep(texp, flags & TIMER_ABSTIME ?
HRTIMER_MODE_ABS : HRTIMER_MODE_REL,
which_clock);
}
/*
* sys_clock_nanosleep() for CLOCK_MONOTONIC and CLOCK_BOOTTIME
*
* Absolute nanosleeps for these clocks are time-namespace adjusted.
*/
static int common_nsleep_timens(const clockid_t which_clock, int flags,
const struct timespec64 *rqtp)
{
ktime_t texp = timespec64_to_ktime(*rqtp);
if (flags & TIMER_ABSTIME)
texp = timens_ktime_to_host(which_clock, texp);
return hrtimer_nanosleep(texp, flags & TIMER_ABSTIME ?
HRTIMER_MODE_ABS : HRTIMER_MODE_REL,
which_clock);
}
SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags,
const struct __kernel_timespec __user *, rqtp,
struct __kernel_timespec __user *, rmtp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock); struct timespec64 t;
if (!kc)
return -EINVAL; if (!kc->nsleep)
return -EOPNOTSUPP;
if (get_timespec64(&t, rqtp))
return -EFAULT;
if (!timespec64_valid(&t))
return -EINVAL;
if (flags & TIMER_ABSTIME)
rmtp = NULL;
current->restart_block.fn = do_no_restart_syscall;
current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
current->restart_block.nanosleep.rmtp = rmtp;
return kc->nsleep(which_clock, flags, &t);
}
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE4(clock_nanosleep_time32, clockid_t, which_clock, int, flags,
struct old_timespec32 __user *, rqtp,
struct old_timespec32 __user *, rmtp)
{
const struct k_clock *kc = clockid_to_kclock(which_clock);
struct timespec64 t;
if (!kc)
return -EINVAL;
if (!kc->nsleep)
return -EOPNOTSUPP;
if (get_old_timespec32(&t, rqtp))
return -EFAULT;
if (!timespec64_valid(&t))
return -EINVAL;
if (flags & TIMER_ABSTIME)
rmtp = NULL;
current->restart_block.fn = do_no_restart_syscall;
current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
current->restart_block.nanosleep.compat_rmtp = rmtp;
return kc->nsleep(which_clock, flags, &t);
}
#endif
static const struct k_clock clock_realtime = {
.clock_getres = posix_get_hrtimer_res,
.clock_get_timespec = posix_get_realtime_timespec,
.clock_get_ktime = posix_get_realtime_ktime,
.clock_set = posix_clock_realtime_set,
.clock_adj = posix_clock_realtime_adj,
.nsleep = common_nsleep,
.timer_create = common_timer_create,
.timer_set = common_timer_set,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
.timer_rearm = common_hrtimer_rearm,
.timer_forward = common_hrtimer_forward,
.timer_remaining = common_hrtimer_remaining,
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
.timer_wait_running = common_timer_wait_running,
.timer_arm = common_hrtimer_arm,
};
static const struct k_clock clock_monotonic = {
.clock_getres = posix_get_hrtimer_res,
.clock_get_timespec = posix_get_monotonic_timespec,
.clock_get_ktime = posix_get_monotonic_ktime,
.nsleep = common_nsleep_timens,
.timer_create = common_timer_create,
.timer_set = common_timer_set,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
.timer_rearm = common_hrtimer_rearm,
.timer_forward = common_hrtimer_forward,
.timer_remaining = common_hrtimer_remaining,
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
.timer_wait_running = common_timer_wait_running,
.timer_arm = common_hrtimer_arm,
};
static const struct k_clock clock_monotonic_raw = {
.clock_getres = posix_get_hrtimer_res,
.clock_get_timespec = posix_get_monotonic_raw,
};
static const struct k_clock clock_realtime_coarse = {
.clock_getres = posix_get_coarse_res,
.clock_get_timespec = posix_get_realtime_coarse,
};
static const struct k_clock clock_monotonic_coarse = {
.clock_getres = posix_get_coarse_res,
.clock_get_timespec = posix_get_monotonic_coarse,
};
static const struct k_clock clock_tai = {
.clock_getres = posix_get_hrtimer_res,
.clock_get_ktime = posix_get_tai_ktime,
.clock_get_timespec = posix_get_tai_timespec,
.nsleep = common_nsleep,
.timer_create = common_timer_create,
.timer_set = common_timer_set,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
.timer_rearm = common_hrtimer_rearm,
.timer_forward = common_hrtimer_forward,
.timer_remaining = common_hrtimer_remaining,
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
.timer_wait_running = common_timer_wait_running,
.timer_arm = common_hrtimer_arm,
};
static const struct k_clock clock_boottime = {
.clock_getres = posix_get_hrtimer_res,
.clock_get_ktime = posix_get_boottime_ktime,
.clock_get_timespec = posix_get_boottime_timespec,
.nsleep = common_nsleep_timens,
.timer_create = common_timer_create,
.timer_set = common_timer_set,
.timer_get = common_timer_get,
.timer_del = common_timer_del,
.timer_rearm = common_hrtimer_rearm,
.timer_forward = common_hrtimer_forward,
.timer_remaining = common_hrtimer_remaining,
.timer_try_to_cancel = common_hrtimer_try_to_cancel,
.timer_wait_running = common_timer_wait_running,
.timer_arm = common_hrtimer_arm,
};
static const struct k_clock * const posix_clocks[] = {
[CLOCK_REALTIME] = &clock_realtime,
[CLOCK_MONOTONIC] = &clock_monotonic,
[CLOCK_PROCESS_CPUTIME_ID] = &clock_process,
[CLOCK_THREAD_CPUTIME_ID] = &clock_thread,
[CLOCK_MONOTONIC_RAW] = &clock_monotonic_raw,
[CLOCK_REALTIME_COARSE] = &clock_realtime_coarse,
[CLOCK_MONOTONIC_COARSE] = &clock_monotonic_coarse,
[CLOCK_BOOTTIME] = &clock_boottime,
[CLOCK_REALTIME_ALARM] = &alarm_clock,
[CLOCK_BOOTTIME_ALARM] = &alarm_clock,
[CLOCK_TAI] = &clock_tai,
};
static const struct k_clock *clockid_to_kclock(const clockid_t id)
{
clockid_t idx = id;
if (id < 0) {
return (id & CLOCKFD_MASK) == CLOCKFD ?
&clock_posix_dynamic : &clock_posix_cpu;
}
if (id >= ARRAY_SIZE(posix_clocks))
return NULL;
return posix_clocks[array_index_nospec(idx, ARRAY_SIZE(posix_clocks))];
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* Backlight Lowlevel Control Abstraction
*
* Copyright (C) 2003,2004 Hewlett-Packard Company
*
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/backlight.h>
#include <linux/notifier.h>
#include <linux/ctype.h>
#include <linux/err.h>
#include <linux/fb.h>
#include <linux/slab.h>
#ifdef CONFIG_PMAC_BACKLIGHT
#include <asm/backlight.h>
#endif
/**
* DOC: overview
*
* The backlight core supports implementing backlight drivers.
*
* A backlight driver registers a driver using
* devm_backlight_device_register(). The properties of the backlight
* driver such as type and max_brightness must be specified.
* When the core detect changes in for example brightness or power state
* the update_status() operation is called. The backlight driver shall
* implement this operation and use it to adjust backlight.
*
* Several sysfs attributes are provided by the backlight core::
*
* - brightness R/W, set the requested brightness level
* - actual_brightness RO, the brightness level used by the HW
* - max_brightness RO, the maximum brightness level supported
*
* See Documentation/ABI/stable/sysfs-class-backlight for the full list.
*
* The backlight can be adjusted using the sysfs interface, and
* the backlight driver may also support adjusting backlight using
* a hot-key or some other platform or firmware specific way.
*
* The driver must implement the get_brightness() operation if
* the HW do not support all the levels that can be specified in
* brightness, thus providing user-space access to the actual level
* via the actual_brightness attribute.
*
* When the backlight changes this is reported to user-space using
* an uevent connected to the actual_brightness attribute.
* When brightness is set by platform specific means, for example
* a hot-key to adjust backlight, the driver must notify the backlight
* core that brightness has changed using backlight_force_update().
*
* The backlight driver core receives notifications from fbdev and
* if the event is FB_EVENT_BLANK and if the value of blank, from the
* FBIOBLANK ioctrl, results in a change in the backlight state the
* update_status() operation is called.
*/
static struct list_head backlight_dev_list;
static struct mutex backlight_dev_list_mutex;
static struct blocking_notifier_head backlight_notifier;
static const char *const backlight_types[] = {
[BACKLIGHT_RAW] = "raw",
[BACKLIGHT_PLATFORM] = "platform",
[BACKLIGHT_FIRMWARE] = "firmware",
};
static const char *const backlight_scale_types[] = {
[BACKLIGHT_SCALE_UNKNOWN] = "unknown",
[BACKLIGHT_SCALE_LINEAR] = "linear",
[BACKLIGHT_SCALE_NON_LINEAR] = "non-linear",
};
#if defined(CONFIG_FB_CORE) || (defined(CONFIG_FB_CORE_MODULE) && \
defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE))
/*
* fb_notifier_callback
*
* This callback gets called when something important happens inside a
* framebuffer driver. The backlight core only cares about FB_BLANK_UNBLANK
* which is reported to the driver using backlight_update_status()
* as a state change.
*
* There may be several fbdev's connected to the backlight device,
* in which case they are kept track of. A state change is only reported
* if there is a change in backlight for the specified fbdev.
*/
static int fb_notifier_callback(struct notifier_block *self,
unsigned long event, void *data)
{
struct backlight_device *bd;
struct fb_event *evdata = data;
int node = evdata->info->node;
int fb_blank = 0;
/* If we aren't interested in this event, skip it immediately ... */
if (event != FB_EVENT_BLANK)
return 0;
bd = container_of(self, struct backlight_device, fb_notif);
mutex_lock(&bd->ops_lock);
if (!bd->ops)
goto out;
if (bd->ops->check_fb && !bd->ops->check_fb(bd, evdata->info))
goto out;
fb_blank = *(int *)evdata->data;
if (fb_blank == FB_BLANK_UNBLANK && !bd->fb_bl_on[node]) {
bd->fb_bl_on[node] = true;
if (!bd->use_count++) {
bd->props.state &= ~BL_CORE_FBBLANK;
bd->props.fb_blank = FB_BLANK_UNBLANK;
backlight_update_status(bd);
}
} else if (fb_blank != FB_BLANK_UNBLANK && bd->fb_bl_on[node]) {
bd->fb_bl_on[node] = false;
if (!(--bd->use_count)) {
bd->props.state |= BL_CORE_FBBLANK;
bd->props.fb_blank = fb_blank;
backlight_update_status(bd);
}
}
out:
mutex_unlock(&bd->ops_lock);
return 0;
}
static int backlight_register_fb(struct backlight_device *bd)
{
memset(&bd->fb_notif, 0, sizeof(bd->fb_notif));
bd->fb_notif.notifier_call = fb_notifier_callback;
return fb_register_client(&bd->fb_notif);
}
static void backlight_unregister_fb(struct backlight_device *bd)
{
fb_unregister_client(&bd->fb_notif);
}
#else
static inline int backlight_register_fb(struct backlight_device *bd)
{
return 0;
}
static inline void backlight_unregister_fb(struct backlight_device *bd)
{
}
#endif /* CONFIG_FB_CORE */
static void backlight_generate_event(struct backlight_device *bd,
enum backlight_update_reason reason)
{
char *envp[2];
switch (reason) {
case BACKLIGHT_UPDATE_SYSFS:
envp[0] = "SOURCE=sysfs";
break;
case BACKLIGHT_UPDATE_HOTKEY:
envp[0] = "SOURCE=hotkey";
break;
default:
envp[0] = "SOURCE=unknown";
break;
}
envp[1] = NULL;
kobject_uevent_env(&bd->dev.kobj, KOBJ_CHANGE, envp);
sysfs_notify(&bd->dev.kobj, NULL, "actual_brightness");
}
static ssize_t bl_power_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct backlight_device *bd = to_backlight_device(dev);
return sprintf(buf, "%d\n", bd->props.power);
}
static ssize_t bl_power_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
int rc;
struct backlight_device *bd = to_backlight_device(dev);
unsigned long power, old_power;
rc = kstrtoul(buf, 0, &power);
if (rc)
return rc;
rc = -ENXIO;
mutex_lock(&bd->ops_lock);
if (bd->ops) {
pr_debug("set power to %lu\n", power);
if (bd->props.power != power) {
old_power = bd->props.power;
bd->props.power = power;
rc = backlight_update_status(bd);
if (rc)
bd->props.power = old_power;
else
rc = count;
} else {
rc = count;
}
}
mutex_unlock(&bd->ops_lock);
return rc;
}
static DEVICE_ATTR_RW(bl_power);
static ssize_t brightness_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct backlight_device *bd = to_backlight_device(dev);
return sprintf(buf, "%d\n", bd->props.brightness);
}
int backlight_device_set_brightness(struct backlight_device *bd,
unsigned long brightness)
{
int rc = -ENXIO;
mutex_lock(&bd->ops_lock);
if (bd->ops) {
if (brightness > bd->props.max_brightness)
rc = -EINVAL;
else {
pr_debug("set brightness to %lu\n", brightness);
bd->props.brightness = brightness;
rc = backlight_update_status(bd);
}
}
mutex_unlock(&bd->ops_lock);
backlight_generate_event(bd, BACKLIGHT_UPDATE_SYSFS);
return rc;
}
EXPORT_SYMBOL(backlight_device_set_brightness);
static ssize_t brightness_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
int rc;
struct backlight_device *bd = to_backlight_device(dev);
unsigned long brightness;
rc = kstrtoul(buf, 0, &brightness);
if (rc)
return rc;
rc = backlight_device_set_brightness(bd, brightness);
return rc ? rc : count;
}
static DEVICE_ATTR_RW(brightness);
static ssize_t type_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct backlight_device *bd = to_backlight_device(dev);
return sprintf(buf, "%s\n", backlight_types[bd->props.type]);
}
static DEVICE_ATTR_RO(type);
static ssize_t max_brightness_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct backlight_device *bd = to_backlight_device(dev);
return sprintf(buf, "%d\n", bd->props.max_brightness);
}
static DEVICE_ATTR_RO(max_brightness);
static ssize_t actual_brightness_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
int rc = -ENXIO;
struct backlight_device *bd = to_backlight_device(dev);
mutex_lock(&bd->ops_lock);
if (bd->ops && bd->ops->get_brightness) {
rc = bd->ops->get_brightness(bd);
if (rc >= 0)
rc = sprintf(buf, "%d\n", rc);
} else {
rc = sprintf(buf, "%d\n", bd->props.brightness);
}
mutex_unlock(&bd->ops_lock);
return rc;
}
static DEVICE_ATTR_RO(actual_brightness);
static ssize_t scale_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct backlight_device *bd = to_backlight_device(dev);
if (WARN_ON(bd->props.scale > BACKLIGHT_SCALE_NON_LINEAR))
return sprintf(buf, "unknown\n");
return sprintf(buf, "%s\n", backlight_scale_types[bd->props.scale]);
}
static DEVICE_ATTR_RO(scale);
static struct class *backlight_class;
#ifdef CONFIG_PM_SLEEP
static int backlight_suspend(struct device *dev)
{
struct backlight_device *bd = to_backlight_device(dev);
mutex_lock(&bd->ops_lock);
if (bd->ops && bd->ops->options & BL_CORE_SUSPENDRESUME) {
bd->props.state |= BL_CORE_SUSPENDED;
backlight_update_status(bd);
}
mutex_unlock(&bd->ops_lock);
return 0;
}
static int backlight_resume(struct device *dev)
{
struct backlight_device *bd = to_backlight_device(dev);
mutex_lock(&bd->ops_lock);
if (bd->ops && bd->ops->options & BL_CORE_SUSPENDRESUME) {
bd->props.state &= ~BL_CORE_SUSPENDED;
backlight_update_status(bd);
}
mutex_unlock(&bd->ops_lock);
return 0;
}
#endif
static SIMPLE_DEV_PM_OPS(backlight_class_dev_pm_ops, backlight_suspend,
backlight_resume);
static void bl_device_release(struct device *dev)
{
struct backlight_device *bd = to_backlight_device(dev);
kfree(bd);
}
static struct attribute *bl_device_attrs[] = {
&dev_attr_bl_power.attr,
&dev_attr_brightness.attr,
&dev_attr_actual_brightness.attr,
&dev_attr_max_brightness.attr,
&dev_attr_scale.attr,
&dev_attr_type.attr,
NULL,
};
ATTRIBUTE_GROUPS(bl_device);
/**
* backlight_force_update - tell the backlight subsystem that hardware state
* has changed
* @bd: the backlight device to update
* @reason: reason for update
*
* Updates the internal state of the backlight in response to a hardware event,
* and generates an uevent to notify userspace. A backlight driver shall call
* backlight_force_update() when the backlight is changed using, for example,
* a hot-key. The updated brightness is read using get_brightness() and the
* brightness value is reported using an uevent.
*/
void backlight_force_update(struct backlight_device *bd,
enum backlight_update_reason reason)
{
int brightness;
mutex_lock(&bd->ops_lock);
if (bd->ops && bd->ops->get_brightness) {
brightness = bd->ops->get_brightness(bd);
if (brightness >= 0)
bd->props.brightness = brightness;
else
dev_err(&bd->dev,
"Could not update brightness from device: %pe\n",
ERR_PTR(brightness));
}
mutex_unlock(&bd->ops_lock);
backlight_generate_event(bd, reason);
}
EXPORT_SYMBOL(backlight_force_update);
/* deprecated - use devm_backlight_device_register() */
struct backlight_device *backlight_device_register(const char *name,
struct device *parent, void *devdata, const struct backlight_ops *ops,
const struct backlight_properties *props)
{
struct backlight_device *new_bd;
int rc;
pr_debug("backlight_device_register: name=%s\n", name);
new_bd = kzalloc(sizeof(struct backlight_device), GFP_KERNEL);
if (!new_bd)
return ERR_PTR(-ENOMEM);
mutex_init(&new_bd->update_lock);
mutex_init(&new_bd->ops_lock);
new_bd->dev.class = backlight_class;
new_bd->dev.parent = parent;
new_bd->dev.release = bl_device_release;
dev_set_name(&new_bd->dev, "%s", name);
dev_set_drvdata(&new_bd->dev, devdata);
/* Set default properties */
if (props) {
memcpy(&new_bd->props, props,
sizeof(struct backlight_properties));
if (props->type <= 0 || props->type >= BACKLIGHT_TYPE_MAX) {
WARN(1, "%s: invalid backlight type", name);
new_bd->props.type = BACKLIGHT_RAW;
}
} else {
new_bd->props.type = BACKLIGHT_RAW;
}
rc = device_register(&new_bd->dev);
if (rc) {
put_device(&new_bd->dev);
return ERR_PTR(rc);
}
rc = backlight_register_fb(new_bd);
if (rc) {
device_unregister(&new_bd->dev);
return ERR_PTR(rc);
}
new_bd->ops = ops;
#ifdef CONFIG_PMAC_BACKLIGHT
mutex_lock(&pmac_backlight_mutex);
if (!pmac_backlight)
pmac_backlight = new_bd;
mutex_unlock(&pmac_backlight_mutex);
#endif
mutex_lock(&backlight_dev_list_mutex);
list_add(&new_bd->entry, &backlight_dev_list);
mutex_unlock(&backlight_dev_list_mutex);
blocking_notifier_call_chain(&backlight_notifier,
BACKLIGHT_REGISTERED, new_bd);
return new_bd;
}
EXPORT_SYMBOL(backlight_device_register);
/** backlight_device_get_by_type - find first backlight device of a type
* @type: the type of backlight device
*
* Look up the first backlight device of the specified type
*
* RETURNS:
*
* Pointer to backlight device if any was found. Otherwise NULL.
*/
struct backlight_device *backlight_device_get_by_type(enum backlight_type type)
{
bool found = false;
struct backlight_device *bd;
mutex_lock(&backlight_dev_list_mutex);
list_for_each_entry(bd, &backlight_dev_list, entry) {
if (bd->props.type == type) {
found = true;
break;
}
}
mutex_unlock(&backlight_dev_list_mutex);
return found ? bd : NULL;
}
EXPORT_SYMBOL(backlight_device_get_by_type);
/**
* backlight_device_get_by_name - Get backlight device by name
* @name: Device name
*
* This function looks up a backlight device by its name. It obtains a reference
* on the backlight device and it is the caller's responsibility to drop the
* reference by calling put_device().
*
* Returns:
* A pointer to the backlight device if found, otherwise NULL.
*/
struct backlight_device *backlight_device_get_by_name(const char *name)
{
struct device *dev;
dev = class_find_device_by_name(backlight_class, name);
return dev ? to_backlight_device(dev) : NULL;
}
EXPORT_SYMBOL(backlight_device_get_by_name);
/* deprecated - use devm_backlight_device_unregister() */
void backlight_device_unregister(struct backlight_device *bd)
{
if (!bd)
return;
mutex_lock(&backlight_dev_list_mutex);
list_del(&bd->entry);
mutex_unlock(&backlight_dev_list_mutex);
#ifdef CONFIG_PMAC_BACKLIGHT
mutex_lock(&pmac_backlight_mutex);
if (pmac_backlight == bd)
pmac_backlight = NULL;
mutex_unlock(&pmac_backlight_mutex);
#endif
blocking_notifier_call_chain(&backlight_notifier,
BACKLIGHT_UNREGISTERED, bd);
mutex_lock(&bd->ops_lock);
bd->ops = NULL;
mutex_unlock(&bd->ops_lock);
backlight_unregister_fb(bd);
device_unregister(&bd->dev);
}
EXPORT_SYMBOL(backlight_device_unregister);
static void devm_backlight_device_release(struct device *dev, void *res)
{
struct backlight_device *backlight = *(struct backlight_device **)res;
backlight_device_unregister(backlight);
}
static int devm_backlight_device_match(struct device *dev, void *res,
void *data)
{
struct backlight_device **r = res;
return *r == data;
}
/**
* backlight_register_notifier - get notified of backlight (un)registration
* @nb: notifier block with the notifier to call on backlight (un)registration
*
* Register a notifier to get notified when backlight devices get registered
* or unregistered.
*
* RETURNS:
*
* 0 on success, otherwise a negative error code
*/
int backlight_register_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_register(&backlight_notifier, nb);
}
EXPORT_SYMBOL(backlight_register_notifier);
/**
* backlight_unregister_notifier - unregister a backlight notifier
* @nb: notifier block to unregister
*
* Register a notifier to get notified when backlight devices get registered
* or unregistered.
*
* RETURNS:
*
* 0 on success, otherwise a negative error code
*/
int backlight_unregister_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(&backlight_notifier, nb);
}
EXPORT_SYMBOL(backlight_unregister_notifier);
/**
* devm_backlight_device_register - register a new backlight device
* @dev: the device to register
* @name: the name of the device
* @parent: a pointer to the parent device (often the same as @dev)
* @devdata: an optional pointer to be stored for private driver use
* @ops: the backlight operations structure
* @props: the backlight properties
*
* Creates and registers new backlight device. When a backlight device
* is registered the configuration must be specified in the @props
* parameter. See description of &backlight_properties.
*
* RETURNS:
*
* struct backlight on success, or an ERR_PTR on error
*/
struct backlight_device *devm_backlight_device_register(struct device *dev,
const char *name, struct device *parent, void *devdata,
const struct backlight_ops *ops,
const struct backlight_properties *props)
{
struct backlight_device **ptr, *backlight;
ptr = devres_alloc(devm_backlight_device_release, sizeof(*ptr),
GFP_KERNEL);
if (!ptr)
return ERR_PTR(-ENOMEM);
backlight = backlight_device_register(name, parent, devdata, ops,
props);
if (!IS_ERR(backlight)) {
*ptr = backlight;
devres_add(dev, ptr);
} else {
devres_free(ptr);
}
return backlight;
}
EXPORT_SYMBOL(devm_backlight_device_register);
/**
* devm_backlight_device_unregister - unregister backlight device
* @dev: the device to unregister
* @bd: the backlight device to unregister
*
* Deallocates a backlight allocated with devm_backlight_device_register().
* Normally this function will not need to be called and the resource management
* code will ensure that the resources are freed.
*/
void devm_backlight_device_unregister(struct device *dev,
struct backlight_device *bd)
{
int rc;
rc = devres_release(dev, devm_backlight_device_release,
devm_backlight_device_match, bd);
WARN_ON(rc);
}
EXPORT_SYMBOL(devm_backlight_device_unregister);
#ifdef CONFIG_OF
static int of_parent_match(struct device *dev, const void *data)
{
return dev->parent && dev->parent->of_node == data;
}
/**
* of_find_backlight_by_node() - find backlight device by device-tree node
* @node: device-tree node of the backlight device
*
* Returns a pointer to the backlight device corresponding to the given DT
* node or NULL if no such backlight device exists or if the device hasn't
* been probed yet.
*
* This function obtains a reference on the backlight device and it is the
* caller's responsibility to drop the reference by calling put_device() on
* the backlight device's .dev field.
*/
struct backlight_device *of_find_backlight_by_node(struct device_node *node)
{
struct device *dev;
dev = class_find_device(backlight_class, NULL, node, of_parent_match);
return dev ? to_backlight_device(dev) : NULL;
}
EXPORT_SYMBOL(of_find_backlight_by_node);
#endif
static struct backlight_device *of_find_backlight(struct device *dev)
{
struct backlight_device *bd = NULL;
struct device_node *np;
if (!dev)
return NULL;
if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
np = of_parse_phandle(dev->of_node, "backlight", 0);
if (np) {
bd = of_find_backlight_by_node(np);
of_node_put(np);
if (!bd)
return ERR_PTR(-EPROBE_DEFER);
}
}
return bd;
}
static void devm_backlight_release(void *data)
{
struct backlight_device *bd = data;
put_device(&bd->dev);
}
/**
* devm_of_find_backlight - find backlight for a device
* @dev: the device
*
* This function looks for a property named 'backlight' on the DT node
* connected to @dev and looks up the backlight device. The lookup is
* device managed so the reference to the backlight device is automatically
* dropped on driver detach.
*
* RETURNS:
*
* A pointer to the backlight device if found.
* Error pointer -EPROBE_DEFER if the DT property is set, but no backlight
* device is found. NULL if there's no backlight property.
*/
struct backlight_device *devm_of_find_backlight(struct device *dev)
{
struct backlight_device *bd;
int ret;
bd = of_find_backlight(dev);
if (IS_ERR_OR_NULL(bd))
return bd;
ret = devm_add_action_or_reset(dev, devm_backlight_release, bd);
if (ret)
return ERR_PTR(ret);
return bd;
}
EXPORT_SYMBOL(devm_of_find_backlight);
static void __exit backlight_class_exit(void)
{
class_destroy(backlight_class);
}
static int __init backlight_class_init(void)
{
backlight_class = class_create("backlight");
if (IS_ERR(backlight_class)) {
pr_warn("Unable to create backlight class; errno = %ld\n",
PTR_ERR(backlight_class));
return PTR_ERR(backlight_class);
}
backlight_class->dev_groups = bl_device_groups;
backlight_class->pm = &backlight_class_dev_pm_ops;
INIT_LIST_HEAD(&backlight_dev_list);
mutex_init(&backlight_dev_list_mutex);
BLOCKING_INIT_NOTIFIER_HEAD(&backlight_notifier);
return 0;
}
/*
* if this is compiled into the kernel, we need to ensure that the
* class is registered before users of the class try to register lcd's
*/
postcore_initcall(backlight_class_init);
module_exit(backlight_class_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Jamey Hicks <jamey.hicks@hp.com>, Andrew Zabolotny <zap@homelink.ru>");
MODULE_DESCRIPTION("Backlight Lowlevel Control Abstraction");
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Christian Brauner <brauner@kernel.org> */
#include <linux/cred.h>
#include <linux/fs.h>
#include <linux/mnt_idmapping.h>
#include <linux/slab.h>
#include <linux/user_namespace.h>
#include "internal.h"
/*
* Outside of this file vfs{g,u}id_t are always created from k{g,u}id_t,
* never from raw values. These are just internal helpers.
*/
#define VFSUIDT_INIT_RAW(val) (vfsuid_t){ val }
#define VFSGIDT_INIT_RAW(val) (vfsgid_t){ val }
struct mnt_idmap {
struct uid_gid_map uid_map;
struct uid_gid_map gid_map;
refcount_t count;
};
/*
* Carries the initial idmapping of 0:0:4294967295 which is an identity
* mapping. This means that {g,u}id 0 is mapped to {g,u}id 0, {g,u}id 1 is
* mapped to {g,u}id 1, [...], {g,u}id 1000 to {g,u}id 1000, [...].
*/
struct mnt_idmap nop_mnt_idmap = {
.count = REFCOUNT_INIT(1),
};
EXPORT_SYMBOL_GPL(nop_mnt_idmap);
/**
* initial_idmapping - check whether this is the initial mapping
* @ns: idmapping to check
*
* Check whether this is the initial mapping, mapping 0 to 0, 1 to 1,
* [...], 1000 to 1000 [...].
*
* Return: true if this is the initial mapping, false if not.
*/
static inline bool initial_idmapping(const struct user_namespace *ns)
{
return ns == &init_user_ns;
}
/**
* make_vfsuid - map a filesystem kuid according to an idmapping
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
* @kuid : kuid to be mapped
*
* Take a @kuid and remap it from @fs_userns into @idmap. Use this
* function when preparing a @kuid to be reported to userspace.
*
* If initial_idmapping() determines that this is not an idmapped mount
* we can simply return @kuid unchanged.
* If initial_idmapping() tells us that the filesystem is not mounted with an
* idmapping we know the value of @kuid won't change when calling
* from_kuid() so we can simply retrieve the value via __kuid_val()
* directly.
*
* Return: @kuid mapped according to @idmap.
* If @kuid has no mapping in either @idmap or @fs_userns INVALID_UID is
* returned.
*/
vfsuid_t make_vfsuid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns,
kuid_t kuid)
{
uid_t uid;
if (idmap == &nop_mnt_idmap)
return VFSUIDT_INIT(kuid); if (initial_idmapping(fs_userns))
uid = __kuid_val(kuid);
else
uid = from_kuid(fs_userns, kuid); if (uid == (uid_t)-1)
return INVALID_VFSUID;
return VFSUIDT_INIT_RAW(map_id_down(&idmap->uid_map, uid));
}
EXPORT_SYMBOL_GPL(make_vfsuid);
/**
* make_vfsgid - map a filesystem kgid according to an idmapping
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
* @kgid : kgid to be mapped
*
* Take a @kgid and remap it from @fs_userns into @idmap. Use this
* function when preparing a @kgid to be reported to userspace.
*
* If initial_idmapping() determines that this is not an idmapped mount
* we can simply return @kgid unchanged.
* If initial_idmapping() tells us that the filesystem is not mounted with an
* idmapping we know the value of @kgid won't change when calling
* from_kgid() so we can simply retrieve the value via __kgid_val()
* directly.
*
* Return: @kgid mapped according to @idmap.
* If @kgid has no mapping in either @idmap or @fs_userns INVALID_GID is
* returned.
*/
vfsgid_t make_vfsgid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, kgid_t kgid)
{
gid_t gid;
if (idmap == &nop_mnt_idmap)
return VFSGIDT_INIT(kgid); if (initial_idmapping(fs_userns))
gid = __kgid_val(kgid);
else
gid = from_kgid(fs_userns, kgid); if (gid == (gid_t)-1)
return INVALID_VFSGID;
return VFSGIDT_INIT_RAW(map_id_down(&idmap->gid_map, gid));
}
EXPORT_SYMBOL_GPL(make_vfsgid);
/**
* from_vfsuid - map a vfsuid into the filesystem idmapping
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
* @vfsuid : vfsuid to be mapped
*
* Map @vfsuid into the filesystem idmapping. This function has to be used in
* order to e.g. write @vfsuid to inode->i_uid.
*
* Return: @vfsuid mapped into the filesystem idmapping
*/
kuid_t from_vfsuid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, vfsuid_t vfsuid)
{
uid_t uid;
if (idmap == &nop_mnt_idmap)
return AS_KUIDT(vfsuid);
uid = map_id_up(&idmap->uid_map, __vfsuid_val(vfsuid));
if (uid == (uid_t)-1)
return INVALID_UID;
if (initial_idmapping(fs_userns)) return KUIDT_INIT(uid); return make_kuid(fs_userns, uid);
}
EXPORT_SYMBOL_GPL(from_vfsuid);
/**
* from_vfsgid - map a vfsgid into the filesystem idmapping
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
* @vfsgid : vfsgid to be mapped
*
* Map @vfsgid into the filesystem idmapping. This function has to be used in
* order to e.g. write @vfsgid to inode->i_gid.
*
* Return: @vfsgid mapped into the filesystem idmapping
*/
kgid_t from_vfsgid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, vfsgid_t vfsgid)
{
gid_t gid;
if (idmap == &nop_mnt_idmap)
return AS_KGIDT(vfsgid);
gid = map_id_up(&idmap->gid_map, __vfsgid_val(vfsgid));
if (gid == (gid_t)-1)
return INVALID_GID;
if (initial_idmapping(fs_userns)) return KGIDT_INIT(gid); return make_kgid(fs_userns, gid);
}
EXPORT_SYMBOL_GPL(from_vfsgid);
#ifdef CONFIG_MULTIUSER
/**
* vfsgid_in_group_p() - check whether a vfsuid matches the caller's groups
* @vfsgid: the mnt gid to match
*
* This function can be used to determine whether @vfsuid matches any of the
* caller's groups.
*
* Return: 1 if vfsuid matches caller's groups, 0 if not.
*/
int vfsgid_in_group_p(vfsgid_t vfsgid)
{
return in_group_p(AS_KGIDT(vfsgid));
}
#else
int vfsgid_in_group_p(vfsgid_t vfsgid)
{
return 1;
}
#endif
EXPORT_SYMBOL_GPL(vfsgid_in_group_p);
static int copy_mnt_idmap(struct uid_gid_map *map_from,
struct uid_gid_map *map_to)
{
struct uid_gid_extent *forward, *reverse;
u32 nr_extents = READ_ONCE(map_from->nr_extents);
/* Pairs with smp_wmb() when writing the idmapping. */
smp_rmb();
/*
* Don't blindly copy @map_to into @map_from if nr_extents is
* smaller or equal to UID_GID_MAP_MAX_BASE_EXTENTS. Since we
* read @nr_extents someone could have written an idmapping and
* then we might end up with inconsistent data. So just don't do
* anything at all.
*/
if (nr_extents == 0)
return -EINVAL;
/*
* Here we know that nr_extents is greater than zero which means
* a map has been written. Since idmappings can't be changed
* once they have been written we know that we can safely copy
* from @map_to into @map_from.
*/
if (nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS) {
*map_to = *map_from;
return 0;
}
forward = kmemdup(map_from->forward,
nr_extents * sizeof(struct uid_gid_extent),
GFP_KERNEL_ACCOUNT);
if (!forward)
return -ENOMEM;
reverse = kmemdup(map_from->reverse,
nr_extents * sizeof(struct uid_gid_extent),
GFP_KERNEL_ACCOUNT);
if (!reverse) {
kfree(forward);
return -ENOMEM;
}
/*
* The idmapping isn't exposed anywhere so we don't need to care
* about ordering between extent pointers and @nr_extents
* initialization.
*/
map_to->forward = forward;
map_to->reverse = reverse;
map_to->nr_extents = nr_extents;
return 0;
}
static void free_mnt_idmap(struct mnt_idmap *idmap)
{
if (idmap->uid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) {
kfree(idmap->uid_map.forward);
kfree(idmap->uid_map.reverse);
}
if (idmap->gid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) {
kfree(idmap->gid_map.forward);
kfree(idmap->gid_map.reverse);
}
kfree(idmap);
}
struct mnt_idmap *alloc_mnt_idmap(struct user_namespace *mnt_userns)
{
struct mnt_idmap *idmap;
int ret;
idmap = kzalloc(sizeof(struct mnt_idmap), GFP_KERNEL_ACCOUNT);
if (!idmap)
return ERR_PTR(-ENOMEM);
refcount_set(&idmap->count, 1);
ret = copy_mnt_idmap(&mnt_userns->uid_map, &idmap->uid_map);
if (!ret)
ret = copy_mnt_idmap(&mnt_userns->gid_map, &idmap->gid_map);
if (ret) {
free_mnt_idmap(idmap);
idmap = ERR_PTR(ret);
}
return idmap;
}
/**
* mnt_idmap_get - get a reference to an idmapping
* @idmap: the idmap to bump the reference on
*
* If @idmap is not the @nop_mnt_idmap bump the reference count.
*
* Return: @idmap with reference count bumped if @not_mnt_idmap isn't passed.
*/
struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap)
{
if (idmap != &nop_mnt_idmap)
refcount_inc(&idmap->count);
return idmap;
}
EXPORT_SYMBOL_GPL(mnt_idmap_get);
/**
* mnt_idmap_put - put a reference to an idmapping
* @idmap: the idmap to put the reference on
*
* If this is a non-initial idmapping, put the reference count when a mount is
* released and free it if we're the last user.
*/
void mnt_idmap_put(struct mnt_idmap *idmap)
{
if (idmap != &nop_mnt_idmap && refcount_dec_and_test(&idmap->count))
free_mnt_idmap(idmap);
}
EXPORT_SYMBOL_GPL(mnt_idmap_put);
// SPDX-License-Identifier: GPL-2.0-only
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/clock.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/timer.h>
#include <linux/acpi_pmtmr.h>
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/clocksource.h>
#include <linux/percpu.h>
#include <linux/timex.h>
#include <linux/static_key.h>
#include <linux/static_call.h>
#include <asm/hpet.h>
#include <asm/timer.h>
#include <asm/vgtod.h>
#include <asm/time.h>
#include <asm/delay.h>
#include <asm/hypervisor.h>
#include <asm/nmi.h>
#include <asm/x86_init.h>
#include <asm/geode.h>
#include <asm/apic.h>
#include <asm/intel-family.h>
#include <asm/i8259.h>
#include <asm/uv/uv.h>
unsigned int __read_mostly cpu_khz; /* TSC clocks / usec, not used here */
EXPORT_SYMBOL(cpu_khz);
unsigned int __read_mostly tsc_khz;
EXPORT_SYMBOL(tsc_khz);
#define KHZ 1000
/*
* TSC can be unstable due to cpufreq or due to unsynced TSCs
*/
static int __read_mostly tsc_unstable;
static unsigned int __initdata tsc_early_khz;
static DEFINE_STATIC_KEY_FALSE(__use_tsc);
int tsc_clocksource_reliable;
static int __read_mostly tsc_force_recalibrate;
static u32 art_to_tsc_numerator;
static u32 art_to_tsc_denominator;
static u64 art_to_tsc_offset;
static bool have_art;
struct cyc2ns {
struct cyc2ns_data data[2]; /* 0 + 2*16 = 32 */
seqcount_latch_t seq; /* 32 + 4 = 36 */
}; /* fits one cacheline */
static DEFINE_PER_CPU_ALIGNED(struct cyc2ns, cyc2ns);
static int __init tsc_early_khz_setup(char *buf)
{
return kstrtouint(buf, 0, &tsc_early_khz);
}
early_param("tsc_early_khz", tsc_early_khz_setup);
__always_inline void __cyc2ns_read(struct cyc2ns_data *data)
{
int seq, idx;
do {
seq = this_cpu_read(cyc2ns.seq.seqcount.sequence);
idx = seq & 1;
data->cyc2ns_offset = this_cpu_read(cyc2ns.data[idx].cyc2ns_offset);
data->cyc2ns_mul = this_cpu_read(cyc2ns.data[idx].cyc2ns_mul);
data->cyc2ns_shift = this_cpu_read(cyc2ns.data[idx].cyc2ns_shift);
} while (unlikely(seq != this_cpu_read(cyc2ns.seq.seqcount.sequence)));
}
__always_inline void cyc2ns_read_begin(struct cyc2ns_data *data)
{
preempt_disable_notrace();
__cyc2ns_read(data);
}
__always_inline void cyc2ns_read_end(void)
{
preempt_enable_notrace();
}
/*
* Accelerators for sched_clock()
* convert from cycles(64bits) => nanoseconds (64bits)
* basic equation:
* ns = cycles / (freq / ns_per_sec)
* ns = cycles * (ns_per_sec / freq)
* ns = cycles * (10^9 / (cpu_khz * 10^3))
* ns = cycles * (10^6 / cpu_khz)
*
* Then we use scaling math (suggested by george@mvista.com) to get:
* ns = cycles * (10^6 * SC / cpu_khz) / SC
* ns = cycles * cyc2ns_scale / SC
*
* And since SC is a constant power of two, we can convert the div
* into a shift. The larger SC is, the more accurate the conversion, but
* cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication
* (64-bit result) can be used.
*
* We can use khz divisor instead of mhz to keep a better precision.
* (mathieu.desnoyers@polymtl.ca)
*
* -johnstul@us.ibm.com "math is hard, lets go shopping!"
*/
static __always_inline unsigned long long __cycles_2_ns(unsigned long long cyc)
{
struct cyc2ns_data data;
unsigned long long ns;
__cyc2ns_read(&data);
ns = data.cyc2ns_offset;
ns += mul_u64_u32_shr(cyc, data.cyc2ns_mul, data.cyc2ns_shift);
return ns;
}
static __always_inline unsigned long long cycles_2_ns(unsigned long long cyc)
{
unsigned long long ns;
preempt_disable_notrace();
ns = __cycles_2_ns(cyc);
preempt_enable_notrace();
return ns;
}
static void __set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now)
{
unsigned long long ns_now;
struct cyc2ns_data data;
struct cyc2ns *c2n;
ns_now = cycles_2_ns(tsc_now);
/*
* Compute a new multiplier as per the above comment and ensure our
* time function is continuous; see the comment near struct
* cyc2ns_data.
*/
clocks_calc_mult_shift(&data.cyc2ns_mul, &data.cyc2ns_shift, khz,
NSEC_PER_MSEC, 0);
/*
* cyc2ns_shift is exported via arch_perf_update_userpage() where it is
* not expected to be greater than 31 due to the original published
* conversion algorithm shifting a 32-bit value (now specifies a 64-bit
* value) - refer perf_event_mmap_page documentation in perf_event.h.
*/
if (data.cyc2ns_shift == 32) {
data.cyc2ns_shift = 31;
data.cyc2ns_mul >>= 1;
}
data.cyc2ns_offset = ns_now -
mul_u64_u32_shr(tsc_now, data.cyc2ns_mul, data.cyc2ns_shift);
c2n = per_cpu_ptr(&cyc2ns, cpu);
raw_write_seqcount_latch(&c2n->seq);
c2n->data[0] = data;
raw_write_seqcount_latch(&c2n->seq);
c2n->data[1] = data;
}
static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now)
{
unsigned long flags;
local_irq_save(flags);
sched_clock_idle_sleep_event();
if (khz)
__set_cyc2ns_scale(khz, cpu, tsc_now);
sched_clock_idle_wakeup_event();
local_irq_restore(flags);
}
/*
* Initialize cyc2ns for boot cpu
*/
static void __init cyc2ns_init_boot_cpu(void)
{
struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns);
seqcount_latch_init(&c2n->seq);
__set_cyc2ns_scale(tsc_khz, smp_processor_id(), rdtsc());
}
/*
* Secondary CPUs do not run through tsc_init(), so set up
* all the scale factors for all CPUs, assuming the same
* speed as the bootup CPU.
*/
static void __init cyc2ns_init_secondary_cpus(void)
{
unsigned int cpu, this_cpu = smp_processor_id();
struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns);
struct cyc2ns_data *data = c2n->data;
for_each_possible_cpu(cpu) {
if (cpu != this_cpu) {
seqcount_latch_init(&c2n->seq);
c2n = per_cpu_ptr(&cyc2ns, cpu);
c2n->data[0] = data[0];
c2n->data[1] = data[1];
}
}
}
/*
* Scheduler clock - returns current time in nanosec units.
*/
noinstr u64 native_sched_clock(void)
{
if (static_branch_likely(&__use_tsc)) {
u64 tsc_now = rdtsc();
/* return the value in ns */
return __cycles_2_ns(tsc_now);
}
/*
* Fall back to jiffies if there's no TSC available:
* ( But note that we still use it if the TSC is marked
* unstable. We do this because unlike Time Of Day,
* the scheduler clock tolerates small errors and it's
* very important for it to be as fast as the platform
* can achieve it. )
*/
/* No locking but a rare wrong value is not a big deal: */
return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
}
/*
* Generate a sched_clock if you already have a TSC value.
*/
u64 native_sched_clock_from_tsc(u64 tsc)
{
return cycles_2_ns(tsc);
}
/* We need to define a real function for sched_clock, to override the
weak default version */
#ifdef CONFIG_PARAVIRT
noinstr u64 sched_clock_noinstr(void)
{
return paravirt_sched_clock();
}
bool using_native_sched_clock(void)
{
return static_call_query(pv_sched_clock) == native_sched_clock;
}
#else
u64 sched_clock_noinstr(void) __attribute__((alias("native_sched_clock")));
bool using_native_sched_clock(void) { return true; }
#endif
notrace u64 sched_clock(void)
{
u64 now;
preempt_disable_notrace();
now = sched_clock_noinstr();
preempt_enable_notrace(); return now;
}
int check_tsc_unstable(void)
{
return tsc_unstable;
}
EXPORT_SYMBOL_GPL(check_tsc_unstable);
#ifdef CONFIG_X86_TSC
int __init notsc_setup(char *str)
{
mark_tsc_unstable("boot parameter notsc");
return 1;
}
#else
/*
* disable flag for tsc. Takes effect by clearing the TSC cpu flag
* in cpu/common.c
*/
int __init notsc_setup(char *str)
{
setup_clear_cpu_cap(X86_FEATURE_TSC);
return 1;
}
#endif
__setup("notsc", notsc_setup);
static int no_sched_irq_time;
static int no_tsc_watchdog;
static int tsc_as_watchdog;
static int __init tsc_setup(char *str)
{
if (!strcmp(str, "reliable"))
tsc_clocksource_reliable = 1;
if (!strncmp(str, "noirqtime", 9))
no_sched_irq_time = 1;
if (!strcmp(str, "unstable"))
mark_tsc_unstable("boot parameter");
if (!strcmp(str, "nowatchdog")) {
no_tsc_watchdog = 1;
if (tsc_as_watchdog)
pr_alert("%s: Overriding earlier tsc=watchdog with tsc=nowatchdog\n",
__func__);
tsc_as_watchdog = 0;
}
if (!strcmp(str, "recalibrate"))
tsc_force_recalibrate = 1;
if (!strcmp(str, "watchdog")) {
if (no_tsc_watchdog)
pr_alert("%s: tsc=watchdog overridden by earlier tsc=nowatchdog\n",
__func__);
else
tsc_as_watchdog = 1;
}
return 1;
}
__setup("tsc=", tsc_setup);
#define MAX_RETRIES 5
#define TSC_DEFAULT_THRESHOLD 0x20000
/*
* Read TSC and the reference counters. Take care of any disturbances
*/
static u64 tsc_read_refs(u64 *p, int hpet)
{
u64 t1, t2;
u64 thresh = tsc_khz ? tsc_khz >> 5 : TSC_DEFAULT_THRESHOLD;
int i;
for (i = 0; i < MAX_RETRIES; i++) {
t1 = get_cycles();
if (hpet)
*p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
else
*p = acpi_pm_read_early();
t2 = get_cycles();
if ((t2 - t1) < thresh)
return t2;
}
return ULLONG_MAX;
}
/*
* Calculate the TSC frequency from HPET reference
*/
static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2)
{
u64 tmp;
if (hpet2 < hpet1)
hpet2 += 0x100000000ULL;
hpet2 -= hpet1;
tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
do_div(tmp, 1000000);
deltatsc = div64_u64(deltatsc, tmp);
return (unsigned long) deltatsc;
}
/*
* Calculate the TSC frequency from PMTimer reference
*/
static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2)
{
u64 tmp;
if (!pm1 && !pm2)
return ULONG_MAX;
if (pm2 < pm1)
pm2 += (u64)ACPI_PM_OVRRUN;
pm2 -= pm1;
tmp = pm2 * 1000000000LL;
do_div(tmp, PMTMR_TICKS_PER_SEC);
do_div(deltatsc, tmp);
return (unsigned long) deltatsc;
}
#define CAL_MS 10
#define CAL_LATCH (PIT_TICK_RATE / (1000 / CAL_MS))
#define CAL_PIT_LOOPS 1000
#define CAL2_MS 50
#define CAL2_LATCH (PIT_TICK_RATE / (1000 / CAL2_MS))
#define CAL2_PIT_LOOPS 5000
/*
* Try to calibrate the TSC against the Programmable
* Interrupt Timer and return the frequency of the TSC
* in kHz.
*
* Return ULONG_MAX on failure to calibrate.
*/
static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
{
u64 tsc, t1, t2, delta;
unsigned long tscmin, tscmax;
int pitcnt;
if (!has_legacy_pic()) {
/*
* Relies on tsc_early_delay_calibrate() to have given us semi
* usable udelay(), wait for the same 50ms we would have with
* the PIT loop below.
*/
udelay(10 * USEC_PER_MSEC);
udelay(10 * USEC_PER_MSEC);
udelay(10 * USEC_PER_MSEC);
udelay(10 * USEC_PER_MSEC);
udelay(10 * USEC_PER_MSEC);
return ULONG_MAX;
}
/* Set the Gate high, disable speaker */
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
/*
* Setup CTC channel 2* for mode 0, (interrupt on terminal
* count mode), binary count. Set the latch register to 50ms
* (LSB then MSB) to begin countdown.
*/
outb(0xb0, 0x43);
outb(latch & 0xff, 0x42);
outb(latch >> 8, 0x42);
tsc = t1 = t2 = get_cycles();
pitcnt = 0;
tscmax = 0;
tscmin = ULONG_MAX;
while ((inb(0x61) & 0x20) == 0) {
t2 = get_cycles();
delta = t2 - tsc;
tsc = t2;
if ((unsigned long) delta < tscmin)
tscmin = (unsigned int) delta;
if ((unsigned long) delta > tscmax)
tscmax = (unsigned int) delta;
pitcnt++;
}
/*
* Sanity checks:
*
* If we were not able to read the PIT more than loopmin
* times, then we have been hit by a massive SMI
*
* If the maximum is 10 times larger than the minimum,
* then we got hit by an SMI as well.
*/
if (pitcnt < loopmin || tscmax > 10 * tscmin)
return ULONG_MAX;
/* Calculate the PIT value */
delta = t2 - t1;
do_div(delta, ms);
return delta;
}
/*
* This reads the current MSB of the PIT counter, and
* checks if we are running on sufficiently fast and
* non-virtualized hardware.
*
* Our expectations are:
*
* - the PIT is running at roughly 1.19MHz
*
* - each IO is going to take about 1us on real hardware,
* but we allow it to be much faster (by a factor of 10) or
* _slightly_ slower (ie we allow up to a 2us read+counter
* update - anything else implies a unacceptably slow CPU
* or PIT for the fast calibration to work.
*
* - with 256 PIT ticks to read the value, we have 214us to
* see the same MSB (and overhead like doing a single TSC
* read per MSB value etc).
*
* - We're doing 2 reads per loop (LSB, MSB), and we expect
* them each to take about a microsecond on real hardware.
* So we expect a count value of around 100. But we'll be
* generous, and accept anything over 50.
*
* - if the PIT is stuck, and we see *many* more reads, we
* return early (and the next caller of pit_expect_msb()
* then consider it a failure when they don't see the
* next expected value).
*
* These expectations mean that we know that we have seen the
* transition from one expected value to another with a fairly
* high accuracy, and we didn't miss any events. We can thus
* use the TSC value at the transitions to calculate a pretty
* good value for the TSC frequency.
*/
static inline int pit_verify_msb(unsigned char val)
{
/* Ignore LSB */
inb(0x42);
return inb(0x42) == val;
}
static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *deltap)
{
int count;
u64 tsc = 0, prev_tsc = 0;
for (count = 0; count < 50000; count++) {
if (!pit_verify_msb(val))
break;
prev_tsc = tsc;
tsc = get_cycles();
}
*deltap = get_cycles() - prev_tsc;
*tscp = tsc;
/*
* We require _some_ success, but the quality control
* will be based on the error terms on the TSC values.
*/
return count > 5;
}
/*
* How many MSB values do we want to see? We aim for
* a maximum error rate of 500ppm (in practice the
* real error is much smaller), but refuse to spend
* more than 50ms on it.
*/
#define MAX_QUICK_PIT_MS 50
#define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
static unsigned long quick_pit_calibrate(void)
{
int i;
u64 tsc, delta;
unsigned long d1, d2;
if (!has_legacy_pic())
return 0;
/* Set the Gate high, disable speaker */
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
/*
* Counter 2, mode 0 (one-shot), binary count
*
* NOTE! Mode 2 decrements by two (and then the
* output is flipped each time, giving the same
* final output frequency as a decrement-by-one),
* so mode 0 is much better when looking at the
* individual counts.
*/
outb(0xb0, 0x43);
/* Start at 0xffff */
outb(0xff, 0x42);
outb(0xff, 0x42);
/*
* The PIT starts counting at the next edge, so we
* need to delay for a microsecond. The easiest way
* to do that is to just read back the 16-bit counter
* once from the PIT.
*/
pit_verify_msb(0);
if (pit_expect_msb(0xff, &tsc, &d1)) {
for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) {
if (!pit_expect_msb(0xff-i, &delta, &d2))
break;
delta -= tsc;
/*
* Extrapolate the error and fail fast if the error will
* never be below 500 ppm.
*/
if (i == 1 &&
d1 + d2 >= (delta * MAX_QUICK_PIT_ITERATIONS) >> 11)
return 0;
/*
* Iterate until the error is less than 500 ppm
*/
if (d1+d2 >= delta >> 11)
continue;
/*
* Check the PIT one more time to verify that
* all TSC reads were stable wrt the PIT.
*
* This also guarantees serialization of the
* last cycle read ('d2') in pit_expect_msb.
*/
if (!pit_verify_msb(0xfe - i))
break;
goto success;
}
}
pr_info("Fast TSC calibration failed\n");
return 0;
success:
/*
* Ok, if we get here, then we've seen the
* MSB of the PIT decrement 'i' times, and the
* error has shrunk to less than 500 ppm.
*
* As a result, we can depend on there not being
* any odd delays anywhere, and the TSC reads are
* reliable (within the error).
*
* kHz = ticks / time-in-seconds / 1000;
* kHz = (t2 - t1) / (I * 256 / PIT_TICK_RATE) / 1000
* kHz = ((t2 - t1) * PIT_TICK_RATE) / (I * 256 * 1000)
*/
delta *= PIT_TICK_RATE;
do_div(delta, i*256*1000);
pr_info("Fast TSC calibration using PIT\n");
return delta;
}
/**
* native_calibrate_tsc - determine TSC frequency
* Determine TSC frequency via CPUID, else return 0.
*/
unsigned long native_calibrate_tsc(void)
{
unsigned int eax_denominator, ebx_numerator, ecx_hz, edx;
unsigned int crystal_khz;
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
return 0;
if (boot_cpu_data.cpuid_level < 0x15)
return 0;
eax_denominator = ebx_numerator = ecx_hz = edx = 0;
/* CPUID 15H TSC/Crystal ratio, plus optionally Crystal Hz */
cpuid(0x15, &eax_denominator, &ebx_numerator, &ecx_hz, &edx);
if (ebx_numerator == 0 || eax_denominator == 0)
return 0;
crystal_khz = ecx_hz / 1000;
/*
* Denverton SoCs don't report crystal clock, and also don't support
* CPUID.0x16 for the calculation below, so hardcode the 25MHz crystal
* clock.
*/
if (crystal_khz == 0 &&
boot_cpu_data.x86_model == INTEL_FAM6_ATOM_GOLDMONT_D)
crystal_khz = 25000;
/*
* TSC frequency reported directly by CPUID is a "hardware reported"
* frequency and is the most accurate one so far we have. This
* is considered a known frequency.
*/
if (crystal_khz != 0)
setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
/*
* Some Intel SoCs like Skylake and Kabylake don't report the crystal
* clock, but we can easily calculate it to a high degree of accuracy
* by considering the crystal ratio and the CPU speed.
*/
if (crystal_khz == 0 && boot_cpu_data.cpuid_level >= 0x16) {
unsigned int eax_base_mhz, ebx, ecx, edx;
cpuid(0x16, &eax_base_mhz, &ebx, &ecx, &edx);
crystal_khz = eax_base_mhz * 1000 *
eax_denominator / ebx_numerator;
}
if (crystal_khz == 0)
return 0;
/*
* For Atom SoCs TSC is the only reliable clocksource.
* Mark TSC reliable so no watchdog on it.
*/
if (boot_cpu_data.x86_model == INTEL_FAM6_ATOM_GOLDMONT)
setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
#ifdef CONFIG_X86_LOCAL_APIC
/*
* The local APIC appears to be fed by the core crystal clock
* (which sounds entirely sensible). We can set the global
* lapic_timer_period here to avoid having to calibrate the APIC
* timer later.
*/
lapic_timer_period = crystal_khz * 1000 / HZ;
#endif
return crystal_khz * ebx_numerator / eax_denominator;
}
static unsigned long cpu_khz_from_cpuid(void)
{
unsigned int eax_base_mhz, ebx_max_mhz, ecx_bus_mhz, edx;
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
return 0;
if (boot_cpu_data.cpuid_level < 0x16)
return 0;
eax_base_mhz = ebx_max_mhz = ecx_bus_mhz = edx = 0;
cpuid(0x16, &eax_base_mhz, &ebx_max_mhz, &ecx_bus_mhz, &edx);
return eax_base_mhz * 1000;
}
/*
* calibrate cpu using pit, hpet, and ptimer methods. They are available
* later in boot after acpi is initialized.
*/
static unsigned long pit_hpet_ptimer_calibrate_cpu(void)
{
u64 tsc1, tsc2, delta, ref1, ref2;
unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
unsigned long flags, latch, ms;
int hpet = is_hpet_enabled(), i, loopmin;
/*
* Run 5 calibration loops to get the lowest frequency value
* (the best estimate). We use two different calibration modes
* here:
*
* 1) PIT loop. We set the PIT Channel 2 to oneshot mode and
* load a timeout of 50ms. We read the time right after we
* started the timer and wait until the PIT count down reaches
* zero. In each wait loop iteration we read the TSC and check
* the delta to the previous read. We keep track of the min
* and max values of that delta. The delta is mostly defined
* by the IO time of the PIT access, so we can detect when
* any disturbance happened between the two reads. If the
* maximum time is significantly larger than the minimum time,
* then we discard the result and have another try.
*
* 2) Reference counter. If available we use the HPET or the
* PMTIMER as a reference to check the sanity of that value.
* We use separate TSC readouts and check inside of the
* reference read for any possible disturbance. We discard
* disturbed values here as well. We do that around the PIT
* calibration delay loop as we have to wait for a certain
* amount of time anyway.
*/
/* Preset PIT loop values */
latch = CAL_LATCH;
ms = CAL_MS;
loopmin = CAL_PIT_LOOPS;
for (i = 0; i < 3; i++) {
unsigned long tsc_pit_khz;
/*
* Read the start value and the reference count of
* hpet/pmtimer when available. Then do the PIT
* calibration, which will take at least 50ms, and
* read the end value.
*/
local_irq_save(flags);
tsc1 = tsc_read_refs(&ref1, hpet);
tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
tsc2 = tsc_read_refs(&ref2, hpet);
local_irq_restore(flags);
/* Pick the lowest PIT TSC calibration so far */
tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
/* hpet or pmtimer available ? */
if (ref1 == ref2)
continue;
/* Check, whether the sampling was disturbed */
if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX)
continue;
tsc2 = (tsc2 - tsc1) * 1000000LL;
if (hpet)
tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
else
tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
/* Check the reference deviation */
delta = ((u64) tsc_pit_min) * 100;
do_div(delta, tsc_ref_min);
/*
* If both calibration results are inside a 10% window
* then we can be sure, that the calibration
* succeeded. We break out of the loop right away. We
* use the reference value, as it is more precise.
*/
if (delta >= 90 && delta <= 110) {
pr_info("PIT calibration matches %s. %d loops\n",
hpet ? "HPET" : "PMTIMER", i + 1);
return tsc_ref_min;
}
/*
* Check whether PIT failed more than once. This
* happens in virtualized environments. We need to
* give the virtual PC a slightly longer timeframe for
* the HPET/PMTIMER to make the result precise.
*/
if (i == 1 && tsc_pit_min == ULONG_MAX) {
latch = CAL2_LATCH;
ms = CAL2_MS;
loopmin = CAL2_PIT_LOOPS;
}
}
/*
* Now check the results.
*/
if (tsc_pit_min == ULONG_MAX) {
/* PIT gave no useful value */
pr_warn("Unable to calibrate against PIT\n");
/* We don't have an alternative source, disable TSC */
if (!hpet && !ref1 && !ref2) {
pr_notice("No reference (HPET/PMTIMER) available\n");
return 0;
}
/* The alternative source failed as well, disable TSC */
if (tsc_ref_min == ULONG_MAX) {
pr_warn("HPET/PMTIMER calibration failed\n");
return 0;
}
/* Use the alternative source */
pr_info("using %s reference calibration\n",
hpet ? "HPET" : "PMTIMER");
return tsc_ref_min;
}
/* We don't have an alternative source, use the PIT calibration value */
if (!hpet && !ref1 && !ref2) {
pr_info("Using PIT calibration value\n");
return tsc_pit_min;
}
/* The alternative source failed, use the PIT calibration value */
if (tsc_ref_min == ULONG_MAX) {
pr_warn("HPET/PMTIMER calibration failed. Using PIT calibration.\n");
return tsc_pit_min;
}
/*
* The calibration values differ too much. In doubt, we use
* the PIT value as we know that there are PMTIMERs around
* running at double speed. At least we let the user know:
*/
pr_warn("PIT calibration deviates from %s: %lu %lu\n",
hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
pr_info("Using PIT calibration value\n");
return tsc_pit_min;
}
/**
* native_calibrate_cpu_early - can calibrate the cpu early in boot
*/
unsigned long native_calibrate_cpu_early(void)
{
unsigned long flags, fast_calibrate = cpu_khz_from_cpuid();
if (!fast_calibrate)
fast_calibrate = cpu_khz_from_msr();
if (!fast_calibrate) {
local_irq_save(flags);
fast_calibrate = quick_pit_calibrate();
local_irq_restore(flags);
}
return fast_calibrate;
}
/**
* native_calibrate_cpu - calibrate the cpu
*/
static unsigned long native_calibrate_cpu(void)
{
unsigned long tsc_freq = native_calibrate_cpu_early();
if (!tsc_freq)
tsc_freq = pit_hpet_ptimer_calibrate_cpu();
return tsc_freq;
}
void recalibrate_cpu_khz(void)
{
#ifndef CONFIG_SMP
unsigned long cpu_khz_old = cpu_khz;
if (!boot_cpu_has(X86_FEATURE_TSC))
return;
cpu_khz = x86_platform.calibrate_cpu();
tsc_khz = x86_platform.calibrate_tsc();
if (tsc_khz == 0)
tsc_khz = cpu_khz;
else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)
cpu_khz = tsc_khz;
cpu_data(0).loops_per_jiffy = cpufreq_scale(cpu_data(0).loops_per_jiffy,
cpu_khz_old, cpu_khz);
#endif
}
EXPORT_SYMBOL_GPL(recalibrate_cpu_khz);
static unsigned long long cyc2ns_suspend;
void tsc_save_sched_clock_state(void)
{
if (!sched_clock_stable())
return;
cyc2ns_suspend = sched_clock();
}
/*
* Even on processors with invariant TSC, TSC gets reset in some the
* ACPI system sleep states. And in some systems BIOS seem to reinit TSC to
* arbitrary value (still sync'd across cpu's) during resume from such sleep
* states. To cope up with this, recompute the cyc2ns_offset for each cpu so
* that sched_clock() continues from the point where it was left off during
* suspend.
*/
void tsc_restore_sched_clock_state(void)
{
unsigned long long offset;
unsigned long flags;
int cpu;
if (!sched_clock_stable())
return;
local_irq_save(flags);
/*
* We're coming out of suspend, there's no concurrency yet; don't
* bother being nice about the RCU stuff, just write to both
* data fields.
*/
this_cpu_write(cyc2ns.data[0].cyc2ns_offset, 0);
this_cpu_write(cyc2ns.data[1].cyc2ns_offset, 0);
offset = cyc2ns_suspend - sched_clock();
for_each_possible_cpu(cpu) {
per_cpu(cyc2ns.data[0].cyc2ns_offset, cpu) = offset;
per_cpu(cyc2ns.data[1].cyc2ns_offset, cpu) = offset;
}
local_irq_restore(flags);
}
#ifdef CONFIG_CPU_FREQ
/*
* Frequency scaling support. Adjust the TSC based timer when the CPU frequency
* changes.
*
* NOTE: On SMP the situation is not fixable in general, so simply mark the TSC
* as unstable and give up in those cases.
*
* Should fix up last_tsc too. Currently gettimeofday in the
* first tick after the change will be slightly wrong.
*/
static unsigned int ref_freq;
static unsigned long loops_per_jiffy_ref;
static unsigned long tsc_khz_ref;
static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
void *data)
{
struct cpufreq_freqs *freq = data;
if (num_online_cpus() > 1) {
mark_tsc_unstable("cpufreq changes on SMP");
return 0;
}
if (!ref_freq) {
ref_freq = freq->old;
loops_per_jiffy_ref = boot_cpu_data.loops_per_jiffy;
tsc_khz_ref = tsc_khz;
}
if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
(val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {
boot_cpu_data.loops_per_jiffy =
cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
if (!(freq->flags & CPUFREQ_CONST_LOOPS))
mark_tsc_unstable("cpufreq changes");
set_cyc2ns_scale(tsc_khz, freq->policy->cpu, rdtsc());
}
return 0;
}
static struct notifier_block time_cpufreq_notifier_block = {
.notifier_call = time_cpufreq_notifier
};
static int __init cpufreq_register_tsc_scaling(void)
{
if (!boot_cpu_has(X86_FEATURE_TSC))
return 0;
if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
return 0;
cpufreq_register_notifier(&time_cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
return 0;
}
core_initcall(cpufreq_register_tsc_scaling);
#endif /* CONFIG_CPU_FREQ */
#define ART_CPUID_LEAF (0x15)
#define ART_MIN_DENOMINATOR (1)
/*
* If ART is present detect the numerator:denominator to convert to TSC
*/
static void __init detect_art(void)
{
unsigned int unused[2];
if (boot_cpu_data.cpuid_level < ART_CPUID_LEAF)
return;
/*
* Don't enable ART in a VM, non-stop TSC and TSC_ADJUST required,
* and the TSC counter resets must not occur asynchronously.
*/
if (boot_cpu_has(X86_FEATURE_HYPERVISOR) ||
!boot_cpu_has(X86_FEATURE_NONSTOP_TSC) ||
!boot_cpu_has(X86_FEATURE_TSC_ADJUST) ||
tsc_async_resets)
return;
cpuid(ART_CPUID_LEAF, &art_to_tsc_denominator,
&art_to_tsc_numerator, unused, unused+1);
if (art_to_tsc_denominator < ART_MIN_DENOMINATOR)
return;
rdmsrl(MSR_IA32_TSC_ADJUST, art_to_tsc_offset);
/* Make this sticky over multiple CPU init calls */
setup_force_cpu_cap(X86_FEATURE_ART);
}
/* clocksource code */
static void tsc_resume(struct clocksource *cs)
{
tsc_verify_tsc_adjust(true);
}
/*
* We used to compare the TSC to the cycle_last value in the clocksource
* structure to avoid a nasty time-warp. This can be observed in a
* very small window right after one CPU updated cycle_last under
* xtime/vsyscall_gtod lock and the other CPU reads a TSC value which
* is smaller than the cycle_last reference value due to a TSC which
* is slightly behind. This delta is nowhere else observable, but in
* that case it results in a forward time jump in the range of hours
* due to the unsigned delta calculation of the time keeping core
* code, which is necessary to support wrapping clocksources like pm
* timer.
*
* This sanity check is now done in the core timekeeping code.
* checking the result of read_tsc() - cycle_last for being negative.
* That works because CLOCKSOURCE_MASK(64) does not mask out any bit.
*/
static u64 read_tsc(struct clocksource *cs)
{
return (u64)rdtsc_ordered();
}
static void tsc_cs_mark_unstable(struct clocksource *cs)
{
if (tsc_unstable)
return;
tsc_unstable = 1;
if (using_native_sched_clock())
clear_sched_clock_stable();
disable_sched_clock_irqtime();
pr_info("Marking TSC unstable due to clocksource watchdog\n");
}
static void tsc_cs_tick_stable(struct clocksource *cs)
{
if (tsc_unstable)
return;
if (using_native_sched_clock())
sched_clock_tick_stable();
}
static int tsc_cs_enable(struct clocksource *cs)
{
vclocks_set_used(VDSO_CLOCKMODE_TSC);
return 0;
}
/*
* .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc()
*/
static struct clocksource clocksource_tsc_early = {
.name = "tsc-early",
.rating = 299,
.uncertainty_margin = 32 * NSEC_PER_MSEC,
.read = read_tsc,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS |
CLOCK_SOURCE_MUST_VERIFY,
.id = CSID_X86_TSC_EARLY,
.vdso_clock_mode = VDSO_CLOCKMODE_TSC,
.enable = tsc_cs_enable,
.resume = tsc_resume,
.mark_unstable = tsc_cs_mark_unstable,
.tick_stable = tsc_cs_tick_stable,
.list = LIST_HEAD_INIT(clocksource_tsc_early.list),
};
/*
* Must mark VALID_FOR_HRES early such that when we unregister tsc_early
* this one will immediately take over. We will only register if TSC has
* been found good.
*/
static struct clocksource clocksource_tsc = {
.name = "tsc",
.rating = 300,
.read = read_tsc,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS |
CLOCK_SOURCE_VALID_FOR_HRES |
CLOCK_SOURCE_MUST_VERIFY |
CLOCK_SOURCE_VERIFY_PERCPU,
.id = CSID_X86_TSC,
.vdso_clock_mode = VDSO_CLOCKMODE_TSC,
.enable = tsc_cs_enable,
.resume = tsc_resume,
.mark_unstable = tsc_cs_mark_unstable,
.tick_stable = tsc_cs_tick_stable,
.list = LIST_HEAD_INIT(clocksource_tsc.list),
};
void mark_tsc_unstable(char *reason)
{
if (tsc_unstable)
return;
tsc_unstable = 1;
if (using_native_sched_clock())
clear_sched_clock_stable();
disable_sched_clock_irqtime();
pr_info("Marking TSC unstable due to %s\n", reason);
clocksource_mark_unstable(&clocksource_tsc_early);
clocksource_mark_unstable(&clocksource_tsc);
}
EXPORT_SYMBOL_GPL(mark_tsc_unstable);
static void __init tsc_disable_clocksource_watchdog(void)
{
clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
}
bool tsc_clocksource_watchdog_disabled(void)
{
return !(clocksource_tsc.flags & CLOCK_SOURCE_MUST_VERIFY) &&
tsc_as_watchdog && !no_tsc_watchdog;
}
static void __init check_system_tsc_reliable(void)
{
#if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC)
if (is_geode_lx()) {
/* RTSC counts during suspend */
#define RTSC_SUSP 0x100
unsigned long res_low, res_high;
rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
/* Geode_LX - the OLPC CPU has a very reliable TSC */
if (res_low & RTSC_SUSP)
tsc_clocksource_reliable = 1;
}
#endif
if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
tsc_clocksource_reliable = 1;
/*
* Disable the clocksource watchdog when the system has:
* - TSC running at constant frequency
* - TSC which does not stop in C-States
* - the TSC_ADJUST register which allows to detect even minimal
* modifications
* - not more than two sockets. As the number of sockets cannot be
* evaluated at the early boot stage where this has to be
* invoked, check the number of online memory nodes as a
* fallback solution which is an reasonable estimate.
*/
if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
boot_cpu_has(X86_FEATURE_NONSTOP_TSC) &&
boot_cpu_has(X86_FEATURE_TSC_ADJUST) &&
nr_online_nodes <= 4)
tsc_disable_clocksource_watchdog();
}
/*
* Make an educated guess if the TSC is trustworthy and synchronized
* over all CPUs.
*/
int unsynchronized_tsc(void)
{
if (!boot_cpu_has(X86_FEATURE_TSC) || tsc_unstable)
return 1;
#ifdef CONFIG_SMP
if (apic_is_clustered_box())
return 1;
#endif
if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
return 0;
if (tsc_clocksource_reliable)
return 0;
/*
* Intel systems are normally all synchronized.
* Exceptions must mark TSC as unstable:
*/
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
/* assume multi socket systems are not synchronized: */
if (num_possible_cpus() > 1)
return 1;
}
return 0;
}
/*
* Convert ART to TSC given numerator/denominator found in detect_art()
*/
struct system_counterval_t convert_art_to_tsc(u64 art)
{
u64 tmp, res, rem;
rem = do_div(art, art_to_tsc_denominator);
res = art * art_to_tsc_numerator;
tmp = rem * art_to_tsc_numerator;
do_div(tmp, art_to_tsc_denominator);
res += tmp + art_to_tsc_offset;
return (struct system_counterval_t) {
.cs_id = have_art ? CSID_X86_TSC : CSID_GENERIC,
.cycles = res,
};
}
EXPORT_SYMBOL(convert_art_to_tsc);
/**
* convert_art_ns_to_tsc() - Convert ART in nanoseconds to TSC.
* @art_ns: ART (Always Running Timer) in unit of nanoseconds
*
* PTM requires all timestamps to be in units of nanoseconds. When user
* software requests a cross-timestamp, this function converts system timestamp
* to TSC.
*
* This is valid when CPU feature flag X86_FEATURE_TSC_KNOWN_FREQ is set
* indicating the tsc_khz is derived from CPUID[15H]. Drivers should check
* that this flag is set before conversion to TSC is attempted.
*
* Return:
* struct system_counterval_t - system counter value with the ID of the
* corresponding clocksource:
* cycles: System counter value
* cs_id: The clocksource ID for validating comparability
*/
struct system_counterval_t convert_art_ns_to_tsc(u64 art_ns)
{
u64 tmp, res, rem;
rem = do_div(art_ns, USEC_PER_SEC);
res = art_ns * tsc_khz;
tmp = rem * tsc_khz;
do_div(tmp, USEC_PER_SEC);
res += tmp;
return (struct system_counterval_t) {
.cs_id = have_art ? CSID_X86_TSC : CSID_GENERIC,
.cycles = res,
};
}
EXPORT_SYMBOL(convert_art_ns_to_tsc);
static void tsc_refine_calibration_work(struct work_struct *work);
static DECLARE_DELAYED_WORK(tsc_irqwork, tsc_refine_calibration_work);
/**
* tsc_refine_calibration_work - Further refine tsc freq calibration
* @work: ignored.
*
* This functions uses delayed work over a period of a
* second to further refine the TSC freq value. Since this is
* timer based, instead of loop based, we don't block the boot
* process while this longer calibration is done.
*
* If there are any calibration anomalies (too many SMIs, etc),
* or the refined calibration is off by 1% of the fast early
* calibration, we throw out the new calibration and use the
* early calibration.
*/
static void tsc_refine_calibration_work(struct work_struct *work)
{
static u64 tsc_start = ULLONG_MAX, ref_start;
static int hpet;
u64 tsc_stop, ref_stop, delta;
unsigned long freq;
int cpu;
/* Don't bother refining TSC on unstable systems */
if (tsc_unstable)
goto unreg;
/*
* Since the work is started early in boot, we may be
* delayed the first time we expire. So set the workqueue
* again once we know timers are working.
*/
if (tsc_start == ULLONG_MAX) {
restart:
/*
* Only set hpet once, to avoid mixing hardware
* if the hpet becomes enabled later.
*/
hpet = is_hpet_enabled();
tsc_start = tsc_read_refs(&ref_start, hpet);
schedule_delayed_work(&tsc_irqwork, HZ);
return;
}
tsc_stop = tsc_read_refs(&ref_stop, hpet);
/* hpet or pmtimer available ? */
if (ref_start == ref_stop)
goto out;
/* Check, whether the sampling was disturbed */
if (tsc_stop == ULLONG_MAX)
goto restart;
delta = tsc_stop - tsc_start;
delta *= 1000000LL;
if (hpet)
freq = calc_hpet_ref(delta, ref_start, ref_stop);
else
freq = calc_pmtimer_ref(delta, ref_start, ref_stop);
/* Will hit this only if tsc_force_recalibrate has been set */
if (boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ)) {
/* Warn if the deviation exceeds 500 ppm */
if (abs(tsc_khz - freq) > (tsc_khz >> 11)) {
pr_warn("Warning: TSC freq calibrated by CPUID/MSR differs from what is calibrated by HW timer, please check with vendor!!\n");
pr_info("Previous calibrated TSC freq:\t %lu.%03lu MHz\n",
(unsigned long)tsc_khz / 1000,
(unsigned long)tsc_khz % 1000);
}
pr_info("TSC freq recalibrated by [%s]:\t %lu.%03lu MHz\n",
hpet ? "HPET" : "PM_TIMER",
(unsigned long)freq / 1000,
(unsigned long)freq % 1000);
return;
}
/* Make sure we're within 1% */
if (abs(tsc_khz - freq) > tsc_khz/100)
goto out;
tsc_khz = freq;
pr_info("Refined TSC clocksource calibration: %lu.%03lu MHz\n",
(unsigned long)tsc_khz / 1000,
(unsigned long)tsc_khz % 1000);
/* Inform the TSC deadline clockevent devices about the recalibration */
lapic_update_tsc_freq();
/* Update the sched_clock() rate to match the clocksource one */
for_each_possible_cpu(cpu)
set_cyc2ns_scale(tsc_khz, cpu, tsc_stop);
out:
if (tsc_unstable)
goto unreg;
if (boot_cpu_has(X86_FEATURE_ART))
have_art = true;
clocksource_register_khz(&clocksource_tsc, tsc_khz);
unreg:
clocksource_unregister(&clocksource_tsc_early);
}
static int __init init_tsc_clocksource(void)
{
if (!boot_cpu_has(X86_FEATURE_TSC) || !tsc_khz)
return 0;
if (tsc_unstable) {
clocksource_unregister(&clocksource_tsc_early);
return 0;
}
if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3))
clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
/*
* When TSC frequency is known (retrieved via MSR or CPUID), we skip
* the refined calibration and directly register it as a clocksource.
*/
if (boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ)) {
if (boot_cpu_has(X86_FEATURE_ART))
have_art = true;
clocksource_register_khz(&clocksource_tsc, tsc_khz);
clocksource_unregister(&clocksource_tsc_early);
if (!tsc_force_recalibrate)
return 0;
}
schedule_delayed_work(&tsc_irqwork, 0);
return 0;
}
/*
* We use device_initcall here, to ensure we run after the hpet
* is fully initialized, which may occur at fs_initcall time.
*/
device_initcall(init_tsc_clocksource);
static bool __init determine_cpu_tsc_frequencies(bool early)
{
/* Make sure that cpu and tsc are not already calibrated */
WARN_ON(cpu_khz || tsc_khz);
if (early) {
cpu_khz = x86_platform.calibrate_cpu();
if (tsc_early_khz)
tsc_khz = tsc_early_khz;
else
tsc_khz = x86_platform.calibrate_tsc();
} else {
/* We should not be here with non-native cpu calibration */
WARN_ON(x86_platform.calibrate_cpu != native_calibrate_cpu);
cpu_khz = pit_hpet_ptimer_calibrate_cpu();
}
/*
* Trust non-zero tsc_khz as authoritative,
* and use it to sanity check cpu_khz,
* which will be off if system timer is off.
*/
if (tsc_khz == 0)
tsc_khz = cpu_khz;
else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)
cpu_khz = tsc_khz;
if (tsc_khz == 0)
return false;
pr_info("Detected %lu.%03lu MHz processor\n",
(unsigned long)cpu_khz / KHZ,
(unsigned long)cpu_khz % KHZ);
if (cpu_khz != tsc_khz) {
pr_info("Detected %lu.%03lu MHz TSC",
(unsigned long)tsc_khz / KHZ,
(unsigned long)tsc_khz % KHZ);
}
return true;
}
static unsigned long __init get_loops_per_jiffy(void)
{
u64 lpj = (u64)tsc_khz * KHZ;
do_div(lpj, HZ);
return lpj;
}
static void __init tsc_enable_sched_clock(void)
{
loops_per_jiffy = get_loops_per_jiffy();
use_tsc_delay();
/* Sanitize TSC ADJUST before cyc2ns gets initialized */
tsc_store_and_check_tsc_adjust(true);
cyc2ns_init_boot_cpu();
static_branch_enable(&__use_tsc);
}
void __init tsc_early_init(void)
{
if (!boot_cpu_has(X86_FEATURE_TSC))
return;
/* Don't change UV TSC multi-chassis synchronization */
if (is_early_uv_system())
return;
if (!determine_cpu_tsc_frequencies(true))
return;
tsc_enable_sched_clock();
}
void __init tsc_init(void)
{
if (!cpu_feature_enabled(X86_FEATURE_TSC)) {
setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
return;
}
/*
* native_calibrate_cpu_early can only calibrate using methods that are
* available early in boot.
*/
if (x86_platform.calibrate_cpu == native_calibrate_cpu_early)
x86_platform.calibrate_cpu = native_calibrate_cpu;
if (!tsc_khz) {
/* We failed to determine frequencies earlier, try again */
if (!determine_cpu_tsc_frequencies(false)) {
mark_tsc_unstable("could not calculate TSC khz");
setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
return;
}
tsc_enable_sched_clock();
}
cyc2ns_init_secondary_cpus();
if (!no_sched_irq_time)
enable_sched_clock_irqtime();
lpj_fine = get_loops_per_jiffy();
check_system_tsc_reliable();
if (unsynchronized_tsc()) {
mark_tsc_unstable("TSCs unsynchronized");
return;
}
if (tsc_clocksource_reliable || no_tsc_watchdog)
tsc_disable_clocksource_watchdog();
clocksource_register_khz(&clocksource_tsc_early, tsc_khz);
detect_art();
}
#ifdef CONFIG_SMP
/*
* Check whether existing calibration data can be reused.
*/
unsigned long calibrate_delay_is_known(void)
{
int sibling, cpu = smp_processor_id();
int constant_tsc = cpu_has(&cpu_data(cpu), X86_FEATURE_CONSTANT_TSC);
const struct cpumask *mask = topology_core_cpumask(cpu);
/*
* If TSC has constant frequency and TSC is synchronized across
* sockets then reuse CPU0 calibration.
*/
if (constant_tsc && !tsc_unstable)
return cpu_data(0).loops_per_jiffy;
/*
* If TSC has constant frequency and TSC is not synchronized across
* sockets and this is not the first CPU in the socket, then reuse
* the calibration value of an already online CPU on that socket.
*
* This assumes that CONSTANT_TSC is consistent for all CPUs in a
* socket.
*/
if (!constant_tsc || !mask)
return 0;
sibling = cpumask_any_but(mask, cpu);
if (sibling < nr_cpu_ids)
return cpu_data(sibling).loops_per_jiffy;
return 0;
}
#endif
/*
BlueZ - Bluetooth protocol stack for Linux
Copyright (C) 2014 Intel Corporation
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License version 2 as
published by the Free Software Foundation;
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS.
IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) AND AUTHOR(S) BE LIABLE FOR ANY
CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
ALL LIABILITY, INCLUDING LIABILITY FOR INFRINGEMENT OF ANY PATENTS,
COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS, RELATING TO USE OF THIS
SOFTWARE IS DISCLAIMED.
*/
#include <linux/sched/signal.h>
#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <net/bluetooth/mgmt.h>
#include "smp.h"
#include "hci_request.h"
#include "msft.h"
#include "eir.h"
void hci_req_init(struct hci_request *req, struct hci_dev *hdev)
{
skb_queue_head_init(&req->cmd_q);
req->hdev = hdev;
req->err = 0;
}
void hci_req_purge(struct hci_request *req)
{
skb_queue_purge(&req->cmd_q);
}
bool hci_req_status_pend(struct hci_dev *hdev)
{
return hdev->req_status == HCI_REQ_PEND;
}
static int req_run(struct hci_request *req, hci_req_complete_t complete,
hci_req_complete_skb_t complete_skb)
{
struct hci_dev *hdev = req->hdev;
struct sk_buff *skb;
unsigned long flags;
bt_dev_dbg(hdev, "length %u", skb_queue_len(&req->cmd_q));
/* If an error occurred during request building, remove all HCI
* commands queued on the HCI request queue.
*/
if (req->err) {
skb_queue_purge(&req->cmd_q);
return req->err;
}
/* Do not allow empty requests */
if (skb_queue_empty(&req->cmd_q))
return -ENODATA;
skb = skb_peek_tail(&req->cmd_q);
if (complete) {
bt_cb(skb)->hci.req_complete = complete;
} else if (complete_skb) {
bt_cb(skb)->hci.req_complete_skb = complete_skb;
bt_cb(skb)->hci.req_flags |= HCI_REQ_SKB;
}
spin_lock_irqsave(&hdev->cmd_q.lock, flags);
skb_queue_splice_tail(&req->cmd_q, &hdev->cmd_q);
spin_unlock_irqrestore(&hdev->cmd_q.lock, flags);
queue_work(hdev->workqueue, &hdev->cmd_work);
return 0;
}
int hci_req_run(struct hci_request *req, hci_req_complete_t complete)
{
return req_run(req, complete, NULL);
}
int hci_req_run_skb(struct hci_request *req, hci_req_complete_skb_t complete)
{
return req_run(req, NULL, complete);
}
void hci_req_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode,
struct sk_buff *skb)
{
bt_dev_dbg(hdev, "result 0x%2.2x", result);
if (hdev->req_status == HCI_REQ_PEND) {
hdev->req_result = result;
hdev->req_status = HCI_REQ_DONE;
if (skb) {
kfree_skb(hdev->req_skb);
hdev->req_skb = skb_get(skb);
}
wake_up_interruptible(&hdev->req_wait_q);
}
}
/* Execute request and wait for completion. */
int __hci_req_sync(struct hci_dev *hdev, int (*func)(struct hci_request *req,
unsigned long opt),
unsigned long opt, u32 timeout, u8 *hci_status)
{
struct hci_request req;
int err = 0;
bt_dev_dbg(hdev, "start");
hci_req_init(&req, hdev);
hdev->req_status = HCI_REQ_PEND;
err = func(&req, opt);
if (err) {
if (hci_status)
*hci_status = HCI_ERROR_UNSPECIFIED;
return err;
}
err = hci_req_run_skb(&req, hci_req_sync_complete);
if (err < 0) {
hdev->req_status = 0;
/* ENODATA means the HCI request command queue is empty.
* This can happen when a request with conditionals doesn't
* trigger any commands to be sent. This is normal behavior
* and should not trigger an error return.
*/
if (err == -ENODATA) {
if (hci_status)
*hci_status = 0;
return 0;
}
if (hci_status)
*hci_status = HCI_ERROR_UNSPECIFIED;
return err;
}
err = wait_event_interruptible_timeout(hdev->req_wait_q,
hdev->req_status != HCI_REQ_PEND, timeout);
if (err == -ERESTARTSYS)
return -EINTR;
switch (hdev->req_status) {
case HCI_REQ_DONE:
err = -bt_to_errno(hdev->req_result);
if (hci_status)
*hci_status = hdev->req_result;
break;
case HCI_REQ_CANCELED:
err = -hdev->req_result;
if (hci_status)
*hci_status = HCI_ERROR_UNSPECIFIED;
break;
default:
err = -ETIMEDOUT;
if (hci_status)
*hci_status = HCI_ERROR_UNSPECIFIED;
break;
}
kfree_skb(hdev->req_skb);
hdev->req_skb = NULL;
hdev->req_status = hdev->req_result = 0;
bt_dev_dbg(hdev, "end: err %d", err);
return err;
}
int hci_req_sync(struct hci_dev *hdev, int (*req)(struct hci_request *req,
unsigned long opt),
unsigned long opt, u32 timeout, u8 *hci_status)
{
int ret;
/* Serialize all requests */
hci_req_sync_lock(hdev);
/* check the state after obtaing the lock to protect the HCI_UP
* against any races from hci_dev_do_close when the controller
* gets removed.
*/
if (test_bit(HCI_UP, &hdev->flags))
ret = __hci_req_sync(hdev, req, opt, timeout, hci_status);
else
ret = -ENETDOWN;
hci_req_sync_unlock(hdev);
return ret;
}
struct sk_buff *hci_prepare_cmd(struct hci_dev *hdev, u16 opcode, u32 plen,
const void *param)
{
int len = HCI_COMMAND_HDR_SIZE + plen;
struct hci_command_hdr *hdr;
struct sk_buff *skb;
skb = bt_skb_alloc(len, GFP_ATOMIC);
if (!skb)
return NULL;
hdr = skb_put(skb, HCI_COMMAND_HDR_SIZE);
hdr->opcode = cpu_to_le16(opcode);
hdr->plen = plen;
if (plen)
skb_put_data(skb, param, plen);
bt_dev_dbg(hdev, "skb len %d", skb->len);
hci_skb_pkt_type(skb) = HCI_COMMAND_PKT;
hci_skb_opcode(skb) = opcode;
return skb;
}
/* Queue a command to an asynchronous HCI request */
void hci_req_add_ev(struct hci_request *req, u16 opcode, u32 plen,
const void *param, u8 event)
{
struct hci_dev *hdev = req->hdev;
struct sk_buff *skb;
bt_dev_dbg(hdev, "opcode 0x%4.4x plen %d", opcode, plen);
/* If an error occurred during request building, there is no point in
* queueing the HCI command. We can simply return.
*/
if (req->err)
return;
skb = hci_prepare_cmd(hdev, opcode, plen, param);
if (!skb) {
bt_dev_err(hdev, "no memory for command (opcode 0x%4.4x)",
opcode);
req->err = -ENOMEM;
return;
}
if (skb_queue_empty(&req->cmd_q))
bt_cb(skb)->hci.req_flags |= HCI_REQ_START;
hci_skb_event(skb) = event;
skb_queue_tail(&req->cmd_q, skb);
}
void hci_req_add(struct hci_request *req, u16 opcode, u32 plen,
const void *param)
{
bt_dev_dbg(req->hdev, "HCI_REQ-0x%4.4x", opcode);
hci_req_add_ev(req, opcode, plen, param, 0);
}
static void start_interleave_scan(struct hci_dev *hdev)
{
hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER;
queue_delayed_work(hdev->req_workqueue,
&hdev->interleave_scan, 0);
}
static bool is_interleave_scanning(struct hci_dev *hdev)
{
return hdev->interleave_scan_state != INTERLEAVE_SCAN_NONE;
}
static void cancel_interleave_scan(struct hci_dev *hdev)
{
bt_dev_dbg(hdev, "cancelling interleave scan"); cancel_delayed_work_sync(&hdev->interleave_scan);
hdev->interleave_scan_state = INTERLEAVE_SCAN_NONE;
}
/* Return true if interleave_scan wasn't started until exiting this function,
* otherwise, return false
*/
static bool __hci_update_interleaved_scan(struct hci_dev *hdev)
{
/* Do interleaved scan only if all of the following are true:
* - There is at least one ADV monitor
* - At least one pending LE connection or one device to be scanned for
* - Monitor offloading is not supported
* If so, we should alternate between allowlist scan and one without
* any filters to save power.
*/
bool use_interleaving = hci_is_adv_monitoring(hdev) &&
!(list_empty(&hdev->pend_le_conns) &&
list_empty(&hdev->pend_le_reports)) &&
hci_get_adv_monitor_offload_ext(hdev) ==
HCI_ADV_MONITOR_EXT_NONE;
bool is_interleaving = is_interleave_scanning(hdev);
if (use_interleaving && !is_interleaving) {
start_interleave_scan(hdev);
bt_dev_dbg(hdev, "starting interleave scan");
return true;
}
if (!use_interleaving && is_interleaving)
cancel_interleave_scan(hdev);
return false;
}
void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn)
{
struct hci_dev *hdev = req->hdev;
if (hdev->scanning_paused) {
bt_dev_dbg(hdev, "Scanning is paused for suspend");
return;
}
if (use_ext_scan(hdev)) {
struct hci_cp_le_set_ext_scan_enable cp;
memset(&cp, 0, sizeof(cp));
cp.enable = LE_SCAN_DISABLE;
hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_ENABLE, sizeof(cp),
&cp);
} else {
struct hci_cp_le_set_scan_enable cp;
memset(&cp, 0, sizeof(cp));
cp.enable = LE_SCAN_DISABLE;
hci_req_add(req, HCI_OP_LE_SET_SCAN_ENABLE, sizeof(cp), &cp);
}
/* Disable address resolution */
if (hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION) && !rpa_le_conn) {
__u8 enable = 0x00;
hci_req_add(req, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, 1, &enable);
}
}
static void del_from_accept_list(struct hci_request *req, bdaddr_t *bdaddr,
u8 bdaddr_type)
{
struct hci_cp_le_del_from_accept_list cp;
cp.bdaddr_type = bdaddr_type;
bacpy(&cp.bdaddr, bdaddr);
bt_dev_dbg(req->hdev, "Remove %pMR (0x%x) from accept list", &cp.bdaddr,
cp.bdaddr_type);
hci_req_add(req, HCI_OP_LE_DEL_FROM_ACCEPT_LIST, sizeof(cp), &cp);
if (use_ll_privacy(req->hdev)) {
struct smp_irk *irk;
irk = hci_find_irk_by_addr(req->hdev, bdaddr, bdaddr_type);
if (irk) {
struct hci_cp_le_del_from_resolv_list cp;
cp.bdaddr_type = bdaddr_type;
bacpy(&cp.bdaddr, bdaddr);
hci_req_add(req, HCI_OP_LE_DEL_FROM_RESOLV_LIST,
sizeof(cp), &cp);
}
}
}
/* Adds connection to accept list if needed. On error, returns -1. */
static int add_to_accept_list(struct hci_request *req,
struct hci_conn_params *params, u8 *num_entries,
bool allow_rpa)
{
struct hci_cp_le_add_to_accept_list cp;
struct hci_dev *hdev = req->hdev;
/* Already in accept list */
if (hci_bdaddr_list_lookup(&hdev->le_accept_list, ¶ms->addr,
params->addr_type))
return 0;
/* Select filter policy to accept all advertising */
if (*num_entries >= hdev->le_accept_list_size)
return -1;
/* Accept list can not be used with RPAs */
if (!allow_rpa &&
!hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY) &&
hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type)) {
return -1;
}
/* During suspend, only wakeable devices can be in accept list */
if (hdev->suspended &&
!(params->flags & HCI_CONN_FLAG_REMOTE_WAKEUP))
return 0;
*num_entries += 1;
cp.bdaddr_type = params->addr_type;
bacpy(&cp.bdaddr, ¶ms->addr);
bt_dev_dbg(hdev, "Add %pMR (0x%x) to accept list", &cp.bdaddr,
cp.bdaddr_type);
hci_req_add(req, HCI_OP_LE_ADD_TO_ACCEPT_LIST, sizeof(cp), &cp);
if (use_ll_privacy(hdev)) {
struct smp_irk *irk;
irk = hci_find_irk_by_addr(hdev, ¶ms->addr,
params->addr_type);
if (irk) {
struct hci_cp_le_add_to_resolv_list cp;
cp.bdaddr_type = params->addr_type;
bacpy(&cp.bdaddr, ¶ms->addr);
memcpy(cp.peer_irk, irk->val, 16);
if (hci_dev_test_flag(hdev, HCI_PRIVACY))
memcpy(cp.local_irk, hdev->irk, 16);
else
memset(cp.local_irk, 0, 16);
hci_req_add(req, HCI_OP_LE_ADD_TO_RESOLV_LIST,
sizeof(cp), &cp);
}
}
return 0;
}
static u8 update_accept_list(struct hci_request *req)
{
struct hci_dev *hdev = req->hdev;
struct hci_conn_params *params;
struct bdaddr_list *b;
u8 num_entries = 0;
bool pend_conn, pend_report;
/* We allow usage of accept list even with RPAs in suspend. In the worst
* case, we won't be able to wake from devices that use the privacy1.2
* features. Additionally, once we support privacy1.2 and IRK
* offloading, we can update this to also check for those conditions.
*/
bool allow_rpa = hdev->suspended;
if (use_ll_privacy(hdev))
allow_rpa = true;
/* Go through the current accept list programmed into the
* controller one by one and check if that address is still
* in the list of pending connections or list of devices to
* report. If not present in either list, then queue the
* command to remove it from the controller.
*/
list_for_each_entry(b, &hdev->le_accept_list, list) {
pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns,
&b->bdaddr,
b->bdaddr_type);
pend_report = hci_pend_le_action_lookup(&hdev->pend_le_reports,
&b->bdaddr,
b->bdaddr_type);
/* If the device is not likely to connect or report,
* remove it from the accept list.
*/
if (!pend_conn && !pend_report) {
del_from_accept_list(req, &b->bdaddr, b->bdaddr_type);
continue;
}
/* Accept list can not be used with RPAs */
if (!allow_rpa &&
!hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY) &&
hci_find_irk_by_addr(hdev, &b->bdaddr, b->bdaddr_type)) {
return 0x00;
}
num_entries++;
}
/* Since all no longer valid accept list entries have been
* removed, walk through the list of pending connections
* and ensure that any new device gets programmed into
* the controller.
*
* If the list of the devices is larger than the list of
* available accept list entries in the controller, then
* just abort and return filer policy value to not use the
* accept list.
*/
list_for_each_entry(params, &hdev->pend_le_conns, action) {
if (add_to_accept_list(req, params, &num_entries, allow_rpa))
return 0x00;
}
/* After adding all new pending connections, walk through
* the list of pending reports and also add these to the
* accept list if there is still space. Abort if space runs out.
*/
list_for_each_entry(params, &hdev->pend_le_reports, action) {
if (add_to_accept_list(req, params, &num_entries, allow_rpa))
return 0x00;
}
/* Use the allowlist unless the following conditions are all true:
* - We are not currently suspending
* - There are 1 or more ADV monitors registered and it's not offloaded
* - Interleaved scanning is not currently using the allowlist
*/
if (!idr_is_empty(&hdev->adv_monitors_idr) && !hdev->suspended &&
hci_get_adv_monitor_offload_ext(hdev) == HCI_ADV_MONITOR_EXT_NONE &&
hdev->interleave_scan_state != INTERLEAVE_SCAN_ALLOWLIST)
return 0x00;
/* Select filter policy to use accept list */
return 0x01;
}
static bool scan_use_rpa(struct hci_dev *hdev)
{
return hci_dev_test_flag(hdev, HCI_PRIVACY);
}
static void hci_req_start_scan(struct hci_request *req, u8 type, u16 interval,
u16 window, u8 own_addr_type, u8 filter_policy,
bool filter_dup, bool addr_resolv)
{
struct hci_dev *hdev = req->hdev;
if (hdev->scanning_paused) {
bt_dev_dbg(hdev, "Scanning is paused for suspend");
return;
}
if (use_ll_privacy(hdev) && addr_resolv) {
u8 enable = 0x01;
hci_req_add(req, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, 1, &enable);
}
/* Use ext scanning if set ext scan param and ext scan enable is
* supported
*/
if (use_ext_scan(hdev)) {
struct hci_cp_le_set_ext_scan_params *ext_param_cp;
struct hci_cp_le_set_ext_scan_enable ext_enable_cp;
struct hci_cp_le_scan_phy_params *phy_params;
u8 data[sizeof(*ext_param_cp) + sizeof(*phy_params) * 2];
u32 plen;
ext_param_cp = (void *)data;
phy_params = (void *)ext_param_cp->data;
memset(ext_param_cp, 0, sizeof(*ext_param_cp));
ext_param_cp->own_addr_type = own_addr_type;
ext_param_cp->filter_policy = filter_policy;
plen = sizeof(*ext_param_cp);
if (scan_1m(hdev) || scan_2m(hdev)) {
ext_param_cp->scanning_phys |= LE_SCAN_PHY_1M;
memset(phy_params, 0, sizeof(*phy_params));
phy_params->type = type;
phy_params->interval = cpu_to_le16(interval);
phy_params->window = cpu_to_le16(window);
plen += sizeof(*phy_params);
phy_params++;
}
if (scan_coded(hdev)) {
ext_param_cp->scanning_phys |= LE_SCAN_PHY_CODED;
memset(phy_params, 0, sizeof(*phy_params));
phy_params->type = type;
phy_params->interval = cpu_to_le16(interval);
phy_params->window = cpu_to_le16(window);
plen += sizeof(*phy_params);
phy_params++;
}
hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_PARAMS,
plen, ext_param_cp);
memset(&ext_enable_cp, 0, sizeof(ext_enable_cp));
ext_enable_cp.enable = LE_SCAN_ENABLE;
ext_enable_cp.filter_dup = filter_dup;
hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_ENABLE,
sizeof(ext_enable_cp), &ext_enable_cp);
} else {
struct hci_cp_le_set_scan_param param_cp;
struct hci_cp_le_set_scan_enable enable_cp;
memset(¶m_cp, 0, sizeof(param_cp));
param_cp.type = type;
param_cp.interval = cpu_to_le16(interval);
param_cp.window = cpu_to_le16(window);
param_cp.own_address_type = own_addr_type;
param_cp.filter_policy = filter_policy;
hci_req_add(req, HCI_OP_LE_SET_SCAN_PARAM, sizeof(param_cp),
¶m_cp);
memset(&enable_cp, 0, sizeof(enable_cp));
enable_cp.enable = LE_SCAN_ENABLE;
enable_cp.filter_dup = filter_dup;
hci_req_add(req, HCI_OP_LE_SET_SCAN_ENABLE, sizeof(enable_cp),
&enable_cp);
}
}
static void set_random_addr(struct hci_request *req, bdaddr_t *rpa);
static int hci_update_random_address(struct hci_request *req,
bool require_privacy, bool use_rpa,
u8 *own_addr_type)
{
struct hci_dev *hdev = req->hdev;
int err;
/* If privacy is enabled use a resolvable private address. If
* current RPA has expired or there is something else than
* the current RPA in use, then generate a new one.
*/
if (use_rpa) {
/* If Controller supports LL Privacy use own address type is
* 0x03
*/
if (use_ll_privacy(hdev))
*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
else
*own_addr_type = ADDR_LE_DEV_RANDOM;
if (rpa_valid(hdev))
return 0;
err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
if (err < 0) {
bt_dev_err(hdev, "failed to generate new RPA");
return err;
}
set_random_addr(req, &hdev->rpa);
return 0;
}
/* In case of required privacy without resolvable private address,
* use an non-resolvable private address. This is useful for active
* scanning and non-connectable advertising.
*/
if (require_privacy) {
bdaddr_t nrpa;
while (true) {
/* The non-resolvable private address is generated
* from random six bytes with the two most significant
* bits cleared.
*/
get_random_bytes(&nrpa, 6);
nrpa.b[5] &= 0x3f;
/* The non-resolvable private address shall not be
* equal to the public address.
*/
if (bacmp(&hdev->bdaddr, &nrpa))
break;
}
*own_addr_type = ADDR_LE_DEV_RANDOM;
set_random_addr(req, &nrpa);
return 0;
}
/* If forcing static address is in use or there is no public
* address use the static address as random address (but skip
* the HCI command if the current random address is already the
* static one.
*
* In case BR/EDR has been disabled on a dual-mode controller
* and a static address has been configured, then use that
* address instead of the public BR/EDR address.
*/
if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) ||
!bacmp(&hdev->bdaddr, BDADDR_ANY) ||
(!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) &&
bacmp(&hdev->static_addr, BDADDR_ANY))) {
*own_addr_type = ADDR_LE_DEV_RANDOM;
if (bacmp(&hdev->static_addr, &hdev->random_addr))
hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6,
&hdev->static_addr);
return 0;
}
/* Neither privacy nor static address is being used so use a
* public address.
*/
*own_addr_type = ADDR_LE_DEV_PUBLIC;
return 0;
}
/* Ensure to call hci_req_add_le_scan_disable() first to disable the
* controller based address resolution to be able to reconfigure
* resolving list.
*/
void hci_req_add_le_passive_scan(struct hci_request *req)
{
struct hci_dev *hdev = req->hdev;
u8 own_addr_type;
u8 filter_policy;
u16 window, interval;
/* Default is to enable duplicates filter */
u8 filter_dup = LE_SCAN_FILTER_DUP_ENABLE;
/* Background scanning should run with address resolution */
bool addr_resolv = true;
if (hdev->scanning_paused) {
bt_dev_dbg(hdev, "Scanning is paused for suspend");
return;
}
/* Set require_privacy to false since no SCAN_REQ are send
* during passive scanning. Not using an non-resolvable address
* here is important so that peer devices using direct
* advertising with our address will be correctly reported
* by the controller.
*/
if (hci_update_random_address(req, false, scan_use_rpa(hdev),
&own_addr_type))
return;
if (hdev->enable_advmon_interleave_scan &&
__hci_update_interleaved_scan(hdev))
return;
bt_dev_dbg(hdev, "interleave state %d", hdev->interleave_scan_state);
/* Adding or removing entries from the accept list must
* happen before enabling scanning. The controller does
* not allow accept list modification while scanning.
*/
filter_policy = update_accept_list(req);
/* When the controller is using random resolvable addresses and
* with that having LE privacy enabled, then controllers with
* Extended Scanner Filter Policies support can now enable support
* for handling directed advertising.
*
* So instead of using filter polices 0x00 (no accept list)
* and 0x01 (accept list enabled) use the new filter policies
* 0x02 (no accept list) and 0x03 (accept list enabled).
*/
if (hci_dev_test_flag(hdev, HCI_PRIVACY) &&
(hdev->le_features[0] & HCI_LE_EXT_SCAN_POLICY))
filter_policy |= 0x02;
if (hdev->suspended) {
window = hdev->le_scan_window_suspend;
interval = hdev->le_scan_int_suspend;
} else if (hci_is_le_conn_scanning(hdev)) {
window = hdev->le_scan_window_connect;
interval = hdev->le_scan_int_connect;
} else if (hci_is_adv_monitoring(hdev)) {
window = hdev->le_scan_window_adv_monitor;
interval = hdev->le_scan_int_adv_monitor;
/* Disable duplicates filter when scanning for advertisement
* monitor for the following reasons.
*
* For HW pattern filtering (ex. MSFT), Realtek and Qualcomm
* controllers ignore RSSI_Sampling_Period when the duplicates
* filter is enabled.
*
* For SW pattern filtering, when we're not doing interleaved
* scanning, it is necessary to disable duplicates filter,
* otherwise hosts can only receive one advertisement and it's
* impossible to know if a peer is still in range.
*/
filter_dup = LE_SCAN_FILTER_DUP_DISABLE;
} else {
window = hdev->le_scan_window;
interval = hdev->le_scan_interval;
}
bt_dev_dbg(hdev, "LE passive scan with accept list = %d",
filter_policy);
hci_req_start_scan(req, LE_SCAN_PASSIVE, interval, window,
own_addr_type, filter_policy, filter_dup,
addr_resolv);
}
static int hci_req_add_le_interleaved_scan(struct hci_request *req,
unsigned long opt)
{
struct hci_dev *hdev = req->hdev;
int ret = 0;
hci_dev_lock(hdev);
if (hci_dev_test_flag(hdev, HCI_LE_SCAN))
hci_req_add_le_scan_disable(req, false);
hci_req_add_le_passive_scan(req);
switch (hdev->interleave_scan_state) {
case INTERLEAVE_SCAN_ALLOWLIST:
bt_dev_dbg(hdev, "next state: allowlist");
hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER;
break;
case INTERLEAVE_SCAN_NO_FILTER:
bt_dev_dbg(hdev, "next state: no filter");
hdev->interleave_scan_state = INTERLEAVE_SCAN_ALLOWLIST;
break;
case INTERLEAVE_SCAN_NONE:
BT_ERR("unexpected error");
ret = -1;
}
hci_dev_unlock(hdev);
return ret;
}
static void interleave_scan_work(struct work_struct *work)
{
struct hci_dev *hdev = container_of(work, struct hci_dev,
interleave_scan.work);
u8 status;
unsigned long timeout;
if (hdev->interleave_scan_state == INTERLEAVE_SCAN_ALLOWLIST) {
timeout = msecs_to_jiffies(hdev->advmon_allowlist_duration);
} else if (hdev->interleave_scan_state == INTERLEAVE_SCAN_NO_FILTER) {
timeout = msecs_to_jiffies(hdev->advmon_no_filter_duration);
} else {
bt_dev_err(hdev, "unexpected error");
return;
}
hci_req_sync(hdev, hci_req_add_le_interleaved_scan, 0,
HCI_CMD_TIMEOUT, &status);
/* Don't continue interleaving if it was canceled */
if (is_interleave_scanning(hdev))
queue_delayed_work(hdev->req_workqueue,
&hdev->interleave_scan, timeout);
}
static void set_random_addr(struct hci_request *req, bdaddr_t *rpa)
{
struct hci_dev *hdev = req->hdev;
/* If we're advertising or initiating an LE connection we can't
* go ahead and change the random address at this time. This is
* because the eventual initiator address used for the
* subsequently created connection will be undefined (some
* controllers use the new address and others the one we had
* when the operation started).
*
* In this kind of scenario skip the update and let the random
* address be updated at the next cycle.
*/
if (hci_dev_test_flag(hdev, HCI_LE_ADV) ||
hci_lookup_le_connect(hdev)) {
bt_dev_dbg(hdev, "Deferring random address update");
hci_dev_set_flag(hdev, HCI_RPA_EXPIRED);
return;
}
hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6, rpa);
}
void hci_request_setup(struct hci_dev *hdev)
{
INIT_DELAYED_WORK(&hdev->interleave_scan, interleave_scan_work);
}
void hci_request_cancel_all(struct hci_dev *hdev)
{
hci_cmd_sync_cancel_sync(hdev, ENODEV);
cancel_interleave_scan(hdev);
}
// SPDX-License-Identifier: GPL-2.0
/*
* linux/fs/char_dev.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/kdev_t.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/major.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/kobject.h>
#include <linux/kobj_map.h>
#include <linux/cdev.h>
#include <linux/mutex.h>
#include <linux/backing-dev.h>
#include <linux/tty.h>
#include "internal.h"
static struct kobj_map *cdev_map __ro_after_init;
static DEFINE_MUTEX(chrdevs_lock);
#define CHRDEV_MAJOR_HASH_SIZE 255
static struct char_device_struct {
struct char_device_struct *next;
unsigned int major;
unsigned int baseminor;
int minorct;
char name[64];
struct cdev *cdev; /* will die */
} *chrdevs[CHRDEV_MAJOR_HASH_SIZE];
/* index in the above */
static inline int major_to_index(unsigned major)
{
return major % CHRDEV_MAJOR_HASH_SIZE;
}
#ifdef CONFIG_PROC_FS
void chrdev_show(struct seq_file *f, off_t offset)
{
struct char_device_struct *cd;
mutex_lock(&chrdevs_lock);
for (cd = chrdevs[major_to_index(offset)]; cd; cd = cd->next) {
if (cd->major == offset)
seq_printf(f, "%3d %s\n", cd->major, cd->name);
}
mutex_unlock(&chrdevs_lock);
}
#endif /* CONFIG_PROC_FS */
static int find_dynamic_major(void)
{
int i;
struct char_device_struct *cd;
for (i = ARRAY_SIZE(chrdevs)-1; i >= CHRDEV_MAJOR_DYN_END; i--) {
if (chrdevs[i] == NULL)
return i;
}
for (i = CHRDEV_MAJOR_DYN_EXT_START;
i >= CHRDEV_MAJOR_DYN_EXT_END; i--) {
for (cd = chrdevs[major_to_index(i)]; cd; cd = cd->next)
if (cd->major == i)
break;
if (cd == NULL)
return i;
}
return -EBUSY;
}
/*
* Register a single major with a specified minor range.
*
* If major == 0 this function will dynamically allocate an unused major.
* If major > 0 this function will attempt to reserve the range of minors
* with given major.
*
*/
static struct char_device_struct *
__register_chrdev_region(unsigned int major, unsigned int baseminor,
int minorct, const char *name)
{
struct char_device_struct *cd, *curr, *prev = NULL;
int ret;
int i;
if (major >= CHRDEV_MAJOR_MAX) {
pr_err("CHRDEV \"%s\" major requested (%u) is greater than the maximum (%u)\n",
name, major, CHRDEV_MAJOR_MAX-1);
return ERR_PTR(-EINVAL);
}
if (minorct > MINORMASK + 1 - baseminor) {
pr_err("CHRDEV \"%s\" minor range requested (%u-%u) is out of range of maximum range (%u-%u) for a single major\n",
name, baseminor, baseminor + minorct - 1, 0, MINORMASK);
return ERR_PTR(-EINVAL);
}
cd = kzalloc(sizeof(struct char_device_struct), GFP_KERNEL);
if (cd == NULL)
return ERR_PTR(-ENOMEM);
mutex_lock(&chrdevs_lock);
if (major == 0) {
ret = find_dynamic_major();
if (ret < 0) {
pr_err("CHRDEV \"%s\" dynamic allocation region is full\n",
name);
goto out;
}
major = ret;
}
ret = -EBUSY;
i = major_to_index(major);
for (curr = chrdevs[i]; curr; prev = curr, curr = curr->next) {
if (curr->major < major)
continue;
if (curr->major > major)
break;
if (curr->baseminor + curr->minorct <= baseminor)
continue;
if (curr->baseminor >= baseminor + minorct)
break;
goto out;
}
cd->major = major;
cd->baseminor = baseminor;
cd->minorct = minorct;
strscpy(cd->name, name, sizeof(cd->name));
if (!prev) {
cd->next = curr;
chrdevs[i] = cd;
} else {
cd->next = prev->next;
prev->next = cd;
}
mutex_unlock(&chrdevs_lock);
return cd;
out:
mutex_unlock(&chrdevs_lock);
kfree(cd);
return ERR_PTR(ret);
}
static struct char_device_struct *
__unregister_chrdev_region(unsigned major, unsigned baseminor, int minorct)
{
struct char_device_struct *cd = NULL, **cp;
int i = major_to_index(major);
mutex_lock(&chrdevs_lock);
for (cp = &chrdevs[i]; *cp; cp = &(*cp)->next)
if ((*cp)->major == major &&
(*cp)->baseminor == baseminor &&
(*cp)->minorct == minorct)
break;
if (*cp) {
cd = *cp;
*cp = cd->next;
}
mutex_unlock(&chrdevs_lock);
return cd;
}
/**
* register_chrdev_region() - register a range of device numbers
* @from: the first in the desired range of device numbers; must include
* the major number.
* @count: the number of consecutive device numbers required
* @name: the name of the device or driver.
*
* Return value is zero on success, a negative error code on failure.
*/
int register_chrdev_region(dev_t from, unsigned count, const char *name)
{
struct char_device_struct *cd;
dev_t to = from + count;
dev_t n, next;
for (n = from; n < to; n = next) {
next = MKDEV(MAJOR(n)+1, 0);
if (next > to)
next = to;
cd = __register_chrdev_region(MAJOR(n), MINOR(n),
next - n, name);
if (IS_ERR(cd))
goto fail;
}
return 0;
fail:
to = n;
for (n = from; n < to; n = next) {
next = MKDEV(MAJOR(n)+1, 0);
kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n));
}
return PTR_ERR(cd);
}
/**
* alloc_chrdev_region() - register a range of char device numbers
* @dev: output parameter for first assigned number
* @baseminor: first of the requested range of minor numbers
* @count: the number of minor numbers required
* @name: the name of the associated device or driver
*
* Allocates a range of char device numbers. The major number will be
* chosen dynamically, and returned (along with the first minor number)
* in @dev. Returns zero or a negative error code.
*/
int alloc_chrdev_region(dev_t *dev, unsigned baseminor, unsigned count,
const char *name)
{
struct char_device_struct *cd;
cd = __register_chrdev_region(0, baseminor, count, name);
if (IS_ERR(cd))
return PTR_ERR(cd);
*dev = MKDEV(cd->major, cd->baseminor);
return 0;
}
/**
* __register_chrdev() - create and register a cdev occupying a range of minors
* @major: major device number or 0 for dynamic allocation
* @baseminor: first of the requested range of minor numbers
* @count: the number of minor numbers required
* @name: name of this range of devices
* @fops: file operations associated with this devices
*
* If @major == 0 this functions will dynamically allocate a major and return
* its number.
*
* If @major > 0 this function will attempt to reserve a device with the given
* major number and will return zero on success.
*
* Returns a -ve errno on failure.
*
* The name of this device has nothing to do with the name of the device in
* /dev. It only helps to keep track of the different owners of devices. If
* your module name has only one type of devices it's ok to use e.g. the name
* of the module here.
*/
int __register_chrdev(unsigned int major, unsigned int baseminor,
unsigned int count, const char *name,
const struct file_operations *fops)
{
struct char_device_struct *cd;
struct cdev *cdev;
int err = -ENOMEM;
cd = __register_chrdev_region(major, baseminor, count, name);
if (IS_ERR(cd))
return PTR_ERR(cd);
cdev = cdev_alloc();
if (!cdev)
goto out2;
cdev->owner = fops->owner;
cdev->ops = fops;
kobject_set_name(&cdev->kobj, "%s", name);
err = cdev_add(cdev, MKDEV(cd->major, baseminor), count);
if (err)
goto out;
cd->cdev = cdev;
return major ? 0 : cd->major;
out:
kobject_put(&cdev->kobj);
out2:
kfree(__unregister_chrdev_region(cd->major, baseminor, count));
return err;
}
/**
* unregister_chrdev_region() - unregister a range of device numbers
* @from: the first in the range of numbers to unregister
* @count: the number of device numbers to unregister
*
* This function will unregister a range of @count device numbers,
* starting with @from. The caller should normally be the one who
* allocated those numbers in the first place...
*/
void unregister_chrdev_region(dev_t from, unsigned count)
{
dev_t to = from + count;
dev_t n, next;
for (n = from; n < to; n = next) {
next = MKDEV(MAJOR(n)+1, 0);
if (next > to)
next = to;
kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n));
}
}
/**
* __unregister_chrdev - unregister and destroy a cdev
* @major: major device number
* @baseminor: first of the range of minor numbers
* @count: the number of minor numbers this cdev is occupying
* @name: name of this range of devices
*
* Unregister and destroy the cdev occupying the region described by
* @major, @baseminor and @count. This function undoes what
* __register_chrdev() did.
*/
void __unregister_chrdev(unsigned int major, unsigned int baseminor,
unsigned int count, const char *name)
{
struct char_device_struct *cd;
cd = __unregister_chrdev_region(major, baseminor, count);
if (cd && cd->cdev)
cdev_del(cd->cdev);
kfree(cd);
}
static DEFINE_SPINLOCK(cdev_lock);
static struct kobject *cdev_get(struct cdev *p)
{
struct module *owner = p->owner;
struct kobject *kobj;
if (!try_module_get(owner))
return NULL;
kobj = kobject_get_unless_zero(&p->kobj); if (!kobj) module_put(owner);
return kobj;
}
void cdev_put(struct cdev *p)
{
if (p) { struct module *owner = p->owner;
kobject_put(&p->kobj);
module_put(owner);
}
}
/*
* Called every time a character special file is opened
*/
static int chrdev_open(struct inode *inode, struct file *filp)
{
const struct file_operations *fops;
struct cdev *p;
struct cdev *new = NULL;
int ret = 0;
spin_lock(&cdev_lock);
p = inode->i_cdev;
if (!p) {
struct kobject *kobj;
int idx;
spin_unlock(&cdev_lock);
kobj = kobj_lookup(cdev_map, inode->i_rdev, &idx);
if (!kobj)
return -ENXIO;
new = container_of(kobj, struct cdev, kobj);
spin_lock(&cdev_lock);
/* Check i_cdev again in case somebody beat us to it while
we dropped the lock. */
p = inode->i_cdev;
if (!p) {
inode->i_cdev = p = new; list_add(&inode->i_devices, &p->list);
new = NULL;
} else if (!cdev_get(p)) ret = -ENXIO; } else if (!cdev_get(p))
ret = -ENXIO;
spin_unlock(&cdev_lock);
cdev_put(new);
if (ret)
return ret;
ret = -ENXIO;
fops = fops_get(p->ops);
if (!fops)
goto out_cdev_put;
replace_fops(filp, fops);
if (filp->f_op->open) {
ret = filp->f_op->open(inode, filp);
if (ret)
goto out_cdev_put;
}
return 0;
out_cdev_put:
cdev_put(p);
return ret;
}
void cd_forget(struct inode *inode)
{
spin_lock(&cdev_lock);
list_del_init(&inode->i_devices);
inode->i_cdev = NULL;
inode->i_mapping = &inode->i_data;
spin_unlock(&cdev_lock);
}
static void cdev_purge(struct cdev *cdev)
{
spin_lock(&cdev_lock);
while (!list_empty(&cdev->list)) {
struct inode *inode;
inode = container_of(cdev->list.next, struct inode, i_devices); list_del_init(&inode->i_devices);
inode->i_cdev = NULL;
}
spin_unlock(&cdev_lock);
}
/*
* Dummy default file-operations: the only thing this does
* is contain the open that then fills in the correct operations
* depending on the special file...
*/
const struct file_operations def_chr_fops = {
.open = chrdev_open,
.llseek = noop_llseek,
};
static struct kobject *exact_match(dev_t dev, int *part, void *data)
{
struct cdev *p = data;
return &p->kobj;
}
static int exact_lock(dev_t dev, void *data)
{
struct cdev *p = data;
return cdev_get(p) ? 0 : -1;
}
/**
* cdev_add() - add a char device to the system
* @p: the cdev structure for the device
* @dev: the first device number for which this device is responsible
* @count: the number of consecutive minor numbers corresponding to this
* device
*
* cdev_add() adds the device represented by @p to the system, making it
* live immediately. A negative error code is returned on failure.
*/
int cdev_add(struct cdev *p, dev_t dev, unsigned count)
{
int error;
p->dev = dev;
p->count = count;
if (WARN_ON(dev == WHITEOUT_DEV)) {
error = -EBUSY;
goto err;
}
error = kobj_map(cdev_map, dev, count, NULL,
exact_match, exact_lock, p);
if (error)
goto err;
kobject_get(p->kobj.parent);
return 0;
err:
kfree_const(p->kobj.name);
p->kobj.name = NULL;
return error;
}
/**
* cdev_set_parent() - set the parent kobject for a char device
* @p: the cdev structure
* @kobj: the kobject to take a reference to
*
* cdev_set_parent() sets a parent kobject which will be referenced
* appropriately so the parent is not freed before the cdev. This
* should be called before cdev_add.
*/
void cdev_set_parent(struct cdev *p, struct kobject *kobj)
{
WARN_ON(!kobj->state_initialized);
p->kobj.parent = kobj;
}
/**
* cdev_device_add() - add a char device and it's corresponding
* struct device, linkink
* @dev: the device structure
* @cdev: the cdev structure
*
* cdev_device_add() adds the char device represented by @cdev to the system,
* just as cdev_add does. It then adds @dev to the system using device_add
* The dev_t for the char device will be taken from the struct device which
* needs to be initialized first. This helper function correctly takes a
* reference to the parent device so the parent will not get released until
* all references to the cdev are released.
*
* This helper uses dev->devt for the device number. If it is not set
* it will not add the cdev and it will be equivalent to device_add.
*
* This function should be used whenever the struct cdev and the
* struct device are members of the same structure whose lifetime is
* managed by the struct device.
*
* NOTE: Callers must assume that userspace was able to open the cdev and
* can call cdev fops callbacks at any time, even if this function fails.
*/
int cdev_device_add(struct cdev *cdev, struct device *dev)
{
int rc = 0;
if (dev->devt) {
cdev_set_parent(cdev, &dev->kobj);
rc = cdev_add(cdev, dev->devt, 1);
if (rc)
return rc;
}
rc = device_add(dev);
if (rc && dev->devt)
cdev_del(cdev);
return rc;
}
/**
* cdev_device_del() - inverse of cdev_device_add
* @dev: the device structure
* @cdev: the cdev structure
*
* cdev_device_del() is a helper function to call cdev_del and device_del.
* It should be used whenever cdev_device_add is used.
*
* If dev->devt is not set it will not remove the cdev and will be equivalent
* to device_del.
*
* NOTE: This guarantees that associated sysfs callbacks are not running
* or runnable, however any cdevs already open will remain and their fops
* will still be callable even after this function returns.
*/
void cdev_device_del(struct cdev *cdev, struct device *dev)
{
device_del(dev);
if (dev->devt)
cdev_del(cdev);
}
static void cdev_unmap(dev_t dev, unsigned count)
{
kobj_unmap(cdev_map, dev, count);
}
/**
* cdev_del() - remove a cdev from the system
* @p: the cdev structure to be removed
*
* cdev_del() removes @p from the system, possibly freeing the structure
* itself.
*
* NOTE: This guarantees that cdev device will no longer be able to be
* opened, however any cdevs already open will remain and their fops will
* still be callable even after cdev_del returns.
*/
void cdev_del(struct cdev *p)
{
cdev_unmap(p->dev, p->count);
kobject_put(&p->kobj);
}
static void cdev_default_release(struct kobject *kobj)
{
struct cdev *p = container_of(kobj, struct cdev, kobj);
struct kobject *parent = kobj->parent;
cdev_purge(p);
kobject_put(parent);
}
static void cdev_dynamic_release(struct kobject *kobj)
{
struct cdev *p = container_of(kobj, struct cdev, kobj);
struct kobject *parent = kobj->parent;
cdev_purge(p);
kfree(p);
kobject_put(parent);
}
static struct kobj_type ktype_cdev_default = {
.release = cdev_default_release,
};
static struct kobj_type ktype_cdev_dynamic = {
.release = cdev_dynamic_release,
};
/**
* cdev_alloc() - allocate a cdev structure
*
* Allocates and returns a cdev structure, or NULL on failure.
*/
struct cdev *cdev_alloc(void)
{
struct cdev *p = kzalloc(sizeof(struct cdev), GFP_KERNEL);
if (p) {
INIT_LIST_HEAD(&p->list);
kobject_init(&p->kobj, &ktype_cdev_dynamic);
}
return p;
}
/**
* cdev_init() - initialize a cdev structure
* @cdev: the structure to initialize
* @fops: the file_operations for this device
*
* Initializes @cdev, remembering @fops, making it ready to add to the
* system with cdev_add().
*/
void cdev_init(struct cdev *cdev, const struct file_operations *fops)
{
memset(cdev, 0, sizeof *cdev);
INIT_LIST_HEAD(&cdev->list);
kobject_init(&cdev->kobj, &ktype_cdev_default);
cdev->ops = fops;
}
static struct kobject *base_probe(dev_t dev, int *part, void *data)
{
if (request_module("char-major-%d-%d", MAJOR(dev), MINOR(dev)) > 0)
/* Make old-style 2.4 aliases work */
request_module("char-major-%d", MAJOR(dev));
return NULL;
}
void __init chrdev_init(void)
{
cdev_map = kobj_map_init(base_probe, &chrdevs_lock);
}
/* Let modules do char dev stuff */
EXPORT_SYMBOL(register_chrdev_region);
EXPORT_SYMBOL(unregister_chrdev_region);
EXPORT_SYMBOL(alloc_chrdev_region);
EXPORT_SYMBOL(cdev_init);
EXPORT_SYMBOL(cdev_alloc);
EXPORT_SYMBOL(cdev_del);
EXPORT_SYMBOL(cdev_add);
EXPORT_SYMBOL(cdev_set_parent);
EXPORT_SYMBOL(cdev_device_add);
EXPORT_SYMBOL(cdev_device_del);
EXPORT_SYMBOL(__register_chrdev);
EXPORT_SYMBOL(__unregister_chrdev);
/*
* Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
* Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
* Copyright (c) 2009-2010, Code Aurora Forum.
* Copyright 2016 Intel Corp.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef _DRM_DRV_H_
#define _DRM_DRV_H_
#include <linux/list.h>
#include <linux/irqreturn.h>
#include <video/nomodeset.h>
#include <drm/drm_device.h>
struct drm_file;
struct drm_gem_object;
struct drm_master;
struct drm_minor;
struct dma_buf;
struct dma_buf_attachment;
struct drm_display_mode;
struct drm_mode_create_dumb;
struct drm_printer;
struct sg_table;
/**
* enum drm_driver_feature - feature flags
*
* See &drm_driver.driver_features, drm_device.driver_features and
* drm_core_check_feature().
*/
enum drm_driver_feature {
/**
* @DRIVER_GEM:
*
* Driver use the GEM memory manager. This should be set for all modern
* drivers.
*/
DRIVER_GEM = BIT(0),
/**
* @DRIVER_MODESET:
*
* Driver supports mode setting interfaces (KMS).
*/
DRIVER_MODESET = BIT(1),
/**
* @DRIVER_RENDER:
*
* Driver supports dedicated render nodes. See also the :ref:`section on
* render nodes <drm_render_node>` for details.
*/
DRIVER_RENDER = BIT(3),
/**
* @DRIVER_ATOMIC:
*
* Driver supports the full atomic modesetting userspace API. Drivers
* which only use atomic internally, but do not support the full
* userspace API (e.g. not all properties converted to atomic, or
* multi-plane updates are not guaranteed to be tear-free) should not
* set this flag.
*/
DRIVER_ATOMIC = BIT(4),
/**
* @DRIVER_SYNCOBJ:
*
* Driver supports &drm_syncobj for explicit synchronization of command
* submission.
*/
DRIVER_SYNCOBJ = BIT(5),
/**
* @DRIVER_SYNCOBJ_TIMELINE:
*
* Driver supports the timeline flavor of &drm_syncobj for explicit
* synchronization of command submission.
*/
DRIVER_SYNCOBJ_TIMELINE = BIT(6),
/**
* @DRIVER_COMPUTE_ACCEL:
*
* Driver supports compute acceleration devices. This flag is mutually exclusive with
* @DRIVER_RENDER and @DRIVER_MODESET. Devices that support both graphics and compute
* acceleration should be handled by two drivers that are connected using auxiliary bus.
*/
DRIVER_COMPUTE_ACCEL = BIT(7),
/**
* @DRIVER_GEM_GPUVA:
*
* Driver supports user defined GPU VA bindings for GEM objects.
*/
DRIVER_GEM_GPUVA = BIT(8),
/**
* @DRIVER_CURSOR_HOTSPOT:
*
* Driver supports and requires cursor hotspot information in the
* cursor plane (e.g. cursor plane has to actually track the mouse
* cursor and the clients are required to set hotspot in order for
* the cursor planes to work correctly).
*/
DRIVER_CURSOR_HOTSPOT = BIT(9),
/* IMPORTANT: Below are all the legacy flags, add new ones above. */
/**
* @DRIVER_USE_AGP:
*
* Set up DRM AGP support, see drm_agp_init(), the DRM core will manage
* AGP resources. New drivers don't need this.
*/
DRIVER_USE_AGP = BIT(25),
/**
* @DRIVER_LEGACY:
*
* Denote a legacy driver using shadow attach. Do not use.
*/
DRIVER_LEGACY = BIT(26),
/**
* @DRIVER_PCI_DMA:
*
* Driver is capable of PCI DMA, mapping of PCI DMA buffers to userspace
* will be enabled. Only for legacy drivers. Do not use.
*/
DRIVER_PCI_DMA = BIT(27),
/**
* @DRIVER_SG:
*
* Driver can perform scatter/gather DMA, allocation and mapping of
* scatter/gather buffers will be enabled. Only for legacy drivers. Do
* not use.
*/
DRIVER_SG = BIT(28),
/**
* @DRIVER_HAVE_DMA:
*
* Driver supports DMA, the userspace DMA API will be supported. Only
* for legacy drivers. Do not use.
*/
DRIVER_HAVE_DMA = BIT(29),
/**
* @DRIVER_HAVE_IRQ:
*
* Legacy irq support. Only for legacy drivers. Do not use.
*/
DRIVER_HAVE_IRQ = BIT(30),
};
/**
* struct drm_driver - DRM driver structure
*
* This structure represent the common code for a family of cards. There will be
* one &struct drm_device for each card present in this family. It contains lots
* of vfunc entries, and a pile of those probably should be moved to more
* appropriate places like &drm_mode_config_funcs or into a new operations
* structure for GEM drivers.
*/
struct drm_driver {
/**
* @load:
*
* Backward-compatible driver callback to complete initialization steps
* after the driver is registered. For this reason, may suffer from
* race conditions and its use is deprecated for new drivers. It is
* therefore only supported for existing drivers not yet converted to
* the new scheme. See devm_drm_dev_alloc() and drm_dev_register() for
* proper and race-free way to set up a &struct drm_device.
*
* This is deprecated, do not use!
*
* Returns:
*
* Zero on success, non-zero value on failure.
*/
int (*load) (struct drm_device *, unsigned long flags);
/**
* @open:
*
* Driver callback when a new &struct drm_file is opened. Useful for
* setting up driver-private data structures like buffer allocators,
* execution contexts or similar things. Such driver-private resources
* must be released again in @postclose.
*
* Since the display/modeset side of DRM can only be owned by exactly
* one &struct drm_file (see &drm_file.is_master and &drm_device.master)
* there should never be a need to set up any modeset related resources
* in this callback. Doing so would be a driver design bug.
*
* Returns:
*
* 0 on success, a negative error code on failure, which will be
* promoted to userspace as the result of the open() system call.
*/
int (*open) (struct drm_device *, struct drm_file *);
/**
* @postclose:
*
* One of the driver callbacks when a new &struct drm_file is closed.
* Useful for tearing down driver-private data structures allocated in
* @open like buffer allocators, execution contexts or similar things.
*
* Since the display/modeset side of DRM can only be owned by exactly
* one &struct drm_file (see &drm_file.is_master and &drm_device.master)
* there should never be a need to tear down any modeset related
* resources in this callback. Doing so would be a driver design bug.
*/
void (*postclose) (struct drm_device *, struct drm_file *);
/**
* @lastclose:
*
* Called when the last &struct drm_file has been closed and there's
* currently no userspace client for the &struct drm_device.
*
* Modern drivers should only use this to force-restore the fbdev
* framebuffer using drm_fb_helper_restore_fbdev_mode_unlocked().
* Anything else would indicate there's something seriously wrong.
* Modern drivers can also use this to execute delayed power switching
* state changes, e.g. in conjunction with the :ref:`vga_switcheroo`
* infrastructure.
*
* This is called after @postclose hook has been called.
*
* NOTE:
*
* All legacy drivers use this callback to de-initialize the hardware.
* This is purely because of the shadow-attach model, where the DRM
* kernel driver does not really own the hardware. Instead ownershipe is
* handled with the help of userspace through an inheritedly racy dance
* to set/unset the VT into raw mode.
*
* Legacy drivers initialize the hardware in the @firstopen callback,
* which isn't even called for modern drivers.
*/
void (*lastclose) (struct drm_device *);
/**
* @unload:
*
* Reverse the effects of the driver load callback. Ideally,
* the clean up performed by the driver should happen in the
* reverse order of the initialization. Similarly to the load
* hook, this handler is deprecated and its usage should be
* dropped in favor of an open-coded teardown function at the
* driver layer. See drm_dev_unregister() and drm_dev_put()
* for the proper way to remove a &struct drm_device.
*
* The unload() hook is called right after unregistering
* the device.
*
*/
void (*unload) (struct drm_device *);
/**
* @release:
*
* Optional callback for destroying device data after the final
* reference is released, i.e. the device is being destroyed.
*
* This is deprecated, clean up all memory allocations associated with a
* &drm_device using drmm_add_action(), drmm_kmalloc() and related
* managed resources functions.
*/
void (*release) (struct drm_device *);
/**
* @master_set:
*
* Called whenever the minor master is set. Only used by vmwgfx.
*/
void (*master_set)(struct drm_device *dev, struct drm_file *file_priv,
bool from_open);
/**
* @master_drop:
*
* Called whenever the minor master is dropped. Only used by vmwgfx.
*/
void (*master_drop)(struct drm_device *dev, struct drm_file *file_priv);
/**
* @debugfs_init:
*
* Allows drivers to create driver-specific debugfs files.
*/
void (*debugfs_init)(struct drm_minor *minor);
/**
* @gem_create_object: constructor for gem objects
*
* Hook for allocating the GEM object struct, for use by the CMA
* and SHMEM GEM helpers. Returns a GEM object on success, or an
* ERR_PTR()-encoded error code otherwise.
*/
struct drm_gem_object *(*gem_create_object)(struct drm_device *dev,
size_t size);
/**
* @prime_handle_to_fd:
*
* PRIME export function. Only used by vmwgfx.
*/
int (*prime_handle_to_fd)(struct drm_device *dev, struct drm_file *file_priv,
uint32_t handle, uint32_t flags, int *prime_fd);
/**
* @prime_fd_to_handle:
*
* PRIME import function. Only used by vmwgfx.
*/
int (*prime_fd_to_handle)(struct drm_device *dev, struct drm_file *file_priv,
int prime_fd, uint32_t *handle);
/**
* @gem_prime_import:
*
* Import hook for GEM drivers.
*
* This defaults to drm_gem_prime_import() if not set.
*/
struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
struct dma_buf *dma_buf);
/**
* @gem_prime_import_sg_table:
*
* Optional hook used by the PRIME helper functions
* drm_gem_prime_import() respectively drm_gem_prime_import_dev().
*/
struct drm_gem_object *(*gem_prime_import_sg_table)(
struct drm_device *dev,
struct dma_buf_attachment *attach,
struct sg_table *sgt);
/**
* @dumb_create:
*
* This creates a new dumb buffer in the driver's backing storage manager (GEM,
* TTM or something else entirely) and returns the resulting buffer handle. This
* handle can then be wrapped up into a framebuffer modeset object.
*
* Note that userspace is not allowed to use such objects for render
* acceleration - drivers must create their own private ioctls for such a use
* case.
*
* Width, height and depth are specified in the &drm_mode_create_dumb
* argument. The callback needs to fill the handle, pitch and size for
* the created buffer.
*
* Called by the user via ioctl.
*
* Returns:
*
* Zero on success, negative errno on failure.
*/
int (*dumb_create)(struct drm_file *file_priv,
struct drm_device *dev,
struct drm_mode_create_dumb *args);
/**
* @dumb_map_offset:
*
* Allocate an offset in the drm device node's address space to be able to
* memory map a dumb buffer.
*
* The default implementation is drm_gem_create_mmap_offset(). GEM based
* drivers must not overwrite this.
*
* Called by the user via ioctl.
*
* Returns:
*
* Zero on success, negative errno on failure.
*/
int (*dumb_map_offset)(struct drm_file *file_priv,
struct drm_device *dev, uint32_t handle,
uint64_t *offset);
/**
* @show_fdinfo:
*
* Print device specific fdinfo. See Documentation/gpu/drm-usage-stats.rst.
*/
void (*show_fdinfo)(struct drm_printer *p, struct drm_file *f);
/** @major: driver major number */
int major;
/** @minor: driver minor number */
int minor;
/** @patchlevel: driver patch level */
int patchlevel;
/** @name: driver name */
char *name;
/** @desc: driver description */
char *desc;
/** @date: driver date */
char *date;
/**
* @driver_features:
* Driver features, see &enum drm_driver_feature. Drivers can disable
* some features on a per-instance basis using
* &drm_device.driver_features.
*/
u32 driver_features;
/**
* @ioctls:
*
* Array of driver-private IOCTL description entries. See the chapter on
* :ref:`IOCTL support in the userland interfaces
* chapter<drm_driver_ioctl>` for the full details.
*/
const struct drm_ioctl_desc *ioctls;
/** @num_ioctls: Number of entries in @ioctls. */
int num_ioctls;
/**
* @fops:
*
* File operations for the DRM device node. See the discussion in
* :ref:`file operations<drm_driver_fops>` for in-depth coverage and
* some examples.
*/
const struct file_operations *fops;
};
void *__devm_drm_dev_alloc(struct device *parent,
const struct drm_driver *driver,
size_t size, size_t offset);
/**
* devm_drm_dev_alloc - Resource managed allocation of a &drm_device instance
* @parent: Parent device object
* @driver: DRM driver
* @type: the type of the struct which contains struct &drm_device
* @member: the name of the &drm_device within @type.
*
* This allocates and initialize a new DRM device. No device registration is done.
* Call drm_dev_register() to advertice the device to user space and register it
* with other core subsystems. This should be done last in the device
* initialization sequence to make sure userspace can't access an inconsistent
* state.
*
* The initial ref-count of the object is 1. Use drm_dev_get() and
* drm_dev_put() to take and drop further ref-counts.
*
* It is recommended that drivers embed &struct drm_device into their own device
* structure.
*
* Note that this manages the lifetime of the resulting &drm_device
* automatically using devres. The DRM device initialized with this function is
* automatically put on driver detach using drm_dev_put().
*
* RETURNS:
* Pointer to new DRM device, or ERR_PTR on failure.
*/
#define devm_drm_dev_alloc(parent, driver, type, member) \
((type *) __devm_drm_dev_alloc(parent, driver, sizeof(type), \
offsetof(type, member)))
struct drm_device *drm_dev_alloc(const struct drm_driver *driver,
struct device *parent);
int drm_dev_register(struct drm_device *dev, unsigned long flags);
void drm_dev_unregister(struct drm_device *dev);
void drm_dev_get(struct drm_device *dev);
void drm_dev_put(struct drm_device *dev);
void drm_put_dev(struct drm_device *dev);
bool drm_dev_enter(struct drm_device *dev, int *idx);
void drm_dev_exit(int idx);
void drm_dev_unplug(struct drm_device *dev);
/**
* drm_dev_is_unplugged - is a DRM device unplugged
* @dev: DRM device
*
* This function can be called to check whether a hotpluggable is unplugged.
* Unplugging itself is singalled through drm_dev_unplug(). If a device is
* unplugged, these two functions guarantee that any store before calling
* drm_dev_unplug() is visible to callers of this function after it completes
*
* WARNING: This function fundamentally races against drm_dev_unplug(). It is
* recommended that drivers instead use the underlying drm_dev_enter() and
* drm_dev_exit() function pairs.
*/
static inline bool drm_dev_is_unplugged(struct drm_device *dev)
{
int idx;
if (drm_dev_enter(dev, &idx)) {
drm_dev_exit(idx);
return false;
}
return true;
}
/**
* drm_core_check_all_features - check driver feature flags mask
* @dev: DRM device to check
* @features: feature flag(s) mask
*
* This checks @dev for driver features, see &drm_driver.driver_features,
* &drm_device.driver_features, and the various &enum drm_driver_feature flags.
*
* Returns true if all features in the @features mask are supported, false
* otherwise.
*/
static inline bool drm_core_check_all_features(const struct drm_device *dev,
u32 features)
{
u32 supported = dev->driver->driver_features & dev->driver_features;
return features && (supported & features) == features;
}
/**
* drm_core_check_feature - check driver feature flags
* @dev: DRM device to check
* @feature: feature flag
*
* This checks @dev for driver features, see &drm_driver.driver_features,
* &drm_device.driver_features, and the various &enum drm_driver_feature flags.
*
* Returns true if the @feature is supported, false otherwise.
*/
static inline bool drm_core_check_feature(const struct drm_device *dev,
enum drm_driver_feature feature)
{
return drm_core_check_all_features(dev, feature);
}
/**
* drm_drv_uses_atomic_modeset - check if the driver implements
* atomic_commit()
* @dev: DRM device
*
* This check is useful if drivers do not have DRIVER_ATOMIC set but
* have atomic modesetting internally implemented.
*/
static inline bool drm_drv_uses_atomic_modeset(struct drm_device *dev)
{
return drm_core_check_feature(dev, DRIVER_ATOMIC) ||
(dev->mode_config.funcs && dev->mode_config.funcs->atomic_commit != NULL);
}
/* TODO: Inline drm_firmware_drivers_only() in all its callers. */
static inline bool drm_firmware_drivers_only(void)
{
return video_firmware_drivers_only();
}
#if defined(CONFIG_DEBUG_FS)
void drm_debugfs_dev_init(struct drm_device *dev, struct dentry *root);
#else
static inline void drm_debugfs_dev_init(struct drm_device *dev, struct dentry *root)
{
}
#endif
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
* Copyright (C) 2005-2006, Thomas Gleixner, Russell King
*
* This file contains the interrupt descriptor management code. Detailed
* information is available in Documentation/core-api/genericirq.rst
*
*/
#include <linux/irq.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/maple_tree.h>
#include <linux/irqdomain.h>
#include <linux/sysfs.h>
#include "internals.h"
/*
* lockdep: we want to handle all irq_desc locks as a single lock-class:
*/
static struct lock_class_key irq_desc_lock_class;
#if defined(CONFIG_SMP)
static int __init irq_affinity_setup(char *str)
{
alloc_bootmem_cpumask_var(&irq_default_affinity);
cpulist_parse(str, irq_default_affinity);
/*
* Set at least the boot cpu. We don't want to end up with
* bugreports caused by random commandline masks
*/
cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
return 1;
}
__setup("irqaffinity=", irq_affinity_setup);
static void __init init_irq_default_affinity(void)
{
if (!cpumask_available(irq_default_affinity))
zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
if (cpumask_empty(irq_default_affinity))
cpumask_setall(irq_default_affinity);
}
#else
static void __init init_irq_default_affinity(void)
{
}
#endif
#ifdef CONFIG_SMP
static int alloc_masks(struct irq_desc *desc, int node)
{
if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
GFP_KERNEL, node))
return -ENOMEM;
#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity,
GFP_KERNEL, node)) {
free_cpumask_var(desc->irq_common_data.affinity);
return -ENOMEM;
}
#endif
#ifdef CONFIG_GENERIC_PENDING_IRQ
if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) {
#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
free_cpumask_var(desc->irq_common_data.effective_affinity);
#endif
free_cpumask_var(desc->irq_common_data.affinity);
return -ENOMEM;
}
#endif
return 0;
}
static void desc_smp_init(struct irq_desc *desc, int node,
const struct cpumask *affinity)
{
if (!affinity)
affinity = irq_default_affinity;
cpumask_copy(desc->irq_common_data.affinity, affinity);
#ifdef CONFIG_GENERIC_PENDING_IRQ
cpumask_clear(desc->pending_mask);
#endif
#ifdef CONFIG_NUMA
desc->irq_common_data.node = node;
#endif
}
static void free_masks(struct irq_desc *desc)
{
#ifdef CONFIG_GENERIC_PENDING_IRQ
free_cpumask_var(desc->pending_mask);
#endif
free_cpumask_var(desc->irq_common_data.affinity);
#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
free_cpumask_var(desc->irq_common_data.effective_affinity);
#endif
}
#else
static inline int
alloc_masks(struct irq_desc *desc, int node) { return 0; }
static inline void
desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }
static inline void free_masks(struct irq_desc *desc) { }
#endif
static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
const struct cpumask *affinity, struct module *owner)
{
int cpu;
desc->irq_common_data.handler_data = NULL;
desc->irq_common_data.msi_desc = NULL;
desc->irq_data.common = &desc->irq_common_data;
desc->irq_data.irq = irq;
desc->irq_data.chip = &no_irq_chip;
desc->irq_data.chip_data = NULL;
irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
desc->handle_irq = handle_bad_irq;
desc->depth = 1;
desc->irq_count = 0;
desc->irqs_unhandled = 0;
desc->tot_count = 0;
desc->name = NULL;
desc->owner = owner;
for_each_possible_cpu(cpu)
*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
desc_smp_init(desc, node, affinity);
}
int nr_irqs = NR_IRQS;
EXPORT_SYMBOL_GPL(nr_irqs);
static DEFINE_MUTEX(sparse_irq_lock);
static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs,
MT_FLAGS_ALLOC_RANGE |
MT_FLAGS_LOCK_EXTERN |
MT_FLAGS_USE_RCU,
sparse_irq_lock);
static int irq_find_free_area(unsigned int from, unsigned int cnt)
{
MA_STATE(mas, &sparse_irqs, 0, 0);
if (mas_empty_area(&mas, from, MAX_SPARSE_IRQS, cnt))
return -ENOSPC;
return mas.index;
}
static unsigned int irq_find_at_or_after(unsigned int offset)
{
unsigned long index = offset;
struct irq_desc *desc = mt_find(&sparse_irqs, &index, nr_irqs);
return desc ? irq_desc_get_irq(desc) : nr_irqs;
}
static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
{
MA_STATE(mas, &sparse_irqs, irq, irq);
WARN_ON(mas_store_gfp(&mas, desc, GFP_KERNEL) != 0);
}
static void delete_irq_desc(unsigned int irq)
{
MA_STATE(mas, &sparse_irqs, irq, irq);
mas_erase(&mas);
}
#ifdef CONFIG_SPARSE_IRQ
static const struct kobj_type irq_kobj_type;
#endif
static int init_desc(struct irq_desc *desc, int irq, int node,
unsigned int flags,
const struct cpumask *affinity,
struct module *owner)
{
desc->kstat_irqs = alloc_percpu(unsigned int);
if (!desc->kstat_irqs)
return -ENOMEM;
if (alloc_masks(desc, node)) {
free_percpu(desc->kstat_irqs);
return -ENOMEM;
}
raw_spin_lock_init(&desc->lock);
lockdep_set_class(&desc->lock, &irq_desc_lock_class);
mutex_init(&desc->request_mutex);
init_waitqueue_head(&desc->wait_for_threads);
desc_set_defaults(irq, desc, node, affinity, owner);
irqd_set(&desc->irq_data, flags);
irq_resend_init(desc);
#ifdef CONFIG_SPARSE_IRQ
kobject_init(&desc->kobj, &irq_kobj_type);
init_rcu_head(&desc->rcu);
#endif
return 0;
}
#ifdef CONFIG_SPARSE_IRQ
static void irq_kobj_release(struct kobject *kobj);
#ifdef CONFIG_SYSFS
static struct kobject *irq_kobj_base;
#define IRQ_ATTR_RO(_name) \
static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
static ssize_t per_cpu_count_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
ssize_t ret = 0;
char *p = "";
int cpu;
for_each_possible_cpu(cpu) {
unsigned int c = irq_desc_kstat_cpu(desc, cpu);
ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c);
p = ",";
}
ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
return ret;
}
IRQ_ATTR_RO(per_cpu_count);
static ssize_t chip_name_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
ssize_t ret = 0;
raw_spin_lock_irq(&desc->lock);
if (desc->irq_data.chip && desc->irq_data.chip->name) {
ret = scnprintf(buf, PAGE_SIZE, "%s\n",
desc->irq_data.chip->name);
}
raw_spin_unlock_irq(&desc->lock);
return ret;
}
IRQ_ATTR_RO(chip_name);
static ssize_t hwirq_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
ssize_t ret = 0;
raw_spin_lock_irq(&desc->lock);
if (desc->irq_data.domain)
ret = sprintf(buf, "%lu\n", desc->irq_data.hwirq);
raw_spin_unlock_irq(&desc->lock);
return ret;
}
IRQ_ATTR_RO(hwirq);
static ssize_t type_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
ssize_t ret = 0;
raw_spin_lock_irq(&desc->lock);
ret = sprintf(buf, "%s\n",
irqd_is_level_type(&desc->irq_data) ? "level" : "edge");
raw_spin_unlock_irq(&desc->lock);
return ret;
}
IRQ_ATTR_RO(type);
static ssize_t wakeup_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
ssize_t ret = 0;
raw_spin_lock_irq(&desc->lock);
ret = sprintf(buf, "%s\n",
irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled");
raw_spin_unlock_irq(&desc->lock);
return ret;
}
IRQ_ATTR_RO(wakeup);
static ssize_t name_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
ssize_t ret = 0;
raw_spin_lock_irq(&desc->lock);
if (desc->name)
ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name);
raw_spin_unlock_irq(&desc->lock);
return ret;
}
IRQ_ATTR_RO(name);
static ssize_t actions_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
struct irqaction *action;
ssize_t ret = 0;
char *p = "";
raw_spin_lock_irq(&desc->lock);
for_each_action_of_desc(desc, action) {
ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s",
p, action->name);
p = ",";
}
raw_spin_unlock_irq(&desc->lock);
if (ret)
ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
return ret;
}
IRQ_ATTR_RO(actions);
static struct attribute *irq_attrs[] = {
&per_cpu_count_attr.attr,
&chip_name_attr.attr,
&hwirq_attr.attr,
&type_attr.attr,
&wakeup_attr.attr,
&name_attr.attr,
&actions_attr.attr,
NULL
};
ATTRIBUTE_GROUPS(irq);
static const struct kobj_type irq_kobj_type = {
.release = irq_kobj_release,
.sysfs_ops = &kobj_sysfs_ops,
.default_groups = irq_groups,
};
static void irq_sysfs_add(int irq, struct irq_desc *desc)
{
if (irq_kobj_base) {
/*
* Continue even in case of failure as this is nothing
* crucial and failures in the late irq_sysfs_init()
* cannot be rolled back.
*/
if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))
pr_warn("Failed to add kobject for irq %d\n", irq);
else
desc->istate |= IRQS_SYSFS;
}
}
static void irq_sysfs_del(struct irq_desc *desc)
{
/*
* Only invoke kobject_del() when kobject_add() was successfully
* invoked for the descriptor. This covers both early boot, where
* sysfs is not initialized yet, and the case of a failed
* kobject_add() invocation.
*/
if (desc->istate & IRQS_SYSFS)
kobject_del(&desc->kobj);
}
static int __init irq_sysfs_init(void)
{
struct irq_desc *desc;
int irq;
/* Prevent concurrent irq alloc/free */
irq_lock_sparse();
irq_kobj_base = kobject_create_and_add("irq", kernel_kobj);
if (!irq_kobj_base) {
irq_unlock_sparse();
return -ENOMEM;
}
/* Add the already allocated interrupts */
for_each_irq_desc(irq, desc)
irq_sysfs_add(irq, desc);
irq_unlock_sparse();
return 0;
}
postcore_initcall(irq_sysfs_init);
#else /* !CONFIG_SYSFS */
static const struct kobj_type irq_kobj_type = {
.release = irq_kobj_release,
};
static void irq_sysfs_add(int irq, struct irq_desc *desc) {}
static void irq_sysfs_del(struct irq_desc *desc) {}
#endif /* CONFIG_SYSFS */
struct irq_desc *irq_to_desc(unsigned int irq)
{
return mtree_load(&sparse_irqs, irq);
}
#ifdef CONFIG_KVM_BOOK3S_64_HV_MODULE
EXPORT_SYMBOL_GPL(irq_to_desc);
#endif
void irq_lock_sparse(void)
{
mutex_lock(&sparse_irq_lock);
}
void irq_unlock_sparse(void)
{
mutex_unlock(&sparse_irq_lock);
}
static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
const struct cpumask *affinity,
struct module *owner)
{
struct irq_desc *desc;
int ret;
desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node);
if (!desc)
return NULL;
ret = init_desc(desc, irq, node, flags, affinity, owner);
if (unlikely(ret)) {
kfree(desc);
return NULL;
}
return desc;
}
static void irq_kobj_release(struct kobject *kobj)
{
struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
free_masks(desc);
free_percpu(desc->kstat_irqs);
kfree(desc);
}
static void delayed_free_desc(struct rcu_head *rhp)
{
struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu);
kobject_put(&desc->kobj);
}
static void free_desc(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
irq_remove_debugfs_entry(desc);
unregister_irq_proc(irq, desc);
/*
* sparse_irq_lock protects also show_interrupts() and
* kstat_irq_usr(). Once we deleted the descriptor from the
* sparse tree we can free it. Access in proc will fail to
* lookup the descriptor.
*
* The sysfs entry must be serialized against a concurrent
* irq_sysfs_init() as well.
*/
irq_sysfs_del(desc);
delete_irq_desc(irq);
/*
* We free the descriptor, masks and stat fields via RCU. That
* allows demultiplex interrupts to do rcu based management of
* the child interrupts.
* This also allows us to use rcu in kstat_irqs_usr().
*/
call_rcu(&desc->rcu, delayed_free_desc);
}
static int alloc_descs(unsigned int start, unsigned int cnt, int node,
const struct irq_affinity_desc *affinity,
struct module *owner)
{
struct irq_desc *desc;
int i;
/* Validate affinity mask(s) */
if (affinity) {
for (i = 0; i < cnt; i++) {
if (cpumask_empty(&affinity[i].mask))
return -EINVAL;
}
}
for (i = 0; i < cnt; i++) {
const struct cpumask *mask = NULL;
unsigned int flags = 0;
if (affinity) {
if (affinity->is_managed) {
flags = IRQD_AFFINITY_MANAGED |
IRQD_MANAGED_SHUTDOWN;
}
mask = &affinity->mask;
node = cpu_to_node(cpumask_first(mask));
affinity++;
}
desc = alloc_desc(start + i, node, flags, mask, owner);
if (!desc)
goto err;
irq_insert_desc(start + i, desc);
irq_sysfs_add(start + i, desc);
irq_add_debugfs_entry(start + i, desc);
}
return start;
err:
for (i--; i >= 0; i--)
free_desc(start + i);
return -ENOMEM;
}
static int irq_expand_nr_irqs(unsigned int nr)
{
if (nr > MAX_SPARSE_IRQS)
return -ENOMEM;
nr_irqs = nr;
return 0;
}
int __init early_irq_init(void)
{
int i, initcnt, node = first_online_node;
struct irq_desc *desc;
init_irq_default_affinity();
/* Let arch update nr_irqs and return the nr of preallocated irqs */
initcnt = arch_probe_nr_irqs();
printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n",
NR_IRQS, nr_irqs, initcnt);
if (WARN_ON(nr_irqs > MAX_SPARSE_IRQS))
nr_irqs = MAX_SPARSE_IRQS;
if (WARN_ON(initcnt > MAX_SPARSE_IRQS))
initcnt = MAX_SPARSE_IRQS;
if (initcnt > nr_irqs)
nr_irqs = initcnt;
for (i = 0; i < initcnt; i++) {
desc = alloc_desc(i, node, 0, NULL, NULL);
irq_insert_desc(i, desc);
}
return arch_early_irq_init();
}
#else /* !CONFIG_SPARSE_IRQ */
struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
[0 ... NR_IRQS-1] = {
.handle_irq = handle_bad_irq,
.depth = 1,
.lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
}
};
int __init early_irq_init(void)
{
int count, i, node = first_online_node;
int ret;
init_irq_default_affinity();
printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS);
count = ARRAY_SIZE(irq_desc);
for (i = 0; i < count; i++) {
ret = init_desc(irq_desc + i, i, node, 0, NULL, NULL);
if (unlikely(ret))
goto __free_desc_res;
}
return arch_early_irq_init();
__free_desc_res:
while (--i >= 0) {
free_masks(irq_desc + i);
free_percpu(irq_desc[i].kstat_irqs);
}
return ret;
}
struct irq_desc *irq_to_desc(unsigned int irq)
{
return (irq < NR_IRQS) ? irq_desc + irq : NULL;
}
EXPORT_SYMBOL(irq_to_desc);
static void free_desc(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
unsigned long flags;
raw_spin_lock_irqsave(&desc->lock, flags);
desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);
raw_spin_unlock_irqrestore(&desc->lock, flags);
delete_irq_desc(irq);
}
static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
const struct irq_affinity_desc *affinity,
struct module *owner)
{
u32 i;
for (i = 0; i < cnt; i++) {
struct irq_desc *desc = irq_to_desc(start + i);
desc->owner = owner;
irq_insert_desc(start + i, desc);
}
return start;
}
static int irq_expand_nr_irqs(unsigned int nr)
{
return -ENOMEM;
}
void irq_mark_irq(unsigned int irq)
{
mutex_lock(&sparse_irq_lock);
irq_insert_desc(irq, irq_desc + irq);
mutex_unlock(&sparse_irq_lock);
}
#ifdef CONFIG_GENERIC_IRQ_LEGACY
void irq_init_desc(unsigned int irq)
{
free_desc(irq);
}
#endif
#endif /* !CONFIG_SPARSE_IRQ */
int handle_irq_desc(struct irq_desc *desc)
{
struct irq_data *data;
if (!desc)
return -EINVAL;
data = irq_desc_get_irq_data(desc);
if (WARN_ON_ONCE(!in_hardirq() && handle_enforce_irqctx(data)))
return -EPERM;
generic_handle_irq_desc(desc);
return 0;
}
/**
* generic_handle_irq - Invoke the handler for a particular irq
* @irq: The irq number to handle
*
* Returns: 0 on success, or -EINVAL if conversion has failed
*
* This function must be called from an IRQ context with irq regs
* initialized.
*/
int generic_handle_irq(unsigned int irq)
{
return handle_irq_desc(irq_to_desc(irq));
}
EXPORT_SYMBOL_GPL(generic_handle_irq);
/**
* generic_handle_irq_safe - Invoke the handler for a particular irq from any
* context.
* @irq: The irq number to handle
*
* Returns: 0 on success, a negative value on error.
*
* This function can be called from any context (IRQ or process context). It
* will report an error if not invoked from IRQ context and the irq has been
* marked to enforce IRQ-context only.
*/
int generic_handle_irq_safe(unsigned int irq)
{
unsigned long flags;
int ret;
local_irq_save(flags);
ret = handle_irq_desc(irq_to_desc(irq));
local_irq_restore(flags);
return ret;
}
EXPORT_SYMBOL_GPL(generic_handle_irq_safe);
#ifdef CONFIG_IRQ_DOMAIN
/**
* generic_handle_domain_irq - Invoke the handler for a HW irq belonging
* to a domain.
* @domain: The domain where to perform the lookup
* @hwirq: The HW irq number to convert to a logical one
*
* Returns: 0 on success, or -EINVAL if conversion has failed
*
* This function must be called from an IRQ context with irq regs
* initialized.
*/
int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq)
{
return handle_irq_desc(irq_resolve_mapping(domain, hwirq));
}
EXPORT_SYMBOL_GPL(generic_handle_domain_irq);
/**
* generic_handle_irq_safe - Invoke the handler for a HW irq belonging
* to a domain from any context.
* @domain: The domain where to perform the lookup
* @hwirq: The HW irq number to convert to a logical one
*
* Returns: 0 on success, a negative value on error.
*
* This function can be called from any context (IRQ or process
* context). If the interrupt is marked as 'enforce IRQ-context only' then
* the function must be invoked from hard interrupt context.
*/
int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq)
{
unsigned long flags;
int ret;
local_irq_save(flags);
ret = handle_irq_desc(irq_resolve_mapping(domain, hwirq));
local_irq_restore(flags);
return ret;
}
EXPORT_SYMBOL_GPL(generic_handle_domain_irq_safe);
/**
* generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging
* to a domain.
* @domain: The domain where to perform the lookup
* @hwirq: The HW irq number to convert to a logical one
*
* Returns: 0 on success, or -EINVAL if conversion has failed
*
* This function must be called from an NMI context with irq regs
* initialized.
**/
int generic_handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq)
{
WARN_ON_ONCE(!in_nmi());
return handle_irq_desc(irq_resolve_mapping(domain, hwirq));
}
#endif
/* Dynamic interrupt handling */
/**
* irq_free_descs - free irq descriptors
* @from: Start of descriptor range
* @cnt: Number of consecutive irqs to free
*/
void irq_free_descs(unsigned int from, unsigned int cnt)
{
int i;
if (from >= nr_irqs || (from + cnt) > nr_irqs)
return;
mutex_lock(&sparse_irq_lock);
for (i = 0; i < cnt; i++)
free_desc(from + i);
mutex_unlock(&sparse_irq_lock);
}
EXPORT_SYMBOL_GPL(irq_free_descs);
/**
* __irq_alloc_descs - allocate and initialize a range of irq descriptors
* @irq: Allocate for specific irq number if irq >= 0
* @from: Start the search from this irq number
* @cnt: Number of consecutive irqs to allocate.
* @node: Preferred node on which the irq descriptor should be allocated
* @owner: Owning module (can be NULL)
* @affinity: Optional pointer to an affinity mask array of size @cnt which
* hints where the irq descriptors should be allocated and which
* default affinities to use
*
* Returns the first irq number or error code
*/
int __ref
__irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
struct module *owner, const struct irq_affinity_desc *affinity)
{
int start, ret;
if (!cnt)
return -EINVAL;
if (irq >= 0) {
if (from > irq)
return -EINVAL;
from = irq;
} else {
/*
* For interrupts which are freely allocated the
* architecture can force a lower bound to the @from
* argument. x86 uses this to exclude the GSI space.
*/
from = arch_dynirq_lower_bound(from);
}
mutex_lock(&sparse_irq_lock);
start = irq_find_free_area(from, cnt);
ret = -EEXIST;
if (irq >=0 && start != irq)
goto unlock;
if (start + cnt > nr_irqs) {
ret = irq_expand_nr_irqs(start + cnt);
if (ret)
goto unlock;
}
ret = alloc_descs(start, cnt, node, affinity, owner);
unlock:
mutex_unlock(&sparse_irq_lock);
return ret;
}
EXPORT_SYMBOL_GPL(__irq_alloc_descs);
/**
* irq_get_next_irq - get next allocated irq number
* @offset: where to start the search
*
* Returns next irq number after offset or nr_irqs if none is found.
*/
unsigned int irq_get_next_irq(unsigned int offset)
{
return irq_find_at_or_after(offset);
}
struct irq_desc *
__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
unsigned int check)
{
struct irq_desc *desc = irq_to_desc(irq);
if (desc) {
if (check & _IRQ_DESC_CHECK) {
if ((check & _IRQ_DESC_PERCPU) &&
!irq_settings_is_per_cpu_devid(desc))
return NULL;
if (!(check & _IRQ_DESC_PERCPU) &&
irq_settings_is_per_cpu_devid(desc))
return NULL;
}
if (bus)
chip_bus_lock(desc);
raw_spin_lock_irqsave(&desc->lock, *flags);
}
return desc;
}
void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
__releases(&desc->lock)
{
raw_spin_unlock_irqrestore(&desc->lock, flags);
if (bus)
chip_bus_sync_unlock(desc);
}
int irq_set_percpu_devid_partition(unsigned int irq,
const struct cpumask *affinity)
{
struct irq_desc *desc = irq_to_desc(irq);
if (!desc)
return -EINVAL;
if (desc->percpu_enabled)
return -EINVAL;
desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
if (!desc->percpu_enabled)
return -ENOMEM;
if (affinity)
desc->percpu_affinity = affinity;
else
desc->percpu_affinity = cpu_possible_mask;
irq_set_percpu_devid_flags(irq);
return 0;
}
int irq_set_percpu_devid(unsigned int irq)
{
return irq_set_percpu_devid_partition(irq, NULL);
}
int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity)
{
struct irq_desc *desc = irq_to_desc(irq);
if (!desc || !desc->percpu_enabled)
return -EINVAL;
if (affinity)
cpumask_copy(affinity, desc->percpu_affinity);
return 0;
}
EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition);
void kstat_incr_irq_this_cpu(unsigned int irq)
{
kstat_incr_irqs_this_cpu(irq_to_desc(irq));
}
/**
* kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
* @irq: The interrupt number
* @cpu: The cpu number
*
* Returns the sum of interrupt counts on @cpu since boot for
* @irq. The caller must ensure that the interrupt is not removed
* concurrently.
*/
unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
{
struct irq_desc *desc = irq_to_desc(irq);
return desc && desc->kstat_irqs ?
*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
}
static bool irq_is_nmi(struct irq_desc *desc)
{
return desc->istate & IRQS_NMI;
}
static unsigned int kstat_irqs(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
unsigned int sum = 0;
int cpu;
if (!desc || !desc->kstat_irqs)
return 0;
if (!irq_settings_is_per_cpu_devid(desc) &&
!irq_settings_is_per_cpu(desc) &&
!irq_is_nmi(desc))
return data_race(desc->tot_count);
for_each_possible_cpu(cpu)
sum += data_race(*per_cpu_ptr(desc->kstat_irqs, cpu));
return sum;
}
/**
* kstat_irqs_usr - Get the statistics for an interrupt from thread context
* @irq: The interrupt number
*
* Returns the sum of interrupt counts on all cpus since boot for @irq.
*
* It uses rcu to protect the access since a concurrent removal of an
* interrupt descriptor is observing an rcu grace period before
* delayed_free_desc()/irq_kobj_release().
*/
unsigned int kstat_irqs_usr(unsigned int irq)
{
unsigned int sum;
rcu_read_lock();
sum = kstat_irqs(irq);
rcu_read_unlock();
return sum;
}
#ifdef CONFIG_LOCKDEP
void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class,
struct lock_class_key *request_class)
{
struct irq_desc *desc = irq_to_desc(irq);
if (desc) {
lockdep_set_class(&desc->lock, lock_class);
lockdep_set_class(&desc->request_mutex, request_class);
}
}
EXPORT_SYMBOL_GPL(__irq_set_lockdep_class);
#endif
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
* This file holds USB constants and structures that are needed for
* USB device APIs. These are used by the USB device model, which is
* defined in chapter 9 of the USB 2.0 specification and in the
* Wireless USB 1.0 spec (now defunct). Linux has several APIs in C that
* need these:
*
* - the master/host side Linux-USB kernel driver API;
* - the "usbfs" user space API; and
* - the Linux "gadget" slave/device/peripheral side driver API.
*
* USB 2.0 adds an additional "On The Go" (OTG) mode, which lets systems
* act either as a USB master/host or as a USB slave/device. That means
* the master and slave side APIs benefit from working well together.
*
* Note all descriptors are declared '__attribute__((packed))' so that:
*
* [a] they never get padded, either internally (USB spec writers
* probably handled that) or externally;
*
* [b] so that accessing bigger-than-a-bytes fields will never
* generate bus errors on any platform, even when the location of
* its descriptor inside a bundle isn't "naturally aligned", and
*
* [c] for consistency, removing all doubt even when it appears to
* someone that the two other points are non-issues for that
* particular descriptor type.
*/
#ifndef _UAPI__LINUX_USB_CH9_H
#define _UAPI__LINUX_USB_CH9_H
#include <linux/types.h> /* __u8 etc */
#include <asm/byteorder.h> /* le16_to_cpu */
/*-------------------------------------------------------------------------*/
/* CONTROL REQUEST SUPPORT */
/*
* USB directions
*
* This bit flag is used in endpoint descriptors' bEndpointAddress field.
* It's also one of three fields in control requests bRequestType.
*/
#define USB_DIR_OUT 0 /* to device */
#define USB_DIR_IN 0x80 /* to host */
/*
* USB types, the second of three bRequestType fields
*/
#define USB_TYPE_MASK (0x03 << 5)
#define USB_TYPE_STANDARD (0x00 << 5)
#define USB_TYPE_CLASS (0x01 << 5)
#define USB_TYPE_VENDOR (0x02 << 5)
#define USB_TYPE_RESERVED (0x03 << 5)
/*
* USB recipients, the third of three bRequestType fields
*/
#define USB_RECIP_MASK 0x1f
#define USB_RECIP_DEVICE 0x00
#define USB_RECIP_INTERFACE 0x01
#define USB_RECIP_ENDPOINT 0x02
#define USB_RECIP_OTHER 0x03
/* From Wireless USB 1.0 */
#define USB_RECIP_PORT 0x04
#define USB_RECIP_RPIPE 0x05
/*
* Standard requests, for the bRequest field of a SETUP packet.
*
* These are qualified by the bRequestType field, so that for example
* TYPE_CLASS or TYPE_VENDOR specific feature flags could be retrieved
* by a GET_STATUS request.
*/
#define USB_REQ_GET_STATUS 0x00
#define USB_REQ_CLEAR_FEATURE 0x01
#define USB_REQ_SET_FEATURE 0x03
#define USB_REQ_SET_ADDRESS 0x05
#define USB_REQ_GET_DESCRIPTOR 0x06
#define USB_REQ_SET_DESCRIPTOR 0x07
#define USB_REQ_GET_CONFIGURATION 0x08
#define USB_REQ_SET_CONFIGURATION 0x09
#define USB_REQ_GET_INTERFACE 0x0A
#define USB_REQ_SET_INTERFACE 0x0B
#define USB_REQ_SYNCH_FRAME 0x0C
#define USB_REQ_SET_SEL 0x30
#define USB_REQ_SET_ISOCH_DELAY 0x31
#define USB_REQ_SET_ENCRYPTION 0x0D /* Wireless USB */
#define USB_REQ_GET_ENCRYPTION 0x0E
#define USB_REQ_RPIPE_ABORT 0x0E
#define USB_REQ_SET_HANDSHAKE 0x0F
#define USB_REQ_RPIPE_RESET 0x0F
#define USB_REQ_GET_HANDSHAKE 0x10
#define USB_REQ_SET_CONNECTION 0x11
#define USB_REQ_SET_SECURITY_DATA 0x12
#define USB_REQ_GET_SECURITY_DATA 0x13
#define USB_REQ_SET_WUSB_DATA 0x14
#define USB_REQ_LOOPBACK_DATA_WRITE 0x15
#define USB_REQ_LOOPBACK_DATA_READ 0x16
#define USB_REQ_SET_INTERFACE_DS 0x17
/* specific requests for USB Power Delivery */
#define USB_REQ_GET_PARTNER_PDO 20
#define USB_REQ_GET_BATTERY_STATUS 21
#define USB_REQ_SET_PDO 22
#define USB_REQ_GET_VDM 23
#define USB_REQ_SEND_VDM 24
/* The Link Power Management (LPM) ECN defines USB_REQ_TEST_AND_SET command,
* used by hubs to put ports into a new L1 suspend state, except that it
* forgot to define its number ...
*/
/*
* USB feature flags are written using USB_REQ_{CLEAR,SET}_FEATURE, and
* are read as a bit array returned by USB_REQ_GET_STATUS. (So there
* are at most sixteen features of each type.) Hubs may also support a
* new USB_REQ_TEST_AND_SET_FEATURE to put ports into L1 suspend.
*/
#define USB_DEVICE_SELF_POWERED 0 /* (read only) */
#define USB_DEVICE_REMOTE_WAKEUP 1 /* dev may initiate wakeup */
#define USB_DEVICE_TEST_MODE 2 /* (wired high speed only) */
#define USB_DEVICE_BATTERY 2 /* (wireless) */
#define USB_DEVICE_B_HNP_ENABLE 3 /* (otg) dev may initiate HNP */
#define USB_DEVICE_WUSB_DEVICE 3 /* (wireless)*/
#define USB_DEVICE_A_HNP_SUPPORT 4 /* (otg) RH port supports HNP */
#define USB_DEVICE_A_ALT_HNP_SUPPORT 5 /* (otg) other RH port does */
#define USB_DEVICE_DEBUG_MODE 6 /* (special devices only) */
/*
* Test Mode Selectors
* See USB 2.0 spec Table 9-7
*/
#define USB_TEST_J 1
#define USB_TEST_K 2
#define USB_TEST_SE0_NAK 3
#define USB_TEST_PACKET 4
#define USB_TEST_FORCE_ENABLE 5
/* Status Type */
#define USB_STATUS_TYPE_STANDARD 0
#define USB_STATUS_TYPE_PTM 1
/*
* New Feature Selectors as added by USB 3.0
* See USB 3.0 spec Table 9-7
*/
#define USB_DEVICE_U1_ENABLE 48 /* dev may initiate U1 transition */
#define USB_DEVICE_U2_ENABLE 49 /* dev may initiate U2 transition */
#define USB_DEVICE_LTM_ENABLE 50 /* dev may send LTM */
#define USB_INTRF_FUNC_SUSPEND 0 /* function suspend */
#define USB_INTR_FUNC_SUSPEND_OPT_MASK 0xFF00
/*
* Suspend Options, Table 9-8 USB 3.0 spec
*/
#define USB_INTRF_FUNC_SUSPEND_LP (1 << (8 + 0))
#define USB_INTRF_FUNC_SUSPEND_RW (1 << (8 + 1))
/*
* Interface status, Figure 9-5 USB 3.0 spec
*/
#define USB_INTRF_STAT_FUNC_RW_CAP 1
#define USB_INTRF_STAT_FUNC_RW 2
#define USB_ENDPOINT_HALT 0 /* IN/OUT will STALL */
/* Bit array elements as returned by the USB_REQ_GET_STATUS request. */
#define USB_DEV_STAT_U1_ENABLED 2 /* transition into U1 state */
#define USB_DEV_STAT_U2_ENABLED 3 /* transition into U2 state */
#define USB_DEV_STAT_LTM_ENABLED 4 /* Latency tolerance messages */
/*
* Feature selectors from Table 9-8 USB Power Delivery spec
*/
#define USB_DEVICE_BATTERY_WAKE_MASK 40
#define USB_DEVICE_OS_IS_PD_AWARE 41
#define USB_DEVICE_POLICY_MODE 42
#define USB_PORT_PR_SWAP 43
#define USB_PORT_GOTO_MIN 44
#define USB_PORT_RETURN_POWER 45
#define USB_PORT_ACCEPT_PD_REQUEST 46
#define USB_PORT_REJECT_PD_REQUEST 47
#define USB_PORT_PORT_PD_RESET 48
#define USB_PORT_C_PORT_PD_CHANGE 49
#define USB_PORT_CABLE_PD_RESET 50
#define USB_DEVICE_CHARGING_POLICY 54
/**
* struct usb_ctrlrequest - SETUP data for a USB device control request
* @bRequestType: matches the USB bmRequestType field
* @bRequest: matches the USB bRequest field
* @wValue: matches the USB wValue field (le16 byte order)
* @wIndex: matches the USB wIndex field (le16 byte order)
* @wLength: matches the USB wLength field (le16 byte order)
*
* This structure is used to send control requests to a USB device. It matches
* the different fields of the USB 2.0 Spec section 9.3, table 9-2. See the
* USB spec for a fuller description of the different fields, and what they are
* used for.
*
* Note that the driver for any interface can issue control requests.
* For most devices, interfaces don't coordinate with each other, so
* such requests may be made at any time.
*/
struct usb_ctrlrequest {
__u8 bRequestType;
__u8 bRequest;
__le16 wValue;
__le16 wIndex;
__le16 wLength;
} __attribute__ ((packed));
/*-------------------------------------------------------------------------*/
/*
* STANDARD DESCRIPTORS ... as returned by GET_DESCRIPTOR, or
* (rarely) accepted by SET_DESCRIPTOR.
*
* Note that all multi-byte values here are encoded in little endian
* byte order "on the wire". Within the kernel and when exposed
* through the Linux-USB APIs, they are not converted to cpu byte
* order; it is the responsibility of the client code to do this.
* The single exception is when device and configuration descriptors (but
* not other descriptors) are read from character devices
* (i.e. /dev/bus/usb/BBB/DDD);
* in this case the fields are converted to host endianness by the kernel.
*/
/*
* Descriptor types ... USB 2.0 spec table 9.5
*/
#define USB_DT_DEVICE 0x01
#define USB_DT_CONFIG 0x02
#define USB_DT_STRING 0x03
#define USB_DT_INTERFACE 0x04
#define USB_DT_ENDPOINT 0x05
#define USB_DT_DEVICE_QUALIFIER 0x06
#define USB_DT_OTHER_SPEED_CONFIG 0x07
#define USB_DT_INTERFACE_POWER 0x08
/* these are from a minor usb 2.0 revision (ECN) */
#define USB_DT_OTG 0x09
#define USB_DT_DEBUG 0x0a
#define USB_DT_INTERFACE_ASSOCIATION 0x0b
/* these are from the Wireless USB spec */
#define USB_DT_SECURITY 0x0c
#define USB_DT_KEY 0x0d
#define USB_DT_ENCRYPTION_TYPE 0x0e
#define USB_DT_BOS 0x0f
#define USB_DT_DEVICE_CAPABILITY 0x10
#define USB_DT_WIRELESS_ENDPOINT_COMP 0x11
#define USB_DT_WIRE_ADAPTER 0x21
#define USB_DT_RPIPE 0x22
#define USB_DT_CS_RADIO_CONTROL 0x23
/* From the T10 UAS specification */
#define USB_DT_PIPE_USAGE 0x24
/* From the USB 3.0 spec */
#define USB_DT_SS_ENDPOINT_COMP 0x30
/* From the USB 3.1 spec */
#define USB_DT_SSP_ISOC_ENDPOINT_COMP 0x31
/* Conventional codes for class-specific descriptors. The convention is
* defined in the USB "Common Class" Spec (3.11). Individual class specs
* are authoritative for their usage, not the "common class" writeup.
*/
#define USB_DT_CS_DEVICE (USB_TYPE_CLASS | USB_DT_DEVICE)
#define USB_DT_CS_CONFIG (USB_TYPE_CLASS | USB_DT_CONFIG)
#define USB_DT_CS_STRING (USB_TYPE_CLASS | USB_DT_STRING)
#define USB_DT_CS_INTERFACE (USB_TYPE_CLASS | USB_DT_INTERFACE)
#define USB_DT_CS_ENDPOINT (USB_TYPE_CLASS | USB_DT_ENDPOINT)
/* All standard descriptors have these 2 fields at the beginning */
struct usb_descriptor_header {
__u8 bLength;
__u8 bDescriptorType;
} __attribute__ ((packed));
/*-------------------------------------------------------------------------*/
/* USB_DT_DEVICE: Device descriptor */
struct usb_device_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__le16 bcdUSB;
__u8 bDeviceClass;
__u8 bDeviceSubClass;
__u8 bDeviceProtocol;
__u8 bMaxPacketSize0;
__le16 idVendor;
__le16 idProduct;
__le16 bcdDevice;
__u8 iManufacturer;
__u8 iProduct;
__u8 iSerialNumber;
__u8 bNumConfigurations;
} __attribute__ ((packed));
#define USB_DT_DEVICE_SIZE 18
/*
* Device and/or Interface Class codes
* as found in bDeviceClass or bInterfaceClass
* and defined by www.usb.org documents
*/
#define USB_CLASS_PER_INTERFACE 0 /* for DeviceClass */
#define USB_CLASS_AUDIO 1
#define USB_CLASS_COMM 2
#define USB_CLASS_HID 3
#define USB_CLASS_PHYSICAL 5
#define USB_CLASS_STILL_IMAGE 6
#define USB_CLASS_PRINTER 7
#define USB_CLASS_MASS_STORAGE 8
#define USB_CLASS_HUB 9
#define USB_CLASS_CDC_DATA 0x0a
#define USB_CLASS_CSCID 0x0b /* chip+ smart card */
#define USB_CLASS_CONTENT_SEC 0x0d /* content security */
#define USB_CLASS_VIDEO 0x0e
#define USB_CLASS_WIRELESS_CONTROLLER 0xe0
#define USB_CLASS_PERSONAL_HEALTHCARE 0x0f
#define USB_CLASS_AUDIO_VIDEO 0x10
#define USB_CLASS_BILLBOARD 0x11
#define USB_CLASS_USB_TYPE_C_BRIDGE 0x12
#define USB_CLASS_MISC 0xef
#define USB_CLASS_APP_SPEC 0xfe
#define USB_CLASS_VENDOR_SPEC 0xff
#define USB_SUBCLASS_VENDOR_SPEC 0xff
/*-------------------------------------------------------------------------*/
/* USB_DT_CONFIG: Configuration descriptor information.
*
* USB_DT_OTHER_SPEED_CONFIG is the same descriptor, except that the
* descriptor type is different. Highspeed-capable devices can look
* different depending on what speed they're currently running. Only
* devices with a USB_DT_DEVICE_QUALIFIER have any OTHER_SPEED_CONFIG
* descriptors.
*/
struct usb_config_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__le16 wTotalLength;
__u8 bNumInterfaces;
__u8 bConfigurationValue;
__u8 iConfiguration;
__u8 bmAttributes;
__u8 bMaxPower;
} __attribute__ ((packed));
#define USB_DT_CONFIG_SIZE 9
/* from config descriptor bmAttributes */
#define USB_CONFIG_ATT_ONE (1 << 7) /* must be set */
#define USB_CONFIG_ATT_SELFPOWER (1 << 6) /* self powered */
#define USB_CONFIG_ATT_WAKEUP (1 << 5) /* can wakeup */
#define USB_CONFIG_ATT_BATTERY (1 << 4) /* battery powered */
/*-------------------------------------------------------------------------*/
/* USB String descriptors can contain at most 126 characters. */
#define USB_MAX_STRING_LEN 126
/* USB_DT_STRING: String descriptor */
struct usb_string_descriptor {
__u8 bLength;
__u8 bDescriptorType;
union {
__le16 legacy_padding;
__DECLARE_FLEX_ARRAY(__le16, wData); /* UTF-16LE encoded */
};
} __attribute__ ((packed));
/* note that "string" zero is special, it holds language codes that
* the device supports, not Unicode characters.
*/
/*-------------------------------------------------------------------------*/
/* USB_DT_INTERFACE: Interface descriptor */
struct usb_interface_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bInterfaceNumber;
__u8 bAlternateSetting;
__u8 bNumEndpoints;
__u8 bInterfaceClass;
__u8 bInterfaceSubClass;
__u8 bInterfaceProtocol;
__u8 iInterface;
} __attribute__ ((packed));
#define USB_DT_INTERFACE_SIZE 9
/*-------------------------------------------------------------------------*/
/* USB_DT_ENDPOINT: Endpoint descriptor */
struct usb_endpoint_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bEndpointAddress;
__u8 bmAttributes;
__le16 wMaxPacketSize;
__u8 bInterval;
/* NOTE: these two are _only_ in audio endpoints. */
/* use USB_DT_ENDPOINT*_SIZE in bLength, not sizeof. */
__u8 bRefresh;
__u8 bSynchAddress;
} __attribute__ ((packed));
#define USB_DT_ENDPOINT_SIZE 7
#define USB_DT_ENDPOINT_AUDIO_SIZE 9 /* Audio extension */
/*
* Endpoints
*/
#define USB_ENDPOINT_NUMBER_MASK 0x0f /* in bEndpointAddress */
#define USB_ENDPOINT_DIR_MASK 0x80
#define USB_ENDPOINT_XFERTYPE_MASK 0x03 /* in bmAttributes */
#define USB_ENDPOINT_XFER_CONTROL 0
#define USB_ENDPOINT_XFER_ISOC 1
#define USB_ENDPOINT_XFER_BULK 2
#define USB_ENDPOINT_XFER_INT 3
#define USB_ENDPOINT_MAX_ADJUSTABLE 0x80
#define USB_ENDPOINT_MAXP_MASK 0x07ff
#define USB_EP_MAXP_MULT_SHIFT 11
#define USB_EP_MAXP_MULT_MASK (3 << USB_EP_MAXP_MULT_SHIFT)
#define USB_EP_MAXP_MULT(m) \
(((m) & USB_EP_MAXP_MULT_MASK) >> USB_EP_MAXP_MULT_SHIFT)
/* The USB 3.0 spec redefines bits 5:4 of bmAttributes as interrupt ep type. */
#define USB_ENDPOINT_INTRTYPE 0x30
#define USB_ENDPOINT_INTR_PERIODIC (0 << 4)
#define USB_ENDPOINT_INTR_NOTIFICATION (1 << 4)
#define USB_ENDPOINT_SYNCTYPE 0x0c
#define USB_ENDPOINT_SYNC_NONE (0 << 2)
#define USB_ENDPOINT_SYNC_ASYNC (1 << 2)
#define USB_ENDPOINT_SYNC_ADAPTIVE (2 << 2)
#define USB_ENDPOINT_SYNC_SYNC (3 << 2)
#define USB_ENDPOINT_USAGE_MASK 0x30
#define USB_ENDPOINT_USAGE_DATA 0x00
#define USB_ENDPOINT_USAGE_FEEDBACK 0x10
#define USB_ENDPOINT_USAGE_IMPLICIT_FB 0x20 /* Implicit feedback Data endpoint */
/*-------------------------------------------------------------------------*/
/**
* usb_endpoint_num - get the endpoint's number
* @epd: endpoint to be checked
*
* Returns @epd's number: 0 to 15.
*/
static inline int usb_endpoint_num(const struct usb_endpoint_descriptor *epd)
{
return epd->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK;
}
/**
* usb_endpoint_type - get the endpoint's transfer type
* @epd: endpoint to be checked
*
* Returns one of USB_ENDPOINT_XFER_{CONTROL, ISOC, BULK, INT} according
* to @epd's transfer type.
*/
static inline int usb_endpoint_type(const struct usb_endpoint_descriptor *epd)
{
return epd->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
}
/**
* usb_endpoint_dir_in - check if the endpoint has IN direction
* @epd: endpoint to be checked
*
* Returns true if the endpoint is of type IN, otherwise it returns false.
*/
static inline int usb_endpoint_dir_in(const struct usb_endpoint_descriptor *epd)
{
return ((epd->bEndpointAddress & USB_ENDPOINT_DIR_MASK) == USB_DIR_IN);
}
/**
* usb_endpoint_dir_out - check if the endpoint has OUT direction
* @epd: endpoint to be checked
*
* Returns true if the endpoint is of type OUT, otherwise it returns false.
*/
static inline int usb_endpoint_dir_out(
const struct usb_endpoint_descriptor *epd)
{
return ((epd->bEndpointAddress & USB_ENDPOINT_DIR_MASK) == USB_DIR_OUT);
}
/**
* usb_endpoint_xfer_bulk - check if the endpoint has bulk transfer type
* @epd: endpoint to be checked
*
* Returns true if the endpoint is of type bulk, otherwise it returns false.
*/
static inline int usb_endpoint_xfer_bulk(
const struct usb_endpoint_descriptor *epd)
{
return ((epd->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
USB_ENDPOINT_XFER_BULK);
}
/**
* usb_endpoint_xfer_control - check if the endpoint has control transfer type
* @epd: endpoint to be checked
*
* Returns true if the endpoint is of type control, otherwise it returns false.
*/
static inline int usb_endpoint_xfer_control(
const struct usb_endpoint_descriptor *epd)
{
return ((epd->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
USB_ENDPOINT_XFER_CONTROL);
}
/**
* usb_endpoint_xfer_int - check if the endpoint has interrupt transfer type
* @epd: endpoint to be checked
*
* Returns true if the endpoint is of type interrupt, otherwise it returns
* false.
*/
static inline int usb_endpoint_xfer_int(
const struct usb_endpoint_descriptor *epd)
{
return ((epd->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
USB_ENDPOINT_XFER_INT);
}
/**
* usb_endpoint_xfer_isoc - check if the endpoint has isochronous transfer type
* @epd: endpoint to be checked
*
* Returns true if the endpoint is of type isochronous, otherwise it returns
* false.
*/
static inline int usb_endpoint_xfer_isoc(
const struct usb_endpoint_descriptor *epd)
{
return ((epd->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
USB_ENDPOINT_XFER_ISOC);
}
/**
* usb_endpoint_is_bulk_in - check if the endpoint is bulk IN
* @epd: endpoint to be checked
*
* Returns true if the endpoint has bulk transfer type and IN direction,
* otherwise it returns false.
*/
static inline int usb_endpoint_is_bulk_in(
const struct usb_endpoint_descriptor *epd)
{
return usb_endpoint_xfer_bulk(epd) && usb_endpoint_dir_in(epd);
}
/**
* usb_endpoint_is_bulk_out - check if the endpoint is bulk OUT
* @epd: endpoint to be checked
*
* Returns true if the endpoint has bulk transfer type and OUT direction,
* otherwise it returns false.
*/
static inline int usb_endpoint_is_bulk_out(
const struct usb_endpoint_descriptor *epd)
{
return usb_endpoint_xfer_bulk(epd) && usb_endpoint_dir_out(epd);
}
/**
* usb_endpoint_is_int_in - check if the endpoint is interrupt IN
* @epd: endpoint to be checked
*
* Returns true if the endpoint has interrupt transfer type and IN direction,
* otherwise it returns false.
*/
static inline int usb_endpoint_is_int_in(
const struct usb_endpoint_descriptor *epd)
{
return usb_endpoint_xfer_int(epd) && usb_endpoint_dir_in(epd);
}
/**
* usb_endpoint_is_int_out - check if the endpoint is interrupt OUT
* @epd: endpoint to be checked
*
* Returns true if the endpoint has interrupt transfer type and OUT direction,
* otherwise it returns false.
*/
static inline int usb_endpoint_is_int_out(
const struct usb_endpoint_descriptor *epd)
{
return usb_endpoint_xfer_int(epd) && usb_endpoint_dir_out(epd);
}
/**
* usb_endpoint_is_isoc_in - check if the endpoint is isochronous IN
* @epd: endpoint to be checked
*
* Returns true if the endpoint has isochronous transfer type and IN direction,
* otherwise it returns false.
*/
static inline int usb_endpoint_is_isoc_in(
const struct usb_endpoint_descriptor *epd)
{
return usb_endpoint_xfer_isoc(epd) && usb_endpoint_dir_in(epd);
}
/**
* usb_endpoint_is_isoc_out - check if the endpoint is isochronous OUT
* @epd: endpoint to be checked
*
* Returns true if the endpoint has isochronous transfer type and OUT direction,
* otherwise it returns false.
*/
static inline int usb_endpoint_is_isoc_out(
const struct usb_endpoint_descriptor *epd)
{
return usb_endpoint_xfer_isoc(epd) && usb_endpoint_dir_out(epd);
}
/**
* usb_endpoint_maxp - get endpoint's max packet size
* @epd: endpoint to be checked
*
* Returns @epd's max packet bits [10:0]
*/
static inline int usb_endpoint_maxp(const struct usb_endpoint_descriptor *epd)
{
return __le16_to_cpu(epd->wMaxPacketSize) & USB_ENDPOINT_MAXP_MASK;
}
/**
* usb_endpoint_maxp_mult - get endpoint's transactional opportunities
* @epd: endpoint to be checked
*
* Return @epd's wMaxPacketSize[12:11] + 1
*/
static inline int
usb_endpoint_maxp_mult(const struct usb_endpoint_descriptor *epd)
{
int maxp = __le16_to_cpu(epd->wMaxPacketSize); return USB_EP_MAXP_MULT(maxp) + 1;
}
static inline int usb_endpoint_interrupt_type(
const struct usb_endpoint_descriptor *epd)
{
return epd->bmAttributes & USB_ENDPOINT_INTRTYPE;
}
/*-------------------------------------------------------------------------*/
/* USB_DT_SSP_ISOC_ENDPOINT_COMP: SuperSpeedPlus Isochronous Endpoint Companion
* descriptor
*/
struct usb_ssp_isoc_ep_comp_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__le16 wReseved;
__le32 dwBytesPerInterval;
} __attribute__ ((packed));
#define USB_DT_SSP_ISOC_EP_COMP_SIZE 8
/*-------------------------------------------------------------------------*/
/* USB_DT_SS_ENDPOINT_COMP: SuperSpeed Endpoint Companion descriptor */
struct usb_ss_ep_comp_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bMaxBurst;
__u8 bmAttributes;
__le16 wBytesPerInterval;
} __attribute__ ((packed));
#define USB_DT_SS_EP_COMP_SIZE 6
/* Bits 4:0 of bmAttributes if this is a bulk endpoint */
static inline int
usb_ss_max_streams(const struct usb_ss_ep_comp_descriptor *comp)
{
int max_streams;
if (!comp)
return 0;
max_streams = comp->bmAttributes & 0x1f;
if (!max_streams)
return 0;
max_streams = 1 << max_streams;
return max_streams;
}
/* Bits 1:0 of bmAttributes if this is an isoc endpoint */
#define USB_SS_MULT(p) (1 + ((p) & 0x3))
/* Bit 7 of bmAttributes if a SSP isoc endpoint companion descriptor exists */
#define USB_SS_SSP_ISOC_COMP(p) ((p) & (1 << 7))
/*-------------------------------------------------------------------------*/
/* USB_DT_DEVICE_QUALIFIER: Device Qualifier descriptor */
struct usb_qualifier_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__le16 bcdUSB;
__u8 bDeviceClass;
__u8 bDeviceSubClass;
__u8 bDeviceProtocol;
__u8 bMaxPacketSize0;
__u8 bNumConfigurations;
__u8 bRESERVED;
} __attribute__ ((packed));
/*-------------------------------------------------------------------------*/
/* USB_DT_OTG (from OTG 1.0a supplement) */
struct usb_otg_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bmAttributes; /* support for HNP, SRP, etc */
} __attribute__ ((packed));
/* USB_DT_OTG (from OTG 2.0 supplement) */
struct usb_otg20_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bmAttributes; /* support for HNP, SRP and ADP, etc */
__le16 bcdOTG; /* OTG and EH supplement release number
* in binary-coded decimal(i.e. 2.0 is 0200H)
*/
} __attribute__ ((packed));
/* from usb_otg_descriptor.bmAttributes */
#define USB_OTG_SRP (1 << 0)
#define USB_OTG_HNP (1 << 1) /* swap host/device roles */
#define USB_OTG_ADP (1 << 2) /* support ADP */
/* OTG 3.0 */
#define USB_OTG_RSP (1 << 3) /* support RSP */
#define OTG_STS_SELECTOR 0xF000 /* OTG status selector */
/*-------------------------------------------------------------------------*/
/* USB_DT_DEBUG: for special highspeed devices, replacing serial console */
struct usb_debug_descriptor {
__u8 bLength;
__u8 bDescriptorType;
/* bulk endpoints with 8 byte maxpacket */
__u8 bDebugInEndpoint;
__u8 bDebugOutEndpoint;
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB_DT_INTERFACE_ASSOCIATION: groups interfaces */
struct usb_interface_assoc_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bFirstInterface;
__u8 bInterfaceCount;
__u8 bFunctionClass;
__u8 bFunctionSubClass;
__u8 bFunctionProtocol;
__u8 iFunction;
} __attribute__ ((packed));
#define USB_DT_INTERFACE_ASSOCIATION_SIZE 8
/*-------------------------------------------------------------------------*/
/* USB_DT_SECURITY: group of wireless security descriptors, including
* encryption types available for setting up a CC/association.
*/
struct usb_security_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__le16 wTotalLength;
__u8 bNumEncryptionTypes;
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB_DT_KEY: used with {GET,SET}_SECURITY_DATA; only public keys
* may be retrieved.
*/
struct usb_key_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 tTKID[3];
__u8 bReserved;
__u8 bKeyData[];
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB_DT_ENCRYPTION_TYPE: bundled in DT_SECURITY groups */
struct usb_encryption_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bEncryptionType;
#define USB_ENC_TYPE_UNSECURE 0
#define USB_ENC_TYPE_WIRED 1 /* non-wireless mode */
#define USB_ENC_TYPE_CCM_1 2 /* aes128/cbc session */
#define USB_ENC_TYPE_RSA_1 3 /* rsa3072/sha1 auth */
__u8 bEncryptionValue; /* use in SET_ENCRYPTION */
__u8 bAuthKeyIndex;
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB_DT_BOS: group of device-level capabilities */
struct usb_bos_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__le16 wTotalLength;
__u8 bNumDeviceCaps;
} __attribute__((packed));
#define USB_DT_BOS_SIZE 5
/*-------------------------------------------------------------------------*/
/* USB_DT_DEVICE_CAPABILITY: grouped with BOS */
struct usb_dev_cap_header {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
} __attribute__((packed));
#define USB_CAP_TYPE_WIRELESS_USB 1
struct usb_wireless_cap_descriptor { /* Ultra Wide Band */
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bmAttributes;
#define USB_WIRELESS_P2P_DRD (1 << 1)
#define USB_WIRELESS_BEACON_MASK (3 << 2)
#define USB_WIRELESS_BEACON_SELF (1 << 2)
#define USB_WIRELESS_BEACON_DIRECTED (2 << 2)
#define USB_WIRELESS_BEACON_NONE (3 << 2)
__le16 wPHYRates; /* bit rates, Mbps */
#define USB_WIRELESS_PHY_53 (1 << 0) /* always set */
#define USB_WIRELESS_PHY_80 (1 << 1)
#define USB_WIRELESS_PHY_107 (1 << 2) /* always set */
#define USB_WIRELESS_PHY_160 (1 << 3)
#define USB_WIRELESS_PHY_200 (1 << 4) /* always set */
#define USB_WIRELESS_PHY_320 (1 << 5)
#define USB_WIRELESS_PHY_400 (1 << 6)
#define USB_WIRELESS_PHY_480 (1 << 7)
__u8 bmTFITXPowerInfo; /* TFI power levels */
__u8 bmFFITXPowerInfo; /* FFI power levels */
__le16 bmBandGroup;
__u8 bReserved;
} __attribute__((packed));
#define USB_DT_USB_WIRELESS_CAP_SIZE 11
/* USB 2.0 Extension descriptor */
#define USB_CAP_TYPE_EXT 2
struct usb_ext_cap_descriptor { /* Link Power Management */
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__le32 bmAttributes;
#define USB_LPM_SUPPORT (1 << 1) /* supports LPM */
#define USB_BESL_SUPPORT (1 << 2) /* supports BESL */
#define USB_BESL_BASELINE_VALID (1 << 3) /* Baseline BESL valid*/
#define USB_BESL_DEEP_VALID (1 << 4) /* Deep BESL valid */
#define USB_SET_BESL_BASELINE(p) (((p) & 0xf) << 8)
#define USB_SET_BESL_DEEP(p) (((p) & 0xf) << 12)
#define USB_GET_BESL_BASELINE(p) (((p) & (0xf << 8)) >> 8)
#define USB_GET_BESL_DEEP(p) (((p) & (0xf << 12)) >> 12)
} __attribute__((packed));
#define USB_DT_USB_EXT_CAP_SIZE 7
/*
* SuperSpeed USB Capability descriptor: Defines the set of SuperSpeed USB
* specific device level capabilities
*/
#define USB_SS_CAP_TYPE 3
struct usb_ss_cap_descriptor { /* Link Power Management */
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bmAttributes;
#define USB_LTM_SUPPORT (1 << 1) /* supports LTM */
__le16 wSpeedSupported;
#define USB_LOW_SPEED_OPERATION (1) /* Low speed operation */
#define USB_FULL_SPEED_OPERATION (1 << 1) /* Full speed operation */
#define USB_HIGH_SPEED_OPERATION (1 << 2) /* High speed operation */
#define USB_5GBPS_OPERATION (1 << 3) /* Operation at 5Gbps */
__u8 bFunctionalitySupport;
__u8 bU1devExitLat;
__le16 bU2DevExitLat;
} __attribute__((packed));
#define USB_DT_USB_SS_CAP_SIZE 10
/*
* Container ID Capability descriptor: Defines the instance unique ID used to
* identify the instance across all operating modes
*/
#define CONTAINER_ID_TYPE 4
struct usb_ss_container_id_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bReserved;
__u8 ContainerID[16]; /* 128-bit number */
} __attribute__((packed));
#define USB_DT_USB_SS_CONTN_ID_SIZE 20
/*
* Platform Device Capability descriptor: Defines platform specific device
* capabilities
*/
#define USB_PLAT_DEV_CAP_TYPE 5
struct usb_plat_dev_cap_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bReserved;
__u8 UUID[16];
__u8 CapabilityData[];
} __attribute__((packed));
#define USB_DT_USB_PLAT_DEV_CAP_SIZE(capability_data_size) (20 + capability_data_size)
/*
* SuperSpeed Plus USB Capability descriptor: Defines the set of
* SuperSpeed Plus USB specific device level capabilities
*/
#define USB_SSP_CAP_TYPE 0xa
struct usb_ssp_cap_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bReserved;
__le32 bmAttributes;
#define USB_SSP_SUBLINK_SPEED_ATTRIBS (0x1f << 0) /* sublink speed entries */
#define USB_SSP_SUBLINK_SPEED_IDS (0xf << 5) /* speed ID entries */
__le16 wFunctionalitySupport;
#define USB_SSP_MIN_SUBLINK_SPEED_ATTRIBUTE_ID (0xf)
#define USB_SSP_MIN_RX_LANE_COUNT (0xf << 8)
#define USB_SSP_MIN_TX_LANE_COUNT (0xf << 12)
__le16 wReserved;
union {
__le32 legacy_padding;
/* list of sublink speed attrib entries */
__DECLARE_FLEX_ARRAY(__le32, bmSublinkSpeedAttr);
};
#define USB_SSP_SUBLINK_SPEED_SSID (0xf) /* sublink speed ID */
#define USB_SSP_SUBLINK_SPEED_LSE (0x3 << 4) /* Lanespeed exponent */
#define USB_SSP_SUBLINK_SPEED_LSE_BPS 0
#define USB_SSP_SUBLINK_SPEED_LSE_KBPS 1
#define USB_SSP_SUBLINK_SPEED_LSE_MBPS 2
#define USB_SSP_SUBLINK_SPEED_LSE_GBPS 3
#define USB_SSP_SUBLINK_SPEED_ST (0x3 << 6) /* Sublink type */
#define USB_SSP_SUBLINK_SPEED_ST_SYM_RX 0
#define USB_SSP_SUBLINK_SPEED_ST_ASYM_RX 1
#define USB_SSP_SUBLINK_SPEED_ST_SYM_TX 2
#define USB_SSP_SUBLINK_SPEED_ST_ASYM_TX 3
#define USB_SSP_SUBLINK_SPEED_RSVD (0x3f << 8) /* Reserved */
#define USB_SSP_SUBLINK_SPEED_LP (0x3 << 14) /* Link protocol */
#define USB_SSP_SUBLINK_SPEED_LP_SS 0
#define USB_SSP_SUBLINK_SPEED_LP_SSP 1
#define USB_SSP_SUBLINK_SPEED_LSM (0xff << 16) /* Lanespeed mantissa */
} __attribute__((packed));
/*
* USB Power Delivery Capability Descriptor:
* Defines capabilities for PD
*/
/* Defines the various PD Capabilities of this device */
#define USB_PD_POWER_DELIVERY_CAPABILITY 0x06
/* Provides information on each battery supported by the device */
#define USB_PD_BATTERY_INFO_CAPABILITY 0x07
/* The Consumer characteristics of a Port on the device */
#define USB_PD_PD_CONSUMER_PORT_CAPABILITY 0x08
/* The provider characteristics of a Port on the device */
#define USB_PD_PD_PROVIDER_PORT_CAPABILITY 0x09
struct usb_pd_cap_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType; /* set to USB_PD_POWER_DELIVERY_CAPABILITY */
__u8 bReserved;
__le32 bmAttributes;
#define USB_PD_CAP_BATTERY_CHARGING (1 << 1) /* supports Battery Charging specification */
#define USB_PD_CAP_USB_PD (1 << 2) /* supports USB Power Delivery specification */
#define USB_PD_CAP_PROVIDER (1 << 3) /* can provide power */
#define USB_PD_CAP_CONSUMER (1 << 4) /* can consume power */
#define USB_PD_CAP_CHARGING_POLICY (1 << 5) /* supports CHARGING_POLICY feature */
#define USB_PD_CAP_TYPE_C_CURRENT (1 << 6) /* supports power capabilities defined in the USB Type-C Specification */
#define USB_PD_CAP_PWR_AC (1 << 8)
#define USB_PD_CAP_PWR_BAT (1 << 9)
#define USB_PD_CAP_PWR_USE_V_BUS (1 << 14)
__le16 bmProviderPorts; /* Bit zero refers to the UFP of the device */
__le16 bmConsumerPorts;
__le16 bcdBCVersion;
__le16 bcdPDVersion;
__le16 bcdUSBTypeCVersion;
} __attribute__((packed));
struct usb_pd_cap_battery_info_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
/* Index of string descriptor shall contain the user friendly name for this battery */
__u8 iBattery;
/* Index of string descriptor shall contain the Serial Number String for this battery */
__u8 iSerial;
__u8 iManufacturer;
__u8 bBatteryId; /* uniquely identifies this battery in status Messages */
__u8 bReserved;
/*
* Shall contain the Battery Charge value above which this
* battery is considered to be fully charged but not necessarily
* “topped off.”
*/
__le32 dwChargedThreshold; /* in mWh */
/*
* Shall contain the minimum charge level of this battery such
* that above this threshold, a device can be assured of being
* able to power up successfully (see Battery Charging 1.2).
*/
__le32 dwWeakThreshold; /* in mWh */
__le32 dwBatteryDesignCapacity; /* in mWh */
__le32 dwBatteryLastFullchargeCapacity; /* in mWh */
} __attribute__((packed));
struct usb_pd_cap_consumer_port_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bReserved;
__u8 bmCapabilities;
/* port will oerate under: */
#define USB_PD_CAP_CONSUMER_BC (1 << 0) /* BC */
#define USB_PD_CAP_CONSUMER_PD (1 << 1) /* PD */
#define USB_PD_CAP_CONSUMER_TYPE_C (1 << 2) /* USB Type-C Current */
__le16 wMinVoltage; /* in 50mV units */
__le16 wMaxVoltage; /* in 50mV units */
__u16 wReserved;
__le32 dwMaxOperatingPower; /* in 10 mW - operating at steady state */
__le32 dwMaxPeakPower; /* in 10mW units - operating at peak power */
__le32 dwMaxPeakPowerTime; /* in 100ms units - duration of peak */
#define USB_PD_CAP_CONSUMER_UNKNOWN_PEAK_POWER_TIME 0xffff
} __attribute__((packed));
struct usb_pd_cap_provider_port_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
__u8 bReserved1;
__u8 bmCapabilities;
/* port will oerate under: */
#define USB_PD_CAP_PROVIDER_BC (1 << 0) /* BC */
#define USB_PD_CAP_PROVIDER_PD (1 << 1) /* PD */
#define USB_PD_CAP_PROVIDER_TYPE_C (1 << 2) /* USB Type-C Current */
__u8 bNumOfPDObjects;
__u8 bReserved2;
__le32 wPowerDataObject[];
} __attribute__((packed));
/*
* Precision time measurement capability descriptor: advertised by devices and
* hubs that support PTM
*/
#define USB_PTM_CAP_TYPE 0xb
struct usb_ptm_cap_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bDevCapabilityType;
} __attribute__((packed));
#define USB_DT_USB_PTM_ID_SIZE 3
/*
* The size of the descriptor for the Sublink Speed Attribute Count
* (SSAC) specified in bmAttributes[4:0]. SSAC is zero-based
*/
#define USB_DT_USB_SSP_CAP_SIZE(ssac) (12 + (ssac + 1) * 4)
/*-------------------------------------------------------------------------*/
/* USB_DT_WIRELESS_ENDPOINT_COMP: companion descriptor associated with
* each endpoint descriptor for a wireless device
*/
struct usb_wireless_ep_comp_descriptor {
__u8 bLength;
__u8 bDescriptorType;
__u8 bMaxBurst;
__u8 bMaxSequence;
__le16 wMaxStreamDelay;
__le16 wOverTheAirPacketSize;
__u8 bOverTheAirInterval;
__u8 bmCompAttributes;
#define USB_ENDPOINT_SWITCH_MASK 0x03 /* in bmCompAttributes */
#define USB_ENDPOINT_SWITCH_NO 0
#define USB_ENDPOINT_SWITCH_SWITCH 1
#define USB_ENDPOINT_SWITCH_SCALE 2
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB_REQ_SET_HANDSHAKE is a four-way handshake used between a wireless
* host and a device for connection set up, mutual authentication, and
* exchanging short lived session keys. The handshake depends on a CC.
*/
struct usb_handshake {
__u8 bMessageNumber;
__u8 bStatus;
__u8 tTKID[3];
__u8 bReserved;
__u8 CDID[16];
__u8 nonce[16];
__u8 MIC[8];
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB_REQ_SET_CONNECTION modifies or revokes a connection context (CC).
* A CC may also be set up using non-wireless secure channels (including
* wired USB!), and some devices may support CCs with multiple hosts.
*/
struct usb_connection_context {
__u8 CHID[16]; /* persistent host id */
__u8 CDID[16]; /* device id (unique w/in host context) */
__u8 CK[16]; /* connection key */
} __attribute__((packed));
/*-------------------------------------------------------------------------*/
/* USB 2.0 defines three speeds, here's how Linux identifies them */
enum usb_device_speed {
USB_SPEED_UNKNOWN = 0, /* enumerating */
USB_SPEED_LOW, USB_SPEED_FULL, /* usb 1.1 */
USB_SPEED_HIGH, /* usb 2.0 */
USB_SPEED_WIRELESS, /* wireless (usb 2.5) */
USB_SPEED_SUPER, /* usb 3.0 */
USB_SPEED_SUPER_PLUS, /* usb 3.1 */
};
enum usb_device_state {
/* NOTATTACHED isn't in the USB spec, and this state acts
* the same as ATTACHED ... but it's clearer this way.
*/
USB_STATE_NOTATTACHED = 0,
/* chapter 9 and authentication (wireless) device states */
USB_STATE_ATTACHED,
USB_STATE_POWERED, /* wired */
USB_STATE_RECONNECTING, /* auth */
USB_STATE_UNAUTHENTICATED, /* auth */
USB_STATE_DEFAULT, /* limited function */
USB_STATE_ADDRESS,
USB_STATE_CONFIGURED, /* most functions */
USB_STATE_SUSPENDED
/* NOTE: there are actually four different SUSPENDED
* states, returning to POWERED, DEFAULT, ADDRESS, or
* CONFIGURED respectively when SOF tokens flow again.
* At this level there's no difference between L1 and L2
* suspend states. (L2 being original USB 1.1 suspend.)
*/
};
enum usb3_link_state {
USB3_LPM_U0 = 0,
USB3_LPM_U1,
USB3_LPM_U2,
USB3_LPM_U3
};
/*
* A U1 timeout of 0x0 means the parent hub will reject any transitions to U1.
* 0xff means the parent hub will accept transitions to U1, but will not
* initiate a transition.
*
* A U1 timeout of 0x1 to 0x7F also causes the hub to initiate a transition to
* U1 after that many microseconds. Timeouts of 0x80 to 0xFE are reserved
* values.
*
* A U2 timeout of 0x0 means the parent hub will reject any transitions to U2.
* 0xff means the parent hub will accept transitions to U2, but will not
* initiate a transition.
*
* A U2 timeout of 0x1 to 0xFE also causes the hub to initiate a transition to
* U2 after N*256 microseconds. Therefore a U2 timeout value of 0x1 means a U2
* idle timer of 256 microseconds, 0x2 means 512 microseconds, 0xFE means
* 65.024ms.
*/
#define USB3_LPM_DISABLED 0x0
#define USB3_LPM_U1_MAX_TIMEOUT 0x7F
#define USB3_LPM_U2_MAX_TIMEOUT 0xFE
#define USB3_LPM_DEVICE_INITIATED 0xFF
struct usb_set_sel_req {
__u8 u1_sel;
__u8 u1_pel;
__le16 u2_sel;
__le16 u2_pel;
} __attribute__ ((packed));
/*
* The Set System Exit Latency control transfer provides one byte each for
* U1 SEL and U1 PEL, so the max exit latency is 0xFF. U2 SEL and U2 PEL each
* are two bytes long.
*/
#define USB3_LPM_MAX_U1_SEL_PEL 0xFF
#define USB3_LPM_MAX_U2_SEL_PEL 0xFFFF
/*-------------------------------------------------------------------------*/
/*
* As per USB compliance update, a device that is actively drawing
* more than 100mA from USB must report itself as bus-powered in
* the GetStatus(DEVICE) call.
* https://compliance.usb.org/index.asp?UpdateFile=Electrical&Format=Standard#34
*/
#define USB_SELF_POWER_VBUS_MAX_DRAW 100
#endif /* _UAPI__LINUX_USB_CH9_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_SEQLOCK_H
#define __LINUX_SEQLOCK_H
/*
* seqcount_t / seqlock_t - a reader-writer consistency mechanism with
* lockless readers (read-only retry loops), and no writer starvation.
*
* See Documentation/locking/seqlock.rst
*
* Copyrights:
* - Based on x86_64 vsyscall gettimeofday: Keith Owens, Andrea Arcangeli
* - Sequence counters with associated locks, (C) 2020 Linutronix GmbH
*/
#include <linux/compiler.h>
#include <linux/kcsan-checks.h>
#include <linux/lockdep.h>
#include <linux/mutex.h>
#include <linux/preempt.h>
#include <linux/seqlock_types.h>
#include <linux/spinlock.h>
#include <asm/processor.h>
/*
* The seqlock seqcount_t interface does not prescribe a precise sequence of
* read begin/retry/end. For readers, typically there is a call to
* read_seqcount_begin() and read_seqcount_retry(), however, there are more
* esoteric cases which do not follow this pattern.
*
* As a consequence, we take the following best-effort approach for raw usage
* via seqcount_t under KCSAN: upon beginning a seq-reader critical section,
* pessimistically mark the next KCSAN_SEQLOCK_REGION_MAX memory accesses as
* atomics; if there is a matching read_seqcount_retry() call, no following
* memory operations are considered atomic. Usage of the seqlock_t interface
* is not affected.
*/
#define KCSAN_SEQLOCK_REGION_MAX 1000
static inline void __seqcount_init(seqcount_t *s, const char *name,
struct lock_class_key *key)
{
/*
* Make sure we are not reinitializing a held lock:
*/
lockdep_init_map(&s->dep_map, name, key, 0);
s->sequence = 0;
}
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define SEQCOUNT_DEP_MAP_INIT(lockname) \
.dep_map = { .name = #lockname }
/**
* seqcount_init() - runtime initializer for seqcount_t
* @s: Pointer to the seqcount_t instance
*/
# define seqcount_init(s) \
do { \
static struct lock_class_key __key; \
__seqcount_init((s), #s, &__key); \
} while (0)
static inline void seqcount_lockdep_reader_access(const seqcount_t *s)
{
seqcount_t *l = (seqcount_t *)s;
unsigned long flags;
local_irq_save(flags); seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_);
seqcount_release(&l->dep_map, _RET_IP_);
local_irq_restore(flags);
}
#else
# define SEQCOUNT_DEP_MAP_INIT(lockname)
# define seqcount_init(s) __seqcount_init(s, NULL, NULL)
# define seqcount_lockdep_reader_access(x)
#endif
/**
* SEQCNT_ZERO() - static initializer for seqcount_t
* @name: Name of the seqcount_t instance
*/
#define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) }
/*
* Sequence counters with associated locks (seqcount_LOCKNAME_t)
*
* A sequence counter which associates the lock used for writer
* serialization at initialization time. This enables lockdep to validate
* that the write side critical section is properly serialized.
*
* For associated locks which do not implicitly disable preemption,
* preemption protection is enforced in the write side function.
*
* Lockdep is never used in any for the raw write variants.
*
* See Documentation/locking/seqlock.rst
*/
/*
* typedef seqcount_LOCKNAME_t - sequence counter with LOCKNAME associated
* @seqcount: The real sequence counter
* @lock: Pointer to the associated lock
*
* A plain sequence counter with external writer synchronization by
* LOCKNAME @lock. The lock is associated to the sequence counter in the
* static initializer or init function. This enables lockdep to validate
* that the write side critical section is properly serialized.
*
* LOCKNAME: raw_spinlock, spinlock, rwlock or mutex
*/
/*
* seqcount_LOCKNAME_init() - runtime initializer for seqcount_LOCKNAME_t
* @s: Pointer to the seqcount_LOCKNAME_t instance
* @lock: Pointer to the associated lock
*/
#define seqcount_LOCKNAME_init(s, _lock, lockname) \
do { \
seqcount_##lockname##_t *____s = (s); \
seqcount_init(&____s->seqcount); \
__SEQ_LOCK(____s->lock = (_lock)); \
} while (0)
#define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock)
#define seqcount_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, spinlock)
#define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock)
#define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex)
/*
* SEQCOUNT_LOCKNAME() - Instantiate seqcount_LOCKNAME_t and helpers
* seqprop_LOCKNAME_*() - Property accessors for seqcount_LOCKNAME_t
*
* @lockname: "LOCKNAME" part of seqcount_LOCKNAME_t
* @locktype: LOCKNAME canonical C data type
* @preemptible: preemptibility of above locktype
* @lockbase: prefix for associated lock/unlock
*/
#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockbase) \
static __always_inline seqcount_t * \
__seqprop_##lockname##_ptr(seqcount_##lockname##_t *s) \
{ \
return &s->seqcount; \
} \
\
static __always_inline const seqcount_t * \
__seqprop_##lockname##_const_ptr(const seqcount_##lockname##_t *s) \
{ \
return &s->seqcount; \
} \
\
static __always_inline unsigned \
__seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s) \
{ \
unsigned seq = READ_ONCE(s->seqcount.sequence); \
\
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \
return seq; \
\
if (preemptible && unlikely(seq & 1)) { \
__SEQ_LOCK(lockbase##_lock(s->lock)); \
__SEQ_LOCK(lockbase##_unlock(s->lock)); \
\
/* \
* Re-read the sequence counter since the (possibly \
* preempted) writer made progress. \
*/ \
seq = READ_ONCE(s->seqcount.sequence); \
} \
\
return seq; \
} \
\
static __always_inline bool \
__seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s) \
{ \
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \
return preemptible; \
\
/* PREEMPT_RT relies on the above LOCK+UNLOCK */ \
return false; \
} \
\
static __always_inline void \
__seqprop_##lockname##_assert(const seqcount_##lockname##_t *s) \
{ \
__SEQ_LOCK(lockdep_assert_held(s->lock)); \
}
/*
* __seqprop() for seqcount_t
*/
static inline seqcount_t *__seqprop_ptr(seqcount_t *s)
{
return s;
}
static inline const seqcount_t *__seqprop_const_ptr(const seqcount_t *s)
{
return s;
}
static inline unsigned __seqprop_sequence(const seqcount_t *s)
{
return READ_ONCE(s->sequence);
}
static inline bool __seqprop_preemptible(const seqcount_t *s)
{
return false;
}
static inline void __seqprop_assert(const seqcount_t *s)
{
lockdep_assert_preemption_disabled();
}
#define __SEQ_RT IS_ENABLED(CONFIG_PREEMPT_RT)
SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t, false, raw_spin)SEQCOUNT_LOCKNAME(spinlock, spinlock_t, __SEQ_RT, spin)
SEQCOUNT_LOCKNAME(rwlock, rwlock_t, __SEQ_RT, read)
SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex)
#undef SEQCOUNT_LOCKNAME
/*
* SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t
* @name: Name of the seqcount_LOCKNAME_t instance
* @lock: Pointer to the associated LOCKNAME
*/
#define SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock) { \
.seqcount = SEQCNT_ZERO(seq_name.seqcount), \
__SEQ_LOCK(.lock = (assoc_lock)) \
}
#define SEQCNT_RAW_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_RWLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define SEQCNT_WW_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define __seqprop_case(s, lockname, prop) \
seqcount_##lockname##_t: __seqprop_##lockname##_##prop
#define __seqprop(s, prop) _Generic(*(s), \
seqcount_t: __seqprop_##prop, \
__seqprop_case((s), raw_spinlock, prop), \
__seqprop_case((s), spinlock, prop), \
__seqprop_case((s), rwlock, prop), \
__seqprop_case((s), mutex, prop))
#define seqprop_ptr(s) __seqprop(s, ptr)(s)
#define seqprop_const_ptr(s) __seqprop(s, const_ptr)(s)
#define seqprop_sequence(s) __seqprop(s, sequence)(s)
#define seqprop_preemptible(s) __seqprop(s, preemptible)(s)
#define seqprop_assert(s) __seqprop(s, assert)(s)
/**
* __read_seqcount_begin() - begin a seqcount_t read section w/o barrier
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb()
* barrier. Callers should ensure that smp_rmb() or equivalent ordering is
* provided before actually loading any of the variables that are to be
* protected in this critical section.
*
* Use carefully, only in critical code, and comment how the barrier is
* provided.
*
* Return: count to be passed to read_seqcount_retry()
*/
#define __read_seqcount_begin(s) \
({ \
unsigned __seq; \
\
while ((__seq = seqprop_sequence(s)) & 1) \
cpu_relax(); \
\
kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \
__seq; \
})
/**
* raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Return: count to be passed to read_seqcount_retry()
*/
#define raw_read_seqcount_begin(s) \
({ \
unsigned _seq = __read_seqcount_begin(s); \
\
smp_rmb(); \
_seq; \
})
/**
* read_seqcount_begin() - begin a seqcount_t read critical section
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Return: count to be passed to read_seqcount_retry()
*/
#define read_seqcount_begin(s) \
({ \
seqcount_lockdep_reader_access(seqprop_const_ptr(s)); \
raw_read_seqcount_begin(s); \
})
/**
* raw_read_seqcount() - read the raw seqcount_t counter value
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* raw_read_seqcount opens a read critical section of the given
* seqcount_t, without any lockdep checking, and without checking or
* masking the sequence counter LSB. Calling code is responsible for
* handling that.
*
* Return: count to be passed to read_seqcount_retry()
*/
#define raw_read_seqcount(s) \
({ \
unsigned __seq = seqprop_sequence(s); \
\
smp_rmb(); \
kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \
__seq; \
})
/**
* raw_seqcount_begin() - begin a seqcount_t read critical section w/o
* lockdep and w/o counter stabilization
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* raw_seqcount_begin opens a read critical section of the given
* seqcount_t. Unlike read_seqcount_begin(), this function will not wait
* for the count to stabilize. If a writer is active when it begins, it
* will fail the read_seqcount_retry() at the end of the read critical
* section instead of stabilizing at the beginning of it.
*
* Use this only in special kernel hot paths where the read section is
* small and has a high probability of success through other external
* means. It will save a single branching instruction.
*
* Return: count to be passed to read_seqcount_retry()
*/
#define raw_seqcount_begin(s) \
({ \
/* \
* If the counter is odd, let read_seqcount_retry() fail \
* by decrementing the counter. \
*/ \
raw_read_seqcount(s) & ~1; \
})
/**
* __read_seqcount_retry() - end a seqcount_t read section w/o barrier
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
* @start: count, from read_seqcount_begin()
*
* __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb()
* barrier. Callers should ensure that smp_rmb() or equivalent ordering is
* provided before actually loading any of the variables that are to be
* protected in this critical section.
*
* Use carefully, only in critical code, and comment how the barrier is
* provided.
*
* Return: true if a read section retry is required, else false
*/
#define __read_seqcount_retry(s, start) \
do___read_seqcount_retry(seqprop_const_ptr(s), start)
static inline int do___read_seqcount_retry(const seqcount_t *s, unsigned start)
{
kcsan_atomic_next(0);
return unlikely(READ_ONCE(s->sequence) != start);
}
/**
* read_seqcount_retry() - end a seqcount_t read critical section
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
* @start: count, from read_seqcount_begin()
*
* read_seqcount_retry closes the read critical section of given
* seqcount_t. If the critical section was invalid, it must be ignored
* (and typically retried).
*
* Return: true if a read section retry is required, else false
*/
#define read_seqcount_retry(s, start) \
do_read_seqcount_retry(seqprop_const_ptr(s), start)
static inline int do_read_seqcount_retry(const seqcount_t *s, unsigned start)
{
smp_rmb();
return do___read_seqcount_retry(s, start);
}
/**
* raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Context: check write_seqcount_begin()
*/
#define raw_write_seqcount_begin(s) \
do { \
if (seqprop_preemptible(s)) \
preempt_disable(); \
\
do_raw_write_seqcount_begin(seqprop_ptr(s)); \
} while (0)
static inline void do_raw_write_seqcount_begin(seqcount_t *s)
{
kcsan_nestable_atomic_begin();
s->sequence++;
smp_wmb();
}
/**
* raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Context: check write_seqcount_end()
*/
#define raw_write_seqcount_end(s) \
do { \
do_raw_write_seqcount_end(seqprop_ptr(s)); \
\
if (seqprop_preemptible(s)) \
preempt_enable(); \
} while (0)
static inline void do_raw_write_seqcount_end(seqcount_t *s)
{
smp_wmb();
s->sequence++;
kcsan_nestable_atomic_end();
}
/**
* write_seqcount_begin_nested() - start a seqcount_t write section with
* custom lockdep nesting level
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
* @subclass: lockdep nesting level
*
* See Documentation/locking/lockdep-design.rst
* Context: check write_seqcount_begin()
*/
#define write_seqcount_begin_nested(s, subclass) \
do { \
seqprop_assert(s); \
\
if (seqprop_preemptible(s)) \
preempt_disable(); \
\
do_write_seqcount_begin_nested(seqprop_ptr(s), subclass); \
} while (0)
static inline void do_write_seqcount_begin_nested(seqcount_t *s, int subclass)
{
seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_);
do_raw_write_seqcount_begin(s);
}
/**
* write_seqcount_begin() - start a seqcount_t write side critical section
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Context: sequence counter write side sections must be serialized and
* non-preemptible. Preemption will be automatically disabled if and
* only if the seqcount write serialization lock is associated, and
* preemptible. If readers can be invoked from hardirq or softirq
* context, interrupts or bottom halves must be respectively disabled.
*/
#define write_seqcount_begin(s) \
do { \
seqprop_assert(s); \
\
if (seqprop_preemptible(s)) \
preempt_disable(); \
\
do_write_seqcount_begin(seqprop_ptr(s)); \
} while (0)
static inline void do_write_seqcount_begin(seqcount_t *s)
{
do_write_seqcount_begin_nested(s, 0);
}
/**
* write_seqcount_end() - end a seqcount_t write side critical section
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Context: Preemption will be automatically re-enabled if and only if
* the seqcount write serialization lock is associated, and preemptible.
*/
#define write_seqcount_end(s) \
do { \
do_write_seqcount_end(seqprop_ptr(s)); \
\
if (seqprop_preemptible(s)) \
preempt_enable(); \
} while (0)
static inline void do_write_seqcount_end(seqcount_t *s)
{
seqcount_release(&s->dep_map, _RET_IP_);
do_raw_write_seqcount_end(s);
}
/**
* raw_write_seqcount_barrier() - do a seqcount_t write barrier
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* This can be used to provide an ordering guarantee instead of the usual
* consistency guarantee. It is one wmb cheaper, because it can collapse
* the two back-to-back wmb()s.
*
* Note that writes surrounding the barrier should be declared atomic (e.g.
* via WRITE_ONCE): a) to ensure the writes become visible to other threads
* atomically, avoiding compiler optimizations; b) to document which writes are
* meant to propagate to the reader critical section. This is necessary because
* neither writes before nor after the barrier are enclosed in a seq-writer
* critical section that would ensure readers are aware of ongoing writes::
*
* seqcount_t seq;
* bool X = true, Y = false;
*
* void read(void)
* {
* bool x, y;
*
* do {
* int s = read_seqcount_begin(&seq);
*
* x = X; y = Y;
*
* } while (read_seqcount_retry(&seq, s));
*
* BUG_ON(!x && !y);
* }
*
* void write(void)
* {
* WRITE_ONCE(Y, true);
*
* raw_write_seqcount_barrier(seq);
*
* WRITE_ONCE(X, false);
* }
*/
#define raw_write_seqcount_barrier(s) \
do_raw_write_seqcount_barrier(seqprop_ptr(s))
static inline void do_raw_write_seqcount_barrier(seqcount_t *s)
{
kcsan_nestable_atomic_begin();
s->sequence++;
smp_wmb();
s->sequence++;
kcsan_nestable_atomic_end();
}
/**
* write_seqcount_invalidate() - invalidate in-progress seqcount_t read
* side operations
* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* After write_seqcount_invalidate, no seqcount_t read side operations
* will complete successfully and see data older than this.
*/
#define write_seqcount_invalidate(s) \
do_write_seqcount_invalidate(seqprop_ptr(s))
static inline void do_write_seqcount_invalidate(seqcount_t *s)
{
smp_wmb();
kcsan_nestable_atomic_begin();
s->sequence+=2;
kcsan_nestable_atomic_end();
}
/*
* Latch sequence counters (seqcount_latch_t)
*
* A sequence counter variant where the counter even/odd value is used to
* switch between two copies of protected data. This allows the read path,
* typically NMIs, to safely interrupt the write side critical section.
*
* As the write sections are fully preemptible, no special handling for
* PREEMPT_RT is needed.
*/
typedef struct {
seqcount_t seqcount;
} seqcount_latch_t;
/**
* SEQCNT_LATCH_ZERO() - static initializer for seqcount_latch_t
* @seq_name: Name of the seqcount_latch_t instance
*/
#define SEQCNT_LATCH_ZERO(seq_name) { \
.seqcount = SEQCNT_ZERO(seq_name.seqcount), \
}
/**
* seqcount_latch_init() - runtime initializer for seqcount_latch_t
* @s: Pointer to the seqcount_latch_t instance
*/
#define seqcount_latch_init(s) seqcount_init(&(s)->seqcount)
/**
* raw_read_seqcount_latch() - pick even/odd latch data copy
* @s: Pointer to seqcount_latch_t
*
* See raw_write_seqcount_latch() for details and a full reader/writer
* usage example.
*
* Return: sequence counter raw value. Use the lowest bit as an index for
* picking which data copy to read. The full counter must then be checked
* with raw_read_seqcount_latch_retry().
*/
static __always_inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s)
{
/*
* Pairs with the first smp_wmb() in raw_write_seqcount_latch().
* Due to the dependent load, a full smp_rmb() is not needed.
*/
return READ_ONCE(s->seqcount.sequence);
}
/**
* raw_read_seqcount_latch_retry() - end a seqcount_latch_t read section
* @s: Pointer to seqcount_latch_t
* @start: count, from raw_read_seqcount_latch()
*
* Return: true if a read section retry is required, else false
*/
static __always_inline int
raw_read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)
{
smp_rmb();
return unlikely(READ_ONCE(s->seqcount.sequence) != start);
}
/**
* raw_write_seqcount_latch() - redirect latch readers to even/odd copy
* @s: Pointer to seqcount_latch_t
*
* The latch technique is a multiversion concurrency control method that allows
* queries during non-atomic modifications. If you can guarantee queries never
* interrupt the modification -- e.g. the concurrency is strictly between CPUs
* -- you most likely do not need this.
*
* Where the traditional RCU/lockless data structures rely on atomic
* modifications to ensure queries observe either the old or the new state the
* latch allows the same for non-atomic updates. The trade-off is doubling the
* cost of storage; we have to maintain two copies of the entire data
* structure.
*
* Very simply put: we first modify one copy and then the other. This ensures
* there is always one copy in a stable state, ready to give us an answer.
*
* The basic form is a data structure like::
*
* struct latch_struct {
* seqcount_latch_t seq;
* struct data_struct data[2];
* };
*
* Where a modification, which is assumed to be externally serialized, does the
* following::
*
* void latch_modify(struct latch_struct *latch, ...)
* {
* smp_wmb(); // Ensure that the last data[1] update is visible
* latch->seq.sequence++;
* smp_wmb(); // Ensure that the seqcount update is visible
*
* modify(latch->data[0], ...);
*
* smp_wmb(); // Ensure that the data[0] update is visible
* latch->seq.sequence++;
* smp_wmb(); // Ensure that the seqcount update is visible
*
* modify(latch->data[1], ...);
* }
*
* The query will have a form like::
*
* struct entry *latch_query(struct latch_struct *latch, ...)
* {
* struct entry *entry;
* unsigned seq, idx;
*
* do {
* seq = raw_read_seqcount_latch(&latch->seq);
*
* idx = seq & 0x01;
* entry = data_query(latch->data[idx], ...);
*
* // This includes needed smp_rmb()
* } while (raw_read_seqcount_latch_retry(&latch->seq, seq));
*
* return entry;
* }
*
* So during the modification, queries are first redirected to data[1]. Then we
* modify data[0]. When that is complete, we redirect queries back to data[0]
* and we can modify data[1].
*
* NOTE:
*
* The non-requirement for atomic modifications does _NOT_ include
* the publishing of new entries in the case where data is a dynamic
* data structure.
*
* An iteration might start in data[0] and get suspended long enough
* to miss an entire modification sequence, once it resumes it might
* observe the new entry.
*
* NOTE2:
*
* When data is a dynamic data structure; one should use regular RCU
* patterns to manage the lifetimes of the objects within.
*/
static inline void raw_write_seqcount_latch(seqcount_latch_t *s)
{
smp_wmb(); /* prior stores before incrementing "sequence" */
s->seqcount.sequence++;
smp_wmb(); /* increment "sequence" before following stores */
}
#define __SEQLOCK_UNLOCKED(lockname) \
{ \
.seqcount = SEQCNT_SPINLOCK_ZERO(lockname, &(lockname).lock), \
.lock = __SPIN_LOCK_UNLOCKED(lockname) \
}
/**
* seqlock_init() - dynamic initializer for seqlock_t
* @sl: Pointer to the seqlock_t instance
*/
#define seqlock_init(sl) \
do { \
spin_lock_init(&(sl)->lock); \
seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock); \
} while (0)
/**
* DEFINE_SEQLOCK(sl) - Define a statically allocated seqlock_t
* @sl: Name of the seqlock_t instance
*/
#define DEFINE_SEQLOCK(sl) \
seqlock_t sl = __SEQLOCK_UNLOCKED(sl)
/**
* read_seqbegin() - start a seqlock_t read side critical section
* @sl: Pointer to seqlock_t
*
* Return: count, to be passed to read_seqretry()
*/
static inline unsigned read_seqbegin(const seqlock_t *sl)
{
unsigned ret = read_seqcount_begin(&sl->seqcount);
kcsan_atomic_next(0); /* non-raw usage, assume closing read_seqretry() */
kcsan_flat_atomic_begin();
return ret;
}
/**
* read_seqretry() - end a seqlock_t read side section
* @sl: Pointer to seqlock_t
* @start: count, from read_seqbegin()
*
* read_seqretry closes the read side critical section of given seqlock_t.
* If the critical section was invalid, it must be ignored (and typically
* retried).
*
* Return: true if a read section retry is required, else false
*/
static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
{
/*
* Assume not nested: read_seqretry() may be called multiple times when
* completing read critical section.
*/
kcsan_flat_atomic_end();
return read_seqcount_retry(&sl->seqcount, start);
}
/*
* For all seqlock_t write side functions, use the internal
* do_write_seqcount_begin() instead of generic write_seqcount_begin().
* This way, no redundant lockdep_assert_held() checks are added.
*/
/**
* write_seqlock() - start a seqlock_t write side critical section
* @sl: Pointer to seqlock_t
*
* write_seqlock opens a write side critical section for the given
* seqlock_t. It also implicitly acquires the spinlock_t embedded inside
* that sequential lock. All seqlock_t write side sections are thus
* automatically serialized and non-preemptible.
*
* Context: if the seqlock_t read section, or other write side critical
* sections, can be invoked from hardirq or softirq contexts, use the
* _irqsave or _bh variants of this function instead.
*/
static inline void write_seqlock(seqlock_t *sl)
{
spin_lock(&sl->lock);
do_write_seqcount_begin(&sl->seqcount.seqcount);
}
/**
* write_sequnlock() - end a seqlock_t write side critical section
* @sl: Pointer to seqlock_t
*
* write_sequnlock closes the (serialized and non-preemptible) write side
* critical section of given seqlock_t.
*/
static inline void write_sequnlock(seqlock_t *sl)
{
do_write_seqcount_end(&sl->seqcount.seqcount);
spin_unlock(&sl->lock);
}
/**
* write_seqlock_bh() - start a softirqs-disabled seqlock_t write section
* @sl: Pointer to seqlock_t
*
* _bh variant of write_seqlock(). Use only if the read side section, or
* other write side sections, can be invoked from softirq contexts.
*/
static inline void write_seqlock_bh(seqlock_t *sl)
{
spin_lock_bh(&sl->lock);
do_write_seqcount_begin(&sl->seqcount.seqcount);
}
/**
* write_sequnlock_bh() - end a softirqs-disabled seqlock_t write section
* @sl: Pointer to seqlock_t
*
* write_sequnlock_bh closes the serialized, non-preemptible, and
* softirqs-disabled, seqlock_t write side critical section opened with
* write_seqlock_bh().
*/
static inline void write_sequnlock_bh(seqlock_t *sl)
{
do_write_seqcount_end(&sl->seqcount.seqcount);
spin_unlock_bh(&sl->lock);
}
/**
* write_seqlock_irq() - start a non-interruptible seqlock_t write section
* @sl: Pointer to seqlock_t
*
* _irq variant of write_seqlock(). Use only if the read side section, or
* other write sections, can be invoked from hardirq contexts.
*/
static inline void write_seqlock_irq(seqlock_t *sl)
{
spin_lock_irq(&sl->lock);
do_write_seqcount_begin(&sl->seqcount.seqcount);
}
/**
* write_sequnlock_irq() - end a non-interruptible seqlock_t write section
* @sl: Pointer to seqlock_t
*
* write_sequnlock_irq closes the serialized and non-interruptible
* seqlock_t write side section opened with write_seqlock_irq().
*/
static inline void write_sequnlock_irq(seqlock_t *sl)
{
do_write_seqcount_end(&sl->seqcount.seqcount);
spin_unlock_irq(&sl->lock);
}
static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl)
{
unsigned long flags;
spin_lock_irqsave(&sl->lock, flags);
do_write_seqcount_begin(&sl->seqcount.seqcount);
return flags;
}
/**
* write_seqlock_irqsave() - start a non-interruptible seqlock_t write
* section
* @lock: Pointer to seqlock_t
* @flags: Stack-allocated storage for saving caller's local interrupt
* state, to be passed to write_sequnlock_irqrestore().
*
* _irqsave variant of write_seqlock(). Use it only if the read side
* section, or other write sections, can be invoked from hardirq context.
*/
#define write_seqlock_irqsave(lock, flags) \
do { flags = __write_seqlock_irqsave(lock); } while (0)
/**
* write_sequnlock_irqrestore() - end non-interruptible seqlock_t write
* section
* @sl: Pointer to seqlock_t
* @flags: Caller's saved interrupt state, from write_seqlock_irqsave()
*
* write_sequnlock_irqrestore closes the serialized and non-interruptible
* seqlock_t write section previously opened with write_seqlock_irqsave().
*/
static inline void
write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
{
do_write_seqcount_end(&sl->seqcount.seqcount);
spin_unlock_irqrestore(&sl->lock, flags);
}
/**
* read_seqlock_excl() - begin a seqlock_t locking reader section
* @sl: Pointer to seqlock_t
*
* read_seqlock_excl opens a seqlock_t locking reader critical section. A
* locking reader exclusively locks out *both* other writers *and* other
* locking readers, but it does not update the embedded sequence number.
*
* Locking readers act like a normal spin_lock()/spin_unlock().
*
* Context: if the seqlock_t write section, *or other read sections*, can
* be invoked from hardirq or softirq contexts, use the _irqsave or _bh
* variant of this function instead.
*
* The opened read section must be closed with read_sequnlock_excl().
*/
static inline void read_seqlock_excl(seqlock_t *sl)
{
spin_lock(&sl->lock);
}
/**
* read_sequnlock_excl() - end a seqlock_t locking reader critical section
* @sl: Pointer to seqlock_t
*/
static inline void read_sequnlock_excl(seqlock_t *sl)
{
spin_unlock(&sl->lock);
}
/**
* read_seqlock_excl_bh() - start a seqlock_t locking reader section with
* softirqs disabled
* @sl: Pointer to seqlock_t
*
* _bh variant of read_seqlock_excl(). Use this variant only if the
* seqlock_t write side section, *or other read sections*, can be invoked
* from softirq contexts.
*/
static inline void read_seqlock_excl_bh(seqlock_t *sl)
{
spin_lock_bh(&sl->lock);
}
/**
* read_sequnlock_excl_bh() - stop a seqlock_t softirq-disabled locking
* reader section
* @sl: Pointer to seqlock_t
*/
static inline void read_sequnlock_excl_bh(seqlock_t *sl)
{
spin_unlock_bh(&sl->lock);
}
/**
* read_seqlock_excl_irq() - start a non-interruptible seqlock_t locking
* reader section
* @sl: Pointer to seqlock_t
*
* _irq variant of read_seqlock_excl(). Use this only if the seqlock_t
* write side section, *or other read sections*, can be invoked from a
* hardirq context.
*/
static inline void read_seqlock_excl_irq(seqlock_t *sl)
{
spin_lock_irq(&sl->lock);
}
/**
* read_sequnlock_excl_irq() - end an interrupts-disabled seqlock_t
* locking reader section
* @sl: Pointer to seqlock_t
*/
static inline void read_sequnlock_excl_irq(seqlock_t *sl)
{
spin_unlock_irq(&sl->lock);
}
static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl)
{
unsigned long flags;
spin_lock_irqsave(&sl->lock, flags);
return flags;
}
/**
* read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t
* locking reader section
* @lock: Pointer to seqlock_t
* @flags: Stack-allocated storage for saving caller's local interrupt
* state, to be passed to read_sequnlock_excl_irqrestore().
*
* _irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t
* write side section, *or other read sections*, can be invoked from a
* hardirq context.
*/
#define read_seqlock_excl_irqsave(lock, flags) \
do { flags = __read_seqlock_excl_irqsave(lock); } while (0)
/**
* read_sequnlock_excl_irqrestore() - end non-interruptible seqlock_t
* locking reader section
* @sl: Pointer to seqlock_t
* @flags: Caller saved interrupt state, from read_seqlock_excl_irqsave()
*/
static inline void
read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags)
{
spin_unlock_irqrestore(&sl->lock, flags);
}
/**
* read_seqbegin_or_lock() - begin a seqlock_t lockless or locking reader
* @lock: Pointer to seqlock_t
* @seq : Marker and return parameter. If the passed value is even, the
* reader will become a *lockless* seqlock_t reader as in read_seqbegin().
* If the passed value is odd, the reader will become a *locking* reader
* as in read_seqlock_excl(). In the first call to this function, the
* caller *must* initialize and pass an even value to @seq; this way, a
* lockless read can be optimistically tried first.
*
* read_seqbegin_or_lock is an API designed to optimistically try a normal
* lockless seqlock_t read section first. If an odd counter is found, the
* lockless read trial has failed, and the next read iteration transforms
* itself into a full seqlock_t locking reader.
*
* This is typically used to avoid seqlock_t lockless readers starvation
* (too much retry loops) in the case of a sharp spike in write side
* activity.
*
* Context: if the seqlock_t write section, *or other read sections*, can
* be invoked from hardirq or softirq contexts, use the _irqsave or _bh
* variant of this function instead.
*
* Check Documentation/locking/seqlock.rst for template example code.
*
* Return: the encountered sequence counter value, through the @seq
* parameter, which is overloaded as a return parameter. This returned
* value must be checked with need_seqretry(). If the read section need to
* be retried, this returned value must also be passed as the @seq
* parameter of the next read_seqbegin_or_lock() iteration.
*/
static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
{
if (!(*seq & 1)) /* Even */ *seq = read_seqbegin(lock);
else /* Odd */
read_seqlock_excl(lock);
}
/**
* need_seqretry() - validate seqlock_t "locking or lockless" read section
* @lock: Pointer to seqlock_t
* @seq: sequence count, from read_seqbegin_or_lock()
*
* Return: true if a read section retry is required, false otherwise
*/
static inline int need_seqretry(seqlock_t *lock, int seq)
{
return !(seq & 1) && read_seqretry(lock, seq);
}
/**
* done_seqretry() - end seqlock_t "locking or lockless" reader section
* @lock: Pointer to seqlock_t
* @seq: count, from read_seqbegin_or_lock()
*
* done_seqretry finishes the seqlock_t read side critical section started
* with read_seqbegin_or_lock() and validated by need_seqretry().
*/
static inline void done_seqretry(seqlock_t *lock, int seq)
{
if (seq & 1) read_sequnlock_excl(lock);
}
/**
* read_seqbegin_or_lock_irqsave() - begin a seqlock_t lockless reader, or
* a non-interruptible locking reader
* @lock: Pointer to seqlock_t
* @seq: Marker and return parameter. Check read_seqbegin_or_lock().
*
* This is the _irqsave variant of read_seqbegin_or_lock(). Use it only if
* the seqlock_t write section, *or other read sections*, can be invoked
* from hardirq context.
*
* Note: Interrupts will be disabled only for "locking reader" mode.
*
* Return:
*
* 1. The saved local interrupts state in case of a locking reader, to
* be passed to done_seqretry_irqrestore().
*
* 2. The encountered sequence counter value, returned through @seq
* overloaded as a return parameter. Check read_seqbegin_or_lock().
*/
static inline unsigned long
read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq)
{
unsigned long flags = 0;
if (!(*seq & 1)) /* Even */
*seq = read_seqbegin(lock);
else /* Odd */
read_seqlock_excl_irqsave(lock, flags);
return flags;
}
/**
* done_seqretry_irqrestore() - end a seqlock_t lockless reader, or a
* non-interruptible locking reader section
* @lock: Pointer to seqlock_t
* @seq: Count, from read_seqbegin_or_lock_irqsave()
* @flags: Caller's saved local interrupt state in case of a locking
* reader, also from read_seqbegin_or_lock_irqsave()
*
* This is the _irqrestore variant of done_seqretry(). The read section
* must've been opened with read_seqbegin_or_lock_irqsave(), and validated
* by need_seqretry().
*/
static inline void
done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags)
{
if (seq & 1)
read_sequnlock_excl_irqrestore(lock, flags);
}
#endif /* __LINUX_SEQLOCK_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Implement CPU time clocks for the POSIX clock interface.
*/
#include <linux/sched/signal.h>
#include <linux/sched/cputime.h>
#include <linux/posix-timers.h>
#include <linux/errno.h>
#include <linux/math64.h>
#include <linux/uaccess.h>
#include <linux/kernel_stat.h>
#include <trace/events/timer.h>
#include <linux/tick.h>
#include <linux/workqueue.h>
#include <linux/compat.h>
#include <linux/sched/deadline.h>
#include <linux/task_work.h>
#include "posix-timers.h"
static void posix_cpu_timer_rearm(struct k_itimer *timer);
void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit)
{
posix_cputimers_init(pct);
if (cpu_limit != RLIM_INFINITY) {
pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC;
pct->timers_active = true;
}
}
/*
* Called after updating RLIMIT_CPU to run cpu timer and update
* tsk->signal->posix_cputimers.bases[clock].nextevt expiration cache if
* necessary. Needs siglock protection since other code may update the
* expiration cache as well.
*
* Returns 0 on success, -ESRCH on failure. Can fail if the task is exiting and
* we cannot lock_task_sighand. Cannot fail if task is current.
*/
int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new)
{
u64 nsecs = rlim_new * NSEC_PER_SEC;
unsigned long irq_fl;
if (!lock_task_sighand(task, &irq_fl))
return -ESRCH;
set_process_cpu_timer(task, CPUCLOCK_PROF, &nsecs, NULL);
unlock_task_sighand(task, &irq_fl);
return 0;
}
/*
* Functions for validating access to tasks.
*/
static struct pid *pid_for_clock(const clockid_t clock, bool gettime)
{
const bool thread = !!CPUCLOCK_PERTHREAD(clock);
const pid_t upid = CPUCLOCK_PID(clock);
struct pid *pid;
if (CPUCLOCK_WHICH(clock) >= CPUCLOCK_MAX)
return NULL;
/*
* If the encoded PID is 0, then the timer is targeted at current
* or the process to which current belongs.
*/
if (upid == 0)
return thread ? task_pid(current) : task_tgid(current);
pid = find_vpid(upid);
if (!pid)
return NULL;
if (thread) {
struct task_struct *tsk = pid_task(pid, PIDTYPE_PID);
return (tsk && same_thread_group(tsk, current)) ? pid : NULL;
}
/*
* For clock_gettime(PROCESS) allow finding the process by
* with the pid of the current task. The code needs the tgid
* of the process so that pid_task(pid, PIDTYPE_TGID) can be
* used to find the process.
*/
if (gettime && (pid == task_pid(current)))
return task_tgid(current);
/*
* For processes require that pid identifies a process.
*/
return pid_has_task(pid, PIDTYPE_TGID) ? pid : NULL;
}
static inline int validate_clock_permissions(const clockid_t clock)
{
int ret;
rcu_read_lock();
ret = pid_for_clock(clock, false) ? 0 : -EINVAL;
rcu_read_unlock();
return ret;
}
static inline enum pid_type clock_pid_type(const clockid_t clock)
{
return CPUCLOCK_PERTHREAD(clock) ? PIDTYPE_PID : PIDTYPE_TGID;
}
static inline struct task_struct *cpu_timer_task_rcu(struct k_itimer *timer)
{
return pid_task(timer->it.cpu.pid, clock_pid_type(timer->it_clock));
}
/*
* Update expiry time from increment, and increase overrun count,
* given the current clock sample.
*/
static u64 bump_cpu_timer(struct k_itimer *timer, u64 now)
{
u64 delta, incr, expires = timer->it.cpu.node.expires;
int i;
if (!timer->it_interval)
return expires;
if (now < expires)
return expires;
incr = timer->it_interval;
delta = now + incr - expires;
/* Don't use (incr*2 < delta), incr*2 might overflow. */
for (i = 0; incr < delta - incr; i++)
incr = incr << 1;
for (; i >= 0; incr >>= 1, i--) {
if (delta < incr)
continue;
timer->it.cpu.node.expires += incr;
timer->it_overrun += 1LL << i;
delta -= incr;
}
return timer->it.cpu.node.expires;
}
/* Check whether all cache entries contain U64_MAX, i.e. eternal expiry time */
static inline bool expiry_cache_is_inactive(const struct posix_cputimers *pct)
{
return !(~pct->bases[CPUCLOCK_PROF].nextevt |
~pct->bases[CPUCLOCK_VIRT].nextevt |
~pct->bases[CPUCLOCK_SCHED].nextevt);
}
static int
posix_cpu_clock_getres(const clockid_t which_clock, struct timespec64 *tp)
{
int error = validate_clock_permissions(which_clock);
if (!error) {
tp->tv_sec = 0;
tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ);
if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
/*
* If sched_clock is using a cycle counter, we
* don't have any idea of its true resolution
* exported, but it is much more than 1s/HZ.
*/
tp->tv_nsec = 1;
}
}
return error;
}
static int
posix_cpu_clock_set(const clockid_t clock, const struct timespec64 *tp)
{
int error = validate_clock_permissions(clock);
/*
* You can never reset a CPU clock, but we check for other errors
* in the call before failing with EPERM.
*/
return error ? : -EPERM;
}
/*
* Sample a per-thread clock for the given task. clkid is validated.
*/
static u64 cpu_clock_sample(const clockid_t clkid, struct task_struct *p)
{
u64 utime, stime;
if (clkid == CPUCLOCK_SCHED)
return task_sched_runtime(p);
task_cputime(p, &utime, &stime);
switch (clkid) {
case CPUCLOCK_PROF:
return utime + stime;
case CPUCLOCK_VIRT:
return utime;
default:
WARN_ON_ONCE(1);
}
return 0;
}
static inline void store_samples(u64 *samples, u64 stime, u64 utime, u64 rtime)
{
samples[CPUCLOCK_PROF] = stime + utime;
samples[CPUCLOCK_VIRT] = utime;
samples[CPUCLOCK_SCHED] = rtime;
}
static void task_sample_cputime(struct task_struct *p, u64 *samples)
{
u64 stime, utime;
task_cputime(p, &utime, &stime);
store_samples(samples, stime, utime, p->se.sum_exec_runtime);
}
static void proc_sample_cputime_atomic(struct task_cputime_atomic *at,
u64 *samples)
{
u64 stime, utime, rtime;
utime = atomic64_read(&at->utime);
stime = atomic64_read(&at->stime);
rtime = atomic64_read(&at->sum_exec_runtime);
store_samples(samples, stime, utime, rtime);
}
/*
* Set cputime to sum_cputime if sum_cputime > cputime. Use cmpxchg
* to avoid race conditions with concurrent updates to cputime.
*/
static inline void __update_gt_cputime(atomic64_t *cputime, u64 sum_cputime)
{
u64 curr_cputime = atomic64_read(cputime);
do {
if (sum_cputime <= curr_cputime)
return;
} while (!atomic64_try_cmpxchg(cputime, &curr_cputime, sum_cputime));
}
static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic,
struct task_cputime *sum)
{
__update_gt_cputime(&cputime_atomic->utime, sum->utime);
__update_gt_cputime(&cputime_atomic->stime, sum->stime);
__update_gt_cputime(&cputime_atomic->sum_exec_runtime, sum->sum_exec_runtime);
}
/**
* thread_group_sample_cputime - Sample cputime for a given task
* @tsk: Task for which cputime needs to be started
* @samples: Storage for time samples
*
* Called from sys_getitimer() to calculate the expiry time of an active
* timer. That means group cputime accounting is already active. Called
* with task sighand lock held.
*
* Updates @times with an uptodate sample of the thread group cputimes.
*/
void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples)
{
struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
struct posix_cputimers *pct = &tsk->signal->posix_cputimers;
WARN_ON_ONCE(!pct->timers_active);
proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
}
/**
* thread_group_start_cputime - Start cputime and return a sample
* @tsk: Task for which cputime needs to be started
* @samples: Storage for time samples
*
* The thread group cputime accounting is avoided when there are no posix
* CPU timers armed. Before starting a timer it's required to check whether
* the time accounting is active. If not, a full update of the atomic
* accounting store needs to be done and the accounting enabled.
*
* Updates @times with an uptodate sample of the thread group cputimes.
*/
static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples)
{
struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
struct posix_cputimers *pct = &tsk->signal->posix_cputimers;
lockdep_assert_task_sighand_held(tsk);
/* Check if cputimer isn't running. This is accessed without locking. */
if (!READ_ONCE(pct->timers_active)) {
struct task_cputime sum;
/*
* The POSIX timer interface allows for absolute time expiry
* values through the TIMER_ABSTIME flag, therefore we have
* to synchronize the timer to the clock every time we start it.
*/
thread_group_cputime(tsk, &sum);
update_gt_cputime(&cputimer->cputime_atomic, &sum);
/*
* We're setting timers_active without a lock. Ensure this
* only gets written to in one operation. We set it after
* update_gt_cputime() as a small optimization, but
* barriers are not required because update_gt_cputime()
* can handle concurrent updates.
*/
WRITE_ONCE(pct->timers_active, true);
}
proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
}
static void __thread_group_cputime(struct task_struct *tsk, u64 *samples)
{
struct task_cputime ct;
thread_group_cputime(tsk, &ct);
store_samples(samples, ct.stime, ct.utime, ct.sum_exec_runtime);
}
/*
* Sample a process (thread group) clock for the given task clkid. If the
* group's cputime accounting is already enabled, read the atomic
* store. Otherwise a full update is required. clkid is already validated.
*/
static u64 cpu_clock_sample_group(const clockid_t clkid, struct task_struct *p,
bool start)
{
struct thread_group_cputimer *cputimer = &p->signal->cputimer;
struct posix_cputimers *pct = &p->signal->posix_cputimers;
u64 samples[CPUCLOCK_MAX];
if (!READ_ONCE(pct->timers_active)) {
if (start)
thread_group_start_cputime(p, samples);
else
__thread_group_cputime(p, samples);
} else {
proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
}
return samples[clkid];
}
static int posix_cpu_clock_get(const clockid_t clock, struct timespec64 *tp)
{
const clockid_t clkid = CPUCLOCK_WHICH(clock);
struct task_struct *tsk;
u64 t;
rcu_read_lock();
tsk = pid_task(pid_for_clock(clock, true), clock_pid_type(clock));
if (!tsk) {
rcu_read_unlock();
return -EINVAL;
}
if (CPUCLOCK_PERTHREAD(clock))
t = cpu_clock_sample(clkid, tsk);
else
t = cpu_clock_sample_group(clkid, tsk, false);
rcu_read_unlock();
*tp = ns_to_timespec64(t);
return 0;
}
/*
* Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
* This is called from sys_timer_create() and do_cpu_nanosleep() with the
* new timer already all-zeros initialized.
*/
static int posix_cpu_timer_create(struct k_itimer *new_timer)
{
static struct lock_class_key posix_cpu_timers_key;
struct pid *pid;
rcu_read_lock();
pid = pid_for_clock(new_timer->it_clock, false);
if (!pid) {
rcu_read_unlock();
return -EINVAL;
}
/*
* If posix timer expiry is handled in task work context then
* timer::it_lock can be taken without disabling interrupts as all
* other locking happens in task context. This requires a separate
* lock class key otherwise regular posix timer expiry would record
* the lock class being taken in interrupt context and generate a
* false positive warning.
*/
if (IS_ENABLED(CONFIG_POSIX_CPU_TIMERS_TASK_WORK))
lockdep_set_class(&new_timer->it_lock, &posix_cpu_timers_key);
new_timer->kclock = &clock_posix_cpu;
timerqueue_init(&new_timer->it.cpu.node);
new_timer->it.cpu.pid = get_pid(pid);
rcu_read_unlock();
return 0;
}
static struct posix_cputimer_base *timer_base(struct k_itimer *timer,
struct task_struct *tsk)
{
int clkidx = CPUCLOCK_WHICH(timer->it_clock);
if (CPUCLOCK_PERTHREAD(timer->it_clock))
return tsk->posix_cputimers.bases + clkidx;
else
return tsk->signal->posix_cputimers.bases + clkidx;
}
/*
* Force recalculating the base earliest expiration on the next tick.
* This will also re-evaluate the need to keep around the process wide
* cputime counter and tick dependency and eventually shut these down
* if necessary.
*/
static void trigger_base_recalc_expires(struct k_itimer *timer,
struct task_struct *tsk)
{
struct posix_cputimer_base *base = timer_base(timer, tsk);
base->nextevt = 0;
}
/*
* Dequeue the timer and reset the base if it was its earliest expiration.
* It makes sure the next tick recalculates the base next expiration so we
* don't keep the costly process wide cputime counter around for a random
* amount of time, along with the tick dependency.
*
* If another timer gets queued between this and the next tick, its
* expiration will update the base next event if necessary on the next
* tick.
*/
static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
{
struct cpu_timer *ctmr = &timer->it.cpu;
struct posix_cputimer_base *base;
if (!cpu_timer_dequeue(ctmr))
return;
base = timer_base(timer, p);
if (cpu_timer_getexpires(ctmr) == base->nextevt)
trigger_base_recalc_expires(timer, p);
}
/*
* Clean up a CPU-clock timer that is about to be destroyed.
* This is called from timer deletion with the timer already locked.
* If we return TIMER_RETRY, it's necessary to release the timer's lock
* and try again. (This happens when the timer is in the middle of firing.)
*/
static int posix_cpu_timer_del(struct k_itimer *timer)
{
struct cpu_timer *ctmr = &timer->it.cpu;
struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
int ret = 0;
rcu_read_lock();
p = cpu_timer_task_rcu(timer);
if (!p)
goto out;
/*
* Protect against sighand release/switch in exit/exec and process/
* thread timer list entry concurrent read/writes.
*/
sighand = lock_task_sighand(p, &flags);
if (unlikely(sighand == NULL)) {
/*
* This raced with the reaping of the task. The exit cleanup
* should have removed this timer from the timer queue.
*/
WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
} else {
if (timer->it.cpu.firing)
ret = TIMER_RETRY;
else
disarm_timer(timer, p);
unlock_task_sighand(p, &flags);
}
out:
rcu_read_unlock();
if (!ret)
put_pid(ctmr->pid);
return ret;
}
static void cleanup_timerqueue(struct timerqueue_head *head)
{
struct timerqueue_node *node;
struct cpu_timer *ctmr;
while ((node = timerqueue_getnext(head))) {
timerqueue_del(head, node);
ctmr = container_of(node, struct cpu_timer, node);
ctmr->head = NULL;
}
}
/*
* Clean out CPU timers which are still armed when a thread exits. The
* timers are only removed from the list. No other updates are done. The
* corresponding posix timers are still accessible, but cannot be rearmed.
*
* This must be called with the siglock held.
*/
static void cleanup_timers(struct posix_cputimers *pct)
{
cleanup_timerqueue(&pct->bases[CPUCLOCK_PROF].tqhead);
cleanup_timerqueue(&pct->bases[CPUCLOCK_VIRT].tqhead);
cleanup_timerqueue(&pct->bases[CPUCLOCK_SCHED].tqhead);
}
/*
* These are both called with the siglock held, when the current thread
* is being reaped. When the final (leader) thread in the group is reaped,
* posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit.
*/
void posix_cpu_timers_exit(struct task_struct *tsk)
{
cleanup_timers(&tsk->posix_cputimers);
}
void posix_cpu_timers_exit_group(struct task_struct *tsk)
{
cleanup_timers(&tsk->signal->posix_cputimers);
}
/*
* Insert the timer on the appropriate list before any timers that
* expire later. This must be called with the sighand lock held.
*/
static void arm_timer(struct k_itimer *timer, struct task_struct *p)
{
struct posix_cputimer_base *base = timer_base(timer, p);
struct cpu_timer *ctmr = &timer->it.cpu;
u64 newexp = cpu_timer_getexpires(ctmr);
if (!cpu_timer_enqueue(&base->tqhead, ctmr))
return;
/*
* We are the new earliest-expiring POSIX 1.b timer, hence
* need to update expiration cache. Take into account that
* for process timers we share expiration cache with itimers
* and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME.
*/
if (newexp < base->nextevt)
base->nextevt = newexp;
if (CPUCLOCK_PERTHREAD(timer->it_clock))
tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER);
else
tick_dep_set_signal(p, TICK_DEP_BIT_POSIX_TIMER);
}
/*
* The timer is locked, fire it and arrange for its reload.
*/
static void cpu_timer_fire(struct k_itimer *timer)
{
struct cpu_timer *ctmr = &timer->it.cpu;
if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {
/*
* User don't want any signal.
*/
cpu_timer_setexpires(ctmr, 0);
} else if (unlikely(timer->sigq == NULL)) {
/*
* This a special case for clock_nanosleep,
* not a normal timer from sys_timer_create.
*/
wake_up_process(timer->it_process);
cpu_timer_setexpires(ctmr, 0);
} else if (!timer->it_interval) {
/*
* One-shot timer. Clear it as soon as it's fired.
*/
posix_timer_event(timer, 0);
cpu_timer_setexpires(ctmr, 0);
} else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {
/*
* The signal did not get queued because the signal
* was ignored, so we won't get any callback to
* reload the timer. But we need to keep it
* ticking in case the signal is deliverable next time.
*/
posix_cpu_timer_rearm(timer);
++timer->it_requeue_pending;
}
}
/*
* Guts of sys_timer_settime for CPU timers.
* This is called with the timer locked and interrupts disabled.
* If we return TIMER_RETRY, it's necessary to release the timer's lock
* and try again. (This happens when the timer is in the middle of firing.)
*/
static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
struct itimerspec64 *new, struct itimerspec64 *old)
{
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
u64 old_expires, new_expires, old_incr, val;
struct cpu_timer *ctmr = &timer->it.cpu;
struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
int ret = 0;
rcu_read_lock();
p = cpu_timer_task_rcu(timer);
if (!p) {
/*
* If p has just been reaped, we can no
* longer get any information about it at all.
*/
rcu_read_unlock();
return -ESRCH;
}
/*
* Use the to_ktime conversion because that clamps the maximum
* value to KTIME_MAX and avoid multiplication overflows.
*/
new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value));
/*
* Protect against sighand release/switch in exit/exec and p->cpu_timers
* and p->signal->cpu_timers read/write in arm_timer()
*/
sighand = lock_task_sighand(p, &flags);
/*
* If p has just been reaped, we can no
* longer get any information about it at all.
*/
if (unlikely(sighand == NULL)) {
rcu_read_unlock();
return -ESRCH;
}
/*
* Disarm any old timer after extracting its expiry time.
*/
old_incr = timer->it_interval;
old_expires = cpu_timer_getexpires(ctmr);
if (unlikely(timer->it.cpu.firing)) {
timer->it.cpu.firing = -1;
ret = TIMER_RETRY;
} else {
cpu_timer_dequeue(ctmr);
}
/*
* We need to sample the current value to convert the new
* value from to relative and absolute, and to convert the
* old value from absolute to relative. To set a process
* timer, we need a sample to balance the thread expiry
* times (in arm_timer). With an absolute time, we must
* check if it's already passed. In short, we need a sample.
*/
if (CPUCLOCK_PERTHREAD(timer->it_clock))
val = cpu_clock_sample(clkid, p);
else
val = cpu_clock_sample_group(clkid, p, true);
if (old) {
if (old_expires == 0) {
old->it_value.tv_sec = 0;
old->it_value.tv_nsec = 0;
} else {
/*
* Update the timer in case it has overrun already.
* If it has, we'll report it as having overrun and
* with the next reloaded timer already ticking,
* though we are swallowing that pending
* notification here to install the new setting.
*/
u64 exp = bump_cpu_timer(timer, val);
if (val < exp) {
old_expires = exp - val;
old->it_value = ns_to_timespec64(old_expires);
} else {
old->it_value.tv_nsec = 1;
old->it_value.tv_sec = 0;
}
}
}
if (unlikely(ret)) {
/*
* We are colliding with the timer actually firing.
* Punt after filling in the timer's old value, and
* disable this firing since we are already reporting
* it as an overrun (thanks to bump_cpu_timer above).
*/
unlock_task_sighand(p, &flags);
goto out;
}
if (new_expires != 0 && !(timer_flags & TIMER_ABSTIME)) {
new_expires += val;
}
/*
* Install the new expiry time (or zero).
* For a timer with no notification action, we don't actually
* arm the timer (we'll just fake it for timer_gettime).
*/
cpu_timer_setexpires(ctmr, new_expires);
if (new_expires != 0 && val < new_expires) {
arm_timer(timer, p);
}
unlock_task_sighand(p, &flags);
/*
* Install the new reload setting, and
* set up the signal and overrun bookkeeping.
*/
timer->it_interval = timespec64_to_ktime(new->it_interval);
/*
* This acts as a modification timestamp for the timer,
* so any automatic reload attempt will punt on seeing
* that we have reset the timer manually.
*/
timer->it_requeue_pending = (timer->it_requeue_pending + 2) &
~REQUEUE_PENDING;
timer->it_overrun_last = 0;
timer->it_overrun = -1;
if (val >= new_expires) {
if (new_expires != 0) {
/*
* The designated time already passed, so we notify
* immediately, even if the thread never runs to
* accumulate more time on this clock.
*/
cpu_timer_fire(timer);
}
/*
* Make sure we don't keep around the process wide cputime
* counter or the tick dependency if they are not necessary.
*/
sighand = lock_task_sighand(p, &flags);
if (!sighand)
goto out;
if (!cpu_timer_queued(ctmr))
trigger_base_recalc_expires(timer, p);
unlock_task_sighand(p, &flags);
}
out:
rcu_read_unlock();
if (old)
old->it_interval = ns_to_timespec64(old_incr);
return ret;
}
static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)
{
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
struct cpu_timer *ctmr = &timer->it.cpu;
u64 now, expires = cpu_timer_getexpires(ctmr);
struct task_struct *p;
rcu_read_lock();
p = cpu_timer_task_rcu(timer);
if (!p)
goto out;
/*
* Easy part: convert the reload time.
*/
itp->it_interval = ktime_to_timespec64(timer->it_interval);
if (!expires)
goto out;
/*
* Sample the clock to take the difference with the expiry time.
*/
if (CPUCLOCK_PERTHREAD(timer->it_clock))
now = cpu_clock_sample(clkid, p);
else
now = cpu_clock_sample_group(clkid, p, false);
if (now < expires) {
itp->it_value = ns_to_timespec64(expires - now);
} else {
/*
* The timer should have expired already, but the firing
* hasn't taken place yet. Say it's just about to expire.
*/
itp->it_value.tv_nsec = 1;
itp->it_value.tv_sec = 0;
}
out:
rcu_read_unlock();
}
#define MAX_COLLECTED 20
static u64 collect_timerqueue(struct timerqueue_head *head,
struct list_head *firing, u64 now)
{
struct timerqueue_node *next;
int i = 0;
while ((next = timerqueue_getnext(head))) {
struct cpu_timer *ctmr;
u64 expires;
ctmr = container_of(next, struct cpu_timer, node);
expires = cpu_timer_getexpires(ctmr);
/* Limit the number of timers to expire at once */
if (++i == MAX_COLLECTED || now < expires)
return expires;
ctmr->firing = 1;
/* See posix_cpu_timer_wait_running() */
rcu_assign_pointer(ctmr->handling, current);
cpu_timer_dequeue(ctmr);
list_add_tail(&ctmr->elist, firing);
}
return U64_MAX;
}
static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples,
struct list_head *firing)
{
struct posix_cputimer_base *base = pct->bases;
int i;
for (i = 0; i < CPUCLOCK_MAX; i++, base++) {
base->nextevt = collect_timerqueue(&base->tqhead, firing,
samples[i]);
}
}
static inline void check_dl_overrun(struct task_struct *tsk)
{
if (tsk->dl.dl_overrun) {
tsk->dl.dl_overrun = 0;
send_signal_locked(SIGXCPU, SEND_SIG_PRIV, tsk, PIDTYPE_TGID);
}
}
static bool check_rlimit(u64 time, u64 limit, int signo, bool rt, bool hard)
{
if (time < limit)
return false;
if (print_fatal_signals) {
pr_info("%s Watchdog Timeout (%s): %s[%d]\n",
rt ? "RT" : "CPU", hard ? "hard" : "soft",
current->comm, task_pid_nr(current));
}
send_signal_locked(signo, SEND_SIG_PRIV, current, PIDTYPE_TGID);
return true;
}
/*
* Check for any per-thread CPU timers that have fired and move them off
* the tsk->cpu_timers[N] list onto the firing list. Here we update the
* tsk->it_*_expires values to reflect the remaining thread CPU timers.
*/
static void check_thread_timers(struct task_struct *tsk,
struct list_head *firing)
{
struct posix_cputimers *pct = &tsk->posix_cputimers;
u64 samples[CPUCLOCK_MAX];
unsigned long soft;
if (dl_task(tsk))
check_dl_overrun(tsk);
if (expiry_cache_is_inactive(pct))
return;
task_sample_cputime(tsk, samples);
collect_posix_cputimers(pct, samples, firing);
/*
* Check for the special case thread timers.
*/
soft = task_rlimit(tsk, RLIMIT_RTTIME);
if (soft != RLIM_INFINITY) {
/* Task RT timeout is accounted in jiffies. RTTIME is usec */
unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ);
unsigned long hard = task_rlimit_max(tsk, RLIMIT_RTTIME);
/* At the hard limit, send SIGKILL. No further action. */
if (hard != RLIM_INFINITY &&
check_rlimit(rttime, hard, SIGKILL, true, true))
return;
/* At the soft limit, send a SIGXCPU every second */
if (check_rlimit(rttime, soft, SIGXCPU, true, false)) {
soft += USEC_PER_SEC;
tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur = soft;
}
}
if (expiry_cache_is_inactive(pct))
tick_dep_clear_task(tsk, TICK_DEP_BIT_POSIX_TIMER);
}
static inline void stop_process_timers(struct signal_struct *sig)
{
struct posix_cputimers *pct = &sig->posix_cputimers;
/* Turn off the active flag. This is done without locking. */
WRITE_ONCE(pct->timers_active, false);
tick_dep_clear_signal(sig, TICK_DEP_BIT_POSIX_TIMER);
}
static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it,
u64 *expires, u64 cur_time, int signo)
{
if (!it->expires)
return;
if (cur_time >= it->expires) {
if (it->incr)
it->expires += it->incr;
else
it->expires = 0;
trace_itimer_expire(signo == SIGPROF ?
ITIMER_PROF : ITIMER_VIRTUAL,
task_tgid(tsk), cur_time);
send_signal_locked(signo, SEND_SIG_PRIV, tsk, PIDTYPE_TGID);
}
if (it->expires && it->expires < *expires)
*expires = it->expires;
}
/*
* Check for any per-thread CPU timers that have fired and move them
* off the tsk->*_timers list onto the firing list. Per-thread timers
* have already been taken off.
*/
static void check_process_timers(struct task_struct *tsk,
struct list_head *firing)
{
struct signal_struct *const sig = tsk->signal;
struct posix_cputimers *pct = &sig->posix_cputimers;
u64 samples[CPUCLOCK_MAX];
unsigned long soft;
/*
* If there are no active process wide timers (POSIX 1.b, itimers,
* RLIMIT_CPU) nothing to check. Also skip the process wide timer
* processing when there is already another task handling them.
*/
if (!READ_ONCE(pct->timers_active) || pct->expiry_active)
return;
/*
* Signify that a thread is checking for process timers.
* Write access to this field is protected by the sighand lock.
*/
pct->expiry_active = true;
/*
* Collect the current process totals. Group accounting is active
* so the sample can be taken directly.
*/
proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples);
collect_posix_cputimers(pct, samples, firing);
/*
* Check for the special case process timers.
*/
check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF],
&pct->bases[CPUCLOCK_PROF].nextevt,
samples[CPUCLOCK_PROF], SIGPROF);
check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT],
&pct->bases[CPUCLOCK_VIRT].nextevt,
samples[CPUCLOCK_VIRT], SIGVTALRM);
soft = task_rlimit(tsk, RLIMIT_CPU);
if (soft != RLIM_INFINITY) {
/* RLIMIT_CPU is in seconds. Samples are nanoseconds */
unsigned long hard = task_rlimit_max(tsk, RLIMIT_CPU);
u64 ptime = samples[CPUCLOCK_PROF];
u64 softns = (u64)soft * NSEC_PER_SEC;
u64 hardns = (u64)hard * NSEC_PER_SEC;
/* At the hard limit, send SIGKILL. No further action. */
if (hard != RLIM_INFINITY &&
check_rlimit(ptime, hardns, SIGKILL, false, true))
return;
/* At the soft limit, send a SIGXCPU every second */
if (check_rlimit(ptime, softns, SIGXCPU, false, false)) {
sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1;
softns += NSEC_PER_SEC;
}
/* Update the expiry cache */
if (softns < pct->bases[CPUCLOCK_PROF].nextevt)
pct->bases[CPUCLOCK_PROF].nextevt = softns;
}
if (expiry_cache_is_inactive(pct))
stop_process_timers(sig);
pct->expiry_active = false;
}
/*
* This is called from the signal code (via posixtimer_rearm)
* when the last timer signal was delivered and we have to reload the timer.
*/
static void posix_cpu_timer_rearm(struct k_itimer *timer)
{
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
struct task_struct *p;
struct sighand_struct *sighand;
unsigned long flags;
u64 now;
rcu_read_lock();
p = cpu_timer_task_rcu(timer);
if (!p)
goto out;
/* Protect timer list r/w in arm_timer() */
sighand = lock_task_sighand(p, &flags);
if (unlikely(sighand == NULL))
goto out;
/*
* Fetch the current sample and update the timer's expiry time.
*/
if (CPUCLOCK_PERTHREAD(timer->it_clock))
now = cpu_clock_sample(clkid, p);
else
now = cpu_clock_sample_group(clkid, p, true);
bump_cpu_timer(timer, now);
/*
* Now re-arm for the new expiry time.
*/
arm_timer(timer, p);
unlock_task_sighand(p, &flags);
out:
rcu_read_unlock();
}
/**
* task_cputimers_expired - Check whether posix CPU timers are expired
*
* @samples: Array of current samples for the CPUCLOCK clocks
* @pct: Pointer to a posix_cputimers container
*
* Returns true if any member of @samples is greater than the corresponding
* member of @pct->bases[CLK].nextevt. False otherwise
*/
static inline bool
task_cputimers_expired(const u64 *samples, struct posix_cputimers *pct)
{
int i;
for (i = 0; i < CPUCLOCK_MAX; i++) { if (samples[i] >= pct->bases[i].nextevt)
return true;
}
return false;
}
/**
* fastpath_timer_check - POSIX CPU timers fast path.
*
* @tsk: The task (thread) being checked.
*
* Check the task and thread group timers. If both are zero (there are no
* timers set) return false. Otherwise snapshot the task and thread group
* timers and compare them with the corresponding expiration times. Return
* true if a timer has expired, else return false.
*/
static inline bool fastpath_timer_check(struct task_struct *tsk)
{
struct posix_cputimers *pct = &tsk->posix_cputimers;
struct signal_struct *sig;
if (!expiry_cache_is_inactive(pct)) {
u64 samples[CPUCLOCK_MAX];
task_sample_cputime(tsk, samples); if (task_cputimers_expired(samples, pct)) return true;
}
sig = tsk->signal;
pct = &sig->posix_cputimers;
/*
* Check if thread group timers expired when timers are active and
* no other thread in the group is already handling expiry for
* thread group cputimers. These fields are read without the
* sighand lock. However, this is fine because this is meant to be
* a fastpath heuristic to determine whether we should try to
* acquire the sighand lock to handle timer expiry.
*
* In the worst case scenario, if concurrently timers_active is set
* or expiry_active is cleared, but the current thread doesn't see
* the change yet, the timer checks are delayed until the next
* thread in the group gets a scheduler interrupt to handle the
* timer. This isn't an issue in practice because these types of
* delays with signals actually getting sent are expected.
*/
if (READ_ONCE(pct->timers_active) && !READ_ONCE(pct->expiry_active)) {
u64 samples[CPUCLOCK_MAX];
proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic,
samples);
if (task_cputimers_expired(samples, pct))
return true;
}
if (dl_task(tsk) && tsk->dl.dl_overrun)
return true;
return false;
}
static void handle_posix_cpu_timers(struct task_struct *tsk);
#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
static void posix_cpu_timers_work(struct callback_head *work)
{
struct posix_cputimers_work *cw = container_of(work, typeof(*cw), work);
mutex_lock(&cw->mutex);
handle_posix_cpu_timers(current);
mutex_unlock(&cw->mutex);
}
/*
* Invoked from the posix-timer core when a cancel operation failed because
* the timer is marked firing. The caller holds rcu_read_lock(), which
* protects the timer and the task which is expiring it from being freed.
*/
static void posix_cpu_timer_wait_running(struct k_itimer *timr)
{
struct task_struct *tsk = rcu_dereference(timr->it.cpu.handling);
/* Has the handling task completed expiry already? */
if (!tsk)
return;
/* Ensure that the task cannot go away */
get_task_struct(tsk);
/* Now drop the RCU protection so the mutex can be locked */
rcu_read_unlock();
/* Wait on the expiry mutex */
mutex_lock(&tsk->posix_cputimers_work.mutex);
/* Release it immediately again. */
mutex_unlock(&tsk->posix_cputimers_work.mutex);
/* Drop the task reference. */
put_task_struct(tsk);
/* Relock RCU so the callsite is balanced */
rcu_read_lock();
}
static void posix_cpu_timer_wait_running_nsleep(struct k_itimer *timr)
{
/* Ensure that timr->it.cpu.handling task cannot go away */
rcu_read_lock();
spin_unlock_irq(&timr->it_lock);
posix_cpu_timer_wait_running(timr);
rcu_read_unlock();
/* @timr is on stack and is valid */
spin_lock_irq(&timr->it_lock);
}
/*
* Clear existing posix CPU timers task work.
*/
void clear_posix_cputimers_work(struct task_struct *p)
{
/*
* A copied work entry from the old task is not meaningful, clear it.
* N.B. init_task_work will not do this.
*/
memset(&p->posix_cputimers_work.work, 0,
sizeof(p->posix_cputimers_work.work));
init_task_work(&p->posix_cputimers_work.work,
posix_cpu_timers_work);
mutex_init(&p->posix_cputimers_work.mutex);
p->posix_cputimers_work.scheduled = false;
}
/*
* Initialize posix CPU timers task work in init task. Out of line to
* keep the callback static and to avoid header recursion hell.
*/
void __init posix_cputimers_init_work(void)
{
clear_posix_cputimers_work(current);
}
/*
* Note: All operations on tsk->posix_cputimer_work.scheduled happen either
* in hard interrupt context or in task context with interrupts
* disabled. Aside of that the writer/reader interaction is always in the
* context of the current task, which means they are strict per CPU.
*/
static inline bool posix_cpu_timers_work_scheduled(struct task_struct *tsk)
{
return tsk->posix_cputimers_work.scheduled;
}
static inline void __run_posix_cpu_timers(struct task_struct *tsk)
{
if (WARN_ON_ONCE(tsk->posix_cputimers_work.scheduled))
return;
/* Schedule task work to actually expire the timers */
tsk->posix_cputimers_work.scheduled = true;
task_work_add(tsk, &tsk->posix_cputimers_work.work, TWA_RESUME);
}
static inline bool posix_cpu_timers_enable_work(struct task_struct *tsk,
unsigned long start)
{
bool ret = true;
/*
* On !RT kernels interrupts are disabled while collecting expired
* timers, so no tick can happen and the fast path check can be
* reenabled without further checks.
*/
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
tsk->posix_cputimers_work.scheduled = false;
return true;
}
/*
* On RT enabled kernels ticks can happen while the expired timers
* are collected under sighand lock. But any tick which observes
* the CPUTIMERS_WORK_SCHEDULED bit set, does not run the fastpath
* checks. So reenabling the tick work has do be done carefully:
*
* Disable interrupts and run the fast path check if jiffies have
* advanced since the collecting of expired timers started. If
* jiffies have not advanced or the fast path check did not find
* newly expired timers, reenable the fast path check in the timer
* interrupt. If there are newly expired timers, return false and
* let the collection loop repeat.
*/
local_irq_disable();
if (start != jiffies && fastpath_timer_check(tsk))
ret = false;
else
tsk->posix_cputimers_work.scheduled = false;
local_irq_enable();
return ret;
}
#else /* CONFIG_POSIX_CPU_TIMERS_TASK_WORK */
static inline void __run_posix_cpu_timers(struct task_struct *tsk)
{
lockdep_posixtimer_enter();
handle_posix_cpu_timers(tsk);
lockdep_posixtimer_exit();
}
static void posix_cpu_timer_wait_running(struct k_itimer *timr)
{
cpu_relax();
}
static void posix_cpu_timer_wait_running_nsleep(struct k_itimer *timr)
{
spin_unlock_irq(&timr->it_lock);
cpu_relax();
spin_lock_irq(&timr->it_lock);
}
static inline bool posix_cpu_timers_work_scheduled(struct task_struct *tsk)
{
return false;
}
static inline bool posix_cpu_timers_enable_work(struct task_struct *tsk,
unsigned long start)
{
return true;
}
#endif /* CONFIG_POSIX_CPU_TIMERS_TASK_WORK */
static void handle_posix_cpu_timers(struct task_struct *tsk)
{
struct k_itimer *timer, *next;
unsigned long flags, start;
LIST_HEAD(firing);
if (!lock_task_sighand(tsk, &flags))
return;
do {
/*
* On RT locking sighand lock does not disable interrupts,
* so this needs to be careful vs. ticks. Store the current
* jiffies value.
*/
start = READ_ONCE(jiffies);
barrier();
/*
* Here we take off tsk->signal->cpu_timers[N] and
* tsk->cpu_timers[N] all the timers that are firing, and
* put them on the firing list.
*/
check_thread_timers(tsk, &firing);
check_process_timers(tsk, &firing);
/*
* The above timer checks have updated the expiry cache and
* because nothing can have queued or modified timers after
* sighand lock was taken above it is guaranteed to be
* consistent. So the next timer interrupt fastpath check
* will find valid data.
*
* If timer expiry runs in the timer interrupt context then
* the loop is not relevant as timers will be directly
* expired in interrupt context. The stub function below
* returns always true which allows the compiler to
* optimize the loop out.
*
* If timer expiry is deferred to task work context then
* the following rules apply:
*
* - On !RT kernels no tick can have happened on this CPU
* after sighand lock was acquired because interrupts are
* disabled. So reenabling task work before dropping
* sighand lock and reenabling interrupts is race free.
*
* - On RT kernels ticks might have happened but the tick
* work ignored posix CPU timer handling because the
* CPUTIMERS_WORK_SCHEDULED bit is set. Reenabling work
* must be done very carefully including a check whether
* ticks have happened since the start of the timer
* expiry checks. posix_cpu_timers_enable_work() takes
* care of that and eventually lets the expiry checks
* run again.
*/
} while (!posix_cpu_timers_enable_work(tsk, start));
/*
* We must release sighand lock before taking any timer's lock.
* There is a potential race with timer deletion here, as the
* siglock now protects our private firing list. We have set
* the firing flag in each timer, so that a deletion attempt
* that gets the timer lock before we do will give it up and
* spin until we've taken care of that timer below.
*/
unlock_task_sighand(tsk, &flags);
/*
* Now that all the timers on our list have the firing flag,
* no one will touch their list entries but us. We'll take
* each timer's lock before clearing its firing flag, so no
* timer call will interfere.
*/
list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {
int cpu_firing;
/*
* spin_lock() is sufficient here even independent of the
* expiry context. If expiry happens in hard interrupt
* context it's obvious. For task work context it's safe
* because all other operations on timer::it_lock happen in
* task context (syscall or exit).
*/
spin_lock(&timer->it_lock);
list_del_init(&timer->it.cpu.elist);
cpu_firing = timer->it.cpu.firing;
timer->it.cpu.firing = 0;
/*
* The firing flag is -1 if we collided with a reset
* of the timer, which already reported this
* almost-firing as an overrun. So don't generate an event.
*/
if (likely(cpu_firing >= 0))
cpu_timer_fire(timer);
/* See posix_cpu_timer_wait_running() */
rcu_assign_pointer(timer->it.cpu.handling, NULL);
spin_unlock(&timer->it_lock);
}
}
/*
* This is called from the timer interrupt handler. The irq handler has
* already updated our counts. We need to check if any timers fire now.
* Interrupts are disabled.
*/
void run_posix_cpu_timers(void)
{
struct task_struct *tsk = current; lockdep_assert_irqs_disabled();
/*
* If the actual expiry is deferred to task work context and the
* work is already scheduled there is no point to do anything here.
*/
if (posix_cpu_timers_work_scheduled(tsk))
return;
/*
* The fast path checks that there are no expired thread or thread
* group timers. If that's so, just return.
*/
if (!fastpath_timer_check(tsk))
return;
__run_posix_cpu_timers(tsk);
}
/*
* Set one of the process-wide special case CPU timers or RLIMIT_CPU.
* The tsk->sighand->siglock must be held by the caller.
*/
void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid,
u64 *newval, u64 *oldval)
{
u64 now, *nextevt;
if (WARN_ON_ONCE(clkid >= CPUCLOCK_SCHED))
return;
nextevt = &tsk->signal->posix_cputimers.bases[clkid].nextevt;
now = cpu_clock_sample_group(clkid, tsk, true);
if (oldval) {
/*
* We are setting itimer. The *oldval is absolute and we update
* it to be relative, *newval argument is relative and we update
* it to be absolute.
*/
if (*oldval) {
if (*oldval <= now) {
/* Just about to fire. */
*oldval = TICK_NSEC;
} else {
*oldval -= now;
}
}
if (*newval)
*newval += now;
}
/*
* Update expiration cache if this is the earliest timer. CPUCLOCK_PROF
* expiry cache is also used by RLIMIT_CPU!.
*/
if (*newval < *nextevt)
*nextevt = *newval;
tick_dep_set_signal(tsk, TICK_DEP_BIT_POSIX_TIMER);
}
static int do_cpu_nanosleep(const clockid_t which_clock, int flags,
const struct timespec64 *rqtp)
{
struct itimerspec64 it;
struct k_itimer timer;
u64 expires;
int error;
/*
* Set up a temporary timer and then wait for it to go off.
*/
memset(&timer, 0, sizeof timer);
spin_lock_init(&timer.it_lock);
timer.it_clock = which_clock;
timer.it_overrun = -1;
error = posix_cpu_timer_create(&timer);
timer.it_process = current;
if (!error) {
static struct itimerspec64 zero_it;
struct restart_block *restart;
memset(&it, 0, sizeof(it));
it.it_value = *rqtp;
spin_lock_irq(&timer.it_lock);
error = posix_cpu_timer_set(&timer, flags, &it, NULL);
if (error) {
spin_unlock_irq(&timer.it_lock);
return error;
}
while (!signal_pending(current)) {
if (!cpu_timer_getexpires(&timer.it.cpu)) {
/*
* Our timer fired and was reset, below
* deletion can not fail.
*/
posix_cpu_timer_del(&timer);
spin_unlock_irq(&timer.it_lock);
return 0;
}
/*
* Block until cpu_timer_fire (or a signal) wakes us.
*/
__set_current_state(TASK_INTERRUPTIBLE);
spin_unlock_irq(&timer.it_lock);
schedule();
spin_lock_irq(&timer.it_lock);
}
/*
* We were interrupted by a signal.
*/
expires = cpu_timer_getexpires(&timer.it.cpu);
error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);
if (!error) {
/* Timer is now unarmed, deletion can not fail. */
posix_cpu_timer_del(&timer);
} else {
while (error == TIMER_RETRY) {
posix_cpu_timer_wait_running_nsleep(&timer);
error = posix_cpu_timer_del(&timer);
}
}
spin_unlock_irq(&timer.it_lock);
if ((it.it_value.tv_sec | it.it_value.tv_nsec) == 0) {
/*
* It actually did fire already.
*/
return 0;
}
error = -ERESTART_RESTARTBLOCK;
/*
* Report back to the user the time still remaining.
*/
restart = ¤t->restart_block;
restart->nanosleep.expires = expires;
if (restart->nanosleep.type != TT_NONE)
error = nanosleep_copyout(restart, &it.it_value);
}
return error;
}
static long posix_cpu_nsleep_restart(struct restart_block *restart_block);
static int posix_cpu_nsleep(const clockid_t which_clock, int flags,
const struct timespec64 *rqtp)
{
struct restart_block *restart_block = ¤t->restart_block;
int error;
/*
* Diagnose required errors first.
*/
if (CPUCLOCK_PERTHREAD(which_clock) &&
(CPUCLOCK_PID(which_clock) == 0 ||
CPUCLOCK_PID(which_clock) == task_pid_vnr(current)))
return -EINVAL;
error = do_cpu_nanosleep(which_clock, flags, rqtp);
if (error == -ERESTART_RESTARTBLOCK) {
if (flags & TIMER_ABSTIME)
return -ERESTARTNOHAND;
restart_block->nanosleep.clockid = which_clock;
set_restart_fn(restart_block, posix_cpu_nsleep_restart);
}
return error;
}
static long posix_cpu_nsleep_restart(struct restart_block *restart_block)
{
clockid_t which_clock = restart_block->nanosleep.clockid;
struct timespec64 t;
t = ns_to_timespec64(restart_block->nanosleep.expires);
return do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t);
}
#define PROCESS_CLOCK make_process_cpuclock(0, CPUCLOCK_SCHED)
#define THREAD_CLOCK make_thread_cpuclock(0, CPUCLOCK_SCHED)
static int process_cpu_clock_getres(const clockid_t which_clock,
struct timespec64 *tp)
{
return posix_cpu_clock_getres(PROCESS_CLOCK, tp);
}
static int process_cpu_clock_get(const clockid_t which_clock,
struct timespec64 *tp)
{
return posix_cpu_clock_get(PROCESS_CLOCK, tp);
}
static int process_cpu_timer_create(struct k_itimer *timer)
{
timer->it_clock = PROCESS_CLOCK;
return posix_cpu_timer_create(timer);
}
static int process_cpu_nsleep(const clockid_t which_clock, int flags,
const struct timespec64 *rqtp)
{
return posix_cpu_nsleep(PROCESS_CLOCK, flags, rqtp);
}
static int thread_cpu_clock_getres(const clockid_t which_clock,
struct timespec64 *tp)
{
return posix_cpu_clock_getres(THREAD_CLOCK, tp);
}
static int thread_cpu_clock_get(const clockid_t which_clock,
struct timespec64 *tp)
{
return posix_cpu_clock_get(THREAD_CLOCK, tp);
}
static int thread_cpu_timer_create(struct k_itimer *timer)
{
timer->it_clock = THREAD_CLOCK;
return posix_cpu_timer_create(timer);
}
const struct k_clock clock_posix_cpu = {
.clock_getres = posix_cpu_clock_getres,
.clock_set = posix_cpu_clock_set,
.clock_get_timespec = posix_cpu_clock_get,
.timer_create = posix_cpu_timer_create,
.nsleep = posix_cpu_nsleep,
.timer_set = posix_cpu_timer_set,
.timer_del = posix_cpu_timer_del,
.timer_get = posix_cpu_timer_get,
.timer_rearm = posix_cpu_timer_rearm,
.timer_wait_running = posix_cpu_timer_wait_running,
};
const struct k_clock clock_process = {
.clock_getres = process_cpu_clock_getres,
.clock_get_timespec = process_cpu_clock_get,
.timer_create = process_cpu_timer_create,
.nsleep = process_cpu_nsleep,
};
const struct k_clock clock_thread = {
.clock_getres = thread_cpu_clock_getres,
.clock_get_timespec = thread_cpu_clock_get,
.timer_create = thread_cpu_timer_create,
};
// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>
/*
* This is an implementation of the notion of "decrement a
* reference count, and return locked if it decremented to zero".
*
* NOTE NOTE NOTE! This is _not_ equivalent to
*
* if (atomic_dec_and_test(&atomic)) {
* spin_lock(&lock);
* return 1;
* }
* return 0;
*
* because the spin-lock and the decrement must be
* "atomic".
*/
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
{
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
if (atomic_add_unless(atomic, -1, 1))
return 0;
/* Otherwise do it the slow way */
spin_lock(lock);
if (atomic_dec_and_test(atomic))
return 1;
spin_unlock(lock);
return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_lock);
int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
unsigned long *flags)
{
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
if (atomic_add_unless(atomic, -1, 1))
return 0;
/* Otherwise do it the slow way */
spin_lock_irqsave(lock, *flags);
if (atomic_dec_and_test(atomic))
return 1;
spin_unlock_irqrestore(lock, *flags);
return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave);
int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
{
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
if (atomic_add_unless(atomic, -1, 1))
return 0;
/* Otherwise do it the slow way */
raw_spin_lock(lock);
if (atomic_dec_and_test(atomic))
return 1;
raw_spin_unlock(lock);
return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_raw_lock);
int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
unsigned long *flags)
{
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
if (atomic_add_unless(atomic, -1, 1))
return 0;
/* Otherwise do it the slow way */
raw_spin_lock_irqsave(lock, *flags);
if (atomic_dec_and_test(atomic))
return 1;
raw_spin_unlock_irqrestore(lock, *flags);
return 0;
}
EXPORT_SYMBOL(_atomic_dec_and_raw_lock_irqsave);
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Media device node
*
* Copyright (C) 2010 Nokia Corporation
*
* Contacts: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
* Sakari Ailus <sakari.ailus@iki.fi>
*
* --
*
* Common functions for media-related drivers to register and unregister media
* device nodes.
*/
#ifndef _MEDIA_DEVNODE_H
#define _MEDIA_DEVNODE_H
#include <linux/poll.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/cdev.h>
struct media_device;
/*
* Flag to mark the media_devnode struct as registered. Drivers must not touch
* this flag directly, it will be set and cleared by media_devnode_register and
* media_devnode_unregister.
*/
#define MEDIA_FLAG_REGISTERED 0
/**
* struct media_file_operations - Media device file operations
*
* @owner: should be filled with %THIS_MODULE
* @read: pointer to the function that implements read() syscall
* @write: pointer to the function that implements write() syscall
* @poll: pointer to the function that implements poll() syscall
* @ioctl: pointer to the function that implements ioctl() syscall
* @compat_ioctl: pointer to the function that will handle 32 bits userspace
* calls to the ioctl() syscall on a Kernel compiled with 64 bits.
* @open: pointer to the function that implements open() syscall
* @release: pointer to the function that will release the resources allocated
* by the @open function.
*/
struct media_file_operations {
struct module *owner;
ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
__poll_t (*poll) (struct file *, struct poll_table_struct *);
long (*ioctl) (struct file *, unsigned int, unsigned long);
long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
int (*open) (struct file *);
int (*release) (struct file *);
};
/**
* struct media_devnode - Media device node
* @media_dev: pointer to struct &media_device
* @fops: pointer to struct &media_file_operations with media device ops
* @dev: pointer to struct &device containing the media controller device
* @cdev: struct cdev pointer character device
* @parent: parent device
* @minor: device node minor number
* @flags: flags, combination of the ``MEDIA_FLAG_*`` constants
* @release: release callback called at the end of ``media_devnode_release()``
* routine at media-device.c.
*
* This structure represents a media-related device node.
*
* The @parent is a physical device. It must be set by core or device drivers
* before registering the node.
*/
struct media_devnode {
struct media_device *media_dev;
/* device ops */
const struct media_file_operations *fops;
/* sysfs */
struct device dev; /* media device */
struct cdev cdev; /* character device */
struct device *parent; /* device parent */
/* device info */
int minor;
unsigned long flags; /* Use bitops to access flags */
/* callbacks */
void (*release)(struct media_devnode *devnode);
};
/* dev to media_devnode */
#define to_media_devnode(cd) container_of(cd, struct media_devnode, dev)
/**
* media_devnode_register - register a media device node
*
* @mdev: struct media_device we want to register a device node
* @devnode: media device node structure we want to register
* @owner: should be filled with %THIS_MODULE
*
* The registration code assigns minor numbers and registers the new device node
* with the kernel. An error is returned if no free minor number can be found,
* or if the registration of the device node fails.
*
* Zero is returned on success.
*
* Note that if the media_devnode_register call fails, the release() callback of
* the media_devnode structure is *not* called, so the caller is responsible for
* freeing any data.
*/
int __must_check media_devnode_register(struct media_device *mdev,
struct media_devnode *devnode,
struct module *owner);
/**
* media_devnode_unregister_prepare - clear the media device node register bit
* @devnode: the device node to prepare for unregister
*
* This clears the passed device register bit. Future open calls will be met
* with errors. Should be called before media_devnode_unregister() to avoid
* races with unregister and device file open calls.
*
* This function can safely be called if the device node has never been
* registered or has already been unregistered.
*/
void media_devnode_unregister_prepare(struct media_devnode *devnode);
/**
* media_devnode_unregister - unregister a media device node
* @devnode: the device node to unregister
*
* This unregisters the passed device. Future open calls will be met with
* errors.
*
* Should be called after media_devnode_unregister_prepare()
*/
void media_devnode_unregister(struct media_devnode *devnode);
/**
* media_devnode_data - returns a pointer to the &media_devnode
*
* @filp: pointer to struct &file
*/
static inline struct media_devnode *media_devnode_data(struct file *filp)
{
return filp->private_data;
}
/**
* media_devnode_is_registered - returns true if &media_devnode is registered;
* false otherwise.
*
* @devnode: pointer to struct &media_devnode.
*
* Note: If mdev is NULL, it also returns false.
*/
static inline int media_devnode_is_registered(struct media_devnode *devnode)
{
if (!devnode)
return false;
return test_bit(MEDIA_FLAG_REGISTERED, &devnode->flags);
}
#endif /* _MEDIA_DEVNODE_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Provides code common for host and device side USB.
*
* If either host side (ie. CONFIG_USB=y) or device side USB stack
* (ie. CONFIG_USB_GADGET=y) is compiled in the kernel, this module is
* compiled-in as well. Otherwise, if either of the two stacks is
* compiled as module, this file is compiled as module as well.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/usb/ch9.h>
#include <linux/usb/of.h>
#include <linux/usb/otg.h>
#include <linux/of_platform.h>
#include <linux/debugfs.h>
#include "common.h"
static const char *const ep_type_names[] = {
[USB_ENDPOINT_XFER_CONTROL] = "ctrl",
[USB_ENDPOINT_XFER_ISOC] = "isoc",
[USB_ENDPOINT_XFER_BULK] = "bulk",
[USB_ENDPOINT_XFER_INT] = "intr",
};
/**
* usb_ep_type_string() - Returns human readable-name of the endpoint type.
* @ep_type: The endpoint type to return human-readable name for. If it's not
* any of the types: USB_ENDPOINT_XFER_{CONTROL, ISOC, BULK, INT},
* usually got by usb_endpoint_type(), the string 'unknown' will be returned.
*/
const char *usb_ep_type_string(int ep_type)
{
if (ep_type < 0 || ep_type >= ARRAY_SIZE(ep_type_names))
return "unknown";
return ep_type_names[ep_type];
}
EXPORT_SYMBOL_GPL(usb_ep_type_string);
const char *usb_otg_state_string(enum usb_otg_state state)
{
static const char *const names[] = {
[OTG_STATE_A_IDLE] = "a_idle",
[OTG_STATE_A_WAIT_VRISE] = "a_wait_vrise",
[OTG_STATE_A_WAIT_BCON] = "a_wait_bcon",
[OTG_STATE_A_HOST] = "a_host",
[OTG_STATE_A_SUSPEND] = "a_suspend",
[OTG_STATE_A_PERIPHERAL] = "a_peripheral",
[OTG_STATE_A_WAIT_VFALL] = "a_wait_vfall",
[OTG_STATE_A_VBUS_ERR] = "a_vbus_err",
[OTG_STATE_B_IDLE] = "b_idle",
[OTG_STATE_B_SRP_INIT] = "b_srp_init",
[OTG_STATE_B_PERIPHERAL] = "b_peripheral",
[OTG_STATE_B_WAIT_ACON] = "b_wait_acon",
[OTG_STATE_B_HOST] = "b_host",
};
if (state < 0 || state >= ARRAY_SIZE(names))
return "UNDEFINED";
return names[state];
}
EXPORT_SYMBOL_GPL(usb_otg_state_string);
static const char *const speed_names[] = {
[USB_SPEED_UNKNOWN] = "UNKNOWN",
[USB_SPEED_LOW] = "low-speed",
[USB_SPEED_FULL] = "full-speed",
[USB_SPEED_HIGH] = "high-speed",
[USB_SPEED_WIRELESS] = "wireless",
[USB_SPEED_SUPER] = "super-speed",
[USB_SPEED_SUPER_PLUS] = "super-speed-plus",
};
static const char *const ssp_rate[] = {
[USB_SSP_GEN_UNKNOWN] = "UNKNOWN",
[USB_SSP_GEN_2x1] = "super-speed-plus-gen2x1",
[USB_SSP_GEN_1x2] = "super-speed-plus-gen1x2",
[USB_SSP_GEN_2x2] = "super-speed-plus-gen2x2",
};
/**
* usb_speed_string() - Returns human readable-name of the speed.
* @speed: The speed to return human-readable name for. If it's not
* any of the speeds defined in usb_device_speed enum, string for
* USB_SPEED_UNKNOWN will be returned.
*/
const char *usb_speed_string(enum usb_device_speed speed)
{
if (speed < 0 || speed >= ARRAY_SIZE(speed_names))
speed = USB_SPEED_UNKNOWN;
return speed_names[speed];
}
EXPORT_SYMBOL_GPL(usb_speed_string);
/**
* usb_get_maximum_speed - Get maximum requested speed for a given USB
* controller.
* @dev: Pointer to the given USB controller device
*
* The function gets the maximum speed string from property "maximum-speed",
* and returns the corresponding enum usb_device_speed.
*/
enum usb_device_speed usb_get_maximum_speed(struct device *dev)
{
const char *maximum_speed;
int ret;
ret = device_property_read_string(dev, "maximum-speed", &maximum_speed);
if (ret < 0)
return USB_SPEED_UNKNOWN;
ret = match_string(ssp_rate, ARRAY_SIZE(ssp_rate), maximum_speed);
if (ret > 0)
return USB_SPEED_SUPER_PLUS;
ret = match_string(speed_names, ARRAY_SIZE(speed_names), maximum_speed);
return (ret < 0) ? USB_SPEED_UNKNOWN : ret;
}
EXPORT_SYMBOL_GPL(usb_get_maximum_speed);
/**
* usb_get_maximum_ssp_rate - Get the signaling rate generation and lane count
* of a SuperSpeed Plus capable device.
* @dev: Pointer to the given USB controller device
*
* If the string from "maximum-speed" property is super-speed-plus-genXxY where
* 'X' is the generation number and 'Y' is the number of lanes, then this
* function returns the corresponding enum usb_ssp_rate.
*/
enum usb_ssp_rate usb_get_maximum_ssp_rate(struct device *dev)
{
const char *maximum_speed;
int ret;
ret = device_property_read_string(dev, "maximum-speed", &maximum_speed);
if (ret < 0)
return USB_SSP_GEN_UNKNOWN;
ret = match_string(ssp_rate, ARRAY_SIZE(ssp_rate), maximum_speed);
return (ret < 0) ? USB_SSP_GEN_UNKNOWN : ret;
}
EXPORT_SYMBOL_GPL(usb_get_maximum_ssp_rate);
/**
* usb_state_string - Returns human readable name for the state.
* @state: The state to return a human-readable name for. If it's not
* any of the states devices in usb_device_state_string enum,
* the string UNKNOWN will be returned.
*/
const char *usb_state_string(enum usb_device_state state)
{
static const char *const names[] = {
[USB_STATE_NOTATTACHED] = "not attached",
[USB_STATE_ATTACHED] = "attached",
[USB_STATE_POWERED] = "powered",
[USB_STATE_RECONNECTING] = "reconnecting",
[USB_STATE_UNAUTHENTICATED] = "unauthenticated",
[USB_STATE_DEFAULT] = "default",
[USB_STATE_ADDRESS] = "addressed",
[USB_STATE_CONFIGURED] = "configured",
[USB_STATE_SUSPENDED] = "suspended",
};
if (state < 0 || state >= ARRAY_SIZE(names))
return "UNKNOWN";
return names[state];
}
EXPORT_SYMBOL_GPL(usb_state_string);
static const char *const usb_dr_modes[] = {
[USB_DR_MODE_UNKNOWN] = "",
[USB_DR_MODE_HOST] = "host",
[USB_DR_MODE_PERIPHERAL] = "peripheral",
[USB_DR_MODE_OTG] = "otg",
};
static enum usb_dr_mode usb_get_dr_mode_from_string(const char *str)
{
int ret;
ret = match_string(usb_dr_modes, ARRAY_SIZE(usb_dr_modes), str);
return (ret < 0) ? USB_DR_MODE_UNKNOWN : ret;
}
enum usb_dr_mode usb_get_dr_mode(struct device *dev)
{
const char *dr_mode;
int err;
err = device_property_read_string(dev, "dr_mode", &dr_mode);
if (err < 0)
return USB_DR_MODE_UNKNOWN;
return usb_get_dr_mode_from_string(dr_mode);
}
EXPORT_SYMBOL_GPL(usb_get_dr_mode);
/**
* usb_get_role_switch_default_mode - Get default mode for given device
* @dev: Pointer to the given device
*
* The function gets string from property 'role-switch-default-mode',
* and returns the corresponding enum usb_dr_mode.
*/
enum usb_dr_mode usb_get_role_switch_default_mode(struct device *dev)
{
const char *str;
int ret;
ret = device_property_read_string(dev, "role-switch-default-mode", &str);
if (ret < 0)
return USB_DR_MODE_UNKNOWN;
return usb_get_dr_mode_from_string(str);
}
EXPORT_SYMBOL_GPL(usb_get_role_switch_default_mode);
/**
* usb_decode_interval - Decode bInterval into the time expressed in 1us unit
* @epd: The descriptor of the endpoint
* @speed: The speed that the endpoint works as
*
* Function returns the interval expressed in 1us unit for servicing
* endpoint for data transfers.
*/
unsigned int usb_decode_interval(const struct usb_endpoint_descriptor *epd,
enum usb_device_speed speed)
{
unsigned int interval = 0;
switch (usb_endpoint_type(epd)) {
case USB_ENDPOINT_XFER_CONTROL:
/* uframes per NAK */
if (speed == USB_SPEED_HIGH)
interval = epd->bInterval;
break;
case USB_ENDPOINT_XFER_ISOC:
interval = 1 << (epd->bInterval - 1);
break;
case USB_ENDPOINT_XFER_BULK:
/* uframes per NAK */
if (speed == USB_SPEED_HIGH && usb_endpoint_dir_out(epd))
interval = epd->bInterval;
break;
case USB_ENDPOINT_XFER_INT:
if (speed >= USB_SPEED_HIGH)
interval = 1 << (epd->bInterval - 1);
else
interval = epd->bInterval;
break;
}
interval *= (speed >= USB_SPEED_HIGH) ? 125 : 1000;
return interval;
}
EXPORT_SYMBOL_GPL(usb_decode_interval);
#ifdef CONFIG_OF
/**
* of_usb_get_dr_mode_by_phy - Get dual role mode for the controller device
* which is associated with the given phy device_node
* @np: Pointer to the given phy device_node
* @arg0: phandle args[0] for phy's with #phy-cells >= 1, or -1 for
* phys which do not have phy-cells
*
* In dts a usb controller associates with phy devices. The function gets
* the string from property 'dr_mode' of the controller associated with the
* given phy device node, and returns the correspondig enum usb_dr_mode.
*/
enum usb_dr_mode of_usb_get_dr_mode_by_phy(struct device_node *np, int arg0)
{
struct device_node *controller = NULL;
struct of_phandle_args args;
const char *dr_mode;
int index;
int err;
do {
controller = of_find_node_with_property(controller, "phys");
if (!of_device_is_available(controller))
continue;
index = 0;
do {
if (arg0 == -1) {
args.np = of_parse_phandle(controller, "phys",
index);
args.args_count = 0;
} else {
err = of_parse_phandle_with_args(controller,
"phys", "#phy-cells",
index, &args);
if (err)
break;
}
of_node_put(args.np);
if (args.np == np && (args.args_count == 0 ||
args.args[0] == arg0))
goto finish;
index++;
} while (args.np);
} while (controller);
finish:
err = of_property_read_string(controller, "dr_mode", &dr_mode);
of_node_put(controller);
if (err < 0)
return USB_DR_MODE_UNKNOWN;
return usb_get_dr_mode_from_string(dr_mode);
}
EXPORT_SYMBOL_GPL(of_usb_get_dr_mode_by_phy);
/**
* of_usb_host_tpl_support - to get if Targeted Peripheral List is supported
* for given targeted hosts (non-PC hosts)
* @np: Pointer to the given device_node
*
* The function gets if the targeted hosts support TPL or not
*/
bool of_usb_host_tpl_support(struct device_node *np)
{
return of_property_read_bool(np, "tpl-support");
}
EXPORT_SYMBOL_GPL(of_usb_host_tpl_support);
/**
* of_usb_update_otg_caps - to update usb otg capabilities according to
* the passed properties in DT.
* @np: Pointer to the given device_node
* @otg_caps: Pointer to the target usb_otg_caps to be set
*
* The function updates the otg capabilities
*/
int of_usb_update_otg_caps(struct device_node *np,
struct usb_otg_caps *otg_caps)
{
u32 otg_rev;
if (!otg_caps)
return -EINVAL;
if (!of_property_read_u32(np, "otg-rev", &otg_rev)) {
switch (otg_rev) {
case 0x0100:
case 0x0120:
case 0x0130:
case 0x0200:
/* Choose the lesser one if it's already been set */
if (otg_caps->otg_rev)
otg_caps->otg_rev = min_t(u16, otg_rev,
otg_caps->otg_rev);
else
otg_caps->otg_rev = otg_rev;
break;
default:
pr_err("%pOF: unsupported otg-rev: 0x%x\n",
np, otg_rev);
return -EINVAL;
}
} else {
/*
* otg-rev is mandatory for otg properties, if not passed
* we set it to be 0 and assume it's a legacy otg device.
* Non-dt platform can set it afterwards.
*/
otg_caps->otg_rev = 0;
}
if (of_property_read_bool(np, "hnp-disable"))
otg_caps->hnp_support = false;
if (of_property_read_bool(np, "srp-disable"))
otg_caps->srp_support = false;
if (of_property_read_bool(np, "adp-disable") ||
(otg_caps->otg_rev < 0x0200))
otg_caps->adp_support = false;
return 0;
}
EXPORT_SYMBOL_GPL(of_usb_update_otg_caps);
/**
* usb_of_get_companion_dev - Find the companion device
* @dev: the device pointer to find a companion
*
* Find the companion device from platform bus.
*
* Takes a reference to the returned struct device which needs to be dropped
* after use.
*
* Return: On success, a pointer to the companion device, %NULL on failure.
*/
struct device *usb_of_get_companion_dev(struct device *dev)
{
struct device_node *node;
struct platform_device *pdev = NULL;
node = of_parse_phandle(dev->of_node, "companion", 0);
if (node)
pdev = of_find_device_by_node(node);
of_node_put(node);
return pdev ? &pdev->dev : NULL;
}
EXPORT_SYMBOL_GPL(usb_of_get_companion_dev);
#endif
struct dentry *usb_debug_root;
EXPORT_SYMBOL_GPL(usb_debug_root);
static int __init usb_common_init(void)
{
usb_debug_root = debugfs_create_dir("usb", NULL);
ledtrig_usb_init();
return 0;
}
static void __exit usb_common_exit(void)
{
ledtrig_usb_exit();
debugfs_remove_recursive(usb_debug_root);
}
subsys_initcall(usb_common_init);
module_exit(usb_common_exit);
MODULE_LICENSE("GPL");
/* SPDX-License-Identifier: GPL-2.0-only */
/* The industrial I/O core
*
* Copyright (c) 2008 Jonathan Cameron
*/
#ifndef _INDUSTRIAL_IO_H_
#define _INDUSTRIAL_IO_H_
#include <linux/device.h>
#include <linux/cdev.h>
#include <linux/cleanup.h>
#include <linux/slab.h>
#include <linux/iio/types.h>
/* IIO TODO LIST */
/*
* Provide means of adjusting timer accuracy.
* Currently assumes nano seconds.
*/
struct fwnode_reference_args;
enum iio_shared_by {
IIO_SEPARATE,
IIO_SHARED_BY_TYPE,
IIO_SHARED_BY_DIR,
IIO_SHARED_BY_ALL
};
enum iio_endian {
IIO_CPU,
IIO_BE,
IIO_LE,
};
struct iio_chan_spec;
struct iio_dev;
/**
* struct iio_chan_spec_ext_info - Extended channel info attribute
* @name: Info attribute name
* @shared: Whether this attribute is shared between all channels.
* @read: Read callback for this info attribute, may be NULL.
* @write: Write callback for this info attribute, may be NULL.
* @private: Data private to the driver.
*/
struct iio_chan_spec_ext_info {
const char *name;
enum iio_shared_by shared;
ssize_t (*read)(struct iio_dev *, uintptr_t private,
struct iio_chan_spec const *, char *buf);
ssize_t (*write)(struct iio_dev *, uintptr_t private,
struct iio_chan_spec const *, const char *buf,
size_t len);
uintptr_t private;
};
/**
* struct iio_enum - Enum channel info attribute
* @items: An array of strings.
* @num_items: Length of the item array.
* @set: Set callback function, may be NULL.
* @get: Get callback function, may be NULL.
*
* The iio_enum struct can be used to implement enum style channel attributes.
* Enum style attributes are those which have a set of strings which map to
* unsigned integer values. The IIO enum helper code takes care of mapping
* between value and string as well as generating a "_available" file which
* contains a list of all available items. The set callback will be called when
* the attribute is updated. The last parameter is the index to the newly
* activated item. The get callback will be used to query the currently active
* item and is supposed to return the index for it.
*/
struct iio_enum {
const char * const *items;
unsigned int num_items;
int (*set)(struct iio_dev *, const struct iio_chan_spec *, unsigned int);
int (*get)(struct iio_dev *, const struct iio_chan_spec *);
};
ssize_t iio_enum_available_read(struct iio_dev *indio_dev,
uintptr_t priv, const struct iio_chan_spec *chan, char *buf);
ssize_t iio_enum_read(struct iio_dev *indio_dev,
uintptr_t priv, const struct iio_chan_spec *chan, char *buf);
ssize_t iio_enum_write(struct iio_dev *indio_dev,
uintptr_t priv, const struct iio_chan_spec *chan, const char *buf,
size_t len);
/**
* IIO_ENUM() - Initialize enum extended channel attribute
* @_name: Attribute name
* @_shared: Whether the attribute is shared between all channels
* @_e: Pointer to an iio_enum struct
*
* This should usually be used together with IIO_ENUM_AVAILABLE()
*/
#define IIO_ENUM(_name, _shared, _e) \
{ \
.name = (_name), \
.shared = (_shared), \
.read = iio_enum_read, \
.write = iio_enum_write, \
.private = (uintptr_t)(_e), \
}
/**
* IIO_ENUM_AVAILABLE() - Initialize enum available extended channel attribute
* @_name: Attribute name ("_available" will be appended to the name)
* @_shared: Whether the attribute is shared between all channels
* @_e: Pointer to an iio_enum struct
*
* Creates a read only attribute which lists all the available enum items in a
* space separated list. This should usually be used together with IIO_ENUM()
*/
#define IIO_ENUM_AVAILABLE(_name, _shared, _e) \
{ \
.name = (_name "_available"), \
.shared = _shared, \
.read = iio_enum_available_read, \
.private = (uintptr_t)(_e), \
}
/**
* struct iio_mount_matrix - iio mounting matrix
* @rotation: 3 dimensional space rotation matrix defining sensor alignment with
* main hardware
*/
struct iio_mount_matrix {
const char *rotation[9];
};
ssize_t iio_show_mount_matrix(struct iio_dev *indio_dev, uintptr_t priv,
const struct iio_chan_spec *chan, char *buf);
int iio_read_mount_matrix(struct device *dev, struct iio_mount_matrix *matrix);
typedef const struct iio_mount_matrix *
(iio_get_mount_matrix_t)(const struct iio_dev *indio_dev,
const struct iio_chan_spec *chan);
/**
* IIO_MOUNT_MATRIX() - Initialize mount matrix extended channel attribute
* @_shared: Whether the attribute is shared between all channels
* @_get: Pointer to an iio_get_mount_matrix_t accessor
*/
#define IIO_MOUNT_MATRIX(_shared, _get) \
{ \
.name = "mount_matrix", \
.shared = (_shared), \
.read = iio_show_mount_matrix, \
.private = (uintptr_t)(_get), \
}
/**
* struct iio_event_spec - specification for a channel event
* @type: Type of the event
* @dir: Direction of the event
* @mask_separate: Bit mask of enum iio_event_info values. Attributes
* set in this mask will be registered per channel.
* @mask_shared_by_type: Bit mask of enum iio_event_info values. Attributes
* set in this mask will be shared by channel type.
* @mask_shared_by_dir: Bit mask of enum iio_event_info values. Attributes
* set in this mask will be shared by channel type and
* direction.
* @mask_shared_by_all: Bit mask of enum iio_event_info values. Attributes
* set in this mask will be shared by all channels.
*/
struct iio_event_spec {
enum iio_event_type type;
enum iio_event_direction dir;
unsigned long mask_separate;
unsigned long mask_shared_by_type;
unsigned long mask_shared_by_dir;
unsigned long mask_shared_by_all;
};
/**
* struct iio_chan_spec - specification of a single channel
* @type: What type of measurement is the channel making.
* @channel: What number do we wish to assign the channel.
* @channel2: If there is a second number for a differential
* channel then this is it. If modified is set then the
* value here specifies the modifier.
* @address: Driver specific identifier.
* @scan_index: Monotonic index to give ordering in scans when read
* from a buffer.
* @scan_type: struct describing the scan type
* @scan_type.sign: 's' or 'u' to specify signed or unsigned
* @scan_type.realbits: Number of valid bits of data
* @scan_type.storagebits: Realbits + padding
* @scan_type.shift: Shift right by this before masking out
* realbits.
* @scan_type.repeat: Number of times real/storage bits repeats.
* When the repeat element is more than 1, then
* the type element in sysfs will show a repeat
* value. Otherwise, the number of repetitions
* is omitted.
* @scan_type.endianness: little or big endian
* @info_mask_separate: What information is to be exported that is specific to
* this channel.
* @info_mask_separate_available: What availability information is to be
* exported that is specific to this channel.
* @info_mask_shared_by_type: What information is to be exported that is shared
* by all channels of the same type.
* @info_mask_shared_by_type_available: What availability information is to be
* exported that is shared by all channels of the same
* type.
* @info_mask_shared_by_dir: What information is to be exported that is shared
* by all channels of the same direction.
* @info_mask_shared_by_dir_available: What availability information is to be
* exported that is shared by all channels of the same
* direction.
* @info_mask_shared_by_all: What information is to be exported that is shared
* by all channels.
* @info_mask_shared_by_all_available: What availability information is to be
* exported that is shared by all channels.
* @event_spec: Array of events which should be registered for this
* channel.
* @num_event_specs: Size of the event_spec array.
* @ext_info: Array of extended info attributes for this channel.
* The array is NULL terminated, the last element should
* have its name field set to NULL.
* @extend_name: Allows labeling of channel attributes with an
* informative name. Note this has no effect codes etc,
* unlike modifiers.
* This field is deprecated in favour of providing
* iio_info->read_label() to override the label, which
* unlike @extend_name does not affect sysfs filenames.
* @datasheet_name: A name used in in-kernel mapping of channels. It should
* correspond to the first name that the channel is referred
* to by in the datasheet (e.g. IND), or the nearest
* possible compound name (e.g. IND-INC).
* @modified: Does a modifier apply to this channel. What these are
* depends on the channel type. Modifier is set in
* channel2. Examples are IIO_MOD_X for axial sensors about
* the 'x' axis.
* @indexed: Specify the channel has a numerical index. If not,
* the channel index number will be suppressed for sysfs
* attributes but not for event codes.
* @output: Channel is output.
* @differential: Channel is differential.
*/
struct iio_chan_spec {
enum iio_chan_type type;
int channel;
int channel2;
unsigned long address;
int scan_index;
struct {
char sign;
u8 realbits;
u8 storagebits;
u8 shift;
u8 repeat;
enum iio_endian endianness;
} scan_type;
long info_mask_separate;
long info_mask_separate_available;
long info_mask_shared_by_type;
long info_mask_shared_by_type_available;
long info_mask_shared_by_dir;
long info_mask_shared_by_dir_available;
long info_mask_shared_by_all;
long info_mask_shared_by_all_available;
const struct iio_event_spec *event_spec;
unsigned int num_event_specs;
const struct iio_chan_spec_ext_info *ext_info;
const char *extend_name;
const char *datasheet_name;
unsigned modified:1;
unsigned indexed:1;
unsigned output:1;
unsigned differential:1;
};
/**
* iio_channel_has_info() - Checks whether a channel supports a info attribute
* @chan: The channel to be queried
* @type: Type of the info attribute to be checked
*
* Returns true if the channels supports reporting values for the given info
* attribute type, false otherwise.
*/
static inline bool iio_channel_has_info(const struct iio_chan_spec *chan,
enum iio_chan_info_enum type)
{
return (chan->info_mask_separate & BIT(type)) |
(chan->info_mask_shared_by_type & BIT(type)) |
(chan->info_mask_shared_by_dir & BIT(type)) |
(chan->info_mask_shared_by_all & BIT(type));
}
/**
* iio_channel_has_available() - Checks if a channel has an available attribute
* @chan: The channel to be queried
* @type: Type of the available attribute to be checked
*
* Returns true if the channel supports reporting available values for the
* given attribute type, false otherwise.
*/
static inline bool iio_channel_has_available(const struct iio_chan_spec *chan,
enum iio_chan_info_enum type)
{
return (chan->info_mask_separate_available & BIT(type)) |
(chan->info_mask_shared_by_type_available & BIT(type)) |
(chan->info_mask_shared_by_dir_available & BIT(type)) |
(chan->info_mask_shared_by_all_available & BIT(type));
}
#define IIO_CHAN_SOFT_TIMESTAMP(_si) { \
.type = IIO_TIMESTAMP, \
.channel = -1, \
.scan_index = _si, \
.scan_type = { \
.sign = 's', \
.realbits = 64, \
.storagebits = 64, \
}, \
}
s64 iio_get_time_ns(const struct iio_dev *indio_dev);
/*
* Device operating modes
* @INDIO_DIRECT_MODE: There is an access to either:
* a) The last single value available for devices that do not provide
* on-demand reads.
* b) A new value after performing an on-demand read otherwise.
* On most devices, this is a single-shot read. On some devices with data
* streams without an 'on-demand' function, this might also be the 'last value'
* feature. Above all, this mode internally means that we are not in any of the
* other modes, and sysfs reads should work.
* Device drivers should inform the core if they support this mode.
* @INDIO_BUFFER_TRIGGERED: Common mode when dealing with kfifo buffers.
* It indicates that an explicit trigger is required. This requests the core to
* attach a poll function when enabling the buffer, which is indicated by the
* _TRIGGERED suffix.
* The core will ensure this mode is set when registering a triggered buffer
* with iio_triggered_buffer_setup().
* @INDIO_BUFFER_SOFTWARE: Another kfifo buffer mode, but not event triggered.
* No poll function can be attached because there is no triggered infrastructure
* we can use to cause capture. There is a kfifo that the driver will fill, but
* not "only one scan at a time". Typically, hardware will have a buffer that
* can hold multiple scans. Software may read one or more scans at a single time
* and push the available data to a Kfifo. This means the core will not attach
* any poll function when enabling the buffer.
* The core will ensure this mode is set when registering a simple kfifo buffer
* with devm_iio_kfifo_buffer_setup().
* @INDIO_BUFFER_HARDWARE: For specific hardware, if unsure do not use this mode.
* Same as above but this time the buffer is not a kfifo where we have direct
* access to the data. Instead, the consumer driver must access the data through
* non software visible channels (or DMA when there is no demux possible in
* software)
* The core will ensure this mode is set when registering a dmaengine buffer
* with devm_iio_dmaengine_buffer_setup().
* @INDIO_EVENT_TRIGGERED: Very unusual mode.
* Triggers usually refer to an external event which will start data capture.
* Here it is kind of the opposite as, a particular state of the data might
* produce an event which can be considered as an event. We don't necessarily
* have access to the data itself, but to the event produced. For example, this
* can be a threshold detector. The internal path of this mode is very close to
* the INDIO_BUFFER_TRIGGERED mode.
* The core will ensure this mode is set when registering a triggered event.
* @INDIO_HARDWARE_TRIGGERED: Very unusual mode.
* Here, triggers can result in data capture and can be routed to multiple
* hardware components, which make them close to regular triggers in the way
* they must be managed by the core, but without the entire interrupts/poll
* functions burden. Interrupts are irrelevant as the data flow is hardware
* mediated and distributed.
*/
#define INDIO_DIRECT_MODE 0x01
#define INDIO_BUFFER_TRIGGERED 0x02
#define INDIO_BUFFER_SOFTWARE 0x04
#define INDIO_BUFFER_HARDWARE 0x08
#define INDIO_EVENT_TRIGGERED 0x10
#define INDIO_HARDWARE_TRIGGERED 0x20
#define INDIO_ALL_BUFFER_MODES \
(INDIO_BUFFER_TRIGGERED | INDIO_BUFFER_HARDWARE | INDIO_BUFFER_SOFTWARE)
#define INDIO_ALL_TRIGGERED_MODES \
(INDIO_BUFFER_TRIGGERED \
| INDIO_EVENT_TRIGGERED \
| INDIO_HARDWARE_TRIGGERED)
#define INDIO_MAX_RAW_ELEMENTS 4
struct iio_val_int_plus_micro {
int integer;
int micro;
};
struct iio_trigger; /* forward declaration */
/**
* struct iio_info - constant information about device
* @event_attrs: event control attributes
* @attrs: general purpose device attributes
* @read_raw: function to request a value from the device.
* mask specifies which value. Note 0 means a reading of
* the channel in question. Return value will specify the
* type of value returned by the device. val and val2 will
* contain the elements making up the returned value.
* @read_raw_multi: function to return values from the device.
* mask specifies which value. Note 0 means a reading of
* the channel in question. Return value will specify the
* type of value returned by the device. vals pointer
* contain the elements making up the returned value.
* max_len specifies maximum number of elements
* vals pointer can contain. val_len is used to return
* length of valid elements in vals.
* @read_avail: function to return the available values from the device.
* mask specifies which value. Note 0 means the available
* values for the channel in question. Return value
* specifies if a IIO_AVAIL_LIST or a IIO_AVAIL_RANGE is
* returned in vals. The type of the vals are returned in
* type and the number of vals is returned in length. For
* ranges, there are always three vals returned; min, step
* and max. For lists, all possible values are enumerated.
* @write_raw: function to write a value to the device.
* Parameters are the same as for read_raw.
* @read_label: function to request label name for a specified label,
* for better channel identification.
* @write_raw_get_fmt: callback function to query the expected
* format/precision. If not set by the driver, write_raw
* returns IIO_VAL_INT_PLUS_MICRO.
* @read_event_config: find out if the event is enabled.
* @write_event_config: set if the event is enabled.
* @read_event_value: read a configuration value associated with the event.
* @write_event_value: write a configuration value for the event.
* @read_event_label: function to request label name for a specified label,
* for better event identification.
* @validate_trigger: function to validate the trigger when the
* current trigger gets changed.
* @update_scan_mode: function to configure device and scan buffer when
* channels have changed
* @debugfs_reg_access: function to read or write register value of device
* @fwnode_xlate: fwnode based function pointer to obtain channel specifier index.
* @hwfifo_set_watermark: function pointer to set the current hardware
* fifo watermark level; see hwfifo_* entries in
* Documentation/ABI/testing/sysfs-bus-iio for details on
* how the hardware fifo operates
* @hwfifo_flush_to_buffer: function pointer to flush the samples stored
* in the hardware fifo to the device buffer. The driver
* should not flush more than count samples. The function
* must return the number of samples flushed, 0 if no
* samples were flushed or a negative integer if no samples
* were flushed and there was an error.
**/
struct iio_info {
const struct attribute_group *event_attrs;
const struct attribute_group *attrs;
int (*read_raw)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
int *val,
int *val2,
long mask);
int (*read_raw_multi)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
int max_len,
int *vals,
int *val_len,
long mask);
int (*read_avail)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
const int **vals,
int *type,
int *length,
long mask);
int (*write_raw)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
int val,
int val2,
long mask);
int (*read_label)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
char *label);
int (*write_raw_get_fmt)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
long mask);
int (*read_event_config)(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan,
enum iio_event_type type,
enum iio_event_direction dir);
int (*write_event_config)(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan,
enum iio_event_type type,
enum iio_event_direction dir,
int state);
int (*read_event_value)(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan,
enum iio_event_type type,
enum iio_event_direction dir,
enum iio_event_info info, int *val, int *val2);
int (*write_event_value)(struct iio_dev *indio_dev,
const struct iio_chan_spec *chan,
enum iio_event_type type,
enum iio_event_direction dir,
enum iio_event_info info, int val, int val2);
int (*read_event_label)(struct iio_dev *indio_dev,
struct iio_chan_spec const *chan,
enum iio_event_type type,
enum iio_event_direction dir,
char *label);
int (*validate_trigger)(struct iio_dev *indio_dev,
struct iio_trigger *trig);
int (*update_scan_mode)(struct iio_dev *indio_dev,
const unsigned long *scan_mask);
int (*debugfs_reg_access)(struct iio_dev *indio_dev,
unsigned reg, unsigned writeval,
unsigned *readval);
int (*fwnode_xlate)(struct iio_dev *indio_dev,
const struct fwnode_reference_args *iiospec);
int (*hwfifo_set_watermark)(struct iio_dev *indio_dev, unsigned val);
int (*hwfifo_flush_to_buffer)(struct iio_dev *indio_dev,
unsigned count);
};
/**
* struct iio_buffer_setup_ops - buffer setup related callbacks
* @preenable: [DRIVER] function to run prior to marking buffer enabled
* @postenable: [DRIVER] function to run after marking buffer enabled
* @predisable: [DRIVER] function to run prior to marking buffer
* disabled
* @postdisable: [DRIVER] function to run after marking buffer disabled
* @validate_scan_mask: [DRIVER] function callback to check whether a given
* scan mask is valid for the device.
*/
struct iio_buffer_setup_ops {
int (*preenable)(struct iio_dev *);
int (*postenable)(struct iio_dev *);
int (*predisable)(struct iio_dev *);
int (*postdisable)(struct iio_dev *);
bool (*validate_scan_mask)(struct iio_dev *indio_dev,
const unsigned long *scan_mask);
};
/**
* struct iio_dev - industrial I/O device
* @modes: [DRIVER] bitmask listing all the operating modes
* supported by the IIO device. This list should be
* initialized before registering the IIO device. It can
* also be filed up by the IIO core, as a result of
* enabling particular features in the driver
* (see iio_triggered_event_setup()).
* @dev: [DRIVER] device structure, should be assigned a parent
* and owner
* @buffer: [DRIVER] any buffer present
* @scan_bytes: [INTERN] num bytes captured to be fed to buffer demux
* @available_scan_masks: [DRIVER] optional array of allowed bitmasks. Sort the
* array in order of preference, the most preferred
* masks first.
* @masklength: [INTERN] the length of the mask established from
* channels
* @active_scan_mask: [INTERN] union of all scan masks requested by buffers
* @scan_timestamp: [INTERN] set if any buffers have requested timestamp
* @trig: [INTERN] current device trigger (buffer modes)
* @pollfunc: [DRIVER] function run on trigger being received
* @pollfunc_event: [DRIVER] function run on events trigger being received
* @channels: [DRIVER] channel specification structure table
* @num_channels: [DRIVER] number of channels specified in @channels.
* @name: [DRIVER] name of the device.
* @label: [DRIVER] unique name to identify which device this is
* @info: [DRIVER] callbacks and constant info from driver
* @setup_ops: [DRIVER] callbacks to call before and after buffer
* enable/disable
* @priv: [DRIVER] reference to driver's private information
* **MUST** be accessed **ONLY** via iio_priv() helper
*/
struct iio_dev {
int modes;
struct device dev;
struct iio_buffer *buffer;
int scan_bytes;
const unsigned long *available_scan_masks;
unsigned masklength;
const unsigned long *active_scan_mask;
bool scan_timestamp;
struct iio_trigger *trig;
struct iio_poll_func *pollfunc;
struct iio_poll_func *pollfunc_event;
struct iio_chan_spec const *channels;
int num_channels;
const char *name;
const char *label;
const struct iio_info *info;
const struct iio_buffer_setup_ops *setup_ops;
void *priv;
};
int iio_device_id(struct iio_dev *indio_dev);
int iio_device_get_current_mode(struct iio_dev *indio_dev);
bool iio_buffer_enabled(struct iio_dev *indio_dev);
const struct iio_chan_spec
*iio_find_channel_from_si(struct iio_dev *indio_dev, int si);
/**
* iio_device_register() - register a device with the IIO subsystem
* @indio_dev: Device structure filled by the device driver
**/
#define iio_device_register(indio_dev) \
__iio_device_register((indio_dev), THIS_MODULE)
int __iio_device_register(struct iio_dev *indio_dev, struct module *this_mod);
void iio_device_unregister(struct iio_dev *indio_dev);
/**
* devm_iio_device_register - Resource-managed iio_device_register()
* @dev: Device to allocate iio_dev for
* @indio_dev: Device structure filled by the device driver
*
* Managed iio_device_register. The IIO device registered with this
* function is automatically unregistered on driver detach. This function
* calls iio_device_register() internally. Refer to that function for more
* information.
*
* RETURNS:
* 0 on success, negative error number on failure.
*/
#define devm_iio_device_register(dev, indio_dev) \
__devm_iio_device_register((dev), (indio_dev), THIS_MODULE)
int __devm_iio_device_register(struct device *dev, struct iio_dev *indio_dev,
struct module *this_mod);
int iio_push_event(struct iio_dev *indio_dev, u64 ev_code, s64 timestamp);
int iio_device_claim_direct_mode(struct iio_dev *indio_dev);
void iio_device_release_direct_mode(struct iio_dev *indio_dev);
/*
* This autocleanup logic is normally used via
* iio_device_claim_direct_scoped().
*/
DEFINE_GUARD(iio_claim_direct, struct iio_dev *, iio_device_claim_direct_mode(_T),
iio_device_release_direct_mode(_T))
DEFINE_GUARD_COND(iio_claim_direct, _try, ({
struct iio_dev *dev;
int d = iio_device_claim_direct_mode(_T);
if (d < 0)
dev = NULL;
else
dev = _T;
dev;
}))
/**
* iio_device_claim_direct_scoped() - Scoped call to iio_device_claim_direct.
* @fail: What to do on failure to claim device.
* @iio_dev: Pointer to the IIO devices structure
*/
#define iio_device_claim_direct_scoped(fail, iio_dev) \
scoped_cond_guard(iio_claim_direct_try, fail, iio_dev)
int iio_device_claim_buffer_mode(struct iio_dev *indio_dev);
void iio_device_release_buffer_mode(struct iio_dev *indio_dev);
extern const struct bus_type iio_bus_type;
/**
* iio_device_put() - reference counted deallocation of struct device
* @indio_dev: IIO device structure containing the device
**/
static inline void iio_device_put(struct iio_dev *indio_dev)
{
if (indio_dev)
put_device(&indio_dev->dev);
}
clockid_t iio_device_get_clock(const struct iio_dev *indio_dev);
int iio_device_set_clock(struct iio_dev *indio_dev, clockid_t clock_id);
/**
* dev_to_iio_dev() - Get IIO device struct from a device struct
* @dev: The device embedded in the IIO device
*
* Note: The device must be a IIO device, otherwise the result is undefined.
*/
static inline struct iio_dev *dev_to_iio_dev(struct device *dev)
{
return container_of(dev, struct iio_dev, dev);
}
/**
* iio_device_get() - increment reference count for the device
* @indio_dev: IIO device structure
*
* Returns: The passed IIO device
**/
static inline struct iio_dev *iio_device_get(struct iio_dev *indio_dev)
{
return indio_dev ? dev_to_iio_dev(get_device(&indio_dev->dev)) : NULL;
}
/**
* iio_device_set_parent() - assign parent device to the IIO device object
* @indio_dev: IIO device structure
* @parent: reference to parent device object
*
* This utility must be called between IIO device allocation
* (via devm_iio_device_alloc()) & IIO device registration
* (via iio_device_register() and devm_iio_device_register())).
* By default, the device allocation will also assign a parent device to
* the IIO device object. In cases where devm_iio_device_alloc() is used,
* sometimes the parent device must be different than the device used to
* manage the allocation.
* In that case, this helper should be used to change the parent, hence the
* requirement to call this between allocation & registration.
**/
static inline void iio_device_set_parent(struct iio_dev *indio_dev,
struct device *parent)
{
indio_dev->dev.parent = parent;
}
/**
* iio_device_set_drvdata() - Set device driver data
* @indio_dev: IIO device structure
* @data: Driver specific data
*
* Allows to attach an arbitrary pointer to an IIO device, which can later be
* retrieved by iio_device_get_drvdata().
*/
static inline void iio_device_set_drvdata(struct iio_dev *indio_dev, void *data)
{
dev_set_drvdata(&indio_dev->dev, data);
}
/**
* iio_device_get_drvdata() - Get device driver data
* @indio_dev: IIO device structure
*
* Returns the data previously set with iio_device_set_drvdata()
*/
static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev)
{
return dev_get_drvdata(&indio_dev->dev);
}
/*
* Used to ensure the iio_priv() structure is aligned to allow that structure
* to in turn include IIO_DMA_MINALIGN'd elements such as buffers which
* must not share cachelines with the rest of the structure, thus making
* them safe for use with non-coherent DMA.
*/
#define IIO_DMA_MINALIGN ARCH_DMA_MINALIGN
struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv);
/* The information at the returned address is guaranteed to be cacheline aligned */
static inline void *iio_priv(const struct iio_dev *indio_dev)
{
return indio_dev->priv;
}
void iio_device_free(struct iio_dev *indio_dev);
struct iio_dev *devm_iio_device_alloc(struct device *parent, int sizeof_priv);
#define devm_iio_trigger_alloc(parent, fmt, ...) \
__devm_iio_trigger_alloc((parent), THIS_MODULE, (fmt), ##__VA_ARGS__)
__printf(3, 4)
struct iio_trigger *__devm_iio_trigger_alloc(struct device *parent,
struct module *this_mod,
const char *fmt, ...);
/**
* iio_get_debugfs_dentry() - helper function to get the debugfs_dentry
* @indio_dev: IIO device structure for device
**/
#if defined(CONFIG_DEBUG_FS)
struct dentry *iio_get_debugfs_dentry(struct iio_dev *indio_dev);
#else
static inline struct dentry *iio_get_debugfs_dentry(struct iio_dev *indio_dev)
{
return NULL;
}
#endif
ssize_t iio_format_value(char *buf, unsigned int type, int size, int *vals);
int iio_str_to_fixpoint(const char *str, int fract_mult, int *integer,
int *fract);
/**
* IIO_DEGREE_TO_RAD() - Convert degree to rad
* @deg: A value in degree
*
* Returns the given value converted from degree to rad
*/
#define IIO_DEGREE_TO_RAD(deg) (((deg) * 314159ULL + 9000000ULL) / 18000000ULL)
/**
* IIO_RAD_TO_DEGREE() - Convert rad to degree
* @rad: A value in rad
*
* Returns the given value converted from rad to degree
*/
#define IIO_RAD_TO_DEGREE(rad) \
(((rad) * 18000000ULL + 314159ULL / 2) / 314159ULL)
/**
* IIO_G_TO_M_S_2() - Convert g to meter / second**2
* @g: A value in g
*
* Returns the given value converted from g to meter / second**2
*/
#define IIO_G_TO_M_S_2(g) ((g) * 980665ULL / 100000ULL)
/**
* IIO_M_S_2_TO_G() - Convert meter / second**2 to g
* @ms2: A value in meter / second**2
*
* Returns the given value converted from meter / second**2 to g
*/
#define IIO_M_S_2_TO_G(ms2) (((ms2) * 100000ULL + 980665ULL / 2) / 980665ULL)
#endif /* _INDUSTRIAL_IO_H_ */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_HIGHMEM_H
#define _LINUX_HIGHMEM_H
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/cacheflush.h>
#include <linux/kmsan.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
#include "highmem-internal.h"
/**
* kmap - Map a page for long term usage
* @page: Pointer to the page to be mapped
*
* Returns: The virtual address of the mapping
*
* Can only be invoked from preemptible task context because on 32bit
* systems with CONFIG_HIGHMEM enabled this function might sleep.
*
* For systems with CONFIG_HIGHMEM=n and for pages in the low memory area
* this returns the virtual address of the direct kernel mapping.
*
* The returned virtual address is globally visible and valid up to the
* point where it is unmapped via kunmap(). The pointer can be handed to
* other contexts.
*
* For highmem pages on 32bit systems this can be slow as the mapping space
* is limited and protected by a global lock. In case that there is no
* mapping slot available the function blocks until a slot is released via
* kunmap().
*/
static inline void *kmap(struct page *page);
/**
* kunmap - Unmap the virtual address mapped by kmap()
* @page: Pointer to the page which was mapped by kmap()
*
* Counterpart to kmap(). A NOOP for CONFIG_HIGHMEM=n and for mappings of
* pages in the low memory area.
*/
static inline void kunmap(struct page *page);
/**
* kmap_to_page - Get the page for a kmap'ed address
* @addr: The address to look up
*
* Returns: The page which is mapped to @addr.
*/
static inline struct page *kmap_to_page(void *addr);
/**
* kmap_flush_unused - Flush all unused kmap mappings in order to
* remove stray mappings
*/
static inline void kmap_flush_unused(void);
/**
* kmap_local_page - Map a page for temporary usage
* @page: Pointer to the page to be mapped
*
* Returns: The virtual address of the mapping
*
* Can be invoked from any context, including interrupts.
*
* Requires careful handling when nesting multiple mappings because the map
* management is stack based. The unmap has to be in the reverse order of
* the map operation:
*
* addr1 = kmap_local_page(page1);
* addr2 = kmap_local_page(page2);
* ...
* kunmap_local(addr2);
* kunmap_local(addr1);
*
* Unmapping addr1 before addr2 is invalid and causes malfunction.
*
* Contrary to kmap() mappings the mapping is only valid in the context of
* the caller and cannot be handed to other contexts.
*
* On CONFIG_HIGHMEM=n kernels and for low memory pages this returns the
* virtual address of the direct mapping. Only real highmem pages are
* temporarily mapped.
*
* While kmap_local_page() is significantly faster than kmap() for the highmem
* case it comes with restrictions about the pointer validity.
*
* On HIGHMEM enabled systems mapping a highmem page has the side effect of
* disabling migration in order to keep the virtual address stable across
* preemption. No caller of kmap_local_page() can rely on this side effect.
*/
static inline void *kmap_local_page(struct page *page);
/**
* kmap_local_folio - Map a page in this folio for temporary usage
* @folio: The folio containing the page.
* @offset: The byte offset within the folio which identifies the page.
*
* Requires careful handling when nesting multiple mappings because the map
* management is stack based. The unmap has to be in the reverse order of
* the map operation::
*
* addr1 = kmap_local_folio(folio1, offset1);
* addr2 = kmap_local_folio(folio2, offset2);
* ...
* kunmap_local(addr2);
* kunmap_local(addr1);
*
* Unmapping addr1 before addr2 is invalid and causes malfunction.
*
* Contrary to kmap() mappings the mapping is only valid in the context of
* the caller and cannot be handed to other contexts.
*
* On CONFIG_HIGHMEM=n kernels and for low memory pages this returns the
* virtual address of the direct mapping. Only real highmem pages are
* temporarily mapped.
*
* While it is significantly faster than kmap() for the highmem case it
* comes with restrictions about the pointer validity.
*
* On HIGHMEM enabled systems mapping a highmem page has the side effect of
* disabling migration in order to keep the virtual address stable across
* preemption. No caller of kmap_local_folio() can rely on this side effect.
*
* Context: Can be invoked from any context.
* Return: The virtual address of @offset.
*/
static inline void *kmap_local_folio(struct folio *folio, size_t offset);
/**
* kmap_atomic - Atomically map a page for temporary usage - Deprecated!
* @page: Pointer to the page to be mapped
*
* Returns: The virtual address of the mapping
*
* In fact a wrapper around kmap_local_page() which also disables pagefaults
* and, depending on PREEMPT_RT configuration, also CPU migration and
* preemption. Therefore users should not count on the latter two side effects.
*
* Mappings should always be released by kunmap_atomic().
*
* Do not use in new code. Use kmap_local_page() instead.
*
* It is used in atomic context when code wants to access the contents of a
* page that might be allocated from high memory (see __GFP_HIGHMEM), for
* example a page in the pagecache. The API has two functions, and they
* can be used in a manner similar to the following::
*
* // Find the page of interest.
* struct page *page = find_get_page(mapping, offset);
*
* // Gain access to the contents of that page.
* void *vaddr = kmap_atomic(page);
*
* // Do something to the contents of that page.
* memset(vaddr, 0, PAGE_SIZE);
*
* // Unmap that page.
* kunmap_atomic(vaddr);
*
* Note that the kunmap_atomic() call takes the result of the kmap_atomic()
* call, not the argument.
*
* If you need to map two pages because you want to copy from one page to
* another you need to keep the kmap_atomic calls strictly nested, like:
*
* vaddr1 = kmap_atomic(page1);
* vaddr2 = kmap_atomic(page2);
*
* memcpy(vaddr1, vaddr2, PAGE_SIZE);
*
* kunmap_atomic(vaddr2);
* kunmap_atomic(vaddr1);
*/
static inline void *kmap_atomic(struct page *page);
/* Highmem related interfaces for management code */
static inline unsigned int nr_free_highpages(void);
static inline unsigned long totalhigh_pages(void);
#ifndef ARCH_HAS_FLUSH_ANON_PAGE
static inline void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
{
}
#endif
#ifndef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE
static inline void flush_kernel_vmap_range(void *vaddr, int size)
{
}
static inline void invalidate_kernel_vmap_range(void *vaddr, int size)
{
}
#endif
/* when CONFIG_HIGHMEM is not set these will be plain clear/copy_page */
#ifndef clear_user_highpage
static inline void clear_user_highpage(struct page *page, unsigned long vaddr)
{
void *addr = kmap_local_page(page);
clear_user_page(addr, vaddr, page);
kunmap_local(addr);
}
#endif
#ifndef vma_alloc_zeroed_movable_folio
/**
* vma_alloc_zeroed_movable_folio - Allocate a zeroed page for a VMA.
* @vma: The VMA the page is to be allocated for.
* @vaddr: The virtual address the page will be inserted into.
*
* This function will allocate a page suitable for inserting into this
* VMA at this virtual address. It may be allocated from highmem or
* the movable zone. An architecture may provide its own implementation.
*
* Return: A folio containing one allocated and zeroed page or NULL if
* we are out of memory.
*/
static inline
struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
unsigned long vaddr)
{
struct folio *folio;
folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, vaddr, false);
if (folio)
clear_user_highpage(&folio->page, vaddr);
return folio;
}
#endif
static inline void clear_highpage(struct page *page)
{
void *kaddr = kmap_local_page(page);
clear_page(kaddr);
kunmap_local(kaddr);
}
static inline void clear_highpage_kasan_tagged(struct page *page)
{
void *kaddr = kmap_local_page(page);
clear_page(kasan_reset_tag(kaddr));
kunmap_local(kaddr);
}
#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE
static inline void tag_clear_highpage(struct page *page)
{
}
#endif
/*
* If we pass in a base or tail page, we can zero up to PAGE_SIZE.
* If we pass in a head page, we can zero up to the size of the compound page.
*/
#ifdef CONFIG_HIGHMEM
void zero_user_segments(struct page *page, unsigned start1, unsigned end1,
unsigned start2, unsigned end2);
#else
static inline void zero_user_segments(struct page *page,
unsigned start1, unsigned end1,
unsigned start2, unsigned end2)
{
void *kaddr = kmap_local_page(page);
unsigned int i;
BUG_ON(end1 > page_size(page) || end2 > page_size(page)); if (end1 > start1) memset(kaddr + start1, 0, end1 - start1); if (end2 > start2) memset(kaddr + start2, 0, end2 - start2);
kunmap_local(kaddr);
for (i = 0; i < compound_nr(page); i++)
flush_dcache_page(page + i);
}
#endif
static inline void zero_user_segment(struct page *page,
unsigned start, unsigned end)
{
zero_user_segments(page, start, end, 0, 0);
}
static inline void zero_user(struct page *page,
unsigned start, unsigned size)
{
zero_user_segments(page, start, start + size, 0, 0);
}
#ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE
static inline void copy_user_highpage(struct page *to, struct page *from,
unsigned long vaddr, struct vm_area_struct *vma)
{
char *vfrom, *vto;
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
copy_user_page(vto, vfrom, vaddr, to);
kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
kunmap_local(vto);
kunmap_local(vfrom);
}
#endif
#ifndef __HAVE_ARCH_COPY_HIGHPAGE
static inline void copy_highpage(struct page *to, struct page *from)
{
char *vfrom, *vto;
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
copy_page(vto, vfrom);
kmsan_copy_page_meta(to, from);
kunmap_local(vto);
kunmap_local(vfrom);
}
#endif
#ifdef copy_mc_to_kernel
/*
* If architecture supports machine check exception handling, define the
* #MC versions of copy_user_highpage and copy_highpage. They copy a memory
* page with #MC in source page (@from) handled, and return the number
* of bytes not copied if there was a #MC, otherwise 0 for success.
*/
static inline int copy_mc_user_highpage(struct page *to, struct page *from,
unsigned long vaddr, struct vm_area_struct *vma)
{
unsigned long ret;
char *vfrom, *vto;
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
ret = copy_mc_to_kernel(vto, vfrom, PAGE_SIZE);
if (!ret)
kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
kunmap_local(vto);
kunmap_local(vfrom);
return ret;
}
static inline int copy_mc_highpage(struct page *to, struct page *from)
{
unsigned long ret;
char *vfrom, *vto;
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
ret = copy_mc_to_kernel(vto, vfrom, PAGE_SIZE);
if (!ret)
kmsan_copy_page_meta(to, from);
kunmap_local(vto);
kunmap_local(vfrom);
return ret;
}
#else
static inline int copy_mc_user_highpage(struct page *to, struct page *from,
unsigned long vaddr, struct vm_area_struct *vma)
{
copy_user_highpage(to, from, vaddr, vma);
return 0;
}
static inline int copy_mc_highpage(struct page *to, struct page *from)
{
copy_highpage(to, from);
return 0;
}
#endif
static inline void memcpy_page(struct page *dst_page, size_t dst_off,
struct page *src_page, size_t src_off,
size_t len)
{
char *dst = kmap_local_page(dst_page);
char *src = kmap_local_page(src_page);
VM_BUG_ON(dst_off + len > PAGE_SIZE || src_off + len > PAGE_SIZE);
memcpy(dst + dst_off, src + src_off, len);
kunmap_local(src);
kunmap_local(dst);
}
static inline void memset_page(struct page *page, size_t offset, int val,
size_t len)
{
char *addr = kmap_local_page(page);
VM_BUG_ON(offset + len > PAGE_SIZE);
memset(addr + offset, val, len);
kunmap_local(addr);
}
static inline void memcpy_from_page(char *to, struct page *page,
size_t offset, size_t len)
{
char *from = kmap_local_page(page);
VM_BUG_ON(offset + len > PAGE_SIZE);
memcpy(to, from + offset, len);
kunmap_local(from);
}
static inline void memcpy_to_page(struct page *page, size_t offset,
const char *from, size_t len)
{
char *to = kmap_local_page(page);
VM_BUG_ON(offset + len > PAGE_SIZE);
memcpy(to + offset, from, len);
flush_dcache_page(page);
kunmap_local(to);
}
static inline void memzero_page(struct page *page, size_t offset, size_t len)
{
char *addr = kmap_local_page(page);
VM_BUG_ON(offset + len > PAGE_SIZE);
memset(addr + offset, 0, len);
flush_dcache_page(page);
kunmap_local(addr);
}
/**
* memcpy_from_folio - Copy a range of bytes from a folio.
* @to: The memory to copy to.
* @folio: The folio to read from.
* @offset: The first byte in the folio to read.
* @len: The number of bytes to copy.
*/
static inline void memcpy_from_folio(char *to, struct folio *folio,
size_t offset, size_t len)
{
VM_BUG_ON(offset + len > folio_size(folio));
do {
const char *from = kmap_local_folio(folio, offset);
size_t chunk = len;
if (folio_test_highmem(folio) &&
chunk > PAGE_SIZE - offset_in_page(offset))
chunk = PAGE_SIZE - offset_in_page(offset);
memcpy(to, from, chunk);
kunmap_local(from);
to += chunk;
offset += chunk;
len -= chunk;
} while (len > 0);
}
/**
* memcpy_to_folio - Copy a range of bytes to a folio.
* @folio: The folio to write to.
* @offset: The first byte in the folio to store to.
* @from: The memory to copy from.
* @len: The number of bytes to copy.
*/
static inline void memcpy_to_folio(struct folio *folio, size_t offset,
const char *from, size_t len)
{
VM_BUG_ON(offset + len > folio_size(folio));
do {
char *to = kmap_local_folio(folio, offset);
size_t chunk = len;
if (folio_test_highmem(folio) &&
chunk > PAGE_SIZE - offset_in_page(offset))
chunk = PAGE_SIZE - offset_in_page(offset);
memcpy(to, from, chunk);
kunmap_local(to);
from += chunk;
offset += chunk;
len -= chunk;
} while (len > 0);
flush_dcache_folio(folio);
}
/**
* folio_zero_tail - Zero the tail of a folio.
* @folio: The folio to zero.
* @offset: The byte offset in the folio to start zeroing at.
* @kaddr: The address the folio is currently mapped to.
*
* If you have already used kmap_local_folio() to map a folio, written
* some data to it and now need to zero the end of the folio (and flush
* the dcache), you can use this function. If you do not have the
* folio kmapped (eg the folio has been partially populated by DMA),
* use folio_zero_range() or folio_zero_segment() instead.
*
* Return: An address which can be passed to kunmap_local().
*/
static inline __must_check void *folio_zero_tail(struct folio *folio,
size_t offset, void *kaddr)
{
size_t len = folio_size(folio) - offset;
if (folio_test_highmem(folio)) {
size_t max = PAGE_SIZE - offset_in_page(offset);
while (len > max) {
memset(kaddr, 0, max);
kunmap_local(kaddr);
len -= max;
offset += max;
max = PAGE_SIZE;
kaddr = kmap_local_folio(folio, offset);
}
}
memset(kaddr, 0, len);
flush_dcache_folio(folio);
return kaddr;
}
/**
* folio_fill_tail - Copy some data to a folio and pad with zeroes.
* @folio: The destination folio.
* @offset: The offset into @folio at which to start copying.
* @from: The data to copy.
* @len: How many bytes of data to copy.
*
* This function is most useful for filesystems which support inline data.
* When they want to copy data from the inode into the page cache, this
* function does everything for them. It supports large folios even on
* HIGHMEM configurations.
*/
static inline void folio_fill_tail(struct folio *folio, size_t offset,
const char *from, size_t len)
{
char *to = kmap_local_folio(folio, offset);
VM_BUG_ON(offset + len > folio_size(folio));
if (folio_test_highmem(folio)) {
size_t max = PAGE_SIZE - offset_in_page(offset);
while (len > max) {
memcpy(to, from, max);
kunmap_local(to);
len -= max;
from += max;
offset += max;
max = PAGE_SIZE;
to = kmap_local_folio(folio, offset);
}
}
memcpy(to, from, len);
to = folio_zero_tail(folio, offset + len, to + len);
kunmap_local(to);
}
/**
* memcpy_from_file_folio - Copy some bytes from a file folio.
* @to: The destination buffer.
* @folio: The folio to copy from.
* @pos: The position in the file.
* @len: The maximum number of bytes to copy.
*
* Copy up to @len bytes from this folio. This may be limited by PAGE_SIZE
* if the folio comes from HIGHMEM, and by the size of the folio.
*
* Return: The number of bytes copied from the folio.
*/
static inline size_t memcpy_from_file_folio(char *to, struct folio *folio,
loff_t pos, size_t len)
{
size_t offset = offset_in_folio(folio, pos);
char *from = kmap_local_folio(folio, offset);
if (folio_test_highmem(folio)) {
offset = offset_in_page(offset);
len = min_t(size_t, len, PAGE_SIZE - offset);
} else
len = min(len, folio_size(folio) - offset);
memcpy(to, from, len);
kunmap_local(from);
return len;
}
/**
* folio_zero_segments() - Zero two byte ranges in a folio.
* @folio: The folio to write to.
* @start1: The first byte to zero.
* @xend1: One more than the last byte in the first range.
* @start2: The first byte to zero in the second range.
* @xend2: One more than the last byte in the second range.
*/
static inline void folio_zero_segments(struct folio *folio,
size_t start1, size_t xend1, size_t start2, size_t xend2)
{
zero_user_segments(&folio->page, start1, xend1, start2, xend2);
}
/**
* folio_zero_segment() - Zero a byte range in a folio.
* @folio: The folio to write to.
* @start: The first byte to zero.
* @xend: One more than the last byte to zero.
*/
static inline void folio_zero_segment(struct folio *folio,
size_t start, size_t xend)
{
zero_user_segments(&folio->page, start, xend, 0, 0);
}
/**
* folio_zero_range() - Zero a byte range in a folio.
* @folio: The folio to write to.
* @start: The first byte to zero.
* @length: The number of bytes to zero.
*/
static inline void folio_zero_range(struct folio *folio,
size_t start, size_t length)
{
zero_user_segments(&folio->page, start, start + length, 0, 0);
}
/**
* folio_release_kmap - Unmap a folio and drop a refcount.
* @folio: The folio to release.
* @addr: The address previously returned by a call to kmap_local_folio().
*
* It is common, eg in directory handling to kmap a folio. This function
* unmaps the folio and drops the refcount that was being held to keep the
* folio alive while we accessed it.
*/
static inline void folio_release_kmap(struct folio *folio, void *addr)
{
kunmap_local(addr);
folio_put(folio);
}
static inline void unmap_and_put_page(struct page *page, void *addr)
{
folio_release_kmap(page_folio(page), addr);
}
#endif /* _LINUX_HIGHMEM_H */
#ifndef _LINUX_JHASH_H
#define _LINUX_JHASH_H
/* jhash.h: Jenkins hash support.
*
* Copyright (C) 2006. Bob Jenkins (bob_jenkins@burtleburtle.net)
*
* https://burtleburtle.net/bob/hash/
*
* These are the credits from Bob's sources:
*
* lookup3.c, by Bob Jenkins, May 2006, Public Domain.
*
* These are functions for producing 32-bit hashes for hash table lookup.
* hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final()
* are externally useful functions. Routines to test the hash are included
* if SELF_TEST is defined. You can use this free for any purpose. It's in
* the public domain. It has no warranty.
*
* Copyright (C) 2009-2010 Jozsef Kadlecsik (kadlec@netfilter.org)
*
* I've modified Bob's hash to be useful in the Linux kernel, and
* any bugs present are my fault.
* Jozsef
*/
#include <linux/bitops.h>
#include <linux/unaligned/packed_struct.h>
/* Best hash sizes are of power of two */
#define jhash_size(n) ((u32)1<<(n))
/* Mask the hash value, i.e (value & jhash_mask(n)) instead of (value % n) */
#define jhash_mask(n) (jhash_size(n)-1)
/* __jhash_mix -- mix 3 32-bit values reversibly. */
#define __jhash_mix(a, b, c) \
{ \
a -= c; a ^= rol32(c, 4); c += b; \
b -= a; b ^= rol32(a, 6); a += c; \
c -= b; c ^= rol32(b, 8); b += a; \
a -= c; a ^= rol32(c, 16); c += b; \
b -= a; b ^= rol32(a, 19); a += c; \
c -= b; c ^= rol32(b, 4); b += a; \
}
/* __jhash_final - final mixing of 3 32-bit values (a,b,c) into c */
#define __jhash_final(a, b, c) \
{ \
c ^= b; c -= rol32(b, 14); \
a ^= c; a -= rol32(c, 11); \
b ^= a; b -= rol32(a, 25); \
c ^= b; c -= rol32(b, 16); \
a ^= c; a -= rol32(c, 4); \
b ^= a; b -= rol32(a, 14); \
c ^= b; c -= rol32(b, 24); \
}
/* An arbitrary initial parameter */
#define JHASH_INITVAL 0xdeadbeef
/* jhash - hash an arbitrary key
* @k: sequence of bytes as key
* @length: the length of the key
* @initval: the previous hash, or an arbitray value
*
* The generic version, hashes an arbitrary sequence of bytes.
* No alignment or length assumptions are made about the input key.
*
* Returns the hash value of the key. The result depends on endianness.
*/
static inline u32 jhash(const void *key, u32 length, u32 initval)
{
u32 a, b, c;
const u8 *k = key;
/* Set up the internal state */
a = b = c = JHASH_INITVAL + length + initval;
/* All but the last block: affect some 32 bits of (a,b,c) */
while (length > 12) {
a += __get_unaligned_cpu32(k);
b += __get_unaligned_cpu32(k + 4);
c += __get_unaligned_cpu32(k + 8);
__jhash_mix(a, b, c);
length -= 12;
k += 12;
}
/* Last block: affect all 32 bits of (c) */
switch (length) {
case 12: c += (u32)k[11]<<24; fallthrough;
case 11: c += (u32)k[10]<<16; fallthrough;
case 10: c += (u32)k[9]<<8; fallthrough;
case 9: c += k[8]; fallthrough;
case 8: b += (u32)k[7]<<24; fallthrough;
case 7: b += (u32)k[6]<<16; fallthrough;
case 6: b += (u32)k[5]<<8; fallthrough;
case 5: b += k[4]; fallthrough;
case 4: a += (u32)k[3]<<24; fallthrough;
case 3: a += (u32)k[2]<<16; fallthrough;
case 2: a += (u32)k[1]<<8; fallthrough;
case 1: a += k[0];
__jhash_final(a, b, c);
break;
case 0: /* Nothing left to add */
break;
}
return c;
}
/* jhash2 - hash an array of u32's
* @k: the key which must be an array of u32's
* @length: the number of u32's in the key
* @initval: the previous hash, or an arbitray value
*
* Returns the hash value of the key.
*/
static inline u32 jhash2(const u32 *k, u32 length, u32 initval)
{
u32 a, b, c;
/* Set up the internal state */
a = b = c = JHASH_INITVAL + (length<<2) + initval;
/* Handle most of the key */
while (length > 3) {
a += k[0];
b += k[1];
c += k[2];
__jhash_mix(a, b, c);
length -= 3;
k += 3;
}
/* Handle the last 3 u32's */
switch (length) {
case 3: c += k[2]; fallthrough;
case 2: b += k[1]; fallthrough;
case 1: a += k[0];
__jhash_final(a, b, c);
break;
case 0: /* Nothing left to add */
break;
}
return c;
}
/* __jhash_nwords - hash exactly 3, 2 or 1 word(s) */
static inline u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
{
a += initval;
b += initval;
c += initval;
__jhash_final(a, b, c);
return c;
}
static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
{
return __jhash_nwords(a, b, c, initval + JHASH_INITVAL + (3 << 2));
}
static inline u32 jhash_2words(u32 a, u32 b, u32 initval)
{
return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
}
static inline u32 jhash_1word(u32 a, u32 initval)
{
return __jhash_nwords(a, 0, 0, initval + JHASH_INITVAL + (1 << 2));
}
#endif /* _LINUX_JHASH_H */
/* SPDX-License-Identifier: GPL-2.0 */
/* interrupt.h */
#ifndef _LINUX_INTERRUPT_H
#define _LINUX_INTERRUPT_H
#include <linux/kernel.h>
#include <linux/bitops.h>
#include <linux/cpumask.h>
#include <linux/irqreturn.h>
#include <linux/irqnr.h>
#include <linux/hardirq.h>
#include <linux/irqflags.h>
#include <linux/hrtimer.h>
#include <linux/kref.h>
#include <linux/workqueue.h>
#include <linux/jump_label.h>
#include <linux/atomic.h>
#include <asm/ptrace.h>
#include <asm/irq.h>
#include <asm/sections.h>
/*
* These correspond to the IORESOURCE_IRQ_* defines in
* linux/ioport.h to select the interrupt line behaviour. When
* requesting an interrupt without specifying a IRQF_TRIGGER, the
* setting should be assumed to be "as already configured", which
* may be as per machine or firmware initialisation.
*/
#define IRQF_TRIGGER_NONE 0x00000000
#define IRQF_TRIGGER_RISING 0x00000001
#define IRQF_TRIGGER_FALLING 0x00000002
#define IRQF_TRIGGER_HIGH 0x00000004
#define IRQF_TRIGGER_LOW 0x00000008
#define IRQF_TRIGGER_MASK (IRQF_TRIGGER_HIGH | IRQF_TRIGGER_LOW | \
IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING)
#define IRQF_TRIGGER_PROBE 0x00000010
/*
* These flags used only by the kernel as part of the
* irq handling routines.
*
* IRQF_SHARED - allow sharing the irq among several devices
* IRQF_PROBE_SHARED - set by callers when they expect sharing mismatches to occur
* IRQF_TIMER - Flag to mark this interrupt as timer interrupt
* IRQF_PERCPU - Interrupt is per cpu
* IRQF_NOBALANCING - Flag to exclude this interrupt from irq balancing
* IRQF_IRQPOLL - Interrupt is used for polling (only the interrupt that is
* registered first in a shared interrupt is considered for
* performance reasons)
* IRQF_ONESHOT - Interrupt is not reenabled after the hardirq handler finished.
* Used by threaded interrupts which need to keep the
* irq line disabled until the threaded handler has been run.
* IRQF_NO_SUSPEND - Do not disable this IRQ during suspend. Does not guarantee
* that this interrupt will wake the system from a suspended
* state. See Documentation/power/suspend-and-interrupts.rst
* IRQF_FORCE_RESUME - Force enable it on resume even if IRQF_NO_SUSPEND is set
* IRQF_NO_THREAD - Interrupt cannot be threaded
* IRQF_EARLY_RESUME - Resume IRQ early during syscore instead of at device
* resume time.
* IRQF_COND_SUSPEND - If the IRQ is shared with a NO_SUSPEND user, execute this
* interrupt handler after suspending interrupts. For system
* wakeup devices users need to implement wakeup detection in
* their interrupt handlers.
* IRQF_NO_AUTOEN - Don't enable IRQ or NMI automatically when users request it.
* Users will enable it explicitly by enable_irq() or enable_nmi()
* later.
* IRQF_NO_DEBUG - Exclude from runnaway detection for IPI and similar handlers,
* depends on IRQF_PERCPU.
* IRQF_COND_ONESHOT - Agree to do IRQF_ONESHOT if already set for a shared
* interrupt.
*/
#define IRQF_SHARED 0x00000080
#define IRQF_PROBE_SHARED 0x00000100
#define __IRQF_TIMER 0x00000200
#define IRQF_PERCPU 0x00000400
#define IRQF_NOBALANCING 0x00000800
#define IRQF_IRQPOLL 0x00001000
#define IRQF_ONESHOT 0x00002000
#define IRQF_NO_SUSPEND 0x00004000
#define IRQF_FORCE_RESUME 0x00008000
#define IRQF_NO_THREAD 0x00010000
#define IRQF_EARLY_RESUME 0x00020000
#define IRQF_COND_SUSPEND 0x00040000
#define IRQF_NO_AUTOEN 0x00080000
#define IRQF_NO_DEBUG 0x00100000
#define IRQF_COND_ONESHOT 0x00200000
#define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD)
/*
* These values can be returned by request_any_context_irq() and
* describe the context the interrupt will be run in.
*
* IRQC_IS_HARDIRQ - interrupt runs in hardirq context
* IRQC_IS_NESTED - interrupt runs in a nested threaded context
*/
enum {
IRQC_IS_HARDIRQ = 0,
IRQC_IS_NESTED,
};
typedef irqreturn_t (*irq_handler_t)(int, void *);
/**
* struct irqaction - per interrupt action descriptor
* @handler: interrupt handler function
* @name: name of the device
* @dev_id: cookie to identify the device
* @percpu_dev_id: cookie to identify the device
* @next: pointer to the next irqaction for shared interrupts
* @irq: interrupt number
* @flags: flags (see IRQF_* above)
* @thread_fn: interrupt handler function for threaded interrupts
* @thread: thread pointer for threaded interrupts
* @secondary: pointer to secondary irqaction (force threading)
* @thread_flags: flags related to @thread
* @thread_mask: bitmask for keeping track of @thread activity
* @dir: pointer to the proc/irq/NN/name entry
*/
struct irqaction {
irq_handler_t handler;
void *dev_id;
void __percpu *percpu_dev_id;
struct irqaction *next;
irq_handler_t thread_fn;
struct task_struct *thread;
struct irqaction *secondary;
unsigned int irq;
unsigned int flags;
unsigned long thread_flags;
unsigned long thread_mask;
const char *name;
struct proc_dir_entry *dir;
} ____cacheline_internodealigned_in_smp;
extern irqreturn_t no_action(int cpl, void *dev_id);
/*
* If a (PCI) device interrupt is not connected we set dev->irq to
* IRQ_NOTCONNECTED. This causes request_irq() to fail with -ENOTCONN, so we
* can distingiush that case from other error returns.
*
* 0x80000000 is guaranteed to be outside the available range of interrupts
* and easy to distinguish from other possible incorrect values.
*/
#define IRQ_NOTCONNECTED (1U << 31)
extern int __must_check
request_threaded_irq(unsigned int irq, irq_handler_t handler,
irq_handler_t thread_fn,
unsigned long flags, const char *name, void *dev);
/**
* request_irq - Add a handler for an interrupt line
* @irq: The interrupt line to allocate
* @handler: Function to be called when the IRQ occurs.
* Primary handler for threaded interrupts
* If NULL, the default primary handler is installed
* @flags: Handling flags
* @name: Name of the device generating this interrupt
* @dev: A cookie passed to the handler function
*
* This call allocates an interrupt and establishes a handler; see
* the documentation for request_threaded_irq() for details.
*/
static inline int __must_check
request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags,
const char *name, void *dev)
{
return request_threaded_irq(irq, handler, NULL, flags, name, dev);
}
extern int __must_check
request_any_context_irq(unsigned int irq, irq_handler_t handler,
unsigned long flags, const char *name, void *dev_id);
extern int __must_check
__request_percpu_irq(unsigned int irq, irq_handler_t handler,
unsigned long flags, const char *devname,
void __percpu *percpu_dev_id);
extern int __must_check
request_nmi(unsigned int irq, irq_handler_t handler, unsigned long flags,
const char *name, void *dev);
static inline int __must_check
request_percpu_irq(unsigned int irq, irq_handler_t handler,
const char *devname, void __percpu *percpu_dev_id)
{
return __request_percpu_irq(irq, handler, 0,
devname, percpu_dev_id);
}
extern int __must_check
request_percpu_nmi(unsigned int irq, irq_handler_t handler,
const char *devname, void __percpu *dev);
extern const void *free_irq(unsigned int, void *);
extern void free_percpu_irq(unsigned int, void __percpu *);
extern const void *free_nmi(unsigned int irq, void *dev_id);
extern void free_percpu_nmi(unsigned int irq, void __percpu *percpu_dev_id);
struct device;
extern int __must_check
devm_request_threaded_irq(struct device *dev, unsigned int irq,
irq_handler_t handler, irq_handler_t thread_fn,
unsigned long irqflags, const char *devname,
void *dev_id);
static inline int __must_check
devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler,
unsigned long irqflags, const char *devname, void *dev_id)
{
return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags,
devname, dev_id);
}
extern int __must_check
devm_request_any_context_irq(struct device *dev, unsigned int irq,
irq_handler_t handler, unsigned long irqflags,
const char *devname, void *dev_id);
extern void devm_free_irq(struct device *dev, unsigned int irq, void *dev_id);
bool irq_has_action(unsigned int irq);
extern void disable_irq_nosync(unsigned int irq);
extern bool disable_hardirq(unsigned int irq);
extern void disable_irq(unsigned int irq);
extern void disable_percpu_irq(unsigned int irq);
extern void enable_irq(unsigned int irq);
extern void enable_percpu_irq(unsigned int irq, unsigned int type);
extern bool irq_percpu_is_enabled(unsigned int irq);
extern void irq_wake_thread(unsigned int irq, void *dev_id);
extern void disable_nmi_nosync(unsigned int irq);
extern void disable_percpu_nmi(unsigned int irq);
extern void enable_nmi(unsigned int irq);
extern void enable_percpu_nmi(unsigned int irq, unsigned int type);
extern int prepare_percpu_nmi(unsigned int irq);
extern void teardown_percpu_nmi(unsigned int irq);
extern int irq_inject_interrupt(unsigned int irq);
/* The following three functions are for the core kernel use only. */
extern void suspend_device_irqs(void);
extern void resume_device_irqs(void);
extern void rearm_wake_irq(unsigned int irq);
/**
* struct irq_affinity_notify - context for notification of IRQ affinity changes
* @irq: Interrupt to which notification applies
* @kref: Reference count, for internal use
* @work: Work item, for internal use
* @notify: Function to be called on change. This will be
* called in process context.
* @release: Function to be called on release. This will be
* called in process context. Once registered, the
* structure must only be freed when this function is
* called or later.
*/
struct irq_affinity_notify {
unsigned int irq;
struct kref kref;
struct work_struct work;
void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask);
void (*release)(struct kref *ref);
};
#define IRQ_AFFINITY_MAX_SETS 4
/**
* struct irq_affinity - Description for automatic irq affinity assignements
* @pre_vectors: Don't apply affinity to @pre_vectors at beginning of
* the MSI(-X) vector space
* @post_vectors: Don't apply affinity to @post_vectors at end of
* the MSI(-X) vector space
* @nr_sets: The number of interrupt sets for which affinity
* spreading is required
* @set_size: Array holding the size of each interrupt set
* @calc_sets: Callback for calculating the number and size
* of interrupt sets
* @priv: Private data for usage by @calc_sets, usually a
* pointer to driver/device specific data.
*/
struct irq_affinity {
unsigned int pre_vectors;
unsigned int post_vectors;
unsigned int nr_sets;
unsigned int set_size[IRQ_AFFINITY_MAX_SETS];
void (*calc_sets)(struct irq_affinity *, unsigned int nvecs);
void *priv;
};
/**
* struct irq_affinity_desc - Interrupt affinity descriptor
* @mask: cpumask to hold the affinity assignment
* @is_managed: 1 if the interrupt is managed internally
*/
struct irq_affinity_desc {
struct cpumask mask;
unsigned int is_managed : 1;
};
#if defined(CONFIG_SMP)
extern cpumask_var_t irq_default_affinity;
extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask);
extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask);
extern int irq_can_set_affinity(unsigned int irq);
extern int irq_select_affinity(unsigned int irq);
extern int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m,
bool setaffinity);
/**
* irq_update_affinity_hint - Update the affinity hint
* @irq: Interrupt to update
* @m: cpumask pointer (NULL to clear the hint)
*
* Updates the affinity hint, but does not change the affinity of the interrupt.
*/
static inline int
irq_update_affinity_hint(unsigned int irq, const struct cpumask *m)
{
return __irq_apply_affinity_hint(irq, m, false);
}
/**
* irq_set_affinity_and_hint - Update the affinity hint and apply the provided
* cpumask to the interrupt
* @irq: Interrupt to update
* @m: cpumask pointer (NULL to clear the hint)
*
* Updates the affinity hint and if @m is not NULL it applies it as the
* affinity of that interrupt.
*/
static inline int
irq_set_affinity_and_hint(unsigned int irq, const struct cpumask *m)
{
return __irq_apply_affinity_hint(irq, m, true);
}
/*
* Deprecated. Use irq_update_affinity_hint() or irq_set_affinity_and_hint()
* instead.
*/
static inline int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
{
return irq_set_affinity_and_hint(irq, m);
}
extern int irq_update_affinity_desc(unsigned int irq,
struct irq_affinity_desc *affinity);
extern int
irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
struct irq_affinity_desc *
irq_create_affinity_masks(unsigned int nvec, struct irq_affinity *affd);
unsigned int irq_calc_affinity_vectors(unsigned int minvec, unsigned int maxvec,
const struct irq_affinity *affd);
#else /* CONFIG_SMP */
static inline int irq_set_affinity(unsigned int irq, const struct cpumask *m)
{
return -EINVAL;
}
static inline int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask)
{
return 0;
}
static inline int irq_can_set_affinity(unsigned int irq)
{
return 0;
}
static inline int irq_select_affinity(unsigned int irq) { return 0; }
static inline int irq_update_affinity_hint(unsigned int irq,
const struct cpumask *m)
{
return -EINVAL;
}
static inline int irq_set_affinity_and_hint(unsigned int irq,
const struct cpumask *m)
{
return -EINVAL;
}
static inline int irq_set_affinity_hint(unsigned int irq,
const struct cpumask *m)
{
return -EINVAL;
}
static inline int irq_update_affinity_desc(unsigned int irq,
struct irq_affinity_desc *affinity)
{
return -EINVAL;
}
static inline int
irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
{
return 0;
}
static inline struct irq_affinity_desc *
irq_create_affinity_masks(unsigned int nvec, struct irq_affinity *affd)
{
return NULL;
}
static inline unsigned int
irq_calc_affinity_vectors(unsigned int minvec, unsigned int maxvec,
const struct irq_affinity *affd)
{
return maxvec;
}
#endif /* CONFIG_SMP */
/*
* Special lockdep variants of irq disabling/enabling.
* These should be used for locking constructs that
* know that a particular irq context which is disabled,
* and which is the only irq-context user of a lock,
* that it's safe to take the lock in the irq-disabled
* section without disabling hardirqs.
*
* On !CONFIG_LOCKDEP they are equivalent to the normal
* irq disable/enable methods.
*/
static inline void disable_irq_nosync_lockdep(unsigned int irq)
{
disable_irq_nosync(irq);
#ifdef CONFIG_LOCKDEP
local_irq_disable();
#endif
}
static inline void disable_irq_nosync_lockdep_irqsave(unsigned int irq, unsigned long *flags)
{
disable_irq_nosync(irq);
#ifdef CONFIG_LOCKDEP
local_irq_save(*flags);
#endif
}
static inline void disable_irq_lockdep(unsigned int irq)
{
disable_irq(irq);
#ifdef CONFIG_LOCKDEP
local_irq_disable();
#endif
}
static inline void enable_irq_lockdep(unsigned int irq)
{
#ifdef CONFIG_LOCKDEP
local_irq_enable();
#endif
enable_irq(irq);
}
static inline void enable_irq_lockdep_irqrestore(unsigned int irq, unsigned long *flags)
{
#ifdef CONFIG_LOCKDEP
local_irq_restore(*flags);
#endif
enable_irq(irq);
}
/* IRQ wakeup (PM) control: */
extern int irq_set_irq_wake(unsigned int irq, unsigned int on);
static inline int enable_irq_wake(unsigned int irq)
{
return irq_set_irq_wake(irq, 1);
}
static inline int disable_irq_wake(unsigned int irq)
{
return irq_set_irq_wake(irq, 0);
}
/*
* irq_get_irqchip_state/irq_set_irqchip_state specific flags
*/
enum irqchip_irq_state {
IRQCHIP_STATE_PENDING, /* Is interrupt pending? */
IRQCHIP_STATE_ACTIVE, /* Is interrupt in progress? */
IRQCHIP_STATE_MASKED, /* Is interrupt masked? */
IRQCHIP_STATE_LINE_LEVEL, /* Is IRQ line high? */
};
extern int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
bool *state);
extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
bool state);
#ifdef CONFIG_IRQ_FORCED_THREADING
# ifdef CONFIG_PREEMPT_RT
# define force_irqthreads() (true)
# else
DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
# define force_irqthreads() (static_branch_unlikely(&force_irqthreads_key))
# endif
#else
#define force_irqthreads() (false)
#endif
#ifndef local_softirq_pending
#ifndef local_softirq_pending_ref
#define local_softirq_pending_ref irq_stat.__softirq_pending
#endif
#define local_softirq_pending() (__this_cpu_read(local_softirq_pending_ref))
#define set_softirq_pending(x) (__this_cpu_write(local_softirq_pending_ref, (x)))
#define or_softirq_pending(x) (__this_cpu_or(local_softirq_pending_ref, (x)))
#endif /* local_softirq_pending */
/* Some architectures might implement lazy enabling/disabling of
* interrupts. In some cases, such as stop_machine, we might want
* to ensure that after a local_irq_disable(), interrupts have
* really been disabled in hardware. Such architectures need to
* implement the following hook.
*/
#ifndef hard_irq_disable
#define hard_irq_disable() do { } while(0)
#endif
/* PLEASE, avoid to allocate new softirqs, if you need not _really_ high
frequency threaded job scheduling. For almost all the purposes
tasklets are more than enough. F.e. all serial device BHs et
al. should be converted to tasklets, not to softirqs.
*/
enum
{
HI_SOFTIRQ=0,
TIMER_SOFTIRQ,
NET_TX_SOFTIRQ,
NET_RX_SOFTIRQ,
BLOCK_SOFTIRQ,
IRQ_POLL_SOFTIRQ,
TASKLET_SOFTIRQ,
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ,
RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS
};
/*
* The following vectors can be safely ignored after ksoftirqd is parked:
*
* _ RCU:
* 1) rcutree_migrate_callbacks() migrates the queue.
* 2) rcutree_report_cpu_dead() reports the final quiescent states.
*
* _ IRQ_POLL: irq_poll_cpu_dead() migrates the queue
*
* _ (HR)TIMER_SOFTIRQ: (hr)timers_dead_cpu() migrates the queue
*/
#define SOFTIRQ_HOTPLUG_SAFE_MASK (BIT(TIMER_SOFTIRQ) | BIT(IRQ_POLL_SOFTIRQ) |\
BIT(HRTIMER_SOFTIRQ) | BIT(RCU_SOFTIRQ))
/* map softirq index to softirq name. update 'softirq_to_name' in
* kernel/softirq.c when adding a new softirq.
*/
extern const char * const softirq_to_name[NR_SOFTIRQS];
/* softirq mask and active fields moved to irq_cpustat_t in
* asm/hardirq.h to get better cache usage. KAO
*/
struct softirq_action
{
void (*action)(struct softirq_action *);
};
asmlinkage void do_softirq(void);
asmlinkage void __do_softirq(void);
#ifdef CONFIG_PREEMPT_RT
extern void do_softirq_post_smp_call_flush(unsigned int was_pending);
#else
static inline void do_softirq_post_smp_call_flush(unsigned int unused)
{
do_softirq();
}
#endif
extern void open_softirq(int nr, void (*action)(struct softirq_action *));
extern void softirq_init(void);
extern void __raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq(unsigned int nr);
DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
static inline struct task_struct *this_cpu_ksoftirqd(void)
{
return this_cpu_read(ksoftirqd);
}
/* Tasklets --- multithreaded analogue of BHs.
This API is deprecated. Please consider using threaded IRQs instead:
https://lore.kernel.org/lkml/20200716081538.2sivhkj4hcyrusem@linutronix.de
Main feature differing them of generic softirqs: tasklet
is running only on one CPU simultaneously.
Main feature differing them of BHs: different tasklets
may be run simultaneously on different CPUs.
Properties:
* If tasklet_schedule() is called, then tasklet is guaranteed
to be executed on some cpu at least once after this.
* If the tasklet is already scheduled, but its execution is still not
started, it will be executed only once.
* If this tasklet is already running on another CPU (or schedule is called
from tasklet itself), it is rescheduled for later.
* Tasklet is strictly serialized wrt itself, but not
wrt another tasklets. If client needs some intertask synchronization,
he makes it with spinlocks.
*/
struct tasklet_struct
{
struct tasklet_struct *next;
unsigned long state;
atomic_t count;
bool use_callback;
union {
void (*func)(unsigned long data);
void (*callback)(struct tasklet_struct *t);
};
unsigned long data;
};
#define DECLARE_TASKLET(name, _callback) \
struct tasklet_struct name = { \
.count = ATOMIC_INIT(0), \
.callback = _callback, \
.use_callback = true, \
}
#define DECLARE_TASKLET_DISABLED(name, _callback) \
struct tasklet_struct name = { \
.count = ATOMIC_INIT(1), \
.callback = _callback, \
.use_callback = true, \
}
#define from_tasklet(var, callback_tasklet, tasklet_fieldname) \
container_of(callback_tasklet, typeof(*var), tasklet_fieldname)
#define DECLARE_TASKLET_OLD(name, _func) \
struct tasklet_struct name = { \
.count = ATOMIC_INIT(0), \
.func = _func, \
}
#define DECLARE_TASKLET_DISABLED_OLD(name, _func) \
struct tasklet_struct name = { \
.count = ATOMIC_INIT(1), \
.func = _func, \
}
enum
{
TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */
TASKLET_STATE_RUN /* Tasklet is running (SMP only) */
};
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
static inline int tasklet_trylock(struct tasklet_struct *t)
{
return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state);
}
void tasklet_unlock(struct tasklet_struct *t);
void tasklet_unlock_wait(struct tasklet_struct *t);
void tasklet_unlock_spin_wait(struct tasklet_struct *t);
#else
static inline int tasklet_trylock(struct tasklet_struct *t) { return 1; }
static inline void tasklet_unlock(struct tasklet_struct *t) { }
static inline void tasklet_unlock_wait(struct tasklet_struct *t) { }
static inline void tasklet_unlock_spin_wait(struct tasklet_struct *t) { }
#endif
extern void __tasklet_schedule(struct tasklet_struct *t);
static inline void tasklet_schedule(struct tasklet_struct *t)
{
if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) __tasklet_schedule(t);
}
extern void __tasklet_hi_schedule(struct tasklet_struct *t);
static inline void tasklet_hi_schedule(struct tasklet_struct *t)
{
if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
__tasklet_hi_schedule(t);
}
static inline void tasklet_disable_nosync(struct tasklet_struct *t)
{
atomic_inc(&t->count);
smp_mb__after_atomic();
}
/*
* Do not use in new code. Disabling tasklets from atomic contexts is
* error prone and should be avoided.
*/
static inline void tasklet_disable_in_atomic(struct tasklet_struct *t)
{
tasklet_disable_nosync(t);
tasklet_unlock_spin_wait(t);
smp_mb();
}
static inline void tasklet_disable(struct tasklet_struct *t)
{
tasklet_disable_nosync(t);
tasklet_unlock_wait(t);
smp_mb();
}
static inline void tasklet_enable(struct tasklet_struct *t)
{
smp_mb__before_atomic();
atomic_dec(&t->count);
}
extern void tasklet_kill(struct tasklet_struct *t);
extern void tasklet_init(struct tasklet_struct *t,
void (*func)(unsigned long), unsigned long data);
extern void tasklet_setup(struct tasklet_struct *t,
void (*callback)(struct tasklet_struct *));
/*
* Autoprobing for irqs:
*
* probe_irq_on() and probe_irq_off() provide robust primitives
* for accurate IRQ probing during kernel initialization. They are
* reasonably simple to use, are not "fooled" by spurious interrupts,
* and, unlike other attempts at IRQ probing, they do not get hung on
* stuck interrupts (such as unused PS2 mouse interfaces on ASUS boards).
*
* For reasonably foolproof probing, use them as follows:
*
* 1. clear and/or mask the device's internal interrupt.
* 2. sti();
* 3. irqs = probe_irq_on(); // "take over" all unassigned idle IRQs
* 4. enable the device and cause it to trigger an interrupt.
* 5. wait for the device to interrupt, using non-intrusive polling or a delay.
* 6. irq = probe_irq_off(irqs); // get IRQ number, 0=none, negative=multiple
* 7. service the device to clear its pending interrupt.
* 8. loop again if paranoia is required.
*
* probe_irq_on() returns a mask of allocated irq's.
*
* probe_irq_off() takes the mask as a parameter,
* and returns the irq number which occurred,
* or zero if none occurred, or a negative irq number
* if more than one irq occurred.
*/
#if !defined(CONFIG_GENERIC_IRQ_PROBE)
static inline unsigned long probe_irq_on(void)
{
return 0;
}
static inline int probe_irq_off(unsigned long val)
{
return 0;
}
static inline unsigned int probe_irq_mask(unsigned long val)
{
return 0;
}
#else
extern unsigned long probe_irq_on(void); /* returns 0 on failure */
extern int probe_irq_off(unsigned long); /* returns 0 or negative on failure */
extern unsigned int probe_irq_mask(unsigned long); /* returns mask of ISA interrupts */
#endif
#ifdef CONFIG_PROC_FS
/* Initialize /proc/irq/ */
extern void init_irq_proc(void);
#else
static inline void init_irq_proc(void)
{
}
#endif
#ifdef CONFIG_IRQ_TIMINGS
void irq_timings_enable(void);
void irq_timings_disable(void);
u64 irq_timings_next_event(u64 now);
#endif
struct seq_file;
int show_interrupts(struct seq_file *p, void *v);
int arch_show_interrupts(struct seq_file *p, int prec);
extern int early_irq_init(void);
extern int arch_probe_nr_irqs(void);
extern int arch_early_irq_init(void);
/*
* We want to know which function is an entrypoint of a hardirq or a softirq.
*/
#ifndef __irq_entry
# define __irq_entry __section(".irqentry.text")
#endif
#define __softirq_entry __section(".softirqentry.text")
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/fs/namespace.c
*
* (C) Copyright Al Viro 2000, 2001
*
* Based on code from fs/super.c, copyright Linus Torvalds and others.
* Heavily rewritten.
*/
#include <linux/syscalls.h>
#include <linux/export.h>
#include <linux/capability.h>
#include <linux/mnt_namespace.h>
#include <linux/user_namespace.h>
#include <linux/namei.h>
#include <linux/security.h>
#include <linux/cred.h>
#include <linux/idr.h>
#include <linux/init.h> /* init_rootfs */
#include <linux/fs_struct.h> /* get_fs_root et.al. */
#include <linux/fsnotify.h> /* fsnotify_vfsmount_delete */
#include <linux/file.h>
#include <linux/uaccess.h>
#include <linux/proc_ns.h>
#include <linux/magic.h>
#include <linux/memblock.h>
#include <linux/proc_fs.h>
#include <linux/task_work.h>
#include <linux/sched/task.h>
#include <uapi/linux/mount.h>
#include <linux/fs_context.h>
#include <linux/shmem_fs.h>
#include <linux/mnt_idmapping.h>
#include <linux/nospec.h>
#include "pnode.h"
#include "internal.h"
/* Maximum number of mounts in a mount namespace */
static unsigned int sysctl_mount_max __read_mostly = 100000;
static unsigned int m_hash_mask __ro_after_init;
static unsigned int m_hash_shift __ro_after_init;
static unsigned int mp_hash_mask __ro_after_init;
static unsigned int mp_hash_shift __ro_after_init;
static __initdata unsigned long mhash_entries;
static int __init set_mhash_entries(char *str)
{
if (!str)
return 0;
mhash_entries = simple_strtoul(str, &str, 0);
return 1;
}
__setup("mhash_entries=", set_mhash_entries);
static __initdata unsigned long mphash_entries;
static int __init set_mphash_entries(char *str)
{
if (!str)
return 0;
mphash_entries = simple_strtoul(str, &str, 0);
return 1;
}
__setup("mphash_entries=", set_mphash_entries);
static u64 event;
static DEFINE_IDA(mnt_id_ida);
static DEFINE_IDA(mnt_group_ida);
/* Don't allow confusion with old 32bit mount ID */
static atomic64_t mnt_id_ctr = ATOMIC64_INIT(1ULL << 32);
static struct hlist_head *mount_hashtable __ro_after_init;
static struct hlist_head *mountpoint_hashtable __ro_after_init;
static struct kmem_cache *mnt_cache __ro_after_init;
static DECLARE_RWSEM(namespace_sem);
static HLIST_HEAD(unmounted); /* protected by namespace_sem */
static LIST_HEAD(ex_mountpoints); /* protected by namespace_sem */
struct mount_kattr {
unsigned int attr_set;
unsigned int attr_clr;
unsigned int propagation;
unsigned int lookup_flags;
bool recurse;
struct user_namespace *mnt_userns;
struct mnt_idmap *mnt_idmap;
};
/* /sys/fs */
struct kobject *fs_kobj __ro_after_init;
EXPORT_SYMBOL_GPL(fs_kobj);
/*
* vfsmount lock may be taken for read to prevent changes to the
* vfsmount hash, ie. during mountpoint lookups or walking back
* up the tree.
*
* It should be taken for write in all cases where the vfsmount
* tree or hash is modified or when a vfsmount structure is modified.
*/
__cacheline_aligned_in_smp DEFINE_SEQLOCK(mount_lock);
static inline void lock_mount_hash(void)
{
write_seqlock(&mount_lock);
}
static inline void unlock_mount_hash(void)
{
write_sequnlock(&mount_lock);
}
static inline struct hlist_head *m_hash(struct vfsmount *mnt, struct dentry *dentry)
{
unsigned long tmp = ((unsigned long)mnt / L1_CACHE_BYTES);
tmp += ((unsigned long)dentry / L1_CACHE_BYTES);
tmp = tmp + (tmp >> m_hash_shift);
return &mount_hashtable[tmp & m_hash_mask];
}
static inline struct hlist_head *mp_hash(struct dentry *dentry)
{
unsigned long tmp = ((unsigned long)dentry / L1_CACHE_BYTES);
tmp = tmp + (tmp >> mp_hash_shift);
return &mountpoint_hashtable[tmp & mp_hash_mask];
}
static int mnt_alloc_id(struct mount *mnt)
{
int res = ida_alloc(&mnt_id_ida, GFP_KERNEL);
if (res < 0)
return res;
mnt->mnt_id = res;
mnt->mnt_id_unique = atomic64_inc_return(&mnt_id_ctr);
return 0;
}
static void mnt_free_id(struct mount *mnt)
{
ida_free(&mnt_id_ida, mnt->mnt_id);
}
/*
* Allocate a new peer group ID
*/
static int mnt_alloc_group_id(struct mount *mnt)
{
int res = ida_alloc_min(&mnt_group_ida, 1, GFP_KERNEL);
if (res < 0)
return res;
mnt->mnt_group_id = res;
return 0;
}
/*
* Release a peer group ID
*/
void mnt_release_group_id(struct mount *mnt)
{
ida_free(&mnt_group_ida, mnt->mnt_group_id);
mnt->mnt_group_id = 0;
}
/*
* vfsmount lock must be held for read
*/
static inline void mnt_add_count(struct mount *mnt, int n)
{
#ifdef CONFIG_SMP
this_cpu_add(mnt->mnt_pcp->mnt_count, n);
#else
preempt_disable();
mnt->mnt_count += n;
preempt_enable();
#endif
}
/*
* vfsmount lock must be held for write
*/
int mnt_get_count(struct mount *mnt)
{
#ifdef CONFIG_SMP
int count = 0;
int cpu;
for_each_possible_cpu(cpu) { count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_count;
}
return count;
#else
return mnt->mnt_count;
#endif
}
static struct mount *alloc_vfsmnt(const char *name)
{
struct mount *mnt = kmem_cache_zalloc(mnt_cache, GFP_KERNEL);
if (mnt) {
int err;
err = mnt_alloc_id(mnt);
if (err)
goto out_free_cache;
if (name) {
mnt->mnt_devname = kstrdup_const(name,
GFP_KERNEL_ACCOUNT);
if (!mnt->mnt_devname)
goto out_free_id;
}
#ifdef CONFIG_SMP
mnt->mnt_pcp = alloc_percpu(struct mnt_pcp);
if (!mnt->mnt_pcp)
goto out_free_devname;
this_cpu_add(mnt->mnt_pcp->mnt_count, 1);
#else
mnt->mnt_count = 1;
mnt->mnt_writers = 0;
#endif
INIT_HLIST_NODE(&mnt->mnt_hash);
INIT_LIST_HEAD(&mnt->mnt_child);
INIT_LIST_HEAD(&mnt->mnt_mounts);
INIT_LIST_HEAD(&mnt->mnt_list);
INIT_LIST_HEAD(&mnt->mnt_expire);
INIT_LIST_HEAD(&mnt->mnt_share);
INIT_LIST_HEAD(&mnt->mnt_slave_list);
INIT_LIST_HEAD(&mnt->mnt_slave);
INIT_HLIST_NODE(&mnt->mnt_mp_list);
INIT_LIST_HEAD(&mnt->mnt_umounting);
INIT_HLIST_HEAD(&mnt->mnt_stuck_children);
mnt->mnt.mnt_idmap = &nop_mnt_idmap;
}
return mnt;
#ifdef CONFIG_SMP
out_free_devname:
kfree_const(mnt->mnt_devname);
#endif
out_free_id:
mnt_free_id(mnt);
out_free_cache:
kmem_cache_free(mnt_cache, mnt);
return NULL;
}
/*
* Most r/o checks on a fs are for operations that take
* discrete amounts of time, like a write() or unlink().
* We must keep track of when those operations start
* (for permission checks) and when they end, so that
* we can determine when writes are able to occur to
* a filesystem.
*/
/*
* __mnt_is_readonly: check whether a mount is read-only
* @mnt: the mount to check for its write status
*
* This shouldn't be used directly ouside of the VFS.
* It does not guarantee that the filesystem will stay
* r/w, just that it is right *now*. This can not and
* should not be used in place of IS_RDONLY(inode).
* mnt_want/drop_write() will _keep_ the filesystem
* r/w.
*/
bool __mnt_is_readonly(struct vfsmount *mnt)
{
return (mnt->mnt_flags & MNT_READONLY) || sb_rdonly(mnt->mnt_sb);
}
EXPORT_SYMBOL_GPL(__mnt_is_readonly);
static inline void mnt_inc_writers(struct mount *mnt)
{
#ifdef CONFIG_SMP
this_cpu_inc(mnt->mnt_pcp->mnt_writers);
#else
mnt->mnt_writers++;
#endif
}
static inline void mnt_dec_writers(struct mount *mnt)
{
#ifdef CONFIG_SMP
this_cpu_dec(mnt->mnt_pcp->mnt_writers);
#else
mnt->mnt_writers--;
#endif
}
static unsigned int mnt_get_writers(struct mount *mnt)
{
#ifdef CONFIG_SMP
unsigned int count = 0;
int cpu;
for_each_possible_cpu(cpu) {
count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_writers;
}
return count;
#else
return mnt->mnt_writers;
#endif
}
static int mnt_is_readonly(struct vfsmount *mnt)
{
if (READ_ONCE(mnt->mnt_sb->s_readonly_remount))
return 1;
/*
* The barrier pairs with the barrier in sb_start_ro_state_change()
* making sure if we don't see s_readonly_remount set yet, we also will
* not see any superblock / mount flag changes done by remount.
* It also pairs with the barrier in sb_end_ro_state_change()
* assuring that if we see s_readonly_remount already cleared, we will
* see the values of superblock / mount flags updated by remount.
*/
smp_rmb(); return __mnt_is_readonly(mnt);
}
/*
* Most r/o & frozen checks on a fs are for operations that take discrete
* amounts of time, like a write() or unlink(). We must keep track of when
* those operations start (for permission checks) and when they end, so that we
* can determine when writes are able to occur to a filesystem.
*/
/**
* mnt_get_write_access - get write access to a mount without freeze protection
* @m: the mount on which to take a write
*
* This tells the low-level filesystem that a write is about to be performed to
* it, and makes sure that writes are allowed (mnt it read-write) before
* returning success. This operation does not protect against filesystem being
* frozen. When the write operation is finished, mnt_put_write_access() must be
* called. This is effectively a refcount.
*/
int mnt_get_write_access(struct vfsmount *m)
{
struct mount *mnt = real_mount(m);
int ret = 0;
preempt_disable();
mnt_inc_writers(mnt);
/*
* The store to mnt_inc_writers must be visible before we pass
* MNT_WRITE_HOLD loop below, so that the slowpath can see our
* incremented count after it has set MNT_WRITE_HOLD.
*/
smp_mb();
might_lock(&mount_lock.lock);
while (READ_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) {
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
cpu_relax();
} else {
/*
* This prevents priority inversion, if the task
* setting MNT_WRITE_HOLD got preempted on a remote
* CPU, and it prevents life lock if the task setting
* MNT_WRITE_HOLD has a lower priority and is bound to
* the same CPU as the task that is spinning here.
*/
preempt_enable();
lock_mount_hash();
unlock_mount_hash();
preempt_disable();
}
}
/*
* The barrier pairs with the barrier sb_start_ro_state_change() making
* sure that if we see MNT_WRITE_HOLD cleared, we will also see
* s_readonly_remount set (or even SB_RDONLY / MNT_READONLY flags) in
* mnt_is_readonly() and bail in case we are racing with remount
* read-only.
*/
smp_rmb(); if (mnt_is_readonly(m)) { mnt_dec_writers(mnt);
ret = -EROFS;
}
preempt_enable(); return ret;
}
EXPORT_SYMBOL_GPL(mnt_get_write_access);
/**
* mnt_want_write - get write access to a mount
* @m: the mount on which to take a write
*
* This tells the low-level filesystem that a write is about to be performed to
* it, and makes sure that writes are allowed (mount is read-write, filesystem
* is not frozen) before returning success. When the write operation is
* finished, mnt_drop_write() must be called. This is effectively a refcount.
*/
int mnt_want_write(struct vfsmount *m)
{
int ret;
sb_start_write(m->mnt_sb); ret = mnt_get_write_access(m);
if (ret)
sb_end_write(m->mnt_sb); return ret;
}
EXPORT_SYMBOL_GPL(mnt_want_write);
/**
* mnt_get_write_access_file - get write access to a file's mount
* @file: the file who's mount on which to take a write
*
* This is like mnt_get_write_access, but if @file is already open for write it
* skips incrementing mnt_writers (since the open file already has a reference)
* and instead only does the check for emergency r/o remounts. This must be
* paired with mnt_put_write_access_file.
*/
int mnt_get_write_access_file(struct file *file)
{
if (file->f_mode & FMODE_WRITER) {
/*
* Superblock may have become readonly while there are still
* writable fd's, e.g. due to a fs error with errors=remount-ro
*/
if (__mnt_is_readonly(file->f_path.mnt)) return -EROFS;
return 0;
}
return mnt_get_write_access(file->f_path.mnt);
}
/**
* mnt_want_write_file - get write access to a file's mount
* @file: the file who's mount on which to take a write
*
* This is like mnt_want_write, but if the file is already open for writing it
* skips incrementing mnt_writers (since the open file already has a reference)
* and instead only does the freeze protection and the check for emergency r/o
* remounts. This must be paired with mnt_drop_write_file.
*/
int mnt_want_write_file(struct file *file)
{
int ret;
sb_start_write(file_inode(file)->i_sb); ret = mnt_get_write_access_file(file); if (ret) sb_end_write(file_inode(file)->i_sb); return ret;
}
EXPORT_SYMBOL_GPL(mnt_want_write_file);
/**
* mnt_put_write_access - give up write access to a mount
* @mnt: the mount on which to give up write access
*
* Tells the low-level filesystem that we are done
* performing writes to it. Must be matched with
* mnt_get_write_access() call above.
*/
void mnt_put_write_access(struct vfsmount *mnt)
{
preempt_disable();
mnt_dec_writers(real_mount(mnt));
preempt_enable();
}
EXPORT_SYMBOL_GPL(mnt_put_write_access);
/**
* mnt_drop_write - give up write access to a mount
* @mnt: the mount on which to give up write access
*
* Tells the low-level filesystem that we are done performing writes to it and
* also allows filesystem to be frozen again. Must be matched with
* mnt_want_write() call above.
*/
void mnt_drop_write(struct vfsmount *mnt)
{
mnt_put_write_access(mnt); sb_end_write(mnt->mnt_sb);
}
EXPORT_SYMBOL_GPL(mnt_drop_write);
void mnt_put_write_access_file(struct file *file)
{
if (!(file->f_mode & FMODE_WRITER)) mnt_put_write_access(file->f_path.mnt);}
void mnt_drop_write_file(struct file *file)
{
mnt_put_write_access_file(file); sb_end_write(file_inode(file)->i_sb);
}
EXPORT_SYMBOL(mnt_drop_write_file);
/**
* mnt_hold_writers - prevent write access to the given mount
* @mnt: mnt to prevent write access to
*
* Prevents write access to @mnt if there are no active writers for @mnt.
* This function needs to be called and return successfully before changing
* properties of @mnt that need to remain stable for callers with write access
* to @mnt.
*
* After this functions has been called successfully callers must pair it with
* a call to mnt_unhold_writers() in order to stop preventing write access to
* @mnt.
*
* Context: This function expects lock_mount_hash() to be held serializing
* setting MNT_WRITE_HOLD.
* Return: On success 0 is returned.
* On error, -EBUSY is returned.
*/
static inline int mnt_hold_writers(struct mount *mnt)
{
mnt->mnt.mnt_flags |= MNT_WRITE_HOLD;
/*
* After storing MNT_WRITE_HOLD, we'll read the counters. This store
* should be visible before we do.
*/
smp_mb();
/*
* With writers on hold, if this value is zero, then there are
* definitely no active writers (although held writers may subsequently
* increment the count, they'll have to wait, and decrement it after
* seeing MNT_READONLY).
*
* It is OK to have counter incremented on one CPU and decremented on
* another: the sum will add up correctly. The danger would be when we
* sum up each counter, if we read a counter before it is incremented,
* but then read another CPU's count which it has been subsequently
* decremented from -- we would see more decrements than we should.
* MNT_WRITE_HOLD protects against this scenario, because
* mnt_want_write first increments count, then smp_mb, then spins on
* MNT_WRITE_HOLD, so it can't be decremented by another CPU while
* we're counting up here.
*/
if (mnt_get_writers(mnt) > 0)
return -EBUSY;
return 0;
}
/**
* mnt_unhold_writers - stop preventing write access to the given mount
* @mnt: mnt to stop preventing write access to
*
* Stop preventing write access to @mnt allowing callers to gain write access
* to @mnt again.
*
* This function can only be called after a successful call to
* mnt_hold_writers().
*
* Context: This function expects lock_mount_hash() to be held.
*/
static inline void mnt_unhold_writers(struct mount *mnt)
{
/*
* MNT_READONLY must become visible before ~MNT_WRITE_HOLD, so writers
* that become unheld will see MNT_READONLY.
*/
smp_wmb();
mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD;
}
static int mnt_make_readonly(struct mount *mnt)
{
int ret;
ret = mnt_hold_writers(mnt);
if (!ret)
mnt->mnt.mnt_flags |= MNT_READONLY;
mnt_unhold_writers(mnt);
return ret;
}
int sb_prepare_remount_readonly(struct super_block *sb)
{
struct mount *mnt;
int err = 0;
/* Racy optimization. Recheck the counter under MNT_WRITE_HOLD */
if (atomic_long_read(&sb->s_remove_count))
return -EBUSY;
lock_mount_hash();
list_for_each_entry(mnt, &sb->s_mounts, mnt_instance) {
if (!(mnt->mnt.mnt_flags & MNT_READONLY)) {
err = mnt_hold_writers(mnt);
if (err)
break;
}
}
if (!err && atomic_long_read(&sb->s_remove_count))
err = -EBUSY;
if (!err)
sb_start_ro_state_change(sb);
list_for_each_entry(mnt, &sb->s_mounts, mnt_instance) {
if (mnt->mnt.mnt_flags & MNT_WRITE_HOLD)
mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD;
}
unlock_mount_hash();
return err;
}
static void free_vfsmnt(struct mount *mnt)
{
mnt_idmap_put(mnt_idmap(&mnt->mnt));
kfree_const(mnt->mnt_devname);
#ifdef CONFIG_SMP
free_percpu(mnt->mnt_pcp);
#endif
kmem_cache_free(mnt_cache, mnt);
}
static void delayed_free_vfsmnt(struct rcu_head *head)
{
free_vfsmnt(container_of(head, struct mount, mnt_rcu));
}
/* call under rcu_read_lock */
int __legitimize_mnt(struct vfsmount *bastard, unsigned seq)
{
struct mount *mnt;
if (read_seqretry(&mount_lock, seq))
return 1;
if (bastard == NULL) return 0;
mnt = real_mount(bastard);
mnt_add_count(mnt, 1);
smp_mb(); // see mntput_no_expire()
if (likely(!read_seqretry(&mount_lock, seq)))
return 0;
if (bastard->mnt_flags & MNT_SYNC_UMOUNT) { mnt_add_count(mnt, -1);
return 1;
}
lock_mount_hash();
if (unlikely(bastard->mnt_flags & MNT_DOOMED)) {
mnt_add_count(mnt, -1);
unlock_mount_hash();
return 1;
}
unlock_mount_hash();
/* caller will mntput() */
return -1;
}
/* call under rcu_read_lock */
static bool legitimize_mnt(struct vfsmount *bastard, unsigned seq)
{
int res = __legitimize_mnt(bastard, seq);
if (likely(!res))
return true;
if (unlikely(res < 0)) {
rcu_read_unlock();
mntput(bastard);
rcu_read_lock();
}
return false;
}
/**
* __lookup_mnt - find first child mount
* @mnt: parent mount
* @dentry: mountpoint
*
* If @mnt has a child mount @c mounted @dentry find and return it.
*
* Note that the child mount @c need not be unique. There are cases
* where shadow mounts are created. For example, during mount
* propagation when a source mount @mnt whose root got overmounted by a
* mount @o after path lookup but before @namespace_sem could be
* acquired gets copied and propagated. So @mnt gets copied including
* @o. When @mnt is propagated to a destination mount @d that already
* has another mount @n mounted at the same mountpoint then the source
* mount @mnt will be tucked beneath @n, i.e., @n will be mounted on
* @mnt and @mnt mounted on @d. Now both @n and @o are mounted at @mnt
* on @dentry.
*
* Return: The first child of @mnt mounted @dentry or NULL.
*/
struct mount *__lookup_mnt(struct vfsmount *mnt, struct dentry *dentry)
{
struct hlist_head *head = m_hash(mnt, dentry);
struct mount *p;
hlist_for_each_entry_rcu(p, head, mnt_hash) if (&p->mnt_parent->mnt == mnt && p->mnt_mountpoint == dentry)
return p;
return NULL;
}
/*
* lookup_mnt - Return the first child mount mounted at path
*
* "First" means first mounted chronologically. If you create the
* following mounts:
*
* mount /dev/sda1 /mnt
* mount /dev/sda2 /mnt
* mount /dev/sda3 /mnt
*
* Then lookup_mnt() on the base /mnt dentry in the root mount will
* return successively the root dentry and vfsmount of /dev/sda1, then
* /dev/sda2, then /dev/sda3, then NULL.
*
* lookup_mnt takes a reference to the found vfsmount.
*/
struct vfsmount *lookup_mnt(const struct path *path)
{
struct mount *child_mnt;
struct vfsmount *m;
unsigned seq;
rcu_read_lock();
do {
seq = read_seqbegin(&mount_lock);
child_mnt = __lookup_mnt(path->mnt, path->dentry);
m = child_mnt ? &child_mnt->mnt : NULL;
} while (!legitimize_mnt(m, seq));
rcu_read_unlock();
return m;
}
/*
* __is_local_mountpoint - Test to see if dentry is a mountpoint in the
* current mount namespace.
*
* The common case is dentries are not mountpoints at all and that
* test is handled inline. For the slow case when we are actually
* dealing with a mountpoint of some kind, walk through all of the
* mounts in the current mount namespace and test to see if the dentry
* is a mountpoint.
*
* The mount_hashtable is not usable in the context because we
* need to identify all mounts that may be in the current mount
* namespace not just a mount that happens to have some specified
* parent mount.
*/
bool __is_local_mountpoint(struct dentry *dentry)
{
struct mnt_namespace *ns = current->nsproxy->mnt_ns;
struct mount *mnt, *n;
bool is_covered = false;
down_read(&namespace_sem);
rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
is_covered = (mnt->mnt_mountpoint == dentry);
if (is_covered)
break;
}
up_read(&namespace_sem);
return is_covered;
}
static struct mountpoint *lookup_mountpoint(struct dentry *dentry)
{
struct hlist_head *chain = mp_hash(dentry);
struct mountpoint *mp;
hlist_for_each_entry(mp, chain, m_hash) {
if (mp->m_dentry == dentry) {
mp->m_count++;
return mp;
}
}
return NULL;
}
static struct mountpoint *get_mountpoint(struct dentry *dentry)
{
struct mountpoint *mp, *new = NULL;
int ret;
if (d_mountpoint(dentry)) {
/* might be worth a WARN_ON() */
if (d_unlinked(dentry))
return ERR_PTR(-ENOENT);
mountpoint:
read_seqlock_excl(&mount_lock);
mp = lookup_mountpoint(dentry);
read_sequnlock_excl(&mount_lock);
if (mp)
goto done;
}
if (!new)
new = kmalloc(sizeof(struct mountpoint), GFP_KERNEL);
if (!new)
return ERR_PTR(-ENOMEM);
/* Exactly one processes may set d_mounted */
ret = d_set_mounted(dentry);
/* Someone else set d_mounted? */
if (ret == -EBUSY)
goto mountpoint;
/* The dentry is not available as a mountpoint? */
mp = ERR_PTR(ret);
if (ret)
goto done;
/* Add the new mountpoint to the hash table */
read_seqlock_excl(&mount_lock);
new->m_dentry = dget(dentry);
new->m_count = 1;
hlist_add_head(&new->m_hash, mp_hash(dentry));
INIT_HLIST_HEAD(&new->m_list);
read_sequnlock_excl(&mount_lock);
mp = new;
new = NULL;
done:
kfree(new);
return mp;
}
/*
* vfsmount lock must be held. Additionally, the caller is responsible
* for serializing calls for given disposal list.
*/
static void __put_mountpoint(struct mountpoint *mp, struct list_head *list)
{
if (!--mp->m_count) {
struct dentry *dentry = mp->m_dentry;
BUG_ON(!hlist_empty(&mp->m_list));
spin_lock(&dentry->d_lock);
dentry->d_flags &= ~DCACHE_MOUNTED;
spin_unlock(&dentry->d_lock);
dput_to_list(dentry, list);
hlist_del(&mp->m_hash);
kfree(mp);
}
}
/* called with namespace_lock and vfsmount lock */
static void put_mountpoint(struct mountpoint *mp)
{
__put_mountpoint(mp, &ex_mountpoints);
}
static inline int check_mnt(struct mount *mnt)
{
return mnt->mnt_ns == current->nsproxy->mnt_ns;
}
/*
* vfsmount lock must be held for write
*/
static void touch_mnt_namespace(struct mnt_namespace *ns)
{
if (ns) {
ns->event = ++event;
wake_up_interruptible(&ns->poll);
}
}
/*
* vfsmount lock must be held for write
*/
static void __touch_mnt_namespace(struct mnt_namespace *ns)
{
if (ns && ns->event != event) {
ns->event = event;
wake_up_interruptible(&ns->poll);
}
}
/*
* vfsmount lock must be held for write
*/
static struct mountpoint *unhash_mnt(struct mount *mnt)
{
struct mountpoint *mp;
mnt->mnt_parent = mnt;
mnt->mnt_mountpoint = mnt->mnt.mnt_root;
list_del_init(&mnt->mnt_child);
hlist_del_init_rcu(&mnt->mnt_hash);
hlist_del_init(&mnt->mnt_mp_list);
mp = mnt->mnt_mp;
mnt->mnt_mp = NULL;
return mp;
}
/*
* vfsmount lock must be held for write
*/
static void umount_mnt(struct mount *mnt)
{
put_mountpoint(unhash_mnt(mnt));
}
/*
* vfsmount lock must be held for write
*/
void mnt_set_mountpoint(struct mount *mnt,
struct mountpoint *mp,
struct mount *child_mnt)
{
mp->m_count++;
mnt_add_count(mnt, 1); /* essentially, that's mntget */
child_mnt->mnt_mountpoint = mp->m_dentry;
child_mnt->mnt_parent = mnt;
child_mnt->mnt_mp = mp;
hlist_add_head(&child_mnt->mnt_mp_list, &mp->m_list);
}
/**
* mnt_set_mountpoint_beneath - mount a mount beneath another one
*
* @new_parent: the source mount
* @top_mnt: the mount beneath which @new_parent is mounted
* @new_mp: the new mountpoint of @top_mnt on @new_parent
*
* Remove @top_mnt from its current mountpoint @top_mnt->mnt_mp and
* parent @top_mnt->mnt_parent and mount it on top of @new_parent at
* @new_mp. And mount @new_parent on the old parent and old
* mountpoint of @top_mnt.
*
* Context: This function expects namespace_lock() and lock_mount_hash()
* to have been acquired in that order.
*/
static void mnt_set_mountpoint_beneath(struct mount *new_parent,
struct mount *top_mnt,
struct mountpoint *new_mp)
{
struct mount *old_top_parent = top_mnt->mnt_parent;
struct mountpoint *old_top_mp = top_mnt->mnt_mp;
mnt_set_mountpoint(old_top_parent, old_top_mp, new_parent);
mnt_change_mountpoint(new_parent, new_mp, top_mnt);
}
static void __attach_mnt(struct mount *mnt, struct mount *parent)
{
hlist_add_head_rcu(&mnt->mnt_hash,
m_hash(&parent->mnt, mnt->mnt_mountpoint));
list_add_tail(&mnt->mnt_child, &parent->mnt_mounts);
}
/**
* attach_mnt - mount a mount, attach to @mount_hashtable and parent's
* list of child mounts
* @parent: the parent
* @mnt: the new mount
* @mp: the new mountpoint
* @beneath: whether to mount @mnt beneath or on top of @parent
*
* If @beneath is false, mount @mnt at @mp on @parent. Then attach @mnt
* to @parent's child mount list and to @mount_hashtable.
*
* If @beneath is true, remove @mnt from its current parent and
* mountpoint and mount it on @mp on @parent, and mount @parent on the
* old parent and old mountpoint of @mnt. Finally, attach @parent to
* @mnt_hashtable and @parent->mnt_parent->mnt_mounts.
*
* Note, when __attach_mnt() is called @mnt->mnt_parent already points
* to the correct parent.
*
* Context: This function expects namespace_lock() and lock_mount_hash()
* to have been acquired in that order.
*/
static void attach_mnt(struct mount *mnt, struct mount *parent,
struct mountpoint *mp, bool beneath)
{
if (beneath)
mnt_set_mountpoint_beneath(mnt, parent, mp);
else
mnt_set_mountpoint(parent, mp, mnt);
/*
* Note, @mnt->mnt_parent has to be used. If @mnt was mounted
* beneath @parent then @mnt will need to be attached to
* @parent's old parent, not @parent. IOW, @mnt->mnt_parent
* isn't the same mount as @parent.
*/
__attach_mnt(mnt, mnt->mnt_parent);
}
void mnt_change_mountpoint(struct mount *parent, struct mountpoint *mp, struct mount *mnt)
{
struct mountpoint *old_mp = mnt->mnt_mp;
struct mount *old_parent = mnt->mnt_parent;
list_del_init(&mnt->mnt_child);
hlist_del_init(&mnt->mnt_mp_list);
hlist_del_init_rcu(&mnt->mnt_hash);
attach_mnt(mnt, parent, mp, false);
put_mountpoint(old_mp);
mnt_add_count(old_parent, -1);
}
static inline struct mount *node_to_mount(struct rb_node *node)
{
return node ? rb_entry(node, struct mount, mnt_node) : NULL;
}
static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt)
{
struct rb_node **link = &ns->mounts.rb_node;
struct rb_node *parent = NULL;
WARN_ON(mnt->mnt.mnt_flags & MNT_ONRB);
mnt->mnt_ns = ns;
while (*link) {
parent = *link;
if (mnt->mnt_id_unique < node_to_mount(parent)->mnt_id_unique)
link = &parent->rb_left;
else
link = &parent->rb_right;
}
rb_link_node(&mnt->mnt_node, parent, link);
rb_insert_color(&mnt->mnt_node, &ns->mounts);
mnt->mnt.mnt_flags |= MNT_ONRB;
}
/*
* vfsmount lock must be held for write
*/
static void commit_tree(struct mount *mnt)
{
struct mount *parent = mnt->mnt_parent;
struct mount *m;
LIST_HEAD(head);
struct mnt_namespace *n = parent->mnt_ns;
BUG_ON(parent == mnt);
list_add_tail(&head, &mnt->mnt_list);
while (!list_empty(&head)) {
m = list_first_entry(&head, typeof(*m), mnt_list);
list_del(&m->mnt_list);
mnt_add_to_ns(n, m);
}
n->nr_mounts += n->pending_mounts;
n->pending_mounts = 0;
__attach_mnt(mnt, parent);
touch_mnt_namespace(n);
}
static struct mount *next_mnt(struct mount *p, struct mount *root)
{
struct list_head *next = p->mnt_mounts.next;
if (next == &p->mnt_mounts) {
while (1) {
if (p == root)
return NULL;
next = p->mnt_child.next;
if (next != &p->mnt_parent->mnt_mounts)
break;
p = p->mnt_parent;
}
}
return list_entry(next, struct mount, mnt_child);
}
static struct mount *skip_mnt_tree(struct mount *p)
{
struct list_head *prev = p->mnt_mounts.prev;
while (prev != &p->mnt_mounts) {
p = list_entry(prev, struct mount, mnt_child);
prev = p->mnt_mounts.prev;
}
return p;
}
/**
* vfs_create_mount - Create a mount for a configured superblock
* @fc: The configuration context with the superblock attached
*
* Create a mount to an already configured superblock. If necessary, the
* caller should invoke vfs_get_tree() before calling this.
*
* Note that this does not attach the mount to anything.
*/
struct vfsmount *vfs_create_mount(struct fs_context *fc)
{
struct mount *mnt;
if (!fc->root)
return ERR_PTR(-EINVAL);
mnt = alloc_vfsmnt(fc->source ?: "none");
if (!mnt)
return ERR_PTR(-ENOMEM);
if (fc->sb_flags & SB_KERNMOUNT)
mnt->mnt.mnt_flags = MNT_INTERNAL;
atomic_inc(&fc->root->d_sb->s_active);
mnt->mnt.mnt_sb = fc->root->d_sb;
mnt->mnt.mnt_root = dget(fc->root);
mnt->mnt_mountpoint = mnt->mnt.mnt_root;
mnt->mnt_parent = mnt;
lock_mount_hash();
list_add_tail(&mnt->mnt_instance, &mnt->mnt.mnt_sb->s_mounts);
unlock_mount_hash();
return &mnt->mnt;
}
EXPORT_SYMBOL(vfs_create_mount);
struct vfsmount *fc_mount(struct fs_context *fc)
{
int err = vfs_get_tree(fc);
if (!err) {
up_write(&fc->root->d_sb->s_umount);
return vfs_create_mount(fc);
}
return ERR_PTR(err);
}
EXPORT_SYMBOL(fc_mount);
struct vfsmount *vfs_kern_mount(struct file_system_type *type,
int flags, const char *name,
void *data)
{
struct fs_context *fc;
struct vfsmount *mnt;
int ret = 0;
if (!type)
return ERR_PTR(-EINVAL);
fc = fs_context_for_mount(type, flags);
if (IS_ERR(fc))
return ERR_CAST(fc);
if (name)
ret = vfs_parse_fs_string(fc, "source",
name, strlen(name));
if (!ret)
ret = parse_monolithic_mount_data(fc, data);
if (!ret)
mnt = fc_mount(fc);
else
mnt = ERR_PTR(ret);
put_fs_context(fc);
return mnt;
}
EXPORT_SYMBOL_GPL(vfs_kern_mount);
struct vfsmount *
vfs_submount(const struct dentry *mountpoint, struct file_system_type *type,
const char *name, void *data)
{
/* Until it is worked out how to pass the user namespace
* through from the parent mount to the submount don't support
* unprivileged mounts with submounts.
*/
if (mountpoint->d_sb->s_user_ns != &init_user_ns)
return ERR_PTR(-EPERM);
return vfs_kern_mount(type, SB_SUBMOUNT, name, data);
}
EXPORT_SYMBOL_GPL(vfs_submount);
static struct mount *clone_mnt(struct mount *old, struct dentry *root,
int flag)
{
struct super_block *sb = old->mnt.mnt_sb;
struct mount *mnt;
int err;
mnt = alloc_vfsmnt(old->mnt_devname);
if (!mnt)
return ERR_PTR(-ENOMEM);
if (flag & (CL_SLAVE | CL_PRIVATE | CL_SHARED_TO_SLAVE))
mnt->mnt_group_id = 0; /* not a peer of original */
else
mnt->mnt_group_id = old->mnt_group_id;
if ((flag & CL_MAKE_SHARED) && !mnt->mnt_group_id) {
err = mnt_alloc_group_id(mnt);
if (err)
goto out_free;
}
mnt->mnt.mnt_flags = old->mnt.mnt_flags;
mnt->mnt.mnt_flags &= ~(MNT_WRITE_HOLD|MNT_MARKED|MNT_INTERNAL|MNT_ONRB);
atomic_inc(&sb->s_active);
mnt->mnt.mnt_idmap = mnt_idmap_get(mnt_idmap(&old->mnt));
mnt->mnt.mnt_sb = sb;
mnt->mnt.mnt_root = dget(root);
mnt->mnt_mountpoint = mnt->mnt.mnt_root;
mnt->mnt_parent = mnt;
lock_mount_hash();
list_add_tail(&mnt->mnt_instance, &sb->s_mounts);
unlock_mount_hash();
if ((flag & CL_SLAVE) ||
((flag & CL_SHARED_TO_SLAVE) && IS_MNT_SHARED(old))) {
list_add(&mnt->mnt_slave, &old->mnt_slave_list);
mnt->mnt_master = old;
CLEAR_MNT_SHARED(mnt);
} else if (!(flag & CL_PRIVATE)) {
if ((flag & CL_MAKE_SHARED) || IS_MNT_SHARED(old))
list_add(&mnt->mnt_share, &old->mnt_share);
if (IS_MNT_SLAVE(old))
list_add(&mnt->mnt_slave, &old->mnt_slave);
mnt->mnt_master = old->mnt_master;
} else {
CLEAR_MNT_SHARED(mnt);
}
if (flag & CL_MAKE_SHARED)
set_mnt_shared(mnt);
/* stick the duplicate mount on the same expiry list
* as the original if that was on one */
if (flag & CL_EXPIRE) {
if (!list_empty(&old->mnt_expire))
list_add(&mnt->mnt_expire, &old->mnt_expire);
}
return mnt;
out_free:
mnt_free_id(mnt);
free_vfsmnt(mnt);
return ERR_PTR(err);
}
static void cleanup_mnt(struct mount *mnt)
{
struct hlist_node *p;
struct mount *m;
/*
* The warning here probably indicates that somebody messed
* up a mnt_want/drop_write() pair. If this happens, the
* filesystem was probably unable to make r/w->r/o transitions.
* The locking used to deal with mnt_count decrement provides barriers,
* so mnt_get_writers() below is safe.
*/
WARN_ON(mnt_get_writers(mnt));
if (unlikely(mnt->mnt_pins.first))
mnt_pin_kill(mnt);
hlist_for_each_entry_safe(m, p, &mnt->mnt_stuck_children, mnt_umount) {
hlist_del(&m->mnt_umount);
mntput(&m->mnt);
}
fsnotify_vfsmount_delete(&mnt->mnt);
dput(mnt->mnt.mnt_root);
deactivate_super(mnt->mnt.mnt_sb);
mnt_free_id(mnt);
call_rcu(&mnt->mnt_rcu, delayed_free_vfsmnt);
}
static void __cleanup_mnt(struct rcu_head *head)
{
cleanup_mnt(container_of(head, struct mount, mnt_rcu));
}
static LLIST_HEAD(delayed_mntput_list);
static void delayed_mntput(struct work_struct *unused)
{
struct llist_node *node = llist_del_all(&delayed_mntput_list);
struct mount *m, *t;
llist_for_each_entry_safe(m, t, node, mnt_llist)
cleanup_mnt(m);
}
static DECLARE_DELAYED_WORK(delayed_mntput_work, delayed_mntput);
static void mntput_no_expire(struct mount *mnt)
{
LIST_HEAD(list);
int count;
rcu_read_lock(); if (likely(READ_ONCE(mnt->mnt_ns))) {
/*
* Since we don't do lock_mount_hash() here,
* ->mnt_ns can change under us. However, if it's
* non-NULL, then there's a reference that won't
* be dropped until after an RCU delay done after
* turning ->mnt_ns NULL. So if we observe it
* non-NULL under rcu_read_lock(), the reference
* we are dropping is not the final one.
*/
mnt_add_count(mnt, -1); rcu_read_unlock();
return;
}
lock_mount_hash();
/*
* make sure that if __legitimize_mnt() has not seen us grab
* mount_lock, we'll see their refcount increment here.
*/
smp_mb();
mnt_add_count(mnt, -1);
count = mnt_get_count(mnt);
if (count != 0) {
WARN_ON(count < 0);
rcu_read_unlock();
unlock_mount_hash();
return;
}
if (unlikely(mnt->mnt.mnt_flags & MNT_DOOMED)) { rcu_read_unlock();
unlock_mount_hash();
return;
}
mnt->mnt.mnt_flags |= MNT_DOOMED; rcu_read_unlock(); list_del(&mnt->mnt_instance);
if (unlikely(!list_empty(&mnt->mnt_mounts))) {
struct mount *p, *tmp;
list_for_each_entry_safe(p, tmp, &mnt->mnt_mounts, mnt_child) { __put_mountpoint(unhash_mnt(p), &list); hlist_add_head(&p->mnt_umount, &mnt->mnt_stuck_children);
}
}
unlock_mount_hash();
shrink_dentry_list(&list);
if (likely(!(mnt->mnt.mnt_flags & MNT_INTERNAL))) {
struct task_struct *task = current;
if (likely(!(task->flags & PF_KTHREAD))) {
init_task_work(&mnt->mnt_rcu, __cleanup_mnt);
if (!task_work_add(task, &mnt->mnt_rcu, TWA_RESUME))
return;
}
if (llist_add(&mnt->mnt_llist, &delayed_mntput_list)) schedule_delayed_work(&delayed_mntput_work, 1);
return;
}
cleanup_mnt(mnt);
}
void mntput(struct vfsmount *mnt)
{
if (mnt) { struct mount *m = real_mount(mnt);
/* avoid cacheline pingpong */
if (unlikely(m->mnt_expiry_mark))
WRITE_ONCE(m->mnt_expiry_mark, 0); mntput_no_expire(m);
}
}
EXPORT_SYMBOL(mntput);
struct vfsmount *mntget(struct vfsmount *mnt)
{
if (mnt) mnt_add_count(real_mount(mnt), 1); return mnt;
}
EXPORT_SYMBOL(mntget);
/*
* Make a mount point inaccessible to new lookups.
* Because there may still be current users, the caller MUST WAIT
* for an RCU grace period before destroying the mount point.
*/
void mnt_make_shortterm(struct vfsmount *mnt)
{
if (mnt)
real_mount(mnt)->mnt_ns = NULL;
}
/**
* path_is_mountpoint() - Check if path is a mount in the current namespace.
* @path: path to check
*
* d_mountpoint() can only be used reliably to establish if a dentry is
* not mounted in any namespace and that common case is handled inline.
* d_mountpoint() isn't aware of the possibility there may be multiple
* mounts using a given dentry in a different namespace. This function
* checks if the passed in path is a mountpoint rather than the dentry
* alone.
*/
bool path_is_mountpoint(const struct path *path)
{
unsigned seq;
bool res;
if (!d_mountpoint(path->dentry))
return false;
rcu_read_lock();
do {
seq = read_seqbegin(&mount_lock);
res = __path_is_mountpoint(path);
} while (read_seqretry(&mount_lock, seq));
rcu_read_unlock();
return res;
}
EXPORT_SYMBOL(path_is_mountpoint);
struct vfsmount *mnt_clone_internal(const struct path *path)
{
struct mount *p;
p = clone_mnt(real_mount(path->mnt), path->dentry, CL_PRIVATE);
if (IS_ERR(p))
return ERR_CAST(p);
p->mnt.mnt_flags |= MNT_INTERNAL;
return &p->mnt;
}
/*
* Returns the mount which either has the specified mnt_id, or has the next
* smallest id afer the specified one.
*/
static struct mount *mnt_find_id_at(struct mnt_namespace *ns, u64 mnt_id)
{
struct rb_node *node = ns->mounts.rb_node;
struct mount *ret = NULL;
while (node) {
struct mount *m = node_to_mount(node);
if (mnt_id <= m->mnt_id_unique) {
ret = node_to_mount(node);
if (mnt_id == m->mnt_id_unique)
break;
node = node->rb_left;
} else {
node = node->rb_right;
}
}
return ret;
}
#ifdef CONFIG_PROC_FS
/* iterator; we want it to have access to namespace_sem, thus here... */
static void *m_start(struct seq_file *m, loff_t *pos)
{
struct proc_mounts *p = m->private;
down_read(&namespace_sem);
return mnt_find_id_at(p->ns, *pos);
}
static void *m_next(struct seq_file *m, void *v, loff_t *pos)
{
struct mount *next = NULL, *mnt = v;
struct rb_node *node = rb_next(&mnt->mnt_node);
++*pos;
if (node) {
next = node_to_mount(node);
*pos = next->mnt_id_unique;
}
return next;
}
static void m_stop(struct seq_file *m, void *v)
{
up_read(&namespace_sem);
}
static int m_show(struct seq_file *m, void *v)
{
struct proc_mounts *p = m->private;
struct mount *r = v;
return p->show(m, &r->mnt);
}
const struct seq_operations mounts_op = {
.start = m_start,
.next = m_next,
.stop = m_stop,
.show = m_show,
};
#endif /* CONFIG_PROC_FS */
/**
* may_umount_tree - check if a mount tree is busy
* @m: root of mount tree
*
* This is called to check if a tree of mounts has any
* open files, pwds, chroots or sub mounts that are
* busy.
*/
int may_umount_tree(struct vfsmount *m)
{
struct mount *mnt = real_mount(m);
int actual_refs = 0;
int minimum_refs = 0;
struct mount *p;
BUG_ON(!m);
/* write lock needed for mnt_get_count */
lock_mount_hash();
for (p = mnt; p; p = next_mnt(p, mnt)) {
actual_refs += mnt_get_count(p);
minimum_refs += 2;
}
unlock_mount_hash();
if (actual_refs > minimum_refs)
return 0;
return 1;
}
EXPORT_SYMBOL(may_umount_tree);
/**
* may_umount - check if a mount point is busy
* @mnt: root of mount
*
* This is called to check if a mount point has any
* open files, pwds, chroots or sub mounts. If the
* mount has sub mounts this will return busy
* regardless of whether the sub mounts are busy.
*
* Doesn't take quota and stuff into account. IOW, in some cases it will
* give false negatives. The main reason why it's here is that we need
* a non-destructive way to look for easily umountable filesystems.
*/
int may_umount(struct vfsmount *mnt)
{
int ret = 1;
down_read(&namespace_sem);
lock_mount_hash();
if (propagate_mount_busy(real_mount(mnt), 2))
ret = 0;
unlock_mount_hash();
up_read(&namespace_sem);
return ret;
}
EXPORT_SYMBOL(may_umount);
static void namespace_unlock(void)
{
struct hlist_head head;
struct hlist_node *p;
struct mount *m;
LIST_HEAD(list);
hlist_move_list(&unmounted, &head);
list_splice_init(&ex_mountpoints, &list);
up_write(&namespace_sem);
shrink_dentry_list(&list);
if (likely(hlist_empty(&head)))
return;
synchronize_rcu_expedited();
hlist_for_each_entry_safe(m, p, &head, mnt_umount) {
hlist_del(&m->mnt_umount);
mntput(&m->mnt);
}
}
static inline void namespace_lock(void)
{
down_write(&namespace_sem);
}
enum umount_tree_flags {
UMOUNT_SYNC = 1,
UMOUNT_PROPAGATE = 2,
UMOUNT_CONNECTED = 4,
};
static bool disconnect_mount(struct mount *mnt, enum umount_tree_flags how)
{
/* Leaving mounts connected is only valid for lazy umounts */
if (how & UMOUNT_SYNC)
return true;
/* A mount without a parent has nothing to be connected to */
if (!mnt_has_parent(mnt))
return true;
/* Because the reference counting rules change when mounts are
* unmounted and connected, umounted mounts may not be
* connected to mounted mounts.
*/
if (!(mnt->mnt_parent->mnt.mnt_flags & MNT_UMOUNT))
return true;
/* Has it been requested that the mount remain connected? */
if (how & UMOUNT_CONNECTED)
return false;
/* Is the mount locked such that it needs to remain connected? */
if (IS_MNT_LOCKED(mnt))
return false;
/* By default disconnect the mount */
return true;
}
/*
* mount_lock must be held
* namespace_sem must be held for write
*/
static void umount_tree(struct mount *mnt, enum umount_tree_flags how)
{
LIST_HEAD(tmp_list);
struct mount *p;
if (how & UMOUNT_PROPAGATE)
propagate_mount_unlock(mnt);
/* Gather the mounts to umount */
for (p = mnt; p; p = next_mnt(p, mnt)) {
p->mnt.mnt_flags |= MNT_UMOUNT;
if (p->mnt.mnt_flags & MNT_ONRB)
move_from_ns(p, &tmp_list);
else
list_move(&p->mnt_list, &tmp_list);
}
/* Hide the mounts from mnt_mounts */
list_for_each_entry(p, &tmp_list, mnt_list) {
list_del_init(&p->mnt_child);
}
/* Add propogated mounts to the tmp_list */
if (how & UMOUNT_PROPAGATE)
propagate_umount(&tmp_list);
while (!list_empty(&tmp_list)) {
struct mnt_namespace *ns;
bool disconnect;
p = list_first_entry(&tmp_list, struct mount, mnt_list);
list_del_init(&p->mnt_expire);
list_del_init(&p->mnt_list);
ns = p->mnt_ns;
if (ns) {
ns->nr_mounts--;
__touch_mnt_namespace(ns);
}
p->mnt_ns = NULL;
if (how & UMOUNT_SYNC)
p->mnt.mnt_flags |= MNT_SYNC_UMOUNT;
disconnect = disconnect_mount(p, how);
if (mnt_has_parent(p)) {
mnt_add_count(p->mnt_parent, -1);
if (!disconnect) {
/* Don't forget about p */
list_add_tail(&p->mnt_child, &p->mnt_parent->mnt_mounts);
} else {
umount_mnt(p);
}
}
change_mnt_propagation(p, MS_PRIVATE);
if (disconnect)
hlist_add_head(&p->mnt_umount, &unmounted);
}
}
static void shrink_submounts(struct mount *mnt);
static int do_umount_root(struct super_block *sb)
{
int ret = 0;
down_write(&sb->s_umount);
if (!sb_rdonly(sb)) {
struct fs_context *fc;
fc = fs_context_for_reconfigure(sb->s_root, SB_RDONLY,
SB_RDONLY);
if (IS_ERR(fc)) {
ret = PTR_ERR(fc);
} else {
ret = parse_monolithic_mount_data(fc, NULL);
if (!ret)
ret = reconfigure_super(fc);
put_fs_context(fc);
}
}
up_write(&sb->s_umount);
return ret;
}
static int do_umount(struct mount *mnt, int flags)
{
struct super_block *sb = mnt->mnt.mnt_sb;
int retval;
retval = security_sb_umount(&mnt->mnt, flags);
if (retval)
return retval;
/*
* Allow userspace to request a mountpoint be expired rather than
* unmounting unconditionally. Unmount only happens if:
* (1) the mark is already set (the mark is cleared by mntput())
* (2) the usage count == 1 [parent vfsmount] + 1 [sys_umount]
*/
if (flags & MNT_EXPIRE) {
if (&mnt->mnt == current->fs->root.mnt ||
flags & (MNT_FORCE | MNT_DETACH))
return -EINVAL;
/*
* probably don't strictly need the lock here if we examined
* all race cases, but it's a slowpath.
*/
lock_mount_hash();
if (mnt_get_count(mnt) != 2) {
unlock_mount_hash();
return -EBUSY;
}
unlock_mount_hash();
if (!xchg(&mnt->mnt_expiry_mark, 1))
return -EAGAIN;
}
/*
* If we may have to abort operations to get out of this
* mount, and they will themselves hold resources we must
* allow the fs to do things. In the Unix tradition of
* 'Gee thats tricky lets do it in userspace' the umount_begin
* might fail to complete on the first run through as other tasks
* must return, and the like. Thats for the mount program to worry
* about for the moment.
*/
if (flags & MNT_FORCE && sb->s_op->umount_begin) {
sb->s_op->umount_begin(sb);
}
/*
* No sense to grab the lock for this test, but test itself looks
* somewhat bogus. Suggestions for better replacement?
* Ho-hum... In principle, we might treat that as umount + switch
* to rootfs. GC would eventually take care of the old vfsmount.
* Actually it makes sense, especially if rootfs would contain a
* /reboot - static binary that would close all descriptors and
* call reboot(9). Then init(8) could umount root and exec /reboot.
*/
if (&mnt->mnt == current->fs->root.mnt && !(flags & MNT_DETACH)) {
/*
* Special case for "unmounting" root ...
* we just try to remount it readonly.
*/
if (!ns_capable(sb->s_user_ns, CAP_SYS_ADMIN))
return -EPERM;
return do_umount_root(sb);
}
namespace_lock();
lock_mount_hash();
/* Recheck MNT_LOCKED with the locks held */
retval = -EINVAL;
if (mnt->mnt.mnt_flags & MNT_LOCKED)
goto out;
event++;
if (flags & MNT_DETACH) {
if (mnt->mnt.mnt_flags & MNT_ONRB ||
!list_empty(&mnt->mnt_list))
umount_tree(mnt, UMOUNT_PROPAGATE);
retval = 0;
} else {
shrink_submounts(mnt);
retval = -EBUSY;
if (!propagate_mount_busy(mnt, 2)) {
if (mnt->mnt.mnt_flags & MNT_ONRB ||
!list_empty(&mnt->mnt_list))
umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC);
retval = 0;
}
}
out:
unlock_mount_hash();
namespace_unlock();
return retval;
}
/*
* __detach_mounts - lazily unmount all mounts on the specified dentry
*
* During unlink, rmdir, and d_drop it is possible to loose the path
* to an existing mountpoint, and wind up leaking the mount.
* detach_mounts allows lazily unmounting those mounts instead of
* leaking them.
*
* The caller may hold dentry->d_inode->i_mutex.
*/
void __detach_mounts(struct dentry *dentry)
{
struct mountpoint *mp;
struct mount *mnt;
namespace_lock();
lock_mount_hash();
mp = lookup_mountpoint(dentry);
if (!mp)
goto out_unlock;
event++;
while (!hlist_empty(&mp->m_list)) {
mnt = hlist_entry(mp->m_list.first, struct mount, mnt_mp_list);
if (mnt->mnt.mnt_flags & MNT_UMOUNT) {
umount_mnt(mnt);
hlist_add_head(&mnt->mnt_umount, &unmounted);
}
else umount_tree(mnt, UMOUNT_CONNECTED);
}
put_mountpoint(mp);
out_unlock:
unlock_mount_hash();
namespace_unlock();
}
/*
* Is the caller allowed to modify his namespace?
*/
bool may_mount(void)
{
return ns_capable(current->nsproxy->mnt_ns->user_ns, CAP_SYS_ADMIN);
}
/**
* path_mounted - check whether path is mounted
* @path: path to check
*
* Determine whether @path refers to the root of a mount.
*
* Return: true if @path is the root of a mount, false if not.
*/
static inline bool path_mounted(const struct path *path)
{
return path->mnt->mnt_root == path->dentry;
}
static void warn_mandlock(void)
{
pr_warn_once("=======================================================\n"
"WARNING: The mand mount option has been deprecated and\n"
" and is ignored by this kernel. Remove the mand\n"
" option from the mount to silence this warning.\n"
"=======================================================\n");
}
static int can_umount(const struct path *path, int flags)
{
struct mount *mnt = real_mount(path->mnt);
if (!may_mount())
return -EPERM;
if (!path_mounted(path))
return -EINVAL;
if (!check_mnt(mnt))
return -EINVAL;
if (mnt->mnt.mnt_flags & MNT_LOCKED) /* Check optimistically */
return -EINVAL;
if (flags & MNT_FORCE && !capable(CAP_SYS_ADMIN))
return -EPERM;
return 0;
}
// caller is responsible for flags being sane
int path_umount(struct path *path, int flags)
{
struct mount *mnt = real_mount(path->mnt);
int ret;
ret = can_umount(path, flags);
if (!ret)
ret = do_umount(mnt, flags);
/* we mustn't call path_put() as that would clear mnt_expiry_mark */
dput(path->dentry);
mntput_no_expire(mnt);
return ret;
}
static int ksys_umount(char __user *name, int flags)
{
int lookup_flags = LOOKUP_MOUNTPOINT;
struct path path;
int ret;
// basic validity checks done first
if (flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW))
return -EINVAL;
if (!(flags & UMOUNT_NOFOLLOW))
lookup_flags |= LOOKUP_FOLLOW;
ret = user_path_at(AT_FDCWD, name, lookup_flags, &path);
if (ret)
return ret;
return path_umount(&path, flags);
}
SYSCALL_DEFINE2(umount, char __user *, name, int, flags)
{
return ksys_umount(name, flags);
}
#ifdef __ARCH_WANT_SYS_OLDUMOUNT
/*
* The 2.0 compatible umount. No flags.
*/
SYSCALL_DEFINE1(oldumount, char __user *, name)
{
return ksys_umount(name, 0);
}
#endif
static bool is_mnt_ns_file(struct dentry *dentry)
{
/* Is this a proxy for a mount namespace? */
return dentry->d_op == &ns_dentry_operations &&
dentry->d_fsdata == &mntns_operations;
}
static struct mnt_namespace *to_mnt_ns(struct ns_common *ns)
{
return container_of(ns, struct mnt_namespace, ns);
}
struct ns_common *from_mnt_ns(struct mnt_namespace *mnt)
{
return &mnt->ns;
}
static bool mnt_ns_loop(struct dentry *dentry)
{
/* Could bind mounting the mount namespace inode cause a
* mount namespace loop?
*/
struct mnt_namespace *mnt_ns;
if (!is_mnt_ns_file(dentry))
return false;
mnt_ns = to_mnt_ns(get_proc_ns(dentry->d_inode));
return current->nsproxy->mnt_ns->seq >= mnt_ns->seq;
}
struct mount *copy_tree(struct mount *mnt, struct dentry *dentry,
int flag)
{
struct mount *res, *p, *q, *r, *parent;
if (!(flag & CL_COPY_UNBINDABLE) && IS_MNT_UNBINDABLE(mnt))
return ERR_PTR(-EINVAL);
if (!(flag & CL_COPY_MNT_NS_FILE) && is_mnt_ns_file(dentry))
return ERR_PTR(-EINVAL);
res = q = clone_mnt(mnt, dentry, flag);
if (IS_ERR(q))
return q;
q->mnt_mountpoint = mnt->mnt_mountpoint;
p = mnt;
list_for_each_entry(r, &mnt->mnt_mounts, mnt_child) {
struct mount *s;
if (!is_subdir(r->mnt_mountpoint, dentry))
continue;
for (s = r; s; s = next_mnt(s, r)) {
if (!(flag & CL_COPY_UNBINDABLE) &&
IS_MNT_UNBINDABLE(s)) {
if (s->mnt.mnt_flags & MNT_LOCKED) {
/* Both unbindable and locked. */
q = ERR_PTR(-EPERM);
goto out;
} else {
s = skip_mnt_tree(s);
continue;
}
}
if (!(flag & CL_COPY_MNT_NS_FILE) &&
is_mnt_ns_file(s->mnt.mnt_root)) {
s = skip_mnt_tree(s);
continue;
}
while (p != s->mnt_parent) {
p = p->mnt_parent;
q = q->mnt_parent;
}
p = s;
parent = q;
q = clone_mnt(p, p->mnt.mnt_root, flag);
if (IS_ERR(q))
goto out;
lock_mount_hash();
list_add_tail(&q->mnt_list, &res->mnt_list);
attach_mnt(q, parent, p->mnt_mp, false);
unlock_mount_hash();
}
}
return res;
out:
if (res) {
lock_mount_hash();
umount_tree(res, UMOUNT_SYNC);
unlock_mount_hash();
}
return q;
}
/* Caller should check returned pointer for errors */
struct vfsmount *collect_mounts(const struct path *path)
{
struct mount *tree;
namespace_lock();
if (!check_mnt(real_mount(path->mnt)))
tree = ERR_PTR(-EINVAL);
else
tree = copy_tree(real_mount(path->mnt), path->dentry,
CL_COPY_ALL | CL_PRIVATE);
namespace_unlock();
if (IS_ERR(tree))
return ERR_CAST(tree);
return &tree->mnt;
}
static void free_mnt_ns(struct mnt_namespace *);
static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *, bool);
void dissolve_on_fput(struct vfsmount *mnt)
{
struct mnt_namespace *ns;
namespace_lock();
lock_mount_hash();
ns = real_mount(mnt)->mnt_ns;
if (ns) {
if (is_anon_ns(ns))
umount_tree(real_mount(mnt), UMOUNT_CONNECTED);
else
ns = NULL;
}
unlock_mount_hash();
namespace_unlock();
if (ns)
free_mnt_ns(ns);
}
void drop_collected_mounts(struct vfsmount *mnt)
{
namespace_lock();
lock_mount_hash();
umount_tree(real_mount(mnt), 0);
unlock_mount_hash();
namespace_unlock();
}
static bool has_locked_children(struct mount *mnt, struct dentry *dentry)
{
struct mount *child;
list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) {
if (!is_subdir(child->mnt_mountpoint, dentry))
continue;
if (child->mnt.mnt_flags & MNT_LOCKED)
return true;
}
return false;
}
/**
* clone_private_mount - create a private clone of a path
* @path: path to clone
*
* This creates a new vfsmount, which will be the clone of @path. The new mount
* will not be attached anywhere in the namespace and will be private (i.e.
* changes to the originating mount won't be propagated into this).
*
* Release with mntput().
*/
struct vfsmount *clone_private_mount(const struct path *path)
{
struct mount *old_mnt = real_mount(path->mnt);
struct mount *new_mnt;
down_read(&namespace_sem);
if (IS_MNT_UNBINDABLE(old_mnt))
goto invalid;
if (!check_mnt(old_mnt))
goto invalid;
if (has_locked_children(old_mnt, path->dentry))
goto invalid;
new_mnt = clone_mnt(old_mnt, path->dentry, CL_PRIVATE);
up_read(&namespace_sem);
if (IS_ERR(new_mnt))
return ERR_CAST(new_mnt);
/* Longterm mount to be removed by kern_unmount*() */
new_mnt->mnt_ns = MNT_NS_INTERNAL;
return &new_mnt->mnt;
invalid:
up_read(&namespace_sem);
return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(clone_private_mount);
int iterate_mounts(int (*f)(struct vfsmount *, void *), void *arg,
struct vfsmount *root)
{
struct mount *mnt;
int res = f(root, arg);
if (res)
return res;
list_for_each_entry(mnt, &real_mount(root)->mnt_list, mnt_list) {
res = f(&mnt->mnt, arg);
if (res)
return res;
}
return 0;
}
static void lock_mnt_tree(struct mount *mnt)
{
struct mount *p;
for (p = mnt; p; p = next_mnt(p, mnt)) {
int flags = p->mnt.mnt_flags;
/* Don't allow unprivileged users to change mount flags */
flags |= MNT_LOCK_ATIME;
if (flags & MNT_READONLY)
flags |= MNT_LOCK_READONLY;
if (flags & MNT_NODEV)
flags |= MNT_LOCK_NODEV;
if (flags & MNT_NOSUID)
flags |= MNT_LOCK_NOSUID;
if (flags & MNT_NOEXEC)
flags |= MNT_LOCK_NOEXEC;
/* Don't allow unprivileged users to reveal what is under a mount */
if (list_empty(&p->mnt_expire))
flags |= MNT_LOCKED;
p->mnt.mnt_flags = flags;
}
}
static void cleanup_group_ids(struct mount *mnt, struct mount *end)
{
struct mount *p;
for (p = mnt; p != end; p = next_mnt(p, mnt)) {
if (p->mnt_group_id && !IS_MNT_SHARED(p))
mnt_release_group_id(p);
}
}
static int invent_group_ids(struct mount *mnt, bool recurse)
{
struct mount *p;
for (p = mnt; p; p = recurse ? next_mnt(p, mnt) : NULL) {
if (!p->mnt_group_id && !IS_MNT_SHARED(p)) {
int err = mnt_alloc_group_id(p);
if (err) {
cleanup_group_ids(mnt, p);
return err;
}
}
}
return 0;
}
int count_mounts(struct mnt_namespace *ns, struct mount *mnt)
{
unsigned int max = READ_ONCE(sysctl_mount_max);
unsigned int mounts = 0;
struct mount *p;
if (ns->nr_mounts >= max)
return -ENOSPC;
max -= ns->nr_mounts;
if (ns->pending_mounts >= max)
return -ENOSPC;
max -= ns->pending_mounts;
for (p = mnt; p; p = next_mnt(p, mnt))
mounts++;
if (mounts > max)
return -ENOSPC;
ns->pending_mounts += mounts;
return 0;
}
enum mnt_tree_flags_t {
MNT_TREE_MOVE = BIT(0),
MNT_TREE_BENEATH = BIT(1),
};
/**
* attach_recursive_mnt - attach a source mount tree
* @source_mnt: mount tree to be attached
* @top_mnt: mount that @source_mnt will be mounted on or mounted beneath
* @dest_mp: the mountpoint @source_mnt will be mounted at
* @flags: modify how @source_mnt is supposed to be attached
*
* NOTE: in the table below explains the semantics when a source mount
* of a given type is attached to a destination mount of a given type.
* ---------------------------------------------------------------------------
* | BIND MOUNT OPERATION |
* |**************************************************************************
* | source-->| shared | private | slave | unbindable |
* | dest | | | | |
* | | | | | | |
* | v | | | | |
* |**************************************************************************
* | shared | shared (++) | shared (+) | shared(+++)| invalid |
* | | | | | |
* |non-shared| shared (+) | private | slave (*) | invalid |
* ***************************************************************************
* A bind operation clones the source mount and mounts the clone on the
* destination mount.
*
* (++) the cloned mount is propagated to all the mounts in the propagation
* tree of the destination mount and the cloned mount is added to
* the peer group of the source mount.
* (+) the cloned mount is created under the destination mount and is marked
* as shared. The cloned mount is added to the peer group of the source
* mount.
* (+++) the mount is propagated to all the mounts in the propagation tree
* of the destination mount and the cloned mount is made slave
* of the same master as that of the source mount. The cloned mount
* is marked as 'shared and slave'.
* (*) the cloned mount is made a slave of the same master as that of the
* source mount.
*
* ---------------------------------------------------------------------------
* | MOVE MOUNT OPERATION |
* |**************************************************************************
* | source-->| shared | private | slave | unbindable |
* | dest | | | | |
* | | | | | | |
* | v | | | | |
* |**************************************************************************
* | shared | shared (+) | shared (+) | shared(+++) | invalid |
* | | | | | |
* |non-shared| shared (+*) | private | slave (*) | unbindable |
* ***************************************************************************
*
* (+) the mount is moved to the destination. And is then propagated to
* all the mounts in the propagation tree of the destination mount.
* (+*) the mount is moved to the destination.
* (+++) the mount is moved to the destination and is then propagated to
* all the mounts belonging to the destination mount's propagation tree.
* the mount is marked as 'shared and slave'.
* (*) the mount continues to be a slave at the new location.
*
* if the source mount is a tree, the operations explained above is
* applied to each mount in the tree.
* Must be called without spinlocks held, since this function can sleep
* in allocations.
*
* Context: The function expects namespace_lock() to be held.
* Return: If @source_mnt was successfully attached 0 is returned.
* Otherwise a negative error code is returned.
*/
static int attach_recursive_mnt(struct mount *source_mnt,
struct mount *top_mnt,
struct mountpoint *dest_mp,
enum mnt_tree_flags_t flags)
{
struct user_namespace *user_ns = current->nsproxy->mnt_ns->user_ns;
HLIST_HEAD(tree_list);
struct mnt_namespace *ns = top_mnt->mnt_ns;
struct mountpoint *smp;
struct mount *child, *dest_mnt, *p;
struct hlist_node *n;
int err = 0;
bool moving = flags & MNT_TREE_MOVE, beneath = flags & MNT_TREE_BENEATH;
/*
* Preallocate a mountpoint in case the new mounts need to be
* mounted beneath mounts on the same mountpoint.
*/
smp = get_mountpoint(source_mnt->mnt.mnt_root);
if (IS_ERR(smp))
return PTR_ERR(smp);
/* Is there space to add these mounts to the mount namespace? */
if (!moving) {
err = count_mounts(ns, source_mnt);
if (err)
goto out;
}
if (beneath)
dest_mnt = top_mnt->mnt_parent;
else
dest_mnt = top_mnt;
if (IS_MNT_SHARED(dest_mnt)) {
err = invent_group_ids(source_mnt, true);
if (err)
goto out;
err = propagate_mnt(dest_mnt, dest_mp, source_mnt, &tree_list);
}
lock_mount_hash();
if (err)
goto out_cleanup_ids;
if (IS_MNT_SHARED(dest_mnt)) {
for (p = source_mnt; p; p = next_mnt(p, source_mnt))
set_mnt_shared(p);
}
if (moving) {
if (beneath)
dest_mp = smp;
unhash_mnt(source_mnt);
attach_mnt(source_mnt, top_mnt, dest_mp, beneath);
touch_mnt_namespace(source_mnt->mnt_ns);
} else {
if (source_mnt->mnt_ns) {
LIST_HEAD(head);
/* move from anon - the caller will destroy */
for (p = source_mnt; p; p = next_mnt(p, source_mnt))
move_from_ns(p, &head);
list_del_init(&head);
}
if (beneath)
mnt_set_mountpoint_beneath(source_mnt, top_mnt, smp);
else
mnt_set_mountpoint(dest_mnt, dest_mp, source_mnt);
commit_tree(source_mnt);
}
hlist_for_each_entry_safe(child, n, &tree_list, mnt_hash) {
struct mount *q;
hlist_del_init(&child->mnt_hash);
q = __lookup_mnt(&child->mnt_parent->mnt,
child->mnt_mountpoint);
if (q)
mnt_change_mountpoint(child, smp, q);
/* Notice when we are propagating across user namespaces */
if (child->mnt_parent->mnt_ns->user_ns != user_ns)
lock_mnt_tree(child);
child->mnt.mnt_flags &= ~MNT_LOCKED;
commit_tree(child);
}
put_mountpoint(smp);
unlock_mount_hash();
return 0;
out_cleanup_ids:
while (!hlist_empty(&tree_list)) {
child = hlist_entry(tree_list.first, struct mount, mnt_hash);
child->mnt_parent->mnt_ns->pending_mounts = 0;
umount_tree(child, UMOUNT_SYNC);
}
unlock_mount_hash();
cleanup_group_ids(source_mnt, NULL);
out:
ns->pending_mounts = 0;
read_seqlock_excl(&mount_lock);
put_mountpoint(smp);
read_sequnlock_excl(&mount_lock);
return err;
}
/**
* do_lock_mount - lock mount and mountpoint
* @path: target path
* @beneath: whether the intention is to mount beneath @path
*
* Follow the mount stack on @path until the top mount @mnt is found. If
* the initial @path->{mnt,dentry} is a mountpoint lookup the first
* mount stacked on top of it. Then simply follow @{mnt,mnt->mnt_root}
* until nothing is stacked on top of it anymore.
*
* Acquire the inode_lock() on the top mount's ->mnt_root to protect
* against concurrent removal of the new mountpoint from another mount
* namespace.
*
* If @beneath is requested, acquire inode_lock() on @mnt's mountpoint
* @mp on @mnt->mnt_parent must be acquired. This protects against a
* concurrent unlink of @mp->mnt_dentry from another mount namespace
* where @mnt doesn't have a child mount mounted @mp. A concurrent
* removal of @mnt->mnt_root doesn't matter as nothing will be mounted
* on top of it for @beneath.
*
* In addition, @beneath needs to make sure that @mnt hasn't been
* unmounted or moved from its current mountpoint in between dropping
* @mount_lock and acquiring @namespace_sem. For the !@beneath case @mnt
* being unmounted would be detected later by e.g., calling
* check_mnt(mnt) in the function it's called from. For the @beneath
* case however, it's useful to detect it directly in do_lock_mount().
* If @mnt hasn't been unmounted then @mnt->mnt_mountpoint still points
* to @mnt->mnt_mp->m_dentry. But if @mnt has been unmounted it will
* point to @mnt->mnt_root and @mnt->mnt_mp will be NULL.
*
* Return: Either the target mountpoint on the top mount or the top
* mount's mountpoint.
*/
static struct mountpoint *do_lock_mount(struct path *path, bool beneath)
{
struct vfsmount *mnt = path->mnt;
struct dentry *dentry;
struct mountpoint *mp = ERR_PTR(-ENOENT);
for (;;) {
struct mount *m;
if (beneath) {
m = real_mount(mnt);
read_seqlock_excl(&mount_lock);
dentry = dget(m->mnt_mountpoint);
read_sequnlock_excl(&mount_lock);
} else {
dentry = path->dentry;
}
inode_lock(dentry->d_inode);
if (unlikely(cant_mount(dentry))) {
inode_unlock(dentry->d_inode);
goto out;
}
namespace_lock();
if (beneath && (!is_mounted(mnt) || m->mnt_mountpoint != dentry)) {
namespace_unlock();
inode_unlock(dentry->d_inode);
goto out;
}
mnt = lookup_mnt(path);
if (likely(!mnt))
break;
namespace_unlock();
inode_unlock(dentry->d_inode);
if (beneath)
dput(dentry);
path_put(path);
path->mnt = mnt;
path->dentry = dget(mnt->mnt_root);
}
mp = get_mountpoint(dentry);
if (IS_ERR(mp)) {
namespace_unlock();
inode_unlock(dentry->d_inode);
}
out:
if (beneath)
dput(dentry);
return mp;
}
static inline struct mountpoint *lock_mount(struct path *path)
{
return do_lock_mount(path, false);
}
static void unlock_mount(struct mountpoint *where)
{
struct dentry *dentry = where->m_dentry;
read_seqlock_excl(&mount_lock);
put_mountpoint(where);
read_sequnlock_excl(&mount_lock);
namespace_unlock();
inode_unlock(dentry->d_inode);
}
static int graft_tree(struct mount *mnt, struct mount *p, struct mountpoint *mp)
{
if (mnt->mnt.mnt_sb->s_flags & SB_NOUSER)
return -EINVAL;
if (d_is_dir(mp->m_dentry) !=
d_is_dir(mnt->mnt.mnt_root))
return -ENOTDIR;
return attach_recursive_mnt(mnt, p, mp, 0);
}
/*
* Sanity check the flags to change_mnt_propagation.
*/
static int flags_to_propagation_type(int ms_flags)
{
int type = ms_flags & ~(MS_REC | MS_SILENT);
/* Fail if any non-propagation flags are set */
if (type & ~(MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE))
return 0;
/* Only one propagation flag should be set */
if (!is_power_of_2(type))
return 0;
return type;
}
/*
* recursively change the type of the mountpoint.
*/
static int do_change_type(struct path *path, int ms_flags)
{
struct mount *m;
struct mount *mnt = real_mount(path->mnt);
int recurse = ms_flags & MS_REC;
int type;
int err = 0;
if (!path_mounted(path))
return -EINVAL;
type = flags_to_propagation_type(ms_flags);
if (!type)
return -EINVAL;
namespace_lock();
if (type == MS_SHARED) {
err = invent_group_ids(mnt, recurse);
if (err)
goto out_unlock;
}
lock_mount_hash();
for (m = mnt; m; m = (recurse ? next_mnt(m, mnt) : NULL))
change_mnt_propagation(m, type);
unlock_mount_hash();
out_unlock:
namespace_unlock();
return err;
}
static struct mount *__do_loopback(struct path *old_path, int recurse)
{
struct mount *mnt = ERR_PTR(-EINVAL), *old = real_mount(old_path->mnt);
if (IS_MNT_UNBINDABLE(old))
return mnt;
if (!check_mnt(old) && old_path->dentry->d_op != &ns_dentry_operations)
return mnt;
if (!recurse && has_locked_children(old, old_path->dentry))
return mnt;
if (recurse)
mnt = copy_tree(old, old_path->dentry, CL_COPY_MNT_NS_FILE);
else
mnt = clone_mnt(old, old_path->dentry, 0);
if (!IS_ERR(mnt))
mnt->mnt.mnt_flags &= ~MNT_LOCKED;
return mnt;
}
/*
* do loopback mount.
*/
static int do_loopback(struct path *path, const char *old_name,
int recurse)
{
struct path old_path;
struct mount *mnt = NULL, *parent;
struct mountpoint *mp;
int err;
if (!old_name || !*old_name)
return -EINVAL;
err = kern_path(old_name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &old_path);
if (err)
return err;
err = -EINVAL;
if (mnt_ns_loop(old_path.dentry))
goto out;
mp = lock_mount(path);
if (IS_ERR(mp)) {
err = PTR_ERR(mp);
goto out;
}
parent = real_mount(path->mnt);
if (!check_mnt(parent))
goto out2;
mnt = __do_loopback(&old_path, recurse);
if (IS_ERR(mnt)) {
err = PTR_ERR(mnt);
goto out2;
}
err = graft_tree(mnt, parent, mp);
if (err) {
lock_mount_hash();
umount_tree(mnt, UMOUNT_SYNC);
unlock_mount_hash();
}
out2:
unlock_mount(mp);
out:
path_put(&old_path);
return err;
}
static struct file *open_detached_copy(struct path *path, bool recursive)
{
struct user_namespace *user_ns = current->nsproxy->mnt_ns->user_ns;
struct mnt_namespace *ns = alloc_mnt_ns(user_ns, true);
struct mount *mnt, *p;
struct file *file;
if (IS_ERR(ns))
return ERR_CAST(ns);
namespace_lock();
mnt = __do_loopback(path, recursive);
if (IS_ERR(mnt)) {
namespace_unlock();
free_mnt_ns(ns);
return ERR_CAST(mnt);
}
lock_mount_hash();
for (p = mnt; p; p = next_mnt(p, mnt)) {
mnt_add_to_ns(ns, p);
ns->nr_mounts++;
}
ns->root = mnt;
mntget(&mnt->mnt);
unlock_mount_hash();
namespace_unlock();
mntput(path->mnt);
path->mnt = &mnt->mnt;
file = dentry_open(path, O_PATH, current_cred());
if (IS_ERR(file))
dissolve_on_fput(path->mnt);
else
file->f_mode |= FMODE_NEED_UNMOUNT;
return file;
}
SYSCALL_DEFINE3(open_tree, int, dfd, const char __user *, filename, unsigned, flags)
{
struct file *file;
struct path path;
int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW;
bool detached = flags & OPEN_TREE_CLONE;
int error;
int fd;
BUILD_BUG_ON(OPEN_TREE_CLOEXEC != O_CLOEXEC);
if (flags & ~(AT_EMPTY_PATH | AT_NO_AUTOMOUNT | AT_RECURSIVE |
AT_SYMLINK_NOFOLLOW | OPEN_TREE_CLONE |
OPEN_TREE_CLOEXEC))
return -EINVAL;
if ((flags & (AT_RECURSIVE | OPEN_TREE_CLONE)) == AT_RECURSIVE)
return -EINVAL;
if (flags & AT_NO_AUTOMOUNT)
lookup_flags &= ~LOOKUP_AUTOMOUNT;
if (flags & AT_SYMLINK_NOFOLLOW)
lookup_flags &= ~LOOKUP_FOLLOW;
if (flags & AT_EMPTY_PATH)
lookup_flags |= LOOKUP_EMPTY;
if (detached && !may_mount())
return -EPERM;
fd = get_unused_fd_flags(flags & O_CLOEXEC);
if (fd < 0)
return fd;
error = user_path_at(dfd, filename, lookup_flags, &path);
if (unlikely(error)) {
file = ERR_PTR(error);
} else {
if (detached)
file = open_detached_copy(&path, flags & AT_RECURSIVE);
else
file = dentry_open(&path, O_PATH, current_cred());
path_put(&path);
}
if (IS_ERR(file)) {
put_unused_fd(fd);
return PTR_ERR(file);
}
fd_install(fd, file);
return fd;
}
/*
* Don't allow locked mount flags to be cleared.
*
* No locks need to be held here while testing the various MNT_LOCK
* flags because those flags can never be cleared once they are set.
*/
static bool can_change_locked_flags(struct mount *mnt, unsigned int mnt_flags)
{
unsigned int fl = mnt->mnt.mnt_flags;
if ((fl & MNT_LOCK_READONLY) &&
!(mnt_flags & MNT_READONLY))
return false;
if ((fl & MNT_LOCK_NODEV) &&
!(mnt_flags & MNT_NODEV))
return false;
if ((fl & MNT_LOCK_NOSUID) &&
!(mnt_flags & MNT_NOSUID))
return false;
if ((fl & MNT_LOCK_NOEXEC) &&
!(mnt_flags & MNT_NOEXEC))
return false;
if ((fl & MNT_LOCK_ATIME) &&
((fl & MNT_ATIME_MASK) != (mnt_flags & MNT_ATIME_MASK)))
return false;
return true;
}
static int change_mount_ro_state(struct mount *mnt, unsigned int mnt_flags)
{
bool readonly_request = (mnt_flags & MNT_READONLY);
if (readonly_request == __mnt_is_readonly(&mnt->mnt))
return 0;
if (readonly_request)
return mnt_make_readonly(mnt);
mnt->mnt.mnt_flags &= ~MNT_READONLY;
return 0;
}
static void set_mount_attributes(struct mount *mnt, unsigned int mnt_flags)
{
mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK;
mnt->mnt.mnt_flags = mnt_flags;
touch_mnt_namespace(mnt->mnt_ns);
}
static void mnt_warn_timestamp_expiry(struct path *mountpoint, struct vfsmount *mnt)
{
struct super_block *sb = mnt->mnt_sb;
if (!__mnt_is_readonly(mnt) &&
(!(sb->s_iflags & SB_I_TS_EXPIRY_WARNED)) &&
(ktime_get_real_seconds() + TIME_UPTIME_SEC_MAX > sb->s_time_max)) {
char *buf = (char *)__get_free_page(GFP_KERNEL);
char *mntpath = buf ? d_path(mountpoint, buf, PAGE_SIZE) : ERR_PTR(-ENOMEM);
pr_warn("%s filesystem being %s at %s supports timestamps until %ptTd (0x%llx)\n",
sb->s_type->name,
is_mounted(mnt) ? "remounted" : "mounted",
mntpath, &sb->s_time_max,
(unsigned long long)sb->s_time_max);
free_page((unsigned long)buf);
sb->s_iflags |= SB_I_TS_EXPIRY_WARNED;
}
}
/*
* Handle reconfiguration of the mountpoint only without alteration of the
* superblock it refers to. This is triggered by specifying MS_REMOUNT|MS_BIND
* to mount(2).
*/
static int do_reconfigure_mnt(struct path *path, unsigned int mnt_flags)
{
struct super_block *sb = path->mnt->mnt_sb;
struct mount *mnt = real_mount(path->mnt);
int ret;
if (!check_mnt(mnt))
return -EINVAL;
if (!path_mounted(path))
return -EINVAL;
if (!can_change_locked_flags(mnt, mnt_flags))
return -EPERM;
/*
* We're only checking whether the superblock is read-only not
* changing it, so only take down_read(&sb->s_umount).
*/
down_read(&sb->s_umount);
lock_mount_hash();
ret = change_mount_ro_state(mnt, mnt_flags);
if (ret == 0)
set_mount_attributes(mnt, mnt_flags);
unlock_mount_hash();
up_read(&sb->s_umount);
mnt_warn_timestamp_expiry(path, &mnt->mnt);
return ret;
}
/*
* change filesystem flags. dir should be a physical root of filesystem.
* If you've mounted a non-root directory somewhere and want to do remount
* on it - tough luck.
*/
static int do_remount(struct path *path, int ms_flags, int sb_flags,
int mnt_flags, void *data)
{
int err;
struct super_block *sb = path->mnt->mnt_sb;
struct mount *mnt = real_mount(path->mnt);
struct fs_context *fc;
if (!check_mnt(mnt))
return -EINVAL;
if (!path_mounted(path))
return -EINVAL;
if (!can_change_locked_flags(mnt, mnt_flags))
return -EPERM;
fc = fs_context_for_reconfigure(path->dentry, sb_flags, MS_RMT_MASK);
if (IS_ERR(fc))
return PTR_ERR(fc);
/*
* Indicate to the filesystem that the remount request is coming
* from the legacy mount system call.
*/
fc->oldapi = true;
err = parse_monolithic_mount_data(fc, data);
if (!err) {
down_write(&sb->s_umount);
err = -EPERM;
if (ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) {
err = reconfigure_super(fc);
if (!err) {
lock_mount_hash();
set_mount_attributes(mnt, mnt_flags);
unlock_mount_hash();
}
}
up_write(&sb->s_umount);
}
mnt_warn_timestamp_expiry(path, &mnt->mnt);
put_fs_context(fc);
return err;
}
static inline int tree_contains_unbindable(struct mount *mnt)
{
struct mount *p;
for (p = mnt; p; p = next_mnt(p, mnt)) {
if (IS_MNT_UNBINDABLE(p))
return 1;
}
return 0;
}
/*
* Check that there aren't references to earlier/same mount namespaces in the
* specified subtree. Such references can act as pins for mount namespaces
* that aren't checked by the mount-cycle checking code, thereby allowing
* cycles to be made.
*/
static bool check_for_nsfs_mounts(struct mount *subtree)
{
struct mount *p;
bool ret = false;
lock_mount_hash();
for (p = subtree; p; p = next_mnt(p, subtree))
if (mnt_ns_loop(p->mnt.mnt_root))
goto out;
ret = true;
out:
unlock_mount_hash();
return ret;
}
static int do_set_group(struct path *from_path, struct path *to_path)
{
struct mount *from, *to;
int err;
from = real_mount(from_path->mnt);
to = real_mount(to_path->mnt);
namespace_lock();
err = -EINVAL;
/* To and From must be mounted */
if (!is_mounted(&from->mnt))
goto out;
if (!is_mounted(&to->mnt))
goto out;
err = -EPERM;
/* We should be allowed to modify mount namespaces of both mounts */
if (!ns_capable(from->mnt_ns->user_ns, CAP_SYS_ADMIN))
goto out;
if (!ns_capable(to->mnt_ns->user_ns, CAP_SYS_ADMIN))
goto out;
err = -EINVAL;
/* To and From paths should be mount roots */
if (!path_mounted(from_path))
goto out;
if (!path_mounted(to_path))
goto out;
/* Setting sharing groups is only allowed across same superblock */
if (from->mnt.mnt_sb != to->mnt.mnt_sb)
goto out;
/* From mount root should be wider than To mount root */
if (!is_subdir(to->mnt.mnt_root, from->mnt.mnt_root))
goto out;
/* From mount should not have locked children in place of To's root */
if (has_locked_children(from, to->mnt.mnt_root))
goto out;
/* Setting sharing groups is only allowed on private mounts */
if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to))
goto out;
/* From should not be private */
if (!IS_MNT_SHARED(from) && !IS_MNT_SLAVE(from))
goto out;
if (IS_MNT_SLAVE(from)) {
struct mount *m = from->mnt_master;
list_add(&to->mnt_slave, &m->mnt_slave_list);
to->mnt_master = m;
}
if (IS_MNT_SHARED(from)) {
to->mnt_group_id = from->mnt_group_id;
list_add(&to->mnt_share, &from->mnt_share);
lock_mount_hash();
set_mnt_shared(to);
unlock_mount_hash();
}
err = 0;
out:
namespace_unlock();
return err;
}
/**
* path_overmounted - check if path is overmounted
* @path: path to check
*
* Check if path is overmounted, i.e., if there's a mount on top of
* @path->mnt with @path->dentry as mountpoint.
*
* Context: This function expects namespace_lock() to be held.
* Return: If path is overmounted true is returned, false if not.
*/
static inline bool path_overmounted(const struct path *path)
{
rcu_read_lock();
if (unlikely(__lookup_mnt(path->mnt, path->dentry))) {
rcu_read_unlock();
return true;
}
rcu_read_unlock();
return false;
}
/**
* can_move_mount_beneath - check that we can mount beneath the top mount
* @from: mount to mount beneath
* @to: mount under which to mount
* @mp: mountpoint of @to
*
* - Make sure that @to->dentry is actually the root of a mount under
* which we can mount another mount.
* - Make sure that nothing can be mounted beneath the caller's current
* root or the rootfs of the namespace.
* - Make sure that the caller can unmount the topmost mount ensuring
* that the caller could reveal the underlying mountpoint.
* - Ensure that nothing has been mounted on top of @from before we
* grabbed @namespace_sem to avoid creating pointless shadow mounts.
* - Prevent mounting beneath a mount if the propagation relationship
* between the source mount, parent mount, and top mount would lead to
* nonsensical mount trees.
*
* Context: This function expects namespace_lock() to be held.
* Return: On success 0, and on error a negative error code is returned.
*/
static int can_move_mount_beneath(const struct path *from,
const struct path *to,
const struct mountpoint *mp)
{
struct mount *mnt_from = real_mount(from->mnt),
*mnt_to = real_mount(to->mnt),
*parent_mnt_to = mnt_to->mnt_parent;
if (!mnt_has_parent(mnt_to))
return -EINVAL;
if (!path_mounted(to))
return -EINVAL;
if (IS_MNT_LOCKED(mnt_to))
return -EINVAL;
/* Avoid creating shadow mounts during mount propagation. */
if (path_overmounted(from))
return -EINVAL;
/*
* Mounting beneath the rootfs only makes sense when the
* semantics of pivot_root(".", ".") are used.
*/
if (&mnt_to->mnt == current->fs->root.mnt)
return -EINVAL;
if (parent_mnt_to == current->nsproxy->mnt_ns->root)
return -EINVAL;
for (struct mount *p = mnt_from; mnt_has_parent(p); p = p->mnt_parent)
if (p == mnt_to)
return -EINVAL;
/*
* If the parent mount propagates to the child mount this would
* mean mounting @mnt_from on @mnt_to->mnt_parent and then
* propagating a copy @c of @mnt_from on top of @mnt_to. This
* defeats the whole purpose of mounting beneath another mount.
*/
if (propagation_would_overmount(parent_mnt_to, mnt_to, mp))
return -EINVAL;
/*
* If @mnt_to->mnt_parent propagates to @mnt_from this would
* mean propagating a copy @c of @mnt_from on top of @mnt_from.
* Afterwards @mnt_from would be mounted on top of
* @mnt_to->mnt_parent and @mnt_to would be unmounted from
* @mnt->mnt_parent and remounted on @mnt_from. But since @c is
* already mounted on @mnt_from, @mnt_to would ultimately be
* remounted on top of @c. Afterwards, @mnt_from would be
* covered by a copy @c of @mnt_from and @c would be covered by
* @mnt_from itself. This defeats the whole purpose of mounting
* @mnt_from beneath @mnt_to.
*/
if (propagation_would_overmount(parent_mnt_to, mnt_from, mp))
return -EINVAL;
return 0;
}
static int do_move_mount(struct path *old_path, struct path *new_path,
bool beneath)
{
struct mnt_namespace *ns;
struct mount *p;
struct mount *old;
struct mount *parent;
struct mountpoint *mp, *old_mp;
int err;
bool attached;
enum mnt_tree_flags_t flags = 0;
mp = do_lock_mount(new_path, beneath);
if (IS_ERR(mp))
return PTR_ERR(mp);
old = real_mount(old_path->mnt);
p = real_mount(new_path->mnt);
parent = old->mnt_parent;
attached = mnt_has_parent(old);
if (attached)
flags |= MNT_TREE_MOVE;
old_mp = old->mnt_mp;
ns = old->mnt_ns;
err = -EINVAL;
/* The mountpoint must be in our namespace. */
if (!check_mnt(p))
goto out;
/* The thing moved must be mounted... */
if (!is_mounted(&old->mnt))
goto out;
/* ... and either ours or the root of anon namespace */
if (!(attached ? check_mnt(old) : is_anon_ns(ns)))
goto out;
if (old->mnt.mnt_flags & MNT_LOCKED)
goto out;
if (!path_mounted(old_path))
goto out;
if (d_is_dir(new_path->dentry) !=
d_is_dir(old_path->dentry))
goto out;
/*
* Don't move a mount residing in a shared parent.
*/
if (attached && IS_MNT_SHARED(parent))
goto out;
if (beneath) {
err = can_move_mount_beneath(old_path, new_path, mp);
if (err)
goto out;
err = -EINVAL;
p = p->mnt_parent;
flags |= MNT_TREE_BENEATH;
}
/*
* Don't move a mount tree containing unbindable mounts to a destination
* mount which is shared.
*/
if (IS_MNT_SHARED(p) && tree_contains_unbindable(old))
goto out;
err = -ELOOP;
if (!check_for_nsfs_mounts(old))
goto out;
for (; mnt_has_parent(p); p = p->mnt_parent)
if (p == old)
goto out;
err = attach_recursive_mnt(old, real_mount(new_path->mnt), mp, flags);
if (err)
goto out;
/* if the mount is moved, it should no longer be expire
* automatically */
list_del_init(&old->mnt_expire);
if (attached)
put_mountpoint(old_mp);
out:
unlock_mount(mp);
if (!err) {
if (attached)
mntput_no_expire(parent);
else
free_mnt_ns(ns);
}
return err;
}
static int do_move_mount_old(struct path *path, const char *old_name)
{
struct path old_path;
int err;
if (!old_name || !*old_name)
return -EINVAL;
err = kern_path(old_name, LOOKUP_FOLLOW, &old_path);
if (err)
return err;
err = do_move_mount(&old_path, path, false);
path_put(&old_path);
return err;
}
/*
* add a mount into a namespace's mount tree
*/
static int do_add_mount(struct mount *newmnt, struct mountpoint *mp,
const struct path *path, int mnt_flags)
{
struct mount *parent = real_mount(path->mnt);
mnt_flags &= ~MNT_INTERNAL_FLAGS;
if (unlikely(!check_mnt(parent))) {
/* that's acceptable only for automounts done in private ns */
if (!(mnt_flags & MNT_SHRINKABLE))
return -EINVAL;
/* ... and for those we'd better have mountpoint still alive */
if (!parent->mnt_ns)
return -EINVAL;
}
/* Refuse the same filesystem on the same mount point */
if (path->mnt->mnt_sb == newmnt->mnt.mnt_sb && path_mounted(path))
return -EBUSY;
if (d_is_symlink(newmnt->mnt.mnt_root))
return -EINVAL;
newmnt->mnt.mnt_flags = mnt_flags;
return graft_tree(newmnt, parent, mp);
}
static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags);
/*
* Create a new mount using a superblock configuration and request it
* be added to the namespace tree.
*/
static int do_new_mount_fc(struct fs_context *fc, struct path *mountpoint,
unsigned int mnt_flags)
{
struct vfsmount *mnt;
struct mountpoint *mp;
struct super_block *sb = fc->root->d_sb;
int error;
error = security_sb_kern_mount(sb);
if (!error && mount_too_revealing(sb, &mnt_flags))
error = -EPERM;
if (unlikely(error)) {
fc_drop_locked(fc);
return error;
}
up_write(&sb->s_umount);
mnt = vfs_create_mount(fc);
if (IS_ERR(mnt))
return PTR_ERR(mnt);
mnt_warn_timestamp_expiry(mountpoint, mnt);
mp = lock_mount(mountpoint);
if (IS_ERR(mp)) {
mntput(mnt);
return PTR_ERR(mp);
}
error = do_add_mount(real_mount(mnt), mp, mountpoint, mnt_flags);
unlock_mount(mp);
if (error < 0)
mntput(mnt);
return error;
}
/*
* create a new mount for userspace and request it to be added into the
* namespace's tree
*/
static int do_new_mount(struct path *path, const char *fstype, int sb_flags,
int mnt_flags, const char *name, void *data)
{
struct file_system_type *type;
struct fs_context *fc;
const char *subtype = NULL;
int err = 0;
if (!fstype)
return -EINVAL;
type = get_fs_type(fstype);
if (!type)
return -ENODEV;
if (type->fs_flags & FS_HAS_SUBTYPE) {
subtype = strchr(fstype, '.');
if (subtype) {
subtype++;
if (!*subtype) {
put_filesystem(type);
return -EINVAL;
}
}
}
fc = fs_context_for_mount(type, sb_flags);
put_filesystem(type);
if (IS_ERR(fc))
return PTR_ERR(fc);
/*
* Indicate to the filesystem that the mount request is coming
* from the legacy mount system call.
*/
fc->oldapi = true;
if (subtype)
err = vfs_parse_fs_string(fc, "subtype",
subtype, strlen(subtype));
if (!err && name)
err = vfs_parse_fs_string(fc, "source", name, strlen(name));
if (!err)
err = parse_monolithic_mount_data(fc, data);
if (!err && !mount_capable(fc))
err = -EPERM;
if (!err)
err = vfs_get_tree(fc);
if (!err)
err = do_new_mount_fc(fc, path, mnt_flags);
put_fs_context(fc);
return err;
}
int finish_automount(struct vfsmount *m, const struct path *path)
{
struct dentry *dentry = path->dentry;
struct mountpoint *mp;
struct mount *mnt;
int err;
if (!m)
return 0;
if (IS_ERR(m))
return PTR_ERR(m);
mnt = real_mount(m);
/* The new mount record should have at least 2 refs to prevent it being
* expired before we get a chance to add it
*/
BUG_ON(mnt_get_count(mnt) < 2);
if (m->mnt_sb == path->mnt->mnt_sb &&
m->mnt_root == dentry) {
err = -ELOOP;
goto discard;
}
/*
* we don't want to use lock_mount() - in this case finding something
* that overmounts our mountpoint to be means "quitely drop what we've
* got", not "try to mount it on top".
*/
inode_lock(dentry->d_inode);
namespace_lock();
if (unlikely(cant_mount(dentry))) {
err = -ENOENT;
goto discard_locked;
}
if (path_overmounted(path)) {
err = 0;
goto discard_locked;
}
mp = get_mountpoint(dentry);
if (IS_ERR(mp)) {
err = PTR_ERR(mp);
goto discard_locked;
}
err = do_add_mount(mnt, mp, path, path->mnt->mnt_flags | MNT_SHRINKABLE);
unlock_mount(mp);
if (unlikely(err))
goto discard;
mntput(m);
return 0;
discard_locked:
namespace_unlock();
inode_unlock(dentry->d_inode);
discard:
/* remove m from any expiration list it may be on */
if (!list_empty(&mnt->mnt_expire)) {
namespace_lock();
list_del_init(&mnt->mnt_expire);
namespace_unlock();
}
mntput(m);
mntput(m);
return err;
}
/**
* mnt_set_expiry - Put a mount on an expiration list
* @mnt: The mount to list.
* @expiry_list: The list to add the mount to.
*/
void mnt_set_expiry(struct vfsmount *mnt, struct list_head *expiry_list)
{
namespace_lock();
list_add_tail(&real_mount(mnt)->mnt_expire, expiry_list);
namespace_unlock();
}
EXPORT_SYMBOL(mnt_set_expiry);
/*
* process a list of expirable mountpoints with the intent of discarding any
* mountpoints that aren't in use and haven't been touched since last we came
* here
*/
void mark_mounts_for_expiry(struct list_head *mounts)
{
struct mount *mnt, *next;
LIST_HEAD(graveyard);
if (list_empty(mounts))
return;
namespace_lock();
lock_mount_hash();
/* extract from the expiration list every vfsmount that matches the
* following criteria:
* - only referenced by its parent vfsmount
* - still marked for expiry (marked on the last call here; marks are
* cleared by mntput())
*/
list_for_each_entry_safe(mnt, next, mounts, mnt_expire) {
if (!xchg(&mnt->mnt_expiry_mark, 1) ||
propagate_mount_busy(mnt, 1))
continue;
list_move(&mnt->mnt_expire, &graveyard);
}
while (!list_empty(&graveyard)) {
mnt = list_first_entry(&graveyard, struct mount, mnt_expire);
touch_mnt_namespace(mnt->mnt_ns);
umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC);
}
unlock_mount_hash();
namespace_unlock();
}
EXPORT_SYMBOL_GPL(mark_mounts_for_expiry);
/*
* Ripoff of 'select_parent()'
*
* search the list of submounts for a given mountpoint, and move any
* shrinkable submounts to the 'graveyard' list.
*/
static int select_submounts(struct mount *parent, struct list_head *graveyard)
{
struct mount *this_parent = parent;
struct list_head *next;
int found = 0;
repeat:
next = this_parent->mnt_mounts.next;
resume:
while (next != &this_parent->mnt_mounts) {
struct list_head *tmp = next;
struct mount *mnt = list_entry(tmp, struct mount, mnt_child);
next = tmp->next;
if (!(mnt->mnt.mnt_flags & MNT_SHRINKABLE))
continue;
/*
* Descend a level if the d_mounts list is non-empty.
*/
if (!list_empty(&mnt->mnt_mounts)) {
this_parent = mnt;
goto repeat;
}
if (!propagate_mount_busy(mnt, 1)) {
list_move_tail(&mnt->mnt_expire, graveyard);
found++;
}
}
/*
* All done at this level ... ascend and resume the search
*/
if (this_parent != parent) {
next = this_parent->mnt_child.next;
this_parent = this_parent->mnt_parent;
goto resume;
}
return found;
}
/*
* process a list of expirable mountpoints with the intent of discarding any
* submounts of a specific parent mountpoint
*
* mount_lock must be held for write
*/
static void shrink_submounts(struct mount *mnt)
{
LIST_HEAD(graveyard);
struct mount *m;
/* extract submounts of 'mountpoint' from the expiration list */
while (select_submounts(mnt, &graveyard)) {
while (!list_empty(&graveyard)) {
m = list_first_entry(&graveyard, struct mount,
mnt_expire);
touch_mnt_namespace(m->mnt_ns);
umount_tree(m, UMOUNT_PROPAGATE|UMOUNT_SYNC);
}
}
}
static void *copy_mount_options(const void __user * data)
{
char *copy;
unsigned left, offset;
if (!data)
return NULL;
copy = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!copy)
return ERR_PTR(-ENOMEM);
left = copy_from_user(copy, data, PAGE_SIZE);
/*
* Not all architectures have an exact copy_from_user(). Resort to
* byte at a time.
*/
offset = PAGE_SIZE - left;
while (left) {
char c;
if (get_user(c, (const char __user *)data + offset))
break;
copy[offset] = c;
left--;
offset++;
}
if (left == PAGE_SIZE) {
kfree(copy);
return ERR_PTR(-EFAULT);
}
return copy;
}
static char *copy_mount_string(const void __user *data)
{
return data ? strndup_user(data, PATH_MAX) : NULL;
}
/*
* Flags is a 32-bit value that allows up to 31 non-fs dependent flags to
* be given to the mount() call (ie: read-only, no-dev, no-suid etc).
*
* data is a (void *) that can point to any structure up to
* PAGE_SIZE-1 bytes, which can contain arbitrary fs-dependent
* information (or be NULL).
*
* Pre-0.97 versions of mount() didn't have a flags word.
* When the flags word was introduced its top half was required
* to have the magic value 0xC0ED, and this remained so until 2.4.0-test9.
* Therefore, if this magic number is present, it carries no information
* and must be discarded.
*/
int path_mount(const char *dev_name, struct path *path,
const char *type_page, unsigned long flags, void *data_page)
{
unsigned int mnt_flags = 0, sb_flags;
int ret;
/* Discard magic */
if ((flags & MS_MGC_MSK) == MS_MGC_VAL)
flags &= ~MS_MGC_MSK;
/* Basic sanity checks */
if (data_page)
((char *)data_page)[PAGE_SIZE - 1] = 0;
if (flags & MS_NOUSER)
return -EINVAL;
ret = security_sb_mount(dev_name, path, type_page, flags, data_page);
if (ret)
return ret;
if (!may_mount())
return -EPERM;
if (flags & SB_MANDLOCK)
warn_mandlock();
/* Default to relatime unless overriden */
if (!(flags & MS_NOATIME))
mnt_flags |= MNT_RELATIME;
/* Separate the per-mountpoint flags */
if (flags & MS_NOSUID)
mnt_flags |= MNT_NOSUID;
if (flags & MS_NODEV)
mnt_flags |= MNT_NODEV;
if (flags & MS_NOEXEC)
mnt_flags |= MNT_NOEXEC;
if (flags & MS_NOATIME)
mnt_flags |= MNT_NOATIME;
if (flags & MS_NODIRATIME)
mnt_flags |= MNT_NODIRATIME;
if (flags & MS_STRICTATIME)
mnt_flags &= ~(MNT_RELATIME | MNT_NOATIME);
if (flags & MS_RDONLY)
mnt_flags |= MNT_READONLY;
if (flags & MS_NOSYMFOLLOW)
mnt_flags |= MNT_NOSYMFOLLOW;
/* The default atime for remount is preservation */
if ((flags & MS_REMOUNT) &&
((flags & (MS_NOATIME | MS_NODIRATIME | MS_RELATIME |
MS_STRICTATIME)) == 0)) {
mnt_flags &= ~MNT_ATIME_MASK;
mnt_flags |= path->mnt->mnt_flags & MNT_ATIME_MASK;
}
sb_flags = flags & (SB_RDONLY |
SB_SYNCHRONOUS |
SB_MANDLOCK |
SB_DIRSYNC |
SB_SILENT |
SB_POSIXACL |
SB_LAZYTIME |
SB_I_VERSION);
if ((flags & (MS_REMOUNT | MS_BIND)) == (MS_REMOUNT | MS_BIND))
return do_reconfigure_mnt(path, mnt_flags);
if (flags & MS_REMOUNT)
return do_remount(path, flags, sb_flags, mnt_flags, data_page);
if (flags & MS_BIND)
return do_loopback(path, dev_name, flags & MS_REC);
if (flags & (MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE))
return do_change_type(path, flags);
if (flags & MS_MOVE)
return do_move_mount_old(path, dev_name);
return do_new_mount(path, type_page, sb_flags, mnt_flags, dev_name,
data_page);
}
long do_mount(const char *dev_name, const char __user *dir_name,
const char *type_page, unsigned long flags, void *data_page)
{
struct path path;
int ret;
ret = user_path_at(AT_FDCWD, dir_name, LOOKUP_FOLLOW, &path);
if (ret)
return ret;
ret = path_mount(dev_name, &path, type_page, flags, data_page);
path_put(&path);
return ret;
}
static struct ucounts *inc_mnt_namespaces(struct user_namespace *ns)
{
return inc_ucount(ns, current_euid(), UCOUNT_MNT_NAMESPACES);
}
static void dec_mnt_namespaces(struct ucounts *ucounts)
{
dec_ucount(ucounts, UCOUNT_MNT_NAMESPACES);
}
static void free_mnt_ns(struct mnt_namespace *ns)
{
if (!is_anon_ns(ns))
ns_free_inum(&ns->ns);
dec_mnt_namespaces(ns->ucounts);
put_user_ns(ns->user_ns);
kfree(ns);
}
/*
* Assign a sequence number so we can detect when we attempt to bind
* mount a reference to an older mount namespace into the current
* mount namespace, preventing reference counting loops. A 64bit
* number incrementing at 10Ghz will take 12,427 years to wrap which
* is effectively never, so we can ignore the possibility.
*/
static atomic64_t mnt_ns_seq = ATOMIC64_INIT(1);
static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns, bool anon)
{
struct mnt_namespace *new_ns;
struct ucounts *ucounts;
int ret;
ucounts = inc_mnt_namespaces(user_ns);
if (!ucounts)
return ERR_PTR(-ENOSPC);
new_ns = kzalloc(sizeof(struct mnt_namespace), GFP_KERNEL_ACCOUNT);
if (!new_ns) {
dec_mnt_namespaces(ucounts);
return ERR_PTR(-ENOMEM);
}
if (!anon) {
ret = ns_alloc_inum(&new_ns->ns);
if (ret) {
kfree(new_ns);
dec_mnt_namespaces(ucounts);
return ERR_PTR(ret);
}
}
new_ns->ns.ops = &mntns_operations;
if (!anon)
new_ns->seq = atomic64_add_return(1, &mnt_ns_seq);
refcount_set(&new_ns->ns.count, 1);
new_ns->mounts = RB_ROOT;
init_waitqueue_head(&new_ns->poll);
new_ns->user_ns = get_user_ns(user_ns);
new_ns->ucounts = ucounts;
return new_ns;
}
__latent_entropy
struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
struct user_namespace *user_ns, struct fs_struct *new_fs)
{
struct mnt_namespace *new_ns;
struct vfsmount *rootmnt = NULL, *pwdmnt = NULL;
struct mount *p, *q;
struct mount *old;
struct mount *new;
int copy_flags;
BUG_ON(!ns);
if (likely(!(flags & CLONE_NEWNS))) {
get_mnt_ns(ns);
return ns;
}
old = ns->root;
new_ns = alloc_mnt_ns(user_ns, false);
if (IS_ERR(new_ns))
return new_ns;
namespace_lock();
/* First pass: copy the tree topology */
copy_flags = CL_COPY_UNBINDABLE | CL_EXPIRE;
if (user_ns != ns->user_ns)
copy_flags |= CL_SHARED_TO_SLAVE;
new = copy_tree(old, old->mnt.mnt_root, copy_flags);
if (IS_ERR(new)) {
namespace_unlock();
free_mnt_ns(new_ns);
return ERR_CAST(new);
}
if (user_ns != ns->user_ns) {
lock_mount_hash();
lock_mnt_tree(new);
unlock_mount_hash();
}
new_ns->root = new;
/*
* Second pass: switch the tsk->fs->* elements and mark new vfsmounts
* as belonging to new namespace. We have already acquired a private
* fs_struct, so tsk->fs->lock is not needed.
*/
p = old;
q = new;
while (p) {
mnt_add_to_ns(new_ns, q);
new_ns->nr_mounts++;
if (new_fs) {
if (&p->mnt == new_fs->root.mnt) {
new_fs->root.mnt = mntget(&q->mnt);
rootmnt = &p->mnt;
}
if (&p->mnt == new_fs->pwd.mnt) {
new_fs->pwd.mnt = mntget(&q->mnt);
pwdmnt = &p->mnt;
}
}
p = next_mnt(p, old);
q = next_mnt(q, new);
if (!q)
break;
// an mntns binding we'd skipped?
while (p->mnt.mnt_root != q->mnt.mnt_root)
p = next_mnt(skip_mnt_tree(p), old);
}
namespace_unlock();
if (rootmnt)
mntput(rootmnt);
if (pwdmnt)
mntput(pwdmnt);
return new_ns;
}
struct dentry *mount_subtree(struct vfsmount *m, const char *name)
{
struct mount *mnt = real_mount(m);
struct mnt_namespace *ns;
struct super_block *s;
struct path path;
int err;
ns = alloc_mnt_ns(&init_user_ns, true);
if (IS_ERR(ns)) {
mntput(m);
return ERR_CAST(ns);
}
ns->root = mnt;
ns->nr_mounts++;
mnt_add_to_ns(ns, mnt);
err = vfs_path_lookup(m->mnt_root, m,
name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &path);
put_mnt_ns(ns);
if (err)
return ERR_PTR(err);
/* trade a vfsmount reference for active sb one */
s = path.mnt->mnt_sb;
atomic_inc(&s->s_active);
mntput(path.mnt);
/* lock the sucker */
down_write(&s->s_umount);
/* ... and return the root of (sub)tree on it */
return path.dentry;
}
EXPORT_SYMBOL(mount_subtree);
SYSCALL_DEFINE5(mount, char __user *, dev_name, char __user *, dir_name,
char __user *, type, unsigned long, flags, void __user *, data)
{
int ret;
char *kernel_type;
char *kernel_dev;
void *options;
kernel_type = copy_mount_string(type);
ret = PTR_ERR(kernel_type);
if (IS_ERR(kernel_type))
goto out_type;
kernel_dev = copy_mount_string(dev_name);
ret = PTR_ERR(kernel_dev);
if (IS_ERR(kernel_dev))
goto out_dev;
options = copy_mount_options(data);
ret = PTR_ERR(options);
if (IS_ERR(options))
goto out_data;
ret = do_mount(kernel_dev, dir_name, kernel_type, flags, options);
kfree(options);
out_data:
kfree(kernel_dev);
out_dev:
kfree(kernel_type);
out_type:
return ret;
}
#define FSMOUNT_VALID_FLAGS \
(MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \
MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME | \
MOUNT_ATTR_NOSYMFOLLOW)
#define MOUNT_SETATTR_VALID_FLAGS (FSMOUNT_VALID_FLAGS | MOUNT_ATTR_IDMAP)
#define MOUNT_SETATTR_PROPAGATION_FLAGS \
(MS_UNBINDABLE | MS_PRIVATE | MS_SLAVE | MS_SHARED)
static unsigned int attr_flags_to_mnt_flags(u64 attr_flags)
{
unsigned int mnt_flags = 0;
if (attr_flags & MOUNT_ATTR_RDONLY)
mnt_flags |= MNT_READONLY;
if (attr_flags & MOUNT_ATTR_NOSUID)
mnt_flags |= MNT_NOSUID;
if (attr_flags & MOUNT_ATTR_NODEV)
mnt_flags |= MNT_NODEV;
if (attr_flags & MOUNT_ATTR_NOEXEC)
mnt_flags |= MNT_NOEXEC;
if (attr_flags & MOUNT_ATTR_NODIRATIME)
mnt_flags |= MNT_NODIRATIME;
if (attr_flags & MOUNT_ATTR_NOSYMFOLLOW)
mnt_flags |= MNT_NOSYMFOLLOW;
return mnt_flags;
}
/*
* Create a kernel mount representation for a new, prepared superblock
* (specified by fs_fd) and attach to an open_tree-like file descriptor.
*/
SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags,
unsigned int, attr_flags)
{
struct mnt_namespace *ns;
struct fs_context *fc;
struct file *file;
struct path newmount;
struct mount *mnt;
struct fd f;
unsigned int mnt_flags = 0;
long ret;
if (!may_mount())
return -EPERM;
if ((flags & ~(FSMOUNT_CLOEXEC)) != 0)
return -EINVAL;
if (attr_flags & ~FSMOUNT_VALID_FLAGS)
return -EINVAL;
mnt_flags = attr_flags_to_mnt_flags(attr_flags);
switch (attr_flags & MOUNT_ATTR__ATIME) {
case MOUNT_ATTR_STRICTATIME:
break;
case MOUNT_ATTR_NOATIME:
mnt_flags |= MNT_NOATIME;
break;
case MOUNT_ATTR_RELATIME:
mnt_flags |= MNT_RELATIME;
break;
default:
return -EINVAL;
}
f = fdget(fs_fd);
if (!f.file)
return -EBADF;
ret = -EINVAL;
if (f.file->f_op != &fscontext_fops)
goto err_fsfd;
fc = f.file->private_data;
ret = mutex_lock_interruptible(&fc->uapi_mutex);
if (ret < 0)
goto err_fsfd;
/* There must be a valid superblock or we can't mount it */
ret = -EINVAL;
if (!fc->root)
goto err_unlock;
ret = -EPERM;
if (mount_too_revealing(fc->root->d_sb, &mnt_flags)) {
pr_warn("VFS: Mount too revealing\n");
goto err_unlock;
}
ret = -EBUSY;
if (fc->phase != FS_CONTEXT_AWAITING_MOUNT)
goto err_unlock;
if (fc->sb_flags & SB_MANDLOCK)
warn_mandlock();
newmount.mnt = vfs_create_mount(fc);
if (IS_ERR(newmount.mnt)) {
ret = PTR_ERR(newmount.mnt);
goto err_unlock;
}
newmount.dentry = dget(fc->root);
newmount.mnt->mnt_flags = mnt_flags;
/* We've done the mount bit - now move the file context into more or
* less the same state as if we'd done an fspick(). We don't want to
* do any memory allocation or anything like that at this point as we
* don't want to have to handle any errors incurred.
*/
vfs_clean_context(fc);
ns = alloc_mnt_ns(current->nsproxy->mnt_ns->user_ns, true);
if (IS_ERR(ns)) {
ret = PTR_ERR(ns);
goto err_path;
}
mnt = real_mount(newmount.mnt);
ns->root = mnt;
ns->nr_mounts = 1;
mnt_add_to_ns(ns, mnt);
mntget(newmount.mnt);
/* Attach to an apparent O_PATH fd with a note that we need to unmount
* it, not just simply put it.
*/
file = dentry_open(&newmount, O_PATH, fc->cred);
if (IS_ERR(file)) {
dissolve_on_fput(newmount.mnt);
ret = PTR_ERR(file);
goto err_path;
}
file->f_mode |= FMODE_NEED_UNMOUNT;
ret = get_unused_fd_flags((flags & FSMOUNT_CLOEXEC) ? O_CLOEXEC : 0);
if (ret >= 0)
fd_install(ret, file);
else
fput(file);
err_path:
path_put(&newmount);
err_unlock:
mutex_unlock(&fc->uapi_mutex);
err_fsfd:
fdput(f);
return ret;
}
/*
* Move a mount from one place to another. In combination with
* fsopen()/fsmount() this is used to install a new mount and in combination
* with open_tree(OPEN_TREE_CLONE [| AT_RECURSIVE]) it can be used to copy
* a mount subtree.
*
* Note the flags value is a combination of MOVE_MOUNT_* flags.
*/
SYSCALL_DEFINE5(move_mount,
int, from_dfd, const char __user *, from_pathname,
int, to_dfd, const char __user *, to_pathname,
unsigned int, flags)
{
struct path from_path, to_path;
unsigned int lflags;
int ret = 0;
if (!may_mount())
return -EPERM;
if (flags & ~MOVE_MOUNT__MASK)
return -EINVAL;
if ((flags & (MOVE_MOUNT_BENEATH | MOVE_MOUNT_SET_GROUP)) ==
(MOVE_MOUNT_BENEATH | MOVE_MOUNT_SET_GROUP))
return -EINVAL;
/* If someone gives a pathname, they aren't permitted to move
* from an fd that requires unmount as we can't get at the flag
* to clear it afterwards.
*/
lflags = 0;
if (flags & MOVE_MOUNT_F_SYMLINKS) lflags |= LOOKUP_FOLLOW;
if (flags & MOVE_MOUNT_F_AUTOMOUNTS) lflags |= LOOKUP_AUTOMOUNT;
if (flags & MOVE_MOUNT_F_EMPTY_PATH) lflags |= LOOKUP_EMPTY;
ret = user_path_at(from_dfd, from_pathname, lflags, &from_path);
if (ret < 0)
return ret;
lflags = 0;
if (flags & MOVE_MOUNT_T_SYMLINKS) lflags |= LOOKUP_FOLLOW;
if (flags & MOVE_MOUNT_T_AUTOMOUNTS) lflags |= LOOKUP_AUTOMOUNT;
if (flags & MOVE_MOUNT_T_EMPTY_PATH) lflags |= LOOKUP_EMPTY;
ret = user_path_at(to_dfd, to_pathname, lflags, &to_path);
if (ret < 0)
goto out_from;
ret = security_move_mount(&from_path, &to_path);
if (ret < 0)
goto out_to;
if (flags & MOVE_MOUNT_SET_GROUP)
ret = do_set_group(&from_path, &to_path);
else
ret = do_move_mount(&from_path, &to_path,
(flags & MOVE_MOUNT_BENEATH));
out_to:
path_put(&to_path);
out_from:
path_put(&from_path);
return ret;
}
/*
* Return true if path is reachable from root
*
* namespace_sem or mount_lock is held
*/
bool is_path_reachable(struct mount *mnt, struct dentry *dentry,
const struct path *root)
{
while (&mnt->mnt != root->mnt && mnt_has_parent(mnt)) {
dentry = mnt->mnt_mountpoint;
mnt = mnt->mnt_parent;
}
return &mnt->mnt == root->mnt && is_subdir(dentry, root->dentry);
}
bool path_is_under(const struct path *path1, const struct path *path2)
{
bool res;
read_seqlock_excl(&mount_lock);
res = is_path_reachable(real_mount(path1->mnt), path1->dentry, path2);
read_sequnlock_excl(&mount_lock);
return res;
}
EXPORT_SYMBOL(path_is_under);
/*
* pivot_root Semantics:
* Moves the root file system of the current process to the directory put_old,
* makes new_root as the new root file system of the current process, and sets
* root/cwd of all processes which had them on the current root to new_root.
*
* Restrictions:
* The new_root and put_old must be directories, and must not be on the
* same file system as the current process root. The put_old must be
* underneath new_root, i.e. adding a non-zero number of /.. to the string
* pointed to by put_old must yield the same directory as new_root. No other
* file system may be mounted on put_old. After all, new_root is a mountpoint.
*
* Also, the current root cannot be on the 'rootfs' (initial ramfs) filesystem.
* See Documentation/filesystems/ramfs-rootfs-initramfs.rst for alternatives
* in this situation.
*
* Notes:
* - we don't move root/cwd if they are not at the root (reason: if something
* cared enough to change them, it's probably wrong to force them elsewhere)
* - it's okay to pick a root that isn't the root of a file system, e.g.
* /nfs/my_root where /nfs is the mount point. It must be a mountpoint,
* though, so you may need to say mount --bind /nfs/my_root /nfs/my_root
* first.
*/
SYSCALL_DEFINE2(pivot_root, const char __user *, new_root,
const char __user *, put_old)
{
struct path new, old, root;
struct mount *new_mnt, *root_mnt, *old_mnt, *root_parent, *ex_parent;
struct mountpoint *old_mp, *root_mp;
int error;
if (!may_mount())
return -EPERM;
error = user_path_at(AT_FDCWD, new_root,
LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &new);
if (error)
goto out0;
error = user_path_at(AT_FDCWD, put_old,
LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &old);
if (error)
goto out1;
error = security_sb_pivotroot(&old, &new);
if (error)
goto out2;
get_fs_root(current->fs, &root);
old_mp = lock_mount(&old);
error = PTR_ERR(old_mp);
if (IS_ERR(old_mp))
goto out3;
error = -EINVAL;
new_mnt = real_mount(new.mnt);
root_mnt = real_mount(root.mnt);
old_mnt = real_mount(old.mnt);
ex_parent = new_mnt->mnt_parent;
root_parent = root_mnt->mnt_parent;
if (IS_MNT_SHARED(old_mnt) ||
IS_MNT_SHARED(ex_parent) ||
IS_MNT_SHARED(root_parent))
goto out4;
if (!check_mnt(root_mnt) || !check_mnt(new_mnt))
goto out4;
if (new_mnt->mnt.mnt_flags & MNT_LOCKED)
goto out4;
error = -ENOENT;
if (d_unlinked(new.dentry))
goto out4;
error = -EBUSY;
if (new_mnt == root_mnt || old_mnt == root_mnt)
goto out4; /* loop, on the same file system */
error = -EINVAL;
if (!path_mounted(&root))
goto out4; /* not a mountpoint */
if (!mnt_has_parent(root_mnt))
goto out4; /* not attached */
if (!path_mounted(&new))
goto out4; /* not a mountpoint */
if (!mnt_has_parent(new_mnt))
goto out4; /* not attached */
/* make sure we can reach put_old from new_root */
if (!is_path_reachable(old_mnt, old.dentry, &new))
goto out4;
/* make certain new is below the root */
if (!is_path_reachable(new_mnt, new.dentry, &root))
goto out4;
lock_mount_hash();
umount_mnt(new_mnt);
root_mp = unhash_mnt(root_mnt); /* we'll need its mountpoint */
if (root_mnt->mnt.mnt_flags & MNT_LOCKED) {
new_mnt->mnt.mnt_flags |= MNT_LOCKED;
root_mnt->mnt.mnt_flags &= ~MNT_LOCKED;
}
/* mount old root on put_old */
attach_mnt(root_mnt, old_mnt, old_mp, false);
/* mount new_root on / */
attach_mnt(new_mnt, root_parent, root_mp, false);
mnt_add_count(root_parent, -1);
touch_mnt_namespace(current->nsproxy->mnt_ns);
/* A moved mount should not expire automatically */
list_del_init(&new_mnt->mnt_expire);
put_mountpoint(root_mp);
unlock_mount_hash();
chroot_fs_refs(&root, &new);
error = 0;
out4:
unlock_mount(old_mp);
if (!error)
mntput_no_expire(ex_parent);
out3:
path_put(&root);
out2:
path_put(&old);
out1:
path_put(&new);
out0:
return error;
}
static unsigned int recalc_flags(struct mount_kattr *kattr, struct mount *mnt)
{
unsigned int flags = mnt->mnt.mnt_flags;
/* flags to clear */
flags &= ~kattr->attr_clr;
/* flags to raise */
flags |= kattr->attr_set;
return flags;
}
static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt)
{
struct vfsmount *m = &mnt->mnt;
struct user_namespace *fs_userns = m->mnt_sb->s_user_ns;
if (!kattr->mnt_idmap)
return 0;
/*
* Creating an idmapped mount with the filesystem wide idmapping
* doesn't make sense so block that. We don't allow mushy semantics.
*/
if (kattr->mnt_userns == m->mnt_sb->s_user_ns)
return -EINVAL;
/*
* Once a mount has been idmapped we don't allow it to change its
* mapping. It makes things simpler and callers can just create
* another bind-mount they can idmap if they want to.
*/
if (is_idmapped_mnt(m))
return -EPERM;
/* The underlying filesystem doesn't support idmapped mounts yet. */
if (!(m->mnt_sb->s_type->fs_flags & FS_ALLOW_IDMAP))
return -EINVAL;
/* We're not controlling the superblock. */
if (!ns_capable(fs_userns, CAP_SYS_ADMIN))
return -EPERM;
/* Mount has already been visible in the filesystem hierarchy. */
if (!is_anon_ns(mnt->mnt_ns))
return -EINVAL;
return 0;
}
/**
* mnt_allow_writers() - check whether the attribute change allows writers
* @kattr: the new mount attributes
* @mnt: the mount to which @kattr will be applied
*
* Check whether thew new mount attributes in @kattr allow concurrent writers.
*
* Return: true if writers need to be held, false if not
*/
static inline bool mnt_allow_writers(const struct mount_kattr *kattr,
const struct mount *mnt)
{
return (!(kattr->attr_set & MNT_READONLY) ||
(mnt->mnt.mnt_flags & MNT_READONLY)) &&
!kattr->mnt_idmap;
}
static int mount_setattr_prepare(struct mount_kattr *kattr, struct mount *mnt)
{
struct mount *m;
int err;
for (m = mnt; m; m = next_mnt(m, mnt)) {
if (!can_change_locked_flags(m, recalc_flags(kattr, m))) {
err = -EPERM;
break;
}
err = can_idmap_mount(kattr, m);
if (err)
break;
if (!mnt_allow_writers(kattr, m)) {
err = mnt_hold_writers(m);
if (err)
break;
}
if (!kattr->recurse)
return 0;
}
if (err) {
struct mount *p;
/*
* If we had to call mnt_hold_writers() MNT_WRITE_HOLD will
* be set in @mnt_flags. The loop unsets MNT_WRITE_HOLD for all
* mounts and needs to take care to include the first mount.
*/
for (p = mnt; p; p = next_mnt(p, mnt)) {
/* If we had to hold writers unblock them. */
if (p->mnt.mnt_flags & MNT_WRITE_HOLD)
mnt_unhold_writers(p);
/*
* We're done once the first mount we changed got
* MNT_WRITE_HOLD unset.
*/
if (p == m)
break;
}
}
return err;
}
static void do_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt)
{
if (!kattr->mnt_idmap)
return;
/*
* Pairs with smp_load_acquire() in mnt_idmap().
*
* Since we only allow a mount to change the idmapping once and
* verified this in can_idmap_mount() we know that the mount has
* @nop_mnt_idmap attached to it. So there's no need to drop any
* references.
*/
smp_store_release(&mnt->mnt.mnt_idmap, mnt_idmap_get(kattr->mnt_idmap));
}
static void mount_setattr_commit(struct mount_kattr *kattr, struct mount *mnt)
{
struct mount *m;
for (m = mnt; m; m = next_mnt(m, mnt)) {
unsigned int flags;
do_idmap_mount(kattr, m);
flags = recalc_flags(kattr, m);
WRITE_ONCE(m->mnt.mnt_flags, flags);
/* If we had to hold writers unblock them. */
if (m->mnt.mnt_flags & MNT_WRITE_HOLD)
mnt_unhold_writers(m);
if (kattr->propagation)
change_mnt_propagation(m, kattr->propagation);
if (!kattr->recurse)
break;
}
touch_mnt_namespace(mnt->mnt_ns);
}
static int do_mount_setattr(struct path *path, struct mount_kattr *kattr)
{
struct mount *mnt = real_mount(path->mnt);
int err = 0;
if (!path_mounted(path))
return -EINVAL;
if (kattr->mnt_userns) {
struct mnt_idmap *mnt_idmap;
mnt_idmap = alloc_mnt_idmap(kattr->mnt_userns);
if (IS_ERR(mnt_idmap))
return PTR_ERR(mnt_idmap);
kattr->mnt_idmap = mnt_idmap;
}
if (kattr->propagation) {
/*
* Only take namespace_lock() if we're actually changing
* propagation.
*/
namespace_lock();
if (kattr->propagation == MS_SHARED) {
err = invent_group_ids(mnt, kattr->recurse);
if (err) {
namespace_unlock();
return err;
}
}
}
err = -EINVAL;
lock_mount_hash();
/* Ensure that this isn't anything purely vfs internal. */
if (!is_mounted(&mnt->mnt))
goto out;
/*
* If this is an attached mount make sure it's located in the callers
* mount namespace. If it's not don't let the caller interact with it.
*
* If this mount doesn't have a parent it's most often simply a
* detached mount with an anonymous mount namespace. IOW, something
* that's simply not attached yet. But there are apparently also users
* that do change mount properties on the rootfs itself. That obviously
* neither has a parent nor is it a detached mount so we cannot
* unconditionally check for detached mounts.
*/
if ((mnt_has_parent(mnt) || !is_anon_ns(mnt->mnt_ns)) && !check_mnt(mnt))
goto out;
/*
* First, we get the mount tree in a shape where we can change mount
* properties without failure. If we succeeded to do so we commit all
* changes and if we failed we clean up.
*/
err = mount_setattr_prepare(kattr, mnt);
if (!err)
mount_setattr_commit(kattr, mnt);
out:
unlock_mount_hash();
if (kattr->propagation) {
if (err)
cleanup_group_ids(mnt, NULL);
namespace_unlock();
}
return err;
}
static int build_mount_idmapped(const struct mount_attr *attr, size_t usize,
struct mount_kattr *kattr, unsigned int flags)
{
int err = 0;
struct ns_common *ns;
struct user_namespace *mnt_userns;
struct fd f;
if (!((attr->attr_set | attr->attr_clr) & MOUNT_ATTR_IDMAP))
return 0;
/*
* We currently do not support clearing an idmapped mount. If this ever
* is a use-case we can revisit this but for now let's keep it simple
* and not allow it.
*/
if (attr->attr_clr & MOUNT_ATTR_IDMAP)
return -EINVAL;
if (attr->userns_fd > INT_MAX)
return -EINVAL;
f = fdget(attr->userns_fd);
if (!f.file)
return -EBADF;
if (!proc_ns_file(f.file)) {
err = -EINVAL;
goto out_fput;
}
ns = get_proc_ns(file_inode(f.file));
if (ns->ops->type != CLONE_NEWUSER) {
err = -EINVAL;
goto out_fput;
}
/*
* The initial idmapping cannot be used to create an idmapped
* mount. We use the initial idmapping as an indicator of a mount
* that is not idmapped. It can simply be passed into helpers that
* are aware of idmapped mounts as a convenient shortcut. A user
* can just create a dedicated identity mapping to achieve the same
* result.
*/
mnt_userns = container_of(ns, struct user_namespace, ns);
if (mnt_userns == &init_user_ns) {
err = -EPERM;
goto out_fput;
}
/* We're not controlling the target namespace. */
if (!ns_capable(mnt_userns, CAP_SYS_ADMIN)) {
err = -EPERM;
goto out_fput;
}
kattr->mnt_userns = get_user_ns(mnt_userns);
out_fput:
fdput(f);
return err;
}
static int build_mount_kattr(const struct mount_attr *attr, size_t usize,
struct mount_kattr *kattr, unsigned int flags)
{
unsigned int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW;
if (flags & AT_NO_AUTOMOUNT)
lookup_flags &= ~LOOKUP_AUTOMOUNT;
if (flags & AT_SYMLINK_NOFOLLOW)
lookup_flags &= ~LOOKUP_FOLLOW;
if (flags & AT_EMPTY_PATH)
lookup_flags |= LOOKUP_EMPTY;
*kattr = (struct mount_kattr) {
.lookup_flags = lookup_flags,
.recurse = !!(flags & AT_RECURSIVE),
};
if (attr->propagation & ~MOUNT_SETATTR_PROPAGATION_FLAGS)
return -EINVAL;
if (hweight32(attr->propagation & MOUNT_SETATTR_PROPAGATION_FLAGS) > 1)
return -EINVAL;
kattr->propagation = attr->propagation;
if ((attr->attr_set | attr->attr_clr) & ~MOUNT_SETATTR_VALID_FLAGS)
return -EINVAL;
kattr->attr_set = attr_flags_to_mnt_flags(attr->attr_set);
kattr->attr_clr = attr_flags_to_mnt_flags(attr->attr_clr);
/*
* Since the MOUNT_ATTR_<atime> values are an enum, not a bitmap,
* users wanting to transition to a different atime setting cannot
* simply specify the atime setting in @attr_set, but must also
* specify MOUNT_ATTR__ATIME in the @attr_clr field.
* So ensure that MOUNT_ATTR__ATIME can't be partially set in
* @attr_clr and that @attr_set can't have any atime bits set if
* MOUNT_ATTR__ATIME isn't set in @attr_clr.
*/
if (attr->attr_clr & MOUNT_ATTR__ATIME) {
if ((attr->attr_clr & MOUNT_ATTR__ATIME) != MOUNT_ATTR__ATIME)
return -EINVAL;
/*
* Clear all previous time settings as they are mutually
* exclusive.
*/
kattr->attr_clr |= MNT_RELATIME | MNT_NOATIME;
switch (attr->attr_set & MOUNT_ATTR__ATIME) {
case MOUNT_ATTR_RELATIME:
kattr->attr_set |= MNT_RELATIME;
break;
case MOUNT_ATTR_NOATIME:
kattr->attr_set |= MNT_NOATIME;
break;
case MOUNT_ATTR_STRICTATIME:
break;
default:
return -EINVAL;
}
} else {
if (attr->attr_set & MOUNT_ATTR__ATIME)
return -EINVAL;
}
return build_mount_idmapped(attr, usize, kattr, flags);
}
static void finish_mount_kattr(struct mount_kattr *kattr)
{
put_user_ns(kattr->mnt_userns);
kattr->mnt_userns = NULL;
if (kattr->mnt_idmap)
mnt_idmap_put(kattr->mnt_idmap);
}
SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path,
unsigned int, flags, struct mount_attr __user *, uattr,
size_t, usize)
{
int err;
struct path target;
struct mount_attr attr;
struct mount_kattr kattr;
BUILD_BUG_ON(sizeof(struct mount_attr) != MOUNT_ATTR_SIZE_VER0);
if (flags & ~(AT_EMPTY_PATH |
AT_RECURSIVE |
AT_SYMLINK_NOFOLLOW |
AT_NO_AUTOMOUNT))
return -EINVAL;
if (unlikely(usize > PAGE_SIZE))
return -E2BIG;
if (unlikely(usize < MOUNT_ATTR_SIZE_VER0))
return -EINVAL;
if (!may_mount())
return -EPERM;
err = copy_struct_from_user(&attr, sizeof(attr), uattr, usize);
if (err)
return err;
/* Don't bother walking through the mounts if this is a nop. */
if (attr.attr_set == 0 &&
attr.attr_clr == 0 &&
attr.propagation == 0)
return 0;
err = build_mount_kattr(&attr, usize, &kattr, flags);
if (err)
return err;
err = user_path_at(dfd, path, kattr.lookup_flags, &target);
if (!err) {
err = do_mount_setattr(&target, &kattr);
path_put(&target);
}
finish_mount_kattr(&kattr);
return err;
}
int show_path(struct seq_file *m, struct dentry *root)
{
if (root->d_sb->s_op->show_path)
return root->d_sb->s_op->show_path(m, root);
seq_dentry(m, root, " \t\n\\");
return 0;
}
static struct vfsmount *lookup_mnt_in_ns(u64 id, struct mnt_namespace *ns)
{
struct mount *mnt = mnt_find_id_at(ns, id);
if (!mnt || mnt->mnt_id_unique != id)
return NULL;
return &mnt->mnt;
}
struct kstatmount {
struct statmount __user *buf;
size_t bufsize;
struct vfsmount *mnt;
u64 mask;
struct path root;
struct statmount sm;
struct seq_file seq;
};
static u64 mnt_to_attr_flags(struct vfsmount *mnt)
{
unsigned int mnt_flags = READ_ONCE(mnt->mnt_flags);
u64 attr_flags = 0;
if (mnt_flags & MNT_READONLY)
attr_flags |= MOUNT_ATTR_RDONLY;
if (mnt_flags & MNT_NOSUID)
attr_flags |= MOUNT_ATTR_NOSUID;
if (mnt_flags & MNT_NODEV)
attr_flags |= MOUNT_ATTR_NODEV;
if (mnt_flags & MNT_NOEXEC)
attr_flags |= MOUNT_ATTR_NOEXEC;
if (mnt_flags & MNT_NODIRATIME)
attr_flags |= MOUNT_ATTR_NODIRATIME;
if (mnt_flags & MNT_NOSYMFOLLOW)
attr_flags |= MOUNT_ATTR_NOSYMFOLLOW;
if (mnt_flags & MNT_NOATIME)
attr_flags |= MOUNT_ATTR_NOATIME;
else if (mnt_flags & MNT_RELATIME)
attr_flags |= MOUNT_ATTR_RELATIME;
else
attr_flags |= MOUNT_ATTR_STRICTATIME;
if (is_idmapped_mnt(mnt))
attr_flags |= MOUNT_ATTR_IDMAP;
return attr_flags;
}
static u64 mnt_to_propagation_flags(struct mount *m)
{
u64 propagation = 0;
if (IS_MNT_SHARED(m))
propagation |= MS_SHARED;
if (IS_MNT_SLAVE(m))
propagation |= MS_SLAVE;
if (IS_MNT_UNBINDABLE(m))
propagation |= MS_UNBINDABLE;
if (!propagation)
propagation |= MS_PRIVATE;
return propagation;
}
static void statmount_sb_basic(struct kstatmount *s)
{
struct super_block *sb = s->mnt->mnt_sb;
s->sm.mask |= STATMOUNT_SB_BASIC;
s->sm.sb_dev_major = MAJOR(sb->s_dev);
s->sm.sb_dev_minor = MINOR(sb->s_dev);
s->sm.sb_magic = sb->s_magic;
s->sm.sb_flags = sb->s_flags & (SB_RDONLY|SB_SYNCHRONOUS|SB_DIRSYNC|SB_LAZYTIME);
}
static void statmount_mnt_basic(struct kstatmount *s)
{
struct mount *m = real_mount(s->mnt);
s->sm.mask |= STATMOUNT_MNT_BASIC;
s->sm.mnt_id = m->mnt_id_unique;
s->sm.mnt_parent_id = m->mnt_parent->mnt_id_unique;
s->sm.mnt_id_old = m->mnt_id;
s->sm.mnt_parent_id_old = m->mnt_parent->mnt_id;
s->sm.mnt_attr = mnt_to_attr_flags(&m->mnt);
s->sm.mnt_propagation = mnt_to_propagation_flags(m);
s->sm.mnt_peer_group = IS_MNT_SHARED(m) ? m->mnt_group_id : 0;
s->sm.mnt_master = IS_MNT_SLAVE(m) ? m->mnt_master->mnt_group_id : 0;
}
static void statmount_propagate_from(struct kstatmount *s)
{
struct mount *m = real_mount(s->mnt);
s->sm.mask |= STATMOUNT_PROPAGATE_FROM;
if (IS_MNT_SLAVE(m))
s->sm.propagate_from = get_dominating_id(m, ¤t->fs->root);
}
static int statmount_mnt_root(struct kstatmount *s, struct seq_file *seq)
{
int ret;
size_t start = seq->count;
ret = show_path(seq, s->mnt->mnt_root);
if (ret)
return ret;
if (unlikely(seq_has_overflowed(seq)))
return -EAGAIN;
/*
* Unescape the result. It would be better if supplied string was not
* escaped in the first place, but that's a pretty invasive change.
*/
seq->buf[seq->count] = '\0';
seq->count = start;
seq_commit(seq, string_unescape_inplace(seq->buf + start, UNESCAPE_OCTAL));
return 0;
}
static int statmount_mnt_point(struct kstatmount *s, struct seq_file *seq)
{
struct vfsmount *mnt = s->mnt;
struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt };
int err;
err = seq_path_root(seq, &mnt_path, &s->root, "");
return err == SEQ_SKIP ? 0 : err;
}
static int statmount_fs_type(struct kstatmount *s, struct seq_file *seq)
{
struct super_block *sb = s->mnt->mnt_sb;
seq_puts(seq, sb->s_type->name);
return 0;
}
static int statmount_string(struct kstatmount *s, u64 flag)
{
int ret;
size_t kbufsize;
struct seq_file *seq = &s->seq;
struct statmount *sm = &s->sm;
switch (flag) {
case STATMOUNT_FS_TYPE:
sm->fs_type = seq->count;
ret = statmount_fs_type(s, seq);
break;
case STATMOUNT_MNT_ROOT:
sm->mnt_root = seq->count;
ret = statmount_mnt_root(s, seq);
break;
case STATMOUNT_MNT_POINT:
sm->mnt_point = seq->count;
ret = statmount_mnt_point(s, seq);
break;
default:
WARN_ON_ONCE(true);
return -EINVAL;
}
if (unlikely(check_add_overflow(sizeof(*sm), seq->count, &kbufsize)))
return -EOVERFLOW;
if (kbufsize >= s->bufsize)
return -EOVERFLOW;
/* signal a retry */
if (unlikely(seq_has_overflowed(seq)))
return -EAGAIN;
if (ret)
return ret;
seq->buf[seq->count++] = '\0';
sm->mask |= flag;
return 0;
}
static int copy_statmount_to_user(struct kstatmount *s)
{
struct statmount *sm = &s->sm;
struct seq_file *seq = &s->seq;
char __user *str = ((char __user *)s->buf) + sizeof(*sm);
size_t copysize = min_t(size_t, s->bufsize, sizeof(*sm));
if (seq->count && copy_to_user(str, seq->buf, seq->count))
return -EFAULT;
/* Return the number of bytes copied to the buffer */
sm->size = copysize + seq->count;
if (copy_to_user(s->buf, sm, copysize))
return -EFAULT;
return 0;
}
static int do_statmount(struct kstatmount *s)
{
struct mount *m = real_mount(s->mnt);
int err;
/*
* Don't trigger audit denials. We just want to determine what
* mounts to show users.
*/
if (!is_path_reachable(m, m->mnt.mnt_root, &s->root) &&
!ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN))
return -EPERM;
err = security_sb_statfs(s->mnt->mnt_root);
if (err)
return err;
if (s->mask & STATMOUNT_SB_BASIC)
statmount_sb_basic(s);
if (s->mask & STATMOUNT_MNT_BASIC)
statmount_mnt_basic(s);
if (s->mask & STATMOUNT_PROPAGATE_FROM)
statmount_propagate_from(s);
if (s->mask & STATMOUNT_FS_TYPE)
err = statmount_string(s, STATMOUNT_FS_TYPE);
if (!err && s->mask & STATMOUNT_MNT_ROOT)
err = statmount_string(s, STATMOUNT_MNT_ROOT);
if (!err && s->mask & STATMOUNT_MNT_POINT)
err = statmount_string(s, STATMOUNT_MNT_POINT);
if (err)
return err;
return 0;
}
static inline bool retry_statmount(const long ret, size_t *seq_size)
{
if (likely(ret != -EAGAIN))
return false;
if (unlikely(check_mul_overflow(*seq_size, 2, seq_size)))
return false;
if (unlikely(*seq_size > MAX_RW_COUNT))
return false;
return true;
}
static int prepare_kstatmount(struct kstatmount *ks, struct mnt_id_req *kreq,
struct statmount __user *buf, size_t bufsize,
size_t seq_size)
{
if (!access_ok(buf, bufsize))
return -EFAULT;
memset(ks, 0, sizeof(*ks));
ks->mask = kreq->param;
ks->buf = buf;
ks->bufsize = bufsize;
ks->seq.size = seq_size;
ks->seq.buf = kvmalloc(seq_size, GFP_KERNEL_ACCOUNT);
if (!ks->seq.buf)
return -ENOMEM;
return 0;
}
static int copy_mnt_id_req(const struct mnt_id_req __user *req,
struct mnt_id_req *kreq)
{
int ret;
size_t usize;
BUILD_BUG_ON(sizeof(struct mnt_id_req) != MNT_ID_REQ_SIZE_VER0);
ret = get_user(usize, &req->size);
if (ret)
return -EFAULT;
if (unlikely(usize > PAGE_SIZE))
return -E2BIG;
if (unlikely(usize < MNT_ID_REQ_SIZE_VER0))
return -EINVAL;
memset(kreq, 0, sizeof(*kreq));
ret = copy_struct_from_user(kreq, sizeof(*kreq), req, usize);
if (ret)
return ret;
if (kreq->spare != 0)
return -EINVAL;
return 0;
}
SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req,
struct statmount __user *, buf, size_t, bufsize,
unsigned int, flags)
{
struct vfsmount *mnt;
struct mnt_id_req kreq;
struct kstatmount ks;
/* We currently support retrieval of 3 strings. */
size_t seq_size = 3 * PATH_MAX;
int ret;
if (flags)
return -EINVAL;
ret = copy_mnt_id_req(req, &kreq);
if (ret)
return ret;
retry:
ret = prepare_kstatmount(&ks, &kreq, buf, bufsize, seq_size);
if (ret)
return ret;
down_read(&namespace_sem);
mnt = lookup_mnt_in_ns(kreq.mnt_id, current->nsproxy->mnt_ns);
if (!mnt) {
up_read(&namespace_sem);
kvfree(ks.seq.buf);
return -ENOENT;
}
ks.mnt = mnt;
get_fs_root(current->fs, &ks.root);
ret = do_statmount(&ks);
path_put(&ks.root);
up_read(&namespace_sem);
if (!ret)
ret = copy_statmount_to_user(&ks);
kvfree(ks.seq.buf);
if (retry_statmount(ret, &seq_size))
goto retry;
return ret;
}
static struct mount *listmnt_next(struct mount *curr)
{
return node_to_mount(rb_next(&curr->mnt_node));
}
static ssize_t do_listmount(struct mount *first, struct path *orig,
u64 mnt_parent_id, u64 __user *mnt_ids,
size_t nr_mnt_ids, const struct path *root)
{
struct mount *r;
ssize_t ret;
/*
* Don't trigger audit denials. We just want to determine what
* mounts to show users.
*/
if (!is_path_reachable(real_mount(orig->mnt), orig->dentry, root) &&
!ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN))
return -EPERM;
ret = security_sb_statfs(orig->dentry);
if (ret)
return ret;
for (ret = 0, r = first; r && nr_mnt_ids; r = listmnt_next(r)) {
if (r->mnt_id_unique == mnt_parent_id)
continue;
if (!is_path_reachable(r, r->mnt.mnt_root, orig))
continue;
if (put_user(r->mnt_id_unique, mnt_ids))
return -EFAULT;
mnt_ids++;
nr_mnt_ids--;
ret++;
}
return ret;
}
SYSCALL_DEFINE4(listmount, const struct mnt_id_req __user *, req, u64 __user *,
mnt_ids, size_t, nr_mnt_ids, unsigned int, flags)
{
struct mnt_namespace *ns = current->nsproxy->mnt_ns;
struct mnt_id_req kreq;
struct mount *first;
struct path root, orig;
u64 mnt_parent_id, last_mnt_id;
const size_t maxcount = (size_t)-1 >> 3;
ssize_t ret;
if (flags)
return -EINVAL;
if (unlikely(nr_mnt_ids > maxcount))
return -EFAULT;
if (!access_ok(mnt_ids, nr_mnt_ids * sizeof(*mnt_ids)))
return -EFAULT;
ret = copy_mnt_id_req(req, &kreq);
if (ret)
return ret;
mnt_parent_id = kreq.mnt_id;
last_mnt_id = kreq.param;
down_read(&namespace_sem);
get_fs_root(current->fs, &root);
if (mnt_parent_id == LSMT_ROOT) {
orig = root;
} else {
ret = -ENOENT;
orig.mnt = lookup_mnt_in_ns(mnt_parent_id, ns);
if (!orig.mnt)
goto err;
orig.dentry = orig.mnt->mnt_root;
}
if (!last_mnt_id)
first = node_to_mount(rb_first(&ns->mounts));
else
first = mnt_find_id_at(ns, last_mnt_id + 1);
ret = do_listmount(first, &orig, mnt_parent_id, mnt_ids, nr_mnt_ids, &root);
err:
path_put(&root);
up_read(&namespace_sem);
return ret;
}
static void __init init_mount_tree(void)
{
struct vfsmount *mnt;
struct mount *m;
struct mnt_namespace *ns;
struct path root;
mnt = vfs_kern_mount(&rootfs_fs_type, 0, "rootfs", NULL);
if (IS_ERR(mnt))
panic("Can't create rootfs");
ns = alloc_mnt_ns(&init_user_ns, false);
if (IS_ERR(ns))
panic("Can't allocate initial namespace");
m = real_mount(mnt);
ns->root = m;
ns->nr_mounts = 1;
mnt_add_to_ns(ns, m);
init_task.nsproxy->mnt_ns = ns;
get_mnt_ns(ns);
root.mnt = mnt;
root.dentry = mnt->mnt_root;
mnt->mnt_flags |= MNT_LOCKED;
set_fs_pwd(current->fs, &root);
set_fs_root(current->fs, &root);
}
void __init mnt_init(void)
{
int err;
mnt_cache = kmem_cache_create("mnt_cache", sizeof(struct mount),
0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
mount_hashtable = alloc_large_system_hash("Mount-cache",
sizeof(struct hlist_head),
mhash_entries, 19,
HASH_ZERO,
&m_hash_shift, &m_hash_mask, 0, 0);
mountpoint_hashtable = alloc_large_system_hash("Mountpoint-cache",
sizeof(struct hlist_head),
mphash_entries, 19,
HASH_ZERO,
&mp_hash_shift, &mp_hash_mask, 0, 0);
if (!mount_hashtable || !mountpoint_hashtable)
panic("Failed to allocate mount hash table\n");
kernfs_init();
err = sysfs_init();
if (err)
printk(KERN_WARNING "%s: sysfs_init error: %d\n",
__func__, err);
fs_kobj = kobject_create_and_add("fs", NULL);
if (!fs_kobj)
printk(KERN_WARNING "%s: kobj create error\n", __func__);
shmem_init();
init_rootfs();
init_mount_tree();
}
void put_mnt_ns(struct mnt_namespace *ns)
{
if (!refcount_dec_and_test(&ns->ns.count))
return;
drop_collected_mounts(&ns->root->mnt);
free_mnt_ns(ns);
}
struct vfsmount *kern_mount(struct file_system_type *type)
{
struct vfsmount *mnt;
mnt = vfs_kern_mount(type, SB_KERNMOUNT, type->name, NULL);
if (!IS_ERR(mnt)) {
/*
* it is a longterm mount, don't release mnt until
* we unmount before file sys is unregistered
*/
real_mount(mnt)->mnt_ns = MNT_NS_INTERNAL;
}
return mnt;
}
EXPORT_SYMBOL_GPL(kern_mount);
void kern_unmount(struct vfsmount *mnt)
{
/* release long term mount so mount point can be released */
if (!IS_ERR(mnt)) {
mnt_make_shortterm(mnt);
synchronize_rcu(); /* yecchhh... */
mntput(mnt);
}
}
EXPORT_SYMBOL(kern_unmount);
void kern_unmount_array(struct vfsmount *mnt[], unsigned int num)
{
unsigned int i;
for (i = 0; i < num; i++)
mnt_make_shortterm(mnt[i]);
synchronize_rcu_expedited();
for (i = 0; i < num; i++)
mntput(mnt[i]);
}
EXPORT_SYMBOL(kern_unmount_array);
bool our_mnt(struct vfsmount *mnt)
{
return check_mnt(real_mount(mnt));
}
bool current_chrooted(void)
{
/* Does the current process have a non-standard root */
struct path ns_root;
struct path fs_root;
bool chrooted;
/* Find the namespace root */
ns_root.mnt = ¤t->nsproxy->mnt_ns->root->mnt;
ns_root.dentry = ns_root.mnt->mnt_root;
path_get(&ns_root);
while (d_mountpoint(ns_root.dentry) && follow_down_one(&ns_root))
;
get_fs_root(current->fs, &fs_root);
chrooted = !path_equal(&fs_root, &ns_root);
path_put(&fs_root);
path_put(&ns_root);
return chrooted;
}
static bool mnt_already_visible(struct mnt_namespace *ns,
const struct super_block *sb,
int *new_mnt_flags)
{
int new_flags = *new_mnt_flags;
struct mount *mnt, *n;
bool visible = false;
down_read(&namespace_sem);
rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
struct mount *child;
int mnt_flags;
if (mnt->mnt.mnt_sb->s_type != sb->s_type)
continue;
/* This mount is not fully visible if it's root directory
* is not the root directory of the filesystem.
*/
if (mnt->mnt.mnt_root != mnt->mnt.mnt_sb->s_root)
continue;
/* A local view of the mount flags */
mnt_flags = mnt->mnt.mnt_flags;
/* Don't miss readonly hidden in the superblock flags */
if (sb_rdonly(mnt->mnt.mnt_sb))
mnt_flags |= MNT_LOCK_READONLY;
/* Verify the mount flags are equal to or more permissive
* than the proposed new mount.
*/
if ((mnt_flags & MNT_LOCK_READONLY) &&
!(new_flags & MNT_READONLY))
continue;
if ((mnt_flags & MNT_LOCK_ATIME) &&
((mnt_flags & MNT_ATIME_MASK) != (new_flags & MNT_ATIME_MASK)))
continue;
/* This mount is not fully visible if there are any
* locked child mounts that cover anything except for
* empty directories.
*/
list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) {
struct inode *inode = child->mnt_mountpoint->d_inode;
/* Only worry about locked mounts */
if (!(child->mnt.mnt_flags & MNT_LOCKED))
continue;
/* Is the directory permanetly empty? */
if (!is_empty_dir_inode(inode))
goto next;
}
/* Preserve the locked attributes */
*new_mnt_flags |= mnt_flags & (MNT_LOCK_READONLY | \
MNT_LOCK_ATIME);
visible = true;
goto found;
next: ;
}
found:
up_read(&namespace_sem);
return visible;
}
static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags)
{
const unsigned long required_iflags = SB_I_NOEXEC | SB_I_NODEV;
struct mnt_namespace *ns = current->nsproxy->mnt_ns;
unsigned long s_iflags;
if (ns->user_ns == &init_user_ns)
return false;
/* Can this filesystem be too revealing? */
s_iflags = sb->s_iflags;
if (!(s_iflags & SB_I_USERNS_VISIBLE))
return false;
if ((s_iflags & required_iflags) != required_iflags) {
WARN_ONCE(1, "Expected s_iflags to contain 0x%lx\n",
required_iflags);
return true;
}
return !mnt_already_visible(ns, sb, new_mnt_flags);
}
bool mnt_may_suid(struct vfsmount *mnt)
{
/*
* Foreign mounts (accessed via fchdir or through /proc
* symlinks) are always treated as if they are nosuid. This
* prevents namespaces from trusting potentially unsafe
* suid/sgid bits, file caps, or security labels that originate
* in other namespaces.
*/
return !(mnt->mnt_flags & MNT_NOSUID) && check_mnt(real_mount(mnt)) &&
current_in_userns(mnt->mnt_sb->s_user_ns);
}
static struct ns_common *mntns_get(struct task_struct *task)
{
struct ns_common *ns = NULL;
struct nsproxy *nsproxy;
task_lock(task);
nsproxy = task->nsproxy;
if (nsproxy) {
ns = &nsproxy->mnt_ns->ns;
get_mnt_ns(to_mnt_ns(ns));
}
task_unlock(task);
return ns;
}
static void mntns_put(struct ns_common *ns)
{
put_mnt_ns(to_mnt_ns(ns));
}
static int mntns_install(struct nsset *nsset, struct ns_common *ns)
{
struct nsproxy *nsproxy = nsset->nsproxy;
struct fs_struct *fs = nsset->fs;
struct mnt_namespace *mnt_ns = to_mnt_ns(ns), *old_mnt_ns;
struct user_namespace *user_ns = nsset->cred->user_ns;
struct path root;
int err;
if (!ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN) ||
!ns_capable(user_ns, CAP_SYS_CHROOT) ||
!ns_capable(user_ns, CAP_SYS_ADMIN))
return -EPERM;
if (is_anon_ns(mnt_ns))
return -EINVAL;
if (fs->users != 1)
return -EINVAL;
get_mnt_ns(mnt_ns);
old_mnt_ns = nsproxy->mnt_ns;
nsproxy->mnt_ns = mnt_ns;
/* Find the root */
err = vfs_path_lookup(mnt_ns->root->mnt.mnt_root, &mnt_ns->root->mnt,
"/", LOOKUP_DOWN, &root);
if (err) {
/* revert to old namespace */
nsproxy->mnt_ns = old_mnt_ns;
put_mnt_ns(mnt_ns);
return err;
}
put_mnt_ns(old_mnt_ns);
/* Update the pwd and root */
set_fs_pwd(fs, &root);
set_fs_root(fs, &root);
path_put(&root);
return 0;
}
static struct user_namespace *mntns_owner(struct ns_common *ns)
{
return to_mnt_ns(ns)->user_ns;
}
const struct proc_ns_operations mntns_operations = {
.name = "mnt",
.type = CLONE_NEWNS,
.get = mntns_get,
.put = mntns_put,
.install = mntns_install,
.owner = mntns_owner,
};
#ifdef CONFIG_SYSCTL
static struct ctl_table fs_namespace_sysctls[] = {
{
.procname = "mount-max",
.data = &sysctl_mount_max,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ONE,
},
};
static int __init init_fs_namespace_sysctls(void)
{
register_sysctl_init("fs", fs_namespace_sysctls);
return 0;
}
fs_initcall(init_fs_namespace_sysctls);
#endif /* CONFIG_SYSCTL */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/lib/cmdline.c
* Helper functions generally used for parsing kernel command line
* and module options.
*
* Code and copyrights come from init/main.c and arch/i386/kernel/setup.c.
*
* GNU Indent formatting options for this file: -kr -i8 -npsl -pcs
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/ctype.h>
/*
* If a hyphen was found in get_option, this will handle the
* range of numbers, M-N. This will expand the range and insert
* the values[M, M+1, ..., N] into the ints array in get_options.
*/
static int get_range(char **str, int *pint, int n)
{
int x, inc_counter, upper_range;
(*str)++;
upper_range = simple_strtol((*str), NULL, 0);
inc_counter = upper_range - *pint;
for (x = *pint; n && x < upper_range; x++, n--)
*pint++ = x;
return inc_counter;
}
/**
* get_option - Parse integer from an option string
* @str: option string
* @pint: (optional output) integer value parsed from @str
*
* Read an int from an option string; if available accept a subsequent
* comma as well.
*
* When @pint is NULL the function can be used as a validator of
* the current option in the string.
*
* Return values:
* 0 - no int in string
* 1 - int found, no subsequent comma
* 2 - int found including a subsequent comma
* 3 - hyphen found to denote a range
*
* Leading hyphen without integer is no integer case, but we consume it
* for the sake of simplification.
*/
int get_option(char **str, int *pint)
{
char *cur = *str;
int value;
if (!cur || !(*cur))
return 0;
if (*cur == '-')
value = -simple_strtoull(++cur, str, 0);
else
value = simple_strtoull(cur, str, 0);
if (pint)
*pint = value;
if (cur == *str)
return 0;
if (**str == ',') {
(*str)++;
return 2;
}
if (**str == '-')
return 3;
return 1;
}
EXPORT_SYMBOL(get_option);
/**
* get_options - Parse a string into a list of integers
* @str: String to be parsed
* @nints: size of integer array
* @ints: integer array (must have room for at least one element)
*
* This function parses a string containing a comma-separated
* list of integers, a hyphen-separated range of _positive_ integers,
* or a combination of both. The parse halts when the array is
* full, or when no more numbers can be retrieved from the
* string.
*
* When @nints is 0, the function just validates the given @str and
* returns the amount of parseable integers as described below.
*
* Returns:
*
* The first element is filled by the number of collected integers
* in the range. The rest is what was parsed from the @str.
*
* Return value is the character in the string which caused
* the parse to end (typically a null terminator, if @str is
* completely parseable).
*/
char *get_options(const char *str, int nints, int *ints)
{
bool validate = (nints == 0);
int res, i = 1;
while (i < nints || validate) {
int *pint = validate ? ints : ints + i;
res = get_option((char **)&str, pint);
if (res == 0)
break;
if (res == 3) {
int n = validate ? 0 : nints - i;
int range_nums;
range_nums = get_range((char **)&str, pint, n);
if (range_nums < 0)
break;
/*
* Decrement the result by one to leave out the
* last number in the range. The next iteration
* will handle the upper number in the range
*/
i += (range_nums - 1);
}
i++;
if (res == 1)
break;
}
ints[0] = i - 1;
return (char *)str;
}
EXPORT_SYMBOL(get_options);
/**
* memparse - parse a string with mem suffixes into a number
* @ptr: Where parse begins
* @retptr: (output) Optional pointer to next char after parse completes
*
* Parses a string into a number. The number stored at @ptr is
* potentially suffixed with K, M, G, T, P, E.
*/
unsigned long long memparse(const char *ptr, char **retptr)
{
char *endptr; /* local pointer to end of parsed string */
unsigned long long ret = simple_strtoull(ptr, &endptr, 0);
switch (*endptr) {
case 'E':
case 'e':
ret <<= 10;
fallthrough;
case 'P':
case 'p':
ret <<= 10;
fallthrough;
case 'T':
case 't':
ret <<= 10;
fallthrough;
case 'G':
case 'g':
ret <<= 10;
fallthrough;
case 'M':
case 'm':
ret <<= 10;
fallthrough;
case 'K':
case 'k':
ret <<= 10;
endptr++;
fallthrough;
default:
break;
}
if (retptr)
*retptr = endptr;
return ret;
}
EXPORT_SYMBOL(memparse);
/**
* parse_option_str - Parse a string and check an option is set or not
* @str: String to be parsed
* @option: option name
*
* This function parses a string containing a comma-separated list of
* strings like a=b,c.
*
* Return true if there's such option in the string, or return false.
*/
bool parse_option_str(const char *str, const char *option)
{
while (*str) { if (!strncmp(str, option, strlen(option))) { str += strlen(option); if (!*str || *str == ',')
return true;
}
while (*str && *str != ',') str++; if (*str == ',') str++;
}
return false;
}
/*
* Parse a string to get a param value pair.
* You can use " around spaces, but can't escape ".
* Hyphens and underscores equivalent in parameter names.
*/
char *next_arg(char *args, char **param, char **val)
{
unsigned int i, equals = 0;
int in_quote = 0, quoted = 0;
if (*args == '"') {
args++;
in_quote = 1;
quoted = 1;
}
for (i = 0; args[i]; i++) {
if (isspace(args[i]) && !in_quote)
break;
if (equals == 0) {
if (args[i] == '=')
equals = i;
}
if (args[i] == '"')
in_quote = !in_quote;
}
*param = args;
if (!equals)
*val = NULL;
else {
args[equals] = '\0';
*val = args + equals + 1;
/* Don't include quotes in value. */
if (**val == '"') {
(*val)++;
if (args[i-1] == '"')
args[i-1] = '\0';
}
}
if (quoted && i > 0 && args[i-1] == '"')
args[i-1] = '\0';
if (args[i]) {
args[i] = '\0';
args += i + 1;
} else
args += i;
/* Chew up trailing spaces. */
return skip_spaces(args);
}
EXPORT_SYMBOL(next_arg);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (c) 2000-2001 Vojtech Pavlik
* Copyright (c) 2006-2010 Jiri Kosina
*
* HID to Linux Input mapping
*/
/*
*
* Should you need to contact me, the author, you can do so either by
* e-mail - mail your message to <vojtech@ucw.cz>, or by paper mail:
* Vojtech Pavlik, Simunkova 1594, Prague 8, 182 00 Czech Republic
*/
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/hid.h>
#include <linux/hid-debug.h>
#include "hid-ids.h"
#define unk KEY_UNKNOWN
static const unsigned char hid_keyboard[256] = {
0, 0, 0, 0, 30, 48, 46, 32, 18, 33, 34, 35, 23, 36, 37, 38,
50, 49, 24, 25, 16, 19, 31, 20, 22, 47, 17, 45, 21, 44, 2, 3,
4, 5, 6, 7, 8, 9, 10, 11, 28, 1, 14, 15, 57, 12, 13, 26,
27, 43, 43, 39, 40, 41, 51, 52, 53, 58, 59, 60, 61, 62, 63, 64,
65, 66, 67, 68, 87, 88, 99, 70,119,110,102,104,111,107,109,106,
105,108,103, 69, 98, 55, 74, 78, 96, 79, 80, 81, 75, 76, 77, 71,
72, 73, 82, 83, 86,127,116,117,183,184,185,186,187,188,189,190,
191,192,193,194,134,138,130,132,128,129,131,137,133,135,136,113,
115,114,unk,unk,unk,121,unk, 89, 93,124, 92, 94, 95,unk,unk,unk,
122,123, 90, 91, 85,unk,unk,unk,unk,unk,unk,unk,111,unk,unk,unk,
unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,
unk,unk,unk,unk,unk,unk,179,180,unk,unk,unk,unk,unk,unk,unk,unk,
unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,unk,
unk,unk,unk,unk,unk,unk,unk,unk,111,unk,unk,unk,unk,unk,unk,unk,
29, 42, 56,125, 97, 54,100,126,164,166,165,163,161,115,114,113,
150,158,159,128,136,177,178,176,142,152,173,140,unk,unk,unk,unk
};
static const struct {
__s32 x;
__s32 y;
} hid_hat_to_axis[] = {{ 0, 0}, { 0,-1}, { 1,-1}, { 1, 0}, { 1, 1}, { 0, 1}, {-1, 1}, {-1, 0}, {-1,-1}};
struct usage_priority {
__u32 usage; /* the HID usage associated */
bool global; /* we assume all usages to be slotted,
* unless global
*/
unsigned int slot_overwrite; /* for globals: allows to set the usage
* before or after the slots
*/
};
/*
* hid-input will convert this list into priorities:
* the first element will have the highest priority
* (the length of the following array) and the last
* element the lowest (1).
*
* hid-input will then shift the priority by 8 bits to leave some space
* in case drivers want to interleave other fields.
*
* To accommodate slotted devices, the slot priority is
* defined in the next 8 bits (defined by 0xff - slot).
*
* If drivers want to add fields before those, hid-input will
* leave out the first 8 bits of the priority value.
*
* This still leaves us 65535 individual priority values.
*/
static const struct usage_priority hidinput_usages_priorities[] = {
{ /* Eraser (eraser touching) must always come before tipswitch */
.usage = HID_DG_ERASER,
},
{ /* Invert must always come before In Range */
.usage = HID_DG_INVERT,
},
{ /* Is the tip of the tool touching? */
.usage = HID_DG_TIPSWITCH,
},
{ /* Tip Pressure might emulate tip switch */
.usage = HID_DG_TIPPRESSURE,
},
{ /* In Range needs to come after the other tool states */
.usage = HID_DG_INRANGE,
},
};
#define map_abs(c) hid_map_usage(hidinput, usage, &bit, &max, EV_ABS, (c))
#define map_rel(c) hid_map_usage(hidinput, usage, &bit, &max, EV_REL, (c))
#define map_key(c) hid_map_usage(hidinput, usage, &bit, &max, EV_KEY, (c))
#define map_led(c) hid_map_usage(hidinput, usage, &bit, &max, EV_LED, (c))
#define map_msc(c) hid_map_usage(hidinput, usage, &bit, &max, EV_MSC, (c))
#define map_abs_clear(c) hid_map_usage_clear(hidinput, usage, &bit, \
&max, EV_ABS, (c))
#define map_key_clear(c) hid_map_usage_clear(hidinput, usage, &bit, \
&max, EV_KEY, (c))
static bool match_scancode(struct hid_usage *usage,
unsigned int cur_idx, unsigned int scancode)
{
return (usage->hid & (HID_USAGE_PAGE | HID_USAGE)) == scancode;
}
static bool match_keycode(struct hid_usage *usage,
unsigned int cur_idx, unsigned int keycode)
{
/*
* We should exclude unmapped usages when doing lookup by keycode.
*/
return (usage->type == EV_KEY && usage->code == keycode);
}
static bool match_index(struct hid_usage *usage,
unsigned int cur_idx, unsigned int idx)
{
return cur_idx == idx;
}
typedef bool (*hid_usage_cmp_t)(struct hid_usage *usage,
unsigned int cur_idx, unsigned int val);
static struct hid_usage *hidinput_find_key(struct hid_device *hid,
hid_usage_cmp_t match,
unsigned int value,
unsigned int *usage_idx)
{
unsigned int i, j, k, cur_idx = 0;
struct hid_report *report;
struct hid_usage *usage;
for (k = HID_INPUT_REPORT; k <= HID_OUTPUT_REPORT; k++) { list_for_each_entry(report, &hid->report_enum[k].report_list, list) { for (i = 0; i < report->maxfield; i++) { for (j = 0; j < report->field[i]->maxusage; j++) { usage = report->field[i]->usage + j;
if (usage->type == EV_KEY || usage->type == 0) {
if (match(usage, cur_idx, value)) { if (usage_idx) *usage_idx = cur_idx;
return usage;
}
cur_idx++;
}
}
}
}
}
return NULL;
}
static struct hid_usage *hidinput_locate_usage(struct hid_device *hid,
const struct input_keymap_entry *ke,
unsigned int *index)
{
struct hid_usage *usage;
unsigned int scancode; if (ke->flags & INPUT_KEYMAP_BY_INDEX) usage = hidinput_find_key(hid, match_index, ke->index, index); else if (input_scancode_to_scalar(ke, &scancode) == 0) usage = hidinput_find_key(hid, match_scancode, scancode, index);
else
usage = NULL;
return usage;
}
static int hidinput_getkeycode(struct input_dev *dev,
struct input_keymap_entry *ke)
{
struct hid_device *hid = input_get_drvdata(dev);
struct hid_usage *usage;
unsigned int scancode, index;
usage = hidinput_locate_usage(hid, ke, &index);
if (usage) {
ke->keycode = usage->type == EV_KEY ? usage->code : KEY_RESERVED;
ke->index = index;
scancode = usage->hid & (HID_USAGE_PAGE | HID_USAGE);
ke->len = sizeof(scancode);
memcpy(ke->scancode, &scancode, sizeof(scancode));
return 0;
}
return -EINVAL;
}
static int hidinput_setkeycode(struct input_dev *dev,
const struct input_keymap_entry *ke,
unsigned int *old_keycode)
{
struct hid_device *hid = input_get_drvdata(dev);
struct hid_usage *usage;
usage = hidinput_locate_usage(hid, ke, NULL);
if (usage) {
*old_keycode = usage->type == EV_KEY ? usage->code : KEY_RESERVED;
usage->type = EV_KEY;
usage->code = ke->keycode;
clear_bit(*old_keycode, dev->keybit);
set_bit(usage->code, dev->keybit);
dbg_hid("Assigned keycode %d to HID usage code %x\n",
usage->code, usage->hid);
/*
* Set the keybit for the old keycode if the old keycode is used
* by another key
*/
if (hidinput_find_key(hid, match_keycode, *old_keycode, NULL)) set_bit(*old_keycode, dev->keybit); return 0;
}
return -EINVAL;
}
/**
* hidinput_calc_abs_res - calculate an absolute axis resolution
* @field: the HID report field to calculate resolution for
* @code: axis code
*
* The formula is:
* (logical_maximum - logical_minimum)
* resolution = ----------------------------------------------------------
* (physical_maximum - physical_minimum) * 10 ^ unit_exponent
*
* as seen in the HID specification v1.11 6.2.2.7 Global Items.
*
* Only exponent 1 length units are processed. Centimeters and inches are
* converted to millimeters. Degrees are converted to radians.
*/
__s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code)
{
__s32 unit_exponent = field->unit_exponent;
__s32 logical_extents = field->logical_maximum -
field->logical_minimum;
__s32 physical_extents = field->physical_maximum -
field->physical_minimum;
__s32 prev;
/* Check if the extents are sane */
if (logical_extents <= 0 || physical_extents <= 0)
return 0;
/*
* Verify and convert units.
* See HID specification v1.11 6.2.2.7 Global Items for unit decoding
*/
switch (code) {
case ABS_X:
case ABS_Y:
case ABS_Z:
case ABS_MT_POSITION_X:
case ABS_MT_POSITION_Y:
case ABS_MT_TOOL_X:
case ABS_MT_TOOL_Y:
case ABS_MT_TOUCH_MAJOR:
case ABS_MT_TOUCH_MINOR:
if (field->unit == 0x11) { /* If centimeters */
/* Convert to millimeters */
unit_exponent += 1;
} else if (field->unit == 0x13) { /* If inches */
/* Convert to millimeters */
prev = physical_extents;
physical_extents *= 254;
if (physical_extents < prev)
return 0;
unit_exponent -= 1;
} else {
return 0;
}
break;
case ABS_RX:
case ABS_RY:
case ABS_RZ:
case ABS_WHEEL:
case ABS_TILT_X:
case ABS_TILT_Y:
if (field->unit == 0x14) { /* If degrees */
/* Convert to radians */
prev = logical_extents;
logical_extents *= 573;
if (logical_extents < prev)
return 0;
unit_exponent += 1;
} else if (field->unit != 0x12) { /* If not radians */
return 0;
}
break;
default:
return 0;
}
/* Apply negative unit exponent */
for (; unit_exponent < 0; unit_exponent++) {
prev = logical_extents;
logical_extents *= 10;
if (logical_extents < prev)
return 0;
}
/* Apply positive unit exponent */
for (; unit_exponent > 0; unit_exponent--) {
prev = physical_extents;
physical_extents *= 10;
if (physical_extents < prev)
return 0;
}
/* Calculate resolution */
return DIV_ROUND_CLOSEST(logical_extents, physical_extents);
}
EXPORT_SYMBOL_GPL(hidinput_calc_abs_res);
#ifdef CONFIG_HID_BATTERY_STRENGTH
static enum power_supply_property hidinput_battery_props[] = {
POWER_SUPPLY_PROP_PRESENT,
POWER_SUPPLY_PROP_ONLINE,
POWER_SUPPLY_PROP_CAPACITY,
POWER_SUPPLY_PROP_MODEL_NAME,
POWER_SUPPLY_PROP_STATUS,
POWER_SUPPLY_PROP_SCOPE,
};
#define HID_BATTERY_QUIRK_PERCENT (1 << 0) /* always reports percent */
#define HID_BATTERY_QUIRK_FEATURE (1 << 1) /* ask for feature report */
#define HID_BATTERY_QUIRK_IGNORE (1 << 2) /* completely ignore the battery */
#define HID_BATTERY_QUIRK_AVOID_QUERY (1 << 3) /* do not query the battery */
static const struct hid_device_id hid_battery_quirks[] = {
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ISO),
HID_BATTERY_QUIRK_PERCENT | HID_BATTERY_QUIRK_FEATURE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ANSI),
HID_BATTERY_QUIRK_PERCENT | HID_BATTERY_QUIRK_FEATURE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_ANSI),
HID_BATTERY_QUIRK_PERCENT | HID_BATTERY_QUIRK_FEATURE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_ISO),
HID_BATTERY_QUIRK_PERCENT | HID_BATTERY_QUIRK_FEATURE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI),
HID_BATTERY_QUIRK_PERCENT | HID_BATTERY_QUIRK_FEATURE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE,
USB_DEVICE_ID_APPLE_MAGICTRACKPAD),
HID_BATTERY_QUIRK_IGNORE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM,
USB_DEVICE_ID_ELECOM_BM084),
HID_BATTERY_QUIRK_IGNORE },
{ HID_USB_DEVICE(USB_VENDOR_ID_SYMBOL,
USB_DEVICE_ID_SYMBOL_SCANNER_3),
HID_BATTERY_QUIRK_IGNORE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ASUSTEK,
USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD),
HID_BATTERY_QUIRK_IGNORE },
{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH,
USB_DEVICE_ID_LOGITECH_DINOVO_EDGE_KBD),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_ASUS_TP420IA_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_ASUS_GV301RA_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, USB_DEVICE_ID_ASUS_UX550VE_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_L),
HID_BATTERY_QUIRK_AVOID_QUERY },
{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_PRO_MW),
HID_BATTERY_QUIRK_AVOID_QUERY },
{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_PRO_SW),
HID_BATTERY_QUIRK_AVOID_QUERY },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_ENVY_X360_15),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_ENVY_X360_15T_DR100),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_ENVY_X360_EU0009NV),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_SPECTRE_X360_15),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_SPECTRE_X360_13_AW0020NG),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_SURFACE_GO_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_SURFACE_GO2_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_LENOVO_YOGA_C630_TOUCHSCREEN),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_SPECTRE_X360_13T_AW100),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_SPECTRE_X360_14T_EA100_V1),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_SPECTRE_X360_14T_EA100_V2),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_HP_ENVY_X360_15_EU0556NG),
HID_BATTERY_QUIRK_IGNORE },
{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_CHROMEBOOK_TROGDOR_POMPOM),
HID_BATTERY_QUIRK_AVOID_QUERY },
{}
};
static unsigned find_battery_quirk(struct hid_device *hdev)
{
unsigned quirks = 0;
const struct hid_device_id *match;
match = hid_match_id(hdev, hid_battery_quirks);
if (match != NULL)
quirks = match->driver_data;
return quirks;
}
static int hidinput_scale_battery_capacity(struct hid_device *dev,
int value)
{
if (dev->battery_min < dev->battery_max &&
value >= dev->battery_min && value <= dev->battery_max)
value = ((value - dev->battery_min) * 100) /
(dev->battery_max - dev->battery_min);
return value;
}
static int hidinput_query_battery_capacity(struct hid_device *dev)
{
u8 *buf;
int ret;
buf = kmalloc(4, GFP_KERNEL);
if (!buf)
return -ENOMEM;
ret = hid_hw_raw_request(dev, dev->battery_report_id, buf, 4,
dev->battery_report_type, HID_REQ_GET_REPORT);
if (ret < 2) {
kfree(buf);
return -ENODATA;
}
ret = hidinput_scale_battery_capacity(dev, buf[1]);
kfree(buf);
return ret;
}
static int hidinput_get_battery_property(struct power_supply *psy,
enum power_supply_property prop,
union power_supply_propval *val)
{
struct hid_device *dev = power_supply_get_drvdata(psy);
int value;
int ret = 0;
switch (prop) {
case POWER_SUPPLY_PROP_PRESENT:
case POWER_SUPPLY_PROP_ONLINE:
val->intval = 1;
break;
case POWER_SUPPLY_PROP_CAPACITY:
if (dev->battery_status != HID_BATTERY_REPORTED &&
!dev->battery_avoid_query) {
value = hidinput_query_battery_capacity(dev);
if (value < 0)
return value;
} else {
value = dev->battery_capacity;
}
val->intval = value;
break;
case POWER_SUPPLY_PROP_MODEL_NAME:
val->strval = dev->name;
break;
case POWER_SUPPLY_PROP_STATUS:
if (dev->battery_status != HID_BATTERY_REPORTED &&
!dev->battery_avoid_query) {
value = hidinput_query_battery_capacity(dev);
if (value < 0)
return value;
dev->battery_capacity = value;
dev->battery_status = HID_BATTERY_QUERIED;
}
if (dev->battery_status == HID_BATTERY_UNKNOWN)
val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
else
val->intval = dev->battery_charge_status;
break;
case POWER_SUPPLY_PROP_SCOPE:
val->intval = POWER_SUPPLY_SCOPE_DEVICE;
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
struct hid_field *field, bool is_percentage)
{
struct power_supply_desc *psy_desc;
struct power_supply_config psy_cfg = { .drv_data = dev, };
unsigned quirks;
s32 min, max;
int error;
if (dev->battery)
return 0; /* already initialized? */
quirks = find_battery_quirk(dev);
hid_dbg(dev, "device %x:%x:%x %d quirks %d\n",
dev->bus, dev->vendor, dev->product, dev->version, quirks);
if (quirks & HID_BATTERY_QUIRK_IGNORE)
return 0;
psy_desc = kzalloc(sizeof(*psy_desc), GFP_KERNEL);
if (!psy_desc)
return -ENOMEM;
psy_desc->name = kasprintf(GFP_KERNEL, "hid-%s-battery",
strlen(dev->uniq) ?
dev->uniq : dev_name(&dev->dev));
if (!psy_desc->name) {
error = -ENOMEM;
goto err_free_mem;
}
psy_desc->type = POWER_SUPPLY_TYPE_BATTERY;
psy_desc->properties = hidinput_battery_props;
psy_desc->num_properties = ARRAY_SIZE(hidinput_battery_props);
psy_desc->use_for_apm = 0;
psy_desc->get_property = hidinput_get_battery_property;
min = field->logical_minimum;
max = field->logical_maximum;
if (is_percentage || (quirks & HID_BATTERY_QUIRK_PERCENT)) {
min = 0;
max = 100;
}
if (quirks & HID_BATTERY_QUIRK_FEATURE)
report_type = HID_FEATURE_REPORT;
dev->battery_min = min;
dev->battery_max = max;
dev->battery_report_type = report_type;
dev->battery_report_id = field->report->id;
dev->battery_charge_status = POWER_SUPPLY_STATUS_DISCHARGING;
/*
* Stylus is normally not connected to the device and thus we
* can't query the device and get meaningful battery strength.
* We have to wait for the device to report it on its own.
*/
dev->battery_avoid_query = report_type == HID_INPUT_REPORT &&
field->physical == HID_DG_STYLUS;
if (quirks & HID_BATTERY_QUIRK_AVOID_QUERY)
dev->battery_avoid_query = true;
dev->battery = power_supply_register(&dev->dev, psy_desc, &psy_cfg);
if (IS_ERR(dev->battery)) {
error = PTR_ERR(dev->battery);
hid_warn(dev, "can't register power supply: %d\n", error);
goto err_free_name;
}
power_supply_powers(dev->battery, &dev->dev);
return 0;
err_free_name:
kfree(psy_desc->name);
err_free_mem:
kfree(psy_desc);
dev->battery = NULL;
return error;
}
static void hidinput_cleanup_battery(struct hid_device *dev)
{
const struct power_supply_desc *psy_desc;
if (!dev->battery)
return;
psy_desc = dev->battery->desc;
power_supply_unregister(dev->battery);
kfree(psy_desc->name);
kfree(psy_desc);
dev->battery = NULL;
}
static void hidinput_update_battery(struct hid_device *dev, int value)
{
int capacity;
if (!dev->battery)
return;
if (value == 0 || value < dev->battery_min || value > dev->battery_max)
return;
capacity = hidinput_scale_battery_capacity(dev, value); if (dev->battery_status != HID_BATTERY_REPORTED || capacity != dev->battery_capacity || ktime_after(ktime_get_coarse(), dev->battery_ratelimit_time)) { dev->battery_capacity = capacity;
dev->battery_status = HID_BATTERY_REPORTED;
dev->battery_ratelimit_time =
ktime_add_ms(ktime_get_coarse(), 30 * 1000);
power_supply_changed(dev->battery);
}
}
static bool hidinput_set_battery_charge_status(struct hid_device *dev,
unsigned int usage, int value)
{
switch (usage) {
case HID_BAT_CHARGING:
dev->battery_charge_status = value ? POWER_SUPPLY_STATUS_CHARGING :
POWER_SUPPLY_STATUS_DISCHARGING;
return true;
}
return false;
}
#else /* !CONFIG_HID_BATTERY_STRENGTH */
static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
struct hid_field *field, bool is_percentage)
{
return 0;
}
static void hidinput_cleanup_battery(struct hid_device *dev)
{
}
static void hidinput_update_battery(struct hid_device *dev, int value)
{
}
static bool hidinput_set_battery_charge_status(struct hid_device *dev,
unsigned int usage, int value)
{
return false;
}
#endif /* CONFIG_HID_BATTERY_STRENGTH */
static bool hidinput_field_in_collection(struct hid_device *device, struct hid_field *field,
unsigned int type, unsigned int usage)
{
struct hid_collection *collection;
collection = &device->collection[field->usage->collection_index];
return collection->type == type && collection->usage == usage;
}
static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_field *field,
struct hid_usage *usage, unsigned int usage_index)
{
struct input_dev *input = hidinput->input;
struct hid_device *device = input_get_drvdata(input);
const struct usage_priority *usage_priority = NULL;
int max = 0, code;
unsigned int i = 0;
unsigned long *bit = NULL;
field->hidinput = hidinput;
if (field->flags & HID_MAIN_ITEM_CONSTANT)
goto ignore;
/* Ignore if report count is out of bounds. */
if (field->report_count < 1)
goto ignore;
/* only LED usages are supported in output fields */
if (field->report_type == HID_OUTPUT_REPORT &&
(usage->hid & HID_USAGE_PAGE) != HID_UP_LED) {
goto ignore;
}
/* assign a priority based on the static list declared here */
for (i = 0; i < ARRAY_SIZE(hidinput_usages_priorities); i++) {
if (usage->hid == hidinput_usages_priorities[i].usage) {
usage_priority = &hidinput_usages_priorities[i];
field->usages_priorities[usage_index] =
(ARRAY_SIZE(hidinput_usages_priorities) - i) << 8;
break;
}
}
/*
* For slotted devices, we need to also add the slot index
* in the priority.
*/
if (usage_priority && usage_priority->global)
field->usages_priorities[usage_index] |=
usage_priority->slot_overwrite;
else
field->usages_priorities[usage_index] |=
(0xff - field->slot_idx) << 16;
if (device->driver->input_mapping) {
int ret = device->driver->input_mapping(device, hidinput, field,
usage, &bit, &max);
if (ret > 0)
goto mapped;
if (ret < 0)
goto ignore;
}
switch (usage->hid & HID_USAGE_PAGE) {
case HID_UP_UNDEFINED:
goto ignore;
case HID_UP_KEYBOARD:
set_bit(EV_REP, input->evbit);
if ((usage->hid & HID_USAGE) < 256) {
if (!hid_keyboard[usage->hid & HID_USAGE]) goto ignore;
map_key_clear(hid_keyboard[usage->hid & HID_USAGE]);
} else
map_key(KEY_UNKNOWN);
break;
case HID_UP_BUTTON:
code = ((usage->hid - 1) & HID_USAGE);
switch (field->application) {
case HID_GD_MOUSE:
case HID_GD_POINTER: code += BTN_MOUSE; break;
case HID_GD_JOYSTICK:
if (code <= 0xf)
code += BTN_JOYSTICK;
else
code += BTN_TRIGGER_HAPPY - 0x10;
break;
case HID_GD_GAMEPAD:
if (code <= 0xf)
code += BTN_GAMEPAD;
else
code += BTN_TRIGGER_HAPPY - 0x10;
break;
case HID_CP_CONSUMER_CONTROL:
if (hidinput_field_in_collection(device, field,
HID_COLLECTION_NAMED_ARRAY,
HID_CP_PROGRAMMABLEBUTTONS)) {
if (code <= 0x1d)
code += KEY_MACRO1;
else
code += BTN_TRIGGER_HAPPY - 0x1e;
break;
}
fallthrough;
default:
switch (field->physical) {
case HID_GD_MOUSE:
case HID_GD_POINTER: code += BTN_MOUSE; break;
case HID_GD_JOYSTICK: code += BTN_JOYSTICK; break;
case HID_GD_GAMEPAD: code += BTN_GAMEPAD; break;
default: code += BTN_MISC;
}
}
map_key(code);
break;
case HID_UP_SIMULATION:
switch (usage->hid & 0xffff) {
case 0xba: map_abs(ABS_RUDDER); break;
case 0xbb: map_abs(ABS_THROTTLE); break;
case 0xc4: map_abs(ABS_GAS); break;
case 0xc5: map_abs(ABS_BRAKE); break;
case 0xc8: map_abs(ABS_WHEEL); break;
default: goto ignore;
}
break;
case HID_UP_GENDESK:
if ((usage->hid & 0xf0) == 0x80) { /* SystemControl */
switch (usage->hid & 0xf) {
case 0x1: map_key_clear(KEY_POWER); break;
case 0x2: map_key_clear(KEY_SLEEP); break;
case 0x3: map_key_clear(KEY_WAKEUP); break;
case 0x4: map_key_clear(KEY_CONTEXT_MENU); break;
case 0x5: map_key_clear(KEY_MENU); break;
case 0x6: map_key_clear(KEY_PROG1); break;
case 0x7: map_key_clear(KEY_HELP); break;
case 0x8: map_key_clear(KEY_EXIT); break;
case 0x9: map_key_clear(KEY_SELECT); break;
case 0xa: map_key_clear(KEY_RIGHT); break;
case 0xb: map_key_clear(KEY_LEFT); break;
case 0xc: map_key_clear(KEY_UP); break;
case 0xd: map_key_clear(KEY_DOWN); break;
case 0xe: map_key_clear(KEY_POWER2); break;
case 0xf: map_key_clear(KEY_RESTART); break;
default: goto unknown;
}
break;
}
if ((usage->hid & 0xf0) == 0xa0) { /* SystemControl */
switch (usage->hid & 0xf) {
case 0x9: map_key_clear(KEY_MICMUTE); break;
default: goto ignore;
}
break;
}
if ((usage->hid & 0xf0) == 0xb0) { /* SC - Display */
switch (usage->hid & 0xf) {
case 0x05: map_key_clear(KEY_SWITCHVIDEOMODE); break;
default: goto ignore;
}
break;
}
/*
* Some lazy vendors declare 255 usages for System Control,
* leading to the creation of ABS_X|Y axis and too many others.
* It wouldn't be a problem if joydev doesn't consider the
* device as a joystick then.
*/
if (field->application == HID_GD_SYSTEM_CONTROL)
goto ignore;
if ((usage->hid & 0xf0) == 0x90) { /* D-pad */
switch (usage->hid) {
case HID_GD_UP: usage->hat_dir = 1; break;
case HID_GD_DOWN: usage->hat_dir = 5; break;
case HID_GD_RIGHT: usage->hat_dir = 3; break;
case HID_GD_LEFT: usage->hat_dir = 7; break;
default: goto unknown;
}
if (field->dpad) {
map_abs(field->dpad);
goto ignore;
}
map_abs(ABS_HAT0X);
break;
}
switch (usage->hid) {
/* These usage IDs map directly to the usage codes. */
case HID_GD_X: case HID_GD_Y: case HID_GD_Z:
case HID_GD_RX: case HID_GD_RY: case HID_GD_RZ:
if (field->flags & HID_MAIN_ITEM_RELATIVE)
map_rel(usage->hid & 0xf);
else
map_abs_clear(usage->hid & 0xf);
break;
case HID_GD_WHEEL:
if (field->flags & HID_MAIN_ITEM_RELATIVE) {
set_bit(REL_WHEEL, input->relbit);
map_rel(REL_WHEEL_HI_RES);
} else {
map_abs(usage->hid & 0xf);
}
break;
case HID_GD_SLIDER: case HID_GD_DIAL:
if (field->flags & HID_MAIN_ITEM_RELATIVE)
map_rel(usage->hid & 0xf);
else
map_abs(usage->hid & 0xf);
break;
case HID_GD_HATSWITCH:
usage->hat_min = field->logical_minimum;
usage->hat_max = field->logical_maximum;
map_abs(ABS_HAT0X);
break;
case HID_GD_START: map_key_clear(BTN_START); break;
case HID_GD_SELECT: map_key_clear(BTN_SELECT); break;
case HID_GD_RFKILL_BTN:
/* MS wireless radio ctl extension, also check CA */
if (field->application == HID_GD_WIRELESS_RADIO_CTLS) {
map_key_clear(KEY_RFKILL);
/* We need to simulate the btn release */
field->flags |= HID_MAIN_ITEM_RELATIVE;
break;
}
goto unknown;
default: goto unknown;
}
break;
case HID_UP_LED:
switch (usage->hid & 0xffff) { /* HID-Value: */
case 0x01: map_led (LED_NUML); break; /* "Num Lock" */
case 0x02: map_led (LED_CAPSL); break; /* "Caps Lock" */
case 0x03: map_led (LED_SCROLLL); break; /* "Scroll Lock" */
case 0x04: map_led (LED_COMPOSE); break; /* "Compose" */
case 0x05: map_led (LED_KANA); break; /* "Kana" */
case 0x27: map_led (LED_SLEEP); break; /* "Stand-By" */
case 0x4c: map_led (LED_SUSPEND); break; /* "System Suspend" */
case 0x09: map_led (LED_MUTE); break; /* "Mute" */
case 0x4b: map_led (LED_MISC); break; /* "Generic Indicator" */
case 0x19: map_led (LED_MAIL); break; /* "Message Waiting" */
case 0x4d: map_led (LED_CHARGING); break; /* "External Power Connected" */
default: goto ignore;
}
break;
case HID_UP_DIGITIZER:
if ((field->application & 0xff) == 0x01) /* Digitizer */
__set_bit(INPUT_PROP_POINTER, input->propbit);
else if ((field->application & 0xff) == 0x02) /* Pen */
__set_bit(INPUT_PROP_DIRECT, input->propbit);
switch (usage->hid & 0xff) {
case 0x00: /* Undefined */
goto ignore;
case 0x30: /* TipPressure */
if (!test_bit(BTN_TOUCH, input->keybit)) {
device->quirks |= HID_QUIRK_NOTOUCH;
set_bit(EV_KEY, input->evbit);
set_bit(BTN_TOUCH, input->keybit);
}
map_abs_clear(ABS_PRESSURE);
break;
case 0x32: /* InRange */
switch (field->physical) {
case HID_DG_PUCK:
map_key(BTN_TOOL_MOUSE);
break;
case HID_DG_FINGER:
map_key(BTN_TOOL_FINGER);
break;
default:
/*
* If the physical is not given,
* rely on the application.
*/
if (!field->physical) {
switch (field->application) {
case HID_DG_TOUCHSCREEN:
case HID_DG_TOUCHPAD:
map_key_clear(BTN_TOOL_FINGER);
break;
default:
map_key_clear(BTN_TOOL_PEN);
}
} else {
map_key(BTN_TOOL_PEN);
}
break;
}
break;
case 0x3b: /* Battery Strength */
hidinput_setup_battery(device, HID_INPUT_REPORT, field, false);
usage->type = EV_PWR;
return;
case 0x3c: /* Invert */
device->quirks &= ~HID_QUIRK_NOINVERT;
map_key_clear(BTN_TOOL_RUBBER);
break;
case 0x3d: /* X Tilt */
map_abs_clear(ABS_TILT_X);
break;
case 0x3e: /* Y Tilt */
map_abs_clear(ABS_TILT_Y);
break;
case 0x33: /* Touch */
case 0x42: /* TipSwitch */
case 0x43: /* TipSwitch2 */
device->quirks &= ~HID_QUIRK_NOTOUCH;
map_key_clear(BTN_TOUCH);
break;
case 0x44: /* BarrelSwitch */
map_key_clear(BTN_STYLUS);
break;
case 0x45: /* ERASER */
/*
* This event is reported when eraser tip touches the surface.
* Actual eraser (BTN_TOOL_RUBBER) is set and released either
* by Invert if tool reports proximity or by Eraser directly.
*/
if (!test_bit(BTN_TOOL_RUBBER, input->keybit)) {
device->quirks |= HID_QUIRK_NOINVERT;
set_bit(BTN_TOOL_RUBBER, input->keybit);
}
map_key_clear(BTN_TOUCH);
break;
case 0x46: /* TabletPick */
case 0x5a: /* SecondaryBarrelSwitch */
map_key_clear(BTN_STYLUS2);
break;
case 0x5b: /* TransducerSerialNumber */
case 0x6e: /* TransducerSerialNumber2 */
map_msc(MSC_SERIAL);
break;
default: goto unknown;
}
break;
case HID_UP_TELEPHONY:
switch (usage->hid & HID_USAGE) {
case 0x2f: map_key_clear(KEY_MICMUTE); break;
case 0xb0: map_key_clear(KEY_NUMERIC_0); break;
case 0xb1: map_key_clear(KEY_NUMERIC_1); break;
case 0xb2: map_key_clear(KEY_NUMERIC_2); break;
case 0xb3: map_key_clear(KEY_NUMERIC_3); break;
case 0xb4: map_key_clear(KEY_NUMERIC_4); break;
case 0xb5: map_key_clear(KEY_NUMERIC_5); break;
case 0xb6: map_key_clear(KEY_NUMERIC_6); break;
case 0xb7: map_key_clear(KEY_NUMERIC_7); break;
case 0xb8: map_key_clear(KEY_NUMERIC_8); break;
case 0xb9: map_key_clear(KEY_NUMERIC_9); break;
case 0xba: map_key_clear(KEY_NUMERIC_STAR); break;
case 0xbb: map_key_clear(KEY_NUMERIC_POUND); break;
case 0xbc: map_key_clear(KEY_NUMERIC_A); break;
case 0xbd: map_key_clear(KEY_NUMERIC_B); break;
case 0xbe: map_key_clear(KEY_NUMERIC_C); break;
case 0xbf: map_key_clear(KEY_NUMERIC_D); break;
default: goto ignore;
}
break;
case HID_UP_CONSUMER: /* USB HUT v1.12, pages 75-84 */
switch (usage->hid & HID_USAGE) {
case 0x000: goto ignore;
case 0x030: map_key_clear(KEY_POWER); break;
case 0x031: map_key_clear(KEY_RESTART); break;
case 0x032: map_key_clear(KEY_SLEEP); break;
case 0x034: map_key_clear(KEY_SLEEP); break;
case 0x035: map_key_clear(KEY_KBDILLUMTOGGLE); break;
case 0x036: map_key_clear(BTN_MISC); break;
case 0x040: map_key_clear(KEY_MENU); break; /* Menu */
case 0x041: map_key_clear(KEY_SELECT); break; /* Menu Pick */
case 0x042: map_key_clear(KEY_UP); break; /* Menu Up */
case 0x043: map_key_clear(KEY_DOWN); break; /* Menu Down */
case 0x044: map_key_clear(KEY_LEFT); break; /* Menu Left */
case 0x045: map_key_clear(KEY_RIGHT); break; /* Menu Right */
case 0x046: map_key_clear(KEY_ESC); break; /* Menu Escape */
case 0x047: map_key_clear(KEY_KPPLUS); break; /* Menu Value Increase */
case 0x048: map_key_clear(KEY_KPMINUS); break; /* Menu Value Decrease */
case 0x060: map_key_clear(KEY_INFO); break; /* Data On Screen */
case 0x061: map_key_clear(KEY_SUBTITLE); break; /* Closed Caption */
case 0x063: map_key_clear(KEY_VCR); break; /* VCR/TV */
case 0x065: map_key_clear(KEY_CAMERA); break; /* Snapshot */
case 0x069: map_key_clear(KEY_RED); break;
case 0x06a: map_key_clear(KEY_GREEN); break;
case 0x06b: map_key_clear(KEY_BLUE); break;
case 0x06c: map_key_clear(KEY_YELLOW); break;
case 0x06d: map_key_clear(KEY_ASPECT_RATIO); break;
case 0x06f: map_key_clear(KEY_BRIGHTNESSUP); break;
case 0x070: map_key_clear(KEY_BRIGHTNESSDOWN); break;
case 0x072: map_key_clear(KEY_BRIGHTNESS_TOGGLE); break;
case 0x073: map_key_clear(KEY_BRIGHTNESS_MIN); break;
case 0x074: map_key_clear(KEY_BRIGHTNESS_MAX); break;
case 0x075: map_key_clear(KEY_BRIGHTNESS_AUTO); break;
case 0x076: map_key_clear(KEY_CAMERA_ACCESS_ENABLE); break;
case 0x077: map_key_clear(KEY_CAMERA_ACCESS_DISABLE); break;
case 0x078: map_key_clear(KEY_CAMERA_ACCESS_TOGGLE); break;
case 0x079: map_key_clear(KEY_KBDILLUMUP); break;
case 0x07a: map_key_clear(KEY_KBDILLUMDOWN); break;
case 0x07c: map_key_clear(KEY_KBDILLUMTOGGLE); break;
case 0x082: map_key_clear(KEY_VIDEO_NEXT); break;
case 0x083: map_key_clear(KEY_LAST); break;
case 0x084: map_key_clear(KEY_ENTER); break;
case 0x088: map_key_clear(KEY_PC); break;
case 0x089: map_key_clear(KEY_TV); break;
case 0x08a: map_key_clear(KEY_WWW); break;
case 0x08b: map_key_clear(KEY_DVD); break;
case 0x08c: map_key_clear(KEY_PHONE); break;
case 0x08d: map_key_clear(KEY_PROGRAM); break;
case 0x08e: map_key_clear(KEY_VIDEOPHONE); break;
case 0x08f: map_key_clear(KEY_GAMES); break;
case 0x090: map_key_clear(KEY_MEMO); break;
case 0x091: map_key_clear(KEY_CD); break;
case 0x092: map_key_clear(KEY_VCR); break;
case 0x093: map_key_clear(KEY_TUNER); break;
case 0x094: map_key_clear(KEY_EXIT); break;
case 0x095: map_key_clear(KEY_HELP); break;
case 0x096: map_key_clear(KEY_TAPE); break;
case 0x097: map_key_clear(KEY_TV2); break;
case 0x098: map_key_clear(KEY_SAT); break;
case 0x09a: map_key_clear(KEY_PVR); break;
case 0x09c: map_key_clear(KEY_CHANNELUP); break;
case 0x09d: map_key_clear(KEY_CHANNELDOWN); break;
case 0x0a0: map_key_clear(KEY_VCR2); break;
case 0x0b0: map_key_clear(KEY_PLAY); break;
case 0x0b1: map_key_clear(KEY_PAUSE); break;
case 0x0b2: map_key_clear(KEY_RECORD); break;
case 0x0b3: map_key_clear(KEY_FASTFORWARD); break;
case 0x0b4: map_key_clear(KEY_REWIND); break;
case 0x0b5: map_key_clear(KEY_NEXTSONG); break;
case 0x0b6: map_key_clear(KEY_PREVIOUSSONG); break;
case 0x0b7: map_key_clear(KEY_STOPCD); break;
case 0x0b8: map_key_clear(KEY_EJECTCD); break;
case 0x0bc: map_key_clear(KEY_MEDIA_REPEAT); break;
case 0x0b9: map_key_clear(KEY_SHUFFLE); break;
case 0x0bf: map_key_clear(KEY_SLOW); break;
case 0x0cd: map_key_clear(KEY_PLAYPAUSE); break;
case 0x0cf: map_key_clear(KEY_VOICECOMMAND); break;
case 0x0d8: map_key_clear(KEY_DICTATE); break;
case 0x0d9: map_key_clear(KEY_EMOJI_PICKER); break;
case 0x0e0: map_abs_clear(ABS_VOLUME); break;
case 0x0e2: map_key_clear(KEY_MUTE); break;
case 0x0e5: map_key_clear(KEY_BASSBOOST); break;
case 0x0e9: map_key_clear(KEY_VOLUMEUP); break;
case 0x0ea: map_key_clear(KEY_VOLUMEDOWN); break;
case 0x0f5: map_key_clear(KEY_SLOW); break;
case 0x181: map_key_clear(KEY_BUTTONCONFIG); break;
case 0x182: map_key_clear(KEY_BOOKMARKS); break;
case 0x183: map_key_clear(KEY_CONFIG); break;
case 0x184: map_key_clear(KEY_WORDPROCESSOR); break;
case 0x185: map_key_clear(KEY_EDITOR); break;
case 0x186: map_key_clear(KEY_SPREADSHEET); break;
case 0x187: map_key_clear(KEY_GRAPHICSEDITOR); break;
case 0x188: map_key_clear(KEY_PRESENTATION); break;
case 0x189: map_key_clear(KEY_DATABASE); break;
case 0x18a: map_key_clear(KEY_MAIL); break;
case 0x18b: map_key_clear(KEY_NEWS); break;
case 0x18c: map_key_clear(KEY_VOICEMAIL); break;
case 0x18d: map_key_clear(KEY_ADDRESSBOOK); break;
case 0x18e: map_key_clear(KEY_CALENDAR); break;
case 0x18f: map_key_clear(KEY_TASKMANAGER); break;
case 0x190: map_key_clear(KEY_JOURNAL); break;
case 0x191: map_key_clear(KEY_FINANCE); break;
case 0x192: map_key_clear(KEY_CALC); break;
case 0x193: map_key_clear(KEY_PLAYER); break;
case 0x194: map_key_clear(KEY_FILE); break;
case 0x196: map_key_clear(KEY_WWW); break;
case 0x199: map_key_clear(KEY_CHAT); break;
case 0x19c: map_key_clear(KEY_LOGOFF); break;
case 0x19e: map_key_clear(KEY_COFFEE); break;
case 0x19f: map_key_clear(KEY_CONTROLPANEL); break;
case 0x1a2: map_key_clear(KEY_APPSELECT); break;
case 0x1a3: map_key_clear(KEY_NEXT); break;
case 0x1a4: map_key_clear(KEY_PREVIOUS); break;
case 0x1a6: map_key_clear(KEY_HELP); break;
case 0x1a7: map_key_clear(KEY_DOCUMENTS); break;
case 0x1ab: map_key_clear(KEY_SPELLCHECK); break;
case 0x1ae: map_key_clear(KEY_KEYBOARD); break;
case 0x1b1: map_key_clear(KEY_SCREENSAVER); break;
case 0x1b4: map_key_clear(KEY_FILE); break;
case 0x1b6: map_key_clear(KEY_IMAGES); break;
case 0x1b7: map_key_clear(KEY_AUDIO); break;
case 0x1b8: map_key_clear(KEY_VIDEO); break;
case 0x1bc: map_key_clear(KEY_MESSENGER); break;
case 0x1bd: map_key_clear(KEY_INFO); break;
case 0x1cb: map_key_clear(KEY_ASSISTANT); break;
case 0x201: map_key_clear(KEY_NEW); break;
case 0x202: map_key_clear(KEY_OPEN); break;
case 0x203: map_key_clear(KEY_CLOSE); break;
case 0x204: map_key_clear(KEY_EXIT); break;
case 0x207: map_key_clear(KEY_SAVE); break;
case 0x208: map_key_clear(KEY_PRINT); break;
case 0x209: map_key_clear(KEY_PROPS); break;
case 0x21a: map_key_clear(KEY_UNDO); break;
case 0x21b: map_key_clear(KEY_COPY); break;
case 0x21c: map_key_clear(KEY_CUT); break;
case 0x21d: map_key_clear(KEY_PASTE); break;
case 0x21f: map_key_clear(KEY_FIND); break;
case 0x221: map_key_clear(KEY_SEARCH); break;
case 0x222: map_key_clear(KEY_GOTO); break;
case 0x223: map_key_clear(KEY_HOMEPAGE); break;
case 0x224: map_key_clear(KEY_BACK); break;
case 0x225: map_key_clear(KEY_FORWARD); break;
case 0x226: map_key_clear(KEY_STOP); break;
case 0x227: map_key_clear(KEY_REFRESH); break;
case 0x22a: map_key_clear(KEY_BOOKMARKS); break;
case 0x22d: map_key_clear(KEY_ZOOMIN); break;
case 0x22e: map_key_clear(KEY_ZOOMOUT); break;
case 0x22f: map_key_clear(KEY_ZOOMRESET); break;
case 0x232: map_key_clear(KEY_FULL_SCREEN); break;
case 0x233: map_key_clear(KEY_SCROLLUP); break;
case 0x234: map_key_clear(KEY_SCROLLDOWN); break;
case 0x238: /* AC Pan */
set_bit(REL_HWHEEL, input->relbit);
map_rel(REL_HWHEEL_HI_RES);
break;
case 0x23d: map_key_clear(KEY_EDIT); break;
case 0x25f: map_key_clear(KEY_CANCEL); break;
case 0x269: map_key_clear(KEY_INSERT); break;
case 0x26a: map_key_clear(KEY_DELETE); break;
case 0x279: map_key_clear(KEY_REDO); break;
case 0x289: map_key_clear(KEY_REPLY); break;
case 0x28b: map_key_clear(KEY_FORWARDMAIL); break;
case 0x28c: map_key_clear(KEY_SEND); break;
case 0x29d: map_key_clear(KEY_KBD_LAYOUT_NEXT); break;
case 0x2a2: map_key_clear(KEY_ALL_APPLICATIONS); break;
case 0x2c7: map_key_clear(KEY_KBDINPUTASSIST_PREV); break;
case 0x2c8: map_key_clear(KEY_KBDINPUTASSIST_NEXT); break;
case 0x2c9: map_key_clear(KEY_KBDINPUTASSIST_PREVGROUP); break;
case 0x2ca: map_key_clear(KEY_KBDINPUTASSIST_NEXTGROUP); break;
case 0x2cb: map_key_clear(KEY_KBDINPUTASSIST_ACCEPT); break;
case 0x2cc: map_key_clear(KEY_KBDINPUTASSIST_CANCEL); break;
case 0x29f: map_key_clear(KEY_SCALE); break;
default: map_key_clear(KEY_UNKNOWN);
}
break;
case HID_UP_GENDEVCTRLS:
switch (usage->hid) {
case HID_DC_BATTERYSTRENGTH:
hidinput_setup_battery(device, HID_INPUT_REPORT, field, false);
usage->type = EV_PWR;
return;
}
goto unknown;
case HID_UP_BATTERY:
switch (usage->hid) {
case HID_BAT_ABSOLUTESTATEOFCHARGE:
hidinput_setup_battery(device, HID_INPUT_REPORT, field, true);
usage->type = EV_PWR;
return;
case HID_BAT_CHARGING:
usage->type = EV_PWR;
return;
}
goto unknown;
case HID_UP_CAMERA:
switch (usage->hid & HID_USAGE) {
case 0x020:
map_key_clear(KEY_CAMERA_FOCUS); break;
case 0x021:
map_key_clear(KEY_CAMERA); break;
default:
goto ignore;
}
break;
case HID_UP_HPVENDOR: /* Reported on a Dutch layout HP5308 */
set_bit(EV_REP, input->evbit);
switch (usage->hid & HID_USAGE) {
case 0x021: map_key_clear(KEY_PRINT); break;
case 0x070: map_key_clear(KEY_HP); break;
case 0x071: map_key_clear(KEY_CAMERA); break;
case 0x072: map_key_clear(KEY_SOUND); break;
case 0x073: map_key_clear(KEY_QUESTION); break;
case 0x080: map_key_clear(KEY_EMAIL); break;
case 0x081: map_key_clear(KEY_CHAT); break;
case 0x082: map_key_clear(KEY_SEARCH); break;
case 0x083: map_key_clear(KEY_CONNECT); break;
case 0x084: map_key_clear(KEY_FINANCE); break;
case 0x085: map_key_clear(KEY_SPORT); break;
case 0x086: map_key_clear(KEY_SHOP); break;
default: goto ignore;
}
break;
case HID_UP_HPVENDOR2:
set_bit(EV_REP, input->evbit);
switch (usage->hid & HID_USAGE) {
case 0x001: map_key_clear(KEY_MICMUTE); break;
case 0x003: map_key_clear(KEY_BRIGHTNESSDOWN); break;
case 0x004: map_key_clear(KEY_BRIGHTNESSUP); break;
default: goto ignore;
}
break;
case HID_UP_MSVENDOR:
goto ignore;
case HID_UP_CUSTOM: /* Reported on Logitech and Apple USB keyboards */
set_bit(EV_REP, input->evbit);
goto ignore;
case HID_UP_LOGIVENDOR:
/* intentional fallback */
case HID_UP_LOGIVENDOR2:
/* intentional fallback */
case HID_UP_LOGIVENDOR3:
goto ignore;
case HID_UP_PID:
switch (usage->hid & HID_USAGE) {
case 0xa4: map_key_clear(BTN_DEAD); break;
default: goto ignore;
}
break;
default:
unknown:
if (field->report_size == 1) {
if (field->report->type == HID_OUTPUT_REPORT) {
map_led(LED_MISC);
break;
}
map_key(BTN_MISC);
break;
}
if (field->flags & HID_MAIN_ITEM_RELATIVE) {
map_rel(REL_MISC);
break;
}
map_abs(ABS_MISC);
break;
}
mapped:
/* Mapping failed, bail out */
if (!bit)
return;
if (device->driver->input_mapped &&
device->driver->input_mapped(device, hidinput, field, usage,
&bit, &max) < 0) {
/*
* The driver indicated that no further generic handling
* of the usage is desired.
*/
return;
}
set_bit(usage->type, input->evbit);
/*
* This part is *really* controversial:
* - HID aims at being generic so we should do our best to export
* all incoming events
* - HID describes what events are, so there is no reason for ABS_X
* to be mapped to ABS_Y
* - HID is using *_MISC+N as a default value, but nothing prevents
* *_MISC+N to overwrite a legitimate even, which confuses userspace
* (for instance ABS_MISC + 7 is ABS_MT_SLOT, which has a different
* processing)
*
* If devices still want to use this (at their own risk), they will
* have to use the quirk HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE, but
* the default should be a reliable mapping.
*/
while (usage->code <= max && test_and_set_bit(usage->code, bit)) {
if (device->quirks & HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE) {
usage->code = find_next_zero_bit(bit,
max + 1,
usage->code);
} else {
device->status |= HID_STAT_DUP_DETECTED;
goto ignore;
}
}
if (usage->code > max)
goto ignore;
if (usage->type == EV_ABS) {
int a = field->logical_minimum;
int b = field->logical_maximum;
if ((device->quirks & HID_QUIRK_BADPAD) && (usage->code == ABS_X || usage->code == ABS_Y)) {
a = field->logical_minimum = 0;
b = field->logical_maximum = 255;
}
if (field->application == HID_GD_GAMEPAD || field->application == HID_GD_JOYSTICK)
input_set_abs_params(input, usage->code, a, b, (b - a) >> 8, (b - a) >> 4);
else input_set_abs_params(input, usage->code, a, b, 0, 0);
input_abs_set_res(input, usage->code,
hidinput_calc_abs_res(field, usage->code));
/* use a larger default input buffer for MT devices */
if (usage->code == ABS_MT_POSITION_X && input->hint_events_per_packet == 0)
input_set_events_per_packet(input, 60);
}
if (usage->type == EV_ABS &&
(usage->hat_min < usage->hat_max || usage->hat_dir)) {
int i;
for (i = usage->code; i < usage->code + 2 && i <= max; i++) {
input_set_abs_params(input, i, -1, 1, 0, 0);
set_bit(i, input->absbit);
}
if (usage->hat_dir && !field->dpad)
field->dpad = usage->code;
}
/* for those devices which produce Consumer volume usage as relative,
* we emulate pressing volumeup/volumedown appropriate number of times
* in hidinput_hid_event()
*/
if ((usage->type == EV_ABS) && (field->flags & HID_MAIN_ITEM_RELATIVE) &&
(usage->code == ABS_VOLUME)) {
set_bit(KEY_VOLUMEUP, input->keybit);
set_bit(KEY_VOLUMEDOWN, input->keybit);
}
if (usage->type == EV_KEY) {
set_bit(EV_MSC, input->evbit);
set_bit(MSC_SCAN, input->mscbit);
}
return;
ignore:
usage->type = 0;
usage->code = 0;
}
static void hidinput_handle_scroll(struct hid_usage *usage,
struct input_dev *input,
__s32 value)
{
int code;
int hi_res, lo_res;
if (value == 0)
return;
if (usage->code == REL_WHEEL_HI_RES)
code = REL_WHEEL;
else
code = REL_HWHEEL;
/*
* Windows reports one wheel click as value 120. Where a high-res
* scroll wheel is present, a fraction of 120 is reported instead.
* Our REL_WHEEL_HI_RES axis does the same because all HW must
* adhere to the 120 expectation.
*/
hi_res = value * 120/usage->resolution_multiplier; usage->wheel_accumulated += hi_res;
lo_res = usage->wheel_accumulated/120;
if (lo_res)
usage->wheel_accumulated -= lo_res * 120; input_event(input, EV_REL, code, lo_res);
input_event(input, EV_REL, usage->code, hi_res);
}
static void hid_report_release_tool(struct hid_report *report, struct input_dev *input,
unsigned int tool)
{
/* if the given tool is not currently reported, ignore */
if (!test_bit(tool, input->key))
return;
/*
* if the given tool was previously set, release it,
* release any TOUCH and send an EV_SYN
*/
input_event(input, EV_KEY, BTN_TOUCH, 0);
input_event(input, EV_KEY, tool, 0);
input_event(input, EV_SYN, SYN_REPORT, 0);
report->tool = 0;
}
static void hid_report_set_tool(struct hid_report *report, struct input_dev *input,
unsigned int new_tool)
{
if (report->tool != new_tool) hid_report_release_tool(report, input, report->tool); input_event(input, EV_KEY, new_tool, 1);
report->tool = new_tool;
}
void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct hid_usage *usage, __s32 value)
{
struct input_dev *input;
struct hid_report *report = field->report;
unsigned *quirks = &hid->quirks;
if (!usage->type)
return;
if (usage->type == EV_PWR) { bool handled = hidinput_set_battery_charge_status(hid, usage->hid, value);
if (!handled)
hidinput_update_battery(hid, value);
return;
}
if (!field->hidinput)
return;
input = field->hidinput->input; if (usage->hat_min < usage->hat_max || usage->hat_dir) { int hat_dir = usage->hat_dir;
if (!hat_dir)
hat_dir = (value - usage->hat_min) * 8 / (usage->hat_max - usage->hat_min + 1) + 1; if (hat_dir < 0 || hat_dir > 8) hat_dir = 0; input_event(input, usage->type, usage->code , hid_hat_to_axis[hat_dir].x);
input_event(input, usage->type, usage->code + 1, hid_hat_to_axis[hat_dir].y);
return;
}
/*
* Ignore out-of-range values as per HID specification,
* section 5.10 and 6.2.25, when NULL state bit is present.
* When it's not, clamp the value to match Microsoft's input
* driver as mentioned in "Required HID usages for digitizers":
* https://msdn.microsoft.com/en-us/library/windows/hardware/dn672278(v=vs.85).asp
*
* The logical_minimum < logical_maximum check is done so that we
* don't unintentionally discard values sent by devices which
* don't specify logical min and max.
*/
if ((field->flags & HID_MAIN_ITEM_VARIABLE) && field->logical_minimum < field->logical_maximum) { if (field->flags & HID_MAIN_ITEM_NULL_STATE && (value < field->logical_minimum ||
value > field->logical_maximum)) {
dbg_hid("Ignoring out-of-range value %x\n", value);
return;
}
value = clamp(value,
field->logical_minimum,
field->logical_maximum);
}
switch (usage->hid) {
case HID_DG_ERASER:
report->tool_active |= !!value;
/*
* if eraser is set, we must enforce BTN_TOOL_RUBBER
* to accommodate for devices not following the spec.
*/
if (value)
hid_report_set_tool(report, input, BTN_TOOL_RUBBER); else if (report->tool != BTN_TOOL_RUBBER)
/* value is off, tool is not rubber, ignore */
return;
else if (*quirks & HID_QUIRK_NOINVERT && !test_bit(BTN_TOUCH, input->key)) {
/*
* There is no invert to release the tool, let hid_input
* send BTN_TOUCH with scancode and release the tool after.
*/
hid_report_release_tool(report, input, BTN_TOOL_RUBBER);
return;
}
/* let hid-input set BTN_TOUCH */
break;
case HID_DG_INVERT:
report->tool_active |= !!value;
/*
* If invert is set, we store BTN_TOOL_RUBBER.
*/
if (value)
hid_report_set_tool(report, input, BTN_TOOL_RUBBER); else if (!report->tool_active)
/* tool_active not set means Invert and Eraser are not set */
hid_report_release_tool(report, input, BTN_TOOL_RUBBER);
/* no further processing */
return;
case HID_DG_INRANGE:
report->tool_active |= !!value;
if (report->tool_active) {
/*
* if tool is not set but is marked as active,
* assume ours
*/
if (!report->tool) report->tool = usage->code;
/* drivers may have changed the value behind our back, resend it */
hid_report_set_tool(report, input, report->tool);
} else {
hid_report_release_tool(report, input, usage->code);
}
/* reset tool_active for the next event */
report->tool_active = false;
/* no further processing */
return;
case HID_DG_TIPSWITCH:
report->tool_active |= !!value;
/* if tool is set to RUBBER we should ignore the current value */
if (report->tool == BTN_TOOL_RUBBER)
return;
break;
case HID_DG_TIPPRESSURE:
if (*quirks & HID_QUIRK_NOTOUCH) { int a = field->logical_minimum;
int b = field->logical_maximum;
if (value > a + ((b - a) >> 3)) {
input_event(input, EV_KEY, BTN_TOUCH, 1);
report->tool_active = true;
}
}
break;
case HID_UP_PID | 0x83UL: /* Simultaneous Effects Max */
dbg_hid("Maximum Effects - %d\n",value);
return;
case HID_UP_PID | 0x7fUL:
dbg_hid("PID Pool Report\n");
return;
}
switch (usage->type) {
case EV_KEY:
if (usage->code == 0) /* Key 0 is "unassigned", not KEY_UNKNOWN */
return;
break;
case EV_REL:
if (usage->code == REL_WHEEL_HI_RES ||
usage->code == REL_HWHEEL_HI_RES) {
hidinput_handle_scroll(usage, input, value);
return;
}
break;
case EV_ABS:
if ((field->flags & HID_MAIN_ITEM_RELATIVE) && usage->code == ABS_VOLUME) { int count = abs(value);
int direction = value > 0 ? KEY_VOLUMEUP : KEY_VOLUMEDOWN;
int i;
for (i = 0; i < count; i++) { input_event(input, EV_KEY, direction, 1);
input_sync(input);
input_event(input, EV_KEY, direction, 0);
input_sync(input);
}
return;
} else if (((*quirks & HID_QUIRK_X_INVERT) && usage->code == ABS_X) || ((*quirks & HID_QUIRK_Y_INVERT) && usage->code == ABS_Y)) value = field->logical_maximum - value;
break;
}
/*
* Ignore reports for absolute data if the data didn't change. This is
* not only an optimization but also fixes 'dead' key reports. Some
* RollOver implementations for localized keys (like BACKSLASH/PIPE; HID
* 0x31 and 0x32) report multiple keys, even though a localized keyboard
* can only have one of them physically available. The 'dead' keys
* report constant 0. As all map to the same keycode, they'd confuse
* the input layer. If we filter the 'dead' keys on the HID level, we
* skip the keycode translation and only forward real events.
*/
if (!(field->flags & (HID_MAIN_ITEM_RELATIVE | HID_MAIN_ITEM_BUFFERED_BYTE)) &&
(field->flags & HID_MAIN_ITEM_VARIABLE) &&
usage->usage_index < field->maxusage && value == field->value[usage->usage_index])
return;
/* report the usage code as scancode if the key status has changed */
if (usage->type == EV_KEY &&
(!test_bit(usage->code, input->key)) == value) input_event(input, EV_MSC, MSC_SCAN, usage->hid); input_event(input, usage->type, usage->code, value);
if ((field->flags & HID_MAIN_ITEM_RELATIVE) &&
usage->type == EV_KEY && value) { input_sync(input);
input_event(input, usage->type, usage->code, 0);
}
}
void hidinput_report_event(struct hid_device *hid, struct hid_report *report)
{
struct hid_input *hidinput;
if (hid->quirks & HID_QUIRK_NO_INPUT_SYNC)
return;
list_for_each_entry(hidinput, &hid->inputs, list) input_sync(hidinput->input);
}
EXPORT_SYMBOL_GPL(hidinput_report_event);
static int hidinput_find_field(struct hid_device *hid, unsigned int type,
unsigned int code, struct hid_field **field)
{
struct hid_report *report;
int i, j;
list_for_each_entry(report, &hid->report_enum[HID_OUTPUT_REPORT].report_list, list) { for (i = 0; i < report->maxfield; i++) { *field = report->field[i]; for (j = 0; j < (*field)->maxusage; j++) if ((*field)->usage[j].type == type && (*field)->usage[j].code == code)
return j;
}
}
return -1;
}
struct hid_field *hidinput_get_led_field(struct hid_device *hid)
{
struct hid_report *report;
struct hid_field *field;
int i, j;
list_for_each_entry(report,
&hid->report_enum[HID_OUTPUT_REPORT].report_list,
list) {
for (i = 0; i < report->maxfield; i++) {
field = report->field[i];
for (j = 0; j < field->maxusage; j++)
if (field->usage[j].type == EV_LED)
return field;
}
}
return NULL;
}
EXPORT_SYMBOL_GPL(hidinput_get_led_field);
unsigned int hidinput_count_leds(struct hid_device *hid)
{
struct hid_report *report;
struct hid_field *field;
int i, j;
unsigned int count = 0;
list_for_each_entry(report,
&hid->report_enum[HID_OUTPUT_REPORT].report_list,
list) {
for (i = 0; i < report->maxfield; i++) {
field = report->field[i];
for (j = 0; j < field->maxusage; j++)
if (field->usage[j].type == EV_LED &&
field->value[j])
count += 1;
}
}
return count;
}
EXPORT_SYMBOL_GPL(hidinput_count_leds);
static void hidinput_led_worker(struct work_struct *work)
{
struct hid_device *hid = container_of(work, struct hid_device,
led_work);
struct hid_field *field;
struct hid_report *report;
int ret;
u32 len;
__u8 *buf;
field = hidinput_get_led_field(hid);
if (!field)
return;
/*
* field->report is accessed unlocked regarding HID core. So there might
* be another incoming SET-LED request from user-space, which changes
* the LED state while we assemble our outgoing buffer. However, this
* doesn't matter as hid_output_report() correctly converts it into a
* boolean value no matter what information is currently set on the LED
* field (even garbage). So the remote device will always get a valid
* request.
* And in case we send a wrong value, a next led worker is spawned
* for every SET-LED request so the following worker will send the
* correct value, guaranteed!
*/
report = field->report;
/* use custom SET_REPORT request if possible (asynchronous) */
if (hid->ll_driver->request)
return hid->ll_driver->request(hid, report, HID_REQ_SET_REPORT);
/* fall back to generic raw-output-report */
len = hid_report_len(report);
buf = hid_alloc_report_buf(report, GFP_KERNEL);
if (!buf)
return;
hid_output_report(report, buf);
/* synchronous output report */
ret = hid_hw_output_report(hid, buf, len);
if (ret == -ENOSYS)
hid_hw_raw_request(hid, report->id, buf, len, HID_OUTPUT_REPORT,
HID_REQ_SET_REPORT);
kfree(buf);
}
static int hidinput_input_event(struct input_dev *dev, unsigned int type,
unsigned int code, int value)
{
struct hid_device *hid = input_get_drvdata(dev);
struct hid_field *field;
int offset;
if (type == EV_FF)
return input_ff_event(dev, type, code, value); if (type != EV_LED)
return -1;
if ((offset = hidinput_find_field(hid, type, code, &field)) == -1) { hid_warn(dev, "event field not found\n");
return -1;
}
hid_set_field(field, offset, value); schedule_work(&hid->led_work);
return 0;
}
static int hidinput_open(struct input_dev *dev)
{
struct hid_device *hid = input_get_drvdata(dev);
return hid_hw_open(hid);
}
static void hidinput_close(struct input_dev *dev)
{
struct hid_device *hid = input_get_drvdata(dev);
hid_hw_close(hid);
}
static bool __hidinput_change_resolution_multipliers(struct hid_device *hid,
struct hid_report *report, bool use_logical_max)
{
struct hid_usage *usage;
bool update_needed = false;
bool get_report_completed = false;
int i, j;
if (report->maxfield == 0)
return false;
for (i = 0; i < report->maxfield; i++) {
__s32 value = use_logical_max ?
report->field[i]->logical_maximum :
report->field[i]->logical_minimum;
/* There is no good reason for a Resolution
* Multiplier to have a count other than 1.
* Ignore that case.
*/
if (report->field[i]->report_count != 1)
continue;
for (j = 0; j < report->field[i]->maxusage; j++) {
usage = &report->field[i]->usage[j];
if (usage->hid != HID_GD_RESOLUTION_MULTIPLIER)
continue;
/*
* If we have more than one feature within this
* report we need to fill in the bits from the
* others before we can overwrite the ones for the
* Resolution Multiplier.
*
* But if we're not allowed to read from the device,
* we just bail. Such a device should not exist
* anyway.
*/
if (!get_report_completed && report->maxfield > 1) {
if (hid->quirks & HID_QUIRK_NO_INIT_REPORTS)
return update_needed;
hid_hw_request(hid, report, HID_REQ_GET_REPORT);
hid_hw_wait(hid);
get_report_completed = true;
}
report->field[i]->value[j] = value;
update_needed = true;
}
}
return update_needed;
}
static void hidinput_change_resolution_multipliers(struct hid_device *hid)
{
struct hid_report_enum *rep_enum;
struct hid_report *rep;
int ret;
rep_enum = &hid->report_enum[HID_FEATURE_REPORT];
list_for_each_entry(rep, &rep_enum->report_list, list) {
bool update_needed = __hidinput_change_resolution_multipliers(hid,
rep, true);
if (update_needed) {
ret = __hid_request(hid, rep, HID_REQ_SET_REPORT);
if (ret) {
__hidinput_change_resolution_multipliers(hid,
rep, false);
return;
}
}
}
/* refresh our structs */
hid_setup_resolution_multiplier(hid);
}
static void report_features(struct hid_device *hid)
{
struct hid_driver *drv = hid->driver;
struct hid_report_enum *rep_enum;
struct hid_report *rep;
struct hid_usage *usage;
int i, j;
rep_enum = &hid->report_enum[HID_FEATURE_REPORT];
list_for_each_entry(rep, &rep_enum->report_list, list)
for (i = 0; i < rep->maxfield; i++) {
/* Ignore if report count is out of bounds. */
if (rep->field[i]->report_count < 1)
continue;
for (j = 0; j < rep->field[i]->maxusage; j++) {
usage = &rep->field[i]->usage[j];
/* Verify if Battery Strength feature is available */
if (usage->hid == HID_DC_BATTERYSTRENGTH)
hidinput_setup_battery(hid, HID_FEATURE_REPORT,
rep->field[i], false);
if (drv->feature_mapping)
drv->feature_mapping(hid, rep->field[i], usage);
}
}
}
static struct hid_input *hidinput_allocate(struct hid_device *hid,
unsigned int application)
{
struct hid_input *hidinput = kzalloc(sizeof(*hidinput), GFP_KERNEL);
struct input_dev *input_dev = input_allocate_device();
const char *suffix = NULL;
size_t suffix_len, name_len;
if (!hidinput || !input_dev)
goto fail;
if ((hid->quirks & HID_QUIRK_INPUT_PER_APP) &&
hid->maxapplication > 1) {
switch (application) {
case HID_GD_KEYBOARD:
suffix = "Keyboard";
break;
case HID_GD_KEYPAD:
suffix = "Keypad";
break;
case HID_GD_MOUSE:
suffix = "Mouse";
break;
case HID_DG_PEN:
/*
* yes, there is an issue here:
* DG_PEN -> "Stylus"
* DG_STYLUS -> "Pen"
* But changing this now means users with config snippets
* will have to change it and the test suite will not be happy.
*/
suffix = "Stylus";
break;
case HID_DG_STYLUS:
suffix = "Pen";
break;
case HID_DG_TOUCHSCREEN:
suffix = "Touchscreen";
break;
case HID_DG_TOUCHPAD:
suffix = "Touchpad";
break;
case HID_GD_SYSTEM_CONTROL:
suffix = "System Control";
break;
case HID_CP_CONSUMER_CONTROL:
suffix = "Consumer Control";
break;
case HID_GD_WIRELESS_RADIO_CTLS:
suffix = "Wireless Radio Control";
break;
case HID_GD_SYSTEM_MULTIAXIS:
suffix = "System Multi Axis";
break;
default:
break;
}
}
if (suffix) {
name_len = strlen(hid->name);
suffix_len = strlen(suffix);
if ((name_len < suffix_len) ||
strcmp(hid->name + name_len - suffix_len, suffix)) {
hidinput->name = kasprintf(GFP_KERNEL, "%s %s",
hid->name, suffix);
if (!hidinput->name)
goto fail;
}
}
input_set_drvdata(input_dev, hid);
input_dev->event = hidinput_input_event;
input_dev->open = hidinput_open;
input_dev->close = hidinput_close;
input_dev->setkeycode = hidinput_setkeycode;
input_dev->getkeycode = hidinput_getkeycode;
input_dev->name = hidinput->name ? hidinput->name : hid->name;
input_dev->phys = hid->phys;
input_dev->uniq = hid->uniq;
input_dev->id.bustype = hid->bus;
input_dev->id.vendor = hid->vendor;
input_dev->id.product = hid->product;
input_dev->id.version = hid->version;
input_dev->dev.parent = &hid->dev;
hidinput->input = input_dev;
hidinput->application = application;
list_add_tail(&hidinput->list, &hid->inputs);
INIT_LIST_HEAD(&hidinput->reports);
return hidinput;
fail:
kfree(hidinput);
input_free_device(input_dev);
hid_err(hid, "Out of memory during hid input probe\n");
return NULL;
}
static bool hidinput_has_been_populated(struct hid_input *hidinput)
{
int i;
unsigned long r = 0;
for (i = 0; i < BITS_TO_LONGS(EV_CNT); i++)
r |= hidinput->input->evbit[i];
for (i = 0; i < BITS_TO_LONGS(KEY_CNT); i++)
r |= hidinput->input->keybit[i];
for (i = 0; i < BITS_TO_LONGS(REL_CNT); i++)
r |= hidinput->input->relbit[i];
for (i = 0; i < BITS_TO_LONGS(ABS_CNT); i++)
r |= hidinput->input->absbit[i];
for (i = 0; i < BITS_TO_LONGS(MSC_CNT); i++)
r |= hidinput->input->mscbit[i];
for (i = 0; i < BITS_TO_LONGS(LED_CNT); i++)
r |= hidinput->input->ledbit[i];
for (i = 0; i < BITS_TO_LONGS(SND_CNT); i++)
r |= hidinput->input->sndbit[i];
for (i = 0; i < BITS_TO_LONGS(FF_CNT); i++)
r |= hidinput->input->ffbit[i];
for (i = 0; i < BITS_TO_LONGS(SW_CNT); i++)
r |= hidinput->input->swbit[i];
return !!r;
}
static void hidinput_cleanup_hidinput(struct hid_device *hid,
struct hid_input *hidinput)
{
struct hid_report *report;
int i, k;
list_del(&hidinput->list);
input_free_device(hidinput->input);
kfree(hidinput->name);
for (k = HID_INPUT_REPORT; k <= HID_OUTPUT_REPORT; k++) {
if (k == HID_OUTPUT_REPORT &&
hid->quirks & HID_QUIRK_SKIP_OUTPUT_REPORTS)
continue;
list_for_each_entry(report, &hid->report_enum[k].report_list,
list) {
for (i = 0; i < report->maxfield; i++)
if (report->field[i]->hidinput == hidinput)
report->field[i]->hidinput = NULL;
}
}
kfree(hidinput);
}
static struct hid_input *hidinput_match(struct hid_report *report)
{
struct hid_device *hid = report->device;
struct hid_input *hidinput;
list_for_each_entry(hidinput, &hid->inputs, list) {
if (hidinput->report &&
hidinput->report->id == report->id)
return hidinput;
}
return NULL;
}
static struct hid_input *hidinput_match_application(struct hid_report *report)
{
struct hid_device *hid = report->device;
struct hid_input *hidinput;
list_for_each_entry(hidinput, &hid->inputs, list) {
if (hidinput->application == report->application)
return hidinput;
/*
* Keep SystemControl and ConsumerControl applications together
* with the main keyboard, if present.
*/
if ((report->application == HID_GD_SYSTEM_CONTROL ||
report->application == HID_CP_CONSUMER_CONTROL) &&
hidinput->application == HID_GD_KEYBOARD) {
return hidinput;
}
}
return NULL;
}
static inline void hidinput_configure_usages(struct hid_input *hidinput,
struct hid_report *report)
{
int i, j, k;
int first_field_index = 0;
int slot_collection_index = -1;
int prev_collection_index = -1;
unsigned int slot_idx = 0;
struct hid_field *field;
/*
* First tag all the fields that are part of a slot,
* a slot needs to have one Contact ID in the collection
*/
for (i = 0; i < report->maxfield; i++) {
field = report->field[i];
/* ignore fields without usage */
if (field->maxusage < 1)
continue;
/*
* janitoring when collection_index changes
*/
if (prev_collection_index != field->usage->collection_index) {
prev_collection_index = field->usage->collection_index;
first_field_index = i;
}
/*
* if we already found a Contact ID in the collection,
* tag and continue to the next.
*/
if (slot_collection_index == field->usage->collection_index) {
field->slot_idx = slot_idx;
continue;
}
/* check if the current field has Contact ID */
for (j = 0; j < field->maxusage; j++) {
if (field->usage[j].hid == HID_DG_CONTACTID) {
slot_collection_index = field->usage->collection_index;
slot_idx++;
/*
* mark all previous fields and this one in the
* current collection to be slotted.
*/
for (k = first_field_index; k <= i; k++)
report->field[k]->slot_idx = slot_idx;
break;
}
}
}
for (i = 0; i < report->maxfield; i++)
for (j = 0; j < report->field[i]->maxusage; j++)
hidinput_configure_usage(hidinput, report->field[i],
report->field[i]->usage + j,
j);
}
/*
* Register the input device; print a message.
* Configure the input layer interface
* Read all reports and initialize the absolute field values.
*/
int hidinput_connect(struct hid_device *hid, unsigned int force)
{
struct hid_driver *drv = hid->driver;
struct hid_report *report;
struct hid_input *next, *hidinput = NULL;
unsigned int application;
int i, k;
INIT_LIST_HEAD(&hid->inputs);
INIT_WORK(&hid->led_work, hidinput_led_worker);
hid->status &= ~HID_STAT_DUP_DETECTED;
if (!force) {
for (i = 0; i < hid->maxcollection; i++) {
struct hid_collection *col = &hid->collection[i];
if (col->type == HID_COLLECTION_APPLICATION ||
col->type == HID_COLLECTION_PHYSICAL)
if (IS_INPUT_APPLICATION(col->usage))
break;
}
if (i == hid->maxcollection)
return -1;
}
report_features(hid);
for (k = HID_INPUT_REPORT; k <= HID_OUTPUT_REPORT; k++) {
if (k == HID_OUTPUT_REPORT &&
hid->quirks & HID_QUIRK_SKIP_OUTPUT_REPORTS)
continue;
list_for_each_entry(report, &hid->report_enum[k].report_list, list) {
if (!report->maxfield)
continue;
application = report->application;
/*
* Find the previous hidinput report attached
* to this report id.
*/
if (hid->quirks & HID_QUIRK_MULTI_INPUT)
hidinput = hidinput_match(report);
else if (hid->maxapplication > 1 &&
(hid->quirks & HID_QUIRK_INPUT_PER_APP))
hidinput = hidinput_match_application(report);
if (!hidinput) {
hidinput = hidinput_allocate(hid, application);
if (!hidinput)
goto out_unwind;
}
hidinput_configure_usages(hidinput, report);
if (hid->quirks & HID_QUIRK_MULTI_INPUT)
hidinput->report = report;
list_add_tail(&report->hidinput_list,
&hidinput->reports);
}
}
hidinput_change_resolution_multipliers(hid);
list_for_each_entry_safe(hidinput, next, &hid->inputs, list) {
if (drv->input_configured &&
drv->input_configured(hid, hidinput))
goto out_unwind;
if (!hidinput_has_been_populated(hidinput)) {
/* no need to register an input device not populated */
hidinput_cleanup_hidinput(hid, hidinput);
continue;
}
if (input_register_device(hidinput->input))
goto out_unwind;
hidinput->registered = true;
}
if (list_empty(&hid->inputs)) {
hid_err(hid, "No inputs registered, leaving\n");
goto out_unwind;
}
if (hid->status & HID_STAT_DUP_DETECTED)
hid_dbg(hid,
"Some usages could not be mapped, please use HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE if this is legitimate.\n");
return 0;
out_unwind:
/* unwind the ones we already registered */
hidinput_disconnect(hid);
return -1;
}
EXPORT_SYMBOL_GPL(hidinput_connect);
void hidinput_disconnect(struct hid_device *hid)
{
struct hid_input *hidinput, *next;
hidinput_cleanup_battery(hid); list_for_each_entry_safe(hidinput, next, &hid->inputs, list) { list_del(&hidinput->list);
if (hidinput->registered)
input_unregister_device(hidinput->input);
else
input_free_device(hidinput->input); kfree(hidinput->name);
kfree(hidinput);
}
/* led_work is spawned by input_dev callbacks, but doesn't access the
* parent input_dev at all. Once all input devices are removed, we
* know that led_work will never get restarted, so we can cancel it
* synchronously and are safe. */
cancel_work_sync(&hid->led_work);
}
EXPORT_SYMBOL_GPL(hidinput_disconnect);
#ifdef CONFIG_HID_KUNIT_TEST
#include "hid-input-test.c"
#endif
// SPDX-License-Identifier: GPL-2.0
#include <linux/spinlock.h>
#include <linux/task_work.h>
#include <linux/resume_user_mode.h>
static struct callback_head work_exited; /* all we need is ->next == NULL */
/**
* task_work_add - ask the @task to execute @work->func()
* @task: the task which should run the callback
* @work: the callback to run
* @notify: how to notify the targeted task
*
* Queue @work for task_work_run() below and notify the @task if @notify
* is @TWA_RESUME, @TWA_SIGNAL, or @TWA_SIGNAL_NO_IPI.
*
* @TWA_SIGNAL works like signals, in that the it will interrupt the targeted
* task and run the task_work, regardless of whether the task is currently
* running in the kernel or userspace.
* @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a
* reschedule IPI to force the targeted task to reschedule and run task_work.
* This can be advantageous if there's no strict requirement that the
* task_work be run as soon as possible, just whenever the task enters the
* kernel anyway.
* @TWA_RESUME work is run only when the task exits the kernel and returns to
* user mode, or before entering guest mode.
*
* Fails if the @task is exiting/exited and thus it can't process this @work.
* Otherwise @work->func() will be called when the @task goes through one of
* the aforementioned transitions, or exits.
*
* If the targeted task is exiting, then an error is returned and the work item
* is not queued. It's up to the caller to arrange for an alternative mechanism
* in that case.
*
* Note: there is no ordering guarantee on works queued here. The task_work
* list is LIFO.
*
* RETURNS:
* 0 if succeeds or -ESRCH.
*/
int task_work_add(struct task_struct *task, struct callback_head *work,
enum task_work_notify_mode notify)
{
struct callback_head *head;
/* record the work call stack in order to print it in KASAN reports */
kasan_record_aux_stack(work);
head = READ_ONCE(task->task_works);
do {
if (unlikely(head == &work_exited))
return -ESRCH;
work->next = head;
} while (!try_cmpxchg(&task->task_works, &head, work));
switch (notify) {
case TWA_NONE:
break;
case TWA_RESUME:
set_notify_resume(task);
break;
case TWA_SIGNAL:
set_notify_signal(task);
break;
case TWA_SIGNAL_NO_IPI:
__set_notify_signal(task);
break;
default:
WARN_ON_ONCE(1);
break;
}
return 0;
}
/**
* task_work_cancel_match - cancel a pending work added by task_work_add()
* @task: the task which should execute the work
* @match: match function to call
* @data: data to be passed in to match function
*
* RETURNS:
* The found work or NULL if not found.
*/
struct callback_head *
task_work_cancel_match(struct task_struct *task,
bool (*match)(struct callback_head *, void *data),
void *data)
{
struct callback_head **pprev = &task->task_works;
struct callback_head *work;
unsigned long flags;
if (likely(!task_work_pending(task)))
return NULL;
/*
* If cmpxchg() fails we continue without updating pprev.
* Either we raced with task_work_add() which added the
* new entry before this work, we will find it again. Or
* we raced with task_work_run(), *pprev == NULL/exited.
*/
raw_spin_lock_irqsave(&task->pi_lock, flags);
work = READ_ONCE(*pprev);
while (work) {
if (!match(work, data)) {
pprev = &work->next;
work = READ_ONCE(*pprev);
} else if (try_cmpxchg(pprev, &work, work->next))
break;
}
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
return work;
}
static bool task_work_func_match(struct callback_head *cb, void *data)
{
return cb->func == data;
}
/**
* task_work_cancel - cancel a pending work added by task_work_add()
* @task: the task which should execute the work
* @func: identifies the work to remove
*
* Find the last queued pending work with ->func == @func and remove
* it from queue.
*
* RETURNS:
* The found work or NULL if not found.
*/
struct callback_head *
task_work_cancel(struct task_struct *task, task_work_func_t func)
{
return task_work_cancel_match(task, task_work_func_match, func);
}
/**
* task_work_run - execute the works added by task_work_add()
*
* Flush the pending works. Should be used by the core kernel code.
* Called before the task returns to the user-mode or stops, or when
* it exits. In the latter case task_work_add() can no longer add the
* new work after task_work_run() returns.
*/
void task_work_run(void)
{
struct task_struct *task = current;
struct callback_head *work, *head, *next;
for (;;) {
/*
* work->func() can do task_work_add(), do not set
* work_exited unless the list is empty.
*/
work = READ_ONCE(task->task_works);
do {
head = NULL;
if (!work) {
if (task->flags & PF_EXITING) head = &work_exited;
else
break;
}
} while (!try_cmpxchg(&task->task_works, &work, head)); if (!work)
break;
/*
* Synchronize with task_work_cancel(). It can not remove
* the first entry == work, cmpxchg(task_works) must fail.
* But it can remove another entry from the ->next list.
*/
raw_spin_lock_irq(&task->pi_lock);
raw_spin_unlock_irq(&task->pi_lock);
do {
next = work->next;
work->func(work);
work = next;
cond_resched();
} while (work);
}
}
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* ALSA sequencer Ports
* Copyright (c) 1998 by Frank van de Pol <fvdpol@coil.demon.nl>
* Jaroslav Kysela <perex@perex.cz>
*/
#include <sound/core.h>
#include <linux/slab.h>
#include <linux/module.h>
#include "seq_system.h"
#include "seq_ports.h"
#include "seq_clientmgr.h"
/*
registration of client ports
*/
/*
NOTE: the current implementation of the port structure as a linked list is
not optimal for clients that have many ports. For sending messages to all
subscribers of a port we first need to find the address of the port
structure, which means we have to traverse the list. A direct access table
(array) would be better, but big preallocated arrays waste memory.
Possible actions:
1) leave it this way, a client does normaly does not have more than a few
ports
2) replace the linked list of ports by a array of pointers which is
dynamicly kmalloced. When a port is added or deleted we can simply allocate
a new array, copy the corresponding pointers, and delete the old one. We
then only need a pointer to this array, and an integer that tells us how
much elements are in array.
*/
/* return pointer to port structure - port is locked if found */
struct snd_seq_client_port *snd_seq_port_use_ptr(struct snd_seq_client *client,
int num)
{
struct snd_seq_client_port *port;
if (client == NULL)
return NULL;
guard(read_lock)(&client->ports_lock); list_for_each_entry(port, &client->ports_list_head, list) { if (port->addr.port == num) { if (port->closing)
break; /* deleting now */
snd_use_lock_use(&port->use_lock);
return port;
}
}
return NULL; /* not found */
}
/* search for the next port - port is locked if found */
struct snd_seq_client_port *snd_seq_port_query_nearest(struct snd_seq_client *client,
struct snd_seq_port_info *pinfo)
{
int num;
struct snd_seq_client_port *port, *found;
bool check_inactive = (pinfo->capability & SNDRV_SEQ_PORT_CAP_INACTIVE);
num = pinfo->addr.port;
found = NULL;
guard(read_lock)(&client->ports_lock);
list_for_each_entry(port, &client->ports_list_head, list) {
if ((port->capability & SNDRV_SEQ_PORT_CAP_INACTIVE) &&
!check_inactive)
continue; /* skip inactive ports */
if (port->addr.port < num)
continue;
if (port->addr.port == num) {
found = port;
break;
}
if (found == NULL || port->addr.port < found->addr.port)
found = port;
}
if (found) {
if (found->closing)
found = NULL;
else
snd_use_lock_use(&found->use_lock);
}
return found;
}
/* initialize snd_seq_port_subs_info */
static void port_subs_info_init(struct snd_seq_port_subs_info *grp)
{
INIT_LIST_HEAD(&grp->list_head);
grp->count = 0;
grp->exclusive = 0;
rwlock_init(&grp->list_lock);
init_rwsem(&grp->list_mutex);
grp->open = NULL;
grp->close = NULL;
}
/* create a port, port number or a negative error code is returned
* the caller needs to unref the port via snd_seq_port_unlock() appropriately
*/
int snd_seq_create_port(struct snd_seq_client *client, int port,
struct snd_seq_client_port **port_ret)
{
struct snd_seq_client_port *new_port, *p;
int num;
*port_ret = NULL;
/* sanity check */
if (snd_BUG_ON(!client))
return -EINVAL;
if (client->num_ports >= SNDRV_SEQ_MAX_PORTS) {
pr_warn("ALSA: seq: too many ports for client %d\n", client->number);
return -EINVAL;
}
/* create a new port */
new_port = kzalloc(sizeof(*new_port), GFP_KERNEL);
if (!new_port)
return -ENOMEM; /* failure, out of memory */
/* init port data */
new_port->addr.client = client->number;
new_port->addr.port = -1;
new_port->owner = THIS_MODULE;
snd_use_lock_init(&new_port->use_lock);
port_subs_info_init(&new_port->c_src);
port_subs_info_init(&new_port->c_dest);
snd_use_lock_use(&new_port->use_lock);
num = max(port, 0);
guard(mutex)(&client->ports_mutex);
guard(write_lock_irq)(&client->ports_lock);
list_for_each_entry(p, &client->ports_list_head, list) {
if (p->addr.port == port) {
kfree(new_port);
return -EBUSY;
}
if (p->addr.port > num)
break;
if (port < 0) /* auto-probe mode */
num = p->addr.port + 1;
}
/* insert the new port */
list_add_tail(&new_port->list, &p->list);
client->num_ports++;
new_port->addr.port = num; /* store the port number in the port */
sprintf(new_port->name, "port-%d", num);
*port_ret = new_port;
return num;
}
/* */
static int subscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_port_subs_info *grp,
struct snd_seq_port_subscribe *info, int send_ack);
static int unsubscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_port_subs_info *grp,
struct snd_seq_port_subscribe *info, int send_ack);
static struct snd_seq_client_port *get_client_port(struct snd_seq_addr *addr,
struct snd_seq_client **cp)
{
struct snd_seq_client_port *p;
*cp = snd_seq_client_use_ptr(addr->client);
if (*cp) {
p = snd_seq_port_use_ptr(*cp, addr->port);
if (! p) {
snd_seq_client_unlock(*cp);
*cp = NULL;
}
return p;
}
return NULL;
}
static void delete_and_unsubscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_subscribers *subs,
bool is_src, bool ack);
static inline struct snd_seq_subscribers *
get_subscriber(struct list_head *p, bool is_src)
{
if (is_src)
return list_entry(p, struct snd_seq_subscribers, src_list);
else
return list_entry(p, struct snd_seq_subscribers, dest_list);
}
/*
* remove all subscribers on the list
* this is called from port_delete, for each src and dest list.
*/
static void clear_subscriber_list(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_port_subs_info *grp,
int is_src)
{
struct list_head *p, *n;
list_for_each_safe(p, n, &grp->list_head) {
struct snd_seq_subscribers *subs;
struct snd_seq_client *c;
struct snd_seq_client_port *aport;
subs = get_subscriber(p, is_src);
if (is_src)
aport = get_client_port(&subs->info.dest, &c);
else
aport = get_client_port(&subs->info.sender, &c);
delete_and_unsubscribe_port(client, port, subs, is_src, false); if (!aport) {
/* looks like the connected port is being deleted.
* we decrease the counter, and when both ports are deleted
* remove the subscriber info
*/
if (atomic_dec_and_test(&subs->ref_count)) kfree(subs);
continue;
}
/* ok we got the connected port */
delete_and_unsubscribe_port(c, aport, subs, !is_src, true); kfree(subs);
snd_seq_port_unlock(aport);
snd_seq_client_unlock(c);
}
}
/* delete port data */
static int port_delete(struct snd_seq_client *client,
struct snd_seq_client_port *port)
{
/* set closing flag and wait for all port access are gone */
port->closing = 1;
snd_use_lock_sync(&port->use_lock);
/* clear subscribers info */
clear_subscriber_list(client, port, &port->c_src, true);
clear_subscriber_list(client, port, &port->c_dest, false);
if (port->private_free)
port->private_free(port->private_data); snd_BUG_ON(port->c_src.count != 0); snd_BUG_ON(port->c_dest.count != 0); kfree(port);
return 0;
}
/* delete a port with the given port id */
int snd_seq_delete_port(struct snd_seq_client *client, int port)
{
struct snd_seq_client_port *found = NULL, *p;
scoped_guard(mutex, &client->ports_mutex) { guard(write_lock_irq)(&client->ports_lock); list_for_each_entry(p, &client->ports_list_head, list) { if (p->addr.port == port) {
/* ok found. delete from the list at first */
list_del(&p->list);
client->num_ports--;
found = p;
break;
}
}
}
if (found)
return port_delete(client, found);
else
return -ENOENT;
}
/* delete the all ports belonging to the given client */
int snd_seq_delete_all_ports(struct snd_seq_client *client)
{
struct list_head deleted_list;
struct snd_seq_client_port *port, *tmp;
/* move the port list to deleted_list, and
* clear the port list in the client data.
*/
guard(mutex)(&client->ports_mutex);
scoped_guard(write_lock_irq, &client->ports_lock) {
if (!list_empty(&client->ports_list_head)) { list_add(&deleted_list, &client->ports_list_head); list_del_init(&client->ports_list_head);
} else {
INIT_LIST_HEAD(&deleted_list);
}
client->num_ports = 0;
}
/* remove each port in deleted_list */
list_for_each_entry_safe(port, tmp, &deleted_list, list) { list_del(&port->list);
snd_seq_system_client_ev_port_exit(port->addr.client, port->addr.port);
port_delete(client, port);
}
return 0;
}
/* set port info fields */
int snd_seq_set_port_info(struct snd_seq_client_port * port,
struct snd_seq_port_info * info)
{
if (snd_BUG_ON(!port || !info))
return -EINVAL;
/* set port name */
if (info->name[0])
strscpy(port->name, info->name, sizeof(port->name));
/* set capabilities */
port->capability = info->capability;
/* get port type */
port->type = info->type;
/* information about supported channels/voices */
port->midi_channels = info->midi_channels;
port->midi_voices = info->midi_voices;
port->synth_voices = info->synth_voices;
/* timestamping */
port->timestamping = (info->flags & SNDRV_SEQ_PORT_FLG_TIMESTAMP) ? 1 : 0;
port->time_real = (info->flags & SNDRV_SEQ_PORT_FLG_TIME_REAL) ? 1 : 0;
port->time_queue = info->time_queue;
/* UMP direction and group */
port->direction = info->direction;
port->ump_group = info->ump_group;
if (port->ump_group > SNDRV_UMP_MAX_GROUPS)
port->ump_group = 0;
/* fill default port direction */
if (!port->direction) {
if (info->capability & SNDRV_SEQ_PORT_CAP_READ)
port->direction |= SNDRV_SEQ_PORT_DIR_INPUT;
if (info->capability & SNDRV_SEQ_PORT_CAP_WRITE)
port->direction |= SNDRV_SEQ_PORT_DIR_OUTPUT;
}
return 0;
}
/* get port info fields */
int snd_seq_get_port_info(struct snd_seq_client_port * port,
struct snd_seq_port_info * info)
{
if (snd_BUG_ON(!port || !info))
return -EINVAL;
/* get port name */
strscpy(info->name, port->name, sizeof(info->name));
/* get capabilities */
info->capability = port->capability;
/* get port type */
info->type = port->type;
/* information about supported channels/voices */
info->midi_channels = port->midi_channels;
info->midi_voices = port->midi_voices;
info->synth_voices = port->synth_voices;
/* get subscriber counts */
info->read_use = port->c_src.count;
info->write_use = port->c_dest.count;
/* timestamping */
info->flags = 0;
if (port->timestamping) {
info->flags |= SNDRV_SEQ_PORT_FLG_TIMESTAMP;
if (port->time_real)
info->flags |= SNDRV_SEQ_PORT_FLG_TIME_REAL;
info->time_queue = port->time_queue;
}
/* UMP direction and group */
info->direction = port->direction;
info->ump_group = port->ump_group;
return 0;
}
/*
* call callback functions (if any):
* the callbacks are invoked only when the first (for connection) or
* the last subscription (for disconnection) is done. Second or later
* subscription results in increment of counter, but no callback is
* invoked.
* This feature is useful if these callbacks are associated with
* initialization or termination of devices (see seq_midi.c).
*/
static int subscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_port_subs_info *grp,
struct snd_seq_port_subscribe *info,
int send_ack)
{
int err = 0;
if (!try_module_get(port->owner))
return -EFAULT;
grp->count++;
if (grp->open && grp->count == 1) {
err = grp->open(port->private_data, info);
if (err < 0) {
module_put(port->owner);
grp->count--;
}
}
if (err >= 0 && send_ack && client->type == USER_CLIENT)
snd_seq_client_notify_subscription(port->addr.client, port->addr.port,
info, SNDRV_SEQ_EVENT_PORT_SUBSCRIBED);
return err;
}
static int unsubscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_port_subs_info *grp,
struct snd_seq_port_subscribe *info,
int send_ack)
{
int err = 0;
if (! grp->count)
return -EINVAL;
grp->count--;
if (grp->close && grp->count == 0)
err = grp->close(port->private_data, info);
if (send_ack && client->type == USER_CLIENT)
snd_seq_client_notify_subscription(port->addr.client, port->addr.port,
info, SNDRV_SEQ_EVENT_PORT_UNSUBSCRIBED);
module_put(port->owner);
return err;
}
/* check if both addresses are identical */
static inline int addr_match(struct snd_seq_addr *r, struct snd_seq_addr *s)
{
return (r->client == s->client) && (r->port == s->port);
}
/* check the two subscribe info match */
/* if flags is zero, checks only sender and destination addresses */
static int match_subs_info(struct snd_seq_port_subscribe *r,
struct snd_seq_port_subscribe *s)
{
if (addr_match(&r->sender, &s->sender) &&
addr_match(&r->dest, &s->dest)) {
if (r->flags && r->flags == s->flags)
return r->queue == s->queue;
else if (! r->flags)
return 1;
}
return 0;
}
static int check_and_subscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_subscribers *subs,
bool is_src, bool exclusive, bool ack)
{
struct snd_seq_port_subs_info *grp;
struct list_head *p;
struct snd_seq_subscribers *s;
int err;
grp = is_src ? &port->c_src : &port->c_dest;
guard(rwsem_write)(&grp->list_mutex);
if (exclusive) {
if (!list_empty(&grp->list_head))
return -EBUSY;
} else {
if (grp->exclusive)
return -EBUSY;
/* check whether already exists */
list_for_each(p, &grp->list_head) {
s = get_subscriber(p, is_src);
if (match_subs_info(&subs->info, &s->info))
return -EBUSY;
}
}
err = subscribe_port(client, port, grp, &subs->info, ack);
if (err < 0) {
grp->exclusive = 0;
return err;
}
/* add to list */
guard(write_lock_irq)(&grp->list_lock);
if (is_src)
list_add_tail(&subs->src_list, &grp->list_head);
else
list_add_tail(&subs->dest_list, &grp->list_head);
grp->exclusive = exclusive;
atomic_inc(&subs->ref_count);
return 0;
}
/* called with grp->list_mutex held */
static void __delete_and_unsubscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_subscribers *subs,
bool is_src, bool ack)
{
struct snd_seq_port_subs_info *grp;
struct list_head *list;
bool empty;
grp = is_src ? &port->c_src : &port->c_dest;
list = is_src ? &subs->src_list : &subs->dest_list;
scoped_guard(write_lock_irq, &grp->list_lock) {
empty = list_empty(list);
if (!empty)
list_del_init(list);
grp->exclusive = 0;
}
if (!empty)
unsubscribe_port(client, port, grp, &subs->info, ack);
}
static void delete_and_unsubscribe_port(struct snd_seq_client *client,
struct snd_seq_client_port *port,
struct snd_seq_subscribers *subs,
bool is_src, bool ack)
{
struct snd_seq_port_subs_info *grp;
grp = is_src ? &port->c_src : &port->c_dest; guard(rwsem_write)(&grp->list_mutex);
__delete_and_unsubscribe_port(client, port, subs, is_src, ack);
}
/* connect two ports */
int snd_seq_port_connect(struct snd_seq_client *connector,
struct snd_seq_client *src_client,
struct snd_seq_client_port *src_port,
struct snd_seq_client *dest_client,
struct snd_seq_client_port *dest_port,
struct snd_seq_port_subscribe *info)
{
struct snd_seq_subscribers *subs;
bool exclusive;
int err;
subs = kzalloc(sizeof(*subs), GFP_KERNEL);
if (!subs)
return -ENOMEM;
subs->info = *info;
atomic_set(&subs->ref_count, 0);
INIT_LIST_HEAD(&subs->src_list);
INIT_LIST_HEAD(&subs->dest_list);
exclusive = !!(info->flags & SNDRV_SEQ_PORT_SUBS_EXCLUSIVE);
err = check_and_subscribe_port(src_client, src_port, subs, true,
exclusive,
connector->number != src_client->number);
if (err < 0)
goto error;
err = check_and_subscribe_port(dest_client, dest_port, subs, false,
exclusive,
connector->number != dest_client->number);
if (err < 0)
goto error_dest;
return 0;
error_dest:
delete_and_unsubscribe_port(src_client, src_port, subs, true,
connector->number != src_client->number);
error:
kfree(subs);
return err;
}
/* remove the connection */
int snd_seq_port_disconnect(struct snd_seq_client *connector,
struct snd_seq_client *src_client,
struct snd_seq_client_port *src_port,
struct snd_seq_client *dest_client,
struct snd_seq_client_port *dest_port,
struct snd_seq_port_subscribe *info)
{
struct snd_seq_port_subs_info *dest = &dest_port->c_dest;
struct snd_seq_subscribers *subs;
int err = -ENOENT;
/* always start from deleting the dest port for avoiding concurrent
* deletions
*/
scoped_guard(rwsem_write, &dest->list_mutex) {
/* look for the connection */
list_for_each_entry(subs, &dest->list_head, dest_list) {
if (match_subs_info(info, &subs->info)) {
__delete_and_unsubscribe_port(dest_client, dest_port,
subs, false,
connector->number != dest_client->number);
err = 0;
break;
}
}
}
if (err < 0)
return err;
delete_and_unsubscribe_port(src_client, src_port, subs, true,
connector->number != src_client->number);
kfree(subs);
return 0;
}
/* get matched subscriber */
int snd_seq_port_get_subscription(struct snd_seq_port_subs_info *src_grp,
struct snd_seq_addr *dest_addr,
struct snd_seq_port_subscribe *subs)
{
struct snd_seq_subscribers *s;
int err = -ENOENT;
guard(rwsem_read)(&src_grp->list_mutex);
list_for_each_entry(s, &src_grp->list_head, src_list) {
if (addr_match(dest_addr, &s->info.dest)) {
*subs = s->info;
err = 0;
break;
}
}
return err;
}
/*
* Attach a device driver that wants to receive events from the
* sequencer. Returns the new port number on success.
* A driver that wants to receive the events converted to midi, will
* use snd_seq_midisynth_register_port().
*/
/* exported */
int snd_seq_event_port_attach(int client,
struct snd_seq_port_callback *pcbp,
int cap, int type, int midi_channels,
int midi_voices, char *portname)
{
struct snd_seq_port_info portinfo;
int ret;
/* Set up the port */
memset(&portinfo, 0, sizeof(portinfo));
portinfo.addr.client = client;
strscpy(portinfo.name, portname ? portname : "Unnamed port",
sizeof(portinfo.name));
portinfo.capability = cap;
portinfo.type = type;
portinfo.kernel = pcbp;
portinfo.midi_channels = midi_channels;
portinfo.midi_voices = midi_voices;
/* Create it */
ret = snd_seq_kernel_client_ctl(client,
SNDRV_SEQ_IOCTL_CREATE_PORT,
&portinfo);
if (ret >= 0)
ret = portinfo.addr.port;
return ret;
}
EXPORT_SYMBOL(snd_seq_event_port_attach);
/*
* Detach the driver from a port.
*/
/* exported */
int snd_seq_event_port_detach(int client, int port)
{
struct snd_seq_port_info portinfo;
int err;
memset(&portinfo, 0, sizeof(portinfo));
portinfo.addr.client = client;
portinfo.addr.port = port;
err = snd_seq_kernel_client_ctl(client,
SNDRV_SEQ_IOCTL_DELETE_PORT,
&portinfo);
return err;
}
EXPORT_SYMBOL(snd_seq_event_port_detach);
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/kdebug.h>
#include <linux/kprobes.h>
#include <linux/export.h>
#include <linux/notifier.h>
#include <linux/rcupdate.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#define CREATE_TRACE_POINTS
#include <trace/events/notifier.h>
/*
* Notifier list for kernel code which wants to be called
* at shutdown. This is used to stop any idling DMA operations
* and the like.
*/
BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
/*
* Notifier chain core routines. The exported routines below
* are layered on top of these, with appropriate locking added.
*/
static int notifier_chain_register(struct notifier_block **nl,
struct notifier_block *n,
bool unique_priority)
{
while ((*nl) != NULL) {
if (unlikely((*nl) == n)) {
WARN(1, "notifier callback %ps already registered",
n->notifier_call);
return -EEXIST;
}
if (n->priority > (*nl)->priority)
break;
if (n->priority == (*nl)->priority && unique_priority)
return -EBUSY;
nl = &((*nl)->next);
}
n->next = *nl;
rcu_assign_pointer(*nl, n);
trace_notifier_register((void *)n->notifier_call);
return 0;
}
static int notifier_chain_unregister(struct notifier_block **nl,
struct notifier_block *n)
{
while ((*nl) != NULL) { if ((*nl) == n) { rcu_assign_pointer(*nl, n->next); trace_notifier_unregister((void *)n->notifier_call);
return 0;
}
nl = &((*nl)->next);
}
return -ENOENT;
}
/**
* notifier_call_chain - Informs the registered notifiers about an event.
* @nl: Pointer to head of the blocking notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
* @nr_to_call: Number of notifier functions to be called. Don't care
* value of this parameter is -1.
* @nr_calls: Records the number of notifications sent. Don't care
* value of this field is NULL.
* Return: notifier_call_chain returns the value returned by the
* last notifier function called.
*/
static int notifier_call_chain(struct notifier_block **nl,
unsigned long val, void *v,
int nr_to_call, int *nr_calls)
{
int ret = NOTIFY_DONE;
struct notifier_block *nb, *next_nb;
nb = rcu_dereference_raw(*nl); while (nb && nr_to_call) { next_nb = rcu_dereference_raw(nb->next);
#ifdef CONFIG_DEBUG_NOTIFIERS
if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
WARN(1, "Invalid notifier called!");
nb = next_nb;
continue;
}
#endif
trace_notifier_run((void *)nb->notifier_call); ret = nb->notifier_call(nb, val, v);
if (nr_calls)
(*nr_calls)++; if (ret & NOTIFY_STOP_MASK)
break;
nb = next_nb;
nr_to_call--;
}
return ret;
}
NOKPROBE_SYMBOL(notifier_call_chain);
/**
* notifier_call_chain_robust - Inform the registered notifiers about an event
* and rollback on error.
* @nl: Pointer to head of the blocking notifier chain
* @val_up: Value passed unmodified to the notifier function
* @val_down: Value passed unmodified to the notifier function when recovering
* from an error on @val_up
* @v: Pointer passed unmodified to the notifier function
*
* NOTE: It is important the @nl chain doesn't change between the two
* invocations of notifier_call_chain() such that we visit the
* exact same notifier callbacks; this rules out any RCU usage.
*
* Return: the return value of the @val_up call.
*/
static int notifier_call_chain_robust(struct notifier_block **nl,
unsigned long val_up, unsigned long val_down,
void *v)
{
int ret, nr = 0;
ret = notifier_call_chain(nl, val_up, v, -1, &nr);
if (ret & NOTIFY_STOP_MASK)
notifier_call_chain(nl, val_down, v, nr-1, NULL);
return ret;
}
/*
* Atomic notifier chain routines. Registration and unregistration
* use a spinlock, and call_chain is synchronized by RCU (no locks).
*/
/**
* atomic_notifier_chain_register - Add notifier to an atomic notifier chain
* @nh: Pointer to head of the atomic notifier chain
* @n: New entry in notifier chain
*
* Adds a notifier to an atomic notifier chain.
*
* Returns 0 on success, %-EEXIST on error.
*/
int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
struct notifier_block *n)
{
unsigned long flags;
int ret;
spin_lock_irqsave(&nh->lock, flags);
ret = notifier_chain_register(&nh->head, n, false);
spin_unlock_irqrestore(&nh->lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
/**
* atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
* @nh: Pointer to head of the atomic notifier chain
* @n: New entry in notifier chain
*
* Adds a notifier to an atomic notifier chain if there is no other
* notifier registered using the same priority.
*
* Returns 0 on success, %-EEXIST or %-EBUSY on error.
*/
int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
struct notifier_block *n)
{
unsigned long flags;
int ret;
spin_lock_irqsave(&nh->lock, flags);
ret = notifier_chain_register(&nh->head, n, true);
spin_unlock_irqrestore(&nh->lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
/**
* atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
* @nh: Pointer to head of the atomic notifier chain
* @n: Entry to remove from notifier chain
*
* Removes a notifier from an atomic notifier chain.
*
* Returns zero on success or %-ENOENT on failure.
*/
int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
struct notifier_block *n)
{
unsigned long flags;
int ret;
spin_lock_irqsave(&nh->lock, flags);
ret = notifier_chain_unregister(&nh->head, n);
spin_unlock_irqrestore(&nh->lock, flags);
synchronize_rcu();
return ret;
}
EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
/**
* atomic_notifier_call_chain - Call functions in an atomic notifier chain
* @nh: Pointer to head of the atomic notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
*
* Calls each function in a notifier chain in turn. The functions
* run in an atomic context, so they must not block.
* This routine uses RCU to synchronize with changes to the chain.
*
* If the return value of the notifier can be and'ed
* with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
* will return immediately, with the return value of
* the notifier function which halted execution.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
unsigned long val, void *v)
{
int ret;
rcu_read_lock(); ret = notifier_call_chain(&nh->head, val, v, -1, NULL); rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
NOKPROBE_SYMBOL(atomic_notifier_call_chain);
/**
* atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
* @nh: Pointer to head of the atomic notifier chain
*
* Checks whether notifier chain is empty.
*
* Returns true is notifier chain is empty, false otherwise.
*/
bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
{
return !rcu_access_pointer(nh->head);
}
/*
* Blocking notifier chain routines. All access to the chain is
* synchronized by an rwsem.
*/
static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
struct notifier_block *n,
bool unique_priority)
{
int ret;
/*
* This code gets used during boot-up, when task switching is
* not yet working and interrupts must remain disabled. At
* such times we must not call down_write().
*/
if (unlikely(system_state == SYSTEM_BOOTING))
return notifier_chain_register(&nh->head, n, unique_priority);
down_write(&nh->rwsem);
ret = notifier_chain_register(&nh->head, n, unique_priority);
up_write(&nh->rwsem);
return ret;
}
/**
* blocking_notifier_chain_register - Add notifier to a blocking notifier chain
* @nh: Pointer to head of the blocking notifier chain
* @n: New entry in notifier chain
*
* Adds a notifier to a blocking notifier chain.
* Must be called in process context.
*
* Returns 0 on success, %-EEXIST on error.
*/
int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
struct notifier_block *n)
{
return __blocking_notifier_chain_register(nh, n, false);
}
EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
/**
* blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
* @nh: Pointer to head of the blocking notifier chain
* @n: New entry in notifier chain
*
* Adds a notifier to an blocking notifier chain if there is no other
* notifier registered using the same priority.
*
* Returns 0 on success, %-EEXIST or %-EBUSY on error.
*/
int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
struct notifier_block *n)
{
return __blocking_notifier_chain_register(nh, n, true);
}
EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
/**
* blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
* @nh: Pointer to head of the blocking notifier chain
* @n: Entry to remove from notifier chain
*
* Removes a notifier from a blocking notifier chain.
* Must be called from process context.
*
* Returns zero on success or %-ENOENT on failure.
*/
int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
struct notifier_block *n)
{
int ret;
/*
* This code gets used during boot-up, when task switching is
* not yet working and interrupts must remain disabled. At
* such times we must not call down_write().
*/
if (unlikely(system_state == SYSTEM_BOOTING)) return notifier_chain_unregister(&nh->head, n); down_write(&nh->rwsem);
ret = notifier_chain_unregister(&nh->head, n);
up_write(&nh->rwsem);
return ret;
}
EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
unsigned long val_up, unsigned long val_down, void *v)
{
int ret = NOTIFY_DONE;
/*
* We check the head outside the lock, but if this access is
* racy then it does not matter what the result of the test
* is, we re-check the list after having taken the lock anyway:
*/
if (rcu_access_pointer(nh->head)) {
down_read(&nh->rwsem);
ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
up_read(&nh->rwsem);
}
return ret;
}
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
/**
* blocking_notifier_call_chain - Call functions in a blocking notifier chain
* @nh: Pointer to head of the blocking notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
*
* Calls each function in a notifier chain in turn. The functions
* run in a process context, so they are allowed to block.
*
* If the return value of the notifier can be and'ed
* with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
* will return immediately, with the return value of
* the notifier function which halted execution.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
unsigned long val, void *v)
{
int ret = NOTIFY_DONE;
/*
* We check the head outside the lock, but if this access is
* racy then it does not matter what the result of the test
* is, we re-check the list after having taken the lock anyway:
*/
if (rcu_access_pointer(nh->head)) { down_read(&nh->rwsem);
ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
up_read(&nh->rwsem);
}
return ret;
}
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
/*
* Raw notifier chain routines. There is no protection;
* the caller must provide it. Use at your own risk!
*/
/**
* raw_notifier_chain_register - Add notifier to a raw notifier chain
* @nh: Pointer to head of the raw notifier chain
* @n: New entry in notifier chain
*
* Adds a notifier to a raw notifier chain.
* All locking must be provided by the caller.
*
* Returns 0 on success, %-EEXIST on error.
*/
int raw_notifier_chain_register(struct raw_notifier_head *nh,
struct notifier_block *n)
{
return notifier_chain_register(&nh->head, n, false);
}
EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
/**
* raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
* @nh: Pointer to head of the raw notifier chain
* @n: Entry to remove from notifier chain
*
* Removes a notifier from a raw notifier chain.
* All locking must be provided by the caller.
*
* Returns zero on success or %-ENOENT on failure.
*/
int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
struct notifier_block *n)
{
return notifier_chain_unregister(&nh->head, n);
}
EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
unsigned long val_up, unsigned long val_down, void *v)
{
return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
}
EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
/**
* raw_notifier_call_chain - Call functions in a raw notifier chain
* @nh: Pointer to head of the raw notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
*
* Calls each function in a notifier chain in turn. The functions
* run in an undefined context.
* All locking must be provided by the caller.
*
* If the return value of the notifier can be and'ed
* with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
* will return immediately, with the return value of
* the notifier function which halted execution.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
int raw_notifier_call_chain(struct raw_notifier_head *nh,
unsigned long val, void *v)
{
return notifier_call_chain(&nh->head, val, v, -1, NULL);
}
EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
/*
* SRCU notifier chain routines. Registration and unregistration
* use a mutex, and call_chain is synchronized by SRCU (no locks).
*/
/**
* srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
* @nh: Pointer to head of the SRCU notifier chain
* @n: New entry in notifier chain
*
* Adds a notifier to an SRCU notifier chain.
* Must be called in process context.
*
* Returns 0 on success, %-EEXIST on error.
*/
int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
struct notifier_block *n)
{
int ret;
/*
* This code gets used during boot-up, when task switching is
* not yet working and interrupts must remain disabled. At
* such times we must not call mutex_lock().
*/
if (unlikely(system_state == SYSTEM_BOOTING))
return notifier_chain_register(&nh->head, n, false);
mutex_lock(&nh->mutex);
ret = notifier_chain_register(&nh->head, n, false);
mutex_unlock(&nh->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
/**
* srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
* @nh: Pointer to head of the SRCU notifier chain
* @n: Entry to remove from notifier chain
*
* Removes a notifier from an SRCU notifier chain.
* Must be called from process context.
*
* Returns zero on success or %-ENOENT on failure.
*/
int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
struct notifier_block *n)
{
int ret;
/*
* This code gets used during boot-up, when task switching is
* not yet working and interrupts must remain disabled. At
* such times we must not call mutex_lock().
*/
if (unlikely(system_state == SYSTEM_BOOTING))
return notifier_chain_unregister(&nh->head, n);
mutex_lock(&nh->mutex);
ret = notifier_chain_unregister(&nh->head, n);
mutex_unlock(&nh->mutex);
synchronize_srcu(&nh->srcu);
return ret;
}
EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
/**
* srcu_notifier_call_chain - Call functions in an SRCU notifier chain
* @nh: Pointer to head of the SRCU notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
*
* Calls each function in a notifier chain in turn. The functions
* run in a process context, so they are allowed to block.
*
* If the return value of the notifier can be and'ed
* with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
* will return immediately, with the return value of
* the notifier function which halted execution.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
unsigned long val, void *v)
{
int ret;
int idx;
idx = srcu_read_lock(&nh->srcu);
ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
srcu_read_unlock(&nh->srcu, idx);
return ret;
}
EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
/**
* srcu_init_notifier_head - Initialize an SRCU notifier head
* @nh: Pointer to head of the srcu notifier chain
*
* Unlike other sorts of notifier heads, SRCU notifier heads require
* dynamic initialization. Be sure to call this routine before
* calling any of the other SRCU notifier routines for this head.
*
* If an SRCU notifier head is deallocated, it must first be cleaned
* up by calling srcu_cleanup_notifier_head(). Otherwise the head's
* per-cpu data (used by the SRCU mechanism) will leak.
*/
void srcu_init_notifier_head(struct srcu_notifier_head *nh)
{
mutex_init(&nh->mutex);
if (init_srcu_struct(&nh->srcu) < 0)
BUG();
nh->head = NULL;
}
EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
static ATOMIC_NOTIFIER_HEAD(die_chain);
int notrace notify_die(enum die_val val, const char *str,
struct pt_regs *regs, long err, int trap, int sig)
{
struct die_args args = {
.regs = regs,
.str = str,
.err = err,
.trapnr = trap,
.signr = sig,
};
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"notify_die called but RCU thinks we're quiescent");
return atomic_notifier_call_chain(&die_chain, val, &args);
}
NOKPROBE_SYMBOL(notify_die);
int register_die_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_register(&die_chain, nb);
}
EXPORT_SYMBOL_GPL(register_die_notifier);
int unregister_die_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_unregister(&die_chain, nb);
}
EXPORT_SYMBOL_GPL(unregister_die_notifier);
// SPDX-License-Identifier: GPL-2.0
/*
* udc.c - Core UDC Framework
*
* Copyright (C) 2010 Texas Instruments
* Author: Felipe Balbi <balbi@ti.com>
*/
#define pr_fmt(fmt) "UDC core: " fmt
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/list.h>
#include <linux/idr.h>
#include <linux/err.h>
#include <linux/dma-mapping.h>
#include <linux/sched/task_stack.h>
#include <linux/workqueue.h>
#include <linux/usb/ch9.h>
#include <linux/usb/gadget.h>
#include <linux/usb.h>
#include "trace.h"
static DEFINE_IDA(gadget_id_numbers);
static const struct bus_type gadget_bus_type;
/**
* struct usb_udc - describes one usb device controller
* @driver: the gadget driver pointer. For use by the class code
* @dev: the child device to the actual controller
* @gadget: the gadget. For use by the class code
* @list: for use by the udc class driver
* @vbus: for udcs who care about vbus status, this value is real vbus status;
* for udcs who do not care about vbus status, this value is always true
* @started: the UDC's started state. True if the UDC had started.
* @allow_connect: Indicates whether UDC is allowed to be pulled up.
* Set/cleared by gadget_(un)bind_driver() after gadget driver is bound or
* unbound.
* @vbus_work: work routine to handle VBUS status change notifications.
* @connect_lock: protects udc->started, gadget->connect,
* gadget->allow_connect and gadget->deactivate. The routines
* usb_gadget_connect_locked(), usb_gadget_disconnect_locked(),
* usb_udc_connect_control_locked(), usb_gadget_udc_start_locked() and
* usb_gadget_udc_stop_locked() are called with this lock held.
*
* This represents the internal data structure which is used by the UDC-class
* to hold information about udc driver and gadget together.
*/
struct usb_udc {
struct usb_gadget_driver *driver;
struct usb_gadget *gadget;
struct device dev;
struct list_head list;
bool vbus;
bool started;
bool allow_connect;
struct work_struct vbus_work;
struct mutex connect_lock;
};
static const struct class udc_class;
static LIST_HEAD(udc_list);
/* Protects udc_list, udc->driver, driver->is_bound, and related calls */
static DEFINE_MUTEX(udc_lock);
/* ------------------------------------------------------------------------- */
/**
* usb_ep_set_maxpacket_limit - set maximum packet size limit for endpoint
* @ep:the endpoint being configured
* @maxpacket_limit:value of maximum packet size limit
*
* This function should be used only in UDC drivers to initialize endpoint
* (usually in probe function).
*/
void usb_ep_set_maxpacket_limit(struct usb_ep *ep,
unsigned maxpacket_limit)
{
ep->maxpacket_limit = maxpacket_limit;
ep->maxpacket = maxpacket_limit;
trace_usb_ep_set_maxpacket_limit(ep, 0);
}
EXPORT_SYMBOL_GPL(usb_ep_set_maxpacket_limit);
/**
* usb_ep_enable - configure endpoint, making it usable
* @ep:the endpoint being configured. may not be the endpoint named "ep0".
* drivers discover endpoints through the ep_list of a usb_gadget.
*
* When configurations are set, or when interface settings change, the driver
* will enable or disable the relevant endpoints. while it is enabled, an
* endpoint may be used for i/o until the driver receives a disconnect() from
* the host or until the endpoint is disabled.
*
* the ep0 implementation (which calls this routine) must ensure that the
* hardware capabilities of each endpoint match the descriptor provided
* for it. for example, an endpoint named "ep2in-bulk" would be usable
* for interrupt transfers as well as bulk, but it likely couldn't be used
* for iso transfers or for endpoint 14. some endpoints are fully
* configurable, with more generic names like "ep-a". (remember that for
* USB, "in" means "towards the USB host".)
*
* This routine may be called in an atomic (interrupt) context.
*
* returns zero, or a negative error code.
*/
int usb_ep_enable(struct usb_ep *ep)
{
int ret = 0;
if (ep->enabled)
goto out;
/* UDC drivers can't handle endpoints with maxpacket size 0 */
if (usb_endpoint_maxp(ep->desc) == 0) {
/*
* We should log an error message here, but we can't call
* dev_err() because there's no way to find the gadget
* given only ep.
*/
ret = -EINVAL;
goto out;
}
ret = ep->ops->enable(ep, ep->desc);
if (ret)
goto out;
ep->enabled = true;
out:
trace_usb_ep_enable(ep, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_enable);
/**
* usb_ep_disable - endpoint is no longer usable
* @ep:the endpoint being unconfigured. may not be the endpoint named "ep0".
*
* no other task may be using this endpoint when this is called.
* any pending and uncompleted requests will complete with status
* indicating disconnect (-ESHUTDOWN) before this call returns.
* gadget drivers must call usb_ep_enable() again before queueing
* requests to the endpoint.
*
* This routine may be called in an atomic (interrupt) context.
*
* returns zero, or a negative error code.
*/
int usb_ep_disable(struct usb_ep *ep)
{
int ret = 0;
if (!ep->enabled)
goto out;
ret = ep->ops->disable(ep);
if (ret)
goto out;
ep->enabled = false;
out:
trace_usb_ep_disable(ep, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_disable);
/**
* usb_ep_alloc_request - allocate a request object to use with this endpoint
* @ep:the endpoint to be used with with the request
* @gfp_flags:GFP_* flags to use
*
* Request objects must be allocated with this call, since they normally
* need controller-specific setup and may even need endpoint-specific
* resources such as allocation of DMA descriptors.
* Requests may be submitted with usb_ep_queue(), and receive a single
* completion callback. Free requests with usb_ep_free_request(), when
* they are no longer needed.
*
* Returns the request, or null if one could not be allocated.
*/
struct usb_request *usb_ep_alloc_request(struct usb_ep *ep,
gfp_t gfp_flags)
{
struct usb_request *req = NULL;
req = ep->ops->alloc_request(ep, gfp_flags); trace_usb_ep_alloc_request(ep, req, req ? 0 : -ENOMEM); return req;
}
EXPORT_SYMBOL_GPL(usb_ep_alloc_request);
/**
* usb_ep_free_request - frees a request object
* @ep:the endpoint associated with the request
* @req:the request being freed
*
* Reverses the effect of usb_ep_alloc_request().
* Caller guarantees the request is not queued, and that it will
* no longer be requeued (or otherwise used).
*/
void usb_ep_free_request(struct usb_ep *ep,
struct usb_request *req)
{
trace_usb_ep_free_request(ep, req, 0); ep->ops->free_request(ep, req);
}
EXPORT_SYMBOL_GPL(usb_ep_free_request);
/**
* usb_ep_queue - queues (submits) an I/O request to an endpoint.
* @ep:the endpoint associated with the request
* @req:the request being submitted
* @gfp_flags: GFP_* flags to use in case the lower level driver couldn't
* pre-allocate all necessary memory with the request.
*
* This tells the device controller to perform the specified request through
* that endpoint (reading or writing a buffer). When the request completes,
* including being canceled by usb_ep_dequeue(), the request's completion
* routine is called to return the request to the driver. Any endpoint
* (except control endpoints like ep0) may have more than one transfer
* request queued; they complete in FIFO order. Once a gadget driver
* submits a request, that request may not be examined or modified until it
* is given back to that driver through the completion callback.
*
* Each request is turned into one or more packets. The controller driver
* never merges adjacent requests into the same packet. OUT transfers
* will sometimes use data that's already buffered in the hardware.
* Drivers can rely on the fact that the first byte of the request's buffer
* always corresponds to the first byte of some USB packet, for both
* IN and OUT transfers.
*
* Bulk endpoints can queue any amount of data; the transfer is packetized
* automatically. The last packet will be short if the request doesn't fill it
* out completely. Zero length packets (ZLPs) should be avoided in portable
* protocols since not all usb hardware can successfully handle zero length
* packets. (ZLPs may be explicitly written, and may be implicitly written if
* the request 'zero' flag is set.) Bulk endpoints may also be used
* for interrupt transfers; but the reverse is not true, and some endpoints
* won't support every interrupt transfer. (Such as 768 byte packets.)
*
* Interrupt-only endpoints are less functional than bulk endpoints, for
* example by not supporting queueing or not handling buffers that are
* larger than the endpoint's maxpacket size. They may also treat data
* toggle differently.
*
* Control endpoints ... after getting a setup() callback, the driver queues
* one response (even if it would be zero length). That enables the
* status ack, after transferring data as specified in the response. Setup
* functions may return negative error codes to generate protocol stalls.
* (Note that some USB device controllers disallow protocol stall responses
* in some cases.) When control responses are deferred (the response is
* written after the setup callback returns), then usb_ep_set_halt() may be
* used on ep0 to trigger protocol stalls. Depending on the controller,
* it may not be possible to trigger a status-stage protocol stall when the
* data stage is over, that is, from within the response's completion
* routine.
*
* For periodic endpoints, like interrupt or isochronous ones, the usb host
* arranges to poll once per interval, and the gadget driver usually will
* have queued some data to transfer at that time.
*
* Note that @req's ->complete() callback must never be called from
* within usb_ep_queue() as that can create deadlock situations.
*
* This routine may be called in interrupt context.
*
* Returns zero, or a negative error code. Endpoints that are not enabled
* report errors; errors will also be
* reported when the usb peripheral is disconnected.
*
* If and only if @req is successfully queued (the return value is zero),
* @req->complete() will be called exactly once, when the Gadget core and
* UDC are finished with the request. When the completion function is called,
* control of the request is returned to the device driver which submitted it.
* The completion handler may then immediately free or reuse @req.
*/
int usb_ep_queue(struct usb_ep *ep,
struct usb_request *req, gfp_t gfp_flags)
{
int ret = 0;
if (!ep->enabled && ep->address) { pr_debug("USB gadget: queue request to disabled ep 0x%x (%s)\n",
ep->address, ep->name);
ret = -ESHUTDOWN;
goto out;
}
ret = ep->ops->queue(ep, req, gfp_flags);
out:
trace_usb_ep_queue(ep, req, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_queue);
/**
* usb_ep_dequeue - dequeues (cancels, unlinks) an I/O request from an endpoint
* @ep:the endpoint associated with the request
* @req:the request being canceled
*
* If the request is still active on the endpoint, it is dequeued and
* eventually its completion routine is called (with status -ECONNRESET);
* else a negative error code is returned. This routine is asynchronous,
* that is, it may return before the completion routine runs.
*
* Note that some hardware can't clear out write fifos (to unlink the request
* at the head of the queue) except as part of disconnecting from usb. Such
* restrictions prevent drivers from supporting configuration changes,
* even to configuration zero (a "chapter 9" requirement).
*
* This routine may be called in interrupt context.
*/
int usb_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
{
int ret;
ret = ep->ops->dequeue(ep, req);
trace_usb_ep_dequeue(ep, req, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_dequeue);
/**
* usb_ep_set_halt - sets the endpoint halt feature.
* @ep: the non-isochronous endpoint being stalled
*
* Use this to stall an endpoint, perhaps as an error report.
* Except for control endpoints,
* the endpoint stays halted (will not stream any data) until the host
* clears this feature; drivers may need to empty the endpoint's request
* queue first, to make sure no inappropriate transfers happen.
*
* Note that while an endpoint CLEAR_FEATURE will be invisible to the
* gadget driver, a SET_INTERFACE will not be. To reset endpoints for the
* current altsetting, see usb_ep_clear_halt(). When switching altsettings,
* it's simplest to use usb_ep_enable() or usb_ep_disable() for the endpoints.
*
* This routine may be called in interrupt context.
*
* Returns zero, or a negative error code. On success, this call sets
* underlying hardware state that blocks data transfers.
* Attempts to halt IN endpoints will fail (returning -EAGAIN) if any
* transfer requests are still queued, or if the controller hardware
* (usually a FIFO) still holds bytes that the host hasn't collected.
*/
int usb_ep_set_halt(struct usb_ep *ep)
{
int ret;
ret = ep->ops->set_halt(ep, 1); trace_usb_ep_set_halt(ep, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_set_halt);
/**
* usb_ep_clear_halt - clears endpoint halt, and resets toggle
* @ep:the bulk or interrupt endpoint being reset
*
* Use this when responding to the standard usb "set interface" request,
* for endpoints that aren't reconfigured, after clearing any other state
* in the endpoint's i/o queue.
*
* This routine may be called in interrupt context.
*
* Returns zero, or a negative error code. On success, this call clears
* the underlying hardware state reflecting endpoint halt and data toggle.
* Note that some hardware can't support this request (like pxa2xx_udc),
* and accordingly can't correctly implement interface altsettings.
*/
int usb_ep_clear_halt(struct usb_ep *ep)
{
int ret;
ret = ep->ops->set_halt(ep, 0); trace_usb_ep_clear_halt(ep, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_clear_halt);
/**
* usb_ep_set_wedge - sets the halt feature and ignores clear requests
* @ep: the endpoint being wedged
*
* Use this to stall an endpoint and ignore CLEAR_FEATURE(HALT_ENDPOINT)
* requests. If the gadget driver clears the halt status, it will
* automatically unwedge the endpoint.
*
* This routine may be called in interrupt context.
*
* Returns zero on success, else negative errno.
*/
int usb_ep_set_wedge(struct usb_ep *ep)
{
int ret;
if (ep->ops->set_wedge) ret = ep->ops->set_wedge(ep);
else
ret = ep->ops->set_halt(ep, 1); trace_usb_ep_set_wedge(ep, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_set_wedge);
/**
* usb_ep_fifo_status - returns number of bytes in fifo, or error
* @ep: the endpoint whose fifo status is being checked.
*
* FIFO endpoints may have "unclaimed data" in them in certain cases,
* such as after aborted transfers. Hosts may not have collected all
* the IN data written by the gadget driver (and reported by a request
* completion). The gadget driver may not have collected all the data
* written OUT to it by the host. Drivers that need precise handling for
* fault reporting or recovery may need to use this call.
*
* This routine may be called in interrupt context.
*
* This returns the number of such bytes in the fifo, or a negative
* errno if the endpoint doesn't use a FIFO or doesn't support such
* precise handling.
*/
int usb_ep_fifo_status(struct usb_ep *ep)
{
int ret;
if (ep->ops->fifo_status)
ret = ep->ops->fifo_status(ep);
else
ret = -EOPNOTSUPP;
trace_usb_ep_fifo_status(ep, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_ep_fifo_status);
/**
* usb_ep_fifo_flush - flushes contents of a fifo
* @ep: the endpoint whose fifo is being flushed.
*
* This call may be used to flush the "unclaimed data" that may exist in
* an endpoint fifo after abnormal transaction terminations. The call
* must never be used except when endpoint is not being used for any
* protocol translation.
*
* This routine may be called in interrupt context.
*/
void usb_ep_fifo_flush(struct usb_ep *ep)
{
if (ep->ops->fifo_flush)
ep->ops->fifo_flush(ep);
trace_usb_ep_fifo_flush(ep, 0);
}
EXPORT_SYMBOL_GPL(usb_ep_fifo_flush);
/* ------------------------------------------------------------------------- */
/**
* usb_gadget_frame_number - returns the current frame number
* @gadget: controller that reports the frame number
*
* Returns the usb frame number, normally eleven bits from a SOF packet,
* or negative errno if this device doesn't support this capability.
*/
int usb_gadget_frame_number(struct usb_gadget *gadget)
{
int ret;
ret = gadget->ops->get_frame(gadget);
trace_usb_gadget_frame_number(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_frame_number);
/**
* usb_gadget_wakeup - tries to wake up the host connected to this gadget
* @gadget: controller used to wake up the host
*
* Returns zero on success, else negative error code if the hardware
* doesn't support such attempts, or its support has not been enabled
* by the usb host. Drivers must return device descriptors that report
* their ability to support this, or hosts won't enable it.
*
* This may also try to use SRP to wake the host and start enumeration,
* even if OTG isn't otherwise in use. OTG devices may also start
* remote wakeup even when hosts don't explicitly enable it.
*/
int usb_gadget_wakeup(struct usb_gadget *gadget)
{
int ret = 0;
if (!gadget->ops->wakeup) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->wakeup(gadget);
out:
trace_usb_gadget_wakeup(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_wakeup);
/**
* usb_gadget_set_remote_wakeup - configures the device remote wakeup feature.
* @gadget:the device being configured for remote wakeup
* @set:value to be configured.
*
* set to one to enable remote wakeup feature and zero to disable it.
*
* returns zero on success, else negative errno.
*/
int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set)
{
int ret = 0;
if (!gadget->ops->set_remote_wakeup) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->set_remote_wakeup(gadget, set);
out:
trace_usb_gadget_set_remote_wakeup(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_set_remote_wakeup);
/**
* usb_gadget_set_selfpowered - sets the device selfpowered feature.
* @gadget:the device being declared as self-powered
*
* this affects the device status reported by the hardware driver
* to reflect that it now has a local power supply.
*
* returns zero on success, else negative errno.
*/
int usb_gadget_set_selfpowered(struct usb_gadget *gadget)
{
int ret = 0;
if (!gadget->ops->set_selfpowered) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->set_selfpowered(gadget, 1);
out:
trace_usb_gadget_set_selfpowered(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_set_selfpowered);
/**
* usb_gadget_clear_selfpowered - clear the device selfpowered feature.
* @gadget:the device being declared as bus-powered
*
* this affects the device status reported by the hardware driver.
* some hardware may not support bus-powered operation, in which
* case this feature's value can never change.
*
* returns zero on success, else negative errno.
*/
int usb_gadget_clear_selfpowered(struct usb_gadget *gadget)
{
int ret = 0;
if (!gadget->ops->set_selfpowered) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->set_selfpowered(gadget, 0);
out:
trace_usb_gadget_clear_selfpowered(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_clear_selfpowered);
/**
* usb_gadget_vbus_connect - Notify controller that VBUS is powered
* @gadget:The device which now has VBUS power.
* Context: can sleep
*
* This call is used by a driver for an external transceiver (or GPIO)
* that detects a VBUS power session starting. Common responses include
* resuming the controller, activating the D+ (or D-) pullup to let the
* host detect that a USB device is attached, and starting to draw power
* (8mA or possibly more, especially after SET_CONFIGURATION).
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_vbus_connect(struct usb_gadget *gadget)
{
int ret = 0;
if (!gadget->ops->vbus_session) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->vbus_session(gadget, 1);
out:
trace_usb_gadget_vbus_connect(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_vbus_connect);
/**
* usb_gadget_vbus_draw - constrain controller's VBUS power usage
* @gadget:The device whose VBUS usage is being described
* @mA:How much current to draw, in milliAmperes. This should be twice
* the value listed in the configuration descriptor bMaxPower field.
*
* This call is used by gadget drivers during SET_CONFIGURATION calls,
* reporting how much power the device may consume. For example, this
* could affect how quickly batteries are recharged.
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_vbus_draw(struct usb_gadget *gadget, unsigned mA)
{
int ret = 0;
if (!gadget->ops->vbus_draw) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->vbus_draw(gadget, mA);
if (!ret)
gadget->mA = mA;
out:
trace_usb_gadget_vbus_draw(gadget, ret); return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_vbus_draw);
/**
* usb_gadget_vbus_disconnect - notify controller about VBUS session end
* @gadget:the device whose VBUS supply is being described
* Context: can sleep
*
* This call is used by a driver for an external transceiver (or GPIO)
* that detects a VBUS power session ending. Common responses include
* reversing everything done in usb_gadget_vbus_connect().
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_vbus_disconnect(struct usb_gadget *gadget)
{
int ret = 0;
if (!gadget->ops->vbus_session) {
ret = -EOPNOTSUPP;
goto out;
}
ret = gadget->ops->vbus_session(gadget, 0);
out:
trace_usb_gadget_vbus_disconnect(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_vbus_disconnect);
static int usb_gadget_connect_locked(struct usb_gadget *gadget)
__must_hold(&gadget->udc->connect_lock)
{
int ret = 0;
if (!gadget->ops->pullup) {
ret = -EOPNOTSUPP;
goto out;
}
if (gadget->deactivated || !gadget->udc->allow_connect || !gadget->udc->started) {
/*
* If the gadget isn't usable (because it is deactivated,
* unbound, or not yet started), we only save the new state.
* The gadget will be connected automatically when it is
* activated/bound/started.
*/
gadget->connected = true;
goto out;
}
ret = gadget->ops->pullup(gadget, 1);
if (!ret)
gadget->connected = 1;
out:
trace_usb_gadget_connect(gadget, ret); return ret;
}
/**
* usb_gadget_connect - software-controlled connect to USB host
* @gadget:the peripheral being connected
*
* Enables the D+ (or potentially D-) pullup. The host will start
* enumerating this gadget when the pullup is active and a VBUS session
* is active (the link is powered).
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_connect(struct usb_gadget *gadget)
{
int ret;
mutex_lock(&gadget->udc->connect_lock);
ret = usb_gadget_connect_locked(gadget);
mutex_unlock(&gadget->udc->connect_lock);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_connect);
static int usb_gadget_disconnect_locked(struct usb_gadget *gadget)
__must_hold(&gadget->udc->connect_lock)
{
int ret = 0;
if (!gadget->ops->pullup) {
ret = -EOPNOTSUPP;
goto out;
}
if (!gadget->connected)
goto out;
if (gadget->deactivated || !gadget->udc->started) {
/*
* If gadget is deactivated we only save new state.
* Gadget will stay disconnected after activation.
*/
gadget->connected = false;
goto out;
}
ret = gadget->ops->pullup(gadget, 0);
if (!ret)
gadget->connected = 0; mutex_lock(&udc_lock);
if (gadget->udc->driver)
gadget->udc->driver->disconnect(gadget); mutex_unlock(&udc_lock);
out:
trace_usb_gadget_disconnect(gadget, ret); return ret;
}
/**
* usb_gadget_disconnect - software-controlled disconnect from USB host
* @gadget:the peripheral being disconnected
*
* Disables the D+ (or potentially D-) pullup, which the host may see
* as a disconnect (when a VBUS session is active). Not all systems
* support software pullup controls.
*
* Following a successful disconnect, invoke the ->disconnect() callback
* for the current gadget driver so that UDC drivers don't need to.
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_disconnect(struct usb_gadget *gadget)
{
int ret;
mutex_lock(&gadget->udc->connect_lock);
ret = usb_gadget_disconnect_locked(gadget);
mutex_unlock(&gadget->udc->connect_lock);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_disconnect);
/**
* usb_gadget_deactivate - deactivate function which is not ready to work
* @gadget: the peripheral being deactivated
*
* This routine may be used during the gadget driver bind() call to prevent
* the peripheral from ever being visible to the USB host, unless later
* usb_gadget_activate() is called. For example, user mode components may
* need to be activated before the system can talk to hosts.
*
* This routine may sleep; it must not be called in interrupt context
* (such as from within a gadget driver's disconnect() callback).
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_deactivate(struct usb_gadget *gadget)
{
int ret = 0;
mutex_lock(&gadget->udc->connect_lock);
if (gadget->deactivated)
goto unlock;
if (gadget->connected) {
ret = usb_gadget_disconnect_locked(gadget);
if (ret)
goto unlock;
/*
* If gadget was being connected before deactivation, we want
* to reconnect it in usb_gadget_activate().
*/
gadget->connected = true;
}
gadget->deactivated = true;
unlock:
mutex_unlock(&gadget->udc->connect_lock);
trace_usb_gadget_deactivate(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_deactivate);
/**
* usb_gadget_activate - activate function which is not ready to work
* @gadget: the peripheral being activated
*
* This routine activates gadget which was previously deactivated with
* usb_gadget_deactivate() call. It calls usb_gadget_connect() if needed.
*
* This routine may sleep; it must not be called in interrupt context.
*
* Returns zero on success, else negative errno.
*/
int usb_gadget_activate(struct usb_gadget *gadget)
{
int ret = 0;
mutex_lock(&gadget->udc->connect_lock);
if (!gadget->deactivated)
goto unlock;
gadget->deactivated = false;
/*
* If gadget has been connected before deactivation, or became connected
* while it was being deactivated, we call usb_gadget_connect().
*/
if (gadget->connected)
ret = usb_gadget_connect_locked(gadget);
unlock:
mutex_unlock(&gadget->udc->connect_lock);
trace_usb_gadget_activate(gadget, ret);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_activate);
/* ------------------------------------------------------------------------- */
#ifdef CONFIG_HAS_DMA
int usb_gadget_map_request_by_dev(struct device *dev,
struct usb_request *req, int is_in)
{
if (req->length == 0)
return 0;
if (req->sg_was_mapped) {
req->num_mapped_sgs = req->num_sgs;
return 0;
}
if (req->num_sgs) {
int mapped;
mapped = dma_map_sg(dev, req->sg, req->num_sgs,
is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
if (mapped == 0) {
dev_err(dev, "failed to map SGs\n");
return -EFAULT;
}
req->num_mapped_sgs = mapped;
} else {
if (is_vmalloc_addr(req->buf)) {
dev_err(dev, "buffer is not dma capable\n");
return -EFAULT;
} else if (object_is_on_stack(req->buf)) {
dev_err(dev, "buffer is on stack\n");
return -EFAULT;
}
req->dma = dma_map_single(dev, req->buf, req->length,
is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
if (dma_mapping_error(dev, req->dma)) {
dev_err(dev, "failed to map buffer\n");
return -EFAULT;
}
req->dma_mapped = 1;
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_gadget_map_request_by_dev);
int usb_gadget_map_request(struct usb_gadget *gadget,
struct usb_request *req, int is_in)
{
return usb_gadget_map_request_by_dev(gadget->dev.parent, req, is_in);
}
EXPORT_SYMBOL_GPL(usb_gadget_map_request);
void usb_gadget_unmap_request_by_dev(struct device *dev,
struct usb_request *req, int is_in)
{
if (req->length == 0 || req->sg_was_mapped)
return;
if (req->num_mapped_sgs) {
dma_unmap_sg(dev, req->sg, req->num_sgs,
is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
req->num_mapped_sgs = 0;
} else if (req->dma_mapped) {
dma_unmap_single(dev, req->dma, req->length,
is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
req->dma_mapped = 0;
}
}
EXPORT_SYMBOL_GPL(usb_gadget_unmap_request_by_dev);
void usb_gadget_unmap_request(struct usb_gadget *gadget,
struct usb_request *req, int is_in)
{
usb_gadget_unmap_request_by_dev(gadget->dev.parent, req, is_in);
}
EXPORT_SYMBOL_GPL(usb_gadget_unmap_request);
#endif /* CONFIG_HAS_DMA */
/* ------------------------------------------------------------------------- */
/**
* usb_gadget_giveback_request - give the request back to the gadget layer
* @ep: the endpoint to be used with with the request
* @req: the request being given back
*
* This is called by device controller drivers in order to return the
* completed request back to the gadget layer.
*/
void usb_gadget_giveback_request(struct usb_ep *ep,
struct usb_request *req)
{
if (likely(req->status == 0)) usb_led_activity(USB_LED_EVENT_GADGET); trace_usb_gadget_giveback_request(ep, req, 0); req->complete(ep, req);
}
EXPORT_SYMBOL_GPL(usb_gadget_giveback_request);
/* ------------------------------------------------------------------------- */
/**
* gadget_find_ep_by_name - returns ep whose name is the same as sting passed
* in second parameter or NULL if searched endpoint not found
* @g: controller to check for quirk
* @name: name of searched endpoint
*/
struct usb_ep *gadget_find_ep_by_name(struct usb_gadget *g, const char *name)
{
struct usb_ep *ep;
gadget_for_each_ep(ep, g) {
if (!strcmp(ep->name, name))
return ep;
}
return NULL;
}
EXPORT_SYMBOL_GPL(gadget_find_ep_by_name);
/* ------------------------------------------------------------------------- */
int usb_gadget_ep_match_desc(struct usb_gadget *gadget,
struct usb_ep *ep, struct usb_endpoint_descriptor *desc,
struct usb_ss_ep_comp_descriptor *ep_comp)
{
u8 type;
u16 max;
int num_req_streams = 0;
/* endpoint already claimed? */
if (ep->claimed) return 0; type = usb_endpoint_type(desc); max = usb_endpoint_maxp(desc); if (usb_endpoint_dir_in(desc) && !ep->caps.dir_in)
return 0;
if (usb_endpoint_dir_out(desc) && !ep->caps.dir_out)
return 0;
if (max > ep->maxpacket_limit)
return 0;
/* "high bandwidth" works only at high speed */
if (!gadget_is_dualspeed(gadget) && usb_endpoint_maxp_mult(desc) > 1)
return 0;
switch (type) {
case USB_ENDPOINT_XFER_CONTROL:
/* only support ep0 for portable CONTROL traffic */
return 0;
case USB_ENDPOINT_XFER_ISOC:
if (!ep->caps.type_iso)
return 0;
/* ISO: limit 1023 bytes full speed, 1024 high/super speed */
if (!gadget_is_dualspeed(gadget) && max > 1023)
return 0;
break;
case USB_ENDPOINT_XFER_BULK:
if (!ep->caps.type_bulk)
return 0;
if (ep_comp && gadget_is_superspeed(gadget)) {
/* Get the number of required streams from the
* EP companion descriptor and see if the EP
* matches it
*/
num_req_streams = ep_comp->bmAttributes & 0x1f;
if (num_req_streams > ep->max_streams)
return 0;
}
break;
case USB_ENDPOINT_XFER_INT:
/* Bulk endpoints handle interrupt transfers,
* except the toggle-quirky iso-synch kind
*/
if (!ep->caps.type_int && !ep->caps.type_bulk)
return 0;
/* INT: limit 64 bytes full speed, 1024 high/super speed */
if (!gadget_is_dualspeed(gadget) && max > 64)
return 0;
break;
}
return 1;
}
EXPORT_SYMBOL_GPL(usb_gadget_ep_match_desc);
/**
* usb_gadget_check_config - checks if the UDC can support the binded
* configuration
* @gadget: controller to check the USB configuration
*
* Ensure that a UDC is able to support the requested resources by a
* configuration, and that there are no resource limitations, such as
* internal memory allocated to all requested endpoints.
*
* Returns zero on success, else a negative errno.
*/
int usb_gadget_check_config(struct usb_gadget *gadget)
{
if (gadget->ops->check_config)
return gadget->ops->check_config(gadget);
return 0;
}
EXPORT_SYMBOL_GPL(usb_gadget_check_config);
/* ------------------------------------------------------------------------- */
static void usb_gadget_state_work(struct work_struct *work)
{
struct usb_gadget *gadget = work_to_gadget(work);
struct usb_udc *udc = gadget->udc;
if (udc)
sysfs_notify(&udc->dev.kobj, NULL, "state");
}
void usb_gadget_set_state(struct usb_gadget *gadget,
enum usb_device_state state)
{
gadget->state = state;
schedule_work(&gadget->work);
}
EXPORT_SYMBOL_GPL(usb_gadget_set_state);
/* ------------------------------------------------------------------------- */
/* Acquire connect_lock before calling this function. */
static int usb_udc_connect_control_locked(struct usb_udc *udc) __must_hold(&udc->connect_lock)
{
if (udc->vbus)
return usb_gadget_connect_locked(udc->gadget);
else
return usb_gadget_disconnect_locked(udc->gadget);
}
static void vbus_event_work(struct work_struct *work)
{
struct usb_udc *udc = container_of(work, struct usb_udc, vbus_work);
mutex_lock(&udc->connect_lock);
usb_udc_connect_control_locked(udc);
mutex_unlock(&udc->connect_lock);
}
/**
* usb_udc_vbus_handler - updates the udc core vbus status, and try to
* connect or disconnect gadget
* @gadget: The gadget which vbus change occurs
* @status: The vbus status
*
* The udc driver calls it when it wants to connect or disconnect gadget
* according to vbus status.
*
* This function can be invoked from interrupt context by irq handlers of
* the gadget drivers, however, usb_udc_connect_control() has to run in
* non-atomic context due to the following:
* a. Some of the gadget driver implementations expect the ->pullup
* callback to be invoked in non-atomic context.
* b. usb_gadget_disconnect() acquires udc_lock which is a mutex.
* Hence offload invocation of usb_udc_connect_control() to workqueue.
*/
void usb_udc_vbus_handler(struct usb_gadget *gadget, bool status)
{
struct usb_udc *udc = gadget->udc;
if (udc) {
udc->vbus = status;
schedule_work(&udc->vbus_work);
}
}
EXPORT_SYMBOL_GPL(usb_udc_vbus_handler);
/**
* usb_gadget_udc_reset - notifies the udc core that bus reset occurs
* @gadget: The gadget which bus reset occurs
* @driver: The gadget driver we want to notify
*
* If the udc driver has bus reset handler, it needs to call this when the bus
* reset occurs, it notifies the gadget driver that the bus reset occurs as
* well as updates gadget state.
*/
void usb_gadget_udc_reset(struct usb_gadget *gadget,
struct usb_gadget_driver *driver)
{
driver->reset(gadget);
usb_gadget_set_state(gadget, USB_STATE_DEFAULT);
}
EXPORT_SYMBOL_GPL(usb_gadget_udc_reset);
/**
* usb_gadget_udc_start_locked - tells usb device controller to start up
* @udc: The UDC to be started
*
* This call is issued by the UDC Class driver when it's about
* to register a gadget driver to the device controller, before
* calling gadget driver's bind() method.
*
* It allows the controller to be powered off until strictly
* necessary to have it powered on.
*
* Returns zero on success, else negative errno.
*
* Caller should acquire connect_lock before invoking this function.
*/
static inline int usb_gadget_udc_start_locked(struct usb_udc *udc)
__must_hold(&udc->connect_lock)
{
int ret;
if (udc->started) {
dev_err(&udc->dev, "UDC had already started\n");
return -EBUSY;
}
ret = udc->gadget->ops->udc_start(udc->gadget, udc->driver);
if (!ret)
udc->started = true;
return ret;
}
/**
* usb_gadget_udc_stop_locked - tells usb device controller we don't need it anymore
* @udc: The UDC to be stopped
*
* This call is issued by the UDC Class driver after calling
* gadget driver's unbind() method.
*
* The details are implementation specific, but it can go as
* far as powering off UDC completely and disable its data
* line pullups.
*
* Caller should acquire connect lock before invoking this function.
*/
static inline void usb_gadget_udc_stop_locked(struct usb_udc *udc)
__must_hold(&udc->connect_lock)
{
if (!udc->started) { dev_err(&udc->dev, "UDC had already stopped\n");
return;
}
udc->gadget->ops->udc_stop(udc->gadget);
udc->started = false;
}
/**
* usb_gadget_udc_set_speed - tells usb device controller speed supported by
* current driver
* @udc: The device we want to set maximum speed
* @speed: The maximum speed to allowed to run
*
* This call is issued by the UDC Class driver before calling
* usb_gadget_udc_start() in order to make sure that we don't try to
* connect on speeds the gadget driver doesn't support.
*/
static inline void usb_gadget_udc_set_speed(struct usb_udc *udc,
enum usb_device_speed speed)
{
struct usb_gadget *gadget = udc->gadget;
enum usb_device_speed s;
if (speed == USB_SPEED_UNKNOWN)
s = gadget->max_speed;
else
s = min(speed, gadget->max_speed); if (s == USB_SPEED_SUPER_PLUS && gadget->ops->udc_set_ssp_rate) gadget->ops->udc_set_ssp_rate(gadget, gadget->max_ssp_rate); else if (gadget->ops->udc_set_speed) gadget->ops->udc_set_speed(gadget, s);
}
/**
* usb_gadget_enable_async_callbacks - tell usb device controller to enable asynchronous callbacks
* @udc: The UDC which should enable async callbacks
*
* This routine is used when binding gadget drivers. It undoes the effect
* of usb_gadget_disable_async_callbacks(); the UDC driver should enable IRQs
* (if necessary) and resume issuing callbacks.
*
* This routine will always be called in process context.
*/
static inline void usb_gadget_enable_async_callbacks(struct usb_udc *udc)
{
struct usb_gadget *gadget = udc->gadget;
if (gadget->ops->udc_async_callbacks)
gadget->ops->udc_async_callbacks(gadget, true);
}
/**
* usb_gadget_disable_async_callbacks - tell usb device controller to disable asynchronous callbacks
* @udc: The UDC which should disable async callbacks
*
* This routine is used when unbinding gadget drivers. It prevents a race:
* The UDC driver doesn't know when the gadget driver's ->unbind callback
* runs, so unless it is told to disable asynchronous callbacks, it might
* issue a callback (such as ->disconnect) after the unbind has completed.
*
* After this function runs, the UDC driver must suppress all ->suspend,
* ->resume, ->disconnect, ->reset, and ->setup callbacks to the gadget driver
* until async callbacks are again enabled. A simple-minded but effective
* way to accomplish this is to tell the UDC hardware not to generate any
* more IRQs.
*
* Request completion callbacks must still be issued. However, it's okay
* to defer them until the request is cancelled, since the pull-up will be
* turned off during the time period when async callbacks are disabled.
*
* This routine will always be called in process context.
*/
static inline void usb_gadget_disable_async_callbacks(struct usb_udc *udc)
{
struct usb_gadget *gadget = udc->gadget;
if (gadget->ops->udc_async_callbacks)
gadget->ops->udc_async_callbacks(gadget, false);
}
/**
* usb_udc_release - release the usb_udc struct
* @dev: the dev member within usb_udc
*
* This is called by driver's core in order to free memory once the last
* reference is released.
*/
static void usb_udc_release(struct device *dev)
{
struct usb_udc *udc;
udc = container_of(dev, struct usb_udc, dev);
dev_dbg(dev, "releasing '%s'\n", dev_name(dev));
kfree(udc);
}
static const struct attribute_group *usb_udc_attr_groups[];
static void usb_udc_nop_release(struct device *dev)
{
dev_vdbg(dev, "%s\n", __func__);
}
/**
* usb_initialize_gadget - initialize a gadget and its embedded struct device
* @parent: the parent device to this udc. Usually the controller driver's
* device.
* @gadget: the gadget to be initialized.
* @release: a gadget release function.
*/
void usb_initialize_gadget(struct device *parent, struct usb_gadget *gadget,
void (*release)(struct device *dev))
{
INIT_WORK(&gadget->work, usb_gadget_state_work);
gadget->dev.parent = parent;
if (release)
gadget->dev.release = release;
else
gadget->dev.release = usb_udc_nop_release;
device_initialize(&gadget->dev);
gadget->dev.bus = &gadget_bus_type;
}
EXPORT_SYMBOL_GPL(usb_initialize_gadget);
/**
* usb_add_gadget - adds a new gadget to the udc class driver list
* @gadget: the gadget to be added to the list.
*
* Returns zero on success, negative errno otherwise.
* Does not do a final usb_put_gadget() if an error occurs.
*/
int usb_add_gadget(struct usb_gadget *gadget)
{
struct usb_udc *udc;
int ret = -ENOMEM;
udc = kzalloc(sizeof(*udc), GFP_KERNEL);
if (!udc)
goto error;
device_initialize(&udc->dev);
udc->dev.release = usb_udc_release;
udc->dev.class = &udc_class;
udc->dev.groups = usb_udc_attr_groups;
udc->dev.parent = gadget->dev.parent;
ret = dev_set_name(&udc->dev, "%s",
kobject_name(&gadget->dev.parent->kobj));
if (ret)
goto err_put_udc;
udc->gadget = gadget;
gadget->udc = udc;
mutex_init(&udc->connect_lock);
udc->started = false;
mutex_lock(&udc_lock);
list_add_tail(&udc->list, &udc_list);
mutex_unlock(&udc_lock);
INIT_WORK(&udc->vbus_work, vbus_event_work);
ret = device_add(&udc->dev);
if (ret)
goto err_unlist_udc;
usb_gadget_set_state(gadget, USB_STATE_NOTATTACHED);
udc->vbus = true;
ret = ida_alloc(&gadget_id_numbers, GFP_KERNEL);
if (ret < 0)
goto err_del_udc;
gadget->id_number = ret;
dev_set_name(&gadget->dev, "gadget.%d", ret);
ret = device_add(&gadget->dev);
if (ret)
goto err_free_id;
ret = sysfs_create_link(&udc->dev.kobj,
&gadget->dev.kobj, "gadget");
if (ret)
goto err_del_gadget;
return 0;
err_del_gadget:
device_del(&gadget->dev);
err_free_id:
ida_free(&gadget_id_numbers, gadget->id_number);
err_del_udc:
flush_work(&gadget->work);
device_del(&udc->dev);
err_unlist_udc:
mutex_lock(&udc_lock);
list_del(&udc->list);
mutex_unlock(&udc_lock);
err_put_udc:
put_device(&udc->dev);
error:
return ret;
}
EXPORT_SYMBOL_GPL(usb_add_gadget);
/**
* usb_add_gadget_udc_release - adds a new gadget to the udc class driver list
* @parent: the parent device to this udc. Usually the controller driver's
* device.
* @gadget: the gadget to be added to the list.
* @release: a gadget release function.
*
* Returns zero on success, negative errno otherwise.
* Calls the gadget release function in the latter case.
*/
int usb_add_gadget_udc_release(struct device *parent, struct usb_gadget *gadget,
void (*release)(struct device *dev))
{
int ret;
usb_initialize_gadget(parent, gadget, release);
ret = usb_add_gadget(gadget);
if (ret)
usb_put_gadget(gadget);
return ret;
}
EXPORT_SYMBOL_GPL(usb_add_gadget_udc_release);
/**
* usb_get_gadget_udc_name - get the name of the first UDC controller
* This functions returns the name of the first UDC controller in the system.
* Please note that this interface is usefull only for legacy drivers which
* assume that there is only one UDC controller in the system and they need to
* get its name before initialization. There is no guarantee that the UDC
* of the returned name will be still available, when gadget driver registers
* itself.
*
* Returns pointer to string with UDC controller name on success, NULL
* otherwise. Caller should kfree() returned string.
*/
char *usb_get_gadget_udc_name(void)
{
struct usb_udc *udc;
char *name = NULL;
/* For now we take the first available UDC */
mutex_lock(&udc_lock);
list_for_each_entry(udc, &udc_list, list) {
if (!udc->driver) {
name = kstrdup(udc->gadget->name, GFP_KERNEL);
break;
}
}
mutex_unlock(&udc_lock);
return name;
}
EXPORT_SYMBOL_GPL(usb_get_gadget_udc_name);
/**
* usb_add_gadget_udc - adds a new gadget to the udc class driver list
* @parent: the parent device to this udc. Usually the controller
* driver's device.
* @gadget: the gadget to be added to the list
*
* Returns zero on success, negative errno otherwise.
*/
int usb_add_gadget_udc(struct device *parent, struct usb_gadget *gadget)
{
return usb_add_gadget_udc_release(parent, gadget, NULL);
}
EXPORT_SYMBOL_GPL(usb_add_gadget_udc);
/**
* usb_del_gadget - deletes a gadget and unregisters its udc
* @gadget: the gadget to be deleted.
*
* This will unbind @gadget, if it is bound.
* It will not do a final usb_put_gadget().
*/
void usb_del_gadget(struct usb_gadget *gadget)
{
struct usb_udc *udc = gadget->udc;
if (!udc)
return;
dev_vdbg(gadget->dev.parent, "unregistering gadget\n");
mutex_lock(&udc_lock);
list_del(&udc->list);
mutex_unlock(&udc_lock);
kobject_uevent(&udc->dev.kobj, KOBJ_REMOVE);
sysfs_remove_link(&udc->dev.kobj, "gadget");
flush_work(&gadget->work);
device_del(&gadget->dev);
ida_free(&gadget_id_numbers, gadget->id_number);
cancel_work_sync(&udc->vbus_work);
device_unregister(&udc->dev);
}
EXPORT_SYMBOL_GPL(usb_del_gadget);
/**
* usb_del_gadget_udc - unregisters a gadget
* @gadget: the gadget to be unregistered.
*
* Calls usb_del_gadget() and does a final usb_put_gadget().
*/
void usb_del_gadget_udc(struct usb_gadget *gadget)
{
usb_del_gadget(gadget);
usb_put_gadget(gadget);
}
EXPORT_SYMBOL_GPL(usb_del_gadget_udc);
/* ------------------------------------------------------------------------- */
static int gadget_match_driver(struct device *dev, struct device_driver *drv)
{
struct usb_gadget *gadget = dev_to_usb_gadget(dev);
struct usb_udc *udc = gadget->udc;
struct usb_gadget_driver *driver = container_of(drv,
struct usb_gadget_driver, driver);
/* If the driver specifies a udc_name, it must match the UDC's name */
if (driver->udc_name && strcmp(driver->udc_name, dev_name(&udc->dev)) != 0)
return 0;
/* If the driver is already bound to a gadget, it doesn't match */
if (driver->is_bound)
return 0;
/* Otherwise any gadget driver matches any UDC */
return 1;
}
static int gadget_bind_driver(struct device *dev)
{
struct usb_gadget *gadget = dev_to_usb_gadget(dev);
struct usb_udc *udc = gadget->udc;
struct usb_gadget_driver *driver = container_of(dev->driver,
struct usb_gadget_driver, driver);
int ret = 0;
mutex_lock(&udc_lock);
if (driver->is_bound) {
mutex_unlock(&udc_lock);
return -ENXIO; /* Driver binds to only one gadget */
}
driver->is_bound = true;
udc->driver = driver;
mutex_unlock(&udc_lock);
dev_dbg(&udc->dev, "binding gadget driver [%s]\n", driver->function); usb_gadget_udc_set_speed(udc, driver->max_speed); ret = driver->bind(udc->gadget, driver);
if (ret)
goto err_bind;
mutex_lock(&udc->connect_lock); ret = usb_gadget_udc_start_locked(udc);
if (ret) {
mutex_unlock(&udc->connect_lock);
goto err_start;
}
usb_gadget_enable_async_callbacks(udc); udc->allow_connect = true; ret = usb_udc_connect_control_locked(udc); if (ret)
goto err_connect_control;
mutex_unlock(&udc->connect_lock);
kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE);
return 0;
err_connect_control:
udc->allow_connect = false; usb_gadget_disable_async_callbacks(udc); if (gadget->irq) synchronize_irq(gadget->irq); usb_gadget_udc_stop_locked(udc); mutex_unlock(&udc->connect_lock);
err_start:
driver->unbind(udc->gadget);
err_bind:
if (ret != -EISNAM) dev_err(&udc->dev, "failed to start %s: %d\n",
driver->function, ret);
mutex_lock(&udc_lock);
udc->driver = NULL;
driver->is_bound = false;
mutex_unlock(&udc_lock);
return ret;
}
static void gadget_unbind_driver(struct device *dev)
{
struct usb_gadget *gadget = dev_to_usb_gadget(dev);
struct usb_udc *udc = gadget->udc;
struct usb_gadget_driver *driver = udc->driver;
dev_dbg(&udc->dev, "unbinding gadget driver [%s]\n", driver->function); udc->allow_connect = false;
cancel_work_sync(&udc->vbus_work);
mutex_lock(&udc->connect_lock);
usb_gadget_disconnect_locked(gadget);
usb_gadget_disable_async_callbacks(udc); if (gadget->irq) synchronize_irq(gadget->irq); mutex_unlock(&udc->connect_lock);
udc->driver->unbind(gadget);
mutex_lock(&udc->connect_lock);
usb_gadget_udc_stop_locked(udc); mutex_unlock(&udc->connect_lock);
mutex_lock(&udc_lock);
driver->is_bound = false;
udc->driver = NULL;
mutex_unlock(&udc_lock);
kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE);
}
/* ------------------------------------------------------------------------- */
int usb_gadget_register_driver_owner(struct usb_gadget_driver *driver,
struct module *owner, const char *mod_name)
{
int ret;
if (!driver || !driver->bind || !driver->setup)
return -EINVAL;
driver->driver.bus = &gadget_bus_type;
driver->driver.owner = owner;
driver->driver.mod_name = mod_name;
ret = driver_register(&driver->driver);
if (ret) {
pr_warn("%s: driver registration failed: %d\n",
driver->function, ret);
return ret;
}
mutex_lock(&udc_lock);
if (!driver->is_bound) {
if (driver->match_existing_only) { pr_warn("%s: couldn't find an available UDC or it's busy\n",
driver->function);
ret = -EBUSY;
} else {
pr_info("%s: couldn't find an available UDC\n",
driver->function);
ret = 0;
}
}
mutex_unlock(&udc_lock); if (ret) driver_unregister(&driver->driver);
return ret;
}
EXPORT_SYMBOL_GPL(usb_gadget_register_driver_owner);
int usb_gadget_unregister_driver(struct usb_gadget_driver *driver)
{
if (!driver || !driver->unbind)
return -EINVAL;
driver_unregister(&driver->driver); return 0;
}
EXPORT_SYMBOL_GPL(usb_gadget_unregister_driver);
/* ------------------------------------------------------------------------- */
static ssize_t srp_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t n)
{
struct usb_udc *udc = container_of(dev, struct usb_udc, dev);
if (sysfs_streq(buf, "1"))
usb_gadget_wakeup(udc->gadget);
return n;
}
static DEVICE_ATTR_WO(srp);
static ssize_t soft_connect_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t n)
{
struct usb_udc *udc = container_of(dev, struct usb_udc, dev);
ssize_t ret;
device_lock(&udc->gadget->dev);
if (!udc->driver) {
dev_err(dev, "soft-connect without a gadget driver\n");
ret = -EOPNOTSUPP;
goto out;
}
if (sysfs_streq(buf, "connect")) {
mutex_lock(&udc->connect_lock);
usb_gadget_udc_start_locked(udc);
usb_gadget_connect_locked(udc->gadget);
mutex_unlock(&udc->connect_lock);
} else if (sysfs_streq(buf, "disconnect")) {
mutex_lock(&udc->connect_lock);
usb_gadget_disconnect_locked(udc->gadget);
usb_gadget_udc_stop_locked(udc);
mutex_unlock(&udc->connect_lock);
} else {
dev_err(dev, "unsupported command '%s'\n", buf);
ret = -EINVAL;
goto out;
}
ret = n;
out:
device_unlock(&udc->gadget->dev);
return ret;
}
static DEVICE_ATTR_WO(soft_connect);
static ssize_t state_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct usb_udc *udc = container_of(dev, struct usb_udc, dev);
struct usb_gadget *gadget = udc->gadget;
return sprintf(buf, "%s\n", usb_state_string(gadget->state));
}
static DEVICE_ATTR_RO(state);
static ssize_t function_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct usb_udc *udc = container_of(dev, struct usb_udc, dev);
struct usb_gadget_driver *drv;
int rc = 0;
mutex_lock(&udc_lock);
drv = udc->driver;
if (drv && drv->function)
rc = scnprintf(buf, PAGE_SIZE, "%s\n", drv->function);
mutex_unlock(&udc_lock);
return rc;
}
static DEVICE_ATTR_RO(function);
#define USB_UDC_SPEED_ATTR(name, param) \
ssize_t name##_show(struct device *dev, \
struct device_attribute *attr, char *buf) \
{ \
struct usb_udc *udc = container_of(dev, struct usb_udc, dev); \
return scnprintf(buf, PAGE_SIZE, "%s\n", \
usb_speed_string(udc->gadget->param)); \
} \
static DEVICE_ATTR_RO(name)
static USB_UDC_SPEED_ATTR(current_speed, speed);
static USB_UDC_SPEED_ATTR(maximum_speed, max_speed);
#define USB_UDC_ATTR(name) \
ssize_t name##_show(struct device *dev, \
struct device_attribute *attr, char *buf) \
{ \
struct usb_udc *udc = container_of(dev, struct usb_udc, dev); \
struct usb_gadget *gadget = udc->gadget; \
\
return scnprintf(buf, PAGE_SIZE, "%d\n", gadget->name); \
} \
static DEVICE_ATTR_RO(name)
static USB_UDC_ATTR(is_otg);
static USB_UDC_ATTR(is_a_peripheral);
static USB_UDC_ATTR(b_hnp_enable);
static USB_UDC_ATTR(a_hnp_support);
static USB_UDC_ATTR(a_alt_hnp_support);
static USB_UDC_ATTR(is_selfpowered);
static struct attribute *usb_udc_attrs[] = {
&dev_attr_srp.attr,
&dev_attr_soft_connect.attr,
&dev_attr_state.attr,
&dev_attr_function.attr,
&dev_attr_current_speed.attr,
&dev_attr_maximum_speed.attr,
&dev_attr_is_otg.attr,
&dev_attr_is_a_peripheral.attr,
&dev_attr_b_hnp_enable.attr,
&dev_attr_a_hnp_support.attr,
&dev_attr_a_alt_hnp_support.attr,
&dev_attr_is_selfpowered.attr,
NULL,
};
static const struct attribute_group usb_udc_attr_group = {
.attrs = usb_udc_attrs,
};
static const struct attribute_group *usb_udc_attr_groups[] = {
&usb_udc_attr_group,
NULL,
};
static int usb_udc_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct usb_udc *udc = container_of(dev, struct usb_udc, dev);
int ret;
ret = add_uevent_var(env, "USB_UDC_NAME=%s", udc->gadget->name);
if (ret) {
dev_err(dev, "failed to add uevent USB_UDC_NAME\n"); return ret;
}
mutex_lock(&udc_lock);
if (udc->driver)
ret = add_uevent_var(env, "USB_UDC_DRIVER=%s",
udc->driver->function);
mutex_unlock(&udc_lock); if (ret) { dev_err(dev, "failed to add uevent USB_UDC_DRIVER\n"); return ret;
}
return 0;
}
static const struct class udc_class = {
.name = "udc",
.dev_uevent = usb_udc_uevent,
};
static const struct bus_type gadget_bus_type = {
.name = "gadget",
.probe = gadget_bind_driver,
.remove = gadget_unbind_driver,
.match = gadget_match_driver,
};
static int __init usb_udc_init(void)
{
int rc;
rc = class_register(&udc_class);
if (rc)
return rc;
rc = bus_register(&gadget_bus_type);
if (rc)
class_unregister(&udc_class);
return rc;
}
subsys_initcall(usb_udc_init);
static void __exit usb_udc_exit(void)
{
bus_unregister(&gadget_bus_type);
class_unregister(&udc_class);
}
module_exit(usb_udc_exit);
MODULE_DESCRIPTION("UDC Framework");
MODULE_AUTHOR("Felipe Balbi <balbi@ti.com>");
MODULE_LICENSE("GPL v2");
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 1999-2002 Vojtech Pavlik
*/
#ifndef _INPUT_H
#define _INPUT_H
#include <linux/time.h>
#include <linux/list.h>
#include <uapi/linux/input.h>
/* Implementation details, userspace should not care about these */
#define ABS_MT_FIRST ABS_MT_TOUCH_MAJOR
#define ABS_MT_LAST ABS_MT_TOOL_Y
/*
* In-kernel definitions.
*/
#include <linux/device.h>
#include <linux/fs.h>
#include <linux/timer.h>
#include <linux/mod_devicetable.h>
struct input_dev_poller;
/**
* struct input_value - input value representation
* @type: type of value (EV_KEY, EV_ABS, etc)
* @code: the value code
* @value: the value
*/
struct input_value {
__u16 type;
__u16 code;
__s32 value;
};
enum input_clock_type {
INPUT_CLK_REAL = 0,
INPUT_CLK_MONO,
INPUT_CLK_BOOT,
INPUT_CLK_MAX
};
/**
* struct input_dev - represents an input device
* @name: name of the device
* @phys: physical path to the device in the system hierarchy
* @uniq: unique identification code for the device (if device has it)
* @id: id of the device (struct input_id)
* @propbit: bitmap of device properties and quirks
* @evbit: bitmap of types of events supported by the device (EV_KEY,
* EV_REL, etc.)
* @keybit: bitmap of keys/buttons this device has
* @relbit: bitmap of relative axes for the device
* @absbit: bitmap of absolute axes for the device
* @mscbit: bitmap of miscellaneous events supported by the device
* @ledbit: bitmap of leds present on the device
* @sndbit: bitmap of sound effects supported by the device
* @ffbit: bitmap of force feedback effects supported by the device
* @swbit: bitmap of switches present on the device
* @hint_events_per_packet: average number of events generated by the
* device in a packet (between EV_SYN/SYN_REPORT events). Used by
* event handlers to estimate size of the buffer needed to hold
* events.
* @keycodemax: size of keycode table
* @keycodesize: size of elements in keycode table
* @keycode: map of scancodes to keycodes for this device
* @getkeycode: optional legacy method to retrieve current keymap.
* @setkeycode: optional method to alter current keymap, used to implement
* sparse keymaps. If not supplied default mechanism will be used.
* The method is being called while holding event_lock and thus must
* not sleep
* @ff: force feedback structure associated with the device if device
* supports force feedback effects
* @poller: poller structure associated with the device if device is
* set up to use polling mode
* @repeat_key: stores key code of the last key pressed; used to implement
* software autorepeat
* @timer: timer for software autorepeat
* @rep: current values for autorepeat parameters (delay, rate)
* @mt: pointer to multitouch state
* @absinfo: array of &struct input_absinfo elements holding information
* about absolute axes (current value, min, max, flat, fuzz,
* resolution)
* @key: reflects current state of device's keys/buttons
* @led: reflects current state of device's LEDs
* @snd: reflects current state of sound effects
* @sw: reflects current state of device's switches
* @open: this method is called when the very first user calls
* input_open_device(). The driver must prepare the device
* to start generating events (start polling thread,
* request an IRQ, submit URB, etc.). The meaning of open() is
* to start providing events to the input core.
* @close: this method is called when the very last user calls
* input_close_device(). The meaning of close() is to stop
* providing events to the input core.
* @flush: purges the device. Most commonly used to get rid of force
* feedback effects loaded into the device when disconnecting
* from it
* @event: event handler for events sent _to_ the device, like EV_LED
* or EV_SND. The device is expected to carry out the requested
* action (turn on a LED, play sound, etc.) The call is protected
* by @event_lock and must not sleep
* @grab: input handle that currently has the device grabbed (via
* EVIOCGRAB ioctl). When a handle grabs a device it becomes sole
* recipient for all input events coming from the device
* @event_lock: this spinlock is taken when input core receives
* and processes a new event for the device (in input_event()).
* Code that accesses and/or modifies parameters of a device
* (such as keymap or absmin, absmax, absfuzz, etc.) after device
* has been registered with input core must take this lock.
* @mutex: serializes calls to open(), close() and flush() methods
* @users: stores number of users (input handlers) that opened this
* device. It is used by input_open_device() and input_close_device()
* to make sure that dev->open() is only called when the first
* user opens device and dev->close() is called when the very
* last user closes the device
* @going_away: marks devices that are in a middle of unregistering and
* causes input_open_device*() fail with -ENODEV.
* @dev: driver model's view of this device
* @h_list: list of input handles associated with the device. When
* accessing the list dev->mutex must be held
* @node: used to place the device onto input_dev_list
* @num_vals: number of values queued in the current frame
* @max_vals: maximum number of values queued in a frame
* @vals: array of values queued in the current frame
* @devres_managed: indicates that devices is managed with devres framework
* and needs not be explicitly unregistered or freed.
* @timestamp: storage for a timestamp set by input_set_timestamp called
* by a driver
* @inhibited: indicates that the input device is inhibited. If that is
* the case then input core ignores any events generated by the device.
* Device's close() is called when it is being inhibited and its open()
* is called when it is being uninhibited.
*/
struct input_dev {
const char *name;
const char *phys;
const char *uniq;
struct input_id id;
unsigned long propbit[BITS_TO_LONGS(INPUT_PROP_CNT)];
unsigned long evbit[BITS_TO_LONGS(EV_CNT)];
unsigned long keybit[BITS_TO_LONGS(KEY_CNT)];
unsigned long relbit[BITS_TO_LONGS(REL_CNT)];
unsigned long absbit[BITS_TO_LONGS(ABS_CNT)];
unsigned long mscbit[BITS_TO_LONGS(MSC_CNT)];
unsigned long ledbit[BITS_TO_LONGS(LED_CNT)];
unsigned long sndbit[BITS_TO_LONGS(SND_CNT)];
unsigned long ffbit[BITS_TO_LONGS(FF_CNT)];
unsigned long swbit[BITS_TO_LONGS(SW_CNT)];
unsigned int hint_events_per_packet;
unsigned int keycodemax;
unsigned int keycodesize;
void *keycode;
int (*setkeycode)(struct input_dev *dev,
const struct input_keymap_entry *ke,
unsigned int *old_keycode);
int (*getkeycode)(struct input_dev *dev,
struct input_keymap_entry *ke);
struct ff_device *ff;
struct input_dev_poller *poller;
unsigned int repeat_key;
struct timer_list timer;
int rep[REP_CNT];
struct input_mt *mt;
struct input_absinfo *absinfo;
unsigned long key[BITS_TO_LONGS(KEY_CNT)];
unsigned long led[BITS_TO_LONGS(LED_CNT)];
unsigned long snd[BITS_TO_LONGS(SND_CNT)];
unsigned long sw[BITS_TO_LONGS(SW_CNT)];
int (*open)(struct input_dev *dev);
void (*close)(struct input_dev *dev);
int (*flush)(struct input_dev *dev, struct file *file);
int (*event)(struct input_dev *dev, unsigned int type, unsigned int code, int value);
struct input_handle __rcu *grab;
spinlock_t event_lock;
struct mutex mutex;
unsigned int users;
bool going_away;
struct device dev;
struct list_head h_list;
struct list_head node;
unsigned int num_vals;
unsigned int max_vals;
struct input_value *vals;
bool devres_managed;
ktime_t timestamp[INPUT_CLK_MAX];
bool inhibited;
};
#define to_input_dev(d) container_of(d, struct input_dev, dev)
/*
* Verify that we are in sync with input_device_id mod_devicetable.h #defines
*/
#if EV_MAX != INPUT_DEVICE_ID_EV_MAX
#error "EV_MAX and INPUT_DEVICE_ID_EV_MAX do not match"
#endif
#if KEY_MIN_INTERESTING != INPUT_DEVICE_ID_KEY_MIN_INTERESTING
#error "KEY_MIN_INTERESTING and INPUT_DEVICE_ID_KEY_MIN_INTERESTING do not match"
#endif
#if KEY_MAX != INPUT_DEVICE_ID_KEY_MAX
#error "KEY_MAX and INPUT_DEVICE_ID_KEY_MAX do not match"
#endif
#if REL_MAX != INPUT_DEVICE_ID_REL_MAX
#error "REL_MAX and INPUT_DEVICE_ID_REL_MAX do not match"
#endif
#if ABS_MAX != INPUT_DEVICE_ID_ABS_MAX
#error "ABS_MAX and INPUT_DEVICE_ID_ABS_MAX do not match"
#endif
#if MSC_MAX != INPUT_DEVICE_ID_MSC_MAX
#error "MSC_MAX and INPUT_DEVICE_ID_MSC_MAX do not match"
#endif
#if LED_MAX != INPUT_DEVICE_ID_LED_MAX
#error "LED_MAX and INPUT_DEVICE_ID_LED_MAX do not match"
#endif
#if SND_MAX != INPUT_DEVICE_ID_SND_MAX
#error "SND_MAX and INPUT_DEVICE_ID_SND_MAX do not match"
#endif
#if FF_MAX != INPUT_DEVICE_ID_FF_MAX
#error "FF_MAX and INPUT_DEVICE_ID_FF_MAX do not match"
#endif
#if SW_MAX != INPUT_DEVICE_ID_SW_MAX
#error "SW_MAX and INPUT_DEVICE_ID_SW_MAX do not match"
#endif
#if INPUT_PROP_MAX != INPUT_DEVICE_ID_PROP_MAX
#error "INPUT_PROP_MAX and INPUT_DEVICE_ID_PROP_MAX do not match"
#endif
#define INPUT_DEVICE_ID_MATCH_DEVICE \
(INPUT_DEVICE_ID_MATCH_BUS | INPUT_DEVICE_ID_MATCH_VENDOR | INPUT_DEVICE_ID_MATCH_PRODUCT)
#define INPUT_DEVICE_ID_MATCH_DEVICE_AND_VERSION \
(INPUT_DEVICE_ID_MATCH_DEVICE | INPUT_DEVICE_ID_MATCH_VERSION)
struct input_handle;
/**
* struct input_handler - implements one of interfaces for input devices
* @private: driver-specific data
* @event: event handler. This method is being called by input core with
* interrupts disabled and dev->event_lock spinlock held and so
* it may not sleep
* @events: event sequence handler. This method is being called by
* input core with interrupts disabled and dev->event_lock
* spinlock held and so it may not sleep
* @filter: similar to @event; separates normal event handlers from
* "filters".
* @match: called after comparing device's id with handler's id_table
* to perform fine-grained matching between device and handler
* @connect: called when attaching a handler to an input device
* @disconnect: disconnects a handler from input device
* @start: starts handler for given handle. This function is called by
* input core right after connect() method and also when a process
* that "grabbed" a device releases it
* @legacy_minors: set to %true by drivers using legacy minor ranges
* @minor: beginning of range of 32 legacy minors for devices this driver
* can provide
* @name: name of the handler, to be shown in /proc/bus/input/handlers
* @id_table: pointer to a table of input_device_ids this driver can
* handle
* @h_list: list of input handles associated with the handler
* @node: for placing the driver onto input_handler_list
*
* Input handlers attach to input devices and create input handles. There
* are likely several handlers attached to any given input device at the
* same time. All of them will get their copy of input event generated by
* the device.
*
* The very same structure is used to implement input filters. Input core
* allows filters to run first and will not pass event to regular handlers
* if any of the filters indicate that the event should be filtered (by
* returning %true from their filter() method).
*
* Note that input core serializes calls to connect() and disconnect()
* methods.
*/
struct input_handler {
void *private;
void (*event)(struct input_handle *handle, unsigned int type, unsigned int code, int value);
void (*events)(struct input_handle *handle,
const struct input_value *vals, unsigned int count);
bool (*filter)(struct input_handle *handle, unsigned int type, unsigned int code, int value);
bool (*match)(struct input_handler *handler, struct input_dev *dev);
int (*connect)(struct input_handler *handler, struct input_dev *dev, const struct input_device_id *id);
void (*disconnect)(struct input_handle *handle);
void (*start)(struct input_handle *handle);
bool legacy_minors;
int minor;
const char *name;
const struct input_device_id *id_table;
struct list_head h_list;
struct list_head node;
};
/**
* struct input_handle - links input device with an input handler
* @private: handler-specific data
* @open: counter showing whether the handle is 'open', i.e. should deliver
* events from its device
* @name: name given to the handle by handler that created it
* @dev: input device the handle is attached to
* @handler: handler that works with the device through this handle
* @d_node: used to put the handle on device's list of attached handles
* @h_node: used to put the handle on handler's list of handles from which
* it gets events
*/
struct input_handle {
void *private;
int open;
const char *name;
struct input_dev *dev;
struct input_handler *handler;
struct list_head d_node;
struct list_head h_node;
};
struct input_dev __must_check *input_allocate_device(void);
struct input_dev __must_check *devm_input_allocate_device(struct device *);
void input_free_device(struct input_dev *dev);
static inline struct input_dev *input_get_device(struct input_dev *dev)
{
return dev ? to_input_dev(get_device(&dev->dev)) : NULL;
}
static inline void input_put_device(struct input_dev *dev)
{
if (dev)
put_device(&dev->dev);
}
static inline void *input_get_drvdata(struct input_dev *dev)
{
return dev_get_drvdata(&dev->dev);
}
static inline void input_set_drvdata(struct input_dev *dev, void *data)
{
dev_set_drvdata(&dev->dev, data);
}
int __must_check input_register_device(struct input_dev *);
void input_unregister_device(struct input_dev *);
void input_reset_device(struct input_dev *);
int input_setup_polling(struct input_dev *dev,
void (*poll_fn)(struct input_dev *dev));
void input_set_poll_interval(struct input_dev *dev, unsigned int interval);
void input_set_min_poll_interval(struct input_dev *dev, unsigned int interval);
void input_set_max_poll_interval(struct input_dev *dev, unsigned int interval);
int input_get_poll_interval(struct input_dev *dev);
int __must_check input_register_handler(struct input_handler *);
void input_unregister_handler(struct input_handler *);
int __must_check input_get_new_minor(int legacy_base, unsigned int legacy_num,
bool allow_dynamic);
void input_free_minor(unsigned int minor);
int input_handler_for_each_handle(struct input_handler *, void *data,
int (*fn)(struct input_handle *, void *));
int input_register_handle(struct input_handle *);
void input_unregister_handle(struct input_handle *);
int input_grab_device(struct input_handle *);
void input_release_device(struct input_handle *);
int input_open_device(struct input_handle *);
void input_close_device(struct input_handle *);
int input_flush_device(struct input_handle *handle, struct file *file);
void input_set_timestamp(struct input_dev *dev, ktime_t timestamp);
ktime_t *input_get_timestamp(struct input_dev *dev);
void input_event(struct input_dev *dev, unsigned int type, unsigned int code, int value);
void input_inject_event(struct input_handle *handle, unsigned int type, unsigned int code, int value);
static inline void input_report_key(struct input_dev *dev, unsigned int code, int value)
{
input_event(dev, EV_KEY, code, !!value);
}
static inline void input_report_rel(struct input_dev *dev, unsigned int code, int value)
{
input_event(dev, EV_REL, code, value);
}
static inline void input_report_abs(struct input_dev *dev, unsigned int code, int value)
{
input_event(dev, EV_ABS, code, value);
}
static inline void input_report_ff_status(struct input_dev *dev, unsigned int code, int value)
{
input_event(dev, EV_FF_STATUS, code, value);
}
static inline void input_report_switch(struct input_dev *dev, unsigned int code, int value)
{
input_event(dev, EV_SW, code, !!value);
}
static inline void input_sync(struct input_dev *dev)
{
input_event(dev, EV_SYN, SYN_REPORT, 0);
}
static inline void input_mt_sync(struct input_dev *dev)
{
input_event(dev, EV_SYN, SYN_MT_REPORT, 0);
}
void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int code);
/**
* input_set_events_per_packet - tell handlers about the driver event rate
* @dev: the input device used by the driver
* @n_events: the average number of events between calls to input_sync()
*
* If the event rate sent from a device is unusually large, use this
* function to set the expected event rate. This will allow handlers
* to set up an appropriate buffer size for the event stream, in order
* to minimize information loss.
*/
static inline void input_set_events_per_packet(struct input_dev *dev, int n_events)
{
dev->hint_events_per_packet = n_events;
}
void input_alloc_absinfo(struct input_dev *dev);
void input_set_abs_params(struct input_dev *dev, unsigned int axis,
int min, int max, int fuzz, int flat);
void input_copy_abs(struct input_dev *dst, unsigned int dst_axis,
const struct input_dev *src, unsigned int src_axis);
#define INPUT_GENERATE_ABS_ACCESSORS(_suffix, _item) \
static inline int input_abs_get_##_suffix(struct input_dev *dev, \
unsigned int axis) \
{ \
return dev->absinfo ? dev->absinfo[axis]._item : 0; \
} \
\
static inline void input_abs_set_##_suffix(struct input_dev *dev, \
unsigned int axis, int val) \
{ \
input_alloc_absinfo(dev); \
if (dev->absinfo) \
dev->absinfo[axis]._item = val; \
}
INPUT_GENERATE_ABS_ACCESSORS(val, value)
INPUT_GENERATE_ABS_ACCESSORS(min, minimum)
INPUT_GENERATE_ABS_ACCESSORS(max, maximum)
INPUT_GENERATE_ABS_ACCESSORS(fuzz, fuzz)
INPUT_GENERATE_ABS_ACCESSORS(flat, flat)
INPUT_GENERATE_ABS_ACCESSORS(res, resolution)
int input_scancode_to_scalar(const struct input_keymap_entry *ke,
unsigned int *scancode);
int input_get_keycode(struct input_dev *dev, struct input_keymap_entry *ke);
int input_set_keycode(struct input_dev *dev,
const struct input_keymap_entry *ke);
bool input_match_device_id(const struct input_dev *dev,
const struct input_device_id *id);
void input_enable_softrepeat(struct input_dev *dev, int delay, int period);
bool input_device_enabled(struct input_dev *dev);
extern const struct class input_class;
/**
* struct ff_device - force-feedback part of an input device
* @upload: Called to upload an new effect into device
* @erase: Called to erase an effect from device
* @playback: Called to request device to start playing specified effect
* @set_gain: Called to set specified gain
* @set_autocenter: Called to auto-center device
* @destroy: called by input core when parent input device is being
* destroyed
* @private: driver-specific data, will be freed automatically
* @ffbit: bitmap of force feedback capabilities truly supported by
* device (not emulated like ones in input_dev->ffbit)
* @mutex: mutex for serializing access to the device
* @max_effects: maximum number of effects supported by device
* @effects: pointer to an array of effects currently loaded into device
* @effect_owners: array of effect owners; when file handle owning
* an effect gets closed the effect is automatically erased
*
* Every force-feedback device must implement upload() and playback()
* methods; erase() is optional. set_gain() and set_autocenter() need
* only be implemented if driver sets up FF_GAIN and FF_AUTOCENTER
* bits.
*
* Note that playback(), set_gain() and set_autocenter() are called with
* dev->event_lock spinlock held and interrupts off and thus may not
* sleep.
*/
struct ff_device {
int (*upload)(struct input_dev *dev, struct ff_effect *effect,
struct ff_effect *old);
int (*erase)(struct input_dev *dev, int effect_id);
int (*playback)(struct input_dev *dev, int effect_id, int value);
void (*set_gain)(struct input_dev *dev, u16 gain);
void (*set_autocenter)(struct input_dev *dev, u16 magnitude);
void (*destroy)(struct ff_device *);
void *private;
unsigned long ffbit[BITS_TO_LONGS(FF_CNT)];
struct mutex mutex;
int max_effects;
struct ff_effect *effects;
struct file *effect_owners[] __counted_by(max_effects);
};
int input_ff_create(struct input_dev *dev, unsigned int max_effects);
void input_ff_destroy(struct input_dev *dev);
int input_ff_event(struct input_dev *dev, unsigned int type, unsigned int code, int value);
int input_ff_upload(struct input_dev *dev, struct ff_effect *effect, struct file *file);
int input_ff_erase(struct input_dev *dev, int effect_id, struct file *file);
int input_ff_flush(struct input_dev *dev, struct file *file);
int input_ff_create_memless(struct input_dev *dev, void *data,
int (*play_effect)(struct input_dev *, void *, struct ff_effect *));
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2002,2003 by Andreas Gruenbacher <a.gruenbacher@computer.org>
*
* Fixes from William Schumacher incorporated on 15 March 2001.
* (Reported by Charles Bertsch, <CBertsch@microtest.com>).
*/
/*
* This file contains generic functions for manipulating
* POSIX 1003.1e draft standard 17 ACLs.
*/
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/atomic.h>
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/cred.h>
#include <linux/posix_acl.h>
#include <linux/posix_acl_xattr.h>
#include <linux/xattr.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/namei.h>
#include <linux/mnt_idmapping.h>
#include <linux/iversion.h>
#include <linux/security.h>
#include <linux/fsnotify.h>
#include <linux/filelock.h>
#include "internal.h"
static struct posix_acl **acl_by_type(struct inode *inode, int type)
{
switch (type) {
case ACL_TYPE_ACCESS:
return &inode->i_acl;
case ACL_TYPE_DEFAULT:
return &inode->i_default_acl;
default:
BUG();
}
}
struct posix_acl *get_cached_acl(struct inode *inode, int type)
{
struct posix_acl **p = acl_by_type(inode, type);
struct posix_acl *acl;
for (;;) {
rcu_read_lock(); acl = rcu_dereference(*p); if (!acl || is_uncached_acl(acl) || refcount_inc_not_zero(&acl->a_refcount))
break;
rcu_read_unlock();
cpu_relax();
}
rcu_read_unlock();
return acl;
}
EXPORT_SYMBOL(get_cached_acl);
struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type)
{
struct posix_acl *acl = rcu_dereference(*acl_by_type(inode, type));
if (acl == ACL_DONT_CACHE) {
struct posix_acl *ret;
ret = inode->i_op->get_inode_acl(inode, type, LOOKUP_RCU);
if (!IS_ERR(ret))
acl = ret;
}
return acl;
}
EXPORT_SYMBOL(get_cached_acl_rcu);
void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl)
{
struct posix_acl **p = acl_by_type(inode, type);
struct posix_acl *old;
old = xchg(p, posix_acl_dup(acl));
if (!is_uncached_acl(old))
posix_acl_release(old);
}
EXPORT_SYMBOL(set_cached_acl);
static void __forget_cached_acl(struct posix_acl **p)
{
struct posix_acl *old;
old = xchg(p, ACL_NOT_CACHED);
if (!is_uncached_acl(old))
posix_acl_release(old);
}
void forget_cached_acl(struct inode *inode, int type)
{
__forget_cached_acl(acl_by_type(inode, type));
}
EXPORT_SYMBOL(forget_cached_acl);
void forget_all_cached_acls(struct inode *inode)
{
__forget_cached_acl(&inode->i_acl);
__forget_cached_acl(&inode->i_default_acl);
}
EXPORT_SYMBOL(forget_all_cached_acls);
static struct posix_acl *__get_acl(struct mnt_idmap *idmap,
struct dentry *dentry, struct inode *inode,
int type)
{
struct posix_acl *sentinel;
struct posix_acl **p;
struct posix_acl *acl;
/*
* The sentinel is used to detect when another operation like
* set_cached_acl() or forget_cached_acl() races with get_inode_acl().
* It is guaranteed that is_uncached_acl(sentinel) is true.
*/
acl = get_cached_acl(inode, type);
if (!is_uncached_acl(acl))
return acl;
if (!IS_POSIXACL(inode))
return NULL;
sentinel = uncached_acl_sentinel(current); p = acl_by_type(inode, type);
/*
* If the ACL isn't being read yet, set our sentinel. Otherwise, the
* current value of the ACL will not be ACL_NOT_CACHED and so our own
* sentinel will not be set; another task will update the cache. We
* could wait for that other task to complete its job, but it's easier
* to just call ->get_inode_acl to fetch the ACL ourself. (This is
* going to be an unlikely race.)
*/
cmpxchg(p, ACL_NOT_CACHED, sentinel);
/*
* Normally, the ACL returned by ->get{_inode}_acl will be cached.
* A filesystem can prevent that by calling
* forget_cached_acl(inode, type) in ->get{_inode}_acl.
*
* If the filesystem doesn't have a get{_inode}_ acl() function at all,
* we'll just create the negative cache entry.
*/
if (dentry && inode->i_op->get_acl) { acl = inode->i_op->get_acl(idmap, dentry, type); } else if (inode->i_op->get_inode_acl) { acl = inode->i_op->get_inode_acl(inode, type, false);
} else {
set_cached_acl(inode, type, NULL);
return NULL;
}
if (IS_ERR(acl)) {
/*
* Remove our sentinel so that we don't block future attempts
* to cache the ACL.
*/
cmpxchg(p, sentinel, ACL_NOT_CACHED);
return acl;
}
/*
* Cache the result, but only if our sentinel is still in place.
*/
posix_acl_dup(acl); if (unlikely(!try_cmpxchg(p, &sentinel, acl))) posix_acl_release(acl);
return acl;
}
struct posix_acl *get_inode_acl(struct inode *inode, int type)
{
return __get_acl(&nop_mnt_idmap, NULL, inode, type);
}
EXPORT_SYMBOL(get_inode_acl);
/*
* Init a fresh posix_acl
*/
void
posix_acl_init(struct posix_acl *acl, int count)
{
refcount_set(&acl->a_refcount, 1);
acl->a_count = count;
}
EXPORT_SYMBOL(posix_acl_init);
/*
* Allocate a new ACL with the specified number of entries.
*/
struct posix_acl *
posix_acl_alloc(int count, gfp_t flags)
{
const size_t size = sizeof(struct posix_acl) +
count * sizeof(struct posix_acl_entry);
struct posix_acl *acl = kmalloc(size, flags);
if (acl)
posix_acl_init(acl, count);
return acl;
}
EXPORT_SYMBOL(posix_acl_alloc);
/*
* Clone an ACL.
*/
struct posix_acl *
posix_acl_clone(const struct posix_acl *acl, gfp_t flags)
{
struct posix_acl *clone = NULL;
if (acl) {
int size = sizeof(struct posix_acl) + acl->a_count *
sizeof(struct posix_acl_entry);
clone = kmemdup(acl, size, flags);
if (clone)
refcount_set(&clone->a_refcount, 1);
}
return clone;
}
EXPORT_SYMBOL_GPL(posix_acl_clone);
/*
* Check if an acl is valid. Returns 0 if it is, or -E... otherwise.
*/
int
posix_acl_valid(struct user_namespace *user_ns, const struct posix_acl *acl)
{
const struct posix_acl_entry *pa, *pe;
int state = ACL_USER_OBJ;
int needs_mask = 0;
FOREACH_ACL_ENTRY(pa, acl, pe) {
if (pa->e_perm & ~(ACL_READ|ACL_WRITE|ACL_EXECUTE))
return -EINVAL;
switch (pa->e_tag) {
case ACL_USER_OBJ:
if (state == ACL_USER_OBJ) {
state = ACL_USER;
break;
}
return -EINVAL;
case ACL_USER:
if (state != ACL_USER)
return -EINVAL;
if (!kuid_has_mapping(user_ns, pa->e_uid))
return -EINVAL;
needs_mask = 1;
break;
case ACL_GROUP_OBJ:
if (state == ACL_USER) {
state = ACL_GROUP;
break;
}
return -EINVAL;
case ACL_GROUP:
if (state != ACL_GROUP)
return -EINVAL;
if (!kgid_has_mapping(user_ns, pa->e_gid))
return -EINVAL;
needs_mask = 1;
break;
case ACL_MASK:
if (state != ACL_GROUP)
return -EINVAL;
state = ACL_OTHER;
break;
case ACL_OTHER:
if (state == ACL_OTHER ||
(state == ACL_GROUP && !needs_mask)) {
state = 0;
break;
}
return -EINVAL;
default:
return -EINVAL;
}
}
if (state == 0)
return 0;
return -EINVAL;
}
EXPORT_SYMBOL(posix_acl_valid);
/*
* Returns 0 if the acl can be exactly represented in the traditional
* file mode permission bits, or else 1. Returns -E... on error.
*/
int
posix_acl_equiv_mode(const struct posix_acl *acl, umode_t *mode_p)
{
const struct posix_acl_entry *pa, *pe;
umode_t mode = 0;
int not_equiv = 0;
/*
* A null ACL can always be presented as mode bits.
*/
if (!acl)
return 0;
FOREACH_ACL_ENTRY(pa, acl, pe) {
switch (pa->e_tag) {
case ACL_USER_OBJ:
mode |= (pa->e_perm & S_IRWXO) << 6;
break;
case ACL_GROUP_OBJ:
mode |= (pa->e_perm & S_IRWXO) << 3;
break;
case ACL_OTHER:
mode |= pa->e_perm & S_IRWXO;
break;
case ACL_MASK:
mode = (mode & ~S_IRWXG) |
((pa->e_perm & S_IRWXO) << 3);
not_equiv = 1;
break;
case ACL_USER:
case ACL_GROUP:
not_equiv = 1;
break;
default:
return -EINVAL;
}
}
if (mode_p)
*mode_p = (*mode_p & ~S_IRWXUGO) | mode;
return not_equiv;
}
EXPORT_SYMBOL(posix_acl_equiv_mode);
/*
* Create an ACL representing the file mode permission bits of an inode.
*/
struct posix_acl *
posix_acl_from_mode(umode_t mode, gfp_t flags)
{
struct posix_acl *acl = posix_acl_alloc(3, flags);
if (!acl)
return ERR_PTR(-ENOMEM);
acl->a_entries[0].e_tag = ACL_USER_OBJ;
acl->a_entries[0].e_perm = (mode & S_IRWXU) >> 6;
acl->a_entries[1].e_tag = ACL_GROUP_OBJ;
acl->a_entries[1].e_perm = (mode & S_IRWXG) >> 3;
acl->a_entries[2].e_tag = ACL_OTHER;
acl->a_entries[2].e_perm = (mode & S_IRWXO);
return acl;
}
EXPORT_SYMBOL(posix_acl_from_mode);
/*
* Return 0 if current is granted want access to the inode
* by the acl. Returns -E... otherwise.
*/
int
posix_acl_permission(struct mnt_idmap *idmap, struct inode *inode,
const struct posix_acl *acl, int want)
{
const struct posix_acl_entry *pa, *pe, *mask_obj;
struct user_namespace *fs_userns = i_user_ns(inode);
int found = 0;
vfsuid_t vfsuid;
vfsgid_t vfsgid;
want &= MAY_READ | MAY_WRITE | MAY_EXEC;
FOREACH_ACL_ENTRY(pa, acl, pe) {
switch(pa->e_tag) {
case ACL_USER_OBJ:
/* (May have been checked already) */
vfsuid = i_uid_into_vfsuid(idmap, inode);
if (vfsuid_eq_kuid(vfsuid, current_fsuid()))
goto check_perm;
break;
case ACL_USER:
vfsuid = make_vfsuid(idmap, fs_userns,
pa->e_uid);
if (vfsuid_eq_kuid(vfsuid, current_fsuid()))
goto mask;
break;
case ACL_GROUP_OBJ:
vfsgid = i_gid_into_vfsgid(idmap, inode);
if (vfsgid_in_group_p(vfsgid)) {
found = 1;
if ((pa->e_perm & want) == want)
goto mask;
}
break;
case ACL_GROUP:
vfsgid = make_vfsgid(idmap, fs_userns,
pa->e_gid);
if (vfsgid_in_group_p(vfsgid)) {
found = 1;
if ((pa->e_perm & want) == want)
goto mask;
}
break;
case ACL_MASK:
break;
case ACL_OTHER:
if (found)
return -EACCES;
else
goto check_perm;
default:
return -EIO;
}
}
return -EIO;
mask:
for (mask_obj = pa+1; mask_obj != pe; mask_obj++) {
if (mask_obj->e_tag == ACL_MASK) {
if ((pa->e_perm & mask_obj->e_perm & want) == want)
return 0;
return -EACCES;
}
}
check_perm:
if ((pa->e_perm & want) == want)
return 0;
return -EACCES;
}
/*
* Modify acl when creating a new inode. The caller must ensure the acl is
* only referenced once.
*
* mode_p initially must contain the mode parameter to the open() / creat()
* system calls. All permissions that are not granted by the acl are removed.
* The permissions in the acl are changed to reflect the mode_p parameter.
*/
static int posix_acl_create_masq(struct posix_acl *acl, umode_t *mode_p)
{
struct posix_acl_entry *pa, *pe;
struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL;
umode_t mode = *mode_p;
int not_equiv = 0;
/* assert(atomic_read(acl->a_refcount) == 1); */
FOREACH_ACL_ENTRY(pa, acl, pe) {
switch(pa->e_tag) {
case ACL_USER_OBJ:
pa->e_perm &= (mode >> 6) | ~S_IRWXO;
mode &= (pa->e_perm << 6) | ~S_IRWXU;
break;
case ACL_USER:
case ACL_GROUP:
not_equiv = 1;
break;
case ACL_GROUP_OBJ:
group_obj = pa;
break;
case ACL_OTHER:
pa->e_perm &= mode | ~S_IRWXO;
mode &= pa->e_perm | ~S_IRWXO;
break;
case ACL_MASK:
mask_obj = pa;
not_equiv = 1;
break;
default:
return -EIO;
}
}
if (mask_obj) {
mask_obj->e_perm &= (mode >> 3) | ~S_IRWXO;
mode &= (mask_obj->e_perm << 3) | ~S_IRWXG;
} else {
if (!group_obj)
return -EIO;
group_obj->e_perm &= (mode >> 3) | ~S_IRWXO;
mode &= (group_obj->e_perm << 3) | ~S_IRWXG;
}
*mode_p = (*mode_p & ~S_IRWXUGO) | mode;
return not_equiv;
}
/*
* Modify the ACL for the chmod syscall.
*/
static int __posix_acl_chmod_masq(struct posix_acl *acl, umode_t mode)
{
struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL;
struct posix_acl_entry *pa, *pe;
/* assert(atomic_read(acl->a_refcount) == 1); */
FOREACH_ACL_ENTRY(pa, acl, pe) {
switch(pa->e_tag) {
case ACL_USER_OBJ:
pa->e_perm = (mode & S_IRWXU) >> 6;
break;
case ACL_USER:
case ACL_GROUP:
break;
case ACL_GROUP_OBJ:
group_obj = pa;
break;
case ACL_MASK:
mask_obj = pa;
break;
case ACL_OTHER:
pa->e_perm = (mode & S_IRWXO);
break;
default:
return -EIO;
}
}
if (mask_obj) {
mask_obj->e_perm = (mode & S_IRWXG) >> 3;
} else {
if (!group_obj)
return -EIO;
group_obj->e_perm = (mode & S_IRWXG) >> 3;
}
return 0;
}
int
__posix_acl_create(struct posix_acl **acl, gfp_t gfp, umode_t *mode_p)
{
struct posix_acl *clone = posix_acl_clone(*acl, gfp);
int err = -ENOMEM;
if (clone) {
err = posix_acl_create_masq(clone, mode_p);
if (err < 0) {
posix_acl_release(clone);
clone = NULL;
}
}
posix_acl_release(*acl);
*acl = clone;
return err;
}
EXPORT_SYMBOL(__posix_acl_create);
int
__posix_acl_chmod(struct posix_acl **acl, gfp_t gfp, umode_t mode)
{
struct posix_acl *clone = posix_acl_clone(*acl, gfp);
int err = -ENOMEM;
if (clone) {
err = __posix_acl_chmod_masq(clone, mode);
if (err) {
posix_acl_release(clone);
clone = NULL;
}
}
posix_acl_release(*acl);
*acl = clone;
return err;
}
EXPORT_SYMBOL(__posix_acl_chmod);
/**
* posix_acl_chmod - chmod a posix acl
*
* @idmap: idmap of the mount @inode was found from
* @dentry: dentry to check permissions on
* @mode: the new mode of @inode
*
* If the dentry has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*/
int
posix_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry,
umode_t mode)
{
struct inode *inode = d_inode(dentry);
struct posix_acl *acl;
int ret = 0;
if (!IS_POSIXACL(inode))
return 0;
if (!inode->i_op->set_acl)
return -EOPNOTSUPP;
acl = get_inode_acl(inode, ACL_TYPE_ACCESS);
if (IS_ERR_OR_NULL(acl)) {
if (acl == ERR_PTR(-EOPNOTSUPP))
return 0;
return PTR_ERR(acl);
}
ret = __posix_acl_chmod(&acl, GFP_KERNEL, mode);
if (ret)
return ret;
ret = inode->i_op->set_acl(idmap, dentry, acl, ACL_TYPE_ACCESS);
posix_acl_release(acl);
return ret;
}
EXPORT_SYMBOL(posix_acl_chmod);
int
posix_acl_create(struct inode *dir, umode_t *mode,
struct posix_acl **default_acl, struct posix_acl **acl)
{
struct posix_acl *p;
struct posix_acl *clone;
int ret;
*acl = NULL;
*default_acl = NULL;
if (S_ISLNK(*mode) || !IS_POSIXACL(dir)) return 0; p = get_inode_acl(dir, ACL_TYPE_DEFAULT); if (!p || p == ERR_PTR(-EOPNOTSUPP)) { *mode &= ~current_umask();
return 0;
}
if (IS_ERR(p)) return PTR_ERR(p);
ret = -ENOMEM;
clone = posix_acl_clone(p, GFP_NOFS);
if (!clone)
goto err_release;
ret = posix_acl_create_masq(clone, mode);
if (ret < 0)
goto err_release_clone;
if (ret == 0) posix_acl_release(clone);
else
*acl = clone; if (!S_ISDIR(*mode)) posix_acl_release(p);
else
*default_acl = p;
return 0;
err_release_clone:
posix_acl_release(clone);
err_release:
posix_acl_release(p);
return ret;
}
EXPORT_SYMBOL_GPL(posix_acl_create);
/**
* posix_acl_update_mode - update mode in set_acl
* @idmap: idmap of the mount @inode was found from
* @inode: target inode
* @mode_p: mode (pointer) for update
* @acl: acl pointer
*
* Update the file mode when setting an ACL: compute the new file permission
* bits based on the ACL. In addition, if the ACL is equivalent to the new
* file mode, set *@acl to NULL to indicate that no ACL should be set.
*
* As with chmod, clear the setgid bit if the caller is not in the owning group
* or capable of CAP_FSETID (see inode_change_ok).
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*
* Called from set_acl inode operations.
*/
int posix_acl_update_mode(struct mnt_idmap *idmap,
struct inode *inode, umode_t *mode_p,
struct posix_acl **acl)
{
umode_t mode = inode->i_mode;
int error;
error = posix_acl_equiv_mode(*acl, &mode);
if (error < 0)
return error;
if (error == 0)
*acl = NULL;
if (!vfsgid_in_group_p(i_gid_into_vfsgid(idmap, inode)) &&
!capable_wrt_inode_uidgid(idmap, inode, CAP_FSETID))
mode &= ~S_ISGID;
*mode_p = mode;
return 0;
}
EXPORT_SYMBOL(posix_acl_update_mode);
/*
* Fix up the uids and gids in posix acl extended attributes in place.
*/
static int posix_acl_fix_xattr_common(const void *value, size_t size)
{
const struct posix_acl_xattr_header *header = value;
int count;
if (!header)
return -EINVAL;
if (size < sizeof(struct posix_acl_xattr_header))
return -EINVAL;
if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION))
return -EOPNOTSUPP;
count = posix_acl_xattr_count(size);
if (count < 0)
return -EINVAL;
if (count == 0)
return 0;
return count;
}
/**
* posix_acl_from_xattr - convert POSIX ACLs from backing store to VFS format
* @userns: the filesystem's idmapping
* @value: the uapi representation of POSIX ACLs
* @size: the size of @void
*
* Filesystems that store POSIX ACLs in the unaltered uapi format should use
* posix_acl_from_xattr() when reading them from the backing store and
* converting them into the struct posix_acl VFS format. The helper is
* specifically intended to be called from the acl inode operation.
*
* The posix_acl_from_xattr() function will map the raw {g,u}id values stored
* in ACL_{GROUP,USER} entries into idmapping in @userns.
*
* Note that posix_acl_from_xattr() does not take idmapped mounts into account.
* If it did it calling it from the get acl inode operation would return POSIX
* ACLs mapped according to an idmapped mount which would mean that the value
* couldn't be cached for the filesystem. Idmapped mounts are taken into
* account on the fly during permission checking or right at the VFS -
* userspace boundary before reporting them to the user.
*
* Return: Allocated struct posix_acl on success, NULL for a valid header but
* without actual POSIX ACL entries, or ERR_PTR() encoded error code.
*/
struct posix_acl *posix_acl_from_xattr(struct user_namespace *userns,
const void *value, size_t size)
{
const struct posix_acl_xattr_header *header = value;
const struct posix_acl_xattr_entry *entry = (const void *)(header + 1), *end;
int count;
struct posix_acl *acl;
struct posix_acl_entry *acl_e;
count = posix_acl_fix_xattr_common(value, size);
if (count < 0)
return ERR_PTR(count);
if (count == 0)
return NULL;
acl = posix_acl_alloc(count, GFP_NOFS);
if (!acl)
return ERR_PTR(-ENOMEM);
acl_e = acl->a_entries;
for (end = entry + count; entry != end; acl_e++, entry++) {
acl_e->e_tag = le16_to_cpu(entry->e_tag);
acl_e->e_perm = le16_to_cpu(entry->e_perm);
switch(acl_e->e_tag) {
case ACL_USER_OBJ:
case ACL_GROUP_OBJ:
case ACL_MASK:
case ACL_OTHER:
break;
case ACL_USER:
acl_e->e_uid = make_kuid(userns,
le32_to_cpu(entry->e_id));
if (!uid_valid(acl_e->e_uid))
goto fail;
break;
case ACL_GROUP:
acl_e->e_gid = make_kgid(userns,
le32_to_cpu(entry->e_id));
if (!gid_valid(acl_e->e_gid))
goto fail;
break;
default:
goto fail;
}
}
return acl;
fail:
posix_acl_release(acl);
return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL (posix_acl_from_xattr);
/*
* Convert from in-memory to extended attribute representation.
*/
int
posix_acl_to_xattr(struct user_namespace *user_ns, const struct posix_acl *acl,
void *buffer, size_t size)
{
struct posix_acl_xattr_header *ext_acl = buffer;
struct posix_acl_xattr_entry *ext_entry;
int real_size, n;
real_size = posix_acl_xattr_size(acl->a_count);
if (!buffer)
return real_size;
if (real_size > size)
return -ERANGE;
ext_entry = (void *)(ext_acl + 1);
ext_acl->a_version = cpu_to_le32(POSIX_ACL_XATTR_VERSION);
for (n=0; n < acl->a_count; n++, ext_entry++) {
const struct posix_acl_entry *acl_e = &acl->a_entries[n];
ext_entry->e_tag = cpu_to_le16(acl_e->e_tag);
ext_entry->e_perm = cpu_to_le16(acl_e->e_perm);
switch(acl_e->e_tag) {
case ACL_USER:
ext_entry->e_id =
cpu_to_le32(from_kuid(user_ns, acl_e->e_uid));
break;
case ACL_GROUP:
ext_entry->e_id =
cpu_to_le32(from_kgid(user_ns, acl_e->e_gid));
break;
default:
ext_entry->e_id = cpu_to_le32(ACL_UNDEFINED_ID);
break;
}
}
return real_size;
}
EXPORT_SYMBOL (posix_acl_to_xattr);
/**
* vfs_posix_acl_to_xattr - convert from kernel to userspace representation
* @idmap: idmap of the mount
* @inode: inode the posix acls are set on
* @acl: the posix acls as represented by the vfs
* @buffer: the buffer into which to convert @acl
* @size: size of @buffer
*
* This converts @acl from the VFS representation in the filesystem idmapping
* to the uapi form reportable to userspace. And mount and caller idmappings
* are handled appropriately.
*
* Return: On success, the size of the stored uapi posix acls, on error a
* negative errno.
*/
static ssize_t vfs_posix_acl_to_xattr(struct mnt_idmap *idmap,
struct inode *inode,
const struct posix_acl *acl, void *buffer,
size_t size)
{
struct posix_acl_xattr_header *ext_acl = buffer;
struct posix_acl_xattr_entry *ext_entry;
struct user_namespace *fs_userns, *caller_userns;
ssize_t real_size, n;
vfsuid_t vfsuid;
vfsgid_t vfsgid;
real_size = posix_acl_xattr_size(acl->a_count);
if (!buffer)
return real_size;
if (real_size > size)
return -ERANGE;
ext_entry = (void *)(ext_acl + 1);
ext_acl->a_version = cpu_to_le32(POSIX_ACL_XATTR_VERSION);
fs_userns = i_user_ns(inode);
caller_userns = current_user_ns();
for (n=0; n < acl->a_count; n++, ext_entry++) {
const struct posix_acl_entry *acl_e = &acl->a_entries[n];
ext_entry->e_tag = cpu_to_le16(acl_e->e_tag);
ext_entry->e_perm = cpu_to_le16(acl_e->e_perm);
switch(acl_e->e_tag) {
case ACL_USER:
vfsuid = make_vfsuid(idmap, fs_userns, acl_e->e_uid);
ext_entry->e_id = cpu_to_le32(from_kuid(
caller_userns, vfsuid_into_kuid(vfsuid)));
break;
case ACL_GROUP:
vfsgid = make_vfsgid(idmap, fs_userns, acl_e->e_gid);
ext_entry->e_id = cpu_to_le32(from_kgid(
caller_userns, vfsgid_into_kgid(vfsgid)));
break;
default:
ext_entry->e_id = cpu_to_le32(ACL_UNDEFINED_ID);
break;
}
}
return real_size;
}
int
set_posix_acl(struct mnt_idmap *idmap, struct dentry *dentry,
int type, struct posix_acl *acl)
{
struct inode *inode = d_inode(dentry);
if (!IS_POSIXACL(inode))
return -EOPNOTSUPP;
if (!inode->i_op->set_acl)
return -EOPNOTSUPP;
if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode))
return acl ? -EACCES : 0;
if (!inode_owner_or_capable(idmap, inode))
return -EPERM;
if (acl) {
int ret = posix_acl_valid(inode->i_sb->s_user_ns, acl);
if (ret)
return ret;
}
return inode->i_op->set_acl(idmap, dentry, acl, type);
}
EXPORT_SYMBOL(set_posix_acl);
int posix_acl_listxattr(struct inode *inode, char **buffer,
ssize_t *remaining_size)
{
int err;
if (!IS_POSIXACL(inode))
return 0;
if (inode->i_acl) {
err = xattr_list_one(buffer, remaining_size,
XATTR_NAME_POSIX_ACL_ACCESS);
if (err)
return err;
}
if (inode->i_default_acl) {
err = xattr_list_one(buffer, remaining_size,
XATTR_NAME_POSIX_ACL_DEFAULT);
if (err)
return err;
}
return 0;
}
static bool
posix_acl_xattr_list(struct dentry *dentry)
{
return IS_POSIXACL(d_backing_inode(dentry));
}
/*
* nop_posix_acl_access - legacy xattr handler for access POSIX ACLs
*
* This is the legacy POSIX ACL access xattr handler. It is used by some
* filesystems to implement their ->listxattr() inode operation. New code
* should never use them.
*/
const struct xattr_handler nop_posix_acl_access = {
.name = XATTR_NAME_POSIX_ACL_ACCESS,
.list = posix_acl_xattr_list,
};
EXPORT_SYMBOL_GPL(nop_posix_acl_access);
/*
* nop_posix_acl_default - legacy xattr handler for default POSIX ACLs
*
* This is the legacy POSIX ACL default xattr handler. It is used by some
* filesystems to implement their ->listxattr() inode operation. New code
* should never use them.
*/
const struct xattr_handler nop_posix_acl_default = {
.name = XATTR_NAME_POSIX_ACL_DEFAULT,
.list = posix_acl_xattr_list,
};
EXPORT_SYMBOL_GPL(nop_posix_acl_default);
int simple_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
struct posix_acl *acl, int type)
{
int error;
struct inode *inode = d_inode(dentry);
if (type == ACL_TYPE_ACCESS) {
error = posix_acl_update_mode(idmap, inode,
&inode->i_mode, &acl);
if (error)
return error;
}
inode_set_ctime_current(inode);
if (IS_I_VERSION(inode))
inode_inc_iversion(inode);
set_cached_acl(inode, type, acl);
return 0;
}
int simple_acl_create(struct inode *dir, struct inode *inode)
{
struct posix_acl *default_acl, *acl;
int error;
error = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); if (error)
return error;
set_cached_acl(inode, ACL_TYPE_DEFAULT, default_acl);
set_cached_acl(inode, ACL_TYPE_ACCESS, acl);
if (default_acl)
posix_acl_release(default_acl); if (acl) posix_acl_release(acl);
return 0;
}
static int vfs_set_acl_idmapped_mnt(struct mnt_idmap *idmap,
struct user_namespace *fs_userns,
struct posix_acl *acl)
{
for (int n = 0; n < acl->a_count; n++) {
struct posix_acl_entry *acl_e = &acl->a_entries[n];
switch (acl_e->e_tag) {
case ACL_USER:
acl_e->e_uid = from_vfsuid(idmap, fs_userns,
VFSUIDT_INIT(acl_e->e_uid));
break;
case ACL_GROUP:
acl_e->e_gid = from_vfsgid(idmap, fs_userns,
VFSGIDT_INIT(acl_e->e_gid));
break;
}
}
return 0;
}
/**
* vfs_set_acl - set posix acls
* @idmap: idmap of the mount
* @dentry: the dentry based on which to set the posix acls
* @acl_name: the name of the posix acl
* @kacl: the posix acls in the appropriate VFS format
*
* This function sets @kacl. The caller must all posix_acl_release() on @kacl
* afterwards.
*
* Return: On success 0, on error negative errno.
*/
int vfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
const char *acl_name, struct posix_acl *kacl)
{
int acl_type;
int error;
struct inode *inode = d_inode(dentry);
struct inode *delegated_inode = NULL;
acl_type = posix_acl_type(acl_name);
if (acl_type < 0)
return -EINVAL;
if (kacl) {
/*
* If we're on an idmapped mount translate from mount specific
* vfs{g,u}id_t into global filesystem k{g,u}id_t.
* Afterwards we can cache the POSIX ACLs filesystem wide and -
* if this is a filesystem with a backing store - ultimately
* translate them to backing store values.
*/
error = vfs_set_acl_idmapped_mnt(idmap, i_user_ns(inode), kacl);
if (error)
return error;
}
retry_deleg:
inode_lock(inode);
/*
* We only care about restrictions the inode struct itself places upon
* us otherwise POSIX ACLs aren't subject to any VFS restrictions.
*/
error = may_write_xattr(idmap, inode);
if (error)
goto out_inode_unlock;
error = security_inode_set_acl(idmap, dentry, acl_name, kacl);
if (error)
goto out_inode_unlock;
error = try_break_deleg(inode, &delegated_inode);
if (error)
goto out_inode_unlock;
if (likely(!is_bad_inode(inode)))
error = set_posix_acl(idmap, dentry, acl_type, kacl);
else
error = -EIO;
if (!error) {
fsnotify_xattr(dentry);
security_inode_post_set_acl(dentry, acl_name, kacl);
}
out_inode_unlock:
inode_unlock(inode);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
return error;
}
EXPORT_SYMBOL_GPL(vfs_set_acl);
/**
* vfs_get_acl - get posix acls
* @idmap: idmap of the mount
* @dentry: the dentry based on which to retrieve the posix acls
* @acl_name: the name of the posix acl
*
* This function retrieves @kacl from the filesystem. The caller must all
* posix_acl_release() on @kacl.
*
* Return: On success POSIX ACLs in VFS format, on error negative errno.
*/
struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap,
struct dentry *dentry, const char *acl_name)
{
struct inode *inode = d_inode(dentry);
struct posix_acl *acl;
int acl_type, error;
acl_type = posix_acl_type(acl_name);
if (acl_type < 0)
return ERR_PTR(-EINVAL);
/*
* The VFS has no restrictions on reading POSIX ACLs so calling
* something like xattr_permission() isn't needed. Only LSMs get a say.
*/
error = security_inode_get_acl(idmap, dentry, acl_name);
if (error)
return ERR_PTR(error);
if (!IS_POSIXACL(inode))
return ERR_PTR(-EOPNOTSUPP);
if (S_ISLNK(inode->i_mode))
return ERR_PTR(-EOPNOTSUPP);
acl = __get_acl(idmap, dentry, inode, acl_type);
if (IS_ERR(acl))
return acl;
if (!acl)
return ERR_PTR(-ENODATA);
return acl;
}
EXPORT_SYMBOL_GPL(vfs_get_acl);
/**
* vfs_remove_acl - remove posix acls
* @idmap: idmap of the mount
* @dentry: the dentry based on which to retrieve the posix acls
* @acl_name: the name of the posix acl
*
* This function removes posix acls.
*
* Return: On success 0, on error negative errno.
*/
int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
const char *acl_name)
{
int acl_type;
int error;
struct inode *inode = d_inode(dentry);
struct inode *delegated_inode = NULL;
acl_type = posix_acl_type(acl_name);
if (acl_type < 0)
return -EINVAL;
retry_deleg:
inode_lock(inode);
/*
* We only care about restrictions the inode struct itself places upon
* us otherwise POSIX ACLs aren't subject to any VFS restrictions.
*/
error = may_write_xattr(idmap, inode);
if (error)
goto out_inode_unlock;
error = security_inode_remove_acl(idmap, dentry, acl_name);
if (error)
goto out_inode_unlock;
error = try_break_deleg(inode, &delegated_inode);
if (error)
goto out_inode_unlock;
if (likely(!is_bad_inode(inode)))
error = set_posix_acl(idmap, dentry, acl_type, NULL);
else
error = -EIO;
if (!error) {
fsnotify_xattr(dentry);
security_inode_post_remove_acl(idmap, dentry, acl_name);
}
out_inode_unlock:
inode_unlock(inode);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
return error;
}
EXPORT_SYMBOL_GPL(vfs_remove_acl);
int do_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
const char *acl_name, const void *kvalue, size_t size)
{
int error;
struct posix_acl *acl = NULL;
if (size) {
/*
* Note that posix_acl_from_xattr() uses GFP_NOFS when it
* probably doesn't need to here.
*/
acl = posix_acl_from_xattr(current_user_ns(), kvalue, size);
if (IS_ERR(acl))
return PTR_ERR(acl);
}
error = vfs_set_acl(idmap, dentry, acl_name, acl);
posix_acl_release(acl);
return error;
}
ssize_t do_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
const char *acl_name, void *kvalue, size_t size)
{
ssize_t error;
struct posix_acl *acl;
acl = vfs_get_acl(idmap, dentry, acl_name);
if (IS_ERR(acl))
return PTR_ERR(acl);
error = vfs_posix_acl_to_xattr(idmap, d_inode(dentry),
acl, kvalue, size);
posix_acl_release(acl);
return error;
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* Helpers for formatting and printing strings
*
* Copyright 31 August 2008 James Bottomley
* Copyright (C) 2013, Intel Corporation
*/
#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/math64.h>
#include <linux/export.h>
#include <linux/ctype.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/limits.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/string_helpers.h>
#include <kunit/test.h>
#include <kunit/test-bug.h>
/**
* string_get_size - get the size in the specified units
* @size: The size to be converted in blocks
* @blk_size: Size of the block (use 1 for size in bytes)
* @units: Units to use (powers of 1000 or 1024), whether to include space separator
* @buf: buffer to format to
* @len: length of buffer
*
* This function returns a string formatted to 3 significant figures
* giving the size in the required units. @buf should have room for
* at least 9 bytes and will always be zero terminated.
*
* Return value: number of characters of output that would have been written
* (which may be greater than len, if output was truncated).
*/
int string_get_size(u64 size, u64 blk_size, const enum string_size_units units,
char *buf, int len)
{
enum string_size_units units_base = units & STRING_UNITS_MASK;
static const char *const units_10[] = {
"", "k", "M", "G", "T", "P", "E", "Z", "Y",
};
static const char *const units_2[] = {
"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi",
};
static const char *const *const units_str[] = {
[STRING_UNITS_10] = units_10,
[STRING_UNITS_2] = units_2,
};
static const unsigned int divisor[] = {
[STRING_UNITS_10] = 1000,
[STRING_UNITS_2] = 1024,
};
static const unsigned int rounding[] = { 500, 50, 5 };
int i = 0, j;
u32 remainder = 0, sf_cap;
char tmp[8];
const char *unit;
tmp[0] = '\0';
if (blk_size == 0)
size = 0;
if (size == 0)
goto out;
/* This is Napier's algorithm. Reduce the original block size to
*
* coefficient * divisor[units_base]^i
*
* we do the reduction so both coefficients are just under 32 bits so
* that multiplying them together won't overflow 64 bits and we keep
* as much precision as possible in the numbers.
*
* Note: it's safe to throw away the remainders here because all the
* precision is in the coefficients.
*/
while (blk_size >> 32) {
do_div(blk_size, divisor[units_base]);
i++;
}
while (size >> 32) {
do_div(size, divisor[units_base]);
i++;
}
/* now perform the actual multiplication keeping i as the sum of the
* two logarithms */
size *= blk_size;
/* and logarithmically reduce it until it's just under the divisor */
while (size >= divisor[units_base]) {
remainder = do_div(size, divisor[units_base]);
i++;
}
/* work out in j how many digits of precision we need from the
* remainder */
sf_cap = size;
for (j = 0; sf_cap*10 < 1000; j++)
sf_cap *= 10;
if (units_base == STRING_UNITS_2) {
/* express the remainder as a decimal. It's currently the
* numerator of a fraction whose denominator is
* divisor[units_base], which is 1 << 10 for STRING_UNITS_2 */
remainder *= 1000;
remainder >>= 10;
}
/* add a 5 to the digit below what will be printed to ensure
* an arithmetical round up and carry it through to size */
remainder += rounding[j];
if (remainder >= 1000) {
remainder -= 1000;
size += 1;
}
if (j) {
snprintf(tmp, sizeof(tmp), ".%03u", remainder);
tmp[j+1] = '\0';
}
out:
if (i >= ARRAY_SIZE(units_2))
unit = "UNK";
else
unit = units_str[units_base][i];
return snprintf(buf, len, "%u%s%s%s%s", (u32)size, tmp,
(units & STRING_UNITS_NO_SPACE) ? "" : " ",
unit,
(units & STRING_UNITS_NO_BYTES) ? "" : "B");
}
EXPORT_SYMBOL(string_get_size);
/**
* parse_int_array_user - Split string into a sequence of integers
* @from: The user space buffer to read from
* @count: The maximum number of bytes to read
* @array: Returned pointer to sequence of integers
*
* On success @array is allocated and initialized with a sequence of
* integers extracted from the @from plus an additional element that
* begins the sequence and specifies the integers count.
*
* Caller takes responsibility for freeing @array when it is no longer
* needed.
*/
int parse_int_array_user(const char __user *from, size_t count, int **array)
{
int *ints, nints;
char *buf;
int ret = 0;
buf = memdup_user_nul(from, count);
if (IS_ERR(buf))
return PTR_ERR(buf);
get_options(buf, 0, &nints);
if (!nints) {
ret = -ENOENT;
goto free_buf;
}
ints = kcalloc(nints + 1, sizeof(*ints), GFP_KERNEL);
if (!ints) {
ret = -ENOMEM;
goto free_buf;
}
get_options(buf, nints + 1, ints);
*array = ints;
free_buf:
kfree(buf);
return ret;
}
EXPORT_SYMBOL(parse_int_array_user);
static bool unescape_space(char **src, char **dst)
{
char *p = *dst, *q = *src;
switch (*q) {
case 'n':
*p = '\n';
break;
case 'r':
*p = '\r';
break;
case 't':
*p = '\t';
break;
case 'v':
*p = '\v';
break;
case 'f':
*p = '\f';
break;
default:
return false;
}
*dst += 1;
*src += 1;
return true;
}
static bool unescape_octal(char **src, char **dst)
{
char *p = *dst, *q = *src;
u8 num;
if (isodigit(*q) == 0)
return false;
num = (*q++) & 7;
while (num < 32 && isodigit(*q) && (q - *src < 3)) {
num <<= 3;
num += (*q++) & 7;
}
*p = num;
*dst += 1;
*src = q;
return true;
}
static bool unescape_hex(char **src, char **dst)
{
char *p = *dst, *q = *src;
int digit;
u8 num;
if (*q++ != 'x')
return false;
num = digit = hex_to_bin(*q++);
if (digit < 0)
return false;
digit = hex_to_bin(*q);
if (digit >= 0) {
q++;
num = (num << 4) | digit;
}
*p = num;
*dst += 1;
*src = q;
return true;
}
static bool unescape_special(char **src, char **dst)
{
char *p = *dst, *q = *src;
switch (*q) {
case '\"':
*p = '\"';
break;
case '\\':
*p = '\\';
break;
case 'a':
*p = '\a';
break;
case 'e':
*p = '\e';
break;
default:
return false;
}
*dst += 1;
*src += 1;
return true;
}
/**
* string_unescape - unquote characters in the given string
* @src: source buffer (escaped)
* @dst: destination buffer (unescaped)
* @size: size of the destination buffer (0 to unlimit)
* @flags: combination of the flags.
*
* Description:
* The function unquotes characters in the given string.
*
* Because the size of the output will be the same as or less than the size of
* the input, the transformation may be performed in place.
*
* Caller must provide valid source and destination pointers. Be aware that
* destination buffer will always be NULL-terminated. Source string must be
* NULL-terminated as well. The supported flags are::
*
* UNESCAPE_SPACE:
* '\f' - form feed
* '\n' - new line
* '\r' - carriage return
* '\t' - horizontal tab
* '\v' - vertical tab
* UNESCAPE_OCTAL:
* '\NNN' - byte with octal value NNN (1 to 3 digits)
* UNESCAPE_HEX:
* '\xHH' - byte with hexadecimal value HH (1 to 2 digits)
* UNESCAPE_SPECIAL:
* '\"' - double quote
* '\\' - backslash
* '\a' - alert (BEL)
* '\e' - escape
* UNESCAPE_ANY:
* all previous together
*
* Return:
* The amount of the characters processed to the destination buffer excluding
* trailing '\0' is returned.
*/
int string_unescape(char *src, char *dst, size_t size, unsigned int flags)
{
char *out = dst;
while (*src && --size) {
if (src[0] == '\\' && src[1] != '\0' && size > 1) {
src++;
size--;
if (flags & UNESCAPE_SPACE &&
unescape_space(&src, &out))
continue;
if (flags & UNESCAPE_OCTAL &&
unescape_octal(&src, &out))
continue;
if (flags & UNESCAPE_HEX &&
unescape_hex(&src, &out))
continue;
if (flags & UNESCAPE_SPECIAL &&
unescape_special(&src, &out))
continue;
*out++ = '\\';
}
*out++ = *src++;
}
*out = '\0';
return out - dst;
}
EXPORT_SYMBOL(string_unescape);
static bool escape_passthrough(unsigned char c, char **dst, char *end)
{
char *out = *dst;
if (out < end)
*out = c;
*dst = out + 1;
return true;
}
static bool escape_space(unsigned char c, char **dst, char *end)
{
char *out = *dst;
unsigned char to;
switch (c) {
case '\n':
to = 'n';
break;
case '\r':
to = 'r';
break;
case '\t':
to = 't';
break;
case '\v':
to = 'v';
break;
case '\f':
to = 'f';
break;
default:
return false;
}
if (out < end)
*out = '\\';
++out;
if (out < end)
*out = to;
++out;
*dst = out;
return true;
}
static bool escape_special(unsigned char c, char **dst, char *end)
{
char *out = *dst;
unsigned char to;
switch (c) {
case '\\':
to = '\\';
break;
case '\a':
to = 'a';
break;
case '\e':
to = 'e';
break;
case '"':
to = '"';
break;
default:
return false;
}
if (out < end)
*out = '\\';
++out;
if (out < end)
*out = to;
++out;
*dst = out;
return true;
}
static bool escape_null(unsigned char c, char **dst, char *end)
{
char *out = *dst;
if (c)
return false;
if (out < end)
*out = '\\';
++out;
if (out < end)
*out = '0';
++out;
*dst = out;
return true;
}
static bool escape_octal(unsigned char c, char **dst, char *end)
{
char *out = *dst;
if (out < end)
*out = '\\';
++out;
if (out < end)
*out = ((c >> 6) & 0x07) + '0';
++out;
if (out < end)
*out = ((c >> 3) & 0x07) + '0';
++out;
if (out < end)
*out = ((c >> 0) & 0x07) + '0';
++out;
*dst = out;
return true;
}
static bool escape_hex(unsigned char c, char **dst, char *end)
{
char *out = *dst;
if (out < end)
*out = '\\';
++out;
if (out < end)
*out = 'x';
++out;
if (out < end)
*out = hex_asc_hi(c);
++out;
if (out < end)
*out = hex_asc_lo(c);
++out;
*dst = out;
return true;
}
/**
* string_escape_mem - quote characters in the given memory buffer
* @src: source buffer (unescaped)
* @isz: source buffer size
* @dst: destination buffer (escaped)
* @osz: destination buffer size
* @flags: combination of the flags
* @only: NULL-terminated string containing characters used to limit
* the selected escape class. If characters are included in @only
* that would not normally be escaped by the classes selected
* in @flags, they will be copied to @dst unescaped.
*
* Description:
* The process of escaping byte buffer includes several parts. They are applied
* in the following sequence.
*
* 1. The character is not matched to the one from @only string and thus
* must go as-is to the output.
* 2. The character is matched to the printable and ASCII classes, if asked,
* and in case of match it passes through to the output.
* 3. The character is matched to the printable or ASCII class, if asked,
* and in case of match it passes through to the output.
* 4. The character is checked if it falls into the class given by @flags.
* %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any
* character. Note that they actually can't go together, otherwise
* %ESCAPE_HEX will be ignored.
*
* Caller must provide valid source and destination pointers. Be aware that
* destination buffer will not be NULL-terminated, thus caller have to append
* it if needs. The supported flags are::
*
* %ESCAPE_SPACE: (special white space, not space itself)
* '\f' - form feed
* '\n' - new line
* '\r' - carriage return
* '\t' - horizontal tab
* '\v' - vertical tab
* %ESCAPE_SPECIAL:
* '\"' - double quote
* '\\' - backslash
* '\a' - alert (BEL)
* '\e' - escape
* %ESCAPE_NULL:
* '\0' - null
* %ESCAPE_OCTAL:
* '\NNN' - byte with octal value NNN (3 digits)
* %ESCAPE_ANY:
* all previous together
* %ESCAPE_NP:
* escape only non-printable characters, checked by isprint()
* %ESCAPE_ANY_NP:
* all previous together
* %ESCAPE_HEX:
* '\xHH' - byte with hexadecimal value HH (2 digits)
* %ESCAPE_NA:
* escape only non-ascii characters, checked by isascii()
* %ESCAPE_NAP:
* escape only non-printable or non-ascii characters
* %ESCAPE_APPEND:
* append characters from @only to be escaped by the given classes
*
* %ESCAPE_APPEND would help to pass additional characters to the escaped, when
* one of %ESCAPE_NP, %ESCAPE_NA, or %ESCAPE_NAP is provided.
*
* One notable caveat, the %ESCAPE_NAP, %ESCAPE_NP and %ESCAPE_NA have the
* higher priority than the rest of the flags (%ESCAPE_NAP is the highest).
* It doesn't make much sense to use either of them without %ESCAPE_OCTAL
* or %ESCAPE_HEX, because they cover most of the other character classes.
* %ESCAPE_NAP can utilize %ESCAPE_SPACE or %ESCAPE_SPECIAL in addition to
* the above.
*
* Return:
* The total size of the escaped output that would be generated for
* the given input and flags. To check whether the output was
* truncated, compare the return value to osz. There is room left in
* dst for a '\0' terminator if and only if ret < osz.
*/
int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz,
unsigned int flags, const char *only)
{
char *p = dst;
char *end = p + osz;
bool is_dict = only && *only;
bool is_append = flags & ESCAPE_APPEND;
while (isz--) {
unsigned char c = *src++;
bool in_dict = is_dict && strchr(only, c);
/*
* Apply rules in the following sequence:
* - the @only string is supplied and does not contain a
* character under question
* - the character is printable and ASCII, when @flags has
* %ESCAPE_NAP bit set
* - the character is printable, when @flags has
* %ESCAPE_NP bit set
* - the character is ASCII, when @flags has
* %ESCAPE_NA bit set
* - the character doesn't fall into a class of symbols
* defined by given @flags
* In these cases we just pass through a character to the
* output buffer.
*
* When %ESCAPE_APPEND is passed, the characters from @only
* have been excluded from the %ESCAPE_NAP, %ESCAPE_NP, and
* %ESCAPE_NA cases.
*/
if (!(is_append || in_dict) && is_dict &&
escape_passthrough(c, &p, end))
continue;
if (!(is_append && in_dict) && isascii(c) && isprint(c) &&
flags & ESCAPE_NAP && escape_passthrough(c, &p, end))
continue;
if (!(is_append && in_dict) && isprint(c) &&
flags & ESCAPE_NP && escape_passthrough(c, &p, end))
continue;
if (!(is_append && in_dict) && isascii(c) &&
flags & ESCAPE_NA && escape_passthrough(c, &p, end))
continue;
if (flags & ESCAPE_SPACE && escape_space(c, &p, end))
continue;
if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end))
continue;
if (flags & ESCAPE_NULL && escape_null(c, &p, end))
continue;
/* ESCAPE_OCTAL and ESCAPE_HEX always go last */
if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end))
continue;
if (flags & ESCAPE_HEX && escape_hex(c, &p, end))
continue;
escape_passthrough(c, &p, end);
}
return p - dst;
}
EXPORT_SYMBOL(string_escape_mem);
/*
* Return an allocated string that has been escaped of special characters
* and double quotes, making it safe to log in quotes.
*/
char *kstrdup_quotable(const char *src, gfp_t gfp)
{
size_t slen, dlen;
char *dst;
const int flags = ESCAPE_HEX;
const char esc[] = "\f\n\r\t\v\a\e\\\"";
if (!src)
return NULL;
slen = strlen(src);
dlen = string_escape_mem(src, slen, NULL, 0, flags, esc);
dst = kmalloc(dlen + 1, gfp);
if (!dst)
return NULL;
WARN_ON(string_escape_mem(src, slen, dst, dlen, flags, esc) != dlen);
dst[dlen] = '\0';
return dst;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable);
/*
* Returns allocated NULL-terminated string containing process
* command line, with inter-argument NULLs replaced with spaces,
* and other special characters escaped.
*/
char *kstrdup_quotable_cmdline(struct task_struct *task, gfp_t gfp)
{
char *buffer, *quoted;
int i, res;
buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!buffer)
return NULL;
res = get_cmdline(task, buffer, PAGE_SIZE - 1);
buffer[res] = '\0';
/* Collapse trailing NULLs, leave res pointing to last non-NULL. */
while (--res >= 0 && buffer[res] == '\0')
;
/* Replace inter-argument NULLs. */
for (i = 0; i <= res; i++)
if (buffer[i] == '\0')
buffer[i] = ' ';
/* Make sure result is printable. */
quoted = kstrdup_quotable(buffer, gfp);
kfree(buffer);
return quoted;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable_cmdline);
/*
* Returns allocated NULL-terminated string containing pathname,
* with special characters escaped, able to be safely logged. If
* there is an error, the leading character will be "<".
*/
char *kstrdup_quotable_file(struct file *file, gfp_t gfp)
{
char *temp, *pathname;
if (!file)
return kstrdup("<unknown>", gfp);
/* We add 11 spaces for ' (deleted)' to be appended */
temp = kmalloc(PATH_MAX + 11, GFP_KERNEL);
if (!temp)
return kstrdup("<no_memory>", gfp);
pathname = file_path(file, temp, PATH_MAX + 11);
if (IS_ERR(pathname))
pathname = kstrdup("<too_long>", gfp);
else
pathname = kstrdup_quotable(pathname, gfp);
kfree(temp);
return pathname;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable_file);
/*
* Returns duplicate string in which the @old characters are replaced by @new.
*/
char *kstrdup_and_replace(const char *src, char old, char new, gfp_t gfp)
{
char *dst;
dst = kstrdup(src, gfp);
if (!dst)
return NULL;
return strreplace(dst, old, new);
}
EXPORT_SYMBOL_GPL(kstrdup_and_replace);
/**
* kasprintf_strarray - allocate and fill array of sequential strings
* @gfp: flags for the slab allocator
* @prefix: prefix to be used
* @n: amount of lines to be allocated and filled
*
* Allocates and fills @n strings using pattern "%s-%zu", where prefix
* is provided by caller. The caller is responsible to free them with
* kfree_strarray() after use.
*
* Returns array of strings or NULL when memory can't be allocated.
*/
char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n)
{
char **names;
size_t i;
names = kcalloc(n + 1, sizeof(char *), gfp);
if (!names)
return NULL;
for (i = 0; i < n; i++) {
names[i] = kasprintf(gfp, "%s-%zu", prefix, i);
if (!names[i]) {
kfree_strarray(names, i);
return NULL;
}
}
return names;
}
EXPORT_SYMBOL_GPL(kasprintf_strarray);
/**
* kfree_strarray - free a number of dynamically allocated strings contained
* in an array and the array itself
*
* @array: Dynamically allocated array of strings to free.
* @n: Number of strings (starting from the beginning of the array) to free.
*
* Passing a non-NULL @array and @n == 0 as well as NULL @array are valid
* use-cases. If @array is NULL, the function does nothing.
*/
void kfree_strarray(char **array, size_t n)
{
unsigned int i;
if (!array)
return;
for (i = 0; i < n; i++)
kfree(array[i]);
kfree(array);
}
EXPORT_SYMBOL_GPL(kfree_strarray);
struct strarray {
char **array;
size_t n;
};
static void devm_kfree_strarray(struct device *dev, void *res)
{
struct strarray *array = res;
kfree_strarray(array->array, array->n);
}
char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n)
{
struct strarray *ptr;
ptr = devres_alloc(devm_kfree_strarray, sizeof(*ptr), GFP_KERNEL);
if (!ptr)
return ERR_PTR(-ENOMEM);
ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n);
if (!ptr->array) {
devres_free(ptr);
return ERR_PTR(-ENOMEM);
}
ptr->n = n;
devres_add(dev, ptr);
return ptr->array;
}
EXPORT_SYMBOL_GPL(devm_kasprintf_strarray);
/**
* skip_spaces - Removes leading whitespace from @str.
* @str: The string to be stripped.
*
* Returns a pointer to the first non-whitespace character in @str.
*/
char *skip_spaces(const char *str)
{
while (isspace(*str)) ++str; return (char *)str;
}
EXPORT_SYMBOL(skip_spaces);
/**
* strim - Removes leading and trailing whitespace from @s.
* @s: The string to be stripped.
*
* Note that the first trailing whitespace is replaced with a %NUL-terminator
* in the given string @s. Returns a pointer to the first non-whitespace
* character in @s.
*/
char *strim(char *s)
{
size_t size;
char *end;
size = strlen(s);
if (!size)
return s;
end = s + size - 1;
while (end >= s && isspace(*end))
end--;
*(end + 1) = '\0';
return skip_spaces(s);
}
EXPORT_SYMBOL(strim);
/**
* sysfs_streq - return true if strings are equal, modulo trailing newline
* @s1: one string
* @s2: another string
*
* This routine returns true iff two strings are equal, treating both
* NUL and newline-then-NUL as equivalent string terminations. It's
* geared for use with sysfs input strings, which generally terminate
* with newlines but are compared against values without newlines.
*/
bool sysfs_streq(const char *s1, const char *s2)
{
while (*s1 && *s1 == *s2) {
s1++;
s2++;
}
if (*s1 == *s2)
return true;
if (!*s1 && *s2 == '\n' && !s2[1])
return true;
if (*s1 == '\n' && !s1[1] && !*s2)
return true;
return false;
}
EXPORT_SYMBOL(sysfs_streq);
/**
* match_string - matches given string in an array
* @array: array of strings
* @n: number of strings in the array or -1 for NULL terminated arrays
* @string: string to match with
*
* This routine will look for a string in an array of strings up to the
* n-th element in the array or until the first NULL element.
*
* Historically the value of -1 for @n, was used to search in arrays that
* are NULL terminated. However, the function does not make a distinction
* when finishing the search: either @n elements have been compared OR
* the first NULL element was found.
*
* Return:
* index of a @string in the @array if matches, or %-EINVAL otherwise.
*/
int match_string(const char * const *array, size_t n, const char *string)
{
int index;
const char *item;
for (index = 0; index < n; index++) {
item = array[index];
if (!item)
break;
if (!strcmp(item, string))
return index;
}
return -EINVAL;
}
EXPORT_SYMBOL(match_string);
/**
* __sysfs_match_string - matches given string in an array
* @array: array of strings
* @n: number of strings in the array or -1 for NULL terminated arrays
* @str: string to match with
*
* Returns index of @str in the @array or -EINVAL, just like match_string().
* Uses sysfs_streq instead of strcmp for matching.
*
* This routine will look for a string in an array of strings up to the
* n-th element in the array or until the first NULL element.
*
* Historically the value of -1 for @n, was used to search in arrays that
* are NULL terminated. However, the function does not make a distinction
* when finishing the search: either @n elements have been compared OR
* the first NULL element was found.
*/
int __sysfs_match_string(const char * const *array, size_t n, const char *str)
{
const char *item;
int index;
for (index = 0; index < n; index++) {
item = array[index];
if (!item)
break;
if (sysfs_streq(item, str))
return index;
}
return -EINVAL;
}
EXPORT_SYMBOL(__sysfs_match_string);
/**
* strreplace - Replace all occurrences of character in string.
* @str: The string to operate on.
* @old: The character being replaced.
* @new: The character @old is replaced with.
*
* Replaces the each @old character with a @new one in the given string @str.
*
* Return: pointer to the string @str itself.
*/
char *strreplace(char *str, char old, char new)
{
char *s = str;
for (; *s; ++s)
if (*s == old)
*s = new;
return str;
}
EXPORT_SYMBOL(strreplace);
/**
* memcpy_and_pad - Copy one buffer to another with padding
* @dest: Where to copy to
* @dest_len: The destination buffer size
* @src: Where to copy from
* @count: The number of bytes to copy
* @pad: Character to use for padding if space is left in destination.
*/
void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count,
int pad)
{
if (dest_len > count) {
memcpy(dest, src, count);
memset(dest + count, pad, dest_len - count);
} else {
memcpy(dest, src, dest_len);
}
}
EXPORT_SYMBOL(memcpy_and_pad);
#ifdef CONFIG_FORTIFY_SOURCE
/* These are placeholders for fortify compile-time warnings. */
void __read_overflow2_field(size_t avail, size_t wanted) { }
EXPORT_SYMBOL(__read_overflow2_field);
void __write_overflow_field(size_t avail, size_t wanted) { }
EXPORT_SYMBOL(__write_overflow_field);
static const char * const fortify_func_name[] = {
#define MAKE_FORTIFY_FUNC_NAME(func) [MAKE_FORTIFY_FUNC(func)] = #func
EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC_NAME)
#undef MAKE_FORTIFY_FUNC_NAME
};
void __fortify_report(const u8 reason, const size_t avail, const size_t size)
{
const u8 func = FORTIFY_REASON_FUNC(reason);
const bool write = FORTIFY_REASON_DIR(reason);
const char *name;
name = fortify_func_name[umin(func, FORTIFY_FUNC_UNKNOWN)];
WARN(1, "%s: detected buffer overflow: %zu byte %s of buffer size %zu\n",
name, size, str_read_write(!write), avail);
}
EXPORT_SYMBOL(__fortify_report);
void __fortify_panic(const u8 reason, const size_t avail, const size_t size)
{
__fortify_report(reason, avail, size);
BUG();
}
EXPORT_SYMBOL(__fortify_panic);
#endif /* CONFIG_FORTIFY_SOURCE */
/*
* Copyright (c) 2016 Intel Corporation
*
* Permission to use, copy, modify, distribute, and sell this software and its
* documentation for any purpose is hereby granted without fee, provided that
* the above copyright notice appear in all copies and that both that copyright
* notice and this permission notice appear in supporting documentation, and
* that the name of the copyright holders not be used in advertising or
* publicity pertaining to distribution of the software without specific,
* written prior permission. The copyright holders make no representations
* about the suitability of this software for any purpose. It is provided "as
* is" without express or implied warranty.
*
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
* EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
* DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
* TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
* OF THIS SOFTWARE.
*/
#include <drm/drm_auth.h>
#include <drm/drm_connector.h>
#include <drm/drm_drv.h>
#include <drm/drm_edid.h>
#include <drm/drm_encoder.h>
#include <drm/drm_file.h>
#include <drm/drm_managed.h>
#include <drm/drm_panel.h>
#include <drm/drm_print.h>
#include <drm/drm_privacy_screen_consumer.h>
#include <drm/drm_sysfs.h>
#include <drm/drm_utils.h>
#include <linux/property.h>
#include <linux/uaccess.h>
#include <video/cmdline.h>
#include "drm_crtc_internal.h"
#include "drm_internal.h"
/**
* DOC: overview
*
* In DRM connectors are the general abstraction for display sinks, and include
* also fixed panels or anything else that can display pixels in some form. As
* opposed to all other KMS objects representing hardware (like CRTC, encoder or
* plane abstractions) connectors can be hotplugged and unplugged at runtime.
* Hence they are reference-counted using drm_connector_get() and
* drm_connector_put().
*
* KMS driver must create, initialize, register and attach at a &struct
* drm_connector for each such sink. The instance is created as other KMS
* objects and initialized by setting the following fields. The connector is
* initialized with a call to drm_connector_init() with a pointer to the
* &struct drm_connector_funcs and a connector type, and then exposed to
* userspace with a call to drm_connector_register().
*
* Connectors must be attached to an encoder to be used. For devices that map
* connectors to encoders 1:1, the connector should be attached at
* initialization time with a call to drm_connector_attach_encoder(). The
* driver must also set the &drm_connector.encoder field to point to the
* attached encoder.
*
* For connectors which are not fixed (like built-in panels) the driver needs to
* support hotplug notifications. The simplest way to do that is by using the
* probe helpers, see drm_kms_helper_poll_init() for connectors which don't have
* hardware support for hotplug interrupts. Connectors with hardware hotplug
* support can instead use e.g. drm_helper_hpd_irq_event().
*/
/*
* Global connector list for drm_connector_find_by_fwnode().
* Note drm_connector_[un]register() first take connector->lock and then
* take the connector_list_lock.
*/
static DEFINE_MUTEX(connector_list_lock);
static LIST_HEAD(connector_list);
struct drm_conn_prop_enum_list {
int type;
const char *name;
struct ida ida;
};
/*
* Connector and encoder types.
*/
static struct drm_conn_prop_enum_list drm_connector_enum_list[] = {
{ DRM_MODE_CONNECTOR_Unknown, "Unknown" },
{ DRM_MODE_CONNECTOR_VGA, "VGA" },
{ DRM_MODE_CONNECTOR_DVII, "DVI-I" },
{ DRM_MODE_CONNECTOR_DVID, "DVI-D" },
{ DRM_MODE_CONNECTOR_DVIA, "DVI-A" },
{ DRM_MODE_CONNECTOR_Composite, "Composite" },
{ DRM_MODE_CONNECTOR_SVIDEO, "SVIDEO" },
{ DRM_MODE_CONNECTOR_LVDS, "LVDS" },
{ DRM_MODE_CONNECTOR_Component, "Component" },
{ DRM_MODE_CONNECTOR_9PinDIN, "DIN" },
{ DRM_MODE_CONNECTOR_DisplayPort, "DP" },
{ DRM_MODE_CONNECTOR_HDMIA, "HDMI-A" },
{ DRM_MODE_CONNECTOR_HDMIB, "HDMI-B" },
{ DRM_MODE_CONNECTOR_TV, "TV" },
{ DRM_MODE_CONNECTOR_eDP, "eDP" },
{ DRM_MODE_CONNECTOR_VIRTUAL, "Virtual" },
{ DRM_MODE_CONNECTOR_DSI, "DSI" },
{ DRM_MODE_CONNECTOR_DPI, "DPI" },
{ DRM_MODE_CONNECTOR_WRITEBACK, "Writeback" },
{ DRM_MODE_CONNECTOR_SPI, "SPI" },
{ DRM_MODE_CONNECTOR_USB, "USB" },
};
void drm_connector_ida_init(void)
{
int i;
for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
ida_init(&drm_connector_enum_list[i].ida);
}
void drm_connector_ida_destroy(void)
{
int i;
for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
ida_destroy(&drm_connector_enum_list[i].ida);
}
/**
* drm_get_connector_type_name - return a string for connector type
* @type: The connector type (DRM_MODE_CONNECTOR_*)
*
* Returns: the name of the connector type, or NULL if the type is not valid.
*/
const char *drm_get_connector_type_name(unsigned int type)
{
if (type < ARRAY_SIZE(drm_connector_enum_list))
return drm_connector_enum_list[type].name;
return NULL;
}
EXPORT_SYMBOL(drm_get_connector_type_name);
/**
* drm_connector_get_cmdline_mode - reads the user's cmdline mode
* @connector: connector to query
*
* The kernel supports per-connector configuration of its consoles through
* use of the video= parameter. This function parses that option and
* extracts the user's specified mode (or enable/disable status) for a
* particular connector. This is typically only used during the early fbdev
* setup.
*/
static void drm_connector_get_cmdline_mode(struct drm_connector *connector)
{
struct drm_cmdline_mode *mode = &connector->cmdline_mode;
const char *option;
option = video_get_options(connector->name);
if (!option)
return;
if (!drm_mode_parse_command_line_for_connector(option,
connector,
mode))
return;
if (mode->force) {
DRM_INFO("forcing %s connector %s\n", connector->name,
drm_get_connector_force_name(mode->force));
connector->force = mode->force;
}
if (mode->panel_orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) {
DRM_INFO("cmdline forces connector %s panel_orientation to %d\n",
connector->name, mode->panel_orientation);
drm_connector_set_panel_orientation(connector,
mode->panel_orientation);
}
DRM_DEBUG_KMS("cmdline mode for connector %s %s %dx%d@%dHz%s%s%s\n",
connector->name, mode->name,
mode->xres, mode->yres,
mode->refresh_specified ? mode->refresh : 60,
mode->rb ? " reduced blanking" : "",
mode->margins ? " with margins" : "",
mode->interlace ? " interlaced" : "");
}
static void drm_connector_free(struct kref *kref)
{
struct drm_connector *connector =
container_of(kref, struct drm_connector, base.refcount);
struct drm_device *dev = connector->dev;
drm_mode_object_unregister(dev, &connector->base);
connector->funcs->destroy(connector);
}
void drm_connector_free_work_fn(struct work_struct *work)
{
struct drm_connector *connector, *n;
struct drm_device *dev =
container_of(work, struct drm_device, mode_config.connector_free_work);
struct drm_mode_config *config = &dev->mode_config;
unsigned long flags;
struct llist_node *freed;
spin_lock_irqsave(&config->connector_list_lock, flags);
freed = llist_del_all(&config->connector_free_list);
spin_unlock_irqrestore(&config->connector_list_lock, flags);
llist_for_each_entry_safe(connector, n, freed, free_node) {
drm_mode_object_unregister(dev, &connector->base);
connector->funcs->destroy(connector);
}
}
static int __drm_connector_init(struct drm_device *dev,
struct drm_connector *connector,
const struct drm_connector_funcs *funcs,
int connector_type,
struct i2c_adapter *ddc)
{
struct drm_mode_config *config = &dev->mode_config;
int ret;
struct ida *connector_ida =
&drm_connector_enum_list[connector_type].ida;
WARN_ON(drm_drv_uses_atomic_modeset(dev) &&
(!funcs->atomic_destroy_state ||
!funcs->atomic_duplicate_state));
ret = __drm_mode_object_add(dev, &connector->base,
DRM_MODE_OBJECT_CONNECTOR,
false, drm_connector_free);
if (ret)
return ret;
connector->base.properties = &connector->properties;
connector->dev = dev;
connector->funcs = funcs;
/* connector index is used with 32bit bitmasks */
ret = ida_alloc_max(&config->connector_ida, 31, GFP_KERNEL);
if (ret < 0) {
DRM_DEBUG_KMS("Failed to allocate %s connector index: %d\n",
drm_connector_enum_list[connector_type].name,
ret);
goto out_put;
}
connector->index = ret;
ret = 0;
connector->connector_type = connector_type;
connector->connector_type_id =
ida_alloc_min(connector_ida, 1, GFP_KERNEL);
if (connector->connector_type_id < 0) {
ret = connector->connector_type_id;
goto out_put_id;
}
connector->name =
kasprintf(GFP_KERNEL, "%s-%d",
drm_connector_enum_list[connector_type].name,
connector->connector_type_id);
if (!connector->name) {
ret = -ENOMEM;
goto out_put_type_id;
}
/* provide ddc symlink in sysfs */
connector->ddc = ddc;
INIT_LIST_HEAD(&connector->global_connector_list_entry);
INIT_LIST_HEAD(&connector->probed_modes);
INIT_LIST_HEAD(&connector->modes);
mutex_init(&connector->mutex);
mutex_init(&connector->edid_override_mutex);
connector->edid_blob_ptr = NULL;
connector->epoch_counter = 0;
connector->tile_blob_ptr = NULL;
connector->status = connector_status_unknown;
connector->display_info.panel_orientation =
DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
drm_connector_get_cmdline_mode(connector);
/* We should add connectors at the end to avoid upsetting the connector
* index too much.
*/
spin_lock_irq(&config->connector_list_lock);
list_add_tail(&connector->head, &config->connector_list);
config->num_connector++;
spin_unlock_irq(&config->connector_list_lock);
if (connector_type != DRM_MODE_CONNECTOR_VIRTUAL &&
connector_type != DRM_MODE_CONNECTOR_WRITEBACK)
drm_connector_attach_edid_property(connector);
drm_object_attach_property(&connector->base,
config->dpms_property, 0);
drm_object_attach_property(&connector->base,
config->link_status_property,
0);
drm_object_attach_property(&connector->base,
config->non_desktop_property,
0);
drm_object_attach_property(&connector->base,
config->tile_property,
0);
if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
drm_object_attach_property(&connector->base, config->prop_crtc_id, 0);
}
connector->debugfs_entry = NULL;
out_put_type_id:
if (ret)
ida_free(connector_ida, connector->connector_type_id);
out_put_id:
if (ret)
ida_free(&config->connector_ida, connector->index);
out_put:
if (ret)
drm_mode_object_unregister(dev, &connector->base);
return ret;
}
/**
* drm_connector_init - Init a preallocated connector
* @dev: DRM device
* @connector: the connector to init
* @funcs: callbacks for this connector
* @connector_type: user visible type of the connector
*
* Initialises a preallocated connector. Connectors should be
* subclassed as part of driver connector objects.
*
* At driver unload time the driver's &drm_connector_funcs.destroy hook
* should call drm_connector_cleanup() and free the connector structure.
* The connector structure should not be allocated with devm_kzalloc().
*
* Note: consider using drmm_connector_init() instead of
* drm_connector_init() to let the DRM managed resource infrastructure
* take care of cleanup and deallocation.
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_connector_init(struct drm_device *dev,
struct drm_connector *connector,
const struct drm_connector_funcs *funcs,
int connector_type)
{
if (drm_WARN_ON(dev, !(funcs && funcs->destroy)))
return -EINVAL;
return __drm_connector_init(dev, connector, funcs, connector_type, NULL);
}
EXPORT_SYMBOL(drm_connector_init);
/**
* drm_connector_init_with_ddc - Init a preallocated connector
* @dev: DRM device
* @connector: the connector to init
* @funcs: callbacks for this connector
* @connector_type: user visible type of the connector
* @ddc: pointer to the associated ddc adapter
*
* Initialises a preallocated connector. Connectors should be
* subclassed as part of driver connector objects.
*
* At driver unload time the driver's &drm_connector_funcs.destroy hook
* should call drm_connector_cleanup() and free the connector structure.
* The connector structure should not be allocated with devm_kzalloc().
*
* Ensures that the ddc field of the connector is correctly set.
*
* Note: consider using drmm_connector_init() instead of
* drm_connector_init_with_ddc() to let the DRM managed resource
* infrastructure take care of cleanup and deallocation.
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_connector_init_with_ddc(struct drm_device *dev,
struct drm_connector *connector,
const struct drm_connector_funcs *funcs,
int connector_type,
struct i2c_adapter *ddc)
{
if (drm_WARN_ON(dev, !(funcs && funcs->destroy)))
return -EINVAL;
return __drm_connector_init(dev, connector, funcs, connector_type, ddc);
}
EXPORT_SYMBOL(drm_connector_init_with_ddc);
static void drm_connector_cleanup_action(struct drm_device *dev,
void *ptr)
{
struct drm_connector *connector = ptr;
drm_connector_cleanup(connector);
}
/**
* drmm_connector_init - Init a preallocated connector
* @dev: DRM device
* @connector: the connector to init
* @funcs: callbacks for this connector
* @connector_type: user visible type of the connector
* @ddc: optional pointer to the associated ddc adapter
*
* Initialises a preallocated connector. Connectors should be
* subclassed as part of driver connector objects.
*
* Cleanup is automatically handled with a call to
* drm_connector_cleanup() in a DRM-managed action.
*
* The connector structure should be allocated with drmm_kzalloc().
*
* Returns:
* Zero on success, error code on failure.
*/
int drmm_connector_init(struct drm_device *dev,
struct drm_connector *connector,
const struct drm_connector_funcs *funcs,
int connector_type,
struct i2c_adapter *ddc)
{
int ret;
if (drm_WARN_ON(dev, funcs && funcs->destroy))
return -EINVAL;
ret = __drm_connector_init(dev, connector, funcs, connector_type, ddc);
if (ret)
return ret;
ret = drmm_add_action_or_reset(dev, drm_connector_cleanup_action,
connector);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(drmm_connector_init);
/**
* drm_connector_attach_edid_property - attach edid property.
* @connector: the connector
*
* Some connector types like DRM_MODE_CONNECTOR_VIRTUAL do not get a
* edid property attached by default. This function can be used to
* explicitly enable the edid property in these cases.
*/
void drm_connector_attach_edid_property(struct drm_connector *connector)
{
struct drm_mode_config *config = &connector->dev->mode_config;
drm_object_attach_property(&connector->base,
config->edid_property,
0);
}
EXPORT_SYMBOL(drm_connector_attach_edid_property);
/**
* drm_connector_attach_encoder - attach a connector to an encoder
* @connector: connector to attach
* @encoder: encoder to attach @connector to
*
* This function links up a connector to an encoder. Note that the routing
* restrictions between encoders and crtcs are exposed to userspace through the
* possible_clones and possible_crtcs bitmasks.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_attach_encoder(struct drm_connector *connector,
struct drm_encoder *encoder)
{
/*
* In the past, drivers have attempted to model the static association
* of connector to encoder in simple connector/encoder devices using a
* direct assignment of connector->encoder = encoder. This connection
* is a logical one and the responsibility of the core, so drivers are
* expected not to mess with this.
*
* Note that the error return should've been enough here, but a large
* majority of drivers ignores the return value, so add in a big WARN
* to get people's attention.
*/
if (WARN_ON(connector->encoder))
return -EINVAL;
connector->possible_encoders |= drm_encoder_mask(encoder);
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_encoder);
/**
* drm_connector_has_possible_encoder - check if the connector and encoder are
* associated with each other
* @connector: the connector
* @encoder: the encoder
*
* Returns:
* True if @encoder is one of the possible encoders for @connector.
*/
bool drm_connector_has_possible_encoder(struct drm_connector *connector,
struct drm_encoder *encoder)
{
return connector->possible_encoders & drm_encoder_mask(encoder);
}
EXPORT_SYMBOL(drm_connector_has_possible_encoder);
static void drm_mode_remove(struct drm_connector *connector,
struct drm_display_mode *mode)
{
list_del(&mode->head);
drm_mode_destroy(connector->dev, mode);
}
/**
* drm_connector_cleanup - cleans up an initialised connector
* @connector: connector to cleanup
*
* Cleans up the connector but doesn't free the object.
*/
void drm_connector_cleanup(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
struct drm_display_mode *mode, *t;
/* The connector should have been removed from userspace long before
* it is finally destroyed.
*/
if (WARN_ON(connector->registration_state ==
DRM_CONNECTOR_REGISTERED))
drm_connector_unregister(connector);
if (connector->privacy_screen) {
drm_privacy_screen_put(connector->privacy_screen);
connector->privacy_screen = NULL;
}
if (connector->tile_group) {
drm_mode_put_tile_group(dev, connector->tile_group);
connector->tile_group = NULL;
}
list_for_each_entry_safe(mode, t, &connector->probed_modes, head)
drm_mode_remove(connector, mode);
list_for_each_entry_safe(mode, t, &connector->modes, head)
drm_mode_remove(connector, mode);
ida_free(&drm_connector_enum_list[connector->connector_type].ida,
connector->connector_type_id);
ida_free(&dev->mode_config.connector_ida, connector->index);
kfree(connector->display_info.bus_formats);
kfree(connector->display_info.vics);
drm_mode_object_unregister(dev, &connector->base);
kfree(connector->name);
connector->name = NULL;
fwnode_handle_put(connector->fwnode);
connector->fwnode = NULL;
spin_lock_irq(&dev->mode_config.connector_list_lock);
list_del(&connector->head);
dev->mode_config.num_connector--;
spin_unlock_irq(&dev->mode_config.connector_list_lock);
WARN_ON(connector->state && !connector->funcs->atomic_destroy_state);
if (connector->state && connector->funcs->atomic_destroy_state)
connector->funcs->atomic_destroy_state(connector,
connector->state);
mutex_destroy(&connector->mutex);
memset(connector, 0, sizeof(*connector));
if (dev->registered)
drm_sysfs_hotplug_event(dev);
}
EXPORT_SYMBOL(drm_connector_cleanup);
/**
* drm_connector_register - register a connector
* @connector: the connector to register
*
* Register userspace interfaces for a connector. Only call this for connectors
* which can be hotplugged after drm_dev_register() has been called already,
* e.g. DP MST connectors. All other connectors will be registered automatically
* when calling drm_dev_register().
*
* When the connector is no longer available, callers must call
* drm_connector_unregister().
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_connector_register(struct drm_connector *connector)
{
int ret = 0;
if (!connector->dev->registered)
return 0;
mutex_lock(&connector->mutex);
if (connector->registration_state != DRM_CONNECTOR_INITIALIZING)
goto unlock;
ret = drm_sysfs_connector_add(connector);
if (ret)
goto unlock;
drm_debugfs_connector_add(connector);
if (connector->funcs->late_register) {
ret = connector->funcs->late_register(connector);
if (ret)
goto err_debugfs;
}
ret = drm_sysfs_connector_add_late(connector);
if (ret)
goto err_late_register;
drm_mode_object_register(connector->dev, &connector->base);
connector->registration_state = DRM_CONNECTOR_REGISTERED;
/* Let userspace know we have a new connector */
drm_sysfs_connector_hotplug_event(connector);
if (connector->privacy_screen)
drm_privacy_screen_register_notifier(connector->privacy_screen,
&connector->privacy_screen_notifier);
mutex_lock(&connector_list_lock);
list_add_tail(&connector->global_connector_list_entry, &connector_list);
mutex_unlock(&connector_list_lock);
goto unlock;
err_late_register:
if (connector->funcs->early_unregister)
connector->funcs->early_unregister(connector);
err_debugfs:
drm_debugfs_connector_remove(connector);
drm_sysfs_connector_remove(connector);
unlock:
mutex_unlock(&connector->mutex);
return ret;
}
EXPORT_SYMBOL(drm_connector_register);
/**
* drm_connector_unregister - unregister a connector
* @connector: the connector to unregister
*
* Unregister userspace interfaces for a connector. Only call this for
* connectors which have been registered explicitly by calling
* drm_connector_register().
*/
void drm_connector_unregister(struct drm_connector *connector)
{
mutex_lock(&connector->mutex);
if (connector->registration_state != DRM_CONNECTOR_REGISTERED) {
mutex_unlock(&connector->mutex);
return;
}
mutex_lock(&connector_list_lock); list_del_init(&connector->global_connector_list_entry);
mutex_unlock(&connector_list_lock);
if (connector->privacy_screen)
drm_privacy_screen_unregister_notifier(
connector->privacy_screen,
&connector->privacy_screen_notifier);
drm_sysfs_connector_remove_early(connector);
if (connector->funcs->early_unregister)
connector->funcs->early_unregister(connector); drm_debugfs_connector_remove(connector);
drm_sysfs_connector_remove(connector);
connector->registration_state = DRM_CONNECTOR_UNREGISTERED;
mutex_unlock(&connector->mutex);
}
EXPORT_SYMBOL(drm_connector_unregister);
void drm_connector_unregister_all(struct drm_device *dev)
{
struct drm_connector *connector;
struct drm_connector_list_iter conn_iter;
drm_connector_list_iter_begin(dev, &conn_iter); drm_for_each_connector_iter(connector, &conn_iter) drm_connector_unregister(connector); drm_connector_list_iter_end(&conn_iter);
}
int drm_connector_register_all(struct drm_device *dev)
{
struct drm_connector *connector;
struct drm_connector_list_iter conn_iter;
int ret = 0;
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) {
ret = drm_connector_register(connector);
if (ret)
break;
}
drm_connector_list_iter_end(&conn_iter);
if (ret)
drm_connector_unregister_all(dev);
return ret;
}
/**
* drm_get_connector_status_name - return a string for connector status
* @status: connector status to compute name of
*
* In contrast to the other drm_get_*_name functions this one here returns a
* const pointer and hence is threadsafe.
*
* Returns: connector status string
*/
const char *drm_get_connector_status_name(enum drm_connector_status status)
{
if (status == connector_status_connected)
return "connected";
else if (status == connector_status_disconnected)
return "disconnected";
else
return "unknown";
}
EXPORT_SYMBOL(drm_get_connector_status_name);
/**
* drm_get_connector_force_name - return a string for connector force
* @force: connector force to get name of
*
* Returns: const pointer to name.
*/
const char *drm_get_connector_force_name(enum drm_connector_force force)
{
switch (force) {
case DRM_FORCE_UNSPECIFIED:
return "unspecified";
case DRM_FORCE_OFF:
return "off";
case DRM_FORCE_ON:
return "on";
case DRM_FORCE_ON_DIGITAL:
return "digital";
default:
return "unknown";
}
}
#ifdef CONFIG_LOCKDEP
static struct lockdep_map connector_list_iter_dep_map = {
.name = "drm_connector_list_iter"
};
#endif
/**
* drm_connector_list_iter_begin - initialize a connector_list iterator
* @dev: DRM device
* @iter: connector_list iterator
*
* Sets @iter up to walk the &drm_mode_config.connector_list of @dev. @iter
* must always be cleaned up again by calling drm_connector_list_iter_end().
* Iteration itself happens using drm_connector_list_iter_next() or
* drm_for_each_connector_iter().
*/
void drm_connector_list_iter_begin(struct drm_device *dev,
struct drm_connector_list_iter *iter)
{
iter->dev = dev;
iter->conn = NULL;
lock_acquire_shared_recursive(&connector_list_iter_dep_map, 0, 1, NULL, _RET_IP_);
}
EXPORT_SYMBOL(drm_connector_list_iter_begin);
/*
* Extra-safe connector put function that works in any context. Should only be
* used from the connector_iter functions, where we never really expect to
* actually release the connector when dropping our final reference.
*/
static void
__drm_connector_put_safe(struct drm_connector *conn)
{
struct drm_mode_config *config = &conn->dev->mode_config; lockdep_assert_held(&config->connector_list_lock); if (!refcount_dec_and_test(&conn->base.refcount.refcount))
return;
llist_add(&conn->free_node, &config->connector_free_list);
schedule_work(&config->connector_free_work);
}
/**
* drm_connector_list_iter_next - return next connector
* @iter: connector_list iterator
*
* Returns: the next connector for @iter, or NULL when the list walk has
* completed.
*/
struct drm_connector *
drm_connector_list_iter_next(struct drm_connector_list_iter *iter)
{
struct drm_connector *old_conn = iter->conn;
struct drm_mode_config *config = &iter->dev->mode_config;
struct list_head *lhead;
unsigned long flags;
spin_lock_irqsave(&config->connector_list_lock, flags);
lhead = old_conn ? &old_conn->head : &config->connector_list;
do {
if (lhead->next == &config->connector_list) { iter->conn = NULL;
break;
}
lhead = lhead->next;
iter->conn = list_entry(lhead, struct drm_connector, head);
/* loop until it's not a zombie connector */
} while (!kref_get_unless_zero(&iter->conn->base.refcount)); if (old_conn) __drm_connector_put_safe(old_conn); spin_unlock_irqrestore(&config->connector_list_lock, flags);
return iter->conn;
}
EXPORT_SYMBOL(drm_connector_list_iter_next);
/**
* drm_connector_list_iter_end - tear down a connector_list iterator
* @iter: connector_list iterator
*
* Tears down @iter and releases any resources (like &drm_connector references)
* acquired while walking the list. This must always be called, both when the
* iteration completes fully or when it was aborted without walking the entire
* list.
*/
void drm_connector_list_iter_end(struct drm_connector_list_iter *iter)
{
struct drm_mode_config *config = &iter->dev->mode_config;
unsigned long flags;
iter->dev = NULL;
if (iter->conn) {
spin_lock_irqsave(&config->connector_list_lock, flags);
__drm_connector_put_safe(iter->conn);
spin_unlock_irqrestore(&config->connector_list_lock, flags);
}
lock_release(&connector_list_iter_dep_map, _RET_IP_);
}
EXPORT_SYMBOL(drm_connector_list_iter_end);
static const struct drm_prop_enum_list drm_subpixel_enum_list[] = {
{ SubPixelUnknown, "Unknown" },
{ SubPixelHorizontalRGB, "Horizontal RGB" },
{ SubPixelHorizontalBGR, "Horizontal BGR" },
{ SubPixelVerticalRGB, "Vertical RGB" },
{ SubPixelVerticalBGR, "Vertical BGR" },
{ SubPixelNone, "None" },
};
/**
* drm_get_subpixel_order_name - return a string for a given subpixel enum
* @order: enum of subpixel_order
*
* Note you could abuse this and return something out of bounds, but that
* would be a caller error. No unscrubbed user data should make it here.
*
* Returns: string describing an enumerated subpixel property
*/
const char *drm_get_subpixel_order_name(enum subpixel_order order)
{
return drm_subpixel_enum_list[order].name;
}
EXPORT_SYMBOL(drm_get_subpixel_order_name);
static const struct drm_prop_enum_list drm_dpms_enum_list[] = {
{ DRM_MODE_DPMS_ON, "On" },
{ DRM_MODE_DPMS_STANDBY, "Standby" },
{ DRM_MODE_DPMS_SUSPEND, "Suspend" },
{ DRM_MODE_DPMS_OFF, "Off" }
};
DRM_ENUM_NAME_FN(drm_get_dpms_name, drm_dpms_enum_list)
static const struct drm_prop_enum_list drm_link_status_enum_list[] = {
{ DRM_MODE_LINK_STATUS_GOOD, "Good" },
{ DRM_MODE_LINK_STATUS_BAD, "Bad" },
};
/**
* drm_display_info_set_bus_formats - set the supported bus formats
* @info: display info to store bus formats in
* @formats: array containing the supported bus formats
* @num_formats: the number of entries in the fmts array
*
* Store the supported bus formats in display info structure.
* See MEDIA_BUS_FMT_* definitions in include/uapi/linux/media-bus-format.h for
* a full list of available formats.
*
* Returns:
* 0 on success or a negative error code on failure.
*/
int drm_display_info_set_bus_formats(struct drm_display_info *info,
const u32 *formats,
unsigned int num_formats)
{
u32 *fmts = NULL;
if (!formats && num_formats)
return -EINVAL;
if (formats && num_formats) {
fmts = kmemdup(formats, sizeof(*formats) * num_formats,
GFP_KERNEL);
if (!fmts)
return -ENOMEM;
}
kfree(info->bus_formats);
info->bus_formats = fmts;
info->num_bus_formats = num_formats;
return 0;
}
EXPORT_SYMBOL(drm_display_info_set_bus_formats);
/* Optional connector properties. */
static const struct drm_prop_enum_list drm_scaling_mode_enum_list[] = {
{ DRM_MODE_SCALE_NONE, "None" },
{ DRM_MODE_SCALE_FULLSCREEN, "Full" },
{ DRM_MODE_SCALE_CENTER, "Center" },
{ DRM_MODE_SCALE_ASPECT, "Full aspect" },
};
static const struct drm_prop_enum_list drm_aspect_ratio_enum_list[] = {
{ DRM_MODE_PICTURE_ASPECT_NONE, "Automatic" },
{ DRM_MODE_PICTURE_ASPECT_4_3, "4:3" },
{ DRM_MODE_PICTURE_ASPECT_16_9, "16:9" },
};
static const struct drm_prop_enum_list drm_content_type_enum_list[] = {
{ DRM_MODE_CONTENT_TYPE_NO_DATA, "No Data" },
{ DRM_MODE_CONTENT_TYPE_GRAPHICS, "Graphics" },
{ DRM_MODE_CONTENT_TYPE_PHOTO, "Photo" },
{ DRM_MODE_CONTENT_TYPE_CINEMA, "Cinema" },
{ DRM_MODE_CONTENT_TYPE_GAME, "Game" },
};
static const struct drm_prop_enum_list drm_panel_orientation_enum_list[] = {
{ DRM_MODE_PANEL_ORIENTATION_NORMAL, "Normal" },
{ DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP, "Upside Down" },
{ DRM_MODE_PANEL_ORIENTATION_LEFT_UP, "Left Side Up" },
{ DRM_MODE_PANEL_ORIENTATION_RIGHT_UP, "Right Side Up" },
};
static const struct drm_prop_enum_list drm_dvi_i_select_enum_list[] = {
{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
{ DRM_MODE_SUBCONNECTOR_DVID, "DVI-D" }, /* DVI-I */
{ DRM_MODE_SUBCONNECTOR_DVIA, "DVI-A" }, /* DVI-I */
};
DRM_ENUM_NAME_FN(drm_get_dvi_i_select_name, drm_dvi_i_select_enum_list)
static const struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] = {
{ DRM_MODE_SUBCONNECTOR_Unknown, "Unknown" }, /* DVI-I, TV-out and DP */
{ DRM_MODE_SUBCONNECTOR_DVID, "DVI-D" }, /* DVI-I */
{ DRM_MODE_SUBCONNECTOR_DVIA, "DVI-A" }, /* DVI-I */
};
DRM_ENUM_NAME_FN(drm_get_dvi_i_subconnector_name,
drm_dvi_i_subconnector_enum_list)
static const struct drm_prop_enum_list drm_tv_mode_enum_list[] = {
{ DRM_MODE_TV_MODE_NTSC, "NTSC" },
{ DRM_MODE_TV_MODE_NTSC_443, "NTSC-443" },
{ DRM_MODE_TV_MODE_NTSC_J, "NTSC-J" },
{ DRM_MODE_TV_MODE_PAL, "PAL" },
{ DRM_MODE_TV_MODE_PAL_M, "PAL-M" },
{ DRM_MODE_TV_MODE_PAL_N, "PAL-N" },
{ DRM_MODE_TV_MODE_SECAM, "SECAM" },
};
DRM_ENUM_NAME_FN(drm_get_tv_mode_name, drm_tv_mode_enum_list)
/**
* drm_get_tv_mode_from_name - Translates a TV mode name into its enum value
* @name: TV Mode name we want to convert
* @len: Length of @name
*
* Translates @name into an enum drm_connector_tv_mode.
*
* Returns: the enum value on success, a negative errno otherwise.
*/
int drm_get_tv_mode_from_name(const char *name, size_t len)
{
unsigned int i;
for (i = 0; i < ARRAY_SIZE(drm_tv_mode_enum_list); i++) {
const struct drm_prop_enum_list *item = &drm_tv_mode_enum_list[i];
if (strlen(item->name) == len && !strncmp(item->name, name, len))
return item->type;
}
return -EINVAL;
}
EXPORT_SYMBOL(drm_get_tv_mode_from_name);
static const struct drm_prop_enum_list drm_tv_select_enum_list[] = {
{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
{ DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
{ DRM_MODE_SUBCONNECTOR_SVIDEO, "SVIDEO" }, /* TV-out */
{ DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
{ DRM_MODE_SUBCONNECTOR_SCART, "SCART" }, /* TV-out */
};
DRM_ENUM_NAME_FN(drm_get_tv_select_name, drm_tv_select_enum_list)
static const struct drm_prop_enum_list drm_tv_subconnector_enum_list[] = {
{ DRM_MODE_SUBCONNECTOR_Unknown, "Unknown" }, /* DVI-I, TV-out and DP */
{ DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
{ DRM_MODE_SUBCONNECTOR_SVIDEO, "SVIDEO" }, /* TV-out */
{ DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
{ DRM_MODE_SUBCONNECTOR_SCART, "SCART" }, /* TV-out */
};
DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
drm_tv_subconnector_enum_list)
static const struct drm_prop_enum_list drm_dp_subconnector_enum_list[] = {
{ DRM_MODE_SUBCONNECTOR_Unknown, "Unknown" }, /* DVI-I, TV-out and DP */
{ DRM_MODE_SUBCONNECTOR_VGA, "VGA" }, /* DP */
{ DRM_MODE_SUBCONNECTOR_DVID, "DVI-D" }, /* DP */
{ DRM_MODE_SUBCONNECTOR_HDMIA, "HDMI" }, /* DP */
{ DRM_MODE_SUBCONNECTOR_DisplayPort, "DP" }, /* DP */
{ DRM_MODE_SUBCONNECTOR_Wireless, "Wireless" }, /* DP */
{ DRM_MODE_SUBCONNECTOR_Native, "Native" }, /* DP */
};
DRM_ENUM_NAME_FN(drm_get_dp_subconnector_name,
drm_dp_subconnector_enum_list)
static const char * const colorspace_names[] = {
/* For Default case, driver will set the colorspace */
[DRM_MODE_COLORIMETRY_DEFAULT] = "Default",
/* Standard Definition Colorimetry based on CEA 861 */
[DRM_MODE_COLORIMETRY_SMPTE_170M_YCC] = "SMPTE_170M_YCC",
[DRM_MODE_COLORIMETRY_BT709_YCC] = "BT709_YCC",
/* Standard Definition Colorimetry based on IEC 61966-2-4 */
[DRM_MODE_COLORIMETRY_XVYCC_601] = "XVYCC_601",
/* High Definition Colorimetry based on IEC 61966-2-4 */
[DRM_MODE_COLORIMETRY_XVYCC_709] = "XVYCC_709",
/* Colorimetry based on IEC 61966-2-1/Amendment 1 */
[DRM_MODE_COLORIMETRY_SYCC_601] = "SYCC_601",
/* Colorimetry based on IEC 61966-2-5 [33] */
[DRM_MODE_COLORIMETRY_OPYCC_601] = "opYCC_601",
/* Colorimetry based on IEC 61966-2-5 */
[DRM_MODE_COLORIMETRY_OPRGB] = "opRGB",
/* Colorimetry based on ITU-R BT.2020 */
[DRM_MODE_COLORIMETRY_BT2020_CYCC] = "BT2020_CYCC",
/* Colorimetry based on ITU-R BT.2020 */
[DRM_MODE_COLORIMETRY_BT2020_RGB] = "BT2020_RGB",
/* Colorimetry based on ITU-R BT.2020 */
[DRM_MODE_COLORIMETRY_BT2020_YCC] = "BT2020_YCC",
/* Added as part of Additional Colorimetry Extension in 861.G */
[DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65] = "DCI-P3_RGB_D65",
[DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER] = "DCI-P3_RGB_Theater",
[DRM_MODE_COLORIMETRY_RGB_WIDE_FIXED] = "RGB_WIDE_FIXED",
/* Colorimetry based on scRGB (IEC 61966-2-2) */
[DRM_MODE_COLORIMETRY_RGB_WIDE_FLOAT] = "RGB_WIDE_FLOAT",
[DRM_MODE_COLORIMETRY_BT601_YCC] = "BT601_YCC",
};
/**
* drm_get_colorspace_name - return a string for color encoding
* @colorspace: color space to compute name of
*
* In contrast to the other drm_get_*_name functions this one here returns a
* const pointer and hence is threadsafe.
*/
const char *drm_get_colorspace_name(enum drm_colorspace colorspace)
{
if (colorspace < ARRAY_SIZE(colorspace_names) && colorspace_names[colorspace])
return colorspace_names[colorspace];
else
return "(null)";
}
static const u32 hdmi_colorspaces =
BIT(DRM_MODE_COLORIMETRY_SMPTE_170M_YCC) |
BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
BIT(DRM_MODE_COLORIMETRY_XVYCC_601) |
BIT(DRM_MODE_COLORIMETRY_XVYCC_709) |
BIT(DRM_MODE_COLORIMETRY_SYCC_601) |
BIT(DRM_MODE_COLORIMETRY_OPYCC_601) |
BIT(DRM_MODE_COLORIMETRY_OPRGB) |
BIT(DRM_MODE_COLORIMETRY_BT2020_CYCC) |
BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) |
BIT(DRM_MODE_COLORIMETRY_BT2020_YCC) |
BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65) |
BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER);
/*
* As per DP 1.4a spec, 2.2.5.7.5 VSC SDP Payload for Pixel Encoding/Colorimetry
* Format Table 2-120
*/
static const u32 dp_colorspaces =
BIT(DRM_MODE_COLORIMETRY_RGB_WIDE_FIXED) |
BIT(DRM_MODE_COLORIMETRY_RGB_WIDE_FLOAT) |
BIT(DRM_MODE_COLORIMETRY_OPRGB) |
BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65) |
BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) |
BIT(DRM_MODE_COLORIMETRY_BT601_YCC) |
BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
BIT(DRM_MODE_COLORIMETRY_XVYCC_601) |
BIT(DRM_MODE_COLORIMETRY_XVYCC_709) |
BIT(DRM_MODE_COLORIMETRY_SYCC_601) |
BIT(DRM_MODE_COLORIMETRY_OPYCC_601) |
BIT(DRM_MODE_COLORIMETRY_BT2020_CYCC) |
BIT(DRM_MODE_COLORIMETRY_BT2020_YCC);
/**
* DOC: standard connector properties
*
* DRM connectors have a few standardized properties:
*
* EDID:
* Blob property which contains the current EDID read from the sink. This
* is useful to parse sink identification information like vendor, model
* and serial. Drivers should update this property by calling
* drm_connector_update_edid_property(), usually after having parsed
* the EDID using drm_add_edid_modes(). Userspace cannot change this
* property.
*
* User-space should not parse the EDID to obtain information exposed via
* other KMS properties (because the kernel might apply limits, quirks or
* fixups to the EDID). For instance, user-space should not try to parse
* mode lists from the EDID.
* DPMS:
* Legacy property for setting the power state of the connector. For atomic
* drivers this is only provided for backwards compatibility with existing
* drivers, it remaps to controlling the "ACTIVE" property on the CRTC the
* connector is linked to. Drivers should never set this property directly,
* it is handled by the DRM core by calling the &drm_connector_funcs.dpms
* callback. For atomic drivers the remapping to the "ACTIVE" property is
* implemented in the DRM core.
*
* Note that this property cannot be set through the MODE_ATOMIC ioctl,
* userspace must use "ACTIVE" on the CRTC instead.
*
* WARNING:
*
* For userspace also running on legacy drivers the "DPMS" semantics are a
* lot more complicated. First, userspace cannot rely on the "DPMS" value
* returned by the GETCONNECTOR actually reflecting reality, because many
* drivers fail to update it. For atomic drivers this is taken care of in
* drm_atomic_helper_update_legacy_modeset_state().
*
* The second issue is that the DPMS state is only well-defined when the
* connector is connected to a CRTC. In atomic the DRM core enforces that
* "ACTIVE" is off in such a case, no such checks exists for "DPMS".
*
* Finally, when enabling an output using the legacy SETCONFIG ioctl then
* "DPMS" is forced to ON. But see above, that might not be reflected in
* the software value on legacy drivers.
*
* Summarizing: Only set "DPMS" when the connector is known to be enabled,
* assume that a successful SETCONFIG call also sets "DPMS" to on, and
* never read back the value of "DPMS" because it can be incorrect.
* PATH:
* Connector path property to identify how this sink is physically
* connected. Used by DP MST. This should be set by calling
* drm_connector_set_path_property(), in the case of DP MST with the
* path property the MST manager created. Userspace cannot change this
* property.
*
* In the case of DP MST, the property has the format
* ``mst:<parent>-<ports>`` where ``<parent>`` is the KMS object ID of the
* parent connector and ``<ports>`` is a hyphen-separated list of DP MST
* port numbers. Note, KMS object IDs are not guaranteed to be stable
* across reboots.
* TILE:
* Connector tile group property to indicate how a set of DRM connector
* compose together into one logical screen. This is used by both high-res
* external screens (often only using a single cable, but exposing multiple
* DP MST sinks), or high-res integrated panels (like dual-link DSI) which
* are not gen-locked. Note that for tiled panels which are genlocked, like
* dual-link LVDS or dual-link DSI, the driver should try to not expose the
* tiling and virtualise both &drm_crtc and &drm_plane if needed. Drivers
* should update this value using drm_connector_set_tile_property().
* Userspace cannot change this property.
* link-status:
* Connector link-status property to indicate the status of link. The
* default value of link-status is "GOOD". If something fails during or
* after modeset, the kernel driver may set this to "BAD" and issue a
* hotplug uevent. Drivers should update this value using
* drm_connector_set_link_status_property().
*
* When user-space receives the hotplug uevent and detects a "BAD"
* link-status, the sink doesn't receive pixels anymore (e.g. the screen
* becomes completely black). The list of available modes may have
* changed. User-space is expected to pick a new mode if the current one
* has disappeared and perform a new modeset with link-status set to
* "GOOD" to re-enable the connector.
*
* If multiple connectors share the same CRTC and one of them gets a "BAD"
* link-status, the other are unaffected (ie. the sinks still continue to
* receive pixels).
*
* When user-space performs an atomic commit on a connector with a "BAD"
* link-status without resetting the property to "GOOD", the sink may
* still not receive pixels. When user-space performs an atomic commit
* which resets the link-status property to "GOOD" without the
* ALLOW_MODESET flag set, it might fail because a modeset is required.
*
* User-space can only change link-status to "GOOD", changing it to "BAD"
* is a no-op.
*
* For backwards compatibility with non-atomic userspace the kernel
* tries to automatically set the link-status back to "GOOD" in the
* SETCRTC IOCTL. This might fail if the mode is no longer valid, similar
* to how it might fail if a different screen has been connected in the
* interim.
* non_desktop:
* Indicates the output should be ignored for purposes of displaying a
* standard desktop environment or console. This is most likely because
* the output device is not rectilinear.
* Content Protection:
* This property is used by userspace to request the kernel protect future
* content communicated over the link. When requested, kernel will apply
* the appropriate means of protection (most often HDCP), and use the
* property to tell userspace the protection is active.
*
* Drivers can set this up by calling
* drm_connector_attach_content_protection_property() on initialization.
*
* The value of this property can be one of the following:
*
* DRM_MODE_CONTENT_PROTECTION_UNDESIRED = 0
* The link is not protected, content is transmitted in the clear.
* DRM_MODE_CONTENT_PROTECTION_DESIRED = 1
* Userspace has requested content protection, but the link is not
* currently protected. When in this state, kernel should enable
* Content Protection as soon as possible.
* DRM_MODE_CONTENT_PROTECTION_ENABLED = 2
* Userspace has requested content protection, and the link is
* protected. Only the driver can set the property to this value.
* If userspace attempts to set to ENABLED, kernel will return
* -EINVAL.
*
* A few guidelines:
*
* - DESIRED state should be preserved until userspace de-asserts it by
* setting the property to UNDESIRED. This means ENABLED should only
* transition to UNDESIRED when the user explicitly requests it.
* - If the state is DESIRED, kernel should attempt to re-authenticate the
* link whenever possible. This includes across disable/enable, dpms,
* hotplug, downstream device changes, link status failures, etc..
* - Kernel sends uevent with the connector id and property id through
* @drm_hdcp_update_content_protection, upon below kernel triggered
* scenarios:
*
* - DESIRED -> ENABLED (authentication success)
* - ENABLED -> DESIRED (termination of authentication)
* - Please note no uevents for userspace triggered property state changes,
* which can't fail such as
*
* - DESIRED/ENABLED -> UNDESIRED
* - UNDESIRED -> DESIRED
* - Userspace is responsible for polling the property or listen to uevents
* to determine when the value transitions from ENABLED to DESIRED.
* This signifies the link is no longer protected and userspace should
* take appropriate action (whatever that might be).
*
* HDCP Content Type:
* This Enum property is used by the userspace to declare the content type
* of the display stream, to kernel. Here display stream stands for any
* display content that userspace intended to display through HDCP
* encryption.
*
* Content Type of a stream is decided by the owner of the stream, as
* "HDCP Type0" or "HDCP Type1".
*
* The value of the property can be one of the below:
* - "HDCP Type0": DRM_MODE_HDCP_CONTENT_TYPE0 = 0
* - "HDCP Type1": DRM_MODE_HDCP_CONTENT_TYPE1 = 1
*
* When kernel starts the HDCP authentication (see "Content Protection"
* for details), it uses the content type in "HDCP Content Type"
* for performing the HDCP authentication with the display sink.
*
* Please note in HDCP spec versions, a link can be authenticated with
* HDCP 2.2 for Content Type 0/Content Type 1. Where as a link can be
* authenticated with HDCP1.4 only for Content Type 0(though it is implicit
* in nature. As there is no reference for Content Type in HDCP1.4).
*
* HDCP2.2 authentication protocol itself takes the "Content Type" as a
* parameter, which is a input for the DP HDCP2.2 encryption algo.
*
* In case of Type 0 content protection request, kernel driver can choose
* either of HDCP spec versions 1.4 and 2.2. When HDCP2.2 is used for
* "HDCP Type 0", a HDCP 2.2 capable repeater in the downstream can send
* that content to a HDCP 1.4 authenticated HDCP sink (Type0 link).
* But if the content is classified as "HDCP Type 1", above mentioned
* HDCP 2.2 repeater wont send the content to the HDCP sink as it can't
* authenticate the HDCP1.4 capable sink for "HDCP Type 1".
*
* Please note userspace can be ignorant of the HDCP versions used by the
* kernel driver to achieve the "HDCP Content Type".
*
* At current scenario, classifying a content as Type 1 ensures that the
* content will be displayed only through the HDCP2.2 encrypted link.
*
* Note that the HDCP Content Type property is introduced at HDCP 2.2, and
* defaults to type 0. It is only exposed by drivers supporting HDCP 2.2
* (hence supporting Type 0 and Type 1). Based on how next versions of
* HDCP specs are defined content Type could be used for higher versions
* too.
*
* If content type is changed when "Content Protection" is not UNDESIRED,
* then kernel will disable the HDCP and re-enable with new type in the
* same atomic commit. And when "Content Protection" is ENABLED, it means
* that link is HDCP authenticated and encrypted, for the transmission of
* the Type of stream mentioned at "HDCP Content Type".
*
* HDR_OUTPUT_METADATA:
* Connector property to enable userspace to send HDR Metadata to
* driver. This metadata is based on the composition and blending
* policies decided by user, taking into account the hardware and
* sink capabilities. The driver gets this metadata and creates a
* Dynamic Range and Mastering Infoframe (DRM) in case of HDMI,
* SDP packet (Non-audio INFOFRAME SDP v1.3) for DP. This is then
* sent to sink. This notifies the sink of the upcoming frame's Color
* Encoding and Luminance parameters.
*
* Userspace first need to detect the HDR capabilities of sink by
* reading and parsing the EDID. Details of HDR metadata for HDMI
* are added in CTA 861.G spec. For DP , its defined in VESA DP
* Standard v1.4. It needs to then get the metadata information
* of the video/game/app content which are encoded in HDR (basically
* using HDR transfer functions). With this information it needs to
* decide on a blending policy and compose the relevant
* layers/overlays into a common format. Once this blending is done,
* userspace will be aware of the metadata of the composed frame to
* be send to sink. It then uses this property to communicate this
* metadata to driver which then make a Infoframe packet and sends
* to sink based on the type of encoder connected.
*
* Userspace will be responsible to do Tone mapping operation in case:
* - Some layers are HDR and others are SDR
* - HDR layers luminance is not same as sink
*
* It will even need to do colorspace conversion and get all layers
* to one common colorspace for blending. It can use either GL, Media
* or display engine to get this done based on the capabilities of the
* associated hardware.
*
* Driver expects metadata to be put in &struct hdr_output_metadata
* structure from userspace. This is received as blob and stored in
* &drm_connector_state.hdr_output_metadata. It parses EDID and saves the
* sink metadata in &struct hdr_sink_metadata, as
* &drm_connector.hdr_sink_metadata. Driver uses
* drm_hdmi_infoframe_set_hdr_metadata() helper to set the HDR metadata,
* hdmi_drm_infoframe_pack() to pack the infoframe as per spec, in case of
* HDMI encoder.
*
* max bpc:
* This range property is used by userspace to limit the bit depth. When
* used the driver would limit the bpc in accordance with the valid range
* supported by the hardware and sink. Drivers to use the function
* drm_connector_attach_max_bpc_property() to create and attach the
* property to the connector during initialization.
*
* Connectors also have one standardized atomic property:
*
* CRTC_ID:
* Mode object ID of the &drm_crtc this connector should be connected to.
*
* Connectors for LCD panels may also have one standardized property:
*
* panel orientation:
* On some devices the LCD panel is mounted in the casing in such a way
* that the up/top side of the panel does not match with the top side of
* the device. Userspace can use this property to check for this.
* Note that input coordinates from touchscreens (input devices with
* INPUT_PROP_DIRECT) will still map 1:1 to the actual LCD panel
* coordinates, so if userspace rotates the picture to adjust for
* the orientation it must also apply the same transformation to the
* touchscreen input coordinates. This property is initialized by calling
* drm_connector_set_panel_orientation() or
* drm_connector_set_panel_orientation_with_quirk()
*
* scaling mode:
* This property defines how a non-native mode is upscaled to the native
* mode of an LCD panel:
*
* None:
* No upscaling happens, scaling is left to the panel. Not all
* drivers expose this mode.
* Full:
* The output is upscaled to the full resolution of the panel,
* ignoring the aspect ratio.
* Center:
* No upscaling happens, the output is centered within the native
* resolution the panel.
* Full aspect:
* The output is upscaled to maximize either the width or height
* while retaining the aspect ratio.
*
* This property should be set up by calling
* drm_connector_attach_scaling_mode_property(). Note that drivers
* can also expose this property to external outputs, in which case they
* must support "None", which should be the default (since external screens
* have a built-in scaler).
*
* subconnector:
* This property is used by DVI-I, TVout and DisplayPort to indicate different
* connector subtypes. Enum values more or less match with those from main
* connector types.
* For DVI-I and TVout there is also a matching property "select subconnector"
* allowing to switch between signal types.
* DP subconnector corresponds to a downstream port.
*
* privacy-screen sw-state, privacy-screen hw-state:
* These 2 optional properties can be used to query the state of the
* electronic privacy screen that is available on some displays; and in
* some cases also control the state. If a driver implements these
* properties then both properties must be present.
*
* "privacy-screen hw-state" is read-only and reflects the actual state
* of the privacy-screen, possible values: "Enabled", "Disabled,
* "Enabled-locked", "Disabled-locked". The locked states indicate
* that the state cannot be changed through the DRM API. E.g. there
* might be devices where the firmware-setup options, or a hardware
* slider-switch, offer always on / off modes.
*
* "privacy-screen sw-state" can be set to change the privacy-screen state
* when not locked. In this case the driver must update the hw-state
* property to reflect the new state on completion of the commit of the
* sw-state property. Setting the sw-state property when the hw-state is
* locked must be interpreted by the driver as a request to change the
* state to the set state when the hw-state becomes unlocked. E.g. if
* "privacy-screen hw-state" is "Enabled-locked" and the sw-state
* gets set to "Disabled" followed by the user unlocking the state by
* changing the slider-switch position, then the driver must set the
* state to "Disabled" upon receiving the unlock event.
*
* In some cases the privacy-screen's actual state might change outside of
* control of the DRM code. E.g. there might be a firmware handled hotkey
* which toggles the actual state, or the actual state might be changed
* through another userspace API such as writing /proc/acpi/ibm/lcdshadow.
* In this case the driver must update both the hw-state and the sw-state
* to reflect the new value, overwriting any pending state requests in the
* sw-state. Any pending sw-state requests are thus discarded.
*
* Note that the ability for the state to change outside of control of
* the DRM master process means that userspace must not cache the value
* of the sw-state. Caching the sw-state value and including it in later
* atomic commits may lead to overriding a state change done through e.g.
* a firmware handled hotkey. Therefor userspace must not include the
* privacy-screen sw-state in an atomic commit unless it wants to change
* its value.
*
* left margin, right margin, top margin, bottom margin:
* Add margins to the connector's viewport. This is typically used to
* mitigate overscan on TVs.
*
* The value is the size in pixels of the black border which will be
* added. The attached CRTC's content will be scaled to fill the whole
* area inside the margin.
*
* The margins configuration might be sent to the sink, e.g. via HDMI AVI
* InfoFrames.
*
* Drivers can set up these properties by calling
* drm_mode_create_tv_margin_properties().
*/
int drm_connector_create_standard_properties(struct drm_device *dev)
{
struct drm_property *prop;
prop = drm_property_create(dev, DRM_MODE_PROP_BLOB |
DRM_MODE_PROP_IMMUTABLE,
"EDID", 0);
if (!prop)
return -ENOMEM;
dev->mode_config.edid_property = prop;
prop = drm_property_create_enum(dev, 0,
"DPMS", drm_dpms_enum_list,
ARRAY_SIZE(drm_dpms_enum_list));
if (!prop)
return -ENOMEM;
dev->mode_config.dpms_property = prop;
prop = drm_property_create(dev,
DRM_MODE_PROP_BLOB |
DRM_MODE_PROP_IMMUTABLE,
"PATH", 0);
if (!prop)
return -ENOMEM;
dev->mode_config.path_property = prop;
prop = drm_property_create(dev,
DRM_MODE_PROP_BLOB |
DRM_MODE_PROP_IMMUTABLE,
"TILE", 0);
if (!prop)
return -ENOMEM;
dev->mode_config.tile_property = prop;
prop = drm_property_create_enum(dev, 0, "link-status",
drm_link_status_enum_list,
ARRAY_SIZE(drm_link_status_enum_list));
if (!prop)
return -ENOMEM;
dev->mode_config.link_status_property = prop;
prop = drm_property_create_bool(dev, DRM_MODE_PROP_IMMUTABLE, "non-desktop");
if (!prop)
return -ENOMEM;
dev->mode_config.non_desktop_property = prop;
prop = drm_property_create(dev, DRM_MODE_PROP_BLOB,
"HDR_OUTPUT_METADATA", 0);
if (!prop)
return -ENOMEM;
dev->mode_config.hdr_output_metadata_property = prop;
return 0;
}
/**
* drm_mode_create_dvi_i_properties - create DVI-I specific connector properties
* @dev: DRM device
*
* Called by a driver the first time a DVI-I connector is made.
*
* Returns: %0
*/
int drm_mode_create_dvi_i_properties(struct drm_device *dev)
{
struct drm_property *dvi_i_selector;
struct drm_property *dvi_i_subconnector;
if (dev->mode_config.dvi_i_select_subconnector_property)
return 0;
dvi_i_selector =
drm_property_create_enum(dev, 0,
"select subconnector",
drm_dvi_i_select_enum_list,
ARRAY_SIZE(drm_dvi_i_select_enum_list));
dev->mode_config.dvi_i_select_subconnector_property = dvi_i_selector;
dvi_i_subconnector = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
"subconnector",
drm_dvi_i_subconnector_enum_list,
ARRAY_SIZE(drm_dvi_i_subconnector_enum_list));
dev->mode_config.dvi_i_subconnector_property = dvi_i_subconnector;
return 0;
}
EXPORT_SYMBOL(drm_mode_create_dvi_i_properties);
/**
* drm_connector_attach_dp_subconnector_property - create subconnector property for DP
* @connector: drm_connector to attach property
*
* Called by a driver when DP connector is created.
*/
void drm_connector_attach_dp_subconnector_property(struct drm_connector *connector)
{
struct drm_mode_config *mode_config = &connector->dev->mode_config;
if (!mode_config->dp_subconnector_property)
mode_config->dp_subconnector_property =
drm_property_create_enum(connector->dev,
DRM_MODE_PROP_IMMUTABLE,
"subconnector",
drm_dp_subconnector_enum_list,
ARRAY_SIZE(drm_dp_subconnector_enum_list));
drm_object_attach_property(&connector->base,
mode_config->dp_subconnector_property,
DRM_MODE_SUBCONNECTOR_Unknown);
}
EXPORT_SYMBOL(drm_connector_attach_dp_subconnector_property);
/**
* DOC: HDMI connector properties
*
* content type (HDMI specific):
* Indicates content type setting to be used in HDMI infoframes to indicate
* content type for the external device, so that it adjusts its display
* settings accordingly.
*
* The value of this property can be one of the following:
*
* No Data:
* Content type is unknown
* Graphics:
* Content type is graphics
* Photo:
* Content type is photo
* Cinema:
* Content type is cinema
* Game:
* Content type is game
*
* The meaning of each content type is defined in CTA-861-G table 15.
*
* Drivers can set up this property by calling
* drm_connector_attach_content_type_property(). Decoding to
* infoframe values is done through drm_hdmi_avi_infoframe_content_type().
*/
/*
* TODO: Document the properties:
* - brightness
* - contrast
* - flicker reduction
* - hue
* - mode
* - overscan
* - saturation
* - select subconnector
*/
/**
* DOC: Analog TV Connector Properties
*
* TV Mode:
* Indicates the TV Mode used on an analog TV connector. The value
* of this property can be one of the following:
*
* NTSC:
* TV Mode is CCIR System M (aka 525-lines) together with
* the NTSC Color Encoding.
*
* NTSC-443:
*
* TV Mode is CCIR System M (aka 525-lines) together with
* the NTSC Color Encoding, but with a color subcarrier
* frequency of 4.43MHz
*
* NTSC-J:
*
* TV Mode is CCIR System M (aka 525-lines) together with
* the NTSC Color Encoding, but with a black level equal to
* the blanking level.
*
* PAL:
*
* TV Mode is CCIR System B (aka 625-lines) together with
* the PAL Color Encoding.
*
* PAL-M:
*
* TV Mode is CCIR System M (aka 525-lines) together with
* the PAL Color Encoding.
*
* PAL-N:
*
* TV Mode is CCIR System N together with the PAL Color
* Encoding, a color subcarrier frequency of 3.58MHz, the
* SECAM color space, and narrower channels than other PAL
* variants.
*
* SECAM:
*
* TV Mode is CCIR System B (aka 625-lines) together with
* the SECAM Color Encoding.
*
* Drivers can set up this property by calling
* drm_mode_create_tv_properties().
*/
/**
* drm_connector_attach_content_type_property - attach content-type property
* @connector: connector to attach content type property on.
*
* Called by a driver the first time a HDMI connector is made.
*
* Returns: %0
*/
int drm_connector_attach_content_type_property(struct drm_connector *connector)
{
if (!drm_mode_create_content_type_property(connector->dev))
drm_object_attach_property(&connector->base,
connector->dev->mode_config.content_type_property,
DRM_MODE_CONTENT_TYPE_NO_DATA);
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_content_type_property);
/**
* drm_connector_attach_tv_margin_properties - attach TV connector margin
* properties
* @connector: DRM connector
*
* Called by a driver when it needs to attach TV margin props to a connector.
* Typically used on SDTV and HDMI connectors.
*/
void drm_connector_attach_tv_margin_properties(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
drm_object_attach_property(&connector->base,
dev->mode_config.tv_left_margin_property,
0);
drm_object_attach_property(&connector->base,
dev->mode_config.tv_right_margin_property,
0);
drm_object_attach_property(&connector->base,
dev->mode_config.tv_top_margin_property,
0);
drm_object_attach_property(&connector->base,
dev->mode_config.tv_bottom_margin_property,
0);
}
EXPORT_SYMBOL(drm_connector_attach_tv_margin_properties);
/**
* drm_mode_create_tv_margin_properties - create TV connector margin properties
* @dev: DRM device
*
* Called by a driver's HDMI connector initialization routine, this function
* creates the TV margin properties for a given device. No need to call this
* function for an SDTV connector, it's already called from
* drm_mode_create_tv_properties_legacy().
*
* Returns:
* 0 on success or a negative error code on failure.
*/
int drm_mode_create_tv_margin_properties(struct drm_device *dev)
{
if (dev->mode_config.tv_left_margin_property)
return 0;
dev->mode_config.tv_left_margin_property =
drm_property_create_range(dev, 0, "left margin", 0, 100);
if (!dev->mode_config.tv_left_margin_property)
return -ENOMEM;
dev->mode_config.tv_right_margin_property =
drm_property_create_range(dev, 0, "right margin", 0, 100);
if (!dev->mode_config.tv_right_margin_property)
return -ENOMEM;
dev->mode_config.tv_top_margin_property =
drm_property_create_range(dev, 0, "top margin", 0, 100);
if (!dev->mode_config.tv_top_margin_property)
return -ENOMEM;
dev->mode_config.tv_bottom_margin_property =
drm_property_create_range(dev, 0, "bottom margin", 0, 100);
if (!dev->mode_config.tv_bottom_margin_property)
return -ENOMEM;
return 0;
}
EXPORT_SYMBOL(drm_mode_create_tv_margin_properties);
/**
* drm_mode_create_tv_properties_legacy - create TV specific connector properties
* @dev: DRM device
* @num_modes: number of different TV formats (modes) supported
* @modes: array of pointers to strings containing name of each format
*
* Called by a driver's TV initialization routine, this function creates
* the TV specific connector properties for a given device. Caller is
* responsible for allocating a list of format names and passing them to
* this routine.
*
* NOTE: This functions registers the deprecated "mode" connector
* property to select the analog TV mode (ie, NTSC, PAL, etc.). New
* drivers must use drm_mode_create_tv_properties() instead.
*
* Returns:
* 0 on success or a negative error code on failure.
*/
int drm_mode_create_tv_properties_legacy(struct drm_device *dev,
unsigned int num_modes,
const char * const modes[])
{
struct drm_property *tv_selector;
struct drm_property *tv_subconnector;
unsigned int i;
if (dev->mode_config.tv_select_subconnector_property)
return 0;
/*
* Basic connector properties
*/
tv_selector = drm_property_create_enum(dev, 0,
"select subconnector",
drm_tv_select_enum_list,
ARRAY_SIZE(drm_tv_select_enum_list));
if (!tv_selector)
goto nomem;
dev->mode_config.tv_select_subconnector_property = tv_selector;
tv_subconnector =
drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
"subconnector",
drm_tv_subconnector_enum_list,
ARRAY_SIZE(drm_tv_subconnector_enum_list));
if (!tv_subconnector)
goto nomem;
dev->mode_config.tv_subconnector_property = tv_subconnector;
/*
* Other, TV specific properties: margins & TV modes.
*/
if (drm_mode_create_tv_margin_properties(dev))
goto nomem;
if (num_modes) {
dev->mode_config.legacy_tv_mode_property =
drm_property_create(dev, DRM_MODE_PROP_ENUM,
"mode", num_modes);
if (!dev->mode_config.legacy_tv_mode_property)
goto nomem;
for (i = 0; i < num_modes; i++)
drm_property_add_enum(dev->mode_config.legacy_tv_mode_property,
i, modes[i]);
}
dev->mode_config.tv_brightness_property =
drm_property_create_range(dev, 0, "brightness", 0, 100);
if (!dev->mode_config.tv_brightness_property)
goto nomem;
dev->mode_config.tv_contrast_property =
drm_property_create_range(dev, 0, "contrast", 0, 100);
if (!dev->mode_config.tv_contrast_property)
goto nomem;
dev->mode_config.tv_flicker_reduction_property =
drm_property_create_range(dev, 0, "flicker reduction", 0, 100);
if (!dev->mode_config.tv_flicker_reduction_property)
goto nomem;
dev->mode_config.tv_overscan_property =
drm_property_create_range(dev, 0, "overscan", 0, 100);
if (!dev->mode_config.tv_overscan_property)
goto nomem;
dev->mode_config.tv_saturation_property =
drm_property_create_range(dev, 0, "saturation", 0, 100);
if (!dev->mode_config.tv_saturation_property)
goto nomem;
dev->mode_config.tv_hue_property =
drm_property_create_range(dev, 0, "hue", 0, 100);
if (!dev->mode_config.tv_hue_property)
goto nomem;
return 0;
nomem:
return -ENOMEM;
}
EXPORT_SYMBOL(drm_mode_create_tv_properties_legacy);
/**
* drm_mode_create_tv_properties - create TV specific connector properties
* @dev: DRM device
* @supported_tv_modes: Bitmask of TV modes supported (See DRM_MODE_TV_MODE_*)
*
* Called by a driver's TV initialization routine, this function creates
* the TV specific connector properties for a given device.
*
* Returns:
* 0 on success or a negative error code on failure.
*/
int drm_mode_create_tv_properties(struct drm_device *dev,
unsigned int supported_tv_modes)
{
struct drm_prop_enum_list tv_mode_list[DRM_MODE_TV_MODE_MAX];
struct drm_property *tv_mode;
unsigned int i, len = 0;
if (dev->mode_config.tv_mode_property)
return 0;
for (i = 0; i < DRM_MODE_TV_MODE_MAX; i++) {
if (!(supported_tv_modes & BIT(i)))
continue;
tv_mode_list[len].type = i;
tv_mode_list[len].name = drm_get_tv_mode_name(i);
len++;
}
tv_mode = drm_property_create_enum(dev, 0, "TV mode",
tv_mode_list, len);
if (!tv_mode)
return -ENOMEM;
dev->mode_config.tv_mode_property = tv_mode;
return drm_mode_create_tv_properties_legacy(dev, 0, NULL);
}
EXPORT_SYMBOL(drm_mode_create_tv_properties);
/**
* drm_mode_create_scaling_mode_property - create scaling mode property
* @dev: DRM device
*
* Called by a driver the first time it's needed, must be attached to desired
* connectors.
*
* Atomic drivers should use drm_connector_attach_scaling_mode_property()
* instead to correctly assign &drm_connector_state.scaling_mode
* in the atomic state.
*
* Returns: %0
*/
int drm_mode_create_scaling_mode_property(struct drm_device *dev)
{
struct drm_property *scaling_mode;
if (dev->mode_config.scaling_mode_property)
return 0;
scaling_mode =
drm_property_create_enum(dev, 0, "scaling mode",
drm_scaling_mode_enum_list,
ARRAY_SIZE(drm_scaling_mode_enum_list));
dev->mode_config.scaling_mode_property = scaling_mode;
return 0;
}
EXPORT_SYMBOL(drm_mode_create_scaling_mode_property);
/**
* DOC: Variable refresh properties
*
* Variable refresh rate capable displays can dynamically adjust their
* refresh rate by extending the duration of their vertical front porch
* until page flip or timeout occurs. This can reduce or remove stuttering
* and latency in scenarios where the page flip does not align with the
* vblank interval.
*
* An example scenario would be an application flipping at a constant rate
* of 48Hz on a 60Hz display. The page flip will frequently miss the vblank
* interval and the same contents will be displayed twice. This can be
* observed as stuttering for content with motion.
*
* If variable refresh rate was active on a display that supported a
* variable refresh range from 35Hz to 60Hz no stuttering would be observable
* for the example scenario. The minimum supported variable refresh rate of
* 35Hz is below the page flip frequency and the vertical front porch can
* be extended until the page flip occurs. The vblank interval will be
* directly aligned to the page flip rate.
*
* Not all userspace content is suitable for use with variable refresh rate.
* Large and frequent changes in vertical front porch duration may worsen
* perceived stuttering for input sensitive applications.
*
* Panel brightness will also vary with vertical front porch duration. Some
* panels may have noticeable differences in brightness between the minimum
* vertical front porch duration and the maximum vertical front porch duration.
* Large and frequent changes in vertical front porch duration may produce
* observable flickering for such panels.
*
* Userspace control for variable refresh rate is supported via properties
* on the &drm_connector and &drm_crtc objects.
*
* "vrr_capable":
* Optional &drm_connector boolean property that drivers should attach
* with drm_connector_attach_vrr_capable_property() on connectors that
* could support variable refresh rates. Drivers should update the
* property value by calling drm_connector_set_vrr_capable_property().
*
* Absence of the property should indicate absence of support.
*
* "VRR_ENABLED":
* Default &drm_crtc boolean property that notifies the driver that the
* content on the CRTC is suitable for variable refresh rate presentation.
* The driver will take this property as a hint to enable variable
* refresh rate support if the receiver supports it, ie. if the
* "vrr_capable" property is true on the &drm_connector object. The
* vertical front porch duration will be extended until page-flip or
* timeout when enabled.
*
* The minimum vertical front porch duration is defined as the vertical
* front porch duration for the current mode.
*
* The maximum vertical front porch duration is greater than or equal to
* the minimum vertical front porch duration. The duration is derived
* from the minimum supported variable refresh rate for the connector.
*
* The driver may place further restrictions within these minimum
* and maximum bounds.
*/
/**
* drm_connector_attach_vrr_capable_property - creates the
* vrr_capable property
* @connector: connector to create the vrr_capable property on.
*
* This is used by atomic drivers to add support for querying
* variable refresh rate capability for a connector.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_attach_vrr_capable_property(
struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
struct drm_property *prop;
if (!connector->vrr_capable_property) {
prop = drm_property_create_bool(dev, DRM_MODE_PROP_IMMUTABLE,
"vrr_capable");
if (!prop)
return -ENOMEM;
connector->vrr_capable_property = prop;
drm_object_attach_property(&connector->base, prop, 0);
}
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_vrr_capable_property);
/**
* drm_connector_attach_scaling_mode_property - attach atomic scaling mode property
* @connector: connector to attach scaling mode property on.
* @scaling_mode_mask: or'ed mask of BIT(%DRM_MODE_SCALE_\*).
*
* This is used to add support for scaling mode to atomic drivers.
* The scaling mode will be set to &drm_connector_state.scaling_mode
* and can be used from &drm_connector_helper_funcs->atomic_check for validation.
*
* This is the atomic version of drm_mode_create_scaling_mode_property().
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_attach_scaling_mode_property(struct drm_connector *connector,
u32 scaling_mode_mask)
{
struct drm_device *dev = connector->dev;
struct drm_property *scaling_mode_property;
int i;
const unsigned valid_scaling_mode_mask =
(1U << ARRAY_SIZE(drm_scaling_mode_enum_list)) - 1;
if (WARN_ON(hweight32(scaling_mode_mask) < 2 ||
scaling_mode_mask & ~valid_scaling_mode_mask))
return -EINVAL;
scaling_mode_property =
drm_property_create(dev, DRM_MODE_PROP_ENUM, "scaling mode",
hweight32(scaling_mode_mask));
if (!scaling_mode_property)
return -ENOMEM;
for (i = 0; i < ARRAY_SIZE(drm_scaling_mode_enum_list); i++) {
int ret;
if (!(BIT(i) & scaling_mode_mask))
continue;
ret = drm_property_add_enum(scaling_mode_property,
drm_scaling_mode_enum_list[i].type,
drm_scaling_mode_enum_list[i].name);
if (ret) {
drm_property_destroy(dev, scaling_mode_property);
return ret;
}
}
drm_object_attach_property(&connector->base,
scaling_mode_property, 0);
connector->scaling_mode_property = scaling_mode_property;
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_scaling_mode_property);
/**
* drm_mode_create_aspect_ratio_property - create aspect ratio property
* @dev: DRM device
*
* Called by a driver the first time it's needed, must be attached to desired
* connectors.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_create_aspect_ratio_property(struct drm_device *dev)
{
if (dev->mode_config.aspect_ratio_property)
return 0;
dev->mode_config.aspect_ratio_property =
drm_property_create_enum(dev, 0, "aspect ratio",
drm_aspect_ratio_enum_list,
ARRAY_SIZE(drm_aspect_ratio_enum_list));
if (dev->mode_config.aspect_ratio_property == NULL)
return -ENOMEM;
return 0;
}
EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);
/**
* DOC: standard connector properties
*
* Colorspace:
* This property helps select a suitable colorspace based on the sink
* capability. Modern sink devices support wider gamut like BT2020.
* This helps switch to BT2020 mode if the BT2020 encoded video stream
* is being played by the user, same for any other colorspace. Thereby
* giving a good visual experience to users.
*
* The expectation from userspace is that it should parse the EDID
* and get supported colorspaces. Use this property and switch to the
* one supported. Sink supported colorspaces should be retrieved by
* userspace from EDID and driver will not explicitly expose them.
*
* Basically the expectation from userspace is:
* - Set up CRTC DEGAMMA/CTM/GAMMA to convert to some sink
* colorspace
* - Set this new property to let the sink know what it
* converted the CRTC output to.
* - This property is just to inform sink what colorspace
* source is trying to drive.
*
* Because between HDMI and DP have different colorspaces,
* drm_mode_create_hdmi_colorspace_property() is used for HDMI connector and
* drm_mode_create_dp_colorspace_property() is used for DP connector.
*/
static int drm_mode_create_colorspace_property(struct drm_connector *connector,
u32 supported_colorspaces)
{
struct drm_device *dev = connector->dev;
u32 colorspaces = supported_colorspaces | BIT(DRM_MODE_COLORIMETRY_DEFAULT);
struct drm_prop_enum_list enum_list[DRM_MODE_COLORIMETRY_COUNT];
int i, len;
if (connector->colorspace_property)
return 0;
if (!supported_colorspaces) {
drm_err(dev, "No supported colorspaces provded on [CONNECTOR:%d:%s]\n",
connector->base.id, connector->name);
return -EINVAL;
}
if ((supported_colorspaces & -BIT(DRM_MODE_COLORIMETRY_COUNT)) != 0) {
drm_err(dev, "Unknown colorspace provded on [CONNECTOR:%d:%s]\n",
connector->base.id, connector->name);
return -EINVAL;
}
len = 0;
for (i = 0; i < DRM_MODE_COLORIMETRY_COUNT; i++) {
if ((colorspaces & BIT(i)) == 0)
continue;
enum_list[len].type = i;
enum_list[len].name = colorspace_names[i];
len++;
}
connector->colorspace_property =
drm_property_create_enum(dev, DRM_MODE_PROP_ENUM, "Colorspace",
enum_list,
len);
if (!connector->colorspace_property)
return -ENOMEM;
return 0;
}
/**
* drm_mode_create_hdmi_colorspace_property - create hdmi colorspace property
* @connector: connector to create the Colorspace property on.
* @supported_colorspaces: bitmap of supported color spaces
*
* Called by a driver the first time it's needed, must be attached to desired
* HDMI connectors.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_create_hdmi_colorspace_property(struct drm_connector *connector,
u32 supported_colorspaces)
{
u32 colorspaces;
if (supported_colorspaces)
colorspaces = supported_colorspaces & hdmi_colorspaces;
else
colorspaces = hdmi_colorspaces;
return drm_mode_create_colorspace_property(connector, colorspaces);
}
EXPORT_SYMBOL(drm_mode_create_hdmi_colorspace_property);
/**
* drm_mode_create_dp_colorspace_property - create dp colorspace property
* @connector: connector to create the Colorspace property on.
* @supported_colorspaces: bitmap of supported color spaces
*
* Called by a driver the first time it's needed, must be attached to desired
* DP connectors.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_create_dp_colorspace_property(struct drm_connector *connector,
u32 supported_colorspaces)
{
u32 colorspaces;
if (supported_colorspaces)
colorspaces = supported_colorspaces & dp_colorspaces;
else
colorspaces = dp_colorspaces;
return drm_mode_create_colorspace_property(connector, colorspaces);
}
EXPORT_SYMBOL(drm_mode_create_dp_colorspace_property);
/**
* drm_mode_create_content_type_property - create content type property
* @dev: DRM device
*
* Called by a driver the first time it's needed, must be attached to desired
* connectors.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_mode_create_content_type_property(struct drm_device *dev)
{
if (dev->mode_config.content_type_property)
return 0;
dev->mode_config.content_type_property =
drm_property_create_enum(dev, 0, "content type",
drm_content_type_enum_list,
ARRAY_SIZE(drm_content_type_enum_list));
if (dev->mode_config.content_type_property == NULL)
return -ENOMEM;
return 0;
}
EXPORT_SYMBOL(drm_mode_create_content_type_property);
/**
* drm_mode_create_suggested_offset_properties - create suggests offset properties
* @dev: DRM device
*
* Create the suggested x/y offset property for connectors.
*
* Returns:
* 0 on success or a negative error code on failure.
*/
int drm_mode_create_suggested_offset_properties(struct drm_device *dev)
{
if (dev->mode_config.suggested_x_property && dev->mode_config.suggested_y_property)
return 0;
dev->mode_config.suggested_x_property =
drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested X", 0, 0xffffffff);
dev->mode_config.suggested_y_property =
drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested Y", 0, 0xffffffff);
if (dev->mode_config.suggested_x_property == NULL ||
dev->mode_config.suggested_y_property == NULL)
return -ENOMEM;
return 0;
}
EXPORT_SYMBOL(drm_mode_create_suggested_offset_properties);
/**
* drm_connector_set_path_property - set tile property on connector
* @connector: connector to set property on.
* @path: path to use for property; must not be NULL.
*
* This creates a property to expose to userspace to specify a
* connector path. This is mainly used for DisplayPort MST where
* connectors have a topology and we want to allow userspace to give
* them more meaningful names.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_set_path_property(struct drm_connector *connector,
const char *path)
{
struct drm_device *dev = connector->dev;
int ret;
ret = drm_property_replace_global_blob(dev,
&connector->path_blob_ptr,
strlen(path) + 1,
path,
&connector->base,
dev->mode_config.path_property);
return ret;
}
EXPORT_SYMBOL(drm_connector_set_path_property);
/**
* drm_connector_set_tile_property - set tile property on connector
* @connector: connector to set property on.
*
* This looks up the tile information for a connector, and creates a
* property for userspace to parse if it exists. The property is of
* the form of 8 integers using ':' as a separator.
* This is used for dual port tiled displays with DisplayPort SST
* or DisplayPort MST connectors.
*
* Returns:
* Zero on success, errno on failure.
*/
int drm_connector_set_tile_property(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
char tile[256];
int ret;
if (!connector->has_tile) {
ret = drm_property_replace_global_blob(dev,
&connector->tile_blob_ptr,
0,
NULL,
&connector->base,
dev->mode_config.tile_property);
return ret;
}
snprintf(tile, 256, "%d:%d:%d:%d:%d:%d:%d:%d",
connector->tile_group->id, connector->tile_is_single_monitor,
connector->num_h_tile, connector->num_v_tile,
connector->tile_h_loc, connector->tile_v_loc,
connector->tile_h_size, connector->tile_v_size);
ret = drm_property_replace_global_blob(dev,
&connector->tile_blob_ptr,
strlen(tile) + 1,
tile,
&connector->base,
dev->mode_config.tile_property);
return ret;
}
EXPORT_SYMBOL(drm_connector_set_tile_property);
/**
* drm_connector_set_link_status_property - Set link status property of a connector
* @connector: drm connector
* @link_status: new value of link status property (0: Good, 1: Bad)
*
* In usual working scenario, this link status property will always be set to
* "GOOD". If something fails during or after a mode set, the kernel driver
* may set this link status property to "BAD". The caller then needs to send a
* hotplug uevent for userspace to re-check the valid modes through
* GET_CONNECTOR_IOCTL and retry modeset.
*
* Note: Drivers cannot rely on userspace to support this property and
* issue a modeset. As such, they may choose to handle issues (like
* re-training a link) without userspace's intervention.
*
* The reason for adding this property is to handle link training failures, but
* it is not limited to DP or link training. For example, if we implement
* asynchronous setcrtc, this property can be used to report any failures in that.
*/
void drm_connector_set_link_status_property(struct drm_connector *connector,
uint64_t link_status)
{
struct drm_device *dev = connector->dev;
drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
connector->state->link_status = link_status;
drm_modeset_unlock(&dev->mode_config.connection_mutex);
}
EXPORT_SYMBOL(drm_connector_set_link_status_property);
/**
* drm_connector_attach_max_bpc_property - attach "max bpc" property
* @connector: connector to attach max bpc property on.
* @min: The minimum bit depth supported by the connector.
* @max: The maximum bit depth supported by the connector.
*
* This is used to add support for limiting the bit depth on a connector.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_attach_max_bpc_property(struct drm_connector *connector,
int min, int max)
{
struct drm_device *dev = connector->dev;
struct drm_property *prop;
prop = connector->max_bpc_property;
if (!prop) {
prop = drm_property_create_range(dev, 0, "max bpc", min, max);
if (!prop)
return -ENOMEM;
connector->max_bpc_property = prop;
}
drm_object_attach_property(&connector->base, prop, max);
connector->state->max_requested_bpc = max;
connector->state->max_bpc = max;
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_max_bpc_property);
/**
* drm_connector_attach_hdr_output_metadata_property - attach "HDR_OUTPUT_METADA" property
* @connector: connector to attach the property on.
*
* This is used to allow the userspace to send HDR Metadata to the
* driver.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector)
{
struct drm_device *dev = connector->dev;
struct drm_property *prop = dev->mode_config.hdr_output_metadata_property;
drm_object_attach_property(&connector->base, prop, 0);
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_hdr_output_metadata_property);
/**
* drm_connector_attach_colorspace_property - attach "Colorspace" property
* @connector: connector to attach the property on.
*
* This is used to allow the userspace to signal the output colorspace
* to the driver.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_attach_colorspace_property(struct drm_connector *connector)
{
struct drm_property *prop = connector->colorspace_property;
drm_object_attach_property(&connector->base, prop, DRM_MODE_COLORIMETRY_DEFAULT);
return 0;
}
EXPORT_SYMBOL(drm_connector_attach_colorspace_property);
/**
* drm_connector_atomic_hdr_metadata_equal - checks if the hdr metadata changed
* @old_state: old connector state to compare
* @new_state: new connector state to compare
*
* This is used by HDR-enabled drivers to test whether the HDR metadata
* have changed between two different connector state (and thus probably
* requires a full blown mode change).
*
* Returns:
* True if the metadata are equal, False otherwise
*/
bool drm_connector_atomic_hdr_metadata_equal(struct drm_connector_state *old_state,
struct drm_connector_state *new_state)
{
struct drm_property_blob *old_blob = old_state->hdr_output_metadata;
struct drm_property_blob *new_blob = new_state->hdr_output_metadata;
if (!old_blob || !new_blob)
return old_blob == new_blob;
if (old_blob->length != new_blob->length)
return false;
return !memcmp(old_blob->data, new_blob->data, old_blob->length);
}
EXPORT_SYMBOL(drm_connector_atomic_hdr_metadata_equal);
/**
* drm_connector_set_vrr_capable_property - sets the variable refresh rate
* capable property for a connector
* @connector: drm connector
* @capable: True if the connector is variable refresh rate capable
*
* Should be used by atomic drivers to update the indicated support for
* variable refresh rate over a connector.
*/
void drm_connector_set_vrr_capable_property(
struct drm_connector *connector, bool capable)
{
if (!connector->vrr_capable_property)
return;
drm_object_property_set_value(&connector->base,
connector->vrr_capable_property,
capable);
}
EXPORT_SYMBOL(drm_connector_set_vrr_capable_property);
/**
* drm_connector_set_panel_orientation - sets the connector's panel_orientation
* @connector: connector for which to set the panel-orientation property.
* @panel_orientation: drm_panel_orientation value to set
*
* This function sets the connector's panel_orientation and attaches
* a "panel orientation" property to the connector.
*
* Calling this function on a connector where the panel_orientation has
* already been set is a no-op (e.g. the orientation has been overridden with
* a kernel commandline option).
*
* It is allowed to call this function with a panel_orientation of
* DRM_MODE_PANEL_ORIENTATION_UNKNOWN, in which case it is a no-op.
*
* The function shouldn't be called in panel after drm is registered (i.e.
* drm_dev_register() is called in drm).
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_set_panel_orientation(
struct drm_connector *connector,
enum drm_panel_orientation panel_orientation)
{
struct drm_device *dev = connector->dev;
struct drm_display_info *info = &connector->display_info;
struct drm_property *prop;
/* Already set? */
if (info->panel_orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
return 0;
/* Don't attach the property if the orientation is unknown */
if (panel_orientation == DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
return 0;
info->panel_orientation = panel_orientation;
prop = dev->mode_config.panel_orientation_property;
if (!prop) {
prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
"panel orientation",
drm_panel_orientation_enum_list,
ARRAY_SIZE(drm_panel_orientation_enum_list));
if (!prop)
return -ENOMEM;
dev->mode_config.panel_orientation_property = prop;
}
drm_object_attach_property(&connector->base, prop,
info->panel_orientation);
return 0;
}
EXPORT_SYMBOL(drm_connector_set_panel_orientation);
/**
* drm_connector_set_panel_orientation_with_quirk - set the
* connector's panel_orientation after checking for quirks
* @connector: connector for which to init the panel-orientation property.
* @panel_orientation: drm_panel_orientation value to set
* @width: width in pixels of the panel, used for panel quirk detection
* @height: height in pixels of the panel, used for panel quirk detection
*
* Like drm_connector_set_panel_orientation(), but with a check for platform
* specific (e.g. DMI based) quirks overriding the passed in panel_orientation.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_set_panel_orientation_with_quirk(
struct drm_connector *connector,
enum drm_panel_orientation panel_orientation,
int width, int height)
{
int orientation_quirk;
orientation_quirk = drm_get_panel_orientation_quirk(width, height);
if (orientation_quirk != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
panel_orientation = orientation_quirk;
return drm_connector_set_panel_orientation(connector,
panel_orientation);
}
EXPORT_SYMBOL(drm_connector_set_panel_orientation_with_quirk);
/**
* drm_connector_set_orientation_from_panel -
* set the connector's panel_orientation from panel's callback.
* @connector: connector for which to init the panel-orientation property.
* @panel: panel that can provide orientation information.
*
* Drm drivers should call this function before drm_dev_register().
* Orientation is obtained from panel's .get_orientation() callback.
*
* Returns:
* Zero on success, negative errno on failure.
*/
int drm_connector_set_orientation_from_panel(
struct drm_connector *connector,
struct drm_panel *panel)
{
enum drm_panel_orientation orientation;
if (panel && panel->funcs && panel->funcs->get_orientation)
orientation = panel->funcs->get_orientation(panel);
else
orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
return drm_connector_set_panel_orientation(connector, orientation);
}
EXPORT_SYMBOL(drm_connector_set_orientation_from_panel);
static const struct drm_prop_enum_list privacy_screen_enum[] = {
{ PRIVACY_SCREEN_DISABLED, "Disabled" },
{ PRIVACY_SCREEN_ENABLED, "Enabled" },
{ PRIVACY_SCREEN_DISABLED_LOCKED, "Disabled-locked" },
{ PRIVACY_SCREEN_ENABLED_LOCKED, "Enabled-locked" },
};
/**
* drm_connector_create_privacy_screen_properties - create the drm connecter's
* privacy-screen properties.
* @connector: connector for which to create the privacy-screen properties
*
* This function creates the "privacy-screen sw-state" and "privacy-screen
* hw-state" properties for the connector. They are not attached.
*/
void
drm_connector_create_privacy_screen_properties(struct drm_connector *connector)
{
if (connector->privacy_screen_sw_state_property)
return;
/* Note sw-state only supports the first 2 values of the enum */
connector->privacy_screen_sw_state_property =
drm_property_create_enum(connector->dev, DRM_MODE_PROP_ENUM,
"privacy-screen sw-state",
privacy_screen_enum, 2);
connector->privacy_screen_hw_state_property =
drm_property_create_enum(connector->dev,
DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_ENUM,
"privacy-screen hw-state",
privacy_screen_enum,
ARRAY_SIZE(privacy_screen_enum));
}
EXPORT_SYMBOL(drm_connector_create_privacy_screen_properties);
/**
* drm_connector_attach_privacy_screen_properties - attach the drm connecter's
* privacy-screen properties.
* @connector: connector on which to attach the privacy-screen properties
*
* This function attaches the "privacy-screen sw-state" and "privacy-screen
* hw-state" properties to the connector. The initial state of both is set
* to "Disabled".
*/
void
drm_connector_attach_privacy_screen_properties(struct drm_connector *connector)
{
if (!connector->privacy_screen_sw_state_property)
return;
drm_object_attach_property(&connector->base,
connector->privacy_screen_sw_state_property,
PRIVACY_SCREEN_DISABLED);
drm_object_attach_property(&connector->base,
connector->privacy_screen_hw_state_property,
PRIVACY_SCREEN_DISABLED);
}
EXPORT_SYMBOL(drm_connector_attach_privacy_screen_properties);
static void drm_connector_update_privacy_screen_properties(
struct drm_connector *connector, bool set_sw_state)
{
enum drm_privacy_screen_status sw_state, hw_state;
drm_privacy_screen_get_state(connector->privacy_screen,
&sw_state, &hw_state);
if (set_sw_state)
connector->state->privacy_screen_sw_state = sw_state;
drm_object_property_set_value(&connector->base,
connector->privacy_screen_hw_state_property, hw_state);
}
static int drm_connector_privacy_screen_notifier(
struct notifier_block *nb, unsigned long action, void *data)
{
struct drm_connector *connector =
container_of(nb, struct drm_connector, privacy_screen_notifier);
struct drm_device *dev = connector->dev;
drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
drm_connector_update_privacy_screen_properties(connector, true);
drm_modeset_unlock(&dev->mode_config.connection_mutex);
drm_sysfs_connector_property_event(connector,
connector->privacy_screen_sw_state_property);
drm_sysfs_connector_property_event(connector,
connector->privacy_screen_hw_state_property);
return NOTIFY_DONE;
}
/**
* drm_connector_attach_privacy_screen_provider - attach a privacy-screen to
* the connector
* @connector: connector to attach the privacy-screen to
* @priv: drm_privacy_screen to attach
*
* Create and attach the standard privacy-screen properties and register
* a generic notifier for generating sysfs-connector-status-events
* on external changes to the privacy-screen status.
* This function takes ownership of the passed in drm_privacy_screen and will
* call drm_privacy_screen_put() on it when the connector is destroyed.
*/
void drm_connector_attach_privacy_screen_provider(
struct drm_connector *connector, struct drm_privacy_screen *priv)
{
connector->privacy_screen = priv;
connector->privacy_screen_notifier.notifier_call =
drm_connector_privacy_screen_notifier;
drm_connector_create_privacy_screen_properties(connector);
drm_connector_update_privacy_screen_properties(connector, true);
drm_connector_attach_privacy_screen_properties(connector);
}
EXPORT_SYMBOL(drm_connector_attach_privacy_screen_provider);
/**
* drm_connector_update_privacy_screen - update connector's privacy-screen sw-state
* @connector_state: connector-state to update the privacy-screen for
*
* This function calls drm_privacy_screen_set_sw_state() on the connector's
* privacy-screen.
*
* If the connector has no privacy-screen, then this is a no-op.
*/
void drm_connector_update_privacy_screen(const struct drm_connector_state *connector_state)
{
struct drm_connector *connector = connector_state->connector;
int ret;
if (!connector->privacy_screen)
return;
ret = drm_privacy_screen_set_sw_state(connector->privacy_screen,
connector_state->privacy_screen_sw_state);
if (ret) {
drm_err(connector->dev, "Error updating privacy-screen sw_state\n");
return;
}
/* The hw_state property value may have changed, update it. */
drm_connector_update_privacy_screen_properties(connector, false);
}
EXPORT_SYMBOL(drm_connector_update_privacy_screen);
int drm_connector_set_obj_prop(struct drm_mode_object *obj,
struct drm_property *property,
uint64_t value)
{
int ret = -EINVAL;
struct drm_connector *connector = obj_to_connector(obj);
/* Do DPMS ourselves */
if (property == connector->dev->mode_config.dpms_property) {
ret = (*connector->funcs->dpms)(connector, (int)value);
} else if (connector->funcs->set_property)
ret = connector->funcs->set_property(connector, property, value);
if (!ret)
drm_object_property_set_value(&connector->base, property, value);
return ret;
}
int drm_connector_property_set_ioctl(struct drm_device *dev,
void *data, struct drm_file *file_priv)
{
struct drm_mode_connector_set_property *conn_set_prop = data;
struct drm_mode_obj_set_property obj_set_prop = {
.value = conn_set_prop->value,
.prop_id = conn_set_prop->prop_id,
.obj_id = conn_set_prop->connector_id,
.obj_type = DRM_MODE_OBJECT_CONNECTOR
};
/* It does all the locking and checking we need */
return drm_mode_obj_set_property_ioctl(dev, &obj_set_prop, file_priv);
}
static struct drm_encoder *drm_connector_get_encoder(struct drm_connector *connector)
{
/* For atomic drivers only state objects are synchronously updated and
* protected by modeset locks, so check those first.
*/
if (connector->state)
return connector->state->best_encoder;
return connector->encoder;
}
static bool
drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
const struct list_head *modes,
const struct drm_file *file_priv)
{
/*
* If user-space hasn't configured the driver to expose the stereo 3D
* modes, don't expose them.
*/
if (!file_priv->stereo_allowed && drm_mode_is_stereo(mode))
return false;
/*
* If user-space hasn't configured the driver to expose the modes
* with aspect-ratio, don't expose them. However if such a mode
* is unique, let it be exposed, but reset the aspect-ratio flags
* while preparing the list of user-modes.
*/
if (!file_priv->aspect_ratio_allowed) {
const struct drm_display_mode *mode_itr;
list_for_each_entry(mode_itr, modes, head) {
if (mode_itr->expose_to_userspace &&
drm_mode_match(mode_itr, mode,
DRM_MODE_MATCH_TIMINGS |
DRM_MODE_MATCH_CLOCK |
DRM_MODE_MATCH_FLAGS |
DRM_MODE_MATCH_3D_FLAGS))
return false;
}
}
return true;
}
int drm_mode_getconnector(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_mode_get_connector *out_resp = data;
struct drm_connector *connector;
struct drm_encoder *encoder;
struct drm_display_mode *mode;
int mode_count = 0;
int encoders_count = 0;
int ret = 0;
int copied = 0;
struct drm_mode_modeinfo u_mode;
struct drm_mode_modeinfo __user *mode_ptr;
uint32_t __user *encoder_ptr;
bool is_current_master;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
memset(&u_mode, 0, sizeof(struct drm_mode_modeinfo));
connector = drm_connector_lookup(dev, file_priv, out_resp->connector_id);
if (!connector)
return -ENOENT;
encoders_count = hweight32(connector->possible_encoders);
if ((out_resp->count_encoders >= encoders_count) && encoders_count) {
copied = 0;
encoder_ptr = (uint32_t __user *)(unsigned long)(out_resp->encoders_ptr);
drm_connector_for_each_possible_encoder(connector, encoder) {
if (put_user(encoder->base.id, encoder_ptr + copied)) {
ret = -EFAULT;
goto out;
}
copied++;
}
}
out_resp->count_encoders = encoders_count;
out_resp->connector_id = connector->base.id;
out_resp->connector_type = connector->connector_type;
out_resp->connector_type_id = connector->connector_type_id;
is_current_master = drm_is_current_master(file_priv);
mutex_lock(&dev->mode_config.mutex);
if (out_resp->count_modes == 0) {
if (is_current_master)
connector->funcs->fill_modes(connector,
dev->mode_config.max_width,
dev->mode_config.max_height);
else
drm_dbg_kms(dev, "User-space requested a forced probe on [CONNECTOR:%d:%s] but is not the DRM master, demoting to read-only probe",
connector->base.id, connector->name);
}
out_resp->mm_width = connector->display_info.width_mm;
out_resp->mm_height = connector->display_info.height_mm;
out_resp->subpixel = connector->display_info.subpixel_order;
out_resp->connection = connector->status;
/* delayed so we get modes regardless of pre-fill_modes state */
list_for_each_entry(mode, &connector->modes, head) {
WARN_ON(mode->expose_to_userspace);
if (drm_mode_expose_to_userspace(mode, &connector->modes,
file_priv)) {
mode->expose_to_userspace = true;
mode_count++;
}
}
/*
* This ioctl is called twice, once to determine how much space is
* needed, and the 2nd time to fill it.
*/
if ((out_resp->count_modes >= mode_count) && mode_count) {
copied = 0;
mode_ptr = (struct drm_mode_modeinfo __user *)(unsigned long)out_resp->modes_ptr;
list_for_each_entry(mode, &connector->modes, head) {
if (!mode->expose_to_userspace)
continue;
/* Clear the tag for the next time around */
mode->expose_to_userspace = false;
drm_mode_convert_to_umode(&u_mode, mode);
/*
* Reset aspect ratio flags of user-mode, if modes with
* aspect-ratio are not supported.
*/
if (!file_priv->aspect_ratio_allowed)
u_mode.flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;
if (copy_to_user(mode_ptr + copied,
&u_mode, sizeof(u_mode))) {
ret = -EFAULT;
/*
* Clear the tag for the rest of
* the modes for the next time around.
*/
list_for_each_entry_continue(mode, &connector->modes, head)
mode->expose_to_userspace = false;
mutex_unlock(&dev->mode_config.mutex);
goto out;
}
copied++;
}
} else {
/* Clear the tag for the next time around */
list_for_each_entry(mode, &connector->modes, head)
mode->expose_to_userspace = false;
}
out_resp->count_modes = mode_count;
mutex_unlock(&dev->mode_config.mutex);
drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
encoder = drm_connector_get_encoder(connector);
if (encoder)
out_resp->encoder_id = encoder->base.id;
else
out_resp->encoder_id = 0;
/* Only grab properties after probing, to make sure EDID and other
* properties reflect the latest status.
*/
ret = drm_mode_object_get_properties(&connector->base, file_priv->atomic,
(uint32_t __user *)(unsigned long)(out_resp->props_ptr),
(uint64_t __user *)(unsigned long)(out_resp->prop_values_ptr),
&out_resp->count_props);
drm_modeset_unlock(&dev->mode_config.connection_mutex);
out:
drm_connector_put(connector);
return ret;
}
/**
* drm_connector_find_by_fwnode - Find a connector based on the associated fwnode
* @fwnode: fwnode for which to find the matching drm_connector
*
* This functions looks up a drm_connector based on its associated fwnode. When
* a connector is found a reference to the connector is returned. The caller must
* call drm_connector_put() to release this reference when it is done with the
* connector.
*
* Returns: A reference to the found connector or an ERR_PTR().
*/
struct drm_connector *drm_connector_find_by_fwnode(struct fwnode_handle *fwnode)
{
struct drm_connector *connector, *found = ERR_PTR(-ENODEV);
if (!fwnode)
return ERR_PTR(-ENODEV);
mutex_lock(&connector_list_lock);
list_for_each_entry(connector, &connector_list, global_connector_list_entry) {
if (connector->fwnode == fwnode ||
(connector->fwnode && connector->fwnode->secondary == fwnode)) {
drm_connector_get(connector);
found = connector;
break;
}
}
mutex_unlock(&connector_list_lock);
return found;
}
/**
* drm_connector_oob_hotplug_event - Report out-of-band hotplug event to connector
* @connector_fwnode: fwnode_handle to report the event on
* @status: hot plug detect logical state
*
* On some hardware a hotplug event notification may come from outside the display
* driver / device. An example of this is some USB Type-C setups where the hardware
* muxes the DisplayPort data and aux-lines but does not pass the altmode HPD
* status bit to the GPU's DP HPD pin.
*
* This function can be used to report these out-of-band events after obtaining
* a drm_connector reference through calling drm_connector_find_by_fwnode().
*/
void drm_connector_oob_hotplug_event(struct fwnode_handle *connector_fwnode,
enum drm_connector_status status)
{
struct drm_connector *connector;
connector = drm_connector_find_by_fwnode(connector_fwnode);
if (IS_ERR(connector))
return;
if (connector->funcs->oob_hotplug_event)
connector->funcs->oob_hotplug_event(connector, status);
drm_connector_put(connector);
}
EXPORT_SYMBOL(drm_connector_oob_hotplug_event);
/**
* DOC: Tile group
*
* Tile groups are used to represent tiled monitors with a unique integer
* identifier. Tiled monitors using DisplayID v1.3 have a unique 8-byte handle,
* we store this in a tile group, so we have a common identifier for all tiles
* in a monitor group. The property is called "TILE". Drivers can manage tile
* groups using drm_mode_create_tile_group(), drm_mode_put_tile_group() and
* drm_mode_get_tile_group(). But this is only needed for internal panels where
* the tile group information is exposed through a non-standard way.
*/
static void drm_tile_group_free(struct kref *kref)
{
struct drm_tile_group *tg = container_of(kref, struct drm_tile_group, refcount);
struct drm_device *dev = tg->dev;
mutex_lock(&dev->mode_config.idr_mutex);
idr_remove(&dev->mode_config.tile_idr, tg->id);
mutex_unlock(&dev->mode_config.idr_mutex);
kfree(tg);
}
/**
* drm_mode_put_tile_group - drop a reference to a tile group.
* @dev: DRM device
* @tg: tile group to drop reference to.
*
* drop reference to tile group and free if 0.
*/
void drm_mode_put_tile_group(struct drm_device *dev,
struct drm_tile_group *tg)
{
kref_put(&tg->refcount, drm_tile_group_free);
}
EXPORT_SYMBOL(drm_mode_put_tile_group);
/**
* drm_mode_get_tile_group - get a reference to an existing tile group
* @dev: DRM device
* @topology: 8-bytes unique per monitor.
*
* Use the unique bytes to get a reference to an existing tile group.
*
* RETURNS:
* tile group or NULL if not found.
*/
struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
const char topology[8])
{
struct drm_tile_group *tg;
int id;
mutex_lock(&dev->mode_config.idr_mutex);
idr_for_each_entry(&dev->mode_config.tile_idr, tg, id) {
if (!memcmp(tg->group_data, topology, 8)) {
if (!kref_get_unless_zero(&tg->refcount))
tg = NULL;
mutex_unlock(&dev->mode_config.idr_mutex);
return tg;
}
}
mutex_unlock(&dev->mode_config.idr_mutex);
return NULL;
}
EXPORT_SYMBOL(drm_mode_get_tile_group);
/**
* drm_mode_create_tile_group - create a tile group from a displayid description
* @dev: DRM device
* @topology: 8-bytes unique per monitor.
*
* Create a tile group for the unique monitor, and get a unique
* identifier for the tile group.
*
* RETURNS:
* new tile group or NULL.
*/
struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
const char topology[8])
{
struct drm_tile_group *tg;
int ret;
tg = kzalloc(sizeof(*tg), GFP_KERNEL);
if (!tg)
return NULL;
kref_init(&tg->refcount);
memcpy(tg->group_data, topology, 8);
tg->dev = dev;
mutex_lock(&dev->mode_config.idr_mutex);
ret = idr_alloc(&dev->mode_config.tile_idr, tg, 1, 0, GFP_KERNEL);
if (ret >= 0) {
tg->id = ret;
} else {
kfree(tg);
tg = NULL;
}
mutex_unlock(&dev->mode_config.idr_mutex);
return tg;
}
EXPORT_SYMBOL(drm_mode_create_tile_group);
// SPDX-License-Identifier: GPL-2.0
/*
* fs/sysfs/dir.c - sysfs core and dir operation implementation
*
* Copyright (c) 2001-3 Patrick Mochel
* Copyright (c) 2007 SUSE Linux Products GmbH
* Copyright (c) 2007 Tejun Heo <teheo@suse.de>
*
* Please see Documentation/filesystems/sysfs.rst for more information.
*/
#define pr_fmt(fmt) "sysfs: " fmt
#include <linux/fs.h>
#include <linux/kobject.h>
#include <linux/slab.h>
#include "sysfs.h"
DEFINE_SPINLOCK(sysfs_symlink_target_lock);
void sysfs_warn_dup(struct kernfs_node *parent, const char *name)
{
char *buf;
buf = kzalloc(PATH_MAX, GFP_KERNEL);
if (buf)
kernfs_path(parent, buf, PATH_MAX);
pr_warn("cannot create duplicate filename '%s/%s'\n", buf, name);
dump_stack();
kfree(buf);
}
/**
* sysfs_create_dir_ns - create a directory for an object with a namespace tag
* @kobj: object we're creating directory for
* @ns: the namespace tag to use
*/
int sysfs_create_dir_ns(struct kobject *kobj, const void *ns)
{
struct kernfs_node *parent, *kn;
kuid_t uid;
kgid_t gid;
if (WARN_ON(!kobj))
return -EINVAL;
if (kobj->parent) parent = kobj->parent->sd;
else
parent = sysfs_root_kn; if (!parent)
return -ENOENT;
kobject_get_ownership(kobj, &uid, &gid);
kn = kernfs_create_dir_ns(parent, kobject_name(kobj), 0755, uid, gid,
kobj, ns);
if (IS_ERR(kn)) {
if (PTR_ERR(kn) == -EEXIST) sysfs_warn_dup(parent, kobject_name(kobj)); return PTR_ERR(kn);
}
kobj->sd = kn; return 0;
}
/**
* sysfs_remove_dir - remove an object's directory.
* @kobj: object.
*
* The only thing special about this is that we remove any files in
* the directory before we remove the directory, and we've inlined
* what used to be sysfs_rmdir() below, instead of calling separately.
*/
void sysfs_remove_dir(struct kobject *kobj)
{
struct kernfs_node *kn = kobj->sd;
/*
* In general, kobject owner is responsible for ensuring removal
* doesn't race with other operations and sysfs doesn't provide any
* protection; however, when @kobj is used as a symlink target, the
* symlinking entity usually doesn't own @kobj and thus has no
* control over removal. @kobj->sd may be removed anytime
* and symlink code may end up dereferencing an already freed node.
*
* sysfs_symlink_target_lock synchronizes @kobj->sd
* disassociation against symlink operations so that symlink code
* can safely dereference @kobj->sd.
*/
spin_lock(&sysfs_symlink_target_lock);
kobj->sd = NULL;
spin_unlock(&sysfs_symlink_target_lock);
if (kn) {
WARN_ON_ONCE(kernfs_type(kn) != KERNFS_DIR); kernfs_remove(kn);
}
}
int sysfs_rename_dir_ns(struct kobject *kobj, const char *new_name,
const void *new_ns)
{
struct kernfs_node *parent;
int ret;
parent = kernfs_get_parent(kobj->sd);
ret = kernfs_rename_ns(kobj->sd, parent, new_name, new_ns);
kernfs_put(parent);
return ret;
}
int sysfs_move_dir_ns(struct kobject *kobj, struct kobject *new_parent_kobj,
const void *new_ns)
{
struct kernfs_node *kn = kobj->sd;
struct kernfs_node *new_parent;
new_parent = new_parent_kobj && new_parent_kobj->sd ?
new_parent_kobj->sd : sysfs_root_kn;
return kernfs_rename_ns(kn, new_parent, kn->name, new_ns);
}
/**
* sysfs_create_mount_point - create an always empty directory
* @parent_kobj: kobject that will contain this always empty directory
* @name: The name of the always empty directory to add
*/
int sysfs_create_mount_point(struct kobject *parent_kobj, const char *name)
{
struct kernfs_node *kn, *parent = parent_kobj->sd;
kn = kernfs_create_empty_dir(parent, name);
if (IS_ERR(kn)) {
if (PTR_ERR(kn) == -EEXIST)
sysfs_warn_dup(parent, name);
return PTR_ERR(kn);
}
return 0;
}
EXPORT_SYMBOL_GPL(sysfs_create_mount_point);
/**
* sysfs_remove_mount_point - remove an always empty directory.
* @parent_kobj: kobject that will contain this always empty directory
* @name: The name of the always empty directory to remove
*
*/
void sysfs_remove_mount_point(struct kobject *parent_kobj, const char *name)
{
struct kernfs_node *parent = parent_kobj->sd;
kernfs_remove_by_name_ns(parent, name, NULL);
}
EXPORT_SYMBOL_GPL(sysfs_remove_mount_point);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* include/linux/writeback.h
*/
#ifndef WRITEBACK_H
#define WRITEBACK_H
#include <linux/sched.h>
#include <linux/workqueue.h>
#include <linux/fs.h>
#include <linux/flex_proportions.h>
#include <linux/backing-dev-defs.h>
#include <linux/blk_types.h>
#include <linux/pagevec.h>
struct bio;
DECLARE_PER_CPU(int, dirty_throttle_leaks);
/*
* The global dirty threshold is normally equal to the global dirty limit,
* except when the system suddenly allocates a lot of anonymous memory and
* knocks down the global dirty threshold quickly, in which case the global
* dirty limit will follow down slowly to prevent livelocking all dirtier tasks.
*/
#define DIRTY_SCOPE 8
struct backing_dev_info;
/*
* fs/fs-writeback.c
*/
enum writeback_sync_modes {
WB_SYNC_NONE, /* Don't wait on anything */
WB_SYNC_ALL, /* Wait on every mapping */
};
/*
* A control structure which tells the writeback code what to do. These are
* always on the stack, and hence need no locking. They are always initialised
* in a manner such that unspecified fields are set to zero.
*/
struct writeback_control {
/* public fields that can be set and/or consumed by the caller: */
long nr_to_write; /* Write this many pages, and decrement
this for each page written */
long pages_skipped; /* Pages which were not written */
/*
* For a_ops->writepages(): if start or end are non-zero then this is
* a hint that the filesystem need only write out the pages inside that
* byterange. The byte at `end' is included in the writeout request.
*/
loff_t range_start;
loff_t range_end;
enum writeback_sync_modes sync_mode;
unsigned for_kupdate:1; /* A kupdate writeback */
unsigned for_background:1; /* A background writeback */
unsigned tagged_writepages:1; /* tag-and-write to avoid livelock */
unsigned for_reclaim:1; /* Invoked from the page allocator */
unsigned range_cyclic:1; /* range_start is cyclic */
unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */
unsigned unpinned_netfs_wb:1; /* Cleared I_PINNING_NETFS_WB */
/*
* When writeback IOs are bounced through async layers, only the
* initial synchronous phase should be accounted towards inode
* cgroup ownership arbitration to avoid confusion. Later stages
* can set the following flag to disable the accounting.
*/
unsigned no_cgroup_owner:1;
/* To enable batching of swap writes to non-block-device backends,
* "plug" can be set point to a 'struct swap_iocb *'. When all swap
* writes have been submitted, if with swap_iocb is not NULL,
* swap_write_unplug() should be called.
*/
struct swap_iocb **swap_plug;
/* internal fields used by the ->writepages implementation: */
struct folio_batch fbatch;
pgoff_t index;
int saved_err;
#ifdef CONFIG_CGROUP_WRITEBACK
struct bdi_writeback *wb; /* wb this writeback is issued under */
struct inode *inode; /* inode being written out */
/* foreign inode detection, see wbc_detach_inode() */
int wb_id; /* current wb id */
int wb_lcand_id; /* last foreign candidate wb id */
int wb_tcand_id; /* this foreign candidate wb id */
size_t wb_bytes; /* bytes written by current wb */
size_t wb_lcand_bytes; /* bytes written by last candidate */
size_t wb_tcand_bytes; /* bytes written by this candidate */
#endif
};
static inline blk_opf_t wbc_to_write_flags(struct writeback_control *wbc)
{
blk_opf_t flags = 0;
if (wbc->sync_mode == WB_SYNC_ALL)
flags |= REQ_SYNC;
else if (wbc->for_kupdate || wbc->for_background)
flags |= REQ_BACKGROUND;
return flags;
}
#ifdef CONFIG_CGROUP_WRITEBACK
#define wbc_blkcg_css(wbc) \
((wbc)->wb ? (wbc)->wb->blkcg_css : blkcg_root_css)
#else
#define wbc_blkcg_css(wbc) (blkcg_root_css)
#endif /* CONFIG_CGROUP_WRITEBACK */
/*
* A wb_domain represents a domain that wb's (bdi_writeback's) belong to
* and are measured against each other in. There always is one global
* domain, global_wb_domain, that every wb in the system is a member of.
* This allows measuring the relative bandwidth of each wb to distribute
* dirtyable memory accordingly.
*/
struct wb_domain {
spinlock_t lock;
/*
* Scale the writeback cache size proportional to the relative
* writeout speed.
*
* We do this by keeping a floating proportion between BDIs, based
* on page writeback completions [end_page_writeback()]. Those
* devices that write out pages fastest will get the larger share,
* while the slower will get a smaller share.
*
* We use page writeout completions because we are interested in
* getting rid of dirty pages. Having them written out is the
* primary goal.
*
* We introduce a concept of time, a period over which we measure
* these events, because demand can/will vary over time. The length
* of this period itself is measured in page writeback completions.
*/
struct fprop_global completions;
struct timer_list period_timer; /* timer for aging of completions */
unsigned long period_time;
/*
* The dirtyable memory and dirty threshold could be suddenly
* knocked down by a large amount (eg. on the startup of KVM in a
* swapless system). This may throw the system into deep dirty
* exceeded state and throttle heavy/light dirtiers alike. To
* retain good responsiveness, maintain global_dirty_limit for
* tracking slowly down to the knocked down dirty threshold.
*
* Both fields are protected by ->lock.
*/
unsigned long dirty_limit_tstamp;
unsigned long dirty_limit;
};
/**
* wb_domain_size_changed - memory available to a wb_domain has changed
* @dom: wb_domain of interest
*
* This function should be called when the amount of memory available to
* @dom has changed. It resets @dom's dirty limit parameters to prevent
* the past values which don't match the current configuration from skewing
* dirty throttling. Without this, when memory size of a wb_domain is
* greatly reduced, the dirty throttling logic may allow too many pages to
* be dirtied leading to consecutive unnecessary OOMs and may get stuck in
* that situation.
*/
static inline void wb_domain_size_changed(struct wb_domain *dom)
{
spin_lock(&dom->lock);
dom->dirty_limit_tstamp = jiffies;
dom->dirty_limit = 0;
spin_unlock(&dom->lock);
}
/*
* fs/fs-writeback.c
*/
struct bdi_writeback;
void writeback_inodes_sb(struct super_block *, enum wb_reason reason);
void writeback_inodes_sb_nr(struct super_block *, unsigned long nr,
enum wb_reason reason);
void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason);
void sync_inodes_sb(struct super_block *);
void wakeup_flusher_threads(enum wb_reason reason);
void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
enum wb_reason reason);
void inode_wait_for_writeback(struct inode *inode);
void inode_io_list_del(struct inode *inode);
/* writeback.h requires fs.h; it, too, is not included from here. */
static inline void wait_on_inode(struct inode *inode)
{
wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE);
}
#ifdef CONFIG_CGROUP_WRITEBACK
#include <linux/cgroup.h>
#include <linux/bio.h>
void __inode_attach_wb(struct inode *inode, struct folio *folio);
void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
struct inode *inode)
__releases(&inode->i_lock);
void wbc_detach_inode(struct writeback_control *wbc);
void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page,
size_t bytes);
int cgroup_writeback_by_id(u64 bdi_id, int memcg_id,
enum wb_reason reason, struct wb_completion *done);
void cgroup_writeback_umount(void);
bool cleanup_offline_cgwb(struct bdi_writeback *wb);
/**
* inode_attach_wb - associate an inode with its wb
* @inode: inode of interest
* @folio: folio being dirtied (may be NULL)
*
* If @inode doesn't have its wb, associate it with the wb matching the
* memcg of @folio or, if @folio is NULL, %current. May be called w/ or w/o
* @inode->i_lock.
*/
static inline void inode_attach_wb(struct inode *inode, struct folio *folio)
{
if (!inode->i_wb)
__inode_attach_wb(inode, folio);
}
/**
* inode_detach_wb - disassociate an inode from its wb
* @inode: inode of interest
*
* @inode is being freed. Detach from its wb.
*/
static inline void inode_detach_wb(struct inode *inode)
{
if (inode->i_wb) { WARN_ON_ONCE(!(inode->i_state & I_CLEAR)); wb_put(inode->i_wb); inode->i_wb = NULL;
}
}
/**
* wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite
* @wbc: writeback_control of interest
* @inode: target inode
*
* This function is to be used by __filemap_fdatawrite_range(), which is an
* alternative entry point into writeback code, and first ensures @inode is
* associated with a bdi_writeback and attaches it to @wbc.
*/
static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
struct inode *inode)
{
spin_lock(&inode->i_lock);
inode_attach_wb(inode, NULL);
wbc_attach_and_unlock_inode(wbc, inode);
}
/**
* wbc_init_bio - writeback specific initializtion of bio
* @wbc: writeback_control for the writeback in progress
* @bio: bio to be initialized
*
* @bio is a part of the writeback in progress controlled by @wbc. Perform
* writeback specific initialization. This is used to apply the cgroup
* writeback context. Must be called after the bio has been associated with
* a device.
*/
static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
{
/*
* pageout() path doesn't attach @wbc to the inode being written
* out. This is intentional as we don't want the function to block
* behind a slow cgroup. Ultimately, we want pageout() to kick off
* regular writeback instead of writing things out itself.
*/
if (wbc->wb)
bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css);
}
#else /* CONFIG_CGROUP_WRITEBACK */
static inline void inode_attach_wb(struct inode *inode, struct folio *folio)
{
}
static inline void inode_detach_wb(struct inode *inode)
{
}
static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
struct inode *inode)
__releases(&inode->i_lock)
{
spin_unlock(&inode->i_lock);
}
static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
struct inode *inode)
{
}
static inline void wbc_detach_inode(struct writeback_control *wbc)
{
}
static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
{
}
static inline void wbc_account_cgroup_owner(struct writeback_control *wbc,
struct page *page, size_t bytes)
{
}
static inline void cgroup_writeback_umount(void)
{
}
#endif /* CONFIG_CGROUP_WRITEBACK */
/*
* mm/page-writeback.c
*/
void laptop_io_completion(struct backing_dev_info *info);
void laptop_sync_completion(void);
void laptop_mode_timer_fn(struct timer_list *t);
bool node_dirty_ok(struct pglist_data *pgdat);
int wb_domain_init(struct wb_domain *dom, gfp_t gfp);
#ifdef CONFIG_CGROUP_WRITEBACK
void wb_domain_exit(struct wb_domain *dom);
#endif
extern struct wb_domain global_wb_domain;
/* These are exported to sysctl. */
extern unsigned int dirty_writeback_interval;
extern unsigned int dirty_expire_interval;
extern unsigned int dirtytime_expire_interval;
extern int laptop_mode;
int dirtytime_interval_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos);
void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty);
unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh);
void wb_update_bandwidth(struct bdi_writeback *wb);
/* Invoke balance dirty pages in async mode. */
#define BDP_ASYNC 0x0001
void balance_dirty_pages_ratelimited(struct address_space *mapping);
int balance_dirty_pages_ratelimited_flags(struct address_space *mapping,
unsigned int flags);
bool wb_over_bg_thresh(struct bdi_writeback *wb);
struct folio *writeback_iter(struct address_space *mapping,
struct writeback_control *wbc, struct folio *folio, int *error);
typedef int (*writepage_t)(struct folio *folio, struct writeback_control *wbc,
void *data);
int write_cache_pages(struct address_space *mapping,
struct writeback_control *wbc, writepage_t writepage,
void *data);
int do_writepages(struct address_space *mapping, struct writeback_control *wbc);
void writeback_set_ratelimit(void);
void tag_pages_for_writeback(struct address_space *mapping,
pgoff_t start, pgoff_t end);
bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio);
bool folio_redirty_for_writepage(struct writeback_control *, struct folio *);
bool redirty_page_for_writepage(struct writeback_control *, struct page *);
void sb_mark_inode_writeback(struct inode *inode);
void sb_clear_inode_writeback(struct inode *inode);
#endif /* WRITEBACK_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MNT_IDMAPPING_H
#define _LINUX_MNT_IDMAPPING_H
#include <linux/types.h>
#include <linux/uidgid.h>
struct mnt_idmap;
struct user_namespace;
extern struct mnt_idmap nop_mnt_idmap;
extern struct user_namespace init_user_ns;
typedef struct {
uid_t val;
} vfsuid_t;
typedef struct {
gid_t val;
} vfsgid_t;
static_assert(sizeof(vfsuid_t) == sizeof(kuid_t));
static_assert(sizeof(vfsgid_t) == sizeof(kgid_t));
static_assert(offsetof(vfsuid_t, val) == offsetof(kuid_t, val));
static_assert(offsetof(vfsgid_t, val) == offsetof(kgid_t, val));
#ifdef CONFIG_MULTIUSER
static inline uid_t __vfsuid_val(vfsuid_t uid)
{
return uid.val;
}
static inline gid_t __vfsgid_val(vfsgid_t gid)
{
return gid.val;
}
#else
static inline uid_t __vfsuid_val(vfsuid_t uid)
{
return 0;
}
static inline gid_t __vfsgid_val(vfsgid_t gid)
{
return 0;
}
#endif
static inline bool vfsuid_valid(vfsuid_t uid)
{
return __vfsuid_val(uid) != (uid_t)-1;
}
static inline bool vfsgid_valid(vfsgid_t gid)
{
return __vfsgid_val(gid) != (gid_t)-1;
}
static inline bool vfsuid_eq(vfsuid_t left, vfsuid_t right)
{
return vfsuid_valid(left) && __vfsuid_val(left) == __vfsuid_val(right);
}
static inline bool vfsgid_eq(vfsgid_t left, vfsgid_t right)
{
return vfsgid_valid(left) && __vfsgid_val(left) == __vfsgid_val(right);
}
/**
* vfsuid_eq_kuid - check whether kuid and vfsuid have the same value
* @vfsuid: the vfsuid to compare
* @kuid: the kuid to compare
*
* Check whether @vfsuid and @kuid have the same values.
*
* Return: true if @vfsuid and @kuid have the same value, false if not.
* Comparison between two invalid uids returns false.
*/
static inline bool vfsuid_eq_kuid(vfsuid_t vfsuid, kuid_t kuid)
{
return vfsuid_valid(vfsuid) && __vfsuid_val(vfsuid) == __kuid_val(kuid);
}
/**
* vfsgid_eq_kgid - check whether kgid and vfsgid have the same value
* @vfsgid: the vfsgid to compare
* @kgid: the kgid to compare
*
* Check whether @vfsgid and @kgid have the same values.
*
* Return: true if @vfsgid and @kgid have the same value, false if not.
* Comparison between two invalid gids returns false.
*/
static inline bool vfsgid_eq_kgid(vfsgid_t vfsgid, kgid_t kgid)
{
return vfsgid_valid(vfsgid) && __vfsgid_val(vfsgid) == __kgid_val(kgid);
}
/*
* vfs{g,u}ids are created from k{g,u}ids.
* We don't allow them to be created from regular {u,g}id.
*/
#define VFSUIDT_INIT(val) (vfsuid_t){ __kuid_val(val) }
#define VFSGIDT_INIT(val) (vfsgid_t){ __kgid_val(val) }
#define INVALID_VFSUID VFSUIDT_INIT(INVALID_UID)
#define INVALID_VFSGID VFSGIDT_INIT(INVALID_GID)
/*
* Allow a vfs{g,u}id to be used as a k{g,u}id where we want to compare
* whether the mapped value is identical to value of a k{g,u}id.
*/
#define AS_KUIDT(val) (kuid_t){ __vfsuid_val(val) }
#define AS_KGIDT(val) (kgid_t){ __vfsgid_val(val) }
int vfsgid_in_group_p(vfsgid_t vfsgid);
struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap);
void mnt_idmap_put(struct mnt_idmap *idmap);
vfsuid_t make_vfsuid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, kuid_t kuid);
vfsgid_t make_vfsgid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, kgid_t kgid);
kuid_t from_vfsuid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, vfsuid_t vfsuid);
kgid_t from_vfsgid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns, vfsgid_t vfsgid);
/**
* vfsuid_has_fsmapping - check whether a vfsuid maps into the filesystem
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
* @vfsuid: vfsuid to be mapped
*
* Check whether @vfsuid has a mapping in the filesystem idmapping. Use this
* function to check whether the filesystem idmapping has a mapping for
* @vfsuid.
*
* Return: true if @vfsuid has a mapping in the filesystem, false if not.
*/
static inline bool vfsuid_has_fsmapping(struct mnt_idmap *idmap,
struct user_namespace *fs_userns,
vfsuid_t vfsuid)
{
return uid_valid(from_vfsuid(idmap, fs_userns, vfsuid));
}
static inline bool vfsuid_has_mapping(struct user_namespace *userns,
vfsuid_t vfsuid)
{
return from_kuid(userns, AS_KUIDT(vfsuid)) != (uid_t)-1;
}
/**
* vfsuid_into_kuid - convert vfsuid into kuid
* @vfsuid: the vfsuid to convert
*
* This can be used when a vfsuid is committed as a kuid.
*
* Return: a kuid with the value of @vfsuid
*/
static inline kuid_t vfsuid_into_kuid(vfsuid_t vfsuid)
{
return AS_KUIDT(vfsuid);
}
/**
* vfsgid_has_fsmapping - check whether a vfsgid maps into the filesystem
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
* @vfsgid: vfsgid to be mapped
*
* Check whether @vfsgid has a mapping in the filesystem idmapping. Use this
* function to check whether the filesystem idmapping has a mapping for
* @vfsgid.
*
* Return: true if @vfsgid has a mapping in the filesystem, false if not.
*/
static inline bool vfsgid_has_fsmapping(struct mnt_idmap *idmap,
struct user_namespace *fs_userns,
vfsgid_t vfsgid)
{
return gid_valid(from_vfsgid(idmap, fs_userns, vfsgid));
}
static inline bool vfsgid_has_mapping(struct user_namespace *userns,
vfsgid_t vfsgid)
{
return from_kgid(userns, AS_KGIDT(vfsgid)) != (gid_t)-1;
}
/**
* vfsgid_into_kgid - convert vfsgid into kgid
* @vfsgid: the vfsgid to convert
*
* This can be used when a vfsgid is committed as a kgid.
*
* Return: a kgid with the value of @vfsgid
*/
static inline kgid_t vfsgid_into_kgid(vfsgid_t vfsgid)
{
return AS_KGIDT(vfsgid);
}
/**
* mapped_fsuid - return caller's fsuid mapped according to an idmapping
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
*
* Use this helper to initialize a new vfs or filesystem object based on
* the caller's fsuid. A common example is initializing the i_uid field of
* a newly allocated inode triggered by a creation event such as mkdir or
* O_CREAT. Other examples include the allocation of quotas for a specific
* user.
*
* Return: the caller's current fsuid mapped up according to @idmap.
*/
static inline kuid_t mapped_fsuid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns)
{
return from_vfsuid(idmap, fs_userns, VFSUIDT_INIT(current_fsuid()));
}
/**
* mapped_fsgid - return caller's fsgid mapped according to an idmapping
* @idmap: the mount's idmapping
* @fs_userns: the filesystem's idmapping
*
* Use this helper to initialize a new vfs or filesystem object based on
* the caller's fsgid. A common example is initializing the i_gid field of
* a newly allocated inode triggered by a creation event such as mkdir or
* O_CREAT. Other examples include the allocation of quotas for a specific
* user.
*
* Return: the caller's current fsgid mapped up according to @idmap.
*/
static inline kgid_t mapped_fsgid(struct mnt_idmap *idmap,
struct user_namespace *fs_userns)
{
return from_vfsgid(idmap, fs_userns, VFSGIDT_INIT(current_fsgid()));
}
#endif /* _LINUX_MNT_IDMAPPING_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* mm/readahead.c - address_space-level file readahead.
*
* Copyright (C) 2002, Linus Torvalds
*
* 09Apr2002 Andrew Morton
* Initial version.
*/
/**
* DOC: Readahead Overview
*
* Readahead is used to read content into the page cache before it is
* explicitly requested by the application. Readahead only ever
* attempts to read folios that are not yet in the page cache. If a
* folio is present but not up-to-date, readahead will not try to read
* it. In that case a simple ->read_folio() will be requested.
*
* Readahead is triggered when an application read request (whether a
* system call or a page fault) finds that the requested folio is not in
* the page cache, or that it is in the page cache and has the
* readahead flag set. This flag indicates that the folio was read
* as part of a previous readahead request and now that it has been
* accessed, it is time for the next readahead.
*
* Each readahead request is partly synchronous read, and partly async
* readahead. This is reflected in the struct file_ra_state which
* contains ->size being the total number of pages, and ->async_size
* which is the number of pages in the async section. The readahead
* flag will be set on the first folio in this async section to trigger
* a subsequent readahead. Once a series of sequential reads has been
* established, there should be no need for a synchronous component and
* all readahead request will be fully asynchronous.
*
* When either of the triggers causes a readahead, three numbers need
* to be determined: the start of the region to read, the size of the
* region, and the size of the async tail.
*
* The start of the region is simply the first page address at or after
* the accessed address, which is not currently populated in the page
* cache. This is found with a simple search in the page cache.
*
* The size of the async tail is determined by subtracting the size that
* was explicitly requested from the determined request size, unless
* this would be less than zero - then zero is used. NOTE THIS
* CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED
* PAGE. ALSO THIS CALCULATION IS NOT USED CONSISTENTLY.
*
* The size of the region is normally determined from the size of the
* previous readahead which loaded the preceding pages. This may be
* discovered from the struct file_ra_state for simple sequential reads,
* or from examining the state of the page cache when multiple
* sequential reads are interleaved. Specifically: where the readahead
* was triggered by the readahead flag, the size of the previous
* readahead is assumed to be the number of pages from the triggering
* page to the start of the new readahead. In these cases, the size of
* the previous readahead is scaled, often doubled, for the new
* readahead, though see get_next_ra_size() for details.
*
* If the size of the previous read cannot be determined, the number of
* preceding pages in the page cache is used to estimate the size of
* a previous read. This estimate could easily be misled by random
* reads being coincidentally adjacent, so it is ignored unless it is
* larger than the current request, and it is not scaled up, unless it
* is at the start of file.
*
* In general readahead is accelerated at the start of the file, as
* reads from there are often sequential. There are other minor
* adjustments to the readahead size in various special cases and these
* are best discovered by reading the code.
*
* The above calculation, based on the previous readahead size,
* determines the size of the readahead, to which any requested read
* size may be added.
*
* Readahead requests are sent to the filesystem using the ->readahead()
* address space operation, for which mpage_readahead() is a canonical
* implementation. ->readahead() should normally initiate reads on all
* folios, but may fail to read any or all folios without causing an I/O
* error. The page cache reading code will issue a ->read_folio() request
* for any folio which ->readahead() did not read, and only an error
* from this will be final.
*
* ->readahead() will generally call readahead_folio() repeatedly to get
* each folio from those prepared for readahead. It may fail to read a
* folio by:
*
* * not calling readahead_folio() sufficiently many times, effectively
* ignoring some folios, as might be appropriate if the path to
* storage is congested.
*
* * failing to actually submit a read request for a given folio,
* possibly due to insufficient resources, or
*
* * getting an error during subsequent processing of a request.
*
* In the last two cases, the folio should be unlocked by the filesystem
* to indicate that the read attempt has failed. In the first case the
* folio will be unlocked by the VFS.
*
* Those folios not in the final ``async_size`` of the request should be
* considered to be important and ->readahead() should not fail them due
* to congestion or temporary resource unavailability, but should wait
* for necessary resources (e.g. memory or indexing information) to
* become available. Folios in the final ``async_size`` may be
* considered less urgent and failure to read them is more acceptable.
* In this case it is best to use filemap_remove_folio() to remove the
* folios from the page cache as is automatically done for folios that
* were not fetched with readahead_folio(). This will allow a
* subsequent synchronous readahead request to try them again. If they
* are left in the page cache, then they will be read individually using
* ->read_folio() which may be less efficient.
*/
#include <linux/blkdev.h>
#include <linux/kernel.h>
#include <linux/dax.h>
#include <linux/gfp.h>
#include <linux/export.h>
#include <linux/backing-dev.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/pagemap.h>
#include <linux/psi.h>
#include <linux/syscalls.h>
#include <linux/file.h>
#include <linux/mm_inline.h>
#include <linux/blk-cgroup.h>
#include <linux/fadvise.h>
#include <linux/sched/mm.h>
#include "internal.h"
/*
* Initialise a struct file's readahead state. Assumes that the caller has
* memset *ra to zero.
*/
void
file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping)
{
ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;
ra->prev_pos = -1;
}
EXPORT_SYMBOL_GPL(file_ra_state_init);
static void read_pages(struct readahead_control *rac)
{
const struct address_space_operations *aops = rac->mapping->a_ops;
struct folio *folio;
struct blk_plug plug;
if (!readahead_count(rac))
return;
if (unlikely(rac->_workingset))
psi_memstall_enter(&rac->_pflags);
blk_start_plug(&plug);
if (aops->readahead) {
aops->readahead(rac);
/*
* Clean up the remaining folios. The sizes in ->ra
* may be used to size the next readahead, so make sure
* they accurately reflect what happened.
*/
while ((folio = readahead_folio(rac)) != NULL) {
unsigned long nr = folio_nr_pages(folio);
folio_get(folio);
rac->ra->size -= nr;
if (rac->ra->async_size >= nr) {
rac->ra->async_size -= nr;
filemap_remove_folio(folio);
}
folio_unlock(folio);
folio_put(folio);
}
} else {
while ((folio = readahead_folio(rac)) != NULL)
aops->read_folio(rac->file, folio);
}
blk_finish_plug(&plug);
if (unlikely(rac->_workingset))
psi_memstall_leave(&rac->_pflags);
rac->_workingset = false;
BUG_ON(readahead_count(rac));
}
/**
* page_cache_ra_unbounded - Start unchecked readahead.
* @ractl: Readahead control.
* @nr_to_read: The number of pages to read.
* @lookahead_size: Where to start the next readahead.
*
* This function is for filesystems to call when they want to start
* readahead beyond a file's stated i_size. This is almost certainly
* not the function you want to call. Use page_cache_async_readahead()
* or page_cache_sync_readahead() instead.
*
* Context: File is referenced by caller. Mutexes may be held by caller.
* May sleep, but will not reenter filesystem to reclaim memory.
*/
void page_cache_ra_unbounded(struct readahead_control *ractl,
unsigned long nr_to_read, unsigned long lookahead_size)
{
struct address_space *mapping = ractl->mapping;
unsigned long index = readahead_index(ractl);
gfp_t gfp_mask = readahead_gfp_mask(mapping);
unsigned long i;
/*
* Partway through the readahead operation, we will have added
* locked pages to the page cache, but will not yet have submitted
* them for I/O. Adding another page may need to allocate memory,
* which can trigger memory reclaim. Telling the VM we're in
* the middle of a filesystem operation will cause it to not
* touch file-backed pages, preventing a deadlock. Most (all?)
* filesystems already specify __GFP_NOFS in their mapping's
* gfp_mask, but let's be explicit here.
*/
unsigned int nofs = memalloc_nofs_save();
filemap_invalidate_lock_shared(mapping);
/*
* Preallocate as many pages as we will need.
*/
for (i = 0; i < nr_to_read; i++) {
struct folio *folio = xa_load(&mapping->i_pages, index + i);
if (folio && !xa_is_value(folio)) {
/*
* Page already present? Kick off the current batch
* of contiguous pages before continuing with the
* next batch. This page may be the one we would
* have intended to mark as Readahead, but we don't
* have a stable reference to this page, and it's
* not worth getting one just for that.
*/
read_pages(ractl);
ractl->_index++;
i = ractl->_index + ractl->_nr_pages - index - 1;
continue;
}
folio = filemap_alloc_folio(gfp_mask, 0);
if (!folio)
break;
if (filemap_add_folio(mapping, folio, index + i,
gfp_mask) < 0) {
folio_put(folio);
read_pages(ractl);
ractl->_index++;
i = ractl->_index + ractl->_nr_pages - index - 1;
continue;
}
if (i == nr_to_read - lookahead_size)
folio_set_readahead(folio);
ractl->_workingset |= folio_test_workingset(folio);
ractl->_nr_pages++;
}
/*
* Now start the IO. We ignore I/O errors - if the folio is not
* uptodate then the caller will launch read_folio again, and
* will then handle the error.
*/
read_pages(ractl);
filemap_invalidate_unlock_shared(mapping);
memalloc_nofs_restore(nofs);
}
EXPORT_SYMBOL_GPL(page_cache_ra_unbounded);
/*
* do_page_cache_ra() actually reads a chunk of disk. It allocates
* the pages first, then submits them for I/O. This avoids the very bad
* behaviour which would occur if page allocations are causing VM writeback.
* We really don't want to intermingle reads and writes like that.
*/
static void do_page_cache_ra(struct readahead_control *ractl,
unsigned long nr_to_read, unsigned long lookahead_size)
{
struct inode *inode = ractl->mapping->host;
unsigned long index = readahead_index(ractl);
loff_t isize = i_size_read(inode);
pgoff_t end_index; /* The last page we want to read */
if (isize == 0)
return;
end_index = (isize - 1) >> PAGE_SHIFT;
if (index > end_index)
return;
/* Don't read past the page containing the last byte of the file */
if (nr_to_read > end_index - index)
nr_to_read = end_index - index + 1;
page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size);
}
/*
* Chunk the readahead into 2 megabyte units, so that we don't pin too much
* memory at once.
*/
void force_page_cache_ra(struct readahead_control *ractl,
unsigned long nr_to_read)
{
struct address_space *mapping = ractl->mapping;
struct file_ra_state *ra = ractl->ra;
struct backing_dev_info *bdi = inode_to_bdi(mapping->host);
unsigned long max_pages, index;
if (unlikely(!mapping->a_ops->read_folio && !mapping->a_ops->readahead))
return;
/*
* If the request exceeds the readahead window, allow the read to
* be up to the optimal hardware IO size
*/
index = readahead_index(ractl);
max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);
nr_to_read = min_t(unsigned long, nr_to_read, max_pages);
while (nr_to_read) {
unsigned long this_chunk = (2 * 1024 * 1024) / PAGE_SIZE;
if (this_chunk > nr_to_read)
this_chunk = nr_to_read;
ractl->_index = index;
do_page_cache_ra(ractl, this_chunk, 0);
index += this_chunk;
nr_to_read -= this_chunk;
}
}
/*
* Set the initial window size, round to next power of 2 and square
* for small size, x 4 for medium, and x 2 for large
* for 128k (32 page) max ra
* 1-2 page = 16k, 3-4 page 32k, 5-8 page = 64k, > 8 page = 128k initial
*/
static unsigned long get_init_ra_size(unsigned long size, unsigned long max)
{
unsigned long newsize = roundup_pow_of_two(size);
if (newsize <= max / 32)
newsize = newsize * 4;
else if (newsize <= max / 4)
newsize = newsize * 2;
else
newsize = max;
return newsize;
}
/*
* Get the previous window size, ramp it up, and
* return it as the new window size.
*/
static unsigned long get_next_ra_size(struct file_ra_state *ra,
unsigned long max)
{
unsigned long cur = ra->size;
if (cur < max / 16)
return 4 * cur;
if (cur <= max / 2)
return 2 * cur;
return max;
}
/*
* On-demand readahead design.
*
* The fields in struct file_ra_state represent the most-recently-executed
* readahead attempt:
*
* |<----- async_size ---------|
* |------------------- size -------------------->|
* |==================#===========================|
* ^start ^page marked with PG_readahead
*
* To overlap application thinking time and disk I/O time, we do
* `readahead pipelining': Do not wait until the application consumed all
* readahead pages and stalled on the missing page at readahead_index;
* Instead, submit an asynchronous readahead I/O as soon as there are
* only async_size pages left in the readahead window. Normally async_size
* will be equal to size, for maximum pipelining.
*
* In interleaved sequential reads, concurrent streams on the same fd can
* be invalidating each other's readahead state. So we flag the new readahead
* page at (start+size-async_size) with PG_readahead, and use it as readahead
* indicator. The flag won't be set on already cached pages, to avoid the
* readahead-for-nothing fuss, saving pointless page cache lookups.
*
* prev_pos tracks the last visited byte in the _previous_ read request.
* It should be maintained by the caller, and will be used for detecting
* small random reads. Note that the readahead algorithm checks loosely
* for sequential patterns. Hence interleaved reads might be served as
* sequential ones.
*
* There is a special-case: if the first page which the application tries to
* read happens to be the first page of the file, it is assumed that a linear
* read is about to happen and the window is immediately set to the initial size
* based on I/O request size and the max_readahead.
*
* The code ramps up the readahead size aggressively at first, but slow down as
* it approaches max_readhead.
*/
/*
* Count contiguously cached pages from @index-1 to @index-@max,
* this count is a conservative estimation of
* - length of the sequential read sequence, or
* - thrashing threshold in memory tight systems
*/
static pgoff_t count_history_pages(struct address_space *mapping,
pgoff_t index, unsigned long max)
{
pgoff_t head;
rcu_read_lock();
head = page_cache_prev_miss(mapping, index - 1, max);
rcu_read_unlock();
return index - 1 - head;
}
/*
* page cache context based readahead
*/
static int try_context_readahead(struct address_space *mapping,
struct file_ra_state *ra,
pgoff_t index,
unsigned long req_size,
unsigned long max)
{
pgoff_t size;
size = count_history_pages(mapping, index, max);
/*
* not enough history pages:
* it could be a random read
*/
if (size <= req_size)
return 0;
/*
* starts from beginning of file:
* it is a strong indication of long-run stream (or whole-file-read)
*/
if (size >= index)
size *= 2;
ra->start = index;
ra->size = min(size + req_size, max);
ra->async_size = 1;
return 1;
}
static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index,
pgoff_t mark, unsigned int order, gfp_t gfp)
{
int err;
struct folio *folio = filemap_alloc_folio(gfp, order);
if (!folio)
return -ENOMEM;
mark = round_down(mark, 1UL << order);
if (index == mark)
folio_set_readahead(folio);
err = filemap_add_folio(ractl->mapping, folio, index, gfp);
if (err) {
folio_put(folio);
return err;
}
ractl->_nr_pages += 1UL << order;
ractl->_workingset |= folio_test_workingset(folio);
return 0;
}
void page_cache_ra_order(struct readahead_control *ractl,
struct file_ra_state *ra, unsigned int new_order)
{
struct address_space *mapping = ractl->mapping;
pgoff_t index = readahead_index(ractl);
pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
pgoff_t mark = index + ra->size - ra->async_size;
int err = 0;
gfp_t gfp = readahead_gfp_mask(mapping);
if (!mapping_large_folio_support(mapping) || ra->size < 4)
goto fallback;
limit = min(limit, index + ra->size - 1);
if (new_order < MAX_PAGECACHE_ORDER) {
new_order += 2;
new_order = min_t(unsigned int, MAX_PAGECACHE_ORDER, new_order);
new_order = min_t(unsigned int, new_order, ilog2(ra->size));
}
filemap_invalidate_lock_shared(mapping);
while (index <= limit) {
unsigned int order = new_order;
/* Align with smaller pages if needed */
if (index & ((1UL << order) - 1))
order = __ffs(index);
/* Don't allocate pages past EOF */
while (index + (1UL << order) - 1 > limit)
order--;
err = ra_alloc_folio(ractl, index, mark, order, gfp);
if (err)
break;
index += 1UL << order;
}
if (index > limit) {
ra->size += index - limit - 1;
ra->async_size += index - limit - 1;
}
read_pages(ractl);
filemap_invalidate_unlock_shared(mapping);
/*
* If there were already pages in the page cache, then we may have
* left some gaps. Let the regular readahead code take care of this
* situation.
*/
if (!err)
return;
fallback:
do_page_cache_ra(ractl, ra->size, ra->async_size);
}
/*
* A minimal readahead algorithm for trivial sequential/random reads.
*/
static void ondemand_readahead(struct readahead_control *ractl,
struct folio *folio, unsigned long req_size)
{
struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);
struct file_ra_state *ra = ractl->ra;
unsigned long max_pages = ra->ra_pages;
unsigned long add_pages;
pgoff_t index = readahead_index(ractl);
pgoff_t expected, prev_index;
unsigned int order = folio ? folio_order(folio) : 0;
/*
* If the request exceeds the readahead window, allow the read to
* be up to the optimal hardware IO size
*/
if (req_size > max_pages && bdi->io_pages > max_pages)
max_pages = min(req_size, bdi->io_pages);
/*
* start of file
*/
if (!index)
goto initial_readahead;
/*
* It's the expected callback index, assume sequential access.
* Ramp up sizes, and push forward the readahead window.
*/
expected = round_down(ra->start + ra->size - ra->async_size,
1UL << order);
if (index == expected || index == (ra->start + ra->size)) {
ra->start += ra->size;
ra->size = get_next_ra_size(ra, max_pages);
ra->async_size = ra->size;
goto readit;
}
/*
* Hit a marked folio without valid readahead state.
* E.g. interleaved reads.
* Query the pagecache for async_size, which normally equals to
* readahead size. Ramp it up and use it as the new readahead size.
*/
if (folio) {
pgoff_t start;
rcu_read_lock();
start = page_cache_next_miss(ractl->mapping, index + 1,
max_pages);
rcu_read_unlock();
if (!start || start - index > max_pages)
return;
ra->start = start;
ra->size = start - index; /* old async_size */
ra->size += req_size;
ra->size = get_next_ra_size(ra, max_pages);
ra->async_size = ra->size;
goto readit;
}
/*
* oversize read
*/
if (req_size > max_pages)
goto initial_readahead;
/*
* sequential cache miss
* trivial case: (index - prev_index) == 1
* unaligned reads: (index - prev_index) == 0
*/
prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;
if (index - prev_index <= 1UL)
goto initial_readahead;
/*
* Query the page cache and look for the traces(cached history pages)
* that a sequential stream would leave behind.
*/
if (try_context_readahead(ractl->mapping, ra, index, req_size,
max_pages))
goto readit;
/*
* standalone, small random read
* Read as is, and do not pollute the readahead state.
*/
do_page_cache_ra(ractl, req_size, 0);
return;
initial_readahead:
ra->start = index;
ra->size = get_init_ra_size(req_size, max_pages);
ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;
readit:
/*
* Will this read hit the readahead marker made by itself?
* If so, trigger the readahead marker hit now, and merge
* the resulted next readahead window into the current one.
* Take care of maximum IO pages as above.
*/
if (index == ra->start && ra->size == ra->async_size) {
add_pages = get_next_ra_size(ra, max_pages);
if (ra->size + add_pages <= max_pages) {
ra->async_size = add_pages;
ra->size += add_pages;
} else {
ra->size = max_pages;
ra->async_size = max_pages >> 1;
}
}
ractl->_index = ra->start;
page_cache_ra_order(ractl, ra, order);
}
void page_cache_sync_ra(struct readahead_control *ractl,
unsigned long req_count)
{
bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);
/*
* Even if readahead is disabled, issue this request as readahead
* as we'll need it to satisfy the requested range. The forced
* readahead will do the right thing and limit the read to just the
* requested range, which we'll set to 1 page for this case.
*/
if (!ractl->ra->ra_pages || blk_cgroup_congested()) {
if (!ractl->file)
return;
req_count = 1;
do_forced_ra = true;
}
/* be dumb */
if (do_forced_ra) {
force_page_cache_ra(ractl, req_count);
return;
}
ondemand_readahead(ractl, NULL, req_count);
}
EXPORT_SYMBOL_GPL(page_cache_sync_ra);
void page_cache_async_ra(struct readahead_control *ractl,
struct folio *folio, unsigned long req_count)
{
/* no readahead */
if (!ractl->ra->ra_pages)
return;
/*
* Same bit is used for PG_readahead and PG_reclaim.
*/
if (folio_test_writeback(folio))
return;
folio_clear_readahead(folio);
if (blk_cgroup_congested())
return;
ondemand_readahead(ractl, folio, req_count);
}
EXPORT_SYMBOL_GPL(page_cache_async_ra);
ssize_t ksys_readahead(int fd, loff_t offset, size_t count)
{
ssize_t ret;
struct fd f;
ret = -EBADF;
f = fdget(fd);
if (!f.file || !(f.file->f_mode & FMODE_READ))
goto out;
/*
* The readahead() syscall is intended to run only on files
* that can execute readahead. If readahead is not possible
* on this file, then we must return -EINVAL.
*/
ret = -EINVAL;
if (!f.file->f_mapping || !f.file->f_mapping->a_ops ||
(!S_ISREG(file_inode(f.file)->i_mode) &&
!S_ISBLK(file_inode(f.file)->i_mode)))
goto out;
ret = vfs_fadvise(f.file, offset, count, POSIX_FADV_WILLNEED);
out:
fdput(f);
return ret;
}
SYSCALL_DEFINE3(readahead, int, fd, loff_t, offset, size_t, count)
{
return ksys_readahead(fd, offset, count);
}
#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_READAHEAD)
COMPAT_SYSCALL_DEFINE4(readahead, int, fd, compat_arg_u64_dual(offset), size_t, count)
{
return ksys_readahead(fd, compat_arg_u64_glue(offset), count);
}
#endif
/**
* readahead_expand - Expand a readahead request
* @ractl: The request to be expanded
* @new_start: The revised start
* @new_len: The revised size of the request
*
* Attempt to expand a readahead request outwards from the current size to the
* specified size by inserting locked pages before and after the current window
* to increase the size to the new window. This may involve the insertion of
* THPs, in which case the window may get expanded even beyond what was
* requested.
*
* The algorithm will stop if it encounters a conflicting page already in the
* pagecache and leave a smaller expansion than requested.
*
* The caller must check for this by examining the revised @ractl object for a
* different expansion than was requested.
*/
void readahead_expand(struct readahead_control *ractl,
loff_t new_start, size_t new_len)
{
struct address_space *mapping = ractl->mapping;
struct file_ra_state *ra = ractl->ra;
pgoff_t new_index, new_nr_pages;
gfp_t gfp_mask = readahead_gfp_mask(mapping);
new_index = new_start / PAGE_SIZE;
/* Expand the leading edge downwards */
while (ractl->_index > new_index) {
unsigned long index = ractl->_index - 1;
struct folio *folio = xa_load(&mapping->i_pages, index);
if (folio && !xa_is_value(folio))
return; /* Folio apparently present */
folio = filemap_alloc_folio(gfp_mask, 0);
if (!folio)
return;
if (filemap_add_folio(mapping, folio, index, gfp_mask) < 0) {
folio_put(folio);
return;
}
if (unlikely(folio_test_workingset(folio)) &&
!ractl->_workingset) {
ractl->_workingset = true;
psi_memstall_enter(&ractl->_pflags);
}
ractl->_nr_pages++;
ractl->_index = folio->index;
}
new_len += new_start - readahead_pos(ractl);
new_nr_pages = DIV_ROUND_UP(new_len, PAGE_SIZE);
/* Expand the trailing edge upwards */
while (ractl->_nr_pages < new_nr_pages) {
unsigned long index = ractl->_index + ractl->_nr_pages;
struct folio *folio = xa_load(&mapping->i_pages, index);
if (folio && !xa_is_value(folio))
return; /* Folio apparently present */
folio = filemap_alloc_folio(gfp_mask, 0);
if (!folio)
return;
if (filemap_add_folio(mapping, folio, index, gfp_mask) < 0) {
folio_put(folio);
return;
}
if (unlikely(folio_test_workingset(folio)) &&
!ractl->_workingset) {
ractl->_workingset = true;
psi_memstall_enter(&ractl->_pflags);
}
ractl->_nr_pages++;
if (ra) {
ra->size++;
ra->async_size++;
}
}
}
EXPORT_SYMBOL(readahead_expand);
// SPDX-License-Identifier: GPL-2.0
/*
* All the USB notify logic
*
* (C) Copyright 2005 Greg Kroah-Hartman <gregkh@suse.de>
*
* notifier functions originally based on those in kernel/sys.c
* but fixed up to not be so broken.
*
* Released under the GPLv2 only.
*/
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/notifier.h>
#include <linux/usb.h>
#include <linux/mutex.h>
#include "usb.h"
static BLOCKING_NOTIFIER_HEAD(usb_notifier_list);
/**
* usb_register_notify - register a notifier callback whenever a usb change happens
* @nb: pointer to the notifier block for the callback events.
*
* These changes are either USB devices or busses being added or removed.
*/
void usb_register_notify(struct notifier_block *nb)
{
blocking_notifier_chain_register(&usb_notifier_list, nb);
}
EXPORT_SYMBOL_GPL(usb_register_notify);
/**
* usb_unregister_notify - unregister a notifier callback
* @nb: pointer to the notifier block for the callback events.
*
* usb_register_notify() must have been previously called for this function
* to work properly.
*/
void usb_unregister_notify(struct notifier_block *nb)
{
blocking_notifier_chain_unregister(&usb_notifier_list, nb);
}
EXPORT_SYMBOL_GPL(usb_unregister_notify);
void usb_notify_add_device(struct usb_device *udev)
{
blocking_notifier_call_chain(&usb_notifier_list, USB_DEVICE_ADD, udev);
}
void usb_notify_remove_device(struct usb_device *udev)
{
blocking_notifier_call_chain(&usb_notifier_list,
USB_DEVICE_REMOVE, udev);
}
void usb_notify_add_bus(struct usb_bus *ubus)
{
blocking_notifier_call_chain(&usb_notifier_list, USB_BUS_ADD, ubus);
}
void usb_notify_remove_bus(struct usb_bus *ubus)
{
blocking_notifier_call_chain(&usb_notifier_list, USB_BUS_REMOVE, ubus);
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_SPINLOCK_H
#define __LINUX_SPINLOCK_H
#define __LINUX_INSIDE_SPINLOCK_H
/*
* include/linux/spinlock.h - generic spinlock/rwlock declarations
*
* here's the role of the various spinlock/rwlock related include files:
*
* on SMP builds:
*
* asm/spinlock_types.h: contains the arch_spinlock_t/arch_rwlock_t and the
* initializers
*
* linux/spinlock_types_raw:
* The raw types and initializers
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* asm/spinlock.h: contains the arch_spin_*()/etc. lowlevel
* implementations, mostly inline assembly code
*
* (also included on UP-debug builds:)
*
* linux/spinlock_api_smp.h:
* contains the prototypes for the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*
* on UP builds:
*
* linux/spinlock_type_up.h:
* contains the generic, simplified UP spinlock type.
* (which is an empty structure on non-debug builds)
*
* linux/spinlock_types_raw:
* The raw RT types and initializers
* linux/spinlock_types.h:
* defines the generic type and initializers
*
* linux/spinlock_up.h:
* contains the arch_spin_*()/etc. version of UP
* builds. (which are NOPs on non-debug, non-preempt
* builds)
*
* (included on UP-non-debug builds:)
*
* linux/spinlock_api_up.h:
* builds the _spin_*() APIs.
*
* linux/spinlock.h: builds the final spin_*() APIs.
*/
#include <linux/typecheck.h>
#include <linux/preempt.h>
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/irqflags.h>
#include <linux/thread_info.h>
#include <linux/stringify.h>
#include <linux/bottom_half.h>
#include <linux/lockdep.h>
#include <linux/cleanup.h>
#include <asm/barrier.h>
#include <asm/mmiowb.h>
/*
* Must define these before including other files, inline functions need them
*/
#define LOCK_SECTION_NAME ".text..lock."KBUILD_BASENAME
#define LOCK_SECTION_START(extra) \
".subsection 1\n\t" \
extra \
".ifndef " LOCK_SECTION_NAME "\n\t" \
LOCK_SECTION_NAME ":\n\t" \
".endif\n"
#define LOCK_SECTION_END \
".previous\n\t"
#define __lockfunc __section(".spinlock.text")
/*
* Pull the arch_spinlock_t and arch_rwlock_t definitions:
*/
#include <linux/spinlock_types.h>
/*
* Pull the arch_spin*() functions/declarations (UP-nondebug doesn't need them):
*/
#ifdef CONFIG_SMP
# include <asm/spinlock.h>
#else
# include <linux/spinlock_up.h>
#endif
#ifdef CONFIG_DEBUG_SPINLOCK
extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
struct lock_class_key *key, short inner);
# define raw_spin_lock_init(lock) \
do { \
static struct lock_class_key __key; \
\
__raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \
} while (0)
#else
# define raw_spin_lock_init(lock) \
do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
#endif
#define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock)
#ifdef arch_spin_is_contended
#define raw_spin_is_contended(lock) arch_spin_is_contended(&(lock)->raw_lock)
#else
#define raw_spin_is_contended(lock) (((void)(lock), 0))
#endif /*arch_spin_is_contended*/
/*
* smp_mb__after_spinlock() provides the equivalent of a full memory barrier
* between program-order earlier lock acquisitions and program-order later
* memory accesses.
*
* This guarantees that the following two properties hold:
*
* 1) Given the snippet:
*
* { X = 0; Y = 0; }
*
* CPU0 CPU1
*
* WRITE_ONCE(X, 1); WRITE_ONCE(Y, 1);
* spin_lock(S); smp_mb();
* smp_mb__after_spinlock(); r1 = READ_ONCE(X);
* r0 = READ_ONCE(Y);
* spin_unlock(S);
*
* it is forbidden that CPU0 does not observe CPU1's store to Y (r0 = 0)
* and CPU1 does not observe CPU0's store to X (r1 = 0); see the comments
* preceding the call to smp_mb__after_spinlock() in __schedule() and in
* try_to_wake_up().
*
* 2) Given the snippet:
*
* { X = 0; Y = 0; }
*
* CPU0 CPU1 CPU2
*
* spin_lock(S); spin_lock(S); r1 = READ_ONCE(Y);
* WRITE_ONCE(X, 1); smp_mb__after_spinlock(); smp_rmb();
* spin_unlock(S); r0 = READ_ONCE(X); r2 = READ_ONCE(X);
* WRITE_ONCE(Y, 1);
* spin_unlock(S);
*
* it is forbidden that CPU0's critical section executes before CPU1's
* critical section (r0 = 1), CPU2 observes CPU1's store to Y (r1 = 1)
* and CPU2 does not observe CPU0's store to X (r2 = 0); see the comments
* preceding the calls to smp_rmb() in try_to_wake_up() for similar
* snippets but "projected" onto two CPUs.
*
* Property (2) upgrades the lock to an RCsc lock.
*
* Since most load-store architectures implement ACQUIRE with an smp_mb() after
* the LL/SC loop, they need no further barriers. Similarly all our TSO
* architectures imply an smp_mb() for each atomic instruction and equally don't
* need more.
*
* Architectures that can implement ACQUIRE better need to take care.
*/
#ifndef smp_mb__after_spinlock
#define smp_mb__after_spinlock() kcsan_mb()
#endif
#ifdef CONFIG_DEBUG_SPINLOCK
extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
extern int do_raw_spin_trylock(raw_spinlock_t *lock);
extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
#else
static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
{
__acquire(lock);
arch_spin_lock(&lock->raw_lock);
mmiowb_spin_lock();
}
static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
{
int ret = arch_spin_trylock(&(lock)->raw_lock);
if (ret)
mmiowb_spin_lock();
return ret;
}
static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
{
mmiowb_spin_unlock();
arch_spin_unlock(&lock->raw_lock);
__release(lock);
}
#endif
/*
* Define the various spin_lock methods. Note we define these
* regardless of whether CONFIG_SMP or CONFIG_PREEMPTION are set. The
* various methods are defined as nops in the case they are not
* required.
*/
#define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock))
#define raw_spin_lock(lock) _raw_spin_lock(lock)
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define raw_spin_lock_nested(lock, subclass) \
_raw_spin_lock_nested(lock, subclass)
# define raw_spin_lock_nest_lock(lock, nest_lock) \
do { \
typecheck(struct lockdep_map *, &(nest_lock)->dep_map);\
_raw_spin_lock_nest_lock(lock, &(nest_lock)->dep_map); \
} while (0)
#else
/*
* Always evaluate the 'subclass' argument to avoid that the compiler
* warns about set-but-not-used variables when building with
* CONFIG_DEBUG_LOCK_ALLOC=n and with W=1.
*/
# define raw_spin_lock_nested(lock, subclass) \
_raw_spin_lock(((void)(subclass), (lock)))
# define raw_spin_lock_nest_lock(lock, nest_lock) _raw_spin_lock(lock)
#endif
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
#define raw_spin_lock_irqsave(lock, flags) \
do { \
typecheck(unsigned long, flags); \
flags = _raw_spin_lock_irqsave(lock); \
} while (0)
#ifdef CONFIG_DEBUG_LOCK_ALLOC
#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
do { \
typecheck(unsigned long, flags); \
flags = _raw_spin_lock_irqsave_nested(lock, subclass); \
} while (0)
#else
#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
do { \
typecheck(unsigned long, flags); \
flags = _raw_spin_lock_irqsave(lock); \
} while (0)
#endif
#else
#define raw_spin_lock_irqsave(lock, flags) \
do { \
typecheck(unsigned long, flags); \
_raw_spin_lock_irqsave(lock, flags); \
} while (0)
#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
raw_spin_lock_irqsave(lock, flags)
#endif
#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
#define raw_spin_unlock_irqrestore(lock, flags) \
do { \
typecheck(unsigned long, flags); \
_raw_spin_unlock_irqrestore(lock, flags); \
} while (0)
#define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
#define raw_spin_trylock_bh(lock) \
__cond_lock(lock, _raw_spin_trylock_bh(lock))
#define raw_spin_trylock_irq(lock) \
({ \
local_irq_disable(); \
raw_spin_trylock(lock) ? \
1 : ({ local_irq_enable(); 0; }); \
})
#define raw_spin_trylock_irqsave(lock, flags) \
({ \
local_irq_save(flags); \
raw_spin_trylock(lock) ? \
1 : ({ local_irq_restore(flags); 0; }); \
})
#ifndef CONFIG_PREEMPT_RT
/* Include rwlock functions for !RT */
#include <linux/rwlock.h>
#endif
/*
* Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
*/
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
# include <linux/spinlock_api_smp.h>
#else
# include <linux/spinlock_api_up.h>
#endif
/* Non PREEMPT_RT kernel, map to raw spinlocks: */
#ifndef CONFIG_PREEMPT_RT
/*
* Map the spin_lock functions to the raw variants for PREEMPT_RT=n
*/
static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
{
return &lock->rlock;
}
#ifdef CONFIG_DEBUG_SPINLOCK
# define spin_lock_init(lock) \
do { \
static struct lock_class_key __key; \
\
__raw_spin_lock_init(spinlock_check(lock), \
#lock, &__key, LD_WAIT_CONFIG); \
} while (0)
#else
# define spin_lock_init(_lock) \
do { \
spinlock_check(_lock); \
*(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \
} while (0)
#endif
static __always_inline void spin_lock(spinlock_t *lock)
{
raw_spin_lock(&lock->rlock);
}
static __always_inline void spin_lock_bh(spinlock_t *lock)
{
raw_spin_lock_bh(&lock->rlock);
}
static __always_inline int spin_trylock(spinlock_t *lock)
{
return raw_spin_trylock(&lock->rlock);
}
#define spin_lock_nested(lock, subclass) \
do { \
raw_spin_lock_nested(spinlock_check(lock), subclass); \
} while (0)
#define spin_lock_nest_lock(lock, nest_lock) \
do { \
raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \
} while (0)
static __always_inline void spin_lock_irq(spinlock_t *lock)
{
raw_spin_lock_irq(&lock->rlock);
}
#define spin_lock_irqsave(lock, flags) \
do { \
raw_spin_lock_irqsave(spinlock_check(lock), flags); \
} while (0)
#define spin_lock_irqsave_nested(lock, flags, subclass) \
do { \
raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
} while (0)
static __always_inline void spin_unlock(spinlock_t *lock)
{
raw_spin_unlock(&lock->rlock);}
static __always_inline void spin_unlock_bh(spinlock_t *lock)
{
raw_spin_unlock_bh(&lock->rlock);
}
static __always_inline void spin_unlock_irq(spinlock_t *lock)
{
raw_spin_unlock_irq(&lock->rlock);
}
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
{
raw_spin_unlock_irqrestore(&lock->rlock, flags);
}
static __always_inline int spin_trylock_bh(spinlock_t *lock)
{
return raw_spin_trylock_bh(&lock->rlock);
}
static __always_inline int spin_trylock_irq(spinlock_t *lock)
{
return raw_spin_trylock_irq(&lock->rlock);
}
#define spin_trylock_irqsave(lock, flags) \
({ \
raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
})
/**
* spin_is_locked() - Check whether a spinlock is locked.
* @lock: Pointer to the spinlock.
*
* This function is NOT required to provide any memory ordering
* guarantees; it could be used for debugging purposes or, when
* additional synchronization is needed, accompanied with other
* constructs (memory barriers) enforcing the synchronization.
*
* Returns: 1 if @lock is locked, 0 otherwise.
*
* Note that the function only tells you that the spinlock is
* seen to be locked, not that it is locked on your CPU.
*
* Further, on CONFIG_SMP=n builds with CONFIG_DEBUG_SPINLOCK=n,
* the return value is always 0 (see include/linux/spinlock_up.h).
* Therefore you should not rely heavily on the return value.
*/
static __always_inline int spin_is_locked(spinlock_t *lock)
{
return raw_spin_is_locked(&lock->rlock);
}
static __always_inline int spin_is_contended(spinlock_t *lock)
{
return raw_spin_is_contended(&lock->rlock);
}
#define assert_spin_locked(lock) assert_raw_spin_locked(&(lock)->rlock)
#else /* !CONFIG_PREEMPT_RT */
# include <linux/spinlock_rt.h>
#endif /* CONFIG_PREEMPT_RT */
/*
* Does a critical section need to be broken due to another
* task waiting?: (technically does not depend on CONFIG_PREEMPTION,
* but a general need for low latency)
*/
static inline int spin_needbreak(spinlock_t *lock)
{
#ifdef CONFIG_PREEMPTION
return spin_is_contended(lock);
#else
return 0;
#endif
}
/*
* Check if a rwlock is contended.
* Returns non-zero if there is another task waiting on the rwlock.
* Returns zero if the lock is not contended or the system / underlying
* rwlock implementation does not support contention detection.
* Technically does not depend on CONFIG_PREEMPTION, but a general need
* for low latency.
*/
static inline int rwlock_needbreak(rwlock_t *lock)
{
#ifdef CONFIG_PREEMPTION
return rwlock_is_contended(lock);
#else
return 0;
#endif
}
/*
* Pull the atomic_t declaration:
* (asm-mips/atomic.h needs above definitions)
*/
#include <linux/atomic.h>
/**
* atomic_dec_and_lock - lock on reaching reference count zero
* @atomic: the atomic counter
* @lock: the spinlock in question
*
* Decrements @atomic by 1. If the result is 0, returns true and locks
* @lock. Returns false for all other cases.
*/
extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
#define atomic_dec_and_lock(atomic, lock) \
__cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
extern int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
unsigned long *flags);
#define atomic_dec_and_lock_irqsave(atomic, lock, flags) \
__cond_lock(lock, _atomic_dec_and_lock_irqsave(atomic, lock, &(flags)))
extern int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock);
#define atomic_dec_and_raw_lock(atomic, lock) \
__cond_lock(lock, _atomic_dec_and_raw_lock(atomic, lock))
extern int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
unsigned long *flags);
#define atomic_dec_and_raw_lock_irqsave(atomic, lock, flags) \
__cond_lock(lock, _atomic_dec_and_raw_lock_irqsave(atomic, lock, &(flags)))
int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask,
size_t max_size, unsigned int cpu_mult,
gfp_t gfp, const char *name,
struct lock_class_key *key);
#define alloc_bucket_spinlocks(locks, lock_mask, max_size, cpu_mult, gfp) \
({ \
static struct lock_class_key key; \
int ret; \
\
ret = __alloc_bucket_spinlocks(locks, lock_mask, max_size, \
cpu_mult, gfp, #locks, &key); \
ret; \
})
void free_bucket_spinlocks(spinlock_t *locks);
DEFINE_LOCK_GUARD_1(raw_spinlock, raw_spinlock_t,
raw_spin_lock(_T->lock),
raw_spin_unlock(_T->lock))
DEFINE_LOCK_GUARD_1_COND(raw_spinlock, _try, raw_spin_trylock(_T->lock))
DEFINE_LOCK_GUARD_1(raw_spinlock_nested, raw_spinlock_t,
raw_spin_lock_nested(_T->lock, SINGLE_DEPTH_NESTING),
raw_spin_unlock(_T->lock))
DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t,
raw_spin_lock_irq(_T->lock),
raw_spin_unlock_irq(_T->lock))
DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock))
DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t,
raw_spin_lock_irqsave(_T->lock, _T->flags),
raw_spin_unlock_irqrestore(_T->lock, _T->flags),
unsigned long flags)
DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try,
raw_spin_trylock_irqsave(_T->lock, _T->flags))
DEFINE_LOCK_GUARD_1(spinlock, spinlock_t,
spin_lock(_T->lock),
spin_unlock(_T->lock))
DEFINE_LOCK_GUARD_1_COND(spinlock, _try, spin_trylock(_T->lock))
DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t,
spin_lock_irq(_T->lock),
spin_unlock_irq(_T->lock))
DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try,
spin_trylock_irq(_T->lock))
DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t,
spin_lock_irqsave(_T->lock, _T->flags),
spin_unlock_irqrestore(_T->lock, _T->flags),
unsigned long flags)
DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try,
spin_trylock_irqsave(_T->lock, _T->flags))
DEFINE_LOCK_GUARD_1(read_lock, rwlock_t,
read_lock(_T->lock),
read_unlock(_T->lock))
DEFINE_LOCK_GUARD_1(read_lock_irq, rwlock_t,
read_lock_irq(_T->lock),
read_unlock_irq(_T->lock))
DEFINE_LOCK_GUARD_1(read_lock_irqsave, rwlock_t,
read_lock_irqsave(_T->lock, _T->flags),
read_unlock_irqrestore(_T->lock, _T->flags),
unsigned long flags)
DEFINE_LOCK_GUARD_1(write_lock, rwlock_t,
write_lock(_T->lock),
write_unlock(_T->lock))
DEFINE_LOCK_GUARD_1(write_lock_irq, rwlock_t,
write_lock_irq(_T->lock),
write_unlock_irq(_T->lock))
DEFINE_LOCK_GUARD_1(write_lock_irqsave, rwlock_t,
write_lock_irqsave(_T->lock, _T->flags),
write_unlock_irqrestore(_T->lock, _T->flags),
unsigned long flags)
#undef __LINUX_INSIDE_SPINLOCK_H
#endif /* __LINUX_SPINLOCK_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* This header provides generic wrappers for memory access instrumentation that
* the compiler cannot emit for: KASAN, KCSAN, KMSAN.
*/
#ifndef _LINUX_INSTRUMENTED_H
#define _LINUX_INSTRUMENTED_H
#include <linux/compiler.h>
#include <linux/kasan-checks.h>
#include <linux/kcsan-checks.h>
#include <linux/kmsan-checks.h>
#include <linux/types.h>
/**
* instrument_read - instrument regular read access
* @v: address of access
* @size: size of access
*
* Instrument a regular read access. The instrumentation should be inserted
* before the actual read happens.
*/
static __always_inline void instrument_read(const volatile void *v, size_t size)
{
kasan_check_read(v, size);
kcsan_check_read(v, size);
}
/**
* instrument_write - instrument regular write access
* @v: address of access
* @size: size of access
*
* Instrument a regular write access. The instrumentation should be inserted
* before the actual write happens.
*/
static __always_inline void instrument_write(const volatile void *v, size_t size)
{
kasan_check_write(v, size);
kcsan_check_write(v, size);
}
/**
* instrument_read_write - instrument regular read-write access
* @v: address of access
* @size: size of access
*
* Instrument a regular write access. The instrumentation should be inserted
* before the actual write happens.
*/
static __always_inline void instrument_read_write(const volatile void *v, size_t size)
{
kasan_check_write(v, size);
kcsan_check_read_write(v, size);
}
/**
* instrument_atomic_read - instrument atomic read access
* @v: address of access
* @size: size of access
*
* Instrument an atomic read access. The instrumentation should be inserted
* before the actual read happens.
*/
static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
{
kasan_check_read(v, size);
kcsan_check_atomic_read(v, size);
}
/**
* instrument_atomic_write - instrument atomic write access
* @v: address of access
* @size: size of access
*
* Instrument an atomic write access. The instrumentation should be inserted
* before the actual write happens.
*/
static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
{
kasan_check_write(v, size);
kcsan_check_atomic_write(v, size);
}
/**
* instrument_atomic_read_write - instrument atomic read-write access
* @v: address of access
* @size: size of access
*
* Instrument an atomic read-write access. The instrumentation should be
* inserted before the actual write happens.
*/
static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
{
kasan_check_write(v, size);
kcsan_check_atomic_read_write(v, size);
}
/**
* instrument_copy_to_user - instrument reads of copy_to_user
* @to: destination address
* @from: source address
* @n: number of bytes to copy
*
* Instrument reads from kernel memory, that are due to copy_to_user (and
* variants). The instrumentation must be inserted before the accesses.
*/
static __always_inline void
instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
{
kasan_check_read(from, n);
kcsan_check_read(from, n);
kmsan_copy_to_user(to, from, n, 0);
}
/**
* instrument_copy_from_user_before - add instrumentation before copy_from_user
* @to: destination address
* @from: source address
* @n: number of bytes to copy
*
* Instrument writes to kernel memory, that are due to copy_from_user (and
* variants). The instrumentation should be inserted before the accesses.
*/
static __always_inline void
instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
{
kasan_check_write(to, n);
kcsan_check_write(to, n);
}
/**
* instrument_copy_from_user_after - add instrumentation after copy_from_user
* @to: destination address
* @from: source address
* @n: number of bytes to copy
* @left: number of bytes not copied (as returned by copy_from_user)
*
* Instrument writes to kernel memory, that are due to copy_from_user (and
* variants). The instrumentation should be inserted after the accesses.
*/
static __always_inline void
instrument_copy_from_user_after(const void *to, const void __user *from,
unsigned long n, unsigned long left)
{
kmsan_unpoison_memory(to, n - left);
}
/**
* instrument_get_user() - add instrumentation to get_user()-like macros
* @to: destination variable, may not be address-taken
*
* get_user() and friends are fragile, so it may depend on the implementation
* whether the instrumentation happens before or after the data is copied from
* the userspace.
*/
#define instrument_get_user(to) \
({ \
u64 __tmp = (u64)(to); \
kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \
to = __tmp; \
})
/**
* instrument_put_user() - add instrumentation to put_user()-like macros
* @from: source address
* @ptr: userspace pointer to copy to
* @size: number of bytes to copy
*
* put_user() and friends are fragile, so it may depend on the implementation
* whether the instrumentation happens before or after the data is copied from
* the userspace.
*/
#define instrument_put_user(from, ptr, size) \
({ \
kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \
})
#endif /* _LINUX_INSTRUMENTED_H */
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* linux/drivers/char/serial_core.h
*
* Copyright (C) 2000 Deep Blue Solutions Ltd.
*/
#ifndef LINUX_SERIAL_CORE_H
#define LINUX_SERIAL_CORE_H
#include <linux/bitops.h>
#include <linux/compiler.h>
#include <linux/console.h>
#include <linux/interrupt.h>
#include <linux/circ_buf.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/tty.h>
#include <linux/mutex.h>
#include <linux/sysrq.h>
#include <uapi/linux/serial_core.h>
#ifdef CONFIG_SERIAL_CORE_CONSOLE
#define uart_console(port) \
((port)->cons && (port)->cons->index == (port)->line)
#else
#define uart_console(port) ({ (void)port; 0; })
#endif
struct uart_port;
struct serial_struct;
struct serial_port_device;
struct device;
struct gpio_desc;
/**
* struct uart_ops -- interface between serial_core and the driver
*
* This structure describes all the operations that can be done on the
* physical hardware.
*
* @tx_empty: ``unsigned int ()(struct uart_port *port)``
*
* This function tests whether the transmitter fifo and shifter for the
* @port is empty. If it is empty, this function should return
* %TIOCSER_TEMT, otherwise return 0. If the port does not support this
* operation, then it should return %TIOCSER_TEMT.
*
* Locking: none.
* Interrupts: caller dependent.
* This call must not sleep
*
* @set_mctrl: ``void ()(struct uart_port *port, unsigned int mctrl)``
*
* This function sets the modem control lines for @port to the state
* described by @mctrl. The relevant bits of @mctrl are:
*
* - %TIOCM_RTS RTS signal.
* - %TIOCM_DTR DTR signal.
* - %TIOCM_OUT1 OUT1 signal.
* - %TIOCM_OUT2 OUT2 signal.
* - %TIOCM_LOOP Set the port into loopback mode.
*
* If the appropriate bit is set, the signal should be driven
* active. If the bit is clear, the signal should be driven
* inactive.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @get_mctrl: ``unsigned int ()(struct uart_port *port)``
*
* Returns the current state of modem control inputs of @port. The state
* of the outputs should not be returned, since the core keeps track of
* their state. The state information should include:
*
* - %TIOCM_CAR state of DCD signal
* - %TIOCM_CTS state of CTS signal
* - %TIOCM_DSR state of DSR signal
* - %TIOCM_RI state of RI signal
*
* The bit is set if the signal is currently driven active. If
* the port does not support CTS, DCD or DSR, the driver should
* indicate that the signal is permanently active. If RI is
* not available, the signal should not be indicated as active.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @stop_tx: ``void ()(struct uart_port *port)``
*
* Stop transmitting characters. This might be due to the CTS line
* becoming inactive or the tty layer indicating we want to stop
* transmission due to an %XOFF character.
*
* The driver should stop transmitting characters as soon as possible.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @start_tx: ``void ()(struct uart_port *port)``
*
* Start transmitting characters.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @throttle: ``void ()(struct uart_port *port)``
*
* Notify the serial driver that input buffers for the line discipline are
* close to full, and it should somehow signal that no more characters
* should be sent to the serial port.
* This will be called only if hardware assisted flow control is enabled.
*
* Locking: serialized with @unthrottle() and termios modification by the
* tty layer.
*
* @unthrottle: ``void ()(struct uart_port *port)``
*
* Notify the serial driver that characters can now be sent to the serial
* port without fear of overrunning the input buffers of the line
* disciplines.
*
* This will be called only if hardware assisted flow control is enabled.
*
* Locking: serialized with @throttle() and termios modification by the
* tty layer.
*
* @send_xchar: ``void ()(struct uart_port *port, char ch)``
*
* Transmit a high priority character, even if the port is stopped. This
* is used to implement XON/XOFF flow control and tcflow(). If the serial
* driver does not implement this function, the tty core will append the
* character to the circular buffer and then call start_tx() / stop_tx()
* to flush the data out.
*
* Do not transmit if @ch == '\0' (%__DISABLED_CHAR).
*
* Locking: none.
* Interrupts: caller dependent.
*
* @start_rx: ``void ()(struct uart_port *port)``
*
* Start receiving characters.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @stop_rx: ``void ()(struct uart_port *port)``
*
* Stop receiving characters; the @port is in the process of being closed.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @enable_ms: ``void ()(struct uart_port *port)``
*
* Enable the modem status interrupts.
*
* This method may be called multiple times. Modem status interrupts
* should be disabled when the @shutdown() method is called.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @break_ctl: ``void ()(struct uart_port *port, int ctl)``
*
* Control the transmission of a break signal. If @ctl is nonzero, the
* break signal should be transmitted. The signal should be terminated
* when another call is made with a zero @ctl.
*
* Locking: caller holds tty_port->mutex
*
* @startup: ``int ()(struct uart_port *port)``
*
* Grab any interrupt resources and initialise any low level driver state.
* Enable the port for reception. It should not activate RTS nor DTR;
* this will be done via a separate call to @set_mctrl().
*
* This method will only be called when the port is initially opened.
*
* Locking: port_sem taken.
* Interrupts: globally disabled.
*
* @shutdown: ``void ()(struct uart_port *port)``
*
* Disable the @port, disable any break condition that may be in effect,
* and free any interrupt resources. It should not disable RTS nor DTR;
* this will have already been done via a separate call to @set_mctrl().
*
* Drivers must not access @port->state once this call has completed.
*
* This method will only be called when there are no more users of this
* @port.
*
* Locking: port_sem taken.
* Interrupts: caller dependent.
*
* @flush_buffer: ``void ()(struct uart_port *port)``
*
* Flush any write buffers, reset any DMA state and stop any ongoing DMA
* transfers.
*
* This will be called whenever the @port->state->xmit circular buffer is
* cleared.
*
* Locking: @port->lock taken.
* Interrupts: locally disabled.
* This call must not sleep
*
* @set_termios: ``void ()(struct uart_port *port, struct ktermios *new,
* struct ktermios *old)``
*
* Change the @port parameters, including word length, parity, stop bits.
* Update @port->read_status_mask and @port->ignore_status_mask to
* indicate the types of events we are interested in receiving. Relevant
* ktermios::c_cflag bits are:
*
* - %CSIZE - word size
* - %CSTOPB - 2 stop bits
* - %PARENB - parity enable
* - %PARODD - odd parity (when %PARENB is in force)
* - %ADDRB - address bit (changed through uart_port::rs485_config()).
* - %CREAD - enable reception of characters (if not set, still receive
* characters from the port, but throw them away).
* - %CRTSCTS - if set, enable CTS status change reporting.
* - %CLOCAL - if not set, enable modem status change reporting.
*
* Relevant ktermios::c_iflag bits are:
*
* - %INPCK - enable frame and parity error events to be passed to the TTY
* layer.
* - %BRKINT / %PARMRK - both of these enable break events to be passed to
* the TTY layer.
* - %IGNPAR - ignore parity and framing errors.
* - %IGNBRK - ignore break errors. If %IGNPAR is also set, ignore overrun
* errors as well.
*
* The interaction of the ktermios::c_iflag bits is as follows (parity
* error given as an example):
*
* ============ ======= ======= =========================================
* Parity error INPCK IGNPAR
* ============ ======= ======= =========================================
* n/a 0 n/a character received, marked as %TTY_NORMAL
* None 1 n/a character received, marked as %TTY_NORMAL
* Yes 1 0 character received, marked as %TTY_PARITY
* Yes 1 1 character discarded
* ============ ======= ======= =========================================
*
* Other flags may be used (eg, xon/xoff characters) if your hardware
* supports hardware "soft" flow control.
*
* Locking: caller holds tty_port->mutex
* Interrupts: caller dependent.
* This call must not sleep
*
* @set_ldisc: ``void ()(struct uart_port *port, struct ktermios *termios)``
*
* Notifier for discipline change. See
* Documentation/driver-api/tty/tty_ldisc.rst.
*
* Locking: caller holds tty_port->mutex
*
* @pm: ``void ()(struct uart_port *port, unsigned int state,
* unsigned int oldstate)``
*
* Perform any power management related activities on the specified @port.
* @state indicates the new state (defined by enum uart_pm_state),
* @oldstate indicates the previous state.
*
* This function should not be used to grab any resources.
*
* This will be called when the @port is initially opened and finally
* closed, except when the @port is also the system console. This will
* occur even if %CONFIG_PM is not set.
*
* Locking: none.
* Interrupts: caller dependent.
*
* @type: ``const char *()(struct uart_port *port)``
*
* Return a pointer to a string constant describing the specified @port,
* or return %NULL, in which case the string 'unknown' is substituted.
*
* Locking: none.
* Interrupts: caller dependent.
*
* @release_port: ``void ()(struct uart_port *port)``
*
* Release any memory and IO region resources currently in use by the
* @port.
*
* Locking: none.
* Interrupts: caller dependent.
*
* @request_port: ``int ()(struct uart_port *port)``
*
* Request any memory and IO region resources required by the port. If any
* fail, no resources should be registered when this function returns, and
* it should return -%EBUSY on failure.
*
* Locking: none.
* Interrupts: caller dependent.
*
* @config_port: ``void ()(struct uart_port *port, int type)``
*
* Perform any autoconfiguration steps required for the @port. @type
* contains a bit mask of the required configuration. %UART_CONFIG_TYPE
* indicates that the port requires detection and identification.
* @port->type should be set to the type found, or %PORT_UNKNOWN if no
* port was detected.
*
* %UART_CONFIG_IRQ indicates autoconfiguration of the interrupt signal,
* which should be probed using standard kernel autoprobing techniques.
* This is not necessary on platforms where ports have interrupts
* internally hard wired (eg, system on a chip implementations).
*
* Locking: none.
* Interrupts: caller dependent.
*
* @verify_port: ``int ()(struct uart_port *port,
* struct serial_struct *serinfo)``
*
* Verify the new serial port information contained within @serinfo is
* suitable for this port type.
*
* Locking: none.
* Interrupts: caller dependent.
*
* @ioctl: ``int ()(struct uart_port *port, unsigned int cmd,
* unsigned long arg)``
*
* Perform any port specific IOCTLs. IOCTL commands must be defined using
* the standard numbering system found in <asm/ioctl.h>.
*
* Locking: none.
* Interrupts: caller dependent.
*
* @poll_init: ``int ()(struct uart_port *port)``
*
* Called by kgdb to perform the minimal hardware initialization needed to
* support @poll_put_char() and @poll_get_char(). Unlike @startup(), this
* should not request interrupts.
*
* Locking: %tty_mutex and tty_port->mutex taken.
* Interrupts: n/a.
*
* @poll_put_char: ``void ()(struct uart_port *port, unsigned char ch)``
*
* Called by kgdb to write a single character @ch directly to the serial
* @port. It can and should block until there is space in the TX FIFO.
*
* Locking: none.
* Interrupts: caller dependent.
* This call must not sleep
*
* @poll_get_char: ``int ()(struct uart_port *port)``
*
* Called by kgdb to read a single character directly from the serial
* port. If data is available, it should be returned; otherwise the
* function should return %NO_POLL_CHAR immediately.
*
* Locking: none.
* Interrupts: caller dependent.
* This call must not sleep
*/
struct uart_ops {
unsigned int (*tx_empty)(struct uart_port *);
void (*set_mctrl)(struct uart_port *, unsigned int mctrl);
unsigned int (*get_mctrl)(struct uart_port *);
void (*stop_tx)(struct uart_port *);
void (*start_tx)(struct uart_port *);
void (*throttle)(struct uart_port *);
void (*unthrottle)(struct uart_port *);
void (*send_xchar)(struct uart_port *, char ch);
void (*stop_rx)(struct uart_port *);
void (*start_rx)(struct uart_port *);
void (*enable_ms)(struct uart_port *);
void (*break_ctl)(struct uart_port *, int ctl);
int (*startup)(struct uart_port *);
void (*shutdown)(struct uart_port *);
void (*flush_buffer)(struct uart_port *);
void (*set_termios)(struct uart_port *, struct ktermios *new,
const struct ktermios *old);
void (*set_ldisc)(struct uart_port *, struct ktermios *);
void (*pm)(struct uart_port *, unsigned int state,
unsigned int oldstate);
const char *(*type)(struct uart_port *);
void (*release_port)(struct uart_port *);
int (*request_port)(struct uart_port *);
void (*config_port)(struct uart_port *, int);
int (*verify_port)(struct uart_port *, struct serial_struct *);
int (*ioctl)(struct uart_port *, unsigned int, unsigned long);
#ifdef CONFIG_CONSOLE_POLL
int (*poll_init)(struct uart_port *);
void (*poll_put_char)(struct uart_port *, unsigned char);
int (*poll_get_char)(struct uart_port *);
#endif
};
#define NO_POLL_CHAR 0x00ff0000
#define UART_CONFIG_TYPE (1 << 0)
#define UART_CONFIG_IRQ (1 << 1)
struct uart_icount {
__u32 cts;
__u32 dsr;
__u32 rng;
__u32 dcd;
__u32 rx;
__u32 tx;
__u32 frame;
__u32 overrun;
__u32 parity;
__u32 brk;
__u32 buf_overrun;
};
typedef u64 __bitwise upf_t;
typedef unsigned int __bitwise upstat_t;
struct uart_port {
spinlock_t lock; /* port lock */
unsigned long iobase; /* in/out[bwl] */
unsigned char __iomem *membase; /* read/write[bwl] */
unsigned int (*serial_in)(struct uart_port *, int);
void (*serial_out)(struct uart_port *, int, int);
void (*set_termios)(struct uart_port *,
struct ktermios *new,
const struct ktermios *old);
void (*set_ldisc)(struct uart_port *,
struct ktermios *);
unsigned int (*get_mctrl)(struct uart_port *);
void (*set_mctrl)(struct uart_port *, unsigned int);
unsigned int (*get_divisor)(struct uart_port *,
unsigned int baud,
unsigned int *frac);
void (*set_divisor)(struct uart_port *,
unsigned int baud,
unsigned int quot,
unsigned int quot_frac);
int (*startup)(struct uart_port *port);
void (*shutdown)(struct uart_port *port);
void (*throttle)(struct uart_port *port);
void (*unthrottle)(struct uart_port *port);
int (*handle_irq)(struct uart_port *);
void (*pm)(struct uart_port *, unsigned int state,
unsigned int old);
void (*handle_break)(struct uart_port *);
int (*rs485_config)(struct uart_port *,
struct ktermios *termios,
struct serial_rs485 *rs485);
int (*iso7816_config)(struct uart_port *,
struct serial_iso7816 *iso7816);
unsigned int ctrl_id; /* optional serial core controller id */
unsigned int port_id; /* optional serial core port id */
unsigned int irq; /* irq number */
unsigned long irqflags; /* irq flags */
unsigned int uartclk; /* base uart clock */
unsigned int fifosize; /* tx fifo size */
unsigned char x_char; /* xon/xoff char */
unsigned char regshift; /* reg offset shift */
unsigned char iotype; /* io access style */
#define UPIO_UNKNOWN ((unsigned char)~0U) /* UCHAR_MAX */
#define UPIO_PORT (SERIAL_IO_PORT) /* 8b I/O port access */
#define UPIO_HUB6 (SERIAL_IO_HUB6) /* Hub6 ISA card */
#define UPIO_MEM (SERIAL_IO_MEM) /* driver-specific */
#define UPIO_MEM32 (SERIAL_IO_MEM32) /* 32b little endian */
#define UPIO_AU (SERIAL_IO_AU) /* Au1x00 and RT288x type IO */
#define UPIO_TSI (SERIAL_IO_TSI) /* Tsi108/109 type IO */
#define UPIO_MEM32BE (SERIAL_IO_MEM32BE) /* 32b big endian */
#define UPIO_MEM16 (SERIAL_IO_MEM16) /* 16b little endian */
unsigned char quirks; /* internal quirks */
/* internal quirks must be updated while holding port mutex */
#define UPQ_NO_TXEN_TEST BIT(0)
unsigned int read_status_mask; /* driver specific */
unsigned int ignore_status_mask; /* driver specific */
struct uart_state *state; /* pointer to parent state */
struct uart_icount icount; /* statistics */
struct console *cons; /* struct console, if any */
/* flags must be updated while holding port mutex */
upf_t flags;
/*
* These flags must be equivalent to the flags defined in
* include/uapi/linux/tty_flags.h which are the userspace definitions
* assigned from the serial_struct flags in uart_set_info()
* [for bit definitions in the UPF_CHANGE_MASK]
*
* Bits [0..ASYNCB_LAST_USER] are userspace defined/visible/changeable
* The remaining bits are serial-core specific and not modifiable by
* userspace.
*/
#define UPF_FOURPORT ((__force upf_t) ASYNC_FOURPORT /* 1 */ )
#define UPF_SAK ((__force upf_t) ASYNC_SAK /* 2 */ )
#define UPF_SPD_HI ((__force upf_t) ASYNC_SPD_HI /* 4 */ )
#define UPF_SPD_VHI ((__force upf_t) ASYNC_SPD_VHI /* 5 */ )
#define UPF_SPD_CUST ((__force upf_t) ASYNC_SPD_CUST /* 0x0030 */ )
#define UPF_SPD_WARP ((__force upf_t) ASYNC_SPD_WARP /* 0x1010 */ )
#define UPF_SPD_MASK ((__force upf_t) ASYNC_SPD_MASK /* 0x1030 */ )
#define UPF_SKIP_TEST ((__force upf_t) ASYNC_SKIP_TEST /* 6 */ )
#define UPF_AUTO_IRQ ((__force upf_t) ASYNC_AUTO_IRQ /* 7 */ )
#define UPF_HARDPPS_CD ((__force upf_t) ASYNC_HARDPPS_CD /* 11 */ )
#define UPF_SPD_SHI ((__force upf_t) ASYNC_SPD_SHI /* 12 */ )
#define UPF_LOW_LATENCY ((__force upf_t) ASYNC_LOW_LATENCY /* 13 */ )
#define UPF_BUGGY_UART ((__force upf_t) ASYNC_BUGGY_UART /* 14 */ )
#define UPF_MAGIC_MULTIPLIER ((__force upf_t) ASYNC_MAGIC_MULTIPLIER /* 16 */ )
#define UPF_NO_THRE_TEST ((__force upf_t) BIT_ULL(19))
/* Port has hardware-assisted h/w flow control */
#define UPF_AUTO_CTS ((__force upf_t) BIT_ULL(20))
#define UPF_AUTO_RTS ((__force upf_t) BIT_ULL(21))
#define UPF_HARD_FLOW ((__force upf_t) (UPF_AUTO_CTS | UPF_AUTO_RTS))
/* Port has hardware-assisted s/w flow control */
#define UPF_SOFT_FLOW ((__force upf_t) BIT_ULL(22))
#define UPF_CONS_FLOW ((__force upf_t) BIT_ULL(23))
#define UPF_SHARE_IRQ ((__force upf_t) BIT_ULL(24))
#define UPF_EXAR_EFR ((__force upf_t) BIT_ULL(25))
#define UPF_BUG_THRE ((__force upf_t) BIT_ULL(26))
/* The exact UART type is known and should not be probed. */
#define UPF_FIXED_TYPE ((__force upf_t) BIT_ULL(27))
#define UPF_BOOT_AUTOCONF ((__force upf_t) BIT_ULL(28))
#define UPF_FIXED_PORT ((__force upf_t) BIT_ULL(29))
#define UPF_DEAD ((__force upf_t) BIT_ULL(30))
#define UPF_IOREMAP ((__force upf_t) BIT_ULL(31))
#define UPF_FULL_PROBE ((__force upf_t) BIT_ULL(32))
#define __UPF_CHANGE_MASK 0x17fff
#define UPF_CHANGE_MASK ((__force upf_t) __UPF_CHANGE_MASK)
#define UPF_USR_MASK ((__force upf_t) (UPF_SPD_MASK|UPF_LOW_LATENCY))
#if __UPF_CHANGE_MASK > ASYNC_FLAGS
#error Change mask not equivalent to userspace-visible bit defines
#endif
/*
* Must hold termios_rwsem, port mutex and port lock to change;
* can hold any one lock to read.
*/
upstat_t status;
#define UPSTAT_CTS_ENABLE ((__force upstat_t) (1 << 0))
#define UPSTAT_DCD_ENABLE ((__force upstat_t) (1 << 1))
#define UPSTAT_AUTORTS ((__force upstat_t) (1 << 2))
#define UPSTAT_AUTOCTS ((__force upstat_t) (1 << 3))
#define UPSTAT_AUTOXOFF ((__force upstat_t) (1 << 4))
#define UPSTAT_SYNC_FIFO ((__force upstat_t) (1 << 5))
bool hw_stopped; /* sw-assisted CTS flow state */
unsigned int mctrl; /* current modem ctrl settings */
unsigned int frame_time; /* frame timing in ns */
unsigned int type; /* port type */
const struct uart_ops *ops;
unsigned int custom_divisor;
unsigned int line; /* port index */
unsigned int minor;
resource_size_t mapbase; /* for ioremap */
resource_size_t mapsize;
struct device *dev; /* serial port physical parent device */
struct serial_port_device *port_dev; /* serial core port device */
unsigned long sysrq; /* sysrq timeout */
u8 sysrq_ch; /* char for sysrq */
unsigned char has_sysrq;
unsigned char sysrq_seq; /* index in sysrq_toggle_seq */
unsigned char hub6; /* this should be in the 8250 driver */
unsigned char suspended;
unsigned char console_reinit;
const char *name; /* port name */
struct attribute_group *attr_group; /* port specific attributes */
const struct attribute_group **tty_groups; /* all attributes (serial core use only) */
struct serial_rs485 rs485;
struct serial_rs485 rs485_supported; /* Supported mask for serial_rs485 */
struct gpio_desc *rs485_term_gpio; /* enable RS485 bus termination */
struct gpio_desc *rs485_rx_during_tx_gpio; /* Output GPIO that sets the state of RS485 RX during TX */
struct serial_iso7816 iso7816;
void *private_data; /* generic platform data pointer */
};
/**
* uart_port_lock - Lock the UART port
* @up: Pointer to UART port structure
*/
static inline void uart_port_lock(struct uart_port *up)
{
spin_lock(&up->lock);
}
/**
* uart_port_lock_irq - Lock the UART port and disable interrupts
* @up: Pointer to UART port structure
*/
static inline void uart_port_lock_irq(struct uart_port *up)
{
spin_lock_irq(&up->lock);
}
/**
* uart_port_lock_irqsave - Lock the UART port, save and disable interrupts
* @up: Pointer to UART port structure
* @flags: Pointer to interrupt flags storage
*/
static inline void uart_port_lock_irqsave(struct uart_port *up, unsigned long *flags)
{
spin_lock_irqsave(&up->lock, *flags);
}
/**
* uart_port_trylock - Try to lock the UART port
* @up: Pointer to UART port structure
*
* Returns: True if lock was acquired, false otherwise
*/
static inline bool uart_port_trylock(struct uart_port *up)
{
return spin_trylock(&up->lock);
}
/**
* uart_port_trylock_irqsave - Try to lock the UART port, save and disable interrupts
* @up: Pointer to UART port structure
* @flags: Pointer to interrupt flags storage
*
* Returns: True if lock was acquired, false otherwise
*/
static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long *flags)
{
return spin_trylock_irqsave(&up->lock, *flags);
}
/**
* uart_port_unlock - Unlock the UART port
* @up: Pointer to UART port structure
*/
static inline void uart_port_unlock(struct uart_port *up)
{
spin_unlock(&up->lock);
}
/**
* uart_port_unlock_irq - Unlock the UART port and re-enable interrupts
* @up: Pointer to UART port structure
*/
static inline void uart_port_unlock_irq(struct uart_port *up)
{
spin_unlock_irq(&up->lock);
}
/**
* uart_port_unlock_irqrestore - Unlock the UART port, restore interrupts
* @up: Pointer to UART port structure
* @flags: The saved interrupt flags for restore
*/
static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags)
{
spin_unlock_irqrestore(&up->lock, flags);
}
static inline int serial_port_in(struct uart_port *up, int offset)
{
return up->serial_in(up, offset);
}
static inline void serial_port_out(struct uart_port *up, int offset, int value)
{
up->serial_out(up, offset, value);
}
/**
* enum uart_pm_state - power states for UARTs
* @UART_PM_STATE_ON: UART is powered, up and operational
* @UART_PM_STATE_OFF: UART is powered off
* @UART_PM_STATE_UNDEFINED: sentinel
*/
enum uart_pm_state {
UART_PM_STATE_ON = 0,
UART_PM_STATE_OFF = 3, /* number taken from ACPI */
UART_PM_STATE_UNDEFINED,
};
/*
* This is the state information which is persistent across opens.
*/
struct uart_state {
struct tty_port port;
enum uart_pm_state pm_state;
struct circ_buf xmit;
atomic_t refcount;
wait_queue_head_t remove_wait;
struct uart_port *uart_port;
};
#define UART_XMIT_SIZE PAGE_SIZE
/* number of characters left in xmit buffer before we ask for more */
#define WAKEUP_CHARS 256
/**
* uart_xmit_advance - Advance xmit buffer and account Tx'ed chars
* @up: uart_port structure describing the port
* @chars: number of characters sent
*
* This function advances the tail of circular xmit buffer by the number of
* @chars transmitted and handles accounting of transmitted bytes (into
* @up's icount.tx).
*/
static inline void uart_xmit_advance(struct uart_port *up, unsigned int chars)
{
struct circ_buf *xmit = &up->state->xmit;
xmit->tail = (xmit->tail + chars) & (UART_XMIT_SIZE - 1);
up->icount.tx += chars;
}
struct module;
struct tty_driver;
struct uart_driver {
struct module *owner;
const char *driver_name;
const char *dev_name;
int major;
int minor;
int nr;
struct console *cons;
/*
* these are private; the low level driver should not
* touch these; they should be initialised to NULL
*/
struct uart_state *state;
struct tty_driver *tty_driver;
};
void uart_write_wakeup(struct uart_port *port);
/**
* enum UART_TX_FLAGS -- flags for uart_port_tx_flags()
*
* @UART_TX_NOSTOP: don't call port->ops->stop_tx() on empty buffer
*/
enum UART_TX_FLAGS {
UART_TX_NOSTOP = BIT(0),
};
#define __uart_port_tx(uport, ch, flags, tx_ready, put_char, tx_done, \
for_test, for_post) \
({ \
struct uart_port *__port = (uport); \
struct circ_buf *xmit = &__port->state->xmit; \
unsigned int pending; \
\
for (; (for_test) && (tx_ready); (for_post), __port->icount.tx++) { \
if (__port->x_char) { \
(ch) = __port->x_char; \
(put_char); \
__port->x_char = 0; \
continue; \
} \
\
if (uart_circ_empty(xmit) || uart_tx_stopped(__port)) \
break; \
\
(ch) = xmit->buf[xmit->tail]; \
(put_char); \
xmit->tail = (xmit->tail + 1) % UART_XMIT_SIZE; \
} \
\
(tx_done); \
\
pending = uart_circ_chars_pending(xmit); \
if (pending < WAKEUP_CHARS) { \
uart_write_wakeup(__port); \
\
if (!((flags) & UART_TX_NOSTOP) && pending == 0 && \
__port->ops->tx_empty(__port)) \
__port->ops->stop_tx(__port); \
} \
\
pending; \
})
/**
* uart_port_tx_limited -- transmit helper for uart_port with count limiting
* @port: uart port
* @ch: variable to store a character to be written to the HW
* @count: a limit of characters to send
* @tx_ready: can HW accept more data function
* @put_char: function to write a character
* @tx_done: function to call after the loop is done
*
* This helper transmits characters from the xmit buffer to the hardware using
* @put_char(). It does so until @count characters are sent and while @tx_ready
* evaluates to true.
*
* Returns: the number of characters in the xmit buffer when done.
*
* The expression in macro parameters shall be designed as follows:
* * **tx_ready:** should evaluate to true if the HW can accept more data to
* be sent. This parameter can be %true, which means the HW is always ready.
* * **put_char:** shall write @ch to the device of @port.
* * **tx_done:** when the write loop is done, this can perform arbitrary
* action before potential invocation of ops->stop_tx() happens. If the
* driver does not need to do anything, use e.g. ({}).
*
* For all of them, @port->lock is held, interrupts are locally disabled and
* the expressions must not sleep.
*/
#define uart_port_tx_limited(port, ch, count, tx_ready, put_char, tx_done) ({ \
unsigned int __count = (count); \
__uart_port_tx(port, ch, 0, tx_ready, put_char, tx_done, __count, \
__count--); \
})
/**
* uart_port_tx -- transmit helper for uart_port
* @port: uart port
* @ch: variable to store a character to be written to the HW
* @tx_ready: can HW accept more data function
* @put_char: function to write a character
*
* See uart_port_tx_limited() for more details.
*/
#define uart_port_tx(port, ch, tx_ready, put_char) \
__uart_port_tx(port, ch, 0, tx_ready, put_char, ({}), true, ({}))
/**
* uart_port_tx_flags -- transmit helper for uart_port with flags
* @port: uart port
* @ch: variable to store a character to be written to the HW
* @flags: %UART_TX_NOSTOP or similar
* @tx_ready: can HW accept more data function
* @put_char: function to write a character
*
* See uart_port_tx_limited() for more details.
*/
#define uart_port_tx_flags(port, ch, flags, tx_ready, put_char) \
__uart_port_tx(port, ch, flags, tx_ready, put_char, ({}), true, ({}))
/*
* Baud rate helpers.
*/
void uart_update_timeout(struct uart_port *port, unsigned int cflag,
unsigned int baud);
unsigned int uart_get_baud_rate(struct uart_port *port, struct ktermios *termios,
const struct ktermios *old, unsigned int min,
unsigned int max);
unsigned int uart_get_divisor(struct uart_port *port, unsigned int baud);
/*
* Calculates FIFO drain time.
*/
static inline unsigned long uart_fifo_timeout(struct uart_port *port)
{
u64 fifo_timeout = (u64)READ_ONCE(port->frame_time) * port->fifosize;
/* Add .02 seconds of slop */
fifo_timeout += 20 * NSEC_PER_MSEC;
return max(nsecs_to_jiffies(fifo_timeout), 1UL);
}
/* Base timer interval for polling */
static inline unsigned long uart_poll_timeout(struct uart_port *port)
{
unsigned long timeout = uart_fifo_timeout(port);
return timeout > 6 ? (timeout / 2 - 2) : 1;
}
/*
* Console helpers.
*/
struct earlycon_device {
struct console *con;
struct uart_port port;
char options[32]; /* e.g., 115200n8 */
unsigned int baud;
};
struct earlycon_id {
char name[15];
char name_term; /* In case compiler didn't '\0' term name */
char compatible[128];
int (*setup)(struct earlycon_device *, const char *options);
};
extern const struct earlycon_id __earlycon_table[];
extern const struct earlycon_id __earlycon_table_end[];
#if defined(CONFIG_SERIAL_EARLYCON) && !defined(MODULE)
#define EARLYCON_USED_OR_UNUSED __used
#else
#define EARLYCON_USED_OR_UNUSED __maybe_unused
#endif
#define OF_EARLYCON_DECLARE(_name, compat, fn) \
static const struct earlycon_id __UNIQUE_ID(__earlycon_##_name) \
EARLYCON_USED_OR_UNUSED __section("__earlycon_table") \
__aligned(__alignof__(struct earlycon_id)) \
= { .name = __stringify(_name), \
.compatible = compat, \
.setup = fn }
#define EARLYCON_DECLARE(_name, fn) OF_EARLYCON_DECLARE(_name, "", fn)
int of_setup_earlycon(const struct earlycon_id *match, unsigned long node,
const char *options);
#ifdef CONFIG_SERIAL_EARLYCON
extern bool earlycon_acpi_spcr_enable __initdata;
int setup_earlycon(char *buf);
#else
static const bool earlycon_acpi_spcr_enable EARLYCON_USED_OR_UNUSED;
static inline int setup_earlycon(char *buf) { return 0; }
#endif
/* Variant of uart_console_registered() when the console_list_lock is held. */
static inline bool uart_console_registered_locked(struct uart_port *port)
{
return uart_console(port) && console_is_registered_locked(port->cons);
}
static inline bool uart_console_registered(struct uart_port *port)
{
return uart_console(port) && console_is_registered(port->cons);
}
struct uart_port *uart_get_console(struct uart_port *ports, int nr,
struct console *c);
int uart_parse_earlycon(char *p, unsigned char *iotype, resource_size_t *addr,
char **options);
void uart_parse_options(const char *options, int *baud, int *parity, int *bits,
int *flow);
int uart_set_options(struct uart_port *port, struct console *co, int baud,
int parity, int bits, int flow);
struct tty_driver *uart_console_device(struct console *co, int *index);
void uart_console_write(struct uart_port *port, const char *s,
unsigned int count,
void (*putchar)(struct uart_port *, unsigned char));
/*
* Port/driver registration/removal
*/
int uart_register_driver(struct uart_driver *uart);
void uart_unregister_driver(struct uart_driver *uart);
int uart_add_one_port(struct uart_driver *reg, struct uart_port *port);
void uart_remove_one_port(struct uart_driver *reg, struct uart_port *port);
int uart_read_port_properties(struct uart_port *port);
int uart_read_and_validate_port_properties(struct uart_port *port);
bool uart_match_port(const struct uart_port *port1,
const struct uart_port *port2);
/*
* Power Management
*/
int uart_suspend_port(struct uart_driver *reg, struct uart_port *port);
int uart_resume_port(struct uart_driver *reg, struct uart_port *port);
#define uart_circ_empty(circ) ((circ)->head == (circ)->tail)
#define uart_circ_clear(circ) ((circ)->head = (circ)->tail = 0)
#define uart_circ_chars_pending(circ) \
(CIRC_CNT((circ)->head, (circ)->tail, UART_XMIT_SIZE))
#define uart_circ_chars_free(circ) \
(CIRC_SPACE((circ)->head, (circ)->tail, UART_XMIT_SIZE))
static inline int uart_tx_stopped(struct uart_port *port)
{
struct tty_struct *tty = port->state->port.tty;
if ((tty && tty->flow.stopped) || port->hw_stopped)
return 1;
return 0;
}
static inline bool uart_cts_enabled(struct uart_port *uport)
{
return !!(uport->status & UPSTAT_CTS_ENABLE);
}
static inline bool uart_softcts_mode(struct uart_port *uport)
{
upstat_t mask = UPSTAT_CTS_ENABLE | UPSTAT_AUTOCTS;
return ((uport->status & mask) == UPSTAT_CTS_ENABLE);
}
/*
* The following are helper functions for the low level drivers.
*/
void uart_handle_dcd_change(struct uart_port *uport, bool active);
void uart_handle_cts_change(struct uart_port *uport, bool active);
void uart_insert_char(struct uart_port *port, unsigned int status,
unsigned int overrun, u8 ch, u8 flag);
void uart_xchar_out(struct uart_port *uport, int offset);
#ifdef CONFIG_MAGIC_SYSRQ_SERIAL
#define SYSRQ_TIMEOUT (HZ * 5)
bool uart_try_toggle_sysrq(struct uart_port *port, u8 ch);
static inline int uart_handle_sysrq_char(struct uart_port *port, u8 ch)
{
if (!port->sysrq)
return 0;
if (ch && time_before(jiffies, port->sysrq)) {
if (sysrq_mask()) {
handle_sysrq(ch);
port->sysrq = 0;
return 1;
}
if (uart_try_toggle_sysrq(port, ch))
return 1;
}
port->sysrq = 0;
return 0;
}
static inline int uart_prepare_sysrq_char(struct uart_port *port, u8 ch)
{
if (!port->sysrq)
return 0;
if (ch && time_before(jiffies, port->sysrq)) {
if (sysrq_mask()) {
port->sysrq_ch = ch;
port->sysrq = 0;
return 1;
}
if (uart_try_toggle_sysrq(port, ch))
return 1;
}
port->sysrq = 0;
return 0;
}
static inline void uart_unlock_and_check_sysrq(struct uart_port *port)
{
u8 sysrq_ch;
if (!port->has_sysrq) {
uart_port_unlock(port);
return;
}
sysrq_ch = port->sysrq_ch;
port->sysrq_ch = 0;
uart_port_unlock(port);
if (sysrq_ch)
handle_sysrq(sysrq_ch);
}
static inline void uart_unlock_and_check_sysrq_irqrestore(struct uart_port *port,
unsigned long flags)
{
u8 sysrq_ch;
if (!port->has_sysrq) {
uart_port_unlock_irqrestore(port, flags);
return;
}
sysrq_ch = port->sysrq_ch;
port->sysrq_ch = 0;
uart_port_unlock_irqrestore(port, flags);
if (sysrq_ch)
handle_sysrq(sysrq_ch);
}
#else /* CONFIG_MAGIC_SYSRQ_SERIAL */
static inline int uart_handle_sysrq_char(struct uart_port *port, u8 ch)
{
return 0;
}
static inline int uart_prepare_sysrq_char(struct uart_port *port, u8 ch)
{
return 0;
}
static inline void uart_unlock_and_check_sysrq(struct uart_port *port)
{
uart_port_unlock(port);
}
static inline void uart_unlock_and_check_sysrq_irqrestore(struct uart_port *port,
unsigned long flags)
{
uart_port_unlock_irqrestore(port, flags);
}
#endif /* CONFIG_MAGIC_SYSRQ_SERIAL */
/*
* We do the SysRQ and SAK checking like this...
*/
static inline int uart_handle_break(struct uart_port *port)
{
struct uart_state *state = port->state;
if (port->handle_break)
port->handle_break(port);
#ifdef CONFIG_MAGIC_SYSRQ_SERIAL
if (port->has_sysrq && uart_console(port)) {
if (!port->sysrq) {
port->sysrq = jiffies + SYSRQ_TIMEOUT;
return 1;
}
port->sysrq = 0;
}
#endif
if (port->flags & UPF_SAK)
do_SAK(state->port.tty);
return 0;
}
/*
* UART_ENABLE_MS - determine if port should enable modem status irqs
*/
#define UART_ENABLE_MS(port,cflag) ((port)->flags & UPF_HARDPPS_CD || \
(cflag) & CRTSCTS || \
!((cflag) & CLOCAL))
int uart_get_rs485_mode(struct uart_port *port);
#endif /* LINUX_SERIAL_CORE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_CGROUP_H
#define _LINUX_CGROUP_H
/*
* cgroup interface
*
* Copyright (C) 2003 BULL SA
* Copyright (C) 2004-2006 Silicon Graphics, Inc.
*
*/
#include <linux/sched.h>
#include <linux/cpumask.h>
#include <linux/nodemask.h>
#include <linux/rculist.h>
#include <linux/cgroupstats.h>
#include <linux/fs.h>
#include <linux/seq_file.h>
#include <linux/kernfs.h>
#include <linux/jump_label.h>
#include <linux/types.h>
#include <linux/ns_common.h>
#include <linux/nsproxy.h>
#include <linux/user_namespace.h>
#include <linux/refcount.h>
#include <linux/kernel_stat.h>
#include <linux/cgroup-defs.h>
struct kernel_clone_args;
#ifdef CONFIG_CGROUPS
/*
* All weight knobs on the default hierarchy should use the following min,
* default and max values. The default value is the logarithmic center of
* MIN and MAX and allows 100x to be expressed in both directions.
*/
#define CGROUP_WEIGHT_MIN 1
#define CGROUP_WEIGHT_DFL 100
#define CGROUP_WEIGHT_MAX 10000
enum {
CSS_TASK_ITER_PROCS = (1U << 0), /* walk only threadgroup leaders */
CSS_TASK_ITER_THREADED = (1U << 1), /* walk all threaded css_sets in the domain */
CSS_TASK_ITER_SKIPPED = (1U << 16), /* internal flags */
};
/* a css_task_iter should be treated as an opaque object */
struct css_task_iter {
struct cgroup_subsys *ss;
unsigned int flags;
struct list_head *cset_pos;
struct list_head *cset_head;
struct list_head *tcset_pos;
struct list_head *tcset_head;
struct list_head *task_pos;
struct list_head *cur_tasks_head;
struct css_set *cur_cset;
struct css_set *cur_dcset;
struct task_struct *cur_task;
struct list_head iters_node; /* css_set->task_iters */
};
extern struct file_system_type cgroup_fs_type;
extern struct cgroup_root cgrp_dfl_root;
extern struct css_set init_css_set;
extern spinlock_t css_set_lock;
#define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys;
#include <linux/cgroup_subsys.h>
#undef SUBSYS
#define SUBSYS(_x) \
extern struct static_key_true _x ## _cgrp_subsys_enabled_key; \
extern struct static_key_true _x ## _cgrp_subsys_on_dfl_key;
#include <linux/cgroup_subsys.h>
#undef SUBSYS
/**
* cgroup_subsys_enabled - fast test on whether a subsys is enabled
* @ss: subsystem in question
*/
#define cgroup_subsys_enabled(ss) \
static_branch_likely(&ss ## _enabled_key)
/**
* cgroup_subsys_on_dfl - fast test on whether a subsys is on default hierarchy
* @ss: subsystem in question
*/
#define cgroup_subsys_on_dfl(ss) \
static_branch_likely(&ss ## _on_dfl_key)
bool css_has_online_children(struct cgroup_subsys_state *css);
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss);
struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgroup,
struct cgroup_subsys *ss);
struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgroup,
struct cgroup_subsys *ss);
struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
struct cgroup_subsys *ss);
struct cgroup *cgroup_get_from_path(const char *path);
struct cgroup *cgroup_get_from_fd(int fd);
struct cgroup *cgroup_v1v2_get_from_fd(int fd);
int cgroup_attach_task_all(struct task_struct *from, struct task_struct *);
int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from);
int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts);
int cgroup_rm_cftypes(struct cftype *cfts);
void cgroup_file_notify(struct cgroup_file *cfile);
void cgroup_file_show(struct cgroup_file *cfile, bool show);
int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry);
int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *tsk);
void cgroup_fork(struct task_struct *p);
extern int cgroup_can_fork(struct task_struct *p,
struct kernel_clone_args *kargs);
extern void cgroup_cancel_fork(struct task_struct *p,
struct kernel_clone_args *kargs);
extern void cgroup_post_fork(struct task_struct *p,
struct kernel_clone_args *kargs);
void cgroup_exit(struct task_struct *p);
void cgroup_release(struct task_struct *p);
void cgroup_free(struct task_struct *p);
int cgroup_init_early(void);
int cgroup_init(void);
int cgroup_parse_float(const char *input, unsigned dec_shift, s64 *v);
/*
* Iteration helpers and macros.
*/
struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
struct cgroup_subsys_state *parent);
struct cgroup_subsys_state *css_next_descendant_pre(struct cgroup_subsys_state *pos,
struct cgroup_subsys_state *css);
struct cgroup_subsys_state *css_rightmost_descendant(struct cgroup_subsys_state *pos);
struct cgroup_subsys_state *css_next_descendant_post(struct cgroup_subsys_state *pos,
struct cgroup_subsys_state *css);
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
struct cgroup_subsys_state **dst_cssp);
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
struct cgroup_subsys_state **dst_cssp);
void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,
struct css_task_iter *it);
struct task_struct *css_task_iter_next(struct css_task_iter *it);
void css_task_iter_end(struct css_task_iter *it);
/**
* css_for_each_child - iterate through children of a css
* @pos: the css * to use as the loop cursor
* @parent: css whose children to walk
*
* Walk @parent's children. Must be called under rcu_read_lock().
*
* If a subsystem synchronizes ->css_online() and the start of iteration, a
* css which finished ->css_online() is guaranteed to be visible in the
* future iterations and will stay visible until the last reference is put.
* A css which hasn't finished ->css_online() or already finished
* ->css_offline() may show up during traversal. It's each subsystem's
* responsibility to synchronize against on/offlining.
*
* It is allowed to temporarily drop RCU read lock during iteration. The
* caller is responsible for ensuring that @pos remains accessible until
* the start of the next iteration by, for example, bumping the css refcnt.
*/
#define css_for_each_child(pos, parent) \
for ((pos) = css_next_child(NULL, (parent)); (pos); \
(pos) = css_next_child((pos), (parent)))
/**
* css_for_each_descendant_pre - pre-order walk of a css's descendants
* @pos: the css * to use as the loop cursor
* @root: css whose descendants to walk
*
* Walk @root's descendants. @root is included in the iteration and the
* first node to be visited. Must be called under rcu_read_lock().
*
* If a subsystem synchronizes ->css_online() and the start of iteration, a
* css which finished ->css_online() is guaranteed to be visible in the
* future iterations and will stay visible until the last reference is put.
* A css which hasn't finished ->css_online() or already finished
* ->css_offline() may show up during traversal. It's each subsystem's
* responsibility to synchronize against on/offlining.
*
* For example, the following guarantees that a descendant can't escape
* state updates of its ancestors.
*
* my_online(@css)
* {
* Lock @css's parent and @css;
* Inherit state from the parent;
* Unlock both.
* }
*
* my_update_state(@css)
* {
* css_for_each_descendant_pre(@pos, @css) {
* Lock @pos;
* if (@pos == @css)
* Update @css's state;
* else
* Verify @pos is alive and inherit state from its parent;
* Unlock @pos;
* }
* }
*
* As long as the inheriting step, including checking the parent state, is
* enclosed inside @pos locking, double-locking the parent isn't necessary
* while inheriting. The state update to the parent is guaranteed to be
* visible by walking order and, as long as inheriting operations to the
* same @pos are atomic to each other, multiple updates racing each other
* still result in the correct state. It's guaranateed that at least one
* inheritance happens for any css after the latest update to its parent.
*
* If checking parent's state requires locking the parent, each inheriting
* iteration should lock and unlock both @pos->parent and @pos.
*
* Alternatively, a subsystem may choose to use a single global lock to
* synchronize ->css_online() and ->css_offline() against tree-walking
* operations.
*
* It is allowed to temporarily drop RCU read lock during iteration. The
* caller is responsible for ensuring that @pos remains accessible until
* the start of the next iteration by, for example, bumping the css refcnt.
*/
#define css_for_each_descendant_pre(pos, css) \
for ((pos) = css_next_descendant_pre(NULL, (css)); (pos); \
(pos) = css_next_descendant_pre((pos), (css)))
/**
* css_for_each_descendant_post - post-order walk of a css's descendants
* @pos: the css * to use as the loop cursor
* @css: css whose descendants to walk
*
* Similar to css_for_each_descendant_pre() but performs post-order
* traversal instead. @root is included in the iteration and the last
* node to be visited.
*
* If a subsystem synchronizes ->css_online() and the start of iteration, a
* css which finished ->css_online() is guaranteed to be visible in the
* future iterations and will stay visible until the last reference is put.
* A css which hasn't finished ->css_online() or already finished
* ->css_offline() may show up during traversal. It's each subsystem's
* responsibility to synchronize against on/offlining.
*
* Note that the walk visibility guarantee example described in pre-order
* walk doesn't apply the same to post-order walks.
*/
#define css_for_each_descendant_post(pos, css) \
for ((pos) = css_next_descendant_post(NULL, (css)); (pos); \
(pos) = css_next_descendant_post((pos), (css)))
/**
* cgroup_taskset_for_each - iterate cgroup_taskset
* @task: the loop cursor
* @dst_css: the destination css
* @tset: taskset to iterate
*
* @tset may contain multiple tasks and they may belong to multiple
* processes.
*
* On the v2 hierarchy, there may be tasks from multiple processes and they
* may not share the source or destination csses.
*
* On traditional hierarchies, when there are multiple tasks in @tset, if a
* task of a process is in @tset, all tasks of the process are in @tset.
* Also, all are guaranteed to share the same source and destination csses.
*
* Iteration is not in any specific order.
*/
#define cgroup_taskset_for_each(task, dst_css, tset) \
for ((task) = cgroup_taskset_first((tset), &(dst_css)); \
(task); \
(task) = cgroup_taskset_next((tset), &(dst_css)))
/**
* cgroup_taskset_for_each_leader - iterate group leaders in a cgroup_taskset
* @leader: the loop cursor
* @dst_css: the destination css
* @tset: taskset to iterate
*
* Iterate threadgroup leaders of @tset. For single-task migrations, @tset
* may not contain any.
*/
#define cgroup_taskset_for_each_leader(leader, dst_css, tset) \
for ((leader) = cgroup_taskset_first((tset), &(dst_css)); \
(leader); \
(leader) = cgroup_taskset_next((tset), &(dst_css))) \
if ((leader) != (leader)->group_leader) \
; \
else
/*
* Inline functions.
*/
#ifdef CONFIG_DEBUG_CGROUP_REF
void css_get(struct cgroup_subsys_state *css);
void css_get_many(struct cgroup_subsys_state *css, unsigned int n);
bool css_tryget(struct cgroup_subsys_state *css);
bool css_tryget_online(struct cgroup_subsys_state *css);
void css_put(struct cgroup_subsys_state *css);
void css_put_many(struct cgroup_subsys_state *css, unsigned int n);
#else
#define CGROUP_REF_FN_ATTRS static inline
#define CGROUP_REF_EXPORT(fn)
#include <linux/cgroup_refcnt.h>
#endif
static inline u64 cgroup_id(const struct cgroup *cgrp)
{
return cgrp->kn->id;
}
/**
* css_is_dying - test whether the specified css is dying
* @css: target css
*
* Test whether @css is in the process of offlining or already offline. In
* most cases, ->css_online() and ->css_offline() callbacks should be
* enough; however, the actual offline operations are RCU delayed and this
* test returns %true also when @css is scheduled to be offlined.
*
* This is useful, for example, when the use case requires synchronous
* behavior with respect to cgroup removal. cgroup removal schedules css
* offlining but the css can seem alive while the operation is being
* delayed. If the delay affects user visible semantics, this test can be
* used to resolve the situation.
*/
static inline bool css_is_dying(struct cgroup_subsys_state *css)
{
return !(css->flags & CSS_NO_REF) && percpu_ref_is_dying(&css->refcnt);
}
static inline void cgroup_get(struct cgroup *cgrp)
{
css_get(&cgrp->self);
}
static inline bool cgroup_tryget(struct cgroup *cgrp)
{
return css_tryget(&cgrp->self);
}
static inline void cgroup_put(struct cgroup *cgrp)
{
css_put(&cgrp->self);
}
extern struct mutex cgroup_mutex;
static inline void cgroup_lock(void)
{
mutex_lock(&cgroup_mutex);
}
static inline void cgroup_unlock(void)
{
mutex_unlock(&cgroup_mutex);
}
/**
* task_css_set_check - obtain a task's css_set with extra access conditions
* @task: the task to obtain css_set for
* @__c: extra condition expression to be passed to rcu_dereference_check()
*
* A task's css_set is RCU protected, initialized and exited while holding
* task_lock(), and can only be modified while holding both cgroup_mutex
* and task_lock() while the task is alive. This macro verifies that the
* caller is inside proper critical section and returns @task's css_set.
*
* The caller can also specify additional allowed conditions via @__c, such
* as locks used during the cgroup_subsys::attach() methods.
*/
#ifdef CONFIG_PROVE_RCU
#define task_css_set_check(task, __c) \
rcu_dereference_check((task)->cgroups, \
rcu_read_lock_sched_held() || \
lockdep_is_held(&cgroup_mutex) || \
lockdep_is_held(&css_set_lock) || \
((task)->flags & PF_EXITING) || (__c))
#else
#define task_css_set_check(task, __c) \
rcu_dereference((task)->cgroups)
#endif
/**
* task_css_check - obtain css for (task, subsys) w/ extra access conds
* @task: the target task
* @subsys_id: the target subsystem ID
* @__c: extra condition expression to be passed to rcu_dereference_check()
*
* Return the cgroup_subsys_state for the (@task, @subsys_id) pair. The
* synchronization rules are the same as task_css_set_check().
*/
#define task_css_check(task, subsys_id, __c) \
task_css_set_check((task), (__c))->subsys[(subsys_id)]
/**
* task_css_set - obtain a task's css_set
* @task: the task to obtain css_set for
*
* See task_css_set_check().
*/
static inline struct css_set *task_css_set(struct task_struct *task)
{
return task_css_set_check(task, false);
}
/**
* task_css - obtain css for (task, subsys)
* @task: the target task
* @subsys_id: the target subsystem ID
*
* See task_css_check().
*/
static inline struct cgroup_subsys_state *task_css(struct task_struct *task,
int subsys_id)
{
return task_css_check(task, subsys_id, false);
}
/**
* task_get_css - find and get the css for (task, subsys)
* @task: the target task
* @subsys_id: the target subsystem ID
*
* Find the css for the (@task, @subsys_id) combination, increment a
* reference on and return it. This function is guaranteed to return a
* valid css. The returned css may already have been offlined.
*/
static inline struct cgroup_subsys_state *
task_get_css(struct task_struct *task, int subsys_id)
{
struct cgroup_subsys_state *css;
rcu_read_lock();
while (true) {
css = task_css(task, subsys_id);
/*
* Can't use css_tryget_online() here. A task which has
* PF_EXITING set may stay associated with an offline css.
* If such task calls this function, css_tryget_online()
* will keep failing.
*/
if (likely(css_tryget(css)))
break;
cpu_relax();
}
rcu_read_unlock();
return css;
}
/**
* task_css_is_root - test whether a task belongs to the root css
* @task: the target task
* @subsys_id: the target subsystem ID
*
* Test whether @task belongs to the root css on the specified subsystem.
* May be invoked in any context.
*/
static inline bool task_css_is_root(struct task_struct *task, int subsys_id)
{
return task_css_check(task, subsys_id, true) ==
init_css_set.subsys[subsys_id];
}
static inline struct cgroup *task_cgroup(struct task_struct *task,
int subsys_id)
{
return task_css(task, subsys_id)->cgroup;
}
static inline struct cgroup *task_dfl_cgroup(struct task_struct *task)
{
return task_css_set(task)->dfl_cgrp;
}
static inline struct cgroup *cgroup_parent(struct cgroup *cgrp)
{
struct cgroup_subsys_state *parent_css = cgrp->self.parent;
if (parent_css)
return container_of(parent_css, struct cgroup, self);
return NULL;
}
/**
* cgroup_is_descendant - test ancestry
* @cgrp: the cgroup to be tested
* @ancestor: possible ancestor of @cgrp
*
* Test whether @cgrp is a descendant of @ancestor. It also returns %true
* if @cgrp == @ancestor. This function is safe to call as long as @cgrp
* and @ancestor are accessible.
*/
static inline bool cgroup_is_descendant(struct cgroup *cgrp,
struct cgroup *ancestor)
{
if (cgrp->root != ancestor->root || cgrp->level < ancestor->level)
return false;
return cgrp->ancestors[ancestor->level] == ancestor;
}
/**
* cgroup_ancestor - find ancestor of cgroup
* @cgrp: cgroup to find ancestor of
* @ancestor_level: level of ancestor to find starting from root
*
* Find ancestor of cgroup at specified level starting from root if it exists
* and return pointer to it. Return NULL if @cgrp doesn't have ancestor at
* @ancestor_level.
*
* This function is safe to call as long as @cgrp is accessible.
*/
static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp,
int ancestor_level)
{
if (ancestor_level < 0 || ancestor_level > cgrp->level)
return NULL;
return cgrp->ancestors[ancestor_level];
}
/**
* task_under_cgroup_hierarchy - test task's membership of cgroup ancestry
* @task: the task to be tested
* @ancestor: possible ancestor of @task's cgroup
*
* Tests whether @task's default cgroup hierarchy is a descendant of @ancestor.
* It follows all the same rules as cgroup_is_descendant, and only applies
* to the default hierarchy.
*/
static inline bool task_under_cgroup_hierarchy(struct task_struct *task,
struct cgroup *ancestor)
{
struct css_set *cset = task_css_set(task);
return cgroup_is_descendant(cset->dfl_cgrp, ancestor);
}
/* no synchronization, the result can only be used as a hint */
static inline bool cgroup_is_populated(struct cgroup *cgrp)
{
return cgrp->nr_populated_csets + cgrp->nr_populated_domain_children +
cgrp->nr_populated_threaded_children;
}
/* returns ino associated with a cgroup */
static inline ino_t cgroup_ino(struct cgroup *cgrp)
{
return kernfs_ino(cgrp->kn);
}
/* cft/css accessors for cftype->write() operation */
static inline struct cftype *of_cft(struct kernfs_open_file *of)
{
return of->kn->priv;
}
struct cgroup_subsys_state *of_css(struct kernfs_open_file *of);
/* cft/css accessors for cftype->seq_*() operations */
static inline struct cftype *seq_cft(struct seq_file *seq)
{
return of_cft(seq->private);
}
static inline struct cgroup_subsys_state *seq_css(struct seq_file *seq)
{
return of_css(seq->private);
}
/*
* Name / path handling functions. All are thin wrappers around the kernfs
* counterparts and can be called under any context.
*/
static inline int cgroup_name(struct cgroup *cgrp, char *buf, size_t buflen)
{
return kernfs_name(cgrp->kn, buf, buflen);
}
static inline int cgroup_path(struct cgroup *cgrp, char *buf, size_t buflen)
{
return kernfs_path(cgrp->kn, buf, buflen);
}
static inline void pr_cont_cgroup_name(struct cgroup *cgrp)
{
pr_cont_kernfs_name(cgrp->kn);
}
static inline void pr_cont_cgroup_path(struct cgroup *cgrp)
{
pr_cont_kernfs_path(cgrp->kn);
}
bool cgroup_psi_enabled(void);
static inline void cgroup_init_kthreadd(void)
{
/*
* kthreadd is inherited by all kthreads, keep it in the root so
* that the new kthreads are guaranteed to stay in the root until
* initialization is finished.
*/
current->no_cgroup_migration = 1;
}
static inline void cgroup_kthread_ready(void)
{
/*
* This kthread finished initialization. The creator should have
* set PF_NO_SETAFFINITY if this kthread should stay in the root.
*/
current->no_cgroup_migration = 0;
}
void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen);
struct cgroup *cgroup_get_from_id(u64 id);
#else /* !CONFIG_CGROUPS */
struct cgroup_subsys_state;
struct cgroup;
static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; }
static inline void css_get(struct cgroup_subsys_state *css) {}
static inline void css_put(struct cgroup_subsys_state *css) {}
static inline void cgroup_lock(void) {}
static inline void cgroup_unlock(void) {}
static inline int cgroup_attach_task_all(struct task_struct *from,
struct task_struct *t) { return 0; }
static inline int cgroupstats_build(struct cgroupstats *stats,
struct dentry *dentry) { return -EINVAL; }
static inline void cgroup_fork(struct task_struct *p) {}
static inline int cgroup_can_fork(struct task_struct *p,
struct kernel_clone_args *kargs) { return 0; }
static inline void cgroup_cancel_fork(struct task_struct *p,
struct kernel_clone_args *kargs) {}
static inline void cgroup_post_fork(struct task_struct *p,
struct kernel_clone_args *kargs) {}
static inline void cgroup_exit(struct task_struct *p) {}
static inline void cgroup_release(struct task_struct *p) {}
static inline void cgroup_free(struct task_struct *p) {}
static inline int cgroup_init_early(void) { return 0; }
static inline int cgroup_init(void) { return 0; }
static inline void cgroup_init_kthreadd(void) {}
static inline void cgroup_kthread_ready(void) {}
static inline struct cgroup *cgroup_parent(struct cgroup *cgrp)
{
return NULL;
}
static inline bool cgroup_psi_enabled(void)
{
return false;
}
static inline bool task_under_cgroup_hierarchy(struct task_struct *task,
struct cgroup *ancestor)
{
return true;
}
static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen)
{}
#endif /* !CONFIG_CGROUPS */
#ifdef CONFIG_CGROUPS
/*
* cgroup scalable recursive statistics.
*/
void cgroup_rstat_updated(struct cgroup *cgrp, int cpu);
void cgroup_rstat_flush(struct cgroup *cgrp);
void cgroup_rstat_flush_hold(struct cgroup *cgrp);
void cgroup_rstat_flush_release(void);
/*
* Basic resource stats.
*/
#ifdef CONFIG_CGROUP_CPUACCT
void cpuacct_charge(struct task_struct *tsk, u64 cputime);
void cpuacct_account_field(struct task_struct *tsk, int index, u64 val);
#else
static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
static inline void cpuacct_account_field(struct task_struct *tsk, int index,
u64 val) {}
#endif
void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec);
void __cgroup_account_cputime_field(struct cgroup *cgrp,
enum cpu_usage_stat index, u64 delta_exec);
static inline void cgroup_account_cputime(struct task_struct *task,
u64 delta_exec)
{
struct cgroup *cgrp;
cpuacct_charge(task, delta_exec);
cgrp = task_dfl_cgroup(task);
if (cgroup_parent(cgrp))
__cgroup_account_cputime(cgrp, delta_exec);
}
static inline void cgroup_account_cputime_field(struct task_struct *task,
enum cpu_usage_stat index,
u64 delta_exec)
{
struct cgroup *cgrp;
cpuacct_account_field(task, index, delta_exec);
cgrp = task_dfl_cgroup(task);
if (cgroup_parent(cgrp))
__cgroup_account_cputime_field(cgrp, index, delta_exec);
}
#else /* CONFIG_CGROUPS */
static inline void cgroup_account_cputime(struct task_struct *task,
u64 delta_exec) {}
static inline void cgroup_account_cputime_field(struct task_struct *task,
enum cpu_usage_stat index,
u64 delta_exec) {}
#endif /* CONFIG_CGROUPS */
/*
* sock->sk_cgrp_data handling. For more info, see sock_cgroup_data
* definition in cgroup-defs.h.
*/
#ifdef CONFIG_SOCK_CGROUP_DATA
void cgroup_sk_alloc(struct sock_cgroup_data *skcd);
void cgroup_sk_clone(struct sock_cgroup_data *skcd);
void cgroup_sk_free(struct sock_cgroup_data *skcd);
static inline struct cgroup *sock_cgroup_ptr(struct sock_cgroup_data *skcd)
{
return skcd->cgroup;
}
#else /* CONFIG_CGROUP_DATA */
static inline void cgroup_sk_alloc(struct sock_cgroup_data *skcd) {}
static inline void cgroup_sk_clone(struct sock_cgroup_data *skcd) {}
static inline void cgroup_sk_free(struct sock_cgroup_data *skcd) {}
#endif /* CONFIG_CGROUP_DATA */
struct cgroup_namespace {
struct ns_common ns;
struct user_namespace *user_ns;
struct ucounts *ucounts;
struct css_set *root_cset;
};
extern struct cgroup_namespace init_cgroup_ns;
#ifdef CONFIG_CGROUPS
void free_cgroup_ns(struct cgroup_namespace *ns);
struct cgroup_namespace *copy_cgroup_ns(unsigned long flags,
struct user_namespace *user_ns,
struct cgroup_namespace *old_ns);
int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
struct cgroup_namespace *ns);
#else /* !CONFIG_CGROUPS */
static inline void free_cgroup_ns(struct cgroup_namespace *ns) { }
static inline struct cgroup_namespace *
copy_cgroup_ns(unsigned long flags, struct user_namespace *user_ns,
struct cgroup_namespace *old_ns)
{
return old_ns;
}
#endif /* !CONFIG_CGROUPS */
static inline void get_cgroup_ns(struct cgroup_namespace *ns)
{
if (ns)
refcount_inc(&ns->ns.count);
}
static inline void put_cgroup_ns(struct cgroup_namespace *ns)
{
if (ns && refcount_dec_and_test(&ns->ns.count))
free_cgroup_ns(ns);
}
#ifdef CONFIG_CGROUPS
void cgroup_enter_frozen(void);
void cgroup_leave_frozen(bool always_leave);
void cgroup_update_frozen(struct cgroup *cgrp);
void cgroup_freeze(struct cgroup *cgrp, bool freeze);
void cgroup_freezer_migrate_task(struct task_struct *task, struct cgroup *src,
struct cgroup *dst);
static inline bool cgroup_task_frozen(struct task_struct *task)
{
return task->frozen;
}
#else /* !CONFIG_CGROUPS */
static inline void cgroup_enter_frozen(void) { }
static inline void cgroup_leave_frozen(bool always_leave) { }
static inline bool cgroup_task_frozen(struct task_struct *task)
{
return false;
}
#endif /* !CONFIG_CGROUPS */
#ifdef CONFIG_CGROUP_BPF
static inline void cgroup_bpf_get(struct cgroup *cgrp)
{
percpu_ref_get(&cgrp->bpf.refcnt);
}
static inline void cgroup_bpf_put(struct cgroup *cgrp)
{
percpu_ref_put(&cgrp->bpf.refcnt);
}
#else /* CONFIG_CGROUP_BPF */
static inline void cgroup_bpf_get(struct cgroup *cgrp) {}
static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
#endif /* CONFIG_CGROUP_BPF */
struct cgroup *task_get_cgroup1(struct task_struct *tsk, int hierarchy_id);
#endif /* _LINUX_CGROUP_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* hosts.c Copyright (C) 1992 Drew Eckhardt
* Copyright (C) 1993, 1994, 1995 Eric Youngdale
* Copyright (C) 2002-2003 Christoph Hellwig
*
* mid to lowlevel SCSI driver interface
* Initial versions: Drew Eckhardt
* Subsequent revisions: Eric Youngdale
*
* <drew@colorado.edu>
*
* Jiffies wrap fixes (host->resetting), 3 Dec 1998 Andrea Arcangeli
* Added QLOGIC QLA1280 SCSI controller kernel host support.
* August 4, 1999 Fred Lewis, Intel DuPont
*
* Updated to reflect the new initialization scheme for the higher
* level of scsi drivers (sd/sr/st)
* September 17, 2000 Torben Mathiasen <tmm@image.dk>
*
* Restructured scsi_host lists and associated functions.
* September 04, 2002 Mike Anderson (andmike@us.ibm.com)
*/
#include <linux/module.h>
#include <linux/blkdev.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/kthread.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/transport_class.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/idr.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_cmnd.h>
#include "scsi_priv.h"
#include "scsi_logging.h"
static int shost_eh_deadline = -1;
module_param_named(eh_deadline, shost_eh_deadline, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(eh_deadline,
"SCSI EH timeout in seconds (should be between 0 and 2^31-1)");
static DEFINE_IDA(host_index_ida);
static void scsi_host_cls_release(struct device *dev)
{
put_device(&class_to_shost(dev)->shost_gendev);
}
static struct class shost_class = {
.name = "scsi_host",
.dev_release = scsi_host_cls_release,
.dev_groups = scsi_shost_groups,
};
/**
* scsi_host_set_state - Take the given host through the host state model.
* @shost: scsi host to change the state of.
* @state: state to change to.
*
* Returns zero if unsuccessful or an error if the requested
* transition is illegal.
**/
int scsi_host_set_state(struct Scsi_Host *shost, enum scsi_host_state state)
{
enum scsi_host_state oldstate = shost->shost_state;
if (state == oldstate)
return 0;
switch (state) {
case SHOST_CREATED:
/* There are no legal states that come back to
* created. This is the manually initialised start
* state */
goto illegal;
case SHOST_RUNNING:
switch (oldstate) {
case SHOST_CREATED:
case SHOST_RECOVERY:
break;
default:
goto illegal;
}
break;
case SHOST_RECOVERY:
switch (oldstate) {
case SHOST_RUNNING:
break;
default:
goto illegal;
}
break;
case SHOST_CANCEL:
switch (oldstate) {
case SHOST_CREATED:
case SHOST_RUNNING:
case SHOST_CANCEL_RECOVERY:
break;
default:
goto illegal;
}
break;
case SHOST_DEL:
switch (oldstate) {
case SHOST_CANCEL:
case SHOST_DEL_RECOVERY:
break;
default:
goto illegal;
}
break;
case SHOST_CANCEL_RECOVERY:
switch (oldstate) {
case SHOST_CANCEL:
case SHOST_RECOVERY:
break;
default:
goto illegal;
}
break;
case SHOST_DEL_RECOVERY:
switch (oldstate) {
case SHOST_CANCEL_RECOVERY:
break;
default:
goto illegal;
}
break;
}
shost->shost_state = state;
return 0;
illegal:
SCSI_LOG_ERROR_RECOVERY(1,
shost_printk(KERN_ERR, shost,
"Illegal host state transition"
"%s->%s\n",
scsi_host_state_name(oldstate),
scsi_host_state_name(state)));
return -EINVAL;
}
/**
* scsi_remove_host - remove a scsi host
* @shost: a pointer to a scsi host to remove
**/
void scsi_remove_host(struct Scsi_Host *shost)
{
unsigned long flags;
mutex_lock(&shost->scan_mutex);
spin_lock_irqsave(shost->host_lock, flags);
if (scsi_host_set_state(shost, SHOST_CANCEL)) if (scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY)) { spin_unlock_irqrestore(shost->host_lock, flags);
mutex_unlock(&shost->scan_mutex);
return;
}
spin_unlock_irqrestore(shost->host_lock, flags);
scsi_autopm_get_host(shost);
flush_workqueue(shost->tmf_work_q);
scsi_forget_host(shost);
mutex_unlock(&shost->scan_mutex);
scsi_proc_host_rm(shost);
scsi_proc_hostdir_rm(shost->hostt);
/*
* New SCSI devices cannot be attached anymore because of the SCSI host
* state so drop the tag set refcnt. Wait until the tag set refcnt drops
* to zero because .exit_cmd_priv implementations may need the host
* pointer.
*/
kref_put(&shost->tagset_refcnt, scsi_mq_free_tags); wait_for_completion(&shost->tagset_freed);
spin_lock_irqsave(shost->host_lock, flags);
if (scsi_host_set_state(shost, SHOST_DEL)) BUG_ON(scsi_host_set_state(shost, SHOST_DEL_RECOVERY)); spin_unlock_irqrestore(shost->host_lock, flags);
transport_unregister_device(&shost->shost_gendev);
device_unregister(&shost->shost_dev);
device_del(&shost->shost_gendev);
}
EXPORT_SYMBOL(scsi_remove_host);
/**
* scsi_add_host_with_dma - add a scsi host with dma device
* @shost: scsi host pointer to add
* @dev: a struct device of type scsi class
* @dma_dev: dma device for the host
*
* Note: You rarely need to worry about this unless you're in a
* virtualised host environments, so use the simpler scsi_add_host()
* function instead.
*
* Return value:
* 0 on success / != 0 for error
**/
int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
struct device *dma_dev)
{
const struct scsi_host_template *sht = shost->hostt;
int error = -EINVAL;
shost_printk(KERN_INFO, shost, "%s\n",
sht->info ? sht->info(shost) : sht->name);
if (!shost->can_queue) {
shost_printk(KERN_ERR, shost,
"can_queue = 0 no longer supported\n");
goto fail;
}
/* Use min_t(int, ...) in case shost->can_queue exceeds SHRT_MAX */
shost->cmd_per_lun = min_t(int, shost->cmd_per_lun,
shost->can_queue);
error = scsi_init_sense_cache(shost);
if (error)
goto fail;
if (!shost->shost_gendev.parent)
shost->shost_gendev.parent = dev ? dev : &platform_bus;
if (!dma_dev)
dma_dev = shost->shost_gendev.parent;
shost->dma_dev = dma_dev;
if (dma_dev->dma_mask) {
shost->max_sectors = min_t(unsigned int, shost->max_sectors,
dma_max_mapping_size(dma_dev) >> SECTOR_SHIFT);
}
error = scsi_mq_setup_tags(shost);
if (error)
goto fail;
kref_init(&shost->tagset_refcnt);
init_completion(&shost->tagset_freed);
/*
* Increase usage count temporarily here so that calling
* scsi_autopm_put_host() will trigger runtime idle if there is
* nothing else preventing suspending the device.
*/
pm_runtime_get_noresume(&shost->shost_gendev);
pm_runtime_set_active(&shost->shost_gendev);
pm_runtime_enable(&shost->shost_gendev);
device_enable_async_suspend(&shost->shost_gendev);
error = device_add(&shost->shost_gendev);
if (error)
goto out_disable_runtime_pm;
scsi_host_set_state(shost, SHOST_RUNNING);
get_device(shost->shost_gendev.parent);
device_enable_async_suspend(&shost->shost_dev);
get_device(&shost->shost_gendev);
error = device_add(&shost->shost_dev);
if (error)
goto out_del_gendev;
if (shost->transportt->host_size) {
shost->shost_data = kzalloc(shost->transportt->host_size,
GFP_KERNEL);
if (shost->shost_data == NULL) {
error = -ENOMEM;
goto out_del_dev;
}
}
if (shost->transportt->create_work_queue) {
snprintf(shost->work_q_name, sizeof(shost->work_q_name),
"scsi_wq_%d", shost->host_no);
shost->work_q = alloc_workqueue("%s",
WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_UNBOUND,
1, shost->work_q_name);
if (!shost->work_q) {
error = -EINVAL;
goto out_del_dev;
}
}
error = scsi_sysfs_add_host(shost);
if (error)
goto out_del_dev;
scsi_proc_host_add(shost);
scsi_autopm_put_host(shost);
return error;
/*
* Any host allocation in this function will be freed in
* scsi_host_dev_release().
*/
out_del_dev:
device_del(&shost->shost_dev);
out_del_gendev:
/*
* Host state is SHOST_RUNNING so we have to explicitly release
* ->shost_dev.
*/
put_device(&shost->shost_dev);
device_del(&shost->shost_gendev);
out_disable_runtime_pm:
device_disable_async_suspend(&shost->shost_gendev);
pm_runtime_disable(&shost->shost_gendev);
pm_runtime_set_suspended(&shost->shost_gendev);
pm_runtime_put_noidle(&shost->shost_gendev);
kref_put(&shost->tagset_refcnt, scsi_mq_free_tags);
fail:
return error;
}
EXPORT_SYMBOL(scsi_add_host_with_dma);
static void scsi_host_dev_release(struct device *dev)
{
struct Scsi_Host *shost = dev_to_shost(dev); struct device *parent = dev->parent;
/* Wait for functions invoked through call_rcu(&scmd->rcu, ...) */
rcu_barrier();
if (shost->tmf_work_q)
destroy_workqueue(shost->tmf_work_q); if (shost->ehandler) kthread_stop(shost->ehandler); if (shost->work_q) destroy_workqueue(shost->work_q); if (shost->shost_state == SHOST_CREATED) {
/*
* Free the shost_dev device name and remove the proc host dir
* here if scsi_host_{alloc,put}() have been called but neither
* scsi_host_add() nor scsi_remove_host() has been called.
* This avoids that the memory allocated for the shost_dev
* name as well as the proc dir structure are leaked.
*/
scsi_proc_hostdir_rm(shost->hostt); kfree(dev_name(&shost->shost_dev));
}
kfree(shost->shost_data);
ida_free(&host_index_ida, shost->host_no);
if (shost->shost_state != SHOST_CREATED)
put_device(parent); kfree(shost);
}
static const struct device_type scsi_host_type = {
.name = "scsi_host",
.release = scsi_host_dev_release,
};
/**
* scsi_host_alloc - register a scsi host adapter instance.
* @sht: pointer to scsi host template
* @privsize: extra bytes to allocate for driver
*
* Note:
* Allocate a new Scsi_Host and perform basic initialization.
* The host is not published to the scsi midlayer until scsi_add_host
* is called.
*
* Return value:
* Pointer to a new Scsi_Host
**/
struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int privsize)
{
struct Scsi_Host *shost;
int index;
shost = kzalloc(sizeof(struct Scsi_Host) + privsize, GFP_KERNEL);
if (!shost)
return NULL;
shost->host_lock = &shost->default_lock;
spin_lock_init(shost->host_lock);
shost->shost_state = SHOST_CREATED;
INIT_LIST_HEAD(&shost->__devices);
INIT_LIST_HEAD(&shost->__targets);
INIT_LIST_HEAD(&shost->eh_abort_list);
INIT_LIST_HEAD(&shost->eh_cmd_q);
INIT_LIST_HEAD(&shost->starved_list);
init_waitqueue_head(&shost->host_wait);
mutex_init(&shost->scan_mutex);
index = ida_alloc(&host_index_ida, GFP_KERNEL);
if (index < 0) {
kfree(shost);
return NULL;
}
shost->host_no = index;
shost->dma_channel = 0xff;
/* These three are default values which can be overridden */
shost->max_channel = 0;
shost->max_id = 8;
shost->max_lun = 8;
/* Give each shost a default transportt */
shost->transportt = &blank_transport_template;
/*
* All drivers right now should be able to handle 12 byte
* commands. Every so often there are requests for 16 byte
* commands, but individual low-level drivers need to certify that
* they actually do something sensible with such commands.
*/
shost->max_cmd_len = 12;
shost->hostt = sht;
shost->this_id = sht->this_id;
shost->can_queue = sht->can_queue;
shost->sg_tablesize = sht->sg_tablesize;
shost->sg_prot_tablesize = sht->sg_prot_tablesize;
shost->cmd_per_lun = sht->cmd_per_lun;
shost->no_write_same = sht->no_write_same;
shost->host_tagset = sht->host_tagset;
shost->queuecommand_may_block = sht->queuecommand_may_block;
if (shost_eh_deadline == -1 || !sht->eh_host_reset_handler)
shost->eh_deadline = -1;
else if ((ulong) shost_eh_deadline * HZ > INT_MAX) {
shost_printk(KERN_WARNING, shost,
"eh_deadline %u too large, setting to %u\n",
shost_eh_deadline, INT_MAX / HZ);
shost->eh_deadline = INT_MAX;
} else
shost->eh_deadline = shost_eh_deadline * HZ;
if (sht->supported_mode == MODE_UNKNOWN)
/* means we didn't set it ... default to INITIATOR */
shost->active_mode = MODE_INITIATOR;
else
shost->active_mode = sht->supported_mode;
if (sht->max_host_blocked)
shost->max_host_blocked = sht->max_host_blocked;
else
shost->max_host_blocked = SCSI_DEFAULT_HOST_BLOCKED;
/*
* If the driver imposes no hard sector transfer limit, start at
* machine infinity initially.
*/
if (sht->max_sectors)
shost->max_sectors = sht->max_sectors;
else
shost->max_sectors = SCSI_DEFAULT_MAX_SECTORS;
if (sht->max_segment_size)
shost->max_segment_size = sht->max_segment_size;
else
shost->max_segment_size = BLK_MAX_SEGMENT_SIZE;
/*
* assume a 4GB boundary, if not set
*/
if (sht->dma_boundary)
shost->dma_boundary = sht->dma_boundary;
else
shost->dma_boundary = 0xffffffff;
if (sht->virt_boundary_mask)
shost->virt_boundary_mask = sht->virt_boundary_mask;
device_initialize(&shost->shost_gendev);
dev_set_name(&shost->shost_gendev, "host%d", shost->host_no);
shost->shost_gendev.bus = &scsi_bus_type;
shost->shost_gendev.type = &scsi_host_type;
scsi_enable_async_suspend(&shost->shost_gendev);
device_initialize(&shost->shost_dev);
shost->shost_dev.parent = &shost->shost_gendev;
shost->shost_dev.class = &shost_class;
dev_set_name(&shost->shost_dev, "host%d", shost->host_no);
shost->shost_dev.groups = sht->shost_groups;
shost->ehandler = kthread_run(scsi_error_handler, shost,
"scsi_eh_%d", shost->host_no);
if (IS_ERR(shost->ehandler)) {
shost_printk(KERN_WARNING, shost,
"error handler thread failed to spawn, error = %ld\n",
PTR_ERR(shost->ehandler));
shost->ehandler = NULL;
goto fail;
}
shost->tmf_work_q = alloc_workqueue("scsi_tmf_%d",
WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS,
1, shost->host_no);
if (!shost->tmf_work_q) {
shost_printk(KERN_WARNING, shost,
"failed to create tmf workq\n");
goto fail;
}
if (scsi_proc_hostdir_add(shost->hostt) < 0)
goto fail;
return shost;
fail:
/*
* Host state is still SHOST_CREATED and that is enough to release
* ->shost_gendev. scsi_host_dev_release() will free
* dev_name(&shost->shost_dev).
*/
put_device(&shost->shost_gendev);
return NULL;
}
EXPORT_SYMBOL(scsi_host_alloc);
static int __scsi_host_match(struct device *dev, const void *data)
{
struct Scsi_Host *p;
const unsigned int *hostnum = data;
p = class_to_shost(dev);
return p->host_no == *hostnum;
}
/**
* scsi_host_lookup - get a reference to a Scsi_Host by host no
* @hostnum: host number to locate
*
* Return value:
* A pointer to located Scsi_Host or NULL.
*
* The caller must do a scsi_host_put() to drop the reference
* that scsi_host_get() took. The put_device() below dropped
* the reference from class_find_device().
**/
struct Scsi_Host *scsi_host_lookup(unsigned int hostnum)
{
struct device *cdev;
struct Scsi_Host *shost = NULL;
cdev = class_find_device(&shost_class, NULL, &hostnum,
__scsi_host_match);
if (cdev) {
shost = scsi_host_get(class_to_shost(cdev));
put_device(cdev);
}
return shost;
}
EXPORT_SYMBOL(scsi_host_lookup);
/**
* scsi_host_get - inc a Scsi_Host ref count
* @shost: Pointer to Scsi_Host to inc.
**/
struct Scsi_Host *scsi_host_get(struct Scsi_Host *shost)
{
if ((shost->shost_state == SHOST_DEL) ||
!get_device(&shost->shost_gendev))
return NULL;
return shost;
}
EXPORT_SYMBOL(scsi_host_get);
static bool scsi_host_check_in_flight(struct request *rq, void *data)
{
int *count = data;
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
if (test_bit(SCMD_STATE_INFLIGHT, &cmd->state))
(*count)++;
return true;
}
/**
* scsi_host_busy - Return the host busy counter
* @shost: Pointer to Scsi_Host to inc.
**/
int scsi_host_busy(struct Scsi_Host *shost)
{
int cnt = 0;
blk_mq_tagset_busy_iter(&shost->tag_set,
scsi_host_check_in_flight, &cnt);
return cnt;
}
EXPORT_SYMBOL(scsi_host_busy);
/**
* scsi_host_put - dec a Scsi_Host ref count
* @shost: Pointer to Scsi_Host to dec.
**/
void scsi_host_put(struct Scsi_Host *shost)
{
put_device(&shost->shost_gendev);
}
EXPORT_SYMBOL(scsi_host_put);
int scsi_init_hosts(void)
{
return class_register(&shost_class);
}
void scsi_exit_hosts(void)
{
class_unregister(&shost_class);
ida_destroy(&host_index_ida);
}
int scsi_is_host_device(const struct device *dev)
{
return dev->type == &scsi_host_type;
}
EXPORT_SYMBOL(scsi_is_host_device);
/**
* scsi_queue_work - Queue work to the Scsi_Host workqueue.
* @shost: Pointer to Scsi_Host.
* @work: Work to queue for execution.
*
* Return value:
* 1 - work queued for execution
* 0 - work is already queued
* -EINVAL - work queue doesn't exist
**/
int scsi_queue_work(struct Scsi_Host *shost, struct work_struct *work)
{
if (unlikely(!shost->work_q)) {
shost_printk(KERN_ERR, shost,
"ERROR: Scsi host '%s' attempted to queue scsi-work, "
"when no workqueue created.\n", shost->hostt->name);
dump_stack();
return -EINVAL;
}
return queue_work(shost->work_q, work);
}
EXPORT_SYMBOL_GPL(scsi_queue_work);
/**
* scsi_flush_work - Flush a Scsi_Host's workqueue.
* @shost: Pointer to Scsi_Host.
**/
void scsi_flush_work(struct Scsi_Host *shost)
{
if (!shost->work_q) {
shost_printk(KERN_ERR, shost,
"ERROR: Scsi host '%s' attempted to flush scsi-work, "
"when no workqueue created.\n", shost->hostt->name);
dump_stack();
return;
}
flush_workqueue(shost->work_q);
}
EXPORT_SYMBOL_GPL(scsi_flush_work);
static bool complete_all_cmds_iter(struct request *rq, void *data)
{
struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
enum scsi_host_status status = *(enum scsi_host_status *)data;
scsi_dma_unmap(scmd);
scmd->result = 0;
set_host_byte(scmd, status);
scsi_done(scmd);
return true;
}
/**
* scsi_host_complete_all_commands - Terminate all running commands
* @shost: Scsi Host on which commands should be terminated
* @status: Status to be set for the terminated commands
*
* There is no protection against modification of the number
* of outstanding commands. It is the responsibility of the
* caller to ensure that concurrent I/O submission and/or
* completion is stopped when calling this function.
*/
void scsi_host_complete_all_commands(struct Scsi_Host *shost,
enum scsi_host_status status)
{
blk_mq_tagset_busy_iter(&shost->tag_set, complete_all_cmds_iter,
&status);
}
EXPORT_SYMBOL_GPL(scsi_host_complete_all_commands);
struct scsi_host_busy_iter_data {
bool (*fn)(struct scsi_cmnd *, void *);
void *priv;
};
static bool __scsi_host_busy_iter_fn(struct request *req, void *priv)
{
struct scsi_host_busy_iter_data *iter_data = priv;
struct scsi_cmnd *sc = blk_mq_rq_to_pdu(req);
return iter_data->fn(sc, iter_data->priv);
}
/**
* scsi_host_busy_iter - Iterate over all busy commands
* @shost: Pointer to Scsi_Host.
* @fn: Function to call on each busy command
* @priv: Data pointer passed to @fn
*
* If locking against concurrent command completions is required
* ithas to be provided by the caller
**/
void scsi_host_busy_iter(struct Scsi_Host *shost,
bool (*fn)(struct scsi_cmnd *, void *),
void *priv)
{
struct scsi_host_busy_iter_data iter_data = {
.fn = fn,
.priv = priv,
};
blk_mq_tagset_busy_iter(&shost->tag_set, __scsi_host_busy_iter_fn,
&iter_data);
}
EXPORT_SYMBOL_GPL(scsi_host_busy_iter);
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
* Author: Joerg Roedel <jroedel@suse.de>
*/
#define pr_fmt(fmt) "iommu: " fmt
#include <linux/amba/bus.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/bits.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/host1x_context_bus.h>
#include <linux/iommu.h>
#include <linux/idr.h>
#include <linux/err.h>
#include <linux/pci.h>
#include <linux/pci-ats.h>
#include <linux/bitops.h>
#include <linux/platform_device.h>
#include <linux/property.h>
#include <linux/fsl/mc.h>
#include <linux/module.h>
#include <linux/cc_platform.h>
#include <linux/cdx/cdx_bus.h>
#include <trace/events/iommu.h>
#include <linux/sched/mm.h>
#include <linux/msi.h>
#include "dma-iommu.h"
#include "iommu-priv.h"
static struct kset *iommu_group_kset;
static DEFINE_IDA(iommu_group_ida);
static DEFINE_IDA(iommu_global_pasid_ida);
static unsigned int iommu_def_domain_type __read_mostly;
static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
static u32 iommu_cmd_line __read_mostly;
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
struct xarray pasid_array;
struct mutex mutex;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *blocking_domain;
struct iommu_domain *domain;
struct list_head entry;
unsigned int owner_cnt;
void *owner;
};
struct group_device {
struct list_head list;
struct device *dev;
char *name;
};
/* Iterate over each struct group_device in a struct iommu_group */
#define for_each_group_device(group, pos) \
list_for_each_entry(pos, &(group)->devices, list)
struct iommu_group_attribute {
struct attribute attr;
ssize_t (*show)(struct iommu_group *group, char *buf);
ssize_t (*store)(struct iommu_group *group,
const char *buf, size_t count);
};
static const char * const iommu_group_resv_type_string[] = {
[IOMMU_RESV_DIRECT] = "direct",
[IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
[IOMMU_RESV_RESERVED] = "reserved",
[IOMMU_RESV_MSI] = "msi",
[IOMMU_RESV_SW_MSI] = "msi",
};
#define IOMMU_CMD_LINE_DMA_API BIT(0)
#define IOMMU_CMD_LINE_STRICT BIT(1)
static int iommu_bus_notifier(struct notifier_block *nb,
unsigned long action, void *data);
static void iommu_release_device(struct device *dev);
static struct iommu_domain *
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type);
static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev);
static int __iommu_attach_group(struct iommu_domain *domain,
struct iommu_group *group);
enum {
IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
};
static int __iommu_device_set_domain(struct iommu_group *group,
struct device *dev,
struct iommu_domain *new_domain,
unsigned int flags);
static int __iommu_group_set_domain_internal(struct iommu_group *group,
struct iommu_domain *new_domain,
unsigned int flags);
static int __iommu_group_set_domain(struct iommu_group *group,
struct iommu_domain *new_domain)
{
return __iommu_group_set_domain_internal(group, new_domain, 0);
}
static void __iommu_group_set_domain_nofail(struct iommu_group *group,
struct iommu_domain *new_domain)
{
WARN_ON(__iommu_group_set_domain_internal(
group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED));
}
static int iommu_setup_default_domain(struct iommu_group *group,
int target_type);
static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
struct device *dev);
static ssize_t iommu_group_store_type(struct iommu_group *group,
const char *buf, size_t count);
static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
struct device *dev);
static void __iommu_group_free_device(struct iommu_group *group,
struct group_device *grp_dev);
#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
struct iommu_group_attribute iommu_group_attr_##_name = \
__ATTR(_name, _mode, _show, _store)
#define to_iommu_group_attr(_attr) \
container_of(_attr, struct iommu_group_attribute, attr)
#define to_iommu_group(_kobj) \
container_of(_kobj, struct iommu_group, kobj)
static LIST_HEAD(iommu_device_list);
static DEFINE_SPINLOCK(iommu_device_lock);
static const struct bus_type * const iommu_buses[] = {
&platform_bus_type,
#ifdef CONFIG_PCI
&pci_bus_type,
#endif
#ifdef CONFIG_ARM_AMBA
&amba_bustype,
#endif
#ifdef CONFIG_FSL_MC_BUS
&fsl_mc_bus_type,
#endif
#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
&host1x_context_device_bus_type,
#endif
#ifdef CONFIG_CDX_BUS
&cdx_bus_type,
#endif
};
/*
* Use a function instead of an array here because the domain-type is a
* bit-field, so an array would waste memory.
*/
static const char *iommu_domain_type_str(unsigned int t)
{
switch (t) {
case IOMMU_DOMAIN_BLOCKED:
return "Blocked";
case IOMMU_DOMAIN_IDENTITY:
return "Passthrough";
case IOMMU_DOMAIN_UNMANAGED:
return "Unmanaged";
case IOMMU_DOMAIN_DMA:
case IOMMU_DOMAIN_DMA_FQ:
return "Translated";
case IOMMU_DOMAIN_PLATFORM:
return "Platform";
default:
return "Unknown";
}
}
static int __init iommu_subsys_init(void)
{
struct notifier_block *nb;
if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) {
if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
iommu_set_default_passthrough(false);
else
iommu_set_default_translated(false);
if (iommu_default_passthrough() && cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
iommu_set_default_translated(false);
}
}
if (!iommu_default_passthrough() && !iommu_dma_strict)
iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ;
pr_info("Default domain type: %s%s\n",
iommu_domain_type_str(iommu_def_domain_type),
(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ?
" (set via kernel command line)" : "");
if (!iommu_default_passthrough())
pr_info("DMA domain TLB invalidation policy: %s mode%s\n",
iommu_dma_strict ? "strict" : "lazy",
(iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ?
" (set via kernel command line)" : "");
nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL);
if (!nb)
return -ENOMEM;
for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
nb[i].notifier_call = iommu_bus_notifier;
bus_register_notifier(iommu_buses[i], &nb[i]);
}
return 0;
}
subsys_initcall(iommu_subsys_init);
static int remove_iommu_group(struct device *dev, void *data)
{
if (dev->iommu && dev->iommu->iommu_dev == data)
iommu_release_device(dev);
return 0;
}
/**
* iommu_device_register() - Register an IOMMU hardware instance
* @iommu: IOMMU handle for the instance
* @ops: IOMMU ops to associate with the instance
* @hwdev: (optional) actual instance device, used for fwnode lookup
*
* Return: 0 on success, or an error.
*/
int iommu_device_register(struct iommu_device *iommu,
const struct iommu_ops *ops, struct device *hwdev)
{
int err = 0;
/* We need to be able to take module references appropriately */
if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner))
return -EINVAL;
iommu->ops = ops;
if (hwdev)
iommu->fwnode = dev_fwnode(hwdev);
spin_lock(&iommu_device_lock);
list_add_tail(&iommu->list, &iommu_device_list);
spin_unlock(&iommu_device_lock);
for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++)
err = bus_iommu_probe(iommu_buses[i]);
if (err)
iommu_device_unregister(iommu);
return err;
}
EXPORT_SYMBOL_GPL(iommu_device_register);
void iommu_device_unregister(struct iommu_device *iommu)
{
for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)
bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);
spin_lock(&iommu_device_lock);
list_del(&iommu->list);
spin_unlock(&iommu_device_lock);
/* Pairs with the alloc in generic_single_device_group() */
iommu_group_put(iommu->singleton_group);
iommu->singleton_group = NULL;
}
EXPORT_SYMBOL_GPL(iommu_device_unregister);
#if IS_ENABLED(CONFIG_IOMMUFD_TEST)
void iommu_device_unregister_bus(struct iommu_device *iommu,
const struct bus_type *bus,
struct notifier_block *nb)
{
bus_unregister_notifier(bus, nb);
iommu_device_unregister(iommu);
}
EXPORT_SYMBOL_GPL(iommu_device_unregister_bus);
/*
* Register an iommu driver against a single bus. This is only used by iommufd
* selftest to create a mock iommu driver. The caller must provide
* some memory to hold a notifier_block.
*/
int iommu_device_register_bus(struct iommu_device *iommu,
const struct iommu_ops *ops,
const struct bus_type *bus,
struct notifier_block *nb)
{
int err;
iommu->ops = ops;
nb->notifier_call = iommu_bus_notifier;
err = bus_register_notifier(bus, nb);
if (err)
return err;
spin_lock(&iommu_device_lock);
list_add_tail(&iommu->list, &iommu_device_list);
spin_unlock(&iommu_device_lock);
err = bus_iommu_probe(bus);
if (err) {
iommu_device_unregister_bus(iommu, bus, nb);
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(iommu_device_register_bus);
#endif
static struct dev_iommu *dev_iommu_get(struct device *dev)
{
struct dev_iommu *param = dev->iommu;
lockdep_assert_held(&iommu_probe_device_lock);
if (param)
return param;
param = kzalloc(sizeof(*param), GFP_KERNEL);
if (!param)
return NULL;
mutex_init(¶m->lock);
dev->iommu = param;
return param;
}
static void dev_iommu_free(struct device *dev)
{
struct dev_iommu *param = dev->iommu;
dev->iommu = NULL;
if (param->fwspec) {
fwnode_handle_put(param->fwspec->iommu_fwnode);
kfree(param->fwspec);
}
kfree(param);
}
/*
* Internal equivalent of device_iommu_mapped() for when we care that a device
* actually has API ops, and don't want false positives from VFIO-only groups.
*/
static bool dev_has_iommu(struct device *dev)
{
return dev->iommu && dev->iommu->iommu_dev;
}
static u32 dev_iommu_get_max_pasids(struct device *dev)
{
u32 max_pasids = 0, bits = 0;
int ret;
if (dev_is_pci(dev)) {
ret = pci_max_pasids(to_pci_dev(dev));
if (ret > 0)
max_pasids = ret;
} else {
ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
if (!ret)
max_pasids = 1UL << bits;
}
return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
}
void dev_iommu_priv_set(struct device *dev, void *priv)
{
/* FSL_PAMU does something weird */
if (!IS_ENABLED(CONFIG_FSL_PAMU))
lockdep_assert_held(&iommu_probe_device_lock);
dev->iommu->priv = priv;
}
EXPORT_SYMBOL_GPL(dev_iommu_priv_set);
/*
* Init the dev->iommu and dev->iommu_group in the struct device and get the
* driver probed
*/
static int iommu_init_device(struct device *dev, const struct iommu_ops *ops)
{
struct iommu_device *iommu_dev;
struct iommu_group *group;
int ret;
if (!dev_iommu_get(dev))
return -ENOMEM;
if (!try_module_get(ops->owner)) {
ret = -EINVAL;
goto err_free;
}
iommu_dev = ops->probe_device(dev);
if (IS_ERR(iommu_dev)) {
ret = PTR_ERR(iommu_dev);
goto err_module_put;
}
dev->iommu->iommu_dev = iommu_dev;
ret = iommu_device_link(iommu_dev, dev);
if (ret)
goto err_release;
group = ops->device_group(dev);
if (WARN_ON_ONCE(group == NULL))
group = ERR_PTR(-EINVAL);
if (IS_ERR(group)) {
ret = PTR_ERR(group);
goto err_unlink;
}
dev->iommu_group = group;
dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
if (ops->is_attach_deferred)
dev->iommu->attach_deferred = ops->is_attach_deferred(dev);
return 0;
err_unlink:
iommu_device_unlink(iommu_dev, dev);
err_release:
if (ops->release_device)
ops->release_device(dev);
err_module_put:
module_put(ops->owner);
err_free:
dev->iommu->iommu_dev = NULL;
dev_iommu_free(dev);
return ret;
}
static void iommu_deinit_device(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
const struct iommu_ops *ops = dev_iommu_ops(dev);
lockdep_assert_held(&group->mutex);
iommu_device_unlink(dev->iommu->iommu_dev, dev);
/*
* release_device() must stop using any attached domain on the device.
* If there are still other devices in the group, they are not affected
* by this callback.
*
* If the iommu driver provides release_domain, the core code ensures
* that domain is attached prior to calling release_device. Drivers can
* use this to enforce a translation on the idle iommu. Typically, the
* global static blocked_domain is a good choice.
*
* Otherwise, the iommu driver must set the device to either an identity
* or a blocking translation in release_device() and stop using any
* domain pointer, as it is going to be freed.
*
* Regardless, if a delayed attach never occurred, then the release
* should still avoid touching any hardware configuration either.
*/
if (!dev->iommu->attach_deferred && ops->release_domain)
ops->release_domain->ops->attach_dev(ops->release_domain, dev);
if (ops->release_device)
ops->release_device(dev);
/*
* If this is the last driver to use the group then we must free the
* domains before we do the module_put().
*/
if (list_empty(&group->devices)) {
if (group->default_domain) {
iommu_domain_free(group->default_domain);
group->default_domain = NULL;
}
if (group->blocking_domain) {
iommu_domain_free(group->blocking_domain);
group->blocking_domain = NULL;
}
group->domain = NULL;
}
/* Caller must put iommu_group */
dev->iommu_group = NULL;
module_put(ops->owner);
dev_iommu_free(dev);
}
DEFINE_MUTEX(iommu_probe_device_lock);
static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
{
const struct iommu_ops *ops;
struct iommu_fwspec *fwspec;
struct iommu_group *group;
struct group_device *gdev;
int ret;
/*
* For FDT-based systems and ACPI IORT/VIOT, drivers register IOMMU
* instances with non-NULL fwnodes, and client devices should have been
* identified with a fwspec by this point. Otherwise, we can currently
* assume that only one of Intel, AMD, s390, PAMU or legacy SMMUv2 can
* be present, and that any of their registered instances has suitable
* ops for probing, and thus cheekily co-opt the same mechanism.
*/
fwspec = dev_iommu_fwspec_get(dev);
if (fwspec && fwspec->ops)
ops = fwspec->ops;
else
ops = iommu_ops_from_fwnode(NULL);
if (!ops)
return -ENODEV;
/*
* Serialise to avoid races between IOMMU drivers registering in
* parallel and/or the "replay" calls from ACPI/OF code via client
* driver probe. Once the latter have been cleaned up we should
* probably be able to use device_lock() here to minimise the scope,
* but for now enforcing a simple global ordering is fine.
*/
lockdep_assert_held(&iommu_probe_device_lock);
/* Device is probed already if in a group */
if (dev->iommu_group)
return 0;
ret = iommu_init_device(dev, ops);
if (ret)
return ret;
group = dev->iommu_group;
gdev = iommu_group_alloc_device(group, dev);
mutex_lock(&group->mutex);
if (IS_ERR(gdev)) {
ret = PTR_ERR(gdev);
goto err_put_group;
}
/*
* The gdev must be in the list before calling
* iommu_setup_default_domain()
*/
list_add_tail(&gdev->list, &group->devices);
WARN_ON(group->default_domain && !group->domain);
if (group->default_domain)
iommu_create_device_direct_mappings(group->default_domain, dev);
if (group->domain) {
ret = __iommu_device_set_domain(group, dev, group->domain, 0);
if (ret)
goto err_remove_gdev;
} else if (!group->default_domain && !group_list) {
ret = iommu_setup_default_domain(group, 0);
if (ret)
goto err_remove_gdev;
} else if (!group->default_domain) {
/*
* With a group_list argument we defer the default_domain setup
* to the caller by providing a de-duplicated list of groups
* that need further setup.
*/
if (list_empty(&group->entry))
list_add_tail(&group->entry, group_list);
}
mutex_unlock(&group->mutex);
if (dev_is_pci(dev))
iommu_dma_set_pci_32bit_workaround(dev);
return 0;
err_remove_gdev:
list_del(&gdev->list);
__iommu_group_free_device(group, gdev);
err_put_group:
iommu_deinit_device(dev);
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
int iommu_probe_device(struct device *dev)
{
const struct iommu_ops *ops;
int ret;
mutex_lock(&iommu_probe_device_lock);
ret = __iommu_probe_device(dev, NULL);
mutex_unlock(&iommu_probe_device_lock);
if (ret)
return ret;
ops = dev_iommu_ops(dev);
if (ops->probe_finalize)
ops->probe_finalize(dev);
return 0;
}
static void __iommu_group_free_device(struct iommu_group *group,
struct group_device *grp_dev)
{
struct device *dev = grp_dev->dev;
sysfs_remove_link(group->devices_kobj, grp_dev->name);
sysfs_remove_link(&dev->kobj, "iommu_group");
trace_remove_device_from_group(group->id, dev);
/*
* If the group has become empty then ownership must have been
* released, and the current domain must be set back to NULL or
* the default domain.
*/
if (list_empty(&group->devices))
WARN_ON(group->owner_cnt ||
group->domain != group->default_domain);
kfree(grp_dev->name);
kfree(grp_dev);
}
/* Remove the iommu_group from the struct device. */
static void __iommu_group_remove_device(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
struct group_device *device;
mutex_lock(&group->mutex);
for_each_group_device(group, device) {
if (device->dev != dev)
continue;
list_del(&device->list);
__iommu_group_free_device(group, device);
if (dev_has_iommu(dev))
iommu_deinit_device(dev);
else
dev->iommu_group = NULL;
break;
}
mutex_unlock(&group->mutex);
/*
* Pairs with the get in iommu_init_device() or
* iommu_group_add_device()
*/
iommu_group_put(group);
}
static void iommu_release_device(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
if (group)
__iommu_group_remove_device(dev);
/* Free any fwspec if no iommu_driver was ever attached */
if (dev->iommu) dev_iommu_free(dev);
}
static int __init iommu_set_def_domain_type(char *str)
{
bool pt;
int ret;
ret = kstrtobool(str, &pt);
if (ret)
return ret;
if (pt)
iommu_set_default_passthrough(true);
else
iommu_set_default_translated(true);
return 0;
}
early_param("iommu.passthrough", iommu_set_def_domain_type);
static int __init iommu_dma_setup(char *str)
{
int ret = kstrtobool(str, &iommu_dma_strict);
if (!ret)
iommu_cmd_line |= IOMMU_CMD_LINE_STRICT;
return ret;
}
early_param("iommu.strict", iommu_dma_setup);
void iommu_set_dma_strict(void)
{
iommu_dma_strict = true;
if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ)
iommu_def_domain_type = IOMMU_DOMAIN_DMA;
}
static ssize_t iommu_group_attr_show(struct kobject *kobj,
struct attribute *__attr, char *buf)
{
struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
struct iommu_group *group = to_iommu_group(kobj);
ssize_t ret = -EIO;
if (attr->show)
ret = attr->show(group, buf);
return ret;
}
static ssize_t iommu_group_attr_store(struct kobject *kobj,
struct attribute *__attr,
const char *buf, size_t count)
{
struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
struct iommu_group *group = to_iommu_group(kobj);
ssize_t ret = -EIO;
if (attr->store)
ret = attr->store(group, buf, count);
return ret;
}
static const struct sysfs_ops iommu_group_sysfs_ops = {
.show = iommu_group_attr_show,
.store = iommu_group_attr_store,
};
static int iommu_group_create_file(struct iommu_group *group,
struct iommu_group_attribute *attr)
{
return sysfs_create_file(&group->kobj, &attr->attr);
}
static void iommu_group_remove_file(struct iommu_group *group,
struct iommu_group_attribute *attr)
{
sysfs_remove_file(&group->kobj, &attr->attr);
}
static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
{
return sysfs_emit(buf, "%s\n", group->name);
}
/**
* iommu_insert_resv_region - Insert a new region in the
* list of reserved regions.
* @new: new region to insert
* @regions: list of regions
*
* Elements are sorted by start address and overlapping segments
* of the same type are merged.
*/
static int iommu_insert_resv_region(struct iommu_resv_region *new,
struct list_head *regions)
{
struct iommu_resv_region *iter, *tmp, *nr, *top;
LIST_HEAD(stack);
nr = iommu_alloc_resv_region(new->start, new->length,
new->prot, new->type, GFP_KERNEL);
if (!nr)
return -ENOMEM;
/* First add the new element based on start address sorting */
list_for_each_entry(iter, regions, list) {
if (nr->start < iter->start ||
(nr->start == iter->start && nr->type <= iter->type))
break;
}
list_add_tail(&nr->list, &iter->list);
/* Merge overlapping segments of type nr->type in @regions, if any */
list_for_each_entry_safe(iter, tmp, regions, list) {
phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
/* no merge needed on elements of different types than @new */
if (iter->type != new->type) {
list_move_tail(&iter->list, &stack);
continue;
}
/* look for the last stack element of same type as @iter */
list_for_each_entry_reverse(top, &stack, list)
if (top->type == iter->type)
goto check_overlap;
list_move_tail(&iter->list, &stack);
continue;
check_overlap:
top_end = top->start + top->length - 1;
if (iter->start > top_end + 1) {
list_move_tail(&iter->list, &stack);
} else {
top->length = max(top_end, iter_end) - top->start + 1;
list_del(&iter->list);
kfree(iter);
}
}
list_splice(&stack, regions);
return 0;
}
static int
iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
struct list_head *group_resv_regions)
{
struct iommu_resv_region *entry;
int ret = 0;
list_for_each_entry(entry, dev_resv_regions, list) {
ret = iommu_insert_resv_region(entry, group_resv_regions);
if (ret)
break;
}
return ret;
}
int iommu_get_group_resv_regions(struct iommu_group *group,
struct list_head *head)
{
struct group_device *device;
int ret = 0;
mutex_lock(&group->mutex);
for_each_group_device(group, device) {
struct list_head dev_resv_regions;
/*
* Non-API groups still expose reserved_regions in sysfs,
* so filter out calls that get here that way.
*/
if (!dev_has_iommu(device->dev))
break;
INIT_LIST_HEAD(&dev_resv_regions);
iommu_get_resv_regions(device->dev, &dev_resv_regions);
ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
iommu_put_resv_regions(device->dev, &dev_resv_regions);
if (ret)
break;
}
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
char *buf)
{
struct iommu_resv_region *region, *next;
struct list_head group_resv_regions;
int offset = 0;
INIT_LIST_HEAD(&group_resv_regions);
iommu_get_group_resv_regions(group, &group_resv_regions);
list_for_each_entry_safe(region, next, &group_resv_regions, list) {
offset += sysfs_emit_at(buf, offset, "0x%016llx 0x%016llx %s\n",
(long long)region->start,
(long long)(region->start +
region->length - 1),
iommu_group_resv_type_string[region->type]);
kfree(region);
}
return offset;
}
static ssize_t iommu_group_show_type(struct iommu_group *group,
char *buf)
{
char *type = "unknown";
mutex_lock(&group->mutex);
if (group->default_domain) {
switch (group->default_domain->type) {
case IOMMU_DOMAIN_BLOCKED:
type = "blocked";
break;
case IOMMU_DOMAIN_IDENTITY:
type = "identity";
break;
case IOMMU_DOMAIN_UNMANAGED:
type = "unmanaged";
break;
case IOMMU_DOMAIN_DMA:
type = "DMA";
break;
case IOMMU_DOMAIN_DMA_FQ:
type = "DMA-FQ";
break;
}
}
mutex_unlock(&group->mutex);
return sysfs_emit(buf, "%s\n", type);
}
static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
static IOMMU_GROUP_ATTR(reserved_regions, 0444,
iommu_group_show_resv_regions, NULL);
static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type,
iommu_group_store_type);
static void iommu_group_release(struct kobject *kobj)
{
struct iommu_group *group = to_iommu_group(kobj);
pr_debug("Releasing group %d\n", group->id);
if (group->iommu_data_release)
group->iommu_data_release(group->iommu_data);
ida_free(&iommu_group_ida, group->id);
/* Domains are free'd by iommu_deinit_device() */
WARN_ON(group->default_domain);
WARN_ON(group->blocking_domain);
kfree(group->name);
kfree(group);
}
static const struct kobj_type iommu_group_ktype = {
.sysfs_ops = &iommu_group_sysfs_ops,
.release = iommu_group_release,
};
/**
* iommu_group_alloc - Allocate a new group
*
* This function is called by an iommu driver to allocate a new iommu
* group. The iommu group represents the minimum granularity of the iommu.
* Upon successful return, the caller holds a reference to the supplied
* group in order to hold the group until devices are added. Use
* iommu_group_put() to release this extra reference count, allowing the
* group to be automatically reclaimed once it has no devices or external
* references.
*/
struct iommu_group *iommu_group_alloc(void)
{
struct iommu_group *group;
int ret;
group = kzalloc(sizeof(*group), GFP_KERNEL);
if (!group)
return ERR_PTR(-ENOMEM);
group->kobj.kset = iommu_group_kset;
mutex_init(&group->mutex);
INIT_LIST_HEAD(&group->devices);
INIT_LIST_HEAD(&group->entry);
xa_init(&group->pasid_array);
ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
if (ret < 0) {
kfree(group);
return ERR_PTR(ret);
}
group->id = ret;
ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
NULL, "%d", group->id);
if (ret) {
kobject_put(&group->kobj);
return ERR_PTR(ret);
}
group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
if (!group->devices_kobj) {
kobject_put(&group->kobj); /* triggers .release & free */
return ERR_PTR(-ENOMEM);
}
/*
* The devices_kobj holds a reference on the group kobject, so
* as long as that exists so will the group. We can therefore
* use the devices_kobj for reference counting.
*/
kobject_put(&group->kobj);
ret = iommu_group_create_file(group,
&iommu_group_attr_reserved_regions);
if (ret) {
kobject_put(group->devices_kobj);
return ERR_PTR(ret);
}
ret = iommu_group_create_file(group, &iommu_group_attr_type);
if (ret) {
kobject_put(group->devices_kobj);
return ERR_PTR(ret);
}
pr_debug("Allocated group %d\n", group->id);
return group;
}
EXPORT_SYMBOL_GPL(iommu_group_alloc);
/**
* iommu_group_get_iommudata - retrieve iommu_data registered for a group
* @group: the group
*
* iommu drivers can store data in the group for use when doing iommu
* operations. This function provides a way to retrieve it. Caller
* should hold a group reference.
*/
void *iommu_group_get_iommudata(struct iommu_group *group)
{
return group->iommu_data;
}
EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
/**
* iommu_group_set_iommudata - set iommu_data for a group
* @group: the group
* @iommu_data: new data
* @release: release function for iommu_data
*
* iommu drivers can store data in the group for use when doing iommu
* operations. This function provides a way to set the data after
* the group has been allocated. Caller should hold a group reference.
*/
void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
void (*release)(void *iommu_data))
{
group->iommu_data = iommu_data;
group->iommu_data_release = release;
}
EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
/**
* iommu_group_set_name - set name for a group
* @group: the group
* @name: name
*
* Allow iommu driver to set a name for a group. When set it will
* appear in a name attribute file under the group in sysfs.
*/
int iommu_group_set_name(struct iommu_group *group, const char *name)
{
int ret;
if (group->name) {
iommu_group_remove_file(group, &iommu_group_attr_name);
kfree(group->name);
group->name = NULL;
if (!name)
return 0;
}
group->name = kstrdup(name, GFP_KERNEL);
if (!group->name)
return -ENOMEM;
ret = iommu_group_create_file(group, &iommu_group_attr_name);
if (ret) {
kfree(group->name);
group->name = NULL;
return ret;
}
return 0;
}
EXPORT_SYMBOL_GPL(iommu_group_set_name);
static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
struct device *dev)
{
struct iommu_resv_region *entry;
struct list_head mappings;
unsigned long pg_size;
int ret = 0;
pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
INIT_LIST_HEAD(&mappings);
if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))
return -EINVAL;
iommu_get_resv_regions(dev, &mappings);
/* We need to consider overlapping regions for different devices */
list_for_each_entry(entry, &mappings, list) {
dma_addr_t start, end, addr;
size_t map_size = 0;
if (entry->type == IOMMU_RESV_DIRECT)
dev->iommu->require_direct = 1;
if ((entry->type != IOMMU_RESV_DIRECT &&
entry->type != IOMMU_RESV_DIRECT_RELAXABLE) ||
!iommu_is_dma_domain(domain))
continue;
start = ALIGN(entry->start, pg_size);
end = ALIGN(entry->start + entry->length, pg_size);
for (addr = start; addr <= end; addr += pg_size) {
phys_addr_t phys_addr;
if (addr == end)
goto map_end;
phys_addr = iommu_iova_to_phys(domain, addr);
if (!phys_addr) {
map_size += pg_size;
continue;
}
map_end:
if (map_size) {
ret = iommu_map(domain, addr - map_size,
addr - map_size, map_size,
entry->prot, GFP_KERNEL);
if (ret)
goto out;
map_size = 0;
}
}
}
if (!list_empty(&mappings) && iommu_is_dma_domain(domain))
iommu_flush_iotlb_all(domain);
out:
iommu_put_resv_regions(dev, &mappings);
return ret;
}
/* This is undone by __iommu_group_free_device() */
static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
struct device *dev)
{
int ret, i = 0;
struct group_device *device;
device = kzalloc(sizeof(*device), GFP_KERNEL);
if (!device)
return ERR_PTR(-ENOMEM);
device->dev = dev;
ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
if (ret)
goto err_free_device;
device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
rename:
if (!device->name) {
ret = -ENOMEM;
goto err_remove_link;
}
ret = sysfs_create_link_nowarn(group->devices_kobj,
&dev->kobj, device->name);
if (ret) {
if (ret == -EEXIST && i >= 0) {
/*
* Account for the slim chance of collision
* and append an instance to the name.
*/
kfree(device->name);
device->name = kasprintf(GFP_KERNEL, "%s.%d",
kobject_name(&dev->kobj), i++);
goto rename;
}
goto err_free_name;
}
trace_add_device_to_group(group->id, dev);
dev_info(dev, "Adding to iommu group %d\n", group->id);
return device;
err_free_name:
kfree(device->name);
err_remove_link:
sysfs_remove_link(&dev->kobj, "iommu_group");
err_free_device:
kfree(device);
dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
return ERR_PTR(ret);
}
/**
* iommu_group_add_device - add a device to an iommu group
* @group: the group into which to add the device (reference should be held)
* @dev: the device
*
* This function is called by an iommu driver to add a device into a
* group. Adding a device increments the group reference count.
*/
int iommu_group_add_device(struct iommu_group *group, struct device *dev)
{
struct group_device *gdev;
gdev = iommu_group_alloc_device(group, dev);
if (IS_ERR(gdev))
return PTR_ERR(gdev);
iommu_group_ref_get(group);
dev->iommu_group = group;
mutex_lock(&group->mutex);
list_add_tail(&gdev->list, &group->devices);
mutex_unlock(&group->mutex);
return 0;
}
EXPORT_SYMBOL_GPL(iommu_group_add_device);
/**
* iommu_group_remove_device - remove a device from it's current group
* @dev: device to be removed
*
* This function is called by an iommu driver to remove the device from
* it's current group. This decrements the iommu group reference count.
*/
void iommu_group_remove_device(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
if (!group)
return;
dev_info(dev, "Removing from iommu group %d\n", group->id);
__iommu_group_remove_device(dev);
}
EXPORT_SYMBOL_GPL(iommu_group_remove_device);
#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
/**
* iommu_group_mutex_assert - Check device group mutex lock
* @dev: the device that has group param set
*
* This function is called by an iommu driver to check whether it holds
* group mutex lock for the given device or not.
*
* Note that this function must be called after device group param is set.
*/
void iommu_group_mutex_assert(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
lockdep_assert_held(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_group_mutex_assert);
#endif
static struct device *iommu_group_first_dev(struct iommu_group *group)
{
lockdep_assert_held(&group->mutex);
return list_first_entry(&group->devices, struct group_device, list)->dev;
}
/**
* iommu_group_for_each_dev - iterate over each device in the group
* @group: the group
* @data: caller opaque data to be passed to callback function
* @fn: caller supplied callback function
*
* This function is called by group users to iterate over group devices.
* Callers should hold a reference count to the group during callback.
* The group->mutex is held across callbacks, which will block calls to
* iommu_group_add/remove_device.
*/
int iommu_group_for_each_dev(struct iommu_group *group, void *data,
int (*fn)(struct device *, void *))
{
struct group_device *device;
int ret = 0;
mutex_lock(&group->mutex);
for_each_group_device(group, device) {
ret = fn(device->dev, data);
if (ret)
break;
}
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
/**
* iommu_group_get - Return the group for a device and increment reference
* @dev: get the group that this device belongs to
*
* This function is called by iommu drivers and users to get the group
* for the specified device. If found, the group is returned and the group
* reference in incremented, else NULL.
*/
struct iommu_group *iommu_group_get(struct device *dev)
{
struct iommu_group *group = dev->iommu_group;
if (group)
kobject_get(group->devices_kobj);
return group;
}
EXPORT_SYMBOL_GPL(iommu_group_get);
/**
* iommu_group_ref_get - Increment reference on a group
* @group: the group to use, must not be NULL
*
* This function is called by iommu drivers to take additional references on an
* existing group. Returns the given group for convenience.
*/
struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
{
kobject_get(group->devices_kobj);
return group;
}
EXPORT_SYMBOL_GPL(iommu_group_ref_get);
/**
* iommu_group_put - Decrement group reference
* @group: the group to use
*
* This function is called by iommu drivers and users to release the
* iommu group. Once the reference count is zero, the group is released.
*/
void iommu_group_put(struct iommu_group *group)
{
if (group)
kobject_put(group->devices_kobj);
}
EXPORT_SYMBOL_GPL(iommu_group_put);
/**
* iommu_group_id - Return ID for a group
* @group: the group to ID
*
* Return the unique ID for the group matching the sysfs group number.
*/
int iommu_group_id(struct iommu_group *group)
{
return group->id;
}
EXPORT_SYMBOL_GPL(iommu_group_id);
static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
unsigned long *devfns);
/*
* To consider a PCI device isolated, we require ACS to support Source
* Validation, Request Redirection, Completer Redirection, and Upstream
* Forwarding. This effectively means that devices cannot spoof their
* requester ID, requests and completions cannot be redirected, and all
* transactions are forwarded upstream, even as it passes through a
* bridge where the target device is downstream.
*/
#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
/*
* For multifunction devices which are not isolated from each other, find
* all the other non-isolated functions and look for existing groups. For
* each function, we also need to look for aliases to or from other devices
* that may already have a group.
*/
static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
unsigned long *devfns)
{
struct pci_dev *tmp = NULL;
struct iommu_group *group;
if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
return NULL;
for_each_pci_dev(tmp) {
if (tmp == pdev || tmp->bus != pdev->bus ||
PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
pci_acs_enabled(tmp, REQ_ACS_FLAGS))
continue;
group = get_pci_alias_group(tmp, devfns);
if (group) {
pci_dev_put(tmp);
return group;
}
}
return NULL;
}
/*
* Look for aliases to or from the given device for existing groups. DMA
* aliases are only supported on the same bus, therefore the search
* space is quite small (especially since we're really only looking at pcie
* device, and therefore only expect multiple slots on the root complex or
* downstream switch ports). It's conceivable though that a pair of
* multifunction devices could have aliases between them that would cause a
* loop. To prevent this, we use a bitmap to track where we've been.
*/
static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
unsigned long *devfns)
{
struct pci_dev *tmp = NULL;
struct iommu_group *group;
if (test_and_set_bit(pdev->devfn & 0xff, devfns))
return NULL;
group = iommu_group_get(&pdev->dev);
if (group)
return group;
for_each_pci_dev(tmp) {
if (tmp == pdev || tmp->bus != pdev->bus)
continue;
/* We alias them or they alias us */
if (pci_devs_are_dma_aliases(pdev, tmp)) {
group = get_pci_alias_group(tmp, devfns);
if (group) {
pci_dev_put(tmp);
return group;
}
group = get_pci_function_alias_group(tmp, devfns);
if (group) {
pci_dev_put(tmp);
return group;
}
}
}
return NULL;
}
struct group_for_pci_data {
struct pci_dev *pdev;
struct iommu_group *group;
};
/*
* DMA alias iterator callback, return the last seen device. Stop and return
* the IOMMU group if we find one along the way.
*/
static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
{
struct group_for_pci_data *data = opaque;
data->pdev = pdev;
data->group = iommu_group_get(&pdev->dev);
return data->group != NULL;
}
/*
* Generic device_group call-back function. It just allocates one
* iommu-group per device.
*/
struct iommu_group *generic_device_group(struct device *dev)
{
return iommu_group_alloc();
}
EXPORT_SYMBOL_GPL(generic_device_group);
/*
* Generic device_group call-back function. It just allocates one
* iommu-group per iommu driver instance shared by every device
* probed by that iommu driver.
*/
struct iommu_group *generic_single_device_group(struct device *dev)
{
struct iommu_device *iommu = dev->iommu->iommu_dev;
if (!iommu->singleton_group) {
struct iommu_group *group;
group = iommu_group_alloc();
if (IS_ERR(group))
return group;
iommu->singleton_group = group;
}
return iommu_group_ref_get(iommu->singleton_group);
}
EXPORT_SYMBOL_GPL(generic_single_device_group);
/*
* Use standard PCI bus topology, isolation features, and DMA alias quirks
* to find or create an IOMMU group for a device.
*/
struct iommu_group *pci_device_group(struct device *dev)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct group_for_pci_data data;
struct pci_bus *bus;
struct iommu_group *group = NULL;
u64 devfns[4] = { 0 };
if (WARN_ON(!dev_is_pci(dev)))
return ERR_PTR(-EINVAL);
/*
* Find the upstream DMA alias for the device. A device must not
* be aliased due to topology in order to have its own IOMMU group.
* If we find an alias along the way that already belongs to a
* group, use it.
*/
if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
return data.group;
pdev = data.pdev;
/*
* Continue upstream from the point of minimum IOMMU granularity
* due to aliases to the point where devices are protected from
* peer-to-peer DMA by PCI ACS. Again, if we find an existing
* group, use it.
*/
for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
if (!bus->self)
continue;
if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
break;
pdev = bus->self;
group = iommu_group_get(&pdev->dev);
if (group)
return group;
}
/*
* Look for existing groups on device aliases. If we alias another
* device or another device aliases us, use the same group.
*/
group = get_pci_alias_group(pdev, (unsigned long *)devfns);
if (group)
return group;
/*
* Look for existing groups on non-isolated functions on the same
* slot and aliases of those funcions, if any. No need to clear
* the search bitmap, the tested devfns are still valid.
*/
group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
if (group)
return group;
/* No shared group found, allocate new */
return iommu_group_alloc();
}
EXPORT_SYMBOL_GPL(pci_device_group);
/* Get the IOMMU group for device on fsl-mc bus */
struct iommu_group *fsl_mc_device_group(struct device *dev)
{
struct device *cont_dev = fsl_mc_cont_dev(dev);
struct iommu_group *group;
group = iommu_group_get(cont_dev);
if (!group)
group = iommu_group_alloc();
return group;
}
EXPORT_SYMBOL_GPL(fsl_mc_device_group);
static struct iommu_domain *
__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
{
if (group->default_domain && group->default_domain->type == req_type)
return group->default_domain;
return __iommu_group_domain_alloc(group, req_type);
}
/*
* req_type of 0 means "auto" which means to select a domain based on
* iommu_def_domain_type or what the driver actually supports.
*/
static struct iommu_domain *
iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
{
const struct iommu_ops *ops = dev_iommu_ops(iommu_group_first_dev(group));
struct iommu_domain *dom;
lockdep_assert_held(&group->mutex);
/*
* Allow legacy drivers to specify the domain that will be the default
* domain. This should always be either an IDENTITY/BLOCKED/PLATFORM
* domain. Do not use in new drivers.
*/
if (ops->default_domain) {
if (req_type != ops->default_domain->type)
return ERR_PTR(-EINVAL);
return ops->default_domain;
}
if (req_type)
return __iommu_group_alloc_default_domain(group, req_type);
/* The driver gave no guidance on what type to use, try the default */
dom = __iommu_group_alloc_default_domain(group, iommu_def_domain_type);
if (!IS_ERR(dom))
return dom;
/* Otherwise IDENTITY and DMA_FQ defaults will try DMA */
if (iommu_def_domain_type == IOMMU_DOMAIN_DMA)
return ERR_PTR(-EINVAL);
dom = __iommu_group_alloc_default_domain(group, IOMMU_DOMAIN_DMA);
if (IS_ERR(dom))
return dom;
pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
iommu_def_domain_type, group->name);
return dom;
}
struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
{
return group->default_domain;
}
static int probe_iommu_group(struct device *dev, void *data)
{
struct list_head *group_list = data;
int ret;
mutex_lock(&iommu_probe_device_lock);
ret = __iommu_probe_device(dev, group_list);
mutex_unlock(&iommu_probe_device_lock);
if (ret == -ENODEV)
ret = 0;
return ret;
}
static int iommu_bus_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct device *dev = data;
if (action == BUS_NOTIFY_ADD_DEVICE) {
int ret;
ret = iommu_probe_device(dev); return (ret) ? NOTIFY_DONE : NOTIFY_OK; } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { iommu_release_device(dev);
return NOTIFY_OK;
}
return 0;
}
/*
* Combine the driver's chosen def_domain_type across all the devices in a
* group. Drivers must give a consistent result.
*/
static int iommu_get_def_domain_type(struct iommu_group *group,
struct device *dev, int cur_type)
{
const struct iommu_ops *ops = dev_iommu_ops(dev);
int type;
if (ops->default_domain) {
/*
* Drivers that declare a global static default_domain will
* always choose that.
*/
type = ops->default_domain->type;
} else {
if (ops->def_domain_type)
type = ops->def_domain_type(dev);
else
return cur_type;
}
if (!type || cur_type == type)
return cur_type;
if (!cur_type)
return type;
dev_err_ratelimited(
dev,
"IOMMU driver error, requesting conflicting def_domain_type, %s and %s, for devices in group %u.\n",
iommu_domain_type_str(cur_type), iommu_domain_type_str(type),
group->id);
/*
* Try to recover, drivers are allowed to force IDENITY or DMA, IDENTITY
* takes precedence.
*/
if (type == IOMMU_DOMAIN_IDENTITY)
return type;
return cur_type;
}
/*
* A target_type of 0 will select the best domain type. 0 can be returned in
* this case meaning the global default should be used.
*/
static int iommu_get_default_domain_type(struct iommu_group *group,
int target_type)
{
struct device *untrusted = NULL;
struct group_device *gdev;
int driver_type = 0;
lockdep_assert_held(&group->mutex);
/*
* ARM32 drivers supporting CONFIG_ARM_DMA_USE_IOMMU can declare an
* identity_domain and it will automatically become their default
* domain. Later on ARM_DMA_USE_IOMMU will install its UNMANAGED domain.
* Override the selection to IDENTITY.
*/
if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)) {
static_assert(!(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) &&
IS_ENABLED(CONFIG_IOMMU_DMA)));
driver_type = IOMMU_DOMAIN_IDENTITY;
}
for_each_group_device(group, gdev) {
driver_type = iommu_get_def_domain_type(group, gdev->dev,
driver_type);
if (dev_is_pci(gdev->dev) && to_pci_dev(gdev->dev)->untrusted) {
/*
* No ARM32 using systems will set untrusted, it cannot
* work.
*/
if (WARN_ON(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)))
return -1;
untrusted = gdev->dev;
}
}
/*
* If the common dma ops are not selected in kconfig then we cannot use
* IOMMU_DOMAIN_DMA at all. Force IDENTITY if nothing else has been
* selected.
*/
if (!IS_ENABLED(CONFIG_IOMMU_DMA)) {
if (WARN_ON(driver_type == IOMMU_DOMAIN_DMA))
return -1;
if (!driver_type)
driver_type = IOMMU_DOMAIN_IDENTITY;
}
if (untrusted) {
if (driver_type && driver_type != IOMMU_DOMAIN_DMA) {
dev_err_ratelimited(
untrusted,
"Device is not trusted, but driver is overriding group %u to %s, refusing to probe.\n",
group->id, iommu_domain_type_str(driver_type));
return -1;
}
driver_type = IOMMU_DOMAIN_DMA;
}
if (target_type) {
if (driver_type && target_type != driver_type)
return -1;
return target_type;
}
return driver_type;
}
static void iommu_group_do_probe_finalize(struct device *dev)
{
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (ops->probe_finalize)
ops->probe_finalize(dev);
}
int bus_iommu_probe(const struct bus_type *bus)
{
struct iommu_group *group, *next;
LIST_HEAD(group_list);
int ret;
ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
if (ret)
return ret;
list_for_each_entry_safe(group, next, &group_list, entry) {
struct group_device *gdev;
mutex_lock(&group->mutex);
/* Remove item from the list */
list_del_init(&group->entry);
/*
* We go to the trouble of deferred default domain creation so
* that the cross-group default domain type and the setup of the
* IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios.
*/
ret = iommu_setup_default_domain(group, 0);
if (ret) {
mutex_unlock(&group->mutex);
return ret;
}
mutex_unlock(&group->mutex);
/*
* FIXME: Mis-locked because the ops->probe_finalize() call-back
* of some IOMMU drivers calls arm_iommu_attach_device() which
* in-turn might call back into IOMMU core code, where it tries
* to take group->mutex, resulting in a deadlock.
*/
for_each_group_device(group, gdev)
iommu_group_do_probe_finalize(gdev->dev);
}
return 0;
}
/**
* iommu_present() - make platform-specific assumptions about an IOMMU
* @bus: bus to check
*
* Do not use this function. You want device_iommu_mapped() instead.
*
* Return: true if some IOMMU is present and aware of devices on the given bus;
* in general it may not be the only IOMMU, and it may not have anything to do
* with whatever device you are ultimately interested in.
*/
bool iommu_present(const struct bus_type *bus)
{
bool ret = false;
for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
if (iommu_buses[i] == bus) {
spin_lock(&iommu_device_lock);
ret = !list_empty(&iommu_device_list);
spin_unlock(&iommu_device_lock);
}
}
return ret;
}
EXPORT_SYMBOL_GPL(iommu_present);
/**
* device_iommu_capable() - check for a general IOMMU capability
* @dev: device to which the capability would be relevant, if available
* @cap: IOMMU capability
*
* Return: true if an IOMMU is present and supports the given capability
* for the given device, otherwise false.
*/
bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
{
const struct iommu_ops *ops;
if (!dev_has_iommu(dev))
return false;
ops = dev_iommu_ops(dev);
if (!ops->capable)
return false;
return ops->capable(dev, cap);
}
EXPORT_SYMBOL_GPL(device_iommu_capable);
/**
* iommu_group_has_isolated_msi() - Compute msi_device_has_isolated_msi()
* for a group
* @group: Group to query
*
* IOMMU groups should not have differing values of
* msi_device_has_isolated_msi() for devices in a group. However nothing
* directly prevents this, so ensure mistakes don't result in isolation failures
* by checking that all the devices are the same.
*/
bool iommu_group_has_isolated_msi(struct iommu_group *group)
{
struct group_device *group_dev;
bool ret = true;
mutex_lock(&group->mutex);
for_each_group_device(group, group_dev)
ret &= msi_device_has_isolated_msi(group_dev->dev);
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_group_has_isolated_msi);
/**
* iommu_set_fault_handler() - set a fault handler for an iommu domain
* @domain: iommu domain
* @handler: fault handler
* @token: user data, will be passed back to the fault handler
*
* This function should be used by IOMMU users which want to be notified
* whenever an IOMMU fault happens.
*
* The fault handler itself should return 0 on success, and an appropriate
* error code otherwise.
*/
void iommu_set_fault_handler(struct iommu_domain *domain,
iommu_fault_handler_t handler,
void *token)
{
BUG_ON(!domain);
domain->handler = handler;
domain->handler_token = token;
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
static struct iommu_domain *__iommu_domain_alloc(const struct iommu_ops *ops,
struct device *dev,
unsigned int type)
{
struct iommu_domain *domain;
unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS;
if (alloc_type == IOMMU_DOMAIN_IDENTITY && ops->identity_domain)
return ops->identity_domain;
else if (alloc_type == IOMMU_DOMAIN_BLOCKED && ops->blocked_domain)
return ops->blocked_domain;
else if (type & __IOMMU_DOMAIN_PAGING && ops->domain_alloc_paging)
domain = ops->domain_alloc_paging(dev);
else if (ops->domain_alloc)
domain = ops->domain_alloc(alloc_type);
else
return ERR_PTR(-EOPNOTSUPP);
/*
* Many domain_alloc ops now return ERR_PTR, make things easier for the
* driver by accepting ERR_PTR from all domain_alloc ops instead of
* having two rules.
*/
if (IS_ERR(domain))
return domain;
if (!domain)
return ERR_PTR(-ENOMEM);
domain->type = type;
domain->owner = ops;
/*
* If not already set, assume all sizes by default; the driver
* may override this later
*/
if (!domain->pgsize_bitmap)
domain->pgsize_bitmap = ops->pgsize_bitmap;
if (!domain->ops)
domain->ops = ops->default_domain_ops;
if (iommu_is_dma_domain(domain)) {
int rc;
rc = iommu_get_dma_cookie(domain);
if (rc) {
iommu_domain_free(domain);
return ERR_PTR(rc);
}
}
return domain;
}
static struct iommu_domain *
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type)
{
struct device *dev = iommu_group_first_dev(group);
return __iommu_domain_alloc(dev_iommu_ops(dev), dev, type);
}
static int __iommu_domain_alloc_dev(struct device *dev, void *data)
{
const struct iommu_ops **ops = data;
if (!dev_has_iommu(dev))
return 0;
if (WARN_ONCE(*ops && *ops != dev_iommu_ops(dev),
"Multiple IOMMU drivers present for bus %s, which the public IOMMU API can't fully support yet. You will still need to disable one or more for this to work, sorry!\n",
dev_bus_name(dev)))
return -EBUSY;
*ops = dev_iommu_ops(dev);
return 0;
}
struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
{
const struct iommu_ops *ops = NULL;
int err = bus_for_each_dev(bus, NULL, &ops, __iommu_domain_alloc_dev);
struct iommu_domain *domain;
if (err || !ops)
return NULL;
domain = __iommu_domain_alloc(ops, NULL, IOMMU_DOMAIN_UNMANAGED);
if (IS_ERR(domain))
return NULL;
return domain;
}
EXPORT_SYMBOL_GPL(iommu_domain_alloc);
void iommu_domain_free(struct iommu_domain *domain)
{
if (domain->type == IOMMU_DOMAIN_SVA)
mmdrop(domain->mm);
iommu_put_dma_cookie(domain);
if (domain->ops->free)
domain->ops->free(domain);
}
EXPORT_SYMBOL_GPL(iommu_domain_free);
/*
* Put the group's domain back to the appropriate core-owned domain - either the
* standard kernel-mode DMA configuration or an all-DMA-blocked domain.
*/
static void __iommu_group_set_core_domain(struct iommu_group *group)
{
struct iommu_domain *new_domain;
if (group->owner)
new_domain = group->blocking_domain;
else
new_domain = group->default_domain;
__iommu_group_set_domain_nofail(group, new_domain);
}
static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
int ret;
if (unlikely(domain->ops->attach_dev == NULL))
return -ENODEV;
ret = domain->ops->attach_dev(domain, dev);
if (ret)
return ret;
dev->iommu->attach_deferred = 0;
trace_attach_device_to_domain(dev);
return 0;
}
/**
* iommu_attach_device - Attach an IOMMU domain to a device
* @domain: IOMMU domain to attach
* @dev: Device that will be attached
*
* Returns 0 on success and error code on failure
*
* Note that EINVAL can be treated as a soft failure, indicating
* that certain configuration of the domain is incompatible with
* the device. In this case attaching a different domain to the
* device may succeed.
*/
int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
int ret;
if (!group)
return -ENODEV;
/*
* Lock the group to make sure the device-count doesn't
* change while we are attaching
*/
mutex_lock(&group->mutex);
ret = -EINVAL;
if (list_count_nodes(&group->devices) != 1)
goto out_unlock;
ret = __iommu_attach_group(domain, group);
out_unlock:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_device);
int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
{
if (dev->iommu && dev->iommu->attach_deferred)
return __iommu_attach_device(domain, dev);
return 0;
}
void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
if (!group)
return;
mutex_lock(&group->mutex);
if (WARN_ON(domain != group->domain) ||
WARN_ON(list_count_nodes(&group->devices) != 1))
goto out_unlock;
__iommu_group_set_core_domain(group);
out_unlock:
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_detach_device);
struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
if (!group)
return NULL;
return group->domain;
}
EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
/*
* For IOMMU_DOMAIN_DMA implementations which already provide their own
* guarantees that the group and its default domain are valid and correct.
*/
struct iommu_domain *iommu_get_dma_domain(struct device *dev)
{
return dev->iommu_group->default_domain;
}
static int __iommu_attach_group(struct iommu_domain *domain,
struct iommu_group *group)
{
struct device *dev;
if (group->domain && group->domain != group->default_domain &&
group->domain != group->blocking_domain)
return -EBUSY;
dev = iommu_group_first_dev(group);
if (!dev_has_iommu(dev) || dev_iommu_ops(dev) != domain->owner)
return -EINVAL;
return __iommu_group_set_domain(group, domain);
}
/**
* iommu_attach_group - Attach an IOMMU domain to an IOMMU group
* @domain: IOMMU domain to attach
* @group: IOMMU group that will be attached
*
* Returns 0 on success and error code on failure
*
* Note that EINVAL can be treated as a soft failure, indicating
* that certain configuration of the domain is incompatible with
* the group. In this case attaching a different domain to the
* group may succeed.
*/
int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
{
int ret;
mutex_lock(&group->mutex);
ret = __iommu_attach_group(domain, group);
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_group);
/**
* iommu_group_replace_domain - replace the domain that a group is attached to
* @new_domain: new IOMMU domain to replace with
* @group: IOMMU group that will be attached to the new domain
*
* This API allows the group to switch domains without being forced to go to
* the blocking domain in-between.
*
* If the currently attached domain is a core domain (e.g. a default_domain),
* it will act just like the iommu_attach_group().
*/
int iommu_group_replace_domain(struct iommu_group *group,
struct iommu_domain *new_domain)
{
int ret;
if (!new_domain)
return -EINVAL;
mutex_lock(&group->mutex);
ret = __iommu_group_set_domain(group, new_domain);
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_NS_GPL(iommu_group_replace_domain, IOMMUFD_INTERNAL);
static int __iommu_device_set_domain(struct iommu_group *group,
struct device *dev,
struct iommu_domain *new_domain,
unsigned int flags)
{
int ret;
/*
* If the device requires IOMMU_RESV_DIRECT then we cannot allow
* the blocking domain to be attached as it does not contain the
* required 1:1 mapping. This test effectively excludes the device
* being used with iommu_group_claim_dma_owner() which will block
* vfio and iommufd as well.
*/
if (dev->iommu->require_direct &&
(new_domain->type == IOMMU_DOMAIN_BLOCKED ||
new_domain == group->blocking_domain)) {
dev_warn(dev,
"Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n");
return -EINVAL;
}
if (dev->iommu->attach_deferred) {
if (new_domain == group->default_domain)
return 0;
dev->iommu->attach_deferred = 0;
}
ret = __iommu_attach_device(new_domain, dev);
if (ret) {
/*
* If we have a blocking domain then try to attach that in hopes
* of avoiding a UAF. Modern drivers should implement blocking
* domains as global statics that cannot fail.
*/
if ((flags & IOMMU_SET_DOMAIN_MUST_SUCCEED) &&
group->blocking_domain &&
group->blocking_domain != new_domain)
__iommu_attach_device(group->blocking_domain, dev);
return ret;
}
return 0;
}
/*
* If 0 is returned the group's domain is new_domain. If an error is returned
* then the group's domain will be set back to the existing domain unless
* IOMMU_SET_DOMAIN_MUST_SUCCEED, otherwise an error is returned and the group's
* domains is left inconsistent. This is a driver bug to fail attach with a
* previously good domain. We try to avoid a kernel UAF because of this.
*
* IOMMU groups are really the natural working unit of the IOMMU, but the IOMMU
* API works on domains and devices. Bridge that gap by iterating over the
* devices in a group. Ideally we'd have a single device which represents the
* requestor ID of the group, but we also allow IOMMU drivers to create policy
* defined minimum sets, where the physical hardware may be able to distiguish
* members, but we wish to group them at a higher level (ex. untrusted
* multi-function PCI devices). Thus we attach each device.
*/
static int __iommu_group_set_domain_internal(struct iommu_group *group,
struct iommu_domain *new_domain,
unsigned int flags)
{
struct group_device *last_gdev;
struct group_device *gdev;
int result;
int ret;
lockdep_assert_held(&group->mutex);
if (group->domain == new_domain)
return 0;
if (WARN_ON(!new_domain))
return -EINVAL;
/*
* Changing the domain is done by calling attach_dev() on the new
* domain. This switch does not have to be atomic and DMA can be
* discarded during the transition. DMA must only be able to access
* either new_domain or group->domain, never something else.
*/
result = 0;
for_each_group_device(group, gdev) {
ret = __iommu_device_set_domain(group, gdev->dev, new_domain,
flags);
if (ret) {
result = ret;
/*
* Keep trying the other devices in the group. If a
* driver fails attach to an otherwise good domain, and
* does not support blocking domains, it should at least
* drop its reference on the current domain so we don't
* UAF.
*/
if (flags & IOMMU_SET_DOMAIN_MUST_SUCCEED)
continue;
goto err_revert;
}
}
group->domain = new_domain;
return result;
err_revert:
/*
* This is called in error unwind paths. A well behaved driver should
* always allow us to attach to a domain that was already attached.
*/
last_gdev = gdev;
for_each_group_device(group, gdev) {
/*
* A NULL domain can happen only for first probe, in which case
* we leave group->domain as NULL and let release clean
* everything up.
*/
if (group->domain)
WARN_ON(__iommu_device_set_domain(
group, gdev->dev, group->domain,
IOMMU_SET_DOMAIN_MUST_SUCCEED));
if (gdev == last_gdev)
break;
}
return ret;
}
void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
{
mutex_lock(&group->mutex);
__iommu_group_set_core_domain(group);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_detach_group);
phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
{
if (domain->type == IOMMU_DOMAIN_IDENTITY)
return iova;
if (domain->type == IOMMU_DOMAIN_BLOCKED)
return 0;
return domain->ops->iova_to_phys(domain, iova);
}
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, size_t *count)
{
unsigned int pgsize_idx, pgsize_idx_next;
unsigned long pgsizes;
size_t offset, pgsize, pgsize_next;
unsigned long addr_merge = paddr | iova;
/* Page sizes supported by the hardware and small enough for @size */
pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
/* Constrain the page sizes further based on the maximum alignment */
if (likely(addr_merge))
pgsizes &= GENMASK(__ffs(addr_merge), 0);
/* Make sure we have at least one suitable page size */
BUG_ON(!pgsizes);
/* Pick the biggest page size remaining */
pgsize_idx = __fls(pgsizes);
pgsize = BIT(pgsize_idx);
if (!count)
return pgsize;
/* Find the next biggest support page size, if it exists */
pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
if (!pgsizes)
goto out_set_count;
pgsize_idx_next = __ffs(pgsizes);
pgsize_next = BIT(pgsize_idx_next);
/*
* There's no point trying a bigger page size unless the virtual
* and physical addresses are similarly offset within the larger page.
*/
if ((iova ^ paddr) & (pgsize_next - 1))
goto out_set_count;
/* Calculate the offset to the next page size alignment boundary */
offset = pgsize_next - (addr_merge & (pgsize_next - 1));
/*
* If size is big enough to accommodate the larger page, reduce
* the number of smaller pages.
*/
if (offset + pgsize_next <= size)
size = offset;
out_set_count:
*count = size >> pgsize_idx;
return pgsize;
}
static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
const struct iommu_domain_ops *ops = domain->ops;
unsigned long orig_iova = iova;
unsigned int min_pagesz;
size_t orig_size = size;
phys_addr_t orig_paddr = paddr;
int ret = 0;
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
return -EINVAL;
if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL))
return -ENODEV;
/* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
/*
* both the virtual address and the physical one, as well as
* the size of the mapping, must be aligned (at least) to the
* size of the smallest page supported by the hardware
*/
if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
iova, &paddr, size, min_pagesz);
return -EINVAL;
}
pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
while (size) {
size_t pgsize, count, mapped = 0;
pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
iova, &paddr, pgsize, count);
ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
gfp, &mapped);
/*
* Some pages may have been mapped, even if an error occurred,
* so we should account for those so they can be unmapped.
*/
size -= mapped;
if (ret)
break;
iova += mapped;
paddr += mapped;
}
/* unroll mapping in case something went wrong */
if (ret)
iommu_unmap(domain, orig_iova, orig_size - size);
else
trace_map(orig_iova, orig_paddr, orig_size);
return ret;
}
int iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
const struct iommu_domain_ops *ops = domain->ops;
int ret;
might_sleep_if(gfpflags_allow_blocking(gfp));
/* Discourage passing strange GFP flags */
if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
__GFP_HIGHMEM)))
return -EINVAL;
ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
if (ret == 0 && ops->iotlb_sync_map) {
ret = ops->iotlb_sync_map(domain, iova, size);
if (ret)
goto out_err;
}
return ret;
out_err:
/* undo mappings already done */
iommu_unmap(domain, iova, size);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_map);
static size_t __iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
struct iommu_iotlb_gather *iotlb_gather)
{
const struct iommu_domain_ops *ops = domain->ops;
size_t unmapped_page, unmapped = 0;
unsigned long orig_iova = iova;
unsigned int min_pagesz;
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
return 0;
if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL))
return 0;
/* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
/*
* The virtual address, as well as the size of the mapping, must be
* aligned (at least) to the size of the smallest page supported
* by the hardware
*/
if (!IS_ALIGNED(iova | size, min_pagesz)) {
pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
iova, size, min_pagesz);
return 0;
}
pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
/*
* Keep iterating until we either unmap 'size' bytes (or more)
* or we hit an area that isn't mapped.
*/
while (unmapped < size) {
size_t pgsize, count;
pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count);
unmapped_page = ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather);
if (!unmapped_page)
break;
pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
iova, unmapped_page);
iova += unmapped_page;
unmapped += unmapped_page;
}
trace_unmap(orig_iova, size, unmapped);
return unmapped;
}
size_t iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct iommu_iotlb_gather iotlb_gather;
size_t ret;
iommu_iotlb_gather_init(&iotlb_gather);
ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
iommu_iotlb_sync(domain, &iotlb_gather);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_unmap);
size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, size_t size,
struct iommu_iotlb_gather *iotlb_gather)
{
return __iommu_unmap(domain, iova, size, iotlb_gather);
}
EXPORT_SYMBOL_GPL(iommu_unmap_fast);
ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
struct scatterlist *sg, unsigned int nents, int prot,
gfp_t gfp)
{
const struct iommu_domain_ops *ops = domain->ops;
size_t len = 0, mapped = 0;
phys_addr_t start;
unsigned int i = 0;
int ret;
might_sleep_if(gfpflags_allow_blocking(gfp));
/* Discourage passing strange GFP flags */
if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
__GFP_HIGHMEM)))
return -EINVAL;
while (i <= nents) {
phys_addr_t s_phys = sg_phys(sg);
if (len && s_phys != start + len) {
ret = __iommu_map(domain, iova + mapped, start,
len, prot, gfp);
if (ret)
goto out_err;
mapped += len;
len = 0;
}
if (sg_dma_is_bus_address(sg))
goto next;
if (len) {
len += sg->length;
} else {
len = sg->length;
start = s_phys;
}
next:
if (++i < nents)
sg = sg_next(sg);
}
if (ops->iotlb_sync_map) {
ret = ops->iotlb_sync_map(domain, iova, mapped);
if (ret)
goto out_err;
}
return mapped;
out_err:
/* undo mappings already done */
iommu_unmap(domain, iova, mapped);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_map_sg);
/**
* report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
* @domain: the iommu domain where the fault has happened
* @dev: the device where the fault has happened
* @iova: the faulting address
* @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
*
* This function should be called by the low-level IOMMU implementations
* whenever IOMMU faults happen, to allow high-level users, that are
* interested in such events, to know about them.
*
* This event may be useful for several possible use cases:
* - mere logging of the event
* - dynamic TLB/PTE loading
* - if restarting of the faulting device is required
*
* Returns 0 on success and an appropriate error code otherwise (if dynamic
* PTE/TLB loading will one day be supported, implementations will be able
* to tell whether it succeeded or not according to this return value).
*
* Specifically, -ENOSYS is returned if a fault handler isn't installed
* (though fault handlers can also return -ENOSYS, in case they want to
* elicit the default behavior of the IOMMU drivers).
*/
int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
unsigned long iova, int flags)
{
int ret = -ENOSYS;
/*
* if upper layers showed interest and installed a fault handler,
* invoke it.
*/
if (domain->handler)
ret = domain->handler(domain, dev, iova, flags,
domain->handler_token);
trace_io_page_fault(dev, iova, flags);
return ret;
}
EXPORT_SYMBOL_GPL(report_iommu_fault);
static int __init iommu_init(void)
{
iommu_group_kset = kset_create_and_add("iommu_groups",
NULL, kernel_kobj);
BUG_ON(!iommu_group_kset);
iommu_debugfs_setup();
return 0;
}
core_initcall(iommu_init);
int iommu_enable_nesting(struct iommu_domain *domain)
{
if (domain->type != IOMMU_DOMAIN_UNMANAGED)
return -EINVAL;
if (!domain->ops->enable_nesting)
return -EINVAL;
return domain->ops->enable_nesting(domain);
}
EXPORT_SYMBOL_GPL(iommu_enable_nesting);
int iommu_set_pgtable_quirks(struct iommu_domain *domain,
unsigned long quirk)
{
if (domain->type != IOMMU_DOMAIN_UNMANAGED)
return -EINVAL;
if (!domain->ops->set_pgtable_quirks)
return -EINVAL;
return domain->ops->set_pgtable_quirks(domain, quirk);
}
EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
/**
* iommu_get_resv_regions - get reserved regions
* @dev: device for which to get reserved regions
* @list: reserved region list for device
*
* This returns a list of reserved IOVA regions specific to this device.
* A domain user should not map IOVA in these ranges.
*/
void iommu_get_resv_regions(struct device *dev, struct list_head *list)
{
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (ops->get_resv_regions)
ops->get_resv_regions(dev, list);
}
EXPORT_SYMBOL_GPL(iommu_get_resv_regions);
/**
* iommu_put_resv_regions - release reserved regions
* @dev: device for which to free reserved regions
* @list: reserved region list for device
*
* This releases a reserved region list acquired by iommu_get_resv_regions().
*/
void iommu_put_resv_regions(struct device *dev, struct list_head *list)
{
struct iommu_resv_region *entry, *next;
list_for_each_entry_safe(entry, next, list, list) {
if (entry->free)
entry->free(dev, entry);
else
kfree(entry);
}
}
EXPORT_SYMBOL(iommu_put_resv_regions);
struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
size_t length, int prot,
enum iommu_resv_type type,
gfp_t gfp)
{
struct iommu_resv_region *region;
region = kzalloc(sizeof(*region), gfp);
if (!region)
return NULL;
INIT_LIST_HEAD(®ion->list);
region->start = start;
region->length = length;
region->prot = prot;
region->type = type;
return region;
}
EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
void iommu_set_default_passthrough(bool cmd_line)
{
if (cmd_line)
iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
}
void iommu_set_default_translated(bool cmd_line)
{
if (cmd_line)
iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
iommu_def_domain_type = IOMMU_DOMAIN_DMA;
}
bool iommu_default_passthrough(void)
{
return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
}
EXPORT_SYMBOL_GPL(iommu_default_passthrough);
const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode)
{
const struct iommu_ops *ops = NULL;
struct iommu_device *iommu;
spin_lock(&iommu_device_lock);
list_for_each_entry(iommu, &iommu_device_list, list)
if (iommu->fwnode == fwnode) {
ops = iommu->ops;
break;
}
spin_unlock(&iommu_device_lock);
return ops;
}
int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
const struct iommu_ops *ops)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
if (fwspec)
return ops == fwspec->ops ? 0 : -EINVAL;
if (!dev_iommu_get(dev))
return -ENOMEM;
/* Preallocate for the overwhelmingly common case of 1 ID */
fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
if (!fwspec)
return -ENOMEM;
of_node_get(to_of_node(iommu_fwnode));
fwspec->iommu_fwnode = iommu_fwnode;
fwspec->ops = ops;
dev_iommu_fwspec_set(dev, fwspec);
return 0;
}
EXPORT_SYMBOL_GPL(iommu_fwspec_init);
void iommu_fwspec_free(struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
if (fwspec) {
fwnode_handle_put(fwspec->iommu_fwnode);
kfree(fwspec);
dev_iommu_fwspec_set(dev, NULL);
}
}
EXPORT_SYMBOL_GPL(iommu_fwspec_free);
int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
int i, new_num;
if (!fwspec)
return -EINVAL;
new_num = fwspec->num_ids + num_ids;
if (new_num > 1) {
fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
GFP_KERNEL);
if (!fwspec)
return -ENOMEM;
dev_iommu_fwspec_set(dev, fwspec);
}
for (i = 0; i < num_ids; i++)
fwspec->ids[fwspec->num_ids + i] = ids[i];
fwspec->num_ids = new_num;
return 0;
}
EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
/*
* Per device IOMMU features.
*/
int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
{
if (dev_has_iommu(dev)) {
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (ops->dev_enable_feat)
return ops->dev_enable_feat(dev, feat);
}
return -ENODEV;
}
EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
/*
* The device drivers should do the necessary cleanups before calling this.
*/
int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
{
if (dev_has_iommu(dev)) {
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (ops->dev_disable_feat)
return ops->dev_disable_feat(dev, feat);
}
return -EBUSY;
}
EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
/**
* iommu_setup_default_domain - Set the default_domain for the group
* @group: Group to change
* @target_type: Domain type to set as the default_domain
*
* Allocate a default domain and set it as the current domain on the group. If
* the group already has a default domain it will be changed to the target_type.
* When target_type is 0 the default domain is selected based on driver and
* system preferences.
*/
static int iommu_setup_default_domain(struct iommu_group *group,
int target_type)
{
struct iommu_domain *old_dom = group->default_domain;
struct group_device *gdev;
struct iommu_domain *dom;
bool direct_failed;
int req_type;
int ret;
lockdep_assert_held(&group->mutex);
req_type = iommu_get_default_domain_type(group, target_type);
if (req_type < 0)
return -EINVAL;
dom = iommu_group_alloc_default_domain(group, req_type);
if (IS_ERR(dom))
return PTR_ERR(dom);
if (group->default_domain == dom)
return 0;
/*
* IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
* mapped before their device is attached, in order to guarantee
* continuity with any FW activity
*/
direct_failed = false;
for_each_group_device(group, gdev) {
if (iommu_create_device_direct_mappings(dom, gdev->dev)) {
direct_failed = true;
dev_warn_once(
gdev->dev->iommu->iommu_dev->dev,
"IOMMU driver was not able to establish FW requested direct mapping.");
}
}
/* We must set default_domain early for __iommu_device_set_domain */
group->default_domain = dom;
if (!group->domain) {
/*
* Drivers are not allowed to fail the first domain attach.
* The only way to recover from this is to fail attaching the
* iommu driver and call ops->release_device. Put the domain
* in group->default_domain so it is freed after.
*/
ret = __iommu_group_set_domain_internal(
group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);
if (WARN_ON(ret))
goto out_free_old;
} else {
ret = __iommu_group_set_domain(group, dom);
if (ret)
goto err_restore_def_domain;
}
/*
* Drivers are supposed to allow mappings to be installed in a domain
* before device attachment, but some don't. Hack around this defect by
* trying again after attaching. If this happens it means the device
* will not continuously have the IOMMU_RESV_DIRECT map.
*/
if (direct_failed) {
for_each_group_device(group, gdev) {
ret = iommu_create_device_direct_mappings(dom, gdev->dev);
if (ret)
goto err_restore_domain;
}
}
out_free_old:
if (old_dom)
iommu_domain_free(old_dom);
return ret;
err_restore_domain:
if (old_dom)
__iommu_group_set_domain_internal(
group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);
err_restore_def_domain:
if (old_dom) {
iommu_domain_free(dom);
group->default_domain = old_dom;
}
return ret;
}
/*
* Changing the default domain through sysfs requires the users to unbind the
* drivers from the devices in the iommu group, except for a DMA -> DMA-FQ
* transition. Return failure if this isn't met.
*
* We need to consider the race between this and the device release path.
* group->mutex is used here to guarantee that the device release path
* will not be entered at the same time.
*/
static ssize_t iommu_group_store_type(struct iommu_group *group,
const char *buf, size_t count)
{
struct group_device *gdev;
int ret, req_type;
if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
return -EACCES;
if (WARN_ON(!group) || !group->default_domain)
return -EINVAL;
if (sysfs_streq(buf, "identity"))
req_type = IOMMU_DOMAIN_IDENTITY;
else if (sysfs_streq(buf, "DMA"))
req_type = IOMMU_DOMAIN_DMA;
else if (sysfs_streq(buf, "DMA-FQ"))
req_type = IOMMU_DOMAIN_DMA_FQ;
else if (sysfs_streq(buf, "auto"))
req_type = 0;
else
return -EINVAL;
mutex_lock(&group->mutex);
/* We can bring up a flush queue without tearing down the domain. */
if (req_type == IOMMU_DOMAIN_DMA_FQ &&
group->default_domain->type == IOMMU_DOMAIN_DMA) {
ret = iommu_dma_init_fq(group->default_domain);
if (ret)
goto out_unlock;
group->default_domain->type = IOMMU_DOMAIN_DMA_FQ;
ret = count;
goto out_unlock;
}
/* Otherwise, ensure that device exists and no driver is bound. */
if (list_empty(&group->devices) || group->owner_cnt) {
ret = -EPERM;
goto out_unlock;
}
ret = iommu_setup_default_domain(group, req_type);
if (ret)
goto out_unlock;
/*
* Release the mutex here because ops->probe_finalize() call-back of
* some vendor IOMMU drivers calls arm_iommu_attach_device() which
* in-turn might call back into IOMMU core code, where it tries to take
* group->mutex, resulting in a deadlock.
*/
mutex_unlock(&group->mutex);
/* Make sure dma_ops is appropriatley set */
for_each_group_device(group, gdev)
iommu_group_do_probe_finalize(gdev->dev);
return count;
out_unlock:
mutex_unlock(&group->mutex);
return ret ?: count;
}
/**
* iommu_device_use_default_domain() - Device driver wants to handle device
* DMA through the kernel DMA API.
* @dev: The device.
*
* The device driver about to bind @dev wants to do DMA through the kernel
* DMA API. Return 0 if it is allowed, otherwise an error.
*/
int iommu_device_use_default_domain(struct device *dev)
{
/* Caller is the driver core during the pre-probe path */
struct iommu_group *group = dev->iommu_group;
int ret = 0;
if (!group)
return 0;
mutex_lock(&group->mutex);
if (group->owner_cnt) {
if (group->domain != group->default_domain || group->owner ||
!xa_empty(&group->pasid_array)) {
ret = -EBUSY;
goto unlock_out;
}
}
group->owner_cnt++;
unlock_out:
mutex_unlock(&group->mutex);
return ret;
}
/**
* iommu_device_unuse_default_domain() - Device driver stops handling device
* DMA through the kernel DMA API.
* @dev: The device.
*
* The device driver doesn't want to do DMA through kernel DMA API anymore.
* It must be called after iommu_device_use_default_domain().
*/
void iommu_device_unuse_default_domain(struct device *dev)
{
/* Caller is the driver core during the post-probe path */
struct iommu_group *group = dev->iommu_group;
if (!group)
return;
mutex_lock(&group->mutex); if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array))) group->owner_cnt--; mutex_unlock(&group->mutex);
}
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
{
struct iommu_domain *domain;
if (group->blocking_domain)
return 0;
domain = __iommu_group_domain_alloc(group, IOMMU_DOMAIN_BLOCKED);
if (IS_ERR(domain)) {
/*
* For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
* create an empty domain instead.
*/
domain = __iommu_group_domain_alloc(group,
IOMMU_DOMAIN_UNMANAGED);
if (IS_ERR(domain))
return PTR_ERR(domain);
}
group->blocking_domain = domain;
return 0;
}
static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)
{
int ret;
if ((group->domain && group->domain != group->default_domain) ||
!xa_empty(&group->pasid_array))
return -EBUSY;
ret = __iommu_group_alloc_blocking_domain(group);
if (ret)
return ret;
ret = __iommu_group_set_domain(group, group->blocking_domain);
if (ret)
return ret;
group->owner = owner;
group->owner_cnt++;
return 0;
}
/**
* iommu_group_claim_dma_owner() - Set DMA ownership of a group
* @group: The group.
* @owner: Caller specified pointer. Used for exclusive ownership.
*
* This is to support backward compatibility for vfio which manages the dma
* ownership in iommu_group level. New invocations on this interface should be
* prohibited. Only a single owner may exist for a group.
*/
int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
{
int ret = 0;
if (WARN_ON(!owner))
return -EINVAL;
mutex_lock(&group->mutex);
if (group->owner_cnt) {
ret = -EPERM;
goto unlock_out;
}
ret = __iommu_take_dma_ownership(group, owner);
unlock_out:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
/**
* iommu_device_claim_dma_owner() - Set DMA ownership of a device
* @dev: The device.
* @owner: Caller specified pointer. Used for exclusive ownership.
*
* Claim the DMA ownership of a device. Multiple devices in the same group may
* concurrently claim ownership if they present the same owner value. Returns 0
* on success and error code on failure
*/
int iommu_device_claim_dma_owner(struct device *dev, void *owner)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
int ret = 0;
if (WARN_ON(!owner))
return -EINVAL;
if (!group)
return -ENODEV;
mutex_lock(&group->mutex);
if (group->owner_cnt) {
if (group->owner != owner) {
ret = -EPERM;
goto unlock_out;
}
group->owner_cnt++;
goto unlock_out;
}
ret = __iommu_take_dma_ownership(group, owner);
unlock_out:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
static void __iommu_release_dma_ownership(struct iommu_group *group)
{
if (WARN_ON(!group->owner_cnt || !group->owner ||
!xa_empty(&group->pasid_array)))
return;
group->owner_cnt = 0;
group->owner = NULL;
__iommu_group_set_domain_nofail(group, group->default_domain);
}
/**
* iommu_group_release_dma_owner() - Release DMA ownership of a group
* @group: The group
*
* Release the DMA ownership claimed by iommu_group_claim_dma_owner().
*/
void iommu_group_release_dma_owner(struct iommu_group *group)
{
mutex_lock(&group->mutex);
__iommu_release_dma_ownership(group);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
/**
* iommu_device_release_dma_owner() - Release DMA ownership of a device
* @dev: The device.
*
* Release the DMA ownership claimed by iommu_device_claim_dma_owner().
*/
void iommu_device_release_dma_owner(struct device *dev)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
mutex_lock(&group->mutex);
if (group->owner_cnt > 1)
group->owner_cnt--;
else
__iommu_release_dma_ownership(group);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
/**
* iommu_group_dma_owner_claimed() - Query group dma ownership status
* @group: The group.
*
* This provides status query on a given group. It is racy and only for
* non-binding status reporting.
*/
bool iommu_group_dma_owner_claimed(struct iommu_group *group)
{
unsigned int user;
mutex_lock(&group->mutex);
user = group->owner_cnt;
mutex_unlock(&group->mutex);
return user;
}
EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
static int __iommu_set_group_pasid(struct iommu_domain *domain,
struct iommu_group *group, ioasid_t pasid)
{
struct group_device *device;
int ret = 0;
for_each_group_device(group, device) {
ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
if (ret)
break;
}
return ret;
}
static void __iommu_remove_group_pasid(struct iommu_group *group,
ioasid_t pasid)
{
struct group_device *device;
const struct iommu_ops *ops;
for_each_group_device(group, device) {
ops = dev_iommu_ops(device->dev);
ops->remove_dev_pasid(device->dev, pasid);
}
}
/*
* iommu_attach_device_pasid() - Attach a domain to pasid of device
* @domain: the iommu domain.
* @dev: the attached device.
* @pasid: the pasid of the device.
*
* Return: 0 on success, or an error.
*/
int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
struct group_device *device;
void *curr;
int ret;
if (!domain->ops->set_dev_pasid)
return -EOPNOTSUPP;
if (!group)
return -ENODEV;
if (!dev_has_iommu(dev) || dev_iommu_ops(dev) != domain->owner ||
pasid == IOMMU_NO_PASID)
return -EINVAL;
mutex_lock(&group->mutex);
for_each_group_device(group, device) {
if (pasid >= device->dev->iommu->max_pasids) {
ret = -EINVAL;
goto out_unlock;
}
}
curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
if (curr) {
ret = xa_err(curr) ? : -EBUSY;
goto out_unlock;
}
ret = __iommu_set_group_pasid(domain, group, pasid);
if (ret) {
__iommu_remove_group_pasid(group, pasid);
xa_erase(&group->pasid_array, pasid);
}
out_unlock:
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
/*
* iommu_detach_device_pasid() - Detach the domain from pasid of device
* @domain: the iommu domain.
* @dev: the attached device.
* @pasid: the pasid of the device.
*
* The @domain must have been attached to @pasid of the @dev with
* iommu_attach_device_pasid().
*/
void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
ioasid_t pasid)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
mutex_lock(&group->mutex);
__iommu_remove_group_pasid(group, pasid);
WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
/*
* iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
* @dev: the queried device
* @pasid: the pasid of the device
* @type: matched domain type, 0 for any match
*
* This is a variant of iommu_get_domain_for_dev(). It returns the existing
* domain attached to pasid of a device. Callers must hold a lock around this
* function, and both iommu_attach/detach_dev_pasid() whenever a domain of
* type is being manipulated. This API does not internally resolve races with
* attach/detach.
*
* Return: attached domain on success, NULL otherwise.
*/
struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
ioasid_t pasid,
unsigned int type)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
struct iommu_domain *domain;
if (!group)
return NULL;
xa_lock(&group->pasid_array);
domain = xa_load(&group->pasid_array, pasid);
if (type && domain && domain->type != type)
domain = ERR_PTR(-EBUSY);
xa_unlock(&group->pasid_array);
return domain;
}
EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
ioasid_t iommu_alloc_global_pasid(struct device *dev)
{
int ret;
/* max_pasids == 0 means that the device does not support PASID */
if (!dev->iommu->max_pasids)
return IOMMU_PASID_INVALID;
/*
* max_pasids is set up by vendor driver based on number of PASID bits
* supported but the IDA allocation is inclusive.
*/
ret = ida_alloc_range(&iommu_global_pasid_ida, IOMMU_FIRST_GLOBAL_PASID,
dev->iommu->max_pasids - 1, GFP_KERNEL);
return ret < 0 ? IOMMU_PASID_INVALID : ret;
}
EXPORT_SYMBOL_GPL(iommu_alloc_global_pasid);
void iommu_free_global_pasid(ioasid_t pasid)
{
if (WARN_ON(pasid == IOMMU_PASID_INVALID))
return;
ida_free(&iommu_global_pasid_ida, pasid);
}
EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
/* CPU control.
* (C) 2001, 2002, 2003, 2004 Rusty Russell
*
* This code is licenced under the GPL.
*/
#include <linux/sched/mm.h>
#include <linux/proc_fs.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/notifier.h>
#include <linux/sched/signal.h>
#include <linux/sched/hotplug.h>
#include <linux/sched/isolation.h>
#include <linux/sched/task.h>
#include <linux/sched/smt.h>
#include <linux/unistd.h>
#include <linux/cpu.h>
#include <linux/oom.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/bug.h>
#include <linux/kthread.h>
#include <linux/stop_machine.h>
#include <linux/mutex.h>
#include <linux/gfp.h>
#include <linux/suspend.h>
#include <linux/lockdep.h>
#include <linux/tick.h>
#include <linux/irq.h>
#include <linux/nmi.h>
#include <linux/smpboot.h>
#include <linux/relay.h>
#include <linux/slab.h>
#include <linux/scs.h>
#include <linux/percpu-rwsem.h>
#include <linux/cpuset.h>
#include <linux/random.h>
#include <linux/cc_platform.h>
#include <trace/events/power.h>
#define CREATE_TRACE_POINTS
#include <trace/events/cpuhp.h>
#include "smpboot.h"
/**
* struct cpuhp_cpu_state - Per cpu hotplug state storage
* @state: The current cpu state
* @target: The target state
* @fail: Current CPU hotplug callback state
* @thread: Pointer to the hotplug thread
* @should_run: Thread should execute
* @rollback: Perform a rollback
* @single: Single callback invocation
* @bringup: Single callback bringup or teardown selector
* @node: Remote CPU node; for multi-instance, do a
* single entry callback for install/remove
* @last: For multi-instance rollback, remember how far we got
* @cb_state: The state for a single callback (install/uninstall)
* @result: Result of the operation
* @ap_sync_state: State for AP synchronization
* @done_up: Signal completion to the issuer of the task for cpu-up
* @done_down: Signal completion to the issuer of the task for cpu-down
*/
struct cpuhp_cpu_state {
enum cpuhp_state state;
enum cpuhp_state target;
enum cpuhp_state fail;
#ifdef CONFIG_SMP
struct task_struct *thread;
bool should_run;
bool rollback;
bool single;
bool bringup;
struct hlist_node *node;
struct hlist_node *last;
enum cpuhp_state cb_state;
int result;
atomic_t ap_sync_state;
struct completion done_up;
struct completion done_down;
#endif
};
static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
.fail = CPUHP_INVALID,
};
#ifdef CONFIG_SMP
cpumask_t cpus_booted_once_mask;
#endif
#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
static struct lockdep_map cpuhp_state_up_map =
STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
static struct lockdep_map cpuhp_state_down_map =
STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
static inline void cpuhp_lock_acquire(bool bringup)
{
lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
}
static inline void cpuhp_lock_release(bool bringup)
{
lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
}
#else
static inline void cpuhp_lock_acquire(bool bringup) { }
static inline void cpuhp_lock_release(bool bringup) { }
#endif
/**
* struct cpuhp_step - Hotplug state machine step
* @name: Name of the step
* @startup: Startup function of the step
* @teardown: Teardown function of the step
* @cant_stop: Bringup/teardown can't be stopped at this step
* @multi_instance: State has multiple instances which get added afterwards
*/
struct cpuhp_step {
const char *name;
union {
int (*single)(unsigned int cpu);
int (*multi)(unsigned int cpu,
struct hlist_node *node);
} startup;
union {
int (*single)(unsigned int cpu);
int (*multi)(unsigned int cpu,
struct hlist_node *node);
} teardown;
/* private: */
struct hlist_head list;
/* public: */
bool cant_stop;
bool multi_instance;
};
static DEFINE_MUTEX(cpuhp_state_mutex);
static struct cpuhp_step cpuhp_hp_states[];
static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
{
return cpuhp_hp_states + state;
}
static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
{
return bringup ? !step->startup.single : !step->teardown.single;
}
/**
* cpuhp_invoke_callback - Invoke the callbacks for a given state
* @cpu: The cpu for which the callback should be invoked
* @state: The state to do callbacks for
* @bringup: True if the bringup callback should be invoked
* @node: For multi-instance, do a single entry callback for install/remove
* @lastp: For multi-instance rollback, remember how far we got
*
* Called from cpu hotplug and from the state register machinery.
*
* Return: %0 on success or a negative errno code
*/
static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
bool bringup, struct hlist_node *node,
struct hlist_node **lastp)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
struct cpuhp_step *step = cpuhp_get_step(state);
int (*cbm)(unsigned int cpu, struct hlist_node *node);
int (*cb)(unsigned int cpu);
int ret, cnt;
if (st->fail == state) {
st->fail = CPUHP_INVALID;
return -EAGAIN;
}
if (cpuhp_step_empty(bringup, step)) {
WARN_ON_ONCE(1);
return 0;
}
if (!step->multi_instance) {
WARN_ON_ONCE(lastp && *lastp);
cb = bringup ? step->startup.single : step->teardown.single;
trace_cpuhp_enter(cpu, st->target, state, cb);
ret = cb(cpu);
trace_cpuhp_exit(cpu, st->state, state, ret);
return ret;
}
cbm = bringup ? step->startup.multi : step->teardown.multi;
/* Single invocation for instance add/remove */
if (node) {
WARN_ON_ONCE(lastp && *lastp);
trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
ret = cbm(cpu, node);
trace_cpuhp_exit(cpu, st->state, state, ret);
return ret;
}
/* State transition. Invoke on all instances */
cnt = 0;
hlist_for_each(node, &step->list) {
if (lastp && node == *lastp)
break;
trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
ret = cbm(cpu, node);
trace_cpuhp_exit(cpu, st->state, state, ret);
if (ret) {
if (!lastp)
goto err;
*lastp = node;
return ret;
}
cnt++;
}
if (lastp)
*lastp = NULL;
return 0;
err:
/* Rollback the instances if one failed */
cbm = !bringup ? step->startup.multi : step->teardown.multi;
if (!cbm)
return ret;
hlist_for_each(node, &step->list) {
if (!cnt--)
break;
trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
ret = cbm(cpu, node);
trace_cpuhp_exit(cpu, st->state, state, ret);
/*
* Rollback must not fail,
*/
WARN_ON_ONCE(ret);
}
return ret;
}
#ifdef CONFIG_SMP
static bool cpuhp_is_ap_state(enum cpuhp_state state)
{
/*
* The extra check for CPUHP_TEARDOWN_CPU is only for documentation
* purposes as that state is handled explicitly in cpu_down.
*/
return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
}
static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
{
struct completion *done = bringup ? &st->done_up : &st->done_down;
wait_for_completion(done);
}
static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
{
struct completion *done = bringup ? &st->done_up : &st->done_down;
complete(done);
}
/*
* The former STARTING/DYING states, ran with IRQs disabled and must not fail.
*/
static bool cpuhp_is_atomic_state(enum cpuhp_state state)
{
return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
}
/* Synchronization state management */
enum cpuhp_sync_state {
SYNC_STATE_DEAD,
SYNC_STATE_KICKED,
SYNC_STATE_SHOULD_DIE,
SYNC_STATE_ALIVE,
SYNC_STATE_SHOULD_ONLINE,
SYNC_STATE_ONLINE,
};
#ifdef CONFIG_HOTPLUG_CORE_SYNC
/**
* cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
* @state: The synchronization state to set
*
* No synchronization point. Just update of the synchronization state, but implies
* a full barrier so that the AP changes are visible before the control CPU proceeds.
*/
static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state)
{
atomic_t *st = this_cpu_ptr(&cpuhp_state.ap_sync_state);
(void)atomic_xchg(st, state);
}
void __weak arch_cpuhp_sync_state_poll(void) { cpu_relax(); }
static bool cpuhp_wait_for_sync_state(unsigned int cpu, enum cpuhp_sync_state state,
enum cpuhp_sync_state next_state)
{
atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
ktime_t now, end, start = ktime_get();
int sync;
end = start + 10ULL * NSEC_PER_SEC;
sync = atomic_read(st);
while (1) {
if (sync == state) {
if (!atomic_try_cmpxchg(st, &sync, next_state))
continue;
return true;
}
now = ktime_get();
if (now > end) {
/* Timeout. Leave the state unchanged */
return false;
} else if (now - start < NSEC_PER_MSEC) {
/* Poll for one millisecond */
arch_cpuhp_sync_state_poll();
} else {
usleep_range_state(USEC_PER_MSEC, 2 * USEC_PER_MSEC, TASK_UNINTERRUPTIBLE);
}
sync = atomic_read(st);
}
return true;
}
#else /* CONFIG_HOTPLUG_CORE_SYNC */
static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state) { }
#endif /* !CONFIG_HOTPLUG_CORE_SYNC */
#ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD
/**
* cpuhp_ap_report_dead - Update synchronization state to DEAD
*
* No synchronization point. Just update of the synchronization state.
*/
void cpuhp_ap_report_dead(void)
{
cpuhp_ap_update_sync_state(SYNC_STATE_DEAD);
}
void __weak arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { }
/*
* Late CPU shutdown synchronization point. Cannot use cpuhp_state::done_down
* because the AP cannot issue complete() at this stage.
*/
static void cpuhp_bp_sync_dead(unsigned int cpu)
{
atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
int sync = atomic_read(st);
do {
/* CPU can have reported dead already. Don't overwrite that! */
if (sync == SYNC_STATE_DEAD)
break;
} while (!atomic_try_cmpxchg(st, &sync, SYNC_STATE_SHOULD_DIE));
if (cpuhp_wait_for_sync_state(cpu, SYNC_STATE_DEAD, SYNC_STATE_DEAD)) {
/* CPU reached dead state. Invoke the cleanup function */
arch_cpuhp_cleanup_dead_cpu(cpu);
return;
}
/* No further action possible. Emit message and give up. */
pr_err("CPU%u failed to report dead state\n", cpu);
}
#else /* CONFIG_HOTPLUG_CORE_SYNC_DEAD */
static inline void cpuhp_bp_sync_dead(unsigned int cpu) { }
#endif /* !CONFIG_HOTPLUG_CORE_SYNC_DEAD */
#ifdef CONFIG_HOTPLUG_CORE_SYNC_FULL
/**
* cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
*
* Updates the AP synchronization state to SYNC_STATE_ALIVE and waits
* for the BP to release it.
*/
void cpuhp_ap_sync_alive(void)
{
atomic_t *st = this_cpu_ptr(&cpuhp_state.ap_sync_state);
cpuhp_ap_update_sync_state(SYNC_STATE_ALIVE);
/* Wait for the control CPU to release it. */
while (atomic_read(st) != SYNC_STATE_SHOULD_ONLINE)
cpu_relax();
}
static bool cpuhp_can_boot_ap(unsigned int cpu)
{
atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
int sync = atomic_read(st);
again:
switch (sync) {
case SYNC_STATE_DEAD:
/* CPU is properly dead */
break;
case SYNC_STATE_KICKED:
/* CPU did not come up in previous attempt */
break;
case SYNC_STATE_ALIVE:
/* CPU is stuck cpuhp_ap_sync_alive(). */
break;
default:
/* CPU failed to report online or dead and is in limbo state. */
return false;
}
/* Prepare for booting */
if (!atomic_try_cmpxchg(st, &sync, SYNC_STATE_KICKED))
goto again;
return true;
}
void __weak arch_cpuhp_cleanup_kick_cpu(unsigned int cpu) { }
/*
* Early CPU bringup synchronization point. Cannot use cpuhp_state::done_up
* because the AP cannot issue complete() so early in the bringup.
*/
static int cpuhp_bp_sync_alive(unsigned int cpu)
{
int ret = 0;
if (!IS_ENABLED(CONFIG_HOTPLUG_CORE_SYNC_FULL))
return 0;
if (!cpuhp_wait_for_sync_state(cpu, SYNC_STATE_ALIVE, SYNC_STATE_SHOULD_ONLINE)) {
pr_err("CPU%u failed to report alive state\n", cpu);
ret = -EIO;
}
/* Let the architecture cleanup the kick alive mechanics. */
arch_cpuhp_cleanup_kick_cpu(cpu);
return ret;
}
#else /* CONFIG_HOTPLUG_CORE_SYNC_FULL */
static inline int cpuhp_bp_sync_alive(unsigned int cpu) { return 0; }
static inline bool cpuhp_can_boot_ap(unsigned int cpu) { return true; }
#endif /* !CONFIG_HOTPLUG_CORE_SYNC_FULL */
/* Serializes the updates to cpu_online_mask, cpu_present_mask */
static DEFINE_MUTEX(cpu_add_remove_lock);
bool cpuhp_tasks_frozen;
EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
/*
* The following two APIs (cpu_maps_update_begin/done) must be used when
* attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
*/
void cpu_maps_update_begin(void)
{
mutex_lock(&cpu_add_remove_lock);
}
void cpu_maps_update_done(void)
{
mutex_unlock(&cpu_add_remove_lock);
}
/*
* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
* Should always be manipulated under cpu_add_remove_lock
*/
static int cpu_hotplug_disabled;
#ifdef CONFIG_HOTPLUG_CPU
DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
void cpus_read_lock(void)
{
percpu_down_read(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(cpus_read_lock);
int cpus_read_trylock(void)
{
return percpu_down_read_trylock(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(cpus_read_trylock);
void cpus_read_unlock(void)
{
percpu_up_read(&cpu_hotplug_lock);
}
EXPORT_SYMBOL_GPL(cpus_read_unlock);
void cpus_write_lock(void)
{
percpu_down_write(&cpu_hotplug_lock);
}
void cpus_write_unlock(void)
{
percpu_up_write(&cpu_hotplug_lock);
}
void lockdep_assert_cpus_held(void)
{
/*
* We can't have hotplug operations before userspace starts running,
* and some init codepaths will knowingly not take the hotplug lock.
* This is all valid, so mute lockdep until it makes sense to report
* unheld locks.
*/
if (system_state < SYSTEM_RUNNING)
return;
percpu_rwsem_assert_held(&cpu_hotplug_lock);
}
#ifdef CONFIG_LOCKDEP
int lockdep_is_cpus_held(void)
{
return percpu_rwsem_is_held(&cpu_hotplug_lock);
}
#endif
static void lockdep_acquire_cpus_lock(void)
{
rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
}
static void lockdep_release_cpus_lock(void)
{
rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
}
/*
* Wait for currently running CPU hotplug operations to complete (if any) and
* disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
* the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
* hotplug path before performing hotplug operations. So acquiring that lock
* guarantees mutual exclusion from any currently running hotplug operations.
*/
void cpu_hotplug_disable(void)
{
cpu_maps_update_begin();
cpu_hotplug_disabled++;
cpu_maps_update_done();
}
EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
static void __cpu_hotplug_enable(void)
{
if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
return;
cpu_hotplug_disabled--;
}
void cpu_hotplug_enable(void)
{
cpu_maps_update_begin();
__cpu_hotplug_enable();
cpu_maps_update_done();
}
EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
#else
static void lockdep_acquire_cpus_lock(void)
{
}
static void lockdep_release_cpus_lock(void)
{
}
#endif /* CONFIG_HOTPLUG_CPU */
/*
* Architectures that need SMT-specific errata handling during SMT hotplug
* should override this.
*/
void __weak arch_smt_update(void) { }
#ifdef CONFIG_HOTPLUG_SMT
enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
static unsigned int cpu_smt_max_threads __ro_after_init;
unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX;
void __init cpu_smt_disable(bool force)
{
if (!cpu_smt_possible())
return;
if (force) {
pr_info("SMT: Force disabled\n");
cpu_smt_control = CPU_SMT_FORCE_DISABLED;
} else {
pr_info("SMT: disabled\n");
cpu_smt_control = CPU_SMT_DISABLED;
}
cpu_smt_num_threads = 1;
}
/*
* The decision whether SMT is supported can only be done after the full
* CPU identification. Called from architecture code.
*/
void __init cpu_smt_set_num_threads(unsigned int num_threads,
unsigned int max_threads)
{
WARN_ON(!num_threads || (num_threads > max_threads));
if (max_threads == 1)
cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
cpu_smt_max_threads = max_threads;
/*
* If SMT has been disabled via the kernel command line or SMT is
* not supported, set cpu_smt_num_threads to 1 for consistency.
* If enabled, take the architecture requested number of threads
* to bring up into account.
*/
if (cpu_smt_control != CPU_SMT_ENABLED)
cpu_smt_num_threads = 1;
else if (num_threads < cpu_smt_num_threads)
cpu_smt_num_threads = num_threads;
}
static int __init smt_cmdline_disable(char *str)
{
cpu_smt_disable(str && !strcmp(str, "force"));
return 0;
}
early_param("nosmt", smt_cmdline_disable);
/*
* For Archicture supporting partial SMT states check if the thread is allowed.
* Otherwise this has already been checked through cpu_smt_max_threads when
* setting the SMT level.
*/
static inline bool cpu_smt_thread_allowed(unsigned int cpu)
{
#ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC
return topology_smt_thread_allowed(cpu);
#else
return true;
#endif
}
static inline bool cpu_bootable(unsigned int cpu)
{
if (cpu_smt_control == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
return true;
/* All CPUs are bootable if controls are not configured */
if (cpu_smt_control == CPU_SMT_NOT_IMPLEMENTED)
return true;
/* All CPUs are bootable if CPU is not SMT capable */
if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
return true;
if (topology_is_primary_thread(cpu))
return true;
/*
* On x86 it's required to boot all logical CPUs at least once so
* that the init code can get a chance to set CR4.MCE on each
* CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
* core will shutdown the machine.
*/
return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
}
/* Returns true if SMT is supported and not forcefully (irreversibly) disabled */
bool cpu_smt_possible(void)
{
return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
}
EXPORT_SYMBOL_GPL(cpu_smt_possible);
#else
static inline bool cpu_bootable(unsigned int cpu) { return true; }
#endif
static inline enum cpuhp_state
cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
{
enum cpuhp_state prev_state = st->state;
bool bringup = st->state < target;
st->rollback = false;
st->last = NULL;
st->target = target;
st->single = false;
st->bringup = bringup;
if (cpu_dying(cpu) != !bringup)
set_cpu_dying(cpu, !bringup);
return prev_state;
}
static inline void
cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
enum cpuhp_state prev_state)
{
bool bringup = !st->bringup;
st->target = prev_state;
/*
* Already rolling back. No need invert the bringup value or to change
* the current state.
*/
if (st->rollback)
return;
st->rollback = true;
/*
* If we have st->last we need to undo partial multi_instance of this
* state first. Otherwise start undo at the previous state.
*/
if (!st->last) {
if (st->bringup)
st->state--;
else
st->state++;
}
st->bringup = bringup;
if (cpu_dying(cpu) != !bringup)
set_cpu_dying(cpu, !bringup);
}
/* Regular hotplug invocation of the AP hotplug thread */
static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
{
if (!st->single && st->state == st->target)
return;
st->result = 0;
/*
* Make sure the above stores are visible before should_run becomes
* true. Paired with the mb() above in cpuhp_thread_fun()
*/
smp_mb();
st->should_run = true;
wake_up_process(st->thread);
wait_for_ap_thread(st, st->bringup);
}
static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
enum cpuhp_state target)
{
enum cpuhp_state prev_state;
int ret;
prev_state = cpuhp_set_state(cpu, st, target);
__cpuhp_kick_ap(st);
if ((ret = st->result)) {
cpuhp_reset_state(cpu, st, prev_state);
__cpuhp_kick_ap(st);
}
return ret;
}
static int bringup_wait_for_ap_online(unsigned int cpu)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
wait_for_ap_thread(st, true);
if (WARN_ON_ONCE((!cpu_online(cpu))))
return -ECANCELED;
/* Unpark the hotplug thread of the target cpu */
kthread_unpark(st->thread);
/*
* SMT soft disabling on X86 requires to bring the CPU out of the
* BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
* CPU marked itself as booted_once in notify_cpu_starting() so the
* cpu_bootable() check will now return false if this is not the
* primary sibling.
*/
if (!cpu_bootable(cpu))
return -ECANCELED;
return 0;
}
#ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
static int cpuhp_kick_ap_alive(unsigned int cpu)
{
if (!cpuhp_can_boot_ap(cpu))
return -EAGAIN;
return arch_cpuhp_kick_ap_alive(cpu, idle_thread_get(cpu));
}
static int cpuhp_bringup_ap(unsigned int cpu)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int ret;
/*
* Some architectures have to walk the irq descriptors to
* setup the vector space for the cpu which comes online.
* Prevent irq alloc/free across the bringup.
*/
irq_lock_sparse();
ret = cpuhp_bp_sync_alive(cpu);
if (ret)
goto out_unlock;
ret = bringup_wait_for_ap_online(cpu);
if (ret)
goto out_unlock;
irq_unlock_sparse();
if (st->target <= CPUHP_AP_ONLINE_IDLE)
return 0;
return cpuhp_kick_ap(cpu, st, st->target);
out_unlock:
irq_unlock_sparse();
return ret;
}
#else
static int bringup_cpu(unsigned int cpu)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
struct task_struct *idle = idle_thread_get(cpu);
int ret;
if (!cpuhp_can_boot_ap(cpu))
return -EAGAIN;
/*
* Some architectures have to walk the irq descriptors to
* setup the vector space for the cpu which comes online.
*
* Prevent irq alloc/free across the bringup by acquiring the
* sparse irq lock. Hold it until the upcoming CPU completes the
* startup in cpuhp_online_idle() which allows to avoid
* intermediate synchronization points in the architecture code.
*/
irq_lock_sparse();
ret = __cpu_up(cpu, idle);
if (ret)
goto out_unlock;
ret = cpuhp_bp_sync_alive(cpu);
if (ret)
goto out_unlock;
ret = bringup_wait_for_ap_online(cpu);
if (ret)
goto out_unlock;
irq_unlock_sparse();
if (st->target <= CPUHP_AP_ONLINE_IDLE)
return 0;
return cpuhp_kick_ap(cpu, st, st->target);
out_unlock:
irq_unlock_sparse();
return ret;
}
#endif
static int finish_cpu(unsigned int cpu)
{
struct task_struct *idle = idle_thread_get(cpu);
struct mm_struct *mm = idle->active_mm;
/*
* idle_task_exit() will have switched to &init_mm, now
* clean up any remaining active_mm state.
*/
if (mm != &init_mm)
idle->active_mm = &init_mm;
mmdrop_lazy_tlb(mm);
return 0;
}
/*
* Hotplug state machine related functions
*/
/*
* Get the next state to run. Empty ones will be skipped. Returns true if a
* state must be run.
*
* st->state will be modified ahead of time, to match state_to_run, as if it
* has already ran.
*/
static bool cpuhp_next_state(bool bringup,
enum cpuhp_state *state_to_run,
struct cpuhp_cpu_state *st,
enum cpuhp_state target)
{
do {
if (bringup) {
if (st->state >= target)
return false;
*state_to_run = ++st->state;
} else {
if (st->state <= target)
return false;
*state_to_run = st->state--;
}
if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
break;
} while (true);
return true;
}
static int __cpuhp_invoke_callback_range(bool bringup,
unsigned int cpu,
struct cpuhp_cpu_state *st,
enum cpuhp_state target,
bool nofail)
{
enum cpuhp_state state;
int ret = 0;
while (cpuhp_next_state(bringup, &state, st, target)) {
int err;
err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
if (!err)
continue;
if (nofail) {
pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
cpu, bringup ? "UP" : "DOWN",
cpuhp_get_step(st->state)->name,
st->state, err);
ret = -1;
} else {
ret = err;
break;
}
}
return ret;
}
static inline int cpuhp_invoke_callback_range(bool bringup,
unsigned int cpu,
struct cpuhp_cpu_state *st,
enum cpuhp_state target)
{
return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);
}
static inline void cpuhp_invoke_callback_range_nofail(bool bringup,
unsigned int cpu,
struct cpuhp_cpu_state *st,
enum cpuhp_state target)
{
__cpuhp_invoke_callback_range(bringup, cpu, st, target, true);
}
static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
{
if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
return true;
/*
* When CPU hotplug is disabled, then taking the CPU down is not
* possible because takedown_cpu() and the architecture and
* subsystem specific mechanisms are not available. So the CPU
* which would be completely unplugged again needs to stay around
* in the current state.
*/
return st->state <= CPUHP_BRINGUP_CPU;
}
static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
enum cpuhp_state target)
{
enum cpuhp_state prev_state = st->state;
int ret = 0;
ret = cpuhp_invoke_callback_range(true, cpu, st, target);
if (ret) {
pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
ret, cpu, cpuhp_get_step(st->state)->name,
st->state);
cpuhp_reset_state(cpu, st, prev_state);
if (can_rollback_cpu(st))
WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
prev_state));
}
return ret;
}
/*
* The cpu hotplug threads manage the bringup and teardown of the cpus
*/
static int cpuhp_should_run(unsigned int cpu)
{
struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
return st->should_run;
}
/*
* Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
* callbacks when a state gets [un]installed at runtime.
*
* Each invocation of this function by the smpboot thread does a single AP
* state callback.
*
* It has 3 modes of operation:
* - single: runs st->cb_state
* - up: runs ++st->state, while st->state < st->target
* - down: runs st->state--, while st->state > st->target
*
* When complete or on error, should_run is cleared and the completion is fired.
*/
static void cpuhp_thread_fun(unsigned int cpu)
{
struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
bool bringup = st->bringup;
enum cpuhp_state state;
if (WARN_ON_ONCE(!st->should_run))
return;
/*
* ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
* that if we see ->should_run we also see the rest of the state.
*/
smp_mb();
/*
* The BP holds the hotplug lock, but we're now running on the AP,
* ensure that anybody asserting the lock is held, will actually find
* it so.
*/
lockdep_acquire_cpus_lock();
cpuhp_lock_acquire(bringup);
if (st->single) {
state = st->cb_state;
st->should_run = false;
} else {
st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
if (!st->should_run)
goto end;
}
WARN_ON_ONCE(!cpuhp_is_ap_state(state));
if (cpuhp_is_atomic_state(state)) {
local_irq_disable();
st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
local_irq_enable();
/*
* STARTING/DYING must not fail!
*/
WARN_ON_ONCE(st->result);
} else {
st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
}
if (st->result) {
/*
* If we fail on a rollback, we're up a creek without no
* paddle, no way forward, no way back. We loose, thanks for
* playing.
*/
WARN_ON_ONCE(st->rollback);
st->should_run = false;
}
end:
cpuhp_lock_release(bringup);
lockdep_release_cpus_lock();
if (!st->should_run)
complete_ap_thread(st, bringup);
}
/* Invoke a single callback on a remote cpu */
static int
cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
struct hlist_node *node)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int ret;
if (!cpu_online(cpu))
return 0;
cpuhp_lock_acquire(false);
cpuhp_lock_release(false);
cpuhp_lock_acquire(true);
cpuhp_lock_release(true);
/*
* If we are up and running, use the hotplug thread. For early calls
* we invoke the thread function directly.
*/
if (!st->thread)
return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
st->rollback = false;
st->last = NULL;
st->node = node;
st->bringup = bringup;
st->cb_state = state;
st->single = true;
__cpuhp_kick_ap(st);
/*
* If we failed and did a partial, do a rollback.
*/
if ((ret = st->result) && st->last) {
st->rollback = true;
st->bringup = !bringup;
__cpuhp_kick_ap(st);
}
/*
* Clean up the leftovers so the next hotplug operation wont use stale
* data.
*/
st->node = st->last = NULL;
return ret;
}
static int cpuhp_kick_ap_work(unsigned int cpu)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
enum cpuhp_state prev_state = st->state;
int ret;
cpuhp_lock_acquire(false);
cpuhp_lock_release(false);
cpuhp_lock_acquire(true);
cpuhp_lock_release(true);
trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
ret = cpuhp_kick_ap(cpu, st, st->target);
trace_cpuhp_exit(cpu, st->state, prev_state, ret);
return ret;
}
static struct smp_hotplug_thread cpuhp_threads = {
.store = &cpuhp_state.thread,
.thread_should_run = cpuhp_should_run,
.thread_fn = cpuhp_thread_fun,
.thread_comm = "cpuhp/%u",
.selfparking = true,
};
static __init void cpuhp_init_state(void)
{
struct cpuhp_cpu_state *st;
int cpu;
for_each_possible_cpu(cpu) {
st = per_cpu_ptr(&cpuhp_state, cpu);
init_completion(&st->done_up);
init_completion(&st->done_down);
}
}
void __init cpuhp_threads_init(void)
{
cpuhp_init_state();
BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
kthread_unpark(this_cpu_read(cpuhp_state.thread));
}
/*
*
* Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
* protected region.
*
* The operation is still serialized against concurrent CPU hotplug via
* cpu_add_remove_lock, i.e. CPU map protection. But it is _not_
* serialized against other hotplug related activity like adding or
* removing of state callbacks and state instances, which invoke either the
* startup or the teardown callback of the affected state.
*
* This is required for subsystems which are unfixable vs. CPU hotplug and
* evade lock inversion problems by scheduling work which has to be
* completed _before_ cpu_up()/_cpu_down() returns.
*
* Don't even think about adding anything to this for any new code or even
* drivers. It's only purpose is to keep existing lock order trainwrecks
* working.
*
* For cpu_down() there might be valid reasons to finish cleanups which are
* not required to be done under cpu_hotplug_lock, but that's a different
* story and would be not invoked via this.
*/
static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
{
/*
* cpusets delegate hotplug operations to a worker to "solve" the
* lock order problems. Wait for the worker, but only if tasks are
* _not_ frozen (suspend, hibernate) as that would wait forever.
*
* The wait is required because otherwise the hotplug operation
* returns with inconsistent state, which could even be observed in
* user space when a new CPU is brought up. The CPU plug uevent
* would be delivered and user space reacting on it would fail to
* move tasks to the newly plugged CPU up to the point where the
* work has finished because up to that point the newly plugged CPU
* is not assignable in cpusets/cgroups. On unplug that's not
* necessarily a visible issue, but it is still inconsistent state,
* which is the real problem which needs to be "fixed". This can't
* prevent the transient state between scheduling the work and
* returning from waiting for it.
*/
if (!tasks_frozen)
cpuset_wait_for_hotplug();
}
#ifdef CONFIG_HOTPLUG_CPU
#ifndef arch_clear_mm_cpumask_cpu
#define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
#endif
/**
* clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
* @cpu: a CPU id
*
* This function walks all processes, finds a valid mm struct for each one and
* then clears a corresponding bit in mm's cpumask. While this all sounds
* trivial, there are various non-obvious corner cases, which this function
* tries to solve in a safe manner.
*
* Also note that the function uses a somewhat relaxed locking scheme, so it may
* be called only for an already offlined CPU.
*/
void clear_tasks_mm_cpumask(int cpu)
{
struct task_struct *p;
/*
* This function is called after the cpu is taken down and marked
* offline, so its not like new tasks will ever get this cpu set in
* their mm mask. -- Peter Zijlstra
* Thus, we may use rcu_read_lock() here, instead of grabbing
* full-fledged tasklist_lock.
*/
WARN_ON(cpu_online(cpu));
rcu_read_lock();
for_each_process(p) {
struct task_struct *t;
/*
* Main thread might exit, but other threads may still have
* a valid mm. Find one.
*/
t = find_lock_task_mm(p);
if (!t)
continue;
arch_clear_mm_cpumask_cpu(cpu, t->mm);
task_unlock(t);
}
rcu_read_unlock();
}
/* Take this CPU down. */
static int take_cpu_down(void *_param)
{
struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
int err, cpu = smp_processor_id();
/* Ensure this CPU doesn't handle any more interrupts. */
err = __cpu_disable();
if (err < 0)
return err;
/*
* Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
* down, that the current state is CPUHP_TEARDOWN_CPU - 1.
*/
WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
/*
* Invoke the former CPU_DYING callbacks. DYING must not fail!
*/
cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
/* Park the stopper thread */
stop_machine_park(cpu);
return 0;
}
static int takedown_cpu(unsigned int cpu)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int err;
/* Park the smpboot threads */
kthread_park(st->thread);
/*
* Prevent irq alloc/free while the dying cpu reorganizes the
* interrupt affinities.
*/
irq_lock_sparse();
/*
* So now all preempt/rcu users must observe !cpu_active().
*/
err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
if (err) {
/* CPU refused to die */
irq_unlock_sparse();
/* Unpark the hotplug thread so we can rollback there */
kthread_unpark(st->thread);
return err;
}
BUG_ON(cpu_online(cpu));
/*
* The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
* all runnable tasks from the CPU, there's only the idle task left now
* that the migration thread is done doing the stop_machine thing.
*
* Wait for the stop thread to go away.
*/
wait_for_ap_thread(st, false);
BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
/* Interrupts are moved away from the dying cpu, reenable alloc/free */
irq_unlock_sparse();
hotplug_cpu__broadcast_tick_pull(cpu);
/* This actually kills the CPU. */
__cpu_die(cpu);
cpuhp_bp_sync_dead(cpu);
tick_cleanup_dead_cpu(cpu);
/*
* Callbacks must be re-integrated right away to the RCU state machine.
* Otherwise an RCU callback could block a further teardown function
* waiting for its completion.
*/
rcutree_migrate_callbacks(cpu);
return 0;
}
static void cpuhp_complete_idle_dead(void *arg)
{
struct cpuhp_cpu_state *st = arg;
complete_ap_thread(st, false);
}
void cpuhp_report_idle_dead(void)
{
struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
BUG_ON(st->state != CPUHP_AP_OFFLINE);
tick_assert_timekeeping_handover();
rcutree_report_cpu_dead();
st->state = CPUHP_AP_IDLE_DEAD;
/*
* We cannot call complete after rcutree_report_cpu_dead() so we delegate it
* to an online cpu.
*/
smp_call_function_single(cpumask_first(cpu_online_mask),
cpuhp_complete_idle_dead, st, 0);
}
static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
enum cpuhp_state target)
{
enum cpuhp_state prev_state = st->state;
int ret = 0;
ret = cpuhp_invoke_callback_range(false, cpu, st, target);
if (ret) {
pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
ret, cpu, cpuhp_get_step(st->state)->name,
st->state);
cpuhp_reset_state(cpu, st, prev_state);
if (st->state < prev_state)
WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
prev_state));
}
return ret;
}
/* Requires cpu_add_remove_lock to be held */
static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
enum cpuhp_state target)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int prev_state, ret = 0;
if (num_online_cpus() == 1)
return -EBUSY;
if (!cpu_present(cpu))
return -EINVAL;
cpus_write_lock();
cpuhp_tasks_frozen = tasks_frozen;
prev_state = cpuhp_set_state(cpu, st, target);
/*
* If the current CPU state is in the range of the AP hotplug thread,
* then we need to kick the thread.
*/
if (st->state > CPUHP_TEARDOWN_CPU) {
st->target = max((int)target, CPUHP_TEARDOWN_CPU);
ret = cpuhp_kick_ap_work(cpu);
/*
* The AP side has done the error rollback already. Just
* return the error code..
*/
if (ret)
goto out;
/*
* We might have stopped still in the range of the AP hotplug
* thread. Nothing to do anymore.
*/
if (st->state > CPUHP_TEARDOWN_CPU)
goto out;
st->target = target;
}
/*
* The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
* to do the further cleanups.
*/
ret = cpuhp_down_callbacks(cpu, st, target);
if (ret && st->state < prev_state) {
if (st->state == CPUHP_TEARDOWN_CPU) {
cpuhp_reset_state(cpu, st, prev_state);
__cpuhp_kick_ap(st);
} else {
WARN(1, "DEAD callback error for CPU%d", cpu);
}
}
out:
cpus_write_unlock();
/*
* Do post unplug cleanup. This is still protected against
* concurrent CPU hotplug via cpu_add_remove_lock.
*/
lockup_detector_cleanup();
arch_smt_update();
cpu_up_down_serialize_trainwrecks(tasks_frozen);
return ret;
}
struct cpu_down_work {
unsigned int cpu;
enum cpuhp_state target;
};
static long __cpu_down_maps_locked(void *arg)
{
struct cpu_down_work *work = arg;
return _cpu_down(work->cpu, 0, work->target);
}
static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
{
struct cpu_down_work work = { .cpu = cpu, .target = target, };
/*
* If the platform does not support hotplug, report it explicitly to
* differentiate it from a transient offlining failure.
*/
if (cc_platform_has(CC_ATTR_HOTPLUG_DISABLED))
return -EOPNOTSUPP;
if (cpu_hotplug_disabled)
return -EBUSY;
/*
* Ensure that the control task does not run on the to be offlined
* CPU to prevent a deadlock against cfs_b->period_timer.
* Also keep at least one housekeeping cpu onlined to avoid generating
* an empty sched_domain span.
*/
for_each_cpu_and(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) {
if (cpu != work.cpu)
return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
}
return -EBUSY;
}
static int cpu_down(unsigned int cpu, enum cpuhp_state target)
{
int err;
cpu_maps_update_begin();
err = cpu_down_maps_locked(cpu, target);
cpu_maps_update_done();
return err;
}
/**
* cpu_device_down - Bring down a cpu device
* @dev: Pointer to the cpu device to offline
*
* This function is meant to be used by device core cpu subsystem only.
*
* Other subsystems should use remove_cpu() instead.
*
* Return: %0 on success or a negative errno code
*/
int cpu_device_down(struct device *dev)
{
return cpu_down(dev->id, CPUHP_OFFLINE);
}
int remove_cpu(unsigned int cpu)
{
int ret;
lock_device_hotplug();
ret = device_offline(get_cpu_device(cpu));
unlock_device_hotplug();
return ret;
}
EXPORT_SYMBOL_GPL(remove_cpu);
void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
{
unsigned int cpu;
int error;
cpu_maps_update_begin();
/*
* Make certain the cpu I'm about to reboot on is online.
*
* This is inline to what migrate_to_reboot_cpu() already do.
*/
if (!cpu_online(primary_cpu))
primary_cpu = cpumask_first(cpu_online_mask);
for_each_online_cpu(cpu) {
if (cpu == primary_cpu)
continue;
error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
if (error) {
pr_err("Failed to offline CPU%d - error=%d",
cpu, error);
break;
}
}
/*
* Ensure all but the reboot CPU are offline.
*/
BUG_ON(num_online_cpus() > 1);
/*
* Make sure the CPUs won't be enabled by someone else after this
* point. Kexec will reboot to a new kernel shortly resetting
* everything along the way.
*/
cpu_hotplug_disabled++;
cpu_maps_update_done();
}
#else
#define takedown_cpu NULL
#endif /*CONFIG_HOTPLUG_CPU*/
/**
* notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
* @cpu: cpu that just started
*
* It must be called by the arch code on the new cpu, before the new cpu
* enables interrupts and before the "boot" cpu returns from __cpu_up().
*/
void notify_cpu_starting(unsigned int cpu)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
rcutree_report_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
cpumask_set_cpu(cpu, &cpus_booted_once_mask);
/*
* STARTING must not fail!
*/
cpuhp_invoke_callback_range_nofail(true, cpu, st, target);
}
/*
* Called from the idle task. Wake up the controlling task which brings the
* hotplug thread of the upcoming CPU up and then delegates the rest of the
* online bringup to the hotplug thread.
*/
void cpuhp_online_idle(enum cpuhp_state state)
{
struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
/* Happens for the boot cpu */
if (state != CPUHP_AP_ONLINE_IDLE)
return;
cpuhp_ap_update_sync_state(SYNC_STATE_ONLINE);
/*
* Unpark the stopper thread before we start the idle loop (and start
* scheduling); this ensures the stopper task is always available.
*/
stop_machine_unpark(smp_processor_id());
st->state = CPUHP_AP_ONLINE_IDLE;
complete_ap_thread(st, true);
}
/* Requires cpu_add_remove_lock to be held */
static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
struct task_struct *idle;
int ret = 0;
cpus_write_lock();
if (!cpu_present(cpu)) {
ret = -EINVAL;
goto out;
}
/*
* The caller of cpu_up() might have raced with another
* caller. Nothing to do.
*/
if (st->state >= target)
goto out;
if (st->state == CPUHP_OFFLINE) {
/* Let it fail before we try to bring the cpu up */
idle = idle_thread_get(cpu);
if (IS_ERR(idle)) {
ret = PTR_ERR(idle);
goto out;
}
/*
* Reset stale stack state from the last time this CPU was online.
*/
scs_task_reset(idle);
kasan_unpoison_task_stack(idle);
}
cpuhp_tasks_frozen = tasks_frozen;
cpuhp_set_state(cpu, st, target);
/*
* If the current CPU state is in the range of the AP hotplug thread,
* then we need to kick the thread once more.
*/
if (st->state > CPUHP_BRINGUP_CPU) {
ret = cpuhp_kick_ap_work(cpu);
/*
* The AP side has done the error rollback already. Just
* return the error code..
*/
if (ret)
goto out;
}
/*
* Try to reach the target state. We max out on the BP at
* CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
* responsible for bringing it up to the target state.
*/
target = min((int)target, CPUHP_BRINGUP_CPU);
ret = cpuhp_up_callbacks(cpu, st, target);
out:
cpus_write_unlock();
arch_smt_update();
cpu_up_down_serialize_trainwrecks(tasks_frozen);
return ret;
}
static int cpu_up(unsigned int cpu, enum cpuhp_state target)
{
int err = 0;
if (!cpu_possible(cpu)) {
pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
cpu);
return -EINVAL;
}
err = try_online_node(cpu_to_node(cpu));
if (err)
return err;
cpu_maps_update_begin();
if (cpu_hotplug_disabled) {
err = -EBUSY;
goto out;
}
if (!cpu_bootable(cpu)) {
err = -EPERM;
goto out;
}
err = _cpu_up(cpu, 0, target);
out:
cpu_maps_update_done();
return err;
}
/**
* cpu_device_up - Bring up a cpu device
* @dev: Pointer to the cpu device to online
*
* This function is meant to be used by device core cpu subsystem only.
*
* Other subsystems should use add_cpu() instead.
*
* Return: %0 on success or a negative errno code
*/
int cpu_device_up(struct device *dev)
{
return cpu_up(dev->id, CPUHP_ONLINE);
}
int add_cpu(unsigned int cpu)
{
int ret;
lock_device_hotplug();
ret = device_online(get_cpu_device(cpu));
unlock_device_hotplug();
return ret;
}
EXPORT_SYMBOL_GPL(add_cpu);
/**
* bringup_hibernate_cpu - Bring up the CPU that we hibernated on
* @sleep_cpu: The cpu we hibernated on and should be brought up.
*
* On some architectures like arm64, we can hibernate on any CPU, but on
* wake up the CPU we hibernated on might be offline as a side effect of
* using maxcpus= for example.
*
* Return: %0 on success or a negative errno code
*/
int bringup_hibernate_cpu(unsigned int sleep_cpu)
{
int ret;
if (!cpu_online(sleep_cpu)) {
pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
if (ret) {
pr_err("Failed to bring hibernate-CPU up!\n");
return ret;
}
}
return 0;
}
static void __init cpuhp_bringup_mask(const struct cpumask *mask, unsigned int ncpus,
enum cpuhp_state target)
{
unsigned int cpu;
for_each_cpu(cpu, mask) {
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
if (cpu_up(cpu, target) && can_rollback_cpu(st)) {
/*
* If this failed then cpu_up() might have only
* rolled back to CPUHP_BP_KICK_AP for the final
* online. Clean it up. NOOP if already rolled back.
*/
WARN_ON(cpuhp_invoke_callback_range(false, cpu, st, CPUHP_OFFLINE));
}
if (!--ncpus)
break;
}
}
#ifdef CONFIG_HOTPLUG_PARALLEL
static bool __cpuhp_parallel_bringup __ro_after_init = true;
static int __init parallel_bringup_parse_param(char *arg)
{
return kstrtobool(arg, &__cpuhp_parallel_bringup);
}
early_param("cpuhp.parallel", parallel_bringup_parse_param);
static inline bool cpuhp_smt_aware(void)
{
return cpu_smt_max_threads > 1;
}
static inline const struct cpumask *cpuhp_get_primary_thread_mask(void)
{
return cpu_primary_thread_mask;
}
/*
* On architectures which have enabled parallel bringup this invokes all BP
* prepare states for each of the to be onlined APs first. The last state
* sends the startup IPI to the APs. The APs proceed through the low level
* bringup code in parallel and then wait for the control CPU to release
* them one by one for the final onlining procedure.
*
* This avoids waiting for each AP to respond to the startup IPI in
* CPUHP_BRINGUP_CPU.
*/
static bool __init cpuhp_bringup_cpus_parallel(unsigned int ncpus)
{
const struct cpumask *mask = cpu_present_mask;
if (__cpuhp_parallel_bringup)
__cpuhp_parallel_bringup = arch_cpuhp_init_parallel_bringup();
if (!__cpuhp_parallel_bringup)
return false;
if (cpuhp_smt_aware()) {
const struct cpumask *pmask = cpuhp_get_primary_thread_mask();
static struct cpumask tmp_mask __initdata;
/*
* X86 requires to prevent that SMT siblings stopped while
* the primary thread does a microcode update for various
* reasons. Bring the primary threads up first.
*/
cpumask_and(&tmp_mask, mask, pmask);
cpuhp_bringup_mask(&tmp_mask, ncpus, CPUHP_BP_KICK_AP);
cpuhp_bringup_mask(&tmp_mask, ncpus, CPUHP_ONLINE);
/* Account for the online CPUs */
ncpus -= num_online_cpus();
if (!ncpus)
return true;
/* Create the mask for secondary CPUs */
cpumask_andnot(&tmp_mask, mask, pmask);
mask = &tmp_mask;
}
/* Bring the not-yet started CPUs up */
cpuhp_bringup_mask(mask, ncpus, CPUHP_BP_KICK_AP);
cpuhp_bringup_mask(mask, ncpus, CPUHP_ONLINE);
return true;
}
#else
static inline bool cpuhp_bringup_cpus_parallel(unsigned int ncpus) { return false; }
#endif /* CONFIG_HOTPLUG_PARALLEL */
void __init bringup_nonboot_cpus(unsigned int max_cpus)
{
/* Try parallel bringup optimization if enabled */
if (cpuhp_bringup_cpus_parallel(max_cpus))
return;
/* Full per CPU serialized bringup */
cpuhp_bringup_mask(cpu_present_mask, max_cpus, CPUHP_ONLINE);
}
#ifdef CONFIG_PM_SLEEP_SMP
static cpumask_var_t frozen_cpus;
int freeze_secondary_cpus(int primary)
{
int cpu, error = 0;
cpu_maps_update_begin();
if (primary == -1) {
primary = cpumask_first(cpu_online_mask);
if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
primary = housekeeping_any_cpu(HK_TYPE_TIMER);
} else {
if (!cpu_online(primary))
primary = cpumask_first(cpu_online_mask);
}
/*
* We take down all of the non-boot CPUs in one shot to avoid races
* with the userspace trying to use the CPU hotplug at the same time
*/
cpumask_clear(frozen_cpus);
pr_info("Disabling non-boot CPUs ...\n");
for_each_online_cpu(cpu) {
if (cpu == primary)
continue;
if (pm_wakeup_pending()) {
pr_info("Wakeup pending. Abort CPU freeze\n");
error = -EBUSY;
break;
}
trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
if (!error)
cpumask_set_cpu(cpu, frozen_cpus);
else {
pr_err("Error taking CPU%d down: %d\n", cpu, error);
break;
}
}
if (!error)
BUG_ON(num_online_cpus() > 1);
else
pr_err("Non-boot CPUs are not disabled\n");
/*
* Make sure the CPUs won't be enabled by someone else. We need to do
* this even in case of failure as all freeze_secondary_cpus() users are
* supposed to do thaw_secondary_cpus() on the failure path.
*/
cpu_hotplug_disabled++;
cpu_maps_update_done();
return error;
}
void __weak arch_thaw_secondary_cpus_begin(void)
{
}
void __weak arch_thaw_secondary_cpus_end(void)
{
}
void thaw_secondary_cpus(void)
{
int cpu, error;
/* Allow everyone to use the CPU hotplug again */
cpu_maps_update_begin();
__cpu_hotplug_enable();
if (cpumask_empty(frozen_cpus))
goto out;
pr_info("Enabling non-boot CPUs ...\n");
arch_thaw_secondary_cpus_begin();
for_each_cpu(cpu, frozen_cpus) {
trace_suspend_resume(TPS("CPU_ON"), cpu, true);
error = _cpu_up(cpu, 1, CPUHP_ONLINE);
trace_suspend_resume(TPS("CPU_ON"), cpu, false);
if (!error) {
pr_info("CPU%d is up\n", cpu);
continue;
}
pr_warn("Error taking CPU%d up: %d\n", cpu, error);
}
arch_thaw_secondary_cpus_end();
cpumask_clear(frozen_cpus);
out:
cpu_maps_update_done();
}
static int __init alloc_frozen_cpus(void)
{
if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
return -ENOMEM;
return 0;
}
core_initcall(alloc_frozen_cpus);
/*
* When callbacks for CPU hotplug notifications are being executed, we must
* ensure that the state of the system with respect to the tasks being frozen
* or not, as reported by the notification, remains unchanged *throughout the
* duration* of the execution of the callbacks.
* Hence we need to prevent the freezer from racing with regular CPU hotplug.
*
* This synchronization is implemented by mutually excluding regular CPU
* hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
* Hibernate notifications.
*/
static int
cpu_hotplug_pm_callback(struct notifier_block *nb,
unsigned long action, void *ptr)
{
switch (action) {
case PM_SUSPEND_PREPARE:
case PM_HIBERNATION_PREPARE:
cpu_hotplug_disable();
break;
case PM_POST_SUSPEND:
case PM_POST_HIBERNATION:
cpu_hotplug_enable();
break;
default:
return NOTIFY_DONE;
}
return NOTIFY_OK;
}
static int __init cpu_hotplug_pm_sync_init(void)
{
/*
* cpu_hotplug_pm_callback has higher priority than x86
* bsp_pm_callback which depends on cpu_hotplug_pm_callback
* to disable cpu hotplug to avoid cpu hotplug race.
*/
pm_notifier(cpu_hotplug_pm_callback, 0);
return 0;
}
core_initcall(cpu_hotplug_pm_sync_init);
#endif /* CONFIG_PM_SLEEP_SMP */
int __boot_cpu_id;
#endif /* CONFIG_SMP */
/* Boot processor state steps */
static struct cpuhp_step cpuhp_hp_states[] = {
[CPUHP_OFFLINE] = {
.name = "offline",
.startup.single = NULL,
.teardown.single = NULL,
},
#ifdef CONFIG_SMP
[CPUHP_CREATE_THREADS]= {
.name = "threads:prepare",
.startup.single = smpboot_create_threads,
.teardown.single = NULL,
.cant_stop = true,
},
[CPUHP_PERF_PREPARE] = {
.name = "perf:prepare",
.startup.single = perf_event_init_cpu,
.teardown.single = perf_event_exit_cpu,
},
[CPUHP_RANDOM_PREPARE] = {
.name = "random:prepare",
.startup.single = random_prepare_cpu,
.teardown.single = NULL,
},
[CPUHP_WORKQUEUE_PREP] = {
.name = "workqueue:prepare",
.startup.single = workqueue_prepare_cpu,
.teardown.single = NULL,
},
[CPUHP_HRTIMERS_PREPARE] = {
.name = "hrtimers:prepare",
.startup.single = hrtimers_prepare_cpu,
.teardown.single = NULL,
},
[CPUHP_SMPCFD_PREPARE] = {
.name = "smpcfd:prepare",
.startup.single = smpcfd_prepare_cpu,
.teardown.single = smpcfd_dead_cpu,
},
[CPUHP_RELAY_PREPARE] = {
.name = "relay:prepare",
.startup.single = relay_prepare_cpu,
.teardown.single = NULL,
},
[CPUHP_RCUTREE_PREP] = {
.name = "RCU/tree:prepare",
.startup.single = rcutree_prepare_cpu,
.teardown.single = rcutree_dead_cpu,
},
/*
* On the tear-down path, timers_dead_cpu() must be invoked
* before blk_mq_queue_reinit_notify() from notify_dead(),
* otherwise a RCU stall occurs.
*/
[CPUHP_TIMERS_PREPARE] = {
.name = "timers:prepare",
.startup.single = timers_prepare_cpu,
.teardown.single = timers_dead_cpu,
},
#ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
/*
* Kicks the AP alive. AP will wait in cpuhp_ap_sync_alive() until
* the next step will release it.
*/
[CPUHP_BP_KICK_AP] = {
.name = "cpu:kick_ap",
.startup.single = cpuhp_kick_ap_alive,
},
/*
* Waits for the AP to reach cpuhp_ap_sync_alive() and then
* releases it for the complete bringup.
*/
[CPUHP_BRINGUP_CPU] = {
.name = "cpu:bringup",
.startup.single = cpuhp_bringup_ap,
.teardown.single = finish_cpu,
.cant_stop = true,
},
#else
/*
* All-in-one CPU bringup state which includes the kick alive.
*/
[CPUHP_BRINGUP_CPU] = {
.name = "cpu:bringup",
.startup.single = bringup_cpu,
.teardown.single = finish_cpu,
.cant_stop = true,
},
#endif
/* Final state before CPU kills itself */
[CPUHP_AP_IDLE_DEAD] = {
.name = "idle:dead",
},
/*
* Last state before CPU enters the idle loop to die. Transient state
* for synchronization.
*/
[CPUHP_AP_OFFLINE] = {
.name = "ap:offline",
.cant_stop = true,
},
/* First state is scheduler control. Interrupts are disabled */
[CPUHP_AP_SCHED_STARTING] = {
.name = "sched:starting",
.startup.single = sched_cpu_starting,
.teardown.single = sched_cpu_dying,
},
[CPUHP_AP_RCUTREE_DYING] = {
.name = "RCU/tree:dying",
.startup.single = NULL,
.teardown.single = rcutree_dying_cpu,
},
[CPUHP_AP_SMPCFD_DYING] = {
.name = "smpcfd:dying",
.startup.single = NULL,
.teardown.single = smpcfd_dying_cpu,
},
[CPUHP_AP_HRTIMERS_DYING] = {
.name = "hrtimers:dying",
.startup.single = NULL,
.teardown.single = hrtimers_cpu_dying,
},
[CPUHP_AP_TICK_DYING] = {
.name = "tick:dying",
.startup.single = NULL,
.teardown.single = tick_cpu_dying,
},
/* Entry state on starting. Interrupts enabled from here on. Transient
* state for synchronsization */
[CPUHP_AP_ONLINE] = {
.name = "ap:online",
},
/*
* Handled on control processor until the plugged processor manages
* this itself.
*/
[CPUHP_TEARDOWN_CPU] = {
.name = "cpu:teardown",
.startup.single = NULL,
.teardown.single = takedown_cpu,
.cant_stop = true,
},
[CPUHP_AP_SCHED_WAIT_EMPTY] = {
.name = "sched:waitempty",
.startup.single = NULL,
.teardown.single = sched_cpu_wait_empty,
},
/* Handle smpboot threads park/unpark */
[CPUHP_AP_SMPBOOT_THREADS] = {
.name = "smpboot/threads:online",
.startup.single = smpboot_unpark_threads,
.teardown.single = smpboot_park_threads,
},
[CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
.name = "irq/affinity:online",
.startup.single = irq_affinity_online_cpu,
.teardown.single = NULL,
},
[CPUHP_AP_PERF_ONLINE] = {
.name = "perf:online",
.startup.single = perf_event_init_cpu,
.teardown.single = perf_event_exit_cpu,
},
[CPUHP_AP_WATCHDOG_ONLINE] = {
.name = "lockup_detector:online",
.startup.single = lockup_detector_online_cpu,
.teardown.single = lockup_detector_offline_cpu,
},
[CPUHP_AP_WORKQUEUE_ONLINE] = {
.name = "workqueue:online",
.startup.single = workqueue_online_cpu,
.teardown.single = workqueue_offline_cpu,
},
[CPUHP_AP_RANDOM_ONLINE] = {
.name = "random:online",
.startup.single = random_online_cpu,
.teardown.single = NULL,
},
[CPUHP_AP_RCUTREE_ONLINE] = {
.name = "RCU/tree:online",
.startup.single = rcutree_online_cpu,
.teardown.single = rcutree_offline_cpu,
},
#endif
/*
* The dynamically registered state space is here
*/
#ifdef CONFIG_SMP
/* Last state is scheduler control setting the cpu active */
[CPUHP_AP_ACTIVE] = {
.name = "sched:active",
.startup.single = sched_cpu_activate,
.teardown.single = sched_cpu_deactivate,
},
#endif
/* CPU is fully up and running. */
[CPUHP_ONLINE] = {
.name = "online",
.startup.single = NULL,
.teardown.single = NULL,
},
};
/* Sanity check for callbacks */
static int cpuhp_cb_check(enum cpuhp_state state)
{
if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
return -EINVAL;
return 0;
}
/*
* Returns a free for dynamic slot assignment of the Online state. The states
* are protected by the cpuhp_slot_states mutex and an empty slot is identified
* by having no name assigned.
*/
static int cpuhp_reserve_state(enum cpuhp_state state)
{
enum cpuhp_state i, end;
struct cpuhp_step *step;
switch (state) {
case CPUHP_AP_ONLINE_DYN:
step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
end = CPUHP_AP_ONLINE_DYN_END;
break;
case CPUHP_BP_PREPARE_DYN:
step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
end = CPUHP_BP_PREPARE_DYN_END;
break;
default:
return -EINVAL;
}
for (i = state; i <= end; i++, step++) {
if (!step->name)
return i;
}
WARN(1, "No more dynamic states available for CPU hotplug\n");
return -ENOSPC;
}
static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
int (*startup)(unsigned int cpu),
int (*teardown)(unsigned int cpu),
bool multi_instance)
{
/* (Un)Install the callbacks for further cpu hotplug operations */
struct cpuhp_step *sp;
int ret = 0;
/*
* If name is NULL, then the state gets removed.
*
* CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
* the first allocation from these dynamic ranges, so the removal
* would trigger a new allocation and clear the wrong (already
* empty) state, leaving the callbacks of the to be cleared state
* dangling, which causes wreckage on the next hotplug operation.
*/
if (name && (state == CPUHP_AP_ONLINE_DYN ||
state == CPUHP_BP_PREPARE_DYN)) {
ret = cpuhp_reserve_state(state);
if (ret < 0)
return ret;
state = ret;
}
sp = cpuhp_get_step(state);
if (name && sp->name)
return -EBUSY;
sp->startup.single = startup;
sp->teardown.single = teardown;
sp->name = name;
sp->multi_instance = multi_instance;
INIT_HLIST_HEAD(&sp->list);
return ret;
}
static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
{
return cpuhp_get_step(state)->teardown.single;
}
/*
* Call the startup/teardown function for a step either on the AP or
* on the current CPU.
*/
static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
struct hlist_node *node)
{
struct cpuhp_step *sp = cpuhp_get_step(state);
int ret;
/*
* If there's nothing to do, we done.
* Relies on the union for multi_instance.
*/
if (cpuhp_step_empty(bringup, sp))
return 0;
/*
* The non AP bound callbacks can fail on bringup. On teardown
* e.g. module removal we crash for now.
*/
#ifdef CONFIG_SMP
if (cpuhp_is_ap_state(state))
ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
else
ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
#else
ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
#endif
BUG_ON(ret && !bringup);
return ret;
}
/*
* Called from __cpuhp_setup_state on a recoverable failure.
*
* Note: The teardown callbacks for rollback are not allowed to fail!
*/
static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
struct hlist_node *node)
{
int cpu;
/* Roll back the already executed steps on the other cpus */
for_each_present_cpu(cpu) {
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int cpustate = st->state;
if (cpu >= failedcpu)
break;
/* Did we invoke the startup call on that cpu ? */
if (cpustate >= state)
cpuhp_issue_call(cpu, state, false, node);
}
}
int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
struct hlist_node *node,
bool invoke)
{
struct cpuhp_step *sp;
int cpu;
int ret;
lockdep_assert_cpus_held();
sp = cpuhp_get_step(state);
if (sp->multi_instance == false)
return -EINVAL;
mutex_lock(&cpuhp_state_mutex);
if (!invoke || !sp->startup.multi)
goto add_node;
/*
* Try to call the startup callback for each present cpu
* depending on the hotplug state of the cpu.
*/
for_each_present_cpu(cpu) {
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int cpustate = st->state;
if (cpustate < state)
continue;
ret = cpuhp_issue_call(cpu, state, true, node);
if (ret) {
if (sp->teardown.multi)
cpuhp_rollback_install(cpu, state, node);
goto unlock;
}
}
add_node:
ret = 0;
hlist_add_head(node, &sp->list);
unlock:
mutex_unlock(&cpuhp_state_mutex);
return ret;
}
int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
bool invoke)
{
int ret;
cpus_read_lock();
ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
cpus_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
/**
* __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
* @state: The state to setup
* @name: Name of the step
* @invoke: If true, the startup function is invoked for cpus where
* cpu state >= @state
* @startup: startup callback function
* @teardown: teardown callback function
* @multi_instance: State is set up for multiple instances which get
* added afterwards.
*
* The caller needs to hold cpus read locked while calling this function.
* Return:
* On success:
* Positive state number if @state is CPUHP_AP_ONLINE_DYN;
* 0 for all other states
* On failure: proper (negative) error code
*/
int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
const char *name, bool invoke,
int (*startup)(unsigned int cpu),
int (*teardown)(unsigned int cpu),
bool multi_instance)
{
int cpu, ret = 0;
bool dynstate;
lockdep_assert_cpus_held();
if (cpuhp_cb_check(state) || !name)
return -EINVAL;
mutex_lock(&cpuhp_state_mutex);
ret = cpuhp_store_callbacks(state, name, startup, teardown,
multi_instance);
dynstate = state == CPUHP_AP_ONLINE_DYN;
if (ret > 0 && dynstate) {
state = ret;
ret = 0;
}
if (ret || !invoke || !startup)
goto out;
/*
* Try to call the startup callback for each present cpu
* depending on the hotplug state of the cpu.
*/
for_each_present_cpu(cpu) {
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int cpustate = st->state;
if (cpustate < state)
continue;
ret = cpuhp_issue_call(cpu, state, true, NULL);
if (ret) {
if (teardown)
cpuhp_rollback_install(cpu, state, NULL);
cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
goto out;
}
}
out:
mutex_unlock(&cpuhp_state_mutex);
/*
* If the requested state is CPUHP_AP_ONLINE_DYN, return the
* dynamically allocated state in case of success.
*/
if (!ret && dynstate)
return state;
return ret;
}
EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
int __cpuhp_setup_state(enum cpuhp_state state,
const char *name, bool invoke,
int (*startup)(unsigned int cpu),
int (*teardown)(unsigned int cpu),
bool multi_instance)
{
int ret;
cpus_read_lock();
ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
teardown, multi_instance);
cpus_read_unlock();
return ret;
}
EXPORT_SYMBOL(__cpuhp_setup_state);
int __cpuhp_state_remove_instance(enum cpuhp_state state,
struct hlist_node *node, bool invoke)
{
struct cpuhp_step *sp = cpuhp_get_step(state);
int cpu;
BUG_ON(cpuhp_cb_check(state));
if (!sp->multi_instance)
return -EINVAL;
cpus_read_lock();
mutex_lock(&cpuhp_state_mutex);
if (!invoke || !cpuhp_get_teardown_cb(state))
goto remove;
/*
* Call the teardown callback for each present cpu depending
* on the hotplug state of the cpu. This function is not
* allowed to fail currently!
*/
for_each_present_cpu(cpu) {
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int cpustate = st->state;
if (cpustate >= state)
cpuhp_issue_call(cpu, state, false, node);
}
remove:
hlist_del(node);
mutex_unlock(&cpuhp_state_mutex);
cpus_read_unlock();
return 0;
}
EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
/**
* __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
* @state: The state to remove
* @invoke: If true, the teardown function is invoked for cpus where
* cpu state >= @state
*
* The caller needs to hold cpus read locked while calling this function.
* The teardown callback is currently not allowed to fail. Think
* about module removal!
*/
void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
{
struct cpuhp_step *sp = cpuhp_get_step(state);
int cpu;
BUG_ON(cpuhp_cb_check(state));
lockdep_assert_cpus_held();
mutex_lock(&cpuhp_state_mutex);
if (sp->multi_instance) {
WARN(!hlist_empty(&sp->list),
"Error: Removing state %d which has instances left.\n",
state);
goto remove;
}
if (!invoke || !cpuhp_get_teardown_cb(state))
goto remove;
/*
* Call the teardown callback for each present cpu depending
* on the hotplug state of the cpu. This function is not
* allowed to fail currently!
*/
for_each_present_cpu(cpu) {
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
int cpustate = st->state;
if (cpustate >= state)
cpuhp_issue_call(cpu, state, false, NULL);
}
remove:
cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
mutex_unlock(&cpuhp_state_mutex);
}
EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
{
cpus_read_lock();
__cpuhp_remove_state_cpuslocked(state, invoke);
cpus_read_unlock();
}
EXPORT_SYMBOL(__cpuhp_remove_state);
#ifdef CONFIG_HOTPLUG_SMT
static void cpuhp_offline_cpu_device(unsigned int cpu)
{
struct device *dev = get_cpu_device(cpu);
dev->offline = true;
/* Tell user space about the state change */
kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
}
static void cpuhp_online_cpu_device(unsigned int cpu)
{
struct device *dev = get_cpu_device(cpu);
dev->offline = false;
/* Tell user space about the state change */
kobject_uevent(&dev->kobj, KOBJ_ONLINE);
}
int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
{
int cpu, ret = 0;
cpu_maps_update_begin();
for_each_online_cpu(cpu) {
if (topology_is_primary_thread(cpu))
continue;
/*
* Disable can be called with CPU_SMT_ENABLED when changing
* from a higher to lower number of SMT threads per core.
*/
if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
continue;
ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
if (ret)
break;
/*
* As this needs to hold the cpu maps lock it's impossible
* to call device_offline() because that ends up calling
* cpu_down() which takes cpu maps lock. cpu maps lock
* needs to be held as this might race against in kernel
* abusers of the hotplug machinery (thermal management).
*
* So nothing would update device:offline state. That would
* leave the sysfs entry stale and prevent onlining after
* smt control has been changed to 'off' again. This is
* called under the sysfs hotplug lock, so it is properly
* serialized against the regular offline usage.
*/
cpuhp_offline_cpu_device(cpu);
}
if (!ret)
cpu_smt_control = ctrlval;
cpu_maps_update_done();
return ret;
}
int cpuhp_smt_enable(void)
{
int cpu, ret = 0;
cpu_maps_update_begin();
cpu_smt_control = CPU_SMT_ENABLED;
for_each_present_cpu(cpu) {
/* Skip online CPUs and CPUs on offline nodes */
if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
continue;
if (!cpu_smt_thread_allowed(cpu))
continue;
ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
if (ret)
break;
/* See comment in cpuhp_smt_disable() */
cpuhp_online_cpu_device(cpu);
}
cpu_maps_update_done();
return ret;
}
#endif
#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
static ssize_t state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
return sprintf(buf, "%d\n", st->state);
}
static DEVICE_ATTR_RO(state);
static ssize_t target_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
struct cpuhp_step *sp;
int target, ret;
ret = kstrtoint(buf, 10, &target);
if (ret)
return ret;
#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
return -EINVAL;
#else
if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
return -EINVAL;
#endif
ret = lock_device_hotplug_sysfs();
if (ret)
return ret;
mutex_lock(&cpuhp_state_mutex);
sp = cpuhp_get_step(target);
ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
mutex_unlock(&cpuhp_state_mutex);
if (ret)
goto out;
if (st->state < target)
ret = cpu_up(dev->id, target);
else if (st->state > target)
ret = cpu_down(dev->id, target);
else if (WARN_ON(st->target != target))
st->target = target;
out:
unlock_device_hotplug();
return ret ? ret : count;
}
static ssize_t target_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
return sprintf(buf, "%d\n", st->target);
}
static DEVICE_ATTR_RW(target);
static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
struct cpuhp_step *sp;
int fail, ret;
ret = kstrtoint(buf, 10, &fail);
if (ret)
return ret;
if (fail == CPUHP_INVALID) {
st->fail = fail;
return count;
}
if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
return -EINVAL;
/*
* Cannot fail STARTING/DYING callbacks.
*/
if (cpuhp_is_atomic_state(fail))
return -EINVAL;
/*
* DEAD callbacks cannot fail...
* ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
* triggering STARTING callbacks, a failure in this state would
* hinder rollback.
*/
if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
return -EINVAL;
/*
* Cannot fail anything that doesn't have callbacks.
*/
mutex_lock(&cpuhp_state_mutex);
sp = cpuhp_get_step(fail);
if (!sp->startup.single && !sp->teardown.single)
ret = -EINVAL;
mutex_unlock(&cpuhp_state_mutex);
if (ret)
return ret;
st->fail = fail;
return count;
}
static ssize_t fail_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
return sprintf(buf, "%d\n", st->fail);
}
static DEVICE_ATTR_RW(fail);
static struct attribute *cpuhp_cpu_attrs[] = {
&dev_attr_state.attr,
&dev_attr_target.attr,
&dev_attr_fail.attr,
NULL
};
static const struct attribute_group cpuhp_cpu_attr_group = {
.attrs = cpuhp_cpu_attrs,
.name = "hotplug",
NULL
};
static ssize_t states_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
ssize_t cur, res = 0;
int i;
mutex_lock(&cpuhp_state_mutex);
for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
struct cpuhp_step *sp = cpuhp_get_step(i);
if (sp->name) {
cur = sprintf(buf, "%3d: %s\n", i, sp->name);
buf += cur;
res += cur;
}
}
mutex_unlock(&cpuhp_state_mutex);
return res;
}
static DEVICE_ATTR_RO(states);
static struct attribute *cpuhp_cpu_root_attrs[] = {
&dev_attr_states.attr,
NULL
};
static const struct attribute_group cpuhp_cpu_root_attr_group = {
.attrs = cpuhp_cpu_root_attrs,
.name = "hotplug",
NULL
};
#ifdef CONFIG_HOTPLUG_SMT
static bool cpu_smt_num_threads_valid(unsigned int threads)
{
if (IS_ENABLED(CONFIG_SMT_NUM_THREADS_DYNAMIC))
return threads >= 1 && threads <= cpu_smt_max_threads;
return threads == 1 || threads == cpu_smt_max_threads;
}
static ssize_t
__store_smt_control(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
int ctrlval, ret, num_threads, orig_threads;
bool force_off;
if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
return -EPERM;
if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
return -ENODEV;
if (sysfs_streq(buf, "on")) {
ctrlval = CPU_SMT_ENABLED;
num_threads = cpu_smt_max_threads;
} else if (sysfs_streq(buf, "off")) {
ctrlval = CPU_SMT_DISABLED;
num_threads = 1;
} else if (sysfs_streq(buf, "forceoff")) {
ctrlval = CPU_SMT_FORCE_DISABLED;
num_threads = 1;
} else if (kstrtoint(buf, 10, &num_threads) == 0) {
if (num_threads == 1)
ctrlval = CPU_SMT_DISABLED;
else if (cpu_smt_num_threads_valid(num_threads))
ctrlval = CPU_SMT_ENABLED;
else
return -EINVAL;
} else {
return -EINVAL;
}
ret = lock_device_hotplug_sysfs();
if (ret)
return ret;
orig_threads = cpu_smt_num_threads;
cpu_smt_num_threads = num_threads;
force_off = ctrlval != cpu_smt_control && ctrlval == CPU_SMT_FORCE_DISABLED;
if (num_threads > orig_threads)
ret = cpuhp_smt_enable();
else if (num_threads < orig_threads || force_off)
ret = cpuhp_smt_disable(ctrlval);
unlock_device_hotplug();
return ret ? ret : count;
}
#else /* !CONFIG_HOTPLUG_SMT */
static ssize_t
__store_smt_control(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
return -ENODEV;
}
#endif /* CONFIG_HOTPLUG_SMT */
static const char *smt_states[] = {
[CPU_SMT_ENABLED] = "on",
[CPU_SMT_DISABLED] = "off",
[CPU_SMT_FORCE_DISABLED] = "forceoff",
[CPU_SMT_NOT_SUPPORTED] = "notsupported",
[CPU_SMT_NOT_IMPLEMENTED] = "notimplemented",
};
static ssize_t control_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
const char *state = smt_states[cpu_smt_control];
#ifdef CONFIG_HOTPLUG_SMT
/*
* If SMT is enabled but not all threads are enabled then show the
* number of threads. If all threads are enabled show "on". Otherwise
* show the state name.
*/
if (cpu_smt_control == CPU_SMT_ENABLED &&
cpu_smt_num_threads != cpu_smt_max_threads)
return sysfs_emit(buf, "%d\n", cpu_smt_num_threads);
#endif
return sysfs_emit(buf, "%s\n", state);
}
static ssize_t control_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
return __store_smt_control(dev, attr, buf, count);
}
static DEVICE_ATTR_RW(control);
static ssize_t active_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
return sysfs_emit(buf, "%d\n", sched_smt_active());
}
static DEVICE_ATTR_RO(active);
static struct attribute *cpuhp_smt_attrs[] = {
&dev_attr_control.attr,
&dev_attr_active.attr,
NULL
};
static const struct attribute_group cpuhp_smt_attr_group = {
.attrs = cpuhp_smt_attrs,
.name = "smt",
NULL
};
static int __init cpu_smt_sysfs_init(void)
{
struct device *dev_root;
int ret = -ENODEV;
dev_root = bus_get_dev_root(&cpu_subsys);
if (dev_root) {
ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group);
put_device(dev_root);
}
return ret;
}
static int __init cpuhp_sysfs_init(void)
{
struct device *dev_root;
int cpu, ret;
ret = cpu_smt_sysfs_init();
if (ret)
return ret;
dev_root = bus_get_dev_root(&cpu_subsys);
if (dev_root) {
ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group);
put_device(dev_root);
if (ret)
return ret;
}
for_each_possible_cpu(cpu) {
struct device *dev = get_cpu_device(cpu);
if (!dev)
continue;
ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
if (ret)
return ret;
}
return 0;
}
device_initcall(cpuhp_sysfs_init);
#endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
/*
* cpu_bit_bitmap[] is a special, "compressed" data structure that
* represents all NR_CPUS bits binary values of 1<<nr.
*
* It is used by cpumask_of() to get a constant address to a CPU
* mask value that has a single bit set only.
*/
/* cpu_bit_bitmap[0] is empty - so we can back into it */
#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
MASK_DECLARE_8(0), MASK_DECLARE_8(8),
MASK_DECLARE_8(16), MASK_DECLARE_8(24),
#if BITS_PER_LONG > 32
MASK_DECLARE_8(32), MASK_DECLARE_8(40),
MASK_DECLARE_8(48), MASK_DECLARE_8(56),
#endif
};
EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
#else
struct cpumask __cpu_possible_mask __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
struct cpumask __cpu_present_mask __read_mostly;
EXPORT_SYMBOL(__cpu_present_mask);
struct cpumask __cpu_active_mask __read_mostly;
EXPORT_SYMBOL(__cpu_active_mask);
struct cpumask __cpu_dying_mask __read_mostly;
EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
}
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
}
void init_cpu_online(const struct cpumask *src)
{
cpumask_copy(&__cpu_online_mask, src);
}
void set_cpu_online(unsigned int cpu, bool online)
{
/*
* atomic_inc/dec() is required to handle the horrid abuse of this
* function by the reboot and kexec code which invoke it from
* IPI/NMI broadcasts when shutting down CPUs. Invocation from
* regular CPU hotplug is properly serialized.
*
* Note, that the fact that __num_online_cpus is of type atomic_t
* does not protect readers which are not serialized against
* concurrent hotplug operations.
*/
if (online) {
if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
atomic_inc(&__num_online_cpus);
} else {
if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
atomic_dec(&__num_online_cpus);
}
}
/*
* Activate the first processor.
*/
void __init boot_cpu_init(void)
{
int cpu = smp_processor_id();
/* Mark the boot cpu "present", "online" etc for SMP and UP case */
set_cpu_online(cpu, true);
set_cpu_active(cpu, true);
set_cpu_present(cpu, true);
set_cpu_possible(cpu, true);
#ifdef CONFIG_SMP
__boot_cpu_id = cpu;
#endif
}
/*
* Must be called _AFTER_ setting up the per_cpu areas
*/
void __init boot_cpu_hotplug_init(void)
{
#ifdef CONFIG_SMP
cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
atomic_set(this_cpu_ptr(&cpuhp_state.ap_sync_state), SYNC_STATE_ONLINE);
#endif
this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
this_cpu_write(cpuhp_state.target, CPUHP_ONLINE);
}
#ifdef CONFIG_CPU_MITIGATIONS
/*
* These are used for a global "mitigations=" cmdline option for toggling
* optional CPU mitigations.
*/
enum cpu_mitigations {
CPU_MITIGATIONS_OFF,
CPU_MITIGATIONS_AUTO,
CPU_MITIGATIONS_AUTO_NOSMT,
};
static enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO;
static int __init mitigations_parse_cmdline(char *arg)
{
if (!strcmp(arg, "off"))
cpu_mitigations = CPU_MITIGATIONS_OFF;
else if (!strcmp(arg, "auto"))
cpu_mitigations = CPU_MITIGATIONS_AUTO;
else if (!strcmp(arg, "auto,nosmt"))
cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
else
pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
arg);
return 0;
}
/* mitigations=off */
bool cpu_mitigations_off(void)
{
return cpu_mitigations == CPU_MITIGATIONS_OFF;
}
EXPORT_SYMBOL_GPL(cpu_mitigations_off);
/* mitigations=auto,nosmt */
bool cpu_mitigations_auto_nosmt(void)
{
return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
}
EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
#else
static int __init mitigations_parse_cmdline(char *arg)
{
pr_crit("Kernel compiled without mitigations, ignoring 'mitigations'; system may still be vulnerable\n");
return 0;
}
#endif
early_param("mitigations", mitigations_parse_cmdline);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Intel SMP support routines.
*
* (c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
* (c) 1998-99, 2000, 2009 Ingo Molnar <mingo@redhat.com>
* (c) 2002,2003 Andi Kleen, SuSE Labs.
*
* i386 and x86_64 integration by Glauber Costa <gcosta@redhat.com>
*/
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/mc146818rtc.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/cpu.h>
#include <linux/gfp.h>
#include <linux/kexec.h>
#include <asm/mtrr.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/proto.h>
#include <asm/apic.h>
#include <asm/cpu.h>
#include <asm/idtentry.h>
#include <asm/nmi.h>
#include <asm/mce.h>
#include <asm/trace/irq_vectors.h>
#include <asm/kexec.h>
#include <asm/reboot.h>
/*
* Some notes on x86 processor bugs affecting SMP operation:
*
* Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
* The Linux implications for SMP are handled as follows:
*
* Pentium III / [Xeon]
* None of the E1AP-E3AP errata are visible to the user.
*
* E1AP. see PII A1AP
* E2AP. see PII A2AP
* E3AP. see PII A3AP
*
* Pentium II / [Xeon]
* None of the A1AP-A3AP errata are visible to the user.
*
* A1AP. see PPro 1AP
* A2AP. see PPro 2AP
* A3AP. see PPro 7AP
*
* Pentium Pro
* None of 1AP-9AP errata are visible to the normal user,
* except occasional delivery of 'spurious interrupt' as trap #15.
* This is very rare and a non-problem.
*
* 1AP. Linux maps APIC as non-cacheable
* 2AP. worked around in hardware
* 3AP. fixed in C0 and above steppings microcode update.
* Linux does not use excessive STARTUP_IPIs.
* 4AP. worked around in hardware
* 5AP. symmetric IO mode (normal Linux operation) not affected.
* 'noapic' mode has vector 0xf filled out properly.
* 6AP. 'noapic' mode might be affected - fixed in later steppings
* 7AP. We do not assume writes to the LVT deasserting IRQs
* 8AP. We do not enable low power mode (deep sleep) during MP bootup
* 9AP. We do not use mixed mode
*
* Pentium
* There is a marginal case where REP MOVS on 100MHz SMP
* machines with B stepping processors can fail. XXX should provide
* an L1cache=Writethrough or L1cache=off option.
*
* B stepping CPUs may hang. There are hardware work arounds
* for this. We warn about it in case your board doesn't have the work
* arounds. Basically that's so I can tell anyone with a B stepping
* CPU and SMP problems "tough".
*
* Specific items [From Pentium Processor Specification Update]
*
* 1AP. Linux doesn't use remote read
* 2AP. Linux doesn't trust APIC errors
* 3AP. We work around this
* 4AP. Linux never generated 3 interrupts of the same priority
* to cause a lost local interrupt.
* 5AP. Remote read is never used
* 6AP. not affected - worked around in hardware
* 7AP. not affected - worked around in hardware
* 8AP. worked around in hardware - we get explicit CS errors if not
* 9AP. only 'noapic' mode affected. Might generate spurious
* interrupts, we log only the first one and count the
* rest silently.
* 10AP. not affected - worked around in hardware
* 11AP. Linux reads the APIC between writes to avoid this, as per
* the documentation. Make sure you preserve this as it affects
* the C stepping chips too.
* 12AP. not affected - worked around in hardware
* 13AP. not affected - worked around in hardware
* 14AP. we always deassert INIT during bootup
* 15AP. not affected - worked around in hardware
* 16AP. not affected - worked around in hardware
* 17AP. not affected - worked around in hardware
* 18AP. not affected - worked around in hardware
* 19AP. not affected - worked around in BIOS
*
* If this sounds worrying believe me these bugs are either ___RARE___,
* or are signal timing bugs worked around in hardware and there's
* about nothing of note with C stepping upwards.
*/
static atomic_t stopping_cpu = ATOMIC_INIT(-1);
static bool smp_no_nmi_ipi = false;
static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs)
{
/* We are registered on stopping cpu too, avoid spurious NMI */
if (raw_smp_processor_id() == atomic_read(&stopping_cpu))
return NMI_HANDLED;
cpu_emergency_disable_virtualization();
stop_this_cpu(NULL);
return NMI_HANDLED;
}
/*
* this function calls the 'stop' function on all other CPUs in the system.
*/
DEFINE_IDTENTRY_SYSVEC(sysvec_reboot)
{
apic_eoi();
cpu_emergency_disable_virtualization();
stop_this_cpu(NULL);
}
static int register_stop_handler(void)
{
return register_nmi_handler(NMI_LOCAL, smp_stop_nmi_callback,
NMI_FLAG_FIRST, "smp_stop");
}
static void native_stop_other_cpus(int wait)
{
unsigned int old_cpu, this_cpu;
unsigned long flags, timeout;
if (reboot_force)
return;
/* Only proceed if this is the first CPU to reach this code */
old_cpu = -1;
this_cpu = smp_processor_id();
if (!atomic_try_cmpxchg(&stopping_cpu, &old_cpu, this_cpu))
return;
/* For kexec, ensure that offline CPUs are out of MWAIT and in HLT */
if (kexec_in_progress)
smp_kick_mwait_play_dead();
/*
* 1) Send an IPI on the reboot vector to all other CPUs.
*
* The other CPUs should react on it after leaving critical
* sections and re-enabling interrupts. They might still hold
* locks, but there is nothing which can be done about that.
*
* 2) Wait for all other CPUs to report that they reached the
* HLT loop in stop_this_cpu()
*
* 3) If #2 timed out send an NMI to the CPUs which did not
* yet report
*
* 4) Wait for all other CPUs to report that they reached the
* HLT loop in stop_this_cpu()
*
* #3 can obviously race against a CPU reaching the HLT loop late.
* That CPU will have reported already and the "have all CPUs
* reached HLT" condition will be true despite the fact that the
* other CPU is still handling the NMI. Again, there is no
* protection against that as "disabled" APICs still respond to
* NMIs.
*/
cpumask_copy(&cpus_stop_mask, cpu_online_mask);
cpumask_clear_cpu(this_cpu, &cpus_stop_mask);
if (!cpumask_empty(&cpus_stop_mask)) {
apic_send_IPI_allbutself(REBOOT_VECTOR);
/*
* Don't wait longer than a second for IPI completion. The
* wait request is not checked here because that would
* prevent an NMI shutdown attempt in case that not all
* CPUs reach shutdown state.
*/
timeout = USEC_PER_SEC;
while (!cpumask_empty(&cpus_stop_mask) && timeout--)
udelay(1);
}
/* if the REBOOT_VECTOR didn't work, try with the NMI */
if (!cpumask_empty(&cpus_stop_mask)) {
/*
* If NMI IPI is enabled, try to register the stop handler
* and send the IPI. In any case try to wait for the other
* CPUs to stop.
*/
if (!smp_no_nmi_ipi && !register_stop_handler()) {
unsigned int cpu;
pr_emerg("Shutting down cpus with NMI\n");
for_each_cpu(cpu, &cpus_stop_mask)
__apic_send_IPI(cpu, NMI_VECTOR);
}
/*
* Don't wait longer than 10 ms if the caller didn't
* request it. If wait is true, the machine hangs here if
* one or more CPUs do not reach shutdown state.
*/
timeout = USEC_PER_MSEC * 10;
while (!cpumask_empty(&cpus_stop_mask) && (wait || timeout--))
udelay(1);
}
local_irq_save(flags);
disable_local_APIC();
mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
local_irq_restore(flags);
/*
* Ensure that the cpus_stop_mask cache lines are invalidated on
* the other CPUs. See comment vs. SME in stop_this_cpu().
*/
cpumask_clear(&cpus_stop_mask);
}
/*
* Reschedule call back. KVM uses this interrupt to force a cpu out of
* guest mode.
*/
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
{
apic_eoi();
trace_reschedule_entry(RESCHEDULE_VECTOR);
inc_irq_stat(irq_resched_count);
scheduler_ipi();
trace_reschedule_exit(RESCHEDULE_VECTOR);
}
DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
{
apic_eoi();
trace_call_function_entry(CALL_FUNCTION_VECTOR);
inc_irq_stat(irq_call_count);
generic_smp_call_function_interrupt();
trace_call_function_exit(CALL_FUNCTION_VECTOR);
}
DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single)
{
apic_eoi(); trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR); inc_irq_stat(irq_call_count);
generic_smp_call_function_single_interrupt();
trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR);
}
static int __init nonmi_ipi_setup(char *str)
{
smp_no_nmi_ipi = true;
return 1;
}
__setup("nonmi_ipi", nonmi_ipi_setup);
struct smp_ops smp_ops = {
.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
.smp_prepare_cpus = native_smp_prepare_cpus,
.smp_cpus_done = native_smp_cpus_done,
.stop_other_cpus = native_stop_other_cpus,
#if defined(CONFIG_CRASH_DUMP)
.crash_stop_other_cpus = kdump_nmi_shootdown_cpus,
#endif
.smp_send_reschedule = native_smp_send_reschedule,
.kick_ap_alive = native_kick_ap,
.cpu_disable = native_cpu_disable,
.play_dead = native_play_dead,
.send_call_func_ipi = native_send_call_func_ipi,
.send_call_func_single_ipi = native_send_call_func_single_ipi,
};
EXPORT_SYMBOL_GPL(smp_ops);
// SPDX-License-Identifier: GPL-2.0
/*
* linux/fs/attr.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
* changes by Thomas Schoebel-Theuer
*/
#include <linux/export.h>
#include <linux/time.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/sched/signal.h>
#include <linux/capability.h>
#include <linux/fsnotify.h>
#include <linux/fcntl.h>
#include <linux/filelock.h>
#include <linux/security.h>
#include "internal.h"
/**
* setattr_should_drop_sgid - determine whether the setgid bit needs to be
* removed
* @idmap: idmap of the mount @inode was found from
* @inode: inode to check
*
* This function determines whether the setgid bit needs to be removed.
* We retain backwards compatibility and require setgid bit to be removed
* unconditionally if S_IXGRP is set. Otherwise we have the exact same
* requirements as setattr_prepare() and setattr_copy().
*
* Return: ATTR_KILL_SGID if setgid bit needs to be removed, 0 otherwise.
*/
int setattr_should_drop_sgid(struct mnt_idmap *idmap,
const struct inode *inode)
{
umode_t mode = inode->i_mode;
if (!(mode & S_ISGID))
return 0;
if (mode & S_IXGRP)
return ATTR_KILL_SGID;
if (!in_group_or_capable(idmap, inode, i_gid_into_vfsgid(idmap, inode)))
return ATTR_KILL_SGID;
return 0;
}
EXPORT_SYMBOL(setattr_should_drop_sgid);
/**
* setattr_should_drop_suidgid - determine whether the set{g,u}id bit needs to
* be dropped
* @idmap: idmap of the mount @inode was found from
* @inode: inode to check
*
* This function determines whether the set{g,u}id bits need to be removed.
* If the setuid bit needs to be removed ATTR_KILL_SUID is returned. If the
* setgid bit needs to be removed ATTR_KILL_SGID is returned. If both
* set{g,u}id bits need to be removed the corresponding mask of both flags is
* returned.
*
* Return: A mask of ATTR_KILL_S{G,U}ID indicating which - if any - setid bits
* to remove, 0 otherwise.
*/
int setattr_should_drop_suidgid(struct mnt_idmap *idmap,
struct inode *inode)
{
umode_t mode = inode->i_mode;
int kill = 0;
/* suid always must be killed */
if (unlikely(mode & S_ISUID))
kill = ATTR_KILL_SUID;
kill |= setattr_should_drop_sgid(idmap, inode); if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
return kill;
return 0;
}
EXPORT_SYMBOL(setattr_should_drop_suidgid);
/**
* chown_ok - verify permissions to chown inode
* @idmap: idmap of the mount @inode was found from
* @inode: inode to check permissions on
* @ia_vfsuid: uid to chown @inode to
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*/
static bool chown_ok(struct mnt_idmap *idmap,
const struct inode *inode, vfsuid_t ia_vfsuid)
{
vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
if (vfsuid_eq_kuid(vfsuid, current_fsuid()) && vfsuid_eq(ia_vfsuid, vfsuid))
return true;
if (capable_wrt_inode_uidgid(idmap, inode, CAP_CHOWN))
return true;
if (!vfsuid_valid(vfsuid) &&
ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN))
return true;
return false;
}
/**
* chgrp_ok - verify permissions to chgrp inode
* @idmap: idmap of the mount @inode was found from
* @inode: inode to check permissions on
* @ia_vfsgid: gid to chown @inode to
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*/
static bool chgrp_ok(struct mnt_idmap *idmap,
const struct inode *inode, vfsgid_t ia_vfsgid)
{
vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);
vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
if (vfsuid_eq_kuid(vfsuid, current_fsuid())) { if (vfsgid_eq(ia_vfsgid, vfsgid))
return true;
if (vfsgid_in_group_p(ia_vfsgid))
return true;
}
if (capable_wrt_inode_uidgid(idmap, inode, CAP_CHOWN))
return true;
if (!vfsgid_valid(vfsgid) && ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN))
return true;
return false;
}
/**
* setattr_prepare - check if attribute changes to a dentry are allowed
* @idmap: idmap of the mount the inode was found from
* @dentry: dentry to check
* @attr: attributes to change
*
* Check if we are allowed to change the attributes contained in @attr
* in the given dentry. This includes the normal unix access permission
* checks, as well as checks for rlimits and others. The function also clears
* SGID bit from mode if user is not allowed to set it. Also file capabilities
* and IMA extended attributes are cleared if ATTR_KILL_PRIV is set.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*
* Should be called as the first thing in ->setattr implementations,
* possibly after taking additional locks.
*/
int setattr_prepare(struct mnt_idmap *idmap, struct dentry *dentry,
struct iattr *attr)
{
struct inode *inode = d_inode(dentry);
unsigned int ia_valid = attr->ia_valid;
/*
* First check size constraints. These can't be overriden using
* ATTR_FORCE.
*/
if (ia_valid & ATTR_SIZE) {
int error = inode_newsize_ok(inode, attr->ia_size);
if (error)
return error;
}
/* If force is set do it anyway. */
if (ia_valid & ATTR_FORCE)
goto kill_priv;
/* Make sure a caller can chown. */
if ((ia_valid & ATTR_UID) && !chown_ok(idmap, inode, attr->ia_vfsuid)) return -EPERM;
/* Make sure caller can chgrp. */
if ((ia_valid & ATTR_GID) && !chgrp_ok(idmap, inode, attr->ia_vfsgid))
return -EPERM;
/* Make sure a caller can chmod. */
if (ia_valid & ATTR_MODE) {
vfsgid_t vfsgid;
if (!inode_owner_or_capable(idmap, inode)) return -EPERM; if (ia_valid & ATTR_GID) vfsgid = attr->ia_vfsgid;
else
vfsgid = i_gid_into_vfsgid(idmap, inode);
/* Also check the setgid bit! */
if (!in_group_or_capable(idmap, inode, vfsgid)) attr->ia_mode &= ~S_ISGID;
}
/* Check for setting the inode time. */
if (ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) { if (!inode_owner_or_capable(idmap, inode))
return -EPERM;
}
kill_priv:
/* User has permission for the change */
if (ia_valid & ATTR_KILL_PRIV) {
int error;
error = security_inode_killpriv(idmap, dentry); if (error)
return error;
}
return 0;
}
EXPORT_SYMBOL(setattr_prepare);
/**
* inode_newsize_ok - may this inode be truncated to a given size
* @inode: the inode to be truncated
* @offset: the new size to assign to the inode
*
* inode_newsize_ok must be called with i_mutex held.
*
* inode_newsize_ok will check filesystem limits and ulimits to check that the
* new inode size is within limits. inode_newsize_ok will also send SIGXFSZ
* when necessary. Caller must not proceed with inode size change if failure is
* returned. @inode must be a file (not directory), with appropriate
* permissions to allow truncate (inode_newsize_ok does NOT check these
* conditions).
*
* Return: 0 on success, -ve errno on failure
*/
int inode_newsize_ok(const struct inode *inode, loff_t offset)
{
if (offset < 0)
return -EINVAL;
if (inode->i_size < offset) {
unsigned long limit;
limit = rlimit(RLIMIT_FSIZE);
if (limit != RLIM_INFINITY && offset > limit)
goto out_sig;
if (offset > inode->i_sb->s_maxbytes)
goto out_big;
} else {
/*
* truncation of in-use swapfiles is disallowed - it would
* cause subsequent swapout to scribble on the now-freed
* blocks.
*/
if (IS_SWAPFILE(inode))
return -ETXTBSY;
}
return 0;
out_sig:
send_sig(SIGXFSZ, current, 0);
out_big:
return -EFBIG;
}
EXPORT_SYMBOL(inode_newsize_ok);
/**
* setattr_copy - copy simple metadata updates into the generic inode
* @idmap: idmap of the mount the inode was found from
* @inode: the inode to be updated
* @attr: the new attributes
*
* setattr_copy must be called with i_mutex held.
*
* setattr_copy updates the inode's metadata with that specified
* in attr on idmapped mounts. Necessary permission checks to determine
* whether or not the S_ISGID property needs to be removed are performed with
* the correct idmapped mount permission helpers.
* Noticeably missing is inode size update, which is more complex
* as it requires pagecache updates.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*
* The inode is not marked as dirty after this operation. The rationale is
* that for "simple" filesystems, the struct inode is the inode storage.
* The caller is free to mark the inode dirty afterwards if needed.
*/
void setattr_copy(struct mnt_idmap *idmap, struct inode *inode,
const struct iattr *attr)
{
unsigned int ia_valid = attr->ia_valid; i_uid_update(idmap, attr, inode); i_gid_update(idmap, attr, inode); if (ia_valid & ATTR_ATIME) inode_set_atime_to_ts(inode, attr->ia_atime); if (ia_valid & ATTR_MTIME) inode_set_mtime_to_ts(inode, attr->ia_mtime); if (ia_valid & ATTR_CTIME) inode_set_ctime_to_ts(inode, attr->ia_ctime); if (ia_valid & ATTR_MODE) { umode_t mode = attr->ia_mode;
if (!in_group_or_capable(idmap, inode,
i_gid_into_vfsgid(idmap, inode)))
mode &= ~S_ISGID; inode->i_mode = mode;
}
}
EXPORT_SYMBOL(setattr_copy);
int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
unsigned int ia_valid)
{
int error;
if (ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID | ATTR_TIMES_SET)) { if (IS_IMMUTABLE(inode) || IS_APPEND(inode))
return -EPERM;
}
/*
* If utimes(2) and friends are called with times == NULL (or both
* times are UTIME_NOW), then we need to check for write permission
*/
if (ia_valid & ATTR_TOUCH) { if (IS_IMMUTABLE(inode))
return -EPERM;
if (!inode_owner_or_capable(idmap, inode)) { error = inode_permission(idmap, inode, MAY_WRITE); if (error)
return error;
}
}
return 0;
}
EXPORT_SYMBOL(may_setattr);
/**
* notify_change - modify attributes of a filesystem object
* @idmap: idmap of the mount the inode was found from
* @dentry: object affected
* @attr: new attributes
* @delegated_inode: returns inode, if the inode is delegated
*
* The caller must hold the i_mutex on the affected object.
*
* If notify_change discovers a delegation in need of breaking,
* it will return -EWOULDBLOCK and return a reference to the inode in
* delegated_inode. The caller should then break the delegation and
* retry. Because breaking a delegation may take a long time, the
* caller should drop the i_mutex before doing so.
*
* Alternatively, a caller may pass NULL for delegated_inode. This may
* be appropriate for callers that expect the underlying filesystem not
* to be NFS exported. Also, passing NULL is fine for callers holding
* the file open for write, as there can be no conflicting delegation in
* that case.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then
* take care to map the inode according to @idmap before checking
* permissions. On non-idmapped mounts or if permission checking is to be
* performed on the raw inode simply pass @nop_mnt_idmap.
*/
int notify_change(struct mnt_idmap *idmap, struct dentry *dentry,
struct iattr *attr, struct inode **delegated_inode)
{
struct inode *inode = dentry->d_inode;
umode_t mode = inode->i_mode;
int error;
struct timespec64 now;
unsigned int ia_valid = attr->ia_valid;
WARN_ON_ONCE(!inode_is_locked(inode)); error = may_setattr(idmap, inode, ia_valid);
if (error)
return error;
if ((ia_valid & ATTR_MODE)) {
/*
* Don't allow changing the mode of symlinks:
*
* (1) The vfs doesn't take the mode of symlinks into account
* during permission checking.
* (2) This has never worked correctly. Most major filesystems
* did return EOPNOTSUPP due to interactions with POSIX ACLs
* but did still updated the mode of the symlink.
* This inconsistency led system call wrapper providers such
* as libc to block changing the mode of symlinks with
* EOPNOTSUPP already.
* (3) To even do this in the first place one would have to use
* specific file descriptors and quite some effort.
*/
if (S_ISLNK(inode->i_mode))
return -EOPNOTSUPP;
/* Flag setting protected by i_mutex */
if (is_sxid(attr->ia_mode)) inode->i_flags &= ~S_NOSEC;
}
now = current_time(inode);
attr->ia_ctime = now;
if (!(ia_valid & ATTR_ATIME_SET))
attr->ia_atime = now;
else
attr->ia_atime = timestamp_truncate(attr->ia_atime, inode); if (!(ia_valid & ATTR_MTIME_SET)) attr->ia_mtime = now;
else
attr->ia_mtime = timestamp_truncate(attr->ia_mtime, inode); if (ia_valid & ATTR_KILL_PRIV) { error = security_inode_need_killpriv(dentry);
if (error < 0)
return error;
if (error == 0) ia_valid = attr->ia_valid &= ~ATTR_KILL_PRIV;
}
/*
* We now pass ATTR_KILL_S*ID to the lower level setattr function so
* that the function has the ability to reinterpret a mode change
* that's due to these bits. This adds an implicit restriction that
* no function will ever call notify_change with both ATTR_MODE and
* ATTR_KILL_S*ID set.
*/
if ((ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) && (ia_valid & ATTR_MODE)) BUG(); if (ia_valid & ATTR_KILL_SUID) { if (mode & S_ISUID) { ia_valid = attr->ia_valid |= ATTR_MODE;
attr->ia_mode = (inode->i_mode & ~S_ISUID);
}
}
if (ia_valid & ATTR_KILL_SGID) { if (mode & S_ISGID) { if (!(ia_valid & ATTR_MODE)) { ia_valid = attr->ia_valid |= ATTR_MODE;
attr->ia_mode = inode->i_mode;
}
attr->ia_mode &= ~S_ISGID;
}
}
if (!(attr->ia_valid & ~(ATTR_KILL_SUID | ATTR_KILL_SGID)))
return 0;
/*
* Verify that uid/gid changes are valid in the target
* namespace of the superblock.
*/
if (ia_valid & ATTR_UID &&
!vfsuid_has_fsmapping(idmap, inode->i_sb->s_user_ns,
attr->ia_vfsuid))
return -EOVERFLOW;
if (ia_valid & ATTR_GID &&
!vfsgid_has_fsmapping(idmap, inode->i_sb->s_user_ns,
attr->ia_vfsgid))
return -EOVERFLOW;
/* Don't allow modifications of files with invalid uids or
* gids unless those uids & gids are being made valid.
*/
if (!(ia_valid & ATTR_UID) &&
!vfsuid_valid(i_uid_into_vfsuid(idmap, inode)))
return -EOVERFLOW;
if (!(ia_valid & ATTR_GID) &&
!vfsgid_valid(i_gid_into_vfsgid(idmap, inode)))
return -EOVERFLOW;
error = security_inode_setattr(idmap, dentry, attr);
if (error)
return error;
error = try_break_deleg(inode, delegated_inode); if (error)
return error;
if (inode->i_op->setattr) error = inode->i_op->setattr(idmap, dentry, attr);
else
error = simple_setattr(idmap, dentry, attr); if (!error) { fsnotify_change(dentry, ia_valid); security_inode_post_setattr(idmap, dentry, ia_valid);
}
return error;
}
EXPORT_SYMBOL(notify_change);
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM vmalloc
#if !defined(_TRACE_VMALLOC_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_VMALLOC_H
#include <linux/tracepoint.h>
/**
* alloc_vmap_area - called when a new vmap allocation occurs
* @addr: an allocated address
* @size: a requested size
* @align: a requested alignment
* @vstart: a requested start range
* @vend: a requested end range
* @failed: an allocation failed or not
*
* This event is used for a debug purpose, it can give an extra
* information for a developer about how often it occurs and which
* parameters are passed for further validation.
*/
TRACE_EVENT(alloc_vmap_area,
TP_PROTO(unsigned long addr, unsigned long size, unsigned long align,
unsigned long vstart, unsigned long vend, int failed),
TP_ARGS(addr, size, align, vstart, vend, failed),
TP_STRUCT__entry(
__field(unsigned long, addr)
__field(unsigned long, size)
__field(unsigned long, align)
__field(unsigned long, vstart)
__field(unsigned long, vend)
__field(int, failed)
),
TP_fast_assign(
__entry->addr = addr;
__entry->size = size;
__entry->align = align;
__entry->vstart = vstart;
__entry->vend = vend;
__entry->failed = failed;
),
TP_printk("va_start: %lu size=%lu align=%lu vstart=0x%lx vend=0x%lx failed=%d",
__entry->addr, __entry->size, __entry->align,
__entry->vstart, __entry->vend, __entry->failed)
);
/**
* purge_vmap_area_lazy - called when vmap areas were lazily freed
* @start: purging start address
* @end: purging end address
* @npurged: numbed of purged vmap areas
*
* This event is used for a debug purpose. It gives some
* indication about start:end range and how many objects
* are released.
*/
TRACE_EVENT(purge_vmap_area_lazy,
TP_PROTO(unsigned long start, unsigned long end,
unsigned int npurged),
TP_ARGS(start, end, npurged),
TP_STRUCT__entry(
__field(unsigned long, start)
__field(unsigned long, end)
__field(unsigned int, npurged)
),
TP_fast_assign(
__entry->start = start;
__entry->end = end;
__entry->npurged = npurged;
),
TP_printk("start=0x%lx end=0x%lx num_purged=%u",
__entry->start, __entry->end, __entry->npurged)
);
/**
* free_vmap_area_noflush - called when a vmap area is freed
* @va_start: a start address of VA
* @nr_lazy: number of current lazy pages
* @nr_lazy_max: number of maximum lazy pages
*
* This event is used for a debug purpose. It gives some
* indication about a VA that is released, number of current
* outstanding areas and a maximum allowed threshold before
* dropping all of them.
*/
TRACE_EVENT(free_vmap_area_noflush,
TP_PROTO(unsigned long va_start, unsigned long nr_lazy,
unsigned long nr_lazy_max),
TP_ARGS(va_start, nr_lazy, nr_lazy_max),
TP_STRUCT__entry(
__field(unsigned long, va_start)
__field(unsigned long, nr_lazy)
__field(unsigned long, nr_lazy_max)
),
TP_fast_assign(
__entry->va_start = va_start;
__entry->nr_lazy = nr_lazy;
__entry->nr_lazy_max = nr_lazy_max;
),
TP_printk("va_start=0x%lx nr_lazy=%lu nr_lazy_max=%lu",
__entry->va_start, __entry->nr_lazy, __entry->nr_lazy_max)
);
#endif /* _TRACE_VMALLOC_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
/* SPDX-License-Identifier: GPL-2.0+ */
/*
* Sleepable Read-Copy Update mechanism for mutual exclusion
*
* Copyright (C) IBM Corporation, 2006
* Copyright (C) Fujitsu, 2012
*
* Author: Paul McKenney <paulmck@linux.ibm.com>
* Lai Jiangshan <laijs@cn.fujitsu.com>
*
* For detailed explanation of Read-Copy Update mechanism see -
* Documentation/RCU/ *.txt
*
*/
#ifndef _LINUX_SRCU_H
#define _LINUX_SRCU_H
#include <linux/mutex.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/rcu_segcblist.h>
struct srcu_struct;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
struct lock_class_key *key);
#define init_srcu_struct(ssp) \
({ \
static struct lock_class_key __srcu_key; \
\
__init_srcu_struct((ssp), #ssp, &__srcu_key); \
})
#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
int init_srcu_struct(struct srcu_struct *ssp);
#define __SRCU_DEP_MAP_INIT(srcu_name)
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
#ifdef CONFIG_TINY_SRCU
#include <linux/srcutiny.h>
#elif defined(CONFIG_TREE_SRCU)
#include <linux/srcutree.h>
#else
#error "Unknown SRCU implementation specified to kernel configuration"
#endif
void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
void (*func)(struct rcu_head *head));
void cleanup_srcu_struct(struct srcu_struct *ssp);
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
void synchronize_srcu(struct srcu_struct *ssp);
unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
#ifdef CONFIG_NEED_SRCU_NMI_SAFE
int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);
#else
static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
{
return __srcu_read_lock(ssp);
}
static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
{
__srcu_read_unlock(ssp, idx);
}
#endif /* CONFIG_NEED_SRCU_NMI_SAFE */
void srcu_init(void);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
/**
* srcu_read_lock_held - might we be in SRCU read-side critical section?
* @ssp: The srcu_struct structure to check
*
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
* read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
* this assumes we are in an SRCU read-side critical section unless it can
* prove otherwise.
*
* Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
* and while lockdep is disabled.
*
* Note that SRCU is based on its own statemachine and it doesn't
* relies on normal RCU, it can be called from the CPU which
* is in the idle loop from an RCU point of view or offline.
*/
static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
if (!debug_lockdep_rcu_enabled())
return 1;
return lock_is_held(&ssp->dep_map);
}
/*
* Annotations provide deadlock detection for SRCU.
*
* Similar to other lockdep annotations, except there is an additional
* srcu_lock_sync(), which is basically an empty *write*-side critical section,
* see lock_sync() for more information.
*/
/* Annotates a srcu_read_lock() */
static inline void srcu_lock_acquire(struct lockdep_map *map)
{
lock_map_acquire_read(map);
}
/* Annotates a srcu_read_lock() */
static inline void srcu_lock_release(struct lockdep_map *map)
{
lock_map_release(map);
}
/* Annotates a synchronize_srcu() */
static inline void srcu_lock_sync(struct lockdep_map *map)
{
lock_map_sync(map);
}
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
return 1;
}
#define srcu_lock_acquire(m) do { } while (0)
#define srcu_lock_release(m) do { } while (0)
#define srcu_lock_sync(m) do { } while (0)
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
#define SRCU_NMI_UNKNOWN 0x0
#define SRCU_NMI_UNSAFE 0x1
#define SRCU_NMI_SAFE 0x2
#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_TREE_SRCU)
void srcu_check_nmi_safety(struct srcu_struct *ssp, bool nmi_safe);
#else
static inline void srcu_check_nmi_safety(struct srcu_struct *ssp,
bool nmi_safe) { }
#endif
/**
* srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
* @p: the pointer to fetch and protect for later dereferencing
* @ssp: pointer to the srcu_struct, which is used to check that we
* really are in an SRCU read-side critical section.
* @c: condition to check for update-side use
*
* If PROVE_RCU is enabled, invoking this outside of an RCU read-side
* critical section will result in an RCU-lockdep splat, unless @c evaluates
* to 1. The @c argument will normally be a logical expression containing
* lockdep_is_held() calls.
*/
#define srcu_dereference_check(p, ssp, c) \
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
(c) || srcu_read_lock_held(ssp), __rcu)
/**
* srcu_dereference - fetch SRCU-protected pointer for later dereferencing
* @p: the pointer to fetch and protect for later dereferencing
* @ssp: pointer to the srcu_struct, which is used to check that we
* really are in an SRCU read-side critical section.
*
* Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
* is enabled, invoking this outside of an RCU read-side critical
* section will result in an RCU-lockdep splat.
*/
#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)
/**
* srcu_dereference_notrace - no tracing and no lockdep calls from here
* @p: the pointer to fetch and protect for later dereferencing
* @ssp: pointer to the srcu_struct, which is used to check that we
* really are in an SRCU read-side critical section.
*/
#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)
/**
* srcu_read_lock - register a new reader for an SRCU-protected structure.
* @ssp: srcu_struct in which to register the new reader.
*
* Enter an SRCU read-side critical section. Note that SRCU read-side
* critical sections may be nested. However, it is illegal to
* call anything that waits on an SRCU grace period for the same
* srcu_struct, whether directly or indirectly. Please note that
* one way to indirectly wait on an SRCU grace period is to acquire
* a mutex that is held elsewhere while calling synchronize_srcu() or
* synchronize_srcu_expedited().
*
* Note that srcu_read_lock() and the matching srcu_read_unlock() must
* occur in the same context, for example, it is illegal to invoke
* srcu_read_unlock() in an irq handler if the matching srcu_read_lock()
* was invoked in process context.
*/
static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
{
int retval;
srcu_check_nmi_safety(ssp, false);
retval = __srcu_read_lock(ssp);
srcu_lock_acquire(&ssp->dep_map);
return retval;
}
/**
* srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
* @ssp: srcu_struct in which to register the new reader.
*
* Enter an SRCU read-side critical section, but in an NMI-safe manner.
* See srcu_read_lock() for more information.
*/
static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp)
{
int retval;
srcu_check_nmi_safety(ssp, true);
retval = __srcu_read_lock_nmisafe(ssp);
rcu_try_lock_acquire(&ssp->dep_map);
return retval;
}
/* Used by tracing, cannot be traced and cannot invoke lockdep. */
static inline notrace int
srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
{
int retval;
srcu_check_nmi_safety(ssp, false);
retval = __srcu_read_lock(ssp);
return retval;
}
/**
* srcu_down_read - register a new reader for an SRCU-protected structure.
* @ssp: srcu_struct in which to register the new reader.
*
* Enter a semaphore-like SRCU read-side critical section. Note that
* SRCU read-side critical sections may be nested. However, it is
* illegal to call anything that waits on an SRCU grace period for the
* same srcu_struct, whether directly or indirectly. Please note that
* one way to indirectly wait on an SRCU grace period is to acquire
* a mutex that is held elsewhere while calling synchronize_srcu() or
* synchronize_srcu_expedited(). But if you want lockdep to help you
* keep this stuff straight, you should instead use srcu_read_lock().
*
* The semaphore-like nature of srcu_down_read() means that the matching
* srcu_up_read() can be invoked from some other context, for example,
* from some other task or from an irq handler. However, neither
* srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
*
* Calls to srcu_down_read() may be nested, similar to the manner in
* which calls to down_read() may be nested.
*/
static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp)
{
WARN_ON_ONCE(in_nmi());
srcu_check_nmi_safety(ssp, false);
return __srcu_read_lock(ssp);
}
/**
* srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
* @ssp: srcu_struct in which to unregister the old reader.
* @idx: return value from corresponding srcu_read_lock().
*
* Exit an SRCU read-side critical section.
*/
static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
__releases(ssp)
{
WARN_ON_ONCE(idx & ~0x1); srcu_check_nmi_safety(ssp, false);
srcu_lock_release(&ssp->dep_map);
__srcu_read_unlock(ssp, idx);
}
/**
* srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
* @ssp: srcu_struct in which to unregister the old reader.
* @idx: return value from corresponding srcu_read_lock().
*
* Exit an SRCU read-side critical section, but in an NMI-safe manner.
*/
static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
__releases(ssp)
{
WARN_ON_ONCE(idx & ~0x1); srcu_check_nmi_safety(ssp, true);
rcu_lock_release(&ssp->dep_map);
__srcu_read_unlock_nmisafe(ssp, idx);
}
/* Used by tracing, cannot be traced and cannot call lockdep. */
static inline notrace void
srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
{
srcu_check_nmi_safety(ssp, false);
__srcu_read_unlock(ssp, idx);
}
/**
* srcu_up_read - unregister a old reader from an SRCU-protected structure.
* @ssp: srcu_struct in which to unregister the old reader.
* @idx: return value from corresponding srcu_read_lock().
*
* Exit an SRCU read-side critical section, but not necessarily from
* the same context as the maching srcu_down_read().
*/
static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
__releases(ssp)
{
WARN_ON_ONCE(idx & ~0x1);
WARN_ON_ONCE(in_nmi());
srcu_check_nmi_safety(ssp, false);
__srcu_read_unlock(ssp, idx);
}
/**
* smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
*
* Converts the preceding srcu_read_unlock into a two-way memory barrier.
*
* Call this after srcu_read_unlock, to guarantee that all memory operations
* that occur after smp_mb__after_srcu_read_unlock will appear to happen after
* the preceding srcu_read_unlock.
*/
static inline void smp_mb__after_srcu_read_unlock(void)
{
/* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
}
DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
_T->idx = srcu_read_lock(_T->lock),
srcu_read_unlock(_T->lock, _T->idx),
int idx)
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/mm/swap.c
*
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
*/
/*
* This file contains the default values for the operation of the
* Linux VM subsystem. Fine-tuning documentation can be found in
* Documentation/admin-guide/sysctl/vm.rst.
* Started 18.12.91
* Swap aging added 23.2.95, Stephen Tweedie.
* Buffermem limits added 12.3.98, Rik van Riel.
*/
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/mm_inline.h>
#include <linux/percpu_counter.h>
#include <linux/memremap.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/backing-dev.h>
#include <linux/memcontrol.h>
#include <linux/gfp.h>
#include <linux/uio.h>
#include <linux/hugetlb.h>
#include <linux/page_idle.h>
#include <linux/local_lock.h>
#include <linux/buffer_head.h>
#include "internal.h"
#define CREATE_TRACE_POINTS
#include <trace/events/pagemap.h>
/* How many pages do we try to swap or page in/out together? As a power of 2 */
int page_cluster;
const int page_cluster_max = 31;
/* Protecting only lru_rotate.fbatch which requires disabling interrupts */
struct lru_rotate {
local_lock_t lock;
struct folio_batch fbatch;
};
static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = {
.lock = INIT_LOCAL_LOCK(lock),
};
/*
* The following folio batches are grouped together because they are protected
* by disabling preemption (and interrupts remain enabled).
*/
struct cpu_fbatches {
local_lock_t lock;
struct folio_batch lru_add;
struct folio_batch lru_deactivate_file;
struct folio_batch lru_deactivate;
struct folio_batch lru_lazyfree;
#ifdef CONFIG_SMP
struct folio_batch activate;
#endif
};
static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = {
.lock = INIT_LOCAL_LOCK(lock),
};
static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp,
unsigned long *flagsp)
{
if (folio_test_lru(folio)) { folio_lruvec_relock_irqsave(folio, lruvecp, flagsp); lruvec_del_folio(*lruvecp, folio); __folio_clear_lru_flags(folio);
}
/*
* In rare cases, when truncation or holepunching raced with
* munlock after VM_LOCKED was cleared, Mlocked may still be
* found set here. This does not indicate a problem, unless
* "unevictable_pgs_cleared" appears worryingly large.
*/
if (unlikely(folio_test_mlocked(folio))) { long nr_pages = folio_nr_pages(folio); __folio_clear_mlocked(folio); zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
}
}
/*
* This path almost never happens for VM activity - pages are normally freed
* in batches. But it gets used by networking - and for compound pages.
*/
static void page_cache_release(struct folio *folio)
{
struct lruvec *lruvec = NULL;
unsigned long flags;
__page_cache_release(folio, &lruvec, &flags);
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
}
static void __folio_put_small(struct folio *folio)
{
page_cache_release(folio); mem_cgroup_uncharge(folio); free_unref_page(&folio->page, 0);
}
static void __folio_put_large(struct folio *folio)
{
/*
* __page_cache_release() is supposed to be called for thp, not for
* hugetlb. This is because hugetlb page does never have PageLRU set
* (it's never listed to any LRU lists) and no memcg routines should
* be called for hugetlb (it has a separate hugetlb_cgroup.)
*/
if (!folio_test_hugetlb(folio)) page_cache_release(folio); destroy_large_folio(folio);
}
void __folio_put(struct folio *folio)
{
if (unlikely(folio_is_zone_device(folio)))
free_zone_device_page(&folio->page);
else if (unlikely(folio_test_large(folio))) __folio_put_large(folio);
else
__folio_put_small(folio);}
EXPORT_SYMBOL(__folio_put);
/**
* put_pages_list() - release a list of pages
* @pages: list of pages threaded on page->lru
*
* Release a list of pages which are strung together on page.lru.
*/
void put_pages_list(struct list_head *pages)
{
struct folio_batch fbatch;
struct folio *folio, *next;
folio_batch_init(&fbatch);
list_for_each_entry_safe(folio, next, pages, lru) {
if (!folio_put_testzero(folio))
continue;
if (folio_test_large(folio)) {
__folio_put_large(folio);
continue;
}
/* LRU flag must be clear because it's passed using the lru */
if (folio_batch_add(&fbatch, folio) > 0)
continue;
free_unref_folios(&fbatch);
}
if (fbatch.nr)
free_unref_folios(&fbatch);
INIT_LIST_HEAD(pages);
}
EXPORT_SYMBOL(put_pages_list);
typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
static void lru_add_fn(struct lruvec *lruvec, struct folio *folio)
{
int was_unevictable = folio_test_clear_unevictable(folio); long nr_pages = folio_nr_pages(folio); VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
/*
* Is an smp_mb__after_atomic() still required here, before
* folio_evictable() tests the mlocked flag, to rule out the possibility
* of stranding an evictable folio on an unevictable LRU? I think
* not, because __munlock_folio() only clears the mlocked flag
* while the LRU lock is held.
*
* (That is not true of __page_cache_release(), and not necessarily
* true of folios_put(): but those only clear the mlocked flag after
* folio_put_testzero() has excluded any other users of the folio.)
*/
if (folio_evictable(folio)) { if (was_unevictable) __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
} else {
folio_clear_active(folio);
folio_set_unevictable(folio);
/*
* folio->mlock_count = !!folio_test_mlocked(folio)?
* But that leaves __mlock_folio() in doubt whether another
* actor has already counted the mlock or not. Err on the
* safe side, underestimate, let page reclaim fix it, rather
* than leaving a page on the unevictable LRU indefinitely.
*/
folio->mlock_count = 0;
if (!was_unevictable)
__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
}
lruvec_add_folio(lruvec, folio); trace_mm_lru_insertion(folio);
}
static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
{
int i;
struct lruvec *lruvec = NULL;
unsigned long flags = 0; for (i = 0; i < folio_batch_count(fbatch); i++) { struct folio *folio = fbatch->folios[i];
/* block memcg migration while the folio moves between lru */
if (move_fn != lru_add_fn && !folio_test_clear_lru(folio))
continue;
folio_lruvec_relock_irqsave(folio, &lruvec, &flags); move_fn(lruvec, folio);
folio_set_lru(folio);
}
if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); folios_put(fbatch);
}
static void folio_batch_add_and_move(struct folio_batch *fbatch,
struct folio *folio, move_fn_t move_fn)
{
if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) && !lru_cache_disabled())
return;
folio_batch_move_lru(fbatch, move_fn);
}
static void lru_move_tail_fn(struct lruvec *lruvec, struct folio *folio)
{
if (!folio_test_unevictable(folio)) {
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
lruvec_add_folio_tail(lruvec, folio);
__count_vm_events(PGROTATED, folio_nr_pages(folio));
}
}
/*
* Writeback is about to end against a folio which has been marked for
* immediate reclaim. If it still appears to be reclaimable, move it
* to the tail of the inactive list.
*
* folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
*/
void folio_rotate_reclaimable(struct folio *folio)
{
if (!folio_test_locked(folio) && !folio_test_dirty(folio) &&
!folio_test_unevictable(folio) && folio_test_lru(folio)) {
struct folio_batch *fbatch;
unsigned long flags;
folio_get(folio);
local_lock_irqsave(&lru_rotate.lock, flags);
fbatch = this_cpu_ptr(&lru_rotate.fbatch);
folio_batch_add_and_move(fbatch, folio, lru_move_tail_fn);
local_unlock_irqrestore(&lru_rotate.lock, flags);
}
}
void lru_note_cost(struct lruvec *lruvec, bool file,
unsigned int nr_io, unsigned int nr_rotated)
{
unsigned long cost;
/*
* Reflect the relative cost of incurring IO and spending CPU
* time on rotations. This doesn't attempt to make a precise
* comparison, it just says: if reloads are about comparable
* between the LRU lists, or rotations are overwhelmingly
* different between them, adjust scan balance for CPU work.
*/
cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated;
do {
unsigned long lrusize;
/*
* Hold lruvec->lru_lock is safe here, since
* 1) The pinned lruvec in reclaim, or
* 2) From a pre-LRU page during refault (which also holds the
* rcu lock, so would be safe even if the page was on the LRU
* and could move simultaneously to a new lruvec).
*/
spin_lock_irq(&lruvec->lru_lock);
/* Record cost event */
if (file)
lruvec->file_cost += cost;
else
lruvec->anon_cost += cost;
/*
* Decay previous events
*
* Because workloads change over time (and to avoid
* overflow) we keep these statistics as a floating
* average, which ends up weighing recent refaults
* more than old ones.
*/
lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
lruvec_page_state(lruvec, NR_ACTIVE_FILE);
if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
lruvec->file_cost /= 2;
lruvec->anon_cost /= 2;
}
spin_unlock_irq(&lruvec->lru_lock);
} while ((lruvec = parent_lruvec(lruvec)));
}
void lru_note_cost_refault(struct folio *folio)
{
lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio),
folio_nr_pages(folio), 0);
}
static void folio_activate_fn(struct lruvec *lruvec, struct folio *folio)
{
if (!folio_test_active(folio) && !folio_test_unevictable(folio)) { long nr_pages = folio_nr_pages(folio); lruvec_del_folio(lruvec, folio);
folio_set_active(folio);
lruvec_add_folio(lruvec, folio); trace_mm_lru_activate(folio); __count_vm_events(PGACTIVATE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE,
nr_pages);
}
}
#ifdef CONFIG_SMP
static void folio_activate_drain(int cpu)
{
struct folio_batch *fbatch = &per_cpu(cpu_fbatches.activate, cpu);
if (folio_batch_count(fbatch))
folio_batch_move_lru(fbatch, folio_activate_fn);
}
void folio_activate(struct folio *folio)
{
if (folio_test_lru(folio) && !folio_test_active(folio) && !folio_test_unevictable(folio)) {
struct folio_batch *fbatch;
folio_get(folio); local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.activate);
folio_batch_add_and_move(fbatch, folio, folio_activate_fn);
local_unlock(&cpu_fbatches.lock);
}
}
#else
static inline void folio_activate_drain(int cpu)
{
}
void folio_activate(struct folio *folio)
{
struct lruvec *lruvec;
if (folio_test_clear_lru(folio)) {
lruvec = folio_lruvec_lock_irq(folio);
folio_activate_fn(lruvec, folio);
unlock_page_lruvec_irq(lruvec);
folio_set_lru(folio);
}
}
#endif
static void __lru_cache_activate_folio(struct folio *folio)
{
struct folio_batch *fbatch;
int i;
local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
/*
* Search backwards on the optimistic assumption that the folio being
* activated has just been added to this batch. Note that only
* the local batch is examined as a !LRU folio could be in the
* process of being released, reclaimed, migrated or on a remote
* batch that is currently being drained. Furthermore, marking
* a remote batch's folio active potentially hits a race where
* a folio is marked active just after it is added to the inactive
* list causing accounting errors and BUG_ON checks to trigger.
*/
for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) { struct folio *batch_folio = fbatch->folios[i];
if (batch_folio == folio) {
folio_set_active(folio);
break;
}
}
local_unlock(&cpu_fbatches.lock);
}
#ifdef CONFIG_LRU_GEN
static void folio_inc_refs(struct folio *folio)
{
unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
if (folio_test_unevictable(folio))
return;
if (!folio_test_referenced(folio)) {
folio_set_referenced(folio);
return;
}
if (!folio_test_workingset(folio)) {
folio_set_workingset(folio);
return;
}
/* see the comment on MAX_NR_TIERS */
do {
new_flags = old_flags & LRU_REFS_MASK;
if (new_flags == LRU_REFS_MASK)
break;
new_flags += BIT(LRU_REFS_PGOFF);
new_flags |= old_flags & ~LRU_REFS_MASK;
} while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
}
#else
static void folio_inc_refs(struct folio *folio)
{
}
#endif /* CONFIG_LRU_GEN */
/*
* Mark a page as having seen activity.
*
* inactive,unreferenced -> inactive,referenced
* inactive,referenced -> active,unreferenced
* active,unreferenced -> active,referenced
*
* When a newly allocated page is not yet visible, so safe for non-atomic ops,
* __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
*/
void folio_mark_accessed(struct folio *folio)
{
if (lru_gen_enabled()) {
folio_inc_refs(folio);
return;
}
if (!folio_test_referenced(folio)) { folio_set_referenced(folio); } else if (folio_test_unevictable(folio)) {
/*
* Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
* this list is never rotated or maintained, so marking an
* unevictable page accessed has no effect.
*/
} else if (!folio_test_active(folio)) {
/*
* If the folio is on the LRU, queue it for activation via
* cpu_fbatches.activate. Otherwise, assume the folio is in a
* folio_batch, mark it active and it'll be moved to the active
* LRU on the next drain.
*/
if (folio_test_lru(folio)) folio_activate(folio);
else
__lru_cache_activate_folio(folio); folio_clear_referenced(folio);
workingset_activation(folio);
}
if (folio_test_idle(folio))
folio_clear_idle(folio);
}
EXPORT_SYMBOL(folio_mark_accessed);
/**
* folio_add_lru - Add a folio to an LRU list.
* @folio: The folio to be added to the LRU.
*
* Queue the folio for addition to the LRU. The decision on whether
* to add the page to the [in]active [file|anon] list is deferred until the
* folio_batch is drained. This gives a chance for the caller of folio_add_lru()
* have the folio added to the active list using folio_mark_accessed().
*/
void folio_add_lru(struct folio *folio)
{
struct folio_batch *fbatch;
VM_BUG_ON_FOLIO(folio_test_active(folio) &&
folio_test_unevictable(folio), folio);
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
/* see the comment in lru_gen_add_folio() */
if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
folio_set_active(folio);
folio_get(folio); local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
folio_batch_add_and_move(fbatch, folio, lru_add_fn);
local_unlock(&cpu_fbatches.lock);
}
EXPORT_SYMBOL(folio_add_lru);
/**
* folio_add_lru_vma() - Add a folio to the appropate LRU list for this VMA.
* @folio: The folio to be added to the LRU.
* @vma: VMA in which the folio is mapped.
*
* If the VMA is mlocked, @folio is added to the unevictable list.
* Otherwise, it is treated the same way as folio_add_lru().
*/
void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)
{
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED)) mlock_new_folio(folio);
else
folio_add_lru(folio);
}
/*
* If the folio cannot be invalidated, it is moved to the
* inactive list to speed up its reclaim. It is moved to the
* head of the list, rather than the tail, to give the flusher
* threads some time to write it out, as this is much more
* effective than the single-page writeout from reclaim.
*
* If the folio isn't mapped and dirty/writeback, the folio
* could be reclaimed asap using the reclaim flag.
*
* 1. active, mapped folio -> none
* 2. active, dirty/writeback folio -> inactive, head, reclaim
* 3. inactive, mapped folio -> none
* 4. inactive, dirty/writeback folio -> inactive, head, reclaim
* 5. inactive, clean -> inactive, tail
* 6. Others -> none
*
* In 4, it moves to the head of the inactive list so the folio is
* written out by flusher threads as this is much more efficient
* than the single-page writeout from reclaim.
*/
static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio)
{
bool active = folio_test_active(folio);
long nr_pages = folio_nr_pages(folio);
if (folio_test_unevictable(folio))
return;
/* Some processes are using the folio */
if (folio_mapped(folio))
return;
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
folio_clear_referenced(folio);
if (folio_test_writeback(folio) || folio_test_dirty(folio)) {
/*
* Setting the reclaim flag could race with
* folio_end_writeback() and confuse readahead. But the
* race window is _really_ small and it's not a critical
* problem.
*/
lruvec_add_folio(lruvec, folio);
folio_set_reclaim(folio);
} else {
/*
* The folio's writeback ended while it was in the batch.
* We move that folio to the tail of the inactive list.
*/
lruvec_add_folio_tail(lruvec, folio);
__count_vm_events(PGROTATED, nr_pages);
}
if (active) {
__count_vm_events(PGDEACTIVATE, nr_pages);
__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
nr_pages);
}
}
static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio)
{
if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) {
long nr_pages = folio_nr_pages(folio);
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
folio_clear_referenced(folio);
lruvec_add_folio(lruvec, folio);
__count_vm_events(PGDEACTIVATE, nr_pages);
__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
nr_pages);
}
}
static void lru_lazyfree_fn(struct lruvec *lruvec, struct folio *folio)
{
if (folio_test_anon(folio) && folio_test_swapbacked(folio) &&
!folio_test_swapcache(folio) && !folio_test_unevictable(folio)) {
long nr_pages = folio_nr_pages(folio);
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
folio_clear_referenced(folio);
/*
* Lazyfree folios are clean anonymous folios. They have
* the swapbacked flag cleared, to distinguish them from normal
* anonymous folios
*/
folio_clear_swapbacked(folio);
lruvec_add_folio(lruvec, folio);
__count_vm_events(PGLAZYFREE, nr_pages);
__count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE,
nr_pages);
}
}
/*
* Drain pages out of the cpu's folio_batch.
* Either "cpu" is the current CPU, and preemption has already been
* disabled; or "cpu" is being hot-unplugged, and is already dead.
*/
void lru_add_drain_cpu(int cpu)
{
struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); struct folio_batch *fbatch = &fbatches->lru_add;
if (folio_batch_count(fbatch))
folio_batch_move_lru(fbatch, lru_add_fn);
fbatch = &per_cpu(lru_rotate.fbatch, cpu);
/* Disabling interrupts below acts as a compiler barrier. */
if (data_race(folio_batch_count(fbatch))) {
unsigned long flags;
/* No harm done if a racing interrupt already did this */
local_lock_irqsave(&lru_rotate.lock, flags);
folio_batch_move_lru(fbatch, lru_move_tail_fn);
local_unlock_irqrestore(&lru_rotate.lock, flags);
}
fbatch = &fbatches->lru_deactivate_file; if (folio_batch_count(fbatch))
folio_batch_move_lru(fbatch, lru_deactivate_file_fn);
fbatch = &fbatches->lru_deactivate; if (folio_batch_count(fbatch))
folio_batch_move_lru(fbatch, lru_deactivate_fn);
fbatch = &fbatches->lru_lazyfree; if (folio_batch_count(fbatch))
folio_batch_move_lru(fbatch, lru_lazyfree_fn);
folio_activate_drain(cpu);
}
/**
* deactivate_file_folio() - Deactivate a file folio.
* @folio: Folio to deactivate.
*
* This function hints to the VM that @folio is a good reclaim candidate,
* for example if its invalidation fails due to the folio being dirty
* or under writeback.
*
* Context: Caller holds a reference on the folio.
*/
void deactivate_file_folio(struct folio *folio)
{
struct folio_batch *fbatch;
/* Deactivating an unevictable folio will not accelerate reclaim */
if (folio_test_unevictable(folio))
return;
folio_get(folio);
local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate_file);
folio_batch_add_and_move(fbatch, folio, lru_deactivate_file_fn);
local_unlock(&cpu_fbatches.lock);
}
/*
* folio_deactivate - deactivate a folio
* @folio: folio to deactivate
*
* folio_deactivate() moves @folio to the inactive list if @folio was on the
* active list and was not unevictable. This is done to accelerate the
* reclaim of @folio.
*/
void folio_deactivate(struct folio *folio)
{
if (folio_test_lru(folio) && !folio_test_unevictable(folio) &&
(folio_test_active(folio) || lru_gen_enabled())) {
struct folio_batch *fbatch;
folio_get(folio);
local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate);
folio_batch_add_and_move(fbatch, folio, lru_deactivate_fn);
local_unlock(&cpu_fbatches.lock);
}
}
/**
* folio_mark_lazyfree - make an anon folio lazyfree
* @folio: folio to deactivate
*
* folio_mark_lazyfree() moves @folio to the inactive file list.
* This is done to accelerate the reclaim of @folio.
*/
void folio_mark_lazyfree(struct folio *folio)
{
if (folio_test_lru(folio) && folio_test_anon(folio) &&
folio_test_swapbacked(folio) && !folio_test_swapcache(folio) &&
!folio_test_unevictable(folio)) {
struct folio_batch *fbatch;
folio_get(folio);
local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.lru_lazyfree);
folio_batch_add_and_move(fbatch, folio, lru_lazyfree_fn);
local_unlock(&cpu_fbatches.lock);
}
}
void lru_add_drain(void)
{
local_lock(&cpu_fbatches.lock);
lru_add_drain_cpu(smp_processor_id());
local_unlock(&cpu_fbatches.lock); mlock_drain_local();
}
/*
* It's called from per-cpu workqueue context in SMP case so
* lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
* the same cpu. It shouldn't be a problem in !SMP case since
* the core is only one and the locks will disable preemption.
*/
static void lru_add_and_bh_lrus_drain(void)
{
local_lock(&cpu_fbatches.lock);
lru_add_drain_cpu(smp_processor_id());
local_unlock(&cpu_fbatches.lock);
invalidate_bh_lrus_cpu();
mlock_drain_local();
}
void lru_add_drain_cpu_zone(struct zone *zone)
{
local_lock(&cpu_fbatches.lock);
lru_add_drain_cpu(smp_processor_id());
drain_local_pages(zone);
local_unlock(&cpu_fbatches.lock);
mlock_drain_local();
}
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
static void lru_add_drain_per_cpu(struct work_struct *dummy)
{
lru_add_and_bh_lrus_drain();
}
static bool cpu_needs_drain(unsigned int cpu)
{
struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
/* Check these in order of likelihood that they're not zero */
return folio_batch_count(&fbatches->lru_add) ||
data_race(folio_batch_count(&per_cpu(lru_rotate.fbatch, cpu))) ||
folio_batch_count(&fbatches->lru_deactivate_file) ||
folio_batch_count(&fbatches->lru_deactivate) ||
folio_batch_count(&fbatches->lru_lazyfree) ||
folio_batch_count(&fbatches->activate) ||
need_mlock_drain(cpu) ||
has_bh_in_lru(cpu, NULL);
}
/*
* Doesn't need any cpu hotplug locking because we do rely on per-cpu
* kworkers being shut down before our page_alloc_cpu_dead callback is
* executed on the offlined cpu.
* Calling this function with cpu hotplug locks held can actually lead
* to obscure indirect dependencies via WQ context.
*/
static inline void __lru_add_drain_all(bool force_all_cpus)
{
/*
* lru_drain_gen - Global pages generation number
*
* (A) Definition: global lru_drain_gen = x implies that all generations
* 0 < n <= x are already *scheduled* for draining.
*
* This is an optimization for the highly-contended use case where a
* user space workload keeps constantly generating a flow of pages for
* each CPU.
*/
static unsigned int lru_drain_gen;
static struct cpumask has_work;
static DEFINE_MUTEX(lock);
unsigned cpu, this_gen;
/*
* Make sure nobody triggers this path before mm_percpu_wq is fully
* initialized.
*/
if (WARN_ON(!mm_percpu_wq))
return;
/*
* Guarantee folio_batch counter stores visible by this CPU
* are visible to other CPUs before loading the current drain
* generation.
*/
smp_mb();
/*
* (B) Locally cache global LRU draining generation number
*
* The read barrier ensures that the counter is loaded before the mutex
* is taken. It pairs with smp_mb() inside the mutex critical section
* at (D).
*/
this_gen = smp_load_acquire(&lru_drain_gen);
mutex_lock(&lock);
/*
* (C) Exit the draining operation if a newer generation, from another
* lru_add_drain_all(), was already scheduled for draining. Check (A).
*/
if (unlikely(this_gen != lru_drain_gen && !force_all_cpus))
goto done;
/*
* (D) Increment global generation number
*
* Pairs with smp_load_acquire() at (B), outside of the critical
* section. Use a full memory barrier to guarantee that the
* new global drain generation number is stored before loading
* folio_batch counters.
*
* This pairing must be done here, before the for_each_online_cpu loop
* below which drains the page vectors.
*
* Let x, y, and z represent some system CPU numbers, where x < y < z.
* Assume CPU #z is in the middle of the for_each_online_cpu loop
* below and has already reached CPU #y's per-cpu data. CPU #x comes
* along, adds some pages to its per-cpu vectors, then calls
* lru_add_drain_all().
*
* If the paired barrier is done at any later step, e.g. after the
* loop, CPU #x will just exit at (C) and miss flushing out all of its
* added pages.
*/
WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
smp_mb();
cpumask_clear(&has_work);
for_each_online_cpu(cpu) {
struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
if (cpu_needs_drain(cpu)) {
INIT_WORK(work, lru_add_drain_per_cpu);
queue_work_on(cpu, mm_percpu_wq, work);
__cpumask_set_cpu(cpu, &has_work);
}
}
for_each_cpu(cpu, &has_work)
flush_work(&per_cpu(lru_add_drain_work, cpu));
done:
mutex_unlock(&lock);
}
void lru_add_drain_all(void)
{
__lru_add_drain_all(false);
}
#else
void lru_add_drain_all(void)
{
lru_add_drain();
}
#endif /* CONFIG_SMP */
atomic_t lru_disable_count = ATOMIC_INIT(0);
/*
* lru_cache_disable() needs to be called before we start compiling
* a list of pages to be migrated using isolate_lru_page().
* It drains pages on LRU cache and then disable on all cpus until
* lru_cache_enable is called.
*
* Must be paired with a call to lru_cache_enable().
*/
void lru_cache_disable(void)
{
atomic_inc(&lru_disable_count);
/*
* Readers of lru_disable_count are protected by either disabling
* preemption or rcu_read_lock:
*
* preempt_disable, local_irq_disable [bh_lru_lock()]
* rcu_read_lock [rt_spin_lock CONFIG_PREEMPT_RT]
* preempt_disable [local_lock !CONFIG_PREEMPT_RT]
*
* Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on
* preempt_disable() regions of code. So any CPU which sees
* lru_disable_count = 0 will have exited the critical
* section when synchronize_rcu() returns.
*/
synchronize_rcu_expedited();
#ifdef CONFIG_SMP
__lru_add_drain_all(true);
#else
lru_add_and_bh_lrus_drain();
#endif
}
/**
* folios_put_refs - Reduce the reference count on a batch of folios.
* @folios: The folios.
* @refs: The number of refs to subtract from each folio.
*
* Like folio_put(), but for a batch of folios. This is more efficient
* than writing the loop yourself as it will optimise the locks which need
* to be taken if the folios are freed. The folios batch is returned
* empty and ready to be reused for another batch; there is no need
* to reinitialise it. If @refs is NULL, we subtract one from each
* folio refcount.
*
* Context: May be called in process or interrupt context, but not in NMI
* context. May be called while holding a spinlock.
*/
void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
{
int i, j;
struct lruvec *lruvec = NULL;
unsigned long flags = 0;
for (i = 0, j = 0; i < folios->nr; i++) { struct folio *folio = folios->folios[i]; unsigned int nr_refs = refs ? refs[i] : 1;
if (is_huge_zero_page(&folio->page))
continue;
if (folio_is_zone_device(folio)) {
if (lruvec) {
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = NULL;
}
if (put_devmap_managed_page_refs(&folio->page, nr_refs))
continue;
if (folio_ref_sub_and_test(folio, nr_refs))
free_zone_device_page(&folio->page);
continue;
}
if (!folio_ref_sub_and_test(folio, nr_refs))
continue;
/* hugetlb has its own memcg */
if (folio_test_hugetlb(folio)) { if (lruvec) { unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = NULL;
}
free_huge_folio(folio);
continue;
}
if (folio_test_large(folio) &&
folio_test_large_rmappable(folio))
folio_undo_large_rmappable(folio);
__page_cache_release(folio, &lruvec, &flags);
if (j != i)
folios->folios[j] = folio; j++;
}
if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); if (!j) { folio_batch_reinit(folios); return;
}
folios->nr = j; mem_cgroup_uncharge_folios(folios); free_unref_folios(folios);
}
EXPORT_SYMBOL(folios_put_refs);
/**
* release_pages - batched put_page()
* @arg: array of pages to release
* @nr: number of pages
*
* Decrement the reference count on all the pages in @arg. If it
* fell to zero, remove the page from the LRU and free it.
*
* Note that the argument can be an array of pages, encoded pages,
* or folio pointers. We ignore any encoded bits, and turn any of
* them into just a folio that gets free'd.
*/
void release_pages(release_pages_arg arg, int nr)
{
struct folio_batch fbatch;
int refs[PAGEVEC_SIZE];
struct encoded_page **encoded = arg.encoded_pages;
int i;
folio_batch_init(&fbatch);
for (i = 0; i < nr; i++) {
/* Turn any of the argument types into a folio */
struct folio *folio = page_folio(encoded_page_ptr(encoded[i]));
/* Is our next entry actually "nr_pages" -> "nr_refs" ? */
refs[fbatch.nr] = 1;
if (unlikely(encoded_page_flags(encoded[i]) &
ENCODED_PAGE_BIT_NR_PAGES_NEXT))
refs[fbatch.nr] = encoded_nr_pages(encoded[++i]);
if (folio_batch_add(&fbatch, folio) > 0)
continue;
folios_put_refs(&fbatch, refs);
}
if (fbatch.nr)
folios_put_refs(&fbatch, refs);
}
EXPORT_SYMBOL(release_pages);
/*
* The folios which we're about to release may be in the deferred lru-addition
* queues. That would prevent them from really being freed right now. That's
* OK from a correctness point of view but is inefficient - those folios may be
* cache-warm and we want to give them back to the page allocator ASAP.
*
* So __folio_batch_release() will drain those queues here.
* folio_batch_move_lru() calls folios_put() directly to avoid
* mutual recursion.
*/
void __folio_batch_release(struct folio_batch *fbatch)
{
if (!fbatch->percpu_pvec_drained) { lru_add_drain();
fbatch->percpu_pvec_drained = true;
}
folios_put(fbatch);
}
EXPORT_SYMBOL(__folio_batch_release);
/**
* folio_batch_remove_exceptionals() - Prune non-folios from a batch.
* @fbatch: The batch to prune
*
* find_get_entries() fills a batch with both folios and shadow/swap/DAX
* entries. This function prunes all the non-folio entries from @fbatch
* without leaving holes, so that it can be passed on to folio-only batch
* operations.
*/
void folio_batch_remove_exceptionals(struct folio_batch *fbatch)
{
unsigned int i, j;
for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) { struct folio *folio = fbatch->folios[i];
if (!xa_is_value(folio))
fbatch->folios[j++] = folio;
}
fbatch->nr = j;
}
/*
* Perform any setup for the swap system
*/
void __init swap_setup(void)
{
unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT);
/* Use a smaller cluster for small-memory machines */
if (megs < 16)
page_cluster = 2;
else
page_cluster = 3;
/*
* Right now other parts of the system means that we
* _really_ don't want to cluster much more
*/
}
// SPDX-License-Identifier: GPL-2.0
/* Device wakeirq helper functions */
#include <linux/device.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/slab.h>
#include <linux/pm_runtime.h>
#include <linux/pm_wakeirq.h>
#include "power.h"
/**
* dev_pm_attach_wake_irq - Attach device interrupt as a wake IRQ
* @dev: Device entry
* @wirq: Wake irq specific data
*
* Internal function to attach a dedicated wake-up interrupt as a wake IRQ.
*/
static int dev_pm_attach_wake_irq(struct device *dev, struct wake_irq *wirq)
{
unsigned long flags;
if (!dev || !wirq)
return -EINVAL;
spin_lock_irqsave(&dev->power.lock, flags);
if (dev_WARN_ONCE(dev, dev->power.wakeirq,
"wake irq already initialized\n")) {
spin_unlock_irqrestore(&dev->power.lock, flags);
return -EEXIST;
}
dev->power.wakeirq = wirq;
device_wakeup_attach_irq(dev, wirq);
spin_unlock_irqrestore(&dev->power.lock, flags);
return 0;
}
/**
* dev_pm_set_wake_irq - Attach device IO interrupt as wake IRQ
* @dev: Device entry
* @irq: Device IO interrupt
*
* Attach a device IO interrupt as a wake IRQ. The wake IRQ gets
* automatically configured for wake-up from suspend based
* on the device specific sysfs wakeup entry. Typically called
* during driver probe after calling device_init_wakeup().
*/
int dev_pm_set_wake_irq(struct device *dev, int irq)
{
struct wake_irq *wirq;
int err;
if (irq < 0)
return -EINVAL;
wirq = kzalloc(sizeof(*wirq), GFP_KERNEL);
if (!wirq)
return -ENOMEM;
wirq->dev = dev;
wirq->irq = irq;
err = dev_pm_attach_wake_irq(dev, wirq);
if (err)
kfree(wirq);
return err;
}
EXPORT_SYMBOL_GPL(dev_pm_set_wake_irq);
/**
* dev_pm_clear_wake_irq - Detach a device IO interrupt wake IRQ
* @dev: Device entry
*
* Detach a device wake IRQ and free resources.
*
* Note that it's OK for drivers to call this without calling
* dev_pm_set_wake_irq() as all the driver instances may not have
* a wake IRQ configured. This avoid adding wake IRQ specific
* checks into the drivers.
*/
void dev_pm_clear_wake_irq(struct device *dev)
{
struct wake_irq *wirq = dev->power.wakeirq;
unsigned long flags;
if (!wirq)
return;
spin_lock_irqsave(&dev->power.lock, flags);
device_wakeup_detach_irq(dev);
dev->power.wakeirq = NULL;
spin_unlock_irqrestore(&dev->power.lock, flags);
if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED) {
free_irq(wirq->irq, wirq);
wirq->status &= ~WAKE_IRQ_DEDICATED_MASK;
}
kfree(wirq->name);
kfree(wirq);
}
EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq);
/**
* handle_threaded_wake_irq - Handler for dedicated wake-up interrupts
* @irq: Device specific dedicated wake-up interrupt
* @_wirq: Wake IRQ data
*
* Some devices have a separate wake-up interrupt in addition to the
* device IO interrupt. The wake-up interrupt signals that a device
* should be woken up from it's idle state. This handler uses device
* specific pm_runtime functions to wake the device, and then it's
* up to the device to do whatever it needs to. Note that as the
* device may need to restore context and start up regulators, we
* use a threaded IRQ.
*
* Also note that we are not resending the lost device interrupts.
* We assume that the wake-up interrupt just needs to wake-up the
* device, and then device's pm_runtime_resume() can deal with the
* situation.
*/
static irqreturn_t handle_threaded_wake_irq(int irq, void *_wirq)
{
struct wake_irq *wirq = _wirq;
int res;
/* Maybe abort suspend? */
if (irqd_is_wakeup_set(irq_get_irq_data(irq))) {
pm_wakeup_event(wirq->dev, 0);
return IRQ_HANDLED;
}
/* We don't want RPM_ASYNC or RPM_NOWAIT here */
res = pm_runtime_resume(wirq->dev);
if (res < 0)
dev_warn(wirq->dev,
"wake IRQ with no resume: %i\n", res);
return IRQ_HANDLED;
}
static int __dev_pm_set_dedicated_wake_irq(struct device *dev, int irq, unsigned int flag)
{
struct wake_irq *wirq;
int err;
if (irq < 0)
return -EINVAL;
wirq = kzalloc(sizeof(*wirq), GFP_KERNEL);
if (!wirq)
return -ENOMEM;
wirq->name = kasprintf(GFP_KERNEL, "%s:wakeup", dev_name(dev));
if (!wirq->name) {
err = -ENOMEM;
goto err_free;
}
wirq->dev = dev;
wirq->irq = irq;
/* Prevent deferred spurious wakeirqs with disable_irq_nosync() */
irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
/*
* Consumer device may need to power up and restore state
* so we use a threaded irq.
*/
err = request_threaded_irq(irq, NULL, handle_threaded_wake_irq,
IRQF_ONESHOT | IRQF_NO_AUTOEN,
wirq->name, wirq);
if (err)
goto err_free_name;
err = dev_pm_attach_wake_irq(dev, wirq);
if (err)
goto err_free_irq;
wirq->status = WAKE_IRQ_DEDICATED_ALLOCATED | flag;
return err;
err_free_irq:
free_irq(irq, wirq);
err_free_name:
kfree(wirq->name);
err_free:
kfree(wirq);
return err;
}
/**
* dev_pm_set_dedicated_wake_irq - Request a dedicated wake-up interrupt
* @dev: Device entry
* @irq: Device wake-up interrupt
*
* Unless your hardware has separate wake-up interrupts in addition
* to the device IO interrupts, you don't need this.
*
* Sets up a threaded interrupt handler for a device that has
* a dedicated wake-up interrupt in addition to the device IO
* interrupt.
*/
int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq)
{
return __dev_pm_set_dedicated_wake_irq(dev, irq, 0);
}
EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq);
/**
* dev_pm_set_dedicated_wake_irq_reverse - Request a dedicated wake-up interrupt
* with reverse enable ordering
* @dev: Device entry
* @irq: Device wake-up interrupt
*
* Unless your hardware has separate wake-up interrupts in addition
* to the device IO interrupts, you don't need this.
*
* Sets up a threaded interrupt handler for a device that has a dedicated
* wake-up interrupt in addition to the device IO interrupt. It sets
* the status of WAKE_IRQ_DEDICATED_REVERSE to tell rpm_suspend()
* to enable dedicated wake-up interrupt after running the runtime suspend
* callback for @dev.
*/
int dev_pm_set_dedicated_wake_irq_reverse(struct device *dev, int irq)
{
return __dev_pm_set_dedicated_wake_irq(dev, irq, WAKE_IRQ_DEDICATED_REVERSE);
}
EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq_reverse);
/**
* dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt
* @dev: Device
* @can_change_status: Can change wake-up interrupt status
*
* Enables wakeirq conditionally. We need to enable wake-up interrupt
* lazily on the first rpm_suspend(). This is needed as the consumer device
* starts in RPM_SUSPENDED state, and the first pm_runtime_get() would
* otherwise try to disable already disabled wakeirq. The wake-up interrupt
* starts disabled with IRQ_NOAUTOEN set.
*
* Should be only called from rpm_suspend() and rpm_resume() path.
* Caller must hold &dev->power.lock to change wirq->status
*/
void dev_pm_enable_wake_irq_check(struct device *dev,
bool can_change_status)
{
struct wake_irq *wirq = dev->power.wakeirq; if (!wirq || !(wirq->status & WAKE_IRQ_DEDICATED_MASK))
return;
if (likely(wirq->status & WAKE_IRQ_DEDICATED_MANAGED)) {
goto enable;
} else if (can_change_status) { wirq->status |= WAKE_IRQ_DEDICATED_MANAGED;
goto enable;
}
return;
enable:
if (!can_change_status || !(wirq->status & WAKE_IRQ_DEDICATED_REVERSE)) { enable_irq(wirq->irq);
wirq->status |= WAKE_IRQ_DEDICATED_ENABLED;
}
}
/**
* dev_pm_disable_wake_irq_check - Checks and disables wake-up interrupt
* @dev: Device
* @cond_disable: if set, also check WAKE_IRQ_DEDICATED_REVERSE
*
* Disables wake-up interrupt conditionally based on status.
* Should be only called from rpm_suspend() and rpm_resume() path.
*/
void dev_pm_disable_wake_irq_check(struct device *dev, bool cond_disable)
{
struct wake_irq *wirq = dev->power.wakeirq; if (!wirq || !(wirq->status & WAKE_IRQ_DEDICATED_MASK))
return;
if (cond_disable && (wirq->status & WAKE_IRQ_DEDICATED_REVERSE))
return;
if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) { wirq->status &= ~WAKE_IRQ_DEDICATED_ENABLED;
disable_irq_nosync(wirq->irq);
}
}
/**
* dev_pm_enable_wake_irq_complete - enable wake IRQ not enabled before
* @dev: Device using the wake IRQ
*
* Enable wake IRQ conditionally based on status, mainly used if want to
* enable wake IRQ after running ->runtime_suspend() which depends on
* WAKE_IRQ_DEDICATED_REVERSE.
*
* Should be only called from rpm_suspend() path.
*/
void dev_pm_enable_wake_irq_complete(struct device *dev)
{
struct wake_irq *wirq = dev->power.wakeirq;
if (!wirq || !(wirq->status & WAKE_IRQ_DEDICATED_MASK))
return;
if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED &&
wirq->status & WAKE_IRQ_DEDICATED_REVERSE) {
enable_irq(wirq->irq);
wirq->status |= WAKE_IRQ_DEDICATED_ENABLED;
}
}
/**
* dev_pm_arm_wake_irq - Arm device wake-up
* @wirq: Device wake-up interrupt
*
* Sets up the wake-up event conditionally based on the
* device_may_wake().
*/
void dev_pm_arm_wake_irq(struct wake_irq *wirq)
{
if (!wirq)
return;
if (device_may_wakeup(wirq->dev)) {
if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED &&
!(wirq->status & WAKE_IRQ_DEDICATED_ENABLED))
enable_irq(wirq->irq);
enable_irq_wake(wirq->irq);
}
}
/**
* dev_pm_disarm_wake_irq - Disarm device wake-up
* @wirq: Device wake-up interrupt
*
* Clears up the wake-up event conditionally based on the
* device_may_wake().
*/
void dev_pm_disarm_wake_irq(struct wake_irq *wirq)
{
if (!wirq)
return;
if (device_may_wakeup(wirq->dev)) {
disable_irq_wake(wirq->irq);
if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED &&
!(wirq->status & WAKE_IRQ_DEDICATED_ENABLED))
disable_irq_nosync(wirq->irq);
}
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* x86 APERF/MPERF KHz calculation for
* /sys/.../cpufreq/scaling_cur_freq
*
* Copyright (C) 2017 Intel Corp.
* Author: Len Brown <len.brown@intel.com>
*/
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/sched/isolation.h>
#include <linux/sched/topology.h>
#include <linux/smp.h>
#include <linux/syscore_ops.h>
#include <asm/cpu.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>
#include "cpu.h"
struct aperfmperf {
seqcount_t seq;
unsigned long last_update;
u64 acnt;
u64 mcnt;
u64 aperf;
u64 mperf;
};
static DEFINE_PER_CPU_SHARED_ALIGNED(struct aperfmperf, cpu_samples) = {
.seq = SEQCNT_ZERO(cpu_samples.seq)
};
static void init_counter_refs(void)
{
u64 aperf, mperf;
rdmsrl(MSR_IA32_APERF, aperf);
rdmsrl(MSR_IA32_MPERF, mperf);
this_cpu_write(cpu_samples.aperf, aperf);
this_cpu_write(cpu_samples.mperf, mperf);
}
#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
/*
* APERF/MPERF frequency ratio computation.
*
* The scheduler wants to do frequency invariant accounting and needs a <1
* ratio to account for the 'current' frequency, corresponding to
* freq_curr / freq_max.
*
* Since the frequency freq_curr on x86 is controlled by micro-controller and
* our P-state setting is little more than a request/hint, we need to observe
* the effective frequency 'BusyMHz', i.e. the average frequency over a time
* interval after discarding idle time. This is given by:
*
* BusyMHz = delta_APERF / delta_MPERF * freq_base
*
* where freq_base is the max non-turbo P-state.
*
* The freq_max term has to be set to a somewhat arbitrary value, because we
* can't know which turbo states will be available at a given point in time:
* it all depends on the thermal headroom of the entire package. We set it to
* the turbo level with 4 cores active.
*
* Benchmarks show that's a good compromise between the 1C turbo ratio
* (freq_curr/freq_max would rarely reach 1) and something close to freq_base,
* which would ignore the entire turbo range (a conspicuous part, making
* freq_curr/freq_max always maxed out).
*
* An exception to the heuristic above is the Atom uarch, where we choose the
* highest turbo level for freq_max since Atom's are generally oriented towards
* power efficiency.
*
* Setting freq_max to anything less than the 1C turbo ratio makes the ratio
* freq_curr / freq_max to eventually grow >1, in which case we clip it to 1.
*/
DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key);
static u64 arch_turbo_freq_ratio = SCHED_CAPACITY_SCALE;
static u64 arch_max_freq_ratio = SCHED_CAPACITY_SCALE;
void arch_set_max_freq_ratio(bool turbo_disabled)
{
arch_max_freq_ratio = turbo_disabled ? SCHED_CAPACITY_SCALE :
arch_turbo_freq_ratio;
}
EXPORT_SYMBOL_GPL(arch_set_max_freq_ratio);
static bool __init turbo_disabled(void)
{
u64 misc_en;
int err;
err = rdmsrl_safe(MSR_IA32_MISC_ENABLE, &misc_en);
if (err)
return false;
return (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
}
static bool __init slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
{
int err;
err = rdmsrl_safe(MSR_ATOM_CORE_RATIOS, base_freq);
if (err)
return false;
err = rdmsrl_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq);
if (err)
return false;
*base_freq = (*base_freq >> 16) & 0x3F; /* max P state */
*turbo_freq = *turbo_freq & 0x3F; /* 1C turbo */
return true;
}
#define X86_MATCH(model) \
X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, \
INTEL_FAM6_##model, X86_FEATURE_APERFMPERF, NULL)
static const struct x86_cpu_id has_knl_turbo_ratio_limits[] __initconst = {
X86_MATCH(XEON_PHI_KNL),
X86_MATCH(XEON_PHI_KNM),
{}
};
static const struct x86_cpu_id has_skx_turbo_ratio_limits[] __initconst = {
X86_MATCH(SKYLAKE_X),
{}
};
static const struct x86_cpu_id has_glm_turbo_ratio_limits[] __initconst = {
X86_MATCH(ATOM_GOLDMONT),
X86_MATCH(ATOM_GOLDMONT_D),
X86_MATCH(ATOM_GOLDMONT_PLUS),
{}
};
static bool __init knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq,
int num_delta_fratio)
{
int fratio, delta_fratio, found;
int err, i;
u64 msr;
err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
if (err)
return false;
*base_freq = (*base_freq >> 8) & 0xFF; /* max P state */
err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
if (err)
return false;
fratio = (msr >> 8) & 0xFF;
i = 16;
found = 0;
do {
if (found >= num_delta_fratio) {
*turbo_freq = fratio;
return true;
}
delta_fratio = (msr >> (i + 5)) & 0x7;
if (delta_fratio) {
found += 1;
fratio -= delta_fratio;
}
i += 8;
} while (i < 64);
return true;
}
static bool __init skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int size)
{
u64 ratios, counts;
u32 group_size;
int err, i;
err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
if (err)
return false;
*base_freq = (*base_freq >> 8) & 0xFF; /* max P state */
err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &ratios);
if (err)
return false;
err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT1, &counts);
if (err)
return false;
for (i = 0; i < 64; i += 8) {
group_size = (counts >> i) & 0xFF;
if (group_size >= size) {
*turbo_freq = (ratios >> i) & 0xFF;
return true;
}
}
return false;
}
static bool __init core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
{
u64 msr;
int err;
err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
if (err)
return false;
err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
if (err)
return false;
*base_freq = (*base_freq >> 8) & 0xFF; /* max P state */
*turbo_freq = (msr >> 24) & 0xFF; /* 4C turbo */
/* The CPU may have less than 4 cores */
if (!*turbo_freq)
*turbo_freq = msr & 0xFF; /* 1C turbo */
return true;
}
static bool __init intel_set_max_freq_ratio(void)
{
u64 base_freq, turbo_freq;
u64 turbo_ratio;
if (slv_set_max_freq_ratio(&base_freq, &turbo_freq))
goto out;
if (x86_match_cpu(has_glm_turbo_ratio_limits) &&
skx_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
goto out;
if (x86_match_cpu(has_knl_turbo_ratio_limits) &&
knl_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
goto out;
if (x86_match_cpu(has_skx_turbo_ratio_limits) &&
skx_set_max_freq_ratio(&base_freq, &turbo_freq, 4))
goto out;
if (core_set_max_freq_ratio(&base_freq, &turbo_freq))
goto out;
return false;
out:
/*
* Some hypervisors advertise X86_FEATURE_APERFMPERF
* but then fill all MSR's with zeroes.
* Some CPUs have turbo boost but don't declare any turbo ratio
* in MSR_TURBO_RATIO_LIMIT.
*/
if (!base_freq || !turbo_freq) {
pr_debug("Couldn't determine cpu base or turbo frequency, necessary for scale-invariant accounting.\n");
return false;
}
turbo_ratio = div_u64(turbo_freq * SCHED_CAPACITY_SCALE, base_freq);
if (!turbo_ratio) {
pr_debug("Non-zero turbo and base frequencies led to a 0 ratio.\n");
return false;
}
arch_turbo_freq_ratio = turbo_ratio;
arch_set_max_freq_ratio(turbo_disabled());
return true;
}
#ifdef CONFIG_PM_SLEEP
static struct syscore_ops freq_invariance_syscore_ops = {
.resume = init_counter_refs,
};
static void register_freq_invariance_syscore_ops(void)
{
register_syscore_ops(&freq_invariance_syscore_ops);
}
#else
static inline void register_freq_invariance_syscore_ops(void) {}
#endif
static void freq_invariance_enable(void)
{
if (static_branch_unlikely(&arch_scale_freq_key)) {
WARN_ON_ONCE(1);
return;
}
static_branch_enable(&arch_scale_freq_key);
register_freq_invariance_syscore_ops();
pr_info("Estimated ratio of average max frequency by base frequency (times 1024): %llu\n", arch_max_freq_ratio);
}
void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled)
{
arch_turbo_freq_ratio = ratio;
arch_set_max_freq_ratio(turbo_disabled);
freq_invariance_enable();
}
static void __init bp_init_freq_invariance(void)
{
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
return;
if (intel_set_max_freq_ratio())
freq_invariance_enable();
}
static void disable_freq_invariance_workfn(struct work_struct *work)
{
int cpu;
static_branch_disable(&arch_scale_freq_key);
/*
* Set arch_freq_scale to a default value on all cpus
* This negates the effect of scaling
*/
for_each_possible_cpu(cpu)
per_cpu(arch_freq_scale, cpu) = SCHED_CAPACITY_SCALE;
}
static DECLARE_WORK(disable_freq_invariance_work,
disable_freq_invariance_workfn);
DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE;
static void scale_freq_tick(u64 acnt, u64 mcnt)
{
u64 freq_scale;
if (!arch_scale_freq_invariant())
return;
if (check_shl_overflow(acnt, 2*SCHED_CAPACITY_SHIFT, &acnt))
goto error;
if (check_mul_overflow(mcnt, arch_max_freq_ratio, &mcnt) || !mcnt)
goto error;
freq_scale = div64_u64(acnt, mcnt);
if (!freq_scale)
goto error;
if (freq_scale > SCHED_CAPACITY_SCALE)
freq_scale = SCHED_CAPACITY_SCALE;
this_cpu_write(arch_freq_scale, freq_scale);
return;
error:
pr_warn("Scheduler frequency invariance went wobbly, disabling!\n");
schedule_work(&disable_freq_invariance_work);
}
#else
static inline void bp_init_freq_invariance(void) { }
static inline void scale_freq_tick(u64 acnt, u64 mcnt) { }
#endif /* CONFIG_X86_64 && CONFIG_SMP */
void arch_scale_freq_tick(void)
{
struct aperfmperf *s = this_cpu_ptr(&cpu_samples);
u64 acnt, mcnt, aperf, mperf;
if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
return;
rdmsrl(MSR_IA32_APERF, aperf); rdmsrl(MSR_IA32_MPERF, mperf);
acnt = aperf - s->aperf;
mcnt = mperf - s->mperf;
s->aperf = aperf;
s->mperf = mperf;
raw_write_seqcount_begin(&s->seq);
s->last_update = jiffies;
s->acnt = acnt;
s->mcnt = mcnt;
raw_write_seqcount_end(&s->seq);
scale_freq_tick(acnt, mcnt);
}
/*
* Discard samples older than the define maximum sample age of 20ms. There
* is no point in sending IPIs in such a case. If the scheduler tick was
* not running then the CPU is either idle or isolated.
*/
#define MAX_SAMPLE_AGE ((unsigned long)HZ / 50)
unsigned int arch_freq_get_on_cpu(int cpu)
{
struct aperfmperf *s = per_cpu_ptr(&cpu_samples, cpu);
unsigned int seq, freq;
unsigned long last;
u64 acnt, mcnt;
if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
goto fallback;
do {
seq = raw_read_seqcount_begin(&s->seq);
last = s->last_update;
acnt = s->acnt;
mcnt = s->mcnt;
} while (read_seqcount_retry(&s->seq, seq));
/*
* Bail on invalid count and when the last update was too long ago,
* which covers idle and NOHZ full CPUs.
*/
if (!mcnt || (jiffies - last) > MAX_SAMPLE_AGE)
goto fallback;
return div64_u64((cpu_khz * acnt), mcnt);
fallback:
freq = cpufreq_quick_get(cpu);
return freq ? freq : cpu_khz;
}
static int __init bp_init_aperfmperf(void)
{
if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
return 0;
init_counter_refs();
bp_init_freq_invariance();
return 0;
}
early_initcall(bp_init_aperfmperf);
void ap_init_aperfmperf(void)
{
if (cpu_feature_enabled(X86_FEATURE_APERFMPERF))
init_counter_refs();
}
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/devres.c - device resource management
*
* Copyright (c) 2006 SUSE Linux Products GmbH
* Copyright (c) 2006 Tejun Heo <teheo@suse.de>
*/
#include <linux/device.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/percpu.h>
#include <asm/sections.h>
#include "base.h"
#include "trace.h"
struct devres_node {
struct list_head entry;
dr_release_t release;
const char *name;
size_t size;
};
struct devres {
struct devres_node node;
/*
* Some archs want to perform DMA into kmalloc caches
* and need a guaranteed alignment larger than
* the alignment of a 64-bit integer.
* Thus we use ARCH_DMA_MINALIGN for data[] which will force the same
* alignment for struct devres when allocated by kmalloc().
*/
u8 __aligned(ARCH_DMA_MINALIGN) data[];
};
struct devres_group {
struct devres_node node[2];
void *id;
int color;
/* -- 8 pointers */
};
static void set_node_dbginfo(struct devres_node *node, const char *name,
size_t size)
{
node->name = name;
node->size = size;
}
#ifdef CONFIG_DEBUG_DEVRES
static int log_devres = 0;
module_param_named(log, log_devres, int, S_IRUGO | S_IWUSR);
static void devres_dbg(struct device *dev, struct devres_node *node,
const char *op)
{
if (unlikely(log_devres)) dev_err(dev, "DEVRES %3s %p %s (%zu bytes)\n",
op, node, node->name, node->size);
}
#else /* CONFIG_DEBUG_DEVRES */
#define devres_dbg(dev, node, op) do {} while (0)
#endif /* CONFIG_DEBUG_DEVRES */
static void devres_log(struct device *dev, struct devres_node *node,
const char *op)
{
trace_devres_log(dev, op, node, node->name, node->size); devres_dbg(dev, node, op);
}
/*
* Release functions for devres group. These callbacks are used only
* for identification.
*/
static void group_open_release(struct device *dev, void *res)
{
/* noop */
}
static void group_close_release(struct device *dev, void *res)
{
/* noop */
}
static struct devres_group * node_to_group(struct devres_node *node)
{
if (node->release == &group_open_release)
return container_of(node, struct devres_group, node[0]);
if (node->release == &group_close_release) return container_of(node, struct devres_group, node[1]);
return NULL;
}
static bool check_dr_size(size_t size, size_t *tot_size)
{
/* We must catch any near-SIZE_MAX cases that could overflow. */
if (unlikely(check_add_overflow(sizeof(struct devres),
size, tot_size)))
return false;
/* Actually allocate the full kmalloc bucket size. */
*tot_size = kmalloc_size_roundup(*tot_size);
return true;
}
static __always_inline struct devres * alloc_dr(dr_release_t release,
size_t size, gfp_t gfp, int nid)
{
size_t tot_size;
struct devres *dr;
if (!check_dr_size(size, &tot_size))
return NULL;
dr = kmalloc_node_track_caller(tot_size, gfp, nid);
if (unlikely(!dr))
return NULL;
/* No need to clear memory twice */
if (!(gfp & __GFP_ZERO))
memset(dr, 0, offsetof(struct devres, data));
INIT_LIST_HEAD(&dr->node.entry);
dr->node.release = release;
return dr;
}
static void add_dr(struct device *dev, struct devres_node *node)
{
devres_log(dev, node, "ADD");
BUG_ON(!list_empty(&node->entry));
list_add_tail(&node->entry, &dev->devres_head);
}
static void replace_dr(struct device *dev,
struct devres_node *old, struct devres_node *new)
{
devres_log(dev, old, "REPLACE");
BUG_ON(!list_empty(&new->entry));
list_replace(&old->entry, &new->entry);
}
/**
* __devres_alloc_node - Allocate device resource data
* @release: Release function devres will be associated with
* @size: Allocation size
* @gfp: Allocation flags
* @nid: NUMA node
* @name: Name of the resource
*
* Allocate devres of @size bytes. The allocated area is zeroed, then
* associated with @release. The returned pointer can be passed to
* other devres_*() functions.
*
* RETURNS:
* Pointer to allocated devres on success, NULL on failure.
*/
void *__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp, int nid,
const char *name)
{
struct devres *dr;
dr = alloc_dr(release, size, gfp | __GFP_ZERO, nid);
if (unlikely(!dr))
return NULL;
set_node_dbginfo(&dr->node, name, size);
return dr->data;
}
EXPORT_SYMBOL_GPL(__devres_alloc_node);
/**
* devres_for_each_res - Resource iterator
* @dev: Device to iterate resource from
* @release: Look for resources associated with this release function
* @match: Match function (optional)
* @match_data: Data for the match function
* @fn: Function to be called for each matched resource.
* @data: Data for @fn, the 3rd parameter of @fn
*
* Call @fn for each devres of @dev which is associated with @release
* and for which @match returns 1.
*
* RETURNS:
* void
*/
void devres_for_each_res(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data,
void (*fn)(struct device *, void *, void *),
void *data)
{
struct devres_node *node;
struct devres_node *tmp;
unsigned long flags;
if (!fn)
return;
spin_lock_irqsave(&dev->devres_lock, flags);
list_for_each_entry_safe_reverse(node, tmp,
&dev->devres_head, entry) {
struct devres *dr = container_of(node, struct devres, node);
if (node->release != release)
continue;
if (match && !match(dev, dr->data, match_data))
continue;
fn(dev, dr->data, data);
}
spin_unlock_irqrestore(&dev->devres_lock, flags);
}
EXPORT_SYMBOL_GPL(devres_for_each_res);
/**
* devres_free - Free device resource data
* @res: Pointer to devres data to free
*
* Free devres created with devres_alloc().
*/
void devres_free(void *res)
{
if (res) {
struct devres *dr = container_of(res, struct devres, data);
BUG_ON(!list_empty(&dr->node.entry));
kfree(dr);
}
}
EXPORT_SYMBOL_GPL(devres_free);
/**
* devres_add - Register device resource
* @dev: Device to add resource to
* @res: Resource to register
*
* Register devres @res to @dev. @res should have been allocated
* using devres_alloc(). On driver detach, the associated release
* function will be invoked and devres will be freed automatically.
*/
void devres_add(struct device *dev, void *res)
{
struct devres *dr = container_of(res, struct devres, data);
unsigned long flags;
spin_lock_irqsave(&dev->devres_lock, flags);
add_dr(dev, &dr->node);
spin_unlock_irqrestore(&dev->devres_lock, flags);
}
EXPORT_SYMBOL_GPL(devres_add);
static struct devres *find_dr(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data)
{
struct devres_node *node;
list_for_each_entry_reverse(node, &dev->devres_head, entry) {
struct devres *dr = container_of(node, struct devres, node);
if (node->release != release)
continue;
if (match && !match(dev, dr->data, match_data))
continue;
return dr;
}
return NULL;
}
/**
* devres_find - Find device resource
* @dev: Device to lookup resource from
* @release: Look for resources associated with this release function
* @match: Match function (optional)
* @match_data: Data for the match function
*
* Find the latest devres of @dev which is associated with @release
* and for which @match returns 1. If @match is NULL, it's considered
* to match all.
*
* RETURNS:
* Pointer to found devres, NULL if not found.
*/
void * devres_find(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data)
{
struct devres *dr;
unsigned long flags;
spin_lock_irqsave(&dev->devres_lock, flags);
dr = find_dr(dev, release, match, match_data);
spin_unlock_irqrestore(&dev->devres_lock, flags);
if (dr)
return dr->data;
return NULL;
}
EXPORT_SYMBOL_GPL(devres_find);
/**
* devres_get - Find devres, if non-existent, add one atomically
* @dev: Device to lookup or add devres for
* @new_res: Pointer to new initialized devres to add if not found
* @match: Match function (optional)
* @match_data: Data for the match function
*
* Find the latest devres of @dev which has the same release function
* as @new_res and for which @match return 1. If found, @new_res is
* freed; otherwise, @new_res is added atomically.
*
* RETURNS:
* Pointer to found or added devres.
*/
void * devres_get(struct device *dev, void *new_res,
dr_match_t match, void *match_data)
{
struct devres *new_dr = container_of(new_res, struct devres, data);
struct devres *dr;
unsigned long flags;
spin_lock_irqsave(&dev->devres_lock, flags);
dr = find_dr(dev, new_dr->node.release, match, match_data);
if (!dr) {
add_dr(dev, &new_dr->node);
dr = new_dr;
new_res = NULL;
}
spin_unlock_irqrestore(&dev->devres_lock, flags);
devres_free(new_res);
return dr->data;
}
EXPORT_SYMBOL_GPL(devres_get);
/**
* devres_remove - Find a device resource and remove it
* @dev: Device to find resource from
* @release: Look for resources associated with this release function
* @match: Match function (optional)
* @match_data: Data for the match function
*
* Find the latest devres of @dev associated with @release and for
* which @match returns 1. If @match is NULL, it's considered to
* match all. If found, the resource is removed atomically and
* returned.
*
* RETURNS:
* Pointer to removed devres on success, NULL if not found.
*/
void * devres_remove(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data)
{
struct devres *dr;
unsigned long flags;
spin_lock_irqsave(&dev->devres_lock, flags);
dr = find_dr(dev, release, match, match_data);
if (dr) {
list_del_init(&dr->node.entry);
devres_log(dev, &dr->node, "REM");
}
spin_unlock_irqrestore(&dev->devres_lock, flags);
if (dr)
return dr->data;
return NULL;
}
EXPORT_SYMBOL_GPL(devres_remove);
/**
* devres_destroy - Find a device resource and destroy it
* @dev: Device to find resource from
* @release: Look for resources associated with this release function
* @match: Match function (optional)
* @match_data: Data for the match function
*
* Find the latest devres of @dev associated with @release and for
* which @match returns 1. If @match is NULL, it's considered to
* match all. If found, the resource is removed atomically and freed.
*
* Note that the release function for the resource will not be called,
* only the devres-allocated data will be freed. The caller becomes
* responsible for freeing any other data.
*
* RETURNS:
* 0 if devres is found and freed, -ENOENT if not found.
*/
int devres_destroy(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data)
{
void *res;
res = devres_remove(dev, release, match, match_data);
if (unlikely(!res))
return -ENOENT;
devres_free(res);
return 0;
}
EXPORT_SYMBOL_GPL(devres_destroy);
/**
* devres_release - Find a device resource and destroy it, calling release
* @dev: Device to find resource from
* @release: Look for resources associated with this release function
* @match: Match function (optional)
* @match_data: Data for the match function
*
* Find the latest devres of @dev associated with @release and for
* which @match returns 1. If @match is NULL, it's considered to
* match all. If found, the resource is removed atomically, the
* release function called and the resource freed.
*
* RETURNS:
* 0 if devres is found and freed, -ENOENT if not found.
*/
int devres_release(struct device *dev, dr_release_t release,
dr_match_t match, void *match_data)
{
void *res;
res = devres_remove(dev, release, match, match_data);
if (unlikely(!res))
return -ENOENT;
(*release)(dev, res);
devres_free(res);
return 0;
}
EXPORT_SYMBOL_GPL(devres_release);
static int remove_nodes(struct device *dev,
struct list_head *first, struct list_head *end,
struct list_head *todo)
{
struct devres_node *node, *n;
int cnt = 0, nr_groups = 0;
/* First pass - move normal devres entries to @todo and clear
* devres_group colors.
*/
node = list_entry(first, struct devres_node, entry);
list_for_each_entry_safe_from(node, n, end, entry) {
struct devres_group *grp;
grp = node_to_group(node); if (grp) {
/* clear color of group markers in the first pass */
grp->color = 0;
nr_groups++;
} else {
/* regular devres entry */
if (&node->entry == first) first = first->next; list_move_tail(&node->entry, todo); cnt++;
}
}
if (!nr_groups)
return cnt;
/* Second pass - Scan groups and color them. A group gets
* color value of two iff the group is wholly contained in
* [current node, end). That is, for a closed group, both opening
* and closing markers should be in the range, while just the
* opening marker is enough for an open group.
*/
node = list_entry(first, struct devres_node, entry);
list_for_each_entry_safe_from(node, n, end, entry) {
struct devres_group *grp;
grp = node_to_group(node); BUG_ON(!grp || list_empty(&grp->node[0].entry)); grp->color++;
if (list_empty(&grp->node[1].entry))
grp->color++; BUG_ON(grp->color <= 0 || grp->color > 2); if (grp->color == 2) {
/* No need to update current node or end. The removed
* nodes are always before both.
*/
list_move_tail(&grp->node[0].entry, todo); list_del_init(&grp->node[1].entry);
}
}
return cnt;
}
static void release_nodes(struct device *dev, struct list_head *todo)
{
struct devres *dr, *tmp;
/* Release. Note that both devres and devres_group are
* handled as devres in the following loop. This is safe.
*/
list_for_each_entry_safe_reverse(dr, tmp, todo, node.entry) { devres_log(dev, &dr->node, "REL");
dr->node.release(dev, dr->data);
kfree(dr);
}
}
/**
* devres_release_all - Release all managed resources
* @dev: Device to release resources for
*
* Release all resources associated with @dev. This function is
* called on driver detach.
*/
int devres_release_all(struct device *dev)
{
unsigned long flags;
LIST_HEAD(todo);
int cnt;
/* Looks like an uninitialized device structure */
if (WARN_ON(dev->devres_head.next == NULL))
return -ENODEV;
/* Nothing to release if list is empty */
if (list_empty(&dev->devres_head))
return 0;
spin_lock_irqsave(&dev->devres_lock, flags);
cnt = remove_nodes(dev, dev->devres_head.next, &dev->devres_head, &todo);
spin_unlock_irqrestore(&dev->devres_lock, flags);
release_nodes(dev, &todo);
return cnt;
}
/**
* devres_open_group - Open a new devres group
* @dev: Device to open devres group for
* @id: Separator ID
* @gfp: Allocation flags
*
* Open a new devres group for @dev with @id. For @id, using a
* pointer to an object which won't be used for another group is
* recommended. If @id is NULL, address-wise unique ID is created.
*
* RETURNS:
* ID of the new group, NULL on failure.
*/
void * devres_open_group(struct device *dev, void *id, gfp_t gfp)
{
struct devres_group *grp;
unsigned long flags;
grp = kmalloc(sizeof(*grp), gfp);
if (unlikely(!grp))
return NULL;
grp->node[0].release = &group_open_release;
grp->node[1].release = &group_close_release;
INIT_LIST_HEAD(&grp->node[0].entry);
INIT_LIST_HEAD(&grp->node[1].entry);
set_node_dbginfo(&grp->node[0], "grp<", 0);
set_node_dbginfo(&grp->node[1], "grp>", 0);
grp->id = grp;
if (id)
grp->id = id;
spin_lock_irqsave(&dev->devres_lock, flags);
add_dr(dev, &grp->node[0]);
spin_unlock_irqrestore(&dev->devres_lock, flags);
return grp->id;
}
EXPORT_SYMBOL_GPL(devres_open_group);
/* Find devres group with ID @id. If @id is NULL, look for the latest. */
static struct devres_group * find_group(struct device *dev, void *id)
{
struct devres_node *node;
list_for_each_entry_reverse(node, &dev->devres_head, entry) {
struct devres_group *grp;
if (node->release != &group_open_release)
continue;
grp = container_of(node, struct devres_group, node[0]);
if (id) { if (grp->id == id)
return grp;
} else if (list_empty(&grp->node[1].entry))
return grp;
}
return NULL;
}
/**
* devres_close_group - Close a devres group
* @dev: Device to close devres group for
* @id: ID of target group, can be NULL
*
* Close the group identified by @id. If @id is NULL, the latest open
* group is selected.
*/
void devres_close_group(struct device *dev, void *id)
{
struct devres_group *grp;
unsigned long flags;
spin_lock_irqsave(&dev->devres_lock, flags);
grp = find_group(dev, id);
if (grp)
add_dr(dev, &grp->node[1]);
else
WARN_ON(1);
spin_unlock_irqrestore(&dev->devres_lock, flags);
}
EXPORT_SYMBOL_GPL(devres_close_group);
/**
* devres_remove_group - Remove a devres group
* @dev: Device to remove group for
* @id: ID of target group, can be NULL
*
* Remove the group identified by @id. If @id is NULL, the latest
* open group is selected. Note that removing a group doesn't affect
* any other resources.
*/
void devres_remove_group(struct device *dev, void *id)
{
struct devres_group *grp;
unsigned long flags;
spin_lock_irqsave(&dev->devres_lock, flags);
grp = find_group(dev, id);
if (grp) {
list_del_init(&grp->node[0].entry);
list_del_init(&grp->node[1].entry);
devres_log(dev, &grp->node[0], "REM");
} else
WARN_ON(1);
spin_unlock_irqrestore(&dev->devres_lock, flags);
kfree(grp);
}
EXPORT_SYMBOL_GPL(devres_remove_group);
/**
* devres_release_group - Release resources in a devres group
* @dev: Device to release group for
* @id: ID of target group, can be NULL
*
* Release all resources in the group identified by @id. If @id is
* NULL, the latest open group is selected. The selected group and
* groups properly nested inside the selected group are removed.
*
* RETURNS:
* The number of released non-group resources.
*/
int devres_release_group(struct device *dev, void *id)
{
struct devres_group *grp;
unsigned long flags;
LIST_HEAD(todo);
int cnt = 0;
spin_lock_irqsave(&dev->devres_lock, flags);
grp = find_group(dev, id); if (grp) { struct list_head *first = &grp->node[0].entry;
struct list_head *end = &dev->devres_head;
if (!list_empty(&grp->node[1].entry))
end = grp->node[1].entry.next; cnt = remove_nodes(dev, first, end, &todo);
spin_unlock_irqrestore(&dev->devres_lock, flags);
release_nodes(dev, &todo);
} else {
WARN_ON(1);
spin_unlock_irqrestore(&dev->devres_lock, flags);
}
return cnt;
}
EXPORT_SYMBOL_GPL(devres_release_group);
/*
* Custom devres actions allow inserting a simple function call
* into the teardown sequence.
*/
struct action_devres {
void *data;
void (*action)(void *);
};
static int devm_action_match(struct device *dev, void *res, void *p)
{
struct action_devres *devres = res;
struct action_devres *target = p;
return devres->action == target->action &&
devres->data == target->data;
}
static void devm_action_release(struct device *dev, void *res)
{
struct action_devres *devres = res;
devres->action(devres->data);
}
/**
* __devm_add_action() - add a custom action to list of managed resources
* @dev: Device that owns the action
* @action: Function that should be called
* @data: Pointer to data passed to @action implementation
* @name: Name of the resource (for debugging purposes)
*
* This adds a custom action to the list of managed resources so that
* it gets executed as part of standard resource unwinding.
*/
int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name)
{
struct action_devres *devres;
devres = __devres_alloc_node(devm_action_release, sizeof(struct action_devres),
GFP_KERNEL, NUMA_NO_NODE, name);
if (!devres)
return -ENOMEM;
devres->data = data;
devres->action = action;
devres_add(dev, devres);
return 0;
}
EXPORT_SYMBOL_GPL(__devm_add_action);
/**
* devm_remove_action() - removes previously added custom action
* @dev: Device that owns the action
* @action: Function implementing the action
* @data: Pointer to data passed to @action implementation
*
* Removes instance of @action previously added by devm_add_action().
* Both action and data should match one of the existing entries.
*/
void devm_remove_action(struct device *dev, void (*action)(void *), void *data)
{
struct action_devres devres = {
.data = data,
.action = action,
};
WARN_ON(devres_destroy(dev, devm_action_release, devm_action_match,
&devres));
}
EXPORT_SYMBOL_GPL(devm_remove_action);
/**
* devm_release_action() - release previously added custom action
* @dev: Device that owns the action
* @action: Function implementing the action
* @data: Pointer to data passed to @action implementation
*
* Releases and removes instance of @action previously added by
* devm_add_action(). Both action and data should match one of the
* existing entries.
*/
void devm_release_action(struct device *dev, void (*action)(void *), void *data)
{
struct action_devres devres = {
.data = data,
.action = action,
};
WARN_ON(devres_release(dev, devm_action_release, devm_action_match,
&devres));
}
EXPORT_SYMBOL_GPL(devm_release_action);
/*
* Managed kmalloc/kfree
*/
static void devm_kmalloc_release(struct device *dev, void *res)
{
/* noop */
}
static int devm_kmalloc_match(struct device *dev, void *res, void *data)
{
return res == data;
}
/**
* devm_kmalloc - Resource-managed kmalloc
* @dev: Device to allocate memory for
* @size: Allocation size
* @gfp: Allocation gfp flags
*
* Managed kmalloc. Memory allocated with this function is
* automatically freed on driver detach. Like all other devres
* resources, guaranteed alignment is unsigned long long.
*
* RETURNS:
* Pointer to allocated memory on success, NULL on failure.
*/
void *devm_kmalloc(struct device *dev, size_t size, gfp_t gfp)
{
struct devres *dr;
if (unlikely(!size))
return ZERO_SIZE_PTR;
/* use raw alloc_dr for kmalloc caller tracing */
dr = alloc_dr(devm_kmalloc_release, size, gfp, dev_to_node(dev));
if (unlikely(!dr))
return NULL;
/*
* This is named devm_kzalloc_release for historical reasons
* The initial implementation did not support kmalloc, only kzalloc
*/
set_node_dbginfo(&dr->node, "devm_kzalloc_release", size);
devres_add(dev, dr->data);
return dr->data;
}
EXPORT_SYMBOL_GPL(devm_kmalloc);
/**
* devm_krealloc - Resource-managed krealloc()
* @dev: Device to re-allocate memory for
* @ptr: Pointer to the memory chunk to re-allocate
* @new_size: New allocation size
* @gfp: Allocation gfp flags
*
* Managed krealloc(). Resizes the memory chunk allocated with devm_kmalloc().
* Behaves similarly to regular krealloc(): if @ptr is NULL or ZERO_SIZE_PTR,
* it's the equivalent of devm_kmalloc(). If new_size is zero, it frees the
* previously allocated memory and returns ZERO_SIZE_PTR. This function doesn't
* change the order in which the release callback for the re-alloc'ed devres
* will be called (except when falling back to devm_kmalloc() or when freeing
* resources when new_size is zero). The contents of the memory are preserved
* up to the lesser of new and old sizes.
*/
void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp)
{
size_t total_new_size, total_old_size;
struct devres *old_dr, *new_dr;
unsigned long flags;
if (unlikely(!new_size)) {
devm_kfree(dev, ptr);
return ZERO_SIZE_PTR;
}
if (unlikely(ZERO_OR_NULL_PTR(ptr)))
return devm_kmalloc(dev, new_size, gfp);
if (WARN_ON(is_kernel_rodata((unsigned long)ptr)))
/*
* We cannot reliably realloc a const string returned by
* devm_kstrdup_const().
*/
return NULL;
if (!check_dr_size(new_size, &total_new_size))
return NULL;
total_old_size = ksize(container_of(ptr, struct devres, data));
if (total_old_size == 0) {
WARN(1, "Pointer doesn't point to dynamically allocated memory.");
return NULL;
}
/*
* If new size is smaller or equal to the actual number of bytes
* allocated previously - just return the same pointer.
*/
if (total_new_size <= total_old_size)
return ptr;
/*
* Otherwise: allocate new, larger chunk. We need to allocate before
* taking the lock as most probably the caller uses GFP_KERNEL.
*/
new_dr = alloc_dr(devm_kmalloc_release,
total_new_size, gfp, dev_to_node(dev));
if (!new_dr)
return NULL;
/*
* The spinlock protects the linked list against concurrent
* modifications but not the resource itself.
*/
spin_lock_irqsave(&dev->devres_lock, flags);
old_dr = find_dr(dev, devm_kmalloc_release, devm_kmalloc_match, ptr);
if (!old_dr) {
spin_unlock_irqrestore(&dev->devres_lock, flags);
kfree(new_dr);
WARN(1, "Memory chunk not managed or managed by a different device.");
return NULL;
}
replace_dr(dev, &old_dr->node, &new_dr->node);
spin_unlock_irqrestore(&dev->devres_lock, flags);
/*
* We can copy the memory contents after releasing the lock as we're
* no longer modifying the list links.
*/
memcpy(new_dr->data, old_dr->data,
total_old_size - offsetof(struct devres, data));
/*
* Same for releasing the old devres - it's now been removed from the
* list. This is also the reason why we must not use devm_kfree() - the
* links are no longer valid.
*/
kfree(old_dr);
return new_dr->data;
}
EXPORT_SYMBOL_GPL(devm_krealloc);
/**
* devm_kstrdup - Allocate resource managed space and
* copy an existing string into that.
* @dev: Device to allocate memory for
* @s: the string to duplicate
* @gfp: the GFP mask used in the devm_kmalloc() call when
* allocating memory
* RETURNS:
* Pointer to allocated string on success, NULL on failure.
*/
char *devm_kstrdup(struct device *dev, const char *s, gfp_t gfp)
{
size_t size;
char *buf;
if (!s)
return NULL;
size = strlen(s) + 1;
buf = devm_kmalloc(dev, size, gfp);
if (buf)
memcpy(buf, s, size);
return buf;
}
EXPORT_SYMBOL_GPL(devm_kstrdup);
/**
* devm_kstrdup_const - resource managed conditional string duplication
* @dev: device for which to duplicate the string
* @s: the string to duplicate
* @gfp: the GFP mask used in the kmalloc() call when allocating memory
*
* Strings allocated by devm_kstrdup_const will be automatically freed when
* the associated device is detached.
*
* RETURNS:
* Source string if it is in .rodata section otherwise it falls back to
* devm_kstrdup.
*/
const char *devm_kstrdup_const(struct device *dev, const char *s, gfp_t gfp)
{
if (is_kernel_rodata((unsigned long)s))
return s;
return devm_kstrdup(dev, s, gfp);
}
EXPORT_SYMBOL_GPL(devm_kstrdup_const);
/**
* devm_kvasprintf - Allocate resource managed space and format a string
* into that.
* @dev: Device to allocate memory for
* @gfp: the GFP mask used in the devm_kmalloc() call when
* allocating memory
* @fmt: The printf()-style format string
* @ap: Arguments for the format string
* RETURNS:
* Pointer to allocated string on success, NULL on failure.
*/
char *devm_kvasprintf(struct device *dev, gfp_t gfp, const char *fmt,
va_list ap)
{
unsigned int len;
char *p;
va_list aq;
va_copy(aq, ap);
len = vsnprintf(NULL, 0, fmt, aq);
va_end(aq);
p = devm_kmalloc(dev, len+1, gfp);
if (!p)
return NULL;
vsnprintf(p, len+1, fmt, ap);
return p;
}
EXPORT_SYMBOL(devm_kvasprintf);
/**
* devm_kasprintf - Allocate resource managed space and format a string
* into that.
* @dev: Device to allocate memory for
* @gfp: the GFP mask used in the devm_kmalloc() call when
* allocating memory
* @fmt: The printf()-style format string
* @...: Arguments for the format string
* RETURNS:
* Pointer to allocated string on success, NULL on failure.
*/
char *devm_kasprintf(struct device *dev, gfp_t gfp, const char *fmt, ...)
{
va_list ap;
char *p;
va_start(ap, fmt);
p = devm_kvasprintf(dev, gfp, fmt, ap);
va_end(ap);
return p;
}
EXPORT_SYMBOL_GPL(devm_kasprintf);
/**
* devm_kfree - Resource-managed kfree
* @dev: Device this memory belongs to
* @p: Memory to free
*
* Free memory allocated with devm_kmalloc().
*/
void devm_kfree(struct device *dev, const void *p)
{
int rc;
/*
* Special cases: pointer to a string in .rodata returned by
* devm_kstrdup_const() or NULL/ZERO ptr.
*/
if (unlikely(is_kernel_rodata((unsigned long)p) || ZERO_OR_NULL_PTR(p)))
return;
rc = devres_destroy(dev, devm_kmalloc_release,
devm_kmalloc_match, (void *)p);
WARN_ON(rc);
}
EXPORT_SYMBOL_GPL(devm_kfree);
/**
* devm_kmemdup - Resource-managed kmemdup
* @dev: Device this memory belongs to
* @src: Memory region to duplicate
* @len: Memory region length
* @gfp: GFP mask to use
*
* Duplicate region of a memory using resource managed kmalloc
*/
void *devm_kmemdup(struct device *dev, const void *src, size_t len, gfp_t gfp)
{
void *p;
p = devm_kmalloc(dev, len, gfp);
if (p)
memcpy(p, src, len);
return p;
}
EXPORT_SYMBOL_GPL(devm_kmemdup);
struct pages_devres {
unsigned long addr;
unsigned int order;
};
static int devm_pages_match(struct device *dev, void *res, void *p)
{
struct pages_devres *devres = res;
struct pages_devres *target = p;
return devres->addr == target->addr;
}
static void devm_pages_release(struct device *dev, void *res)
{
struct pages_devres *devres = res;
free_pages(devres->addr, devres->order);
}
/**
* devm_get_free_pages - Resource-managed __get_free_pages
* @dev: Device to allocate memory for
* @gfp_mask: Allocation gfp flags
* @order: Allocation size is (1 << order) pages
*
* Managed get_free_pages. Memory allocated with this function is
* automatically freed on driver detach.
*
* RETURNS:
* Address of allocated memory on success, 0 on failure.
*/
unsigned long devm_get_free_pages(struct device *dev,
gfp_t gfp_mask, unsigned int order)
{
struct pages_devres *devres;
unsigned long addr;
addr = __get_free_pages(gfp_mask, order);
if (unlikely(!addr))
return 0;
devres = devres_alloc(devm_pages_release,
sizeof(struct pages_devres), GFP_KERNEL);
if (unlikely(!devres)) {
free_pages(addr, order);
return 0;
}
devres->addr = addr;
devres->order = order;
devres_add(dev, devres);
return addr;
}
EXPORT_SYMBOL_GPL(devm_get_free_pages);
/**
* devm_free_pages - Resource-managed free_pages
* @dev: Device this memory belongs to
* @addr: Memory to free
*
* Free memory allocated with devm_get_free_pages(). Unlike free_pages,
* there is no need to supply the @order.
*/
void devm_free_pages(struct device *dev, unsigned long addr)
{
struct pages_devres devres = { .addr = addr };
WARN_ON(devres_release(dev, devm_pages_release, devm_pages_match,
&devres));
}
EXPORT_SYMBOL_GPL(devm_free_pages);
static void devm_percpu_release(struct device *dev, void *pdata)
{
void __percpu *p;
p = *(void __percpu **)pdata;
free_percpu(p);
}
static int devm_percpu_match(struct device *dev, void *data, void *p)
{
struct devres *devr = container_of(data, struct devres, data);
return *(void **)devr->data == p;
}
/**
* __devm_alloc_percpu - Resource-managed alloc_percpu
* @dev: Device to allocate per-cpu memory for
* @size: Size of per-cpu memory to allocate
* @align: Alignment of per-cpu memory to allocate
*
* Managed alloc_percpu. Per-cpu memory allocated with this function is
* automatically freed on driver detach.
*
* RETURNS:
* Pointer to allocated memory on success, NULL on failure.
*/
void __percpu *__devm_alloc_percpu(struct device *dev, size_t size,
size_t align)
{
void *p;
void __percpu *pcpu;
pcpu = __alloc_percpu(size, align);
if (!pcpu)
return NULL;
p = devres_alloc(devm_percpu_release, sizeof(void *), GFP_KERNEL);
if (!p) {
free_percpu(pcpu);
return NULL;
}
*(void __percpu **)p = pcpu;
devres_add(dev, p);
return pcpu;
}
EXPORT_SYMBOL_GPL(__devm_alloc_percpu);
/**
* devm_free_percpu - Resource-managed free_percpu
* @dev: Device this memory belongs to
* @pdata: Per-cpu memory to free
*
* Free memory allocated with devm_alloc_percpu().
*/
void devm_free_percpu(struct device *dev, void __percpu *pdata)
{
WARN_ON(devres_destroy(dev, devm_percpu_release, devm_percpu_match,
(__force void *)pdata));
}
EXPORT_SYMBOL_GPL(devm_free_percpu);
// SPDX-License-Identifier: GPL-2.0+
/*
* Copyright (c) 2001-2002 by David Brownell
*/
#ifndef __USB_CORE_HCD_H
#define __USB_CORE_HCD_H
#ifdef __KERNEL__
#include <linux/rwsem.h>
#include <linux/interrupt.h>
#include <linux/idr.h>
#define MAX_TOPO_LEVEL 6
/* This file contains declarations of usbcore internals that are mostly
* used or exposed by Host Controller Drivers.
*/
/*
* USB Packet IDs (PIDs)
*/
#define USB_PID_EXT 0xf0 /* USB 2.0 LPM ECN */
#define USB_PID_OUT 0xe1
#define USB_PID_ACK 0xd2
#define USB_PID_DATA0 0xc3
#define USB_PID_PING 0xb4 /* USB 2.0 */
#define USB_PID_SOF 0xa5
#define USB_PID_NYET 0x96 /* USB 2.0 */
#define USB_PID_DATA2 0x87 /* USB 2.0 */
#define USB_PID_SPLIT 0x78 /* USB 2.0 */
#define USB_PID_IN 0x69
#define USB_PID_NAK 0x5a
#define USB_PID_DATA1 0x4b
#define USB_PID_PREAMBLE 0x3c /* Token mode */
#define USB_PID_ERR 0x3c /* USB 2.0: handshake mode */
#define USB_PID_SETUP 0x2d
#define USB_PID_STALL 0x1e
#define USB_PID_MDATA 0x0f /* USB 2.0 */
/*-------------------------------------------------------------------------*/
/*
* USB Host Controller Driver (usb_hcd) framework
*
* Since "struct usb_bus" is so thin, you can't share much code in it.
* This framework is a layer over that, and should be more shareable.
*/
/*-------------------------------------------------------------------------*/
struct giveback_urb_bh {
bool running;
bool high_prio;
spinlock_t lock;
struct list_head head;
struct work_struct bh;
struct usb_host_endpoint *completing_ep;
};
enum usb_dev_authorize_policy {
USB_DEVICE_AUTHORIZE_NONE = 0,
USB_DEVICE_AUTHORIZE_ALL = 1,
USB_DEVICE_AUTHORIZE_INTERNAL = 2,
};
struct usb_hcd {
/*
* housekeeping
*/
struct usb_bus self; /* hcd is-a bus */
struct kref kref; /* reference counter */
const char *product_desc; /* product/vendor string */
int speed; /* Speed for this roothub.
* May be different from
* hcd->driver->flags & HCD_MASK
*/
char irq_descr[24]; /* driver + bus # */
struct timer_list rh_timer; /* drives root-hub polling */
struct urb *status_urb; /* the current status urb */
#ifdef CONFIG_PM
struct work_struct wakeup_work; /* for remote wakeup */
#endif
struct work_struct died_work; /* for when the device dies */
/*
* hardware info/state
*/
const struct hc_driver *driver; /* hw-specific hooks */
/*
* OTG and some Host controllers need software interaction with phys;
* other external phys should be software-transparent
*/
struct usb_phy *usb_phy;
struct usb_phy_roothub *phy_roothub;
/* Flags that need to be manipulated atomically because they can
* change while the host controller is running. Always use
* set_bit() or clear_bit() to change their values.
*/
unsigned long flags;
#define HCD_FLAG_HW_ACCESSIBLE 0 /* at full power */
#define HCD_FLAG_POLL_RH 2 /* poll for rh status? */
#define HCD_FLAG_POLL_PENDING 3 /* status has changed? */
#define HCD_FLAG_WAKEUP_PENDING 4 /* root hub is resuming? */
#define HCD_FLAG_RH_RUNNING 5 /* root hub is running? */
#define HCD_FLAG_DEAD 6 /* controller has died? */
#define HCD_FLAG_INTF_AUTHORIZED 7 /* authorize interfaces? */
#define HCD_FLAG_DEFER_RH_REGISTER 8 /* Defer roothub registration */
/* The flags can be tested using these macros; they are likely to
* be slightly faster than test_bit().
*/
#define HCD_HW_ACCESSIBLE(hcd) ((hcd)->flags & (1U << HCD_FLAG_HW_ACCESSIBLE))
#define HCD_POLL_RH(hcd) ((hcd)->flags & (1U << HCD_FLAG_POLL_RH))
#define HCD_POLL_PENDING(hcd) ((hcd)->flags & (1U << HCD_FLAG_POLL_PENDING))
#define HCD_WAKEUP_PENDING(hcd) ((hcd)->flags & (1U << HCD_FLAG_WAKEUP_PENDING))
#define HCD_RH_RUNNING(hcd) ((hcd)->flags & (1U << HCD_FLAG_RH_RUNNING))
#define HCD_DEAD(hcd) ((hcd)->flags & (1U << HCD_FLAG_DEAD))
#define HCD_DEFER_RH_REGISTER(hcd) ((hcd)->flags & (1U << HCD_FLAG_DEFER_RH_REGISTER))
/*
* Specifies if interfaces are authorized by default
* or they require explicit user space authorization; this bit is
* settable through /sys/class/usb_host/X/interface_authorized_default
*/
#define HCD_INTF_AUTHORIZED(hcd) \
((hcd)->flags & (1U << HCD_FLAG_INTF_AUTHORIZED))
/*
* Specifies if devices are authorized by default
* or they require explicit user space authorization; this bit is
* settable through /sys/class/usb_host/X/authorized_default
*/
enum usb_dev_authorize_policy dev_policy;
/* Flags that get set only during HCD registration or removal. */
unsigned rh_registered:1;/* is root hub registered? */
unsigned rh_pollable:1; /* may we poll the root hub? */
unsigned msix_enabled:1; /* driver has MSI-X enabled? */
unsigned msi_enabled:1; /* driver has MSI enabled? */
/*
* do not manage the PHY state in the HCD core, instead let the driver
* handle this (for example if the PHY can only be turned on after a
* specific event)
*/
unsigned skip_phy_initialization:1;
/* The next flag is a stopgap, to be removed when all the HCDs
* support the new root-hub polling mechanism. */
unsigned uses_new_polling:1;
unsigned has_tt:1; /* Integrated TT in root hub */
unsigned amd_resume_bug:1; /* AMD remote wakeup quirk */
unsigned can_do_streams:1; /* HC supports streams */
unsigned tpl_support:1; /* OTG & EH TPL support */
unsigned cant_recv_wakeups:1;
/* wakeup requests from downstream aren't received */
unsigned int irq; /* irq allocated */
void __iomem *regs; /* device memory/io */
resource_size_t rsrc_start; /* memory/io resource start */
resource_size_t rsrc_len; /* memory/io resource length */
unsigned power_budget; /* in mA, 0 = no limit */
struct giveback_urb_bh high_prio_bh;
struct giveback_urb_bh low_prio_bh;
/* bandwidth_mutex should be taken before adding or removing
* any new bus bandwidth constraints:
* 1. Before adding a configuration for a new device.
* 2. Before removing the configuration to put the device into
* the addressed state.
* 3. Before selecting a different configuration.
* 4. Before selecting an alternate interface setting.
*
* bandwidth_mutex should be dropped after a successful control message
* to the device, or resetting the bandwidth after a failed attempt.
*/
struct mutex *address0_mutex;
struct mutex *bandwidth_mutex;
struct usb_hcd *shared_hcd;
struct usb_hcd *primary_hcd;
#define HCD_BUFFER_POOLS 4
struct dma_pool *pool[HCD_BUFFER_POOLS];
int state;
# define __ACTIVE 0x01
# define __SUSPEND 0x04
# define __TRANSIENT 0x80
# define HC_STATE_HALT 0
# define HC_STATE_RUNNING (__ACTIVE)
# define HC_STATE_QUIESCING (__SUSPEND|__TRANSIENT|__ACTIVE)
# define HC_STATE_RESUMING (__SUSPEND|__TRANSIENT)
# define HC_STATE_SUSPENDED (__SUSPEND)
#define HC_IS_RUNNING(state) ((state) & __ACTIVE)
#define HC_IS_SUSPENDED(state) ((state) & __SUSPEND)
/* memory pool for HCs having local memory, or %NULL */
struct gen_pool *localmem_pool;
/* more shared queuing code would be good; it should support
* smarter scheduling, handle transaction translators, etc;
* input size of periodic table to an interrupt scheduler.
* (ohci 32, uhci 1024, ehci 256/512/1024).
*/
/* The HC driver's private data is stored at the end of
* this structure.
*/
unsigned long hcd_priv[]
__attribute__ ((aligned(sizeof(s64))));
};
/* 2.4 does this a bit differently ... */
static inline struct usb_bus *hcd_to_bus(struct usb_hcd *hcd)
{
return &hcd->self;
}
static inline struct usb_hcd *bus_to_hcd(struct usb_bus *bus)
{
return container_of(bus, struct usb_hcd, self);
}
/*-------------------------------------------------------------------------*/
struct hc_driver {
const char *description; /* "ehci-hcd" etc */
const char *product_desc; /* product/vendor string */
size_t hcd_priv_size; /* size of private data */
/* irq handler */
irqreturn_t (*irq) (struct usb_hcd *hcd);
int flags;
#define HCD_MEMORY 0x0001 /* HC regs use memory (else I/O) */
#define HCD_DMA 0x0002 /* HC uses DMA */
#define HCD_SHARED 0x0004 /* Two (or more) usb_hcds share HW */
#define HCD_USB11 0x0010 /* USB 1.1 */
#define HCD_USB2 0x0020 /* USB 2.0 */
#define HCD_USB3 0x0040 /* USB 3.0 */
#define HCD_USB31 0x0050 /* USB 3.1 */
#define HCD_USB32 0x0060 /* USB 3.2 */
#define HCD_MASK 0x0070
#define HCD_BH 0x0100 /* URB complete in BH context */
/* called to init HCD and root hub */
int (*reset) (struct usb_hcd *hcd);
int (*start) (struct usb_hcd *hcd);
/* NOTE: these suspend/resume calls relate to the HC as
* a whole, not just the root hub; they're for PCI bus glue.
*/
/* called after suspending the hub, before entering D3 etc */
int (*pci_suspend)(struct usb_hcd *hcd, bool do_wakeup);
/* called after entering D0 (etc), before resuming the hub */
int (*pci_resume)(struct usb_hcd *hcd, pm_message_t state);
/* called just before hibernate final D3 state, allows host to poweroff parts */
int (*pci_poweroff_late)(struct usb_hcd *hcd, bool do_wakeup);
/* cleanly make HCD stop writing memory and doing I/O */
void (*stop) (struct usb_hcd *hcd);
/* shutdown HCD */
void (*shutdown) (struct usb_hcd *hcd);
/* return current frame number */
int (*get_frame_number) (struct usb_hcd *hcd);
/* manage i/o requests, device state */
int (*urb_enqueue)(struct usb_hcd *hcd,
struct urb *urb, gfp_t mem_flags);
int (*urb_dequeue)(struct usb_hcd *hcd,
struct urb *urb, int status);
/*
* (optional) these hooks allow an HCD to override the default DMA
* mapping and unmapping routines. In general, they shouldn't be
* necessary unless the host controller has special DMA requirements,
* such as alignment constraints. If these are not specified, the
* general usb_hcd_(un)?map_urb_for_dma functions will be used instead
* (and it may be a good idea to call these functions in your HCD
* implementation)
*/
int (*map_urb_for_dma)(struct usb_hcd *hcd, struct urb *urb,
gfp_t mem_flags);
void (*unmap_urb_for_dma)(struct usb_hcd *hcd, struct urb *urb);
/* hw synch, freeing endpoint resources that urb_dequeue can't */
void (*endpoint_disable)(struct usb_hcd *hcd,
struct usb_host_endpoint *ep);
/* (optional) reset any endpoint state such as sequence number
and current window */
void (*endpoint_reset)(struct usb_hcd *hcd,
struct usb_host_endpoint *ep);
/* root hub support */
int (*hub_status_data) (struct usb_hcd *hcd, char *buf);
int (*hub_control) (struct usb_hcd *hcd,
u16 typeReq, u16 wValue, u16 wIndex,
char *buf, u16 wLength);
int (*bus_suspend)(struct usb_hcd *);
int (*bus_resume)(struct usb_hcd *);
int (*start_port_reset)(struct usb_hcd *, unsigned port_num);
unsigned long (*get_resuming_ports)(struct usb_hcd *);
/* force handover of high-speed port to full-speed companion */
void (*relinquish_port)(struct usb_hcd *, int);
/* has a port been handed over to a companion? */
int (*port_handed_over)(struct usb_hcd *, int);
/* CLEAR_TT_BUFFER completion callback */
void (*clear_tt_buffer_complete)(struct usb_hcd *,
struct usb_host_endpoint *);
/* xHCI specific functions */
/* Called by usb_alloc_dev to alloc HC device structures */
int (*alloc_dev)(struct usb_hcd *, struct usb_device *);
/* Called by usb_disconnect to free HC device structures */
void (*free_dev)(struct usb_hcd *, struct usb_device *);
/* Change a group of bulk endpoints to support multiple stream IDs */
int (*alloc_streams)(struct usb_hcd *hcd, struct usb_device *udev,
struct usb_host_endpoint **eps, unsigned int num_eps,
unsigned int num_streams, gfp_t mem_flags);
/* Reverts a group of bulk endpoints back to not using stream IDs.
* Can fail if we run out of memory.
*/
int (*free_streams)(struct usb_hcd *hcd, struct usb_device *udev,
struct usb_host_endpoint **eps, unsigned int num_eps,
gfp_t mem_flags);
/* Bandwidth computation functions */
/* Note that add_endpoint() can only be called once per endpoint before
* check_bandwidth() or reset_bandwidth() must be called.
* drop_endpoint() can only be called once per endpoint also.
* A call to xhci_drop_endpoint() followed by a call to
* xhci_add_endpoint() will add the endpoint to the schedule with
* possibly new parameters denoted by a different endpoint descriptor
* in usb_host_endpoint. A call to xhci_add_endpoint() followed by a
* call to xhci_drop_endpoint() is not allowed.
*/
/* Allocate endpoint resources and add them to a new schedule */
int (*add_endpoint)(struct usb_hcd *, struct usb_device *,
struct usb_host_endpoint *);
/* Drop an endpoint from a new schedule */
int (*drop_endpoint)(struct usb_hcd *, struct usb_device *,
struct usb_host_endpoint *);
/* Check that a new hardware configuration, set using
* endpoint_enable and endpoint_disable, does not exceed bus
* bandwidth. This must be called before any set configuration
* or set interface requests are sent to the device.
*/
int (*check_bandwidth)(struct usb_hcd *, struct usb_device *);
/* Reset the device schedule to the last known good schedule,
* which was set from a previous successful call to
* check_bandwidth(). This reverts any add_endpoint() and
* drop_endpoint() calls since that last successful call.
* Used for when a check_bandwidth() call fails due to resource
* or bandwidth constraints.
*/
void (*reset_bandwidth)(struct usb_hcd *, struct usb_device *);
/* Set the hardware-chosen device address */
int (*address_device)(struct usb_hcd *, struct usb_device *udev,
unsigned int timeout_ms);
/* prepares the hardware to send commands to the device */
int (*enable_device)(struct usb_hcd *, struct usb_device *udev);
/* Notifies the HCD after a hub descriptor is fetched.
* Will block.
*/
int (*update_hub_device)(struct usb_hcd *, struct usb_device *hdev,
struct usb_tt *tt, gfp_t mem_flags);
int (*reset_device)(struct usb_hcd *, struct usb_device *);
/* Notifies the HCD after a device is connected and its
* address is set
*/
int (*update_device)(struct usb_hcd *, struct usb_device *);
int (*set_usb2_hw_lpm)(struct usb_hcd *, struct usb_device *, int);
/* USB 3.0 Link Power Management */
/* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
int (*enable_usb3_lpm_timeout)(struct usb_hcd *,
struct usb_device *, enum usb3_link_state state);
/* The xHCI host controller can still fail the command to
* disable the LPM timeouts, so this can return an error code.
*/
int (*disable_usb3_lpm_timeout)(struct usb_hcd *,
struct usb_device *, enum usb3_link_state state);
int (*find_raw_port_number)(struct usb_hcd *, int);
/* Call for power on/off the port if necessary */
int (*port_power)(struct usb_hcd *hcd, int portnum, bool enable);
/* Call for SINGLE_STEP_SET_FEATURE Test for USB2 EH certification */
#define EHSET_TEST_SINGLE_STEP_SET_FEATURE 0x06
int (*submit_single_step_set_feature)(struct usb_hcd *,
struct urb *, int);
};
static inline int hcd_giveback_urb_in_bh(struct usb_hcd *hcd)
{
return hcd->driver->flags & HCD_BH;
}
static inline bool hcd_periodic_completion_in_progress(struct usb_hcd *hcd,
struct usb_host_endpoint *ep)
{
return hcd->high_prio_bh.completing_ep == ep;
}
static inline bool hcd_uses_dma(struct usb_hcd *hcd)
{
return IS_ENABLED(CONFIG_HAS_DMA) && (hcd->driver->flags & HCD_DMA);
}
extern int usb_hcd_link_urb_to_ep(struct usb_hcd *hcd, struct urb *urb);
extern int usb_hcd_check_unlink_urb(struct usb_hcd *hcd, struct urb *urb,
int status);
extern void usb_hcd_unlink_urb_from_ep(struct usb_hcd *hcd, struct urb *urb);
extern int usb_hcd_submit_urb(struct urb *urb, gfp_t mem_flags);
extern int usb_hcd_unlink_urb(struct urb *urb, int status);
extern void usb_hcd_giveback_urb(struct usb_hcd *hcd, struct urb *urb,
int status);
extern int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
gfp_t mem_flags);
extern void usb_hcd_unmap_urb_setup_for_dma(struct usb_hcd *, struct urb *);
extern void usb_hcd_unmap_urb_for_dma(struct usb_hcd *, struct urb *);
extern void usb_hcd_flush_endpoint(struct usb_device *udev,
struct usb_host_endpoint *ep);
extern void usb_hcd_disable_endpoint(struct usb_device *udev,
struct usb_host_endpoint *ep);
extern void usb_hcd_reset_endpoint(struct usb_device *udev,
struct usb_host_endpoint *ep);
extern void usb_hcd_synchronize_unlinks(struct usb_device *udev);
extern int usb_hcd_alloc_bandwidth(struct usb_device *udev,
struct usb_host_config *new_config,
struct usb_host_interface *old_alt,
struct usb_host_interface *new_alt);
extern int usb_hcd_get_frame_number(struct usb_device *udev);
struct usb_hcd *__usb_create_hcd(const struct hc_driver *driver,
struct device *sysdev, struct device *dev, const char *bus_name,
struct usb_hcd *primary_hcd);
extern struct usb_hcd *usb_create_hcd(const struct hc_driver *driver,
struct device *dev, const char *bus_name);
extern struct usb_hcd *usb_create_shared_hcd(const struct hc_driver *driver,
struct device *dev, const char *bus_name,
struct usb_hcd *shared_hcd);
extern struct usb_hcd *usb_get_hcd(struct usb_hcd *hcd);
extern void usb_put_hcd(struct usb_hcd *hcd);
extern int usb_hcd_is_primary_hcd(struct usb_hcd *hcd);
extern int usb_add_hcd(struct usb_hcd *hcd,
unsigned int irqnum, unsigned long irqflags);
extern void usb_remove_hcd(struct usb_hcd *hcd);
extern int usb_hcd_find_raw_port_number(struct usb_hcd *hcd, int port1);
int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr,
dma_addr_t dma, size_t size);
struct platform_device;
extern void usb_hcd_platform_shutdown(struct platform_device *dev);
#ifdef CONFIG_USB_HCD_TEST_MODE
extern int ehset_single_step_set_feature(struct usb_hcd *hcd, int port);
#else
static inline int ehset_single_step_set_feature(struct usb_hcd *hcd, int port)
{
return 0;
}
#endif /* CONFIG_USB_HCD_TEST_MODE */
#ifdef CONFIG_USB_PCI
struct pci_dev;
struct pci_device_id;
extern int usb_hcd_pci_probe(struct pci_dev *dev,
const struct hc_driver *driver);
extern void usb_hcd_pci_remove(struct pci_dev *dev);
extern void usb_hcd_pci_shutdown(struct pci_dev *dev);
#ifdef CONFIG_USB_PCI_AMD
extern int usb_hcd_amd_remote_wakeup_quirk(struct pci_dev *dev);
static inline bool usb_hcd_amd_resume_bug(struct pci_dev *dev,
const struct hc_driver *driver)
{
if (!usb_hcd_amd_remote_wakeup_quirk(dev))
return false;
if (driver->flags & (HCD_USB11 | HCD_USB3))
return true;
return false;
}
#else /* CONFIG_USB_PCI_AMD */
static inline bool usb_hcd_amd_resume_bug(struct pci_dev *dev,
const struct hc_driver *driver)
{
return false;
}
#endif
extern const struct dev_pm_ops usb_hcd_pci_pm_ops;
#endif /* CONFIG_USB_PCI */
/* pci-ish (pdev null is ok) buffer alloc/mapping support */
void usb_init_pool_max(void);
int hcd_buffer_create(struct usb_hcd *hcd);
void hcd_buffer_destroy(struct usb_hcd *hcd);
void *hcd_buffer_alloc(struct usb_bus *bus, size_t size,
gfp_t mem_flags, dma_addr_t *dma);
void hcd_buffer_free(struct usb_bus *bus, size_t size,
void *addr, dma_addr_t dma);
void *hcd_buffer_alloc_pages(struct usb_hcd *hcd,
size_t size, gfp_t mem_flags, dma_addr_t *dma);
void hcd_buffer_free_pages(struct usb_hcd *hcd,
size_t size, void *addr, dma_addr_t dma);
/* generic bus glue, needed for host controllers that don't use PCI */
extern irqreturn_t usb_hcd_irq(int irq, void *__hcd);
extern void usb_hc_died(struct usb_hcd *hcd);
extern void usb_hcd_poll_rh_status(struct usb_hcd *hcd);
extern void usb_wakeup_notification(struct usb_device *hdev,
unsigned int portnum);
extern void usb_hcd_start_port_resume(struct usb_bus *bus, int portnum);
extern void usb_hcd_end_port_resume(struct usb_bus *bus, int portnum);
/* The D0/D1 toggle bits ... USE WITH CAUTION (they're almost hcd-internal) */
#define usb_gettoggle(dev, ep, out) (((dev)->toggle[out] >> (ep)) & 1)
#define usb_dotoggle(dev, ep, out) ((dev)->toggle[out] ^= (1 << (ep)))
#define usb_settoggle(dev, ep, out, bit) \
((dev)->toggle[out] = ((dev)->toggle[out] & ~(1 << (ep))) | \
((bit) << (ep)))
/* -------------------------------------------------------------------------- */
/* Enumeration is only for the hub driver, or HCD virtual root hubs */
extern struct usb_device *usb_alloc_dev(struct usb_device *parent,
struct usb_bus *, unsigned port);
extern int usb_new_device(struct usb_device *dev);
extern void usb_disconnect(struct usb_device **);
extern int usb_get_configuration(struct usb_device *dev);
extern void usb_destroy_configuration(struct usb_device *dev);
/*-------------------------------------------------------------------------*/
/*
* HCD Root Hub support
*/
#include <linux/usb/ch11.h>
/*
* As of USB 2.0, full/low speed devices are segregated into trees.
* One type grows from USB 1.1 host controllers (OHCI, UHCI etc).
* The other type grows from high speed hubs when they connect to
* full/low speed devices using "Transaction Translators" (TTs).
*
* TTs should only be known to the hub driver, and high speed bus
* drivers (only EHCI for now). They affect periodic scheduling and
* sometimes control/bulk error recovery.
*/
struct usb_device;
struct usb_tt {
struct usb_device *hub; /* upstream highspeed hub */
int multi; /* true means one TT per port */
unsigned think_time; /* think time in ns */
void *hcpriv; /* HCD private data */
/* for control/bulk error recovery (CLEAR_TT_BUFFER) */
spinlock_t lock;
struct list_head clear_list; /* of usb_tt_clear */
struct work_struct clear_work;
};
struct usb_tt_clear {
struct list_head clear_list;
unsigned tt;
u16 devinfo;
struct usb_hcd *hcd;
struct usb_host_endpoint *ep;
};
extern int usb_hub_clear_tt_buffer(struct urb *urb);
extern void usb_ep0_reinit(struct usb_device *);
/* (shifted) direction/type/recipient from the USB 2.0 spec, table 9.2 */
#define DeviceRequest \
((USB_DIR_IN|USB_TYPE_STANDARD|USB_RECIP_DEVICE)<<8)
#define DeviceOutRequest \
((USB_DIR_OUT|USB_TYPE_STANDARD|USB_RECIP_DEVICE)<<8)
#define InterfaceRequest \
((USB_DIR_IN|USB_TYPE_STANDARD|USB_RECIP_INTERFACE)<<8)
#define EndpointRequest \
((USB_DIR_IN|USB_TYPE_STANDARD|USB_RECIP_ENDPOINT)<<8)
#define EndpointOutRequest \
((USB_DIR_OUT|USB_TYPE_STANDARD|USB_RECIP_ENDPOINT)<<8)
/* class requests from the USB 2.0 hub spec, table 11-15 */
#define HUB_CLASS_REQ(dir, type, request) ((((dir) | (type)) << 8) | (request))
/* GetBusState and SetHubDescriptor are optional, omitted */
#define ClearHubFeature HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_HUB, USB_REQ_CLEAR_FEATURE)
#define ClearPortFeature HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, USB_REQ_CLEAR_FEATURE)
#define GetHubDescriptor HUB_CLASS_REQ(USB_DIR_IN, USB_RT_HUB, USB_REQ_GET_DESCRIPTOR)
#define GetHubStatus HUB_CLASS_REQ(USB_DIR_IN, USB_RT_HUB, USB_REQ_GET_STATUS)
#define GetPortStatus HUB_CLASS_REQ(USB_DIR_IN, USB_RT_PORT, USB_REQ_GET_STATUS)
#define SetHubFeature HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_HUB, USB_REQ_SET_FEATURE)
#define SetPortFeature HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, USB_REQ_SET_FEATURE)
#define ClearTTBuffer HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, HUB_CLEAR_TT_BUFFER)
#define ResetTT HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, HUB_RESET_TT)
#define GetTTState HUB_CLASS_REQ(USB_DIR_IN, USB_RT_PORT, HUB_GET_TT_STATE)
#define StopTT HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, HUB_STOP_TT)
/*-------------------------------------------------------------------------*/
/* class requests from USB 3.1 hub spec, table 10-7 */
#define SetHubDepth HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_HUB, HUB_SET_DEPTH)
#define GetPortErrorCount HUB_CLASS_REQ(USB_DIR_IN, USB_RT_PORT, HUB_GET_PORT_ERR_COUNT)
/*
* Generic bandwidth allocation constants/support
*/
#define FRAME_TIME_USECS 1000L
#define BitTime(bytecount) (7 * 8 * bytecount / 6) /* with integer truncation */
/* Trying not to use worst-case bit-stuffing
* of (7/6 * 8 * bytecount) = 9.33 * bytecount */
/* bytecount = data payload byte count */
#define NS_TO_US(ns) DIV_ROUND_UP(ns, 1000L)
/* convert nanoseconds to microseconds, rounding up */
/*
* Full/low speed bandwidth allocation constants/support.
*/
#define BW_HOST_DELAY 1000L /* nanoseconds */
#define BW_HUB_LS_SETUP 333L /* nanoseconds */
/* 4 full-speed bit times (est.) */
#define FRAME_TIME_BITS 12000L /* frame = 1 millisecond */
#define FRAME_TIME_MAX_BITS_ALLOC (90L * FRAME_TIME_BITS / 100L)
#define FRAME_TIME_MAX_USECS_ALLOC (90L * FRAME_TIME_USECS / 100L)
/*
* Ceiling [nano/micro]seconds (typical) for that many bytes at high speed
* ISO is a bit less, no ACK ... from USB 2.0 spec, 5.11.3 (and needed
* to preallocate bandwidth)
*/
#define USB2_HOST_DELAY 5 /* nsec, guess */
#define HS_NSECS(bytes) (((55 * 8 * 2083) \
+ (2083UL * (3 + BitTime(bytes))))/1000 \
+ USB2_HOST_DELAY)
#define HS_NSECS_ISO(bytes) (((38 * 8 * 2083) \
+ (2083UL * (3 + BitTime(bytes))))/1000 \
+ USB2_HOST_DELAY)
#define HS_USECS(bytes) NS_TO_US(HS_NSECS(bytes))
#define HS_USECS_ISO(bytes) NS_TO_US(HS_NSECS_ISO(bytes))
extern long usb_calc_bus_time(int speed, int is_input,
int isoc, int bytecount);
/*-------------------------------------------------------------------------*/
extern void usb_set_device_state(struct usb_device *udev,
enum usb_device_state new_state);
/*-------------------------------------------------------------------------*/
/* exported only within usbcore */
extern struct idr usb_bus_idr;
extern struct mutex usb_bus_idr_lock;
extern wait_queue_head_t usb_kill_urb_queue;
#define usb_endpoint_out(ep_dir) (!((ep_dir) & USB_DIR_IN))
#ifdef CONFIG_PM
extern unsigned usb_wakeup_enabled_descendants(struct usb_device *udev);
extern void usb_root_hub_lost_power(struct usb_device *rhdev);
extern int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg);
extern int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg);
extern void usb_hcd_resume_root_hub(struct usb_hcd *hcd);
#else
static inline unsigned usb_wakeup_enabled_descendants(struct usb_device *udev)
{
return 0;
}
static inline void usb_hcd_resume_root_hub(struct usb_hcd *hcd)
{
return;
}
#endif /* CONFIG_PM */
/*-------------------------------------------------------------------------*/
#if defined(CONFIG_USB_MON) || defined(CONFIG_USB_MON_MODULE)
struct usb_mon_operations {
void (*urb_submit)(struct usb_bus *bus, struct urb *urb);
void (*urb_submit_error)(struct usb_bus *bus, struct urb *urb, int err);
void (*urb_complete)(struct usb_bus *bus, struct urb *urb, int status);
/* void (*urb_unlink)(struct usb_bus *bus, struct urb *urb); */
};
extern const struct usb_mon_operations *mon_ops;
static inline void usbmon_urb_submit(struct usb_bus *bus, struct urb *urb)
{
if (bus->monitored)
(*mon_ops->urb_submit)(bus, urb);
}
static inline void usbmon_urb_submit_error(struct usb_bus *bus, struct urb *urb,
int error)
{
if (bus->monitored) (*mon_ops->urb_submit_error)(bus, urb, error);
}
static inline void usbmon_urb_complete(struct usb_bus *bus, struct urb *urb,
int status)
{
if (bus->monitored) (*mon_ops->urb_complete)(bus, urb, status);
}
int usb_mon_register(const struct usb_mon_operations *ops);
void usb_mon_deregister(void);
#else
static inline void usbmon_urb_submit(struct usb_bus *bus, struct urb *urb) {}
static inline void usbmon_urb_submit_error(struct usb_bus *bus, struct urb *urb,
int error) {}
static inline void usbmon_urb_complete(struct usb_bus *bus, struct urb *urb,
int status) {}
#endif /* CONFIG_USB_MON || CONFIG_USB_MON_MODULE */
/*-------------------------------------------------------------------------*/
/* random stuff */
/* This rwsem is for use only by the hub driver and ehci-hcd.
* Nobody else should touch it.
*/
extern struct rw_semaphore ehci_cf_port_reset_rwsem;
/* Keep track of which host controller drivers are loaded */
#define USB_UHCI_LOADED 0
#define USB_OHCI_LOADED 1
#define USB_EHCI_LOADED 2
extern unsigned long usb_hcds_loaded;
#endif /* __KERNEL__ */
#endif /* __USB_CORE_HCD_H */
// SPDX-License-Identifier: GPL-2.0
/*
* inode.c - part of debugfs, a tiny little debug file system
*
* Copyright (C) 2004,2019 Greg Kroah-Hartman <greg@kroah.com>
* Copyright (C) 2004 IBM Inc.
* Copyright (C) 2019 Linux Foundation <gregkh@linuxfoundation.org>
*
* debugfs is for people to use instead of /proc or /sys.
* See ./Documentation/core-api/kernel-api.rst for more details.
*/
#define pr_fmt(fmt) "debugfs: " fmt
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/kobject.h>
#include <linux/namei.h>
#include <linux/debugfs.h>
#include <linux/fsnotify.h>
#include <linux/string.h>
#include <linux/seq_file.h>
#include <linux/parser.h>
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/security.h>
#include "internal.h"
#define DEBUGFS_DEFAULT_MODE 0700
static struct vfsmount *debugfs_mount;
static int debugfs_mount_count;
static bool debugfs_registered;
static unsigned int debugfs_allow __ro_after_init = DEFAULT_DEBUGFS_ALLOW_BITS;
/*
* Don't allow access attributes to be changed whilst the kernel is locked down
* so that we can use the file mode as part of a heuristic to determine whether
* to lock down individual files.
*/
static int debugfs_setattr(struct mnt_idmap *idmap,
struct dentry *dentry, struct iattr *ia)
{
int ret;
if (ia->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) {
ret = security_locked_down(LOCKDOWN_DEBUGFS);
if (ret)
return ret;
}
return simple_setattr(&nop_mnt_idmap, dentry, ia);
}
static const struct inode_operations debugfs_file_inode_operations = {
.setattr = debugfs_setattr,
};
static const struct inode_operations debugfs_dir_inode_operations = {
.lookup = simple_lookup,
.setattr = debugfs_setattr,
};
static const struct inode_operations debugfs_symlink_inode_operations = {
.get_link = simple_get_link,
.setattr = debugfs_setattr,
};
static struct inode *debugfs_get_inode(struct super_block *sb)
{
struct inode *inode = new_inode(sb);
if (inode) {
inode->i_ino = get_next_ino();
simple_inode_init_ts(inode);
}
return inode;
}
struct debugfs_mount_opts {
kuid_t uid;
kgid_t gid;
umode_t mode;
/* Opt_* bitfield. */
unsigned int opts;
};
enum {
Opt_uid,
Opt_gid,
Opt_mode,
Opt_err
};
static const match_table_t tokens = {
{Opt_uid, "uid=%u"},
{Opt_gid, "gid=%u"},
{Opt_mode, "mode=%o"},
{Opt_err, NULL}
};
struct debugfs_fs_info {
struct debugfs_mount_opts mount_opts;
};
static int debugfs_parse_options(char *data, struct debugfs_mount_opts *opts)
{
substring_t args[MAX_OPT_ARGS];
int option;
int token;
kuid_t uid;
kgid_t gid;
char *p;
opts->opts = 0;
opts->mode = DEBUGFS_DEFAULT_MODE;
while ((p = strsep(&data, ",")) != NULL) {
if (!*p)
continue;
token = match_token(p, tokens, args);
switch (token) {
case Opt_uid:
if (match_int(&args[0], &option))
return -EINVAL;
uid = make_kuid(current_user_ns(), option);
if (!uid_valid(uid))
return -EINVAL;
opts->uid = uid;
break;
case Opt_gid:
if (match_int(&args[0], &option))
return -EINVAL;
gid = make_kgid(current_user_ns(), option);
if (!gid_valid(gid))
return -EINVAL;
opts->gid = gid;
break;
case Opt_mode:
if (match_octal(&args[0], &option))
return -EINVAL;
opts->mode = option & S_IALLUGO;
break;
/*
* We might like to report bad mount options here;
* but traditionally debugfs has ignored all mount options
*/
}
opts->opts |= BIT(token);
}
return 0;
}
static void _debugfs_apply_options(struct super_block *sb, bool remount)
{
struct debugfs_fs_info *fsi = sb->s_fs_info;
struct inode *inode = d_inode(sb->s_root);
struct debugfs_mount_opts *opts = &fsi->mount_opts;
/*
* On remount, only reset mode/uid/gid if they were provided as mount
* options.
*/
if (!remount || opts->opts & BIT(Opt_mode)) {
inode->i_mode &= ~S_IALLUGO;
inode->i_mode |= opts->mode;
}
if (!remount || opts->opts & BIT(Opt_uid))
inode->i_uid = opts->uid;
if (!remount || opts->opts & BIT(Opt_gid))
inode->i_gid = opts->gid;
}
static void debugfs_apply_options(struct super_block *sb)
{
_debugfs_apply_options(sb, false);
}
static void debugfs_apply_options_remount(struct super_block *sb)
{
_debugfs_apply_options(sb, true);
}
static int debugfs_remount(struct super_block *sb, int *flags, char *data)
{
int err;
struct debugfs_fs_info *fsi = sb->s_fs_info;
sync_filesystem(sb);
err = debugfs_parse_options(data, &fsi->mount_opts);
if (err)
goto fail;
debugfs_apply_options_remount(sb);
fail:
return err;
}
static int debugfs_show_options(struct seq_file *m, struct dentry *root)
{
struct debugfs_fs_info *fsi = root->d_sb->s_fs_info;
struct debugfs_mount_opts *opts = &fsi->mount_opts;
if (!uid_eq(opts->uid, GLOBAL_ROOT_UID))
seq_printf(m, ",uid=%u",
from_kuid_munged(&init_user_ns, opts->uid));
if (!gid_eq(opts->gid, GLOBAL_ROOT_GID))
seq_printf(m, ",gid=%u",
from_kgid_munged(&init_user_ns, opts->gid));
if (opts->mode != DEBUGFS_DEFAULT_MODE)
seq_printf(m, ",mode=%o", opts->mode);
return 0;
}
static void debugfs_free_inode(struct inode *inode)
{
if (S_ISLNK(inode->i_mode))
kfree(inode->i_link);
free_inode_nonrcu(inode);
}
static const struct super_operations debugfs_super_operations = {
.statfs = simple_statfs,
.remount_fs = debugfs_remount,
.show_options = debugfs_show_options,
.free_inode = debugfs_free_inode,
};
static void debugfs_release_dentry(struct dentry *dentry)
{
struct debugfs_fsdata *fsd = dentry->d_fsdata;
if ((unsigned long)fsd & DEBUGFS_FSDATA_IS_REAL_FOPS_BIT)
return;
/* check it wasn't a dir (no fsdata) or automount (no real_fops) */
if (fsd && fsd->real_fops) { WARN_ON(!list_empty(&fsd->cancellations)); mutex_destroy(&fsd->cancellations_mtx);
}
kfree(fsd);
}
static struct vfsmount *debugfs_automount(struct path *path)
{
struct debugfs_fsdata *fsd = path->dentry->d_fsdata;
return fsd->automount(path->dentry, d_inode(path->dentry)->i_private);
}
static const struct dentry_operations debugfs_dops = {
.d_delete = always_delete_dentry,
.d_release = debugfs_release_dentry,
.d_automount = debugfs_automount,
};
static int debug_fill_super(struct super_block *sb, void *data, int silent)
{
static const struct tree_descr debug_files[] = {{""}};
struct debugfs_fs_info *fsi;
int err;
fsi = kzalloc(sizeof(struct debugfs_fs_info), GFP_KERNEL);
sb->s_fs_info = fsi;
if (!fsi) {
err = -ENOMEM;
goto fail;
}
err = debugfs_parse_options(data, &fsi->mount_opts);
if (err)
goto fail;
err = simple_fill_super(sb, DEBUGFS_MAGIC, debug_files);
if (err)
goto fail;
sb->s_op = &debugfs_super_operations;
sb->s_d_op = &debugfs_dops;
debugfs_apply_options(sb);
return 0;
fail:
kfree(fsi);
sb->s_fs_info = NULL;
return err;
}
static struct dentry *debug_mount(struct file_system_type *fs_type,
int flags, const char *dev_name,
void *data)
{
if (!(debugfs_allow & DEBUGFS_ALLOW_API))
return ERR_PTR(-EPERM);
return mount_single(fs_type, flags, data, debug_fill_super);
}
static struct file_system_type debug_fs_type = {
.owner = THIS_MODULE,
.name = "debugfs",
.mount = debug_mount,
.kill_sb = kill_litter_super,
};
MODULE_ALIAS_FS("debugfs");
/**
* debugfs_lookup() - look up an existing debugfs file
* @name: a pointer to a string containing the name of the file to look up.
* @parent: a pointer to the parent dentry of the file.
*
* This function will return a pointer to a dentry if it succeeds. If the file
* doesn't exist or an error occurs, %NULL will be returned. The returned
* dentry must be passed to dput() when it is no longer needed.
*
* If debugfs is not enabled in the kernel, the value -%ENODEV will be
* returned.
*/
struct dentry *debugfs_lookup(const char *name, struct dentry *parent)
{
struct dentry *dentry;
if (!debugfs_initialized() || IS_ERR_OR_NULL(name) || IS_ERR(parent))
return NULL;
if (!parent)
parent = debugfs_mount->mnt_root;
dentry = lookup_positive_unlocked(name, parent, strlen(name));
if (IS_ERR(dentry))
return NULL;
return dentry;
}
EXPORT_SYMBOL_GPL(debugfs_lookup);
static struct dentry *start_creating(const char *name, struct dentry *parent)
{
struct dentry *dentry;
int error;
if (!(debugfs_allow & DEBUGFS_ALLOW_API))
return ERR_PTR(-EPERM);
if (!debugfs_initialized())
return ERR_PTR(-ENOENT);
pr_debug("creating file '%s'\n", name);
if (IS_ERR(parent))
return parent;
error = simple_pin_fs(&debug_fs_type, &debugfs_mount,
&debugfs_mount_count);
if (error) {
pr_err("Unable to pin filesystem for file '%s'\n", name);
return ERR_PTR(error);
}
/* If the parent is not specified, we create it in the root.
* We need the root dentry to do this, which is in the super
* block. A pointer to that is in the struct vfsmount that we
* have around.
*/
if (!parent)
parent = debugfs_mount->mnt_root;
inode_lock(d_inode(parent));
if (unlikely(IS_DEADDIR(d_inode(parent))))
dentry = ERR_PTR(-ENOENT);
else
dentry = lookup_one_len(name, parent, strlen(name));
if (!IS_ERR(dentry) && d_really_is_positive(dentry)) {
if (d_is_dir(dentry))
pr_err("Directory '%s' with parent '%s' already present!\n",
name, parent->d_name.name);
else
pr_err("File '%s' in directory '%s' already present!\n",
name, parent->d_name.name);
dput(dentry);
dentry = ERR_PTR(-EEXIST);
}
if (IS_ERR(dentry)) {
inode_unlock(d_inode(parent));
simple_release_fs(&debugfs_mount, &debugfs_mount_count);
}
return dentry;
}
static struct dentry *failed_creating(struct dentry *dentry)
{
inode_unlock(d_inode(dentry->d_parent));
dput(dentry);
simple_release_fs(&debugfs_mount, &debugfs_mount_count);
return ERR_PTR(-ENOMEM);
}
static struct dentry *end_creating(struct dentry *dentry)
{
inode_unlock(d_inode(dentry->d_parent));
return dentry;
}
static struct dentry *__debugfs_create_file(const char *name, umode_t mode,
struct dentry *parent, void *data,
const struct file_operations *proxy_fops,
const struct file_operations *real_fops)
{
struct dentry *dentry;
struct inode *inode;
if (!(mode & S_IFMT))
mode |= S_IFREG;
BUG_ON(!S_ISREG(mode));
dentry = start_creating(name, parent);
if (IS_ERR(dentry))
return dentry;
if (!(debugfs_allow & DEBUGFS_ALLOW_API)) {
failed_creating(dentry);
return ERR_PTR(-EPERM);
}
inode = debugfs_get_inode(dentry->d_sb);
if (unlikely(!inode)) {
pr_err("out of free dentries, can not create file '%s'\n",
name);
return failed_creating(dentry);
}
inode->i_mode = mode;
inode->i_private = data;
inode->i_op = &debugfs_file_inode_operations;
inode->i_fop = proxy_fops;
dentry->d_fsdata = (void *)((unsigned long)real_fops |
DEBUGFS_FSDATA_IS_REAL_FOPS_BIT);
d_instantiate(dentry, inode);
fsnotify_create(d_inode(dentry->d_parent), dentry);
return end_creating(dentry);
}
/**
* debugfs_create_file - create a file in the debugfs filesystem
* @name: a pointer to a string containing the name of the file to create.
* @mode: the permission that the file should have.
* @parent: a pointer to the parent dentry for this file. This should be a
* directory dentry if set. If this parameter is NULL, then the
* file will be created in the root of the debugfs filesystem.
* @data: a pointer to something that the caller will want to get to later
* on. The inode.i_private pointer will point to this value on
* the open() call.
* @fops: a pointer to a struct file_operations that should be used for
* this file.
*
* This is the basic "create a file" function for debugfs. It allows for a
* wide range of flexibility in creating a file, or a directory (if you want
* to create a directory, the debugfs_create_dir() function is
* recommended to be used instead.)
*
* This function will return a pointer to a dentry if it succeeds. This
* pointer must be passed to the debugfs_remove() function when the file is
* to be removed (no automatic cleanup happens if your module is unloaded,
* you are responsible here.) If an error occurs, ERR_PTR(-ERROR) will be
* returned.
*
* If debugfs is not enabled in the kernel, the value -%ENODEV will be
* returned.
*
* NOTE: it's expected that most callers should _ignore_ the errors returned
* by this function. Other debugfs functions handle the fact that the "dentry"
* passed to them could be an error and they don't crash in that case.
* Drivers should generally work fine even if debugfs fails to init anyway.
*/
struct dentry *debugfs_create_file(const char *name, umode_t mode,
struct dentry *parent, void *data,
const struct file_operations *fops)
{
return __debugfs_create_file(name, mode, parent, data,
fops ? &debugfs_full_proxy_file_operations :
&debugfs_noop_file_operations,
fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_file);
/**
* debugfs_create_file_unsafe - create a file in the debugfs filesystem
* @name: a pointer to a string containing the name of the file to create.
* @mode: the permission that the file should have.
* @parent: a pointer to the parent dentry for this file. This should be a
* directory dentry if set. If this parameter is NULL, then the
* file will be created in the root of the debugfs filesystem.
* @data: a pointer to something that the caller will want to get to later
* on. The inode.i_private pointer will point to this value on
* the open() call.
* @fops: a pointer to a struct file_operations that should be used for
* this file.
*
* debugfs_create_file_unsafe() is completely analogous to
* debugfs_create_file(), the only difference being that the fops
* handed it will not get protected against file removals by the
* debugfs core.
*
* It is your responsibility to protect your struct file_operation
* methods against file removals by means of debugfs_file_get()
* and debugfs_file_put(). ->open() is still protected by
* debugfs though.
*
* Any struct file_operations defined by means of
* DEFINE_DEBUGFS_ATTRIBUTE() is protected against file removals and
* thus, may be used here.
*/
struct dentry *debugfs_create_file_unsafe(const char *name, umode_t mode,
struct dentry *parent, void *data,
const struct file_operations *fops)
{
return __debugfs_create_file(name, mode, parent, data,
fops ? &debugfs_open_proxy_file_operations :
&debugfs_noop_file_operations,
fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_file_unsafe);
/**
* debugfs_create_file_size - create a file in the debugfs filesystem
* @name: a pointer to a string containing the name of the file to create.
* @mode: the permission that the file should have.
* @parent: a pointer to the parent dentry for this file. This should be a
* directory dentry if set. If this parameter is NULL, then the
* file will be created in the root of the debugfs filesystem.
* @data: a pointer to something that the caller will want to get to later
* on. The inode.i_private pointer will point to this value on
* the open() call.
* @fops: a pointer to a struct file_operations that should be used for
* this file.
* @file_size: initial file size
*
* This is the basic "create a file" function for debugfs. It allows for a
* wide range of flexibility in creating a file, or a directory (if you want
* to create a directory, the debugfs_create_dir() function is
* recommended to be used instead.)
*/
void debugfs_create_file_size(const char *name, umode_t mode,
struct dentry *parent, void *data,
const struct file_operations *fops,
loff_t file_size)
{
struct dentry *de = debugfs_create_file(name, mode, parent, data, fops);
if (!IS_ERR(de))
d_inode(de)->i_size = file_size;
}
EXPORT_SYMBOL_GPL(debugfs_create_file_size);
/**
* debugfs_create_dir - create a directory in the debugfs filesystem
* @name: a pointer to a string containing the name of the directory to
* create.
* @parent: a pointer to the parent dentry for this file. This should be a
* directory dentry if set. If this parameter is NULL, then the
* directory will be created in the root of the debugfs filesystem.
*
* This function creates a directory in debugfs with the given name.
*
* This function will return a pointer to a dentry if it succeeds. This
* pointer must be passed to the debugfs_remove() function when the file is
* to be removed (no automatic cleanup happens if your module is unloaded,
* you are responsible here.) If an error occurs, ERR_PTR(-ERROR) will be
* returned.
*
* If debugfs is not enabled in the kernel, the value -%ENODEV will be
* returned.
*
* NOTE: it's expected that most callers should _ignore_ the errors returned
* by this function. Other debugfs functions handle the fact that the "dentry"
* passed to them could be an error and they don't crash in that case.
* Drivers should generally work fine even if debugfs fails to init anyway.
*/
struct dentry *debugfs_create_dir(const char *name, struct dentry *parent)
{
struct dentry *dentry = start_creating(name, parent);
struct inode *inode;
if (IS_ERR(dentry))
return dentry;
if (!(debugfs_allow & DEBUGFS_ALLOW_API)) {
failed_creating(dentry);
return ERR_PTR(-EPERM);
}
inode = debugfs_get_inode(dentry->d_sb);
if (unlikely(!inode)) {
pr_err("out of free dentries, can not create directory '%s'\n",
name);
return failed_creating(dentry);
}
inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO;
inode->i_op = &debugfs_dir_inode_operations;
inode->i_fop = &simple_dir_operations;
/* directory inodes start off with i_nlink == 2 (for "." entry) */
inc_nlink(inode);
d_instantiate(dentry, inode);
inc_nlink(d_inode(dentry->d_parent));
fsnotify_mkdir(d_inode(dentry->d_parent), dentry);
return end_creating(dentry);
}
EXPORT_SYMBOL_GPL(debugfs_create_dir);
/**
* debugfs_create_automount - create automount point in the debugfs filesystem
* @name: a pointer to a string containing the name of the file to create.
* @parent: a pointer to the parent dentry for this file. This should be a
* directory dentry if set. If this parameter is NULL, then the
* file will be created in the root of the debugfs filesystem.
* @f: function to be called when pathname resolution steps on that one.
* @data: opaque argument to pass to f().
*
* @f should return what ->d_automount() would.
*/
struct dentry *debugfs_create_automount(const char *name,
struct dentry *parent,
debugfs_automount_t f,
void *data)
{
struct dentry *dentry = start_creating(name, parent);
struct debugfs_fsdata *fsd;
struct inode *inode;
if (IS_ERR(dentry))
return dentry;
fsd = kzalloc(sizeof(*fsd), GFP_KERNEL);
if (!fsd) {
failed_creating(dentry);
return ERR_PTR(-ENOMEM);
}
fsd->automount = f;
if (!(debugfs_allow & DEBUGFS_ALLOW_API)) {
failed_creating(dentry);
kfree(fsd);
return ERR_PTR(-EPERM);
}
inode = debugfs_get_inode(dentry->d_sb);
if (unlikely(!inode)) {
pr_err("out of free dentries, can not create automount '%s'\n",
name);
kfree(fsd);
return failed_creating(dentry);
}
make_empty_dir_inode(inode);
inode->i_flags |= S_AUTOMOUNT;
inode->i_private = data;
dentry->d_fsdata = fsd;
/* directory inodes start off with i_nlink == 2 (for "." entry) */
inc_nlink(inode);
d_instantiate(dentry, inode);
inc_nlink(d_inode(dentry->d_parent));
fsnotify_mkdir(d_inode(dentry->d_parent), dentry);
return end_creating(dentry);
}
EXPORT_SYMBOL(debugfs_create_automount);
/**
* debugfs_create_symlink- create a symbolic link in the debugfs filesystem
* @name: a pointer to a string containing the name of the symbolic link to
* create.
* @parent: a pointer to the parent dentry for this symbolic link. This
* should be a directory dentry if set. If this parameter is NULL,
* then the symbolic link will be created in the root of the debugfs
* filesystem.
* @target: a pointer to a string containing the path to the target of the
* symbolic link.
*
* This function creates a symbolic link with the given name in debugfs that
* links to the given target path.
*
* This function will return a pointer to a dentry if it succeeds. This
* pointer must be passed to the debugfs_remove() function when the symbolic
* link is to be removed (no automatic cleanup happens if your module is
* unloaded, you are responsible here.) If an error occurs, ERR_PTR(-ERROR)
* will be returned.
*
* If debugfs is not enabled in the kernel, the value -%ENODEV will be
* returned.
*/
struct dentry *debugfs_create_symlink(const char *name, struct dentry *parent,
const char *target)
{
struct dentry *dentry;
struct inode *inode;
char *link = kstrdup(target, GFP_KERNEL);
if (!link)
return ERR_PTR(-ENOMEM);
dentry = start_creating(name, parent);
if (IS_ERR(dentry)) {
kfree(link);
return dentry;
}
inode = debugfs_get_inode(dentry->d_sb);
if (unlikely(!inode)) {
pr_err("out of free dentries, can not create symlink '%s'\n",
name);
kfree(link);
return failed_creating(dentry);
}
inode->i_mode = S_IFLNK | S_IRWXUGO;
inode->i_op = &debugfs_symlink_inode_operations;
inode->i_link = link;
d_instantiate(dentry, inode);
return end_creating(dentry);
}
EXPORT_SYMBOL_GPL(debugfs_create_symlink);
static void __debugfs_file_removed(struct dentry *dentry)
{
struct debugfs_fsdata *fsd;
/*
* Paired with the closing smp_mb() implied by a successful
* cmpxchg() in debugfs_file_get(): either
* debugfs_file_get() must see a dead dentry or we must see a
* debugfs_fsdata instance at ->d_fsdata here (or both).
*/
smp_mb();
fsd = READ_ONCE(dentry->d_fsdata);
if ((unsigned long)fsd & DEBUGFS_FSDATA_IS_REAL_FOPS_BIT)
return;
/* if this was the last reference, we're done */
if (refcount_dec_and_test(&fsd->active_users))
return;
/*
* If there's still a reference, the code that obtained it can
* be in different states:
* - The common case of not using cancellations, or already
* after debugfs_leave_cancellation(), where we just need
* to wait for debugfs_file_put() which signals the completion;
* - inside a cancellation section, i.e. between
* debugfs_enter_cancellation() and debugfs_leave_cancellation(),
* in which case we need to trigger the ->cancel() function,
* and then wait for debugfs_file_put() just like in the
* previous case;
* - before debugfs_enter_cancellation() (but obviously after
* debugfs_file_get()), in which case we may not see the
* cancellation in the list on the first round of the loop,
* but debugfs_enter_cancellation() signals the completion
* after adding it, so this code gets woken up to call the
* ->cancel() function.
*/
while (refcount_read(&fsd->active_users)) {
struct debugfs_cancellation *c;
/*
* Lock the cancellations. Note that the cancellations
* structs are meant to be on the stack, so we need to
* ensure we either use them here or don't touch them,
* and debugfs_leave_cancellation() will wait for this
* to be finished processing before exiting one. It may
* of course win and remove the cancellation, but then
* chances are we never even got into this bit, we only
* do if the refcount isn't zero already.
*/
mutex_lock(&fsd->cancellations_mtx); while ((c = list_first_entry_or_null(&fsd->cancellations,
typeof(*c), list))) {
list_del_init(&c->list);
c->cancel(dentry, c->cancel_data);
}
mutex_unlock(&fsd->cancellations_mtx);
wait_for_completion(&fsd->active_users_drained);
}
}
static void remove_one(struct dentry *victim)
{
if (d_is_reg(victim)) __debugfs_file_removed(victim); simple_release_fs(&debugfs_mount, &debugfs_mount_count);
}
/**
* debugfs_remove - recursively removes a directory
* @dentry: a pointer to a the dentry of the directory to be removed. If this
* parameter is NULL or an error value, nothing will be done.
*
* This function recursively removes a directory tree in debugfs that
* was previously created with a call to another debugfs function
* (like debugfs_create_file() or variants thereof.)
*
* This function is required to be called in order for the file to be
* removed, no automatic cleanup of files will happen when a module is
* removed, you are responsible here.
*/
void debugfs_remove(struct dentry *dentry)
{
if (IS_ERR_OR_NULL(dentry))
return;
simple_pin_fs(&debug_fs_type, &debugfs_mount, &debugfs_mount_count);
simple_recursive_removal(dentry, remove_one);
simple_release_fs(&debugfs_mount, &debugfs_mount_count);
}
EXPORT_SYMBOL_GPL(debugfs_remove);
/**
* debugfs_lookup_and_remove - lookup a directory or file and recursively remove it
* @name: a pointer to a string containing the name of the item to look up.
* @parent: a pointer to the parent dentry of the item.
*
* This is the equlivant of doing something like
* debugfs_remove(debugfs_lookup(..)) but with the proper reference counting
* handled for the directory being looked up.
*/
void debugfs_lookup_and_remove(const char *name, struct dentry *parent)
{
struct dentry *dentry;
dentry = debugfs_lookup(name, parent);
if (!dentry)
return;
debugfs_remove(dentry);
dput(dentry);
}
EXPORT_SYMBOL_GPL(debugfs_lookup_and_remove);
/**
* debugfs_rename - rename a file/directory in the debugfs filesystem
* @old_dir: a pointer to the parent dentry for the renamed object. This
* should be a directory dentry.
* @old_dentry: dentry of an object to be renamed.
* @new_dir: a pointer to the parent dentry where the object should be
* moved. This should be a directory dentry.
* @new_name: a pointer to a string containing the target name.
*
* This function renames a file/directory in debugfs. The target must not
* exist for rename to succeed.
*
* This function will return a pointer to old_dentry (which is updated to
* reflect renaming) if it succeeds. If an error occurs, ERR_PTR(-ERROR)
* will be returned.
*
* If debugfs is not enabled in the kernel, the value -%ENODEV will be
* returned.
*/
struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
struct dentry *new_dir, const char *new_name)
{
int error;
struct dentry *dentry = NULL, *trap;
struct name_snapshot old_name;
if (IS_ERR(old_dir))
return old_dir;
if (IS_ERR(new_dir))
return new_dir;
if (IS_ERR_OR_NULL(old_dentry))
return old_dentry;
trap = lock_rename(new_dir, old_dir);
/* Source or destination directories don't exist? */
if (d_really_is_negative(old_dir) || d_really_is_negative(new_dir))
goto exit;
/* Source does not exist, cyclic rename, or mountpoint? */
if (d_really_is_negative(old_dentry) || old_dentry == trap ||
d_mountpoint(old_dentry))
goto exit;
dentry = lookup_one_len(new_name, new_dir, strlen(new_name));
/* Lookup failed, cyclic rename or target exists? */
if (IS_ERR(dentry) || dentry == trap || d_really_is_positive(dentry))
goto exit;
take_dentry_name_snapshot(&old_name, old_dentry);
error = simple_rename(&nop_mnt_idmap, d_inode(old_dir), old_dentry,
d_inode(new_dir), dentry, 0);
if (error) {
release_dentry_name_snapshot(&old_name);
goto exit;
}
d_move(old_dentry, dentry);
fsnotify_move(d_inode(old_dir), d_inode(new_dir), &old_name.name,
d_is_dir(old_dentry),
NULL, old_dentry);
release_dentry_name_snapshot(&old_name);
unlock_rename(new_dir, old_dir);
dput(dentry);
return old_dentry;
exit:
if (dentry && !IS_ERR(dentry))
dput(dentry);
unlock_rename(new_dir, old_dir);
if (IS_ERR(dentry))
return dentry;
return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(debugfs_rename);
/**
* debugfs_initialized - Tells whether debugfs has been registered
*/
bool debugfs_initialized(void)
{
return debugfs_registered;
}
EXPORT_SYMBOL_GPL(debugfs_initialized);
static int __init debugfs_kernel(char *str)
{
if (str) {
if (!strcmp(str, "on"))
debugfs_allow = DEBUGFS_ALLOW_API | DEBUGFS_ALLOW_MOUNT;
else if (!strcmp(str, "no-mount"))
debugfs_allow = DEBUGFS_ALLOW_API;
else if (!strcmp(str, "off"))
debugfs_allow = 0;
}
return 0;
}
early_param("debugfs", debugfs_kernel);
static int __init debugfs_init(void)
{
int retval;
if (!(debugfs_allow & DEBUGFS_ALLOW_MOUNT))
return -EPERM;
retval = sysfs_create_mount_point(kernel_kobj, "debug");
if (retval)
return retval;
retval = register_filesystem(&debug_fs_type);
if (retval)
sysfs_remove_mount_point(kernel_kobj, "debug");
else
debugfs_registered = true;
return retval;
}
core_initcall(debugfs_init);
// SPDX-License-Identifier: GPL-2.0
/*
* linux/fs/namei.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
/*
* Some corrections by tytso.
*/
/* [Feb 1997 T. Schoebel-Theuer] Complete rewrite of the pathname
* lookup logic.
*/
/* [Feb-Apr 2000, AV] Rewrite to the new namespace architecture.
*/
#include <linux/init.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/wordpart.h>
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/namei.h>
#include <linux/pagemap.h>
#include <linux/sched/mm.h>
#include <linux/fsnotify.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/mount.h>
#include <linux/audit.h>
#include <linux/capability.h>
#include <linux/file.h>
#include <linux/fcntl.h>
#include <linux/device_cgroup.h>
#include <linux/fs_struct.h>
#include <linux/posix_acl.h>
#include <linux/hash.h>
#include <linux/bitops.h>
#include <linux/init_task.h>
#include <linux/uaccess.h>
#include "internal.h"
#include "mount.h"
/* [Feb-1997 T. Schoebel-Theuer]
* Fundamental changes in the pathname lookup mechanisms (namei)
* were necessary because of omirr. The reason is that omirr needs
* to know the _real_ pathname, not the user-supplied one, in case
* of symlinks (and also when transname replacements occur).
*
* The new code replaces the old recursive symlink resolution with
* an iterative one (in case of non-nested symlink chains). It does
* this with calls to <fs>_follow_link().
* As a side effect, dir_namei(), _namei() and follow_link() are now
* replaced with a single function lookup_dentry() that can handle all
* the special cases of the former code.
*
* With the new dcache, the pathname is stored at each inode, at least as
* long as the refcount of the inode is positive. As a side effect, the
* size of the dcache depends on the inode cache and thus is dynamic.
*
* [29-Apr-1998 C. Scott Ananian] Updated above description of symlink
* resolution to correspond with current state of the code.
*
* Note that the symlink resolution is not *completely* iterative.
* There is still a significant amount of tail- and mid- recursion in
* the algorithm. Also, note that <fs>_readlink() is not used in
* lookup_dentry(): lookup_dentry() on the result of <fs>_readlink()
* may return different results than <fs>_follow_link(). Many virtual
* filesystems (including /proc) exhibit this behavior.
*/
/* [24-Feb-97 T. Schoebel-Theuer] Side effects caused by new implementation:
* New symlink semantics: when open() is called with flags O_CREAT | O_EXCL
* and the name already exists in form of a symlink, try to create the new
* name indicated by the symlink. The old code always complained that the
* name already exists, due to not following the symlink even if its target
* is nonexistent. The new semantics affects also mknod() and link() when
* the name is a symlink pointing to a non-existent name.
*
* I don't know which semantics is the right one, since I have no access
* to standards. But I found by trial that HP-UX 9.0 has the full "new"
* semantics implemented, while SunOS 4.1.1 and Solaris (SunOS 5.4) have the
* "old" one. Personally, I think the new semantics is much more logical.
* Note that "ln old new" where "new" is a symlink pointing to a non-existing
* file does succeed in both HP-UX and SunOs, but not in Solaris
* and in the old Linux semantics.
*/
/* [16-Dec-97 Kevin Buhr] For security reasons, we change some symlink
* semantics. See the comments in "open_namei" and "do_link" below.
*
* [10-Sep-98 Alan Modra] Another symlink change.
*/
/* [Feb-Apr 2000 AV] Complete rewrite. Rules for symlinks:
* inside the path - always follow.
* in the last component in creation/removal/renaming - never follow.
* if LOOKUP_FOLLOW passed - follow.
* if the pathname has trailing slashes - follow.
* otherwise - don't follow.
* (applied in that order).
*
* [Jun 2000 AV] Inconsistent behaviour of open() in case if flags==O_CREAT
* restored for 2.4. This is the last surviving part of old 4.2BSD bug.
* During the 2.4 we need to fix the userland stuff depending on it -
* hopefully we will be able to get rid of that wart in 2.5. So far only
* XEmacs seems to be relying on it...
*/
/*
* [Sep 2001 AV] Single-semaphore locking scheme (kudos to David Holland)
* implemented. Let's see if raised priority of ->s_vfs_rename_mutex gives
* any extra contention...
*/
/* In order to reduce some races, while at the same time doing additional
* checking and hopefully speeding things up, we copy filenames to the
* kernel data space before using them..
*
* POSIX.1 2.4: an empty pathname is invalid (ENOENT).
* PATH_MAX includes the nul terminator --RR.
*/
#define EMBEDDED_NAME_MAX (PATH_MAX - offsetof(struct filename, iname))
struct filename *
getname_flags(const char __user *filename, int flags, int *empty)
{
struct filename *result;
char *kname;
int len;
result = audit_reusename(filename);
if (result)
return result;
result = __getname();
if (unlikely(!result))
return ERR_PTR(-ENOMEM);
/*
* First, try to embed the struct filename inside the names_cache
* allocation
*/
kname = (char *)result->iname;
result->name = kname;
len = strncpy_from_user(kname, filename, EMBEDDED_NAME_MAX);
if (unlikely(len < 0)) {
__putname(result);
return ERR_PTR(len);
}
/*
* Uh-oh. We have a name that's approaching PATH_MAX. Allocate a
* separate struct filename so we can dedicate the entire
* names_cache allocation for the pathname, and re-do the copy from
* userland.
*/
if (unlikely(len == EMBEDDED_NAME_MAX)) {
const size_t size = offsetof(struct filename, iname[1]);
kname = (char *)result;
/*
* size is chosen that way we to guarantee that
* result->iname[0] is within the same object and that
* kname can't be equal to result->iname, no matter what.
*/
result = kzalloc(size, GFP_KERNEL);
if (unlikely(!result)) {
__putname(kname);
return ERR_PTR(-ENOMEM);
}
result->name = kname;
len = strncpy_from_user(kname, filename, PATH_MAX);
if (unlikely(len < 0)) {
__putname(kname);
kfree(result);
return ERR_PTR(len);
}
if (unlikely(len == PATH_MAX)) { __putname(kname);
kfree(result);
return ERR_PTR(-ENAMETOOLONG);
}
}
atomic_set(&result->refcnt, 1);
/* The empty path is special. */
if (unlikely(!len)) {
if (empty) *empty = 1; if (!(flags & LOOKUP_EMPTY)) { putname(result);
return ERR_PTR(-ENOENT);
}
}
result->uptr = filename;
result->aname = NULL;
audit_getname(result);
return result;
}
struct filename *
getname_uflags(const char __user *filename, int uflags)
{
int flags = (uflags & AT_EMPTY_PATH) ? LOOKUP_EMPTY : 0;
return getname_flags(filename, flags, NULL);
}
struct filename *
getname(const char __user * filename)
{
return getname_flags(filename, 0, NULL);
}
struct filename *
getname_kernel(const char * filename)
{
struct filename *result;
int len = strlen(filename) + 1;
result = __getname();
if (unlikely(!result))
return ERR_PTR(-ENOMEM);
if (len <= EMBEDDED_NAME_MAX) {
result->name = (char *)result->iname;
} else if (len <= PATH_MAX) {
const size_t size = offsetof(struct filename, iname[1]);
struct filename *tmp;
tmp = kmalloc(size, GFP_KERNEL);
if (unlikely(!tmp)) {
__putname(result);
return ERR_PTR(-ENOMEM);
}
tmp->name = (char *)result;
result = tmp;
} else {
__putname(result);
return ERR_PTR(-ENAMETOOLONG);
}
memcpy((char *)result->name, filename, len);
result->uptr = NULL;
result->aname = NULL;
atomic_set(&result->refcnt, 1);
audit_getname(result);
return result;
}
EXPORT_SYMBOL(getname_kernel);
void putname(struct filename *name)
{
if (IS_ERR(name))
return;
if (WARN_ON_ONCE(!atomic_read(&name->refcnt)))
return;
if (!atomic_dec_and_test(&name->refcnt))
return;
if (name->name != name->iname) { __putname(name->name);
kfree(name);
} else
__putname(name);
}
EXPORT_SYMBOL(putname);
/**
* check_acl - perform ACL permission checking
* @idmap: idmap of the mount the inode was found from
* @inode: inode to check permissions on
* @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
*
* This function performs the ACL permission checking. Since this function
* retrieve POSIX acls it needs to know whether it is called from a blocking or
* non-blocking context and thus cares about the MAY_NOT_BLOCK bit.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
static int check_acl(struct mnt_idmap *idmap,
struct inode *inode, int mask)
{
#ifdef CONFIG_FS_POSIX_ACL
struct posix_acl *acl;
if (mask & MAY_NOT_BLOCK) { acl = get_cached_acl_rcu(inode, ACL_TYPE_ACCESS);
if (!acl)
return -EAGAIN;
/* no ->get_inode_acl() calls in RCU mode... */
if (is_uncached_acl(acl))
return -ECHILD;
return posix_acl_permission(idmap, inode, acl, mask);
}
acl = get_inode_acl(inode, ACL_TYPE_ACCESS);
if (IS_ERR(acl))
return PTR_ERR(acl); if (acl) { int error = posix_acl_permission(idmap, inode, acl, mask); posix_acl_release(acl);
return error;
}
#endif
return -EAGAIN;
}
/**
* acl_permission_check - perform basic UNIX permission checking
* @idmap: idmap of the mount the inode was found from
* @inode: inode to check permissions on
* @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
*
* This function performs the basic UNIX permission checking. Since this
* function may retrieve POSIX acls it needs to know whether it is called from a
* blocking or non-blocking context and thus cares about the MAY_NOT_BLOCK bit.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
static int acl_permission_check(struct mnt_idmap *idmap,
struct inode *inode, int mask)
{
unsigned int mode = inode->i_mode;
vfsuid_t vfsuid;
/* Are we the owner? If so, ACL's don't matter */
vfsuid = i_uid_into_vfsuid(idmap, inode);
if (likely(vfsuid_eq_kuid(vfsuid, current_fsuid()))) { mask &= 7;
mode >>= 6;
return (mask & ~mode) ? -EACCES : 0;
}
/* Do we have ACL's? */
if (IS_POSIXACL(inode) && (mode & S_IRWXG)) { int error = check_acl(idmap, inode, mask); if (error != -EAGAIN)
return error;
}
/* Only RWX matters for group/other mode bits */
mask &= 7;
/*
* Are the group permissions different from
* the other permissions in the bits we care
* about? Need to check group ownership if so.
*/
if (mask & (mode ^ (mode >> 3))) {
vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);
if (vfsgid_in_group_p(vfsgid))
mode >>= 3;
}
/* Bits in 'mode' clear that we require? */
return (mask & ~mode) ? -EACCES : 0;
}
/**
* generic_permission - check for access rights on a Posix-like filesystem
* @idmap: idmap of the mount the inode was found from
* @inode: inode to check access rights for
* @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC,
* %MAY_NOT_BLOCK ...)
*
* Used to check for read/write/execute permissions on a file.
* We use "fsuid" for this, letting us set arbitrary permissions
* for filesystem access without changing the "normal" uids which
* are used for other things.
*
* generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk
* request cannot be satisfied (eg. requires blocking or too much complexity).
* It would then be called again in ref-walk mode.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int generic_permission(struct mnt_idmap *idmap, struct inode *inode,
int mask)
{
int ret;
/*
* Do the basic permission checks.
*/
ret = acl_permission_check(idmap, inode, mask); if (ret != -EACCES)
return ret;
if (S_ISDIR(inode->i_mode)) {
/* DACs are overridable for directories */
if (!(mask & MAY_WRITE)) if (capable_wrt_inode_uidgid(idmap, inode,
CAP_DAC_READ_SEARCH))
return 0;
if (capable_wrt_inode_uidgid(idmap, inode,
CAP_DAC_OVERRIDE))
return 0;
return -EACCES;
}
/*
* Searching includes executable on directories, else just read.
*/
mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
if (mask == MAY_READ)
if (capable_wrt_inode_uidgid(idmap, inode,
CAP_DAC_READ_SEARCH))
return 0;
/*
* Read/write DACs are always overridable.
* Executable DACs are overridable when there is
* at least one exec bit set.
*/
if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO)) if (capable_wrt_inode_uidgid(idmap, inode,
CAP_DAC_OVERRIDE))
return 0;
return -EACCES;
}
EXPORT_SYMBOL(generic_permission);
/**
* do_inode_permission - UNIX permission checking
* @idmap: idmap of the mount the inode was found from
* @inode: inode to check permissions on
* @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
*
* We _really_ want to just do "generic_permission()" without
* even looking at the inode->i_op values. So we keep a cache
* flag in inode->i_opflags, that says "this has not special
* permission function, use the fast case".
*/
static inline int do_inode_permission(struct mnt_idmap *idmap,
struct inode *inode, int mask)
{
if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) { if (likely(inode->i_op->permission)) return inode->i_op->permission(idmap, inode, mask);
/* This gets set once for the inode lifetime */
spin_lock(&inode->i_lock);
inode->i_opflags |= IOP_FASTPERM;
spin_unlock(&inode->i_lock);
}
return generic_permission(idmap, inode, mask);
}
/**
* sb_permission - Check superblock-level permissions
* @sb: Superblock of inode to check permission on
* @inode: Inode to check permission on
* @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
*
* Separate out file-system wide checks from inode-specific permission checks.
*/
static int sb_permission(struct super_block *sb, struct inode *inode, int mask)
{
if (unlikely(mask & MAY_WRITE)) {
umode_t mode = inode->i_mode;
/* Nobody gets write access to a read-only fs. */
if (sb_rdonly(sb) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)))
return -EROFS;
}
return 0;
}
/**
* inode_permission - Check for access rights to a given inode
* @idmap: idmap of the mount the inode was found from
* @inode: Inode to check permission on
* @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
*
* Check for read/write/execute permissions on an inode. We use fs[ug]id for
* this, letting us set arbitrary permissions for filesystem access without
* changing the "normal" UIDs which are used for other things.
*
* When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
*/
int inode_permission(struct mnt_idmap *idmap,
struct inode *inode, int mask)
{
int retval;
retval = sb_permission(inode->i_sb, inode, mask);
if (retval)
return retval;
if (unlikely(mask & MAY_WRITE)) {
/*
* Nobody gets write access to an immutable file.
*/
if (IS_IMMUTABLE(inode))
return -EPERM;
/*
* Updating mtime will likely cause i_uid and i_gid to be
* written back improperly if their true value is unknown
* to the vfs.
*/
if (HAS_UNMAPPED_ID(idmap, inode))
return -EACCES;
}
retval = do_inode_permission(idmap, inode, mask); if (retval)
return retval;
retval = devcgroup_inode_permission(inode, mask);
if (retval)
return retval;
return security_inode_permission(inode, mask);
}
EXPORT_SYMBOL(inode_permission);
/**
* path_get - get a reference to a path
* @path: path to get the reference to
*
* Given a path increment the reference count to the dentry and the vfsmount.
*/
void path_get(const struct path *path)
{
mntget(path->mnt); dget(path->dentry);
}
EXPORT_SYMBOL(path_get);
/**
* path_put - put a reference to a path
* @path: path to put the reference to
*
* Given a path decrement the reference count to the dentry and the vfsmount.
*/
void path_put(const struct path *path)
{
dput(path->dentry);
mntput(path->mnt);
}
EXPORT_SYMBOL(path_put);
#define EMBEDDED_LEVELS 2
struct nameidata {
struct path path;
struct qstr last;
struct path root;
struct inode *inode; /* path.dentry.d_inode */
unsigned int flags, state;
unsigned seq, next_seq, m_seq, r_seq;
int last_type;
unsigned depth;
int total_link_count;
struct saved {
struct path link;
struct delayed_call done;
const char *name;
unsigned seq;
} *stack, internal[EMBEDDED_LEVELS];
struct filename *name;
struct nameidata *saved;
unsigned root_seq;
int dfd;
vfsuid_t dir_vfsuid;
umode_t dir_mode;
} __randomize_layout;
#define ND_ROOT_PRESET 1
#define ND_ROOT_GRABBED 2
#define ND_JUMPED 4
static void __set_nameidata(struct nameidata *p, int dfd, struct filename *name)
{
struct nameidata *old = current->nameidata;
p->stack = p->internal;
p->depth = 0;
p->dfd = dfd;
p->name = name;
p->path.mnt = NULL;
p->path.dentry = NULL;
p->total_link_count = old ? old->total_link_count : 0;
p->saved = old;
current->nameidata = p;
}
static inline void set_nameidata(struct nameidata *p, int dfd, struct filename *name,
const struct path *root)
{
__set_nameidata(p, dfd, name);
p->state = 0;
if (unlikely(root)) {
p->state = ND_ROOT_PRESET;
p->root = *root;
}
}
static void restore_nameidata(void)
{
struct nameidata *now = current->nameidata, *old = now->saved;
current->nameidata = old;
if (old)
old->total_link_count = now->total_link_count; if (now->stack != now->internal) kfree(now->stack);
}
static bool nd_alloc_stack(struct nameidata *nd)
{
struct saved *p;
p= kmalloc_array(MAXSYMLINKS, sizeof(struct saved),
nd->flags & LOOKUP_RCU ? GFP_ATOMIC : GFP_KERNEL);
if (unlikely(!p))
return false;
memcpy(p, nd->internal, sizeof(nd->internal));
nd->stack = p;
return true;
}
/**
* path_connected - Verify that a dentry is below mnt.mnt_root
* @mnt: The mountpoint to check.
* @dentry: The dentry to check.
*
* Rename can sometimes move a file or directory outside of a bind
* mount, path_connected allows those cases to be detected.
*/
static bool path_connected(struct vfsmount *mnt, struct dentry *dentry)
{
struct super_block *sb = mnt->mnt_sb;
/* Bind mounts can have disconnected paths */
if (mnt->mnt_root == sb->s_root)
return true;
return is_subdir(dentry, mnt->mnt_root);
}
static void drop_links(struct nameidata *nd)
{
int i = nd->depth;
while (i--) {
struct saved *last = nd->stack + i; do_delayed_call(&last->done); clear_delayed_call(&last->done);
}
}
static void leave_rcu(struct nameidata *nd)
{
nd->flags &= ~LOOKUP_RCU;
nd->seq = nd->next_seq = 0;
rcu_read_unlock();
}
static void terminate_walk(struct nameidata *nd)
{
drop_links(nd); if (!(nd->flags & LOOKUP_RCU)) {
int i;
path_put(&nd->path);
for (i = 0; i < nd->depth; i++)
path_put(&nd->stack[i].link); if (nd->state & ND_ROOT_GRABBED) { path_put(&nd->root);
nd->state &= ~ND_ROOT_GRABBED;
}
} else {
leave_rcu(nd);
}
nd->depth = 0;
nd->path.mnt = NULL;
nd->path.dentry = NULL;
}
/* path_put is needed afterwards regardless of success or failure */
static bool __legitimize_path(struct path *path, unsigned seq, unsigned mseq)
{
int res = __legitimize_mnt(path->mnt, mseq);
if (unlikely(res)) {
if (res > 0) path->mnt = NULL; path->dentry = NULL; return false;
}
if (unlikely(!lockref_get_not_dead(&path->dentry->d_lockref))) {
path->dentry = NULL;
return false;
}
return !read_seqcount_retry(&path->dentry->d_seq, seq);
}
static inline bool legitimize_path(struct nameidata *nd,
struct path *path, unsigned seq)
{
return __legitimize_path(path, seq, nd->m_seq);
}
static bool legitimize_links(struct nameidata *nd)
{
int i;
if (unlikely(nd->flags & LOOKUP_CACHED)) { drop_links(nd); nd->depth = 0; return false;
}
for (i = 0; i < nd->depth; i++) { struct saved *last = nd->stack + i;
if (unlikely(!legitimize_path(nd, &last->link, last->seq))) {
drop_links(nd); nd->depth = i + 1;
return false;
}
}
return true;
}
static bool legitimize_root(struct nameidata *nd)
{
/* Nothing to do if nd->root is zero or is managed by the VFS user. */
if (!nd->root.mnt || (nd->state & ND_ROOT_PRESET))
return true;
nd->state |= ND_ROOT_GRABBED;
return legitimize_path(nd, &nd->root, nd->root_seq);
}
/*
* Path walking has 2 modes, rcu-walk and ref-walk (see
* Documentation/filesystems/path-lookup.txt). In situations when we can't
* continue in RCU mode, we attempt to drop out of rcu-walk mode and grab
* normal reference counts on dentries and vfsmounts to transition to ref-walk
* mode. Refcounts are grabbed at the last known good point before rcu-walk
* got stuck, so ref-walk may continue from there. If this is not successful
* (eg. a seqcount has changed), then failure is returned and it's up to caller
* to restart the path walk from the beginning in ref-walk mode.
*/
/**
* try_to_unlazy - try to switch to ref-walk mode.
* @nd: nameidata pathwalk data
* Returns: true on success, false on failure
*
* try_to_unlazy attempts to legitimize the current nd->path and nd->root
* for ref-walk mode.
* Must be called from rcu-walk context.
* Nothing should touch nameidata between try_to_unlazy() failure and
* terminate_walk().
*/
static bool try_to_unlazy(struct nameidata *nd)
{
struct dentry *parent = nd->path.dentry; BUG_ON(!(nd->flags & LOOKUP_RCU)); if (unlikely(!legitimize_links(nd)))
goto out1;
if (unlikely(!legitimize_path(nd, &nd->path, nd->seq)))
goto out;
if (unlikely(!legitimize_root(nd)))
goto out;
leave_rcu(nd); BUG_ON(nd->inode != parent->d_inode);
return true;
out1:
nd->path.mnt = NULL;
nd->path.dentry = NULL;
out:
leave_rcu(nd);
return false;
}
/**
* try_to_unlazy_next - try to switch to ref-walk mode.
* @nd: nameidata pathwalk data
* @dentry: next dentry to step into
* Returns: true on success, false on failure
*
* Similar to try_to_unlazy(), but here we have the next dentry already
* picked by rcu-walk and want to legitimize that in addition to the current
* nd->path and nd->root for ref-walk mode. Must be called from rcu-walk context.
* Nothing should touch nameidata between try_to_unlazy_next() failure and
* terminate_walk().
*/
static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry)
{
int res;
BUG_ON(!(nd->flags & LOOKUP_RCU));
if (unlikely(!legitimize_links(nd)))
goto out2;
res = __legitimize_mnt(nd->path.mnt, nd->m_seq);
if (unlikely(res)) {
if (res > 0)
goto out2;
goto out1;
}
if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref)))
goto out1;
/*
* We need to move both the parent and the dentry from the RCU domain
* to be properly refcounted. And the sequence number in the dentry
* validates *both* dentry counters, since we checked the sequence
* number of the parent after we got the child sequence number. So we
* know the parent must still be valid if the child sequence number is
*/
if (unlikely(!lockref_get_not_dead(&dentry->d_lockref)))
goto out;
if (read_seqcount_retry(&dentry->d_seq, nd->next_seq))
goto out_dput;
/*
* Sequence counts matched. Now make sure that the root is
* still valid and get it if required.
*/
if (unlikely(!legitimize_root(nd)))
goto out_dput;
leave_rcu(nd);
return true;
out2:
nd->path.mnt = NULL;
out1:
nd->path.dentry = NULL;
out:
leave_rcu(nd);
return false;
out_dput:
leave_rcu(nd);
dput(dentry);
return false;
}
static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
{
if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE)) return dentry->d_op->d_revalidate(dentry, flags);
else
return 1;
}
/**
* complete_walk - successful completion of path walk
* @nd: pointer nameidata
*
* If we had been in RCU mode, drop out of it and legitimize nd->path.
* Revalidate the final result, unless we'd already done that during
* the path walk or the filesystem doesn't ask for it. Return 0 on
* success, -error on failure. In case of failure caller does not
* need to drop nd->path.
*/
static int complete_walk(struct nameidata *nd)
{
struct dentry *dentry = nd->path.dentry;
int status;
if (nd->flags & LOOKUP_RCU) {
/*
* We don't want to zero nd->root for scoped-lookups or
* externally-managed nd->root.
*/
if (!(nd->state & ND_ROOT_PRESET))
if (!(nd->flags & LOOKUP_IS_SCOPED))
nd->root.mnt = NULL; nd->flags &= ~LOOKUP_CACHED;
if (!try_to_unlazy(nd))
return -ECHILD;
}
if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
/*
* While the guarantee of LOOKUP_IS_SCOPED is (roughly) "don't
* ever step outside the root during lookup" and should already
* be guaranteed by the rest of namei, we want to avoid a namei
* BUG resulting in userspace being given a path that was not
* scoped within the root at some point during the lookup.
*
* So, do a final sanity-check to make sure that in the
* worst-case scenario (a complete bypass of LOOKUP_IS_SCOPED)
* we won't silently return an fd completely outside of the
* requested root to userspace.
*
* Userspace could move the path outside the root after this
* check, but as discussed elsewhere this is not a concern (the
* resolved file was inside the root at some point).
*/
if (!path_is_under(&nd->path, &nd->root))
return -EXDEV;
}
if (likely(!(nd->state & ND_JUMPED))) return 0; if (likely(!(dentry->d_flags & DCACHE_OP_WEAK_REVALIDATE)))
return 0;
status = dentry->d_op->d_weak_revalidate(dentry, nd->flags);
if (status > 0)
return 0;
if (!status)
status = -ESTALE;
return status;
}
static int set_root(struct nameidata *nd)
{
struct fs_struct *fs = current->fs;
/*
* Jumping to the real root in a scoped-lookup is a BUG in namei, but we
* still have to ensure it doesn't happen because it will cause a breakout
* from the dirfd.
*/
if (WARN_ON(nd->flags & LOOKUP_IS_SCOPED))
return -ENOTRECOVERABLE;
if (nd->flags & LOOKUP_RCU) {
unsigned seq;
do {
seq = read_seqcount_begin(&fs->seq);
nd->root = fs->root;
nd->root_seq = __read_seqcount_begin(&nd->root.dentry->d_seq);
} while (read_seqcount_retry(&fs->seq, seq));
} else {
get_fs_root(fs, &nd->root);
nd->state |= ND_ROOT_GRABBED;
}
return 0;
}
static int nd_jump_root(struct nameidata *nd)
{
if (unlikely(nd->flags & LOOKUP_BENEATH))
return -EXDEV;
if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
/* Absolute path arguments to path_init() are allowed. */
if (nd->path.mnt != NULL && nd->path.mnt != nd->root.mnt)
return -EXDEV;
}
if (!nd->root.mnt) { int error = set_root(nd);
if (error)
return error;
}
if (nd->flags & LOOKUP_RCU) {
struct dentry *d;
nd->path = nd->root;
d = nd->path.dentry;
nd->inode = d->d_inode;
nd->seq = nd->root_seq;
if (read_seqcount_retry(&d->d_seq, nd->seq))
return -ECHILD;
} else {
path_put(&nd->path);
nd->path = nd->root;
path_get(&nd->path); nd->inode = nd->path.dentry->d_inode;
}
nd->state |= ND_JUMPED; return 0;
}
/*
* Helper to directly jump to a known parsed path from ->get_link,
* caller must have taken a reference to path beforehand.
*/
int nd_jump_link(const struct path *path)
{
int error = -ELOOP;
struct nameidata *nd = current->nameidata;
if (unlikely(nd->flags & LOOKUP_NO_MAGICLINKS))
goto err;
error = -EXDEV;
if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
if (nd->path.mnt != path->mnt)
goto err;
}
/* Not currently safe for scoped-lookups. */
if (unlikely(nd->flags & LOOKUP_IS_SCOPED))
goto err;
path_put(&nd->path);
nd->path = *path;
nd->inode = nd->path.dentry->d_inode;
nd->state |= ND_JUMPED;
return 0;
err:
path_put(path);
return error;
}
static inline void put_link(struct nameidata *nd)
{
struct saved *last = nd->stack + --nd->depth; do_delayed_call(&last->done); if (!(nd->flags & LOOKUP_RCU)) path_put(&last->link);
}
static int sysctl_protected_symlinks __read_mostly;
static int sysctl_protected_hardlinks __read_mostly;
static int sysctl_protected_fifos __read_mostly;
static int sysctl_protected_regular __read_mostly;
#ifdef CONFIG_SYSCTL
static struct ctl_table namei_sysctls[] = {
{
.procname = "protected_symlinks",
.data = &sysctl_protected_symlinks,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{
.procname = "protected_hardlinks",
.data = &sysctl_protected_hardlinks,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{
.procname = "protected_fifos",
.data = &sysctl_protected_fifos,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_TWO,
},
{
.procname = "protected_regular",
.data = &sysctl_protected_regular,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_TWO,
},
};
static int __init init_fs_namei_sysctls(void)
{
register_sysctl_init("fs", namei_sysctls);
return 0;
}
fs_initcall(init_fs_namei_sysctls);
#endif /* CONFIG_SYSCTL */
/**
* may_follow_link - Check symlink following for unsafe situations
* @nd: nameidata pathwalk data
* @inode: Used for idmapping.
*
* In the case of the sysctl_protected_symlinks sysctl being enabled,
* CAP_DAC_OVERRIDE needs to be specifically ignored if the symlink is
* in a sticky world-writable directory. This is to protect privileged
* processes from failing races against path names that may change out
* from under them by way of other users creating malicious symlinks.
* It will permit symlinks to be followed only when outside a sticky
* world-writable directory, or when the uid of the symlink and follower
* match, or when the directory owner matches the symlink's owner.
*
* Returns 0 if following the symlink is allowed, -ve on error.
*/
static inline int may_follow_link(struct nameidata *nd, const struct inode *inode)
{
struct mnt_idmap *idmap;
vfsuid_t vfsuid;
if (!sysctl_protected_symlinks)
return 0;
idmap = mnt_idmap(nd->path.mnt);
vfsuid = i_uid_into_vfsuid(idmap, inode);
/* Allowed if owner and follower match. */
if (vfsuid_eq_kuid(vfsuid, current_fsuid()))
return 0;
/* Allowed if parent directory not sticky and world-writable. */
if ((nd->dir_mode & (S_ISVTX|S_IWOTH)) != (S_ISVTX|S_IWOTH))
return 0;
/* Allowed if parent directory and link owner match. */
if (vfsuid_valid(nd->dir_vfsuid) && vfsuid_eq(nd->dir_vfsuid, vfsuid))
return 0;
if (nd->flags & LOOKUP_RCU)
return -ECHILD;
audit_inode(nd->name, nd->stack[0].link.dentry, 0); audit_log_path_denied(AUDIT_ANOM_LINK, "follow_link");
return -EACCES;
}
/**
* safe_hardlink_source - Check for safe hardlink conditions
* @idmap: idmap of the mount the inode was found from
* @inode: the source inode to hardlink from
*
* Return false if at least one of the following conditions:
* - inode is not a regular file
* - inode is setuid
* - inode is setgid and group-exec
* - access failure for read and write
*
* Otherwise returns true.
*/
static bool safe_hardlink_source(struct mnt_idmap *idmap,
struct inode *inode)
{
umode_t mode = inode->i_mode;
/* Special files should not get pinned to the filesystem. */
if (!S_ISREG(mode))
return false;
/* Setuid files should not get pinned to the filesystem. */
if (mode & S_ISUID)
return false;
/* Executable setgid files should not get pinned to the filesystem. */
if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP))
return false;
/* Hardlinking to unreadable or unwritable sources is dangerous. */
if (inode_permission(idmap, inode, MAY_READ | MAY_WRITE))
return false;
return true;
}
/**
* may_linkat - Check permissions for creating a hardlink
* @idmap: idmap of the mount the inode was found from
* @link: the source to hardlink from
*
* Block hardlink when all of:
* - sysctl_protected_hardlinks enabled
* - fsuid does not match inode
* - hardlink source is unsafe (see safe_hardlink_source() above)
* - not CAP_FOWNER in a namespace with the inode owner uid mapped
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*
* Returns 0 if successful, -ve on error.
*/
int may_linkat(struct mnt_idmap *idmap, const struct path *link)
{
struct inode *inode = link->dentry->d_inode;
/* Inode writeback is not safe when the uid or gid are invalid. */
if (!vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) ||
!vfsgid_valid(i_gid_into_vfsgid(idmap, inode)))
return -EOVERFLOW;
if (!sysctl_protected_hardlinks)
return 0;
/* Source inode owner (or CAP_FOWNER) can hardlink all they like,
* otherwise, it must be a safe source.
*/
if (safe_hardlink_source(idmap, inode) ||
inode_owner_or_capable(idmap, inode))
return 0;
audit_log_path_denied(AUDIT_ANOM_LINK, "linkat");
return -EPERM;
}
/**
* may_create_in_sticky - Check whether an O_CREAT open in a sticky directory
* should be allowed, or not, on files that already
* exist.
* @idmap: idmap of the mount the inode was found from
* @nd: nameidata pathwalk data
* @inode: the inode of the file to open
*
* Block an O_CREAT open of a FIFO (or a regular file) when:
* - sysctl_protected_fifos (or sysctl_protected_regular) is enabled
* - the file already exists
* - we are in a sticky directory
* - we don't own the file
* - the owner of the directory doesn't own the file
* - the directory is world writable
* If the sysctl_protected_fifos (or sysctl_protected_regular) is set to 2
* the directory doesn't have to be world writable: being group writable will
* be enough.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*
* Returns 0 if the open is allowed, -ve on error.
*/
static int may_create_in_sticky(struct mnt_idmap *idmap,
struct nameidata *nd, struct inode *const inode)
{
umode_t dir_mode = nd->dir_mode;
vfsuid_t dir_vfsuid = nd->dir_vfsuid;
if ((!sysctl_protected_fifos && S_ISFIFO(inode->i_mode)) || (!sysctl_protected_regular && S_ISREG(inode->i_mode)) ||
likely(!(dir_mode & S_ISVTX)) ||
vfsuid_eq(i_uid_into_vfsuid(idmap, inode), dir_vfsuid) || vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid()))
return 0;
if (likely(dir_mode & 0002) ||
(dir_mode & 0020 &&
((sysctl_protected_fifos >= 2 && S_ISFIFO(inode->i_mode)) || (sysctl_protected_regular >= 2 && S_ISREG(inode->i_mode))))) { const char *operation = S_ISFIFO(inode->i_mode) ?
"sticky_create_fifo" :
"sticky_create_regular";
audit_log_path_denied(AUDIT_ANOM_CREAT, operation);
return -EACCES;
}
return 0;
}
/*
* follow_up - Find the mountpoint of path's vfsmount
*
* Given a path, find the mountpoint of its source file system.
* Replace @path with the path of the mountpoint in the parent mount.
* Up is towards /.
*
* Return 1 if we went up a level and 0 if we were already at the
* root.
*/
int follow_up(struct path *path)
{
struct mount *mnt = real_mount(path->mnt);
struct mount *parent;
struct dentry *mountpoint;
read_seqlock_excl(&mount_lock);
parent = mnt->mnt_parent;
if (parent == mnt) {
read_sequnlock_excl(&mount_lock);
return 0;
}
mntget(&parent->mnt);
mountpoint = dget(mnt->mnt_mountpoint);
read_sequnlock_excl(&mount_lock);
dput(path->dentry);
path->dentry = mountpoint;
mntput(path->mnt);
path->mnt = &parent->mnt;
return 1;
}
EXPORT_SYMBOL(follow_up);
static bool choose_mountpoint_rcu(struct mount *m, const struct path *root,
struct path *path, unsigned *seqp)
{
while (mnt_has_parent(m)) {
struct dentry *mountpoint = m->mnt_mountpoint;
m = m->mnt_parent;
if (unlikely(root->dentry == mountpoint &&
root->mnt == &m->mnt))
break;
if (mountpoint != m->mnt.mnt_root) {
path->mnt = &m->mnt;
path->dentry = mountpoint;
*seqp = read_seqcount_begin(&mountpoint->d_seq);
return true;
}
}
return false;
}
static bool choose_mountpoint(struct mount *m, const struct path *root,
struct path *path)
{
bool found;
rcu_read_lock();
while (1) {
unsigned seq, mseq = read_seqbegin(&mount_lock);
found = choose_mountpoint_rcu(m, root, path, &seq);
if (unlikely(!found)) { if (!read_seqretry(&mount_lock, mseq))
break;
} else {
if (likely(__legitimize_path(path, seq, mseq)))
break;
rcu_read_unlock();
path_put(path);
rcu_read_lock();
}
}
rcu_read_unlock();
return found;
}
/*
* Perform an automount
* - return -EISDIR to tell follow_managed() to stop and return the path we
* were called with.
*/
static int follow_automount(struct path *path, int *count, unsigned lookup_flags)
{
struct dentry *dentry = path->dentry;
/* We don't want to mount if someone's just doing a stat -
* unless they're stat'ing a directory and appended a '/' to
* the name.
*
* We do, however, want to mount if someone wants to open or
* create a file of any type under the mountpoint, wants to
* traverse through the mountpoint or wants to open the
* mounted directory. Also, autofs may mark negative dentries
* as being automount points. These will need the attentions
* of the daemon to instantiate them before they can be used.
*/
if (!(lookup_flags & (LOOKUP_PARENT | LOOKUP_DIRECTORY |
LOOKUP_OPEN | LOOKUP_CREATE | LOOKUP_AUTOMOUNT)) &&
dentry->d_inode)
return -EISDIR;
if (count && (*count)++ >= MAXSYMLINKS)
return -ELOOP;
return finish_automount(dentry->d_op->d_automount(path), path);
}
/*
* mount traversal - out-of-line part. One note on ->d_flags accesses -
* dentries are pinned but not locked here, so negative dentry can go
* positive right under us. Use of smp_load_acquire() provides a barrier
* sufficient for ->d_inode and ->d_flags consistency.
*/
static int __traverse_mounts(struct path *path, unsigned flags, bool *jumped,
int *count, unsigned lookup_flags)
{
struct vfsmount *mnt = path->mnt;
bool need_mntput = false;
int ret = 0;
while (flags & DCACHE_MANAGED_DENTRY) {
/* Allow the filesystem to manage the transit without i_mutex
* being held. */
if (flags & DCACHE_MANAGE_TRANSIT) {
ret = path->dentry->d_op->d_manage(path, false);
flags = smp_load_acquire(&path->dentry->d_flags);
if (ret < 0)
break;
}
if (flags & DCACHE_MOUNTED) { // something's mounted on it..
struct vfsmount *mounted = lookup_mnt(path);
if (mounted) { // ... in our namespace
dput(path->dentry);
if (need_mntput)
mntput(path->mnt);
path->mnt = mounted;
path->dentry = dget(mounted->mnt_root);
// here we know it's positive
flags = path->dentry->d_flags;
need_mntput = true;
continue;
}
}
if (!(flags & DCACHE_NEED_AUTOMOUNT))
break;
// uncovered automount point
ret = follow_automount(path, count, lookup_flags);
flags = smp_load_acquire(&path->dentry->d_flags);
if (ret < 0)
break;
}
if (ret == -EISDIR)
ret = 0;
// possible if you race with several mount --move
if (need_mntput && path->mnt == mnt)
mntput(path->mnt);
if (!ret && unlikely(d_flags_negative(flags)))
ret = -ENOENT;
*jumped = need_mntput;
return ret;
}
static inline int traverse_mounts(struct path *path, bool *jumped,
int *count, unsigned lookup_flags)
{
unsigned flags = smp_load_acquire(&path->dentry->d_flags);
/* fastpath */
if (likely(!(flags & DCACHE_MANAGED_DENTRY))) {
*jumped = false; if (unlikely(d_flags_negative(flags)))
return -ENOENT;
return 0;
}
return __traverse_mounts(path, flags, jumped, count, lookup_flags);
}
int follow_down_one(struct path *path)
{
struct vfsmount *mounted;
mounted = lookup_mnt(path);
if (mounted) {
dput(path->dentry);
mntput(path->mnt);
path->mnt = mounted;
path->dentry = dget(mounted->mnt_root);
return 1;
}
return 0;
}
EXPORT_SYMBOL(follow_down_one);
/*
* Follow down to the covering mount currently visible to userspace. At each
* point, the filesystem owning that dentry may be queried as to whether the
* caller is permitted to proceed or not.
*/
int follow_down(struct path *path, unsigned int flags)
{
struct vfsmount *mnt = path->mnt;
bool jumped;
int ret = traverse_mounts(path, &jumped, NULL, flags);
if (path->mnt != mnt)
mntput(mnt);
return ret;
}
EXPORT_SYMBOL(follow_down);
/*
* Try to skip to top of mountpoint pile in rcuwalk mode. Fail if
* we meet a managed dentry that would need blocking.
*/
static bool __follow_mount_rcu(struct nameidata *nd, struct path *path)
{
struct dentry *dentry = path->dentry;
unsigned int flags = dentry->d_flags;
if (likely(!(flags & DCACHE_MANAGED_DENTRY)))
return true;
if (unlikely(nd->flags & LOOKUP_NO_XDEV))
return false;
for (;;) {
/*
* Don't forget we might have a non-mountpoint managed dentry
* that wants to block transit.
*/
if (unlikely(flags & DCACHE_MANAGE_TRANSIT)) { int res = dentry->d_op->d_manage(path, true);
if (res)
return res == -EISDIR; flags = dentry->d_flags;
}
if (flags & DCACHE_MOUNTED) { struct mount *mounted = __lookup_mnt(path->mnt, dentry);
if (mounted) {
path->mnt = &mounted->mnt;
dentry = path->dentry = mounted->mnt.mnt_root;
nd->state |= ND_JUMPED;
nd->next_seq = read_seqcount_begin(&dentry->d_seq);
flags = dentry->d_flags;
// makes sure that non-RCU pathwalk could reach
// this state.
if (read_seqretry(&mount_lock, nd->m_seq))
return false;
continue;
}
if (read_seqretry(&mount_lock, nd->m_seq))
return false;
}
return !(flags & DCACHE_NEED_AUTOMOUNT);
}
}
static inline int handle_mounts(struct nameidata *nd, struct dentry *dentry,
struct path *path)
{
bool jumped;
int ret;
path->mnt = nd->path.mnt;
path->dentry = dentry;
if (nd->flags & LOOKUP_RCU) {
unsigned int seq = nd->next_seq; if (likely(__follow_mount_rcu(nd, path)))
return 0;
// *path and nd->next_seq might've been clobbered
path->mnt = nd->path.mnt;
path->dentry = dentry;
nd->next_seq = seq;
if (!try_to_unlazy_next(nd, dentry))
return -ECHILD;
}
ret = traverse_mounts(path, &jumped, &nd->total_link_count, nd->flags);
if (jumped) {
if (unlikely(nd->flags & LOOKUP_NO_XDEV))
ret = -EXDEV;
else
nd->state |= ND_JUMPED;
}
if (unlikely(ret)) {
dput(path->dentry);
if (path->mnt != nd->path.mnt)
mntput(path->mnt);
}
return ret;
}
/*
* This looks up the name in dcache and possibly revalidates the found dentry.
* NULL is returned if the dentry does not exist in the cache.
*/
static struct dentry *lookup_dcache(const struct qstr *name,
struct dentry *dir,
unsigned int flags)
{
struct dentry *dentry = d_lookup(dir, name);
if (dentry) {
int error = d_revalidate(dentry, flags);
if (unlikely(error <= 0)) {
if (!error)
d_invalidate(dentry);
dput(dentry);
return ERR_PTR(error);
}
}
return dentry;
}
/*
* Parent directory has inode locked exclusive. This is one
* and only case when ->lookup() gets called on non in-lookup
* dentries - as the matter of fact, this only gets called
* when directory is guaranteed to have no in-lookup children
* at all.
*/
struct dentry *lookup_one_qstr_excl(const struct qstr *name,
struct dentry *base,
unsigned int flags)
{
struct dentry *dentry = lookup_dcache(name, base, flags);
struct dentry *old;
struct inode *dir = base->d_inode;
if (dentry)
return dentry;
/* Don't create child dentry for a dead directory. */
if (unlikely(IS_DEADDIR(dir)))
return ERR_PTR(-ENOENT);
dentry = d_alloc(base, name);
if (unlikely(!dentry))
return ERR_PTR(-ENOMEM);
old = dir->i_op->lookup(dir, dentry, flags);
if (unlikely(old)) {
dput(dentry);
dentry = old;
}
return dentry;
}
EXPORT_SYMBOL(lookup_one_qstr_excl);
static struct dentry *lookup_fast(struct nameidata *nd)
{
struct dentry *dentry, *parent = nd->path.dentry;
int status = 1;
/*
* Rename seqlock is not required here because in the off chance
* of a false negative due to a concurrent rename, the caller is
* going to fall back to non-racy lookup.
*/
if (nd->flags & LOOKUP_RCU) {
dentry = __d_lookup_rcu(parent, &nd->last, &nd->next_seq);
if (unlikely(!dentry)) {
if (!try_to_unlazy(nd))
return ERR_PTR(-ECHILD);
return NULL;
}
/*
* This sequence count validates that the parent had no
* changes while we did the lookup of the dentry above.
*/
if (read_seqcount_retry(&parent->d_seq, nd->seq))
return ERR_PTR(-ECHILD);
status = d_revalidate(dentry, nd->flags);
if (likely(status > 0))
return dentry;
if (!try_to_unlazy_next(nd, dentry))
return ERR_PTR(-ECHILD);
if (status == -ECHILD)
/* we'd been told to redo it in non-rcu mode */
status = d_revalidate(dentry, nd->flags);
} else {
dentry = __d_lookup(parent, &nd->last);
if (unlikely(!dentry))
return NULL;
status = d_revalidate(dentry, nd->flags);
}
if (unlikely(status <= 0)) { if (!status) d_invalidate(dentry); dput(dentry);
return ERR_PTR(status);
}
return dentry;
}
/* Fast lookup failed, do it the slow way */
static struct dentry *__lookup_slow(const struct qstr *name,
struct dentry *dir,
unsigned int flags)
{
struct dentry *dentry, *old;
struct inode *inode = dir->d_inode;
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
/* Don't go there if it's already dead */
if (unlikely(IS_DEADDIR(inode)))
return ERR_PTR(-ENOENT);
again:
dentry = d_alloc_parallel(dir, name, &wq);
if (IS_ERR(dentry))
return dentry;
if (unlikely(!d_in_lookup(dentry))) { int error = d_revalidate(dentry, flags);
if (unlikely(error <= 0)) {
if (!error) { d_invalidate(dentry);
dput(dentry);
goto again;
}
dput(dentry);
dentry = ERR_PTR(error);
}
} else {
old = inode->i_op->lookup(inode, dentry, flags); d_lookup_done(dentry); if (unlikely(old)) { dput(dentry);
dentry = old;
}
}
return dentry;
}
static struct dentry *lookup_slow(const struct qstr *name,
struct dentry *dir,
unsigned int flags)
{
struct inode *inode = dir->d_inode;
struct dentry *res;
inode_lock_shared(inode);
res = __lookup_slow(name, dir, flags);
inode_unlock_shared(inode);
return res;
}
static inline int may_lookup(struct mnt_idmap *idmap,
struct nameidata *nd)
{
if (nd->flags & LOOKUP_RCU) {
int err = inode_permission(idmap, nd->inode, MAY_EXEC|MAY_NOT_BLOCK);
if (!err) // success, keep going
return 0;
if (!try_to_unlazy(nd))
return -ECHILD; // redo it all non-lazy
if (err != -ECHILD) // hard error
return err;
}
return inode_permission(idmap, nd->inode, MAY_EXEC);
}
static int reserve_stack(struct nameidata *nd, struct path *link)
{
if (unlikely(nd->total_link_count++ >= MAXSYMLINKS))
return -ELOOP;
if (likely(nd->depth != EMBEDDED_LEVELS))
return 0;
if (likely(nd->stack != nd->internal))
return 0;
if (likely(nd_alloc_stack(nd)))
return 0;
if (nd->flags & LOOKUP_RCU) {
// we need to grab link before we do unlazy. And we can't skip
// unlazy even if we fail to grab the link - cleanup needs it
bool grabbed_link = legitimize_path(nd, link, nd->next_seq); if (!try_to_unlazy(nd) || !grabbed_link)
return -ECHILD;
if (nd_alloc_stack(nd))
return 0;
}
return -ENOMEM;
}
enum {WALK_TRAILING = 1, WALK_MORE = 2, WALK_NOFOLLOW = 4};
static const char *pick_link(struct nameidata *nd, struct path *link,
struct inode *inode, int flags)
{
struct saved *last;
const char *res;
int error = reserve_stack(nd, link);
if (unlikely(error)) {
if (!(nd->flags & LOOKUP_RCU)) path_put(link);
return ERR_PTR(error);
}
last = nd->stack + nd->depth++;
last->link = *link;
clear_delayed_call(&last->done);
last->seq = nd->next_seq;
if (flags & WALK_TRAILING) {
error = may_follow_link(nd, inode);
if (unlikely(error))
return ERR_PTR(error);
}
if (unlikely(nd->flags & LOOKUP_NO_SYMLINKS) || unlikely(link->mnt->mnt_flags & MNT_NOSYMFOLLOW))
return ERR_PTR(-ELOOP);
if (!(nd->flags & LOOKUP_RCU)) { touch_atime(&last->link);
cond_resched();
} else if (atime_needs_update(&last->link, inode)) { if (!try_to_unlazy(nd))
return ERR_PTR(-ECHILD);
touch_atime(&last->link);
}
error = security_inode_follow_link(link->dentry, inode,
nd->flags & LOOKUP_RCU);
if (unlikely(error))
return ERR_PTR(error); res = READ_ONCE(inode->i_link);
if (!res) {
const char * (*get)(struct dentry *, struct inode *,
struct delayed_call *);
get = inode->i_op->get_link;
if (nd->flags & LOOKUP_RCU) {
res = get(NULL, inode, &last->done); if (res == ERR_PTR(-ECHILD) && try_to_unlazy(nd))
res = get(link->dentry, inode, &last->done);
} else {
res = get(link->dentry, inode, &last->done);
}
if (!res)
goto all_done;
if (IS_ERR(res))
return res;
}
if (*res == '/') { error = nd_jump_root(nd);
if (unlikely(error))
return ERR_PTR(error); while (unlikely(*++res == '/'))
;
}
if (*res)
return res;
all_done: // pure jump
put_link(nd); return NULL;
}
/*
* Do we need to follow links? We _really_ want to be able
* to do this check without having to look at inode->i_op,
* so we keep a cache of "no, this doesn't need follow_link"
* for the common case.
*
* NOTE: dentry must be what nd->next_seq had been sampled from.
*/
static const char *step_into(struct nameidata *nd, int flags,
struct dentry *dentry)
{
struct path path;
struct inode *inode;
int err = handle_mounts(nd, dentry, &path); if (err < 0)
return ERR_PTR(err);
inode = path.dentry->d_inode;
if (likely(!d_is_symlink(path.dentry)) ||
((flags & WALK_TRAILING) && !(nd->flags & LOOKUP_FOLLOW)) || (flags & WALK_NOFOLLOW)) {
/* not a symlink or should not follow */
if (nd->flags & LOOKUP_RCU) { if (read_seqcount_retry(&path.dentry->d_seq, nd->next_seq))
return ERR_PTR(-ECHILD);
if (unlikely(!inode))
return ERR_PTR(-ENOENT);
} else {
dput(nd->path.dentry);
if (nd->path.mnt != path.mnt)
mntput(nd->path.mnt);
}
nd->path = path;
nd->inode = inode;
nd->seq = nd->next_seq;
return NULL;
}
if (nd->flags & LOOKUP_RCU) {
/* make sure that d_is_symlink above matches inode */
if (read_seqcount_retry(&path.dentry->d_seq, nd->next_seq))
return ERR_PTR(-ECHILD);
} else {
if (path.mnt == nd->path.mnt) mntget(path.mnt);
}
return pick_link(nd, &path, inode, flags);
}
static struct dentry *follow_dotdot_rcu(struct nameidata *nd)
{
struct dentry *parent, *old;
if (path_equal(&nd->path, &nd->root))
goto in_root;
if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) { struct path path;
unsigned seq;
if (!choose_mountpoint_rcu(real_mount(nd->path.mnt),
&nd->root, &path, &seq))
goto in_root; if (unlikely(nd->flags & LOOKUP_NO_XDEV)) return ERR_PTR(-ECHILD); nd->path = path;
nd->inode = path.dentry->d_inode;
nd->seq = seq;
// makes sure that non-RCU pathwalk could reach this state
if (read_seqretry(&mount_lock, nd->m_seq))
return ERR_PTR(-ECHILD);
/* we know that mountpoint was pinned */
}
old = nd->path.dentry;
parent = old->d_parent; nd->next_seq = read_seqcount_begin(&parent->d_seq);
// makes sure that non-RCU pathwalk could reach this state
if (read_seqcount_retry(&old->d_seq, nd->seq))
return ERR_PTR(-ECHILD);
if (unlikely(!path_connected(nd->path.mnt, parent)))
return ERR_PTR(-ECHILD);
return parent;
in_root:
if (read_seqretry(&mount_lock, nd->m_seq))
return ERR_PTR(-ECHILD);
if (unlikely(nd->flags & LOOKUP_BENEATH))
return ERR_PTR(-ECHILD);
nd->next_seq = nd->seq;
return nd->path.dentry;
}
static struct dentry *follow_dotdot(struct nameidata *nd)
{
struct dentry *parent;
if (path_equal(&nd->path, &nd->root))
goto in_root;
if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) { struct path path; if (!choose_mountpoint(real_mount(nd->path.mnt),
&nd->root, &path))
goto in_root; path_put(&nd->path);
nd->path = path;
nd->inode = path.dentry->d_inode;
if (unlikely(nd->flags & LOOKUP_NO_XDEV)) return ERR_PTR(-EXDEV);
}
/* rare case of legitimate dget_parent()... */
parent = dget_parent(nd->path.dentry); if (unlikely(!path_connected(nd->path.mnt, parent))) { dput(parent);
return ERR_PTR(-ENOENT);
}
return parent;
in_root:
if (unlikely(nd->flags & LOOKUP_BENEATH))
return ERR_PTR(-EXDEV);
return dget(nd->path.dentry);
}
static const char *handle_dots(struct nameidata *nd, int type)
{
if (type == LAST_DOTDOT) {
const char *error = NULL;
struct dentry *parent;
if (!nd->root.mnt) { error = ERR_PTR(set_root(nd));
if (error)
return error;
}
if (nd->flags & LOOKUP_RCU) parent = follow_dotdot_rcu(nd);
else
parent = follow_dotdot(nd); if (IS_ERR(parent))
return ERR_CAST(parent);
error = step_into(nd, WALK_NOFOLLOW, parent);
if (unlikely(error))
return error;
if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
/*
* If there was a racing rename or mount along our
* path, then we can't be sure that ".." hasn't jumped
* above nd->root (and so userspace should retry or use
* some fallback).
*/
smp_rmb();
if (__read_seqcount_retry(&mount_lock.seqcount, nd->m_seq))
return ERR_PTR(-EAGAIN);
if (__read_seqcount_retry(&rename_lock.seqcount, nd->r_seq))
return ERR_PTR(-EAGAIN);
}
}
return NULL;
}
static const char *walk_component(struct nameidata *nd, int flags)
{
struct dentry *dentry;
/*
* "." and ".." are special - ".." especially so because it has
* to be able to know about the current root directory and
* parent relationships.
*/
if (unlikely(nd->last_type != LAST_NORM)) { if (!(flags & WALK_MORE) && nd->depth) put_link(nd); return handle_dots(nd, nd->last_type);
}
dentry = lookup_fast(nd);
if (IS_ERR(dentry))
return ERR_CAST(dentry);
if (unlikely(!dentry)) { dentry = lookup_slow(&nd->last, nd->path.dentry, nd->flags);
if (IS_ERR(dentry))
return ERR_CAST(dentry);
}
if (!(flags & WALK_MORE) && nd->depth) put_link(nd); return step_into(nd, flags, dentry);
}
/*
* We can do the critical dentry name comparison and hashing
* operations one word at a time, but we are limited to:
*
* - Architectures with fast unaligned word accesses. We could
* do a "get_unaligned()" if this helps and is sufficiently
* fast.
*
* - non-CONFIG_DEBUG_PAGEALLOC configurations (so that we
* do not trap on the (extremely unlikely) case of a page
* crossing operation.
*
* - Furthermore, we need an efficient 64-bit compile for the
* 64-bit case in order to generate the "number of bytes in
* the final mask". Again, that could be replaced with a
* efficient population count instruction or similar.
*/
#ifdef CONFIG_DCACHE_WORD_ACCESS
#include <asm/word-at-a-time.h>
#ifdef HASH_MIX
/* Architecture provides HASH_MIX and fold_hash() in <asm/hash.h> */
#elif defined(CONFIG_64BIT)
/*
* Register pressure in the mixing function is an issue, particularly
* on 32-bit x86, but almost any function requires one state value and
* one temporary. Instead, use a function designed for two state values
* and no temporaries.
*
* This function cannot create a collision in only two iterations, so
* we have two iterations to achieve avalanche. In those two iterations,
* we have six layers of mixing, which is enough to spread one bit's
* influence out to 2^6 = 64 state bits.
*
* Rotate constants are scored by considering either 64 one-bit input
* deltas or 64*63/2 = 2016 two-bit input deltas, and finding the
* probability of that delta causing a change to each of the 128 output
* bits, using a sample of random initial states.
*
* The Shannon entropy of the computed probabilities is then summed
* to produce a score. Ideally, any input change has a 50% chance of
* toggling any given output bit.
*
* Mixing scores (in bits) for (12,45):
* Input delta: 1-bit 2-bit
* 1 round: 713.3 42542.6
* 2 rounds: 2753.7 140389.8
* 3 rounds: 5954.1 233458.2
* 4 rounds: 7862.6 256672.2
* Perfect: 8192 258048
* (64*128) (64*63/2 * 128)
*/
#define HASH_MIX(x, y, a) \
( x ^= (a), \
y ^= x, x = rol64(x,12),\
x += y, y = rol64(y,45),\
y *= 9 )
/*
* Fold two longs into one 32-bit hash value. This must be fast, but
* latency isn't quite as critical, as there is a fair bit of additional
* work done before the hash value is used.
*/
static inline unsigned int fold_hash(unsigned long x, unsigned long y)
{
y ^= x * GOLDEN_RATIO_64; y *= GOLDEN_RATIO_64;
return y >> 32;
}
#else /* 32-bit case */
/*
* Mixing scores (in bits) for (7,20):
* Input delta: 1-bit 2-bit
* 1 round: 330.3 9201.6
* 2 rounds: 1246.4 25475.4
* 3 rounds: 1907.1 31295.1
* 4 rounds: 2042.3 31718.6
* Perfect: 2048 31744
* (32*64) (32*31/2 * 64)
*/
#define HASH_MIX(x, y, a) \
( x ^= (a), \
y ^= x, x = rol32(x, 7),\
x += y, y = rol32(y,20),\
y *= 9 )
static inline unsigned int fold_hash(unsigned long x, unsigned long y)
{
/* Use arch-optimized multiply if one exists */
return __hash_32(y ^ __hash_32(x));
}
#endif
/*
* Return the hash of a string of known length. This is carfully
* designed to match hash_name(), which is the more critical function.
* In particular, we must end by hashing a final word containing 0..7
* payload bytes, to match the way that hash_name() iterates until it
* finds the delimiter after the name.
*/
unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
{
unsigned long a, x = 0, y = (unsigned long)salt;
for (;;) {
if (!len)
goto done;
a = load_unaligned_zeropad(name);
if (len < sizeof(unsigned long))
break;
HASH_MIX(x, y, a);
name += sizeof(unsigned long);
len -= sizeof(unsigned long);
}
x ^= a & bytemask_from_count(len);
done:
return fold_hash(x, y);
}
EXPORT_SYMBOL(full_name_hash);
/* Return the "hash_len" (hash and length) of a null-terminated string */
u64 hashlen_string(const void *salt, const char *name)
{
unsigned long a = 0, x = 0, y = (unsigned long)salt;
unsigned long adata, mask, len;
const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
len = 0;
goto inside;
do {
HASH_MIX(x, y, a);
len += sizeof(unsigned long);
inside:
a = load_unaligned_zeropad(name+len);
} while (!has_zero(a, &adata, &constants));
adata = prep_zero_mask(a, adata, &constants);
mask = create_zero_mask(adata);
x ^= a & zero_bytemask(mask);
return hashlen_create(fold_hash(x, y), len + find_zero(mask));
}
EXPORT_SYMBOL(hashlen_string);
/*
* Calculate the length and hash of the path component, and
* return the "hash_len" as the result.
*/
static inline u64 hash_name(const void *salt, const char *name)
{
unsigned long a = 0, b, x = 0, y = (unsigned long)salt;
unsigned long adata, bdata, mask, len;
const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
len = 0;
goto inside;
do {
HASH_MIX(x, y, a);
len += sizeof(unsigned long);
inside:
a = load_unaligned_zeropad(name+len);
b = a ^ REPEAT_BYTE('/');
} while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));
adata = prep_zero_mask(a, adata, &constants);
bdata = prep_zero_mask(b, bdata, &constants);
mask = create_zero_mask(adata | bdata);
x ^= a & zero_bytemask(mask);
return hashlen_create(fold_hash(x, y), len + find_zero(mask));
}
#else /* !CONFIG_DCACHE_WORD_ACCESS: Slow, byte-at-a-time version */
/* Return the hash of a string of known length */
unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
{
unsigned long hash = init_name_hash(salt);
while (len--)
hash = partial_name_hash((unsigned char)*name++, hash);
return end_name_hash(hash);
}
EXPORT_SYMBOL(full_name_hash);
/* Return the "hash_len" (hash and length) of a null-terminated string */
u64 hashlen_string(const void *salt, const char *name)
{
unsigned long hash = init_name_hash(salt);
unsigned long len = 0, c;
c = (unsigned char)*name;
while (c) {
len++;
hash = partial_name_hash(c, hash);
c = (unsigned char)name[len];
}
return hashlen_create(end_name_hash(hash), len);
}
EXPORT_SYMBOL(hashlen_string);
/*
* We know there's a real path component here of at least
* one character.
*/
static inline u64 hash_name(const void *salt, const char *name)
{
unsigned long hash = init_name_hash(salt);
unsigned long len = 0, c;
c = (unsigned char)*name;
do {
len++;
hash = partial_name_hash(c, hash);
c = (unsigned char)name[len];
} while (c && c != '/');
return hashlen_create(end_name_hash(hash), len);
}
#endif
/*
* Name resolution.
* This is the basic name resolution function, turning a pathname into
* the final dentry. We expect 'base' to be positive and a directory.
*
* Returns 0 and nd will have valid dentry and mnt on success.
* Returns error and drops reference to input namei data on failure.
*/
static int link_path_walk(const char *name, struct nameidata *nd)
{
int depth = 0; // depth <= nd->depth
int err;
nd->last_type = LAST_ROOT; nd->flags |= LOOKUP_PARENT; if (IS_ERR(name)) return PTR_ERR(name); while (*name=='/') name++; if (!*name) { nd->dir_mode = 0; // short-circuit the 'hardening' idiocy
return 0;
}
/* At this point we know we have a real path component. */
for(;;) {
struct mnt_idmap *idmap;
const char *link;
u64 hash_len;
int type;
idmap = mnt_idmap(nd->path.mnt); err = may_lookup(idmap, nd);
if (err)
return err;
hash_len = hash_name(nd->path.dentry, name);
type = LAST_NORM;
if (name[0] == '.') switch (hashlen_len(hash_len)) {
case 2:
if (name[1] == '.') {
type = LAST_DOTDOT;
nd->state |= ND_JUMPED;
}
break;
case 1:
type = LAST_DOT;
}
if (likely(type == LAST_NORM)) {
struct dentry *parent = nd->path.dentry;
nd->state &= ~ND_JUMPED;
if (unlikely(parent->d_flags & DCACHE_OP_HASH)) {
struct qstr this = { { .hash_len = hash_len }, .name = name };
err = parent->d_op->d_hash(parent, &this);
if (err < 0)
return err;
hash_len = this.hash_len;
name = this.name;
}
}
nd->last.hash_len = hash_len;
nd->last.name = name;
nd->last_type = type;
name += hashlen_len(hash_len);
if (!*name)
goto OK;
/*
* If it wasn't NUL, we know it was '/'. Skip that
* slash, and continue until no more slashes.
*/
do {
name++;
} while (unlikely(*name == '/'));
if (unlikely(!*name)) {
OK:
/* pathname or trailing symlink, done */
if (!depth) { nd->dir_vfsuid = i_uid_into_vfsuid(idmap, nd->inode);
nd->dir_mode = nd->inode->i_mode;
nd->flags &= ~LOOKUP_PARENT;
return 0;
}
/* last component of nested symlink */
name = nd->stack[--depth].name;
link = walk_component(nd, 0);
} else {
/* not the last component */
link = walk_component(nd, WALK_MORE);
}
if (unlikely(link)) { if (IS_ERR(link)) return PTR_ERR(link);
/* a symlink to follow */
nd->stack[depth++].name = name;
name = link;
continue;
}
if (unlikely(!d_can_lookup(nd->path.dentry))) { if (nd->flags & LOOKUP_RCU) { if (!try_to_unlazy(nd))
return -ECHILD;
}
return -ENOTDIR;
}
}
}
/* must be paired with terminate_walk() */
static const char *path_init(struct nameidata *nd, unsigned flags)
{
int error;
const char *s = nd->name->name;
/* LOOKUP_CACHED requires RCU, ask caller to retry */
if ((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED)
return ERR_PTR(-EAGAIN);
if (!*s)
flags &= ~LOOKUP_RCU; if (flags & LOOKUP_RCU) rcu_read_lock();
else
nd->seq = nd->next_seq = 0; nd->flags = flags;
nd->state |= ND_JUMPED;
nd->m_seq = __read_seqcount_begin(&mount_lock.seqcount); nd->r_seq = __read_seqcount_begin(&rename_lock.seqcount);
smp_rmb();
if (nd->state & ND_ROOT_PRESET) {
struct dentry *root = nd->root.dentry;
struct inode *inode = root->d_inode;
if (*s && unlikely(!d_can_lookup(root)))
return ERR_PTR(-ENOTDIR);
nd->path = nd->root;
nd->inode = inode;
if (flags & LOOKUP_RCU) {
nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
nd->root_seq = nd->seq;
} else {
path_get(&nd->path);
}
return s;
}
nd->root.mnt = NULL;
/* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */
if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) { error = nd_jump_root(nd);
if (unlikely(error))
return ERR_PTR(error);
return s;
}
/* Relative pathname -- get the starting-point it is relative to. */
if (nd->dfd == AT_FDCWD) { if (flags & LOOKUP_RCU) { struct fs_struct *fs = current->fs;
unsigned seq;
do {
seq = read_seqcount_begin(&fs->seq);
nd->path = fs->pwd;
nd->inode = nd->path.dentry->d_inode;
nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
} while (read_seqcount_retry(&fs->seq, seq));
} else {
get_fs_pwd(current->fs, &nd->path);
nd->inode = nd->path.dentry->d_inode;
}
} else {
/* Caller must check execute permissions on the starting path component */
struct fd f = fdget_raw(nd->dfd);
struct dentry *dentry;
if (!f.file)
return ERR_PTR(-EBADF);
dentry = f.file->f_path.dentry;
if (*s && unlikely(!d_can_lookup(dentry))) { fdput(f);
return ERR_PTR(-ENOTDIR);
}
nd->path = f.file->f_path;
if (flags & LOOKUP_RCU) {
nd->inode = nd->path.dentry->d_inode; nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
} else {
path_get(&nd->path); nd->inode = nd->path.dentry->d_inode;
}
fdput(f);
}
/* For scoped-lookups we need to set the root to the dirfd as well. */
if (flags & LOOKUP_IS_SCOPED) { nd->root = nd->path;
if (flags & LOOKUP_RCU) {
nd->root_seq = nd->seq;
} else {
path_get(&nd->root); nd->state |= ND_ROOT_GRABBED;
}
}
return s;
}
static inline const char *lookup_last(struct nameidata *nd)
{
if (nd->last_type == LAST_NORM && nd->last.name[nd->last.len]) nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY; return walk_component(nd, WALK_TRAILING);
}
static int handle_lookup_down(struct nameidata *nd)
{
if (!(nd->flags & LOOKUP_RCU)) dget(nd->path.dentry); nd->next_seq = nd->seq;
return PTR_ERR(step_into(nd, WALK_NOFOLLOW, nd->path.dentry));
}
/* Returns 0 and nd will be valid on success; Returns error, otherwise. */
static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path)
{
const char *s = path_init(nd, flags);
int err;
if (unlikely(flags & LOOKUP_DOWN) && !IS_ERR(s)) { err = handle_lookup_down(nd);
if (unlikely(err < 0))
s = ERR_PTR(err);
}
while (!(err = link_path_walk(s, nd)) && (s = lookup_last(nd)) != NULL)
;
if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) { err = handle_lookup_down(nd);
nd->state &= ~ND_JUMPED; // no d_weak_revalidate(), please...
}
if (!err)
err = complete_walk(nd); if (!err && nd->flags & LOOKUP_DIRECTORY) if (!d_can_lookup(nd->path.dentry))
err = -ENOTDIR;
if (!err) {
*path = nd->path;
nd->path.mnt = NULL;
nd->path.dentry = NULL;
}
terminate_walk(nd);
return err;
}
int filename_lookup(int dfd, struct filename *name, unsigned flags,
struct path *path, struct path *root)
{
int retval;
struct nameidata nd;
if (IS_ERR(name))
return PTR_ERR(name);
set_nameidata(&nd, dfd, name, root);
retval = path_lookupat(&nd, flags | LOOKUP_RCU, path);
if (unlikely(retval == -ECHILD))
retval = path_lookupat(&nd, flags, path);
if (unlikely(retval == -ESTALE))
retval = path_lookupat(&nd, flags | LOOKUP_REVAL, path);
if (likely(!retval))
audit_inode(name, path->dentry,
flags & LOOKUP_MOUNTPOINT ? AUDIT_INODE_NOEVAL : 0);
restore_nameidata();
return retval;
}
/* Returns 0 and nd will be valid on success; Returns error, otherwise. */
static int path_parentat(struct nameidata *nd, unsigned flags,
struct path *parent)
{
const char *s = path_init(nd, flags);
int err = link_path_walk(s, nd);
if (!err)
err = complete_walk(nd);
if (!err) {
*parent = nd->path;
nd->path.mnt = NULL;
nd->path.dentry = NULL;
}
terminate_walk(nd);
return err;
}
/* Note: this does not consume "name" */
static int __filename_parentat(int dfd, struct filename *name,
unsigned int flags, struct path *parent,
struct qstr *last, int *type,
const struct path *root)
{
int retval;
struct nameidata nd;
if (IS_ERR(name))
return PTR_ERR(name);
set_nameidata(&nd, dfd, name, root);
retval = path_parentat(&nd, flags | LOOKUP_RCU, parent);
if (unlikely(retval == -ECHILD))
retval = path_parentat(&nd, flags, parent);
if (unlikely(retval == -ESTALE))
retval = path_parentat(&nd, flags | LOOKUP_REVAL, parent);
if (likely(!retval)) {
*last = nd.last;
*type = nd.last_type;
audit_inode(name, parent->dentry, AUDIT_INODE_PARENT);
}
restore_nameidata();
return retval;
}
static int filename_parentat(int dfd, struct filename *name,
unsigned int flags, struct path *parent,
struct qstr *last, int *type)
{
return __filename_parentat(dfd, name, flags, parent, last, type, NULL);
}
/* does lookup, returns the object with parent locked */
static struct dentry *__kern_path_locked(int dfd, struct filename *name, struct path *path)
{
struct dentry *d;
struct qstr last;
int type, error;
error = filename_parentat(dfd, name, 0, path, &last, &type);
if (error)
return ERR_PTR(error);
if (unlikely(type != LAST_NORM)) {
path_put(path);
return ERR_PTR(-EINVAL);
}
inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
d = lookup_one_qstr_excl(&last, path->dentry, 0);
if (IS_ERR(d)) {
inode_unlock(path->dentry->d_inode);
path_put(path);
}
return d;
}
struct dentry *kern_path_locked(const char *name, struct path *path)
{
struct filename *filename = getname_kernel(name);
struct dentry *res = __kern_path_locked(AT_FDCWD, filename, path);
putname(filename);
return res;
}
struct dentry *user_path_locked_at(int dfd, const char __user *name, struct path *path)
{
struct filename *filename = getname(name);
struct dentry *res = __kern_path_locked(dfd, filename, path);
putname(filename);
return res;
}
EXPORT_SYMBOL(user_path_locked_at);
int kern_path(const char *name, unsigned int flags, struct path *path)
{
struct filename *filename = getname_kernel(name);
int ret = filename_lookup(AT_FDCWD, filename, flags, path, NULL);
putname(filename);
return ret;
}
EXPORT_SYMBOL(kern_path);
/**
* vfs_path_parent_lookup - lookup a parent path relative to a dentry-vfsmount pair
* @filename: filename structure
* @flags: lookup flags
* @parent: pointer to struct path to fill
* @last: last component
* @type: type of the last component
* @root: pointer to struct path of the base directory
*/
int vfs_path_parent_lookup(struct filename *filename, unsigned int flags,
struct path *parent, struct qstr *last, int *type,
const struct path *root)
{
return __filename_parentat(AT_FDCWD, filename, flags, parent, last,
type, root);
}
EXPORT_SYMBOL(vfs_path_parent_lookup);
/**
* vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair
* @dentry: pointer to dentry of the base directory
* @mnt: pointer to vfs mount of the base directory
* @name: pointer to file name
* @flags: lookup flags
* @path: pointer to struct path to fill
*/
int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt,
const char *name, unsigned int flags,
struct path *path)
{
struct filename *filename;
struct path root = {.mnt = mnt, .dentry = dentry};
int ret;
filename = getname_kernel(name);
/* the first argument of filename_lookup() is ignored with root */
ret = filename_lookup(AT_FDCWD, filename, flags, path, &root);
putname(filename);
return ret;
}
EXPORT_SYMBOL(vfs_path_lookup);
static int lookup_one_common(struct mnt_idmap *idmap,
const char *name, struct dentry *base, int len,
struct qstr *this)
{
this->name = name;
this->len = len;
this->hash = full_name_hash(base, name, len);
if (!len)
return -EACCES;
if (is_dot_dotdot(name, len))
return -EACCES;
while (len--) {
unsigned int c = *(const unsigned char *)name++;
if (c == '/' || c == '\0')
return -EACCES;
}
/*
* See if the low-level filesystem might want
* to use its own hash..
*/
if (base->d_flags & DCACHE_OP_HASH) {
int err = base->d_op->d_hash(base, this);
if (err < 0)
return err;
}
return inode_permission(idmap, base->d_inode, MAY_EXEC);
}
/**
* try_lookup_one_len - filesystem helper to lookup single pathname component
* @name: pathname component to lookup
* @base: base directory to lookup from
* @len: maximum length @len should be interpreted to
*
* Look up a dentry by name in the dcache, returning NULL if it does not
* currently exist. The function does not try to create a dentry.
*
* Note that this routine is purely a helper for filesystem usage and should
* not be called by generic code.
*
* The caller must hold base->i_mutex.
*/
struct dentry *try_lookup_one_len(const char *name, struct dentry *base, int len)
{
struct qstr this;
int err;
WARN_ON_ONCE(!inode_is_locked(base->d_inode));
err = lookup_one_common(&nop_mnt_idmap, name, base, len, &this);
if (err)
return ERR_PTR(err);
return lookup_dcache(&this, base, 0);
}
EXPORT_SYMBOL(try_lookup_one_len);
/**
* lookup_one_len - filesystem helper to lookup single pathname component
* @name: pathname component to lookup
* @base: base directory to lookup from
* @len: maximum length @len should be interpreted to
*
* Note that this routine is purely a helper for filesystem usage and should
* not be called by generic code.
*
* The caller must hold base->i_mutex.
*/
struct dentry *lookup_one_len(const char *name, struct dentry *base, int len)
{
struct dentry *dentry;
struct qstr this;
int err;
WARN_ON_ONCE(!inode_is_locked(base->d_inode));
err = lookup_one_common(&nop_mnt_idmap, name, base, len, &this);
if (err)
return ERR_PTR(err);
dentry = lookup_dcache(&this, base, 0);
return dentry ? dentry : __lookup_slow(&this, base, 0);
}
EXPORT_SYMBOL(lookup_one_len);
/**
* lookup_one - filesystem helper to lookup single pathname component
* @idmap: idmap of the mount the lookup is performed from
* @name: pathname component to lookup
* @base: base directory to lookup from
* @len: maximum length @len should be interpreted to
*
* Note that this routine is purely a helper for filesystem usage and should
* not be called by generic code.
*
* The caller must hold base->i_mutex.
*/
struct dentry *lookup_one(struct mnt_idmap *idmap, const char *name,
struct dentry *base, int len)
{
struct dentry *dentry;
struct qstr this;
int err;
WARN_ON_ONCE(!inode_is_locked(base->d_inode));
err = lookup_one_common(idmap, name, base, len, &this);
if (err)
return ERR_PTR(err);
dentry = lookup_dcache(&this, base, 0);
return dentry ? dentry : __lookup_slow(&this, base, 0);
}
EXPORT_SYMBOL(lookup_one);
/**
* lookup_one_unlocked - filesystem helper to lookup single pathname component
* @idmap: idmap of the mount the lookup is performed from
* @name: pathname component to lookup
* @base: base directory to lookup from
* @len: maximum length @len should be interpreted to
*
* Note that this routine is purely a helper for filesystem usage and should
* not be called by generic code.
*
* Unlike lookup_one_len, it should be called without the parent
* i_mutex held, and will take the i_mutex itself if necessary.
*/
struct dentry *lookup_one_unlocked(struct mnt_idmap *idmap,
const char *name, struct dentry *base,
int len)
{
struct qstr this;
int err;
struct dentry *ret;
err = lookup_one_common(idmap, name, base, len, &this);
if (err)
return ERR_PTR(err);
ret = lookup_dcache(&this, base, 0);
if (!ret)
ret = lookup_slow(&this, base, 0);
return ret;
}
EXPORT_SYMBOL(lookup_one_unlocked);
/**
* lookup_one_positive_unlocked - filesystem helper to lookup single
* pathname component
* @idmap: idmap of the mount the lookup is performed from
* @name: pathname component to lookup
* @base: base directory to lookup from
* @len: maximum length @len should be interpreted to
*
* This helper will yield ERR_PTR(-ENOENT) on negatives. The helper returns
* known positive or ERR_PTR(). This is what most of the users want.
*
* Note that pinned negative with unlocked parent _can_ become positive at any
* time, so callers of lookup_one_unlocked() need to be very careful; pinned
* positives have >d_inode stable, so this one avoids such problems.
*
* Note that this routine is purely a helper for filesystem usage and should
* not be called by generic code.
*
* The helper should be called without i_mutex held.
*/
struct dentry *lookup_one_positive_unlocked(struct mnt_idmap *idmap,
const char *name,
struct dentry *base, int len)
{
struct dentry *ret = lookup_one_unlocked(idmap, name, base, len);
if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) {
dput(ret);
ret = ERR_PTR(-ENOENT);
}
return ret;
}
EXPORT_SYMBOL(lookup_one_positive_unlocked);
/**
* lookup_one_len_unlocked - filesystem helper to lookup single pathname component
* @name: pathname component to lookup
* @base: base directory to lookup from
* @len: maximum length @len should be interpreted to
*
* Note that this routine is purely a helper for filesystem usage and should
* not be called by generic code.
*
* Unlike lookup_one_len, it should be called without the parent
* i_mutex held, and will take the i_mutex itself if necessary.
*/
struct dentry *lookup_one_len_unlocked(const char *name,
struct dentry *base, int len)
{
return lookup_one_unlocked(&nop_mnt_idmap, name, base, len);
}
EXPORT_SYMBOL(lookup_one_len_unlocked);
/*
* Like lookup_one_len_unlocked(), except that it yields ERR_PTR(-ENOENT)
* on negatives. Returns known positive or ERR_PTR(); that's what
* most of the users want. Note that pinned negative with unlocked parent
* _can_ become positive at any time, so callers of lookup_one_len_unlocked()
* need to be very careful; pinned positives have ->d_inode stable, so
* this one avoids such problems.
*/
struct dentry *lookup_positive_unlocked(const char *name,
struct dentry *base, int len)
{
return lookup_one_positive_unlocked(&nop_mnt_idmap, name, base, len);
}
EXPORT_SYMBOL(lookup_positive_unlocked);
#ifdef CONFIG_UNIX98_PTYS
int path_pts(struct path *path)
{
/* Find something mounted on "pts" in the same directory as
* the input path.
*/
struct dentry *parent = dget_parent(path->dentry);
struct dentry *child;
struct qstr this = QSTR_INIT("pts", 3);
if (unlikely(!path_connected(path->mnt, parent))) {
dput(parent);
return -ENOENT;
}
dput(path->dentry);
path->dentry = parent;
child = d_hash_and_lookup(parent, &this);
if (IS_ERR_OR_NULL(child))
return -ENOENT;
path->dentry = child;
dput(parent);
follow_down(path, 0);
return 0;
}
#endif
int user_path_at_empty(int dfd, const char __user *name, unsigned flags,
struct path *path, int *empty)
{
struct filename *filename = getname_flags(name, flags, empty);
int ret = filename_lookup(dfd, filename, flags, path, NULL);
putname(filename);
return ret;
}
EXPORT_SYMBOL(user_path_at_empty);
int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
struct inode *inode)
{
kuid_t fsuid = current_fsuid();
if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), fsuid))
return 0;
if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, dir), fsuid))
return 0;
return !capable_wrt_inode_uidgid(idmap, inode, CAP_FOWNER);
}
EXPORT_SYMBOL(__check_sticky);
/*
* Check whether we can remove a link victim from directory dir, check
* whether the type of victim is right.
* 1. We can't do it if dir is read-only (done in permission())
* 2. We should have write and exec permissions on dir
* 3. We can't remove anything from append-only dir
* 4. We can't do anything with immutable dir (done in permission())
* 5. If the sticky bit on dir is set we should either
* a. be owner of dir, or
* b. be owner of victim, or
* c. have CAP_FOWNER capability
* 6. If the victim is append-only or immutable we can't do antyhing with
* links pointing to it.
* 7. If the victim has an unknown uid or gid we can't change the inode.
* 8. If we were asked to remove a directory and victim isn't one - ENOTDIR.
* 9. If we were asked to remove a non-directory and victim isn't one - EISDIR.
* 10. We can't remove a root or mountpoint.
* 11. We don't allow removal of NFS sillyrenamed files; it's handled by
* nfs_async_unlink().
*/
static int may_delete(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *victim, bool isdir)
{
struct inode *inode = d_backing_inode(victim);
int error;
if (d_is_negative(victim))
return -ENOENT;
BUG_ON(!inode);
BUG_ON(victim->d_parent->d_inode != dir);
/* Inode writeback is not safe when the uid or gid are invalid. */
if (!vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) ||
!vfsgid_valid(i_gid_into_vfsgid(idmap, inode)))
return -EOVERFLOW;
audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE);
error = inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC);
if (error)
return error;
if (IS_APPEND(dir))
return -EPERM;
if (check_sticky(idmap, dir, inode) || IS_APPEND(inode) ||
IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) ||
HAS_UNMAPPED_ID(idmap, inode))
return -EPERM;
if (isdir) {
if (!d_is_dir(victim))
return -ENOTDIR;
if (IS_ROOT(victim))
return -EBUSY;
} else if (d_is_dir(victim))
return -EISDIR;
if (IS_DEADDIR(dir))
return -ENOENT;
if (victim->d_flags & DCACHE_NFSFS_RENAMED)
return -EBUSY;
return 0;
}
/* Check whether we can create an object with dentry child in directory
* dir.
* 1. We can't do it if child already exists (open has special treatment for
* this case, but since we are inlined it's OK)
* 2. We can't do it if dir is read-only (done in permission())
* 3. We can't do it if the fs can't represent the fsuid or fsgid.
* 4. We should have write and exec permissions on dir
* 5. We can't do it if dir is immutable (done in permission())
*/
static inline int may_create(struct mnt_idmap *idmap,
struct inode *dir, struct dentry *child)
{
audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE);
if (child->d_inode)
return -EEXIST;
if (IS_DEADDIR(dir))
return -ENOENT;
if (!fsuidgid_has_mapping(dir->i_sb, idmap))
return -EOVERFLOW;
return inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC);
}
// p1 != p2, both are on the same filesystem, ->s_vfs_rename_mutex is held
static struct dentry *lock_two_directories(struct dentry *p1, struct dentry *p2)
{
struct dentry *p = p1, *q = p2, *r;
while ((r = p->d_parent) != p2 && r != p)
p = r;
if (r == p2) {
// p is a child of p2 and an ancestor of p1 or p1 itself
inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
inode_lock_nested(p1->d_inode, I_MUTEX_PARENT2);
return p;
}
// p is the root of connected component that contains p1
// p2 does not occur on the path from p to p1
while ((r = q->d_parent) != p1 && r != p && r != q)
q = r;
if (r == p1) {
// q is a child of p1 and an ancestor of p2 or p2 itself
inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2);
return q;
} else if (likely(r == p)) {
// both p2 and p1 are descendents of p
inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2);
return NULL;
} else { // no common ancestor at the time we'd been called
mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
return ERR_PTR(-EXDEV);
}
}
/*
* p1 and p2 should be directories on the same fs.
*/
struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
{
if (p1 == p2) {
inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
return NULL;
}
mutex_lock(&p1->d_sb->s_vfs_rename_mutex);
return lock_two_directories(p1, p2);
}
EXPORT_SYMBOL(lock_rename);
/*
* c1 and p2 should be on the same fs.
*/
struct dentry *lock_rename_child(struct dentry *c1, struct dentry *p2)
{
if (READ_ONCE(c1->d_parent) == p2) {
/*
* hopefully won't need to touch ->s_vfs_rename_mutex at all.
*/
inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
/*
* now that p2 is locked, nobody can move in or out of it,
* so the test below is safe.
*/
if (likely(c1->d_parent == p2))
return NULL;
/*
* c1 got moved out of p2 while we'd been taking locks;
* unlock and fall back to slow case.
*/
inode_unlock(p2->d_inode);
}
mutex_lock(&c1->d_sb->s_vfs_rename_mutex);
/*
* nobody can move out of any directories on this fs.
*/
if (likely(c1->d_parent != p2))
return lock_two_directories(c1->d_parent, p2);
/*
* c1 got moved into p2 while we were taking locks;
* we need p2 locked and ->s_vfs_rename_mutex unlocked,
* for consistency with lock_rename().
*/
inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
mutex_unlock(&c1->d_sb->s_vfs_rename_mutex);
return NULL;
}
EXPORT_SYMBOL(lock_rename_child);
void unlock_rename(struct dentry *p1, struct dentry *p2)
{
inode_unlock(p1->d_inode);
if (p1 != p2) {
inode_unlock(p2->d_inode);
mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
}
}
EXPORT_SYMBOL(unlock_rename);
/**
* vfs_prepare_mode - prepare the mode to be used for a new inode
* @idmap: idmap of the mount the inode was found from
* @dir: parent directory of the new inode
* @mode: mode of the new inode
* @mask_perms: allowed permission by the vfs
* @type: type of file to be created
*
* This helper consolidates and enforces vfs restrictions on the @mode of a new
* object to be created.
*
* Umask stripping depends on whether the filesystem supports POSIX ACLs (see
* the kernel documentation for mode_strip_umask()). Moving umask stripping
* after setgid stripping allows the same ordering for both non-POSIX ACL and
* POSIX ACL supporting filesystems.
*
* Note that it's currently valid for @type to be 0 if a directory is created.
* Filesystems raise that flag individually and we need to check whether each
* filesystem can deal with receiving S_IFDIR from the vfs before we enforce a
* non-zero type.
*
* Returns: mode to be passed to the filesystem
*/
static inline umode_t vfs_prepare_mode(struct mnt_idmap *idmap,
const struct inode *dir, umode_t mode,
umode_t mask_perms, umode_t type)
{
mode = mode_strip_sgid(idmap, dir, mode);
mode = mode_strip_umask(dir, mode);
/*
* Apply the vfs mandated allowed permission mask and set the type of
* file to be created before we call into the filesystem.
*/
mode &= (mask_perms & ~S_IFMT);
mode |= (type & S_IFMT);
return mode;
}
/**
* vfs_create - create new file
* @idmap: idmap of the mount the inode was found from
* @dir: inode of @dentry
* @dentry: pointer to dentry of the base directory
* @mode: mode of the new file
* @want_excl: whether the file must not yet exist
*
* Create a new file.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_create(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry, umode_t mode, bool want_excl)
{
int error;
error = may_create(idmap, dir, dentry);
if (error)
return error;
if (!dir->i_op->create)
return -EACCES; /* shouldn't it be ENOSYS? */
mode = vfs_prepare_mode(idmap, dir, mode, S_IALLUGO, S_IFREG);
error = security_inode_create(dir, dentry, mode);
if (error)
return error;
error = dir->i_op->create(idmap, dir, dentry, mode, want_excl);
if (!error)
fsnotify_create(dir, dentry);
return error;
}
EXPORT_SYMBOL(vfs_create);
int vfs_mkobj(struct dentry *dentry, umode_t mode,
int (*f)(struct dentry *, umode_t, void *),
void *arg)
{
struct inode *dir = dentry->d_parent->d_inode;
int error = may_create(&nop_mnt_idmap, dir, dentry);
if (error)
return error;
mode &= S_IALLUGO;
mode |= S_IFREG;
error = security_inode_create(dir, dentry, mode);
if (error)
return error;
error = f(dentry, mode, arg);
if (!error)
fsnotify_create(dir, dentry);
return error;
}
EXPORT_SYMBOL(vfs_mkobj);
bool may_open_dev(const struct path *path)
{
return !(path->mnt->mnt_flags & MNT_NODEV) && !(path->mnt->mnt_sb->s_iflags & SB_I_NODEV);
}
static int may_open(struct mnt_idmap *idmap, const struct path *path,
int acc_mode, int flag)
{
struct dentry *dentry = path->dentry;
struct inode *inode = dentry->d_inode;
int error;
if (!inode)
return -ENOENT;
switch (inode->i_mode & S_IFMT) {
case S_IFLNK:
return -ELOOP;
case S_IFDIR:
if (acc_mode & MAY_WRITE)
return -EISDIR;
if (acc_mode & MAY_EXEC)
return -EACCES;
break;
case S_IFBLK:
case S_IFCHR:
if (!may_open_dev(path))
return -EACCES;
fallthrough;
case S_IFIFO:
case S_IFSOCK:
if (acc_mode & MAY_EXEC)
return -EACCES;
flag &= ~O_TRUNC;
break;
case S_IFREG:
if ((acc_mode & MAY_EXEC) && path_noexec(path))
return -EACCES;
break;
}
error = inode_permission(idmap, inode, MAY_OPEN | acc_mode);
if (error)
return error;
/*
* An append-only file must be opened in append mode for writing.
*/
if (IS_APPEND(inode)) { if ((flag & O_ACCMODE) != O_RDONLY && !(flag & O_APPEND))
return -EPERM;
if (flag & O_TRUNC)
return -EPERM;
}
/* O_NOATIME can only be set by the owner or superuser */
if (flag & O_NOATIME && !inode_owner_or_capable(idmap, inode))
return -EPERM;
return 0;
}
static int handle_truncate(struct mnt_idmap *idmap, struct file *filp)
{
const struct path *path = &filp->f_path;
struct inode *inode = path->dentry->d_inode; int error = get_write_access(inode);
if (error)
return error;
error = security_file_truncate(filp);
if (!error) {
error = do_truncate(idmap, path->dentry, 0,
ATTR_MTIME|ATTR_CTIME|ATTR_OPEN,
filp);
}
put_write_access(inode);
return error;
}
static inline int open_to_namei_flags(int flag)
{
if ((flag & O_ACCMODE) == 3)
flag--;
return flag;
}
static int may_o_create(struct mnt_idmap *idmap,
const struct path *dir, struct dentry *dentry,
umode_t mode)
{
int error = security_path_mknod(dir, dentry, mode, 0);
if (error)
return error;
if (!fsuidgid_has_mapping(dir->dentry->d_sb, idmap))
return -EOVERFLOW;
error = inode_permission(idmap, dir->dentry->d_inode,
MAY_WRITE | MAY_EXEC);
if (error)
return error;
return security_inode_create(dir->dentry->d_inode, dentry, mode);
}
/*
* Attempt to atomically look up, create and open a file from a negative
* dentry.
*
* Returns 0 if successful. The file will have been created and attached to
* @file by the filesystem calling finish_open().
*
* If the file was looked up only or didn't need creating, FMODE_OPENED won't
* be set. The caller will need to perform the open themselves. @path will
* have been updated to point to the new dentry. This may be negative.
*
* Returns an error code otherwise.
*/
static struct dentry *atomic_open(struct nameidata *nd, struct dentry *dentry,
struct file *file,
int open_flag, umode_t mode)
{
struct dentry *const DENTRY_NOT_SET = (void *) -1UL;
struct inode *dir = nd->path.dentry->d_inode;
int error;
if (nd->flags & LOOKUP_DIRECTORY)
open_flag |= O_DIRECTORY; file->f_path.dentry = DENTRY_NOT_SET;
file->f_path.mnt = nd->path.mnt;
error = dir->i_op->atomic_open(dir, dentry, file, open_to_namei_flags(open_flag), mode); d_lookup_done(dentry); if (!error) { if (file->f_mode & FMODE_OPENED) { if (unlikely(dentry != file->f_path.dentry)) { dput(dentry); dentry = dget(file->f_path.dentry);
}
} else if (WARN_ON(file->f_path.dentry == DENTRY_NOT_SET)) {
error = -EIO;
} else {
if (file->f_path.dentry) { dput(dentry);
dentry = file->f_path.dentry;
}
if (unlikely(d_is_negative(dentry)))
error = -ENOENT;
}
}
if (error) {
dput(dentry);
dentry = ERR_PTR(error);
}
return dentry;
}
/*
* Look up and maybe create and open the last component.
*
* Must be called with parent locked (exclusive in O_CREAT case).
*
* Returns 0 on success, that is, if
* the file was successfully atomically created (if necessary) and opened, or
* the file was not completely opened at this time, though lookups and
* creations were performed.
* These case are distinguished by presence of FMODE_OPENED on file->f_mode.
* In the latter case dentry returned in @path might be negative if O_CREAT
* hadn't been specified.
*
* An error code is returned on failure.
*/
static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
const struct open_flags *op,
bool got_write)
{
struct mnt_idmap *idmap;
struct dentry *dir = nd->path.dentry;
struct inode *dir_inode = dir->d_inode;
int open_flag = op->open_flag;
struct dentry *dentry;
int error, create_error = 0;
umode_t mode = op->mode;
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
if (unlikely(IS_DEADDIR(dir_inode)))
return ERR_PTR(-ENOENT);
file->f_mode &= ~FMODE_CREATED;
dentry = d_lookup(dir, &nd->last);
for (;;) {
if (!dentry) { dentry = d_alloc_parallel(dir, &nd->last, &wq);
if (IS_ERR(dentry))
return dentry;
}
if (d_in_lookup(dentry))
break;
error = d_revalidate(dentry, nd->flags);
if (likely(error > 0))
break;
if (error)
goto out_dput;
d_invalidate(dentry);
dput(dentry);
dentry = NULL;
}
if (dentry->d_inode) {
/* Cached positive dentry: will open in f_op->open */
return dentry;
}
/*
* Checking write permission is tricky, bacuse we don't know if we are
* going to actually need it: O_CREAT opens should work as long as the
* file exists. But checking existence breaks atomicity. The trick is
* to check access and if not granted clear O_CREAT from the flags.
*
* Another problem is returing the "right" error value (e.g. for an
* O_EXCL open we want to return EEXIST not EROFS).
*/
if (unlikely(!got_write)) open_flag &= ~O_TRUNC; idmap = mnt_idmap(nd->path.mnt);
if (open_flag & O_CREAT) {
if (open_flag & O_EXCL) open_flag &= ~O_TRUNC; mode = vfs_prepare_mode(idmap, dir->d_inode, mode, mode, mode);
if (likely(got_write))
create_error = may_o_create(idmap, &nd->path,
dentry, mode);
else
create_error = -EROFS;
}
if (create_error)
open_flag &= ~O_CREAT; if (dir_inode->i_op->atomic_open) { dentry = atomic_open(nd, dentry, file, open_flag, mode); if (unlikely(create_error) && dentry == ERR_PTR(-ENOENT)) dentry = ERR_PTR(create_error);
return dentry;
}
if (d_in_lookup(dentry)) { struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry,
nd->flags);
d_lookup_done(dentry); if (unlikely(res)) { if (IS_ERR(res)) {
error = PTR_ERR(res);
goto out_dput;
}
dput(dentry);
dentry = res;
}
}
/* Negative dentry, just create the file */
if (!dentry->d_inode && (open_flag & O_CREAT)) { file->f_mode |= FMODE_CREATED; audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE); if (!dir_inode->i_op->create) {
error = -EACCES;
goto out_dput;
}
error = dir_inode->i_op->create(idmap, dir_inode, dentry,
mode, open_flag & O_EXCL);
if (error)
goto out_dput;
}
if (unlikely(create_error) && !dentry->d_inode) {
error = create_error;
goto out_dput;
}
return dentry;
out_dput:
dput(dentry); return ERR_PTR(error);
}
static const char *open_last_lookups(struct nameidata *nd,
struct file *file, const struct open_flags *op)
{
struct dentry *dir = nd->path.dentry;
int open_flag = op->open_flag;
bool got_write = false;
struct dentry *dentry;
const char *res;
nd->flags |= op->intent;
if (nd->last_type != LAST_NORM) {
if (nd->depth) put_link(nd); return handle_dots(nd, nd->last_type);
}
if (!(open_flag & O_CREAT)) { if (nd->last.name[nd->last.len]) nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
/* we _can_ be in RCU mode here */
dentry = lookup_fast(nd);
if (IS_ERR(dentry))
return ERR_CAST(dentry);
if (likely(dentry))
goto finish_lookup;
if (WARN_ON_ONCE(nd->flags & LOOKUP_RCU))
return ERR_PTR(-ECHILD);
} else {
/* create side of things */
if (nd->flags & LOOKUP_RCU) { if (!try_to_unlazy(nd))
return ERR_PTR(-ECHILD);
}
audit_inode(nd->name, dir, AUDIT_INODE_PARENT);
/* trailing slashes? */
if (unlikely(nd->last.name[nd->last.len]))
return ERR_PTR(-EISDIR);
}
if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) { got_write = !mnt_want_write(nd->path.mnt);
/*
* do _not_ fail yet - we might not need that or fail with
* a different error; let lookup_open() decide; we'll be
* dropping this one anyway.
*/
}
if (open_flag & O_CREAT)
inode_lock(dir->d_inode);
else
inode_lock_shared(dir->d_inode); dentry = lookup_open(nd, file, op, got_write); if (!IS_ERR(dentry) && (file->f_mode & FMODE_CREATED)) fsnotify_create(dir->d_inode, dentry);
if (open_flag & O_CREAT)
inode_unlock(dir->d_inode);
else
inode_unlock_shared(dir->d_inode);
if (got_write)
mnt_drop_write(nd->path.mnt);
if (IS_ERR(dentry))
return ERR_CAST(dentry);
if (file->f_mode & (FMODE_OPENED | FMODE_CREATED)) { dput(nd->path.dentry);
nd->path.dentry = dentry;
return NULL;
}
finish_lookup:
if (nd->depth) put_link(nd); res = step_into(nd, WALK_TRAILING, dentry);
if (unlikely(res))
nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL);
return res;
}
/*
* Handle the last step of open()
*/
static int do_open(struct nameidata *nd,
struct file *file, const struct open_flags *op)
{
struct mnt_idmap *idmap;
int open_flag = op->open_flag;
bool do_truncate;
int acc_mode;
int error;
if (!(file->f_mode & (FMODE_OPENED | FMODE_CREATED))) {
error = complete_walk(nd);
if (error)
return error;
}
if (!(file->f_mode & FMODE_CREATED)) audit_inode(nd->name, nd->path.dentry, 0); idmap = mnt_idmap(nd->path.mnt);
if (open_flag & O_CREAT) {
if ((open_flag & O_EXCL) && !(file->f_mode & FMODE_CREATED))
return -EEXIST;
if (d_is_dir(nd->path.dentry))
return -EISDIR;
error = may_create_in_sticky(idmap, nd,
d_backing_inode(nd->path.dentry));
if (unlikely(error))
return error;
}
if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry))
return -ENOTDIR;
do_truncate = false;
acc_mode = op->acc_mode;
if (file->f_mode & FMODE_CREATED) {
/* Don't check for write permission, don't truncate */
open_flag &= ~O_TRUNC;
acc_mode = 0;
} else if (d_is_reg(nd->path.dentry) && open_flag & O_TRUNC) { error = mnt_want_write(nd->path.mnt);
if (error)
return error;
do_truncate = true;
}
error = may_open(idmap, &nd->path, acc_mode, open_flag); if (!error && !(file->f_mode & FMODE_OPENED)) error = vfs_open(&nd->path, file);
if (!error)
error = security_file_post_open(file, op->acc_mode); if (!error && do_truncate) error = handle_truncate(idmap, file); if (unlikely(error > 0)) { WARN_ON(1);
error = -EINVAL;
}
if (do_truncate) mnt_drop_write(nd->path.mnt);
return error;
}
/**
* vfs_tmpfile - create tmpfile
* @idmap: idmap of the mount the inode was found from
* @parentpath: pointer to the path of the base directory
* @file: file descriptor of the new tmpfile
* @mode: mode of the new tmpfile
*
* Create a temporary file.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
static int vfs_tmpfile(struct mnt_idmap *idmap,
const struct path *parentpath,
struct file *file, umode_t mode)
{
struct dentry *child;
struct inode *dir = d_inode(parentpath->dentry);
struct inode *inode;
int error;
int open_flag = file->f_flags;
/* we want directory to be writable */
error = inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC);
if (error)
return error;
if (!dir->i_op->tmpfile)
return -EOPNOTSUPP;
child = d_alloc(parentpath->dentry, &slash_name);
if (unlikely(!child))
return -ENOMEM;
file->f_path.mnt = parentpath->mnt;
file->f_path.dentry = child;
mode = vfs_prepare_mode(idmap, dir, mode, mode, mode);
error = dir->i_op->tmpfile(idmap, dir, file, mode);
dput(child);
if (error)
return error;
/* Don't check for other permissions, the inode was just created */
error = may_open(idmap, &file->f_path, 0, file->f_flags);
if (error)
return error;
inode = file_inode(file);
if (!(open_flag & O_EXCL)) {
spin_lock(&inode->i_lock);
inode->i_state |= I_LINKABLE;
spin_unlock(&inode->i_lock);
}
security_inode_post_create_tmpfile(idmap, inode);
return 0;
}
/**
* kernel_tmpfile_open - open a tmpfile for kernel internal use
* @idmap: idmap of the mount the inode was found from
* @parentpath: path of the base directory
* @mode: mode of the new tmpfile
* @open_flag: flags
* @cred: credentials for open
*
* Create and open a temporary file. The file is not accounted in nr_files,
* hence this is only for kernel internal use, and must not be installed into
* file tables or such.
*/
struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
const struct path *parentpath,
umode_t mode, int open_flag,
const struct cred *cred)
{
struct file *file;
int error;
file = alloc_empty_file_noaccount(open_flag, cred);
if (IS_ERR(file))
return file;
error = vfs_tmpfile(idmap, parentpath, file, mode);
if (error) {
fput(file);
file = ERR_PTR(error);
}
return file;
}
EXPORT_SYMBOL(kernel_tmpfile_open);
static int do_tmpfile(struct nameidata *nd, unsigned flags,
const struct open_flags *op,
struct file *file)
{
struct path path;
int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
if (unlikely(error))
return error;
error = mnt_want_write(path.mnt);
if (unlikely(error))
goto out;
error = vfs_tmpfile(mnt_idmap(path.mnt), &path, file, op->mode);
if (error)
goto out2;
audit_inode(nd->name, file->f_path.dentry, 0);
out2:
mnt_drop_write(path.mnt);
out:
path_put(&path);
return error;
}
static int do_o_path(struct nameidata *nd, unsigned flags, struct file *file)
{
struct path path;
int error = path_lookupat(nd, flags, &path);
if (!error) {
audit_inode(nd->name, path.dentry, 0); error = vfs_open(&path, file);
path_put(&path);
}
return error;
}
static struct file *path_openat(struct nameidata *nd,
const struct open_flags *op, unsigned flags)
{
struct file *file;
int error;
file = alloc_empty_file(op->open_flag, current_cred());
if (IS_ERR(file))
return file;
if (unlikely(file->f_flags & __O_TMPFILE)) { error = do_tmpfile(nd, flags, op, file); } else if (unlikely(file->f_flags & O_PATH)) { error = do_o_path(nd, flags, file);
} else {
const char *s = path_init(nd, flags); while (!(error = link_path_walk(s, nd)) && (s = open_last_lookups(nd, file, op)) != NULL)
;
if (!error) error = do_open(nd, file, op); terminate_walk(nd);
}
if (likely(!error)) { if (likely(file->f_mode & FMODE_OPENED))
return file;
WARN_ON(1);
error = -EINVAL;
}
fput(file); if (error == -EOPENSTALE) { if (flags & LOOKUP_RCU)
error = -ECHILD;
else
error = -ESTALE;
}
return ERR_PTR(error);
}
struct file *do_filp_open(int dfd, struct filename *pathname,
const struct open_flags *op)
{
struct nameidata nd;
int flags = op->lookup_flags;
struct file *filp;
set_nameidata(&nd, dfd, pathname, NULL);
filp = path_openat(&nd, op, flags | LOOKUP_RCU);
if (unlikely(filp == ERR_PTR(-ECHILD)))
filp = path_openat(&nd, op, flags); if (unlikely(filp == ERR_PTR(-ESTALE))) filp = path_openat(&nd, op, flags | LOOKUP_REVAL); restore_nameidata(); return filp;
}
struct file *do_file_open_root(const struct path *root,
const char *name, const struct open_flags *op)
{
struct nameidata nd;
struct file *file;
struct filename *filename;
int flags = op->lookup_flags;
if (d_is_symlink(root->dentry) && op->intent & LOOKUP_OPEN)
return ERR_PTR(-ELOOP);
filename = getname_kernel(name);
if (IS_ERR(filename))
return ERR_CAST(filename);
set_nameidata(&nd, -1, filename, root);
file = path_openat(&nd, op, flags | LOOKUP_RCU);
if (unlikely(file == ERR_PTR(-ECHILD)))
file = path_openat(&nd, op, flags);
if (unlikely(file == ERR_PTR(-ESTALE)))
file = path_openat(&nd, op, flags | LOOKUP_REVAL);
restore_nameidata();
putname(filename);
return file;
}
static struct dentry *filename_create(int dfd, struct filename *name,
struct path *path, unsigned int lookup_flags)
{
struct dentry *dentry = ERR_PTR(-EEXIST);
struct qstr last;
bool want_dir = lookup_flags & LOOKUP_DIRECTORY;
unsigned int reval_flag = lookup_flags & LOOKUP_REVAL;
unsigned int create_flags = LOOKUP_CREATE | LOOKUP_EXCL;
int type;
int err2;
int error;
error = filename_parentat(dfd, name, reval_flag, path, &last, &type);
if (error)
return ERR_PTR(error);
/*
* Yucky last component or no last component at all?
* (foo/., foo/.., /////)
*/
if (unlikely(type != LAST_NORM))
goto out;
/* don't fail immediately if it's r/o, at least try to report other errors */
err2 = mnt_want_write(path->mnt);
/*
* Do the final lookup. Suppress 'create' if there is a trailing
* '/', and a directory wasn't requested.
*/
if (last.name[last.len] && !want_dir)
create_flags = 0;
inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
dentry = lookup_one_qstr_excl(&last, path->dentry,
reval_flag | create_flags);
if (IS_ERR(dentry))
goto unlock;
error = -EEXIST;
if (d_is_positive(dentry))
goto fail;
/*
* Special case - lookup gave negative, but... we had foo/bar/
* From the vfs_mknod() POV we just have a negative dentry -
* all is fine. Let's be bastards - you had / on the end, you've
* been asking for (non-existent) directory. -ENOENT for you.
*/
if (unlikely(!create_flags)) {
error = -ENOENT;
goto fail;
}
if (unlikely(err2)) {
error = err2;
goto fail;
}
return dentry;
fail:
dput(dentry);
dentry = ERR_PTR(error);
unlock:
inode_unlock(path->dentry->d_inode);
if (!err2)
mnt_drop_write(path->mnt);
out:
path_put(path);
return dentry;
}
struct dentry *kern_path_create(int dfd, const char *pathname,
struct path *path, unsigned int lookup_flags)
{
struct filename *filename = getname_kernel(pathname);
struct dentry *res = filename_create(dfd, filename, path, lookup_flags);
putname(filename);
return res;
}
EXPORT_SYMBOL(kern_path_create);
void done_path_create(struct path *path, struct dentry *dentry)
{
dput(dentry);
inode_unlock(path->dentry->d_inode);
mnt_drop_write(path->mnt);
path_put(path);
}
EXPORT_SYMBOL(done_path_create);
inline struct dentry *user_path_create(int dfd, const char __user *pathname,
struct path *path, unsigned int lookup_flags)
{
struct filename *filename = getname(pathname);
struct dentry *res = filename_create(dfd, filename, path, lookup_flags);
putname(filename);
return res;
}
EXPORT_SYMBOL(user_path_create);
/**
* vfs_mknod - create device node or file
* @idmap: idmap of the mount the inode was found from
* @dir: inode of @dentry
* @dentry: pointer to dentry of the base directory
* @mode: mode of the new device node or file
* @dev: device number of device to create
*
* Create a device node or file.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry, umode_t mode, dev_t dev)
{
bool is_whiteout = S_ISCHR(mode) && dev == WHITEOUT_DEV;
int error = may_create(idmap, dir, dentry);
if (error)
return error;
if ((S_ISCHR(mode) || S_ISBLK(mode)) && !is_whiteout &&
!capable(CAP_MKNOD))
return -EPERM;
if (!dir->i_op->mknod)
return -EPERM;
mode = vfs_prepare_mode(idmap, dir, mode, mode, mode);
error = devcgroup_inode_mknod(mode, dev);
if (error)
return error;
error = security_inode_mknod(dir, dentry, mode, dev);
if (error)
return error;
error = dir->i_op->mknod(idmap, dir, dentry, mode, dev);
if (!error)
fsnotify_create(dir, dentry);
return error;
}
EXPORT_SYMBOL(vfs_mknod);
static int may_mknod(umode_t mode)
{
switch (mode & S_IFMT) {
case S_IFREG:
case S_IFCHR:
case S_IFBLK:
case S_IFIFO:
case S_IFSOCK:
case 0: /* zero mode translates to S_IFREG */
return 0;
case S_IFDIR:
return -EPERM;
default:
return -EINVAL;
}
}
static int do_mknodat(int dfd, struct filename *name, umode_t mode,
unsigned int dev)
{
struct mnt_idmap *idmap;
struct dentry *dentry;
struct path path;
int error;
unsigned int lookup_flags = 0;
error = may_mknod(mode);
if (error)
goto out1;
retry:
dentry = filename_create(dfd, name, &path, lookup_flags);
error = PTR_ERR(dentry);
if (IS_ERR(dentry))
goto out1;
error = security_path_mknod(&path, dentry,
mode_strip_umask(path.dentry->d_inode, mode), dev);
if (error)
goto out2;
idmap = mnt_idmap(path.mnt);
switch (mode & S_IFMT) {
case 0: case S_IFREG:
error = vfs_create(idmap, path.dentry->d_inode,
dentry, mode, true);
if (!error)
security_path_post_mknod(idmap, dentry);
break;
case S_IFCHR: case S_IFBLK:
error = vfs_mknod(idmap, path.dentry->d_inode,
dentry, mode, new_decode_dev(dev));
break;
case S_IFIFO: case S_IFSOCK:
error = vfs_mknod(idmap, path.dentry->d_inode,
dentry, mode, 0);
break;
}
out2:
done_path_create(&path, dentry);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out1:
putname(name);
return error;
}
SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode,
unsigned int, dev)
{
return do_mknodat(dfd, getname(filename), mode, dev);
}
SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev)
{
return do_mknodat(AT_FDCWD, getname(filename), mode, dev);
}
/**
* vfs_mkdir - create directory
* @idmap: idmap of the mount the inode was found from
* @dir: inode of @dentry
* @dentry: pointer to dentry of the base directory
* @mode: mode of the new directory
*
* Create a directory.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry, umode_t mode)
{
int error;
unsigned max_links = dir->i_sb->s_max_links;
error = may_create(idmap, dir, dentry);
if (error)
return error;
if (!dir->i_op->mkdir)
return -EPERM;
mode = vfs_prepare_mode(idmap, dir, mode, S_IRWXUGO | S_ISVTX, 0);
error = security_inode_mkdir(dir, dentry, mode);
if (error)
return error;
if (max_links && dir->i_nlink >= max_links)
return -EMLINK;
error = dir->i_op->mkdir(idmap, dir, dentry, mode);
if (!error)
fsnotify_mkdir(dir, dentry);
return error;
}
EXPORT_SYMBOL(vfs_mkdir);
int do_mkdirat(int dfd, struct filename *name, umode_t mode)
{
struct dentry *dentry;
struct path path;
int error;
unsigned int lookup_flags = LOOKUP_DIRECTORY;
retry:
dentry = filename_create(dfd, name, &path, lookup_flags);
error = PTR_ERR(dentry);
if (IS_ERR(dentry))
goto out_putname;
error = security_path_mkdir(&path, dentry,
mode_strip_umask(path.dentry->d_inode, mode));
if (!error) {
error = vfs_mkdir(mnt_idmap(path.mnt), path.dentry->d_inode,
dentry, mode);
}
done_path_create(&path, dentry);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out_putname:
putname(name);
return error;
}
SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode)
{
return do_mkdirat(dfd, getname(pathname), mode);
}
SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode)
{
return do_mkdirat(AT_FDCWD, getname(pathname), mode);
}
/**
* vfs_rmdir - remove directory
* @idmap: idmap of the mount the inode was found from
* @dir: inode of @dentry
* @dentry: pointer to dentry of the base directory
*
* Remove a directory.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_rmdir(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry)
{
int error = may_delete(idmap, dir, dentry, 1);
if (error)
return error;
if (!dir->i_op->rmdir)
return -EPERM;
dget(dentry);
inode_lock(dentry->d_inode);
error = -EBUSY;
if (is_local_mountpoint(dentry) ||
(dentry->d_inode->i_flags & S_KERNEL_FILE))
goto out;
error = security_inode_rmdir(dir, dentry);
if (error)
goto out;
error = dir->i_op->rmdir(dir, dentry);
if (error)
goto out;
shrink_dcache_parent(dentry);
dentry->d_inode->i_flags |= S_DEAD;
dont_mount(dentry);
detach_mounts(dentry);
out:
inode_unlock(dentry->d_inode);
dput(dentry);
if (!error)
d_delete_notify(dir, dentry);
return error;
}
EXPORT_SYMBOL(vfs_rmdir);
int do_rmdir(int dfd, struct filename *name)
{
int error;
struct dentry *dentry;
struct path path;
struct qstr last;
int type;
unsigned int lookup_flags = 0;
retry:
error = filename_parentat(dfd, name, lookup_flags, &path, &last, &type);
if (error)
goto exit1;
switch (type) {
case LAST_DOTDOT:
error = -ENOTEMPTY;
goto exit2;
case LAST_DOT:
error = -EINVAL;
goto exit2;
case LAST_ROOT:
error = -EBUSY;
goto exit2;
}
error = mnt_want_write(path.mnt);
if (error)
goto exit2;
inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
dentry = lookup_one_qstr_excl(&last, path.dentry, lookup_flags);
error = PTR_ERR(dentry);
if (IS_ERR(dentry))
goto exit3;
if (!dentry->d_inode) {
error = -ENOENT;
goto exit4;
}
error = security_path_rmdir(&path, dentry);
if (error)
goto exit4;
error = vfs_rmdir(mnt_idmap(path.mnt), path.dentry->d_inode, dentry);
exit4:
dput(dentry);
exit3:
inode_unlock(path.dentry->d_inode);
mnt_drop_write(path.mnt);
exit2:
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
exit1:
putname(name);
return error;
}
SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
{
return do_rmdir(AT_FDCWD, getname(pathname));
}
/**
* vfs_unlink - unlink a filesystem object
* @idmap: idmap of the mount the inode was found from
* @dir: parent directory
* @dentry: victim
* @delegated_inode: returns victim inode, if the inode is delegated.
*
* The caller must hold dir->i_mutex.
*
* If vfs_unlink discovers a delegation, it will return -EWOULDBLOCK and
* return a reference to the inode in delegated_inode. The caller
* should then break the delegation on that inode and retry. Because
* breaking a delegation may take a long time, the caller should drop
* dir->i_mutex before doing so.
*
* Alternatively, a caller may pass NULL for delegated_inode. This may
* be appropriate for callers that expect the underlying filesystem not
* to be NFS exported.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_unlink(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry, struct inode **delegated_inode)
{
struct inode *target = dentry->d_inode;
int error = may_delete(idmap, dir, dentry, 0);
if (error)
return error;
if (!dir->i_op->unlink)
return -EPERM;
inode_lock(target);
if (IS_SWAPFILE(target))
error = -EPERM;
else if (is_local_mountpoint(dentry))
error = -EBUSY;
else {
error = security_inode_unlink(dir, dentry);
if (!error) {
error = try_break_deleg(target, delegated_inode);
if (error)
goto out;
error = dir->i_op->unlink(dir, dentry);
if (!error) {
dont_mount(dentry);
detach_mounts(dentry);
}
}
}
out:
inode_unlock(target);
/* We don't d_delete() NFS sillyrenamed files--they still exist. */
if (!error && dentry->d_flags & DCACHE_NFSFS_RENAMED) {
fsnotify_unlink(dir, dentry);
} else if (!error) {
fsnotify_link_count(target);
d_delete_notify(dir, dentry);
}
return error;
}
EXPORT_SYMBOL(vfs_unlink);
/*
* Make sure that the actual truncation of the file will occur outside its
* directory's i_mutex. Truncate can take a long time if there is a lot of
* writeout happening, and we don't want to prevent access to the directory
* while waiting on the I/O.
*/
int do_unlinkat(int dfd, struct filename *name)
{
int error;
struct dentry *dentry;
struct path path;
struct qstr last;
int type;
struct inode *inode = NULL;
struct inode *delegated_inode = NULL;
unsigned int lookup_flags = 0;
retry:
error = filename_parentat(dfd, name, lookup_flags, &path, &last, &type);
if (error)
goto exit1;
error = -EISDIR;
if (type != LAST_NORM)
goto exit2;
error = mnt_want_write(path.mnt);
if (error)
goto exit2;
retry_deleg:
inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
dentry = lookup_one_qstr_excl(&last, path.dentry, lookup_flags);
error = PTR_ERR(dentry);
if (!IS_ERR(dentry)) {
/* Why not before? Because we want correct error value */
if (last.name[last.len] || d_is_negative(dentry))
goto slashes;
inode = dentry->d_inode;
ihold(inode);
error = security_path_unlink(&path, dentry);
if (error)
goto exit3;
error = vfs_unlink(mnt_idmap(path.mnt), path.dentry->d_inode,
dentry, &delegated_inode);
exit3:
dput(dentry);
}
inode_unlock(path.dentry->d_inode);
if (inode)
iput(inode); /* truncate the inode here */
inode = NULL;
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
mnt_drop_write(path.mnt);
exit2:
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
inode = NULL;
goto retry;
}
exit1:
putname(name);
return error;
slashes:
if (d_is_negative(dentry))
error = -ENOENT;
else if (d_is_dir(dentry))
error = -EISDIR;
else
error = -ENOTDIR;
goto exit3;
}
SYSCALL_DEFINE3(unlinkat, int, dfd, const char __user *, pathname, int, flag)
{
if ((flag & ~AT_REMOVEDIR) != 0)
return -EINVAL;
if (flag & AT_REMOVEDIR)
return do_rmdir(dfd, getname(pathname));
return do_unlinkat(dfd, getname(pathname));
}
SYSCALL_DEFINE1(unlink, const char __user *, pathname)
{
return do_unlinkat(AT_FDCWD, getname(pathname));
}
/**
* vfs_symlink - create symlink
* @idmap: idmap of the mount the inode was found from
* @dir: inode of @dentry
* @dentry: pointer to dentry of the base directory
* @oldname: name of the file to link to
*
* Create a symlink.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
struct dentry *dentry, const char *oldname)
{
int error;
error = may_create(idmap, dir, dentry);
if (error)
return error;
if (!dir->i_op->symlink)
return -EPERM;
error = security_inode_symlink(dir, dentry, oldname);
if (error)
return error;
error = dir->i_op->symlink(idmap, dir, dentry, oldname);
if (!error)
fsnotify_create(dir, dentry);
return error;
}
EXPORT_SYMBOL(vfs_symlink);
int do_symlinkat(struct filename *from, int newdfd, struct filename *to)
{
int error;
struct dentry *dentry;
struct path path;
unsigned int lookup_flags = 0;
if (IS_ERR(from)) {
error = PTR_ERR(from);
goto out_putnames;
}
retry:
dentry = filename_create(newdfd, to, &path, lookup_flags);
error = PTR_ERR(dentry);
if (IS_ERR(dentry))
goto out_putnames;
error = security_path_symlink(&path, dentry, from->name);
if (!error)
error = vfs_symlink(mnt_idmap(path.mnt), path.dentry->d_inode,
dentry, from->name);
done_path_create(&path, dentry);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
out_putnames:
putname(to);
putname(from);
return error;
}
SYSCALL_DEFINE3(symlinkat, const char __user *, oldname,
int, newdfd, const char __user *, newname)
{
return do_symlinkat(getname(oldname), newdfd, getname(newname));
}
SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newname)
{
return do_symlinkat(getname(oldname), AT_FDCWD, getname(newname));
}
/**
* vfs_link - create a new link
* @old_dentry: object to be linked
* @idmap: idmap of the mount
* @dir: new parent
* @new_dentry: where to create the new link
* @delegated_inode: returns inode needing a delegation break
*
* The caller must hold dir->i_mutex
*
* If vfs_link discovers a delegation on the to-be-linked file in need
* of breaking, it will return -EWOULDBLOCK and return a reference to the
* inode in delegated_inode. The caller should then break the delegation
* and retry. Because breaking a delegation may take a long time, the
* caller should drop the i_mutex before doing so.
*
* Alternatively, a caller may pass NULL for delegated_inode. This may
* be appropriate for callers that expect the underlying filesystem not
* to be NFS exported.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
int vfs_link(struct dentry *old_dentry, struct mnt_idmap *idmap,
struct inode *dir, struct dentry *new_dentry,
struct inode **delegated_inode)
{
struct inode *inode = old_dentry->d_inode;
unsigned max_links = dir->i_sb->s_max_links;
int error;
if (!inode)
return -ENOENT;
error = may_create(idmap, dir, new_dentry);
if (error)
return error;
if (dir->i_sb != inode->i_sb)
return -EXDEV;
/*
* A link to an append-only or immutable file cannot be created.
*/
if (IS_APPEND(inode) || IS_IMMUTABLE(inode))
return -EPERM;
/*
* Updating the link count will likely cause i_uid and i_gid to
* be writen back improperly if their true value is unknown to
* the vfs.
*/
if (HAS_UNMAPPED_ID(idmap, inode))
return -EPERM;
if (!dir->i_op->link)
return -EPERM;
if (S_ISDIR(inode->i_mode))
return -EPERM;
error = security_inode_link(old_dentry, dir, new_dentry);
if (error)
return error;
inode_lock(inode);
/* Make sure we don't allow creating hardlink to an unlinked file */
if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE))
error = -ENOENT;
else if (max_links && inode->i_nlink >= max_links)
error = -EMLINK;
else {
error = try_break_deleg(inode, delegated_inode);
if (!error)
error = dir->i_op->link(old_dentry, dir, new_dentry);
}
if (!error && (inode->i_state & I_LINKABLE)) {
spin_lock(&inode->i_lock);
inode->i_state &= ~I_LINKABLE;
spin_unlock(&inode->i_lock);
}
inode_unlock(inode);
if (!error)
fsnotify_link(dir, inode, new_dentry);
return error;
}
EXPORT_SYMBOL(vfs_link);
/*
* Hardlinks are often used in delicate situations. We avoid
* security-related surprises by not following symlinks on the
* newname. --KAB
*
* We don't follow them on the oldname either to be compatible
* with linux 2.0, and to avoid hard-linking to directories
* and other special files. --ADM
*/
int do_linkat(int olddfd, struct filename *old, int newdfd,
struct filename *new, int flags)
{
struct mnt_idmap *idmap;
struct dentry *new_dentry;
struct path old_path, new_path;
struct inode *delegated_inode = NULL;
int how = 0;
int error;
if ((flags & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH)) != 0) {
error = -EINVAL;
goto out_putnames;
}
/*
* To use null names we require CAP_DAC_READ_SEARCH
* This ensures that not everyone will be able to create
* handlink using the passed filedescriptor.
*/
if (flags & AT_EMPTY_PATH && !capable(CAP_DAC_READ_SEARCH)) {
error = -ENOENT;
goto out_putnames;
}
if (flags & AT_SYMLINK_FOLLOW)
how |= LOOKUP_FOLLOW;
retry:
error = filename_lookup(olddfd, old, how, &old_path, NULL);
if (error)
goto out_putnames;
new_dentry = filename_create(newdfd, new, &new_path,
(how & LOOKUP_REVAL));
error = PTR_ERR(new_dentry);
if (IS_ERR(new_dentry))
goto out_putpath;
error = -EXDEV;
if (old_path.mnt != new_path.mnt)
goto out_dput;
idmap = mnt_idmap(new_path.mnt);
error = may_linkat(idmap, &old_path);
if (unlikely(error))
goto out_dput;
error = security_path_link(old_path.dentry, &new_path, new_dentry);
if (error)
goto out_dput;
error = vfs_link(old_path.dentry, idmap, new_path.dentry->d_inode,
new_dentry, &delegated_inode);
out_dput:
done_path_create(&new_path, new_dentry);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error) {
path_put(&old_path);
goto retry;
}
}
if (retry_estale(error, how)) {
path_put(&old_path);
how |= LOOKUP_REVAL;
goto retry;
}
out_putpath:
path_put(&old_path);
out_putnames:
putname(old);
putname(new);
return error;
}
SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname,
int, newdfd, const char __user *, newname, int, flags)
{
return do_linkat(olddfd, getname_uflags(oldname, flags),
newdfd, getname(newname), flags);
}
SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname)
{
return do_linkat(AT_FDCWD, getname(oldname), AT_FDCWD, getname(newname), 0);
}
/**
* vfs_rename - rename a filesystem object
* @rd: pointer to &struct renamedata info
*
* The caller must hold multiple mutexes--see lock_rename()).
*
* If vfs_rename discovers a delegation in need of breaking at either
* the source or destination, it will return -EWOULDBLOCK and return a
* reference to the inode in delegated_inode. The caller should then
* break the delegation and retry. Because breaking a delegation may
* take a long time, the caller should drop all locks before doing
* so.
*
* Alternatively, a caller may pass NULL for delegated_inode. This may
* be appropriate for callers that expect the underlying filesystem not
* to be NFS exported.
*
* The worst of all namespace operations - renaming directory. "Perverted"
* doesn't even start to describe it. Somebody in UCB had a heck of a trip...
* Problems:
*
* a) we can get into loop creation.
* b) race potential - two innocent renames can create a loop together.
* That's where 4.4BSD screws up. Current fix: serialization on
* sb->s_vfs_rename_mutex. We might be more accurate, but that's another
* story.
* c) we may have to lock up to _four_ objects - parents and victim (if it exists),
* and source (if it's a non-directory or a subdirectory that moves to
* different parent).
* And that - after we got ->i_mutex on parents (until then we don't know
* whether the target exists). Solution: try to be smart with locking
* order for inodes. We rely on the fact that tree topology may change
* only under ->s_vfs_rename_mutex _and_ that parent of the object we
* move will be locked. Thus we can rank directories by the tree
* (ancestors first) and rank all non-directories after them.
* That works since everybody except rename does "lock parent, lookup,
* lock child" and rename is under ->s_vfs_rename_mutex.
* HOWEVER, it relies on the assumption that any object with ->lookup()
* has no more than 1 dentry. If "hybrid" objects will ever appear,
* we'd better make sure that there's no link(2) for them.
* d) conversion from fhandle to dentry may come in the wrong moment - when
* we are removing the target. Solution: we will have to grab ->i_mutex
* in the fhandle_to_dentry code. [FIXME - current nfsfh.c relies on
* ->i_mutex on parents, which works but leads to some truly excessive
* locking].
*/
int vfs_rename(struct renamedata *rd)
{
int error;
struct inode *old_dir = rd->old_dir, *new_dir = rd->new_dir;
struct dentry *old_dentry = rd->old_dentry;
struct dentry *new_dentry = rd->new_dentry;
struct inode **delegated_inode = rd->delegated_inode;
unsigned int flags = rd->flags;
bool is_dir = d_is_dir(old_dentry);
struct inode *source = old_dentry->d_inode;
struct inode *target = new_dentry->d_inode;
bool new_is_dir = false;
unsigned max_links = new_dir->i_sb->s_max_links;
struct name_snapshot old_name;
bool lock_old_subdir, lock_new_subdir;
if (source == target)
return 0;
error = may_delete(rd->old_mnt_idmap, old_dir, old_dentry, is_dir);
if (error)
return error;
if (!target) {
error = may_create(rd->new_mnt_idmap, new_dir, new_dentry);
} else {
new_is_dir = d_is_dir(new_dentry);
if (!(flags & RENAME_EXCHANGE))
error = may_delete(rd->new_mnt_idmap, new_dir,
new_dentry, is_dir);
else
error = may_delete(rd->new_mnt_idmap, new_dir,
new_dentry, new_is_dir);
}
if (error)
return error;
if (!old_dir->i_op->rename)
return -EPERM;
/*
* If we are going to change the parent - check write permissions,
* we'll need to flip '..'.
*/
if (new_dir != old_dir) {
if (is_dir) {
error = inode_permission(rd->old_mnt_idmap, source,
MAY_WRITE);
if (error)
return error;
}
if ((flags & RENAME_EXCHANGE) && new_is_dir) {
error = inode_permission(rd->new_mnt_idmap, target,
MAY_WRITE);
if (error)
return error;
}
}
error = security_inode_rename(old_dir, old_dentry, new_dir, new_dentry,
flags);
if (error)
return error;
take_dentry_name_snapshot(&old_name, old_dentry);
dget(new_dentry);
/*
* Lock children.
* The source subdirectory needs to be locked on cross-directory
* rename or cross-directory exchange since its parent changes.
* The target subdirectory needs to be locked on cross-directory
* exchange due to parent change and on any rename due to becoming
* a victim.
* Non-directories need locking in all cases (for NFS reasons);
* they get locked after any subdirectories (in inode address order).
*
* NOTE: WE ONLY LOCK UNRELATED DIRECTORIES IN CROSS-DIRECTORY CASE.
* NEVER, EVER DO THAT WITHOUT ->s_vfs_rename_mutex.
*/
lock_old_subdir = new_dir != old_dir;
lock_new_subdir = new_dir != old_dir || !(flags & RENAME_EXCHANGE);
if (is_dir) {
if (lock_old_subdir)
inode_lock_nested(source, I_MUTEX_CHILD);
if (target && (!new_is_dir || lock_new_subdir))
inode_lock(target);
} else if (new_is_dir) {
if (lock_new_subdir)
inode_lock_nested(target, I_MUTEX_CHILD);
inode_lock(source);
} else {
lock_two_nondirectories(source, target);
}
error = -EPERM;
if (IS_SWAPFILE(source) || (target && IS_SWAPFILE(target)))
goto out;
error = -EBUSY;
if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry))
goto out;
if (max_links && new_dir != old_dir) {
error = -EMLINK;
if (is_dir && !new_is_dir && new_dir->i_nlink >= max_links)
goto out;
if ((flags & RENAME_EXCHANGE) && !is_dir && new_is_dir &&
old_dir->i_nlink >= max_links)
goto out;
}
if (!is_dir) {
error = try_break_deleg(source, delegated_inode);
if (error)
goto out;
}
if (target && !new_is_dir) {
error = try_break_deleg(target, delegated_inode);
if (error)
goto out;
}
error = old_dir->i_op->rename(rd->new_mnt_idmap, old_dir, old_dentry,
new_dir, new_dentry, flags);
if (error)
goto out;
if (!(flags & RENAME_EXCHANGE) && target) {
if (is_dir) {
shrink_dcache_parent(new_dentry);
target->i_flags |= S_DEAD;
}
dont_mount(new_dentry);
detach_mounts(new_dentry);
}
if (!(old_dir->i_sb->s_type->fs_flags & FS_RENAME_DOES_D_MOVE)) {
if (!(flags & RENAME_EXCHANGE))
d_move(old_dentry, new_dentry);
else
d_exchange(old_dentry, new_dentry);
}
out:
if (!is_dir || lock_old_subdir)
inode_unlock(source);
if (target && (!new_is_dir || lock_new_subdir))
inode_unlock(target);
dput(new_dentry);
if (!error) {
fsnotify_move(old_dir, new_dir, &old_name.name, is_dir,
!(flags & RENAME_EXCHANGE) ? target : NULL, old_dentry);
if (flags & RENAME_EXCHANGE) {
fsnotify_move(new_dir, old_dir, &old_dentry->d_name,
new_is_dir, NULL, new_dentry);
}
}
release_dentry_name_snapshot(&old_name);
return error;
}
EXPORT_SYMBOL(vfs_rename);
int do_renameat2(int olddfd, struct filename *from, int newdfd,
struct filename *to, unsigned int flags)
{
struct renamedata rd;
struct dentry *old_dentry, *new_dentry;
struct dentry *trap;
struct path old_path, new_path;
struct qstr old_last, new_last;
int old_type, new_type;
struct inode *delegated_inode = NULL;
unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET;
bool should_retry = false;
int error = -EINVAL;
if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
goto put_names;
if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) &&
(flags & RENAME_EXCHANGE))
goto put_names;
if (flags & RENAME_EXCHANGE)
target_flags = 0;
retry:
error = filename_parentat(olddfd, from, lookup_flags, &old_path,
&old_last, &old_type);
if (error)
goto put_names;
error = filename_parentat(newdfd, to, lookup_flags, &new_path, &new_last,
&new_type);
if (error)
goto exit1;
error = -EXDEV;
if (old_path.mnt != new_path.mnt)
goto exit2;
error = -EBUSY;
if (old_type != LAST_NORM)
goto exit2;
if (flags & RENAME_NOREPLACE)
error = -EEXIST;
if (new_type != LAST_NORM)
goto exit2;
error = mnt_want_write(old_path.mnt);
if (error)
goto exit2;
retry_deleg:
trap = lock_rename(new_path.dentry, old_path.dentry);
if (IS_ERR(trap)) {
error = PTR_ERR(trap);
goto exit_lock_rename;
}
old_dentry = lookup_one_qstr_excl(&old_last, old_path.dentry,
lookup_flags);
error = PTR_ERR(old_dentry);
if (IS_ERR(old_dentry))
goto exit3;
/* source must exist */
error = -ENOENT;
if (d_is_negative(old_dentry))
goto exit4;
new_dentry = lookup_one_qstr_excl(&new_last, new_path.dentry,
lookup_flags | target_flags);
error = PTR_ERR(new_dentry);
if (IS_ERR(new_dentry))
goto exit4;
error = -EEXIST;
if ((flags & RENAME_NOREPLACE) && d_is_positive(new_dentry))
goto exit5;
if (flags & RENAME_EXCHANGE) {
error = -ENOENT;
if (d_is_negative(new_dentry))
goto exit5;
if (!d_is_dir(new_dentry)) {
error = -ENOTDIR;
if (new_last.name[new_last.len])
goto exit5;
}
}
/* unless the source is a directory trailing slashes give -ENOTDIR */
if (!d_is_dir(old_dentry)) {
error = -ENOTDIR;
if (old_last.name[old_last.len])
goto exit5;
if (!(flags & RENAME_EXCHANGE) && new_last.name[new_last.len])
goto exit5;
}
/* source should not be ancestor of target */
error = -EINVAL;
if (old_dentry == trap)
goto exit5;
/* target should not be an ancestor of source */
if (!(flags & RENAME_EXCHANGE))
error = -ENOTEMPTY;
if (new_dentry == trap)
goto exit5;
error = security_path_rename(&old_path, old_dentry,
&new_path, new_dentry, flags);
if (error)
goto exit5;
rd.old_dir = old_path.dentry->d_inode;
rd.old_dentry = old_dentry;
rd.old_mnt_idmap = mnt_idmap(old_path.mnt);
rd.new_dir = new_path.dentry->d_inode;
rd.new_dentry = new_dentry;
rd.new_mnt_idmap = mnt_idmap(new_path.mnt);
rd.delegated_inode = &delegated_inode;
rd.flags = flags;
error = vfs_rename(&rd);
exit5:
dput(new_dentry);
exit4:
dput(old_dentry);
exit3:
unlock_rename(new_path.dentry, old_path.dentry);
exit_lock_rename:
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
mnt_drop_write(old_path.mnt);
exit2:
if (retry_estale(error, lookup_flags))
should_retry = true;
path_put(&new_path);
exit1:
path_put(&old_path);
if (should_retry) {
should_retry = false;
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
put_names:
putname(from);
putname(to);
return error;
}
SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname,
int, newdfd, const char __user *, newname, unsigned int, flags)
{
return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname),
flags);
}
SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname,
int, newdfd, const char __user *, newname)
{
return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname),
0);
}
SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname)
{
return do_renameat2(AT_FDCWD, getname(oldname), AT_FDCWD,
getname(newname), 0);
}
int readlink_copy(char __user *buffer, int buflen, const char *link)
{
int len = PTR_ERR(link);
if (IS_ERR(link))
goto out;
len = strlen(link);
if (len > (unsigned) buflen)
len = buflen;
if (copy_to_user(buffer, link, len))
len = -EFAULT;
out:
return len;
}
/**
* vfs_readlink - copy symlink body into userspace buffer
* @dentry: dentry on which to get symbolic link
* @buffer: user memory pointer
* @buflen: size of buffer
*
* Does not touch atime. That's up to the caller if necessary
*
* Does not call security hook.
*/
int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
{
struct inode *inode = d_inode(dentry);
DEFINE_DELAYED_CALL(done);
const char *link;
int res;
if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
if (unlikely(inode->i_op->readlink))
return inode->i_op->readlink(dentry, buffer, buflen);
if (!d_is_symlink(dentry))
return -EINVAL;
spin_lock(&inode->i_lock);
inode->i_opflags |= IOP_DEFAULT_READLINK;
spin_unlock(&inode->i_lock);
}
link = READ_ONCE(inode->i_link);
if (!link) {
link = inode->i_op->get_link(dentry, inode, &done);
if (IS_ERR(link))
return PTR_ERR(link);
}
res = readlink_copy(buffer, buflen, link);
do_delayed_call(&done);
return res;
}
EXPORT_SYMBOL(vfs_readlink);
/**
* vfs_get_link - get symlink body
* @dentry: dentry on which to get symbolic link
* @done: caller needs to free returned data with this
*
* Calls security hook and i_op->get_link() on the supplied inode.
*
* It does not touch atime. That's up to the caller if necessary.
*
* Does not work on "special" symlinks like /proc/$$/fd/N
*/
const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done)
{
const char *res = ERR_PTR(-EINVAL);
struct inode *inode = d_inode(dentry);
if (d_is_symlink(dentry)) {
res = ERR_PTR(security_inode_readlink(dentry));
if (!res)
res = inode->i_op->get_link(dentry, inode, done);
}
return res;
}
EXPORT_SYMBOL(vfs_get_link);
/* get the link contents into pagecache */
const char *page_get_link(struct dentry *dentry, struct inode *inode,
struct delayed_call *callback)
{
char *kaddr;
struct page *page;
struct address_space *mapping = inode->i_mapping;
if (!dentry) {
page = find_get_page(mapping, 0);
if (!page)
return ERR_PTR(-ECHILD);
if (!PageUptodate(page)) {
put_page(page);
return ERR_PTR(-ECHILD);
}
} else {
page = read_mapping_page(mapping, 0, NULL);
if (IS_ERR(page))
return (char*)page;
}
set_delayed_call(callback, page_put_link, page);
BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM);
kaddr = page_address(page);
nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1);
return kaddr;
}
EXPORT_SYMBOL(page_get_link);
void page_put_link(void *arg)
{
put_page(arg);
}
EXPORT_SYMBOL(page_put_link);
int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
{
DEFINE_DELAYED_CALL(done);
int res = readlink_copy(buffer, buflen,
page_get_link(dentry, d_inode(dentry),
&done));
do_delayed_call(&done);
return res;
}
EXPORT_SYMBOL(page_readlink);
int page_symlink(struct inode *inode, const char *symname, int len)
{
struct address_space *mapping = inode->i_mapping;
const struct address_space_operations *aops = mapping->a_ops;
bool nofs = !mapping_gfp_constraint(mapping, __GFP_FS);
struct page *page;
void *fsdata = NULL;
int err;
unsigned int flags;
retry:
if (nofs)
flags = memalloc_nofs_save();
err = aops->write_begin(NULL, mapping, 0, len-1, &page, &fsdata);
if (nofs)
memalloc_nofs_restore(flags);
if (err)
goto fail;
memcpy(page_address(page), symname, len-1);
err = aops->write_end(NULL, mapping, 0, len-1, len-1,
page, fsdata);
if (err < 0)
goto fail;
if (err < len-1)
goto retry;
mark_inode_dirty(inode);
return 0;
fail:
return err;
}
EXPORT_SYMBOL(page_symlink);
const struct inode_operations page_symlink_inode_operations = {
.get_link = page_get_link,
};
EXPORT_SYMBOL(page_symlink_inode_operations);
/*
* Linux Security Module interfaces
*
* Copyright (C) 2001 WireX Communications, Inc <chris@wirex.com>
* Copyright (C) 2001 Greg Kroah-Hartman <greg@kroah.com>
* Copyright (C) 2001 Networks Associates Technology, Inc <ssmalley@nai.com>
* Copyright (C) 2001 James Morris <jmorris@intercode.com.au>
* Copyright (C) 2001 Silicon Graphics, Inc. (Trust Technology Group)
* Copyright (C) 2015 Intel Corporation.
* Copyright (C) 2015 Casey Schaufler <casey@schaufler-ca.com>
* Copyright (C) 2016 Mellanox Techonologies
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* Due to this file being licensed under the GPL there is controversy over
* whether this permits you to write a module that #includes this file
* without placing your module under the GPL. Please consult a lawyer for
* advice before doing this.
*
*/
#ifndef __LINUX_LSM_HOOKS_H
#define __LINUX_LSM_HOOKS_H
#include <uapi/linux/lsm.h>
#include <linux/security.h>
#include <linux/init.h>
#include <linux/rculist.h>
#include <linux/xattr.h>
union security_list_options {
#define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__);
#include "lsm_hook_defs.h"
#undef LSM_HOOK
};
struct security_hook_heads {
#define LSM_HOOK(RET, DEFAULT, NAME, ...) struct hlist_head NAME;
#include "lsm_hook_defs.h"
#undef LSM_HOOK
} __randomize_layout;
/**
* struct lsm_id - Identify a Linux Security Module.
* @lsm: name of the LSM, must be approved by the LSM maintainers
* @id: LSM ID number from uapi/linux/lsm.h
*
* Contains the information that identifies the LSM.
*/
struct lsm_id {
const char *name;
u64 id;
};
/*
* Security module hook list structure.
* For use with generic list macros for common operations.
*/
struct security_hook_list {
struct hlist_node list;
struct hlist_head *head;
union security_list_options hook;
const struct lsm_id *lsmid;
} __randomize_layout;
/*
* Security blob size or offset data.
*/
struct lsm_blob_sizes {
int lbs_cred;
int lbs_file;
int lbs_inode;
int lbs_superblock;
int lbs_ipc;
int lbs_msg_msg;
int lbs_task;
int lbs_xattr_count; /* number of xattr slots in new_xattrs array */
};
/**
* lsm_get_xattr_slot - Return the next available slot and increment the index
* @xattrs: array storing LSM-provided xattrs
* @xattr_count: number of already stored xattrs (updated)
*
* Retrieve the first available slot in the @xattrs array to fill with an xattr,
* and increment @xattr_count.
*
* Return: The slot to fill in @xattrs if non-NULL, NULL otherwise.
*/
static inline struct xattr *lsm_get_xattr_slot(struct xattr *xattrs,
int *xattr_count)
{
if (unlikely(!xattrs))
return NULL;
return &xattrs[(*xattr_count)++];
}
/*
* LSM_RET_VOID is used as the default value in LSM_HOOK definitions for void
* LSM hooks (in include/linux/lsm_hook_defs.h).
*/
#define LSM_RET_VOID ((void) 0)
/*
* Initializing a security_hook_list structure takes
* up a lot of space in a source file. This macro takes
* care of the common case and reduces the amount of
* text involved.
*/
#define LSM_HOOK_INIT(HEAD, HOOK) \
{ .head = &security_hook_heads.HEAD, .hook = { .HEAD = HOOK } }
extern struct security_hook_heads security_hook_heads;
extern char *lsm_names;
extern void security_add_hooks(struct security_hook_list *hooks, int count,
const struct lsm_id *lsmid);
#define LSM_FLAG_LEGACY_MAJOR BIT(0)
#define LSM_FLAG_EXCLUSIVE BIT(1)
enum lsm_order {
LSM_ORDER_FIRST = -1, /* This is only for capabilities. */
LSM_ORDER_MUTABLE = 0,
LSM_ORDER_LAST = 1, /* This is only for integrity. */
};
struct lsm_info {
const char *name; /* Required. */
enum lsm_order order; /* Optional: default is LSM_ORDER_MUTABLE */
unsigned long flags; /* Optional: flags describing LSM */
int *enabled; /* Optional: controlled by CONFIG_LSM */
int (*init)(void); /* Required. */
struct lsm_blob_sizes *blobs; /* Optional: for blob sharing. */
};
extern struct lsm_info __start_lsm_info[], __end_lsm_info[];
extern struct lsm_info __start_early_lsm_info[], __end_early_lsm_info[];
#define DEFINE_LSM(lsm) \
static struct lsm_info __lsm_##lsm \
__used __section(".lsm_info.init") \
__aligned(sizeof(unsigned long))
#define DEFINE_EARLY_LSM(lsm) \
static struct lsm_info __early_lsm_##lsm \
__used __section(".early_lsm_info.init") \
__aligned(sizeof(unsigned long))
extern int lsm_inode_alloc(struct inode *inode);
#endif /* ! __LINUX_LSM_HOOKS_H */
/* SPDX-License-Identifier: GPL-2.0 */
/* linux/include/linux/clockchips.h
*
* This file contains the structure definitions for clockchips.
*
* If you are not a clockchip, or the time of day code, you should
* not be including this file!
*/
#ifndef _LINUX_CLOCKCHIPS_H
#define _LINUX_CLOCKCHIPS_H
#ifdef CONFIG_GENERIC_CLOCKEVENTS
# include <linux/clocksource.h>
# include <linux/cpumask.h>
# include <linux/ktime.h>
# include <linux/notifier.h>
struct clock_event_device;
struct module;
/*
* Possible states of a clock event device.
*
* DETACHED: Device is not used by clockevents core. Initial state or can be
* reached from SHUTDOWN.
* SHUTDOWN: Device is powered-off. Can be reached from PERIODIC or ONESHOT.
* PERIODIC: Device is programmed to generate events periodically. Can be
* reached from DETACHED or SHUTDOWN.
* ONESHOT: Device is programmed to generate event only once. Can be reached
* from DETACHED or SHUTDOWN.
* ONESHOT_STOPPED: Device was programmed in ONESHOT mode and is temporarily
* stopped.
*/
enum clock_event_state {
CLOCK_EVT_STATE_DETACHED,
CLOCK_EVT_STATE_SHUTDOWN,
CLOCK_EVT_STATE_PERIODIC,
CLOCK_EVT_STATE_ONESHOT,
CLOCK_EVT_STATE_ONESHOT_STOPPED,
};
/*
* Clock event features
*/
# define CLOCK_EVT_FEAT_PERIODIC 0x000001
# define CLOCK_EVT_FEAT_ONESHOT 0x000002
# define CLOCK_EVT_FEAT_KTIME 0x000004
/*
* x86(64) specific (mis)features:
*
* - Clockevent source stops in C3 State and needs broadcast support.
* - Local APIC timer is used as a dummy device.
*/
# define CLOCK_EVT_FEAT_C3STOP 0x000008
# define CLOCK_EVT_FEAT_DUMMY 0x000010
/*
* Core shall set the interrupt affinity dynamically in broadcast mode
*/
# define CLOCK_EVT_FEAT_DYNIRQ 0x000020
# define CLOCK_EVT_FEAT_PERCPU 0x000040
/*
* Clockevent device is based on a hrtimer for broadcast
*/
# define CLOCK_EVT_FEAT_HRTIMER 0x000080
/**
* struct clock_event_device - clock event device descriptor
* @event_handler: Assigned by the framework to be called by the low
* level handler of the event source
* @set_next_event: set next event function using a clocksource delta
* @set_next_ktime: set next event function using a direct ktime value
* @next_event: local storage for the next event in oneshot mode
* @max_delta_ns: maximum delta value in ns
* @min_delta_ns: minimum delta value in ns
* @mult: nanosecond to cycles multiplier
* @shift: nanoseconds to cycles divisor (power of two)
* @state_use_accessors:current state of the device, assigned by the core code
* @features: features
* @retries: number of forced programming retries
* @set_state_periodic: switch state to periodic
* @set_state_oneshot: switch state to oneshot
* @set_state_oneshot_stopped: switch state to oneshot_stopped
* @set_state_shutdown: switch state to shutdown
* @tick_resume: resume clkevt device
* @broadcast: function to broadcast events
* @min_delta_ticks: minimum delta value in ticks stored for reconfiguration
* @max_delta_ticks: maximum delta value in ticks stored for reconfiguration
* @name: ptr to clock event name
* @rating: variable to rate clock event devices
* @irq: IRQ number (only for non CPU local devices)
* @bound_on: Bound on CPU
* @cpumask: cpumask to indicate for which CPUs this device works
* @list: list head for the management code
* @owner: module reference
*/
struct clock_event_device {
void (*event_handler)(struct clock_event_device *);
int (*set_next_event)(unsigned long evt, struct clock_event_device *);
int (*set_next_ktime)(ktime_t expires, struct clock_event_device *);
ktime_t next_event;
u64 max_delta_ns;
u64 min_delta_ns;
u32 mult;
u32 shift;
enum clock_event_state state_use_accessors;
unsigned int features;
unsigned long retries;
int (*set_state_periodic)(struct clock_event_device *);
int (*set_state_oneshot)(struct clock_event_device *);
int (*set_state_oneshot_stopped)(struct clock_event_device *);
int (*set_state_shutdown)(struct clock_event_device *);
int (*tick_resume)(struct clock_event_device *);
void (*broadcast)(const struct cpumask *mask);
void (*suspend)(struct clock_event_device *);
void (*resume)(struct clock_event_device *);
unsigned long min_delta_ticks;
unsigned long max_delta_ticks;
const char *name;
int rating;
int irq;
int bound_on;
const struct cpumask *cpumask;
struct list_head list;
struct module *owner;
} ____cacheline_aligned;
/* Helpers to verify state of a clockevent device */
static inline bool clockevent_state_detached(struct clock_event_device *dev)
{
return dev->state_use_accessors == CLOCK_EVT_STATE_DETACHED;
}
static inline bool clockevent_state_shutdown(struct clock_event_device *dev)
{
return dev->state_use_accessors == CLOCK_EVT_STATE_SHUTDOWN;
}
static inline bool clockevent_state_periodic(struct clock_event_device *dev)
{
return dev->state_use_accessors == CLOCK_EVT_STATE_PERIODIC;
}
static inline bool clockevent_state_oneshot(struct clock_event_device *dev)
{
return dev->state_use_accessors == CLOCK_EVT_STATE_ONESHOT;
}
static inline bool clockevent_state_oneshot_stopped(struct clock_event_device *dev)
{
return dev->state_use_accessors == CLOCK_EVT_STATE_ONESHOT_STOPPED;
}
/*
* Calculate a multiplication factor for scaled math, which is used to convert
* nanoseconds based values to clock ticks:
*
* clock_ticks = (nanoseconds * factor) >> shift.
*
* div_sc is the rearranged equation to calculate a factor from a given clock
* ticks / nanoseconds ratio:
*
* factor = (clock_ticks << shift) / nanoseconds
*/
static inline unsigned long
div_sc(unsigned long ticks, unsigned long nsec, int shift)
{
u64 tmp = ((u64)ticks) << shift;
do_div(tmp, nsec);
return (unsigned long) tmp;
}
/* Clock event layer functions */
extern u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt);
extern void clockevents_register_device(struct clock_event_device *dev);
extern int clockevents_unbind_device(struct clock_event_device *ced, int cpu);
extern void clockevents_config_and_register(struct clock_event_device *dev,
u32 freq, unsigned long min_delta,
unsigned long max_delta);
extern int clockevents_update_freq(struct clock_event_device *ce, u32 freq);
static inline void
clockevents_calc_mult_shift(struct clock_event_device *ce, u32 freq, u32 maxsec)
{
return clocks_calc_mult_shift(&ce->mult, &ce->shift, NSEC_PER_SEC, freq, maxsec);
}
extern void clockevents_suspend(void);
extern void clockevents_resume(void);
# ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
# ifdef CONFIG_ARCH_HAS_TICK_BROADCAST
extern void tick_broadcast(const struct cpumask *mask);
# else
# define tick_broadcast NULL
# endif
extern int tick_receive_broadcast(void);
# endif
# if defined(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) && defined(CONFIG_TICK_ONESHOT)
extern void tick_setup_hrtimer_broadcast(void);
extern int tick_check_broadcast_expired(void);
# else
static __always_inline int tick_check_broadcast_expired(void) { return 0; }
static inline void tick_setup_hrtimer_broadcast(void) { }
# endif
#else /* !CONFIG_GENERIC_CLOCKEVENTS: */
static inline void clockevents_suspend(void) { }
static inline void clockevents_resume(void) { }
static __always_inline int tick_check_broadcast_expired(void) { return 0; }
static inline void tick_setup_hrtimer_broadcast(void) { }
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
#endif /* _LINUX_CLOCKCHIPS_H */
// SPDX-License-Identifier: GPL-2.0
// Generated by scripts/atomic/gen-atomic-long.sh
// DO NOT MODIFY THIS FILE DIRECTLY
#ifndef _LINUX_ATOMIC_LONG_H
#define _LINUX_ATOMIC_LONG_H
#include <linux/compiler.h>
#include <asm/types.h>
#ifdef CONFIG_64BIT
typedef atomic64_t atomic_long_t;
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
#define atomic_long_cond_read_acquire atomic64_cond_read_acquire
#define atomic_long_cond_read_relaxed atomic64_cond_read_relaxed
#else
typedef atomic_t atomic_long_t;
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
#define atomic_long_cond_read_acquire atomic_cond_read_acquire
#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
#endif
/**
* raw_atomic_long_read() - atomic load with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically loads the value of @v with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_read() elsewhere.
*
* Return: The value loaded from @v.
*/
static __always_inline long
raw_atomic_long_read(const atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_read(v);
#else
return raw_atomic_read(v);
#endif
}
/**
* raw_atomic_long_read_acquire() - atomic load with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically loads the value of @v with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_read_acquire() elsewhere.
*
* Return: The value loaded from @v.
*/
static __always_inline long
raw_atomic_long_read_acquire(const atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_read_acquire(v);
#else
return raw_atomic_read_acquire(v);
#endif
}
/**
* raw_atomic_long_set() - atomic set with relaxed ordering
* @v: pointer to atomic_long_t
* @i: long value to assign
*
* Atomically sets @v to @i with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_set() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_set(atomic_long_t *v, long i)
{
#ifdef CONFIG_64BIT
raw_atomic64_set(v, i);
#else
raw_atomic_set(v, i);
#endif
}
/**
* raw_atomic_long_set_release() - atomic set with release ordering
* @v: pointer to atomic_long_t
* @i: long value to assign
*
* Atomically sets @v to @i with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_set_release() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_set_release(atomic_long_t *v, long i)
{
#ifdef CONFIG_64BIT
raw_atomic64_set_release(v, i);
#else
raw_atomic_set_release(v, i);
#endif
}
/**
* raw_atomic_long_add() - atomic add with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_add(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_add(i, v);
#else
raw_atomic_add(i, v);
#endif
}
/**
* raw_atomic_long_add_return() - atomic add with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_return() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_add_return(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_return(i, v);
#else
return raw_atomic_add_return(i, v);
#endif
}
/**
* raw_atomic_long_add_return_acquire() - atomic add with acquire ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_return_acquire() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_add_return_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_return_acquire(i, v);
#else
return raw_atomic_add_return_acquire(i, v);
#endif
}
/**
* raw_atomic_long_add_return_release() - atomic add with release ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_return_release() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_add_return_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_return_release(i, v);
#else
return raw_atomic_add_return_release(i, v);
#endif
}
/**
* raw_atomic_long_add_return_relaxed() - atomic add with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_return_relaxed() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_return_relaxed(i, v);
#else
return raw_atomic_add_return_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_fetch_add() - atomic add with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_add() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_add(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_add(i, v);
#else
return raw_atomic_fetch_add(i, v);
#endif
}
/**
* raw_atomic_long_fetch_add_acquire() - atomic add with acquire ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_add_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_add_acquire(i, v);
#else
return raw_atomic_fetch_add_acquire(i, v);
#endif
}
/**
* raw_atomic_long_fetch_add_release() - atomic add with release ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_add_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_add_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_add_release(i, v);
#else
return raw_atomic_fetch_add_release(i, v);
#endif
}
/**
* raw_atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_add_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_add_relaxed(i, v);
#else
return raw_atomic_fetch_add_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_sub() - atomic subtract with relaxed ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_sub() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_sub(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_sub(i, v);
#else
raw_atomic_sub(i, v);
#endif
}
/**
* raw_atomic_long_sub_return() - atomic subtract with full ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_sub_return() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_sub_return(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_sub_return(i, v);
#else
return raw_atomic_sub_return(i, v);
#endif
}
/**
* raw_atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_sub_return_acquire() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_sub_return_acquire(i, v);
#else
return raw_atomic_sub_return_acquire(i, v);
#endif
}
/**
* raw_atomic_long_sub_return_release() - atomic subtract with release ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_sub_return_release() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_sub_return_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_sub_return_release(i, v);
#else
return raw_atomic_sub_return_release(i, v);
#endif
}
/**
* raw_atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_sub_return_relaxed() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_sub_return_relaxed(i, v);
#else
return raw_atomic_sub_return_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_fetch_sub() - atomic subtract with full ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_sub() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_sub(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_sub(i, v);
#else
return raw_atomic_fetch_sub(i, v);
#endif
}
/**
* raw_atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_sub_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_sub_acquire(i, v);
#else
return raw_atomic_fetch_sub_acquire(i, v);
#endif
}
/**
* raw_atomic_long_fetch_sub_release() - atomic subtract with release ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_sub_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_sub_release(i, v);
#else
return raw_atomic_fetch_sub_release(i, v);
#endif
}
/**
* raw_atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_sub_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_sub_relaxed(i, v);
#else
return raw_atomic_fetch_sub_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_inc() - atomic increment with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_inc() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_inc(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_inc(v);
#else
raw_atomic_inc(v);
#endif
}
/**
* raw_atomic_long_inc_return() - atomic increment with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_inc_return() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_inc_return(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_return(v);
#else
return raw_atomic_inc_return(v);
#endif
}
/**
* raw_atomic_long_inc_return_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_inc_return_acquire() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_inc_return_acquire(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_return_acquire(v);
#else
return raw_atomic_inc_return_acquire(v);
#endif
}
/**
* raw_atomic_long_inc_return_release() - atomic increment with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_inc_return_release() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_inc_return_release(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_return_release(v);
#else
return raw_atomic_inc_return_release(v);
#endif
}
/**
* raw_atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_inc_return_relaxed() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_inc_return_relaxed(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_return_relaxed(v);
#else
return raw_atomic_inc_return_relaxed(v);
#endif
}
/**
* raw_atomic_long_fetch_inc() - atomic increment with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_inc() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_inc(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_inc(v);
#else
return raw_atomic_fetch_inc(v);
#endif
}
/**
* raw_atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_inc_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_inc_acquire(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_inc_acquire(v);
#else
return raw_atomic_fetch_inc_acquire(v);
#endif
}
/**
* raw_atomic_long_fetch_inc_release() - atomic increment with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_inc_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_inc_release(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_inc_release(v);
#else
return raw_atomic_fetch_inc_release(v);
#endif
}
/**
* raw_atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_inc_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_inc_relaxed(v);
#else
return raw_atomic_fetch_inc_relaxed(v);
#endif
}
/**
* raw_atomic_long_dec() - atomic decrement with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_dec() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_dec(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_dec(v);
#else
raw_atomic_dec(v);
#endif
}
/**
* raw_atomic_long_dec_return() - atomic decrement with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_dec_return() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_dec_return(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_return(v);
#else
return raw_atomic_dec_return(v);
#endif
}
/**
* raw_atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_dec_return_acquire() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_dec_return_acquire(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_return_acquire(v);
#else
return raw_atomic_dec_return_acquire(v);
#endif
}
/**
* raw_atomic_long_dec_return_release() - atomic decrement with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_dec_return_release() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_dec_return_release(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_return_release(v);
#else
return raw_atomic_dec_return_release(v);
#endif
}
/**
* raw_atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_dec_return_relaxed() elsewhere.
*
* Return: The updated value of @v.
*/
static __always_inline long
raw_atomic_long_dec_return_relaxed(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_return_relaxed(v);
#else
return raw_atomic_dec_return_relaxed(v);
#endif
}
/**
* raw_atomic_long_fetch_dec() - atomic decrement with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_dec() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_dec(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_dec(v);
#else
return raw_atomic_fetch_dec(v);
#endif
}
/**
* raw_atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_dec_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_dec_acquire(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_dec_acquire(v);
#else
return raw_atomic_fetch_dec_acquire(v);
#endif
}
/**
* raw_atomic_long_fetch_dec_release() - atomic decrement with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_dec_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_dec_release(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_dec_release(v);
#else
return raw_atomic_fetch_dec_release(v);
#endif
}
/**
* raw_atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_dec_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_dec_relaxed(v);
#else
return raw_atomic_fetch_dec_relaxed(v);
#endif
}
/**
* raw_atomic_long_and() - atomic bitwise AND with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_and() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_and(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_and(i, v);
#else
raw_atomic_and(i, v);
#endif
}
/**
* raw_atomic_long_fetch_and() - atomic bitwise AND with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_and() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_and(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_and(i, v);
#else
return raw_atomic_fetch_and(i, v);
#endif
}
/**
* raw_atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_and_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_and_acquire(i, v);
#else
return raw_atomic_fetch_and_acquire(i, v);
#endif
}
/**
* raw_atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_and_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_and_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_and_release(i, v);
#else
return raw_atomic_fetch_and_release(i, v);
#endif
}
/**
* raw_atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_and_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_and_relaxed(i, v);
#else
return raw_atomic_fetch_and_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_andnot() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_andnot(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_andnot(i, v);
#else
raw_atomic_andnot(i, v);
#endif
}
/**
* raw_atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_andnot() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_andnot(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_andnot(i, v);
#else
return raw_atomic_fetch_andnot(i, v);
#endif
}
/**
* raw_atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_andnot_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_andnot_acquire(i, v);
#else
return raw_atomic_fetch_andnot_acquire(i, v);
#endif
}
/**
* raw_atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_andnot_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_andnot_release(i, v);
#else
return raw_atomic_fetch_andnot_release(i, v);
#endif
}
/**
* raw_atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_andnot_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_andnot_relaxed(i, v);
#else
return raw_atomic_fetch_andnot_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_or() - atomic bitwise OR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_or() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_or(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_or(i, v);
#else
raw_atomic_or(i, v);
#endif
}
/**
* raw_atomic_long_fetch_or() - atomic bitwise OR with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_or() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_or(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_or(i, v);
#else
return raw_atomic_fetch_or(i, v);
#endif
}
/**
* raw_atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_or_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_or_acquire(i, v);
#else
return raw_atomic_fetch_or_acquire(i, v);
#endif
}
/**
* raw_atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_or_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_or_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_or_release(i, v);
#else
return raw_atomic_fetch_or_release(i, v);
#endif
}
/**
* raw_atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_or_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_or_relaxed(i, v);
#else
return raw_atomic_fetch_or_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_xor() - atomic bitwise XOR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_xor() elsewhere.
*
* Return: Nothing.
*/
static __always_inline void
raw_atomic_long_xor(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
raw_atomic64_xor(i, v);
#else
raw_atomic_xor(i, v);
#endif
}
/**
* raw_atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_xor() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_xor(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_xor(i, v);
#else
return raw_atomic_fetch_xor(i, v);
#endif
}
/**
* raw_atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_xor_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_xor_acquire(i, v);
#else
return raw_atomic_fetch_xor_acquire(i, v);
#endif
}
/**
* raw_atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_xor_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_xor_release(i, v);
#else
return raw_atomic_fetch_xor_release(i, v);
#endif
}
/**
* raw_atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_xor_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_xor_relaxed(i, v);
#else
return raw_atomic_fetch_xor_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_xchg() - atomic exchange with full ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_xchg() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_xchg(atomic_long_t *v, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_xchg(v, new);
#else
return raw_atomic_xchg(v, new);
#endif
}
/**
* raw_atomic_long_xchg_acquire() - atomic exchange with acquire ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_xchg_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_xchg_acquire(atomic_long_t *v, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_xchg_acquire(v, new);
#else
return raw_atomic_xchg_acquire(v, new);
#endif
}
/**
* raw_atomic_long_xchg_release() - atomic exchange with release ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_xchg_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_xchg_release(atomic_long_t *v, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_xchg_release(v, new);
#else
return raw_atomic_xchg_release(v, new);
#endif
}
/**
* raw_atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_xchg_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_xchg_relaxed(v, new);
#else
return raw_atomic_xchg_relaxed(v, new);
#endif
}
/**
* raw_atomic_long_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_cmpxchg() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_cmpxchg(v, old, new);
#else
return raw_atomic_cmpxchg(v, old, new);
#endif
}
/**
* raw_atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_cmpxchg_acquire() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_cmpxchg_acquire(v, old, new);
#else
return raw_atomic_cmpxchg_acquire(v, old, new);
#endif
}
/**
* raw_atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_cmpxchg_release() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_cmpxchg_release(v, old, new);
#else
return raw_atomic_cmpxchg_release(v, old, new);
#endif
}
/**
* raw_atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_cmpxchg_relaxed() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_cmpxchg_relaxed(v, old, new);
#else
return raw_atomic_cmpxchg_relaxed(v, old, new);
#endif
}
/**
* raw_atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_try_cmpxchg(v, (s64 *)old, new);
#else
return raw_atomic_try_cmpxchg(v, (int *)old, new);
#endif
}
/**
* raw_atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
#else
return raw_atomic_try_cmpxchg_acquire(v, (int *)old, new);
#endif
}
/**
* raw_atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
#else
return raw_atomic_try_cmpxchg_release(v, (int *)old, new);
#endif
}
/**
* raw_atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
{
#ifdef CONFIG_64BIT
return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
#else
return raw_atomic_try_cmpxchg_relaxed(v, (int *)old, new);
#endif
}
/**
* raw_atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_sub_and_test() elsewhere.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_sub_and_test(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_sub_and_test(i, v);
#else
return raw_atomic_sub_and_test(i, v);
#endif
}
/**
* raw_atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_dec_and_test() elsewhere.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_dec_and_test(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_and_test(v);
#else
return raw_atomic_dec_and_test(v);
#endif
}
/**
* raw_atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_inc_and_test() elsewhere.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_inc_and_test(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_and_test(v);
#else
return raw_atomic_inc_and_test(v);
#endif
}
/**
* raw_atomic_long_add_negative() - atomic add and test if negative with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_negative() elsewhere.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_add_negative(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_negative(i, v);
#else
return raw_atomic_add_negative(i, v);
#endif
}
/**
* raw_atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_negative_acquire() elsewhere.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_negative_acquire(i, v);
#else
return raw_atomic_add_negative_acquire(i, v);
#endif
}
/**
* raw_atomic_long_add_negative_release() - atomic add and test if negative with release ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_negative_release() elsewhere.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_add_negative_release(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_negative_release(i, v);
#else
return raw_atomic_add_negative_release(i, v);
#endif
}
/**
* raw_atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Safe to use in noinstr code; prefer atomic_long_add_negative_relaxed() elsewhere.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_negative_relaxed(i, v);
#else
return raw_atomic_add_negative_relaxed(i, v);
#endif
}
/**
* raw_atomic_long_fetch_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic_long_t
* @a: long value to add
* @u: long value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_fetch_add_unless() elsewhere.
*
* Return: The original value of @v.
*/
static __always_inline long
raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
{
#ifdef CONFIG_64BIT
return raw_atomic64_fetch_add_unless(v, a, u);
#else
return raw_atomic_fetch_add_unless(v, a, u);
#endif
}
/**
* raw_atomic_long_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic_long_t
* @a: long value to add
* @u: long value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_add_unless() elsewhere.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_add_unless(atomic_long_t *v, long a, long u)
{
#ifdef CONFIG_64BIT
return raw_atomic64_add_unless(v, a, u);
#else
return raw_atomic_add_unless(v, a, u);
#endif
}
/**
* raw_atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
* @v: pointer to atomic_long_t
*
* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_inc_not_zero() elsewhere.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_inc_not_zero(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_not_zero(v);
#else
return raw_atomic_inc_not_zero(v);
#endif
}
/**
* raw_atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
* @v: pointer to atomic_long_t
*
* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_inc_unless_negative() elsewhere.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_inc_unless_negative(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_inc_unless_negative(v);
#else
return raw_atomic_inc_unless_negative(v);
#endif
}
/**
* raw_atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
* @v: pointer to atomic_long_t
*
* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_dec_unless_positive() elsewhere.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
raw_atomic_long_dec_unless_positive(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_unless_positive(v);
#else
return raw_atomic_dec_unless_positive(v);
#endif
}
/**
* raw_atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
* @v: pointer to atomic_long_t
*
* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere.
*
* Return: The old value of (@v - 1), regardless of whether @v was updated.
*/
static __always_inline long
raw_atomic_long_dec_if_positive(atomic_long_t *v)
{
#ifdef CONFIG_64BIT
return raw_atomic64_dec_if_positive(v);
#else
return raw_atomic_dec_if_positive(v);
#endif
}
#endif /* _LINUX_ATOMIC_LONG_H */
// 1c4a26fc77f345342953770ebe3c4d08e7ce2f9a
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/unistd.h>
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <asm/ucontext.h>
#include <asm/fpu/signal.h>
#include <asm/sighandling.h>
#include <asm/syscall.h>
#include <asm/sigframe.h>
#include <asm/signal.h>
/*
* If regs->ss will cause an IRET fault, change it. Otherwise leave it
* alone. Using this generally makes no sense unless
* user_64bit_mode(regs) would return true.
*/
static void force_valid_ss(struct pt_regs *regs)
{
u32 ar;
asm volatile ("lar %[old_ss], %[ar]\n\t"
"jz 1f\n\t" /* If invalid: */
"xorl %[ar], %[ar]\n\t" /* set ar = 0 */
"1:"
: [ar] "=r" (ar)
: [old_ss] "rm" ((u16)regs->ss));
/*
* For a valid 64-bit user context, we need DPL 3, type
* read-write data or read-write exp-down data, and S and P
* set. We can't use VERW because VERW doesn't check the
* P bit.
*/
ar &= AR_DPL_MASK | AR_S | AR_P | AR_TYPE_MASK;
if (ar != (AR_DPL3 | AR_S | AR_P | AR_TYPE_RWDATA) &&
ar != (AR_DPL3 | AR_S | AR_P | AR_TYPE_RWDATA_EXPDOWN))
regs->ss = __USER_DS;
}
static bool restore_sigcontext(struct pt_regs *regs,
struct sigcontext __user *usc,
unsigned long uc_flags)
{
struct sigcontext sc;
/* Always make any pending restarted system calls return -EINTR */
current->restart_block.fn = do_no_restart_syscall;
if (copy_from_user(&sc, usc, offsetof(struct sigcontext, reserved1)))
return false;
regs->bx = sc.bx;
regs->cx = sc.cx;
regs->dx = sc.dx;
regs->si = sc.si;
regs->di = sc.di;
regs->bp = sc.bp;
regs->ax = sc.ax;
regs->sp = sc.sp;
regs->ip = sc.ip;
regs->r8 = sc.r8;
regs->r9 = sc.r9;
regs->r10 = sc.r10;
regs->r11 = sc.r11;
regs->r12 = sc.r12;
regs->r13 = sc.r13;
regs->r14 = sc.r14;
regs->r15 = sc.r15;
/* Get CS/SS and force CPL3 */
regs->cs = sc.cs | 0x03;
regs->ss = sc.ss | 0x03;
regs->flags = (regs->flags & ~FIX_EFLAGS) | (sc.flags & FIX_EFLAGS);
/* disable syscall checks */
regs->orig_ax = -1;
/*
* Fix up SS if needed for the benefit of old DOSEMU and
* CRIU.
*/
if (unlikely(!(uc_flags & UC_STRICT_RESTORE_SS) && user_64bit_mode(regs)))
force_valid_ss(regs);
return fpu__restore_sig((void __user *)sc.fpstate, 0);
}
static __always_inline int
__unsafe_setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
struct pt_regs *regs, unsigned long mask)
{
unsafe_put_user(regs->di, &sc->di, Efault); unsafe_put_user(regs->si, &sc->si, Efault); unsafe_put_user(regs->bp, &sc->bp, Efault); unsafe_put_user(regs->sp, &sc->sp, Efault); unsafe_put_user(regs->bx, &sc->bx, Efault); unsafe_put_user(regs->dx, &sc->dx, Efault); unsafe_put_user(regs->cx, &sc->cx, Efault); unsafe_put_user(regs->ax, &sc->ax, Efault); unsafe_put_user(regs->r8, &sc->r8, Efault); unsafe_put_user(regs->r9, &sc->r9, Efault); unsafe_put_user(regs->r10, &sc->r10, Efault); unsafe_put_user(regs->r11, &sc->r11, Efault); unsafe_put_user(regs->r12, &sc->r12, Efault); unsafe_put_user(regs->r13, &sc->r13, Efault); unsafe_put_user(regs->r14, &sc->r14, Efault); unsafe_put_user(regs->r15, &sc->r15, Efault); unsafe_put_user(current->thread.trap_nr, &sc->trapno, Efault); unsafe_put_user(current->thread.error_code, &sc->err, Efault); unsafe_put_user(regs->ip, &sc->ip, Efault); unsafe_put_user(regs->flags, &sc->flags, Efault); unsafe_put_user(regs->cs, &sc->cs, Efault); unsafe_put_user(0, &sc->gs, Efault); unsafe_put_user(0, &sc->fs, Efault); unsafe_put_user(regs->ss, &sc->ss, Efault);
unsafe_put_user(fpstate, (unsigned long __user *)&sc->fpstate, Efault);
/* non-iBCS2 extensions.. */
unsafe_put_user(mask, &sc->oldmask, Efault);
unsafe_put_user(current->thread.cr2, &sc->cr2, Efault);
return 0;
Efault:
return -EFAULT;
}
#define unsafe_put_sigcontext(sc, fp, regs, set, label) \
do { \
if (__unsafe_setup_sigcontext(sc, fp, regs, set->sig[0])) \
goto label; \
} while(0);
#define unsafe_put_sigmask(set, frame, label) \
unsafe_put_user(*(__u64 *)(set), \
(__u64 __user *)&(frame)->uc.uc_sigmask, \
label)
static unsigned long frame_uc_flags(struct pt_regs *regs)
{
unsigned long flags;
if (boot_cpu_has(X86_FEATURE_XSAVE))
flags = UC_FP_XSTATE | UC_SIGCONTEXT_SS;
else
flags = UC_SIGCONTEXT_SS;
if (likely(user_64bit_mode(regs)))
flags |= UC_STRICT_RESTORE_SS;
return flags;
}
int x64_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
{
sigset_t *set = sigmask_to_save();
struct rt_sigframe __user *frame;
void __user *fp = NULL;
unsigned long uc_flags;
/* x86-64 should always use SA_RESTORER. */
if (!(ksig->ka.sa.sa_flags & SA_RESTORER))
return -EFAULT; frame = get_sigframe(ksig, regs, sizeof(struct rt_sigframe), &fp); uc_flags = frame_uc_flags(regs); if (!user_access_begin(frame, sizeof(*frame)))
return -EFAULT;
/* Create the ucontext. */
unsafe_put_user(uc_flags, &frame->uc.uc_flags, Efault);
unsafe_put_user(0, &frame->uc.uc_link, Efault); unsafe_save_altstack(&frame->uc.uc_stack, regs->sp, Efault);
/* Set up to return from userspace. If provided, use a stub
already in userspace. */
unsafe_put_user(ksig->ka.sa.sa_restorer, &frame->pretcode, Efault); unsafe_put_sigcontext(&frame->uc.uc_mcontext, fp, regs, set, Efault); unsafe_put_sigmask(set, frame, Efault); user_access_end();
if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
if (copy_siginfo_to_user(&frame->info, &ksig->info))
return -EFAULT;
}
if (setup_signal_shadow_stack(ksig))
return -EFAULT;
/* Set up registers for signal handler */
regs->di = ksig->sig;
/* In case the signal handler was declared without prototypes */
regs->ax = 0;
/* This also works for non SA_SIGINFO handlers because they expect the
next argument after the signal number on the stack. */
regs->si = (unsigned long)&frame->info;
regs->dx = (unsigned long)&frame->uc;
regs->ip = (unsigned long) ksig->ka.sa.sa_handler;
regs->sp = (unsigned long)frame;
/*
* Set up the CS and SS registers to run signal handlers in
* 64-bit mode, even if the handler happens to be interrupting
* 32-bit or 16-bit code.
*
* SS is subtle. In 64-bit mode, we don't need any particular
* SS descriptor, but we do need SS to be valid. It's possible
* that the old SS is entirely bogus -- this can happen if the
* signal we're trying to deliver is #GP or #SS caused by a bad
* SS value. We also have a compatibility issue here: DOSEMU
* relies on the contents of the SS register indicating the
* SS value at the time of the signal, even though that code in
* DOSEMU predates sigreturn's ability to restore SS. (DOSEMU
* avoids relying on sigreturn to restore SS; instead it uses
* a trampoline.) So we do our best: if the old SS was valid,
* we keep it. Otherwise we replace it.
*/
regs->cs = __USER_CS;
if (unlikely(regs->ss != __USER_DS))
force_valid_ss(regs);
return 0;
Efault:
user_access_end();
return -EFAULT;
}
/*
* Do a signal return; undo the signal stack.
*/
SYSCALL_DEFINE0(rt_sigreturn)
{
struct pt_regs *regs = current_pt_regs();
struct rt_sigframe __user *frame;
sigset_t set;
unsigned long uc_flags;
frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long));
if (!access_ok(frame, sizeof(*frame)))
goto badframe;
if (__get_user(*(__u64 *)&set, (__u64 __user *)&frame->uc.uc_sigmask))
goto badframe;
if (__get_user(uc_flags, &frame->uc.uc_flags))
goto badframe;
set_current_blocked(&set);
if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))
goto badframe;
if (restore_signal_shadow_stack())
goto badframe;
if (restore_altstack(&frame->uc.uc_stack))
goto badframe;
return regs->ax;
badframe:
signal_fault(regs, frame, "rt_sigreturn");
return 0;
}
#ifdef CONFIG_X86_X32_ABI
static int x32_copy_siginfo_to_user(struct compat_siginfo __user *to,
const struct kernel_siginfo *from)
{
struct compat_siginfo new;
copy_siginfo_to_external32(&new, from);
if (from->si_signo == SIGCHLD) {
new._sifields._sigchld_x32._utime = from->si_utime;
new._sifields._sigchld_x32._stime = from->si_stime;
}
if (copy_to_user(to, &new, sizeof(struct compat_siginfo)))
return -EFAULT;
return 0;
}
int copy_siginfo_to_user32(struct compat_siginfo __user *to,
const struct kernel_siginfo *from)
{
if (in_x32_syscall())
return x32_copy_siginfo_to_user(to, from);
return __copy_siginfo_to_user32(to, from);
}
int x32_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
{
compat_sigset_t *set = (compat_sigset_t *) sigmask_to_save();
struct rt_sigframe_x32 __user *frame;
unsigned long uc_flags;
void __user *restorer;
void __user *fp = NULL;
if (!(ksig->ka.sa.sa_flags & SA_RESTORER))
return -EFAULT;
frame = get_sigframe(ksig, regs, sizeof(*frame), &fp);
uc_flags = frame_uc_flags(regs);
if (!user_access_begin(frame, sizeof(*frame)))
return -EFAULT;
/* Create the ucontext. */
unsafe_put_user(uc_flags, &frame->uc.uc_flags, Efault);
unsafe_put_user(0, &frame->uc.uc_link, Efault);
unsafe_compat_save_altstack(&frame->uc.uc_stack, regs->sp, Efault);
unsafe_put_user(0, &frame->uc.uc__pad0, Efault);
restorer = ksig->ka.sa.sa_restorer;
unsafe_put_user(restorer, (unsigned long __user *)&frame->pretcode, Efault);
unsafe_put_sigcontext(&frame->uc.uc_mcontext, fp, regs, set, Efault);
unsafe_put_sigmask(set, frame, Efault);
user_access_end();
if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
if (x32_copy_siginfo_to_user(&frame->info, &ksig->info))
return -EFAULT;
}
/* Set up registers for signal handler */
regs->sp = (unsigned long) frame;
regs->ip = (unsigned long) ksig->ka.sa.sa_handler;
/* We use the x32 calling convention here... */
regs->di = ksig->sig;
regs->si = (unsigned long) &frame->info;
regs->dx = (unsigned long) &frame->uc;
loadsegment(ds, __USER_DS);
loadsegment(es, __USER_DS);
regs->cs = __USER_CS;
regs->ss = __USER_DS;
return 0;
Efault:
user_access_end();
return -EFAULT;
}
COMPAT_SYSCALL_DEFINE0(x32_rt_sigreturn)
{
struct pt_regs *regs = current_pt_regs();
struct rt_sigframe_x32 __user *frame;
sigset_t set;
unsigned long uc_flags;
frame = (struct rt_sigframe_x32 __user *)(regs->sp - 8);
if (!access_ok(frame, sizeof(*frame)))
goto badframe;
if (__get_user(set.sig[0], (__u64 __user *)&frame->uc.uc_sigmask))
goto badframe;
if (__get_user(uc_flags, &frame->uc.uc_flags))
goto badframe;
set_current_blocked(&set);
if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))
goto badframe;
if (compat_restore_altstack(&frame->uc.uc_stack))
goto badframe;
return regs->ax;
badframe:
signal_fault(regs, frame, "x32 rt_sigreturn");
return 0;
}
#endif /* CONFIG_X86_X32_ABI */
#ifdef CONFIG_COMPAT
void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oact)
{
if (!act)
return;
if (in_ia32_syscall())
act->sa.sa_flags |= SA_IA32_ABI;
if (in_x32_syscall())
act->sa.sa_flags |= SA_X32_ABI;
}
#endif /* CONFIG_COMPAT */
/*
* If adding a new si_code, there is probably new data in
* the siginfo. Make sure folks bumping the si_code
* limits also have to look at this code. Make sure any
* new fields are handled in copy_siginfo_to_user32()!
*/
static_assert(NSIGILL == 11);
static_assert(NSIGFPE == 15);
static_assert(NSIGSEGV == 10);
static_assert(NSIGBUS == 5);
static_assert(NSIGTRAP == 6);
static_assert(NSIGCHLD == 6);
static_assert(NSIGSYS == 2);
/* This is part of the ABI and can never change in size: */
static_assert(sizeof(siginfo_t) == 128);
/* This is a part of the ABI and can never change in alignment */
static_assert(__alignof__(siginfo_t) == 8);
/*
* The offsets of all the (unioned) si_fields are fixed
* in the ABI, of course. Make sure none of them ever
* move and are always at the beginning:
*/
static_assert(offsetof(siginfo_t, si_signo) == 0);
static_assert(offsetof(siginfo_t, si_errno) == 4);
static_assert(offsetof(siginfo_t, si_code) == 8);
/*
* Ensure that the size of each si_field never changes.
* If it does, it is a sign that the
* copy_siginfo_to_user32() code below needs to updated
* along with the size in the CHECK_SI_SIZE().
*
* We repeat this check for both the generic and compat
* siginfos.
*
* Note: it is OK for these to grow as long as the whole
* structure stays within the padding size (checked
* above).
*/
#define CHECK_SI_OFFSET(name) \
static_assert(offsetof(siginfo_t, _sifields) == \
offsetof(siginfo_t, _sifields.name))
#define CHECK_SI_SIZE(name, size) \
static_assert(sizeof_field(siginfo_t, _sifields.name) == size)
CHECK_SI_OFFSET(_kill);
CHECK_SI_SIZE (_kill, 2*sizeof(int));
static_assert(offsetof(siginfo_t, si_pid) == 0x10);
static_assert(offsetof(siginfo_t, si_uid) == 0x14);
CHECK_SI_OFFSET(_timer);
CHECK_SI_SIZE (_timer, 6*sizeof(int));
static_assert(offsetof(siginfo_t, si_tid) == 0x10);
static_assert(offsetof(siginfo_t, si_overrun) == 0x14);
static_assert(offsetof(siginfo_t, si_value) == 0x18);
CHECK_SI_OFFSET(_rt);
CHECK_SI_SIZE (_rt, 4*sizeof(int));
static_assert(offsetof(siginfo_t, si_pid) == 0x10);
static_assert(offsetof(siginfo_t, si_uid) == 0x14);
static_assert(offsetof(siginfo_t, si_value) == 0x18);
CHECK_SI_OFFSET(_sigchld);
CHECK_SI_SIZE (_sigchld, 8*sizeof(int));
static_assert(offsetof(siginfo_t, si_pid) == 0x10);
static_assert(offsetof(siginfo_t, si_uid) == 0x14);
static_assert(offsetof(siginfo_t, si_status) == 0x18);
static_assert(offsetof(siginfo_t, si_utime) == 0x20);
static_assert(offsetof(siginfo_t, si_stime) == 0x28);
#ifdef CONFIG_X86_X32_ABI
/* no _sigchld_x32 in the generic siginfo_t */
static_assert(sizeof_field(compat_siginfo_t, _sifields._sigchld_x32) ==
7*sizeof(int));
static_assert(offsetof(compat_siginfo_t, _sifields) ==
offsetof(compat_siginfo_t, _sifields._sigchld_x32));
static_assert(offsetof(compat_siginfo_t, _sifields._sigchld_x32._utime) == 0x18);
static_assert(offsetof(compat_siginfo_t, _sifields._sigchld_x32._stime) == 0x20);
#endif
CHECK_SI_OFFSET(_sigfault);
CHECK_SI_SIZE (_sigfault, 8*sizeof(int));
static_assert(offsetof(siginfo_t, si_addr) == 0x10);
static_assert(offsetof(siginfo_t, si_trapno) == 0x18);
static_assert(offsetof(siginfo_t, si_addr_lsb) == 0x18);
static_assert(offsetof(siginfo_t, si_lower) == 0x20);
static_assert(offsetof(siginfo_t, si_upper) == 0x28);
static_assert(offsetof(siginfo_t, si_pkey) == 0x20);
static_assert(offsetof(siginfo_t, si_perf_data) == 0x18);
static_assert(offsetof(siginfo_t, si_perf_type) == 0x20);
static_assert(offsetof(siginfo_t, si_perf_flags) == 0x24);
CHECK_SI_OFFSET(_sigpoll);
CHECK_SI_SIZE (_sigpoll, 4*sizeof(int));
static_assert(offsetof(siginfo_t, si_band) == 0x10);
static_assert(offsetof(siginfo_t, si_fd) == 0x18);
CHECK_SI_OFFSET(_sigsys);
CHECK_SI_SIZE (_sigsys, 4*sizeof(int));
static_assert(offsetof(siginfo_t, si_call_addr) == 0x10);
static_assert(offsetof(siginfo_t, si_syscall) == 0x18);
static_assert(offsetof(siginfo_t, si_arch) == 0x1C);
/* any new si_fields should be added here */
// SPDX-License-Identifier: GPL-2.0
// rc-main.c - Remote Controller core module
//
// Copyright (C) 2009-2010 by Mauro Carvalho Chehab
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <media/rc-core.h>
#include <linux/bsearch.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/input.h>
#include <linux/leds.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/device.h>
#include <linux/module.h>
#include "rc-core-priv.h"
/* Sizes are in bytes, 256 bytes allows for 32 entries on x64 */
#define IR_TAB_MIN_SIZE 256
#define IR_TAB_MAX_SIZE 8192
static const struct {
const char *name;
unsigned int repeat_period;
unsigned int scancode_bits;
} protocols[] = {
[RC_PROTO_UNKNOWN] = { .name = "unknown", .repeat_period = 125 },
[RC_PROTO_OTHER] = { .name = "other", .repeat_period = 125 },
[RC_PROTO_RC5] = { .name = "rc-5",
.scancode_bits = 0x1f7f, .repeat_period = 114 },
[RC_PROTO_RC5X_20] = { .name = "rc-5x-20",
.scancode_bits = 0x1f7f3f, .repeat_period = 114 },
[RC_PROTO_RC5_SZ] = { .name = "rc-5-sz",
.scancode_bits = 0x2fff, .repeat_period = 114 },
[RC_PROTO_JVC] = { .name = "jvc",
.scancode_bits = 0xffff, .repeat_period = 125 },
[RC_PROTO_SONY12] = { .name = "sony-12",
.scancode_bits = 0x1f007f, .repeat_period = 100 },
[RC_PROTO_SONY15] = { .name = "sony-15",
.scancode_bits = 0xff007f, .repeat_period = 100 },
[RC_PROTO_SONY20] = { .name = "sony-20",
.scancode_bits = 0x1fff7f, .repeat_period = 100 },
[RC_PROTO_NEC] = { .name = "nec",
.scancode_bits = 0xffff, .repeat_period = 110 },
[RC_PROTO_NECX] = { .name = "nec-x",
.scancode_bits = 0xffffff, .repeat_period = 110 },
[RC_PROTO_NEC32] = { .name = "nec-32",
.scancode_bits = 0xffffffff, .repeat_period = 110 },
[RC_PROTO_SANYO] = { .name = "sanyo",
.scancode_bits = 0x1fffff, .repeat_period = 125 },
[RC_PROTO_MCIR2_KBD] = { .name = "mcir2-kbd",
.scancode_bits = 0xffffff, .repeat_period = 100 },
[RC_PROTO_MCIR2_MSE] = { .name = "mcir2-mse",
.scancode_bits = 0x1fffff, .repeat_period = 100 },
[RC_PROTO_RC6_0] = { .name = "rc-6-0",
.scancode_bits = 0xffff, .repeat_period = 114 },
[RC_PROTO_RC6_6A_20] = { .name = "rc-6-6a-20",
.scancode_bits = 0xfffff, .repeat_period = 114 },
[RC_PROTO_RC6_6A_24] = { .name = "rc-6-6a-24",
.scancode_bits = 0xffffff, .repeat_period = 114 },
[RC_PROTO_RC6_6A_32] = { .name = "rc-6-6a-32",
.scancode_bits = 0xffffffff, .repeat_period = 114 },
[RC_PROTO_RC6_MCE] = { .name = "rc-6-mce",
.scancode_bits = 0xffff7fff, .repeat_period = 114 },
[RC_PROTO_SHARP] = { .name = "sharp",
.scancode_bits = 0x1fff, .repeat_period = 125 },
[RC_PROTO_XMP] = { .name = "xmp", .repeat_period = 125 },
[RC_PROTO_CEC] = { .name = "cec", .repeat_period = 0 },
[RC_PROTO_IMON] = { .name = "imon",
.scancode_bits = 0x7fffffff, .repeat_period = 114 },
[RC_PROTO_RCMM12] = { .name = "rc-mm-12",
.scancode_bits = 0x00000fff, .repeat_period = 114 },
[RC_PROTO_RCMM24] = { .name = "rc-mm-24",
.scancode_bits = 0x00ffffff, .repeat_period = 114 },
[RC_PROTO_RCMM32] = { .name = "rc-mm-32",
.scancode_bits = 0xffffffff, .repeat_period = 114 },
[RC_PROTO_XBOX_DVD] = { .name = "xbox-dvd", .repeat_period = 64 },
};
/* Used to keep track of known keymaps */
static LIST_HEAD(rc_map_list);
static DEFINE_SPINLOCK(rc_map_lock);
static struct led_trigger *led_feedback;
/* Used to keep track of rc devices */
static DEFINE_IDA(rc_ida);
static struct rc_map_list *seek_rc_map(const char *name)
{
struct rc_map_list *map = NULL;
spin_lock(&rc_map_lock);
list_for_each_entry(map, &rc_map_list, list) {
if (!strcmp(name, map->map.name)) {
spin_unlock(&rc_map_lock);
return map;
}
}
spin_unlock(&rc_map_lock);
return NULL;
}
struct rc_map *rc_map_get(const char *name)
{
struct rc_map_list *map;
map = seek_rc_map(name);
#ifdef CONFIG_MODULES
if (!map) {
int rc = request_module("%s", name);
if (rc < 0) {
pr_err("Couldn't load IR keymap %s\n", name);
return NULL;
}
msleep(20); /* Give some time for IR to register */
map = seek_rc_map(name);
}
#endif
if (!map) {
pr_err("IR keymap %s not found\n", name);
return NULL;
}
printk(KERN_INFO "Registered IR keymap %s\n", map->map.name);
return &map->map;
}
EXPORT_SYMBOL_GPL(rc_map_get);
int rc_map_register(struct rc_map_list *map)
{
spin_lock(&rc_map_lock);
list_add_tail(&map->list, &rc_map_list);
spin_unlock(&rc_map_lock);
return 0;
}
EXPORT_SYMBOL_GPL(rc_map_register);
void rc_map_unregister(struct rc_map_list *map)
{
spin_lock(&rc_map_lock);
list_del(&map->list);
spin_unlock(&rc_map_lock);
}
EXPORT_SYMBOL_GPL(rc_map_unregister);
static struct rc_map_table empty[] = {
{ 0x2a, KEY_COFFEE },
};
static struct rc_map_list empty_map = {
.map = {
.scan = empty,
.size = ARRAY_SIZE(empty),
.rc_proto = RC_PROTO_UNKNOWN, /* Legacy IR type */
.name = RC_MAP_EMPTY,
}
};
/**
* scancode_to_u64() - converts scancode in &struct input_keymap_entry
* @ke: keymap entry containing scancode to be converted.
* @scancode: pointer to the location where converted scancode should
* be stored.
*
* This function is a version of input_scancode_to_scalar specialized for
* rc-core.
*/
static int scancode_to_u64(const struct input_keymap_entry *ke, u64 *scancode)
{
switch (ke->len) {
case 1:
*scancode = *((u8 *)ke->scancode);
break;
case 2:
*scancode = *((u16 *)ke->scancode);
break;
case 4:
*scancode = *((u32 *)ke->scancode);
break;
case 8:
*scancode = *((u64 *)ke->scancode);
break;
default:
return -EINVAL;
}
return 0;
}
/**
* ir_create_table() - initializes a scancode table
* @dev: the rc_dev device
* @rc_map: the rc_map to initialize
* @name: name to assign to the table
* @rc_proto: ir type to assign to the new table
* @size: initial size of the table
*
* This routine will initialize the rc_map and will allocate
* memory to hold at least the specified number of elements.
*
* return: zero on success or a negative error code
*/
static int ir_create_table(struct rc_dev *dev, struct rc_map *rc_map,
const char *name, u64 rc_proto, size_t size)
{
rc_map->name = kstrdup(name, GFP_KERNEL);
if (!rc_map->name)
return -ENOMEM;
rc_map->rc_proto = rc_proto;
rc_map->alloc = roundup_pow_of_two(size * sizeof(struct rc_map_table));
rc_map->size = rc_map->alloc / sizeof(struct rc_map_table);
rc_map->scan = kmalloc(rc_map->alloc, GFP_KERNEL);
if (!rc_map->scan) {
kfree(rc_map->name);
rc_map->name = NULL;
return -ENOMEM;
}
dev_dbg(&dev->dev, "Allocated space for %u keycode entries (%u bytes)\n",
rc_map->size, rc_map->alloc);
return 0;
}
/**
* ir_free_table() - frees memory allocated by a scancode table
* @rc_map: the table whose mappings need to be freed
*
* This routine will free memory alloctaed for key mappings used by given
* scancode table.
*/
static void ir_free_table(struct rc_map *rc_map)
{
rc_map->size = 0;
kfree(rc_map->name);
rc_map->name = NULL;
kfree(rc_map->scan);
rc_map->scan = NULL;
}
/**
* ir_resize_table() - resizes a scancode table if necessary
* @dev: the rc_dev device
* @rc_map: the rc_map to resize
* @gfp_flags: gfp flags to use when allocating memory
*
* This routine will shrink the rc_map if it has lots of
* unused entries and grow it if it is full.
*
* return: zero on success or a negative error code
*/
static int ir_resize_table(struct rc_dev *dev, struct rc_map *rc_map,
gfp_t gfp_flags)
{
unsigned int oldalloc = rc_map->alloc;
unsigned int newalloc = oldalloc;
struct rc_map_table *oldscan = rc_map->scan;
struct rc_map_table *newscan;
if (rc_map->size == rc_map->len) {
/* All entries in use -> grow keytable */
if (rc_map->alloc >= IR_TAB_MAX_SIZE)
return -ENOMEM;
newalloc *= 2; dev_dbg(&dev->dev, "Growing table to %u bytes\n", newalloc);
}
if ((rc_map->len * 3 < rc_map->size) && (oldalloc > IR_TAB_MIN_SIZE)) {
/* Less than 1/3 of entries in use -> shrink keytable */
newalloc /= 2; dev_dbg(&dev->dev, "Shrinking table to %u bytes\n", newalloc);
}
if (newalloc == oldalloc) return 0; newscan = kmalloc(newalloc, gfp_flags);
if (!newscan)
return -ENOMEM;
memcpy(newscan, rc_map->scan, rc_map->len * sizeof(struct rc_map_table));
rc_map->scan = newscan;
rc_map->alloc = newalloc;
rc_map->size = rc_map->alloc / sizeof(struct rc_map_table);
kfree(oldscan);
return 0;
}
/**
* ir_update_mapping() - set a keycode in the scancode->keycode table
* @dev: the struct rc_dev device descriptor
* @rc_map: scancode table to be adjusted
* @index: index of the mapping that needs to be updated
* @new_keycode: the desired keycode
*
* This routine is used to update scancode->keycode mapping at given
* position.
*
* return: previous keycode assigned to the mapping
*
*/
static unsigned int ir_update_mapping(struct rc_dev *dev,
struct rc_map *rc_map,
unsigned int index,
unsigned int new_keycode)
{
int old_keycode = rc_map->scan[index].keycode;
int i;
/* Did the user wish to remove the mapping? */
if (new_keycode == KEY_RESERVED || new_keycode == KEY_UNKNOWN) { dev_dbg(&dev->dev, "#%d: Deleting scan 0x%04llx\n",
index, rc_map->scan[index].scancode);
rc_map->len--;
memmove(&rc_map->scan[index], &rc_map->scan[index+ 1],
(rc_map->len - index) * sizeof(struct rc_map_table));
} else {
dev_dbg(&dev->dev, "#%d: %s scan 0x%04llx with key 0x%04x\n",
index,
old_keycode == KEY_RESERVED ? "New" : "Replacing",
rc_map->scan[index].scancode, new_keycode);
rc_map->scan[index].keycode = new_keycode;
__set_bit(new_keycode, dev->input_dev->keybit);
}
if (old_keycode != KEY_RESERVED) {
/* A previous mapping was updated... */
__clear_bit(old_keycode, dev->input_dev->keybit);
/* ... but another scancode might use the same keycode */
for (i = 0; i < rc_map->len; i++) { if (rc_map->scan[i].keycode == old_keycode) { __set_bit(old_keycode, dev->input_dev->keybit);
break;
}
}
/* Possibly shrink the keytable, failure is not a problem */
ir_resize_table(dev, rc_map, GFP_ATOMIC);
}
return old_keycode;
}
/**
* ir_establish_scancode() - set a keycode in the scancode->keycode table
* @dev: the struct rc_dev device descriptor
* @rc_map: scancode table to be searched
* @scancode: the desired scancode
* @resize: controls whether we allowed to resize the table to
* accommodate not yet present scancodes
*
* This routine is used to locate given scancode in rc_map.
* If scancode is not yet present the routine will allocate a new slot
* for it.
*
* return: index of the mapping containing scancode in question
* or -1U in case of failure.
*/
static unsigned int ir_establish_scancode(struct rc_dev *dev,
struct rc_map *rc_map,
u64 scancode, bool resize)
{
unsigned int i;
/*
* Unfortunately, some hardware-based IR decoders don't provide
* all bits for the complete IR code. In general, they provide only
* the command part of the IR code. Yet, as it is possible to replace
* the provided IR with another one, it is needed to allow loading
* IR tables from other remotes. So, we support specifying a mask to
* indicate the valid bits of the scancodes.
*/
if (dev->scancode_mask) scancode &= dev->scancode_mask;
/* First check if we already have a mapping for this ir command */
for (i = 0; i < rc_map->len; i++) { if (rc_map->scan[i].scancode == scancode)
return i;
/* Keytable is sorted from lowest to highest scancode */
if (rc_map->scan[i].scancode >= scancode)
break;
}
/* No previous mapping found, we might need to grow the table */
if (rc_map->size == rc_map->len) { if (!resize || ir_resize_table(dev, rc_map, GFP_ATOMIC))
return -1U;
}
/* i is the proper index to insert our new keycode */
if (i < rc_map->len) memmove(&rc_map->scan[i + 1], &rc_map->scan[i],
(rc_map->len - i) * sizeof(struct rc_map_table));
rc_map->scan[i].scancode = scancode;
rc_map->scan[i].keycode = KEY_RESERVED;
rc_map->len++;
return i;
}
/**
* ir_setkeycode() - set a keycode in the scancode->keycode table
* @idev: the struct input_dev device descriptor
* @ke: Input keymap entry
* @old_keycode: result
*
* This routine is used to handle evdev EVIOCSKEY ioctl.
*
* return: -EINVAL if the keycode could not be inserted, otherwise zero.
*/
static int ir_setkeycode(struct input_dev *idev,
const struct input_keymap_entry *ke,
unsigned int *old_keycode)
{
struct rc_dev *rdev = input_get_drvdata(idev);
struct rc_map *rc_map = &rdev->rc_map;
unsigned int index;
u64 scancode;
int retval = 0;
unsigned long flags;
spin_lock_irqsave(&rc_map->lock, flags);
if (ke->flags & INPUT_KEYMAP_BY_INDEX) {
index = ke->index;
if (index >= rc_map->len) {
retval = -EINVAL;
goto out;
}
} else {
retval = scancode_to_u64(ke, &scancode);
if (retval)
goto out;
index = ir_establish_scancode(rdev, rc_map, scancode, true);
if (index >= rc_map->len) {
retval = -ENOMEM;
goto out;
}
}
*old_keycode = ir_update_mapping(rdev, rc_map, index, ke->keycode);
out:
spin_unlock_irqrestore(&rc_map->lock, flags);
return retval;
}
/**
* ir_setkeytable() - sets several entries in the scancode->keycode table
* @dev: the struct rc_dev device descriptor
* @from: the struct rc_map to copy entries from
*
* This routine is used to handle table initialization.
*
* return: -ENOMEM if all keycodes could not be inserted, otherwise zero.
*/
static int ir_setkeytable(struct rc_dev *dev, const struct rc_map *from)
{
struct rc_map *rc_map = &dev->rc_map;
unsigned int i, index;
int rc;
rc = ir_create_table(dev, rc_map, from->name, from->rc_proto,
from->size);
if (rc)
return rc;
for (i = 0; i < from->size; i++) {
index = ir_establish_scancode(dev, rc_map,
from->scan[i].scancode, false);
if (index >= rc_map->len) {
rc = -ENOMEM;
break;
}
ir_update_mapping(dev, rc_map, index,
from->scan[i].keycode);
}
if (rc)
ir_free_table(rc_map);
return rc;
}
static int rc_map_cmp(const void *key, const void *elt)
{
const u64 *scancode = key;
const struct rc_map_table *e = elt;
if (*scancode < e->scancode)
return -1;
else if (*scancode > e->scancode)
return 1;
return 0;
}
/**
* ir_lookup_by_scancode() - locate mapping by scancode
* @rc_map: the struct rc_map to search
* @scancode: scancode to look for in the table
*
* This routine performs binary search in RC keykeymap table for
* given scancode.
*
* return: index in the table, -1U if not found
*/
static unsigned int ir_lookup_by_scancode(const struct rc_map *rc_map,
u64 scancode)
{
struct rc_map_table *res;
res = bsearch(&scancode, rc_map->scan, rc_map->len,
sizeof(struct rc_map_table), rc_map_cmp);
if (!res)
return -1U;
else
return res - rc_map->scan;
}
/**
* ir_getkeycode() - get a keycode from the scancode->keycode table
* @idev: the struct input_dev device descriptor
* @ke: Input keymap entry
*
* This routine is used to handle evdev EVIOCGKEY ioctl.
*
* return: always returns zero.
*/
static int ir_getkeycode(struct input_dev *idev,
struct input_keymap_entry *ke)
{
struct rc_dev *rdev = input_get_drvdata(idev);
struct rc_map *rc_map = &rdev->rc_map;
struct rc_map_table *entry;
unsigned long flags;
unsigned int index;
u64 scancode;
int retval;
spin_lock_irqsave(&rc_map->lock, flags);
if (ke->flags & INPUT_KEYMAP_BY_INDEX) {
index = ke->index;
} else {
retval = scancode_to_u64(ke, &scancode);
if (retval)
goto out;
index = ir_lookup_by_scancode(rc_map, scancode);
}
if (index < rc_map->len) {
entry = &rc_map->scan[index]; ke->index = index;
ke->keycode = entry->keycode;
ke->len = sizeof(entry->scancode);
memcpy(ke->scancode, &entry->scancode, sizeof(entry->scancode));
} else if (!(ke->flags & INPUT_KEYMAP_BY_INDEX)) {
/*
* We do not really know the valid range of scancodes
* so let's respond with KEY_RESERVED to anything we
* do not have mapping for [yet].
*/
ke->index = index;
ke->keycode = KEY_RESERVED;
} else {
retval = -EINVAL;
goto out;
}
retval = 0;
out:
spin_unlock_irqrestore(&rc_map->lock, flags);
return retval;
}
/**
* rc_g_keycode_from_table() - gets the keycode that corresponds to a scancode
* @dev: the struct rc_dev descriptor of the device
* @scancode: the scancode to look for
*
* This routine is used by drivers which need to convert a scancode to a
* keycode. Normally it should not be used since drivers should have no
* interest in keycodes.
*
* return: the corresponding keycode, or KEY_RESERVED
*/
u32 rc_g_keycode_from_table(struct rc_dev *dev, u64 scancode)
{
struct rc_map *rc_map = &dev->rc_map;
unsigned int keycode;
unsigned int index;
unsigned long flags;
spin_lock_irqsave(&rc_map->lock, flags);
index = ir_lookup_by_scancode(rc_map, scancode);
keycode = index < rc_map->len ?
rc_map->scan[index].keycode : KEY_RESERVED;
spin_unlock_irqrestore(&rc_map->lock, flags);
if (keycode != KEY_RESERVED)
dev_dbg(&dev->dev, "%s: scancode 0x%04llx keycode 0x%02x\n",
dev->device_name, scancode, keycode);
return keycode;
}
EXPORT_SYMBOL_GPL(rc_g_keycode_from_table);
/**
* ir_do_keyup() - internal function to signal the release of a keypress
* @dev: the struct rc_dev descriptor of the device
* @sync: whether or not to call input_sync
*
* This function is used internally to release a keypress, it must be
* called with keylock held.
*/
static void ir_do_keyup(struct rc_dev *dev, bool sync)
{
if (!dev->keypressed)
return;
dev_dbg(&dev->dev, "keyup key 0x%04x\n", dev->last_keycode);
del_timer(&dev->timer_repeat);
input_report_key(dev->input_dev, dev->last_keycode, 0);
led_trigger_event(led_feedback, LED_OFF);
if (sync)
input_sync(dev->input_dev);
dev->keypressed = false;
}
/**
* rc_keyup() - signals the release of a keypress
* @dev: the struct rc_dev descriptor of the device
*
* This routine is used to signal that a key has been released on the
* remote control.
*/
void rc_keyup(struct rc_dev *dev)
{
unsigned long flags;
spin_lock_irqsave(&dev->keylock, flags);
ir_do_keyup(dev, true);
spin_unlock_irqrestore(&dev->keylock, flags);
}
EXPORT_SYMBOL_GPL(rc_keyup);
/**
* ir_timer_keyup() - generates a keyup event after a timeout
*
* @t: a pointer to the struct timer_list
*
* This routine will generate a keyup event some time after a keydown event
* is generated when no further activity has been detected.
*/
static void ir_timer_keyup(struct timer_list *t)
{
struct rc_dev *dev = from_timer(dev, t, timer_keyup);
unsigned long flags;
/*
* ir->keyup_jiffies is used to prevent a race condition if a
* hardware interrupt occurs at this point and the keyup timer
* event is moved further into the future as a result.
*
* The timer will then be reactivated and this function called
* again in the future. We need to exit gracefully in that case
* to allow the input subsystem to do its auto-repeat magic or
* a keyup event might follow immediately after the keydown.
*/
spin_lock_irqsave(&dev->keylock, flags);
if (time_is_before_eq_jiffies(dev->keyup_jiffies))
ir_do_keyup(dev, true);
spin_unlock_irqrestore(&dev->keylock, flags);
}
/**
* ir_timer_repeat() - generates a repeat event after a timeout
*
* @t: a pointer to the struct timer_list
*
* This routine will generate a soft repeat event every REP_PERIOD
* milliseconds.
*/
static void ir_timer_repeat(struct timer_list *t)
{
struct rc_dev *dev = from_timer(dev, t, timer_repeat);
struct input_dev *input = dev->input_dev;
unsigned long flags;
spin_lock_irqsave(&dev->keylock, flags);
if (dev->keypressed) {
input_event(input, EV_KEY, dev->last_keycode, 2);
input_sync(input);
if (input->rep[REP_PERIOD])
mod_timer(&dev->timer_repeat, jiffies +
msecs_to_jiffies(input->rep[REP_PERIOD]));
}
spin_unlock_irqrestore(&dev->keylock, flags);
}
static unsigned int repeat_period(int protocol)
{
if (protocol >= ARRAY_SIZE(protocols))
return 100;
return protocols[protocol].repeat_period;
}
/**
* rc_repeat() - signals that a key is still pressed
* @dev: the struct rc_dev descriptor of the device
*
* This routine is used by IR decoders when a repeat message which does
* not include the necessary bits to reproduce the scancode has been
* received.
*/
void rc_repeat(struct rc_dev *dev)
{
unsigned long flags;
unsigned int timeout = usecs_to_jiffies(dev->timeout) +
msecs_to_jiffies(repeat_period(dev->last_protocol));
struct lirc_scancode sc = {
.scancode = dev->last_scancode, .rc_proto = dev->last_protocol,
.keycode = dev->keypressed ? dev->last_keycode : KEY_RESERVED,
.flags = LIRC_SCANCODE_FLAG_REPEAT |
(dev->last_toggle ? LIRC_SCANCODE_FLAG_TOGGLE : 0)
};
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
lirc_scancode_event(dev, &sc);
spin_lock_irqsave(&dev->keylock, flags);
if (dev->last_scancode <= U32_MAX) {
input_event(dev->input_dev, EV_MSC, MSC_SCAN,
dev->last_scancode);
input_sync(dev->input_dev);
}
if (dev->keypressed) {
dev->keyup_jiffies = jiffies + timeout;
mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
}
spin_unlock_irqrestore(&dev->keylock, flags);
}
EXPORT_SYMBOL_GPL(rc_repeat);
/**
* ir_do_keydown() - internal function to process a keypress
* @dev: the struct rc_dev descriptor of the device
* @protocol: the protocol of the keypress
* @scancode: the scancode of the keypress
* @keycode: the keycode of the keypress
* @toggle: the toggle value of the keypress
*
* This function is used internally to register a keypress, it must be
* called with keylock held.
*/
static void ir_do_keydown(struct rc_dev *dev, enum rc_proto protocol,
u64 scancode, u32 keycode, u8 toggle)
{
bool new_event = (!dev->keypressed ||
dev->last_protocol != protocol ||
dev->last_scancode != scancode ||
dev->last_toggle != toggle);
struct lirc_scancode sc = {
.scancode = scancode, .rc_proto = protocol,
.flags = (toggle ? LIRC_SCANCODE_FLAG_TOGGLE : 0) |
(!new_event ? LIRC_SCANCODE_FLAG_REPEAT : 0),
.keycode = keycode
};
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
lirc_scancode_event(dev, &sc);
if (new_event && dev->keypressed)
ir_do_keyup(dev, false);
if (scancode <= U32_MAX)
input_event(dev->input_dev, EV_MSC, MSC_SCAN, scancode);
dev->last_protocol = protocol;
dev->last_scancode = scancode;
dev->last_toggle = toggle;
dev->last_keycode = keycode;
if (new_event && keycode != KEY_RESERVED) {
/* Register a keypress */
dev->keypressed = true;
dev_dbg(&dev->dev, "%s: key down event, key 0x%04x, protocol 0x%04x, scancode 0x%08llx\n",
dev->device_name, keycode, protocol, scancode);
input_report_key(dev->input_dev, keycode, 1);
led_trigger_event(led_feedback, LED_FULL);
}
/*
* For CEC, start sending repeat messages as soon as the first
* repeated message is sent, as long as REP_DELAY = 0 and REP_PERIOD
* is non-zero. Otherwise, the input layer will generate repeat
* messages.
*/
if (!new_event && keycode != KEY_RESERVED &&
dev->allowed_protocols == RC_PROTO_BIT_CEC &&
!timer_pending(&dev->timer_repeat) &&
dev->input_dev->rep[REP_PERIOD] &&
!dev->input_dev->rep[REP_DELAY]) {
input_event(dev->input_dev, EV_KEY, keycode, 2);
mod_timer(&dev->timer_repeat, jiffies +
msecs_to_jiffies(dev->input_dev->rep[REP_PERIOD]));
}
input_sync(dev->input_dev);
}
/**
* rc_keydown() - generates input event for a key press
* @dev: the struct rc_dev descriptor of the device
* @protocol: the protocol for the keypress
* @scancode: the scancode for the keypress
* @toggle: the toggle value (protocol dependent, if the protocol doesn't
* support toggle values, this should be set to zero)
*
* This routine is used to signal that a key has been pressed on the
* remote control.
*/
void rc_keydown(struct rc_dev *dev, enum rc_proto protocol, u64 scancode,
u8 toggle)
{
unsigned long flags;
u32 keycode = rc_g_keycode_from_table(dev, scancode);
spin_lock_irqsave(&dev->keylock, flags);
ir_do_keydown(dev, protocol, scancode, keycode, toggle);
if (dev->keypressed) {
dev->keyup_jiffies = jiffies + usecs_to_jiffies(dev->timeout) +
msecs_to_jiffies(repeat_period(protocol));
mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
}
spin_unlock_irqrestore(&dev->keylock, flags);
}
EXPORT_SYMBOL_GPL(rc_keydown);
/**
* rc_keydown_notimeout() - generates input event for a key press without
* an automatic keyup event at a later time
* @dev: the struct rc_dev descriptor of the device
* @protocol: the protocol for the keypress
* @scancode: the scancode for the keypress
* @toggle: the toggle value (protocol dependent, if the protocol doesn't
* support toggle values, this should be set to zero)
*
* This routine is used to signal that a key has been pressed on the
* remote control. The driver must manually call rc_keyup() at a later stage.
*/
void rc_keydown_notimeout(struct rc_dev *dev, enum rc_proto protocol,
u64 scancode, u8 toggle)
{
unsigned long flags;
u32 keycode = rc_g_keycode_from_table(dev, scancode);
spin_lock_irqsave(&dev->keylock, flags);
ir_do_keydown(dev, protocol, scancode, keycode, toggle);
spin_unlock_irqrestore(&dev->keylock, flags);
}
EXPORT_SYMBOL_GPL(rc_keydown_notimeout);
/**
* rc_validate_scancode() - checks that a scancode is valid for a protocol.
* For nec, it should do the opposite of ir_nec_bytes_to_scancode()
* @proto: protocol
* @scancode: scancode
*/
bool rc_validate_scancode(enum rc_proto proto, u32 scancode)
{
switch (proto) {
/*
* NECX has a 16-bit address; if the lower 8 bits match the upper
* 8 bits inverted, then the address would match regular nec.
*/
case RC_PROTO_NECX:
if ((((scancode >> 16) ^ ~(scancode >> 8)) & 0xff) == 0)
return false;
break;
/*
* NEC32 has a 16 bit address and 16 bit command. If the lower 8 bits
* of the command match the upper 8 bits inverted, then it would
* be either NEC or NECX.
*/
case RC_PROTO_NEC32:
if ((((scancode >> 8) ^ ~scancode) & 0xff) == 0)
return false;
break;
/*
* If the customer code (top 32-bit) is 0x800f, it is MCE else it
* is regular mode-6a 32 bit
*/
case RC_PROTO_RC6_MCE:
if ((scancode & 0xffff0000) != 0x800f0000)
return false;
break;
case RC_PROTO_RC6_6A_32:
if ((scancode & 0xffff0000) == 0x800f0000)
return false;
break;
default:
break;
}
return true;
}
/**
* rc_validate_filter() - checks that the scancode and mask are valid and
* provides sensible defaults
* @dev: the struct rc_dev descriptor of the device
* @filter: the scancode and mask
*
* return: 0 or -EINVAL if the filter is not valid
*/
static int rc_validate_filter(struct rc_dev *dev,
struct rc_scancode_filter *filter)
{
u32 mask, s = filter->data;
enum rc_proto protocol = dev->wakeup_protocol;
if (protocol >= ARRAY_SIZE(protocols))
return -EINVAL;
mask = protocols[protocol].scancode_bits;
if (!rc_validate_scancode(protocol, s))
return -EINVAL;
filter->data &= mask;
filter->mask &= mask;
/*
* If we have to raw encode the IR for wakeup, we cannot have a mask
*/
if (dev->encode_wakeup && filter->mask != 0 && filter->mask != mask)
return -EINVAL;
return 0;
}
int rc_open(struct rc_dev *rdev)
{
int rval = 0;
if (!rdev)
return -EINVAL;
mutex_lock(&rdev->lock);
if (!rdev->registered) {
rval = -ENODEV;
} else {
if (!rdev->users++ && rdev->open)
rval = rdev->open(rdev);
if (rval)
rdev->users--;
}
mutex_unlock(&rdev->lock);
return rval;
}
static int ir_open(struct input_dev *idev)
{
struct rc_dev *rdev = input_get_drvdata(idev);
return rc_open(rdev);
}
void rc_close(struct rc_dev *rdev)
{
if (rdev) {
mutex_lock(&rdev->lock);
if (!--rdev->users && rdev->close && rdev->registered)
rdev->close(rdev);
mutex_unlock(&rdev->lock);
}
}
static void ir_close(struct input_dev *idev)
{
struct rc_dev *rdev = input_get_drvdata(idev);
rc_close(rdev);
}
/* class for /sys/class/rc */
static char *rc_devnode(const struct device *dev, umode_t *mode)
{
return kasprintf(GFP_KERNEL, "rc/%s", dev_name(dev));
}
static struct class rc_class = {
.name = "rc",
.devnode = rc_devnode,
};
/*
* These are the protocol textual descriptions that are
* used by the sysfs protocols file. Note that the order
* of the entries is relevant.
*/
static const struct {
u64 type;
const char *name;
const char *module_name;
} proto_names[] = {
{ RC_PROTO_BIT_NONE, "none", NULL },
{ RC_PROTO_BIT_OTHER, "other", NULL },
{ RC_PROTO_BIT_UNKNOWN, "unknown", NULL },
{ RC_PROTO_BIT_RC5 |
RC_PROTO_BIT_RC5X_20, "rc-5", "ir-rc5-decoder" },
{ RC_PROTO_BIT_NEC |
RC_PROTO_BIT_NECX |
RC_PROTO_BIT_NEC32, "nec", "ir-nec-decoder" },
{ RC_PROTO_BIT_RC6_0 |
RC_PROTO_BIT_RC6_6A_20 |
RC_PROTO_BIT_RC6_6A_24 |
RC_PROTO_BIT_RC6_6A_32 |
RC_PROTO_BIT_RC6_MCE, "rc-6", "ir-rc6-decoder" },
{ RC_PROTO_BIT_JVC, "jvc", "ir-jvc-decoder" },
{ RC_PROTO_BIT_SONY12 |
RC_PROTO_BIT_SONY15 |
RC_PROTO_BIT_SONY20, "sony", "ir-sony-decoder" },
{ RC_PROTO_BIT_RC5_SZ, "rc-5-sz", "ir-rc5-decoder" },
{ RC_PROTO_BIT_SANYO, "sanyo", "ir-sanyo-decoder" },
{ RC_PROTO_BIT_SHARP, "sharp", "ir-sharp-decoder" },
{ RC_PROTO_BIT_MCIR2_KBD |
RC_PROTO_BIT_MCIR2_MSE, "mce_kbd", "ir-mce_kbd-decoder" },
{ RC_PROTO_BIT_XMP, "xmp", "ir-xmp-decoder" },
{ RC_PROTO_BIT_CEC, "cec", NULL },
{ RC_PROTO_BIT_IMON, "imon", "ir-imon-decoder" },
{ RC_PROTO_BIT_RCMM12 |
RC_PROTO_BIT_RCMM24 |
RC_PROTO_BIT_RCMM32, "rc-mm", "ir-rcmm-decoder" },
{ RC_PROTO_BIT_XBOX_DVD, "xbox-dvd", NULL },
};
/**
* struct rc_filter_attribute - Device attribute relating to a filter type.
* @attr: Device attribute.
* @type: Filter type.
* @mask: false for filter value, true for filter mask.
*/
struct rc_filter_attribute {
struct device_attribute attr;
enum rc_filter_type type;
bool mask;
};
#define to_rc_filter_attr(a) container_of(a, struct rc_filter_attribute, attr)
#define RC_FILTER_ATTR(_name, _mode, _show, _store, _type, _mask) \
struct rc_filter_attribute dev_attr_##_name = { \
.attr = __ATTR(_name, _mode, _show, _store), \
.type = (_type), \
.mask = (_mask), \
}
/**
* show_protocols() - shows the current IR protocol(s)
* @device: the device descriptor
* @mattr: the device attribute struct
* @buf: a pointer to the output buffer
*
* This routine is a callback routine for input read the IR protocol type(s).
* it is triggered by reading /sys/class/rc/rc?/protocols.
* It returns the protocol names of supported protocols.
* Enabled protocols are printed in brackets.
*
* dev->lock is taken to guard against races between
* store_protocols and show_protocols.
*/
static ssize_t show_protocols(struct device *device,
struct device_attribute *mattr, char *buf)
{
struct rc_dev *dev = to_rc_dev(device);
u64 allowed, enabled;
char *tmp = buf;
int i;
mutex_lock(&dev->lock);
enabled = dev->enabled_protocols;
allowed = dev->allowed_protocols;
if (dev->raw && !allowed)
allowed = ir_raw_get_allowed_protocols();
mutex_unlock(&dev->lock);
dev_dbg(&dev->dev, "%s: allowed - 0x%llx, enabled - 0x%llx\n",
__func__, (long long)allowed, (long long)enabled);
for (i = 0; i < ARRAY_SIZE(proto_names); i++) {
if (allowed & enabled & proto_names[i].type)
tmp += sprintf(tmp, "[%s] ", proto_names[i].name);
else if (allowed & proto_names[i].type)
tmp += sprintf(tmp, "%s ", proto_names[i].name);
if (allowed & proto_names[i].type)
allowed &= ~proto_names[i].type;
}
#ifdef CONFIG_LIRC
if (dev->driver_type == RC_DRIVER_IR_RAW)
tmp += sprintf(tmp, "[lirc] ");
#endif
if (tmp != buf)
tmp--;
*tmp = '\n';
return tmp + 1 - buf;
}
/**
* parse_protocol_change() - parses a protocol change request
* @dev: rc_dev device
* @protocols: pointer to the bitmask of current protocols
* @buf: pointer to the buffer with a list of changes
*
* Writing "+proto" will add a protocol to the protocol mask.
* Writing "-proto" will remove a protocol from protocol mask.
* Writing "proto" will enable only "proto".
* Writing "none" will disable all protocols.
* Returns the number of changes performed or a negative error code.
*/
static int parse_protocol_change(struct rc_dev *dev, u64 *protocols,
const char *buf)
{
const char *tmp;
unsigned count = 0;
bool enable, disable;
u64 mask;
int i;
while ((tmp = strsep((char **)&buf, " \n")) != NULL) {
if (!*tmp)
break;
if (*tmp == '+') {
enable = true;
disable = false;
tmp++;
} else if (*tmp == '-') {
enable = false;
disable = true;
tmp++;
} else {
enable = false;
disable = false;
}
for (i = 0; i < ARRAY_SIZE(proto_names); i++) {
if (!strcasecmp(tmp, proto_names[i].name)) {
mask = proto_names[i].type;
break;
}
}
if (i == ARRAY_SIZE(proto_names)) {
if (!strcasecmp(tmp, "lirc"))
mask = 0;
else {
dev_dbg(&dev->dev, "Unknown protocol: '%s'\n",
tmp);
return -EINVAL;
}
}
count++;
if (enable)
*protocols |= mask;
else if (disable)
*protocols &= ~mask;
else
*protocols = mask;
}
if (!count) {
dev_dbg(&dev->dev, "Protocol not specified\n");
return -EINVAL;
}
return count;
}
void ir_raw_load_modules(u64 *protocols)
{
u64 available;
int i, ret;
for (i = 0; i < ARRAY_SIZE(proto_names); i++) {
if (proto_names[i].type == RC_PROTO_BIT_NONE ||
proto_names[i].type & (RC_PROTO_BIT_OTHER |
RC_PROTO_BIT_UNKNOWN))
continue;
available = ir_raw_get_allowed_protocols();
if (!(*protocols & proto_names[i].type & ~available))
continue;
if (!proto_names[i].module_name) {
pr_err("Can't enable IR protocol %s\n",
proto_names[i].name);
*protocols &= ~proto_names[i].type;
continue;
}
ret = request_module("%s", proto_names[i].module_name);
if (ret < 0) {
pr_err("Couldn't load IR protocol module %s\n",
proto_names[i].module_name);
*protocols &= ~proto_names[i].type;
continue;
}
msleep(20);
available = ir_raw_get_allowed_protocols();
if (!(*protocols & proto_names[i].type & ~available))
continue;
pr_err("Loaded IR protocol module %s, but protocol %s still not available\n",
proto_names[i].module_name,
proto_names[i].name);
*protocols &= ~proto_names[i].type;
}
}
/**
* store_protocols() - changes the current/wakeup IR protocol(s)
* @device: the device descriptor
* @mattr: the device attribute struct
* @buf: a pointer to the input buffer
* @len: length of the input buffer
*
* This routine is for changing the IR protocol type.
* It is triggered by writing to /sys/class/rc/rc?/[wakeup_]protocols.
* See parse_protocol_change() for the valid commands.
* Returns @len on success or a negative error code.
*
* dev->lock is taken to guard against races between
* store_protocols and show_protocols.
*/
static ssize_t store_protocols(struct device *device,
struct device_attribute *mattr,
const char *buf, size_t len)
{
struct rc_dev *dev = to_rc_dev(device);
u64 *current_protocols;
struct rc_scancode_filter *filter;
u64 old_protocols, new_protocols;
ssize_t rc;
dev_dbg(&dev->dev, "Normal protocol change requested\n");
current_protocols = &dev->enabled_protocols;
filter = &dev->scancode_filter;
if (!dev->change_protocol) {
dev_dbg(&dev->dev, "Protocol switching not supported\n");
return -EINVAL;
}
mutex_lock(&dev->lock);
if (!dev->registered) {
mutex_unlock(&dev->lock);
return -ENODEV;
}
old_protocols = *current_protocols;
new_protocols = old_protocols;
rc = parse_protocol_change(dev, &new_protocols, buf);
if (rc < 0)
goto out;
if (dev->driver_type == RC_DRIVER_IR_RAW)
ir_raw_load_modules(&new_protocols);
rc = dev->change_protocol(dev, &new_protocols);
if (rc < 0) {
dev_dbg(&dev->dev, "Error setting protocols to 0x%llx\n",
(long long)new_protocols);
goto out;
}
if (new_protocols != old_protocols) {
*current_protocols = new_protocols;
dev_dbg(&dev->dev, "Protocols changed to 0x%llx\n",
(long long)new_protocols);
}
/*
* If a protocol change was attempted the filter may need updating, even
* if the actual protocol mask hasn't changed (since the driver may have
* cleared the filter).
* Try setting the same filter with the new protocol (if any).
* Fall back to clearing the filter.
*/
if (dev->s_filter && filter->mask) {
if (new_protocols)
rc = dev->s_filter(dev, filter);
else
rc = -1;
if (rc < 0) {
filter->data = 0;
filter->mask = 0;
dev->s_filter(dev, filter);
}
}
rc = len;
out:
mutex_unlock(&dev->lock);
return rc;
}
/**
* show_filter() - shows the current scancode filter value or mask
* @device: the device descriptor
* @attr: the device attribute struct
* @buf: a pointer to the output buffer
*
* This routine is a callback routine to read a scancode filter value or mask.
* It is triggered by reading /sys/class/rc/rc?/[wakeup_]filter[_mask].
* It prints the current scancode filter value or mask of the appropriate filter
* type in hexadecimal into @buf and returns the size of the buffer.
*
* Bits of the filter value corresponding to set bits in the filter mask are
* compared against input scancodes and non-matching scancodes are discarded.
*
* dev->lock is taken to guard against races between
* store_filter and show_filter.
*/
static ssize_t show_filter(struct device *device,
struct device_attribute *attr,
char *buf)
{
struct rc_dev *dev = to_rc_dev(device);
struct rc_filter_attribute *fattr = to_rc_filter_attr(attr);
struct rc_scancode_filter *filter;
u32 val;
mutex_lock(&dev->lock);
if (fattr->type == RC_FILTER_NORMAL)
filter = &dev->scancode_filter;
else
filter = &dev->scancode_wakeup_filter;
if (fattr->mask)
val = filter->mask;
else
val = filter->data;
mutex_unlock(&dev->lock);
return sprintf(buf, "%#x\n", val);
}
/**
* store_filter() - changes the scancode filter value
* @device: the device descriptor
* @attr: the device attribute struct
* @buf: a pointer to the input buffer
* @len: length of the input buffer
*
* This routine is for changing a scancode filter value or mask.
* It is triggered by writing to /sys/class/rc/rc?/[wakeup_]filter[_mask].
* Returns -EINVAL if an invalid filter value for the current protocol was
* specified or if scancode filtering is not supported by the driver, otherwise
* returns @len.
*
* Bits of the filter value corresponding to set bits in the filter mask are
* compared against input scancodes and non-matching scancodes are discarded.
*
* dev->lock is taken to guard against races between
* store_filter and show_filter.
*/
static ssize_t store_filter(struct device *device,
struct device_attribute *attr,
const char *buf, size_t len)
{
struct rc_dev *dev = to_rc_dev(device);
struct rc_filter_attribute *fattr = to_rc_filter_attr(attr);
struct rc_scancode_filter new_filter, *filter;
int ret;
unsigned long val;
int (*set_filter)(struct rc_dev *dev, struct rc_scancode_filter *filter);
ret = kstrtoul(buf, 0, &val);
if (ret < 0)
return ret;
if (fattr->type == RC_FILTER_NORMAL) {
set_filter = dev->s_filter;
filter = &dev->scancode_filter;
} else {
set_filter = dev->s_wakeup_filter;
filter = &dev->scancode_wakeup_filter;
}
if (!set_filter)
return -EINVAL;
mutex_lock(&dev->lock);
if (!dev->registered) {
mutex_unlock(&dev->lock);
return -ENODEV;
}
new_filter = *filter;
if (fattr->mask)
new_filter.mask = val;
else
new_filter.data = val;
if (fattr->type == RC_FILTER_WAKEUP) {
/*
* Refuse to set a filter unless a protocol is enabled
* and the filter is valid for that protocol
*/
if (dev->wakeup_protocol != RC_PROTO_UNKNOWN)
ret = rc_validate_filter(dev, &new_filter);
else
ret = -EINVAL;
if (ret != 0)
goto unlock;
}
if (fattr->type == RC_FILTER_NORMAL && !dev->enabled_protocols &&
val) {
/* refuse to set a filter unless a protocol is enabled */
ret = -EINVAL;
goto unlock;
}
ret = set_filter(dev, &new_filter);
if (ret < 0)
goto unlock;
*filter = new_filter;
unlock:
mutex_unlock(&dev->lock);
return (ret < 0) ? ret : len;
}
/**
* show_wakeup_protocols() - shows the wakeup IR protocol
* @device: the device descriptor
* @mattr: the device attribute struct
* @buf: a pointer to the output buffer
*
* This routine is a callback routine for input read the IR protocol type(s).
* it is triggered by reading /sys/class/rc/rc?/wakeup_protocols.
* It returns the protocol names of supported protocols.
* The enabled protocols are printed in brackets.
*
* dev->lock is taken to guard against races between
* store_wakeup_protocols and show_wakeup_protocols.
*/
static ssize_t show_wakeup_protocols(struct device *device,
struct device_attribute *mattr,
char *buf)
{
struct rc_dev *dev = to_rc_dev(device);
u64 allowed;
enum rc_proto enabled;
char *tmp = buf;
int i;
mutex_lock(&dev->lock);
allowed = dev->allowed_wakeup_protocols;
enabled = dev->wakeup_protocol;
mutex_unlock(&dev->lock);
dev_dbg(&dev->dev, "%s: allowed - 0x%llx, enabled - %d\n",
__func__, (long long)allowed, enabled);
for (i = 0; i < ARRAY_SIZE(protocols); i++) {
if (allowed & (1ULL << i)) {
if (i == enabled)
tmp += sprintf(tmp, "[%s] ", protocols[i].name);
else
tmp += sprintf(tmp, "%s ", protocols[i].name);
}
}
if (tmp != buf)
tmp--;
*tmp = '\n';
return tmp + 1 - buf;
}
/**
* store_wakeup_protocols() - changes the wakeup IR protocol(s)
* @device: the device descriptor
* @mattr: the device attribute struct
* @buf: a pointer to the input buffer
* @len: length of the input buffer
*
* This routine is for changing the IR protocol type.
* It is triggered by writing to /sys/class/rc/rc?/wakeup_protocols.
* Returns @len on success or a negative error code.
*
* dev->lock is taken to guard against races between
* store_wakeup_protocols and show_wakeup_protocols.
*/
static ssize_t store_wakeup_protocols(struct device *device,
struct device_attribute *mattr,
const char *buf, size_t len)
{
struct rc_dev *dev = to_rc_dev(device);
enum rc_proto protocol = RC_PROTO_UNKNOWN;
ssize_t rc;
u64 allowed;
int i;
mutex_lock(&dev->lock);
if (!dev->registered) {
mutex_unlock(&dev->lock);
return -ENODEV;
}
allowed = dev->allowed_wakeup_protocols;
if (!sysfs_streq(buf, "none")) {
for (i = 0; i < ARRAY_SIZE(protocols); i++) {
if ((allowed & (1ULL << i)) &&
sysfs_streq(buf, protocols[i].name)) {
protocol = i;
break;
}
}
if (i == ARRAY_SIZE(protocols)) {
rc = -EINVAL;
goto out;
}
if (dev->encode_wakeup) {
u64 mask = 1ULL << protocol;
ir_raw_load_modules(&mask);
if (!mask) {
rc = -EINVAL;
goto out;
}
}
}
if (dev->wakeup_protocol != protocol) {
dev->wakeup_protocol = protocol;
dev_dbg(&dev->dev, "Wakeup protocol changed to %d\n", protocol);
if (protocol == RC_PROTO_RC6_MCE)
dev->scancode_wakeup_filter.data = 0x800f0000;
else
dev->scancode_wakeup_filter.data = 0;
dev->scancode_wakeup_filter.mask = 0;
rc = dev->s_wakeup_filter(dev, &dev->scancode_wakeup_filter);
if (rc == 0)
rc = len;
} else {
rc = len;
}
out:
mutex_unlock(&dev->lock);
return rc;
}
static void rc_dev_release(struct device *device)
{
struct rc_dev *dev = to_rc_dev(device);
kfree(dev);
}
static int rc_dev_uevent(const struct device *device, struct kobj_uevent_env *env)
{
struct rc_dev *dev = to_rc_dev(device);
int ret = 0;
mutex_lock(&dev->lock);
if (!dev->registered)
ret = -ENODEV;
if (ret == 0 && dev->rc_map.name) ret = add_uevent_var(env, "NAME=%s", dev->rc_map.name); if (ret == 0 && dev->driver_name) ret = add_uevent_var(env, "DRV_NAME=%s", dev->driver_name); if (ret == 0 && dev->device_name) ret = add_uevent_var(env, "DEV_NAME=%s", dev->device_name); mutex_unlock(&dev->lock);
return ret;
}
/*
* Static device attribute struct with the sysfs attributes for IR's
*/
static struct device_attribute dev_attr_ro_protocols =
__ATTR(protocols, 0444, show_protocols, NULL);
static struct device_attribute dev_attr_rw_protocols =
__ATTR(protocols, 0644, show_protocols, store_protocols);
static DEVICE_ATTR(wakeup_protocols, 0644, show_wakeup_protocols,
store_wakeup_protocols);
static RC_FILTER_ATTR(filter, S_IRUGO|S_IWUSR,
show_filter, store_filter, RC_FILTER_NORMAL, false);
static RC_FILTER_ATTR(filter_mask, S_IRUGO|S_IWUSR,
show_filter, store_filter, RC_FILTER_NORMAL, true);
static RC_FILTER_ATTR(wakeup_filter, S_IRUGO|S_IWUSR,
show_filter, store_filter, RC_FILTER_WAKEUP, false);
static RC_FILTER_ATTR(wakeup_filter_mask, S_IRUGO|S_IWUSR,
show_filter, store_filter, RC_FILTER_WAKEUP, true);
static struct attribute *rc_dev_rw_protocol_attrs[] = {
&dev_attr_rw_protocols.attr,
NULL,
};
static const struct attribute_group rc_dev_rw_protocol_attr_grp = {
.attrs = rc_dev_rw_protocol_attrs,
};
static struct attribute *rc_dev_ro_protocol_attrs[] = {
&dev_attr_ro_protocols.attr,
NULL,
};
static const struct attribute_group rc_dev_ro_protocol_attr_grp = {
.attrs = rc_dev_ro_protocol_attrs,
};
static struct attribute *rc_dev_filter_attrs[] = {
&dev_attr_filter.attr.attr,
&dev_attr_filter_mask.attr.attr,
NULL,
};
static const struct attribute_group rc_dev_filter_attr_grp = {
.attrs = rc_dev_filter_attrs,
};
static struct attribute *rc_dev_wakeup_filter_attrs[] = {
&dev_attr_wakeup_filter.attr.attr,
&dev_attr_wakeup_filter_mask.attr.attr,
&dev_attr_wakeup_protocols.attr,
NULL,
};
static const struct attribute_group rc_dev_wakeup_filter_attr_grp = {
.attrs = rc_dev_wakeup_filter_attrs,
};
static const struct device_type rc_dev_type = {
.release = rc_dev_release,
.uevent = rc_dev_uevent,
};
struct rc_dev *rc_allocate_device(enum rc_driver_type type)
{
struct rc_dev *dev;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return NULL;
if (type != RC_DRIVER_IR_RAW_TX) {
dev->input_dev = input_allocate_device();
if (!dev->input_dev) {
kfree(dev);
return NULL;
}
dev->input_dev->getkeycode = ir_getkeycode;
dev->input_dev->setkeycode = ir_setkeycode;
input_set_drvdata(dev->input_dev, dev);
dev->timeout = IR_DEFAULT_TIMEOUT;
timer_setup(&dev->timer_keyup, ir_timer_keyup, 0);
timer_setup(&dev->timer_repeat, ir_timer_repeat, 0);
spin_lock_init(&dev->rc_map.lock);
spin_lock_init(&dev->keylock);
}
mutex_init(&dev->lock);
dev->dev.type = &rc_dev_type;
dev->dev.class = &rc_class;
device_initialize(&dev->dev);
dev->driver_type = type;
__module_get(THIS_MODULE);
return dev;
}
EXPORT_SYMBOL_GPL(rc_allocate_device);
void rc_free_device(struct rc_dev *dev)
{
if (!dev)
return;
input_free_device(dev->input_dev);
put_device(&dev->dev);
/* kfree(dev) will be called by the callback function
rc_dev_release() */
module_put(THIS_MODULE);
}
EXPORT_SYMBOL_GPL(rc_free_device);
static void devm_rc_alloc_release(struct device *dev, void *res)
{
rc_free_device(*(struct rc_dev **)res);
}
struct rc_dev *devm_rc_allocate_device(struct device *dev,
enum rc_driver_type type)
{
struct rc_dev **dr, *rc;
dr = devres_alloc(devm_rc_alloc_release, sizeof(*dr), GFP_KERNEL);
if (!dr)
return NULL;
rc = rc_allocate_device(type);
if (!rc) {
devres_free(dr);
return NULL;
}
rc->dev.parent = dev;
rc->managed_alloc = true;
*dr = rc;
devres_add(dev, dr);
return rc;
}
EXPORT_SYMBOL_GPL(devm_rc_allocate_device);
static int rc_prepare_rx_device(struct rc_dev *dev)
{
int rc;
struct rc_map *rc_map;
u64 rc_proto;
if (!dev->map_name)
return -EINVAL;
rc_map = rc_map_get(dev->map_name);
if (!rc_map)
rc_map = rc_map_get(RC_MAP_EMPTY);
if (!rc_map || !rc_map->scan || rc_map->size == 0)
return -EINVAL;
rc = ir_setkeytable(dev, rc_map);
if (rc)
return rc;
rc_proto = BIT_ULL(rc_map->rc_proto);
if (dev->driver_type == RC_DRIVER_SCANCODE && !dev->change_protocol)
dev->enabled_protocols = dev->allowed_protocols;
if (dev->driver_type == RC_DRIVER_IR_RAW)
ir_raw_load_modules(&rc_proto);
if (dev->change_protocol) {
rc = dev->change_protocol(dev, &rc_proto);
if (rc < 0)
goto out_table;
dev->enabled_protocols = rc_proto;
}
/* Keyboard events */
set_bit(EV_KEY, dev->input_dev->evbit);
set_bit(EV_REP, dev->input_dev->evbit);
set_bit(EV_MSC, dev->input_dev->evbit);
set_bit(MSC_SCAN, dev->input_dev->mscbit);
/* Pointer/mouse events */
set_bit(INPUT_PROP_POINTING_STICK, dev->input_dev->propbit);
set_bit(EV_REL, dev->input_dev->evbit);
set_bit(REL_X, dev->input_dev->relbit);
set_bit(REL_Y, dev->input_dev->relbit);
if (dev->open)
dev->input_dev->open = ir_open;
if (dev->close)
dev->input_dev->close = ir_close;
dev->input_dev->dev.parent = &dev->dev;
memcpy(&dev->input_dev->id, &dev->input_id, sizeof(dev->input_id));
dev->input_dev->phys = dev->input_phys;
dev->input_dev->name = dev->device_name;
return 0;
out_table:
ir_free_table(&dev->rc_map);
return rc;
}
static int rc_setup_rx_device(struct rc_dev *dev)
{
int rc;
/* rc_open will be called here */
rc = input_register_device(dev->input_dev);
if (rc)
return rc;
/*
* Default delay of 250ms is too short for some protocols, especially
* since the timeout is currently set to 250ms. Increase it to 500ms,
* to avoid wrong repetition of the keycodes. Note that this must be
* set after the call to input_register_device().
*/
if (dev->allowed_protocols == RC_PROTO_BIT_CEC)
dev->input_dev->rep[REP_DELAY] = 0;
else
dev->input_dev->rep[REP_DELAY] = 500;
/*
* As a repeat event on protocols like RC-5 and NEC take as long as
* 110/114ms, using 33ms as a repeat period is not the right thing
* to do.
*/
dev->input_dev->rep[REP_PERIOD] = 125;
return 0;
}
static void rc_free_rx_device(struct rc_dev *dev)
{
if (!dev)
return;
if (dev->input_dev) {
input_unregister_device(dev->input_dev);
dev->input_dev = NULL;
}
ir_free_table(&dev->rc_map);
}
int rc_register_device(struct rc_dev *dev)
{
const char *path;
int attr = 0;
int minor;
int rc;
if (!dev)
return -EINVAL;
minor = ida_alloc_max(&rc_ida, RC_DEV_MAX - 1, GFP_KERNEL);
if (minor < 0)
return minor;
dev->minor = minor;
dev_set_name(&dev->dev, "rc%u", dev->minor);
dev_set_drvdata(&dev->dev, dev);
dev->dev.groups = dev->sysfs_groups;
if (dev->driver_type == RC_DRIVER_SCANCODE && !dev->change_protocol)
dev->sysfs_groups[attr++] = &rc_dev_ro_protocol_attr_grp;
else if (dev->driver_type != RC_DRIVER_IR_RAW_TX)
dev->sysfs_groups[attr++] = &rc_dev_rw_protocol_attr_grp;
if (dev->s_filter)
dev->sysfs_groups[attr++] = &rc_dev_filter_attr_grp;
if (dev->s_wakeup_filter)
dev->sysfs_groups[attr++] = &rc_dev_wakeup_filter_attr_grp;
dev->sysfs_groups[attr++] = NULL;
if (dev->driver_type == RC_DRIVER_IR_RAW) {
rc = ir_raw_event_prepare(dev);
if (rc < 0)
goto out_minor;
}
if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
rc = rc_prepare_rx_device(dev);
if (rc)
goto out_raw;
}
dev->registered = true;
rc = device_add(&dev->dev);
if (rc)
goto out_rx_free;
path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL);
dev_info(&dev->dev, "%s as %s\n",
dev->device_name ?: "Unspecified device", path ?: "N/A");
kfree(path);
/*
* once the input device is registered in rc_setup_rx_device,
* userspace can open the input device and rc_open() will be called
* as a result. This results in driver code being allowed to submit
* keycodes with rc_keydown, so lirc must be registered first.
*/
if (dev->allowed_protocols != RC_PROTO_BIT_CEC) {
rc = lirc_register(dev);
if (rc < 0)
goto out_dev;
}
if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
rc = rc_setup_rx_device(dev);
if (rc)
goto out_lirc;
}
if (dev->driver_type == RC_DRIVER_IR_RAW) {
rc = ir_raw_event_register(dev);
if (rc < 0)
goto out_rx;
}
dev_dbg(&dev->dev, "Registered rc%u (driver: %s)\n", dev->minor,
dev->driver_name ? dev->driver_name : "unknown");
return 0;
out_rx:
rc_free_rx_device(dev);
out_lirc:
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
lirc_unregister(dev);
out_dev:
device_del(&dev->dev);
out_rx_free:
ir_free_table(&dev->rc_map);
out_raw:
ir_raw_event_free(dev);
out_minor:
ida_free(&rc_ida, minor);
return rc;
}
EXPORT_SYMBOL_GPL(rc_register_device);
static void devm_rc_release(struct device *dev, void *res)
{
rc_unregister_device(*(struct rc_dev **)res);
}
int devm_rc_register_device(struct device *parent, struct rc_dev *dev)
{
struct rc_dev **dr;
int ret;
dr = devres_alloc(devm_rc_release, sizeof(*dr), GFP_KERNEL);
if (!dr)
return -ENOMEM;
ret = rc_register_device(dev);
if (ret) {
devres_free(dr);
return ret;
}
*dr = dev;
devres_add(parent, dr);
return 0;
}
EXPORT_SYMBOL_GPL(devm_rc_register_device);
void rc_unregister_device(struct rc_dev *dev)
{
if (!dev)
return;
if (dev->driver_type == RC_DRIVER_IR_RAW) ir_raw_event_unregister(dev); del_timer_sync(&dev->timer_keyup);
del_timer_sync(&dev->timer_repeat);
mutex_lock(&dev->lock);
if (dev->users && dev->close) dev->close(dev); dev->registered = false;
mutex_unlock(&dev->lock);
rc_free_rx_device(dev);
/*
* lirc device should be freed with dev->registered = false, so
* that userspace polling will get notified.
*/
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
lirc_unregister(dev);
device_del(&dev->dev);
ida_free(&rc_ida, dev->minor);
if (!dev->managed_alloc)
rc_free_device(dev);
}
EXPORT_SYMBOL_GPL(rc_unregister_device);
/*
* Init/exit code for the module. Basically, creates/removes /sys/class/rc
*/
static int __init rc_core_init(void)
{
int rc = class_register(&rc_class);
if (rc) {
pr_err("rc_core: unable to register rc class\n");
return rc;
}
rc = lirc_dev_init();
if (rc) {
pr_err("rc_core: unable to init lirc\n");
class_unregister(&rc_class);
return rc;
}
led_trigger_register_simple("rc-feedback", &led_feedback);
rc_map_register(&empty_map);
#ifdef CONFIG_MEDIA_CEC_RC
rc_map_register(&cec_map);
#endif
return 0;
}
static void __exit rc_core_exit(void)
{
lirc_dev_exit();
class_unregister(&rc_class);
led_trigger_unregister_simple(led_feedback);
#ifdef CONFIG_MEDIA_CEC_RC
rc_map_unregister(&cec_map);
#endif
rc_map_unregister(&empty_map);
}
subsys_initcall(rc_core_init);
module_exit(rc_core_exit);
MODULE_AUTHOR("Mauro Carvalho Chehab");
MODULE_LICENSE("GPL v2");
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_CPUMASK_H
#define __LINUX_CPUMASK_H
/*
* Cpumasks provide a bitmap suitable for representing the
* set of CPUs in a system, one bit position per CPU number. In general,
* only nr_cpu_ids (<= NR_CPUS) bits are valid.
*/
#include <linux/cleanup.h>
#include <linux/kernel.h>
#include <linux/threads.h>
#include <linux/bitmap.h>
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/gfp_types.h>
#include <linux/numa.h>
/* Don't assign or return these: may not be this big! */
typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
/**
* cpumask_bits - get the bits in a cpumask
* @maskp: the struct cpumask *
*
* You should only assume nr_cpu_ids bits of this mask are valid. This is
* a macro so it's const-correct.
*/
#define cpumask_bits(maskp) ((maskp)->bits)
/**
* cpumask_pr_args - printf args to output a cpumask
* @maskp: cpumask to be printed
*
* Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
*/
#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
#define nr_cpu_ids ((unsigned int)NR_CPUS)
#else
extern unsigned int nr_cpu_ids;
#endif
static inline void set_nr_cpu_ids(unsigned int nr)
{
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
WARN_ON(nr != nr_cpu_ids);
#else
nr_cpu_ids = nr;
#endif
}
/*
* We have several different "preferred sizes" for the cpumask
* operations, depending on operation.
*
* For example, the bitmap scanning and operating operations have
* optimized routines that work for the single-word case, but only when
* the size is constant. So if NR_CPUS fits in one single word, we are
* better off using that small constant, in order to trigger the
* optimized bit finding. That is 'small_cpumask_size'.
*
* The clearing and copying operations will similarly perform better
* with a constant size, but we limit that size arbitrarily to four
* words. We call this 'large_cpumask_size'.
*
* Finally, some operations just want the exact limit, either because
* they set bits or just don't have any faster fixed-sized versions. We
* call this just 'nr_cpumask_bits'.
*
* Note that these optional constants are always guaranteed to be at
* least as big as 'nr_cpu_ids' itself is, and all our cpumask
* allocations are at least that size (see cpumask_size()). The
* optimization comes from being able to potentially use a compile-time
* constant instead of a run-time generated exact number of CPUs.
*/
#if NR_CPUS <= BITS_PER_LONG
#define small_cpumask_bits ((unsigned int)NR_CPUS)
#define large_cpumask_bits ((unsigned int)NR_CPUS)
#elif NR_CPUS <= 4*BITS_PER_LONG
#define small_cpumask_bits nr_cpu_ids
#define large_cpumask_bits ((unsigned int)NR_CPUS)
#else
#define small_cpumask_bits nr_cpu_ids
#define large_cpumask_bits nr_cpu_ids
#endif
#define nr_cpumask_bits nr_cpu_ids
/*
* The following particular system cpumasks and operations manage
* possible, present, active and online cpus.
*
* cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
* cpu_present_mask - has bit 'cpu' set iff cpu is populated
* cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
* cpu_active_mask - has bit 'cpu' set iff cpu available to migration
*
* If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
*
* The cpu_possible_mask is fixed at boot time, as the set of CPU IDs
* that it is possible might ever be plugged in at anytime during the
* life of that system boot. The cpu_present_mask is dynamic(*),
* representing which CPUs are currently plugged in. And
* cpu_online_mask is the dynamic subset of cpu_present_mask,
* indicating those CPUs available for scheduling.
*
* If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
* depending on what ACPI reports as currently plugged in, otherwise
* cpu_present_mask is just a copy of cpu_possible_mask.
*
* (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
* hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
*
* Subtleties:
* 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
* assumption that their single CPU is online. The UP
* cpu_{online,possible,present}_masks are placebos. Changing them
* will have no useful affect on the following num_*_cpus()
* and cpu_*() macros in the UP case. This ugliness is a UP
* optimization - don't waste any instructions or memory references
* asking if you're online or how many CPUs there are if there is
* only one CPU.
*/
extern struct cpumask __cpu_possible_mask;
extern struct cpumask __cpu_online_mask;
extern struct cpumask __cpu_present_mask;
extern struct cpumask __cpu_active_mask;
extern struct cpumask __cpu_dying_mask;
#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
extern cpumask_t cpus_booted_once_mask;
static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
{
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
WARN_ON_ONCE(cpu >= bits);
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
}
/* verify cpu argument to cpumask_* operators */
static __always_inline unsigned int cpumask_check(unsigned int cpu)
{
cpu_max_bits_warn(cpu, small_cpumask_bits);
return cpu;
}
/**
* cpumask_first - get the first cpu in a cpumask
* @srcp: the cpumask pointer
*
* Return: >= nr_cpu_ids if no cpus set.
*/
static inline unsigned int cpumask_first(const struct cpumask *srcp)
{
return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
}
/**
* cpumask_first_zero - get the first unset cpu in a cpumask
* @srcp: the cpumask pointer
*
* Return: >= nr_cpu_ids if all cpus are set.
*/
static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
{
return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
}
/**
* cpumask_first_and - return the first cpu from *srcp1 & *srcp2
* @srcp1: the first input
* @srcp2: the second input
*
* Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
*/
static inline
unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
{
return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
}
/**
* cpumask_last - get the last CPU in a cpumask
* @srcp: - the cpumask pointer
*
* Return: >= nr_cpumask_bits if no CPUs set.
*/
static inline unsigned int cpumask_last(const struct cpumask *srcp)
{
return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
}
/**
* cpumask_next - get the next cpu in a cpumask
* @n: the cpu prior to the place to search (i.e. return will be > @n)
* @srcp: the cpumask pointer
*
* Return: >= nr_cpu_ids if no further cpus set.
*/
static inline
unsigned int cpumask_next(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);
}
/**
* cpumask_next_zero - get the next unset cpu in a cpumask
* @n: the cpu prior to the place to search (i.e. return will be > @n)
* @srcp: the cpumask pointer
*
* Return: >= nr_cpu_ids if no further cpus unset.
*/
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);
}
#if NR_CPUS == 1
/* Uniprocessor: there is only one valid CPU */
static inline unsigned int cpumask_local_spread(unsigned int i, int node)
{
return 0;
}
static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
const struct cpumask *src2p)
{
return cpumask_first_and(src1p, src2p);
}
static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
{
return cpumask_first(srcp);
}
#else
unsigned int cpumask_local_spread(unsigned int i, int node);
unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
const struct cpumask *src2p);
unsigned int cpumask_any_distribute(const struct cpumask *srcp);
#endif /* NR_CPUS */
/**
* cpumask_next_and - get the next cpu in *src1p & *src2p
* @n: the cpu prior to the place to search (i.e. return will be > @n)
* @src1p: the first cpumask pointer
* @src2p: the second cpumask pointer
*
* Return: >= nr_cpu_ids if no further cpus set in both.
*/
static inline
unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
const struct cpumask *src2p)
{
/* -1 is a legal arg here. */
if (n != -1) cpumask_check(n); return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits, n + 1);
}
/**
* for_each_cpu - iterate over every cpu in a mask
* @cpu: the (optionally unsigned) integer iterator
* @mask: the cpumask pointer
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu(cpu, mask) \
for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
#if NR_CPUS == 1
static inline
unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
{
cpumask_check(start);
if (n != -1)
cpumask_check(n);
/*
* Return the first available CPU when wrapping, or when starting before cpu0,
* since there is only one valid option.
*/
if (wrap && n >= 0)
return nr_cpumask_bits;
return cpumask_first(mask);
}
#else
unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
#endif
/**
* for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
* @cpu: the (optionally unsigned) integer iterator
* @mask: the cpumask pointer
* @start: the start location
*
* The implementation does not assume any bit in @mask is set (including @start).
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_wrap(cpu, mask, start) \
for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
/**
* for_each_cpu_and - iterate over every cpu in both masks
* @cpu: the (optionally unsigned) integer iterator
* @mask1: the first cpumask pointer
* @mask2: the second cpumask pointer
*
* This saves a temporary CPU mask in many places. It is equivalent to:
* struct cpumask tmp;
* cpumask_and(&tmp, &mask1, &mask2);
* for_each_cpu(cpu, &tmp)
* ...
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_and(cpu, mask1, mask2) \
for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
/**
* for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
* those present in another.
* @cpu: the (optionally unsigned) integer iterator
* @mask1: the first cpumask pointer
* @mask2: the second cpumask pointer
*
* This saves a temporary CPU mask in many places. It is equivalent to:
* struct cpumask tmp;
* cpumask_andnot(&tmp, &mask1, &mask2);
* for_each_cpu(cpu, &tmp)
* ...
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_andnot(cpu, mask1, mask2) \
for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
/**
* for_each_cpu_or - iterate over every cpu present in either mask
* @cpu: the (optionally unsigned) integer iterator
* @mask1: the first cpumask pointer
* @mask2: the second cpumask pointer
*
* This saves a temporary CPU mask in many places. It is equivalent to:
* struct cpumask tmp;
* cpumask_or(&tmp, &mask1, &mask2);
* for_each_cpu(cpu, &tmp)
* ...
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_or(cpu, mask1, mask2) \
for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
/**
* cpumask_any_but - return a "random" in a cpumask, but not this one.
* @mask: the cpumask to search
* @cpu: the cpu to ignore.
*
* Often used to find any cpu but smp_processor_id() in a mask.
* Return: >= nr_cpu_ids if no cpus set.
*/
static inline
unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
{
unsigned int i;
cpumask_check(cpu);
for_each_cpu(i, mask)
if (i != cpu)
break;
return i;
}
/**
* cpumask_nth - get the Nth cpu in a cpumask
* @srcp: the cpumask pointer
* @cpu: the Nth cpu to find, starting from 0
*
* Return: >= nr_cpu_ids if such cpu doesn't exist.
*/
static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
{
return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
}
/**
* cpumask_nth_and - get the Nth cpu in 2 cpumasks
* @srcp1: the cpumask pointer
* @srcp2: the cpumask pointer
* @cpu: the Nth cpu to find, starting from 0
*
* Return: >= nr_cpu_ids if such cpu doesn't exist.
*/
static inline
unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
small_cpumask_bits, cpumask_check(cpu));
}
/**
* cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd.
* @srcp1: the cpumask pointer
* @srcp2: the cpumask pointer
* @cpu: the Nth cpu to find, starting from 0
*
* Return: >= nr_cpu_ids if such cpu doesn't exist.
*/
static inline
unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
small_cpumask_bits, cpumask_check(cpu));
}
/**
* cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
* @srcp1: the cpumask pointer
* @srcp2: the cpumask pointer
* @srcp3: the cpumask pointer
* @cpu: the Nth cpu to find, starting from 0
*
* Return: >= nr_cpu_ids if such cpu doesn't exist.
*/
static __always_inline
unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,
const struct cpumask *srcp2,
const struct cpumask *srcp3)
{
return find_nth_and_andnot_bit(cpumask_bits(srcp1),
cpumask_bits(srcp2),
cpumask_bits(srcp3),
small_cpumask_bits, cpumask_check(cpu));
}
#define CPU_BITS_NONE \
{ \
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
}
#define CPU_BITS_CPU0 \
{ \
[0] = 1UL \
}
/**
* cpumask_set_cpu - set a cpu in a cpumask
* @cpu: cpu number (< nr_cpu_ids)
* @dstp: the cpumask pointer
*/
static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
{
set_bit(cpumask_check(cpu), cpumask_bits(dstp));
}
static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
{
__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
}
/**
* cpumask_clear_cpu - clear a cpu in a cpumask
* @cpu: cpu number (< nr_cpu_ids)
* @dstp: the cpumask pointer
*/
static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{
clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
}
static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{
__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
}
/**
* cpumask_test_cpu - test for a cpu in a cpumask
* @cpu: cpu number (< nr_cpu_ids)
* @cpumask: the cpumask pointer
*
* Return: true if @cpu is set in @cpumask, else returns false
*/
static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
{
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
}
/**
* cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
* @cpu: cpu number (< nr_cpu_ids)
* @cpumask: the cpumask pointer
*
* test_and_set_bit wrapper for cpumasks.
*
* Return: true if @cpu is set in old bitmap of @cpumask, else returns false
*/
static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
{
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
}
/**
* cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
* @cpu: cpu number (< nr_cpu_ids)
* @cpumask: the cpumask pointer
*
* test_and_clear_bit wrapper for cpumasks.
*
* Return: true if @cpu is set in old bitmap of @cpumask, else returns false
*/
static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
{
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
}
/**
* cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
* @dstp: the cpumask pointer
*/
static inline void cpumask_setall(struct cpumask *dstp)
{
if (small_const_nbits(small_cpumask_bits)) {
cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
return;
}
bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
}
/**
* cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
* @dstp: the cpumask pointer
*/
static inline void cpumask_clear(struct cpumask *dstp)
{
bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
}
/**
* cpumask_and - *dstp = *src1p & *src2p
* @dstp: the cpumask result
* @src1p: the first input
* @src2p: the second input
*
* Return: false if *@dstp is empty, else returns true
*/
static inline bool cpumask_and(struct cpumask *dstp,
const struct cpumask *src1p,
const struct cpumask *src2p)
{
return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
cpumask_bits(src2p), small_cpumask_bits);
}
/**
* cpumask_or - *dstp = *src1p | *src2p
* @dstp: the cpumask result
* @src1p: the first input
* @src2p: the second input
*/
static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
const struct cpumask *src2p)
{
bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
cpumask_bits(src2p), small_cpumask_bits);
}
/**
* cpumask_xor - *dstp = *src1p ^ *src2p
* @dstp: the cpumask result
* @src1p: the first input
* @src2p: the second input
*/
static inline void cpumask_xor(struct cpumask *dstp,
const struct cpumask *src1p,
const struct cpumask *src2p)
{
bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
cpumask_bits(src2p), small_cpumask_bits);
}
/**
* cpumask_andnot - *dstp = *src1p & ~*src2p
* @dstp: the cpumask result
* @src1p: the first input
* @src2p: the second input
*
* Return: false if *@dstp is empty, else returns true
*/
static inline bool cpumask_andnot(struct cpumask *dstp,
const struct cpumask *src1p,
const struct cpumask *src2p)
{
return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
cpumask_bits(src2p), small_cpumask_bits);
}
/**
* cpumask_equal - *src1p == *src2p
* @src1p: the first input
* @src2p: the second input
*
* Return: true if the cpumasks are equal, false if not
*/
static inline bool cpumask_equal(const struct cpumask *src1p,
const struct cpumask *src2p)
{
return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
small_cpumask_bits);
}
/**
* cpumask_or_equal - *src1p | *src2p == *src3p
* @src1p: the first input
* @src2p: the second input
* @src3p: the third input
*
* Return: true if first cpumask ORed with second cpumask == third cpumask,
* otherwise false
*/
static inline bool cpumask_or_equal(const struct cpumask *src1p,
const struct cpumask *src2p,
const struct cpumask *src3p)
{
return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
cpumask_bits(src3p), small_cpumask_bits);
}
/**
* cpumask_intersects - (*src1p & *src2p) != 0
* @src1p: the first input
* @src2p: the second input
*
* Return: true if first cpumask ANDed with second cpumask is non-empty,
* otherwise false
*/
static inline bool cpumask_intersects(const struct cpumask *src1p,
const struct cpumask *src2p)
{
return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
small_cpumask_bits);
}
/**
* cpumask_subset - (*src1p & ~*src2p) == 0
* @src1p: the first input
* @src2p: the second input
*
* Return: true if *@src1p is a subset of *@src2p, else returns false
*/
static inline bool cpumask_subset(const struct cpumask *src1p,
const struct cpumask *src2p)
{
return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
small_cpumask_bits);
}
/**
* cpumask_empty - *srcp == 0
* @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
*
* Return: true if srcp is empty (has no bits set), else false
*/
static inline bool cpumask_empty(const struct cpumask *srcp)
{
return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
}
/**
* cpumask_full - *srcp == 0xFFFFFFFF...
* @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
*
* Return: true if srcp is full (has all bits set), else false
*/
static inline bool cpumask_full(const struct cpumask *srcp)
{
return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
}
/**
* cpumask_weight - Count of bits in *srcp
* @srcp: the cpumask to count bits (< nr_cpu_ids) in.
*
* Return: count of bits set in *srcp
*/
static inline unsigned int cpumask_weight(const struct cpumask *srcp)
{
return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
}
/**
* cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
* @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
* @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
*
* Return: count of bits set in both *srcp1 and *srcp2
*/
static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
}
/**
* cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2)
* @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
* @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
*
* Return: count of bits set in both *srcp1 and *srcp2
*/
static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
}
/**
* cpumask_shift_right - *dstp = *srcp >> n
* @dstp: the cpumask result
* @srcp: the input to shift
* @n: the number of bits to shift by
*/
static inline void cpumask_shift_right(struct cpumask *dstp,
const struct cpumask *srcp, int n)
{
bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
small_cpumask_bits);
}
/**
* cpumask_shift_left - *dstp = *srcp << n
* @dstp: the cpumask result
* @srcp: the input to shift
* @n: the number of bits to shift by
*/
static inline void cpumask_shift_left(struct cpumask *dstp,
const struct cpumask *srcp, int n)
{
bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
nr_cpumask_bits);
}
/**
* cpumask_copy - *dstp = *srcp
* @dstp: the result
* @srcp: the input cpumask
*/
static inline void cpumask_copy(struct cpumask *dstp,
const struct cpumask *srcp)
{
bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
}
/**
* cpumask_any - pick a "random" cpu from *srcp
* @srcp: the input cpumask
*
* Return: >= nr_cpu_ids if no cpus set.
*/
#define cpumask_any(srcp) cpumask_first(srcp)
/**
* cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
* @mask1: the first input cpumask
* @mask2: the second input cpumask
*
* Return: >= nr_cpu_ids if no cpus set.
*/
#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
/**
* cpumask_of - the cpumask containing just a given cpu
* @cpu: the cpu (<= nr_cpu_ids)
*/
#define cpumask_of(cpu) (get_cpu_mask(cpu))
/**
* cpumask_parse_user - extract a cpumask from a user string
* @buf: the buffer to extract from
* @len: the length of the buffer
* @dstp: the cpumask to set.
*
* Return: -errno, or 0 for success.
*/
static inline int cpumask_parse_user(const char __user *buf, int len,
struct cpumask *dstp)
{
return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
}
/**
* cpumask_parselist_user - extract a cpumask from a user string
* @buf: the buffer to extract from
* @len: the length of the buffer
* @dstp: the cpumask to set.
*
* Return: -errno, or 0 for success.
*/
static inline int cpumask_parselist_user(const char __user *buf, int len,
struct cpumask *dstp)
{
return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
nr_cpumask_bits);
}
/**
* cpumask_parse - extract a cpumask from a string
* @buf: the buffer to extract from
* @dstp: the cpumask to set.
*
* Return: -errno, or 0 for success.
*/
static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
{
return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
}
/**
* cpulist_parse - extract a cpumask from a user string of ranges
* @buf: the buffer to extract from
* @dstp: the cpumask to set.
*
* Return: -errno, or 0 for success.
*/
static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
{
return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
}
/**
* cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes
*
* Return: size to allocate for a &struct cpumask in bytes
*/
static inline unsigned int cpumask_size(void)
{
return BITS_TO_LONGS(large_cpumask_bits) * sizeof(long);
}
/*
* cpumask_var_t: struct cpumask for stack usage.
*
* Oh, the wicked games we play! In order to make kernel coding a
* little more difficult, we typedef cpumask_var_t to an array or a
* pointer: doing &mask on an array is a noop, so it still works.
*
* i.e.
* cpumask_var_t tmpmask;
* if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
* return -ENOMEM;
*
* ... use 'tmpmask' like a normal struct cpumask * ...
*
* free_cpumask_var(tmpmask);
*
*
* However, one notable exception is there. alloc_cpumask_var() allocates
* only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
* NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
*
* cpumask_var_t tmpmask;
* if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
* return -ENOMEM;
*
* var = *tmpmask;
*
* This code makes NR_CPUS length memcopy and brings to a memory corruption.
* cpumask_copy() provide safe copy functionality.
*
* Note that there is another evil here: If you define a cpumask_var_t
* as a percpu variable then the way to obtain the address of the cpumask
* structure differently influences what this_cpu_* operation needs to be
* used. Please use this_cpu_cpumask_var_t in those cases. The direct use
* of this_cpu_ptr() or this_cpu_read() will lead to failures when the
* other type of cpumask_var_t implementation is configured.
*
* Please also note that __cpumask_var_read_mostly can be used to declare
* a cpumask_var_t variable itself (not its content) as read mostly.
*/
#ifdef CONFIG_CPUMASK_OFFSTACK
typedef struct cpumask *cpumask_var_t;
#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
#define __cpumask_var_read_mostly __read_mostly
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
static inline
bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
{
return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
}
/**
* alloc_cpumask_var - allocate a struct cpumask
* @mask: pointer to cpumask_var_t where the cpumask is returned
* @flags: GFP_ flags
*
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
* a nop returning a constant 1 (in <linux/cpumask.h>).
*
* See alloc_cpumask_var_node.
*
* Return: %true if allocation succeeded, %false if not
*/
static inline
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
}
static inline
bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return alloc_cpumask_var(mask, flags | __GFP_ZERO);
}
void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
void free_cpumask_var(cpumask_var_t mask);
void free_bootmem_cpumask_var(cpumask_var_t mask);
static inline bool cpumask_available(cpumask_var_t mask)
{
return mask != NULL;
}
#else
typedef struct cpumask cpumask_var_t[1];
#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
#define __cpumask_var_read_mostly
static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
return true;
}
static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
int node)
{
return true;
}
static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
{
cpumask_clear(*mask);
return true;
}
static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
int node)
{
cpumask_clear(*mask);
return true;
}
static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
{
}
static inline void free_cpumask_var(cpumask_var_t mask)
{
}
static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
{
}
static inline bool cpumask_available(cpumask_var_t mask)
{
return true;
}
#endif /* CONFIG_CPUMASK_OFFSTACK */
DEFINE_FREE(free_cpumask_var, struct cpumask *, if (_T) free_cpumask_var(_T));
/* It's common to want to use cpu_all_mask in struct member initializers,
* so it has to refer to an address rather than a pointer. */
extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
#define cpu_all_mask to_cpumask(cpu_all_bits)
/* First bits of cpu_bit_bitmap are in fact unset. */
#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
#if NR_CPUS == 1
/* Uniprocessor: the possible/online/present masks are always "1" */
#define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
#define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
#define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
#else
#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
#endif
/* Wrappers for arch boot code to manipulate normally-constant masks */
void init_cpu_present(const struct cpumask *src);
void init_cpu_possible(const struct cpumask *src);
void init_cpu_online(const struct cpumask *src);
static inline void reset_cpu_possible_mask(void)
{
bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
}
static inline void
set_cpu_possible(unsigned int cpu, bool possible)
{
if (possible)
cpumask_set_cpu(cpu, &__cpu_possible_mask);
else
cpumask_clear_cpu(cpu, &__cpu_possible_mask);
}
static inline void
set_cpu_present(unsigned int cpu, bool present)
{
if (present)
cpumask_set_cpu(cpu, &__cpu_present_mask);
else
cpumask_clear_cpu(cpu, &__cpu_present_mask);
}
void set_cpu_online(unsigned int cpu, bool online);
static inline void
set_cpu_active(unsigned int cpu, bool active)
{
if (active)
cpumask_set_cpu(cpu, &__cpu_active_mask);
else
cpumask_clear_cpu(cpu, &__cpu_active_mask);
}
static inline void
set_cpu_dying(unsigned int cpu, bool dying)
{
if (dying)
cpumask_set_cpu(cpu, &__cpu_dying_mask);
else
cpumask_clear_cpu(cpu, &__cpu_dying_mask);
}
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
* @bitmap: the bitmap
*
* There are a few places where cpumask_var_t isn't appropriate and
* static cpumasks must be used (eg. very early boot), yet we don't
* expose the definition of 'struct cpumask'.
*
* This does the conversion, and can be used as a constant initializer.
*/
#define to_cpumask(bitmap) \
((struct cpumask *)(1 ? (bitmap) \
: (void *)sizeof(__check_is_bitmap(bitmap))))
static inline int __check_is_bitmap(const unsigned long *bitmap)
{
return 1;
}
/*
* Special-case data structure for "single bit set only" constant CPU masks.
*
* We pre-generate all the 64 (or 32) possible bit positions, with enough
* padding to the left and the right, and return the constant pointer
* appropriately offset.
*/
extern const unsigned long
cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
{
const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
p -= cpu / BITS_PER_LONG;
return to_cpumask(p);
}
#if NR_CPUS > 1
/**
* num_online_cpus() - Read the number of online CPUs
*
* Despite the fact that __num_online_cpus is of type atomic_t, this
* interface gives only a momentary snapshot and is not protected against
* concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
* region.
*
* Return: momentary snapshot of the number of online CPUs
*/
static __always_inline unsigned int num_online_cpus(void)
{
return raw_atomic_read(&__num_online_cpus);
}
#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
static inline bool cpu_online(unsigned int cpu)
{
return cpumask_test_cpu(cpu, cpu_online_mask);
}
static inline bool cpu_possible(unsigned int cpu)
{
return cpumask_test_cpu(cpu, cpu_possible_mask);
}
static inline bool cpu_present(unsigned int cpu)
{
return cpumask_test_cpu(cpu, cpu_present_mask);
}
static inline bool cpu_active(unsigned int cpu)
{
return cpumask_test_cpu(cpu, cpu_active_mask);
}
static inline bool cpu_dying(unsigned int cpu)
{
return cpumask_test_cpu(cpu, cpu_dying_mask);
}
#else
#define num_online_cpus() 1U
#define num_possible_cpus() 1U
#define num_present_cpus() 1U
#define num_active_cpus() 1U
static inline bool cpu_online(unsigned int cpu)
{
return cpu == 0;
}
static inline bool cpu_possible(unsigned int cpu)
{
return cpu == 0;
}
static inline bool cpu_present(unsigned int cpu)
{
return cpu == 0;
}
static inline bool cpu_active(unsigned int cpu)
{
return cpu == 0;
}
static inline bool cpu_dying(unsigned int cpu)
{
return false;
}
#endif /* NR_CPUS > 1 */
#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
#if NR_CPUS <= BITS_PER_LONG
#define CPU_BITS_ALL \
{ \
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
}
#else /* NR_CPUS > BITS_PER_LONG */
#define CPU_BITS_ALL \
{ \
[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
}
#endif /* NR_CPUS > BITS_PER_LONG */
/**
* cpumap_print_to_pagebuf - copies the cpumask into the buffer either
* as comma-separated list of cpus or hex values of cpumask
* @list: indicates whether the cpumap must be list
* @mask: the cpumask to copy
* @buf: the buffer to copy into
*
* Return: the length of the (null-terminated) @buf string, zero if
* nothing is copied.
*/
static inline ssize_t
cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
{
return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
nr_cpu_ids);
}
/**
* cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
* hex values of cpumask
*
* @buf: the buffer to copy into
* @mask: the cpumask to copy
* @off: in the string from which we are copying, we copy to @buf
* @count: the maximum number of bytes to print
*
* The function prints the cpumask into the buffer as hex values of
* cpumask; Typically used by bin_attribute to export cpumask bitmask
* ABI.
*
* Return: the length of how many bytes have been copied, excluding
* terminating '\0'.
*/
static inline ssize_t
cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
loff_t off, size_t count)
{
return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
nr_cpu_ids, off, count) - 1;
}
/**
* cpumap_print_list_to_buf - copies the cpumask into the buffer as
* comma-separated list of cpus
* @buf: the buffer to copy into
* @mask: the cpumask to copy
* @off: in the string from which we are copying, we copy to @buf
* @count: the maximum number of bytes to print
*
* Everything is same with the above cpumap_print_bitmask_to_buf()
* except the print format.
*
* Return: the length of how many bytes have been copied, excluding
* terminating '\0'.
*/
static inline ssize_t
cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
loff_t off, size_t count)
{
return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
nr_cpu_ids, off, count) - 1;
}
#if NR_CPUS <= BITS_PER_LONG
#define CPU_MASK_ALL \
(cpumask_t) { { \
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
} }
#else
#define CPU_MASK_ALL \
(cpumask_t) { { \
[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
} }
#endif /* NR_CPUS > BITS_PER_LONG */
#define CPU_MASK_NONE \
(cpumask_t) { { \
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
} }
#define CPU_MASK_CPU0 \
(cpumask_t) { { \
[0] = 1UL \
} }
/*
* Provide a valid theoretical max size for cpumap and cpulist sysfs files
* to avoid breaking userspace which may allocate a buffer based on the size
* reported by e.g. fstat.
*
* for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
*
* For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
* to 2 orders of magnitude larger than 8192. And then we divide by 2 to
* cover a worst-case of every other cpu being on one of two nodes for a
* very large NR_CPUS.
*
* Use PAGE_SIZE as a minimum for smaller configurations while avoiding
* unsigned comparison to -1.
*/
#define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \
? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
#define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
#endif /* __LINUX_CPUMASK_H */
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* internal.h: mm/ internal definitions
*
* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*/
#ifndef __MM_INTERNAL_H
#define __MM_INTERNAL_H
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/tracepoint-defs.h>
struct folio_batch;
/*
* The set of flags that only affect watermark checking and reclaim
* behaviour. This is used by the MM to obey the caller constraints
* about IO, FS and watermark checking while ignoring placement
* hints such as HIGHMEM usage.
*/
#define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\
__GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\
__GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\
__GFP_NOLOCKDEP)
/* The GFP flags allowed during early boot */
#define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS))
/* Control allocation cpuset and node placement constraints */
#define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE)
/* Do not use these with a slab allocator */
#define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)
/*
* Different from WARN_ON_ONCE(), no warning will be issued
* when we specify __GFP_NOWARN.
*/
#define WARN_ON_ONCE_GFP(cond, gfp) ({ \
static bool __section(".data.once") __warned; \
int __ret_warn_once = !!(cond); \
\
if (unlikely(!(gfp & __GFP_NOWARN) && __ret_warn_once && !__warned)) { \
__warned = true; \
WARN_ON(1); \
} \
unlikely(__ret_warn_once); \
})
void page_writeback_init(void);
/*
* If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages,
* its nr_pages_mapped would be 0x400000: choose the ENTIRELY_MAPPED bit
* above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently
* leaves nr_pages_mapped at 0, but avoid surprise if it participates later.
*/
#define ENTIRELY_MAPPED 0x800000
#define FOLIO_PAGES_MAPPED (ENTIRELY_MAPPED - 1)
/*
* Flags passed to __show_mem() and show_free_areas() to suppress output in
* various contexts.
*/
#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
/*
* How many individual pages have an elevated _mapcount. Excludes
* the folio's entire_mapcount.
*/
static inline int folio_nr_pages_mapped(struct folio *folio)
{
return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED;
}
static inline void *folio_raw_mapping(struct folio *folio)
{
unsigned long mapping = (unsigned long)folio->mapping;
return (void *)(mapping & ~PAGE_MAPPING_FLAGS);
}
#ifdef CONFIG_MMU
/* Flags for folio_pte_batch(). */
typedef int __bitwise fpb_t;
/* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */
#define FPB_IGNORE_DIRTY ((__force fpb_t)BIT(0))
/* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */
#define FPB_IGNORE_SOFT_DIRTY ((__force fpb_t)BIT(1))
static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
{
if (flags & FPB_IGNORE_DIRTY)
pte = pte_mkclean(pte);
if (likely(flags & FPB_IGNORE_SOFT_DIRTY))
pte = pte_clear_soft_dirty(pte);
return pte_wrprotect(pte_mkold(pte));
}
/**
* folio_pte_batch - detect a PTE batch for a large folio
* @folio: The large folio to detect a PTE batch for.
* @addr: The user virtual address the first page is mapped at.
* @start_ptep: Page table pointer for the first entry.
* @pte: Page table entry for the first page.
* @max_nr: The maximum number of table entries to consider.
* @flags: Flags to modify the PTE batch semantics.
* @any_writable: Optional pointer to indicate whether any entry except the
* first one is writable.
*
* Detect a PTE batch: consecutive (present) PTEs that map consecutive
* pages of the same large folio.
*
* All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,
* the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and
* soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY).
*
* start_ptep must map any page of the folio. max_nr must be at least one and
* must be limited by the caller so scanning cannot exceed a single page table.
*
* Return: the number of table entries in the batch.
*/
static inline int folio_pte_batch(struct folio *folio, unsigned long addr,
pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags,
bool *any_writable)
{
unsigned long folio_end_pfn = folio_pfn(folio) + folio_nr_pages(folio);
const pte_t *end_ptep = start_ptep + max_nr;
pte_t expected_pte, *ptep;
bool writable;
int nr;
if (any_writable)
*any_writable = false;
VM_WARN_ON_FOLIO(!pte_present(pte), folio);
VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio);
VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio);
nr = pte_batch_hint(start_ptep, pte);
expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags);
ptep = start_ptep + nr;
while (ptep < end_ptep) {
pte = ptep_get(ptep);
if (any_writable)
writable = !!pte_write(pte);
pte = __pte_batch_clear_ignored(pte, flags);
if (!pte_same(pte, expected_pte))
break;
/*
* Stop immediately once we reached the end of the folio. In
* corner cases the next PFN might fall into a different
* folio.
*/
if (pte_pfn(pte) >= folio_end_pfn)
break;
if (any_writable)
*any_writable |= writable;
nr = pte_batch_hint(ptep, pte);
expected_pte = pte_advance_pfn(expected_pte, nr);
ptep += nr;
}
return min(ptep - start_ptep, max_nr);
}
#endif /* CONFIG_MMU */
void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
int nr_throttled);
static inline void acct_reclaim_writeback(struct folio *folio)
{
pg_data_t *pgdat = folio_pgdat(folio);
int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled);
if (nr_throttled)
__acct_reclaim_writeback(pgdat, folio, nr_throttled);
}
static inline void wake_throttle_isolated(pg_data_t *pgdat)
{
wait_queue_head_t *wqh;
wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED];
if (waitqueue_active(wqh))
wake_up(wqh);
}
vm_fault_t vmf_anon_prepare(struct vm_fault *vmf);
vm_fault_t do_swap_page(struct vm_fault *vmf);
void folio_rotate_reclaimable(struct folio *folio);
bool __folio_end_writeback(struct folio *folio);
void deactivate_file_folio(struct folio *folio);
void folio_activate(struct folio *folio);
void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
struct vm_area_struct *start_vma, unsigned long floor,
unsigned long ceiling, bool mm_wr_locked);
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
struct zap_details;
void unmap_page_range(struct mmu_gather *tlb,
struct vm_area_struct *vma,
unsigned long addr, unsigned long end,
struct zap_details *details);
void page_cache_ra_order(struct readahead_control *, struct file_ra_state *,
unsigned int order);
void force_page_cache_ra(struct readahead_control *, unsigned long nr);
static inline void force_page_cache_readahead(struct address_space *mapping,
struct file *file, pgoff_t index, unsigned long nr_to_read)
{
DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index);
force_page_cache_ra(&ractl, nr_to_read);
}
unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
void filemap_free_folio(struct address_space *mapping, struct folio *folio);
int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
loff_t end);
long mapping_evict_folio(struct address_space *mapping, struct folio *folio);
unsigned long mapping_try_invalidate(struct address_space *mapping,
pgoff_t start, pgoff_t end, unsigned long *nr_failed);
/**
* folio_evictable - Test whether a folio is evictable.
* @folio: The folio to test.
*
* Test whether @folio is evictable -- i.e., should be placed on
* active/inactive lists vs unevictable list.
*
* Reasons folio might not be evictable:
* 1. folio's mapping marked unevictable
* 2. One of the pages in the folio is part of an mlocked VMA
*/
static inline bool folio_evictable(struct folio *folio)
{
bool ret;
/* Prevent address_space of inode and swap cache from being freed */
rcu_read_lock(); ret = !mapping_unevictable(folio_mapping(folio)) && !folio_test_mlocked(folio); rcu_read_unlock();
return ret;
}
/*
* Turn a non-refcounted page (->_refcount == 0) into refcounted with
* a count of one.
*/
static inline void set_page_refcounted(struct page *page)
{
VM_BUG_ON_PAGE(PageTail(page), page);
VM_BUG_ON_PAGE(page_ref_count(page), page);
set_page_count(page, 1);
}
/*
* Return true if a folio needs ->release_folio() calling upon it.
*/
static inline bool folio_needs_release(struct folio *folio)
{
struct address_space *mapping = folio_mapping(folio);
return folio_has_private(folio) ||
(mapping && mapping_release_always(mapping));
}
extern unsigned long highest_memmap_pfn;
/*
* Maximum number of reclaim retries without progress before the OOM
* killer is consider the only way forward.
*/
#define MAX_RECLAIM_RETRIES 16
/*
* in mm/vmscan.c:
*/
bool isolate_lru_page(struct page *page);
bool folio_isolate_lru(struct folio *folio);
void putback_lru_page(struct page *page);
void folio_putback_lru(struct folio *folio);
extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);
/*
* in mm/rmap.c:
*/
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
/*
* in mm/page_alloc.c
*/
#define K(x) ((x) << (PAGE_SHIFT-10))
extern char * const zone_names[MAX_NR_ZONES];
/* perform sanity checks on struct pages being allocated or freed */
DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
extern int min_free_kbytes;
void setup_per_zone_wmarks(void);
void calculate_min_free_kbytes(void);
int __meminit init_per_zone_wmark_min(void);
void page_alloc_sysctl_init(void);
/*
* Structure for holding the mostly immutable allocation parameters passed
* between functions involved in allocations, including the alloc_pages*
* family of functions.
*
* nodemask, migratetype and highest_zoneidx are initialized only once in
* __alloc_pages() and then never change.
*
* zonelist, preferred_zone and highest_zoneidx are set first in
* __alloc_pages() for the fast path, and might be later changed
* in __alloc_pages_slowpath(). All other functions pass the whole structure
* by a const pointer.
*/
struct alloc_context {
struct zonelist *zonelist;
nodemask_t *nodemask;
struct zoneref *preferred_zoneref;
int migratetype;
/*
* highest_zoneidx represents highest usable zone index of
* the allocation request. Due to the nature of the zone,
* memory on lower zone than the highest_zoneidx will be
* protected by lowmem_reserve[highest_zoneidx].
*
* highest_zoneidx is also used by reclaim/compaction to limit
* the target zone since higher zone than this index cannot be
* usable for this allocation request.
*/
enum zone_type highest_zoneidx;
bool spread_dirty_pages;
};
/*
* This function returns the order of a free page in the buddy system. In
* general, page_zone(page)->lock must be held by the caller to prevent the
* page from being allocated in parallel and returning garbage as the order.
* If a caller does not hold page_zone(page)->lock, it must guarantee that the
* page cannot be allocated or merged in parallel. Alternatively, it must
* handle invalid values gracefully, and use buddy_order_unsafe() below.
*/
static inline unsigned int buddy_order(struct page *page)
{
/* PageBuddy() must be checked by the caller */
return page_private(page);
}
/*
* Like buddy_order(), but for callers who cannot afford to hold the zone lock.
* PageBuddy() should be checked first by the caller to minimize race window,
* and invalid values must be handled gracefully.
*
* READ_ONCE is used so that if the caller assigns the result into a local
* variable and e.g. tests it for valid range before using, the compiler cannot
* decide to remove the variable and inline the page_private(page) multiple
* times, potentially observing different values in the tests and the actual
* use of the result.
*/
#define buddy_order_unsafe(page) READ_ONCE(page_private(page))
/*
* This function checks whether a page is free && is the buddy
* we can coalesce a page and its buddy if
* (a) the buddy is not in a hole (check before calling!) &&
* (b) the buddy is in the buddy system &&
* (c) a page and its buddy have the same order &&
* (d) a page and its buddy are in the same zone.
*
* For recording whether a page is in the buddy system, we set PageBuddy.
* Setting, clearing, and testing PageBuddy is serialized by zone->lock.
*
* For recording page's order, we use page_private(page).
*/
static inline bool page_is_buddy(struct page *page, struct page *buddy,
unsigned int order)
{
if (!page_is_guard(buddy) && !PageBuddy(buddy))
return false;
if (buddy_order(buddy) != order)
return false;
/*
* zone check is done late to avoid uselessly calculating
* zone/node ids for pages that could never merge.
*/
if (page_zone_id(page) != page_zone_id(buddy))
return false;
VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
return true;
}
/*
* Locate the struct page for both the matching buddy in our
* pair (buddy1) and the combined O(n+1) page they form (page).
*
* 1) Any buddy B1 will have an order O twin B2 which satisfies
* the following equation:
* B2 = B1 ^ (1 << O)
* For example, if the starting buddy (buddy2) is #8 its order
* 1 buddy is #10:
* B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
*
* 2) Any buddy B will have an order O+1 parent P which
* satisfies the following equation:
* P = B & ~(1 << O)
*
* Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER
*/
static inline unsigned long
__find_buddy_pfn(unsigned long page_pfn, unsigned int order)
{
return page_pfn ^ (1 << order);
}
/*
* Find the buddy of @page and validate it.
* @page: The input page
* @pfn: The pfn of the page, it saves a call to page_to_pfn() when the
* function is used in the performance-critical __free_one_page().
* @order: The order of the page
* @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to
* page_to_pfn().
*
* The found buddy can be a non PageBuddy, out of @page's zone, or its order is
* not the same as @page. The validation is necessary before use it.
*
* Return: the found buddy page or NULL if not found.
*/
static inline struct page *find_buddy_page_pfn(struct page *page,
unsigned long pfn, unsigned int order, unsigned long *buddy_pfn)
{
unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order);
struct page *buddy;
buddy = page + (__buddy_pfn - pfn);
if (buddy_pfn)
*buddy_pfn = __buddy_pfn;
if (page_is_buddy(page, buddy, order))
return buddy;
return NULL;
}
extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
unsigned long end_pfn, struct zone *zone);
static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
unsigned long end_pfn, struct zone *zone)
{
if (zone->contiguous)
return pfn_to_page(start_pfn);
return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
}
void set_zone_contiguous(struct zone *zone);
static inline void clear_zone_contiguous(struct zone *zone)
{
zone->contiguous = false;
}
extern int __isolate_free_page(struct page *page, unsigned int order);
extern void __putback_isolated_page(struct page *page, unsigned int order,
int mt);
extern void memblock_free_pages(struct page *page, unsigned long pfn,
unsigned int order);
extern void __free_pages_core(struct page *page, unsigned int order);
/*
* This will have no effect, other than possibly generating a warning, if the
* caller passes in a non-large folio.
*/
static inline void folio_set_order(struct folio *folio, unsigned int order)
{
if (WARN_ON_ONCE(!order || !folio_test_large(folio)))
return;
folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order;
#ifdef CONFIG_64BIT
folio->_folio_nr_pages = 1U << order;
#endif
}
void folio_undo_large_rmappable(struct folio *folio);
static inline struct folio *page_rmappable_folio(struct page *page)
{
struct folio *folio = (struct folio *)page;
folio_prep_large_rmappable(folio);
return folio;
}
static inline void prep_compound_head(struct page *page, unsigned int order)
{
struct folio *folio = (struct folio *)page;
folio_set_order(folio, order);
atomic_set(&folio->_entire_mapcount, -1);
atomic_set(&folio->_nr_pages_mapped, 0);
atomic_set(&folio->_pincount, 0);
}
static inline void prep_compound_tail(struct page *head, int tail_idx)
{
struct page *p = head + tail_idx;
p->mapping = TAIL_MAPPING;
set_compound_head(p, head);
set_page_private(p, 0);
}
extern void prep_compound_page(struct page *page, unsigned int order);
extern void post_alloc_hook(struct page *page, unsigned int order,
gfp_t gfp_flags);
extern bool free_pages_prepare(struct page *page, unsigned int order);
extern int user_min_free_kbytes;
void free_unref_page(struct page *page, unsigned int order);
void free_unref_folios(struct folio_batch *fbatch);
extern void zone_pcp_reset(struct zone *zone);
extern void zone_pcp_disable(struct zone *zone);
extern void zone_pcp_enable(struct zone *zone);
extern void zone_pcp_init(struct zone *zone);
extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,
phys_addr_t min_addr,
int nid, bool exact_nid);
void memmap_init_range(unsigned long, int, unsigned long, unsigned long,
unsigned long, enum meminit_context, struct vmem_altmap *, int);
int split_free_page(struct page *free_page,
unsigned int order, unsigned long split_pfn_offset);
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
/*
* in mm/compaction.c
*/
/*
* compact_control is used to track pages being migrated and the free pages
* they are being migrated to during memory compaction. The free_pfn starts
* at the end of a zone and migrate_pfn begins at the start. Movable pages
* are moved to the end of a zone during a compaction run and the run
* completes when free_pfn <= migrate_pfn
*/
struct compact_control {
struct list_head freepages[NR_PAGE_ORDERS]; /* List of free pages to migrate to */
struct list_head migratepages; /* List of pages being migrated */
unsigned int nr_freepages; /* Number of isolated free pages */
unsigned int nr_migratepages; /* Number of pages to migrate */
unsigned long free_pfn; /* isolate_freepages search base */
/*
* Acts as an in/out parameter to page isolation for migration.
* isolate_migratepages uses it as a search base.
* isolate_migratepages_block will update the value to the next pfn
* after the last isolated one.
*/
unsigned long migrate_pfn;
unsigned long fast_start_pfn; /* a pfn to start linear scan from */
struct zone *zone;
unsigned long total_migrate_scanned;
unsigned long total_free_scanned;
unsigned short fast_search_fail;/* failures to use free list searches */
short search_order; /* order to start a fast search at */
const gfp_t gfp_mask; /* gfp mask of a direct compactor */
int order; /* order a direct compactor needs */
int migratetype; /* migratetype of direct compactor */
const unsigned int alloc_flags; /* alloc flags of a direct compactor */
const int highest_zoneidx; /* zone index of a direct compactor */
enum migrate_mode mode; /* Async or sync migration mode */
bool ignore_skip_hint; /* Scan blocks even if marked skip */
bool no_set_skip_hint; /* Don't mark blocks for skipping */
bool ignore_block_suitable; /* Scan blocks considered unsuitable */
bool direct_compaction; /* False from kcompactd or /proc/... */
bool proactive_compaction; /* kcompactd proactive compaction */
bool whole_zone; /* Whole zone should/has been scanned */
bool contended; /* Signal lock contention */
bool finish_pageblock; /* Scan the remainder of a pageblock. Used
* when there are potentially transient
* isolation or migration failures to
* ensure forward progress.
*/
bool alloc_contig; /* alloc_contig_range allocation */
};
/*
* Used in direct compaction when a page should be taken from the freelists
* immediately when one is created during the free path.
*/
struct capture_control {
struct compact_control *cc;
struct page *page;
};
unsigned long
isolate_freepages_range(struct compact_control *cc,
unsigned long start_pfn, unsigned long end_pfn);
int
isolate_migratepages_range(struct compact_control *cc,
unsigned long low_pfn, unsigned long end_pfn);
int __alloc_contig_migrate_range(struct compact_control *cc,
unsigned long start, unsigned long end,
int migratetype);
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
void init_cma_reserved_pageblock(struct page *page);
#endif /* CONFIG_COMPACTION || CONFIG_CMA */
int find_suitable_fallback(struct free_area *area, unsigned int order,
int migratetype, bool only_stealable, bool *can_steal);
static inline bool free_area_empty(struct free_area *area, int migratetype)
{
return list_empty(&area->free_list[migratetype]);
}
/*
* These three helpers classifies VMAs for virtual memory accounting.
*/
/*
* Executable code area - executable, not writable, not stack
*/
static inline bool is_exec_mapping(vm_flags_t flags)
{
return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
}
/*
* Stack area (including shadow stacks)
*
* VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
* do_mmap() forbids all other combinations.
*/
static inline bool is_stack_mapping(vm_flags_t flags)
{
return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
}
/*
* Data area - private, writable, not stack
*/
static inline bool is_data_mapping(vm_flags_t flags)
{
return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
}
/* mm/util.c */
struct anon_vma *folio_anon_vma(struct folio *folio);
#ifdef CONFIG_MMU
void unmap_mapping_folio(struct folio *folio);
extern long populate_vma_page_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end, int *locked);
extern long faultin_page_range(struct mm_struct *mm, unsigned long start,
unsigned long end, bool write, int *locked);
extern bool mlock_future_ok(struct mm_struct *mm, unsigned long flags,
unsigned long bytes);
/*
* NOTE: This function can't tell whether the folio is "fully mapped" in the
* range.
* "fully mapped" means all the pages of folio is associated with the page
* table of range while this function just check whether the folio range is
* within the range [start, end). Function caller needs to do page table
* check if it cares about the page table association.
*
* Typical usage (like mlock or madvise) is:
* Caller knows at least 1 page of folio is associated with page table of VMA
* and the range [start, end) is intersect with the VMA range. Caller wants
* to know whether the folio is fully associated with the range. It calls
* this function to check whether the folio is in the range first. Then checks
* the page table to know whether the folio is fully mapped to the range.
*/
static inline bool
folio_within_range(struct folio *folio, struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
pgoff_t pgoff, addr;
unsigned long vma_pglen = vma_pages(vma);
VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio);
if (start > end)
return false;
if (start < vma->vm_start)
start = vma->vm_start;
if (end > vma->vm_end)
end = vma->vm_end;
pgoff = folio_pgoff(folio);
/* if folio start address is not in vma range */
if (!in_range(pgoff, vma->vm_pgoff, vma_pglen))
return false;
addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
return !(addr < start || end - addr < folio_size(folio));
}
static inline bool
folio_within_vma(struct folio *folio, struct vm_area_struct *vma)
{
return folio_within_range(folio, vma, vma->vm_start, vma->vm_end);
}
/*
* mlock_vma_folio() and munlock_vma_folio():
* should be called with vma's mmap_lock held for read or write,
* under page table lock for the pte/pmd being added or removed.
*
* mlock is usually called at the end of folio_add_*_rmap_*(), munlock at
* the end of folio_remove_rmap_*(); but new anon folios are managed by
* folio_add_lru_vma() calling mlock_new_folio().
*/
void mlock_folio(struct folio *folio);
static inline void mlock_vma_folio(struct folio *folio,
struct vm_area_struct *vma)
{
/*
* The VM_SPECIAL check here serves two purposes.
* 1) VM_IO check prevents migration from double-counting during mlock.
* 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED
* is never left set on a VM_SPECIAL vma, there is an interval while
* file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may
* still be set while VM_SPECIAL bits are added: so ignore it then.
*/
if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED)) mlock_folio(folio);
}
void munlock_folio(struct folio *folio);
static inline void munlock_vma_folio(struct folio *folio,
struct vm_area_struct *vma)
{
/*
* munlock if the function is called. Ideally, we should only
* do munlock if any page of folio is unmapped from VMA and
* cause folio not fully mapped to VMA.
*
* But it's not easy to confirm that's the situation. So we
* always munlock the folio and page reclaim will correct it
* if it's wrong.
*/
if (unlikely(vma->vm_flags & VM_LOCKED))
munlock_folio(folio);
}
void mlock_new_folio(struct folio *folio);
bool need_mlock_drain(int cpu);
void mlock_drain_local(void);
void mlock_drain_remote(int cpu);
extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
/*
* Return the start of user virtual address at the specific offset within
* a vma.
*/
static inline unsigned long
vma_pgoff_address(pgoff_t pgoff, unsigned long nr_pages,
struct vm_area_struct *vma)
{
unsigned long address;
if (pgoff >= vma->vm_pgoff) {
address = vma->vm_start +
((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
/* Check for address beyond vma (or wrapped through 0?) */
if (address < vma->vm_start || address >= vma->vm_end)
address = -EFAULT;
} else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) {
/* Test above avoids possibility of wrap to 0 on 32-bit */
address = vma->vm_start;
} else {
address = -EFAULT;
}
return address;
}
/*
* Return the start of user virtual address of a page within a vma.
* Returns -EFAULT if all of the page is outside the range of vma.
* If page is a compound head, the entire compound page is considered.
*/
static inline unsigned long
vma_address(struct page *page, struct vm_area_struct *vma)
{
VM_BUG_ON_PAGE(PageKsm(page), page); /* KSM page->index unusable */
return vma_pgoff_address(page_to_pgoff(page), compound_nr(page), vma);
}
/*
* Then at what user virtual address will none of the range be found in vma?
* Assumes that vma_address() already returned a good starting address.
*/
static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
{
struct vm_area_struct *vma = pvmw->vma;
pgoff_t pgoff;
unsigned long address;
/* Common case, plus ->pgoff is invalid for KSM */
if (pvmw->nr_pages == 1)
return pvmw->address + PAGE_SIZE;
pgoff = pvmw->pgoff + pvmw->nr_pages;
address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
/* Check for address beyond vma (or wrapped through 0?) */
if (address < vma->vm_start || address > vma->vm_end)
address = vma->vm_end;
return address;
}
static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
struct file *fpin)
{
int flags = vmf->flags;
if (fpin)
return fpin;
/*
* FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
* anything, so we only pin the file and drop the mmap_lock if only
* FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt.
*/
if (fault_flag_allow_retry_first(flags) &&
!(flags & FAULT_FLAG_RETRY_NOWAIT)) {
fpin = get_file(vmf->vma->vm_file);
release_fault_lock(vmf);
}
return fpin;
}
#else /* !CONFIG_MMU */
static inline void unmap_mapping_folio(struct folio *folio) { }
static inline void mlock_new_folio(struct folio *folio) { }
static inline bool need_mlock_drain(int cpu) { return false; }
static inline void mlock_drain_local(void) { }
static inline void mlock_drain_remote(int cpu) { }
static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
{
}
#endif /* !CONFIG_MMU */
/* Memory initialisation debug and verification */
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
DECLARE_STATIC_KEY_TRUE(deferred_pages);
bool __init deferred_grow_zone(struct zone *zone, unsigned int order);
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
enum mminit_level {
MMINIT_WARNING,
MMINIT_VERIFY,
MMINIT_TRACE
};
#ifdef CONFIG_DEBUG_MEMORY_INIT
extern int mminit_loglevel;
#define mminit_dprintk(level, prefix, fmt, arg...) \
do { \
if (level < mminit_loglevel) { \
if (level <= MMINIT_WARNING) \
pr_warn("mminit::" prefix " " fmt, ##arg); \
else \
printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
} \
} while (0)
extern void mminit_verify_pageflags_layout(void);
extern void mminit_verify_zonelist(void);
#else
static inline void mminit_dprintk(enum mminit_level level,
const char *prefix, const char *fmt, ...)
{
}
static inline void mminit_verify_pageflags_layout(void)
{
}
static inline void mminit_verify_zonelist(void)
{
}
#endif /* CONFIG_DEBUG_MEMORY_INIT */
#define NODE_RECLAIM_NOSCAN -2
#define NODE_RECLAIM_FULL -1
#define NODE_RECLAIM_SOME 0
#define NODE_RECLAIM_SUCCESS 1
#ifdef CONFIG_NUMA
extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int);
extern int find_next_best_node(int node, nodemask_t *used_node_mask);
#else
static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask,
unsigned int order)
{
return NODE_RECLAIM_NOSCAN;
}
static inline int find_next_best_node(int node, nodemask_t *used_node_mask)
{
return NUMA_NO_NODE;
}
#endif
/*
* mm/memory-failure.c
*/
extern int hwpoison_filter(struct page *p);
extern u32 hwpoison_filter_dev_major;
extern u32 hwpoison_filter_dev_minor;
extern u64 hwpoison_filter_flags_mask;
extern u64 hwpoison_filter_flags_value;
extern u64 hwpoison_filter_memcg;
extern u32 hwpoison_filter_enable;
extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long,
unsigned long, unsigned long,
unsigned long, unsigned long);
extern void set_pageblock_order(void);
unsigned long reclaim_pages(struct list_head *folio_list, bool ignore_references);
unsigned int reclaim_clean_pages_from_list(struct zone *zone,
struct list_head *folio_list);
/* The ALLOC_WMARK bits are used as an index to zone->watermark */
#define ALLOC_WMARK_MIN WMARK_MIN
#define ALLOC_WMARK_LOW WMARK_LOW
#define ALLOC_WMARK_HIGH WMARK_HIGH
#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
/* Mask to get the watermark bits */
#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
/*
* Only MMU archs have async oom victim reclaim - aka oom_reaper so we
* cannot assume a reduced access to memory reserves is sufficient for
* !MMU
*/
#ifdef CONFIG_MMU
#define ALLOC_OOM 0x08
#else
#define ALLOC_OOM ALLOC_NO_WATERMARKS
#endif
#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access
* to 25% of the min watermark or
* 62.5% if __GFP_HIGH is set.
*/
#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50%
* of the min watermark.
*/
#define ALLOC_CPUSET 0x40 /* check for correct cpuset */
#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */
#ifdef CONFIG_ZONE_DMA32
#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */
#else
#define ALLOC_NOFRAGMENT 0x0
#endif
#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */
#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
/* Flags that allow allocations below the min watermark. */
#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
enum ttu_flags;
struct tlbflush_unmap_batch;
/*
* only for MM internal work items which do not depend on
* any allocations or locks which might depend on allocations
*/
extern struct workqueue_struct *mm_percpu_wq;
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
void try_to_unmap_flush(void);
void try_to_unmap_flush_dirty(void);
void flush_tlb_batched_pending(struct mm_struct *mm);
#else
static inline void try_to_unmap_flush(void)
{
}
static inline void try_to_unmap_flush_dirty(void)
{
}
static inline void flush_tlb_batched_pending(struct mm_struct *mm)
{
}
#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
extern const struct trace_print_flags pageflag_names[];
extern const struct trace_print_flags pagetype_names[];
extern const struct trace_print_flags vmaflag_names[];
extern const struct trace_print_flags gfpflag_names[];
static inline bool is_migrate_highatomic(enum migratetype migratetype)
{
return migratetype == MIGRATE_HIGHATOMIC;
}
static inline bool is_migrate_highatomic_page(struct page *page)
{
return get_pageblock_migratetype(page) == MIGRATE_HIGHATOMIC;
}
void setup_zone_pageset(struct zone *zone);
struct migration_target_control {
int nid; /* preferred node id */
nodemask_t *nmask;
gfp_t gfp_mask;
};
/*
* mm/filemap.c
*/
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size);
/*
* mm/vmalloc.c
*/
#ifdef CONFIG_MMU
void __init vmalloc_init(void);
int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift);
#else
static inline void vmalloc_init(void)
{
}
static inline
int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
return -EINVAL;
}
#endif
int __must_check __vmap_pages_range_noflush(unsigned long addr,
unsigned long end, pgprot_t prot,
struct page **pages, unsigned int page_shift);
void vunmap_range_noflush(unsigned long start, unsigned long end);
void __vunmap_range_noflush(unsigned long start, unsigned long end);
int numa_migrate_prep(struct folio *folio, struct vm_area_struct *vma,
unsigned long addr, int page_nid, int *flags);
void free_zone_device_page(struct page *page);
int migrate_device_coherent_page(struct page *page);
/*
* mm/gup.c
*/
struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
int __must_check try_grab_page(struct page *page, unsigned int flags);
/*
* mm/huge_memory.c
*/
struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
unsigned long addr, pmd_t *pmd,
unsigned int flags);
/*
* mm/mmap.c
*/
struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
struct vm_area_struct *vma,
unsigned long delta);
enum {
/* mark page accessed */
FOLL_TOUCH = 1 << 16,
/* a retry, previous pass started an IO */
FOLL_TRIED = 1 << 17,
/* we are working on non-current tsk/mm */
FOLL_REMOTE = 1 << 18,
/* pages must be released via unpin_user_page */
FOLL_PIN = 1 << 19,
/* gup_fast: prevent fall-back to slow gup */
FOLL_FAST_ONLY = 1 << 20,
/* allow unlocking the mmap lock */
FOLL_UNLOCKABLE = 1 << 21,
/* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */
FOLL_MADV_POPULATE = 1 << 22,
};
#define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \
FOLL_FAST_ONLY | FOLL_UNLOCKABLE | \
FOLL_MADV_POPULATE)
/*
* Indicates for which pages that are write-protected in the page table,
* whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the
* GUP pin will remain consistent with the pages mapped into the page tables
* of the MM.
*
* Temporary unmapping of PageAnonExclusive() pages or clearing of
* PageAnonExclusive() has to protect against concurrent GUP:
* * Ordinary GUP: Using the PT lock
* * GUP-fast and fork(): mm->write_protect_seq
* * GUP-fast and KSM or temporary unmapping (swap, migration): see
* folio_try_share_anon_rmap_*()
*
* Must be called with the (sub)page that's actually referenced via the
* page table entry, which might not necessarily be the head page for a
* PTE-mapped THP.
*
* If the vma is NULL, we're coming from the GUP-fast path and might have
* to fallback to the slow path just to lookup the vma.
*/
static inline bool gup_must_unshare(struct vm_area_struct *vma,
unsigned int flags, struct page *page)
{
/*
* FOLL_WRITE is implicitly handled correctly as the page table entry
* has to be writable -- and if it references (part of) an anonymous
* folio, that part is required to be marked exclusive.
*/
if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN)
return false;
/*
* Note: PageAnon(page) is stable until the page is actually getting
* freed.
*/
if (!PageAnon(page)) {
/*
* We only care about R/O long-term pining: R/O short-term
* pinning does not have the semantics to observe successive
* changes through the process page tables.
*/
if (!(flags & FOLL_LONGTERM))
return false;
/* We really need the vma ... */
if (!vma)
return true;
/*
* ... because we only care about writable private ("COW")
* mappings where we have to break COW early.
*/
return is_cow_mapping(vma->vm_flags);
}
/* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
smp_rmb();
/*
* During GUP-fast we might not get called on the head page for a
* hugetlb page that is mapped using cont-PTE, because GUP-fast does
* not work with the abstracted hugetlb PTEs that always point at the
* head page. For hugetlb, PageAnonExclusive only applies on the head
* page (as it cannot be partially COW-shared), so lookup the head page.
*/
if (unlikely(!PageHead(page) && PageHuge(page)))
page = compound_head(page);
/*
* Note that PageKsm() pages cannot be exclusive, and consequently,
* cannot get pinned.
*/
return !PageAnonExclusive(page);
}
extern bool mirrored_kernelcore;
extern bool memblock_has_mirror(void);
static __always_inline void vma_set_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
pgoff_t pgoff)
{
vma->vm_start = start;
vma->vm_end = end;
vma->vm_pgoff = pgoff;
}
static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
{
/*
* NOTE: we must check this before VM_SOFTDIRTY on soft-dirty
* enablements, because when without soft-dirty being compiled in,
* VM_SOFTDIRTY is defined as 0x0, then !(vm_flags & VM_SOFTDIRTY)
* will be constantly true.
*/
if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
return false;
/*
* Soft-dirty is kind of special: its tracking is enabled when the
* vma flags not set.
*/
return !(vma->vm_flags & VM_SOFTDIRTY);
}
static inline void vma_iter_config(struct vma_iterator *vmi,
unsigned long index, unsigned long last)
{
__mas_set_range(&vmi->mas, index, last - 1);
}
/*
* VMA Iterator functions shared between nommu and mmap
*/
static inline int vma_iter_prealloc(struct vma_iterator *vmi,
struct vm_area_struct *vma)
{
return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
}
static inline void vma_iter_clear(struct vma_iterator *vmi)
{
mas_store_prealloc(&vmi->mas, NULL);
}
static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
{
return mas_walk(&vmi->mas);
}
/* Store a VMA with preallocated memory */
static inline void vma_iter_store(struct vma_iterator *vmi,
struct vm_area_struct *vma)
{
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
vmi->mas.index > vma->vm_start)) {
pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
vmi->mas.index, vma->vm_start, vma->vm_start,
vma->vm_end, vmi->mas.index, vmi->mas.last);
}
if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
vmi->mas.last < vma->vm_start)) {
pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
vmi->mas.index, vmi->mas.last);
}
#endif
if (vmi->mas.status != ma_start &&
((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
vma_iter_invalidate(vmi);
__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
mas_store_prealloc(&vmi->mas, vma);
}
static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
struct vm_area_struct *vma, gfp_t gfp)
{
if (vmi->mas.status != ma_start &&
((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
vma_iter_invalidate(vmi);
__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
mas_store_gfp(&vmi->mas, vma, gfp);
if (unlikely(mas_is_err(&vmi->mas)))
return -ENOMEM;
return 0;
}
/*
* VMA lock generalization
*/
struct vma_prepare {
struct vm_area_struct *vma;
struct vm_area_struct *adj_next;
struct file *file;
struct address_space *mapping;
struct anon_vma *anon_vma;
struct vm_area_struct *insert;
struct vm_area_struct *remove;
struct vm_area_struct *remove2;
};
void __meminit __init_single_page(struct page *page, unsigned long pfn,
unsigned long zone, int nid);
/* shrinker related functions */
unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
int priority);
#ifdef CONFIG_SHRINKER_DEBUG
static inline __printf(2, 0) int shrinker_debugfs_name_alloc(
struct shrinker *shrinker, const char *fmt, va_list ap)
{
shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
return shrinker->name ? 0 : -ENOMEM;
}
static inline void shrinker_debugfs_name_free(struct shrinker *shrinker)
{
kfree_const(shrinker->name);
shrinker->name = NULL;
}
extern int shrinker_debugfs_add(struct shrinker *shrinker);
extern struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker,
int *debugfs_id);
extern void shrinker_debugfs_remove(struct dentry *debugfs_entry,
int debugfs_id);
#else /* CONFIG_SHRINKER_DEBUG */
static inline int shrinker_debugfs_add(struct shrinker *shrinker)
{
return 0;
}
static inline int shrinker_debugfs_name_alloc(struct shrinker *shrinker,
const char *fmt, va_list ap)
{
return 0;
}
static inline void shrinker_debugfs_name_free(struct shrinker *shrinker)
{
}
static inline struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker,
int *debugfs_id)
{
*debugfs_id = -1;
return NULL;
}
static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry,
int debugfs_id)
{
}
#endif /* CONFIG_SHRINKER_DEBUG */
/* Only track the nodes of mappings with shadow entries */
void workingset_update_node(struct xa_node *node);
extern struct list_lru shadow_nodes;
#endif /* __MM_INTERNAL_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2012-2014 Andy Lutomirski <luto@amacapital.net>
*
* Based on the original implementation which is:
* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
* Copyright 2003 Andi Kleen, SuSE Labs.
*
* Parts of the original code have been moved to arch/x86/vdso/vma.c
*
* This file implements vsyscall emulation. vsyscalls are a legacy ABI:
* Userspace can request certain kernel services by calling fixed
* addresses. This concept is problematic:
*
* - It interferes with ASLR.
* - It's awkward to write code that lives in kernel addresses but is
* callable by userspace at fixed addresses.
* - The whole concept is impossible for 32-bit compat userspace.
* - UML cannot easily virtualize a vsyscall.
*
* As of mid-2014, I believe that there is no new userspace code that
* will use a vsyscall if the vDSO is present. I hope that there will
* soon be no new userspace code that will ever use a vsyscall.
*
* The code in this file emulates vsyscalls when notified of a page
* fault to a vsyscall address.
*/
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/sched/signal.h>
#include <linux/mm_types.h>
#include <linux/syscalls.h>
#include <linux/ratelimit.h>
#include <asm/vsyscall.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/traps.h>
#include <asm/paravirt.h>
#define CREATE_TRACE_POINTS
#include "vsyscall_trace.h"
static enum { EMULATE, XONLY, NONE } vsyscall_mode __ro_after_init =
#ifdef CONFIG_LEGACY_VSYSCALL_NONE
NONE;
#elif defined(CONFIG_LEGACY_VSYSCALL_XONLY)
XONLY;
#else
#error VSYSCALL config is broken
#endif
static int __init vsyscall_setup(char *str)
{
if (str) {
if (!strcmp("emulate", str))
vsyscall_mode = EMULATE;
else if (!strcmp("xonly", str))
vsyscall_mode = XONLY;
else if (!strcmp("none", str))
vsyscall_mode = NONE;
else
return -EINVAL;
return 0;
}
return -EINVAL;
}
early_param("vsyscall", vsyscall_setup);
static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
const char *message)
{
if (!show_unhandled_signals)
return;
printk_ratelimited("%s%s[%d] %s ip:%lx cs:%x sp:%lx ax:%lx si:%lx di:%lx\n",
level, current->comm, task_pid_nr(current),
message, regs->ip, regs->cs,
regs->sp, regs->ax, regs->si, regs->di);
}
static int addr_to_vsyscall_nr(unsigned long addr)
{
int nr;
if ((addr & ~0xC00UL) != VSYSCALL_ADDR)
return -EINVAL;
nr = (addr & 0xC00UL) >> 10;
if (nr >= 3)
return -EINVAL;
return nr;
}
static bool write_ok_or_segv(unsigned long ptr, size_t size)
{
if (!access_ok((void __user *)ptr, size)) { struct thread_struct *thread = ¤t->thread;
thread->error_code = X86_PF_USER | X86_PF_WRITE;
thread->cr2 = ptr;
thread->trap_nr = X86_TRAP_PF;
force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)ptr);
return false;
} else {
return true;
}
}
bool emulate_vsyscall(unsigned long error_code,
struct pt_regs *regs, unsigned long address)
{
unsigned long caller;
int vsyscall_nr, syscall_nr, tmp;
long ret;
unsigned long orig_dx;
/* Write faults or kernel-privilege faults never get fixed up. */
if ((error_code & (X86_PF_WRITE | X86_PF_USER)) != X86_PF_USER) return false; if (!(error_code & X86_PF_INSTR)) {
/* Failed vsyscall read */
if (vsyscall_mode == EMULATE)
return false;
/*
* User code tried and failed to read the vsyscall page.
*/
warn_bad_vsyscall(KERN_INFO, regs, "vsyscall read attempt denied -- look up the vsyscall kernel parameter if you need a workaround");
return false;
}
/*
* No point in checking CS -- the only way to get here is a user mode
* trap to a high address, which means that we're in 64-bit user code.
*/
WARN_ON_ONCE(address != regs->ip); if (vsyscall_mode == NONE) { warn_bad_vsyscall(KERN_INFO, regs,
"vsyscall attempted with vsyscall=none");
return false;
}
vsyscall_nr = addr_to_vsyscall_nr(address); trace_emulate_vsyscall(vsyscall_nr); if (vsyscall_nr < 0) { warn_bad_vsyscall(KERN_WARNING, regs,
"misaligned vsyscall (exploit attempt or buggy program) -- look up the vsyscall kernel parameter if you need a workaround");
goto sigsegv;
}
if (get_user(caller, (unsigned long __user *)regs->sp) != 0) { warn_bad_vsyscall(KERN_WARNING, regs,
"vsyscall with bad stack (exploit attempt?)");
goto sigsegv;
}
/*
* Check for access_ok violations and find the syscall nr.
*
* NULL is a valid user pointer (in the access_ok sense) on 32-bit and
* 64-bit, so we don't need to special-case it here. For all the
* vsyscalls, NULL means "don't write anything" not "write it at
* address 0".
*/
switch (vsyscall_nr) {
case 0:
if (!write_ok_or_segv(regs->di, sizeof(struct __kernel_old_timeval)) || !write_ok_or_segv(regs->si, sizeof(struct timezone))) {
ret = -EFAULT;
goto check_fault;
}
syscall_nr = __NR_gettimeofday;
break;
case 1:
if (!write_ok_or_segv(regs->di, sizeof(__kernel_old_time_t))) {
ret = -EFAULT;
goto check_fault;
}
syscall_nr = __NR_time;
break;
case 2:
if (!write_ok_or_segv(regs->di, sizeof(unsigned)) || !write_ok_or_segv(regs->si, sizeof(unsigned))) {
ret = -EFAULT;
goto check_fault;
}
syscall_nr = __NR_getcpu;
break;
}
/*
* Handle seccomp. regs->ip must be the original value.
* See seccomp_send_sigsys and Documentation/userspace-api/seccomp_filter.rst.
*
* We could optimize the seccomp disabled case, but performance
* here doesn't matter.
*/
regs->orig_ax = syscall_nr;
regs->ax = -ENOSYS;
tmp = secure_computing(); if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) { warn_bad_vsyscall(KERN_DEBUG, regs,
"seccomp tried to change syscall nr or ip");
force_exit_sig(SIGSYS);
return true;
}
regs->orig_ax = -1;
if (tmp)
goto do_ret; /* skip requested */
/*
* With a real vsyscall, page faults cause SIGSEGV.
*/
ret = -EFAULT;
switch (vsyscall_nr) {
case 0:
/* this decodes regs->di and regs->si on its own */
ret = __x64_sys_gettimeofday(regs);
break;
case 1:
/* this decodes regs->di on its own */
ret = __x64_sys_time(regs);
break;
case 2:
/* while we could clobber regs->dx, we didn't in the past... */
orig_dx = regs->dx;
regs->dx = 0;
/* this decodes regs->di, regs->si and regs->dx on its own */
ret = __x64_sys_getcpu(regs);
regs->dx = orig_dx;
break;
}
check_fault:
if (ret == -EFAULT) {
/* Bad news -- userspace fed a bad pointer to a vsyscall. */
warn_bad_vsyscall(KERN_INFO, regs,
"vsyscall fault (exploit attempt?)");
goto sigsegv;
}
regs->ax = ret;
do_ret:
/* Emulate a ret instruction. */
regs->ip = caller;
regs->sp += 8;
return true;
sigsegv:
force_sig(SIGSEGV);
return true;
}
/*
* A pseudo VMA to allow ptrace access for the vsyscall page. This only
* covers the 64bit vsyscall page now. 32bit has a real VMA now and does
* not need special handling anymore:
*/
static const char *gate_vma_name(struct vm_area_struct *vma)
{
return "[vsyscall]";
}
static const struct vm_operations_struct gate_vma_ops = {
.name = gate_vma_name,
};
static struct vm_area_struct gate_vma __ro_after_init = {
.vm_start = VSYSCALL_ADDR,
.vm_end = VSYSCALL_ADDR + PAGE_SIZE,
.vm_page_prot = PAGE_READONLY_EXEC,
.vm_flags = VM_READ | VM_EXEC,
.vm_ops = &gate_vma_ops,
};
struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
#ifdef CONFIG_COMPAT
if (!mm || !test_bit(MM_CONTEXT_HAS_VSYSCALL, &mm->context.flags))
return NULL;
#endif
if (vsyscall_mode == NONE)
return NULL;
return &gate_vma;
}
int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma = get_gate_vma(mm);
if (!vma)
return 0;
return (addr >= vma->vm_start) && (addr < vma->vm_end);
}
/*
* Use this when you have no reliable mm, typically from interrupt
* context. It is less reliable than using a task's mm and may give
* false positives.
*/
int in_gate_area_no_mm(unsigned long addr)
{
return vsyscall_mode != NONE && (addr & PAGE_MASK) == VSYSCALL_ADDR;
}
/*
* The VSYSCALL page is the only user-accessible page in the kernel address
* range. Normally, the kernel page tables can have _PAGE_USER clear, but
* the tables covering VSYSCALL_ADDR need _PAGE_USER set if vsyscalls
* are enabled.
*
* Some day we may create a "minimal" vsyscall mode in which we emulate
* vsyscalls but leave the page not present. If so, we skip calling
* this.
*/
void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset_pgd(root, VSYSCALL_ADDR);
set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
p4d = p4d_offset(pgd, VSYSCALL_ADDR);
#if CONFIG_PGTABLE_LEVELS >= 5
set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
#endif
pud = pud_offset(p4d, VSYSCALL_ADDR);
set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
pmd = pmd_offset(pud, VSYSCALL_ADDR);
set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_USER));
}
void __init map_vsyscall(void)
{
extern char __vsyscall_page;
unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);
/*
* For full emulation, the page needs to exist for real. In
* execute-only mode, there is no PTE at all backing the vsyscall
* page.
*/
if (vsyscall_mode == EMULATE) {
__set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall,
PAGE_KERNEL_VVAR);
set_vsyscall_pgtable_user_bits(swapper_pg_dir);
}
if (vsyscall_mode == XONLY)
vm_flags_init(&gate_vma, VM_EXEC);
BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_PAGE) !=
(unsigned long)VSYSCALL_ADDR);
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef BLK_MQ_H
#define BLK_MQ_H
#include <linux/blkdev.h>
#include <linux/sbitmap.h>
#include <linux/lockdep.h>
#include <linux/scatterlist.h>
#include <linux/prefetch.h>
#include <linux/srcu.h>
#include <linux/rw_hint.h>
struct blk_mq_tags;
struct blk_flush_queue;
#define BLKDEV_MIN_RQ 4
#define BLKDEV_DEFAULT_RQ 128
enum rq_end_io_ret {
RQ_END_IO_NONE,
RQ_END_IO_FREE,
};
typedef enum rq_end_io_ret (rq_end_io_fn)(struct request *, blk_status_t);
/*
* request flags */
typedef __u32 __bitwise req_flags_t;
/* drive already may have started this one */
#define RQF_STARTED ((__force req_flags_t)(1 << 1))
/* request for flush sequence */
#define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))
/* merge of different types, fail separately */
#define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))
/* don't call prep for this one */
#define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
/* use hctx->sched_tags */
#define RQF_SCHED_TAGS ((__force req_flags_t)(1 << 8))
/* use an I/O scheduler for this request */
#define RQF_USE_SCHED ((__force req_flags_t)(1 << 9))
/* vaguely specified driver internal error. Ignored by the block layer */
#define RQF_FAILED ((__force req_flags_t)(1 << 10))
/* don't warn about errors */
#define RQF_QUIET ((__force req_flags_t)(1 << 11))
/* account into disk and partition IO statistics */
#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
/* runtime pm request */
#define RQF_PM ((__force req_flags_t)(1 << 15))
/* on IO scheduler merge hash */
#define RQF_HASHED ((__force req_flags_t)(1 << 16))
/* track IO completion time */
#define RQF_STATS ((__force req_flags_t)(1 << 17))
/* Look at ->special_vec for the actual data payload instead of the
bio chain. */
#define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))
/* The per-zone write lock is held for this request */
#define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19))
/* ->timeout has been called, don't expire again */
#define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21))
#define RQF_RESV ((__force req_flags_t)(1 << 23))
/* flags that prevent us from merging requests: */
#define RQF_NOMERGE_FLAGS \
(RQF_STARTED | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
enum mq_rq_state {
MQ_RQ_IDLE = 0,
MQ_RQ_IN_FLIGHT = 1,
MQ_RQ_COMPLETE = 2,
};
/*
* Try to put the fields that are referenced together in the same cacheline.
*
* If you modify this structure, make sure to update blk_rq_init() and
* especially blk_mq_rq_ctx_init() to take care of the added fields.
*/
struct request {
struct request_queue *q;
struct blk_mq_ctx *mq_ctx;
struct blk_mq_hw_ctx *mq_hctx;
blk_opf_t cmd_flags; /* op and common flags */
req_flags_t rq_flags;
int tag;
int internal_tag;
unsigned int timeout;
/* the following two fields are internal, NEVER access directly */
unsigned int __data_len; /* total data len */
sector_t __sector; /* sector cursor */
struct bio *bio;
struct bio *biotail;
union {
struct list_head queuelist;
struct request *rq_next;
};
struct block_device *part;
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
/* Time that the first bio started allocating this request. */
u64 alloc_time_ns;
#endif
/* Time that this request was allocated for this IO. */
u64 start_time_ns;
/* Time that I/O was submitted to the device. */
u64 io_start_time_ns;
#ifdef CONFIG_BLK_WBT
unsigned short wbt_flags;
#endif
/*
* rq sectors used for blk stats. It has the same value
* with blk_rq_sectors(rq), except that it never be zeroed
* by completion.
*/
unsigned short stats_sectors;
/*
* Number of scatter-gather DMA addr+len pairs after
* physical address coalescing is performed.
*/
unsigned short nr_phys_segments;
#ifdef CONFIG_BLK_DEV_INTEGRITY
unsigned short nr_integrity_segments;
#endif
#ifdef CONFIG_BLK_INLINE_ENCRYPTION
struct bio_crypt_ctx *crypt_ctx;
struct blk_crypto_keyslot *crypt_keyslot;
#endif
enum rw_hint write_hint;
unsigned short ioprio;
enum mq_rq_state state;
atomic_t ref;
unsigned long deadline;
/*
* The hash is used inside the scheduler, and killed once the
* request reaches the dispatch list. The ipi_list is only used
* to queue the request for softirq completion, which is long
* after the request has been unhashed (and even removed from
* the dispatch list).
*/
union {
struct hlist_node hash; /* merge hash */
struct llist_node ipi_list;
};
/*
* The rb_node is only used inside the io scheduler, requests
* are pruned when moved to the dispatch queue. special_vec must
* only be used if RQF_SPECIAL_PAYLOAD is set, and those cannot be
* insert into an IO scheduler.
*/
union {
struct rb_node rb_node; /* sort/lookup */
struct bio_vec special_vec;
};
/*
* Three pointers are available for the IO schedulers, if they need
* more they have to dynamically allocate it.
*/
struct {
struct io_cq *icq;
void *priv[2];
} elv;
struct {
unsigned int seq;
rq_end_io_fn *saved_end_io;
} flush;
u64 fifo_time;
/*
* completion callback.
*/
rq_end_io_fn *end_io;
void *end_io_data;
};
static inline enum req_op req_op(const struct request *req)
{
return req->cmd_flags & REQ_OP_MASK;
}
static inline bool blk_rq_is_passthrough(struct request *rq)
{
return blk_op_is_passthrough(rq->cmd_flags);
}
static inline unsigned short req_get_ioprio(struct request *req)
{
return req->ioprio;
}
#define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
#define rq_dma_dir(rq) \
(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
#define rq_list_add(listptr, rq) do { \
(rq)->rq_next = *(listptr); \
*(listptr) = rq; \
} while (0)
#define rq_list_add_tail(lastpptr, rq) do { \
(rq)->rq_next = NULL; \
**(lastpptr) = rq; \
*(lastpptr) = &rq->rq_next; \
} while (0)
#define rq_list_pop(listptr) \
({ \
struct request *__req = NULL; \
if ((listptr) && *(listptr)) { \
__req = *(listptr); \
*(listptr) = __req->rq_next; \
} \
__req; \
})
#define rq_list_peek(listptr) \
({ \
struct request *__req = NULL; \
if ((listptr) && *(listptr)) \
__req = *(listptr); \
__req; \
})
#define rq_list_for_each(listptr, pos) \
for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))
#define rq_list_for_each_safe(listptr, pos, nxt) \
for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos); \
pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL)
#define rq_list_next(rq) (rq)->rq_next
#define rq_list_empty(list) ((list) == (struct request *) NULL)
/**
* rq_list_move() - move a struct request from one list to another
* @src: The source list @rq is currently in
* @dst: The destination list that @rq will be appended to
* @rq: The request to move
* @prev: The request preceding @rq in @src (NULL if @rq is the head)
*/
static inline void rq_list_move(struct request **src, struct request **dst,
struct request *rq, struct request *prev)
{
if (prev)
prev->rq_next = rq->rq_next;
else
*src = rq->rq_next;
rq_list_add(dst, rq);
}
/**
* enum blk_eh_timer_return - How the timeout handler should proceed
* @BLK_EH_DONE: The block driver completed the command or will complete it at
* a later time.
* @BLK_EH_RESET_TIMER: Reset the request timer and continue waiting for the
* request to complete.
*/
enum blk_eh_timer_return {
BLK_EH_DONE,
BLK_EH_RESET_TIMER,
};
#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
/**
* struct blk_mq_hw_ctx - State for a hardware queue facing the hardware
* block device
*/
struct blk_mq_hw_ctx {
struct {
/** @lock: Protects the dispatch list. */
spinlock_t lock;
/**
* @dispatch: Used for requests that are ready to be
* dispatched to the hardware but for some reason (e.g. lack of
* resources) could not be sent to the hardware. As soon as the
* driver can send new requests, requests at this list will
* be sent first for a fairer dispatch.
*/
struct list_head dispatch;
/**
* @state: BLK_MQ_S_* flags. Defines the state of the hw
* queue (active, scheduled to restart, stopped).
*/
unsigned long state;
} ____cacheline_aligned_in_smp;
/**
* @run_work: Used for scheduling a hardware queue run at a later time.
*/
struct delayed_work run_work;
/** @cpumask: Map of available CPUs where this hctx can run. */
cpumask_var_t cpumask;
/**
* @next_cpu: Used by blk_mq_hctx_next_cpu() for round-robin CPU
* selection from @cpumask.
*/
int next_cpu;
/**
* @next_cpu_batch: Counter of how many works left in the batch before
* changing to the next CPU.
*/
int next_cpu_batch;
/** @flags: BLK_MQ_F_* flags. Defines the behaviour of the queue. */
unsigned long flags;
/**
* @sched_data: Pointer owned by the IO scheduler attached to a request
* queue. It's up to the IO scheduler how to use this pointer.
*/
void *sched_data;
/**
* @queue: Pointer to the request queue that owns this hardware context.
*/
struct request_queue *queue;
/** @fq: Queue of requests that need to perform a flush operation. */
struct blk_flush_queue *fq;
/**
* @driver_data: Pointer to data owned by the block driver that created
* this hctx
*/
void *driver_data;
/**
* @ctx_map: Bitmap for each software queue. If bit is on, there is a
* pending request in that software queue.
*/
struct sbitmap ctx_map;
/**
* @dispatch_from: Software queue to be used when no scheduler was
* selected.
*/
struct blk_mq_ctx *dispatch_from;
/**
* @dispatch_busy: Number used by blk_mq_update_dispatch_busy() to
* decide if the hw_queue is busy using Exponential Weighted Moving
* Average algorithm.
*/
unsigned int dispatch_busy;
/** @type: HCTX_TYPE_* flags. Type of hardware queue. */
unsigned short type;
/** @nr_ctx: Number of software queues. */
unsigned short nr_ctx;
/** @ctxs: Array of software queues. */
struct blk_mq_ctx **ctxs;
/** @dispatch_wait_lock: Lock for dispatch_wait queue. */
spinlock_t dispatch_wait_lock;
/**
* @dispatch_wait: Waitqueue to put requests when there is no tag
* available at the moment, to wait for another try in the future.
*/
wait_queue_entry_t dispatch_wait;
/**
* @wait_index: Index of next available dispatch_wait queue to insert
* requests.
*/
atomic_t wait_index;
/**
* @tags: Tags owned by the block driver. A tag at this set is only
* assigned when a request is dispatched from a hardware queue.
*/
struct blk_mq_tags *tags;
/**
* @sched_tags: Tags owned by I/O scheduler. If there is an I/O
* scheduler associated with a request queue, a tag is assigned when
* that request is allocated. Else, this member is not used.
*/
struct blk_mq_tags *sched_tags;
/** @numa_node: NUMA node the storage adapter has been connected to. */
unsigned int numa_node;
/** @queue_num: Index of this hardware queue. */
unsigned int queue_num;
/**
* @nr_active: Number of active requests. Only used when a tag set is
* shared across request queues.
*/
atomic_t nr_active;
/** @cpuhp_online: List to store request if CPU is going to die */
struct hlist_node cpuhp_online;
/** @cpuhp_dead: List to store request if some CPU die. */
struct hlist_node cpuhp_dead;
/** @kobj: Kernel object for sysfs. */
struct kobject kobj;
#ifdef CONFIG_BLK_DEBUG_FS
/**
* @debugfs_dir: debugfs directory for this hardware queue. Named
* as cpu<cpu_number>.
*/
struct dentry *debugfs_dir;
/** @sched_debugfs_dir: debugfs directory for the scheduler. */
struct dentry *sched_debugfs_dir;
#endif
/**
* @hctx_list: if this hctx is not in use, this is an entry in
* q->unused_hctx_list.
*/
struct list_head hctx_list;
};
/**
* struct blk_mq_queue_map - Map software queues to hardware queues
* @mq_map: CPU ID to hardware queue index map. This is an array
* with nr_cpu_ids elements. Each element has a value in the range
* [@queue_offset, @queue_offset + @nr_queues).
* @nr_queues: Number of hardware queues to map CPU IDs onto.
* @queue_offset: First hardware queue to map onto. Used by the PCIe NVMe
* driver to map each hardware queue type (enum hctx_type) onto a distinct
* set of hardware queues.
*/
struct blk_mq_queue_map {
unsigned int *mq_map;
unsigned int nr_queues;
unsigned int queue_offset;
};
/**
* enum hctx_type - Type of hardware queue
* @HCTX_TYPE_DEFAULT: All I/O not otherwise accounted for.
* @HCTX_TYPE_READ: Just for READ I/O.
* @HCTX_TYPE_POLL: Polled I/O of any kind.
* @HCTX_MAX_TYPES: Number of types of hctx.
*/
enum hctx_type {
HCTX_TYPE_DEFAULT,
HCTX_TYPE_READ,
HCTX_TYPE_POLL,
HCTX_MAX_TYPES,
};
/**
* struct blk_mq_tag_set - tag set that can be shared between request queues
* @ops: Pointers to functions that implement block driver behavior.
* @map: One or more ctx -> hctx mappings. One map exists for each
* hardware queue type (enum hctx_type) that the driver wishes
* to support. There are no restrictions on maps being of the
* same size, and it's perfectly legal to share maps between
* types.
* @nr_maps: Number of elements in the @map array. A number in the range
* [1, HCTX_MAX_TYPES].
* @nr_hw_queues: Number of hardware queues supported by the block driver that
* owns this data structure.
* @queue_depth: Number of tags per hardware queue, reserved tags included.
* @reserved_tags: Number of tags to set aside for BLK_MQ_REQ_RESERVED tag
* allocations.
* @cmd_size: Number of additional bytes to allocate per request. The block
* driver owns these additional bytes.
* @numa_node: NUMA node the storage adapter has been connected to.
* @timeout: Request processing timeout in jiffies.
* @flags: Zero or more BLK_MQ_F_* flags.
* @driver_data: Pointer to data owned by the block driver that created this
* tag set.
* @tags: Tag sets. One tag set per hardware queue. Has @nr_hw_queues
* elements.
* @shared_tags:
* Shared set of tags. Has @nr_hw_queues elements. If set,
* shared by all @tags.
* @tag_list_lock: Serializes tag_list accesses.
* @tag_list: List of the request queues that use this tag set. See also
* request_queue.tag_set_list.
* @srcu: Use as lock when type of the request queue is blocking
* (BLK_MQ_F_BLOCKING).
*/
struct blk_mq_tag_set {
const struct blk_mq_ops *ops;
struct blk_mq_queue_map map[HCTX_MAX_TYPES];
unsigned int nr_maps;
unsigned int nr_hw_queues;
unsigned int queue_depth;
unsigned int reserved_tags;
unsigned int cmd_size;
int numa_node;
unsigned int timeout;
unsigned int flags;
void *driver_data;
struct blk_mq_tags **tags;
struct blk_mq_tags *shared_tags;
struct mutex tag_list_lock;
struct list_head tag_list;
struct srcu_struct *srcu;
};
/**
* struct blk_mq_queue_data - Data about a request inserted in a queue
*
* @rq: Request pointer.
* @last: If it is the last request in the queue.
*/
struct blk_mq_queue_data {
struct request *rq;
bool last;
};
typedef bool (busy_tag_iter_fn)(struct request *, void *);
/**
* struct blk_mq_ops - Callback functions that implements block driver
* behaviour.
*/
struct blk_mq_ops {
/**
* @queue_rq: Queue a new request from block IO.
*/
blk_status_t (*queue_rq)(struct blk_mq_hw_ctx *,
const struct blk_mq_queue_data *);
/**
* @commit_rqs: If a driver uses bd->last to judge when to submit
* requests to hardware, it must define this function. In case of errors
* that make us stop issuing further requests, this hook serves the
* purpose of kicking the hardware (which the last request otherwise
* would have done).
*/
void (*commit_rqs)(struct blk_mq_hw_ctx *);
/**
* @queue_rqs: Queue a list of new requests. Driver is guaranteed
* that each request belongs to the same queue. If the driver doesn't
* empty the @rqlist completely, then the rest will be queued
* individually by the block layer upon return.
*/
void (*queue_rqs)(struct request **rqlist);
/**
* @get_budget: Reserve budget before queue request, once .queue_rq is
* run, it is driver's responsibility to release the
* reserved budget. Also we have to handle failure case
* of .get_budget for avoiding I/O deadlock.
*/
int (*get_budget)(struct request_queue *);
/**
* @put_budget: Release the reserved budget.
*/
void (*put_budget)(struct request_queue *, int);
/**
* @set_rq_budget_token: store rq's budget token
*/
void (*set_rq_budget_token)(struct request *, int);
/**
* @get_rq_budget_token: retrieve rq's budget token
*/
int (*get_rq_budget_token)(struct request *);
/**
* @timeout: Called on request timeout.
*/
enum blk_eh_timer_return (*timeout)(struct request *);
/**
* @poll: Called to poll for completion of a specific tag.
*/
int (*poll)(struct blk_mq_hw_ctx *, struct io_comp_batch *);
/**
* @complete: Mark the request as complete.
*/
void (*complete)(struct request *);
/**
* @init_hctx: Called when the block layer side of a hardware queue has
* been set up, allowing the driver to allocate/init matching
* structures.
*/
int (*init_hctx)(struct blk_mq_hw_ctx *, void *, unsigned int);
/**
* @exit_hctx: Ditto for exit/teardown.
*/
void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int);
/**
* @init_request: Called for every command allocated by the block layer
* to allow the driver to set up driver specific data.
*
* Tag greater than or equal to queue_depth is for setting up
* flush request.
*/
int (*init_request)(struct blk_mq_tag_set *set, struct request *,
unsigned int, unsigned int);
/**
* @exit_request: Ditto for exit/teardown.
*/
void (*exit_request)(struct blk_mq_tag_set *set, struct request *,
unsigned int);
/**
* @cleanup_rq: Called before freeing one request which isn't completed
* yet, and usually for freeing the driver private data.
*/
void (*cleanup_rq)(struct request *);
/**
* @busy: If set, returns whether or not this queue currently is busy.
*/
bool (*busy)(struct request_queue *);
/**
* @map_queues: This allows drivers specify their own queue mapping by
* overriding the setup-time function that builds the mq_map.
*/
void (*map_queues)(struct blk_mq_tag_set *set);
#ifdef CONFIG_BLK_DEBUG_FS
/**
* @show_rq: Used by the debugfs implementation to show driver-specific
* information about a request.
*/
void (*show_rq)(struct seq_file *m, struct request *rq);
#endif
};
enum {
BLK_MQ_F_SHOULD_MERGE = 1 << 0,
BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1,
/*
* Set when this device requires underlying blk-mq device for
* completing IO:
*/
BLK_MQ_F_STACKING = 1 << 2,
BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3,
BLK_MQ_F_BLOCKING = 1 << 5,
/* Do not allow an I/O scheduler to be configured. */
BLK_MQ_F_NO_SCHED = 1 << 6,
/*
* Select 'none' during queue registration in case of a single hwq
* or shared hwqs instead of 'mq-deadline'.
*/
BLK_MQ_F_NO_SCHED_BY_DEFAULT = 1 << 7,
BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
BLK_MQ_F_ALLOC_POLICY_BITS = 1,
BLK_MQ_S_STOPPED = 0,
BLK_MQ_S_TAG_ACTIVE = 1,
BLK_MQ_S_SCHED_RESTART = 2,
/* hw queue is inactive after all its CPUs become offline */
BLK_MQ_S_INACTIVE = 3,
BLK_MQ_MAX_DEPTH = 10240,
BLK_MQ_CPU_WORK_BATCH = 8,
};
#define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \
((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \
((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1))
#define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \
((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \
<< BLK_MQ_F_ALLOC_POLICY_START_BIT)
#define BLK_MQ_NO_HCTX_IDX (-1U)
struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,
struct queue_limits *lim, void *queuedata,
struct lock_class_key *lkclass);
#define blk_mq_alloc_disk(set, lim, queuedata) \
({ \
static struct lock_class_key __key; \
\
__blk_mq_alloc_disk(set, lim, queuedata, &__key); \
})
struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
struct lock_class_key *lkclass);
struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set,
struct queue_limits *lim, void *queuedata);
int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
struct request_queue *q);
void blk_mq_destroy_queue(struct request_queue *);
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set);
int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
const struct blk_mq_ops *ops, unsigned int queue_depth,
unsigned int set_flags);
void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
void blk_mq_free_request(struct request *rq);
int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
unsigned int poll_flags);
bool blk_mq_queue_inflight(struct request_queue *q);
enum {
/* return when out of requests */
BLK_MQ_REQ_NOWAIT = (__force blk_mq_req_flags_t)(1 << 0),
/* allocate from reserved pool */
BLK_MQ_REQ_RESERVED = (__force blk_mq_req_flags_t)(1 << 1),
/* set RQF_PM */
BLK_MQ_REQ_PM = (__force blk_mq_req_flags_t)(1 << 2),
};
struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
blk_mq_req_flags_t flags);
struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
blk_opf_t opf, blk_mq_req_flags_t flags,
unsigned int hctx_idx);
/*
* Tag address space map.
*/
struct blk_mq_tags {
unsigned int nr_tags;
unsigned int nr_reserved_tags;
unsigned int active_queues;
struct sbitmap_queue bitmap_tags;
struct sbitmap_queue breserved_tags;
struct request **rqs;
struct request **static_rqs;
struct list_head page_list;
/*
* used to clear request reference in rqs[] before freeing one
* request pool
*/
spinlock_t lock;
};
static inline struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags,
unsigned int tag)
{
if (tag < tags->nr_tags) {
prefetch(tags->rqs[tag]);
return tags->rqs[tag];
}
return NULL;
}
enum {
BLK_MQ_UNIQUE_TAG_BITS = 16,
BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1,
};
u32 blk_mq_unique_tag(struct request *rq);
static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag)
{
return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS;
}
static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag)
{
return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;
}
/**
* blk_mq_rq_state() - read the current MQ_RQ_* state of a request
* @rq: target request.
*/
static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)
{
return READ_ONCE(rq->state);
}
static inline int blk_mq_request_started(struct request *rq)
{
return blk_mq_rq_state(rq) != MQ_RQ_IDLE;
}
static inline int blk_mq_request_completed(struct request *rq)
{
return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE;
}
/*
*
* Set the state to complete when completing a request from inside ->queue_rq.
* This is used by drivers that want to ensure special complete actions that
* need access to the request are called on failure, e.g. by nvme for
* multipathing.
*/
static inline void blk_mq_set_request_complete(struct request *rq)
{
WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
}
/*
* Complete the request directly instead of deferring it to softirq or
* completing it another CPU. Useful in preemptible instead of an interrupt.
*/
static inline void blk_mq_complete_request_direct(struct request *rq,
void (*complete)(struct request *rq))
{
WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
complete(rq);
}
void blk_mq_start_request(struct request *rq);
void blk_mq_end_request(struct request *rq, blk_status_t error);
void __blk_mq_end_request(struct request *rq, blk_status_t error);
void blk_mq_end_request_batch(struct io_comp_batch *ib);
/*
* Only need start/end time stamping if we have iostat or
* blk stats enabled, or using an IO scheduler.
*/
static inline bool blk_mq_need_time_stamp(struct request *rq)
{
/*
* passthrough io doesn't use iostat accounting, cgroup stats
* and io scheduler functionalities.
*/
if (blk_rq_is_passthrough(rq))
return false;
return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED));
}
static inline bool blk_mq_is_reserved_rq(struct request *rq)
{
return rq->rq_flags & RQF_RESV;
}
/*
* Batched completions only work when there is no I/O error and no special
* ->end_io handler.
*/
static inline bool blk_mq_add_to_batch(struct request *req,
struct io_comp_batch *iob, int ioerror,
void (*complete)(struct io_comp_batch *))
{
/*
* blk_mq_end_request_batch() can't end request allocated from
* sched tags
*/
if (!iob || (req->rq_flags & RQF_SCHED_TAGS) || ioerror ||
(req->end_io && !blk_rq_is_passthrough(req)))
return false;
if (!iob->complete)
iob->complete = complete;
else if (iob->complete != complete)
return false;
iob->need_ts |= blk_mq_need_time_stamp(req);
rq_list_add(&iob->req_list, req);
return true;
}
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
void blk_mq_kick_requeue_list(struct request_queue *q);
void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
void blk_mq_complete_request(struct request *rq);
bool blk_mq_complete_request_remote(struct request *rq);
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
void blk_mq_stop_hw_queues(struct request_queue *q);
void blk_mq_start_hw_queues(struct request_queue *q);
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
void blk_mq_quiesce_queue(struct request_queue *q);
void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set);
void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set);
void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set);
void blk_mq_unquiesce_queue(struct request_queue *q);
void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
void blk_mq_run_hw_queues(struct request_queue *q, bool async);
void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs);
void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
busy_tag_iter_fn *fn, void *priv);
void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset);
void blk_mq_freeze_queue(struct request_queue *q);
void blk_mq_unfreeze_queue(struct request_queue *q);
void blk_freeze_queue_start(struct request_queue *q);
void blk_mq_freeze_queue_wait(struct request_queue *q);
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
unsigned long timeout);
void blk_mq_map_queues(struct blk_mq_queue_map *qmap);
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
void blk_mq_quiesce_queue_nowait(struct request_queue *q);
unsigned int blk_mq_rq_cpu(struct request *rq);
bool __blk_should_fake_timeout(struct request_queue *q);
static inline bool blk_should_fake_timeout(struct request_queue *q)
{
if (IS_ENABLED(CONFIG_FAIL_IO_TIMEOUT) &&
test_bit(QUEUE_FLAG_FAIL_IO, &q->queue_flags))
return __blk_should_fake_timeout(q);
return false;
}
/**
* blk_mq_rq_from_pdu - cast a PDU to a request
* @pdu: the PDU (Protocol Data Unit) to be casted
*
* Return: request
*
* Driver command data is immediately after the request. So subtract request
* size to get back to the original request.
*/
static inline struct request *blk_mq_rq_from_pdu(void *pdu)
{
return pdu - sizeof(struct request);
}
/**
* blk_mq_rq_to_pdu - cast a request to a PDU
* @rq: the request to be casted
*
* Return: pointer to the PDU
*
* Driver command data is immediately after the request. So add request to get
* the PDU.
*/
static inline void *blk_mq_rq_to_pdu(struct request *rq)
{
return rq + 1;
}
#define queue_for_each_hw_ctx(q, hctx, i) \
xa_for_each(&(q)->hctx_table, (i), (hctx))
#define hctx_for_each_ctx(hctx, ctx, i) \
for ((i) = 0; (i) < (hctx)->nr_ctx && \
({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++)
static inline void blk_mq_cleanup_rq(struct request *rq)
{
if (rq->q->mq_ops->cleanup_rq)
rq->q->mq_ops->cleanup_rq(rq);
}
static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,
unsigned int nr_segs)
{
rq->nr_phys_segments = nr_segs;
rq->__data_len = bio->bi_iter.bi_size;
rq->bio = rq->biotail = bio;
rq->ioprio = bio_prio(bio);
}
void blk_mq_hctx_set_fq_lock_class(struct blk_mq_hw_ctx *hctx,
struct lock_class_key *key);
static inline bool rq_is_sync(struct request *rq)
{
return op_is_sync(rq->cmd_flags);
}
void blk_rq_init(struct request_queue *q, struct request *rq);
int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
struct bio_set *bs, gfp_t gfp_mask,
int (*bio_ctr)(struct bio *, struct bio *, void *), void *data);
void blk_rq_unprep_clone(struct request *rq);
blk_status_t blk_insert_cloned_request(struct request *rq);
struct rq_map_data {
struct page **pages;
unsigned long offset;
unsigned short page_order;
unsigned short nr_entries;
bool null_mapped;
bool from_user;
};
int blk_rq_map_user(struct request_queue *, struct request *,
struct rq_map_data *, void __user *, unsigned long, gfp_t);
int blk_rq_map_user_io(struct request *, struct rq_map_data *,
void __user *, unsigned long, gfp_t, bool, int, bool, int);
int blk_rq_map_user_iov(struct request_queue *, struct request *,
struct rq_map_data *, const struct iov_iter *, gfp_t);
int blk_rq_unmap_user(struct bio *);
int blk_rq_map_kern(struct request_queue *, struct request *, void *,
unsigned int, gfp_t);
int blk_rq_append_bio(struct request *rq, struct bio *bio);
void blk_execute_rq_nowait(struct request *rq, bool at_head);
blk_status_t blk_execute_rq(struct request *rq, bool at_head);
bool blk_rq_is_poll(struct request *rq);
struct req_iterator {
struct bvec_iter iter;
struct bio *bio;
};
#define __rq_for_each_bio(_bio, rq) \
if ((rq->bio)) \
for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
#define rq_for_each_segment(bvl, _rq, _iter) \
__rq_for_each_bio(_iter.bio, _rq) \
bio_for_each_segment(bvl, _iter.bio, _iter.iter)
#define rq_for_each_bvec(bvl, _rq, _iter) \
__rq_for_each_bio(_iter.bio, _rq) \
bio_for_each_bvec(bvl, _iter.bio, _iter.iter)
#define rq_iter_last(bvec, _iter) \
(_iter.bio->bi_next == NULL && \
bio_iter_last(bvec, _iter.iter))
/*
* blk_rq_pos() : the current sector
* blk_rq_bytes() : bytes left in the entire request
* blk_rq_cur_bytes() : bytes left in the current segment
* blk_rq_sectors() : sectors left in the entire request
* blk_rq_cur_sectors() : sectors left in the current segment
* blk_rq_stats_sectors() : sectors of the entire request used for stats
*/
static inline sector_t blk_rq_pos(const struct request *rq)
{
return rq->__sector;
}
static inline unsigned int blk_rq_bytes(const struct request *rq)
{
return rq->__data_len;
}
static inline int blk_rq_cur_bytes(const struct request *rq)
{
if (!rq->bio)
return 0;
if (!bio_has_data(rq->bio)) /* dataless requests such as discard */
return rq->bio->bi_iter.bi_size;
return bio_iovec(rq->bio).bv_len;
}
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
return blk_rq_bytes(rq) >> SECTOR_SHIFT;
}
static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
}
static inline unsigned int blk_rq_stats_sectors(const struct request *rq)
{
return rq->stats_sectors;
}
/*
* Some commands like WRITE SAME have a payload or data transfer size which
* is different from the size of the request. Any driver that supports such
* commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
* calculate the data transfer size.
*/
static inline unsigned int blk_rq_payload_bytes(struct request *rq)
{
if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
return rq->special_vec.bv_len;
return blk_rq_bytes(rq);
}
/*
* Return the first full biovec in the request. The caller needs to check that
* there are any bvecs before calling this helper.
*/
static inline struct bio_vec req_bvec(struct request *rq)
{
if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
return rq->special_vec;
return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);
}
static inline unsigned int blk_rq_count_bios(struct request *rq)
{
unsigned int nr_bios = 0;
struct bio *bio;
__rq_for_each_bio(bio, rq)
nr_bios++;
return nr_bios;
}
void blk_steal_bios(struct bio_list *list, struct request *rq);
/*
* Request completion related functions.
*
* blk_update_request() completes given number of bytes and updates
* the request without completing it.
*/
bool blk_update_request(struct request *rq, blk_status_t error,
unsigned int nr_bytes);
void blk_abort_request(struct request *);
/*
* Number of physical segments as sent to the device.
*
* Normally this is the number of discontiguous data segments sent by the
* submitter. But for data-less command like discard we might have no
* actual data segments submitted, but the driver might have to add it's
* own special payload. In that case we still return 1 here so that this
* special payload will be mapped.
*/
static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
{
if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
return 1;
return rq->nr_phys_segments;
}
/*
* Number of discard segments (or ranges) the driver needs to fill in.
* Each discard bio merged into a request is counted as one segment.
*/
static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
{
return max_t(unsigned short, rq->nr_phys_segments, 1);
}
int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
struct scatterlist *sglist, struct scatterlist **last_sg);
static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,
struct scatterlist *sglist)
{
struct scatterlist *last_sg = NULL;
return __blk_rq_map_sg(q, rq, sglist, &last_sg);
}
void blk_dump_rq_flags(struct request *, char *);
#ifdef CONFIG_BLK_DEV_ZONED
static inline unsigned int blk_rq_zone_no(struct request *rq)
{
return disk_zone_no(rq->q->disk, blk_rq_pos(rq));
}
static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
{
return disk_zone_is_seq(rq->q->disk, blk_rq_pos(rq));
}
/**
* blk_rq_is_seq_zoned_write() - Check if @rq requires write serialization.
* @rq: Request to examine.
*
* Note: REQ_OP_ZONE_APPEND requests do not require serialization.
*/
static inline bool blk_rq_is_seq_zoned_write(struct request *rq)
{
return op_needs_zoned_write_locking(req_op(rq)) &&
blk_rq_zone_is_seq(rq);
}
bool blk_req_needs_zone_write_lock(struct request *rq);
bool blk_req_zone_write_trylock(struct request *rq);
void __blk_req_zone_write_lock(struct request *rq);
void __blk_req_zone_write_unlock(struct request *rq);
static inline void blk_req_zone_write_lock(struct request *rq)
{
if (blk_req_needs_zone_write_lock(rq))
__blk_req_zone_write_lock(rq);
}
static inline void blk_req_zone_write_unlock(struct request *rq)
{
if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
__blk_req_zone_write_unlock(rq);
}
static inline bool blk_req_zone_is_write_locked(struct request *rq)
{
return rq->q->disk->seq_zones_wlock &&
test_bit(blk_rq_zone_no(rq), rq->q->disk->seq_zones_wlock);
}
static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
{
if (!blk_req_needs_zone_write_lock(rq))
return true;
return !blk_req_zone_is_write_locked(rq);
}
#else /* CONFIG_BLK_DEV_ZONED */
static inline bool blk_rq_is_seq_zoned_write(struct request *rq)
{
return false;
}
static inline bool blk_req_needs_zone_write_lock(struct request *rq)
{
return false;
}
static inline void blk_req_zone_write_lock(struct request *rq)
{
}
static inline void blk_req_zone_write_unlock(struct request *rq)
{
}
static inline bool blk_req_zone_is_write_locked(struct request *rq)
{
return false;
}
static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
{
return true;
}
#endif /* CONFIG_BLK_DEV_ZONED */
#endif /* BLK_MQ_H */
// SPDX-License-Identifier: GPL-2.0
#define pr_fmt(fmt) "irq: " fmt
#include <linux/acpi.h>
#include <linux/debugfs.h>
#include <linux/hardirq.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/irqdomain.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/topology.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/fs.h>
static LIST_HEAD(irq_domain_list);
static DEFINE_MUTEX(irq_domain_mutex);
static struct irq_domain *irq_default_domain;
static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,
unsigned int nr_irqs, int node, void *arg,
bool realloc, const struct irq_affinity_desc *affinity);
static void irq_domain_check_hierarchy(struct irq_domain *domain);
static void irq_domain_free_one_irq(struct irq_domain *domain, unsigned int virq);
struct irqchip_fwid {
struct fwnode_handle fwnode;
unsigned int type;
char *name;
phys_addr_t *pa;
};
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
static void debugfs_add_domain_dir(struct irq_domain *d);
static void debugfs_remove_domain_dir(struct irq_domain *d);
#else
static inline void debugfs_add_domain_dir(struct irq_domain *d) { }
static inline void debugfs_remove_domain_dir(struct irq_domain *d) { }
#endif
static const char *irqchip_fwnode_get_name(const struct fwnode_handle *fwnode)
{
struct irqchip_fwid *fwid = container_of(fwnode, struct irqchip_fwid, fwnode);
return fwid->name;
}
const struct fwnode_operations irqchip_fwnode_ops = {
.get_name = irqchip_fwnode_get_name,
};
EXPORT_SYMBOL_GPL(irqchip_fwnode_ops);
/**
* __irq_domain_alloc_fwnode - Allocate a fwnode_handle suitable for
* identifying an irq domain
* @type: Type of irqchip_fwnode. See linux/irqdomain.h
* @id: Optional user provided id if name != NULL
* @name: Optional user provided domain name
* @pa: Optional user-provided physical address
*
* Allocate a struct irqchip_fwid, and return a pointer to the embedded
* fwnode_handle (or NULL on failure).
*
* Note: The types IRQCHIP_FWNODE_NAMED and IRQCHIP_FWNODE_NAMED_ID are
* solely to transport name information to irqdomain creation code. The
* node is not stored. For other types the pointer is kept in the irq
* domain struct.
*/
struct fwnode_handle *__irq_domain_alloc_fwnode(unsigned int type, int id,
const char *name,
phys_addr_t *pa)
{
struct irqchip_fwid *fwid;
char *n;
fwid = kzalloc(sizeof(*fwid), GFP_KERNEL);
switch (type) {
case IRQCHIP_FWNODE_NAMED:
n = kasprintf(GFP_KERNEL, "%s", name);
break;
case IRQCHIP_FWNODE_NAMED_ID:
n = kasprintf(GFP_KERNEL, "%s-%d", name, id);
break;
default:
n = kasprintf(GFP_KERNEL, "irqchip@%pa", pa);
break;
}
if (!fwid || !n) {
kfree(fwid);
kfree(n);
return NULL;
}
fwid->type = type;
fwid->name = n;
fwid->pa = pa;
fwnode_init(&fwid->fwnode, &irqchip_fwnode_ops);
return &fwid->fwnode;
}
EXPORT_SYMBOL_GPL(__irq_domain_alloc_fwnode);
/**
* irq_domain_free_fwnode - Free a non-OF-backed fwnode_handle
*
* Free a fwnode_handle allocated with irq_domain_alloc_fwnode.
*/
void irq_domain_free_fwnode(struct fwnode_handle *fwnode)
{
struct irqchip_fwid *fwid;
if (!fwnode || WARN_ON(!is_fwnode_irqchip(fwnode)))
return;
fwid = container_of(fwnode, struct irqchip_fwid, fwnode);
kfree(fwid->name);
kfree(fwid);
}
EXPORT_SYMBOL_GPL(irq_domain_free_fwnode);
static struct irq_domain *__irq_domain_create(struct fwnode_handle *fwnode,
unsigned int size,
irq_hw_number_t hwirq_max,
int direct_max,
const struct irq_domain_ops *ops,
void *host_data)
{
struct irqchip_fwid *fwid;
struct irq_domain *domain;
static atomic_t unknown_domains;
if (WARN_ON((size && direct_max) ||
(!IS_ENABLED(CONFIG_IRQ_DOMAIN_NOMAP) && direct_max) ||
(direct_max && (direct_max != hwirq_max))))
return NULL;
domain = kzalloc_node(struct_size(domain, revmap, size),
GFP_KERNEL, of_node_to_nid(to_of_node(fwnode)));
if (!domain)
return NULL;
if (is_fwnode_irqchip(fwnode)) {
fwid = container_of(fwnode, struct irqchip_fwid, fwnode);
switch (fwid->type) {
case IRQCHIP_FWNODE_NAMED:
case IRQCHIP_FWNODE_NAMED_ID:
domain->fwnode = fwnode;
domain->name = kstrdup(fwid->name, GFP_KERNEL);
if (!domain->name) {
kfree(domain);
return NULL;
}
domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
break;
default:
domain->fwnode = fwnode;
domain->name = fwid->name;
break;
}
} else if (is_of_node(fwnode) || is_acpi_device_node(fwnode) ||
is_software_node(fwnode)) {
char *name;
/*
* fwnode paths contain '/', which debugfs is legitimately
* unhappy about. Replace them with ':', which does
* the trick and is not as offensive as '\'...
*/
name = kasprintf(GFP_KERNEL, "%pfw", fwnode);
if (!name) {
kfree(domain);
return NULL;
}
domain->name = strreplace(name, '/', ':');
domain->fwnode = fwnode;
domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
}
if (!domain->name) {
if (fwnode)
pr_err("Invalid fwnode type for irqdomain\n");
domain->name = kasprintf(GFP_KERNEL, "unknown-%d",
atomic_inc_return(&unknown_domains));
if (!domain->name) {
kfree(domain);
return NULL;
}
domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
}
fwnode_handle_get(fwnode);
fwnode_dev_initialized(fwnode, true);
/* Fill structure */
INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL);
domain->ops = ops;
domain->host_data = host_data;
domain->hwirq_max = hwirq_max;
if (direct_max)
domain->flags |= IRQ_DOMAIN_FLAG_NO_MAP;
domain->revmap_size = size;
/*
* Hierarchical domains use the domain lock of the root domain
* (innermost domain).
*
* For non-hierarchical domains (as for root domains), the root
* pointer is set to the domain itself so that &domain->root->mutex
* always points to the right lock.
*/
mutex_init(&domain->mutex);
domain->root = domain;
irq_domain_check_hierarchy(domain);
return domain;
}
static void __irq_domain_publish(struct irq_domain *domain)
{
mutex_lock(&irq_domain_mutex);
debugfs_add_domain_dir(domain);
list_add(&domain->link, &irq_domain_list);
mutex_unlock(&irq_domain_mutex);
pr_debug("Added domain %s\n", domain->name);
}
/**
* __irq_domain_add() - Allocate a new irq_domain data structure
* @fwnode: firmware node for the interrupt controller
* @size: Size of linear map; 0 for radix mapping only
* @hwirq_max: Maximum number of interrupts supported by controller
* @direct_max: Maximum value of direct maps; Use ~0 for no limit; 0 for no
* direct mapping
* @ops: domain callbacks
* @host_data: Controller private data pointer
*
* Allocates and initializes an irq_domain structure.
* Returns pointer to IRQ domain, or NULL on failure.
*/
struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, unsigned int size,
irq_hw_number_t hwirq_max, int direct_max,
const struct irq_domain_ops *ops,
void *host_data)
{
struct irq_domain *domain;
domain = __irq_domain_create(fwnode, size, hwirq_max, direct_max,
ops, host_data);
if (domain)
__irq_domain_publish(domain);
return domain;
}
EXPORT_SYMBOL_GPL(__irq_domain_add);
/**
* irq_domain_remove() - Remove an irq domain.
* @domain: domain to remove
*
* This routine is used to remove an irq domain. The caller must ensure
* that all mappings within the domain have been disposed of prior to
* use, depending on the revmap type.
*/
void irq_domain_remove(struct irq_domain *domain)
{
mutex_lock(&irq_domain_mutex);
debugfs_remove_domain_dir(domain);
WARN_ON(!radix_tree_empty(&domain->revmap_tree)); list_del(&domain->link);
/*
* If the going away domain is the default one, reset it.
*/
if (unlikely(irq_default_domain == domain))
irq_set_default_host(NULL); mutex_unlock(&irq_domain_mutex); pr_debug("Removed domain %s\n", domain->name); fwnode_dev_initialized(domain->fwnode, false); fwnode_handle_put(domain->fwnode);
if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED)
kfree(domain->name); kfree(domain);
}
EXPORT_SYMBOL_GPL(irq_domain_remove);
void irq_domain_update_bus_token(struct irq_domain *domain,
enum irq_domain_bus_token bus_token)
{
char *name;
if (domain->bus_token == bus_token)
return;
mutex_lock(&irq_domain_mutex);
domain->bus_token = bus_token;
name = kasprintf(GFP_KERNEL, "%s-%d", domain->name, bus_token);
if (!name) {
mutex_unlock(&irq_domain_mutex);
return;
}
debugfs_remove_domain_dir(domain);
if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED)
kfree(domain->name);
else
domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
domain->name = name;
debugfs_add_domain_dir(domain);
mutex_unlock(&irq_domain_mutex);
}
EXPORT_SYMBOL_GPL(irq_domain_update_bus_token);
/**
* irq_domain_create_simple() - Register an irq_domain and optionally map a range of irqs
* @fwnode: firmware node for the interrupt controller
* @size: total number of irqs in mapping
* @first_irq: first number of irq block assigned to the domain,
* pass zero to assign irqs on-the-fly. If first_irq is non-zero, then
* pre-map all of the irqs in the domain to virqs starting at first_irq.
* @ops: domain callbacks
* @host_data: Controller private data pointer
*
* Allocates an irq_domain, and optionally if first_irq is positive then also
* allocate irq_descs and map all of the hwirqs to virqs starting at first_irq.
*
* This is intended to implement the expected behaviour for most
* interrupt controllers. If device tree is used, then first_irq will be 0 and
* irqs get mapped dynamically on the fly. However, if the controller requires
* static virq assignments (non-DT boot) then it will set that up correctly.
*/
struct irq_domain *irq_domain_create_simple(struct fwnode_handle *fwnode,
unsigned int size,
unsigned int first_irq,
const struct irq_domain_ops *ops,
void *host_data)
{
struct irq_domain *domain;
domain = __irq_domain_add(fwnode, size, size, 0, ops, host_data);
if (!domain)
return NULL;
if (first_irq > 0) {
if (IS_ENABLED(CONFIG_SPARSE_IRQ)) {
/* attempt to allocated irq_descs */
int rc = irq_alloc_descs(first_irq, first_irq, size,
of_node_to_nid(to_of_node(fwnode)));
if (rc < 0)
pr_info("Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n",
first_irq);
}
irq_domain_associate_many(domain, first_irq, 0, size);
}
return domain;
}
EXPORT_SYMBOL_GPL(irq_domain_create_simple);
/**
* irq_domain_add_legacy() - Allocate and register a legacy revmap irq_domain.
* @of_node: pointer to interrupt controller's device tree node.
* @size: total number of irqs in legacy mapping
* @first_irq: first number of irq block assigned to the domain
* @first_hwirq: first hwirq number to use for the translation. Should normally
* be '0', but a positive integer can be used if the effective
* hwirqs numbering does not begin at zero.
* @ops: map/unmap domain callbacks
* @host_data: Controller private data pointer
*
* Note: the map() callback will be called before this function returns
* for all legacy interrupts except 0 (which is always the invalid irq for
* a legacy controller).
*/
struct irq_domain *irq_domain_add_legacy(struct device_node *of_node,
unsigned int size,
unsigned int first_irq,
irq_hw_number_t first_hwirq,
const struct irq_domain_ops *ops,
void *host_data)
{
return irq_domain_create_legacy(of_node_to_fwnode(of_node), size,
first_irq, first_hwirq, ops, host_data);
}
EXPORT_SYMBOL_GPL(irq_domain_add_legacy);
struct irq_domain *irq_domain_create_legacy(struct fwnode_handle *fwnode,
unsigned int size,
unsigned int first_irq,
irq_hw_number_t first_hwirq,
const struct irq_domain_ops *ops,
void *host_data)
{
struct irq_domain *domain;
domain = __irq_domain_add(fwnode, first_hwirq + size, first_hwirq + size, 0, ops, host_data);
if (domain)
irq_domain_associate_many(domain, first_irq, first_hwirq, size);
return domain;
}
EXPORT_SYMBOL_GPL(irq_domain_create_legacy);
/**
* irq_find_matching_fwspec() - Locates a domain for a given fwspec
* @fwspec: FW specifier for an interrupt
* @bus_token: domain-specific data
*/
struct irq_domain *irq_find_matching_fwspec(struct irq_fwspec *fwspec,
enum irq_domain_bus_token bus_token)
{
struct irq_domain *h, *found = NULL;
struct fwnode_handle *fwnode = fwspec->fwnode;
int rc;
/* We might want to match the legacy controller last since
* it might potentially be set to match all interrupts in
* the absence of a device node. This isn't a problem so far
* yet though...
*
* bus_token == DOMAIN_BUS_ANY matches any domain, any other
* values must generate an exact match for the domain to be
* selected.
*/
mutex_lock(&irq_domain_mutex);
list_for_each_entry(h, &irq_domain_list, link) {
if (h->ops->select && bus_token != DOMAIN_BUS_ANY)
rc = h->ops->select(h, fwspec, bus_token);
else if (h->ops->match)
rc = h->ops->match(h, to_of_node(fwnode), bus_token);
else
rc = ((fwnode != NULL) && (h->fwnode == fwnode) &&
((bus_token == DOMAIN_BUS_ANY) ||
(h->bus_token == bus_token)));
if (rc) {
found = h;
break;
}
}
mutex_unlock(&irq_domain_mutex);
return found;
}
EXPORT_SYMBOL_GPL(irq_find_matching_fwspec);
/**
* irq_set_default_host() - Set a "default" irq domain
* @domain: default domain pointer
*
* For convenience, it's possible to set a "default" domain that will be used
* whenever NULL is passed to irq_create_mapping(). It makes life easier for
* platforms that want to manipulate a few hard coded interrupt numbers that
* aren't properly represented in the device-tree.
*/
void irq_set_default_host(struct irq_domain *domain)
{
pr_debug("Default domain set to @0x%p\n", domain); irq_default_domain = domain;
}
EXPORT_SYMBOL_GPL(irq_set_default_host);
/**
* irq_get_default_host() - Retrieve the "default" irq domain
*
* Returns: the default domain, if any.
*
* Modern code should never use this. This should only be used on
* systems that cannot implement a firmware->fwnode mapping (which
* both DT and ACPI provide).
*/
struct irq_domain *irq_get_default_host(void)
{
return irq_default_domain;
}
EXPORT_SYMBOL_GPL(irq_get_default_host);
static bool irq_domain_is_nomap(struct irq_domain *domain)
{
return IS_ENABLED(CONFIG_IRQ_DOMAIN_NOMAP) &&
(domain->flags & IRQ_DOMAIN_FLAG_NO_MAP);
}
static void irq_domain_clear_mapping(struct irq_domain *domain,
irq_hw_number_t hwirq)
{
lockdep_assert_held(&domain->root->mutex);
if (irq_domain_is_nomap(domain))
return;
if (hwirq < domain->revmap_size)
rcu_assign_pointer(domain->revmap[hwirq], NULL);
else
radix_tree_delete(&domain->revmap_tree, hwirq);
}
static void irq_domain_set_mapping(struct irq_domain *domain,
irq_hw_number_t hwirq,
struct irq_data *irq_data)
{
/*
* This also makes sure that all domains point to the same root when
* called from irq_domain_insert_irq() for each domain in a hierarchy.
*/
lockdep_assert_held(&domain->root->mutex);
if (irq_domain_is_nomap(domain))
return;
if (hwirq < domain->revmap_size)
rcu_assign_pointer(domain->revmap[hwirq], irq_data);
else
radix_tree_insert(&domain->revmap_tree, hwirq, irq_data);
}
static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq)
{
struct irq_data *irq_data = irq_get_irq_data(irq);
irq_hw_number_t hwirq;
if (WARN(!irq_data || irq_data->domain != domain,
"virq%i doesn't exist; cannot disassociate\n", irq))
return;
hwirq = irq_data->hwirq;
mutex_lock(&domain->root->mutex);
irq_set_status_flags(irq, IRQ_NOREQUEST);
/* remove chip and handler */
irq_set_chip_and_handler(irq, NULL, NULL);
/* Make sure it's completed */
synchronize_irq(irq);
/* Tell the PIC about it */
if (domain->ops->unmap)
domain->ops->unmap(domain, irq); smp_mb();
irq_data->domain = NULL;
irq_data->hwirq = 0;
domain->mapcount--;
/* Clear reverse map for this hwirq */
irq_domain_clear_mapping(domain, hwirq);
mutex_unlock(&domain->root->mutex);
}
static int irq_domain_associate_locked(struct irq_domain *domain, unsigned int virq,
irq_hw_number_t hwirq)
{
struct irq_data *irq_data = irq_get_irq_data(virq);
int ret;
if (WARN(hwirq >= domain->hwirq_max,
"error: hwirq 0x%x is too large for %s\n", (int)hwirq, domain->name))
return -EINVAL;
if (WARN(!irq_data, "error: virq%i is not allocated", virq))
return -EINVAL;
if (WARN(irq_data->domain, "error: virq%i is already associated", virq))
return -EINVAL;
irq_data->hwirq = hwirq;
irq_data->domain = domain;
if (domain->ops->map) {
ret = domain->ops->map(domain, virq, hwirq);
if (ret != 0) {
/*
* If map() returns -EPERM, this interrupt is protected
* by the firmware or some other service and shall not
* be mapped. Don't bother telling the user about it.
*/
if (ret != -EPERM) {
pr_info("%s didn't like hwirq-0x%lx to VIRQ%i mapping (rc=%d)\n",
domain->name, hwirq, virq, ret);
}
irq_data->domain = NULL;
irq_data->hwirq = 0;
return ret;
}
}
domain->mapcount++;
irq_domain_set_mapping(domain, hwirq, irq_data);
irq_clear_status_flags(virq, IRQ_NOREQUEST);
return 0;
}
int irq_domain_associate(struct irq_domain *domain, unsigned int virq,
irq_hw_number_t hwirq)
{
int ret;
mutex_lock(&domain->root->mutex);
ret = irq_domain_associate_locked(domain, virq, hwirq);
mutex_unlock(&domain->root->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(irq_domain_associate);
void irq_domain_associate_many(struct irq_domain *domain, unsigned int irq_base,
irq_hw_number_t hwirq_base, int count)
{
struct device_node *of_node;
int i;
of_node = irq_domain_get_of_node(domain);
pr_debug("%s(%s, irqbase=%i, hwbase=%i, count=%i)\n", __func__,
of_node_full_name(of_node), irq_base, (int)hwirq_base, count);
for (i = 0; i < count; i++)
irq_domain_associate(domain, irq_base + i, hwirq_base + i);
}
EXPORT_SYMBOL_GPL(irq_domain_associate_many);
#ifdef CONFIG_IRQ_DOMAIN_NOMAP
/**
* irq_create_direct_mapping() - Allocate an irq for direct mapping
* @domain: domain to allocate the irq for or NULL for default domain
*
* This routine is used for irq controllers which can choose the hardware
* interrupt numbers they generate. In such a case it's simplest to use
* the linux irq as the hardware interrupt number. It still uses the linear
* or radix tree to store the mapping, but the irq controller can optimize
* the revmap path by using the hwirq directly.
*/
unsigned int irq_create_direct_mapping(struct irq_domain *domain)
{
struct device_node *of_node;
unsigned int virq;
if (domain == NULL)
domain = irq_default_domain;
of_node = irq_domain_get_of_node(domain);
virq = irq_alloc_desc_from(1, of_node_to_nid(of_node));
if (!virq) {
pr_debug("create_direct virq allocation failed\n");
return 0;
}
if (virq >= domain->hwirq_max) {
pr_err("ERROR: no free irqs available below %lu maximum\n",
domain->hwirq_max);
irq_free_desc(virq);
return 0;
}
pr_debug("create_direct obtained virq %d\n", virq);
if (irq_domain_associate(domain, virq, virq)) {
irq_free_desc(virq);
return 0;
}
return virq;
}
EXPORT_SYMBOL_GPL(irq_create_direct_mapping);
#endif
static unsigned int irq_create_mapping_affinity_locked(struct irq_domain *domain,
irq_hw_number_t hwirq,
const struct irq_affinity_desc *affinity)
{
struct device_node *of_node = irq_domain_get_of_node(domain);
int virq;
pr_debug("irq_create_mapping(0x%p, 0x%lx)\n", domain, hwirq);
/* Allocate a virtual interrupt number */
virq = irq_domain_alloc_descs(-1, 1, hwirq, of_node_to_nid(of_node),
affinity);
if (virq <= 0) {
pr_debug("-> virq allocation failed\n");
return 0;
}
if (irq_domain_associate_locked(domain, virq, hwirq)) {
irq_free_desc(virq);
return 0;
}
pr_debug("irq %lu on domain %s mapped to virtual irq %u\n",
hwirq, of_node_full_name(of_node), virq);
return virq;
}
/**
* irq_create_mapping_affinity() - Map a hardware interrupt into linux irq space
* @domain: domain owning this hardware interrupt or NULL for default domain
* @hwirq: hardware irq number in that domain space
* @affinity: irq affinity
*
* Only one mapping per hardware interrupt is permitted. Returns a linux
* irq number.
* If the sense/trigger is to be specified, set_irq_type() should be called
* on the number returned from that call.
*/
unsigned int irq_create_mapping_affinity(struct irq_domain *domain,
irq_hw_number_t hwirq,
const struct irq_affinity_desc *affinity)
{
int virq;
/* Look for default domain if necessary */
if (domain == NULL)
domain = irq_default_domain;
if (domain == NULL) {
WARN(1, "%s(, %lx) called with NULL domain\n", __func__, hwirq);
return 0;
}
mutex_lock(&domain->root->mutex);
/* Check if mapping already exists */
virq = irq_find_mapping(domain, hwirq);
if (virq) {
pr_debug("existing mapping on virq %d\n", virq);
goto out;
}
virq = irq_create_mapping_affinity_locked(domain, hwirq, affinity);
out:
mutex_unlock(&domain->root->mutex);
return virq;
}
EXPORT_SYMBOL_GPL(irq_create_mapping_affinity);
static int irq_domain_translate(struct irq_domain *d,
struct irq_fwspec *fwspec,
irq_hw_number_t *hwirq, unsigned int *type)
{
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
if (d->ops->translate)
return d->ops->translate(d, fwspec, hwirq, type);
#endif
if (d->ops->xlate)
return d->ops->xlate(d, to_of_node(fwspec->fwnode),
fwspec->param, fwspec->param_count,
hwirq, type);
/* If domain has no translation, then we assume interrupt line */
*hwirq = fwspec->param[0];
return 0;
}
void of_phandle_args_to_fwspec(struct device_node *np, const u32 *args,
unsigned int count, struct irq_fwspec *fwspec)
{
int i;
fwspec->fwnode = of_node_to_fwnode(np);
fwspec->param_count = count;
for (i = 0; i < count; i++)
fwspec->param[i] = args[i];
}
EXPORT_SYMBOL_GPL(of_phandle_args_to_fwspec);
unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec)
{
struct irq_domain *domain;
struct irq_data *irq_data;
irq_hw_number_t hwirq;
unsigned int type = IRQ_TYPE_NONE;
int virq;
if (fwspec->fwnode) {
domain = irq_find_matching_fwspec(fwspec, DOMAIN_BUS_WIRED);
if (!domain)
domain = irq_find_matching_fwspec(fwspec, DOMAIN_BUS_ANY);
} else {
domain = irq_default_domain;
}
if (!domain) {
pr_warn("no irq domain found for %s !\n",
of_node_full_name(to_of_node(fwspec->fwnode)));
return 0;
}
if (irq_domain_translate(domain, fwspec, &hwirq, &type))
return 0;
/*
* WARN if the irqchip returns a type with bits
* outside the sense mask set and clear these bits.
*/
if (WARN_ON(type & ~IRQ_TYPE_SENSE_MASK))
type &= IRQ_TYPE_SENSE_MASK;
mutex_lock(&domain->root->mutex);
/*
* If we've already configured this interrupt,
* don't do it again, or hell will break loose.
*/
virq = irq_find_mapping(domain, hwirq);
if (virq) {
/*
* If the trigger type is not specified or matches the
* current trigger type then we are done so return the
* interrupt number.
*/
if (type == IRQ_TYPE_NONE || type == irq_get_trigger_type(virq))
goto out;
/*
* If the trigger type has not been set yet, then set
* it now and return the interrupt number.
*/
if (irq_get_trigger_type(virq) == IRQ_TYPE_NONE) {
irq_data = irq_get_irq_data(virq);
if (!irq_data) {
virq = 0;
goto out;
}
irqd_set_trigger_type(irq_data, type);
goto out;
}
pr_warn("type mismatch, failed to map hwirq-%lu for %s!\n",
hwirq, of_node_full_name(to_of_node(fwspec->fwnode)));
virq = 0;
goto out;
}
if (irq_domain_is_hierarchy(domain)) {
if (irq_domain_is_msi_device(domain)) {
mutex_unlock(&domain->root->mutex);
virq = msi_device_domain_alloc_wired(domain, hwirq, type);
mutex_lock(&domain->root->mutex);
} else
virq = irq_domain_alloc_irqs_locked(domain, -1, 1, NUMA_NO_NODE,
fwspec, false, NULL);
if (virq <= 0) {
virq = 0;
goto out;
}
} else {
/* Create mapping */
virq = irq_create_mapping_affinity_locked(domain, hwirq, NULL);
if (!virq)
goto out;
}
irq_data = irq_get_irq_data(virq);
if (WARN_ON(!irq_data)) {
virq = 0;
goto out;
}
/* Store trigger type */
irqd_set_trigger_type(irq_data, type);
out:
mutex_unlock(&domain->root->mutex);
return virq;
}
EXPORT_SYMBOL_GPL(irq_create_fwspec_mapping);
unsigned int irq_create_of_mapping(struct of_phandle_args *irq_data)
{
struct irq_fwspec fwspec;
of_phandle_args_to_fwspec(irq_data->np, irq_data->args,
irq_data->args_count, &fwspec);
return irq_create_fwspec_mapping(&fwspec);
}
EXPORT_SYMBOL_GPL(irq_create_of_mapping);
/**
* irq_dispose_mapping() - Unmap an interrupt
* @virq: linux irq number of the interrupt to unmap
*/
void irq_dispose_mapping(unsigned int virq)
{
struct irq_data *irq_data = irq_get_irq_data(virq);
struct irq_domain *domain;
if (!virq || !irq_data)
return;
domain = irq_data->domain; if (WARN_ON(domain == NULL))
return;
if (irq_domain_is_hierarchy(domain)) { irq_domain_free_one_irq(domain, virq);
} else {
irq_domain_disassociate(domain, virq); irq_free_desc(virq);
}
}
EXPORT_SYMBOL_GPL(irq_dispose_mapping);
/**
* __irq_resolve_mapping() - Find a linux irq from a hw irq number.
* @domain: domain owning this hardware interrupt
* @hwirq: hardware irq number in that domain space
* @irq: optional pointer to return the Linux irq if required
*
* Returns the interrupt descriptor.
*/
struct irq_desc *__irq_resolve_mapping(struct irq_domain *domain,
irq_hw_number_t hwirq,
unsigned int *irq)
{
struct irq_desc *desc = NULL;
struct irq_data *data;
/* Look for default domain if necessary */
if (domain == NULL) domain = irq_default_domain;
if (domain == NULL)
return desc;
if (irq_domain_is_nomap(domain)) {
if (hwirq < domain->hwirq_max) {
data = irq_domain_get_irq_data(domain, hwirq);
if (data && data->hwirq == hwirq)
desc = irq_data_to_desc(data);
if (irq && desc)
*irq = hwirq;
}
return desc;
}
rcu_read_lock();
/* Check if the hwirq is in the linear revmap. */
if (hwirq < domain->revmap_size) data = rcu_dereference(domain->revmap[hwirq]);
else
data = radix_tree_lookup(&domain->revmap_tree, hwirq); if (likely(data)) { desc = irq_data_to_desc(data);
if (irq)
*irq = data->irq;
}
rcu_read_unlock();
return desc;
}
EXPORT_SYMBOL_GPL(__irq_resolve_mapping);
/**
* irq_domain_xlate_onecell() - Generic xlate for direct one cell bindings
*
* Device Tree IRQ specifier translation function which works with one cell
* bindings where the cell value maps directly to the hwirq number.
*/
int irq_domain_xlate_onecell(struct irq_domain *d, struct device_node *ctrlr,
const u32 *intspec, unsigned int intsize,
unsigned long *out_hwirq, unsigned int *out_type)
{
if (WARN_ON(intsize < 1))
return -EINVAL;
*out_hwirq = intspec[0];
*out_type = IRQ_TYPE_NONE;
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_xlate_onecell);
/**
* irq_domain_xlate_twocell() - Generic xlate for direct two cell bindings
*
* Device Tree IRQ specifier translation function which works with two cell
* bindings where the cell values map directly to the hwirq number
* and linux irq flags.
*/
int irq_domain_xlate_twocell(struct irq_domain *d, struct device_node *ctrlr,
const u32 *intspec, unsigned int intsize,
irq_hw_number_t *out_hwirq, unsigned int *out_type)
{
struct irq_fwspec fwspec;
of_phandle_args_to_fwspec(ctrlr, intspec, intsize, &fwspec);
return irq_domain_translate_twocell(d, &fwspec, out_hwirq, out_type);
}
EXPORT_SYMBOL_GPL(irq_domain_xlate_twocell);
/**
* irq_domain_xlate_onetwocell() - Generic xlate for one or two cell bindings
*
* Device Tree IRQ specifier translation function which works with either one
* or two cell bindings where the cell values map directly to the hwirq number
* and linux irq flags.
*
* Note: don't use this function unless your interrupt controller explicitly
* supports both one and two cell bindings. For the majority of controllers
* the _onecell() or _twocell() variants above should be used.
*/
int irq_domain_xlate_onetwocell(struct irq_domain *d,
struct device_node *ctrlr,
const u32 *intspec, unsigned int intsize,
unsigned long *out_hwirq, unsigned int *out_type)
{
if (WARN_ON(intsize < 1))
return -EINVAL;
*out_hwirq = intspec[0];
if (intsize > 1)
*out_type = intspec[1] & IRQ_TYPE_SENSE_MASK;
else
*out_type = IRQ_TYPE_NONE;
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_xlate_onetwocell);
const struct irq_domain_ops irq_domain_simple_ops = {
.xlate = irq_domain_xlate_onetwocell,
};
EXPORT_SYMBOL_GPL(irq_domain_simple_ops);
/**
* irq_domain_translate_onecell() - Generic translate for direct one cell
* bindings
*/
int irq_domain_translate_onecell(struct irq_domain *d,
struct irq_fwspec *fwspec,
unsigned long *out_hwirq,
unsigned int *out_type)
{
if (WARN_ON(fwspec->param_count < 1))
return -EINVAL;
*out_hwirq = fwspec->param[0];
*out_type = IRQ_TYPE_NONE;
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_translate_onecell);
/**
* irq_domain_translate_twocell() - Generic translate for direct two cell
* bindings
*
* Device Tree IRQ specifier translation function which works with two cell
* bindings where the cell values map directly to the hwirq number
* and linux irq flags.
*/
int irq_domain_translate_twocell(struct irq_domain *d,
struct irq_fwspec *fwspec,
unsigned long *out_hwirq,
unsigned int *out_type)
{
if (WARN_ON(fwspec->param_count < 2))
return -EINVAL;
*out_hwirq = fwspec->param[0];
*out_type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK;
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_translate_twocell);
int irq_domain_alloc_descs(int virq, unsigned int cnt, irq_hw_number_t hwirq,
int node, const struct irq_affinity_desc *affinity)
{
unsigned int hint;
if (virq >= 0) {
virq = __irq_alloc_descs(virq, virq, cnt, node, THIS_MODULE,
affinity);
} else {
hint = hwirq % nr_irqs;
if (hint == 0)
hint++;
virq = __irq_alloc_descs(-1, hint, cnt, node, THIS_MODULE,
affinity);
if (virq <= 0 && hint > 1) {
virq = __irq_alloc_descs(-1, 1, cnt, node, THIS_MODULE,
affinity);
}
}
return virq;
}
/**
* irq_domain_reset_irq_data - Clear hwirq, chip and chip_data in @irq_data
* @irq_data: The pointer to irq_data
*/
void irq_domain_reset_irq_data(struct irq_data *irq_data)
{
irq_data->hwirq = 0;
irq_data->chip = &no_irq_chip;
irq_data->chip_data = NULL;
}
EXPORT_SYMBOL_GPL(irq_domain_reset_irq_data);
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
/**
* irq_domain_create_hierarchy - Add a irqdomain into the hierarchy
* @parent: Parent irq domain to associate with the new domain
* @flags: Irq domain flags associated to the domain
* @size: Size of the domain. See below
* @fwnode: Optional fwnode of the interrupt controller
* @ops: Pointer to the interrupt domain callbacks
* @host_data: Controller private data pointer
*
* If @size is 0 a tree domain is created, otherwise a linear domain.
*
* If successful the parent is associated to the new domain and the
* domain flags are set.
* Returns pointer to IRQ domain, or NULL on failure.
*/
struct irq_domain *irq_domain_create_hierarchy(struct irq_domain *parent,
unsigned int flags,
unsigned int size,
struct fwnode_handle *fwnode,
const struct irq_domain_ops *ops,
void *host_data)
{
struct irq_domain *domain;
if (size)
domain = __irq_domain_create(fwnode, size, size, 0, ops, host_data);
else
domain = __irq_domain_create(fwnode, 0, ~0, 0, ops, host_data);
if (domain) {
if (parent)
domain->root = parent->root;
domain->parent = parent;
domain->flags |= flags;
__irq_domain_publish(domain);
}
return domain;
}
EXPORT_SYMBOL_GPL(irq_domain_create_hierarchy);
static void irq_domain_insert_irq(int virq)
{
struct irq_data *data;
for (data = irq_get_irq_data(virq); data; data = data->parent_data) {
struct irq_domain *domain = data->domain;
domain->mapcount++;
irq_domain_set_mapping(domain, data->hwirq, data);
}
irq_clear_status_flags(virq, IRQ_NOREQUEST);
}
static void irq_domain_remove_irq(int virq)
{
struct irq_data *data;
irq_set_status_flags(virq, IRQ_NOREQUEST);
irq_set_chip_and_handler(virq, NULL, NULL);
synchronize_irq(virq);
smp_mb();
for (data = irq_get_irq_data(virq); data; data = data->parent_data) {
struct irq_domain *domain = data->domain;
irq_hw_number_t hwirq = data->hwirq;
domain->mapcount--;
irq_domain_clear_mapping(domain, hwirq);
}
}
static struct irq_data *irq_domain_insert_irq_data(struct irq_domain *domain,
struct irq_data *child)
{
struct irq_data *irq_data;
irq_data = kzalloc_node(sizeof(*irq_data), GFP_KERNEL,
irq_data_get_node(child));
if (irq_data) {
child->parent_data = irq_data;
irq_data->irq = child->irq;
irq_data->common = child->common;
irq_data->domain = domain;
}
return irq_data;
}
static void __irq_domain_free_hierarchy(struct irq_data *irq_data)
{
struct irq_data *tmp;
while (irq_data) {
tmp = irq_data;
irq_data = irq_data->parent_data;
kfree(tmp);
}
}
static void irq_domain_free_irq_data(unsigned int virq, unsigned int nr_irqs)
{
struct irq_data *irq_data, *tmp;
int i;
for (i = 0; i < nr_irqs; i++) {
irq_data = irq_get_irq_data(virq + i);
tmp = irq_data->parent_data;
irq_data->parent_data = NULL;
irq_data->domain = NULL;
__irq_domain_free_hierarchy(tmp);
}
}
/**
* irq_domain_disconnect_hierarchy - Mark the first unused level of a hierarchy
* @domain: IRQ domain from which the hierarchy is to be disconnected
* @virq: IRQ number where the hierarchy is to be trimmed
*
* Marks the @virq level belonging to @domain as disconnected.
* Returns -EINVAL if @virq doesn't have a valid irq_data pointing
* to @domain.
*
* Its only use is to be able to trim levels of hierarchy that do not
* have any real meaning for this interrupt, and that the driver marks
* as such from its .alloc() callback.
*/
int irq_domain_disconnect_hierarchy(struct irq_domain *domain,
unsigned int virq)
{
struct irq_data *irqd;
irqd = irq_domain_get_irq_data(domain, virq);
if (!irqd)
return -EINVAL;
irqd->chip = ERR_PTR(-ENOTCONN);
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_disconnect_hierarchy);
static int irq_domain_trim_hierarchy(unsigned int virq)
{
struct irq_data *tail, *irqd, *irq_data;
irq_data = irq_get_irq_data(virq);
tail = NULL;
/* The first entry must have a valid irqchip */
if (!irq_data->chip || IS_ERR(irq_data->chip))
return -EINVAL;
/*
* Validate that the irq_data chain is sane in the presence of
* a hierarchy trimming marker.
*/
for (irqd = irq_data->parent_data; irqd; irq_data = irqd, irqd = irqd->parent_data) {
/* Can't have a valid irqchip after a trim marker */
if (irqd->chip && tail)
return -EINVAL;
/* Can't have an empty irqchip before a trim marker */
if (!irqd->chip && !tail)
return -EINVAL;
if (IS_ERR(irqd->chip)) {
/* Only -ENOTCONN is a valid trim marker */
if (PTR_ERR(irqd->chip) != -ENOTCONN)
return -EINVAL;
tail = irq_data;
}
}
/* No trim marker, nothing to do */
if (!tail)
return 0;
pr_info("IRQ%d: trimming hierarchy from %s\n",
virq, tail->parent_data->domain->name);
/* Sever the inner part of the hierarchy... */
irqd = tail;
tail = tail->parent_data;
irqd->parent_data = NULL;
__irq_domain_free_hierarchy(tail);
return 0;
}
static int irq_domain_alloc_irq_data(struct irq_domain *domain,
unsigned int virq, unsigned int nr_irqs)
{
struct irq_data *irq_data;
struct irq_domain *parent;
int i;
/* The outermost irq_data is embedded in struct irq_desc */
for (i = 0; i < nr_irqs; i++) {
irq_data = irq_get_irq_data(virq + i);
irq_data->domain = domain;
for (parent = domain->parent; parent; parent = parent->parent) {
irq_data = irq_domain_insert_irq_data(parent, irq_data);
if (!irq_data) {
irq_domain_free_irq_data(virq, i + 1);
return -ENOMEM;
}
}
}
return 0;
}
/**
* irq_domain_get_irq_data - Get irq_data associated with @virq and @domain
* @domain: domain to match
* @virq: IRQ number to get irq_data
*/
struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain,
unsigned int virq)
{
struct irq_data *irq_data;
for (irq_data = irq_get_irq_data(virq); irq_data;
irq_data = irq_data->parent_data)
if (irq_data->domain == domain)
return irq_data;
return NULL;
}
EXPORT_SYMBOL_GPL(irq_domain_get_irq_data);
/**
* irq_domain_set_hwirq_and_chip - Set hwirq and irqchip of @virq at @domain
* @domain: Interrupt domain to match
* @virq: IRQ number
* @hwirq: The hwirq number
* @chip: The associated interrupt chip
* @chip_data: The associated chip data
*/
int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, unsigned int virq,
irq_hw_number_t hwirq,
const struct irq_chip *chip,
void *chip_data)
{
struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq);
if (!irq_data)
return -ENOENT;
irq_data->hwirq = hwirq;
irq_data->chip = (struct irq_chip *)(chip ? chip : &no_irq_chip);
irq_data->chip_data = chip_data;
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_set_hwirq_and_chip);
/**
* irq_domain_set_info - Set the complete data for a @virq in @domain
* @domain: Interrupt domain to match
* @virq: IRQ number
* @hwirq: The hardware interrupt number
* @chip: The associated interrupt chip
* @chip_data: The associated interrupt chip data
* @handler: The interrupt flow handler
* @handler_data: The interrupt flow handler data
* @handler_name: The interrupt handler name
*/
void irq_domain_set_info(struct irq_domain *domain, unsigned int virq,
irq_hw_number_t hwirq, const struct irq_chip *chip,
void *chip_data, irq_flow_handler_t handler,
void *handler_data, const char *handler_name)
{
irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip, chip_data);
__irq_set_handler(virq, handler, 0, handler_name);
irq_set_handler_data(virq, handler_data);
}
EXPORT_SYMBOL(irq_domain_set_info);
/**
* irq_domain_free_irqs_common - Clear irq_data and free the parent
* @domain: Interrupt domain to match
* @virq: IRQ number to start with
* @nr_irqs: The number of irqs to free
*/
void irq_domain_free_irqs_common(struct irq_domain *domain, unsigned int virq,
unsigned int nr_irqs)
{
struct irq_data *irq_data;
int i;
for (i = 0; i < nr_irqs; i++) {
irq_data = irq_domain_get_irq_data(domain, virq + i);
if (irq_data)
irq_domain_reset_irq_data(irq_data);
}
irq_domain_free_irqs_parent(domain, virq, nr_irqs);
}
EXPORT_SYMBOL_GPL(irq_domain_free_irqs_common);
/**
* irq_domain_free_irqs_top - Clear handler and handler data, clear irqdata and free parent
* @domain: Interrupt domain to match
* @virq: IRQ number to start with
* @nr_irqs: The number of irqs to free
*/
void irq_domain_free_irqs_top(struct irq_domain *domain, unsigned int virq,
unsigned int nr_irqs)
{
int i;
for (i = 0; i < nr_irqs; i++) {
irq_set_handler_data(virq + i, NULL);
irq_set_handler(virq + i, NULL);
}
irq_domain_free_irqs_common(domain, virq, nr_irqs);
}
static void irq_domain_free_irqs_hierarchy(struct irq_domain *domain,
unsigned int irq_base,
unsigned int nr_irqs)
{
unsigned int i;
if (!domain->ops->free)
return;
for (i = 0; i < nr_irqs; i++) {
if (irq_domain_get_irq_data(domain, irq_base + i))
domain->ops->free(domain, irq_base + i, 1);
}
}
int irq_domain_alloc_irqs_hierarchy(struct irq_domain *domain,
unsigned int irq_base,
unsigned int nr_irqs, void *arg)
{
if (!domain->ops->alloc) {
pr_debug("domain->ops->alloc() is NULL\n");
return -ENOSYS;
}
return domain->ops->alloc(domain, irq_base, nr_irqs, arg);
}
static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,
unsigned int nr_irqs, int node, void *arg,
bool realloc, const struct irq_affinity_desc *affinity)
{
int i, ret, virq;
if (realloc && irq_base >= 0) {
virq = irq_base;
} else {
virq = irq_domain_alloc_descs(irq_base, nr_irqs, 0, node,
affinity);
if (virq < 0) {
pr_debug("cannot allocate IRQ(base %d, count %d)\n",
irq_base, nr_irqs);
return virq;
}
}
if (irq_domain_alloc_irq_data(domain, virq, nr_irqs)) {
pr_debug("cannot allocate memory for IRQ%d\n", virq);
ret = -ENOMEM;
goto out_free_desc;
}
ret = irq_domain_alloc_irqs_hierarchy(domain, virq, nr_irqs, arg);
if (ret < 0)
goto out_free_irq_data;
for (i = 0; i < nr_irqs; i++) {
ret = irq_domain_trim_hierarchy(virq + i);
if (ret)
goto out_free_irq_data;
}
for (i = 0; i < nr_irqs; i++)
irq_domain_insert_irq(virq + i);
return virq;
out_free_irq_data:
irq_domain_free_irq_data(virq, nr_irqs);
out_free_desc:
irq_free_descs(virq, nr_irqs);
return ret;
}
/**
* __irq_domain_alloc_irqs - Allocate IRQs from domain
* @domain: domain to allocate from
* @irq_base: allocate specified IRQ number if irq_base >= 0
* @nr_irqs: number of IRQs to allocate
* @node: NUMA node id for memory allocation
* @arg: domain specific argument
* @realloc: IRQ descriptors have already been allocated if true
* @affinity: Optional irq affinity mask for multiqueue devices
*
* Allocate IRQ numbers and initialized all data structures to support
* hierarchy IRQ domains.
* Parameter @realloc is mainly to support legacy IRQs.
* Returns error code or allocated IRQ number
*
* The whole process to setup an IRQ has been split into two steps.
* The first step, __irq_domain_alloc_irqs(), is to allocate IRQ
* descriptor and required hardware resources. The second step,
* irq_domain_activate_irq(), is to program the hardware with preallocated
* resources. In this way, it's easier to rollback when failing to
* allocate resources.
*/
int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base,
unsigned int nr_irqs, int node, void *arg,
bool realloc, const struct irq_affinity_desc *affinity)
{
int ret;
if (domain == NULL) {
domain = irq_default_domain;
if (WARN(!domain, "domain is NULL; cannot allocate IRQ\n"))
return -EINVAL;
}
mutex_lock(&domain->root->mutex);
ret = irq_domain_alloc_irqs_locked(domain, irq_base, nr_irqs, node, arg,
realloc, affinity);
mutex_unlock(&domain->root->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(__irq_domain_alloc_irqs);
/* The irq_data was moved, fix the revmap to refer to the new location */
static void irq_domain_fix_revmap(struct irq_data *d)
{
void __rcu **slot;
lockdep_assert_held(&d->domain->root->mutex);
if (irq_domain_is_nomap(d->domain))
return;
/* Fix up the revmap. */
if (d->hwirq < d->domain->revmap_size) {
/* Not using radix tree */
rcu_assign_pointer(d->domain->revmap[d->hwirq], d);
} else {
slot = radix_tree_lookup_slot(&d->domain->revmap_tree, d->hwirq);
if (slot)
radix_tree_replace_slot(&d->domain->revmap_tree, slot, d);
}
}
/**
* irq_domain_push_irq() - Push a domain in to the top of a hierarchy.
* @domain: Domain to push.
* @virq: Irq to push the domain in to.
* @arg: Passed to the irq_domain_ops alloc() function.
*
* For an already existing irqdomain hierarchy, as might be obtained
* via a call to pci_enable_msix(), add an additional domain to the
* head of the processing chain. Must be called before request_irq()
* has been called.
*/
int irq_domain_push_irq(struct irq_domain *domain, int virq, void *arg)
{
struct irq_data *irq_data = irq_get_irq_data(virq);
struct irq_data *parent_irq_data;
struct irq_desc *desc;
int rv = 0;
/*
* Check that no action has been set, which indicates the virq
* is in a state where this function doesn't have to deal with
* races between interrupt handling and maintaining the
* hierarchy. This will catch gross misuse. Attempting to
* make the check race free would require holding locks across
* calls to struct irq_domain_ops->alloc(), which could lead
* to deadlock, so we just do a simple check before starting.
*/
desc = irq_to_desc(virq);
if (!desc)
return -EINVAL;
if (WARN_ON(desc->action))
return -EBUSY;
if (domain == NULL)
return -EINVAL;
if (WARN_ON(!irq_domain_is_hierarchy(domain)))
return -EINVAL;
if (!irq_data)
return -EINVAL;
if (domain->parent != irq_data->domain)
return -EINVAL;
parent_irq_data = kzalloc_node(sizeof(*parent_irq_data), GFP_KERNEL,
irq_data_get_node(irq_data));
if (!parent_irq_data)
return -ENOMEM;
mutex_lock(&domain->root->mutex);
/* Copy the original irq_data. */
*parent_irq_data = *irq_data;
/*
* Overwrite the irq_data, which is embedded in struct irq_desc, with
* values for this domain.
*/
irq_data->parent_data = parent_irq_data;
irq_data->domain = domain;
irq_data->mask = 0;
irq_data->hwirq = 0;
irq_data->chip = NULL;
irq_data->chip_data = NULL;
/* May (probably does) set hwirq, chip, etc. */
rv = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg);
if (rv) {
/* Restore the original irq_data. */
*irq_data = *parent_irq_data;
kfree(parent_irq_data);
goto error;
}
irq_domain_fix_revmap(parent_irq_data);
irq_domain_set_mapping(domain, irq_data->hwirq, irq_data);
error:
mutex_unlock(&domain->root->mutex);
return rv;
}
EXPORT_SYMBOL_GPL(irq_domain_push_irq);
/**
* irq_domain_pop_irq() - Remove a domain from the top of a hierarchy.
* @domain: Domain to remove.
* @virq: Irq to remove the domain from.
*
* Undo the effects of a call to irq_domain_push_irq(). Must be
* called either before request_irq() or after free_irq().
*/
int irq_domain_pop_irq(struct irq_domain *domain, int virq)
{
struct irq_data *irq_data = irq_get_irq_data(virq);
struct irq_data *parent_irq_data;
struct irq_data *tmp_irq_data;
struct irq_desc *desc;
/*
* Check that no action is set, which indicates the virq is in
* a state where this function doesn't have to deal with races
* between interrupt handling and maintaining the hierarchy.
* This will catch gross misuse. Attempting to make the check
* race free would require holding locks across calls to
* struct irq_domain_ops->free(), which could lead to
* deadlock, so we just do a simple check before starting.
*/
desc = irq_to_desc(virq);
if (!desc)
return -EINVAL;
if (WARN_ON(desc->action))
return -EBUSY;
if (domain == NULL)
return -EINVAL;
if (!irq_data)
return -EINVAL;
tmp_irq_data = irq_domain_get_irq_data(domain, virq);
/* We can only "pop" if this domain is at the top of the list */
if (WARN_ON(irq_data != tmp_irq_data))
return -EINVAL;
if (WARN_ON(irq_data->domain != domain))
return -EINVAL;
parent_irq_data = irq_data->parent_data;
if (WARN_ON(!parent_irq_data))
return -EINVAL;
mutex_lock(&domain->root->mutex);
irq_data->parent_data = NULL;
irq_domain_clear_mapping(domain, irq_data->hwirq);
irq_domain_free_irqs_hierarchy(domain, virq, 1);
/* Restore the original irq_data. */
*irq_data = *parent_irq_data;
irq_domain_fix_revmap(irq_data);
mutex_unlock(&domain->root->mutex);
kfree(parent_irq_data);
return 0;
}
EXPORT_SYMBOL_GPL(irq_domain_pop_irq);
/**
* irq_domain_free_irqs - Free IRQ number and associated data structures
* @virq: base IRQ number
* @nr_irqs: number of IRQs to free
*/
void irq_domain_free_irqs(unsigned int virq, unsigned int nr_irqs)
{
struct irq_data *data = irq_get_irq_data(virq);
struct irq_domain *domain;
int i;
if (WARN(!data || !data->domain || !data->domain->ops->free,
"NULL pointer, cannot free irq\n"))
return;
domain = data->domain;
mutex_lock(&domain->root->mutex);
for (i = 0; i < nr_irqs; i++)
irq_domain_remove_irq(virq + i);
irq_domain_free_irqs_hierarchy(domain, virq, nr_irqs);
mutex_unlock(&domain->root->mutex);
irq_domain_free_irq_data(virq, nr_irqs);
irq_free_descs(virq, nr_irqs);
}
static void irq_domain_free_one_irq(struct irq_domain *domain, unsigned int virq)
{
if (irq_domain_is_msi_device(domain)) msi_device_domain_free_wired(domain, virq);
else
irq_domain_free_irqs(virq, 1);
}
/**
* irq_domain_alloc_irqs_parent - Allocate interrupts from parent domain
* @domain: Domain below which interrupts must be allocated
* @irq_base: Base IRQ number
* @nr_irqs: Number of IRQs to allocate
* @arg: Allocation data (arch/domain specific)
*/
int irq_domain_alloc_irqs_parent(struct irq_domain *domain,
unsigned int irq_base, unsigned int nr_irqs,
void *arg)
{
if (!domain->parent)
return -ENOSYS;
return irq_domain_alloc_irqs_hierarchy(domain->parent, irq_base,
nr_irqs, arg);
}
EXPORT_SYMBOL_GPL(irq_domain_alloc_irqs_parent);
/**
* irq_domain_free_irqs_parent - Free interrupts from parent domain
* @domain: Domain below which interrupts must be freed
* @irq_base: Base IRQ number
* @nr_irqs: Number of IRQs to free
*/
void irq_domain_free_irqs_parent(struct irq_domain *domain,
unsigned int irq_base, unsigned int nr_irqs)
{
if (!domain->parent)
return;
irq_domain_free_irqs_hierarchy(domain->parent, irq_base, nr_irqs);
}
EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent);
static void __irq_domain_deactivate_irq(struct irq_data *irq_data)
{
if (irq_data && irq_data->domain) {
struct irq_domain *domain = irq_data->domain;
if (domain->ops->deactivate)
domain->ops->deactivate(domain, irq_data);
if (irq_data->parent_data)
__irq_domain_deactivate_irq(irq_data->parent_data);
}
}
static int __irq_domain_activate_irq(struct irq_data *irqd, bool reserve)
{
int ret = 0;
if (irqd && irqd->domain) {
struct irq_domain *domain = irqd->domain;
if (irqd->parent_data)
ret = __irq_domain_activate_irq(irqd->parent_data,
reserve);
if (!ret && domain->ops->activate) {
ret = domain->ops->activate(domain, irqd, reserve);
/* Rollback in case of error */
if (ret && irqd->parent_data)
__irq_domain_deactivate_irq(irqd->parent_data);
}
}
return ret;
}
/**
* irq_domain_activate_irq - Call domain_ops->activate recursively to activate
* interrupt
* @irq_data: Outermost irq_data associated with interrupt
* @reserve: If set only reserve an interrupt vector instead of assigning one
*
* This is the second step to call domain_ops->activate to program interrupt
* controllers, so the interrupt could actually get delivered.
*/
int irq_domain_activate_irq(struct irq_data *irq_data, bool reserve)
{
int ret = 0;
if (!irqd_is_activated(irq_data))
ret = __irq_domain_activate_irq(irq_data, reserve);
if (!ret)
irqd_set_activated(irq_data);
return ret;
}
/**
* irq_domain_deactivate_irq - Call domain_ops->deactivate recursively to
* deactivate interrupt
* @irq_data: outermost irq_data associated with interrupt
*
* It calls domain_ops->deactivate to program interrupt controllers to disable
* interrupt delivery.
*/
void irq_domain_deactivate_irq(struct irq_data *irq_data)
{
if (irqd_is_activated(irq_data)) {
__irq_domain_deactivate_irq(irq_data);
irqd_clr_activated(irq_data);
}
}
static void irq_domain_check_hierarchy(struct irq_domain *domain)
{
/* Hierarchy irq_domains must implement callback alloc() */
if (domain->ops->alloc)
domain->flags |= IRQ_DOMAIN_FLAG_HIERARCHY;
}
#else /* CONFIG_IRQ_DOMAIN_HIERARCHY */
/**
* irq_domain_get_irq_data - Get irq_data associated with @virq and @domain
* @domain: domain to match
* @virq: IRQ number to get irq_data
*/
struct irq_data *irq_domain_get_irq_data(struct irq_domain *domain,
unsigned int virq)
{
struct irq_data *irq_data = irq_get_irq_data(virq);
return (irq_data && irq_data->domain == domain) ? irq_data : NULL;
}
EXPORT_SYMBOL_GPL(irq_domain_get_irq_data);
/**
* irq_domain_set_info - Set the complete data for a @virq in @domain
* @domain: Interrupt domain to match
* @virq: IRQ number
* @hwirq: The hardware interrupt number
* @chip: The associated interrupt chip
* @chip_data: The associated interrupt chip data
* @handler: The interrupt flow handler
* @handler_data: The interrupt flow handler data
* @handler_name: The interrupt handler name
*/
void irq_domain_set_info(struct irq_domain *domain, unsigned int virq,
irq_hw_number_t hwirq, const struct irq_chip *chip,
void *chip_data, irq_flow_handler_t handler,
void *handler_data, const char *handler_name)
{
irq_set_chip_and_handler_name(virq, chip, handler, handler_name);
irq_set_chip_data(virq, chip_data);
irq_set_handler_data(virq, handler_data);
}
static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,
unsigned int nr_irqs, int node, void *arg,
bool realloc, const struct irq_affinity_desc *affinity)
{
return -EINVAL;
}
static void irq_domain_check_hierarchy(struct irq_domain *domain) { }
static void irq_domain_free_one_irq(struct irq_domain *domain, unsigned int virq) { }
#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
#include "internals.h"
static struct dentry *domain_dir;
static void
irq_domain_debug_show_one(struct seq_file *m, struct irq_domain *d, int ind)
{
seq_printf(m, "%*sname: %s\n", ind, "", d->name);
seq_printf(m, "%*ssize: %u\n", ind + 1, "", d->revmap_size);
seq_printf(m, "%*smapped: %u\n", ind + 1, "", d->mapcount);
seq_printf(m, "%*sflags: 0x%08x\n", ind +1 , "", d->flags);
if (d->ops && d->ops->debug_show)
d->ops->debug_show(m, d, NULL, ind + 1);
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
if (!d->parent)
return;
seq_printf(m, "%*sparent: %s\n", ind + 1, "", d->parent->name);
irq_domain_debug_show_one(m, d->parent, ind + 4);
#endif
}
static int irq_domain_debug_show(struct seq_file *m, void *p)
{
struct irq_domain *d = m->private;
/* Default domain? Might be NULL */
if (!d) {
if (!irq_default_domain)
return 0;
d = irq_default_domain;
}
irq_domain_debug_show_one(m, d, 0);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(irq_domain_debug);
static void debugfs_add_domain_dir(struct irq_domain *d)
{
if (!d->name || !domain_dir)
return;
debugfs_create_file(d->name, 0444, domain_dir, d,
&irq_domain_debug_fops);
}
static void debugfs_remove_domain_dir(struct irq_domain *d)
{
debugfs_lookup_and_remove(d->name, domain_dir);
}
void __init irq_domain_debugfs_init(struct dentry *root)
{
struct irq_domain *d;
domain_dir = debugfs_create_dir("domains", root);
debugfs_create_file("default", 0444, domain_dir, NULL,
&irq_domain_debug_fops);
mutex_lock(&irq_domain_mutex);
list_for_each_entry(d, &irq_domain_list, link)
debugfs_add_domain_dir(d);
mutex_unlock(&irq_domain_mutex);
}
#endif
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
Red Black Trees
(C) 1999 Andrea Arcangeli <andrea@suse.de>
linux/include/linux/rbtree.h
To use rbtrees you'll have to implement your own insert and search cores.
This will avoid us to use callbacks and to drop drammatically performances.
I know it's not the cleaner way, but in C (not in C++) to get
performances and genericity...
See Documentation/core-api/rbtree.rst for documentation and samples.
*/
#ifndef _LINUX_RBTREE_H
#define _LINUX_RBTREE_H
#include <linux/container_of.h>
#include <linux/rbtree_types.h>
#include <linux/stddef.h>
#include <linux/rcupdate.h>
#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3))
#define rb_entry(ptr, type, member) container_of(ptr, type, member)
#define RB_EMPTY_ROOT(root) (READ_ONCE((root)->rb_node) == NULL)
/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */
#define RB_EMPTY_NODE(node) \
((node)->__rb_parent_color == (unsigned long)(node))
#define RB_CLEAR_NODE(node) \
((node)->__rb_parent_color = (unsigned long)(node))
extern void rb_insert_color(struct rb_node *, struct rb_root *);
extern void rb_erase(struct rb_node *, struct rb_root *);
/* Find logical next and previous nodes in a tree */
extern struct rb_node *rb_next(const struct rb_node *);
extern struct rb_node *rb_prev(const struct rb_node *);
extern struct rb_node *rb_first(const struct rb_root *);
extern struct rb_node *rb_last(const struct rb_root *);
/* Postorder iteration - always visit the parent after its children */
extern struct rb_node *rb_first_postorder(const struct rb_root *);
extern struct rb_node *rb_next_postorder(const struct rb_node *);
/* Fast replacement of a single node without remove/rebalance/add/rebalance */
extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
struct rb_root *root);
extern void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new,
struct rb_root *root);
static inline void rb_link_node(struct rb_node *node, struct rb_node *parent,
struct rb_node **rb_link)
{
node->__rb_parent_color = (unsigned long)parent;
node->rb_left = node->rb_right = NULL;
*rb_link = node;
}
static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,
struct rb_node **rb_link)
{
node->__rb_parent_color = (unsigned long)parent;
node->rb_left = node->rb_right = NULL;
rcu_assign_pointer(*rb_link, node);
}
#define rb_entry_safe(ptr, type, member) \
({ typeof(ptr) ____ptr = (ptr); \
____ptr ? rb_entry(____ptr, type, member) : NULL; \
})
/**
* rbtree_postorder_for_each_entry_safe - iterate in post-order over rb_root of
* given type allowing the backing memory of @pos to be invalidated
*
* @pos: the 'type *' to use as a loop cursor.
* @n: another 'type *' to use as temporary storage
* @root: 'rb_root *' of the rbtree.
* @field: the name of the rb_node field within 'type'.
*
* rbtree_postorder_for_each_entry_safe() provides a similar guarantee as
* list_for_each_entry_safe() and allows the iteration to continue independent
* of changes to @pos by the body of the loop.
*
* Note, however, that it cannot handle other modifications that re-order the
* rbtree it is iterating over. This includes calling rb_erase() on @pos, as
* rb_erase() may rebalance the tree, causing us to miss some nodes.
*/
#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \
for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \
pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \
typeof(*pos), field); 1; }); \
pos = n)
/* Same as rb_first(), but O(1) */
#define rb_first_cached(root) (root)->rb_leftmost
static inline void rb_insert_color_cached(struct rb_node *node,
struct rb_root_cached *root,
bool leftmost)
{
if (leftmost)
root->rb_leftmost = node; rb_insert_color(node, &root->rb_root);
}
static inline struct rb_node *
rb_erase_cached(struct rb_node *node, struct rb_root_cached *root)
{
struct rb_node *leftmost = NULL;
if (root->rb_leftmost == node)
leftmost = root->rb_leftmost = rb_next(node); rb_erase(node, &root->rb_root);
return leftmost;
}
static inline void rb_replace_node_cached(struct rb_node *victim,
struct rb_node *new,
struct rb_root_cached *root)
{
if (root->rb_leftmost == victim)
root->rb_leftmost = new;
rb_replace_node(victim, new, &root->rb_root);
}
/*
* The below helper functions use 2 operators with 3 different
* calling conventions. The operators are related like:
*
* comp(a->key,b) < 0 := less(a,b)
* comp(a->key,b) > 0 := less(b,a)
* comp(a->key,b) == 0 := !less(a,b) && !less(b,a)
*
* If these operators define a partial order on the elements we make no
* guarantee on which of the elements matching the key is found. See
* rb_find().
*
* The reason for this is to allow the find() interface without requiring an
* on-stack dummy object, which might not be feasible due to object size.
*/
/**
* rb_add_cached() - insert @node into the leftmost cached tree @tree
* @node: node to insert
* @tree: leftmost cached tree to insert @node into
* @less: operator defining the (partial) node order
*
* Returns @node when it is the new leftmost, or NULL.
*/
static __always_inline struct rb_node *
rb_add_cached(struct rb_node *node, struct rb_root_cached *tree,
bool (*less)(struct rb_node *, const struct rb_node *))
{
struct rb_node **link = &tree->rb_root.rb_node;
struct rb_node *parent = NULL;
bool leftmost = true;
while (*link) {
parent = *link;
if (less(node, parent)) { link = &parent->rb_left;
} else {
link = &parent->rb_right;
leftmost = false;
}
}
rb_link_node(node, parent, link); rb_insert_color_cached(node, tree, leftmost);
return leftmost ? node : NULL;
}
/**
* rb_add() - insert @node into @tree
* @node: node to insert
* @tree: tree to insert @node into
* @less: operator defining the (partial) node order
*/
static __always_inline void
rb_add(struct rb_node *node, struct rb_root *tree,
bool (*less)(struct rb_node *, const struct rb_node *))
{
struct rb_node **link = &tree->rb_node;
struct rb_node *parent = NULL;
while (*link) {
parent = *link;
if (less(node, parent))
link = &parent->rb_left;
else
link = &parent->rb_right;
}
rb_link_node(node, parent, link);
rb_insert_color(node, tree);
}
/**
* rb_find_add() - find equivalent @node in @tree, or add @node
* @node: node to look-for / insert
* @tree: tree to search / modify
* @cmp: operator defining the node order
*
* Returns the rb_node matching @node, or NULL when no match is found and @node
* is inserted.
*/
static __always_inline struct rb_node *
rb_find_add(struct rb_node *node, struct rb_root *tree,
int (*cmp)(struct rb_node *, const struct rb_node *))
{
struct rb_node **link = &tree->rb_node;
struct rb_node *parent = NULL;
int c;
while (*link) {
parent = *link;
c = cmp(node, parent);
if (c < 0)
link = &parent->rb_left;
else if (c > 0)
link = &parent->rb_right;
else
return parent;
}
rb_link_node(node, parent, link);
rb_insert_color(node, tree);
return NULL;
}
/**
* rb_find() - find @key in tree @tree
* @key: key to match
* @tree: tree to search
* @cmp: operator defining the node order
*
* Returns the rb_node matching @key or NULL.
*/
static __always_inline struct rb_node *
rb_find(const void *key, const struct rb_root *tree,
int (*cmp)(const void *key, const struct rb_node *))
{
struct rb_node *node = tree->rb_node;
while (node) { int c = cmp(key, node);
if (c < 0)
node = node->rb_left; else if (c > 0) node = node->rb_right;
else
return node;
}
return NULL;
}
/**
* rb_find_first() - find the first @key in @tree
* @key: key to match
* @tree: tree to search
* @cmp: operator defining node order
*
* Returns the leftmost node matching @key, or NULL.
*/
static __always_inline struct rb_node *
rb_find_first(const void *key, const struct rb_root *tree,
int (*cmp)(const void *key, const struct rb_node *))
{
struct rb_node *node = tree->rb_node;
struct rb_node *match = NULL;
while (node) {
int c = cmp(key, node);
if (c <= 0) {
if (!c)
match = node;
node = node->rb_left;
} else if (c > 0) {
node = node->rb_right;
}
}
return match;
}
/**
* rb_next_match() - find the next @key in @tree
* @key: key to match
* @tree: tree to search
* @cmp: operator defining node order
*
* Returns the next node matching @key, or NULL.
*/
static __always_inline struct rb_node *
rb_next_match(const void *key, struct rb_node *node,
int (*cmp)(const void *key, const struct rb_node *))
{
node = rb_next(node);
if (node && cmp(key, node))
node = NULL;
return node;
}
/**
* rb_for_each() - iterates a subtree matching @key
* @node: iterator
* @key: key to match
* @tree: tree to search
* @cmp: operator defining node order
*/
#define rb_for_each(node, key, tree, cmp) \
for ((node) = rb_find_first((key), (tree), (cmp)); \
(node); (node) = rb_next_match((key), (node), (cmp)))
#endif /* _LINUX_RBTREE_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 1994 Linus Torvalds
*
* Pentium III FXSR, SSE support
* General FPU state handling cleanups
* Gareth Hughes <gareth@valinux.com>, May 2000
*/
#include <asm/fpu/api.h>
#include <asm/fpu/regset.h>
#include <asm/fpu/sched.h>
#include <asm/fpu/signal.h>
#include <asm/fpu/types.h>
#include <asm/traps.h>
#include <asm/irq_regs.h>
#include <uapi/asm/kvm.h>
#include <linux/hardirq.h>
#include <linux/pkeys.h>
#include <linux/vmalloc.h>
#include "context.h"
#include "internal.h"
#include "legacy.h"
#include "xstate.h"
#define CREATE_TRACE_POINTS
#include <asm/trace/fpu.h>
#ifdef CONFIG_X86_64
DEFINE_STATIC_KEY_FALSE(__fpu_state_size_dynamic);
DEFINE_PER_CPU(u64, xfd_state);
#endif
/* The FPU state configuration data for kernel and user space */
struct fpu_state_config fpu_kernel_cfg __ro_after_init;
struct fpu_state_config fpu_user_cfg __ro_after_init;
/*
* Represents the initial FPU state. It's mostly (but not completely) zeroes,
* depending on the FPU hardware format:
*/
struct fpstate init_fpstate __ro_after_init;
/* Track in-kernel FPU usage */
static DEFINE_PER_CPU(bool, in_kernel_fpu);
/*
* Track which context is using the FPU on the CPU:
*/
DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
/*
* Can we use the FPU in kernel mode with the
* whole "kernel_fpu_begin/end()" sequence?
*/
bool irq_fpu_usable(void)
{
if (WARN_ON_ONCE(in_nmi()))
return false;
/* In kernel FPU usage already active? */
if (this_cpu_read(in_kernel_fpu))
return false;
/*
* When not in NMI or hard interrupt context, FPU can be used in:
*
* - Task context except from within fpregs_lock()'ed critical
* regions.
*
* - Soft interrupt processing context which cannot happen
* while in a fpregs_lock()'ed critical region.
*/
if (!in_hardirq())
return true;
/*
* In hard interrupt context it's safe when soft interrupts
* are enabled, which means the interrupt did not hit in
* a fpregs_lock()'ed critical region.
*/
return !softirq_count();
}
EXPORT_SYMBOL(irq_fpu_usable);
/*
* Track AVX512 state use because it is known to slow the max clock
* speed of the core.
*/
static void update_avx_timestamp(struct fpu *fpu)
{
#define AVX512_TRACKING_MASK (XFEATURE_MASK_ZMM_Hi256 | XFEATURE_MASK_Hi16_ZMM)
if (fpu->fpstate->regs.xsave.header.xfeatures & AVX512_TRACKING_MASK)
fpu->avx512_timestamp = jiffies;
}
/*
* Save the FPU register state in fpu->fpstate->regs. The register state is
* preserved.
*
* Must be called with fpregs_lock() held.
*
* The legacy FNSAVE instruction clears all FPU state unconditionally, so
* register state has to be reloaded. That might be a pointless exercise
* when the FPU is going to be used by another task right after that. But
* this only affects 20+ years old 32bit systems and avoids conditionals all
* over the place.
*
* FXSAVE and all XSAVE variants preserve the FPU register state.
*/
void save_fpregs_to_fpstate(struct fpu *fpu)
{
if (likely(use_xsave())) {
os_xsave(fpu->fpstate);
update_avx_timestamp(fpu);
return;
}
if (likely(use_fxsr())) {
fxsave(&fpu->fpstate->regs.fxsave);
return;
}
/*
* Legacy FPU register saving, FNSAVE always clears FPU registers,
* so we have to reload them from the memory state.
*/
asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->fpstate->regs.fsave));
frstor(&fpu->fpstate->regs.fsave);
}
void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)
{
/*
* AMD K7/K8 and later CPUs up to Zen don't save/restore
* FDP/FIP/FOP unless an exception is pending. Clear the x87 state
* here by setting it to fixed values. "m" is a random variable
* that should be in L1.
*/
if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) { asm volatile(
"fnclex\n\t"
"emms\n\t"
"fildl %P[addr]" /* set F?P to defined value */
: : [addr] "m" (fpstate));
}
if (use_xsave()) {
/*
* Dynamically enabled features are enabled in XCR0, but
* usage requires also that the corresponding bits in XFD
* are cleared. If the bits are set then using a related
* instruction will raise #NM. This allows to do the
* allocation of the larger FPU buffer lazy from #NM or if
* the task has no permission to kill it which would happen
* via #UD if the feature is disabled in XCR0.
*
* XFD state is following the same life time rules as
* XSTATE and to restore state correctly XFD has to be
* updated before XRSTORS otherwise the component would
* stay in or go into init state even if the bits are set
* in fpstate::regs::xsave::xfeatures.
*/
xfd_update_state(fpstate);
/*
* Restoring state always needs to modify all features
* which are in @mask even if the current task cannot use
* extended features.
*
* So fpstate->xfeatures cannot be used here, because then
* a feature for which the task has no permission but was
* used by the previous task would not go into init state.
*/
mask = fpu_kernel_cfg.max_features & mask; os_xrstor(fpstate, mask);
} else {
if (use_fxsr())
fxrstor(&fpstate->regs.fxsave);
else
frstor(&fpstate->regs.fsave);
}
}
void fpu_reset_from_exception_fixup(void)
{
restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE);
}
#if IS_ENABLED(CONFIG_KVM)
static void __fpstate_reset(struct fpstate *fpstate, u64 xfd);
static void fpu_init_guest_permissions(struct fpu_guest *gfpu)
{
struct fpu_state_perm *fpuperm;
u64 perm;
if (!IS_ENABLED(CONFIG_X86_64))
return;
spin_lock_irq(¤t->sighand->siglock);
fpuperm = ¤t->group_leader->thread.fpu.guest_perm;
perm = fpuperm->__state_perm;
/* First fpstate allocation locks down permissions. */
WRITE_ONCE(fpuperm->__state_perm, perm | FPU_GUEST_PERM_LOCKED);
spin_unlock_irq(¤t->sighand->siglock);
gfpu->perm = perm & ~FPU_GUEST_PERM_LOCKED;
}
bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu)
{
struct fpstate *fpstate;
unsigned int size;
size = fpu_user_cfg.default_size + ALIGN(offsetof(struct fpstate, regs), 64);
fpstate = vzalloc(size);
if (!fpstate)
return false;
/* Leave xfd to 0 (the reset value defined by spec) */
__fpstate_reset(fpstate, 0);
fpstate_init_user(fpstate);
fpstate->is_valloc = true;
fpstate->is_guest = true;
gfpu->fpstate = fpstate;
gfpu->xfeatures = fpu_user_cfg.default_features;
gfpu->perm = fpu_user_cfg.default_features;
/*
* KVM sets the FP+SSE bits in the XSAVE header when copying FPU state
* to userspace, even when XSAVE is unsupported, so that restoring FPU
* state on a different CPU that does support XSAVE can cleanly load
* the incoming state using its natural XSAVE. In other words, KVM's
* uABI size may be larger than this host's default size. Conversely,
* the default size should never be larger than KVM's base uABI size;
* all features that can expand the uABI size must be opt-in.
*/
gfpu->uabi_size = sizeof(struct kvm_xsave);
if (WARN_ON_ONCE(fpu_user_cfg.default_size > gfpu->uabi_size))
gfpu->uabi_size = fpu_user_cfg.default_size;
fpu_init_guest_permissions(gfpu);
return true;
}
EXPORT_SYMBOL_GPL(fpu_alloc_guest_fpstate);
void fpu_free_guest_fpstate(struct fpu_guest *gfpu)
{
struct fpstate *fps = gfpu->fpstate;
if (!fps)
return;
if (WARN_ON_ONCE(!fps->is_valloc || !fps->is_guest || fps->in_use))
return;
gfpu->fpstate = NULL;
vfree(fps);
}
EXPORT_SYMBOL_GPL(fpu_free_guest_fpstate);
/*
* fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable
* @guest_fpu: Pointer to the guest FPU container
* @xfeatures: Features requested by guest CPUID
*
* Enable all dynamic xfeatures according to guest perm and requested CPUID.
*
* Return: 0 on success, error code otherwise
*/
int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures)
{
lockdep_assert_preemption_enabled();
/* Nothing to do if all requested features are already enabled. */
xfeatures &= ~guest_fpu->xfeatures;
if (!xfeatures)
return 0;
return __xfd_enable_feature(xfeatures, guest_fpu);
}
EXPORT_SYMBOL_GPL(fpu_enable_guest_xfd_features);
#ifdef CONFIG_X86_64
void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd)
{
fpregs_lock();
guest_fpu->fpstate->xfd = xfd;
if (guest_fpu->fpstate->in_use)
xfd_update_state(guest_fpu->fpstate);
fpregs_unlock();
}
EXPORT_SYMBOL_GPL(fpu_update_guest_xfd);
/**
* fpu_sync_guest_vmexit_xfd_state - Synchronize XFD MSR and software state
*
* Must be invoked from KVM after a VMEXIT before enabling interrupts when
* XFD write emulation is disabled. This is required because the guest can
* freely modify XFD and the state at VMEXIT is not guaranteed to be the
* same as the state on VMENTER. So software state has to be updated before
* any operation which depends on it can take place.
*
* Note: It can be invoked unconditionally even when write emulation is
* enabled for the price of a then pointless MSR read.
*/
void fpu_sync_guest_vmexit_xfd_state(void)
{
struct fpstate *fps = current->thread.fpu.fpstate;
lockdep_assert_irqs_disabled();
if (fpu_state_size_dynamic()) {
rdmsrl(MSR_IA32_XFD, fps->xfd);
__this_cpu_write(xfd_state, fps->xfd);
}
}
EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state);
#endif /* CONFIG_X86_64 */
int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
{
struct fpstate *guest_fps = guest_fpu->fpstate;
struct fpu *fpu = ¤t->thread.fpu;
struct fpstate *cur_fps = fpu->fpstate;
fpregs_lock();
if (!cur_fps->is_confidential && !test_thread_flag(TIF_NEED_FPU_LOAD))
save_fpregs_to_fpstate(fpu);
/* Swap fpstate */
if (enter_guest) {
fpu->__task_fpstate = cur_fps;
fpu->fpstate = guest_fps;
guest_fps->in_use = true;
} else {
guest_fps->in_use = false;
fpu->fpstate = fpu->__task_fpstate;
fpu->__task_fpstate = NULL;
}
cur_fps = fpu->fpstate;
if (!cur_fps->is_confidential) {
/* Includes XFD update */
restore_fpregs_from_fpstate(cur_fps, XFEATURE_MASK_FPSTATE);
} else {
/*
* XSTATE is restored by firmware from encrypted
* memory. Make sure XFD state is correct while
* running with guest fpstate
*/
xfd_update_state(cur_fps);
}
fpregs_mark_activate();
fpregs_unlock();
return 0;
}
EXPORT_SYMBOL_GPL(fpu_swap_kvm_fpstate);
void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf,
unsigned int size, u64 xfeatures, u32 pkru)
{
struct fpstate *kstate = gfpu->fpstate;
union fpregs_state *ustate = buf;
struct membuf mb = { .p = buf, .left = size };
if (cpu_feature_enabled(X86_FEATURE_XSAVE)) {
__copy_xstate_to_uabi_buf(mb, kstate, xfeatures, pkru,
XSTATE_COPY_XSAVE);
} else {
memcpy(&ustate->fxsave, &kstate->regs.fxsave,
sizeof(ustate->fxsave));
/* Make it restorable on a XSAVE enabled host */
ustate->xsave.header.xfeatures = XFEATURE_MASK_FPSSE;
}
}
EXPORT_SYMBOL_GPL(fpu_copy_guest_fpstate_to_uabi);
int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf,
u64 xcr0, u32 *vpkru)
{
struct fpstate *kstate = gfpu->fpstate;
const union fpregs_state *ustate = buf;
if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) {
if (ustate->xsave.header.xfeatures & ~XFEATURE_MASK_FPSSE)
return -EINVAL;
if (ustate->fxsave.mxcsr & ~mxcsr_feature_mask)
return -EINVAL;
memcpy(&kstate->regs.fxsave, &ustate->fxsave, sizeof(ustate->fxsave));
return 0;
}
if (ustate->xsave.header.xfeatures & ~xcr0)
return -EINVAL;
/*
* Nullify @vpkru to preserve its current value if PKRU's bit isn't set
* in the header. KVM's odd ABI is to leave PKRU untouched in this
* case (all other components are eventually re-initialized).
*/
if (!(ustate->xsave.header.xfeatures & XFEATURE_MASK_PKRU))
vpkru = NULL;
return copy_uabi_from_kernel_to_xstate(kstate, ustate, vpkru);
}
EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate);
#endif /* CONFIG_KVM */
void kernel_fpu_begin_mask(unsigned int kfpu_mask)
{
preempt_disable();
WARN_ON_FPU(!irq_fpu_usable());
WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, true);
if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) &&
!test_thread_flag(TIF_NEED_FPU_LOAD)) {
set_thread_flag(TIF_NEED_FPU_LOAD);
save_fpregs_to_fpstate(¤t->thread.fpu);
}
__cpu_invalidate_fpregs_state();
/* Put sane initial values into the control registers. */
if (likely(kfpu_mask & KFPU_MXCSR) && boot_cpu_has(X86_FEATURE_XMM))
ldmxcsr(MXCSR_DEFAULT);
if (unlikely(kfpu_mask & KFPU_387) && boot_cpu_has(X86_FEATURE_FPU))
asm volatile ("fninit");
}
EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask);
void kernel_fpu_end(void)
{
WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, false);
preempt_enable();
}
EXPORT_SYMBOL_GPL(kernel_fpu_end);
/*
* Sync the FPU register state to current's memory register state when the
* current task owns the FPU. The hardware register state is preserved.
*/
void fpu_sync_fpstate(struct fpu *fpu)
{
WARN_ON_FPU(fpu != ¤t->thread.fpu);
fpregs_lock();
trace_x86_fpu_before_save(fpu);
if (!test_thread_flag(TIF_NEED_FPU_LOAD))
save_fpregs_to_fpstate(fpu);
trace_x86_fpu_after_save(fpu);
fpregs_unlock();
}
static inline unsigned int init_fpstate_copy_size(void)
{
if (!use_xsave())
return fpu_kernel_cfg.default_size;
/* XSAVE(S) just needs the legacy and the xstate header part */
return sizeof(init_fpstate.regs.xsave);
}
static inline void fpstate_init_fxstate(struct fpstate *fpstate)
{
fpstate->regs.fxsave.cwd = 0x37f;
fpstate->regs.fxsave.mxcsr = MXCSR_DEFAULT;
}
/*
* Legacy x87 fpstate state init:
*/
static inline void fpstate_init_fstate(struct fpstate *fpstate)
{
fpstate->regs.fsave.cwd = 0xffff037fu;
fpstate->regs.fsave.swd = 0xffff0000u;
fpstate->regs.fsave.twd = 0xffffffffu;
fpstate->regs.fsave.fos = 0xffff0000u;
}
/*
* Used in two places:
* 1) Early boot to setup init_fpstate for non XSAVE systems
* 2) fpu_init_fpstate_user() which is invoked from KVM
*/
void fpstate_init_user(struct fpstate *fpstate)
{
if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
fpstate_init_soft(&fpstate->regs.soft);
return;
}
xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures);
if (cpu_feature_enabled(X86_FEATURE_FXSR))
fpstate_init_fxstate(fpstate);
else
fpstate_init_fstate(fpstate);
}
static void __fpstate_reset(struct fpstate *fpstate, u64 xfd)
{
/* Initialize sizes and feature masks */
fpstate->size = fpu_kernel_cfg.default_size;
fpstate->user_size = fpu_user_cfg.default_size;
fpstate->xfeatures = fpu_kernel_cfg.default_features;
fpstate->user_xfeatures = fpu_user_cfg.default_features;
fpstate->xfd = xfd;
}
void fpstate_reset(struct fpu *fpu)
{
/* Set the fpstate pointer to the default fpstate */
fpu->fpstate = &fpu->__fpstate;
__fpstate_reset(fpu->fpstate, init_fpstate.xfd);
/* Initialize the permission related info in fpu */
fpu->perm.__state_perm = fpu_kernel_cfg.default_features;
fpu->perm.__state_size = fpu_kernel_cfg.default_size;
fpu->perm.__user_state_size = fpu_user_cfg.default_size;
/* Same defaults for guests */
fpu->guest_perm = fpu->perm;
}
static inline void fpu_inherit_perms(struct fpu *dst_fpu)
{
if (fpu_state_size_dynamic()) {
struct fpu *src_fpu = ¤t->group_leader->thread.fpu;
spin_lock_irq(¤t->sighand->siglock);
/* Fork also inherits the permissions of the parent */
dst_fpu->perm = src_fpu->perm;
dst_fpu->guest_perm = src_fpu->guest_perm;
spin_unlock_irq(¤t->sighand->siglock);
}
}
/* A passed ssp of zero will not cause any update */
static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp)
{
#ifdef CONFIG_X86_USER_SHADOW_STACK
struct cet_user_state *xstate;
/* If ssp update is not needed. */
if (!ssp)
return 0;
xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave,
XFEATURE_CET_USER);
/*
* If there is a non-zero ssp, then 'dst' must be configured with a shadow
* stack and the fpu state should be up to date since it was just copied
* from the parent in fpu_clone(). So there must be a valid non-init CET
* state location in the buffer.
*/
if (WARN_ON_ONCE(!xstate))
return 1;
xstate->user_ssp = (u64)ssp;
#endif
return 0;
}
/* Clone current's FPU state on fork */
int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,
unsigned long ssp)
{
struct fpu *src_fpu = ¤t->thread.fpu;
struct fpu *dst_fpu = &dst->thread.fpu;
/* The new task's FPU state cannot be valid in the hardware. */
dst_fpu->last_cpu = -1;
fpstate_reset(dst_fpu);
if (!cpu_feature_enabled(X86_FEATURE_FPU))
return 0;
/*
* Enforce reload for user space tasks and prevent kernel threads
* from trying to save the FPU registers on context switch.
*/
set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD);
/*
* No FPU state inheritance for kernel threads and IO
* worker threads.
*/
if (minimal) {
/* Clear out the minimal state */
memcpy(&dst_fpu->fpstate->regs, &init_fpstate.regs,
init_fpstate_copy_size());
return 0;
}
/*
* If a new feature is added, ensure all dynamic features are
* caller-saved from here!
*/
BUILD_BUG_ON(XFEATURE_MASK_USER_DYNAMIC != XFEATURE_MASK_XTILE_DATA);
/*
* Save the default portion of the current FPU state into the
* clone. Assume all dynamic features to be defined as caller-
* saved, which enables skipping both the expansion of fpstate
* and the copying of any dynamic state.
*
* Do not use memcpy() when TIF_NEED_FPU_LOAD is set because
* copying is not valid when current uses non-default states.
*/
fpregs_lock();
if (test_thread_flag(TIF_NEED_FPU_LOAD))
fpregs_restore_userregs();
save_fpregs_to_fpstate(dst_fpu);
fpregs_unlock();
if (!(clone_flags & CLONE_THREAD))
fpu_inherit_perms(dst_fpu);
/*
* Children never inherit PASID state.
* Force it to have its init value:
*/
if (use_xsave())
dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID;
/*
* Update shadow stack pointer, in case it changed during clone.
*/
if (update_fpu_shstk(dst, ssp))
return 1;
trace_x86_fpu_copy_src(src_fpu);
trace_x86_fpu_copy_dst(dst_fpu);
return 0;
}
/*
* Whitelist the FPU register state embedded into task_struct for hardened
* usercopy.
*/
void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size)
{
*offset = offsetof(struct thread_struct, fpu.__fpstate.regs);
*size = fpu_kernel_cfg.default_size;
}
/*
* Drops current FPU state: deactivates the fpregs and
* the fpstate. NOTE: it still leaves previous contents
* in the fpregs in the eager-FPU case.
*
* This function can be used in cases where we know that
* a state-restore is coming: either an explicit one,
* or a reschedule.
*/
void fpu__drop(struct fpu *fpu)
{
preempt_disable();
if (fpu == ¤t->thread.fpu) {
/* Ignore delayed exceptions from user space */
asm volatile("1: fwait\n"
"2:\n"
_ASM_EXTABLE(1b, 2b));
fpregs_deactivate(fpu);
}
trace_x86_fpu_dropped(fpu);
preempt_enable();
}
/*
* Clear FPU registers by setting them up from the init fpstate.
* Caller must do fpregs_[un]lock() around it.
*/
static inline void restore_fpregs_from_init_fpstate(u64 features_mask)
{
if (use_xsave()) os_xrstor(&init_fpstate, features_mask);
else if (use_fxsr())
fxrstor(&init_fpstate.regs.fxsave);
else
frstor(&init_fpstate.regs.fsave);
pkru_write_default();
}
/*
* Reset current->fpu memory state to the init values.
*/
static void fpu_reset_fpregs(void)
{
struct fpu *fpu = ¤t->thread.fpu;
fpregs_lock();
__fpu_invalidate_fpregs_state(fpu);
/*
* This does not change the actual hardware registers. It just
* resets the memory image and sets TIF_NEED_FPU_LOAD so a
* subsequent return to usermode will reload the registers from the
* task's memory image.
*
* Do not use fpstate_init() here. Just copy init_fpstate which has
* the correct content already except for PKRU.
*
* PKRU handling does not rely on the xstate when restoring for
* user space as PKRU is eagerly written in switch_to() and
* flush_thread().
*/
memcpy(&fpu->fpstate->regs, &init_fpstate.regs, init_fpstate_copy_size());
set_thread_flag(TIF_NEED_FPU_LOAD);
fpregs_unlock();
}
/*
* Reset current's user FPU states to the init states. current's
* supervisor states, if any, are not modified by this function. The
* caller guarantees that the XSTATE header in memory is intact.
*/
void fpu__clear_user_states(struct fpu *fpu)
{
WARN_ON_FPU(fpu != ¤t->thread.fpu); fpregs_lock();
if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
fpu_reset_fpregs();
fpregs_unlock();
return;
}
/*
* Ensure that current's supervisor states are loaded into their
* corresponding registers.
*/
if (xfeatures_mask_supervisor() &&
!fpregs_state_valid(fpu, smp_processor_id())) os_xrstor_supervisor(fpu->fpstate);
/* Reset user states in registers. */
restore_fpregs_from_init_fpstate(XFEATURE_MASK_USER_RESTORE);
/*
* Now all FPU registers have their desired values. Inform the FPU
* state machine that current's FPU registers are in the hardware
* registers. The memory image does not need to be updated because
* any operation relying on it has to save the registers first when
* current's FPU is marked active.
*/
fpregs_mark_activate();
fpregs_unlock();
}
void fpu_flush_thread(void)
{
fpstate_reset(¤t->thread.fpu);
fpu_reset_fpregs();
}
/*
* Load FPU context before returning to userspace.
*/
void switch_fpu_return(void)
{
if (!static_cpu_has(X86_FEATURE_FPU))
return;
fpregs_restore_userregs();
}
EXPORT_SYMBOL_GPL(switch_fpu_return);
void fpregs_lock_and_load(void)
{
/*
* fpregs_lock() only disables preemption (mostly). So modifying state
* in an interrupt could screw up some in progress fpregs operation.
* Warn about it.
*/
WARN_ON_ONCE(!irq_fpu_usable());
WARN_ON_ONCE(current->flags & PF_KTHREAD);
fpregs_lock();
fpregs_assert_state_consistent();
if (test_thread_flag(TIF_NEED_FPU_LOAD))
fpregs_restore_userregs();
}
#ifdef CONFIG_X86_DEBUG_FPU
/*
* If current FPU state according to its tracking (loaded FPU context on this
* CPU) is not valid then we must have TIF_NEED_FPU_LOAD set so the context is
* loaded on return to userland.
*/
void fpregs_assert_state_consistent(void)
{
struct fpu *fpu = ¤t->thread.fpu;
if (test_thread_flag(TIF_NEED_FPU_LOAD))
return;
WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id()));
}
EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent);
#endif
void fpregs_mark_activate(void)
{
struct fpu *fpu = ¤t->thread.fpu; fpregs_activate(fpu); fpu->last_cpu = smp_processor_id();
clear_thread_flag(TIF_NEED_FPU_LOAD);
}
/*
* x87 math exception handling:
*/
int fpu__exception_code(struct fpu *fpu, int trap_nr)
{
int err;
if (trap_nr == X86_TRAP_MF) {
unsigned short cwd, swd;
/*
* (~cwd & swd) will mask out exceptions that are not set to unmasked
* status. 0x3f is the exception bits in these regs, 0x200 is the
* C1 reg you need in case of a stack fault, 0x040 is the stack
* fault bit. We should only be taking one exception at a time,
* so if this combination doesn't produce any single exception,
* then we have a bad program that isn't synchronizing its FPU usage
* and it will suffer the consequences since we won't be able to
* fully reproduce the context of the exception.
*/
if (boot_cpu_has(X86_FEATURE_FXSR)) {
cwd = fpu->fpstate->regs.fxsave.cwd;
swd = fpu->fpstate->regs.fxsave.swd;
} else {
cwd = (unsigned short)fpu->fpstate->regs.fsave.cwd;
swd = (unsigned short)fpu->fpstate->regs.fsave.swd;
}
err = swd & ~cwd;
} else {
/*
* The SIMD FPU exceptions are handled a little differently, as there
* is only a single status/control register. Thus, to determine which
* unmasked exception was caught we must mask the exception mask bits
* at 0x1f80, and then use these to mask the exception bits at 0x3f.
*/
unsigned short mxcsr = MXCSR_DEFAULT;
if (boot_cpu_has(X86_FEATURE_XMM))
mxcsr = fpu->fpstate->regs.fxsave.mxcsr;
err = ~(mxcsr >> 7) & mxcsr;
}
if (err & 0x001) { /* Invalid op */
/*
* swd & 0x240 == 0x040: Stack Underflow
* swd & 0x240 == 0x240: Stack Overflow
* User must clear the SF bit (0x40) if set
*/
return FPE_FLTINV;
} else if (err & 0x004) { /* Divide by Zero */
return FPE_FLTDIV;
} else if (err & 0x008) { /* Overflow */
return FPE_FLTOVF;
} else if (err & 0x012) { /* Denormal, Underflow */
return FPE_FLTUND;
} else if (err & 0x020) { /* Precision */
return FPE_FLTRES;
}
/*
* If we're using IRQ 13, or supposedly even some trap
* X86_TRAP_MF implementations, it's possible
* we get a spurious trap, which is not an error.
*/
return 0;
}
/*
* Initialize register state that may prevent from entering low-power idle.
* This function will be invoked from the cpuidle driver only when needed.
*/
noinstr void fpu_idle_fpregs(void)
{
/* Note: AMX_TILE being enabled implies XGETBV1 support */
if (cpu_feature_enabled(X86_FEATURE_AMX_TILE) &&
(xfeatures_in_use() & XFEATURE_MASK_XTILE)) {
tile_release();
__this_cpu_write(fpu_fpregs_owner_ctx, NULL);
}
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_COMPLETION_H
#define __LINUX_COMPLETION_H
/*
* (C) Copyright 2001 Linus Torvalds
*
* Atomic wait-for-completion handler data structures.
* See kernel/sched/completion.c for details.
*/
#include <linux/swait.h>
/*
* struct completion - structure used to maintain state for a "completion"
*
* This is the opaque structure used to maintain the state for a "completion".
* Completions currently use a FIFO to queue threads that have to wait for
* the "completion" event.
*
* See also: complete(), wait_for_completion() (and friends _timeout,
* _interruptible, _interruptible_timeout, and _killable), init_completion(),
* reinit_completion(), and macros DECLARE_COMPLETION(),
* DECLARE_COMPLETION_ONSTACK().
*/
struct completion {
unsigned int done;
struct swait_queue_head wait;
};
#define init_completion_map(x, m) init_completion(x)
static inline void complete_acquire(struct completion *x) {}
static inline void complete_release(struct completion *x) {}
#define COMPLETION_INITIALIZER(work) \
{ 0, __SWAIT_QUEUE_HEAD_INITIALIZER((work).wait) }
#define COMPLETION_INITIALIZER_ONSTACK_MAP(work, map) \
(*({ init_completion_map(&(work), &(map)); &(work); }))
#define COMPLETION_INITIALIZER_ONSTACK(work) \
(*({ init_completion(&work); &work; }))
/**
* DECLARE_COMPLETION - declare and initialize a completion structure
* @work: identifier for the completion structure
*
* This macro declares and initializes a completion structure. Generally used
* for static declarations. You should use the _ONSTACK variant for automatic
* variables.
*/
#define DECLARE_COMPLETION(work) \
struct completion work = COMPLETION_INITIALIZER(work)
/*
* Lockdep needs to run a non-constant initializer for on-stack
* completions - so we use the _ONSTACK() variant for those that
* are on the kernel stack:
*/
/**
* DECLARE_COMPLETION_ONSTACK - declare and initialize a completion structure
* @work: identifier for the completion structure
*
* This macro declares and initializes a completion structure on the kernel
* stack.
*/
#ifdef CONFIG_LOCKDEP
# define DECLARE_COMPLETION_ONSTACK(work) \
struct completion work = COMPLETION_INITIALIZER_ONSTACK(work)
# define DECLARE_COMPLETION_ONSTACK_MAP(work, map) \
struct completion work = COMPLETION_INITIALIZER_ONSTACK_MAP(work, map)
#else
# define DECLARE_COMPLETION_ONSTACK(work) DECLARE_COMPLETION(work)
# define DECLARE_COMPLETION_ONSTACK_MAP(work, map) DECLARE_COMPLETION(work)
#endif
/**
* init_completion - Initialize a dynamically allocated completion
* @x: pointer to completion structure that is to be initialized
*
* This inline function will initialize a dynamically created completion
* structure.
*/
static inline void init_completion(struct completion *x)
{
x->done = 0;
init_swait_queue_head(&x->wait);
}
/**
* reinit_completion - reinitialize a completion structure
* @x: pointer to completion structure that is to be reinitialized
*
* This inline function should be used to reinitialize a completion structure so it can
* be reused. This is especially important after complete_all() is used.
*/
static inline void reinit_completion(struct completion *x)
{
x->done = 0;
}
extern void wait_for_completion(struct completion *);
extern void wait_for_completion_io(struct completion *);
extern int wait_for_completion_interruptible(struct completion *x);
extern int wait_for_completion_killable(struct completion *x);
extern int wait_for_completion_state(struct completion *x, unsigned int state);
extern unsigned long wait_for_completion_timeout(struct completion *x,
unsigned long timeout);
extern unsigned long wait_for_completion_io_timeout(struct completion *x,
unsigned long timeout);
extern long wait_for_completion_interruptible_timeout(
struct completion *x, unsigned long timeout);
extern long wait_for_completion_killable_timeout(
struct completion *x, unsigned long timeout);
extern bool try_wait_for_completion(struct completion *x);
extern bool completion_done(struct completion *x);
extern void complete(struct completion *);
extern void complete_on_current_cpu(struct completion *x);
extern void complete_all(struct completion *);
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_BITOPS_H
#define _ASM_X86_BITOPS_H
/*
* Copyright 1992, Linus Torvalds.
*
* Note: inlines with more than a single statement should be marked
* __always_inline to avoid problems with older gcc's inlining heuristics.
*/
#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif
#include <linux/compiler.h>
#include <asm/alternative.h>
#include <asm/rmwcc.h>
#include <asm/barrier.h>
#if BITS_PER_LONG == 32
# define _BITOPS_LONG_SHIFT 5
#elif BITS_PER_LONG == 64
# define _BITOPS_LONG_SHIFT 6
#else
# error "Unexpected BITS_PER_LONG"
#endif
#define BIT_64(n) (U64_C(1) << (n))
/*
* These have to be done with inline assembly: that way the bit-setting
* is guaranteed to be atomic. All bit operations return 0 if the bit
* was cleared before the operation and != 0 if it was not.
*
* bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
*/
#define RLONG_ADDR(x) "m" (*(volatile long *) (x))
#define WBYTE_ADDR(x) "+m" (*(volatile char *) (x))
#define ADDR RLONG_ADDR(addr)
/*
* We do the locked ops that don't return the old value as
* a mask operation on a byte.
*/
#define CONST_MASK_ADDR(nr, addr) WBYTE_ADDR((void *)(addr) + ((nr)>>3))
#define CONST_MASK(nr) (1 << ((nr) & 7))
static __always_inline void
arch_set_bit(long nr, volatile unsigned long *addr)
{
if (__builtin_constant_p(nr)) {
asm volatile(LOCK_PREFIX "orb %b1,%0"
: CONST_MASK_ADDR(nr, addr)
: "iq" (CONST_MASK(nr))
: "memory");
} else {
asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0"
: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
}
}
static __always_inline void
arch___set_bit(unsigned long nr, volatile unsigned long *addr)
{
asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
}
static __always_inline void
arch_clear_bit(long nr, volatile unsigned long *addr)
{
if (__builtin_constant_p(nr)) {
asm volatile(LOCK_PREFIX "andb %b1,%0"
: CONST_MASK_ADDR(nr, addr)
: "iq" (~CONST_MASK(nr)));
} else {
asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0"
: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
}
}
static __always_inline void
arch_clear_bit_unlock(long nr, volatile unsigned long *addr)
{
barrier();
arch_clear_bit(nr, addr);
}
static __always_inline void
arch___clear_bit(unsigned long nr, volatile unsigned long *addr)
{
asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
}
static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
volatile unsigned long *addr)
{
bool negative;
asm volatile(LOCK_PREFIX "xorb %2,%1"
CC_SET(s)
: CC_OUT(s) (negative), WBYTE_ADDR(addr)
: "iq" ((char)mask) : "memory");
return negative;
}
#define arch_xor_unlock_is_negative_byte arch_xor_unlock_is_negative_byte
static __always_inline void
arch___clear_bit_unlock(long nr, volatile unsigned long *addr)
{
arch___clear_bit(nr, addr);
}
static __always_inline void
arch___change_bit(unsigned long nr, volatile unsigned long *addr)
{
asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
}
static __always_inline void
arch_change_bit(long nr, volatile unsigned long *addr)
{
if (__builtin_constant_p(nr)) {
asm volatile(LOCK_PREFIX "xorb %b1,%0"
: CONST_MASK_ADDR(nr, addr)
: "iq" (CONST_MASK(nr)));
} else {
asm volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
}
}
static __always_inline bool
arch_test_and_set_bit(long nr, volatile unsigned long *addr)
{
return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), *addr, c, "Ir", nr);
}
static __always_inline bool
arch_test_and_set_bit_lock(long nr, volatile unsigned long *addr)
{
return arch_test_and_set_bit(nr, addr);
}
static __always_inline bool
arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
{
bool oldbit;
asm(__ASM_SIZE(bts) " %2,%1"
CC_SET(c)
: CC_OUT(c) (oldbit)
: ADDR, "Ir" (nr) : "memory");
return oldbit;
}
static __always_inline bool
arch_test_and_clear_bit(long nr, volatile unsigned long *addr)
{
return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), *addr, c, "Ir", nr);
}
/*
* Note: the operation is performed atomically with respect to
* the local CPU, but not other CPUs. Portable code should not
* rely on this behaviour.
* KVM relies on this behaviour on x86 for modifying memory that is also
* accessed from a hypervisor on the same CPU if running in a VM: don't change
* this without also updating arch/x86/kernel/kvm.c
*/
static __always_inline bool
arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
{
bool oldbit;
asm volatile(__ASM_SIZE(btr) " %2,%1"
CC_SET(c)
: CC_OUT(c) (oldbit)
: ADDR, "Ir" (nr) : "memory");
return oldbit;
}
static __always_inline bool
arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
{
bool oldbit;
asm volatile(__ASM_SIZE(btc) " %2,%1"
CC_SET(c)
: CC_OUT(c) (oldbit)
: ADDR, "Ir" (nr) : "memory");
return oldbit;
}
static __always_inline bool
arch_test_and_change_bit(long nr, volatile unsigned long *addr)
{
return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), *addr, c, "Ir", nr);
}
static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
{
return ((1UL << (nr & (BITS_PER_LONG-1))) &
(addr[nr >> _BITOPS_LONG_SHIFT])) != 0;
}
static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr)
{
bool oldbit;
asm volatile("testb %2,%1"
CC_SET(nz)
: CC_OUT(nz) (oldbit)
: "m" (((unsigned char *)addr)[nr >> 3]),
"i" (1 << (nr & 7))
:"memory");
return oldbit;
}
static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)
{
bool oldbit;
asm volatile(__ASM_SIZE(bt) " %2,%1"
CC_SET(c)
: CC_OUT(c) (oldbit)
: "m" (*(unsigned long *)addr), "Ir" (nr) : "memory");
return oldbit;
}
static __always_inline bool
arch_test_bit(unsigned long nr, const volatile unsigned long *addr)
{
return __builtin_constant_p(nr) ? constant_test_bit(nr, addr) : variable_test_bit(nr, addr);
}
static __always_inline bool
arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
{
return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) :
variable_test_bit(nr, addr);
}
static __always_inline unsigned long variable__ffs(unsigned long word)
{
asm("rep; bsf %1,%0"
: "=r" (word)
: "rm" (word));
return word;
}
/**
* __ffs - find first set bit in word
* @word: The word to search
*
* Undefined if no bit exists, so code should check against 0 first.
*/
#define __ffs(word) \
(__builtin_constant_p(word) ? \
(unsigned long)__builtin_ctzl(word) : \
variable__ffs(word))
static __always_inline unsigned long variable_ffz(unsigned long word)
{
asm("rep; bsf %1,%0"
: "=r" (word)
: "r" (~word));
return word;
}
/**
* ffz - find first zero bit in word
* @word: The word to search
*
* Undefined if no zero exists, so code should check against ~0UL first.
*/
#define ffz(word) \
(__builtin_constant_p(word) ? \
(unsigned long)__builtin_ctzl(~word) : \
variable_ffz(word))
/*
* __fls: find last set bit in word
* @word: The word to search
*
* Undefined if no set bit exists, so code should check against 0 first.
*/
static __always_inline unsigned long __fls(unsigned long word)
{
if (__builtin_constant_p(word))
return BITS_PER_LONG - 1 - __builtin_clzl(word);
asm("bsr %1,%0"
: "=r" (word)
: "rm" (word));
return word;
}
#undef ADDR
#ifdef __KERNEL__
static __always_inline int variable_ffs(int x)
{
int r;
#ifdef CONFIG_X86_64
/*
* AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the
* dest reg is undefined if x==0, but their CPU architect says its
* value is written to set it to the same as before, except that the
* top 32 bits will be cleared.
*
* We cannot do this on 32 bits because at the very least some
* 486 CPUs did not behave this way.
*/
asm("bsfl %1,%0"
: "=r" (r)
: "rm" (x), "0" (-1));
#elif defined(CONFIG_X86_CMOV)
asm("bsfl %1,%0\n\t"
"cmovzl %2,%0"
: "=&r" (r) : "rm" (x), "r" (-1));
#else
asm("bsfl %1,%0\n\t"
"jnz 1f\n\t"
"movl $-1,%0\n"
"1:" : "=r" (r) : "rm" (x));
#endif
return r + 1;
}
/**
* ffs - find first set bit in word
* @x: the word to search
*
* This is defined the same way as the libc and compiler builtin ffs
* routines, therefore differs in spirit from the other bitops.
*
* ffs(value) returns 0 if value is 0 or the position of the first
* set bit if value is nonzero. The first (least significant) bit
* is at position 1.
*/
#define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x))
/**
* fls - find last set bit in word
* @x: the word to search
*
* This is defined in a similar way as the libc and compiler builtin
* ffs, but returns the position of the most significant set bit.
*
* fls(value) returns 0 if value is 0 or the position of the last
* set bit if value is nonzero. The last (most significant) bit is
* at position 32.
*/
static __always_inline int fls(unsigned int x)
{
int r;
if (__builtin_constant_p(x))
return x ? 32 - __builtin_clz(x) : 0;
#ifdef CONFIG_X86_64
/*
* AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the
* dest reg is undefined if x==0, but their CPU architect says its
* value is written to set it to the same as before, except that the
* top 32 bits will be cleared.
*
* We cannot do this on 32 bits because at the very least some
* 486 CPUs did not behave this way.
*/
asm("bsrl %1,%0"
: "=r" (r)
: "rm" (x), "0" (-1));
#elif defined(CONFIG_X86_CMOV)
asm("bsrl %1,%0\n\t"
"cmovzl %2,%0"
: "=&r" (r) : "rm" (x), "rm" (-1));
#else
asm("bsrl %1,%0\n\t"
"jnz 1f\n\t"
"movl $-1,%0\n"
"1:" : "=r" (r) : "rm" (x));
#endif
return r + 1;
}
/**
* fls64 - find last set bit in a 64-bit word
* @x: the word to search
*
* This is defined in a similar way as the libc and compiler builtin
* ffsll, but returns the position of the most significant set bit.
*
* fls64(value) returns 0 if value is 0 or the position of the last
* set bit if value is nonzero. The last (most significant) bit is
* at position 64.
*/
#ifdef CONFIG_X86_64
static __always_inline int fls64(__u64 x)
{
int bitpos = -1;
if (__builtin_constant_p(x))
return x ? 64 - __builtin_clzll(x) : 0;
/*
* AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the
* dest reg is undefined if x==0, but their CPU architect says its
* value is written to set it to the same as before.
*/
asm("bsrq %1,%q0"
: "+r" (bitpos)
: "rm" (x));
return bitpos + 1;
}
#else
#include <asm-generic/bitops/fls64.h>
#endif
#include <asm-generic/bitops/sched.h>
#include <asm/arch_hweight.h>
#include <asm-generic/bitops/const_hweight.h>
#include <asm-generic/bitops/instrumented-atomic.h>
#include <asm-generic/bitops/instrumented-non-atomic.h>
#include <asm-generic/bitops/instrumented-lock.h>
#include <asm-generic/bitops/le.h>
#include <asm-generic/bitops/ext2-atomic-setbit.h>
#endif /* __KERNEL__ */
#endif /* _ASM_X86_BITOPS_H */
// SPDX-License-Identifier: GPL-2.0
/*
* bus.c - bus driver management
*
* Copyright (c) 2002-3 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
* Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de>
* Copyright (c) 2007 Novell Inc.
* Copyright (c) 2023 Greg Kroah-Hartman <gregkh@linuxfoundation.org>
*/
#include <linux/async.h>
#include <linux/device/bus.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/mutex.h>
#include <linux/sysfs.h>
#include "base.h"
#include "power/power.h"
/* /sys/devices/system */
static struct kset *system_kset;
/* /sys/bus */
static struct kset *bus_kset;
#define to_bus_attr(_attr) container_of(_attr, struct bus_attribute, attr)
/*
* sysfs bindings for drivers
*/
#define to_drv_attr(_attr) container_of(_attr, struct driver_attribute, attr)
#define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \
struct driver_attribute driver_attr_##_name = \
__ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store)
static int __must_check bus_rescan_devices_helper(struct device *dev,
void *data);
/**
* bus_to_subsys - Turn a struct bus_type into a struct subsys_private
*
* @bus: pointer to the struct bus_type to look up
*
* The driver core internals needs to work on the subsys_private structure, not
* the external struct bus_type pointer. This function walks the list of
* registered busses in the system and finds the matching one and returns the
* internal struct subsys_private that relates to that bus.
*
* Note, the reference count of the return value is INCREMENTED if it is not
* NULL. A call to subsys_put() must be done when finished with the pointer in
* order for it to be properly freed.
*/
static struct subsys_private *bus_to_subsys(const struct bus_type *bus)
{
struct subsys_private *sp = NULL;
struct kobject *kobj;
if (!bus || !bus_kset)
return NULL;
spin_lock(&bus_kset->list_lock);
if (list_empty(&bus_kset->list))
goto done;
list_for_each_entry(kobj, &bus_kset->list, entry) { struct kset *kset = container_of(kobj, struct kset, kobj);
sp = container_of_const(kset, struct subsys_private, subsys);
if (sp->bus == bus)
goto done;
}
sp = NULL;
done:
sp = subsys_get(sp); spin_unlock(&bus_kset->list_lock); return sp;
}
static const struct bus_type *bus_get(const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
if (sp)
return bus;
return NULL;
}
static void bus_put(const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
/* two puts are required as the call to bus_to_subsys incremented it again */
subsys_put(sp);
subsys_put(sp);
}
static ssize_t drv_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct driver_attribute *drv_attr = to_drv_attr(attr);
struct driver_private *drv_priv = to_driver(kobj);
ssize_t ret = -EIO;
if (drv_attr->show)
ret = drv_attr->show(drv_priv->driver, buf);
return ret;
}
static ssize_t drv_attr_store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t count)
{
struct driver_attribute *drv_attr = to_drv_attr(attr);
struct driver_private *drv_priv = to_driver(kobj);
ssize_t ret = -EIO;
if (drv_attr->store)
ret = drv_attr->store(drv_priv->driver, buf, count);
return ret;
}
static const struct sysfs_ops driver_sysfs_ops = {
.show = drv_attr_show,
.store = drv_attr_store,
};
static void driver_release(struct kobject *kobj)
{
struct driver_private *drv_priv = to_driver(kobj);
pr_debug("driver: '%s': %s\n", kobject_name(kobj), __func__); kfree(drv_priv);
}
static const struct kobj_type driver_ktype = {
.sysfs_ops = &driver_sysfs_ops,
.release = driver_release,
};
/*
* sysfs bindings for buses
*/
static ssize_t bus_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct bus_attribute *bus_attr = to_bus_attr(attr);
struct subsys_private *subsys_priv = to_subsys_private(kobj);
ssize_t ret = 0;
if (bus_attr->show)
ret = bus_attr->show(subsys_priv->bus, buf);
return ret;
}
static ssize_t bus_attr_store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t count)
{
struct bus_attribute *bus_attr = to_bus_attr(attr);
struct subsys_private *subsys_priv = to_subsys_private(kobj);
ssize_t ret = 0;
if (bus_attr->store)
ret = bus_attr->store(subsys_priv->bus, buf, count);
return ret;
}
static const struct sysfs_ops bus_sysfs_ops = {
.show = bus_attr_show,
.store = bus_attr_store,
};
int bus_create_file(const struct bus_type *bus, struct bus_attribute *attr)
{
struct subsys_private *sp = bus_to_subsys(bus);
int error;
if (!sp)
return -EINVAL;
error = sysfs_create_file(&sp->subsys.kobj, &attr->attr);
subsys_put(sp);
return error;
}
EXPORT_SYMBOL_GPL(bus_create_file);
void bus_remove_file(const struct bus_type *bus, struct bus_attribute *attr)
{
struct subsys_private *sp = bus_to_subsys(bus);
if (!sp)
return;
sysfs_remove_file(&sp->subsys.kobj, &attr->attr);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(bus_remove_file);
static void bus_release(struct kobject *kobj)
{
struct subsys_private *priv = to_subsys_private(kobj);
lockdep_unregister_key(&priv->lock_key);
kfree(priv);
}
static const struct kobj_type bus_ktype = {
.sysfs_ops = &bus_sysfs_ops,
.release = bus_release,
};
static int bus_uevent_filter(const struct kobject *kobj)
{
const struct kobj_type *ktype = get_ktype(kobj);
if (ktype == &bus_ktype)
return 1;
return 0;
}
static const struct kset_uevent_ops bus_uevent_ops = {
.filter = bus_uevent_filter,
};
/* Manually detach a device from its associated driver. */
static ssize_t unbind_store(struct device_driver *drv, const char *buf,
size_t count)
{
const struct bus_type *bus = bus_get(drv->bus);
struct device *dev;
int err = -ENODEV;
dev = bus_find_device_by_name(bus, NULL, buf);
if (dev && dev->driver == drv) {
device_driver_detach(dev);
err = count;
}
put_device(dev);
bus_put(bus);
return err;
}
static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store);
/*
* Manually attach a device to a driver.
* Note: the driver must want to bind to the device,
* it is not possible to override the driver's id table.
*/
static ssize_t bind_store(struct device_driver *drv, const char *buf,
size_t count)
{
const struct bus_type *bus = bus_get(drv->bus);
struct device *dev;
int err = -ENODEV;
dev = bus_find_device_by_name(bus, NULL, buf);
if (dev && driver_match_device(drv, dev)) {
err = device_driver_attach(drv, dev);
if (!err) {
/* success */
err = count;
}
}
put_device(dev);
bus_put(bus);
return err;
}
static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store);
static ssize_t drivers_autoprobe_show(const struct bus_type *bus, char *buf)
{
struct subsys_private *sp = bus_to_subsys(bus);
int ret;
if (!sp)
return -EINVAL;
ret = sysfs_emit(buf, "%d\n", sp->drivers_autoprobe);
subsys_put(sp);
return ret;
}
static ssize_t drivers_autoprobe_store(const struct bus_type *bus,
const char *buf, size_t count)
{
struct subsys_private *sp = bus_to_subsys(bus);
if (!sp)
return -EINVAL;
if (buf[0] == '0')
sp->drivers_autoprobe = 0;
else
sp->drivers_autoprobe = 1;
subsys_put(sp);
return count;
}
static ssize_t drivers_probe_store(const struct bus_type *bus,
const char *buf, size_t count)
{
struct device *dev;
int err = -EINVAL;
dev = bus_find_device_by_name(bus, NULL, buf);
if (!dev)
return -ENODEV;
if (bus_rescan_devices_helper(dev, NULL) == 0)
err = count;
put_device(dev);
return err;
}
static struct device *next_device(struct klist_iter *i)
{
struct klist_node *n = klist_next(i);
struct device *dev = NULL;
struct device_private *dev_prv;
if (n) {
dev_prv = to_device_private_bus(n);
dev = dev_prv->device;
}
return dev;
}
/**
* bus_for_each_dev - device iterator.
* @bus: bus type.
* @start: device to start iterating from.
* @data: data for the callback.
* @fn: function to be called for each device.
*
* Iterate over @bus's list of devices, and call @fn for each,
* passing it @data. If @start is not NULL, we use that device to
* begin iterating from.
*
* We check the return of @fn each time. If it returns anything
* other than 0, we break out and return that value.
*
* NOTE: The device that returns a non-zero value is not retained
* in any way, nor is its refcount incremented. If the caller needs
* to retain this data, it should do so, and increment the reference
* count in the supplied callback.
*/
int bus_for_each_dev(const struct bus_type *bus, struct device *start,
void *data, int (*fn)(struct device *, void *))
{
struct subsys_private *sp = bus_to_subsys(bus);
struct klist_iter i;
struct device *dev;
int error = 0;
if (!sp)
return -EINVAL;
klist_iter_init_node(&sp->klist_devices, &i, (start ? &start->p->knode_bus : NULL)); while (!error && (dev = next_device(&i))) error = fn(dev, data); klist_iter_exit(&i);
subsys_put(sp);
return error;
}
EXPORT_SYMBOL_GPL(bus_for_each_dev);
/**
* bus_find_device - device iterator for locating a particular device.
* @bus: bus type
* @start: Device to begin with
* @data: Data to pass to match function
* @match: Callback function to check device
*
* This is similar to the bus_for_each_dev() function above, but it
* returns a reference to a device that is 'found' for later use, as
* determined by the @match callback.
*
* The callback should return 0 if the device doesn't match and non-zero
* if it does. If the callback returns non-zero, this function will
* return to the caller and not iterate over any more devices.
*/
struct device *bus_find_device(const struct bus_type *bus,
struct device *start, const void *data,
int (*match)(struct device *dev, const void *data))
{
struct subsys_private *sp = bus_to_subsys(bus);
struct klist_iter i;
struct device *dev;
if (!sp)
return NULL;
klist_iter_init_node(&sp->klist_devices, &i, (start ? &start->p->knode_bus : NULL)); while ((dev = next_device(&i))) if (match(dev, data) && get_device(dev))
break;
klist_iter_exit(&i);
subsys_put(sp);
return dev;
}
EXPORT_SYMBOL_GPL(bus_find_device);
static struct device_driver *next_driver(struct klist_iter *i)
{
struct klist_node *n = klist_next(i);
struct driver_private *drv_priv;
if (n) {
drv_priv = container_of(n, struct driver_private, knode_bus);
return drv_priv->driver;
}
return NULL;
}
/**
* bus_for_each_drv - driver iterator
* @bus: bus we're dealing with.
* @start: driver to start iterating on.
* @data: data to pass to the callback.
* @fn: function to call for each driver.
*
* This is nearly identical to the device iterator above.
* We iterate over each driver that belongs to @bus, and call
* @fn for each. If @fn returns anything but 0, we break out
* and return it. If @start is not NULL, we use it as the head
* of the list.
*
* NOTE: we don't return the driver that returns a non-zero
* value, nor do we leave the reference count incremented for that
* driver. If the caller needs to know that info, it must set it
* in the callback. It must also be sure to increment the refcount
* so it doesn't disappear before returning to the caller.
*/
int bus_for_each_drv(const struct bus_type *bus, struct device_driver *start,
void *data, int (*fn)(struct device_driver *, void *))
{
struct subsys_private *sp = bus_to_subsys(bus);
struct klist_iter i;
struct device_driver *drv;
int error = 0;
if (!sp)
return -EINVAL;
klist_iter_init_node(&sp->klist_drivers, &i, start ? &start->p->knode_bus : NULL); while ((drv = next_driver(&i)) && !error) error = fn(drv, data); klist_iter_exit(&i);
subsys_put(sp);
return error;
}
EXPORT_SYMBOL_GPL(bus_for_each_drv);
/**
* bus_add_device - add device to bus
* @dev: device being added
*
* - Add device's bus attributes.
* - Create links to device's bus.
* - Add the device to its bus's list of devices.
*/
int bus_add_device(struct device *dev)
{
struct subsys_private *sp = bus_to_subsys(dev->bus);
int error;
if (!sp) {
/*
* This is a normal operation for many devices that do not
* have a bus assigned to them, just say that all went
* well.
*/
return 0;
}
/*
* Reference in sp is now incremented and will be dropped when
* the device is removed from the bus
*/
pr_debug("bus: '%s': add device %s\n", sp->bus->name, dev_name(dev)); error = device_add_groups(dev, sp->bus->dev_groups);
if (error)
goto out_put;
error = sysfs_create_link(&sp->devices_kset->kobj, &dev->kobj, dev_name(dev));
if (error)
goto out_groups;
error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem");
if (error)
goto out_subsys;
klist_add_tail(&dev->p->knode_bus, &sp->klist_devices);
return 0;
out_subsys:
sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev));
out_groups:
device_remove_groups(dev, sp->bus->dev_groups);
out_put:
subsys_put(sp); return error;
}
/**
* bus_probe_device - probe drivers for a new device
* @dev: device to probe
*
* - Automatically probe for a driver if the bus allows it.
*/
void bus_probe_device(struct device *dev)
{
struct subsys_private *sp = bus_to_subsys(dev->bus);
struct subsys_interface *sif;
if (!sp)
return;
if (sp->drivers_autoprobe) device_initial_probe(dev); mutex_lock(&sp->mutex); list_for_each_entry(sif, &sp->interfaces, node) if (sif->add_dev) sif->add_dev(dev, sif); mutex_unlock(&sp->mutex);
subsys_put(sp);
}
/**
* bus_remove_device - remove device from bus
* @dev: device to be removed
*
* - Remove device from all interfaces.
* - Remove symlink from bus' directory.
* - Delete device from bus's list.
* - Detach from its driver.
* - Drop reference taken in bus_add_device().
*/
void bus_remove_device(struct device *dev)
{
struct subsys_private *sp = bus_to_subsys(dev->bus);
struct subsys_interface *sif;
if (!sp)
return;
mutex_lock(&sp->mutex); list_for_each_entry(sif, &sp->interfaces, node) if (sif->remove_dev) sif->remove_dev(dev, sif); mutex_unlock(&sp->mutex);
sysfs_remove_link(&dev->kobj, "subsystem");
sysfs_remove_link(&sp->devices_kset->kobj, dev_name(dev));
device_remove_groups(dev, dev->bus->dev_groups);
if (klist_node_attached(&dev->p->knode_bus))
klist_del(&dev->p->knode_bus); pr_debug("bus: '%s': remove device %s\n",
dev->bus->name, dev_name(dev));
device_release_driver(dev);
/*
* Decrement the reference count twice, once for the bus_to_subsys()
* call in the start of this function, and the second one from the
* reference increment in bus_add_device()
*/
subsys_put(sp);
subsys_put(sp);
}
static int __must_check add_bind_files(struct device_driver *drv)
{
int ret;
ret = driver_create_file(drv, &driver_attr_unbind);
if (ret == 0) {
ret = driver_create_file(drv, &driver_attr_bind);
if (ret)
driver_remove_file(drv, &driver_attr_unbind);
}
return ret;
}
static void remove_bind_files(struct device_driver *drv)
{
driver_remove_file(drv, &driver_attr_bind);
driver_remove_file(drv, &driver_attr_unbind);
}
static BUS_ATTR_WO(drivers_probe);
static BUS_ATTR_RW(drivers_autoprobe);
static int add_probe_files(const struct bus_type *bus)
{
int retval;
retval = bus_create_file(bus, &bus_attr_drivers_probe);
if (retval)
goto out;
retval = bus_create_file(bus, &bus_attr_drivers_autoprobe);
if (retval)
bus_remove_file(bus, &bus_attr_drivers_probe);
out:
return retval;
}
static void remove_probe_files(const struct bus_type *bus)
{
bus_remove_file(bus, &bus_attr_drivers_autoprobe);
bus_remove_file(bus, &bus_attr_drivers_probe);
}
static ssize_t uevent_store(struct device_driver *drv, const char *buf,
size_t count)
{
int rc;
rc = kobject_synth_uevent(&drv->p->kobj, buf, count);
return rc ? rc : count;
}
static DRIVER_ATTR_WO(uevent);
/**
* bus_add_driver - Add a driver to the bus.
* @drv: driver.
*/
int bus_add_driver(struct device_driver *drv)
{
struct subsys_private *sp = bus_to_subsys(drv->bus);
struct driver_private *priv;
int error = 0;
if (!sp)
return -EINVAL;
/*
* Reference in sp is now incremented and will be dropped when
* the driver is removed from the bus
*/
pr_debug("bus: '%s': add driver %s\n", sp->bus->name, drv->name); priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv) {
error = -ENOMEM;
goto out_put_bus;
}
klist_init(&priv->klist_devices, NULL, NULL);
priv->driver = drv;
drv->p = priv;
priv->kobj.kset = sp->drivers_kset;
error = kobject_init_and_add(&priv->kobj, &driver_ktype, NULL,
"%s", drv->name);
if (error)
goto out_unregister;
klist_add_tail(&priv->knode_bus, &sp->klist_drivers);
if (sp->drivers_autoprobe) {
error = driver_attach(drv);
if (error)
goto out_del_list;
}
module_add_driver(drv->owner, drv);
error = driver_create_file(drv, &driver_attr_uevent);
if (error) {
printk(KERN_ERR "%s: uevent attr (%s) failed\n",
__func__, drv->name);
}
error = driver_add_groups(drv, sp->bus->drv_groups);
if (error) {
/* How the hell do we get out of this pickle? Give up */
printk(KERN_ERR "%s: driver_add_groups(%s) failed\n",
__func__, drv->name);
}
if (!drv->suppress_bind_attrs) { error = add_bind_files(drv);
if (error) {
/* Ditto */
printk(KERN_ERR "%s: add_bind_files(%s) failed\n",
__func__, drv->name);
}
}
return 0;
out_del_list:
klist_del(&priv->knode_bus);
out_unregister:
kobject_put(&priv->kobj);
/* drv->p is freed in driver_release() */
drv->p = NULL;
out_put_bus:
subsys_put(sp); return error;
}
/**
* bus_remove_driver - delete driver from bus's knowledge.
* @drv: driver.
*
* Detach the driver from the devices it controls, and remove
* it from its bus's list of drivers. Finally, we drop the reference
* to the bus we took in bus_add_driver().
*/
void bus_remove_driver(struct device_driver *drv)
{
struct subsys_private *sp = bus_to_subsys(drv->bus);
if (!sp)
return;
pr_debug("bus: '%s': remove driver %s\n", sp->bus->name, drv->name); if (!drv->suppress_bind_attrs) remove_bind_files(drv); driver_remove_groups(drv, sp->bus->drv_groups);
driver_remove_file(drv, &driver_attr_uevent);
klist_remove(&drv->p->knode_bus);
driver_detach(drv);
module_remove_driver(drv);
kobject_put(&drv->p->kobj);
/*
* Decrement the reference count twice, once for the bus_to_subsys()
* call in the start of this function, and the second one from the
* reference increment in bus_add_driver()
*/
subsys_put(sp);
subsys_put(sp);
}
/* Helper for bus_rescan_devices's iter */
static int __must_check bus_rescan_devices_helper(struct device *dev,
void *data)
{
int ret = 0;
if (!dev->driver) {
if (dev->parent && dev->bus->need_parent_lock)
device_lock(dev->parent);
ret = device_attach(dev);
if (dev->parent && dev->bus->need_parent_lock)
device_unlock(dev->parent);
}
return ret < 0 ? ret : 0;
}
/**
* bus_rescan_devices - rescan devices on the bus for possible drivers
* @bus: the bus to scan.
*
* This function will look for devices on the bus with no driver
* attached and rescan it against existing drivers to see if it matches
* any by calling device_attach() for the unbound devices.
*/
int bus_rescan_devices(const struct bus_type *bus)
{
return bus_for_each_dev(bus, NULL, NULL, bus_rescan_devices_helper);
}
EXPORT_SYMBOL_GPL(bus_rescan_devices);
/**
* device_reprobe - remove driver for a device and probe for a new driver
* @dev: the device to reprobe
*
* This function detaches the attached driver (if any) for the given
* device and restarts the driver probing process. It is intended
* to use if probing criteria changed during a devices lifetime and
* driver attachment should change accordingly.
*/
int device_reprobe(struct device *dev)
{
if (dev->driver)
device_driver_detach(dev);
return bus_rescan_devices_helper(dev, NULL);
}
EXPORT_SYMBOL_GPL(device_reprobe);
static void klist_devices_get(struct klist_node *n)
{
struct device_private *dev_prv = to_device_private_bus(n);
struct device *dev = dev_prv->device;
get_device(dev);
}
static void klist_devices_put(struct klist_node *n)
{
struct device_private *dev_prv = to_device_private_bus(n);
struct device *dev = dev_prv->device;
put_device(dev);
}
static ssize_t bus_uevent_store(const struct bus_type *bus,
const char *buf, size_t count)
{
struct subsys_private *sp = bus_to_subsys(bus);
int ret;
if (!sp)
return -EINVAL;
ret = kobject_synth_uevent(&sp->subsys.kobj, buf, count);
subsys_put(sp);
if (ret)
return ret;
return count;
}
/*
* "open code" the old BUS_ATTR() macro here. We want to use BUS_ATTR_WO()
* here, but can not use it as earlier in the file we have
* DEVICE_ATTR_WO(uevent), which would cause a clash with the with the store
* function name.
*/
static struct bus_attribute bus_attr_uevent = __ATTR(uevent, 0200, NULL,
bus_uevent_store);
/**
* bus_register - register a driver-core subsystem
* @bus: bus to register
*
* Once we have that, we register the bus with the kobject
* infrastructure, then register the children subsystems it has:
* the devices and drivers that belong to the subsystem.
*/
int bus_register(const struct bus_type *bus)
{
int retval;
struct subsys_private *priv;
struct kobject *bus_kobj;
struct lock_class_key *key;
priv = kzalloc(sizeof(struct subsys_private), GFP_KERNEL);
if (!priv)
return -ENOMEM;
priv->bus = bus;
BLOCKING_INIT_NOTIFIER_HEAD(&priv->bus_notifier);
bus_kobj = &priv->subsys.kobj;
retval = kobject_set_name(bus_kobj, "%s", bus->name);
if (retval)
goto out;
bus_kobj->kset = bus_kset;
bus_kobj->ktype = &bus_ktype;
priv->drivers_autoprobe = 1;
retval = kset_register(&priv->subsys);
if (retval)
goto out;
retval = bus_create_file(bus, &bus_attr_uevent);
if (retval)
goto bus_uevent_fail;
priv->devices_kset = kset_create_and_add("devices", NULL, bus_kobj);
if (!priv->devices_kset) {
retval = -ENOMEM;
goto bus_devices_fail;
}
priv->drivers_kset = kset_create_and_add("drivers", NULL, bus_kobj);
if (!priv->drivers_kset) {
retval = -ENOMEM;
goto bus_drivers_fail;
}
INIT_LIST_HEAD(&priv->interfaces);
key = &priv->lock_key;
lockdep_register_key(key);
__mutex_init(&priv->mutex, "subsys mutex", key);
klist_init(&priv->klist_devices, klist_devices_get, klist_devices_put);
klist_init(&priv->klist_drivers, NULL, NULL);
retval = add_probe_files(bus);
if (retval)
goto bus_probe_files_fail;
retval = sysfs_create_groups(bus_kobj, bus->bus_groups);
if (retval)
goto bus_groups_fail;
pr_debug("bus: '%s': registered\n", bus->name);
return 0;
bus_groups_fail:
remove_probe_files(bus);
bus_probe_files_fail:
kset_unregister(priv->drivers_kset);
bus_drivers_fail:
kset_unregister(priv->devices_kset);
bus_devices_fail:
bus_remove_file(bus, &bus_attr_uevent);
bus_uevent_fail:
kset_unregister(&priv->subsys);
out:
kfree(priv);
return retval;
}
EXPORT_SYMBOL_GPL(bus_register);
/**
* bus_unregister - remove a bus from the system
* @bus: bus.
*
* Unregister the child subsystems and the bus itself.
* Finally, we call bus_put() to release the refcount
*/
void bus_unregister(const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
struct kobject *bus_kobj;
if (!sp)
return;
pr_debug("bus: '%s': unregistering\n", bus->name);
if (sp->dev_root)
device_unregister(sp->dev_root);
bus_kobj = &sp->subsys.kobj;
sysfs_remove_groups(bus_kobj, bus->bus_groups);
remove_probe_files(bus);
bus_remove_file(bus, &bus_attr_uevent);
kset_unregister(sp->drivers_kset);
kset_unregister(sp->devices_kset);
kset_unregister(&sp->subsys);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(bus_unregister);
int bus_register_notifier(const struct bus_type *bus, struct notifier_block *nb)
{
struct subsys_private *sp = bus_to_subsys(bus);
int retval;
if (!sp)
return -EINVAL;
retval = blocking_notifier_chain_register(&sp->bus_notifier, nb);
subsys_put(sp);
return retval;
}
EXPORT_SYMBOL_GPL(bus_register_notifier);
int bus_unregister_notifier(const struct bus_type *bus, struct notifier_block *nb)
{
struct subsys_private *sp = bus_to_subsys(bus);
int retval;
if (!sp)
return -EINVAL;
retval = blocking_notifier_chain_unregister(&sp->bus_notifier, nb);
subsys_put(sp);
return retval;
}
EXPORT_SYMBOL_GPL(bus_unregister_notifier);
void bus_notify(struct device *dev, enum bus_notifier_event value)
{
struct subsys_private *sp = bus_to_subsys(dev->bus);
if (!sp)
return;
blocking_notifier_call_chain(&sp->bus_notifier, value, dev);
subsys_put(sp);
}
struct kset *bus_get_kset(const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
struct kset *kset;
if (!sp)
return NULL;
kset = &sp->subsys;
subsys_put(sp);
return kset;
}
EXPORT_SYMBOL_GPL(bus_get_kset);
/*
* Yes, this forcibly breaks the klist abstraction temporarily. It
* just wants to sort the klist, not change reference counts and
* take/drop locks rapidly in the process. It does all this while
* holding the lock for the list, so objects can't otherwise be
* added/removed while we're swizzling.
*/
static void device_insertion_sort_klist(struct device *a, struct list_head *list,
int (*compare)(const struct device *a,
const struct device *b))
{
struct klist_node *n;
struct device_private *dev_prv;
struct device *b;
list_for_each_entry(n, list, n_node) {
dev_prv = to_device_private_bus(n);
b = dev_prv->device;
if (compare(a, b) <= 0) {
list_move_tail(&a->p->knode_bus.n_node,
&b->p->knode_bus.n_node);
return;
}
}
list_move_tail(&a->p->knode_bus.n_node, list);
}
void bus_sort_breadthfirst(const struct bus_type *bus,
int (*compare)(const struct device *a,
const struct device *b))
{
struct subsys_private *sp = bus_to_subsys(bus);
LIST_HEAD(sorted_devices);
struct klist_node *n, *tmp;
struct device_private *dev_prv;
struct device *dev;
struct klist *device_klist;
if (!sp)
return;
device_klist = &sp->klist_devices;
spin_lock(&device_klist->k_lock);
list_for_each_entry_safe(n, tmp, &device_klist->k_list, n_node) {
dev_prv = to_device_private_bus(n);
dev = dev_prv->device;
device_insertion_sort_klist(dev, &sorted_devices, compare);
}
list_splice(&sorted_devices, &device_klist->k_list);
spin_unlock(&device_klist->k_lock);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(bus_sort_breadthfirst);
struct subsys_dev_iter {
struct klist_iter ki;
const struct device_type *type;
};
/**
* subsys_dev_iter_init - initialize subsys device iterator
* @iter: subsys iterator to initialize
* @sp: the subsys private (i.e. bus) we wanna iterate over
* @start: the device to start iterating from, if any
* @type: device_type of the devices to iterate over, NULL for all
*
* Initialize subsys iterator @iter such that it iterates over devices
* of @subsys. If @start is set, the list iteration will start there,
* otherwise if it is NULL, the iteration starts at the beginning of
* the list.
*/
static void subsys_dev_iter_init(struct subsys_dev_iter *iter, struct subsys_private *sp,
struct device *start, const struct device_type *type)
{
struct klist_node *start_knode = NULL;
if (start)
start_knode = &start->p->knode_bus;
klist_iter_init_node(&sp->klist_devices, &iter->ki, start_knode);
iter->type = type;
}
/**
* subsys_dev_iter_next - iterate to the next device
* @iter: subsys iterator to proceed
*
* Proceed @iter to the next device and return it. Returns NULL if
* iteration is complete.
*
* The returned device is referenced and won't be released till
* iterator is proceed to the next device or exited. The caller is
* free to do whatever it wants to do with the device including
* calling back into subsys code.
*/
static struct device *subsys_dev_iter_next(struct subsys_dev_iter *iter)
{
struct klist_node *knode;
struct device *dev;
for (;;) {
knode = klist_next(&iter->ki);
if (!knode)
return NULL;
dev = to_device_private_bus(knode)->device;
if (!iter->type || iter->type == dev->type)
return dev;
}
}
/**
* subsys_dev_iter_exit - finish iteration
* @iter: subsys iterator to finish
*
* Finish an iteration. Always call this function after iteration is
* complete whether the iteration ran till the end or not.
*/
static void subsys_dev_iter_exit(struct subsys_dev_iter *iter)
{
klist_iter_exit(&iter->ki);
}
int subsys_interface_register(struct subsys_interface *sif)
{
struct subsys_private *sp;
struct subsys_dev_iter iter;
struct device *dev;
if (!sif || !sif->subsys)
return -ENODEV;
sp = bus_to_subsys(sif->subsys);
if (!sp)
return -EINVAL;
/*
* Reference in sp is now incremented and will be dropped when
* the interface is removed from the bus
*/
mutex_lock(&sp->mutex);
list_add_tail(&sif->node, &sp->interfaces);
if (sif->add_dev) {
subsys_dev_iter_init(&iter, sp, NULL, NULL);
while ((dev = subsys_dev_iter_next(&iter)))
sif->add_dev(dev, sif);
subsys_dev_iter_exit(&iter);
}
mutex_unlock(&sp->mutex);
return 0;
}
EXPORT_SYMBOL_GPL(subsys_interface_register);
void subsys_interface_unregister(struct subsys_interface *sif)
{
struct subsys_private *sp;
struct subsys_dev_iter iter;
struct device *dev;
if (!sif || !sif->subsys)
return;
sp = bus_to_subsys(sif->subsys);
if (!sp)
return;
mutex_lock(&sp->mutex);
list_del_init(&sif->node);
if (sif->remove_dev) {
subsys_dev_iter_init(&iter, sp, NULL, NULL);
while ((dev = subsys_dev_iter_next(&iter)))
sif->remove_dev(dev, sif);
subsys_dev_iter_exit(&iter);
}
mutex_unlock(&sp->mutex);
/*
* Decrement the reference count twice, once for the bus_to_subsys()
* call in the start of this function, and the second one from the
* reference increment in subsys_interface_register()
*/
subsys_put(sp);
subsys_put(sp);
}
EXPORT_SYMBOL_GPL(subsys_interface_unregister);
static void system_root_device_release(struct device *dev)
{
kfree(dev);
}
static int subsys_register(const struct bus_type *subsys,
const struct attribute_group **groups,
struct kobject *parent_of_root)
{
struct subsys_private *sp;
struct device *dev;
int err;
err = bus_register(subsys);
if (err < 0)
return err;
sp = bus_to_subsys(subsys);
if (!sp) {
err = -EINVAL;
goto err_sp;
}
dev = kzalloc(sizeof(struct device), GFP_KERNEL);
if (!dev) {
err = -ENOMEM;
goto err_dev;
}
err = dev_set_name(dev, "%s", subsys->name);
if (err < 0)
goto err_name;
dev->kobj.parent = parent_of_root;
dev->groups = groups;
dev->release = system_root_device_release;
err = device_register(dev);
if (err < 0)
goto err_dev_reg;
sp->dev_root = dev;
subsys_put(sp);
return 0;
err_dev_reg:
put_device(dev);
dev = NULL;
err_name:
kfree(dev);
err_dev:
subsys_put(sp);
err_sp:
bus_unregister(subsys);
return err;
}
/**
* subsys_system_register - register a subsystem at /sys/devices/system/
* @subsys: system subsystem
* @groups: default attributes for the root device
*
* All 'system' subsystems have a /sys/devices/system/<name> root device
* with the name of the subsystem. The root device can carry subsystem-
* wide attributes. All registered devices are below this single root
* device and are named after the subsystem with a simple enumeration
* number appended. The registered devices are not explicitly named;
* only 'id' in the device needs to be set.
*
* Do not use this interface for anything new, it exists for compatibility
* with bad ideas only. New subsystems should use plain subsystems; and
* add the subsystem-wide attributes should be added to the subsystem
* directory itself and not some create fake root-device placed in
* /sys/devices/system/<name>.
*/
int subsys_system_register(const struct bus_type *subsys,
const struct attribute_group **groups)
{
return subsys_register(subsys, groups, &system_kset->kobj);
}
EXPORT_SYMBOL_GPL(subsys_system_register);
/**
* subsys_virtual_register - register a subsystem at /sys/devices/virtual/
* @subsys: virtual subsystem
* @groups: default attributes for the root device
*
* All 'virtual' subsystems have a /sys/devices/system/<name> root device
* with the name of the subystem. The root device can carry subsystem-wide
* attributes. All registered devices are below this single root device.
* There's no restriction on device naming. This is for kernel software
* constructs which need sysfs interface.
*/
int subsys_virtual_register(const struct bus_type *subsys,
const struct attribute_group **groups)
{
struct kobject *virtual_dir;
virtual_dir = virtual_device_parent(NULL);
if (!virtual_dir)
return -ENOMEM;
return subsys_register(subsys, groups, virtual_dir);
}
EXPORT_SYMBOL_GPL(subsys_virtual_register);
/**
* driver_find - locate driver on a bus by its name.
* @name: name of the driver.
* @bus: bus to scan for the driver.
*
* Call kset_find_obj() to iterate over list of drivers on
* a bus to find driver by name. Return driver if found.
*
* This routine provides no locking to prevent the driver it returns
* from being unregistered or unloaded while the caller is using it.
* The caller is responsible for preventing this.
*/
struct device_driver *driver_find(const char *name, const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
struct kobject *k;
struct driver_private *priv;
if (!sp)
return NULL;
k = kset_find_obj(sp->drivers_kset, name);
subsys_put(sp);
if (!k)
return NULL;
priv = to_driver(k);
/* Drop reference added by kset_find_obj() */
kobject_put(k); return priv->driver;}
EXPORT_SYMBOL_GPL(driver_find);
/*
* Warning, the value could go to "removed" instantly after calling this function, so be very
* careful when calling it...
*/
bool bus_is_registered(const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
bool is_initialized = false;
if (sp) {
is_initialized = true;
subsys_put(sp);
}
return is_initialized;
}
/**
* bus_get_dev_root - return a pointer to the "device root" of a bus
* @bus: bus to return the device root of.
*
* If a bus has a "device root" structure, return it, WITH THE REFERENCE
* COUNT INCREMENTED.
*
* Note, when finished with the device, a call to put_device() is required.
*
* If the device root is not present (or bus is not a valid pointer), NULL
* will be returned.
*/
struct device *bus_get_dev_root(const struct bus_type *bus)
{
struct subsys_private *sp = bus_to_subsys(bus);
struct device *dev_root;
if (!sp)
return NULL;
dev_root = get_device(sp->dev_root);
subsys_put(sp);
return dev_root;
}
EXPORT_SYMBOL_GPL(bus_get_dev_root);
int __init buses_init(void)
{
bus_kset = kset_create_and_add("bus", &bus_uevent_ops, NULL);
if (!bus_kset)
return -ENOMEM;
system_kset = kset_create_and_add("system", NULL, &devices_kset->kobj);
if (!system_kset)
return -ENOMEM;
return 0;
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/lib/vsprintf.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
/* vsprintf.c -- Lars Wirzenius & Linus Torvalds. */
/*
* Wirzenius wrote this portably, Torvalds fucked it up :-)
*/
/*
* Fri Jul 13 2001 Crutcher Dunnavant <crutcher+kernel@datastacks.com>
* - changed to provide snprintf and vsnprintf functions
* So Feb 1 16:51:32 CET 2004 Juergen Quade <quade@hsnr.de>
* - scnprintf and vscnprintf
*/
#include <linux/stdarg.h>
#include <linux/build_bug.h>
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/errname.h>
#include <linux/module.h> /* for KSYM_SYMBOL_LEN */
#include <linux/types.h>
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/kernel.h>
#include <linux/kallsyms.h>
#include <linux/math64.h>
#include <linux/uaccess.h>
#include <linux/ioport.h>
#include <linux/dcache.h>
#include <linux/cred.h>
#include <linux/rtc.h>
#include <linux/sprintf.h>
#include <linux/time.h>
#include <linux/uuid.h>
#include <linux/of.h>
#include <net/addrconf.h>
#include <linux/siphash.h>
#include <linux/compiler.h>
#include <linux/property.h>
#include <linux/notifier.h>
#ifdef CONFIG_BLOCK
#include <linux/blkdev.h>
#endif
#include "../mm/internal.h" /* For the trace_print_flags arrays */
#include <asm/page.h> /* for PAGE_SIZE */
#include <asm/byteorder.h> /* cpu_to_le16 */
#include <asm/unaligned.h>
#include <linux/string_helpers.h>
#include "kstrtox.h"
/* Disable pointer hashing if requested */
bool no_hash_pointers __ro_after_init;
EXPORT_SYMBOL_GPL(no_hash_pointers);
noinline
static unsigned long long simple_strntoull(const char *startp, char **endp, unsigned int base, size_t max_chars)
{
const char *cp;
unsigned long long result = 0ULL;
size_t prefix_chars;
unsigned int rv;
cp = _parse_integer_fixup_radix(startp, &base);
prefix_chars = cp - startp;
if (prefix_chars < max_chars) {
rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars);
/* FIXME */
cp += (rv & ~KSTRTOX_OVERFLOW);
} else {
/* Field too short for prefix + digit, skip over without converting */
cp = startp + max_chars;
}
if (endp) *endp = (char *)cp; return result;
}
/**
* simple_strtoull - convert a string to an unsigned long long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function has caveats. Please use kstrtoull instead.
*/
noinline
unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base)
{
return simple_strntoull(cp, endp, base, INT_MAX);
}
EXPORT_SYMBOL(simple_strtoull);
/**
* simple_strtoul - convert a string to an unsigned long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function has caveats. Please use kstrtoul instead.
*/
unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base)
{
return simple_strtoull(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtoul);
/**
* simple_strtol - convert a string to a signed long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function has caveats. Please use kstrtol instead.
*/
long simple_strtol(const char *cp, char **endp, unsigned int base)
{
if (*cp == '-')
return -simple_strtoul(cp + 1, endp, base);
return simple_strtoul(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtol);
noinline
static long long simple_strntoll(const char *cp, char **endp, unsigned int base, size_t max_chars)
{
/*
* simple_strntoull() safely handles receiving max_chars==0 in the
* case cp[0] == '-' && max_chars == 1.
* If max_chars == 0 we can drop through and pass it to simple_strntoull()
* and the content of *cp is irrelevant.
*/
if (*cp == '-' && max_chars > 0) return -simple_strntoull(cp + 1, endp, base, max_chars - 1); return simple_strntoull(cp, endp, base, max_chars);
}
/**
* simple_strtoll - convert a string to a signed long long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function has caveats. Please use kstrtoll instead.
*/
long long simple_strtoll(const char *cp, char **endp, unsigned int base)
{
return simple_strntoll(cp, endp, base, INT_MAX);
}
EXPORT_SYMBOL(simple_strtoll);
static noinline_for_stack
int skip_atoi(const char **s)
{
int i = 0;
do {
i = i*10 + *((*s)++) - '0';
} while (isdigit(**s));
return i;
}
/*
* Decimal conversion is by far the most typical, and is used for
* /proc and /sys data. This directly impacts e.g. top performance
* with many processes running. We optimize it for speed by emitting
* two characters at a time, using a 200 byte lookup table. This
* roughly halves the number of multiplications compared to computing
* the digits one at a time. Implementation strongly inspired by the
* previous version, which in turn used ideas described at
* <http://www.cs.uiowa.edu/~jones/bcd/divide.html> (with permission
* from the author, Douglas W. Jones).
*
* It turns out there is precisely one 26 bit fixed-point
* approximation a of 64/100 for which x/100 == (x * (u64)a) >> 32
* holds for all x in [0, 10^8-1], namely a = 0x28f5c29. The actual
* range happens to be somewhat larger (x <= 1073741898), but that's
* irrelevant for our purpose.
*
* For dividing a number in the range [10^4, 10^6-1] by 100, we still
* need a 32x32->64 bit multiply, so we simply use the same constant.
*
* For dividing a number in the range [100, 10^4-1] by 100, there are
* several options. The simplest is (x * 0x147b) >> 19, which is valid
* for all x <= 43698.
*/
static const u16 decpair[100] = {
#define _(x) (__force u16) cpu_to_le16(((x % 10) | ((x / 10) << 8)) + 0x3030)
_( 0), _( 1), _( 2), _( 3), _( 4), _( 5), _( 6), _( 7), _( 8), _( 9),
_(10), _(11), _(12), _(13), _(14), _(15), _(16), _(17), _(18), _(19),
_(20), _(21), _(22), _(23), _(24), _(25), _(26), _(27), _(28), _(29),
_(30), _(31), _(32), _(33), _(34), _(35), _(36), _(37), _(38), _(39),
_(40), _(41), _(42), _(43), _(44), _(45), _(46), _(47), _(48), _(49),
_(50), _(51), _(52), _(53), _(54), _(55), _(56), _(57), _(58), _(59),
_(60), _(61), _(62), _(63), _(64), _(65), _(66), _(67), _(68), _(69),
_(70), _(71), _(72), _(73), _(74), _(75), _(76), _(77), _(78), _(79),
_(80), _(81), _(82), _(83), _(84), _(85), _(86), _(87), _(88), _(89),
_(90), _(91), _(92), _(93), _(94), _(95), _(96), _(97), _(98), _(99),
#undef _
};
/*
* This will print a single '0' even if r == 0, since we would
* immediately jump to out_r where two 0s would be written but only
* one of them accounted for in buf. This is needed by ip4_string
* below. All other callers pass a non-zero value of r.
*/
static noinline_for_stack
char *put_dec_trunc8(char *buf, unsigned r)
{
unsigned q;
/* 1 <= r < 10^8 */
if (r < 100)
goto out_r;
/* 100 <= r < 10^8 */
q = (r * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
/* 1 <= q < 10^6 */
if (q < 100)
goto out_q;
/* 100 <= q < 10^6 */
r = (q * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[q - 100*r];
buf += 2;
/* 1 <= r < 10^4 */
if (r < 100)
goto out_r;
/* 100 <= r < 10^4 */
q = (r * 0x147b) >> 19;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
out_q:
/* 1 <= q < 100 */
r = q;
out_r:
/* 1 <= r < 100 */
*((u16 *)buf) = decpair[r]; buf += r < 10 ? 1 : 2;
return buf;
}
#if BITS_PER_LONG == 64 && BITS_PER_LONG_LONG == 64
static noinline_for_stack
char *put_dec_full8(char *buf, unsigned r)
{
unsigned q;
/* 0 <= r < 10^8 */
q = (r * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
/* 0 <= q < 10^6 */
r = (q * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[q - 100*r];
buf += 2;
/* 0 <= r < 10^4 */
q = (r * 0x147b) >> 19;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
/* 0 <= q < 100 */
*((u16 *)buf) = decpair[q];
buf += 2;
return buf;
}
static noinline_for_stack
char *put_dec(char *buf, unsigned long long n)
{
if (n >= 100*1000*1000) buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
/* 1 <= n <= 1.6e11 */
if (n >= 100*1000*1000)
buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
/* 1 <= n < 1e8 */
return put_dec_trunc8(buf, n);
}
#elif BITS_PER_LONG == 32 && BITS_PER_LONG_LONG == 64
static void
put_dec_full4(char *buf, unsigned r)
{
unsigned q;
/* 0 <= r < 10^4 */
q = (r * 0x147b) >> 19;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
/* 0 <= q < 100 */
*((u16 *)buf) = decpair[q];
}
/*
* Call put_dec_full4 on x % 10000, return x / 10000.
* The approximation x/10000 == (x * 0x346DC5D7) >> 43
* holds for all x < 1,128,869,999. The largest value this
* helper will ever be asked to convert is 1,125,520,955.
* (second call in the put_dec code, assuming n is all-ones).
*/
static noinline_for_stack
unsigned put_dec_helper4(char *buf, unsigned x)
{
uint32_t q = (x * (uint64_t)0x346DC5D7) >> 43;
put_dec_full4(buf, x - q * 10000);
return q;
}
/* Based on code by Douglas W. Jones found at
* <http://www.cs.uiowa.edu/~jones/bcd/decimal.html#sixtyfour>
* (with permission from the author).
* Performs no 64-bit division and hence should be fast on 32-bit machines.
*/
static
char *put_dec(char *buf, unsigned long long n)
{
uint32_t d3, d2, d1, q, h;
if (n < 100*1000*1000)
return put_dec_trunc8(buf, n);
d1 = ((uint32_t)n >> 16); /* implicit "& 0xffff" */
h = (n >> 32);
d2 = (h ) & 0xffff;
d3 = (h >> 16); /* implicit "& 0xffff" */
/* n = 2^48 d3 + 2^32 d2 + 2^16 d1 + d0
= 281_4749_7671_0656 d3 + 42_9496_7296 d2 + 6_5536 d1 + d0 */
q = 656 * d3 + 7296 * d2 + 5536 * d1 + ((uint32_t)n & 0xffff);
q = put_dec_helper4(buf, q);
q += 7671 * d3 + 9496 * d2 + 6 * d1;
q = put_dec_helper4(buf+4, q);
q += 4749 * d3 + 42 * d2;
q = put_dec_helper4(buf+8, q);
q += 281 * d3;
buf += 12;
if (q)
buf = put_dec_trunc8(buf, q);
else while (buf[-1] == '0')
--buf;
return buf;
}
#endif
/*
* Convert passed number to decimal string.
* Returns the length of string. On buffer overflow, returns 0.
*
* If speed is not important, use snprintf(). It's easy to read the code.
*/
int num_to_str(char *buf, int size, unsigned long long num, unsigned int width)
{
/* put_dec requires 2-byte alignment of the buffer. */
char tmp[sizeof(num) * 3] __aligned(2);
int idx, len;
/* put_dec() may work incorrectly for num = 0 (generate "", not "0") */
if (num <= 9) {
tmp[0] = '0' + num;
len = 1;
} else {
len = put_dec(tmp, num) - tmp;
}
if (len > size || width > size)
return 0;
if (width > len) {
width = width - len;
for (idx = 0; idx < width; idx++)
buf[idx] = ' ';
} else {
width = 0;
}
for (idx = 0; idx < len; ++idx)
buf[idx + width] = tmp[len - idx - 1];
return len + width;
}
#define SIGN 1 /* unsigned/signed, must be 1 */
#define LEFT 2 /* left justified */
#define PLUS 4 /* show plus */
#define SPACE 8 /* space if plus */
#define ZEROPAD 16 /* pad with zero, must be 16 == '0' - ' ' */
#define SMALL 32 /* use lowercase in hex (must be 32 == 0x20) */
#define SPECIAL 64 /* prefix hex with "0x", octal with "0" */
static_assert(SIGN == 1);
static_assert(ZEROPAD == ('0' - ' '));
static_assert(SMALL == ('a' ^ 'A'));
enum format_type {
FORMAT_TYPE_NONE, /* Just a string part */
FORMAT_TYPE_WIDTH,
FORMAT_TYPE_PRECISION,
FORMAT_TYPE_CHAR,
FORMAT_TYPE_STR,
FORMAT_TYPE_PTR,
FORMAT_TYPE_PERCENT_CHAR,
FORMAT_TYPE_INVALID,
FORMAT_TYPE_LONG_LONG,
FORMAT_TYPE_ULONG,
FORMAT_TYPE_LONG,
FORMAT_TYPE_UBYTE,
FORMAT_TYPE_BYTE,
FORMAT_TYPE_USHORT,
FORMAT_TYPE_SHORT,
FORMAT_TYPE_UINT,
FORMAT_TYPE_INT,
FORMAT_TYPE_SIZE_T,
FORMAT_TYPE_PTRDIFF
};
struct printf_spec {
unsigned int type:8; /* format_type enum */
signed int field_width:24; /* width of output field */
unsigned int flags:8; /* flags to number() */
unsigned int base:8; /* number base, 8, 10 or 16 only */
signed int precision:16; /* # of digits/chars */
} __packed;
static_assert(sizeof(struct printf_spec) == 8);
#define FIELD_WIDTH_MAX ((1 << 23) - 1)
#define PRECISION_MAX ((1 << 15) - 1)
static noinline_for_stack
char *number(char *buf, char *end, unsigned long long num,
struct printf_spec spec)
{
/* put_dec requires 2-byte alignment of the buffer. */
char tmp[3 * sizeof(num)] __aligned(2);
char sign;
char locase;
int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10);
int i;
bool is_zero = num == 0LL;
int field_width = spec.field_width;
int precision = spec.precision;
/* locase = 0 or 0x20. ORing digits or letters with 'locase'
* produces same digits or (maybe lowercased) letters */
locase = (spec.flags & SMALL);
if (spec.flags & LEFT) spec.flags &= ~ZEROPAD;
sign = 0;
if (spec.flags & SIGN) {
if ((signed long long)num < 0) {
sign = '-';
num = -(signed long long)num;
field_width--;
} else if (spec.flags & PLUS) {
sign = '+';
field_width--; } else if (spec.flags & SPACE) {
sign = ' ';
field_width--;
}
}
if (need_pfx) { if (spec.base == 16) field_width -= 2; else if (!is_zero) field_width--;
}
/* generate full string in tmp[], in reverse order */
i = 0;
if (num < spec.base) tmp[i++] = hex_asc_upper[num] | locase; else if (spec.base != 10) { /* 8 or 16 */ int mask = spec.base - 1;
int shift = 3;
if (spec.base == 16)
shift = 4;
do {
tmp[i++] = (hex_asc_upper[((unsigned char)num) & mask] | locase);
num >>= shift;
} while (num);
} else { /* base 10 */
i = put_dec(tmp, num) - tmp;
}
/* printing 100 using %2d gives "100", not "00" */
if (i > precision)
precision = i;
/* leading space padding */
field_width -= precision;
if (!(spec.flags & (ZEROPAD | LEFT))) {
while (--field_width >= 0) { if (buf < end) *buf = ' '; ++buf;
}
}
/* sign */
if (sign) { if (buf < end) *buf = sign; ++buf;
}
/* "0x" / "0" prefix */
if (need_pfx) { if (spec.base == 16 || !is_zero) {
if (buf < end)
*buf = '0'; ++buf;
}
if (spec.base == 16) {
if (buf < end) *buf = ('X' | locase); ++buf;
}
}
/* zero or space padding */
if (!(spec.flags & LEFT)) {
char c = ' ' + (spec.flags & ZEROPAD);
while (--field_width >= 0) { if (buf < end) *buf = c; ++buf;
}
}
/* hmm even more zero padding? */
while (i <= --precision) { if (buf < end) *buf = '0'; ++buf;
}
/* actual digits of result */
while (--i >= 0) { if (buf < end) *buf = tmp[i]; ++buf;
}
/* trailing space padding */
while (--field_width >= 0) { if (buf < end) *buf = ' '; ++buf;
}
return buf;
}
static noinline_for_stack
char *special_hex_number(char *buf, char *end, unsigned long long num, int size)
{
struct printf_spec spec;
spec.type = FORMAT_TYPE_PTR;
spec.field_width = 2 + 2 * size; /* 0x + hex */
spec.flags = SPECIAL | SMALL | ZEROPAD;
spec.base = 16;
spec.precision = -1;
return number(buf, end, num, spec);
}
static void move_right(char *buf, char *end, unsigned len, unsigned spaces)
{
size_t size;
if (buf >= end) /* nowhere to put anything */
return;
size = end - buf;
if (size <= spaces) {
memset(buf, ' ', size);
return;
}
if (len) {
if (len > size - spaces) len = size - spaces; memmove(buf + spaces, buf, len);
}
memset(buf, ' ', spaces);
}
/*
* Handle field width padding for a string.
* @buf: current buffer position
* @n: length of string
* @end: end of output buffer
* @spec: for field width and flags
* Returns: new buffer position after padding.
*/
static noinline_for_stack
char *widen_string(char *buf, int n, char *end, struct printf_spec spec)
{
unsigned spaces;
if (likely(n >= spec.field_width))
return buf;
/* we want to pad the sucker */
spaces = spec.field_width - n;
if (!(spec.flags & LEFT)) {
move_right(buf - n, end, n, spaces); return buf + spaces;
}
while (spaces--) { if (buf < end) *buf = ' '; ++buf;
}
return buf;
}
/* Handle string from a well known address. */
static char *string_nocheck(char *buf, char *end, const char *s,
struct printf_spec spec)
{
int len = 0;
int lim = spec.precision;
while (lim--) {
char c = *s++;
if (!c)
break;
if (buf < end) *buf = c; ++buf;
++len;
}
return widen_string(buf, len, end, spec);
}
static char *err_ptr(char *buf, char *end, void *ptr,
struct printf_spec spec)
{
int err = PTR_ERR(ptr);
const char *sym = errname(err);
if (sym)
return string_nocheck(buf, end, sym, spec);
/*
* Somebody passed ERR_PTR(-1234) or some other non-existing
* Efoo - or perhaps CONFIG_SYMBOLIC_ERRNAME=n. Fall back to
* printing it as its decimal representation.
*/
spec.flags |= SIGN;
spec.base = 10;
return number(buf, end, err, spec);
}
/* Be careful: error messages must fit into the given buffer. */
static char *error_string(char *buf, char *end, const char *s,
struct printf_spec spec)
{
/*
* Hard limit to avoid a completely insane messages. It actually
* works pretty well because most error messages are in
* the many pointer format modifiers.
*/
if (spec.precision == -1)
spec.precision = 2 * sizeof(void *);
return string_nocheck(buf, end, s, spec);
}
/*
* Do not call any complex external code here. Nested printk()/vsprintf()
* might cause infinite loops. Failures might break printk() and would
* be hard to debug.
*/
static const char *check_pointer_msg(const void *ptr)
{
if (!ptr)
return "(null)";
if ((unsigned long)ptr < PAGE_SIZE || IS_ERR_VALUE(ptr))
return "(efault)";
return NULL;
}
static int check_pointer(char **buf, char *end, const void *ptr,
struct printf_spec spec)
{
const char *err_msg;
err_msg = check_pointer_msg(ptr);
if (err_msg) {
*buf = error_string(*buf, end, err_msg, spec);
return -EFAULT;
}
return 0;
}
static noinline_for_stack
char *string(char *buf, char *end, const char *s,
struct printf_spec spec)
{
if (check_pointer(&buf, end, s, spec))
return buf;
return string_nocheck(buf, end, s, spec);
}
static char *pointer_string(char *buf, char *end,
const void *ptr,
struct printf_spec spec)
{
spec.base = 16;
spec.flags |= SMALL;
if (spec.field_width == -1) {
spec.field_width = 2 * sizeof(ptr);
spec.flags |= ZEROPAD;
}
return number(buf, end, (unsigned long int)ptr, spec);
}
/* Make pointers available for printing early in the boot sequence. */
static int debug_boot_weak_hash __ro_after_init;
static int __init debug_boot_weak_hash_enable(char *str)
{
debug_boot_weak_hash = 1;
pr_info("debug_boot_weak_hash enabled\n");
return 0;
}
early_param("debug_boot_weak_hash", debug_boot_weak_hash_enable);
static bool filled_random_ptr_key __read_mostly;
static siphash_key_t ptr_key __read_mostly;
static int fill_ptr_key(struct notifier_block *nb, unsigned long action, void *data)
{
get_random_bytes(&ptr_key, sizeof(ptr_key));
/* Pairs with smp_rmb() before reading ptr_key. */
smp_wmb();
WRITE_ONCE(filled_random_ptr_key, true);
return NOTIFY_DONE;
}
static int __init vsprintf_init_hashval(void)
{
static struct notifier_block fill_ptr_key_nb = { .notifier_call = fill_ptr_key };
execute_with_initialized_rng(&fill_ptr_key_nb);
return 0;
}
subsys_initcall(vsprintf_init_hashval)
/* Maps a pointer to a 32 bit unique identifier. */
static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
{
unsigned long hashval;
if (!READ_ONCE(filled_random_ptr_key))
return -EBUSY;
/* Pairs with smp_wmb() after writing ptr_key. */
smp_rmb();
#ifdef CONFIG_64BIT
hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
/*
* Mask off the first 32 bits, this makes explicit that we have
* modified the address (and 32 bits is plenty for a unique ID).
*/
hashval = hashval & 0xffffffff;
#else
hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
#endif
*hashval_out = hashval;
return 0;
}
int ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
{
return __ptr_to_hashval(ptr, hashval_out);
}
static char *ptr_to_id(char *buf, char *end, const void *ptr,
struct printf_spec spec)
{
const char *str = sizeof(ptr) == 8 ? "(____ptrval____)" : "(ptrval)";
unsigned long hashval;
int ret;
/*
* Print the real pointer value for NULL and error pointers,
* as they are not actual addresses.
*/
if (IS_ERR_OR_NULL(ptr)) return pointer_string(buf, end, ptr, spec);
/* When debugging early boot use non-cryptographically secure hash. */
if (unlikely(debug_boot_weak_hash)) { hashval = hash_long((unsigned long)ptr, 32); return pointer_string(buf, end, (const void *)hashval, spec);
}
ret = __ptr_to_hashval(ptr, &hashval);
if (ret) {
spec.field_width = 2 * sizeof(ptr);
/* string length must be less than default_width */
return error_string(buf, end, str, spec);
}
return pointer_string(buf, end, (const void *)hashval, spec);
}
static char *default_pointer(char *buf, char *end, const void *ptr,
struct printf_spec spec)
{
/*
* default is to _not_ leak addresses, so hash before printing,
* unless no_hash_pointers is specified on the command line.
*/
if (unlikely(no_hash_pointers)) return pointer_string(buf, end, ptr, spec); return ptr_to_id(buf, end, ptr, spec);
}
int kptr_restrict __read_mostly;
static noinline_for_stack
char *restricted_pointer(char *buf, char *end, const void *ptr,
struct printf_spec spec)
{
switch (kptr_restrict) {
case 0:
/* Handle as %p, hash and do _not_ leak addresses. */
return default_pointer(buf, end, ptr, spec);
case 1: {
const struct cred *cred;
/*
* kptr_restrict==1 cannot be used in IRQ context
* because its test for CAP_SYSLOG would be meaningless.
*/
if (in_hardirq() || in_serving_softirq() || in_nmi()) {
if (spec.field_width == -1)
spec.field_width = 2 * sizeof(ptr);
return error_string(buf, end, "pK-error", spec);
}
/*
* Only print the real pointer value if the current
* process has CAP_SYSLOG and is running with the
* same credentials it started with. This is because
* access to files is checked at open() time, but %pK
* checks permission at read() time. We don't want to
* leak pointer values if a binary opens a file using
* %pK and then elevates privileges before reading it.
*/
cred = current_cred();
if (!has_capability_noaudit(current, CAP_SYSLOG) ||
!uid_eq(cred->euid, cred->uid) ||
!gid_eq(cred->egid, cred->gid))
ptr = NULL;
break;
}
case 2:
default:
/* Always print 0's for %pK */
ptr = NULL;
break;
}
return pointer_string(buf, end, ptr, spec);
}
static noinline_for_stack
char *dentry_name(char *buf, char *end, const struct dentry *d, struct printf_spec spec,
const char *fmt)
{
const char *array[4], *s;
const struct dentry *p;
int depth;
int i, n;
switch (fmt[1]) {
case '2': case '3': case '4':
depth = fmt[1] - '0';
break;
default:
depth = 1;
}
rcu_read_lock();
for (i = 0; i < depth; i++, d = p) {
if (check_pointer(&buf, end, d, spec)) {
rcu_read_unlock();
return buf;
}
p = READ_ONCE(d->d_parent);
array[i] = READ_ONCE(d->d_name.name);
if (p == d) {
if (i)
array[i] = "";
i++;
break;
}
}
s = array[--i];
for (n = 0; n != spec.precision; n++, buf++) {
char c = *s++;
if (!c) {
if (!i)
break;
c = '/';
s = array[--i];
}
if (buf < end)
*buf = c;
}
rcu_read_unlock();
return widen_string(buf, n, end, spec);
}
static noinline_for_stack
char *file_dentry_name(char *buf, char *end, const struct file *f,
struct printf_spec spec, const char *fmt)
{
if (check_pointer(&buf, end, f, spec))
return buf;
return dentry_name(buf, end, f->f_path.dentry, spec, fmt);
}
#ifdef CONFIG_BLOCK
static noinline_for_stack
char *bdev_name(char *buf, char *end, struct block_device *bdev,
struct printf_spec spec, const char *fmt)
{
struct gendisk *hd;
if (check_pointer(&buf, end, bdev, spec))
return buf;
hd = bdev->bd_disk;
buf = string(buf, end, hd->disk_name, spec);
if (bdev->bd_partno) {
if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) {
if (buf < end)
*buf = 'p';
buf++;
}
buf = number(buf, end, bdev->bd_partno, spec);
}
return buf;
}
#endif
static noinline_for_stack
char *symbol_string(char *buf, char *end, void *ptr,
struct printf_spec spec, const char *fmt)
{
unsigned long value;
#ifdef CONFIG_KALLSYMS
char sym[KSYM_SYMBOL_LEN];
#endif
if (fmt[1] == 'R')
ptr = __builtin_extract_return_addr(ptr);
value = (unsigned long)ptr;
#ifdef CONFIG_KALLSYMS
if (*fmt == 'B' && fmt[1] == 'b')
sprint_backtrace_build_id(sym, value);
else if (*fmt == 'B')
sprint_backtrace(sym, value);
else if (*fmt == 'S' && (fmt[1] == 'b' || (fmt[1] == 'R' && fmt[2] == 'b')))
sprint_symbol_build_id(sym, value);
else if (*fmt != 's')
sprint_symbol(sym, value);
else
sprint_symbol_no_offset(sym, value);
return string_nocheck(buf, end, sym, spec);
#else
return special_hex_number(buf, end, value, sizeof(void *));
#endif
}
static const struct printf_spec default_str_spec = {
.field_width = -1,
.precision = -1,
};
static const struct printf_spec default_flag_spec = {
.base = 16,
.precision = -1,
.flags = SPECIAL | SMALL,
};
static const struct printf_spec default_dec_spec = {
.base = 10,
.precision = -1,
};
static const struct printf_spec default_dec02_spec = {
.base = 10,
.field_width = 2,
.precision = -1,
.flags = ZEROPAD,
};
static const struct printf_spec default_dec04_spec = {
.base = 10,
.field_width = 4,
.precision = -1,
.flags = ZEROPAD,
};
static noinline_for_stack
char *resource_string(char *buf, char *end, struct resource *res,
struct printf_spec spec, const char *fmt)
{
#ifndef IO_RSRC_PRINTK_SIZE
#define IO_RSRC_PRINTK_SIZE 6
#endif
#ifndef MEM_RSRC_PRINTK_SIZE
#define MEM_RSRC_PRINTK_SIZE 10
#endif
static const struct printf_spec io_spec = {
.base = 16,
.field_width = IO_RSRC_PRINTK_SIZE,
.precision = -1,
.flags = SPECIAL | SMALL | ZEROPAD,
};
static const struct printf_spec mem_spec = {
.base = 16,
.field_width = MEM_RSRC_PRINTK_SIZE,
.precision = -1,
.flags = SPECIAL | SMALL | ZEROPAD,
};
static const struct printf_spec bus_spec = {
.base = 16,
.field_width = 2,
.precision = -1,
.flags = SMALL | ZEROPAD,
};
static const struct printf_spec str_spec = {
.field_width = -1,
.precision = 10,
.flags = LEFT,
};
/* 32-bit res (sizeof==4): 10 chars in dec, 10 in hex ("0x" + 8)
* 64-bit res (sizeof==8): 20 chars in dec, 18 in hex ("0x" + 16) */
#define RSRC_BUF_SIZE ((2 * sizeof(resource_size_t)) + 4)
#define FLAG_BUF_SIZE (2 * sizeof(res->flags))
#define DECODED_BUF_SIZE sizeof("[mem - 64bit pref window disabled]")
#define RAW_BUF_SIZE sizeof("[mem - flags 0x]")
char sym[max(2*RSRC_BUF_SIZE + DECODED_BUF_SIZE,
2*RSRC_BUF_SIZE + FLAG_BUF_SIZE + RAW_BUF_SIZE)];
char *p = sym, *pend = sym + sizeof(sym);
int decode = (fmt[0] == 'R') ? 1 : 0;
const struct printf_spec *specp;
if (check_pointer(&buf, end, res, spec))
return buf;
*p++ = '[';
if (res->flags & IORESOURCE_IO) {
p = string_nocheck(p, pend, "io ", str_spec);
specp = &io_spec;
} else if (res->flags & IORESOURCE_MEM) {
p = string_nocheck(p, pend, "mem ", str_spec);
specp = &mem_spec;
} else if (res->flags & IORESOURCE_IRQ) {
p = string_nocheck(p, pend, "irq ", str_spec);
specp = &default_dec_spec;
} else if (res->flags & IORESOURCE_DMA) {
p = string_nocheck(p, pend, "dma ", str_spec);
specp = &default_dec_spec;
} else if (res->flags & IORESOURCE_BUS) {
p = string_nocheck(p, pend, "bus ", str_spec);
specp = &bus_spec;
} else {
p = string_nocheck(p, pend, "??? ", str_spec);
specp = &mem_spec;
decode = 0;
}
if (decode && res->flags & IORESOURCE_UNSET) {
p = string_nocheck(p, pend, "size ", str_spec);
p = number(p, pend, resource_size(res), *specp);
} else {
p = number(p, pend, res->start, *specp);
if (res->start != res->end) {
*p++ = '-';
p = number(p, pend, res->end, *specp);
}
}
if (decode) {
if (res->flags & IORESOURCE_MEM_64)
p = string_nocheck(p, pend, " 64bit", str_spec);
if (res->flags & IORESOURCE_PREFETCH)
p = string_nocheck(p, pend, " pref", str_spec);
if (res->flags & IORESOURCE_WINDOW)
p = string_nocheck(p, pend, " window", str_spec);
if (res->flags & IORESOURCE_DISABLED)
p = string_nocheck(p, pend, " disabled", str_spec);
} else {
p = string_nocheck(p, pend, " flags ", str_spec);
p = number(p, pend, res->flags, default_flag_spec);
}
*p++ = ']';
*p = '\0';
return string_nocheck(buf, end, sym, spec);
}
static noinline_for_stack
char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
const char *fmt)
{
int i, len = 1; /* if we pass '%ph[CDN]', field width remains
negative value, fallback to the default */
char separator;
if (spec.field_width == 0)
/* nothing to print */
return buf;
if (check_pointer(&buf, end, addr, spec))
return buf;
switch (fmt[1]) {
case 'C':
separator = ':';
break;
case 'D':
separator = '-';
break;
case 'N':
separator = 0;
break;
default:
separator = ' ';
break;
}
if (spec.field_width > 0)
len = min_t(int, spec.field_width, 64);
for (i = 0; i < len; ++i) {
if (buf < end)
*buf = hex_asc_hi(addr[i]);
++buf;
if (buf < end)
*buf = hex_asc_lo(addr[i]);
++buf;
if (separator && i != len - 1) {
if (buf < end)
*buf = separator;
++buf;
}
}
return buf;
}
static noinline_for_stack
char *bitmap_string(char *buf, char *end, const unsigned long *bitmap,
struct printf_spec spec, const char *fmt)
{
const int CHUNKSZ = 32;
int nr_bits = max_t(int, spec.field_width, 0);
int i, chunksz;
bool first = true;
if (check_pointer(&buf, end, bitmap, spec))
return buf;
/* reused to print numbers */
spec = (struct printf_spec){ .flags = SMALL | ZEROPAD, .base = 16 };
chunksz = nr_bits & (CHUNKSZ - 1);
if (chunksz == 0)
chunksz = CHUNKSZ;
i = ALIGN(nr_bits, CHUNKSZ) - CHUNKSZ;
for (; i >= 0; i -= CHUNKSZ) {
u32 chunkmask, val;
int word, bit;
chunkmask = ((1ULL << chunksz) - 1);
word = i / BITS_PER_LONG;
bit = i % BITS_PER_LONG;
val = (bitmap[word] >> bit) & chunkmask;
if (!first) {
if (buf < end)
*buf = ',';
buf++;
}
first = false;
spec.field_width = DIV_ROUND_UP(chunksz, 4);
buf = number(buf, end, val, spec);
chunksz = CHUNKSZ;
}
return buf;
}
static noinline_for_stack
char *bitmap_list_string(char *buf, char *end, const unsigned long *bitmap,
struct printf_spec spec, const char *fmt)
{
int nr_bits = max_t(int, spec.field_width, 0);
bool first = true;
int rbot, rtop;
if (check_pointer(&buf, end, bitmap, spec))
return buf;
for_each_set_bitrange(rbot, rtop, bitmap, nr_bits) {
if (!first) {
if (buf < end)
*buf = ',';
buf++;
}
first = false;
buf = number(buf, end, rbot, default_dec_spec);
if (rtop == rbot + 1)
continue;
if (buf < end)
*buf = '-';
buf = number(++buf, end, rtop - 1, default_dec_spec);
}
return buf;
}
static noinline_for_stack
char *mac_address_string(char *buf, char *end, u8 *addr,
struct printf_spec spec, const char *fmt)
{
char mac_addr[sizeof("xx:xx:xx:xx:xx:xx")];
char *p = mac_addr;
int i;
char separator;
bool reversed = false;
if (check_pointer(&buf, end, addr, spec))
return buf;
switch (fmt[1]) {
case 'F':
separator = '-';
break;
case 'R':
reversed = true;
fallthrough;
default:
separator = ':';
break;
}
for (i = 0; i < 6; i++) {
if (reversed)
p = hex_byte_pack(p, addr[5 - i]);
else
p = hex_byte_pack(p, addr[i]);
if (fmt[0] == 'M' && i != 5)
*p++ = separator;
}
*p = '\0';
return string_nocheck(buf, end, mac_addr, spec);
}
static noinline_for_stack
char *ip4_string(char *p, const u8 *addr, const char *fmt)
{
int i;
bool leading_zeros = (fmt[0] == 'i');
int index;
int step;
switch (fmt[2]) {
case 'h':
#ifdef __BIG_ENDIAN
index = 0;
step = 1;
#else
index = 3;
step = -1;
#endif
break;
case 'l':
index = 3;
step = -1;
break;
case 'n':
case 'b':
default:
index = 0;
step = 1;
break;
}
for (i = 0; i < 4; i++) {
char temp[4] __aligned(2); /* hold each IP quad in reverse order */
int digits = put_dec_trunc8(temp, addr[index]) - temp;
if (leading_zeros) {
if (digits < 3)
*p++ = '0';
if (digits < 2)
*p++ = '0';
}
/* reverse the digits in the quad */
while (digits--)
*p++ = temp[digits];
if (i < 3)
*p++ = '.';
index += step;
}
*p = '\0';
return p;
}
static noinline_for_stack
char *ip6_compressed_string(char *p, const char *addr)
{
int i, j, range;
unsigned char zerolength[8];
int longest = 1;
int colonpos = -1;
u16 word;
u8 hi, lo;
bool needcolon = false;
bool useIPv4;
struct in6_addr in6;
memcpy(&in6, addr, sizeof(struct in6_addr));
useIPv4 = ipv6_addr_v4mapped(&in6) || ipv6_addr_is_isatap(&in6);
memset(zerolength, 0, sizeof(zerolength));
if (useIPv4)
range = 6;
else
range = 8;
/* find position of longest 0 run */
for (i = 0; i < range; i++) {
for (j = i; j < range; j++) {
if (in6.s6_addr16[j] != 0)
break;
zerolength[i]++;
}
}
for (i = 0; i < range; i++) {
if (zerolength[i] > longest) {
longest = zerolength[i];
colonpos = i;
}
}
if (longest == 1) /* don't compress a single 0 */
colonpos = -1;
/* emit address */
for (i = 0; i < range; i++) {
if (i == colonpos) {
if (needcolon || i == 0)
*p++ = ':';
*p++ = ':';
needcolon = false;
i += longest - 1;
continue;
}
if (needcolon) {
*p++ = ':';
needcolon = false;
}
/* hex u16 without leading 0s */
word = ntohs(in6.s6_addr16[i]);
hi = word >> 8;
lo = word & 0xff;
if (hi) {
if (hi > 0x0f)
p = hex_byte_pack(p, hi);
else
*p++ = hex_asc_lo(hi);
p = hex_byte_pack(p, lo);
}
else if (lo > 0x0f)
p = hex_byte_pack(p, lo);
else
*p++ = hex_asc_lo(lo);
needcolon = true;
}
if (useIPv4) {
if (needcolon)
*p++ = ':';
p = ip4_string(p, &in6.s6_addr[12], "I4");
}
*p = '\0';
return p;
}
static noinline_for_stack
char *ip6_string(char *p, const char *addr, const char *fmt)
{
int i;
for (i = 0; i < 8; i++) {
p = hex_byte_pack(p, *addr++);
p = hex_byte_pack(p, *addr++);
if (fmt[0] == 'I' && i != 7)
*p++ = ':';
}
*p = '\0';
return p;
}
static noinline_for_stack
char *ip6_addr_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec, const char *fmt)
{
char ip6_addr[sizeof("xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255")];
if (fmt[0] == 'I' && fmt[2] == 'c')
ip6_compressed_string(ip6_addr, addr);
else
ip6_string(ip6_addr, addr, fmt);
return string_nocheck(buf, end, ip6_addr, spec);
}
static noinline_for_stack
char *ip4_addr_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec, const char *fmt)
{
char ip4_addr[sizeof("255.255.255.255")];
ip4_string(ip4_addr, addr, fmt);
return string_nocheck(buf, end, ip4_addr, spec);
}
static noinline_for_stack
char *ip6_addr_string_sa(char *buf, char *end, const struct sockaddr_in6 *sa,
struct printf_spec spec, const char *fmt)
{
bool have_p = false, have_s = false, have_f = false, have_c = false;
char ip6_addr[sizeof("[xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255]") +
sizeof(":12345") + sizeof("/123456789") +
sizeof("%1234567890")];
char *p = ip6_addr, *pend = ip6_addr + sizeof(ip6_addr);
const u8 *addr = (const u8 *) &sa->sin6_addr;
char fmt6[2] = { fmt[0], '6' };
u8 off = 0;
fmt++;
while (isalpha(*++fmt)) {
switch (*fmt) {
case 'p':
have_p = true;
break;
case 'f':
have_f = true;
break;
case 's':
have_s = true;
break;
case 'c':
have_c = true;
break;
}
}
if (have_p || have_s || have_f) {
*p = '[';
off = 1;
}
if (fmt6[0] == 'I' && have_c)
p = ip6_compressed_string(ip6_addr + off, addr);
else
p = ip6_string(ip6_addr + off, addr, fmt6);
if (have_p || have_s || have_f)
*p++ = ']';
if (have_p) {
*p++ = ':';
p = number(p, pend, ntohs(sa->sin6_port), spec);
}
if (have_f) {
*p++ = '/';
p = number(p, pend, ntohl(sa->sin6_flowinfo &
IPV6_FLOWINFO_MASK), spec);
}
if (have_s) {
*p++ = '%';
p = number(p, pend, sa->sin6_scope_id, spec);
}
*p = '\0';
return string_nocheck(buf, end, ip6_addr, spec);
}
static noinline_for_stack
char *ip4_addr_string_sa(char *buf, char *end, const struct sockaddr_in *sa,
struct printf_spec spec, const char *fmt)
{
bool have_p = false;
char *p, ip4_addr[sizeof("255.255.255.255") + sizeof(":12345")];
char *pend = ip4_addr + sizeof(ip4_addr);
const u8 *addr = (const u8 *) &sa->sin_addr.s_addr;
char fmt4[3] = { fmt[0], '4', 0 };
fmt++;
while (isalpha(*++fmt)) {
switch (*fmt) {
case 'p':
have_p = true;
break;
case 'h':
case 'l':
case 'n':
case 'b':
fmt4[2] = *fmt;
break;
}
}
p = ip4_string(ip4_addr, addr, fmt4);
if (have_p) {
*p++ = ':';
p = number(p, pend, ntohs(sa->sin_port), spec);
}
*p = '\0';
return string_nocheck(buf, end, ip4_addr, spec);
}
static noinline_for_stack
char *ip_addr_string(char *buf, char *end, const void *ptr,
struct printf_spec spec, const char *fmt)
{
char *err_fmt_msg;
if (check_pointer(&buf, end, ptr, spec))
return buf;
switch (fmt[1]) {
case '6':
return ip6_addr_string(buf, end, ptr, spec, fmt);
case '4':
return ip4_addr_string(buf, end, ptr, spec, fmt);
case 'S': {
const union {
struct sockaddr raw;
struct sockaddr_in v4;
struct sockaddr_in6 v6;
} *sa = ptr;
switch (sa->raw.sa_family) {
case AF_INET:
return ip4_addr_string_sa(buf, end, &sa->v4, spec, fmt);
case AF_INET6:
return ip6_addr_string_sa(buf, end, &sa->v6, spec, fmt);
default:
return error_string(buf, end, "(einval)", spec);
}}
}
err_fmt_msg = fmt[0] == 'i' ? "(%pi?)" : "(%pI?)";
return error_string(buf, end, err_fmt_msg, spec);
}
static noinline_for_stack
char *escaped_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
const char *fmt)
{
bool found = true;
int count = 1;
unsigned int flags = 0;
int len;
if (spec.field_width == 0)
return buf; /* nothing to print */
if (check_pointer(&buf, end, addr, spec))
return buf;
do {
switch (fmt[count++]) {
case 'a':
flags |= ESCAPE_ANY;
break;
case 'c':
flags |= ESCAPE_SPECIAL;
break;
case 'h':
flags |= ESCAPE_HEX;
break;
case 'n':
flags |= ESCAPE_NULL;
break;
case 'o':
flags |= ESCAPE_OCTAL;
break;
case 'p':
flags |= ESCAPE_NP;
break;
case 's':
flags |= ESCAPE_SPACE;
break;
default:
found = false;
break;
}
} while (found);
if (!flags)
flags = ESCAPE_ANY_NP;
len = spec.field_width < 0 ? 1 : spec.field_width;
/*
* string_escape_mem() writes as many characters as it can to
* the given buffer, and returns the total size of the output
* had the buffer been big enough.
*/
buf += string_escape_mem(addr, len, buf, buf < end ? end - buf : 0, flags, NULL);
return buf;
}
static char *va_format(char *buf, char *end, struct va_format *va_fmt,
struct printf_spec spec, const char *fmt)
{
va_list va; if (check_pointer(&buf, end, va_fmt, spec))
return buf;
va_copy(va, *va_fmt->va);
buf += vsnprintf(buf, end > buf ? end - buf : 0, va_fmt->fmt, va);
va_end(va);
return buf;
}
static noinline_for_stack
char *uuid_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec, const char *fmt)
{
char uuid[UUID_STRING_LEN + 1];
char *p = uuid;
int i;
const u8 *index = uuid_index;
bool uc = false;
if (check_pointer(&buf, end, addr, spec))
return buf;
switch (*(++fmt)) {
case 'L':
uc = true;
fallthrough;
case 'l':
index = guid_index;
break;
case 'B':
uc = true;
break;
}
for (i = 0; i < 16; i++) {
if (uc)
p = hex_byte_pack_upper(p, addr[index[i]]);
else
p = hex_byte_pack(p, addr[index[i]]);
switch (i) {
case 3:
case 5:
case 7:
case 9:
*p++ = '-';
break;
}
}
*p = 0;
return string_nocheck(buf, end, uuid, spec);
}
static noinline_for_stack
char *netdev_bits(char *buf, char *end, const void *addr,
struct printf_spec spec, const char *fmt)
{
unsigned long long num;
int size;
if (check_pointer(&buf, end, addr, spec))
return buf;
switch (fmt[1]) {
case 'F':
num = *(const netdev_features_t *)addr;
size = sizeof(netdev_features_t);
break;
default:
return error_string(buf, end, "(%pN?)", spec);
}
return special_hex_number(buf, end, num, size);
}
static noinline_for_stack
char *fourcc_string(char *buf, char *end, const u32 *fourcc,
struct printf_spec spec, const char *fmt)
{
char output[sizeof("0123 little-endian (0x01234567)")];
char *p = output;
unsigned int i;
u32 orig, val;
if (fmt[1] != 'c' || fmt[2] != 'c')
return error_string(buf, end, "(%p4?)", spec);
if (check_pointer(&buf, end, fourcc, spec))
return buf;
orig = get_unaligned(fourcc);
val = orig & ~BIT(31);
for (i = 0; i < sizeof(u32); i++) {
unsigned char c = val >> (i * 8);
/* Print non-control ASCII characters as-is, dot otherwise */
*p++ = isascii(c) && isprint(c) ? c : '.';
}
*p++ = ' ';
strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian");
p += strlen(p);
*p++ = ' ';
*p++ = '(';
p = special_hex_number(p, output + sizeof(output) - 2, orig, sizeof(u32));
*p++ = ')';
*p = '\0';
return string(buf, end, output, spec);
}
static noinline_for_stack
char *address_val(char *buf, char *end, const void *addr,
struct printf_spec spec, const char *fmt)
{
unsigned long long num;
int size;
if (check_pointer(&buf, end, addr, spec))
return buf;
switch (fmt[1]) {
case 'd':
num = *(const dma_addr_t *)addr;
size = sizeof(dma_addr_t);
break;
case 'p':
default:
num = *(const phys_addr_t *)addr;
size = sizeof(phys_addr_t);
break;
}
return special_hex_number(buf, end, num, size);
}
static noinline_for_stack
char *date_str(char *buf, char *end, const struct rtc_time *tm, bool r)
{
int year = tm->tm_year + (r ? 0 : 1900);
int mon = tm->tm_mon + (r ? 0 : 1);
buf = number(buf, end, year, default_dec04_spec);
if (buf < end)
*buf = '-';
buf++;
buf = number(buf, end, mon, default_dec02_spec);
if (buf < end)
*buf = '-';
buf++;
return number(buf, end, tm->tm_mday, default_dec02_spec);
}
static noinline_for_stack
char *time_str(char *buf, char *end, const struct rtc_time *tm, bool r)
{
buf = number(buf, end, tm->tm_hour, default_dec02_spec);
if (buf < end)
*buf = ':';
buf++;
buf = number(buf, end, tm->tm_min, default_dec02_spec);
if (buf < end)
*buf = ':';
buf++;
return number(buf, end, tm->tm_sec, default_dec02_spec);
}
static noinline_for_stack
char *rtc_str(char *buf, char *end, const struct rtc_time *tm,
struct printf_spec spec, const char *fmt)
{
bool have_t = true, have_d = true;
bool raw = false, iso8601_separator = true;
bool found = true;
int count = 2;
if (check_pointer(&buf, end, tm, spec))
return buf;
switch (fmt[count]) {
case 'd':
have_t = false;
count++;
break;
case 't':
have_d = false;
count++;
break;
}
do {
switch (fmt[count++]) {
case 'r':
raw = true;
break;
case 's':
iso8601_separator = false;
break;
default:
found = false;
break;
}
} while (found);
if (have_d)
buf = date_str(buf, end, tm, raw);
if (have_d && have_t) {
if (buf < end)
*buf = iso8601_separator ? 'T' : ' ';
buf++;
}
if (have_t)
buf = time_str(buf, end, tm, raw);
return buf;
}
static noinline_for_stack
char *time64_str(char *buf, char *end, const time64_t time,
struct printf_spec spec, const char *fmt)
{
struct rtc_time rtc_time;
struct tm tm;
time64_to_tm(time, 0, &tm);
rtc_time.tm_sec = tm.tm_sec;
rtc_time.tm_min = tm.tm_min;
rtc_time.tm_hour = tm.tm_hour;
rtc_time.tm_mday = tm.tm_mday;
rtc_time.tm_mon = tm.tm_mon;
rtc_time.tm_year = tm.tm_year;
rtc_time.tm_wday = tm.tm_wday;
rtc_time.tm_yday = tm.tm_yday;
rtc_time.tm_isdst = 0;
return rtc_str(buf, end, &rtc_time, spec, fmt);
}
static noinline_for_stack
char *time_and_date(char *buf, char *end, void *ptr, struct printf_spec spec,
const char *fmt)
{
switch (fmt[1]) {
case 'R':
return rtc_str(buf, end, (const struct rtc_time *)ptr, spec, fmt);
case 'T':
return time64_str(buf, end, *(const time64_t *)ptr, spec, fmt);
default:
return error_string(buf, end, "(%pt?)", spec);
}
}
static noinline_for_stack
char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec,
const char *fmt)
{
if (!IS_ENABLED(CONFIG_HAVE_CLK))
return error_string(buf, end, "(%pC?)", spec);
if (check_pointer(&buf, end, clk, spec))
return buf;
switch (fmt[1]) {
case 'n':
default:
#ifdef CONFIG_COMMON_CLK
return string(buf, end, __clk_get_name(clk), spec);
#else
return ptr_to_id(buf, end, clk, spec);
#endif
}
}
static
char *format_flags(char *buf, char *end, unsigned long flags,
const struct trace_print_flags *names)
{
unsigned long mask;
for ( ; flags && names->name; names++) {
mask = names->mask;
if ((flags & mask) != mask)
continue;
buf = string(buf, end, names->name, default_str_spec);
flags &= ~mask;
if (flags) {
if (buf < end)
*buf = '|';
buf++;
}
}
if (flags)
buf = number(buf, end, flags, default_flag_spec);
return buf;
}
struct page_flags_fields {
int width;
int shift;
int mask;
const struct printf_spec *spec;
const char *name;
};
static const struct page_flags_fields pff[] = {
{SECTIONS_WIDTH, SECTIONS_PGSHIFT, SECTIONS_MASK,
&default_dec_spec, "section"},
{NODES_WIDTH, NODES_PGSHIFT, NODES_MASK,
&default_dec_spec, "node"},
{ZONES_WIDTH, ZONES_PGSHIFT, ZONES_MASK,
&default_dec_spec, "zone"},
{LAST_CPUPID_WIDTH, LAST_CPUPID_PGSHIFT, LAST_CPUPID_MASK,
&default_flag_spec, "lastcpupid"},
{KASAN_TAG_WIDTH, KASAN_TAG_PGSHIFT, KASAN_TAG_MASK,
&default_flag_spec, "kasantag"},
};
static
char *format_page_flags(char *buf, char *end, unsigned long flags)
{
unsigned long main_flags = flags & PAGEFLAGS_MASK;
bool append = false;
int i;
buf = number(buf, end, flags, default_flag_spec);
if (buf < end)
*buf = '(';
buf++;
/* Page flags from the main area. */
if (main_flags) {
buf = format_flags(buf, end, main_flags, pageflag_names);
append = true;
}
/* Page flags from the fields area */
for (i = 0; i < ARRAY_SIZE(pff); i++) {
/* Skip undefined fields. */
if (!pff[i].width)
continue;
/* Format: Flag Name + '=' (equals sign) + Number + '|' (separator) */
if (append) {
if (buf < end)
*buf = '|';
buf++;
}
buf = string(buf, end, pff[i].name, default_str_spec);
if (buf < end)
*buf = '=';
buf++;
buf = number(buf, end, (flags >> pff[i].shift) & pff[i].mask,
*pff[i].spec);
append = true;
}
if (buf < end)
*buf = ')';
buf++;
return buf;
}
static
char *format_page_type(char *buf, char *end, unsigned int page_type)
{
buf = number(buf, end, page_type, default_flag_spec);
if (buf < end)
*buf = '(';
buf++;
if (page_type_has_type(page_type))
buf = format_flags(buf, end, ~page_type, pagetype_names);
if (buf < end)
*buf = ')';
buf++;
return buf;
}
static noinline_for_stack
char *flags_string(char *buf, char *end, void *flags_ptr,
struct printf_spec spec, const char *fmt)
{
unsigned long flags;
const struct trace_print_flags *names;
if (check_pointer(&buf, end, flags_ptr, spec))
return buf;
switch (fmt[1]) {
case 'p':
return format_page_flags(buf, end, *(unsigned long *)flags_ptr);
case 't':
return format_page_type(buf, end, *(unsigned int *)flags_ptr);
case 'v':
flags = *(unsigned long *)flags_ptr;
names = vmaflag_names;
break;
case 'g':
flags = (__force unsigned long)(*(gfp_t *)flags_ptr);
names = gfpflag_names;
break;
default:
return error_string(buf, end, "(%pG?)", spec);
}
return format_flags(buf, end, flags, names);
}
static noinline_for_stack
char *fwnode_full_name_string(struct fwnode_handle *fwnode, char *buf,
char *end)
{
int depth;
/* Loop starting from the root node to the current node. */
for (depth = fwnode_count_parents(fwnode); depth >= 0; depth--) {
/*
* Only get a reference for other nodes (i.e. parent nodes).
* fwnode refcount may be 0 here.
*/
struct fwnode_handle *__fwnode = depth ?
fwnode_get_nth_parent(fwnode, depth) : fwnode;
buf = string(buf, end, fwnode_get_name_prefix(__fwnode),
default_str_spec);
buf = string(buf, end, fwnode_get_name(__fwnode),
default_str_spec);
if (depth)
fwnode_handle_put(__fwnode);
}
return buf;
}
static noinline_for_stack
char *device_node_string(char *buf, char *end, struct device_node *dn,
struct printf_spec spec, const char *fmt)
{
char tbuf[sizeof("xxxx") + 1];
const char *p;
int ret;
char *buf_start = buf;
struct property *prop;
bool has_mult, pass;
struct printf_spec str_spec = spec;
str_spec.field_width = -1;
if (fmt[0] != 'F')
return error_string(buf, end, "(%pO?)", spec);
if (!IS_ENABLED(CONFIG_OF))
return error_string(buf, end, "(%pOF?)", spec);
if (check_pointer(&buf, end, dn, spec))
return buf;
/* simple case without anything any more format specifiers */
fmt++;
if (fmt[0] == '\0' || strcspn(fmt,"fnpPFcC") > 0)
fmt = "f";
for (pass = false; strspn(fmt,"fnpPFcC"); fmt++, pass = true) {
int precision;
if (pass) {
if (buf < end)
*buf = ':';
buf++;
}
switch (*fmt) {
case 'f': /* full_name */
buf = fwnode_full_name_string(of_fwnode_handle(dn), buf,
end);
break;
case 'n': /* name */
p = fwnode_get_name(of_fwnode_handle(dn));
precision = str_spec.precision;
str_spec.precision = strchrnul(p, '@') - p;
buf = string(buf, end, p, str_spec);
str_spec.precision = precision;
break;
case 'p': /* phandle */
buf = number(buf, end, (unsigned int)dn->phandle, default_dec_spec);
break;
case 'P': /* path-spec */
p = fwnode_get_name(of_fwnode_handle(dn));
if (!p[1])
p = "/";
buf = string(buf, end, p, str_spec);
break;
case 'F': /* flags */
tbuf[0] = of_node_check_flag(dn, OF_DYNAMIC) ? 'D' : '-';
tbuf[1] = of_node_check_flag(dn, OF_DETACHED) ? 'd' : '-';
tbuf[2] = of_node_check_flag(dn, OF_POPULATED) ? 'P' : '-';
tbuf[3] = of_node_check_flag(dn, OF_POPULATED_BUS) ? 'B' : '-';
tbuf[4] = 0;
buf = string_nocheck(buf, end, tbuf, str_spec);
break;
case 'c': /* major compatible string */
ret = of_property_read_string(dn, "compatible", &p);
if (!ret)
buf = string(buf, end, p, str_spec);
break;
case 'C': /* full compatible string */
has_mult = false;
of_property_for_each_string(dn, "compatible", prop, p) {
if (has_mult)
buf = string_nocheck(buf, end, ",", str_spec);
buf = string_nocheck(buf, end, "\"", str_spec);
buf = string(buf, end, p, str_spec);
buf = string_nocheck(buf, end, "\"", str_spec);
has_mult = true;
}
break;
default:
break;
}
}
return widen_string(buf, buf - buf_start, end, spec);
}
static noinline_for_stack
char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode,
struct printf_spec spec, const char *fmt)
{
struct printf_spec str_spec = spec;
char *buf_start = buf;
str_spec.field_width = -1;
if (*fmt != 'w')
return error_string(buf, end, "(%pf?)", spec);
if (check_pointer(&buf, end, fwnode, spec))
return buf;
fmt++;
switch (*fmt) {
case 'P': /* name */
buf = string(buf, end, fwnode_get_name(fwnode), str_spec);
break;
case 'f': /* full_name */
default:
buf = fwnode_full_name_string(fwnode, buf, end);
break;
}
return widen_string(buf, buf - buf_start, end, spec);
}
int __init no_hash_pointers_enable(char *str)
{
if (no_hash_pointers)
return 0;
no_hash_pointers = true;
pr_warn("**********************************************************\n");
pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
pr_warn("** **\n");
pr_warn("** This system shows unhashed kernel memory addresses **\n");
pr_warn("** via the console, logs, and other interfaces. This **\n");
pr_warn("** might reduce the security of your system. **\n");
pr_warn("** **\n");
pr_warn("** If you see this message and you are not debugging **\n");
pr_warn("** the kernel, report this immediately to your system **\n");
pr_warn("** administrator! **\n");
pr_warn("** **\n");
pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
pr_warn("**********************************************************\n");
return 0;
}
early_param("no_hash_pointers", no_hash_pointers_enable);
/* Used for Rust formatting ('%pA'). */
char *rust_fmt_argument(char *buf, char *end, void *ptr);
/*
* Show a '%p' thing. A kernel extension is that the '%p' is followed
* by an extra set of alphanumeric characters that are extended format
* specifiers.
*
* Please update scripts/checkpatch.pl when adding/removing conversion
* characters. (Search for "check for vsprintf extension").
*
* Right now we handle:
*
* - 'S' For symbolic direct pointers (or function descriptors) with offset
* - 's' For symbolic direct pointers (or function descriptors) without offset
* - '[Ss]R' as above with __builtin_extract_return_addr() translation
* - 'S[R]b' as above with module build ID (for use in backtraces)
* - '[Ff]' %pf and %pF were obsoleted and later removed in favor of
* %ps and %pS. Be careful when re-using these specifiers.
* - 'B' For backtraced symbolic direct pointers with offset
* - 'Bb' as above with module build ID (for use in backtraces)
* - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref]
* - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201]
* - 'b[l]' For a bitmap, the number of bits is determined by the field
* width which must be explicitly specified either as part of the
* format string '%32b[l]' or through '%*b[l]', [l] selects
* range-list format instead of hex format
* - 'M' For a 6-byte MAC address, it prints the address in the
* usual colon-separated hex notation
* - 'm' For a 6-byte MAC address, it prints the hex address without colons
* - 'MF' For a 6-byte MAC FDDI address, it prints the address
* with a dash-separated hex notation
* - '[mM]R' For a 6-byte MAC address, Reverse order (Bluetooth)
* - 'I' [46] for IPv4/IPv6 addresses printed in the usual way
* IPv4 uses dot-separated decimal without leading 0's (1.2.3.4)
* IPv6 uses colon separated network-order 16 bit hex with leading 0's
* [S][pfs]
* Generic IPv4/IPv6 address (struct sockaddr *) that falls back to
* [4] or [6] and is able to print port [p], flowinfo [f], scope [s]
* - 'i' [46] for 'raw' IPv4/IPv6 addresses
* IPv6 omits the colons (01020304...0f)
* IPv4 uses dot-separated decimal with leading 0's (010.123.045.006)
* [S][pfs]
* Generic IPv4/IPv6 address (struct sockaddr *) that falls back to
* [4] or [6] and is able to print port [p], flowinfo [f], scope [s]
* - '[Ii][4S][hnbl]' IPv4 addresses in host, network, big or little endian order
* - 'I[6S]c' for IPv6 addresses printed as specified by
* https://tools.ietf.org/html/rfc5952
* - 'E[achnops]' For an escaped buffer, where rules are defined by combination
* of the following flags (see string_escape_mem() for the
* details):
* a - ESCAPE_ANY
* c - ESCAPE_SPECIAL
* h - ESCAPE_HEX
* n - ESCAPE_NULL
* o - ESCAPE_OCTAL
* p - ESCAPE_NP
* s - ESCAPE_SPACE
* By default ESCAPE_ANY_NP is used.
* - 'U' For a 16 byte UUID/GUID, it prints the UUID/GUID in the form
* "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
* Options for %pU are:
* b big endian lower case hex (default)
* B big endian UPPER case hex
* l little endian lower case hex
* L little endian UPPER case hex
* big endian output byte order is:
* [0][1][2][3]-[4][5]-[6][7]-[8][9]-[10][11][12][13][14][15]
* little endian output byte order is:
* [3][2][1][0]-[5][4]-[7][6]-[8][9]-[10][11][12][13][14][15]
* - 'V' For a struct va_format which contains a format string * and va_list *,
* call vsnprintf(->format, *->va_list).
* Implements a "recursive vsnprintf".
* Do not use this feature without some mechanism to verify the
* correctness of the format string and va_list arguments.
* - 'K' For a kernel pointer that should be hidden from unprivileged users.
* Use only for procfs, sysfs and similar files, not printk(); please
* read the documentation (path below) first.
* - 'NF' For a netdev_features_t
* - '4cc' V4L2 or DRM FourCC code, with endianness and raw numerical value.
* - 'h[CDN]' For a variable-length buffer, it prints it as a hex string with
* a certain separator (' ' by default):
* C colon
* D dash
* N no separator
* The maximum supported length is 64 bytes of the input. Consider
* to use print_hex_dump() for the larger input.
* - 'a[pd]' For address types [p] phys_addr_t, [d] dma_addr_t and derivatives
* (default assumed to be phys_addr_t, passed by reference)
* - 'd[234]' For a dentry name (optionally 2-4 last components)
* - 'D[234]' Same as 'd' but for a struct file
* - 'g' For block_device name (gendisk + partition number)
* - 't[RT][dt][r][s]' For time and date as represented by:
* R struct rtc_time
* T time64_t
* - 'C' For a clock, it prints the name (Common Clock Framework) or address
* (legacy clock framework) of the clock
* - 'Cn' For a clock, it prints the name (Common Clock Framework) or address
* (legacy clock framework) of the clock
* - 'G' For flags to be printed as a collection of symbolic strings that would
* construct the specific value. Supported flags given by option:
* p page flags (see struct page) given as pointer to unsigned long
* g gfp flags (GFP_* and __GFP_*) given as pointer to gfp_t
* v vma flags (VM_*) given as pointer to unsigned long
* - 'OF[fnpPcCF]' For a device tree object
* Without any optional arguments prints the full_name
* f device node full_name
* n device node name
* p device node phandle
* P device node path spec (name + @unit)
* F device node flags
* c major compatible string
* C full compatible string
* - 'fw[fP]' For a firmware node (struct fwnode_handle) pointer
* Without an option prints the full name of the node
* f full name
* P node name, including a possible unit address
* - 'x' For printing the address unmodified. Equivalent to "%lx".
* Please read the documentation (path below) before using!
* - '[ku]s' For a BPF/tracing related format specifier, e.g. used out of
* bpf_trace_printk() where [ku] prefix specifies either kernel (k)
* or user (u) memory to probe, and:
* s a string, equivalent to "%s" on direct vsnprintf() use
*
* ** When making changes please also update:
* Documentation/core-api/printk-formats.rst
*
* Note: The default behaviour (unadorned %p) is to hash the address,
* rendering it useful as a unique identifier.
*
* There is also a '%pA' format specifier, but it is only intended to be used
* from Rust code to format core::fmt::Arguments. Do *not* use it from C.
* See rust/kernel/print.rs for details.
*/
static noinline_for_stack
char *pointer(const char *fmt, char *buf, char *end, void *ptr,
struct printf_spec spec)
{
switch (*fmt) {
case 'S':
case 's':
ptr = dereference_symbol_descriptor(ptr);
fallthrough;
case 'B':
return symbol_string(buf, end, ptr, spec, fmt);
case 'R':
case 'r':
return resource_string(buf, end, ptr, spec, fmt);
case 'h':
return hex_string(buf, end, ptr, spec, fmt);
case 'b':
switch (fmt[1]) {
case 'l':
return bitmap_list_string(buf, end, ptr, spec, fmt);
default:
return bitmap_string(buf, end, ptr, spec, fmt);
}
case 'M': /* Colon separated: 00:01:02:03:04:05 */
case 'm': /* Contiguous: 000102030405 */
/* [mM]F (FDDI) */
/* [mM]R (Reverse order; Bluetooth) */
return mac_address_string(buf, end, ptr, spec, fmt);
case 'I': /* Formatted IP supported
* 4: 1.2.3.4
* 6: 0001:0203:...:0708
* 6c: 1::708 or 1::1.2.3.4
*/
case 'i': /* Contiguous:
* 4: 001.002.003.004
* 6: 000102...0f
*/
return ip_addr_string(buf, end, ptr, spec, fmt);
case 'E':
return escaped_string(buf, end, ptr, spec, fmt);
case 'U':
return uuid_string(buf, end, ptr, spec, fmt);
case 'V':
return va_format(buf, end, ptr, spec, fmt);
case 'K':
return restricted_pointer(buf, end, ptr, spec);
case 'N':
return netdev_bits(buf, end, ptr, spec, fmt);
case '4':
return fourcc_string(buf, end, ptr, spec, fmt);
case 'a':
return address_val(buf, end, ptr, spec, fmt);
case 'd':
return dentry_name(buf, end, ptr, spec, fmt);
case 't':
return time_and_date(buf, end, ptr, spec, fmt);
case 'C':
return clock(buf, end, ptr, spec, fmt);
case 'D':
return file_dentry_name(buf, end, ptr, spec, fmt);
#ifdef CONFIG_BLOCK
case 'g':
return bdev_name(buf, end, ptr, spec, fmt);
#endif
case 'G':
return flags_string(buf, end, ptr, spec, fmt);
case 'O':
return device_node_string(buf, end, ptr, spec, fmt + 1);
case 'f':
return fwnode_string(buf, end, ptr, spec, fmt + 1);
case 'A':
if (!IS_ENABLED(CONFIG_RUST)) {
WARN_ONCE(1, "Please remove %%pA from non-Rust code\n"); return error_string(buf, end, "(%pA?)", spec);
}
return rust_fmt_argument(buf, end, ptr);
case 'x':
return pointer_string(buf, end, ptr, spec);
case 'e':
/* %pe with a non-ERR_PTR gets treated as plain %p */
if (!IS_ERR(ptr))
return default_pointer(buf, end, ptr, spec);
return err_ptr(buf, end, ptr, spec);
case 'u':
case 'k':
switch (fmt[1]) {
case 's':
return string(buf, end, ptr, spec);
default:
return error_string(buf, end, "(einval)", spec);
}
default:
return default_pointer(buf, end, ptr, spec);
}
}
/*
* Helper function to decode printf style format.
* Each call decode a token from the format and return the
* number of characters read (or likely the delta where it wants
* to go on the next call).
* The decoded token is returned through the parameters
*
* 'h', 'l', or 'L' for integer fields
* 'z' support added 23/7/1999 S.H.
* 'z' changed to 'Z' --davidm 1/25/99
* 'Z' changed to 'z' --adobriyan 2017-01-25
* 't' added for ptrdiff_t
*
* @fmt: the format string
* @type of the token returned
* @flags: various flags such as +, -, # tokens..
* @field_width: overwritten width
* @base: base of the number (octal, hex, ...)
* @precision: precision of a number
* @qualifier: qualifier of a number (long, size_t, ...)
*/
static noinline_for_stack
int format_decode(const char *fmt, struct printf_spec *spec)
{
const char *start = fmt;
char qualifier;
/* we finished early by reading the field width */
if (spec->type == FORMAT_TYPE_WIDTH) {
if (spec->field_width < 0) { spec->field_width = -spec->field_width;
spec->flags |= LEFT;
}
spec->type = FORMAT_TYPE_NONE;
goto precision;
}
/* we finished early by reading the precision */
if (spec->type == FORMAT_TYPE_PRECISION) { if (spec->precision < 0) spec->precision = 0; spec->type = FORMAT_TYPE_NONE;
goto qualifier;
}
/* By default */
spec->type = FORMAT_TYPE_NONE; for (; *fmt ; ++fmt) { if (*fmt == '%')
break;
}
/* Return the current non-format string */
if (fmt != start || !*fmt) return fmt - start;
/* Process flags */
spec->flags = 0;
while (1) { /* this also skips first '%' */
bool found = true;
++fmt;
switch (*fmt) {
case '-': spec->flags |= LEFT; break; case '+': spec->flags |= PLUS; break; case ' ': spec->flags |= SPACE; break; case '#': spec->flags |= SPECIAL; break; case '0': spec->flags |= ZEROPAD; break;
default: found = false;
}
if (!found)
break;
}
/* get field width */
spec->field_width = -1;
if (isdigit(*fmt))
spec->field_width = skip_atoi(&fmt); else if (*fmt == '*') {
/* it's the next argument */
spec->type = FORMAT_TYPE_WIDTH;
return ++fmt - start;
}
precision:
/* get the precision */
spec->precision = -1;
if (*fmt == '.') {
++fmt;
if (isdigit(*fmt)) {
spec->precision = skip_atoi(&fmt);
if (spec->precision < 0)
spec->precision = 0; } else if (*fmt == '*') {
/* it's the next argument */
spec->type = FORMAT_TYPE_PRECISION;
return ++fmt - start;
}
}
qualifier:
/* get the conversion qualifier */
qualifier = 0;
if (*fmt == 'h' || _tolower(*fmt) == 'l' || *fmt == 'z' || *fmt == 't') { qualifier = *fmt++;
if (unlikely(qualifier == *fmt)) {
if (qualifier == 'l') {
qualifier = 'L';
++fmt;
} else if (qualifier == 'h') {
qualifier = 'H';
++fmt;
}
}
}
/* default base */
spec->base = 10;
switch (*fmt) {
case 'c':
spec->type = FORMAT_TYPE_CHAR;
return ++fmt - start;
case 's':
spec->type = FORMAT_TYPE_STR; return ++fmt - start;
case 'p':
spec->type = FORMAT_TYPE_PTR;
return ++fmt - start;
case '%':
spec->type = FORMAT_TYPE_PERCENT_CHAR;
return ++fmt - start;
/* integer number formats - set up the flags and "break" */
case 'o':
spec->base = 8;
break;
case 'x':
spec->flags |= SMALL;
fallthrough;
case 'X':
spec->base = 16;
break;
case 'd':
case 'i':
spec->flags |= SIGN;
break;
case 'u':
break;
case 'n':
/*
* Since %n poses a greater security risk than
* utility, treat it as any other invalid or
* unsupported format specifier.
*/
fallthrough;
default:
WARN_ONCE(1, "Please remove unsupported %%%c in format string\n", *fmt); spec->type = FORMAT_TYPE_INVALID;
return fmt - start;
}
if (qualifier == 'L') spec->type = FORMAT_TYPE_LONG_LONG; else if (qualifier == 'l') {
BUILD_BUG_ON(FORMAT_TYPE_ULONG + SIGN != FORMAT_TYPE_LONG);
spec->type = FORMAT_TYPE_ULONG + (spec->flags & SIGN); } else if (qualifier == 'z') {
spec->type = FORMAT_TYPE_SIZE_T;
} else if (qualifier == 't') {
spec->type = FORMAT_TYPE_PTRDIFF;
} else if (qualifier == 'H') {
BUILD_BUG_ON(FORMAT_TYPE_UBYTE + SIGN != FORMAT_TYPE_BYTE);
spec->type = FORMAT_TYPE_UBYTE + (spec->flags & SIGN); } else if (qualifier == 'h') {
BUILD_BUG_ON(FORMAT_TYPE_USHORT + SIGN != FORMAT_TYPE_SHORT);
spec->type = FORMAT_TYPE_USHORT + (spec->flags & SIGN);
} else {
BUILD_BUG_ON(FORMAT_TYPE_UINT + SIGN != FORMAT_TYPE_INT);
spec->type = FORMAT_TYPE_UINT + (spec->flags & SIGN);
}
return ++fmt - start;
}
static void
set_field_width(struct printf_spec *spec, int width)
{
spec->field_width = width;
if (WARN_ONCE(spec->field_width != width, "field width %d too large", width)) {
spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX);
}
}
static void
set_precision(struct printf_spec *spec, int prec)
{
spec->precision = prec;
if (WARN_ONCE(spec->precision != prec, "precision %d too large", prec)) {
spec->precision = clamp(prec, 0, PRECISION_MAX);
}
}
/**
* vsnprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @args: Arguments for the format string
*
* This function generally follows C99 vsnprintf, but has some
* extensions and a few limitations:
*
* - ``%n`` is unsupported
* - ``%p*`` is handled by pointer()
*
* See pointer() or Documentation/core-api/printk-formats.rst for more
* extensive description.
*
* **Please update the documentation in both places when making changes**
*
* The return value is the number of characters which would
* be generated for the given input, excluding the trailing
* '\0', as per ISO C99. If you want to have the exact
* number of characters written into @buf as return value
* (not including the trailing '\0'), use vscnprintf(). If the
* return is greater than or equal to @size, the resulting
* string is truncated.
*
* If you're not already dealing with a va_list consider using snprintf().
*/
int vsnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
unsigned long long num;
char *str, *end;
struct printf_spec spec = {0};
/* Reject out-of-range values early. Large positive sizes are
used for unknown buffer sizes. */
if (WARN_ON_ONCE(size > INT_MAX))
return 0;
str = buf;
end = buf + size;
/* Make sure end is always >= buf */
if (end < buf) {
end = ((void *)-1);
size = end - buf;
}
while (*fmt) {
const char *old_fmt = fmt;
int read = format_decode(fmt, &spec);
fmt += read;
switch (spec.type) {
case FORMAT_TYPE_NONE: {
int copy = read;
if (str < end) { if (copy > end - str) copy = end - str; memcpy(str, old_fmt, copy);
}
str += read;
break;
}
case FORMAT_TYPE_WIDTH:
set_field_width(&spec, va_arg(args, int));
break;
case FORMAT_TYPE_PRECISION:
set_precision(&spec, va_arg(args, int));
break;
case FORMAT_TYPE_CHAR: {
char c;
if (!(spec.flags & LEFT)) { while (--spec.field_width > 0) { if (str < end) *str = ' '; ++str;
}
}
c = (unsigned char) va_arg(args, int); if (str < end)
*str = c;
++str; while (--spec.field_width > 0) { if (str < end) *str = ' '; ++str;
}
break;
}
case FORMAT_TYPE_STR:
str = string(str, end, va_arg(args, char *), spec);
break;
case FORMAT_TYPE_PTR:
str = pointer(fmt, str, end, va_arg(args, void *),
spec);
while (isalnum(*fmt))
fmt++;
break;
case FORMAT_TYPE_PERCENT_CHAR:
if (str < end) *str = '%'; ++str;
break;
case FORMAT_TYPE_INVALID:
/*
* Presumably the arguments passed gcc's type
* checking, but there is no safe or sane way
* for us to continue parsing the format and
* fetching from the va_list; the remaining
* specifiers and arguments would be out of
* sync.
*/
goto out;
default:
switch (spec.type) {
case FORMAT_TYPE_LONG_LONG:
num = va_arg(args, long long);
break;
case FORMAT_TYPE_ULONG:
num = va_arg(args, unsigned long);
break;
case FORMAT_TYPE_LONG:
num = va_arg(args, long);
break;
case FORMAT_TYPE_SIZE_T:
if (spec.flags & SIGN) num = va_arg(args, ssize_t);
else
num = va_arg(args, size_t);
break;
case FORMAT_TYPE_PTRDIFF:
num = va_arg(args, ptrdiff_t);
break;
case FORMAT_TYPE_UBYTE:
num = (unsigned char) va_arg(args, int);
break;
case FORMAT_TYPE_BYTE:
num = (signed char) va_arg(args, int);
break;
case FORMAT_TYPE_USHORT:
num = (unsigned short) va_arg(args, int);
break;
case FORMAT_TYPE_SHORT:
num = (short) va_arg(args, int);
break;
case FORMAT_TYPE_INT:
num = (int) va_arg(args, int);
break;
default:
num = va_arg(args, unsigned int);
}
str = number(str, end, num, spec);
}
}
out:
if (size > 0) { if (str < end) *str = '\0';
else
end[-1] = '\0';
}
/* the trailing null byte doesn't count towards the total */
return str-buf;
}
EXPORT_SYMBOL(vsnprintf);
/**
* vscnprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @args: Arguments for the format string
*
* The return value is the number of characters which have been written into
* the @buf not including the trailing '\0'. If @size is == 0 the function
* returns 0.
*
* If you're not already dealing with a va_list consider using scnprintf().
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int vscnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
int i;
if (unlikely(!size))
return 0;
i = vsnprintf(buf, size, fmt, args); if (likely(i < size))
return i;
return size - 1;}
EXPORT_SYMBOL(vscnprintf);
/**
* snprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The return value is the number of characters which would be
* generated for the given input, excluding the trailing null,
* as per ISO C99. If the return is greater than or equal to
* @size, the resulting string is truncated.
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int snprintf(char *buf, size_t size, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vsnprintf(buf, size, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(snprintf);
/**
* scnprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The return value is the number of characters written into @buf not including
* the trailing '\0'. If @size is == 0 the function returns 0.
*/
int scnprintf(char *buf, size_t size, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vscnprintf(buf, size, fmt, args); va_end(args);
return i;
}
EXPORT_SYMBOL(scnprintf);
/**
* vsprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @fmt: The format string to use
* @args: Arguments for the format string
*
* The function returns the number of characters written
* into @buf. Use vsnprintf() or vscnprintf() in order to avoid
* buffer overflows.
*
* If you're not already dealing with a va_list consider using sprintf().
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int vsprintf(char *buf, const char *fmt, va_list args)
{
return vsnprintf(buf, INT_MAX, fmt, args);
}
EXPORT_SYMBOL(vsprintf);
/**
* sprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The function returns the number of characters written
* into @buf. Use snprintf() or scnprintf() in order to avoid
* buffer overflows.
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int sprintf(char *buf, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vsnprintf(buf, INT_MAX, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(sprintf);
#ifdef CONFIG_BINARY_PRINTF
/*
* bprintf service:
* vbin_printf() - VA arguments to binary data
* bstr_printf() - Binary data to text string
*/
/**
* vbin_printf - Parse a format string and place args' binary value in a buffer
* @bin_buf: The buffer to place args' binary value
* @size: The size of the buffer(by words(32bits), not characters)
* @fmt: The format string to use
* @args: Arguments for the format string
*
* The format follows C99 vsnprintf, except %n is ignored, and its argument
* is skipped.
*
* The return value is the number of words(32bits) which would be generated for
* the given input.
*
* NOTE:
* If the return value is greater than @size, the resulting bin_buf is NOT
* valid for bstr_printf().
*/
int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args)
{
struct printf_spec spec = {0};
char *str, *end;
int width;
str = (char *)bin_buf;
end = (char *)(bin_buf + size);
#define save_arg(type) \
({ \
unsigned long long value; \
if (sizeof(type) == 8) { \
unsigned long long val8; \
str = PTR_ALIGN(str, sizeof(u32)); \
val8 = va_arg(args, unsigned long long); \
if (str + sizeof(type) <= end) { \
*(u32 *)str = *(u32 *)&val8; \
*(u32 *)(str + 4) = *((u32 *)&val8 + 1); \
} \
value = val8; \
} else { \
unsigned int val4; \
str = PTR_ALIGN(str, sizeof(type)); \
val4 = va_arg(args, int); \
if (str + sizeof(type) <= end) \
*(typeof(type) *)str = (type)(long)val4; \
value = (unsigned long long)val4; \
} \
str += sizeof(type); \
value; \
})
while (*fmt) {
int read = format_decode(fmt, &spec);
fmt += read;
switch (spec.type) {
case FORMAT_TYPE_NONE:
case FORMAT_TYPE_PERCENT_CHAR:
break;
case FORMAT_TYPE_INVALID:
goto out;
case FORMAT_TYPE_WIDTH:
case FORMAT_TYPE_PRECISION:
width = (int)save_arg(int);
/* Pointers may require the width */
if (*fmt == 'p')
set_field_width(&spec, width);
break;
case FORMAT_TYPE_CHAR:
save_arg(char);
break;
case FORMAT_TYPE_STR: {
const char *save_str = va_arg(args, char *);
const char *err_msg;
size_t len;
err_msg = check_pointer_msg(save_str);
if (err_msg)
save_str = err_msg;
len = strlen(save_str) + 1;
if (str + len < end)
memcpy(str, save_str, len);
str += len;
break;
}
case FORMAT_TYPE_PTR:
/* Dereferenced pointers must be done now */
switch (*fmt) {
/* Dereference of functions is still OK */
case 'S':
case 's':
case 'x':
case 'K':
case 'e':
save_arg(void *);
break;
default:
if (!isalnum(*fmt)) {
save_arg(void *);
break;
}
str = pointer(fmt, str, end, va_arg(args, void *),
spec);
if (str + 1 < end)
*str++ = '\0';
else
end[-1] = '\0'; /* Must be nul terminated */
}
/* skip all alphanumeric pointer suffixes */
while (isalnum(*fmt))
fmt++;
break;
default:
switch (spec.type) {
case FORMAT_TYPE_LONG_LONG:
save_arg(long long);
break;
case FORMAT_TYPE_ULONG:
case FORMAT_TYPE_LONG:
save_arg(unsigned long);
break;
case FORMAT_TYPE_SIZE_T:
save_arg(size_t);
break;
case FORMAT_TYPE_PTRDIFF:
save_arg(ptrdiff_t);
break;
case FORMAT_TYPE_UBYTE:
case FORMAT_TYPE_BYTE:
save_arg(char);
break;
case FORMAT_TYPE_USHORT:
case FORMAT_TYPE_SHORT:
save_arg(short);
break;
default:
save_arg(int);
}
}
}
out:
return (u32 *)(PTR_ALIGN(str, sizeof(u32))) - bin_buf;
#undef save_arg
}
EXPORT_SYMBOL_GPL(vbin_printf);
/**
* bstr_printf - Format a string from binary arguments and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @bin_buf: Binary arguments for the format string
*
* This function like C99 vsnprintf, but the difference is that vsnprintf gets
* arguments from stack, and bstr_printf gets arguments from @bin_buf which is
* a binary buffer that generated by vbin_printf.
*
* The format follows C99 vsnprintf, but has some extensions:
* see vsnprintf comment for details.
*
* The return value is the number of characters which would
* be generated for the given input, excluding the trailing
* '\0', as per ISO C99. If you want to have the exact
* number of characters written into @buf as return value
* (not including the trailing '\0'), use vscnprintf(). If the
* return is greater than or equal to @size, the resulting
* string is truncated.
*/
int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf)
{
struct printf_spec spec = {0};
char *str, *end;
const char *args = (const char *)bin_buf;
if (WARN_ON_ONCE(size > INT_MAX))
return 0;
str = buf;
end = buf + size;
#define get_arg(type) \
({ \
typeof(type) value; \
if (sizeof(type) == 8) { \
args = PTR_ALIGN(args, sizeof(u32)); \
*(u32 *)&value = *(u32 *)args; \
*((u32 *)&value + 1) = *(u32 *)(args + 4); \
} else { \
args = PTR_ALIGN(args, sizeof(type)); \
value = *(typeof(type) *)args; \
} \
args += sizeof(type); \
value; \
})
/* Make sure end is always >= buf */
if (end < buf) {
end = ((void *)-1);
size = end - buf;
}
while (*fmt) {
const char *old_fmt = fmt;
int read = format_decode(fmt, &spec);
fmt += read;
switch (spec.type) {
case FORMAT_TYPE_NONE: {
int copy = read;
if (str < end) {
if (copy > end - str)
copy = end - str;
memcpy(str, old_fmt, copy);
}
str += read;
break;
}
case FORMAT_TYPE_WIDTH:
set_field_width(&spec, get_arg(int));
break;
case FORMAT_TYPE_PRECISION:
set_precision(&spec, get_arg(int));
break;
case FORMAT_TYPE_CHAR: {
char c;
if (!(spec.flags & LEFT)) {
while (--spec.field_width > 0) {
if (str < end)
*str = ' ';
++str;
}
}
c = (unsigned char) get_arg(char);
if (str < end)
*str = c;
++str;
while (--spec.field_width > 0) {
if (str < end)
*str = ' ';
++str;
}
break;
}
case FORMAT_TYPE_STR: {
const char *str_arg = args;
args += strlen(str_arg) + 1;
str = string(str, end, (char *)str_arg, spec);
break;
}
case FORMAT_TYPE_PTR: {
bool process = false;
int copy, len;
/* Non function dereferences were already done */
switch (*fmt) {
case 'S':
case 's':
case 'x':
case 'K':
case 'e':
process = true;
break;
default:
if (!isalnum(*fmt)) {
process = true;
break;
}
/* Pointer dereference was already processed */
if (str < end) {
len = copy = strlen(args);
if (copy > end - str)
copy = end - str;
memcpy(str, args, copy);
str += len;
args += len + 1;
}
}
if (process)
str = pointer(fmt, str, end, get_arg(void *), spec);
while (isalnum(*fmt))
fmt++;
break;
}
case FORMAT_TYPE_PERCENT_CHAR:
if (str < end)
*str = '%';
++str;
break;
case FORMAT_TYPE_INVALID:
goto out;
default: {
unsigned long long num;
switch (spec.type) {
case FORMAT_TYPE_LONG_LONG:
num = get_arg(long long);
break;
case FORMAT_TYPE_ULONG:
case FORMAT_TYPE_LONG:
num = get_arg(unsigned long);
break;
case FORMAT_TYPE_SIZE_T:
num = get_arg(size_t);
break;
case FORMAT_TYPE_PTRDIFF:
num = get_arg(ptrdiff_t);
break;
case FORMAT_TYPE_UBYTE:
num = get_arg(unsigned char);
break;
case FORMAT_TYPE_BYTE:
num = get_arg(signed char);
break;
case FORMAT_TYPE_USHORT:
num = get_arg(unsigned short);
break;
case FORMAT_TYPE_SHORT:
num = get_arg(short);
break;
case FORMAT_TYPE_UINT:
num = get_arg(unsigned int);
break;
default:
num = get_arg(int);
}
str = number(str, end, num, spec);
} /* default: */
} /* switch(spec.type) */
} /* while(*fmt) */
out:
if (size > 0) {
if (str < end)
*str = '\0';
else
end[-1] = '\0';
}
#undef get_arg
/* the trailing null byte doesn't count towards the total */
return str - buf;
}
EXPORT_SYMBOL_GPL(bstr_printf);
/**
* bprintf - Parse a format string and place args' binary value in a buffer
* @bin_buf: The buffer to place args' binary value
* @size: The size of the buffer(by words(32bits), not characters)
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The function returns the number of words(u32) written
* into @bin_buf.
*/
int bprintf(u32 *bin_buf, size_t size, const char *fmt, ...)
{
va_list args;
int ret;
va_start(args, fmt);
ret = vbin_printf(bin_buf, size, fmt, args);
va_end(args);
return ret;
}
EXPORT_SYMBOL_GPL(bprintf);
#endif /* CONFIG_BINARY_PRINTF */
/**
* vsscanf - Unformat a buffer into a list of arguments
* @buf: input buffer
* @fmt: format of buffer
* @args: arguments
*/
int vsscanf(const char *buf, const char *fmt, va_list args)
{
const char *str = buf;
char *next;
char digit;
int num = 0;
u8 qualifier;
unsigned int base;
union {
long long s;
unsigned long long u;
} val;
s16 field_width;
bool is_sign;
while (*fmt) {
/* skip any white space in format */
/* white space in format matches any amount of
* white space, including none, in the input.
*/
if (isspace(*fmt)) { fmt = skip_spaces(++fmt);
str = skip_spaces(str);
}
/* anything that is not a conversion must match exactly */
if (*fmt != '%' && *fmt) { if (*fmt++ != *str++)
break;
continue;
}
if (!*fmt)
break;
++fmt;
/* skip this conversion.
* advance both strings to next white space
*/
if (*fmt == '*') {
if (!*str)
break;
while (!isspace(*fmt) && *fmt != '%' && *fmt) {
/* '%*[' not yet supported, invalid format */
if (*fmt == '[')
return num;
fmt++;
}
while (!isspace(*str) && *str) str++;
continue;
}
/* get field width */
field_width = -1;
if (isdigit(*fmt)) { field_width = skip_atoi(&fmt);
if (field_width <= 0)
break;
}
/* get conversion qualifier */
qualifier = -1;
if (*fmt == 'h' || _tolower(*fmt) == 'l' ||
*fmt == 'z') {
qualifier = *fmt++;
if (unlikely(qualifier == *fmt)) {
if (qualifier == 'h') {
qualifier = 'H';
fmt++; } else if (qualifier == 'l') {
qualifier = 'L';
fmt++;
}
}
}
if (!*fmt)
break;
if (*fmt == 'n') {
/* return number of characters read so far */
*va_arg(args, int *) = str - buf;
++fmt;
continue;
}
if (!*str)
break;
base = 10;
is_sign = false;
switch (*fmt++) {
case 'c':
{
char *s = (char *)va_arg(args, char*);
if (field_width == -1)
field_width = 1;
do {
*s++ = *str++; } while (--field_width > 0 && *str); num++;
}
continue;
case 's':
{
char *s = (char *)va_arg(args, char *);
if (field_width == -1)
field_width = SHRT_MAX;
/* first, skip leading white space in buffer */
str = skip_spaces(str);
/* now copy until next white space */
while (*str && !isspace(*str) && field_width--) *s++ = *str++; *s = '\0';
num++;
}
continue;
/*
* Warning: This implementation of the '[' conversion specifier
* deviates from its glibc counterpart in the following ways:
* (1) It does NOT support ranges i.e. '-' is NOT a special
* character
* (2) It cannot match the closing bracket ']' itself
* (3) A field width is required
* (4) '%*[' (discard matching input) is currently not supported
*
* Example usage:
* ret = sscanf("00:0a:95","%2[^:]:%2[^:]:%2[^:]",
* buf1, buf2, buf3);
* if (ret < 3)
* // etc..
*/
case '[':
{
char *s = (char *)va_arg(args, char *);
DECLARE_BITMAP(set, 256) = {0};
unsigned int len = 0;
bool negate = (*fmt == '^');
/* field width is required */
if (field_width == -1)
return num; if (negate) ++fmt; for ( ; *fmt && *fmt != ']'; ++fmt, ++len) __set_bit((u8)*fmt, set);
/* no ']' or no character set found */
if (!*fmt || !len)
return num;
++fmt;
if (negate) {
bitmap_complement(set, set, 256);
/* exclude null '\0' byte */
__clear_bit(0, set);
}
/* match must be non-empty */
if (!test_bit((u8)*str, set))
return num;
while (test_bit((u8)*str, set) && field_width--) *s++ = *str++; *s = '\0';
++num;
}
continue;
case 'o':
base = 8;
break;
case 'x':
case 'X':
base = 16;
break;
case 'i':
base = 0;
fallthrough;
case 'd':
is_sign = true;
fallthrough;
case 'u':
break;
case '%':
/* looking for '%' in str */
if (*str++ != '%')
return num;
continue;
default:
/* invalid format; stop here */
return num;
}
/* have some sort of integer conversion.
* first, skip white space in buffer.
*/
str = skip_spaces(str);
digit = *str;
if (is_sign && digit == '-') { if (field_width == 1)
break;
digit = *(str + 1);
}
if (!digit || (base == 16 && !isxdigit(digit)) || (base == 10 && !isdigit(digit)) || (base == 8 && !isodigit(digit)) || (base == 0 && !isdigit(digit)))
break;
if (is_sign) val.s = simple_strntoll(str, &next, base,
field_width >= 0 ? field_width : INT_MAX);
else
val.u = simple_strntoull(str, &next, base,
field_width >= 0 ? field_width : INT_MAX);
switch (qualifier) {
case 'H': /* that's 'hh' in format */
if (is_sign)
*va_arg(args, signed char *) = val.s;
else
*va_arg(args, unsigned char *) = val.u;
break;
case 'h':
if (is_sign)
*va_arg(args, short *) = val.s;
else
*va_arg(args, unsigned short *) = val.u;
break;
case 'l':
if (is_sign)
*va_arg(args, long *) = val.s;
else
*va_arg(args, unsigned long *) = val.u;
break;
case 'L':
if (is_sign)
*va_arg(args, long long *) = val.s;
else
*va_arg(args, unsigned long long *) = val.u;
break;
case 'z':
*va_arg(args, size_t *) = val.u;
break;
default:
if (is_sign)
*va_arg(args, int *) = val.s;
else
*va_arg(args, unsigned int *) = val.u;
break;
}
num++;
if (!next)
break;
str = next;
}
return num;
}
EXPORT_SYMBOL(vsscanf);
/**
* sscanf - Unformat a buffer into a list of arguments
* @buf: input buffer
* @fmt: formatting of buffer
* @...: resulting arguments
*/
int sscanf(const char *buf, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vsscanf(buf, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(sscanf);
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2008 IBM Corporation
*
* Authors:
* Mimi Zohar <zohar@us.ibm.com>
*
* File: ima_iint.c
* - implements the IMA hook: ima_inode_free
* - cache integrity information in the inode security blob
*/
#include <linux/slab.h>
#include "ima.h"
static struct kmem_cache *ima_iint_cache __ro_after_init;
/**
* ima_iint_find - Return the iint associated with an inode
* @inode: Pointer to the inode
*
* Return the IMA integrity information (iint) associated with an inode, if the
* inode was processed by IMA.
*
* Return: Found iint or NULL.
*/
struct ima_iint_cache *ima_iint_find(struct inode *inode)
{
if (!IS_IMA(inode))
return NULL;
return ima_inode_get_iint(inode);}
#define IMA_MAX_NESTING (FILESYSTEM_MAX_STACK_DEPTH + 1)
/*
* It is not clear that IMA should be nested at all, but as long is it measures
* files both on overlayfs and on underlying fs, we need to annotate the iint
* mutex to avoid lockdep false positives related to IMA + overlayfs.
* See ovl_lockdep_annotate_inode_mutex_key() for more details.
*/
static inline void ima_iint_lockdep_annotate(struct ima_iint_cache *iint,
struct inode *inode)
{
#ifdef CONFIG_LOCKDEP
static struct lock_class_key ima_iint_mutex_key[IMA_MAX_NESTING];
int depth = inode->i_sb->s_stack_depth;
if (WARN_ON_ONCE(depth < 0 || depth >= IMA_MAX_NESTING))
depth = 0;
lockdep_set_class(&iint->mutex, &ima_iint_mutex_key[depth]);
#endif
}
static void ima_iint_init_always(struct ima_iint_cache *iint,
struct inode *inode)
{
iint->ima_hash = NULL;
iint->version = 0;
iint->flags = 0UL;
iint->atomic_flags = 0UL;
iint->ima_file_status = INTEGRITY_UNKNOWN;
iint->ima_mmap_status = INTEGRITY_UNKNOWN;
iint->ima_bprm_status = INTEGRITY_UNKNOWN;
iint->ima_read_status = INTEGRITY_UNKNOWN;
iint->ima_creds_status = INTEGRITY_UNKNOWN;
iint->measured_pcrs = 0;
mutex_init(&iint->mutex);
ima_iint_lockdep_annotate(iint, inode);
}
static void ima_iint_free(struct ima_iint_cache *iint)
{
kfree(iint->ima_hash);
mutex_destroy(&iint->mutex);
kmem_cache_free(ima_iint_cache, iint);
}
/**
* ima_inode_get - Find or allocate an iint associated with an inode
* @inode: Pointer to the inode
*
* Find an iint associated with an inode, and allocate a new one if not found.
* Caller must lock i_mutex.
*
* Return: An iint on success, NULL on error.
*/
struct ima_iint_cache *ima_inode_get(struct inode *inode)
{
struct ima_iint_cache *iint;
iint = ima_iint_find(inode);
if (iint)
return iint;
iint = kmem_cache_alloc(ima_iint_cache, GFP_NOFS);
if (!iint)
return NULL;
ima_iint_init_always(iint, inode);
inode->i_flags |= S_IMA;
ima_inode_set_iint(inode, iint);
return iint;
}
/**
* ima_inode_free - Called on inode free
* @inode: Pointer to the inode
*
* Free the iint associated with an inode.
*/
void ima_inode_free(struct inode *inode)
{
struct ima_iint_cache *iint;
if (!IS_IMA(inode))
return;
iint = ima_iint_find(inode);
ima_inode_set_iint(inode, NULL);
ima_iint_free(iint);
}
static void ima_iint_init_once(void *foo)
{
struct ima_iint_cache *iint = (struct ima_iint_cache *)foo;
memset(iint, 0, sizeof(*iint));
}
void __init ima_iintcache_init(void)
{
ima_iint_cache =
kmem_cache_create("ima_iint_cache", sizeof(struct ima_iint_cache),
0, SLAB_PANIC, ima_iint_init_once);
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/fs/locks.c
*
* We implement four types of file locks: BSD locks, posix locks, open
* file description locks, and leases. For details about BSD locks,
* see the flock(2) man page; for details about the other three, see
* fcntl(2).
*
*
* Locking conflicts and dependencies:
* If multiple threads attempt to lock the same byte (or flock the same file)
* only one can be granted the lock, and other must wait their turn.
* The first lock has been "applied" or "granted", the others are "waiting"
* and are "blocked" by the "applied" lock..
*
* Waiting and applied locks are all kept in trees whose properties are:
*
* - the root of a tree may be an applied or waiting lock.
* - every other node in the tree is a waiting lock that
* conflicts with every ancestor of that node.
*
* Every such tree begins life as a waiting singleton which obviously
* satisfies the above properties.
*
* The only ways we modify trees preserve these properties:
*
* 1. We may add a new leaf node, but only after first verifying that it
* conflicts with all of its ancestors.
* 2. We may remove the root of a tree, creating a new singleton
* tree from the root and N new trees rooted in the immediate
* children.
* 3. If the root of a tree is not currently an applied lock, we may
* apply it (if possible).
* 4. We may upgrade the root of the tree (either extend its range,
* or upgrade its entire range from read to write).
*
* When an applied lock is modified in a way that reduces or downgrades any
* part of its range, we remove all its children (2 above). This particularly
* happens when a lock is unlocked.
*
* For each of those child trees we "wake up" the thread which is
* waiting for the lock so it can continue handling as follows: if the
* root of the tree applies, we do so (3). If it doesn't, it must
* conflict with some applied lock. We remove (wake up) all of its children
* (2), and add it is a new leaf to the tree rooted in the applied
* lock (1). We then repeat the process recursively with those
* children.
*
*/
#include <linux/capability.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/filelock.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/security.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/time.h>
#include <linux/rcupdate.h>
#include <linux/pid_namespace.h>
#include <linux/hashtable.h>
#include <linux/percpu.h>
#include <linux/sysctl.h>
#define CREATE_TRACE_POINTS
#include <trace/events/filelock.h>
#include <linux/uaccess.h>
static struct file_lock *file_lock(struct file_lock_core *flc)
{
return container_of(flc, struct file_lock, c);
}
static struct file_lease *file_lease(struct file_lock_core *flc)
{
return container_of(flc, struct file_lease, c);
}
static bool lease_breaking(struct file_lease *fl)
{
return fl->c.flc_flags & (FL_UNLOCK_PENDING | FL_DOWNGRADE_PENDING);
}
static int target_leasetype(struct file_lease *fl)
{
if (fl->c.flc_flags & FL_UNLOCK_PENDING)
return F_UNLCK;
if (fl->c.flc_flags & FL_DOWNGRADE_PENDING)
return F_RDLCK;
return fl->c.flc_type;
}
static int leases_enable = 1;
static int lease_break_time = 45;
#ifdef CONFIG_SYSCTL
static struct ctl_table locks_sysctls[] = {
{
.procname = "leases-enable",
.data = &leases_enable,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
#ifdef CONFIG_MMU
{
.procname = "lease-break-time",
.data = &lease_break_time,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
#endif /* CONFIG_MMU */
};
static int __init init_fs_locks_sysctls(void)
{
register_sysctl_init("fs", locks_sysctls);
return 0;
}
early_initcall(init_fs_locks_sysctls);
#endif /* CONFIG_SYSCTL */
/*
* The global file_lock_list is only used for displaying /proc/locks, so we
* keep a list on each CPU, with each list protected by its own spinlock.
* Global serialization is done using file_rwsem.
*
* Note that alterations to the list also require that the relevant flc_lock is
* held.
*/
struct file_lock_list_struct {
spinlock_t lock;
struct hlist_head hlist;
};
static DEFINE_PER_CPU(struct file_lock_list_struct, file_lock_list);
DEFINE_STATIC_PERCPU_RWSEM(file_rwsem);
/*
* The blocked_hash is used to find POSIX lock loops for deadlock detection.
* It is protected by blocked_lock_lock.
*
* We hash locks by lockowner in order to optimize searching for the lock a
* particular lockowner is waiting on.
*
* FIXME: make this value scale via some heuristic? We generally will want more
* buckets when we have more lockowners holding locks, but that's a little
* difficult to determine without knowing what the workload will look like.
*/
#define BLOCKED_HASH_BITS 7
static DEFINE_HASHTABLE(blocked_hash, BLOCKED_HASH_BITS);
/*
* This lock protects the blocked_hash. Generally, if you're accessing it, you
* want to be holding this lock.
*
* In addition, it also protects the fl->fl_blocked_requests list, and the
* fl->fl_blocker pointer for file_lock structures that are acting as lock
* requests (in contrast to those that are acting as records of acquired locks).
*
* Note that when we acquire this lock in order to change the above fields,
* we often hold the flc_lock as well. In certain cases, when reading the fields
* protected by this lock, we can skip acquiring it iff we already hold the
* flc_lock.
*/
static DEFINE_SPINLOCK(blocked_lock_lock);
static struct kmem_cache *flctx_cache __ro_after_init;
static struct kmem_cache *filelock_cache __ro_after_init;
static struct kmem_cache *filelease_cache __ro_after_init;
static struct file_lock_context *
locks_get_lock_context(struct inode *inode, int type)
{
struct file_lock_context *ctx;
/* paired with cmpxchg() below */
ctx = locks_inode_context(inode);
if (likely(ctx) || type == F_UNLCK)
goto out;
ctx = kmem_cache_alloc(flctx_cache, GFP_KERNEL);
if (!ctx)
goto out;
spin_lock_init(&ctx->flc_lock);
INIT_LIST_HEAD(&ctx->flc_flock);
INIT_LIST_HEAD(&ctx->flc_posix);
INIT_LIST_HEAD(&ctx->flc_lease);
/*
* Assign the pointer if it's not already assigned. If it is, then
* free the context we just allocated.
*/
if (cmpxchg(&inode->i_flctx, NULL, ctx)) {
kmem_cache_free(flctx_cache, ctx);
ctx = locks_inode_context(inode);
}
out:
trace_locks_get_lock_context(inode, type, ctx);
return ctx;
}
static void
locks_dump_ctx_list(struct list_head *list, char *list_type)
{
struct file_lock_core *flc;
list_for_each_entry(flc, list, flc_list) pr_warn("%s: fl_owner=%p fl_flags=0x%x fl_type=0x%x fl_pid=%u\n",
list_type, flc->flc_owner, flc->flc_flags,
flc->flc_type, flc->flc_pid);
}
static void
locks_check_ctx_lists(struct inode *inode)
{
struct file_lock_context *ctx = inode->i_flctx; if (unlikely(!list_empty(&ctx->flc_flock) ||
!list_empty(&ctx->flc_posix) ||
!list_empty(&ctx->flc_lease))) {
pr_warn("Leaked locks on dev=0x%x:0x%x ino=0x%lx:\n",
MAJOR(inode->i_sb->s_dev), MINOR(inode->i_sb->s_dev),
inode->i_ino);
locks_dump_ctx_list(&ctx->flc_flock, "FLOCK"); locks_dump_ctx_list(&ctx->flc_posix, "POSIX"); locks_dump_ctx_list(&ctx->flc_lease, "LEASE");
}
}
static void
locks_check_ctx_file_list(struct file *filp, struct list_head *list, char *list_type)
{
struct file_lock_core *flc;
struct inode *inode = file_inode(filp);
list_for_each_entry(flc, list, flc_list)
if (flc->flc_file == filp)
pr_warn("Leaked %s lock on dev=0x%x:0x%x ino=0x%lx "
" fl_owner=%p fl_flags=0x%x fl_type=0x%x fl_pid=%u\n",
list_type, MAJOR(inode->i_sb->s_dev),
MINOR(inode->i_sb->s_dev), inode->i_ino,
flc->flc_owner, flc->flc_flags,
flc->flc_type, flc->flc_pid);
}
void
locks_free_lock_context(struct inode *inode)
{
struct file_lock_context *ctx = locks_inode_context(inode);
if (unlikely(ctx)) {
locks_check_ctx_lists(inode); kmem_cache_free(flctx_cache, ctx);
}
}
static void locks_init_lock_heads(struct file_lock_core *flc)
{
INIT_HLIST_NODE(&flc->flc_link);
INIT_LIST_HEAD(&flc->flc_list);
INIT_LIST_HEAD(&flc->flc_blocked_requests);
INIT_LIST_HEAD(&flc->flc_blocked_member);
init_waitqueue_head(&flc->flc_wait);
}
/* Allocate an empty lock structure. */
struct file_lock *locks_alloc_lock(void)
{
struct file_lock *fl = kmem_cache_zalloc(filelock_cache, GFP_KERNEL);
if (fl)
locks_init_lock_heads(&fl->c);
return fl;
}
EXPORT_SYMBOL_GPL(locks_alloc_lock);
/* Allocate an empty lock structure. */
struct file_lease *locks_alloc_lease(void)
{
struct file_lease *fl = kmem_cache_zalloc(filelease_cache, GFP_KERNEL);
if (fl)
locks_init_lock_heads(&fl->c);
return fl;
}
EXPORT_SYMBOL_GPL(locks_alloc_lease);
void locks_release_private(struct file_lock *fl)
{
struct file_lock_core *flc = &fl->c;
BUG_ON(waitqueue_active(&flc->flc_wait));
BUG_ON(!list_empty(&flc->flc_list));
BUG_ON(!list_empty(&flc->flc_blocked_requests));
BUG_ON(!list_empty(&flc->flc_blocked_member));
BUG_ON(!hlist_unhashed(&flc->flc_link));
if (fl->fl_ops) {
if (fl->fl_ops->fl_release_private)
fl->fl_ops->fl_release_private(fl);
fl->fl_ops = NULL;
}
if (fl->fl_lmops) {
if (fl->fl_lmops->lm_put_owner) {
fl->fl_lmops->lm_put_owner(flc->flc_owner);
flc->flc_owner = NULL;
}
fl->fl_lmops = NULL;
}
}
EXPORT_SYMBOL_GPL(locks_release_private);
/**
* locks_owner_has_blockers - Check for blocking lock requests
* @flctx: file lock context
* @owner: lock owner
*
* Return values:
* %true: @owner has at least one blocker
* %false: @owner has no blockers
*/
bool locks_owner_has_blockers(struct file_lock_context *flctx, fl_owner_t owner)
{
struct file_lock_core *flc;
spin_lock(&flctx->flc_lock);
list_for_each_entry(flc, &flctx->flc_posix, flc_list) {
if (flc->flc_owner != owner)
continue;
if (!list_empty(&flc->flc_blocked_requests)) {
spin_unlock(&flctx->flc_lock);
return true;
}
}
spin_unlock(&flctx->flc_lock);
return false;
}
EXPORT_SYMBOL_GPL(locks_owner_has_blockers);
/* Free a lock which is not in use. */
void locks_free_lock(struct file_lock *fl)
{
locks_release_private(fl);
kmem_cache_free(filelock_cache, fl);
}
EXPORT_SYMBOL(locks_free_lock);
/* Free a lease which is not in use. */
void locks_free_lease(struct file_lease *fl)
{
kmem_cache_free(filelease_cache, fl);
}
EXPORT_SYMBOL(locks_free_lease);
static void
locks_dispose_list(struct list_head *dispose)
{
struct file_lock_core *flc;
while (!list_empty(dispose)) {
flc = list_first_entry(dispose, struct file_lock_core, flc_list);
list_del_init(&flc->flc_list);
if (flc->flc_flags & (FL_LEASE|FL_DELEG|FL_LAYOUT))
locks_free_lease(file_lease(flc));
else
locks_free_lock(file_lock(flc));
}
}
void locks_init_lock(struct file_lock *fl)
{
memset(fl, 0, sizeof(struct file_lock));
locks_init_lock_heads(&fl->c);
}
EXPORT_SYMBOL(locks_init_lock);
void locks_init_lease(struct file_lease *fl)
{
memset(fl, 0, sizeof(*fl));
locks_init_lock_heads(&fl->c);
}
EXPORT_SYMBOL(locks_init_lease);
/*
* Initialize a new lock from an existing file_lock structure.
*/
void locks_copy_conflock(struct file_lock *new, struct file_lock *fl)
{
new->c.flc_owner = fl->c.flc_owner;
new->c.flc_pid = fl->c.flc_pid;
new->c.flc_file = NULL;
new->c.flc_flags = fl->c.flc_flags;
new->c.flc_type = fl->c.flc_type;
new->fl_start = fl->fl_start;
new->fl_end = fl->fl_end;
new->fl_lmops = fl->fl_lmops;
new->fl_ops = NULL;
if (fl->fl_lmops) {
if (fl->fl_lmops->lm_get_owner)
fl->fl_lmops->lm_get_owner(fl->c.flc_owner);
}
}
EXPORT_SYMBOL(locks_copy_conflock);
void locks_copy_lock(struct file_lock *new, struct file_lock *fl)
{
/* "new" must be a freshly-initialized lock */
WARN_ON_ONCE(new->fl_ops);
locks_copy_conflock(new, fl);
new->c.flc_file = fl->c.flc_file;
new->fl_ops = fl->fl_ops;
if (fl->fl_ops) {
if (fl->fl_ops->fl_copy_lock)
fl->fl_ops->fl_copy_lock(new, fl);
}
}
EXPORT_SYMBOL(locks_copy_lock);
static void locks_move_blocks(struct file_lock *new, struct file_lock *fl)
{
struct file_lock *f;
/*
* As ctx->flc_lock is held, new requests cannot be added to
* ->flc_blocked_requests, so we don't need a lock to check if it
* is empty.
*/
if (list_empty(&fl->c.flc_blocked_requests))
return;
spin_lock(&blocked_lock_lock);
list_splice_init(&fl->c.flc_blocked_requests,
&new->c.flc_blocked_requests);
list_for_each_entry(f, &new->c.flc_blocked_requests,
c.flc_blocked_member)
f->c.flc_blocker = &new->c;
spin_unlock(&blocked_lock_lock);
}
static inline int flock_translate_cmd(int cmd) {
switch (cmd) {
case LOCK_SH:
return F_RDLCK;
case LOCK_EX:
return F_WRLCK;
case LOCK_UN:
return F_UNLCK;
}
return -EINVAL;
}
/* Fill in a file_lock structure with an appropriate FLOCK lock. */
static void flock_make_lock(struct file *filp, struct file_lock *fl, int type)
{
locks_init_lock(fl);
fl->c.flc_file = filp;
fl->c.flc_owner = filp;
fl->c.flc_pid = current->tgid;
fl->c.flc_flags = FL_FLOCK;
fl->c.flc_type = type;
fl->fl_end = OFFSET_MAX;
}
static int assign_type(struct file_lock_core *flc, int type)
{
switch (type) {
case F_RDLCK:
case F_WRLCK:
case F_UNLCK:
flc->flc_type = type;
break;
default:
return -EINVAL;
}
return 0;
}
static int flock64_to_posix_lock(struct file *filp, struct file_lock *fl,
struct flock64 *l)
{
switch (l->l_whence) {
case SEEK_SET:
fl->fl_start = 0;
break;
case SEEK_CUR:
fl->fl_start = filp->f_pos;
break;
case SEEK_END:
fl->fl_start = i_size_read(file_inode(filp));
break;
default:
return -EINVAL;
}
if (l->l_start > OFFSET_MAX - fl->fl_start)
return -EOVERFLOW;
fl->fl_start += l->l_start;
if (fl->fl_start < 0)
return -EINVAL;
/* POSIX-1996 leaves the case l->l_len < 0 undefined;
POSIX-2001 defines it. */
if (l->l_len > 0) {
if (l->l_len - 1 > OFFSET_MAX - fl->fl_start)
return -EOVERFLOW;
fl->fl_end = fl->fl_start + (l->l_len - 1);
} else if (l->l_len < 0) {
if (fl->fl_start + l->l_len < 0)
return -EINVAL;
fl->fl_end = fl->fl_start - 1;
fl->fl_start += l->l_len;
} else
fl->fl_end = OFFSET_MAX;
fl->c.flc_owner = current->files;
fl->c.flc_pid = current->tgid;
fl->c.flc_file = filp;
fl->c.flc_flags = FL_POSIX;
fl->fl_ops = NULL;
fl->fl_lmops = NULL;
return assign_type(&fl->c, l->l_type);
}
/* Verify a "struct flock" and copy it to a "struct file_lock" as a POSIX
* style lock.
*/
static int flock_to_posix_lock(struct file *filp, struct file_lock *fl,
struct flock *l)
{
struct flock64 ll = {
.l_type = l->l_type,
.l_whence = l->l_whence,
.l_start = l->l_start,
.l_len = l->l_len,
};
return flock64_to_posix_lock(filp, fl, &ll);
}
/* default lease lock manager operations */
static bool
lease_break_callback(struct file_lease *fl)
{
kill_fasync(&fl->fl_fasync, SIGIO, POLL_MSG);
return false;
}
static void
lease_setup(struct file_lease *fl, void **priv)
{
struct file *filp = fl->c.flc_file;
struct fasync_struct *fa = *priv;
/*
* fasync_insert_entry() returns the old entry if any. If there was no
* old entry, then it used "priv" and inserted it into the fasync list.
* Clear the pointer to indicate that it shouldn't be freed.
*/
if (!fasync_insert_entry(fa->fa_fd, filp, &fl->fl_fasync, fa))
*priv = NULL;
__f_setown(filp, task_pid(current), PIDTYPE_TGID, 0);
}
static const struct lease_manager_operations lease_manager_ops = {
.lm_break = lease_break_callback,
.lm_change = lease_modify,
.lm_setup = lease_setup,
};
/*
* Initialize a lease, use the default lock manager operations
*/
static int lease_init(struct file *filp, int type, struct file_lease *fl)
{
if (assign_type(&fl->c, type) != 0)
return -EINVAL;
fl->c.flc_owner = filp;
fl->c.flc_pid = current->tgid;
fl->c.flc_file = filp;
fl->c.flc_flags = FL_LEASE;
fl->fl_lmops = &lease_manager_ops;
return 0;
}
/* Allocate a file_lock initialised to this type of lease */
static struct file_lease *lease_alloc(struct file *filp, int type)
{
struct file_lease *fl = locks_alloc_lease();
int error = -ENOMEM;
if (fl == NULL)
return ERR_PTR(error);
error = lease_init(filp, type, fl);
if (error) {
locks_free_lease(fl);
return ERR_PTR(error);
}
return fl;
}
/* Check if two locks overlap each other.
*/
static inline int locks_overlap(struct file_lock *fl1, struct file_lock *fl2)
{
return ((fl1->fl_end >= fl2->fl_start) &&
(fl2->fl_end >= fl1->fl_start));
}
/*
* Check whether two locks have the same owner.
*/
static int posix_same_owner(struct file_lock_core *fl1, struct file_lock_core *fl2)
{
return fl1->flc_owner == fl2->flc_owner;
}
/* Must be called with the flc_lock held! */
static void locks_insert_global_locks(struct file_lock_core *flc)
{
struct file_lock_list_struct *fll = this_cpu_ptr(&file_lock_list);
percpu_rwsem_assert_held(&file_rwsem);
spin_lock(&fll->lock);
flc->flc_link_cpu = smp_processor_id();
hlist_add_head(&flc->flc_link, &fll->hlist);
spin_unlock(&fll->lock);
}
/* Must be called with the flc_lock held! */
static void locks_delete_global_locks(struct file_lock_core *flc)
{
struct file_lock_list_struct *fll;
percpu_rwsem_assert_held(&file_rwsem);
/*
* Avoid taking lock if already unhashed. This is safe since this check
* is done while holding the flc_lock, and new insertions into the list
* also require that it be held.
*/
if (hlist_unhashed(&flc->flc_link))
return;
fll = per_cpu_ptr(&file_lock_list, flc->flc_link_cpu);
spin_lock(&fll->lock);
hlist_del_init(&flc->flc_link);
spin_unlock(&fll->lock);
}
static unsigned long
posix_owner_key(struct file_lock_core *flc)
{
return (unsigned long) flc->flc_owner;
}
static void locks_insert_global_blocked(struct file_lock_core *waiter)
{
lockdep_assert_held(&blocked_lock_lock);
hash_add(blocked_hash, &waiter->flc_link, posix_owner_key(waiter));
}
static void locks_delete_global_blocked(struct file_lock_core *waiter)
{
lockdep_assert_held(&blocked_lock_lock);
hash_del(&waiter->flc_link);
}
/* Remove waiter from blocker's block list.
* When blocker ends up pointing to itself then the list is empty.
*
* Must be called with blocked_lock_lock held.
*/
static void __locks_unlink_block(struct file_lock_core *waiter)
{
locks_delete_global_blocked(waiter);
list_del_init(&waiter->flc_blocked_member);
}
static void __locks_wake_up_blocks(struct file_lock_core *blocker)
{
while (!list_empty(&blocker->flc_blocked_requests)) {
struct file_lock_core *waiter;
struct file_lock *fl;
waiter = list_first_entry(&blocker->flc_blocked_requests,
struct file_lock_core, flc_blocked_member);
fl = file_lock(waiter);
__locks_unlink_block(waiter);
if ((waiter->flc_flags & (FL_POSIX | FL_FLOCK)) &&
fl->fl_lmops && fl->fl_lmops->lm_notify)
fl->fl_lmops->lm_notify(fl);
else
locks_wake_up(fl);
/*
* The setting of flc_blocker to NULL marks the "done"
* point in deleting a block. Paired with acquire at the top
* of locks_delete_block().
*/
smp_store_release(&waiter->flc_blocker, NULL);
}
}
static int __locks_delete_block(struct file_lock_core *waiter)
{
int status = -ENOENT;
/*
* If fl_blocker is NULL, it won't be set again as this thread "owns"
* the lock and is the only one that might try to claim the lock.
*
* We use acquire/release to manage fl_blocker so that we can
* optimize away taking the blocked_lock_lock in many cases.
*
* The smp_load_acquire guarantees two things:
*
* 1/ that fl_blocked_requests can be tested locklessly. If something
* was recently added to that list it must have been in a locked region
* *before* the locked region when fl_blocker was set to NULL.
*
* 2/ that no other thread is accessing 'waiter', so it is safe to free
* it. __locks_wake_up_blocks is careful not to touch waiter after
* fl_blocker is released.
*
* If a lockless check of fl_blocker shows it to be NULL, we know that
* no new locks can be inserted into its fl_blocked_requests list, and
* can avoid doing anything further if the list is empty.
*/
if (!smp_load_acquire(&waiter->flc_blocker) &&
list_empty(&waiter->flc_blocked_requests))
return status;
spin_lock(&blocked_lock_lock);
if (waiter->flc_blocker)
status = 0;
__locks_wake_up_blocks(waiter);
__locks_unlink_block(waiter);
/*
* The setting of fl_blocker to NULL marks the "done" point in deleting
* a block. Paired with acquire at the top of this function.
*/
smp_store_release(&waiter->flc_blocker, NULL);
spin_unlock(&blocked_lock_lock);
return status;
}
/**
* locks_delete_block - stop waiting for a file lock
* @waiter: the lock which was waiting
*
* lockd/nfsd need to disconnect the lock while working on it.
*/
int locks_delete_block(struct file_lock *waiter)
{
return __locks_delete_block(&waiter->c);
}
EXPORT_SYMBOL(locks_delete_block);
/* Insert waiter into blocker's block list.
* We use a circular list so that processes can be easily woken up in
* the order they blocked. The documentation doesn't require this but
* it seems like the reasonable thing to do.
*
* Must be called with both the flc_lock and blocked_lock_lock held. The
* fl_blocked_requests list itself is protected by the blocked_lock_lock,
* but by ensuring that the flc_lock is also held on insertions we can avoid
* taking the blocked_lock_lock in some cases when we see that the
* fl_blocked_requests list is empty.
*
* Rather than just adding to the list, we check for conflicts with any existing
* waiters, and add beneath any waiter that blocks the new waiter.
* Thus wakeups don't happen until needed.
*/
static void __locks_insert_block(struct file_lock_core *blocker,
struct file_lock_core *waiter,
bool conflict(struct file_lock_core *,
struct file_lock_core *))
{
struct file_lock_core *flc;
BUG_ON(!list_empty(&waiter->flc_blocked_member));
new_blocker:
list_for_each_entry(flc, &blocker->flc_blocked_requests, flc_blocked_member)
if (conflict(flc, waiter)) {
blocker = flc;
goto new_blocker;
}
waiter->flc_blocker = blocker;
list_add_tail(&waiter->flc_blocked_member,
&blocker->flc_blocked_requests);
if ((blocker->flc_flags & (FL_POSIX|FL_OFDLCK)) == FL_POSIX)
locks_insert_global_blocked(waiter);
/* The requests in waiter->flc_blocked are known to conflict with
* waiter, but might not conflict with blocker, or the requests
* and lock which block it. So they all need to be woken.
*/
__locks_wake_up_blocks(waiter);
}
/* Must be called with flc_lock held. */
static void locks_insert_block(struct file_lock_core *blocker,
struct file_lock_core *waiter,
bool conflict(struct file_lock_core *,
struct file_lock_core *))
{
spin_lock(&blocked_lock_lock);
__locks_insert_block(blocker, waiter, conflict);
spin_unlock(&blocked_lock_lock);
}
/*
* Wake up processes blocked waiting for blocker.
*
* Must be called with the inode->flc_lock held!
*/
static void locks_wake_up_blocks(struct file_lock_core *blocker)
{
/*
* Avoid taking global lock if list is empty. This is safe since new
* blocked requests are only added to the list under the flc_lock, and
* the flc_lock is always held here. Note that removal from the
* fl_blocked_requests list does not require the flc_lock, so we must
* recheck list_empty() after acquiring the blocked_lock_lock.
*/
if (list_empty(&blocker->flc_blocked_requests))
return;
spin_lock(&blocked_lock_lock);
__locks_wake_up_blocks(blocker);
spin_unlock(&blocked_lock_lock);
}
static void
locks_insert_lock_ctx(struct file_lock_core *fl, struct list_head *before)
{
list_add_tail(&fl->flc_list, before);
locks_insert_global_locks(fl);
}
static void
locks_unlink_lock_ctx(struct file_lock_core *fl)
{
locks_delete_global_locks(fl);
list_del_init(&fl->flc_list);
locks_wake_up_blocks(fl);
}
static void
locks_delete_lock_ctx(struct file_lock_core *fl, struct list_head *dispose)
{
locks_unlink_lock_ctx(fl);
if (dispose)
list_add(&fl->flc_list, dispose);
else
locks_free_lock(file_lock(fl));
}
/* Determine if lock sys_fl blocks lock caller_fl. Common functionality
* checks for shared/exclusive status of overlapping locks.
*/
static bool locks_conflict(struct file_lock_core *caller_flc,
struct file_lock_core *sys_flc)
{
if (sys_flc->flc_type == F_WRLCK)
return true;
if (caller_flc->flc_type == F_WRLCK)
return true;
return false;
}
/* Determine if lock sys_fl blocks lock caller_fl. POSIX specific
* checking before calling the locks_conflict().
*/
static bool posix_locks_conflict(struct file_lock_core *caller_flc,
struct file_lock_core *sys_flc)
{
struct file_lock *caller_fl = file_lock(caller_flc);
struct file_lock *sys_fl = file_lock(sys_flc);
/* POSIX locks owned by the same process do not conflict with
* each other.
*/
if (posix_same_owner(caller_flc, sys_flc))
return false;
/* Check whether they overlap */
if (!locks_overlap(caller_fl, sys_fl))
return false;
return locks_conflict(caller_flc, sys_flc);
}
/* Determine if lock sys_fl blocks lock caller_fl. Used on xx_GETLK
* path so checks for additional GETLK-specific things like F_UNLCK.
*/
static bool posix_test_locks_conflict(struct file_lock *caller_fl,
struct file_lock *sys_fl)
{
struct file_lock_core *caller = &caller_fl->c;
struct file_lock_core *sys = &sys_fl->c;
/* F_UNLCK checks any locks on the same fd. */
if (lock_is_unlock(caller_fl)) {
if (!posix_same_owner(caller, sys))
return false;
return locks_overlap(caller_fl, sys_fl);
}
return posix_locks_conflict(caller, sys);
}
/* Determine if lock sys_fl blocks lock caller_fl. FLOCK specific
* checking before calling the locks_conflict().
*/
static bool flock_locks_conflict(struct file_lock_core *caller_flc,
struct file_lock_core *sys_flc)
{
/* FLOCK locks referring to the same filp do not conflict with
* each other.
*/
if (caller_flc->flc_file == sys_flc->flc_file)
return false;
return locks_conflict(caller_flc, sys_flc);
}
void
posix_test_lock(struct file *filp, struct file_lock *fl)
{
struct file_lock *cfl;
struct file_lock_context *ctx;
struct inode *inode = file_inode(filp);
void *owner;
void (*func)(void);
ctx = locks_inode_context(inode);
if (!ctx || list_empty_careful(&ctx->flc_posix)) {
fl->c.flc_type = F_UNLCK;
return;
}
retry:
spin_lock(&ctx->flc_lock);
list_for_each_entry(cfl, &ctx->flc_posix, c.flc_list) {
if (!posix_test_locks_conflict(fl, cfl))
continue;
if (cfl->fl_lmops && cfl->fl_lmops->lm_lock_expirable
&& (*cfl->fl_lmops->lm_lock_expirable)(cfl)) {
owner = cfl->fl_lmops->lm_mod_owner;
func = cfl->fl_lmops->lm_expire_lock;
__module_get(owner);
spin_unlock(&ctx->flc_lock);
(*func)();
module_put(owner);
goto retry;
}
locks_copy_conflock(fl, cfl);
goto out;
}
fl->c.flc_type = F_UNLCK;
out:
spin_unlock(&ctx->flc_lock);
return;
}
EXPORT_SYMBOL(posix_test_lock);
/*
* Deadlock detection:
*
* We attempt to detect deadlocks that are due purely to posix file
* locks.
*
* We assume that a task can be waiting for at most one lock at a time.
* So for any acquired lock, the process holding that lock may be
* waiting on at most one other lock. That lock in turns may be held by
* someone waiting for at most one other lock. Given a requested lock
* caller_fl which is about to wait for a conflicting lock block_fl, we
* follow this chain of waiters to ensure we are not about to create a
* cycle.
*
* Since we do this before we ever put a process to sleep on a lock, we
* are ensured that there is never a cycle; that is what guarantees that
* the while() loop in posix_locks_deadlock() eventually completes.
*
* Note: the above assumption may not be true when handling lock
* requests from a broken NFS client. It may also fail in the presence
* of tasks (such as posix threads) sharing the same open file table.
* To handle those cases, we just bail out after a few iterations.
*
* For FL_OFDLCK locks, the owner is the filp, not the files_struct.
* Because the owner is not even nominally tied to a thread of
* execution, the deadlock detection below can't reasonably work well. Just
* skip it for those.
*
* In principle, we could do a more limited deadlock detection on FL_OFDLCK
* locks that just checks for the case where two tasks are attempting to
* upgrade from read to write locks on the same inode.
*/
#define MAX_DEADLK_ITERATIONS 10
/* Find a lock that the owner of the given @blocker is blocking on. */
static struct file_lock_core *what_owner_is_waiting_for(struct file_lock_core *blocker)
{
struct file_lock_core *flc;
hash_for_each_possible(blocked_hash, flc, flc_link, posix_owner_key(blocker)) {
if (posix_same_owner(flc, blocker)) {
while (flc->flc_blocker)
flc = flc->flc_blocker;
return flc;
}
}
return NULL;
}
/* Must be called with the blocked_lock_lock held! */
static bool posix_locks_deadlock(struct file_lock *caller_fl,
struct file_lock *block_fl)
{
struct file_lock_core *caller = &caller_fl->c;
struct file_lock_core *blocker = &block_fl->c;
int i = 0;
lockdep_assert_held(&blocked_lock_lock);
/*
* This deadlock detector can't reasonably detect deadlocks with
* FL_OFDLCK locks, since they aren't owned by a process, per-se.
*/
if (caller->flc_flags & FL_OFDLCK)
return false;
while ((blocker = what_owner_is_waiting_for(blocker))) {
if (i++ > MAX_DEADLK_ITERATIONS)
return false;
if (posix_same_owner(caller, blocker))
return true;
}
return false;
}
/* Try to create a FLOCK lock on filp. We always insert new FLOCK locks
* after any leases, but before any posix locks.
*
* Note that if called with an FL_EXISTS argument, the caller may determine
* whether or not a lock was successfully freed by testing the return
* value for -ENOENT.
*/
static int flock_lock_inode(struct inode *inode, struct file_lock *request)
{
struct file_lock *new_fl = NULL;
struct file_lock *fl;
struct file_lock_context *ctx;
int error = 0;
bool found = false;
LIST_HEAD(dispose);
ctx = locks_get_lock_context(inode, request->c.flc_type);
if (!ctx) {
if (request->c.flc_type != F_UNLCK)
return -ENOMEM;
return (request->c.flc_flags & FL_EXISTS) ? -ENOENT : 0;
}
if (!(request->c.flc_flags & FL_ACCESS) && (request->c.flc_type != F_UNLCK)) {
new_fl = locks_alloc_lock();
if (!new_fl)
return -ENOMEM;
}
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
if (request->c.flc_flags & FL_ACCESS)
goto find_conflict;
list_for_each_entry(fl, &ctx->flc_flock, c.flc_list) {
if (request->c.flc_file != fl->c.flc_file)
continue;
if (request->c.flc_type == fl->c.flc_type)
goto out;
found = true;
locks_delete_lock_ctx(&fl->c, &dispose);
break;
}
if (lock_is_unlock(request)) {
if ((request->c.flc_flags & FL_EXISTS) && !found)
error = -ENOENT;
goto out;
}
find_conflict:
list_for_each_entry(fl, &ctx->flc_flock, c.flc_list) {
if (!flock_locks_conflict(&request->c, &fl->c))
continue;
error = -EAGAIN;
if (!(request->c.flc_flags & FL_SLEEP))
goto out;
error = FILE_LOCK_DEFERRED;
locks_insert_block(&fl->c, &request->c, flock_locks_conflict);
goto out;
}
if (request->c.flc_flags & FL_ACCESS)
goto out;
locks_copy_lock(new_fl, request);
locks_move_blocks(new_fl, request);
locks_insert_lock_ctx(&new_fl->c, &ctx->flc_flock);
new_fl = NULL;
error = 0;
out:
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
if (new_fl)
locks_free_lock(new_fl);
locks_dispose_list(&dispose);
trace_flock_lock_inode(inode, request, error);
return error;
}
static int posix_lock_inode(struct inode *inode, struct file_lock *request,
struct file_lock *conflock)
{
struct file_lock *fl, *tmp;
struct file_lock *new_fl = NULL;
struct file_lock *new_fl2 = NULL;
struct file_lock *left = NULL;
struct file_lock *right = NULL;
struct file_lock_context *ctx;
int error;
bool added = false;
LIST_HEAD(dispose);
void *owner;
void (*func)(void);
ctx = locks_get_lock_context(inode, request->c.flc_type);
if (!ctx)
return lock_is_unlock(request) ? 0 : -ENOMEM;
/*
* We may need two file_lock structures for this operation,
* so we get them in advance to avoid races.
*
* In some cases we can be sure, that no new locks will be needed
*/
if (!(request->c.flc_flags & FL_ACCESS) &&
(request->c.flc_type != F_UNLCK ||
request->fl_start != 0 || request->fl_end != OFFSET_MAX)) {
new_fl = locks_alloc_lock();
new_fl2 = locks_alloc_lock();
}
retry:
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
/*
* New lock request. Walk all POSIX locks and look for conflicts. If
* there are any, either return error or put the request on the
* blocker's list of waiters and the global blocked_hash.
*/
if (request->c.flc_type != F_UNLCK) {
list_for_each_entry(fl, &ctx->flc_posix, c.flc_list) {
if (!posix_locks_conflict(&request->c, &fl->c))
continue;
if (fl->fl_lmops && fl->fl_lmops->lm_lock_expirable
&& (*fl->fl_lmops->lm_lock_expirable)(fl)) {
owner = fl->fl_lmops->lm_mod_owner;
func = fl->fl_lmops->lm_expire_lock;
__module_get(owner);
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
(*func)();
module_put(owner);
goto retry;
}
if (conflock)
locks_copy_conflock(conflock, fl);
error = -EAGAIN;
if (!(request->c.flc_flags & FL_SLEEP))
goto out;
/*
* Deadlock detection and insertion into the blocked
* locks list must be done while holding the same lock!
*/
error = -EDEADLK;
spin_lock(&blocked_lock_lock);
/*
* Ensure that we don't find any locks blocked on this
* request during deadlock detection.
*/
__locks_wake_up_blocks(&request->c);
if (likely(!posix_locks_deadlock(request, fl))) {
error = FILE_LOCK_DEFERRED;
__locks_insert_block(&fl->c, &request->c,
posix_locks_conflict);
}
spin_unlock(&blocked_lock_lock);
goto out;
}
}
/* If we're just looking for a conflict, we're done. */
error = 0;
if (request->c.flc_flags & FL_ACCESS)
goto out;
/* Find the first old lock with the same owner as the new lock */
list_for_each_entry(fl, &ctx->flc_posix, c.flc_list) {
if (posix_same_owner(&request->c, &fl->c))
break;
}
/* Process locks with this owner. */
list_for_each_entry_safe_from(fl, tmp, &ctx->flc_posix, c.flc_list) {
if (!posix_same_owner(&request->c, &fl->c))
break;
/* Detect adjacent or overlapping regions (if same lock type) */
if (request->c.flc_type == fl->c.flc_type) {
/* In all comparisons of start vs end, use
* "start - 1" rather than "end + 1". If end
* is OFFSET_MAX, end + 1 will become negative.
*/
if (fl->fl_end < request->fl_start - 1)
continue;
/* If the next lock in the list has entirely bigger
* addresses than the new one, insert the lock here.
*/
if (fl->fl_start - 1 > request->fl_end)
break;
/* If we come here, the new and old lock are of the
* same type and adjacent or overlapping. Make one
* lock yielding from the lower start address of both
* locks to the higher end address.
*/
if (fl->fl_start > request->fl_start)
fl->fl_start = request->fl_start;
else
request->fl_start = fl->fl_start;
if (fl->fl_end < request->fl_end)
fl->fl_end = request->fl_end;
else
request->fl_end = fl->fl_end;
if (added) {
locks_delete_lock_ctx(&fl->c, &dispose);
continue;
}
request = fl;
added = true;
} else {
/* Processing for different lock types is a bit
* more complex.
*/
if (fl->fl_end < request->fl_start)
continue;
if (fl->fl_start > request->fl_end)
break;
if (lock_is_unlock(request))
added = true;
if (fl->fl_start < request->fl_start)
left = fl;
/* If the next lock in the list has a higher end
* address than the new one, insert the new one here.
*/
if (fl->fl_end > request->fl_end) {
right = fl;
break;
}
if (fl->fl_start >= request->fl_start) {
/* The new lock completely replaces an old
* one (This may happen several times).
*/
if (added) {
locks_delete_lock_ctx(&fl->c, &dispose);
continue;
}
/*
* Replace the old lock with new_fl, and
* remove the old one. It's safe to do the
* insert here since we know that we won't be
* using new_fl later, and that the lock is
* just replacing an existing lock.
*/
error = -ENOLCK;
if (!new_fl)
goto out;
locks_copy_lock(new_fl, request);
locks_move_blocks(new_fl, request);
request = new_fl;
new_fl = NULL;
locks_insert_lock_ctx(&request->c,
&fl->c.flc_list);
locks_delete_lock_ctx(&fl->c, &dispose);
added = true;
}
}
}
/*
* The above code only modifies existing locks in case of merging or
* replacing. If new lock(s) need to be inserted all modifications are
* done below this, so it's safe yet to bail out.
*/
error = -ENOLCK; /* "no luck" */
if (right && left == right && !new_fl2)
goto out;
error = 0;
if (!added) {
if (lock_is_unlock(request)) {
if (request->c.flc_flags & FL_EXISTS)
error = -ENOENT;
goto out;
}
if (!new_fl) {
error = -ENOLCK;
goto out;
}
locks_copy_lock(new_fl, request);
locks_move_blocks(new_fl, request);
locks_insert_lock_ctx(&new_fl->c, &fl->c.flc_list);
fl = new_fl;
new_fl = NULL;
}
if (right) {
if (left == right) {
/* The new lock breaks the old one in two pieces,
* so we have to use the second new lock.
*/
left = new_fl2;
new_fl2 = NULL;
locks_copy_lock(left, right);
locks_insert_lock_ctx(&left->c, &fl->c.flc_list);
}
right->fl_start = request->fl_end + 1;
locks_wake_up_blocks(&right->c);
}
if (left) {
left->fl_end = request->fl_start - 1;
locks_wake_up_blocks(&left->c);
}
out:
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
trace_posix_lock_inode(inode, request, error);
/*
* Free any unused locks.
*/
if (new_fl)
locks_free_lock(new_fl);
if (new_fl2)
locks_free_lock(new_fl2);
locks_dispose_list(&dispose);
return error;
}
/**
* posix_lock_file - Apply a POSIX-style lock to a file
* @filp: The file to apply the lock to
* @fl: The lock to be applied
* @conflock: Place to return a copy of the conflicting lock, if found.
*
* Add a POSIX style lock to a file.
* We merge adjacent & overlapping locks whenever possible.
* POSIX locks are sorted by owner task, then by starting address
*
* Note that if called with an FL_EXISTS argument, the caller may determine
* whether or not a lock was successfully freed by testing the return
* value for -ENOENT.
*/
int posix_lock_file(struct file *filp, struct file_lock *fl,
struct file_lock *conflock)
{
return posix_lock_inode(file_inode(filp), fl, conflock);
}
EXPORT_SYMBOL(posix_lock_file);
/**
* posix_lock_inode_wait - Apply a POSIX-style lock to a file
* @inode: inode of file to which lock request should be applied
* @fl: The lock to be applied
*
* Apply a POSIX style lock request to an inode.
*/
static int posix_lock_inode_wait(struct inode *inode, struct file_lock *fl)
{
int error;
might_sleep ();
for (;;) {
error = posix_lock_inode(inode, fl, NULL);
if (error != FILE_LOCK_DEFERRED)
break;
error = wait_event_interruptible(fl->c.flc_wait,
list_empty(&fl->c.flc_blocked_member));
if (error)
break;
}
locks_delete_block(fl);
return error;
}
static void lease_clear_pending(struct file_lease *fl, int arg)
{
switch (arg) {
case F_UNLCK:
fl->c.flc_flags &= ~FL_UNLOCK_PENDING;
fallthrough;
case F_RDLCK:
fl->c.flc_flags &= ~FL_DOWNGRADE_PENDING;
}
}
/* We already had a lease on this file; just change its type */
int lease_modify(struct file_lease *fl, int arg, struct list_head *dispose)
{
int error = assign_type(&fl->c, arg);
if (error)
return error;
lease_clear_pending(fl, arg);
locks_wake_up_blocks(&fl->c);
if (arg == F_UNLCK) {
struct file *filp = fl->c.flc_file;
f_delown(filp);
filp->f_owner.signum = 0;
fasync_helper(0, fl->c.flc_file, 0, &fl->fl_fasync);
if (fl->fl_fasync != NULL) {
printk(KERN_ERR "locks_delete_lock: fasync == %p\n", fl->fl_fasync);
fl->fl_fasync = NULL;
}
locks_delete_lock_ctx(&fl->c, dispose);
}
return 0;
}
EXPORT_SYMBOL(lease_modify);
static bool past_time(unsigned long then)
{
if (!then)
/* 0 is a special value meaning "this never expires": */
return false;
return time_after(jiffies, then);
}
static void time_out_leases(struct inode *inode, struct list_head *dispose)
{
struct file_lock_context *ctx = inode->i_flctx;
struct file_lease *fl, *tmp;
lockdep_assert_held(&ctx->flc_lock);
list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) {
trace_time_out_leases(inode, fl);
if (past_time(fl->fl_downgrade_time))
lease_modify(fl, F_RDLCK, dispose);
if (past_time(fl->fl_break_time))
lease_modify(fl, F_UNLCK, dispose);
}
}
static bool leases_conflict(struct file_lock_core *lc, struct file_lock_core *bc)
{
bool rc;
struct file_lease *lease = file_lease(lc);
struct file_lease *breaker = file_lease(bc);
if (lease->fl_lmops->lm_breaker_owns_lease
&& lease->fl_lmops->lm_breaker_owns_lease(lease))
return false;
if ((bc->flc_flags & FL_LAYOUT) != (lc->flc_flags & FL_LAYOUT)) {
rc = false;
goto trace;
}
if ((bc->flc_flags & FL_DELEG) && (lc->flc_flags & FL_LEASE)) {
rc = false;
goto trace;
}
rc = locks_conflict(bc, lc);
trace:
trace_leases_conflict(rc, lease, breaker);
return rc;
}
static bool
any_leases_conflict(struct inode *inode, struct file_lease *breaker)
{
struct file_lock_context *ctx = inode->i_flctx;
struct file_lock_core *flc;
lockdep_assert_held(&ctx->flc_lock);
list_for_each_entry(flc, &ctx->flc_lease, flc_list) {
if (leases_conflict(flc, &breaker->c))
return true;
}
return false;
}
/**
* __break_lease - revoke all outstanding leases on file
* @inode: the inode of the file to return
* @mode: O_RDONLY: break only write leases; O_WRONLY or O_RDWR:
* break all leases
* @type: FL_LEASE: break leases and delegations; FL_DELEG: break
* only delegations
*
* break_lease (inlined for speed) has checked there already is at least
* some kind of lock (maybe a lease) on this file. Leases are broken on
* a call to open() or truncate(). This function can sleep unless you
* specified %O_NONBLOCK to your open().
*/
int __break_lease(struct inode *inode, unsigned int mode, unsigned int type)
{
int error = 0;
struct file_lock_context *ctx;
struct file_lease *new_fl, *fl, *tmp;
unsigned long break_time;
int want_write = (mode & O_ACCMODE) != O_RDONLY;
LIST_HEAD(dispose);
new_fl = lease_alloc(NULL, want_write ? F_WRLCK : F_RDLCK);
if (IS_ERR(new_fl))
return PTR_ERR(new_fl);
new_fl->c.flc_flags = type;
/* typically we will check that ctx is non-NULL before calling */
ctx = locks_inode_context(inode);
if (!ctx) {
WARN_ON_ONCE(1);
goto free_lock;
}
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
time_out_leases(inode, &dispose);
if (!any_leases_conflict(inode, new_fl))
goto out;
break_time = 0;
if (lease_break_time > 0) {
break_time = jiffies + lease_break_time * HZ;
if (break_time == 0)
break_time++; /* so that 0 means no break time */
}
list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) {
if (!leases_conflict(&fl->c, &new_fl->c))
continue;
if (want_write) {
if (fl->c.flc_flags & FL_UNLOCK_PENDING)
continue;
fl->c.flc_flags |= FL_UNLOCK_PENDING;
fl->fl_break_time = break_time;
} else {
if (lease_breaking(fl))
continue;
fl->c.flc_flags |= FL_DOWNGRADE_PENDING;
fl->fl_downgrade_time = break_time;
}
if (fl->fl_lmops->lm_break(fl))
locks_delete_lock_ctx(&fl->c, &dispose);
}
if (list_empty(&ctx->flc_lease))
goto out;
if (mode & O_NONBLOCK) {
trace_break_lease_noblock(inode, new_fl);
error = -EWOULDBLOCK;
goto out;
}
restart:
fl = list_first_entry(&ctx->flc_lease, struct file_lease, c.flc_list);
break_time = fl->fl_break_time;
if (break_time != 0)
break_time -= jiffies;
if (break_time == 0)
break_time++;
locks_insert_block(&fl->c, &new_fl->c, leases_conflict);
trace_break_lease_block(inode, new_fl);
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
locks_dispose_list(&dispose);
error = wait_event_interruptible_timeout(new_fl->c.flc_wait,
list_empty(&new_fl->c.flc_blocked_member),
break_time);
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
trace_break_lease_unblock(inode, new_fl);
__locks_delete_block(&new_fl->c);
if (error >= 0) {
/*
* Wait for the next conflicting lease that has not been
* broken yet
*/
if (error == 0)
time_out_leases(inode, &dispose);
if (any_leases_conflict(inode, new_fl))
goto restart;
error = 0;
}
out:
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
locks_dispose_list(&dispose);
free_lock:
locks_free_lease(new_fl);
return error;
}
EXPORT_SYMBOL(__break_lease);
/**
* lease_get_mtime - update modified time of an inode with exclusive lease
* @inode: the inode
* @time: pointer to a timespec which contains the last modified time
*
* This is to force NFS clients to flush their caches for files with
* exclusive leases. The justification is that if someone has an
* exclusive lease, then they could be modifying it.
*/
void lease_get_mtime(struct inode *inode, struct timespec64 *time)
{
bool has_lease = false;
struct file_lock_context *ctx;
struct file_lock_core *flc;
ctx = locks_inode_context(inode);
if (ctx && !list_empty_careful(&ctx->flc_lease)) {
spin_lock(&ctx->flc_lock);
flc = list_first_entry_or_null(&ctx->flc_lease,
struct file_lock_core, flc_list);
if (flc && flc->flc_type == F_WRLCK)
has_lease = true;
spin_unlock(&ctx->flc_lock);
}
if (has_lease)
*time = current_time(inode);
}
EXPORT_SYMBOL(lease_get_mtime);
/**
* fcntl_getlease - Enquire what lease is currently active
* @filp: the file
*
* The value returned by this function will be one of
* (if no lease break is pending):
*
* %F_RDLCK to indicate a shared lease is held.
*
* %F_WRLCK to indicate an exclusive lease is held.
*
* %F_UNLCK to indicate no lease is held.
*
* (if a lease break is pending):
*
* %F_RDLCK to indicate an exclusive lease needs to be
* changed to a shared lease (or removed).
*
* %F_UNLCK to indicate the lease needs to be removed.
*
* XXX: sfr & willy disagree over whether F_INPROGRESS
* should be returned to userspace.
*/
int fcntl_getlease(struct file *filp)
{
struct file_lease *fl;
struct inode *inode = file_inode(filp);
struct file_lock_context *ctx;
int type = F_UNLCK;
LIST_HEAD(dispose);
ctx = locks_inode_context(inode);
if (ctx && !list_empty_careful(&ctx->flc_lease)) {
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
time_out_leases(inode, &dispose);
list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) {
if (fl->c.flc_file != filp)
continue;
type = target_leasetype(fl);
break;
}
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
locks_dispose_list(&dispose);
}
return type;
}
/**
* check_conflicting_open - see if the given file points to an inode that has
* an existing open that would conflict with the
* desired lease.
* @filp: file to check
* @arg: type of lease that we're trying to acquire
* @flags: current lock flags
*
* Check to see if there's an existing open fd on this file that would
* conflict with the lease we're trying to set.
*/
static int
check_conflicting_open(struct file *filp, const int arg, int flags)
{
struct inode *inode = file_inode(filp);
int self_wcount = 0, self_rcount = 0;
if (flags & FL_LAYOUT)
return 0;
if (flags & FL_DELEG)
/* We leave these checks to the caller */
return 0;
if (arg == F_RDLCK)
return inode_is_open_for_write(inode) ? -EAGAIN : 0;
else if (arg != F_WRLCK)
return 0;
/*
* Make sure that only read/write count is from lease requestor.
* Note that this will result in denying write leases when i_writecount
* is negative, which is what we want. (We shouldn't grant write leases
* on files open for execution.)
*/
if (filp->f_mode & FMODE_WRITE)
self_wcount = 1;
else if (filp->f_mode & FMODE_READ)
self_rcount = 1;
if (atomic_read(&inode->i_writecount) != self_wcount ||
atomic_read(&inode->i_readcount) != self_rcount)
return -EAGAIN;
return 0;
}
static int
generic_add_lease(struct file *filp, int arg, struct file_lease **flp, void **priv)
{
struct file_lease *fl, *my_fl = NULL, *lease;
struct inode *inode = file_inode(filp);
struct file_lock_context *ctx;
bool is_deleg = (*flp)->c.flc_flags & FL_DELEG;
int error;
LIST_HEAD(dispose);
lease = *flp;
trace_generic_add_lease(inode, lease);
/* Note that arg is never F_UNLCK here */
ctx = locks_get_lock_context(inode, arg);
if (!ctx)
return -ENOMEM;
/*
* In the delegation case we need mutual exclusion with
* a number of operations that take the i_mutex. We trylock
* because delegations are an optional optimization, and if
* there's some chance of a conflict--we'd rather not
* bother, maybe that's a sign this just isn't a good file to
* hand out a delegation on.
*/
if (is_deleg && !inode_trylock(inode))
return -EAGAIN;
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
time_out_leases(inode, &dispose);
error = check_conflicting_open(filp, arg, lease->c.flc_flags);
if (error)
goto out;
/*
* At this point, we know that if there is an exclusive
* lease on this file, then we hold it on this filp
* (otherwise our open of this file would have blocked).
* And if we are trying to acquire an exclusive lease,
* then the file is not open by anyone (including us)
* except for this filp.
*/
error = -EAGAIN;
list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) {
if (fl->c.flc_file == filp &&
fl->c.flc_owner == lease->c.flc_owner) {
my_fl = fl;
continue;
}
/*
* No exclusive leases if someone else has a lease on
* this file:
*/
if (arg == F_WRLCK)
goto out;
/*
* Modifying our existing lease is OK, but no getting a
* new lease if someone else is opening for write:
*/
if (fl->c.flc_flags & FL_UNLOCK_PENDING)
goto out;
}
if (my_fl != NULL) {
lease = my_fl;
error = lease->fl_lmops->lm_change(lease, arg, &dispose);
if (error)
goto out;
goto out_setup;
}
error = -EINVAL;
if (!leases_enable)
goto out;
locks_insert_lock_ctx(&lease->c, &ctx->flc_lease);
/*
* The check in break_lease() is lockless. It's possible for another
* open to race in after we did the earlier check for a conflicting
* open but before the lease was inserted. Check again for a
* conflicting open and cancel the lease if there is one.
*
* We also add a barrier here to ensure that the insertion of the lock
* precedes these checks.
*/
smp_mb();
error = check_conflicting_open(filp, arg, lease->c.flc_flags);
if (error) {
locks_unlink_lock_ctx(&lease->c);
goto out;
}
out_setup:
if (lease->fl_lmops->lm_setup)
lease->fl_lmops->lm_setup(lease, priv);
out:
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
locks_dispose_list(&dispose);
if (is_deleg)
inode_unlock(inode);
if (!error && !my_fl)
*flp = NULL;
return error;
}
static int generic_delete_lease(struct file *filp, void *owner)
{
int error = -EAGAIN;
struct file_lease *fl, *victim = NULL;
struct inode *inode = file_inode(filp);
struct file_lock_context *ctx;
LIST_HEAD(dispose);
ctx = locks_inode_context(inode);
if (!ctx) {
trace_generic_delete_lease(inode, NULL);
return error;
}
percpu_down_read(&file_rwsem);
spin_lock(&ctx->flc_lock);
list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) {
if (fl->c.flc_file == filp &&
fl->c.flc_owner == owner) {
victim = fl;
break;
}
}
trace_generic_delete_lease(inode, victim);
if (victim)
error = fl->fl_lmops->lm_change(victim, F_UNLCK, &dispose);
spin_unlock(&ctx->flc_lock);
percpu_up_read(&file_rwsem);
locks_dispose_list(&dispose);
return error;
}
/**
* generic_setlease - sets a lease on an open file
* @filp: file pointer
* @arg: type of lease to obtain
* @flp: input - file_lock to use, output - file_lock inserted
* @priv: private data for lm_setup (may be NULL if lm_setup
* doesn't require it)
*
* The (input) flp->fl_lmops->lm_break function is required
* by break_lease().
*/
int generic_setlease(struct file *filp, int arg, struct file_lease **flp,
void **priv)
{
switch (arg) {
case F_UNLCK:
return generic_delete_lease(filp, *priv);
case F_RDLCK:
case F_WRLCK:
if (!(*flp)->fl_lmops->lm_break) {
WARN_ON_ONCE(1);
return -ENOLCK;
}
return generic_add_lease(filp, arg, flp, priv);
default:
return -EINVAL;
}
}
EXPORT_SYMBOL(generic_setlease);
/*
* Kernel subsystems can register to be notified on any attempt to set
* a new lease with the lease_notifier_chain. This is used by (e.g.) nfsd
* to close files that it may have cached when there is an attempt to set a
* conflicting lease.
*/
static struct srcu_notifier_head lease_notifier_chain;
static inline void
lease_notifier_chain_init(void)
{
srcu_init_notifier_head(&lease_notifier_chain);
}
static inline void
setlease_notifier(int arg, struct file_lease *lease)
{
if (arg != F_UNLCK)
srcu_notifier_call_chain(&lease_notifier_chain, arg, lease);
}
int lease_register_notifier(struct notifier_block *nb)
{
return srcu_notifier_chain_register(&lease_notifier_chain, nb);
}
EXPORT_SYMBOL_GPL(lease_register_notifier);
void lease_unregister_notifier(struct notifier_block *nb)
{
srcu_notifier_chain_unregister(&lease_notifier_chain, nb);
}
EXPORT_SYMBOL_GPL(lease_unregister_notifier);
int
kernel_setlease(struct file *filp, int arg, struct file_lease **lease, void **priv)
{
if (lease)
setlease_notifier(arg, *lease);
if (filp->f_op->setlease)
return filp->f_op->setlease(filp, arg, lease, priv);
else
return generic_setlease(filp, arg, lease, priv);
}
EXPORT_SYMBOL_GPL(kernel_setlease);
/**
* vfs_setlease - sets a lease on an open file
* @filp: file pointer
* @arg: type of lease to obtain
* @lease: file_lock to use when adding a lease
* @priv: private info for lm_setup when adding a lease (may be
* NULL if lm_setup doesn't require it)
*
* Call this to establish a lease on the file. The "lease" argument is not
* used for F_UNLCK requests and may be NULL. For commands that set or alter
* an existing lease, the ``(*lease)->fl_lmops->lm_break`` operation must be
* set; if not, this function will return -ENOLCK (and generate a scary-looking
* stack trace).
*
* The "priv" pointer is passed directly to the lm_setup function as-is. It
* may be NULL if the lm_setup operation doesn't require it.
*/
int
vfs_setlease(struct file *filp, int arg, struct file_lease **lease, void **priv)
{
struct inode *inode = file_inode(filp);
vfsuid_t vfsuid = i_uid_into_vfsuid(file_mnt_idmap(filp), inode);
int error;
if ((!vfsuid_eq_kuid(vfsuid, current_fsuid())) && !capable(CAP_LEASE))
return -EACCES;
if (!S_ISREG(inode->i_mode))
return -EINVAL;
error = security_file_lock(filp, arg);
if (error)
return error;
return kernel_setlease(filp, arg, lease, priv);
}
EXPORT_SYMBOL_GPL(vfs_setlease);
static int do_fcntl_add_lease(unsigned int fd, struct file *filp, int arg)
{
struct file_lease *fl;
struct fasync_struct *new;
int error;
fl = lease_alloc(filp, arg);
if (IS_ERR(fl))
return PTR_ERR(fl);
new = fasync_alloc();
if (!new) {
locks_free_lease(fl);
return -ENOMEM;
}
new->fa_fd = fd;
error = vfs_setlease(filp, arg, &fl, (void **)&new);
if (fl)
locks_free_lease(fl);
if (new)
fasync_free(new);
return error;
}
/**
* fcntl_setlease - sets a lease on an open file
* @fd: open file descriptor
* @filp: file pointer
* @arg: type of lease to obtain
*
* Call this fcntl to establish a lease on the file.
* Note that you also need to call %F_SETSIG to
* receive a signal when the lease is broken.
*/
int fcntl_setlease(unsigned int fd, struct file *filp, int arg)
{
if (arg == F_UNLCK)
return vfs_setlease(filp, F_UNLCK, NULL, (void **)&filp);
return do_fcntl_add_lease(fd, filp, arg);
}
/**
* flock_lock_inode_wait - Apply a FLOCK-style lock to a file
* @inode: inode of the file to apply to
* @fl: The lock to be applied
*
* Apply a FLOCK style lock request to an inode.
*/
static int flock_lock_inode_wait(struct inode *inode, struct file_lock *fl)
{
int error;
might_sleep();
for (;;) {
error = flock_lock_inode(inode, fl);
if (error != FILE_LOCK_DEFERRED)
break;
error = wait_event_interruptible(fl->c.flc_wait,
list_empty(&fl->c.flc_blocked_member));
if (error)
break;
}
locks_delete_block(fl);
return error;
}
/**
* locks_lock_inode_wait - Apply a lock to an inode
* @inode: inode of the file to apply to
* @fl: The lock to be applied
*
* Apply a POSIX or FLOCK style lock request to an inode.
*/
int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl)
{
int res = 0;
switch (fl->c.flc_flags & (FL_POSIX|FL_FLOCK)) {
case FL_POSIX:
res = posix_lock_inode_wait(inode, fl);
break;
case FL_FLOCK:
res = flock_lock_inode_wait(inode, fl);
break;
default:
BUG();
}
return res;
}
EXPORT_SYMBOL(locks_lock_inode_wait);
/**
* sys_flock: - flock() system call.
* @fd: the file descriptor to lock.
* @cmd: the type of lock to apply.
*
* Apply a %FL_FLOCK style lock to an open file descriptor.
* The @cmd can be one of:
*
* - %LOCK_SH -- a shared lock.
* - %LOCK_EX -- an exclusive lock.
* - %LOCK_UN -- remove an existing lock.
* - %LOCK_MAND -- a 'mandatory' flock. (DEPRECATED)
*
* %LOCK_MAND support has been removed from the kernel.
*/
SYSCALL_DEFINE2(flock, unsigned int, fd, unsigned int, cmd)
{
int can_sleep, error, type;
struct file_lock fl;
struct fd f;
/*
* LOCK_MAND locks were broken for a long time in that they never
* conflicted with one another and didn't prevent any sort of open,
* read or write activity.
*
* Just ignore these requests now, to preserve legacy behavior, but
* throw a warning to let people know that they don't actually work.
*/
if (cmd & LOCK_MAND) {
pr_warn_once("%s(%d): Attempt to set a LOCK_MAND lock via flock(2). This support has been removed and the request ignored.\n", current->comm, current->pid);
return 0;
}
type = flock_translate_cmd(cmd & ~LOCK_NB);
if (type < 0)
return type;
error = -EBADF;
f = fdget(fd);
if (!f.file)
return error;
if (type != F_UNLCK && !(f.file->f_mode & (FMODE_READ | FMODE_WRITE)))
goto out_putf;
flock_make_lock(f.file, &fl, type);
error = security_file_lock(f.file, fl.c.flc_type);
if (error)
goto out_putf;
can_sleep = !(cmd & LOCK_NB);
if (can_sleep)
fl.c.flc_flags |= FL_SLEEP;
if (f.file->f_op->flock)
error = f.file->f_op->flock(f.file,
(can_sleep) ? F_SETLKW : F_SETLK,
&fl);
else
error = locks_lock_file_wait(f.file, &fl);
locks_release_private(&fl);
out_putf:
fdput(f);
return error;
}
/**
* vfs_test_lock - test file byte range lock
* @filp: The file to test lock for
* @fl: The lock to test; also used to hold result
*
* Returns -ERRNO on failure. Indicates presence of conflicting lock by
* setting conf->fl_type to something other than F_UNLCK.
*/
int vfs_test_lock(struct file *filp, struct file_lock *fl)
{
WARN_ON_ONCE(filp != fl->c.flc_file);
if (filp->f_op->lock)
return filp->f_op->lock(filp, F_GETLK, fl);
posix_test_lock(filp, fl);
return 0;
}
EXPORT_SYMBOL_GPL(vfs_test_lock);
/**
* locks_translate_pid - translate a file_lock's fl_pid number into a namespace
* @fl: The file_lock who's fl_pid should be translated
* @ns: The namespace into which the pid should be translated
*
* Used to translate a fl_pid into a namespace virtual pid number
*/
static pid_t locks_translate_pid(struct file_lock_core *fl, struct pid_namespace *ns)
{
pid_t vnr;
struct pid *pid;
if (fl->flc_flags & FL_OFDLCK)
return -1;
/* Remote locks report a negative pid value */
if (fl->flc_pid <= 0)
return fl->flc_pid;
/*
* If the flock owner process is dead and its pid has been already
* freed, the translation below won't work, but we still want to show
* flock owner pid number in init pidns.
*/
if (ns == &init_pid_ns)
return (pid_t) fl->flc_pid;
rcu_read_lock();
pid = find_pid_ns(fl->flc_pid, &init_pid_ns);
vnr = pid_nr_ns(pid, ns);
rcu_read_unlock();
return vnr;
}
static int posix_lock_to_flock(struct flock *flock, struct file_lock *fl)
{
flock->l_pid = locks_translate_pid(&fl->c, task_active_pid_ns(current));
#if BITS_PER_LONG == 32
/*
* Make sure we can represent the posix lock via
* legacy 32bit flock.
*/
if (fl->fl_start > OFFT_OFFSET_MAX)
return -EOVERFLOW;
if (fl->fl_end != OFFSET_MAX && fl->fl_end > OFFT_OFFSET_MAX)
return -EOVERFLOW;
#endif
flock->l_start = fl->fl_start;
flock->l_len = fl->fl_end == OFFSET_MAX ? 0 :
fl->fl_end - fl->fl_start + 1;
flock->l_whence = 0;
flock->l_type = fl->c.flc_type;
return 0;
}
#if BITS_PER_LONG == 32
static void posix_lock_to_flock64(struct flock64 *flock, struct file_lock *fl)
{
flock->l_pid = locks_translate_pid(&fl->c, task_active_pid_ns(current));
flock->l_start = fl->fl_start;
flock->l_len = fl->fl_end == OFFSET_MAX ? 0 :
fl->fl_end - fl->fl_start + 1;
flock->l_whence = 0;
flock->l_type = fl->c.flc_type;
}
#endif
/* Report the first existing lock that would conflict with l.
* This implements the F_GETLK command of fcntl().
*/
int fcntl_getlk(struct file *filp, unsigned int cmd, struct flock *flock)
{
struct file_lock *fl;
int error;
fl = locks_alloc_lock();
if (fl == NULL)
return -ENOMEM;
error = -EINVAL;
if (cmd != F_OFD_GETLK && flock->l_type != F_RDLCK
&& flock->l_type != F_WRLCK)
goto out;
error = flock_to_posix_lock(filp, fl, flock);
if (error)
goto out;
if (cmd == F_OFD_GETLK) {
error = -EINVAL;
if (flock->l_pid != 0)
goto out;
fl->c.flc_flags |= FL_OFDLCK;
fl->c.flc_owner = filp;
}
error = vfs_test_lock(filp, fl);
if (error)
goto out;
flock->l_type = fl->c.flc_type;
if (fl->c.flc_type != F_UNLCK) {
error = posix_lock_to_flock(flock, fl);
if (error)
goto out;
}
out:
locks_free_lock(fl);
return error;
}
/**
* vfs_lock_file - file byte range lock
* @filp: The file to apply the lock to
* @cmd: type of locking operation (F_SETLK, F_GETLK, etc.)
* @fl: The lock to be applied
* @conf: Place to return a copy of the conflicting lock, if found.
*
* A caller that doesn't care about the conflicting lock may pass NULL
* as the final argument.
*
* If the filesystem defines a private ->lock() method, then @conf will
* be left unchanged; so a caller that cares should initialize it to
* some acceptable default.
*
* To avoid blocking kernel daemons, such as lockd, that need to acquire POSIX
* locks, the ->lock() interface may return asynchronously, before the lock has
* been granted or denied by the underlying filesystem, if (and only if)
* lm_grant is set. Additionally EXPORT_OP_ASYNC_LOCK in export_operations
* flags need to be set.
*
* Callers expecting ->lock() to return asynchronously will only use F_SETLK,
* not F_SETLKW; they will set FL_SLEEP if (and only if) the request is for a
* blocking lock. When ->lock() does return asynchronously, it must return
* FILE_LOCK_DEFERRED, and call ->lm_grant() when the lock request completes.
* If the request is for non-blocking lock the file system should return
* FILE_LOCK_DEFERRED then try to get the lock and call the callback routine
* with the result. If the request timed out the callback routine will return a
* nonzero return code and the file system should release the lock. The file
* system is also responsible to keep a corresponding posix lock when it
* grants a lock so the VFS can find out which locks are locally held and do
* the correct lock cleanup when required.
* The underlying filesystem must not drop the kernel lock or call
* ->lm_grant() before returning to the caller with a FILE_LOCK_DEFERRED
* return code.
*/
int vfs_lock_file(struct file *filp, unsigned int cmd, struct file_lock *fl, struct file_lock *conf)
{
WARN_ON_ONCE(filp != fl->c.flc_file);
if (filp->f_op->lock)
return filp->f_op->lock(filp, cmd, fl);
else
return posix_lock_file(filp, fl, conf);
}
EXPORT_SYMBOL_GPL(vfs_lock_file);
static int do_lock_file_wait(struct file *filp, unsigned int cmd,
struct file_lock *fl)
{
int error;
error = security_file_lock(filp, fl->c.flc_type);
if (error)
return error;
for (;;) {
error = vfs_lock_file(filp, cmd, fl, NULL);
if (error != FILE_LOCK_DEFERRED)
break;
error = wait_event_interruptible(fl->c.flc_wait,
list_empty(&fl->c.flc_blocked_member));
if (error)
break;
}
locks_delete_block(fl);
return error;
}
/* Ensure that fl->fl_file has compatible f_mode for F_SETLK calls */
static int
check_fmode_for_setlk(struct file_lock *fl)
{
switch (fl->c.flc_type) {
case F_RDLCK:
if (!(fl->c.flc_file->f_mode & FMODE_READ))
return -EBADF;
break;
case F_WRLCK:
if (!(fl->c.flc_file->f_mode & FMODE_WRITE))
return -EBADF;
}
return 0;
}
/* Apply the lock described by l to an open file descriptor.
* This implements both the F_SETLK and F_SETLKW commands of fcntl().
*/
int fcntl_setlk(unsigned int fd, struct file *filp, unsigned int cmd,
struct flock *flock)
{
struct file_lock *file_lock = locks_alloc_lock();
struct inode *inode = file_inode(filp);
struct file *f;
int error;
if (file_lock == NULL)
return -ENOLCK;
error = flock_to_posix_lock(filp, file_lock, flock);
if (error)
goto out;
error = check_fmode_for_setlk(file_lock);
if (error)
goto out;
/*
* If the cmd is requesting file-private locks, then set the
* FL_OFDLCK flag and override the owner.
*/
switch (cmd) {
case F_OFD_SETLK:
error = -EINVAL;
if (flock->l_pid != 0)
goto out;
cmd = F_SETLK;
file_lock->c.flc_flags |= FL_OFDLCK;
file_lock->c.flc_owner = filp;
break;
case F_OFD_SETLKW:
error = -EINVAL;
if (flock->l_pid != 0)
goto out;
cmd = F_SETLKW;
file_lock->c.flc_flags |= FL_OFDLCK;
file_lock->c.flc_owner = filp;
fallthrough;
case F_SETLKW:
file_lock->c.flc_flags |= FL_SLEEP;
}
error = do_lock_file_wait(filp, cmd, file_lock);
/*
* Attempt to detect a close/fcntl race and recover by releasing the
* lock that was just acquired. There is no need to do that when we're
* unlocking though, or for OFD locks.
*/
if (!error && file_lock->c.flc_type != F_UNLCK &&
!(file_lock->c.flc_flags & FL_OFDLCK)) {
struct files_struct *files = current->files;
/*
* We need that spin_lock here - it prevents reordering between
* update of i_flctx->flc_posix and check for it done in
* close(). rcu_read_lock() wouldn't do.
*/
spin_lock(&files->file_lock);
f = files_lookup_fd_locked(files, fd);
spin_unlock(&files->file_lock);
if (f != filp) {
file_lock->c.flc_type = F_UNLCK;
error = do_lock_file_wait(filp, cmd, file_lock);
WARN_ON_ONCE(error);
error = -EBADF;
}
}
out:
trace_fcntl_setlk(inode, file_lock, error);
locks_free_lock(file_lock);
return error;
}
#if BITS_PER_LONG == 32
/* Report the first existing lock that would conflict with l.
* This implements the F_GETLK command of fcntl().
*/
int fcntl_getlk64(struct file *filp, unsigned int cmd, struct flock64 *flock)
{
struct file_lock *fl;
int error;
fl = locks_alloc_lock();
if (fl == NULL)
return -ENOMEM;
error = -EINVAL;
if (cmd != F_OFD_GETLK && flock->l_type != F_RDLCK
&& flock->l_type != F_WRLCK)
goto out;
error = flock64_to_posix_lock(filp, fl, flock);
if (error)
goto out;
if (cmd == F_OFD_GETLK) {
error = -EINVAL;
if (flock->l_pid != 0)
goto out;
fl->c.flc_flags |= FL_OFDLCK;
fl->c.flc_owner = filp;
}
error = vfs_test_lock(filp, fl);
if (error)
goto out;
flock->l_type = fl->c.flc_type;
if (fl->c.flc_type != F_UNLCK)
posix_lock_to_flock64(flock, fl);
out:
locks_free_lock(fl);
return error;
}
/* Apply the lock described by l to an open file descriptor.
* This implements both the F_SETLK and F_SETLKW commands of fcntl().
*/
int fcntl_setlk64(unsigned int fd, struct file *filp, unsigned int cmd,
struct flock64 *flock)
{
struct file_lock *file_lock = locks_alloc_lock();
struct file *f;
int error;
if (file_lock == NULL)
return -ENOLCK;
error = flock64_to_posix_lock(filp, file_lock, flock);
if (error)
goto out;
error = check_fmode_for_setlk(file_lock);
if (error)
goto out;
/*
* If the cmd is requesting file-private locks, then set the
* FL_OFDLCK flag and override the owner.
*/
switch (cmd) {
case F_OFD_SETLK:
error = -EINVAL;
if (flock->l_pid != 0)
goto out;
cmd = F_SETLK64;
file_lock->c.flc_flags |= FL_OFDLCK;
file_lock->c.flc_owner = filp;
break;
case F_OFD_SETLKW:
error = -EINVAL;
if (flock->l_pid != 0)
goto out;
cmd = F_SETLKW64;
file_lock->c.flc_flags |= FL_OFDLCK;
file_lock->c.flc_owner = filp;
fallthrough;
case F_SETLKW64:
file_lock->c.flc_flags |= FL_SLEEP;
}
error = do_lock_file_wait(filp, cmd, file_lock);
/*
* Attempt to detect a close/fcntl race and recover by releasing the
* lock that was just acquired. There is no need to do that when we're
* unlocking though, or for OFD locks.
*/
if (!error && file_lock->c.flc_type != F_UNLCK &&
!(file_lock->c.flc_flags & FL_OFDLCK)) {
struct files_struct *files = current->files;
/*
* We need that spin_lock here - it prevents reordering between
* update of i_flctx->flc_posix and check for it done in
* close(). rcu_read_lock() wouldn't do.
*/
spin_lock(&files->file_lock);
f = files_lookup_fd_locked(files, fd);
spin_unlock(&files->file_lock);
if (f != filp) {
file_lock->c.flc_type = F_UNLCK;
error = do_lock_file_wait(filp, cmd, file_lock);
WARN_ON_ONCE(error);
error = -EBADF;
}
}
out:
locks_free_lock(file_lock);
return error;
}
#endif /* BITS_PER_LONG == 32 */
/*
* This function is called when the file is being removed
* from the task's fd array. POSIX locks belonging to this task
* are deleted at this time.
*/
void locks_remove_posix(struct file *filp, fl_owner_t owner)
{
int error;
struct inode *inode = file_inode(filp);
struct file_lock lock;
struct file_lock_context *ctx;
/*
* If there are no locks held on this file, we don't need to call
* posix_lock_file(). Another process could be setting a lock on this
* file at the same time, but we wouldn't remove that lock anyway.
*/
ctx = locks_inode_context(inode);
if (!ctx || list_empty(&ctx->flc_posix)) return; locks_init_lock(&lock);
lock.c.flc_type = F_UNLCK;
lock.c.flc_flags = FL_POSIX | FL_CLOSE;
lock.fl_start = 0;
lock.fl_end = OFFSET_MAX;
lock.c.flc_owner = owner;
lock.c.flc_pid = current->tgid;
lock.c.flc_file = filp;
lock.fl_ops = NULL;
lock.fl_lmops = NULL;
error = vfs_lock_file(filp, F_SETLK, &lock, NULL);
if (lock.fl_ops && lock.fl_ops->fl_release_private) lock.fl_ops->fl_release_private(&lock); trace_locks_remove_posix(inode, &lock, error);
}
EXPORT_SYMBOL(locks_remove_posix);
/* The i_flctx must be valid when calling into here */
static void
locks_remove_flock(struct file *filp, struct file_lock_context *flctx)
{
struct file_lock fl;
struct inode *inode = file_inode(filp);
if (list_empty(&flctx->flc_flock))
return;
flock_make_lock(filp, &fl, F_UNLCK);
fl.c.flc_flags |= FL_CLOSE;
if (filp->f_op->flock)
filp->f_op->flock(filp, F_SETLKW, &fl);
else
flock_lock_inode(inode, &fl);
if (fl.fl_ops && fl.fl_ops->fl_release_private)
fl.fl_ops->fl_release_private(&fl);
}
/* The i_flctx must be valid when calling into here */
static void
locks_remove_lease(struct file *filp, struct file_lock_context *ctx)
{
struct file_lease *fl, *tmp;
LIST_HEAD(dispose);
if (list_empty(&ctx->flc_lease))
return; percpu_down_read(&file_rwsem); spin_lock(&ctx->flc_lock); list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) if (filp == fl->c.flc_file) lease_modify(fl, F_UNLCK, &dispose); spin_unlock(&ctx->flc_lock); percpu_up_read(&file_rwsem); locks_dispose_list(&dispose);
}
/*
* This function is called on the last close of an open file.
*/
void locks_remove_file(struct file *filp)
{
struct file_lock_context *ctx;
ctx = locks_inode_context(file_inode(filp));
if (!ctx)
return;
/* remove any OFD locks */
locks_remove_posix(filp, filp);
/* remove flock locks */
locks_remove_flock(filp, ctx);
/* remove any leases */
locks_remove_lease(filp, ctx);
spin_lock(&ctx->flc_lock);
locks_check_ctx_file_list(filp, &ctx->flc_posix, "POSIX");
locks_check_ctx_file_list(filp, &ctx->flc_flock, "FLOCK");
locks_check_ctx_file_list(filp, &ctx->flc_lease, "LEASE");
spin_unlock(&ctx->flc_lock);
}
/**
* vfs_cancel_lock - file byte range unblock lock
* @filp: The file to apply the unblock to
* @fl: The lock to be unblocked
*
* Used by lock managers to cancel blocked requests
*/
int vfs_cancel_lock(struct file *filp, struct file_lock *fl)
{
WARN_ON_ONCE(filp != fl->c.flc_file);
if (filp->f_op->lock)
return filp->f_op->lock(filp, F_CANCELLK, fl);
return 0;
}
EXPORT_SYMBOL_GPL(vfs_cancel_lock);
/**
* vfs_inode_has_locks - are any file locks held on @inode?
* @inode: inode to check for locks
*
* Return true if there are any FL_POSIX or FL_FLOCK locks currently
* set on @inode.
*/
bool vfs_inode_has_locks(struct inode *inode)
{
struct file_lock_context *ctx;
bool ret;
ctx = locks_inode_context(inode);
if (!ctx)
return false;
spin_lock(&ctx->flc_lock);
ret = !list_empty(&ctx->flc_posix) || !list_empty(&ctx->flc_flock);
spin_unlock(&ctx->flc_lock);
return ret;
}
EXPORT_SYMBOL_GPL(vfs_inode_has_locks);
#ifdef CONFIG_PROC_FS
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
struct locks_iterator {
int li_cpu;
loff_t li_pos;
};
static void lock_get_status(struct seq_file *f, struct file_lock_core *flc,
loff_t id, char *pfx, int repeat)
{
struct inode *inode = NULL;
unsigned int pid;
struct pid_namespace *proc_pidns = proc_pid_ns(file_inode(f->file)->i_sb);
int type = flc->flc_type;
struct file_lock *fl = file_lock(flc);
pid = locks_translate_pid(flc, proc_pidns);
/*
* If lock owner is dead (and pid is freed) or not visible in current
* pidns, zero is shown as a pid value. Check lock info from
* init_pid_ns to get saved lock pid value.
*/
if (flc->flc_file != NULL)
inode = file_inode(flc->flc_file);
seq_printf(f, "%lld: ", id);
if (repeat)
seq_printf(f, "%*s", repeat - 1 + (int)strlen(pfx), pfx);
if (flc->flc_flags & FL_POSIX) {
if (flc->flc_flags & FL_ACCESS)
seq_puts(f, "ACCESS");
else if (flc->flc_flags & FL_OFDLCK)
seq_puts(f, "OFDLCK");
else
seq_puts(f, "POSIX ");
seq_printf(f, " %s ",
(inode == NULL) ? "*NOINODE*" : "ADVISORY ");
} else if (flc->flc_flags & FL_FLOCK) {
seq_puts(f, "FLOCK ADVISORY ");
} else if (flc->flc_flags & (FL_LEASE|FL_DELEG|FL_LAYOUT)) {
struct file_lease *lease = file_lease(flc);
type = target_leasetype(lease);
if (flc->flc_flags & FL_DELEG)
seq_puts(f, "DELEG ");
else
seq_puts(f, "LEASE ");
if (lease_breaking(lease))
seq_puts(f, "BREAKING ");
else if (flc->flc_file)
seq_puts(f, "ACTIVE ");
else
seq_puts(f, "BREAKER ");
} else {
seq_puts(f, "UNKNOWN UNKNOWN ");
}
seq_printf(f, "%s ", (type == F_WRLCK) ? "WRITE" :
(type == F_RDLCK) ? "READ" : "UNLCK");
if (inode) {
/* userspace relies on this representation of dev_t */
seq_printf(f, "%d %02x:%02x:%lu ", pid,
MAJOR(inode->i_sb->s_dev),
MINOR(inode->i_sb->s_dev), inode->i_ino);
} else {
seq_printf(f, "%d <none>:0 ", pid);
}
if (flc->flc_flags & FL_POSIX) {
if (fl->fl_end == OFFSET_MAX)
seq_printf(f, "%Ld EOF\n", fl->fl_start);
else
seq_printf(f, "%Ld %Ld\n", fl->fl_start, fl->fl_end);
} else {
seq_puts(f, "0 EOF\n");
}
}
static struct file_lock_core *get_next_blocked_member(struct file_lock_core *node)
{
struct file_lock_core *tmp;
/* NULL node or root node */
if (node == NULL || node->flc_blocker == NULL)
return NULL;
/* Next member in the linked list could be itself */
tmp = list_next_entry(node, flc_blocked_member);
if (list_entry_is_head(tmp, &node->flc_blocker->flc_blocked_requests,
flc_blocked_member)
|| tmp == node) {
return NULL;
}
return tmp;
}
static int locks_show(struct seq_file *f, void *v)
{
struct locks_iterator *iter = f->private;
struct file_lock_core *cur, *tmp;
struct pid_namespace *proc_pidns = proc_pid_ns(file_inode(f->file)->i_sb);
int level = 0;
cur = hlist_entry(v, struct file_lock_core, flc_link);
if (locks_translate_pid(cur, proc_pidns) == 0)
return 0;
/* View this crossed linked list as a binary tree, the first member of flc_blocked_requests
* is the left child of current node, the next silibing in flc_blocked_member is the
* right child, we can alse get the parent of current node from flc_blocker, so this
* question becomes traversal of a binary tree
*/
while (cur != NULL) {
if (level)
lock_get_status(f, cur, iter->li_pos, "-> ", level);
else
lock_get_status(f, cur, iter->li_pos, "", level);
if (!list_empty(&cur->flc_blocked_requests)) {
/* Turn left */
cur = list_first_entry_or_null(&cur->flc_blocked_requests,
struct file_lock_core,
flc_blocked_member);
level++;
} else {
/* Turn right */
tmp = get_next_blocked_member(cur);
/* Fall back to parent node */
while (tmp == NULL && cur->flc_blocker != NULL) {
cur = cur->flc_blocker;
level--;
tmp = get_next_blocked_member(cur);
}
cur = tmp;
}
}
return 0;
}
static void __show_fd_locks(struct seq_file *f,
struct list_head *head, int *id,
struct file *filp, struct files_struct *files)
{
struct file_lock_core *fl;
list_for_each_entry(fl, head, flc_list) {
if (filp != fl->flc_file)
continue;
if (fl->flc_owner != files && fl->flc_owner != filp)
continue;
(*id)++;
seq_puts(f, "lock:\t");
lock_get_status(f, fl, *id, "", 0);
}
}
void show_fd_locks(struct seq_file *f,
struct file *filp, struct files_struct *files)
{
struct inode *inode = file_inode(filp);
struct file_lock_context *ctx;
int id = 0;
ctx = locks_inode_context(inode);
if (!ctx)
return;
spin_lock(&ctx->flc_lock);
__show_fd_locks(f, &ctx->flc_flock, &id, filp, files);
__show_fd_locks(f, &ctx->flc_posix, &id, filp, files);
__show_fd_locks(f, &ctx->flc_lease, &id, filp, files);
spin_unlock(&ctx->flc_lock);
}
static void *locks_start(struct seq_file *f, loff_t *pos)
__acquires(&blocked_lock_lock)
{
struct locks_iterator *iter = f->private;
iter->li_pos = *pos + 1;
percpu_down_write(&file_rwsem);
spin_lock(&blocked_lock_lock);
return seq_hlist_start_percpu(&file_lock_list.hlist, &iter->li_cpu, *pos);
}
static void *locks_next(struct seq_file *f, void *v, loff_t *pos)
{
struct locks_iterator *iter = f->private;
++iter->li_pos;
return seq_hlist_next_percpu(v, &file_lock_list.hlist, &iter->li_cpu, pos);
}
static void locks_stop(struct seq_file *f, void *v)
__releases(&blocked_lock_lock)
{
spin_unlock(&blocked_lock_lock);
percpu_up_write(&file_rwsem);
}
static const struct seq_operations locks_seq_operations = {
.start = locks_start,
.next = locks_next,
.stop = locks_stop,
.show = locks_show,
};
static int __init proc_locks_init(void)
{
proc_create_seq_private("locks", 0, NULL, &locks_seq_operations,
sizeof(struct locks_iterator), NULL);
return 0;
}
fs_initcall(proc_locks_init);
#endif
static int __init filelock_init(void)
{
int i;
flctx_cache = kmem_cache_create("file_lock_ctx",
sizeof(struct file_lock_context), 0, SLAB_PANIC, NULL);
filelock_cache = kmem_cache_create("file_lock_cache",
sizeof(struct file_lock), 0, SLAB_PANIC, NULL);
filelease_cache = kmem_cache_create("file_lock_cache",
sizeof(struct file_lease), 0, SLAB_PANIC, NULL);
for_each_possible_cpu(i) {
struct file_lock_list_struct *fll = per_cpu_ptr(&file_lock_list, i);
spin_lock_init(&fll->lock);
INIT_HLIST_HEAD(&fll->hlist);
}
lease_notifier_chain_init();
return 0;
}
core_initcall(filelock_init);
// SPDX-License-Identifier: GPL-2.0
// rc-ir-raw.c - handle IR pulse/space events
//
// Copyright (C) 2010 by Mauro Carvalho Chehab
#include <linux/export.h>
#include <linux/kthread.h>
#include <linux/mutex.h>
#include <linux/kmod.h>
#include <linux/sched.h>
#include "rc-core-priv.h"
/* Used to keep track of IR raw clients, protected by ir_raw_handler_lock */
static LIST_HEAD(ir_raw_client_list);
/* Used to handle IR raw handler extensions */
DEFINE_MUTEX(ir_raw_handler_lock);
static LIST_HEAD(ir_raw_handler_list);
static atomic64_t available_protocols = ATOMIC64_INIT(0);
static int ir_raw_event_thread(void *data)
{
struct ir_raw_event ev;
struct ir_raw_handler *handler;
struct ir_raw_event_ctrl *raw = data;
struct rc_dev *dev = raw->dev;
while (1) {
mutex_lock(&ir_raw_handler_lock);
while (kfifo_out(&raw->kfifo, &ev, 1)) {
if (is_timing_event(ev)) {
if (ev.duration == 0)
dev_warn_once(&dev->dev, "nonsensical timing event of duration 0");
if (is_timing_event(raw->prev_ev) &&
!is_transition(&ev, &raw->prev_ev))
dev_warn_once(&dev->dev, "two consecutive events of type %s",
TO_STR(ev.pulse));
}
list_for_each_entry(handler, &ir_raw_handler_list, list)
if (dev->enabled_protocols &
handler->protocols || !handler->protocols)
handler->decode(dev, ev);
lirc_raw_event(dev, ev);
raw->prev_ev = ev;
}
mutex_unlock(&ir_raw_handler_lock);
set_current_state(TASK_INTERRUPTIBLE);
if (kthread_should_stop()) {
__set_current_state(TASK_RUNNING);
break;
} else if (!kfifo_is_empty(&raw->kfifo))
set_current_state(TASK_RUNNING);
schedule();
}
return 0;
}
/**
* ir_raw_event_store() - pass a pulse/space duration to the raw ir decoders
* @dev: the struct rc_dev device descriptor
* @ev: the struct ir_raw_event descriptor of the pulse/space
*
* This routine (which may be called from an interrupt context) stores a
* pulse/space duration for the raw ir decoding state machines. Pulses are
* signalled as positive values and spaces as negative values. A zero value
* will reset the decoding state machines.
*/
int ir_raw_event_store(struct rc_dev *dev, struct ir_raw_event *ev)
{
if (!dev->raw)
return -EINVAL;
dev_dbg(&dev->dev, "sample: (%05dus %s)\n",
ev->duration, TO_STR(ev->pulse));
if (!kfifo_put(&dev->raw->kfifo, *ev)) {
dev_err(&dev->dev, "IR event FIFO is full!\n");
return -ENOSPC;
}
return 0;
}
EXPORT_SYMBOL_GPL(ir_raw_event_store);
/**
* ir_raw_event_store_edge() - notify raw ir decoders of the start of a pulse/space
* @dev: the struct rc_dev device descriptor
* @pulse: true for pulse, false for space
*
* This routine (which may be called from an interrupt context) is used to
* store the beginning of an ir pulse or space (or the start/end of ir
* reception) for the raw ir decoding state machines. This is used by
* hardware which does not provide durations directly but only interrupts
* (or similar events) on state change.
*/
int ir_raw_event_store_edge(struct rc_dev *dev, bool pulse)
{
ktime_t now;
struct ir_raw_event ev = {};
if (!dev->raw)
return -EINVAL;
now = ktime_get();
ev.duration = ktime_to_us(ktime_sub(now, dev->raw->last_event));
ev.pulse = !pulse;
return ir_raw_event_store_with_timeout(dev, &ev);
}
EXPORT_SYMBOL_GPL(ir_raw_event_store_edge);
/*
* ir_raw_event_store_with_timeout() - pass a pulse/space duration to the raw
* ir decoders, schedule decoding and
* timeout
* @dev: the struct rc_dev device descriptor
* @ev: the struct ir_raw_event descriptor of the pulse/space
*
* This routine (which may be called from an interrupt context) stores a
* pulse/space duration for the raw ir decoding state machines, schedules
* decoding and generates a timeout.
*/
int ir_raw_event_store_with_timeout(struct rc_dev *dev, struct ir_raw_event *ev)
{
ktime_t now;
int rc = 0;
if (!dev->raw)
return -EINVAL;
now = ktime_get();
spin_lock(&dev->raw->edge_spinlock);
rc = ir_raw_event_store(dev, ev);
dev->raw->last_event = now;
/* timer could be set to timeout (125ms by default) */
if (!timer_pending(&dev->raw->edge_handle) ||
time_after(dev->raw->edge_handle.expires,
jiffies + msecs_to_jiffies(15))) {
mod_timer(&dev->raw->edge_handle,
jiffies + msecs_to_jiffies(15));
}
spin_unlock(&dev->raw->edge_spinlock);
return rc;
}
EXPORT_SYMBOL_GPL(ir_raw_event_store_with_timeout);
/**
* ir_raw_event_store_with_filter() - pass next pulse/space to decoders with some processing
* @dev: the struct rc_dev device descriptor
* @ev: the event that has occurred
*
* This routine (which may be called from an interrupt context) works
* in similar manner to ir_raw_event_store_edge.
* This routine is intended for devices with limited internal buffer
* It automerges samples of same type, and handles timeouts. Returns non-zero
* if the event was added, and zero if the event was ignored due to idle
* processing.
*/
int ir_raw_event_store_with_filter(struct rc_dev *dev, struct ir_raw_event *ev)
{
if (!dev->raw)
return -EINVAL;
/* Ignore spaces in idle mode */
if (dev->idle && !ev->pulse)
return 0;
else if (dev->idle)
ir_raw_event_set_idle(dev, false);
if (!dev->raw->this_ev.duration)
dev->raw->this_ev = *ev;
else if (ev->pulse == dev->raw->this_ev.pulse)
dev->raw->this_ev.duration += ev->duration;
else {
ir_raw_event_store(dev, &dev->raw->this_ev);
dev->raw->this_ev = *ev;
}
/* Enter idle mode if necessary */
if (!ev->pulse && dev->timeout &&
dev->raw->this_ev.duration >= dev->timeout)
ir_raw_event_set_idle(dev, true);
return 1;
}
EXPORT_SYMBOL_GPL(ir_raw_event_store_with_filter);
/**
* ir_raw_event_set_idle() - provide hint to rc-core when the device is idle or not
* @dev: the struct rc_dev device descriptor
* @idle: whether the device is idle or not
*/
void ir_raw_event_set_idle(struct rc_dev *dev, bool idle)
{
if (!dev->raw)
return;
dev_dbg(&dev->dev, "%s idle mode\n", idle ? "enter" : "leave");
if (idle) {
dev->raw->this_ev.timeout = true;
ir_raw_event_store(dev, &dev->raw->this_ev);
dev->raw->this_ev = (struct ir_raw_event) {};
}
if (dev->s_idle)
dev->s_idle(dev, idle);
dev->idle = idle;
}
EXPORT_SYMBOL_GPL(ir_raw_event_set_idle);
/**
* ir_raw_event_handle() - schedules the decoding of stored ir data
* @dev: the struct rc_dev device descriptor
*
* This routine will tell rc-core to start decoding stored ir data.
*/
void ir_raw_event_handle(struct rc_dev *dev)
{
if (!dev->raw || !dev->raw->thread)
return;
wake_up_process(dev->raw->thread);
}
EXPORT_SYMBOL_GPL(ir_raw_event_handle);
/* used internally by the sysfs interface */
u64
ir_raw_get_allowed_protocols(void)
{
return atomic64_read(&available_protocols);
}
static int change_protocol(struct rc_dev *dev, u64 *rc_proto)
{
struct ir_raw_handler *handler;
u32 timeout = 0;
mutex_lock(&ir_raw_handler_lock);
list_for_each_entry(handler, &ir_raw_handler_list, list) {
if (!(dev->enabled_protocols & handler->protocols) &&
(*rc_proto & handler->protocols) && handler->raw_register)
handler->raw_register(dev);
if ((dev->enabled_protocols & handler->protocols) &&
!(*rc_proto & handler->protocols) &&
handler->raw_unregister)
handler->raw_unregister(dev);
}
mutex_unlock(&ir_raw_handler_lock);
if (!dev->max_timeout)
return 0;
mutex_lock(&ir_raw_handler_lock);
list_for_each_entry(handler, &ir_raw_handler_list, list) {
if (handler->protocols & *rc_proto) {
if (timeout < handler->min_timeout)
timeout = handler->min_timeout;
}
}
mutex_unlock(&ir_raw_handler_lock);
if (timeout == 0)
timeout = IR_DEFAULT_TIMEOUT;
else
timeout += MS_TO_US(10);
if (timeout < dev->min_timeout)
timeout = dev->min_timeout;
else if (timeout > dev->max_timeout)
timeout = dev->max_timeout;
if (dev->s_timeout)
dev->s_timeout(dev, timeout);
else
dev->timeout = timeout;
return 0;
}
static void ir_raw_disable_protocols(struct rc_dev *dev, u64 protocols)
{
mutex_lock(&dev->lock);
dev->enabled_protocols &= ~protocols;
mutex_unlock(&dev->lock);
}
/**
* ir_raw_gen_manchester() - Encode data with Manchester (bi-phase) modulation.
* @ev: Pointer to pointer to next free event. *@ev is incremented for
* each raw event filled.
* @max: Maximum number of raw events to fill.
* @timings: Manchester modulation timings.
* @n: Number of bits of data.
* @data: Data bits to encode.
*
* Encodes the @n least significant bits of @data using Manchester (bi-phase)
* modulation with the timing characteristics described by @timings, writing up
* to @max raw IR events using the *@ev pointer.
*
* Returns: 0 on success.
* -ENOBUFS if there isn't enough space in the array to fit the
* full encoded data. In this case all @max events will have been
* written.
*/
int ir_raw_gen_manchester(struct ir_raw_event **ev, unsigned int max,
const struct ir_raw_timings_manchester *timings,
unsigned int n, u64 data)
{
bool need_pulse;
u64 i;
int ret = -ENOBUFS;
i = BIT_ULL(n - 1);
if (timings->leader_pulse) {
if (!max--)
return ret;
init_ir_raw_event_duration((*ev), 1, timings->leader_pulse);
if (timings->leader_space) {
if (!max--)
return ret;
init_ir_raw_event_duration(++(*ev), 0,
timings->leader_space);
}
} else {
/* continue existing signal */
--(*ev);
}
/* from here on *ev will point to the last event rather than the next */
while (n && i > 0) {
need_pulse = !(data & i);
if (timings->invert)
need_pulse = !need_pulse;
if (need_pulse == !!(*ev)->pulse) {
(*ev)->duration += timings->clock;
} else {
if (!max--)
goto nobufs;
init_ir_raw_event_duration(++(*ev), need_pulse,
timings->clock);
}
if (!max--)
goto nobufs;
init_ir_raw_event_duration(++(*ev), !need_pulse,
timings->clock);
i >>= 1;
}
if (timings->trailer_space) {
if (!(*ev)->pulse)
(*ev)->duration += timings->trailer_space;
else if (!max--)
goto nobufs;
else
init_ir_raw_event_duration(++(*ev), 0,
timings->trailer_space);
}
ret = 0;
nobufs:
/* point to the next event rather than last event before returning */
++(*ev);
return ret;
}
EXPORT_SYMBOL(ir_raw_gen_manchester);
/**
* ir_raw_gen_pd() - Encode data to raw events with pulse-distance modulation.
* @ev: Pointer to pointer to next free event. *@ev is incremented for
* each raw event filled.
* @max: Maximum number of raw events to fill.
* @timings: Pulse distance modulation timings.
* @n: Number of bits of data.
* @data: Data bits to encode.
*
* Encodes the @n least significant bits of @data using pulse-distance
* modulation with the timing characteristics described by @timings, writing up
* to @max raw IR events using the *@ev pointer.
*
* Returns: 0 on success.
* -ENOBUFS if there isn't enough space in the array to fit the
* full encoded data. In this case all @max events will have been
* written.
*/
int ir_raw_gen_pd(struct ir_raw_event **ev, unsigned int max,
const struct ir_raw_timings_pd *timings,
unsigned int n, u64 data)
{
int i;
int ret;
unsigned int space;
if (timings->header_pulse) {
ret = ir_raw_gen_pulse_space(ev, &max, timings->header_pulse,
timings->header_space);
if (ret)
return ret;
}
if (timings->msb_first) {
for (i = n - 1; i >= 0; --i) {
space = timings->bit_space[(data >> i) & 1];
ret = ir_raw_gen_pulse_space(ev, &max,
timings->bit_pulse,
space);
if (ret)
return ret;
}
} else {
for (i = 0; i < n; ++i, data >>= 1) {
space = timings->bit_space[data & 1];
ret = ir_raw_gen_pulse_space(ev, &max,
timings->bit_pulse,
space);
if (ret)
return ret;
}
}
ret = ir_raw_gen_pulse_space(ev, &max, timings->trailer_pulse,
timings->trailer_space);
return ret;
}
EXPORT_SYMBOL(ir_raw_gen_pd);
/**
* ir_raw_gen_pl() - Encode data to raw events with pulse-length modulation.
* @ev: Pointer to pointer to next free event. *@ev is incremented for
* each raw event filled.
* @max: Maximum number of raw events to fill.
* @timings: Pulse distance modulation timings.
* @n: Number of bits of data.
* @data: Data bits to encode.
*
* Encodes the @n least significant bits of @data using space-distance
* modulation with the timing characteristics described by @timings, writing up
* to @max raw IR events using the *@ev pointer.
*
* Returns: 0 on success.
* -ENOBUFS if there isn't enough space in the array to fit the
* full encoded data. In this case all @max events will have been
* written.
*/
int ir_raw_gen_pl(struct ir_raw_event **ev, unsigned int max,
const struct ir_raw_timings_pl *timings,
unsigned int n, u64 data)
{
int i;
int ret = -ENOBUFS;
unsigned int pulse;
if (!max--)
return ret;
init_ir_raw_event_duration((*ev)++, 1, timings->header_pulse);
if (timings->msb_first) {
for (i = n - 1; i >= 0; --i) {
if (!max--)
return ret;
init_ir_raw_event_duration((*ev)++, 0,
timings->bit_space);
if (!max--)
return ret;
pulse = timings->bit_pulse[(data >> i) & 1];
init_ir_raw_event_duration((*ev)++, 1, pulse);
}
} else {
for (i = 0; i < n; ++i, data >>= 1) {
if (!max--)
return ret;
init_ir_raw_event_duration((*ev)++, 0,
timings->bit_space);
if (!max--)
return ret;
pulse = timings->bit_pulse[data & 1];
init_ir_raw_event_duration((*ev)++, 1, pulse);
}
}
if (!max--)
return ret;
init_ir_raw_event_duration((*ev)++, 0, timings->trailer_space);
return 0;
}
EXPORT_SYMBOL(ir_raw_gen_pl);
/**
* ir_raw_encode_scancode() - Encode a scancode as raw events
*
* @protocol: protocol
* @scancode: scancode filter describing a single scancode
* @events: array of raw events to write into
* @max: max number of raw events
*
* Attempts to encode the scancode as raw events.
*
* Returns: The number of events written.
* -ENOBUFS if there isn't enough space in the array to fit the
* encoding. In this case all @max events will have been written.
* -EINVAL if the scancode is ambiguous or invalid, or if no
* compatible encoder was found.
*/
int ir_raw_encode_scancode(enum rc_proto protocol, u32 scancode,
struct ir_raw_event *events, unsigned int max)
{
struct ir_raw_handler *handler;
int ret = -EINVAL;
u64 mask = 1ULL << protocol;
ir_raw_load_modules(&mask);
mutex_lock(&ir_raw_handler_lock);
list_for_each_entry(handler, &ir_raw_handler_list, list) {
if (handler->protocols & mask && handler->encode) {
ret = handler->encode(protocol, scancode, events, max);
if (ret >= 0 || ret == -ENOBUFS)
break;
}
}
mutex_unlock(&ir_raw_handler_lock);
return ret;
}
EXPORT_SYMBOL(ir_raw_encode_scancode);
/**
* ir_raw_edge_handle() - Handle ir_raw_event_store_edge() processing
*
* @t: timer_list
*
* This callback is armed by ir_raw_event_store_edge(). It does two things:
* first of all, rather than calling ir_raw_event_handle() for each
* edge and waking up the rc thread, 15 ms after the first edge
* ir_raw_event_handle() is called. Secondly, generate a timeout event
* no more IR is received after the rc_dev timeout.
*/
static void ir_raw_edge_handle(struct timer_list *t)
{
struct ir_raw_event_ctrl *raw = from_timer(raw, t, edge_handle);
struct rc_dev *dev = raw->dev;
unsigned long flags;
ktime_t interval;
spin_lock_irqsave(&dev->raw->edge_spinlock, flags);
interval = ktime_sub(ktime_get(), dev->raw->last_event);
if (ktime_to_us(interval) >= dev->timeout) {
struct ir_raw_event ev = {
.timeout = true,
.duration = ktime_to_us(interval)
};
ir_raw_event_store(dev, &ev);
} else {
mod_timer(&dev->raw->edge_handle,
jiffies + usecs_to_jiffies(dev->timeout -
ktime_to_us(interval)));
}
spin_unlock_irqrestore(&dev->raw->edge_spinlock, flags);
ir_raw_event_handle(dev);
}
/**
* ir_raw_encode_carrier() - Get carrier used for protocol
*
* @protocol: protocol
*
* Attempts to find the carrier for the specified protocol
*
* Returns: The carrier in Hz
* -EINVAL if the protocol is invalid, or if no
* compatible encoder was found.
*/
int ir_raw_encode_carrier(enum rc_proto protocol)
{
struct ir_raw_handler *handler;
int ret = -EINVAL;
u64 mask = BIT_ULL(protocol);
mutex_lock(&ir_raw_handler_lock);
list_for_each_entry(handler, &ir_raw_handler_list, list) {
if (handler->protocols & mask && handler->encode) {
ret = handler->carrier;
break;
}
}
mutex_unlock(&ir_raw_handler_lock);
return ret;
}
EXPORT_SYMBOL(ir_raw_encode_carrier);
/*
* Used to (un)register raw event clients
*/
int ir_raw_event_prepare(struct rc_dev *dev)
{
if (!dev)
return -EINVAL;
dev->raw = kzalloc(sizeof(*dev->raw), GFP_KERNEL);
if (!dev->raw)
return -ENOMEM;
dev->raw->dev = dev;
dev->change_protocol = change_protocol;
dev->idle = true;
spin_lock_init(&dev->raw->edge_spinlock);
timer_setup(&dev->raw->edge_handle, ir_raw_edge_handle, 0);
INIT_KFIFO(dev->raw->kfifo);
return 0;
}
int ir_raw_event_register(struct rc_dev *dev)
{
struct task_struct *thread;
thread = kthread_run(ir_raw_event_thread, dev->raw, "rc%u", dev->minor);
if (IS_ERR(thread))
return PTR_ERR(thread);
dev->raw->thread = thread;
mutex_lock(&ir_raw_handler_lock);
list_add_tail(&dev->raw->list, &ir_raw_client_list);
mutex_unlock(&ir_raw_handler_lock);
return 0;
}
void ir_raw_event_free(struct rc_dev *dev)
{
if (!dev)
return;
kfree(dev->raw);
dev->raw = NULL;
}
void ir_raw_event_unregister(struct rc_dev *dev)
{
struct ir_raw_handler *handler;
if (!dev || !dev->raw)
return;
kthread_stop(dev->raw->thread);
del_timer_sync(&dev->raw->edge_handle);
mutex_lock(&ir_raw_handler_lock);
list_del(&dev->raw->list); list_for_each_entry(handler, &ir_raw_handler_list, list) if (handler->raw_unregister && (handler->protocols & dev->enabled_protocols)) handler->raw_unregister(dev);
lirc_bpf_free(dev);
ir_raw_event_free(dev);
/*
* A user can be calling bpf(BPF_PROG_{QUERY|ATTACH|DETACH}), so
* ensure that the raw member is null on unlock; this is how
* "device gone" is checked.
*/
mutex_unlock(&ir_raw_handler_lock);
}
/*
* Extension interface - used to register the IR decoders
*/
int ir_raw_handler_register(struct ir_raw_handler *ir_raw_handler)
{
mutex_lock(&ir_raw_handler_lock);
list_add_tail(&ir_raw_handler->list, &ir_raw_handler_list);
atomic64_or(ir_raw_handler->protocols, &available_protocols);
mutex_unlock(&ir_raw_handler_lock);
return 0;
}
EXPORT_SYMBOL(ir_raw_handler_register);
void ir_raw_handler_unregister(struct ir_raw_handler *ir_raw_handler)
{
struct ir_raw_event_ctrl *raw;
u64 protocols = ir_raw_handler->protocols;
mutex_lock(&ir_raw_handler_lock);
list_del(&ir_raw_handler->list);
list_for_each_entry(raw, &ir_raw_client_list, list) {
if (ir_raw_handler->raw_unregister &&
(raw->dev->enabled_protocols & protocols))
ir_raw_handler->raw_unregister(raw->dev);
ir_raw_disable_protocols(raw->dev, protocols);
}
atomic64_andnot(protocols, &available_protocols);
mutex_unlock(&ir_raw_handler_lock);
}
EXPORT_SYMBOL(ir_raw_handler_unregister);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_IRQDESC_H
#define _LINUX_IRQDESC_H
#include <linux/rcupdate.h>
#include <linux/kobject.h>
#include <linux/mutex.h>
/*
* Core internal functions to deal with irq descriptors
*/
struct irq_affinity_notify;
struct proc_dir_entry;
struct module;
struct irq_desc;
struct irq_domain;
struct pt_regs;
/**
* struct irq_desc - interrupt descriptor
* @irq_common_data: per irq and chip data passed down to chip functions
* @kstat_irqs: irq stats per cpu
* @handle_irq: highlevel irq-events handler
* @action: the irq action chain
* @status_use_accessors: status information
* @core_internal_state__do_not_mess_with_it: core internal status information
* @depth: disable-depth, for nested irq_disable() calls
* @wake_depth: enable depth, for multiple irq_set_irq_wake() callers
* @tot_count: stats field for non-percpu irqs
* @irq_count: stats field to detect stalled irqs
* @last_unhandled: aging timer for unhandled count
* @irqs_unhandled: stats field for spurious unhandled interrupts
* @threads_handled: stats field for deferred spurious detection of threaded handlers
* @threads_handled_last: comparator field for deferred spurious detection of threaded handlers
* @lock: locking for SMP
* @affinity_hint: hint to user space for preferred irq affinity
* @affinity_notify: context for notification of affinity changes
* @pending_mask: pending rebalanced interrupts
* @threads_oneshot: bitfield to handle shared oneshot threads
* @threads_active: number of irqaction threads currently running
* @wait_for_threads: wait queue for sync_irq to wait for threaded handlers
* @nr_actions: number of installed actions on this descriptor
* @no_suspend_depth: number of irqactions on a irq descriptor with
* IRQF_NO_SUSPEND set
* @force_resume_depth: number of irqactions on a irq descriptor with
* IRQF_FORCE_RESUME set
* @rcu: rcu head for delayed free
* @kobj: kobject used to represent this struct in sysfs
* @request_mutex: mutex to protect request/free before locking desc->lock
* @dir: /proc/irq/ procfs entry
* @debugfs_file: dentry for the debugfs file
* @name: flow handler name for /proc/interrupts output
*/
struct irq_desc {
struct irq_common_data irq_common_data;
struct irq_data irq_data;
unsigned int __percpu *kstat_irqs;
irq_flow_handler_t handle_irq;
struct irqaction *action; /* IRQ action list */
unsigned int status_use_accessors;
unsigned int core_internal_state__do_not_mess_with_it;
unsigned int depth; /* nested irq disables */
unsigned int wake_depth; /* nested wake enables */
unsigned int tot_count;
unsigned int irq_count; /* For detecting broken IRQs */
unsigned long last_unhandled; /* Aging timer for unhandled count */
unsigned int irqs_unhandled;
atomic_t threads_handled;
int threads_handled_last;
raw_spinlock_t lock;
struct cpumask *percpu_enabled;
const struct cpumask *percpu_affinity;
#ifdef CONFIG_SMP
const struct cpumask *affinity_hint;
struct irq_affinity_notify *affinity_notify;
#ifdef CONFIG_GENERIC_PENDING_IRQ
cpumask_var_t pending_mask;
#endif
#endif
unsigned long threads_oneshot;
atomic_t threads_active;
wait_queue_head_t wait_for_threads;
#ifdef CONFIG_PM_SLEEP
unsigned int nr_actions;
unsigned int no_suspend_depth;
unsigned int cond_suspend_depth;
unsigned int force_resume_depth;
#endif
#ifdef CONFIG_PROC_FS
struct proc_dir_entry *dir;
#endif
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
struct dentry *debugfs_file;
const char *dev_name;
#endif
#ifdef CONFIG_SPARSE_IRQ
struct rcu_head rcu;
struct kobject kobj;
#endif
struct mutex request_mutex;
int parent_irq;
struct module *owner;
const char *name;
#ifdef CONFIG_HARDIRQS_SW_RESEND
struct hlist_node resend_node;
#endif
} ____cacheline_internodealigned_in_smp;
#ifdef CONFIG_SPARSE_IRQ
extern void irq_lock_sparse(void);
extern void irq_unlock_sparse(void);
#else
static inline void irq_lock_sparse(void) { }
static inline void irq_unlock_sparse(void) { }
extern struct irq_desc irq_desc[NR_IRQS];
#endif
static inline unsigned int irq_desc_kstat_cpu(struct irq_desc *desc,
unsigned int cpu)
{
return desc->kstat_irqs ? *per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
}
static inline struct irq_desc *irq_data_to_desc(struct irq_data *data)
{
return container_of(data->common, struct irq_desc, irq_common_data);
}
static inline unsigned int irq_desc_get_irq(struct irq_desc *desc)
{
return desc->irq_data.irq;
}
static inline struct irq_data *irq_desc_get_irq_data(struct irq_desc *desc)
{
return &desc->irq_data;
}
static inline struct irq_chip *irq_desc_get_chip(struct irq_desc *desc)
{
return desc->irq_data.chip;
}
static inline void *irq_desc_get_chip_data(struct irq_desc *desc)
{
return desc->irq_data.chip_data;
}
static inline void *irq_desc_get_handler_data(struct irq_desc *desc)
{
return desc->irq_common_data.handler_data;
}
/*
* Architectures call this to let the generic IRQ layer
* handle an interrupt.
*/
static inline void generic_handle_irq_desc(struct irq_desc *desc)
{
desc->handle_irq(desc);
}
int handle_irq_desc(struct irq_desc *desc);
int generic_handle_irq(unsigned int irq);
int generic_handle_irq_safe(unsigned int irq);
#ifdef CONFIG_IRQ_DOMAIN
/*
* Convert a HW interrupt number to a logical one using a IRQ domain,
* and handle the result interrupt number. Return -EINVAL if
* conversion failed.
*/
int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq);
int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq);
int generic_handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq);
#endif
/* Test to see if a driver has successfully requested an irq */
static inline int irq_desc_has_action(struct irq_desc *desc)
{
return desc && desc->action != NULL;
}
/**
* irq_set_handler_locked - Set irq handler from a locked region
* @data: Pointer to the irq_data structure which identifies the irq
* @handler: Flow control handler function for this interrupt
*
* Sets the handler in the irq descriptor associated to @data.
*
* Must be called with irq_desc locked and valid parameters. Typical
* call site is the irq_set_type() callback.
*/
static inline void irq_set_handler_locked(struct irq_data *data,
irq_flow_handler_t handler)
{
struct irq_desc *desc = irq_data_to_desc(data);
desc->handle_irq = handler;
}
/**
* irq_set_chip_handler_name_locked - Set chip, handler and name from a locked region
* @data: Pointer to the irq_data structure for which the chip is set
* @chip: Pointer to the new irq chip
* @handler: Flow control handler function for this interrupt
* @name: Name of the interrupt
*
* Replace the irq chip at the proper hierarchy level in @data and
* sets the handler and name in the associated irq descriptor.
*
* Must be called with irq_desc locked and valid parameters.
*/
static inline void
irq_set_chip_handler_name_locked(struct irq_data *data,
const struct irq_chip *chip,
irq_flow_handler_t handler, const char *name)
{
struct irq_desc *desc = irq_data_to_desc(data);
desc->handle_irq = handler;
desc->name = name;
data->chip = (struct irq_chip *)chip;
}
bool irq_check_status_bit(unsigned int irq, unsigned int bitmask);
static inline bool irq_balancing_disabled(unsigned int irq)
{
return irq_check_status_bit(irq, IRQ_NO_BALANCING_MASK);
}
static inline bool irq_is_percpu(unsigned int irq)
{
return irq_check_status_bit(irq, IRQ_PER_CPU);
}
static inline bool irq_is_percpu_devid(unsigned int irq)
{
return irq_check_status_bit(irq, IRQ_PER_CPU_DEVID);
}
void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class,
struct lock_class_key *request_class);
static inline void
irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class,
struct lock_class_key *request_class)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
__irq_set_lockdep_class(irq, lock_class, request_class);
}
#endif
/*
* mm/rmap.c - physical to virtual reverse mappings
*
* Copyright 2001, Rik van Riel <riel@conectiva.com.br>
* Released under the General Public License (GPL).
*
* Simple, low overhead reverse mapping scheme.
* Please try to keep this thing as modular as possible.
*
* Provides methods for unmapping each kind of mapped page:
* the anon methods track anonymous pages, and
* the file methods track pages belonging to an inode.
*
* Original design by Rik van Riel <riel@conectiva.com.br> 2001
* File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
* Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
* Contributions by Hugh Dickins 2003, 2004
*/
/*
* Lock ordering in mm:
*
* inode->i_rwsem (while writing or truncating, not reading or faulting)
* mm->mmap_lock
* mapping->invalidate_lock (in filemap_fault)
* page->flags PG_locked (lock_page)
* hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
* vma_start_write
* mapping->i_mmap_rwsem
* anon_vma->rwsem
* mm->page_table_lock or pte_lock
* swap_lock (in swap_duplicate, swap_info_get)
* mmlist_lock (in mmput, drain_mmlist and others)
* mapping->private_lock (in block_dirty_folio)
* folio_lock_memcg move_lock (in block_dirty_folio)
* i_pages lock (widely used)
* lruvec->lru_lock (in folio_lruvec_lock_irq)
* inode->i_lock (in set_page_dirty's __mark_inode_dirty)
* bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
* sb_lock (within inode_lock in fs/fs-writeback.c)
* i_pages lock (widely used, in set_page_dirty,
* in arch-dependent flush_dcache_mmap_lock,
* within bdi.wb->list_lock in __sync_single_inode)
*
* anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon)
* ->tasklist_lock
* pte map lock
*
* hugetlbfs PageHuge() take locks in this order:
* hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
* vma_lock (hugetlb specific lock for pmd_sharing)
* mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
* page->flags PG_locked (lock_page)
*/
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/rcupdate.h>
#include <linux/export.h>
#include <linux/memcontrol.h>
#include <linux/mmu_notifier.h>
#include <linux/migrate.h>
#include <linux/hugetlb.h>
#include <linux/huge_mm.h>
#include <linux/backing-dev.h>
#include <linux/page_idle.h>
#include <linux/memremap.h>
#include <linux/userfaultfd_k.h>
#include <linux/mm_inline.h>
#include <asm/tlbflush.h>
#define CREATE_TRACE_POINTS
#include <trace/events/tlb.h>
#include <trace/events/migrate.h>
#include "internal.h"
static struct kmem_cache *anon_vma_cachep;
static struct kmem_cache *anon_vma_chain_cachep;
static inline struct anon_vma *anon_vma_alloc(void)
{
struct anon_vma *anon_vma;
anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
if (anon_vma) {
atomic_set(&anon_vma->refcount, 1);
anon_vma->num_children = 0;
anon_vma->num_active_vmas = 0;
anon_vma->parent = anon_vma;
/*
* Initialise the anon_vma root to point to itself. If called
* from fork, the root will be reset to the parents anon_vma.
*/
anon_vma->root = anon_vma;
}
return anon_vma;
}
static inline void anon_vma_free(struct anon_vma *anon_vma)
{
VM_BUG_ON(atomic_read(&anon_vma->refcount));
/*
* Synchronize against folio_lock_anon_vma_read() such that
* we can safely hold the lock without the anon_vma getting
* freed.
*
* Relies on the full mb implied by the atomic_dec_and_test() from
* put_anon_vma() against the acquire barrier implied by
* down_read_trylock() from folio_lock_anon_vma_read(). This orders:
*
* folio_lock_anon_vma_read() VS put_anon_vma()
* down_read_trylock() atomic_dec_and_test()
* LOCK MB
* atomic_read() rwsem_is_locked()
*
* LOCK should suffice since the actual taking of the lock must
* happen _before_ what follows.
*/
might_sleep();
if (rwsem_is_locked(&anon_vma->root->rwsem)) {
anon_vma_lock_write(anon_vma);
anon_vma_unlock_write(anon_vma);
}
kmem_cache_free(anon_vma_cachep, anon_vma);
}
static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
{
return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
}
static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
{
kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
}
static void anon_vma_chain_link(struct vm_area_struct *vma,
struct anon_vma_chain *avc,
struct anon_vma *anon_vma)
{
avc->vma = vma;
avc->anon_vma = anon_vma;
list_add(&avc->same_vma, &vma->anon_vma_chain);
anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
}
/**
* __anon_vma_prepare - attach an anon_vma to a memory region
* @vma: the memory region in question
*
* This makes sure the memory mapping described by 'vma' has
* an 'anon_vma' attached to it, so that we can associate the
* anonymous pages mapped into it with that anon_vma.
*
* The common case will be that we already have one, which
* is handled inline by anon_vma_prepare(). But if
* not we either need to find an adjacent mapping that we
* can re-use the anon_vma from (very common when the only
* reason for splitting a vma has been mprotect()), or we
* allocate a new one.
*
* Anon-vma allocations are very subtle, because we may have
* optimistically looked up an anon_vma in folio_lock_anon_vma_read()
* and that may actually touch the rwsem even in the newly
* allocated vma (it depends on RCU to make sure that the
* anon_vma isn't actually destroyed).
*
* As a result, we need to do proper anon_vma locking even
* for the new allocation. At the same time, we do not want
* to do any locking for the common case of already having
* an anon_vma.
*
* This must be called with the mmap_lock held for reading.
*/
int __anon_vma_prepare(struct vm_area_struct *vma)
{
struct mm_struct *mm = vma->vm_mm;
struct anon_vma *anon_vma, *allocated;
struct anon_vma_chain *avc;
might_sleep();
avc = anon_vma_chain_alloc(GFP_KERNEL);
if (!avc)
goto out_enomem;
anon_vma = find_mergeable_anon_vma(vma);
allocated = NULL;
if (!anon_vma) {
anon_vma = anon_vma_alloc();
if (unlikely(!anon_vma))
goto out_enomem_free_avc;
anon_vma->num_children++; /* self-parent link for new root */
allocated = anon_vma;
}
anon_vma_lock_write(anon_vma);
/* page_table_lock to protect against threads */
spin_lock(&mm->page_table_lock);
if (likely(!vma->anon_vma)) {
vma->anon_vma = anon_vma;
anon_vma_chain_link(vma, avc, anon_vma);
anon_vma->num_active_vmas++;
allocated = NULL;
avc = NULL;
}
spin_unlock(&mm->page_table_lock);
anon_vma_unlock_write(anon_vma);
if (unlikely(allocated))
put_anon_vma(allocated);
if (unlikely(avc))
anon_vma_chain_free(avc);
return 0;
out_enomem_free_avc:
anon_vma_chain_free(avc);
out_enomem:
return -ENOMEM;
}
/*
* This is a useful helper function for locking the anon_vma root as
* we traverse the vma->anon_vma_chain, looping over anon_vma's that
* have the same vma.
*
* Such anon_vma's should have the same root, so you'd expect to see
* just a single mutex_lock for the whole traversal.
*/
static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
{
struct anon_vma *new_root = anon_vma->root;
if (new_root != root) {
if (WARN_ON_ONCE(root))
up_write(&root->rwsem);
root = new_root;
down_write(&root->rwsem);
}
return root;
}
static inline void unlock_anon_vma_root(struct anon_vma *root)
{
if (root)
up_write(&root->rwsem);
}
/*
* Attach the anon_vmas from src to dst.
* Returns 0 on success, -ENOMEM on failure.
*
* anon_vma_clone() is called by vma_expand(), vma_merge(), __split_vma(),
* copy_vma() and anon_vma_fork(). The first four want an exact copy of src,
* while the last one, anon_vma_fork(), may try to reuse an existing anon_vma to
* prevent endless growth of anon_vma. Since dst->anon_vma is set to NULL before
* call, we can identify this case by checking (!dst->anon_vma &&
* src->anon_vma).
*
* If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
* and reuse existing anon_vma which has no vmas and only one child anon_vma.
* This prevents degradation of anon_vma hierarchy to endless linear chain in
* case of constantly forking task. On the other hand, an anon_vma with more
* than one child isn't reused even if there was no alive vma, thus rmap
* walker has a good chance of avoiding scanning the whole hierarchy when it
* searches where page is mapped.
*/
int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
{
struct anon_vma_chain *avc, *pavc;
struct anon_vma *root = NULL;
list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
struct anon_vma *anon_vma;
avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
if (unlikely(!avc)) {
unlock_anon_vma_root(root);
root = NULL;
avc = anon_vma_chain_alloc(GFP_KERNEL);
if (!avc)
goto enomem_failure;
}
anon_vma = pavc->anon_vma;
root = lock_anon_vma_root(root, anon_vma);
anon_vma_chain_link(dst, avc, anon_vma);
/*
* Reuse existing anon_vma if it has no vma and only one
* anon_vma child.
*
* Root anon_vma is never reused:
* it has self-parent reference and at least one child.
*/
if (!dst->anon_vma && src->anon_vma &&
anon_vma->num_children < 2 &&
anon_vma->num_active_vmas == 0)
dst->anon_vma = anon_vma;
}
if (dst->anon_vma)
dst->anon_vma->num_active_vmas++;
unlock_anon_vma_root(root);
return 0;
enomem_failure:
/*
* dst->anon_vma is dropped here otherwise its num_active_vmas can
* be incorrectly decremented in unlink_anon_vmas().
* We can safely do this because callers of anon_vma_clone() don't care
* about dst->anon_vma if anon_vma_clone() failed.
*/
dst->anon_vma = NULL;
unlink_anon_vmas(dst);
return -ENOMEM;
}
/*
* Attach vma to its own anon_vma, as well as to the anon_vmas that
* the corresponding VMA in the parent process is attached to.
* Returns 0 on success, non-zero on failure.
*/
int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
{
struct anon_vma_chain *avc;
struct anon_vma *anon_vma;
int error;
/* Don't bother if the parent process has no anon_vma here. */
if (!pvma->anon_vma)
return 0;
/* Drop inherited anon_vma, we'll reuse existing or allocate new. */
vma->anon_vma = NULL;
/*
* First, attach the new VMA to the parent VMA's anon_vmas,
* so rmap can find non-COWed pages in child processes.
*/
error = anon_vma_clone(vma, pvma);
if (error)
return error;
/* An existing anon_vma has been reused, all done then. */
if (vma->anon_vma)
return 0;
/* Then add our own anon_vma. */
anon_vma = anon_vma_alloc();
if (!anon_vma)
goto out_error;
anon_vma->num_active_vmas++;
avc = anon_vma_chain_alloc(GFP_KERNEL);
if (!avc)
goto out_error_free_anon_vma;
/*
* The root anon_vma's rwsem is the lock actually used when we
* lock any of the anon_vmas in this anon_vma tree.
*/
anon_vma->root = pvma->anon_vma->root;
anon_vma->parent = pvma->anon_vma;
/*
* With refcounts, an anon_vma can stay around longer than the
* process it belongs to. The root anon_vma needs to be pinned until
* this anon_vma is freed, because the lock lives in the root.
*/
get_anon_vma(anon_vma->root);
/* Mark this anon_vma as the one where our new (COWed) pages go. */
vma->anon_vma = anon_vma;
anon_vma_lock_write(anon_vma);
anon_vma_chain_link(vma, avc, anon_vma);
anon_vma->parent->num_children++;
anon_vma_unlock_write(anon_vma);
return 0;
out_error_free_anon_vma:
put_anon_vma(anon_vma);
out_error:
unlink_anon_vmas(vma);
return -ENOMEM;
}
void unlink_anon_vmas(struct vm_area_struct *vma)
{
struct anon_vma_chain *avc, *next;
struct anon_vma *root = NULL;
/*
* Unlink each anon_vma chained to the VMA. This list is ordered
* from newest to oldest, ensuring the root anon_vma gets freed last.
*/
list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
struct anon_vma *anon_vma = avc->anon_vma;
root = lock_anon_vma_root(root, anon_vma);
anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
/*
* Leave empty anon_vmas on the list - we'll need
* to free them outside the lock.
*/
if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
anon_vma->parent->num_children--;
continue;
}
list_del(&avc->same_vma);
anon_vma_chain_free(avc);
}
if (vma->anon_vma) {
vma->anon_vma->num_active_vmas--;
/*
* vma would still be needed after unlink, and anon_vma will be prepared
* when handle fault.
*/
vma->anon_vma = NULL;
}
unlock_anon_vma_root(root);
/*
* Iterate the list once more, it now only contains empty and unlinked
* anon_vmas, destroy them. Could not do before due to __put_anon_vma()
* needing to write-acquire the anon_vma->root->rwsem.
*/
list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
struct anon_vma *anon_vma = avc->anon_vma;
VM_WARN_ON(anon_vma->num_children);
VM_WARN_ON(anon_vma->num_active_vmas);
put_anon_vma(anon_vma);
list_del(&avc->same_vma);
anon_vma_chain_free(avc);
}
}
static void anon_vma_ctor(void *data)
{
struct anon_vma *anon_vma = data;
init_rwsem(&anon_vma->rwsem);
atomic_set(&anon_vma->refcount, 0);
anon_vma->rb_root = RB_ROOT_CACHED;
}
void __init anon_vma_init(void)
{
anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
anon_vma_ctor);
anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
SLAB_PANIC|SLAB_ACCOUNT);
}
/*
* Getting a lock on a stable anon_vma from a page off the LRU is tricky!
*
* Since there is no serialization what so ever against folio_remove_rmap_*()
* the best this function can do is return a refcount increased anon_vma
* that might have been relevant to this page.
*
* The page might have been remapped to a different anon_vma or the anon_vma
* returned may already be freed (and even reused).
*
* In case it was remapped to a different anon_vma, the new anon_vma will be a
* child of the old anon_vma, and the anon_vma lifetime rules will therefore
* ensure that any anon_vma obtained from the page will still be valid for as
* long as we observe page_mapped() [ hence all those page_mapped() tests ].
*
* All users of this function must be very careful when walking the anon_vma
* chain and verify that the page in question is indeed mapped in it
* [ something equivalent to page_mapped_in_vma() ].
*
* Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
* folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid
* if there is a mapcount, we can dereference the anon_vma after observing
* those.
*
* NOTE: the caller should normally hold folio lock when calling this. If
* not, the caller needs to double check the anon_vma didn't change after
* taking the anon_vma lock for either read or write (UFFDIO_MOVE can modify it
* concurrently without folio lock protection). See folio_lock_anon_vma_read()
* which has already covered that, and comment above remap_pages().
*/
struct anon_vma *folio_get_anon_vma(struct folio *folio)
{
struct anon_vma *anon_vma = NULL;
unsigned long anon_mapping;
rcu_read_lock();
anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
goto out;
if (!folio_mapped(folio))
goto out;
anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
if (!atomic_inc_not_zero(&anon_vma->refcount)) {
anon_vma = NULL;
goto out;
}
/*
* If this folio is still mapped, then its anon_vma cannot have been
* freed. But if it has been unmapped, we have no security against the
* anon_vma structure being freed and reused (for another anon_vma:
* SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
* above cannot corrupt).
*/
if (!folio_mapped(folio)) {
rcu_read_unlock();
put_anon_vma(anon_vma);
return NULL;
}
out:
rcu_read_unlock();
return anon_vma;
}
/*
* Similar to folio_get_anon_vma() except it locks the anon_vma.
*
* Its a little more complex as it tries to keep the fast path to a single
* atomic op -- the trylock. If we fail the trylock, we fall back to getting a
* reference like with folio_get_anon_vma() and then block on the mutex
* on !rwc->try_lock case.
*/
struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,
struct rmap_walk_control *rwc)
{
struct anon_vma *anon_vma = NULL;
struct anon_vma *root_anon_vma;
unsigned long anon_mapping;
retry:
rcu_read_lock();
anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
goto out;
if (!folio_mapped(folio))
goto out;
anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
root_anon_vma = READ_ONCE(anon_vma->root);
if (down_read_trylock(&root_anon_vma->rwsem)) {
/*
* folio_move_anon_rmap() might have changed the anon_vma as we
* might not hold the folio lock here.
*/
if (unlikely((unsigned long)READ_ONCE(folio->mapping) !=
anon_mapping)) {
up_read(&root_anon_vma->rwsem);
rcu_read_unlock();
goto retry;
}
/*
* If the folio is still mapped, then this anon_vma is still
* its anon_vma, and holding the mutex ensures that it will
* not go away, see anon_vma_free().
*/
if (!folio_mapped(folio)) {
up_read(&root_anon_vma->rwsem);
anon_vma = NULL;
}
goto out;
}
if (rwc && rwc->try_lock) {
anon_vma = NULL;
rwc->contended = true;
goto out;
}
/* trylock failed, we got to sleep */
if (!atomic_inc_not_zero(&anon_vma->refcount)) {
anon_vma = NULL;
goto out;
}
if (!folio_mapped(folio)) {
rcu_read_unlock();
put_anon_vma(anon_vma);
return NULL;
}
/* we pinned the anon_vma, its safe to sleep */
rcu_read_unlock();
anon_vma_lock_read(anon_vma);
/*
* folio_move_anon_rmap() might have changed the anon_vma as we might
* not hold the folio lock here.
*/
if (unlikely((unsigned long)READ_ONCE(folio->mapping) !=
anon_mapping)) {
anon_vma_unlock_read(anon_vma);
put_anon_vma(anon_vma);
anon_vma = NULL;
goto retry;
}
if (atomic_dec_and_test(&anon_vma->refcount)) {
/*
* Oops, we held the last refcount, release the lock
* and bail -- can't simply use put_anon_vma() because
* we'll deadlock on the anon_vma_lock_write() recursion.
*/
anon_vma_unlock_read(anon_vma);
__put_anon_vma(anon_vma);
anon_vma = NULL;
}
return anon_vma;
out:
rcu_read_unlock();
return anon_vma;
}
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
/*
* Flush TLB entries for recently unmapped pages from remote CPUs. It is
* important if a PTE was dirty when it was unmapped that it's flushed
* before any IO is initiated on the page to prevent lost writes. Similarly,
* it must be flushed before freeing to prevent data leakage.
*/
void try_to_unmap_flush(void)
{
struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
if (!tlb_ubc->flush_required)
return;
arch_tlbbatch_flush(&tlb_ubc->arch);
tlb_ubc->flush_required = false;
tlb_ubc->writable = false;
}
/* Flush iff there are potentially writable TLB entries that can race with IO */
void try_to_unmap_flush_dirty(void)
{
struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
if (tlb_ubc->writable)
try_to_unmap_flush();
}
/*
* Bits 0-14 of mm->tlb_flush_batched record pending generations.
* Bits 16-30 of mm->tlb_flush_batched bit record flushed generations.
*/
#define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16
#define TLB_FLUSH_BATCH_PENDING_MASK \
((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1)
#define TLB_FLUSH_BATCH_PENDING_LARGE \
(TLB_FLUSH_BATCH_PENDING_MASK / 2)
static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
unsigned long uaddr)
{
struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
int batch;
bool writable = pte_dirty(pteval);
if (!pte_accessible(mm, pteval))
return;
arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, uaddr);
tlb_ubc->flush_required = true;
/*
* Ensure compiler does not re-order the setting of tlb_flush_batched
* before the PTE is cleared.
*/
barrier();
batch = atomic_read(&mm->tlb_flush_batched);
retry:
if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) {
/*
* Prevent `pending' from catching up with `flushed' because of
* overflow. Reset `pending' and `flushed' to be 1 and 0 if
* `pending' becomes large.
*/
if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1))
goto retry;
} else {
atomic_inc(&mm->tlb_flush_batched);
}
/*
* If the PTE was dirty then it's best to assume it's writable. The
* caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
* before the page is queued for IO.
*/
if (writable)
tlb_ubc->writable = true;
}
/*
* Returns true if the TLB flush should be deferred to the end of a batch of
* unmap operations to reduce IPIs.
*/
static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
{
if (!(flags & TTU_BATCH_FLUSH))
return false;
return arch_tlbbatch_should_defer(mm);
}
/*
* Reclaim unmaps pages under the PTL but do not flush the TLB prior to
* releasing the PTL if TLB flushes are batched. It's possible for a parallel
* operation such as mprotect or munmap to race between reclaim unmapping
* the page and flushing the page. If this race occurs, it potentially allows
* access to data via a stale TLB entry. Tracking all mm's that have TLB
* batching in flight would be expensive during reclaim so instead track
* whether TLB batching occurred in the past and if so then do a flush here
* if required. This will cost one additional flush per reclaim cycle paid
* by the first operation at risk such as mprotect and mumap.
*
* This must be called under the PTL so that an access to tlb_flush_batched
* that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
* via the PTL.
*/
void flush_tlb_batched_pending(struct mm_struct *mm)
{
int batch = atomic_read(&mm->tlb_flush_batched);
int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK;
int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT;
if (pending != flushed) {
arch_flush_tlb_batched_pending(mm);
/*
* If the new TLB flushing is pending during flushing, leave
* mm->tlb_flush_batched as is, to avoid losing flushing.
*/
atomic_cmpxchg(&mm->tlb_flush_batched, batch,
pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT));
}
}
#else
static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
unsigned long uaddr)
{
}
static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
{
return false;
}
#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
/*
* At what user virtual address is page expected in vma?
* Caller should check the page is actually part of the vma.
*/
unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
{
struct folio *folio = page_folio(page);
if (folio_test_anon(folio)) {
struct anon_vma *page__anon_vma = folio_anon_vma(folio);
/*
* Note: swapoff's unuse_vma() is more efficient with this
* check, and needs it to match anon_vma when KSM is active.
*/
if (!vma->anon_vma || !page__anon_vma ||
vma->anon_vma->root != page__anon_vma->root)
return -EFAULT;
} else if (!vma->vm_file) {
return -EFAULT;
} else if (vma->vm_file->f_mapping != folio->mapping) {
return -EFAULT;
}
return vma_address(page, vma);
}
/*
* Returns the actual pmd_t* where we expect 'address' to be mapped from, or
* NULL if it doesn't exist. No guarantees / checks on what the pmd_t*
* represents.
*/
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd = NULL;
pgd = pgd_offset(mm, address);
if (!pgd_present(*pgd))
goto out;
p4d = p4d_offset(pgd, address);
if (!p4d_present(*p4d))
goto out;
pud = pud_offset(p4d, address);
if (!pud_present(*pud))
goto out;
pmd = pmd_offset(pud, address);
out:
return pmd;
}
struct folio_referenced_arg {
int mapcount;
int referenced;
unsigned long vm_flags;
struct mem_cgroup *memcg;
};
/*
* arg: folio_referenced_arg will be passed
*/
static bool folio_referenced_one(struct folio *folio,
struct vm_area_struct *vma, unsigned long address, void *arg)
{
struct folio_referenced_arg *pra = arg;
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
int referenced = 0;
unsigned long start = address, ptes = 0;
while (page_vma_mapped_walk(&pvmw)) {
address = pvmw.address;
if (vma->vm_flags & VM_LOCKED) {
if (!folio_test_large(folio) || !pvmw.pte) {
/* Restore the mlock which got missed */
mlock_vma_folio(folio, vma);
page_vma_mapped_walk_done(&pvmw);
pra->vm_flags |= VM_LOCKED;
return false; /* To break the loop */
}
/*
* For large folio fully mapped to VMA, will
* be handled after the pvmw loop.
*
* For large folio cross VMA boundaries, it's
* expected to be picked by page reclaim. But
* should skip reference of pages which are in
* the range of VM_LOCKED vma. As page reclaim
* should just count the reference of pages out
* the range of VM_LOCKED vma.
*/
ptes++;
pra->mapcount--;
continue;
}
if (pvmw.pte) {
if (lru_gen_enabled() &&
pte_young(ptep_get(pvmw.pte))) {
lru_gen_look_around(&pvmw);
referenced++;
}
if (ptep_clear_flush_young_notify(vma, address,
pvmw.pte))
referenced++;
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
if (pmdp_clear_flush_young_notify(vma, address,
pvmw.pmd))
referenced++;
} else {
/* unexpected pmd-mapped folio? */
WARN_ON_ONCE(1);
}
pra->mapcount--;
}
if ((vma->vm_flags & VM_LOCKED) &&
folio_test_large(folio) &&
folio_within_vma(folio, vma)) {
unsigned long s_align, e_align;
s_align = ALIGN_DOWN(start, PMD_SIZE);
e_align = ALIGN_DOWN(start + folio_size(folio) - 1, PMD_SIZE);
/* folio doesn't cross page table boundary and fully mapped */
if ((s_align == e_align) && (ptes == folio_nr_pages(folio))) {
/* Restore the mlock which got missed */
mlock_vma_folio(folio, vma);
pra->vm_flags |= VM_LOCKED;
return false; /* To break the loop */
}
}
if (referenced)
folio_clear_idle(folio);
if (folio_test_clear_young(folio))
referenced++;
if (referenced) {
pra->referenced++;
pra->vm_flags |= vma->vm_flags & ~VM_LOCKED;
}
if (!pra->mapcount)
return false; /* To break the loop */
return true;
}
static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
{
struct folio_referenced_arg *pra = arg;
struct mem_cgroup *memcg = pra->memcg;
/*
* Ignore references from this mapping if it has no recency. If the
* folio has been used in another mapping, we will catch it; if this
* other mapping is already gone, the unmap path will have set the
* referenced flag or activated the folio in zap_pte_range().
*/
if (!vma_has_recency(vma))
return true;
/*
* If we are reclaiming on behalf of a cgroup, skip counting on behalf
* of references from different cgroups.
*/
if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
return true;
return false;
}
/**
* folio_referenced() - Test if the folio was referenced.
* @folio: The folio to test.
* @is_locked: Caller holds lock on the folio.
* @memcg: target memory cgroup
* @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
*
* Quick test_and_clear_referenced for all mappings of a folio,
*
* Return: The number of mappings which referenced the folio. Return -1 if
* the function bailed out due to rmap lock contention.
*/
int folio_referenced(struct folio *folio, int is_locked,
struct mem_cgroup *memcg, unsigned long *vm_flags)
{
int we_locked = 0;
struct folio_referenced_arg pra = {
.mapcount = folio_mapcount(folio),
.memcg = memcg,
};
struct rmap_walk_control rwc = {
.rmap_one = folio_referenced_one,
.arg = (void *)&pra,
.anon_lock = folio_lock_anon_vma_read,
.try_lock = true,
.invalid_vma = invalid_folio_referenced_vma,
};
*vm_flags = 0;
if (!pra.mapcount)
return 0;
if (!folio_raw_mapping(folio))
return 0;
if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) {
we_locked = folio_trylock(folio);
if (!we_locked)
return 1;
}
rmap_walk(folio, &rwc);
*vm_flags = pra.vm_flags;
if (we_locked)
folio_unlock(folio);
return rwc.contended ? -1 : pra.referenced;
}
static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw)
{
int cleaned = 0;
struct vm_area_struct *vma = pvmw->vma;
struct mmu_notifier_range range;
unsigned long address = pvmw->address;
/*
* We have to assume the worse case ie pmd for invalidation. Note that
* the folio can not be freed from this function.
*/
mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
vma->vm_mm, address, vma_address_end(pvmw));
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(pvmw)) {
int ret = 0;
address = pvmw->address;
if (pvmw->pte) {
pte_t *pte = pvmw->pte;
pte_t entry = ptep_get(pte);
if (!pte_dirty(entry) && !pte_write(entry))
continue;
flush_cache_page(vma, address, pte_pfn(entry));
entry = ptep_clear_flush(vma, address, pte);
entry = pte_wrprotect(entry);
entry = pte_mkclean(entry);
set_pte_at(vma->vm_mm, address, pte, entry);
ret = 1;
} else {
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmd_t *pmd = pvmw->pmd;
pmd_t entry;
if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
continue;
flush_cache_range(vma, address,
address + HPAGE_PMD_SIZE);
entry = pmdp_invalidate(vma, address, pmd);
entry = pmd_wrprotect(entry);
entry = pmd_mkclean(entry);
set_pmd_at(vma->vm_mm, address, pmd, entry);
ret = 1;
#else
/* unexpected pmd-mapped folio? */
WARN_ON_ONCE(1);
#endif
}
if (ret)
cleaned++;
}
mmu_notifier_invalidate_range_end(&range);
return cleaned;
}
static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
unsigned long address, void *arg)
{
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
int *cleaned = arg;
*cleaned += page_vma_mkclean_one(&pvmw);
return true;
}
static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
{
if (vma->vm_flags & VM_SHARED)
return false;
return true;
}
int folio_mkclean(struct folio *folio)
{
int cleaned = 0;
struct address_space *mapping;
struct rmap_walk_control rwc = {
.arg = (void *)&cleaned,
.rmap_one = page_mkclean_one,
.invalid_vma = invalid_mkclean_vma,
};
BUG_ON(!folio_test_locked(folio));
if (!folio_mapped(folio))
return 0;
mapping = folio_mapping(folio);
if (!mapping)
return 0;
rmap_walk(folio, &rwc);
return cleaned;
}
EXPORT_SYMBOL_GPL(folio_mkclean);
/**
* pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of
* [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff)
* within the @vma of shared mappings. And since clean PTEs
* should also be readonly, write protects them too.
* @pfn: start pfn.
* @nr_pages: number of physically contiguous pages srarting with @pfn.
* @pgoff: page offset that the @pfn mapped with.
* @vma: vma that @pfn mapped within.
*
* Returns the number of cleaned PTEs (including PMDs).
*/
int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
struct vm_area_struct *vma)
{
struct page_vma_mapped_walk pvmw = {
.pfn = pfn,
.nr_pages = nr_pages,
.pgoff = pgoff,
.vma = vma,
.flags = PVMW_SYNC,
};
if (invalid_mkclean_vma(vma, NULL))
return 0;
pvmw.address = vma_pgoff_address(pgoff, nr_pages, vma);
VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);
return page_vma_mkclean_one(&pvmw);
}
int folio_total_mapcount(struct folio *folio)
{
int mapcount = folio_entire_mapcount(folio);
int nr_pages;
int i;
/* In the common case, avoid the loop when no pages mapped by PTE */
if (folio_nr_pages_mapped(folio) == 0)
return mapcount;
/*
* Add all the PTE mappings of those pages mapped by PTE.
* Limit the loop to folio_nr_pages_mapped()?
* Perhaps: given all the raciness, that may be a good or a bad idea.
*/
nr_pages = folio_nr_pages(folio);
for (i = 0; i < nr_pages; i++)
mapcount += atomic_read(&folio_page(folio, i)->_mapcount);
/* But each of those _mapcounts was based on -1 */
mapcount += nr_pages;
return mapcount;
}
static __always_inline unsigned int __folio_add_rmap(struct folio *folio,
struct page *page, int nr_pages, enum rmap_level level,
int *nr_pmdmapped)
{
atomic_t *mapped = &folio->_nr_pages_mapped;
int first, nr = 0;
__folio_rmap_sanity_checks(folio, page, nr_pages, level);
switch (level) {
case RMAP_LEVEL_PTE:
do {
first = atomic_inc_and_test(&page->_mapcount); if (first && folio_test_large(folio)) { first = atomic_inc_return_relaxed(mapped);
first = (first < ENTIRELY_MAPPED);
}
if (first)
nr++; } while (page++, --nr_pages > 0);
break;
case RMAP_LEVEL_PMD:
first = atomic_inc_and_test(&folio->_entire_mapcount);
if (first) {
nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);
if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {
*nr_pmdmapped = folio_nr_pages(folio);
nr = *nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
/* Raced ahead of a remove and another add? */
if (unlikely(nr < 0))
nr = 0;
} else {
/* Raced ahead of a remove of ENTIRELY_MAPPED */
nr = 0;
}
}
break;
}
return nr;
}
/**
* folio_move_anon_rmap - move a folio to our anon_vma
* @folio: The folio to move to our anon_vma
* @vma: The vma the folio belongs to
*
* When a folio belongs exclusively to one process after a COW event,
* that folio can be moved into the anon_vma that belongs to just that
* process, so the rmap code will not search the parent or sibling processes.
*/
void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma)
{
void *anon_vma = vma->anon_vma;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_VMA(!anon_vma, vma);
anon_vma += PAGE_MAPPING_ANON;
/*
* Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
* simultaneously, so a concurrent reader (eg folio_referenced()'s
* folio_test_anon()) will not see one without the other.
*/
WRITE_ONCE(folio->mapping, anon_vma);
}
/**
* __folio_set_anon - set up a new anonymous rmap for a folio
* @folio: The folio to set up the new anonymous rmap for.
* @vma: VM area to add the folio to.
* @address: User virtual address of the mapping
* @exclusive: Whether the folio is exclusive to the process.
*/
static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma,
unsigned long address, bool exclusive)
{
struct anon_vma *anon_vma = vma->anon_vma;
BUG_ON(!anon_vma);
/*
* If the folio isn't exclusive to this vma, we must use the _oldest_
* possible anon_vma for the folio mapping!
*/
if (!exclusive)
anon_vma = anon_vma->root;
/*
* page_idle does a lockless/optimistic rmap scan on folio->mapping.
* Make sure the compiler doesn't split the stores of anon_vma and
* the PAGE_MAPPING_ANON type identifier, otherwise the rmap code
* could mistake the mapping for a struct address_space and crash.
*/
anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
folio->index = linear_page_index(vma, address);
}
/**
* __page_check_anon_rmap - sanity check anonymous rmap addition
* @folio: The folio containing @page.
* @page: the page to check the mapping of
* @vma: the vm area in which the mapping is added
* @address: the user virtual address mapped
*/
static void __page_check_anon_rmap(struct folio *folio, struct page *page,
struct vm_area_struct *vma, unsigned long address)
{
/*
* The page's anon-rmap details (mapping and index) are guaranteed to
* be set up correctly at this point.
*
* We have exclusion against folio_add_anon_rmap_*() because the caller
* always holds the page locked.
*
* We have exclusion against folio_add_new_anon_rmap because those pages
* are initially only visible via the pagetables, and the pte is locked
* over the call to folio_add_new_anon_rmap.
*/
VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
folio);
VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
page);
}
static __always_inline void __folio_add_anon_rmap(struct folio *folio,
struct page *page, int nr_pages, struct vm_area_struct *vma,
unsigned long address, rmap_t flags, enum rmap_level level)
{
int i, nr, nr_pmdmapped = 0;
nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);
if (nr_pmdmapped)
__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped);
if (nr)
__lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr);
if (unlikely(!folio_test_anon(folio))) {
VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
/*
* For a PTE-mapped large folio, we only know that the single
* PTE is exclusive. Further, __folio_set_anon() might not get
* folio->index right when not given the address of the head
* page.
*/
VM_WARN_ON_FOLIO(folio_test_large(folio) &&
level != RMAP_LEVEL_PMD, folio);
__folio_set_anon(folio, vma, address,
!!(flags & RMAP_EXCLUSIVE));
} else if (likely(!folio_test_ksm(folio))) {
__page_check_anon_rmap(folio, page, vma, address);
}
if (flags & RMAP_EXCLUSIVE) {
switch (level) {
case RMAP_LEVEL_PTE:
for (i = 0; i < nr_pages; i++)
SetPageAnonExclusive(page + i);
break;
case RMAP_LEVEL_PMD:
SetPageAnonExclusive(page);
break;
}
}
for (i = 0; i < nr_pages; i++) {
struct page *cur_page = page + i;
/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
(folio_test_large(folio) &&
folio_entire_mapcount(folio) > 1)) &&
PageAnonExclusive(cur_page), folio);
}
/*
* For large folio, only mlock it if it's fully mapped to VMA. It's
* not easy to check whether the large folio is fully mapped to VMA
* here. Only mlock normal 4K folio and leave page reclaim to handle
* large folio.
*/
if (!folio_test_large(folio))
mlock_vma_folio(folio, vma);
}
/**
* folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
* @folio: The folio to add the mappings to
* @page: The first page to add
* @nr_pages: The number of pages which will be mapped
* @vma: The vm area in which the mappings are added
* @address: The user virtual address of the first page to map
* @flags: The rmap flags
*
* The page range of folio is defined by [first_page, first_page + nr_pages)
*
* The caller needs to hold the page table lock, and the page must be locked in
* the anon_vma case: to serialize mapping,index checking after setting,
* and to ensure that an anon folio is not being upgraded racily to a KSM folio
* (but KSM folios are never downgraded).
*/
void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
int nr_pages, struct vm_area_struct *vma, unsigned long address,
rmap_t flags)
{
__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
RMAP_LEVEL_PTE);
}
/**
* folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
* @folio: The folio to add the mapping to
* @page: The first page to add
* @vma: The vm area in which the mapping is added
* @address: The user virtual address of the first page to map
* @flags: The rmap flags
*
* The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
*
* The caller needs to hold the page table lock, and the page must be locked in
* the anon_vma case: to serialize mapping,index checking after setting.
*/
void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
struct vm_area_struct *vma, unsigned long address, rmap_t flags)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
RMAP_LEVEL_PMD);
#else
WARN_ON_ONCE(true);
#endif
}
/**
* folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
* @folio: The folio to add the mapping to.
* @vma: the vm area in which the mapping is added
* @address: the user virtual address mapped
*
* Like folio_add_anon_rmap_*() but must only be called on *new* folios.
* This means the inc-and-test can be bypassed.
* The folio does not have to be locked.
*
* If the folio is pmd-mappable, it is accounted as a THP. As the folio
* is new, it's assumed to be mapped exclusively by a single process.
*/
void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
unsigned long address)
{
int nr = folio_nr_pages(folio); VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); VM_BUG_ON_VMA(address < vma->vm_start ||
address + (nr << PAGE_SHIFT) > vma->vm_end, vma);
__folio_set_swapbacked(folio); __folio_set_anon(folio, vma, address, true);
if (likely(!folio_test_large(folio))) {
/* increment count (starts at -1) */
atomic_set(&folio->_mapcount, 0); SetPageAnonExclusive(&folio->page);
} else if (!folio_test_pmd_mappable(folio)) {
int i;
for (i = 0; i < nr; i++) { struct page *page = folio_page(folio, i);
/* increment count (starts at -1) */
atomic_set(&page->_mapcount, 0);
SetPageAnonExclusive(page);
}
atomic_set(&folio->_nr_pages_mapped, nr);
} else {
/* increment count (starts at -1) */
atomic_set(&folio->_entire_mapcount, 0);
atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);
SetPageAnonExclusive(&folio->page);
__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr);
}
__lruvec_stat_mod_folio(folio, NR_ANON_MAPPED, nr);
}
static __always_inline void __folio_add_file_rmap(struct folio *folio,
struct page *page, int nr_pages, struct vm_area_struct *vma,
enum rmap_level level)
{
int nr, nr_pmdmapped = 0;
VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); nr = __folio_add_rmap(folio, page, nr_pages, level, &nr_pmdmapped);
if (nr_pmdmapped)
__lruvec_stat_mod_folio(folio, folio_test_swapbacked(folio) ?
NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped);
if (nr) __lruvec_stat_mod_folio(folio, NR_FILE_MAPPED, nr);
/* See comments in folio_add_anon_rmap_*() */
if (!folio_test_large(folio)) mlock_vma_folio(folio, vma);
}
/**
* folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio
* @folio: The folio to add the mappings to
* @page: The first page to add
* @nr_pages: The number of pages that will be mapped using PTEs
* @vma: The vm area in which the mappings are added
*
* The page range of the folio is defined by [page, page + nr_pages)
*
* The caller needs to hold the page table lock.
*/
void folio_add_file_rmap_ptes(struct folio *folio, struct page *page,
int nr_pages, struct vm_area_struct *vma)
{
__folio_add_file_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);
}
/**
* folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio
* @folio: The folio to add the mapping to
* @page: The first page to add
* @vma: The vm area in which the mapping is added
*
* The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
*
* The caller needs to hold the page table lock.
*/
void folio_add_file_rmap_pmd(struct folio *folio, struct page *page,
struct vm_area_struct *vma)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
__folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);
#else
WARN_ON_ONCE(true);
#endif
}
static __always_inline void __folio_remove_rmap(struct folio *folio,
struct page *page, int nr_pages, struct vm_area_struct *vma,
enum rmap_level level)
{
atomic_t *mapped = &folio->_nr_pages_mapped;
int last, nr = 0, nr_pmdmapped = 0;
enum node_stat_item idx;
__folio_rmap_sanity_checks(folio, page, nr_pages, level);
switch (level) {
case RMAP_LEVEL_PTE:
do {
last = atomic_add_negative(-1, &page->_mapcount);
if (last && folio_test_large(folio)) {
last = atomic_dec_return_relaxed(mapped);
last = (last < ENTIRELY_MAPPED);
}
if (last)
nr++;
} while (page++, --nr_pages > 0);
break;
case RMAP_LEVEL_PMD:
last = atomic_add_negative(-1, &folio->_entire_mapcount);
if (last) {
nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);
if (likely(nr < ENTIRELY_MAPPED)) {
nr_pmdmapped = folio_nr_pages(folio);
nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
/* Raced ahead of another remove and an add? */
if (unlikely(nr < 0))
nr = 0;
} else {
/* An add of ENTIRELY_MAPPED raced ahead */
nr = 0;
}
}
break;
}
if (nr_pmdmapped) {
if (folio_test_anon(folio))
idx = NR_ANON_THPS;
else if (folio_test_swapbacked(folio))
idx = NR_SHMEM_PMDMAPPED;
else
idx = NR_FILE_PMDMAPPED;
__lruvec_stat_mod_folio(folio, idx, -nr_pmdmapped);
}
if (nr) {
idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;
__lruvec_stat_mod_folio(folio, idx, -nr);
/*
* Queue anon large folio for deferred split if at least one
* page of the folio is unmapped and at least one page
* is still mapped.
*/
if (folio_test_large(folio) && folio_test_anon(folio))
if (level == RMAP_LEVEL_PTE || nr < nr_pmdmapped)
deferred_split_folio(folio);
}
/*
* It would be tidy to reset folio_test_anon mapping when fully
* unmapped, but that might overwrite a racing folio_add_anon_rmap_*()
* which increments mapcount after us but sets mapping before us:
* so leave the reset to free_pages_prepare, and remember that
* it's only reliable while mapped.
*/
munlock_vma_folio(folio, vma);
}
/**
* folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio
* @folio: The folio to remove the mappings from
* @page: The first page to remove
* @nr_pages: The number of pages that will be removed from the mapping
* @vma: The vm area from which the mappings are removed
*
* The page range of the folio is defined by [page, page + nr_pages)
*
* The caller needs to hold the page table lock.
*/
void folio_remove_rmap_ptes(struct folio *folio, struct page *page,
int nr_pages, struct vm_area_struct *vma)
{
__folio_remove_rmap(folio, page, nr_pages, vma, RMAP_LEVEL_PTE);
}
/**
* folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio
* @folio: The folio to remove the mapping from
* @page: The first page to remove
* @vma: The vm area from which the mapping is removed
*
* The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
*
* The caller needs to hold the page table lock.
*/
void folio_remove_rmap_pmd(struct folio *folio, struct page *page,
struct vm_area_struct *vma)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
__folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, RMAP_LEVEL_PMD);
#else
WARN_ON_ONCE(true);
#endif
}
/*
* @arg: enum ttu_flags will be passed to this argument
*/
static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
unsigned long address, void *arg)
{
struct mm_struct *mm = vma->vm_mm;
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
pte_t pteval;
struct page *subpage;
bool anon_exclusive, ret = true;
struct mmu_notifier_range range;
enum ttu_flags flags = (enum ttu_flags)(long)arg;
unsigned long pfn;
unsigned long hsz = 0;
/*
* When racing against e.g. zap_pte_range() on another cpu,
* in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
* try_to_unmap() may return before page_mapped() has become false,
* if page table locking is skipped: use TTU_SYNC to wait for that.
*/
if (flags & TTU_SYNC)
pvmw.flags = PVMW_SYNC;
if (flags & TTU_SPLIT_HUGE_PMD)
split_huge_pmd_address(vma, address, false, folio);
/*
* For THP, we have to assume the worse case ie pmd for invalidation.
* For hugetlb, it could be much worse if we need to do pud
* invalidation in the case of pmd sharing.
*
* Note that the folio can not be freed in this function as call of
* try_to_unmap() must hold a reference on the folio.
*/
range.end = vma_address_end(&pvmw);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
address, range.end);
if (folio_test_hugetlb(folio)) {
/*
* If sharing is possible, start and end will be adjusted
* accordingly.
*/
adjust_range_if_pmd_sharing_possible(vma, &range.start,
&range.end);
/* We need the huge page size for set_huge_pte_at() */
hsz = huge_page_size(hstate_vma(vma));
}
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(&pvmw)) {
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
/*
* If the folio is in an mlock()d vma, we must not swap it out.
*/
if (!(flags & TTU_IGNORE_MLOCK) &&
(vma->vm_flags & VM_LOCKED)) {
/* Restore the mlock which got missed */
if (!folio_test_large(folio))
mlock_vma_folio(folio, vma);
page_vma_mapped_walk_done(&pvmw);
ret = false;
break;
}
pfn = pte_pfn(ptep_get(pvmw.pte));
subpage = folio_page(folio, pfn - folio_pfn(folio));
address = pvmw.address;
anon_exclusive = folio_test_anon(folio) &&
PageAnonExclusive(subpage);
if (folio_test_hugetlb(folio)) {
bool anon = folio_test_anon(folio);
/*
* The try_to_unmap() is only passed a hugetlb page
* in the case where the hugetlb page is poisoned.
*/
VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage);
/*
* huge_pmd_unshare may unmap an entire PMD page.
* There is no way of knowing exactly which PMDs may
* be cached for this mm, so we must flush them all.
* start/end were already adjusted above to cover this
* range.
*/
flush_cache_range(vma, range.start, range.end);
/*
* To call huge_pmd_unshare, i_mmap_rwsem must be
* held in write mode. Caller needs to explicitly
* do this outside rmap routines.
*
* We also must hold hugetlb vma_lock in write mode.
* Lock order dictates acquiring vma_lock BEFORE
* i_mmap_rwsem. We can only try lock here and fail
* if unsuccessful.
*/
if (!anon) {
VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
if (!hugetlb_vma_trylock_write(vma)) {
page_vma_mapped_walk_done(&pvmw);
ret = false;
break;
}
if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
hugetlb_vma_unlock_write(vma);
flush_tlb_range(vma,
range.start, range.end);
/*
* The ref count of the PMD page was
* dropped which is part of the way map
* counting is done for shared PMDs.
* Return 'true' here. When there is
* no other sharing, huge_pmd_unshare
* returns false and we will unmap the
* actual page and drop map count
* to zero.
*/
page_vma_mapped_walk_done(&pvmw);
break;
}
hugetlb_vma_unlock_write(vma);
}
pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
} else {
flush_cache_page(vma, address, pfn);
/* Nuke the page table entry. */
if (should_defer_flush(mm, flags)) {
/*
* We clear the PTE but do not flush so potentially
* a remote CPU could still be writing to the folio.
* If the entry was previously clean then the
* architecture must guarantee that a clear->dirty
* transition on a cached TLB entry is written through
* and traps if the PTE is unmapped.
*/
pteval = ptep_get_and_clear(mm, address, pvmw.pte);
set_tlb_ubc_flush_pending(mm, pteval, address);
} else {
pteval = ptep_clear_flush(vma, address, pvmw.pte);
}
}
/*
* Now the pte is cleared. If this pte was uffd-wp armed,
* we may want to replace a none pte with a marker pte if
* it's file-backed, so we don't lose the tracking info.
*/
pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);
/* Set the dirty flag on the folio now the pte is gone. */
if (pte_dirty(pteval))
folio_mark_dirty(folio);
/* Update high watermark before we lower rss */
update_hiwater_rss(mm);
if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) {
pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
if (folio_test_hugetlb(folio)) {
hugetlb_count_sub(folio_nr_pages(folio), mm);
set_huge_pte_at(mm, address, pvmw.pte, pteval,
hsz);
} else {
dec_mm_counter(mm, mm_counter(folio));
set_pte_at(mm, address, pvmw.pte, pteval);
}
} else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
/*
* The guest indicated that the page content is of no
* interest anymore. Simply discard the pte, vmscan
* will take care of the rest.
* A future reference will then fault in a new zero
* page. When userfaultfd is active, we must not drop
* this page though, as its main user (postcopy
* migration) will not expect userfaults on already
* copied pages.
*/
dec_mm_counter(mm, mm_counter(folio));
} else if (folio_test_anon(folio)) {
swp_entry_t entry = page_swap_entry(subpage);
pte_t swp_pte;
/*
* Store the swap location in the pte.
* See handle_pte_fault() ...
*/
if (unlikely(folio_test_swapbacked(folio) !=
folio_test_swapcache(folio))) {
WARN_ON_ONCE(1);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
/* MADV_FREE page check */
if (!folio_test_swapbacked(folio)) {
int ref_count, map_count;
/*
* Synchronize with gup_pte_range():
* - clear PTE; barrier; read refcount
* - inc refcount; barrier; read PTE
*/
smp_mb();
ref_count = folio_ref_count(folio);
map_count = folio_mapcount(folio);
/*
* Order reads for page refcount and dirty flag
* (see comments in __remove_mapping()).
*/
smp_rmb();
/*
* The only page refs must be one from isolation
* plus the rmap(s) (dropped by discard:).
*/
if (ref_count == 1 + map_count &&
!folio_test_dirty(folio)) {
dec_mm_counter(mm, MM_ANONPAGES);
goto discard;
}
/*
* If the folio was redirtied, it cannot be
* discarded. Remap the page to page table.
*/
set_pte_at(mm, address, pvmw.pte, pteval);
folio_set_swapbacked(folio);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
if (swap_duplicate(entry) < 0) {
set_pte_at(mm, address, pvmw.pte, pteval);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
if (arch_unmap_one(mm, vma, address, pteval) < 0) {
swap_free(entry);
set_pte_at(mm, address, pvmw.pte, pteval);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
/* See folio_try_share_anon_rmap(): clear PTE first. */
if (anon_exclusive &&
folio_try_share_anon_rmap_pte(folio, subpage)) {
swap_free(entry);
set_pte_at(mm, address, pvmw.pte, pteval);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
if (list_empty(&mm->mmlist)) {
spin_lock(&mmlist_lock);
if (list_empty(&mm->mmlist))
list_add(&mm->mmlist, &init_mm.mmlist);
spin_unlock(&mmlist_lock);
}
dec_mm_counter(mm, MM_ANONPAGES);
inc_mm_counter(mm, MM_SWAPENTS);
swp_pte = swp_entry_to_pte(entry);
if (anon_exclusive)
swp_pte = pte_swp_mkexclusive(swp_pte);
if (pte_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte);
if (pte_uffd_wp(pteval))
swp_pte = pte_swp_mkuffd_wp(swp_pte);
set_pte_at(mm, address, pvmw.pte, swp_pte);
} else {
/*
* This is a locked file-backed folio,
* so it cannot be removed from the page
* cache and replaced by a new folio before
* mmu_notifier_invalidate_range_end, so no
* concurrent thread might update its page table
* to point at a new folio while a device is
* still using this folio.
*
* See Documentation/mm/mmu_notifier.rst
*/
dec_mm_counter(mm, mm_counter_file(folio));
}
discard:
if (unlikely(folio_test_hugetlb(folio)))
hugetlb_remove_rmap(folio);
else
folio_remove_rmap_pte(folio, subpage, vma);
if (vma->vm_flags & VM_LOCKED)
mlock_drain_local();
folio_put(folio);
}
mmu_notifier_invalidate_range_end(&range);
return ret;
}
static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
{
return vma_is_temporary_stack(vma);
}
static int folio_not_mapped(struct folio *folio)
{
return !folio_mapped(folio);
}
/**
* try_to_unmap - Try to remove all page table mappings to a folio.
* @folio: The folio to unmap.
* @flags: action and flags
*
* Tries to remove all the page table entries which are mapping this
* folio. It is the caller's responsibility to check if the folio is
* still mapped if needed (use TTU_SYNC to prevent accounting races).
*
* Context: Caller must hold the folio lock.
*/
void try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
struct rmap_walk_control rwc = {
.rmap_one = try_to_unmap_one,
.arg = (void *)flags,
.done = folio_not_mapped,
.anon_lock = folio_lock_anon_vma_read,
};
if (flags & TTU_RMAP_LOCKED)
rmap_walk_locked(folio, &rwc);
else
rmap_walk(folio, &rwc);
}
/*
* @arg: enum ttu_flags will be passed to this argument.
*
* If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
* containing migration entries.
*/
static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
unsigned long address, void *arg)
{
struct mm_struct *mm = vma->vm_mm;
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
pte_t pteval;
struct page *subpage;
bool anon_exclusive, ret = true;
struct mmu_notifier_range range;
enum ttu_flags flags = (enum ttu_flags)(long)arg;
unsigned long pfn;
unsigned long hsz = 0;
/*
* When racing against e.g. zap_pte_range() on another cpu,
* in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
* try_to_migrate() may return before page_mapped() has become false,
* if page table locking is skipped: use TTU_SYNC to wait for that.
*/
if (flags & TTU_SYNC)
pvmw.flags = PVMW_SYNC;
/*
* unmap_page() in mm/huge_memory.c is the only user of migration with
* TTU_SPLIT_HUGE_PMD and it wants to freeze.
*/
if (flags & TTU_SPLIT_HUGE_PMD)
split_huge_pmd_address(vma, address, true, folio);
/*
* For THP, we have to assume the worse case ie pmd for invalidation.
* For hugetlb, it could be much worse if we need to do pud
* invalidation in the case of pmd sharing.
*
* Note that the page can not be free in this function as call of
* try_to_unmap() must hold a reference on the page.
*/
range.end = vma_address_end(&pvmw);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
address, range.end);
if (folio_test_hugetlb(folio)) {
/*
* If sharing is possible, start and end will be adjusted
* accordingly.
*/
adjust_range_if_pmd_sharing_possible(vma, &range.start,
&range.end);
/* We need the huge page size for set_huge_pte_at() */
hsz = huge_page_size(hstate_vma(vma));
}
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(&pvmw)) {
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
/* PMD-mapped THP migration entry */
if (!pvmw.pte) {
subpage = folio_page(folio,
pmd_pfn(*pvmw.pmd) - folio_pfn(folio));
VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
!folio_test_pmd_mappable(folio), folio);
if (set_pmd_migration_entry(&pvmw, subpage)) {
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
continue;
}
#endif
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
pfn = pte_pfn(ptep_get(pvmw.pte));
if (folio_is_zone_device(folio)) {
/*
* Our PTE is a non-present device exclusive entry and
* calculating the subpage as for the common case would
* result in an invalid pointer.
*
* Since only PAGE_SIZE pages can currently be
* migrated, just set it to page. This will need to be
* changed when hugepage migrations to device private
* memory are supported.
*/
VM_BUG_ON_FOLIO(folio_nr_pages(folio) > 1, folio);
subpage = &folio->page;
} else {
subpage = folio_page(folio, pfn - folio_pfn(folio));
}
address = pvmw.address;
anon_exclusive = folio_test_anon(folio) &&
PageAnonExclusive(subpage);
if (folio_test_hugetlb(folio)) {
bool anon = folio_test_anon(folio);
/*
* huge_pmd_unshare may unmap an entire PMD page.
* There is no way of knowing exactly which PMDs may
* be cached for this mm, so we must flush them all.
* start/end were already adjusted above to cover this
* range.
*/
flush_cache_range(vma, range.start, range.end);
/*
* To call huge_pmd_unshare, i_mmap_rwsem must be
* held in write mode. Caller needs to explicitly
* do this outside rmap routines.
*
* We also must hold hugetlb vma_lock in write mode.
* Lock order dictates acquiring vma_lock BEFORE
* i_mmap_rwsem. We can only try lock here and
* fail if unsuccessful.
*/
if (!anon) {
VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
if (!hugetlb_vma_trylock_write(vma)) {
page_vma_mapped_walk_done(&pvmw);
ret = false;
break;
}
if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
hugetlb_vma_unlock_write(vma);
flush_tlb_range(vma,
range.start, range.end);
/*
* The ref count of the PMD page was
* dropped which is part of the way map
* counting is done for shared PMDs.
* Return 'true' here. When there is
* no other sharing, huge_pmd_unshare
* returns false and we will unmap the
* actual page and drop map count
* to zero.
*/
page_vma_mapped_walk_done(&pvmw);
break;
}
hugetlb_vma_unlock_write(vma);
}
/* Nuke the hugetlb page table entry */
pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
} else {
flush_cache_page(vma, address, pfn);
/* Nuke the page table entry. */
if (should_defer_flush(mm, flags)) {
/*
* We clear the PTE but do not flush so potentially
* a remote CPU could still be writing to the folio.
* If the entry was previously clean then the
* architecture must guarantee that a clear->dirty
* transition on a cached TLB entry is written through
* and traps if the PTE is unmapped.
*/
pteval = ptep_get_and_clear(mm, address, pvmw.pte);
set_tlb_ubc_flush_pending(mm, pteval, address);
} else {
pteval = ptep_clear_flush(vma, address, pvmw.pte);
}
}
/* Set the dirty flag on the folio now the pte is gone. */
if (pte_dirty(pteval))
folio_mark_dirty(folio);
/* Update high watermark before we lower rss */
update_hiwater_rss(mm);
if (folio_is_device_private(folio)) {
unsigned long pfn = folio_pfn(folio);
swp_entry_t entry;
pte_t swp_pte;
if (anon_exclusive)
WARN_ON_ONCE(folio_try_share_anon_rmap_pte(folio,
subpage));
/*
* Store the pfn of the page in a special migration
* pte. do_swap_page() will wait until the migration
* pte is removed and then restart fault handling.
*/
entry = pte_to_swp_entry(pteval);
if (is_writable_device_private_entry(entry))
entry = make_writable_migration_entry(pfn);
else if (anon_exclusive)
entry = make_readable_exclusive_migration_entry(pfn);
else
entry = make_readable_migration_entry(pfn);
swp_pte = swp_entry_to_pte(entry);
/*
* pteval maps a zone device page and is therefore
* a swap pte.
*/
if (pte_swp_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte);
if (pte_swp_uffd_wp(pteval))
swp_pte = pte_swp_mkuffd_wp(swp_pte);
set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
trace_set_migration_pte(pvmw.address, pte_val(swp_pte),
folio_order(folio));
/*
* No need to invalidate here it will synchronize on
* against the special swap migration pte.
*/
} else if (PageHWPoison(subpage)) {
pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
if (folio_test_hugetlb(folio)) {
hugetlb_count_sub(folio_nr_pages(folio), mm);
set_huge_pte_at(mm, address, pvmw.pte, pteval,
hsz);
} else {
dec_mm_counter(mm, mm_counter(folio));
set_pte_at(mm, address, pvmw.pte, pteval);
}
} else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
/*
* The guest indicated that the page content is of no
* interest anymore. Simply discard the pte, vmscan
* will take care of the rest.
* A future reference will then fault in a new zero
* page. When userfaultfd is active, we must not drop
* this page though, as its main user (postcopy
* migration) will not expect userfaults on already
* copied pages.
*/
dec_mm_counter(mm, mm_counter(folio));
} else {
swp_entry_t entry;
pte_t swp_pte;
if (arch_unmap_one(mm, vma, address, pteval) < 0) {
if (folio_test_hugetlb(folio))
set_huge_pte_at(mm, address, pvmw.pte,
pteval, hsz);
else
set_pte_at(mm, address, pvmw.pte, pteval);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
VM_BUG_ON_PAGE(pte_write(pteval) && folio_test_anon(folio) &&
!anon_exclusive, subpage);
/* See folio_try_share_anon_rmap_pte(): clear PTE first. */
if (folio_test_hugetlb(folio)) {
if (anon_exclusive &&
hugetlb_try_share_anon_rmap(folio)) {
set_huge_pte_at(mm, address, pvmw.pte,
pteval, hsz);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
} else if (anon_exclusive &&
folio_try_share_anon_rmap_pte(folio, subpage)) {
set_pte_at(mm, address, pvmw.pte, pteval);
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
/*
* Store the pfn of the page in a special migration
* pte. do_swap_page() will wait until the migration
* pte is removed and then restart fault handling.
*/
if (pte_write(pteval))
entry = make_writable_migration_entry(
page_to_pfn(subpage));
else if (anon_exclusive)
entry = make_readable_exclusive_migration_entry(
page_to_pfn(subpage));
else
entry = make_readable_migration_entry(
page_to_pfn(subpage));
if (pte_young(pteval))
entry = make_migration_entry_young(entry);
if (pte_dirty(pteval))
entry = make_migration_entry_dirty(entry);
swp_pte = swp_entry_to_pte(entry);
if (pte_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte);
if (pte_uffd_wp(pteval))
swp_pte = pte_swp_mkuffd_wp(swp_pte);
if (folio_test_hugetlb(folio))
set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
hsz);
else
set_pte_at(mm, address, pvmw.pte, swp_pte);
trace_set_migration_pte(address, pte_val(swp_pte),
folio_order(folio));
/*
* No need to invalidate here it will synchronize on
* against the special swap migration pte.
*/
}
if (unlikely(folio_test_hugetlb(folio)))
hugetlb_remove_rmap(folio);
else
folio_remove_rmap_pte(folio, subpage, vma);
if (vma->vm_flags & VM_LOCKED)
mlock_drain_local();
folio_put(folio);
}
mmu_notifier_invalidate_range_end(&range);
return ret;
}
/**
* try_to_migrate - try to replace all page table mappings with swap entries
* @folio: the folio to replace page table entries for
* @flags: action and flags
*
* Tries to remove all the page table entries which are mapping this folio and
* replace them with special swap entries. Caller must hold the folio lock.
*/
void try_to_migrate(struct folio *folio, enum ttu_flags flags)
{
struct rmap_walk_control rwc = {
.rmap_one = try_to_migrate_one,
.arg = (void *)flags,
.done = folio_not_mapped,
.anon_lock = folio_lock_anon_vma_read,
};
/*
* Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
* TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
*/
if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
TTU_SYNC | TTU_BATCH_FLUSH)))
return;
if (folio_is_zone_device(folio) &&
(!folio_is_device_private(folio) && !folio_is_device_coherent(folio)))
return;
/*
* During exec, a temporary VMA is setup and later moved.
* The VMA is moved under the anon_vma lock but not the
* page tables leading to a race where migration cannot
* find the migration ptes. Rather than increasing the
* locking requirements of exec(), migration skips
* temporary VMAs until after exec() completes.
*/
if (!folio_test_ksm(folio) && folio_test_anon(folio))
rwc.invalid_vma = invalid_migration_vma;
if (flags & TTU_RMAP_LOCKED)
rmap_walk_locked(folio, &rwc);
else
rmap_walk(folio, &rwc);
}
#ifdef CONFIG_DEVICE_PRIVATE
struct make_exclusive_args {
struct mm_struct *mm;
unsigned long address;
void *owner;
bool valid;
};
static bool page_make_device_exclusive_one(struct folio *folio,
struct vm_area_struct *vma, unsigned long address, void *priv)
{
struct mm_struct *mm = vma->vm_mm;
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
struct make_exclusive_args *args = priv;
pte_t pteval;
struct page *subpage;
bool ret = true;
struct mmu_notifier_range range;
swp_entry_t entry;
pte_t swp_pte;
pte_t ptent;
mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
vma->vm_mm, address, min(vma->vm_end,
address + folio_size(folio)),
args->owner);
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(&pvmw)) {
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
ptent = ptep_get(pvmw.pte);
if (!pte_present(ptent)) {
ret = false;
page_vma_mapped_walk_done(&pvmw);
break;
}
subpage = folio_page(folio,
pte_pfn(ptent) - folio_pfn(folio));
address = pvmw.address;
/* Nuke the page table entry. */
flush_cache_page(vma, address, pte_pfn(ptent));
pteval = ptep_clear_flush(vma, address, pvmw.pte);
/* Set the dirty flag on the folio now the pte is gone. */
if (pte_dirty(pteval))
folio_mark_dirty(folio);
/*
* Check that our target page is still mapped at the expected
* address.
*/
if (args->mm == mm && args->address == address &&
pte_write(pteval))
args->valid = true;
/*
* Store the pfn of the page in a special migration
* pte. do_swap_page() will wait until the migration
* pte is removed and then restart fault handling.
*/
if (pte_write(pteval))
entry = make_writable_device_exclusive_entry(
page_to_pfn(subpage));
else
entry = make_readable_device_exclusive_entry(
page_to_pfn(subpage));
swp_pte = swp_entry_to_pte(entry);
if (pte_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte);
if (pte_uffd_wp(pteval))
swp_pte = pte_swp_mkuffd_wp(swp_pte);
set_pte_at(mm, address, pvmw.pte, swp_pte);
/*
* There is a reference on the page for the swap entry which has
* been removed, so shouldn't take another.
*/
folio_remove_rmap_pte(folio, subpage, vma);
}
mmu_notifier_invalidate_range_end(&range);
return ret;
}
/**
* folio_make_device_exclusive - Mark the folio exclusively owned by a device.
* @folio: The folio to replace page table entries for.
* @mm: The mm_struct where the folio is expected to be mapped.
* @address: Address where the folio is expected to be mapped.
* @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks
*
* Tries to remove all the page table entries which are mapping this
* folio and replace them with special device exclusive swap entries to
* grant a device exclusive access to the folio.
*
* Context: Caller must hold the folio lock.
* Return: false if the page is still mapped, or if it could not be unmapped
* from the expected address. Otherwise returns true (success).
*/
static bool folio_make_device_exclusive(struct folio *folio,
struct mm_struct *mm, unsigned long address, void *owner)
{
struct make_exclusive_args args = {
.mm = mm,
.address = address,
.owner = owner,
.valid = false,
};
struct rmap_walk_control rwc = {
.rmap_one = page_make_device_exclusive_one,
.done = folio_not_mapped,
.anon_lock = folio_lock_anon_vma_read,
.arg = &args,
};
/*
* Restrict to anonymous folios for now to avoid potential writeback
* issues.
*/
if (!folio_test_anon(folio))
return false;
rmap_walk(folio, &rwc);
return args.valid && !folio_mapcount(folio);
}
/**
* make_device_exclusive_range() - Mark a range for exclusive use by a device
* @mm: mm_struct of associated target process
* @start: start of the region to mark for exclusive device access
* @end: end address of region
* @pages: returns the pages which were successfully marked for exclusive access
* @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering
*
* Returns: number of pages found in the range by GUP. A page is marked for
* exclusive access only if the page pointer is non-NULL.
*
* This function finds ptes mapping page(s) to the given address range, locks
* them and replaces mappings with special swap entries preventing userspace CPU
* access. On fault these entries are replaced with the original mapping after
* calling MMU notifiers.
*
* A driver using this to program access from a device must use a mmu notifier
* critical section to hold a device specific lock during programming. Once
* programming is complete it should drop the page lock and reference after
* which point CPU access to the page will revoke the exclusive access.
*/
int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
unsigned long end, struct page **pages,
void *owner)
{
long npages = (end - start) >> PAGE_SHIFT;
long i;
npages = get_user_pages_remote(mm, start, npages,
FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD,
pages, NULL);
if (npages < 0)
return npages;
for (i = 0; i < npages; i++, start += PAGE_SIZE) {
struct folio *folio = page_folio(pages[i]);
if (PageTail(pages[i]) || !folio_trylock(folio)) {
folio_put(folio);
pages[i] = NULL;
continue;
}
if (!folio_make_device_exclusive(folio, mm, start, owner)) {
folio_unlock(folio);
folio_put(folio);
pages[i] = NULL;
}
}
return npages;
}
EXPORT_SYMBOL_GPL(make_device_exclusive_range);
#endif
void __put_anon_vma(struct anon_vma *anon_vma)
{
struct anon_vma *root = anon_vma->root;
anon_vma_free(anon_vma);
if (root != anon_vma && atomic_dec_and_test(&root->refcount))
anon_vma_free(root);
}
static struct anon_vma *rmap_walk_anon_lock(struct folio *folio,
struct rmap_walk_control *rwc)
{
struct anon_vma *anon_vma;
if (rwc->anon_lock)
return rwc->anon_lock(folio, rwc);
/*
* Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
* because that depends on page_mapped(); but not all its usages
* are holding mmap_lock. Users without mmap_lock are required to
* take a reference count to prevent the anon_vma disappearing
*/
anon_vma = folio_anon_vma(folio);
if (!anon_vma)
return NULL;
if (anon_vma_trylock_read(anon_vma))
goto out;
if (rwc->try_lock) {
anon_vma = NULL;
rwc->contended = true;
goto out;
}
anon_vma_lock_read(anon_vma);
out:
return anon_vma;
}
/*
* rmap_walk_anon - do something to anonymous page using the object-based
* rmap method
* @folio: the folio to be handled
* @rwc: control variable according to each walk type
* @locked: caller holds relevant rmap lock
*
* Find all the mappings of a folio using the mapping pointer and the vma
* chains contained in the anon_vma struct it points to.
*/
static void rmap_walk_anon(struct folio *folio,
struct rmap_walk_control *rwc, bool locked)
{
struct anon_vma *anon_vma;
pgoff_t pgoff_start, pgoff_end;
struct anon_vma_chain *avc;
if (locked) {
anon_vma = folio_anon_vma(folio);
/* anon_vma disappear under us? */
VM_BUG_ON_FOLIO(!anon_vma, folio);
} else {
anon_vma = rmap_walk_anon_lock(folio, rwc);
}
if (!anon_vma)
return;
pgoff_start = folio_pgoff(folio);
pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
pgoff_start, pgoff_end) {
struct vm_area_struct *vma = avc->vma;
unsigned long address = vma_address(&folio->page, vma);
VM_BUG_ON_VMA(address == -EFAULT, vma);
cond_resched();
if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
continue;
if (!rwc->rmap_one(folio, vma, address, rwc->arg))
break;
if (rwc->done && rwc->done(folio))
break;
}
if (!locked)
anon_vma_unlock_read(anon_vma);
}
/*
* rmap_walk_file - do something to file page using the object-based rmap method
* @folio: the folio to be handled
* @rwc: control variable according to each walk type
* @locked: caller holds relevant rmap lock
*
* Find all the mappings of a folio using the mapping pointer and the vma chains
* contained in the address_space struct it points to.
*/
static void rmap_walk_file(struct folio *folio,
struct rmap_walk_control *rwc, bool locked)
{
struct address_space *mapping = folio_mapping(folio);
pgoff_t pgoff_start, pgoff_end;
struct vm_area_struct *vma;
/*
* The page lock not only makes sure that page->mapping cannot
* suddenly be NULLified by truncation, it makes sure that the
* structure at mapping cannot be freed and reused yet,
* so we can safely take mapping->i_mmap_rwsem.
*/
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (!mapping)
return;
pgoff_start = folio_pgoff(folio);
pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
if (!locked) {
if (i_mmap_trylock_read(mapping))
goto lookup;
if (rwc->try_lock) {
rwc->contended = true;
return;
}
i_mmap_lock_read(mapping);
}
lookup:
vma_interval_tree_foreach(vma, &mapping->i_mmap,
pgoff_start, pgoff_end) {
unsigned long address = vma_address(&folio->page, vma);
VM_BUG_ON_VMA(address == -EFAULT, vma);
cond_resched();
if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
continue;
if (!rwc->rmap_one(folio, vma, address, rwc->arg))
goto done;
if (rwc->done && rwc->done(folio))
goto done;
}
done:
if (!locked)
i_mmap_unlock_read(mapping);
}
void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc)
{
if (unlikely(folio_test_ksm(folio)))
rmap_walk_ksm(folio, rwc);
else if (folio_test_anon(folio))
rmap_walk_anon(folio, rwc, false);
else
rmap_walk_file(folio, rwc, false);
}
/* Like rmap_walk, but caller holds relevant rmap lock */
void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)
{
/* no ksm support for now */
VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
if (folio_test_anon(folio))
rmap_walk_anon(folio, rwc, true);
else
rmap_walk_file(folio, rwc, true);
}
#ifdef CONFIG_HUGETLB_PAGE
/*
* The following two functions are for anonymous (private mapped) hugepages.
* Unlike common anonymous pages, anonymous hugepages have no accounting code
* and no lru code, because we handle hugepages differently from common pages.
*/
void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
unsigned long address, rmap_t flags)
{
VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
atomic_inc(&folio->_entire_mapcount);
if (flags & RMAP_EXCLUSIVE)
SetPageAnonExclusive(&folio->page);
VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 &&
PageAnonExclusive(&folio->page), folio);
}
void hugetlb_add_new_anon_rmap(struct folio *folio,
struct vm_area_struct *vma, unsigned long address)
{
VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
BUG_ON(address < vma->vm_start || address >= vma->vm_end);
/* increment count (starts at -1) */
atomic_set(&folio->_entire_mapcount, 0);
folio_clear_hugetlb_restore_reserve(folio);
__folio_set_anon(folio, vma, address, true);
SetPageAnonExclusive(&folio->page);
}
#endif /* CONFIG_HUGETLB_PAGE */
// SPDX-License-Identifier: GPL-2.0
/*
* transport_class.c - implementation of generic transport classes
* using attribute_containers
*
* Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com>
*
* The basic idea here is to allow any "device controller" (which
* would most often be a Host Bus Adapter to use the services of one
* or more tranport classes for performing transport specific
* services. Transport specific services are things that the generic
* command layer doesn't want to know about (speed settings, line
* condidtioning, etc), but which the user might be interested in.
* Thus, the HBA's use the routines exported by the transport classes
* to perform these functions. The transport classes export certain
* values to the user via sysfs using attribute containers.
*
* Note: because not every HBA will care about every transport
* attribute, there's a many to one relationship that goes like this:
*
* transport class<-----attribute container<----class device
*
* Usually the attribute container is per-HBA, but the design doesn't
* mandate that. Although most of the services will be specific to
* the actual external storage connection used by the HBA, the generic
* transport class is framed entirely in terms of generic devices to
* allow it to be used by any physical HBA in the system.
*/
#include <linux/export.h>
#include <linux/attribute_container.h>
#include <linux/transport_class.h>
static int transport_remove_classdev(struct attribute_container *cont,
struct device *dev,
struct device *classdev);
/**
* transport_class_register - register an initial transport class
*
* @tclass: a pointer to the transport class structure to be initialised
*
* The transport class contains an embedded class which is used to
* identify it. The caller should initialise this structure with
* zeros and then generic class must have been initialised with the
* actual transport class unique name. There's a macro
* DECLARE_TRANSPORT_CLASS() to do this (declared classes still must
* be registered).
*
* Returns 0 on success or error on failure.
*/
int transport_class_register(struct transport_class *tclass)
{
return class_register(&tclass->class);
}
EXPORT_SYMBOL_GPL(transport_class_register);
/**
* transport_class_unregister - unregister a previously registered class
*
* @tclass: The transport class to unregister
*
* Must be called prior to deallocating the memory for the transport
* class.
*/
void transport_class_unregister(struct transport_class *tclass)
{
class_unregister(&tclass->class);
}
EXPORT_SYMBOL_GPL(transport_class_unregister);
static int anon_transport_dummy_function(struct transport_container *tc,
struct device *dev,
struct device *cdev)
{
/* do nothing */
return 0;
}
/**
* anon_transport_class_register - register an anonymous class
*
* @atc: The anon transport class to register
*
* The anonymous transport class contains both a transport class and a
* container. The idea of an anonymous class is that it never
* actually has any device attributes associated with it (and thus
* saves on container storage). So it can only be used for triggering
* events. Use prezero and then use DECLARE_ANON_TRANSPORT_CLASS() to
* initialise the anon transport class storage.
*/
int anon_transport_class_register(struct anon_transport_class *atc)
{
int error;
atc->container.class = &atc->tclass.class;
attribute_container_set_no_classdevs(&atc->container);
error = attribute_container_register(&atc->container);
if (error)
return error;
atc->tclass.setup = anon_transport_dummy_function;
atc->tclass.remove = anon_transport_dummy_function;
return 0;
}
EXPORT_SYMBOL_GPL(anon_transport_class_register);
/**
* anon_transport_class_unregister - unregister an anon class
*
* @atc: Pointer to the anon transport class to unregister
*
* Must be called prior to deallocating the memory for the anon
* transport class.
*/
void anon_transport_class_unregister(struct anon_transport_class *atc)
{
if (unlikely(attribute_container_unregister(&atc->container)))
BUG();
}
EXPORT_SYMBOL_GPL(anon_transport_class_unregister);
static int transport_setup_classdev(struct attribute_container *cont,
struct device *dev,
struct device *classdev)
{
struct transport_class *tclass = class_to_transport_class(cont->class);
struct transport_container *tcont = attribute_container_to_transport_container(cont);
if (tclass->setup)
tclass->setup(tcont, dev, classdev);
return 0;
}
/**
* transport_setup_device - declare a new dev for transport class association but don't make it visible yet.
* @dev: the generic device representing the entity being added
*
* Usually, dev represents some component in the HBA system (either
* the HBA itself or a device remote across the HBA bus). This
* routine is simply a trigger point to see if any set of transport
* classes wishes to associate with the added device. This allocates
* storage for the class device and initialises it, but does not yet
* add it to the system or add attributes to it (you do this with
* transport_add_device). If you have no need for a separate setup
* and add operations, use transport_register_device (see
* transport_class.h).
*/
void transport_setup_device(struct device *dev)
{
attribute_container_add_device(dev, transport_setup_classdev);
}
EXPORT_SYMBOL_GPL(transport_setup_device);
static int transport_add_class_device(struct attribute_container *cont,
struct device *dev,
struct device *classdev)
{
struct transport_class *tclass = class_to_transport_class(cont->class);
int error = attribute_container_add_class_device(classdev);
struct transport_container *tcont =
attribute_container_to_transport_container(cont);
if (error)
goto err_remove;
if (tcont->statistics) {
error = sysfs_create_group(&classdev->kobj, tcont->statistics);
if (error)
goto err_del;
}
return 0;
err_del:
attribute_container_class_device_del(classdev);
err_remove:
if (tclass->remove)
tclass->remove(tcont, dev, classdev);
return error;
}
/**
* transport_add_device - declare a new dev for transport class association
*
* @dev: the generic device representing the entity being added
*
* Usually, dev represents some component in the HBA system (either
* the HBA itself or a device remote across the HBA bus). This
* routine is simply a trigger point used to add the device to the
* system and register attributes for it.
*/
int transport_add_device(struct device *dev)
{
return attribute_container_device_trigger_safe(dev,
transport_add_class_device,
transport_remove_classdev);
}
EXPORT_SYMBOL_GPL(transport_add_device);
static int transport_configure(struct attribute_container *cont,
struct device *dev,
struct device *cdev)
{
struct transport_class *tclass = class_to_transport_class(cont->class);
struct transport_container *tcont = attribute_container_to_transport_container(cont);
if (tclass->configure)
tclass->configure(tcont, dev, cdev);
return 0;
}
/**
* transport_configure_device - configure an already set up device
*
* @dev: generic device representing device to be configured
*
* The idea of configure is simply to provide a point within the setup
* process to allow the transport class to extract information from a
* device after it has been setup. This is used in SCSI because we
* have to have a setup device to begin using the HBA, but after we
* send the initial inquiry, we use configure to extract the device
* parameters. The device need not have been added to be configured.
*/
void transport_configure_device(struct device *dev)
{
attribute_container_device_trigger(dev, transport_configure);
}
EXPORT_SYMBOL_GPL(transport_configure_device);
static int transport_remove_classdev(struct attribute_container *cont,
struct device *dev,
struct device *classdev)
{
struct transport_container *tcont =
attribute_container_to_transport_container(cont);
struct transport_class *tclass = class_to_transport_class(cont->class);
if (tclass->remove)
tclass->remove(tcont, dev, classdev);
if (tclass->remove != anon_transport_dummy_function) {
if (tcont->statistics)
sysfs_remove_group(&classdev->kobj, tcont->statistics);
attribute_container_class_device_del(classdev);
}
return 0;
}
/**
* transport_remove_device - remove the visibility of a device
*
* @dev: generic device to remove
*
* This call removes the visibility of the device (to the user from
* sysfs), but does not destroy it. To eliminate a device entirely
* you must also call transport_destroy_device. If you don't need to
* do remove and destroy as separate operations, use
* transport_unregister_device() (see transport_class.h) which will
* perform both calls for you.
*/
void transport_remove_device(struct device *dev)
{
attribute_container_device_trigger(dev, transport_remove_classdev);
}
EXPORT_SYMBOL_GPL(transport_remove_device);
static void transport_destroy_classdev(struct attribute_container *cont,
struct device *dev,
struct device *classdev)
{
struct transport_class *tclass = class_to_transport_class(cont->class);
if (tclass->remove != anon_transport_dummy_function)
put_device(classdev);
}
/**
* transport_destroy_device - destroy a removed device
*
* @dev: device to eliminate from the transport class.
*
* This call triggers the elimination of storage associated with the
* transport classdev. Note: all it really does is relinquish a
* reference to the classdev. The memory will not be freed until the
* last reference goes to zero. Note also that the classdev retains a
* reference count on dev, so dev too will remain for as long as the
* transport class device remains around.
*/
void transport_destroy_device(struct device *dev)
{
attribute_container_remove_device(dev, transport_destroy_classdev);
}
EXPORT_SYMBOL_GPL(transport_destroy_device);
// SPDX-License-Identifier: GPL-2.0
#include <linux/bitops.h>
#include <linux/fault-inject-usercopy.h>
#include <linux/instrumented.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
/* out-of-line parts */
#ifndef INLINE_COPY_FROM_USER
unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned long res = n;
might_fault(); if (!should_fail_usercopy() && likely(access_ok(from, n))) {
/*
* Ensure that bad access_ok() speculation will not
* lead to nasty side effects *after* the copy is
* finished:
*/
barrier_nospec();
instrument_copy_from_user_before(to, from, n);
res = raw_copy_from_user(to, from, n);
instrument_copy_from_user_after(to, from, n, res);
}
if (unlikely(res)) memset(to + (n - res), 0, res); return res;
}
EXPORT_SYMBOL(_copy_from_user);
#endif
#ifndef INLINE_COPY_TO_USER
unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n)
{
might_fault(); if (should_fail_usercopy())
return n;
if (likely(access_ok(to, n))) { instrument_copy_to_user(to, from, n);
n = raw_copy_to_user(to, from, n);
}
return n;
}
EXPORT_SYMBOL(_copy_to_user);
#endif
/**
* check_zeroed_user: check if a userspace buffer only contains zero bytes
* @from: Source address, in userspace.
* @size: Size of buffer.
*
* This is effectively shorthand for "memchr_inv(from, 0, size) == NULL" for
* userspace addresses (and is more efficient because we don't care where the
* first non-zero byte is).
*
* Returns:
* * 0: There were non-zero bytes present in the buffer.
* * 1: The buffer was full of zero bytes.
* * -EFAULT: access to userspace failed.
*/
int check_zeroed_user(const void __user *from, size_t size)
{
unsigned long val;
uintptr_t align = (uintptr_t) from % sizeof(unsigned long);
if (unlikely(size == 0))
return 1;
from -= align;
size += align;
if (!user_read_access_begin(from, size))
return -EFAULT;
unsafe_get_user(val, (unsigned long __user *) from, err_fault);
if (align)
val &= ~aligned_byte_mask(align);
while (size > sizeof(unsigned long)) {
if (unlikely(val))
goto done;
from += sizeof(unsigned long);
size -= sizeof(unsigned long);
unsafe_get_user(val, (unsigned long __user *) from, err_fault);
}
if (size < sizeof(unsigned long))
val &= aligned_byte_mask(size);
done:
user_read_access_end();
return (val == 0);
err_fault:
user_read_access_end();
return -EFAULT;
}
EXPORT_SYMBOL(check_zeroed_user);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* This file provides wrappers with sanitizer instrumentation for bit
* locking operations.
*
* To use this functionality, an arch's bitops.h file needs to define each of
* the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
* arch___set_bit(), etc.).
*/
#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H
#define _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H
#include <linux/instrumented.h>
/**
* clear_bit_unlock - Clear a bit in memory, for unlock
* @nr: the bit to set
* @addr: the address to start counting from
*
* This operation is atomic and provides release barrier semantics.
*/
static inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
{
kcsan_release();
instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
arch_clear_bit_unlock(nr, addr);
}
/**
* __clear_bit_unlock - Clears a bit in memory
* @nr: Bit to clear
* @addr: Address to start counting from
*
* This is a non-atomic operation but implies a release barrier before the
* memory operation. It can be used for an unlock if no other CPUs can
* concurrently modify other bits in the word.
*/
static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
{
kcsan_release();
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___clear_bit_unlock(nr, addr);
}
/**
* test_and_set_bit_lock - Set a bit and return its old value, for lock
* @nr: Bit to set
* @addr: Address to count from
*
* This operation is atomic and provides acquire barrier semantics if
* the returned value is 0.
* It can be used to implement bit locks.
*/
static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr)
{
instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_set_bit_lock(nr, addr);
}
/**
* xor_unlock_is_negative_byte - XOR a single byte in memory and test if
* it is negative, for unlock.
* @mask: Change the bits which are set in this mask.
* @addr: The address of the word containing the byte to change.
*
* Changes some of bits 0-6 in the word pointed to by @addr.
* This operation is atomic and provides release barrier semantics.
* Used to optimise some folio operations which are commonly paired
* with an unlock or end of writeback. Bit 7 is used as PG_waiters to
* indicate whether anybody is waiting for the unlock.
*
* Return: Whether the top bit of the byte is set.
*/
static inline bool xor_unlock_is_negative_byte(unsigned long mask,
volatile unsigned long *addr)
{
kcsan_release();
instrument_atomic_write(addr, sizeof(long));
return arch_xor_unlock_is_negative_byte(mask, addr);
}
#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_UACCESS_H
#define _ASM_X86_UACCESS_H
/*
* User space memory access functions
*/
#include <linux/compiler.h>
#include <linux/instrumented.h>
#include <linux/kasan-checks.h>
#include <linux/mm_types.h>
#include <linux/string.h>
#include <linux/mmap_lock.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/smap.h>
#include <asm/extable.h>
#include <asm/tlbflush.h>
#ifdef CONFIG_X86_32
# include <asm/uaccess_32.h>
#else
# include <asm/uaccess_64.h>
#endif
#include <asm-generic/access_ok.h>
extern int __get_user_1(void);
extern int __get_user_2(void);
extern int __get_user_4(void);
extern int __get_user_8(void);
extern int __get_user_nocheck_1(void);
extern int __get_user_nocheck_2(void);
extern int __get_user_nocheck_4(void);
extern int __get_user_nocheck_8(void);
extern int __get_user_bad(void);
#define __uaccess_begin() stac()
#define __uaccess_end() clac()
#define __uaccess_begin_nospec() \
({ \
stac(); \
barrier_nospec(); \
})
/*
* This is the smallest unsigned integer type that can fit a value
* (up to 'long long')
*/
#define __inttype(x) __typeof__( \
__typefits(x,char, \
__typefits(x,short, \
__typefits(x,int, \
__typefits(x,long,0ULL)))))
#define __typefits(x,type,not) \
__builtin_choose_expr(sizeof(x)<=sizeof(type),(unsigned type)0,not)
/*
* This is used for both get_user() and __get_user() to expand to
* the proper special function call that has odd calling conventions
* due to returning both a value and an error, and that depends on
* the size of the pointer passed in.
*
* Careful: we have to cast the result to the type of the pointer
* for sign reasons.
*
* The use of _ASM_DX as the register specifier is a bit of a
* simplification, as gcc only cares about it as the starting point
* and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
* (%ecx being the next register in gcc's x86 register sequence), and
* %rdx on 64 bits.
*
* Clang/LLVM cares about the size of the register, but still wants
* the base register for something that ends up being a pair.
*/
#define do_get_user_call(fn,x,ptr) \
({ \
int __ret_gu; \
register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \
__chk_user_ptr(ptr); \
asm volatile("call __" #fn "_%P4" \
: "=a" (__ret_gu), "=r" (__val_gu), \
ASM_CALL_CONSTRAINT \
: "0" (ptr), "i" (sizeof(*(ptr)))); \
instrument_get_user(__val_gu); \
(x) = (__force __typeof__(*(ptr))) __val_gu; \
__builtin_expect(__ret_gu, 0); \
})
/**
* get_user - Get a simple variable from user space.
* @x: Variable to store result.
* @ptr: Source address, in user space.
*
* Context: User context only. This function may sleep if pagefaults are
* enabled.
*
* This macro copies a single simple variable from user space to kernel
* space. It supports simple types like char and int, but not larger
* data types like structures or arrays.
*
* @ptr must have pointer-to-simple-variable type, and the result of
* dereferencing @ptr must be assignable to @x without a cast.
*
* Return: zero on success, or -EFAULT on error.
* On error, the variable @x is set to zero.
*/
#define get_user(x,ptr) ({ might_fault(); do_get_user_call(get_user,x,ptr); })
/**
* __get_user - Get a simple variable from user space, with less checking.
* @x: Variable to store result.
* @ptr: Source address, in user space.
*
* Context: User context only. This function may sleep if pagefaults are
* enabled.
*
* This macro copies a single simple variable from user space to kernel
* space. It supports simple types like char and int, but not larger
* data types like structures or arrays.
*
* @ptr must have pointer-to-simple-variable type, and the result of
* dereferencing @ptr must be assignable to @x without a cast.
*
* Caller must check the pointer with access_ok() before calling this
* function.
*
* Return: zero on success, or -EFAULT on error.
* On error, the variable @x is set to zero.
*/
#define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
#ifdef CONFIG_X86_32
#define __put_user_goto_u64(x, addr, label) \
asm goto("\n" \
"1: movl %%eax,0(%1)\n" \
"2: movl %%edx,4(%1)\n" \
_ASM_EXTABLE_UA(1b, %l2) \
_ASM_EXTABLE_UA(2b, %l2) \
: : "A" (x), "r" (addr) \
: : label)
#else
#define __put_user_goto_u64(x, ptr, label) \
__put_user_goto(x, ptr, "q", "er", label)
#endif
extern void __put_user_bad(void);
/*
* Strange magic calling convention: pointer in %ecx,
* value in %eax(:%edx), return value in %ecx. clobbers %rbx
*/
extern void __put_user_1(void);
extern void __put_user_2(void);
extern void __put_user_4(void);
extern void __put_user_8(void);
extern void __put_user_nocheck_1(void);
extern void __put_user_nocheck_2(void);
extern void __put_user_nocheck_4(void);
extern void __put_user_nocheck_8(void);
/*
* ptr must be evaluated and assigned to the temporary __ptr_pu before
* the assignment of x to __val_pu, to avoid any function calls
* involved in the ptr expression (possibly implicitly generated due
* to KASAN) from clobbering %ax.
*/
#define do_put_user_call(fn,x,ptr) \
({ \
int __ret_pu; \
void __user *__ptr_pu; \
register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX); \
__typeof__(*(ptr)) __x = (x); /* eval x once */ \
__typeof__(ptr) __ptr = (ptr); /* eval ptr once */ \
__chk_user_ptr(__ptr); \
__ptr_pu = __ptr; \
__val_pu = __x; \
asm volatile("call __" #fn "_%P[size]" \
: "=c" (__ret_pu), \
ASM_CALL_CONSTRAINT \
: "0" (__ptr_pu), \
"r" (__val_pu), \
[size] "i" (sizeof(*(ptr))) \
:"ebx"); \
instrument_put_user(__x, __ptr, sizeof(*(ptr))); \
__builtin_expect(__ret_pu, 0); \
})
/**
* put_user - Write a simple value into user space.
* @x: Value to copy to user space.
* @ptr: Destination address, in user space.
*
* Context: User context only. This function may sleep if pagefaults are
* enabled.
*
* This macro copies a single simple value from kernel space to user
* space. It supports simple types like char and int, but not larger
* data types like structures or arrays.
*
* @ptr must have pointer-to-simple-variable type, and @x must be assignable
* to the result of dereferencing @ptr.
*
* Return: zero on success, or -EFAULT on error.
*/
#define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
/**
* __put_user - Write a simple value into user space, with less checking.
* @x: Value to copy to user space.
* @ptr: Destination address, in user space.
*
* Context: User context only. This function may sleep if pagefaults are
* enabled.
*
* This macro copies a single simple value from kernel space to user
* space. It supports simple types like char and int, but not larger
* data types like structures or arrays.
*
* @ptr must have pointer-to-simple-variable type, and @x must be assignable
* to the result of dereferencing @ptr.
*
* Caller must check the pointer with access_ok() before calling this
* function.
*
* Return: zero on success, or -EFAULT on error.
*/
#define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
#define __put_user_size(x, ptr, size, label) \
do { \
__typeof__(*(ptr)) __x = (x); /* eval x once */ \
__typeof__(ptr) __ptr = (ptr); /* eval ptr once */ \
__chk_user_ptr(__ptr); \
switch (size) { \
case 1: \
__put_user_goto(__x, __ptr, "b", "iq", label); \
break; \
case 2: \
__put_user_goto(__x, __ptr, "w", "ir", label); \
break; \
case 4: \
__put_user_goto(__x, __ptr, "l", "ir", label); \
break; \
case 8: \
__put_user_goto_u64(__x, __ptr, label); \
break; \
default: \
__put_user_bad(); \
} \
instrument_put_user(__x, __ptr, size); \
} while (0)
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, label) do { \
unsigned int __gu_low, __gu_high; \
const unsigned int __user *__gu_ptr; \
__gu_ptr = (const void __user *)(ptr); \
__get_user_asm(__gu_low, __gu_ptr, "l", "=r", label); \
__get_user_asm(__gu_high, __gu_ptr+1, "l", "=r", label); \
(x) = ((unsigned long long)__gu_high << 32) | __gu_low; \
} while (0)
#else
#define __get_user_asm_u64(x, ptr, label) \
__get_user_asm(x, ptr, "q", "=r", label)
#endif
#define __get_user_size(x, ptr, size, label) \
do { \
__chk_user_ptr(ptr); \
switch (size) { \
case 1: { \
unsigned char x_u8__; \
__get_user_asm(x_u8__, ptr, "b", "=q", label); \
(x) = x_u8__; \
break; \
} \
case 2: \
__get_user_asm(x, ptr, "w", "=r", label); \
break; \
case 4: \
__get_user_asm(x, ptr, "l", "=r", label); \
break; \
case 8: \
__get_user_asm_u64(x, ptr, label); \
break; \
default: \
(x) = __get_user_bad(); \
} \
instrument_get_user(x); \
} while (0)
#define __get_user_asm(x, addr, itype, ltype, label) \
asm_goto_output("\n" \
"1: mov"itype" %[umem],%[output]\n" \
_ASM_EXTABLE_UA(1b, %l2) \
: [output] ltype(x) \
: [umem] "m" (__m(addr)) \
: : label)
#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, retval) \
({ \
__typeof__(ptr) __ptr = (ptr); \
asm volatile("\n" \
"1: movl %[lowbits],%%eax\n" \
"2: movl %[highbits],%%edx\n" \
"3:\n" \
_ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG | \
EX_FLAG_CLEAR_AX_DX, \
%[errout]) \
_ASM_EXTABLE_TYPE_REG(2b, 3b, EX_TYPE_EFAULT_REG | \
EX_FLAG_CLEAR_AX_DX, \
%[errout]) \
: [errout] "=r" (retval), \
[output] "=&A"(x) \
: [lowbits] "m" (__m(__ptr)), \
[highbits] "m" __m(((u32 __user *)(__ptr)) + 1), \
"0" (retval)); \
})
#else
#define __get_user_asm_u64(x, ptr, retval) \
__get_user_asm(x, ptr, retval, "q")
#endif
#define __get_user_size(x, ptr, size, retval) \
do { \
unsigned char x_u8__; \
\
retval = 0; \
__chk_user_ptr(ptr); \
switch (size) { \
case 1: \
__get_user_asm(x_u8__, ptr, retval, "b"); \
(x) = x_u8__; \
break; \
case 2: \
__get_user_asm(x, ptr, retval, "w"); \
break; \
case 4: \
__get_user_asm(x, ptr, retval, "l"); \
break; \
case 8: \
__get_user_asm_u64(x, ptr, retval); \
break; \
default: \
(x) = __get_user_bad(); \
} \
} while (0)
#define __get_user_asm(x, addr, err, itype) \
asm volatile("\n" \
"1: mov"itype" %[umem],%[output]\n" \
"2:\n" \
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG | \
EX_FLAG_CLEAR_AX, \
%[errout]) \
: [errout] "=r" (err), \
[output] "=a" (x) \
: [umem] "m" (__m(addr)), \
"0" (err))
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
bool success; \
__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
__typeof__(*(_ptr)) __old = *_old; \
__typeof__(*(_ptr)) __new = (_new); \
asm_goto_output("\n" \
"1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
_ASM_EXTABLE_UA(1b, %l[label]) \
: CC_OUT(z) (success), \
[ptr] "+m" (*_ptr), \
[old] "+a" (__old) \
: [new] ltype (__new) \
: "memory" \
: label); \
if (unlikely(!success)) \
*_old = __old; \
likely(success); })
#ifdef CONFIG_X86_32
#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
bool success; \
__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
__typeof__(*(_ptr)) __old = *_old; \
__typeof__(*(_ptr)) __new = (_new); \
asm_goto_output("\n" \
"1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
_ASM_EXTABLE_UA(1b, %l[label]) \
: CC_OUT(z) (success), \
"+A" (__old), \
[ptr] "+m" (*_ptr) \
: "b" ((u32)__new), \
"c" ((u32)((u64)__new >> 32)) \
: "memory" \
: label); \
if (unlikely(!success)) \
*_old = __old; \
likely(success); })
#endif // CONFIG_X86_32
#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \
int __err = 0; \
bool success; \
__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
__typeof__(*(_ptr)) __old = *_old; \
__typeof__(*(_ptr)) __new = (_new); \
asm volatile("\n" \
"1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\
CC_SET(z) \
"2:\n" \
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \
%[errout]) \
: CC_OUT(z) (success), \
[errout] "+r" (__err), \
[ptr] "+m" (*_ptr), \
[old] "+a" (__old) \
: [new] ltype (__new) \
: "memory"); \
if (unlikely(__err)) \
goto label; \
if (unlikely(!success)) \
*_old = __old; \
likely(success); })
#ifdef CONFIG_X86_32
/*
* Unlike the normal CMPXCHG, use output GPR for both success/fail and error.
* There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
* hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses
* both ESI and EDI for the memory operand, compilation will fail if the error
* is an input+output as there will be no register available for input.
*/
#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \
int __result; \
__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \
__typeof__(*(_ptr)) __old = *_old; \
__typeof__(*(_ptr)) __new = (_new); \
asm volatile("\n" \
"1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \
"mov $0, %[result]\n\t" \
"setz %b[result]\n" \
"2:\n" \
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \
%[result]) \
: [result] "=q" (__result), \
"+A" (__old), \
[ptr] "+m" (*_ptr) \
: "b" ((u32)__new), \
"c" ((u32)((u64)__new >> 32)) \
: "memory", "cc"); \
if (unlikely(__result < 0)) \
goto label; \
if (unlikely(!__result)) \
*_old = __old; \
likely(__result); })
#endif // CONFIG_X86_32
#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT
/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x))
/*
* Tell gcc we read from memory instead of writing: this is because
* we do not write to any memory gcc knows about, so there are no
* aliasing issues.
*/
#define __put_user_goto(x, addr, itype, ltype, label) \
asm goto("\n" \
"1: mov"itype" %0,%1\n" \
_ASM_EXTABLE_UA(1b, %l2) \
: : ltype(x), "m" (__m(addr)) \
: : label)
extern unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
extern __must_check long
strncpy_from_user(char *dst, const char __user *src, long count);
extern __must_check long strnlen_user(const char __user *str, long n);
#ifdef CONFIG_ARCH_HAS_COPY_MC
unsigned long __must_check
copy_mc_to_kernel(void *to, const void *from, unsigned len);
#define copy_mc_to_kernel copy_mc_to_kernel
unsigned long __must_check
copy_mc_to_user(void __user *to, const void *from, unsigned len);
#endif
/*
* movsl can be slow when source and dest are not both 8-byte aligned
*/
#ifdef CONFIG_X86_INTEL_USERCOPY
extern struct movsl_mask {
int mask;
} ____cacheline_aligned_in_smp movsl_mask;
#endif
#define ARCH_HAS_NOCACHE_UACCESS 1
/*
* The "unsafe" user accesses aren't really "unsafe", but the naming
* is a big fat warning: you have to not only do the access_ok()
* checking before using them, but you have to surround them with the
* user_access_begin/end() pair.
*/
static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
{
if (unlikely(!access_ok(ptr,len)))
return 0;
__uaccess_begin_nospec();
return 1;
}
#define user_access_begin(a,b) user_access_begin(a,b)
#define user_access_end() __uaccess_end()
#define user_access_save() smap_save()
#define user_access_restore(x) smap_restore(x)
#define unsafe_put_user(x, ptr, label) \
__put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define unsafe_get_user(x, ptr, err_label) \
do { \
__inttype(*(ptr)) __gu_val; \
__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label); \
(x) = (__force __typeof__(*(ptr)))__gu_val; \
} while (0)
#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define unsafe_get_user(x, ptr, err_label) \
do { \
int __gu_err; \
__inttype(*(ptr)) __gu_val; \
__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err); \
(x) = (__force __typeof__(*(ptr)))__gu_val; \
if (unlikely(__gu_err)) goto err_label; \
} while (0)
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
extern void __try_cmpxchg_user_wrong_size(void);
#ifndef CONFIG_X86_32
#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \
__try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label)
#endif
/*
* Force the pointer to u<size> to match the size expected by the asm helper.
* clang/LLVM compiles all cases and only discards the unused paths after
* processing errors, which breaks i386 if the pointer is an 8-byte value.
*/
#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
bool __ret; \
__chk_user_ptr(_ptr); \
switch (sizeof(*(_ptr))) { \
case 1: __ret = __try_cmpxchg_user_asm("b", "q", \
(__force u8 *)(_ptr), (_oldp), \
(_nval), _label); \
break; \
case 2: __ret = __try_cmpxchg_user_asm("w", "r", \
(__force u16 *)(_ptr), (_oldp), \
(_nval), _label); \
break; \
case 4: __ret = __try_cmpxchg_user_asm("l", "r", \
(__force u32 *)(_ptr), (_oldp), \
(_nval), _label); \
break; \
case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\
(_nval), _label); \
break; \
default: __try_cmpxchg_user_wrong_size(); \
} \
__ret; })
/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */
#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \
int __ret = -EFAULT; \
__uaccess_begin_nospec(); \
__ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \
_label: \
__uaccess_end(); \
__ret; \
})
/*
* We want the unsafe accessors to always be inlined and use
* the error labels - thus the macro games.
*/
#define unsafe_copy_loop(dst, src, len, type, label) \
while (len >= sizeof(type)) { \
unsafe_put_user(*(type *)(src),(type __user *)(dst),label); \
dst += sizeof(type); \
src += sizeof(type); \
len -= sizeof(type); \
}
#define unsafe_copy_to_user(_dst,_src,_len,label) \
do { \
char __user *__ucu_dst = (_dst); \
const char *__ucu_src = (_src); \
size_t __ucu_len = (_len); \
unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u64, label); \
unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u32, label); \
unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u16, label); \
unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label); \
} while (0)
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __get_kernel_nofault(dst, src, type, err_label) \
__get_user_size(*((type *)(dst)), (__force type __user *)(src), \
sizeof(type), err_label)
#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __get_kernel_nofault(dst, src, type, err_label) \
do { \
int __kr_err; \
\
__get_user_size(*((type *)(dst)), (__force type __user *)(src), \
sizeof(type), __kr_err); \
if (unlikely(__kr_err)) \
goto err_label; \
} while (0)
#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __put_kernel_nofault(dst, src, type, err_label) \
__put_user_size(*((type *)(src)), (__force type __user *)(dst), \
sizeof(type), err_label)
#endif /* _ASM_X86_UACCESS_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_MSR_H
#define _ASM_X86_MSR_H
#include "msr-index.h"
#ifndef __ASSEMBLY__
#include <asm/asm.h>
#include <asm/errno.h>
#include <asm/cpumask.h>
#include <uapi/asm/msr.h>
#include <asm/shared/msr.h>
#include <linux/percpu.h>
struct msr_info {
u32 msr_no;
struct msr reg;
struct msr __percpu *msrs;
int err;
};
struct msr_regs_info {
u32 *regs;
int err;
};
struct saved_msr {
bool valid;
struct msr_info info;
};
struct saved_msrs {
unsigned int num;
struct saved_msr *array;
};
/*
* both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
* constraint has different meanings. For i386, "A" means exactly
* edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
* it means rax *or* rdx.
*/
#ifdef CONFIG_X86_64
/* Using 64-bit values saves one instruction clearing the high half of low */
#define DECLARE_ARGS(val, low, high) unsigned long low, high
#define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32)
#define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high)
#else
#define DECLARE_ARGS(val, low, high) unsigned long long val
#define EAX_EDX_VAL(val, low, high) (val)
#define EAX_EDX_RET(val, low, high) "=A" (val)
#endif
/*
* Be very careful with includes. This header is prone to include loops.
*/
#include <asm/atomic.h>
#include <linux/tracepoint-defs.h>
#ifdef CONFIG_TRACEPOINTS
DECLARE_TRACEPOINT(read_msr);
DECLARE_TRACEPOINT(write_msr);
DECLARE_TRACEPOINT(rdpmc);
extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
#else
static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
#endif
/*
* __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR
* accessors and should not have any tracing or other functionality piggybacking
* on them - those are *purely* for accessing MSRs and nothing more. So don't even
* think of extending them - you will be slapped with a stinking trout or a frozen
* shark will reach you, wherever you are! You've been warned.
*/
static __always_inline unsigned long long __rdmsr(unsigned int msr)
{
DECLARE_ARGS(val, low, high);
asm volatile("1: rdmsr\n"
"2:\n"
_ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR)
: EAX_EDX_RET(val, low, high) : "c" (msr));
return EAX_EDX_VAL(val, low, high);
}
static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high)
{
asm volatile("1: wrmsr\n"
"2:\n"
_ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR)
: : "c" (msr), "a"(low), "d" (high) : "memory");
}
/*
* WRMSRNS behaves exactly like WRMSR with the only difference being
* that it is not a serializing instruction by default.
*/
static __always_inline void __wrmsrns(u32 msr, u32 low, u32 high)
{
/* Instruction opcode for WRMSRNS; supported in binutils >= 2.40. */
asm volatile("1: .byte 0x0f,0x01,0xc6\n"
"2:\n"
_ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR)
: : "c" (msr), "a"(low), "d" (high));
}
#define native_rdmsr(msr, val1, val2) \
do { \
u64 __val = __rdmsr((msr)); \
(void)((val1) = (u32)__val); \
(void)((val2) = (u32)(__val >> 32)); \
} while (0)
#define native_wrmsr(msr, low, high) \
__wrmsr(msr, low, high)
#define native_wrmsrl(msr, val) \
__wrmsr((msr), (u32)((u64)(val)), \
(u32)((u64)(val) >> 32))
static inline unsigned long long native_read_msr(unsigned int msr)
{
unsigned long long val;
val = __rdmsr(msr);
if (tracepoint_enabled(read_msr))
do_trace_read_msr(msr, val, 0);
return val;
}
static inline unsigned long long native_read_msr_safe(unsigned int msr,
int *err)
{
DECLARE_ARGS(val, low, high);
asm volatile("1: rdmsr ; xor %[err],%[err]\n"
"2:\n\t"
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err])
: [err] "=r" (*err), EAX_EDX_RET(val, low, high)
: "c" (msr));
if (tracepoint_enabled(read_msr))
do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
return EAX_EDX_VAL(val, low, high);
}
/* Can be uninlined because referenced by paravirt */
static inline void notrace
native_write_msr(unsigned int msr, u32 low, u32 high)
{
__wrmsr(msr, low, high);
if (tracepoint_enabled(write_msr))
do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
}
/* Can be uninlined because referenced by paravirt */
static inline int notrace
native_write_msr_safe(unsigned int msr, u32 low, u32 high)
{
int err;
asm volatile("1: wrmsr ; xor %[err],%[err]\n"
"2:\n\t"
_ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err])
: [err] "=a" (err)
: "c" (msr), "0" (low), "d" (high)
: "memory");
if (tracepoint_enabled(write_msr))
do_trace_write_msr(msr, ((u64)high << 32 | low), err);
return err;
}
extern int rdmsr_safe_regs(u32 regs[8]);
extern int wrmsr_safe_regs(u32 regs[8]);
/**
* rdtsc() - returns the current TSC without ordering constraints
*
* rdtsc() returns the result of RDTSC as a 64-bit integer. The
* only ordering constraint it supplies is the ordering implied by
* "asm volatile": it will put the RDTSC in the place you expect. The
* CPU can and will speculatively execute that RDTSC, though, so the
* results can be non-monotonic if compared on different CPUs.
*/
static __always_inline unsigned long long rdtsc(void)
{
DECLARE_ARGS(val, low, high);
asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
return EAX_EDX_VAL(val, low, high);
}
/**
* rdtsc_ordered() - read the current TSC in program order
*
* rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
* It is ordered like a load to a global in-memory counter. It should
* be impossible to observe non-monotonic rdtsc_unordered() behavior
* across multiple CPUs as long as the TSC is synced.
*/
static __always_inline unsigned long long rdtsc_ordered(void)
{
DECLARE_ARGS(val, low, high);
/*
* The RDTSC instruction is not ordered relative to memory
* access. The Intel SDM and the AMD APM are both vague on this
* point, but empirically an RDTSC instruction can be
* speculatively executed before prior loads. An RDTSC
* immediately after an appropriate barrier appears to be
* ordered as a normal load, that is, it provides the same
* ordering guarantees as reading from a global memory location
* that some other imaginary CPU is updating continuously with a
* time stamp.
*
* Thus, use the preferred barrier on the respective CPU, aiming for
* RDTSCP as the default.
*/
asm volatile(ALTERNATIVE_2("rdtsc",
"lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
"rdtscp", X86_FEATURE_RDTSCP)
: EAX_EDX_RET(val, low, high)
/* RDTSCP clobbers ECX with MSR_TSC_AUX. */
:: "ecx");
return EAX_EDX_VAL(val, low, high);
}
static inline unsigned long long native_read_pmc(int counter)
{
DECLARE_ARGS(val, low, high);
asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
if (tracepoint_enabled(rdpmc))
do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0);
return EAX_EDX_VAL(val, low, high);
}
#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
#include <linux/errno.h>
/*
* Access to machine-specific registers (available on 586 and better only)
* Note: the rd* operations modify the parameters directly (without using
* pointer indirection), this allows gcc to optimize better
*/
#define rdmsr(msr, low, high) \
do { \
u64 __val = native_read_msr((msr)); \
(void)((low) = (u32)__val); \
(void)((high) = (u32)(__val >> 32)); \
} while (0)
static inline void wrmsr(unsigned int msr, u32 low, u32 high)
{
native_write_msr(msr, low, high);
}
#define rdmsrl(msr, val) \
((val) = native_read_msr((msr)))
static inline void wrmsrl(unsigned int msr, u64 val)
{
native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
}
/* wrmsr with exception handling */
static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
{
return native_write_msr_safe(msr, low, high);
}
/* rdmsr with exception handling */
#define rdmsr_safe(msr, low, high) \
({ \
int __err; \
u64 __val = native_read_msr_safe((msr), &__err); \
(*low) = (u32)__val; \
(*high) = (u32)(__val >> 32); \
__err; \
})
static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
{
int err;
*p = native_read_msr_safe(msr, &err);
return err;
}
#define rdpmc(counter, low, high) \
do { \
u64 _l = native_read_pmc((counter)); \
(low) = (u32)_l; \
(high) = (u32)(_l >> 32); \
} while (0)
#define rdpmcl(counter, val) ((val) = native_read_pmc(counter))
#endif /* !CONFIG_PARAVIRT_XXL */
static __always_inline void wrmsrns(u32 msr, u64 val)
{
__wrmsrns(msr, val, val >> 32);
}
/*
* 64-bit version of wrmsr_safe():
*/
static inline int wrmsrl_safe(u32 msr, u64 val)
{
return wrmsr_safe(msr, (u32)val, (u32)(val >> 32));
}
struct msr __percpu *msrs_alloc(void);
void msrs_free(struct msr __percpu *msrs);
int msr_set_bit(u32 msr, u8 bit);
int msr_clear_bit(u32 msr, u8 bit);
#ifdef CONFIG_SMP
int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs);
void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr __percpu *msrs);
int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
#else /* CONFIG_SMP */
static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
{
rdmsr(msr_no, *l, *h);
return 0;
}
static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
{
wrmsr(msr_no, l, h);
return 0;
}
static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
{
rdmsrl(msr_no, *q);
return 0;
}
static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
{
wrmsrl(msr_no, q);
return 0;
}
static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
struct msr __percpu *msrs)
{
rdmsr_on_cpu(0, msr_no, raw_cpu_ptr(&msrs->l), raw_cpu_ptr(&msrs->h));
}
static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no,
struct msr __percpu *msrs)
{
wrmsr_on_cpu(0, msr_no, raw_cpu_read(msrs->l), raw_cpu_read(msrs->h));
}
static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
u32 *l, u32 *h)
{
return rdmsr_safe(msr_no, l, h);
}
static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
{
return wrmsr_safe(msr_no, l, h);
}
static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
{
return rdmsrl_safe(msr_no, q);
}
static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
{
return wrmsrl_safe(msr_no, q);
}
static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
{
return rdmsr_safe_regs(regs);
}
static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
{
return wrmsr_safe_regs(regs);
}
#endif /* CONFIG_SMP */
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_MSR_H */
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* include/linux/eventpoll.h ( Efficient event polling implementation )
* Copyright (C) 2001,...,2006 Davide Libenzi
*
* Davide Libenzi <davidel@xmailserver.org>
*/
#ifndef _LINUX_EVENTPOLL_H
#define _LINUX_EVENTPOLL_H
#include <uapi/linux/eventpoll.h>
#include <uapi/linux/kcmp.h>
/* Forward declarations to avoid compiler errors */
struct file;
#ifdef CONFIG_EPOLL
#ifdef CONFIG_KCMP
struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, unsigned long toff);
#endif
/* Used to release the epoll bits inside the "struct file" */
void eventpoll_release_file(struct file *file);
/*
* This is called from inside fs/file_table.c:__fput() to unlink files
* from the eventpoll interface. We need to have this facility to cleanup
* correctly files that are closed without being removed from the eventpoll
* interface.
*/
static inline void eventpoll_release(struct file *file)
{
/*
* Fast check to avoid the get/release of the semaphore. Since
* we're doing this outside the semaphore lock, it might return
* false negatives, but we don't care. It'll help in 99.99% of cases
* to avoid the semaphore lock. False positives simply cannot happen
* because the file in on the way to be removed and nobody ( but
* eventpoll ) has still a reference to this file.
*/
if (likely(!file->f_ep))
return;
/*
* The file is being closed while it is still linked to an epoll
* descriptor. We need to handle this by correctly unlinking it
* from its containers.
*/
eventpoll_release_file(file);
}
int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
bool nonblock);
/* Tells if the epoll_ctl(2) operation needs an event copy from userspace */
static inline int ep_op_has_event(int op)
{
return op != EPOLL_CTL_DEL;
}
#else
static inline void eventpoll_release(struct file *file) {}
#endif
#if defined(CONFIG_ARM) && defined(CONFIG_OABI_COMPAT)
/* ARM OABI has an incompatible struct layout and needs a special handler */
extern struct epoll_event __user *
epoll_put_uevent(__poll_t revents, __u64 data,
struct epoll_event __user *uevent);
#else
static inline struct epoll_event __user *
epoll_put_uevent(__poll_t revents, __u64 data,
struct epoll_event __user *uevent)
{
if (__put_user(revents, &uevent->events) ||
__put_user(data, &uevent->data))
return NULL;
return uevent+1;
}
#endif
#endif /* #ifndef _LINUX_EVENTPOLL_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* workqueue.h --- work queue handling for Linux.
*/
#ifndef _LINUX_WORKQUEUE_H
#define _LINUX_WORKQUEUE_H
#include <linux/timer.h>
#include <linux/linkage.h>
#include <linux/bitops.h>
#include <linux/lockdep.h>
#include <linux/threads.h>
#include <linux/atomic.h>
#include <linux/cpumask.h>
#include <linux/rcupdate.h>
#include <linux/workqueue_types.h>
/*
* The first word is the work queue pointer and the flags rolled into
* one
*/
#define work_data_bits(work) ((unsigned long *)(&(work)->data))
enum work_bits {
WORK_STRUCT_PENDING_BIT = 0, /* work item is pending execution */
WORK_STRUCT_INACTIVE_BIT, /* work item is inactive */
WORK_STRUCT_PWQ_BIT, /* data points to pwq */
WORK_STRUCT_LINKED_BIT, /* next work is linked to this one */
#ifdef CONFIG_DEBUG_OBJECTS_WORK
WORK_STRUCT_STATIC_BIT, /* static initializer (debugobjects) */
#endif
WORK_STRUCT_FLAG_BITS,
/* color for workqueue flushing */
WORK_STRUCT_COLOR_SHIFT = WORK_STRUCT_FLAG_BITS,
WORK_STRUCT_COLOR_BITS = 4,
/*
* When WORK_STRUCT_PWQ is set, reserve 8 bits off of pwq pointer w/
* debugobjects turned off. This makes pwqs aligned to 256 bytes (512
* bytes w/ DEBUG_OBJECTS_WORK) and allows 16 workqueue flush colors.
*
* MSB
* [ pwq pointer ] [ flush color ] [ STRUCT flags ]
* 4 bits 4 or 5 bits
*/
WORK_STRUCT_PWQ_SHIFT = WORK_STRUCT_COLOR_SHIFT + WORK_STRUCT_COLOR_BITS,
/*
* data contains off-queue information when !WORK_STRUCT_PWQ.
*
* MSB
* [ pool ID ] [ OFFQ flags ] [ STRUCT flags ]
* 1 bit 4 or 5 bits
*/
WORK_OFFQ_FLAG_SHIFT = WORK_STRUCT_FLAG_BITS,
WORK_OFFQ_CANCELING_BIT = WORK_OFFQ_FLAG_SHIFT,
WORK_OFFQ_FLAG_END,
WORK_OFFQ_FLAG_BITS = WORK_OFFQ_FLAG_END - WORK_OFFQ_FLAG_SHIFT,
/*
* When a work item is off queue, the high bits encode off-queue flags
* and the last pool it was on. Cap pool ID to 31 bits and use the
* highest number to indicate that no pool is associated.
*/
WORK_OFFQ_POOL_SHIFT = WORK_OFFQ_FLAG_SHIFT + WORK_OFFQ_FLAG_BITS,
WORK_OFFQ_LEFT = BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT,
WORK_OFFQ_POOL_BITS = WORK_OFFQ_LEFT <= 31 ? WORK_OFFQ_LEFT : 31,
};
enum work_flags {
WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT,
WORK_STRUCT_INACTIVE = 1 << WORK_STRUCT_INACTIVE_BIT,
WORK_STRUCT_PWQ = 1 << WORK_STRUCT_PWQ_BIT,
WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT,
#ifdef CONFIG_DEBUG_OBJECTS_WORK
WORK_STRUCT_STATIC = 1 << WORK_STRUCT_STATIC_BIT,
#else
WORK_STRUCT_STATIC = 0,
#endif
};
enum wq_misc_consts {
WORK_NR_COLORS = (1 << WORK_STRUCT_COLOR_BITS),
/* not bound to any CPU, prefer the local CPU */
WORK_CPU_UNBOUND = NR_CPUS,
/* bit mask for work_busy() return values */
WORK_BUSY_PENDING = 1 << 0,
WORK_BUSY_RUNNING = 1 << 1,
/* maximum string length for set_worker_desc() */
WORKER_DESC_LEN = 24,
};
/* Convenience constants - of type 'unsigned long', not 'enum'! */
#define WORK_OFFQ_CANCELING (1ul << WORK_OFFQ_CANCELING_BIT)
#define WORK_OFFQ_POOL_NONE ((1ul << WORK_OFFQ_POOL_BITS) - 1)
#define WORK_STRUCT_NO_POOL (WORK_OFFQ_POOL_NONE << WORK_OFFQ_POOL_SHIFT)
#define WORK_STRUCT_PWQ_MASK (~((1ul << WORK_STRUCT_PWQ_SHIFT) - 1))
#define WORK_DATA_INIT() ATOMIC_LONG_INIT((unsigned long)WORK_STRUCT_NO_POOL)
#define WORK_DATA_STATIC_INIT() \
ATOMIC_LONG_INIT((unsigned long)(WORK_STRUCT_NO_POOL | WORK_STRUCT_STATIC))
struct delayed_work {
struct work_struct work;
struct timer_list timer;
/* target workqueue and CPU ->timer uses to queue ->work */
struct workqueue_struct *wq;
int cpu;
};
struct rcu_work {
struct work_struct work;
struct rcu_head rcu;
/* target workqueue ->rcu uses to queue ->work */
struct workqueue_struct *wq;
};
enum wq_affn_scope {
WQ_AFFN_DFL, /* use system default */
WQ_AFFN_CPU, /* one pod per CPU */
WQ_AFFN_SMT, /* one pod poer SMT */
WQ_AFFN_CACHE, /* one pod per LLC */
WQ_AFFN_NUMA, /* one pod per NUMA node */
WQ_AFFN_SYSTEM, /* one pod across the whole system */
WQ_AFFN_NR_TYPES,
};
/**
* struct workqueue_attrs - A struct for workqueue attributes.
*
* This can be used to change attributes of an unbound workqueue.
*/
struct workqueue_attrs {
/**
* @nice: nice level
*/
int nice;
/**
* @cpumask: allowed CPUs
*
* Work items in this workqueue are affine to these CPUs and not allowed
* to execute on other CPUs. A pool serving a workqueue must have the
* same @cpumask.
*/
cpumask_var_t cpumask;
/**
* @__pod_cpumask: internal attribute used to create per-pod pools
*
* Internal use only.
*
* Per-pod unbound worker pools are used to improve locality. Always a
* subset of ->cpumask. A workqueue can be associated with multiple
* worker pools with disjoint @__pod_cpumask's. Whether the enforcement
* of a pool's @__pod_cpumask is strict depends on @affn_strict.
*/
cpumask_var_t __pod_cpumask;
/**
* @affn_strict: affinity scope is strict
*
* If clear, workqueue will make a best-effort attempt at starting the
* worker inside @__pod_cpumask but the scheduler is free to migrate it
* outside.
*
* If set, workers are only allowed to run inside @__pod_cpumask.
*/
bool affn_strict;
/*
* Below fields aren't properties of a worker_pool. They only modify how
* :c:func:`apply_workqueue_attrs` select pools and thus don't
* participate in pool hash calculations or equality comparisons.
*/
/**
* @affn_scope: unbound CPU affinity scope
*
* CPU pods are used to improve execution locality of unbound work
* items. There are multiple pod types, one for each wq_affn_scope, and
* every CPU in the system belongs to one pod in every pod type. CPUs
* that belong to the same pod share the worker pool. For example,
* selecting %WQ_AFFN_NUMA makes the workqueue use a separate worker
* pool for each NUMA node.
*/
enum wq_affn_scope affn_scope;
/**
* @ordered: work items must be executed one by one in queueing order
*/
bool ordered;
};
static inline struct delayed_work *to_delayed_work(struct work_struct *work)
{
return container_of(work, struct delayed_work, work);
}
static inline struct rcu_work *to_rcu_work(struct work_struct *work)
{
return container_of(work, struct rcu_work, work);
}
struct execute_work {
struct work_struct work;
};
#ifdef CONFIG_LOCKDEP
/*
* NB: because we have to copy the lockdep_map, setting _key
* here is required, otherwise it could get initialised to the
* copy of the lockdep_map!
*/
#define __WORK_INIT_LOCKDEP_MAP(n, k) \
.lockdep_map = STATIC_LOCKDEP_MAP_INIT(n, k),
#else
#define __WORK_INIT_LOCKDEP_MAP(n, k)
#endif
#define __WORK_INITIALIZER(n, f) { \
.data = WORK_DATA_STATIC_INIT(), \
.entry = { &(n).entry, &(n).entry }, \
.func = (f), \
__WORK_INIT_LOCKDEP_MAP(#n, &(n)) \
}
#define __DELAYED_WORK_INITIALIZER(n, f, tflags) { \
.work = __WORK_INITIALIZER((n).work, (f)), \
.timer = __TIMER_INITIALIZER(delayed_work_timer_fn,\
(tflags) | TIMER_IRQSAFE), \
}
#define DECLARE_WORK(n, f) \
struct work_struct n = __WORK_INITIALIZER(n, f)
#define DECLARE_DELAYED_WORK(n, f) \
struct delayed_work n = __DELAYED_WORK_INITIALIZER(n, f, 0)
#define DECLARE_DEFERRABLE_WORK(n, f) \
struct delayed_work n = __DELAYED_WORK_INITIALIZER(n, f, TIMER_DEFERRABLE)
#ifdef CONFIG_DEBUG_OBJECTS_WORK
extern void __init_work(struct work_struct *work, int onstack);
extern void destroy_work_on_stack(struct work_struct *work);
extern void destroy_delayed_work_on_stack(struct delayed_work *work);
static inline unsigned int work_static(struct work_struct *work)
{
return *work_data_bits(work) & WORK_STRUCT_STATIC;
}
#else
static inline void __init_work(struct work_struct *work, int onstack) { }
static inline void destroy_work_on_stack(struct work_struct *work) { }
static inline void destroy_delayed_work_on_stack(struct delayed_work *work) { }
static inline unsigned int work_static(struct work_struct *work) { return 0; }
#endif
/*
* initialize all of a work item in one go
*
* NOTE! No point in using "atomic_long_set()": using a direct
* assignment of the work data initializer allows the compiler
* to generate better code.
*/
#ifdef CONFIG_LOCKDEP
#define __INIT_WORK_KEY(_work, _func, _onstack, _key) \
do { \
__init_work((_work), _onstack); \
(_work)->data = (atomic_long_t) WORK_DATA_INIT(); \
lockdep_init_map(&(_work)->lockdep_map, "(work_completion)"#_work, (_key), 0); \
INIT_LIST_HEAD(&(_work)->entry); \
(_work)->func = (_func); \
} while (0)
#else
#define __INIT_WORK_KEY(_work, _func, _onstack, _key) \
do { \
__init_work((_work), _onstack); \
(_work)->data = (atomic_long_t) WORK_DATA_INIT(); \
INIT_LIST_HEAD(&(_work)->entry); \
(_work)->func = (_func); \
} while (0)
#endif
#define __INIT_WORK(_work, _func, _onstack) \
do { \
static __maybe_unused struct lock_class_key __key; \
\
__INIT_WORK_KEY(_work, _func, _onstack, &__key); \
} while (0)
#define INIT_WORK(_work, _func) \
__INIT_WORK((_work), (_func), 0)
#define INIT_WORK_ONSTACK(_work, _func) \
__INIT_WORK((_work), (_func), 1)
#define INIT_WORK_ONSTACK_KEY(_work, _func, _key) \
__INIT_WORK_KEY((_work), (_func), 1, _key)
#define __INIT_DELAYED_WORK(_work, _func, _tflags) \
do { \
INIT_WORK(&(_work)->work, (_func)); \
__init_timer(&(_work)->timer, \
delayed_work_timer_fn, \
(_tflags) | TIMER_IRQSAFE); \
} while (0)
#define __INIT_DELAYED_WORK_ONSTACK(_work, _func, _tflags) \
do { \
INIT_WORK_ONSTACK(&(_work)->work, (_func)); \
__init_timer_on_stack(&(_work)->timer, \
delayed_work_timer_fn, \
(_tflags) | TIMER_IRQSAFE); \
} while (0)
#define INIT_DELAYED_WORK(_work, _func) \
__INIT_DELAYED_WORK(_work, _func, 0)
#define INIT_DELAYED_WORK_ONSTACK(_work, _func) \
__INIT_DELAYED_WORK_ONSTACK(_work, _func, 0)
#define INIT_DEFERRABLE_WORK(_work, _func) \
__INIT_DELAYED_WORK(_work, _func, TIMER_DEFERRABLE)
#define INIT_DEFERRABLE_WORK_ONSTACK(_work, _func) \
__INIT_DELAYED_WORK_ONSTACK(_work, _func, TIMER_DEFERRABLE)
#define INIT_RCU_WORK(_work, _func) \
INIT_WORK(&(_work)->work, (_func))
#define INIT_RCU_WORK_ONSTACK(_work, _func) \
INIT_WORK_ONSTACK(&(_work)->work, (_func))
/**
* work_pending - Find out whether a work item is currently pending
* @work: The work item in question
*/
#define work_pending(work) \
test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))
/**
* delayed_work_pending - Find out whether a delayable work item is currently
* pending
* @w: The work item in question
*/
#define delayed_work_pending(w) \
work_pending(&(w)->work)
/*
* Workqueue flags and constants. For details, please refer to
* Documentation/core-api/workqueue.rst.
*/
enum wq_flags {
WQ_BH = 1 << 0, /* execute in bottom half (softirq) context */
WQ_UNBOUND = 1 << 1, /* not bound to any cpu */
WQ_FREEZABLE = 1 << 2, /* freeze during suspend */
WQ_MEM_RECLAIM = 1 << 3, /* may be used for memory reclaim */
WQ_HIGHPRI = 1 << 4, /* high priority */
WQ_CPU_INTENSIVE = 1 << 5, /* cpu intensive workqueue */
WQ_SYSFS = 1 << 6, /* visible in sysfs, see workqueue_sysfs_register() */
/*
* Per-cpu workqueues are generally preferred because they tend to
* show better performance thanks to cache locality. Per-cpu
* workqueues exclude the scheduler from choosing the CPU to
* execute the worker threads, which has an unfortunate side effect
* of increasing power consumption.
*
* The scheduler considers a CPU idle if it doesn't have any task
* to execute and tries to keep idle cores idle to conserve power;
* however, for example, a per-cpu work item scheduled from an
* interrupt handler on an idle CPU will force the scheduler to
* execute the work item on that CPU breaking the idleness, which in
* turn may lead to more scheduling choices which are sub-optimal
* in terms of power consumption.
*
* Workqueues marked with WQ_POWER_EFFICIENT are per-cpu by default
* but become unbound if workqueue.power_efficient kernel param is
* specified. Per-cpu workqueues which are identified to
* contribute significantly to power-consumption are identified and
* marked with this flag and enabling the power_efficient mode
* leads to noticeable power saving at the cost of small
* performance disadvantage.
*
* http://thread.gmane.org/gmane.linux.kernel/1480396
*/
WQ_POWER_EFFICIENT = 1 << 7,
__WQ_DESTROYING = 1 << 15, /* internal: workqueue is destroying */
__WQ_DRAINING = 1 << 16, /* internal: workqueue is draining */
__WQ_ORDERED = 1 << 17, /* internal: workqueue is ordered */
__WQ_LEGACY = 1 << 18, /* internal: create*_workqueue() */
/* BH wq only allows the following flags */
__WQ_BH_ALLOWS = WQ_BH | WQ_HIGHPRI,
};
enum wq_consts {
WQ_MAX_ACTIVE = 512, /* I like 512, better ideas? */
WQ_UNBOUND_MAX_ACTIVE = WQ_MAX_ACTIVE,
WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2,
/*
* Per-node default cap on min_active. Unless explicitly set, min_active
* is set to min(max_active, WQ_DFL_MIN_ACTIVE). For more details, see
* workqueue_struct->min_active definition.
*/
WQ_DFL_MIN_ACTIVE = 8,
};
/*
* System-wide workqueues which are always present.
*
* system_wq is the one used by schedule[_delayed]_work[_on]().
* Multi-CPU multi-threaded. There are users which expect relatively
* short queue flush time. Don't queue works which can run for too
* long.
*
* system_highpri_wq is similar to system_wq but for work items which
* require WQ_HIGHPRI.
*
* system_long_wq is similar to system_wq but may host long running
* works. Queue flushing might take relatively long.
*
* system_unbound_wq is unbound workqueue. Workers are not bound to
* any specific CPU, not concurrency managed, and all queued works are
* executed immediately as long as max_active limit is not reached and
* resources are available.
*
* system_freezable_wq is equivalent to system_wq except that it's
* freezable.
*
* *_power_efficient_wq are inclined towards saving power and converted
* into WQ_UNBOUND variants if 'wq_power_efficient' is enabled; otherwise,
* they are same as their non-power-efficient counterparts - e.g.
* system_power_efficient_wq is identical to system_wq if
* 'wq_power_efficient' is disabled. See WQ_POWER_EFFICIENT for more info.
*
* system_bh[_highpri]_wq are convenience interface to softirq. BH work items
* are executed in the queueing CPU's BH context in the queueing order.
*/
extern struct workqueue_struct *system_wq;
extern struct workqueue_struct *system_highpri_wq;
extern struct workqueue_struct *system_long_wq;
extern struct workqueue_struct *system_unbound_wq;
extern struct workqueue_struct *system_freezable_wq;
extern struct workqueue_struct *system_power_efficient_wq;
extern struct workqueue_struct *system_freezable_power_efficient_wq;
extern struct workqueue_struct *system_bh_wq;
extern struct workqueue_struct *system_bh_highpri_wq;
void workqueue_softirq_action(bool highpri);
void workqueue_softirq_dead(unsigned int cpu);
/**
* alloc_workqueue - allocate a workqueue
* @fmt: printf format for the name of the workqueue
* @flags: WQ_* flags
* @max_active: max in-flight work items, 0 for default
* remaining args: args for @fmt
*
* For a per-cpu workqueue, @max_active limits the number of in-flight work
* items for each CPU. e.g. @max_active of 1 indicates that each CPU can be
* executing at most one work item for the workqueue.
*
* For unbound workqueues, @max_active limits the number of in-flight work items
* for the whole system. e.g. @max_active of 16 indicates that that there can be
* at most 16 work items executing for the workqueue in the whole system.
*
* As sharing the same active counter for an unbound workqueue across multiple
* NUMA nodes can be expensive, @max_active is distributed to each NUMA node
* according to the proportion of the number of online CPUs and enforced
* independently.
*
* Depending on online CPU distribution, a node may end up with per-node
* max_active which is significantly lower than @max_active, which can lead to
* deadlocks if the per-node concurrency limit is lower than the maximum number
* of interdependent work items for the workqueue.
*
* To guarantee forward progress regardless of online CPU distribution, the
* concurrency limit on every node is guaranteed to be equal to or greater than
* min_active which is set to min(@max_active, %WQ_DFL_MIN_ACTIVE). This means
* that the sum of per-node max_active's may be larger than @max_active.
*
* For detailed information on %WQ_* flags, please refer to
* Documentation/core-api/workqueue.rst.
*
* RETURNS:
* Pointer to the allocated workqueue on success, %NULL on failure.
*/
__printf(1, 4) struct workqueue_struct *
alloc_workqueue(const char *fmt, unsigned int flags, int max_active, ...);
/**
* alloc_ordered_workqueue - allocate an ordered workqueue
* @fmt: printf format for the name of the workqueue
* @flags: WQ_* flags (only WQ_FREEZABLE and WQ_MEM_RECLAIM are meaningful)
* @args: args for @fmt
*
* Allocate an ordered workqueue. An ordered workqueue executes at
* most one work item at any given time in the queued order. They are
* implemented as unbound workqueues with @max_active of one.
*
* RETURNS:
* Pointer to the allocated workqueue on success, %NULL on failure.
*/
#define alloc_ordered_workqueue(fmt, flags, args...) \
alloc_workqueue(fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args)
#define create_workqueue(name) \
alloc_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, 1, (name))
#define create_freezable_workqueue(name) \
alloc_workqueue("%s", __WQ_LEGACY | WQ_FREEZABLE | WQ_UNBOUND | \
WQ_MEM_RECLAIM, 1, (name))
#define create_singlethread_workqueue(name) \
alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
#define from_work(var, callback_work, work_fieldname) \
container_of(callback_work, typeof(*var), work_fieldname)
extern void destroy_workqueue(struct workqueue_struct *wq);
struct workqueue_attrs *alloc_workqueue_attrs(void);
void free_workqueue_attrs(struct workqueue_attrs *attrs);
int apply_workqueue_attrs(struct workqueue_struct *wq,
const struct workqueue_attrs *attrs);
extern int workqueue_unbound_exclude_cpumask(cpumask_var_t cpumask);
extern bool queue_work_on(int cpu, struct workqueue_struct *wq,
struct work_struct *work);
extern bool queue_work_node(int node, struct workqueue_struct *wq,
struct work_struct *work);
extern bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
struct delayed_work *work, unsigned long delay);
extern bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
struct delayed_work *dwork, unsigned long delay);
extern bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork);
extern void __flush_workqueue(struct workqueue_struct *wq);
extern void drain_workqueue(struct workqueue_struct *wq);
extern int schedule_on_each_cpu(work_func_t func);
int execute_in_process_context(work_func_t fn, struct execute_work *);
extern bool flush_work(struct work_struct *work);
extern bool cancel_work(struct work_struct *work);
extern bool cancel_work_sync(struct work_struct *work);
extern bool flush_delayed_work(struct delayed_work *dwork);
extern bool cancel_delayed_work(struct delayed_work *dwork);
extern bool cancel_delayed_work_sync(struct delayed_work *dwork);
extern bool flush_rcu_work(struct rcu_work *rwork);
extern void workqueue_set_max_active(struct workqueue_struct *wq,
int max_active);
extern void workqueue_set_min_active(struct workqueue_struct *wq,
int min_active);
extern struct work_struct *current_work(void);
extern bool current_is_workqueue_rescuer(void);
extern bool workqueue_congested(int cpu, struct workqueue_struct *wq);
extern unsigned int work_busy(struct work_struct *work);
extern __printf(1, 2) void set_worker_desc(const char *fmt, ...);
extern void print_worker_info(const char *log_lvl, struct task_struct *task);
extern void show_all_workqueues(void);
extern void show_freezable_workqueues(void);
extern void show_one_workqueue(struct workqueue_struct *wq);
extern void wq_worker_comm(char *buf, size_t size, struct task_struct *task);
/**
* queue_work - queue work on a workqueue
* @wq: workqueue to use
* @work: work to queue
*
* Returns %false if @work was already on a queue, %true otherwise.
*
* We queue the work to the CPU on which it was submitted, but if the CPU dies
* it can be processed by another CPU.
*
* Memory-ordering properties: If it returns %true, guarantees that all stores
* preceding the call to queue_work() in the program order will be visible from
* the CPU which will execute @work by the time such work executes, e.g.,
*
* { x is initially 0 }
*
* CPU0 CPU1
*
* WRITE_ONCE(x, 1); [ @work is being executed ]
* r0 = queue_work(wq, work); r1 = READ_ONCE(x);
*
* Forbids: r0 == true && r1 == 0
*/
static inline bool queue_work(struct workqueue_struct *wq,
struct work_struct *work)
{
return queue_work_on(WORK_CPU_UNBOUND, wq, work);
}
/**
* queue_delayed_work - queue work on a workqueue after delay
* @wq: workqueue to use
* @dwork: delayable work to queue
* @delay: number of jiffies to wait before queueing
*
* Equivalent to queue_delayed_work_on() but tries to use the local CPU.
*/
static inline bool queue_delayed_work(struct workqueue_struct *wq,
struct delayed_work *dwork,
unsigned long delay)
{
return queue_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay);
}
/**
* mod_delayed_work - modify delay of or queue a delayed work
* @wq: workqueue to use
* @dwork: work to queue
* @delay: number of jiffies to wait before queueing
*
* mod_delayed_work_on() on local CPU.
*/
static inline bool mod_delayed_work(struct workqueue_struct *wq,
struct delayed_work *dwork,
unsigned long delay)
{
return mod_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay);
}
/**
* schedule_work_on - put work task on a specific cpu
* @cpu: cpu to put the work task on
* @work: job to be done
*
* This puts a job on a specific cpu
*/
static inline bool schedule_work_on(int cpu, struct work_struct *work)
{
return queue_work_on(cpu, system_wq, work);
}
/**
* schedule_work - put work task in global workqueue
* @work: job to be done
*
* Returns %false if @work was already on the kernel-global workqueue and
* %true otherwise.
*
* This puts a job in the kernel-global workqueue if it was not already
* queued and leaves it in the same position on the kernel-global
* workqueue otherwise.
*
* Shares the same memory-ordering properties of queue_work(), cf. the
* DocBook header of queue_work().
*/
static inline bool schedule_work(struct work_struct *work)
{
return queue_work(system_wq, work);
}
/*
* Detect attempt to flush system-wide workqueues at compile time when possible.
* Warn attempt to flush system-wide workqueues at runtime.
*
* See https://lkml.kernel.org/r/49925af7-78a8-a3dd-bce6-cfc02e1a9236@I-love.SAKURA.ne.jp
* for reasons and steps for converting system-wide workqueues into local workqueues.
*/
extern void __warn_flushing_systemwide_wq(void)
__compiletime_warning("Please avoid flushing system-wide workqueues.");
/* Please stop using this function, for this function will be removed in near future. */
#define flush_scheduled_work() \
({ \
__warn_flushing_systemwide_wq(); \
__flush_workqueue(system_wq); \
})
#define flush_workqueue(wq) \
({ \
struct workqueue_struct *_wq = (wq); \
\
if ((__builtin_constant_p(_wq == system_wq) && \
_wq == system_wq) || \
(__builtin_constant_p(_wq == system_highpri_wq) && \
_wq == system_highpri_wq) || \
(__builtin_constant_p(_wq == system_long_wq) && \
_wq == system_long_wq) || \
(__builtin_constant_p(_wq == system_unbound_wq) && \
_wq == system_unbound_wq) || \
(__builtin_constant_p(_wq == system_freezable_wq) && \
_wq == system_freezable_wq) || \
(__builtin_constant_p(_wq == system_power_efficient_wq) && \
_wq == system_power_efficient_wq) || \
(__builtin_constant_p(_wq == system_freezable_power_efficient_wq) && \
_wq == system_freezable_power_efficient_wq)) \
__warn_flushing_systemwide_wq(); \
__flush_workqueue(_wq); \
})
/**
* schedule_delayed_work_on - queue work in global workqueue on CPU after delay
* @cpu: cpu to use
* @dwork: job to be done
* @delay: number of jiffies to wait
*
* After waiting for a given time this puts a job in the kernel-global
* workqueue on the specified CPU.
*/
static inline bool schedule_delayed_work_on(int cpu, struct delayed_work *dwork,
unsigned long delay)
{
return queue_delayed_work_on(cpu, system_wq, dwork, delay);
}
/**
* schedule_delayed_work - put work task in global workqueue after delay
* @dwork: job to be done
* @delay: number of jiffies to wait or 0 for immediate execution
*
* After waiting for a given time this puts a job in the kernel-global
* workqueue.
*/
static inline bool schedule_delayed_work(struct delayed_work *dwork,
unsigned long delay)
{
return queue_delayed_work(system_wq, dwork, delay);
}
#ifndef CONFIG_SMP
static inline long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
{
return fn(arg);
}
static inline long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg)
{
return fn(arg);
}
#else
long work_on_cpu_key(int cpu, long (*fn)(void *),
void *arg, struct lock_class_key *key);
/*
* A new key is defined for each caller to make sure the work
* associated with the function doesn't share its locking class.
*/
#define work_on_cpu(_cpu, _fn, _arg) \
({ \
static struct lock_class_key __key; \
\
work_on_cpu_key(_cpu, _fn, _arg, &__key); \
})
long work_on_cpu_safe_key(int cpu, long (*fn)(void *),
void *arg, struct lock_class_key *key);
/*
* A new key is defined for each caller to make sure the work
* associated with the function doesn't share its locking class.
*/
#define work_on_cpu_safe(_cpu, _fn, _arg) \
({ \
static struct lock_class_key __key; \
\
work_on_cpu_safe_key(_cpu, _fn, _arg, &__key); \
})
#endif /* CONFIG_SMP */
#ifdef CONFIG_FREEZER
extern void freeze_workqueues_begin(void);
extern bool freeze_workqueues_busy(void);
extern void thaw_workqueues(void);
#endif /* CONFIG_FREEZER */
#ifdef CONFIG_SYSFS
int workqueue_sysfs_register(struct workqueue_struct *wq);
#else /* CONFIG_SYSFS */
static inline int workqueue_sysfs_register(struct workqueue_struct *wq)
{ return 0; }
#endif /* CONFIG_SYSFS */
#ifdef CONFIG_WQ_WATCHDOG
void wq_watchdog_touch(int cpu);
#else /* CONFIG_WQ_WATCHDOG */
static inline void wq_watchdog_touch(int cpu) { }
#endif /* CONFIG_WQ_WATCHDOG */
#ifdef CONFIG_SMP
int workqueue_prepare_cpu(unsigned int cpu);
int workqueue_online_cpu(unsigned int cpu);
int workqueue_offline_cpu(unsigned int cpu);
#endif
void __init workqueue_init_early(void);
void __init workqueue_init(void);
void __init workqueue_init_topology(void);
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* Lock-less NULL terminated single linked list
*
* The basic atomic operation of this list is cmpxchg on long. On
* architectures that don't have NMI-safe cmpxchg implementation, the
* list can NOT be used in NMI handlers. So code that uses the list in
* an NMI handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
*
* Copyright 2010,2011 Intel Corp.
* Author: Huang Ying <ying.huang@intel.com>
*/
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/llist.h>
/**
* llist_add_batch - add several linked entries in batch
* @new_first: first entry in batch to be added
* @new_last: last entry in batch to be added
* @head: the head for your lock-less list
*
* Return whether list is empty before adding.
*/
bool llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
struct llist_head *head)
{
struct llist_node *first = READ_ONCE(head->first);
do {
new_last->next = first;
} while (!try_cmpxchg(&head->first, &first, new_first));
return !first;
}
EXPORT_SYMBOL_GPL(llist_add_batch);
/**
* llist_del_first - delete the first entry of lock-less list
* @head: the head for your lock-less list
*
* If list is empty, return NULL, otherwise, return the first entry
* deleted, this is the newest added one.
*
* Only one llist_del_first user can be used simultaneously with
* multiple llist_add users without lock. Because otherwise
* llist_del_first, llist_add, llist_add (or llist_del_all, llist_add,
* llist_add) sequence in another user may change @head->first->next,
* but keep @head->first. If multiple consumers are needed, please
* use llist_del_all or use lock between consumers.
*/
struct llist_node *llist_del_first(struct llist_head *head)
{
struct llist_node *entry, *next; entry = smp_load_acquire(&head->first);
do {
if (entry == NULL)
return NULL;
next = READ_ONCE(entry->next);
} while (!try_cmpxchg(&head->first, &entry, next));
return entry;
}
EXPORT_SYMBOL_GPL(llist_del_first);
/**
* llist_del_first_this - delete given entry of lock-less list if it is first
* @head: the head for your lock-less list
* @this: a list entry.
*
* If head of the list is given entry, delete and return %true else
* return %false.
*
* Multiple callers can safely call this concurrently with multiple
* llist_add() callers, providing all the callers offer a different @this.
*/
bool llist_del_first_this(struct llist_head *head,
struct llist_node *this)
{
struct llist_node *entry, *next;
/* acquire ensures orderig wrt try_cmpxchg() is llist_del_first() */
entry = smp_load_acquire(&head->first);
do {
if (entry != this)
return false;
next = READ_ONCE(entry->next);
} while (!try_cmpxchg(&head->first, &entry, next));
return true;
}
EXPORT_SYMBOL_GPL(llist_del_first_this);
/**
* llist_reverse_order - reverse order of a llist chain
* @head: first item of the list to be reversed
*
* Reverse the order of a chain of llist entries and return the
* new first entry.
*/
struct llist_node *llist_reverse_order(struct llist_node *head)
{
struct llist_node *new_head = NULL;
while (head) {
struct llist_node *tmp = head;
head = head->next;
tmp->next = new_head;
new_head = tmp;
}
return new_head;
}
EXPORT_SYMBOL_GPL(llist_reverse_order);
/*
* Copyright (c) 2016 Intel Corporation
*
* Permission to use, copy, modify, distribute, and sell this software and its
* documentation for any purpose is hereby granted without fee, provided that
* the above copyright notice appear in all copies and that both that copyright
* notice and this permission notice appear in supporting documentation, and
* that the name of the copyright holders not be used in advertising or
* publicity pertaining to distribution of the software without specific,
* written prior permission. The copyright holders make no representations
* about the suitability of this software for any purpose. It is provided "as
* is" without express or implied warranty.
*
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
* EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
* DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
* TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
* OF THIS SOFTWARE.
*/
#include <linux/export.h>
#include <drm/drm_bridge.h>
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_encoder.h>
#include <drm/drm_managed.h>
#include <drm/drm_print.h>
#include "drm_crtc_internal.h"
#include "drm_internal.h"
/**
* DOC: overview
*
* Encoders represent the connecting element between the CRTC (as the overall
* pixel pipeline, represented by &struct drm_crtc) and the connectors (as the
* generic sink entity, represented by &struct drm_connector). An encoder takes
* pixel data from a CRTC and converts it to a format suitable for any attached
* connector. Encoders are objects exposed to userspace, originally to allow
* userspace to infer cloning and connector/CRTC restrictions. Unfortunately
* almost all drivers get this wrong, making the uabi pretty much useless. On
* top of that the exposed restrictions are too simple for today's hardware, and
* the recommended way to infer restrictions is by using the
* DRM_MODE_ATOMIC_TEST_ONLY flag for the atomic IOCTL.
*
* Otherwise encoders aren't used in the uapi at all (any modeset request from
* userspace directly connects a connector with a CRTC), drivers are therefore
* free to use them however they wish. Modeset helper libraries make strong use
* of encoders to facilitate code sharing. But for more complex settings it is
* usually better to move shared code into a separate &drm_bridge. Compared to
* encoders, bridges also have the benefit of being purely an internal
* abstraction since they are not exposed to userspace at all.
*
* Encoders are initialized with drm_encoder_init() and cleaned up using
* drm_encoder_cleanup().
*/
static const struct drm_prop_enum_list drm_encoder_enum_list[] = {
{ DRM_MODE_ENCODER_NONE, "None" },
{ DRM_MODE_ENCODER_DAC, "DAC" },
{ DRM_MODE_ENCODER_TMDS, "TMDS" },
{ DRM_MODE_ENCODER_LVDS, "LVDS" },
{ DRM_MODE_ENCODER_TVDAC, "TV" },
{ DRM_MODE_ENCODER_VIRTUAL, "Virtual" },
{ DRM_MODE_ENCODER_DSI, "DSI" },
{ DRM_MODE_ENCODER_DPMST, "DP MST" },
{ DRM_MODE_ENCODER_DPI, "DPI" },
};
int drm_encoder_register_all(struct drm_device *dev)
{
struct drm_encoder *encoder;
int ret = 0;
drm_for_each_encoder(encoder, dev) {
drm_debugfs_encoder_add(encoder);
if (encoder->funcs && encoder->funcs->late_register)
ret = encoder->funcs->late_register(encoder);
if (ret)
return ret;
}
return 0;
}
void drm_encoder_unregister_all(struct drm_device *dev)
{
struct drm_encoder *encoder;
drm_for_each_encoder(encoder, dev) { if (encoder->funcs && encoder->funcs->early_unregister) encoder->funcs->early_unregister(encoder); drm_debugfs_encoder_remove(encoder);
}
}
__printf(5, 0)
static int __drm_encoder_init(struct drm_device *dev,
struct drm_encoder *encoder,
const struct drm_encoder_funcs *funcs,
int encoder_type, const char *name, va_list ap)
{
int ret;
/* encoder index is used with 32bit bitmasks */
if (WARN_ON(dev->mode_config.num_encoder >= 32))
return -EINVAL;
ret = drm_mode_object_add(dev, &encoder->base, DRM_MODE_OBJECT_ENCODER);
if (ret)
return ret;
encoder->dev = dev;
encoder->encoder_type = encoder_type;
encoder->funcs = funcs;
if (name) {
encoder->name = kvasprintf(GFP_KERNEL, name, ap);
} else {
encoder->name = kasprintf(GFP_KERNEL, "%s-%d",
drm_encoder_enum_list[encoder_type].name,
encoder->base.id);
}
if (!encoder->name) {
ret = -ENOMEM;
goto out_put;
}
INIT_LIST_HEAD(&encoder->bridge_chain);
list_add_tail(&encoder->head, &dev->mode_config.encoder_list);
encoder->index = dev->mode_config.num_encoder++;
out_put:
if (ret)
drm_mode_object_unregister(dev, &encoder->base);
return ret;
}
/**
* drm_encoder_init - Init a preallocated encoder
* @dev: drm device
* @encoder: the encoder to init
* @funcs: callbacks for this encoder
* @encoder_type: user visible type of the encoder
* @name: printf style format string for the encoder name, or NULL for default name
*
* Initializes a preallocated encoder. Encoder should be subclassed as part of
* driver encoder objects. At driver unload time the driver's
* &drm_encoder_funcs.destroy hook should call drm_encoder_cleanup() and kfree()
* the encoder structure. The encoder structure should not be allocated with
* devm_kzalloc().
*
* Note: consider using drmm_encoder_alloc() or drmm_encoder_init()
* instead of drm_encoder_init() to let the DRM managed resource
* infrastructure take care of cleanup and deallocation.
*
* Returns:
* Zero on success, error code on failure.
*/
int drm_encoder_init(struct drm_device *dev,
struct drm_encoder *encoder,
const struct drm_encoder_funcs *funcs,
int encoder_type, const char *name, ...)
{
va_list ap;
int ret;
WARN_ON(!funcs->destroy);
va_start(ap, name);
ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
va_end(ap);
return ret;
}
EXPORT_SYMBOL(drm_encoder_init);
/**
* drm_encoder_cleanup - cleans up an initialised encoder
* @encoder: encoder to cleanup
*
* Cleans up the encoder but doesn't free the object.
*/
void drm_encoder_cleanup(struct drm_encoder *encoder)
{
struct drm_device *dev = encoder->dev;
struct drm_bridge *bridge, *next;
/* Note that the encoder_list is considered to be static; should we
* remove the drm_encoder at runtime we would have to decrement all
* the indices on the drm_encoder after us in the encoder_list.
*/
list_for_each_entry_safe(bridge, next, &encoder->bridge_chain,
chain_node)
drm_bridge_detach(bridge);
drm_mode_object_unregister(dev, &encoder->base);
kfree(encoder->name);
list_del(&encoder->head);
dev->mode_config.num_encoder--;
memset(encoder, 0, sizeof(*encoder));
}
EXPORT_SYMBOL(drm_encoder_cleanup);
static void drmm_encoder_alloc_release(struct drm_device *dev, void *ptr)
{
struct drm_encoder *encoder = ptr;
if (WARN_ON(!encoder->dev))
return;
drm_encoder_cleanup(encoder);
}
__printf(5, 0)
static int __drmm_encoder_init(struct drm_device *dev,
struct drm_encoder *encoder,
const struct drm_encoder_funcs *funcs,
int encoder_type,
const char *name,
va_list args)
{
int ret;
if (drm_WARN_ON(dev, funcs && funcs->destroy))
return -EINVAL;
ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, args);
if (ret)
return ret;
ret = drmm_add_action_or_reset(dev, drmm_encoder_alloc_release, encoder);
if (ret)
return ret;
return 0;
}
void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset,
const struct drm_encoder_funcs *funcs,
int encoder_type, const char *name, ...)
{
void *container;
struct drm_encoder *encoder;
va_list ap;
int ret;
container = drmm_kzalloc(dev, size, GFP_KERNEL);
if (!container)
return ERR_PTR(-ENOMEM);
encoder = container + offset;
va_start(ap, name);
ret = __drmm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
va_end(ap);
if (ret)
return ERR_PTR(ret);
return container;
}
EXPORT_SYMBOL(__drmm_encoder_alloc);
/**
* drmm_encoder_init - Initialize a preallocated encoder
* @dev: drm device
* @encoder: the encoder to init
* @funcs: callbacks for this encoder (optional)
* @encoder_type: user visible type of the encoder
* @name: printf style format string for the encoder name, or NULL for default name
*
* Initializes a preallocated encoder. Encoder should be subclassed as
* part of driver encoder objects. Cleanup is automatically handled
* through registering drm_encoder_cleanup() with drmm_add_action(). The
* encoder structure should be allocated with drmm_kzalloc().
*
* The @drm_encoder_funcs.destroy hook must be NULL.
*
* Returns:
* Zero on success, error code on failure.
*/
int drmm_encoder_init(struct drm_device *dev, struct drm_encoder *encoder,
const struct drm_encoder_funcs *funcs,
int encoder_type, const char *name, ...)
{
va_list ap;
int ret;
va_start(ap, name);
ret = __drmm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
va_end(ap);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(drmm_encoder_init);
static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder)
{
struct drm_connector *connector;
struct drm_device *dev = encoder->dev;
bool uses_atomic = false;
struct drm_connector_list_iter conn_iter;
/* For atomic drivers only state objects are synchronously updated and
* protected by modeset locks, so check those first. */
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) {
if (!connector->state)
continue;
uses_atomic = true;
if (connector->state->best_encoder != encoder)
continue;
drm_connector_list_iter_end(&conn_iter);
return connector->state->crtc;
}
drm_connector_list_iter_end(&conn_iter);
/* Don't return stale data (e.g. pending async disable). */
if (uses_atomic)
return NULL;
return encoder->crtc;
}
int drm_mode_getencoder(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_mode_get_encoder *enc_resp = data;
struct drm_encoder *encoder;
struct drm_crtc *crtc;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return -EOPNOTSUPP;
encoder = drm_encoder_find(dev, file_priv, enc_resp->encoder_id);
if (!encoder)
return -ENOENT;
drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
crtc = drm_encoder_get_crtc(encoder);
if (crtc && drm_lease_held(file_priv, crtc->base.id))
enc_resp->crtc_id = crtc->base.id;
else
enc_resp->crtc_id = 0;
drm_modeset_unlock(&dev->mode_config.connection_mutex);
enc_resp->encoder_type = encoder->encoder_type;
enc_resp->encoder_id = encoder->base.id;
enc_resp->possible_crtcs = drm_lease_filter_crtcs(file_priv,
encoder->possible_crtcs);
enc_resp->possible_clones = encoder->possible_clones;
return 0;
}
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io
*/
/* Devmaps primary use is as a backend map for XDP BPF helper call
* bpf_redirect_map(). Because XDP is mostly concerned with performance we
* spent some effort to ensure the datapath with redirect maps does not use
* any locking. This is a quick note on the details.
*
* We have three possible paths to get into the devmap control plane bpf
* syscalls, bpf programs, and driver side xmit/flush operations. A bpf syscall
* will invoke an update, delete, or lookup operation. To ensure updates and
* deletes appear atomic from the datapath side xchg() is used to modify the
* netdev_map array. Then because the datapath does a lookup into the netdev_map
* array (read-only) from an RCU critical section we use call_rcu() to wait for
* an rcu grace period before free'ing the old data structures. This ensures the
* datapath always has a valid copy. However, the datapath does a "flush"
* operation that pushes any pending packets in the driver outside the RCU
* critical section. Each bpf_dtab_netdev tracks these pending operations using
* a per-cpu flush list. The bpf_dtab_netdev object will not be destroyed until
* this list is empty, indicating outstanding flush operations have completed.
*
* BPF syscalls may race with BPF program calls on any of the update, delete
* or lookup operations. As noted above the xchg() operation also keep the
* netdev_map consistent in this case. From the devmap side BPF programs
* calling into these operations are the same as multiple user space threads
* making system calls.
*
* Finally, any of the above may race with a netdev_unregister notifier. The
* unregister notifier must search for net devices in the map structure that
* contain a reference to the net device and remove them. This is a two step
* process (a) dereference the bpf_dtab_netdev object in netdev_map and (b)
* check to see if the ifindex is the same as the net_device being removed.
* When removing the dev a cmpxchg() is used to ensure the correct dev is
* removed, in the case of a concurrent update or delete operation it is
* possible that the initially referenced dev is no longer in the map. As the
* notifier hook walks the map we know that new dev references can not be
* added by the user because core infrastructure ensures dev_get_by_index()
* calls will fail at this point.
*
* The devmap_hash type is a map type which interprets keys as ifindexes and
* indexes these using a hashmap. This allows maps that use ifindex as key to be
* densely packed instead of having holes in the lookup array for unused
* ifindexes. The setup and packet enqueue/send code is shared between the two
* types of devmap; only the lookup and insertion is different.
*/
#include <linux/bpf.h>
#include <net/xdp.h>
#include <linux/filter.h>
#include <trace/events/xdp.h>
#include <linux/btf_ids.h>
#define DEV_CREATE_FLAG_MASK \
(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
struct xdp_dev_bulk_queue {
struct xdp_frame *q[DEV_MAP_BULK_SIZE];
struct list_head flush_node;
struct net_device *dev;
struct net_device *dev_rx;
struct bpf_prog *xdp_prog;
unsigned int count;
};
struct bpf_dtab_netdev {
struct net_device *dev; /* must be first member, due to tracepoint */
struct hlist_node index_hlist;
struct bpf_prog *xdp_prog;
struct rcu_head rcu;
unsigned int idx;
struct bpf_devmap_val val;
};
struct bpf_dtab {
struct bpf_map map;
struct bpf_dtab_netdev __rcu **netdev_map; /* DEVMAP type only */
struct list_head list;
/* these are only used for DEVMAP_HASH type maps */
struct hlist_head *dev_index_head;
spinlock_t index_lock;
unsigned int items;
u32 n_buckets;
};
static DEFINE_PER_CPU(struct list_head, dev_flush_list);
static DEFINE_SPINLOCK(dev_map_lock);
static LIST_HEAD(dev_map_list);
static struct hlist_head *dev_map_create_hash(unsigned int entries,
int numa_node)
{
int i;
struct hlist_head *hash;
hash = bpf_map_area_alloc((u64) entries * sizeof(*hash), numa_node);
if (hash != NULL)
for (i = 0; i < entries; i++)
INIT_HLIST_HEAD(&hash[i]);
return hash;
}
static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab,
int idx)
{
return &dtab->dev_index_head[idx & (dtab->n_buckets - 1)];
}
static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr)
{
u32 valsize = attr->value_size;
/* check sanity of attributes. 2 value sizes supported:
* 4 bytes: ifindex
* 8 bytes: ifindex + prog fd
*/
if (attr->max_entries == 0 || attr->key_size != 4 ||
(valsize != offsetofend(struct bpf_devmap_val, ifindex) &&
valsize != offsetofend(struct bpf_devmap_val, bpf_prog.fd)) ||
attr->map_flags & ~DEV_CREATE_FLAG_MASK)
return -EINVAL;
/* Lookup returns a pointer straight to dev->ifindex, so make sure the
* verifier prevents writes from the BPF side
*/
attr->map_flags |= BPF_F_RDONLY_PROG;
bpf_map_init_from_attr(&dtab->map, attr);
if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
/* hash table size must be power of 2; roundup_pow_of_two() can
* overflow into UB on 32-bit arches, so check that first
*/
if (dtab->map.max_entries > 1UL << 31)
return -EINVAL;
dtab->n_buckets = roundup_pow_of_two(dtab->map.max_entries);
dtab->dev_index_head = dev_map_create_hash(dtab->n_buckets,
dtab->map.numa_node);
if (!dtab->dev_index_head)
return -ENOMEM;
spin_lock_init(&dtab->index_lock);
} else {
dtab->netdev_map = bpf_map_area_alloc((u64) dtab->map.max_entries *
sizeof(struct bpf_dtab_netdev *),
dtab->map.numa_node);
if (!dtab->netdev_map)
return -ENOMEM;
}
return 0;
}
static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
{
struct bpf_dtab *dtab;
int err;
dtab = bpf_map_area_alloc(sizeof(*dtab), NUMA_NO_NODE);
if (!dtab)
return ERR_PTR(-ENOMEM);
err = dev_map_init_map(dtab, attr);
if (err) {
bpf_map_area_free(dtab);
return ERR_PTR(err);
}
spin_lock(&dev_map_lock);
list_add_tail_rcu(&dtab->list, &dev_map_list);
spin_unlock(&dev_map_lock);
return &dtab->map;
}
static void dev_map_free(struct bpf_map *map)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
int i;
/* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
* so the programs (can be more than one that used this map) were
* disconnected from events. The following synchronize_rcu() guarantees
* both rcu read critical sections complete and waits for
* preempt-disable regions (NAPI being the relevant context here) so we
* are certain there will be no further reads against the netdev_map and
* all flush operations are complete. Flush operations can only be done
* from NAPI context for this reason.
*/
spin_lock(&dev_map_lock);
list_del_rcu(&dtab->list);
spin_unlock(&dev_map_lock);
bpf_clear_redirect_map(map);
synchronize_rcu();
/* Make sure prior __dev_map_entry_free() have completed. */
rcu_barrier();
if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
for (i = 0; i < dtab->n_buckets; i++) {
struct bpf_dtab_netdev *dev;
struct hlist_head *head;
struct hlist_node *next;
head = dev_map_index_hash(dtab, i);
hlist_for_each_entry_safe(dev, next, head, index_hlist) {
hlist_del_rcu(&dev->index_hlist);
if (dev->xdp_prog)
bpf_prog_put(dev->xdp_prog);
dev_put(dev->dev);
kfree(dev);
}
}
bpf_map_area_free(dtab->dev_index_head);
} else {
for (i = 0; i < dtab->map.max_entries; i++) {
struct bpf_dtab_netdev *dev;
dev = rcu_dereference_raw(dtab->netdev_map[i]);
if (!dev)
continue;
if (dev->xdp_prog)
bpf_prog_put(dev->xdp_prog);
dev_put(dev->dev);
kfree(dev);
}
bpf_map_area_free(dtab->netdev_map);
}
bpf_map_area_free(dtab);
}
static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
u32 index = key ? *(u32 *)key : U32_MAX;
u32 *next = next_key;
if (index >= dtab->map.max_entries) {
*next = 0;
return 0;
}
if (index == dtab->map.max_entries - 1)
return -ENOENT;
*next = index + 1;
return 0;
}
/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
* by local_bh_disable() (from XDP calls inside NAPI). The
* rcu_read_lock_bh_held() below makes lockdep accept both.
*/
static void *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct hlist_head *head = dev_map_index_hash(dtab, key);
struct bpf_dtab_netdev *dev;
hlist_for_each_entry_rcu(dev, head, index_hlist,
lockdep_is_held(&dtab->index_lock))
if (dev->idx == key)
return dev;
return NULL;
}
static int dev_map_hash_get_next_key(struct bpf_map *map, void *key,
void *next_key)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
u32 idx, *next = next_key;
struct bpf_dtab_netdev *dev, *next_dev;
struct hlist_head *head;
int i = 0;
if (!key)
goto find_first;
idx = *(u32 *)key;
dev = __dev_map_hash_lookup_elem(map, idx);
if (!dev)
goto find_first;
next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&dev->index_hlist)),
struct bpf_dtab_netdev, index_hlist);
if (next_dev) {
*next = next_dev->idx;
return 0;
}
i = idx & (dtab->n_buckets - 1);
i++;
find_first:
for (; i < dtab->n_buckets; i++) {
head = dev_map_index_hash(dtab, i);
next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),
struct bpf_dtab_netdev,
index_hlist);
if (next_dev) {
*next = next_dev->idx;
return 0;
}
}
return -ENOENT;
}
static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog,
struct xdp_frame **frames, int n,
struct net_device *dev)
{
struct xdp_txq_info txq = { .dev = dev };
struct xdp_buff xdp;
int i, nframes = 0;
for (i = 0; i < n; i++) {
struct xdp_frame *xdpf = frames[i];
u32 act;
int err;
xdp_convert_frame_to_buff(xdpf, &xdp);
xdp.txq = &txq;
act = bpf_prog_run_xdp(xdp_prog, &xdp);
switch (act) {
case XDP_PASS:
err = xdp_update_frame_from_buff(&xdp, xdpf);
if (unlikely(err < 0))
xdp_return_frame_rx_napi(xdpf);
else
frames[nframes++] = xdpf;
break;
default:
bpf_warn_invalid_xdp_action(NULL, xdp_prog, act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(dev, xdp_prog, act);
fallthrough;
case XDP_DROP:
xdp_return_frame_rx_napi(xdpf);
break;
}
}
return nframes; /* sent frames count */
}
static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
{
struct net_device *dev = bq->dev;
unsigned int cnt = bq->count;
int sent = 0, err = 0;
int to_send = cnt;
int i;
if (unlikely(!cnt))
return;
for (i = 0; i < cnt; i++) {
struct xdp_frame *xdpf = bq->q[i];
prefetch(xdpf);
}
if (bq->xdp_prog) {
to_send = dev_map_bpf_prog_run(bq->xdp_prog, bq->q, cnt, dev);
if (!to_send)
goto out;
}
sent = dev->netdev_ops->ndo_xdp_xmit(dev, to_send, bq->q, flags);
if (sent < 0) {
/* If ndo_xdp_xmit fails with an errno, no frames have
* been xmit'ed.
*/
err = sent;
sent = 0;
}
/* If not all frames have been transmitted, it is our
* responsibility to free them
*/
for (i = sent; unlikely(i < to_send); i++)
xdp_return_frame_rx_napi(bq->q[i]);
out:
bq->count = 0;
trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, cnt - sent, err);
}
/* __dev_flush is called from xdp_do_flush() which _must_ be signalled from the
* driver before returning from its napi->poll() routine. See the comment above
* xdp_do_flush() in filter.c.
*/
void __dev_flush(void)
{
struct list_head *flush_list = this_cpu_ptr(&dev_flush_list);
struct xdp_dev_bulk_queue *bq, *tmp;
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
bq_xmit_all(bq, XDP_XMIT_FLUSH);
bq->dev_rx = NULL;
bq->xdp_prog = NULL;
__list_del_clearprev(&bq->flush_node);
}
}
#ifdef CONFIG_DEBUG_NET
bool dev_check_flush(void)
{
if (list_empty(this_cpu_ptr(&dev_flush_list)))
return false;
__dev_flush();
return true;
}
#endif
/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
* by local_bh_disable() (from XDP calls inside NAPI). The
* rcu_read_lock_bh_held() below makes lockdep accept both.
*/
static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *obj;
if (key >= map->max_entries)
return NULL;
obj = rcu_dereference_check(dtab->netdev_map[key],
rcu_read_lock_bh_held());
return obj;
}
/* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu
* variable access, and map elements stick around. See comment above
* xdp_do_flush() in filter.c.
*/
static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
struct net_device *dev_rx, struct bpf_prog *xdp_prog)
{
struct list_head *flush_list = this_cpu_ptr(&dev_flush_list);
struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
if (unlikely(bq->count == DEV_MAP_BULK_SIZE))
bq_xmit_all(bq, 0);
/* Ingress dev_rx will be the same for all xdp_frame's in
* bulk_queue, because bq stored per-CPU and must be flushed
* from net_device drivers NAPI func end.
*
* Do the same with xdp_prog and flush_list since these fields
* are only ever modified together.
*/
if (!bq->dev_rx) {
bq->dev_rx = dev_rx;
bq->xdp_prog = xdp_prog;
list_add(&bq->flush_node, flush_list);
}
bq->q[bq->count++] = xdpf;
}
static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
struct net_device *dev_rx,
struct bpf_prog *xdp_prog)
{
int err;
if (!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT))
return -EOPNOTSUPP;
if (unlikely(!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) &&
xdp_frame_has_frags(xdpf)))
return -EOPNOTSUPP;
err = xdp_ok_fwd_dev(dev, xdp_get_frame_len(xdpf));
if (unlikely(err))
return err;
bq_enqueue(dev, xdpf, dev_rx, xdp_prog);
return 0;
}
static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev *dst)
{
struct xdp_txq_info txq = { .dev = dst->dev };
struct xdp_buff xdp;
u32 act;
if (!dst->xdp_prog)
return XDP_PASS;
__skb_pull(skb, skb->mac_len);
xdp.txq = &txq;
act = bpf_prog_run_generic_xdp(skb, &xdp, dst->xdp_prog);
switch (act) {
case XDP_PASS:
__skb_push(skb, skb->mac_len);
break;
default:
bpf_warn_invalid_xdp_action(NULL, dst->xdp_prog, act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(dst->dev, dst->xdp_prog, act);
fallthrough;
case XDP_DROP:
kfree_skb(skb);
break;
}
return act;
}
int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
struct net_device *dev_rx)
{
return __xdp_enqueue(dev, xdpf, dev_rx, NULL);
}
int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
struct net_device *dev_rx)
{
struct net_device *dev = dst->dev;
return __xdp_enqueue(dev, xdpf, dev_rx, dst->xdp_prog);
}
static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf)
{
if (!obj)
return false;
if (!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT))
return false;
if (unlikely(!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) &&
xdp_frame_has_frags(xdpf)))
return false;
if (xdp_ok_fwd_dev(obj->dev, xdp_get_frame_len(xdpf)))
return false;
return true;
}
static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj,
struct net_device *dev_rx,
struct xdp_frame *xdpf)
{
struct xdp_frame *nxdpf;
nxdpf = xdpf_clone(xdpf);
if (!nxdpf)
return -ENOMEM;
bq_enqueue(obj->dev, nxdpf, dev_rx, obj->xdp_prog);
return 0;
}
static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifindex)
{
while (num_excluded--) {
if (ifindex == excluded[num_excluded])
return true;
}
return false;
}
/* Get ifindex of each upper device. 'indexes' must be able to hold at
* least MAX_NEST_DEV elements.
* Returns the number of ifindexes added.
*/
static int get_upper_ifindexes(struct net_device *dev, int *indexes)
{
struct net_device *upper;
struct list_head *iter;
int n = 0;
netdev_for_each_upper_dev_rcu(dev, upper, iter) {
indexes[n++] = upper->ifindex;
}
return n;
}
int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
struct bpf_map *map, bool exclude_ingress)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *dst, *last_dst = NULL;
int excluded_devices[1+MAX_NEST_DEV];
struct hlist_head *head;
int num_excluded = 0;
unsigned int i;
int err;
if (exclude_ingress) {
num_excluded = get_upper_ifindexes(dev_rx, excluded_devices);
excluded_devices[num_excluded++] = dev_rx->ifindex;
}
if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
for (i = 0; i < map->max_entries; i++) {
dst = rcu_dereference_check(dtab->netdev_map[i],
rcu_read_lock_bh_held());
if (!is_valid_dst(dst, xdpf))
continue;
if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))
continue;
/* we only need n-1 clones; last_dst enqueued below */
if (!last_dst) {
last_dst = dst;
continue;
}
err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);
if (err)
return err;
last_dst = dst;
}
} else { /* BPF_MAP_TYPE_DEVMAP_HASH */
for (i = 0; i < dtab->n_buckets; i++) {
head = dev_map_index_hash(dtab, i);
hlist_for_each_entry_rcu(dst, head, index_hlist,
lockdep_is_held(&dtab->index_lock)) {
if (!is_valid_dst(dst, xdpf))
continue;
if (is_ifindex_excluded(excluded_devices, num_excluded,
dst->dev->ifindex))
continue;
/* we only need n-1 clones; last_dst enqueued below */
if (!last_dst) {
last_dst = dst;
continue;
}
err = dev_map_enqueue_clone(last_dst, dev_rx, xdpf);
if (err)
return err;
last_dst = dst;
}
}
}
/* consume the last copy of the frame */
if (last_dst)
bq_enqueue(last_dst->dev, xdpf, dev_rx, last_dst->xdp_prog);
else
xdp_return_frame_rx_napi(xdpf); /* dtab is empty */
return 0;
}
int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
struct bpf_prog *xdp_prog)
{
int err;
err = xdp_ok_fwd_dev(dst->dev, skb->len);
if (unlikely(err))
return err;
/* Redirect has already succeeded semantically at this point, so we just
* return 0 even if packet is dropped. Helper below takes care of
* freeing skb.
*/
if (dev_map_bpf_prog_run_skb(skb, dst) != XDP_PASS)
return 0;
skb->dev = dst->dev;
generic_xdp_tx(skb, xdp_prog);
return 0;
}
static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst,
struct sk_buff *skb,
struct bpf_prog *xdp_prog)
{
struct sk_buff *nskb;
int err;
nskb = skb_clone(skb, GFP_ATOMIC);
if (!nskb)
return -ENOMEM;
err = dev_map_generic_redirect(dst, nskb, xdp_prog);
if (unlikely(err)) {
consume_skb(nskb);
return err;
}
return 0;
}
int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
struct bpf_prog *xdp_prog, struct bpf_map *map,
bool exclude_ingress)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *dst, *last_dst = NULL;
int excluded_devices[1+MAX_NEST_DEV];
struct hlist_head *head;
struct hlist_node *next;
int num_excluded = 0;
unsigned int i;
int err;
if (exclude_ingress) {
num_excluded = get_upper_ifindexes(dev, excluded_devices);
excluded_devices[num_excluded++] = dev->ifindex;
}
if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
for (i = 0; i < map->max_entries; i++) {
dst = rcu_dereference_check(dtab->netdev_map[i],
rcu_read_lock_bh_held());
if (!dst)
continue;
if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex))
continue;
/* we only need n-1 clones; last_dst enqueued below */
if (!last_dst) {
last_dst = dst;
continue;
}
err = dev_map_redirect_clone(last_dst, skb, xdp_prog);
if (err)
return err;
last_dst = dst;
}
} else { /* BPF_MAP_TYPE_DEVMAP_HASH */
for (i = 0; i < dtab->n_buckets; i++) {
head = dev_map_index_hash(dtab, i);
hlist_for_each_entry_safe(dst, next, head, index_hlist) {
if (!dst)
continue;
if (is_ifindex_excluded(excluded_devices, num_excluded,
dst->dev->ifindex))
continue;
/* we only need n-1 clones; last_dst enqueued below */
if (!last_dst) {
last_dst = dst;
continue;
}
err = dev_map_redirect_clone(last_dst, skb, xdp_prog);
if (err)
return err;
last_dst = dst;
}
}
}
/* consume the first skb and return */
if (last_dst)
return dev_map_generic_redirect(last_dst, skb, xdp_prog);
/* dtab is empty */
consume_skb(skb);
return 0;
}
static void *dev_map_lookup_elem(struct bpf_map *map, void *key)
{
struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, *(u32 *)key);
return obj ? &obj->val : NULL;
}
static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key)
{
struct bpf_dtab_netdev *obj = __dev_map_hash_lookup_elem(map,
*(u32 *)key);
return obj ? &obj->val : NULL;
}
static void __dev_map_entry_free(struct rcu_head *rcu)
{
struct bpf_dtab_netdev *dev;
dev = container_of(rcu, struct bpf_dtab_netdev, rcu);
if (dev->xdp_prog)
bpf_prog_put(dev->xdp_prog);
dev_put(dev->dev);
kfree(dev);
}
static long dev_map_delete_elem(struct bpf_map *map, void *key)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *old_dev;
int k = *(u32 *)key;
if (k >= map->max_entries)
return -EINVAL;
old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL));
if (old_dev) {
call_rcu(&old_dev->rcu, __dev_map_entry_free);
atomic_dec((atomic_t *)&dtab->items);
}
return 0;
}
static long dev_map_hash_delete_elem(struct bpf_map *map, void *key)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *old_dev;
int k = *(u32 *)key;
unsigned long flags;
int ret = -ENOENT;
spin_lock_irqsave(&dtab->index_lock, flags);
old_dev = __dev_map_hash_lookup_elem(map, k);
if (old_dev) {
dtab->items--;
hlist_del_init_rcu(&old_dev->index_hlist);
call_rcu(&old_dev->rcu, __dev_map_entry_free);
ret = 0;
}
spin_unlock_irqrestore(&dtab->index_lock, flags);
return ret;
}
static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net,
struct bpf_dtab *dtab,
struct bpf_devmap_val *val,
unsigned int idx)
{
struct bpf_prog *prog = NULL;
struct bpf_dtab_netdev *dev;
dev = bpf_map_kmalloc_node(&dtab->map, sizeof(*dev),
GFP_NOWAIT | __GFP_NOWARN,
dtab->map.numa_node);
if (!dev)
return ERR_PTR(-ENOMEM);
dev->dev = dev_get_by_index(net, val->ifindex);
if (!dev->dev)
goto err_out;
if (val->bpf_prog.fd > 0) {
prog = bpf_prog_get_type_dev(val->bpf_prog.fd,
BPF_PROG_TYPE_XDP, false);
if (IS_ERR(prog))
goto err_put_dev;
if (prog->expected_attach_type != BPF_XDP_DEVMAP ||
!bpf_prog_map_compatible(&dtab->map, prog))
goto err_put_prog;
}
dev->idx = idx;
if (prog) {
dev->xdp_prog = prog;
dev->val.bpf_prog.id = prog->aux->id;
} else {
dev->xdp_prog = NULL;
dev->val.bpf_prog.id = 0;
}
dev->val.ifindex = val->ifindex;
return dev;
err_put_prog:
bpf_prog_put(prog);
err_put_dev:
dev_put(dev->dev);
err_out:
kfree(dev);
return ERR_PTR(-EINVAL);
}
static long __dev_map_update_elem(struct net *net, struct bpf_map *map,
void *key, void *value, u64 map_flags)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *dev, *old_dev;
struct bpf_devmap_val val = {};
u32 i = *(u32 *)key;
if (unlikely(map_flags > BPF_EXIST))
return -EINVAL;
if (unlikely(i >= dtab->map.max_entries))
return -E2BIG;
if (unlikely(map_flags == BPF_NOEXIST))
return -EEXIST;
/* already verified value_size <= sizeof val */
memcpy(&val, value, map->value_size);
if (!val.ifindex) {
dev = NULL;
/* can not specify fd if ifindex is 0 */
if (val.bpf_prog.fd > 0)
return -EINVAL;
} else {
dev = __dev_map_alloc_node(net, dtab, &val, i);
if (IS_ERR(dev))
return PTR_ERR(dev);
}
/* Use call_rcu() here to ensure rcu critical sections have completed
* Remembering the driver side flush operation will happen before the
* net device is removed.
*/
old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev)));
if (old_dev)
call_rcu(&old_dev->rcu, __dev_map_entry_free);
else
atomic_inc((atomic_t *)&dtab->items);
return 0;
}
static long dev_map_update_elem(struct bpf_map *map, void *key, void *value,
u64 map_flags)
{
return __dev_map_update_elem(current->nsproxy->net_ns,
map, key, value, map_flags);
}
static long __dev_map_hash_update_elem(struct net *net, struct bpf_map *map,
void *key, void *value, u64 map_flags)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
struct bpf_dtab_netdev *dev, *old_dev;
struct bpf_devmap_val val = {};
u32 idx = *(u32 *)key;
unsigned long flags;
int err = -EEXIST;
/* already verified value_size <= sizeof val */
memcpy(&val, value, map->value_size);
if (unlikely(map_flags > BPF_EXIST || !val.ifindex))
return -EINVAL;
spin_lock_irqsave(&dtab->index_lock, flags);
old_dev = __dev_map_hash_lookup_elem(map, idx);
if (old_dev && (map_flags & BPF_NOEXIST))
goto out_err;
dev = __dev_map_alloc_node(net, dtab, &val, idx);
if (IS_ERR(dev)) {
err = PTR_ERR(dev);
goto out_err;
}
if (old_dev) {
hlist_del_rcu(&old_dev->index_hlist);
} else {
if (dtab->items >= dtab->map.max_entries) {
spin_unlock_irqrestore(&dtab->index_lock, flags);
call_rcu(&dev->rcu, __dev_map_entry_free);
return -E2BIG;
}
dtab->items++;
}
hlist_add_head_rcu(&dev->index_hlist,
dev_map_index_hash(dtab, idx));
spin_unlock_irqrestore(&dtab->index_lock, flags);
if (old_dev)
call_rcu(&old_dev->rcu, __dev_map_entry_free);
return 0;
out_err:
spin_unlock_irqrestore(&dtab->index_lock, flags);
return err;
}
static long dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value,
u64 map_flags)
{
return __dev_map_hash_update_elem(current->nsproxy->net_ns,
map, key, value, map_flags);
}
static long dev_map_redirect(struct bpf_map *map, u64 ifindex, u64 flags)
{
return __bpf_xdp_redirect_map(map, ifindex, flags,
BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS,
__dev_map_lookup_elem);
}
static long dev_hash_map_redirect(struct bpf_map *map, u64 ifindex, u64 flags)
{
return __bpf_xdp_redirect_map(map, ifindex, flags,
BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS,
__dev_map_hash_lookup_elem);
}
static u64 dev_map_mem_usage(const struct bpf_map *map)
{
struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
u64 usage = sizeof(struct bpf_dtab);
if (map->map_type == BPF_MAP_TYPE_DEVMAP_HASH)
usage += (u64)dtab->n_buckets * sizeof(struct hlist_head);
else
usage += (u64)map->max_entries * sizeof(struct bpf_dtab_netdev *);
usage += atomic_read((atomic_t *)&dtab->items) *
(u64)sizeof(struct bpf_dtab_netdev);
return usage;
}
BTF_ID_LIST_SINGLE(dev_map_btf_ids, struct, bpf_dtab)
const struct bpf_map_ops dev_map_ops = {
.map_meta_equal = bpf_map_meta_equal,
.map_alloc = dev_map_alloc,
.map_free = dev_map_free,
.map_get_next_key = dev_map_get_next_key,
.map_lookup_elem = dev_map_lookup_elem,
.map_update_elem = dev_map_update_elem,
.map_delete_elem = dev_map_delete_elem,
.map_check_btf = map_check_no_btf,
.map_mem_usage = dev_map_mem_usage,
.map_btf_id = &dev_map_btf_ids[0],
.map_redirect = dev_map_redirect,
};
const struct bpf_map_ops dev_map_hash_ops = {
.map_meta_equal = bpf_map_meta_equal,
.map_alloc = dev_map_alloc,
.map_free = dev_map_free,
.map_get_next_key = dev_map_hash_get_next_key,
.map_lookup_elem = dev_map_hash_lookup_elem,
.map_update_elem = dev_map_hash_update_elem,
.map_delete_elem = dev_map_hash_delete_elem,
.map_check_btf = map_check_no_btf,
.map_mem_usage = dev_map_mem_usage,
.map_btf_id = &dev_map_btf_ids[0],
.map_redirect = dev_hash_map_redirect,
};
static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab,
struct net_device *netdev)
{
unsigned long flags;
u32 i;
spin_lock_irqsave(&dtab->index_lock, flags); for (i = 0; i < dtab->n_buckets; i++) {
struct bpf_dtab_netdev *dev;
struct hlist_head *head;
struct hlist_node *next;
head = dev_map_index_hash(dtab, i); hlist_for_each_entry_safe(dev, next, head, index_hlist) {
if (netdev != dev->dev)
continue;
dtab->items--; hlist_del_rcu(&dev->index_hlist);
call_rcu(&dev->rcu, __dev_map_entry_free);
}
}
spin_unlock_irqrestore(&dtab->index_lock, flags);
}
static int dev_map_notification(struct notifier_block *notifier,
ulong event, void *ptr)
{
struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
struct bpf_dtab *dtab;
int i, cpu;
switch (event) {
case NETDEV_REGISTER:
if (!netdev->netdev_ops->ndo_xdp_xmit || netdev->xdp_bulkq)
break;
/* will be freed in free_netdev() */
netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue);
if (!netdev->xdp_bulkq)
return NOTIFY_BAD;
for_each_possible_cpu(cpu) per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev;
break;
case NETDEV_UNREGISTER:
/* This rcu_read_lock/unlock pair is needed because
* dev_map_list is an RCU list AND to ensure a delete
* operation does not free a netdev_map entry while we
* are comparing it against the netdev being unregistered.
*/
rcu_read_lock(); list_for_each_entry_rcu(dtab, &dev_map_list, list) { if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { dev_map_hash_remove_netdev(dtab, netdev);
continue;
}
for (i = 0; i < dtab->map.max_entries; i++) {
struct bpf_dtab_netdev *dev, *odev;
dev = rcu_dereference(dtab->netdev_map[i]); if (!dev || netdev != dev->dev)
continue;
odev = unrcu_pointer(cmpxchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev), NULL));
if (dev == odev) {
call_rcu(&dev->rcu,
__dev_map_entry_free);
atomic_dec((atomic_t *)&dtab->items);
}
}
}
rcu_read_unlock();
break;
default:
break;
}
return NOTIFY_OK;
}
static struct notifier_block dev_map_notifier = {
.notifier_call = dev_map_notification,
};
static int __init dev_map_init(void)
{
int cpu;
/* Assure tracepoint shadow struct _bpf_dtab_netdev is in sync */
BUILD_BUG_ON(offsetof(struct bpf_dtab_netdev, dev) !=
offsetof(struct _bpf_dtab_netdev, dev));
register_netdevice_notifier(&dev_map_notifier);
for_each_possible_cpu(cpu)
INIT_LIST_HEAD(&per_cpu(dev_flush_list, cpu));
return 0;
}
subsys_initcall(dev_map_init);
// SPDX-License-Identifier: GPL-2.0
/*
* Devices PM QoS constraints management
*
* Copyright (C) 2011 Texas Instruments, Inc.
*
* This module exposes the interface to kernel space for specifying
* per-device PM QoS dependencies. It provides infrastructure for registration
* of:
*
* Dependents on a QoS value : register requests
* Watchers of QoS value : get notified when target QoS value changes
*
* This QoS design is best effort based. Dependents register their QoS needs.
* Watchers register to keep track of the current QoS needs of the system.
* Watchers can register a per-device notification callback using the
* dev_pm_qos_*_notifier API. The notification chain data is stored in the
* per-device constraint data struct.
*
* Note about the per-device constraint data struct allocation:
* . The per-device constraints data struct ptr is stored into the device
* dev_pm_info.
* . To minimize the data usage by the per-device constraints, the data struct
* is only allocated at the first call to dev_pm_qos_add_request.
* . The data is later free'd when the device is removed from the system.
* . A global mutex protects the constraints users from the data being
* allocated and free'd.
*/
#include <linux/pm_qos.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/device.h>
#include <linux/mutex.h>
#include <linux/export.h>
#include <linux/pm_runtime.h>
#include <linux/err.h>
#include <trace/events/power.h>
#include "power.h"
static DEFINE_MUTEX(dev_pm_qos_mtx);
static DEFINE_MUTEX(dev_pm_qos_sysfs_mtx);
/**
* __dev_pm_qos_flags - Check PM QoS flags for a given device.
* @dev: Device to check the PM QoS flags for.
* @mask: Flags to check against.
*
* This routine must be called with dev->power.lock held.
*/
enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev, s32 mask)
{
struct dev_pm_qos *qos = dev->power.qos;
struct pm_qos_flags *pqf;
s32 val;
lockdep_assert_held(&dev->power.lock); if (IS_ERR_OR_NULL(qos))
return PM_QOS_FLAGS_UNDEFINED;
pqf = &qos->flags;
if (list_empty(&pqf->list))
return PM_QOS_FLAGS_UNDEFINED;
val = pqf->effective_flags & mask;
if (val)
return (val == mask) ? PM_QOS_FLAGS_ALL : PM_QOS_FLAGS_SOME;
return PM_QOS_FLAGS_NONE;
}
/**
* dev_pm_qos_flags - Check PM QoS flags for a given device (locked).
* @dev: Device to check the PM QoS flags for.
* @mask: Flags to check against.
*/
enum pm_qos_flags_status dev_pm_qos_flags(struct device *dev, s32 mask)
{
unsigned long irqflags;
enum pm_qos_flags_status ret;
spin_lock_irqsave(&dev->power.lock, irqflags);
ret = __dev_pm_qos_flags(dev, mask);
spin_unlock_irqrestore(&dev->power.lock, irqflags);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_flags);
/**
* __dev_pm_qos_resume_latency - Get resume latency constraint for a given device.
* @dev: Device to get the PM QoS constraint value for.
*
* This routine must be called with dev->power.lock held.
*/
s32 __dev_pm_qos_resume_latency(struct device *dev)
{
lockdep_assert_held(&dev->power.lock); return dev_pm_qos_raw_resume_latency(dev);
}
/**
* dev_pm_qos_read_value - Get PM QoS constraint for a given device (locked).
* @dev: Device to get the PM QoS constraint value for.
* @type: QoS request type.
*/
s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type)
{
struct dev_pm_qos *qos = dev->power.qos;
unsigned long flags;
s32 ret;
spin_lock_irqsave(&dev->power.lock, flags);
switch (type) {
case DEV_PM_QOS_RESUME_LATENCY:
ret = IS_ERR_OR_NULL(qos) ? PM_QOS_RESUME_LATENCY_NO_CONSTRAINT
: pm_qos_read_value(&qos->resume_latency);
break;
case DEV_PM_QOS_MIN_FREQUENCY:
ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE
: freq_qos_read_value(&qos->freq, FREQ_QOS_MIN);
break;
case DEV_PM_QOS_MAX_FREQUENCY:
ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE
: freq_qos_read_value(&qos->freq, FREQ_QOS_MAX);
break;
default:
WARN_ON(1);
ret = 0;
}
spin_unlock_irqrestore(&dev->power.lock, flags);
return ret;
}
/**
* apply_constraint - Add/modify/remove device PM QoS request.
* @req: Constraint request to apply
* @action: Action to perform (add/update/remove).
* @value: Value to assign to the QoS request.
*
* Internal function to update the constraints list using the PM QoS core
* code and if needed call the per-device callbacks.
*/
static int apply_constraint(struct dev_pm_qos_request *req,
enum pm_qos_req_action action, s32 value)
{
struct dev_pm_qos *qos = req->dev->power.qos;
int ret;
switch(req->type) {
case DEV_PM_QOS_RESUME_LATENCY:
if (WARN_ON(action != PM_QOS_REMOVE_REQ && value < 0))
value = 0;
ret = pm_qos_update_target(&qos->resume_latency,
&req->data.pnode, action, value);
break;
case DEV_PM_QOS_LATENCY_TOLERANCE:
ret = pm_qos_update_target(&qos->latency_tolerance,
&req->data.pnode, action, value);
if (ret) {
value = pm_qos_read_value(&qos->latency_tolerance);
req->dev->power.set_latency_tolerance(req->dev, value);
}
break;
case DEV_PM_QOS_MIN_FREQUENCY:
case DEV_PM_QOS_MAX_FREQUENCY:
ret = freq_qos_apply(&req->data.freq, action, value);
break;
case DEV_PM_QOS_FLAGS:
ret = pm_qos_update_flags(&qos->flags, &req->data.flr,
action, value);
break;
default:
ret = -EINVAL;
}
return ret;
}
/*
* dev_pm_qos_constraints_allocate
* @dev: device to allocate data for
*
* Called at the first call to add_request, for constraint data allocation
* Must be called with the dev_pm_qos_mtx mutex held
*/
static int dev_pm_qos_constraints_allocate(struct device *dev)
{
struct dev_pm_qos *qos;
struct pm_qos_constraints *c;
struct blocking_notifier_head *n;
qos = kzalloc(sizeof(*qos), GFP_KERNEL);
if (!qos)
return -ENOMEM;
n = kcalloc(3, sizeof(*n), GFP_KERNEL);
if (!n) {
kfree(qos);
return -ENOMEM;
}
c = &qos->resume_latency;
plist_head_init(&c->list);
c->target_value = PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
c->default_value = PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
c->no_constraint_value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
c->type = PM_QOS_MIN;
c->notifiers = n;
BLOCKING_INIT_NOTIFIER_HEAD(n);
c = &qos->latency_tolerance;
plist_head_init(&c->list);
c->target_value = PM_QOS_LATENCY_TOLERANCE_DEFAULT_VALUE;
c->default_value = PM_QOS_LATENCY_TOLERANCE_DEFAULT_VALUE;
c->no_constraint_value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT;
c->type = PM_QOS_MIN;
freq_constraints_init(&qos->freq);
INIT_LIST_HEAD(&qos->flags.list);
spin_lock_irq(&dev->power.lock);
dev->power.qos = qos;
spin_unlock_irq(&dev->power.lock);
return 0;
}
static void __dev_pm_qos_hide_latency_limit(struct device *dev);
static void __dev_pm_qos_hide_flags(struct device *dev);
/**
* dev_pm_qos_constraints_destroy
* @dev: target device
*
* Called from the device PM subsystem on device removal under device_pm_lock().
*/
void dev_pm_qos_constraints_destroy(struct device *dev)
{
struct dev_pm_qos *qos;
struct dev_pm_qos_request *req, *tmp;
struct pm_qos_constraints *c;
struct pm_qos_flags *f;
mutex_lock(&dev_pm_qos_sysfs_mtx);
/*
* If the device's PM QoS resume latency limit or PM QoS flags have been
* exposed to user space, they have to be hidden at this point.
*/
pm_qos_sysfs_remove_resume_latency(dev);
pm_qos_sysfs_remove_flags(dev);
mutex_lock(&dev_pm_qos_mtx);
__dev_pm_qos_hide_latency_limit(dev); __dev_pm_qos_hide_flags(dev); qos = dev->power.qos;
if (!qos)
goto out;
/* Flush the constraints lists for the device. */
c = &qos->resume_latency;
plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) {
/*
* Update constraints list and call the notification
* callbacks if needed
*/
apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
c = &qos->latency_tolerance;
plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
c = &qos->freq.min_freq;
plist_for_each_entry_safe(req, tmp, &c->list, data.freq.pnode) { apply_constraint(req, PM_QOS_REMOVE_REQ,
PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
c = &qos->freq.max_freq;
plist_for_each_entry_safe(req, tmp, &c->list, data.freq.pnode) { apply_constraint(req, PM_QOS_REMOVE_REQ,
PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
f = &qos->flags;
list_for_each_entry_safe(req, tmp, &f->list, data.flr.node) { apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
}
spin_lock_irq(&dev->power.lock);
dev->power.qos = ERR_PTR(-ENODEV);
spin_unlock_irq(&dev->power.lock);
kfree(qos->resume_latency.notifiers);
kfree(qos);
out:
mutex_unlock(&dev_pm_qos_mtx);
mutex_unlock(&dev_pm_qos_sysfs_mtx);
}
static bool dev_pm_qos_invalid_req_type(struct device *dev,
enum dev_pm_qos_req_type type)
{
return type == DEV_PM_QOS_LATENCY_TOLERANCE &&
!dev->power.set_latency_tolerance;
}
static int __dev_pm_qos_add_request(struct device *dev,
struct dev_pm_qos_request *req,
enum dev_pm_qos_req_type type, s32 value)
{
int ret = 0;
if (!dev || !req || dev_pm_qos_invalid_req_type(dev, type))
return -EINVAL;
if (WARN(dev_pm_qos_request_active(req),
"%s() called for already added request\n", __func__))
return -EINVAL;
if (IS_ERR(dev->power.qos))
ret = -ENODEV;
else if (!dev->power.qos)
ret = dev_pm_qos_constraints_allocate(dev);
trace_dev_pm_qos_add_request(dev_name(dev), type, value);
if (ret)
return ret;
req->dev = dev;
req->type = type;
if (req->type == DEV_PM_QOS_MIN_FREQUENCY)
ret = freq_qos_add_request(&dev->power.qos->freq,
&req->data.freq,
FREQ_QOS_MIN, value);
else if (req->type == DEV_PM_QOS_MAX_FREQUENCY)
ret = freq_qos_add_request(&dev->power.qos->freq,
&req->data.freq,
FREQ_QOS_MAX, value);
else
ret = apply_constraint(req, PM_QOS_ADD_REQ, value);
return ret;
}
/**
* dev_pm_qos_add_request - inserts new qos request into the list
* @dev: target device for the constraint
* @req: pointer to a preallocated handle
* @type: type of the request
* @value: defines the qos request
*
* This function inserts a new entry in the device constraints list of
* requested qos performance characteristics. It recomputes the aggregate
* QoS expectations of parameters and initializes the dev_pm_qos_request
* handle. Caller needs to save this handle for later use in updates and
* removal.
*
* Returns 1 if the aggregated constraint value has changed,
* 0 if the aggregated constraint value has not changed,
* -EINVAL in case of wrong parameters, -ENOMEM if there's not enough memory
* to allocate for data structures, -ENODEV if the device has just been removed
* from the system.
*
* Callers should ensure that the target device is not RPM_SUSPENDED before
* using this function for requests of type DEV_PM_QOS_FLAGS.
*/
int dev_pm_qos_add_request(struct device *dev, struct dev_pm_qos_request *req,
enum dev_pm_qos_req_type type, s32 value)
{
int ret;
mutex_lock(&dev_pm_qos_mtx);
ret = __dev_pm_qos_add_request(dev, req, type, value);
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_add_request);
/**
* __dev_pm_qos_update_request - Modify an existing device PM QoS request.
* @req : PM QoS request to modify.
* @new_value: New value to request.
*/
static int __dev_pm_qos_update_request(struct dev_pm_qos_request *req,
s32 new_value)
{
s32 curr_value;
int ret = 0;
if (!req) /*guard against callers passing in null */
return -EINVAL;
if (WARN(!dev_pm_qos_request_active(req),
"%s() called for unknown object\n", __func__))
return -EINVAL;
if (IS_ERR_OR_NULL(req->dev->power.qos))
return -ENODEV;
switch(req->type) {
case DEV_PM_QOS_RESUME_LATENCY:
case DEV_PM_QOS_LATENCY_TOLERANCE:
curr_value = req->data.pnode.prio;
break;
case DEV_PM_QOS_MIN_FREQUENCY:
case DEV_PM_QOS_MAX_FREQUENCY:
curr_value = req->data.freq.pnode.prio;
break;
case DEV_PM_QOS_FLAGS:
curr_value = req->data.flr.flags;
break;
default:
return -EINVAL;
}
trace_dev_pm_qos_update_request(dev_name(req->dev), req->type,
new_value);
if (curr_value != new_value)
ret = apply_constraint(req, PM_QOS_UPDATE_REQ, new_value);
return ret;
}
/**
* dev_pm_qos_update_request - modifies an existing qos request
* @req : handle to list element holding a dev_pm_qos request to use
* @new_value: defines the qos request
*
* Updates an existing dev PM qos request along with updating the
* target value.
*
* Attempts are made to make this code callable on hot code paths.
*
* Returns 1 if the aggregated constraint value has changed,
* 0 if the aggregated constraint value has not changed,
* -EINVAL in case of wrong parameters, -ENODEV if the device has been
* removed from the system
*
* Callers should ensure that the target device is not RPM_SUSPENDED before
* using this function for requests of type DEV_PM_QOS_FLAGS.
*/
int dev_pm_qos_update_request(struct dev_pm_qos_request *req, s32 new_value)
{
int ret;
mutex_lock(&dev_pm_qos_mtx);
ret = __dev_pm_qos_update_request(req, new_value);
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_update_request);
static int __dev_pm_qos_remove_request(struct dev_pm_qos_request *req)
{
int ret;
if (!req) /*guard against callers passing in null */
return -EINVAL;
if (WARN(!dev_pm_qos_request_active(req),
"%s() called for unknown object\n", __func__))
return -EINVAL;
if (IS_ERR_OR_NULL(req->dev->power.qos))
return -ENODEV;
trace_dev_pm_qos_remove_request(dev_name(req->dev), req->type,
PM_QOS_DEFAULT_VALUE);
ret = apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE);
memset(req, 0, sizeof(*req));
return ret;
}
/**
* dev_pm_qos_remove_request - modifies an existing qos request
* @req: handle to request list element
*
* Will remove pm qos request from the list of constraints and
* recompute the current target value. Call this on slow code paths.
*
* Returns 1 if the aggregated constraint value has changed,
* 0 if the aggregated constraint value has not changed,
* -EINVAL in case of wrong parameters, -ENODEV if the device has been
* removed from the system
*
* Callers should ensure that the target device is not RPM_SUSPENDED before
* using this function for requests of type DEV_PM_QOS_FLAGS.
*/
int dev_pm_qos_remove_request(struct dev_pm_qos_request *req)
{
int ret;
mutex_lock(&dev_pm_qos_mtx);
ret = __dev_pm_qos_remove_request(req);
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request);
/**
* dev_pm_qos_add_notifier - sets notification entry for changes to target value
* of per-device PM QoS constraints
*
* @dev: target device for the constraint
* @notifier: notifier block managed by caller.
* @type: request type.
*
* Will register the notifier into a notification chain that gets called
* upon changes to the target value for the device.
*
* If the device's constraints object doesn't exist when this routine is called,
* it will be created (or error code will be returned if that fails).
*/
int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier,
enum dev_pm_qos_req_type type)
{
int ret = 0;
mutex_lock(&dev_pm_qos_mtx);
if (IS_ERR(dev->power.qos))
ret = -ENODEV;
else if (!dev->power.qos)
ret = dev_pm_qos_constraints_allocate(dev);
if (ret)
goto unlock;
switch (type) {
case DEV_PM_QOS_RESUME_LATENCY:
ret = blocking_notifier_chain_register(dev->power.qos->resume_latency.notifiers,
notifier);
break;
case DEV_PM_QOS_MIN_FREQUENCY:
ret = freq_qos_add_notifier(&dev->power.qos->freq,
FREQ_QOS_MIN, notifier);
break;
case DEV_PM_QOS_MAX_FREQUENCY:
ret = freq_qos_add_notifier(&dev->power.qos->freq,
FREQ_QOS_MAX, notifier);
break;
default:
WARN_ON(1);
ret = -EINVAL;
}
unlock:
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_add_notifier);
/**
* dev_pm_qos_remove_notifier - deletes notification for changes to target value
* of per-device PM QoS constraints
*
* @dev: target device for the constraint
* @notifier: notifier block to be removed.
* @type: request type.
*
* Will remove the notifier from the notification chain that gets called
* upon changes to the target value.
*/
int dev_pm_qos_remove_notifier(struct device *dev,
struct notifier_block *notifier,
enum dev_pm_qos_req_type type)
{
int ret = 0;
mutex_lock(&dev_pm_qos_mtx);
/* Silently return if the constraints object is not present. */
if (IS_ERR_OR_NULL(dev->power.qos))
goto unlock;
switch (type) {
case DEV_PM_QOS_RESUME_LATENCY:
ret = blocking_notifier_chain_unregister(dev->power.qos->resume_latency.notifiers,
notifier);
break;
case DEV_PM_QOS_MIN_FREQUENCY:
ret = freq_qos_remove_notifier(&dev->power.qos->freq,
FREQ_QOS_MIN, notifier);
break;
case DEV_PM_QOS_MAX_FREQUENCY:
ret = freq_qos_remove_notifier(&dev->power.qos->freq,
FREQ_QOS_MAX, notifier);
break;
default:
WARN_ON(1);
ret = -EINVAL;
}
unlock:
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_remove_notifier);
/**
* dev_pm_qos_add_ancestor_request - Add PM QoS request for device's ancestor.
* @dev: Device whose ancestor to add the request for.
* @req: Pointer to the preallocated handle.
* @type: Type of the request.
* @value: Constraint latency value.
*/
int dev_pm_qos_add_ancestor_request(struct device *dev,
struct dev_pm_qos_request *req,
enum dev_pm_qos_req_type type, s32 value)
{
struct device *ancestor = dev->parent;
int ret = -ENODEV;
switch (type) {
case DEV_PM_QOS_RESUME_LATENCY:
while (ancestor && !ancestor->power.ignore_children)
ancestor = ancestor->parent;
break;
case DEV_PM_QOS_LATENCY_TOLERANCE:
while (ancestor && !ancestor->power.set_latency_tolerance)
ancestor = ancestor->parent;
break;
default:
ancestor = NULL;
}
if (ancestor)
ret = dev_pm_qos_add_request(ancestor, req, type, value);
if (ret < 0)
req->dev = NULL;
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_add_ancestor_request);
static void __dev_pm_qos_drop_user_request(struct device *dev,
enum dev_pm_qos_req_type type)
{
struct dev_pm_qos_request *req = NULL;
switch(type) {
case DEV_PM_QOS_RESUME_LATENCY:
req = dev->power.qos->resume_latency_req;
dev->power.qos->resume_latency_req = NULL;
break;
case DEV_PM_QOS_LATENCY_TOLERANCE:
req = dev->power.qos->latency_tolerance_req;
dev->power.qos->latency_tolerance_req = NULL;
break;
case DEV_PM_QOS_FLAGS:
req = dev->power.qos->flags_req;
dev->power.qos->flags_req = NULL;
break;
default:
WARN_ON(1);
return;
}
__dev_pm_qos_remove_request(req);
kfree(req);
}
static void dev_pm_qos_drop_user_request(struct device *dev,
enum dev_pm_qos_req_type type)
{
mutex_lock(&dev_pm_qos_mtx);
__dev_pm_qos_drop_user_request(dev, type);
mutex_unlock(&dev_pm_qos_mtx);
}
/**
* dev_pm_qos_expose_latency_limit - Expose PM QoS latency limit to user space.
* @dev: Device whose PM QoS latency limit is to be exposed to user space.
* @value: Initial value of the latency limit.
*/
int dev_pm_qos_expose_latency_limit(struct device *dev, s32 value)
{
struct dev_pm_qos_request *req;
int ret;
if (!device_is_registered(dev) || value < 0)
return -EINVAL;
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (!req)
return -ENOMEM;
ret = dev_pm_qos_add_request(dev, req, DEV_PM_QOS_RESUME_LATENCY, value);
if (ret < 0) {
kfree(req);
return ret;
}
mutex_lock(&dev_pm_qos_sysfs_mtx);
mutex_lock(&dev_pm_qos_mtx);
if (IS_ERR_OR_NULL(dev->power.qos))
ret = -ENODEV;
else if (dev->power.qos->resume_latency_req)
ret = -EEXIST;
if (ret < 0) {
__dev_pm_qos_remove_request(req);
kfree(req);
mutex_unlock(&dev_pm_qos_mtx);
goto out;
}
dev->power.qos->resume_latency_req = req;
mutex_unlock(&dev_pm_qos_mtx);
ret = pm_qos_sysfs_add_resume_latency(dev);
if (ret)
dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_RESUME_LATENCY);
out:
mutex_unlock(&dev_pm_qos_sysfs_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_expose_latency_limit);
static void __dev_pm_qos_hide_latency_limit(struct device *dev)
{
if (!IS_ERR_OR_NULL(dev->power.qos) && dev->power.qos->resume_latency_req) __dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_RESUME_LATENCY);
}
/**
* dev_pm_qos_hide_latency_limit - Hide PM QoS latency limit from user space.
* @dev: Device whose PM QoS latency limit is to be hidden from user space.
*/
void dev_pm_qos_hide_latency_limit(struct device *dev)
{
mutex_lock(&dev_pm_qos_sysfs_mtx);
pm_qos_sysfs_remove_resume_latency(dev);
mutex_lock(&dev_pm_qos_mtx);
__dev_pm_qos_hide_latency_limit(dev);
mutex_unlock(&dev_pm_qos_mtx);
mutex_unlock(&dev_pm_qos_sysfs_mtx);
}
EXPORT_SYMBOL_GPL(dev_pm_qos_hide_latency_limit);
/**
* dev_pm_qos_expose_flags - Expose PM QoS flags of a device to user space.
* @dev: Device whose PM QoS flags are to be exposed to user space.
* @val: Initial values of the flags.
*/
int dev_pm_qos_expose_flags(struct device *dev, s32 val)
{
struct dev_pm_qos_request *req;
int ret;
if (!device_is_registered(dev))
return -EINVAL;
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (!req)
return -ENOMEM;
ret = dev_pm_qos_add_request(dev, req, DEV_PM_QOS_FLAGS, val);
if (ret < 0) {
kfree(req);
return ret;
}
pm_runtime_get_sync(dev);
mutex_lock(&dev_pm_qos_sysfs_mtx);
mutex_lock(&dev_pm_qos_mtx);
if (IS_ERR_OR_NULL(dev->power.qos))
ret = -ENODEV;
else if (dev->power.qos->flags_req)
ret = -EEXIST;
if (ret < 0) {
__dev_pm_qos_remove_request(req);
kfree(req);
mutex_unlock(&dev_pm_qos_mtx);
goto out;
}
dev->power.qos->flags_req = req;
mutex_unlock(&dev_pm_qos_mtx);
ret = pm_qos_sysfs_add_flags(dev);
if (ret)
dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_FLAGS);
out:
mutex_unlock(&dev_pm_qos_sysfs_mtx);
pm_runtime_put(dev);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_expose_flags);
static void __dev_pm_qos_hide_flags(struct device *dev)
{
if (!IS_ERR_OR_NULL(dev->power.qos) && dev->power.qos->flags_req) __dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_FLAGS);
}
/**
* dev_pm_qos_hide_flags - Hide PM QoS flags of a device from user space.
* @dev: Device whose PM QoS flags are to be hidden from user space.
*/
void dev_pm_qos_hide_flags(struct device *dev)
{
pm_runtime_get_sync(dev);
mutex_lock(&dev_pm_qos_sysfs_mtx);
pm_qos_sysfs_remove_flags(dev);
mutex_lock(&dev_pm_qos_mtx);
__dev_pm_qos_hide_flags(dev);
mutex_unlock(&dev_pm_qos_mtx);
mutex_unlock(&dev_pm_qos_sysfs_mtx);
pm_runtime_put(dev);
}
EXPORT_SYMBOL_GPL(dev_pm_qos_hide_flags);
/**
* dev_pm_qos_update_flags - Update PM QoS flags request owned by user space.
* @dev: Device to update the PM QoS flags request for.
* @mask: Flags to set/clear.
* @set: Whether to set or clear the flags (true means set).
*/
int dev_pm_qos_update_flags(struct device *dev, s32 mask, bool set)
{
s32 value;
int ret;
pm_runtime_get_sync(dev);
mutex_lock(&dev_pm_qos_mtx);
if (IS_ERR_OR_NULL(dev->power.qos) || !dev->power.qos->flags_req) {
ret = -EINVAL;
goto out;
}
value = dev_pm_qos_requested_flags(dev);
if (set)
value |= mask;
else
value &= ~mask;
ret = __dev_pm_qos_update_request(dev->power.qos->flags_req, value);
out:
mutex_unlock(&dev_pm_qos_mtx);
pm_runtime_put(dev);
return ret;
}
/**
* dev_pm_qos_get_user_latency_tolerance - Get user space latency tolerance.
* @dev: Device to obtain the user space latency tolerance for.
*/
s32 dev_pm_qos_get_user_latency_tolerance(struct device *dev)
{
s32 ret;
mutex_lock(&dev_pm_qos_mtx);
ret = IS_ERR_OR_NULL(dev->power.qos)
|| !dev->power.qos->latency_tolerance_req ?
PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT :
dev->power.qos->latency_tolerance_req->data.pnode.prio;
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
/**
* dev_pm_qos_update_user_latency_tolerance - Update user space latency tolerance.
* @dev: Device to update the user space latency tolerance for.
* @val: New user space latency tolerance for @dev (negative values disable).
*/
int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val)
{
int ret;
mutex_lock(&dev_pm_qos_mtx);
if (IS_ERR_OR_NULL(dev->power.qos)
|| !dev->power.qos->latency_tolerance_req) {
struct dev_pm_qos_request *req;
if (val < 0) {
if (val == PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT)
ret = 0;
else
ret = -EINVAL;
goto out;
}
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (!req) {
ret = -ENOMEM;
goto out;
}
ret = __dev_pm_qos_add_request(dev, req, DEV_PM_QOS_LATENCY_TOLERANCE, val);
if (ret < 0) {
kfree(req);
goto out;
}
dev->power.qos->latency_tolerance_req = req;
} else {
if (val < 0) {
__dev_pm_qos_drop_user_request(dev, DEV_PM_QOS_LATENCY_TOLERANCE);
ret = 0;
} else {
ret = __dev_pm_qos_update_request(dev->power.qos->latency_tolerance_req, val);
}
}
out:
mutex_unlock(&dev_pm_qos_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_update_user_latency_tolerance);
/**
* dev_pm_qos_expose_latency_tolerance - Expose latency tolerance to userspace
* @dev: Device whose latency tolerance to expose
*/
int dev_pm_qos_expose_latency_tolerance(struct device *dev)
{
int ret;
if (!dev->power.set_latency_tolerance)
return -EINVAL;
mutex_lock(&dev_pm_qos_sysfs_mtx);
ret = pm_qos_sysfs_add_latency_tolerance(dev);
mutex_unlock(&dev_pm_qos_sysfs_mtx);
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_qos_expose_latency_tolerance);
/**
* dev_pm_qos_hide_latency_tolerance - Hide latency tolerance from userspace
* @dev: Device whose latency tolerance to hide
*/
void dev_pm_qos_hide_latency_tolerance(struct device *dev)
{
mutex_lock(&dev_pm_qos_sysfs_mtx);
pm_qos_sysfs_remove_latency_tolerance(dev);
mutex_unlock(&dev_pm_qos_sysfs_mtx);
/* Remove the request from user space now */
pm_runtime_get_sync(dev);
dev_pm_qos_update_user_latency_tolerance(dev,
PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT);
pm_runtime_put(dev);
}
EXPORT_SYMBOL_GPL(dev_pm_qos_hide_latency_tolerance);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MMZONE_H
#define _LINUX_MMZONE_H
#ifndef __ASSEMBLY__
#ifndef __GENERATING_BOUNDS_H
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/list_nulls.h>
#include <linux/wait.h>
#include <linux/bitops.h>
#include <linux/cache.h>
#include <linux/threads.h>
#include <linux/numa.h>
#include <linux/init.h>
#include <linux/seqlock.h>
#include <linux/nodemask.h>
#include <linux/pageblock-flags.h>
#include <linux/page-flags-layout.h>
#include <linux/atomic.h>
#include <linux/mm_types.h>
#include <linux/page-flags.h>
#include <linux/local_lock.h>
#include <linux/zswap.h>
#include <asm/page.h>
/* Free memory management - zoned buddy allocator. */
#ifndef CONFIG_ARCH_FORCE_MAX_ORDER
#define MAX_PAGE_ORDER 10
#else
#define MAX_PAGE_ORDER CONFIG_ARCH_FORCE_MAX_ORDER
#endif
#define MAX_ORDER_NR_PAGES (1 << MAX_PAGE_ORDER)
#define IS_MAX_ORDER_ALIGNED(pfn) IS_ALIGNED(pfn, MAX_ORDER_NR_PAGES)
#define NR_PAGE_ORDERS (MAX_PAGE_ORDER + 1)
/*
* PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
* costly to service. That is between allocation orders which should
* coalesce naturally under reasonable reclaim pressure and those which
* will not.
*/
#define PAGE_ALLOC_COSTLY_ORDER 3
enum migratetype {
MIGRATE_UNMOVABLE,
MIGRATE_MOVABLE,
MIGRATE_RECLAIMABLE,
MIGRATE_PCPTYPES, /* the number of types on the pcp lists */
MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,
#ifdef CONFIG_CMA
/*
* MIGRATE_CMA migration type is designed to mimic the way
* ZONE_MOVABLE works. Only movable pages can be allocated
* from MIGRATE_CMA pageblocks and page allocator never
* implicitly change migration type of MIGRATE_CMA pageblock.
*
* The way to use it is to change migratetype of a range of
* pageblocks to MIGRATE_CMA which can be done by
* __free_pageblock_cma() function.
*/
MIGRATE_CMA,
#endif
#ifdef CONFIG_MEMORY_ISOLATION
MIGRATE_ISOLATE, /* can't allocate from here */
#endif
MIGRATE_TYPES
};
/* In mm/page_alloc.c; keep in sync also with show_migration_types() there */
extern const char * const migratetype_names[MIGRATE_TYPES];
#ifdef CONFIG_CMA
# define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
# define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)
# define is_migrate_cma_folio(folio, pfn) (MIGRATE_CMA == \
get_pfnblock_flags_mask(&folio->page, pfn, MIGRATETYPE_MASK))
#else
# define is_migrate_cma(migratetype) false
# define is_migrate_cma_page(_page) false
# define is_migrate_cma_folio(folio, pfn) false
#endif
static inline bool is_migrate_movable(int mt)
{
return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
}
/*
* Check whether a migratetype can be merged with another migratetype.
*
* It is only mergeable when it can fall back to other migratetypes for
* allocation. See fallbacks[MIGRATE_TYPES][3] in page_alloc.c.
*/
static inline bool migratetype_is_mergeable(int mt)
{
return mt < MIGRATE_PCPTYPES;
}
#define for_each_migratetype_order(order, type) \
for (order = 0; order < NR_PAGE_ORDERS; order++) \
for (type = 0; type < MIGRATE_TYPES; type++)
extern int page_group_by_mobility_disabled;
#define MIGRATETYPE_MASK ((1UL << PB_migratetype_bits) - 1)
#define get_pageblock_migratetype(page) \
get_pfnblock_flags_mask(page, page_to_pfn(page), MIGRATETYPE_MASK)
#define folio_migratetype(folio) \
get_pfnblock_flags_mask(&folio->page, folio_pfn(folio), \
MIGRATETYPE_MASK)
struct free_area {
struct list_head free_list[MIGRATE_TYPES];
unsigned long nr_free;
};
struct pglist_data;
#ifdef CONFIG_NUMA
enum numa_stat_item {
NUMA_HIT, /* allocated in intended node */
NUMA_MISS, /* allocated in non intended node */
NUMA_FOREIGN, /* was intended here, hit elsewhere */
NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */
NUMA_LOCAL, /* allocation from local node */
NUMA_OTHER, /* allocation from other node */
NR_VM_NUMA_EVENT_ITEMS
};
#else
#define NR_VM_NUMA_EVENT_ITEMS 0
#endif
enum zone_stat_item {
/* First 128 byte cacheline (assuming 64 bit words) */
NR_FREE_PAGES,
NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */
NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,
NR_ZONE_ACTIVE_ANON,
NR_ZONE_INACTIVE_FILE,
NR_ZONE_ACTIVE_FILE,
NR_ZONE_UNEVICTABLE,
NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages */
NR_MLOCK, /* mlock()ed pages found and moved off LRU */
/* Second 128 byte cacheline */
NR_BOUNCE,
#if IS_ENABLED(CONFIG_ZSMALLOC)
NR_ZSPAGES, /* allocated in zsmalloc */
#endif
NR_FREE_CMA_PAGES,
#ifdef CONFIG_UNACCEPTED_MEMORY
NR_UNACCEPTED,
#endif
NR_VM_ZONE_STAT_ITEMS };
enum node_stat_item {
NR_LRU_BASE,
NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
NR_ACTIVE_ANON, /* " " " " " */
NR_INACTIVE_FILE, /* " " " " " */
NR_ACTIVE_FILE, /* " " " " " */
NR_UNEVICTABLE, /* " " " " " */
NR_SLAB_RECLAIMABLE_B,
NR_SLAB_UNRECLAIMABLE_B,
NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */
NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */
WORKINGSET_NODES,
WORKINGSET_REFAULT_BASE,
WORKINGSET_REFAULT_ANON = WORKINGSET_REFAULT_BASE,
WORKINGSET_REFAULT_FILE,
WORKINGSET_ACTIVATE_BASE,
WORKINGSET_ACTIVATE_ANON = WORKINGSET_ACTIVATE_BASE,
WORKINGSET_ACTIVATE_FILE,
WORKINGSET_RESTORE_BASE,
WORKINGSET_RESTORE_ANON = WORKINGSET_RESTORE_BASE,
WORKINGSET_RESTORE_FILE,
WORKINGSET_NODERECLAIM,
NR_ANON_MAPPED, /* Mapped anonymous pages */
NR_FILE_MAPPED, /* pagecache pages mapped into pagetables.
only modified from process context */
NR_FILE_PAGES,
NR_FILE_DIRTY,
NR_WRITEBACK,
NR_WRITEBACK_TEMP, /* Writeback using temporary buffers */
NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */
NR_SHMEM_THPS,
NR_SHMEM_PMDMAPPED,
NR_FILE_THPS,
NR_FILE_PMDMAPPED,
NR_ANON_THPS,
NR_VMSCAN_WRITE,
NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */
NR_DIRTIED, /* page dirtyings since bootup */
NR_WRITTEN, /* page writings since bootup */
NR_THROTTLED_WRITTEN, /* NR_WRITTEN while reclaim throttled */
NR_KERNEL_MISC_RECLAIMABLE, /* reclaimable non-slab kernel pages */
NR_FOLL_PIN_ACQUIRED, /* via: pin_user_page(), gup flag: FOLL_PIN */
NR_FOLL_PIN_RELEASED, /* pages returned via unpin_user_page() */
NR_KERNEL_STACK_KB, /* measured in KiB */
#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
NR_KERNEL_SCS_KB, /* measured in KiB */
#endif
NR_PAGETABLE, /* used for pagetables */
NR_SECONDARY_PAGETABLE, /* secondary pagetables, e.g. KVM pagetables */
#ifdef CONFIG_SWAP
NR_SWAPCACHE,
#endif
#ifdef CONFIG_NUMA_BALANCING
PGPROMOTE_SUCCESS, /* promote successfully */
PGPROMOTE_CANDIDATE, /* candidate pages to promote */
#endif
/* PGDEMOTE_*: pages demoted */
PGDEMOTE_KSWAPD,
PGDEMOTE_DIRECT,
PGDEMOTE_KHUGEPAGED,
NR_VM_NODE_STAT_ITEMS
};
/*
* Returns true if the item should be printed in THPs (/proc/vmstat
* currently prints number of anon, file and shmem THPs. But the item
* is charged in pages).
*/
static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item)
{
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
return false;
return item == NR_ANON_THPS ||
item == NR_FILE_THPS ||
item == NR_SHMEM_THPS ||
item == NR_SHMEM_PMDMAPPED ||
item == NR_FILE_PMDMAPPED;
}
/*
* Returns true if the value is measured in bytes (most vmstat values are
* measured in pages). This defines the API part, the internal representation
* might be different.
*/
static __always_inline bool vmstat_item_in_bytes(int idx)
{
/*
* Global and per-node slab counters track slab pages.
* It's expected that changes are multiples of PAGE_SIZE.
* Internally values are stored in pages.
*
* Per-memcg and per-lruvec counters track memory, consumed
* by individual slab objects. These counters are actually
* byte-precise.
*/
return (idx == NR_SLAB_RECLAIMABLE_B ||
idx == NR_SLAB_UNRECLAIMABLE_B);
}
/*
* We do arithmetic on the LRU lists in various places in the code,
* so it is important to keep the active lists LRU_ACTIVE higher in
* the array than the corresponding inactive lists, and to keep
* the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
*
* This has to be kept in sync with the statistics in zone_stat_item
* above and the descriptions in vmstat_text in mm/vmstat.c
*/
#define LRU_BASE 0
#define LRU_ACTIVE 1
#define LRU_FILE 2
enum lru_list {
LRU_INACTIVE_ANON = LRU_BASE,
LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
LRU_UNEVICTABLE,
NR_LRU_LISTS
};
enum vmscan_throttle_state {
VMSCAN_THROTTLE_WRITEBACK,
VMSCAN_THROTTLE_ISOLATED,
VMSCAN_THROTTLE_NOPROGRESS,
VMSCAN_THROTTLE_CONGESTED,
NR_VMSCAN_THROTTLE,
};
#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
static inline bool is_file_lru(enum lru_list lru)
{
return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
}
static inline bool is_active_lru(enum lru_list lru)
{
return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
}
#define WORKINGSET_ANON 0
#define WORKINGSET_FILE 1
#define ANON_AND_FILE 2
enum lruvec_flags {
/*
* An lruvec has many dirty pages backed by a congested BDI:
* 1. LRUVEC_CGROUP_CONGESTED is set by cgroup-level reclaim.
* It can be cleared by cgroup reclaim or kswapd.
* 2. LRUVEC_NODE_CONGESTED is set by kswapd node-level reclaim.
* It can only be cleared by kswapd.
*
* Essentially, kswapd can unthrottle an lruvec throttled by cgroup
* reclaim, but not vice versa. This only applies to the root cgroup.
* The goal is to prevent cgroup reclaim on the root cgroup (e.g.
* memory.reclaim) to unthrottle an unbalanced node (that was throttled
* by kswapd).
*/
LRUVEC_CGROUP_CONGESTED,
LRUVEC_NODE_CONGESTED,
};
#endif /* !__GENERATING_BOUNDS_H */
/*
* Evictable pages are divided into multiple generations. The youngest and the
* oldest generation numbers, max_seq and min_seq, are monotonically increasing.
* They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
* offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the
* corresponding generation. The gen counter in folio->flags stores gen+1 while
* a page is on one of lrugen->folios[]. Otherwise it stores 0.
*
* A page is added to the youngest generation on faulting. The aging needs to
* check the accessed bit at least twice before handing this page over to the
* eviction. The first check takes care of the accessed bit set on the initial
* fault; the second check makes sure this page hasn't been used since then.
* This process, AKA second chance, requires a minimum of two generations,
* hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive
* LRU, e.g., /proc/vmstat, these two generations are considered active; the
* rest of generations, if they exist, are considered inactive. See
* lru_gen_is_active().
*
* PG_active is always cleared while a page is on one of lrugen->folios[] so
* that the aging needs not to worry about it. And it's set again when a page
* considered active is isolated for non-reclaiming purposes, e.g., migration.
* See lru_gen_add_folio() and lru_gen_del_folio().
*
* MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the
* number of categories of the active/inactive LRU when keeping track of
* accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits
* in folio->flags.
*/
#define MIN_NR_GENS 2U
#define MAX_NR_GENS 4U
/*
* Each generation is divided into multiple tiers. A page accessed N times
* through file descriptors is in tier order_base_2(N). A page in the first tier
* (N=0,1) is marked by PG_referenced unless it was faulted in through page
* tables or read ahead. A page in any other tier (N>1) is marked by
* PG_referenced and PG_workingset. This implies a minimum of two tiers is
* supported without using additional bits in folio->flags.
*
* In contrast to moving across generations which requires the LRU lock, moving
* across tiers only involves atomic operations on folio->flags and therefore
* has a negligible cost in the buffered access path. In the eviction path,
* comparisons of refaulted/(evicted+protected) from the first tier and the
* rest infer whether pages accessed multiple times through file descriptors
* are statistically hot and thus worth protecting.
*
* MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
* number of categories of the active/inactive LRU when keeping track of
* accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
* folio->flags.
*/
#define MAX_NR_TIERS 4U
#ifndef __GENERATING_BOUNDS_H
struct lruvec;
struct page_vma_mapped_walk;
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
#ifdef CONFIG_LRU_GEN
enum {
LRU_GEN_ANON,
LRU_GEN_FILE,
};
enum {
LRU_GEN_CORE,
LRU_GEN_MM_WALK,
LRU_GEN_NONLEAF_YOUNG,
NR_LRU_GEN_CAPS
};
#define MIN_LRU_BATCH BITS_PER_LONG
#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64)
/* whether to keep historical stats from evicted generations */
#ifdef CONFIG_LRU_GEN_STATS
#define NR_HIST_GENS MAX_NR_GENS
#else
#define NR_HIST_GENS 1U
#endif
/*
* The youngest generation number is stored in max_seq for both anon and file
* types as they are aged on an equal footing. The oldest generation numbers are
* stored in min_seq[] separately for anon and file types as clean file pages
* can be evicted regardless of swap constraints.
*
* Normally anon and file min_seq are in sync. But if swapping is constrained,
* e.g., out of swap space, file min_seq is allowed to advance and leave anon
* min_seq behind.
*
* The number of pages in each generation is eventually consistent and therefore
* can be transiently negative when reset_batch_size() is pending.
*/
struct lru_gen_folio {
/* the aging increments the youngest generation number */
unsigned long max_seq;
/* the eviction increments the oldest generation numbers */
unsigned long min_seq[ANON_AND_FILE];
/* the birth time of each generation in jiffies */
unsigned long timestamps[MAX_NR_GENS];
/* the multi-gen LRU lists, lazily sorted on eviction */
struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the multi-gen LRU sizes, eventually consistent */
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
/* the exponential moving average of evicted+protected */
unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS];
/* the first tier doesn't need protection, hence the minus one */
unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1];
/* can be modified without holding the LRU lock */
atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
/* whether the multi-gen LRU is enabled */
bool enabled;
/* the memcg generation this lru_gen_folio belongs to */
u8 gen;
/* the list segment this lru_gen_folio belongs to */
u8 seg;
/* per-node lru_gen_folio list for global reclaim */
struct hlist_nulls_node list;
};
enum {
MM_LEAF_TOTAL, /* total leaf entries */
MM_LEAF_OLD, /* old leaf entries */
MM_LEAF_YOUNG, /* young leaf entries */
MM_NONLEAF_TOTAL, /* total non-leaf entries */
MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */
MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */
NR_MM_STATS
};
/* double-buffering Bloom filters */
#define NR_BLOOM_FILTERS 2
struct lru_gen_mm_state {
/* synced with max_seq after each iteration */
unsigned long seq;
/* where the current iteration continues after */
struct list_head *head;
/* where the last iteration ended before */
struct list_head *tail;
/* Bloom filters flip after each iteration */
unsigned long *filters[NR_BLOOM_FILTERS];
/* the mm stats for debugging */
unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
};
struct lru_gen_mm_walk {
/* the lruvec under reclaim */
struct lruvec *lruvec;
/* max_seq from lru_gen_folio: can be out of date */
unsigned long seq;
/* the next address within an mm to scan */
unsigned long next_addr;
/* to batch promoted pages */
int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* to batch the mm stats */
int mm_stats[NR_MM_STATS];
/* total batched items */
int batched;
bool can_swap;
bool force_scan;
};
/*
* For each node, memcgs are divided into two generations: the old and the
* young. For each generation, memcgs are randomly sharded into multiple bins
* to improve scalability. For each bin, the hlist_nulls is virtually divided
* into three segments: the head, the tail and the default.
*
* An onlining memcg is added to the tail of a random bin in the old generation.
* The eviction starts at the head of a random bin in the old generation. The
* per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes
* the old generation, is incremented when all its bins become empty.
*
* There are four operations:
* 1. MEMCG_LRU_HEAD, which moves a memcg to the head of a random bin in its
* current generation (old or young) and updates its "seg" to "head";
* 2. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its
* current generation (old or young) and updates its "seg" to "tail";
* 3. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old
* generation, updates its "gen" to "old" and resets its "seg" to "default";
* 4. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the
* young generation, updates its "gen" to "young" and resets its "seg" to
* "default".
*
* The events that trigger the above operations are:
* 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD;
* 2. The first attempt to reclaim a memcg below low, which triggers
* MEMCG_LRU_TAIL;
* 3. The first attempt to reclaim a memcg offlined or below reclaimable size
* threshold, which triggers MEMCG_LRU_TAIL;
* 4. The second attempt to reclaim a memcg offlined or below reclaimable size
* threshold, which triggers MEMCG_LRU_YOUNG;
* 5. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG;
* 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG;
* 7. Offlining a memcg, which triggers MEMCG_LRU_OLD.
*
* Notes:
* 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing
* of their max_seq counters ensures the eventual fairness to all eligible
* memcgs. For memcg reclaim, it still relies on mem_cgroup_iter().
* 2. There are only two valid generations: old (seq) and young (seq+1).
* MEMCG_NR_GENS is set to three so that when reading the generation counter
* locklessly, a stale value (seq-1) does not wraparound to young.
*/
#define MEMCG_NR_GENS 3
#define MEMCG_NR_BINS 8
struct lru_gen_memcg {
/* the per-node memcg generation counter */
unsigned long seq;
/* each memcg has one lru_gen_folio per node */
unsigned long nr_memcgs[MEMCG_NR_GENS];
/* per-node lru_gen_folio list for global reclaim */
struct hlist_nulls_head fifo[MEMCG_NR_GENS][MEMCG_NR_BINS];
/* protects the above */
spinlock_t lock;
};
void lru_gen_init_pgdat(struct pglist_data *pgdat);
void lru_gen_init_lruvec(struct lruvec *lruvec);
void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
void lru_gen_init_memcg(struct mem_cgroup *memcg);
void lru_gen_exit_memcg(struct mem_cgroup *memcg);
void lru_gen_online_memcg(struct mem_cgroup *memcg);
void lru_gen_offline_memcg(struct mem_cgroup *memcg);
void lru_gen_release_memcg(struct mem_cgroup *memcg);
void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid);
#else /* !CONFIG_LRU_GEN */
static inline void lru_gen_init_pgdat(struct pglist_data *pgdat)
{
}
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
{
}
static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
{
}
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_online_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_offline_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_release_memcg(struct mem_cgroup *memcg)
{
}
static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
{
}
#endif /* CONFIG_LRU_GEN */
struct lruvec {
struct list_head lists[NR_LRU_LISTS];
/* per lruvec lru_lock for memcg */
spinlock_t lru_lock;
/*
* These track the cost of reclaiming one LRU - file or anon -
* over the other. As the observed cost of reclaiming one LRU
* increases, the reclaim scan balance tips toward the other.
*/
unsigned long anon_cost;
unsigned long file_cost;
/* Non-resident age, driven by LRU movement */
atomic_long_t nonresident_age;
/* Refaults at the time of last reclaim cycle */
unsigned long refaults[ANON_AND_FILE];
/* Various lruvec state flags (enum lruvec_flags) */
unsigned long flags;
#ifdef CONFIG_LRU_GEN
/* evictable pages divided into generations */
struct lru_gen_folio lrugen;
#ifdef CONFIG_LRU_GEN_WALKS_MMU
/* to concurrently iterate lru_gen_mm_list */
struct lru_gen_mm_state mm_state;
#endif
#endif /* CONFIG_LRU_GEN */
#ifdef CONFIG_MEMCG
struct pglist_data *pgdat;
#endif
struct zswap_lruvec_state zswap_lruvec_state;
};
/* Isolate for asynchronous migration */
#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
/* Isolate unevictable pages */
#define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
/* LRU Isolation modes. */
typedef unsigned __bitwise isolate_mode_t;
enum zone_watermarks {
WMARK_MIN,
WMARK_LOW,
WMARK_HIGH,
WMARK_PROMO,
NR_WMARK
};
/*
* One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. One additional list
* for THP which will usually be GFP_MOVABLE. Even if it is another type,
* it should not contribute to serious fragmentation causing THP allocation
* failures.
*/
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define NR_PCP_THP 1
#else
#define NR_PCP_THP 0
#endif
#define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1))
#define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP)
#define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost)
#define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost)
#define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
#define wmark_pages(z, i) (z->_watermark[i] + z->watermark_boost)
/*
* Flags used in pcp->flags field.
*
* PCPF_PREV_FREE_HIGH_ORDER: a high-order page is freed in the
* previous page freeing. To avoid to drain PCP for an accident
* high-order page freeing.
*
* PCPF_FREE_HIGH_BATCH: preserve "pcp->batch" pages in PCP before
* draining PCP for consecutive high-order pages freeing without
* allocation if data cache slice of CPU is large enough. To reduce
* zone lock contention and keep cache-hot pages reusing.
*/
#define PCPF_PREV_FREE_HIGH_ORDER BIT(0)
#define PCPF_FREE_HIGH_BATCH BIT(1)
struct per_cpu_pages {
spinlock_t lock; /* Protects lists field */
int count; /* number of pages in the list */
int high; /* high watermark, emptying needed */
int high_min; /* min high watermark */
int high_max; /* max high watermark */
int batch; /* chunk size for buddy add/remove */
u8 flags; /* protected by pcp->lock */
u8 alloc_factor; /* batch scaling factor during allocate */
#ifdef CONFIG_NUMA
u8 expire; /* When 0, remote pagesets are drained */
#endif
short free_count; /* consecutive free count */
/* Lists of pages, one per migrate type stored on the pcp-lists */
struct list_head lists[NR_PCP_LISTS];
} ____cacheline_aligned_in_smp;
struct per_cpu_zonestat {
#ifdef CONFIG_SMP
s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
s8 stat_threshold;
#endif
#ifdef CONFIG_NUMA
/*
* Low priority inaccurate counters that are only folded
* on demand. Use a large type to avoid the overhead of
* folding during refresh_cpu_vm_stats.
*/
unsigned long vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
#endif
};
struct per_cpu_nodestat {
s8 stat_threshold;
s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];
};
#endif /* !__GENERATING_BOUNDS.H */
enum zone_type {
/*
* ZONE_DMA and ZONE_DMA32 are used when there are peripherals not able
* to DMA to all of the addressable memory (ZONE_NORMAL).
* On architectures where this area covers the whole 32 bit address
* space ZONE_DMA32 is used. ZONE_DMA is left for the ones with smaller
* DMA addressing constraints. This distinction is important as a 32bit
* DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit
* platforms may need both zones as they support peripherals with
* different DMA addressing limitations.
*/
#ifdef CONFIG_ZONE_DMA
ZONE_DMA,
#endif
#ifdef CONFIG_ZONE_DMA32
ZONE_DMA32,
#endif
/*
* Normal addressable memory is in ZONE_NORMAL. DMA operations can be
* performed on pages in ZONE_NORMAL if the DMA devices support
* transfers to all addressable memory.
*/
ZONE_NORMAL,
#ifdef CONFIG_HIGHMEM
/*
* A memory area that is only addressable by the kernel through
* mapping portions into its own address space. This is for example
* used by i386 to allow the kernel to address the memory beyond
* 900MB. The kernel will set up special mappings (page
* table entries on i386) for each page that the kernel needs to
* access.
*/
ZONE_HIGHMEM,
#endif
/*
* ZONE_MOVABLE is similar to ZONE_NORMAL, except that it contains
* movable pages with few exceptional cases described below. Main use
* cases for ZONE_MOVABLE are to make memory offlining/unplug more
* likely to succeed, and to locally limit unmovable allocations - e.g.,
* to increase the number of THP/huge pages. Notable special cases are:
*
* 1. Pinned pages: (long-term) pinning of movable pages might
* essentially turn such pages unmovable. Therefore, we do not allow
* pinning long-term pages in ZONE_MOVABLE. When pages are pinned and
* faulted, they come from the right zone right away. However, it is
* still possible that address space already has pages in
* ZONE_MOVABLE at the time when pages are pinned (i.e. user has
* touches that memory before pinning). In such case we migrate them
* to a different zone. When migration fails - pinning fails.
* 2. memblock allocations: kernelcore/movablecore setups might create
* situations where ZONE_MOVABLE contains unmovable allocations
* after boot. Memory offlining and allocations fail early.
* 3. Memory holes: kernelcore/movablecore setups might create very rare
* situations where ZONE_MOVABLE contains memory holes after boot,
* for example, if we have sections that are only partially
* populated. Memory offlining and allocations fail early.
* 4. PG_hwpoison pages: while poisoned pages can be skipped during
* memory offlining, such pages cannot be allocated.
* 5. Unmovable PG_offline pages: in paravirtualized environments,
* hotplugged memory blocks might only partially be managed by the
* buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The
* parts not manged by the buddy are unmovable PG_offline pages. In
* some cases (virtio-mem), such pages can be skipped during
* memory offlining, however, cannot be moved/allocated. These
* techniques might use alloc_contig_range() to hide previously
* exposed pages from the buddy again (e.g., to implement some sort
* of memory unplug in virtio-mem).
* 6. ZERO_PAGE(0), kernelcore/movablecore setups might create
* situations where ZERO_PAGE(0) which is allocated differently
* on different platforms may end up in a movable zone. ZERO_PAGE(0)
* cannot be migrated.
* 7. Memory-hotplug: when using memmap_on_memory and onlining the
* memory to the MOVABLE zone, the vmemmap pages are also placed in
* such zone. Such pages cannot be really moved around as they are
* self-stored in the range, but they are treated as movable when
* the range they describe is about to be offlined.
*
* In general, no unmovable allocations that degrade memory offlining
* should end up in ZONE_MOVABLE. Allocators (like alloc_contig_range())
* have to expect that migrating pages in ZONE_MOVABLE can fail (even
* if has_unmovable_pages() states that there are no unmovable pages,
* there can be false negatives).
*/
ZONE_MOVABLE,
#ifdef CONFIG_ZONE_DEVICE
ZONE_DEVICE,
#endif
__MAX_NR_ZONES
};
#ifndef __GENERATING_BOUNDS_H
#define ASYNC_AND_SYNC 2
struct zone {
/* Read-mostly fields */
/* zone watermarks, access with *_wmark_pages(zone) macros */
unsigned long _watermark[NR_WMARK];
unsigned long watermark_boost;
unsigned long nr_reserved_highatomic;
/*
* We don't know if the memory that we're going to allocate will be
* freeable or/and it will be released eventually, so to avoid totally
* wasting several GB of ram we must reserve some of the lower zone
* memory (otherwise we risk to run OOM on the lower zones despite
* there being tons of freeable ram on the higher zones). This array is
* recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl
* changes.
*/
long lowmem_reserve[MAX_NR_ZONES];
#ifdef CONFIG_NUMA
int node;
#endif
struct pglist_data *zone_pgdat;
struct per_cpu_pages __percpu *per_cpu_pageset;
struct per_cpu_zonestat __percpu *per_cpu_zonestats;
/*
* the high and batch values are copied to individual pagesets for
* faster access
*/
int pageset_high_min;
int pageset_high_max;
int pageset_batch;
#ifndef CONFIG_SPARSEMEM
/*
* Flags for a pageblock_nr_pages block. See pageblock-flags.h.
* In SPARSEMEM, this map is stored in struct mem_section
*/
unsigned long *pageblock_flags;
#endif /* CONFIG_SPARSEMEM */
/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
unsigned long zone_start_pfn;
/*
* spanned_pages is the total pages spanned by the zone, including
* holes, which is calculated as:
* spanned_pages = zone_end_pfn - zone_start_pfn;
*
* present_pages is physical pages existing within the zone, which
* is calculated as:
* present_pages = spanned_pages - absent_pages(pages in holes);
*
* present_early_pages is present pages existing within the zone
* located on memory available since early boot, excluding hotplugged
* memory.
*
* managed_pages is present pages managed by the buddy system, which
* is calculated as (reserved_pages includes pages allocated by the
* bootmem allocator):
* managed_pages = present_pages - reserved_pages;
*
* cma pages is present pages that are assigned for CMA use
* (MIGRATE_CMA).
*
* So present_pages may be used by memory hotplug or memory power
* management logic to figure out unmanaged pages by checking
* (present_pages - managed_pages). And managed_pages should be used
* by page allocator and vm scanner to calculate all kinds of watermarks
* and thresholds.
*
* Locking rules:
*
* zone_start_pfn and spanned_pages are protected by span_seqlock.
* It is a seqlock because it has to be read outside of zone->lock,
* and it is done in the main allocator path. But, it is written
* quite infrequently.
*
* The span_seq lock is declared along with zone->lock because it is
* frequently read in proximity to zone->lock. It's good to
* give them a chance of being in the same cacheline.
*
* Write access to present_pages at runtime should be protected by
* mem_hotplug_begin/done(). Any reader who can't tolerant drift of
* present_pages should use get_online_mems() to get a stable value.
*/
atomic_long_t managed_pages;
unsigned long spanned_pages;
unsigned long present_pages;
#if defined(CONFIG_MEMORY_HOTPLUG)
unsigned long present_early_pages;
#endif
#ifdef CONFIG_CMA
unsigned long cma_pages;
#endif
const char *name;
#ifdef CONFIG_MEMORY_ISOLATION
/*
* Number of isolated pageblock. It is used to solve incorrect
* freepage counting problem due to racy retrieving migratetype
* of pageblock. Protected by zone->lock.
*/
unsigned long nr_isolate_pageblock;
#endif
#ifdef CONFIG_MEMORY_HOTPLUG
/* see spanned/present_pages for more description */
seqlock_t span_seqlock;
#endif
int initialized;
/* Write-intensive fields used from the page allocator */
CACHELINE_PADDING(_pad1_);
/* free areas of different sizes */
struct free_area free_area[NR_PAGE_ORDERS];
#ifdef CONFIG_UNACCEPTED_MEMORY
/* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */
struct list_head unaccepted_pages;
#endif
/* zone flags, see below */
unsigned long flags;
/* Primarily protects free_area */
spinlock_t lock;
/* Write-intensive fields used by compaction and vmstats. */
CACHELINE_PADDING(_pad2_);
/*
* When free pages are below this point, additional steps are taken
* when reading the number of free pages to avoid per-cpu counter
* drift allowing watermarks to be breached
*/
unsigned long percpu_drift_mark;
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
/* pfn where compaction free scanner should start */
unsigned long compact_cached_free_pfn;
/* pfn where compaction migration scanner should start */
unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC];
unsigned long compact_init_migrate_pfn;
unsigned long compact_init_free_pfn;
#endif
#ifdef CONFIG_COMPACTION
/*
* On compaction failure, 1<<compact_defer_shift compactions
* are skipped before trying again. The number attempted since
* last failure is tracked with compact_considered.
* compact_order_failed is the minimum compaction failed order.
*/
unsigned int compact_considered;
unsigned int compact_defer_shift;
int compact_order_failed;
#endif
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
/* Set to true when the PG_migrate_skip bits should be cleared */
bool compact_blockskip_flush;
#endif
bool contiguous;
CACHELINE_PADDING(_pad3_);
/* Zone statistics */
atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
} ____cacheline_internodealigned_in_smp;
enum pgdat_flags {
PGDAT_DIRTY, /* reclaim scanning has recently found
* many dirty file pages at the tail
* of the LRU.
*/
PGDAT_WRITEBACK, /* reclaim scanning has recently found
* many pages under writeback
*/
PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */
};
enum zone_flags {
ZONE_BOOSTED_WATERMARK, /* zone recently boosted watermarks.
* Cleared when kswapd is woken.
*/
ZONE_RECLAIM_ACTIVE, /* kswapd may be scanning the zone. */
ZONE_BELOW_HIGH, /* zone is below high watermark. */
};
static inline unsigned long zone_managed_pages(struct zone *zone)
{
return (unsigned long)atomic_long_read(&zone->managed_pages);
}
static inline unsigned long zone_cma_pages(struct zone *zone)
{
#ifdef CONFIG_CMA
return zone->cma_pages;
#else
return 0;
#endif
}
static inline unsigned long zone_end_pfn(const struct zone *zone)
{
return zone->zone_start_pfn + zone->spanned_pages;
}
static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
{
return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
}
static inline bool zone_is_initialized(struct zone *zone)
{
return zone->initialized;
}
static inline bool zone_is_empty(struct zone *zone)
{
return zone->spanned_pages == 0;
}
#ifndef BUILD_VDSO32_64
/*
* The zone field is never updated after free_area_init_core()
* sets it, so none of the operations on it need to be atomic.
*/
/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH)
#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH)
/*
* Define the bit shifts to access each section. For non-existent
* sections we define the shift as 0; that plus a 0 mask ensures
* the compiler will optimise away reference to them.
*/
#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
#define KASAN_TAG_PGSHIFT (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0))
/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
#ifdef NODE_NOT_IN_PAGE_FLAGS
#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF) ? \
SECTIONS_PGOFF : ZONES_PGOFF)
#else
#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF) ? \
NODES_PGOFF : ZONES_PGOFF)
#endif
#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
#define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1)
#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
static inline enum zone_type page_zonenum(const struct page *page)
{
ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT);
return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
}
static inline enum zone_type folio_zonenum(const struct folio *folio)
{
return page_zonenum(&folio->page);
}
#ifdef CONFIG_ZONE_DEVICE
static inline bool is_zone_device_page(const struct page *page)
{
return page_zonenum(page) == ZONE_DEVICE;
}
/*
* Consecutive zone device pages should not be merged into the same sgl
* or bvec segment with other types of pages or if they belong to different
* pgmaps. Otherwise getting the pgmap of a given segment is not possible
* without scanning the entire segment. This helper returns true either if
* both pages are not zone device pages or both pages are zone device pages
* with the same pgmap.
*/
static inline bool zone_device_pages_have_same_pgmap(const struct page *a,
const struct page *b)
{
if (is_zone_device_page(a) != is_zone_device_page(b))
return false;
if (!is_zone_device_page(a))
return true;
return a->pgmap == b->pgmap;
}
extern void memmap_init_zone_device(struct zone *, unsigned long,
unsigned long, struct dev_pagemap *);
#else
static inline bool is_zone_device_page(const struct page *page)
{
return false;
}
static inline bool zone_device_pages_have_same_pgmap(const struct page *a,
const struct page *b)
{
return true;
}
#endif
static inline bool folio_is_zone_device(const struct folio *folio)
{
return is_zone_device_page(&folio->page);
}
static inline bool is_zone_movable_page(const struct page *page)
{
return page_zonenum(page) == ZONE_MOVABLE;
}
static inline bool folio_is_zone_movable(const struct folio *folio)
{
return folio_zonenum(folio) == ZONE_MOVABLE;
}
#endif
/*
* Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty
* intersection with the given zone
*/
static inline bool zone_intersects(struct zone *zone,
unsigned long start_pfn, unsigned long nr_pages)
{
if (zone_is_empty(zone))
return false;
if (start_pfn >= zone_end_pfn(zone) ||
start_pfn + nr_pages <= zone->zone_start_pfn)
return false;
return true;
}
/*
* The "priority" of VM scanning is how much of the queues we will scan in one
* go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
* queues ("queue_length >> 12") during an aging round.
*/
#define DEF_PRIORITY 12
/* Maximum number of zones on a zonelist */
#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
enum {
ZONELIST_FALLBACK, /* zonelist with fallback */
#ifdef CONFIG_NUMA
/*
* The NUMA zonelists are doubled because we need zonelists that
* restrict the allocations to a single node for __GFP_THISNODE.
*/
ZONELIST_NOFALLBACK, /* zonelist without fallback (__GFP_THISNODE) */
#endif
MAX_ZONELISTS
};
/*
* This struct contains information about a zone in a zonelist. It is stored
* here to avoid dereferences into large structures and lookups of tables
*/
struct zoneref {
struct zone *zone; /* Pointer to actual zone */
int zone_idx; /* zone_idx(zoneref->zone) */
};
/*
* One allocation request operates on a zonelist. A zonelist
* is a list of zones, the first one is the 'goal' of the
* allocation, the other zones are fallback zones, in decreasing
* priority.
*
* To speed the reading of the zonelist, the zonerefs contain the zone index
* of the entry being read. Helper functions to access information given
* a struct zoneref are
*
* zonelist_zone() - Return the struct zone * for an entry in _zonerefs
* zonelist_zone_idx() - Return the index of the zone for an entry
* zonelist_node_idx() - Return the index of the node for an entry
*/
struct zonelist {
struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
};
/*
* The array of struct pages for flatmem.
* It must be declared for SPARSEMEM as well because there are configurations
* that rely on that.
*/
extern struct page *mem_map;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct deferred_split {
spinlock_t split_queue_lock;
struct list_head split_queue;
unsigned long split_queue_len;
};
#endif
#ifdef CONFIG_MEMORY_FAILURE
/*
* Per NUMA node memory failure handling statistics.
*/
struct memory_failure_stats {
/*
* Number of raw pages poisoned.
* Cases not accounted: memory outside kernel control, offline page,
* arch-specific memory_failure (SGX), hwpoison_filter() filtered
* error events, and unpoison actions from hwpoison_unpoison.
*/
unsigned long total;
/*
* Recovery results of poisoned raw pages handled by memory_failure,
* in sync with mf_result.
* total = ignored + failed + delayed + recovered.
* total * PAGE_SIZE * #nodes = /proc/meminfo/HardwareCorrupted.
*/
unsigned long ignored;
unsigned long failed;
unsigned long delayed;
unsigned long recovered;
};
#endif
/*
* On NUMA machines, each NUMA node would have a pg_data_t to describe
* it's memory layout. On UMA machines there is a single pglist_data which
* describes the whole memory.
*
* Memory statistics and page replacement data structures are maintained on a
* per-zone basis.
*/
typedef struct pglist_data {
/*
* node_zones contains just the zones for THIS node. Not all of the
* zones may be populated, but it is the full list. It is referenced by
* this node's node_zonelists as well as other node's node_zonelists.
*/
struct zone node_zones[MAX_NR_ZONES];
/*
* node_zonelists contains references to all zones in all nodes.
* Generally the first zones will be references to this node's
* node_zones.
*/
struct zonelist node_zonelists[MAX_ZONELISTS];
int nr_zones; /* number of populated zones in this node */
#ifdef CONFIG_FLATMEM /* means !SPARSEMEM */
struct page *node_mem_map;
#ifdef CONFIG_PAGE_EXTENSION
struct page_ext *node_page_ext;
#endif
#endif
#if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
/*
* Must be held any time you expect node_start_pfn,
* node_present_pages, node_spanned_pages or nr_zones to stay constant.
* Also synchronizes pgdat->first_deferred_pfn during deferred page
* init.
*
* pgdat_resize_lock() and pgdat_resize_unlock() are provided to
* manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG
* or CONFIG_DEFERRED_STRUCT_PAGE_INIT.
*
* Nests above zone->lock and zone->span_seqlock
*/
spinlock_t node_size_lock;
#endif
unsigned long node_start_pfn;
unsigned long node_present_pages; /* total number of physical pages */
unsigned long node_spanned_pages; /* total size of physical page
range, including holes */
int node_id;
wait_queue_head_t kswapd_wait;
wait_queue_head_t pfmemalloc_wait;
/* workqueues for throttling reclaim for different reasons. */
wait_queue_head_t reclaim_wait[NR_VMSCAN_THROTTLE];
atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */
unsigned long nr_reclaim_start; /* nr pages written while throttled
* when throttling started. */
#ifdef CONFIG_MEMORY_HOTPLUG
struct mutex kswapd_lock;
#endif
struct task_struct *kswapd; /* Protected by kswapd_lock */
int kswapd_order;
enum zone_type kswapd_highest_zoneidx;
int kswapd_failures; /* Number of 'reclaimed == 0' runs */
#ifdef CONFIG_COMPACTION
int kcompactd_max_order;
enum zone_type kcompactd_highest_zoneidx;
wait_queue_head_t kcompactd_wait;
struct task_struct *kcompactd;
bool proactive_compact_trigger;
#endif
/*
* This is a per-node reserve of pages that are not available
* to userspace allocations.
*/
unsigned long totalreserve_pages;
#ifdef CONFIG_NUMA
/*
* node reclaim becomes active if more unmapped pages exist.
*/
unsigned long min_unmapped_pages;
unsigned long min_slab_pages;
#endif /* CONFIG_NUMA */
/* Write-intensive fields used by page reclaim */
CACHELINE_PADDING(_pad1_);
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
/*
* If memory initialisation on large machines is deferred then this
* is the first PFN that needs to be initialised.
*/
unsigned long first_deferred_pfn;
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct deferred_split deferred_split_queue;
#endif
#ifdef CONFIG_NUMA_BALANCING
/* start time in ms of current promote rate limit period */
unsigned int nbp_rl_start;
/* number of promote candidate pages at start time of current rate limit period */
unsigned long nbp_rl_nr_cand;
/* promote threshold in ms */
unsigned int nbp_threshold;
/* start time in ms of current promote threshold adjustment period */
unsigned int nbp_th_start;
/*
* number of promote candidate pages at start time of current promote
* threshold adjustment period
*/
unsigned long nbp_th_nr_cand;
#endif
/* Fields commonly accessed by the page reclaim scanner */
/*
* NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
*
* Use mem_cgroup_lruvec() to look up lruvecs.
*/
struct lruvec __lruvec;
unsigned long flags;
#ifdef CONFIG_LRU_GEN
/* kswap mm walk data */
struct lru_gen_mm_walk mm_walk;
/* lru_gen_folio list */
struct lru_gen_memcg memcg_lru;
#endif
CACHELINE_PADDING(_pad2_);
/* Per-node vmstats */
struct per_cpu_nodestat __percpu *per_cpu_nodestats;
atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
#ifdef CONFIG_NUMA
struct memory_tier __rcu *memtier;
#endif
#ifdef CONFIG_MEMORY_FAILURE
struct memory_failure_stats mf_stats;
#endif
} pg_data_t;
#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
{
return pgdat->node_start_pfn + pgdat->node_spanned_pages;
}
#include <linux/memory_hotplug.h>
void build_all_zonelists(pg_data_t *pgdat);
void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order,
enum zone_type highest_zoneidx);
bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
int highest_zoneidx, unsigned int alloc_flags,
long free_pages);
bool zone_watermark_ok(struct zone *z, unsigned int order,
unsigned long mark, int highest_zoneidx,
unsigned int alloc_flags);
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
unsigned long mark, int highest_zoneidx);
/*
* Memory initialization context, use to differentiate memory added by
* the platform statically or via memory hotplug interface.
*/
enum meminit_context {
MEMINIT_EARLY,
MEMINIT_HOTPLUG,
};
extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
unsigned long size);
extern void lruvec_init(struct lruvec *lruvec);
static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
{
#ifdef CONFIG_MEMCG
return lruvec->pgdat;
#else
return container_of(lruvec, struct pglist_data, __lruvec);
#endif
}
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
int local_memory_node(int node_id);
#else
static inline int local_memory_node(int node_id) { return node_id; };
#endif
/*
* zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
*/
#define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
#ifdef CONFIG_ZONE_DEVICE
static inline bool zone_is_zone_device(struct zone *zone)
{
return zone_idx(zone) == ZONE_DEVICE;
}
#else
static inline bool zone_is_zone_device(struct zone *zone)
{
return false;
}
#endif
/*
* Returns true if a zone has pages managed by the buddy allocator.
* All the reclaim decisions have to use this function rather than
* populated_zone(). If the whole zone is reserved then we can easily
* end up with populated_zone() && !managed_zone().
*/
static inline bool managed_zone(struct zone *zone)
{
return zone_managed_pages(zone);
}
/* Returns true if a zone has memory */
static inline bool populated_zone(struct zone *zone)
{
return zone->present_pages;
}
#ifdef CONFIG_NUMA
static inline int zone_to_nid(struct zone *zone)
{
return zone->node;
}
static inline void zone_set_nid(struct zone *zone, int nid)
{
zone->node = nid;
}
#else
static inline int zone_to_nid(struct zone *zone)
{
return 0;
}
static inline void zone_set_nid(struct zone *zone, int nid) {}
#endif
extern int movable_zone;
static inline int is_highmem_idx(enum zone_type idx)
{
#ifdef CONFIG_HIGHMEM
return (idx == ZONE_HIGHMEM ||
(idx == ZONE_MOVABLE && movable_zone == ZONE_HIGHMEM));
#else
return 0;
#endif
}
/**
* is_highmem - helper function to quickly check if a struct zone is a
* highmem zone or not. This is an attempt to keep references
* to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
* @zone: pointer to struct zone variable
* Return: 1 for a highmem zone, 0 otherwise
*/
static inline int is_highmem(struct zone *zone)
{
return is_highmem_idx(zone_idx(zone));
}
#ifdef CONFIG_ZONE_DMA
bool has_managed_dma(void);
#else
static inline bool has_managed_dma(void)
{
return false;
}
#endif
#ifndef CONFIG_NUMA
extern struct pglist_data contig_page_data;
static inline struct pglist_data *NODE_DATA(int nid)
{
return &contig_page_data;
}
#else /* CONFIG_NUMA */
#include <asm/mmzone.h>
#endif /* !CONFIG_NUMA */
extern struct pglist_data *first_online_pgdat(void);
extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
extern struct zone *next_zone(struct zone *zone);
/**
* for_each_online_pgdat - helper macro to iterate over all online nodes
* @pgdat: pointer to a pg_data_t variable
*/
#define for_each_online_pgdat(pgdat) \
for (pgdat = first_online_pgdat(); \
pgdat; \
pgdat = next_online_pgdat(pgdat))
/**
* for_each_zone - helper macro to iterate over all memory zones
* @zone: pointer to struct zone variable
*
* The user only needs to declare the zone variable, for_each_zone
* fills it in.
*/
#define for_each_zone(zone) \
for (zone = (first_online_pgdat())->node_zones; \
zone; \
zone = next_zone(zone))
#define for_each_populated_zone(zone) \
for (zone = (first_online_pgdat())->node_zones; \
zone; \
zone = next_zone(zone)) \
if (!populated_zone(zone)) \
; /* do nothing */ \
else
static inline struct zone *zonelist_zone(struct zoneref *zoneref)
{
return zoneref->zone;
}
static inline int zonelist_zone_idx(struct zoneref *zoneref)
{
return zoneref->zone_idx;
}
static inline int zonelist_node_idx(struct zoneref *zoneref)
{
return zone_to_nid(zoneref->zone);
}
struct zoneref *__next_zones_zonelist(struct zoneref *z,
enum zone_type highest_zoneidx,
nodemask_t *nodes);
/**
* next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
* @z: The cursor used as a starting point for the search
* @highest_zoneidx: The zone index of the highest zone to return
* @nodes: An optional nodemask to filter the zonelist with
*
* This function returns the next zone at or below a given zone index that is
* within the allowed nodemask using a cursor as the starting point for the
* search. The zoneref returned is a cursor that represents the current zone
* being examined. It should be advanced by one before calling
* next_zones_zonelist again.
*
* Return: the next zone at or below highest_zoneidx within the allowed
* nodemask using a cursor within a zonelist as a starting point
*/
static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
enum zone_type highest_zoneidx,
nodemask_t *nodes)
{
if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx))
return z;
return __next_zones_zonelist(z, highest_zoneidx, nodes);
}
/**
* first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
* @zonelist: The zonelist to search for a suitable zone
* @highest_zoneidx: The zone index of the highest zone to return
* @nodes: An optional nodemask to filter the zonelist with
*
* This function returns the first zone at or below a given zone index that is
* within the allowed nodemask. The zoneref returned is a cursor that can be
* used to iterate the zonelist with next_zones_zonelist by advancing it by
* one before calling.
*
* When no eligible zone is found, zoneref->zone is NULL (zoneref itself is
* never NULL). This may happen either genuinely, or due to concurrent nodemask
* update due to cpuset modification.
*
* Return: Zoneref pointer for the first suitable zone found
*/
static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
enum zone_type highest_zoneidx,
nodemask_t *nodes)
{
return next_zones_zonelist(zonelist->_zonerefs,
highest_zoneidx, nodes);
}
/**
* for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
* @zone: The current zone in the iterator
* @z: The current pointer within zonelist->_zonerefs being iterated
* @zlist: The zonelist being iterated
* @highidx: The zone index of the highest zone to return
* @nodemask: Nodemask allowed by the allocator
*
* This iterator iterates though all zones at or below a given zone index and
* within a given nodemask
*/
#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \
zone; \
z = next_zones_zonelist(++z, highidx, nodemask), \
zone = zonelist_zone(z))
#define for_next_zone_zonelist_nodemask(zone, z, highidx, nodemask) \
for (zone = z->zone; \
zone; \
z = next_zones_zonelist(++z, highidx, nodemask), \
zone = zonelist_zone(z))
/**
* for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
* @zone: The current zone in the iterator
* @z: The current pointer within zonelist->zones being iterated
* @zlist: The zonelist being iterated
* @highidx: The zone index of the highest zone to return
*
* This iterator iterates though all zones at or below a given zone index.
*/
#define for_each_zone_zonelist(zone, z, zlist, highidx) \
for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
/* Whether the 'nodes' are all movable nodes */
static inline bool movable_only_nodes(nodemask_t *nodes)
{
struct zonelist *zonelist;
struct zoneref *z;
int nid;
if (nodes_empty(*nodes))
return false;
/*
* We can chose arbitrary node from the nodemask to get a
* zonelist as they are interlinked. We just need to find
* at least one zone that can satisfy kernel allocations.
*/
nid = first_node(*nodes);
zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
z = first_zones_zonelist(zonelist, ZONE_NORMAL, nodes);
return (!z->zone) ? true : false;
}
#ifdef CONFIG_SPARSEMEM
#include <asm/sparsemem.h>
#endif
#ifdef CONFIG_FLATMEM
#define pfn_to_nid(pfn) (0)
#endif
#ifdef CONFIG_SPARSEMEM
/*
* PA_SECTION_SHIFT physical address to/from section number
* PFN_SECTION_SHIFT pfn to/from section number
*/
#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
#define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)
#define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
#define SECTION_BLOCKFLAGS_BITS \
((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
#if (MAX_PAGE_ORDER + PAGE_SHIFT) > SECTION_SIZE_BITS
#error Allocator MAX_PAGE_ORDER exceeds SECTION_SIZE
#endif
static inline unsigned long pfn_to_section_nr(unsigned long pfn)
{
return pfn >> PFN_SECTION_SHIFT;
}
static inline unsigned long section_nr_to_pfn(unsigned long sec)
{
return sec << PFN_SECTION_SHIFT;
}
#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
#define SUBSECTION_SHIFT 21
#define SUBSECTION_SIZE (1UL << SUBSECTION_SHIFT)
#define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)
#define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT)
#define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
#if SUBSECTION_SHIFT > SECTION_SIZE_BITS
#error Subsection size exceeds section size
#else
#define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))
#endif
#define SUBSECTION_ALIGN_UP(pfn) ALIGN((pfn), PAGES_PER_SUBSECTION)
#define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
struct mem_section_usage {
struct rcu_head rcu;
#ifdef CONFIG_SPARSEMEM_VMEMMAP
DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);
#endif
/* See declaration of similar field in struct zone */
unsigned long pageblock_flags[0];
};
void subsection_map_init(unsigned long pfn, unsigned long nr_pages);
struct page;
struct page_ext;
struct mem_section {
/*
* This is, logically, a pointer to an array of struct
* pages. However, it is stored with some other magic.
* (see sparse.c::sparse_init_one_section())
*
* Additionally during early boot we encode node id of
* the location of the section here to guide allocation.
* (see sparse.c::memory_present())
*
* Making it a UL at least makes someone do a cast
* before using it wrong.
*/
unsigned long section_mem_map;
struct mem_section_usage *usage;
#ifdef CONFIG_PAGE_EXTENSION
/*
* If SPARSEMEM, pgdat doesn't have page_ext pointer. We use
* section. (see page_ext.h about this.)
*/
struct page_ext *page_ext;
unsigned long pad;
#endif
/*
* WARNING: mem_section must be a power-of-2 in size for the
* calculation and use of SECTION_ROOT_MASK to make sense.
*/
};
#ifdef CONFIG_SPARSEMEM_EXTREME
#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
#else
#define SECTIONS_PER_ROOT 1
#endif
#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
#define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
#ifdef CONFIG_SPARSEMEM_EXTREME
extern struct mem_section **mem_section;
#else
extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
#endif
static inline unsigned long *section_to_usemap(struct mem_section *ms)
{
return ms->usage->pageblock_flags;
}
static inline struct mem_section *__nr_to_section(unsigned long nr)
{
unsigned long root = SECTION_NR_TO_ROOT(nr);
if (unlikely(root >= NR_SECTION_ROOTS))
return NULL;
#ifdef CONFIG_SPARSEMEM_EXTREME
if (!mem_section || !mem_section[root])
return NULL;
#endif
return &mem_section[root][nr & SECTION_ROOT_MASK];
}
extern size_t mem_section_usage_size(void);
/*
* We use the lower bits of the mem_map pointer to store
* a little bit of information. The pointer is calculated
* as mem_map - section_nr_to_pfn(pnum). The result is
* aligned to the minimum alignment of the two values:
* 1. All mem_map arrays are page-aligned.
* 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT
* lowest bits. PFN_SECTION_SHIFT is arch-specific
* (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the
* worst combination is powerpc with 256k pages,
* which results in PFN_SECTION_SHIFT equal 6.
* To sum it up, at least 6 bits are available on all architectures.
* However, we can exceed 6 bits on some other architectures except
* powerpc (e.g. 15 bits are available on x86_64, 13 bits are available
* with the worst case of 64K pages on arm64) if we make sure the
* exceeded bit is not applicable to powerpc.
*/
enum {
SECTION_MARKED_PRESENT_BIT,
SECTION_HAS_MEM_MAP_BIT,
SECTION_IS_ONLINE_BIT,
SECTION_IS_EARLY_BIT,
#ifdef CONFIG_ZONE_DEVICE
SECTION_TAINT_ZONE_DEVICE_BIT,
#endif
SECTION_MAP_LAST_BIT,
};
#define SECTION_MARKED_PRESENT BIT(SECTION_MARKED_PRESENT_BIT)
#define SECTION_HAS_MEM_MAP BIT(SECTION_HAS_MEM_MAP_BIT)
#define SECTION_IS_ONLINE BIT(SECTION_IS_ONLINE_BIT)
#define SECTION_IS_EARLY BIT(SECTION_IS_EARLY_BIT)
#ifdef CONFIG_ZONE_DEVICE
#define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT)
#endif
#define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1))
#define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT
static inline struct page *__section_mem_map_addr(struct mem_section *section)
{
unsigned long map = section->section_mem_map;
map &= SECTION_MAP_MASK;
return (struct page *)map;
}
static inline int present_section(struct mem_section *section)
{
return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
}
static inline int present_section_nr(unsigned long nr)
{
return present_section(__nr_to_section(nr));
}
static inline int valid_section(struct mem_section *section)
{
return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
}
static inline int early_section(struct mem_section *section)
{
return (section && (section->section_mem_map & SECTION_IS_EARLY));
}
static inline int valid_section_nr(unsigned long nr)
{
return valid_section(__nr_to_section(nr));
}
static inline int online_section(struct mem_section *section)
{
return (section && (section->section_mem_map & SECTION_IS_ONLINE));
}
#ifdef CONFIG_ZONE_DEVICE
static inline int online_device_section(struct mem_section *section)
{
unsigned long flags = SECTION_IS_ONLINE | SECTION_TAINT_ZONE_DEVICE;
return section && ((section->section_mem_map & flags) == flags);
}
#else
static inline int online_device_section(struct mem_section *section)
{
return 0;
}
#endif
static inline int online_section_nr(unsigned long nr)
{
return online_section(__nr_to_section(nr));
}
#ifdef CONFIG_MEMORY_HOTPLUG
void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
#endif
static inline struct mem_section *__pfn_to_section(unsigned long pfn)
{
return __nr_to_section(pfn_to_section_nr(pfn));
}
extern unsigned long __highest_present_section_nr;
static inline int subsection_map_index(unsigned long pfn)
{
return (pfn & ~(PAGE_SECTION_MASK)) / PAGES_PER_SUBSECTION;
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
{
int idx = subsection_map_index(pfn);
return test_bit(idx, READ_ONCE(ms->usage)->subsection_map);
}
#else
static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
{
return 1;
}
#endif
#ifndef CONFIG_HAVE_ARCH_PFN_VALID
/**
* pfn_valid - check if there is a valid memory map entry for a PFN
* @pfn: the page frame number to check
*
* Check if there is a valid memory map entry aka struct page for the @pfn.
* Note, that availability of the memory map entry does not imply that
* there is actual usable memory at that @pfn. The struct page may
* represent a hole or an unusable page frame.
*
* Return: 1 for PFNs that have memory map entries and 0 otherwise
*/
static inline int pfn_valid(unsigned long pfn)
{
struct mem_section *ms;
int ret;
/*
* Ensure the upper PAGE_SHIFT bits are clear in the
* pfn. Else it might lead to false positives when
* some of the upper bits are set, but the lower bits
* match a valid pfn.
*/
if (PHYS_PFN(PFN_PHYS(pfn)) != pfn)
return 0;
if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
return 0;
ms = __pfn_to_section(pfn); rcu_read_lock_sched(); if (!valid_section(ms)) { rcu_read_unlock_sched();
return 0;
}
/*
* Traditionally early sections always returned pfn_valid() for
* the entire section-sized span.
*/
ret = early_section(ms) || pfn_section_valid(ms, pfn); rcu_read_unlock_sched();
return ret;
}
#endif
static inline int pfn_in_present_section(unsigned long pfn)
{
if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
return 0;
return present_section(__pfn_to_section(pfn));
}
static inline unsigned long next_present_section_nr(unsigned long section_nr)
{
while (++section_nr <= __highest_present_section_nr) {
if (present_section_nr(section_nr))
return section_nr;
}
return -1;
}
/*
* These are _only_ used during initialisation, therefore they
* can use __initdata ... They could have names to indicate
* this restriction.
*/
#ifdef CONFIG_NUMA
#define pfn_to_nid(pfn) \
({ \
unsigned long __pfn_to_nid_pfn = (pfn); \
page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \
})
#else
#define pfn_to_nid(pfn) (0)
#endif
void sparse_init(void);
#else
#define sparse_init() do {} while (0)
#define sparse_index_init(_sec, _nid) do {} while (0)
#define pfn_in_present_section pfn_valid
#define subsection_map_init(_pfn, _nr_pages) do {} while (0)
#endif /* CONFIG_SPARSEMEM */
#endif /* !__GENERATING_BOUNDS.H */
#endif /* !__ASSEMBLY__ */
#endif /* _LINUX_MMZONE_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Filesystem access notification for Linux
*
* Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
*/
#ifndef __LINUX_FSNOTIFY_BACKEND_H
#define __LINUX_FSNOTIFY_BACKEND_H
#ifdef __KERNEL__
#include <linux/idr.h> /* inotify uses this */
#include <linux/fs.h> /* struct inode */
#include <linux/list.h>
#include <linux/path.h> /* struct path */
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/user_namespace.h>
#include <linux/refcount.h>
#include <linux/mempool.h>
#include <linux/sched/mm.h>
/*
* IN_* from inotfy.h lines up EXACTLY with FS_*, this is so we can easily
* convert between them. dnotify only needs conversion at watch creation
* so no perf loss there. fanotify isn't defined yet, so it can use the
* wholes if it needs more events.
*/
#define FS_ACCESS 0x00000001 /* File was accessed */
#define FS_MODIFY 0x00000002 /* File was modified */
#define FS_ATTRIB 0x00000004 /* Metadata changed */
#define FS_CLOSE_WRITE 0x00000008 /* Writable file was closed */
#define FS_CLOSE_NOWRITE 0x00000010 /* Unwritable file closed */
#define FS_OPEN 0x00000020 /* File was opened */
#define FS_MOVED_FROM 0x00000040 /* File was moved from X */
#define FS_MOVED_TO 0x00000080 /* File was moved to Y */
#define FS_CREATE 0x00000100 /* Subfile was created */
#define FS_DELETE 0x00000200 /* Subfile was deleted */
#define FS_DELETE_SELF 0x00000400 /* Self was deleted */
#define FS_MOVE_SELF 0x00000800 /* Self was moved */
#define FS_OPEN_EXEC 0x00001000 /* File was opened for exec */
#define FS_UNMOUNT 0x00002000 /* inode on umount fs */
#define FS_Q_OVERFLOW 0x00004000 /* Event queued overflowed */
#define FS_ERROR 0x00008000 /* Filesystem Error (fanotify) */
/*
* FS_IN_IGNORED overloads FS_ERROR. It is only used internally by inotify
* which does not support FS_ERROR.
*/
#define FS_IN_IGNORED 0x00008000 /* last inotify event here */
#define FS_OPEN_PERM 0x00010000 /* open event in an permission hook */
#define FS_ACCESS_PERM 0x00020000 /* access event in a permissions hook */
#define FS_OPEN_EXEC_PERM 0x00040000 /* open/exec event in a permission hook */
/*
* Set on inode mark that cares about things that happen to its children.
* Always set for dnotify and inotify.
* Set on inode/sb/mount marks that care about parent/name info.
*/
#define FS_EVENT_ON_CHILD 0x08000000
#define FS_RENAME 0x10000000 /* File was renamed */
#define FS_DN_MULTISHOT 0x20000000 /* dnotify multishot */
#define FS_ISDIR 0x40000000 /* event occurred against dir */
#define FS_MOVE (FS_MOVED_FROM | FS_MOVED_TO)
/*
* Directory entry modification events - reported only to directory
* where entry is modified and not to a watching parent.
* The watching parent may get an FS_ATTRIB|FS_EVENT_ON_CHILD event
* when a directory entry inside a child subdir changes.
*/
#define ALL_FSNOTIFY_DIRENT_EVENTS (FS_CREATE | FS_DELETE | FS_MOVE | FS_RENAME)
#define ALL_FSNOTIFY_PERM_EVENTS (FS_OPEN_PERM | FS_ACCESS_PERM | \
FS_OPEN_EXEC_PERM)
/*
* This is a list of all events that may get sent to a parent that is watching
* with flag FS_EVENT_ON_CHILD based on fs event on a child of that directory.
*/
#define FS_EVENTS_POSS_ON_CHILD (ALL_FSNOTIFY_PERM_EVENTS | \
FS_ACCESS | FS_MODIFY | FS_ATTRIB | \
FS_CLOSE_WRITE | FS_CLOSE_NOWRITE | \
FS_OPEN | FS_OPEN_EXEC)
/*
* This is a list of all events that may get sent with the parent inode as the
* @to_tell argument of fsnotify().
* It may include events that can be sent to an inode/sb/mount mark, but cannot
* be sent to a parent watching children.
*/
#define FS_EVENTS_POSS_TO_PARENT (FS_EVENTS_POSS_ON_CHILD)
/* Events that can be reported to backends */
#define ALL_FSNOTIFY_EVENTS (ALL_FSNOTIFY_DIRENT_EVENTS | \
FS_EVENTS_POSS_ON_CHILD | \
FS_DELETE_SELF | FS_MOVE_SELF | \
FS_UNMOUNT | FS_Q_OVERFLOW | FS_IN_IGNORED | \
FS_ERROR)
/* Extra flags that may be reported with event or control handling of events */
#define ALL_FSNOTIFY_FLAGS (FS_ISDIR | FS_EVENT_ON_CHILD | FS_DN_MULTISHOT)
#define ALL_FSNOTIFY_BITS (ALL_FSNOTIFY_EVENTS | ALL_FSNOTIFY_FLAGS)
struct fsnotify_group;
struct fsnotify_event;
struct fsnotify_mark;
struct fsnotify_event_private_data;
struct fsnotify_fname;
struct fsnotify_iter_info;
struct mem_cgroup;
/*
* Each group much define these ops. The fsnotify infrastructure will call
* these operations for each relevant group.
*
* handle_event - main call for a group to handle an fs event
* @group: group to notify
* @mask: event type and flags
* @data: object that event happened on
* @data_type: type of object for fanotify_data_XXX() accessors
* @dir: optional directory associated with event -
* if @file_name is not NULL, this is the directory that
* @file_name is relative to
* @file_name: optional file name associated with event
* @cookie: inotify rename cookie
* @iter_info: array of marks from this group that are interested in the event
*
* handle_inode_event - simple variant of handle_event() for groups that only
* have inode marks and don't have ignore mask
* @mark: mark to notify
* @mask: event type and flags
* @inode: inode that event happened on
* @dir: optional directory associated with event -
* if @file_name is not NULL, this is the directory that
* @file_name is relative to.
* Either @inode or @dir must be non-NULL.
* @file_name: optional file name associated with event
* @cookie: inotify rename cookie
*
* free_group_priv - called when a group refcnt hits 0 to clean up the private union
* freeing_mark - called when a mark is being destroyed for some reason. The group
* MUST be holding a reference on each mark and that reference must be
* dropped in this function. inotify uses this function to send
* userspace messages that marks have been removed.
*/
struct fsnotify_ops {
int (*handle_event)(struct fsnotify_group *group, u32 mask,
const void *data, int data_type, struct inode *dir,
const struct qstr *file_name, u32 cookie,
struct fsnotify_iter_info *iter_info);
int (*handle_inode_event)(struct fsnotify_mark *mark, u32 mask,
struct inode *inode, struct inode *dir,
const struct qstr *file_name, u32 cookie);
void (*free_group_priv)(struct fsnotify_group *group);
void (*freeing_mark)(struct fsnotify_mark *mark, struct fsnotify_group *group);
void (*free_event)(struct fsnotify_group *group, struct fsnotify_event *event);
/* called on final put+free to free memory */
void (*free_mark)(struct fsnotify_mark *mark);
};
/*
* all of the information about the original object we want to now send to
* a group. If you want to carry more info from the accessing task to the
* listener this structure is where you need to be adding fields.
*/
struct fsnotify_event {
struct list_head list;
};
/*
* A group is a "thing" that wants to receive notification about filesystem
* events. The mask holds the subset of event types this group cares about.
* refcnt on a group is up to the implementor and at any moment if it goes 0
* everything will be cleaned up.
*/
struct fsnotify_group {
const struct fsnotify_ops *ops; /* how this group handles things */
/*
* How the refcnt is used is up to each group. When the refcnt hits 0
* fsnotify will clean up all of the resources associated with this group.
* As an example, the dnotify group will always have a refcnt=1 and that
* will never change. Inotify, on the other hand, has a group per
* inotify_init() and the refcnt will hit 0 only when that fd has been
* closed.
*/
refcount_t refcnt; /* things with interest in this group */
/* needed to send notification to userspace */
spinlock_t notification_lock; /* protect the notification_list */
struct list_head notification_list; /* list of event_holder this group needs to send to userspace */
wait_queue_head_t notification_waitq; /* read() on the notification file blocks on this waitq */
unsigned int q_len; /* events on the queue */
unsigned int max_events; /* maximum events allowed on the list */
/*
* Valid fsnotify group priorities. Events are send in order from highest
* priority to lowest priority. We default to the lowest priority.
*/
#define FS_PRIO_0 0 /* normal notifiers, no permissions */
#define FS_PRIO_1 1 /* fanotify content based access control */
#define FS_PRIO_2 2 /* fanotify pre-content access */
unsigned int priority;
bool shutdown; /* group is being shut down, don't queue more events */
#define FSNOTIFY_GROUP_USER 0x01 /* user allocated group */
#define FSNOTIFY_GROUP_DUPS 0x02 /* allow multiple marks per object */
#define FSNOTIFY_GROUP_NOFS 0x04 /* group lock is not direct reclaim safe */
int flags;
unsigned int owner_flags; /* stored flags of mark_mutex owner */
/* stores all fastpath marks assoc with this group so they can be cleaned on unregister */
struct mutex mark_mutex; /* protect marks_list */
atomic_t user_waits; /* Number of tasks waiting for user
* response */
struct list_head marks_list; /* all inode marks for this group */
struct fasync_struct *fsn_fa; /* async notification */
struct fsnotify_event *overflow_event; /* Event we queue when the
* notification list is too
* full */
struct mem_cgroup *memcg; /* memcg to charge allocations */
/* groups can define private fields here or use the void *private */
union {
void *private;
#ifdef CONFIG_INOTIFY_USER
struct inotify_group_private_data {
spinlock_t idr_lock;
struct idr idr;
struct ucounts *ucounts;
} inotify_data;
#endif
#ifdef CONFIG_FANOTIFY
struct fanotify_group_private_data {
/* Hash table of events for merge */
struct hlist_head *merge_hash;
/* allows a group to block waiting for a userspace response */
struct list_head access_list;
wait_queue_head_t access_waitq;
int flags; /* flags from fanotify_init() */
int f_flags; /* event_f_flags from fanotify_init() */
struct ucounts *ucounts;
mempool_t error_events_pool;
} fanotify_data;
#endif /* CONFIG_FANOTIFY */
};
};
/*
* These helpers are used to prevent deadlock when reclaiming inodes with
* evictable marks of the same group that is allocating a new mark.
*/
static inline void fsnotify_group_lock(struct fsnotify_group *group)
{
mutex_lock(&group->mark_mutex);
if (group->flags & FSNOTIFY_GROUP_NOFS)
group->owner_flags = memalloc_nofs_save();
}
static inline void fsnotify_group_unlock(struct fsnotify_group *group)
{
if (group->flags & FSNOTIFY_GROUP_NOFS)
memalloc_nofs_restore(group->owner_flags); mutex_unlock(&group->mark_mutex);
}
static inline void fsnotify_group_assert_locked(struct fsnotify_group *group)
{
WARN_ON_ONCE(!mutex_is_locked(&group->mark_mutex));
if (group->flags & FSNOTIFY_GROUP_NOFS)
WARN_ON_ONCE(!(current->flags & PF_MEMALLOC_NOFS));
}
/* When calling fsnotify tell it if the data is a path or inode */
enum fsnotify_data_type {
FSNOTIFY_EVENT_NONE,
FSNOTIFY_EVENT_PATH,
FSNOTIFY_EVENT_INODE,
FSNOTIFY_EVENT_DENTRY,
FSNOTIFY_EVENT_ERROR,
};
struct fs_error_report {
int error;
struct inode *inode;
struct super_block *sb;
};
static inline struct inode *fsnotify_data_inode(const void *data, int data_type)
{
switch (data_type) {
case FSNOTIFY_EVENT_INODE:
return (struct inode *)data;
case FSNOTIFY_EVENT_DENTRY:
return d_inode(data);
case FSNOTIFY_EVENT_PATH:
return d_inode(((const struct path *)data)->dentry);
case FSNOTIFY_EVENT_ERROR:
return ((struct fs_error_report *)data)->inode;
default:
return NULL;
}
}
static inline struct dentry *fsnotify_data_dentry(const void *data, int data_type)
{
switch (data_type) {
case FSNOTIFY_EVENT_DENTRY:
/* Non const is needed for dget() */
return (struct dentry *)data;
case FSNOTIFY_EVENT_PATH:
return ((const struct path *)data)->dentry;
default:
return NULL;
}
}
static inline const struct path *fsnotify_data_path(const void *data,
int data_type)
{
switch (data_type) {
case FSNOTIFY_EVENT_PATH:
return data;
default:
return NULL;
}
}
static inline struct super_block *fsnotify_data_sb(const void *data,
int data_type)
{
switch (data_type) {
case FSNOTIFY_EVENT_INODE:
return ((struct inode *)data)->i_sb;
case FSNOTIFY_EVENT_DENTRY:
return ((struct dentry *)data)->d_sb;
case FSNOTIFY_EVENT_PATH:
return ((const struct path *)data)->dentry->d_sb;
case FSNOTIFY_EVENT_ERROR:
return ((struct fs_error_report *) data)->sb;
default:
return NULL;
}
}
static inline struct fs_error_report *fsnotify_data_error_report(
const void *data,
int data_type)
{
switch (data_type) {
case FSNOTIFY_EVENT_ERROR:
return (struct fs_error_report *) data;
default:
return NULL;
}
}
/*
* Index to merged marks iterator array that correlates to a type of watch.
* The type of watched object can be deduced from the iterator type, but not
* the other way around, because an event can match different watched objects
* of the same object type.
* For example, both parent and child are watching an object of type inode.
*/
enum fsnotify_iter_type {
FSNOTIFY_ITER_TYPE_INODE,
FSNOTIFY_ITER_TYPE_VFSMOUNT,
FSNOTIFY_ITER_TYPE_SB,
FSNOTIFY_ITER_TYPE_PARENT,
FSNOTIFY_ITER_TYPE_INODE2,
FSNOTIFY_ITER_TYPE_COUNT
};
/* The type of object that a mark is attached to */
enum fsnotify_obj_type {
FSNOTIFY_OBJ_TYPE_ANY = -1,
FSNOTIFY_OBJ_TYPE_INODE,
FSNOTIFY_OBJ_TYPE_VFSMOUNT,
FSNOTIFY_OBJ_TYPE_SB,
FSNOTIFY_OBJ_TYPE_COUNT,
FSNOTIFY_OBJ_TYPE_DETACHED = FSNOTIFY_OBJ_TYPE_COUNT
};
static inline bool fsnotify_valid_obj_type(unsigned int obj_type)
{
return (obj_type < FSNOTIFY_OBJ_TYPE_COUNT);
}
struct fsnotify_iter_info {
struct fsnotify_mark *marks[FSNOTIFY_ITER_TYPE_COUNT];
struct fsnotify_group *current_group;
unsigned int report_mask;
int srcu_idx;
};
static inline bool fsnotify_iter_should_report_type(
struct fsnotify_iter_info *iter_info, int iter_type)
{
return (iter_info->report_mask & (1U << iter_type));
}
static inline void fsnotify_iter_set_report_type(
struct fsnotify_iter_info *iter_info, int iter_type)
{
iter_info->report_mask |= (1U << iter_type);
}
static inline struct fsnotify_mark *fsnotify_iter_mark(
struct fsnotify_iter_info *iter_info, int iter_type)
{
if (fsnotify_iter_should_report_type(iter_info, iter_type)) return iter_info->marks[iter_type];
return NULL;
}
static inline int fsnotify_iter_step(struct fsnotify_iter_info *iter, int type,
struct fsnotify_mark **markp)
{
while (type < FSNOTIFY_ITER_TYPE_COUNT) { *markp = fsnotify_iter_mark(iter, type);
if (*markp)
break;
type++;
}
return type;
}
#define FSNOTIFY_ITER_FUNCS(name, NAME) \
static inline struct fsnotify_mark *fsnotify_iter_##name##_mark( \
struct fsnotify_iter_info *iter_info) \
{ \
return fsnotify_iter_mark(iter_info, FSNOTIFY_ITER_TYPE_##NAME); \
}
FSNOTIFY_ITER_FUNCS(inode, INODE)FSNOTIFY_ITER_FUNCS(parent, PARENT)FSNOTIFY_ITER_FUNCS(vfsmount, VFSMOUNT)FSNOTIFY_ITER_FUNCS(sb, SB)
#define fsnotify_foreach_iter_type(type) \
for (type = 0; type < FSNOTIFY_ITER_TYPE_COUNT; type++)
#define fsnotify_foreach_iter_mark_type(iter, mark, type) \
for (type = 0; \
type = fsnotify_iter_step(iter, type, &mark), \
type < FSNOTIFY_ITER_TYPE_COUNT; \
type++)
/*
* fsnotify_connp_t is what we embed in objects which connector can be attached
* to. fsnotify_connp_t * is how we refer from connector back to object.
*/
struct fsnotify_mark_connector;
typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t;
/*
* Inode/vfsmount/sb point to this structure which tracks all marks attached to
* the inode/vfsmount/sb. The reference to inode/vfsmount/sb is held by this
* structure. We destroy this structure when there are no more marks attached
* to it. The structure is protected by fsnotify_mark_srcu.
*/
struct fsnotify_mark_connector {
spinlock_t lock;
unsigned short type; /* Type of object [lock] */
#define FSNOTIFY_CONN_FLAG_HAS_IREF 0x02
unsigned short flags; /* flags [lock] */
union {
/* Object pointer [lock] */
fsnotify_connp_t *obj;
/* Used listing heads to free after srcu period expires */
struct fsnotify_mark_connector *destroy_next;
};
struct hlist_head list;
};
/*
* A mark is simply an object attached to an in core inode which allows an
* fsnotify listener to indicate they are either no longer interested in events
* of a type matching mask or only interested in those events.
*
* These are flushed when an inode is evicted from core and may be flushed
* when the inode is modified (as seen by fsnotify_access). Some fsnotify
* users (such as dnotify) will flush these when the open fd is closed and not
* at inode eviction or modification.
*
* Text in brackets is showing the lock(s) protecting modifications of a
* particular entry. obj_lock means either inode->i_lock or
* mnt->mnt_root->d_lock depending on the mark type.
*/
struct fsnotify_mark {
/* Mask this mark is for [mark->lock, group->mark_mutex] */
__u32 mask;
/* We hold one for presence in g_list. Also one ref for each 'thing'
* in kernel that found and may be using this mark. */
refcount_t refcnt;
/* Group this mark is for. Set on mark creation, stable until last ref
* is dropped */
struct fsnotify_group *group;
/* List of marks by group->marks_list. Also reused for queueing
* mark into destroy_list when it's waiting for the end of SRCU period
* before it can be freed. [group->mark_mutex] */
struct list_head g_list;
/* Protects inode / mnt pointers, flags, masks */
spinlock_t lock;
/* List of marks for inode / vfsmount [connector->lock, mark ref] */
struct hlist_node obj_list;
/* Head of list of marks for an object [mark ref] */
struct fsnotify_mark_connector *connector;
/* Events types and flags to ignore [mark->lock, group->mark_mutex] */
__u32 ignore_mask;
/* General fsnotify mark flags */
#define FSNOTIFY_MARK_FLAG_ALIVE 0x0001
#define FSNOTIFY_MARK_FLAG_ATTACHED 0x0002
/* inotify mark flags */
#define FSNOTIFY_MARK_FLAG_EXCL_UNLINK 0x0010
#define FSNOTIFY_MARK_FLAG_IN_ONESHOT 0x0020
/* fanotify mark flags */
#define FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY 0x0100
#define FSNOTIFY_MARK_FLAG_NO_IREF 0x0200
#define FSNOTIFY_MARK_FLAG_HAS_IGNORE_FLAGS 0x0400
#define FSNOTIFY_MARK_FLAG_HAS_FSID 0x0800
#define FSNOTIFY_MARK_FLAG_WEAK_FSID 0x1000
unsigned int flags; /* flags [mark->lock] */
};
#ifdef CONFIG_FSNOTIFY
/* called from the vfs helpers */
/* main fsnotify call to send events */
extern int fsnotify(__u32 mask, const void *data, int data_type,
struct inode *dir, const struct qstr *name,
struct inode *inode, u32 cookie);
extern int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
int data_type);
extern void __fsnotify_inode_delete(struct inode *inode);
extern void __fsnotify_vfsmount_delete(struct vfsmount *mnt);
extern void fsnotify_sb_delete(struct super_block *sb);
extern u32 fsnotify_get_cookie(void);
static inline __u32 fsnotify_parent_needed_mask(__u32 mask)
{
/* FS_EVENT_ON_CHILD is set on marks that want parent/name info */
if (!(mask & FS_EVENT_ON_CHILD))
return 0;
/*
* This object might be watched by a mark that cares about parent/name
* info, does it care about the specific set of events that can be
* reported with parent/name info?
*/
return mask & FS_EVENTS_POSS_TO_PARENT;
}
static inline int fsnotify_inode_watches_children(struct inode *inode)
{
/* FS_EVENT_ON_CHILD is set if the inode may care */
if (!(inode->i_fsnotify_mask & FS_EVENT_ON_CHILD))
return 0;
/* this inode might care about child events, does it care about the
* specific set of events that can happen on a child? */
return inode->i_fsnotify_mask & FS_EVENTS_POSS_ON_CHILD;
}
/*
* Update the dentry with a flag indicating the interest of its parent to receive
* filesystem events when those events happens to this dentry->d_inode.
*/
static inline void fsnotify_update_flags(struct dentry *dentry)
{
assert_spin_locked(&dentry->d_lock);
/*
* Serialisation of setting PARENT_WATCHED on the dentries is provided
* by d_lock. If inotify_inode_watched changes after we have taken
* d_lock, the following __fsnotify_update_child_dentry_flags call will
* find our entry, so it will spin until we complete here, and update
* us with the new state.
*/
if (fsnotify_inode_watches_children(dentry->d_parent->d_inode)) dentry->d_flags |= DCACHE_FSNOTIFY_PARENT_WATCHED;
else
dentry->d_flags &= ~DCACHE_FSNOTIFY_PARENT_WATCHED;
}
/* called from fsnotify listeners, such as fanotify or dnotify */
/* create a new group */
extern struct fsnotify_group *fsnotify_alloc_group(
const struct fsnotify_ops *ops,
int flags);
/* get reference to a group */
extern void fsnotify_get_group(struct fsnotify_group *group);
/* drop reference on a group from fsnotify_alloc_group */
extern void fsnotify_put_group(struct fsnotify_group *group);
/* group destruction begins, stop queuing new events */
extern void fsnotify_group_stop_queueing(struct fsnotify_group *group);
/* destroy group */
extern void fsnotify_destroy_group(struct fsnotify_group *group);
/* fasync handler function */
extern int fsnotify_fasync(int fd, struct file *file, int on);
/* Free event from memory */
extern void fsnotify_destroy_event(struct fsnotify_group *group,
struct fsnotify_event *event);
/* attach the event to the group notification queue */
extern int fsnotify_insert_event(struct fsnotify_group *group,
struct fsnotify_event *event,
int (*merge)(struct fsnotify_group *,
struct fsnotify_event *),
void (*insert)(struct fsnotify_group *,
struct fsnotify_event *));
static inline int fsnotify_add_event(struct fsnotify_group *group,
struct fsnotify_event *event,
int (*merge)(struct fsnotify_group *,
struct fsnotify_event *))
{
return fsnotify_insert_event(group, event, merge, NULL);
}
/* Queue overflow event to a notification group */
static inline void fsnotify_queue_overflow(struct fsnotify_group *group)
{
fsnotify_add_event(group, group->overflow_event, NULL);
}
static inline bool fsnotify_is_overflow_event(u32 mask)
{
return mask & FS_Q_OVERFLOW;
}
static inline bool fsnotify_notify_queue_is_empty(struct fsnotify_group *group)
{
assert_spin_locked(&group->notification_lock);
return list_empty(&group->notification_list);
}
extern bool fsnotify_notify_queue_is_empty(struct fsnotify_group *group);
/* return, but do not dequeue the first event on the notification queue */
extern struct fsnotify_event *fsnotify_peek_first_event(struct fsnotify_group *group);
/* return AND dequeue the first event on the notification queue */
extern struct fsnotify_event *fsnotify_remove_first_event(struct fsnotify_group *group);
/* Remove event queued in the notification list */
extern void fsnotify_remove_queued_event(struct fsnotify_group *group,
struct fsnotify_event *event);
/* functions used to manipulate the marks attached to inodes */
/*
* Canonical "ignore mask" including event flags.
*
* Note the subtle semantic difference from the legacy ->ignored_mask.
* ->ignored_mask traditionally only meant which events should be ignored,
* while ->ignore_mask also includes flags regarding the type of objects on
* which events should be ignored.
*/
static inline __u32 fsnotify_ignore_mask(struct fsnotify_mark *mark)
{
__u32 ignore_mask = mark->ignore_mask;
/* The event flags in ignore mask take effect */
if (mark->flags & FSNOTIFY_MARK_FLAG_HAS_IGNORE_FLAGS)
return ignore_mask;
/*
* Legacy behavior:
* - Always ignore events on dir
* - Ignore events on child if parent is watching children
*/
ignore_mask |= FS_ISDIR;
ignore_mask &= ~FS_EVENT_ON_CHILD;
ignore_mask |= mark->mask & FS_EVENT_ON_CHILD;
return ignore_mask;
}
/* Legacy ignored_mask - only event types to ignore */
static inline __u32 fsnotify_ignored_events(struct fsnotify_mark *mark)
{
return mark->ignore_mask & ALL_FSNOTIFY_EVENTS;
}
/*
* Check if mask (or ignore mask) should be applied depending if victim is a
* directory and whether it is reported to a watching parent.
*/
static inline bool fsnotify_mask_applicable(__u32 mask, bool is_dir,
int iter_type)
{
/* Should mask be applied to a directory? */
if (is_dir && !(mask & FS_ISDIR))
return false;
/* Should mask be applied to a child? */
if (iter_type == FSNOTIFY_ITER_TYPE_PARENT && !(mask & FS_EVENT_ON_CHILD))
return false;
return true;
}
/*
* Effective ignore mask taking into account if event victim is a
* directory and whether it is reported to a watching parent.
*/
static inline __u32 fsnotify_effective_ignore_mask(struct fsnotify_mark *mark,
bool is_dir, int iter_type)
{
__u32 ignore_mask = fsnotify_ignored_events(mark);
if (!ignore_mask)
return 0;
/* For non-dir and non-child, no need to consult the event flags */
if (!is_dir && iter_type != FSNOTIFY_ITER_TYPE_PARENT)
return ignore_mask;
ignore_mask = fsnotify_ignore_mask(mark); if (!fsnotify_mask_applicable(ignore_mask, is_dir, iter_type))
return 0;
return ignore_mask & ALL_FSNOTIFY_EVENTS;
}
/* Get mask for calculating object interest taking ignore mask into account */
static inline __u32 fsnotify_calc_mask(struct fsnotify_mark *mark)
{
__u32 mask = mark->mask;
if (!fsnotify_ignored_events(mark))
return mask;
/* Interest in FS_MODIFY may be needed for clearing ignore mask */
if (!(mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY))
mask |= FS_MODIFY;
/*
* If mark is interested in ignoring events on children, the object must
* show interest in those events for fsnotify_parent() to notice it.
*/
return mask | mark->ignore_mask;
}
/* Get mask of events for a list of marks */
extern __u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn);
/* Calculate mask of events for a list of marks */
extern void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn);
extern void fsnotify_init_mark(struct fsnotify_mark *mark,
struct fsnotify_group *group);
/* Find mark belonging to given group in the list of marks */
extern struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp,
struct fsnotify_group *group);
/* attach the mark to the object */
extern int fsnotify_add_mark(struct fsnotify_mark *mark,
fsnotify_connp_t *connp, unsigned int obj_type,
int add_flags);
extern int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
fsnotify_connp_t *connp,
unsigned int obj_type, int add_flags);
/* attach the mark to the inode */
static inline int fsnotify_add_inode_mark(struct fsnotify_mark *mark,
struct inode *inode,
int add_flags)
{
return fsnotify_add_mark(mark, &inode->i_fsnotify_marks,
FSNOTIFY_OBJ_TYPE_INODE, add_flags);
}
static inline int fsnotify_add_inode_mark_locked(struct fsnotify_mark *mark,
struct inode *inode,
int add_flags)
{
return fsnotify_add_mark_locked(mark, &inode->i_fsnotify_marks,
FSNOTIFY_OBJ_TYPE_INODE, add_flags);
}
/* given a group and a mark, flag mark to be freed when all references are dropped */
extern void fsnotify_destroy_mark(struct fsnotify_mark *mark,
struct fsnotify_group *group);
/* detach mark from inode / mount list, group list, drop inode reference */
extern void fsnotify_detach_mark(struct fsnotify_mark *mark);
/* free mark */
extern void fsnotify_free_mark(struct fsnotify_mark *mark);
/* Wait until all marks queued for destruction are destroyed */
extern void fsnotify_wait_marks_destroyed(void);
/* Clear all of the marks of a group attached to a given object type */
extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
unsigned int obj_type);
/* run all the marks in a group, and clear all of the vfsmount marks */
static inline void fsnotify_clear_vfsmount_marks_by_group(struct fsnotify_group *group)
{
fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_VFSMOUNT);
}
/* run all the marks in a group, and clear all of the inode marks */
static inline void fsnotify_clear_inode_marks_by_group(struct fsnotify_group *group)
{
fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_INODE);
}
/* run all the marks in a group, and clear all of the sn marks */
static inline void fsnotify_clear_sb_marks_by_group(struct fsnotify_group *group)
{
fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_SB);
}
extern void fsnotify_get_mark(struct fsnotify_mark *mark);
extern void fsnotify_put_mark(struct fsnotify_mark *mark);
extern void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info);
extern bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info);
static inline void fsnotify_init_event(struct fsnotify_event *event)
{
INIT_LIST_HEAD(&event->list);
}
#else
static inline int fsnotify(__u32 mask, const void *data, int data_type,
struct inode *dir, const struct qstr *name,
struct inode *inode, u32 cookie)
{
return 0;
}
static inline int __fsnotify_parent(struct dentry *dentry, __u32 mask,
const void *data, int data_type)
{
return 0;
}
static inline void __fsnotify_inode_delete(struct inode *inode)
{}
static inline void __fsnotify_vfsmount_delete(struct vfsmount *mnt)
{}
static inline void fsnotify_sb_delete(struct super_block *sb)
{}
static inline void fsnotify_update_flags(struct dentry *dentry)
{}
static inline u32 fsnotify_get_cookie(void)
{
return 0;
}
static inline void fsnotify_unmount_inodes(struct super_block *sb)
{}
#endif /* CONFIG_FSNOTIFY */
#endif /* __KERNEL __ */
#endif /* __LINUX_FSNOTIFY_BACKEND_H */
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/power/common.c - Common device power management code.
*
* Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
*/
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/pm_clock.h>
#include <linux/acpi.h>
#include <linux/pm_domain.h>
#include "power.h"
/**
* dev_pm_get_subsys_data - Create or refcount power.subsys_data for device.
* @dev: Device to handle.
*
* If power.subsys_data is NULL, point it to a new object, otherwise increment
* its reference counter. Return 0 if new object has been created or refcount
* increased, otherwise negative error code.
*/
int dev_pm_get_subsys_data(struct device *dev)
{
struct pm_subsys_data *psd;
psd = kzalloc(sizeof(*psd), GFP_KERNEL);
if (!psd)
return -ENOMEM;
spin_lock_irq(&dev->power.lock);
if (dev->power.subsys_data) {
dev->power.subsys_data->refcount++;
} else {
spin_lock_init(&psd->lock);
psd->refcount = 1;
dev->power.subsys_data = psd;
pm_clk_init(dev);
psd = NULL;
}
spin_unlock_irq(&dev->power.lock);
/* kfree() verifies that its argument is nonzero. */
kfree(psd);
return 0;
}
EXPORT_SYMBOL_GPL(dev_pm_get_subsys_data);
/**
* dev_pm_put_subsys_data - Drop reference to power.subsys_data.
* @dev: Device to handle.
*
* If the reference counter of power.subsys_data is zero after dropping the
* reference, power.subsys_data is removed.
*/
void dev_pm_put_subsys_data(struct device *dev)
{
struct pm_subsys_data *psd;
spin_lock_irq(&dev->power.lock);
psd = dev_to_psd(dev);
if (!psd)
goto out;
if (--psd->refcount == 0)
dev->power.subsys_data = NULL;
else
psd = NULL;
out:
spin_unlock_irq(&dev->power.lock);
kfree(psd);
}
EXPORT_SYMBOL_GPL(dev_pm_put_subsys_data);
/**
* dev_pm_domain_attach - Attach a device to its PM domain.
* @dev: Device to attach.
* @power_on: Used to indicate whether we should power on the device.
*
* The @dev may only be attached to a single PM domain. By iterating through
* the available alternatives we try to find a valid PM domain for the device.
* As attachment succeeds, the ->detach() callback in the struct dev_pm_domain
* should be assigned by the corresponding attach function.
*
* This function should typically be invoked from subsystem level code during
* the probe phase. Especially for those that holds devices which requires
* power management through PM domains.
*
* Callers must ensure proper synchronization of this function with power
* management callbacks.
*
* Returns 0 on successfully attached PM domain, or when it is found that the
* device doesn't need a PM domain, else a negative error code.
*/
int dev_pm_domain_attach(struct device *dev, bool power_on)
{
int ret;
if (dev->pm_domain)
return 0;
ret = acpi_dev_pm_attach(dev, power_on);
if (!ret)
ret = genpd_dev_pm_attach(dev);
return ret < 0 ? ret : 0;
}
EXPORT_SYMBOL_GPL(dev_pm_domain_attach);
/**
* dev_pm_domain_attach_by_id - Associate a device with one of its PM domains.
* @dev: The device used to lookup the PM domain.
* @index: The index of the PM domain.
*
* As @dev may only be attached to a single PM domain, the backend PM domain
* provider creates a virtual device to attach instead. If attachment succeeds,
* the ->detach() callback in the struct dev_pm_domain are assigned by the
* corresponding backend attach function, as to deal with detaching of the
* created virtual device.
*
* This function should typically be invoked by a driver during the probe phase,
* in case its device requires power management through multiple PM domains. The
* driver may benefit from using the received device, to configure device-links
* towards its original device. Depending on the use-case and if needed, the
* links may be dynamically changed by the driver, which allows it to control
* the power to the PM domains independently from each other.
*
* Callers must ensure proper synchronization of this function with power
* management callbacks.
*
* Returns the virtual created device when successfully attached to its PM
* domain, NULL in case @dev don't need a PM domain, else an ERR_PTR().
* Note that, to detach the returned virtual device, the driver shall call
* dev_pm_domain_detach() on it, typically during the remove phase.
*/
struct device *dev_pm_domain_attach_by_id(struct device *dev,
unsigned int index)
{
if (dev->pm_domain)
return ERR_PTR(-EEXIST);
return genpd_dev_pm_attach_by_id(dev, index);
}
EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_id);
/**
* dev_pm_domain_attach_by_name - Associate a device with one of its PM domains.
* @dev: The device used to lookup the PM domain.
* @name: The name of the PM domain.
*
* For a detailed function description, see dev_pm_domain_attach_by_id().
*/
struct device *dev_pm_domain_attach_by_name(struct device *dev,
const char *name)
{
if (dev->pm_domain)
return ERR_PTR(-EEXIST);
return genpd_dev_pm_attach_by_name(dev, name);
}
EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_name);
/**
* dev_pm_domain_attach_list - Associate a device with its PM domains.
* @dev: The device used to lookup the PM domains for.
* @data: The data used for attaching to the PM domains.
* @list: An out-parameter with an allocated list of attached PM domains.
*
* This function helps to attach a device to its multiple PM domains. The
* caller, which is typically a driver's probe function, may provide a list of
* names for the PM domains that we should try to attach the device to, but it
* may also provide an empty list, in case the attach should be done for all of
* the available PM domains.
*
* Callers must ensure proper synchronization of this function with power
* management callbacks.
*
* Returns the number of attached PM domains or a negative error code in case of
* a failure. Note that, to detach the list of PM domains, the driver shall call
* dev_pm_domain_detach_list(), typically during the remove phase.
*/
int dev_pm_domain_attach_list(struct device *dev,
const struct dev_pm_domain_attach_data *data,
struct dev_pm_domain_list **list)
{
struct device_node *np = dev->of_node;
struct dev_pm_domain_list *pds;
struct device *pd_dev = NULL;
int ret, i, num_pds = 0;
bool by_id = true;
u32 pd_flags = data ? data->pd_flags : 0;
u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 :
DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME;
if (dev->pm_domain)
return -EEXIST;
/* For now this is limited to OF based platforms. */
if (!np)
return 0;
if (data && data->pd_names) {
num_pds = data->num_pd_names;
by_id = false;
} else {
num_pds = of_count_phandle_with_args(np, "power-domains",
"#power-domain-cells");
}
if (num_pds <= 0)
return 0;
pds = devm_kzalloc(dev, sizeof(*pds), GFP_KERNEL);
if (!pds)
return -ENOMEM;
pds->pd_devs = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_devs),
GFP_KERNEL);
if (!pds->pd_devs)
return -ENOMEM;
pds->pd_links = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_links),
GFP_KERNEL);
if (!pds->pd_links)
return -ENOMEM;
if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON)
link_flags |= DL_FLAG_RPM_ACTIVE;
for (i = 0; i < num_pds; i++) {
if (by_id)
pd_dev = dev_pm_domain_attach_by_id(dev, i);
else
pd_dev = dev_pm_domain_attach_by_name(dev,
data->pd_names[i]);
if (IS_ERR_OR_NULL(pd_dev)) {
ret = pd_dev ? PTR_ERR(pd_dev) : -ENODEV;
goto err_attach;
}
if (link_flags) {
struct device_link *link;
link = device_link_add(dev, pd_dev, link_flags);
if (!link) {
ret = -ENODEV;
goto err_link;
}
pds->pd_links[i] = link;
}
pds->pd_devs[i] = pd_dev;
}
pds->num_pds = num_pds;
*list = pds;
return num_pds;
err_link:
dev_pm_domain_detach(pd_dev, true);
err_attach:
while (--i >= 0) {
if (pds->pd_links[i])
device_link_del(pds->pd_links[i]);
dev_pm_domain_detach(pds->pd_devs[i], true);
}
return ret;
}
EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list);
/**
* dev_pm_domain_detach - Detach a device from its PM domain.
* @dev: Device to detach.
* @power_off: Used to indicate whether we should power off the device.
*
* This functions will reverse the actions from dev_pm_domain_attach(),
* dev_pm_domain_attach_by_id() and dev_pm_domain_attach_by_name(), thus it
* detaches @dev from its PM domain. Typically it should be invoked during the
* remove phase, either from subsystem level code or from drivers.
*
* Callers must ensure proper synchronization of this function with power
* management callbacks.
*/
void dev_pm_domain_detach(struct device *dev, bool power_off)
{
if (dev->pm_domain && dev->pm_domain->detach) dev->pm_domain->detach(dev, power_off);
}
EXPORT_SYMBOL_GPL(dev_pm_domain_detach);
/**
* dev_pm_domain_detach_list - Detach a list of PM domains.
* @list: The list of PM domains to detach.
*
* This function reverse the actions from dev_pm_domain_attach_list().
* Typically it should be invoked during the remove phase from drivers.
*
* Callers must ensure proper synchronization of this function with power
* management callbacks.
*/
void dev_pm_domain_detach_list(struct dev_pm_domain_list *list)
{
int i;
if (!list)
return;
for (i = 0; i < list->num_pds; i++) {
if (list->pd_links[i])
device_link_del(list->pd_links[i]);
dev_pm_domain_detach(list->pd_devs[i], true);
}
}
EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list);
/**
* dev_pm_domain_start - Start the device through its PM domain.
* @dev: Device to start.
*
* This function should typically be called during probe by a subsystem/driver,
* when it needs to start its device from the PM domain's perspective. Note
* that, it's assumed that the PM domain is already powered on when this
* function is called.
*
* Returns 0 on success and negative error values on failures.
*/
int dev_pm_domain_start(struct device *dev)
{
if (dev->pm_domain && dev->pm_domain->start)
return dev->pm_domain->start(dev);
return 0;
}
EXPORT_SYMBOL_GPL(dev_pm_domain_start);
/**
* dev_pm_domain_set - Set PM domain of a device.
* @dev: Device whose PM domain is to be set.
* @pd: PM domain to be set, or NULL.
*
* Sets the PM domain the device belongs to. The PM domain of a device needs
* to be set before its probe finishes (it's bound to a driver).
*
* This function must be called with the device lock held.
*/
void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd)
{
if (dev->pm_domain == pd)
return;
WARN(pd && device_is_bound(dev),
"PM domains can only be changed for unbound devices\n");
dev->pm_domain = pd;
device_pm_check_callbacks(dev);
}
EXPORT_SYMBOL_GPL(dev_pm_domain_set);
/**
* dev_pm_domain_set_performance_state - Request a new performance state.
* @dev: The device to make the request for.
* @state: Target performance state for the device.
*
* This function should be called when a new performance state needs to be
* requested for a device that is attached to a PM domain. Note that, the
* support for performance scaling for PM domains is optional.
*
* Returns 0 on success and when performance scaling isn't supported, negative
* error code on failure.
*/
int dev_pm_domain_set_performance_state(struct device *dev, unsigned int state)
{
if (dev->pm_domain && dev->pm_domain->set_performance_state)
return dev->pm_domain->set_performance_state(dev, state);
return 0;
}
EXPORT_SYMBOL_GPL(dev_pm_domain_set_performance_state);
/* SPDX-License-Identifier: GPL-2.0 */
/*
*
* V 4 L 2 D R I V E R H E L P E R A P I
*
* Moved from videodev2.h
*
* Some commonly needed functions for drivers (v4l2-common.o module)
*/
#ifndef _V4L2_DEV_H
#define _V4L2_DEV_H
#include <linux/poll.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/cdev.h>
#include <linux/mutex.h>
#include <linux/videodev2.h>
#include <media/media-entity.h>
#define VIDEO_MAJOR 81
/**
* enum vfl_devnode_type - type of V4L2 device node
*
* @VFL_TYPE_VIDEO: for video input/output devices
* @VFL_TYPE_VBI: for vertical blank data (i.e. closed captions, teletext)
* @VFL_TYPE_RADIO: for radio tuners
* @VFL_TYPE_SUBDEV: for V4L2 subdevices
* @VFL_TYPE_SDR: for Software Defined Radio tuners
* @VFL_TYPE_TOUCH: for touch sensors
* @VFL_TYPE_MAX: number of VFL types, must always be last in the enum
*/
enum vfl_devnode_type {
VFL_TYPE_VIDEO,
VFL_TYPE_VBI,
VFL_TYPE_RADIO,
VFL_TYPE_SUBDEV,
VFL_TYPE_SDR,
VFL_TYPE_TOUCH,
VFL_TYPE_MAX /* Shall be the last one */
};
/**
* enum vfl_devnode_direction - Identifies if a &struct video_device
* corresponds to a receiver, a transmitter or a mem-to-mem device.
*
* @VFL_DIR_RX: device is a receiver.
* @VFL_DIR_TX: device is a transmitter.
* @VFL_DIR_M2M: device is a memory to memory device.
*
* Note: Ignored if &enum vfl_devnode_type is %VFL_TYPE_SUBDEV.
*/
enum vfl_devnode_direction {
VFL_DIR_RX,
VFL_DIR_TX,
VFL_DIR_M2M,
};
struct v4l2_ioctl_callbacks;
struct video_device;
struct v4l2_device;
struct v4l2_ctrl_handler;
/**
* enum v4l2_video_device_flags - Flags used by &struct video_device
*
* @V4L2_FL_REGISTERED:
* indicates that a &struct video_device is registered.
* Drivers can clear this flag if they want to block all future
* device access. It is cleared by video_unregister_device.
* @V4L2_FL_USES_V4L2_FH:
* indicates that file->private_data points to &struct v4l2_fh.
* This flag is set by the core when v4l2_fh_init() is called.
* All new drivers should use it.
* @V4L2_FL_QUIRK_INVERTED_CROP:
* some old M2M drivers use g/s_crop/cropcap incorrectly: crop and
* compose are swapped. If this flag is set, then the selection
* targets are swapped in the g/s_crop/cropcap functions in v4l2-ioctl.c.
* This allows those drivers to correctly implement the selection API,
* but the old crop API will still work as expected in order to preserve
* backwards compatibility.
* Never set this flag for new drivers.
* @V4L2_FL_SUBDEV_RO_DEVNODE:
* indicates that the video device node is registered in read-only mode.
* The flag only applies to device nodes registered for sub-devices, it is
* set by the core when the sub-devices device nodes are registered with
* v4l2_device_register_ro_subdev_nodes() and used by the sub-device ioctl
* handler to restrict access to some ioctl calls.
*/
enum v4l2_video_device_flags {
V4L2_FL_REGISTERED = 0,
V4L2_FL_USES_V4L2_FH = 1,
V4L2_FL_QUIRK_INVERTED_CROP = 2,
V4L2_FL_SUBDEV_RO_DEVNODE = 3,
};
/* Priority helper functions */
/**
* struct v4l2_prio_state - stores the priority states
*
* @prios: array with elements to store the array priorities
*
*
* .. note::
* The size of @prios array matches the number of priority types defined
* by enum &v4l2_priority.
*/
struct v4l2_prio_state {
atomic_t prios[4];
};
/**
* v4l2_prio_init - initializes a struct v4l2_prio_state
*
* @global: pointer to &struct v4l2_prio_state
*/
void v4l2_prio_init(struct v4l2_prio_state *global);
/**
* v4l2_prio_change - changes the v4l2 file handler priority
*
* @global: pointer to the &struct v4l2_prio_state of the device node.
* @local: pointer to the desired priority, as defined by enum &v4l2_priority
* @new: Priority type requested, as defined by enum &v4l2_priority.
*
* .. note::
* This function should be used only by the V4L2 core.
*/
int v4l2_prio_change(struct v4l2_prio_state *global, enum v4l2_priority *local,
enum v4l2_priority new);
/**
* v4l2_prio_open - Implements the priority logic for a file handler open
*
* @global: pointer to the &struct v4l2_prio_state of the device node.
* @local: pointer to the desired priority, as defined by enum &v4l2_priority
*
* .. note::
* This function should be used only by the V4L2 core.
*/
void v4l2_prio_open(struct v4l2_prio_state *global, enum v4l2_priority *local);
/**
* v4l2_prio_close - Implements the priority logic for a file handler close
*
* @global: pointer to the &struct v4l2_prio_state of the device node.
* @local: priority to be released, as defined by enum &v4l2_priority
*
* .. note::
* This function should be used only by the V4L2 core.
*/
void v4l2_prio_close(struct v4l2_prio_state *global, enum v4l2_priority local);
/**
* v4l2_prio_max - Return the maximum priority, as stored at the @global array.
*
* @global: pointer to the &struct v4l2_prio_state of the device node.
*
* .. note::
* This function should be used only by the V4L2 core.
*/
enum v4l2_priority v4l2_prio_max(struct v4l2_prio_state *global);
/**
* v4l2_prio_check - Implements the priority logic for a file handler close
*
* @global: pointer to the &struct v4l2_prio_state of the device node.
* @local: desired priority, as defined by enum &v4l2_priority local
*
* .. note::
* This function should be used only by the V4L2 core.
*/
int v4l2_prio_check(struct v4l2_prio_state *global, enum v4l2_priority local);
/**
* struct v4l2_file_operations - fs operations used by a V4L2 device
*
* @owner: pointer to struct module
* @read: operations needed to implement the read() syscall
* @write: operations needed to implement the write() syscall
* @poll: operations needed to implement the poll() syscall
* @unlocked_ioctl: operations needed to implement the ioctl() syscall
* @compat_ioctl32: operations needed to implement the ioctl() syscall for
* the special case where the Kernel uses 64 bits instructions, but
* the userspace uses 32 bits.
* @get_unmapped_area: called by the mmap() syscall, used when %!CONFIG_MMU
* @mmap: operations needed to implement the mmap() syscall
* @open: operations needed to implement the open() syscall
* @release: operations needed to implement the release() syscall
*
* .. note::
*
* Those operations are used to implemente the fs struct file_operations
* at the V4L2 drivers. The V4L2 core overrides the fs ops with some
* extra logic needed by the subsystem.
*/
struct v4l2_file_operations {
struct module *owner;
ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
__poll_t (*poll) (struct file *, struct poll_table_struct *);
long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
#ifdef CONFIG_COMPAT
long (*compat_ioctl32) (struct file *, unsigned int, unsigned long);
#endif
unsigned long (*get_unmapped_area) (struct file *, unsigned long,
unsigned long, unsigned long, unsigned long);
int (*mmap) (struct file *, struct vm_area_struct *);
int (*open) (struct file *);
int (*release) (struct file *);
};
/*
* Newer version of video_device, handled by videodev2.c
* This version moves redundant code from video device code to
* the common handler
*/
/**
* struct video_device - Structure used to create and manage the V4L2 device
* nodes.
*
* @entity: &struct media_entity
* @intf_devnode: pointer to &struct media_intf_devnode
* @pipe: &struct media_pipeline
* @fops: pointer to &struct v4l2_file_operations for the video device
* @device_caps: device capabilities as used in v4l2_capabilities
* @dev: &struct device for the video device
* @cdev: character device
* @v4l2_dev: pointer to &struct v4l2_device parent
* @dev_parent: pointer to &struct device parent
* @ctrl_handler: Control handler associated with this device node.
* May be NULL.
* @queue: &struct vb2_queue associated with this device node. May be NULL.
* @prio: pointer to &struct v4l2_prio_state with device's Priority state.
* If NULL, then v4l2_dev->prio will be used.
* @name: video device name
* @vfl_type: V4L device type, as defined by &enum vfl_devnode_type
* @vfl_dir: V4L receiver, transmitter or m2m
* @minor: device node 'minor'. It is set to -1 if the registration failed
* @num: number of the video device node
* @flags: video device flags. Use bitops to set/clear/test flags.
* Contains a set of &enum v4l2_video_device_flags.
* @index: attribute to differentiate multiple indices on one physical device
* @fh_lock: Lock for all v4l2_fhs
* @fh_list: List of &struct v4l2_fh
* @dev_debug: Internal device debug flags, not for use by drivers
* @tvnorms: Supported tv norms
*
* @release: video device release() callback
* @ioctl_ops: pointer to &struct v4l2_ioctl_ops with ioctl callbacks
*
* @valid_ioctls: bitmap with the valid ioctls for this device
* @lock: pointer to &struct mutex serialization lock
*
* .. note::
* Only set @dev_parent if that can't be deduced from @v4l2_dev.
*/
struct video_device {
#if defined(CONFIG_MEDIA_CONTROLLER)
struct media_entity entity;
struct media_intf_devnode *intf_devnode;
struct media_pipeline pipe;
#endif
const struct v4l2_file_operations *fops;
u32 device_caps;
/* sysfs */
struct device dev;
struct cdev *cdev;
struct v4l2_device *v4l2_dev;
struct device *dev_parent;
struct v4l2_ctrl_handler *ctrl_handler;
struct vb2_queue *queue;
struct v4l2_prio_state *prio;
/* device info */
char name[64];
enum vfl_devnode_type vfl_type;
enum vfl_devnode_direction vfl_dir;
int minor;
u16 num;
unsigned long flags;
int index;
/* V4L2 file handles */
spinlock_t fh_lock;
struct list_head fh_list;
int dev_debug;
v4l2_std_id tvnorms;
/* callbacks */
void (*release)(struct video_device *vdev);
const struct v4l2_ioctl_ops *ioctl_ops;
DECLARE_BITMAP(valid_ioctls, BASE_VIDIOC_PRIVATE);
struct mutex *lock;
};
/**
* media_entity_to_video_device - Returns a &struct video_device from
* the &struct media_entity embedded on it.
*
* @__entity: pointer to &struct media_entity
*/
#define media_entity_to_video_device(__entity) \
container_of(__entity, struct video_device, entity)
/**
* to_video_device - Returns a &struct video_device from the
* &struct device embedded on it.
*
* @cd: pointer to &struct device
*/
#define to_video_device(cd) container_of(cd, struct video_device, dev)
/**
* __video_register_device - register video4linux devices
*
* @vdev: struct video_device to register
* @type: type of device to register, as defined by &enum vfl_devnode_type
* @nr: which device node number is desired:
* (0 == /dev/video0, 1 == /dev/video1, ..., -1 == first free)
* @warn_if_nr_in_use: warn if the desired device node number
* was already in use and another number was chosen instead.
* @owner: module that owns the video device node
*
* The registration code assigns minor numbers and device node numbers
* based on the requested type and registers the new device node with
* the kernel.
*
* This function assumes that struct video_device was zeroed when it
* was allocated and does not contain any stale date.
*
* An error is returned if no free minor or device node number could be
* found, or if the registration of the device node failed.
*
* Returns 0 on success.
*
* .. note::
*
* This function is meant to be used only inside the V4L2 core.
* Drivers should use video_register_device() or
* video_register_device_no_warn().
*/
int __must_check __video_register_device(struct video_device *vdev,
enum vfl_devnode_type type,
int nr, int warn_if_nr_in_use,
struct module *owner);
/**
* video_register_device - register video4linux devices
*
* @vdev: struct video_device to register
* @type: type of device to register, as defined by &enum vfl_devnode_type
* @nr: which device node number is desired:
* (0 == /dev/video0, 1 == /dev/video1, ..., -1 == first free)
*
* Internally, it calls __video_register_device(). Please see its
* documentation for more details.
*
* .. note::
* if video_register_device fails, the release() callback of
* &struct video_device structure is *not* called, so the caller
* is responsible for freeing any data. Usually that means that
* you video_device_release() should be called on failure.
*/
static inline int __must_check video_register_device(struct video_device *vdev,
enum vfl_devnode_type type,
int nr)
{
return __video_register_device(vdev, type, nr, 1, vdev->fops->owner);
}
/**
* video_register_device_no_warn - register video4linux devices
*
* @vdev: struct video_device to register
* @type: type of device to register, as defined by &enum vfl_devnode_type
* @nr: which device node number is desired:
* (0 == /dev/video0, 1 == /dev/video1, ..., -1 == first free)
*
* This function is identical to video_register_device() except that no
* warning is issued if the desired device node number was already in use.
*
* Internally, it calls __video_register_device(). Please see its
* documentation for more details.
*
* .. note::
* if video_register_device fails, the release() callback of
* &struct video_device structure is *not* called, so the caller
* is responsible for freeing any data. Usually that means that
* you video_device_release() should be called on failure.
*/
static inline int __must_check
video_register_device_no_warn(struct video_device *vdev,
enum vfl_devnode_type type, int nr)
{
return __video_register_device(vdev, type, nr, 0, vdev->fops->owner);
}
/**
* video_unregister_device - Unregister video devices.
*
* @vdev: &struct video_device to register
*
* Does nothing if vdev == NULL or if video_is_registered() returns false.
*/
void video_unregister_device(struct video_device *vdev);
/**
* video_device_alloc - helper function to alloc &struct video_device
*
* Returns NULL if %-ENOMEM or a &struct video_device on success.
*/
struct video_device * __must_check video_device_alloc(void);
/**
* video_device_release - helper function to release &struct video_device
*
* @vdev: pointer to &struct video_device
*
* Can also be used for video_device->release\(\).
*/
void video_device_release(struct video_device *vdev);
/**
* video_device_release_empty - helper function to implement the
* video_device->release\(\) callback.
*
* @vdev: pointer to &struct video_device
*
* This release function does nothing.
*
* It should be used when the video_device is a static global struct.
*
* .. note::
* Having a static video_device is a dubious construction at best.
*/
void video_device_release_empty(struct video_device *vdev);
/**
* v4l2_disable_ioctl- mark that a given command isn't implemented.
* shouldn't use core locking
*
* @vdev: pointer to &struct video_device
* @cmd: ioctl command
*
* This function allows drivers to provide just one v4l2_ioctl_ops struct, but
* disable ioctls based on the specific card that is actually found.
*
* .. note::
*
* This must be called before video_register_device.
* See also the comments for determine_valid_ioctls().
*/
static inline void v4l2_disable_ioctl(struct video_device *vdev,
unsigned int cmd)
{
if (_IOC_NR(cmd) < BASE_VIDIOC_PRIVATE)
set_bit(_IOC_NR(cmd), vdev->valid_ioctls);
}
/**
* video_get_drvdata - gets private data from &struct video_device.
*
* @vdev: pointer to &struct video_device
*
* returns a pointer to the private data
*/
static inline void *video_get_drvdata(struct video_device *vdev)
{
return dev_get_drvdata(&vdev->dev);
}
/**
* video_set_drvdata - sets private data from &struct video_device.
*
* @vdev: pointer to &struct video_device
* @data: private data pointer
*/
static inline void video_set_drvdata(struct video_device *vdev, void *data)
{
dev_set_drvdata(&vdev->dev, data);
}
/**
* video_devdata - gets &struct video_device from struct file.
*
* @file: pointer to struct file
*/
struct video_device *video_devdata(struct file *file);
/**
* video_drvdata - gets private data from &struct video_device using the
* struct file.
*
* @file: pointer to struct file
*
* This is function combines both video_get_drvdata() and video_devdata()
* as this is used very often.
*/
static inline void *video_drvdata(struct file *file)
{
return video_get_drvdata(video_devdata(file));
}
/**
* video_device_node_name - returns the video device name
*
* @vdev: pointer to &struct video_device
*
* Returns the device name string
*/
static inline const char *video_device_node_name(struct video_device *vdev)
{
return dev_name(&vdev->dev);
}
/**
* video_is_registered - returns true if the &struct video_device is registered.
*
*
* @vdev: pointer to &struct video_device
*/
static inline int video_is_registered(struct video_device *vdev)
{
return test_bit(V4L2_FL_REGISTERED, &vdev->flags);
}
#if defined(CONFIG_MEDIA_CONTROLLER)
/**
* video_device_pipeline_start - Mark a pipeline as streaming
* @vdev: Starting video device
* @pipe: Media pipeline to be assigned to all entities in the pipeline.
*
* Mark all entities connected to a given video device through enabled links,
* either directly or indirectly, as streaming. The given pipeline object is
* assigned to every pad in the pipeline and stored in the media_pad pipe
* field.
*
* Calls to this function can be nested, in which case the same number of
* video_device_pipeline_stop() calls will be required to stop streaming. The
* pipeline pointer must be identical for all nested calls to
* video_device_pipeline_start().
*
* The video device must contain a single pad.
*
* This is a convenience wrapper around media_pipeline_start().
*/
__must_check int video_device_pipeline_start(struct video_device *vdev,
struct media_pipeline *pipe);
/**
* __video_device_pipeline_start - Mark a pipeline as streaming
* @vdev: Starting video device
* @pipe: Media pipeline to be assigned to all entities in the pipeline.
*
* ..note:: This is the non-locking version of video_device_pipeline_start()
*
* The video device must contain a single pad.
*
* This is a convenience wrapper around __media_pipeline_start().
*/
__must_check int __video_device_pipeline_start(struct video_device *vdev,
struct media_pipeline *pipe);
/**
* video_device_pipeline_stop - Mark a pipeline as not streaming
* @vdev: Starting video device
*
* Mark all entities connected to a given video device through enabled links,
* either directly or indirectly, as not streaming. The media_pad pipe field
* is reset to %NULL.
*
* If multiple calls to media_pipeline_start() have been made, the same
* number of calls to this function are required to mark the pipeline as not
* streaming.
*
* The video device must contain a single pad.
*
* This is a convenience wrapper around media_pipeline_stop().
*/
void video_device_pipeline_stop(struct video_device *vdev);
/**
* __video_device_pipeline_stop - Mark a pipeline as not streaming
* @vdev: Starting video device
*
* .. note:: This is the non-locking version of media_pipeline_stop()
*
* The video device must contain a single pad.
*
* This is a convenience wrapper around __media_pipeline_stop().
*/
void __video_device_pipeline_stop(struct video_device *vdev);
/**
* video_device_pipeline_alloc_start - Mark a pipeline as streaming
* @vdev: Starting video device
*
* video_device_pipeline_alloc_start() is similar to video_device_pipeline_start()
* but instead of working on a given pipeline the function will use an
* existing pipeline if the video device is already part of a pipeline, or
* allocate a new pipeline.
*
* Calls to video_device_pipeline_alloc_start() must be matched with
* video_device_pipeline_stop().
*/
__must_check int video_device_pipeline_alloc_start(struct video_device *vdev);
/**
* video_device_pipeline - Get the media pipeline a video device is part of
* @vdev: The video device
*
* This function returns the media pipeline that a video device has been
* associated with when constructing the pipeline with
* video_device_pipeline_start(). The pointer remains valid until
* video_device_pipeline_stop() is called.
*
* Return: The media_pipeline the video device is part of, or NULL if the video
* device is not part of any pipeline.
*
* The video device must contain a single pad.
*
* This is a convenience wrapper around media_entity_pipeline().
*/
struct media_pipeline *video_device_pipeline(struct video_device *vdev);
#endif /* CONFIG_MEDIA_CONTROLLER */
#endif /* _V4L2_DEV_H */
// SPDX-License-Identifier: GPL-2.0
/*
* linux/mm/mlock.c
*
* (C) Copyright 1995 Linus Torvalds
* (C) Copyright 2002 Christoph Hellwig
*/
#include <linux/capability.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/sched/user.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/pagewalk.h>
#include <linux/mempolicy.h>
#include <linux/syscalls.h>
#include <linux/sched.h>
#include <linux/export.h>
#include <linux/rmap.h>
#include <linux/mmzone.h>
#include <linux/hugetlb.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/secretmem.h>
#include "internal.h"
struct mlock_fbatch {
local_lock_t lock;
struct folio_batch fbatch;
};
static DEFINE_PER_CPU(struct mlock_fbatch, mlock_fbatch) = {
.lock = INIT_LOCAL_LOCK(lock),
};
bool can_do_mlock(void)
{
if (rlimit(RLIMIT_MEMLOCK) != 0)
return true;
if (capable(CAP_IPC_LOCK))
return true;
return false;
}
EXPORT_SYMBOL(can_do_mlock);
/*
* Mlocked folios are marked with the PG_mlocked flag for efficient testing
* in vmscan and, possibly, the fault path; and to support semi-accurate
* statistics.
*
* An mlocked folio [folio_test_mlocked(folio)] is unevictable. As such, it
* will be ostensibly placed on the LRU "unevictable" list (actually no such
* list exists), rather than the [in]active lists. PG_unevictable is set to
* indicate the unevictable state.
*/
static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec)
{
/* There is nothing more we can do while it's off LRU */
if (!folio_test_clear_lru(folio))
return lruvec;
lruvec = folio_lruvec_relock_irq(folio, lruvec);
if (unlikely(folio_evictable(folio))) {
/*
* This is a little surprising, but quite possible: PG_mlocked
* must have got cleared already by another CPU. Could this
* folio be unevictable? I'm not sure, but move it now if so.
*/
if (folio_test_unevictable(folio)) {
lruvec_del_folio(lruvec, folio);
folio_clear_unevictable(folio);
lruvec_add_folio(lruvec, folio);
__count_vm_events(UNEVICTABLE_PGRESCUED,
folio_nr_pages(folio));
}
goto out;
}
if (folio_test_unevictable(folio)) {
if (folio_test_mlocked(folio))
folio->mlock_count++;
goto out;
}
lruvec_del_folio(lruvec, folio);
folio_clear_active(folio);
folio_set_unevictable(folio);
folio->mlock_count = !!folio_test_mlocked(folio);
lruvec_add_folio(lruvec, folio);
__count_vm_events(UNEVICTABLE_PGCULLED, folio_nr_pages(folio));
out:
folio_set_lru(folio);
return lruvec;
}
static struct lruvec *__mlock_new_folio(struct folio *folio, struct lruvec *lruvec)
{
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
lruvec = folio_lruvec_relock_irq(folio, lruvec);
/* As above, this is a little surprising, but possible */
if (unlikely(folio_evictable(folio)))
goto out;
folio_set_unevictable(folio);
folio->mlock_count = !!folio_test_mlocked(folio);
__count_vm_events(UNEVICTABLE_PGCULLED, folio_nr_pages(folio));
out:
lruvec_add_folio(lruvec, folio);
folio_set_lru(folio);
return lruvec;
}
static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec)
{
int nr_pages = folio_nr_pages(folio);
bool isolated = false;
if (!folio_test_clear_lru(folio))
goto munlock;
isolated = true;
lruvec = folio_lruvec_relock_irq(folio, lruvec);
if (folio_test_unevictable(folio)) {
/* Then mlock_count is maintained, but might undercount */
if (folio->mlock_count)
folio->mlock_count--;
if (folio->mlock_count)
goto out;
}
/* else assume that was the last mlock: reclaim will fix it if not */
munlock:
if (folio_test_clear_mlocked(folio)) {
__zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
if (isolated || !folio_test_unevictable(folio))
__count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
else
__count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);
}
/* folio_evictable() has to be checked *after* clearing Mlocked */
if (isolated && folio_test_unevictable(folio) && folio_evictable(folio)) {
lruvec_del_folio(lruvec, folio);
folio_clear_unevictable(folio);
lruvec_add_folio(lruvec, folio);
__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
}
out:
if (isolated)
folio_set_lru(folio);
return lruvec;
}
/*
* Flags held in the low bits of a struct folio pointer on the mlock_fbatch.
*/
#define LRU_FOLIO 0x1
#define NEW_FOLIO 0x2
static inline struct folio *mlock_lru(struct folio *folio)
{
return (struct folio *)((unsigned long)folio + LRU_FOLIO);
}
static inline struct folio *mlock_new(struct folio *folio)
{
return (struct folio *)((unsigned long)folio + NEW_FOLIO);
}
/*
* mlock_folio_batch() is derived from folio_batch_move_lru(): perhaps that can
* make use of such folio pointer flags in future, but for now just keep it for
* mlock. We could use three separate folio batches instead, but one feels
* better (munlocking a full folio batch does not need to drain mlocking folio
* batches first).
*/
static void mlock_folio_batch(struct folio_batch *fbatch)
{
struct lruvec *lruvec = NULL;
unsigned long mlock;
struct folio *folio;
int i;
for (i = 0; i < folio_batch_count(fbatch); i++) {
folio = fbatch->folios[i];
mlock = (unsigned long)folio & (LRU_FOLIO | NEW_FOLIO);
folio = (struct folio *)((unsigned long)folio - mlock);
fbatch->folios[i] = folio;
if (mlock & LRU_FOLIO)
lruvec = __mlock_folio(folio, lruvec);
else if (mlock & NEW_FOLIO)
lruvec = __mlock_new_folio(folio, lruvec);
else
lruvec = __munlock_folio(folio, lruvec);
}
if (lruvec)
unlock_page_lruvec_irq(lruvec);
folios_put(fbatch);
}
void mlock_drain_local(void)
{
struct folio_batch *fbatch;
local_lock(&mlock_fbatch.lock);
fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
if (folio_batch_count(fbatch))
mlock_folio_batch(fbatch); local_unlock(&mlock_fbatch.lock);
}
void mlock_drain_remote(int cpu)
{
struct folio_batch *fbatch;
WARN_ON_ONCE(cpu_online(cpu));
fbatch = &per_cpu(mlock_fbatch.fbatch, cpu);
if (folio_batch_count(fbatch))
mlock_folio_batch(fbatch);
}
bool need_mlock_drain(int cpu)
{
return folio_batch_count(&per_cpu(mlock_fbatch.fbatch, cpu));
}
/**
* mlock_folio - mlock a folio already on (or temporarily off) LRU
* @folio: folio to be mlocked.
*/
void mlock_folio(struct folio *folio)
{
struct folio_batch *fbatch;
local_lock(&mlock_fbatch.lock);
fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
if (!folio_test_set_mlocked(folio)) {
int nr_pages = folio_nr_pages(folio);
zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
__count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
}
folio_get(folio);
if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
folio_test_large(folio) || lru_cache_disabled())
mlock_folio_batch(fbatch);
local_unlock(&mlock_fbatch.lock);
}
/**
* mlock_new_folio - mlock a newly allocated folio not yet on LRU
* @folio: folio to be mlocked, either normal or a THP head.
*/
void mlock_new_folio(struct folio *folio)
{
struct folio_batch *fbatch;
int nr_pages = folio_nr_pages(folio);
local_lock(&mlock_fbatch.lock);
fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
folio_set_mlocked(folio);
zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
__count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
folio_get(folio);
if (!folio_batch_add(fbatch, mlock_new(folio)) ||
folio_test_large(folio) || lru_cache_disabled())
mlock_folio_batch(fbatch);
local_unlock(&mlock_fbatch.lock);
}
/**
* munlock_folio - munlock a folio
* @folio: folio to be munlocked, either normal or a THP head.
*/
void munlock_folio(struct folio *folio)
{
struct folio_batch *fbatch;
local_lock(&mlock_fbatch.lock);
fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
/*
* folio_test_clear_mlocked(folio) must be left to __munlock_folio(),
* which will check whether the folio is multiply mlocked.
*/
folio_get(folio);
if (!folio_batch_add(fbatch, folio) ||
folio_test_large(folio) || lru_cache_disabled())
mlock_folio_batch(fbatch);
local_unlock(&mlock_fbatch.lock);
}
static inline unsigned int folio_mlock_step(struct folio *folio,
pte_t *pte, unsigned long addr, unsigned long end)
{
unsigned int count, i, nr = folio_nr_pages(folio);
unsigned long pfn = folio_pfn(folio);
pte_t ptent = ptep_get(pte);
if (!folio_test_large(folio))
return 1;
count = pfn + nr - pte_pfn(ptent);
count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
for (i = 0; i < count; i++, pte++) {
pte_t entry = ptep_get(pte);
if (!pte_present(entry))
break;
if (pte_pfn(entry) - pfn >= nr)
break;
}
return i;
}
static inline bool allow_mlock_munlock(struct folio *folio,
struct vm_area_struct *vma, unsigned long start,
unsigned long end, unsigned int step)
{
/*
* For unlock, allow munlock large folio which is partially
* mapped to VMA. As it's possible that large folio is
* mlocked and VMA is split later.
*
* During memory pressure, such kind of large folio can
* be split. And the pages are not in VM_LOCKed VMA
* can be reclaimed.
*/
if (!(vma->vm_flags & VM_LOCKED))
return true;
/* folio_within_range() cannot take KSM, but any small folio is OK */
if (!folio_test_large(folio))
return true;
/* folio not in range [start, end), skip mlock */
if (!folio_within_range(folio, vma, start, end))
return false;
/* folio is not fully mapped, skip mlock */
if (step != folio_nr_pages(folio))
return false;
return true;
}
static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
unsigned long end, struct mm_walk *walk)
{
struct vm_area_struct *vma = walk->vma;
spinlock_t *ptl;
pte_t *start_pte, *pte;
pte_t ptent;
struct folio *folio;
unsigned int step = 1;
unsigned long start = addr;
ptl = pmd_trans_huge_lock(pmd, vma);
if (ptl) {
if (!pmd_present(*pmd))
goto out;
if (is_huge_zero_pmd(*pmd))
goto out;
folio = page_folio(pmd_page(*pmd));
if (vma->vm_flags & VM_LOCKED)
mlock_folio(folio);
else
munlock_folio(folio);
goto out;
}
start_pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
if (!start_pte) {
walk->action = ACTION_AGAIN;
return 0;
}
for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) {
ptent = ptep_get(pte);
if (!pte_present(ptent))
continue;
folio = vm_normal_folio(vma, addr, ptent);
if (!folio || folio_is_zone_device(folio))
continue;
step = folio_mlock_step(folio, pte, addr, end);
if (!allow_mlock_munlock(folio, vma, start, end, step))
goto next_entry;
if (vma->vm_flags & VM_LOCKED)
mlock_folio(folio);
else
munlock_folio(folio);
next_entry:
pte += step - 1;
addr += (step - 1) << PAGE_SHIFT;
}
pte_unmap(start_pte);
out:
spin_unlock(ptl);
cond_resched();
return 0;
}
/*
* mlock_vma_pages_range() - mlock any pages already in the range,
* or munlock all pages in the range.
* @vma - vma containing range to be mlock()ed or munlock()ed
* @start - start address in @vma of the range
* @end - end of range in @vma
* @newflags - the new set of flags for @vma.
*
* Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
* called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
*/
static void mlock_vma_pages_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end, vm_flags_t newflags)
{
static const struct mm_walk_ops mlock_walk_ops = {
.pmd_entry = mlock_pte_range,
.walk_lock = PGWALK_WRLOCK_VERIFY,
};
/*
* There is a slight chance that concurrent page migration,
* or page reclaim finding a page of this now-VM_LOCKED vma,
* will call mlock_vma_folio() and raise page's mlock_count:
* double counting, leaving the page unevictable indefinitely.
* Communicate this danger to mlock_vma_folio() with VM_IO,
* which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
* mmap_lock is held in write mode here, so this weird
* combination should not be visible to other mmap_lock users;
* but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
*/
if (newflags & VM_LOCKED)
newflags |= VM_IO;
vma_start_write(vma);
vm_flags_reset_once(vma, newflags);
lru_add_drain();
walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL);
lru_add_drain();
if (newflags & VM_IO) {
newflags &= ~VM_IO;
vm_flags_reset_once(vma, newflags);
}
}
/*
* mlock_fixup - handle mlock[all]/munlock[all] requests.
*
* Filters out "special" vmas -- VM_LOCKED never gets set for these, and
* munlock is a no-op. However, for some special vmas, we go ahead and
* populate the ptes.
*
* For vmas that pass the filters, merge/split as appropriate.
*/
static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
struct vm_area_struct **prev, unsigned long start,
unsigned long end, vm_flags_t newflags)
{
struct mm_struct *mm = vma->vm_mm;
int nr_pages;
int ret = 0;
vm_flags_t oldflags = vma->vm_flags;
if (newflags == oldflags || (oldflags & VM_SPECIAL) ||
is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm) ||
vma_is_dax(vma) || vma_is_secretmem(vma))
/* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
goto out;
vma = vma_modify_flags(vmi, *prev, vma, start, end, newflags);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto out;
}
/*
* Keep track of amount of locked VM.
*/
nr_pages = (end - start) >> PAGE_SHIFT;
if (!(newflags & VM_LOCKED))
nr_pages = -nr_pages;
else if (oldflags & VM_LOCKED)
nr_pages = 0;
mm->locked_vm += nr_pages;
/*
* vm_flags is protected by the mmap_lock held in write mode.
* It's okay if try_to_unmap_one unmaps a page just after we
* set VM_LOCKED, populate_vma_page_range will bring it back.
*/
if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) {
/* No work to do, and mlocking twice would be wrong */
vma_start_write(vma);
vm_flags_reset(vma, newflags);
} else {
mlock_vma_pages_range(vma, start, end, newflags);
}
out:
*prev = vma;
return ret;
}
static int apply_vma_lock_flags(unsigned long start, size_t len,
vm_flags_t flags)
{
unsigned long nstart, end, tmp;
struct vm_area_struct *vma, *prev;
VMA_ITERATOR(vmi, current->mm, start);
VM_BUG_ON(offset_in_page(start));
VM_BUG_ON(len != PAGE_ALIGN(len));
end = start + len;
if (end < start)
return -EINVAL;
if (end == start)
return 0;
vma = vma_iter_load(&vmi);
if (!vma)
return -ENOMEM;
prev = vma_prev(&vmi);
if (start > vma->vm_start)
prev = vma;
nstart = start;
tmp = vma->vm_start;
for_each_vma_range(vmi, vma, end) {
int error;
vm_flags_t newflags;
if (vma->vm_start != tmp)
return -ENOMEM;
newflags = vma->vm_flags & ~VM_LOCKED_MASK;
newflags |= flags;
/* Here we know that vma->vm_start <= nstart < vma->vm_end. */
tmp = vma->vm_end;
if (tmp > end)
tmp = end;
error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags);
if (error)
return error;
tmp = vma_iter_end(&vmi);
nstart = tmp;
}
if (tmp < end)
return -ENOMEM;
return 0;
}
/*
* Go through vma areas and sum size of mlocked
* vma pages, as return value.
* Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT)
* is also counted.
* Return value: previously mlocked page counts
*/
static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
unsigned long start, size_t len)
{
struct vm_area_struct *vma;
unsigned long count = 0;
unsigned long end;
VMA_ITERATOR(vmi, mm, start);
/* Don't overflow past ULONG_MAX */
if (unlikely(ULONG_MAX - len < start))
end = ULONG_MAX;
else
end = start + len;
for_each_vma_range(vmi, vma, end) {
if (vma->vm_flags & VM_LOCKED) {
if (start > vma->vm_start)
count -= (start - vma->vm_start);
if (end < vma->vm_end) {
count += end - vma->vm_start;
break;
}
count += vma->vm_end - vma->vm_start;
}
}
return count >> PAGE_SHIFT;
}
/*
* convert get_user_pages() return value to posix mlock() error
*/
static int __mlock_posix_error_return(long retval)
{
if (retval == -EFAULT)
retval = -ENOMEM;
else if (retval == -ENOMEM)
retval = -EAGAIN;
return retval;
}
static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
{
unsigned long locked;
unsigned long lock_limit;
int error = -ENOMEM;
start = untagged_addr(start);
if (!can_do_mlock())
return -EPERM;
len = PAGE_ALIGN(len + (offset_in_page(start)));
start &= PAGE_MASK;
lock_limit = rlimit(RLIMIT_MEMLOCK);
lock_limit >>= PAGE_SHIFT;
locked = len >> PAGE_SHIFT;
if (mmap_write_lock_killable(current->mm))
return -EINTR;
locked += current->mm->locked_vm;
if ((locked > lock_limit) && (!capable(CAP_IPC_LOCK))) {
/*
* It is possible that the regions requested intersect with
* previously mlocked areas, that part area in "mm->locked_vm"
* should not be counted to new mlock increment count. So check
* and adjust locked count if necessary.
*/
locked -= count_mm_mlocked_page_nr(current->mm,
start, len);
}
/* check against resource limits */
if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
error = apply_vma_lock_flags(start, len, flags);
mmap_write_unlock(current->mm);
if (error)
return error;
error = __mm_populate(start, len, 0);
if (error)
return __mlock_posix_error_return(error);
return 0;
}
SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len)
{
return do_mlock(start, len, VM_LOCKED);
}
SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
{
vm_flags_t vm_flags = VM_LOCKED;
if (flags & ~MLOCK_ONFAULT)
return -EINVAL;
if (flags & MLOCK_ONFAULT)
vm_flags |= VM_LOCKONFAULT;
return do_mlock(start, len, vm_flags);
}
SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
{
int ret;
start = untagged_addr(start);
len = PAGE_ALIGN(len + (offset_in_page(start)));
start &= PAGE_MASK;
if (mmap_write_lock_killable(current->mm))
return -EINTR;
ret = apply_vma_lock_flags(start, len, 0);
mmap_write_unlock(current->mm);
return ret;
}
/*
* Take the MCL_* flags passed into mlockall (or 0 if called from munlockall)
* and translate into the appropriate modifications to mm->def_flags and/or the
* flags for all current VMAs.
*
* There are a couple of subtleties with this. If mlockall() is called multiple
* times with different flags, the values do not necessarily stack. If mlockall
* is called once including the MCL_FUTURE flag and then a second time without
* it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags.
*/
static int apply_mlockall_flags(int flags)
{
VMA_ITERATOR(vmi, current->mm, 0);
struct vm_area_struct *vma, *prev = NULL;
vm_flags_t to_add = 0;
current->mm->def_flags &= ~VM_LOCKED_MASK;
if (flags & MCL_FUTURE) {
current->mm->def_flags |= VM_LOCKED;
if (flags & MCL_ONFAULT)
current->mm->def_flags |= VM_LOCKONFAULT;
if (!(flags & MCL_CURRENT))
goto out;
}
if (flags & MCL_CURRENT) {
to_add |= VM_LOCKED;
if (flags & MCL_ONFAULT)
to_add |= VM_LOCKONFAULT;
}
for_each_vma(vmi, vma) {
vm_flags_t newflags;
newflags = vma->vm_flags & ~VM_LOCKED_MASK;
newflags |= to_add;
/* Ignore errors */
mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end,
newflags);
cond_resched();
}
out:
return 0;
}
SYSCALL_DEFINE1(mlockall, int, flags)
{
unsigned long lock_limit;
int ret;
if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)) ||
flags == MCL_ONFAULT)
return -EINVAL;
if (!can_do_mlock())
return -EPERM;
lock_limit = rlimit(RLIMIT_MEMLOCK);
lock_limit >>= PAGE_SHIFT;
if (mmap_write_lock_killable(current->mm))
return -EINTR;
ret = -ENOMEM;
if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
capable(CAP_IPC_LOCK))
ret = apply_mlockall_flags(flags);
mmap_write_unlock(current->mm);
if (!ret && (flags & MCL_CURRENT))
mm_populate(0, TASK_SIZE);
return ret;
}
SYSCALL_DEFINE0(munlockall)
{
int ret;
if (mmap_write_lock_killable(current->mm))
return -EINTR;
ret = apply_mlockall_flags(0);
mmap_write_unlock(current->mm);
return ret;
}
/*
* Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB
* shm segments) get accounted against the user_struct instead.
*/
static DEFINE_SPINLOCK(shmlock_user_lock);
int user_shm_lock(size_t size, struct ucounts *ucounts)
{
unsigned long lock_limit, locked;
long memlock;
int allowed = 0;
locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
lock_limit = rlimit(RLIMIT_MEMLOCK);
if (lock_limit != RLIM_INFINITY)
lock_limit >>= PAGE_SHIFT;
spin_lock(&shmlock_user_lock);
memlock = inc_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
if ((memlock == LONG_MAX || memlock > lock_limit) && !capable(CAP_IPC_LOCK)) {
dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
goto out;
}
if (!get_ucounts(ucounts)) {
dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
allowed = 0;
goto out;
}
allowed = 1;
out:
spin_unlock(&shmlock_user_lock);
return allowed;
}
void user_shm_unlock(size_t size, struct ucounts *ucounts)
{
spin_lock(&shmlock_user_lock);
dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, (size + PAGE_SIZE - 1) >> PAGE_SHIFT);
spin_unlock(&shmlock_user_lock);
put_ucounts(ucounts);
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* This implements the various checks for CONFIG_HARDENED_USERCOPY*,
* which are designed to protect kernel memory from needless exposure
* and overwrite under many unintended conditions. This code is based
* on PAX_USERCOPY, which is:
*
* Copyright (C) 2001-2016 PaX Team, Bradley Spengler, Open Source
* Security Inc.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/kstrtox.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/thread_info.h>
#include <linux/vmalloc.h>
#include <linux/atomic.h>
#include <linux/jump_label.h>
#include <asm/sections.h>
#include "slab.h"
/*
* Checks if a given pointer and length is contained by the current
* stack frame (if possible).
*
* Returns:
* NOT_STACK: not at all on the stack
* GOOD_FRAME: fully within a valid stack frame
* GOOD_STACK: within the current stack (when can't frame-check exactly)
* BAD_STACK: error condition (invalid stack position or bad stack frame)
*/
static noinline int check_stack_object(const void *obj, unsigned long len)
{
const void * const stack = task_stack_page(current); const void * const stackend = stack + THREAD_SIZE;
int ret;
/* Object is not on the stack at all. */
if (obj + len <= stack || stackend <= obj)
return NOT_STACK;
/*
* Reject: object partially overlaps the stack (passing the
* check above means at least one end is within the stack,
* so if this check fails, the other end is outside the stack).
*/
if (obj < stack || stackend < obj + len)
return BAD_STACK;
/* Check if object is safely within a valid frame. */
ret = arch_within_stack_frames(stack, stackend, obj, len);
if (ret)
return ret;
/* Finally, check stack depth if possible. */
#ifdef CONFIG_ARCH_HAS_CURRENT_STACK_POINTER
if (IS_ENABLED(CONFIG_STACK_GROWSUP)) {
if ((void *)current_stack_pointer < obj + len)
return BAD_STACK;
} else {
if (obj < (void *)current_stack_pointer)
return BAD_STACK;
}
#endif
return GOOD_STACK;
}
/*
* If these functions are reached, then CONFIG_HARDENED_USERCOPY has found
* an unexpected state during a copy_from_user() or copy_to_user() call.
* There are several checks being performed on the buffer by the
* __check_object_size() function. Normal stack buffer usage should never
* trip the checks, and kernel text addressing will always trip the check.
* For cache objects, it is checking that only the whitelisted range of
* bytes for a given cache is being accessed (via the cache's usersize and
* useroffset fields). To adjust a cache whitelist, use the usercopy-aware
* kmem_cache_create_usercopy() function to create the cache (and
* carefully audit the whitelist range).
*/
void __noreturn usercopy_abort(const char *name, const char *detail,
bool to_user, unsigned long offset,
unsigned long len)
{
pr_emerg("Kernel memory %s attempt detected %s %s%s%s%s (offset %lu, size %lu)!\n",
to_user ? "exposure" : "overwrite",
to_user ? "from" : "to",
name ? : "unknown?!",
detail ? " '" : "", detail ? : "", detail ? "'" : "",
offset, len);
/*
* For greater effect, it would be nice to do do_group_exit(),
* but BUG() actually hooks all the lock-breaking and per-arch
* Oops code, so that is used here instead.
*/
BUG();
}
/* Returns true if any portion of [ptr,ptr+n) over laps with [low,high). */
static bool overlaps(const unsigned long ptr, unsigned long n,
unsigned long low, unsigned long high)
{
const unsigned long check_low = ptr;
unsigned long check_high = check_low + n;
/* Does not overlap if entirely above or entirely below. */
if (check_low >= high || check_high <= low)
return false;
return true;
}
/* Is this address range in the kernel text area? */
static inline void check_kernel_text_object(const unsigned long ptr,
unsigned long n, bool to_user)
{
unsigned long textlow = (unsigned long)_stext;
unsigned long texthigh = (unsigned long)_etext;
unsigned long textlow_linear, texthigh_linear;
if (overlaps(ptr, n, textlow, texthigh)) usercopy_abort("kernel text", NULL, to_user, ptr - textlow, n);
/*
* Some architectures have virtual memory mappings with a secondary
* mapping of the kernel text, i.e. there is more than one virtual
* kernel address that points to the kernel image. It is usually
* when there is a separate linear physical memory mapping, in that
* __pa() is not just the reverse of __va(). This can be detected
* and checked:
*/
textlow_linear = (unsigned long)lm_alias(textlow);
/* No different mapping: we're done. */
if (textlow_linear == textlow)
return;
/* Check the secondary mapping... */
texthigh_linear = (unsigned long)lm_alias(texthigh); if (overlaps(ptr, n, textlow_linear, texthigh_linear)) usercopy_abort("linear kernel text", NULL, to_user,
ptr - textlow_linear, n);
}
static inline void check_bogus_address(const unsigned long ptr, unsigned long n,
bool to_user)
{
/* Reject if object wraps past end of memory. */
if (ptr + (n - 1) < ptr)
usercopy_abort("wrapped address", NULL, to_user, 0, ptr + n);
/* Reject if NULL or ZERO-allocation. */
if (ZERO_OR_NULL_PTR(ptr)) usercopy_abort("null address", NULL, to_user, ptr, n);
}
static inline void check_heap_object(const void *ptr, unsigned long n,
bool to_user)
{
unsigned long addr = (unsigned long)ptr;
unsigned long offset;
struct folio *folio;
if (is_kmap_addr(ptr)) {
offset = offset_in_page(ptr);
if (n > PAGE_SIZE - offset)
usercopy_abort("kmap", NULL, to_user, offset, n);
return;
}
if (is_vmalloc_addr(ptr) && !pagefault_disabled()) { struct vmap_area *area = find_vmap_area(addr);
if (!area)
usercopy_abort("vmalloc", "no area", to_user, 0, n); if (n > area->va_end - addr) { offset = addr - area->va_start;
usercopy_abort("vmalloc", NULL, to_user, offset, n);
}
return;
}
if (!virt_addr_valid(ptr))
return;
folio = virt_to_folio(ptr); if (folio_test_slab(folio)) {
/* Check slab allocator for flags and size. */
__check_heap_object(ptr, n, folio_slab(folio), to_user); } else if (folio_test_large(folio)) { offset = ptr - folio_address(folio); if (n > folio_size(folio) - offset) usercopy_abort("page alloc", NULL, to_user, offset, n);
}
}
static DEFINE_STATIC_KEY_FALSE_RO(bypass_usercopy_checks);
/*
* Validates that the given object is:
* - not bogus address
* - fully contained by stack (or stack frame, when available)
* - fully within SLAB object (or object whitelist area, when available)
* - not in kernel text
*/
void __check_object_size(const void *ptr, unsigned long n, bool to_user)
{
if (static_branch_unlikely(&bypass_usercopy_checks))
return;
/* Skip all tests if size is zero. */
if (!n)
return;
/* Check for invalid addresses. */
check_bogus_address((const unsigned long)ptr, n, to_user);
/* Check for bad stack object. */
switch (check_stack_object(ptr, n)) {
case NOT_STACK:
/* Object is not touching the current process stack. */
break;
case GOOD_FRAME:
case GOOD_STACK:
/*
* Object is either in the correct frame (when it
* is possible to check) or just generally on the
* process stack (when frame checking not available).
*/
return;
default:
usercopy_abort("process stack", NULL, to_user,
#ifdef CONFIG_ARCH_HAS_CURRENT_STACK_POINTER
IS_ENABLED(CONFIG_STACK_GROWSUP) ?
ptr - (void *)current_stack_pointer :
(void *)current_stack_pointer - ptr,
#else
0,
#endif
n);
}
/* Check for bad heap object. */
check_heap_object(ptr, n, to_user);
/* Check for object in kernel to avoid text exposure. */
check_kernel_text_object((const unsigned long)ptr, n, to_user);}
EXPORT_SYMBOL(__check_object_size);
static bool enable_checks __initdata = true;
static int __init parse_hardened_usercopy(char *str)
{
if (kstrtobool(str, &enable_checks))
pr_warn("Invalid option string for hardened_usercopy: '%s'\n",
str);
return 1;
}
__setup("hardened_usercopy=", parse_hardened_usercopy);
static int __init set_hardened_usercopy(void)
{
if (enable_checks == false)
static_branch_enable(&bypass_usercopy_checks);
return 1;
}
late_initcall(set_hardened_usercopy);
// SPDX-License-Identifier: GPL-2.0+
/*
* XArray implementation
* Copyright (c) 2017-2018 Microsoft Corporation
* Copyright (c) 2018-2020 Oracle
* Author: Matthew Wilcox <willy@infradead.org>
*/
#include <linux/bitmap.h>
#include <linux/export.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/xarray.h>
#include "radix-tree.h"
/*
* Coding conventions in this file:
*
* @xa is used to refer to the entire xarray.
* @xas is the 'xarray operation state'. It may be either a pointer to
* an xa_state, or an xa_state stored on the stack. This is an unfortunate
* ambiguity.
* @index is the index of the entry being operated on
* @mark is an xa_mark_t; a small number indicating one of the mark bits.
* @node refers to an xa_node; usually the primary one being operated on by
* this function.
* @offset is the index into the slots array inside an xa_node.
* @parent refers to the @xa_node closer to the head than @node.
* @entry refers to something stored in a slot in the xarray
*/
static inline unsigned int xa_lock_type(const struct xarray *xa)
{
return (__force unsigned int)xa->xa_flags & 3;
}
static inline void xas_lock_type(struct xa_state *xas, unsigned int lock_type)
{
if (lock_type == XA_LOCK_IRQ)
xas_lock_irq(xas);
else if (lock_type == XA_LOCK_BH)
xas_lock_bh(xas);
else
xas_lock(xas);
}
static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type)
{
if (lock_type == XA_LOCK_IRQ)
xas_unlock_irq(xas);
else if (lock_type == XA_LOCK_BH)
xas_unlock_bh(xas);
else
xas_unlock(xas);
}
static inline bool xa_track_free(const struct xarray *xa)
{
return xa->xa_flags & XA_FLAGS_TRACK_FREE;
}
static inline bool xa_zero_busy(const struct xarray *xa)
{
return xa->xa_flags & XA_FLAGS_ZERO_BUSY;
}
static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)
{
if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))
xa->xa_flags |= XA_FLAGS_MARK(mark);
}
static inline void xa_mark_clear(struct xarray *xa, xa_mark_t mark)
{
if (xa->xa_flags & XA_FLAGS_MARK(mark)) xa->xa_flags &= ~(XA_FLAGS_MARK(mark));
}
static inline unsigned long *node_marks(struct xa_node *node, xa_mark_t mark)
{
return node->marks[(__force unsigned)mark];
}
static inline bool node_get_mark(struct xa_node *node,
unsigned int offset, xa_mark_t mark)
{
return test_bit(offset, node_marks(node, mark));
}
/* returns true if the bit was set */
static inline bool node_set_mark(struct xa_node *node, unsigned int offset,
xa_mark_t mark)
{
return __test_and_set_bit(offset, node_marks(node, mark));
}
/* returns true if the bit was set */
static inline bool node_clear_mark(struct xa_node *node, unsigned int offset,
xa_mark_t mark)
{
return __test_and_clear_bit(offset, node_marks(node, mark));
}
static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark)
{
return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE);
}
static inline void node_mark_all(struct xa_node *node, xa_mark_t mark)
{
bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE);
}
#define mark_inc(mark) do { \
mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \
} while (0)
/*
* xas_squash_marks() - Merge all marks to the first entry
* @xas: Array operation state.
*
* Set a mark on the first entry if any entry has it set. Clear marks on
* all sibling entries.
*/
static void xas_squash_marks(const struct xa_state *xas)
{
unsigned int mark = 0;
unsigned int limit = xas->xa_offset + xas->xa_sibs + 1;
if (!xas->xa_sibs)
return;
do {
unsigned long *marks = xas->xa_node->marks[mark];
if (find_next_bit(marks, limit, xas->xa_offset + 1) == limit)
continue;
__set_bit(xas->xa_offset, marks);
bitmap_clear(marks, xas->xa_offset + 1, xas->xa_sibs);
} while (mark++ != (__force unsigned)XA_MARK_MAX);
}
/* extracts the offset within this node from the index */
static unsigned int get_offset(unsigned long index, struct xa_node *node)
{
return (index >> node->shift) & XA_CHUNK_MASK;
}
static void xas_set_offset(struct xa_state *xas)
{
xas->xa_offset = get_offset(xas->xa_index, xas->xa_node);
}
/* move the index either forwards (find) or backwards (sibling slot) */
static void xas_move_index(struct xa_state *xas, unsigned long offset)
{
unsigned int shift = xas->xa_node->shift; xas->xa_index &= ~XA_CHUNK_MASK << shift; xas->xa_index += offset << shift;
}
static void xas_next_offset(struct xa_state *xas)
{
xas->xa_offset++;
xas_move_index(xas, xas->xa_offset);
}
static void *set_bounds(struct xa_state *xas)
{
xas->xa_node = XAS_BOUNDS;
return NULL;
}
/*
* Starts a walk. If the @xas is already valid, we assume that it's on
* the right path and just return where we've got to. If we're in an
* error state, return NULL. If the index is outside the current scope
* of the xarray, return NULL without changing @xas->xa_node. Otherwise
* set @xas->xa_node to NULL and return the current head of the array.
*/
static void *xas_start(struct xa_state *xas)
{
void *entry;
if (xas_valid(xas)) return xas_reload(xas); if (xas_error(xas))
return NULL;
entry = xa_head(xas->xa); if (!xa_is_node(entry)) { if (xas->xa_index) return set_bounds(xas);
} else {
if ((xas->xa_index >> xa_to_node(entry)->shift) > XA_CHUNK_MASK)
return set_bounds(xas);
}
xas->xa_node = NULL; return entry;
}
static void *xas_descend(struct xa_state *xas, struct xa_node *node)
{
unsigned int offset = get_offset(xas->xa_index, node); void *entry = xa_entry(xas->xa, node, offset); xas->xa_node = node; while (xa_is_sibling(entry)) { offset = xa_to_sibling(entry); entry = xa_entry(xas->xa, node, offset); if (node->shift && xa_is_node(entry))
entry = XA_RETRY_ENTRY;
}
xas->xa_offset = offset;
return entry;
}
/**
* xas_load() - Load an entry from the XArray (advanced).
* @xas: XArray operation state.
*
* Usually walks the @xas to the appropriate state to load the entry
* stored at xa_index. However, it will do nothing and return %NULL if
* @xas is in an error state. xas_load() will never expand the tree.
*
* If the xa_state is set up to operate on a multi-index entry, xas_load()
* may return %NULL or an internal entry, even if there are entries
* present within the range specified by @xas.
*
* Context: Any context. The caller should hold the xa_lock or the RCU lock.
* Return: Usually an entry in the XArray, but see description for exceptions.
*/
void *xas_load(struct xa_state *xas)
{
void *entry = xas_start(xas); while (xa_is_node(entry)) { struct xa_node *node = xa_to_node(entry);
if (xas->xa_shift > node->shift)
break;
entry = xas_descend(xas, node);
if (node->shift == 0)
break;
}
return entry;
}
EXPORT_SYMBOL_GPL(xas_load);
#define XA_RCU_FREE ((struct xarray *)1)
static void xa_node_free(struct xa_node *node)
{
XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
node->array = XA_RCU_FREE;
call_rcu(&node->rcu_head, radix_tree_node_rcu_free);
}
/*
* xas_destroy() - Free any resources allocated during the XArray operation.
* @xas: XArray operation state.
*
* Most users will not need to call this function; it is called for you
* by xas_nomem().
*/
void xas_destroy(struct xa_state *xas)
{
struct xa_node *next, *node = xas->xa_alloc;
while (node) {
XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
next = rcu_dereference_raw(node->parent);
radix_tree_node_rcu_free(&node->rcu_head);
xas->xa_alloc = node = next;
}
}
/**
* xas_nomem() - Allocate memory if needed.
* @xas: XArray operation state.
* @gfp: Memory allocation flags.
*
* If we need to add new nodes to the XArray, we try to allocate memory
* with GFP_NOWAIT while holding the lock, which will usually succeed.
* If it fails, @xas is flagged as needing memory to continue. The caller
* should drop the lock and call xas_nomem(). If xas_nomem() succeeds,
* the caller should retry the operation.
*
* Forward progress is guaranteed as one node is allocated here and
* stored in the xa_state where it will be found by xas_alloc(). More
* nodes will likely be found in the slab allocator, but we do not tie
* them up here.
*
* Return: true if memory was needed, and was successfully allocated.
*/
bool xas_nomem(struct xa_state *xas, gfp_t gfp)
{
if (xas->xa_node != XA_ERROR(-ENOMEM)) { xas_destroy(xas);
return false;
}
if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) gfp |= __GFP_ACCOUNT; xas->xa_alloc = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp);
if (!xas->xa_alloc)
return false;
xas->xa_alloc->parent = NULL;
XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list));
xas->xa_node = XAS_RESTART;
return true;
}
EXPORT_SYMBOL_GPL(xas_nomem);
/*
* __xas_nomem() - Drop locks and allocate memory if needed.
* @xas: XArray operation state.
* @gfp: Memory allocation flags.
*
* Internal variant of xas_nomem().
*
* Return: true if memory was needed, and was successfully allocated.
*/
static bool __xas_nomem(struct xa_state *xas, gfp_t gfp)
__must_hold(xas->xa->xa_lock)
{
unsigned int lock_type = xa_lock_type(xas->xa);
if (xas->xa_node != XA_ERROR(-ENOMEM)) {
xas_destroy(xas);
return false;
}
if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT)
gfp |= __GFP_ACCOUNT;
if (gfpflags_allow_blocking(gfp)) {
xas_unlock_type(xas, lock_type);
xas->xa_alloc = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp);
xas_lock_type(xas, lock_type);
} else {
xas->xa_alloc = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp);
}
if (!xas->xa_alloc)
return false;
xas->xa_alloc->parent = NULL;
XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list));
xas->xa_node = XAS_RESTART;
return true;
}
static void xas_update(struct xa_state *xas, struct xa_node *node)
{
if (xas->xa_update) xas->xa_update(node);
else
XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
}
static void *xas_alloc(struct xa_state *xas, unsigned int shift)
{
struct xa_node *parent = xas->xa_node;
struct xa_node *node = xas->xa_alloc;
if (xas_invalid(xas))
return NULL;
if (node) { xas->xa_alloc = NULL;
} else {
gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN;
if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT)
gfp |= __GFP_ACCOUNT;
node = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp);
if (!node) {
xas_set_err(xas, -ENOMEM);
return NULL;
}
}
if (parent) { node->offset = xas->xa_offset;
parent->count++;
XA_NODE_BUG_ON(node, parent->count > XA_CHUNK_SIZE);
xas_update(xas, parent);
}
XA_NODE_BUG_ON(node, shift > BITS_PER_LONG);
XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
node->shift = shift;
node->count = 0;
node->nr_values = 0;
RCU_INIT_POINTER(node->parent, xas->xa_node);
node->array = xas->xa;
return node;
}
#ifdef CONFIG_XARRAY_MULTI
/* Returns the number of indices covered by a given xa_state */
static unsigned long xas_size(const struct xa_state *xas)
{
return (xas->xa_sibs + 1UL) << xas->xa_shift;
}
#endif
/*
* Use this to calculate the maximum index that will need to be created
* in order to add the entry described by @xas. Because we cannot store a
* multi-index entry at index 0, the calculation is a little more complex
* than you might expect.
*/
static unsigned long xas_max(struct xa_state *xas)
{
unsigned long max = xas->xa_index;
#ifdef CONFIG_XARRAY_MULTI
if (xas->xa_shift || xas->xa_sibs) { unsigned long mask = xas_size(xas) - 1;
max |= mask;
if (mask == max)
max++;
}
#endif
return max;
}
/* The maximum index that can be contained in the array without expanding it */
static unsigned long max_index(void *entry)
{
if (!xa_is_node(entry))
return 0;
return (XA_CHUNK_SIZE << xa_to_node(entry)->shift) - 1;
}
static void xas_shrink(struct xa_state *xas)
{
struct xarray *xa = xas->xa;
struct xa_node *node = xas->xa_node;
for (;;) {
void *entry;
XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE);
if (node->count != 1)
break;
entry = xa_entry_locked(xa, node, 0);
if (!entry)
break;
if (!xa_is_node(entry) && node->shift)
break;
if (xa_is_zero(entry) && xa_zero_busy(xa))
entry = NULL;
xas->xa_node = XAS_BOUNDS;
RCU_INIT_POINTER(xa->xa_head, entry);
if (xa_track_free(xa) && !node_get_mark(node, 0, XA_FREE_MARK)) xa_mark_clear(xa, XA_FREE_MARK); node->count = 0;
node->nr_values = 0;
if (!xa_is_node(entry)) RCU_INIT_POINTER(node->slots[0], XA_RETRY_ENTRY); xas_update(xas, node); xa_node_free(node); if (!xa_is_node(entry))
break;
node = xa_to_node(entry);
node->parent = NULL;
}
}
/*
* xas_delete_node() - Attempt to delete an xa_node
* @xas: Array operation state.
*
* Attempts to delete the @xas->xa_node. This will fail if xa->node has
* a non-zero reference count.
*/
static void xas_delete_node(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;
for (;;) {
struct xa_node *parent;
XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE);
if (node->count)
break;
parent = xa_parent_locked(xas->xa, node);
xas->xa_node = parent;
xas->xa_offset = node->offset;
xa_node_free(node);
if (!parent) {
xas->xa->xa_head = NULL;
xas->xa_node = XAS_BOUNDS;
return;
}
parent->slots[xas->xa_offset] = NULL;
parent->count--;
XA_NODE_BUG_ON(parent, parent->count > XA_CHUNK_SIZE);
node = parent;
xas_update(xas, node);
}
if (!node->parent) xas_shrink(xas);
}
/**
* xas_free_nodes() - Free this node and all nodes that it references
* @xas: Array operation state.
* @top: Node to free
*
* This node has been removed from the tree. We must now free it and all
* of its subnodes. There may be RCU walkers with references into the tree,
* so we must replace all entries with retry markers.
*/
static void xas_free_nodes(struct xa_state *xas, struct xa_node *top)
{
unsigned int offset = 0;
struct xa_node *node = top;
for (;;) {
void *entry = xa_entry_locked(xas->xa, node, offset); if (node->shift && xa_is_node(entry)) { node = xa_to_node(entry);
offset = 0;
continue;
}
if (entry) RCU_INIT_POINTER(node->slots[offset], XA_RETRY_ENTRY); offset++;
while (offset == XA_CHUNK_SIZE) {
struct xa_node *parent;
parent = xa_parent_locked(xas->xa, node);
offset = node->offset + 1;
node->count = 0;
node->nr_values = 0;
xas_update(xas, node); xa_node_free(node);
if (node == top)
return;
node = parent;
}
}
}
/*
* xas_expand adds nodes to the head of the tree until it has reached
* sufficient height to be able to contain @xas->xa_index
*/
static int xas_expand(struct xa_state *xas, void *head)
{
struct xarray *xa = xas->xa;
struct xa_node *node = NULL;
unsigned int shift = 0;
unsigned long max = xas_max(xas); if (!head) { if (max == 0)
return 0;
while ((max >> shift) >= XA_CHUNK_SIZE)
shift += XA_CHUNK_SHIFT;
return shift + XA_CHUNK_SHIFT; } else if (xa_is_node(head)) { node = xa_to_node(head);
shift = node->shift + XA_CHUNK_SHIFT;
}
xas->xa_node = NULL; while (max > max_index(head)) {
xa_mark_t mark = 0;
XA_NODE_BUG_ON(node, shift > BITS_PER_LONG);
node = xas_alloc(xas, shift);
if (!node)
return -ENOMEM;
node->count = 1;
if (xa_is_value(head))
node->nr_values = 1; RCU_INIT_POINTER(node->slots[0], head);
/* Propagate the aggregated mark info to the new child */
for (;;) {
if (xa_track_free(xa) && mark == XA_FREE_MARK) { node_mark_all(node, XA_FREE_MARK);
if (!xa_marked(xa, XA_FREE_MARK)) {
node_clear_mark(node, 0, XA_FREE_MARK); xa_mark_set(xa, XA_FREE_MARK);
}
} else if (xa_marked(xa, mark)) { node_set_mark(node, 0, mark);
}
if (mark == XA_MARK_MAX)
break;
mark_inc(mark);
}
/*
* Now that the new node is fully initialised, we can add
* it to the tree
*/
if (xa_is_node(head)) { xa_to_node(head)->offset = 0;
rcu_assign_pointer(xa_to_node(head)->parent, node);
}
head = xa_mk_node(node);
rcu_assign_pointer(xa->xa_head, head);
xas_update(xas, node); shift += XA_CHUNK_SHIFT;
}
xas->xa_node = node;
return shift;
}
/*
* xas_create() - Create a slot to store an entry in.
* @xas: XArray operation state.
* @allow_root: %true if we can store the entry in the root directly
*
* Most users will not need to call this function directly, as it is called
* by xas_store(). It is useful for doing conditional store operations
* (see the xa_cmpxchg() implementation for an example).
*
* Return: If the slot already existed, returns the contents of this slot.
* If the slot was newly created, returns %NULL. If it failed to create the
* slot, returns %NULL and indicates the error in @xas.
*/
static void *xas_create(struct xa_state *xas, bool allow_root)
{
struct xarray *xa = xas->xa;
void *entry;
void __rcu **slot;
struct xa_node *node = xas->xa_node;
int shift;
unsigned int order = xas->xa_shift;
if (xas_top(node)) {
entry = xa_head_locked(xa);
xas->xa_node = NULL;
if (!entry && xa_zero_busy(xa))
entry = XA_ZERO_ENTRY;
shift = xas_expand(xas, entry); if (shift < 0)
return NULL;
if (!shift && !allow_root)
shift = XA_CHUNK_SHIFT;
entry = xa_head_locked(xa);
slot = &xa->xa_head;
} else if (xas_error(xas)) {
return NULL;
} else if (node) {
unsigned int offset = xas->xa_offset;
shift = node->shift;
entry = xa_entry_locked(xa, node, offset);
slot = &node->slots[offset];
} else {
shift = 0;
entry = xa_head_locked(xa);
slot = &xa->xa_head;
}
while (shift > order) { shift -= XA_CHUNK_SHIFT;
if (!entry) {
node = xas_alloc(xas, shift);
if (!node)
break;
if (xa_track_free(xa)) node_mark_all(node, XA_FREE_MARK); rcu_assign_pointer(*slot, xa_mk_node(node)); } else if (xa_is_node(entry)) { node = xa_to_node(entry);
} else {
break;
}
entry = xas_descend(xas, node);
slot = &node->slots[xas->xa_offset];
}
return entry;
}
/**
* xas_create_range() - Ensure that stores to this range will succeed
* @xas: XArray operation state.
*
* Creates all of the slots in the range covered by @xas. Sets @xas to
* create single-index entries and positions it at the beginning of the
* range. This is for the benefit of users which have not yet been
* converted to use multi-index entries.
*/
void xas_create_range(struct xa_state *xas)
{
unsigned long index = xas->xa_index;
unsigned char shift = xas->xa_shift;
unsigned char sibs = xas->xa_sibs;
xas->xa_index |= ((sibs + 1UL) << shift) - 1;
if (xas_is_node(xas) && xas->xa_node->shift == xas->xa_shift)
xas->xa_offset |= sibs;
xas->xa_shift = 0;
xas->xa_sibs = 0;
for (;;) {
xas_create(xas, true);
if (xas_error(xas))
goto restore;
if (xas->xa_index <= (index | XA_CHUNK_MASK))
goto success;
xas->xa_index -= XA_CHUNK_SIZE;
for (;;) {
struct xa_node *node = xas->xa_node;
if (node->shift >= shift)
break;
xas->xa_node = xa_parent_locked(xas->xa, node);
xas->xa_offset = node->offset - 1;
if (node->offset != 0)
break;
}
}
restore:
xas->xa_shift = shift;
xas->xa_sibs = sibs;
xas->xa_index = index;
return;
success:
xas->xa_index = index;
if (xas->xa_node)
xas_set_offset(xas);
}
EXPORT_SYMBOL_GPL(xas_create_range);
static void update_node(struct xa_state *xas, struct xa_node *node,
int count, int values)
{
if (!node || (!count && !values))
return;
node->count += count;
node->nr_values += values;
XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE);
XA_NODE_BUG_ON(node, node->nr_values > XA_CHUNK_SIZE);
xas_update(xas, node); if (count < 0) xas_delete_node(xas);
}
/**
* xas_store() - Store this entry in the XArray.
* @xas: XArray operation state.
* @entry: New entry.
*
* If @xas is operating on a multi-index entry, the entry returned by this
* function is essentially meaningless (it may be an internal entry or it
* may be %NULL, even if there are non-NULL entries at some of the indices
* covered by the range). This is not a problem for any current users,
* and can be changed if needed.
*
* Return: The old entry at this index.
*/
void *xas_store(struct xa_state *xas, void *entry)
{
struct xa_node *node;
void __rcu **slot = &xas->xa->xa_head;
unsigned int offset, max;
int count = 0;
int values = 0;
void *first, *next;
bool value = xa_is_value(entry);
if (entry) {
bool allow_root = !xa_is_node(entry) && !xa_is_zero(entry); first = xas_create(xas, allow_root);
} else {
first = xas_load(xas);
}
if (xas_invalid(xas))
return first;
node = xas->xa_node;
if (node && (xas->xa_shift < node->shift)) xas->xa_sibs = 0; if ((first == entry) && !xas->xa_sibs)
return first;
next = first;
offset = xas->xa_offset; max = xas->xa_offset + xas->xa_sibs;
if (node) {
slot = &node->slots[offset];
if (xas->xa_sibs)
xas_squash_marks(xas);
}
if (!entry) xas_init_marks(xas);
for (;;) {
/*
* Must clear the marks before setting the entry to NULL,
* otherwise xas_for_each_marked may find a NULL entry and
* stop early. rcu_assign_pointer contains a release barrier
* so the mark clearing will appear to happen before the
* entry is set to NULL.
*/
rcu_assign_pointer(*slot, entry); if (xa_is_node(next) && (!node || node->shift)) xas_free_nodes(xas, xa_to_node(next)); if (!node)
break;
count += !next - !entry;
values += !xa_is_value(first) - !value;
if (entry) {
if (offset == max)
break;
if (!xa_is_sibling(entry)) entry = xa_mk_sibling(xas->xa_offset);
} else {
if (offset == XA_CHUNK_MASK)
break;
}
next = xa_entry_locked(xas->xa, node, ++offset); if (!xa_is_sibling(next)) { if (!entry && (offset > max))
break;
first = next;
}
slot++;
}
update_node(xas, node, count, values);
return first;
}
EXPORT_SYMBOL_GPL(xas_store);
/**
* xas_get_mark() - Returns the state of this mark.
* @xas: XArray operation state.
* @mark: Mark number.
*
* Return: true if the mark is set, false if the mark is clear or @xas
* is in an error state.
*/
bool xas_get_mark(const struct xa_state *xas, xa_mark_t mark)
{
if (xas_invalid(xas))
return false;
if (!xas->xa_node)
return xa_marked(xas->xa, mark);
return node_get_mark(xas->xa_node, xas->xa_offset, mark);
}
EXPORT_SYMBOL_GPL(xas_get_mark);
/**
* xas_set_mark() - Sets the mark on this entry and its parents.
* @xas: XArray operation state.
* @mark: Mark number.
*
* Sets the specified mark on this entry, and walks up the tree setting it
* on all the ancestor entries. Does nothing if @xas has not been walked to
* an entry, or is in an error state.
*/
void xas_set_mark(const struct xa_state *xas, xa_mark_t mark)
{
struct xa_node *node = xas->xa_node; unsigned int offset = xas->xa_offset;
if (xas_invalid(xas))
return;
while (node) { if (node_set_mark(node, offset, mark))
return;
offset = node->offset; node = xa_parent_locked(xas->xa, node);
}
if (!xa_marked(xas->xa, mark)) xa_mark_set(xas->xa, mark);
}
EXPORT_SYMBOL_GPL(xas_set_mark);
/**
* xas_clear_mark() - Clears the mark on this entry and its parents.
* @xas: XArray operation state.
* @mark: Mark number.
*
* Clears the specified mark on this entry, and walks back to the head
* attempting to clear it on all the ancestor entries. Does nothing if
* @xas has not been walked to an entry, or is in an error state.
*/
void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark)
{
struct xa_node *node = xas->xa_node; unsigned int offset = xas->xa_offset;
if (xas_invalid(xas))
return;
while (node) { if (!node_clear_mark(node, offset, mark))
return;
if (node_any_mark(node, mark))
return;
offset = node->offset; node = xa_parent_locked(xas->xa, node);
}
if (xa_marked(xas->xa, mark)) xa_mark_clear(xas->xa, mark);
}
EXPORT_SYMBOL_GPL(xas_clear_mark);
/**
* xas_init_marks() - Initialise all marks for the entry
* @xas: Array operations state.
*
* Initialise all marks for the entry specified by @xas. If we're tracking
* free entries with a mark, we need to set it on all entries. All other
* marks are cleared.
*
* This implementation is not as efficient as it could be; we may walk
* up the tree multiple times.
*/
void xas_init_marks(const struct xa_state *xas)
{
xa_mark_t mark = 0;
for (;;) {
if (xa_track_free(xas->xa) && mark == XA_FREE_MARK) xas_set_mark(xas, mark);
else
xas_clear_mark(xas, mark);
if (mark == XA_MARK_MAX)
break;
mark_inc(mark);
}
}
EXPORT_SYMBOL_GPL(xas_init_marks);
#ifdef CONFIG_XARRAY_MULTI
static unsigned int node_get_marks(struct xa_node *node, unsigned int offset)
{
unsigned int marks = 0;
xa_mark_t mark = XA_MARK_0;
for (;;) {
if (node_get_mark(node, offset, mark))
marks |= 1 << (__force unsigned int)mark;
if (mark == XA_MARK_MAX)
break;
mark_inc(mark);
}
return marks;
}
static void node_set_marks(struct xa_node *node, unsigned int offset,
struct xa_node *child, unsigned int marks)
{
xa_mark_t mark = XA_MARK_0;
for (;;) {
if (marks & (1 << (__force unsigned int)mark)) {
node_set_mark(node, offset, mark);
if (child)
node_mark_all(child, mark);
}
if (mark == XA_MARK_MAX)
break;
mark_inc(mark);
}
}
/**
* xas_split_alloc() - Allocate memory for splitting an entry.
* @xas: XArray operation state.
* @entry: New entry which will be stored in the array.
* @order: Current entry order.
* @gfp: Memory allocation flags.
*
* This function should be called before calling xas_split().
* If necessary, it will allocate new nodes (and fill them with @entry)
* to prepare for the upcoming split of an entry of @order size into
* entries of the order stored in the @xas.
*
* Context: May sleep if @gfp flags permit.
*/
void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order,
gfp_t gfp)
{
unsigned int sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1;
unsigned int mask = xas->xa_sibs;
/* XXX: no support for splitting really large entries yet */
if (WARN_ON(xas->xa_shift + 2 * XA_CHUNK_SHIFT < order))
goto nomem;
if (xas->xa_shift + XA_CHUNK_SHIFT > order)
return;
do {
unsigned int i;
void *sibling = NULL;
struct xa_node *node;
node = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp);
if (!node)
goto nomem;
node->array = xas->xa;
for (i = 0; i < XA_CHUNK_SIZE; i++) {
if ((i & mask) == 0) {
RCU_INIT_POINTER(node->slots[i], entry);
sibling = xa_mk_sibling(i);
} else {
RCU_INIT_POINTER(node->slots[i], sibling);
}
}
RCU_INIT_POINTER(node->parent, xas->xa_alloc);
xas->xa_alloc = node;
} while (sibs-- > 0);
return;
nomem:
xas_destroy(xas);
xas_set_err(xas, -ENOMEM);
}
EXPORT_SYMBOL_GPL(xas_split_alloc);
/**
* xas_split() - Split a multi-index entry into smaller entries.
* @xas: XArray operation state.
* @entry: New entry to store in the array.
* @order: Current entry order.
*
* The size of the new entries is set in @xas. The value in @entry is
* copied to all the replacement entries.
*
* Context: Any context. The caller should hold the xa_lock.
*/
void xas_split(struct xa_state *xas, void *entry, unsigned int order)
{
unsigned int sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1;
unsigned int offset, marks;
struct xa_node *node;
void *curr = xas_load(xas);
int values = 0;
node = xas->xa_node;
if (xas_top(node))
return;
marks = node_get_marks(node, xas->xa_offset);
offset = xas->xa_offset + sibs;
do {
if (xas->xa_shift < node->shift) {
struct xa_node *child = xas->xa_alloc;
xas->xa_alloc = rcu_dereference_raw(child->parent);
child->shift = node->shift - XA_CHUNK_SHIFT;
child->offset = offset;
child->count = XA_CHUNK_SIZE;
child->nr_values = xa_is_value(entry) ?
XA_CHUNK_SIZE : 0;
RCU_INIT_POINTER(child->parent, node);
node_set_marks(node, offset, child, marks);
rcu_assign_pointer(node->slots[offset],
xa_mk_node(child));
if (xa_is_value(curr))
values--;
xas_update(xas, child);
} else {
unsigned int canon = offset - xas->xa_sibs;
node_set_marks(node, canon, NULL, marks);
rcu_assign_pointer(node->slots[canon], entry);
while (offset > canon)
rcu_assign_pointer(node->slots[offset--],
xa_mk_sibling(canon));
values += (xa_is_value(entry) - xa_is_value(curr)) *
(xas->xa_sibs + 1);
}
} while (offset-- > xas->xa_offset);
node->nr_values += values;
xas_update(xas, node);
}
EXPORT_SYMBOL_GPL(xas_split);
#endif
/**
* xas_pause() - Pause a walk to drop a lock.
* @xas: XArray operation state.
*
* Some users need to pause a walk and drop the lock they're holding in
* order to yield to a higher priority thread or carry out an operation
* on an entry. Those users should call this function before they drop
* the lock. It resets the @xas to be suitable for the next iteration
* of the loop after the user has reacquired the lock. If most entries
* found during a walk require you to call xas_pause(), the xa_for_each()
* iterator may be more appropriate.
*
* Note that xas_pause() only works for forward iteration. If a user needs
* to pause a reverse iteration, we will need a xas_pause_rev().
*/
void xas_pause(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;
if (xas_invalid(xas))
return;
xas->xa_node = XAS_RESTART;
if (node) {
unsigned long offset = xas->xa_offset;
while (++offset < XA_CHUNK_SIZE) {
if (!xa_is_sibling(xa_entry(xas->xa, node, offset)))
break;
}
xas->xa_index += (offset - xas->xa_offset) << node->shift;
if (xas->xa_index == 0)
xas->xa_node = XAS_BOUNDS;
} else {
xas->xa_index++;
}
}
EXPORT_SYMBOL_GPL(xas_pause);
/*
* __xas_prev() - Find the previous entry in the XArray.
* @xas: XArray operation state.
*
* Helper function for xas_prev() which handles all the complex cases
* out of line.
*/
void *__xas_prev(struct xa_state *xas)
{
void *entry;
if (!xas_frozen(xas->xa_node))
xas->xa_index--;
if (!xas->xa_node)
return set_bounds(xas);
if (xas_not_node(xas->xa_node))
return xas_load(xas);
if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
xas->xa_offset--;
while (xas->xa_offset == 255) {
xas->xa_offset = xas->xa_node->offset - 1;
xas->xa_node = xa_parent(xas->xa, xas->xa_node);
if (!xas->xa_node)
return set_bounds(xas);
}
for (;;) {
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
if (!xa_is_node(entry))
return entry;
xas->xa_node = xa_to_node(entry);
xas_set_offset(xas);
}
}
EXPORT_SYMBOL_GPL(__xas_prev);
/*
* __xas_next() - Find the next entry in the XArray.
* @xas: XArray operation state.
*
* Helper function for xas_next() which handles all the complex cases
* out of line.
*/
void *__xas_next(struct xa_state *xas)
{
void *entry;
if (!xas_frozen(xas->xa_node))
xas->xa_index++;
if (!xas->xa_node)
return set_bounds(xas);
if (xas_not_node(xas->xa_node))
return xas_load(xas);
if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
xas->xa_offset++;
while (xas->xa_offset == XA_CHUNK_SIZE) {
xas->xa_offset = xas->xa_node->offset + 1;
xas->xa_node = xa_parent(xas->xa, xas->xa_node);
if (!xas->xa_node)
return set_bounds(xas);
}
for (;;) {
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
if (!xa_is_node(entry))
return entry;
xas->xa_node = xa_to_node(entry);
xas_set_offset(xas);
}
}
EXPORT_SYMBOL_GPL(__xas_next);
/**
* xas_find() - Find the next present entry in the XArray.
* @xas: XArray operation state.
* @max: Highest index to return.
*
* If the @xas has not yet been walked to an entry, return the entry
* which has an index >= xas.xa_index. If it has been walked, the entry
* currently being pointed at has been processed, and so we move to the
* next entry.
*
* If no entry is found and the array is smaller than @max, the iterator
* is set to the smallest index not yet in the array. This allows @xas
* to be immediately passed to xas_store().
*
* Return: The entry, if found, otherwise %NULL.
*/
void *xas_find(struct xa_state *xas, unsigned long max)
{
void *entry;
if (xas_error(xas) || xas->xa_node == XAS_BOUNDS)
return NULL;
if (xas->xa_index > max) return set_bounds(xas); if (!xas->xa_node) { xas->xa_index = 1;
return set_bounds(xas);
} else if (xas->xa_node == XAS_RESTART) { entry = xas_load(xas); if (entry || xas_not_node(xas->xa_node))
return entry;
} else if (!xas->xa_node->shift && xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)) { xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
}
xas_next_offset(xas); while (xas->xa_node && (xas->xa_index <= max)) { if (unlikely(xas->xa_offset == XA_CHUNK_SIZE)) { xas->xa_offset = xas->xa_node->offset + 1; xas->xa_node = xa_parent(xas->xa, xas->xa_node);
continue;
}
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); if (xa_is_node(entry)) { xas->xa_node = xa_to_node(entry);
xas->xa_offset = 0;
continue;
}
if (entry && !xa_is_sibling(entry))
return entry;
xas_next_offset(xas);
}
if (!xas->xa_node)
xas->xa_node = XAS_BOUNDS;
return NULL;
}
EXPORT_SYMBOL_GPL(xas_find);
/**
* xas_find_marked() - Find the next marked entry in the XArray.
* @xas: XArray operation state.
* @max: Highest index to return.
* @mark: Mark number to search for.
*
* If the @xas has not yet been walked to an entry, return the marked entry
* which has an index >= xas.xa_index. If it has been walked, the entry
* currently being pointed at has been processed, and so we return the
* first marked entry with an index > xas.xa_index.
*
* If no marked entry is found and the array is smaller than @max, @xas is
* set to the bounds state and xas->xa_index is set to the smallest index
* not yet in the array. This allows @xas to be immediately passed to
* xas_store().
*
* If no entry is found before @max is reached, @xas is set to the restart
* state.
*
* Return: The entry, if found, otherwise %NULL.
*/
void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark)
{
bool advance = true;
unsigned int offset;
void *entry;
if (xas_error(xas))
return NULL;
if (xas->xa_index > max)
goto max;
if (!xas->xa_node) { xas->xa_index = 1;
goto out;
} else if (xas_top(xas->xa_node)) {
advance = false;
entry = xa_head(xas->xa); xas->xa_node = NULL; if (xas->xa_index > max_index(entry))
goto out;
if (!xa_is_node(entry)) {
if (xa_marked(xas->xa, mark))
return entry;
xas->xa_index = 1;
goto out;
}
xas->xa_node = xa_to_node(entry); xas->xa_offset = xas->xa_index >> xas->xa_node->shift;
}
while (xas->xa_index <= max) { if (unlikely(xas->xa_offset == XA_CHUNK_SIZE)) { xas->xa_offset = xas->xa_node->offset + 1; xas->xa_node = xa_parent(xas->xa, xas->xa_node);
if (!xas->xa_node)
break;
advance = false;
continue;
}
if (!advance) {
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); if (xa_is_sibling(entry)) { xas->xa_offset = xa_to_sibling(entry);
xas_move_index(xas, xas->xa_offset);
}
}
offset = xas_find_chunk(xas, advance, mark); if (offset > xas->xa_offset) {
advance = false;
xas_move_index(xas, offset);
/* Mind the wrap */
if ((xas->xa_index - 1) >= max)
goto max;
xas->xa_offset = offset;
if (offset == XA_CHUNK_SIZE)
continue;
}
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); if (!entry && !(xa_track_free(xas->xa) && mark == XA_FREE_MARK))
continue;
if (!xa_is_node(entry))
return entry;
xas->xa_node = xa_to_node(entry); xas_set_offset(xas);
}
out:
if (xas->xa_index > max)
goto max;
return set_bounds(xas);
max:
xas->xa_node = XAS_RESTART;
return NULL;
}
EXPORT_SYMBOL_GPL(xas_find_marked);
/**
* xas_find_conflict() - Find the next present entry in a range.
* @xas: XArray operation state.
*
* The @xas describes both a range and a position within that range.
*
* Context: Any context. Expects xa_lock to be held.
* Return: The next entry in the range covered by @xas or %NULL.
*/
void *xas_find_conflict(struct xa_state *xas)
{
void *curr;
if (xas_error(xas))
return NULL;
if (!xas->xa_node)
return NULL;
if (xas_top(xas->xa_node)) { curr = xas_start(xas);
if (!curr)
return NULL;
while (xa_is_node(curr)) { struct xa_node *node = xa_to_node(curr);
curr = xas_descend(xas, node);
}
if (curr)
return curr;
}
if (xas->xa_node->shift > xas->xa_shift)
return NULL;
for (;;) {
if (xas->xa_node->shift == xas->xa_shift) { if ((xas->xa_offset & xas->xa_sibs) == xas->xa_sibs)
break;
} else if (xas->xa_offset == XA_CHUNK_MASK) {
xas->xa_offset = xas->xa_node->offset;
xas->xa_node = xa_parent_locked(xas->xa, xas->xa_node);
if (!xas->xa_node)
break;
continue;
}
curr = xa_entry_locked(xas->xa, xas->xa_node, ++xas->xa_offset); if (xa_is_sibling(curr))
continue;
while (xa_is_node(curr)) { xas->xa_node = xa_to_node(curr);
xas->xa_offset = 0;
curr = xa_entry_locked(xas->xa, xas->xa_node, 0);
}
if (curr)
return curr;
}
xas->xa_offset -= xas->xa_sibs;
return NULL;
}
EXPORT_SYMBOL_GPL(xas_find_conflict);
/**
* xa_load() - Load an entry from an XArray.
* @xa: XArray.
* @index: index into array.
*
* Context: Any context. Takes and releases the RCU lock.
* Return: The entry at @index in @xa.
*/
void *xa_load(struct xarray *xa, unsigned long index)
{
XA_STATE(xas, xa, index);
void *entry;
rcu_read_lock();
do {
entry = xas_load(&xas);
if (xa_is_zero(entry))
entry = NULL;
} while (xas_retry(&xas, entry)); rcu_read_unlock();
return entry;
}
EXPORT_SYMBOL(xa_load);
static void *xas_result(struct xa_state *xas, void *curr)
{
if (xa_is_zero(curr))
return NULL;
if (xas_error(xas))
curr = xas->xa_node;
return curr;
}
/**
* __xa_erase() - Erase this entry from the XArray while locked.
* @xa: XArray.
* @index: Index into array.
*
* After this function returns, loading from @index will return %NULL.
* If the index is part of a multi-index entry, all indices will be erased
* and none of the entries will be part of a multi-index entry.
*
* Context: Any context. Expects xa_lock to be held on entry.
* Return: The entry which used to be at this index.
*/
void *__xa_erase(struct xarray *xa, unsigned long index)
{
XA_STATE(xas, xa, index); return xas_result(&xas, xas_store(&xas, NULL));
}
EXPORT_SYMBOL(__xa_erase);
/**
* xa_erase() - Erase this entry from the XArray.
* @xa: XArray.
* @index: Index of entry.
*
* After this function returns, loading from @index will return %NULL.
* If the index is part of a multi-index entry, all indices will be erased
* and none of the entries will be part of a multi-index entry.
*
* Context: Any context. Takes and releases the xa_lock.
* Return: The entry which used to be at this index.
*/
void *xa_erase(struct xarray *xa, unsigned long index)
{
void *entry;
xa_lock(xa);
entry = __xa_erase(xa, index);
xa_unlock(xa);
return entry;
}
EXPORT_SYMBOL(xa_erase);
/**
* __xa_store() - Store this entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* You must already be holding the xa_lock when calling this function.
* It will drop the lock if needed to allocate memory, and then reacquire
* it afterwards.
*
* Context: Any context. Expects xa_lock to be held on entry. May
* release and reacquire xa_lock if @gfp flags permit.
* Return: The old entry at this index or xa_err() if an error happened.
*/
void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
{
XA_STATE(xas, xa, index);
void *curr;
if (WARN_ON_ONCE(xa_is_advanced(entry)))
return XA_ERROR(-EINVAL);
if (xa_track_free(xa) && !entry)
entry = XA_ZERO_ENTRY;
do {
curr = xas_store(&xas, entry);
if (xa_track_free(xa))
xas_clear_mark(&xas, XA_FREE_MARK);
} while (__xas_nomem(&xas, gfp));
return xas_result(&xas, curr);
}
EXPORT_SYMBOL(__xa_store);
/**
* xa_store() - Store this entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* After this function returns, loads from this index will return @entry.
* Storing into an existing multi-index entry updates the entry of every index.
* The marks associated with @index are unaffected unless @entry is %NULL.
*
* Context: Any context. Takes and releases the xa_lock.
* May sleep if the @gfp flags permit.
* Return: The old entry at this index on success, xa_err(-EINVAL) if @entry
* cannot be stored in an XArray, or xa_err(-ENOMEM) if memory allocation
* failed.
*/
void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
{
void *curr;
xa_lock(xa);
curr = __xa_store(xa, index, entry, gfp);
xa_unlock(xa);
return curr;
}
EXPORT_SYMBOL(xa_store);
/**
* __xa_cmpxchg() - Store this entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @old: Old value to test against.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* You must already be holding the xa_lock when calling this function.
* It will drop the lock if needed to allocate memory, and then reacquire
* it afterwards.
*
* Context: Any context. Expects xa_lock to be held on entry. May
* release and reacquire xa_lock if @gfp flags permit.
* Return: The old entry at this index or xa_err() if an error happened.
*/
void *__xa_cmpxchg(struct xarray *xa, unsigned long index,
void *old, void *entry, gfp_t gfp)
{
XA_STATE(xas, xa, index);
void *curr;
if (WARN_ON_ONCE(xa_is_advanced(entry)))
return XA_ERROR(-EINVAL);
do {
curr = xas_load(&xas);
if (curr == old) {
xas_store(&xas, entry);
if (xa_track_free(xa) && entry && !curr)
xas_clear_mark(&xas, XA_FREE_MARK);
}
} while (__xas_nomem(&xas, gfp));
return xas_result(&xas, curr);
}
EXPORT_SYMBOL(__xa_cmpxchg);
/**
* __xa_insert() - Store this entry in the XArray if no entry is present.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* Inserting a NULL entry will store a reserved entry (like xa_reserve())
* if no entry is present. Inserting will fail if a reserved entry is
* present, even though loading from this index will return NULL.
*
* Context: Any context. Expects xa_lock to be held on entry. May
* release and reacquire xa_lock if @gfp flags permit.
* Return: 0 if the store succeeded. -EBUSY if another entry was present.
* -ENOMEM if memory could not be allocated.
*/
int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
{
XA_STATE(xas, xa, index);
void *curr;
if (WARN_ON_ONCE(xa_is_advanced(entry)))
return -EINVAL;
if (!entry)
entry = XA_ZERO_ENTRY;
do {
curr = xas_load(&xas);
if (!curr) {
xas_store(&xas, entry);
if (xa_track_free(xa))
xas_clear_mark(&xas, XA_FREE_MARK);
} else {
xas_set_err(&xas, -EBUSY);
}
} while (__xas_nomem(&xas, gfp));
return xas_error(&xas);
}
EXPORT_SYMBOL(__xa_insert);
#ifdef CONFIG_XARRAY_MULTI
static void xas_set_range(struct xa_state *xas, unsigned long first,
unsigned long last)
{
unsigned int shift = 0;
unsigned long sibs = last - first;
unsigned int offset = XA_CHUNK_MASK;
xas_set(xas, first);
while ((first & XA_CHUNK_MASK) == 0) {
if (sibs < XA_CHUNK_MASK)
break;
if ((sibs == XA_CHUNK_MASK) && (offset < XA_CHUNK_MASK))
break;
shift += XA_CHUNK_SHIFT;
if (offset == XA_CHUNK_MASK)
offset = sibs & XA_CHUNK_MASK;
sibs >>= XA_CHUNK_SHIFT;
first >>= XA_CHUNK_SHIFT;
}
offset = first & XA_CHUNK_MASK;
if (offset + sibs > XA_CHUNK_MASK)
sibs = XA_CHUNK_MASK - offset;
if ((((first + sibs + 1) << shift) - 1) > last)
sibs -= 1;
xas->xa_shift = shift;
xas->xa_sibs = sibs;
}
/**
* xa_store_range() - Store this entry at a range of indices in the XArray.
* @xa: XArray.
* @first: First index to affect.
* @last: Last index to affect.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* After this function returns, loads from any index between @first and @last,
* inclusive will return @entry.
* Storing into an existing multi-index entry updates the entry of every index.
* The marks associated with @index are unaffected unless @entry is %NULL.
*
* Context: Process context. Takes and releases the xa_lock. May sleep
* if the @gfp flags permit.
* Return: %NULL on success, xa_err(-EINVAL) if @entry cannot be stored in
* an XArray, or xa_err(-ENOMEM) if memory allocation failed.
*/
void *xa_store_range(struct xarray *xa, unsigned long first,
unsigned long last, void *entry, gfp_t gfp)
{
XA_STATE(xas, xa, 0);
if (WARN_ON_ONCE(xa_is_internal(entry)))
return XA_ERROR(-EINVAL);
if (last < first)
return XA_ERROR(-EINVAL);
do {
xas_lock(&xas);
if (entry) {
unsigned int order = BITS_PER_LONG;
if (last + 1)
order = __ffs(last + 1);
xas_set_order(&xas, last, order);
xas_create(&xas, true);
if (xas_error(&xas))
goto unlock;
}
do {
xas_set_range(&xas, first, last);
xas_store(&xas, entry);
if (xas_error(&xas))
goto unlock;
first += xas_size(&xas);
} while (first <= last);
unlock:
xas_unlock(&xas);
} while (xas_nomem(&xas, gfp));
return xas_result(&xas, NULL);
}
EXPORT_SYMBOL(xa_store_range);
/**
* xa_get_order() - Get the order of an entry.
* @xa: XArray.
* @index: Index of the entry.
*
* Return: A number between 0 and 63 indicating the order of the entry.
*/
int xa_get_order(struct xarray *xa, unsigned long index)
{
XA_STATE(xas, xa, index);
void *entry;
int order = 0;
rcu_read_lock();
entry = xas_load(&xas);
if (!entry)
goto unlock;
if (!xas.xa_node)
goto unlock;
for (;;) {
unsigned int slot = xas.xa_offset + (1 << order);
if (slot >= XA_CHUNK_SIZE)
break;
if (!xa_is_sibling(xas.xa_node->slots[slot]))
break;
order++;
}
order += xas.xa_node->shift;
unlock:
rcu_read_unlock();
return order;
}
EXPORT_SYMBOL(xa_get_order);
#endif /* CONFIG_XARRAY_MULTI */
/**
* __xa_alloc() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @limit: Range for allocated ID.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Any context. Expects xa_lock to be held on entry. May
* release and reacquire xa_lock if @gfp flags permit.
* Return: 0 on success, -ENOMEM if memory could not be allocated or
* -EBUSY if there are no free entries in @limit.
*/
int __xa_alloc(struct xarray *xa, u32 *id, void *entry,
struct xa_limit limit, gfp_t gfp)
{
XA_STATE(xas, xa, 0);
if (WARN_ON_ONCE(xa_is_advanced(entry)))
return -EINVAL;
if (WARN_ON_ONCE(!xa_track_free(xa)))
return -EINVAL;
if (!entry)
entry = XA_ZERO_ENTRY;
do {
xas.xa_index = limit.min;
xas_find_marked(&xas, limit.max, XA_FREE_MARK);
if (xas.xa_node == XAS_RESTART)
xas_set_err(&xas, -EBUSY);
else
*id = xas.xa_index;
xas_store(&xas, entry);
xas_clear_mark(&xas, XA_FREE_MARK);
} while (__xas_nomem(&xas, gfp));
return xas_error(&xas);
}
EXPORT_SYMBOL(__xa_alloc);
/**
* __xa_alloc_cyclic() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of allocated ID.
* @next: Pointer to next ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
* The search for an empty entry will start at @next and will wrap
* around if necessary.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Any context. Expects xa_lock to be held on entry. May
* release and reacquire xa_lock if @gfp flags permit.
* Return: 0 if the allocation succeeded without wrapping. 1 if the
* allocation succeeded after wrapping, -ENOMEM if memory could not be
* allocated or -EBUSY if there are no free entries in @limit.
*/
int __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
struct xa_limit limit, u32 *next, gfp_t gfp)
{
u32 min = limit.min;
int ret;
limit.min = max(min, *next);
ret = __xa_alloc(xa, id, entry, limit, gfp);
if ((xa->xa_flags & XA_FLAGS_ALLOC_WRAPPED) && ret == 0) {
xa->xa_flags &= ~XA_FLAGS_ALLOC_WRAPPED;
ret = 1;
}
if (ret < 0 && limit.min > min) {
limit.min = min;
ret = __xa_alloc(xa, id, entry, limit, gfp);
if (ret == 0)
ret = 1;
}
if (ret >= 0) {
*next = *id + 1;
if (*next == 0)
xa->xa_flags |= XA_FLAGS_ALLOC_WRAPPED;
}
return ret;
}
EXPORT_SYMBOL(__xa_alloc_cyclic);
/**
* __xa_set_mark() - Set this mark on this entry while locked.
* @xa: XArray.
* @index: Index of entry.
* @mark: Mark number.
*
* Attempting to set a mark on a %NULL entry does not succeed.
*
* Context: Any context. Expects xa_lock to be held on entry.
*/
void __xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
{
XA_STATE(xas, xa, index);
void *entry = xas_load(&xas);
if (entry)
xas_set_mark(&xas, mark);
}
EXPORT_SYMBOL(__xa_set_mark);
/**
* __xa_clear_mark() - Clear this mark on this entry while locked.
* @xa: XArray.
* @index: Index of entry.
* @mark: Mark number.
*
* Context: Any context. Expects xa_lock to be held on entry.
*/
void __xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
{
XA_STATE(xas, xa, index);
void *entry = xas_load(&xas);
if (entry)
xas_clear_mark(&xas, mark);
}
EXPORT_SYMBOL(__xa_clear_mark);
/**
* xa_get_mark() - Inquire whether this mark is set on this entry.
* @xa: XArray.
* @index: Index of entry.
* @mark: Mark number.
*
* This function uses the RCU read lock, so the result may be out of date
* by the time it returns. If you need the result to be stable, use a lock.
*
* Context: Any context. Takes and releases the RCU lock.
* Return: True if the entry at @index has this mark set, false if it doesn't.
*/
bool xa_get_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
{
XA_STATE(xas, xa, index);
void *entry;
rcu_read_lock();
entry = xas_start(&xas);
while (xas_get_mark(&xas, mark)) {
if (!xa_is_node(entry))
goto found;
entry = xas_descend(&xas, xa_to_node(entry));
}
rcu_read_unlock();
return false;
found:
rcu_read_unlock();
return true;
}
EXPORT_SYMBOL(xa_get_mark);
/**
* xa_set_mark() - Set this mark on this entry.
* @xa: XArray.
* @index: Index of entry.
* @mark: Mark number.
*
* Attempting to set a mark on a %NULL entry does not succeed.
*
* Context: Process context. Takes and releases the xa_lock.
*/
void xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
{
xa_lock(xa);
__xa_set_mark(xa, index, mark);
xa_unlock(xa);
}
EXPORT_SYMBOL(xa_set_mark);
/**
* xa_clear_mark() - Clear this mark on this entry.
* @xa: XArray.
* @index: Index of entry.
* @mark: Mark number.
*
* Clearing a mark always succeeds.
*
* Context: Process context. Takes and releases the xa_lock.
*/
void xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
{
xa_lock(xa);
__xa_clear_mark(xa, index, mark);
xa_unlock(xa);
}
EXPORT_SYMBOL(xa_clear_mark);
/**
* xa_find() - Search the XArray for an entry.
* @xa: XArray.
* @indexp: Pointer to an index.
* @max: Maximum index to search to.
* @filter: Selection criterion.
*
* Finds the entry in @xa which matches the @filter, and has the lowest
* index that is at least @indexp and no more than @max.
* If an entry is found, @indexp is updated to be the index of the entry.
* This function is protected by the RCU read lock, so it may not find
* entries which are being simultaneously added. It will not return an
* %XA_RETRY_ENTRY; if you need to see retry entries, use xas_find().
*
* Context: Any context. Takes and releases the RCU lock.
* Return: The entry, if found, otherwise %NULL.
*/
void *xa_find(struct xarray *xa, unsigned long *indexp,
unsigned long max, xa_mark_t filter)
{
XA_STATE(xas, xa, *indexp);
void *entry;
rcu_read_lock();
do {
if ((__force unsigned int)filter < XA_MAX_MARKS)
entry = xas_find_marked(&xas, max, filter);
else
entry = xas_find(&xas, max);
} while (xas_retry(&xas, entry));
rcu_read_unlock();
if (entry)
*indexp = xas.xa_index;
return entry;
}
EXPORT_SYMBOL(xa_find);
static bool xas_sibling(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;
unsigned long mask;
if (!IS_ENABLED(CONFIG_XARRAY_MULTI) || !node)
return false;
mask = (XA_CHUNK_SIZE << node->shift) - 1;
return (xas->xa_index & mask) >
((unsigned long)xas->xa_offset << node->shift);
}
/**
* xa_find_after() - Search the XArray for a present entry.
* @xa: XArray.
* @indexp: Pointer to an index.
* @max: Maximum index to search to.
* @filter: Selection criterion.
*
* Finds the entry in @xa which matches the @filter and has the lowest
* index that is above @indexp and no more than @max.
* If an entry is found, @indexp is updated to be the index of the entry.
* This function is protected by the RCU read lock, so it may miss entries
* which are being simultaneously added. It will not return an
* %XA_RETRY_ENTRY; if you need to see retry entries, use xas_find().
*
* Context: Any context. Takes and releases the RCU lock.
* Return: The pointer, if found, otherwise %NULL.
*/
void *xa_find_after(struct xarray *xa, unsigned long *indexp,
unsigned long max, xa_mark_t filter)
{
XA_STATE(xas, xa, *indexp + 1);
void *entry;
if (xas.xa_index == 0)
return NULL;
rcu_read_lock();
for (;;) {
if ((__force unsigned int)filter < XA_MAX_MARKS)
entry = xas_find_marked(&xas, max, filter);
else
entry = xas_find(&xas, max);
if (xas_invalid(&xas))
break;
if (xas_sibling(&xas))
continue;
if (!xas_retry(&xas, entry))
break;
}
rcu_read_unlock();
if (entry)
*indexp = xas.xa_index;
return entry;
}
EXPORT_SYMBOL(xa_find_after);
static unsigned int xas_extract_present(struct xa_state *xas, void **dst,
unsigned long max, unsigned int n)
{
void *entry;
unsigned int i = 0;
rcu_read_lock();
xas_for_each(xas, entry, max) {
if (xas_retry(xas, entry))
continue;
dst[i++] = entry;
if (i == n)
break;
}
rcu_read_unlock();
return i;
}
static unsigned int xas_extract_marked(struct xa_state *xas, void **dst,
unsigned long max, unsigned int n, xa_mark_t mark)
{
void *entry;
unsigned int i = 0;
rcu_read_lock();
xas_for_each_marked(xas, entry, max, mark) {
if (xas_retry(xas, entry))
continue;
dst[i++] = entry;
if (i == n)
break;
}
rcu_read_unlock();
return i;
}
/**
* xa_extract() - Copy selected entries from the XArray into a normal array.
* @xa: The source XArray to copy from.
* @dst: The buffer to copy entries into.
* @start: The first index in the XArray eligible to be selected.
* @max: The last index in the XArray eligible to be selected.
* @n: The maximum number of entries to copy.
* @filter: Selection criterion.
*
* Copies up to @n entries that match @filter from the XArray. The
* copied entries will have indices between @start and @max, inclusive.
*
* The @filter may be an XArray mark value, in which case entries which are
* marked with that mark will be copied. It may also be %XA_PRESENT, in
* which case all entries which are not %NULL will be copied.
*
* The entries returned may not represent a snapshot of the XArray at a
* moment in time. For example, if another thread stores to index 5, then
* index 10, calling xa_extract() may return the old contents of index 5
* and the new contents of index 10. Indices not modified while this
* function is running will not be skipped.
*
* If you need stronger guarantees, holding the xa_lock across calls to this
* function will prevent concurrent modification.
*
* Context: Any context. Takes and releases the RCU lock.
* Return: The number of entries copied.
*/
unsigned int xa_extract(struct xarray *xa, void **dst, unsigned long start,
unsigned long max, unsigned int n, xa_mark_t filter)
{
XA_STATE(xas, xa, start);
if (!n)
return 0;
if ((__force unsigned int)filter < XA_MAX_MARKS)
return xas_extract_marked(&xas, dst, max, n, filter);
return xas_extract_present(&xas, dst, max, n);
}
EXPORT_SYMBOL(xa_extract);
/**
* xa_delete_node() - Private interface for workingset code.
* @node: Node to be removed from the tree.
* @update: Function to call to update ancestor nodes.
*
* Context: xa_lock must be held on entry and will not be released.
*/
void xa_delete_node(struct xa_node *node, xa_update_node_t update)
{
struct xa_state xas = {
.xa = node->array,
.xa_index = (unsigned long)node->offset <<
(node->shift + XA_CHUNK_SHIFT),
.xa_shift = node->shift + XA_CHUNK_SHIFT,
.xa_offset = node->offset,
.xa_node = xa_parent_locked(node->array, node),
.xa_update = update,
};
xas_store(&xas, NULL);
}
EXPORT_SYMBOL_GPL(xa_delete_node); /* For the benefit of the test suite */
/**
* xa_destroy() - Free all internal data structures.
* @xa: XArray.
*
* After calling this function, the XArray is empty and has freed all memory
* allocated for its internal data structures. You are responsible for
* freeing the objects referenced by the XArray.
*
* Context: Any context. Takes and releases the xa_lock, interrupt-safe.
*/
void xa_destroy(struct xarray *xa)
{
XA_STATE(xas, xa, 0);
unsigned long flags;
void *entry;
xas.xa_node = NULL;
xas_lock_irqsave(&xas, flags);
entry = xa_head_locked(xa);
RCU_INIT_POINTER(xa->xa_head, NULL);
xas_init_marks(&xas); if (xa_zero_busy(xa)) xa_mark_clear(xa, XA_FREE_MARK);
/* lockdep checks we're still holding the lock in xas_free_nodes() */
if (xa_is_node(entry)) xas_free_nodes(&xas, xa_to_node(entry)); xas_unlock_irqrestore(&xas, flags);
}
EXPORT_SYMBOL(xa_destroy);
#ifdef XA_DEBUG
void xa_dump_node(const struct xa_node *node)
{
unsigned i, j;
if (!node)
return;
if ((unsigned long)node & 3) {
pr_cont("node %px\n", node);
return;
}
pr_cont("node %px %s %d parent %px shift %d count %d values %d "
"array %px list %px %px marks",
node, node->parent ? "offset" : "max", node->offset,
node->parent, node->shift, node->count, node->nr_values,
node->array, node->private_list.prev, node->private_list.next);
for (i = 0; i < XA_MAX_MARKS; i++)
for (j = 0; j < XA_MARK_LONGS; j++)
pr_cont(" %lx", node->marks[i][j]);
pr_cont("\n");
}
void xa_dump_index(unsigned long index, unsigned int shift)
{
if (!shift)
pr_info("%lu: ", index);
else if (shift >= BITS_PER_LONG)
pr_info("0-%lu: ", ~0UL);
else
pr_info("%lu-%lu: ", index, index | ((1UL << shift) - 1));
}
void xa_dump_entry(const void *entry, unsigned long index, unsigned long shift)
{
if (!entry)
return;
xa_dump_index(index, shift);
if (xa_is_node(entry)) {
if (shift == 0) {
pr_cont("%px\n", entry);
} else {
unsigned long i;
struct xa_node *node = xa_to_node(entry);
xa_dump_node(node);
for (i = 0; i < XA_CHUNK_SIZE; i++)
xa_dump_entry(node->slots[i],
index + (i << node->shift), node->shift);
}
} else if (xa_is_value(entry))
pr_cont("value %ld (0x%lx) [%px]\n", xa_to_value(entry),
xa_to_value(entry), entry);
else if (!xa_is_internal(entry))
pr_cont("%px\n", entry);
else if (xa_is_retry(entry))
pr_cont("retry (%ld)\n", xa_to_internal(entry));
else if (xa_is_sibling(entry))
pr_cont("sibling (slot %ld)\n", xa_to_sibling(entry));
else if (xa_is_zero(entry))
pr_cont("zero (%ld)\n", xa_to_internal(entry));
else
pr_cont("UNKNOWN ENTRY (%px)\n", entry);
}
void xa_dump(const struct xarray *xa)
{
void *entry = xa->xa_head;
unsigned int shift = 0;
pr_info("xarray: %px head %px flags %x marks %d %d %d\n", xa, entry,
xa->xa_flags, xa_marked(xa, XA_MARK_0),
xa_marked(xa, XA_MARK_1), xa_marked(xa, XA_MARK_2));
if (xa_is_node(entry))
shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT;
xa_dump_entry(entry, 0, shift);
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_DCACHE_H
#define __LINUX_DCACHE_H
#include <linux/atomic.h>
#include <linux/list.h>
#include <linux/math.h>
#include <linux/rculist.h>
#include <linux/rculist_bl.h>
#include <linux/spinlock.h>
#include <linux/seqlock.h>
#include <linux/cache.h>
#include <linux/rcupdate.h>
#include <linux/lockref.h>
#include <linux/stringhash.h>
#include <linux/wait.h>
struct path;
struct file;
struct vfsmount;
/*
* linux/include/linux/dcache.h
*
* Dirent cache data structures
*
* (C) Copyright 1997 Thomas Schoebel-Theuer,
* with heavy changes by Linus Torvalds
*/
#define IS_ROOT(x) ((x) == (x)->d_parent)
/* The hash is always the low bits of hash_len */
#ifdef __LITTLE_ENDIAN
#define HASH_LEN_DECLARE u32 hash; u32 len
#define bytemask_from_count(cnt) (~(~0ul << (cnt)*8))
#else
#define HASH_LEN_DECLARE u32 len; u32 hash
#define bytemask_from_count(cnt) (~(~0ul >> (cnt)*8))
#endif
/*
* "quick string" -- eases parameter passing, but more importantly
* saves "metadata" about the string (ie length and the hash).
*
* hash comes first so it snuggles against d_parent in the
* dentry.
*/
struct qstr {
union {
struct {
HASH_LEN_DECLARE;
};
u64 hash_len;
};
const unsigned char *name;
};
#define QSTR_INIT(n,l) { { { .len = l } }, .name = n }
extern const struct qstr empty_name;
extern const struct qstr slash_name;
extern const struct qstr dotdot_name;
/*
* Try to keep struct dentry aligned on 64 byte cachelines (this will
* give reasonable cacheline footprint with larger lines without the
* large memory footprint increase).
*/
#ifdef CONFIG_64BIT
# define DNAME_INLINE_LEN 40 /* 192 bytes */
#else
# ifdef CONFIG_SMP
# define DNAME_INLINE_LEN 40 /* 128 bytes */
# else
# define DNAME_INLINE_LEN 44 /* 128 bytes */
# endif
#endif
#define d_lock d_lockref.lock
struct dentry {
/* RCU lookup touched fields */
unsigned int d_flags; /* protected by d_lock */
seqcount_spinlock_t d_seq; /* per dentry seqlock */
struct hlist_bl_node d_hash; /* lookup hash list */
struct dentry *d_parent; /* parent directory */
struct qstr d_name;
struct inode *d_inode; /* Where the name belongs to - NULL is
* negative */
unsigned char d_iname[DNAME_INLINE_LEN]; /* small names */
/* Ref lookup also touches following */
struct lockref d_lockref; /* per-dentry lock and refcount */
const struct dentry_operations *d_op;
struct super_block *d_sb; /* The root of the dentry tree */
unsigned long d_time; /* used by d_revalidate */
void *d_fsdata; /* fs-specific data */
union {
struct list_head d_lru; /* LRU list */
wait_queue_head_t *d_wait; /* in-lookup ones only */
};
struct hlist_node d_sib; /* child of parent list */
struct hlist_head d_children; /* our children */
/*
* d_alias and d_rcu can share memory
*/
union {
struct hlist_node d_alias; /* inode alias list */
struct hlist_bl_node d_in_lookup_hash; /* only for in-lookup ones */
struct rcu_head d_rcu;
} d_u;
};
/*
* dentry->d_lock spinlock nesting subclasses:
*
* 0: normal
* 1: nested
*/
enum dentry_d_lock_class
{
DENTRY_D_LOCK_NORMAL, /* implicitly used by plain spin_lock() APIs. */
DENTRY_D_LOCK_NESTED
};
enum d_real_type {
D_REAL_DATA,
D_REAL_METADATA,
};
struct dentry_operations {
int (*d_revalidate)(struct dentry *, unsigned int);
int (*d_weak_revalidate)(struct dentry *, unsigned int);
int (*d_hash)(const struct dentry *, struct qstr *);
int (*d_compare)(const struct dentry *,
unsigned int, const char *, const struct qstr *);
int (*d_delete)(const struct dentry *);
int (*d_init)(struct dentry *);
void (*d_release)(struct dentry *);
void (*d_prune)(struct dentry *);
void (*d_iput)(struct dentry *, struct inode *);
char *(*d_dname)(struct dentry *, char *, int);
struct vfsmount *(*d_automount)(struct path *);
int (*d_manage)(const struct path *, bool);
struct dentry *(*d_real)(struct dentry *, enum d_real_type type);
} ____cacheline_aligned;
/*
* Locking rules for dentry_operations callbacks are to be found in
* Documentation/filesystems/locking.rst. Keep it updated!
*
* FUrther descriptions are found in Documentation/filesystems/vfs.rst.
* Keep it updated too!
*/
/* d_flags entries */
#define DCACHE_OP_HASH BIT(0)
#define DCACHE_OP_COMPARE BIT(1)
#define DCACHE_OP_REVALIDATE BIT(2)
#define DCACHE_OP_DELETE BIT(3)
#define DCACHE_OP_PRUNE BIT(4)
#define DCACHE_DISCONNECTED BIT(5)
/* This dentry is possibly not currently connected to the dcache tree, in
* which case its parent will either be itself, or will have this flag as
* well. nfsd will not use a dentry with this bit set, but will first
* endeavour to clear the bit either by discovering that it is connected,
* or by performing lookup operations. Any filesystem which supports
* nfsd_operations MUST have a lookup function which, if it finds a
* directory inode with a DCACHE_DISCONNECTED dentry, will d_move that
* dentry into place and return that dentry rather than the passed one,
* typically using d_splice_alias. */
#define DCACHE_REFERENCED BIT(6) /* Recently used, don't discard. */
#define DCACHE_DONTCACHE BIT(7) /* Purge from memory on final dput() */
#define DCACHE_CANT_MOUNT BIT(8)
#define DCACHE_GENOCIDE BIT(9)
#define DCACHE_SHRINK_LIST BIT(10)
#define DCACHE_OP_WEAK_REVALIDATE BIT(11)
#define DCACHE_NFSFS_RENAMED BIT(12)
/* this dentry has been "silly renamed" and has to be deleted on the last
* dput() */
#define DCACHE_FSNOTIFY_PARENT_WATCHED BIT(14)
/* Parent inode is watched by some fsnotify listener */
#define DCACHE_DENTRY_KILLED BIT(15)
#define DCACHE_MOUNTED BIT(16) /* is a mountpoint */
#define DCACHE_NEED_AUTOMOUNT BIT(17) /* handle automount on this dir */
#define DCACHE_MANAGE_TRANSIT BIT(18) /* manage transit from this dirent */
#define DCACHE_MANAGED_DENTRY \
(DCACHE_MOUNTED|DCACHE_NEED_AUTOMOUNT|DCACHE_MANAGE_TRANSIT)
#define DCACHE_LRU_LIST BIT(19)
#define DCACHE_ENTRY_TYPE (7 << 20) /* bits 20..22 are for storing type: */
#define DCACHE_MISS_TYPE (0 << 20) /* Negative dentry */
#define DCACHE_WHITEOUT_TYPE (1 << 20) /* Whiteout dentry (stop pathwalk) */
#define DCACHE_DIRECTORY_TYPE (2 << 20) /* Normal directory */
#define DCACHE_AUTODIR_TYPE (3 << 20) /* Lookupless directory (presumed automount) */
#define DCACHE_REGULAR_TYPE (4 << 20) /* Regular file type */
#define DCACHE_SPECIAL_TYPE (5 << 20) /* Other file type */
#define DCACHE_SYMLINK_TYPE (6 << 20) /* Symlink */
#define DCACHE_NOKEY_NAME BIT(25) /* Encrypted name encoded without key */
#define DCACHE_OP_REAL BIT(26)
#define DCACHE_PAR_LOOKUP BIT(28) /* being looked up (with parent locked shared) */
#define DCACHE_DENTRY_CURSOR BIT(29)
#define DCACHE_NORCU BIT(30) /* No RCU delay for freeing */
extern seqlock_t rename_lock;
/*
* These are the low-level FS interfaces to the dcache..
*/
extern void d_instantiate(struct dentry *, struct inode *);
extern void d_instantiate_new(struct dentry *, struct inode *);
extern void __d_drop(struct dentry *dentry);
extern void d_drop(struct dentry *dentry);
extern void d_delete(struct dentry *);
extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op);
/* allocate/de-allocate */
extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
extern struct dentry * d_alloc_anon(struct super_block *);
extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *,
wait_queue_head_t *);
extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *);
extern bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
const struct qstr *name);
extern struct dentry * d_exact_alias(struct dentry *, struct inode *);
extern struct dentry *d_find_any_alias(struct inode *inode);
extern struct dentry * d_obtain_alias(struct inode *);
extern struct dentry * d_obtain_root(struct inode *);
extern void shrink_dcache_sb(struct super_block *);
extern void shrink_dcache_parent(struct dentry *);
extern void d_invalidate(struct dentry *);
/* only used at mount-time */
extern struct dentry * d_make_root(struct inode *);
extern void d_mark_tmpfile(struct file *, struct inode *);
extern void d_tmpfile(struct file *, struct inode *);
extern struct dentry *d_find_alias(struct inode *);
extern void d_prune_aliases(struct inode *);
extern struct dentry *d_find_alias_rcu(struct inode *);
/* test whether we have any submounts in a subdir tree */
extern int path_has_submounts(const struct path *);
/*
* This adds the entry to the hash queues.
*/
extern void d_rehash(struct dentry *);
extern void d_add(struct dentry *, struct inode *);
/* used for rename() and baskets */
extern void d_move(struct dentry *, struct dentry *);
extern void d_exchange(struct dentry *, struct dentry *);
extern struct dentry *d_ancestor(struct dentry *, struct dentry *);
extern struct dentry *d_lookup(const struct dentry *, const struct qstr *);
extern struct dentry *d_hash_and_lookup(struct dentry *, struct qstr *);
static inline unsigned d_count(const struct dentry *dentry)
{
return dentry->d_lockref.count;
}
/*
* helper function for dentry_operations.d_dname() members
*/
extern __printf(3, 4)
char *dynamic_dname(char *, int, const char *, ...);
extern char *__d_path(const struct path *, const struct path *, char *, int);
extern char *d_absolute_path(const struct path *, char *, int);
extern char *d_path(const struct path *, char *, int);
extern char *dentry_path_raw(const struct dentry *, char *, int);
extern char *dentry_path(const struct dentry *, char *, int);
/* Allocation counts.. */
/**
* dget_dlock - get a reference to a dentry
* @dentry: dentry to get a reference to
*
* Given a live dentry, increment the reference count and return the dentry.
* Caller must hold @dentry->d_lock. Making sure that dentry is alive is
* caller's resonsibility. There are many conditions sufficient to guarantee
* that; e.g. anything with non-negative refcount is alive, so's anything
* hashed, anything positive, anyone's parent, etc.
*/
static inline struct dentry *dget_dlock(struct dentry *dentry)
{
dentry->d_lockref.count++;
return dentry;
}
/**
* dget - get a reference to a dentry
* @dentry: dentry to get a reference to
*
* Given a dentry or %NULL pointer increment the reference count
* if appropriate and return the dentry. A dentry will not be
* destroyed when it has references. Conversely, a dentry with
* no references can disappear for any number of reasons, starting
* with memory pressure. In other words, that primitive is
* used to clone an existing reference; using it on something with
* zero refcount is a bug.
*
* NOTE: it will spin if @dentry->d_lock is held. From the deadlock
* avoidance point of view it is equivalent to spin_lock()/increment
* refcount/spin_unlock(), so calling it under @dentry->d_lock is
* always a bug; so's calling it under ->d_lock on any of its descendents.
*
*/
static inline struct dentry *dget(struct dentry *dentry)
{
if (dentry) lockref_get(&dentry->d_lockref);
return dentry;
}
extern struct dentry *dget_parent(struct dentry *dentry);
/**
* d_unhashed - is dentry hashed
* @dentry: entry to check
*
* Returns true if the dentry passed is not currently hashed.
*/
static inline int d_unhashed(const struct dentry *dentry)
{
return hlist_bl_unhashed(&dentry->d_hash);
}
static inline int d_unlinked(const struct dentry *dentry)
{
return d_unhashed(dentry) && !IS_ROOT(dentry);
}
static inline int cant_mount(const struct dentry *dentry)
{
return (dentry->d_flags & DCACHE_CANT_MOUNT);
}
static inline void dont_mount(struct dentry *dentry)
{
spin_lock(&dentry->d_lock);
dentry->d_flags |= DCACHE_CANT_MOUNT;
spin_unlock(&dentry->d_lock);
}
extern void __d_lookup_unhash_wake(struct dentry *dentry);
static inline int d_in_lookup(const struct dentry *dentry)
{
return dentry->d_flags & DCACHE_PAR_LOOKUP;
}
static inline void d_lookup_done(struct dentry *dentry)
{
if (unlikely(d_in_lookup(dentry)))
__d_lookup_unhash_wake(dentry);
}
extern void dput(struct dentry *);
static inline bool d_managed(const struct dentry *dentry)
{
return dentry->d_flags & DCACHE_MANAGED_DENTRY;
}
static inline bool d_mountpoint(const struct dentry *dentry)
{
return dentry->d_flags & DCACHE_MOUNTED;
}
/*
* Directory cache entry type accessor functions.
*/
static inline unsigned __d_entry_type(const struct dentry *dentry)
{
return dentry->d_flags & DCACHE_ENTRY_TYPE;
}
static inline bool d_is_miss(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_MISS_TYPE;
}
static inline bool d_is_whiteout(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_WHITEOUT_TYPE;
}
static inline bool d_can_lookup(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_DIRECTORY_TYPE;
}
static inline bool d_is_autodir(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_AUTODIR_TYPE;
}
static inline bool d_is_dir(const struct dentry *dentry)
{
return d_can_lookup(dentry) || d_is_autodir(dentry);
}
static inline bool d_is_symlink(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_SYMLINK_TYPE;
}
static inline bool d_is_reg(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_REGULAR_TYPE;
}
static inline bool d_is_special(const struct dentry *dentry)
{
return __d_entry_type(dentry) == DCACHE_SPECIAL_TYPE;
}
static inline bool d_is_file(const struct dentry *dentry)
{
return d_is_reg(dentry) || d_is_special(dentry);
}
static inline bool d_is_negative(const struct dentry *dentry)
{
// TODO: check d_is_whiteout(dentry) also.
return d_is_miss(dentry);
}
static inline bool d_flags_negative(unsigned flags)
{
return (flags & DCACHE_ENTRY_TYPE) == DCACHE_MISS_TYPE;
}
static inline bool d_is_positive(const struct dentry *dentry)
{
return !d_is_negative(dentry);
}
/**
* d_really_is_negative - Determine if a dentry is really negative (ignoring fallthroughs)
* @dentry: The dentry in question
*
* Returns true if the dentry represents either an absent name or a name that
* doesn't map to an inode (ie. ->d_inode is NULL). The dentry could represent
* a true miss, a whiteout that isn't represented by a 0,0 chardev or a
* fallthrough marker in an opaque directory.
*
* Note! (1) This should be used *only* by a filesystem to examine its own
* dentries. It should not be used to look at some other filesystem's
* dentries. (2) It should also be used in combination with d_inode() to get
* the inode. (3) The dentry may have something attached to ->d_lower and the
* type field of the flags may be set to something other than miss or whiteout.
*/
static inline bool d_really_is_negative(const struct dentry *dentry)
{
return dentry->d_inode == NULL;
}
/**
* d_really_is_positive - Determine if a dentry is really positive (ignoring fallthroughs)
* @dentry: The dentry in question
*
* Returns true if the dentry represents a name that maps to an inode
* (ie. ->d_inode is not NULL). The dentry might still represent a whiteout if
* that is represented on medium as a 0,0 chardev.
*
* Note! (1) This should be used *only* by a filesystem to examine its own
* dentries. It should not be used to look at some other filesystem's
* dentries. (2) It should also be used in combination with d_inode() to get
* the inode.
*/
static inline bool d_really_is_positive(const struct dentry *dentry)
{
return dentry->d_inode != NULL;
}
static inline int simple_positive(const struct dentry *dentry)
{
return d_really_is_positive(dentry) && !d_unhashed(dentry);
}
extern int sysctl_vfs_cache_pressure;
static inline unsigned long vfs_pressure_ratio(unsigned long val)
{
return mult_frac(val, sysctl_vfs_cache_pressure, 100);
}
/**
* d_inode - Get the actual inode of this dentry
* @dentry: The dentry to query
*
* This is the helper normal filesystems should use to get at their own inodes
* in their own dentries and ignore the layering superimposed upon them.
*/
static inline struct inode *d_inode(const struct dentry *dentry)
{
return dentry->d_inode;
}
/**
* d_inode_rcu - Get the actual inode of this dentry with READ_ONCE()
* @dentry: The dentry to query
*
* This is the helper normal filesystems should use to get at their own inodes
* in their own dentries and ignore the layering superimposed upon them.
*/
static inline struct inode *d_inode_rcu(const struct dentry *dentry)
{
return READ_ONCE(dentry->d_inode);
}
/**
* d_backing_inode - Get upper or lower inode we should be using
* @upper: The upper layer
*
* This is the helper that should be used to get at the inode that will be used
* if this dentry were to be opened as a file. The inode may be on the upper
* dentry or it may be on a lower dentry pinned by the upper.
*
* Normal filesystems should not use this to access their own inodes.
*/
static inline struct inode *d_backing_inode(const struct dentry *upper)
{
struct inode *inode = upper->d_inode;
return inode;
}
/**
* d_real - Return the real dentry
* @dentry: the dentry to query
* @type: the type of real dentry (data or metadata)
*
* If dentry is on a union/overlay, then return the underlying, real dentry.
* Otherwise return the dentry itself.
*
* See also: Documentation/filesystems/vfs.rst
*/
static inline struct dentry *d_real(struct dentry *dentry, enum d_real_type type)
{
if (unlikely(dentry->d_flags & DCACHE_OP_REAL)) return dentry->d_op->d_real(dentry, type);
else
return dentry;
}
/**
* d_real_inode - Return the real inode hosting the data
* @dentry: The dentry to query
*
* If dentry is on a union/overlay, then return the underlying, real inode.
* Otherwise return d_inode().
*/
static inline struct inode *d_real_inode(const struct dentry *dentry)
{
/* This usage of d_real() results in const dentry */
return d_inode(d_real((struct dentry *) dentry, D_REAL_DATA));
}
struct name_snapshot {
struct qstr name;
unsigned char inline_name[DNAME_INLINE_LEN];
};
void take_dentry_name_snapshot(struct name_snapshot *, struct dentry *);
void release_dentry_name_snapshot(struct name_snapshot *);
static inline struct dentry *d_first_child(const struct dentry *dentry)
{
return hlist_entry_safe(dentry->d_children.first, struct dentry, d_sib);
}
static inline struct dentry *d_next_sibling(const struct dentry *dentry)
{
return hlist_entry_safe(dentry->d_sib.next, struct dentry, d_sib);
}
#endif /* __LINUX_DCACHE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MM_TYPES_H
#define _LINUX_MM_TYPES_H
#include <linux/mm_types_task.h>
#include <linux/auxvec.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/rbtree.h>
#include <linux/maple_tree.h>
#include <linux/rwsem.h>
#include <linux/completion.h>
#include <linux/cpumask.h>
#include <linux/uprobes.h>
#include <linux/rcupdate.h>
#include <linux/page-flags-layout.h>
#include <linux/workqueue.h>
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <asm/mmu.h>
#ifndef AT_VECTOR_SIZE_ARCH
#define AT_VECTOR_SIZE_ARCH 0
#endif
#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1))
#define INIT_PASID 0
struct address_space;
struct mem_cgroup;
/*
* Each physical page in the system has a struct page associated with
* it to keep track of whatever it is we are using the page for at the
* moment. Note that we have no way to track which tasks are using
* a page, though if it is a pagecache page, rmap structures can tell us
* who is mapping it.
*
* If you allocate the page using alloc_pages(), you can use some of the
* space in struct page for your own purposes. The five words in the main
* union are available, except for bit 0 of the first word which must be
* kept clear. Many users use this word to store a pointer to an object
* which is guaranteed to be aligned. If you use the same storage as
* page->mapping, you must restore it to NULL before freeing the page.
*
* If your page will not be mapped to userspace, you can also use the four
* bytes in the mapcount union, but you must call page_mapcount_reset()
* before freeing it.
*
* If you want to use the refcount field, it must be used in such a way
* that other CPUs temporarily incrementing and then decrementing the
* refcount does not cause problems. On receiving the page from
* alloc_pages(), the refcount will be positive.
*
* If you allocate pages of order > 0, you can use some of the fields
* in each subpage, but you may need to restore some of their values
* afterwards.
*
* SLUB uses cmpxchg_double() to atomically update its freelist and counters.
* That requires that freelist & counters in struct slab be adjacent and
* double-word aligned. Because struct slab currently just reinterprets the
* bits of struct page, we align all struct pages to double-word boundaries,
* and ensure that 'freelist' is aligned within struct slab.
*/
#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE
#define _struct_page_alignment __aligned(2 * sizeof(unsigned long))
#else
#define _struct_page_alignment __aligned(sizeof(unsigned long))
#endif
struct page {
unsigned long flags; /* Atomic flags, some possibly
* updated asynchronously */
/*
* Five words (20/40 bytes) are available in this union.
* WARNING: bit 0 of the first word is used for PageTail(). That
* means the other users of this union MUST NOT use the bit to
* avoid collision and false-positive PageTail().
*/
union {
struct { /* Page cache and anonymous pages */
/**
* @lru: Pageout list, eg. active_list protected by
* lruvec->lru_lock. Sometimes used as a generic list
* by the page owner.
*/
union {
struct list_head lru;
/* Or, for the Unevictable "LRU list" slot */
struct {
/* Always even, to negate PageTail */
void *__filler;
/* Count page's or folio's mlocks */
unsigned int mlock_count;
};
/* Or, free page */
struct list_head buddy_list;
struct list_head pcp_list;
};
/* See page-flags.h for PAGE_MAPPING_FLAGS */
struct address_space *mapping;
union {
pgoff_t index; /* Our offset within mapping. */
unsigned long share; /* share count for fsdax */
};
/**
* @private: Mapping-private opaque data.
* Usually used for buffer_heads if PagePrivate.
* Used for swp_entry_t if PageSwapCache.
* Indicates order in the buddy system if PageBuddy.
*/
unsigned long private;
};
struct { /* page_pool used by netstack */
/**
* @pp_magic: magic value to avoid recycling non
* page_pool allocated pages.
*/
unsigned long pp_magic;
struct page_pool *pp;
unsigned long _pp_mapping_pad;
unsigned long dma_addr;
atomic_long_t pp_ref_count;
};
struct { /* Tail pages of compound page */
unsigned long compound_head; /* Bit zero is set */
};
struct { /* ZONE_DEVICE pages */
/** @pgmap: Points to the hosting device page map. */
struct dev_pagemap *pgmap;
void *zone_device_data;
/*
* ZONE_DEVICE private pages are counted as being
* mapped so the next 3 words hold the mapping, index,
* and private fields from the source anonymous or
* page cache page while the page is migrated to device
* private memory.
* ZONE_DEVICE MEMORY_DEVICE_FS_DAX pages also
* use the mapping, index, and private fields when
* pmem backed DAX files are mapped.
*/
};
/** @rcu_head: You can use this to free a page by RCU. */
struct rcu_head rcu_head;
};
union { /* This union is 4 bytes in size. */
/*
* If the page can be mapped to userspace, encodes the number
* of times this page is referenced by a page table.
*/
atomic_t _mapcount;
/*
* If the page is neither PageSlab nor mappable to userspace,
* the value stored here may help determine what this page
* is used for. See page-flags.h for a list of page types
* which are currently stored here.
*/
unsigned int page_type;
};
/* Usage count. *DO NOT USE DIRECTLY*. See page_ref.h */
atomic_t _refcount;
#ifdef CONFIG_MEMCG
unsigned long memcg_data;
#endif
/*
* On machines where all RAM is mapped into kernel address space,
* we can simply calculate the virtual address. On machines with
* highmem some memory is mapped into kernel virtual memory
* dynamically, so we need a place to store that address.
* Note that this field could be 16 bits on x86 ... ;)
*
* Architectures with slow multiplication can define
* WANT_PAGE_VIRTUAL in asm/page.h
*/
#if defined(WANT_PAGE_VIRTUAL)
void *virtual; /* Kernel virtual address (NULL if
not kmapped, ie. highmem) */
#endif /* WANT_PAGE_VIRTUAL */
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
int _last_cpupid;
#endif
#ifdef CONFIG_KMSAN
/*
* KMSAN metadata for this page:
* - shadow page: every bit indicates whether the corresponding
* bit of the original page is initialized (0) or not (1);
* - origin page: every 4 bytes contain an id of the stack trace
* where the uninitialized value was created.
*/
struct page *kmsan_shadow;
struct page *kmsan_origin;
#endif
} _struct_page_alignment;
/*
* struct encoded_page - a nonexistent type marking this pointer
*
* An 'encoded_page' pointer is a pointer to a regular 'struct page', but
* with the low bits of the pointer indicating extra context-dependent
* information. Only used in mmu_gather handling, and this acts as a type
* system check on that use.
*
* We only really have two guaranteed bits in general, although you could
* play with 'struct page' alignment (see CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
* for more.
*
* Use the supplied helper functions to endcode/decode the pointer and bits.
*/
struct encoded_page;
#define ENCODED_PAGE_BITS 3ul
/* Perform rmap removal after we have flushed the TLB. */
#define ENCODED_PAGE_BIT_DELAY_RMAP 1ul
/*
* The next item in an encoded_page array is the "nr_pages" argument, specifying
* the number of consecutive pages starting from this page, that all belong to
* the same folio. For example, "nr_pages" corresponds to the number of folio
* references that must be dropped. If this bit is not set, "nr_pages" is
* implicitly 1.
*/
#define ENCODED_PAGE_BIT_NR_PAGES_NEXT 2ul
static __always_inline struct encoded_page *encode_page(struct page *page, unsigned long flags)
{
BUILD_BUG_ON(flags > ENCODED_PAGE_BITS);
return (struct encoded_page *)(flags | (unsigned long)page);
}
static inline unsigned long encoded_page_flags(struct encoded_page *page)
{
return ENCODED_PAGE_BITS & (unsigned long)page;
}
static inline struct page *encoded_page_ptr(struct encoded_page *page)
{
return (struct page *)(~ENCODED_PAGE_BITS & (unsigned long)page);
}
static __always_inline struct encoded_page *encode_nr_pages(unsigned long nr)
{
VM_WARN_ON_ONCE((nr << 2) >> 2 != nr);
return (struct encoded_page *)(nr << 2);
}
static __always_inline unsigned long encoded_nr_pages(struct encoded_page *page)
{
return ((unsigned long)page) >> 2;
}
/*
* A swap entry has to fit into a "unsigned long", as the entry is hidden
* in the "index" field of the swapper address space.
*/
typedef struct {
unsigned long val;
} swp_entry_t;
/**
* struct folio - Represents a contiguous set of bytes.
* @flags: Identical to the page flags.
* @lru: Least Recently Used list; tracks how recently this folio was used.
* @mlock_count: Number of times this folio has been pinned by mlock().
* @mapping: The file this page belongs to, or refers to the anon_vma for
* anonymous memory.
* @index: Offset within the file, in units of pages. For anonymous memory,
* this is the index from the beginning of the mmap.
* @private: Filesystem per-folio data (see folio_attach_private()).
* @swap: Used for swp_entry_t if folio_test_swapcache().
* @_mapcount: Do not access this member directly. Use folio_mapcount() to
* find out how many times this folio is mapped by userspace.
* @_refcount: Do not access this member directly. Use folio_ref_count()
* to find how many references there are to this folio.
* @memcg_data: Memory Control Group data.
* @virtual: Virtual address in the kernel direct map.
* @_last_cpupid: IDs of last CPU and last process that accessed the folio.
* @_entire_mapcount: Do not use directly, call folio_entire_mapcount().
* @_nr_pages_mapped: Do not use directly, call folio_mapcount().
* @_pincount: Do not use directly, call folio_maybe_dma_pinned().
* @_folio_nr_pages: Do not use directly, call folio_nr_pages().
* @_hugetlb_subpool: Do not use directly, use accessor in hugetlb.h.
* @_hugetlb_cgroup: Do not use directly, use accessor in hugetlb_cgroup.h.
* @_hugetlb_cgroup_rsvd: Do not use directly, use accessor in hugetlb_cgroup.h.
* @_hugetlb_hwpoison: Do not use directly, call raw_hwp_list_head().
* @_deferred_list: Folios to be split under memory pressure.
*
* A folio is a physically, virtually and logically contiguous set
* of bytes. It is a power-of-two in size, and it is aligned to that
* same power-of-two. It is at least as large as %PAGE_SIZE. If it is
* in the page cache, it is at a file offset which is a multiple of that
* power-of-two. It may be mapped into userspace at an address which is
* at an arbitrary page offset, but its kernel virtual address is aligned
* to its size.
*/
struct folio {
/* private: don't document the anon union */
union {
struct {
/* public: */
unsigned long flags;
union {
struct list_head lru;
/* private: avoid cluttering the output */
struct {
void *__filler;
/* public: */
unsigned int mlock_count;
/* private: */
};
/* public: */
};
struct address_space *mapping;
pgoff_t index;
union {
void *private;
swp_entry_t swap;
};
atomic_t _mapcount;
atomic_t _refcount;
#ifdef CONFIG_MEMCG
unsigned long memcg_data;
#endif
#if defined(WANT_PAGE_VIRTUAL)
void *virtual;
#endif
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
int _last_cpupid;
#endif
/* private: the union with struct page is transitional */
};
struct page page;
};
union {
struct {
unsigned long _flags_1;
unsigned long _head_1;
unsigned long _folio_avail;
/* public: */
atomic_t _entire_mapcount;
atomic_t _nr_pages_mapped;
atomic_t _pincount;
#ifdef CONFIG_64BIT
unsigned int _folio_nr_pages;
#endif
/* private: the union with struct page is transitional */
};
struct page __page_1;
};
union {
struct {
unsigned long _flags_2;
unsigned long _head_2;
/* public: */
void *_hugetlb_subpool;
void *_hugetlb_cgroup;
void *_hugetlb_cgroup_rsvd;
void *_hugetlb_hwpoison;
/* private: the union with struct page is transitional */
};
struct {
unsigned long _flags_2a;
unsigned long _head_2a;
/* public: */
struct list_head _deferred_list;
/* private: the union with struct page is transitional */
};
struct page __page_2;
};
};
#define FOLIO_MATCH(pg, fl) \
static_assert(offsetof(struct page, pg) == offsetof(struct folio, fl))
FOLIO_MATCH(flags, flags);
FOLIO_MATCH(lru, lru);
FOLIO_MATCH(mapping, mapping);
FOLIO_MATCH(compound_head, lru);
FOLIO_MATCH(index, index);
FOLIO_MATCH(private, private);
FOLIO_MATCH(_mapcount, _mapcount);
FOLIO_MATCH(_refcount, _refcount);
#ifdef CONFIG_MEMCG
FOLIO_MATCH(memcg_data, memcg_data);
#endif
#if defined(WANT_PAGE_VIRTUAL)
FOLIO_MATCH(virtual, virtual);
#endif
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
FOLIO_MATCH(_last_cpupid, _last_cpupid);
#endif
#undef FOLIO_MATCH
#define FOLIO_MATCH(pg, fl) \
static_assert(offsetof(struct folio, fl) == \
offsetof(struct page, pg) + sizeof(struct page))
FOLIO_MATCH(flags, _flags_1);
FOLIO_MATCH(compound_head, _head_1);
#undef FOLIO_MATCH
#define FOLIO_MATCH(pg, fl) \
static_assert(offsetof(struct folio, fl) == \
offsetof(struct page, pg) + 2 * sizeof(struct page))
FOLIO_MATCH(flags, _flags_2);
FOLIO_MATCH(compound_head, _head_2);
FOLIO_MATCH(flags, _flags_2a);
FOLIO_MATCH(compound_head, _head_2a);
#undef FOLIO_MATCH
/**
* struct ptdesc - Memory descriptor for page tables.
* @__page_flags: Same as page flags. Powerpc only.
* @pt_rcu_head: For freeing page table pages.
* @pt_list: List of used page tables. Used for s390 and x86.
* @_pt_pad_1: Padding that aliases with page's compound head.
* @pmd_huge_pte: Protected by ptdesc->ptl, used for THPs.
* @__page_mapping: Aliases with page->mapping. Unused for page tables.
* @pt_index: Used for s390 gmap.
* @pt_mm: Used for x86 pgds.
* @pt_frag_refcount: For fragmented page table tracking. Powerpc only.
* @_pt_pad_2: Padding to ensure proper alignment.
* @ptl: Lock for the page table.
* @__page_type: Same as page->page_type. Unused for page tables.
* @__page_refcount: Same as page refcount.
* @pt_memcg_data: Memcg data. Tracked for page tables here.
*
* This struct overlays struct page for now. Do not modify without a good
* understanding of the issues.
*/
struct ptdesc {
unsigned long __page_flags;
union {
struct rcu_head pt_rcu_head;
struct list_head pt_list;
struct {
unsigned long _pt_pad_1;
pgtable_t pmd_huge_pte;
};
};
unsigned long __page_mapping;
union {
pgoff_t pt_index;
struct mm_struct *pt_mm;
atomic_t pt_frag_refcount;
};
union {
unsigned long _pt_pad_2;
#if ALLOC_SPLIT_PTLOCKS
spinlock_t *ptl;
#else
spinlock_t ptl;
#endif
};
unsigned int __page_type;
atomic_t __page_refcount;
#ifdef CONFIG_MEMCG
unsigned long pt_memcg_data;
#endif
};
#define TABLE_MATCH(pg, pt) \
static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt))
TABLE_MATCH(flags, __page_flags);
TABLE_MATCH(compound_head, pt_list);
TABLE_MATCH(compound_head, _pt_pad_1);
TABLE_MATCH(mapping, __page_mapping);
TABLE_MATCH(index, pt_index);
TABLE_MATCH(rcu_head, pt_rcu_head);
TABLE_MATCH(page_type, __page_type);
TABLE_MATCH(_refcount, __page_refcount);
#ifdef CONFIG_MEMCG
TABLE_MATCH(memcg_data, pt_memcg_data);
#endif
#undef TABLE_MATCH
static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
#define ptdesc_page(pt) (_Generic((pt), \
const struct ptdesc *: (const struct page *)(pt), \
struct ptdesc *: (struct page *)(pt)))
#define ptdesc_folio(pt) (_Generic((pt), \
const struct ptdesc *: (const struct folio *)(pt), \
struct ptdesc *: (struct folio *)(pt)))
#define page_ptdesc(p) (_Generic((p), \
const struct page *: (const struct ptdesc *)(p), \
struct page *: (struct ptdesc *)(p)))
/*
* Used for sizing the vmemmap region on some architectures
*/
#define STRUCT_PAGE_MAX_SHIFT (order_base_2(sizeof(struct page)))
#define PAGE_FRAG_CACHE_MAX_SIZE __ALIGN_MASK(32768, ~PAGE_MASK)
#define PAGE_FRAG_CACHE_MAX_ORDER get_order(PAGE_FRAG_CACHE_MAX_SIZE)
/*
* page_private can be used on tail pages. However, PagePrivate is only
* checked by the VM on the head page. So page_private on the tail pages
* should be used for data that's ancillary to the head page (eg attaching
* buffer heads to tail pages after attaching buffer heads to the head page)
*/
#define page_private(page) ((page)->private)
static inline void set_page_private(struct page *page, unsigned long private)
{
page->private = private;
}
static inline void *folio_get_private(struct folio *folio)
{
return folio->private;
}
struct page_frag_cache {
void * va;
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
__u16 offset;
__u16 size;
#else
__u32 offset;
#endif
/* we maintain a pagecount bias, so that we dont dirty cache line
* containing page->_refcount every time we allocate a fragment.
*/
unsigned int pagecnt_bias;
bool pfmemalloc;
};
typedef unsigned long vm_flags_t;
/*
* A region containing a mapping of a non-memory backed file under NOMMU
* conditions. These are held in a global tree and are pinned by the VMAs that
* map parts of them.
*/
struct vm_region {
struct rb_node vm_rb; /* link in global region tree */
vm_flags_t vm_flags; /* VMA vm_flags */
unsigned long vm_start; /* start address of region */
unsigned long vm_end; /* region initialised to here */
unsigned long vm_top; /* region allocated to here */
unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */
struct file *vm_file; /* the backing file or NULL */
int vm_usage; /* region usage count (access under nommu_region_sem) */
bool vm_icache_flushed : 1; /* true if the icache has been flushed for
* this region */
};
#ifdef CONFIG_USERFAULTFD
#define NULL_VM_UFFD_CTX ((struct vm_userfaultfd_ctx) { NULL, })
struct vm_userfaultfd_ctx {
struct userfaultfd_ctx *ctx;
};
#else /* CONFIG_USERFAULTFD */
#define NULL_VM_UFFD_CTX ((struct vm_userfaultfd_ctx) {})
struct vm_userfaultfd_ctx {};
#endif /* CONFIG_USERFAULTFD */
struct anon_vma_name {
struct kref kref;
/* The name needs to be at the end because it is dynamically sized. */
char name[];
};
#ifdef CONFIG_ANON_VMA_NAME
/*
* mmap_lock should be read-locked when calling anon_vma_name(). Caller should
* either keep holding the lock while using the returned pointer or it should
* raise anon_vma_name refcount before releasing the lock.
*/
struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma);
struct anon_vma_name *anon_vma_name_alloc(const char *name);
void anon_vma_name_free(struct kref *kref);
#else /* CONFIG_ANON_VMA_NAME */
static inline struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
{
return NULL;
}
static inline struct anon_vma_name *anon_vma_name_alloc(const char *name)
{
return NULL;
}
#endif
struct vma_lock {
struct rw_semaphore lock;
};
struct vma_numab_state {
/*
* Initialised as time in 'jiffies' after which VMA
* should be scanned. Delays first scan of new VMA by at
* least sysctl_numa_balancing_scan_delay:
*/
unsigned long next_scan;
/*
* Time in jiffies when pids_active[] is reset to
* detect phase change behaviour:
*/
unsigned long pids_active_reset;
/*
* Approximate tracking of PIDs that trapped a NUMA hinting
* fault. May produce false positives due to hash collisions.
*
* [0] Previous PID tracking
* [1] Current PID tracking
*
* Window moves after next_pid_reset has expired approximately
* every VMA_PID_RESET_PERIOD jiffies:
*/
unsigned long pids_active[2];
/* MM scan sequence ID when scan first started after VMA creation */
int start_scan_seq;
/*
* MM scan sequence ID when the VMA was last completely scanned.
* A VMA is not eligible for scanning if prev_scan_seq == numa_scan_seq
*/
int prev_scan_seq;
};
/*
* This struct describes a virtual memory area. There is one of these
* per VM-area/task. A VM area is any part of the process virtual memory
* space that has a special rule for the page-fault handlers (ie a shared
* library, the executable area etc).
*/
struct vm_area_struct {
/* The first cache line has the info for VMA tree walking. */
union {
struct {
/* VMA covers [vm_start; vm_end) addresses within mm */
unsigned long vm_start;
unsigned long vm_end;
};
#ifdef CONFIG_PER_VMA_LOCK
struct rcu_head vm_rcu; /* Used for deferred freeing. */
#endif
};
struct mm_struct *vm_mm; /* The address space we belong to. */
pgprot_t vm_page_prot; /* Access permissions of this VMA. */
/*
* Flags, see mm.h.
* To modify use vm_flags_{init|reset|set|clear|mod} functions.
*/
union {
const vm_flags_t vm_flags;
vm_flags_t __private __vm_flags;
};
#ifdef CONFIG_PER_VMA_LOCK
/*
* Can only be written (using WRITE_ONCE()) while holding both:
* - mmap_lock (in write mode)
* - vm_lock->lock (in write mode)
* Can be read reliably while holding one of:
* - mmap_lock (in read or write mode)
* - vm_lock->lock (in read or write mode)
* Can be read unreliably (using READ_ONCE()) for pessimistic bailout
* while holding nothing (except RCU to keep the VMA struct allocated).
*
* This sequence counter is explicitly allowed to overflow; sequence
* counter reuse can only lead to occasional unnecessary use of the
* slowpath.
*/
int vm_lock_seq;
struct vma_lock *vm_lock;
/* Flag to indicate areas detached from the mm->mm_mt tree */
bool detached;
#endif
/*
* For areas with an address space and backing store,
* linkage into the address_space->i_mmap interval tree.
*
*/
struct {
struct rb_node rb;
unsigned long rb_subtree_last;
} shared;
/*
* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
* list, after a COW of one of the file pages. A MAP_SHARED vma
* can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack
* or brk vma (with NULL file) can only be in an anon_vma list.
*/
struct list_head anon_vma_chain; /* Serialized by mmap_lock &
* page_table_lock */
struct anon_vma *anon_vma; /* Serialized by page_table_lock */
/* Function pointers to deal with this struct. */
const struct vm_operations_struct *vm_ops;
/* Information about our backing store: */
unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE
units */
struct file * vm_file; /* File we map to (can be NULL). */
void * vm_private_data; /* was vm_pte (shared mem) */
#ifdef CONFIG_ANON_VMA_NAME
/*
* For private and shared anonymous mappings, a pointer to a null
* terminated string containing the name given to the vma, or NULL if
* unnamed. Serialized by mmap_lock. Use anon_vma_name to access.
*/
struct anon_vma_name *anon_name;
#endif
#ifdef CONFIG_SWAP
atomic_long_t swap_readahead_info;
#endif
#ifndef CONFIG_MMU
struct vm_region *vm_region; /* NOMMU mapping region */
#endif
#ifdef CONFIG_NUMA
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
#endif
#ifdef CONFIG_NUMA_BALANCING
struct vma_numab_state *numab_state; /* NUMA Balancing state */
#endif
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
} __randomize_layout;
#ifdef CONFIG_NUMA
#define vma_policy(vma) ((vma)->vm_policy)
#else
#define vma_policy(vma) NULL
#endif
#ifdef CONFIG_SCHED_MM_CID
struct mm_cid {
u64 time;
int cid;
};
#endif
struct kioctx_table;
struct iommu_mm_data;
struct mm_struct {
struct {
/*
* Fields which are often written to are placed in a separate
* cache line.
*/
struct {
/**
* @mm_count: The number of references to &struct
* mm_struct (@mm_users count as 1).
*
* Use mmgrab()/mmdrop() to modify. When this drops to
* 0, the &struct mm_struct is freed.
*/
atomic_t mm_count;
} ____cacheline_aligned_in_smp;
struct maple_tree mm_mt;
#ifdef CONFIG_MMU
unsigned long (*get_unmapped_area) (struct file *filp,
unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags);
#endif
unsigned long mmap_base; /* base of mmap area */
unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */
#ifdef CONFIG_HAVE_ARCH_COMPAT_MMAP_BASES
/* Base addresses for compatible mmap() */
unsigned long mmap_compat_base;
unsigned long mmap_compat_legacy_base;
#endif
unsigned long task_size; /* size of task vm space */
pgd_t * pgd;
#ifdef CONFIG_MEMBARRIER
/**
* @membarrier_state: Flags controlling membarrier behavior.
*
* This field is close to @pgd to hopefully fit in the same
* cache-line, which needs to be touched by switch_mm().
*/
atomic_t membarrier_state;
#endif
/**
* @mm_users: The number of users including userspace.
*
* Use mmget()/mmget_not_zero()/mmput() to modify. When this
* drops to 0 (i.e. when the task exits and there are no other
* temporary reference holders), we also release a reference on
* @mm_count (which may then free the &struct mm_struct if
* @mm_count also drops to 0).
*/
atomic_t mm_users;
#ifdef CONFIG_SCHED_MM_CID
/**
* @pcpu_cid: Per-cpu current cid.
*
* Keep track of the currently allocated mm_cid for each cpu.
* The per-cpu mm_cid values are serialized by their respective
* runqueue locks.
*/
struct mm_cid __percpu *pcpu_cid;
/*
* @mm_cid_next_scan: Next mm_cid scan (in jiffies).
*
* When the next mm_cid scan is due (in jiffies).
*/
unsigned long mm_cid_next_scan;
#endif
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
int map_count; /* number of VMAs */
spinlock_t page_table_lock; /* Protects page tables and some
* counters
*/
/*
* With some kernel config, the current mmap_lock's offset
* inside 'mm_struct' is at 0x120, which is very optimal, as
* its two hot fields 'count' and 'owner' sit in 2 different
* cachelines, and when mmap_lock is highly contended, both
* of the 2 fields will be accessed frequently, current layout
* will help to reduce cache bouncing.
*
* So please be careful with adding new fields before
* mmap_lock, which can easily push the 2 fields into one
* cacheline.
*/
struct rw_semaphore mmap_lock;
struct list_head mmlist; /* List of maybe swapped mm's. These
* are globally strung together off
* init_mm.mmlist, and are protected
* by mmlist_lock
*/
#ifdef CONFIG_PER_VMA_LOCK
/*
* This field has lock-like semantics, meaning it is sometimes
* accessed with ACQUIRE/RELEASE semantics.
* Roughly speaking, incrementing the sequence number is
* equivalent to releasing locks on VMAs; reading the sequence
* number can be part of taking a read lock on a VMA.
*
* Can be modified under write mmap_lock using RELEASE
* semantics.
* Can be read with no other protection when holding write
* mmap_lock.
* Can be read with ACQUIRE semantics if not holding write
* mmap_lock.
*/
int mm_lock_seq;
#endif
unsigned long hiwater_rss; /* High-watermark of RSS usage */
unsigned long hiwater_vm; /* High-water virtual memory usage */
unsigned long total_vm; /* Total pages mapped */
unsigned long locked_vm; /* Pages that have PG_mlocked set */
atomic64_t pinned_vm; /* Refcount permanently increased */
unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */
unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
unsigned long stack_vm; /* VM_STACK */
unsigned long def_flags;
/**
* @write_protect_seq: Locked when any thread is write
* protecting pages mapped by this mm to enforce a later COW,
* for instance during page table copying for fork().
*/
seqcount_t write_protect_seq;
spinlock_t arg_lock; /* protect the below fields */
unsigned long start_code, end_code, start_data, end_data;
unsigned long start_brk, brk, start_stack;
unsigned long arg_start, arg_end, env_start, env_end;
unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
struct percpu_counter rss_stat[NR_MM_COUNTERS];
struct linux_binfmt *binfmt;
/* Architecture-specific MM context */
mm_context_t context;
unsigned long flags; /* Must use atomic bitops to access */
#ifdef CONFIG_AIO
spinlock_t ioctx_lock;
struct kioctx_table __rcu *ioctx_table;
#endif
#ifdef CONFIG_MEMCG
/*
* "owner" points to a task that is regarded as the canonical
* user/owner of this mm. All of the following must be true in
* order for it to be changed:
*
* current == mm->owner
* current->mm != mm
* new_owner->mm == mm
* new_owner->alloc_lock is held
*/
struct task_struct __rcu *owner;
#endif
struct user_namespace *user_ns;
/* store ref to file /proc/<pid>/exe symlink points to */
struct file __rcu *exe_file;
#ifdef CONFIG_MMU_NOTIFIER
struct mmu_notifier_subscriptions *notifier_subscriptions;
#endif
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
pgtable_t pmd_huge_pte; /* protected by page_table_lock */
#endif
#ifdef CONFIG_NUMA_BALANCING
/*
* numa_next_scan is the next time that PTEs will be remapped
* PROT_NONE to trigger NUMA hinting faults; such faults gather
* statistics and migrate pages to new nodes if necessary.
*/
unsigned long numa_next_scan;
/* Restart point for scanning and remapping PTEs. */
unsigned long numa_scan_offset;
/* numa_scan_seq prevents two threads remapping PTEs. */
int numa_scan_seq;
#endif
/*
* An operation with batched TLB flushing is going on. Anything
* that can move process memory needs to flush the TLB when
* moving a PROT_NONE mapped page.
*/
atomic_t tlb_flush_pending;
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
/* See flush_tlb_batched_pending() */
atomic_t tlb_flush_batched;
#endif
struct uprobes_state uprobes_state;
#ifdef CONFIG_PREEMPT_RT
struct rcu_head delayed_drop;
#endif
#ifdef CONFIG_HUGETLB_PAGE
atomic_long_t hugetlb_usage;
#endif
struct work_struct async_put_work;
#ifdef CONFIG_IOMMU_MM_DATA
struct iommu_mm_data *iommu_mm;
#endif
#ifdef CONFIG_KSM
/*
* Represent how many pages of this process are involved in KSM
* merging (not including ksm_zero_pages).
*/
unsigned long ksm_merging_pages;
/*
* Represent how many pages are checked for ksm merging
* including merged and not merged.
*/
unsigned long ksm_rmap_items;
/*
* Represent how many empty pages are merged with kernel zero
* pages when enabling KSM use_zero_pages.
*/
unsigned long ksm_zero_pages;
#endif /* CONFIG_KSM */
#ifdef CONFIG_LRU_GEN_WALKS_MMU
struct {
/* this mm_struct is on lru_gen_mm_list */
struct list_head list;
/*
* Set when switching to this mm_struct, as a hint of
* whether it has been used since the last time per-node
* page table walkers cleared the corresponding bits.
*/
unsigned long bitmap;
#ifdef CONFIG_MEMCG
/* points to the memcg of "owner" above */
struct mem_cgroup *memcg;
#endif
} lru_gen;
#endif /* CONFIG_LRU_GEN_WALKS_MMU */
} __randomize_layout;
/*
* The mm_cpumask needs to be at the end of mm_struct, because it
* is dynamically sized based on nr_cpu_ids.
*/
unsigned long cpu_bitmap[];
};
#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN | \
MT_FLAGS_USE_RCU)
extern struct mm_struct init_mm;
/* Pointer magic because the dynamic array size confuses some compilers. */
static inline void mm_init_cpumask(struct mm_struct *mm)
{
unsigned long cpu_bitmap = (unsigned long)mm;
cpu_bitmap += offsetof(struct mm_struct, cpu_bitmap);
cpumask_clear((struct cpumask *)cpu_bitmap);
}
/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
{
return (struct cpumask *)&mm->cpu_bitmap;
}
#ifdef CONFIG_LRU_GEN
struct lru_gen_mm_list {
/* mm_struct list for page table walkers */
struct list_head fifo;
/* protects the list above */
spinlock_t lock;
};
#endif /* CONFIG_LRU_GEN */
#ifdef CONFIG_LRU_GEN_WALKS_MMU
void lru_gen_add_mm(struct mm_struct *mm);
void lru_gen_del_mm(struct mm_struct *mm);
void lru_gen_migrate_mm(struct mm_struct *mm);
static inline void lru_gen_init_mm(struct mm_struct *mm)
{
INIT_LIST_HEAD(&mm->lru_gen.list);
mm->lru_gen.bitmap = 0;
#ifdef CONFIG_MEMCG
mm->lru_gen.memcg = NULL;
#endif
}
static inline void lru_gen_use_mm(struct mm_struct *mm)
{
/*
* When the bitmap is set, page reclaim knows this mm_struct has been
* used since the last time it cleared the bitmap. So it might be worth
* walking the page tables of this mm_struct to clear the accessed bit.
*/
WRITE_ONCE(mm->lru_gen.bitmap, -1);
}
#else /* !CONFIG_LRU_GEN_WALKS_MMU */
static inline void lru_gen_add_mm(struct mm_struct *mm)
{
}
static inline void lru_gen_del_mm(struct mm_struct *mm)
{
}
static inline void lru_gen_migrate_mm(struct mm_struct *mm)
{
}
static inline void lru_gen_init_mm(struct mm_struct *mm)
{
}
static inline void lru_gen_use_mm(struct mm_struct *mm)
{
}
#endif /* CONFIG_LRU_GEN_WALKS_MMU */
struct vma_iterator {
struct ma_state mas;
};
#define VMA_ITERATOR(name, __mm, __addr) \
struct vma_iterator name = { \
.mas = { \
.tree = &(__mm)->mm_mt, \
.index = __addr, \
.node = NULL, \
.status = ma_start, \
}, \
}
static inline void vma_iter_init(struct vma_iterator *vmi,
struct mm_struct *mm, unsigned long addr)
{
mas_init(&vmi->mas, &mm->mm_mt, addr);
}
#ifdef CONFIG_SCHED_MM_CID
enum mm_cid_state {
MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
MM_CID_LAZY_PUT = (1U << 31),
};
static inline bool mm_cid_is_unset(int cid)
{
return cid == MM_CID_UNSET;
}
static inline bool mm_cid_is_lazy_put(int cid)
{
return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
}
static inline bool mm_cid_is_valid(int cid)
{
return !(cid & MM_CID_LAZY_PUT);
}
static inline int mm_cid_set_lazy_put(int cid)
{
return cid | MM_CID_LAZY_PUT;
}
static inline int mm_cid_clear_lazy_put(int cid)
{
return cid & ~MM_CID_LAZY_PUT;
}
/* Accessor for struct mm_struct's cidmask. */
static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm;
cid_bitmap += offsetof(struct mm_struct, cpu_bitmap);
/* Skip cpu_bitmap */
cid_bitmap += cpumask_size();
return (struct cpumask *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm)
{
int i;
for_each_possible_cpu(i) {
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
pcpu_cid->cid = MM_CID_UNSET;
pcpu_cid->time = 0;
}
cpumask_clear(mm_cidmask(mm));
}
static inline int mm_alloc_cid(struct mm_struct *mm)
{
mm->pcpu_cid = alloc_percpu(struct mm_cid);
if (!mm->pcpu_cid)
return -ENOMEM;
mm_init_cid(mm);
return 0;
}
static inline void mm_destroy_cid(struct mm_struct *mm)
{
free_percpu(mm->pcpu_cid);
mm->pcpu_cid = NULL;
}
static inline unsigned int mm_cid_size(void)
{
return cpumask_size();
}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm) { }
static inline int mm_alloc_cid(struct mm_struct *mm) { return 0; }
static inline void mm_destroy_cid(struct mm_struct *mm) { }
static inline unsigned int mm_cid_size(void)
{
return 0;
}
#endif /* CONFIG_SCHED_MM_CID */
struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_finish_mmu(struct mmu_gather *tlb);
struct vm_fault;
/**
* typedef vm_fault_t - Return type for page fault handlers.
*
* Page fault handlers return a bitmask of %VM_FAULT values.
*/
typedef __bitwise unsigned int vm_fault_t;
/**
* enum vm_fault_reason - Page fault handlers return a bitmask of
* these values to tell the core VM what happened when handling the
* fault. Used to decide whether a process gets delivered SIGBUS or
* just gets major/minor fault counters bumped up.
*
* @VM_FAULT_OOM: Out Of Memory
* @VM_FAULT_SIGBUS: Bad access
* @VM_FAULT_MAJOR: Page read from storage
* @VM_FAULT_HWPOISON: Hit poisoned small page
* @VM_FAULT_HWPOISON_LARGE: Hit poisoned large page. Index encoded
* in upper bits
* @VM_FAULT_SIGSEGV: segmentation fault
* @VM_FAULT_NOPAGE: ->fault installed the pte, not return page
* @VM_FAULT_LOCKED: ->fault locked the returned page
* @VM_FAULT_RETRY: ->fault blocked, must retry
* @VM_FAULT_FALLBACK: huge page fault failed, fall back to small
* @VM_FAULT_DONE_COW: ->fault has fully handled COW
* @VM_FAULT_NEEDDSYNC: ->fault did not modify page tables and needs
* fsync() to complete (for synchronous page faults
* in DAX)
* @VM_FAULT_COMPLETED: ->fault completed, meanwhile mmap lock released
* @VM_FAULT_HINDEX_MASK: mask HINDEX value
*
*/
enum vm_fault_reason {
VM_FAULT_OOM = (__force vm_fault_t)0x000001,
VM_FAULT_SIGBUS = (__force vm_fault_t)0x000002,
VM_FAULT_MAJOR = (__force vm_fault_t)0x000004,
VM_FAULT_HWPOISON = (__force vm_fault_t)0x000010,
VM_FAULT_HWPOISON_LARGE = (__force vm_fault_t)0x000020,
VM_FAULT_SIGSEGV = (__force vm_fault_t)0x000040,
VM_FAULT_NOPAGE = (__force vm_fault_t)0x000100,
VM_FAULT_LOCKED = (__force vm_fault_t)0x000200,
VM_FAULT_RETRY = (__force vm_fault_t)0x000400,
VM_FAULT_FALLBACK = (__force vm_fault_t)0x000800,
VM_FAULT_DONE_COW = (__force vm_fault_t)0x001000,
VM_FAULT_NEEDDSYNC = (__force vm_fault_t)0x002000,
VM_FAULT_COMPLETED = (__force vm_fault_t)0x004000,
VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000,
};
/* Encode hstate index for a hwpoisoned large page */
#define VM_FAULT_SET_HINDEX(x) ((__force vm_fault_t)((x) << 16))
#define VM_FAULT_GET_HINDEX(x) (((__force unsigned int)(x) >> 16) & 0xf)
#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | \
VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON | \
VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)
#define VM_FAULT_RESULT_TRACE \
{ VM_FAULT_OOM, "OOM" }, \
{ VM_FAULT_SIGBUS, "SIGBUS" }, \
{ VM_FAULT_MAJOR, "MAJOR" }, \
{ VM_FAULT_HWPOISON, "HWPOISON" }, \
{ VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \
{ VM_FAULT_SIGSEGV, "SIGSEGV" }, \
{ VM_FAULT_NOPAGE, "NOPAGE" }, \
{ VM_FAULT_LOCKED, "LOCKED" }, \
{ VM_FAULT_RETRY, "RETRY" }, \
{ VM_FAULT_FALLBACK, "FALLBACK" }, \
{ VM_FAULT_DONE_COW, "DONE_COW" }, \
{ VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \
{ VM_FAULT_COMPLETED, "COMPLETED" }
struct vm_special_mapping {
const char *name; /* The name, e.g. "[vdso]". */
/*
* If .fault is not provided, this points to a
* NULL-terminated array of pages that back the special mapping.
*
* This must not be NULL unless .fault is provided.
*/
struct page **pages;
/*
* If non-NULL, then this is called to resolve page faults
* on the special mapping. If used, .pages is not checked.
*/
vm_fault_t (*fault)(const struct vm_special_mapping *sm,
struct vm_area_struct *vma,
struct vm_fault *vmf);
int (*mremap)(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma);
};
enum tlb_flush_reason {
TLB_FLUSH_ON_TASK_SWITCH,
TLB_REMOTE_SHOOTDOWN,
TLB_LOCAL_SHOOTDOWN,
TLB_LOCAL_MM_SHOOTDOWN,
TLB_REMOTE_SEND_IPI,
NR_TLB_FLUSH_REASONS,
};
/**
* enum fault_flag - Fault flag definitions.
* @FAULT_FLAG_WRITE: Fault was a write fault.
* @FAULT_FLAG_MKWRITE: Fault was mkwrite of existing PTE.
* @FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked.
* @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_lock and wait when retrying.
* @FAULT_FLAG_KILLABLE: The fault task is in SIGKILL killable region.
* @FAULT_FLAG_TRIED: The fault has been tried once.
* @FAULT_FLAG_USER: The fault originated in userspace.
* @FAULT_FLAG_REMOTE: The fault is not for current task/mm.
* @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch.
* @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals.
* @FAULT_FLAG_UNSHARE: The fault is an unsharing request to break COW in a
* COW mapping, making sure that an exclusive anon page is
* mapped after the fault.
* @FAULT_FLAG_ORIG_PTE_VALID: whether the fault has vmf->orig_pte cached.
* We should only access orig_pte if this flag set.
* @FAULT_FLAG_VMA_LOCK: The fault is handled under VMA lock.
*
* About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify
* whether we would allow page faults to retry by specifying these two
* fault flags correctly. Currently there can be three legal combinations:
*
* (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and
* this is the first try
*
* (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and
* we've already tried at least once
*
* (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry
*
* The unlisted combination (!ALLOW_RETRY && TRIED) is illegal and should never
* be used. Note that page faults can be allowed to retry for multiple times,
* in which case we'll have an initial fault with flags (a) then later on
* continuous faults with flags (b). We should always try to detect pending
* signals before a retry to make sure the continuous page faults can still be
* interrupted if necessary.
*
* The combination FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE is illegal.
* FAULT_FLAG_UNSHARE is ignored and treated like an ordinary read fault when
* applied to mappings that are not COW mappings.
*/
enum fault_flag {
FAULT_FLAG_WRITE = 1 << 0,
FAULT_FLAG_MKWRITE = 1 << 1,
FAULT_FLAG_ALLOW_RETRY = 1 << 2,
FAULT_FLAG_RETRY_NOWAIT = 1 << 3,
FAULT_FLAG_KILLABLE = 1 << 4,
FAULT_FLAG_TRIED = 1 << 5,
FAULT_FLAG_USER = 1 << 6,
FAULT_FLAG_REMOTE = 1 << 7,
FAULT_FLAG_INSTRUCTION = 1 << 8,
FAULT_FLAG_INTERRUPTIBLE = 1 << 9,
FAULT_FLAG_UNSHARE = 1 << 10,
FAULT_FLAG_ORIG_PTE_VALID = 1 << 11,
FAULT_FLAG_VMA_LOCK = 1 << 12,
};
typedef unsigned int __bitwise zap_flags_t;
/*
* FOLL_PIN and FOLL_LONGTERM may be used in various combinations with each
* other. Here is what they mean, and how to use them:
*
*
* FIXME: For pages which are part of a filesystem, mappings are subject to the
* lifetime enforced by the filesystem and we need guarantees that longterm
* users like RDMA and V4L2 only establish mappings which coordinate usage with
* the filesystem. Ideas for this coordination include revoking the longterm
* pin, delaying writeback, bounce buffer page writeback, etc. As FS DAX was
* added after the problem with filesystems was found FS DAX VMAs are
* specifically failed. Filesystem pages are still subject to bugs and use of
* FOLL_LONGTERM should be avoided on those pages.
*
* In the CMA case: long term pins in a CMA region would unnecessarily fragment
* that region. And so, CMA attempts to migrate the page before pinning, when
* FOLL_LONGTERM is specified.
*
* FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
* but an additional pin counting system) will be invoked. This is intended for
* anything that gets a page reference and then touches page data (for example,
* Direct IO). This lets the filesystem know that some non-file-system entity is
* potentially changing the pages' data. In contrast to FOLL_GET (whose pages
* are released via put_page()), FOLL_PIN pages must be released, ultimately, by
* a call to unpin_user_page().
*
* FOLL_PIN is similar to FOLL_GET: both of these pin pages. They use different
* and separate refcounting mechanisms, however, and that means that each has
* its own acquire and release mechanisms:
*
* FOLL_GET: get_user_pages*() to acquire, and put_page() to release.
*
* FOLL_PIN: pin_user_pages*() to acquire, and unpin_user_pages to release.
*
* FOLL_PIN and FOLL_GET are mutually exclusive for a given function call.
* (The underlying pages may experience both FOLL_GET-based and FOLL_PIN-based
* calls applied to them, and that's perfectly OK. This is a constraint on the
* callers, not on the pages.)
*
* FOLL_PIN should be set internally by the pin_user_pages*() APIs, never
* directly by the caller. That's in order to help avoid mismatches when
* releasing pages: get_user_pages*() pages must be released via put_page(),
* while pin_user_pages*() pages must be released via unpin_user_page().
*
* Please see Documentation/core-api/pin_user_pages.rst for more information.
*/
enum {
/* check pte is writable */
FOLL_WRITE = 1 << 0,
/* do get_page on page */
FOLL_GET = 1 << 1,
/* give error on hole if it would be zero */
FOLL_DUMP = 1 << 2,
/* get_user_pages read/write w/o permission */
FOLL_FORCE = 1 << 3,
/*
* if a disk transfer is needed, start the IO and return without waiting
* upon it
*/
FOLL_NOWAIT = 1 << 4,
/* do not fault in pages */
FOLL_NOFAULT = 1 << 5,
/* check page is hwpoisoned */
FOLL_HWPOISON = 1 << 6,
/* don't do file mappings */
FOLL_ANON = 1 << 7,
/*
* FOLL_LONGTERM indicates that the page will be held for an indefinite
* time period _often_ under userspace control. This is in contrast to
* iov_iter_get_pages(), whose usages are transient.
*/
FOLL_LONGTERM = 1 << 8,
/* split huge pmd before returning */
FOLL_SPLIT_PMD = 1 << 9,
/* allow returning PCI P2PDMA pages */
FOLL_PCI_P2PDMA = 1 << 10,
/* allow interrupts from generic signals */
FOLL_INTERRUPTIBLE = 1 << 11,
/*
* Always honor (trigger) NUMA hinting faults.
*
* FOLL_WRITE implicitly honors NUMA hinting faults because a
* PROT_NONE-mapped page is not writable (exceptions with FOLL_FORCE
* apply). get_user_pages_fast_only() always implicitly honors NUMA
* hinting faults.
*/
FOLL_HONOR_NUMA_FAULT = 1 << 12,
/* See also internal only FOLL flags in mm/internal.h */
};
#endif /* _LINUX_MM_TYPES_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Dynamic DMA mapping support.
*
* This implementation is a fallback for platforms that do not support
* I/O TLBs (aka DMA address translation hardware).
* Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
* Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
* Copyright (C) 2000, 2003 Hewlett-Packard Co
* David Mosberger-Tang <davidm@hpl.hp.com>
*
* 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
* 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
* unnecessary i-cache flushing.
* 04/07/.. ak Better overflow handling. Assorted fixes.
* 05/09/10 linville Add support for syncing ranges, support syncing for
* DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
* 08/12/11 beckyb Add highmem support
*/
#define pr_fmt(fmt) "software IO TLB: " fmt
#include <linux/cache.h>
#include <linux/cc_platform.h>
#include <linux/ctype.h>
#include <linux/debugfs.h>
#include <linux/dma-direct.h>
#include <linux/dma-map-ops.h>
#include <linux/export.h>
#include <linux/gfp.h>
#include <linux/highmem.h>
#include <linux/io.h>
#include <linux/iommu-helper.h>
#include <linux/init.h>
#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/pfn.h>
#include <linux/rculist.h>
#include <linux/scatterlist.h>
#include <linux/set_memory.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/swiotlb.h>
#include <linux/types.h>
#ifdef CONFIG_DMA_RESTRICTED_POOL
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/of_reserved_mem.h>
#include <linux/slab.h>
#endif
#define CREATE_TRACE_POINTS
#include <trace/events/swiotlb.h>
#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
/*
* Minimum IO TLB size to bother booting with. Systems with mainly
* 64bit capable cards will only lightly use the swiotlb. If we can't
* allocate a contiguous 1MB, we're probably in trouble anyway.
*/
#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
/**
* struct io_tlb_slot - IO TLB slot descriptor
* @orig_addr: The original address corresponding to a mapped entry.
* @alloc_size: Size of the allocated buffer.
* @list: The free list describing the number of free entries available
* from each index.
* @pad_slots: Number of preceding padding slots. Valid only in the first
* allocated non-padding slot.
*/
struct io_tlb_slot {
phys_addr_t orig_addr;
size_t alloc_size;
unsigned short list;
unsigned short pad_slots;
};
static bool swiotlb_force_bounce;
static bool swiotlb_force_disable;
#ifdef CONFIG_SWIOTLB_DYNAMIC
static void swiotlb_dyn_alloc(struct work_struct *work);
static struct io_tlb_mem io_tlb_default_mem = {
.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
swiotlb_dyn_alloc),
};
#else /* !CONFIG_SWIOTLB_DYNAMIC */
static struct io_tlb_mem io_tlb_default_mem;
#endif /* CONFIG_SWIOTLB_DYNAMIC */
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
static unsigned long default_nareas;
/**
* struct io_tlb_area - IO TLB memory area descriptor
*
* This is a single area with a single lock.
*
* @used: The number of used IO TLB block.
* @index: The slot index to start searching in this area for next round.
* @lock: The lock to protect the above data structures in the map and
* unmap calls.
*/
struct io_tlb_area {
unsigned long used;
unsigned int index;
spinlock_t lock;
};
/*
* Round up number of slabs to the next power of 2. The last area is going
* be smaller than the rest if default_nslabs is not power of two.
* The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
* otherwise a segment may span two or more areas. It conflicts with free
* contiguous slots tracking: free slots are treated contiguous no matter
* whether they cross an area boundary.
*
* Return true if default_nslabs is rounded up.
*/
static bool round_up_default_nslabs(void)
{
if (!default_nareas)
return false;
if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
default_nslabs = IO_TLB_SEGSIZE * default_nareas;
else if (is_power_of_2(default_nslabs))
return false;
default_nslabs = roundup_pow_of_two(default_nslabs);
return true;
}
/**
* swiotlb_adjust_nareas() - adjust the number of areas and slots
* @nareas: Desired number of areas. Zero is treated as 1.
*
* Adjust the default number of areas in a memory pool.
* The default size of the memory pool may also change to meet minimum area
* size requirements.
*/
static void swiotlb_adjust_nareas(unsigned int nareas)
{
if (!nareas)
nareas = 1;
else if (!is_power_of_2(nareas))
nareas = roundup_pow_of_two(nareas);
default_nareas = nareas;
pr_info("area num %d.\n", nareas);
if (round_up_default_nslabs())
pr_info("SWIOTLB bounce buffer size roundup to %luMB",
(default_nslabs << IO_TLB_SHIFT) >> 20);
}
/**
* limit_nareas() - get the maximum number of areas for a given memory pool size
* @nareas: Desired number of areas.
* @nslots: Total number of slots in the memory pool.
*
* Limit the number of areas to the maximum possible number of areas in
* a memory pool of the given size.
*
* Return: Maximum possible number of areas.
*/
static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
{
if (nslots < nareas * IO_TLB_SEGSIZE)
return nslots / IO_TLB_SEGSIZE;
return nareas;
}
static int __init
setup_io_tlb_npages(char *str)
{
if (isdigit(*str)) {
/* avoid tail segment of size < IO_TLB_SEGSIZE */
default_nslabs =
ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
}
if (*str == ',')
++str;
if (isdigit(*str))
swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
if (*str == ',')
++str;
if (!strcmp(str, "force"))
swiotlb_force_bounce = true;
else if (!strcmp(str, "noforce"))
swiotlb_force_disable = true;
return 0;
}
early_param("swiotlb", setup_io_tlb_npages);
unsigned long swiotlb_size_or_default(void)
{
return default_nslabs << IO_TLB_SHIFT;
}
void __init swiotlb_adjust_size(unsigned long size)
{
/*
* If swiotlb parameter has not been specified, give a chance to
* architectures such as those supporting memory encryption to
* adjust/expand SWIOTLB size for their use.
*/
if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
return;
size = ALIGN(size, IO_TLB_SIZE);
default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
if (round_up_default_nslabs())
size = default_nslabs << IO_TLB_SHIFT;
pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
}
void swiotlb_print_info(void)
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
if (!mem->nslabs) {
pr_warn("No low mem\n");
return;
}
pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
(mem->nslabs << IO_TLB_SHIFT) >> 20);
}
static inline unsigned long io_tlb_offset(unsigned long val)
{
return val & (IO_TLB_SEGSIZE - 1);
}
static inline unsigned long nr_slots(u64 val)
{
return DIV_ROUND_UP(val, IO_TLB_SIZE);
}
/*
* Early SWIOTLB allocation may be too early to allow an architecture to
* perform the desired operations. This function allows the architecture to
* call SWIOTLB when the operations are possible. It needs to be called
* before the SWIOTLB memory is used.
*/
void __init swiotlb_update_mem_attributes(void)
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long bytes;
if (!mem->nslabs || mem->late_alloc)
return;
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
}
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
unsigned long nslabs, bool late_alloc, unsigned int nareas)
{
void *vaddr = phys_to_virt(start);
unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
mem->nslabs = nslabs;
mem->start = start;
mem->end = mem->start + bytes;
mem->late_alloc = late_alloc;
mem->nareas = nareas;
mem->area_nslabs = nslabs / mem->nareas;
for (i = 0; i < mem->nareas; i++) {
spin_lock_init(&mem->areas[i].lock);
mem->areas[i].index = 0;
mem->areas[i].used = 0;
}
for (i = 0; i < mem->nslabs; i++) {
mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
mem->nslabs - i);
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
mem->slots[i].alloc_size = 0;
mem->slots[i].pad_slots = 0;
}
memset(vaddr, 0, bytes);
mem->vaddr = vaddr;
return;
}
/**
* add_mem_pool() - add a memory pool to the allocator
* @mem: Software IO TLB allocator.
* @pool: Memory pool to be added.
*/
static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
{
#ifdef CONFIG_SWIOTLB_DYNAMIC
spin_lock(&mem->lock);
list_add_rcu(&pool->node, &mem->pools);
mem->nslabs += pool->nslabs;
spin_unlock(&mem->lock);
#else
mem->nslabs = pool->nslabs;
#endif
}
static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
unsigned int flags,
int (*remap)(void *tlb, unsigned long nslabs))
{
size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
void *tlb;
/*
* By default allocate the bounce buffer memory from low memory, but
* allow to pick a location everywhere for hypervisors with guest
* memory encryption.
*/
if (flags & SWIOTLB_ANY)
tlb = memblock_alloc(bytes, PAGE_SIZE);
else
tlb = memblock_alloc_low(bytes, PAGE_SIZE);
if (!tlb) {
pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
__func__, bytes);
return NULL;
}
if (remap && remap(tlb, nslabs) < 0) {
memblock_free(tlb, PAGE_ALIGN(bytes));
pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
return NULL;
}
return tlb;
}
/*
* Statically reserve bounce buffer space and initialize bounce buffer data
* structures for the software IO TLB used to implement the DMA API.
*/
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
int (*remap)(void *tlb, unsigned long nslabs))
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long nslabs;
unsigned int nareas;
size_t alloc_size;
void *tlb;
if (!addressing_limit && !swiotlb_force_bounce)
return;
if (swiotlb_force_disable)
return;
io_tlb_default_mem.force_bounce =
swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
#ifdef CONFIG_SWIOTLB_DYNAMIC
if (!remap)
io_tlb_default_mem.can_grow = true;
if (flags & SWIOTLB_ANY)
io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
else
io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
#endif
if (!default_nareas)
swiotlb_adjust_nareas(num_possible_cpus());
nslabs = default_nslabs;
nareas = limit_nareas(default_nareas, nslabs);
while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
if (nslabs <= IO_TLB_MIN_SLABS)
return;
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
nareas = limit_nareas(nareas, nslabs);
}
if (default_nslabs != nslabs) {
pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
default_nslabs, nslabs);
default_nslabs = nslabs;
}
alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
if (!mem->slots) {
pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
__func__, alloc_size, PAGE_SIZE);
return;
}
mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
nareas), SMP_CACHE_BYTES);
if (!mem->areas) {
pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
return;
}
swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas);
add_mem_pool(&io_tlb_default_mem, mem);
if (flags & SWIOTLB_VERBOSE)
swiotlb_print_info();
}
void __init swiotlb_init(bool addressing_limit, unsigned int flags)
{
swiotlb_init_remap(addressing_limit, flags, NULL);
}
/*
* Systems with larger DMA zones (those that don't support ISA) can
* initialize the swiotlb later using the slab allocator if needed.
* This should be just like above, but with some error catching.
*/
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
int (*remap)(void *tlb, unsigned long nslabs))
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
unsigned int nareas;
unsigned char *vstart = NULL;
unsigned int order, area_order;
bool retried = false;
int rc = 0;
if (io_tlb_default_mem.nslabs)
return 0;
if (swiotlb_force_disable)
return 0;
io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
#ifdef CONFIG_SWIOTLB_DYNAMIC
if (!remap)
io_tlb_default_mem.can_grow = true;
if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
io_tlb_default_mem.phys_limit = DMA_BIT_MASK(zone_dma_bits);
else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
io_tlb_default_mem.phys_limit = DMA_BIT_MASK(32);
else
io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
#endif
if (!default_nareas)
swiotlb_adjust_nareas(num_possible_cpus());
retry:
order = get_order(nslabs << IO_TLB_SHIFT);
nslabs = SLABS_PER_PAGE << order;
while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
order);
if (vstart)
break;
order--;
nslabs = SLABS_PER_PAGE << order;
retried = true;
}
if (!vstart)
return -ENOMEM;
if (remap)
rc = remap(vstart, nslabs);
if (rc) {
free_pages((unsigned long)vstart, order);
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
if (nslabs < IO_TLB_MIN_SLABS)
return rc;
retried = true;
goto retry;
}
if (retried) {
pr_warn("only able to allocate %ld MB\n",
(PAGE_SIZE << order) >> 20);
}
nareas = limit_nareas(default_nareas, nslabs);
area_order = get_order(array_size(sizeof(*mem->areas), nareas));
mem->areas = (struct io_tlb_area *)
__get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
if (!mem->areas)
goto error_area;
mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(array_size(sizeof(*mem->slots), nslabs)));
if (!mem->slots)
goto error_slots;
set_memory_decrypted((unsigned long)vstart,
(nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
nareas);
add_mem_pool(&io_tlb_default_mem, mem);
swiotlb_print_info();
return 0;
error_slots:
free_pages((unsigned long)mem->areas, area_order);
error_area:
free_pages((unsigned long)vstart, order);
return -ENOMEM;
}
void __init swiotlb_exit(void)
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long tbl_vaddr;
size_t tbl_size, slots_size;
unsigned int area_order;
if (swiotlb_force_bounce)
return;
if (!mem->nslabs)
return;
pr_info("tearing down default memory pool\n");
tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
tbl_size = PAGE_ALIGN(mem->end - mem->start);
slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
if (mem->late_alloc) {
area_order = get_order(array_size(sizeof(*mem->areas),
mem->nareas));
free_pages((unsigned long)mem->areas, area_order);
free_pages(tbl_vaddr, get_order(tbl_size));
free_pages((unsigned long)mem->slots, get_order(slots_size));
} else {
memblock_free_late(__pa(mem->areas),
array_size(sizeof(*mem->areas), mem->nareas));
memblock_free_late(mem->start, tbl_size);
memblock_free_late(__pa(mem->slots), slots_size);
}
memset(mem, 0, sizeof(*mem));
}
#ifdef CONFIG_SWIOTLB_DYNAMIC
/**
* alloc_dma_pages() - allocate pages to be used for DMA
* @gfp: GFP flags for the allocation.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
*
* Allocate pages from the buddy allocator. If successful, make the allocated
* pages decrypted that they can be used for DMA.
*
* Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
* if the allocated physical address was above @phys_limit.
*/
static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
{
unsigned int order = get_order(bytes);
struct page *page;
phys_addr_t paddr;
void *vaddr;
page = alloc_pages(gfp, order);
if (!page)
return NULL;
paddr = page_to_phys(page);
if (paddr + bytes - 1 > phys_limit) {
__free_pages(page, order);
return ERR_PTR(-EAGAIN);
}
vaddr = phys_to_virt(paddr);
if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
goto error;
return page;
error:
/* Intentional leak if pages cannot be encrypted again. */
if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(page, order);
return NULL;
}
/**
* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
* @dev: Device for which a memory pool is allocated.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
* @gfp: GFP flags for the allocation.
*
* Return: Allocated pages, or %NULL on allocation failure.
*/
static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
u64 phys_limit, gfp_t gfp)
{
struct page *page;
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
*/
if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
void *vaddr;
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
dma_coherent_ok);
}
gfp &= ~GFP_ZONEMASK;
if (phys_limit <= DMA_BIT_MASK(zone_dma_bits))
gfp |= __GFP_DMA;
else if (phys_limit <= DMA_BIT_MASK(32))
gfp |= __GFP_DMA32;
while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
phys_limit < DMA_BIT_MASK(64) &&
!(gfp & (__GFP_DMA32 | __GFP_DMA)))
gfp |= __GFP_DMA32;
else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
!(gfp & __GFP_DMA))
gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
else
return NULL;
}
return page;
}
/**
* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
* @vaddr: Virtual address of the buffer.
* @bytes: Size of the buffer.
*/
static void swiotlb_free_tlb(void *vaddr, size_t bytes)
{
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(NULL, vaddr, bytes))
return;
/* Intentional leak if pages cannot be encrypted again. */
if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(virt_to_page(vaddr), get_order(bytes));
}
/**
* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
* @dev: Device for which a memory pool is allocated.
* @minslabs: Minimum number of slabs.
* @nslabs: Desired (maximum) number of slabs.
* @nareas: Number of areas.
* @phys_limit: Maximum DMA buffer physical address.
* @gfp: GFP flags for the allocations.
*
* Allocate and initialize a new IO TLB memory pool. The actual number of
* slabs may be reduced if allocation of @nslabs fails. If even
* @minslabs cannot be allocated, this function fails.
*
* Return: New memory pool, or %NULL on allocation failure.
*/
static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
unsigned long minslabs, unsigned long nslabs,
unsigned int nareas, u64 phys_limit, gfp_t gfp)
{
struct io_tlb_pool *pool;
unsigned int slot_order;
struct page *tlb;
size_t pool_size;
size_t tlb_size;
if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
nareas = limit_nareas(nareas, nslabs);
}
pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
pool = kzalloc(pool_size, gfp);
if (!pool)
goto error;
pool->areas = (void *)pool + sizeof(*pool);
tlb_size = nslabs << IO_TLB_SHIFT;
while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
if (nslabs <= minslabs)
goto error_tlb;
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
nareas = limit_nareas(nareas, nslabs);
tlb_size = nslabs << IO_TLB_SHIFT;
}
slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
pool->slots = (struct io_tlb_slot *)
__get_free_pages(gfp, slot_order);
if (!pool->slots)
goto error_slots;
swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
return pool;
error_slots:
swiotlb_free_tlb(page_address(tlb), tlb_size);
error_tlb:
kfree(pool);
error:
return NULL;
}
/**
* swiotlb_dyn_alloc() - dynamic memory pool allocation worker
* @work: Pointer to dyn_alloc in struct io_tlb_mem.
*/
static void swiotlb_dyn_alloc(struct work_struct *work)
{
struct io_tlb_mem *mem =
container_of(work, struct io_tlb_mem, dyn_alloc);
struct io_tlb_pool *pool;
pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
default_nareas, mem->phys_limit, GFP_KERNEL);
if (!pool) {
pr_warn_ratelimited("Failed to allocate new pool");
return;
}
add_mem_pool(mem, pool);
}
/**
* swiotlb_dyn_free() - RCU callback to free a memory pool
* @rcu: RCU head in the corresponding struct io_tlb_pool.
*/
static void swiotlb_dyn_free(struct rcu_head *rcu)
{
struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
size_t tlb_size = pool->end - pool->start;
free_pages((unsigned long)pool->slots, get_order(slots_size));
swiotlb_free_tlb(pool->vaddr, tlb_size);
kfree(pool);
}
/**
* swiotlb_find_pool() - find the IO TLB pool for a physical address
* @dev: Device which has mapped the DMA buffer.
* @paddr: Physical address within the DMA buffer.
*
* Find the IO TLB memory pool descriptor which contains the given physical
* address, if any.
*
* Return: Memory pool which contains @paddr, or %NULL if none.
*/
struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
rcu_read_lock();
list_for_each_entry_rcu(pool, &mem->pools, node) {
if (paddr >= pool->start && paddr < pool->end)
goto out;
}
list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
if (paddr >= pool->start && paddr < pool->end)
goto out;
}
pool = NULL;
out:
rcu_read_unlock();
return pool;
}
/**
* swiotlb_del_pool() - remove an IO TLB pool from a device
* @dev: Owning device.
* @pool: Memory pool to be removed.
*/
static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
{
unsigned long flags;
spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
list_del_rcu(&pool->node);
spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
call_rcu(&pool->rcu, swiotlb_dyn_free);
}
#endif /* CONFIG_SWIOTLB_DYNAMIC */
/**
* swiotlb_dev_init() - initialize swiotlb fields in &struct device
* @dev: Device to be initialized.
*/
void swiotlb_dev_init(struct device *dev)
{
dev->dma_io_tlb_mem = &io_tlb_default_mem;
#ifdef CONFIG_SWIOTLB_DYNAMIC
INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
spin_lock_init(&dev->dma_io_tlb_lock);
dev->dma_uses_io_tlb = false;
#endif
}
/**
* swiotlb_align_offset() - Get required offset into an IO TLB allocation.
* @dev: Owning device.
* @align_mask: Allocation alignment mask.
* @addr: DMA address.
*
* Return the minimum offset from the start of an IO TLB allocation which is
* required for a given buffer address and allocation alignment to keep the
* device happy.
*
* First, the address bits covered by min_align_mask must be identical in the
* original address and the bounce buffer address. High bits are preserved by
* choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
* padding bytes before the bounce buffer.
*
* Second, @align_mask specifies which bits of the first allocated slot must
* be zero. This may require allocating additional padding slots, and then the
* offset (in bytes) from the first such padding slot is returned.
*/
static unsigned int swiotlb_align_offset(struct device *dev,
unsigned int align_mask, u64 addr)
{
return addr & dma_get_min_align_mask(dev) &
(align_mask | (IO_TLB_SIZE - 1));
}
/*
* Bounce: copy the swiotlb buffer from or back to the original dma location
*/
static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
enum dma_data_direction dir)
{
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
phys_addr_t orig_addr = mem->slots[index].orig_addr;
size_t alloc_size = mem->slots[index].alloc_size;
unsigned long pfn = PFN_DOWN(orig_addr);
unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
int tlb_offset;
if (orig_addr == INVALID_PHYS_ADDR)
return;
/*
* It's valid for tlb_offset to be negative. This can happen when the
* "offset" returned by swiotlb_align_offset() is non-zero, and the
* tlb_addr is pointing within the first "offset" bytes of the second
* or subsequent slots of the allocated swiotlb area. While it's not
* valid for tlb_addr to be pointing within the first "offset" bytes
* of the first slot, there's no way to check for such an error since
* this function can't distinguish the first slot from the second and
* subsequent slots.
*/
tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
swiotlb_align_offset(dev, 0, orig_addr);
orig_addr += tlb_offset;
alloc_size -= tlb_offset;
if (size > alloc_size) {
dev_WARN_ONCE(dev, 1,
"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
alloc_size, size);
size = alloc_size;
}
if (PageHighMem(pfn_to_page(pfn))) {
unsigned int offset = orig_addr & ~PAGE_MASK;
struct page *page;
unsigned int sz = 0;
unsigned long flags;
while (size) {
sz = min_t(size_t, PAGE_SIZE - offset, size);
local_irq_save(flags);
page = pfn_to_page(pfn);
if (dir == DMA_TO_DEVICE)
memcpy_from_page(vaddr, page, offset, sz);
else
memcpy_to_page(page, offset, vaddr, sz);
local_irq_restore(flags);
size -= sz;
pfn++;
vaddr += sz;
offset = 0;
}
} else if (dir == DMA_TO_DEVICE) {
memcpy(vaddr, phys_to_virt(orig_addr), size);
} else {
memcpy(phys_to_virt(orig_addr), vaddr, size);
}
}
static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
{
return start + (idx << IO_TLB_SHIFT);
}
/*
* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
*/
static inline unsigned long get_max_slots(unsigned long boundary_mask)
{
return (boundary_mask >> IO_TLB_SHIFT) + 1;
}
static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
{
if (index >= mem->area_nslabs)
return 0;
return index;
}
/*
* Track the total used slots with a global atomic value in order to have
* correct information to determine the high water mark. The mem_used()
* function gives imprecise results because there's no locking across
* multiple areas.
*/
#ifdef CONFIG_DEBUG_FS
static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
{
unsigned long old_hiwater, new_used;
new_used = atomic_long_add_return(nslots, &mem->total_used);
old_hiwater = atomic_long_read(&mem->used_hiwater);
do {
if (new_used <= old_hiwater)
break;
} while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
&old_hiwater, new_used));
}
static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
{
atomic_long_sub(nslots, &mem->total_used);
}
#else /* !CONFIG_DEBUG_FS */
static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
{
}
static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
{
}
#endif /* CONFIG_DEBUG_FS */
#ifdef CONFIG_SWIOTLB_DYNAMIC
#ifdef CONFIG_DEBUG_FS
static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
{
atomic_long_add(nslots, &mem->transient_nslabs);
}
static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
{
atomic_long_sub(nslots, &mem->transient_nslabs);
}
#else /* !CONFIG_DEBUG_FS */
static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
{
}
static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
{
}
#endif /* CONFIG_DEBUG_FS */
#endif /* CONFIG_SWIOTLB_DYNAMIC */
/**
* swiotlb_search_pool_area() - search one memory area in one pool
* @dev: Device which maps the buffer.
* @pool: Memory pool to be searched.
* @area_index: Index of the IO TLB memory area to be searched.
* @orig_addr: Original (non-bounced) IO buffer address.
* @alloc_size: Total requested size of the bounce buffer,
* including initial alignment padding.
* @alloc_align_mask: Required alignment of the allocated buffer.
*
* Find a suitable sequence of IO TLB entries for the request and allocate
* a buffer from the given IO TLB memory area.
* This function takes care of locking.
*
* Return: Index of the first allocated slot, or -1 on error.
*/
static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
int area_index, phys_addr_t orig_addr, size_t alloc_size,
unsigned int alloc_align_mask)
{
struct io_tlb_area *area = pool->areas + area_index;
unsigned long boundary_mask = dma_get_seg_boundary(dev);
dma_addr_t tbl_dma_addr =
phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
unsigned int nslots = nr_slots(alloc_size), stride;
unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
unsigned int index, slots_checked, count = 0, i;
unsigned long flags;
unsigned int slot_base;
unsigned int slot_index;
BUG_ON(!nslots);
BUG_ON(area_index >= pool->nareas);
/*
* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
* page-aligned in the absence of any other alignment requirements.
* 'alloc_align_mask' was later introduced to specify the alignment
* explicitly, however this is passed as zero for streaming mappings
* and so we preserve the old behaviour there in case any drivers are
* relying on it.
*/
if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
alloc_align_mask = PAGE_SIZE - 1;
/*
* Ensure that the allocation is at least slot-aligned and update
* 'iotlb_align_mask' to ignore bits that will be preserved when
* offsetting into the allocation.
*/
alloc_align_mask |= (IO_TLB_SIZE - 1);
iotlb_align_mask &= ~alloc_align_mask;
/*
* For mappings with an alignment requirement don't bother looping to
* unaligned slots once we found an aligned one.
*/
stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
spin_lock_irqsave(&area->lock, flags);
if (unlikely(nslots > pool->area_nslabs - area->used))
goto not_found;
slot_base = area_index * pool->area_nslabs;
index = area->index;
for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
phys_addr_t tlb_addr;
slot_index = slot_base + index;
tlb_addr = slot_addr(tbl_dma_addr, slot_index);
if ((tlb_addr & alloc_align_mask) ||
(orig_addr && (tlb_addr & iotlb_align_mask) !=
(orig_addr & iotlb_align_mask))) {
index = wrap_area_index(pool, index + 1);
slots_checked++;
continue;
}
if (!iommu_is_span_boundary(slot_index, nslots,
nr_slots(tbl_dma_addr),
max_slots)) {
if (pool->slots[slot_index].list >= nslots)
goto found;
}
index = wrap_area_index(pool, index + stride);
slots_checked += stride;
}
not_found:
spin_unlock_irqrestore(&area->lock, flags);
return -1;
found:
/*
* If we find a slot that indicates we have 'nslots' number of
* contiguous buffers, we allocate the buffers from that slot onwards
* and set the list of free entries to '0' indicating unavailable.
*/
for (i = slot_index; i < slot_index + nslots; i++) {
pool->slots[i].list = 0;
pool->slots[i].alloc_size = alloc_size - (offset +
((i - slot_index) << IO_TLB_SHIFT));
}
for (i = slot_index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
pool->slots[i].list; i--)
pool->slots[i].list = ++count;
/*
* Update the indices to avoid searching in the next round.
*/
area->index = wrap_area_index(pool, index + nslots);
area->used += nslots;
spin_unlock_irqrestore(&area->lock, flags);
inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
return slot_index;
}
#ifdef CONFIG_SWIOTLB_DYNAMIC
/**
* swiotlb_search_area() - search one memory area in all pools
* @dev: Device which maps the buffer.
* @start_cpu: Start CPU number.
* @cpu_offset: Offset from @start_cpu.
* @orig_addr: Original (non-bounced) IO buffer address.
* @alloc_size: Total requested size of the bounce buffer,
* including initial alignment padding.
* @alloc_align_mask: Required alignment of the allocated buffer.
* @retpool: Used memory pool, updated on return.
*
* Search one memory area in all pools for a sequence of slots that match the
* allocation constraints.
*
* Return: Index of the first allocated slot, or -1 on error.
*/
static int swiotlb_search_area(struct device *dev, int start_cpu,
int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
int area_index;
int index = -1;
rcu_read_lock();
list_for_each_entry_rcu(pool, &mem->pools, node) {
if (cpu_offset >= pool->nareas)
continue;
area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
index = swiotlb_search_pool_area(dev, pool, area_index,
orig_addr, alloc_size,
alloc_align_mask);
if (index >= 0) {
*retpool = pool;
break;
}
}
rcu_read_unlock();
return index;
}
/**
* swiotlb_find_slots() - search for slots in the whole swiotlb
* @dev: Device which maps the buffer.
* @orig_addr: Original (non-bounced) IO buffer address.
* @alloc_size: Total requested size of the bounce buffer,
* including initial alignment padding.
* @alloc_align_mask: Required alignment of the allocated buffer.
* @retpool: Used memory pool, updated on return.
*
* Search through the whole software IO TLB to find a sequence of slots that
* match the allocation constraints.
*
* Return: Index of the first allocated slot, or -1 on error.
*/
static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
size_t alloc_size, unsigned int alloc_align_mask,
struct io_tlb_pool **retpool)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
unsigned long nslabs;
unsigned long flags;
u64 phys_limit;
int cpu, i;
int index;
if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
return -1;
cpu = raw_smp_processor_id();
for (i = 0; i < default_nareas; ++i) {
index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
alloc_align_mask, &pool);
if (index >= 0)
goto found;
}
if (!mem->can_grow)
return -1;
schedule_work(&mem->dyn_alloc);
nslabs = nr_slots(alloc_size);
phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
GFP_NOWAIT | __GFP_NOWARN);
if (!pool)
return -1;
index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
alloc_size, alloc_align_mask);
if (index < 0) {
swiotlb_dyn_free(&pool->rcu);
return -1;
}
pool->transient = true;
spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
inc_transient_used(mem, pool->nslabs);
found:
WRITE_ONCE(dev->dma_uses_io_tlb, true);
/*
* The general barrier orders reads and writes against a presumed store
* of the SWIOTLB buffer address by a device driver (to a driver private
* data structure). It serves two purposes.
*
* First, the store to dev->dma_uses_io_tlb must be ordered before the
* presumed store. This guarantees that the returned buffer address
* cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
*
* Second, the load from mem->pools must be ordered before the same
* presumed store. This guarantees that the returned buffer address
* cannot be observed by another CPU before an update of the RCU list
* that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
* atomicity).
*
* See also the comment in is_swiotlb_buffer().
*/
smp_mb();
*retpool = pool;
return index;
}
#else /* !CONFIG_SWIOTLB_DYNAMIC */
static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
size_t alloc_size, unsigned int alloc_align_mask,
struct io_tlb_pool **retpool)
{
struct io_tlb_pool *pool;
int start, i;
int index;
*retpool = pool = &dev->dma_io_tlb_mem->defpool;
i = start = raw_smp_processor_id() & (pool->nareas - 1);
do {
index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
alloc_size, alloc_align_mask);
if (index >= 0)
return index;
if (++i >= pool->nareas)
i = 0;
} while (i != start);
return -1;
}
#endif /* CONFIG_SWIOTLB_DYNAMIC */
#ifdef CONFIG_DEBUG_FS
/**
* mem_used() - get number of used slots in an allocator
* @mem: Software IO TLB allocator.
*
* The result is accurate in this version of the function, because an atomic
* counter is available if CONFIG_DEBUG_FS is set.
*
* Return: Number of used slots.
*/
static unsigned long mem_used(struct io_tlb_mem *mem)
{
return atomic_long_read(&mem->total_used);
}
#else /* !CONFIG_DEBUG_FS */
/**
* mem_pool_used() - get number of used slots in a memory pool
* @pool: Software IO TLB memory pool.
*
* The result is not accurate, see mem_used().
*
* Return: Approximate number of used slots.
*/
static unsigned long mem_pool_used(struct io_tlb_pool *pool)
{
int i;
unsigned long used = 0;
for (i = 0; i < pool->nareas; i++)
used += pool->areas[i].used;
return used;
}
/**
* mem_used() - get number of used slots in an allocator
* @mem: Software IO TLB allocator.
*
* The result is not accurate, because there is no locking of individual
* areas.
*
* Return: Approximate number of used slots.
*/
static unsigned long mem_used(struct io_tlb_mem *mem)
{
#ifdef CONFIG_SWIOTLB_DYNAMIC
struct io_tlb_pool *pool;
unsigned long used = 0;
rcu_read_lock();
list_for_each_entry_rcu(pool, &mem->pools, node)
used += mem_pool_used(pool);
rcu_read_unlock();
return used;
#else
return mem_pool_used(&mem->defpool);
#endif
}
#endif /* CONFIG_DEBUG_FS */
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, size_t alloc_size,
unsigned int alloc_align_mask, enum dma_data_direction dir,
unsigned long attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned int offset;
struct io_tlb_pool *pool;
unsigned int i;
int index;
phys_addr_t tlb_addr;
unsigned short pad_slots;
if (!mem || !mem->nslabs) {
dev_warn_ratelimited(dev,
"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
return (phys_addr_t)DMA_MAPPING_ERROR;
}
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
if (mapping_size > alloc_size) {
dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
mapping_size, alloc_size);
return (phys_addr_t)DMA_MAPPING_ERROR;
}
offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
index = swiotlb_find_slots(dev, orig_addr,
alloc_size + offset, alloc_align_mask, &pool);
if (index == -1) {
if (!(attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
alloc_size, mem->nslabs, mem_used(mem));
return (phys_addr_t)DMA_MAPPING_ERROR;
}
/*
* Save away the mapping from the original address to the DMA address.
* This is needed when we sync the memory. Then we sync the buffer if
* needed.
*/
pad_slots = offset >> IO_TLB_SHIFT;
offset &= (IO_TLB_SIZE - 1);
index += pad_slots;
pool->slots[index].pad_slots = pad_slots;
for (i = 0; i < nr_slots(alloc_size + offset); i++)
pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
tlb_addr = slot_addr(pool->start, index) + offset;
/*
* When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
* the original buffer to the TLB buffer before initiating DMA in order
* to preserve the original's data if the device does a partial write,
* i.e. if the device doesn't overwrite the entire buffer. Preserving
* the original data, even if it's garbage, is necessary to match
* hardware behavior. Use of swiotlb is supposed to be transparent,
* i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
*/
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE);
return tlb_addr;
}
static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
{
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
unsigned long flags;
unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
int index, nslots, aindex;
struct io_tlb_area *area;
int count, i;
index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
index -= mem->slots[index].pad_slots;
nslots = nr_slots(mem->slots[index].alloc_size + offset);
aindex = index / mem->area_nslabs;
area = &mem->areas[aindex];
/*
* Return the buffer to the free list by setting the corresponding
* entries to indicate the number of contiguous entries available.
* While returning the entries to the free list, we merge the entries
* with slots below and above the pool being returned.
*/
BUG_ON(aindex >= mem->nareas);
spin_lock_irqsave(&area->lock, flags);
if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
count = mem->slots[index + nslots].list;
else
count = 0;
/*
* Step 1: return the slots to the free list, merging the slots with
* superceeding slots
*/
for (i = index + nslots - 1; i >= index; i--) {
mem->slots[i].list = ++count;
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
mem->slots[i].alloc_size = 0;
mem->slots[i].pad_slots = 0;
}
/*
* Step 2: merge the returned slots with the preceding slots, if
* available (non zero)
*/
for (i = index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
i--)
mem->slots[i].list = ++count;
area->used -= nslots;
spin_unlock_irqrestore(&area->lock, flags);
dec_used(dev->dma_io_tlb_mem, nslots);
}
#ifdef CONFIG_SWIOTLB_DYNAMIC
/**
* swiotlb_del_transient() - delete a transient memory pool
* @dev: Device which mapped the buffer.
* @tlb_addr: Physical address within a bounce buffer.
*
* Check whether the address belongs to a transient SWIOTLB memory pool.
* If yes, then delete the pool.
*
* Return: %true if @tlb_addr belonged to a transient pool that was released.
*/
static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr)
{
struct io_tlb_pool *pool;
pool = swiotlb_find_pool(dev, tlb_addr);
if (!pool->transient)
return false;
dec_used(dev->dma_io_tlb_mem, pool->nslabs);
swiotlb_del_pool(dev, pool);
dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);
return true;
}
#else /* !CONFIG_SWIOTLB_DYNAMIC */
static inline bool swiotlb_del_transient(struct device *dev,
phys_addr_t tlb_addr)
{
return false;
}
#endif /* CONFIG_SWIOTLB_DYNAMIC */
/*
* tlb_addr is the physical address of the bounce buffer to unmap.
*/
void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
size_t mapping_size, enum dma_data_direction dir,
unsigned long attrs)
{
/*
* First, sync the memory before unmapping the entry
*/
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
if (swiotlb_del_transient(dev, tlb_addr))
return;
swiotlb_release_slots(dev, tlb_addr);
}
void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir)
{
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE);
else
BUG_ON(dir != DMA_FROM_DEVICE);
}
void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir)
{
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE);
else
BUG_ON(dir != DMA_TO_DEVICE);
}
/*
* Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
* to the device copy the data into it as well.
*/
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
phys_addr_t swiotlb_addr;
dma_addr_t dma_addr;
trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir,
attrs);
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
/* Ensure that the address returned is DMA'ble */
dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC);
dev_WARN_ONCE(dev, 1,
"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
return DMA_MAPPING_ERROR;
}
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
arch_sync_dma_for_device(swiotlb_addr, size, dir);
return dma_addr;
}
size_t swiotlb_max_mapping_size(struct device *dev)
{
int min_align_mask = dma_get_min_align_mask(dev);
int min_align = 0;
/*
* swiotlb_find_slots() skips slots according to
* min align mask. This affects max mapping size.
* Take it into acount here.
*/
if (min_align_mask)
min_align = roundup(min_align_mask, IO_TLB_SIZE);
return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
}
/**
* is_swiotlb_allocated() - check if the default software IO TLB is initialized
*/
bool is_swiotlb_allocated(void)
{
return io_tlb_default_mem.nslabs;
}
bool is_swiotlb_active(struct device *dev)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
return mem && mem->nslabs;
}
/**
* default_swiotlb_base() - get the base address of the default SWIOTLB
*
* Get the lowest physical address used by the default software IO TLB pool.
*/
phys_addr_t default_swiotlb_base(void)
{
#ifdef CONFIG_SWIOTLB_DYNAMIC
io_tlb_default_mem.can_grow = false;
#endif
return io_tlb_default_mem.defpool.start;
}
/**
* default_swiotlb_limit() - get the address limit of the default SWIOTLB
*
* Get the highest physical address used by the default software IO TLB pool.
*/
phys_addr_t default_swiotlb_limit(void)
{
#ifdef CONFIG_SWIOTLB_DYNAMIC
return io_tlb_default_mem.phys_limit;
#else
return io_tlb_default_mem.defpool.end - 1;
#endif
}
#ifdef CONFIG_DEBUG_FS
#ifdef CONFIG_SWIOTLB_DYNAMIC
static unsigned long mem_transient_used(struct io_tlb_mem *mem)
{
return atomic_long_read(&mem->transient_nslabs);
}
static int io_tlb_transient_used_get(void *data, u64 *val)
{
struct io_tlb_mem *mem = data;
*val = mem_transient_used(mem);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
NULL, "%llu\n");
#endif /* CONFIG_SWIOTLB_DYNAMIC */
static int io_tlb_used_get(void *data, u64 *val)
{
struct io_tlb_mem *mem = data;
*val = mem_used(mem);
return 0;
}
static int io_tlb_hiwater_get(void *data, u64 *val)
{
struct io_tlb_mem *mem = data;
*val = atomic_long_read(&mem->used_hiwater);
return 0;
}
static int io_tlb_hiwater_set(void *data, u64 val)
{
struct io_tlb_mem *mem = data;
/* Only allow setting to zero */
if (val != 0)
return -EINVAL;
atomic_long_set(&mem->used_hiwater, val);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
io_tlb_hiwater_set, "%llu\n");
static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
const char *dirname)
{
mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
if (!mem->nslabs)
return;
debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
&fops_io_tlb_used);
debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
&fops_io_tlb_hiwater);
#ifdef CONFIG_SWIOTLB_DYNAMIC
debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
mem, &fops_io_tlb_transient_used);
#endif
}
static int __init swiotlb_create_default_debugfs(void)
{
swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
return 0;
}
late_initcall(swiotlb_create_default_debugfs);
#else /* !CONFIG_DEBUG_FS */
static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
const char *dirname)
{
}
#endif /* CONFIG_DEBUG_FS */
#ifdef CONFIG_DMA_RESTRICTED_POOL
struct page *swiotlb_alloc(struct device *dev, size_t size)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
phys_addr_t tlb_addr;
unsigned int align;
int index;
if (!mem)
return NULL;
align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
index = swiotlb_find_slots(dev, 0, size, align, &pool);
if (index == -1)
return NULL;
tlb_addr = slot_addr(pool->start, index);
if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
&tlb_addr);
swiotlb_release_slots(dev, tlb_addr);
return NULL;
}
return pfn_to_page(PFN_DOWN(tlb_addr));
}
bool swiotlb_free(struct device *dev, struct page *page, size_t size)
{
phys_addr_t tlb_addr = page_to_phys(page);
if (!is_swiotlb_buffer(dev, tlb_addr))
return false;
swiotlb_release_slots(dev, tlb_addr);
return true;
}
static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
struct device *dev)
{
struct io_tlb_mem *mem = rmem->priv;
unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
/* Set Per-device io tlb area to one */
unsigned int nareas = 1;
if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
return -EINVAL;
}
/*
* Since multiple devices can share the same pool, the private data,
* io_tlb_mem struct, will be initialized by the first device attached
* to it.
*/
if (!mem) {
struct io_tlb_pool *pool;
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
if (!mem)
return -ENOMEM;
pool = &mem->defpool;
pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
if (!pool->slots) {
kfree(mem);
return -ENOMEM;
}
pool->areas = kcalloc(nareas, sizeof(*pool->areas),
GFP_KERNEL);
if (!pool->areas) {
kfree(pool->slots);
kfree(mem);
return -ENOMEM;
}
set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
rmem->size >> PAGE_SHIFT);
swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
false, nareas);
mem->force_bounce = true;
mem->for_alloc = true;
#ifdef CONFIG_SWIOTLB_DYNAMIC
spin_lock_init(&mem->lock);
INIT_LIST_HEAD_RCU(&mem->pools);
#endif
add_mem_pool(mem, pool);
rmem->priv = mem;
swiotlb_create_debugfs_files(mem, rmem->name);
}
dev->dma_io_tlb_mem = mem;
return 0;
}
static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
struct device *dev)
{
dev->dma_io_tlb_mem = &io_tlb_default_mem;
}
static const struct reserved_mem_ops rmem_swiotlb_ops = {
.device_init = rmem_swiotlb_device_init,
.device_release = rmem_swiotlb_device_release,
};
static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
{
unsigned long node = rmem->fdt_node;
if (of_get_flat_dt_prop(node, "reusable", NULL) ||
of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
of_get_flat_dt_prop(node, "no-map", NULL))
return -EINVAL;
rmem->ops = &rmem_swiotlb_ops;
pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
&rmem->base, (unsigned long)rmem->size / SZ_1M);
return 0;
}
RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
#endif /* CONFIG_DMA_RESTRICTED_POOL */
// SPDX-License-Identifier: GPL-2.0-only
/*
* (C) 1997 Linus Torvalds
* (C) 1999 Andrea Arcangeli <andrea@suse.de> (dynamic inode allocation)
*/
#include <linux/export.h>
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/mm.h>
#include <linux/backing-dev.h>
#include <linux/hash.h>
#include <linux/swap.h>
#include <linux/security.h>
#include <linux/cdev.h>
#include <linux/memblock.h>
#include <linux/fsnotify.h>
#include <linux/mount.h>
#include <linux/posix_acl.h>
#include <linux/buffer_head.h> /* for inode_has_buffers */
#include <linux/ratelimit.h>
#include <linux/list_lru.h>
#include <linux/iversion.h>
#include <linux/rw_hint.h>
#include <trace/events/writeback.h>
#include "internal.h"
/*
* Inode locking rules:
*
* inode->i_lock protects:
* inode->i_state, inode->i_hash, __iget(), inode->i_io_list
* Inode LRU list locks protect:
* inode->i_sb->s_inode_lru, inode->i_lru
* inode->i_sb->s_inode_list_lock protects:
* inode->i_sb->s_inodes, inode->i_sb_list
* bdi->wb.list_lock protects:
* bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_io_list
* inode_hash_lock protects:
* inode_hashtable, inode->i_hash
*
* Lock ordering:
*
* inode->i_sb->s_inode_list_lock
* inode->i_lock
* Inode LRU list locks
*
* bdi->wb.list_lock
* inode->i_lock
*
* inode_hash_lock
* inode->i_sb->s_inode_list_lock
* inode->i_lock
*
* iunique_lock
* inode_hash_lock
*/
static unsigned int i_hash_mask __ro_after_init;
static unsigned int i_hash_shift __ro_after_init;
static struct hlist_head *inode_hashtable __ro_after_init;
static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
/*
* Empty aops. Can be used for the cases where the user does not
* define any of the address_space operations.
*/
const struct address_space_operations empty_aops = {
};
EXPORT_SYMBOL(empty_aops);
static DEFINE_PER_CPU(unsigned long, nr_inodes);
static DEFINE_PER_CPU(unsigned long, nr_unused);
static struct kmem_cache *inode_cachep __ro_after_init;
static long get_nr_inodes(void)
{
int i;
long sum = 0;
for_each_possible_cpu(i)
sum += per_cpu(nr_inodes, i);
return sum < 0 ? 0 : sum;
}
static inline long get_nr_inodes_unused(void)
{
int i;
long sum = 0;
for_each_possible_cpu(i)
sum += per_cpu(nr_unused, i);
return sum < 0 ? 0 : sum;
}
long get_nr_dirty_inodes(void)
{
/* not actually dirty inodes, but a wild approximation */
long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
return nr_dirty > 0 ? nr_dirty : 0;
}
/*
* Handle nr_inode sysctl
*/
#ifdef CONFIG_SYSCTL
/*
* Statistics gathering..
*/
static struct inodes_stat_t inodes_stat;
static int proc_nr_inodes(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
inodes_stat.nr_inodes = get_nr_inodes();
inodes_stat.nr_unused = get_nr_inodes_unused();
return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
}
static struct ctl_table inodes_sysctls[] = {
{
.procname = "inode-nr",
.data = &inodes_stat,
.maxlen = 2*sizeof(long),
.mode = 0444,
.proc_handler = proc_nr_inodes,
},
{
.procname = "inode-state",
.data = &inodes_stat,
.maxlen = 7*sizeof(long),
.mode = 0444,
.proc_handler = proc_nr_inodes,
},
};
static int __init init_fs_inode_sysctls(void)
{
register_sysctl_init("fs", inodes_sysctls);
return 0;
}
early_initcall(init_fs_inode_sysctls);
#endif
static int no_open(struct inode *inode, struct file *file)
{
return -ENXIO;
}
/**
* inode_init_always - perform inode structure initialisation
* @sb: superblock inode belongs to
* @inode: inode to initialise
*
* These are initializations that need to be done on every inode
* allocation as the fields are not initialised by slab allocation.
*/
int inode_init_always(struct super_block *sb, struct inode *inode)
{
static const struct inode_operations empty_iops;
static const struct file_operations no_open_fops = {.open = no_open};
struct address_space *const mapping = &inode->i_data;
inode->i_sb = sb;
inode->i_blkbits = sb->s_blocksize_bits;
inode->i_flags = 0;
atomic64_set(&inode->i_sequence, 0);
atomic_set(&inode->i_count, 1);
inode->i_op = &empty_iops;
inode->i_fop = &no_open_fops;
inode->i_ino = 0;
inode->__i_nlink = 1;
inode->i_opflags = 0;
if (sb->s_xattr)
inode->i_opflags |= IOP_XATTR;
i_uid_write(inode, 0);
i_gid_write(inode, 0);
atomic_set(&inode->i_writecount, 0);
inode->i_size = 0;
inode->i_write_hint = WRITE_LIFE_NOT_SET;
inode->i_blocks = 0;
inode->i_bytes = 0;
inode->i_generation = 0;
inode->i_pipe = NULL;
inode->i_cdev = NULL;
inode->i_link = NULL;
inode->i_dir_seq = 0;
inode->i_rdev = 0;
inode->dirtied_when = 0;
#ifdef CONFIG_CGROUP_WRITEBACK
inode->i_wb_frn_winner = 0;
inode->i_wb_frn_avg_time = 0;
inode->i_wb_frn_history = 0;
#endif
spin_lock_init(&inode->i_lock);
lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
init_rwsem(&inode->i_rwsem);
lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
atomic_set(&inode->i_dio_count, 0);
mapping->a_ops = &empty_aops;
mapping->host = inode;
mapping->flags = 0;
mapping->wb_err = 0;
atomic_set(&mapping->i_mmap_writable, 0);
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
atomic_set(&mapping->nr_thps, 0);
#endif
mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
mapping->i_private_data = NULL;
mapping->writeback_index = 0;
init_rwsem(&mapping->invalidate_lock);
lockdep_set_class_and_name(&mapping->invalidate_lock,
&sb->s_type->invalidate_lock_key,
"mapping.invalidate_lock");
if (sb->s_iflags & SB_I_STABLE_WRITES)
mapping_set_stable_writes(mapping); inode->i_private = NULL;
inode->i_mapping = mapping;
INIT_HLIST_HEAD(&inode->i_dentry); /* buggered by rcu freeing */
#ifdef CONFIG_FS_POSIX_ACL
inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
#endif
#ifdef CONFIG_FSNOTIFY
inode->i_fsnotify_mask = 0;
#endif
inode->i_flctx = NULL;
if (unlikely(security_inode_alloc(inode)))
return -ENOMEM;
this_cpu_inc(nr_inodes); return 0;
}
EXPORT_SYMBOL(inode_init_always);
void free_inode_nonrcu(struct inode *inode)
{
kmem_cache_free(inode_cachep, inode);
}
EXPORT_SYMBOL(free_inode_nonrcu);
static void i_callback(struct rcu_head *head)
{
struct inode *inode = container_of(head, struct inode, i_rcu);
if (inode->free_inode)
inode->free_inode(inode);
else
free_inode_nonrcu(inode);
}
static struct inode *alloc_inode(struct super_block *sb)
{
const struct super_operations *ops = sb->s_op;
struct inode *inode;
if (ops->alloc_inode)
inode = ops->alloc_inode(sb);
else
inode = alloc_inode_sb(sb, inode_cachep, GFP_KERNEL); if (!inode)
return NULL;
if (unlikely(inode_init_always(sb, inode))) { if (ops->destroy_inode) { ops->destroy_inode(inode);
if (!ops->free_inode)
return NULL;
}
inode->free_inode = ops->free_inode; i_callback(&inode->i_rcu);
return NULL;
}
return inode;
}
void __destroy_inode(struct inode *inode)
{
BUG_ON(inode_has_buffers(inode)); inode_detach_wb(inode); security_inode_free(inode);
fsnotify_inode_delete(inode);
locks_free_lock_context(inode);
if (!inode->i_nlink) {
WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0); atomic_long_dec(&inode->i_sb->s_remove_count);
}
#ifdef CONFIG_FS_POSIX_ACL
if (inode->i_acl && !is_uncached_acl(inode->i_acl)) posix_acl_release(inode->i_acl); if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl)) posix_acl_release(inode->i_default_acl);
#endif
this_cpu_dec(nr_inodes);
}
EXPORT_SYMBOL(__destroy_inode);
static void destroy_inode(struct inode *inode)
{
const struct super_operations *ops = inode->i_sb->s_op; BUG_ON(!list_empty(&inode->i_lru)); __destroy_inode(inode);
if (ops->destroy_inode) {
ops->destroy_inode(inode);
if (!ops->free_inode)
return;
}
inode->free_inode = ops->free_inode;
call_rcu(&inode->i_rcu, i_callback);
}
/**
* drop_nlink - directly drop an inode's link count
* @inode: inode
*
* This is a low-level filesystem helper to replace any
* direct filesystem manipulation of i_nlink. In cases
* where we are attempting to track writes to the
* filesystem, a decrement to zero means an imminent
* write when the file is truncated and actually unlinked
* on the filesystem.
*/
void drop_nlink(struct inode *inode)
{
WARN_ON(inode->i_nlink == 0); inode->__i_nlink--;
if (!inode->i_nlink)
atomic_long_inc(&inode->i_sb->s_remove_count);
}
EXPORT_SYMBOL(drop_nlink);
/**
* clear_nlink - directly zero an inode's link count
* @inode: inode
*
* This is a low-level filesystem helper to replace any
* direct filesystem manipulation of i_nlink. See
* drop_nlink() for why we care about i_nlink hitting zero.
*/
void clear_nlink(struct inode *inode)
{
if (inode->i_nlink) { inode->__i_nlink = 0;
atomic_long_inc(&inode->i_sb->s_remove_count);
}
}
EXPORT_SYMBOL(clear_nlink);
/**
* set_nlink - directly set an inode's link count
* @inode: inode
* @nlink: new nlink (should be non-zero)
*
* This is a low-level filesystem helper to replace any
* direct filesystem manipulation of i_nlink.
*/
void set_nlink(struct inode *inode, unsigned int nlink)
{
if (!nlink) {
clear_nlink(inode);
} else {
/* Yes, some filesystems do change nlink from zero to one */
if (inode->i_nlink == 0)
atomic_long_dec(&inode->i_sb->s_remove_count);
inode->__i_nlink = nlink;
}
}
EXPORT_SYMBOL(set_nlink);
/**
* inc_nlink - directly increment an inode's link count
* @inode: inode
*
* This is a low-level filesystem helper to replace any
* direct filesystem manipulation of i_nlink. Currently,
* it is only here for parity with dec_nlink().
*/
void inc_nlink(struct inode *inode)
{
if (unlikely(inode->i_nlink == 0)) {
WARN_ON(!(inode->i_state & I_LINKABLE));
atomic_long_dec(&inode->i_sb->s_remove_count);
}
inode->__i_nlink++;
}
EXPORT_SYMBOL(inc_nlink);
static void __address_space_init_once(struct address_space *mapping)
{
xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
init_rwsem(&mapping->i_mmap_rwsem);
INIT_LIST_HEAD(&mapping->i_private_list);
spin_lock_init(&mapping->i_private_lock);
mapping->i_mmap = RB_ROOT_CACHED;
}
void address_space_init_once(struct address_space *mapping)
{
memset(mapping, 0, sizeof(*mapping));
__address_space_init_once(mapping);
}
EXPORT_SYMBOL(address_space_init_once);
/*
* These are initializations that only need to be done
* once, because the fields are idempotent across use
* of the inode, so let the slab aware of that.
*/
void inode_init_once(struct inode *inode)
{
memset(inode, 0, sizeof(*inode));
INIT_HLIST_NODE(&inode->i_hash);
INIT_LIST_HEAD(&inode->i_devices);
INIT_LIST_HEAD(&inode->i_io_list);
INIT_LIST_HEAD(&inode->i_wb_list);
INIT_LIST_HEAD(&inode->i_lru);
INIT_LIST_HEAD(&inode->i_sb_list);
__address_space_init_once(&inode->i_data);
i_size_ordered_init(inode);
}
EXPORT_SYMBOL(inode_init_once);
static void init_once(void *foo)
{
struct inode *inode = (struct inode *) foo;
inode_init_once(inode);
}
/*
* inode->i_lock must be held
*/
void __iget(struct inode *inode)
{
atomic_inc(&inode->i_count);
}
/*
* get additional reference to inode; caller must already hold one.
*/
void ihold(struct inode *inode)
{
WARN_ON(atomic_inc_return(&inode->i_count) < 2);
}
EXPORT_SYMBOL(ihold);
static void __inode_add_lru(struct inode *inode, bool rotate)
{
if (inode->i_state & (I_DIRTY_ALL | I_SYNC | I_FREEING | I_WILL_FREE))
return;
if (atomic_read(&inode->i_count))
return;
if (!(inode->i_sb->s_flags & SB_ACTIVE))
return;
if (!mapping_shrinkable(&inode->i_data))
return;
if (list_lru_add_obj(&inode->i_sb->s_inode_lru, &inode->i_lru))
this_cpu_inc(nr_unused);
else if (rotate)
inode->i_state |= I_REFERENCED;
}
/*
* Add inode to LRU if needed (inode is unused and clean).
*
* Needs inode->i_lock held.
*/
void inode_add_lru(struct inode *inode)
{
__inode_add_lru(inode, false);
}
static void inode_lru_list_del(struct inode *inode)
{
if (list_lru_del_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) this_cpu_dec(nr_unused);
}
/**
* inode_sb_list_add - add inode to the superblock list of inodes
* @inode: inode to add
*/
void inode_sb_list_add(struct inode *inode)
{
spin_lock(&inode->i_sb->s_inode_list_lock);
list_add(&inode->i_sb_list, &inode->i_sb->s_inodes); spin_unlock(&inode->i_sb->s_inode_list_lock);
}
EXPORT_SYMBOL_GPL(inode_sb_list_add);
static inline void inode_sb_list_del(struct inode *inode)
{
if (!list_empty(&inode->i_sb_list)) { spin_lock(&inode->i_sb->s_inode_list_lock); list_del_init(&inode->i_sb_list);
spin_unlock(&inode->i_sb->s_inode_list_lock);
}
}
static unsigned long hash(struct super_block *sb, unsigned long hashval)
{
unsigned long tmp;
tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
L1_CACHE_BYTES;
tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
return tmp & i_hash_mask;
}
/**
* __insert_inode_hash - hash an inode
* @inode: unhashed inode
* @hashval: unsigned long value used to locate this object in the
* inode_hashtable.
*
* Add an inode to the inode hash for this superblock.
*/
void __insert_inode_hash(struct inode *inode, unsigned long hashval)
{
struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
spin_lock(&inode_hash_lock);
spin_lock(&inode->i_lock);
hlist_add_head_rcu(&inode->i_hash, b);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_hash_lock);
}
EXPORT_SYMBOL(__insert_inode_hash);
/**
* __remove_inode_hash - remove an inode from the hash
* @inode: inode to unhash
*
* Remove an inode from the superblock.
*/
void __remove_inode_hash(struct inode *inode)
{
spin_lock(&inode_hash_lock);
spin_lock(&inode->i_lock);
hlist_del_init_rcu(&inode->i_hash); spin_unlock(&inode->i_lock);
spin_unlock(&inode_hash_lock);
}
EXPORT_SYMBOL(__remove_inode_hash);
void dump_mapping(const struct address_space *mapping)
{
struct inode *host;
const struct address_space_operations *a_ops;
struct hlist_node *dentry_first;
struct dentry *dentry_ptr;
struct dentry dentry;
unsigned long ino;
/*
* If mapping is an invalid pointer, we don't want to crash
* accessing it, so probe everything depending on it carefully.
*/
if (get_kernel_nofault(host, &mapping->host) ||
get_kernel_nofault(a_ops, &mapping->a_ops)) {
pr_warn("invalid mapping:%px\n", mapping);
return;
}
if (!host) {
pr_warn("aops:%ps\n", a_ops);
return;
}
if (get_kernel_nofault(dentry_first, &host->i_dentry.first) ||
get_kernel_nofault(ino, &host->i_ino)) {
pr_warn("aops:%ps invalid inode:%px\n", a_ops, host);
return;
}
if (!dentry_first) {
pr_warn("aops:%ps ino:%lx\n", a_ops, ino);
return;
}
dentry_ptr = container_of(dentry_first, struct dentry, d_u.d_alias);
if (get_kernel_nofault(dentry, dentry_ptr) ||
!dentry.d_parent || !dentry.d_name.name) {
pr_warn("aops:%ps ino:%lx invalid dentry:%px\n",
a_ops, ino, dentry_ptr);
return;
}
/*
* if dentry is corrupted, the %pd handler may still crash,
* but it's unlikely that we reach here with a corrupt mapping
*/
pr_warn("aops:%ps ino:%lx dentry name:\"%pd\"\n", a_ops, ino, &dentry);
}
void clear_inode(struct inode *inode)
{
/*
* We have to cycle the i_pages lock here because reclaim can be in the
* process of removing the last page (in __filemap_remove_folio())
* and we must not free the mapping under it.
*/
xa_lock_irq(&inode->i_data.i_pages); BUG_ON(inode->i_data.nrpages);
/*
* Almost always, mapping_empty(&inode->i_data) here; but there are
* two known and long-standing ways in which nodes may get left behind
* (when deep radix-tree node allocation failed partway; or when THP
* collapse_file() failed). Until those two known cases are cleaned up,
* or a cleanup function is called here, do not BUG_ON(!mapping_empty),
* nor even WARN_ON(!mapping_empty).
*/
xa_unlock_irq(&inode->i_data.i_pages); BUG_ON(!list_empty(&inode->i_data.i_private_list)); BUG_ON(!(inode->i_state & I_FREEING)); BUG_ON(inode->i_state & I_CLEAR); BUG_ON(!list_empty(&inode->i_wb_list));
/* don't need i_lock here, no concurrent mods to i_state */
inode->i_state = I_FREEING | I_CLEAR;
}
EXPORT_SYMBOL(clear_inode);
/*
* Free the inode passed in, removing it from the lists it is still connected
* to. We remove any pages still attached to the inode and wait for any IO that
* is still in progress before finally destroying the inode.
*
* An inode must already be marked I_FREEING so that we avoid the inode being
* moved back onto lists if we race with other code that manipulates the lists
* (e.g. writeback_single_inode). The caller is responsible for setting this.
*
* An inode must already be removed from the LRU list before being evicted from
* the cache. This should occur atomically with setting the I_FREEING state
* flag, so no inodes here should ever be on the LRU when being evicted.
*/
static void evict(struct inode *inode)
{
const struct super_operations *op = inode->i_sb->s_op; BUG_ON(!(inode->i_state & I_FREEING)); BUG_ON(!list_empty(&inode->i_lru));
if (!list_empty(&inode->i_io_list))
inode_io_list_del(inode); inode_sb_list_del(inode);
/*
* Wait for flusher thread to be done with the inode so that filesystem
* does not start destroying it while writeback is still running. Since
* the inode has I_FREEING set, flusher thread won't start new work on
* the inode. We just have to wait for running writeback to finish.
*/
inode_wait_for_writeback(inode);
if (op->evict_inode) {
op->evict_inode(inode);
} else {
truncate_inode_pages_final(&inode->i_data);
clear_inode(inode);
}
if (S_ISCHR(inode->i_mode) && inode->i_cdev) cd_forget(inode); remove_inode_hash(inode); spin_lock(&inode->i_lock);
wake_up_bit(&inode->i_state, __I_NEW);
BUG_ON(inode->i_state != (I_FREEING | I_CLEAR)); spin_unlock(&inode->i_lock);
destroy_inode(inode);
}
/*
* dispose_list - dispose of the contents of a local list
* @head: the head of the list to free
*
* Dispose-list gets a local list with local inodes in it, so it doesn't
* need to worry about list corruption and SMP locks.
*/
static void dispose_list(struct list_head *head)
{
while (!list_empty(head)) {
struct inode *inode;
inode = list_first_entry(head, struct inode, i_lru);
list_del_init(&inode->i_lru);
evict(inode);
cond_resched();
}
}
/**
* evict_inodes - evict all evictable inodes for a superblock
* @sb: superblock to operate on
*
* Make sure that no inodes with zero refcount are retained. This is
* called by superblock shutdown after having SB_ACTIVE flag removed,
* so any inode reaching zero refcount during or after that call will
* be immediately evicted.
*/
void evict_inodes(struct super_block *sb)
{
struct inode *inode, *next;
LIST_HEAD(dispose);
again:
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
if (atomic_read(&inode->i_count))
continue;
spin_lock(&inode->i_lock);
if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
spin_unlock(&inode->i_lock);
continue;
}
inode->i_state |= I_FREEING;
inode_lru_list_del(inode);
spin_unlock(&inode->i_lock);
list_add(&inode->i_lru, &dispose);
/*
* We can have a ton of inodes to evict at unmount time given
* enough memory, check to see if we need to go to sleep for a
* bit so we don't livelock.
*/
if (need_resched()) {
spin_unlock(&sb->s_inode_list_lock);
cond_resched();
dispose_list(&dispose);
goto again;
}
}
spin_unlock(&sb->s_inode_list_lock);
dispose_list(&dispose);
}
EXPORT_SYMBOL_GPL(evict_inodes);
/**
* invalidate_inodes - attempt to free all inodes on a superblock
* @sb: superblock to operate on
*
* Attempts to free all inodes (including dirty inodes) for a given superblock.
*/
void invalidate_inodes(struct super_block *sb)
{
struct inode *inode, *next;
LIST_HEAD(dispose);
again:
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
spin_lock(&inode->i_lock);
if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
spin_unlock(&inode->i_lock);
continue;
}
if (atomic_read(&inode->i_count)) {
spin_unlock(&inode->i_lock);
continue;
}
inode->i_state |= I_FREEING;
inode_lru_list_del(inode);
spin_unlock(&inode->i_lock);
list_add(&inode->i_lru, &dispose);
if (need_resched()) {
spin_unlock(&sb->s_inode_list_lock);
cond_resched();
dispose_list(&dispose);
goto again;
}
}
spin_unlock(&sb->s_inode_list_lock);
dispose_list(&dispose);
}
/*
* Isolate the inode from the LRU in preparation for freeing it.
*
* If the inode has the I_REFERENCED flag set, then it means that it has been
* used recently - the flag is set in iput_final(). When we encounter such an
* inode, clear the flag and move it to the back of the LRU so it gets another
* pass through the LRU before it gets reclaimed. This is necessary because of
* the fact we are doing lazy LRU updates to minimise lock contention so the
* LRU does not have strict ordering. Hence we don't want to reclaim inodes
* with this flag set because they are the inodes that are out of order.
*/
static enum lru_status inode_lru_isolate(struct list_head *item,
struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
{
struct list_head *freeable = arg;
struct inode *inode = container_of(item, struct inode, i_lru);
/*
* We are inverting the lru lock/inode->i_lock here, so use a
* trylock. If we fail to get the lock, just skip it.
*/
if (!spin_trylock(&inode->i_lock))
return LRU_SKIP;
/*
* Inodes can get referenced, redirtied, or repopulated while
* they're already on the LRU, and this can make them
* unreclaimable for a while. Remove them lazily here; iput,
* sync, or the last page cache deletion will requeue them.
*/
if (atomic_read(&inode->i_count) ||
(inode->i_state & ~I_REFERENCED) ||
!mapping_shrinkable(&inode->i_data)) {
list_lru_isolate(lru, &inode->i_lru);
spin_unlock(&inode->i_lock);
this_cpu_dec(nr_unused);
return LRU_REMOVED;
}
/* Recently referenced inodes get one more pass */
if (inode->i_state & I_REFERENCED) {
inode->i_state &= ~I_REFERENCED;
spin_unlock(&inode->i_lock);
return LRU_ROTATE;
}
/*
* On highmem systems, mapping_shrinkable() permits dropping
* page cache in order to free up struct inodes: lowmem might
* be under pressure before the cache inside the highmem zone.
*/
if (inode_has_buffers(inode) || !mapping_empty(&inode->i_data)) {
__iget(inode);
spin_unlock(&inode->i_lock);
spin_unlock(lru_lock);
if (remove_inode_buffers(inode)) {
unsigned long reap;
reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
if (current_is_kswapd())
__count_vm_events(KSWAPD_INODESTEAL, reap);
else
__count_vm_events(PGINODESTEAL, reap);
mm_account_reclaimed_pages(reap);
}
iput(inode);
spin_lock(lru_lock);
return LRU_RETRY;
}
WARN_ON(inode->i_state & I_NEW);
inode->i_state |= I_FREEING;
list_lru_isolate_move(lru, &inode->i_lru, freeable);
spin_unlock(&inode->i_lock);
this_cpu_dec(nr_unused);
return LRU_REMOVED;
}
/*
* Walk the superblock inode LRU for freeable inodes and attempt to free them.
* This is called from the superblock shrinker function with a number of inodes
* to trim from the LRU. Inodes to be freed are moved to a temporary list and
* then are freed outside inode_lock by dispose_list().
*/
long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
{
LIST_HEAD(freeable);
long freed;
freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
inode_lru_isolate, &freeable);
dispose_list(&freeable);
return freed;
}
static void __wait_on_freeing_inode(struct inode *inode);
/*
* Called with the inode lock held.
*/
static struct inode *find_inode(struct super_block *sb,
struct hlist_head *head,
int (*test)(struct inode *, void *),
void *data)
{
struct inode *inode = NULL;
repeat:
hlist_for_each_entry(inode, head, i_hash) {
if (inode->i_sb != sb)
continue;
if (!test(inode, data))
continue;
spin_lock(&inode->i_lock);
if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
__wait_on_freeing_inode(inode);
goto repeat;
}
if (unlikely(inode->i_state & I_CREATING)) {
spin_unlock(&inode->i_lock);
return ERR_PTR(-ESTALE);
}
__iget(inode);
spin_unlock(&inode->i_lock);
return inode;
}
return NULL;
}
/*
* find_inode_fast is the fast path version of find_inode, see the comment at
* iget_locked for details.
*/
static struct inode *find_inode_fast(struct super_block *sb,
struct hlist_head *head, unsigned long ino)
{
struct inode *inode = NULL;
repeat:
hlist_for_each_entry(inode, head, i_hash) {
if (inode->i_ino != ino)
continue;
if (inode->i_sb != sb)
continue;
spin_lock(&inode->i_lock);
if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
__wait_on_freeing_inode(inode);
goto repeat;
}
if (unlikely(inode->i_state & I_CREATING)) {
spin_unlock(&inode->i_lock);
return ERR_PTR(-ESTALE);
}
__iget(inode);
spin_unlock(&inode->i_lock);
return inode;
}
return NULL;
}
/*
* Each cpu owns a range of LAST_INO_BATCH numbers.
* 'shared_last_ino' is dirtied only once out of LAST_INO_BATCH allocations,
* to renew the exhausted range.
*
* This does not significantly increase overflow rate because every CPU can
* consume at most LAST_INO_BATCH-1 unused inode numbers. So there is
* NR_CPUS*(LAST_INO_BATCH-1) wastage. At 4096 and 1024, this is ~0.1% of the
* 2^32 range, and is a worst-case. Even a 50% wastage would only increase
* overflow rate by 2x, which does not seem too significant.
*
* On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
* error if st_ino won't fit in target struct field. Use 32bit counter
* here to attempt to avoid that.
*/
#define LAST_INO_BATCH 1024
static DEFINE_PER_CPU(unsigned int, last_ino);
unsigned int get_next_ino(void)
{
unsigned int *p = &get_cpu_var(last_ino);
unsigned int res = *p;
#ifdef CONFIG_SMP
if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
static atomic_t shared_last_ino;
int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
res = next - LAST_INO_BATCH;
}
#endif
res++;
/* get_next_ino should not provide a 0 inode number */
if (unlikely(!res))
res++;
*p = res;
put_cpu_var(last_ino);
return res;
}
EXPORT_SYMBOL(get_next_ino);
/**
* new_inode_pseudo - obtain an inode
* @sb: superblock
*
* Allocates a new inode for given superblock.
* Inode wont be chained in superblock s_inodes list
* This means :
* - fs can't be unmount
* - quotas, fsnotify, writeback can't work
*/
struct inode *new_inode_pseudo(struct super_block *sb)
{
struct inode *inode = alloc_inode(sb);
if (inode) {
spin_lock(&inode->i_lock);
inode->i_state = 0;
spin_unlock(&inode->i_lock);
}
return inode;
}
/**
* new_inode - obtain an inode
* @sb: superblock
*
* Allocates a new inode for given superblock. The default gfp_mask
* for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE.
* If HIGHMEM pages are unsuitable or it is known that pages allocated
* for the page cache are not reclaimable or migratable,
* mapping_set_gfp_mask() must be called with suitable flags on the
* newly created inode's mapping
*
*/
struct inode *new_inode(struct super_block *sb)
{
struct inode *inode;
inode = new_inode_pseudo(sb);
if (inode)
inode_sb_list_add(inode); return inode;
}
EXPORT_SYMBOL(new_inode);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void lockdep_annotate_inode_mutex_key(struct inode *inode)
{
if (S_ISDIR(inode->i_mode)) { struct file_system_type *type = inode->i_sb->s_type;
/* Set new key only if filesystem hasn't already changed it */
if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
/*
* ensure nobody is actually holding i_mutex
*/
// mutex_destroy(&inode->i_mutex);
init_rwsem(&inode->i_rwsem);
lockdep_set_class(&inode->i_rwsem,
&type->i_mutex_dir_key);
}
}
}
EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
#endif
/**
* unlock_new_inode - clear the I_NEW state and wake up any waiters
* @inode: new inode to unlock
*
* Called when the inode is fully initialised to clear the new state of the
* inode and wake up anyone waiting for the inode to finish initialisation.
*/
void unlock_new_inode(struct inode *inode)
{
lockdep_annotate_inode_mutex_key(inode);
spin_lock(&inode->i_lock);
WARN_ON(!(inode->i_state & I_NEW));
inode->i_state &= ~I_NEW & ~I_CREATING;
smp_mb();
wake_up_bit(&inode->i_state, __I_NEW);
spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL(unlock_new_inode);
void discard_new_inode(struct inode *inode)
{
lockdep_annotate_inode_mutex_key(inode);
spin_lock(&inode->i_lock);
WARN_ON(!(inode->i_state & I_NEW));
inode->i_state &= ~I_NEW;
smp_mb();
wake_up_bit(&inode->i_state, __I_NEW);
spin_unlock(&inode->i_lock);
iput(inode);
}
EXPORT_SYMBOL(discard_new_inode);
/**
* lock_two_nondirectories - take two i_mutexes on non-directory objects
*
* Lock any non-NULL argument. Passed objects must not be directories.
* Zero, one or two objects may be locked by this function.
*
* @inode1: first inode to lock
* @inode2: second inode to lock
*/
void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
{
if (inode1)
WARN_ON_ONCE(S_ISDIR(inode1->i_mode));
if (inode2)
WARN_ON_ONCE(S_ISDIR(inode2->i_mode));
if (inode1 > inode2)
swap(inode1, inode2);
if (inode1)
inode_lock(inode1);
if (inode2 && inode2 != inode1)
inode_lock_nested(inode2, I_MUTEX_NONDIR2);
}
EXPORT_SYMBOL(lock_two_nondirectories);
/**
* unlock_two_nondirectories - release locks from lock_two_nondirectories()
* @inode1: first inode to unlock
* @inode2: second inode to unlock
*/
void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
{
if (inode1) {
WARN_ON_ONCE(S_ISDIR(inode1->i_mode));
inode_unlock(inode1);
}
if (inode2 && inode2 != inode1) {
WARN_ON_ONCE(S_ISDIR(inode2->i_mode));
inode_unlock(inode2);
}
}
EXPORT_SYMBOL(unlock_two_nondirectories);
/**
* inode_insert5 - obtain an inode from a mounted file system
* @inode: pre-allocated inode to use for insert to cache
* @hashval: hash value (usually inode number) to get
* @test: callback used for comparisons between inodes
* @set: callback used to initialize a new struct inode
* @data: opaque data pointer to pass to @test and @set
*
* Search for the inode specified by @hashval and @data in the inode cache,
* and if present it is return it with an increased reference count. This is
* a variant of iget5_locked() for callers that don't want to fail on memory
* allocation of inode.
*
* If the inode is not in cache, insert the pre-allocated inode to cache and
* return it locked, hashed, and with the I_NEW flag set. The file system gets
* to fill it in before unlocking it via unlock_new_inode().
*
* Note both @test and @set are called with the inode_hash_lock held, so can't
* sleep.
*/
struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
int (*test)(struct inode *, void *),
int (*set)(struct inode *, void *), void *data)
{
struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
struct inode *old;
again:
spin_lock(&inode_hash_lock);
old = find_inode(inode->i_sb, head, test, data);
if (unlikely(old)) {
/*
* Uhhuh, somebody else created the same inode under us.
* Use the old inode instead of the preallocated one.
*/
spin_unlock(&inode_hash_lock);
if (IS_ERR(old))
return NULL;
wait_on_inode(old);
if (unlikely(inode_unhashed(old))) {
iput(old);
goto again;
}
return old;
}
if (set && unlikely(set(inode, data))) {
inode = NULL;
goto unlock;
}
/*
* Return the locked inode with I_NEW set, the
* caller is responsible for filling in the contents
*/
spin_lock(&inode->i_lock);
inode->i_state |= I_NEW;
hlist_add_head_rcu(&inode->i_hash, head);
spin_unlock(&inode->i_lock);
/*
* Add inode to the sb list if it's not already. It has I_NEW at this
* point, so it should be safe to test i_sb_list locklessly.
*/
if (list_empty(&inode->i_sb_list))
inode_sb_list_add(inode);
unlock:
spin_unlock(&inode_hash_lock);
return inode;
}
EXPORT_SYMBOL(inode_insert5);
/**
* iget5_locked - obtain an inode from a mounted file system
* @sb: super block of file system
* @hashval: hash value (usually inode number) to get
* @test: callback used for comparisons between inodes
* @set: callback used to initialize a new struct inode
* @data: opaque data pointer to pass to @test and @set
*
* Search for the inode specified by @hashval and @data in the inode cache,
* and if present it is return it with an increased reference count. This is
* a generalized version of iget_locked() for file systems where the inode
* number is not sufficient for unique identification of an inode.
*
* If the inode is not in cache, allocate a new inode and return it locked,
* hashed, and with the I_NEW flag set. The file system gets to fill it in
* before unlocking it via unlock_new_inode().
*
* Note both @test and @set are called with the inode_hash_lock held, so can't
* sleep.
*/
struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
int (*test)(struct inode *, void *),
int (*set)(struct inode *, void *), void *data)
{
struct inode *inode = ilookup5(sb, hashval, test, data);
if (!inode) {
struct inode *new = alloc_inode(sb);
if (new) {
new->i_state = 0;
inode = inode_insert5(new, hashval, test, set, data);
if (unlikely(inode != new))
destroy_inode(new);
}
}
return inode;
}
EXPORT_SYMBOL(iget5_locked);
/**
* iget_locked - obtain an inode from a mounted file system
* @sb: super block of file system
* @ino: inode number to get
*
* Search for the inode specified by @ino in the inode cache and if present
* return it with an increased reference count. This is for file systems
* where the inode number is sufficient for unique identification of an inode.
*
* If the inode is not in cache, allocate a new inode and return it locked,
* hashed, and with the I_NEW flag set. The file system gets to fill it in
* before unlocking it via unlock_new_inode().
*/
struct inode *iget_locked(struct super_block *sb, unsigned long ino)
{
struct hlist_head *head = inode_hashtable + hash(sb, ino);
struct inode *inode;
again:
spin_lock(&inode_hash_lock);
inode = find_inode_fast(sb, head, ino);
spin_unlock(&inode_hash_lock);
if (inode) {
if (IS_ERR(inode))
return NULL;
wait_on_inode(inode);
if (unlikely(inode_unhashed(inode))) {
iput(inode);
goto again;
}
return inode;
}
inode = alloc_inode(sb);
if (inode) {
struct inode *old;
spin_lock(&inode_hash_lock);
/* We released the lock, so.. */
old = find_inode_fast(sb, head, ino);
if (!old) {
inode->i_ino = ino;
spin_lock(&inode->i_lock);
inode->i_state = I_NEW;
hlist_add_head_rcu(&inode->i_hash, head);
spin_unlock(&inode->i_lock);
inode_sb_list_add(inode);
spin_unlock(&inode_hash_lock);
/* Return the locked inode with I_NEW set, the
* caller is responsible for filling in the contents
*/
return inode;
}
/*
* Uhhuh, somebody else created the same inode under
* us. Use the old inode instead of the one we just
* allocated.
*/
spin_unlock(&inode_hash_lock);
destroy_inode(inode);
if (IS_ERR(old))
return NULL;
inode = old;
wait_on_inode(inode);
if (unlikely(inode_unhashed(inode))) {
iput(inode);
goto again;
}
}
return inode;
}
EXPORT_SYMBOL(iget_locked);
/*
* search the inode cache for a matching inode number.
* If we find one, then the inode number we are trying to
* allocate is not unique and so we should not use it.
*
* Returns 1 if the inode number is unique, 0 if it is not.
*/
static int test_inode_iunique(struct super_block *sb, unsigned long ino)
{
struct hlist_head *b = inode_hashtable + hash(sb, ino);
struct inode *inode;
hlist_for_each_entry_rcu(inode, b, i_hash) {
if (inode->i_ino == ino && inode->i_sb == sb)
return 0;
}
return 1;
}
/**
* iunique - get a unique inode number
* @sb: superblock
* @max_reserved: highest reserved inode number
*
* Obtain an inode number that is unique on the system for a given
* superblock. This is used by file systems that have no natural
* permanent inode numbering system. An inode number is returned that
* is higher than the reserved limit but unique.
*
* BUGS:
* With a large number of inodes live on the file system this function
* currently becomes quite slow.
*/
ino_t iunique(struct super_block *sb, ino_t max_reserved)
{
/*
* On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
* error if st_ino won't fit in target struct field. Use 32bit counter
* here to attempt to avoid that.
*/
static DEFINE_SPINLOCK(iunique_lock);
static unsigned int counter;
ino_t res;
rcu_read_lock();
spin_lock(&iunique_lock);
do {
if (counter <= max_reserved)
counter = max_reserved + 1;
res = counter++;
} while (!test_inode_iunique(sb, res));
spin_unlock(&iunique_lock);
rcu_read_unlock();
return res;
}
EXPORT_SYMBOL(iunique);
struct inode *igrab(struct inode *inode)
{
spin_lock(&inode->i_lock);
if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
__iget(inode);
spin_unlock(&inode->i_lock);
} else {
spin_unlock(&inode->i_lock);
/*
* Handle the case where s_op->clear_inode is not been
* called yet, and somebody is calling igrab
* while the inode is getting freed.
*/
inode = NULL;
}
return inode;
}
EXPORT_SYMBOL(igrab);
/**
* ilookup5_nowait - search for an inode in the inode cache
* @sb: super block of file system to search
* @hashval: hash value (usually inode number) to search for
* @test: callback used for comparisons between inodes
* @data: opaque data pointer to pass to @test
*
* Search for the inode specified by @hashval and @data in the inode cache.
* If the inode is in the cache, the inode is returned with an incremented
* reference count.
*
* Note: I_NEW is not waited upon so you have to be very careful what you do
* with the returned inode. You probably should be using ilookup5() instead.
*
* Note2: @test is called with the inode_hash_lock held, so can't sleep.
*/
struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
int (*test)(struct inode *, void *), void *data)
{
struct hlist_head *head = inode_hashtable + hash(sb, hashval);
struct inode *inode;
spin_lock(&inode_hash_lock);
inode = find_inode(sb, head, test, data);
spin_unlock(&inode_hash_lock);
return IS_ERR(inode) ? NULL : inode;
}
EXPORT_SYMBOL(ilookup5_nowait);
/**
* ilookup5 - search for an inode in the inode cache
* @sb: super block of file system to search
* @hashval: hash value (usually inode number) to search for
* @test: callback used for comparisons between inodes
* @data: opaque data pointer to pass to @test
*
* Search for the inode specified by @hashval and @data in the inode cache,
* and if the inode is in the cache, return the inode with an incremented
* reference count. Waits on I_NEW before returning the inode.
* returned with an incremented reference count.
*
* This is a generalized version of ilookup() for file systems where the
* inode number is not sufficient for unique identification of an inode.
*
* Note: @test is called with the inode_hash_lock held, so can't sleep.
*/
struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
int (*test)(struct inode *, void *), void *data)
{
struct inode *inode;
again:
inode = ilookup5_nowait(sb, hashval, test, data);
if (inode) {
wait_on_inode(inode);
if (unlikely(inode_unhashed(inode))) {
iput(inode);
goto again;
}
}
return inode;
}
EXPORT_SYMBOL(ilookup5);
/**
* ilookup - search for an inode in the inode cache
* @sb: super block of file system to search
* @ino: inode number to search for
*
* Search for the inode @ino in the inode cache, and if the inode is in the
* cache, the inode is returned with an incremented reference count.
*/
struct inode *ilookup(struct super_block *sb, unsigned long ino)
{
struct hlist_head *head = inode_hashtable + hash(sb, ino);
struct inode *inode;
again:
spin_lock(&inode_hash_lock);
inode = find_inode_fast(sb, head, ino);
spin_unlock(&inode_hash_lock);
if (inode) {
if (IS_ERR(inode))
return NULL;
wait_on_inode(inode);
if (unlikely(inode_unhashed(inode))) {
iput(inode);
goto again;
}
}
return inode;
}
EXPORT_SYMBOL(ilookup);
/**
* find_inode_nowait - find an inode in the inode cache
* @sb: super block of file system to search
* @hashval: hash value (usually inode number) to search for
* @match: callback used for comparisons between inodes
* @data: opaque data pointer to pass to @match
*
* Search for the inode specified by @hashval and @data in the inode
* cache, where the helper function @match will return 0 if the inode
* does not match, 1 if the inode does match, and -1 if the search
* should be stopped. The @match function must be responsible for
* taking the i_lock spin_lock and checking i_state for an inode being
* freed or being initialized, and incrementing the reference count
* before returning 1. It also must not sleep, since it is called with
* the inode_hash_lock spinlock held.
*
* This is a even more generalized version of ilookup5() when the
* function must never block --- find_inode() can block in
* __wait_on_freeing_inode() --- or when the caller can not increment
* the reference count because the resulting iput() might cause an
* inode eviction. The tradeoff is that the @match funtion must be
* very carefully implemented.
*/
struct inode *find_inode_nowait(struct super_block *sb,
unsigned long hashval,
int (*match)(struct inode *, unsigned long,
void *),
void *data)
{
struct hlist_head *head = inode_hashtable + hash(sb, hashval);
struct inode *inode, *ret_inode = NULL;
int mval;
spin_lock(&inode_hash_lock);
hlist_for_each_entry(inode, head, i_hash) {
if (inode->i_sb != sb)
continue;
mval = match(inode, hashval, data);
if (mval == 0)
continue;
if (mval == 1)
ret_inode = inode;
goto out;
}
out:
spin_unlock(&inode_hash_lock);
return ret_inode;
}
EXPORT_SYMBOL(find_inode_nowait);
/**
* find_inode_rcu - find an inode in the inode cache
* @sb: Super block of file system to search
* @hashval: Key to hash
* @test: Function to test match on an inode
* @data: Data for test function
*
* Search for the inode specified by @hashval and @data in the inode cache,
* where the helper function @test will return 0 if the inode does not match
* and 1 if it does. The @test function must be responsible for taking the
* i_lock spin_lock and checking i_state for an inode being freed or being
* initialized.
*
* If successful, this will return the inode for which the @test function
* returned 1 and NULL otherwise.
*
* The @test function is not permitted to take a ref on any inode presented.
* It is also not permitted to sleep.
*
* The caller must hold the RCU read lock.
*/
struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,
int (*test)(struct inode *, void *), void *data)
{
struct hlist_head *head = inode_hashtable + hash(sb, hashval);
struct inode *inode;
RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
"suspicious find_inode_rcu() usage");
hlist_for_each_entry_rcu(inode, head, i_hash) {
if (inode->i_sb == sb &&
!(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&
test(inode, data))
return inode;
}
return NULL;
}
EXPORT_SYMBOL(find_inode_rcu);
/**
* find_inode_by_ino_rcu - Find an inode in the inode cache
* @sb: Super block of file system to search
* @ino: The inode number to match
*
* Search for the inode specified by @hashval and @data in the inode cache,
* where the helper function @test will return 0 if the inode does not match
* and 1 if it does. The @test function must be responsible for taking the
* i_lock spin_lock and checking i_state for an inode being freed or being
* initialized.
*
* If successful, this will return the inode for which the @test function
* returned 1 and NULL otherwise.
*
* The @test function is not permitted to take a ref on any inode presented.
* It is also not permitted to sleep.
*
* The caller must hold the RCU read lock.
*/
struct inode *find_inode_by_ino_rcu(struct super_block *sb,
unsigned long ino)
{
struct hlist_head *head = inode_hashtable + hash(sb, ino);
struct inode *inode;
RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
"suspicious find_inode_by_ino_rcu() usage");
hlist_for_each_entry_rcu(inode, head, i_hash) {
if (inode->i_ino == ino &&
inode->i_sb == sb &&
!(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))
return inode;
}
return NULL;
}
EXPORT_SYMBOL(find_inode_by_ino_rcu);
int insert_inode_locked(struct inode *inode)
{
struct super_block *sb = inode->i_sb;
ino_t ino = inode->i_ino;
struct hlist_head *head = inode_hashtable + hash(sb, ino);
while (1) {
struct inode *old = NULL;
spin_lock(&inode_hash_lock);
hlist_for_each_entry(old, head, i_hash) {
if (old->i_ino != ino)
continue;
if (old->i_sb != sb)
continue;
spin_lock(&old->i_lock);
if (old->i_state & (I_FREEING|I_WILL_FREE)) {
spin_unlock(&old->i_lock);
continue;
}
break;
}
if (likely(!old)) {
spin_lock(&inode->i_lock);
inode->i_state |= I_NEW | I_CREATING;
hlist_add_head_rcu(&inode->i_hash, head);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_hash_lock);
return 0;
}
if (unlikely(old->i_state & I_CREATING)) {
spin_unlock(&old->i_lock);
spin_unlock(&inode_hash_lock);
return -EBUSY;
}
__iget(old);
spin_unlock(&old->i_lock);
spin_unlock(&inode_hash_lock);
wait_on_inode(old);
if (unlikely(!inode_unhashed(old))) {
iput(old);
return -EBUSY;
}
iput(old);
}
}
EXPORT_SYMBOL(insert_inode_locked);
int insert_inode_locked4(struct inode *inode, unsigned long hashval,
int (*test)(struct inode *, void *), void *data)
{
struct inode *old;
inode->i_state |= I_CREATING;
old = inode_insert5(inode, hashval, test, NULL, data);
if (old != inode) {
iput(old);
return -EBUSY;
}
return 0;
}
EXPORT_SYMBOL(insert_inode_locked4);
int generic_delete_inode(struct inode *inode)
{
return 1;
}
EXPORT_SYMBOL(generic_delete_inode);
/*
* Called when we're dropping the last reference
* to an inode.
*
* Call the FS "drop_inode()" function, defaulting to
* the legacy UNIX filesystem behaviour. If it tells
* us to evict inode, do so. Otherwise, retain inode
* in cache if fs is alive, sync and evict if fs is
* shutting down.
*/
static void iput_final(struct inode *inode)
{
struct super_block *sb = inode->i_sb;
const struct super_operations *op = inode->i_sb->s_op;
unsigned long state;
int drop;
WARN_ON(inode->i_state & I_NEW); if (op->drop_inode) drop = op->drop_inode(inode);
else
drop = generic_drop_inode(inode);
if (!drop &&
!(inode->i_state & I_DONTCACHE) && (sb->s_flags & SB_ACTIVE)) { __inode_add_lru(inode, true); spin_unlock(&inode->i_lock);
return;
}
state = inode->i_state;
if (!drop) {
WRITE_ONCE(inode->i_state, state | I_WILL_FREE);
spin_unlock(&inode->i_lock);
write_inode_now(inode, 1);
spin_lock(&inode->i_lock);
state = inode->i_state;
WARN_ON(state & I_NEW); state &= ~I_WILL_FREE;
}
WRITE_ONCE(inode->i_state, state | I_FREEING);
if (!list_empty(&inode->i_lru))
inode_lru_list_del(inode); spin_unlock(&inode->i_lock);
evict(inode);
}
/**
* iput - put an inode
* @inode: inode to put
*
* Puts an inode, dropping its usage count. If the inode use count hits
* zero, the inode is then freed and may also be destroyed.
*
* Consequently, iput() can sleep.
*/
void iput(struct inode *inode)
{
if (!inode)
return;
BUG_ON(inode->i_state & I_CLEAR);
retry:
if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) { if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) { atomic_inc(&inode->i_count);
spin_unlock(&inode->i_lock);
trace_writeback_lazytime_iput(inode); mark_inode_dirty_sync(inode);
goto retry;
}
iput_final(inode);
}
}
EXPORT_SYMBOL(iput);
#ifdef CONFIG_BLOCK
/**
* bmap - find a block number in a file
* @inode: inode owning the block number being requested
* @block: pointer containing the block to find
*
* Replaces the value in ``*block`` with the block number on the device holding
* corresponding to the requested block number in the file.
* That is, asked for block 4 of inode 1 the function will replace the
* 4 in ``*block``, with disk block relative to the disk start that holds that
* block of the file.
*
* Returns -EINVAL in case of error, 0 otherwise. If mapping falls into a
* hole, returns 0 and ``*block`` is also set to 0.
*/
int bmap(struct inode *inode, sector_t *block)
{
if (!inode->i_mapping->a_ops->bmap)
return -EINVAL;
*block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);
return 0;
}
EXPORT_SYMBOL(bmap);
#endif
/*
* With relative atime, only update atime if the previous atime is
* earlier than or equal to either the ctime or mtime,
* or if at least a day has passed since the last atime update.
*/
static bool relatime_need_update(struct vfsmount *mnt, struct inode *inode,
struct timespec64 now)
{
struct timespec64 atime, mtime, ctime;
if (!(mnt->mnt_flags & MNT_RELATIME))
return true;
/*
* Is mtime younger than or equal to atime? If yes, update atime:
*/
atime = inode_get_atime(inode);
mtime = inode_get_mtime(inode);
if (timespec64_compare(&mtime, &atime) >= 0)
return true;
/*
* Is ctime younger than or equal to atime? If yes, update atime:
*/
ctime = inode_get_ctime(inode); if (timespec64_compare(&ctime, &atime) >= 0)
return true;
/*
* Is the previous atime value older than a day? If yes,
* update atime:
*/
if ((long)(now.tv_sec - atime.tv_sec) >= 24*60*60)
return true;
/*
* Good, we can skip the atime update:
*/
return false;
}
/**
* inode_update_timestamps - update the timestamps on the inode
* @inode: inode to be updated
* @flags: S_* flags that needed to be updated
*
* The update_time function is called when an inode's timestamps need to be
* updated for a read or write operation. This function handles updating the
* actual timestamps. It's up to the caller to ensure that the inode is marked
* dirty appropriately.
*
* In the case where any of S_MTIME, S_CTIME, or S_VERSION need to be updated,
* attempt to update all three of them. S_ATIME updates can be handled
* independently of the rest.
*
* Returns a set of S_* flags indicating which values changed.
*/
int inode_update_timestamps(struct inode *inode, int flags)
{
int updated = 0;
struct timespec64 now;
if (flags & (S_MTIME|S_CTIME|S_VERSION)) { struct timespec64 ctime = inode_get_ctime(inode);
struct timespec64 mtime = inode_get_mtime(inode);
now = inode_set_ctime_current(inode);
if (!timespec64_equal(&now, &ctime))
updated |= S_CTIME;
if (!timespec64_equal(&now, &mtime)) { inode_set_mtime_to_ts(inode, now);
updated |= S_MTIME;
}
if (IS_I_VERSION(inode) && inode_maybe_inc_iversion(inode, updated)) updated |= S_VERSION;
} else {
now = current_time(inode);
}
if (flags & S_ATIME) { struct timespec64 atime = inode_get_atime(inode); if (!timespec64_equal(&now, &atime)) { inode_set_atime_to_ts(inode, now);
updated |= S_ATIME;
}
}
return updated;
}
EXPORT_SYMBOL(inode_update_timestamps);
/**
* generic_update_time - update the timestamps on the inode
* @inode: inode to be updated
* @flags: S_* flags that needed to be updated
*
* The update_time function is called when an inode's timestamps need to be
* updated for a read or write operation. In the case where any of S_MTIME, S_CTIME,
* or S_VERSION need to be updated we attempt to update all three of them. S_ATIME
* updates can be handled done independently of the rest.
*
* Returns a S_* mask indicating which fields were updated.
*/
int generic_update_time(struct inode *inode, int flags)
{
int updated = inode_update_timestamps(inode, flags);
int dirty_flags = 0;
if (updated & (S_ATIME|S_MTIME|S_CTIME))
dirty_flags = inode->i_sb->s_flags & SB_LAZYTIME ? I_DIRTY_TIME : I_DIRTY_SYNC; if (updated & S_VERSION) dirty_flags |= I_DIRTY_SYNC; __mark_inode_dirty(inode, dirty_flags);
return updated;
}
EXPORT_SYMBOL(generic_update_time);
/*
* This does the actual work of updating an inodes time or version. Must have
* had called mnt_want_write() before calling this.
*/
int inode_update_time(struct inode *inode, int flags)
{
if (inode->i_op->update_time) return inode->i_op->update_time(inode, flags); generic_update_time(inode, flags);
return 0;
}
EXPORT_SYMBOL(inode_update_time);
/**
* atime_needs_update - update the access time
* @path: the &struct path to update
* @inode: inode to update
*
* Update the accessed time on an inode and mark it for writeback.
* This function automatically handles read only file systems and media,
* as well as the "noatime" flag and inode specific "noatime" markers.
*/
bool atime_needs_update(const struct path *path, struct inode *inode)
{
struct vfsmount *mnt = path->mnt;
struct timespec64 now, atime;
if (inode->i_flags & S_NOATIME)
return false;
/* Atime updates will likely cause i_uid and i_gid to be written
* back improprely if their true value is unknown to the vfs.
*/
if (HAS_UNMAPPED_ID(mnt_idmap(mnt), inode))
return false;
if (IS_NOATIME(inode))
return false;
if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
return false;
if (mnt->mnt_flags & MNT_NOATIME)
return false;
if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
return false;
now = current_time(inode); if (!relatime_need_update(mnt, inode, now))
return false;
atime = inode_get_atime(inode);
if (timespec64_equal(&atime, &now))
return false;
return true;
}
void touch_atime(const struct path *path)
{
struct vfsmount *mnt = path->mnt;
struct inode *inode = d_inode(path->dentry);
if (!atime_needs_update(path, inode))
return;
if (!sb_start_write_trylock(inode->i_sb))
return;
if (mnt_get_write_access(mnt) != 0)
goto skip_update;
/*
* File systems can error out when updating inodes if they need to
* allocate new space to modify an inode (such is the case for
* Btrfs), but since we touch atime while walking down the path we
* really don't care if we failed to update the atime of the file,
* so just ignore the return value.
* We may also fail on filesystems that have the ability to make parts
* of the fs read only, e.g. subvolumes in Btrfs.
*/
inode_update_time(inode, S_ATIME); mnt_put_write_access(mnt);
skip_update:
sb_end_write(inode->i_sb);
}
EXPORT_SYMBOL(touch_atime);
/*
* Return mask of changes for notify_change() that need to be done as a
* response to write or truncate. Return 0 if nothing has to be changed.
* Negative value on error (change should be denied).
*/
int dentry_needs_remove_privs(struct mnt_idmap *idmap,
struct dentry *dentry)
{
struct inode *inode = d_inode(dentry);
int mask = 0;
int ret;
if (IS_NOSEC(inode))
return 0;
mask = setattr_should_drop_suidgid(idmap, inode);
ret = security_inode_need_killpriv(dentry);
if (ret < 0)
return ret;
if (ret) mask |= ATTR_KILL_PRIV;
return mask;
}
static int __remove_privs(struct mnt_idmap *idmap,
struct dentry *dentry, int kill)
{
struct iattr newattrs;
newattrs.ia_valid = ATTR_FORCE | kill;
/*
* Note we call this on write, so notify_change will not
* encounter any conflicting delegations:
*/
return notify_change(idmap, dentry, &newattrs, NULL);
}
int file_remove_privs_flags(struct file *file, unsigned int flags)
{
struct dentry *dentry = file_dentry(file);
struct inode *inode = file_inode(file);
int error = 0;
int kill;
if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode)) return 0; kill = dentry_needs_remove_privs(file_mnt_idmap(file), dentry); if (kill < 0)
return kill;
if (kill) { if (flags & IOCB_NOWAIT)
return -EAGAIN;
error = __remove_privs(file_mnt_idmap(file), dentry, kill);
}
if (!error)
inode_has_no_xattr(inode);
return error;
}
EXPORT_SYMBOL_GPL(file_remove_privs_flags);
/**
* file_remove_privs - remove special file privileges (suid, capabilities)
* @file: file to remove privileges from
*
* When file is modified by a write or truncation ensure that special
* file privileges are removed.
*
* Return: 0 on success, negative errno on failure.
*/
int file_remove_privs(struct file *file)
{
return file_remove_privs_flags(file, 0);
}
EXPORT_SYMBOL(file_remove_privs);
static int inode_needs_update_time(struct inode *inode)
{
int sync_it = 0;
struct timespec64 now = current_time(inode);
struct timespec64 ts;
/* First try to exhaust all avenues to not sync */
if (IS_NOCMTIME(inode))
return 0;
ts = inode_get_mtime(inode);
if (!timespec64_equal(&ts, &now))
sync_it = S_MTIME;
ts = inode_get_ctime(inode); if (!timespec64_equal(&ts, &now)) sync_it |= S_CTIME; if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode)) sync_it |= S_VERSION;
return sync_it;
}
static int __file_update_time(struct file *file, int sync_mode)
{
int ret = 0; struct inode *inode = file_inode(file);
/* try to update time settings */
if (!mnt_get_write_access_file(file)) {
ret = inode_update_time(inode, sync_mode); mnt_put_write_access_file(file);
}
return ret;
}
/**
* file_update_time - update mtime and ctime time
* @file: file accessed
*
* Update the mtime and ctime members of an inode and mark the inode for
* writeback. Note that this function is meant exclusively for usage in
* the file write path of filesystems, and filesystems may choose to
* explicitly ignore updates via this function with the _NOCMTIME inode
* flag, e.g. for network filesystem where these imestamps are handled
* by the server. This can return an error for file systems who need to
* allocate space in order to update an inode.
*
* Return: 0 on success, negative errno on failure.
*/
int file_update_time(struct file *file)
{
int ret;
struct inode *inode = file_inode(file);
ret = inode_needs_update_time(inode);
if (ret <= 0)
return ret;
return __file_update_time(file, ret);
}
EXPORT_SYMBOL(file_update_time);
/**
* file_modified_flags - handle mandated vfs changes when modifying a file
* @file: file that was modified
* @flags: kiocb flags
*
* When file has been modified ensure that special
* file privileges are removed and time settings are updated.
*
* If IOCB_NOWAIT is set, special file privileges will not be removed and
* time settings will not be updated. It will return -EAGAIN.
*
* Context: Caller must hold the file's inode lock.
*
* Return: 0 on success, negative errno on failure.
*/
static int file_modified_flags(struct file *file, int flags)
{
int ret;
struct inode *inode = file_inode(file);
/*
* Clear the security bits if the process is not being run by root.
* This keeps people from modifying setuid and setgid binaries.
*/
ret = file_remove_privs_flags(file, flags);
if (ret)
return ret;
if (unlikely(file->f_mode & FMODE_NOCMTIME))
return 0;
ret = inode_needs_update_time(inode);
if (ret <= 0)
return ret;
if (flags & IOCB_NOWAIT)
return -EAGAIN;
return __file_update_time(file, ret);
}
/**
* file_modified - handle mandated vfs changes when modifying a file
* @file: file that was modified
*
* When file has been modified ensure that special
* file privileges are removed and time settings are updated.
*
* Context: Caller must hold the file's inode lock.
*
* Return: 0 on success, negative errno on failure.
*/
int file_modified(struct file *file)
{
return file_modified_flags(file, 0);
}
EXPORT_SYMBOL(file_modified);
/**
* kiocb_modified - handle mandated vfs changes when modifying a file
* @iocb: iocb that was modified
*
* When file has been modified ensure that special
* file privileges are removed and time settings are updated.
*
* Context: Caller must hold the file's inode lock.
*
* Return: 0 on success, negative errno on failure.
*/
int kiocb_modified(struct kiocb *iocb)
{
return file_modified_flags(iocb->ki_filp, iocb->ki_flags);
}
EXPORT_SYMBOL_GPL(kiocb_modified);
int inode_needs_sync(struct inode *inode)
{
if (IS_SYNC(inode))
return 1;
if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
return 1;
return 0;
}
EXPORT_SYMBOL(inode_needs_sync);
/*
* If we try to find an inode in the inode hash while it is being
* deleted, we have to wait until the filesystem completes its
* deletion before reporting that it isn't found. This function waits
* until the deletion _might_ have completed. Callers are responsible
* to recheck inode state.
*
* It doesn't matter if I_NEW is not set initially, a call to
* wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list
* will DTRT.
*/
static void __wait_on_freeing_inode(struct inode *inode)
{
wait_queue_head_t *wq;
DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
wq = bit_waitqueue(&inode->i_state, __I_NEW);
prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
spin_unlock(&inode->i_lock);
spin_unlock(&inode_hash_lock);
schedule();
finish_wait(wq, &wait.wq_entry);
spin_lock(&inode_hash_lock);
}
static __initdata unsigned long ihash_entries;
static int __init set_ihash_entries(char *str)
{
if (!str)
return 0;
ihash_entries = simple_strtoul(str, &str, 0);
return 1;
}
__setup("ihash_entries=", set_ihash_entries);
/*
* Initialize the waitqueues and inode hash table.
*/
void __init inode_init_early(void)
{
/* If hashes are distributed across NUMA nodes, defer
* hash allocation until vmalloc space is available.
*/
if (hashdist)
return;
inode_hashtable =
alloc_large_system_hash("Inode-cache",
sizeof(struct hlist_head),
ihash_entries,
14,
HASH_EARLY | HASH_ZERO,
&i_hash_shift,
&i_hash_mask,
0,
0);
}
void __init inode_init(void)
{
/* inode slab cache */
inode_cachep = kmem_cache_create("inode_cache",
sizeof(struct inode),
0,
(SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
SLAB_ACCOUNT),
init_once);
/* Hash may have been set up in inode_init_early */
if (!hashdist)
return;
inode_hashtable =
alloc_large_system_hash("Inode-cache",
sizeof(struct hlist_head),
ihash_entries,
14,
HASH_ZERO,
&i_hash_shift,
&i_hash_mask,
0,
0);
}
void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
{
inode->i_mode = mode;
if (S_ISCHR(mode)) {
inode->i_fop = &def_chr_fops;
inode->i_rdev = rdev;
} else if (S_ISBLK(mode)) {
if (IS_ENABLED(CONFIG_BLOCK))
inode->i_fop = &def_blk_fops;
inode->i_rdev = rdev;
} else if (S_ISFIFO(mode))
inode->i_fop = &pipefifo_fops;
else if (S_ISSOCK(mode))
; /* leave it no_open_fops */
else
printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
" inode %s:%lu\n", mode, inode->i_sb->s_id,
inode->i_ino);
}
EXPORT_SYMBOL(init_special_inode);
/**
* inode_init_owner - Init uid,gid,mode for new inode according to posix standards
* @idmap: idmap of the mount the inode was created from
* @inode: New inode
* @dir: Directory inode
* @mode: mode of the new inode
*
* If the inode has been created through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions
* and initializing i_uid and i_gid. On non-idmapped mounts or if permission
* checking is to be performed on the raw inode simply pass @nop_mnt_idmap.
*/
void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
const struct inode *dir, umode_t mode)
{
inode_fsuid_set(inode, idmap); if (dir && dir->i_mode & S_ISGID) { inode->i_gid = dir->i_gid;
/* Directories are special, and always inherit S_ISGID */
if (S_ISDIR(mode))
mode |= S_ISGID;
} else
inode_fsgid_set(inode, idmap); inode->i_mode = mode;
}
EXPORT_SYMBOL(inode_init_owner);
/**
* inode_owner_or_capable - check current task permissions to inode
* @idmap: idmap of the mount the inode was found from
* @inode: inode being checked
*
* Return true if current either has CAP_FOWNER in a namespace with the
* inode owner uid mapped, or owns the file.
*
* If the inode has been found through an idmapped mount the idmap of
* the vfsmount must be passed through @idmap. This function will then take
* care to map the inode according to @idmap before checking permissions.
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply pass @nop_mnt_idmap.
*/
bool inode_owner_or_capable(struct mnt_idmap *idmap,
const struct inode *inode)
{
vfsuid_t vfsuid;
struct user_namespace *ns;
vfsuid = i_uid_into_vfsuid(idmap, inode); if (vfsuid_eq_kuid(vfsuid, current_fsuid()))
return true;
ns = current_user_ns(); if (vfsuid_has_mapping(ns, vfsuid) && ns_capable(ns, CAP_FOWNER))
return true;
return false;
}
EXPORT_SYMBOL(inode_owner_or_capable);
/*
* Direct i/o helper functions
*/
static void __inode_dio_wait(struct inode *inode)
{
wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
do {
prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
if (atomic_read(&inode->i_dio_count))
schedule();
} while (atomic_read(&inode->i_dio_count));
finish_wait(wq, &q.wq_entry);
}
/**
* inode_dio_wait - wait for outstanding DIO requests to finish
* @inode: inode to wait for
*
* Waits for all pending direct I/O requests to finish so that we can
* proceed with a truncate or equivalent operation.
*
* Must be called under a lock that serializes taking new references
* to i_dio_count, usually by inode->i_mutex.
*/
void inode_dio_wait(struct inode *inode)
{
if (atomic_read(&inode->i_dio_count))
__inode_dio_wait(inode);
}
EXPORT_SYMBOL(inode_dio_wait);
/*
* inode_set_flags - atomically set some inode flags
*
* Note: the caller should be holding i_mutex, or else be sure that
* they have exclusive access to the inode structure (i.e., while the
* inode is being instantiated). The reason for the cmpxchg() loop
* --- which wouldn't be necessary if all code paths which modify
* i_flags actually followed this rule, is that there is at least one
* code path which doesn't today so we use cmpxchg() out of an abundance
* of caution.
*
* In the long run, i_mutex is overkill, and we should probably look
* at using the i_lock spinlock to protect i_flags, and then make sure
* it is so documented in include/linux/fs.h and that all code follows
* the locking convention!!
*/
void inode_set_flags(struct inode *inode, unsigned int flags,
unsigned int mask)
{
WARN_ON_ONCE(flags & ~mask); set_mask_bits(&inode->i_flags, mask, flags);
}
EXPORT_SYMBOL(inode_set_flags);
void inode_nohighmem(struct inode *inode)
{
mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
}
EXPORT_SYMBOL(inode_nohighmem);
/**
* timestamp_truncate - Truncate timespec to a granularity
* @t: Timespec
* @inode: inode being updated
*
* Truncate a timespec to the granularity supported by the fs
* containing the inode. Always rounds down. gran must
* not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns).
*/
struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)
{
struct super_block *sb = inode->i_sb;
unsigned int gran = sb->s_time_gran;
t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max); if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min))
t.tv_nsec = 0;
/* Avoid division in the common cases 1 ns and 1 s. */
if (gran == 1)
; /* nothing */
else if (gran == NSEC_PER_SEC)
t.tv_nsec = 0;
else if (gran > 1 && gran < NSEC_PER_SEC) t.tv_nsec -= t.tv_nsec % gran;
else
WARN(1, "invalid file time granularity: %u", gran); return t;
}
EXPORT_SYMBOL(timestamp_truncate);
/**
* current_time - Return FS time
* @inode: inode.
*
* Return the current time truncated to the time granularity supported by
* the fs.
*
* Note that inode and inode->sb cannot be NULL.
* Otherwise, the function warns and returns time without truncation.
*/
struct timespec64 current_time(struct inode *inode)
{
struct timespec64 now;
ktime_get_coarse_real_ts64(&now);
return timestamp_truncate(now, inode);
}
EXPORT_SYMBOL(current_time);
/**
* inode_set_ctime_current - set the ctime to current_time
* @inode: inode
*
* Set the inode->i_ctime to the current value for the inode. Returns
* the current value that was assigned to i_ctime.
*/
struct timespec64 inode_set_ctime_current(struct inode *inode)
{
struct timespec64 now = current_time(inode);
inode_set_ctime_to_ts(inode, now);
return now;
}
EXPORT_SYMBOL(inode_set_ctime_current);
/**
* in_group_or_capable - check whether caller is CAP_FSETID privileged
* @idmap: idmap of the mount @inode was found from
* @inode: inode to check
* @vfsgid: the new/current vfsgid of @inode
*
* Check wether @vfsgid is in the caller's group list or if the caller is
* privileged with CAP_FSETID over @inode. This can be used to determine
* whether the setgid bit can be kept or must be dropped.
*
* Return: true if the caller is sufficiently privileged, false if not.
*/
bool in_group_or_capable(struct mnt_idmap *idmap,
const struct inode *inode, vfsgid_t vfsgid)
{
if (vfsgid_in_group_p(vfsgid))
return true;
if (capable_wrt_inode_uidgid(idmap, inode, CAP_FSETID))
return true;
return false;
}
/**
* mode_strip_sgid - handle the sgid bit for non-directories
* @idmap: idmap of the mount the inode was created from
* @dir: parent directory inode
* @mode: mode of the file to be created in @dir
*
* If the @mode of the new file has both the S_ISGID and S_IXGRP bit
* raised and @dir has the S_ISGID bit raised ensure that the caller is
* either in the group of the parent directory or they have CAP_FSETID
* in their user namespace and are privileged over the parent directory.
* In all other cases, strip the S_ISGID bit from @mode.
*
* Return: the new mode to use for the file
*/
umode_t mode_strip_sgid(struct mnt_idmap *idmap,
const struct inode *dir, umode_t mode)
{
if ((mode & (S_ISGID | S_IXGRP)) != (S_ISGID | S_IXGRP))
return mode;
if (S_ISDIR(mode) || !dir || !(dir->i_mode & S_ISGID))
return mode;
if (in_group_or_capable(idmap, dir, i_gid_into_vfsgid(idmap, dir)))
return mode;
return mode & ~S_ISGID;
}
EXPORT_SYMBOL(mode_strip_sgid);
// SPDX-License-Identifier: GPL-2.0
/*
* kobject.c - library routines for handling generic kernel objects
*
* Copyright (c) 2002-2003 Patrick Mochel <mochel@osdl.org>
* Copyright (c) 2006-2007 Greg Kroah-Hartman <greg@kroah.com>
* Copyright (c) 2006-2007 Novell Inc.
*
* Please see the file Documentation/core-api/kobject.rst for critical information
* about using the kobject interface.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kobject.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/stat.h>
#include <linux/slab.h>
#include <linux/random.h>
/**
* kobject_namespace() - Return @kobj's namespace tag.
* @kobj: kobject in question
*
* Returns namespace tag of @kobj if its parent has namespace ops enabled
* and thus @kobj should have a namespace tag associated with it. Returns
* %NULL otherwise.
*/
const void *kobject_namespace(const struct kobject *kobj)
{
const struct kobj_ns_type_operations *ns_ops = kobj_ns_ops(kobj); if (!ns_ops || ns_ops->type == KOBJ_NS_TYPE_NONE)
return NULL;
return kobj->ktype->namespace(kobj);}
/**
* kobject_get_ownership() - Get sysfs ownership data for @kobj.
* @kobj: kobject in question
* @uid: kernel user ID for sysfs objects
* @gid: kernel group ID for sysfs objects
*
* Returns initial uid/gid pair that should be used when creating sysfs
* representation of given kobject. Normally used to adjust ownership of
* objects in a container.
*/
void kobject_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid)
{
*uid = GLOBAL_ROOT_UID;
*gid = GLOBAL_ROOT_GID;
if (kobj->ktype->get_ownership)
kobj->ktype->get_ownership(kobj, uid, gid);
}
static bool kobj_ns_type_is_valid(enum kobj_ns_type type)
{
if ((type <= KOBJ_NS_TYPE_NONE) || (type >= KOBJ_NS_TYPES))
return false;
return true;
}
static int create_dir(struct kobject *kobj)
{
const struct kobj_type *ktype = get_ktype(kobj);
const struct kobj_ns_type_operations *ops;
int error;
error = sysfs_create_dir_ns(kobj, kobject_namespace(kobj));
if (error)
return error;
if (ktype) { error = sysfs_create_groups(kobj, ktype->default_groups);
if (error) {
sysfs_remove_dir(kobj);
return error;
}
}
/*
* @kobj->sd may be deleted by an ancestor going away. Hold an
* extra reference so that it stays until @kobj is gone.
*/
sysfs_get(kobj->sd);
/*
* If @kobj has ns_ops, its children need to be filtered based on
* their namespace tags. Enable namespace support on @kobj->sd.
*/
ops = kobj_child_ns_ops(kobj);
if (ops) {
BUG_ON(!kobj_ns_type_is_valid(ops->type)); BUG_ON(!kobj_ns_type_registered(ops->type)); sysfs_enable_ns(kobj->sd);
}
return 0;
}
static int get_kobj_path_length(const struct kobject *kobj)
{
int length = 1;
const struct kobject *parent = kobj;
/* walk up the ancestors until we hit the one pointing to the
* root.
* Add 1 to strlen for leading '/' of each level.
*/
do {
if (kobject_name(parent) == NULL)
return 0;
length += strlen(kobject_name(parent)) + 1;
parent = parent->parent;
} while (parent);
return length;
}
static int fill_kobj_path(const struct kobject *kobj, char *path, int length)
{
const struct kobject *parent;
--length;
for (parent = kobj; parent; parent = parent->parent) {
int cur = strlen(kobject_name(parent));
/* back up enough to print this name with '/' */
length -= cur;
if (length <= 0)
return -EINVAL;
memcpy(path + length, kobject_name(parent), cur);
*(path + --length) = '/';
}
pr_debug("'%s' (%p): %s: path = '%s'\n", kobject_name(kobj),
kobj, __func__, path);
return 0;
}
/**
* kobject_get_path() - Allocate memory and fill in the path for @kobj.
* @kobj: kobject in question, with which to build the path
* @gfp_mask: the allocation type used to allocate the path
*
* Return: The newly allocated memory, caller must free with kfree().
*/
char *kobject_get_path(const struct kobject *kobj, gfp_t gfp_mask)
{
char *path;
int len;
retry:
len = get_kobj_path_length(kobj); if (len == 0) return NULL; path = kzalloc(len, gfp_mask);
if (!path)
return NULL;
if (fill_kobj_path(kobj, path, len)) { kfree(path);
goto retry;
}
return path;
}
EXPORT_SYMBOL_GPL(kobject_get_path);
/* add the kobject to its kset's list */
static void kobj_kset_join(struct kobject *kobj)
{
if (!kobj->kset)
return;
kset_get(kobj->kset);
spin_lock(&kobj->kset->list_lock);
list_add_tail(&kobj->entry, &kobj->kset->list); spin_unlock(&kobj->kset->list_lock);
}
/* remove the kobject from its kset's list */
static void kobj_kset_leave(struct kobject *kobj)
{
if (!kobj->kset)
return;
spin_lock(&kobj->kset->list_lock); list_del_init(&kobj->entry);
spin_unlock(&kobj->kset->list_lock);
kset_put(kobj->kset);
}
static void kobject_init_internal(struct kobject *kobj)
{
if (!kobj)
return;
kref_init(&kobj->kref);
INIT_LIST_HEAD(&kobj->entry);
kobj->state_in_sysfs = 0;
kobj->state_add_uevent_sent = 0;
kobj->state_remove_uevent_sent = 0;
kobj->state_initialized = 1;
}
static int kobject_add_internal(struct kobject *kobj)
{
int error = 0;
struct kobject *parent;
if (!kobj)
return -ENOENT;
if (!kobj->name || !kobj->name[0]) { WARN(1,
"kobject: (%p): attempted to be registered with empty name!\n",
kobj);
return -EINVAL;
}
parent = kobject_get(kobj->parent);
/* join kset if set, use it as parent if we do not already have one */
if (kobj->kset) {
if (!parent) parent = kobject_get(&kobj->kset->kobj); kobj_kset_join(kobj); kobj->parent = parent;
}
pr_debug("'%s' (%p): %s: parent: '%s', set: '%s'\n",
kobject_name(kobj), kobj, __func__,
parent ? kobject_name(parent) : "<NULL>",
kobj->kset ? kobject_name(&kobj->kset->kobj) : "<NULL>");
error = create_dir(kobj);
if (error) {
kobj_kset_leave(kobj);
kobject_put(parent);
kobj->parent = NULL;
/* be noisy on error issues */
if (error == -EEXIST)
pr_err("%s failed for %s with -EEXIST, don't try to register things with the same name in the same directory.\n",
__func__, kobject_name(kobj));
else
pr_err("%s failed for %s (error: %d parent: %s)\n",
__func__, kobject_name(kobj), error,
parent ? kobject_name(parent) : "'none'");
} else
kobj->state_in_sysfs = 1;
return error;
}
/**
* kobject_set_name_vargs() - Set the name of a kobject.
* @kobj: struct kobject to set the name of
* @fmt: format string used to build the name
* @vargs: vargs to format the string.
*/
int kobject_set_name_vargs(struct kobject *kobj, const char *fmt,
va_list vargs)
{
const char *s;
if (kobj->name && !fmt) return 0; s = kvasprintf_const(GFP_KERNEL, fmt, vargs);
if (!s)
return -ENOMEM;
/*
* ewww... some of these buggers have '/' in the name ... If
* that's the case, we need to make sure we have an actual
* allocated copy to modify, since kvasprintf_const may have
* returned something from .rodata.
*/
if (strchr(s, '/')) {
char *t;
t = kstrdup(s, GFP_KERNEL);
kfree_const(s);
if (!t)
return -ENOMEM;
s = strreplace(t, '/', '!');
}
kfree_const(kobj->name);
kobj->name = s;
return 0;
}
/**
* kobject_set_name() - Set the name of a kobject.
* @kobj: struct kobject to set the name of
* @fmt: format string used to build the name
*
* This sets the name of the kobject. If you have already added the
* kobject to the system, you must call kobject_rename() in order to
* change the name of the kobject.
*/
int kobject_set_name(struct kobject *kobj, const char *fmt, ...)
{
va_list vargs;
int retval;
va_start(vargs, fmt);
retval = kobject_set_name_vargs(kobj, fmt, vargs);
va_end(vargs);
return retval;
}
EXPORT_SYMBOL(kobject_set_name);
/**
* kobject_init() - Initialize a kobject structure.
* @kobj: pointer to the kobject to initialize
* @ktype: pointer to the ktype for this kobject.
*
* This function will properly initialize a kobject such that it can then
* be passed to the kobject_add() call.
*
* After this function is called, the kobject MUST be cleaned up by a call
* to kobject_put(), not by a call to kfree directly to ensure that all of
* the memory is cleaned up properly.
*/
void kobject_init(struct kobject *kobj, const struct kobj_type *ktype)
{
char *err_str;
if (!kobj) {
err_str = "invalid kobject pointer!";
goto error;
}
if (!ktype) {
err_str = "must have a ktype to be initialized properly!\n";
goto error;
}
if (kobj->state_initialized) {
/* do not error out as sometimes we can recover */
pr_err("kobject (%p): tried to init an initialized object, something is seriously wrong.\n",
kobj);
dump_stack_lvl(KERN_ERR);
}
kobject_init_internal(kobj);
kobj->ktype = ktype;
return;
error:
pr_err("kobject (%p): %s\n", kobj, err_str);
dump_stack_lvl(KERN_ERR);
}
EXPORT_SYMBOL(kobject_init);
static __printf(3, 0) int kobject_add_varg(struct kobject *kobj,
struct kobject *parent,
const char *fmt, va_list vargs)
{
int retval;
retval = kobject_set_name_vargs(kobj, fmt, vargs);
if (retval) {
pr_err("can not set name properly!\n");
return retval;
}
kobj->parent = parent;
return kobject_add_internal(kobj);
}
/**
* kobject_add() - The main kobject add function.
* @kobj: the kobject to add
* @parent: pointer to the parent of the kobject.
* @fmt: format to name the kobject with.
*
* The kobject name is set and added to the kobject hierarchy in this
* function.
*
* If @parent is set, then the parent of the @kobj will be set to it.
* If @parent is NULL, then the parent of the @kobj will be set to the
* kobject associated with the kset assigned to this kobject. If no kset
* is assigned to the kobject, then the kobject will be located in the
* root of the sysfs tree.
*
* Note, no "add" uevent will be created with this call, the caller should set
* up all of the necessary sysfs files for the object and then call
* kobject_uevent() with the UEVENT_ADD parameter to ensure that
* userspace is properly notified of this kobject's creation.
*
* Return: If this function returns an error, kobject_put() must be
* called to properly clean up the memory associated with the
* object. Under no instance should the kobject that is passed
* to this function be directly freed with a call to kfree(),
* that can leak memory.
*
* If this function returns success, kobject_put() must also be called
* in order to properly clean up the memory associated with the object.
*
* In short, once this function is called, kobject_put() MUST be called
* when the use of the object is finished in order to properly free
* everything.
*/
int kobject_add(struct kobject *kobj, struct kobject *parent,
const char *fmt, ...)
{
va_list args;
int retval;
if (!kobj)
return -EINVAL;
if (!kobj->state_initialized) { pr_err("kobject '%s' (%p): tried to add an uninitialized object, something is seriously wrong.\n",
kobject_name(kobj), kobj);
dump_stack_lvl(KERN_ERR);
return -EINVAL;
}
va_start(args, fmt); retval = kobject_add_varg(kobj, parent, fmt, args); va_end(args); return retval;
}
EXPORT_SYMBOL(kobject_add);
/**
* kobject_init_and_add() - Initialize a kobject structure and add it to
* the kobject hierarchy.
* @kobj: pointer to the kobject to initialize
* @ktype: pointer to the ktype for this kobject.
* @parent: pointer to the parent of this kobject.
* @fmt: the name of the kobject.
*
* This function combines the call to kobject_init() and kobject_add().
*
* If this function returns an error, kobject_put() must be called to
* properly clean up the memory associated with the object. This is the
* same type of error handling after a call to kobject_add() and kobject
* lifetime rules are the same here.
*/
int kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype,
struct kobject *parent, const char *fmt, ...)
{
va_list args;
int retval;
kobject_init(kobj, ktype);
va_start(args, fmt);
retval = kobject_add_varg(kobj, parent, fmt, args); va_end(args);
return retval;
}
EXPORT_SYMBOL_GPL(kobject_init_and_add);
/**
* kobject_rename() - Change the name of an object.
* @kobj: object in question.
* @new_name: object's new name
*
* It is the responsibility of the caller to provide mutual
* exclusion between two different calls of kobject_rename
* on the same kobject and to ensure that new_name is valid and
* won't conflict with other kobjects.
*/
int kobject_rename(struct kobject *kobj, const char *new_name)
{
int error = 0;
const char *devpath = NULL;
const char *dup_name = NULL, *name;
char *devpath_string = NULL;
char *envp[2];
kobj = kobject_get(kobj);
if (!kobj)
return -EINVAL;
if (!kobj->parent) {
kobject_put(kobj);
return -EINVAL;
}
devpath = kobject_get_path(kobj, GFP_KERNEL);
if (!devpath) {
error = -ENOMEM;
goto out;
}
devpath_string = kmalloc(strlen(devpath) + 15, GFP_KERNEL);
if (!devpath_string) {
error = -ENOMEM;
goto out;
}
sprintf(devpath_string, "DEVPATH_OLD=%s", devpath);
envp[0] = devpath_string;
envp[1] = NULL;
name = dup_name = kstrdup_const(new_name, GFP_KERNEL);
if (!name) {
error = -ENOMEM;
goto out;
}
error = sysfs_rename_dir_ns(kobj, new_name, kobject_namespace(kobj));
if (error)
goto out;
/* Install the new kobject name */
dup_name = kobj->name;
kobj->name = name;
/* This function is mostly/only used for network interface.
* Some hotplug package track interfaces by their name and
* therefore want to know when the name is changed by the user. */
kobject_uevent_env(kobj, KOBJ_MOVE, envp);
out:
kfree_const(dup_name);
kfree(devpath_string);
kfree(devpath);
kobject_put(kobj);
return error;
}
EXPORT_SYMBOL_GPL(kobject_rename);
/**
* kobject_move() - Move object to another parent.
* @kobj: object in question.
* @new_parent: object's new parent (can be NULL)
*/
int kobject_move(struct kobject *kobj, struct kobject *new_parent)
{
int error;
struct kobject *old_parent;
const char *devpath = NULL;
char *devpath_string = NULL;
char *envp[2];
kobj = kobject_get(kobj);
if (!kobj)
return -EINVAL;
new_parent = kobject_get(new_parent);
if (!new_parent) {
if (kobj->kset)
new_parent = kobject_get(&kobj->kset->kobj);
}
/* old object path */
devpath = kobject_get_path(kobj, GFP_KERNEL);
if (!devpath) {
error = -ENOMEM;
goto out;
}
devpath_string = kmalloc(strlen(devpath) + 15, GFP_KERNEL);
if (!devpath_string) {
error = -ENOMEM;
goto out;
}
sprintf(devpath_string, "DEVPATH_OLD=%s", devpath);
envp[0] = devpath_string;
envp[1] = NULL;
error = sysfs_move_dir_ns(kobj, new_parent, kobject_namespace(kobj));
if (error)
goto out;
old_parent = kobj->parent;
kobj->parent = new_parent;
new_parent = NULL;
kobject_put(old_parent);
kobject_uevent_env(kobj, KOBJ_MOVE, envp);
out:
kobject_put(new_parent);
kobject_put(kobj);
kfree(devpath_string);
kfree(devpath);
return error;
}
EXPORT_SYMBOL_GPL(kobject_move);
static void __kobject_del(struct kobject *kobj)
{
struct kernfs_node *sd;
const struct kobj_type *ktype;
sd = kobj->sd;
ktype = get_ktype(kobj);
if (ktype)
sysfs_remove_groups(kobj, ktype->default_groups);
/* send "remove" if the caller did not do it but sent "add" */
if (kobj->state_add_uevent_sent && !kobj->state_remove_uevent_sent) { pr_debug("'%s' (%p): auto cleanup 'remove' event\n",
kobject_name(kobj), kobj);
kobject_uevent(kobj, KOBJ_REMOVE);
}
sysfs_remove_dir(kobj);
sysfs_put(sd);
kobj->state_in_sysfs = 0;
kobj_kset_leave(kobj);
kobj->parent = NULL;
}
/**
* kobject_del() - Unlink kobject from hierarchy.
* @kobj: object.
*
* This is the function that should be called to delete an object
* successfully added via kobject_add().
*/
void kobject_del(struct kobject *kobj)
{
struct kobject *parent;
if (!kobj)
return;
parent = kobj->parent;
__kobject_del(kobj);
kobject_put(parent);
}
EXPORT_SYMBOL(kobject_del);
/**
* kobject_get() - Increment refcount for object.
* @kobj: object.
*/
struct kobject *kobject_get(struct kobject *kobj)
{
if (kobj) { if (!kobj->state_initialized) WARN(1, KERN_WARNING
"kobject: '%s' (%p): is not initialized, yet kobject_get() is being called.\n",
kobject_name(kobj), kobj);
kref_get(&kobj->kref);
}
return kobj;
}
EXPORT_SYMBOL(kobject_get);
struct kobject * __must_check kobject_get_unless_zero(struct kobject *kobj)
{
if (!kobj)
return NULL;
if (!kref_get_unless_zero(&kobj->kref))
kobj = NULL;
return kobj;
}
EXPORT_SYMBOL(kobject_get_unless_zero);
/*
* kobject_cleanup - free kobject resources.
* @kobj: object to cleanup
*/
static void kobject_cleanup(struct kobject *kobj)
{
struct kobject *parent = kobj->parent;
const struct kobj_type *t = get_ktype(kobj);
const char *name = kobj->name;
pr_debug("'%s' (%p): %s, parent %p\n",
kobject_name(kobj), kobj, __func__, kobj->parent);
if (t && !t->release) pr_debug("'%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n",
kobject_name(kobj), kobj);
/* remove from sysfs if the caller did not do it */
if (kobj->state_in_sysfs) { pr_debug("'%s' (%p): auto cleanup kobject_del\n",
kobject_name(kobj), kobj);
__kobject_del(kobj);
} else {
/* avoid dropping the parent reference unnecessarily */
parent = NULL;
}
if (t && t->release) { pr_debug("'%s' (%p): calling ktype release\n",
kobject_name(kobj), kobj);
t->release(kobj);
}
/* free name if we allocated it */
if (name) { pr_debug("'%s': free name\n", name); kfree_const(name);
}
kobject_put(parent);
}
#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
static void kobject_delayed_cleanup(struct work_struct *work)
{
kobject_cleanup(container_of(to_delayed_work(work),
struct kobject, release));
}
#endif
static void kobject_release(struct kref *kref)
{
struct kobject *kobj = container_of(kref, struct kobject, kref);
#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
unsigned long delay = HZ + HZ * get_random_u32_below(4);
pr_info("'%s' (%p): %s, parent %p (delayed %ld)\n",
kobject_name(kobj), kobj, __func__, kobj->parent, delay);
INIT_DELAYED_WORK(&kobj->release, kobject_delayed_cleanup);
schedule_delayed_work(&kobj->release, delay);
#else
kobject_cleanup(kobj);
#endif
}
/**
* kobject_put() - Decrement refcount for object.
* @kobj: object.
*
* Decrement the refcount, and if 0, call kobject_cleanup().
*/
void kobject_put(struct kobject *kobj)
{
if (kobj) { if (!kobj->state_initialized) WARN(1, KERN_WARNING
"kobject: '%s' (%p): is not initialized, yet kobject_put() is being called.\n",
kobject_name(kobj), kobj);
kref_put(&kobj->kref, kobject_release);
}
}
EXPORT_SYMBOL(kobject_put);
static void dynamic_kobj_release(struct kobject *kobj)
{
pr_debug("(%p): %s\n", kobj, __func__); kfree(kobj);
}
static const struct kobj_type dynamic_kobj_ktype = {
.release = dynamic_kobj_release,
.sysfs_ops = &kobj_sysfs_ops,
};
/**
* kobject_create() - Create a struct kobject dynamically.
*
* This function creates a kobject structure dynamically and sets it up
* to be a "dynamic" kobject with a default release function set up.
*
* If the kobject was not able to be created, NULL will be returned.
* The kobject structure returned from here must be cleaned up with a
* call to kobject_put() and not kfree(), as kobject_init() has
* already been called on this structure.
*/
static struct kobject *kobject_create(void)
{
struct kobject *kobj;
kobj = kzalloc(sizeof(*kobj), GFP_KERNEL);
if (!kobj)
return NULL;
kobject_init(kobj, &dynamic_kobj_ktype);
return kobj;
}
/**
* kobject_create_and_add() - Create a struct kobject dynamically and
* register it with sysfs.
* @name: the name for the kobject
* @parent: the parent kobject of this kobject, if any.
*
* This function creates a kobject structure dynamically and registers it
* with sysfs. When you are finished with this structure, call
* kobject_put() and the structure will be dynamically freed when
* it is no longer being used.
*
* If the kobject was not able to be created, NULL will be returned.
*/
struct kobject *kobject_create_and_add(const char *name, struct kobject *parent)
{
struct kobject *kobj;
int retval;
kobj = kobject_create();
if (!kobj)
return NULL;
retval = kobject_add(kobj, parent, "%s", name);
if (retval) {
pr_warn("%s: kobject_add error: %d\n", __func__, retval);
kobject_put(kobj);
kobj = NULL;
}
return kobj;
}
EXPORT_SYMBOL_GPL(kobject_create_and_add);
/**
* kset_init() - Initialize a kset for use.
* @k: kset
*/
void kset_init(struct kset *k)
{
kobject_init_internal(&k->kobj);
INIT_LIST_HEAD(&k->list);
spin_lock_init(&k->list_lock);
}
/* default kobject attribute operations */
static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct kobj_attribute *kattr;
ssize_t ret = -EIO;
kattr = container_of(attr, struct kobj_attribute, attr);
if (kattr->show)
ret = kattr->show(kobj, kattr, buf);
return ret;
}
static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t count)
{
struct kobj_attribute *kattr;
ssize_t ret = -EIO;
kattr = container_of(attr, struct kobj_attribute, attr);
if (kattr->store)
ret = kattr->store(kobj, kattr, buf, count);
return ret;
}
const struct sysfs_ops kobj_sysfs_ops = {
.show = kobj_attr_show,
.store = kobj_attr_store,
};
EXPORT_SYMBOL_GPL(kobj_sysfs_ops);
/**
* kset_register() - Initialize and add a kset.
* @k: kset.
*
* NOTE: On error, the kset.kobj.name allocated by() kobj_set_name()
* is freed, it can not be used any more.
*/
int kset_register(struct kset *k)
{
int err;
if (!k)
return -EINVAL;
if (!k->kobj.ktype) {
pr_err("must have a ktype to be initialized properly!\n");
return -EINVAL;
}
kset_init(k);
err = kobject_add_internal(&k->kobj);
if (err) {
kfree_const(k->kobj.name);
/* Set it to NULL to avoid accessing bad pointer in callers. */
k->kobj.name = NULL;
return err;
}
kobject_uevent(&k->kobj, KOBJ_ADD);
return 0;
}
EXPORT_SYMBOL(kset_register);
/**
* kset_unregister() - Remove a kset.
* @k: kset.
*/
void kset_unregister(struct kset *k)
{
if (!k)
return;
kobject_del(&k->kobj); kobject_put(&k->kobj);
}
EXPORT_SYMBOL(kset_unregister);
/**
* kset_find_obj() - Search for object in kset.
* @kset: kset we're looking in.
* @name: object's name.
*
* Lock kset via @kset->subsys, and iterate over @kset->list,
* looking for a matching kobject. If matching object is found
* take a reference and return the object.
*/
struct kobject *kset_find_obj(struct kset *kset, const char *name)
{
struct kobject *k;
struct kobject *ret = NULL;
spin_lock(&kset->list_lock); list_for_each_entry(k, &kset->list, entry) { if (kobject_name(k) && !strcmp(kobject_name(k), name)) { ret = kobject_get_unless_zero(k);
break;
}
}
spin_unlock(&kset->list_lock);
return ret;
}
EXPORT_SYMBOL_GPL(kset_find_obj);
static void kset_release(struct kobject *kobj)
{
struct kset *kset = container_of(kobj, struct kset, kobj); pr_debug("'%s' (%p): %s\n",
kobject_name(kobj), kobj, __func__);
kfree(kset);
}
static void kset_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid)
{
if (kobj->parent)
kobject_get_ownership(kobj->parent, uid, gid);
}
static const struct kobj_type kset_ktype = {
.sysfs_ops = &kobj_sysfs_ops,
.release = kset_release,
.get_ownership = kset_get_ownership,
};
/**
* kset_create() - Create a struct kset dynamically.
*
* @name: the name for the kset
* @uevent_ops: a struct kset_uevent_ops for the kset
* @parent_kobj: the parent kobject of this kset, if any.
*
* This function creates a kset structure dynamically. This structure can
* then be registered with the system and show up in sysfs with a call to
* kset_register(). When you are finished with this structure, if
* kset_register() has been called, call kset_unregister() and the
* structure will be dynamically freed when it is no longer being used.
*
* If the kset was not able to be created, NULL will be returned.
*/
static struct kset *kset_create(const char *name,
const struct kset_uevent_ops *uevent_ops,
struct kobject *parent_kobj)
{
struct kset *kset;
int retval;
kset = kzalloc(sizeof(*kset), GFP_KERNEL);
if (!kset)
return NULL;
retval = kobject_set_name(&kset->kobj, "%s", name);
if (retval) {
kfree(kset);
return NULL;
}
kset->uevent_ops = uevent_ops;
kset->kobj.parent = parent_kobj;
/*
* The kobject of this kset will have a type of kset_ktype and belong to
* no kset itself. That way we can properly free it when it is
* finished being used.
*/
kset->kobj.ktype = &kset_ktype;
kset->kobj.kset = NULL;
return kset;
}
/**
* kset_create_and_add() - Create a struct kset dynamically and add it to sysfs.
*
* @name: the name for the kset
* @uevent_ops: a struct kset_uevent_ops for the kset
* @parent_kobj: the parent kobject of this kset, if any.
*
* This function creates a kset structure dynamically and registers it
* with sysfs. When you are finished with this structure, call
* kset_unregister() and the structure will be dynamically freed when it
* is no longer being used.
*
* If the kset was not able to be created, NULL will be returned.
*/
struct kset *kset_create_and_add(const char *name,
const struct kset_uevent_ops *uevent_ops,
struct kobject *parent_kobj)
{
struct kset *kset;
int error;
kset = kset_create(name, uevent_ops, parent_kobj);
if (!kset)
return NULL;
error = kset_register(kset);
if (error) {
kfree(kset);
return NULL;
}
return kset;
}
EXPORT_SYMBOL_GPL(kset_create_and_add);
static DEFINE_SPINLOCK(kobj_ns_type_lock);
static const struct kobj_ns_type_operations *kobj_ns_ops_tbl[KOBJ_NS_TYPES];
int kobj_ns_type_register(const struct kobj_ns_type_operations *ops)
{
enum kobj_ns_type type = ops->type;
int error;
spin_lock(&kobj_ns_type_lock);
error = -EINVAL;
if (!kobj_ns_type_is_valid(type))
goto out;
error = -EBUSY;
if (kobj_ns_ops_tbl[type])
goto out;
error = 0;
kobj_ns_ops_tbl[type] = ops;
out:
spin_unlock(&kobj_ns_type_lock);
return error;
}
int kobj_ns_type_registered(enum kobj_ns_type type)
{
int registered = 0;
spin_lock(&kobj_ns_type_lock);
if (kobj_ns_type_is_valid(type))
registered = kobj_ns_ops_tbl[type] != NULL;
spin_unlock(&kobj_ns_type_lock);
return registered;
}
const struct kobj_ns_type_operations *kobj_child_ns_ops(const struct kobject *parent)
{
const struct kobj_ns_type_operations *ops = NULL;
if (parent && parent->ktype && parent->ktype->child_ns_type) ops = parent->ktype->child_ns_type(parent);
return ops;
}
const struct kobj_ns_type_operations *kobj_ns_ops(const struct kobject *kobj)
{
return kobj_child_ns_ops(kobj->parent);
}
bool kobj_ns_current_may_mount(enum kobj_ns_type type)
{
bool may_mount = true;
spin_lock(&kobj_ns_type_lock);
if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type])
may_mount = kobj_ns_ops_tbl[type]->current_may_mount();
spin_unlock(&kobj_ns_type_lock);
return may_mount;
}
void *kobj_ns_grab_current(enum kobj_ns_type type)
{
void *ns = NULL;
spin_lock(&kobj_ns_type_lock);
if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type])
ns = kobj_ns_ops_tbl[type]->grab_current_ns();
spin_unlock(&kobj_ns_type_lock);
return ns;
}
EXPORT_SYMBOL_GPL(kobj_ns_grab_current);
const void *kobj_ns_netlink(enum kobj_ns_type type, struct sock *sk)
{
const void *ns = NULL;
spin_lock(&kobj_ns_type_lock);
if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type])
ns = kobj_ns_ops_tbl[type]->netlink_ns(sk);
spin_unlock(&kobj_ns_type_lock);
return ns;
}
const void *kobj_ns_initial(enum kobj_ns_type type)
{
const void *ns = NULL;
spin_lock(&kobj_ns_type_lock);
if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type])
ns = kobj_ns_ops_tbl[type]->initial_ns();
spin_unlock(&kobj_ns_type_lock);
return ns;
}
void kobj_ns_drop(enum kobj_ns_type type, void *ns)
{
spin_lock(&kobj_ns_type_lock);
if (kobj_ns_type_is_valid(type) &&
kobj_ns_ops_tbl[type] && kobj_ns_ops_tbl[type]->drop_ns)
kobj_ns_ops_tbl[type]->drop_ns(ns);
spin_unlock(&kobj_ns_type_lock);
}
EXPORT_SYMBOL_GPL(kobj_ns_drop);
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2002-2005, Instant802 Networks, Inc.
* Copyright 2006-2007 Jiri Benc <jbenc@suse.cz>
* Copyright 2013-2014 Intel Mobile Communications GmbH
* Copyright (C) 2015 - 2017 Intel Deutschland GmbH
* Copyright (C) 2018-2023 Intel Corporation
*/
#include <linux/module.h>
#include <linux/init.h>
#include <linux/etherdevice.h>
#include <linux/netdevice.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/if_arp.h>
#include <linux/timer.h>
#include <linux/rtnetlink.h>
#include <net/codel.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"
#include "sta_info.h"
#include "debugfs_sta.h"
#include "mesh.h"
#include "wme.h"
/**
* DOC: STA information lifetime rules
*
* STA info structures (&struct sta_info) are managed in a hash table
* for faster lookup and a list for iteration. They are managed using
* RCU, i.e. access to the list and hash table is protected by RCU.
*
* Upon allocating a STA info structure with sta_info_alloc(), the caller
* owns that structure. It must then insert it into the hash table using
* either sta_info_insert() or sta_info_insert_rcu(); only in the latter
* case (which acquires an rcu read section but must not be called from
* within one) will the pointer still be valid after the call. Note that
* the caller may not do much with the STA info before inserting it; in
* particular, it may not start any mesh peer link management or add
* encryption keys.
*
* When the insertion fails (sta_info_insert()) returns non-zero), the
* structure will have been freed by sta_info_insert()!
*
* Station entries are added by mac80211 when you establish a link with a
* peer. This means different things for the different type of interfaces
* we support. For a regular station this mean we add the AP sta when we
* receive an association response from the AP. For IBSS this occurs when
* get to know about a peer on the same IBSS. For WDS we add the sta for
* the peer immediately upon device open. When using AP mode we add stations
* for each respective station upon request from userspace through nl80211.
*
* In order to remove a STA info structure, various sta_info_destroy_*()
* calls are available.
*
* There is no concept of ownership on a STA entry; each structure is
* owned by the global hash table/list until it is removed. All users of
* the structure need to be RCU protected so that the structure won't be
* freed before they are done using it.
*/
struct sta_link_alloc {
struct link_sta_info info;
struct ieee80211_link_sta sta;
struct rcu_head rcu_head;
};
static const struct rhashtable_params sta_rht_params = {
.nelem_hint = 3, /* start small */
.automatic_shrinking = true,
.head_offset = offsetof(struct sta_info, hash_node),
.key_offset = offsetof(struct sta_info, addr),
.key_len = ETH_ALEN,
.max_size = CONFIG_MAC80211_STA_HASH_MAX_SIZE,
};
static const struct rhashtable_params link_sta_rht_params = {
.nelem_hint = 3, /* start small */
.automatic_shrinking = true,
.head_offset = offsetof(struct link_sta_info, link_hash_node),
.key_offset = offsetof(struct link_sta_info, addr),
.key_len = ETH_ALEN,
.max_size = CONFIG_MAC80211_STA_HASH_MAX_SIZE,
};
static int sta_info_hash_del(struct ieee80211_local *local,
struct sta_info *sta)
{
return rhltable_remove(&local->sta_hash, &sta->hash_node,
sta_rht_params);
}
static int link_sta_info_hash_add(struct ieee80211_local *local,
struct link_sta_info *link_sta)
{
lockdep_assert_wiphy(local->hw.wiphy);
return rhltable_insert(&local->link_sta_hash,
&link_sta->link_hash_node, link_sta_rht_params);
}
static int link_sta_info_hash_del(struct ieee80211_local *local,
struct link_sta_info *link_sta)
{
lockdep_assert_wiphy(local->hw.wiphy);
return rhltable_remove(&local->link_sta_hash,
&link_sta->link_hash_node, link_sta_rht_params);
}
void ieee80211_purge_sta_txqs(struct sta_info *sta)
{
struct ieee80211_local *local = sta->sdata->local;
int i;
for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
struct txq_info *txqi;
if (!sta->sta.txq[i])
continue;
txqi = to_txq_info(sta->sta.txq[i]);
ieee80211_txq_purge(local, txqi);
}
}
static void __cleanup_single_sta(struct sta_info *sta)
{
int ac, i;
struct tid_ampdu_tx *tid_tx;
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
struct ps_data *ps;
if (test_sta_flag(sta, WLAN_STA_PS_STA) ||
test_sta_flag(sta, WLAN_STA_PS_DRIVER) ||
test_sta_flag(sta, WLAN_STA_PS_DELIVER)) {
if (sta->sdata->vif.type == NL80211_IFTYPE_AP ||
sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
ps = &sdata->bss->ps;
else if (ieee80211_vif_is_mesh(&sdata->vif))
ps = &sdata->u.mesh.ps;
else
return;
clear_sta_flag(sta, WLAN_STA_PS_STA);
clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
clear_sta_flag(sta, WLAN_STA_PS_DELIVER);
atomic_dec(&ps->num_sta_ps);
}
ieee80211_purge_sta_txqs(sta);
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
local->total_ps_buffered -= skb_queue_len(&sta->ps_tx_buf[ac]);
ieee80211_purge_tx_queue(&local->hw, &sta->ps_tx_buf[ac]);
ieee80211_purge_tx_queue(&local->hw, &sta->tx_filtered[ac]);
}
if (ieee80211_vif_is_mesh(&sdata->vif))
mesh_sta_cleanup(sta);
cancel_work_sync(&sta->drv_deliver_wk);
/*
* Destroy aggregation state here. It would be nice to wait for the
* driver to finish aggregation stop and then clean up, but for now
* drivers have to handle aggregation stop being requested, followed
* directly by station destruction.
*/
for (i = 0; i < IEEE80211_NUM_TIDS; i++) {
kfree(sta->ampdu_mlme.tid_start_tx[i]);
tid_tx = rcu_dereference_raw(sta->ampdu_mlme.tid_tx[i]);
if (!tid_tx)
continue;
ieee80211_purge_tx_queue(&local->hw, &tid_tx->pending);
kfree(tid_tx);
}
}
static void cleanup_single_sta(struct sta_info *sta)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
__cleanup_single_sta(sta);
sta_info_free(local, sta);
}
struct rhlist_head *sta_info_hash_lookup(struct ieee80211_local *local,
const u8 *addr)
{
return rhltable_lookup(&local->sta_hash, addr, sta_rht_params);
}
/* protected by RCU */
struct sta_info *sta_info_get(struct ieee80211_sub_if_data *sdata,
const u8 *addr)
{
struct ieee80211_local *local = sdata->local;
struct rhlist_head *tmp;
struct sta_info *sta;
rcu_read_lock();
for_each_sta_info(local, addr, sta, tmp) {
if (sta->sdata == sdata) {
rcu_read_unlock();
/* this is safe as the caller must already hold
* another rcu read section or the mutex
*/
return sta;
}
}
rcu_read_unlock();
return NULL;
}
/*
* Get sta info either from the specified interface
* or from one of its vlans
*/
struct sta_info *sta_info_get_bss(struct ieee80211_sub_if_data *sdata,
const u8 *addr)
{
struct ieee80211_local *local = sdata->local;
struct rhlist_head *tmp;
struct sta_info *sta;
rcu_read_lock();
for_each_sta_info(local, addr, sta, tmp) {
if (sta->sdata == sdata ||
(sta->sdata->bss && sta->sdata->bss == sdata->bss)) {
rcu_read_unlock();
/* this is safe as the caller must already hold
* another rcu read section or the mutex
*/
return sta;
}
}
rcu_read_unlock();
return NULL;
}
struct rhlist_head *link_sta_info_hash_lookup(struct ieee80211_local *local,
const u8 *addr)
{
return rhltable_lookup(&local->link_sta_hash, addr,
link_sta_rht_params);
}
struct link_sta_info *
link_sta_info_get_bss(struct ieee80211_sub_if_data *sdata, const u8 *addr)
{
struct ieee80211_local *local = sdata->local;
struct rhlist_head *tmp;
struct link_sta_info *link_sta;
rcu_read_lock();
for_each_link_sta_info(local, addr, link_sta, tmp) {
struct sta_info *sta = link_sta->sta;
if (sta->sdata == sdata ||
(sta->sdata->bss && sta->sdata->bss == sdata->bss)) {
rcu_read_unlock();
/* this is safe as the caller must already hold
* another rcu read section or the mutex
*/
return link_sta;
}
}
rcu_read_unlock();
return NULL;
}
struct ieee80211_sta *
ieee80211_find_sta_by_link_addrs(struct ieee80211_hw *hw,
const u8 *addr,
const u8 *localaddr,
unsigned int *link_id)
{
struct ieee80211_local *local = hw_to_local(hw);
struct link_sta_info *link_sta;
struct rhlist_head *tmp;
for_each_link_sta_info(local, addr, link_sta, tmp) {
struct sta_info *sta = link_sta->sta;
struct ieee80211_link_data *link;
u8 _link_id = link_sta->link_id;
if (!localaddr) {
if (link_id)
*link_id = _link_id;
return &sta->sta;
}
link = rcu_dereference(sta->sdata->link[_link_id]);
if (!link)
continue;
if (memcmp(link->conf->addr, localaddr, ETH_ALEN))
continue;
if (link_id)
*link_id = _link_id;
return &sta->sta;
}
return NULL;
}
EXPORT_SYMBOL_GPL(ieee80211_find_sta_by_link_addrs);
struct sta_info *sta_info_get_by_addrs(struct ieee80211_local *local,
const u8 *sta_addr, const u8 *vif_addr)
{
struct rhlist_head *tmp;
struct sta_info *sta;
for_each_sta_info(local, sta_addr, sta, tmp) {
if (ether_addr_equal(vif_addr, sta->sdata->vif.addr))
return sta;
}
return NULL;
}
struct sta_info *sta_info_get_by_idx(struct ieee80211_sub_if_data *sdata,
int idx)
{
struct ieee80211_local *local = sdata->local;
struct sta_info *sta;
int i = 0;
list_for_each_entry_rcu(sta, &local->sta_list, list,
lockdep_is_held(&local->hw.wiphy->mtx)) {
if (sdata != sta->sdata)
continue;
if (i < idx) {
++i;
continue;
}
return sta;
}
return NULL;
}
static void sta_info_free_link(struct link_sta_info *link_sta)
{
free_percpu(link_sta->pcpu_rx_stats);
}
static void sta_remove_link(struct sta_info *sta, unsigned int link_id,
bool unhash)
{
struct sta_link_alloc *alloc = NULL;
struct link_sta_info *link_sta;
lockdep_assert_wiphy(sta->local->hw.wiphy);
link_sta = rcu_access_pointer(sta->link[link_id]);
if (WARN_ON(!link_sta))
return;
if (unhash)
link_sta_info_hash_del(sta->local, link_sta);
if (test_sta_flag(sta, WLAN_STA_INSERTED))
ieee80211_link_sta_debugfs_remove(link_sta);
if (link_sta != &sta->deflink)
alloc = container_of(link_sta, typeof(*alloc), info);
sta->sta.valid_links &= ~BIT(link_id);
RCU_INIT_POINTER(sta->link[link_id], NULL);
RCU_INIT_POINTER(sta->sta.link[link_id], NULL);
if (alloc) {
sta_info_free_link(&alloc->info);
kfree_rcu(alloc, rcu_head);
}
ieee80211_sta_recalc_aggregates(&sta->sta);
}
/**
* sta_info_free - free STA
*
* @local: pointer to the global information
* @sta: STA info to free
*
* This function must undo everything done by sta_info_alloc()
* that may happen before sta_info_insert(). It may only be
* called when sta_info_insert() has not been attempted (and
* if that fails, the station is freed anyway.)
*/
void sta_info_free(struct ieee80211_local *local, struct sta_info *sta)
{
int i;
for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
struct link_sta_info *link_sta;
link_sta = rcu_access_pointer(sta->link[i]);
if (!link_sta)
continue;
sta_remove_link(sta, i, false);
}
/*
* If we had used sta_info_pre_move_state() then we might not
* have gone through the state transitions down again, so do
* it here now (and warn if it's inserted).
*
* This will clear state such as fast TX/RX that may have been
* allocated during state transitions.
*/
while (sta->sta_state > IEEE80211_STA_NONE) {
int ret;
WARN_ON_ONCE(test_sta_flag(sta, WLAN_STA_INSERTED));
ret = sta_info_move_state(sta, sta->sta_state - 1);
if (WARN_ONCE(ret, "sta_info_move_state() returned %d\n", ret))
break;
}
if (sta->rate_ctrl)
rate_control_free_sta(sta);
sta_dbg(sta->sdata, "Destroyed STA %pM\n", sta->sta.addr);
kfree(to_txq_info(sta->sta.txq[0]));
kfree(rcu_dereference_raw(sta->sta.rates));
#ifdef CONFIG_MAC80211_MESH
kfree(sta->mesh);
#endif
sta_info_free_link(&sta->deflink);
kfree(sta);
}
static int sta_info_hash_add(struct ieee80211_local *local,
struct sta_info *sta)
{
return rhltable_insert(&local->sta_hash, &sta->hash_node,
sta_rht_params);
}
static void sta_deliver_ps_frames(struct work_struct *wk)
{
struct sta_info *sta;
sta = container_of(wk, struct sta_info, drv_deliver_wk);
if (sta->dead)
return;
local_bh_disable();
if (!test_sta_flag(sta, WLAN_STA_PS_STA))
ieee80211_sta_ps_deliver_wakeup(sta);
else if (test_and_clear_sta_flag(sta, WLAN_STA_PSPOLL))
ieee80211_sta_ps_deliver_poll_response(sta);
else if (test_and_clear_sta_flag(sta, WLAN_STA_UAPSD))
ieee80211_sta_ps_deliver_uapsd(sta);
local_bh_enable();
}
static int sta_prepare_rate_control(struct ieee80211_local *local,
struct sta_info *sta, gfp_t gfp)
{
if (ieee80211_hw_check(&local->hw, HAS_RATE_CONTROL))
return 0;
sta->rate_ctrl = local->rate_ctrl;
sta->rate_ctrl_priv = rate_control_alloc_sta(sta->rate_ctrl,
sta, gfp);
if (!sta->rate_ctrl_priv)
return -ENOMEM;
return 0;
}
static int sta_info_alloc_link(struct ieee80211_local *local,
struct link_sta_info *link_info,
gfp_t gfp)
{
struct ieee80211_hw *hw = &local->hw;
int i;
if (ieee80211_hw_check(hw, USES_RSS)) {
link_info->pcpu_rx_stats =
alloc_percpu_gfp(struct ieee80211_sta_rx_stats, gfp);
if (!link_info->pcpu_rx_stats)
return -ENOMEM;
}
link_info->rx_stats.last_rx = jiffies;
u64_stats_init(&link_info->rx_stats.syncp);
ewma_signal_init(&link_info->rx_stats_avg.signal);
ewma_avg_signal_init(&link_info->status_stats.avg_ack_signal);
for (i = 0; i < ARRAY_SIZE(link_info->rx_stats_avg.chain_signal); i++)
ewma_signal_init(&link_info->rx_stats_avg.chain_signal[i]);
return 0;
}
static void sta_info_add_link(struct sta_info *sta,
unsigned int link_id,
struct link_sta_info *link_info,
struct ieee80211_link_sta *link_sta)
{
link_info->sta = sta;
link_info->link_id = link_id;
link_info->pub = link_sta;
link_info->pub->sta = &sta->sta;
link_sta->link_id = link_id;
rcu_assign_pointer(sta->link[link_id], link_info);
rcu_assign_pointer(sta->sta.link[link_id], link_sta);
link_sta->smps_mode = IEEE80211_SMPS_OFF;
link_sta->agg.max_rc_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA;
}
static struct sta_info *
__sta_info_alloc(struct ieee80211_sub_if_data *sdata,
const u8 *addr, int link_id, const u8 *link_addr,
gfp_t gfp)
{
struct ieee80211_local *local = sdata->local;
struct ieee80211_hw *hw = &local->hw;
struct sta_info *sta;
void *txq_data;
int size;
int i;
sta = kzalloc(sizeof(*sta) + hw->sta_data_size, gfp);
if (!sta)
return NULL;
sta->local = local;
sta->sdata = sdata;
if (sta_info_alloc_link(local, &sta->deflink, gfp))
goto free;
if (link_id >= 0) {
sta_info_add_link(sta, link_id, &sta->deflink,
&sta->sta.deflink);
sta->sta.valid_links = BIT(link_id);
} else {
sta_info_add_link(sta, 0, &sta->deflink, &sta->sta.deflink);
}
sta->sta.cur = &sta->sta.deflink.agg;
spin_lock_init(&sta->lock);
spin_lock_init(&sta->ps_lock);
INIT_WORK(&sta->drv_deliver_wk, sta_deliver_ps_frames);
wiphy_work_init(&sta->ampdu_mlme.work, ieee80211_ba_session_work);
#ifdef CONFIG_MAC80211_MESH
if (ieee80211_vif_is_mesh(&sdata->vif)) {
sta->mesh = kzalloc(sizeof(*sta->mesh), gfp);
if (!sta->mesh)
goto free;
sta->mesh->plink_sta = sta;
spin_lock_init(&sta->mesh->plink_lock);
if (!sdata->u.mesh.user_mpm)
timer_setup(&sta->mesh->plink_timer, mesh_plink_timer,
0);
sta->mesh->nonpeer_pm = NL80211_MESH_POWER_ACTIVE;
}
#endif
memcpy(sta->addr, addr, ETH_ALEN);
memcpy(sta->sta.addr, addr, ETH_ALEN);
memcpy(sta->deflink.addr, link_addr, ETH_ALEN);
memcpy(sta->sta.deflink.addr, link_addr, ETH_ALEN);
sta->sta.max_rx_aggregation_subframes =
local->hw.max_rx_aggregation_subframes;
/* TODO link specific alloc and assignments for MLO Link STA */
/* Extended Key ID needs to install keys for keyid 0 and 1 Rx-only.
* The Tx path starts to use a key as soon as the key slot ptk_idx
* references to is not NULL. To not use the initial Rx-only key
* prematurely for Tx initialize ptk_idx to an impossible PTK keyid
* which always will refer to a NULL key.
*/
BUILD_BUG_ON(ARRAY_SIZE(sta->ptk) <= INVALID_PTK_KEYIDX);
sta->ptk_idx = INVALID_PTK_KEYIDX;
ieee80211_init_frag_cache(&sta->frags);
sta->sta_state = IEEE80211_STA_NONE;
if (sdata->vif.type == NL80211_IFTYPE_MESH_POINT)
sta->amsdu_mesh_control = -1;
/* Mark TID as unreserved */
sta->reserved_tid = IEEE80211_TID_UNRESERVED;
sta->last_connected = ktime_get_seconds();
size = sizeof(struct txq_info) +
ALIGN(hw->txq_data_size, sizeof(void *));
txq_data = kcalloc(ARRAY_SIZE(sta->sta.txq), size, gfp);
if (!txq_data)
goto free;
for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
struct txq_info *txq = txq_data + i * size;
/* might not do anything for the (bufferable) MMPDU TXQ */
ieee80211_txq_init(sdata, sta, txq, i);
}
if (sta_prepare_rate_control(local, sta, gfp))
goto free_txq;
sta->airtime_weight = IEEE80211_DEFAULT_AIRTIME_WEIGHT;
for (i = 0; i < IEEE80211_NUM_ACS; i++) {
skb_queue_head_init(&sta->ps_tx_buf[i]);
skb_queue_head_init(&sta->tx_filtered[i]);
sta->airtime[i].deficit = sta->airtime_weight;
atomic_set(&sta->airtime[i].aql_tx_pending, 0);
sta->airtime[i].aql_limit_low = local->aql_txq_limit_low[i];
sta->airtime[i].aql_limit_high = local->aql_txq_limit_high[i];
}
for (i = 0; i < IEEE80211_NUM_TIDS; i++)
sta->last_seq_ctrl[i] = cpu_to_le16(USHRT_MAX);
for (i = 0; i < NUM_NL80211_BANDS; i++) {
u32 mandatory = 0;
int r;
if (!hw->wiphy->bands[i])
continue;
switch (i) {
case NL80211_BAND_2GHZ:
case NL80211_BAND_LC:
/*
* We use both here, even if we cannot really know for
* sure the station will support both, but the only use
* for this is when we don't know anything yet and send
* management frames, and then we'll pick the lowest
* possible rate anyway.
* If we don't include _G here, we cannot find a rate
* in P2P, and thus trigger the WARN_ONCE() in rate.c
*/
mandatory = IEEE80211_RATE_MANDATORY_B |
IEEE80211_RATE_MANDATORY_G;
break;
case NL80211_BAND_5GHZ:
mandatory = IEEE80211_RATE_MANDATORY_A;
break;
case NL80211_BAND_60GHZ:
WARN_ON(1);
mandatory = 0;
break;
}
for (r = 0; r < hw->wiphy->bands[i]->n_bitrates; r++) {
struct ieee80211_rate *rate;
rate = &hw->wiphy->bands[i]->bitrates[r];
if (!(rate->flags & mandatory))
continue;
sta->sta.deflink.supp_rates[i] |= BIT(r);
}
}
sta->cparams.ce_threshold = CODEL_DISABLED_THRESHOLD;
sta->cparams.target = MS2TIME(20);
sta->cparams.interval = MS2TIME(100);
sta->cparams.ecn = true;
sta->cparams.ce_threshold_selector = 0;
sta->cparams.ce_threshold_mask = 0;
sta_dbg(sdata, "Allocated STA %pM\n", sta->sta.addr);
return sta;
free_txq:
kfree(to_txq_info(sta->sta.txq[0]));
free:
sta_info_free_link(&sta->deflink);
#ifdef CONFIG_MAC80211_MESH
kfree(sta->mesh);
#endif
kfree(sta);
return NULL;
}
struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata,
const u8 *addr, gfp_t gfp)
{
return __sta_info_alloc(sdata, addr, -1, addr, gfp);
}
struct sta_info *sta_info_alloc_with_link(struct ieee80211_sub_if_data *sdata,
const u8 *mld_addr,
unsigned int link_id,
const u8 *link_addr,
gfp_t gfp)
{
return __sta_info_alloc(sdata, mld_addr, link_id, link_addr, gfp);
}
static int sta_info_insert_check(struct sta_info *sta)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
lockdep_assert_wiphy(sdata->local->hw.wiphy);
/*
* Can't be a WARN_ON because it can be triggered through a race:
* something inserts a STA (on one CPU) without holding the RTNL
* and another CPU turns off the net device.
*/
if (unlikely(!ieee80211_sdata_running(sdata)))
return -ENETDOWN;
if (WARN_ON(ether_addr_equal(sta->sta.addr, sdata->vif.addr) ||
!is_valid_ether_addr(sta->sta.addr)))
return -EINVAL;
/* The RCU read lock is required by rhashtable due to
* asynchronous resize/rehash. We also require the mutex
* for correctness.
*/
rcu_read_lock();
if (ieee80211_hw_check(&sdata->local->hw, NEEDS_UNIQUE_STA_ADDR) &&
ieee80211_find_sta_by_ifaddr(&sdata->local->hw, sta->addr, NULL)) {
rcu_read_unlock();
return -ENOTUNIQ;
}
rcu_read_unlock();
return 0;
}
static int sta_info_insert_drv_state(struct ieee80211_local *local,
struct ieee80211_sub_if_data *sdata,
struct sta_info *sta)
{
enum ieee80211_sta_state state;
int err = 0;
for (state = IEEE80211_STA_NOTEXIST; state < sta->sta_state; state++) {
err = drv_sta_state(local, sdata, sta, state, state + 1);
if (err)
break;
}
if (!err) {
/*
* Drivers using legacy sta_add/sta_remove callbacks only
* get uploaded set to true after sta_add is called.
*/
if (!local->ops->sta_add)
sta->uploaded = true;
return 0;
}
if (sdata->vif.type == NL80211_IFTYPE_ADHOC) {
sdata_info(sdata,
"failed to move IBSS STA %pM to state %d (%d) - keeping it anyway\n",
sta->sta.addr, state + 1, err);
err = 0;
}
/* unwind on error */
for (; state > IEEE80211_STA_NOTEXIST; state--)
WARN_ON(drv_sta_state(local, sdata, sta, state, state - 1));
return err;
}
static void
ieee80211_recalc_p2p_go_ps_allowed(struct ieee80211_sub_if_data *sdata)
{
struct ieee80211_local *local = sdata->local;
bool allow_p2p_go_ps = sdata->vif.p2p;
struct sta_info *sta;
rcu_read_lock();
list_for_each_entry_rcu(sta, &local->sta_list, list) {
if (sdata != sta->sdata ||
!test_sta_flag(sta, WLAN_STA_ASSOC))
continue;
if (!sta->sta.support_p2p_ps) {
allow_p2p_go_ps = false;
break;
}
}
rcu_read_unlock();
if (allow_p2p_go_ps != sdata->vif.bss_conf.allow_p2p_go_ps) {
sdata->vif.bss_conf.allow_p2p_go_ps = allow_p2p_go_ps;
ieee80211_link_info_change_notify(sdata, &sdata->deflink,
BSS_CHANGED_P2P_PS);
}
}
static int sta_info_insert_finish(struct sta_info *sta) __acquires(RCU)
{
struct ieee80211_local *local = sta->local;
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct station_info *sinfo = NULL;
int err = 0;
lockdep_assert_wiphy(local->hw.wiphy);
/* check if STA exists already */
if (sta_info_get_bss(sdata, sta->sta.addr)) {
err = -EEXIST;
goto out_cleanup;
}
sinfo = kzalloc(sizeof(struct station_info), GFP_KERNEL);
if (!sinfo) {
err = -ENOMEM;
goto out_cleanup;
}
local->num_sta++;
local->sta_generation++;
smp_mb();
/* simplify things and don't accept BA sessions yet */
set_sta_flag(sta, WLAN_STA_BLOCK_BA);
/* make the station visible */
err = sta_info_hash_add(local, sta);
if (err)
goto out_drop_sta;
if (sta->sta.valid_links) {
err = link_sta_info_hash_add(local, &sta->deflink);
if (err) {
sta_info_hash_del(local, sta);
goto out_drop_sta;
}
}
list_add_tail_rcu(&sta->list, &local->sta_list);
/* update channel context before notifying the driver about state
* change, this enables driver using the updated channel context right away.
*/
if (sta->sta_state >= IEEE80211_STA_ASSOC) {
ieee80211_recalc_min_chandef(sta->sdata, -1);
if (!sta->sta.support_p2p_ps)
ieee80211_recalc_p2p_go_ps_allowed(sta->sdata);
}
/* notify driver */
err = sta_info_insert_drv_state(local, sdata, sta);
if (err)
goto out_remove;
set_sta_flag(sta, WLAN_STA_INSERTED);
/* accept BA sessions now */
clear_sta_flag(sta, WLAN_STA_BLOCK_BA);
ieee80211_sta_debugfs_add(sta);
rate_control_add_sta_debugfs(sta);
if (sta->sta.valid_links) {
int i;
for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
struct link_sta_info *link_sta;
link_sta = rcu_dereference_protected(sta->link[i],
lockdep_is_held(&local->hw.wiphy->mtx));
if (!link_sta)
continue;
ieee80211_link_sta_debugfs_add(link_sta);
if (sdata->vif.active_links & BIT(i))
ieee80211_link_sta_debugfs_drv_add(link_sta);
}
} else {
ieee80211_link_sta_debugfs_add(&sta->deflink);
ieee80211_link_sta_debugfs_drv_add(&sta->deflink);
}
sinfo->generation = local->sta_generation;
cfg80211_new_sta(sdata->dev, sta->sta.addr, sinfo, GFP_KERNEL);
kfree(sinfo);
sta_dbg(sdata, "Inserted STA %pM\n", sta->sta.addr);
/* move reference to rcu-protected */
rcu_read_lock();
if (ieee80211_vif_is_mesh(&sdata->vif))
mesh_accept_plinks_update(sdata);
ieee80211_check_fast_xmit(sta);
return 0;
out_remove:
if (sta->sta.valid_links)
link_sta_info_hash_del(local, &sta->deflink);
sta_info_hash_del(local, sta);
list_del_rcu(&sta->list);
out_drop_sta:
local->num_sta--;
synchronize_net();
out_cleanup:
cleanup_single_sta(sta);
kfree(sinfo);
rcu_read_lock();
return err;
}
int sta_info_insert_rcu(struct sta_info *sta) __acquires(RCU)
{
struct ieee80211_local *local = sta->local;
int err;
might_sleep();
lockdep_assert_wiphy(local->hw.wiphy);
err = sta_info_insert_check(sta);
if (err) {
sta_info_free(local, sta);
rcu_read_lock();
return err;
}
return sta_info_insert_finish(sta);
}
int sta_info_insert(struct sta_info *sta)
{
int err = sta_info_insert_rcu(sta);
rcu_read_unlock();
return err;
}
static inline void __bss_tim_set(u8 *tim, u16 id)
{
/*
* This format has been mandated by the IEEE specifications,
* so this line may not be changed to use the __set_bit() format.
*/
tim[id / 8] |= (1 << (id % 8));
}
static inline void __bss_tim_clear(u8 *tim, u16 id)
{
/*
* This format has been mandated by the IEEE specifications,
* so this line may not be changed to use the __clear_bit() format.
*/
tim[id / 8] &= ~(1 << (id % 8));
}
static inline bool __bss_tim_get(u8 *tim, u16 id)
{
/*
* This format has been mandated by the IEEE specifications,
* so this line may not be changed to use the test_bit() format.
*/
return tim[id / 8] & (1 << (id % 8));
}
static unsigned long ieee80211_tids_for_ac(int ac)
{
/* If we ever support TIDs > 7, this obviously needs to be adjusted */
switch (ac) {
case IEEE80211_AC_VO:
return BIT(6) | BIT(7);
case IEEE80211_AC_VI:
return BIT(4) | BIT(5);
case IEEE80211_AC_BE:
return BIT(0) | BIT(3);
case IEEE80211_AC_BK:
return BIT(1) | BIT(2);
default:
WARN_ON(1);
return 0;
}
}
static void __sta_info_recalc_tim(struct sta_info *sta, bool ignore_pending)
{
struct ieee80211_local *local = sta->local;
struct ps_data *ps;
bool indicate_tim = false;
u8 ignore_for_tim = sta->sta.uapsd_queues;
int ac;
u16 id = sta->sta.aid;
if (sta->sdata->vif.type == NL80211_IFTYPE_AP ||
sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN) {
if (WARN_ON_ONCE(!sta->sdata->bss))
return;
ps = &sta->sdata->bss->ps;
#ifdef CONFIG_MAC80211_MESH
} else if (ieee80211_vif_is_mesh(&sta->sdata->vif)) {
ps = &sta->sdata->u.mesh.ps;
#endif
} else {
return;
}
/* No need to do anything if the driver does all */
if (ieee80211_hw_check(&local->hw, AP_LINK_PS) && !local->ops->set_tim)
return;
if (sta->dead)
goto done;
/*
* If all ACs are delivery-enabled then we should build
* the TIM bit for all ACs anyway; if only some are then
* we ignore those and build the TIM bit using only the
* non-enabled ones.
*/
if (ignore_for_tim == BIT(IEEE80211_NUM_ACS) - 1)
ignore_for_tim = 0;
if (ignore_pending)
ignore_for_tim = BIT(IEEE80211_NUM_ACS) - 1;
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
unsigned long tids;
if (ignore_for_tim & ieee80211_ac_to_qos_mask[ac])
continue;
indicate_tim |= !skb_queue_empty(&sta->tx_filtered[ac]) ||
!skb_queue_empty(&sta->ps_tx_buf[ac]);
if (indicate_tim)
break;
tids = ieee80211_tids_for_ac(ac);
indicate_tim |=
sta->driver_buffered_tids & tids;
indicate_tim |=
sta->txq_buffered_tids & tids;
}
done:
spin_lock_bh(&local->tim_lock);
if (indicate_tim == __bss_tim_get(ps->tim, id))
goto out_unlock;
if (indicate_tim)
__bss_tim_set(ps->tim, id);
else
__bss_tim_clear(ps->tim, id);
if (local->ops->set_tim && !WARN_ON(sta->dead)) {
local->tim_in_locked_section = true;
drv_set_tim(local, &sta->sta, indicate_tim);
local->tim_in_locked_section = false;
}
out_unlock:
spin_unlock_bh(&local->tim_lock);
}
void sta_info_recalc_tim(struct sta_info *sta)
{
__sta_info_recalc_tim(sta, false);
}
static bool sta_info_buffer_expired(struct sta_info *sta, struct sk_buff *skb)
{
struct ieee80211_tx_info *info;
int timeout;
if (!skb)
return false;
info = IEEE80211_SKB_CB(skb);
/* Timeout: (2 * listen_interval * beacon_int * 1024 / 1000000) sec */
timeout = (sta->listen_interval *
sta->sdata->vif.bss_conf.beacon_int *
32 / 15625) * HZ;
if (timeout < STA_TX_BUFFER_EXPIRE)
timeout = STA_TX_BUFFER_EXPIRE;
return time_after(jiffies, info->control.jiffies + timeout);
}
static bool sta_info_cleanup_expire_buffered_ac(struct ieee80211_local *local,
struct sta_info *sta, int ac)
{
unsigned long flags;
struct sk_buff *skb;
/*
* First check for frames that should expire on the filtered
* queue. Frames here were rejected by the driver and are on
* a separate queue to avoid reordering with normal PS-buffered
* frames. They also aren't accounted for right now in the
* total_ps_buffered counter.
*/
for (;;) {
spin_lock_irqsave(&sta->tx_filtered[ac].lock, flags);
skb = skb_peek(&sta->tx_filtered[ac]);
if (sta_info_buffer_expired(sta, skb))
skb = __skb_dequeue(&sta->tx_filtered[ac]);
else
skb = NULL;
spin_unlock_irqrestore(&sta->tx_filtered[ac].lock, flags);
/*
* Frames are queued in order, so if this one
* hasn't expired yet we can stop testing. If
* we actually reached the end of the queue we
* also need to stop, of course.
*/
if (!skb)
break;
ieee80211_free_txskb(&local->hw, skb);
}
/*
* Now also check the normal PS-buffered queue, this will
* only find something if the filtered queue was emptied
* since the filtered frames are all before the normal PS
* buffered frames.
*/
for (;;) {
spin_lock_irqsave(&sta->ps_tx_buf[ac].lock, flags);
skb = skb_peek(&sta->ps_tx_buf[ac]);
if (sta_info_buffer_expired(sta, skb))
skb = __skb_dequeue(&sta->ps_tx_buf[ac]);
else
skb = NULL;
spin_unlock_irqrestore(&sta->ps_tx_buf[ac].lock, flags);
/*
* frames are queued in order, so if this one
* hasn't expired yet (or we reached the end of
* the queue) we can stop testing
*/
if (!skb)
break;
local->total_ps_buffered--;
ps_dbg(sta->sdata, "Buffered frame expired (STA %pM)\n",
sta->sta.addr);
ieee80211_free_txskb(&local->hw, skb);
}
/*
* Finally, recalculate the TIM bit for this station -- it might
* now be clear because the station was too slow to retrieve its
* frames.
*/
sta_info_recalc_tim(sta);
/*
* Return whether there are any frames still buffered, this is
* used to check whether the cleanup timer still needs to run,
* if there are no frames we don't need to rearm the timer.
*/
return !(skb_queue_empty(&sta->ps_tx_buf[ac]) &&
skb_queue_empty(&sta->tx_filtered[ac]));
}
static bool sta_info_cleanup_expire_buffered(struct ieee80211_local *local,
struct sta_info *sta)
{
bool have_buffered = false;
int ac;
/* This is only necessary for stations on BSS/MBSS interfaces */
if (!sta->sdata->bss &&
!ieee80211_vif_is_mesh(&sta->sdata->vif))
return false;
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
have_buffered |=
sta_info_cleanup_expire_buffered_ac(local, sta, ac);
return have_buffered;
}
static int __must_check __sta_info_destroy_part1(struct sta_info *sta)
{
struct ieee80211_local *local;
struct ieee80211_sub_if_data *sdata;
int ret, i;
might_sleep();
if (!sta)
return -ENOENT;
local = sta->local;
sdata = sta->sdata;
lockdep_assert_wiphy(local->hw.wiphy);
/*
* Before removing the station from the driver and
* rate control, it might still start new aggregation
* sessions -- block that to make sure the tear-down
* will be sufficient.
*/
set_sta_flag(sta, WLAN_STA_BLOCK_BA);
ieee80211_sta_tear_down_BA_sessions(sta, AGG_STOP_DESTROY_STA);
/*
* Before removing the station from the driver there might be pending
* rx frames on RSS queues sent prior to the disassociation - wait for
* all such frames to be processed.
*/
drv_sync_rx_queues(local, sta);
for (i = 0; i < ARRAY_SIZE(sta->link); i++) {
struct link_sta_info *link_sta;
if (!(sta->sta.valid_links & BIT(i)))
continue;
link_sta = rcu_dereference_protected(sta->link[i],
lockdep_is_held(&local->hw.wiphy->mtx));
link_sta_info_hash_del(local, link_sta);
}
ret = sta_info_hash_del(local, sta);
if (WARN_ON(ret))
return ret;
/*
* for TDLS peers, make sure to return to the base channel before
* removal.
*/
if (test_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL)) {
drv_tdls_cancel_channel_switch(local, sdata, &sta->sta);
clear_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL);
}
list_del_rcu(&sta->list);
sta->removed = true;
if (sta->uploaded)
drv_sta_pre_rcu_remove(local, sta->sdata, sta);
if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN &&
rcu_access_pointer(sdata->u.vlan.sta) == sta)
RCU_INIT_POINTER(sdata->u.vlan.sta, NULL);
return 0;
}
static int _sta_info_move_state(struct sta_info *sta,
enum ieee80211_sta_state new_state,
bool recalc)
{
struct ieee80211_local *local = sta->local;
might_sleep();
if (sta->sta_state == new_state)
return 0;
/* check allowed transitions first */
switch (new_state) {
case IEEE80211_STA_NONE:
if (sta->sta_state != IEEE80211_STA_AUTH)
return -EINVAL;
break;
case IEEE80211_STA_AUTH:
if (sta->sta_state != IEEE80211_STA_NONE &&
sta->sta_state != IEEE80211_STA_ASSOC)
return -EINVAL;
break;
case IEEE80211_STA_ASSOC:
if (sta->sta_state != IEEE80211_STA_AUTH &&
sta->sta_state != IEEE80211_STA_AUTHORIZED)
return -EINVAL;
break;
case IEEE80211_STA_AUTHORIZED:
if (sta->sta_state != IEEE80211_STA_ASSOC)
return -EINVAL;
break;
default:
WARN(1, "invalid state %d", new_state);
return -EINVAL;
}
sta_dbg(sta->sdata, "moving STA %pM to state %d\n",
sta->sta.addr, new_state);
/* notify the driver before the actual changes so it can
* fail the transition
*/
if (test_sta_flag(sta, WLAN_STA_INSERTED)) {
int err = drv_sta_state(sta->local, sta->sdata, sta,
sta->sta_state, new_state);
if (err)
return err;
}
/* reflect the change in all state variables */
switch (new_state) {
case IEEE80211_STA_NONE:
if (sta->sta_state == IEEE80211_STA_AUTH)
clear_bit(WLAN_STA_AUTH, &sta->_flags);
break;
case IEEE80211_STA_AUTH:
if (sta->sta_state == IEEE80211_STA_NONE) {
set_bit(WLAN_STA_AUTH, &sta->_flags);
} else if (sta->sta_state == IEEE80211_STA_ASSOC) {
clear_bit(WLAN_STA_ASSOC, &sta->_flags);
if (recalc) {
ieee80211_recalc_min_chandef(sta->sdata, -1);
if (!sta->sta.support_p2p_ps)
ieee80211_recalc_p2p_go_ps_allowed(sta->sdata);
}
}
break;
case IEEE80211_STA_ASSOC:
if (sta->sta_state == IEEE80211_STA_AUTH) {
set_bit(WLAN_STA_ASSOC, &sta->_flags);
sta->assoc_at = ktime_get_boottime_ns();
if (recalc) {
ieee80211_recalc_min_chandef(sta->sdata, -1);
if (!sta->sta.support_p2p_ps)
ieee80211_recalc_p2p_go_ps_allowed(sta->sdata);
}
} else if (sta->sta_state == IEEE80211_STA_AUTHORIZED) {
ieee80211_vif_dec_num_mcast(sta->sdata);
clear_bit(WLAN_STA_AUTHORIZED, &sta->_flags);
/*
* If we have encryption offload, flush (station) queues
* (after ensuring concurrent TX completed) so we won't
* transmit anything later unencrypted if/when keys are
* also removed, which might otherwise happen depending
* on how the hardware offload works.
*/
if (local->ops->set_key) {
synchronize_net();
if (local->ops->flush_sta)
drv_flush_sta(local, sta->sdata, sta);
else
ieee80211_flush_queues(local,
sta->sdata,
false);
}
ieee80211_clear_fast_xmit(sta);
ieee80211_clear_fast_rx(sta);
}
break;
case IEEE80211_STA_AUTHORIZED:
if (sta->sta_state == IEEE80211_STA_ASSOC) {
ieee80211_vif_inc_num_mcast(sta->sdata);
set_bit(WLAN_STA_AUTHORIZED, &sta->_flags);
ieee80211_check_fast_xmit(sta);
ieee80211_check_fast_rx(sta);
}
if (sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN ||
sta->sdata->vif.type == NL80211_IFTYPE_AP)
cfg80211_send_layer2_update(sta->sdata->dev,
sta->sta.addr);
break;
default:
break;
}
sta->sta_state = new_state;
return 0;
}
int sta_info_move_state(struct sta_info *sta,
enum ieee80211_sta_state new_state)
{
return _sta_info_move_state(sta, new_state, true);
}
static void __sta_info_destroy_part2(struct sta_info *sta, bool recalc)
{
struct ieee80211_local *local = sta->local;
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct station_info *sinfo;
int ret;
/*
* NOTE: This assumes at least synchronize_net() was done
* after _part1 and before _part2!
*/
/*
* There's a potential race in _part1 where we set WLAN_STA_BLOCK_BA
* but someone might have just gotten past a check, and not yet into
* queuing the work/creating the data/etc.
*
* Do another round of destruction so that the worker is certainly
* canceled before we later free the station.
*
* Since this is after synchronize_rcu()/synchronize_net() we're now
* certain that nobody can actually hold a reference to the STA and
* be calling e.g. ieee80211_start_tx_ba_session().
*/
ieee80211_sta_tear_down_BA_sessions(sta, AGG_STOP_DESTROY_STA);
might_sleep();
lockdep_assert_wiphy(local->hw.wiphy);
if (sta->sta_state == IEEE80211_STA_AUTHORIZED) {
ret = _sta_info_move_state(sta, IEEE80211_STA_ASSOC, recalc);
WARN_ON_ONCE(ret);
}
/* now keys can no longer be reached */
ieee80211_free_sta_keys(local, sta);
/* disable TIM bit - last chance to tell driver */
__sta_info_recalc_tim(sta, true);
sta->dead = true;
local->num_sta--;
local->sta_generation++;
while (sta->sta_state > IEEE80211_STA_NONE) {
ret = _sta_info_move_state(sta, sta->sta_state - 1, recalc);
if (ret) {
WARN_ON_ONCE(1);
break;
}
}
if (sta->uploaded) {
ret = drv_sta_state(local, sdata, sta, IEEE80211_STA_NONE,
IEEE80211_STA_NOTEXIST);
WARN_ON_ONCE(ret != 0);
}
sta_dbg(sdata, "Removed STA %pM\n", sta->sta.addr);
sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL);
if (sinfo)
sta_set_sinfo(sta, sinfo, true);
cfg80211_del_sta_sinfo(sdata->dev, sta->sta.addr, sinfo, GFP_KERNEL);
kfree(sinfo);
ieee80211_sta_debugfs_remove(sta);
ieee80211_destroy_frag_cache(&sta->frags);
cleanup_single_sta(sta);
}
int __must_check __sta_info_destroy(struct sta_info *sta)
{
int err = __sta_info_destroy_part1(sta);
if (err)
return err;
synchronize_net();
__sta_info_destroy_part2(sta, true);
return 0;
}
int sta_info_destroy_addr(struct ieee80211_sub_if_data *sdata, const u8 *addr)
{
struct sta_info *sta;
lockdep_assert_wiphy(sdata->local->hw.wiphy);
sta = sta_info_get(sdata, addr);
return __sta_info_destroy(sta);
}
int sta_info_destroy_addr_bss(struct ieee80211_sub_if_data *sdata,
const u8 *addr)
{
struct sta_info *sta;
lockdep_assert_wiphy(sdata->local->hw.wiphy);
sta = sta_info_get_bss(sdata, addr);
return __sta_info_destroy(sta);
}
static void sta_info_cleanup(struct timer_list *t)
{
struct ieee80211_local *local = from_timer(local, t, sta_cleanup);
struct sta_info *sta;
bool timer_needed = false;
rcu_read_lock();
list_for_each_entry_rcu(sta, &local->sta_list, list)
if (sta_info_cleanup_expire_buffered(local, sta))
timer_needed = true;
rcu_read_unlock();
if (local->quiescing)
return;
if (!timer_needed)
return;
mod_timer(&local->sta_cleanup,
round_jiffies(jiffies + STA_INFO_CLEANUP_INTERVAL));
}
int sta_info_init(struct ieee80211_local *local)
{
int err;
err = rhltable_init(&local->sta_hash, &sta_rht_params);
if (err)
return err;
err = rhltable_init(&local->link_sta_hash, &link_sta_rht_params);
if (err) {
rhltable_destroy(&local->sta_hash);
return err;
}
spin_lock_init(&local->tim_lock);
INIT_LIST_HEAD(&local->sta_list);
timer_setup(&local->sta_cleanup, sta_info_cleanup, 0);
return 0;
}
void sta_info_stop(struct ieee80211_local *local)
{
del_timer_sync(&local->sta_cleanup);
rhltable_destroy(&local->sta_hash);
rhltable_destroy(&local->link_sta_hash);
}
int __sta_info_flush(struct ieee80211_sub_if_data *sdata, bool vlans,
int link_id)
{
struct ieee80211_local *local = sdata->local;
struct sta_info *sta, *tmp;
LIST_HEAD(free_list);
int ret = 0;
might_sleep();
lockdep_assert_wiphy(local->hw.wiphy);
WARN_ON(vlans && sdata->vif.type != NL80211_IFTYPE_AP);
WARN_ON(vlans && !sdata->bss);
list_for_each_entry_safe(sta, tmp, &local->sta_list, list) {
if (sdata != sta->sdata &&
(!vlans || sdata->bss != sta->sdata->bss))
continue;
if (link_id >= 0 && sta->sta.valid_links &&
!(sta->sta.valid_links & BIT(link_id)))
continue;
if (!WARN_ON(__sta_info_destroy_part1(sta)))
list_add(&sta->free_list, &free_list);
ret++;
}
if (!list_empty(&free_list)) {
bool support_p2p_ps = true;
synchronize_net();
list_for_each_entry_safe(sta, tmp, &free_list, free_list) {
if (!sta->sta.support_p2p_ps)
support_p2p_ps = false;
__sta_info_destroy_part2(sta, false);
}
ieee80211_recalc_min_chandef(sdata, -1);
if (!support_p2p_ps)
ieee80211_recalc_p2p_go_ps_allowed(sdata);
}
return ret;
}
void ieee80211_sta_expire(struct ieee80211_sub_if_data *sdata,
unsigned long exp_time)
{
struct ieee80211_local *local = sdata->local;
struct sta_info *sta, *tmp;
lockdep_assert_wiphy(local->hw.wiphy);
list_for_each_entry_safe(sta, tmp, &local->sta_list, list) {
unsigned long last_active = ieee80211_sta_last_active(sta);
if (sdata != sta->sdata)
continue;
if (time_is_before_jiffies(last_active + exp_time)) {
sta_dbg(sta->sdata, "expiring inactive STA %pM\n",
sta->sta.addr);
if (ieee80211_vif_is_mesh(&sdata->vif) &&
test_sta_flag(sta, WLAN_STA_PS_STA))
atomic_dec(&sdata->u.mesh.ps.num_sta_ps);
WARN_ON(__sta_info_destroy(sta));
}
}
}
struct ieee80211_sta *ieee80211_find_sta_by_ifaddr(struct ieee80211_hw *hw,
const u8 *addr,
const u8 *localaddr)
{
struct ieee80211_local *local = hw_to_local(hw);
struct rhlist_head *tmp;
struct sta_info *sta;
/*
* Just return a random station if localaddr is NULL
* ... first in list.
*/
for_each_sta_info(local, addr, sta, tmp) {
if (localaddr &&
!ether_addr_equal(sta->sdata->vif.addr, localaddr))
continue;
if (!sta->uploaded)
return NULL;
return &sta->sta;
}
return NULL;
}
EXPORT_SYMBOL_GPL(ieee80211_find_sta_by_ifaddr);
struct ieee80211_sta *ieee80211_find_sta(struct ieee80211_vif *vif,
const u8 *addr)
{
struct sta_info *sta;
if (!vif)
return NULL;
sta = sta_info_get_bss(vif_to_sdata(vif), addr);
if (!sta)
return NULL;
if (!sta->uploaded)
return NULL;
return &sta->sta;
}
EXPORT_SYMBOL(ieee80211_find_sta);
/* powersave support code */
void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
struct sk_buff_head pending;
int filtered = 0, buffered = 0, ac, i;
unsigned long flags;
struct ps_data *ps;
if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
sdata = container_of(sdata->bss, struct ieee80211_sub_if_data,
u.ap);
if (sdata->vif.type == NL80211_IFTYPE_AP)
ps = &sdata->bss->ps;
else if (ieee80211_vif_is_mesh(&sdata->vif))
ps = &sdata->u.mesh.ps;
else
return;
clear_sta_flag(sta, WLAN_STA_SP);
BUILD_BUG_ON(BITS_TO_LONGS(IEEE80211_NUM_TIDS) > 1);
sta->driver_buffered_tids = 0;
sta->txq_buffered_tids = 0;
if (!ieee80211_hw_check(&local->hw, AP_LINK_PS))
drv_sta_notify(local, sdata, STA_NOTIFY_AWAKE, &sta->sta);
for (i = 0; i < ARRAY_SIZE(sta->sta.txq); i++) {
if (!sta->sta.txq[i] || !txq_has_queue(sta->sta.txq[i]))
continue;
schedule_and_wake_txq(local, to_txq_info(sta->sta.txq[i]));
}
skb_queue_head_init(&pending);
/* sync with ieee80211_tx_h_unicast_ps_buf */
spin_lock(&sta->ps_lock);
/* Send all buffered frames to the station */
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
int count = skb_queue_len(&pending), tmp;
spin_lock_irqsave(&sta->tx_filtered[ac].lock, flags);
skb_queue_splice_tail_init(&sta->tx_filtered[ac], &pending);
spin_unlock_irqrestore(&sta->tx_filtered[ac].lock, flags);
tmp = skb_queue_len(&pending);
filtered += tmp - count;
count = tmp;
spin_lock_irqsave(&sta->ps_tx_buf[ac].lock, flags);
skb_queue_splice_tail_init(&sta->ps_tx_buf[ac], &pending);
spin_unlock_irqrestore(&sta->ps_tx_buf[ac].lock, flags);
tmp = skb_queue_len(&pending);
buffered += tmp - count;
}
ieee80211_add_pending_skbs(local, &pending);
/* now we're no longer in the deliver code */
clear_sta_flag(sta, WLAN_STA_PS_DELIVER);
/* The station might have polled and then woken up before we responded,
* so clear these flags now to avoid them sticking around.
*/
clear_sta_flag(sta, WLAN_STA_PSPOLL);
clear_sta_flag(sta, WLAN_STA_UAPSD);
spin_unlock(&sta->ps_lock);
atomic_dec(&ps->num_sta_ps);
local->total_ps_buffered -= buffered;
sta_info_recalc_tim(sta);
ps_dbg(sdata,
"STA %pM aid %d sending %d filtered/%d PS frames since STA woke up\n",
sta->sta.addr, sta->sta.aid, filtered, buffered);
ieee80211_check_fast_xmit(sta);
}
static void ieee80211_send_null_response(struct sta_info *sta, int tid,
enum ieee80211_frame_release_type reason,
bool call_driver, bool more_data)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
struct ieee80211_qos_hdr *nullfunc;
struct sk_buff *skb;
int size = sizeof(*nullfunc);
__le16 fc;
bool qos = sta->sta.wme;
struct ieee80211_tx_info *info;
struct ieee80211_chanctx_conf *chanctx_conf;
if (qos) {
fc = cpu_to_le16(IEEE80211_FTYPE_DATA |
IEEE80211_STYPE_QOS_NULLFUNC |
IEEE80211_FCTL_FROMDS);
} else {
size -= 2;
fc = cpu_to_le16(IEEE80211_FTYPE_DATA |
IEEE80211_STYPE_NULLFUNC |
IEEE80211_FCTL_FROMDS);
}
skb = dev_alloc_skb(local->hw.extra_tx_headroom + size);
if (!skb)
return;
skb_reserve(skb, local->hw.extra_tx_headroom);
nullfunc = skb_put(skb, size);
nullfunc->frame_control = fc;
nullfunc->duration_id = 0;
memcpy(nullfunc->addr1, sta->sta.addr, ETH_ALEN);
memcpy(nullfunc->addr2, sdata->vif.addr, ETH_ALEN);
memcpy(nullfunc->addr3, sdata->vif.addr, ETH_ALEN);
nullfunc->seq_ctrl = 0;
skb->priority = tid;
skb_set_queue_mapping(skb, ieee802_1d_to_ac[tid]);
if (qos) {
nullfunc->qos_ctrl = cpu_to_le16(tid);
if (reason == IEEE80211_FRAME_RELEASE_UAPSD) {
nullfunc->qos_ctrl |=
cpu_to_le16(IEEE80211_QOS_CTL_EOSP);
if (more_data)
nullfunc->frame_control |=
cpu_to_le16(IEEE80211_FCTL_MOREDATA);
}
}
info = IEEE80211_SKB_CB(skb);
/*
* Tell TX path to send this frame even though the
* STA may still remain is PS mode after this frame
* exchange. Also set EOSP to indicate this packet
* ends the poll/service period.
*/
info->flags |= IEEE80211_TX_CTL_NO_PS_BUFFER |
IEEE80211_TX_STATUS_EOSP |
IEEE80211_TX_CTL_REQ_TX_STATUS;
info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;
if (call_driver)
drv_allow_buffered_frames(local, sta, BIT(tid), 1,
reason, false);
skb->dev = sdata->dev;
rcu_read_lock();
chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
if (WARN_ON(!chanctx_conf)) {
rcu_read_unlock();
kfree_skb(skb);
return;
}
info->band = chanctx_conf->def.chan->band;
ieee80211_xmit(sdata, sta, skb);
rcu_read_unlock();
}
static int find_highest_prio_tid(unsigned long tids)
{
/* lower 3 TIDs aren't ordered perfectly */
if (tids & 0xF8)
return fls(tids) - 1;
/* TID 0 is BE just like TID 3 */
if (tids & BIT(0))
return 0;
return fls(tids) - 1;
}
/* Indicates if the MORE_DATA bit should be set in the last
* frame obtained by ieee80211_sta_ps_get_frames.
* Note that driver_release_tids is relevant only if
* reason = IEEE80211_FRAME_RELEASE_PSPOLL
*/
static bool
ieee80211_sta_ps_more_data(struct sta_info *sta, u8 ignored_acs,
enum ieee80211_frame_release_type reason,
unsigned long driver_release_tids)
{
int ac;
/* If the driver has data on more than one TID then
* certainly there's more data if we release just a
* single frame now (from a single TID). This will
* only happen for PS-Poll.
*/
if (reason == IEEE80211_FRAME_RELEASE_PSPOLL &&
hweight16(driver_release_tids) > 1)
return true;
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
if (ignored_acs & ieee80211_ac_to_qos_mask[ac])
continue;
if (!skb_queue_empty(&sta->tx_filtered[ac]) ||
!skb_queue_empty(&sta->ps_tx_buf[ac]))
return true;
}
return false;
}
static void
ieee80211_sta_ps_get_frames(struct sta_info *sta, int n_frames, u8 ignored_acs,
enum ieee80211_frame_release_type reason,
struct sk_buff_head *frames,
unsigned long *driver_release_tids)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
int ac;
/* Get response frame(s) and more data bit for the last one. */
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
unsigned long tids;
if (ignored_acs & ieee80211_ac_to_qos_mask[ac])
continue;
tids = ieee80211_tids_for_ac(ac);
/* if we already have frames from software, then we can't also
* release from hardware queues
*/
if (skb_queue_empty(frames)) {
*driver_release_tids |=
sta->driver_buffered_tids & tids;
*driver_release_tids |= sta->txq_buffered_tids & tids;
}
if (!*driver_release_tids) {
struct sk_buff *skb;
while (n_frames > 0) {
skb = skb_dequeue(&sta->tx_filtered[ac]);
if (!skb) {
skb = skb_dequeue(
&sta->ps_tx_buf[ac]);
if (skb)
local->total_ps_buffered--;
}
if (!skb)
break;
n_frames--;
__skb_queue_tail(frames, skb);
}
}
/* If we have more frames buffered on this AC, then abort the
* loop since we can't send more data from other ACs before
* the buffered frames from this.
*/
if (!skb_queue_empty(&sta->tx_filtered[ac]) ||
!skb_queue_empty(&sta->ps_tx_buf[ac]))
break;
}
}
static void
ieee80211_sta_ps_deliver_response(struct sta_info *sta,
int n_frames, u8 ignored_acs,
enum ieee80211_frame_release_type reason)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
unsigned long driver_release_tids = 0;
struct sk_buff_head frames;
bool more_data;
/* Service or PS-Poll period starts */
set_sta_flag(sta, WLAN_STA_SP);
__skb_queue_head_init(&frames);
ieee80211_sta_ps_get_frames(sta, n_frames, ignored_acs, reason,
&frames, &driver_release_tids);
more_data = ieee80211_sta_ps_more_data(sta, ignored_acs, reason, driver_release_tids);
if (driver_release_tids && reason == IEEE80211_FRAME_RELEASE_PSPOLL)
driver_release_tids =
BIT(find_highest_prio_tid(driver_release_tids));
if (skb_queue_empty(&frames) && !driver_release_tids) {
int tid, ac;
/*
* For PS-Poll, this can only happen due to a race condition
* when we set the TIM bit and the station notices it, but
* before it can poll for the frame we expire it.
*
* For uAPSD, this is said in the standard (11.2.1.5 h):
* At each unscheduled SP for a non-AP STA, the AP shall
* attempt to transmit at least one MSDU or MMPDU, but no
* more than the value specified in the Max SP Length field
* in the QoS Capability element from delivery-enabled ACs,
* that are destined for the non-AP STA.
*
* Since we have no other MSDU/MMPDU, transmit a QoS null frame.
*/
/* This will evaluate to 1, 3, 5 or 7. */
for (ac = IEEE80211_AC_VO; ac < IEEE80211_NUM_ACS; ac++)
if (!(ignored_acs & ieee80211_ac_to_qos_mask[ac]))
break;
tid = 7 - 2 * ac;
ieee80211_send_null_response(sta, tid, reason, true, false);
} else if (!driver_release_tids) {
struct sk_buff_head pending;
struct sk_buff *skb;
int num = 0;
u16 tids = 0;
bool need_null = false;
skb_queue_head_init(&pending);
while ((skb = __skb_dequeue(&frames))) {
struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
struct ieee80211_hdr *hdr = (void *) skb->data;
u8 *qoshdr = NULL;
num++;
/*
* Tell TX path to send this frame even though the
* STA may still remain is PS mode after this frame
* exchange.
*/
info->flags |= IEEE80211_TX_CTL_NO_PS_BUFFER;
info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;
/*
* Use MoreData flag to indicate whether there are
* more buffered frames for this STA
*/
if (more_data || !skb_queue_empty(&frames))
hdr->frame_control |=
cpu_to_le16(IEEE80211_FCTL_MOREDATA);
else
hdr->frame_control &=
cpu_to_le16(~IEEE80211_FCTL_MOREDATA);
if (ieee80211_is_data_qos(hdr->frame_control) ||
ieee80211_is_qos_nullfunc(hdr->frame_control))
qoshdr = ieee80211_get_qos_ctl(hdr);
tids |= BIT(skb->priority);
__skb_queue_tail(&pending, skb);
/* end service period after last frame or add one */
if (!skb_queue_empty(&frames))
continue;
if (reason != IEEE80211_FRAME_RELEASE_UAPSD) {
/* for PS-Poll, there's only one frame */
info->flags |= IEEE80211_TX_STATUS_EOSP |
IEEE80211_TX_CTL_REQ_TX_STATUS;
break;
}
/* For uAPSD, things are a bit more complicated. If the
* last frame has a QoS header (i.e. is a QoS-data or
* QoS-nulldata frame) then just set the EOSP bit there
* and be done.
* If the frame doesn't have a QoS header (which means
* it should be a bufferable MMPDU) then we can't set
* the EOSP bit in the QoS header; add a QoS-nulldata
* frame to the list to send it after the MMPDU.
*
* Note that this code is only in the mac80211-release
* code path, we assume that the driver will not buffer
* anything but QoS-data frames, or if it does, will
* create the QoS-nulldata frame by itself if needed.
*
* Cf. 802.11-2012 10.2.1.10 (c).
*/
if (qoshdr) {
*qoshdr |= IEEE80211_QOS_CTL_EOSP;
info->flags |= IEEE80211_TX_STATUS_EOSP |
IEEE80211_TX_CTL_REQ_TX_STATUS;
} else {
/* The standard isn't completely clear on this
* as it says the more-data bit should be set
* if there are more BUs. The QoS-Null frame
* we're about to send isn't buffered yet, we
* only create it below, but let's pretend it
* was buffered just in case some clients only
* expect more-data=0 when eosp=1.
*/
hdr->frame_control |=
cpu_to_le16(IEEE80211_FCTL_MOREDATA);
need_null = true;
num++;
}
break;
}
drv_allow_buffered_frames(local, sta, tids, num,
reason, more_data);
ieee80211_add_pending_skbs(local, &pending);
if (need_null)
ieee80211_send_null_response(
sta, find_highest_prio_tid(tids),
reason, false, false);
sta_info_recalc_tim(sta);
} else {
int tid;
/*
* We need to release a frame that is buffered somewhere in the
* driver ... it'll have to handle that.
* Note that the driver also has to check the number of frames
* on the TIDs we're releasing from - if there are more than
* n_frames it has to set the more-data bit (if we didn't ask
* it to set it anyway due to other buffered frames); if there
* are fewer than n_frames it has to make sure to adjust that
* to allow the service period to end properly.
*/
drv_release_buffered_frames(local, sta, driver_release_tids,
n_frames, reason, more_data);
/*
* Note that we don't recalculate the TIM bit here as it would
* most likely have no effect at all unless the driver told us
* that the TID(s) became empty before returning here from the
* release function.
* Either way, however, when the driver tells us that the TID(s)
* became empty or we find that a txq became empty, we'll do the
* TIM recalculation.
*/
for (tid = 0; tid < ARRAY_SIZE(sta->sta.txq); tid++) {
if (!sta->sta.txq[tid] ||
!(driver_release_tids & BIT(tid)) ||
txq_has_queue(sta->sta.txq[tid]))
continue;
sta_info_recalc_tim(sta);
break;
}
}
}
void ieee80211_sta_ps_deliver_poll_response(struct sta_info *sta)
{
u8 ignore_for_response = sta->sta.uapsd_queues;
/*
* If all ACs are delivery-enabled then we should reply
* from any of them, if only some are enabled we reply
* only from the non-enabled ones.
*/
if (ignore_for_response == BIT(IEEE80211_NUM_ACS) - 1)
ignore_for_response = 0;
ieee80211_sta_ps_deliver_response(sta, 1, ignore_for_response,
IEEE80211_FRAME_RELEASE_PSPOLL);
}
void ieee80211_sta_ps_deliver_uapsd(struct sta_info *sta)
{
int n_frames = sta->sta.max_sp;
u8 delivery_enabled = sta->sta.uapsd_queues;
/*
* If we ever grow support for TSPEC this might happen if
* the TSPEC update from hostapd comes in between a trigger
* frame setting WLAN_STA_UAPSD in the RX path and this
* actually getting called.
*/
if (!delivery_enabled)
return;
switch (sta->sta.max_sp) {
case 1:
n_frames = 2;
break;
case 2:
n_frames = 4;
break;
case 3:
n_frames = 6;
break;
case 0:
/* XXX: what is a good value? */
n_frames = 128;
break;
}
ieee80211_sta_ps_deliver_response(sta, n_frames, ~delivery_enabled,
IEEE80211_FRAME_RELEASE_UAPSD);
}
void ieee80211_sta_block_awake(struct ieee80211_hw *hw,
struct ieee80211_sta *pubsta, bool block)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
trace_api_sta_block_awake(sta->local, pubsta, block);
if (block) {
set_sta_flag(sta, WLAN_STA_PS_DRIVER);
ieee80211_clear_fast_xmit(sta);
return;
}
if (!test_sta_flag(sta, WLAN_STA_PS_DRIVER))
return;
if (!test_sta_flag(sta, WLAN_STA_PS_STA)) {
set_sta_flag(sta, WLAN_STA_PS_DELIVER);
clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
ieee80211_queue_work(hw, &sta->drv_deliver_wk);
} else if (test_sta_flag(sta, WLAN_STA_PSPOLL) ||
test_sta_flag(sta, WLAN_STA_UAPSD)) {
/* must be asleep in this case */
clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
ieee80211_queue_work(hw, &sta->drv_deliver_wk);
} else {
clear_sta_flag(sta, WLAN_STA_PS_DRIVER);
ieee80211_check_fast_xmit(sta);
}
}
EXPORT_SYMBOL(ieee80211_sta_block_awake);
void ieee80211_sta_eosp(struct ieee80211_sta *pubsta)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
struct ieee80211_local *local = sta->local;
trace_api_eosp(local, pubsta);
clear_sta_flag(sta, WLAN_STA_SP);
}
EXPORT_SYMBOL(ieee80211_sta_eosp);
void ieee80211_send_eosp_nullfunc(struct ieee80211_sta *pubsta, int tid)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
enum ieee80211_frame_release_type reason;
bool more_data;
trace_api_send_eosp_nullfunc(sta->local, pubsta, tid);
reason = IEEE80211_FRAME_RELEASE_UAPSD;
more_data = ieee80211_sta_ps_more_data(sta, ~sta->sta.uapsd_queues,
reason, 0);
ieee80211_send_null_response(sta, tid, reason, false, more_data);
}
EXPORT_SYMBOL(ieee80211_send_eosp_nullfunc);
void ieee80211_sta_set_buffered(struct ieee80211_sta *pubsta,
u8 tid, bool buffered)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
if (WARN_ON(tid >= IEEE80211_NUM_TIDS))
return;
trace_api_sta_set_buffered(sta->local, pubsta, tid, buffered);
if (buffered)
set_bit(tid, &sta->driver_buffered_tids);
else
clear_bit(tid, &sta->driver_buffered_tids);
sta_info_recalc_tim(sta);
}
EXPORT_SYMBOL(ieee80211_sta_set_buffered);
void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid,
u32 tx_airtime, u32 rx_airtime)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
struct ieee80211_local *local = sta->sdata->local;
u8 ac = ieee80211_ac_from_tid(tid);
u32 airtime = 0;
if (sta->local->airtime_flags & AIRTIME_USE_TX)
airtime += tx_airtime;
if (sta->local->airtime_flags & AIRTIME_USE_RX)
airtime += rx_airtime;
spin_lock_bh(&local->active_txq_lock[ac]);
sta->airtime[ac].tx_airtime += tx_airtime;
sta->airtime[ac].rx_airtime += rx_airtime;
if (ieee80211_sta_keep_active(sta, ac))
sta->airtime[ac].deficit -= airtime;
spin_unlock_bh(&local->active_txq_lock[ac]);
}
EXPORT_SYMBOL(ieee80211_sta_register_airtime);
void __ieee80211_sta_recalc_aggregates(struct sta_info *sta, u16 active_links)
{
bool first = true;
int link_id;
if (!sta->sta.valid_links || !sta->sta.mlo) {
sta->sta.cur = &sta->sta.deflink.agg;
return;
}
rcu_read_lock();
for (link_id = 0; link_id < ARRAY_SIZE((sta)->link); link_id++) {
struct ieee80211_link_sta *link_sta;
int i;
if (!(active_links & BIT(link_id)))
continue;
link_sta = rcu_dereference(sta->sta.link[link_id]);
if (!link_sta)
continue;
if (first) {
sta->cur = sta->sta.deflink.agg;
first = false;
continue;
}
sta->cur.max_amsdu_len =
min(sta->cur.max_amsdu_len,
link_sta->agg.max_amsdu_len);
sta->cur.max_rc_amsdu_len =
min(sta->cur.max_rc_amsdu_len,
link_sta->agg.max_rc_amsdu_len);
for (i = 0; i < ARRAY_SIZE(sta->cur.max_tid_amsdu_len); i++)
sta->cur.max_tid_amsdu_len[i] =
min(sta->cur.max_tid_amsdu_len[i],
link_sta->agg.max_tid_amsdu_len[i]);
}
rcu_read_unlock();
sta->sta.cur = &sta->cur;
}
void ieee80211_sta_recalc_aggregates(struct ieee80211_sta *pubsta)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
__ieee80211_sta_recalc_aggregates(sta, sta->sdata->vif.active_links);
}
EXPORT_SYMBOL(ieee80211_sta_recalc_aggregates);
void ieee80211_sta_update_pending_airtime(struct ieee80211_local *local,
struct sta_info *sta, u8 ac,
u16 tx_airtime, bool tx_completed)
{
int tx_pending;
if (!wiphy_ext_feature_isset(local->hw.wiphy, NL80211_EXT_FEATURE_AQL))
return;
if (!tx_completed) {
if (sta)
atomic_add(tx_airtime,
&sta->airtime[ac].aql_tx_pending);
atomic_add(tx_airtime, &local->aql_total_pending_airtime);
atomic_add(tx_airtime, &local->aql_ac_pending_airtime[ac]);
return;
}
if (sta) {
tx_pending = atomic_sub_return(tx_airtime,
&sta->airtime[ac].aql_tx_pending);
if (tx_pending < 0)
atomic_cmpxchg(&sta->airtime[ac].aql_tx_pending,
tx_pending, 0);
}
atomic_sub(tx_airtime, &local->aql_total_pending_airtime);
tx_pending = atomic_sub_return(tx_airtime,
&local->aql_ac_pending_airtime[ac]);
if (WARN_ONCE(tx_pending < 0,
"Device %s AC %d pending airtime underflow: %u, %u",
wiphy_name(local->hw.wiphy), ac, tx_pending,
tx_airtime)) {
atomic_cmpxchg(&local->aql_ac_pending_airtime[ac],
tx_pending, 0);
atomic_sub(tx_pending, &local->aql_total_pending_airtime);
}
}
static struct ieee80211_sta_rx_stats *
sta_get_last_rx_stats(struct sta_info *sta)
{
struct ieee80211_sta_rx_stats *stats = &sta->deflink.rx_stats;
int cpu;
if (!sta->deflink.pcpu_rx_stats)
return stats;
for_each_possible_cpu(cpu) {
struct ieee80211_sta_rx_stats *cpustats;
cpustats = per_cpu_ptr(sta->deflink.pcpu_rx_stats, cpu);
if (time_after(cpustats->last_rx, stats->last_rx))
stats = cpustats;
}
return stats;
}
static void sta_stats_decode_rate(struct ieee80211_local *local, u32 rate,
struct rate_info *rinfo)
{
rinfo->bw = STA_STATS_GET(BW, rate);
switch (STA_STATS_GET(TYPE, rate)) {
case STA_STATS_RATE_TYPE_VHT:
rinfo->flags = RATE_INFO_FLAGS_VHT_MCS;
rinfo->mcs = STA_STATS_GET(VHT_MCS, rate);
rinfo->nss = STA_STATS_GET(VHT_NSS, rate);
if (STA_STATS_GET(SGI, rate))
rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
break;
case STA_STATS_RATE_TYPE_HT:
rinfo->flags = RATE_INFO_FLAGS_MCS;
rinfo->mcs = STA_STATS_GET(HT_MCS, rate);
if (STA_STATS_GET(SGI, rate))
rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
break;
case STA_STATS_RATE_TYPE_LEGACY: {
struct ieee80211_supported_band *sband;
u16 brate;
unsigned int shift;
int band = STA_STATS_GET(LEGACY_BAND, rate);
int rate_idx = STA_STATS_GET(LEGACY_IDX, rate);
sband = local->hw.wiphy->bands[band];
if (WARN_ON_ONCE(!sband->bitrates))
break;
brate = sband->bitrates[rate_idx].bitrate;
if (rinfo->bw == RATE_INFO_BW_5)
shift = 2;
else if (rinfo->bw == RATE_INFO_BW_10)
shift = 1;
else
shift = 0;
rinfo->legacy = DIV_ROUND_UP(brate, 1 << shift);
break;
}
case STA_STATS_RATE_TYPE_HE:
rinfo->flags = RATE_INFO_FLAGS_HE_MCS;
rinfo->mcs = STA_STATS_GET(HE_MCS, rate);
rinfo->nss = STA_STATS_GET(HE_NSS, rate);
rinfo->he_gi = STA_STATS_GET(HE_GI, rate);
rinfo->he_ru_alloc = STA_STATS_GET(HE_RU, rate);
rinfo->he_dcm = STA_STATS_GET(HE_DCM, rate);
break;
case STA_STATS_RATE_TYPE_EHT:
rinfo->flags = RATE_INFO_FLAGS_EHT_MCS;
rinfo->mcs = STA_STATS_GET(EHT_MCS, rate);
rinfo->nss = STA_STATS_GET(EHT_NSS, rate);
rinfo->eht_gi = STA_STATS_GET(EHT_GI, rate);
rinfo->eht_ru_alloc = STA_STATS_GET(EHT_RU, rate);
break;
}
}
static int sta_set_rate_info_rx(struct sta_info *sta, struct rate_info *rinfo)
{
u32 rate = READ_ONCE(sta_get_last_rx_stats(sta)->last_rate);
if (rate == STA_STATS_RATE_INVALID)
return -EINVAL;
sta_stats_decode_rate(sta->local, rate, rinfo);
return 0;
}
static inline u64 sta_get_tidstats_msdu(struct ieee80211_sta_rx_stats *rxstats,
int tid)
{
unsigned int start;
u64 value;
do {
start = u64_stats_fetch_begin(&rxstats->syncp);
value = rxstats->msdu[tid];
} while (u64_stats_fetch_retry(&rxstats->syncp, start));
return value;
}
static void sta_set_tidstats(struct sta_info *sta,
struct cfg80211_tid_stats *tidstats,
int tid)
{
struct ieee80211_local *local = sta->local;
int cpu;
if (!(tidstats->filled & BIT(NL80211_TID_STATS_RX_MSDU))) {
tidstats->rx_msdu += sta_get_tidstats_msdu(&sta->deflink.rx_stats,
tid);
if (sta->deflink.pcpu_rx_stats) {
for_each_possible_cpu(cpu) {
struct ieee80211_sta_rx_stats *cpurxs;
cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats,
cpu);
tidstats->rx_msdu +=
sta_get_tidstats_msdu(cpurxs, tid);
}
}
tidstats->filled |= BIT(NL80211_TID_STATS_RX_MSDU);
}
if (!(tidstats->filled & BIT(NL80211_TID_STATS_TX_MSDU))) {
tidstats->filled |= BIT(NL80211_TID_STATS_TX_MSDU);
tidstats->tx_msdu = sta->deflink.tx_stats.msdu[tid];
}
if (!(tidstats->filled & BIT(NL80211_TID_STATS_TX_MSDU_RETRIES)) &&
ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
tidstats->filled |= BIT(NL80211_TID_STATS_TX_MSDU_RETRIES);
tidstats->tx_msdu_retries = sta->deflink.status_stats.msdu_retries[tid];
}
if (!(tidstats->filled & BIT(NL80211_TID_STATS_TX_MSDU_FAILED)) &&
ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
tidstats->filled |= BIT(NL80211_TID_STATS_TX_MSDU_FAILED);
tidstats->tx_msdu_failed = sta->deflink.status_stats.msdu_failed[tid];
}
if (tid < IEEE80211_NUM_TIDS) {
spin_lock_bh(&local->fq.lock);
rcu_read_lock();
tidstats->filled |= BIT(NL80211_TID_STATS_TXQ_STATS);
ieee80211_fill_txq_stats(&tidstats->txq_stats,
to_txq_info(sta->sta.txq[tid]));
rcu_read_unlock();
spin_unlock_bh(&local->fq.lock);
}
}
static inline u64 sta_get_stats_bytes(struct ieee80211_sta_rx_stats *rxstats)
{
unsigned int start;
u64 value;
do {
start = u64_stats_fetch_begin(&rxstats->syncp);
value = rxstats->bytes;
} while (u64_stats_fetch_retry(&rxstats->syncp, start));
return value;
}
void sta_set_sinfo(struct sta_info *sta, struct station_info *sinfo,
bool tidstats)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
u32 thr = 0;
int i, ac, cpu;
struct ieee80211_sta_rx_stats *last_rxstats;
last_rxstats = sta_get_last_rx_stats(sta);
sinfo->generation = sdata->local->sta_generation;
/* do before driver, so beacon filtering drivers have a
* chance to e.g. just add the number of filtered beacons
* (or just modify the value entirely, of course)
*/
if (sdata->vif.type == NL80211_IFTYPE_STATION)
sinfo->rx_beacon = sdata->deflink.u.mgd.count_beacon_signal;
drv_sta_statistics(local, sdata, &sta->sta, sinfo);
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME) |
BIT_ULL(NL80211_STA_INFO_STA_FLAGS) |
BIT_ULL(NL80211_STA_INFO_BSS_PARAM) |
BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) |
BIT_ULL(NL80211_STA_INFO_ASSOC_AT_BOOTTIME) |
BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC);
if (sdata->vif.type == NL80211_IFTYPE_STATION) {
sinfo->beacon_loss_count =
sdata->deflink.u.mgd.beacon_loss_count;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_LOSS);
}
sinfo->connected_time = ktime_get_seconds() - sta->last_connected;
sinfo->assoc_at = sta->assoc_at;
sinfo->inactive_time =
jiffies_to_msecs(jiffies - ieee80211_sta_last_active(sta));
if (!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_TX_BYTES64) |
BIT_ULL(NL80211_STA_INFO_TX_BYTES)))) {
sinfo->tx_bytes = 0;
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
sinfo->tx_bytes += sta->deflink.tx_stats.bytes[ac];
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_PACKETS))) {
sinfo->tx_packets = 0;
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
sinfo->tx_packets += sta->deflink.tx_stats.packets[ac];
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
}
if (!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_RX_BYTES64) |
BIT_ULL(NL80211_STA_INFO_RX_BYTES)))) {
sinfo->rx_bytes += sta_get_stats_bytes(&sta->deflink.rx_stats);
if (sta->deflink.pcpu_rx_stats) {
for_each_possible_cpu(cpu) {
struct ieee80211_sta_rx_stats *cpurxs;
cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats,
cpu);
sinfo->rx_bytes += sta_get_stats_bytes(cpurxs);
}
}
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES64);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_PACKETS))) {
sinfo->rx_packets = sta->deflink.rx_stats.packets;
if (sta->deflink.pcpu_rx_stats) {
for_each_possible_cpu(cpu) {
struct ieee80211_sta_rx_stats *cpurxs;
cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats,
cpu);
sinfo->rx_packets += cpurxs->packets;
}
}
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_RETRIES))) {
sinfo->tx_retries = sta->deflink.status_stats.retry_count;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_FAILED))) {
sinfo->tx_failed = sta->deflink.status_stats.retry_failed;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_DURATION))) {
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
sinfo->rx_duration += sta->airtime[ac].rx_airtime;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_DURATION))) {
for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
sinfo->tx_duration += sta->airtime[ac].tx_airtime;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_AIRTIME_WEIGHT))) {
sinfo->airtime_weight = sta->airtime_weight;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_AIRTIME_WEIGHT);
}
sinfo->rx_dropped_misc = sta->deflink.rx_stats.dropped;
if (sta->deflink.pcpu_rx_stats) {
for_each_possible_cpu(cpu) {
struct ieee80211_sta_rx_stats *cpurxs;
cpurxs = per_cpu_ptr(sta->deflink.pcpu_rx_stats, cpu);
sinfo->rx_dropped_misc += cpurxs->dropped;
}
}
if (sdata->vif.type == NL80211_IFTYPE_STATION &&
!(sdata->vif.driver_flags & IEEE80211_VIF_BEACON_FILTER)) {
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_RX) |
BIT_ULL(NL80211_STA_INFO_BEACON_SIGNAL_AVG);
sinfo->rx_beacon_signal_avg = ieee80211_ave_rssi(&sdata->vif);
}
if (ieee80211_hw_check(&sta->local->hw, SIGNAL_DBM) ||
ieee80211_hw_check(&sta->local->hw, SIGNAL_UNSPEC)) {
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_SIGNAL))) {
sinfo->signal = (s8)last_rxstats->last_signal;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
}
if (!sta->deflink.pcpu_rx_stats &&
!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG))) {
sinfo->signal_avg =
-ewma_signal_read(&sta->deflink.rx_stats_avg.signal);
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
}
}
/* for the average - if pcpu_rx_stats isn't set - rxstats must point to
* the sta->rx_stats struct, so the check here is fine with and without
* pcpu statistics
*/
if (last_rxstats->chains &&
!(sinfo->filled & (BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL) |
BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG)))) {
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL);
if (!sta->deflink.pcpu_rx_stats)
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG);
sinfo->chains = last_rxstats->chains;
for (i = 0; i < ARRAY_SIZE(sinfo->chain_signal); i++) {
sinfo->chain_signal[i] =
last_rxstats->chain_signal_last[i];
sinfo->chain_signal_avg[i] =
-ewma_signal_read(&sta->deflink.rx_stats_avg.chain_signal[i]);
}
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_TX_BITRATE)) &&
!sta->sta.valid_links &&
ieee80211_rate_valid(&sta->deflink.tx_stats.last_rate)) {
sta_set_rate_info_tx(sta, &sta->deflink.tx_stats.last_rate,
&sinfo->txrate);
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_RX_BITRATE)) &&
!sta->sta.valid_links) {
if (sta_set_rate_info_rx(sta, &sinfo->rxrate) == 0)
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE);
}
if (tidstats && !cfg80211_sinfo_alloc_tid_stats(sinfo, GFP_KERNEL)) {
for (i = 0; i < IEEE80211_NUM_TIDS + 1; i++)
sta_set_tidstats(sta, &sinfo->pertid[i], i);
}
if (ieee80211_vif_is_mesh(&sdata->vif)) {
#ifdef CONFIG_MAC80211_MESH
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_LLID) |
BIT_ULL(NL80211_STA_INFO_PLID) |
BIT_ULL(NL80211_STA_INFO_PLINK_STATE) |
BIT_ULL(NL80211_STA_INFO_LOCAL_PM) |
BIT_ULL(NL80211_STA_INFO_PEER_PM) |
BIT_ULL(NL80211_STA_INFO_NONPEER_PM) |
BIT_ULL(NL80211_STA_INFO_CONNECTED_TO_GATE) |
BIT_ULL(NL80211_STA_INFO_CONNECTED_TO_AS);
sinfo->llid = sta->mesh->llid;
sinfo->plid = sta->mesh->plid;
sinfo->plink_state = sta->mesh->plink_state;
if (test_sta_flag(sta, WLAN_STA_TOFFSET_KNOWN)) {
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_T_OFFSET);
sinfo->t_offset = sta->mesh->t_offset;
}
sinfo->local_pm = sta->mesh->local_pm;
sinfo->peer_pm = sta->mesh->peer_pm;
sinfo->nonpeer_pm = sta->mesh->nonpeer_pm;
sinfo->connected_to_gate = sta->mesh->connected_to_gate;
sinfo->connected_to_as = sta->mesh->connected_to_as;
#endif
}
sinfo->bss_param.flags = 0;
if (sdata->vif.bss_conf.use_cts_prot)
sinfo->bss_param.flags |= BSS_PARAM_FLAGS_CTS_PROT;
if (sdata->vif.bss_conf.use_short_preamble)
sinfo->bss_param.flags |= BSS_PARAM_FLAGS_SHORT_PREAMBLE;
if (sdata->vif.bss_conf.use_short_slot)
sinfo->bss_param.flags |= BSS_PARAM_FLAGS_SHORT_SLOT_TIME;
sinfo->bss_param.dtim_period = sdata->vif.bss_conf.dtim_period;
sinfo->bss_param.beacon_interval = sdata->vif.bss_conf.beacon_int;
sinfo->sta_flags.set = 0;
sinfo->sta_flags.mask = BIT(NL80211_STA_FLAG_AUTHORIZED) |
BIT(NL80211_STA_FLAG_SHORT_PREAMBLE) |
BIT(NL80211_STA_FLAG_WME) |
BIT(NL80211_STA_FLAG_MFP) |
BIT(NL80211_STA_FLAG_AUTHENTICATED) |
BIT(NL80211_STA_FLAG_ASSOCIATED) |
BIT(NL80211_STA_FLAG_TDLS_PEER);
if (test_sta_flag(sta, WLAN_STA_AUTHORIZED))
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_AUTHORIZED);
if (test_sta_flag(sta, WLAN_STA_SHORT_PREAMBLE))
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_SHORT_PREAMBLE);
if (sta->sta.wme)
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_WME);
if (test_sta_flag(sta, WLAN_STA_MFP))
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_MFP);
if (test_sta_flag(sta, WLAN_STA_AUTH))
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_AUTHENTICATED);
if (test_sta_flag(sta, WLAN_STA_ASSOC))
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_ASSOCIATED);
if (test_sta_flag(sta, WLAN_STA_TDLS_PEER))
sinfo->sta_flags.set |= BIT(NL80211_STA_FLAG_TDLS_PEER);
thr = sta_get_expected_throughput(sta);
if (thr != 0) {
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_EXPECTED_THROUGHPUT);
sinfo->expected_throughput = thr;
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL)) &&
sta->deflink.status_stats.ack_signal_filled) {
sinfo->ack_signal = sta->deflink.status_stats.last_ack_signal;
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL);
}
if (!(sinfo->filled & BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG)) &&
sta->deflink.status_stats.ack_signal_filled) {
sinfo->avg_ack_signal =
-(s8)ewma_avg_signal_read(
&sta->deflink.status_stats.avg_ack_signal);
sinfo->filled |=
BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG);
}
if (ieee80211_vif_is_mesh(&sdata->vif)) {
sinfo->filled |= BIT_ULL(NL80211_STA_INFO_AIRTIME_LINK_METRIC);
sinfo->airtime_link_metric =
airtime_link_metric_get(local, sta);
}
}
u32 sta_get_expected_throughput(struct sta_info *sta)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct ieee80211_local *local = sdata->local;
struct rate_control_ref *ref = NULL;
u32 thr = 0;
if (test_sta_flag(sta, WLAN_STA_RATE_CONTROL))
ref = local->rate_ctrl;
/* check if the driver has a SW RC implementation */
if (ref && ref->ops->get_expected_throughput)
thr = ref->ops->get_expected_throughput(sta->rate_ctrl_priv);
else
thr = drv_get_expected_throughput(local, sta);
return thr;
}
unsigned long ieee80211_sta_last_active(struct sta_info *sta)
{
struct ieee80211_sta_rx_stats *stats = sta_get_last_rx_stats(sta);
if (!sta->deflink.status_stats.last_ack ||
time_after(stats->last_rx, sta->deflink.status_stats.last_ack))
return stats->last_rx;
return sta->deflink.status_stats.last_ack;
}
static void sta_update_codel_params(struct sta_info *sta, u32 thr)
{
if (thr && thr < STA_SLOW_THRESHOLD * sta->local->num_sta) {
sta->cparams.target = MS2TIME(50);
sta->cparams.interval = MS2TIME(300);
sta->cparams.ecn = false;
} else {
sta->cparams.target = MS2TIME(20);
sta->cparams.interval = MS2TIME(100);
sta->cparams.ecn = true;
}
}
void ieee80211_sta_set_expected_throughput(struct ieee80211_sta *pubsta,
u32 thr)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
sta_update_codel_params(sta, thr);
}
int ieee80211_sta_allocate_link(struct sta_info *sta, unsigned int link_id)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct sta_link_alloc *alloc;
int ret;
lockdep_assert_wiphy(sdata->local->hw.wiphy);
WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED));
/* must represent an MLD from the start */
if (WARN_ON(!sta->sta.valid_links))
return -EINVAL;
if (WARN_ON(sta->sta.valid_links & BIT(link_id) ||
sta->link[link_id]))
return -EBUSY;
alloc = kzalloc(sizeof(*alloc), GFP_KERNEL);
if (!alloc)
return -ENOMEM;
ret = sta_info_alloc_link(sdata->local, &alloc->info, GFP_KERNEL);
if (ret) {
kfree(alloc);
return ret;
}
sta_info_add_link(sta, link_id, &alloc->info, &alloc->sta);
ieee80211_link_sta_debugfs_add(&alloc->info);
return 0;
}
void ieee80211_sta_free_link(struct sta_info *sta, unsigned int link_id)
{
lockdep_assert_wiphy(sta->sdata->local->hw.wiphy);
WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED));
sta_remove_link(sta, link_id, false);
}
int ieee80211_sta_activate_link(struct sta_info *sta, unsigned int link_id)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
struct link_sta_info *link_sta;
u16 old_links = sta->sta.valid_links;
u16 new_links = old_links | BIT(link_id);
int ret;
link_sta = rcu_dereference_protected(sta->link[link_id],
lockdep_is_held(&sdata->local->hw.wiphy->mtx));
if (WARN_ON(old_links == new_links || !link_sta))
return -EINVAL;
rcu_read_lock();
if (link_sta_info_hash_lookup(sdata->local, link_sta->addr)) {
rcu_read_unlock();
return -EALREADY;
}
/* we only modify under the mutex so this is fine */
rcu_read_unlock();
sta->sta.valid_links = new_links;
if (WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED)))
goto hash;
ieee80211_recalc_min_chandef(sdata, link_id);
/* Ensure the values are updated for the driver,
* redone by sta_remove_link on failure.
*/
ieee80211_sta_recalc_aggregates(&sta->sta);
ret = drv_change_sta_links(sdata->local, sdata, &sta->sta,
old_links, new_links);
if (ret) {
sta->sta.valid_links = old_links;
sta_remove_link(sta, link_id, false);
return ret;
}
hash:
ret = link_sta_info_hash_add(sdata->local, link_sta);
WARN_ON(ret);
return 0;
}
void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id)
{
struct ieee80211_sub_if_data *sdata = sta->sdata;
u16 old_links = sta->sta.valid_links;
lockdep_assert_wiphy(sdata->local->hw.wiphy);
sta->sta.valid_links &= ~BIT(link_id);
if (!WARN_ON(!test_sta_flag(sta, WLAN_STA_INSERTED)))
drv_change_sta_links(sdata->local, sdata, &sta->sta,
old_links, sta->sta.valid_links);
sta_remove_link(sta, link_id, true);
}
void ieee80211_sta_set_max_amsdu_subframes(struct sta_info *sta,
const u8 *ext_capab,
unsigned int ext_capab_len)
{
u8 val;
sta->sta.max_amsdu_subframes = 0;
if (ext_capab_len < 8)
return;
/* The sender might not have sent the last bit, consider it to be 0 */
val = u8_get_bits(ext_capab[7], WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB);
/* we did get all the bits, take the MSB as well */
if (ext_capab_len >= 9)
val |= u8_get_bits(ext_capab[8],
WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB) << 1;
if (val)
sta->sta.max_amsdu_subframes = 4 << (4 - val);
}
#ifdef CONFIG_LOCKDEP
bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta)
{
struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
return lockdep_is_held(&sta->local->hw.wiphy->mtx);
}
EXPORT_SYMBOL(lockdep_sta_mutex_held);
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/kernel/printk.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*
* Modified to make sys_syslog() more flexible: added commands to
* return the last 4k of kernel messages, regardless of whether
* they've been read or not. Added option to suppress kernel printk's
* to the console. Added hook for sending the console messages
* elsewhere, in preparation for a serial line console (someday).
* Ted Ts'o, 2/11/93.
* Modified for sysctl support, 1/8/97, Chris Horn.
* Fixed SMP synchronization, 08/08/99, Manfred Spraul
* manfred@colorfullife.com
* Rewrote bits to get rid of console_lock
* 01Mar01 Andrew Morton
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/console.h>
#include <linux/init.h>
#include <linux/jiffies.h>
#include <linux/nmi.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/security.h>
#include <linux/memblock.h>
#include <linux/syscalls.h>
#include <linux/vmcore_info.h>
#include <linux/ratelimit.h>
#include <linux/kmsg_dump.h>
#include <linux/syslog.h>
#include <linux/cpu.h>
#include <linux/rculist.h>
#include <linux/poll.h>
#include <linux/irq_work.h>
#include <linux/ctype.h>
#include <linux/uio.h>
#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/task_stack.h>
#include <linux/uaccess.h>
#include <asm/sections.h>
#include <trace/events/initcall.h>
#define CREATE_TRACE_POINTS
#include <trace/events/printk.h>
#include "printk_ringbuffer.h"
#include "console_cmdline.h"
#include "braille.h"
#include "internal.h"
int console_printk[4] = {
CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */
MESSAGE_LOGLEVEL_DEFAULT, /* default_message_loglevel */
CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */
CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */
};
EXPORT_SYMBOL_GPL(console_printk);
atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0);
EXPORT_SYMBOL(ignore_console_lock_warning);
EXPORT_TRACEPOINT_SYMBOL_GPL(console);
/*
* Low level drivers may need that to know if they can schedule in
* their unblank() callback or not. So let's export it.
*/
int oops_in_progress;
EXPORT_SYMBOL(oops_in_progress);
/*
* console_mutex protects console_list updates and console->flags updates.
* The flags are synchronized only for consoles that are registered, i.e.
* accessible via the console list.
*/
static DEFINE_MUTEX(console_mutex);
/*
* console_sem protects updates to console->seq
* and also provides serialization for console printing.
*/
static DEFINE_SEMAPHORE(console_sem, 1);
HLIST_HEAD(console_list);
EXPORT_SYMBOL_GPL(console_list);
DEFINE_STATIC_SRCU(console_srcu);
/*
* System may need to suppress printk message under certain
* circumstances, like after kernel panic happens.
*/
int __read_mostly suppress_printk;
#ifdef CONFIG_LOCKDEP
static struct lockdep_map console_lock_dep_map = {
.name = "console_lock"
};
void lockdep_assert_console_list_lock_held(void)
{
lockdep_assert_held(&console_mutex);
}
EXPORT_SYMBOL(lockdep_assert_console_list_lock_held);
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
bool console_srcu_read_lock_is_held(void)
{
return srcu_read_lock_held(&console_srcu);
}
EXPORT_SYMBOL(console_srcu_read_lock_is_held);
#endif
enum devkmsg_log_bits {
__DEVKMSG_LOG_BIT_ON = 0,
__DEVKMSG_LOG_BIT_OFF,
__DEVKMSG_LOG_BIT_LOCK,
};
enum devkmsg_log_masks {
DEVKMSG_LOG_MASK_ON = BIT(__DEVKMSG_LOG_BIT_ON),
DEVKMSG_LOG_MASK_OFF = BIT(__DEVKMSG_LOG_BIT_OFF),
DEVKMSG_LOG_MASK_LOCK = BIT(__DEVKMSG_LOG_BIT_LOCK),
};
/* Keep both the 'on' and 'off' bits clear, i.e. ratelimit by default: */
#define DEVKMSG_LOG_MASK_DEFAULT 0
static unsigned int __read_mostly devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
static int __control_devkmsg(char *str)
{
size_t len;
if (!str)
return -EINVAL;
len = str_has_prefix(str, "on");
if (len) {
devkmsg_log = DEVKMSG_LOG_MASK_ON;
return len;
}
len = str_has_prefix(str, "off");
if (len) {
devkmsg_log = DEVKMSG_LOG_MASK_OFF;
return len;
}
len = str_has_prefix(str, "ratelimit");
if (len) {
devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
return len;
}
return -EINVAL;
}
static int __init control_devkmsg(char *str)
{
if (__control_devkmsg(str) < 0) {
pr_warn("printk.devkmsg: bad option string '%s'\n", str);
return 1;
}
/*
* Set sysctl string accordingly:
*/
if (devkmsg_log == DEVKMSG_LOG_MASK_ON)
strcpy(devkmsg_log_str, "on");
else if (devkmsg_log == DEVKMSG_LOG_MASK_OFF)
strcpy(devkmsg_log_str, "off");
/* else "ratelimit" which is set by default. */
/*
* Sysctl cannot change it anymore. The kernel command line setting of
* this parameter is to force the setting to be permanent throughout the
* runtime of the system. This is a precation measure against userspace
* trying to be a smarta** and attempting to change it up on us.
*/
devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;
return 1;
}
__setup("printk.devkmsg=", control_devkmsg);
char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
char old_str[DEVKMSG_STR_MAX_SIZE];
unsigned int old;
int err;
if (write) {
if (devkmsg_log & DEVKMSG_LOG_MASK_LOCK)
return -EINVAL;
old = devkmsg_log;
strncpy(old_str, devkmsg_log_str, DEVKMSG_STR_MAX_SIZE);
}
err = proc_dostring(table, write, buffer, lenp, ppos);
if (err)
return err;
if (write) {
err = __control_devkmsg(devkmsg_log_str);
/*
* Do not accept an unknown string OR a known string with
* trailing crap...
*/
if (err < 0 || (err + 1 != *lenp)) {
/* ... and restore old setting. */
devkmsg_log = old;
strncpy(devkmsg_log_str, old_str, DEVKMSG_STR_MAX_SIZE);
return -EINVAL;
}
}
return 0;
}
#endif /* CONFIG_PRINTK && CONFIG_SYSCTL */
/**
* console_list_lock - Lock the console list
*
* For console list or console->flags updates
*/
void console_list_lock(void)
{
/*
* In unregister_console() and console_force_preferred_locked(),
* synchronize_srcu() is called with the console_list_lock held.
* Therefore it is not allowed that the console_list_lock is taken
* with the srcu_lock held.
*
* Detecting if this context is really in the read-side critical
* section is only possible if the appropriate debug options are
* enabled.
*/
WARN_ON_ONCE(debug_lockdep_rcu_enabled() &&
srcu_read_lock_held(&console_srcu));
mutex_lock(&console_mutex);
}
EXPORT_SYMBOL(console_list_lock);
/**
* console_list_unlock - Unlock the console list
*
* Counterpart to console_list_lock()
*/
void console_list_unlock(void)
{
mutex_unlock(&console_mutex);
}
EXPORT_SYMBOL(console_list_unlock);
/**
* console_srcu_read_lock - Register a new reader for the
* SRCU-protected console list
*
* Use for_each_console_srcu() to iterate the console list
*
* Context: Any context.
* Return: A cookie to pass to console_srcu_read_unlock().
*/
int console_srcu_read_lock(void)
{
return srcu_read_lock_nmisafe(&console_srcu);
}
EXPORT_SYMBOL(console_srcu_read_lock);
/**
* console_srcu_read_unlock - Unregister an old reader from
* the SRCU-protected console list
* @cookie: cookie returned from console_srcu_read_lock()
*
* Counterpart to console_srcu_read_lock()
*/
void console_srcu_read_unlock(int cookie)
{
srcu_read_unlock_nmisafe(&console_srcu, cookie);
}
EXPORT_SYMBOL(console_srcu_read_unlock);
/*
* Helper macros to handle lockdep when locking/unlocking console_sem. We use
* macros instead of functions so that _RET_IP_ contains useful information.
*/
#define down_console_sem() do { \
down(&console_sem);\
mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);\
} while (0)
static int __down_trylock_console_sem(unsigned long ip)
{
int lock_failed;
unsigned long flags;
/*
* Here and in __up_console_sem() we need to be in safe mode,
* because spindump/WARN/etc from under console ->lock will
* deadlock in printk()->down_trylock_console_sem() otherwise.
*/
printk_safe_enter_irqsave(flags);
lock_failed = down_trylock(&console_sem);
printk_safe_exit_irqrestore(flags); if (lock_failed)
return 1;
mutex_acquire(&console_lock_dep_map, 0, 1, ip); return 0;
}
#define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)
static void __up_console_sem(unsigned long ip)
{
unsigned long flags;
mutex_release(&console_lock_dep_map, ip);
printk_safe_enter_irqsave(flags);
up(&console_sem);
printk_safe_exit_irqrestore(flags);
}
#define up_console_sem() __up_console_sem(_RET_IP_)
static bool panic_in_progress(void)
{
return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
}
/* Return true if a panic is in progress on the current CPU. */
bool this_cpu_in_panic(void)
{
/*
* We can use raw_smp_processor_id() here because it is impossible for
* the task to be migrated to the panic_cpu, or away from it. If
* panic_cpu has already been set, and we're not currently executing on
* that CPU, then we never will be.
*/
return unlikely(atomic_read(&panic_cpu) == raw_smp_processor_id());
}
/*
* Return true if a panic is in progress on a remote CPU.
*
* On true, the local CPU should immediately release any printing resources
* that may be needed by the panic CPU.
*/
bool other_cpu_in_panic(void)
{
return (panic_in_progress() && !this_cpu_in_panic());
}
/*
* This is used for debugging the mess that is the VT code by
* keeping track if we have the console semaphore held. It's
* definitely not the perfect debug tool (we don't know if _WE_
* hold it and are racing, but it helps tracking those weird code
* paths in the console code where we end up in places I want
* locked without the console semaphore held).
*/
static int console_locked;
/*
* Array of consoles built from command line options (console=)
*/
#define MAX_CMDLINECONSOLES 8
static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
static int preferred_console = -1;
int console_set_on_cmdline;
EXPORT_SYMBOL(console_set_on_cmdline);
/* Flag: console code may call schedule() */
static int console_may_schedule;
enum con_msg_format_flags {
MSG_FORMAT_DEFAULT = 0,
MSG_FORMAT_SYSLOG = (1 << 0),
};
static int console_msg_format = MSG_FORMAT_DEFAULT;
/*
* The printk log buffer consists of a sequenced collection of records, each
* containing variable length message text. Every record also contains its
* own meta-data (@info).
*
* Every record meta-data carries the timestamp in microseconds, as well as
* the standard userspace syslog level and syslog facility. The usual kernel
* messages use LOG_KERN; userspace-injected messages always carry a matching
* syslog facility, by default LOG_USER. The origin of every message can be
* reliably determined that way.
*
* The human readable log message of a record is available in @text, the
* length of the message text in @text_len. The stored message is not
* terminated.
*
* Optionally, a record can carry a dictionary of properties (key/value
* pairs), to provide userspace with a machine-readable message context.
*
* Examples for well-defined, commonly used property names are:
* DEVICE=b12:8 device identifier
* b12:8 block dev_t
* c127:3 char dev_t
* n8 netdev ifindex
* +sound:card0 subsystem:devname
* SUBSYSTEM=pci driver-core subsystem name
*
* Valid characters in property names are [a-zA-Z0-9.-_]. Property names
* and values are terminated by a '\0' character.
*
* Example of record values:
* record.text_buf = "it's a line" (unterminated)
* record.info.seq = 56
* record.info.ts_nsec = 36863
* record.info.text_len = 11
* record.info.facility = 0 (LOG_KERN)
* record.info.flags = 0
* record.info.level = 3 (LOG_ERR)
* record.info.caller_id = 299 (task 299)
* record.info.dev_info.subsystem = "pci" (terminated)
* record.info.dev_info.device = "+pci:0000:00:01.0" (terminated)
*
* The 'struct printk_info' buffer must never be directly exported to
* userspace, it is a kernel-private implementation detail that might
* need to be changed in the future, when the requirements change.
*
* /dev/kmsg exports the structured data in the following line format:
* "<level>,<sequnum>,<timestamp>,<contflag>[,additional_values, ... ];<message text>\n"
*
* Users of the export format should ignore possible additional values
* separated by ',', and find the message after the ';' character.
*
* The optional key/value pairs are attached as continuation lines starting
* with a space character and terminated by a newline. All possible
* non-prinatable characters are escaped in the "\xff" notation.
*/
/* syslog_lock protects syslog_* variables and write access to clear_seq. */
static DEFINE_MUTEX(syslog_lock);
#ifdef CONFIG_PRINTK
DECLARE_WAIT_QUEUE_HEAD(log_wait);
/* All 3 protected by @syslog_lock. */
/* the next printk record to read by syslog(READ) or /proc/kmsg */
static u64 syslog_seq;
static size_t syslog_partial;
static bool syslog_time;
struct latched_seq {
seqcount_latch_t latch;
u64 val[2];
};
/*
* The next printk record to read after the last 'clear' command. There are
* two copies (updated with seqcount_latch) so that reads can locklessly
* access a valid value. Writers are synchronized by @syslog_lock.
*/
static struct latched_seq clear_seq = {
.latch = SEQCNT_LATCH_ZERO(clear_seq.latch),
.val[0] = 0,
.val[1] = 0,
};
#define LOG_LEVEL(v) ((v) & 0x07)
#define LOG_FACILITY(v) ((v) >> 3 & 0xff)
/* record buffer */
#define LOG_ALIGN __alignof__(unsigned long)
#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
#define LOG_BUF_LEN_MAX (u32)(1 << 31)
static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
static char *log_buf = __log_buf;
static u32 log_buf_len = __LOG_BUF_LEN;
/*
* Define the average message size. This only affects the number of
* descriptors that will be available. Underestimating is better than
* overestimating (too many available descriptors is better than not enough).
*/
#define PRB_AVGBITS 5 /* 32 character average length */
#if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
#error CONFIG_LOG_BUF_SHIFT value too small.
#endif
_DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
PRB_AVGBITS, &__log_buf[0]);
static struct printk_ringbuffer printk_rb_dynamic;
struct printk_ringbuffer *prb = &printk_rb_static;
/*
* We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
* per_cpu_areas are initialised. This variable is set to true when
* it's safe to access per-CPU data.
*/
static bool __printk_percpu_data_ready __ro_after_init;
bool printk_percpu_data_ready(void)
{
return __printk_percpu_data_ready;
}
/* Must be called under syslog_lock. */
static void latched_seq_write(struct latched_seq *ls, u64 val)
{
raw_write_seqcount_latch(&ls->latch);
ls->val[0] = val;
raw_write_seqcount_latch(&ls->latch);
ls->val[1] = val;
}
/* Can be called from any context. */
static u64 latched_seq_read_nolock(struct latched_seq *ls)
{
unsigned int seq;
unsigned int idx;
u64 val;
do {
seq = raw_read_seqcount_latch(&ls->latch);
idx = seq & 0x1;
val = ls->val[idx];
} while (raw_read_seqcount_latch_retry(&ls->latch, seq));
return val;
}
/* Return log buffer address */
char *log_buf_addr_get(void)
{
return log_buf;
}
/* Return log buffer size */
u32 log_buf_len_get(void)
{
return log_buf_len;
}
/*
* Define how much of the log buffer we could take at maximum. The value
* must be greater than two. Note that only half of the buffer is available
* when the index points to the middle.
*/
#define MAX_LOG_TAKE_PART 4
static const char trunc_msg[] = "<truncated>";
static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
{
/*
* The message should not take the whole buffer. Otherwise, it might
* get removed too soon.
*/
u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
if (*text_len > max_text_len)
*text_len = max_text_len;
/* enable the warning message (if there is room) */
*trunc_msg_len = strlen(trunc_msg);
if (*text_len >= *trunc_msg_len) *text_len -= *trunc_msg_len;
else
*trunc_msg_len = 0;
}
int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
static int syslog_action_restricted(int type)
{
if (dmesg_restrict)
return 1;
/*
* Unless restricted, we allow "read all" and "get buffer size"
* for everybody.
*/
return type != SYSLOG_ACTION_READ_ALL &&
type != SYSLOG_ACTION_SIZE_BUFFER;
}
static int check_syslog_permissions(int type, int source)
{
/*
* If this is from /proc/kmsg and we've already opened it, then we've
* already done the capabilities checks at open time.
*/
if (source == SYSLOG_FROM_PROC && type != SYSLOG_ACTION_OPEN)
goto ok;
if (syslog_action_restricted(type)) {
if (capable(CAP_SYSLOG))
goto ok;
return -EPERM;
}
ok:
return security_syslog(type);
}
static void append_char(char **pp, char *e, char c)
{
if (*pp < e)
*(*pp)++ = c;
}
static ssize_t info_print_ext_header(char *buf, size_t size,
struct printk_info *info)
{
u64 ts_usec = info->ts_nsec;
char caller[20];
#ifdef CONFIG_PRINTK_CALLER
u32 id = info->caller_id;
snprintf(caller, sizeof(caller), ",caller=%c%u",
id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
#else
caller[0] = '\0';
#endif
do_div(ts_usec, 1000);
return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",
(info->facility << 3) | info->level, info->seq,
ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
}
static ssize_t msg_add_ext_text(char *buf, size_t size,
const char *text, size_t text_len,
unsigned char endc)
{
char *p = buf, *e = buf + size;
size_t i;
/* escape non-printable characters */
for (i = 0; i < text_len; i++) {
unsigned char c = text[i];
if (c < ' ' || c >= 127 || c == '\\')
p += scnprintf(p, e - p, "\\x%02x", c);
else
append_char(&p, e, c);
}
append_char(&p, e, endc);
return p - buf;
}
static ssize_t msg_add_dict_text(char *buf, size_t size,
const char *key, const char *val)
{
size_t val_len = strlen(val);
ssize_t len;
if (!val_len)
return 0;
len = msg_add_ext_text(buf, size, "", 0, ' '); /* dict prefix */
len += msg_add_ext_text(buf + len, size - len, key, strlen(key), '=');
len += msg_add_ext_text(buf + len, size - len, val, val_len, '\n');
return len;
}
static ssize_t msg_print_ext_body(char *buf, size_t size,
char *text, size_t text_len,
struct dev_printk_info *dev_info)
{
ssize_t len;
len = msg_add_ext_text(buf, size, text, text_len, '\n');
if (!dev_info)
goto out;
len += msg_add_dict_text(buf + len, size - len, "SUBSYSTEM",
dev_info->subsystem);
len += msg_add_dict_text(buf + len, size - len, "DEVICE",
dev_info->device);
out:
return len;
}
/* /dev/kmsg - userspace message inject/listen interface */
struct devkmsg_user {
atomic64_t seq;
struct ratelimit_state rs;
struct mutex lock;
struct printk_buffers pbufs;
};
static __printf(3, 4) __cold
int devkmsg_emit(int facility, int level, const char *fmt, ...)
{
va_list args;
int r;
va_start(args, fmt);
r = vprintk_emit(facility, level, NULL, fmt, args);
va_end(args);
return r;
}
static ssize_t devkmsg_write(struct kiocb *iocb, struct iov_iter *from)
{
char *buf, *line;
int level = default_message_loglevel;
int facility = 1; /* LOG_USER */
struct file *file = iocb->ki_filp;
struct devkmsg_user *user = file->private_data;
size_t len = iov_iter_count(from);
ssize_t ret = len;
if (len > PRINTKRB_RECORD_MAX)
return -EINVAL;
/* Ignore when user logging is disabled. */
if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
return len;
/* Ratelimit when not explicitly enabled. */
if (!(devkmsg_log & DEVKMSG_LOG_MASK_ON)) {
if (!___ratelimit(&user->rs, current->comm))
return ret;
}
buf = kmalloc(len+1, GFP_KERNEL);
if (buf == NULL)
return -ENOMEM;
buf[len] = '\0';
if (!copy_from_iter_full(buf, len, from)) {
kfree(buf);
return -EFAULT;
}
/*
* Extract and skip the syslog prefix <[0-9]*>. Coming from userspace
* the decimal value represents 32bit, the lower 3 bit are the log
* level, the rest are the log facility.
*
* If no prefix or no userspace facility is specified, we
* enforce LOG_USER, to be able to reliably distinguish
* kernel-generated messages from userspace-injected ones.
*/
line = buf;
if (line[0] == '<') {
char *endp = NULL;
unsigned int u;
u = simple_strtoul(line + 1, &endp, 10);
if (endp && endp[0] == '>') {
level = LOG_LEVEL(u);
if (LOG_FACILITY(u) != 0)
facility = LOG_FACILITY(u);
endp++;
line = endp;
}
}
devkmsg_emit(facility, level, "%s", line);
kfree(buf);
return ret;
}
static ssize_t devkmsg_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
struct devkmsg_user *user = file->private_data;
char *outbuf = &user->pbufs.outbuf[0];
struct printk_message pmsg = {
.pbufs = &user->pbufs,
};
ssize_t ret;
ret = mutex_lock_interruptible(&user->lock);
if (ret)
return ret;
if (!printk_get_next_message(&pmsg, atomic64_read(&user->seq), true, false)) {
if (file->f_flags & O_NONBLOCK) {
ret = -EAGAIN;
goto out;
}
/*
* Guarantee this task is visible on the waitqueue before
* checking the wake condition.
*
* The full memory barrier within set_current_state() of
* prepare_to_wait_event() pairs with the full memory barrier
* within wq_has_sleeper().
*
* This pairs with __wake_up_klogd:A.
*/
ret = wait_event_interruptible(log_wait,
printk_get_next_message(&pmsg, atomic64_read(&user->seq), true,
false)); /* LMM(devkmsg_read:A) */
if (ret)
goto out;
}
if (pmsg.dropped) {
/* our last seen message is gone, return error and reset */
atomic64_set(&user->seq, pmsg.seq);
ret = -EPIPE;
goto out;
}
atomic64_set(&user->seq, pmsg.seq + 1);
if (pmsg.outbuf_len > count) {
ret = -EINVAL;
goto out;
}
if (copy_to_user(buf, outbuf, pmsg.outbuf_len)) {
ret = -EFAULT;
goto out;
}
ret = pmsg.outbuf_len;
out:
mutex_unlock(&user->lock);
return ret;
}
/*
* Be careful when modifying this function!!!
*
* Only few operations are supported because the device works only with the
* entire variable length messages (records). Non-standard values are
* returned in the other cases and has been this way for quite some time.
* User space applications might depend on this behavior.
*/
static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
{
struct devkmsg_user *user = file->private_data;
loff_t ret = 0;
if (offset)
return -ESPIPE;
switch (whence) {
case SEEK_SET:
/* the first record */
atomic64_set(&user->seq, prb_first_valid_seq(prb));
break;
case SEEK_DATA:
/*
* The first record after the last SYSLOG_ACTION_CLEAR,
* like issued by 'dmesg -c'. Reading /dev/kmsg itself
* changes no global state, and does not clear anything.
*/
atomic64_set(&user->seq, latched_seq_read_nolock(&clear_seq));
break;
case SEEK_END:
/* after the last record */
atomic64_set(&user->seq, prb_next_seq(prb));
break;
default:
ret = -EINVAL;
}
return ret;
}
static __poll_t devkmsg_poll(struct file *file, poll_table *wait)
{
struct devkmsg_user *user = file->private_data;
struct printk_info info;
__poll_t ret = 0;
poll_wait(file, &log_wait, wait);
if (prb_read_valid_info(prb, atomic64_read(&user->seq), &info, NULL)) {
/* return error when data has vanished underneath us */
if (info.seq != atomic64_read(&user->seq))
ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
else
ret = EPOLLIN|EPOLLRDNORM;
}
return ret;
}
static int devkmsg_open(struct inode *inode, struct file *file)
{
struct devkmsg_user *user;
int err;
if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
return -EPERM;
/* write-only does not need any file context */
if ((file->f_flags & O_ACCMODE) != O_WRONLY) {
err = check_syslog_permissions(SYSLOG_ACTION_READ_ALL,
SYSLOG_FROM_READER);
if (err)
return err;
}
user = kvmalloc(sizeof(struct devkmsg_user), GFP_KERNEL);
if (!user)
return -ENOMEM;
ratelimit_default_init(&user->rs);
ratelimit_set_flags(&user->rs, RATELIMIT_MSG_ON_RELEASE);
mutex_init(&user->lock);
atomic64_set(&user->seq, prb_first_valid_seq(prb));
file->private_data = user;
return 0;
}
static int devkmsg_release(struct inode *inode, struct file *file)
{
struct devkmsg_user *user = file->private_data;
ratelimit_state_exit(&user->rs);
mutex_destroy(&user->lock);
kvfree(user);
return 0;
}
const struct file_operations kmsg_fops = {
.open = devkmsg_open,
.read = devkmsg_read,
.write_iter = devkmsg_write,
.llseek = devkmsg_llseek,
.poll = devkmsg_poll,
.release = devkmsg_release,
};
#ifdef CONFIG_VMCORE_INFO
/*
* This appends the listed symbols to /proc/vmcore
*
* /proc/vmcore is used by various utilities, like crash and makedumpfile to
* obtain access to symbols that are otherwise very difficult to locate. These
* symbols are specifically used so that utilities can access and extract the
* dmesg log from a vmcore file after a crash.
*/
void log_buf_vmcoreinfo_setup(void)
{
struct dev_printk_info *dev_info = NULL;
VMCOREINFO_SYMBOL(prb);
VMCOREINFO_SYMBOL(printk_rb_static);
VMCOREINFO_SYMBOL(clear_seq);
/*
* Export struct size and field offsets. User space tools can
* parse it and detect any changes to structure down the line.
*/
VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
VMCOREINFO_OFFSET(printk_ringbuffer, fail);
VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
VMCOREINFO_OFFSET(prb_desc_ring, descs);
VMCOREINFO_OFFSET(prb_desc_ring, infos);
VMCOREINFO_OFFSET(prb_desc_ring, head_id);
VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
VMCOREINFO_STRUCT_SIZE(prb_desc);
VMCOREINFO_OFFSET(prb_desc, state_var);
VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
VMCOREINFO_STRUCT_SIZE(printk_info);
VMCOREINFO_OFFSET(printk_info, seq);
VMCOREINFO_OFFSET(printk_info, ts_nsec);
VMCOREINFO_OFFSET(printk_info, text_len);
VMCOREINFO_OFFSET(printk_info, caller_id);
VMCOREINFO_OFFSET(printk_info, dev_info);
VMCOREINFO_STRUCT_SIZE(dev_printk_info);
VMCOREINFO_OFFSET(dev_printk_info, subsystem);
VMCOREINFO_LENGTH(printk_info_subsystem, sizeof(dev_info->subsystem));
VMCOREINFO_OFFSET(dev_printk_info, device);
VMCOREINFO_LENGTH(printk_info_device, sizeof(dev_info->device));
VMCOREINFO_STRUCT_SIZE(prb_data_ring);
VMCOREINFO_OFFSET(prb_data_ring, size_bits);
VMCOREINFO_OFFSET(prb_data_ring, data);
VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
VMCOREINFO_SIZE(atomic_long_t);
VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
VMCOREINFO_STRUCT_SIZE(latched_seq);
VMCOREINFO_OFFSET(latched_seq, val);
}
#endif
/* requested log_buf_len from kernel cmdline */
static unsigned long __initdata new_log_buf_len;
/* we practice scaling the ring buffer by powers of 2 */
static void __init log_buf_len_update(u64 size)
{
if (size > (u64)LOG_BUF_LEN_MAX) {
size = (u64)LOG_BUF_LEN_MAX;
pr_err("log_buf over 2G is not supported.\n");
}
if (size)
size = roundup_pow_of_two(size);
if (size > log_buf_len)
new_log_buf_len = (unsigned long)size;
}
/* save requested log_buf_len since it's too early to process it */
static int __init log_buf_len_setup(char *str)
{
u64 size;
if (!str)
return -EINVAL;
size = memparse(str, &str);
log_buf_len_update(size);
return 0;
}
early_param("log_buf_len", log_buf_len_setup);
#ifdef CONFIG_SMP
#define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT)
static void __init log_buf_add_cpu(void)
{
unsigned int cpu_extra;
/*
* archs should set up cpu_possible_bits properly with
* set_cpu_possible() after setup_arch() but just in
* case lets ensure this is valid.
*/
if (num_possible_cpus() == 1)
return;
cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;
/* by default this will only continue through for large > 64 CPUs */
if (cpu_extra <= __LOG_BUF_LEN / 2)
return;
pr_info("log_buf_len individual max cpu contribution: %d bytes\n",
__LOG_CPU_MAX_BUF_LEN);
pr_info("log_buf_len total cpu_extra contributions: %d bytes\n",
cpu_extra);
pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN);
log_buf_len_update(cpu_extra + __LOG_BUF_LEN);
}
#else /* !CONFIG_SMP */
static inline void log_buf_add_cpu(void) {}
#endif /* CONFIG_SMP */
static void __init set_percpu_data_ready(void)
{
__printk_percpu_data_ready = true;
}
static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
struct printk_record *r)
{
struct prb_reserved_entry e;
struct printk_record dest_r;
prb_rec_init_wr(&dest_r, r->info->text_len);
if (!prb_reserve(&e, rb, &dest_r))
return 0;
memcpy(&dest_r.text_buf[0], &r->text_buf[0], r->info->text_len);
dest_r.info->text_len = r->info->text_len;
dest_r.info->facility = r->info->facility;
dest_r.info->level = r->info->level;
dest_r.info->flags = r->info->flags;
dest_r.info->ts_nsec = r->info->ts_nsec;
dest_r.info->caller_id = r->info->caller_id;
memcpy(&dest_r.info->dev_info, &r->info->dev_info, sizeof(dest_r.info->dev_info));
prb_final_commit(&e);
return prb_record_text_space(&e);
}
static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata;
void __init setup_log_buf(int early)
{
struct printk_info *new_infos;
unsigned int new_descs_count;
struct prb_desc *new_descs;
struct printk_info info;
struct printk_record r;
unsigned int text_size;
size_t new_descs_size;
size_t new_infos_size;
unsigned long flags;
char *new_log_buf;
unsigned int free;
u64 seq;
/*
* Some archs call setup_log_buf() multiple times - first is very
* early, e.g. from setup_arch(), and second - when percpu_areas
* are initialised.
*/
if (!early)
set_percpu_data_ready();
if (log_buf != __log_buf)
return;
if (!early && !new_log_buf_len)
log_buf_add_cpu();
if (!new_log_buf_len)
return;
new_descs_count = new_log_buf_len >> PRB_AVGBITS;
if (new_descs_count == 0) {
pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
return;
}
new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
if (unlikely(!new_log_buf)) {
pr_err("log_buf_len: %lu text bytes not available\n",
new_log_buf_len);
return;
}
new_descs_size = new_descs_count * sizeof(struct prb_desc);
new_descs = memblock_alloc(new_descs_size, LOG_ALIGN);
if (unlikely(!new_descs)) {
pr_err("log_buf_len: %zu desc bytes not available\n",
new_descs_size);
goto err_free_log_buf;
}
new_infos_size = new_descs_count * sizeof(struct printk_info);
new_infos = memblock_alloc(new_infos_size, LOG_ALIGN);
if (unlikely(!new_infos)) {
pr_err("log_buf_len: %zu info bytes not available\n",
new_infos_size);
goto err_free_descs;
}
prb_rec_init_rd(&r, &info, &setup_text_buf[0], sizeof(setup_text_buf));
prb_init(&printk_rb_dynamic,
new_log_buf, ilog2(new_log_buf_len),
new_descs, ilog2(new_descs_count),
new_infos);
local_irq_save(flags);
log_buf_len = new_log_buf_len;
log_buf = new_log_buf;
new_log_buf_len = 0;
free = __LOG_BUF_LEN;
prb_for_each_record(0, &printk_rb_static, seq, &r) {
text_size = add_to_rb(&printk_rb_dynamic, &r);
if (text_size > free)
free = 0;
else
free -= text_size;
}
prb = &printk_rb_dynamic;
local_irq_restore(flags);
/*
* Copy any remaining messages that might have appeared from
* NMI context after copying but before switching to the
* dynamic buffer.
*/
prb_for_each_record(seq, &printk_rb_static, seq, &r) {
text_size = add_to_rb(&printk_rb_dynamic, &r);
if (text_size > free)
free = 0;
else
free -= text_size;
}
if (seq != prb_next_seq(&printk_rb_static)) {
pr_err("dropped %llu messages\n",
prb_next_seq(&printk_rb_static) - seq);
}
pr_info("log_buf_len: %u bytes\n", log_buf_len);
pr_info("early log buf free: %u(%u%%)\n",
free, (free * 100) / __LOG_BUF_LEN);
return;
err_free_descs:
memblock_free(new_descs, new_descs_size);
err_free_log_buf:
memblock_free(new_log_buf, new_log_buf_len);
}
static bool __read_mostly ignore_loglevel;
static int __init ignore_loglevel_setup(char *str)
{
ignore_loglevel = true;
pr_info("debug: ignoring loglevel setting.\n");
return 0;
}
early_param("ignore_loglevel", ignore_loglevel_setup);
module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(ignore_loglevel,
"ignore loglevel setting (prints all kernel messages to the console)");
static bool suppress_message_printing(int level)
{
return (level >= console_loglevel && !ignore_loglevel);
}
#ifdef CONFIG_BOOT_PRINTK_DELAY
static int boot_delay; /* msecs delay after each printk during bootup */
static unsigned long long loops_per_msec; /* based on boot_delay */
static int __init boot_delay_setup(char *str)
{
unsigned long lpj;
lpj = preset_lpj ? preset_lpj : 1000000; /* some guess */
loops_per_msec = (unsigned long long)lpj / 1000 * HZ;
get_option(&str, &boot_delay);
if (boot_delay > 10 * 1000)
boot_delay = 0;
pr_debug("boot_delay: %u, preset_lpj: %ld, lpj: %lu, "
"HZ: %d, loops_per_msec: %llu\n",
boot_delay, preset_lpj, lpj, HZ, loops_per_msec);
return 0;
}
early_param("boot_delay", boot_delay_setup);
static void boot_delay_msec(int level)
{
unsigned long long k;
unsigned long timeout;
if ((boot_delay == 0 || system_state >= SYSTEM_RUNNING)
|| suppress_message_printing(level)) {
return;
}
k = (unsigned long long)loops_per_msec * boot_delay;
timeout = jiffies + msecs_to_jiffies(boot_delay);
while (k) {
k--;
cpu_relax();
/*
* use (volatile) jiffies to prevent
* compiler reduction; loop termination via jiffies
* is secondary and may or may not happen.
*/
if (time_after(jiffies, timeout))
break;
touch_nmi_watchdog();
}
}
#else
static inline void boot_delay_msec(int level)
{
}
#endif
static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME);
module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR);
static size_t print_syslog(unsigned int level, char *buf)
{
return sprintf(buf, "<%u>", level);
}
static size_t print_time(u64 ts, char *buf)
{
unsigned long rem_nsec = do_div(ts, 1000000000);
return sprintf(buf, "[%5lu.%06lu]",
(unsigned long)ts, rem_nsec / 1000);
}
#ifdef CONFIG_PRINTK_CALLER
static size_t print_caller(u32 id, char *buf)
{
char caller[12];
snprintf(caller, sizeof(caller), "%c%u",
id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
return sprintf(buf, "[%6s]", caller);
}
#else
#define print_caller(id, buf) 0
#endif
static size_t info_print_prefix(const struct printk_info *info, bool syslog,
bool time, char *buf)
{
size_t len = 0;
if (syslog) len = print_syslog((info->facility << 3) | info->level, buf); if (time) len += print_time(info->ts_nsec, buf + len); len += print_caller(info->caller_id, buf + len);
if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
buf[len++] = ' ';
buf[len] = '\0';
}
return len;
}
/*
* Prepare the record for printing. The text is shifted within the given
* buffer to avoid a need for another one. The following operations are
* done:
*
* - Add prefix for each line.
* - Drop truncated lines that no longer fit into the buffer.
* - Add the trailing newline that has been removed in vprintk_store().
* - Add a string terminator.
*
* Since the produced string is always terminated, the maximum possible
* return value is @r->text_buf_size - 1;
*
* Return: The length of the updated/prepared text, including the added
* prefixes and the newline. The terminator is not counted. The dropped
* line(s) are not counted.
*/
static size_t record_print_text(struct printk_record *r, bool syslog,
bool time)
{
size_t text_len = r->info->text_len;
size_t buf_size = r->text_buf_size;
char *text = r->text_buf;
char prefix[PRINTK_PREFIX_MAX];
bool truncated = false;
size_t prefix_len;
size_t line_len;
size_t len = 0;
char *next;
/*
* If the message was truncated because the buffer was not large
* enough, treat the available text as if it were the full text.
*/
if (text_len > buf_size)
text_len = buf_size;
prefix_len = info_print_prefix(r->info, syslog, time, prefix);
/*
* @text_len: bytes of unprocessed text
* @line_len: bytes of current line _without_ newline
* @text: pointer to beginning of current line
* @len: number of bytes prepared in r->text_buf
*/
for (;;) {
next = memchr(text, '\n', text_len);
if (next) {
line_len = next - text;
} else {
/* Drop truncated line(s). */
if (truncated)
break;
line_len = text_len;
}
/*
* Truncate the text if there is not enough space to add the
* prefix and a trailing newline and a terminator.
*/
if (len + prefix_len + text_len + 1 + 1 > buf_size) {
/* Drop even the current line if no space. */
if (len + prefix_len + line_len + 1 + 1 > buf_size)
break;
text_len = buf_size - len - prefix_len - 1 - 1;
truncated = true;
}
memmove(text + prefix_len, text, text_len); memcpy(text, prefix, prefix_len);
/*
* Increment the prepared length to include the text and
* prefix that were just moved+copied. Also increment for the
* newline at the end of this line. If this is the last line,
* there is no newline, but it will be added immediately below.
*/
len += prefix_len + line_len + 1;
if (text_len == line_len) {
/*
* This is the last line. Add the trailing newline
* removed in vprintk_store().
*/
text[prefix_len + line_len] = '\n';
break;
}
/*
* Advance beyond the added prefix and the related line with
* its newline.
*/
text += prefix_len + line_len + 1;
/*
* The remaining text has only decreased by the line with its
* newline.
*
* Note that @text_len can become zero. It happens when @text
* ended with a newline (either due to truncation or the
* original string ending with "\n\n"). The loop is correctly
* repeated and (if not truncated) an empty line with a prefix
* will be prepared.
*/
text_len -= line_len + 1;
}
/*
* If a buffer was provided, it will be terminated. Space for the
* string terminator is guaranteed to be available. The terminator is
* not counted in the return value.
*/
if (buf_size > 0) r->text_buf[len] = 0; return len;
}
static size_t get_record_print_text_size(struct printk_info *info,
unsigned int line_count,
bool syslog, bool time)
{
char prefix[PRINTK_PREFIX_MAX];
size_t prefix_len;
prefix_len = info_print_prefix(info, syslog, time, prefix);
/*
* Each line will be preceded with a prefix. The intermediate
* newlines are already within the text, but a final trailing
* newline will be added.
*/
return ((prefix_len * line_count) + info->text_len + 1);
}
/*
* Beginning with @start_seq, find the first record where it and all following
* records up to (but not including) @max_seq fit into @size.
*
* @max_seq is simply an upper bound and does not need to exist. If the caller
* does not require an upper bound, -1 can be used for @max_seq.
*/
static u64 find_first_fitting_seq(u64 start_seq, u64 max_seq, size_t size,
bool syslog, bool time)
{
struct printk_info info;
unsigned int line_count;
size_t len = 0;
u64 seq;
/* Determine the size of the records up to @max_seq. */
prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
if (info.seq >= max_seq)
break;
len += get_record_print_text_size(&info, line_count, syslog, time);
}
/*
* Adjust the upper bound for the next loop to avoid subtracting
* lengths that were never added.
*/
if (seq < max_seq)
max_seq = seq;
/*
* Move first record forward until length fits into the buffer. Ignore
* newest messages that were not counted in the above cycle. Messages
* might appear and get lost in the meantime. This is a best effort
* that prevents an infinite loop that could occur with a retry.
*/
prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
if (len <= size || info.seq >= max_seq)
break;
len -= get_record_print_text_size(&info, line_count, syslog, time);
}
return seq;
}
/* The caller is responsible for making sure @size is greater than 0. */
static int syslog_print(char __user *buf, int size)
{
struct printk_info info;
struct printk_record r;
char *text;
int len = 0;
u64 seq;
text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
if (!text)
return -ENOMEM;
prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);
mutex_lock(&syslog_lock);
/*
* Wait for the @syslog_seq record to be available. @syslog_seq may
* change while waiting.
*/
do {
seq = syslog_seq;
mutex_unlock(&syslog_lock);
/*
* Guarantee this task is visible on the waitqueue before
* checking the wake condition.
*
* The full memory barrier within set_current_state() of
* prepare_to_wait_event() pairs with the full memory barrier
* within wq_has_sleeper().
*
* This pairs with __wake_up_klogd:A.
*/
len = wait_event_interruptible(log_wait,
prb_read_valid(prb, seq, NULL)); /* LMM(syslog_print:A) */
mutex_lock(&syslog_lock);
if (len)
goto out;
} while (syslog_seq != seq);
/*
* Copy records that fit into the buffer. The above cycle makes sure
* that the first record is always available.
*/
do {
size_t n;
size_t skip;
int err;
if (!prb_read_valid(prb, syslog_seq, &r))
break;
if (r.info->seq != syslog_seq) {
/* message is gone, move to next valid one */
syslog_seq = r.info->seq;
syslog_partial = 0;
}
/*
* To keep reading/counting partial line consistent,
* use printk_time value as of the beginning of a line.
*/
if (!syslog_partial)
syslog_time = printk_time;
skip = syslog_partial;
n = record_print_text(&r, true, syslog_time);
if (n - syslog_partial <= size) {
/* message fits into buffer, move forward */
syslog_seq = r.info->seq + 1;
n -= syslog_partial;
syslog_partial = 0;
} else if (!len){
/* partial read(), remember position */
n = size;
syslog_partial += n;
} else
n = 0;
if (!n)
break;
mutex_unlock(&syslog_lock);
err = copy_to_user(buf, text + skip, n);
mutex_lock(&syslog_lock);
if (err) {
if (!len)
len = -EFAULT;
break;
}
len += n;
size -= n;
buf += n;
} while (size);
out:
mutex_unlock(&syslog_lock);
kfree(text);
return len;
}
static int syslog_print_all(char __user *buf, int size, bool clear)
{
struct printk_info info;
struct printk_record r;
char *text;
int len = 0;
u64 seq;
bool time;
text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
if (!text)
return -ENOMEM;
time = printk_time;
/*
* Find first record that fits, including all following records,
* into the user-provided buffer for this dump.
*/
seq = find_first_fitting_seq(latched_seq_read_nolock(&clear_seq), -1,
size, true, time);
prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);
prb_for_each_record(seq, prb, seq, &r) {
int textlen;
textlen = record_print_text(&r, true, time);
if (len + textlen > size) {
seq--;
break;
}
if (copy_to_user(buf + len, text, textlen))
len = -EFAULT;
else
len += textlen;
if (len < 0)
break;
}
if (clear) {
mutex_lock(&syslog_lock);
latched_seq_write(&clear_seq, seq);
mutex_unlock(&syslog_lock);
}
kfree(text);
return len;
}
static void syslog_clear(void)
{
mutex_lock(&syslog_lock);
latched_seq_write(&clear_seq, prb_next_seq(prb));
mutex_unlock(&syslog_lock);
}
int do_syslog(int type, char __user *buf, int len, int source)
{
struct printk_info info;
bool clear = false;
static int saved_console_loglevel = LOGLEVEL_DEFAULT;
int error;
error = check_syslog_permissions(type, source);
if (error)
return error;
switch (type) {
case SYSLOG_ACTION_CLOSE: /* Close log */
break;
case SYSLOG_ACTION_OPEN: /* Open log */
break;
case SYSLOG_ACTION_READ: /* Read from log */
if (!buf || len < 0)
return -EINVAL;
if (!len)
return 0;
if (!access_ok(buf, len))
return -EFAULT;
error = syslog_print(buf, len);
break;
/* Read/clear last kernel messages */
case SYSLOG_ACTION_READ_CLEAR:
clear = true;
fallthrough;
/* Read last kernel messages */
case SYSLOG_ACTION_READ_ALL:
if (!buf || len < 0)
return -EINVAL;
if (!len)
return 0;
if (!access_ok(buf, len))
return -EFAULT;
error = syslog_print_all(buf, len, clear);
break;
/* Clear ring buffer */
case SYSLOG_ACTION_CLEAR:
syslog_clear();
break;
/* Disable logging to console */
case SYSLOG_ACTION_CONSOLE_OFF:
if (saved_console_loglevel == LOGLEVEL_DEFAULT)
saved_console_loglevel = console_loglevel;
console_loglevel = minimum_console_loglevel;
break;
/* Enable logging to console */
case SYSLOG_ACTION_CONSOLE_ON:
if (saved_console_loglevel != LOGLEVEL_DEFAULT) {
console_loglevel = saved_console_loglevel;
saved_console_loglevel = LOGLEVEL_DEFAULT;
}
break;
/* Set level of messages printed to console */
case SYSLOG_ACTION_CONSOLE_LEVEL:
if (len < 1 || len > 8)
return -EINVAL;
if (len < minimum_console_loglevel)
len = minimum_console_loglevel;
console_loglevel = len;
/* Implicitly re-enable logging to console */
saved_console_loglevel = LOGLEVEL_DEFAULT;
break;
/* Number of chars in the log buffer */
case SYSLOG_ACTION_SIZE_UNREAD:
mutex_lock(&syslog_lock);
if (!prb_read_valid_info(prb, syslog_seq, &info, NULL)) {
/* No unread messages. */
mutex_unlock(&syslog_lock);
return 0;
}
if (info.seq != syslog_seq) {
/* messages are gone, move to first one */
syslog_seq = info.seq;
syslog_partial = 0;
}
if (source == SYSLOG_FROM_PROC) {
/*
* Short-cut for poll(/"proc/kmsg") which simply checks
* for pending data, not the size; return the count of
* records, not the length.
*/
error = prb_next_seq(prb) - syslog_seq;
} else {
bool time = syslog_partial ? syslog_time : printk_time;
unsigned int line_count;
u64 seq;
prb_for_each_info(syslog_seq, prb, seq, &info,
&line_count) {
error += get_record_print_text_size(&info, line_count,
true, time);
time = printk_time;
}
error -= syslog_partial;
}
mutex_unlock(&syslog_lock);
break;
/* Size of the log buffer */
case SYSLOG_ACTION_SIZE_BUFFER:
error = log_buf_len;
break;
default:
error = -EINVAL;
break;
}
return error;
}
SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
{
return do_syslog(type, buf, len, SYSLOG_FROM_READER);
}
/*
* Special console_lock variants that help to reduce the risk of soft-lockups.
* They allow to pass console_lock to another printk() call using a busy wait.
*/
#ifdef CONFIG_LOCKDEP
static struct lockdep_map console_owner_dep_map = {
.name = "console_owner"
};
#endif
static DEFINE_RAW_SPINLOCK(console_owner_lock);
static struct task_struct *console_owner;
static bool console_waiter;
/**
* console_lock_spinning_enable - mark beginning of code where another
* thread might safely busy wait
*
* This basically converts console_lock into a spinlock. This marks
* the section where the console_lock owner can not sleep, because
* there may be a waiter spinning (like a spinlock). Also it must be
* ready to hand over the lock at the end of the section.
*/
static void console_lock_spinning_enable(void)
{
/*
* Do not use spinning in panic(). The panic CPU wants to keep the lock.
* Non-panic CPUs abandon the flush anyway.
*
* Just keep the lockdep annotation. The panic-CPU should avoid
* taking console_owner_lock because it might cause a deadlock.
* This looks like the easiest way how to prevent false lockdep
* reports without handling races a lockless way.
*/
if (panic_in_progress())
goto lockdep;
raw_spin_lock(&console_owner_lock);
console_owner = current;
raw_spin_unlock(&console_owner_lock);
lockdep:
/* The waiter may spin on us after setting console_owner */
spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
}
/**
* console_lock_spinning_disable_and_check - mark end of code where another
* thread was able to busy wait and check if there is a waiter
* @cookie: cookie returned from console_srcu_read_lock()
*
* This is called at the end of the section where spinning is allowed.
* It has two functions. First, it is a signal that it is no longer
* safe to start busy waiting for the lock. Second, it checks if
* there is a busy waiter and passes the lock rights to her.
*
* Important: Callers lose both the console_lock and the SRCU read lock if
* there was a busy waiter. They must not touch items synchronized by
* console_lock or SRCU read lock in this case.
*
* Return: 1 if the lock rights were passed, 0 otherwise.
*/
static int console_lock_spinning_disable_and_check(int cookie)
{
int waiter;
/*
* Ignore spinning waiters during panic() because they might get stopped
* or blocked at any time,
*
* It is safe because nobody is allowed to start spinning during panic
* in the first place. If there has been a waiter then non panic CPUs
* might stay spinning. They would get stopped anyway. The panic context
* will never start spinning and an interrupted spin on panic CPU will
* never continue.
*/
if (panic_in_progress()) {
/* Keep lockdep happy. */
spin_release(&console_owner_dep_map, _THIS_IP_);
return 0;
}
raw_spin_lock(&console_owner_lock);
waiter = READ_ONCE(console_waiter);
console_owner = NULL;
raw_spin_unlock(&console_owner_lock);
if (!waiter) {
spin_release(&console_owner_dep_map, _THIS_IP_);
return 0;
}
/* The waiter is now free to continue */
WRITE_ONCE(console_waiter, false);
spin_release(&console_owner_dep_map, _THIS_IP_);
/*
* Preserve lockdep lock ordering. Release the SRCU read lock before
* releasing the console_lock.
*/
console_srcu_read_unlock(cookie);
/*
* Hand off console_lock to waiter. The waiter will perform
* the up(). After this, the waiter is the console_lock owner.
*/
mutex_release(&console_lock_dep_map, _THIS_IP_);
return 1;
}
/**
* console_trylock_spinning - try to get console_lock by busy waiting
*
* This allows to busy wait for the console_lock when the current
* owner is running in specially marked sections. It means that
* the current owner is running and cannot reschedule until it
* is ready to lose the lock.
*
* Return: 1 if we got the lock, 0 othrewise
*/
static int console_trylock_spinning(void)
{
struct task_struct *owner = NULL;
bool waiter;
bool spin = false;
unsigned long flags;
if (console_trylock())
return 1;
/*
* It's unsafe to spin once a panic has begun. If we are the
* panic CPU, we may have already halted the owner of the
* console_sem. If we are not the panic CPU, then we should
* avoid taking console_sem, so the panic CPU has a better
* chance of cleanly acquiring it later.
*/
if (panic_in_progress())
return 0;
printk_safe_enter_irqsave(flags);
raw_spin_lock(&console_owner_lock);
owner = READ_ONCE(console_owner);
waiter = READ_ONCE(console_waiter);
if (!waiter && owner && owner != current) { WRITE_ONCE(console_waiter, true);
spin = true;
}
raw_spin_unlock(&console_owner_lock);
/*
* If there is an active printk() writing to the
* consoles, instead of having it write our data too,
* see if we can offload that load from the active
* printer, and do some printing ourselves.
* Go into a spin only if there isn't already a waiter
* spinning, and there is an active printer, and
* that active printer isn't us (recursive printk?).
*/
if (!spin) {
printk_safe_exit_irqrestore(flags);
return 0;
}
/* We spin waiting for the owner to release us */
spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
/* Owner will clear console_waiter on hand off */
while (READ_ONCE(console_waiter))
cpu_relax(); spin_release(&console_owner_dep_map, _THIS_IP_); printk_safe_exit_irqrestore(flags);
/*
* The owner passed the console lock to us.
* Since we did not spin on console lock, annotate
* this as a trylock. Otherwise lockdep will
* complain.
*/
mutex_acquire(&console_lock_dep_map, 0, 1, _THIS_IP_);
/*
* Update @console_may_schedule for trylock because the previous
* owner may have been schedulable.
*/
console_may_schedule = 0;
return 1;
}
/*
* Recursion is tracked separately on each CPU. If NMIs are supported, an
* additional NMI context per CPU is also separately tracked. Until per-CPU
* is available, a separate "early tracking" is performed.
*/
static DEFINE_PER_CPU(u8, printk_count);
static u8 printk_count_early;
#ifdef CONFIG_HAVE_NMI
static DEFINE_PER_CPU(u8, printk_count_nmi);
static u8 printk_count_nmi_early;
#endif
/*
* Recursion is limited to keep the output sane. printk() should not require
* more than 1 level of recursion (allowing, for example, printk() to trigger
* a WARN), but a higher value is used in case some printk-internal errors
* exist, such as the ringbuffer validation checks failing.
*/
#define PRINTK_MAX_RECURSION 3
/*
* Return a pointer to the dedicated counter for the CPU+context of the
* caller.
*/
static u8 *__printk_recursion_counter(void)
{
#ifdef CONFIG_HAVE_NMI
if (in_nmi()) { if (printk_percpu_data_ready()) return this_cpu_ptr(&printk_count_nmi);
return &printk_count_nmi_early;
}
#endif
if (printk_percpu_data_ready()) return this_cpu_ptr(&printk_count);
return &printk_count_early;
}
/*
* Enter recursion tracking. Interrupts are disabled to simplify tracking.
* The caller must check the boolean return value to see if the recursion is
* allowed. On failure, interrupts are not disabled.
*
* @recursion_ptr must be a variable of type (u8 *) and is the same variable
* that is passed to printk_exit_irqrestore().
*/
#define printk_enter_irqsave(recursion_ptr, flags) \
({ \
bool success = true; \
\
typecheck(u8 *, recursion_ptr); \
local_irq_save(flags); \
(recursion_ptr) = __printk_recursion_counter(); \
if (*(recursion_ptr) > PRINTK_MAX_RECURSION) { \
local_irq_restore(flags); \
success = false; \
} else { \
(*(recursion_ptr))++; \
} \
success; \
})
/* Exit recursion tracking, restoring interrupts. */
#define printk_exit_irqrestore(recursion_ptr, flags) \
do { \
typecheck(u8 *, recursion_ptr); \
(*(recursion_ptr))--; \
local_irq_restore(flags); \
} while (0)
int printk_delay_msec __read_mostly;
static inline void printk_delay(int level)
{
boot_delay_msec(level);
if (unlikely(printk_delay_msec)) {
int m = printk_delay_msec;
while (m--) { mdelay(1);
touch_nmi_watchdog();
}
}
}
static inline u32 printk_caller_id(void)
{
return in_task() ? task_pid_nr(current) : 0x80000000 + smp_processor_id();
}
/**
* printk_parse_prefix - Parse level and control flags.
*
* @text: The terminated text message.
* @level: A pointer to the current level value, will be updated.
* @flags: A pointer to the current printk_info flags, will be updated.
*
* @level may be NULL if the caller is not interested in the parsed value.
* Otherwise the variable pointed to by @level must be set to
* LOGLEVEL_DEFAULT in order to be updated with the parsed value.
*
* @flags may be NULL if the caller is not interested in the parsed value.
* Otherwise the variable pointed to by @flags will be OR'd with the parsed
* value.
*
* Return: The length of the parsed level and control flags.
*/
u16 printk_parse_prefix(const char *text, int *level,
enum printk_info_flags *flags)
{
u16 prefix_len = 0;
int kern_level;
while (*text) { kern_level = printk_get_level(text);
if (!kern_level)
break;
switch (kern_level) {
case '0' ... '7':
if (level && *level == LOGLEVEL_DEFAULT)
*level = kern_level - '0';
break;
case 'c': /* KERN_CONT */
if (flags) *flags |= LOG_CONT;
}
prefix_len += 2;
text += 2;
}
return prefix_len;
}
__printf(5, 0)
static u16 printk_sprint(char *text, u16 size, int facility,
enum printk_info_flags *flags, const char *fmt,
va_list args)
{
u16 text_len;
text_len = vscnprintf(text, size, fmt, args);
/* Mark and strip a trailing newline. */
if (text_len && text[text_len - 1] == '\n') { text_len--;
*flags |= LOG_NEWLINE;
}
/* Strip log level and control flags. */
if (facility == 0) {
u16 prefix_len;
prefix_len = printk_parse_prefix(text, NULL, NULL);
if (prefix_len) {
text_len -= prefix_len;
memmove(text, text + prefix_len, text_len);
}
}
trace_console(text, text_len); return text_len;
}
__printf(4, 0)
int vprintk_store(int facility, int level,
const struct dev_printk_info *dev_info,
const char *fmt, va_list args)
{
struct prb_reserved_entry e;
enum printk_info_flags flags = 0;
struct printk_record r;
unsigned long irqflags;
u16 trunc_msg_len = 0;
char prefix_buf[8];
u8 *recursion_ptr;
u16 reserve_size;
va_list args2;
u32 caller_id;
u16 text_len;
int ret = 0;
u64 ts_nsec;
if (!printk_enter_irqsave(recursion_ptr, irqflags)) return 0;
/*
* Since the duration of printk() can vary depending on the message
* and state of the ringbuffer, grab the timestamp now so that it is
* close to the call of printk(). This provides a more deterministic
* timestamp with respect to the caller.
*/
ts_nsec = local_clock();
caller_id = printk_caller_id();
/*
* The sprintf needs to come first since the syslog prefix might be
* passed in as a parameter. An extra byte must be reserved so that
* later the vscnprintf() into the reserved buffer has room for the
* terminating '\0', which is not counted by vsnprintf().
*/
va_copy(args2, args);
reserve_size = vsnprintf(&prefix_buf[0], sizeof(prefix_buf), fmt, args2) + 1;
va_end(args2);
if (reserve_size > PRINTKRB_RECORD_MAX)
reserve_size = PRINTKRB_RECORD_MAX;
/* Extract log level or control flags. */
if (facility == 0)
printk_parse_prefix(&prefix_buf[0], &level, &flags); if (level == LOGLEVEL_DEFAULT) level = default_message_loglevel;
if (dev_info)
flags |= LOG_NEWLINE; if (flags & LOG_CONT) { prb_rec_init_wr(&r, reserve_size);
if (prb_reserve_in_last(&e, prb, &r, caller_id, PRINTKRB_RECORD_MAX)) {
text_len = printk_sprint(&r.text_buf[r.info->text_len], reserve_size,
facility, &flags, fmt, args);
r.info->text_len += text_len;
if (flags & LOG_NEWLINE) {
r.info->flags |= LOG_NEWLINE;
prb_final_commit(&e);
} else {
prb_commit(&e);
}
ret = text_len;
goto out;
}
}
/*
* Explicitly initialize the record before every prb_reserve() call.
* prb_reserve_in_last() and prb_reserve() purposely invalidate the
* structure when they fail.
*/
prb_rec_init_wr(&r, reserve_size);
if (!prb_reserve(&e, prb, &r)) {
/* truncate the message if it is too long for empty buffer */
truncate_msg(&reserve_size, &trunc_msg_len); prb_rec_init_wr(&r, reserve_size + trunc_msg_len);
if (!prb_reserve(&e, prb, &r))
goto out;
}
/* fill message */
text_len = printk_sprint(&r.text_buf[0], reserve_size, facility, &flags, fmt, args);
if (trunc_msg_len)
memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len); r.info->text_len = text_len + trunc_msg_len;
r.info->facility = facility;
r.info->level = level & 7;
r.info->flags = flags & 0x1f;
r.info->ts_nsec = ts_nsec;
r.info->caller_id = caller_id;
if (dev_info)
memcpy(&r.info->dev_info, dev_info, sizeof(r.info->dev_info));
/* A message without a trailing newline can be continued. */
if (!(flags & LOG_NEWLINE)) prb_commit(&e);
else
prb_final_commit(&e); ret = text_len + trunc_msg_len;
out:
printk_exit_irqrestore(recursion_ptr, irqflags);
return ret;
}
asmlinkage int vprintk_emit(int facility, int level,
const struct dev_printk_info *dev_info,
const char *fmt, va_list args)
{
int printed_len;
bool in_sched = false;
/* Suppress unimportant messages after panic happens */
if (unlikely(suppress_printk))
return 0;
/*
* The messages on the panic CPU are the most important. If
* non-panic CPUs are generating any messages, they will be
* silently dropped.
*/
if (other_cpu_in_panic())
return 0;
if (level == LOGLEVEL_SCHED) {
level = LOGLEVEL_DEFAULT;
in_sched = true;
}
printk_delay(level); printed_len = vprintk_store(facility, level, dev_info, fmt, args);
/* If called from the scheduler, we can not call up(). */
if (!in_sched) {
/*
* The caller may be holding system-critical or
* timing-sensitive locks. Disable preemption during
* printing of all remaining records to all consoles so that
* this context can return as soon as possible. Hopefully
* another printk() caller will take over the printing.
*/
preempt_disable();
/*
* Try to acquire and then immediately release the console
* semaphore. The release will print out buffers. With the
* spinning variant, this context tries to take over the
* printing from another printing context.
*/
if (console_trylock_spinning())
console_unlock();
preempt_enable();
}
if (in_sched)
defer_console_output();
else
wake_up_klogd();
return printed_len;
}
EXPORT_SYMBOL(vprintk_emit);
int vprintk_default(const char *fmt, va_list args)
{
return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, fmt, args);
}
EXPORT_SYMBOL_GPL(vprintk_default);
asmlinkage __visible int _printk(const char *fmt, ...)
{
va_list args;
int r;
va_start(args, fmt);
r = vprintk(fmt, args);
va_end(args);
return r;
}
EXPORT_SYMBOL(_printk);
static bool pr_flush(int timeout_ms, bool reset_on_progress);
static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
#else /* CONFIG_PRINTK */
#define printk_time false
#define prb_read_valid(rb, seq, r) false
#define prb_first_valid_seq(rb) 0
#define prb_next_seq(rb) 0
static u64 syslog_seq;
static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }
static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
#endif /* CONFIG_PRINTK */
#ifdef CONFIG_EARLY_PRINTK
struct console *early_console;
asmlinkage __visible void early_printk(const char *fmt, ...)
{
va_list ap;
char buf[512];
int n;
if (!early_console)
return;
va_start(ap, fmt);
n = vscnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
early_console->write(early_console, buf, n);
}
#endif
static void set_user_specified(struct console_cmdline *c, bool user_specified)
{
if (!user_specified)
return;
/*
* @c console was defined by the user on the command line.
* Do not clear when added twice also by SPCR or the device tree.
*/
c->user_specified = true;
/* At least one console defined by the user on the command line. */
console_set_on_cmdline = 1;
}
static int __add_preferred_console(const char *name, const short idx, char *options,
char *brl_options, bool user_specified)
{
struct console_cmdline *c;
int i;
/*
* We use a signed short index for struct console for device drivers to
* indicate a not yet assigned index or port. However, a negative index
* value is not valid for preferred console.
*/
if (idx < 0)
return -EINVAL;
/*
* See if this tty is not yet registered, and
* if we have a slot free.
*/
for (i = 0, c = console_cmdline;
i < MAX_CMDLINECONSOLES && c->name[0];
i++, c++) {
if (strcmp(c->name, name) == 0 && c->index == idx) {
if (!brl_options)
preferred_console = i;
set_user_specified(c, user_specified);
return 0;
}
}
if (i == MAX_CMDLINECONSOLES)
return -E2BIG;
if (!brl_options)
preferred_console = i;
strscpy(c->name, name, sizeof(c->name));
c->options = options;
set_user_specified(c, user_specified);
braille_set_options(c, brl_options);
c->index = idx;
return 0;
}
static int __init console_msg_format_setup(char *str)
{
if (!strcmp(str, "syslog"))
console_msg_format = MSG_FORMAT_SYSLOG;
if (!strcmp(str, "default"))
console_msg_format = MSG_FORMAT_DEFAULT;
return 1;
}
__setup("console_msg_format=", console_msg_format_setup);
/*
* Set up a console. Called via do_early_param() in init/main.c
* for each "console=" parameter in the boot command line.
*/
static int __init console_setup(char *str)
{
char buf[sizeof(console_cmdline[0].name) + 4]; /* 4 for "ttyS" */
char *s, *options, *brl_options = NULL;
int idx;
/*
* console="" or console=null have been suggested as a way to
* disable console output. Use ttynull that has been created
* for exactly this purpose.
*/
if (str[0] == 0 || strcmp(str, "null") == 0) {
__add_preferred_console("ttynull", 0, NULL, NULL, true);
return 1;
}
if (_braille_console_setup(&str, &brl_options))
return 1;
/*
* Decode str into name, index, options.
*/
if (str[0] >= '0' && str[0] <= '9') {
strcpy(buf, "ttyS");
strncpy(buf + 4, str, sizeof(buf) - 5);
} else {
strncpy(buf, str, sizeof(buf) - 1);
}
buf[sizeof(buf) - 1] = 0;
options = strchr(str, ',');
if (options)
*(options++) = 0;
#ifdef __sparc__
if (!strcmp(str, "ttya"))
strcpy(buf, "ttyS0");
if (!strcmp(str, "ttyb"))
strcpy(buf, "ttyS1");
#endif
for (s = buf; *s; s++)
if (isdigit(*s) || *s == ',')
break;
idx = simple_strtoul(s, NULL, 10);
*s = 0;
__add_preferred_console(buf, idx, options, brl_options, true);
return 1;
}
__setup("console=", console_setup);
/**
* add_preferred_console - add a device to the list of preferred consoles.
* @name: device name
* @idx: device index
* @options: options for this console
*
* The last preferred console added will be used for kernel messages
* and stdin/out/err for init. Normally this is used by console_setup
* above to handle user-supplied console arguments; however it can also
* be used by arch-specific code either to override the user or more
* commonly to provide a default console (ie from PROM variables) when
* the user has not supplied one.
*/
int add_preferred_console(const char *name, const short idx, char *options)
{
return __add_preferred_console(name, idx, options, NULL, false);
}
bool console_suspend_enabled = true;
EXPORT_SYMBOL(console_suspend_enabled);
static int __init console_suspend_disable(char *str)
{
console_suspend_enabled = false;
return 1;
}
__setup("no_console_suspend", console_suspend_disable);
module_param_named(console_suspend, console_suspend_enabled,
bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(console_suspend, "suspend console during suspend"
" and hibernate operations");
static bool printk_console_no_auto_verbose;
void console_verbose(void)
{
if (console_loglevel && !printk_console_no_auto_verbose)
console_loglevel = CONSOLE_LOGLEVEL_MOTORMOUTH;
}
EXPORT_SYMBOL_GPL(console_verbose);
module_param_named(console_no_auto_verbose, printk_console_no_auto_verbose, bool, 0644);
MODULE_PARM_DESC(console_no_auto_verbose, "Disable console loglevel raise to highest on oops/panic/etc");
/**
* suspend_console - suspend the console subsystem
*
* This disables printk() while we go into suspend states
*/
void suspend_console(void)
{
struct console *con;
if (!console_suspend_enabled)
return;
pr_info("Suspending console(s) (use no_console_suspend to debug)\n");
pr_flush(1000, true);
console_list_lock();
for_each_console(con)
console_srcu_write_flags(con, con->flags | CON_SUSPENDED);
console_list_unlock();
/*
* Ensure that all SRCU list walks have completed. All printing
* contexts must be able to see that they are suspended so that it
* is guaranteed that all printing has stopped when this function
* completes.
*/
synchronize_srcu(&console_srcu);
}
void resume_console(void)
{
struct console *con;
if (!console_suspend_enabled)
return;
console_list_lock();
for_each_console(con)
console_srcu_write_flags(con, con->flags & ~CON_SUSPENDED);
console_list_unlock();
/*
* Ensure that all SRCU list walks have completed. All printing
* contexts must be able to see they are no longer suspended so
* that they are guaranteed to wake up and resume printing.
*/
synchronize_srcu(&console_srcu);
pr_flush(1000, true);
}
/**
* console_cpu_notify - print deferred console messages after CPU hotplug
* @cpu: unused
*
* If printk() is called from a CPU that is not online yet, the messages
* will be printed on the console only if there are CON_ANYTIME consoles.
* This function is called when a new CPU comes online (or fails to come
* up) or goes offline.
*/
static int console_cpu_notify(unsigned int cpu)
{
if (!cpuhp_tasks_frozen) {
/* If trylock fails, someone else is doing the printing */
if (console_trylock())
console_unlock();
}
return 0;
}
/**
* console_lock - block the console subsystem from printing
*
* Acquires a lock which guarantees that no consoles will
* be in or enter their write() callback.
*
* Can sleep, returns nothing.
*/
void console_lock(void)
{
might_sleep();
/* On panic, the console_lock must be left to the panic cpu. */
while (other_cpu_in_panic())
msleep(1000);
down_console_sem();
console_locked = 1;
console_may_schedule = 1;
}
EXPORT_SYMBOL(console_lock);
/**
* console_trylock - try to block the console subsystem from printing
*
* Try to acquire a lock which guarantees that no consoles will
* be in or enter their write() callback.
*
* returns 1 on success, and 0 on failure to acquire the lock.
*/
int console_trylock(void)
{
/* On panic, the console_lock must be left to the panic cpu. */
if (other_cpu_in_panic())
return 0;
if (down_trylock_console_sem())
return 0;
console_locked = 1;
console_may_schedule = 0;
return 1;
}
EXPORT_SYMBOL(console_trylock);
int is_console_locked(void)
{
return console_locked;
}
EXPORT_SYMBOL(is_console_locked);
/*
* Check if the given console is currently capable and allowed to print
* records.
*
* Requires the console_srcu_read_lock.
*/
static inline bool console_is_usable(struct console *con)
{
short flags = console_srcu_read_flags(con);
if (!(flags & CON_ENABLED))
return false;
if ((flags & CON_SUSPENDED))
return false;
if (!con->write)
return false;
/*
* Console drivers may assume that per-cpu resources have been
* allocated. So unless they're explicitly marked as being able to
* cope (CON_ANYTIME) don't call them until this CPU is officially up.
*/
if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
return false;
return true;
}
static void __console_unlock(void)
{
console_locked = 0; up_console_sem();
}
#ifdef CONFIG_PRINTK
/*
* Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This
* is achieved by shifting the existing message over and inserting the dropped
* message.
*
* @pmsg is the printk message to prepend.
*
* @dropped is the dropped count to report in the dropped message.
*
* If the message text in @pmsg->pbufs->outbuf does not have enough space for
* the dropped message, the message text will be sufficiently truncated.
*
* If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
*/
void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
{
struct printk_buffers *pbufs = pmsg->pbufs;
const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
const size_t outbuf_sz = sizeof(pbufs->outbuf);
char *scratchbuf = &pbufs->scratchbuf[0];
char *outbuf = &pbufs->outbuf[0];
size_t len;
len = scnprintf(scratchbuf, scratchbuf_sz,
"** %lu printk messages dropped **\n", dropped);
/*
* Make sure outbuf is sufficiently large before prepending.
* Keep at least the prefix when the message must be truncated.
* It is a rather theoretical problem when someone tries to
* use a minimalist buffer.
*/
if (WARN_ON_ONCE(len + PRINTK_PREFIX_MAX >= outbuf_sz))
return;
if (pmsg->outbuf_len + len >= outbuf_sz) {
/* Truncate the message, but keep it terminated. */
pmsg->outbuf_len = outbuf_sz - (len + 1);
outbuf[pmsg->outbuf_len] = 0;
}
memmove(outbuf + len, outbuf, pmsg->outbuf_len + 1);
memcpy(outbuf, scratchbuf, len);
pmsg->outbuf_len += len;
}
/*
* Read and format the specified record (or a later record if the specified
* record is not available).
*
* @pmsg will contain the formatted result. @pmsg->pbufs must point to a
* struct printk_buffers.
*
* @seq is the record to read and format. If it is not available, the next
* valid record is read.
*
* @is_extended specifies if the message should be formatted for extended
* console output.
*
* @may_supress specifies if records may be skipped based on loglevel.
*
* Returns false if no record is available. Otherwise true and all fields
* of @pmsg are valid. (See the documentation of struct printk_message
* for information about the @pmsg fields.)
*/
bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
bool is_extended, bool may_suppress)
{
struct printk_buffers *pbufs = pmsg->pbufs;
const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
const size_t outbuf_sz = sizeof(pbufs->outbuf);
char *scratchbuf = &pbufs->scratchbuf[0];
char *outbuf = &pbufs->outbuf[0];
struct printk_info info;
struct printk_record r;
size_t len = 0;
/*
* Formatting extended messages requires a separate buffer, so use the
* scratch buffer to read in the ringbuffer text.
*
* Formatting normal messages is done in-place, so read the ringbuffer
* text directly into the output buffer.
*/
if (is_extended)
prb_rec_init_rd(&r, &info, scratchbuf, scratchbuf_sz);
else
prb_rec_init_rd(&r, &info, outbuf, outbuf_sz);
if (!prb_read_valid(prb, seq, &r))
return false;
pmsg->seq = r.info->seq;
pmsg->dropped = r.info->seq - seq;
/* Skip record that has level above the console loglevel. */
if (may_suppress && suppress_message_printing(r.info->level))
goto out;
if (is_extended) { len = info_print_ext_header(outbuf, outbuf_sz, r.info); len += msg_print_ext_body(outbuf + len, outbuf_sz - len,
&r.text_buf[0], r.info->text_len, &r.info->dev_info);
} else {
len = record_print_text(&r, console_msg_format & MSG_FORMAT_SYSLOG, printk_time);
}
out:
pmsg->outbuf_len = len; return true;
}
/*
* Used as the printk buffers for non-panic, serialized console printing.
* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.
* Its usage requires the console_lock held.
*/
struct printk_buffers printk_shared_pbufs;
/*
* Print one record for the given console. The record printed is whatever
* record is the next available record for the given console.
*
* @handover will be set to true if a printk waiter has taken over the
* console_lock, in which case the caller is no longer holding both the
* console_lock and the SRCU read lock. Otherwise it is set to false.
*
* @cookie is the cookie from the SRCU read lock.
*
* Returns false if the given console has no next record to print, otherwise
* true.
*
* Requires the console_lock and the SRCU read lock.
*/
static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
{
bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
char *outbuf = &printk_shared_pbufs.outbuf[0];
struct printk_message pmsg = {
.pbufs = &printk_shared_pbufs,
};
unsigned long flags;
*handover = false;
if (!printk_get_next_message(&pmsg, con->seq, is_extended, true))
return false;
con->dropped += pmsg.dropped;
/* Skip messages of formatted length 0. */
if (pmsg.outbuf_len == 0) {
con->seq = pmsg.seq + 1;
goto skip;
}
if (con->dropped && !is_extended) { console_prepend_dropped(&pmsg, con->dropped);
con->dropped = 0;
}
/*
* While actively printing out messages, if another printk()
* were to occur on another CPU, it may wait for this one to
* finish. This task can not be preempted if there is a
* waiter waiting to take over.
*
* Interrupts are disabled because the hand over to a waiter
* must not be interrupted until the hand over is completed
* (@console_waiter is cleared).
*/
printk_safe_enter_irqsave(flags); console_lock_spinning_enable();
/* Do not trace print latency. */
stop_critical_timings();
/* Write everything out to the hardware. */
con->write(con, outbuf, pmsg.outbuf_len);
start_critical_timings();
con->seq = pmsg.seq + 1;
*handover = console_lock_spinning_disable_and_check(cookie); printk_safe_exit_irqrestore(flags);
skip:
return true;
}
#else
static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
{
*handover = false;
return false;
}
#endif /* CONFIG_PRINTK */
/*
* Print out all remaining records to all consoles.
*
* @do_cond_resched is set by the caller. It can be true only in schedulable
* context.
*
* @next_seq is set to the sequence number after the last available record.
* The value is valid only when this function returns true. It means that all
* usable consoles are completely flushed.
*
* @handover will be set to true if a printk waiter has taken over the
* console_lock, in which case the caller is no longer holding the
* console_lock. Otherwise it is set to false.
*
* Returns true when there was at least one usable console and all messages
* were flushed to all usable consoles. A returned false informs the caller
* that everything was not flushed (either there were no usable consoles or
* another context has taken over printing or it is a panic situation and this
* is not the panic CPU). Regardless the reason, the caller should assume it
* is not useful to immediately try again.
*
* Requires the console_lock.
*/
static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover)
{
bool any_usable = false;
struct console *con;
bool any_progress;
int cookie;
*next_seq = 0;
*handover = false;
do {
any_progress = false;
cookie = console_srcu_read_lock(); for_each_console_srcu(con) {
bool progress;
if (!console_is_usable(con))
continue;
any_usable = true;
progress = console_emit_next_record(con, handover, cookie);
/*
* If a handover has occurred, the SRCU read lock
* is already released.
*/
if (*handover)
return false;
/* Track the next of the highest seq flushed. */
if (con->seq > *next_seq) *next_seq = con->seq;
if (!progress)
continue;
any_progress = true;
/* Allow panic_cpu to take over the consoles safely. */
if (other_cpu_in_panic())
goto abandon;
if (do_cond_resched) cond_resched();
}
console_srcu_read_unlock(cookie); } while (any_progress);
return any_usable;
abandon:
console_srcu_read_unlock(cookie);
return false;
}
/**
* console_unlock - unblock the console subsystem from printing
*
* Releases the console_lock which the caller holds to block printing of
* the console subsystem.
*
* While the console_lock was held, console output may have been buffered
* by printk(). If this is the case, console_unlock(); emits
* the output prior to releasing the lock.
*
* console_unlock(); may be called from any context.
*/
void console_unlock(void)
{
bool do_cond_resched;
bool handover;
bool flushed;
u64 next_seq;
/*
* Console drivers are called with interrupts disabled, so
* @console_may_schedule should be cleared before; however, we may
* end up dumping a lot of lines, for example, if called from
* console registration path, and should invoke cond_resched()
* between lines if allowable. Not doing so can cause a very long
* scheduling stall on a slow console leading to RCU stall and
* softlockup warnings which exacerbate the issue with more
* messages practically incapacitating the system. Therefore, create
* a local to use for the printing loop.
*/
do_cond_resched = console_may_schedule;
do {
console_may_schedule = 0;
flushed = console_flush_all(do_cond_resched, &next_seq, &handover);
if (!handover)
__console_unlock();
/*
* Abort if there was a failure to flush all messages to all
* usable consoles. Either it is not possible to flush (in
* which case it would be an infinite loop of retrying) or
* another context has taken over printing.
*/
if (!flushed)
break;
/*
* Some context may have added new records after
* console_flush_all() but before unlocking the console.
* Re-check if there is a new record to flush. If the trylock
* fails, another context is already handling the printing.
*/
} while (prb_read_valid(prb, next_seq, NULL) && console_trylock());
}
EXPORT_SYMBOL(console_unlock);
/**
* console_conditional_schedule - yield the CPU if required
*
* If the console code is currently allowed to sleep, and
* if this CPU should yield the CPU to another task, do
* so here.
*
* Must be called within console_lock();.
*/
void __sched console_conditional_schedule(void)
{
if (console_may_schedule)
cond_resched();
}
EXPORT_SYMBOL(console_conditional_schedule);
void console_unblank(void)
{
bool found_unblank = false;
struct console *c;
int cookie;
/*
* First check if there are any consoles implementing the unblank()
* callback. If not, there is no reason to continue and take the
* console lock, which in particular can be dangerous if
* @oops_in_progress is set.
*/
cookie = console_srcu_read_lock();
for_each_console_srcu(c) {
if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank) {
found_unblank = true;
break;
}
}
console_srcu_read_unlock(cookie);
if (!found_unblank)
return;
/*
* Stop console printing because the unblank() callback may
* assume the console is not within its write() callback.
*
* If @oops_in_progress is set, this may be an atomic context.
* In that case, attempt a trylock as best-effort.
*/
if (oops_in_progress) {
/* Semaphores are not NMI-safe. */
if (in_nmi())
return;
/*
* Attempting to trylock the console lock can deadlock
* if another CPU was stopped while modifying the
* semaphore. "Hope and pray" that this is not the
* current situation.
*/
if (down_trylock_console_sem() != 0)
return;
} else
console_lock();
console_locked = 1;
console_may_schedule = 0;
cookie = console_srcu_read_lock();
for_each_console_srcu(c) {
if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank)
c->unblank();
}
console_srcu_read_unlock(cookie);
console_unlock();
if (!oops_in_progress)
pr_flush(1000, true);
}
/**
* console_flush_on_panic - flush console content on panic
* @mode: flush all messages in buffer or just the pending ones
*
* Immediately output all pending messages no matter what.
*/
void console_flush_on_panic(enum con_flush_mode mode)
{
bool handover;
u64 next_seq;
/*
* Ignore the console lock and flush out the messages. Attempting a
* trylock would not be useful because:
*
* - if it is contended, it must be ignored anyway
* - console_lock() and console_trylock() block and fail
* respectively in panic for non-panic CPUs
* - semaphores are not NMI-safe
*/
/*
* If another context is holding the console lock,
* @console_may_schedule might be set. Clear it so that
* this context does not call cond_resched() while flushing.
*/
console_may_schedule = 0;
if (mode == CONSOLE_REPLAY_ALL) {
struct console *c;
short flags;
int cookie;
u64 seq;
seq = prb_first_valid_seq(prb);
cookie = console_srcu_read_lock();
for_each_console_srcu(c) {
flags = console_srcu_read_flags(c);
if (flags & CON_NBCON) {
nbcon_seq_force(c, seq);
} else {
/*
* This is an unsynchronized assignment. On
* panic legacy consoles are only best effort.
*/
c->seq = seq;
}
}
console_srcu_read_unlock(cookie);
}
console_flush_all(false, &next_seq, &handover);
}
/*
* Return the console tty driver structure and its associated index
*/
struct tty_driver *console_device(int *index)
{
struct console *c;
struct tty_driver *driver = NULL;
int cookie;
/*
* Take console_lock to serialize device() callback with
* other console operations. For example, fg_console is
* modified under console_lock when switching vt.
*/
console_lock();
cookie = console_srcu_read_lock();
for_each_console_srcu(c) {
if (!c->device)
continue;
driver = c->device(c, index);
if (driver)
break;
}
console_srcu_read_unlock(cookie);
console_unlock();
return driver;
}
/*
* Prevent further output on the passed console device so that (for example)
* serial drivers can disable console output before suspending a port, and can
* re-enable output afterwards.
*/
void console_stop(struct console *console)
{
__pr_flush(console, 1000, true);
console_list_lock();
console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
console_list_unlock();
/*
* Ensure that all SRCU list walks have completed. All contexts must
* be able to see that this console is disabled so that (for example)
* the caller can suspend the port without risk of another context
* using the port.
*/
synchronize_srcu(&console_srcu);
}
EXPORT_SYMBOL(console_stop);
void console_start(struct console *console)
{
console_list_lock();
console_srcu_write_flags(console, console->flags | CON_ENABLED);
console_list_unlock();
__pr_flush(console, 1000, true);
}
EXPORT_SYMBOL(console_start);
static int __read_mostly keep_bootcon;
static int __init keep_bootcon_setup(char *str)
{
keep_bootcon = 1;
pr_info("debug: skip boot console de-registration.\n");
return 0;
}
early_param("keep_bootcon", keep_bootcon_setup);
static int console_call_setup(struct console *newcon, char *options)
{
int err;
if (!newcon->setup)
return 0;
/* Synchronize with possible boot console. */
console_lock();
err = newcon->setup(newcon, options);
console_unlock();
return err;
}
/*
* This is called by register_console() to try to match
* the newly registered console with any of the ones selected
* by either the command line or add_preferred_console() and
* setup/enable it.
*
* Care need to be taken with consoles that are statically
* enabled such as netconsole
*/
static int try_enable_preferred_console(struct console *newcon,
bool user_specified)
{
struct console_cmdline *c;
int i, err;
for (i = 0, c = console_cmdline;
i < MAX_CMDLINECONSOLES && c->name[0];
i++, c++) {
if (c->user_specified != user_specified)
continue;
if (!newcon->match ||
newcon->match(newcon, c->name, c->index, c->options) != 0) {
/* default matching */
BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
if (strcmp(c->name, newcon->name) != 0)
continue;
if (newcon->index >= 0 &&
newcon->index != c->index)
continue;
if (newcon->index < 0)
newcon->index = c->index;
if (_braille_register_console(newcon, c))
return 0;
err = console_call_setup(newcon, c->options);
if (err)
return err;
}
newcon->flags |= CON_ENABLED;
if (i == preferred_console)
newcon->flags |= CON_CONSDEV;
return 0;
}
/*
* Some consoles, such as pstore and netconsole, can be enabled even
* without matching. Accept the pre-enabled consoles only when match()
* and setup() had a chance to be called.
*/
if (newcon->flags & CON_ENABLED && c->user_specified == user_specified)
return 0;
return -ENOENT;
}
/* Try to enable the console unconditionally */
static void try_enable_default_console(struct console *newcon)
{
if (newcon->index < 0)
newcon->index = 0;
if (console_call_setup(newcon, NULL) != 0)
return;
newcon->flags |= CON_ENABLED;
if (newcon->device)
newcon->flags |= CON_CONSDEV;
}
static void console_init_seq(struct console *newcon, bool bootcon_registered)
{
struct console *con;
bool handover;
if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) {
/* Get a consistent copy of @syslog_seq. */
mutex_lock(&syslog_lock);
newcon->seq = syslog_seq;
mutex_unlock(&syslog_lock);
} else {
/* Begin with next message added to ringbuffer. */
newcon->seq = prb_next_seq(prb);
/*
* If any enabled boot consoles are due to be unregistered
* shortly, some may not be caught up and may be the same
* device as @newcon. Since it is not known which boot console
* is the same device, flush all consoles and, if necessary,
* start with the message of the enabled boot console that is
* the furthest behind.
*/
if (bootcon_registered && !keep_bootcon) {
/*
* Hold the console_lock to stop console printing and
* guarantee safe access to console->seq.
*/
console_lock();
/*
* Flush all consoles and set the console to start at
* the next unprinted sequence number.
*/
if (!console_flush_all(true, &newcon->seq, &handover)) {
/*
* Flushing failed. Just choose the lowest
* sequence of the enabled boot consoles.
*/
/*
* If there was a handover, this context no
* longer holds the console_lock.
*/
if (handover)
console_lock();
newcon->seq = prb_next_seq(prb);
for_each_console(con) {
if ((con->flags & CON_BOOT) &&
(con->flags & CON_ENABLED) &&
con->seq < newcon->seq) {
newcon->seq = con->seq;
}
}
}
console_unlock();
}
}
}
#define console_first() \
hlist_entry(console_list.first, struct console, node)
static int unregister_console_locked(struct console *console);
/*
* The console driver calls this routine during kernel initialization
* to register the console printing procedure with printk() and to
* print any messages that were printed by the kernel before the
* console driver was initialized.
*
* This can happen pretty early during the boot process (because of
* early_printk) - sometimes before setup_arch() completes - be careful
* of what kernel features are used - they may not be initialised yet.
*
* There are two types of consoles - bootconsoles (early_printk) and
* "real" consoles (everything which is not a bootconsole) which are
* handled differently.
* - Any number of bootconsoles can be registered at any time.
* - As soon as a "real" console is registered, all bootconsoles
* will be unregistered automatically.
* - Once a "real" console is registered, any attempt to register a
* bootconsoles will be rejected
*/
void register_console(struct console *newcon)
{
struct console *con;
bool bootcon_registered = false;
bool realcon_registered = false;
int err;
console_list_lock();
for_each_console(con) {
if (WARN(con == newcon, "console '%s%d' already registered\n",
con->name, con->index)) {
goto unlock;
}
if (con->flags & CON_BOOT)
bootcon_registered = true;
else
realcon_registered = true;
}
/* Do not register boot consoles when there already is a real one. */
if ((newcon->flags & CON_BOOT) && realcon_registered) {
pr_info("Too late to register bootconsole %s%d\n",
newcon->name, newcon->index);
goto unlock;
}
if (newcon->flags & CON_NBCON) {
/*
* Ensure the nbcon console buffers can be allocated
* before modifying any global data.
*/
if (!nbcon_alloc(newcon))
goto unlock;
}
/*
* See if we want to enable this console driver by default.
*
* Nope when a console is preferred by the command line, device
* tree, or SPCR.
*
* The first real console with tty binding (driver) wins. More
* consoles might get enabled before the right one is found.
*
* Note that a console with tty binding will have CON_CONSDEV
* flag set and will be first in the list.
*/
if (preferred_console < 0) {
if (hlist_empty(&console_list) || !console_first()->device ||
console_first()->flags & CON_BOOT) {
try_enable_default_console(newcon);
}
}
/* See if this console matches one we selected on the command line */
err = try_enable_preferred_console(newcon, true);
/* If not, try to match against the platform default(s) */
if (err == -ENOENT)
err = try_enable_preferred_console(newcon, false);
/* printk() messages are not printed to the Braille console. */
if (err || newcon->flags & CON_BRL) {
if (newcon->flags & CON_NBCON)
nbcon_free(newcon);
goto unlock;
}
/*
* If we have a bootconsole, and are switching to a real console,
* don't print everything out again, since when the boot console, and
* the real console are the same physical device, it's annoying to
* see the beginning boot messages twice
*/
if (bootcon_registered &&
((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
newcon->flags &= ~CON_PRINTBUFFER;
}
newcon->dropped = 0;
console_init_seq(newcon, bootcon_registered);
if (newcon->flags & CON_NBCON)
nbcon_init(newcon);
/*
* Put this console in the list - keep the
* preferred driver at the head of the list.
*/
if (hlist_empty(&console_list)) {
/* Ensure CON_CONSDEV is always set for the head. */
newcon->flags |= CON_CONSDEV;
hlist_add_head_rcu(&newcon->node, &console_list);
} else if (newcon->flags & CON_CONSDEV) {
/* Only the new head can have CON_CONSDEV set. */
console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV);
hlist_add_head_rcu(&newcon->node, &console_list);
} else {
hlist_add_behind_rcu(&newcon->node, console_list.first);
}
/*
* No need to synchronize SRCU here! The caller does not rely
* on all contexts being able to see the new console before
* register_console() completes.
*/
console_sysfs_notify();
/*
* By unregistering the bootconsoles after we enable the real console
* we get the "console xxx enabled" message on all the consoles -
* boot consoles, real consoles, etc - this is to ensure that end
* users know there might be something in the kernel's log buffer that
* went to the bootconsole (that they do not see on the real console)
*/
con_printk(KERN_INFO, newcon, "enabled\n");
if (bootcon_registered &&
((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) &&
!keep_bootcon) {
struct hlist_node *tmp;
hlist_for_each_entry_safe(con, tmp, &console_list, node) {
if (con->flags & CON_BOOT)
unregister_console_locked(con);
}
}
unlock:
console_list_unlock();
}
EXPORT_SYMBOL(register_console);
/* Must be called under console_list_lock(). */
static int unregister_console_locked(struct console *console)
{
int res;
lockdep_assert_console_list_lock_held();
con_printk(KERN_INFO, console, "disabled\n");
res = _braille_unregister_console(console);
if (res < 0)
return res;
if (res > 0)
return 0;
/* Disable it unconditionally */
console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
if (!console_is_registered_locked(console))
return -ENODEV;
hlist_del_init_rcu(&console->node);
/*
* <HISTORICAL>
* If this isn't the last console and it has CON_CONSDEV set, we
* need to set it on the next preferred console.
* </HISTORICAL>
*
* The above makes no sense as there is no guarantee that the next
* console has any device attached. Oh well....
*/
if (!hlist_empty(&console_list) && console->flags & CON_CONSDEV)
console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV);
/*
* Ensure that all SRCU list walks have completed. All contexts
* must not be able to see this console in the list so that any
* exit/cleanup routines can be performed safely.
*/
synchronize_srcu(&console_srcu);
if (console->flags & CON_NBCON)
nbcon_free(console);
console_sysfs_notify();
if (console->exit)
res = console->exit(console);
return res;
}
int unregister_console(struct console *console)
{
int res;
console_list_lock();
res = unregister_console_locked(console);
console_list_unlock();
return res;
}
EXPORT_SYMBOL(unregister_console);
/**
* console_force_preferred_locked - force a registered console preferred
* @con: The registered console to force preferred.
*
* Must be called under console_list_lock().
*/
void console_force_preferred_locked(struct console *con)
{
struct console *cur_pref_con;
if (!console_is_registered_locked(con))
return;
cur_pref_con = console_first();
/* Already preferred? */
if (cur_pref_con == con)
return;
/*
* Delete, but do not re-initialize the entry. This allows the console
* to continue to appear registered (via any hlist_unhashed_lockless()
* checks), even though it was briefly removed from the console list.
*/
hlist_del_rcu(&con->node);
/*
* Ensure that all SRCU list walks have completed so that the console
* can be added to the beginning of the console list and its forward
* list pointer can be re-initialized.
*/
synchronize_srcu(&console_srcu);
con->flags |= CON_CONSDEV;
WARN_ON(!con->device);
/* Only the new head can have CON_CONSDEV set. */
console_srcu_write_flags(cur_pref_con, cur_pref_con->flags & ~CON_CONSDEV);
hlist_add_head_rcu(&con->node, &console_list);
}
EXPORT_SYMBOL(console_force_preferred_locked);
/*
* Initialize the console device. This is called *early*, so
* we can't necessarily depend on lots of kernel help here.
* Just do some early initializations, and do the complex setup
* later.
*/
void __init console_init(void)
{
int ret;
initcall_t call;
initcall_entry_t *ce;
/* Setup the default TTY line discipline. */
n_tty_init();
/*
* set up the console device so that later boot sequences can
* inform about problems etc..
*/
ce = __con_initcall_start;
trace_initcall_level("console");
while (ce < __con_initcall_end) {
call = initcall_from_entry(ce);
trace_initcall_start(call);
ret = call();
trace_initcall_finish(call, ret);
ce++;
}
}
/*
* Some boot consoles access data that is in the init section and which will
* be discarded after the initcalls have been run. To make sure that no code
* will access this data, unregister the boot consoles in a late initcall.
*
* If for some reason, such as deferred probe or the driver being a loadable
* module, the real console hasn't registered yet at this point, there will
* be a brief interval in which no messages are logged to the console, which
* makes it difficult to diagnose problems that occur during this time.
*
* To mitigate this problem somewhat, only unregister consoles whose memory
* intersects with the init section. Note that all other boot consoles will
* get unregistered when the real preferred console is registered.
*/
static int __init printk_late_init(void)
{
struct hlist_node *tmp;
struct console *con;
int ret;
console_list_lock();
hlist_for_each_entry_safe(con, tmp, &console_list, node) {
if (!(con->flags & CON_BOOT))
continue;
/* Check addresses that might be used for enabled consoles. */
if (init_section_intersects(con, sizeof(*con)) ||
init_section_contains(con->write, 0) ||
init_section_contains(con->read, 0) ||
init_section_contains(con->device, 0) ||
init_section_contains(con->unblank, 0) ||
init_section_contains(con->data, 0)) {
/*
* Please, consider moving the reported consoles out
* of the init section.
*/
pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
con->name, con->index);
unregister_console_locked(con);
}
}
console_list_unlock();
ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL,
console_cpu_notify);
WARN_ON(ret < 0);
ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online",
console_cpu_notify, NULL);
WARN_ON(ret < 0);
printk_sysctl_init();
return 0;
}
late_initcall(printk_late_init);
#if defined CONFIG_PRINTK
/* If @con is specified, only wait for that console. Otherwise wait for all. */
static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress)
{
unsigned long timeout_jiffies = msecs_to_jiffies(timeout_ms);
unsigned long remaining_jiffies = timeout_jiffies;
struct console *c;
u64 last_diff = 0;
u64 printk_seq;
short flags;
int cookie;
u64 diff;
u64 seq;
might_sleep();
seq = prb_next_reserve_seq(prb);
/* Flush the consoles so that records up to @seq are printed. */
console_lock();
console_unlock();
for (;;) {
unsigned long begin_jiffies;
unsigned long slept_jiffies;
diff = 0;
/*
* Hold the console_lock to guarantee safe access to
* console->seq. Releasing console_lock flushes more
* records in case @seq is still not printed on all
* usable consoles.
*/
console_lock();
cookie = console_srcu_read_lock();
for_each_console_srcu(c) {
if (con && con != c)
continue;
flags = console_srcu_read_flags(c);
/*
* If consoles are not usable, it cannot be expected
* that they make forward progress, so only increment
* @diff for usable consoles.
*/
if (!console_is_usable(c))
continue;
if (flags & CON_NBCON) {
printk_seq = nbcon_seq_read(c);
} else {
printk_seq = c->seq;
}
if (printk_seq < seq)
diff += seq - printk_seq;
}
console_srcu_read_unlock(cookie);
if (diff != last_diff && reset_on_progress)
remaining_jiffies = timeout_jiffies;
console_unlock();
/* Note: @diff is 0 if there are no usable consoles. */
if (diff == 0 || remaining_jiffies == 0)
break;
/* msleep(1) might sleep much longer. Check time by jiffies. */
begin_jiffies = jiffies;
msleep(1);
slept_jiffies = jiffies - begin_jiffies;
remaining_jiffies -= min(slept_jiffies, remaining_jiffies);
last_diff = diff;
}
return (diff == 0);
}
/**
* pr_flush() - Wait for printing threads to catch up.
*
* @timeout_ms: The maximum time (in ms) to wait.
* @reset_on_progress: Reset the timeout if forward progress is seen.
*
* A value of 0 for @timeout_ms means no waiting will occur. A value of -1
* represents infinite waiting.
*
* If @reset_on_progress is true, the timeout will be reset whenever any
* printer has been seen to make some forward progress.
*
* Context: Process context. May sleep while acquiring console lock.
* Return: true if all usable printers are caught up.
*/
static bool pr_flush(int timeout_ms, bool reset_on_progress)
{
return __pr_flush(NULL, timeout_ms, reset_on_progress);
}
/*
* Delayed printk version, for scheduler-internal messages:
*/
#define PRINTK_PENDING_WAKEUP 0x01
#define PRINTK_PENDING_OUTPUT 0x02
static DEFINE_PER_CPU(int, printk_pending);
static void wake_up_klogd_work_func(struct irq_work *irq_work)
{
int pending = this_cpu_xchg(printk_pending, 0);
if (pending & PRINTK_PENDING_OUTPUT) {
/* If trylock fails, someone else is doing the printing */
if (console_trylock())
console_unlock();
}
if (pending & PRINTK_PENDING_WAKEUP)
wake_up_interruptible(&log_wait);
}
static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
static void __wake_up_klogd(int val)
{
if (!printk_percpu_data_ready())
return;
preempt_disable();
/*
* Guarantee any new records can be seen by tasks preparing to wait
* before this context checks if the wait queue is empty.
*
* The full memory barrier within wq_has_sleeper() pairs with the full
* memory barrier within set_current_state() of
* prepare_to_wait_event(), which is called after ___wait_event() adds
* the waiter but before it has checked the wait condition.
*
* This pairs with devkmsg_read:A and syslog_print:A.
*/
if (wq_has_sleeper(&log_wait) || /* LMM(__wake_up_klogd:A) */
(val & PRINTK_PENDING_OUTPUT)) { this_cpu_or(printk_pending, val);
irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
}
preempt_enable();
}
/**
* wake_up_klogd - Wake kernel logging daemon
*
* Use this function when new records have been added to the ringbuffer
* and the console printing of those records has already occurred or is
* known to be handled by some other context. This function will only
* wake the logging daemon.
*
* Context: Any context.
*/
void wake_up_klogd(void)
{
__wake_up_klogd(PRINTK_PENDING_WAKEUP);
}
/**
* defer_console_output - Wake kernel logging daemon and trigger
* console printing in a deferred context
*
* Use this function when new records have been added to the ringbuffer,
* this context is responsible for console printing those records, but
* the current context is not allowed to perform the console printing.
* Trigger an irq_work context to perform the console printing. This
* function also wakes the logging daemon.
*
* Context: Any context.
*/
void defer_console_output(void)
{
/*
* New messages may have been added directly to the ringbuffer
* using vprintk_store(), so wake any waiters as well.
*/
__wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
}
void printk_trigger_flush(void)
{
defer_console_output();
}
int vprintk_deferred(const char *fmt, va_list args)
{
return vprintk_emit(0, LOGLEVEL_SCHED, NULL, fmt, args);
}
int _printk_deferred(const char *fmt, ...)
{
va_list args;
int r;
va_start(args, fmt);
r = vprintk_deferred(fmt, args);
va_end(args);
return r;
}
/*
* printk rate limiting, lifted from the networking subsystem.
*
* This enforces a rate limit: not more than 10 kernel messages
* every 5s to make a denial-of-service attack impossible.
*/
DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);
int __printk_ratelimit(const char *func)
{
return ___ratelimit(&printk_ratelimit_state, func);
}
EXPORT_SYMBOL(__printk_ratelimit);
/**
* printk_timed_ratelimit - caller-controlled printk ratelimiting
* @caller_jiffies: pointer to caller's state
* @interval_msecs: minimum interval between prints
*
* printk_timed_ratelimit() returns true if more than @interval_msecs
* milliseconds have elapsed since the last time printk_timed_ratelimit()
* returned true.
*/
bool printk_timed_ratelimit(unsigned long *caller_jiffies,
unsigned int interval_msecs)
{
unsigned long elapsed = jiffies - *caller_jiffies;
if (*caller_jiffies && elapsed <= msecs_to_jiffies(interval_msecs))
return false;
*caller_jiffies = jiffies;
return true;
}
EXPORT_SYMBOL(printk_timed_ratelimit);
static DEFINE_SPINLOCK(dump_list_lock);
static LIST_HEAD(dump_list);
/**
* kmsg_dump_register - register a kernel log dumper.
* @dumper: pointer to the kmsg_dumper structure
*
* Adds a kernel log dumper to the system. The dump callback in the
* structure will be called when the kernel oopses or panics and must be
* set. Returns zero on success and %-EINVAL or %-EBUSY otherwise.
*/
int kmsg_dump_register(struct kmsg_dumper *dumper)
{
unsigned long flags;
int err = -EBUSY;
/* The dump callback needs to be set */
if (!dumper->dump)
return -EINVAL;
spin_lock_irqsave(&dump_list_lock, flags);
/* Don't allow registering multiple times */
if (!dumper->registered) {
dumper->registered = 1;
list_add_tail_rcu(&dumper->list, &dump_list);
err = 0;
}
spin_unlock_irqrestore(&dump_list_lock, flags);
return err;
}
EXPORT_SYMBOL_GPL(kmsg_dump_register);
/**
* kmsg_dump_unregister - unregister a kmsg dumper.
* @dumper: pointer to the kmsg_dumper structure
*
* Removes a dump device from the system. Returns zero on success and
* %-EINVAL otherwise.
*/
int kmsg_dump_unregister(struct kmsg_dumper *dumper)
{
unsigned long flags;
int err = -EINVAL;
spin_lock_irqsave(&dump_list_lock, flags);
if (dumper->registered) {
dumper->registered = 0;
list_del_rcu(&dumper->list);
err = 0;
}
spin_unlock_irqrestore(&dump_list_lock, flags);
synchronize_rcu();
return err;
}
EXPORT_SYMBOL_GPL(kmsg_dump_unregister);
static bool always_kmsg_dump;
module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR);
const char *kmsg_dump_reason_str(enum kmsg_dump_reason reason)
{
switch (reason) {
case KMSG_DUMP_PANIC:
return "Panic";
case KMSG_DUMP_OOPS:
return "Oops";
case KMSG_DUMP_EMERG:
return "Emergency";
case KMSG_DUMP_SHUTDOWN:
return "Shutdown";
default:
return "Unknown";
}
}
EXPORT_SYMBOL_GPL(kmsg_dump_reason_str);
/**
* kmsg_dump - dump kernel log to kernel message dumpers.
* @reason: the reason (oops, panic etc) for dumping
*
* Call each of the registered dumper's dump() callback, which can
* retrieve the kmsg records with kmsg_dump_get_line() or
* kmsg_dump_get_buffer().
*/
void kmsg_dump(enum kmsg_dump_reason reason)
{
struct kmsg_dumper *dumper;
rcu_read_lock();
list_for_each_entry_rcu(dumper, &dump_list, list) {
enum kmsg_dump_reason max_reason = dumper->max_reason;
/*
* If client has not provided a specific max_reason, default
* to KMSG_DUMP_OOPS, unless always_kmsg_dump was set.
*/
if (max_reason == KMSG_DUMP_UNDEF) {
max_reason = always_kmsg_dump ? KMSG_DUMP_MAX :
KMSG_DUMP_OOPS;
}
if (reason > max_reason)
continue;
/* invoke dumper which will iterate over records */
dumper->dump(dumper, reason);
}
rcu_read_unlock();
}
/**
* kmsg_dump_get_line - retrieve one kmsg log line
* @iter: kmsg dump iterator
* @syslog: include the "<4>" prefixes
* @line: buffer to copy the line to
* @size: maximum size of the buffer
* @len: length of line placed into buffer
*
* Start at the beginning of the kmsg buffer, with the oldest kmsg
* record, and copy one record into the provided buffer.
*
* Consecutive calls will return the next available record moving
* towards the end of the buffer with the youngest messages.
*
* A return value of FALSE indicates that there are no more records to
* read.
*/
bool kmsg_dump_get_line(struct kmsg_dump_iter *iter, bool syslog,
char *line, size_t size, size_t *len)
{
u64 min_seq = latched_seq_read_nolock(&clear_seq);
struct printk_info info;
unsigned int line_count;
struct printk_record r;
size_t l = 0;
bool ret = false;
if (iter->cur_seq < min_seq)
iter->cur_seq = min_seq;
prb_rec_init_rd(&r, &info, line, size);
/* Read text or count text lines? */
if (line) {
if (!prb_read_valid(prb, iter->cur_seq, &r))
goto out;
l = record_print_text(&r, syslog, printk_time);
} else {
if (!prb_read_valid_info(prb, iter->cur_seq,
&info, &line_count)) {
goto out;
}
l = get_record_print_text_size(&info, line_count, syslog,
printk_time);
}
iter->cur_seq = r.info->seq + 1;
ret = true;
out:
if (len)
*len = l;
return ret;
}
EXPORT_SYMBOL_GPL(kmsg_dump_get_line);
/**
* kmsg_dump_get_buffer - copy kmsg log lines
* @iter: kmsg dump iterator
* @syslog: include the "<4>" prefixes
* @buf: buffer to copy the line to
* @size: maximum size of the buffer
* @len_out: length of line placed into buffer
*
* Start at the end of the kmsg buffer and fill the provided buffer
* with as many of the *youngest* kmsg records that fit into it.
* If the buffer is large enough, all available kmsg records will be
* copied with a single call.
*
* Consecutive calls will fill the buffer with the next block of
* available older records, not including the earlier retrieved ones.
*
* A return value of FALSE indicates that there are no more records to
* read.
*/
bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog,
char *buf, size_t size, size_t *len_out)
{
u64 min_seq = latched_seq_read_nolock(&clear_seq);
struct printk_info info;
struct printk_record r;
u64 seq;
u64 next_seq;
size_t len = 0;
bool ret = false;
bool time = printk_time;
if (!buf || !size)
goto out;
if (iter->cur_seq < min_seq)
iter->cur_seq = min_seq;
if (prb_read_valid_info(prb, iter->cur_seq, &info, NULL)) {
if (info.seq != iter->cur_seq) {
/* messages are gone, move to first available one */
iter->cur_seq = info.seq;
}
}
/* last entry */
if (iter->cur_seq >= iter->next_seq)
goto out;
/*
* Find first record that fits, including all following records,
* into the user-provided buffer for this dump. Pass in size-1
* because this function (by way of record_print_text()) will
* not write more than size-1 bytes of text into @buf.
*/
seq = find_first_fitting_seq(iter->cur_seq, iter->next_seq,
size - 1, syslog, time);
/*
* Next kmsg_dump_get_buffer() invocation will dump block of
* older records stored right before this one.
*/
next_seq = seq;
prb_rec_init_rd(&r, &info, buf, size);
prb_for_each_record(seq, prb, seq, &r) {
if (r.info->seq >= iter->next_seq)
break;
len += record_print_text(&r, syslog, time);
/* Adjust record to store to remaining buffer space. */
prb_rec_init_rd(&r, &info, buf + len, size - len);
}
iter->next_seq = next_seq;
ret = true;
out:
if (len_out)
*len_out = len;
return ret;
}
EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer);
/**
* kmsg_dump_rewind - reset the iterator
* @iter: kmsg dump iterator
*
* Reset the dumper's iterator so that kmsg_dump_get_line() and
* kmsg_dump_get_buffer() can be called again and used multiple
* times within the same dumper.dump() callback.
*/
void kmsg_dump_rewind(struct kmsg_dump_iter *iter)
{
iter->cur_seq = latched_seq_read_nolock(&clear_seq);
iter->next_seq = prb_next_seq(prb);
}
EXPORT_SYMBOL_GPL(kmsg_dump_rewind);
#endif
#ifdef CONFIG_SMP
static atomic_t printk_cpu_sync_owner = ATOMIC_INIT(-1);
static atomic_t printk_cpu_sync_nested = ATOMIC_INIT(0);
/**
* __printk_cpu_sync_wait() - Busy wait until the printk cpu-reentrant
* spinning lock is not owned by any CPU.
*
* Context: Any context.
*/
void __printk_cpu_sync_wait(void)
{
do {
cpu_relax();
} while (atomic_read(&printk_cpu_sync_owner) != -1);
}
EXPORT_SYMBOL(__printk_cpu_sync_wait);
/**
* __printk_cpu_sync_try_get() - Try to acquire the printk cpu-reentrant
* spinning lock.
*
* If no processor has the lock, the calling processor takes the lock and
* becomes the owner. If the calling processor is already the owner of the
* lock, this function succeeds immediately.
*
* Context: Any context. Expects interrupts to be disabled.
* Return: 1 on success, otherwise 0.
*/
int __printk_cpu_sync_try_get(void)
{
int cpu;
int old;
cpu = smp_processor_id();
/*
* Guarantee loads and stores from this CPU when it is the lock owner
* are _not_ visible to the previous lock owner. This pairs with
* __printk_cpu_sync_put:B.
*
* Memory barrier involvement:
*
* If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
* then __printk_cpu_sync_put:A can never read from
* __printk_cpu_sync_try_get:B.
*
* Relies on:
*
* RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
* of the previous CPU
* matching
* ACQUIRE from __printk_cpu_sync_try_get:A to
* __printk_cpu_sync_try_get:B of this CPU
*/
old = atomic_cmpxchg_acquire(&printk_cpu_sync_owner, -1,
cpu); /* LMM(__printk_cpu_sync_try_get:A) */
if (old == -1) {
/*
* This CPU is now the owner and begins loading/storing
* data: LMM(__printk_cpu_sync_try_get:B)
*/
return 1;
} else if (old == cpu) {
/* This CPU is already the owner. */
atomic_inc(&printk_cpu_sync_nested);
return 1;
}
return 0;
}
EXPORT_SYMBOL(__printk_cpu_sync_try_get);
/**
* __printk_cpu_sync_put() - Release the printk cpu-reentrant spinning lock.
*
* The calling processor must be the owner of the lock.
*
* Context: Any context. Expects interrupts to be disabled.
*/
void __printk_cpu_sync_put(void)
{
if (atomic_read(&printk_cpu_sync_nested)) {
atomic_dec(&printk_cpu_sync_nested);
return;
}
/*
* This CPU is finished loading/storing data:
* LMM(__printk_cpu_sync_put:A)
*/
/*
* Guarantee loads and stores from this CPU when it was the
* lock owner are visible to the next lock owner. This pairs
* with __printk_cpu_sync_try_get:A.
*
* Memory barrier involvement:
*
* If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
* then __printk_cpu_sync_try_get:B reads from __printk_cpu_sync_put:A.
*
* Relies on:
*
* RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
* of this CPU
* matching
* ACQUIRE from __printk_cpu_sync_try_get:A to
* __printk_cpu_sync_try_get:B of the next CPU
*/
atomic_set_release(&printk_cpu_sync_owner,
-1); /* LMM(__printk_cpu_sync_put:B) */
}
EXPORT_SYMBOL(__printk_cpu_sync_put);
#endif /* CONFIG_SMP */
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
#ifndef __LINUX_OVERFLOW_H
#define __LINUX_OVERFLOW_H
#include <linux/compiler.h>
#include <linux/limits.h>
#include <linux/const.h>
/*
* We need to compute the minimum and maximum values representable in a given
* type. These macros may also be useful elsewhere. It would seem more obvious
* to do something like:
*
* #define type_min(T) (T)(is_signed_type(T) ? (T)1 << (8*sizeof(T)-1) : 0)
* #define type_max(T) (T)(is_signed_type(T) ? ((T)1 << (8*sizeof(T)-1)) - 1 : ~(T)0)
*
* Unfortunately, the middle expressions, strictly speaking, have
* undefined behaviour, and at least some versions of gcc warn about
* the type_max expression (but not if -fsanitize=undefined is in
* effect; in that case, the warning is deferred to runtime...).
*
* The slightly excessive casting in type_min is to make sure the
* macros also produce sensible values for the exotic type _Bool. [The
* overflow checkers only almost work for _Bool, but that's
* a-feature-not-a-bug, since people shouldn't be doing arithmetic on
* _Bools. Besides, the gcc builtins don't allow _Bool* as third
* argument.]
*
* Idea stolen from
* https://mail-index.netbsd.org/tech-misc/2007/02/05/0000.html -
* credit to Christian Biere.
*/
#define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type)))
#define __type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T)))
#define type_max(t) __type_max(typeof(t))
#define __type_min(T) ((T)((T)-type_max(T)-(T)1))
#define type_min(t) __type_min(typeof(t))
/*
* Avoids triggering -Wtype-limits compilation warning,
* while using unsigned data types to check a < 0.
*/
#define is_non_negative(a) ((a) > 0 || (a) == 0)
#define is_negative(a) (!(is_non_negative(a)))
/*
* Allows for effectively applying __must_check to a macro so we can have
* both the type-agnostic benefits of the macros while also being able to
* enforce that the return value is, in fact, checked.
*/
static inline bool __must_check __must_check_overflow(bool overflow)
{
return unlikely(overflow);
}
/**
* check_add_overflow() - Calculate addition with overflow checking
* @a: first addend
* @b: second addend
* @d: pointer to store sum
*
* Returns true on wrap-around, false otherwise.
*
* *@d holds the results of the attempted addition, regardless of whether
* wrap-around occurred.
*/
#define check_add_overflow(a, b, d) \
__must_check_overflow(__builtin_add_overflow(a, b, d))
/**
* wrapping_add() - Intentionally perform a wrapping addition
* @type: type for result of calculation
* @a: first addend
* @b: second addend
*
* Return the potentially wrapped-around addition without
* tripping any wrap-around sanitizers that may be enabled.
*/
#define wrapping_add(type, a, b) \
({ \
type __val; \
__builtin_add_overflow(a, b, &__val); \
__val; \
})
/**
* wrapping_assign_add() - Intentionally perform a wrapping increment assignment
* @var: variable to be incremented
* @offset: amount to add
*
* Increments @var by @offset with wrap-around. Returns the resulting
* value of @var. Will not trip any wrap-around sanitizers.
*
* Returns the new value of @var.
*/
#define wrapping_assign_add(var, offset) \
({ \
typeof(var) *__ptr = &(var); \
*__ptr = wrapping_add(typeof(var), *__ptr, offset); \
})
/**
* check_sub_overflow() - Calculate subtraction with overflow checking
* @a: minuend; value to subtract from
* @b: subtrahend; value to subtract from @a
* @d: pointer to store difference
*
* Returns true on wrap-around, false otherwise.
*
* *@d holds the results of the attempted subtraction, regardless of whether
* wrap-around occurred.
*/
#define check_sub_overflow(a, b, d) \
__must_check_overflow(__builtin_sub_overflow(a, b, d))
/**
* wrapping_sub() - Intentionally perform a wrapping subtraction
* @type: type for result of calculation
* @a: minuend; value to subtract from
* @b: subtrahend; value to subtract from @a
*
* Return the potentially wrapped-around subtraction without
* tripping any wrap-around sanitizers that may be enabled.
*/
#define wrapping_sub(type, a, b) \
({ \
type __val; \
__builtin_sub_overflow(a, b, &__val); \
__val; \
})
/**
* wrapping_assign_sub() - Intentionally perform a wrapping decrement assign
* @var: variable to be decremented
* @offset: amount to subtract
*
* Decrements @var by @offset with wrap-around. Returns the resulting
* value of @var. Will not trip any wrap-around sanitizers.
*
* Returns the new value of @var.
*/
#define wrapping_assign_sub(var, offset) \
({ \
typeof(var) *__ptr = &(var); \
*__ptr = wrapping_sub(typeof(var), *__ptr, offset); \
})
/**
* check_mul_overflow() - Calculate multiplication with overflow checking
* @a: first factor
* @b: second factor
* @d: pointer to store product
*
* Returns true on wrap-around, false otherwise.
*
* *@d holds the results of the attempted multiplication, regardless of whether
* wrap-around occurred.
*/
#define check_mul_overflow(a, b, d) \
__must_check_overflow(__builtin_mul_overflow(a, b, d))
/**
* wrapping_mul() - Intentionally perform a wrapping multiplication
* @type: type for result of calculation
* @a: first factor
* @b: second factor
*
* Return the potentially wrapped-around multiplication without
* tripping any wrap-around sanitizers that may be enabled.
*/
#define wrapping_mul(type, a, b) \
({ \
type __val; \
__builtin_mul_overflow(a, b, &__val); \
__val; \
})
/**
* check_shl_overflow() - Calculate a left-shifted value and check overflow
* @a: Value to be shifted
* @s: How many bits left to shift
* @d: Pointer to where to store the result
*
* Computes *@d = (@a << @s)
*
* Returns true if '*@d' cannot hold the result or when '@a << @s' doesn't
* make sense. Example conditions:
*
* - '@a << @s' causes bits to be lost when stored in *@d.
* - '@s' is garbage (e.g. negative) or so large that the result of
* '@a << @s' is guaranteed to be 0.
* - '@a' is negative.
* - '@a << @s' sets the sign bit, if any, in '*@d'.
*
* '*@d' will hold the results of the attempted shift, but is not
* considered "safe for use" if true is returned.
*/
#define check_shl_overflow(a, s, d) __must_check_overflow(({ \
typeof(a) _a = a; \
typeof(s) _s = s; \
typeof(d) _d = d; \
unsigned long long _a_full = _a; \
unsigned int _to_shift = \
is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0; \
*_d = (_a_full << _to_shift); \
(_to_shift != _s || is_negative(*_d) || is_negative(_a) || \
(*_d >> _to_shift) != _a); \
}))
#define __overflows_type_constexpr(x, T) ( \
is_unsigned_type(typeof(x)) ? \
(x) > type_max(T) : \
is_unsigned_type(typeof(T)) ? \
(x) < 0 || (x) > type_max(T) : \
(x) < type_min(T) || (x) > type_max(T))
#define __overflows_type(x, T) ({ \
typeof(T) v = 0; \
check_add_overflow((x), v, &v); \
})
/**
* overflows_type - helper for checking the overflows between value, variables,
* or data type
*
* @n: source constant value or variable to be checked
* @T: destination variable or data type proposed to store @x
*
* Compares the @x expression for whether or not it can safely fit in
* the storage of the type in @T. @x and @T can have different types.
* If @x is a constant expression, this will also resolve to a constant
* expression.
*
* Returns: true if overflow can occur, false otherwise.
*/
#define overflows_type(n, T) \
__builtin_choose_expr(__is_constexpr(n), \
__overflows_type_constexpr(n, T), \
__overflows_type(n, T))
/**
* castable_to_type - like __same_type(), but also allows for casted literals
*
* @n: variable or constant value
* @T: variable or data type
*
* Unlike the __same_type() macro, this allows a constant value as the
* first argument. If this value would not overflow into an assignment
* of the second argument's type, it returns true. Otherwise, this falls
* back to __same_type().
*/
#define castable_to_type(n, T) \
__builtin_choose_expr(__is_constexpr(n), \
!__overflows_type_constexpr(n, T), \
__same_type(n, T))
/**
* size_mul() - Calculate size_t multiplication with saturation at SIZE_MAX
* @factor1: first factor
* @factor2: second factor
*
* Returns: calculate @factor1 * @factor2, both promoted to size_t,
* with any overflow causing the return value to be SIZE_MAX. The
* lvalue must be size_t to avoid implicit type conversion.
*/
static inline size_t __must_check size_mul(size_t factor1, size_t factor2)
{
size_t bytes;
if (check_mul_overflow(factor1, factor2, &bytes))
return SIZE_MAX;
return bytes;
}
/**
* size_add() - Calculate size_t addition with saturation at SIZE_MAX
* @addend1: first addend
* @addend2: second addend
*
* Returns: calculate @addend1 + @addend2, both promoted to size_t,
* with any overflow causing the return value to be SIZE_MAX. The
* lvalue must be size_t to avoid implicit type conversion.
*/
static inline size_t __must_check size_add(size_t addend1, size_t addend2)
{
size_t bytes;
if (check_add_overflow(addend1, addend2, &bytes))
return SIZE_MAX;
return bytes;
}
/**
* size_sub() - Calculate size_t subtraction with saturation at SIZE_MAX
* @minuend: value to subtract from
* @subtrahend: value to subtract from @minuend
*
* Returns: calculate @minuend - @subtrahend, both promoted to size_t,
* with any overflow causing the return value to be SIZE_MAX. For
* composition with the size_add() and size_mul() helpers, neither
* argument may be SIZE_MAX (or the result with be forced to SIZE_MAX).
* The lvalue must be size_t to avoid implicit type conversion.
*/
static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend)
{
size_t bytes;
if (minuend == SIZE_MAX || subtrahend == SIZE_MAX ||
check_sub_overflow(minuend, subtrahend, &bytes))
return SIZE_MAX;
return bytes;
}
/**
* array_size() - Calculate size of 2-dimensional array.
* @a: dimension one
* @b: dimension two
*
* Calculates size of 2-dimensional array: @a * @b.
*
* Returns: number of bytes needed to represent the array or SIZE_MAX on
* overflow.
*/
#define array_size(a, b) size_mul(a, b)
/**
* array3_size() - Calculate size of 3-dimensional array.
* @a: dimension one
* @b: dimension two
* @c: dimension three
*
* Calculates size of 3-dimensional array: @a * @b * @c.
*
* Returns: number of bytes needed to represent the array or SIZE_MAX on
* overflow.
*/
#define array3_size(a, b, c) size_mul(size_mul(a, b), c)
/**
* flex_array_size() - Calculate size of a flexible array member
* within an enclosing structure.
* @p: Pointer to the structure.
* @member: Name of the flexible array member.
* @count: Number of elements in the array.
*
* Calculates size of a flexible array of @count number of @member
* elements, at the end of structure @p.
*
* Return: number of bytes needed or SIZE_MAX on overflow.
*/
#define flex_array_size(p, member, count) \
__builtin_choose_expr(__is_constexpr(count), \
(count) * sizeof(*(p)->member) + __must_be_array((p)->member), \
size_mul(count, sizeof(*(p)->member) + __must_be_array((p)->member)))
/**
* struct_size() - Calculate size of structure with trailing flexible array.
* @p: Pointer to the structure.
* @member: Name of the array member.
* @count: Number of elements in the array.
*
* Calculates size of memory needed for structure of @p followed by an
* array of @count number of @member elements.
*
* Return: number of bytes needed or SIZE_MAX on overflow.
*/
#define struct_size(p, member, count) \
__builtin_choose_expr(__is_constexpr(count), \
sizeof(*(p)) + flex_array_size(p, member, count), \
size_add(sizeof(*(p)), flex_array_size(p, member, count)))
/**
* struct_size_t() - Calculate size of structure with trailing flexible array
* @type: structure type name.
* @member: Name of the array member.
* @count: Number of elements in the array.
*
* Calculates size of memory needed for structure @type followed by an
* array of @count number of @member elements. Prefer using struct_size()
* when possible instead, to keep calculations associated with a specific
* instance variable of type @type.
*
* Return: number of bytes needed or SIZE_MAX on overflow.
*/
#define struct_size_t(type, member, count) \
struct_size((type *)NULL, member, count)
/**
* _DEFINE_FLEX() - helper macro for DEFINE_FLEX() family.
* Enables caller macro to pass (different) initializer.
*
* @type: structure type name, including "struct" keyword.
* @name: Name for a variable to define.
* @member: Name of the array member.
* @count: Number of elements in the array; must be compile-time const.
* @initializer: initializer expression (could be empty for no init).
*/
#define _DEFINE_FLEX(type, name, member, count, initializer...) \
_Static_assert(__builtin_constant_p(count), \
"onstack flex array members require compile-time const count"); \
union { \
u8 bytes[struct_size_t(type, member, count)]; \
type obj; \
} name##_u initializer; \
type *name = (type *)&name##_u
/**
* DEFINE_RAW_FLEX() - Define an on-stack instance of structure with a trailing
* flexible array member, when it does not have a __counted_by annotation.
*
* @type: structure type name, including "struct" keyword.
* @name: Name for a variable to define.
* @member: Name of the array member.
* @count: Number of elements in the array; must be compile-time const.
*
* Define a zeroed, on-stack, instance of @type structure with a trailing
* flexible array member.
* Use __struct_size(@name) to get compile-time size of it afterwards.
*/
#define DEFINE_RAW_FLEX(type, name, member, count) \
_DEFINE_FLEX(type, name, member, count, = {})
/**
* DEFINE_FLEX() - Define an on-stack instance of structure with a trailing
* flexible array member.
*
* @TYPE: structure type name, including "struct" keyword.
* @NAME: Name for a variable to define.
* @MEMBER: Name of the array member.
* @COUNTER: Name of the __counted_by member.
* @COUNT: Number of elements in the array; must be compile-time const.
*
* Define a zeroed, on-stack, instance of @TYPE structure with a trailing
* flexible array member.
* Use __struct_size(@NAME) to get compile-time size of it afterwards.
*/
#define DEFINE_FLEX(TYPE, NAME, MEMBER, COUNTER, COUNT) \
_DEFINE_FLEX(TYPE, NAME, MEMBER, COUNT, = { .obj.COUNTER = COUNT, })
#endif /* __LINUX_OVERFLOW_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PKEYS_H
#define _ASM_X86_PKEYS_H
/*
* If more than 16 keys are ever supported, a thorough audit
* will be necessary to ensure that the types that store key
* numbers and masks have sufficient capacity.
*/
#define arch_max_pkey() (cpu_feature_enabled(X86_FEATURE_OSPKE) ? 16 : 1)
extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
unsigned long init_val);
static inline bool arch_pkeys_enabled(void)
{
return cpu_feature_enabled(X86_FEATURE_OSPKE);
}
/*
* Try to dedicate one of the protection keys to be used as an
* execute-only protection key.
*/
extern int __execute_only_pkey(struct mm_struct *mm);
static inline int execute_only_pkey(struct mm_struct *mm)
{
if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
return ARCH_DEFAULT_PKEY;
return __execute_only_pkey(mm);
}
extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma,
int prot, int pkey);
static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma,
int prot, int pkey)
{
if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
return 0;
return __arch_override_mprotect_pkey(vma, prot, pkey);
}
#define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | VM_PKEY_BIT3)
#define mm_pkey_allocation_map(mm) (mm->context.pkey_allocation_map)
#define mm_set_pkey_allocated(mm, pkey) do { \
mm_pkey_allocation_map(mm) |= (1U << pkey); \
} while (0)
#define mm_set_pkey_free(mm, pkey) do { \
mm_pkey_allocation_map(mm) &= ~(1U << pkey); \
} while (0)
static inline
bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey)
{
/*
* "Allocated" pkeys are those that have been returned
* from pkey_alloc() or pkey 0 which is allocated
* implicitly when the mm is created.
*/
if (pkey < 0)
return false;
if (pkey >= arch_max_pkey())
return false;
/*
* The exec-only pkey is set in the allocation map, but
* is not available to any of the user interfaces like
* mprotect_pkey().
*/
if (pkey == mm->context.execute_only_pkey)
return false;
return mm_pkey_allocation_map(mm) & (1U << pkey);
}
/*
* Returns a positive, 4-bit key on success, or -1 on failure.
*/
static inline
int mm_pkey_alloc(struct mm_struct *mm)
{
/*
* Note: this is the one and only place we make sure
* that the pkey is valid as far as the hardware is
* concerned. The rest of the kernel trusts that
* only good, valid pkeys come out of here.
*/
u16 all_pkeys_mask = ((1U << arch_max_pkey()) - 1);
int ret;
/*
* Are we out of pkeys? We must handle this specially
* because ffz() behavior is undefined if there are no
* zeros.
*/
if (mm_pkey_allocation_map(mm) == all_pkeys_mask)
return -1;
ret = ffz(mm_pkey_allocation_map(mm));
mm_set_pkey_allocated(mm, ret);
return ret;
}
static inline
int mm_pkey_free(struct mm_struct *mm, int pkey)
{
if (!mm_pkey_is_allocated(mm, pkey))
return -EINVAL;
mm_set_pkey_free(mm, pkey);
return 0;
}
static inline int vma_pkey(struct vm_area_struct *vma)
{
unsigned long vma_pkey_mask = VM_PKEY_BIT0 | VM_PKEY_BIT1 |
VM_PKEY_BIT2 | VM_PKEY_BIT3;
return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT;
}
#endif /*_ASM_X86_PKEYS_H */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
*/
/*
* fsnotify inode mark locking/lifetime/and refcnting
*
* REFCNT:
* The group->recnt and mark->refcnt tell how many "things" in the kernel
* currently are referencing the objects. Both kind of objects typically will
* live inside the kernel with a refcnt of 2, one for its creation and one for
* the reference a group and a mark hold to each other.
* If you are holding the appropriate locks, you can take a reference and the
* object itself is guaranteed to survive until the reference is dropped.
*
* LOCKING:
* There are 3 locks involved with fsnotify inode marks and they MUST be taken
* in order as follows:
*
* group->mark_mutex
* mark->lock
* mark->connector->lock
*
* group->mark_mutex protects the marks_list anchored inside a given group and
* each mark is hooked via the g_list. It also protects the groups private
* data (i.e group limits).
* mark->lock protects the marks attributes like its masks and flags.
* Furthermore it protects the access to a reference of the group that the mark
* is assigned to as well as the access to a reference of the inode/vfsmount
* that is being watched by the mark.
*
* mark->connector->lock protects the list of marks anchored inside an
* inode / vfsmount and each mark is hooked via the i_list.
*
* A list of notification marks relating to inode / mnt is contained in
* fsnotify_mark_connector. That structure is alive as long as there are any
* marks in the list and is also protected by fsnotify_mark_srcu. A mark gets
* detached from fsnotify_mark_connector when last reference to the mark is
* dropped. Thus having mark reference is enough to protect mark->connector
* pointer and to make sure fsnotify_mark_connector cannot disappear. Also
* because we remove mark from g_list before dropping mark reference associated
* with that, any mark found through g_list is guaranteed to have
* mark->connector set until we drop group->mark_mutex.
*
* LIFETIME:
* Inode marks survive between when they are added to an inode and when their
* refcnt==0. Marks are also protected by fsnotify_mark_srcu.
*
* The inode mark can be cleared for a number of different reasons including:
* - The inode is unlinked for the last time. (fsnotify_inode_remove)
* - The inode is being evicted from cache. (fsnotify_inode_delete)
* - The fs the inode is on is unmounted. (fsnotify_inode_delete/fsnotify_unmount_inodes)
* - Something explicitly requests that it be removed. (fsnotify_destroy_mark)
* - The fsnotify_group associated with the mark is going away and all such marks
* need to be cleaned up. (fsnotify_clear_marks_by_group)
*
* This has the very interesting property of being able to run concurrently with
* any (or all) other directions.
*/
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/srcu.h>
#include <linux/ratelimit.h>
#include <linux/atomic.h>
#include <linux/fsnotify_backend.h>
#include "fsnotify.h"
#define FSNOTIFY_REAPER_DELAY (1) /* 1 jiffy */
struct srcu_struct fsnotify_mark_srcu;
struct kmem_cache *fsnotify_mark_connector_cachep;
static DEFINE_SPINLOCK(destroy_lock);
static LIST_HEAD(destroy_list);
static struct fsnotify_mark_connector *connector_destroy_list;
static void fsnotify_mark_destroy_workfn(struct work_struct *work);
static DECLARE_DELAYED_WORK(reaper_work, fsnotify_mark_destroy_workfn);
static void fsnotify_connector_destroy_workfn(struct work_struct *work);
static DECLARE_WORK(connector_reaper_work, fsnotify_connector_destroy_workfn);
void fsnotify_get_mark(struct fsnotify_mark *mark)
{
WARN_ON_ONCE(!refcount_read(&mark->refcnt));
refcount_inc(&mark->refcnt);
}
static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)
{
if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
return &fsnotify_conn_inode(conn)->i_fsnotify_mask;
else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)
return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;
else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)
return &fsnotify_conn_sb(conn)->s_fsnotify_mask;
return NULL;
}
__u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)
{
if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))
return 0;
return *fsnotify_conn_mask_p(conn);
}
static void fsnotify_get_inode_ref(struct inode *inode)
{
ihold(inode);
atomic_long_inc(&inode->i_sb->s_fsnotify_connectors);
}
/*
* Grab or drop inode reference for the connector if needed.
*
* When it's time to drop the reference, we only clear the HAS_IREF flag and
* return the inode object. fsnotify_drop_object() will be resonsible for doing
* iput() outside of spinlocks. This happens when last mark that wanted iref is
* detached.
*/
static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,
bool want_iref)
{
bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
struct inode *inode = NULL;
if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||
want_iref == has_iref)
return NULL;
if (want_iref) {
/* Pin inode if any mark wants inode refcount held */
fsnotify_get_inode_ref(fsnotify_conn_inode(conn));
conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;
} else {
/* Unpin inode after detach of last mark that wanted iref */
inode = fsnotify_conn_inode(conn);
conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;
}
return inode;
}
static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
u32 new_mask = 0;
bool want_iref = false;
struct fsnotify_mark *mark;
assert_spin_locked(&conn->lock);
/* We can get detached connector here when inode is getting unlinked. */
if (!fsnotify_valid_obj_type(conn->type))
return NULL;
hlist_for_each_entry(mark, &conn->list, obj_list) {
if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED))
continue;
new_mask |= fsnotify_calc_mask(mark);
if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&
!(mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF))
want_iref = true;
}
*fsnotify_conn_mask_p(conn) = new_mask;
return fsnotify_update_iref(conn, want_iref);
}
/*
* Calculate mask of events for a list of marks. The caller must make sure
* connector and connector->obj cannot disappear under us. Callers achieve
* this by holding a mark->lock or mark->group->mark_mutex for a mark on this
* list.
*/
void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
if (!conn)
return;
spin_lock(&conn->lock);
__fsnotify_recalc_mask(conn);
spin_unlock(&conn->lock);
if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
__fsnotify_update_child_dentry_flags(
fsnotify_conn_inode(conn));
}
/* Free all connectors queued for freeing once SRCU period ends */
static void fsnotify_connector_destroy_workfn(struct work_struct *work)
{
struct fsnotify_mark_connector *conn, *free;
spin_lock(&destroy_lock);
conn = connector_destroy_list;
connector_destroy_list = NULL;
spin_unlock(&destroy_lock);
synchronize_srcu(&fsnotify_mark_srcu);
while (conn) {
free = conn;
conn = conn->destroy_next;
kmem_cache_free(fsnotify_mark_connector_cachep, free);
}
}
static void fsnotify_put_inode_ref(struct inode *inode)
{
struct super_block *sb = inode->i_sb;
iput(inode);
if (atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
wake_up_var(&sb->s_fsnotify_connectors);
}
static void fsnotify_get_sb_connectors(struct fsnotify_mark_connector *conn)
{
struct super_block *sb = fsnotify_connector_sb(conn);
if (sb)
atomic_long_inc(&sb->s_fsnotify_connectors);
}
static void fsnotify_put_sb_connectors(struct fsnotify_mark_connector *conn)
{
struct super_block *sb = fsnotify_connector_sb(conn);
if (sb && atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
wake_up_var(&sb->s_fsnotify_connectors);
}
static void *fsnotify_detach_connector_from_object(
struct fsnotify_mark_connector *conn,
unsigned int *type)
{
struct inode *inode = NULL;
*type = conn->type;
if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)
return NULL;
if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
inode = fsnotify_conn_inode(conn);
inode->i_fsnotify_mask = 0;
/* Unpin inode when detaching from connector */
if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))
inode = NULL;
} else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;
} else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {
fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
}
fsnotify_put_sb_connectors(conn);
rcu_assign_pointer(*(conn->obj), NULL);
conn->obj = NULL;
conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
return inode;
}
static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)
{
struct fsnotify_group *group = mark->group;
if (WARN_ON_ONCE(!group))
return;
group->ops->free_mark(mark);
fsnotify_put_group(group);
}
/* Drop object reference originally held by a connector */
static void fsnotify_drop_object(unsigned int type, void *objp)
{
if (!objp)
return;
/* Currently only inode references are passed to be dropped */
if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE))
return;
fsnotify_put_inode_ref(objp);
}
void fsnotify_put_mark(struct fsnotify_mark *mark)
{
struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);
void *objp = NULL;
unsigned int type = FSNOTIFY_OBJ_TYPE_DETACHED;
bool free_conn = false;
/* Catch marks that were actually never attached to object */
if (!conn) {
if (refcount_dec_and_test(&mark->refcnt))
fsnotify_final_mark_destroy(mark);
return;
}
/*
* We have to be careful so that traversals of obj_list under lock can
* safely grab mark reference.
*/
if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))
return;
hlist_del_init_rcu(&mark->obj_list);
if (hlist_empty(&conn->list)) {
objp = fsnotify_detach_connector_from_object(conn, &type);
free_conn = true;
} else {
objp = __fsnotify_recalc_mask(conn);
type = conn->type;
}
WRITE_ONCE(mark->connector, NULL);
spin_unlock(&conn->lock);
fsnotify_drop_object(type, objp);
if (free_conn) {
spin_lock(&destroy_lock);
conn->destroy_next = connector_destroy_list;
connector_destroy_list = conn;
spin_unlock(&destroy_lock);
queue_work(system_unbound_wq, &connector_reaper_work);
}
/*
* Note that we didn't update flags telling whether inode cares about
* what's happening with children. We update these flags from
* __fsnotify_parent() lazily when next event happens on one of our
* children.
*/
spin_lock(&destroy_lock);
list_add(&mark->g_list, &destroy_list);
spin_unlock(&destroy_lock);
queue_delayed_work(system_unbound_wq, &reaper_work,
FSNOTIFY_REAPER_DELAY);
}
EXPORT_SYMBOL_GPL(fsnotify_put_mark);
/*
* Get mark reference when we found the mark via lockless traversal of object
* list. Mark can be already removed from the list by now and on its way to be
* destroyed once SRCU period ends.
*
* Also pin the group so it doesn't disappear under us.
*/
static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)
{
if (!mark)
return true;
if (refcount_inc_not_zero(&mark->refcnt)) {
spin_lock(&mark->lock);
if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {
/* mark is attached, group is still alive then */
atomic_inc(&mark->group->user_waits);
spin_unlock(&mark->lock);
return true;
}
spin_unlock(&mark->lock);
fsnotify_put_mark(mark);
}
return false;
}
/*
* Puts marks and wakes up group destruction if necessary.
*
* Pairs with fsnotify_get_mark_safe()
*/
static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)
{
if (mark) {
struct fsnotify_group *group = mark->group;
fsnotify_put_mark(mark);
/*
* We abuse notification_waitq on group shutdown for waiting for
* all marks pinned when waiting for userspace.
*/
if (atomic_dec_and_test(&group->user_waits) && group->shutdown)
wake_up(&group->notification_waitq);
}
}
bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
__releases(&fsnotify_mark_srcu)
{
int type;
fsnotify_foreach_iter_type(type) {
/* This can fail if mark is being removed */
if (!fsnotify_get_mark_safe(iter_info->marks[type])) {
__release(&fsnotify_mark_srcu);
goto fail;
}
}
/*
* Now that both marks are pinned by refcount in the inode / vfsmount
* lists, we can drop SRCU lock, and safely resume the list iteration
* once userspace returns.
*/
srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx);
return true;
fail:
for (type--; type >= 0; type--)
fsnotify_put_mark_wake(iter_info->marks[type]);
return false;
}
void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info)
__acquires(&fsnotify_mark_srcu)
{
int type;
iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
fsnotify_foreach_iter_type(type)
fsnotify_put_mark_wake(iter_info->marks[type]);
}
/*
* Mark mark as detached, remove it from group list. Mark still stays in object
* list until its last reference is dropped. Note that we rely on mark being
* removed from group list before corresponding reference to it is dropped. In
* particular we rely on mark->connector being valid while we hold
* group->mark_mutex if we found the mark through g_list.
*
* Must be called with group->mark_mutex held. The caller must either hold
* reference to the mark or be protected by fsnotify_mark_srcu.
*/
void fsnotify_detach_mark(struct fsnotify_mark *mark)
{
fsnotify_group_assert_locked(mark->group);
WARN_ON_ONCE(!srcu_read_lock_held(&fsnotify_mark_srcu) &&
refcount_read(&mark->refcnt) < 1 +
!!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));
spin_lock(&mark->lock);
/* something else already called this function on this mark */
if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
spin_unlock(&mark->lock);
return;
}
mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;
list_del_init(&mark->g_list);
spin_unlock(&mark->lock);
/* Drop mark reference acquired in fsnotify_add_mark_locked() */
fsnotify_put_mark(mark);
}
/*
* Free fsnotify mark. The mark is actually only marked as being freed. The
* freeing is actually happening only once last reference to the mark is
* dropped from a workqueue which first waits for srcu period end.
*
* Caller must have a reference to the mark or be protected by
* fsnotify_mark_srcu.
*/
void fsnotify_free_mark(struct fsnotify_mark *mark)
{
struct fsnotify_group *group = mark->group;
spin_lock(&mark->lock);
/* something else already called this function on this mark */
if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {
spin_unlock(&mark->lock);
return;
}
mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;
spin_unlock(&mark->lock);
/*
* Some groups like to know that marks are being freed. This is a
* callback to the group function to let it know that this mark
* is being freed.
*/
if (group->ops->freeing_mark)
group->ops->freeing_mark(mark, group);
}
void fsnotify_destroy_mark(struct fsnotify_mark *mark,
struct fsnotify_group *group)
{
fsnotify_group_lock(group); fsnotify_detach_mark(mark); fsnotify_group_unlock(group);
fsnotify_free_mark(mark);
}
EXPORT_SYMBOL_GPL(fsnotify_destroy_mark);
/*
* Sorting function for lists of fsnotify marks.
*
* Fanotify supports different notification classes (reflected as priority of
* notification group). Events shall be passed to notification groups in
* decreasing priority order. To achieve this marks in notification lists for
* inodes and vfsmounts are sorted so that priorities of corresponding groups
* are descending.
*
* Furthermore correct handling of the ignore mask requires processing inode
* and vfsmount marks of each group together. Using the group address as
* further sort criterion provides a unique sorting order and thus we can
* merge inode and vfsmount lists of marks in linear time and find groups
* present in both lists.
*
* A return value of 1 signifies that b has priority over a.
* A return value of 0 signifies that the two marks have to be handled together.
* A return value of -1 signifies that a has priority over b.
*/
int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)
{
if (a == b)
return 0;
if (!a)
return 1;
if (!b)
return -1;
if (a->priority < b->priority)
return 1;
if (a->priority > b->priority)
return -1;
if (a < b)
return 1;
return -1;
}
static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
unsigned int obj_type)
{
struct fsnotify_mark_connector *conn;
conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, GFP_KERNEL);
if (!conn)
return -ENOMEM;
spin_lock_init(&conn->lock);
INIT_HLIST_HEAD(&conn->list);
conn->flags = 0;
conn->type = obj_type;
conn->obj = connp;
conn->flags = 0;
fsnotify_get_sb_connectors(conn);
/*
* cmpxchg() provides the barrier so that readers of *connp can see
* only initialized structure
*/
if (cmpxchg(connp, NULL, conn)) {
/* Someone else created list structure for us */
fsnotify_put_sb_connectors(conn);
kmem_cache_free(fsnotify_mark_connector_cachep, conn);
}
return 0;
}
/*
* Get mark connector, make sure it is alive and return with its lock held.
* This is for users that get connector pointer from inode or mount. Users that
* hold reference to a mark on the list may directly lock connector->lock as
* they are sure list cannot go away under them.
*/
static struct fsnotify_mark_connector *fsnotify_grab_connector(
fsnotify_connp_t *connp)
{
struct fsnotify_mark_connector *conn;
int idx;
idx = srcu_read_lock(&fsnotify_mark_srcu); conn = srcu_dereference(*connp, &fsnotify_mark_srcu);
if (!conn)
goto out;
spin_lock(&conn->lock);
if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {
spin_unlock(&conn->lock); srcu_read_unlock(&fsnotify_mark_srcu, idx);
return NULL;
}
out:
srcu_read_unlock(&fsnotify_mark_srcu, idx);
return conn;
}
/*
* Add mark into proper place in given list of marks. These marks may be used
* for the fsnotify backend to determine which event types should be delivered
* to which group and for which inodes. These marks are ordered according to
* priority, highest number first, and then by the group's location in memory.
*/
static int fsnotify_add_mark_list(struct fsnotify_mark *mark,
fsnotify_connp_t *connp,
unsigned int obj_type, int add_flags)
{
struct fsnotify_mark *lmark, *last = NULL;
struct fsnotify_mark_connector *conn;
int cmp;
int err = 0;
if (WARN_ON(!fsnotify_valid_obj_type(obj_type)))
return -EINVAL;
restart:
spin_lock(&mark->lock);
conn = fsnotify_grab_connector(connp);
if (!conn) {
spin_unlock(&mark->lock);
err = fsnotify_attach_connector_to_object(connp, obj_type);
if (err)
return err;
goto restart;
}
/* is mark the first mark? */
if (hlist_empty(&conn->list)) {
hlist_add_head_rcu(&mark->obj_list, &conn->list);
goto added;
}
/* should mark be in the middle of the current list? */
hlist_for_each_entry(lmark, &conn->list, obj_list) {
last = lmark;
if ((lmark->group == mark->group) &&
(lmark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) &&
!(mark->group->flags & FSNOTIFY_GROUP_DUPS)) {
err = -EEXIST;
goto out_err;
}
cmp = fsnotify_compare_groups(lmark->group, mark->group);
if (cmp >= 0) {
hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);
goto added;
}
}
BUG_ON(last == NULL);
/* mark should be the last entry. last is the current last entry */
hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);
added:
/*
* Since connector is attached to object using cmpxchg() we are
* guaranteed that connector initialization is fully visible by anyone
* seeing mark->connector set.
*/
WRITE_ONCE(mark->connector, conn);
out_err:
spin_unlock(&conn->lock);
spin_unlock(&mark->lock);
return err;
}
/*
* Attach an initialized mark to a given group and fs object.
* These marks may be used for the fsnotify backend to determine which
* event types should be delivered to which group.
*/
int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
fsnotify_connp_t *connp, unsigned int obj_type,
int add_flags)
{
struct fsnotify_group *group = mark->group;
int ret = 0;
fsnotify_group_assert_locked(group);
/*
* LOCKING ORDER!!!!
* group->mark_mutex
* mark->lock
* mark->connector->lock
*/
spin_lock(&mark->lock);
mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;
list_add(&mark->g_list, &group->marks_list);
fsnotify_get_mark(mark); /* for g_list */
spin_unlock(&mark->lock);
ret = fsnotify_add_mark_list(mark, connp, obj_type, add_flags);
if (ret)
goto err;
fsnotify_recalc_mask(mark->connector);
return ret;
err:
spin_lock(&mark->lock);
mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |
FSNOTIFY_MARK_FLAG_ATTACHED);
list_del_init(&mark->g_list);
spin_unlock(&mark->lock);
fsnotify_put_mark(mark);
return ret;
}
int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp,
unsigned int obj_type, int add_flags)
{
int ret;
struct fsnotify_group *group = mark->group;
fsnotify_group_lock(group);
ret = fsnotify_add_mark_locked(mark, connp, obj_type, add_flags);
fsnotify_group_unlock(group);
return ret;
}
EXPORT_SYMBOL_GPL(fsnotify_add_mark);
/*
* Given a list of marks, find the mark associated with given group. If found
* take a reference to that mark and return it, else return NULL.
*/
struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp,
struct fsnotify_group *group)
{
struct fsnotify_mark_connector *conn;
struct fsnotify_mark *mark;
conn = fsnotify_grab_connector(connp);
if (!conn)
return NULL;
hlist_for_each_entry(mark, &conn->list, obj_list) {
if (mark->group == group &&
(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
fsnotify_get_mark(mark);
spin_unlock(&conn->lock);
return mark;
}
}
spin_unlock(&conn->lock);
return NULL;
}
EXPORT_SYMBOL_GPL(fsnotify_find_mark);
/* Clear any marks in a group with given type mask */
void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
unsigned int obj_type)
{
struct fsnotify_mark *lmark, *mark;
LIST_HEAD(to_free);
struct list_head *head = &to_free;
/* Skip selection step if we want to clear all marks. */
if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) {
head = &group->marks_list;
goto clear;
}
/*
* We have to be really careful here. Anytime we drop mark_mutex, e.g.
* fsnotify_clear_marks_by_inode() can come and free marks. Even in our
* to_free list so we have to use mark_mutex even when accessing that
* list. And freeing mark requires us to drop mark_mutex. So we can
* reliably free only the first mark in the list. That's why we first
* move marks to free to to_free list in one go and then free marks in
* to_free list one by one.
*/
fsnotify_group_lock(group);
list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {
if (mark->connector->type == obj_type)
list_move(&mark->g_list, &to_free);
}
fsnotify_group_unlock(group);
clear:
while (1) {
fsnotify_group_lock(group);
if (list_empty(head)) {
fsnotify_group_unlock(group);
break;
}
mark = list_first_entry(head, struct fsnotify_mark, g_list);
fsnotify_get_mark(mark);
fsnotify_detach_mark(mark);
fsnotify_group_unlock(group);
fsnotify_free_mark(mark);
fsnotify_put_mark(mark);
}
}
/* Destroy all marks attached to an object via connector */
void fsnotify_destroy_marks(fsnotify_connp_t *connp)
{
struct fsnotify_mark_connector *conn;
struct fsnotify_mark *mark, *old_mark = NULL;
void *objp;
unsigned int type;
conn = fsnotify_grab_connector(connp);
if (!conn)
return;
/*
* We have to be careful since we can race with e.g.
* fsnotify_clear_marks_by_group() and once we drop the conn->lock, the
* list can get modified. However we are holding mark reference and
* thus our mark cannot be removed from obj_list so we can continue
* iteration after regaining conn->lock.
*/
hlist_for_each_entry(mark, &conn->list, obj_list) { fsnotify_get_mark(mark);
spin_unlock(&conn->lock);
if (old_mark)
fsnotify_put_mark(old_mark);
old_mark = mark;
fsnotify_destroy_mark(mark, mark->group);
spin_lock(&conn->lock);
}
/*
* Detach list from object now so that we don't pin inode until all
* mark references get dropped. It would lead to strange results such
* as delaying inode deletion or blocking unmount.
*/
objp = fsnotify_detach_connector_from_object(conn, &type);
spin_unlock(&conn->lock);
if (old_mark)
fsnotify_put_mark(old_mark); fsnotify_drop_object(type, objp);
}
/*
* Nothing fancy, just initialize lists and locks and counters.
*/
void fsnotify_init_mark(struct fsnotify_mark *mark,
struct fsnotify_group *group)
{
memset(mark, 0, sizeof(*mark));
spin_lock_init(&mark->lock);
refcount_set(&mark->refcnt, 1);
fsnotify_get_group(group);
mark->group = group;
WRITE_ONCE(mark->connector, NULL);
}
EXPORT_SYMBOL_GPL(fsnotify_init_mark);
/*
* Destroy all marks in destroy_list, waits for SRCU period to finish before
* actually freeing marks.
*/
static void fsnotify_mark_destroy_workfn(struct work_struct *work)
{
struct fsnotify_mark *mark, *next;
struct list_head private_destroy_list;
spin_lock(&destroy_lock);
/* exchange the list head */
list_replace_init(&destroy_list, &private_destroy_list);
spin_unlock(&destroy_lock);
synchronize_srcu(&fsnotify_mark_srcu);
list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {
list_del_init(&mark->g_list);
fsnotify_final_mark_destroy(mark);
}
}
/* Wait for all marks queued for destruction to be actually destroyed */
void fsnotify_wait_marks_destroyed(void)
{
flush_delayed_work(&reaper_work);
}
EXPORT_SYMBOL_GPL(fsnotify_wait_marks_destroyed);
/* SPDX-License-Identifier: GPL-2.0 */
/* rwsem.h: R/W semaphores, public interface
*
* Written by David Howells (dhowells@redhat.com).
* Derived from asm-i386/semaphore.h
*/
#ifndef _LINUX_RWSEM_H
#define _LINUX_RWSEM_H
#include <linux/linkage.h>
#include <linux/types.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>
#include <linux/err.h>
#include <linux/cleanup.h>
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define __RWSEM_DEP_MAP_INIT(lockname) \
.dep_map = { \
.name = #lockname, \
.wait_type_inner = LD_WAIT_SLEEP, \
},
#else
# define __RWSEM_DEP_MAP_INIT(lockname)
#endif
#ifndef CONFIG_PREEMPT_RT
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
#include <linux/osq_lock.h>
#endif
/*
* For an uncontended rwsem, count and owner are the only fields a task
* needs to touch when acquiring the rwsem. So they are put next to each
* other to increase the chance that they will share the same cacheline.
*
* In a contended rwsem, the owner is likely the most frequently accessed
* field in the structure as the optimistic waiter that holds the osq lock
* will spin on owner. For an embedded rwsem, other hot fields in the
* containing structure should be moved further away from the rwsem to
* reduce the chance that they will share the same cacheline causing
* cacheline bouncing problem.
*/
struct rw_semaphore {
atomic_long_t count;
/*
* Write owner or one of the read owners as well flags regarding
* the current state of the rwsem. Can be used as a speculative
* check to see if the write owner is running on the cpu.
*/
atomic_long_t owner;
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
struct optimistic_spin_queue osq; /* spinner MCS lock */
#endif
raw_spinlock_t wait_lock;
struct list_head wait_list;
#ifdef CONFIG_DEBUG_RWSEMS
void *magic;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif
};
#define RWSEM_UNLOCKED_VALUE 0UL
#define RWSEM_WRITER_LOCKED (1UL << 0)
#define __RWSEM_COUNT_INIT(name) .count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE)
static inline int rwsem_is_locked(struct rw_semaphore *sem)
{
return atomic_long_read(&sem->count) != RWSEM_UNLOCKED_VALUE;
}
static inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem)
{
WARN_ON(atomic_long_read(&sem->count) == RWSEM_UNLOCKED_VALUE);
}
static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem)
{
WARN_ON(!(atomic_long_read(&sem->count) & RWSEM_WRITER_LOCKED));
}
/* Common initializer macros and functions */
#ifdef CONFIG_DEBUG_RWSEMS
# define __RWSEM_DEBUG_INIT(lockname) .magic = &lockname,
#else
# define __RWSEM_DEBUG_INIT(lockname)
#endif
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
#define __RWSEM_OPT_INIT(lockname) .osq = OSQ_LOCK_UNLOCKED,
#else
#define __RWSEM_OPT_INIT(lockname)
#endif
#define __RWSEM_INITIALIZER(name) \
{ __RWSEM_COUNT_INIT(name), \
.owner = ATOMIC_LONG_INIT(0), \
__RWSEM_OPT_INIT(name) \
.wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\
.wait_list = LIST_HEAD_INIT((name).wait_list), \
__RWSEM_DEBUG_INIT(name) \
__RWSEM_DEP_MAP_INIT(name) }
#define DECLARE_RWSEM(name) \
struct rw_semaphore name = __RWSEM_INITIALIZER(name)
extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
struct lock_class_key *key);
#define init_rwsem(sem) \
do { \
static struct lock_class_key __key; \
\
__init_rwsem((sem), #sem, &__key); \
} while (0)
/*
* This is the same regardless of which rwsem implementation that is being used.
* It is just a heuristic meant to be called by somebody already holding the
* rwsem to see if somebody from an incompatible type is wanting access to the
* lock.
*/
static inline int rwsem_is_contended(struct rw_semaphore *sem)
{
return !list_empty(&sem->wait_list);
}
#else /* !CONFIG_PREEMPT_RT */
#include <linux/rwbase_rt.h>
struct rw_semaphore {
struct rwbase_rt rwbase;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif
};
#define __RWSEM_INITIALIZER(name) \
{ \
.rwbase = __RWBASE_INITIALIZER(name), \
__RWSEM_DEP_MAP_INIT(name) \
}
#define DECLARE_RWSEM(lockname) \
struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname)
extern void __init_rwsem(struct rw_semaphore *rwsem, const char *name,
struct lock_class_key *key);
#define init_rwsem(sem) \
do { \
static struct lock_class_key __key; \
\
__init_rwsem((sem), #sem, &__key); \
} while (0)
static __always_inline int rwsem_is_locked(const struct rw_semaphore *sem)
{
return rw_base_is_locked(&sem->rwbase);
}
static __always_inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem)
{
WARN_ON(!rwsem_is_locked(sem));
}
static __always_inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem)
{
WARN_ON(!rw_base_is_write_locked(&sem->rwbase));
}
static __always_inline int rwsem_is_contended(struct rw_semaphore *sem)
{
return rw_base_is_contended(&sem->rwbase);
}
#endif /* CONFIG_PREEMPT_RT */
/*
* The functions below are the same for all rwsem implementations including
* the RT specific variant.
*/
static inline void rwsem_assert_held(const struct rw_semaphore *sem)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
lockdep_assert_held(sem);
else
rwsem_assert_held_nolockdep(sem);
}
static inline void rwsem_assert_held_write(const struct rw_semaphore *sem)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
lockdep_assert_held_write(sem);
else
rwsem_assert_held_write_nolockdep(sem);
}
/*
* lock for reading
*/
extern void down_read(struct rw_semaphore *sem);
extern int __must_check down_read_interruptible(struct rw_semaphore *sem);
extern int __must_check down_read_killable(struct rw_semaphore *sem);
/*
* trylock for reading -- returns 1 if successful, 0 if contention
*/
extern int down_read_trylock(struct rw_semaphore *sem);
/*
* lock for writing
*/
extern void down_write(struct rw_semaphore *sem);
extern int __must_check down_write_killable(struct rw_semaphore *sem);
/*
* trylock for writing -- returns 1 if successful, 0 if contention
*/
extern int down_write_trylock(struct rw_semaphore *sem);
/*
* release a read lock
*/
extern void up_read(struct rw_semaphore *sem);
/*
* release a write lock
*/
extern void up_write(struct rw_semaphore *sem);
DEFINE_GUARD(rwsem_read, struct rw_semaphore *, down_read(_T), up_read(_T))
DEFINE_GUARD_COND(rwsem_read, _try, down_read_trylock(_T))
DEFINE_GUARD_COND(rwsem_read, _intr, down_read_interruptible(_T) == 0)
DEFINE_GUARD(rwsem_write, struct rw_semaphore *, down_write(_T), up_write(_T))
DEFINE_GUARD_COND(rwsem_write, _try, down_write_trylock(_T))
/*
* downgrade write lock to read lock
*/
extern void downgrade_write(struct rw_semaphore *sem);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
/*
* nested locking. NOTE: rwsems are not allowed to recurse
* (which occurs if the same task tries to acquire the same
* lock instance multiple times), but multiple locks of the
* same lock class might be taken, if the order of the locks
* is always the same. This ordering rule can be expressed
* to lockdep via the _nested() APIs, but enumerating the
* subclasses that are used. (If the nesting relationship is
* static then another method for expressing nested locking is
* the explicit definition of lock class keys and the use of
* lockdep_set_class() at lock initialization time.
* See Documentation/locking/lockdep-design.rst for more details.)
*/
extern void down_read_nested(struct rw_semaphore *sem, int subclass);
extern int __must_check down_read_killable_nested(struct rw_semaphore *sem, int subclass);
extern void down_write_nested(struct rw_semaphore *sem, int subclass);
extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass);
extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock);
# define down_write_nest_lock(sem, nest_lock) \
do { \
typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
_down_write_nest_lock(sem, &(nest_lock)->dep_map); \
} while (0)
/*
* Take/release a lock when not the owner will release it.
*
* [ This API should be avoided as much as possible - the
* proper abstraction for this case is completions. ]
*/
extern void down_read_non_owner(struct rw_semaphore *sem);
extern void up_read_non_owner(struct rw_semaphore *sem);
#else
# define down_read_nested(sem, subclass) down_read(sem)
# define down_read_killable_nested(sem, subclass) down_read_killable(sem)
# define down_write_nest_lock(sem, nest_lock) down_write(sem)
# define down_write_nested(sem, subclass) down_write(sem)
# define down_write_killable_nested(sem, subclass) down_write_killable(sem)
# define down_read_non_owner(sem) down_read(sem)
# define up_read_non_owner(sem) up_read(sem)
#endif
#endif /* _LINUX_RWSEM_H */
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2014 Felix Fietkau <nbd@nbd.name>
* Copyright (C) 2004 - 2009 Ivo van Doorn <IvDoorn@gmail.com>
*/
#ifndef _LINUX_BITFIELD_H
#define _LINUX_BITFIELD_H
#include <linux/build_bug.h>
#include <asm/byteorder.h>
/*
* Bitfield access macros
*
* FIELD_{GET,PREP} macros take as first parameter shifted mask
* from which they extract the base mask and shift amount.
* Mask must be a compilation time constant.
*
* Example:
*
* #include <linux/bitfield.h>
* #include <linux/bits.h>
*
* #define REG_FIELD_A GENMASK(6, 0)
* #define REG_FIELD_B BIT(7)
* #define REG_FIELD_C GENMASK(15, 8)
* #define REG_FIELD_D GENMASK(31, 16)
*
* Get:
* a = FIELD_GET(REG_FIELD_A, reg);
* b = FIELD_GET(REG_FIELD_B, reg);
*
* Set:
* reg = FIELD_PREP(REG_FIELD_A, 1) |
* FIELD_PREP(REG_FIELD_B, 0) |
* FIELD_PREP(REG_FIELD_C, c) |
* FIELD_PREP(REG_FIELD_D, 0x40);
*
* Modify:
* reg &= ~REG_FIELD_C;
* reg |= FIELD_PREP(REG_FIELD_C, c);
*/
#define __bf_shf(x) (__builtin_ffsll(x) - 1)
#define __scalar_type_to_unsigned_cases(type) \
unsigned type: (unsigned type)0, \
signed type: (unsigned type)0
#define __unsigned_scalar_typeof(x) typeof( \
_Generic((x), \
char: (unsigned char)0, \
__scalar_type_to_unsigned_cases(char), \
__scalar_type_to_unsigned_cases(short), \
__scalar_type_to_unsigned_cases(int), \
__scalar_type_to_unsigned_cases(long), \
__scalar_type_to_unsigned_cases(long long), \
default: (x)))
#define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x))
#define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \
({ \
BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \
_pfx "mask is not constant"); \
BUILD_BUG_ON_MSG((_mask) == 0, _pfx "mask is zero"); \
BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \
~((_mask) >> __bf_shf(_mask)) & \
(0 + (_val)) : 0, \
_pfx "value too large for the field"); \
BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \
__bf_cast_unsigned(_reg, ~0ull), \
_pfx "type of reg too small for mask"); \
__BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \
(1ULL << __bf_shf(_mask))); \
})
/**
* FIELD_MAX() - produce the maximum value representable by a field
* @_mask: shifted mask defining the field's length and position
*
* FIELD_MAX() returns the maximum value that can be held in the field
* specified by @_mask.
*/
#define FIELD_MAX(_mask) \
({ \
__BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_MAX: "); \
(typeof(_mask))((_mask) >> __bf_shf(_mask)); \
})
/**
* FIELD_FIT() - check if value fits in the field
* @_mask: shifted mask defining the field's length and position
* @_val: value to test against the field
*
* Return: true if @_val can fit inside @_mask, false if @_val is too big.
*/
#define FIELD_FIT(_mask, _val) \
({ \
__BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_FIT: "); \
!((((typeof(_mask))_val) << __bf_shf(_mask)) & ~(_mask)); \
})
/**
* FIELD_PREP() - prepare a bitfield element
* @_mask: shifted mask defining the field's length and position
* @_val: value to put in the field
*
* FIELD_PREP() masks and shifts up the value. The result should
* be combined with other fields of the bitfield using logical OR.
*/
#define FIELD_PREP(_mask, _val) \
({ \
__BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_PREP: "); \
((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask); \
})
#define __BF_CHECK_POW2(n) BUILD_BUG_ON_ZERO(((n) & ((n) - 1)) != 0)
/**
* FIELD_PREP_CONST() - prepare a constant bitfield element
* @_mask: shifted mask defining the field's length and position
* @_val: value to put in the field
*
* FIELD_PREP_CONST() masks and shifts up the value. The result should
* be combined with other fields of the bitfield using logical OR.
*
* Unlike FIELD_PREP() this is a constant expression and can therefore
* be used in initializers. Error checking is less comfortable for this
* version, and non-constant masks cannot be used.
*/
#define FIELD_PREP_CONST(_mask, _val) \
( \
/* mask must be non-zero */ \
BUILD_BUG_ON_ZERO((_mask) == 0) + \
/* check if value fits */ \
BUILD_BUG_ON_ZERO(~((_mask) >> __bf_shf(_mask)) & (_val)) + \
/* check if mask is contiguous */ \
__BF_CHECK_POW2((_mask) + (1ULL << __bf_shf(_mask))) + \
/* and create the value */ \
(((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask)) \
)
/**
* FIELD_GET() - extract a bitfield element
* @_mask: shifted mask defining the field's length and position
* @_reg: value of entire bitfield
*
* FIELD_GET() extracts the field specified by @_mask from the
* bitfield passed in as @_reg by masking and shifting it down.
*/
#define FIELD_GET(_mask, _reg) \
({ \
__BF_FIELD_CHECK(_mask, _reg, 0U, "FIELD_GET: "); \
(typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \
})
extern void __compiletime_error("value doesn't fit into mask")
__field_overflow(void);
extern void __compiletime_error("bad bitfield mask")
__bad_mask(void);
static __always_inline u64 field_multiplier(u64 field)
{
if ((field | (field - 1)) & ((field | (field - 1)) + 1))
__bad_mask();
return field & -field;
}
static __always_inline u64 field_mask(u64 field)
{
return field / field_multiplier(field);
}
#define field_max(field) ((typeof(field))field_mask(field))
#define ____MAKE_OP(type,base,to,from) \
static __always_inline __##type type##_encode_bits(base v, base field) \
{ \
if (__builtin_constant_p(v) && (v & ~field_mask(field))) \
__field_overflow(); \
return to((v & field_mask(field)) * field_multiplier(field)); \
} \
static __always_inline __##type type##_replace_bits(__##type old, \
base val, base field) \
{ \
return (old & ~to(field)) | type##_encode_bits(val, field); \
} \
static __always_inline void type##p_replace_bits(__##type *p, \
base val, base field) \
{ \
*p = (*p & ~to(field)) | type##_encode_bits(val, field); \
} \
static __always_inline base type##_get_bits(__##type v, base field) \
{ \
return (from(v) & field)/field_multiplier(field); \
}
#define __MAKE_OP(size) \
____MAKE_OP(le##size,u##size,cpu_to_le##size,le##size##_to_cpu) \
____MAKE_OP(be##size,u##size,cpu_to_be##size,be##size##_to_cpu) \
____MAKE_OP(u##size,u##size,,)
____MAKE_OP(u8,u8,,)
__MAKE_OP(16)
__MAKE_OP(32)
__MAKE_OP(64)
#undef __MAKE_OP
#undef ____MAKE_OP
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM printk
#if !defined(_TRACE_PRINTK_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PRINTK_H
#include <linux/tracepoint.h>
TRACE_EVENT(console,
TP_PROTO(const char *text, size_t len),
TP_ARGS(text, len),
TP_STRUCT__entry(
__dynamic_array(char, msg, len + 1)
),
TP_fast_assign(
/*
* Each trace entry is printed in a new line.
* If the msg finishes with '\n', cut it off
* to avoid blank lines in the trace.
*/
if ((len > 0) && (text[len-1] == '\n'))
len -= 1;
memcpy(__get_str(msg), text, len);
__get_str(msg)[len] = 0;
),
TP_printk("%s", __get_str(msg))
);
#endif /* _TRACE_PRINTK_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_WAIT_H
#define _LINUX_WAIT_H
/*
* Linux wait queue related types and methods
*/
#include <linux/list.h>
#include <linux/stddef.h>
#include <linux/spinlock.h>
#include <asm/current.h>
typedef struct wait_queue_entry wait_queue_entry_t;
typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
/* wait_queue_entry::flags */
#define WQ_FLAG_EXCLUSIVE 0x01
#define WQ_FLAG_WOKEN 0x02
#define WQ_FLAG_CUSTOM 0x04
#define WQ_FLAG_DONE 0x08
#define WQ_FLAG_PRIORITY 0x10
/*
* A single wait-queue entry structure:
*/
struct wait_queue_entry {
unsigned int flags;
void *private;
wait_queue_func_t func;
struct list_head entry;
};
struct wait_queue_head {
spinlock_t lock;
struct list_head head;
};
typedef struct wait_queue_head wait_queue_head_t;
struct task_struct;
/*
* Macros for declaration and initialisaton of the datatypes
*/
#define __WAITQUEUE_INITIALIZER(name, tsk) { \
.private = tsk, \
.func = default_wake_function, \
.entry = { NULL, NULL } }
#define DECLARE_WAITQUEUE(name, tsk) \
struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
#define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.head = LIST_HEAD_INIT(name.head) }
#define DECLARE_WAIT_QUEUE_HEAD(name) \
struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
#define init_waitqueue_head(wq_head) \
do { \
static struct lock_class_key __key; \
\
__init_waitqueue_head((wq_head), #wq_head, &__key); \
} while (0)
#ifdef CONFIG_LOCKDEP
# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
({ init_waitqueue_head(&name); name; })
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
#else
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
#endif
static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
{
wq_entry->flags = 0;
wq_entry->private = p;
wq_entry->func = default_wake_function;
}
static inline void
init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
{
wq_entry->flags = 0;
wq_entry->private = NULL;
wq_entry->func = func;
}
/**
* waitqueue_active -- locklessly test for waiters on the queue
* @wq_head: the waitqueue to test for waiters
*
* returns true if the wait list is not empty
*
* NOTE: this function is lockless and requires care, incorrect usage _will_
* lead to sporadic and non-obvious failure.
*
* Use either while holding wait_queue_head::lock or when used for wakeups
* with an extra smp_mb() like::
*
* CPU0 - waker CPU1 - waiter
*
* for (;;) {
* @cond = true; prepare_to_wait(&wq_head, &wait, state);
* smp_mb(); // smp_mb() from set_current_state()
* if (waitqueue_active(wq_head)) if (@cond)
* wake_up(wq_head); break;
* schedule();
* }
* finish_wait(&wq_head, &wait);
*
* Because without the explicit smp_mb() it's possible for the
* waitqueue_active() load to get hoisted over the @cond store such that we'll
* observe an empty wait list while the waiter might not observe @cond.
*
* Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
* which (when the lock is uncontended) are of roughly equal cost.
*/
static inline int waitqueue_active(struct wait_queue_head *wq_head)
{
return !list_empty(&wq_head->head);
}
/**
* wq_has_single_sleeper - check if there is only one sleeper
* @wq_head: wait queue head
*
* Returns true of wq_head has only one sleeper on the list.
*
* Please refer to the comment for waitqueue_active.
*/
static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
{
return list_is_singular(&wq_head->head);
}
/**
* wq_has_sleeper - check if there are any waiting processes
* @wq_head: wait queue head
*
* Returns true if wq_head has waiting processes
*
* Please refer to the comment for waitqueue_active.
*/
static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
{
/*
* We need to be sure we are in sync with the
* add_wait_queue modifications to the wait queue.
*
* This memory barrier should be paired with one on the
* waiting side.
*/
smp_mb();
return waitqueue_active(wq_head);
}
extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
struct list_head *head = &wq_head->head;
struct wait_queue_entry *wq;
list_for_each_entry(wq, &wq_head->head, entry) {
if (!(wq->flags & WQ_FLAG_PRIORITY))
break;
head = &wq->entry;
}
list_add(&wq_entry->entry, head);
}
/*
* Used for wake-one threads:
*/
static inline void
__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
__add_wait_queue(wq_head, wq_entry);
}
static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
list_add_tail(&wq_entry->entry, &wq_head->head);
}
static inline void
__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
__add_wait_queue_entry_tail(wq_head, wq_entry);
}
static inline void
__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
list_del(&wq_entry->entry);
}
int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
void __wake_up_pollfree(struct wait_queue_head *wq_head);
#define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
#define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
#define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
#define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
#define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
#define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
#define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
#define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
#define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
/*
* Wakeup macros to be used to report events to the targets.
*/
#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
#define wake_up_poll(x, m) \
__wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
#define wake_up_poll_on_current_cpu(x, m) \
__wake_up_on_current_cpu(x, TASK_NORMAL, poll_to_key(m))
#define wake_up_locked_poll(x, m) \
__wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
#define wake_up_interruptible_poll(x, m) \
__wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
#define wake_up_interruptible_sync_poll(x, m) \
__wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
#define wake_up_interruptible_sync_poll_locked(x, m) \
__wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
/**
* wake_up_pollfree - signal that a polled waitqueue is going away
* @wq_head: the wait queue head
*
* In the very rare cases where a ->poll() implementation uses a waitqueue whose
* lifetime is tied to a task rather than to the 'struct file' being polled,
* this function must be called before the waitqueue is freed so that
* non-blocking polls (e.g. epoll) are notified that the queue is going away.
*
* The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
* an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
*/
static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
{
/*
* For performance reasons, we don't always take the queue lock here.
* Therefore, we might race with someone removing the last entry from
* the queue, and proceed while they still hold the queue lock.
* However, rcu_read_lock() is required to be held in such cases, so we
* can safely proceed with an RCU-delayed free.
*/
if (waitqueue_active(wq_head))
__wake_up_pollfree(wq_head);
}
#define ___wait_cond_timeout(condition) \
({ \
bool __cond = (condition); \
if (__cond && !__ret) \
__ret = 1; \
__cond || !__ret; \
})
#define ___wait_is_interruptible(state) \
(!__builtin_constant_p(state) || \
(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
/*
* The below macro ___wait_event() has an explicit shadow of the __ret
* variable when used from the wait_event_*() macros.
*
* This is so that both can use the ___wait_cond_timeout() construct
* to wrap the condition.
*
* The type inconsistency of the wait_event_*() __ret variable is also
* on purpose; we use long where we can return timeout values and int
* otherwise.
*/
#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
({ \
__label__ __out; \
struct wait_queue_entry __wq_entry; \
long __ret = ret; /* explicit shadow */ \
\
init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
for (;;) { \
long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
\
if (condition) \
break; \
\
if (___wait_is_interruptible(state) && __int) { \
__ret = __int; \
goto __out; \
} \
\
cmd; \
} \
finish_wait(&wq_head, &__wq_entry); \
__out: __ret; \
})
#define __wait_event(wq_head, condition) \
(void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
schedule())
/**
* wait_event - sleep until a condition gets true
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*/
#define wait_event(wq_head, condition) \
do { \
might_sleep(); \
if (condition) \
break; \
__wait_event(wq_head, condition); \
} while (0)
#define __io_wait_event(wq_head, condition) \
(void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
io_schedule())
/*
* io_wait_event() -- like wait_event() but with io_schedule()
*/
#define io_wait_event(wq_head, condition) \
do { \
might_sleep(); \
if (condition) \
break; \
__io_wait_event(wq_head, condition); \
} while (0)
#define __wait_event_freezable(wq_head, condition) \
___wait_event(wq_head, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), \
0, 0, schedule())
/**
* wait_event_freezable - sleep (or freeze) until a condition gets true
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
* to system load) until the @condition evaluates to true. The
* @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*/
#define wait_event_freezable(wq_head, condition) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_freezable(wq_head, condition); \
__ret; \
})
#define __wait_event_timeout(wq_head, condition, timeout) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
TASK_UNINTERRUPTIBLE, 0, timeout, \
__ret = schedule_timeout(__ret))
/**
* wait_event_timeout - sleep until a condition gets true or a timeout elapses
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, in jiffies
*
* The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* Returns:
* 0 if the @condition evaluated to %false after the @timeout elapsed,
* 1 if the @condition evaluated to %true after the @timeout elapsed,
* or the remaining jiffies (at least 1) if the @condition evaluated
* to %true before the @timeout elapsed.
*/
#define wait_event_timeout(wq_head, condition, timeout) \
({ \
long __ret = timeout; \
might_sleep(); \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_timeout(wq_head, condition, timeout); \
__ret; \
})
#define __wait_event_freezable_timeout(wq_head, condition, timeout) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
(TASK_INTERRUPTIBLE|TASK_FREEZABLE), 0, timeout, \
__ret = schedule_timeout(__ret))
/*
* like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
* increasing load and is freezable.
*/
#define wait_event_freezable_timeout(wq_head, condition, timeout) \
({ \
long __ret = timeout; \
might_sleep(); \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
__ret; \
})
#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
(void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
cmd1; schedule(); cmd2)
/*
* Just like wait_event_cmd(), except it sets exclusive flag
*/
#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
do { \
if (condition) \
break; \
__wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
} while (0)
#define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
(void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
cmd1; schedule(); cmd2)
/**
* wait_event_cmd - sleep until a condition gets true
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @cmd1: the command will be executed before sleep
* @cmd2: the command will be executed after sleep
*
* The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*/
#define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
do { \
if (condition) \
break; \
__wait_event_cmd(wq_head, condition, cmd1, cmd2); \
} while (0)
#define __wait_event_interruptible(wq_head, condition) \
___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
schedule())
/**
* wait_event_interruptible - sleep until a condition gets true
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a
* signal and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible(wq_head, condition) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_interruptible(wq_head, condition); \
__ret; \
})
#define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
TASK_INTERRUPTIBLE, 0, timeout, \
__ret = schedule_timeout(__ret))
/**
* wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, in jiffies
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* Returns:
* 0 if the @condition evaluated to %false after the @timeout elapsed,
* 1 if the @condition evaluated to %true after the @timeout elapsed,
* the remaining jiffies (at least 1) if the @condition evaluated
* to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
* interrupted by a signal.
*/
#define wait_event_interruptible_timeout(wq_head, condition, timeout) \
({ \
long __ret = timeout; \
might_sleep(); \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_interruptible_timeout(wq_head, \
condition, timeout); \
__ret; \
})
#define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
({ \
int __ret = 0; \
struct hrtimer_sleeper __t; \
\
hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
HRTIMER_MODE_REL); \
if ((timeout) != KTIME_MAX) { \
hrtimer_set_expires_range_ns(&__t.timer, timeout, \
current->timer_slack_ns); \
hrtimer_sleeper_start_expires(&__t, HRTIMER_MODE_REL); \
} \
\
__ret = ___wait_event(wq_head, condition, state, 0, 0, \
if (!__t.task) { \
__ret = -ETIME; \
break; \
} \
schedule()); \
\
hrtimer_cancel(&__t.timer); \
destroy_hrtimer_on_stack(&__t.timer); \
__ret; \
})
/**
* wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, as a ktime_t
*
* The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function returns 0 if @condition became true, or -ETIME if the timeout
* elapsed.
*/
#define wait_event_hrtimeout(wq_head, condition, timeout) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
TASK_UNINTERRUPTIBLE); \
__ret; \
})
/**
* wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
* @wq: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, as a ktime_t
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function returns 0 if @condition became true, -ERESTARTSYS if it was
* interrupted by a signal, or -ETIME if the timeout elapsed.
*/
#define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
({ \
long __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_hrtimeout(wq, condition, timeout, \
TASK_INTERRUPTIBLE); \
__ret; \
})
#define __wait_event_interruptible_exclusive(wq, condition) \
___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
schedule())
#define wait_event_interruptible_exclusive(wq, condition) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_interruptible_exclusive(wq, condition); \
__ret; \
})
#define __wait_event_killable_exclusive(wq, condition) \
___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
schedule())
#define wait_event_killable_exclusive(wq, condition) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_killable_exclusive(wq, condition); \
__ret; \
})
#define __wait_event_freezable_exclusive(wq, condition) \
___wait_event(wq, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 1, 0,\
schedule())
#define wait_event_freezable_exclusive(wq, condition) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_freezable_exclusive(wq, condition); \
__ret; \
})
/**
* wait_event_idle - wait for a condition without contributing to system load
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_IDLE) until the
* @condition evaluates to true.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
*/
#define wait_event_idle(wq_head, condition) \
do { \
might_sleep(); \
if (!(condition)) \
___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
} while (0)
/**
* wait_event_idle_exclusive - wait for a condition with contributing to system load
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_IDLE) until the
* @condition evaluates to true.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
* set thus if other processes wait on the same list, when this
* process is woken further processes are not considered.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
*/
#define wait_event_idle_exclusive(wq_head, condition) \
do { \
might_sleep(); \
if (!(condition)) \
___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
} while (0)
#define __wait_event_idle_timeout(wq_head, condition, timeout) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
TASK_IDLE, 0, timeout, \
__ret = schedule_timeout(__ret))
/**
* wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, in jiffies
*
* The process is put to sleep (TASK_IDLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* Returns:
* 0 if the @condition evaluated to %false after the @timeout elapsed,
* 1 if the @condition evaluated to %true after the @timeout elapsed,
* or the remaining jiffies (at least 1) if the @condition evaluated
* to %true before the @timeout elapsed.
*/
#define wait_event_idle_timeout(wq_head, condition, timeout) \
({ \
long __ret = timeout; \
might_sleep(); \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
__ret; \
})
#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
TASK_IDLE, 1, timeout, \
__ret = schedule_timeout(__ret))
/**
* wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, in jiffies
*
* The process is put to sleep (TASK_IDLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
* set thus if other processes wait on the same list, when this
* process is woken further processes are not considered.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* Returns:
* 0 if the @condition evaluated to %false after the @timeout elapsed,
* 1 if the @condition evaluated to %true after the @timeout elapsed,
* or the remaining jiffies (at least 1) if the @condition evaluated
* to %true before the @timeout elapsed.
*/
#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
({ \
long __ret = timeout; \
might_sleep(); \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
__ret; \
})
extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
#define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
({ \
int __ret; \
DEFINE_WAIT(__wait); \
if (exclusive) \
__wait.flags |= WQ_FLAG_EXCLUSIVE; \
do { \
__ret = fn(&(wq), &__wait); \
if (__ret) \
break; \
} while (!(condition)); \
__remove_wait_queue(&(wq), &__wait); \
__set_current_state(TASK_RUNNING); \
__ret; \
})
/**
* wait_event_interruptible_locked - sleep until a condition gets true
* @wq: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq is woken up.
*
* It must be called with wq.lock being held. This spinlock is
* unlocked while sleeping but @condition testing is done while lock
* is held and when this macro exits the lock is held.
*
* The lock is locked/unlocked using spin_lock()/spin_unlock()
* functions which must match the way they are locked/unlocked outside
* of this macro.
*
* wake_up_locked() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a
* signal and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible_locked(wq, condition) \
((condition) \
? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
/**
* wait_event_interruptible_locked_irq - sleep until a condition gets true
* @wq: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq is woken up.
*
* It must be called with wq.lock being held. This spinlock is
* unlocked while sleeping but @condition testing is done while lock
* is held and when this macro exits the lock is held.
*
* The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
* functions which must match the way they are locked/unlocked outside
* of this macro.
*
* wake_up_locked() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a
* signal and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible_locked_irq(wq, condition) \
((condition) \
? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
/**
* wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
* @wq: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq is woken up.
*
* It must be called with wq.lock being held. This spinlock is
* unlocked while sleeping but @condition testing is done while lock
* is held and when this macro exits the lock is held.
*
* The lock is locked/unlocked using spin_lock()/spin_unlock()
* functions which must match the way they are locked/unlocked outside
* of this macro.
*
* The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
* set thus when other process waits process on the list if this
* process is awaken further processes are not considered.
*
* wake_up_locked() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a
* signal and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible_exclusive_locked(wq, condition) \
((condition) \
? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
/**
* wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
* @wq: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq is woken up.
*
* It must be called with wq.lock being held. This spinlock is
* unlocked while sleeping but @condition testing is done while lock
* is held and when this macro exits the lock is held.
*
* The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
* functions which must match the way they are locked/unlocked outside
* of this macro.
*
* The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
* set thus when other process waits process on the list if this
* process is awaken further processes are not considered.
*
* wake_up_locked() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a
* signal and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
((condition) \
? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
#define __wait_event_killable(wq, condition) \
___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
/**
* wait_event_killable - sleep until a condition gets true
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
*
* The process is put to sleep (TASK_KILLABLE) until the
* @condition evaluates to true or a signal is received.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a
* signal and 0 if @condition evaluated to true.
*/
#define wait_event_killable(wq_head, condition) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_killable(wq_head, condition); \
__ret; \
})
#define __wait_event_state(wq, condition, state) \
___wait_event(wq, condition, state, 0, 0, schedule())
/**
* wait_event_state - sleep until a condition gets true
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @state: state to sleep in
*
* The process is put to sleep (@state) until the @condition evaluates to true
* or a signal is received (when allowed by @state). The @condition is checked
* each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* The function will return -ERESTARTSYS if it was interrupted by a signal
* (when allowed by @state) and 0 if @condition evaluated to true.
*/
#define wait_event_state(wq_head, condition, state) \
({ \
int __ret = 0; \
might_sleep(); \
if (!(condition)) \
__ret = __wait_event_state(wq_head, condition, state); \
__ret; \
})
#define __wait_event_killable_timeout(wq_head, condition, timeout) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
TASK_KILLABLE, 0, timeout, \
__ret = schedule_timeout(__ret))
/**
* wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @timeout: timeout, in jiffies
*
* The process is put to sleep (TASK_KILLABLE) until the
* @condition evaluates to true or a kill signal is received.
* The @condition is checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* Returns:
* 0 if the @condition evaluated to %false after the @timeout elapsed,
* 1 if the @condition evaluated to %true after the @timeout elapsed,
* the remaining jiffies (at least 1) if the @condition evaluated
* to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
* interrupted by a kill signal.
*
* Only kill signals interrupt this process.
*/
#define wait_event_killable_timeout(wq_head, condition, timeout) \
({ \
long __ret = timeout; \
might_sleep(); \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_killable_timeout(wq_head, \
condition, timeout); \
__ret; \
})
#define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
(void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
spin_unlock_irq(&lock); \
cmd; \
schedule(); \
spin_lock_irq(&lock))
/**
* wait_event_lock_irq_cmd - sleep until a condition gets true. The
* condition is checked under the lock. This
* is expected to be called with the lock
* taken.
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @lock: a locked spinlock_t, which will be released before cmd
* and schedule() and reacquired afterwards.
* @cmd: a command which is invoked outside the critical section before
* sleep
*
* The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* This is supposed to be called while holding the lock. The lock is
* dropped before invoking the cmd and going to sleep and is reacquired
* afterwards.
*/
#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
do { \
if (condition) \
break; \
__wait_event_lock_irq(wq_head, condition, lock, cmd); \
} while (0)
/**
* wait_event_lock_irq - sleep until a condition gets true. The
* condition is checked under the lock. This
* is expected to be called with the lock
* taken.
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @lock: a locked spinlock_t, which will be released before schedule()
* and reacquired afterwards.
*
* The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
* @condition evaluates to true. The @condition is checked each time
* the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* This is supposed to be called while holding the lock. The lock is
* dropped before going to sleep and is reacquired afterwards.
*/
#define wait_event_lock_irq(wq_head, condition, lock) \
do { \
if (condition) \
break; \
__wait_event_lock_irq(wq_head, condition, lock, ); \
} while (0)
#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
spin_unlock_irq(&lock); \
cmd; \
schedule(); \
spin_lock_irq(&lock))
/**
* wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
* The condition is checked under the lock. This is expected to
* be called with the lock taken.
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @lock: a locked spinlock_t, which will be released before cmd and
* schedule() and reacquired afterwards.
* @cmd: a command which is invoked outside the critical section before
* sleep
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or a signal is received. The @condition is
* checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* This is supposed to be called while holding the lock. The lock is
* dropped before invoking the cmd and going to sleep and is reacquired
* afterwards.
*
* The macro will return -ERESTARTSYS if it was interrupted by a signal
* and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
({ \
int __ret = 0; \
if (!(condition)) \
__ret = __wait_event_interruptible_lock_irq(wq_head, \
condition, lock, cmd); \
__ret; \
})
/**
* wait_event_interruptible_lock_irq - sleep until a condition gets true.
* The condition is checked under the lock. This is expected
* to be called with the lock taken.
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @lock: a locked spinlock_t, which will be released before schedule()
* and reacquired afterwards.
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or signal is received. The @condition is
* checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* This is supposed to be called while holding the lock. The lock is
* dropped before going to sleep and is reacquired afterwards.
*
* The macro will return -ERESTARTSYS if it was interrupted by a signal
* and 0 if @condition evaluated to true.
*/
#define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
({ \
int __ret = 0; \
if (!(condition)) \
__ret = __wait_event_interruptible_lock_irq(wq_head, \
condition, lock,); \
__ret; \
})
#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
___wait_event(wq_head, ___wait_cond_timeout(condition), \
state, 0, timeout, \
spin_unlock_irq(&lock); \
__ret = schedule_timeout(__ret); \
spin_lock_irq(&lock));
/**
* wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
* true or a timeout elapses. The condition is checked under
* the lock. This is expected to be called with the lock taken.
* @wq_head: the waitqueue to wait on
* @condition: a C expression for the event to wait for
* @lock: a locked spinlock_t, which will be released before schedule()
* and reacquired afterwards.
* @timeout: timeout, in jiffies
*
* The process is put to sleep (TASK_INTERRUPTIBLE) until the
* @condition evaluates to true or signal is received. The @condition is
* checked each time the waitqueue @wq_head is woken up.
*
* wake_up() has to be called after changing any variable that could
* change the result of the wait condition.
*
* This is supposed to be called while holding the lock. The lock is
* dropped before going to sleep and is reacquired afterwards.
*
* The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
* was interrupted by a signal, and the remaining jiffies otherwise
* if the condition evaluated to true before the timeout elapsed.
*/
#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
timeout) \
({ \
long __ret = timeout; \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_lock_irq_timeout( \
wq_head, condition, lock, timeout, \
TASK_INTERRUPTIBLE); \
__ret; \
})
#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
({ \
long __ret = timeout; \
if (!___wait_cond_timeout(condition)) \
__ret = __wait_event_lock_irq_timeout( \
wq_head, condition, lock, timeout, \
TASK_UNINTERRUPTIBLE); \
__ret; \
})
/*
* Waitqueues which are removed from the waitqueue_head at wakeup time
*/
void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
#define DEFINE_WAIT_FUNC(name, function) \
struct wait_queue_entry name = { \
.private = current, \
.func = function, \
.entry = LIST_HEAD_INIT((name).entry), \
}
#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
#define init_wait(wait) \
do { \
(wait)->private = current; \
(wait)->func = autoremove_wake_function; \
INIT_LIST_HEAD(&(wait)->entry); \
(wait)->flags = 0; \
} while (0)
typedef int (*task_call_f)(struct task_struct *p, void *arg);
extern int task_call_func(struct task_struct *p, task_call_f func, void *arg);
#endif /* _LINUX_WAIT_H */
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/memblock.h>
#include <linux/page_ext.h>
#include <linux/memory.h>
#include <linux/vmalloc.h>
#include <linux/kmemleak.h>
#include <linux/page_owner.h>
#include <linux/page_idle.h>
#include <linux/page_table_check.h>
#include <linux/rcupdate.h>
/*
* struct page extension
*
* This is the feature to manage memory for extended data per page.
*
* Until now, we must modify struct page itself to store extra data per page.
* This requires rebuilding the kernel and it is really time consuming process.
* And, sometimes, rebuild is impossible due to third party module dependency.
* At last, enlarging struct page could cause un-wanted system behaviour change.
*
* This feature is intended to overcome above mentioned problems. This feature
* allocates memory for extended data per page in certain place rather than
* the struct page itself. This memory can be accessed by the accessor
* functions provided by this code. During the boot process, it checks whether
* allocation of huge chunk of memory is needed or not. If not, it avoids
* allocating memory at all. With this advantage, we can include this feature
* into the kernel in default and can avoid rebuild and solve related problems.
*
* To help these things to work well, there are two callbacks for clients. One
* is the need callback which is mandatory if user wants to avoid useless
* memory allocation at boot-time. The other is optional, init callback, which
* is used to do proper initialization after memory is allocated.
*
* The need callback is used to decide whether extended memory allocation is
* needed or not. Sometimes users want to deactivate some features in this
* boot and extra memory would be unnecessary. In this case, to avoid
* allocating huge chunk of memory, each clients represent their need of
* extra memory through the need callback. If one of the need callbacks
* returns true, it means that someone needs extra memory so that
* page extension core should allocates memory for page extension. If
* none of need callbacks return true, memory isn't needed at all in this boot
* and page extension core can skip to allocate memory. As result,
* none of memory is wasted.
*
* When need callback returns true, page_ext checks if there is a request for
* extra memory through size in struct page_ext_operations. If it is non-zero,
* extra space is allocated for each page_ext entry and offset is returned to
* user through offset in struct page_ext_operations.
*
* The init callback is used to do proper initialization after page extension
* is completely initialized. In sparse memory system, extra memory is
* allocated some time later than memmap is allocated. In other words, lifetime
* of memory for page extension isn't same with memmap for struct page.
* Therefore, clients can't store extra data until page extension is
* initialized, even if pages are allocated and used freely. This could
* cause inadequate state of extra data per page, so, to prevent it, client
* can utilize this callback to initialize the state of it correctly.
*/
#ifdef CONFIG_SPARSEMEM
#define PAGE_EXT_INVALID (0x1)
#endif
#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
static bool need_page_idle(void)
{
return true;
}
static struct page_ext_operations page_idle_ops __initdata = {
.need = need_page_idle,
.need_shared_flags = true,
};
#endif
static struct page_ext_operations *page_ext_ops[] __initdata = {
#ifdef CONFIG_PAGE_OWNER
&page_owner_ops,
#endif
#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
&page_idle_ops,
#endif
#ifdef CONFIG_PAGE_TABLE_CHECK
&page_table_check_ops,
#endif
};
unsigned long page_ext_size;
static unsigned long total_usage;
bool early_page_ext __meminitdata;
static int __init setup_early_page_ext(char *str)
{
early_page_ext = true;
return 0;
}
early_param("early_page_ext", setup_early_page_ext);
static bool __init invoke_need_callbacks(void)
{
int i;
int entries = ARRAY_SIZE(page_ext_ops);
bool need = false;
for (i = 0; i < entries; i++) {
if (page_ext_ops[i]->need()) {
if (page_ext_ops[i]->need_shared_flags) {
page_ext_size = sizeof(struct page_ext);
break;
}
}
}
for (i = 0; i < entries; i++) {
if (page_ext_ops[i]->need()) {
page_ext_ops[i]->offset = page_ext_size;
page_ext_size += page_ext_ops[i]->size;
need = true;
}
}
return need;
}
static void __init invoke_init_callbacks(void)
{
int i;
int entries = ARRAY_SIZE(page_ext_ops);
for (i = 0; i < entries; i++) {
if (page_ext_ops[i]->init)
page_ext_ops[i]->init();
}
}
static inline struct page_ext *get_entry(void *base, unsigned long index)
{
return base + page_ext_size * index;
}
#ifndef CONFIG_SPARSEMEM
void __init page_ext_init_flatmem_late(void)
{
invoke_init_callbacks();
}
void __meminit pgdat_page_ext_init(struct pglist_data *pgdat)
{
pgdat->node_page_ext = NULL;
}
static struct page_ext *lookup_page_ext(const struct page *page)
{
unsigned long pfn = page_to_pfn(page);
unsigned long index;
struct page_ext *base;
WARN_ON_ONCE(!rcu_read_lock_held());
base = NODE_DATA(page_to_nid(page))->node_page_ext;
/*
* The sanity checks the page allocator does upon freeing a
* page can reach here before the page_ext arrays are
* allocated when feeding a range of pages to the allocator
* for the first time during bootup or memory hotplug.
*/
if (unlikely(!base))
return NULL;
index = pfn - round_down(node_start_pfn(page_to_nid(page)),
MAX_ORDER_NR_PAGES);
return get_entry(base, index);
}
static int __init alloc_node_page_ext(int nid)
{
struct page_ext *base;
unsigned long table_size;
unsigned long nr_pages;
nr_pages = NODE_DATA(nid)->node_spanned_pages;
if (!nr_pages)
return 0;
/*
* Need extra space if node range is not aligned with
* MAX_ORDER_NR_PAGES. When page allocator's buddy algorithm
* checks buddy's status, range could be out of exact node range.
*/
if (!IS_ALIGNED(node_start_pfn(nid), MAX_ORDER_NR_PAGES) ||
!IS_ALIGNED(node_end_pfn(nid), MAX_ORDER_NR_PAGES))
nr_pages += MAX_ORDER_NR_PAGES;
table_size = page_ext_size * nr_pages;
base = memblock_alloc_try_nid(
table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS),
MEMBLOCK_ALLOC_ACCESSIBLE, nid);
if (!base)
return -ENOMEM;
NODE_DATA(nid)->node_page_ext = base;
total_usage += table_size;
return 0;
}
void __init page_ext_init_flatmem(void)
{
int nid, fail;
if (!invoke_need_callbacks())
return;
for_each_online_node(nid) {
fail = alloc_node_page_ext(nid);
if (fail)
goto fail;
}
pr_info("allocated %ld bytes of page_ext\n", total_usage);
return;
fail:
pr_crit("allocation of page_ext failed.\n");
panic("Out of memory");
}
#else /* CONFIG_SPARSEMEM */
static bool page_ext_invalid(struct page_ext *page_ext)
{
return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
}
static struct page_ext *lookup_page_ext(const struct page *page)
{
unsigned long pfn = page_to_pfn(page); struct mem_section *section = __pfn_to_section(pfn); struct page_ext *page_ext = READ_ONCE(section->page_ext); WARN_ON_ONCE(!rcu_read_lock_held());
/*
* The sanity checks the page allocator does upon freeing a
* page can reach here before the page_ext arrays are
* allocated when feeding a range of pages to the allocator
* for the first time during bootup or memory hotplug.
*/
if (page_ext_invalid(page_ext))
return NULL;
return get_entry(page_ext, pfn);
}
static void *__meminit alloc_page_ext(size_t size, int nid)
{
gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN;
void *addr = NULL;
addr = alloc_pages_exact_nid(nid, size, flags);
if (addr) {
kmemleak_alloc(addr, size, 1, flags);
return addr;
}
addr = vzalloc_node(size, nid);
return addr;
}
static int __meminit init_section_page_ext(unsigned long pfn, int nid)
{
struct mem_section *section;
struct page_ext *base;
unsigned long table_size;
section = __pfn_to_section(pfn);
if (section->page_ext)
return 0;
table_size = page_ext_size * PAGES_PER_SECTION;
base = alloc_page_ext(table_size, nid);
/*
* The value stored in section->page_ext is (base - pfn)
* and it does not point to the memory block allocated above,
* causing kmemleak false positives.
*/
kmemleak_not_leak(base);
if (!base) {
pr_err("page ext allocation failure\n");
return -ENOMEM;
}
/*
* The passed "pfn" may not be aligned to SECTION. For the calculation
* we need to apply a mask.
*/
pfn &= PAGE_SECTION_MASK;
section->page_ext = (void *)base - page_ext_size * pfn;
total_usage += table_size;
return 0;
}
static void free_page_ext(void *addr)
{
if (is_vmalloc_addr(addr)) {
vfree(addr);
} else {
struct page *page = virt_to_page(addr);
size_t table_size;
table_size = page_ext_size * PAGES_PER_SECTION;
BUG_ON(PageReserved(page));
kmemleak_free(addr);
free_pages_exact(addr, table_size);
}
}
static void __free_page_ext(unsigned long pfn)
{
struct mem_section *ms;
struct page_ext *base;
ms = __pfn_to_section(pfn);
if (!ms || !ms->page_ext)
return;
base = READ_ONCE(ms->page_ext);
/*
* page_ext here can be valid while doing the roll back
* operation in online_page_ext().
*/
if (page_ext_invalid(base))
base = (void *)base - PAGE_EXT_INVALID;
WRITE_ONCE(ms->page_ext, NULL);
base = get_entry(base, pfn);
free_page_ext(base);
}
static void __invalidate_page_ext(unsigned long pfn)
{
struct mem_section *ms;
void *val;
ms = __pfn_to_section(pfn);
if (!ms || !ms->page_ext)
return;
val = (void *)ms->page_ext + PAGE_EXT_INVALID;
WRITE_ONCE(ms->page_ext, val);
}
static int __meminit online_page_ext(unsigned long start_pfn,
unsigned long nr_pages,
int nid)
{
unsigned long start, end, pfn;
int fail = 0;
start = SECTION_ALIGN_DOWN(start_pfn);
end = SECTION_ALIGN_UP(start_pfn + nr_pages);
if (nid == NUMA_NO_NODE) {
/*
* In this case, "nid" already exists and contains valid memory.
* "start_pfn" passed to us is a pfn which is an arg for
* online__pages(), and start_pfn should exist.
*/
nid = pfn_to_nid(start_pfn);
VM_BUG_ON(!node_online(nid));
}
for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION)
fail = init_section_page_ext(pfn, nid);
if (!fail)
return 0;
/* rollback */
end = pfn - PAGES_PER_SECTION;
for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
__free_page_ext(pfn);
return -ENOMEM;
}
static void __meminit offline_page_ext(unsigned long start_pfn,
unsigned long nr_pages)
{
unsigned long start, end, pfn;
start = SECTION_ALIGN_DOWN(start_pfn);
end = SECTION_ALIGN_UP(start_pfn + nr_pages);
/*
* Freeing of page_ext is done in 3 steps to avoid
* use-after-free of it:
* 1) Traverse all the sections and mark their page_ext
* as invalid.
* 2) Wait for all the existing users of page_ext who
* started before invalidation to finish.
* 3) Free the page_ext.
*/
for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
__invalidate_page_ext(pfn);
synchronize_rcu();
for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
__free_page_ext(pfn);
}
static int __meminit page_ext_callback(struct notifier_block *self,
unsigned long action, void *arg)
{
struct memory_notify *mn = arg;
int ret = 0;
switch (action) {
case MEM_GOING_ONLINE:
ret = online_page_ext(mn->start_pfn,
mn->nr_pages, mn->status_change_nid);
break;
case MEM_OFFLINE:
offline_page_ext(mn->start_pfn,
mn->nr_pages);
break;
case MEM_CANCEL_ONLINE:
offline_page_ext(mn->start_pfn,
mn->nr_pages);
break;
case MEM_GOING_OFFLINE:
break;
case MEM_ONLINE:
case MEM_CANCEL_OFFLINE:
break;
}
return notifier_from_errno(ret);
}
void __init page_ext_init(void)
{
unsigned long pfn;
int nid;
if (!invoke_need_callbacks())
return;
for_each_node_state(nid, N_MEMORY) {
unsigned long start_pfn, end_pfn;
start_pfn = node_start_pfn(nid);
end_pfn = node_end_pfn(nid);
/*
* start_pfn and end_pfn may not be aligned to SECTION and the
* page->flags of out of node pages are not initialized. So we
* scan [start_pfn, the biggest section's pfn < end_pfn) here.
*/
for (pfn = start_pfn; pfn < end_pfn;
pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {
if (!pfn_valid(pfn))
continue;
/*
* Nodes's pfns can be overlapping.
* We know some arch can have a nodes layout such as
* -------------pfn-------------->
* N0 | N1 | N2 | N0 | N1 | N2|....
*/
if (pfn_to_nid(pfn) != nid)
continue;
if (init_section_page_ext(pfn, nid))
goto oom;
cond_resched();
}
}
hotplug_memory_notifier(page_ext_callback, DEFAULT_CALLBACK_PRI);
pr_info("allocated %ld bytes of page_ext\n", total_usage);
invoke_init_callbacks();
return;
oom:
panic("Out of memory");
}
void __meminit pgdat_page_ext_init(struct pglist_data *pgdat)
{
}
#endif
/**
* page_ext_get() - Get the extended information for a page.
* @page: The page we're interested in.
*
* Ensures that the page_ext will remain valid until page_ext_put()
* is called.
*
* Return: NULL if no page_ext exists for this page.
* Context: Any context. Caller may not sleep until they have called
* page_ext_put().
*/
struct page_ext *page_ext_get(struct page *page)
{
struct page_ext *page_ext;
rcu_read_lock(); page_ext = lookup_page_ext(page); if (!page_ext) { rcu_read_unlock();
return NULL;
}
return page_ext;
}
/**
* page_ext_put() - Working with page extended information is done.
* @page_ext: Page extended information received from page_ext_get().
*
* The page extended information of the page may not be valid after this
* function is called.
*
* Return: None.
* Context: Any context with corresponding page_ext_get() is called.
*/
void page_ext_put(struct page_ext *page_ext)
{
if (unlikely(!page_ext))
return;
rcu_read_unlock();
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* keyspan_remote: USB driver for the Keyspan DMR
*
* Copyright (C) 2005 Zymeta Corporation - Michael Downey (downey@zymeta.com)
*
* This driver has been put together with the support of Innosys, Inc.
* and Keyspan, Inc the manufacturers of the Keyspan USB DMR product.
*/
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/usb/input.h>
/* Parameters that can be passed to the driver. */
static int debug;
module_param(debug, int, 0444);
MODULE_PARM_DESC(debug, "Enable extra debug messages and information");
/* Vendor and product ids */
#define USB_KEYSPAN_VENDOR_ID 0x06CD
#define USB_KEYSPAN_PRODUCT_UIA11 0x0202
/* Defines for converting the data from the remote. */
#define ZERO 0x18
#define ZERO_MASK 0x1F /* 5 bits for a 0 */
#define ONE 0x3C
#define ONE_MASK 0x3F /* 6 bits for a 1 */
#define SYNC 0x3F80
#define SYNC_MASK 0x3FFF /* 14 bits for a SYNC sequence */
#define STOP 0x00
#define STOP_MASK 0x1F /* 5 bits for the STOP sequence */
#define GAP 0xFF
#define RECV_SIZE 8 /* The UIA-11 type have a 8 byte limit. */
/*
* Table that maps the 31 possible keycodes to input keys.
* Currently there are 15 and 17 button models so RESERVED codes
* are blank areas in the mapping.
*/
static const unsigned short keyspan_key_table[] = {
KEY_RESERVED, /* 0 is just a place holder. */
KEY_RESERVED,
KEY_STOP,
KEY_PLAYCD,
KEY_RESERVED,
KEY_PREVIOUSSONG,
KEY_REWIND,
KEY_FORWARD,
KEY_NEXTSONG,
KEY_RESERVED,
KEY_RESERVED,
KEY_RESERVED,
KEY_PAUSE,
KEY_VOLUMEUP,
KEY_RESERVED,
KEY_RESERVED,
KEY_RESERVED,
KEY_VOLUMEDOWN,
KEY_RESERVED,
KEY_UP,
KEY_RESERVED,
KEY_MUTE,
KEY_LEFT,
KEY_ENTER,
KEY_RIGHT,
KEY_RESERVED,
KEY_RESERVED,
KEY_DOWN,
KEY_RESERVED,
KEY_KPASTERISK,
KEY_RESERVED,
KEY_MENU
};
/* table of devices that work with this driver */
static const struct usb_device_id keyspan_table[] = {
{ USB_DEVICE(USB_KEYSPAN_VENDOR_ID, USB_KEYSPAN_PRODUCT_UIA11) },
{ } /* Terminating entry */
};
/* Structure to store all the real stuff that a remote sends to us. */
struct keyspan_message {
u16 system;
u8 button;
u8 toggle;
};
/* Structure used for all the bit testing magic needed to be done. */
struct bit_tester {
u32 tester;
int len;
int pos;
int bits_left;
u8 buffer[32];
};
/* Structure to hold all of our driver specific stuff */
struct usb_keyspan {
char name[128];
char phys[64];
unsigned short keymap[ARRAY_SIZE(keyspan_key_table)];
struct usb_device *udev;
struct input_dev *input;
struct usb_interface *interface;
struct usb_endpoint_descriptor *in_endpoint;
struct urb* irq_urb;
int open;
dma_addr_t in_dma;
unsigned char *in_buffer;
/* variables used to parse messages from remote. */
struct bit_tester data;
int stage;
int toggle;
};
static struct usb_driver keyspan_driver;
/*
* Debug routine that prints out what we've received from the remote.
*/
static void keyspan_print(struct usb_keyspan* dev) /*unsigned char* data)*/
{
char codes[4 * RECV_SIZE];
int i;
for (i = 0; i < RECV_SIZE; i++)
snprintf(codes + i * 3, 4, "%02x ", dev->in_buffer[i]);
dev_info(&dev->udev->dev, "%s\n", codes);
}
/*
* Routine that manages the bit_tester structure. It makes sure that there are
* at least bits_needed bits loaded into the tester.
*/
static int keyspan_load_tester(struct usb_keyspan* dev, int bits_needed)
{
if (dev->data.bits_left >= bits_needed)
return 0;
/*
* Somehow we've missed the last message. The message will be repeated
* though so it's not too big a deal
*/
if (dev->data.pos >= dev->data.len) {
dev_dbg(&dev->interface->dev,
"%s - Error ran out of data. pos: %d, len: %d\n",
__func__, dev->data.pos, dev->data.len);
return -1;
}
/* Load as much as we can into the tester. */
while ((dev->data.bits_left + 7 < (sizeof(dev->data.tester) * 8)) &&
(dev->data.pos < dev->data.len)) {
dev->data.tester += (dev->data.buffer[dev->data.pos++] << dev->data.bits_left);
dev->data.bits_left += 8;
}
return 0;
}
static void keyspan_report_button(struct usb_keyspan *remote, int button, int press)
{
struct input_dev *input = remote->input;
input_event(input, EV_MSC, MSC_SCAN, button);
input_report_key(input, remote->keymap[button], press);
input_sync(input);
}
/*
* Routine that handles all the logic needed to parse out the message from the remote.
*/
static void keyspan_check_data(struct usb_keyspan *remote)
{
int i;
int found = 0;
struct keyspan_message message;
switch(remote->stage) {
case 0:
/*
* In stage 0 we want to find the start of a message. The remote sends a 0xFF as filler.
* So the first byte that isn't a FF should be the start of a new message.
*/
for (i = 0; i < RECV_SIZE && remote->in_buffer[i] == GAP; ++i);
if (i < RECV_SIZE) {
memcpy(remote->data.buffer, remote->in_buffer, RECV_SIZE);
remote->data.len = RECV_SIZE;
remote->data.pos = 0;
remote->data.tester = 0;
remote->data.bits_left = 0;
remote->stage = 1;
}
break;
case 1:
/*
* Stage 1 we should have 16 bytes and should be able to detect a
* SYNC. The SYNC is 14 bits, 7 0's and then 7 1's.
*/
memcpy(remote->data.buffer + remote->data.len, remote->in_buffer, RECV_SIZE);
remote->data.len += RECV_SIZE;
found = 0;
while ((remote->data.bits_left >= 14 || remote->data.pos < remote->data.len) && !found) {
for (i = 0; i < 8; ++i) {
if (keyspan_load_tester(remote, 14) != 0) {
remote->stage = 0;
return;
}
if ((remote->data.tester & SYNC_MASK) == SYNC) {
remote->data.tester = remote->data.tester >> 14;
remote->data.bits_left -= 14;
found = 1;
break;
} else {
remote->data.tester = remote->data.tester >> 1;
--remote->data.bits_left;
}
}
}
if (!found) {
remote->stage = 0;
remote->data.len = 0;
} else {
remote->stage = 2;
}
break;
case 2:
/*
* Stage 2 we should have 24 bytes which will be enough for a full
* message. We need to parse out the system code, button code,
* toggle code, and stop.
*/
memcpy(remote->data.buffer + remote->data.len, remote->in_buffer, RECV_SIZE);
remote->data.len += RECV_SIZE;
message.system = 0;
for (i = 0; i < 9; i++) {
keyspan_load_tester(remote, 6);
if ((remote->data.tester & ZERO_MASK) == ZERO) {
message.system = message.system << 1;
remote->data.tester = remote->data.tester >> 5;
remote->data.bits_left -= 5;
} else if ((remote->data.tester & ONE_MASK) == ONE) {
message.system = (message.system << 1) + 1;
remote->data.tester = remote->data.tester >> 6;
remote->data.bits_left -= 6;
} else {
dev_err(&remote->interface->dev,
"%s - Unknown sequence found in system data.\n",
__func__);
remote->stage = 0;
return;
}
}
message.button = 0;
for (i = 0; i < 5; i++) {
keyspan_load_tester(remote, 6);
if ((remote->data.tester & ZERO_MASK) == ZERO) {
message.button = message.button << 1;
remote->data.tester = remote->data.tester >> 5;
remote->data.bits_left -= 5;
} else if ((remote->data.tester & ONE_MASK) == ONE) {
message.button = (message.button << 1) + 1;
remote->data.tester = remote->data.tester >> 6;
remote->data.bits_left -= 6;
} else {
dev_err(&remote->interface->dev,
"%s - Unknown sequence found in button data.\n",
__func__);
remote->stage = 0;
return;
}
}
keyspan_load_tester(remote, 6);
if ((remote->data.tester & ZERO_MASK) == ZERO) {
message.toggle = 0;
remote->data.tester = remote->data.tester >> 5;
remote->data.bits_left -= 5;
} else if ((remote->data.tester & ONE_MASK) == ONE) {
message.toggle = 1;
remote->data.tester = remote->data.tester >> 6;
remote->data.bits_left -= 6;
} else {
dev_err(&remote->interface->dev,
"%s - Error in message, invalid toggle.\n",
__func__);
remote->stage = 0;
return;
}
keyspan_load_tester(remote, 5);
if ((remote->data.tester & STOP_MASK) == STOP) {
remote->data.tester = remote->data.tester >> 5;
remote->data.bits_left -= 5;
} else {
dev_err(&remote->interface->dev,
"Bad message received, no stop bit found.\n");
}
dev_dbg(&remote->interface->dev,
"%s found valid message: system: %d, button: %d, toggle: %d\n",
__func__, message.system, message.button, message.toggle);
if (message.toggle != remote->toggle) {
keyspan_report_button(remote, message.button, 1);
keyspan_report_button(remote, message.button, 0);
remote->toggle = message.toggle;
}
remote->stage = 0;
break;
}
}
/*
* Routine for sending all the initialization messages to the remote.
*/
static int keyspan_setup(struct usb_device* dev)
{
int retval = 0;
retval = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
0x11, 0x40, 0x5601, 0x0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
if (retval) {
dev_dbg(&dev->dev, "%s - failed to set bit rate due to error: %d\n",
__func__, retval);
return(retval);
}
retval = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
0x44, 0x40, 0x0, 0x0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
if (retval) {
dev_dbg(&dev->dev, "%s - failed to set resume sensitivity due to error: %d\n",
__func__, retval);
return(retval);
}
retval = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
0x22, 0x40, 0x0, 0x0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
if (retval) {
dev_dbg(&dev->dev, "%s - failed to turn receive on due to error: %d\n",
__func__, retval);
return(retval);
}
dev_dbg(&dev->dev, "%s - Setup complete.\n", __func__);
return(retval);
}
/*
* Routine used to handle a new message that has come in.
*/
static void keyspan_irq_recv(struct urb *urb)
{
struct usb_keyspan *dev = urb->context;
int retval;
/* Check our status in case we need to bail out early. */
switch (urb->status) {
case 0:
break;
/* Device went away so don't keep trying to read from it. */
case -ECONNRESET:
case -ENOENT:
case -ESHUTDOWN:
return;
default:
goto resubmit;
}
if (debug)
keyspan_print(dev);
keyspan_check_data(dev);
resubmit:
retval = usb_submit_urb(urb, GFP_ATOMIC);
if (retval)
dev_err(&dev->interface->dev,
"%s - usb_submit_urb failed with result: %d\n",
__func__, retval);
}
static int keyspan_open(struct input_dev *dev)
{
struct usb_keyspan *remote = input_get_drvdata(dev);
remote->irq_urb->dev = remote->udev;
if (usb_submit_urb(remote->irq_urb, GFP_KERNEL))
return -EIO;
return 0;
}
static void keyspan_close(struct input_dev *dev)
{
struct usb_keyspan *remote = input_get_drvdata(dev);
usb_kill_urb(remote->irq_urb);
}
static struct usb_endpoint_descriptor *keyspan_get_in_endpoint(struct usb_host_interface *iface)
{
struct usb_endpoint_descriptor *endpoint;
int i;
for (i = 0; i < iface->desc.bNumEndpoints; ++i) {
endpoint = &iface->endpoint[i].desc;
if (usb_endpoint_is_int_in(endpoint)) {
/* we found our interrupt in endpoint */
return endpoint;
}
}
return NULL;
}
/*
* Routine that sets up the driver to handle a specific USB device detected on the bus.
*/
static int keyspan_probe(struct usb_interface *interface, const struct usb_device_id *id)
{
struct usb_device *udev = interface_to_usbdev(interface);
struct usb_endpoint_descriptor *endpoint;
struct usb_keyspan *remote;
struct input_dev *input_dev;
int i, error;
endpoint = keyspan_get_in_endpoint(interface->cur_altsetting);
if (!endpoint)
return -ENODEV;
remote = kzalloc(sizeof(*remote), GFP_KERNEL);
input_dev = input_allocate_device();
if (!remote || !input_dev) {
error = -ENOMEM;
goto fail1;
}
remote->udev = udev;
remote->input = input_dev;
remote->interface = interface;
remote->in_endpoint = endpoint;
remote->toggle = -1; /* Set to -1 so we will always not match the toggle from the first remote message. */
remote->in_buffer = usb_alloc_coherent(udev, RECV_SIZE, GFP_KERNEL, &remote->in_dma);
if (!remote->in_buffer) {
error = -ENOMEM;
goto fail1;
}
remote->irq_urb = usb_alloc_urb(0, GFP_KERNEL);
if (!remote->irq_urb) {
error = -ENOMEM;
goto fail2;
}
error = keyspan_setup(udev);
if (error) {
error = -ENODEV;
goto fail3;
}
if (udev->manufacturer)
strscpy(remote->name, udev->manufacturer, sizeof(remote->name));
if (udev->product) {
if (udev->manufacturer)
strlcat(remote->name, " ", sizeof(remote->name));
strlcat(remote->name, udev->product, sizeof(remote->name));
}
if (!strlen(remote->name))
snprintf(remote->name, sizeof(remote->name),
"USB Keyspan Remote %04x:%04x",
le16_to_cpu(udev->descriptor.idVendor),
le16_to_cpu(udev->descriptor.idProduct));
usb_make_path(udev, remote->phys, sizeof(remote->phys));
strlcat(remote->phys, "/input0", sizeof(remote->phys));
memcpy(remote->keymap, keyspan_key_table, sizeof(remote->keymap));
input_dev->name = remote->name;
input_dev->phys = remote->phys;
usb_to_input_id(udev, &input_dev->id);
input_dev->dev.parent = &interface->dev;
input_dev->keycode = remote->keymap;
input_dev->keycodesize = sizeof(unsigned short);
input_dev->keycodemax = ARRAY_SIZE(remote->keymap);
input_set_capability(input_dev, EV_MSC, MSC_SCAN);
__set_bit(EV_KEY, input_dev->evbit);
for (i = 0; i < ARRAY_SIZE(keyspan_key_table); i++)
__set_bit(keyspan_key_table[i], input_dev->keybit);
__clear_bit(KEY_RESERVED, input_dev->keybit);
input_set_drvdata(input_dev, remote);
input_dev->open = keyspan_open;
input_dev->close = keyspan_close;
/*
* Initialize the URB to access the device.
* The urb gets sent to the device in keyspan_open()
*/
usb_fill_int_urb(remote->irq_urb,
remote->udev,
usb_rcvintpipe(remote->udev, endpoint->bEndpointAddress),
remote->in_buffer, RECV_SIZE, keyspan_irq_recv, remote,
endpoint->bInterval);
remote->irq_urb->transfer_dma = remote->in_dma;
remote->irq_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
/* we can register the device now, as it is ready */
error = input_register_device(remote->input);
if (error)
goto fail3;
/* save our data pointer in this interface device */
usb_set_intfdata(interface, remote);
return 0;
fail3: usb_free_urb(remote->irq_urb);
fail2: usb_free_coherent(udev, RECV_SIZE, remote->in_buffer, remote->in_dma);
fail1: kfree(remote);
input_free_device(input_dev);
return error;
}
/*
* Routine called when a device is disconnected from the USB.
*/
static void keyspan_disconnect(struct usb_interface *interface)
{
struct usb_keyspan *remote;
remote = usb_get_intfdata(interface);
usb_set_intfdata(interface, NULL);
if (remote) { /* We have a valid driver structure so clean up everything we allocated. */
input_unregister_device(remote->input);
usb_kill_urb(remote->irq_urb);
usb_free_urb(remote->irq_urb);
usb_free_coherent(remote->udev, RECV_SIZE, remote->in_buffer, remote->in_dma);
kfree(remote);
}
}
/*
* Standard driver set up sections
*/
static struct usb_driver keyspan_driver =
{
.name = "keyspan_remote",
.probe = keyspan_probe,
.disconnect = keyspan_disconnect,
.id_table = keyspan_table
};
module_usb_driver(keyspan_driver);
MODULE_DEVICE_TABLE(usb, keyspan_table);
MODULE_AUTHOR("Michael Downey <downey@zymeta.com>");
MODULE_DESCRIPTION("Driver for the USB Keyspan remote control.");
MODULE_LICENSE("GPL");
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* INET An implementation of the TCP/IP protocol suite for the LINUX
* operating system. INET is implemented using the BSD Socket
* interface as the means of communication with the user level.
*
* Definitions for the Interfaces handler.
*
* Version: @(#)dev.h 1.0.10 08/12/93
*
* Authors: Ross Biro
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
* Corey Minyard <wf-rch!minyard@relay.EU.net>
* Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
* Alan Cox, <alan@lxorguk.ukuu.org.uk>
* Bjorn Ekwall. <bj0rn@blox.se>
* Pekka Riikonen <priikone@poseidon.pspt.fi>
*
* Moved to /usr/include/linux for NET3
*/
#ifndef _LINUX_NETDEVICE_H
#define _LINUX_NETDEVICE_H
#include <linux/timer.h>
#include <linux/bug.h>
#include <linux/delay.h>
#include <linux/atomic.h>
#include <linux/prefetch.h>
#include <asm/cache.h>
#include <asm/byteorder.h>
#include <asm/local.h>
#include <linux/percpu.h>
#include <linux/rculist.h>
#include <linux/workqueue.h>
#include <linux/dynamic_queue_limits.h>
#include <net/net_namespace.h>
#ifdef CONFIG_DCB
#include <net/dcbnl.h>
#endif
#include <net/netprio_cgroup.h>
#include <linux/netdev_features.h>
#include <linux/neighbour.h>
#include <uapi/linux/netdevice.h>
#include <uapi/linux/if_bonding.h>
#include <uapi/linux/pkt_cls.h>
#include <uapi/linux/netdev.h>
#include <linux/hashtable.h>
#include <linux/rbtree.h>
#include <net/net_trackers.h>
#include <net/net_debug.h>
#include <net/dropreason-core.h>
struct netpoll_info;
struct device;
struct ethtool_ops;
struct kernel_hwtstamp_config;
struct phy_device;
struct dsa_port;
struct ip_tunnel_parm;
struct macsec_context;
struct macsec_ops;
struct netdev_name_node;
struct sd_flow_limit;
struct sfp_bus;
/* 802.11 specific */
struct wireless_dev;
/* 802.15.4 specific */
struct wpan_dev;
struct mpls_dev;
/* UDP Tunnel offloads */
struct udp_tunnel_info;
struct udp_tunnel_nic_info;
struct udp_tunnel_nic;
struct bpf_prog;
struct xdp_buff;
struct xdp_frame;
struct xdp_metadata_ops;
struct xdp_md;
typedef u32 xdp_features_t;
void synchronize_net(void);
void netdev_set_default_ethtool_ops(struct net_device *dev,
const struct ethtool_ops *ops);
void netdev_sw_irq_coalesce_default_on(struct net_device *dev);
/* Backlog congestion levels */
#define NET_RX_SUCCESS 0 /* keep 'em coming, baby */
#define NET_RX_DROP 1 /* packet dropped */
#define MAX_NEST_DEV 8
/*
* Transmit return codes: transmit return codes originate from three different
* namespaces:
*
* - qdisc return codes
* - driver transmit return codes
* - errno values
*
* Drivers are allowed to return any one of those in their hard_start_xmit()
* function. Real network devices commonly used with qdiscs should only return
* the driver transmit return codes though - when qdiscs are used, the actual
* transmission happens asynchronously, so the value is not propagated to
* higher layers. Virtual network devices transmit synchronously; in this case
* the driver transmit return codes are consumed by dev_queue_xmit(), and all
* others are propagated to higher layers.
*/
/* qdisc ->enqueue() return codes. */
#define NET_XMIT_SUCCESS 0x00
#define NET_XMIT_DROP 0x01 /* skb dropped */
#define NET_XMIT_CN 0x02 /* congestion notification */
#define NET_XMIT_MASK 0x0f /* qdisc flags in net/sch_generic.h */
/* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
* indicates that the device will soon be dropping packets, or already drops
* some packets of the same priority; prompting us to send less aggressively. */
#define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
#define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
/* Driver transmit return codes */
#define NETDEV_TX_MASK 0xf0
enum netdev_tx {
__NETDEV_TX_MIN = INT_MIN, /* make sure enum is signed */
NETDEV_TX_OK = 0x00, /* driver took care of packet */
NETDEV_TX_BUSY = 0x10, /* driver tx path was busy*/
};
typedef enum netdev_tx netdev_tx_t;
/*
* Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
* hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
*/
static inline bool dev_xmit_complete(int rc)
{
/*
* Positive cases with an skb consumed by a driver:
* - successful transmission (rc == NETDEV_TX_OK)
* - error while transmitting (rc < 0)
* - error while queueing to a different device (rc & NET_XMIT_MASK)
*/
if (likely(rc < NET_XMIT_MASK))
return true;
return false;
}
/*
* Compute the worst-case header length according to the protocols
* used.
*/
#if defined(CONFIG_HYPERV_NET)
# define LL_MAX_HEADER 128
#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
# if defined(CONFIG_MAC80211_MESH)
# define LL_MAX_HEADER 128
# else
# define LL_MAX_HEADER 96
# endif
#else
# define LL_MAX_HEADER 32
#endif
#if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
!IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
#define MAX_HEADER LL_MAX_HEADER
#else
#define MAX_HEADER (LL_MAX_HEADER + 48)
#endif
/*
* Old network device statistics. Fields are native words
* (unsigned long) so they can be read and written atomically.
*/
#define NET_DEV_STAT(FIELD) \
union { \
unsigned long FIELD; \
atomic_long_t __##FIELD; \
}
struct net_device_stats {
NET_DEV_STAT(rx_packets);
NET_DEV_STAT(tx_packets);
NET_DEV_STAT(rx_bytes);
NET_DEV_STAT(tx_bytes);
NET_DEV_STAT(rx_errors);
NET_DEV_STAT(tx_errors);
NET_DEV_STAT(rx_dropped);
NET_DEV_STAT(tx_dropped);
NET_DEV_STAT(multicast);
NET_DEV_STAT(collisions);
NET_DEV_STAT(rx_length_errors);
NET_DEV_STAT(rx_over_errors);
NET_DEV_STAT(rx_crc_errors);
NET_DEV_STAT(rx_frame_errors);
NET_DEV_STAT(rx_fifo_errors);
NET_DEV_STAT(rx_missed_errors);
NET_DEV_STAT(tx_aborted_errors);
NET_DEV_STAT(tx_carrier_errors);
NET_DEV_STAT(tx_fifo_errors);
NET_DEV_STAT(tx_heartbeat_errors);
NET_DEV_STAT(tx_window_errors);
NET_DEV_STAT(rx_compressed);
NET_DEV_STAT(tx_compressed);
};
#undef NET_DEV_STAT
/* per-cpu stats, allocated on demand.
* Try to fit them in a single cache line, for dev_get_stats() sake.
*/
struct net_device_core_stats {
unsigned long rx_dropped;
unsigned long tx_dropped;
unsigned long rx_nohandler;
unsigned long rx_otherhost_dropped;
} __aligned(4 * sizeof(unsigned long));
#include <linux/cache.h>
#include <linux/skbuff.h>
struct neighbour;
struct neigh_parms;
struct sk_buff;
struct netdev_hw_addr {
struct list_head list;
struct rb_node node;
unsigned char addr[MAX_ADDR_LEN];
unsigned char type;
#define NETDEV_HW_ADDR_T_LAN 1
#define NETDEV_HW_ADDR_T_SAN 2
#define NETDEV_HW_ADDR_T_UNICAST 3
#define NETDEV_HW_ADDR_T_MULTICAST 4
bool global_use;
int sync_cnt;
int refcount;
int synced;
struct rcu_head rcu_head;
};
struct netdev_hw_addr_list {
struct list_head list;
int count;
/* Auxiliary tree for faster lookup on addition and deletion */
struct rb_root tree;
};
#define netdev_hw_addr_list_count(l) ((l)->count)
#define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
#define netdev_hw_addr_list_for_each(ha, l) \
list_for_each_entry(ha, &(l)->list, list)
#define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
#define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
#define netdev_for_each_uc_addr(ha, dev) \
netdev_hw_addr_list_for_each(ha, &(dev)->uc)
#define netdev_for_each_synced_uc_addr(_ha, _dev) \
netdev_for_each_uc_addr((_ha), (_dev)) \
if ((_ha)->sync_cnt)
#define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
#define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
#define netdev_for_each_mc_addr(ha, dev) \
netdev_hw_addr_list_for_each(ha, &(dev)->mc)
#define netdev_for_each_synced_mc_addr(_ha, _dev) \
netdev_for_each_mc_addr((_ha), (_dev)) \
if ((_ha)->sync_cnt)
struct hh_cache {
unsigned int hh_len;
seqlock_t hh_lock;
/* cached hardware header; allow for machine alignment needs. */
#define HH_DATA_MOD 16
#define HH_DATA_OFF(__len) \
(HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
#define HH_DATA_ALIGN(__len) \
(((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
};
/* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
* Alternative is:
* dev->hard_header_len ? (dev->hard_header_len +
* (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
*
* We could use other alignment values, but we must maintain the
* relationship HH alignment <= LL alignment.
*/
#define LL_RESERVED_SPACE(dev) \
((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \
& ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
#define LL_RESERVED_SPACE_EXTRA(dev,extra) \
((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \
& ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
struct header_ops {
int (*create) (struct sk_buff *skb, struct net_device *dev,
unsigned short type, const void *daddr,
const void *saddr, unsigned int len);
int (*parse)(const struct sk_buff *skb, unsigned char *haddr);
int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
void (*cache_update)(struct hh_cache *hh,
const struct net_device *dev,
const unsigned char *haddr);
bool (*validate)(const char *ll_header, unsigned int len);
__be16 (*parse_protocol)(const struct sk_buff *skb);
};
/* These flag bits are private to the generic network queueing
* layer; they may not be explicitly referenced by any other
* code.
*/
enum netdev_state_t {
__LINK_STATE_START,
__LINK_STATE_PRESENT,
__LINK_STATE_NOCARRIER,
__LINK_STATE_LINKWATCH_PENDING,
__LINK_STATE_DORMANT,
__LINK_STATE_TESTING,
};
struct gro_list {
struct list_head list;
int count;
};
/*
* size of gro hash buckets, must less than bit number of
* napi_struct::gro_bitmask
*/
#define GRO_HASH_BUCKETS 8
/*
* Structure for NAPI scheduling similar to tasklet but with weighting
*/
struct napi_struct {
/* The poll_list must only be managed by the entity which
* changes the state of the NAPI_STATE_SCHED bit. This means
* whoever atomically sets that bit can add this napi_struct
* to the per-CPU poll_list, and whoever clears that bit
* can remove from the list right before clearing the bit.
*/
struct list_head poll_list;
unsigned long state;
int weight;
int defer_hard_irqs_count;
unsigned long gro_bitmask;
int (*poll)(struct napi_struct *, int);
#ifdef CONFIG_NETPOLL
/* CPU actively polling if netpoll is configured */
int poll_owner;
#endif
/* CPU on which NAPI has been scheduled for processing */
int list_owner;
struct net_device *dev;
struct gro_list gro_hash[GRO_HASH_BUCKETS];
struct sk_buff *skb;
struct list_head rx_list; /* Pending GRO_NORMAL skbs */
int rx_count; /* length of rx_list */
unsigned int napi_id;
struct hrtimer timer;
struct task_struct *thread;
/* control-path-only fields follow */
struct list_head dev_list;
struct hlist_node napi_hash_node;
int irq;
};
enum {
NAPI_STATE_SCHED, /* Poll is scheduled */
NAPI_STATE_MISSED, /* reschedule a napi */
NAPI_STATE_DISABLE, /* Disable pending */
NAPI_STATE_NPSVC, /* Netpoll - don't dequeue from poll_list */
NAPI_STATE_LISTED, /* NAPI added to system lists */
NAPI_STATE_NO_BUSY_POLL, /* Do not add in napi_hash, no busy polling */
NAPI_STATE_IN_BUSY_POLL, /* sk_busy_loop() owns this NAPI */
NAPI_STATE_PREFER_BUSY_POLL, /* prefer busy-polling over softirq processing*/
NAPI_STATE_THREADED, /* The poll is performed inside its own thread*/
NAPI_STATE_SCHED_THREADED, /* Napi is currently scheduled in threaded mode */
};
enum {
NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
NAPIF_STATE_LISTED = BIT(NAPI_STATE_LISTED),
NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
NAPIF_STATE_PREFER_BUSY_POLL = BIT(NAPI_STATE_PREFER_BUSY_POLL),
NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED),
NAPIF_STATE_SCHED_THREADED = BIT(NAPI_STATE_SCHED_THREADED),
};
enum gro_result {
GRO_MERGED,
GRO_MERGED_FREE,
GRO_HELD,
GRO_NORMAL,
GRO_CONSUMED,
};
typedef enum gro_result gro_result_t;
/*
* enum rx_handler_result - Possible return values for rx_handlers.
* @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
* further.
* @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
* case skb->dev was changed by rx_handler.
* @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
* @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
*
* rx_handlers are functions called from inside __netif_receive_skb(), to do
* special processing of the skb, prior to delivery to protocol handlers.
*
* Currently, a net_device can only have a single rx_handler registered. Trying
* to register a second rx_handler will return -EBUSY.
*
* To register a rx_handler on a net_device, use netdev_rx_handler_register().
* To unregister a rx_handler on a net_device, use
* netdev_rx_handler_unregister().
*
* Upon return, rx_handler is expected to tell __netif_receive_skb() what to
* do with the skb.
*
* If the rx_handler consumed the skb in some way, it should return
* RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
* the skb to be delivered in some other way.
*
* If the rx_handler changed skb->dev, to divert the skb to another
* net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
* new device will be called if it exists.
*
* If the rx_handler decides the skb should be ignored, it should return
* RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
* are registered on exact device (ptype->dev == skb->dev).
*
* If the rx_handler didn't change skb->dev, but wants the skb to be normally
* delivered, it should return RX_HANDLER_PASS.
*
* A device without a registered rx_handler will behave as if rx_handler
* returned RX_HANDLER_PASS.
*/
enum rx_handler_result {
RX_HANDLER_CONSUMED,
RX_HANDLER_ANOTHER,
RX_HANDLER_EXACT,
RX_HANDLER_PASS,
};
typedef enum rx_handler_result rx_handler_result_t;
typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
void __napi_schedule(struct napi_struct *n);
void __napi_schedule_irqoff(struct napi_struct *n);
static inline bool napi_disable_pending(struct napi_struct *n)
{
return test_bit(NAPI_STATE_DISABLE, &n->state);
}
static inline bool napi_prefer_busy_poll(struct napi_struct *n)
{
return test_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
}
/**
* napi_is_scheduled - test if NAPI is scheduled
* @n: NAPI context
*
* This check is "best-effort". With no locking implemented,
* a NAPI can be scheduled or terminate right after this check
* and produce not precise results.
*
* NAPI_STATE_SCHED is an internal state, napi_is_scheduled
* should not be used normally and napi_schedule should be
* used instead.
*
* Use only if the driver really needs to check if a NAPI
* is scheduled for example in the context of delayed timer
* that can be skipped if a NAPI is already scheduled.
*
* Return True if NAPI is scheduled, False otherwise.
*/
static inline bool napi_is_scheduled(struct napi_struct *n)
{
return test_bit(NAPI_STATE_SCHED, &n->state);
}
bool napi_schedule_prep(struct napi_struct *n);
/**
* napi_schedule - schedule NAPI poll
* @n: NAPI context
*
* Schedule NAPI poll routine to be called if it is not already
* running.
* Return true if we schedule a NAPI or false if not.
* Refer to napi_schedule_prep() for additional reason on why
* a NAPI might not be scheduled.
*/
static inline bool napi_schedule(struct napi_struct *n)
{
if (napi_schedule_prep(n)) {
__napi_schedule(n);
return true;
}
return false;
}
/**
* napi_schedule_irqoff - schedule NAPI poll
* @n: NAPI context
*
* Variant of napi_schedule(), assuming hard irqs are masked.
*/
static inline void napi_schedule_irqoff(struct napi_struct *n)
{
if (napi_schedule_prep(n))
__napi_schedule_irqoff(n);
}
/**
* napi_complete_done - NAPI processing complete
* @n: NAPI context
* @work_done: number of packets processed
*
* Mark NAPI processing as complete. Should only be called if poll budget
* has not been completely consumed.
* Prefer over napi_complete().
* Return false if device should avoid rearming interrupts.
*/
bool napi_complete_done(struct napi_struct *n, int work_done);
static inline bool napi_complete(struct napi_struct *n)
{
return napi_complete_done(n, 0);
}
int dev_set_threaded(struct net_device *dev, bool threaded);
/**
* napi_disable - prevent NAPI from scheduling
* @n: NAPI context
*
* Stop NAPI from being scheduled on this context.
* Waits till any outstanding processing completes.
*/
void napi_disable(struct napi_struct *n);
void napi_enable(struct napi_struct *n);
/**
* napi_synchronize - wait until NAPI is not running
* @n: NAPI context
*
* Wait until NAPI is done being scheduled on this context.
* Waits till any outstanding processing completes but
* does not disable future activations.
*/
static inline void napi_synchronize(const struct napi_struct *n)
{
if (IS_ENABLED(CONFIG_SMP))
while (test_bit(NAPI_STATE_SCHED, &n->state))
msleep(1);
else
barrier();
}
/**
* napi_if_scheduled_mark_missed - if napi is running, set the
* NAPIF_STATE_MISSED
* @n: NAPI context
*
* If napi is running, set the NAPIF_STATE_MISSED, and return true if
* NAPI is scheduled.
**/
static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
{
unsigned long val, new;
val = READ_ONCE(n->state);
do {
if (val & NAPIF_STATE_DISABLE)
return true;
if (!(val & NAPIF_STATE_SCHED))
return false;
new = val | NAPIF_STATE_MISSED;
} while (!try_cmpxchg(&n->state, &val, new));
return true;
}
enum netdev_queue_state_t {
__QUEUE_STATE_DRV_XOFF,
__QUEUE_STATE_STACK_XOFF,
__QUEUE_STATE_FROZEN,
};
#define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
#define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
#define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
#define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
#define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
QUEUE_STATE_FROZEN)
#define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
QUEUE_STATE_FROZEN)
/*
* __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue. The
* netif_tx_* functions below are used to manipulate this flag. The
* __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
* queue independently. The netif_xmit_*stopped functions below are called
* to check if the queue has been stopped by the driver or stack (either
* of the XOFF bits are set in the state). Drivers should not need to call
* netif_xmit*stopped functions, they should only be using netif_tx_*.
*/
struct netdev_queue {
/*
* read-mostly part
*/
struct net_device *dev;
netdevice_tracker dev_tracker;
struct Qdisc __rcu *qdisc;
struct Qdisc __rcu *qdisc_sleeping;
#ifdef CONFIG_SYSFS
struct kobject kobj;
#endif
#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
int numa_node;
#endif
unsigned long tx_maxrate;
/*
* Number of TX timeouts for this queue
* (/sys/class/net/DEV/Q/trans_timeout)
*/
atomic_long_t trans_timeout;
/* Subordinate device that the queue has been assigned to */
struct net_device *sb_dev;
#ifdef CONFIG_XDP_SOCKETS
struct xsk_buff_pool *pool;
#endif
/* NAPI instance for the queue
* Readers and writers must hold RTNL
*/
struct napi_struct *napi;
/*
* write-mostly part
*/
spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
int xmit_lock_owner;
/*
* Time (in jiffies) of last Tx
*/
unsigned long trans_start;
unsigned long state;
#ifdef CONFIG_BQL
struct dql dql;
#endif
} ____cacheline_aligned_in_smp;
extern int sysctl_fb_tunnels_only_for_init_net;
extern int sysctl_devconf_inherit_init_net;
/*
* sysctl_fb_tunnels_only_for_init_net == 0 : For all netns
* == 1 : For initns only
* == 2 : For none.
*/
static inline bool net_has_fallback_tunnels(const struct net *net)
{
#if IS_ENABLED(CONFIG_SYSCTL)
int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net);
return !fb_tunnels_only_for_init_net ||
(net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1);
#else
return true;
#endif
}
static inline int net_inherit_devconf(void)
{
#if IS_ENABLED(CONFIG_SYSCTL)
return READ_ONCE(sysctl_devconf_inherit_init_net);
#else
return 0;
#endif
}
static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
{
#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
return q->numa_node;
#else
return NUMA_NO_NODE;
#endif
}
static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
{
#if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
q->numa_node = node;
#endif
}
#ifdef CONFIG_RFS_ACCEL
bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
u16 filter_id);
#endif
/* XPS map type and offset of the xps map within net_device->xps_maps[]. */
enum xps_map_type {
XPS_CPUS = 0,
XPS_RXQS,
XPS_MAPS_MAX,
};
#ifdef CONFIG_XPS
/*
* This structure holds an XPS map which can be of variable length. The
* map is an array of queues.
*/
struct xps_map {
unsigned int len;
unsigned int alloc_len;
struct rcu_head rcu;
u16 queues[];
};
#define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
#define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
- sizeof(struct xps_map)) / sizeof(u16))
/*
* This structure holds all XPS maps for device. Maps are indexed by CPU.
*
* We keep track of the number of cpus/rxqs used when the struct is allocated,
* in nr_ids. This will help not accessing out-of-bound memory.
*
* We keep track of the number of traffic classes used when the struct is
* allocated, in num_tc. This will be used to navigate the maps, to ensure we're
* not crossing its upper bound, as the original dev->num_tc can be updated in
* the meantime.
*/
struct xps_dev_maps {
struct rcu_head rcu;
unsigned int nr_ids;
s16 num_tc;
struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
};
#define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
(nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
#define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
(_rxqs * (_tcs) * sizeof(struct xps_map *)))
#endif /* CONFIG_XPS */
#define TC_MAX_QUEUE 16
#define TC_BITMASK 15
/* HW offloaded queuing disciplines txq count and offset maps */
struct netdev_tc_txq {
u16 count;
u16 offset;
};
#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
/*
* This structure is to hold information about the device
* configured to run FCoE protocol stack.
*/
struct netdev_fcoe_hbainfo {
char manufacturer[64];
char serial_number[64];
char hardware_version[64];
char driver_version[64];
char optionrom_version[64];
char firmware_version[64];
char model[256];
char model_description[256];
};
#endif
#define MAX_PHYS_ITEM_ID_LEN 32
/* This structure holds a unique identifier to identify some
* physical item (port for example) used by a netdevice.
*/
struct netdev_phys_item_id {
unsigned char id[MAX_PHYS_ITEM_ID_LEN];
unsigned char id_len;
};
static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
struct netdev_phys_item_id *b)
{
return a->id_len == b->id_len &&
memcmp(a->id, b->id, a->id_len) == 0;
}
typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
struct sk_buff *skb,
struct net_device *sb_dev);
enum net_device_path_type {
DEV_PATH_ETHERNET = 0,
DEV_PATH_VLAN,
DEV_PATH_BRIDGE,
DEV_PATH_PPPOE,
DEV_PATH_DSA,
DEV_PATH_MTK_WDMA,
};
struct net_device_path {
enum net_device_path_type type;
const struct net_device *dev;
union {
struct {
u16 id;
__be16 proto;
u8 h_dest[ETH_ALEN];
} encap;
struct {
enum {
DEV_PATH_BR_VLAN_KEEP,
DEV_PATH_BR_VLAN_TAG,
DEV_PATH_BR_VLAN_UNTAG,
DEV_PATH_BR_VLAN_UNTAG_HW,
} vlan_mode;
u16 vlan_id;
__be16 vlan_proto;
} bridge;
struct {
int port;
u16 proto;
} dsa;
struct {
u8 wdma_idx;
u8 queue;
u16 wcid;
u8 bss;
u8 amsdu;
} mtk_wdma;
};
};
#define NET_DEVICE_PATH_STACK_MAX 5
#define NET_DEVICE_PATH_VLAN_MAX 2
struct net_device_path_stack {
int num_paths;
struct net_device_path path[NET_DEVICE_PATH_STACK_MAX];
};
struct net_device_path_ctx {
const struct net_device *dev;
u8 daddr[ETH_ALEN];
int num_vlans;
struct {
u16 id;
__be16 proto;
} vlan[NET_DEVICE_PATH_VLAN_MAX];
};
enum tc_setup_type {
TC_QUERY_CAPS,
TC_SETUP_QDISC_MQPRIO,
TC_SETUP_CLSU32,
TC_SETUP_CLSFLOWER,
TC_SETUP_CLSMATCHALL,
TC_SETUP_CLSBPF,
TC_SETUP_BLOCK,
TC_SETUP_QDISC_CBS,
TC_SETUP_QDISC_RED,
TC_SETUP_QDISC_PRIO,
TC_SETUP_QDISC_MQ,
TC_SETUP_QDISC_ETF,
TC_SETUP_ROOT_QDISC,
TC_SETUP_QDISC_GRED,
TC_SETUP_QDISC_TAPRIO,
TC_SETUP_FT,
TC_SETUP_QDISC_ETS,
TC_SETUP_QDISC_TBF,
TC_SETUP_QDISC_FIFO,
TC_SETUP_QDISC_HTB,
TC_SETUP_ACT,
};
/* These structures hold the attributes of bpf state that are being passed
* to the netdevice through the bpf op.
*/
enum bpf_netdev_command {
/* Set or clear a bpf program used in the earliest stages of packet
* rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
* is responsible for calling bpf_prog_put on any old progs that are
* stored. In case of error, the callee need not release the new prog
* reference, but on success it takes ownership and must bpf_prog_put
* when it is no longer used.
*/
XDP_SETUP_PROG,
XDP_SETUP_PROG_HW,
/* BPF program for offload callbacks, invoked at program load time. */
BPF_OFFLOAD_MAP_ALLOC,
BPF_OFFLOAD_MAP_FREE,
XDP_SETUP_XSK_POOL,
};
struct bpf_prog_offload_ops;
struct netlink_ext_ack;
struct xdp_umem;
struct xdp_dev_bulk_queue;
struct bpf_xdp_link;
enum bpf_xdp_mode {
XDP_MODE_SKB = 0,
XDP_MODE_DRV = 1,
XDP_MODE_HW = 2,
__MAX_XDP_MODE
};
struct bpf_xdp_entity {
struct bpf_prog *prog;
struct bpf_xdp_link *link;
};
struct netdev_bpf {
enum bpf_netdev_command command;
union {
/* XDP_SETUP_PROG */
struct {
u32 flags;
struct bpf_prog *prog;
struct netlink_ext_ack *extack;
};
/* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
struct {
struct bpf_offloaded_map *offmap;
};
/* XDP_SETUP_XSK_POOL */
struct {
struct xsk_buff_pool *pool;
u16 queue_id;
} xsk;
};
};
/* Flags for ndo_xsk_wakeup. */
#define XDP_WAKEUP_RX (1 << 0)
#define XDP_WAKEUP_TX (1 << 1)
#ifdef CONFIG_XFRM_OFFLOAD
struct xfrmdev_ops {
int (*xdo_dev_state_add) (struct xfrm_state *x, struct netlink_ext_ack *extack);
void (*xdo_dev_state_delete) (struct xfrm_state *x);
void (*xdo_dev_state_free) (struct xfrm_state *x);
bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
struct xfrm_state *x);
void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
void (*xdo_dev_state_update_stats) (struct xfrm_state *x);
int (*xdo_dev_policy_add) (struct xfrm_policy *x, struct netlink_ext_ack *extack);
void (*xdo_dev_policy_delete) (struct xfrm_policy *x);
void (*xdo_dev_policy_free) (struct xfrm_policy *x);
};
#endif
struct dev_ifalias {
struct rcu_head rcuhead;
char ifalias[];
};
struct devlink;
struct tlsdev_ops;
struct netdev_net_notifier {
struct list_head list;
struct notifier_block *nb;
};
/*
* This structure defines the management hooks for network devices.
* The following hooks can be defined; unless noted otherwise, they are
* optional and can be filled with a null pointer.
*
* int (*ndo_init)(struct net_device *dev);
* This function is called once when a network device is registered.
* The network device can use this for any late stage initialization
* or semantic validation. It can fail with an error code which will
* be propagated back to register_netdev.
*
* void (*ndo_uninit)(struct net_device *dev);
* This function is called when device is unregistered or when registration
* fails. It is not called if init fails.
*
* int (*ndo_open)(struct net_device *dev);
* This function is called when a network device transitions to the up
* state.
*
* int (*ndo_stop)(struct net_device *dev);
* This function is called when a network device transitions to the down
* state.
*
* netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
* struct net_device *dev);
* Called when a packet needs to be transmitted.
* Returns NETDEV_TX_OK. Can return NETDEV_TX_BUSY, but you should stop
* the queue before that can happen; it's for obsolete devices and weird
* corner cases, but the stack really does a non-trivial amount
* of useless work if you return NETDEV_TX_BUSY.
* Required; cannot be NULL.
*
* netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
* struct net_device *dev
* netdev_features_t features);
* Called by core transmit path to determine if device is capable of
* performing offload operations on a given packet. This is to give
* the device an opportunity to implement any restrictions that cannot
* be otherwise expressed by feature flags. The check is called with
* the set of features that the stack has calculated and it returns
* those the driver believes to be appropriate.
*
* u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
* struct net_device *sb_dev);
* Called to decide which queue to use when device supports multiple
* transmit queues.
*
* void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
* This function is called to allow device receiver to make
* changes to configuration when multicast or promiscuous is enabled.
*
* void (*ndo_set_rx_mode)(struct net_device *dev);
* This function is called device changes address list filtering.
* If driver handles unicast address filtering, it should set
* IFF_UNICAST_FLT in its priv_flags.
*
* int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
* This function is called when the Media Access Control address
* needs to be changed. If this interface is not defined, the
* MAC address can not be changed.
*
* int (*ndo_validate_addr)(struct net_device *dev);
* Test if Media Access Control address is valid for the device.
*
* int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
* Old-style ioctl entry point. This is used internally by the
* appletalk and ieee802154 subsystems but is no longer called by
* the device ioctl handler.
*
* int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd);
* Used by the bonding driver for its device specific ioctls:
* SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE,
* SIOCBONDSLAVEINFOQUERY, and SIOCBONDINFOQUERY
*
* * int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
* Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG,
* SIOCSMIIREG, SIOCSHWTSTAMP and SIOCGHWTSTAMP.
*
* int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
* Used to set network devices bus interface parameters. This interface
* is retained for legacy reasons; new devices should use the bus
* interface (PCI) for low level management.
*
* int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
* Called when a user wants to change the Maximum Transfer Unit
* of a device.
*
* void (*ndo_tx_timeout)(struct net_device *dev, unsigned int txqueue);
* Callback used when the transmitter has not made any progress
* for dev->watchdog ticks.
*
* void (*ndo_get_stats64)(struct net_device *dev,
* struct rtnl_link_stats64 *storage);
* struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
* Called when a user wants to get the network device usage
* statistics. Drivers must do one of the following:
* 1. Define @ndo_get_stats64 to fill in a zero-initialised
* rtnl_link_stats64 structure passed by the caller.
* 2. Define @ndo_get_stats to update a net_device_stats structure
* (which should normally be dev->stats) and return a pointer to
* it. The structure may be changed asynchronously only if each
* field is written atomically.
* 3. Update dev->stats asynchronously and atomically, and define
* neither operation.
*
* bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
* Return true if this device supports offload stats of this attr_id.
*
* int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
* void *attr_data)
* Get statistics for offload operations by attr_id. Write it into the
* attr_data pointer.
*
* int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
* If device supports VLAN filtering this function is called when a
* VLAN id is registered.
*
* int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
* If device supports VLAN filtering this function is called when a
* VLAN id is unregistered.
*
* void (*ndo_poll_controller)(struct net_device *dev);
*
* SR-IOV management functions.
* int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
* int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
* u8 qos, __be16 proto);
* int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
* int max_tx_rate);
* int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
* int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
* int (*ndo_get_vf_config)(struct net_device *dev,
* int vf, struct ifla_vf_info *ivf);
* int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
* int (*ndo_set_vf_port)(struct net_device *dev, int vf,
* struct nlattr *port[]);
*
* Enable or disable the VF ability to query its RSS Redirection Table and
* Hash Key. This is needed since on some devices VF share this information
* with PF and querying it may introduce a theoretical security risk.
* int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
* int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
* int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
* void *type_data);
* Called to setup any 'tc' scheduler, classifier or action on @dev.
* This is always called from the stack with the rtnl lock held and netif
* tx queues stopped. This allows the netdevice to perform queue
* management safely.
*
* Fiber Channel over Ethernet (FCoE) offload functions.
* int (*ndo_fcoe_enable)(struct net_device *dev);
* Called when the FCoE protocol stack wants to start using LLD for FCoE
* so the underlying device can perform whatever needed configuration or
* initialization to support acceleration of FCoE traffic.
*
* int (*ndo_fcoe_disable)(struct net_device *dev);
* Called when the FCoE protocol stack wants to stop using LLD for FCoE
* so the underlying device can perform whatever needed clean-ups to
* stop supporting acceleration of FCoE traffic.
*
* int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
* struct scatterlist *sgl, unsigned int sgc);
* Called when the FCoE Initiator wants to initialize an I/O that
* is a possible candidate for Direct Data Placement (DDP). The LLD can
* perform necessary setup and returns 1 to indicate the device is set up
* successfully to perform DDP on this I/O, otherwise this returns 0.
*
* int (*ndo_fcoe_ddp_done)(struct net_device *dev, u16 xid);
* Called when the FCoE Initiator/Target is done with the DDPed I/O as
* indicated by the FC exchange id 'xid', so the underlying device can
* clean up and reuse resources for later DDP requests.
*
* int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
* struct scatterlist *sgl, unsigned int sgc);
* Called when the FCoE Target wants to initialize an I/O that
* is a possible candidate for Direct Data Placement (DDP). The LLD can
* perform necessary setup and returns 1 to indicate the device is set up
* successfully to perform DDP on this I/O, otherwise this returns 0.
*
* int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
* struct netdev_fcoe_hbainfo *hbainfo);
* Called when the FCoE Protocol stack wants information on the underlying
* device. This information is utilized by the FCoE protocol stack to
* register attributes with Fiber Channel management service as per the
* FC-GS Fabric Device Management Information(FDMI) specification.
*
* int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
* Called when the underlying device wants to override default World Wide
* Name (WWN) generation mechanism in FCoE protocol stack to pass its own
* World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
* protocol stack to use.
*
* RFS acceleration.
* int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
* u16 rxq_index, u32 flow_id);
* Set hardware filter for RFS. rxq_index is the target queue index;
* flow_id is a flow ID to be passed to rps_may_expire_flow() later.
* Return the filter ID on success, or a negative error code.
*
* Slave management functions (for bridge, bonding, etc).
* int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
* Called to make another netdev an underling.
*
* int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
* Called to release previously enslaved netdev.
*
* struct net_device *(*ndo_get_xmit_slave)(struct net_device *dev,
* struct sk_buff *skb,
* bool all_slaves);
* Get the xmit slave of master device. If all_slaves is true, function
* assume all the slaves can transmit.
*
* Feature/offload setting functions.
* netdev_features_t (*ndo_fix_features)(struct net_device *dev,
* netdev_features_t features);
* Adjusts the requested feature flags according to device-specific
* constraints, and returns the resulting flags. Must not modify
* the device state.
*
* int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
* Called to update device configuration to new features. Passed
* feature set might be less than what was returned by ndo_fix_features()).
* Must return >0 or -errno if it changed dev->features itself.
*
* int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
* struct net_device *dev,
* const unsigned char *addr, u16 vid, u16 flags,
* struct netlink_ext_ack *extack);
* Adds an FDB entry to dev for addr.
* int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
* struct net_device *dev,
* const unsigned char *addr, u16 vid)
* Deletes the FDB entry from dev coresponding to addr.
* int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh, struct net_device *dev,
* struct netlink_ext_ack *extack);
* int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
* struct net_device *dev, struct net_device *filter_dev,
* int *idx)
* Used to add FDB entries to dump requests. Implementers should add
* entries to skb and update idx with the number of entries.
*
* int (*ndo_mdb_add)(struct net_device *dev, struct nlattr *tb[],
* u16 nlmsg_flags, struct netlink_ext_ack *extack);
* Adds an MDB entry to dev.
* int (*ndo_mdb_del)(struct net_device *dev, struct nlattr *tb[],
* struct netlink_ext_ack *extack);
* Deletes the MDB entry from dev.
* int (*ndo_mdb_del_bulk)(struct net_device *dev, struct nlattr *tb[],
* struct netlink_ext_ack *extack);
* Bulk deletes MDB entries from dev.
* int (*ndo_mdb_dump)(struct net_device *dev, struct sk_buff *skb,
* struct netlink_callback *cb);
* Dumps MDB entries from dev. The first argument (marker) in the netlink
* callback is used by core rtnetlink code.
*
* int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
* u16 flags, struct netlink_ext_ack *extack)
* int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
* struct net_device *dev, u32 filter_mask,
* int nlflags)
* int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
* u16 flags);
*
* int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
* Called to change device carrier. Soft-devices (like dummy, team, etc)
* which do not represent real hardware may define this to allow their
* userspace components to manage their virtual carrier state. Devices
* that determine carrier state from physical hardware properties (eg
* network cables) or protocol-dependent mechanisms (eg
* USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
*
* int (*ndo_get_phys_port_id)(struct net_device *dev,
* struct netdev_phys_item_id *ppid);
* Called to get ID of physical port of this device. If driver does
* not implement this, it is assumed that the hw is not able to have
* multiple net devices on single physical port.
*
* int (*ndo_get_port_parent_id)(struct net_device *dev,
* struct netdev_phys_item_id *ppid)
* Called to get the parent ID of the physical port of this device.
*
* void* (*ndo_dfwd_add_station)(struct net_device *pdev,
* struct net_device *dev)
* Called by upper layer devices to accelerate switching or other
* station functionality into hardware. 'pdev is the lowerdev
* to use for the offload and 'dev' is the net device that will
* back the offload. Returns a pointer to the private structure
* the upper layer will maintain.
* void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
* Called by upper layer device to delete the station created
* by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
* the station and priv is the structure returned by the add
* operation.
* int (*ndo_set_tx_maxrate)(struct net_device *dev,
* int queue_index, u32 maxrate);
* Called when a user wants to set a max-rate limitation of specific
* TX queue.
* int (*ndo_get_iflink)(const struct net_device *dev);
* Called to get the iflink value of this device.
* int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
* This function is used to get egress tunnel information for given skb.
* This is useful for retrieving outer tunnel header parameters while
* sampling packet.
* void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
* This function is used to specify the headroom that the skb must
* consider when allocation skb during packet reception. Setting
* appropriate rx headroom value allows avoiding skb head copy on
* forward. Setting a negative value resets the rx headroom to the
* default value.
* int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
* This function is used to set or query state related to XDP on the
* netdevice and manage BPF offload. See definition of
* enum bpf_netdev_command for details.
* int (*ndo_xdp_xmit)(struct net_device *dev, int n, struct xdp_frame **xdp,
* u32 flags);
* This function is used to submit @n XDP packets for transmit on a
* netdevice. Returns number of frames successfully transmitted, frames
* that got dropped are freed/returned via xdp_return_frame().
* Returns negative number, means general error invoking ndo, meaning
* no frames were xmit'ed and core-caller will free all frames.
* struct net_device *(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
* struct xdp_buff *xdp);
* Get the xmit slave of master device based on the xdp_buff.
* int (*ndo_xsk_wakeup)(struct net_device *dev, u32 queue_id, u32 flags);
* This function is used to wake up the softirq, ksoftirqd or kthread
* responsible for sending and/or receiving packets on a specific
* queue id bound to an AF_XDP socket. The flags field specifies if
* only RX, only Tx, or both should be woken up using the flags
* XDP_WAKEUP_RX and XDP_WAKEUP_TX.
* int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm *p,
* int cmd);
* Add, change, delete or get information on an IPv4 tunnel.
* struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
* If a device is paired with a peer device, return the peer instance.
* The caller must be under RCU read context.
* int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx, struct net_device_path *path);
* Get the forwarding path to reach the real device from the HW destination address
* ktime_t (*ndo_get_tstamp)(struct net_device *dev,
* const struct skb_shared_hwtstamps *hwtstamps,
* bool cycles);
* Get hardware timestamp based on normal/adjustable time or free running
* cycle counter. This function is required if physical clock supports a
* free running cycle counter.
*
* int (*ndo_hwtstamp_get)(struct net_device *dev,
* struct kernel_hwtstamp_config *kernel_config);
* Get the currently configured hardware timestamping parameters for the
* NIC device.
*
* int (*ndo_hwtstamp_set)(struct net_device *dev,
* struct kernel_hwtstamp_config *kernel_config,
* struct netlink_ext_ack *extack);
* Change the hardware timestamping parameters for NIC device.
*/
struct net_device_ops {
int (*ndo_init)(struct net_device *dev);
void (*ndo_uninit)(struct net_device *dev);
int (*ndo_open)(struct net_device *dev);
int (*ndo_stop)(struct net_device *dev);
netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
struct net_device *dev);
netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features);
u16 (*ndo_select_queue)(struct net_device *dev,
struct sk_buff *skb,
struct net_device *sb_dev);
void (*ndo_change_rx_flags)(struct net_device *dev,
int flags);
void (*ndo_set_rx_mode)(struct net_device *dev);
int (*ndo_set_mac_address)(struct net_device *dev,
void *addr);
int (*ndo_validate_addr)(struct net_device *dev);
int (*ndo_do_ioctl)(struct net_device *dev,
struct ifreq *ifr, int cmd);
int (*ndo_eth_ioctl)(struct net_device *dev,
struct ifreq *ifr, int cmd);
int (*ndo_siocbond)(struct net_device *dev,
struct ifreq *ifr, int cmd);
int (*ndo_siocwandev)(struct net_device *dev,
struct if_settings *ifs);
int (*ndo_siocdevprivate)(struct net_device *dev,
struct ifreq *ifr,
void __user *data, int cmd);
int (*ndo_set_config)(struct net_device *dev,
struct ifmap *map);
int (*ndo_change_mtu)(struct net_device *dev,
int new_mtu);
int (*ndo_neigh_setup)(struct net_device *dev,
struct neigh_parms *);
void (*ndo_tx_timeout) (struct net_device *dev,
unsigned int txqueue);
void (*ndo_get_stats64)(struct net_device *dev,
struct rtnl_link_stats64 *storage);
bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
int (*ndo_get_offload_stats)(int attr_id,
const struct net_device *dev,
void *attr_data);
struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
__be16 proto, u16 vid);
int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
__be16 proto, u16 vid);
#ifdef CONFIG_NET_POLL_CONTROLLER
void (*ndo_poll_controller)(struct net_device *dev);
int (*ndo_netpoll_setup)(struct net_device *dev,
struct netpoll_info *info);
void (*ndo_netpoll_cleanup)(struct net_device *dev);
#endif
int (*ndo_set_vf_mac)(struct net_device *dev,
int queue, u8 *mac);
int (*ndo_set_vf_vlan)(struct net_device *dev,
int queue, u16 vlan,
u8 qos, __be16 proto);
int (*ndo_set_vf_rate)(struct net_device *dev,
int vf, int min_tx_rate,
int max_tx_rate);
int (*ndo_set_vf_spoofchk)(struct net_device *dev,
int vf, bool setting);
int (*ndo_set_vf_trust)(struct net_device *dev,
int vf, bool setting);
int (*ndo_get_vf_config)(struct net_device *dev,
int vf,
struct ifla_vf_info *ivf);
int (*ndo_set_vf_link_state)(struct net_device *dev,
int vf, int link_state);
int (*ndo_get_vf_stats)(struct net_device *dev,
int vf,
struct ifla_vf_stats
*vf_stats);
int (*ndo_set_vf_port)(struct net_device *dev,
int vf,
struct nlattr *port[]);
int (*ndo_get_vf_port)(struct net_device *dev,
int vf, struct sk_buff *skb);
int (*ndo_get_vf_guid)(struct net_device *dev,
int vf,
struct ifla_vf_guid *node_guid,
struct ifla_vf_guid *port_guid);
int (*ndo_set_vf_guid)(struct net_device *dev,
int vf, u64 guid,
int guid_type);
int (*ndo_set_vf_rss_query_en)(
struct net_device *dev,
int vf, bool setting);
int (*ndo_setup_tc)(struct net_device *dev,
enum tc_setup_type type,
void *type_data);
#if IS_ENABLED(CONFIG_FCOE)
int (*ndo_fcoe_enable)(struct net_device *dev);
int (*ndo_fcoe_disable)(struct net_device *dev);
int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
u16 xid,
struct scatterlist *sgl,
unsigned int sgc);
int (*ndo_fcoe_ddp_done)(struct net_device *dev,
u16 xid);
int (*ndo_fcoe_ddp_target)(struct net_device *dev,
u16 xid,
struct scatterlist *sgl,
unsigned int sgc);
int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
struct netdev_fcoe_hbainfo *hbainfo);
#endif
#if IS_ENABLED(CONFIG_LIBFCOE)
#define NETDEV_FCOE_WWNN 0
#define NETDEV_FCOE_WWPN 1
int (*ndo_fcoe_get_wwn)(struct net_device *dev,
u64 *wwn, int type);
#endif
#ifdef CONFIG_RFS_ACCEL
int (*ndo_rx_flow_steer)(struct net_device *dev,
const struct sk_buff *skb,
u16 rxq_index,
u32 flow_id);
#endif
int (*ndo_add_slave)(struct net_device *dev,
struct net_device *slave_dev,
struct netlink_ext_ack *extack);
int (*ndo_del_slave)(struct net_device *dev,
struct net_device *slave_dev);
struct net_device* (*ndo_get_xmit_slave)(struct net_device *dev,
struct sk_buff *skb,
bool all_slaves);
struct net_device* (*ndo_sk_get_lower_dev)(struct net_device *dev,
struct sock *sk);
netdev_features_t (*ndo_fix_features)(struct net_device *dev,
netdev_features_t features);
int (*ndo_set_features)(struct net_device *dev,
netdev_features_t features);
int (*ndo_neigh_construct)(struct net_device *dev,
struct neighbour *n);
void (*ndo_neigh_destroy)(struct net_device *dev,
struct neighbour *n);
int (*ndo_fdb_add)(struct ndmsg *ndm,
struct nlattr *tb[],
struct net_device *dev,
const unsigned char *addr,
u16 vid,
u16 flags,
struct netlink_ext_ack *extack);
int (*ndo_fdb_del)(struct ndmsg *ndm,
struct nlattr *tb[],
struct net_device *dev,
const unsigned char *addr,
u16 vid, struct netlink_ext_ack *extack);
int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh,
struct net_device *dev,
struct netlink_ext_ack *extack);
int (*ndo_fdb_dump)(struct sk_buff *skb,
struct netlink_callback *cb,
struct net_device *dev,
struct net_device *filter_dev,
int *idx);
int (*ndo_fdb_get)(struct sk_buff *skb,
struct nlattr *tb[],
struct net_device *dev,
const unsigned char *addr,
u16 vid, u32 portid, u32 seq,
struct netlink_ext_ack *extack);
int (*ndo_mdb_add)(struct net_device *dev,
struct nlattr *tb[],
u16 nlmsg_flags,
struct netlink_ext_ack *extack);
int (*ndo_mdb_del)(struct net_device *dev,
struct nlattr *tb[],
struct netlink_ext_ack *extack);
int (*ndo_mdb_del_bulk)(struct net_device *dev,
struct nlattr *tb[],
struct netlink_ext_ack *extack);
int (*ndo_mdb_dump)(struct net_device *dev,
struct sk_buff *skb,
struct netlink_callback *cb);
int (*ndo_mdb_get)(struct net_device *dev,
struct nlattr *tb[], u32 portid,
u32 seq,
struct netlink_ext_ack *extack);
int (*ndo_bridge_setlink)(struct net_device *dev,
struct nlmsghdr *nlh,
u16 flags,
struct netlink_ext_ack *extack);
int (*ndo_bridge_getlink)(struct sk_buff *skb,
u32 pid, u32 seq,
struct net_device *dev,
u32 filter_mask,
int nlflags);
int (*ndo_bridge_dellink)(struct net_device *dev,
struct nlmsghdr *nlh,
u16 flags);
int (*ndo_change_carrier)(struct net_device *dev,
bool new_carrier);
int (*ndo_get_phys_port_id)(struct net_device *dev,
struct netdev_phys_item_id *ppid);
int (*ndo_get_port_parent_id)(struct net_device *dev,
struct netdev_phys_item_id *ppid);
int (*ndo_get_phys_port_name)(struct net_device *dev,
char *name, size_t len);
void* (*ndo_dfwd_add_station)(struct net_device *pdev,
struct net_device *dev);
void (*ndo_dfwd_del_station)(struct net_device *pdev,
void *priv);
int (*ndo_set_tx_maxrate)(struct net_device *dev,
int queue_index,
u32 maxrate);
int (*ndo_get_iflink)(const struct net_device *dev);
int (*ndo_fill_metadata_dst)(struct net_device *dev,
struct sk_buff *skb);
void (*ndo_set_rx_headroom)(struct net_device *dev,
int needed_headroom);
int (*ndo_bpf)(struct net_device *dev,
struct netdev_bpf *bpf);
int (*ndo_xdp_xmit)(struct net_device *dev, int n,
struct xdp_frame **xdp,
u32 flags);
struct net_device * (*ndo_xdp_get_xmit_slave)(struct net_device *dev,
struct xdp_buff *xdp);
int (*ndo_xsk_wakeup)(struct net_device *dev,
u32 queue_id, u32 flags);
int (*ndo_tunnel_ctl)(struct net_device *dev,
struct ip_tunnel_parm *p, int cmd);
struct net_device * (*ndo_get_peer_dev)(struct net_device *dev);
int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
struct net_device_path *path);
ktime_t (*ndo_get_tstamp)(struct net_device *dev,
const struct skb_shared_hwtstamps *hwtstamps,
bool cycles);
int (*ndo_hwtstamp_get)(struct net_device *dev,
struct kernel_hwtstamp_config *kernel_config);
int (*ndo_hwtstamp_set)(struct net_device *dev,
struct kernel_hwtstamp_config *kernel_config,
struct netlink_ext_ack *extack);
};
/**
* enum netdev_priv_flags - &struct net_device priv_flags
*
* These are the &struct net_device, they are only set internally
* by drivers and used in the kernel. These flags are invisible to
* userspace; this means that the order of these flags can change
* during any kernel release.
*
* You should have a pretty good reason to be extending these flags.
*
* @IFF_802_1Q_VLAN: 802.1Q VLAN device
* @IFF_EBRIDGE: Ethernet bridging device
* @IFF_BONDING: bonding master or slave
* @IFF_ISATAP: ISATAP interface (RFC4214)
* @IFF_WAN_HDLC: WAN HDLC device
* @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
* release skb->dst
* @IFF_DONT_BRIDGE: disallow bridging this ether dev
* @IFF_DISABLE_NETPOLL: disable netpoll at run-time
* @IFF_MACVLAN_PORT: device used as macvlan port
* @IFF_BRIDGE_PORT: device used as bridge port
* @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
* @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
* @IFF_UNICAST_FLT: Supports unicast filtering
* @IFF_TEAM_PORT: device used as team port
* @IFF_SUPP_NOFCS: device supports sending custom FCS
* @IFF_LIVE_ADDR_CHANGE: device supports hardware address
* change when it's running
* @IFF_MACVLAN: Macvlan device
* @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
* underlying stacked devices
* @IFF_L3MDEV_MASTER: device is an L3 master device
* @IFF_NO_QUEUE: device can run without qdisc attached
* @IFF_OPENVSWITCH: device is a Open vSwitch master
* @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
* @IFF_TEAM: device is a team device
* @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
* @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
* entity (i.e. the master device for bridged veth)
* @IFF_MACSEC: device is a MACsec device
* @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
* @IFF_FAILOVER: device is a failover master device
* @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
* @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
* @IFF_NO_ADDRCONF: prevent ipv6 addrconf
* @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with
* skb_headlen(skb) == 0 (data starts from frag0)
* @IFF_CHANGE_PROTO_DOWN: device supports setting carrier via IFLA_PROTO_DOWN
* @IFF_SEE_ALL_HWTSTAMP_REQUESTS: device wants to see calls to
* ndo_hwtstamp_set() for all timestamp requests regardless of source,
* even if those aren't HWTSTAMP_SOURCE_NETDEV.
*/
enum netdev_priv_flags {
IFF_802_1Q_VLAN = 1<<0,
IFF_EBRIDGE = 1<<1,
IFF_BONDING = 1<<2,
IFF_ISATAP = 1<<3,
IFF_WAN_HDLC = 1<<4,
IFF_XMIT_DST_RELEASE = 1<<5,
IFF_DONT_BRIDGE = 1<<6,
IFF_DISABLE_NETPOLL = 1<<7,
IFF_MACVLAN_PORT = 1<<8,
IFF_BRIDGE_PORT = 1<<9,
IFF_OVS_DATAPATH = 1<<10,
IFF_TX_SKB_SHARING = 1<<11,
IFF_UNICAST_FLT = 1<<12,
IFF_TEAM_PORT = 1<<13,
IFF_SUPP_NOFCS = 1<<14,
IFF_LIVE_ADDR_CHANGE = 1<<15,
IFF_MACVLAN = 1<<16,
IFF_XMIT_DST_RELEASE_PERM = 1<<17,
IFF_L3MDEV_MASTER = 1<<18,
IFF_NO_QUEUE = 1<<19,
IFF_OPENVSWITCH = 1<<20,
IFF_L3MDEV_SLAVE = 1<<21,
IFF_TEAM = 1<<22,
IFF_RXFH_CONFIGURED = 1<<23,
IFF_PHONY_HEADROOM = 1<<24,
IFF_MACSEC = 1<<25,
IFF_NO_RX_HANDLER = 1<<26,
IFF_FAILOVER = 1<<27,
IFF_FAILOVER_SLAVE = 1<<28,
IFF_L3MDEV_RX_HANDLER = 1<<29,
IFF_NO_ADDRCONF = BIT_ULL(30),
IFF_TX_SKB_NO_LINEAR = BIT_ULL(31),
IFF_CHANGE_PROTO_DOWN = BIT_ULL(32),
IFF_SEE_ALL_HWTSTAMP_REQUESTS = BIT_ULL(33),
};
#define IFF_802_1Q_VLAN IFF_802_1Q_VLAN
#define IFF_EBRIDGE IFF_EBRIDGE
#define IFF_BONDING IFF_BONDING
#define IFF_ISATAP IFF_ISATAP
#define IFF_WAN_HDLC IFF_WAN_HDLC
#define IFF_XMIT_DST_RELEASE IFF_XMIT_DST_RELEASE
#define IFF_DONT_BRIDGE IFF_DONT_BRIDGE
#define IFF_DISABLE_NETPOLL IFF_DISABLE_NETPOLL
#define IFF_MACVLAN_PORT IFF_MACVLAN_PORT
#define IFF_BRIDGE_PORT IFF_BRIDGE_PORT
#define IFF_OVS_DATAPATH IFF_OVS_DATAPATH
#define IFF_TX_SKB_SHARING IFF_TX_SKB_SHARING
#define IFF_UNICAST_FLT IFF_UNICAST_FLT
#define IFF_TEAM_PORT IFF_TEAM_PORT
#define IFF_SUPP_NOFCS IFF_SUPP_NOFCS
#define IFF_LIVE_ADDR_CHANGE IFF_LIVE_ADDR_CHANGE
#define IFF_MACVLAN IFF_MACVLAN
#define IFF_XMIT_DST_RELEASE_PERM IFF_XMIT_DST_RELEASE_PERM
#define IFF_L3MDEV_MASTER IFF_L3MDEV_MASTER
#define IFF_NO_QUEUE IFF_NO_QUEUE
#define IFF_OPENVSWITCH IFF_OPENVSWITCH
#define IFF_L3MDEV_SLAVE IFF_L3MDEV_SLAVE
#define IFF_TEAM IFF_TEAM
#define IFF_RXFH_CONFIGURED IFF_RXFH_CONFIGURED
#define IFF_PHONY_HEADROOM IFF_PHONY_HEADROOM
#define IFF_MACSEC IFF_MACSEC
#define IFF_NO_RX_HANDLER IFF_NO_RX_HANDLER
#define IFF_FAILOVER IFF_FAILOVER
#define IFF_FAILOVER_SLAVE IFF_FAILOVER_SLAVE
#define IFF_L3MDEV_RX_HANDLER IFF_L3MDEV_RX_HANDLER
#define IFF_TX_SKB_NO_LINEAR IFF_TX_SKB_NO_LINEAR
/* Specifies the type of the struct net_device::ml_priv pointer */
enum netdev_ml_priv_type {
ML_PRIV_NONE,
ML_PRIV_CAN,
};
enum netdev_stat_type {
NETDEV_PCPU_STAT_NONE,
NETDEV_PCPU_STAT_LSTATS, /* struct pcpu_lstats */
NETDEV_PCPU_STAT_TSTATS, /* struct pcpu_sw_netstats */
NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */
};
enum netdev_reg_state {
NETREG_UNINITIALIZED = 0,
NETREG_REGISTERED, /* completed register_netdevice */
NETREG_UNREGISTERING, /* called unregister_netdevice */
NETREG_UNREGISTERED, /* completed unregister todo */
NETREG_RELEASED, /* called free_netdev */
NETREG_DUMMY, /* dummy device for NAPI poll */
};
/**
* struct net_device - The DEVICE structure.
*
* Actually, this whole structure is a big mistake. It mixes I/O
* data with strictly "high-level" data, and it has to know about
* almost every data structure used in the INET module.
*
* @name: This is the first field of the "visible" part of this structure
* (i.e. as seen by users in the "Space.c" file). It is the name
* of the interface.
*
* @name_node: Name hashlist node
* @ifalias: SNMP alias
* @mem_end: Shared memory end
* @mem_start: Shared memory start
* @base_addr: Device I/O address
* @irq: Device IRQ number
*
* @state: Generic network queuing layer state, see netdev_state_t
* @dev_list: The global list of network devices
* @napi_list: List entry used for polling NAPI devices
* @unreg_list: List entry when we are unregistering the
* device; see the function unregister_netdev
* @close_list: List entry used when we are closing the device
* @ptype_all: Device-specific packet handlers for all protocols
* @ptype_specific: Device-specific, protocol-specific packet handlers
*
* @adj_list: Directly linked devices, like slaves for bonding
* @features: Currently active device features
* @hw_features: User-changeable features
*
* @wanted_features: User-requested features
* @vlan_features: Mask of features inheritable by VLAN devices
*
* @hw_enc_features: Mask of features inherited by encapsulating devices
* This field indicates what encapsulation
* offloads the hardware is capable of doing,
* and drivers will need to set them appropriately.
*
* @mpls_features: Mask of features inheritable by MPLS
* @gso_partial_features: value(s) from NETIF_F_GSO\*
*
* @ifindex: interface index
* @group: The group the device belongs to
*
* @stats: Statistics struct, which was left as a legacy, use
* rtnl_link_stats64 instead
*
* @core_stats: core networking counters,
* do not use this in drivers
* @carrier_up_count: Number of times the carrier has been up
* @carrier_down_count: Number of times the carrier has been down
*
* @wireless_handlers: List of functions to handle Wireless Extensions,
* instead of ioctl,
* see <net/iw_handler.h> for details.
* @wireless_data: Instance data managed by the core of wireless extensions
*
* @netdev_ops: Includes several pointers to callbacks,
* if one wants to override the ndo_*() functions
* @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
* @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
* @ethtool_ops: Management operations
* @l3mdev_ops: Layer 3 master device operations
* @ndisc_ops: Includes callbacks for different IPv6 neighbour
* discovery handling. Necessary for e.g. 6LoWPAN.
* @xfrmdev_ops: Transformation offload operations
* @tlsdev_ops: Transport Layer Security offload operations
* @header_ops: Includes callbacks for creating,parsing,caching,etc
* of Layer 2 headers.
*
* @flags: Interface flags (a la BSD)
* @xdp_features: XDP capability supported by the device
* @priv_flags: Like 'flags' but invisible to userspace,
* see if.h for the definitions
* @gflags: Global flags ( kept as legacy )
* @padded: How much padding added by alloc_netdev()
* @operstate: RFC2863 operstate
* @link_mode: Mapping policy to operstate
* @if_port: Selectable AUI, TP, ...
* @dma: DMA channel
* @mtu: Interface MTU value
* @min_mtu: Interface Minimum MTU value
* @max_mtu: Interface Maximum MTU value
* @type: Interface hardware type
* @hard_header_len: Maximum hardware header length.
* @min_header_len: Minimum hardware header length
*
* @needed_headroom: Extra headroom the hardware may need, but not in all
* cases can this be guaranteed
* @needed_tailroom: Extra tailroom the hardware may need, but not in all
* cases can this be guaranteed. Some cases also use
* LL_MAX_HEADER instead to allocate the skb
*
* interface address info:
*
* @perm_addr: Permanent hw address
* @addr_assign_type: Hw address assignment type
* @addr_len: Hardware address length
* @upper_level: Maximum depth level of upper devices.
* @lower_level: Maximum depth level of lower devices.
* @neigh_priv_len: Used in neigh_alloc()
* @dev_id: Used to differentiate devices that share
* the same link layer address
* @dev_port: Used to differentiate devices that share
* the same function
* @addr_list_lock: XXX: need comments on this one
* @name_assign_type: network interface name assignment type
* @uc_promisc: Counter that indicates promiscuous mode
* has been enabled due to the need to listen to
* additional unicast addresses in a device that
* does not implement ndo_set_rx_mode()
* @uc: unicast mac addresses
* @mc: multicast mac addresses
* @dev_addrs: list of device hw addresses
* @queues_kset: Group of all Kobjects in the Tx and RX queues
* @promiscuity: Number of times the NIC is told to work in
* promiscuous mode; if it becomes 0 the NIC will
* exit promiscuous mode
* @allmulti: Counter, enables or disables allmulticast mode
*
* @vlan_info: VLAN info
* @dsa_ptr: dsa specific data
* @tipc_ptr: TIPC specific data
* @atalk_ptr: AppleTalk link
* @ip_ptr: IPv4 specific data
* @ip6_ptr: IPv6 specific data
* @ax25_ptr: AX.25 specific data
* @ieee80211_ptr: IEEE 802.11 specific data, assign before registering
* @ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network
* device struct
* @mpls_ptr: mpls_dev struct pointer
* @mctp_ptr: MCTP specific data
*
* @dev_addr: Hw address (before bcast,
* because most packets are unicast)
*
* @_rx: Array of RX queues
* @num_rx_queues: Number of RX queues
* allocated at register_netdev() time
* @real_num_rx_queues: Number of RX queues currently active in device
* @xdp_prog: XDP sockets filter program pointer
* @gro_flush_timeout: timeout for GRO layer in NAPI
* @napi_defer_hard_irqs: If not zero, provides a counter that would
* allow to avoid NIC hard IRQ, on busy queues.
*
* @rx_handler: handler for received packets
* @rx_handler_data: XXX: need comments on this one
* @tcx_ingress: BPF & clsact qdisc specific data for ingress processing
* @ingress_queue: XXX: need comments on this one
* @nf_hooks_ingress: netfilter hooks executed for ingress packets
* @broadcast: hw bcast address
*
* @rx_cpu_rmap: CPU reverse-mapping for RX completion interrupts,
* indexed by RX queue number. Assigned by driver.
* This must only be set if the ndo_rx_flow_steer
* operation is defined
* @index_hlist: Device index hash chain
*
* @_tx: Array of TX queues
* @num_tx_queues: Number of TX queues allocated at alloc_netdev_mq() time
* @real_num_tx_queues: Number of TX queues currently active in device
* @qdisc: Root qdisc from userspace point of view
* @tx_queue_len: Max frames per queue allowed
* @tx_global_lock: XXX: need comments on this one
* @xdp_bulkq: XDP device bulk queue
* @xps_maps: all CPUs/RXQs maps for XPS device
*
* @xps_maps: XXX: need comments on this one
* @tcx_egress: BPF & clsact qdisc specific data for egress processing
* @nf_hooks_egress: netfilter hooks executed for egress packets
* @qdisc_hash: qdisc hash table
* @watchdog_timeo: Represents the timeout that is used by
* the watchdog (see dev_watchdog())
* @watchdog_timer: List of timers
*
* @proto_down_reason: reason a netdev interface is held down
* @pcpu_refcnt: Number of references to this device
* @dev_refcnt: Number of references to this device
* @refcnt_tracker: Tracker directory for tracked references to this device
* @todo_list: Delayed register/unregister
* @link_watch_list: XXX: need comments on this one
*
* @reg_state: Register/unregister state machine
* @dismantle: Device is going to be freed
* @rtnl_link_state: This enum represents the phases of creating
* a new link
*
* @needs_free_netdev: Should unregister perform free_netdev?
* @priv_destructor: Called from unregister
* @npinfo: XXX: need comments on this one
* @nd_net: Network namespace this network device is inside
*
* @ml_priv: Mid-layer private
* @ml_priv_type: Mid-layer private type
*
* @pcpu_stat_type: Type of device statistics which the core should
* allocate/free: none, lstats, tstats, dstats. none
* means the driver is handling statistics allocation/
* freeing internally.
* @lstats: Loopback statistics: packets, bytes
* @tstats: Tunnel statistics: RX/TX packets, RX/TX bytes
* @dstats: Dummy statistics: RX/TX/drop packets, RX/TX bytes
*
* @garp_port: GARP
* @mrp_port: MRP
*
* @dm_private: Drop monitor private
*
* @dev: Class/net/name entry
* @sysfs_groups: Space for optional device, statistics and wireless
* sysfs groups
*
* @sysfs_rx_queue_group: Space for optional per-rx queue attributes
* @rtnl_link_ops: Rtnl_link_ops
* @stat_ops: Optional ops for queue-aware statistics
*
* @gso_max_size: Maximum size of generic segmentation offload
* @tso_max_size: Device (as in HW) limit on the max TSO request size
* @gso_max_segs: Maximum number of segments that can be passed to the
* NIC for GSO
* @tso_max_segs: Device (as in HW) limit on the max TSO segment count
* @gso_ipv4_max_size: Maximum size of generic segmentation offload,
* for IPv4.
*
* @dcbnl_ops: Data Center Bridging netlink ops
* @num_tc: Number of traffic classes in the net device
* @tc_to_txq: XXX: need comments on this one
* @prio_tc_map: XXX: need comments on this one
*
* @fcoe_ddp_xid: Max exchange id for FCoE LRO by ddp
*
* @priomap: XXX: need comments on this one
* @phydev: Physical device may attach itself
* for hardware timestamping
* @sfp_bus: attached &struct sfp_bus structure.
*
* @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
*
* @proto_down: protocol port state information can be sent to the
* switch driver and used to set the phys state of the
* switch port.
*
* @wol_enabled: Wake-on-LAN is enabled
*
* @threaded: napi threaded mode is enabled
*
* @net_notifier_list: List of per-net netdev notifier block
* that follow this device when it is moved
* to another network namespace.
*
* @macsec_ops: MACsec offloading ops
*
* @udp_tunnel_nic_info: static structure describing the UDP tunnel
* offload capabilities of the device
* @udp_tunnel_nic: UDP tunnel offload state
* @xdp_state: stores info on attached XDP BPF programs
*
* @nested_level: Used as a parameter of spin_lock_nested() of
* dev->addr_list_lock.
* @unlink_list: As netif_addr_lock() can be called recursively,
* keep a list of interfaces to be deleted.
* @gro_max_size: Maximum size of aggregated packet in generic
* receive offload (GRO)
* @gro_ipv4_max_size: Maximum size of aggregated packet in generic
* receive offload (GRO), for IPv4.
* @xdp_zc_max_segs: Maximum number of segments supported by AF_XDP
* zero copy driver
*
* @dev_addr_shadow: Copy of @dev_addr to catch direct writes.
* @linkwatch_dev_tracker: refcount tracker used by linkwatch.
* @watchdog_dev_tracker: refcount tracker used by watchdog.
* @dev_registered_tracker: tracker for reference held while
* registered
* @offload_xstats_l3: L3 HW stats for this netdevice.
*
* @devlink_port: Pointer to related devlink port structure.
* Assigned by a driver before netdev registration using
* SET_NETDEV_DEVLINK_PORT macro. This pointer is static
* during the time netdevice is registered.
*
* @dpll_pin: Pointer to the SyncE source pin of a DPLL subsystem,
* where the clock is recovered.
*
* FIXME: cleanup struct net_device such that network protocol info
* moves out.
*/
struct net_device {
/* Cacheline organization can be found documented in
* Documentation/networking/net_cachelines/net_device.rst.
* Please update the document when adding new fields.
*/
/* TX read-mostly hotpath */
__cacheline_group_begin(net_device_read_tx);
unsigned long long priv_flags;
const struct net_device_ops *netdev_ops;
const struct header_ops *header_ops;
struct netdev_queue *_tx;
netdev_features_t gso_partial_features;
unsigned int real_num_tx_queues;
unsigned int gso_max_size;
unsigned int gso_ipv4_max_size;
u16 gso_max_segs;
s16 num_tc;
/* Note : dev->mtu is often read without holding a lock.
* Writers usually hold RTNL.
* It is recommended to use READ_ONCE() to annotate the reads,
* and to use WRITE_ONCE() to annotate the writes.
*/
unsigned int mtu;
unsigned short needed_headroom;
struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
#ifdef CONFIG_XPS
struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX];
#endif
#ifdef CONFIG_NETFILTER_EGRESS
struct nf_hook_entries __rcu *nf_hooks_egress;
#endif
#ifdef CONFIG_NET_XGRESS
struct bpf_mprog_entry __rcu *tcx_egress;
#endif
__cacheline_group_end(net_device_read_tx);
/* TXRX read-mostly hotpath */
__cacheline_group_begin(net_device_read_txrx);
union {
struct pcpu_lstats __percpu *lstats;
struct pcpu_sw_netstats __percpu *tstats;
struct pcpu_dstats __percpu *dstats;
};
unsigned long state;
unsigned int flags;
unsigned short hard_header_len;
netdev_features_t features;
struct inet6_dev __rcu *ip6_ptr;
__cacheline_group_end(net_device_read_txrx);
/* RX read-mostly hotpath */
__cacheline_group_begin(net_device_read_rx);
struct bpf_prog __rcu *xdp_prog;
struct list_head ptype_specific;
int ifindex;
unsigned int real_num_rx_queues;
struct netdev_rx_queue *_rx;
unsigned long gro_flush_timeout;
int napi_defer_hard_irqs;
unsigned int gro_max_size;
unsigned int gro_ipv4_max_size;
rx_handler_func_t __rcu *rx_handler;
void __rcu *rx_handler_data;
possible_net_t nd_net;
#ifdef CONFIG_NETPOLL
struct netpoll_info __rcu *npinfo;
#endif
#ifdef CONFIG_NET_XGRESS
struct bpf_mprog_entry __rcu *tcx_ingress;
#endif
__cacheline_group_end(net_device_read_rx);
char name[IFNAMSIZ];
struct netdev_name_node *name_node;
struct dev_ifalias __rcu *ifalias;
/*
* I/O specific fields
* FIXME: Merge these and struct ifmap into one
*/
unsigned long mem_end;
unsigned long mem_start;
unsigned long base_addr;
/*
* Some hardware also needs these fields (state,dev_list,
* napi_list,unreg_list,close_list) but they are not
* part of the usual set specified in Space.c.
*/
struct list_head dev_list;
struct list_head napi_list;
struct list_head unreg_list;
struct list_head close_list;
struct list_head ptype_all;
struct {
struct list_head upper;
struct list_head lower;
} adj_list;
/* Read-mostly cache-line for fast-path access */
xdp_features_t xdp_features;
const struct xdp_metadata_ops *xdp_metadata_ops;
const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
unsigned short gflags;
unsigned short needed_tailroom;
netdev_features_t hw_features;
netdev_features_t wanted_features;
netdev_features_t vlan_features;
netdev_features_t hw_enc_features;
netdev_features_t mpls_features;
unsigned int min_mtu;
unsigned int max_mtu;
unsigned short type;
unsigned char min_header_len;
unsigned char name_assign_type;
int group;
struct net_device_stats stats; /* not used by modern drivers */
struct net_device_core_stats __percpu *core_stats;
/* Stats to monitor link on/off, flapping */
atomic_t carrier_up_count;
atomic_t carrier_down_count;
#ifdef CONFIG_WIRELESS_EXT
const struct iw_handler_def *wireless_handlers;
struct iw_public_data *wireless_data;
#endif
const struct ethtool_ops *ethtool_ops;
#ifdef CONFIG_NET_L3_MASTER_DEV
const struct l3mdev_ops *l3mdev_ops;
#endif
#if IS_ENABLED(CONFIG_IPV6)
const struct ndisc_ops *ndisc_ops;
#endif
#ifdef CONFIG_XFRM_OFFLOAD
const struct xfrmdev_ops *xfrmdev_ops;
#endif
#if IS_ENABLED(CONFIG_TLS_DEVICE)
const struct tlsdev_ops *tlsdev_ops;
#endif
unsigned int operstate;
unsigned char link_mode;
unsigned char if_port;
unsigned char dma;
/* Interface address info. */
unsigned char perm_addr[MAX_ADDR_LEN];
unsigned char addr_assign_type;
unsigned char addr_len;
unsigned char upper_level;
unsigned char lower_level;
unsigned short neigh_priv_len;
unsigned short dev_id;
unsigned short dev_port;
unsigned short padded;
spinlock_t addr_list_lock;
int irq;
struct netdev_hw_addr_list uc;
struct netdev_hw_addr_list mc;
struct netdev_hw_addr_list dev_addrs;
#ifdef CONFIG_SYSFS
struct kset *queues_kset;
#endif
#ifdef CONFIG_LOCKDEP
struct list_head unlink_list;
#endif
unsigned int promiscuity;
unsigned int allmulti;
bool uc_promisc;
#ifdef CONFIG_LOCKDEP
unsigned char nested_level;
#endif
/* Protocol-specific pointers */
struct in_device __rcu *ip_ptr;
#if IS_ENABLED(CONFIG_VLAN_8021Q)
struct vlan_info __rcu *vlan_info;
#endif
#if IS_ENABLED(CONFIG_NET_DSA)
struct dsa_port *dsa_ptr;
#endif
#if IS_ENABLED(CONFIG_TIPC)
struct tipc_bearer __rcu *tipc_ptr;
#endif
#if IS_ENABLED(CONFIG_ATALK)
void *atalk_ptr;
#endif
#if IS_ENABLED(CONFIG_AX25)
void *ax25_ptr;
#endif
#if IS_ENABLED(CONFIG_CFG80211)
struct wireless_dev *ieee80211_ptr;
#endif
#if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN)
struct wpan_dev *ieee802154_ptr;
#endif
#if IS_ENABLED(CONFIG_MPLS_ROUTING)
struct mpls_dev __rcu *mpls_ptr;
#endif
#if IS_ENABLED(CONFIG_MCTP)
struct mctp_dev __rcu *mctp_ptr;
#endif
/*
* Cache lines mostly used on receive path (including eth_type_trans())
*/
/* Interface address info used in eth_type_trans() */
const unsigned char *dev_addr;
unsigned int num_rx_queues;
#define GRO_LEGACY_MAX_SIZE 65536u
/* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
* and shinfo->gso_segs is a 16bit field.
*/
#define GRO_MAX_SIZE (8 * 65535u)
unsigned int xdp_zc_max_segs;
struct netdev_queue __rcu *ingress_queue;
#ifdef CONFIG_NETFILTER_INGRESS
struct nf_hook_entries __rcu *nf_hooks_ingress;
#endif
unsigned char broadcast[MAX_ADDR_LEN];
#ifdef CONFIG_RFS_ACCEL
struct cpu_rmap *rx_cpu_rmap;
#endif
struct hlist_node index_hlist;
/*
* Cache lines mostly used on transmit path
*/
unsigned int num_tx_queues;
struct Qdisc __rcu *qdisc;
unsigned int tx_queue_len;
spinlock_t tx_global_lock;
struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
#ifdef CONFIG_NET_SCHED
DECLARE_HASHTABLE (qdisc_hash, 4);
#endif
/* These may be needed for future network-power-down code. */
struct timer_list watchdog_timer;
int watchdog_timeo;
u32 proto_down_reason;
struct list_head todo_list;
#ifdef CONFIG_PCPU_DEV_REFCNT
int __percpu *pcpu_refcnt;
#else
refcount_t dev_refcnt;
#endif
struct ref_tracker_dir refcnt_tracker;
struct list_head link_watch_list;
u8 reg_state;
bool dismantle;
enum {
RTNL_LINK_INITIALIZED,
RTNL_LINK_INITIALIZING,
} rtnl_link_state:16;
bool needs_free_netdev;
void (*priv_destructor)(struct net_device *dev);
/* mid-layer private */
void *ml_priv;
enum netdev_ml_priv_type ml_priv_type;
enum netdev_stat_type pcpu_stat_type:8;
#if IS_ENABLED(CONFIG_GARP)
struct garp_port __rcu *garp_port;
#endif
#if IS_ENABLED(CONFIG_MRP)
struct mrp_port __rcu *mrp_port;
#endif
#if IS_ENABLED(CONFIG_NET_DROP_MONITOR)
struct dm_hw_stat_delta __rcu *dm_private;
#endif
struct device dev;
const struct attribute_group *sysfs_groups[4];
const struct attribute_group *sysfs_rx_queue_group;
const struct rtnl_link_ops *rtnl_link_ops;
const struct netdev_stat_ops *stat_ops;
/* for setting kernel sock attribute on TCP connection setup */
#define GSO_MAX_SEGS 65535u
#define GSO_LEGACY_MAX_SIZE 65536u
/* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
* and shinfo->gso_segs is a 16bit field.
*/
#define GSO_MAX_SIZE (8 * GSO_MAX_SEGS)
#define TSO_LEGACY_MAX_SIZE 65536
#define TSO_MAX_SIZE UINT_MAX
unsigned int tso_max_size;
#define TSO_MAX_SEGS U16_MAX
u16 tso_max_segs;
#ifdef CONFIG_DCB
const struct dcbnl_rtnl_ops *dcbnl_ops;
#endif
u8 prio_tc_map[TC_BITMASK + 1];
#if IS_ENABLED(CONFIG_FCOE)
unsigned int fcoe_ddp_xid;
#endif
#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
struct netprio_map __rcu *priomap;
#endif
struct phy_device *phydev;
struct sfp_bus *sfp_bus;
struct lock_class_key *qdisc_tx_busylock;
bool proto_down;
unsigned wol_enabled:1;
unsigned threaded:1;
struct list_head net_notifier_list;
#if IS_ENABLED(CONFIG_MACSEC)
/* MACsec management functions */
const struct macsec_ops *macsec_ops;
#endif
const struct udp_tunnel_nic_info *udp_tunnel_nic_info;
struct udp_tunnel_nic *udp_tunnel_nic;
/* protected by rtnl_lock */
struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE];
u8 dev_addr_shadow[MAX_ADDR_LEN];
netdevice_tracker linkwatch_dev_tracker;
netdevice_tracker watchdog_dev_tracker;
netdevice_tracker dev_registered_tracker;
struct rtnl_hw_stats64 *offload_xstats_l3;
struct devlink_port *devlink_port;
#if IS_ENABLED(CONFIG_DPLL)
struct dpll_pin __rcu *dpll_pin;
#endif
#if IS_ENABLED(CONFIG_PAGE_POOL)
/** @page_pools: page pools created for this netdevice */
struct hlist_head page_pools;
#endif
};
#define to_net_dev(d) container_of(d, struct net_device, dev)
/*
* Driver should use this to assign devlink port instance to a netdevice
* before it registers the netdevice. Therefore devlink_port is static
* during the netdev lifetime after it is registered.
*/
#define SET_NETDEV_DEVLINK_PORT(dev, port) \
({ \
WARN_ON((dev)->reg_state != NETREG_UNINITIALIZED); \
((dev)->devlink_port = (port)); \
})
static inline bool netif_elide_gro(const struct net_device *dev)
{
if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
return true;
return false;
}
#define NETDEV_ALIGN 32
static inline
int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
{
return dev->prio_tc_map[prio & TC_BITMASK];
}
static inline
int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
{
if (tc >= dev->num_tc)
return -EINVAL;
dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
return 0;
}
int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
void netdev_reset_tc(struct net_device *dev);
int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
static inline
int netdev_get_num_tc(struct net_device *dev)
{
return dev->num_tc;
}
static inline void net_prefetch(void *p)
{
prefetch(p);
#if L1_CACHE_BYTES < 128
prefetch((u8 *)p + L1_CACHE_BYTES);
#endif
}
static inline void net_prefetchw(void *p)
{
prefetchw(p);
#if L1_CACHE_BYTES < 128
prefetchw((u8 *)p + L1_CACHE_BYTES);
#endif
}
void netdev_unbind_sb_channel(struct net_device *dev,
struct net_device *sb_dev);
int netdev_bind_sb_channel_queue(struct net_device *dev,
struct net_device *sb_dev,
u8 tc, u16 count, u16 offset);
int netdev_set_sb_channel(struct net_device *dev, u16 channel);
static inline int netdev_get_sb_channel(struct net_device *dev)
{
return max_t(int, -dev->num_tc, 0);
}
static inline
struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
unsigned int index)
{
DEBUG_NET_WARN_ON_ONCE(index >= dev->num_tx_queues); return &dev->_tx[index];
}
static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
const struct sk_buff *skb)
{
return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
}
static inline void netdev_for_each_tx_queue(struct net_device *dev,
void (*f)(struct net_device *,
struct netdev_queue *,
void *),
void *arg)
{
unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) f(dev, &dev->_tx[i], arg);
}
#define netdev_lockdep_set_classes(dev) \
{ \
static struct lock_class_key qdisc_tx_busylock_key; \
static struct lock_class_key qdisc_xmit_lock_key; \
static struct lock_class_key dev_addr_list_lock_key; \
unsigned int i; \
\
(dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key; \
lockdep_set_class(&(dev)->addr_list_lock, \
&dev_addr_list_lock_key); \
for (i = 0; i < (dev)->num_tx_queues; i++) \
lockdep_set_class(&(dev)->_tx[i]._xmit_lock, \
&qdisc_xmit_lock_key); \
}
u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
struct net_device *sb_dev);
struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
struct sk_buff *skb,
struct net_device *sb_dev);
/* returns the headroom that the master device needs to take in account
* when forwarding to this dev
*/
static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
{
return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
}
static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
{
if (dev->netdev_ops->ndo_set_rx_headroom)
dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
}
/* set the device rx headroom to the dev's default */
static inline void netdev_reset_rx_headroom(struct net_device *dev)
{
netdev_set_rx_headroom(dev, -1);
}
static inline void *netdev_get_ml_priv(struct net_device *dev,
enum netdev_ml_priv_type type)
{
if (dev->ml_priv_type != type)
return NULL;
return dev->ml_priv;
}
static inline void netdev_set_ml_priv(struct net_device *dev,
void *ml_priv,
enum netdev_ml_priv_type type)
{
WARN(dev->ml_priv_type && dev->ml_priv_type != type,
"Overwriting already set ml_priv_type (%u) with different ml_priv_type (%u)!\n",
dev->ml_priv_type, type);
WARN(!dev->ml_priv_type && dev->ml_priv,
"Overwriting already set ml_priv and ml_priv_type is ML_PRIV_NONE!\n");
dev->ml_priv = ml_priv;
dev->ml_priv_type = type;
}
/*
* Net namespace inlines
*/
static inline
struct net *dev_net(const struct net_device *dev)
{
return read_pnet(&dev->nd_net);
}
static inline
void dev_net_set(struct net_device *dev, struct net *net)
{
write_pnet(&dev->nd_net, net);
}
/**
* netdev_priv - access network device private data
* @dev: network device
*
* Get network device private data
*/
static inline void *netdev_priv(const struct net_device *dev)
{
return (char *)dev + ALIGN(sizeof(struct net_device), NETDEV_ALIGN);
}
/* Set the sysfs physical device reference for the network logical device
* if set prior to registration will cause a symlink during initialization.
*/
#define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
/* Set the sysfs device type for the network logical device to allow
* fine-grained identification of different network device types. For
* example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
*/
#define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
enum netdev_queue_type type,
struct napi_struct *napi);
static inline void netif_napi_set_irq(struct napi_struct *napi, int irq)
{
napi->irq = irq;
}
/* Default NAPI poll() weight
* Device drivers are strongly advised to not use bigger value
*/
#define NAPI_POLL_WEIGHT 64
void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
int (*poll)(struct napi_struct *, int), int weight);
/**
* netif_napi_add() - initialize a NAPI context
* @dev: network device
* @napi: NAPI context
* @poll: polling function
*
* netif_napi_add() must be used to initialize a NAPI context prior to calling
* *any* of the other NAPI-related functions.
*/
static inline void
netif_napi_add(struct net_device *dev, struct napi_struct *napi,
int (*poll)(struct napi_struct *, int))
{
netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
}
static inline void
netif_napi_add_tx_weight(struct net_device *dev,
struct napi_struct *napi,
int (*poll)(struct napi_struct *, int),
int weight)
{
set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
netif_napi_add_weight(dev, napi, poll, weight);
}
/**
* netif_napi_add_tx() - initialize a NAPI context to be used for Tx only
* @dev: network device
* @napi: NAPI context
* @poll: polling function
*
* This variant of netif_napi_add() should be used from drivers using NAPI
* to exclusively poll a TX queue.
* This will avoid we add it into napi_hash[], thus polluting this hash table.
*/
static inline void netif_napi_add_tx(struct net_device *dev,
struct napi_struct *napi,
int (*poll)(struct napi_struct *, int))
{
netif_napi_add_tx_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
}
/**
* __netif_napi_del - remove a NAPI context
* @napi: NAPI context
*
* Warning: caller must observe RCU grace period before freeing memory
* containing @napi. Drivers might want to call this helper to combine
* all the needed RCU grace periods into a single one.
*/
void __netif_napi_del(struct napi_struct *napi);
/**
* netif_napi_del - remove a NAPI context
* @napi: NAPI context
*
* netif_napi_del() removes a NAPI context from the network device NAPI list
*/
static inline void netif_napi_del(struct napi_struct *napi)
{
__netif_napi_del(napi);
synchronize_net();
}
struct packet_type {
__be16 type; /* This is really htons(ether_type). */
bool ignore_outgoing;
struct net_device *dev; /* NULL is wildcarded here */
netdevice_tracker dev_tracker;
int (*func) (struct sk_buff *,
struct net_device *,
struct packet_type *,
struct net_device *);
void (*list_func) (struct list_head *,
struct packet_type *,
struct net_device *);
bool (*id_match)(struct packet_type *ptype,
struct sock *sk);
struct net *af_packet_net;
void *af_packet_priv;
struct list_head list;
};
struct offload_callbacks {
struct sk_buff *(*gso_segment)(struct sk_buff *skb,
netdev_features_t features);
struct sk_buff *(*gro_receive)(struct list_head *head,
struct sk_buff *skb);
int (*gro_complete)(struct sk_buff *skb, int nhoff);
};
struct packet_offload {
__be16 type; /* This is really htons(ether_type). */
u16 priority;
struct offload_callbacks callbacks;
struct list_head list;
};
/* often modified stats are per-CPU, other are shared (netdev->stats) */
struct pcpu_sw_netstats {
u64_stats_t rx_packets;
u64_stats_t rx_bytes;
u64_stats_t tx_packets;
u64_stats_t tx_bytes;
struct u64_stats_sync syncp;
} __aligned(4 * sizeof(u64));
struct pcpu_dstats {
u64 rx_packets;
u64 rx_bytes;
u64 rx_drops;
u64 tx_packets;
u64 tx_bytes;
u64 tx_drops;
struct u64_stats_sync syncp;
} __aligned(8 * sizeof(u64));
struct pcpu_lstats {
u64_stats_t packets;
u64_stats_t bytes;
struct u64_stats_sync syncp;
} __aligned(2 * sizeof(u64));
void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);
static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
{
struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
u64_stats_update_begin(&tstats->syncp);
u64_stats_add(&tstats->rx_bytes, len);
u64_stats_inc(&tstats->rx_packets);
u64_stats_update_end(&tstats->syncp);
}
static inline void dev_sw_netstats_tx_add(struct net_device *dev,
unsigned int packets,
unsigned int len)
{
struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
u64_stats_update_begin(&tstats->syncp);
u64_stats_add(&tstats->tx_bytes, len);
u64_stats_add(&tstats->tx_packets, packets);
u64_stats_update_end(&tstats->syncp);
}
static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
{
struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);
u64_stats_update_begin(&lstats->syncp);
u64_stats_add(&lstats->bytes, len);
u64_stats_inc(&lstats->packets);
u64_stats_update_end(&lstats->syncp);
}
#define __netdev_alloc_pcpu_stats(type, gfp) \
({ \
typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
if (pcpu_stats) { \
int __cpu; \
for_each_possible_cpu(__cpu) { \
typeof(type) *stat; \
stat = per_cpu_ptr(pcpu_stats, __cpu); \
u64_stats_init(&stat->syncp); \
} \
} \
pcpu_stats; \
})
#define netdev_alloc_pcpu_stats(type) \
__netdev_alloc_pcpu_stats(type, GFP_KERNEL)
#define devm_netdev_alloc_pcpu_stats(dev, type) \
({ \
typeof(type) __percpu *pcpu_stats = devm_alloc_percpu(dev, type);\
if (pcpu_stats) { \
int __cpu; \
for_each_possible_cpu(__cpu) { \
typeof(type) *stat; \
stat = per_cpu_ptr(pcpu_stats, __cpu); \
u64_stats_init(&stat->syncp); \
} \
} \
pcpu_stats; \
})
enum netdev_lag_tx_type {
NETDEV_LAG_TX_TYPE_UNKNOWN,
NETDEV_LAG_TX_TYPE_RANDOM,
NETDEV_LAG_TX_TYPE_BROADCAST,
NETDEV_LAG_TX_TYPE_ROUNDROBIN,
NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
NETDEV_LAG_TX_TYPE_HASH,
};
enum netdev_lag_hash {
NETDEV_LAG_HASH_NONE,
NETDEV_LAG_HASH_L2,
NETDEV_LAG_HASH_L34,
NETDEV_LAG_HASH_L23,
NETDEV_LAG_HASH_E23,
NETDEV_LAG_HASH_E34,
NETDEV_LAG_HASH_VLAN_SRCMAC,
NETDEV_LAG_HASH_UNKNOWN,
};
struct netdev_lag_upper_info {
enum netdev_lag_tx_type tx_type;
enum netdev_lag_hash hash_type;
};
struct netdev_lag_lower_state_info {
u8 link_up : 1,
tx_enabled : 1;
};
#include <linux/notifier.h>
/* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
* and the rtnetlink notification exclusion list in rtnetlink_event() when
* adding new types.
*/
enum netdev_cmd {
NETDEV_UP = 1, /* For now you can't veto a device up/down */
NETDEV_DOWN,
NETDEV_REBOOT, /* Tell a protocol stack a network interface
detected a hardware crash and restarted
- we can use this eg to kick tcp sessions
once done */
NETDEV_CHANGE, /* Notify device state change */
NETDEV_REGISTER,
NETDEV_UNREGISTER,
NETDEV_CHANGEMTU, /* notify after mtu change happened */
NETDEV_CHANGEADDR, /* notify after the address change */
NETDEV_PRE_CHANGEADDR, /* notify before the address change */
NETDEV_GOING_DOWN,
NETDEV_CHANGENAME,
NETDEV_FEAT_CHANGE,
NETDEV_BONDING_FAILOVER,
NETDEV_PRE_UP,
NETDEV_PRE_TYPE_CHANGE,
NETDEV_POST_TYPE_CHANGE,
NETDEV_POST_INIT,
NETDEV_PRE_UNINIT,
NETDEV_RELEASE,
NETDEV_NOTIFY_PEERS,
NETDEV_JOIN,
NETDEV_CHANGEUPPER,
NETDEV_RESEND_IGMP,
NETDEV_PRECHANGEMTU, /* notify before mtu change happened */
NETDEV_CHANGEINFODATA,
NETDEV_BONDING_INFO,
NETDEV_PRECHANGEUPPER,
NETDEV_CHANGELOWERSTATE,
NETDEV_UDP_TUNNEL_PUSH_INFO,
NETDEV_UDP_TUNNEL_DROP_INFO,
NETDEV_CHANGE_TX_QUEUE_LEN,
NETDEV_CVLAN_FILTER_PUSH_INFO,
NETDEV_CVLAN_FILTER_DROP_INFO,
NETDEV_SVLAN_FILTER_PUSH_INFO,
NETDEV_SVLAN_FILTER_DROP_INFO,
NETDEV_OFFLOAD_XSTATS_ENABLE,
NETDEV_OFFLOAD_XSTATS_DISABLE,
NETDEV_OFFLOAD_XSTATS_REPORT_USED,
NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
NETDEV_XDP_FEAT_CHANGE,
};
const char *netdev_cmd_to_name(enum netdev_cmd cmd);
int register_netdevice_notifier(struct notifier_block *nb);
int unregister_netdevice_notifier(struct notifier_block *nb);
int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
int unregister_netdevice_notifier_net(struct net *net,
struct notifier_block *nb);
int register_netdevice_notifier_dev_net(struct net_device *dev,
struct notifier_block *nb,
struct netdev_net_notifier *nn);
int unregister_netdevice_notifier_dev_net(struct net_device *dev,
struct notifier_block *nb,
struct netdev_net_notifier *nn);
struct netdev_notifier_info {
struct net_device *dev;
struct netlink_ext_ack *extack;
};
struct netdev_notifier_info_ext {
struct netdev_notifier_info info; /* must be first */
union {
u32 mtu;
} ext;
};
struct netdev_notifier_change_info {
struct netdev_notifier_info info; /* must be first */
unsigned int flags_changed;
};
struct netdev_notifier_changeupper_info {
struct netdev_notifier_info info; /* must be first */
struct net_device *upper_dev; /* new upper dev */
bool master; /* is upper dev master */
bool linking; /* is the notification for link or unlink */
void *upper_info; /* upper dev info */
};
struct netdev_notifier_changelowerstate_info {
struct netdev_notifier_info info; /* must be first */
void *lower_state_info; /* is lower dev state */
};
struct netdev_notifier_pre_changeaddr_info {
struct netdev_notifier_info info; /* must be first */
const unsigned char *dev_addr;
};
enum netdev_offload_xstats_type {
NETDEV_OFFLOAD_XSTATS_TYPE_L3 = 1,
};
struct netdev_notifier_offload_xstats_info {
struct netdev_notifier_info info; /* must be first */
enum netdev_offload_xstats_type type;
union {
/* NETDEV_OFFLOAD_XSTATS_REPORT_DELTA */
struct netdev_notifier_offload_xstats_rd *report_delta;
/* NETDEV_OFFLOAD_XSTATS_REPORT_USED */
struct netdev_notifier_offload_xstats_ru *report_used;
};
};
int netdev_offload_xstats_enable(struct net_device *dev,
enum netdev_offload_xstats_type type,
struct netlink_ext_ack *extack);
int netdev_offload_xstats_disable(struct net_device *dev,
enum netdev_offload_xstats_type type);
bool netdev_offload_xstats_enabled(const struct net_device *dev,
enum netdev_offload_xstats_type type);
int netdev_offload_xstats_get(struct net_device *dev,
enum netdev_offload_xstats_type type,
struct rtnl_hw_stats64 *stats, bool *used,
struct netlink_ext_ack *extack);
void
netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *rd,
const struct rtnl_hw_stats64 *stats);
void
netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *ru);
void netdev_offload_xstats_push_delta(struct net_device *dev,
enum netdev_offload_xstats_type type,
const struct rtnl_hw_stats64 *stats);
static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
struct net_device *dev)
{
info->dev = dev;
info->extack = NULL;
}
static inline struct net_device *
netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
{
return info->dev;
}
static inline struct netlink_ext_ack *
netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
{
return info->extack;
}
int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
int call_netdevice_notifiers_info(unsigned long val,
struct netdev_notifier_info *info);
#define for_each_netdev(net, d) \
list_for_each_entry(d, &(net)->dev_base_head, dev_list)
#define for_each_netdev_reverse(net, d) \
list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
#define for_each_netdev_rcu(net, d) \
list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
#define for_each_netdev_safe(net, d, n) \
list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
#define for_each_netdev_continue(net, d) \
list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
#define for_each_netdev_continue_reverse(net, d) \
list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
dev_list)
#define for_each_netdev_continue_rcu(net, d) \
list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
#define for_each_netdev_in_bond_rcu(bond, slave) \
for_each_netdev_rcu(&init_net, slave) \
if (netdev_master_upper_dev_get_rcu(slave) == (bond))
#define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
#define for_each_netdev_dump(net, d, ifindex) \
xa_for_each_start(&(net)->dev_by_index, (ifindex), (d), (ifindex))
static inline struct net_device *next_net_device(struct net_device *dev)
{
struct list_head *lh;
struct net *net;
net = dev_net(dev);
lh = dev->dev_list.next;
return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
}
static inline struct net_device *next_net_device_rcu(struct net_device *dev)
{
struct list_head *lh;
struct net *net;
net = dev_net(dev);
lh = rcu_dereference(list_next_rcu(&dev->dev_list));
return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
}
static inline struct net_device *first_net_device(struct net *net)
{
return list_empty(&net->dev_base_head) ? NULL :
net_device_entry(net->dev_base_head.next);
}
static inline struct net_device *first_net_device_rcu(struct net *net)
{
struct list_head *lh = rcu_dereference(list_next_rcu(&net->dev_base_head));
return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
}
int netdev_boot_setup_check(struct net_device *dev);
struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
const char *hwaddr);
struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
void dev_add_pack(struct packet_type *pt);
void dev_remove_pack(struct packet_type *pt);
void __dev_remove_pack(struct packet_type *pt);
void dev_add_offload(struct packet_offload *po);
void dev_remove_offload(struct packet_offload *po);
int dev_get_iflink(const struct net_device *dev);
int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
struct net_device_path_stack *stack);
struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
unsigned short mask);
struct net_device *dev_get_by_name(struct net *net, const char *name);
struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
struct net_device *__dev_get_by_name(struct net *net, const char *name);
bool netdev_name_in_use(struct net *net, const char *name);
int dev_alloc_name(struct net_device *dev, const char *name);
int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
void dev_close(struct net_device *dev);
void dev_close_many(struct list_head *head, bool unlink);
void dev_disable_lro(struct net_device *dev);
int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
struct net_device *sb_dev);
u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
struct net_device *sb_dev);
int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev);
int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
static inline int dev_queue_xmit(struct sk_buff *skb)
{
return __dev_queue_xmit(skb, NULL);
}
static inline int dev_queue_xmit_accel(struct sk_buff *skb,
struct net_device *sb_dev)
{
return __dev_queue_xmit(skb, sb_dev);
}
static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
{
int ret;
ret = __dev_direct_xmit(skb, queue_id);
if (!dev_xmit_complete(ret))
kfree_skb(skb);
return ret;
}
int register_netdevice(struct net_device *dev);
void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
void unregister_netdevice_many(struct list_head *head);
static inline void unregister_netdevice(struct net_device *dev)
{
unregister_netdevice_queue(dev, NULL);
}
int netdev_refcnt_read(const struct net_device *dev);
void free_netdev(struct net_device *dev);
void netdev_freemem(struct net_device *dev);
void init_dummy_netdev(struct net_device *dev);
struct net_device *netdev_get_xmit_slave(struct net_device *dev,
struct sk_buff *skb,
bool all_slaves);
struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
struct sock *sk);
struct net_device *dev_get_by_index(struct net *net, int ifindex);
struct net_device *__dev_get_by_index(struct net *net, int ifindex);
struct net_device *netdev_get_by_index(struct net *net, int ifindex,
netdevice_tracker *tracker, gfp_t gfp);
struct net_device *netdev_get_by_name(struct net *net, const char *name,
netdevice_tracker *tracker, gfp_t gfp);
struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
struct net_device *dev_get_by_napi_id(unsigned int napi_id);
static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
unsigned short type,
const void *daddr, const void *saddr,
unsigned int len)
{
if (!dev->header_ops || !dev->header_ops->create)
return 0;
return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
}
static inline int dev_parse_header(const struct sk_buff *skb,
unsigned char *haddr)
{
const struct net_device *dev = skb->dev;
if (!dev->header_ops || !dev->header_ops->parse)
return 0;
return dev->header_ops->parse(skb, haddr);
}
static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
{
const struct net_device *dev = skb->dev;
if (!dev->header_ops || !dev->header_ops->parse_protocol)
return 0;
return dev->header_ops->parse_protocol(skb);
}
/* ll_header must have at least hard_header_len allocated */
static inline bool dev_validate_header(const struct net_device *dev,
char *ll_header, int len)
{
if (likely(len >= dev->hard_header_len))
return true;
if (len < dev->min_header_len)
return false;
if (capable(CAP_SYS_RAWIO)) {
memset(ll_header + len, 0, dev->hard_header_len - len);
return true;
}
if (dev->header_ops && dev->header_ops->validate)
return dev->header_ops->validate(ll_header, len);
return false;
}
static inline bool dev_has_header(const struct net_device *dev)
{
return dev->header_ops && dev->header_ops->create;
}
/*
* Incoming packets are placed on per-CPU queues
*/
struct softnet_data {
struct list_head poll_list;
struct sk_buff_head process_queue;
/* stats */
unsigned int processed;
unsigned int time_squeeze;
#ifdef CONFIG_RPS
struct softnet_data *rps_ipi_list;
#endif
bool in_net_rx_action;
bool in_napi_threaded_poll;
#ifdef CONFIG_NET_FLOW_LIMIT
struct sd_flow_limit __rcu *flow_limit;
#endif
struct Qdisc *output_queue;
struct Qdisc **output_queue_tailp;
struct sk_buff *completion_queue;
#ifdef CONFIG_XFRM_OFFLOAD
struct sk_buff_head xfrm_backlog;
#endif
/* written and read only by owning cpu: */
struct {
u16 recursion;
u8 more;
#ifdef CONFIG_NET_EGRESS
u8 skip_txqueue;
#endif
} xmit;
#ifdef CONFIG_RPS
/* input_queue_head should be written by cpu owning this struct,
* and only read by other cpus. Worth using a cache line.
*/
unsigned int input_queue_head ____cacheline_aligned_in_smp;
/* Elements below can be accessed between CPUs for RPS/RFS */
call_single_data_t csd ____cacheline_aligned_in_smp;
struct softnet_data *rps_ipi_next;
unsigned int cpu;
unsigned int input_queue_tail;
#endif
unsigned int received_rps;
unsigned int dropped;
struct sk_buff_head input_pkt_queue;
struct napi_struct backlog;
/* Another possibly contended cache line */
spinlock_t defer_lock ____cacheline_aligned_in_smp;
int defer_count;
int defer_ipi_scheduled;
struct sk_buff *defer_list;
call_single_data_t defer_csd;
};
static inline void input_queue_head_incr(struct softnet_data *sd)
{
#ifdef CONFIG_RPS
sd->input_queue_head++;
#endif
}
static inline void input_queue_tail_incr_save(struct softnet_data *sd,
unsigned int *qtail)
{
#ifdef CONFIG_RPS
*qtail = ++sd->input_queue_tail;
#endif
}
DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
static inline int dev_recursion_level(void)
{
return this_cpu_read(softnet_data.xmit.recursion);
}
#define XMIT_RECURSION_LIMIT 8
static inline bool dev_xmit_recursion(void)
{
return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
XMIT_RECURSION_LIMIT);
}
static inline void dev_xmit_recursion_inc(void)
{
__this_cpu_inc(softnet_data.xmit.recursion);
}
static inline void dev_xmit_recursion_dec(void)
{
__this_cpu_dec(softnet_data.xmit.recursion);
}
void __netif_schedule(struct Qdisc *q);
void netif_schedule_queue(struct netdev_queue *txq);
static inline void netif_tx_schedule_all(struct net_device *dev)
{
unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++)
netif_schedule_queue(netdev_get_tx_queue(dev, i));
}
static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
{
clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
}
/**
* netif_start_queue - allow transmit
* @dev: network device
*
* Allow upper layers to call the device hard_start_xmit routine.
*/
static inline void netif_start_queue(struct net_device *dev)
{
netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
}
static inline void netif_tx_start_all_queues(struct net_device *dev)
{
unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
netif_tx_start_queue(txq);
}
}
void netif_tx_wake_queue(struct netdev_queue *dev_queue);
/**
* netif_wake_queue - restart transmit
* @dev: network device
*
* Allow upper layers to call the device hard_start_xmit routine.
* Used for flow control when transmit resources are available.
*/
static inline void netif_wake_queue(struct net_device *dev)
{
netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
}
static inline void netif_tx_wake_all_queues(struct net_device *dev)
{
unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
netif_tx_wake_queue(txq);
}
}
static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
{
/* Must be an atomic op see netif_txq_try_stop() */
set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
}
/**
* netif_stop_queue - stop transmitted packets
* @dev: network device
*
* Stop upper layers calling the device hard_start_xmit routine.
* Used for flow control when transmit resources are unavailable.
*/
static inline void netif_stop_queue(struct net_device *dev)
{
netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
}
void netif_tx_stop_all_queues(struct net_device *dev);
static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
{
return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
}
/**
* netif_queue_stopped - test if transmit queue is flowblocked
* @dev: network device
*
* Test if transmit queue on device is currently unable to send.
*/
static inline bool netif_queue_stopped(const struct net_device *dev)
{
return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
}
static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
{
return dev_queue->state & QUEUE_STATE_ANY_XOFF;
}
static inline bool
netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
{
return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
}
static inline bool
netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
{
return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
}
/**
* netdev_queue_set_dql_min_limit - set dql minimum limit
* @dev_queue: pointer to transmit queue
* @min_limit: dql minimum limit
*
* Forces xmit_more() to return true until the minimum threshold
* defined by @min_limit is reached (or until the tx queue is
* empty). Warning: to be use with care, misuse will impact the
* latency.
*/
static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,
unsigned int min_limit)
{
#ifdef CONFIG_BQL
dev_queue->dql.min_limit = min_limit;
#endif
}
static inline int netdev_queue_dql_avail(const struct netdev_queue *txq)
{
#ifdef CONFIG_BQL
/* Non-BQL migrated drivers will return 0, too. */
return dql_avail(&txq->dql);
#else
return 0;
#endif
}
/**
* netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
* @dev_queue: pointer to transmit queue
*
* BQL enabled drivers might use this helper in their ndo_start_xmit(),
* to give appropriate hint to the CPU.
*/
static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
{
#ifdef CONFIG_BQL
prefetchw(&dev_queue->dql.num_queued);
#endif
}
/**
* netdev_txq_bql_complete_prefetchw - prefetch bql data for write
* @dev_queue: pointer to transmit queue
*
* BQL enabled drivers might use this helper in their TX completion path,
* to give appropriate hint to the CPU.
*/
static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
{
#ifdef CONFIG_BQL
prefetchw(&dev_queue->dql.limit);
#endif
}
/**
* netdev_tx_sent_queue - report the number of bytes queued to a given tx queue
* @dev_queue: network device queue
* @bytes: number of bytes queued to the device queue
*
* Report the number of bytes queued for sending/completion to the network
* device hardware queue. @bytes should be a good approximation and should
* exactly match netdev_completed_queue() @bytes.
* This is typically called once per packet, from ndo_start_xmit().
*/
static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
unsigned int bytes)
{
#ifdef CONFIG_BQL
dql_queued(&dev_queue->dql, bytes);
if (likely(dql_avail(&dev_queue->dql) >= 0))
return;
set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
/*
* The XOFF flag must be set before checking the dql_avail below,
* because in netdev_tx_completed_queue we update the dql_completed
* before checking the XOFF flag.
*/
smp_mb();
/* check again in case another CPU has just made room avail */
if (unlikely(dql_avail(&dev_queue->dql) >= 0))
clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
#endif
}
/* Variant of netdev_tx_sent_queue() for drivers that are aware
* that they should not test BQL status themselves.
* We do want to change __QUEUE_STATE_STACK_XOFF only for the last
* skb of a batch.
* Returns true if the doorbell must be used to kick the NIC.
*/
static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
unsigned int bytes,
bool xmit_more)
{
if (xmit_more) {
#ifdef CONFIG_BQL
dql_queued(&dev_queue->dql, bytes);
#endif
return netif_tx_queue_stopped(dev_queue);
}
netdev_tx_sent_queue(dev_queue, bytes);
return true;
}
/**
* netdev_sent_queue - report the number of bytes queued to hardware
* @dev: network device
* @bytes: number of bytes queued to the hardware device queue
*
* Report the number of bytes queued for sending/completion to the network
* device hardware queue#0. @bytes should be a good approximation and should
* exactly match netdev_completed_queue() @bytes.
* This is typically called once per packet, from ndo_start_xmit().
*/
static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
{
netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
}
static inline bool __netdev_sent_queue(struct net_device *dev,
unsigned int bytes,
bool xmit_more)
{
return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
xmit_more);
}
/**
* netdev_tx_completed_queue - report number of packets/bytes at TX completion.
* @dev_queue: network device queue
* @pkts: number of packets (currently ignored)
* @bytes: number of bytes dequeued from the device queue
*
* Must be called at most once per TX completion round (and not per
* individual packet), so that BQL can adjust its limits appropriately.
*/
static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
unsigned int pkts, unsigned int bytes)
{
#ifdef CONFIG_BQL
if (unlikely(!bytes))
return;
dql_completed(&dev_queue->dql, bytes);
/*
* Without the memory barrier there is a small possiblity that
* netdev_tx_sent_queue will miss the update and cause the queue to
* be stopped forever
*/
smp_mb(); /* NOTE: netdev_txq_completed_mb() assumes this exists */
if (unlikely(dql_avail(&dev_queue->dql) < 0))
return;
if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
netif_schedule_queue(dev_queue);
#endif
}
/**
* netdev_completed_queue - report bytes and packets completed by device
* @dev: network device
* @pkts: actual number of packets sent over the medium
* @bytes: actual number of bytes sent over the medium
*
* Report the number of bytes and packets transmitted by the network device
* hardware queue over the physical medium, @bytes must exactly match the
* @bytes amount passed to netdev_sent_queue()
*/
static inline void netdev_completed_queue(struct net_device *dev,
unsigned int pkts, unsigned int bytes)
{
netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
}
static inline void netdev_tx_reset_queue(struct netdev_queue *q)
{
#ifdef CONFIG_BQL
clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
dql_reset(&q->dql);
#endif
}
/**
* netdev_reset_queue - reset the packets and bytes count of a network device
* @dev_queue: network device
*
* Reset the bytes and packet count of a network device and clear the
* software flow control OFF bit for this network device
*/
static inline void netdev_reset_queue(struct net_device *dev_queue)
{
netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0));
}
/**
* netdev_cap_txqueue - check if selected tx queue exceeds device queues
* @dev: network device
* @queue_index: given tx queue index
*
* Returns 0 if given tx queue index >= number of device tx queues,
* otherwise returns the originally passed tx queue index.
*/
static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
{
if (unlikely(queue_index >= dev->real_num_tx_queues)) {
net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
dev->name, queue_index,
dev->real_num_tx_queues);
return 0;
}
return queue_index;
}
/**
* netif_running - test if up
* @dev: network device
*
* Test if the device has been brought up.
*/
static inline bool netif_running(const struct net_device *dev)
{
return test_bit(__LINK_STATE_START, &dev->state);
}
/*
* Routines to manage the subqueues on a device. We only need start,
* stop, and a check if it's stopped. All other device management is
* done at the overall netdevice level.
* Also test the device if we're multiqueue.
*/
/**
* netif_start_subqueue - allow sending packets on subqueue
* @dev: network device
* @queue_index: sub queue index
*
* Start individual transmit queue of a device with multiple transmit queues.
*/
static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
{
struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
netif_tx_start_queue(txq);
}
/**
* netif_stop_subqueue - stop sending packets on subqueue
* @dev: network device
* @queue_index: sub queue index
*
* Stop individual transmit queue of a device with multiple transmit queues.
*/
static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
{
struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
netif_tx_stop_queue(txq);
}
/**
* __netif_subqueue_stopped - test status of subqueue
* @dev: network device
* @queue_index: sub queue index
*
* Check individual transmit queue of a device with multiple transmit queues.
*/
static inline bool __netif_subqueue_stopped(const struct net_device *dev,
u16 queue_index)
{
struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
return netif_tx_queue_stopped(txq);
}
/**
* netif_subqueue_stopped - test status of subqueue
* @dev: network device
* @skb: sub queue buffer pointer
*
* Check individual transmit queue of a device with multiple transmit queues.
*/
static inline bool netif_subqueue_stopped(const struct net_device *dev,
struct sk_buff *skb)
{
return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
}
/**
* netif_wake_subqueue - allow sending packets on subqueue
* @dev: network device
* @queue_index: sub queue index
*
* Resume individual transmit queue of a device with multiple transmit queues.
*/
static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
{
struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
netif_tx_wake_queue(txq);
}
#ifdef CONFIG_XPS
int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
u16 index);
int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
u16 index, enum xps_map_type type);
/**
* netif_attr_test_mask - Test a CPU or Rx queue set in a mask
* @j: CPU/Rx queue index
* @mask: bitmask of all cpus/rx queues
* @nr_bits: number of bits in the bitmask
*
* Test if a CPU or Rx queue index is set in a mask of all CPU/Rx queues.
*/
static inline bool netif_attr_test_mask(unsigned long j,
const unsigned long *mask,
unsigned int nr_bits)
{
cpu_max_bits_warn(j, nr_bits);
return test_bit(j, mask);
}
/**
* netif_attr_test_online - Test for online CPU/Rx queue
* @j: CPU/Rx queue index
* @online_mask: bitmask for CPUs/Rx queues that are online
* @nr_bits: number of bits in the bitmask
*
* Returns true if a CPU/Rx queue is online.
*/
static inline bool netif_attr_test_online(unsigned long j,
const unsigned long *online_mask,
unsigned int nr_bits)
{
cpu_max_bits_warn(j, nr_bits);
if (online_mask)
return test_bit(j, online_mask);
return (j < nr_bits);
}
/**
* netif_attrmask_next - get the next CPU/Rx queue in a cpu/Rx queues mask
* @n: CPU/Rx queue index
* @srcp: the cpumask/Rx queue mask pointer
* @nr_bits: number of bits in the bitmask
*
* Returns >= nr_bits if no further CPUs/Rx queues set.
*/
static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
unsigned int nr_bits)
{
/* -1 is a legal arg here. */
if (n != -1)
cpu_max_bits_warn(n, nr_bits);
if (srcp)
return find_next_bit(srcp, nr_bits, n + 1);
return n + 1;
}
/**
* netif_attrmask_next_and - get the next CPU/Rx queue in \*src1p & \*src2p
* @n: CPU/Rx queue index
* @src1p: the first CPUs/Rx queues mask pointer
* @src2p: the second CPUs/Rx queues mask pointer
* @nr_bits: number of bits in the bitmask
*
* Returns >= nr_bits if no further CPUs/Rx queues set in both.
*/
static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
const unsigned long *src2p,
unsigned int nr_bits)
{
/* -1 is a legal arg here. */
if (n != -1)
cpu_max_bits_warn(n, nr_bits);
if (src1p && src2p)
return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
else if (src1p)
return find_next_bit(src1p, nr_bits, n + 1);
else if (src2p)
return find_next_bit(src2p, nr_bits, n + 1);
return n + 1;
}
#else
static inline int netif_set_xps_queue(struct net_device *dev,
const struct cpumask *mask,
u16 index)
{
return 0;
}
static inline int __netif_set_xps_queue(struct net_device *dev,
const unsigned long *mask,
u16 index, enum xps_map_type type)
{
return 0;
}
#endif
/**
* netif_is_multiqueue - test if device has multiple transmit queues
* @dev: network device
*
* Check if device has multiple transmit queues
*/
static inline bool netif_is_multiqueue(const struct net_device *dev)
{
return dev->num_tx_queues > 1;
}
int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
#ifdef CONFIG_SYSFS
int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
#else
static inline int netif_set_real_num_rx_queues(struct net_device *dev,
unsigned int rxqs)
{
dev->real_num_rx_queues = rxqs;
return 0;
}
#endif
int netif_set_real_num_queues(struct net_device *dev,
unsigned int txq, unsigned int rxq);
int netif_get_num_default_rss_queues(void);
void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason);
void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason);
/*
* It is not allowed to call kfree_skb() or consume_skb() from hardware
* interrupt context or with hardware interrupts being disabled.
* (in_hardirq() || irqs_disabled())
*
* We provide four helpers that can be used in following contexts :
*
* dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
* replacing kfree_skb(skb)
*
* dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
* Typically used in place of consume_skb(skb) in TX completion path
*
* dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
* replacing kfree_skb(skb)
*
* dev_consume_skb_any(skb) when caller doesn't know its current irq context,
* and consumed a packet. Used in place of consume_skb(skb)
*/
static inline void dev_kfree_skb_irq(struct sk_buff *skb)
{
dev_kfree_skb_irq_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
}
static inline void dev_consume_skb_irq(struct sk_buff *skb)
{
dev_kfree_skb_irq_reason(skb, SKB_CONSUMED);
}
static inline void dev_kfree_skb_any(struct sk_buff *skb)
{
dev_kfree_skb_any_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
}
static inline void dev_consume_skb_any(struct sk_buff *skb)
{
dev_kfree_skb_any_reason(skb, SKB_CONSUMED);
}
u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
struct bpf_prog *xdp_prog);
void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb);
int netif_rx(struct sk_buff *skb);
int __netif_rx(struct sk_buff *skb);
int netif_receive_skb(struct sk_buff *skb);
int netif_receive_skb_core(struct sk_buff *skb);
void netif_receive_skb_list_internal(struct list_head *head);
void netif_receive_skb_list(struct list_head *head);
gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
void napi_gro_flush(struct napi_struct *napi, bool flush_old);
struct sk_buff *napi_get_frags(struct napi_struct *napi);
void napi_get_frags_check(struct napi_struct *napi);
gro_result_t napi_gro_frags(struct napi_struct *napi);
static inline void napi_free_frags(struct napi_struct *napi)
{
kfree_skb(napi->skb);
napi->skb = NULL;
}
bool netdev_is_rx_handler_busy(struct net_device *dev);
int netdev_rx_handler_register(struct net_device *dev,
rx_handler_func_t *rx_handler,
void *rx_handler_data);
void netdev_rx_handler_unregister(struct net_device *dev);
bool dev_valid_name(const char *name);
static inline bool is_socket_ioctl_cmd(unsigned int cmd)
{
return _IOC_TYPE(cmd) == SOCK_IOC_TYPE;
}
int get_user_ifreq(struct ifreq *ifr, void __user **ifrdata, void __user *arg);
int put_user_ifreq(struct ifreq *ifr, void __user *arg);
int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
void __user *data, bool *need_copyout);
int dev_ifconf(struct net *net, struct ifconf __user *ifc);
int generic_hwtstamp_get_lower(struct net_device *dev,
struct kernel_hwtstamp_config *kernel_cfg);
int generic_hwtstamp_set_lower(struct net_device *dev,
struct kernel_hwtstamp_config *kernel_cfg,
struct netlink_ext_ack *extack);
int dev_set_hwtstamp_phylib(struct net_device *dev,
struct kernel_hwtstamp_config *cfg,
struct netlink_ext_ack *extack);
int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
unsigned int dev_get_flags(const struct net_device *);
int __dev_change_flags(struct net_device *dev, unsigned int flags,
struct netlink_ext_ack *extack);
int dev_change_flags(struct net_device *dev, unsigned int flags,
struct netlink_ext_ack *extack);
int dev_set_alias(struct net_device *, const char *, size_t);
int dev_get_alias(const struct net_device *, char *, size_t);
int __dev_change_net_namespace(struct net_device *dev, struct net *net,
const char *pat, int new_ifindex);
static inline
int dev_change_net_namespace(struct net_device *dev, struct net *net,
const char *pat)
{
return __dev_change_net_namespace(dev, net, pat, 0);
}
int __dev_set_mtu(struct net_device *, int);
int dev_set_mtu(struct net_device *, int);
int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
struct netlink_ext_ack *extack);
int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
struct netlink_ext_ack *extack);
int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
struct netlink_ext_ack *extack);
int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name);
int dev_get_port_parent_id(struct net_device *dev,
struct netdev_phys_item_id *ppid, bool recurse);
bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
struct netdev_queue *txq, int *ret);
int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
u8 dev_xdp_prog_count(struct net_device *dev);
u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
bool is_skb_forwardable(const struct net_device *dev,
const struct sk_buff *skb);
static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
const struct sk_buff *skb,
const bool check_mtu)
{
const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
unsigned int len;
if (!(dev->flags & IFF_UP))
return false;
if (!check_mtu)
return true;
len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
if (skb->len <= len)
return true;
/* if TSO is enabled, we don't care about the length as the packet
* could be forwarded without being segmented before
*/
if (skb_is_gso(skb))
return true;
return false;
}
void netdev_core_stats_inc(struct net_device *dev, u32 offset);
#define DEV_CORE_STATS_INC(FIELD) \
static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev) \
{ \
netdev_core_stats_inc(dev, \
offsetof(struct net_device_core_stats, FIELD)); \
}
DEV_CORE_STATS_INC(rx_dropped)
DEV_CORE_STATS_INC(tx_dropped)
DEV_CORE_STATS_INC(rx_nohandler)
DEV_CORE_STATS_INC(rx_otherhost_dropped)
#undef DEV_CORE_STATS_INC
static __always_inline int ____dev_forward_skb(struct net_device *dev,
struct sk_buff *skb,
const bool check_mtu)
{
if (skb_orphan_frags(skb, GFP_ATOMIC) ||
unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
dev_core_stats_rx_dropped_inc(dev);
kfree_skb(skb);
return NET_RX_DROP;
}
skb_scrub_packet(skb, !net_eq(dev_net(dev), dev_net(skb->dev)));
skb->priority = 0;
return 0;
}
bool dev_nit_active(struct net_device *dev);
void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
static inline void __dev_put(struct net_device *dev)
{
if (dev) {
#ifdef CONFIG_PCPU_DEV_REFCNT
this_cpu_dec(*dev->pcpu_refcnt);
#else
refcount_dec(&dev->dev_refcnt);
#endif
}
}
static inline void __dev_hold(struct net_device *dev)
{
if (dev) {
#ifdef CONFIG_PCPU_DEV_REFCNT
this_cpu_inc(*dev->pcpu_refcnt);
#else
refcount_inc(&dev->dev_refcnt);
#endif
}
}
static inline void __netdev_tracker_alloc(struct net_device *dev,
netdevice_tracker *tracker,
gfp_t gfp)
{
#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp);
#endif
}
/* netdev_tracker_alloc() can upgrade a prior untracked reference
* taken by dev_get_by_name()/dev_get_by_index() to a tracked one.
*/
static inline void netdev_tracker_alloc(struct net_device *dev,
netdevice_tracker *tracker, gfp_t gfp)
{
#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
refcount_dec(&dev->refcnt_tracker.no_tracker);
__netdev_tracker_alloc(dev, tracker, gfp);
#endif
}
static inline void netdev_tracker_free(struct net_device *dev,
netdevice_tracker *tracker)
{
#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
ref_tracker_free(&dev->refcnt_tracker, tracker);
#endif
}
static inline void netdev_hold(struct net_device *dev,
netdevice_tracker *tracker, gfp_t gfp)
{
if (dev) { __dev_hold(dev); __netdev_tracker_alloc(dev, tracker, gfp);
}
}
static inline void netdev_put(struct net_device *dev,
netdevice_tracker *tracker)
{
if (dev) { netdev_tracker_free(dev, tracker); __dev_put(dev);
}
}
/**
* dev_hold - get reference to device
* @dev: network device
*
* Hold reference to device to keep it from being freed.
* Try using netdev_hold() instead.
*/
static inline void dev_hold(struct net_device *dev)
{
netdev_hold(dev, NULL, GFP_ATOMIC);
}
/**
* dev_put - release reference to device
* @dev: network device
*
* Release reference to device to allow it to be freed.
* Try using netdev_put() instead.
*/
static inline void dev_put(struct net_device *dev)
{
netdev_put(dev, NULL);
}
static inline void netdev_ref_replace(struct net_device *odev,
struct net_device *ndev,
netdevice_tracker *tracker,
gfp_t gfp)
{
if (odev)
netdev_tracker_free(odev, tracker); __dev_hold(ndev); __dev_put(odev); if (ndev) __netdev_tracker_alloc(ndev, tracker, gfp);
}
/* Carrier loss detection, dial on demand. The functions netif_carrier_on
* and _off may be called from IRQ context, but it is caller
* who is responsible for serialization of these calls.
*
* The name carrier is inappropriate, these functions should really be
* called netif_lowerlayer_*() because they represent the state of any
* kind of lower layer not just hardware media.
*/
void linkwatch_fire_event(struct net_device *dev);
/**
* linkwatch_sync_dev - sync linkwatch for the given device
* @dev: network device to sync linkwatch for
*
* Sync linkwatch for the given device, removing it from the
* pending work list (if queued).
*/
void linkwatch_sync_dev(struct net_device *dev);
/**
* netif_carrier_ok - test if carrier present
* @dev: network device
*
* Check if carrier is present on device
*/
static inline bool netif_carrier_ok(const struct net_device *dev)
{
return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
}
unsigned long dev_trans_start(struct net_device *dev);
void __netdev_watchdog_up(struct net_device *dev);
void netif_carrier_on(struct net_device *dev);
void netif_carrier_off(struct net_device *dev);
void netif_carrier_event(struct net_device *dev);
/**
* netif_dormant_on - mark device as dormant.
* @dev: network device
*
* Mark device as dormant (as per RFC2863).
*
* The dormant state indicates that the relevant interface is not
* actually in a condition to pass packets (i.e., it is not 'up') but is
* in a "pending" state, waiting for some external event. For "on-
* demand" interfaces, this new state identifies the situation where the
* interface is waiting for events to place it in the up state.
*/
static inline void netif_dormant_on(struct net_device *dev)
{
if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
linkwatch_fire_event(dev);
}
/**
* netif_dormant_off - set device as not dormant.
* @dev: network device
*
* Device is not in dormant state.
*/
static inline void netif_dormant_off(struct net_device *dev)
{
if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
linkwatch_fire_event(dev);
}
/**
* netif_dormant - test if device is dormant
* @dev: network device
*
* Check if device is dormant.
*/
static inline bool netif_dormant(const struct net_device *dev)
{
return test_bit(__LINK_STATE_DORMANT, &dev->state);
}
/**
* netif_testing_on - mark device as under test.
* @dev: network device
*
* Mark device as under test (as per RFC2863).
*
* The testing state indicates that some test(s) must be performed on
* the interface. After completion, of the test, the interface state
* will change to up, dormant, or down, as appropriate.
*/
static inline void netif_testing_on(struct net_device *dev)
{
if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
linkwatch_fire_event(dev);
}
/**
* netif_testing_off - set device as not under test.
* @dev: network device
*
* Device is not in testing state.
*/
static inline void netif_testing_off(struct net_device *dev)
{
if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
linkwatch_fire_event(dev);
}
/**
* netif_testing - test if device is under test
* @dev: network device
*
* Check if device is under test
*/
static inline bool netif_testing(const struct net_device *dev)
{
return test_bit(__LINK_STATE_TESTING, &dev->state);
}
/**
* netif_oper_up - test if device is operational
* @dev: network device
*
* Check if carrier is operational
*/
static inline bool netif_oper_up(const struct net_device *dev)
{
unsigned int operstate = READ_ONCE(dev->operstate); return operstate == IF_OPER_UP ||
operstate == IF_OPER_UNKNOWN /* backward compat */;
}
/**
* netif_device_present - is device available or removed
* @dev: network device
*
* Check if device has not been removed from system.
*/
static inline bool netif_device_present(const struct net_device *dev)
{
return test_bit(__LINK_STATE_PRESENT, &dev->state);
}
void netif_device_detach(struct net_device *dev);
void netif_device_attach(struct net_device *dev);
/*
* Network interface message level settings
*/
enum {
NETIF_MSG_DRV_BIT,
NETIF_MSG_PROBE_BIT,
NETIF_MSG_LINK_BIT,
NETIF_MSG_TIMER_BIT,
NETIF_MSG_IFDOWN_BIT,
NETIF_MSG_IFUP_BIT,
NETIF_MSG_RX_ERR_BIT,
NETIF_MSG_TX_ERR_BIT,
NETIF_MSG_TX_QUEUED_BIT,
NETIF_MSG_INTR_BIT,
NETIF_MSG_TX_DONE_BIT,
NETIF_MSG_RX_STATUS_BIT,
NETIF_MSG_PKTDATA_BIT,
NETIF_MSG_HW_BIT,
NETIF_MSG_WOL_BIT,
/* When you add a new bit above, update netif_msg_class_names array
* in net/ethtool/common.c
*/
NETIF_MSG_CLASS_COUNT,
};
/* Both ethtool_ops interface and internal driver implementation use u32 */
static_assert(NETIF_MSG_CLASS_COUNT <= 32);
#define __NETIF_MSG_BIT(bit) ((u32)1 << (bit))
#define __NETIF_MSG(name) __NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)
#define NETIF_MSG_DRV __NETIF_MSG(DRV)
#define NETIF_MSG_PROBE __NETIF_MSG(PROBE)
#define NETIF_MSG_LINK __NETIF_MSG(LINK)
#define NETIF_MSG_TIMER __NETIF_MSG(TIMER)
#define NETIF_MSG_IFDOWN __NETIF_MSG(IFDOWN)
#define NETIF_MSG_IFUP __NETIF_MSG(IFUP)
#define NETIF_MSG_RX_ERR __NETIF_MSG(RX_ERR)
#define NETIF_MSG_TX_ERR __NETIF_MSG(TX_ERR)
#define NETIF_MSG_TX_QUEUED __NETIF_MSG(TX_QUEUED)
#define NETIF_MSG_INTR __NETIF_MSG(INTR)
#define NETIF_MSG_TX_DONE __NETIF_MSG(TX_DONE)
#define NETIF_MSG_RX_STATUS __NETIF_MSG(RX_STATUS)
#define NETIF_MSG_PKTDATA __NETIF_MSG(PKTDATA)
#define NETIF_MSG_HW __NETIF_MSG(HW)
#define NETIF_MSG_WOL __NETIF_MSG(WOL)
#define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
#define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
#define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
#define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
#define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
#define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
#define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
#define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
#define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
#define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
#define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
#define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
#define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
#define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
#define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
{
/* use default */
if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
return default_msg_enable_bits;
if (debug_value == 0) /* no output */
return 0;
/* set low N bits */
return (1U << debug_value) - 1;
}
static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
{
spin_lock(&txq->_xmit_lock);
/* Pairs with READ_ONCE() in __dev_queue_xmit() */
WRITE_ONCE(txq->xmit_lock_owner, cpu);
}
static inline bool __netif_tx_acquire(struct netdev_queue *txq)
{
__acquire(&txq->_xmit_lock);
return true;
}
static inline void __netif_tx_release(struct netdev_queue *txq)
{
__release(&txq->_xmit_lock);
}
static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
{
spin_lock_bh(&txq->_xmit_lock);
/* Pairs with READ_ONCE() in __dev_queue_xmit() */
WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
}
static inline bool __netif_tx_trylock(struct netdev_queue *txq)
{
bool ok = spin_trylock(&txq->_xmit_lock);
if (likely(ok)) {
/* Pairs with READ_ONCE() in __dev_queue_xmit() */
WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
}
return ok;
}
static inline void __netif_tx_unlock(struct netdev_queue *txq)
{
/* Pairs with READ_ONCE() in __dev_queue_xmit() */
WRITE_ONCE(txq->xmit_lock_owner, -1);
spin_unlock(&txq->_xmit_lock);
}
static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
{
/* Pairs with READ_ONCE() in __dev_queue_xmit() */
WRITE_ONCE(txq->xmit_lock_owner, -1);
spin_unlock_bh(&txq->_xmit_lock);
}
/*
* txq->trans_start can be read locklessly from dev_watchdog()
*/
static inline void txq_trans_update(struct netdev_queue *txq)
{
if (txq->xmit_lock_owner != -1)
WRITE_ONCE(txq->trans_start, jiffies);
}
static inline void txq_trans_cond_update(struct netdev_queue *txq)
{
unsigned long now = jiffies;
if (READ_ONCE(txq->trans_start) != now)
WRITE_ONCE(txq->trans_start, now);
}
/* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
static inline void netif_trans_update(struct net_device *dev)
{
struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
txq_trans_cond_update(txq);
}
/**
* netif_tx_lock - grab network device transmit lock
* @dev: network device
*
* Get network device transmit lock
*/
void netif_tx_lock(struct net_device *dev);
static inline void netif_tx_lock_bh(struct net_device *dev)
{
local_bh_disable();
netif_tx_lock(dev);
}
void netif_tx_unlock(struct net_device *dev);
static inline void netif_tx_unlock_bh(struct net_device *dev)
{
netif_tx_unlock(dev);
local_bh_enable();
}
#define HARD_TX_LOCK(dev, txq, cpu) { \
if ((dev->features & NETIF_F_LLTX) == 0) { \
__netif_tx_lock(txq, cpu); \
} else { \
__netif_tx_acquire(txq); \
} \
}
#define HARD_TX_TRYLOCK(dev, txq) \
(((dev->features & NETIF_F_LLTX) == 0) ? \
__netif_tx_trylock(txq) : \
__netif_tx_acquire(txq))
#define HARD_TX_UNLOCK(dev, txq) { \
if ((dev->features & NETIF_F_LLTX) == 0) { \
__netif_tx_unlock(txq); \
} else { \
__netif_tx_release(txq); \
} \
}
static inline void netif_tx_disable(struct net_device *dev)
{
unsigned int i;
int cpu;
local_bh_disable();
cpu = smp_processor_id();
spin_lock(&dev->tx_global_lock);
for (i = 0; i < dev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
__netif_tx_lock(txq, cpu);
netif_tx_stop_queue(txq);
__netif_tx_unlock(txq);
}
spin_unlock(&dev->tx_global_lock);
local_bh_enable();
}
static inline void netif_addr_lock(struct net_device *dev)
{
unsigned char nest_level = 0;
#ifdef CONFIG_LOCKDEP
nest_level = dev->nested_level;
#endif
spin_lock_nested(&dev->addr_list_lock, nest_level);
}
static inline void netif_addr_lock_bh(struct net_device *dev)
{
unsigned char nest_level = 0;
#ifdef CONFIG_LOCKDEP
nest_level = dev->nested_level;
#endif
local_bh_disable();
spin_lock_nested(&dev->addr_list_lock, nest_level);
}
static inline void netif_addr_unlock(struct net_device *dev)
{
spin_unlock(&dev->addr_list_lock);
}
static inline void netif_addr_unlock_bh(struct net_device *dev)
{
spin_unlock_bh(&dev->addr_list_lock);
}
/*
* dev_addrs walker. Should be used only for read access. Call with
* rcu_read_lock held.
*/
#define for_each_dev_addr(dev, ha) \
list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
/* These functions live elsewhere (drivers/net/net_init.c, but related) */
void ether_setup(struct net_device *dev);
/* Support for loadable net-drivers */
struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
unsigned char name_assign_type,
void (*setup)(struct net_device *),
unsigned int txqs, unsigned int rxqs);
#define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
#define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
count)
int register_netdev(struct net_device *dev);
void unregister_netdev(struct net_device *dev);
int devm_register_netdev(struct device *dev, struct net_device *ndev);
/* General hardware address lists handling functions */
int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr_list *from_list, int addr_len);
void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
struct netdev_hw_addr_list *from_list, int addr_len);
int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*sync)(struct net_device *, const unsigned char *),
int (*unsync)(struct net_device *,
const unsigned char *));
int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*sync)(struct net_device *,
const unsigned char *, int),
int (*unsync)(struct net_device *,
const unsigned char *, int));
void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*unsync)(struct net_device *,
const unsigned char *, int));
void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
struct net_device *dev,
int (*unsync)(struct net_device *,
const unsigned char *));
void __hw_addr_init(struct netdev_hw_addr_list *list);
/* Functions used for device addresses handling */
void dev_addr_mod(struct net_device *dev, unsigned int offset,
const void *addr, size_t len);
static inline void
__dev_addr_set(struct net_device *dev, const void *addr, size_t len)
{
dev_addr_mod(dev, 0, addr, len);
}
static inline void dev_addr_set(struct net_device *dev, const u8 *addr)
{
__dev_addr_set(dev, addr, dev->addr_len);
}
int dev_addr_add(struct net_device *dev, const unsigned char *addr,
unsigned char addr_type);
int dev_addr_del(struct net_device *dev, const unsigned char *addr,
unsigned char addr_type);
/* Functions used for unicast addresses handling */
int dev_uc_add(struct net_device *dev, const unsigned char *addr);
int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
int dev_uc_del(struct net_device *dev, const unsigned char *addr);
int dev_uc_sync(struct net_device *to, struct net_device *from);
int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
void dev_uc_unsync(struct net_device *to, struct net_device *from);
void dev_uc_flush(struct net_device *dev);
void dev_uc_init(struct net_device *dev);
/**
* __dev_uc_sync - Synchonize device's unicast list
* @dev: device to sync
* @sync: function to call if address should be added
* @unsync: function to call if address should be removed
*
* Add newly added addresses to the interface, and release
* addresses that have been deleted.
*/
static inline int __dev_uc_sync(struct net_device *dev,
int (*sync)(struct net_device *,
const unsigned char *),
int (*unsync)(struct net_device *,
const unsigned char *))
{
return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
}
/**
* __dev_uc_unsync - Remove synchronized addresses from device
* @dev: device to sync
* @unsync: function to call if address should be removed
*
* Remove all addresses that were added to the device by dev_uc_sync().
*/
static inline void __dev_uc_unsync(struct net_device *dev,
int (*unsync)(struct net_device *,
const unsigned char *))
{
__hw_addr_unsync_dev(&dev->uc, dev, unsync);
}
/* Functions used for multicast addresses handling */
int dev_mc_add(struct net_device *dev, const unsigned char *addr);
int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
int dev_mc_del(struct net_device *dev, const unsigned char *addr);
int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
int dev_mc_sync(struct net_device *to, struct net_device *from);
int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
void dev_mc_unsync(struct net_device *to, struct net_device *from);
void dev_mc_flush(struct net_device *dev);
void dev_mc_init(struct net_device *dev);
/**
* __dev_mc_sync - Synchonize device's multicast list
* @dev: device to sync
* @sync: function to call if address should be added
* @unsync: function to call if address should be removed
*
* Add newly added addresses to the interface, and release
* addresses that have been deleted.
*/
static inline int __dev_mc_sync(struct net_device *dev,
int (*sync)(struct net_device *,
const unsigned char *),
int (*unsync)(struct net_device *,
const unsigned char *))
{
return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
}
/**
* __dev_mc_unsync - Remove synchronized addresses from device
* @dev: device to sync
* @unsync: function to call if address should be removed
*
* Remove all addresses that were added to the device by dev_mc_sync().
*/
static inline void __dev_mc_unsync(struct net_device *dev,
int (*unsync)(struct net_device *,
const unsigned char *))
{
__hw_addr_unsync_dev(&dev->mc, dev, unsync);
}
/* Functions used for secondary unicast and multicast support */
void dev_set_rx_mode(struct net_device *dev);
int dev_set_promiscuity(struct net_device *dev, int inc);
int dev_set_allmulti(struct net_device *dev, int inc);
void netdev_state_change(struct net_device *dev);
void __netdev_notify_peers(struct net_device *dev);
void netdev_notify_peers(struct net_device *dev);
void netdev_features_change(struct net_device *dev);
/* Load a device via the kmod */
void dev_load(struct net *net, const char *name);
struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
struct rtnl_link_stats64 *storage);
void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
const struct net_device_stats *netdev_stats);
void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
const struct pcpu_sw_netstats __percpu *netstats);
void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
enum {
NESTED_SYNC_IMM_BIT,
NESTED_SYNC_TODO_BIT,
};
#define __NESTED_SYNC_BIT(bit) ((u32)1 << (bit))
#define __NESTED_SYNC(name) __NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
#define NESTED_SYNC_IMM __NESTED_SYNC(IMM)
#define NESTED_SYNC_TODO __NESTED_SYNC(TODO)
struct netdev_nested_priv {
unsigned char flags;
void *data;
};
bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
struct list_head **iter);
/* iterate through upper list, must be called under RCU read lock */
#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
for (iter = &(dev)->adj_list.upper, \
updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
updev; \
updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *upper_dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv);
bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
struct net_device *upper_dev);
bool netdev_has_any_upper_dev(struct net_device *dev);
void *netdev_lower_get_next_private(struct net_device *dev,
struct list_head **iter);
void *netdev_lower_get_next_private_rcu(struct net_device *dev,
struct list_head **iter);
#define netdev_for_each_lower_private(dev, priv, iter) \
for (iter = (dev)->adj_list.lower.next, \
priv = netdev_lower_get_next_private(dev, &(iter)); \
priv; \
priv = netdev_lower_get_next_private(dev, &(iter)))
#define netdev_for_each_lower_private_rcu(dev, priv, iter) \
for (iter = &(dev)->adj_list.lower, \
priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
priv; \
priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
void *netdev_lower_get_next(struct net_device *dev,
struct list_head **iter);
#define netdev_for_each_lower_dev(dev, ldev, iter) \
for (iter = (dev)->adj_list.lower.next, \
ldev = netdev_lower_get_next(dev, &(iter)); \
ldev; \
ldev = netdev_lower_get_next(dev, &(iter)))
struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
struct list_head **iter);
int netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *lower_dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv);
int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *lower_dev,
struct netdev_nested_priv *priv),
struct netdev_nested_priv *priv);
void *netdev_adjacent_get_private(struct list_head *adj_list);
void *netdev_lower_get_first_private_rcu(struct net_device *dev);
struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
struct netlink_ext_ack *extack);
int netdev_master_upper_dev_link(struct net_device *dev,
struct net_device *upper_dev,
void *upper_priv, void *upper_info,
struct netlink_ext_ack *extack);
void netdev_upper_dev_unlink(struct net_device *dev,
struct net_device *upper_dev);
int netdev_adjacent_change_prepare(struct net_device *old_dev,
struct net_device *new_dev,
struct net_device *dev,
struct netlink_ext_ack *extack);
void netdev_adjacent_change_commit(struct net_device *old_dev,
struct net_device *new_dev,
struct net_device *dev);
void netdev_adjacent_change_abort(struct net_device *old_dev,
struct net_device *new_dev,
struct net_device *dev);
void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
void *netdev_lower_dev_get_private(struct net_device *dev,
struct net_device *lower_dev);
void netdev_lower_state_changed(struct net_device *lower_dev,
void *lower_state_info);
/* RSS keys are 40 or 52 bytes long */
#define NETDEV_RSS_KEY_LEN 52
extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
void netdev_rss_key_fill(void *buffer, size_t len);
int skb_checksum_help(struct sk_buff *skb);
int skb_crc32c_csum_help(struct sk_buff *skb);
int skb_csum_hwoffload_help(struct sk_buff *skb,
const netdev_features_t features);
struct netdev_bonding_info {
ifslave slave;
ifbond master;
};
struct netdev_notifier_bonding_info {
struct netdev_notifier_info info; /* must be first */
struct netdev_bonding_info bonding_info;
};
void netdev_bonding_info_change(struct net_device *dev,
struct netdev_bonding_info *bonding_info);
#if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
void ethtool_notify(struct net_device *dev, unsigned int cmd, const void *data);
#else
static inline void ethtool_notify(struct net_device *dev, unsigned int cmd,
const void *data)
{
}
#endif
__be16 skb_network_protocol(struct sk_buff *skb, int *depth);
static inline bool can_checksum_protocol(netdev_features_t features,
__be16 protocol)
{
if (protocol == htons(ETH_P_FCOE))
return !!(features & NETIF_F_FCOE_CRC);
/* Assume this is an IP checksum (not SCTP CRC) */
if (features & NETIF_F_HW_CSUM) {
/* Can checksum everything */
return true;
}
switch (protocol) {
case htons(ETH_P_IP):
return !!(features & NETIF_F_IP_CSUM);
case htons(ETH_P_IPV6):
return !!(features & NETIF_F_IPV6_CSUM);
default:
return false;
}
}
#ifdef CONFIG_BUG
void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
#else
static inline void netdev_rx_csum_fault(struct net_device *dev,
struct sk_buff *skb)
{
}
#endif
/* rx skb timestamps */
void net_enable_timestamp(void);
void net_disable_timestamp(void);
static inline ktime_t netdev_get_tstamp(struct net_device *dev,
const struct skb_shared_hwtstamps *hwtstamps,
bool cycles)
{
const struct net_device_ops *ops = dev->netdev_ops;
if (ops->ndo_get_tstamp)
return ops->ndo_get_tstamp(dev, hwtstamps, cycles);
return hwtstamps->hwtstamp;
}
static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
struct sk_buff *skb, struct net_device *dev,
bool more)
{
__this_cpu_write(softnet_data.xmit.more, more);
return ops->ndo_start_xmit(skb, dev);
}
static inline bool netdev_xmit_more(void)
{
return __this_cpu_read(softnet_data.xmit.more);
}
static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
struct netdev_queue *txq, bool more)
{
const struct net_device_ops *ops = dev->netdev_ops;
netdev_tx_t rc;
rc = __netdev_start_xmit(ops, skb, dev, more);
if (rc == NETDEV_TX_OK)
txq_trans_update(txq);
return rc;
}
int netdev_class_create_file_ns(const struct class_attribute *class_attr,
const void *ns);
void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
const void *ns);
extern const struct kobj_ns_type_operations net_ns_type_operations;
const char *netdev_drivername(const struct net_device *dev);
static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
netdev_features_t f2)
{
if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
if (f1 & NETIF_F_HW_CSUM)
f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
else
f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
}
return f1 & f2;
}
static inline netdev_features_t netdev_get_wanted_features(
struct net_device *dev)
{
return (dev->features & ~dev->hw_features) | dev->wanted_features;
}
netdev_features_t netdev_increment_features(netdev_features_t all,
netdev_features_t one, netdev_features_t mask);
/* Allow TSO being used on stacked device :
* Performing the GSO segmentation before last device
* is a performance improvement.
*/
static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
netdev_features_t mask)
{
return netdev_increment_features(features, NETIF_F_ALL_TSO, mask);
}
int __netdev_update_features(struct net_device *dev);
void netdev_update_features(struct net_device *dev);
void netdev_change_features(struct net_device *dev);
void netif_stacked_transfer_operstate(const struct net_device *rootdev,
struct net_device *dev);
netdev_features_t passthru_features_check(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features);
netdev_features_t netif_skb_features(struct sk_buff *skb);
void skb_warn_bad_offload(const struct sk_buff *skb);
static inline bool net_gso_ok(netdev_features_t features, int gso_type)
{
netdev_features_t feature = (netdev_features_t)gso_type << NETIF_F_GSO_SHIFT;
/* check flags correspondence */
BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
return (features & feature) == feature;
}
static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
{
return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
(!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
}
static inline bool netif_needs_gso(struct sk_buff *skb,
netdev_features_t features)
{
return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
(skb->ip_summed != CHECKSUM_UNNECESSARY)));
}
void netif_set_tso_max_size(struct net_device *dev, unsigned int size);
void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs);
void netif_inherit_tso_max(struct net_device *to,
const struct net_device *from);
static inline bool netif_is_macsec(const struct net_device *dev)
{
return dev->priv_flags & IFF_MACSEC;
}
static inline bool netif_is_macvlan(const struct net_device *dev)
{
return dev->priv_flags & IFF_MACVLAN;
}
static inline bool netif_is_macvlan_port(const struct net_device *dev)
{
return dev->priv_flags & IFF_MACVLAN_PORT;
}
static inline bool netif_is_bond_master(const struct net_device *dev)
{
return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
}
static inline bool netif_is_bond_slave(const struct net_device *dev)
{
return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
}
static inline bool netif_supports_nofcs(struct net_device *dev)
{
return dev->priv_flags & IFF_SUPP_NOFCS;
}
static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
{
return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
}
static inline bool netif_is_l3_master(const struct net_device *dev)
{
return dev->priv_flags & IFF_L3MDEV_MASTER;
}
static inline bool netif_is_l3_slave(const struct net_device *dev)
{
return dev->priv_flags & IFF_L3MDEV_SLAVE;
}
static inline int dev_sdif(const struct net_device *dev)
{
#ifdef CONFIG_NET_L3_MASTER_DEV
if (netif_is_l3_slave(dev))
return dev->ifindex;
#endif
return 0;
}
static inline bool netif_is_bridge_master(const struct net_device *dev)
{
return dev->priv_flags & IFF_EBRIDGE;
}
static inline bool netif_is_bridge_port(const struct net_device *dev)
{
return dev->priv_flags & IFF_BRIDGE_PORT;
}
static inline bool netif_is_ovs_master(const struct net_device *dev)
{
return dev->priv_flags & IFF_OPENVSWITCH;
}
static inline bool netif_is_ovs_port(const struct net_device *dev)
{
return dev->priv_flags & IFF_OVS_DATAPATH;
}
static inline bool netif_is_any_bridge_master(const struct net_device *dev)
{
return netif_is_bridge_master(dev) || netif_is_ovs_master(dev);
}
static inline bool netif_is_any_bridge_port(const struct net_device *dev)
{
return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
}
static inline bool netif_is_team_master(const struct net_device *dev)
{
return dev->priv_flags & IFF_TEAM;
}
static inline bool netif_is_team_port(const struct net_device *dev)
{
return dev->priv_flags & IFF_TEAM_PORT;
}
static inline bool netif_is_lag_master(const struct net_device *dev)
{
return netif_is_bond_master(dev) || netif_is_team_master(dev);
}
static inline bool netif_is_lag_port(const struct net_device *dev)
{
return netif_is_bond_slave(dev) || netif_is_team_port(dev);
}
static inline bool netif_is_rxfh_configured(const struct net_device *dev)
{
return dev->priv_flags & IFF_RXFH_CONFIGURED;
}
static inline bool netif_is_failover(const struct net_device *dev)
{
return dev->priv_flags & IFF_FAILOVER;
}
static inline bool netif_is_failover_slave(const struct net_device *dev)
{
return dev->priv_flags & IFF_FAILOVER_SLAVE;
}
/* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
static inline void netif_keep_dst(struct net_device *dev)
{
dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
}
/* return true if dev can't cope with mtu frames that need vlan tag insertion */
static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
{
/* TODO: reserve and use an additional IFF bit, if we get more users */
return netif_is_macsec(dev);
}
extern struct pernet_operations __net_initdata loopback_net_ops;
/* Logging, debugging and troubleshooting/diagnostic helpers. */
/* netdev_printk helpers, similar to dev_printk */
static inline const char *netdev_name(const struct net_device *dev)
{
if (!dev->name[0] || strchr(dev->name, '%'))
return "(unnamed net_device)";
return dev->name;
}
static inline const char *netdev_reg_state(const struct net_device *dev)
{
u8 reg_state = READ_ONCE(dev->reg_state); switch (reg_state) {
case NETREG_UNINITIALIZED: return " (uninitialized)";
case NETREG_REGISTERED: return "";
case NETREG_UNREGISTERING: return " (unregistering)";
case NETREG_UNREGISTERED: return " (unregistered)";
case NETREG_RELEASED: return " (released)";
case NETREG_DUMMY: return " (dummy)";
}
WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, reg_state);
return " (unknown)";
}
#define MODULE_ALIAS_NETDEV(device) \
MODULE_ALIAS("netdev-" device)
/*
* netdev_WARN() acts like dev_printk(), but with the key difference
* of using a WARN/WARN_ON to get the message out, including the
* file/line information and a backtrace.
*/
#define netdev_WARN(dev, format, args...) \
WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
netdev_reg_state(dev), ##args)
#define netdev_WARN_ONCE(dev, format, args...) \
WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
netdev_reg_state(dev), ##args)
/*
* The list of packet types we will receive (as opposed to discard)
* and the routines to invoke.
*
* Why 16. Because with 16 the only overlap we get on a hash of the
* low nibble of the protocol value is RARP/SNAP/X.25.
*
* 0800 IP
* 0001 802.3
* 0002 AX.25
* 0004 802.2
* 8035 RARP
* 0005 SNAP
* 0805 X.25
* 0806 ARP
* 8137 IPX
* 0009 Localtalk
* 86DD IPv6
*/
#define PTYPE_HASH_SIZE (16)
#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
extern struct net_device *blackhole_netdev;
/* Note: Avoid these macros in fast path, prefer per-cpu or per-queue counters. */
#define DEV_STATS_INC(DEV, FIELD) atomic_long_inc(&(DEV)->stats.__##FIELD)
#define DEV_STATS_ADD(DEV, FIELD, VAL) \
atomic_long_add((VAL), &(DEV)->stats.__##FIELD)
#define DEV_STATS_READ(DEV, FIELD) atomic_long_read(&(DEV)->stats.__##FIELD)
#endif /* _LINUX_NETDEVICE_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Tty buffer allocation management
*/
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/minmax.h>
#include <linux/tty.h>
#include <linux/tty_buffer.h>
#include <linux/tty_driver.h>
#include <linux/tty_flip.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/bitops.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/ratelimit.h>
#include "tty.h"
#define MIN_TTYB_SIZE 256
#define TTYB_ALIGN_MASK 0xff
/*
* Byte threshold to limit memory consumption for flip buffers.
* The actual memory limit is > 2x this amount.
*/
#define TTYB_DEFAULT_MEM_LIMIT (640 * 1024UL)
/*
* We default to dicing tty buffer allocations to this many characters
* in order to avoid multiple page allocations. We know the size of
* tty_buffer itself but it must also be taken into account that the
* buffer is 256 byte aligned. See tty_buffer_find for the allocation
* logic this must match.
*/
#define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~TTYB_ALIGN_MASK)
/**
* tty_buffer_lock_exclusive - gain exclusive access to buffer
* @port: tty port owning the flip buffer
*
* Guarantees safe use of the &tty_ldisc_ops.receive_buf() method by excluding
* the buffer work and any pending flush from using the flip buffer. Data can
* continue to be added concurrently to the flip buffer from the driver side.
*
* See also tty_buffer_unlock_exclusive().
*/
void tty_buffer_lock_exclusive(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
atomic_inc(&buf->priority);
mutex_lock(&buf->lock);
}
EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive);
/**
* tty_buffer_unlock_exclusive - release exclusive access
* @port: tty port owning the flip buffer
*
* The buffer work is restarted if there is data in the flip buffer.
*
* See also tty_buffer_lock_exclusive().
*/
void tty_buffer_unlock_exclusive(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
bool restart = buf->head->commit != buf->head->read;
atomic_dec(&buf->priority);
mutex_unlock(&buf->lock);
if (restart)
queue_work(system_unbound_wq, &buf->work);
}
EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive);
/**
* tty_buffer_space_avail - return unused buffer space
* @port: tty port owning the flip buffer
*
* Returns: the # of bytes which can be written by the driver without reaching
* the buffer limit.
*
* Note: this does not guarantee that memory is available to write the returned
* # of bytes (use tty_prepare_flip_string() to pre-allocate if memory
* guarantee is required).
*/
unsigned int tty_buffer_space_avail(struct tty_port *port)
{
int space = port->buf.mem_limit - atomic_read(&port->buf.mem_used);
return max(space, 0);
}
EXPORT_SYMBOL_GPL(tty_buffer_space_avail);
static void tty_buffer_reset(struct tty_buffer *p, size_t size)
{
p->used = 0;
p->size = size;
p->next = NULL;
p->commit = 0;
p->lookahead = 0;
p->read = 0;
p->flags = true;
}
/**
* tty_buffer_free_all - free buffers used by a tty
* @port: tty port to free from
*
* Remove all the buffers pending on a tty whether queued with data or in the
* free ring. Must be called when the tty is no longer in use.
*/
void tty_buffer_free_all(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
struct tty_buffer *p, *next;
struct llist_node *llist;
unsigned int freed = 0;
int still_used;
while ((p = buf->head) != NULL) { buf->head = p->next;
freed += p->size;
if (p->size > 0)
kfree(p);
}
llist = llist_del_all(&buf->free); llist_for_each_entry_safe(p, next, llist, free)
kfree(p);
tty_buffer_reset(&buf->sentinel, 0);
buf->head = &buf->sentinel;
buf->tail = &buf->sentinel;
still_used = atomic_xchg(&buf->mem_used, 0);
WARN(still_used != freed, "we still have not freed %d bytes!",
still_used - freed);
}
/**
* tty_buffer_alloc - allocate a tty buffer
* @port: tty port
* @size: desired size (characters)
*
* Allocate a new tty buffer to hold the desired number of characters. We
* round our buffers off in 256 character chunks to get better allocation
* behaviour.
*
* Returns: %NULL if out of memory or the allocation would exceed the per
* device queue.
*/
static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
{
struct llist_node *free;
struct tty_buffer *p;
/* Round the buffer size out */
size = __ALIGN_MASK(size, TTYB_ALIGN_MASK);
if (size <= MIN_TTYB_SIZE) {
free = llist_del_first(&port->buf.free);
if (free) {
p = llist_entry(free, struct tty_buffer, free);
goto found;
}
}
/* Should possibly check if this fails for the largest buffer we
* have queued and recycle that ?
*/
if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit)
return NULL;
p = kmalloc(struct_size(p, data, 2 * size), GFP_ATOMIC | __GFP_NOWARN);
if (p == NULL)
return NULL;
found:
tty_buffer_reset(p, size);
atomic_add(size, &port->buf.mem_used);
return p;
}
/**
* tty_buffer_free - free a tty buffer
* @port: tty port owning the buffer
* @b: the buffer to free
*
* Free a tty buffer, or add it to the free list according to our internal
* strategy.
*/
static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
{
struct tty_bufhead *buf = &port->buf;
/* Dumb strategy for now - should keep some stats */
WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0);
if (b->size > MIN_TTYB_SIZE)
kfree(b);
else if (b->size > 0)
llist_add(&b->free, &buf->free);
}
/**
* tty_buffer_flush - flush full tty buffers
* @tty: tty to flush
* @ld: optional ldisc ptr (must be referenced)
*
* Flush all the buffers containing receive data. If @ld != %NULL, flush the
* ldisc input buffer.
*
* Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'.
*/
void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld)
{
struct tty_port *port = tty->port;
struct tty_bufhead *buf = &port->buf;
struct tty_buffer *next;
atomic_inc(&buf->priority);
mutex_lock(&buf->lock);
/* paired w/ release in __tty_buffer_request_room; ensures there are
* no pending memory accesses to the freed buffer
*/
while ((next = smp_load_acquire(&buf->head->next)) != NULL) {
tty_buffer_free(port, buf->head);
buf->head = next;
}
buf->head->read = buf->head->commit;
buf->head->lookahead = buf->head->read;
if (ld && ld->ops->flush_buffer)
ld->ops->flush_buffer(tty);
atomic_dec(&buf->priority);
mutex_unlock(&buf->lock);
}
/**
* __tty_buffer_request_room - grow tty buffer if needed
* @port: tty port
* @size: size desired
* @flags: buffer has to store flags along character data
*
* Make at least @size bytes of linear space available for the tty buffer.
*
* Will change over to a new buffer if the current buffer is encoded as
* %TTY_NORMAL (so has no flags buffer) and the new buffer requires a flags
* buffer.
*
* Returns: the size we managed to find.
*/
static int __tty_buffer_request_room(struct tty_port *port, size_t size,
bool flags)
{
struct tty_bufhead *buf = &port->buf;
struct tty_buffer *n, *b = buf->tail; size_t left = (b->flags ? 1 : 2) * b->size - b->used; bool change = !b->flags && flags; if (!change && left >= size) return size;
/* This is the slow path - looking for new buffers to use */
n = tty_buffer_alloc(port, size);
if (n == NULL)
return change ? 0 : left; n->flags = flags;
buf->tail = n;
/*
* Paired w/ acquire in flush_to_ldisc() and lookahead_bufs()
* ensures they see all buffer data.
*/
smp_store_release(&b->commit, b->used);
/*
* Paired w/ acquire in flush_to_ldisc() and lookahead_bufs()
* ensures the latest commit value can be read before the head
* is advanced to the next buffer.
*/
smp_store_release(&b->next, n);
return size;
}
int tty_buffer_request_room(struct tty_port *port, size_t size)
{
return __tty_buffer_request_room(port, size, true);
}
EXPORT_SYMBOL_GPL(tty_buffer_request_room);
size_t __tty_insert_flip_string_flags(struct tty_port *port, const u8 *chars,
const u8 *flags, bool mutable_flags,
size_t size)
{
bool need_flags = mutable_flags || flags[0] != TTY_NORMAL;
size_t copied = 0;
do {
size_t goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
size_t space = __tty_buffer_request_room(port, goal, need_flags);
struct tty_buffer *tb = port->buf.tail;
if (unlikely(space == 0))
break;
memcpy(char_buf_ptr(tb, tb->used), chars, space);
if (mutable_flags) {
memcpy(flag_buf_ptr(tb, tb->used), flags, space);
flags += space;
} else if (tb->flags) {
memset(flag_buf_ptr(tb, tb->used), flags[0], space);
} else {
/* tb->flags should be available once requested */
WARN_ON_ONCE(need_flags);
}
tb->used += space;
copied += space;
chars += space;
/* There is a small chance that we need to split the data over
* several buffers. If this is the case we must loop.
*/
} while (unlikely(size > copied));
return copied;
}
EXPORT_SYMBOL(__tty_insert_flip_string_flags);
/**
* tty_prepare_flip_string - make room for characters
* @port: tty port
* @chars: return pointer for character write area
* @size: desired size
*
* Prepare a block of space in the buffer for data.
*
* This is used for drivers that need their own block copy routines into the
* buffer. There is no guarantee the buffer is a DMA target!
*
* Returns: the length available and buffer pointer (@chars) to the space which
* is now allocated and accounted for as ready for normal characters.
*/
size_t tty_prepare_flip_string(struct tty_port *port, u8 **chars, size_t size)
{
size_t space = __tty_buffer_request_room(port, size, false);
if (likely(space)) {
struct tty_buffer *tb = port->buf.tail;
*chars = char_buf_ptr(tb, tb->used);
if (tb->flags)
memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space);
tb->used += space;
}
return space;
}
EXPORT_SYMBOL_GPL(tty_prepare_flip_string);
/**
* tty_ldisc_receive_buf - forward data to line discipline
* @ld: line discipline to process input
* @p: char buffer
* @f: %TTY_NORMAL, %TTY_BREAK, etc. flags buffer
* @count: number of bytes to process
*
* Callers other than flush_to_ldisc() need to exclude the kworker from
* concurrent use of the line discipline, see paste_selection().
*
* Returns: the number of bytes processed.
*/
size_t tty_ldisc_receive_buf(struct tty_ldisc *ld, const u8 *p, const u8 *f,
size_t count)
{
if (ld->ops->receive_buf2)
count = ld->ops->receive_buf2(ld->tty, p, f, count);
else {
count = min_t(size_t, count, ld->tty->receive_room);
if (count && ld->ops->receive_buf)
ld->ops->receive_buf(ld->tty, p, f, count);
}
return count;
}
EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf);
static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head)
{
head->lookahead = max(head->lookahead, head->read);
while (head) {
struct tty_buffer *next;
unsigned int count;
/*
* Paired w/ release in __tty_buffer_request_room();
* ensures commit value read is not stale if the head
* is advancing to the next buffer.
*/
next = smp_load_acquire(&head->next);
/*
* Paired w/ release in __tty_buffer_request_room() or in
* tty_buffer_flush(); ensures we see the committed buffer data.
*/
count = smp_load_acquire(&head->commit) - head->lookahead;
if (!count) {
head = next;
continue;
}
if (port->client_ops->lookahead_buf) {
u8 *p, *f = NULL;
p = char_buf_ptr(head, head->lookahead);
if (head->flags)
f = flag_buf_ptr(head, head->lookahead);
port->client_ops->lookahead_buf(port, p, f, count);
}
head->lookahead += count;
}
}
static size_t
receive_buf(struct tty_port *port, struct tty_buffer *head, size_t count)
{
u8 *p = char_buf_ptr(head, head->read);
const u8 *f = NULL;
size_t n;
if (head->flags)
f = flag_buf_ptr(head, head->read);
n = port->client_ops->receive_buf(port, p, f, count);
if (n > 0)
memset(p, 0, n);
return n;
}
/**
* flush_to_ldisc - flush data from buffer to ldisc
* @work: tty structure passed from work queue.
*
* This routine is called out of the software interrupt to flush data from the
* buffer chain to the line discipline.
*
* The receive_buf() method is single threaded for each tty instance.
*
* Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'.
*/
static void flush_to_ldisc(struct work_struct *work)
{
struct tty_port *port = container_of(work, struct tty_port, buf.work);
struct tty_bufhead *buf = &port->buf;
mutex_lock(&buf->lock);
while (1) {
struct tty_buffer *head = buf->head;
struct tty_buffer *next;
size_t count, rcvd;
/* Ldisc or user is trying to gain exclusive access */
if (atomic_read(&buf->priority))
break;
/* paired w/ release in __tty_buffer_request_room();
* ensures commit value read is not stale if the head
* is advancing to the next buffer
*/
next = smp_load_acquire(&head->next);
/* paired w/ release in __tty_buffer_request_room() or in
* tty_buffer_flush(); ensures we see the committed buffer data
*/
count = smp_load_acquire(&head->commit) - head->read;
if (!count) {
if (next == NULL)
break;
buf->head = next;
tty_buffer_free(port, head);
continue;
}
rcvd = receive_buf(port, head, count);
head->read += rcvd;
if (rcvd < count)
lookahead_bufs(port, head);
if (!rcvd)
break;
if (need_resched())
cond_resched();
}
mutex_unlock(&buf->lock);
}
static inline void tty_flip_buffer_commit(struct tty_buffer *tail)
{
/*
* Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees
* buffer data.
*/
smp_store_release(&tail->commit, tail->used);
}
/**
* tty_flip_buffer_push - push terminal buffers
* @port: tty port to push
*
* Queue a push of the terminal flip buffers to the line discipline. Can be
* called from IRQ/atomic context.
*
* In the event of the queue being busy for flipping the work will be held off
* and retried later.
*/
void tty_flip_buffer_push(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
tty_flip_buffer_commit(buf->tail);
queue_work(system_unbound_wq, &buf->work);
}
EXPORT_SYMBOL(tty_flip_buffer_push);
/**
* tty_insert_flip_string_and_push_buffer - add characters to the tty buffer and
* push
* @port: tty port
* @chars: characters
* @size: size
*
* The function combines tty_insert_flip_string() and tty_flip_buffer_push()
* with the exception of properly holding the @port->lock.
*
* To be used only internally (by pty currently).
*
* Returns: the number added.
*/
int tty_insert_flip_string_and_push_buffer(struct tty_port *port,
const u8 *chars, size_t size)
{
struct tty_bufhead *buf = &port->buf;
unsigned long flags;
spin_lock_irqsave(&port->lock, flags);
size = tty_insert_flip_string(port, chars, size);
if (size)
tty_flip_buffer_commit(buf->tail);
spin_unlock_irqrestore(&port->lock, flags);
queue_work(system_unbound_wq, &buf->work);
return size;
}
/**
* tty_buffer_init - prepare a tty buffer structure
* @port: tty port to initialise
*
* Set up the initial state of the buffer management for a tty device. Must be
* called before the other tty buffer functions are used.
*/
void tty_buffer_init(struct tty_port *port)
{
struct tty_bufhead *buf = &port->buf;
mutex_init(&buf->lock);
tty_buffer_reset(&buf->sentinel, 0);
buf->head = &buf->sentinel;
buf->tail = &buf->sentinel;
init_llist_head(&buf->free);
atomic_set(&buf->mem_used, 0);
atomic_set(&buf->priority, 0);
INIT_WORK(&buf->work, flush_to_ldisc);
buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT;
}
/**
* tty_buffer_set_limit - change the tty buffer memory limit
* @port: tty port to change
* @limit: memory limit to set
*
* Change the tty buffer memory limit.
*
* Must be called before the other tty buffer functions are used.
*/
int tty_buffer_set_limit(struct tty_port *port, int limit)
{
if (limit < MIN_TTYB_SIZE)
return -EINVAL;
port->buf.mem_limit = limit;
return 0;
}
EXPORT_SYMBOL_GPL(tty_buffer_set_limit);
/* slave ptys can claim nested buffer lock when handling BRK and INTR */
void tty_buffer_set_lock_subclass(struct tty_port *port)
{
lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE);
}
bool tty_buffer_restart_work(struct tty_port *port)
{
return queue_work(system_unbound_wq, &port->buf.work);
}
bool tty_buffer_cancel_work(struct tty_port *port)
{
return cancel_work_sync(&port->buf.work);
}
void tty_buffer_flush_work(struct tty_port *port)
{
flush_work(&port->buf.work);
}
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/dd.c - The core device/driver interactions.
*
* This file contains the (sometimes tricky) code that controls the
* interactions between devices and drivers, which primarily includes
* driver binding and unbinding.
*
* All of this code used to exist in drivers/base/bus.c, but was
* relocated to here in the name of compartmentalization (since it wasn't
* strictly code just for the 'struct bus_type'.
*
* Copyright (c) 2002-5 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
* Copyright (c) 2007-2009 Greg Kroah-Hartman <gregkh@suse.de>
* Copyright (c) 2007-2009 Novell Inc.
*/
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/dma-map-ops.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/async.h>
#include <linux/pm_runtime.h>
#include <linux/pinctrl/devinfo.h>
#include <linux/slab.h>
#include "base.h"
#include "power/power.h"
/*
* Deferred Probe infrastructure.
*
* Sometimes driver probe order matters, but the kernel doesn't always have
* dependency information which means some drivers will get probed before a
* resource it depends on is available. For example, an SDHCI driver may
* first need a GPIO line from an i2c GPIO controller before it can be
* initialized. If a required resource is not available yet, a driver can
* request probing to be deferred by returning -EPROBE_DEFER from its probe hook
*
* Deferred probe maintains two lists of devices, a pending list and an active
* list. A driver returning -EPROBE_DEFER causes the device to be added to the
* pending list. A successful driver probe will trigger moving all devices
* from the pending to the active list so that the workqueue will eventually
* retry them.
*
* The deferred_probe_mutex must be held any time the deferred_probe_*_list
* of the (struct device*)->p->deferred_probe pointers are manipulated
*/
static DEFINE_MUTEX(deferred_probe_mutex);
static LIST_HEAD(deferred_probe_pending_list);
static LIST_HEAD(deferred_probe_active_list);
static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
static bool initcalls_done;
/* Save the async probe drivers' name from kernel cmdline */
#define ASYNC_DRV_NAMES_MAX_LEN 256
static char async_probe_drv_names[ASYNC_DRV_NAMES_MAX_LEN];
static bool async_probe_default;
/*
* In some cases, like suspend to RAM or hibernation, It might be reasonable
* to prohibit probing of devices as it could be unsafe.
* Once defer_all_probes is true all drivers probes will be forcibly deferred.
*/
static bool defer_all_probes;
static void __device_set_deferred_probe_reason(const struct device *dev, char *reason)
{
kfree(dev->p->deferred_probe_reason);
dev->p->deferred_probe_reason = reason;
}
/*
* deferred_probe_work_func() - Retry probing devices in the active list.
*/
static void deferred_probe_work_func(struct work_struct *work)
{
struct device *dev;
struct device_private *private;
/*
* This block processes every device in the deferred 'active' list.
* Each device is removed from the active list and passed to
* bus_probe_device() to re-attempt the probe. The loop continues
* until every device in the active list is removed and retried.
*
* Note: Once the device is removed from the list and the mutex is
* released, it is possible for the device get freed by another thread
* and cause a illegal pointer dereference. This code uses
* get/put_device() to ensure the device structure cannot disappear
* from under our feet.
*/
mutex_lock(&deferred_probe_mutex);
while (!list_empty(&deferred_probe_active_list)) {
private = list_first_entry(&deferred_probe_active_list,
typeof(*dev->p), deferred_probe);
dev = private->device;
list_del_init(&private->deferred_probe);
get_device(dev);
__device_set_deferred_probe_reason(dev, NULL);
/*
* Drop the mutex while probing each device; the probe path may
* manipulate the deferred list
*/
mutex_unlock(&deferred_probe_mutex);
/*
* Force the device to the end of the dpm_list since
* the PM code assumes that the order we add things to
* the list is a good order for suspend but deferred
* probe makes that very unsafe.
*/
device_pm_move_to_tail(dev);
dev_dbg(dev, "Retrying from deferred list\n");
bus_probe_device(dev);
mutex_lock(&deferred_probe_mutex);
put_device(dev);
}
mutex_unlock(&deferred_probe_mutex);
}
static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func);
void driver_deferred_probe_add(struct device *dev)
{
if (!dev->can_match)
return;
mutex_lock(&deferred_probe_mutex);
if (list_empty(&dev->p->deferred_probe)) {
dev_dbg(dev, "Added to deferred list\n");
list_add_tail(&dev->p->deferred_probe, &deferred_probe_pending_list);
}
mutex_unlock(&deferred_probe_mutex);
}
void driver_deferred_probe_del(struct device *dev)
{
mutex_lock(&deferred_probe_mutex);
if (!list_empty(&dev->p->deferred_probe)) {
dev_dbg(dev, "Removed from deferred list\n"); list_del_init(&dev->p->deferred_probe);
__device_set_deferred_probe_reason(dev, NULL);
}
mutex_unlock(&deferred_probe_mutex);
}
static bool driver_deferred_probe_enable;
/**
* driver_deferred_probe_trigger() - Kick off re-probing deferred devices
*
* This functions moves all devices from the pending list to the active
* list and schedules the deferred probe workqueue to process them. It
* should be called anytime a driver is successfully bound to a device.
*
* Note, there is a race condition in multi-threaded probe. In the case where
* more than one device is probing at the same time, it is possible for one
* probe to complete successfully while another is about to defer. If the second
* depends on the first, then it will get put on the pending list after the
* trigger event has already occurred and will be stuck there.
*
* The atomic 'deferred_trigger_count' is used to determine if a successful
* trigger has occurred in the midst of probing a driver. If the trigger count
* changes in the midst of a probe, then deferred processing should be triggered
* again.
*/
void driver_deferred_probe_trigger(void)
{
if (!driver_deferred_probe_enable)
return;
/*
* A successful probe means that all the devices in the pending list
* should be triggered to be reprobed. Move all the deferred devices
* into the active list so they can be retried by the workqueue
*/
mutex_lock(&deferred_probe_mutex);
atomic_inc(&deferred_trigger_count);
list_splice_tail_init(&deferred_probe_pending_list,
&deferred_probe_active_list);
mutex_unlock(&deferred_probe_mutex);
/*
* Kick the re-probe thread. It may already be scheduled, but it is
* safe to kick it again.
*/
queue_work(system_unbound_wq, &deferred_probe_work);
}
/**
* device_block_probing() - Block/defer device's probes
*
* It will disable probing of devices and defer their probes instead.
*/
void device_block_probing(void)
{
defer_all_probes = true;
/* sync with probes to avoid races. */
wait_for_device_probe();
}
/**
* device_unblock_probing() - Unblock/enable device's probes
*
* It will restore normal behavior and trigger re-probing of deferred
* devices.
*/
void device_unblock_probing(void)
{
defer_all_probes = false;
driver_deferred_probe_trigger();
}
/**
* device_set_deferred_probe_reason() - Set defer probe reason message for device
* @dev: the pointer to the struct device
* @vaf: the pointer to va_format structure with message
*/
void device_set_deferred_probe_reason(const struct device *dev, struct va_format *vaf)
{
const char *drv = dev_driver_string(dev);
char *reason;
mutex_lock(&deferred_probe_mutex);
reason = kasprintf(GFP_KERNEL, "%s: %pV", drv, vaf);
__device_set_deferred_probe_reason(dev, reason);
mutex_unlock(&deferred_probe_mutex);
}
/*
* deferred_devs_show() - Show the devices in the deferred probe pending list.
*/
static int deferred_devs_show(struct seq_file *s, void *data)
{
struct device_private *curr;
mutex_lock(&deferred_probe_mutex);
list_for_each_entry(curr, &deferred_probe_pending_list, deferred_probe)
seq_printf(s, "%s\t%s", dev_name(curr->device),
curr->device->p->deferred_probe_reason ?: "\n");
mutex_unlock(&deferred_probe_mutex);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(deferred_devs);
#ifdef CONFIG_MODULES
static int driver_deferred_probe_timeout = 10;
#else
static int driver_deferred_probe_timeout;
#endif
static int __init deferred_probe_timeout_setup(char *str)
{
int timeout;
if (!kstrtoint(str, 10, &timeout))
driver_deferred_probe_timeout = timeout;
return 1;
}
__setup("deferred_probe_timeout=", deferred_probe_timeout_setup);
/**
* driver_deferred_probe_check_state() - Check deferred probe state
* @dev: device to check
*
* Return:
* * -ENODEV if initcalls have completed and modules are disabled.
* * -ETIMEDOUT if the deferred probe timeout was set and has expired
* and modules are enabled.
* * -EPROBE_DEFER in other cases.
*
* Drivers or subsystems can opt-in to calling this function instead of directly
* returning -EPROBE_DEFER.
*/
int driver_deferred_probe_check_state(struct device *dev)
{
if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) {
dev_warn(dev, "ignoring dependency for device, assuming no driver\n");
return -ENODEV;
}
if (!driver_deferred_probe_timeout && initcalls_done) {
dev_warn(dev, "deferred probe timeout, ignoring dependency\n");
return -ETIMEDOUT;
}
return -EPROBE_DEFER;
}
EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state);
static void deferred_probe_timeout_work_func(struct work_struct *work)
{
struct device_private *p;
fw_devlink_drivers_done();
driver_deferred_probe_timeout = 0;
driver_deferred_probe_trigger();
flush_work(&deferred_probe_work);
mutex_lock(&deferred_probe_mutex);
list_for_each_entry(p, &deferred_probe_pending_list, deferred_probe)
dev_warn(p->device, "deferred probe pending: %s", p->deferred_probe_reason ?: "(reason unknown)\n");
mutex_unlock(&deferred_probe_mutex);
fw_devlink_probing_done();
}
static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func);
void deferred_probe_extend_timeout(void)
{
/*
* If the work hasn't been queued yet or if the work expired, don't
* start a new one.
*/
if (cancel_delayed_work(&deferred_probe_timeout_work)) {
schedule_delayed_work(&deferred_probe_timeout_work,
driver_deferred_probe_timeout * HZ); pr_debug("Extended deferred probe timeout by %d secs\n",
driver_deferred_probe_timeout);
}
}
/**
* deferred_probe_initcall() - Enable probing of deferred devices
*
* We don't want to get in the way when the bulk of drivers are getting probed.
* Instead, this initcall makes sure that deferred probing is delayed until
* late_initcall time.
*/
static int deferred_probe_initcall(void)
{
debugfs_create_file("devices_deferred", 0444, NULL, NULL,
&deferred_devs_fops);
driver_deferred_probe_enable = true;
driver_deferred_probe_trigger();
/* Sort as many dependencies as possible before exiting initcalls */
flush_work(&deferred_probe_work);
initcalls_done = true;
if (!IS_ENABLED(CONFIG_MODULES))
fw_devlink_drivers_done();
/*
* Trigger deferred probe again, this time we won't defer anything
* that is optional
*/
driver_deferred_probe_trigger();
flush_work(&deferred_probe_work);
if (driver_deferred_probe_timeout > 0) {
schedule_delayed_work(&deferred_probe_timeout_work,
driver_deferred_probe_timeout * HZ);
}
if (!IS_ENABLED(CONFIG_MODULES))
fw_devlink_probing_done();
return 0;
}
late_initcall(deferred_probe_initcall);
static void __exit deferred_probe_exit(void)
{
debugfs_lookup_and_remove("devices_deferred", NULL);
}
__exitcall(deferred_probe_exit);
/**
* device_is_bound() - Check if device is bound to a driver
* @dev: device to check
*
* Returns true if passed device has already finished probing successfully
* against a driver.
*
* This function must be called with the device lock held.
*/
bool device_is_bound(struct device *dev)
{
return dev->p && klist_node_attached(&dev->p->knode_driver);
}
static void driver_bound(struct device *dev)
{
if (device_is_bound(dev)) { dev_warn(dev, "%s: device already bound\n", __func__);
return;
}
dev_dbg(dev, "driver: '%s': %s: bound to device\n", dev->driver->name,
__func__);
klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices);
device_links_driver_bound(dev);
device_pm_check_callbacks(dev);
/*
* Make sure the device is no longer in one of the deferred lists and
* kick off retrying all pending devices
*/
driver_deferred_probe_del(dev);
driver_deferred_probe_trigger(); bus_notify(dev, BUS_NOTIFY_BOUND_DRIVER);
kobject_uevent(&dev->kobj, KOBJ_BIND);
}
static ssize_t coredump_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
device_lock(dev);
dev->driver->coredump(dev);
device_unlock(dev);
return count;
}
static DEVICE_ATTR_WO(coredump);
static int driver_sysfs_add(struct device *dev)
{
int ret;
bus_notify(dev, BUS_NOTIFY_BIND_DRIVER);
ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj,
kobject_name(&dev->kobj));
if (ret)
goto fail;
ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj,
"driver");
if (ret)
goto rm_dev;
if (!IS_ENABLED(CONFIG_DEV_COREDUMP) || !dev->driver->coredump)
return 0;
ret = device_create_file(dev, &dev_attr_coredump);
if (!ret)
return 0;
sysfs_remove_link(&dev->kobj, "driver");
rm_dev:
sysfs_remove_link(&dev->driver->p->kobj,
kobject_name(&dev->kobj));
fail:
return ret;
}
static void driver_sysfs_remove(struct device *dev)
{
struct device_driver *drv = dev->driver;
if (drv) {
if (drv->coredump) device_remove_file(dev, &dev_attr_coredump); sysfs_remove_link(&drv->p->kobj, kobject_name(&dev->kobj));
sysfs_remove_link(&dev->kobj, "driver");
}
}
/**
* device_bind_driver - bind a driver to one device.
* @dev: device.
*
* Allow manual attachment of a driver to a device.
* Caller must have already set @dev->driver.
*
* Note that this does not modify the bus reference count.
* Please verify that is accounted for before calling this.
* (It is ok to call with no other effort from a driver's probe() method.)
*
* This function must be called with the device lock held.
*
* Callers should prefer to use device_driver_attach() instead.
*/
int device_bind_driver(struct device *dev)
{
int ret;
ret = driver_sysfs_add(dev);
if (!ret) {
device_links_force_bind(dev);
driver_bound(dev);
}
else
bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND);
return ret;
}
EXPORT_SYMBOL_GPL(device_bind_driver);
static atomic_t probe_count = ATOMIC_INIT(0);
static DECLARE_WAIT_QUEUE_HEAD(probe_waitqueue);
static ssize_t state_synced_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
int ret = 0;
if (strcmp("1", buf))
return -EINVAL;
device_lock(dev);
if (!dev->state_synced) {
dev->state_synced = true;
dev_sync_state(dev);
} else {
ret = -EINVAL;
}
device_unlock(dev);
return ret ? ret : count;
}
static ssize_t state_synced_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
bool val;
device_lock(dev);
val = dev->state_synced;
device_unlock(dev);
return sysfs_emit(buf, "%u\n", val);
}
static DEVICE_ATTR_RW(state_synced);
static void device_unbind_cleanup(struct device *dev)
{
devres_release_all(dev);
arch_teardown_dma_ops(dev);
kfree(dev->dma_range_map);
dev->dma_range_map = NULL;
dev->driver = NULL;
dev_set_drvdata(dev, NULL);
if (dev->pm_domain && dev->pm_domain->dismiss) dev->pm_domain->dismiss(dev); pm_runtime_reinit(dev);
dev_pm_set_driver_flags(dev, 0);
}
static void device_remove(struct device *dev)
{
device_remove_file(dev, &dev_attr_state_synced);
device_remove_groups(dev, dev->driver->dev_groups);
if (dev->bus && dev->bus->remove) dev->bus->remove(dev); else if (dev->driver->remove) dev->driver->remove(dev);
}
static int call_driver_probe(struct device *dev, struct device_driver *drv)
{
int ret = 0;
if (dev->bus->probe)
ret = dev->bus->probe(dev);
else if (drv->probe) ret = drv->probe(dev); switch (ret) {
case 0:
break;
case -EPROBE_DEFER:
/* Driver requested deferred probing */
dev_dbg(dev, "Driver %s requests probe deferral\n", drv->name);
break;
case -ENODEV:
case -ENXIO:
dev_dbg(dev, "probe with driver %s rejects match %d\n",
drv->name, ret);
break;
default:
/* driver matched but the probe failed */
dev_err(dev, "probe with driver %s failed with error %d\n",
drv->name, ret);
break;
}
return ret;
}
static int really_probe(struct device *dev, struct device_driver *drv)
{
bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE) &&
!drv->suppress_bind_attrs;
int ret, link_ret;
if (defer_all_probes) {
/*
* Value of defer_all_probes can be set only by
* device_block_probing() which, in turn, will call
* wait_for_device_probe() right after that to avoid any races.
*/
dev_dbg(dev, "Driver %s force probe deferral\n", drv->name);
return -EPROBE_DEFER;
}
link_ret = device_links_check_suppliers(dev);
if (link_ret == -EPROBE_DEFER)
return link_ret;
dev_dbg(dev, "bus: '%s': %s: probing driver %s with device\n",
drv->bus->name, __func__, drv->name);
if (!list_empty(&dev->devres_head)) { dev_crit(dev, "Resources present before probing\n");
ret = -EBUSY;
goto done;
}
re_probe:
dev->driver = drv;
/* If using pinctrl, bind pins now before probing */
ret = pinctrl_bind_pins(dev);
if (ret)
goto pinctrl_bind_failed;
if (dev->bus->dma_configure) {
ret = dev->bus->dma_configure(dev);
if (ret)
goto pinctrl_bind_failed;
}
ret = driver_sysfs_add(dev);
if (ret) {
dev_err(dev, "%s: driver_sysfs_add failed\n", __func__);
goto sysfs_failed;
}
if (dev->pm_domain && dev->pm_domain->activate) { ret = dev->pm_domain->activate(dev);
if (ret)
goto probe_failed;
}
ret = call_driver_probe(dev, drv);
if (ret) {
/*
* If fw_devlink_best_effort is active (denoted by -EAGAIN), the
* device might actually probe properly once some of its missing
* suppliers have probed. So, treat this as if the driver
* returned -EPROBE_DEFER.
*/
if (link_ret == -EAGAIN)
ret = -EPROBE_DEFER;
/*
* Return probe errors as positive values so that the callers
* can distinguish them from other errors.
*/
ret = -ret;
goto probe_failed;
}
ret = device_add_groups(dev, drv->dev_groups);
if (ret) {
dev_err(dev, "device_add_groups() failed\n");
goto dev_groups_failed;
}
if (dev_has_sync_state(dev)) { ret = device_create_file(dev, &dev_attr_state_synced);
if (ret) {
dev_err(dev, "state_synced sysfs add failed\n");
goto dev_sysfs_state_synced_failed;
}
}
if (test_remove) {
test_remove = false;
device_remove(dev);
driver_sysfs_remove(dev);
if (dev->bus && dev->bus->dma_cleanup)
dev->bus->dma_cleanup(dev);
device_unbind_cleanup(dev);
goto re_probe;
}
pinctrl_init_done(dev);
if (dev->pm_domain && dev->pm_domain->sync) dev->pm_domain->sync(dev); driver_bound(dev); dev_dbg(dev, "bus: '%s': %s: bound device to driver %s\n",
drv->bus->name, __func__, drv->name);
goto done;
dev_sysfs_state_synced_failed:
dev_groups_failed:
device_remove(dev);
probe_failed:
driver_sysfs_remove(dev);
sysfs_failed:
bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND); if (dev->bus && dev->bus->dma_cleanup) dev->bus->dma_cleanup(dev);
pinctrl_bind_failed:
device_links_no_driver(dev); device_unbind_cleanup(dev);
done:
return ret;
}
/*
* For initcall_debug, show the driver probe time.
*/
static int really_probe_debug(struct device *dev, struct device_driver *drv)
{
ktime_t calltime, rettime;
int ret;
calltime = ktime_get();
ret = really_probe(dev, drv);
rettime = ktime_get();
/*
* Don't change this to pr_debug() because that requires
* CONFIG_DYNAMIC_DEBUG and we want a simple 'initcall_debug' on the
* kernel commandline to print this all the time at the debug level.
*/
printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n",
dev_name(dev), ret, ktime_us_delta(rettime, calltime));
return ret;
}
/**
* driver_probe_done
* Determine if the probe sequence is finished or not.
*
* Should somehow figure out how to use a semaphore, not an atomic variable...
*/
bool __init driver_probe_done(void)
{
int local_probe_count = atomic_read(&probe_count);
pr_debug("%s: probe_count = %d\n", __func__, local_probe_count);
return !local_probe_count;
}
/**
* wait_for_device_probe
* Wait for device probing to be completed.
*/
void wait_for_device_probe(void)
{
/* wait for the deferred probe workqueue to finish */
flush_work(&deferred_probe_work);
/* wait for the known devices to complete their probing */
wait_event(probe_waitqueue, atomic_read(&probe_count) == 0);
async_synchronize_full();
}
EXPORT_SYMBOL_GPL(wait_for_device_probe);
static int __driver_probe_device(struct device_driver *drv, struct device *dev)
{
int ret = 0;
if (dev->p->dead || !device_is_registered(dev))
return -ENODEV;
if (dev->driver)
return -EBUSY;
dev->can_match = true; dev_dbg(dev, "bus: '%s': %s: matched device with driver %s\n",
drv->bus->name, __func__, drv->name);
pm_runtime_get_suppliers(dev);
if (dev->parent)
pm_runtime_get_sync(dev->parent); pm_runtime_barrier(dev);
if (initcall_debug)
ret = really_probe_debug(dev, drv);
else
ret = really_probe(dev, drv); pm_request_idle(dev);
if (dev->parent)
pm_runtime_put(dev->parent); pm_runtime_put_suppliers(dev); return ret;
}
/**
* driver_probe_device - attempt to bind device & driver together
* @drv: driver to bind a device to
* @dev: device to try to bind to the driver
*
* This function returns -ENODEV if the device is not registered, -EBUSY if it
* already has a driver, 0 if the device is bound successfully and a positive
* (inverted) error code for failures from the ->probe method.
*
* This function must be called with @dev lock held. When called for a
* USB interface, @dev->parent lock must be held as well.
*
* If the device has a parent, runtime-resume the parent before driver probing.
*/
static int driver_probe_device(struct device_driver *drv, struct device *dev)
{
int trigger_count = atomic_read(&deferred_trigger_count);
int ret;
atomic_inc(&probe_count);
ret = __driver_probe_device(drv, dev);
if (ret == -EPROBE_DEFER || ret == EPROBE_DEFER) { driver_deferred_probe_add(dev);
/*
* Did a trigger occur while probing? Need to re-trigger if yes
*/
if (trigger_count != atomic_read(&deferred_trigger_count) && !defer_all_probes) driver_deferred_probe_trigger();
}
atomic_dec(&probe_count);
wake_up_all(&probe_waitqueue);
return ret;
}
static inline bool cmdline_requested_async_probing(const char *drv_name)
{
bool async_drv;
async_drv = parse_option_str(async_probe_drv_names, drv_name);
return (async_probe_default != async_drv);
}
/* The option format is "driver_async_probe=drv_name1,drv_name2,..." */
static int __init save_async_options(char *buf)
{
if (strlen(buf) >= ASYNC_DRV_NAMES_MAX_LEN)
pr_warn("Too long list of driver names for 'driver_async_probe'!\n");
strscpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN);
async_probe_default = parse_option_str(async_probe_drv_names, "*");
return 1;
}
__setup("driver_async_probe=", save_async_options);
static bool driver_allows_async_probing(struct device_driver *drv)
{
switch (drv->probe_type) {
case PROBE_PREFER_ASYNCHRONOUS:
return true;
case PROBE_FORCE_SYNCHRONOUS:
return false;
default:
if (cmdline_requested_async_probing(drv->name))
return true;
if (module_requested_async_probing(drv->owner))
return true;
return false;
}
}
struct device_attach_data {
struct device *dev;
/*
* Indicates whether we are considering asynchronous probing or
* not. Only initial binding after device or driver registration
* (including deferral processing) may be done asynchronously, the
* rest is always synchronous, as we expect it is being done by
* request from userspace.
*/
bool check_async;
/*
* Indicates if we are binding synchronous or asynchronous drivers.
* When asynchronous probing is enabled we'll execute 2 passes
* over drivers: first pass doing synchronous probing and second
* doing asynchronous probing (if synchronous did not succeed -
* most likely because there was no driver requiring synchronous
* probing - and we found asynchronous driver during first pass).
* The 2 passes are done because we can't shoot asynchronous
* probe for given device and driver from bus_for_each_drv() since
* driver pointer is not guaranteed to stay valid once
* bus_for_each_drv() iterates to the next driver on the bus.
*/
bool want_async;
/*
* We'll set have_async to 'true' if, while scanning for matching
* driver, we'll encounter one that requests asynchronous probing.
*/
bool have_async;
};
static int __device_attach_driver(struct device_driver *drv, void *_data)
{
struct device_attach_data *data = _data;
struct device *dev = data->dev;
bool async_allowed;
int ret;
ret = driver_match_device(drv, dev);
if (ret == 0) {
/* no match */
return 0; } else if (ret == -EPROBE_DEFER) { dev_dbg(dev, "Device match requests probe deferral\n"); dev->can_match = true;
driver_deferred_probe_add(dev);
/*
* Device can't match with a driver right now, so don't attempt
* to match or bind with other drivers on the bus.
*/
return ret;
} else if (ret < 0) { dev_dbg(dev, "Bus failed to match device: %d\n", ret);
return ret;
} /* ret > 0 means positive match */
async_allowed = driver_allows_async_probing(drv);
if (async_allowed)
data->have_async = true; if (data->check_async && async_allowed != data->want_async)
return 0;
/*
* Ignore errors returned by ->probe so that the next driver can try
* its luck.
*/
ret = driver_probe_device(drv, dev);
if (ret < 0)
return ret;
return ret == 0;
}
static void __device_attach_async_helper(void *_dev, async_cookie_t cookie)
{
struct device *dev = _dev;
struct device_attach_data data = {
.dev = dev,
.check_async = true,
.want_async = true,
};
device_lock(dev);
/*
* Check if device has already been removed or claimed. This may
* happen with driver loading, device discovery/registration,
* and deferred probe processing happens all at once with
* multiple threads.
*/
if (dev->p->dead || dev->driver)
goto out_unlock;
if (dev->parent)
pm_runtime_get_sync(dev->parent);
bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver);
dev_dbg(dev, "async probe completed\n");
pm_request_idle(dev);
if (dev->parent)
pm_runtime_put(dev->parent);
out_unlock:
device_unlock(dev);
put_device(dev);
}
static int __device_attach(struct device *dev, bool allow_async)
{
int ret = 0;
bool async = false;
device_lock(dev);
if (dev->p->dead) {
goto out_unlock;
} else if (dev->driver) { if (device_is_bound(dev)) {
ret = 1;
goto out_unlock;
}
ret = device_bind_driver(dev);
if (ret == 0)
ret = 1;
else {
dev->driver = NULL;
ret = 0;
}
} else {
struct device_attach_data data = {
.dev = dev,
.check_async = allow_async,
.want_async = false,
};
if (dev->parent)
pm_runtime_get_sync(dev->parent); ret = bus_for_each_drv(dev->bus, NULL, &data,
__device_attach_driver);
if (!ret && allow_async && data.have_async) {
/*
* If we could not find appropriate driver
* synchronously and we are allowed to do
* async probes and there are drivers that
* want to probe asynchronously, we'll
* try them.
*/
dev_dbg(dev, "scheduling asynchronous probe\n"); get_device(dev);
async = true;
} else {
pm_request_idle(dev);
}
if (dev->parent)
pm_runtime_put(dev->parent);
}
out_unlock:
device_unlock(dev); if (async) async_schedule_dev(__device_attach_async_helper, dev); return ret;
}
/**
* device_attach - try to attach device to a driver.
* @dev: device.
*
* Walk the list of drivers that the bus has and call
* driver_probe_device() for each pair. If a compatible
* pair is found, break out and return.
*
* Returns 1 if the device was bound to a driver;
* 0 if no matching driver was found;
* -ENODEV if the device is not registered.
*
* When called for a USB interface, @dev->parent lock must be held.
*/
int device_attach(struct device *dev)
{
return __device_attach(dev, false);
}
EXPORT_SYMBOL_GPL(device_attach);
void device_initial_probe(struct device *dev)
{
__device_attach(dev, true);
}
/*
* __device_driver_lock - acquire locks needed to manipulate dev->drv
* @dev: Device we will update driver info for
* @parent: Parent device. Needed if the bus requires parent lock
*
* This function will take the required locks for manipulating dev->drv.
* Normally this will just be the @dev lock, but when called for a USB
* interface, @parent lock will be held as well.
*/
static void __device_driver_lock(struct device *dev, struct device *parent)
{
if (parent && dev->bus->need_parent_lock) device_lock(parent); device_lock(dev);
}
/*
* __device_driver_unlock - release locks needed to manipulate dev->drv
* @dev: Device we will update driver info for
* @parent: Parent device. Needed if the bus requires parent lock
*
* This function will release the required locks for manipulating dev->drv.
* Normally this will just be the @dev lock, but when called for a
* USB interface, @parent lock will be released as well.
*/
static void __device_driver_unlock(struct device *dev, struct device *parent)
{
device_unlock(dev); if (parent && dev->bus->need_parent_lock) device_unlock(parent);
}
/**
* device_driver_attach - attach a specific driver to a specific device
* @drv: Driver to attach
* @dev: Device to attach it to
*
* Manually attach driver to a device. Will acquire both @dev lock and
* @dev->parent lock if needed. Returns 0 on success, -ERR on failure.
*/
int device_driver_attach(struct device_driver *drv, struct device *dev)
{
int ret;
__device_driver_lock(dev, dev->parent);
ret = __driver_probe_device(drv, dev);
__device_driver_unlock(dev, dev->parent);
/* also return probe errors as normal negative errnos */
if (ret > 0)
ret = -ret;
if (ret == -EPROBE_DEFER)
return -EAGAIN;
return ret;
}
EXPORT_SYMBOL_GPL(device_driver_attach);
static void __driver_attach_async_helper(void *_dev, async_cookie_t cookie)
{
struct device *dev = _dev;
struct device_driver *drv;
int ret;
__device_driver_lock(dev, dev->parent);
drv = dev->p->async_driver;
dev->p->async_driver = NULL;
ret = driver_probe_device(drv, dev);
__device_driver_unlock(dev, dev->parent);
dev_dbg(dev, "driver %s async attach completed: %d\n", drv->name, ret);
put_device(dev);
}
static int __driver_attach(struct device *dev, void *data)
{
struct device_driver *drv = data;
bool async = false;
int ret;
/*
* Lock device and try to bind to it. We drop the error
* here and always return 0, because we need to keep trying
* to bind to devices and some drivers will return an error
* simply if it didn't support the device.
*
* driver_probe_device() will spit a warning if there
* is an error.
*/
ret = driver_match_device(drv, dev);
if (ret == 0) {
/* no match */
return 0;
} else if (ret == -EPROBE_DEFER) { dev_dbg(dev, "Device match requests probe deferral\n"); dev->can_match = true;
driver_deferred_probe_add(dev);
/*
* Driver could not match with device, but may match with
* another device on the bus.
*/
return 0; } else if (ret < 0) { dev_dbg(dev, "Bus failed to match device: %d\n", ret);
/*
* Driver could not match with device, but may match with
* another device on the bus.
*/
return 0;
} /* ret > 0 means positive match */
if (driver_allows_async_probing(drv)) {
/*
* Instead of probing the device synchronously we will
* probe it asynchronously to allow for more parallelism.
*
* We only take the device lock here in order to guarantee
* that the dev->driver and async_driver fields are protected
*/
dev_dbg(dev, "probing driver %s asynchronously\n", drv->name); device_lock(dev); if (!dev->driver && !dev->p->async_driver) { get_device(dev);
dev->p->async_driver = drv;
async = true;
}
device_unlock(dev); if (async)
async_schedule_dev(__driver_attach_async_helper, dev);
return 0;
}
__device_driver_lock(dev, dev->parent);
driver_probe_device(drv, dev);
__device_driver_unlock(dev, dev->parent);
return 0;
}
/**
* driver_attach - try to bind driver to devices.
* @drv: driver.
*
* Walk the list of devices that the bus has on it and try to
* match the driver with each one. If driver_probe_device()
* returns 0 and the @dev->driver is set, we've found a
* compatible pair.
*/
int driver_attach(struct device_driver *drv)
{
return bus_for_each_dev(drv->bus, NULL, drv, __driver_attach);
}
EXPORT_SYMBOL_GPL(driver_attach);
/*
* __device_release_driver() must be called with @dev lock held.
* When called for a USB interface, @dev->parent lock must be held as well.
*/
static void __device_release_driver(struct device *dev, struct device *parent)
{
struct device_driver *drv;
drv = dev->driver;
if (drv) {
pm_runtime_get_sync(dev); while (device_links_busy(dev)) { __device_driver_unlock(dev, parent); device_links_unbind_consumers(dev); __device_driver_lock(dev, parent);
/*
* A concurrent invocation of the same function might
* have released the driver successfully while this one
* was waiting, so check for that.
*/
if (dev->driver != drv) {
pm_runtime_put(dev);
return;
}
}
driver_sysfs_remove(dev); bus_notify(dev, BUS_NOTIFY_UNBIND_DRIVER);
pm_runtime_put_sync(dev);
device_remove(dev);
if (dev->bus && dev->bus->dma_cleanup) dev->bus->dma_cleanup(dev); device_unbind_cleanup(dev);
device_links_driver_cleanup(dev);
klist_remove(&dev->p->knode_driver);
device_pm_check_callbacks(dev);
bus_notify(dev, BUS_NOTIFY_UNBOUND_DRIVER);
kobject_uevent(&dev->kobj, KOBJ_UNBIND);
}
}
void device_release_driver_internal(struct device *dev,
struct device_driver *drv,
struct device *parent)
{
__device_driver_lock(dev, parent); if (!drv || drv == dev->driver) __device_release_driver(dev, parent); __device_driver_unlock(dev, parent);
}
/**
* device_release_driver - manually detach device from driver.
* @dev: device.
*
* Manually detach device from driver.
* When called for a USB interface, @dev->parent lock must be held.
*
* If this function is to be called with @dev->parent lock held, ensure that
* the device's consumers are unbound in advance or that their locks can be
* acquired under the @dev->parent lock.
*/
void device_release_driver(struct device *dev)
{
/*
* If anyone calls device_release_driver() recursively from
* within their ->remove callback for the same device, they
* will deadlock right here.
*/
device_release_driver_internal(dev, NULL, NULL);
}
EXPORT_SYMBOL_GPL(device_release_driver);
/**
* device_driver_detach - detach driver from a specific device
* @dev: device to detach driver from
*
* Detach driver from device. Will acquire both @dev lock and @dev->parent
* lock if needed.
*/
void device_driver_detach(struct device *dev)
{
device_release_driver_internal(dev, NULL, dev->parent);
}
/**
* driver_detach - detach driver from all devices it controls.
* @drv: driver.
*/
void driver_detach(struct device_driver *drv)
{
struct device_private *dev_prv;
struct device *dev;
if (driver_allows_async_probing(drv)) async_synchronize_full();
for (;;) {
spin_lock(&drv->p->klist_devices.k_lock);
if (list_empty(&drv->p->klist_devices.k_list)) {
spin_unlock(&drv->p->klist_devices.k_lock);
break;
}
dev_prv = list_last_entry(&drv->p->klist_devices.k_list,
struct device_private,
knode_driver.n_node);
dev = dev_prv->device;
get_device(dev);
spin_unlock(&drv->p->klist_devices.k_lock);
device_release_driver_internal(dev, drv, dev->parent);
put_device(dev);
}
}
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Fast and scalable bitmaps.
*
* Copyright (C) 2016 Facebook
* Copyright (C) 2013-2014 Jens Axboe
*/
#ifndef __LINUX_SCALE_BITMAP_H
#define __LINUX_SCALE_BITMAP_H
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/cache.h>
#include <linux/list.h>
#include <linux/log2.h>
#include <linux/minmax.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/types.h>
#include <linux/wait.h>
struct seq_file;
/**
* struct sbitmap_word - Word in a &struct sbitmap.
*/
struct sbitmap_word {
/**
* @word: word holding free bits
*/
unsigned long word;
/**
* @cleared: word holding cleared bits
*/
unsigned long cleared ____cacheline_aligned_in_smp;
} ____cacheline_aligned_in_smp;
/**
* struct sbitmap - Scalable bitmap.
*
* A &struct sbitmap is spread over multiple cachelines to avoid ping-pong. This
* trades off higher memory usage for better scalability.
*/
struct sbitmap {
/**
* @depth: Number of bits used in the whole bitmap.
*/
unsigned int depth;
/**
* @shift: log2(number of bits used per word)
*/
unsigned int shift;
/**
* @map_nr: Number of words (cachelines) being used for the bitmap.
*/
unsigned int map_nr;
/**
* @round_robin: Allocate bits in strict round-robin order.
*/
bool round_robin;
/**
* @map: Allocated bitmap.
*/
struct sbitmap_word *map;
/*
* @alloc_hint: Cache of last successfully allocated or freed bit.
*
* This is per-cpu, which allows multiple users to stick to different
* cachelines until the map is exhausted.
*/
unsigned int __percpu *alloc_hint;
};
#define SBQ_WAIT_QUEUES 8
#define SBQ_WAKE_BATCH 8
/**
* struct sbq_wait_state - Wait queue in a &struct sbitmap_queue.
*/
struct sbq_wait_state {
/**
* @wait: Wait queue.
*/
wait_queue_head_t wait;
} ____cacheline_aligned_in_smp;
/**
* struct sbitmap_queue - Scalable bitmap with the added ability to wait on free
* bits.
*
* A &struct sbitmap_queue uses multiple wait queues and rolling wakeups to
* avoid contention on the wait queue spinlock. This ensures that we don't hit a
* scalability wall when we run out of free bits and have to start putting tasks
* to sleep.
*/
struct sbitmap_queue {
/**
* @sb: Scalable bitmap.
*/
struct sbitmap sb;
/**
* @wake_batch: Number of bits which must be freed before we wake up any
* waiters.
*/
unsigned int wake_batch;
/**
* @wake_index: Next wait queue in @ws to wake up.
*/
atomic_t wake_index;
/**
* @ws: Wait queues.
*/
struct sbq_wait_state *ws;
/*
* @ws_active: count of currently active ws waitqueues
*/
atomic_t ws_active;
/**
* @min_shallow_depth: The minimum shallow depth which may be passed to
* sbitmap_queue_get_shallow()
*/
unsigned int min_shallow_depth;
/**
* @completion_cnt: Number of bits cleared passed to the
* wakeup function.
*/
atomic_t completion_cnt;
/**
* @wakeup_cnt: Number of thread wake ups issued.
*/
atomic_t wakeup_cnt;
};
/**
* sbitmap_init_node() - Initialize a &struct sbitmap on a specific memory node.
* @sb: Bitmap to initialize.
* @depth: Number of bits to allocate.
* @shift: Use 2^@shift bits per word in the bitmap; if a negative number if
* given, a good default is chosen.
* @flags: Allocation flags.
* @node: Memory node to allocate on.
* @round_robin: If true, be stricter about allocation order; always allocate
* starting from the last allocated bit. This is less efficient
* than the default behavior (false).
* @alloc_hint: If true, apply percpu hint for where to start searching for
* a free bit.
*
* Return: Zero on success or negative errno on failure.
*/
int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
gfp_t flags, int node, bool round_robin, bool alloc_hint);
/* sbitmap internal helper */
static inline unsigned int __map_depth(const struct sbitmap *sb, int index)
{
if (index == sb->map_nr - 1)
return sb->depth - (index << sb->shift);
return 1U << sb->shift;
}
/**
* sbitmap_free() - Free memory used by a &struct sbitmap.
* @sb: Bitmap to free.
*/
static inline void sbitmap_free(struct sbitmap *sb)
{
free_percpu(sb->alloc_hint);
kvfree(sb->map);
sb->map = NULL;
}
/**
* sbitmap_resize() - Resize a &struct sbitmap.
* @sb: Bitmap to resize.
* @depth: New number of bits to resize to.
*
* Doesn't reallocate anything. It's up to the caller to ensure that the new
* depth doesn't exceed the depth that the sb was initialized with.
*/
void sbitmap_resize(struct sbitmap *sb, unsigned int depth);
/**
* sbitmap_get() - Try to allocate a free bit from a &struct sbitmap.
* @sb: Bitmap to allocate from.
*
* This operation provides acquire barrier semantics if it succeeds.
*
* Return: Non-negative allocated bit number if successful, -1 otherwise.
*/
int sbitmap_get(struct sbitmap *sb);
/**
* sbitmap_get_shallow() - Try to allocate a free bit from a &struct sbitmap,
* limiting the depth used from each word.
* @sb: Bitmap to allocate from.
* @shallow_depth: The maximum number of bits to allocate from a single word.
*
* This rather specific operation allows for having multiple users with
* different allocation limits. E.g., there can be a high-priority class that
* uses sbitmap_get() and a low-priority class that uses sbitmap_get_shallow()
* with a @shallow_depth of (1 << (@sb->shift - 1)). Then, the low-priority
* class can only allocate half of the total bits in the bitmap, preventing it
* from starving out the high-priority class.
*
* Return: Non-negative allocated bit number if successful, -1 otherwise.
*/
int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth);
/**
* sbitmap_any_bit_set() - Check for a set bit in a &struct sbitmap.
* @sb: Bitmap to check.
*
* Return: true if any bit in the bitmap is set, false otherwise.
*/
bool sbitmap_any_bit_set(const struct sbitmap *sb);
#define SB_NR_TO_INDEX(sb, bitnr) ((bitnr) >> (sb)->shift)
#define SB_NR_TO_BIT(sb, bitnr) ((bitnr) & ((1U << (sb)->shift) - 1U))
typedef bool (*sb_for_each_fn)(struct sbitmap *, unsigned int, void *);
/**
* __sbitmap_for_each_set() - Iterate over each set bit in a &struct sbitmap.
* @start: Where to start the iteration.
* @sb: Bitmap to iterate over.
* @fn: Callback. Should return true to continue or false to break early.
* @data: Pointer to pass to callback.
*
* This is inline even though it's non-trivial so that the function calls to the
* callback will hopefully get optimized away.
*/
static inline void __sbitmap_for_each_set(struct sbitmap *sb,
unsigned int start,
sb_for_each_fn fn, void *data)
{
unsigned int index;
unsigned int nr;
unsigned int scanned = 0;
if (start >= sb->depth)
start = 0;
index = SB_NR_TO_INDEX(sb, start);
nr = SB_NR_TO_BIT(sb, start);
while (scanned < sb->depth) {
unsigned long word;
unsigned int depth = min_t(unsigned int,
__map_depth(sb, index) - nr,
sb->depth - scanned);
scanned += depth;
word = sb->map[index].word & ~sb->map[index].cleared;
if (!word)
goto next;
/*
* On the first iteration of the outer loop, we need to add the
* bit offset back to the size of the word for find_next_bit().
* On all other iterations, nr is zero, so this is a noop.
*/
depth += nr;
while (1) {
nr = find_next_bit(&word, depth, nr);
if (nr >= depth)
break;
if (!fn(sb, (index << sb->shift) + nr, data))
return;
nr++;
}
next:
nr = 0;
if (++index >= sb->map_nr)
index = 0;
}
}
/**
* sbitmap_for_each_set() - Iterate over each set bit in a &struct sbitmap.
* @sb: Bitmap to iterate over.
* @fn: Callback. Should return true to continue or false to break early.
* @data: Pointer to pass to callback.
*/
static inline void sbitmap_for_each_set(struct sbitmap *sb, sb_for_each_fn fn,
void *data)
{
__sbitmap_for_each_set(sb, 0, fn, data);
}
static inline unsigned long *__sbitmap_word(struct sbitmap *sb,
unsigned int bitnr)
{
return &sb->map[SB_NR_TO_INDEX(sb, bitnr)].word;
}
/* Helpers equivalent to the operations in asm/bitops.h and linux/bitmap.h */
static inline void sbitmap_set_bit(struct sbitmap *sb, unsigned int bitnr)
{
set_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));
}
static inline void sbitmap_clear_bit(struct sbitmap *sb, unsigned int bitnr)
{
clear_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));
}
/*
* This one is special, since it doesn't actually clear the bit, rather it
* sets the corresponding bit in the ->cleared mask instead. Paired with
* the caller doing sbitmap_deferred_clear() if a given index is full, which
* will clear the previously freed entries in the corresponding ->word.
*/
static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int bitnr)
{
unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared;
set_bit(SB_NR_TO_BIT(sb, bitnr), addr);
}
/*
* Pair of sbitmap_get, and this one applies both cleared bit and
* allocation hint.
*/
static inline void sbitmap_put(struct sbitmap *sb, unsigned int bitnr)
{
sbitmap_deferred_clear_bit(sb, bitnr);
if (likely(sb->alloc_hint && !sb->round_robin && bitnr < sb->depth))
*raw_cpu_ptr(sb->alloc_hint) = bitnr;
}
static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr)
{
return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));
}
static inline int sbitmap_calculate_shift(unsigned int depth)
{
int shift = ilog2(BITS_PER_LONG);
/*
* If the bitmap is small, shrink the number of bits per word so
* we spread over a few cachelines, at least. If less than 4
* bits, just forget about it, it's not going to work optimally
* anyway.
*/
if (depth >= 4) {
while ((4U << shift) > depth)
shift--;
}
return shift;
}
/**
* sbitmap_show() - Dump &struct sbitmap information to a &struct seq_file.
* @sb: Bitmap to show.
* @m: struct seq_file to write to.
*
* This is intended for debugging. The format may change at any time.
*/
void sbitmap_show(struct sbitmap *sb, struct seq_file *m);
/**
* sbitmap_weight() - Return how many set and not cleared bits in a &struct
* sbitmap.
* @sb: Bitmap to check.
*
* Return: How many set and not cleared bits set
*/
unsigned int sbitmap_weight(const struct sbitmap *sb);
/**
* sbitmap_bitmap_show() - Write a hex dump of a &struct sbitmap to a &struct
* seq_file.
* @sb: Bitmap to show.
* @m: struct seq_file to write to.
*
* This is intended for debugging. The output isn't guaranteed to be internally
* consistent.
*/
void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m);
/**
* sbitmap_queue_init_node() - Initialize a &struct sbitmap_queue on a specific
* memory node.
* @sbq: Bitmap queue to initialize.
* @depth: See sbitmap_init_node().
* @shift: See sbitmap_init_node().
* @round_robin: See sbitmap_get().
* @flags: Allocation flags.
* @node: Memory node to allocate on.
*
* Return: Zero on success or negative errno on failure.
*/
int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
int shift, bool round_robin, gfp_t flags, int node);
/**
* sbitmap_queue_free() - Free memory used by a &struct sbitmap_queue.
*
* @sbq: Bitmap queue to free.
*/
static inline void sbitmap_queue_free(struct sbitmap_queue *sbq)
{
kfree(sbq->ws);
sbitmap_free(&sbq->sb);
}
/**
* sbitmap_queue_recalculate_wake_batch() - Recalculate wake batch
* @sbq: Bitmap queue to recalculate wake batch.
* @users: Number of shares.
*
* Like sbitmap_queue_update_wake_batch(), this will calculate wake batch
* by depth. This interface is for HCTX shared tags or queue shared tags.
*/
void sbitmap_queue_recalculate_wake_batch(struct sbitmap_queue *sbq,
unsigned int users);
/**
* sbitmap_queue_resize() - Resize a &struct sbitmap_queue.
* @sbq: Bitmap queue to resize.
* @depth: New number of bits to resize to.
*
* Like sbitmap_resize(), this doesn't reallocate anything. It has to do
* some extra work on the &struct sbitmap_queue, so it's not safe to just
* resize the underlying &struct sbitmap.
*/
void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth);
/**
* __sbitmap_queue_get() - Try to allocate a free bit from a &struct
* sbitmap_queue with preemption already disabled.
* @sbq: Bitmap queue to allocate from.
*
* Return: Non-negative allocated bit number if successful, -1 otherwise.
*/
int __sbitmap_queue_get(struct sbitmap_queue *sbq);
/**
* __sbitmap_queue_get_batch() - Try to allocate a batch of free bits
* @sbq: Bitmap queue to allocate from.
* @nr_tags: number of tags requested
* @offset: offset to add to returned bits
*
* Return: Mask of allocated tags, 0 if none are found. Each tag allocated is
* a bit in the mask returned, and the caller must add @offset to the value to
* get the absolute tag value.
*/
unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
unsigned int *offset);
/**
* sbitmap_queue_get_shallow() - Try to allocate a free bit from a &struct
* sbitmap_queue, limiting the depth used from each word, with preemption
* already disabled.
* @sbq: Bitmap queue to allocate from.
* @shallow_depth: The maximum number of bits to allocate from a single word.
* See sbitmap_get_shallow().
*
* If you call this, make sure to call sbitmap_queue_min_shallow_depth() after
* initializing @sbq.
*
* Return: Non-negative allocated bit number if successful, -1 otherwise.
*/
int sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
unsigned int shallow_depth);
/**
* sbitmap_queue_get() - Try to allocate a free bit from a &struct
* sbitmap_queue.
* @sbq: Bitmap queue to allocate from.
* @cpu: Output parameter; will contain the CPU we ran on (e.g., to be passed to
* sbitmap_queue_clear()).
*
* Return: Non-negative allocated bit number if successful, -1 otherwise.
*/
static inline int sbitmap_queue_get(struct sbitmap_queue *sbq,
unsigned int *cpu)
{
int nr;
*cpu = get_cpu();
nr = __sbitmap_queue_get(sbq);
put_cpu();
return nr;
}
/**
* sbitmap_queue_min_shallow_depth() - Inform a &struct sbitmap_queue of the
* minimum shallow depth that will be used.
* @sbq: Bitmap queue in question.
* @min_shallow_depth: The minimum shallow depth that will be passed to
* sbitmap_queue_get_shallow() or __sbitmap_queue_get_shallow().
*
* sbitmap_queue_clear() batches wakeups as an optimization. The batch size
* depends on the depth of the bitmap. Since the shallow allocation functions
* effectively operate with a different depth, the shallow depth must be taken
* into account when calculating the batch size. This function must be called
* with the minimum shallow depth that will be used. Failure to do so can result
* in missed wakeups.
*/
void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq,
unsigned int min_shallow_depth);
/**
* sbitmap_queue_clear() - Free an allocated bit and wake up waiters on a
* &struct sbitmap_queue.
* @sbq: Bitmap to free from.
* @nr: Bit number to free.
* @cpu: CPU the bit was allocated on.
*/
void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
unsigned int cpu);
/**
* sbitmap_queue_clear_batch() - Free a batch of allocated bits
* &struct sbitmap_queue.
* @sbq: Bitmap to free from.
* @offset: offset for each tag in array
* @tags: array of tags
* @nr_tags: number of tags in array
*/
void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset,
int *tags, int nr_tags);
static inline int sbq_index_inc(int index)
{
return (index + 1) & (SBQ_WAIT_QUEUES - 1);
}
static inline void sbq_index_atomic_inc(atomic_t *index)
{
int old = atomic_read(index);
int new = sbq_index_inc(old);
atomic_cmpxchg(index, old, new);
}
/**
* sbq_wait_ptr() - Get the next wait queue to use for a &struct
* sbitmap_queue.
* @sbq: Bitmap queue to wait on.
* @wait_index: A counter per "user" of @sbq.
*/
static inline struct sbq_wait_state *sbq_wait_ptr(struct sbitmap_queue *sbq,
atomic_t *wait_index)
{
struct sbq_wait_state *ws;
ws = &sbq->ws[atomic_read(wait_index)];
sbq_index_atomic_inc(wait_index);
return ws;
}
/**
* sbitmap_queue_wake_all() - Wake up everything waiting on a &struct
* sbitmap_queue.
* @sbq: Bitmap queue to wake up.
*/
void sbitmap_queue_wake_all(struct sbitmap_queue *sbq);
/**
* sbitmap_queue_wake_up() - Wake up some of waiters in one waitqueue
* on a &struct sbitmap_queue.
* @sbq: Bitmap queue to wake up.
* @nr: Number of bits cleared.
*/
void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr);
/**
* sbitmap_queue_show() - Dump &struct sbitmap_queue information to a &struct
* seq_file.
* @sbq: Bitmap queue to show.
* @m: struct seq_file to write to.
*
* This is intended for debugging. The format may change at any time.
*/
void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m);
struct sbq_wait {
struct sbitmap_queue *sbq; /* if set, sbq_wait is accounted */
struct wait_queue_entry wait;
};
#define DEFINE_SBQ_WAIT(name) \
struct sbq_wait name = { \
.sbq = NULL, \
.wait = { \
.private = current, \
.func = autoremove_wake_function, \
.entry = LIST_HEAD_INIT((name).wait.entry), \
} \
}
/*
* Wrapper around prepare_to_wait_exclusive(), which maintains some extra
* internal state.
*/
void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
struct sbq_wait_state *ws,
struct sbq_wait *sbq_wait, int state);
/*
* Must be paired with sbitmap_prepare_to_wait().
*/
void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
struct sbq_wait *sbq_wait);
/*
* Wrapper around add_wait_queue(), which maintains some extra internal state
*/
void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
struct sbq_wait_state *ws,
struct sbq_wait *sbq_wait);
/*
* Must be paired with sbitmap_add_wait_queue()
*/
void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait);
#endif /* __LINUX_SCALE_BITMAP_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __INCLUDE_LINUX_OOM_H
#define __INCLUDE_LINUX_OOM_H
#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/nodemask.h>
#include <uapi/linux/oom.h>
#include <linux/sched/coredump.h> /* MMF_* */
#include <linux/mm.h> /* VM_FAULT* */
struct zonelist;
struct notifier_block;
struct mem_cgroup;
struct task_struct;
enum oom_constraint {
CONSTRAINT_NONE,
CONSTRAINT_CPUSET,
CONSTRAINT_MEMORY_POLICY,
CONSTRAINT_MEMCG,
};
/*
* Details of the page allocation that triggered the oom killer that are used to
* determine what should be killed.
*/
struct oom_control {
/* Used to determine cpuset */
struct zonelist *zonelist;
/* Used to determine mempolicy */
nodemask_t *nodemask;
/* Memory cgroup in which oom is invoked, or NULL for global oom */
struct mem_cgroup *memcg;
/* Used to determine cpuset and node locality requirement */
const gfp_t gfp_mask;
/*
* order == -1 means the oom kill is required by sysrq, otherwise only
* for display purposes.
*/
const int order;
/* Used by oom implementation, do not set */
unsigned long totalpages;
struct task_struct *chosen;
long chosen_points;
/* Used to print the constraint info. */
enum oom_constraint constraint;
};
extern struct mutex oom_lock;
extern struct mutex oom_adj_mutex;
static inline void set_current_oom_origin(void)
{
current->signal->oom_flag_origin = true;
}
static inline void clear_current_oom_origin(void)
{
current->signal->oom_flag_origin = false;
}
static inline bool oom_task_origin(const struct task_struct *p)
{
return p->signal->oom_flag_origin;
}
static inline bool tsk_is_oom_victim(struct task_struct * tsk)
{
return tsk->signal->oom_mm;
}
/*
* Checks whether a page fault on the given mm is still reliable.
* This is no longer true if the oom reaper started to reap the
* address space which is reflected by MMF_UNSTABLE flag set in
* the mm. At that moment any !shared mapping would lose the content
* and could cause a memory corruption (zero pages instead of the
* original content).
*
* User should call this before establishing a page table entry for
* a !shared mapping and under the proper page table lock.
*
* Return 0 when the PF is safe VM_FAULT_SIGBUS otherwise.
*/
static inline vm_fault_t check_stable_address_space(struct mm_struct *mm)
{
if (unlikely(test_bit(MMF_UNSTABLE, &mm->flags)))
return VM_FAULT_SIGBUS;
return 0;
}
long oom_badness(struct task_struct *p,
unsigned long totalpages);
extern bool out_of_memory(struct oom_control *oc);
extern void exit_oom_victim(void);
extern int register_oom_notifier(struct notifier_block *nb);
extern int unregister_oom_notifier(struct notifier_block *nb);
extern bool oom_killer_disable(signed long timeout);
extern void oom_killer_enable(void);
extern struct task_struct *find_lock_task_mm(struct task_struct *p);
#endif /* _INCLUDE_LINUX_OOM_H */
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* include/linux/idr.h
*
* 2002-10-18 written by Jim Houston jim.houston@ccur.com
* Copyright (C) 2002 by Concurrent Computer Corporation
*
* Small id to pointer translation service avoiding fixed sized
* tables.
*/
#ifndef __IDR_H__
#define __IDR_H__
#include <linux/radix-tree.h>
#include <linux/gfp.h>
#include <linux/percpu.h>
struct idr {
struct radix_tree_root idr_rt;
unsigned int idr_base;
unsigned int idr_next;
};
/*
* The IDR API does not expose the tagging functionality of the radix tree
* to users. Use tag 0 to track whether a node has free space below it.
*/
#define IDR_FREE 0
/* Set the IDR flag and the IDR_FREE tag */
#define IDR_RT_MARKER (ROOT_IS_IDR | (__force gfp_t) \
(1 << (ROOT_TAG_SHIFT + IDR_FREE)))
#define IDR_INIT_BASE(name, base) { \
.idr_rt = RADIX_TREE_INIT(name, IDR_RT_MARKER), \
.idr_base = (base), \
.idr_next = 0, \
}
/**
* IDR_INIT() - Initialise an IDR.
* @name: Name of IDR.
*
* A freshly-initialised IDR contains no IDs.
*/
#define IDR_INIT(name) IDR_INIT_BASE(name, 0)
/**
* DEFINE_IDR() - Define a statically-allocated IDR.
* @name: Name of IDR.
*
* An IDR defined using this macro is ready for use with no additional
* initialisation required. It contains no IDs.
*/
#define DEFINE_IDR(name) struct idr name = IDR_INIT(name)
/**
* idr_get_cursor - Return the current position of the cyclic allocator
* @idr: idr handle
*
* The value returned is the value that will be next returned from
* idr_alloc_cyclic() if it is free (otherwise the search will start from
* this position).
*/
static inline unsigned int idr_get_cursor(const struct idr *idr)
{
return READ_ONCE(idr->idr_next);
}
/**
* idr_set_cursor - Set the current position of the cyclic allocator
* @idr: idr handle
* @val: new position
*
* The next call to idr_alloc_cyclic() will return @val if it is free
* (otherwise the search will start from this position).
*/
static inline void idr_set_cursor(struct idr *idr, unsigned int val)
{
WRITE_ONCE(idr->idr_next, val);
}
/**
* DOC: idr sync
* idr synchronization (stolen from radix-tree.h)
*
* idr_find() is able to be called locklessly, using RCU. The caller must
* ensure calls to this function are made within rcu_read_lock() regions.
* Other readers (lock-free or otherwise) and modifications may be running
* concurrently.
*
* It is still required that the caller manage the synchronization and
* lifetimes of the items. So if RCU lock-free lookups are used, typically
* this would mean that the items have their own locks, or are amenable to
* lock-free access; and that the items are freed by RCU (or only freed after
* having been deleted from the idr tree *and* a synchronize_rcu() grace
* period).
*/
#define idr_lock(idr) xa_lock(&(idr)->idr_rt)
#define idr_unlock(idr) xa_unlock(&(idr)->idr_rt)
#define idr_lock_bh(idr) xa_lock_bh(&(idr)->idr_rt)
#define idr_unlock_bh(idr) xa_unlock_bh(&(idr)->idr_rt)
#define idr_lock_irq(idr) xa_lock_irq(&(idr)->idr_rt)
#define idr_unlock_irq(idr) xa_unlock_irq(&(idr)->idr_rt)
#define idr_lock_irqsave(idr, flags) \
xa_lock_irqsave(&(idr)->idr_rt, flags)
#define idr_unlock_irqrestore(idr, flags) \
xa_unlock_irqrestore(&(idr)->idr_rt, flags)
void idr_preload(gfp_t gfp_mask);
int idr_alloc(struct idr *, void *ptr, int start, int end, gfp_t);
int __must_check idr_alloc_u32(struct idr *, void *ptr, u32 *id,
unsigned long max, gfp_t);
int idr_alloc_cyclic(struct idr *, void *ptr, int start, int end, gfp_t);
void *idr_remove(struct idr *, unsigned long id);
void *idr_find(const struct idr *, unsigned long id);
int idr_for_each(const struct idr *,
int (*fn)(int id, void *p, void *data), void *data);
void *idr_get_next(struct idr *, int *nextid);
void *idr_get_next_ul(struct idr *, unsigned long *nextid);
void *idr_replace(struct idr *, void *, unsigned long id);
void idr_destroy(struct idr *);
/**
* idr_init_base() - Initialise an IDR.
* @idr: IDR handle.
* @base: The base value for the IDR.
*
* This variation of idr_init() creates an IDR which will allocate IDs
* starting at %base.
*/
static inline void idr_init_base(struct idr *idr, int base)
{
INIT_RADIX_TREE(&idr->idr_rt, IDR_RT_MARKER);
idr->idr_base = base;
idr->idr_next = 0;
}
/**
* idr_init() - Initialise an IDR.
* @idr: IDR handle.
*
* Initialise a dynamically allocated IDR. To initialise a
* statically allocated IDR, use DEFINE_IDR().
*/
static inline void idr_init(struct idr *idr)
{
idr_init_base(idr, 0);
}
/**
* idr_is_empty() - Are there any IDs allocated?
* @idr: IDR handle.
*
* Return: %true if any IDs have been allocated from this IDR.
*/
static inline bool idr_is_empty(const struct idr *idr)
{
return radix_tree_empty(&idr->idr_rt) &&
radix_tree_tagged(&idr->idr_rt, IDR_FREE);
}
/**
* idr_preload_end - end preload section started with idr_preload()
*
* Each idr_preload() should be matched with an invocation of this
* function. See idr_preload() for details.
*/
static inline void idr_preload_end(void)
{
local_unlock(&radix_tree_preloads.lock);
}
/**
* idr_for_each_entry() - Iterate over an IDR's elements of a given type.
* @idr: IDR handle.
* @entry: The type * to use as cursor
* @id: Entry ID.
*
* @entry and @id do not need to be initialized before the loop, and
* after normal termination @entry is left with the value NULL. This
* is convenient for a "not found" value.
*/
#define idr_for_each_entry(idr, entry, id) \
for (id = 0; ((entry) = idr_get_next(idr, &(id))) != NULL; id += 1U)
/**
* idr_for_each_entry_ul() - Iterate over an IDR's elements of a given type.
* @idr: IDR handle.
* @entry: The type * to use as cursor.
* @tmp: A temporary placeholder for ID.
* @id: Entry ID.
*
* @entry and @id do not need to be initialized before the loop, and
* after normal termination @entry is left with the value NULL. This
* is convenient for a "not found" value.
*/
#define idr_for_each_entry_ul(idr, entry, tmp, id) \
for (tmp = 0, id = 0; \
((entry) = tmp <= id ? idr_get_next_ul(idr, &(id)) : NULL) != NULL; \
tmp = id, ++id)
/**
* idr_for_each_entry_continue() - Continue iteration over an IDR's elements of a given type
* @idr: IDR handle.
* @entry: The type * to use as a cursor.
* @id: Entry ID.
*
* Continue to iterate over entries, continuing after the current position.
*/
#define idr_for_each_entry_continue(idr, entry, id) \
for ((entry) = idr_get_next((idr), &(id)); \
entry; \
++id, (entry) = idr_get_next((idr), &(id)))
/**
* idr_for_each_entry_continue_ul() - Continue iteration over an IDR's elements of a given type
* @idr: IDR handle.
* @entry: The type * to use as a cursor.
* @tmp: A temporary placeholder for ID.
* @id: Entry ID.
*
* Continue to iterate over entries, continuing after the current position.
* After normal termination @entry is left with the value NULL. This
* is convenient for a "not found" value.
*/
#define idr_for_each_entry_continue_ul(idr, entry, tmp, id) \
for (tmp = id; \
((entry) = tmp <= id ? idr_get_next_ul(idr, &(id)) : NULL) != NULL; \
tmp = id, ++id)
/*
* IDA - ID Allocator, use when translation from id to pointer isn't necessary.
*/
#define IDA_CHUNK_SIZE 128 /* 128 bytes per chunk */
#define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long))
#define IDA_BITMAP_BITS (IDA_BITMAP_LONGS * sizeof(long) * 8)
struct ida_bitmap {
unsigned long bitmap[IDA_BITMAP_LONGS];
};
struct ida {
struct xarray xa;
};
#define IDA_INIT_FLAGS (XA_FLAGS_LOCK_IRQ | XA_FLAGS_ALLOC)
#define IDA_INIT(name) { \
.xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \
}
#define DEFINE_IDA(name) struct ida name = IDA_INIT(name)
int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t);
void ida_free(struct ida *, unsigned int id);
void ida_destroy(struct ida *ida);
/**
* ida_alloc() - Allocate an unused ID.
* @ida: IDA handle.
* @gfp: Memory allocation flags.
*
* Allocate an ID between 0 and %INT_MAX, inclusive.
*
* Context: Any context. It is safe to call this function without
* locking in your code.
* Return: The allocated ID, or %-ENOMEM if memory could not be allocated,
* or %-ENOSPC if there are no free IDs.
*/
static inline int ida_alloc(struct ida *ida, gfp_t gfp)
{
return ida_alloc_range(ida, 0, ~0, gfp);
}
/**
* ida_alloc_min() - Allocate an unused ID.
* @ida: IDA handle.
* @min: Lowest ID to allocate.
* @gfp: Memory allocation flags.
*
* Allocate an ID between @min and %INT_MAX, inclusive.
*
* Context: Any context. It is safe to call this function without
* locking in your code.
* Return: The allocated ID, or %-ENOMEM if memory could not be allocated,
* or %-ENOSPC if there are no free IDs.
*/
static inline int ida_alloc_min(struct ida *ida, unsigned int min, gfp_t gfp)
{
return ida_alloc_range(ida, min, ~0, gfp);
}
/**
* ida_alloc_max() - Allocate an unused ID.
* @ida: IDA handle.
* @max: Highest ID to allocate.
* @gfp: Memory allocation flags.
*
* Allocate an ID between 0 and @max, inclusive.
*
* Context: Any context. It is safe to call this function without
* locking in your code.
* Return: The allocated ID, or %-ENOMEM if memory could not be allocated,
* or %-ENOSPC if there are no free IDs.
*/
static inline int ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp)
{
return ida_alloc_range(ida, 0, max, gfp);
}
static inline void ida_init(struct ida *ida)
{
xa_init_flags(&ida->xa, IDA_INIT_FLAGS);
}
/*
* ida_simple_get() and ida_simple_remove() are deprecated. Use
* ida_alloc() and ida_free() instead respectively.
*/
#define ida_simple_get(ida, start, end, gfp) \
ida_alloc_range(ida, start, (end) - 1, gfp)
#define ida_simple_remove(ida, id) ida_free(ida, id)
static inline bool ida_is_empty(const struct ida *ida)
{
return xa_empty(&ida->xa);
}
#endif /* __IDR_H__ */
// SPDX-License-Identifier: GPL-2.0
/*
* driver.c - centralized device driver management
*
* Copyright (c) 2002-3 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
* Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de>
* Copyright (c) 2007 Novell Inc.
*/
#include <linux/device/driver.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/sysfs.h>
#include "base.h"
static struct device *next_device(struct klist_iter *i)
{
struct klist_node *n = klist_next(i);
struct device *dev = NULL;
struct device_private *dev_prv;
if (n) {
dev_prv = to_device_private_driver(n);
dev = dev_prv->device;
}
return dev;
}
/**
* driver_set_override() - Helper to set or clear driver override.
* @dev: Device to change
* @override: Address of string to change (e.g. &device->driver_override);
* The contents will be freed and hold newly allocated override.
* @s: NUL-terminated string, new driver name to force a match, pass empty
* string to clear it ("" or "\n", where the latter is only for sysfs
* interface).
* @len: length of @s
*
* Helper to set or clear driver override in a device, intended for the cases
* when the driver_override field is allocated by driver/bus code.
*
* Returns: 0 on success or a negative error code on failure.
*/
int driver_set_override(struct device *dev, const char **override,
const char *s, size_t len)
{
const char *new, *old;
char *cp;
if (!override || !s)
return -EINVAL;
/*
* The stored value will be used in sysfs show callback (sysfs_emit()),
* which has a length limit of PAGE_SIZE and adds a trailing newline.
* Thus we can store one character less to avoid truncation during sysfs
* show.
*/
if (len >= (PAGE_SIZE - 1))
return -EINVAL;
/*
* Compute the real length of the string in case userspace sends us a
* bunch of \0 characters like python likes to do.
*/
len = strlen(s);
if (!len) {
/* Empty string passed - clear override */
device_lock(dev);
old = *override;
*override = NULL;
device_unlock(dev);
kfree(old);
return 0;
}
cp = strnchr(s, len, '\n');
if (cp)
len = cp - s;
new = kstrndup(s, len, GFP_KERNEL);
if (!new)
return -ENOMEM;
device_lock(dev);
old = *override;
if (cp != s) {
*override = new;
} else {
/* "\n" passed - clear override */
kfree(new);
*override = NULL;
}
device_unlock(dev);
kfree(old);
return 0;
}
EXPORT_SYMBOL_GPL(driver_set_override);
/**
* driver_for_each_device - Iterator for devices bound to a driver.
* @drv: Driver we're iterating.
* @start: Device to begin with
* @data: Data to pass to the callback.
* @fn: Function to call for each device.
*
* Iterate over the @drv's list of devices calling @fn for each one.
*/
int driver_for_each_device(struct device_driver *drv, struct device *start,
void *data, int (*fn)(struct device *, void *))
{
struct klist_iter i;
struct device *dev;
int error = 0;
if (!drv)
return -EINVAL;
klist_iter_init_node(&drv->p->klist_devices, &i,
start ? &start->p->knode_driver : NULL);
while (!error && (dev = next_device(&i)))
error = fn(dev, data);
klist_iter_exit(&i);
return error;
}
EXPORT_SYMBOL_GPL(driver_for_each_device);
/**
* driver_find_device - device iterator for locating a particular device.
* @drv: The device's driver
* @start: Device to begin with
* @data: Data to pass to match function
* @match: Callback function to check device
*
* This is similar to the driver_for_each_device() function above, but
* it returns a reference to a device that is 'found' for later use, as
* determined by the @match callback.
*
* The callback should return 0 if the device doesn't match and non-zero
* if it does. If the callback returns non-zero, this function will
* return to the caller and not iterate over any more devices.
*/
struct device *driver_find_device(struct device_driver *drv,
struct device *start, const void *data,
int (*match)(struct device *dev, const void *data))
{
struct klist_iter i;
struct device *dev;
if (!drv || !drv->p)
return NULL;
klist_iter_init_node(&drv->p->klist_devices, &i,
(start ? &start->p->knode_driver : NULL));
while ((dev = next_device(&i)))
if (match(dev, data) && get_device(dev))
break;
klist_iter_exit(&i);
return dev;
}
EXPORT_SYMBOL_GPL(driver_find_device);
/**
* driver_create_file - create sysfs file for driver.
* @drv: driver.
* @attr: driver attribute descriptor.
*/
int driver_create_file(struct device_driver *drv,
const struct driver_attribute *attr)
{
int error;
if (drv) error = sysfs_create_file(&drv->p->kobj, &attr->attr);
else
error = -EINVAL;
return error;
}
EXPORT_SYMBOL_GPL(driver_create_file);
/**
* driver_remove_file - remove sysfs file for driver.
* @drv: driver.
* @attr: driver attribute descriptor.
*/
void driver_remove_file(struct device_driver *drv,
const struct driver_attribute *attr)
{
if (drv)
sysfs_remove_file(&drv->p->kobj, &attr->attr);
}
EXPORT_SYMBOL_GPL(driver_remove_file);
int driver_add_groups(struct device_driver *drv,
const struct attribute_group **groups)
{
return sysfs_create_groups(&drv->p->kobj, groups);
}
void driver_remove_groups(struct device_driver *drv,
const struct attribute_group **groups)
{
sysfs_remove_groups(&drv->p->kobj, groups);
}
/**
* driver_register - register driver with bus
* @drv: driver to register
*
* We pass off most of the work to the bus_add_driver() call,
* since most of the things we have to do deal with the bus
* structures.
*/
int driver_register(struct device_driver *drv)
{
int ret;
struct device_driver *other;
if (!bus_is_registered(drv->bus)) { pr_err("Driver '%s' was unable to register with bus_type '%s' because the bus was not initialized.\n",
drv->name, drv->bus->name);
return -EINVAL;
}
if ((drv->bus->probe && drv->probe) || (drv->bus->remove && drv->remove) || (drv->bus->shutdown && drv->shutdown)) pr_warn("Driver '%s' needs updating - please use "
"bus_type methods\n", drv->name);
other = driver_find(drv->name, drv->bus);
if (other) {
pr_err("Error: Driver '%s' is already registered, "
"aborting...\n", drv->name);
return -EBUSY;
}
ret = bus_add_driver(drv); if (ret)
return ret;
ret = driver_add_groups(drv, drv->groups);
if (ret) {
bus_remove_driver(drv); return ret;
}
kobject_uevent(&drv->p->kobj, KOBJ_ADD);
deferred_probe_extend_timeout();
return ret;
}
EXPORT_SYMBOL_GPL(driver_register);
/**
* driver_unregister - remove driver from system.
* @drv: driver.
*
* Again, we pass off most of the work to the bus-level call.
*/
void driver_unregister(struct device_driver *drv)
{
if (!drv || !drv->p) { WARN(1, "Unexpected driver unregister!\n");
return;
}
driver_remove_groups(drv, drv->groups);
bus_remove_driver(drv);
}
EXPORT_SYMBOL_GPL(driver_unregister);
// SPDX-License-Identifier: GPL-2.0-only
/*
*
* Copyright (C) 2005 Mike Isely <isely@pobox.com>
*/
#include <linux/i2c.h>
#include <linux/module.h>
#include <media/i2c/ir-kbd-i2c.h>
#include "pvrusb2-i2c-core.h"
#include "pvrusb2-hdw-internal.h"
#include "pvrusb2-debug.h"
#include "pvrusb2-fx2-cmd.h"
#include "pvrusb2.h"
#define trace_i2c(...) pvr2_trace(PVR2_TRACE_I2C,__VA_ARGS__)
/*
This module attempts to implement a compliant I2C adapter for the pvrusb2
device.
*/
static unsigned int i2c_scan;
module_param(i2c_scan, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(i2c_scan,"scan i2c bus at insmod time");
static int ir_mode[PVR_NUM] = { [0 ... PVR_NUM-1] = 1 };
module_param_array(ir_mode, int, NULL, 0444);
MODULE_PARM_DESC(ir_mode,"specify: 0=disable IR reception, 1=normal IR");
static int pvr2_disable_ir_video;
module_param_named(disable_autoload_ir_video, pvr2_disable_ir_video,
int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(disable_autoload_ir_video,
"1=do not try to autoload ir_video IR receiver");
static int pvr2_i2c_write(struct pvr2_hdw *hdw, /* Context */
u8 i2c_addr, /* I2C address we're talking to */
u8 *data, /* Data to write */
u16 length) /* Size of data to write */
{
/* Return value - default 0 means success */
int ret;
if (!data) length = 0;
if (length > (sizeof(hdw->cmd_buffer) - 3)) {
pvr2_trace(PVR2_TRACE_ERROR_LEGS,
"Killing an I2C write to %u that is too large (desired=%u limit=%u)",
i2c_addr,
length,(unsigned int)(sizeof(hdw->cmd_buffer) - 3));
return -ENOTSUPP;
}
LOCK_TAKE(hdw->ctl_lock);
/* Clear the command buffer (likely to be paranoia) */
memset(hdw->cmd_buffer, 0, sizeof(hdw->cmd_buffer));
/* Set up command buffer for an I2C write */
hdw->cmd_buffer[0] = FX2CMD_I2C_WRITE; /* write prefix */
hdw->cmd_buffer[1] = i2c_addr; /* i2c addr of chip */
hdw->cmd_buffer[2] = length; /* length of what follows */
if (length) memcpy(hdw->cmd_buffer + 3, data, length);
/* Do the operation */
ret = pvr2_send_request(hdw,
hdw->cmd_buffer,
length + 3,
hdw->cmd_buffer,
1);
if (!ret) {
if (hdw->cmd_buffer[0] != 8) {
ret = -EIO;
if (hdw->cmd_buffer[0] != 7) {
trace_i2c("unexpected status from i2_write[%d]: %d",
i2c_addr,hdw->cmd_buffer[0]);
}
}
}
LOCK_GIVE(hdw->ctl_lock);
return ret;
}
static int pvr2_i2c_read(struct pvr2_hdw *hdw, /* Context */
u8 i2c_addr, /* I2C address we're talking to */
u8 *data, /* Data to write */
u16 dlen, /* Size of data to write */
u8 *res, /* Where to put data we read */
u16 rlen) /* Amount of data to read */
{
/* Return value - default 0 means success */
int ret;
if (!data) dlen = 0;
if (dlen > (sizeof(hdw->cmd_buffer) - 4)) {
pvr2_trace(PVR2_TRACE_ERROR_LEGS,
"Killing an I2C read to %u that has wlen too large (desired=%u limit=%u)",
i2c_addr,
dlen,(unsigned int)(sizeof(hdw->cmd_buffer) - 4));
return -ENOTSUPP;
}
if (res && (rlen > (sizeof(hdw->cmd_buffer) - 1))) {
pvr2_trace(PVR2_TRACE_ERROR_LEGS,
"Killing an I2C read to %u that has rlen too large (desired=%u limit=%u)",
i2c_addr,
rlen,(unsigned int)(sizeof(hdw->cmd_buffer) - 1));
return -ENOTSUPP;
}
LOCK_TAKE(hdw->ctl_lock);
/* Clear the command buffer (likely to be paranoia) */
memset(hdw->cmd_buffer, 0, sizeof(hdw->cmd_buffer));
/* Set up command buffer for an I2C write followed by a read */
hdw->cmd_buffer[0] = FX2CMD_I2C_READ; /* read prefix */
hdw->cmd_buffer[1] = dlen; /* arg length */
hdw->cmd_buffer[2] = rlen; /* answer length. Device will send one
more byte (status). */
hdw->cmd_buffer[3] = i2c_addr; /* i2c addr of chip */
if (dlen) memcpy(hdw->cmd_buffer + 4, data, dlen);
/* Do the operation */
ret = pvr2_send_request(hdw,
hdw->cmd_buffer,
4 + dlen,
hdw->cmd_buffer,
rlen + 1);
if (!ret) {
if (hdw->cmd_buffer[0] != 8) {
ret = -EIO;
if (hdw->cmd_buffer[0] != 7) {
trace_i2c("unexpected status from i2_read[%d]: %d",
i2c_addr,hdw->cmd_buffer[0]);
}
}
}
/* Copy back the result */
if (res && rlen) {
if (ret) {
/* Error, just blank out the return buffer */
memset(res, 0, rlen);
} else {
memcpy(res, hdw->cmd_buffer + 1, rlen);
}
}
LOCK_GIVE(hdw->ctl_lock);
return ret;
}
/* This is the common low level entry point for doing I2C operations to the
hardware. */
static int pvr2_i2c_basic_op(struct pvr2_hdw *hdw,
u8 i2c_addr,
u8 *wdata,
u16 wlen,
u8 *rdata,
u16 rlen)
{
if (!rdata) rlen = 0;
if (!wdata) wlen = 0;
if (rlen || !wlen) {
return pvr2_i2c_read(hdw,i2c_addr,wdata,wlen,rdata,rlen);
} else {
return pvr2_i2c_write(hdw,i2c_addr,wdata,wlen);
}
}
/* This is a special entry point for cases of I2C transaction attempts to
the IR receiver. The implementation here simulates the IR receiver by
issuing a command to the FX2 firmware and using that response to return
what the real I2C receiver would have returned. We use this for 24xxx
devices, where the IR receiver chip has been removed and replaced with
FX2 related logic. */
static int i2c_24xxx_ir(struct pvr2_hdw *hdw,
u8 i2c_addr,u8 *wdata,u16 wlen,u8 *rdata,u16 rlen)
{
u8 dat[4];
unsigned int stat;
if (!(rlen || wlen)) {
/* This is a probe attempt. Just let it succeed. */
return 0;
}
/* We don't understand this kind of transaction */
if ((wlen != 0) || (rlen == 0)) return -EIO;
if (rlen < 3) {
/* Mike Isely <isely@pobox.com> Appears to be a probe
attempt from lirc. Just fill in zeroes and return. If
we try instead to do the full transaction here, then bad
things seem to happen within the lirc driver module
(version 0.8.0-7 sources from Debian, when run under
vanilla 2.6.17.6 kernel) - and I don't have the patience
to chase it down. */
if (rlen > 0) rdata[0] = 0;
if (rlen > 1) rdata[1] = 0;
return 0;
}
/* Issue a command to the FX2 to read the IR receiver. */
LOCK_TAKE(hdw->ctl_lock); do {
hdw->cmd_buffer[0] = FX2CMD_GET_IR_CODE;
stat = pvr2_send_request(hdw,
hdw->cmd_buffer,1,
hdw->cmd_buffer,4);
dat[0] = hdw->cmd_buffer[0];
dat[1] = hdw->cmd_buffer[1];
dat[2] = hdw->cmd_buffer[2];
dat[3] = hdw->cmd_buffer[3];
} while (0); LOCK_GIVE(hdw->ctl_lock);
/* Give up if that operation failed. */
if (stat != 0) return stat;
/* Mangle the results into something that looks like the real IR
receiver. */
rdata[2] = 0xc1;
if (dat[0] != 1) {
/* No code received. */
rdata[0] = 0;
rdata[1] = 0;
} else {
u16 val;
/* Mash the FX2 firmware-provided IR code into something
that the normal i2c chip-level driver expects. */
val = dat[1];
val <<= 8;
val |= dat[2];
val >>= 1;
val &= ~0x0003;
val |= 0x8000;
rdata[0] = (val >> 8) & 0xffu;
rdata[1] = val & 0xffu;
}
return 0;
}
/* This is a special entry point that is entered if an I2C operation is
attempted to a wm8775 chip on model 24xxx hardware. Autodetect of this
part doesn't work, but we know it is really there. So let's look for
the autodetect attempt and just return success if we see that. */
static int i2c_hack_wm8775(struct pvr2_hdw *hdw,
u8 i2c_addr,u8 *wdata,u16 wlen,u8 *rdata,u16 rlen)
{
if (!(rlen || wlen)) {
// This is a probe attempt. Just let it succeed.
return 0;
}
return pvr2_i2c_basic_op(hdw,i2c_addr,wdata,wlen,rdata,rlen);
}
/* This is an entry point designed to always fail any attempt to perform a
transfer. We use this to cause certain I2C addresses to not be
probed. */
static int i2c_black_hole(struct pvr2_hdw *hdw,
u8 i2c_addr,u8 *wdata,u16 wlen,u8 *rdata,u16 rlen)
{
return -EIO;
}
/* This is a special entry point that is entered if an I2C operation is
attempted to a cx25840 chip on model 24xxx hardware. This chip can
sometimes wedge itself. Worse still, when this happens msp3400 can
falsely detect this part and then the system gets hosed up after msp3400
gets confused and dies. What we want to do here is try to keep msp3400
away and also try to notice if the chip is wedged and send a warning to
the system log. */
static int i2c_hack_cx25840(struct pvr2_hdw *hdw,
u8 i2c_addr,u8 *wdata,u16 wlen,u8 *rdata,u16 rlen)
{
int ret;
unsigned int subaddr;
u8 wbuf[2];
int state = hdw->i2c_cx25840_hack_state;
if (!(rlen || wlen)) {
// Probe attempt - always just succeed and don't bother the
// hardware (this helps to make the state machine further
// down somewhat easier).
return 0;
}
if (state == 3) {
return pvr2_i2c_basic_op(hdw,i2c_addr,wdata,wlen,rdata,rlen);
}
/* We're looking for the exact pattern where the revision register
is being read. The cx25840 module will always look at the
revision register first. Any other pattern of access therefore
has to be a probe attempt from somebody else so we'll reject it.
Normally we could just let each client just probe the part
anyway, but when the cx25840 is wedged, msp3400 will get a false
positive and that just screws things up... */
if (wlen == 0) {
switch (state) {
case 1: subaddr = 0x0100; break;
case 2: subaddr = 0x0101; break;
default: goto fail;
}
} else if (wlen == 2) {
subaddr = (wdata[0] << 8) | wdata[1];
switch (subaddr) {
case 0x0100: state = 1; break;
case 0x0101: state = 2; break;
default: goto fail;
}
} else {
goto fail;
}
if (!rlen) goto success;
state = 0;
if (rlen != 1) goto fail;
/* If we get to here then we have a legitimate read for one of the
two revision bytes, so pass it through. */
wbuf[0] = subaddr >> 8;
wbuf[1] = subaddr;
ret = pvr2_i2c_basic_op(hdw,i2c_addr,wbuf,2,rdata,rlen);
if ((ret != 0) || (*rdata == 0x04) || (*rdata == 0x0a)) {
pvr2_trace(PVR2_TRACE_ERROR_LEGS,
"***WARNING*** Detected a wedged cx25840 chip; the device will not work.");
pvr2_trace(PVR2_TRACE_ERROR_LEGS,
"***WARNING*** Try power cycling the pvrusb2 device.");
pvr2_trace(PVR2_TRACE_ERROR_LEGS,
"***WARNING*** Disabling further access to the device to prevent other foul-ups.");
// This blocks all further communication with the part.
hdw->i2c_func[0x44] = NULL;
pvr2_hdw_render_useless(hdw);
goto fail;
}
/* Success! */
pvr2_trace(PVR2_TRACE_CHIPS,"cx25840 appears to be OK.");
state = 3;
success:
hdw->i2c_cx25840_hack_state = state;
return 0;
fail:
hdw->i2c_cx25840_hack_state = state;
return -EIO;
}
/* This is a very, very limited I2C adapter implementation. We can only
support what we actually know will work on the device... */
static int pvr2_i2c_xfer(struct i2c_adapter *i2c_adap,
struct i2c_msg msgs[],
int num)
{
int ret = -ENOTSUPP;
pvr2_i2c_func funcp = NULL;
struct pvr2_hdw *hdw = (struct pvr2_hdw *)(i2c_adap->algo_data);
if (!num) {
ret = -EINVAL;
goto done;
}
if (msgs[0].addr < PVR2_I2C_FUNC_CNT) { funcp = hdw->i2c_func[msgs[0].addr];
}
if (!funcp) {
ret = -EIO;
goto done;
}
if (num == 1) { if (msgs[0].flags & I2C_M_RD) {
/* Simple read */
u16 tcnt,bcnt,offs;
if (!msgs[0].len) {
/* Length == 0 read. This is a probe. */
if (funcp(hdw,msgs[0].addr,NULL,0,NULL,0)) {
ret = -EIO;
goto done;
}
ret = 1;
goto done;
}
/* If the read is short enough we'll do the whole
thing atomically. Otherwise we have no choice
but to break apart the reads. */
tcnt = msgs[0].len;
offs = 0;
while (tcnt) {
bcnt = tcnt;
if (bcnt > sizeof(hdw->cmd_buffer)-1) {
bcnt = sizeof(hdw->cmd_buffer)-1;
}
if (funcp(hdw,msgs[0].addr,NULL,0,
msgs[0].buf+offs,bcnt)) {
ret = -EIO;
goto done;
}
offs += bcnt;
tcnt -= bcnt;
}
ret = 1;
goto done;
} else {
/* Simple write */
ret = 1;
if (funcp(hdw,msgs[0].addr,
msgs[0].buf,msgs[0].len,NULL,0)) {
ret = -EIO;
}
goto done;
}
} else if (num == 2) { if (msgs[0].addr != msgs[1].addr) { trace_i2c("i2c refusing 2 phase transfer with conflicting target addresses");
ret = -ENOTSUPP;
goto done;
}
if ((!((msgs[0].flags & I2C_M_RD))) && (msgs[1].flags & I2C_M_RD)) {
u16 tcnt,bcnt,wcnt,offs;
/* Write followed by atomic read. If the read
portion is short enough we'll do the whole thing
atomically. Otherwise we have no choice but to
break apart the reads. */
tcnt = msgs[1].len;
wcnt = msgs[0].len;
offs = 0;
while (tcnt || wcnt) {
bcnt = tcnt;
if (bcnt > sizeof(hdw->cmd_buffer)-1) {
bcnt = sizeof(hdw->cmd_buffer)-1;
}
if (funcp(hdw,msgs[0].addr,
msgs[0].buf,wcnt,
msgs[1].buf+offs,bcnt)) {
ret = -EIO;
goto done;
}
offs += bcnt;
tcnt -= bcnt;
wcnt = 0;
}
ret = 2;
goto done;
} else {
trace_i2c("i2c refusing complex transfer read0=%d read1=%d",
(msgs[0].flags & I2C_M_RD),
(msgs[1].flags & I2C_M_RD));
}
} else {
trace_i2c("i2c refusing %d phase transfer",num);
}
done:
if (pvrusb2_debug & PVR2_TRACE_I2C_TRAF) {
unsigned int idx,offs,cnt;
for (idx = 0; idx < num; idx++) { cnt = msgs[idx].len; pr_info("pvrusb2 i2c xfer %u/%u: addr=0x%x len=%d %s",
idx+1,num,
msgs[idx].addr,
cnt,
(msgs[idx].flags & I2C_M_RD ?
"read" : "write"));
if ((ret > 0) || !(msgs[idx].flags & I2C_M_RD)) { if (cnt > 8) cnt = 8;
pr_cont(" [");
for (offs = 0; offs < cnt; offs++) {
if (offs) pr_cont(" "); pr_cont("%02x", msgs[idx].buf[offs]);
}
if (offs < cnt) pr_cont(" ...");
pr_cont("]");
}
if (idx+1 == num) { pr_cont(" result=%d", ret);
}
pr_cont("\n");
}
if (!num) {
pr_info("pvrusb2 i2c xfer null transfer result=%d\n",
ret);
}
}
return ret;
}
static u32 pvr2_i2c_functionality(struct i2c_adapter *adap)
{
return I2C_FUNC_SMBUS_EMUL | I2C_FUNC_I2C;
}
static const struct i2c_algorithm pvr2_i2c_algo_template = {
.master_xfer = pvr2_i2c_xfer,
.functionality = pvr2_i2c_functionality,
};
static const struct i2c_adapter pvr2_i2c_adap_template = {
.owner = THIS_MODULE,
.class = 0,
};
/* Return true if device exists at given address */
static int do_i2c_probe(struct pvr2_hdw *hdw, int addr)
{
struct i2c_msg msg[1];
int rc;
msg[0].addr = 0;
msg[0].flags = I2C_M_RD;
msg[0].len = 0;
msg[0].buf = NULL;
msg[0].addr = addr;
rc = i2c_transfer(&hdw->i2c_adap, msg, ARRAY_SIZE(msg));
return rc == 1;
}
static void do_i2c_scan(struct pvr2_hdw *hdw)
{
int i;
pr_info("%s: i2c scan beginning\n", hdw->name);
for (i = 0; i < 128; i++) {
if (do_i2c_probe(hdw, i)) {
pr_info("%s: i2c scan: found device @ 0x%x\n",
hdw->name, i);
}
}
pr_info("%s: i2c scan done.\n", hdw->name);
}
static void pvr2_i2c_register_ir(struct pvr2_hdw *hdw)
{
struct i2c_board_info info;
struct IR_i2c_init_data *init_data = &hdw->ir_init_data;
if (pvr2_disable_ir_video) {
pvr2_trace(PVR2_TRACE_INFO,
"Automatic binding of ir_video has been disabled.");
return;
}
memset(&info, 0, sizeof(struct i2c_board_info));
switch (hdw->ir_scheme_active) {
case PVR2_IR_SCHEME_24XXX: /* FX2-controlled IR */
case PVR2_IR_SCHEME_29XXX: /* Original 29xxx device */
init_data->ir_codes = RC_MAP_HAUPPAUGE;
init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP;
init_data->type = RC_PROTO_BIT_RC5;
init_data->name = hdw->hdw_desc->description;
init_data->polling_interval = 100; /* ms From ir-kbd-i2c */
/* IR Receiver */
info.addr = 0x18;
info.platform_data = init_data;
strscpy(info.type, "ir_video", I2C_NAME_SIZE);
pvr2_trace(PVR2_TRACE_INFO, "Binding %s to i2c address 0x%02x.",
info.type, info.addr);
i2c_new_client_device(&hdw->i2c_adap, &info);
break;
case PVR2_IR_SCHEME_ZILOG: /* HVR-1950 style */
case PVR2_IR_SCHEME_24XXX_MCE: /* 24xxx MCE device */
init_data->ir_codes = RC_MAP_HAUPPAUGE;
init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
RC_PROTO_BIT_RC6_6A_32;
init_data->name = hdw->hdw_desc->description;
/* IR Transceiver */
info.addr = 0x71;
info.platform_data = init_data;
strscpy(info.type, "ir_z8f0811_haup", I2C_NAME_SIZE);
pvr2_trace(PVR2_TRACE_INFO, "Binding %s to i2c address 0x%02x.",
info.type, info.addr);
i2c_new_client_device(&hdw->i2c_adap, &info);
break;
default:
/* The device either doesn't support I2C-based IR or we
don't know (yet) how to operate IR on the device. */
break;
}
}
void pvr2_i2c_core_init(struct pvr2_hdw *hdw)
{
unsigned int idx;
/* The default action for all possible I2C addresses is just to do
the transfer normally. */
for (idx = 0; idx < PVR2_I2C_FUNC_CNT; idx++) {
hdw->i2c_func[idx] = pvr2_i2c_basic_op;
}
/* However, deal with various special cases for 24xxx hardware. */
if (ir_mode[hdw->unit_number] == 0) {
pr_info("%s: IR disabled\n", hdw->name);
hdw->i2c_func[0x18] = i2c_black_hole;
} else if (ir_mode[hdw->unit_number] == 1) {
if (hdw->ir_scheme_active == PVR2_IR_SCHEME_24XXX) {
/* Set up translation so that our IR looks like a
29xxx device */
hdw->i2c_func[0x18] = i2c_24xxx_ir;
}
}
if (hdw->hdw_desc->flag_has_cx25840) {
hdw->i2c_func[0x44] = i2c_hack_cx25840;
}
if (hdw->hdw_desc->flag_has_wm8775) {
hdw->i2c_func[0x1b] = i2c_hack_wm8775;
}
// Configure the adapter and set up everything else related to it.
hdw->i2c_adap = pvr2_i2c_adap_template;
hdw->i2c_algo = pvr2_i2c_algo_template;
strscpy(hdw->i2c_adap.name, hdw->name, sizeof(hdw->i2c_adap.name));
hdw->i2c_adap.dev.parent = &hdw->usb_dev->dev;
hdw->i2c_adap.algo = &hdw->i2c_algo;
hdw->i2c_adap.algo_data = hdw;
hdw->i2c_linked = !0;
i2c_set_adapdata(&hdw->i2c_adap, &hdw->v4l2_dev);
i2c_add_adapter(&hdw->i2c_adap);
if (hdw->i2c_func[0x18] == i2c_24xxx_ir) {
/* Probe for a different type of IR receiver on this
device. This is really the only way to differentiate
older 24xxx devices from 24xxx variants that include an
IR blaster. If the IR blaster is present, the IR
receiver is part of that chip and thus we must disable
the emulated IR receiver. */
if (do_i2c_probe(hdw, 0x71)) {
pvr2_trace(PVR2_TRACE_INFO,
"Device has newer IR hardware; disabling unneeded virtual IR device");
hdw->i2c_func[0x18] = NULL;
/* Remember that this is a different device... */
hdw->ir_scheme_active = PVR2_IR_SCHEME_24XXX_MCE;
}
}
if (i2c_scan) do_i2c_scan(hdw);
pvr2_i2c_register_ir(hdw);
}
void pvr2_i2c_core_done(struct pvr2_hdw *hdw)
{
if (hdw->i2c_linked) { i2c_del_adapter(&hdw->i2c_adap);
hdw->i2c_linked = 0;
}
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_WORD_AT_A_TIME_H
#define _ASM_WORD_AT_A_TIME_H
#include <linux/bitops.h>
#include <linux/wordpart.h>
/*
* This is largely generic for little-endian machines, but the
* optimal byte mask counting is probably going to be something
* that is architecture-specific. If you have a reliably fast
* bit count instruction, that might be better than the multiply
* and shift, for example.
*/
struct word_at_a_time {
const unsigned long one_bits, high_bits;
};
#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
#ifdef CONFIG_64BIT
/*
* Jan Achrenius on G+: microoptimized version of
* the simpler "(mask & ONEBYTES) * ONEBYTES >> 56"
* that works for the bytemasks without having to
* mask them first.
*/
static inline long count_masked_bytes(unsigned long mask)
{
return mask*0x0001020304050608ul >> 56;
}
#else /* 32-bit case */
/* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
static inline long count_masked_bytes(long mask)
{
/* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
long a = (0x0ff0001+mask) >> 23;
/* Fix the 1 for 00 case */
return a & mask;
}
#endif
/* Return nonzero if it has a zero */
static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
{
unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
*bits = mask;
return mask;
}
static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
{
return bits;
}
static inline unsigned long create_zero_mask(unsigned long bits)
{
bits = (bits - 1) & ~bits;
return bits >> 7;
}
/* The mask we created is directly usable as a bytemask */
#define zero_bytemask(mask) (mask)
static inline unsigned long find_zero(unsigned long mask)
{
return count_masked_bytes(mask);
}
/*
* Load an unaligned word from kernel space.
*
* In the (very unlikely) case of the word being a page-crosser
* and the next page not being mapped, take the exception and
* return zeroes in the non-existing part.
*/
static inline unsigned long load_unaligned_zeropad(const void *addr)
{
unsigned long ret;
asm volatile(
"1: mov %[mem], %[ret]\n"
"2:\n"
_ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_ZEROPAD)
: [ret] "=r" (ret)
: [mem] "m" (*(unsigned long *)addr));
return ret;
}
#endif /* _ASM_WORD_AT_A_TIME_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* mm/mmap.c
*
* Written by obz.
*
* Address space accounting code <alan@lxorguk.ukuu.org.uk>
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/backing-dev.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/syscalls.h>
#include <linux/capability.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/hugetlb.h>
#include <linux/shmem_fs.h>
#include <linux/profile.h>
#include <linux/export.h>
#include <linux/mount.h>
#include <linux/mempolicy.h>
#include <linux/rmap.h>
#include <linux/mmu_notifier.h>
#include <linux/mmdebug.h>
#include <linux/perf_event.h>
#include <linux/audit.h>
#include <linux/khugepaged.h>
#include <linux/uprobes.h>
#include <linux/notifier.h>
#include <linux/memory.h>
#include <linux/printk.h>
#include <linux/userfaultfd_k.h>
#include <linux/moduleparam.h>
#include <linux/pkeys.h>
#include <linux/oom.h>
#include <linux/sched/mm.h>
#include <linux/ksm.h>
#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>
#include <asm/mmu_context.h>
#define CREATE_TRACE_POINTS
#include <trace/events/mmap.h>
#include "internal.h"
#ifndef arch_mmap_check
#define arch_mmap_check(addr, len, flags) (0)
#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX;
int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
#endif
static bool ignore_rlimit_data;
core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
struct vm_area_struct *vma, struct vm_area_struct *prev,
struct vm_area_struct *next, unsigned long start,
unsigned long end, unsigned long tree_end, bool mm_wr_locked);
static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
{
return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
}
/* Update vma->vm_page_prot to reflect vma->vm_flags. */
void vma_set_page_prot(struct vm_area_struct *vma)
{
unsigned long vm_flags = vma->vm_flags;
pgprot_t vm_page_prot;
vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
if (vma_wants_writenotify(vma, vm_page_prot)) {
vm_flags &= ~VM_SHARED;
vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
}
/* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */
WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
}
/*
* Requires inode->i_mapping->i_mmap_rwsem
*/
static void __remove_shared_vm_struct(struct vm_area_struct *vma,
struct address_space *mapping)
{
if (vma_is_shared_maywrite(vma))
mapping_unmap_writable(mapping);
flush_dcache_mmap_lock(mapping);
vma_interval_tree_remove(vma, &mapping->i_mmap);
flush_dcache_mmap_unlock(mapping);
}
/*
* Unlink a file-based vm structure from its interval tree, to hide
* vma from rmap and vmtruncate before freeing its page tables.
*/
void unlink_file_vma(struct vm_area_struct *vma)
{
struct file *file = vma->vm_file;
if (file) {
struct address_space *mapping = file->f_mapping;
i_mmap_lock_write(mapping);
__remove_shared_vm_struct(vma, mapping);
i_mmap_unlock_write(mapping);
}
}
/*
* Close a vm structure and free it.
*/
static void remove_vma(struct vm_area_struct *vma, bool unreachable)
{
might_sleep();
if (vma->vm_ops && vma->vm_ops->close)
vma->vm_ops->close(vma);
if (vma->vm_file)
fput(vma->vm_file);
mpol_put(vma_policy(vma));
if (unreachable)
__vm_area_free(vma);
else
vm_area_free(vma);
}
static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
unsigned long min)
{
return mas_prev(&vmi->mas, min);
}
/*
* check_brk_limits() - Use platform specific check of range & verify mlock
* limits.
* @addr: The address to check
* @len: The size of increase.
*
* Return: 0 on success.
*/
static int check_brk_limits(unsigned long addr, unsigned long len)
{
unsigned long mapped_addr;
mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
if (IS_ERR_VALUE(mapped_addr))
return mapped_addr;
return mlock_future_ok(current->mm, current->mm->def_flags, len)
? 0 : -EAGAIN;
}
static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
unsigned long addr, unsigned long request, unsigned long flags);
SYSCALL_DEFINE1(brk, unsigned long, brk)
{
unsigned long newbrk, oldbrk, origbrk;
struct mm_struct *mm = current->mm;
struct vm_area_struct *brkvma, *next = NULL;
unsigned long min_brk;
bool populate = false;
LIST_HEAD(uf);
struct vma_iterator vmi;
if (mmap_write_lock_killable(mm))
return -EINTR;
origbrk = mm->brk;
#ifdef CONFIG_COMPAT_BRK
/*
* CONFIG_COMPAT_BRK can still be overridden by setting
* randomize_va_space to 2, which will still cause mm->start_brk
* to be arbitrarily shifted
*/
if (current->brk_randomized)
min_brk = mm->start_brk;
else
min_brk = mm->end_data;
#else
min_brk = mm->start_brk;
#endif
if (brk < min_brk)
goto out;
/*
* Check against rlimit here. If this check is done later after the test
* of oldbrk with newbrk then it can escape the test and let the data
* segment grow beyond its set limit the in case where the limit is
* not page aligned -Ram Gupta
*/
if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
mm->end_data, mm->start_data))
goto out;
newbrk = PAGE_ALIGN(brk);
oldbrk = PAGE_ALIGN(mm->brk);
if (oldbrk == newbrk) {
mm->brk = brk;
goto success;
}
/* Always allow shrinking brk. */
if (brk <= mm->brk) {
/* Search one past newbrk */
vma_iter_init(&vmi, mm, newbrk);
brkvma = vma_find(&vmi, oldbrk);
if (!brkvma || brkvma->vm_start >= oldbrk)
goto out; /* mapping intersects with an existing non-brk vma. */
/*
* mm->brk must be protected by write mmap_lock.
* do_vma_munmap() will drop the lock on success, so update it
* before calling do_vma_munmap().
*/
mm->brk = brk;
if (do_vma_munmap(&vmi, brkvma, newbrk, oldbrk, &uf, true))
goto out;
goto success_unlocked;
}
if (check_brk_limits(oldbrk, newbrk - oldbrk))
goto out;
/*
* Only check if the next VMA is within the stack_guard_gap of the
* expansion area
*/
vma_iter_init(&vmi, mm, oldbrk);
next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap);
if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
goto out;
brkvma = vma_prev_limit(&vmi, mm->start_brk);
/* Ok, looks good - let it rip. */
if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
goto out;
mm->brk = brk;
if (mm->def_flags & VM_LOCKED)
populate = true;
success:
mmap_write_unlock(mm);
success_unlocked:
userfaultfd_unmap_complete(mm, &uf);
if (populate)
mm_populate(oldbrk, newbrk - oldbrk);
return brk;
out:
mm->brk = origbrk;
mmap_write_unlock(mm);
return origbrk;
}
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
static void validate_mm(struct mm_struct *mm)
{
int bug = 0;
int i = 0;
struct vm_area_struct *vma;
VMA_ITERATOR(vmi, mm, 0);
mt_validate(&mm->mm_mt);
for_each_vma(vmi, vma) {
#ifdef CONFIG_DEBUG_VM_RB
struct anon_vma *anon_vma = vma->anon_vma;
struct anon_vma_chain *avc;
#endif
unsigned long vmi_start, vmi_end;
bool warn = 0;
vmi_start = vma_iter_addr(&vmi);
vmi_end = vma_iter_end(&vmi);
if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
warn = 1;
if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
warn = 1;
if (warn) { pr_emerg("issue in %s\n", current->comm);
dump_stack();
dump_vma(vma);
pr_emerg("tree range: %px start %lx end %lx\n", vma,
vmi_start, vmi_end - 1);
vma_iter_dump_tree(&vmi);
}
#ifdef CONFIG_DEBUG_VM_RB
if (anon_vma) { anon_vma_lock_read(anon_vma);
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
anon_vma_interval_tree_verify(avc); anon_vma_unlock_read(anon_vma);
}
#endif
i++;
}
if (i != mm->map_count) { pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
bug = 1;
}
VM_BUG_ON_MM(bug, mm);
}
#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */
#define validate_mm(mm) do { } while (0)
#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
/*
* vma has some anon_vma assigned, and is already inserted on that
* anon_vma's interval trees.
*
* Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
* vma must be removed from the anon_vma's interval trees using
* anon_vma_interval_tree_pre_update_vma().
*
* After the update, the vma will be reinserted using
* anon_vma_interval_tree_post_update_vma().
*
* The entire update must be protected by exclusive mmap_lock and by
* the root anon_vma's mutex.
*/
static inline void
anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
{
struct anon_vma_chain *avc;
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
}
static inline void
anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
{
struct anon_vma_chain *avc;
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
}
static unsigned long count_vma_pages_range(struct mm_struct *mm,
unsigned long addr, unsigned long end)
{
VMA_ITERATOR(vmi, mm, addr);
struct vm_area_struct *vma;
unsigned long nr_pages = 0;
for_each_vma_range(vmi, vma, end) {
unsigned long vm_start = max(addr, vma->vm_start);
unsigned long vm_end = min(end, vma->vm_end);
nr_pages += PHYS_PFN(vm_end - vm_start);
}
return nr_pages;
}
static void __vma_link_file(struct vm_area_struct *vma,
struct address_space *mapping)
{
if (vma_is_shared_maywrite(vma))
mapping_allow_writable(mapping);
flush_dcache_mmap_lock(mapping);
vma_interval_tree_insert(vma, &mapping->i_mmap);
flush_dcache_mmap_unlock(mapping);
}
static void vma_link_file(struct vm_area_struct *vma)
{
struct file *file = vma->vm_file;
struct address_space *mapping;
if (file) {
mapping = file->f_mapping;
i_mmap_lock_write(mapping);
__vma_link_file(vma, mapping);
i_mmap_unlock_write(mapping);
}
}
static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
{
VMA_ITERATOR(vmi, mm, 0);
vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
if (vma_iter_prealloc(&vmi, vma))
return -ENOMEM;
vma_start_write(vma);
vma_iter_store(&vmi, vma);
vma_link_file(vma);
mm->map_count++;
validate_mm(mm);
return 0;
}
/*
* init_multi_vma_prep() - Initializer for struct vma_prepare
* @vp: The vma_prepare struct
* @vma: The vma that will be altered once locked
* @next: The next vma if it is to be adjusted
* @remove: The first vma to be removed
* @remove2: The second vma to be removed
*/
static inline void init_multi_vma_prep(struct vma_prepare *vp,
struct vm_area_struct *vma, struct vm_area_struct *next,
struct vm_area_struct *remove, struct vm_area_struct *remove2)
{
memset(vp, 0, sizeof(struct vma_prepare));
vp->vma = vma;
vp->anon_vma = vma->anon_vma;
vp->remove = remove;
vp->remove2 = remove2;
vp->adj_next = next;
if (!vp->anon_vma && next)
vp->anon_vma = next->anon_vma;
vp->file = vma->vm_file;
if (vp->file)
vp->mapping = vma->vm_file->f_mapping;
}
/*
* init_vma_prep() - Initializer wrapper for vma_prepare struct
* @vp: The vma_prepare struct
* @vma: The vma that will be altered once locked
*/
static inline void init_vma_prep(struct vma_prepare *vp,
struct vm_area_struct *vma)
{
init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
}
/*
* vma_prepare() - Helper function for handling locking VMAs prior to altering
* @vp: The initialized vma_prepare struct
*/
static inline void vma_prepare(struct vma_prepare *vp)
{
if (vp->file) {
uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
if (vp->adj_next)
uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
vp->adj_next->vm_end);
i_mmap_lock_write(vp->mapping);
if (vp->insert && vp->insert->vm_file) {
/*
* Put into interval tree now, so instantiated pages
* are visible to arm/parisc __flush_dcache_page
* throughout; but we cannot insert into address
* space until vma start or end is updated.
*/
__vma_link_file(vp->insert,
vp->insert->vm_file->f_mapping);
}
}
if (vp->anon_vma) {
anon_vma_lock_write(vp->anon_vma);
anon_vma_interval_tree_pre_update_vma(vp->vma);
if (vp->adj_next)
anon_vma_interval_tree_pre_update_vma(vp->adj_next);
}
if (vp->file) {
flush_dcache_mmap_lock(vp->mapping);
vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
if (vp->adj_next)
vma_interval_tree_remove(vp->adj_next,
&vp->mapping->i_mmap);
}
}
/*
* vma_complete- Helper function for handling the unlocking after altering VMAs,
* or for inserting a VMA.
*
* @vp: The vma_prepare struct
* @vmi: The vma iterator
* @mm: The mm_struct
*/
static inline void vma_complete(struct vma_prepare *vp,
struct vma_iterator *vmi, struct mm_struct *mm)
{
if (vp->file) {
if (vp->adj_next)
vma_interval_tree_insert(vp->adj_next,
&vp->mapping->i_mmap);
vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
flush_dcache_mmap_unlock(vp->mapping);
}
if (vp->remove && vp->file) {
__remove_shared_vm_struct(vp->remove, vp->mapping);
if (vp->remove2)
__remove_shared_vm_struct(vp->remove2, vp->mapping);
} else if (vp->insert) {
/*
* split_vma has split insert from vma, and needs
* us to insert it before dropping the locks
* (it may either follow vma or precede it).
*/
vma_iter_store(vmi, vp->insert);
mm->map_count++;
}
if (vp->anon_vma) {
anon_vma_interval_tree_post_update_vma(vp->vma);
if (vp->adj_next)
anon_vma_interval_tree_post_update_vma(vp->adj_next);
anon_vma_unlock_write(vp->anon_vma);
}
if (vp->file) {
i_mmap_unlock_write(vp->mapping);
uprobe_mmap(vp->vma);
if (vp->adj_next)
uprobe_mmap(vp->adj_next);
}
if (vp->remove) {
again:
vma_mark_detached(vp->remove, true);
if (vp->file) {
uprobe_munmap(vp->remove, vp->remove->vm_start,
vp->remove->vm_end);
fput(vp->file);
}
if (vp->remove->anon_vma)
anon_vma_merge(vp->vma, vp->remove);
mm->map_count--;
mpol_put(vma_policy(vp->remove));
if (!vp->remove2)
WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
vm_area_free(vp->remove);
/*
* In mprotect's case 6 (see comments on vma_merge),
* we are removing both mid and next vmas
*/
if (vp->remove2) {
vp->remove = vp->remove2;
vp->remove2 = NULL;
goto again;
}
}
if (vp->insert && vp->file)
uprobe_mmap(vp->insert);
validate_mm(mm);
}
/*
* dup_anon_vma() - Helper function to duplicate anon_vma
* @dst: The destination VMA
* @src: The source VMA
* @dup: Pointer to the destination VMA when successful.
*
* Returns: 0 on success.
*/
static inline int dup_anon_vma(struct vm_area_struct *dst,
struct vm_area_struct *src, struct vm_area_struct **dup)
{
/*
* Easily overlooked: when mprotect shifts the boundary, make sure the
* expanding vma has anon_vma set if the shrinking vma had, to cover any
* anon pages imported.
*/
if (src->anon_vma && !dst->anon_vma) {
int ret;
vma_assert_write_locked(dst);
dst->anon_vma = src->anon_vma;
ret = anon_vma_clone(dst, src);
if (ret)
return ret;
*dup = dst;
}
return 0;
}
/*
* vma_expand - Expand an existing VMA
*
* @vmi: The vma iterator
* @vma: The vma to expand
* @start: The start of the vma
* @end: The exclusive end of the vma
* @pgoff: The page offset of vma
* @next: The current of next vma.
*
* Expand @vma to @start and @end. Can expand off the start and end. Will
* expand over @next if it's different from @vma and @end == @next->vm_end.
* Checking if the @vma can expand and merge with @next needs to be handled by
* the caller.
*
* Returns: 0 on success
*/
int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long start, unsigned long end, pgoff_t pgoff,
struct vm_area_struct *next)
{
struct vm_area_struct *anon_dup = NULL;
bool remove_next = false;
struct vma_prepare vp;
vma_start_write(vma);
if (next && (vma != next) && (end == next->vm_end)) {
int ret;
remove_next = true;
vma_start_write(next);
ret = dup_anon_vma(vma, next, &anon_dup);
if (ret)
return ret;
}
init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL);
/* Not merging but overwriting any part of next is not handled. */
VM_WARN_ON(next && !vp.remove &&
next != vma && end > next->vm_start);
/* Only handles expanding */
VM_WARN_ON(vma->vm_start < start || vma->vm_end > end);
/* Note: vma iterator must be pointing to 'start' */
vma_iter_config(vmi, start, end);
if (vma_iter_prealloc(vmi, vma))
goto nomem;
vma_prepare(&vp);
vma_adjust_trans_huge(vma, start, end, 0);
vma_set_range(vma, start, end, pgoff);
vma_iter_store(vmi, vma);
vma_complete(&vp, vmi, vma->vm_mm);
return 0;
nomem:
if (anon_dup)
unlink_anon_vmas(anon_dup);
return -ENOMEM;
}
/*
* vma_shrink() - Reduce an existing VMAs memory area
* @vmi: The vma iterator
* @vma: The VMA to modify
* @start: The new start
* @end: The new end
*
* Returns: 0 on success, -ENOMEM otherwise
*/
int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long start, unsigned long end, pgoff_t pgoff)
{
struct vma_prepare vp;
WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
if (vma->vm_start < start)
vma_iter_config(vmi, vma->vm_start, start);
else
vma_iter_config(vmi, end, vma->vm_end);
if (vma_iter_prealloc(vmi, NULL))
return -ENOMEM;
vma_start_write(vma);
init_vma_prep(&vp, vma);
vma_prepare(&vp);
vma_adjust_trans_huge(vma, start, end, 0);
vma_iter_clear(vmi);
vma_set_range(vma, start, end, pgoff);
vma_complete(&vp, vmi, vma->vm_mm);
return 0;
}
/*
* If the vma has a ->close operation then the driver probably needs to release
* per-vma resources, so we don't attempt to merge those if the caller indicates
* the current vma may be removed as part of the merge.
*/
static inline bool is_mergeable_vma(struct vm_area_struct *vma,
struct file *file, unsigned long vm_flags,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name, bool may_remove_vma)
{
/*
* VM_SOFTDIRTY should not prevent from VMA merging, if we
* match the flags but dirty bit -- the caller should mark
* merged VMA as dirty. If dirty bit won't be excluded from
* comparison, we increase pressure on the memory system forcing
* the kernel to generate new VMAs when old one could be
* extended instead.
*/
if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
return false;
if (vma->vm_file != file)
return false;
if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
return false;
if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
return false;
if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
return false;
return true;
}
static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
struct anon_vma *anon_vma2, struct vm_area_struct *vma)
{
/*
* The list_is_singular() test is to avoid merging VMA cloned from
* parents. This can improve scalability caused by anon_vma lock.
*/
if ((!anon_vma1 || !anon_vma2) && (!vma ||
list_is_singular(&vma->anon_vma_chain)))
return true;
return anon_vma1 == anon_vma2;
}
/*
* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
* in front of (at a lower virtual address and file offset than) the vma.
*
* We cannot merge two vmas if they have differently assigned (non-NULL)
* anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
*
* We don't check here for the merged mmap wrapping around the end of pagecache
* indices (16TB on ia32) because do_mmap() does not permit mmap's which
* wrap, nor mmaps which cover the final page at index -1UL.
*
* We assume the vma may be removed as part of the merge.
*/
static bool
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file,
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name)
{
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
if (vma->vm_pgoff == vm_pgoff)
return true;
}
return false;
}
/*
* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
* beyond (at a higher virtual address and file offset than) the vma.
*
* We cannot merge two vmas if they have differently assigned (non-NULL)
* anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
*
* We assume that vma is not removed as part of the merge.
*/
static bool
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file,
pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name)
{
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
pgoff_t vm_pglen;
vm_pglen = vma_pages(vma);
if (vma->vm_pgoff + vm_pglen == vm_pgoff)
return true;
}
return false;
}
/*
* Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
* figure out whether that can be merged with its predecessor or its
* successor. Or both (it neatly fills a hole).
*
* In most cases - when called for mmap, brk or mremap - [addr,end) is
* certain not to be mapped by the time vma_merge is called; but when
* called for mprotect, it is certain to be already mapped (either at
* an offset within prev, or at the start of next), and the flags of
* this area are about to be changed to vm_flags - and the no-change
* case has already been eliminated.
*
* The following mprotect cases have to be considered, where **** is
* the area passed down from mprotect_fixup, never extending beyond one
* vma, PPPP is the previous vma, CCCC is a concurrent vma that starts
* at the same address as **** and is of the same or larger span, and
* NNNN the next vma after ****:
*
* **** **** ****
* PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPCCCCCC
* cannot merge might become might become
* PPNNNNNNNNNN PPPPPPPPPPCC
* mmap, brk or case 4 below case 5 below
* mremap move:
* **** ****
* PPPP NNNN PPPPCCCCNNNN
* might become might become
* PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or
* PPPPPPPPNNNN 2 or PPPPPPPPNNNN 7 or
* PPPPNNNNNNNN 3 PPPPNNNNNNNN 8
*
* It is important for case 8 that the vma CCCC overlapping the
* region **** is never going to extended over NNNN. Instead NNNN must
* be extended in region **** and CCCC must be removed. This way in
* all cases where vma_merge succeeds, the moment vma_merge drops the
* rmap_locks, the properties of the merged vma will be already
* correct for the whole merged range. Some of those properties like
* vm_page_prot/vm_flags may be accessed by rmap_walks and they must
* be correct for the whole merged range immediately after the
* rmap_locks are released. Otherwise if NNNN would be removed and
* CCCC would be extended over the NNNN range, remove_migration_ptes
* or other rmap walkers (if working on addresses beyond the "end"
* parameter) may establish ptes with the wrong permissions of CCCC
* instead of the right permissions of NNNN.
*
* In the code below:
* PPPP is represented by *prev
* CCCC is represented by *curr or not represented at all (NULL)
* NNNN is represented by *next or not represented at all (NULL)
* **** is not represented - it will be merged and the vma containing the
* area is returned, or the function will return NULL
*/
static struct vm_area_struct
*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
struct vm_area_struct *src, unsigned long addr, unsigned long end,
unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
struct anon_vma_name *anon_name)
{
struct mm_struct *mm = src->vm_mm;
struct anon_vma *anon_vma = src->anon_vma;
struct file *file = src->vm_file;
struct vm_area_struct *curr, *next, *res;
struct vm_area_struct *vma, *adjust, *remove, *remove2;
struct vm_area_struct *anon_dup = NULL;
struct vma_prepare vp;
pgoff_t vma_pgoff;
int err = 0;
bool merge_prev = false;
bool merge_next = false;
bool vma_expanded = false;
unsigned long vma_start = addr;
unsigned long vma_end = end;
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
long adj_start = 0;
/*
* We later require that vma->vm_flags == vm_flags,
* so this tests vma->vm_flags & VM_SPECIAL, too.
*/
if (vm_flags & VM_SPECIAL)
return NULL;
/* Does the input range span an existing VMA? (cases 5 - 8) */
curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end);
if (!curr || /* cases 1 - 4 */
end == curr->vm_end) /* cases 6 - 8, adjacent VMA */
next = vma_lookup(mm, end);
else
next = NULL; /* case 5 */
if (prev) {
vma_start = prev->vm_start;
vma_pgoff = prev->vm_pgoff;
/* Can we merge the predecessor? */
if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)
&& can_vma_merge_after(prev, vm_flags, anon_vma, file,
pgoff, vm_userfaultfd_ctx, anon_name)) {
merge_prev = true;
vma_prev(vmi);
}
}
/* Can we merge the successor? */
if (next && mpol_equal(policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,
vm_userfaultfd_ctx, anon_name)) {
merge_next = true;
}
/* Verify some invariant that must be enforced by the caller. */
VM_WARN_ON(prev && addr <= prev->vm_start);
VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end));
VM_WARN_ON(addr >= end);
if (!merge_prev && !merge_next)
return NULL; /* Not mergeable. */
if (merge_prev)
vma_start_write(prev);
res = vma = prev;
remove = remove2 = adjust = NULL;
/* Can we merge both the predecessor and the successor? */
if (merge_prev && merge_next &&
is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) {
vma_start_write(next);
remove = next; /* case 1 */
vma_end = next->vm_end;
err = dup_anon_vma(prev, next, &anon_dup);
if (curr) { /* case 6 */
vma_start_write(curr);
remove = curr;
remove2 = next;
/*
* Note that the dup_anon_vma below cannot overwrite err
* since the first caller would do nothing unless next
* has an anon_vma.
*/
if (!next->anon_vma)
err = dup_anon_vma(prev, curr, &anon_dup);
}
} else if (merge_prev) { /* case 2 */
if (curr) {
vma_start_write(curr);
if (end == curr->vm_end) { /* case 7 */
/*
* can_vma_merge_after() assumed we would not be
* removing prev vma, so it skipped the check
* for vm_ops->close, but we are removing curr
*/
if (curr->vm_ops && curr->vm_ops->close)
err = -EINVAL;
remove = curr;
} else { /* case 5 */
adjust = curr;
adj_start = (end - curr->vm_start);
}
if (!err)
err = dup_anon_vma(prev, curr, &anon_dup);
}
} else { /* merge_next */
vma_start_write(next);
res = next;
if (prev && addr < prev->vm_end) { /* case 4 */
vma_start_write(prev);
vma_end = addr;
adjust = next;
adj_start = -(prev->vm_end - addr);
err = dup_anon_vma(next, prev, &anon_dup);
} else {
/*
* Note that cases 3 and 8 are the ONLY ones where prev
* is permitted to be (but is not necessarily) NULL.
*/
vma = next; /* case 3 */
vma_start = addr;
vma_end = next->vm_end;
vma_pgoff = next->vm_pgoff - pglen;
if (curr) { /* case 8 */
vma_pgoff = curr->vm_pgoff;
vma_start_write(curr);
remove = curr;
err = dup_anon_vma(next, curr, &anon_dup);
}
}
}
/* Error in anon_vma clone. */
if (err)
goto anon_vma_fail;
if (vma_start < vma->vm_start || vma_end > vma->vm_end)
vma_expanded = true;
if (vma_expanded) {
vma_iter_config(vmi, vma_start, vma_end);
} else {
vma_iter_config(vmi, adjust->vm_start + adj_start,
adjust->vm_end);
}
if (vma_iter_prealloc(vmi, vma))
goto prealloc_fail;
init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
vp.anon_vma != adjust->anon_vma);
vma_prepare(&vp);
vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start);
vma_set_range(vma, vma_start, vma_end, vma_pgoff);
if (vma_expanded)
vma_iter_store(vmi, vma);
if (adj_start) {
adjust->vm_start += adj_start;
adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
if (adj_start < 0) {
WARN_ON(vma_expanded);
vma_iter_store(vmi, next);
}
}
vma_complete(&vp, vmi, mm);
khugepaged_enter_vma(res, vm_flags);
return res;
prealloc_fail:
if (anon_dup)
unlink_anon_vmas(anon_dup);
anon_vma_fail:
vma_iter_set(vmi, addr);
vma_iter_load(vmi);
return NULL;
}
/*
* Rough compatibility check to quickly see if it's even worth looking
* at sharing an anon_vma.
*
* They need to have the same vm_file, and the flags can only differ
* in things that mprotect may change.
*
* NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
* we can merge the two vma's. For example, we refuse to merge a vma if
* there is a vm_ops->close() function, because that indicates that the
* driver is doing some kind of reference counting. But that doesn't
* really matter for the anon_vma sharing case.
*/
static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
{
return a->vm_end == b->vm_start &&
mpol_equal(vma_policy(a), vma_policy(b)) &&
a->vm_file == b->vm_file &&
!((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
}
/*
* Do some basic sanity checking to see if we can re-use the anon_vma
* from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
* the same as 'old', the other will be the new one that is trying
* to share the anon_vma.
*
* NOTE! This runs with mmap_lock held for reading, so it is possible that
* the anon_vma of 'old' is concurrently in the process of being set up
* by another page fault trying to merge _that_. But that's ok: if it
* is being set up, that automatically means that it will be a singleton
* acceptable for merging, so we can do all of this optimistically. But
* we do that READ_ONCE() to make sure that we never re-load the pointer.
*
* IOW: that the "list_is_singular()" test on the anon_vma_chain only
* matters for the 'stable anon_vma' case (ie the thing we want to avoid
* is to return an anon_vma that is "complex" due to having gone through
* a fork).
*
* We also make sure that the two vma's are compatible (adjacent,
* and with the same memory policies). That's all stable, even with just
* a read lock on the mmap_lock.
*/
static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
{
if (anon_vma_compatible(a, b)) {
struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
if (anon_vma && list_is_singular(&old->anon_vma_chain))
return anon_vma;
}
return NULL;
}
/*
* find_mergeable_anon_vma is used by anon_vma_prepare, to check
* neighbouring vmas for a suitable anon_vma, before it goes off
* to allocate a new anon_vma. It checks because a repetitive
* sequence of mprotects and faults may otherwise lead to distinct
* anon_vmas being allocated, preventing vma merge in subsequent
* mprotect.
*/
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
{
MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end);
struct anon_vma *anon_vma = NULL;
struct vm_area_struct *prev, *next;
/* Try next first. */
next = mas_walk(&mas);
if (next) {
anon_vma = reusable_anon_vma(next, vma, next);
if (anon_vma)
return anon_vma;
}
prev = mas_prev(&mas, 0);
VM_BUG_ON_VMA(prev != vma, vma);
prev = mas_prev(&mas, 0);
/* Try prev next. */
if (prev)
anon_vma = reusable_anon_vma(prev, prev, vma);
/*
* We might reach here with anon_vma == NULL if we can't find
* any reusable anon_vma.
* There's no absolute need to look only at touching neighbours:
* we could search further afield for "compatible" anon_vmas.
* But it would probably just be a waste of time searching,
* or lead to too many vmas hanging off the same anon_vma.
* We're trying to allow mprotect remerging later on,
* not trying to minimize memory used for anon_vmas.
*/
return anon_vma;
}
/*
* If a hint addr is less than mmap_min_addr change hint to be as
* low as possible but still greater than mmap_min_addr
*/
static inline unsigned long round_hint_to_min(unsigned long hint)
{
hint &= PAGE_MASK;
if (((void *)hint != NULL) &&
(hint < mmap_min_addr))
return PAGE_ALIGN(mmap_min_addr);
return hint;
}
bool mlock_future_ok(struct mm_struct *mm, unsigned long flags,
unsigned long bytes)
{
unsigned long locked_pages, limit_pages;
if (!(flags & VM_LOCKED) || capable(CAP_IPC_LOCK))
return true;
locked_pages = bytes >> PAGE_SHIFT;
locked_pages += mm->locked_vm;
limit_pages = rlimit(RLIMIT_MEMLOCK);
limit_pages >>= PAGE_SHIFT;
return locked_pages <= limit_pages;
}
static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
{
if (S_ISREG(inode->i_mode))
return MAX_LFS_FILESIZE;
if (S_ISBLK(inode->i_mode))
return MAX_LFS_FILESIZE;
if (S_ISSOCK(inode->i_mode))
return MAX_LFS_FILESIZE;
/* Special "we do even unsigned file positions" case */
if (file->f_mode & FMODE_UNSIGNED_OFFSET)
return 0;
/* Yes, random drivers might want more. But I'm tired of buggy drivers */
return ULONG_MAX;
}
static inline bool file_mmap_ok(struct file *file, struct inode *inode,
unsigned long pgoff, unsigned long len)
{
u64 maxsize = file_mmap_size_max(file, inode);
if (maxsize && len > maxsize)
return false;
maxsize -= len;
if (pgoff > maxsize >> PAGE_SHIFT)
return false;
return true;
}
/*
* The caller must write-lock current->mm->mmap_lock.
*/
unsigned long do_mmap(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flags, vm_flags_t vm_flags,
unsigned long pgoff, unsigned long *populate,
struct list_head *uf)
{
struct mm_struct *mm = current->mm;
int pkey = 0;
*populate = 0;
if (!len)
return -EINVAL;
/*
* Does the application expect PROT_READ to imply PROT_EXEC?
*
* (the exception is when the underlying filesystem is noexec
* mounted, in which case we don't add PROT_EXEC.)
*/
if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
if (!(file && path_noexec(&file->f_path)))
prot |= PROT_EXEC;
/* force arch specific MAP_FIXED handling in get_unmapped_area */
if (flags & MAP_FIXED_NOREPLACE)
flags |= MAP_FIXED;
if (!(flags & MAP_FIXED))
addr = round_hint_to_min(addr);
/* Careful about overflows.. */
len = PAGE_ALIGN(len);
if (!len)
return -ENOMEM;
/* offset overflow? */
if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
return -EOVERFLOW;
/* Too many mappings? */
if (mm->map_count > sysctl_max_map_count)
return -ENOMEM;
/* Obtain the address to map to. we verify (or select) it and ensure
* that it represents a valid section of the address space.
*/
addr = get_unmapped_area(file, addr, len, pgoff, flags);
if (IS_ERR_VALUE(addr))
return addr;
if (flags & MAP_FIXED_NOREPLACE) {
if (find_vma_intersection(mm, addr, addr + len))
return -EEXIST;
}
if (prot == PROT_EXEC) {
pkey = execute_only_pkey(mm);
if (pkey < 0)
pkey = 0;
}
/* Do simple checking here so the lower-level routines won't have
* to. we assume access permissions have been handled by the open
* of the memory object, so we don't do any here.
*/
vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
if (flags & MAP_LOCKED)
if (!can_do_mlock())
return -EPERM;
if (!mlock_future_ok(mm, vm_flags, len))
return -EAGAIN;
if (file) {
struct inode *inode = file_inode(file);
unsigned long flags_mask;
if (!file_mmap_ok(file, inode, pgoff, len))
return -EOVERFLOW;
flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags;
switch (flags & MAP_TYPE) {
case MAP_SHARED:
/*
* Force use of MAP_SHARED_VALIDATE with non-legacy
* flags. E.g. MAP_SYNC is dangerous to use with
* MAP_SHARED as you don't know which consistency model
* you will get. We silently ignore unsupported flags
* with MAP_SHARED to preserve backward compatibility.
*/
flags &= LEGACY_MAP_MASK;
fallthrough;
case MAP_SHARED_VALIDATE:
if (flags & ~flags_mask)
return -EOPNOTSUPP;
if (prot & PROT_WRITE) {
if (!(file->f_mode & FMODE_WRITE))
return -EACCES;
if (IS_SWAPFILE(file->f_mapping->host))
return -ETXTBSY;
}
/*
* Make sure we don't allow writing to an append-only
* file..
*/
if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
return -EACCES;
vm_flags |= VM_SHARED | VM_MAYSHARE;
if (!(file->f_mode & FMODE_WRITE))
vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
fallthrough;
case MAP_PRIVATE:
if (!(file->f_mode & FMODE_READ))
return -EACCES;
if (path_noexec(&file->f_path)) {
if (vm_flags & VM_EXEC)
return -EPERM;
vm_flags &= ~VM_MAYEXEC;
}
if (!file->f_op->mmap)
return -ENODEV;
if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
return -EINVAL;
break;
default:
return -EINVAL;
}
} else {
switch (flags & MAP_TYPE) {
case MAP_SHARED:
if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
return -EINVAL;
/*
* Ignore pgoff.
*/
pgoff = 0;
vm_flags |= VM_SHARED | VM_MAYSHARE;
break;
case MAP_PRIVATE:
/*
* Set pgoff according to addr for anon_vma.
*/
pgoff = addr >> PAGE_SHIFT;
break;
default:
return -EINVAL;
}
}
/*
* Set 'VM_NORESERVE' if we should not account for the
* memory use of this mapping.
*/
if (flags & MAP_NORESERVE) {
/* We honor MAP_NORESERVE if allowed to overcommit */
if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
vm_flags |= VM_NORESERVE;
/* hugetlb applies strict overcommit unless MAP_NORESERVE */
if (file && is_file_hugepages(file))
vm_flags |= VM_NORESERVE;
}
addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
if (!IS_ERR_VALUE(addr) &&
((vm_flags & VM_LOCKED) ||
(flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
*populate = len;
return addr;
}
unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
unsigned long prot, unsigned long flags,
unsigned long fd, unsigned long pgoff)
{
struct file *file = NULL;
unsigned long retval;
if (!(flags & MAP_ANONYMOUS)) {
audit_mmap_fd(fd, flags);
file = fget(fd);
if (!file)
return -EBADF;
if (is_file_hugepages(file)) {
len = ALIGN(len, huge_page_size(hstate_file(file)));
} else if (unlikely(flags & MAP_HUGETLB)) {
retval = -EINVAL;
goto out_fput;
}
} else if (flags & MAP_HUGETLB) {
struct hstate *hs;
hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
if (!hs)
return -EINVAL;
len = ALIGN(len, huge_page_size(hs));
/*
* VM_NORESERVE is used because the reservations will be
* taken when vm_ops->mmap() is called
*/
file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
VM_NORESERVE,
HUGETLB_ANONHUGE_INODE,
(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
if (IS_ERR(file))
return PTR_ERR(file);
}
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
out_fput:
if (file)
fput(file);
return retval;
}
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
unsigned long, prot, unsigned long, flags,
unsigned long, fd, unsigned long, pgoff)
{
return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
}
#ifdef __ARCH_WANT_SYS_OLD_MMAP
struct mmap_arg_struct {
unsigned long addr;
unsigned long len;
unsigned long prot;
unsigned long flags;
unsigned long fd;
unsigned long offset;
};
SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
{
struct mmap_arg_struct a;
if (copy_from_user(&a, arg, sizeof(a)))
return -EFAULT;
if (offset_in_page(a.offset))
return -EINVAL;
return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
a.offset >> PAGE_SHIFT);
}
#endif /* __ARCH_WANT_SYS_OLD_MMAP */
static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
{
return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
}
static bool vma_is_shared_writable(struct vm_area_struct *vma)
{
return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
(VM_WRITE | VM_SHARED);
}
static bool vma_fs_can_writeback(struct vm_area_struct *vma)
{
/* No managed pages to writeback. */
if (vma->vm_flags & VM_PFNMAP)
return false;
return vma->vm_file && vma->vm_file->f_mapping &&
mapping_can_writeback(vma->vm_file->f_mapping);
}
/*
* Does this VMA require the underlying folios to have their dirty state
* tracked?
*/
bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
{
/* Only shared, writable VMAs require dirty tracking. */
if (!vma_is_shared_writable(vma))
return false;
/* Does the filesystem need to be notified? */
if (vm_ops_needs_writenotify(vma->vm_ops))
return true;
/*
* Even if the filesystem doesn't indicate a need for writenotify, if it
* can writeback, dirty tracking is still required.
*/
return vma_fs_can_writeback(vma);
}
/*
* Some shared mappings will want the pages marked read-only
* to track write events. If so, we'll downgrade vm_page_prot
* to the private version (using protection_map[] without the
* VM_SHARED bit).
*/
int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
{
/* If it was private or non-writable, the write bit is already clear */
if (!vma_is_shared_writable(vma))
return 0;
/* The backer wishes to know when pages are first written to? */
if (vm_ops_needs_writenotify(vma->vm_ops))
return 1;
/* The open routine did something to the protections that pgprot_modify
* won't preserve? */
if (pgprot_val(vm_page_prot) !=
pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
return 0;
/*
* Do we need to track softdirty? hugetlb does not support softdirty
* tracking yet.
*/
if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
return 1;
/* Do we need write faults for uffd-wp tracking? */
if (userfaultfd_wp(vma))
return 1;
/* Can the mapping track the dirty pages? */
return vma_fs_can_writeback(vma);
}
/*
* We account for memory if it's a private writeable mapping,
* not hugepages and VM_NORESERVE wasn't set.
*/
static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
{
/*
* hugetlb has its own accounting separate from the core VM
* VM_HUGETLB may not be set yet so we cannot check for that flag.
*/
if (file && is_file_hugepages(file))
return 0;
return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
}
/**
* unmapped_area() - Find an area between the low_limit and the high_limit with
* the correct alignment and offset, all from @info. Note: current->mm is used
* for the search.
*
* @info: The unmapped area information including the range [low_limit -
* high_limit), the alignment offset and mask.
*
* Return: A memory address or -ENOMEM.
*/
static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
{
unsigned long length, gap;
unsigned long low_limit, high_limit;
struct vm_area_struct *tmp;
MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
/* Adjust search length to account for worst case alignment overhead */
length = info->length + info->align_mask;
if (length < info->length)
return -ENOMEM;
low_limit = info->low_limit;
if (low_limit < mmap_min_addr)
low_limit = mmap_min_addr;
high_limit = info->high_limit;
retry:
if (mas_empty_area(&mas, low_limit, high_limit - 1, length))
return -ENOMEM;
gap = mas.index;
gap += (info->align_offset - gap) & info->align_mask;
tmp = mas_next(&mas, ULONG_MAX);
if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
if (vm_start_gap(tmp) < gap + length - 1) {
low_limit = tmp->vm_end;
mas_reset(&mas);
goto retry;
}
} else {
tmp = mas_prev(&mas, 0);
if (tmp && vm_end_gap(tmp) > gap) {
low_limit = vm_end_gap(tmp);
mas_reset(&mas);
goto retry;
}
}
return gap;
}
/**
* unmapped_area_topdown() - Find an area between the low_limit and the
* high_limit with the correct alignment and offset at the highest available
* address, all from @info. Note: current->mm is used for the search.
*
* @info: The unmapped area information including the range [low_limit -
* high_limit), the alignment offset and mask.
*
* Return: A memory address or -ENOMEM.
*/
static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
{
unsigned long length, gap, gap_end;
unsigned long low_limit, high_limit;
struct vm_area_struct *tmp;
MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
/* Adjust search length to account for worst case alignment overhead */
length = info->length + info->align_mask;
if (length < info->length)
return -ENOMEM;
low_limit = info->low_limit;
if (low_limit < mmap_min_addr)
low_limit = mmap_min_addr;
high_limit = info->high_limit;
retry:
if (mas_empty_area_rev(&mas, low_limit, high_limit - 1, length))
return -ENOMEM;
gap = mas.last + 1 - info->length;
gap -= (gap - info->align_offset) & info->align_mask;
gap_end = mas.last;
tmp = mas_next(&mas, ULONG_MAX);
if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
if (vm_start_gap(tmp) <= gap_end) {
high_limit = vm_start_gap(tmp);
mas_reset(&mas);
goto retry;
}
} else {
tmp = mas_prev(&mas, 0);
if (tmp && vm_end_gap(tmp) > gap) {
high_limit = tmp->vm_start;
mas_reset(&mas);
goto retry;
}
}
return gap;
}
/*
* Search for an unmapped address range.
*
* We are looking for a range that:
* - does not intersect with any VMA;
* - is contained within the [low_limit, high_limit) interval;
* - is at least the desired size.
* - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
*/
unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
{
unsigned long addr;
if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
addr = unmapped_area_topdown(info);
else
addr = unmapped_area(info);
trace_vm_unmapped_area(addr, info);
return addr;
}
/* Get an address range which is currently unmapped.
* For shmat() with addr=0.
*
* Ugly calling convention alert:
* Return value with the low bits set means error value,
* ie
* if (ret & ~PAGE_MASK)
* error = ret;
*
* This function "knows" that -ENOMEM has the bits set.
*/
unsigned long
generic_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma, *prev;
struct vm_unmapped_area_info info;
const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
if (len > mmap_end - mmap_min_addr)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma_prev(mm, addr, &prev);
if (mmap_end - len >= addr && addr >= mmap_min_addr &&
(!vma || addr + len <= vm_start_gap(vma)) &&
(!prev || addr >= vm_end_gap(prev)))
return addr;
}
info.flags = 0;
info.length = len;
info.low_limit = mm->mmap_base;
info.high_limit = mmap_end;
info.align_mask = 0;
info.align_offset = 0;
return vm_unmapped_area(&info);
}
#ifndef HAVE_ARCH_UNMAPPED_AREA
unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags)
{
return generic_get_unmapped_area(filp, addr, len, pgoff, flags);
}
#endif
/*
* This mmap-allocator allocates new areas top-down from below the
* stack's low limit (the base):
*/
unsigned long
generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags)
{
struct vm_area_struct *vma, *prev;
struct mm_struct *mm = current->mm;
struct vm_unmapped_area_info info;
const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
/* requested length too big for entire address space */
if (len > mmap_end - mmap_min_addr)
return -ENOMEM;
if (flags & MAP_FIXED)
return addr;
/* requesting a specific address */
if (addr) {
addr = PAGE_ALIGN(addr);
vma = find_vma_prev(mm, addr, &prev);
if (mmap_end - len >= addr && addr >= mmap_min_addr &&
(!vma || addr + len <= vm_start_gap(vma)) &&
(!prev || addr >= vm_end_gap(prev)))
return addr;
}
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
info.length = len;
info.low_limit = PAGE_SIZE;
info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
info.align_mask = 0;
info.align_offset = 0;
addr = vm_unmapped_area(&info);
/*
* A failed mmap() very likely causes application failure,
* so fall back to the bottom-up function here. This scenario
* can happen with large stack limits and large mmap()
* allocations.
*/
if (offset_in_page(addr)) {
VM_BUG_ON(addr != -ENOMEM);
info.flags = 0;
info.low_limit = TASK_UNMAPPED_BASE;
info.high_limit = mmap_end;
addr = vm_unmapped_area(&info);
}
return addr;
}
#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
unsigned long
arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags)
{
return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags);
}
#endif
unsigned long
get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
unsigned long (*get_area)(struct file *, unsigned long,
unsigned long, unsigned long, unsigned long);
unsigned long error = arch_mmap_check(addr, len, flags);
if (error)
return error;
/* Careful about overflows.. */
if (len > TASK_SIZE)
return -ENOMEM;
get_area = current->mm->get_unmapped_area;
if (file) {
if (file->f_op->get_unmapped_area)
get_area = file->f_op->get_unmapped_area;
} else if (flags & MAP_SHARED) {
/*
* mmap_region() will call shmem_zero_setup() to create a file,
* so use shmem's get_unmapped_area in case it can be huge.
*/
get_area = shmem_get_unmapped_area;
} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
/* Ensures that larger anonymous mappings are THP aligned. */
get_area = thp_get_unmapped_area;
}
/* Always treat pgoff as zero for anonymous memory. */
if (!file)
pgoff = 0;
addr = get_area(file, addr, len, pgoff, flags);
if (IS_ERR_VALUE(addr))
return addr;
if (addr > TASK_SIZE - len)
return -ENOMEM;
if (offset_in_page(addr))
return -EINVAL;
error = security_mmap_addr(addr);
return error ? error : addr;
}
EXPORT_SYMBOL(get_unmapped_area);
/**
* find_vma_intersection() - Look up the first VMA which intersects the interval
* @mm: The process address space.
* @start_addr: The inclusive start user address.
* @end_addr: The exclusive end user address.
*
* Returns: The first VMA within the provided range, %NULL otherwise. Assumes
* start_addr < end_addr.
*/
struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
unsigned long start_addr,
unsigned long end_addr)
{
unsigned long index = start_addr;
mmap_assert_locked(mm);
return mt_find(&mm->mm_mt, &index, end_addr - 1);
}
EXPORT_SYMBOL(find_vma_intersection);
/**
* find_vma() - Find the VMA for a given address, or the next VMA.
* @mm: The mm_struct to check
* @addr: The address
*
* Returns: The VMA associated with addr, or the next VMA.
* May return %NULL in the case of no VMA at addr or above.
*/
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
unsigned long index = addr; mmap_assert_locked(mm); return mt_find(&mm->mm_mt, &index, ULONG_MAX);
}
EXPORT_SYMBOL(find_vma);
/**
* find_vma_prev() - Find the VMA for a given address, or the next vma and
* set %pprev to the previous VMA, if any.
* @mm: The mm_struct to check
* @addr: The address
* @pprev: The pointer to set to the previous VMA
*
* Note that RCU lock is missing here since the external mmap_lock() is used
* instead.
*
* Returns: The VMA associated with @addr, or the next vma.
* May return %NULL in the case of no vma at addr or above.
*/
struct vm_area_struct *
find_vma_prev(struct mm_struct *mm, unsigned long addr,
struct vm_area_struct **pprev)
{
struct vm_area_struct *vma;
MA_STATE(mas, &mm->mm_mt, addr, addr);
vma = mas_walk(&mas);
*pprev = mas_prev(&mas, 0);
if (!vma)
vma = mas_next(&mas, ULONG_MAX);
return vma;
}
/*
* Verify that the stack growth is acceptable and
* update accounting. This is shared with both the
* grow-up and grow-down cases.
*/
static int acct_stack_growth(struct vm_area_struct *vma,
unsigned long size, unsigned long grow)
{
struct mm_struct *mm = vma->vm_mm;
unsigned long new_start;
/* address space limit tests */
if (!may_expand_vm(mm, vma->vm_flags, grow))
return -ENOMEM;
/* Stack limit test */
if (size > rlimit(RLIMIT_STACK))
return -ENOMEM;
/* mlock limit tests */
if (!mlock_future_ok(mm, vma->vm_flags, grow << PAGE_SHIFT))
return -ENOMEM;
/* Check to ensure the stack will not grow into a hugetlb-only region */
new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
vma->vm_end - size;
if (is_hugepage_only_range(vma->vm_mm, new_start, size))
return -EFAULT;
/*
* Overcommit.. This must be the final test, as it will
* update security statistics.
*/
if (security_vm_enough_memory_mm(mm, grow))
return -ENOMEM;
return 0;
}
#if defined(CONFIG_STACK_GROWSUP)
/*
* PA-RISC uses this for its stack.
* vma is the last one with address > vma->vm_end. Have to extend vma.
*/
static int expand_upwards(struct vm_area_struct *vma, unsigned long address)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *next;
unsigned long gap_addr;
int error = 0;
MA_STATE(mas, &mm->mm_mt, vma->vm_start, address);
if (!(vma->vm_flags & VM_GROWSUP))
return -EFAULT;
/* Guard against exceeding limits of the address space. */
address &= PAGE_MASK;
if (address >= (TASK_SIZE & PAGE_MASK))
return -ENOMEM;
address += PAGE_SIZE;
/* Enforce stack_guard_gap */
gap_addr = address + stack_guard_gap;
/* Guard against overflow */
if (gap_addr < address || gap_addr > TASK_SIZE)
gap_addr = TASK_SIZE;
next = find_vma_intersection(mm, vma->vm_end, gap_addr);
if (next && vma_is_accessible(next)) {
if (!(next->vm_flags & VM_GROWSUP))
return -ENOMEM;
/* Check that both stack segments have the same anon_vma? */
}
if (next)
mas_prev_range(&mas, address);
__mas_set_range(&mas, vma->vm_start, address - 1);
if (mas_preallocate(&mas, vma, GFP_KERNEL))
return -ENOMEM;
/* We must make sure the anon_vma is allocated. */
if (unlikely(anon_vma_prepare(vma))) {
mas_destroy(&mas);
return -ENOMEM;
}
/* Lock the VMA before expanding to prevent concurrent page faults */
vma_start_write(vma);
/*
* vma->vm_start/vm_end cannot change under us because the caller
* is required to hold the mmap_lock in read mode. We need the
* anon_vma lock to serialize against concurrent expand_stacks.
*/
anon_vma_lock_write(vma->anon_vma);
/* Somebody else might have raced and expanded it already */
if (address > vma->vm_end) {
unsigned long size, grow;
size = address - vma->vm_start;
grow = (address - vma->vm_end) >> PAGE_SHIFT;
error = -ENOMEM;
if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
error = acct_stack_growth(vma, size, grow);
if (!error) {
/*
* We only hold a shared mmap_lock lock here, so
* we need to protect against concurrent vma
* expansions. anon_vma_lock_write() doesn't
* help here, as we don't guarantee that all
* growable vmas in a mm share the same root
* anon vma. So, we reuse mm->page_table_lock
* to guard against concurrent vma expansions.
*/
spin_lock(&mm->page_table_lock);
if (vma->vm_flags & VM_LOCKED)
mm->locked_vm += grow;
vm_stat_account(mm, vma->vm_flags, grow);
anon_vma_interval_tree_pre_update_vma(vma);
vma->vm_end = address;
/* Overwrite old entry in mtree. */
mas_store_prealloc(&mas, vma);
anon_vma_interval_tree_post_update_vma(vma);
spin_unlock(&mm->page_table_lock);
perf_event_mmap(vma);
}
}
}
anon_vma_unlock_write(vma->anon_vma);
mas_destroy(&mas);
validate_mm(mm);
return error;
}
#endif /* CONFIG_STACK_GROWSUP */
/*
* vma is the first one with address < vma->vm_start. Have to extend vma.
* mmap_lock held for writing.
*/
int expand_downwards(struct vm_area_struct *vma, unsigned long address)
{
struct mm_struct *mm = vma->vm_mm;
MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
struct vm_area_struct *prev;
int error = 0;
if (!(vma->vm_flags & VM_GROWSDOWN))
return -EFAULT;
address &= PAGE_MASK;
if (address < mmap_min_addr || address < FIRST_USER_ADDRESS)
return -EPERM;
/* Enforce stack_guard_gap */
prev = mas_prev(&mas, 0);
/* Check that both stack segments have the same anon_vma? */
if (prev) {
if (!(prev->vm_flags & VM_GROWSDOWN) && vma_is_accessible(prev) && (address - prev->vm_end < stack_guard_gap))
return -ENOMEM;
}
if (prev)
mas_next_range(&mas, vma->vm_start); __mas_set_range(&mas, address, vma->vm_end - 1);
if (mas_preallocate(&mas, vma, GFP_KERNEL))
return -ENOMEM;
/* We must make sure the anon_vma is allocated. */
if (unlikely(anon_vma_prepare(vma))) { mas_destroy(&mas);
return -ENOMEM;
}
/* Lock the VMA before expanding to prevent concurrent page faults */
vma_start_write(vma);
/*
* vma->vm_start/vm_end cannot change under us because the caller
* is required to hold the mmap_lock in read mode. We need the
* anon_vma lock to serialize against concurrent expand_stacks.
*/
anon_vma_lock_write(vma->anon_vma);
/* Somebody else might have raced and expanded it already */
if (address < vma->vm_start) {
unsigned long size, grow;
size = vma->vm_end - address;
grow = (vma->vm_start - address) >> PAGE_SHIFT;
error = -ENOMEM;
if (grow <= vma->vm_pgoff) {
error = acct_stack_growth(vma, size, grow);
if (!error) {
/*
* We only hold a shared mmap_lock lock here, so
* we need to protect against concurrent vma
* expansions. anon_vma_lock_write() doesn't
* help here, as we don't guarantee that all
* growable vmas in a mm share the same root
* anon vma. So, we reuse mm->page_table_lock
* to guard against concurrent vma expansions.
*/
spin_lock(&mm->page_table_lock);
if (vma->vm_flags & VM_LOCKED)
mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address;
vma->vm_pgoff -= grow;
/* Overwrite old entry in mtree. */
mas_store_prealloc(&mas, vma);
anon_vma_interval_tree_post_update_vma(vma); spin_unlock(&mm->page_table_lock);
perf_event_mmap(vma);
}
}
}
anon_vma_unlock_write(vma->anon_vma);
mas_destroy(&mas);
validate_mm(mm);
return error;
}
/* enforced gap between the expanding stack and other mappings. */
unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
static int __init cmdline_parse_stack_guard_gap(char *p)
{
unsigned long val;
char *endptr;
val = simple_strtoul(p, &endptr, 10);
if (!*endptr)
stack_guard_gap = val << PAGE_SHIFT;
return 1;
}
__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
#ifdef CONFIG_STACK_GROWSUP
int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
{
return expand_upwards(vma, address);
}
struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma, *prev;
addr &= PAGE_MASK;
vma = find_vma_prev(mm, addr, &prev);
if (vma && (vma->vm_start <= addr))
return vma;
if (!prev)
return NULL;
if (expand_stack_locked(prev, addr))
return NULL;
if (prev->vm_flags & VM_LOCKED)
populate_vma_page_range(prev, addr, prev->vm_end, NULL);
return prev;
}
#else
int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
{
return expand_downwards(vma, address);
}
struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma;
unsigned long start;
addr &= PAGE_MASK;
vma = find_vma(mm, addr);
if (!vma)
return NULL;
if (vma->vm_start <= addr)
return vma;
start = vma->vm_start;
if (expand_stack_locked(vma, addr))
return NULL;
if (vma->vm_flags & VM_LOCKED)
populate_vma_page_range(vma, addr, start, NULL);
return vma;
}
#endif
#if defined(CONFIG_STACK_GROWSUP)
#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
#define vma_expand_down(vma, addr) (-EFAULT)
#else
#define vma_expand_up(vma,addr) (-EFAULT)
#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
#endif
/*
* expand_stack(): legacy interface for page faulting. Don't use unless
* you have to.
*
* This is called with the mm locked for reading, drops the lock, takes
* the lock for writing, tries to look up a vma again, expands it if
* necessary, and downgrades the lock to reading again.
*
* If no vma is found or it can't be expanded, it returns NULL and has
* dropped the lock.
*/
struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma, *prev;
mmap_read_unlock(mm);
if (mmap_write_lock_killable(mm))
return NULL;
vma = find_vma_prev(mm, addr, &prev);
if (vma && vma->vm_start <= addr)
goto success;
if (prev && !vma_expand_up(prev, addr)) {
vma = prev;
goto success;
}
if (vma && !vma_expand_down(vma, addr))
goto success;
mmap_write_unlock(mm);
return NULL;
success:
mmap_write_downgrade(mm);
return vma;
}
/*
* Ok - we have the memory areas we should free on a maple tree so release them,
* and do the vma updates.
*
* Called with the mm semaphore held.
*/
static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
{
unsigned long nr_accounted = 0;
struct vm_area_struct *vma;
/* Update high watermark before we lower total_vm */
update_hiwater_vm(mm);
mas_for_each(mas, vma, ULONG_MAX) {
long nrpages = vma_pages(vma);
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += nrpages;
vm_stat_account(mm, vma->vm_flags, -nrpages);
remove_vma(vma, false);
}
vm_unacct_memory(nr_accounted);
}
/*
* Get rid of page table information in the indicated region.
*
* Called with the mm semaphore held.
*/
static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
struct vm_area_struct *vma, struct vm_area_struct *prev,
struct vm_area_struct *next, unsigned long start,
unsigned long end, unsigned long tree_end, bool mm_wr_locked)
{
struct mmu_gather tlb;
unsigned long mt_start = mas->index;
lru_add_drain();
tlb_gather_mmu(&tlb, mm);
update_hiwater_rss(mm);
unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked);
mas_set(mas, mt_start);
free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
next ? next->vm_start : USER_PGTABLES_CEILING,
mm_wr_locked);
tlb_finish_mmu(&tlb);
}
/*
* __split_vma() bypasses sysctl_max_map_count checking. We use this where it
* has already been checked or doesn't make sense to fail.
* VMA Iterator will point to the end VMA.
*/
static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long addr, int new_below)
{
struct vma_prepare vp;
struct vm_area_struct *new;
int err;
WARN_ON(vma->vm_start >= addr);
WARN_ON(vma->vm_end <= addr);
if (vma->vm_ops && vma->vm_ops->may_split) {
err = vma->vm_ops->may_split(vma, addr);
if (err)
return err;
}
new = vm_area_dup(vma);
if (!new)
return -ENOMEM;
if (new_below) {
new->vm_end = addr;
} else {
new->vm_start = addr;
new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
}
err = -ENOMEM;
vma_iter_config(vmi, new->vm_start, new->vm_end);
if (vma_iter_prealloc(vmi, new))
goto out_free_vma;
err = vma_dup_policy(vma, new);
if (err)
goto out_free_vmi;
err = anon_vma_clone(new, vma);
if (err)
goto out_free_mpol;
if (new->vm_file)
get_file(new->vm_file);
if (new->vm_ops && new->vm_ops->open)
new->vm_ops->open(new);
vma_start_write(vma);
vma_start_write(new);
init_vma_prep(&vp, vma);
vp.insert = new;
vma_prepare(&vp);
vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
if (new_below) {
vma->vm_start = addr;
vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
} else {
vma->vm_end = addr;
}
/* vma_complete stores the new vma */
vma_complete(&vp, vmi, vma->vm_mm);
/* Success. */
if (new_below)
vma_next(vmi);
return 0;
out_free_mpol:
mpol_put(vma_policy(new));
out_free_vmi:
vma_iter_free(vmi);
out_free_vma:
vm_area_free(new);
return err;
}
/*
* Split a vma into two pieces at address 'addr', a new vma is allocated
* either for the first part or the tail.
*/
static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long addr, int new_below)
{
if (vma->vm_mm->map_count >= sysctl_max_map_count)
return -ENOMEM;
return __split_vma(vmi, vma, addr, new_below);
}
/*
* We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
* context and anonymous VMA name within the range [start, end).
*
* As a result, we might be able to merge the newly modified VMA range with an
* adjacent VMA with identical properties.
*
* If no merge is possible and the range does not span the entirety of the VMA,
* we then need to split the VMA to accommodate the change.
*
* The function returns either the merged VMA, the original VMA if a split was
* required instead, or an error if the split failed.
*/
struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long vm_flags,
struct mempolicy *policy,
struct vm_userfaultfd_ctx uffd_ctx,
struct anon_vma_name *anon_name)
{
pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
struct vm_area_struct *merged;
merged = vma_merge(vmi, prev, vma, start, end, vm_flags,
pgoff, policy, uffd_ctx, anon_name);
if (merged)
return merged;
if (vma->vm_start < start) {
int err = split_vma(vmi, vma, start, 1);
if (err)
return ERR_PTR(err);
}
if (vma->vm_end > end) {
int err = split_vma(vmi, vma, end, 0);
if (err)
return ERR_PTR(err);
}
return vma;
}
/*
* Attempt to merge a newly mapped VMA with those adjacent to it. The caller
* must ensure that [start, end) does not overlap any existing VMA.
*/
static struct vm_area_struct
*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff)
{
return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,
vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
}
/*
* Expand vma by delta bytes, potentially merging with an immediately adjacent
* VMA with identical properties.
*/
struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
struct vm_area_struct *vma,
unsigned long delta)
{
pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);
/* vma is specified as prev, so case 1 or 2 will apply. */
return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,
vma->vm_flags, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_vma_name(vma));
}
/*
* do_vmi_align_munmap() - munmap the aligned region from @start to @end.
* @vmi: The vma iterator
* @vma: The starting vm_area_struct
* @mm: The mm_struct
* @start: The aligned start address to munmap.
* @end: The aligned end address to munmap.
* @uf: The userfaultfd list_head
* @unlock: Set to true to drop the mmap_lock. unlocking only happens on
* success.
*
* Return: 0 on success and drops the lock if so directed, error and leaves the
* lock held otherwise.
*/
static int
do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
struct mm_struct *mm, unsigned long start,
unsigned long end, struct list_head *uf, bool unlock)
{
struct vm_area_struct *prev, *next = NULL;
struct maple_tree mt_detach;
int count = 0;
int error = -ENOMEM;
unsigned long locked_vm = 0;
MA_STATE(mas_detach, &mt_detach, 0, 0);
mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
mt_on_stack(mt_detach);
/*
* If we need to split any vma, do it now to save pain later.
*
* Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
* unmapped vm_area_struct will remain in use: so lower split_vma
* places tmp vma above, and higher split_vma places tmp vma below.
*/
/* Does it split the first one? */
if (start > vma->vm_start) {
/*
* Make sure that map_count on return from munmap() will
* not exceed its limit; but let map_count go just above
* its limit temporarily, to help free resources as expected.
*/
if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
goto map_count_exceeded;
error = __split_vma(vmi, vma, start, 1);
if (error)
goto start_split_failed;
}
/*
* Detach a range of VMAs from the mm. Using next as a temp variable as
* it is always overwritten.
*/
next = vma;
do {
/* Does it split the end? */
if (next->vm_end > end) {
error = __split_vma(vmi, next, end, 0);
if (error)
goto end_split_failed;
}
vma_start_write(next);
mas_set(&mas_detach, count);
error = mas_store_gfp(&mas_detach, next, GFP_KERNEL);
if (error)
goto munmap_gather_failed;
vma_mark_detached(next, true);
if (next->vm_flags & VM_LOCKED)
locked_vm += vma_pages(next);
count++;
if (unlikely(uf)) {
/*
* If userfaultfd_unmap_prep returns an error the vmas
* will remain split, but userland will get a
* highly unexpected error anyway. This is no
* different than the case where the first of the two
* __split_vma fails, but we don't undo the first
* split, despite we could. This is unlikely enough
* failure that it's not worth optimizing it for.
*/
error = userfaultfd_unmap_prep(next, start, end, uf);
if (error)
goto userfaultfd_error;
}
#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
BUG_ON(next->vm_start < start);
BUG_ON(next->vm_start > end);
#endif
} for_each_vma_range(*vmi, next, end);
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
/* Make sure no VMAs are about to be lost. */
{
MA_STATE(test, &mt_detach, 0, 0);
struct vm_area_struct *vma_mas, *vma_test;
int test_count = 0;
vma_iter_set(vmi, start);
rcu_read_lock();
vma_test = mas_find(&test, count - 1);
for_each_vma_range(*vmi, vma_mas, end) {
BUG_ON(vma_mas != vma_test);
test_count++;
vma_test = mas_next(&test, count - 1);
}
rcu_read_unlock();
BUG_ON(count != test_count);
}
#endif
while (vma_iter_addr(vmi) > start)
vma_iter_prev_range(vmi);
error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
if (error)
goto clear_tree_failed;
/* Point of no return */
mm->locked_vm -= locked_vm;
mm->map_count -= count;
if (unlock)
mmap_write_downgrade(mm);
prev = vma_iter_prev_range(vmi);
next = vma_next(vmi);
if (next)
vma_iter_prev_range(vmi);
/*
* We can free page tables without write-locking mmap_lock because VMAs
* were isolated before we downgraded mmap_lock.
*/
mas_set(&mas_detach, 1);
unmap_region(mm, &mas_detach, vma, prev, next, start, end, count,
!unlock);
/* Statistics and freeing VMAs */
mas_set(&mas_detach, 0);
remove_mt(mm, &mas_detach);
validate_mm(mm);
if (unlock)
mmap_read_unlock(mm);
__mt_destroy(&mt_detach);
return 0;
clear_tree_failed:
userfaultfd_error:
munmap_gather_failed:
end_split_failed:
mas_set(&mas_detach, 0);
mas_for_each(&mas_detach, next, end)
vma_mark_detached(next, false);
__mt_destroy(&mt_detach);
start_split_failed:
map_count_exceeded:
validate_mm(mm);
return error;
}
/*
* do_vmi_munmap() - munmap a given range.
* @vmi: The vma iterator
* @mm: The mm_struct
* @start: The start address to munmap
* @len: The length of the range to munmap
* @uf: The userfaultfd list_head
* @unlock: set to true if the user wants to drop the mmap_lock on success
*
* This function takes a @mas that is either pointing to the previous VMA or set
* to MA_START and sets it up to remove the mapping(s). The @len will be
* aligned and any arch_unmap work will be preformed.
*
* Return: 0 on success and drops the lock if so directed, error and leaves the
* lock held otherwise.
*/
int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
unsigned long start, size_t len, struct list_head *uf,
bool unlock)
{
unsigned long end;
struct vm_area_struct *vma;
if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
return -EINVAL;
end = start + PAGE_ALIGN(len);
if (end == start)
return -EINVAL;
/* arch_unmap() might do unmaps itself. */
arch_unmap(mm, start, end);
/* Find the first overlapping VMA */
vma = vma_find(vmi, end);
if (!vma) {
if (unlock)
mmap_write_unlock(mm);
return 0;
}
return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
}
/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
* @mm: The mm_struct
* @start: The start address to munmap
* @len: The length to be munmapped.
* @uf: The userfaultfd list_head
*
* Return: 0 on success, error otherwise.
*/
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
struct list_head *uf)
{
VMA_ITERATOR(vmi, mm, start);
return do_vmi_munmap(&vmi, mm, start, len, uf, false);
}
unsigned long mmap_region(struct file *file, unsigned long addr,
unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
struct list_head *uf)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma = NULL;
struct vm_area_struct *next, *prev, *merge;
pgoff_t pglen = len >> PAGE_SHIFT;
unsigned long charged = 0;
unsigned long end = addr + len;
unsigned long merge_start = addr, merge_end = end;
bool writable_file_mapping = false;
pgoff_t vm_pgoff;
int error;
VMA_ITERATOR(vmi, mm, addr);
/* Check against address space limit. */
if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
unsigned long nr_pages;
/*
* MAP_FIXED may remove pages of mappings that intersects with
* requested mapping. Account for the pages it would unmap.
*/
nr_pages = count_vma_pages_range(mm, addr, end);
if (!may_expand_vm(mm, vm_flags,
(len >> PAGE_SHIFT) - nr_pages))
return -ENOMEM;
}
/* Unmap any existing mapping in the area */
if (do_vmi_munmap(&vmi, mm, addr, len, uf, false))
return -ENOMEM;
/*
* Private writable mapping: check memory availability
*/
if (accountable_mapping(file, vm_flags)) {
charged = len >> PAGE_SHIFT;
if (security_vm_enough_memory_mm(mm, charged))
return -ENOMEM;
vm_flags |= VM_ACCOUNT;
}
next = vma_next(&vmi);
prev = vma_prev(&vmi);
if (vm_flags & VM_SPECIAL) {
if (prev)
vma_iter_next_range(&vmi);
goto cannot_expand;
}
/* Attempt to expand an old mapping */
/* Check next */
if (next && next->vm_start == end && !vma_policy(next) &&
can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen,
NULL_VM_UFFD_CTX, NULL)) {
merge_end = next->vm_end;
vma = next;
vm_pgoff = next->vm_pgoff - pglen;
}
/* Check prev */
if (prev && prev->vm_end == addr && !vma_policy(prev) &&
(vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file,
pgoff, vma->vm_userfaultfd_ctx, NULL) :
can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,
NULL_VM_UFFD_CTX, NULL))) {
merge_start = prev->vm_start;
vma = prev;
vm_pgoff = prev->vm_pgoff;
} else if (prev) {
vma_iter_next_range(&vmi);
}
/* Actually expand, if possible */
if (vma &&
!vma_expand(&vmi, vma, merge_start, merge_end, vm_pgoff, next)) {
khugepaged_enter_vma(vma, vm_flags);
goto expanded;
}
if (vma == prev)
vma_iter_set(&vmi, addr);
cannot_expand:
/*
* Determine the object being mapped and call the appropriate
* specific mapper. the address has already been validated, but
* not unmapped, but the maps are removed from the list.
*/
vma = vm_area_alloc(mm);
if (!vma) {
error = -ENOMEM;
goto unacct_error;
}
vma_iter_config(&vmi, addr, end);
vma_set_range(vma, addr, end, pgoff);
vm_flags_init(vma, vm_flags);
vma->vm_page_prot = vm_get_page_prot(vm_flags);
if (file) {
vma->vm_file = get_file(file);
error = call_mmap(file, vma);
if (error)
goto unmap_and_free_vma;
if (vma_is_shared_maywrite(vma)) {
error = mapping_map_writable(file->f_mapping);
if (error)
goto close_and_free_vma;
writable_file_mapping = true;
}
/*
* Expansion is handled above, merging is handled below.
* Drivers should not alter the address of the VMA.
*/
error = -EINVAL;
if (WARN_ON((addr != vma->vm_start)))
goto close_and_free_vma;
vma_iter_config(&vmi, addr, end);
/*
* If vm_flags changed after call_mmap(), we should try merge
* vma again as we may succeed this time.
*/
if (unlikely(vm_flags != vma->vm_flags && prev)) {
merge = vma_merge_new_vma(&vmi, prev, vma,
vma->vm_start, vma->vm_end,
vma->vm_pgoff);
if (merge) {
/*
* ->mmap() can change vma->vm_file and fput
* the original file. So fput the vma->vm_file
* here or we would add an extra fput for file
* and cause general protection fault
* ultimately.
*/
fput(vma->vm_file);
vm_area_free(vma);
vma = merge;
/* Update vm_flags to pick up the change. */
vm_flags = vma->vm_flags;
goto unmap_writable;
}
}
vm_flags = vma->vm_flags;
} else if (vm_flags & VM_SHARED) {
error = shmem_zero_setup(vma);
if (error)
goto free_vma;
} else {
vma_set_anonymous(vma);
}
if (map_deny_write_exec(vma, vma->vm_flags)) {
error = -EACCES;
goto close_and_free_vma;
}
/* Allow architectures to sanity-check the vm_flags */
error = -EINVAL;
if (!arch_validate_flags(vma->vm_flags))
goto close_and_free_vma;
error = -ENOMEM;
if (vma_iter_prealloc(&vmi, vma))
goto close_and_free_vma;
/* Lock the VMA since it is modified after insertion into VMA tree */
vma_start_write(vma);
vma_iter_store(&vmi, vma);
mm->map_count++;
vma_link_file(vma);
/*
* vma_merge() calls khugepaged_enter_vma() either, the below
* call covers the non-merge case.
*/
khugepaged_enter_vma(vma, vma->vm_flags);
/* Once vma denies write, undo our temporary denial count */
unmap_writable:
if (writable_file_mapping)
mapping_unmap_writable(file->f_mapping);
file = vma->vm_file;
ksm_add_vma(vma);
expanded:
perf_event_mmap(vma);
vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
if (vm_flags & VM_LOCKED) {
if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
is_vm_hugetlb_page(vma) ||
vma == get_gate_vma(current->mm))
vm_flags_clear(vma, VM_LOCKED_MASK);
else
mm->locked_vm += (len >> PAGE_SHIFT);
}
if (file)
uprobe_mmap(vma);
/*
* New (or expanded) vma always get soft dirty status.
* Otherwise user-space soft-dirty page tracker won't
* be able to distinguish situation when vma area unmapped,
* then new mapped in-place (which must be aimed as
* a completely new data area).
*/
vm_flags_set(vma, VM_SOFTDIRTY);
vma_set_page_prot(vma);
validate_mm(mm);
return addr;
close_and_free_vma:
if (file && vma->vm_ops && vma->vm_ops->close)
vma->vm_ops->close(vma);
if (file || vma->vm_file) {
unmap_and_free_vma:
fput(vma->vm_file);
vma->vm_file = NULL;
vma_iter_set(&vmi, vma->vm_end);
/* Undo any partial mapping done by a device driver. */
unmap_region(mm, &vmi.mas, vma, prev, next, vma->vm_start,
vma->vm_end, vma->vm_end, true);
}
if (writable_file_mapping)
mapping_unmap_writable(file->f_mapping);
free_vma:
vm_area_free(vma);
unacct_error:
if (charged)
vm_unacct_memory(charged);
validate_mm(mm);
return error;
}
static int __vm_munmap(unsigned long start, size_t len, bool unlock)
{
int ret;
struct mm_struct *mm = current->mm;
LIST_HEAD(uf);
VMA_ITERATOR(vmi, mm, start);
if (mmap_write_lock_killable(mm))
return -EINTR;
ret = do_vmi_munmap(&vmi, mm, start, len, &uf, unlock);
if (ret || !unlock)
mmap_write_unlock(mm);
userfaultfd_unmap_complete(mm, &uf);
return ret;
}
int vm_munmap(unsigned long start, size_t len)
{
return __vm_munmap(start, len, false);
}
EXPORT_SYMBOL(vm_munmap);
SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
{
addr = untagged_addr(addr);
return __vm_munmap(addr, len, true);
}
/*
* Emulation of deprecated remap_file_pages() syscall.
*/
SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long populate = 0;
unsigned long ret = -EINVAL;
struct file *file;
pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n",
current->comm, current->pid);
if (prot)
return ret;
start = start & PAGE_MASK;
size = size & PAGE_MASK;
if (start + size <= start)
return ret;
/* Does pgoff wrap? */
if (pgoff + (size >> PAGE_SHIFT) < pgoff)
return ret;
if (mmap_write_lock_killable(mm))
return -EINTR;
vma = vma_lookup(mm, start);
if (!vma || !(vma->vm_flags & VM_SHARED))
goto out;
if (start + size > vma->vm_end) {
VMA_ITERATOR(vmi, mm, vma->vm_end);
struct vm_area_struct *next, *prev = vma;
for_each_vma_range(vmi, next, start + size) {
/* hole between vmas ? */
if (next->vm_start != prev->vm_end)
goto out;
if (next->vm_file != vma->vm_file)
goto out;
if (next->vm_flags != vma->vm_flags)
goto out;
if (start + size <= next->vm_end)
break;
prev = next;
}
if (!next)
goto out;
}
prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
flags &= MAP_NONBLOCK;
flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
if (vma->vm_flags & VM_LOCKED)
flags |= MAP_LOCKED;
file = get_file(vma->vm_file);
ret = do_mmap(vma->vm_file, start, size,
prot, flags, 0, pgoff, &populate, NULL);
fput(file);
out:
mmap_write_unlock(mm);
if (populate)
mm_populate(ret, populate);
if (!IS_ERR_VALUE(ret))
ret = 0;
return ret;
}
/*
* do_vma_munmap() - Unmap a full or partial vma.
* @vmi: The vma iterator pointing at the vma
* @vma: The first vma to be munmapped
* @start: the start of the address to unmap
* @end: The end of the address to unmap
* @uf: The userfaultfd list_head
* @unlock: Drop the lock on success
*
* unmaps a VMA mapping when the vma iterator is already in position.
* Does not handle alignment.
*
* Return: 0 on success drops the lock of so directed, error on failure and will
* still hold the lock.
*/
int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long start, unsigned long end, struct list_head *uf,
bool unlock)
{
struct mm_struct *mm = vma->vm_mm;
arch_unmap(mm, start, end);
return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
}
/*
* do_brk_flags() - Increase the brk vma if the flags match.
* @vmi: The vma iterator
* @addr: The start address
* @len: The length of the increase
* @vma: The vma,
* @flags: The VMA Flags
*
* Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags
* do not match then create a new anonymous VMA. Eventually we may be able to
* do some brk-specific accounting here.
*/
static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long addr, unsigned long len, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vma_prepare vp;
/*
* Check against address space limits by the changed size
* Note: This happens *after* clearing old mappings in some code paths.
*/
flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
return -ENOMEM;
if (mm->map_count > sysctl_max_map_count)
return -ENOMEM;
if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
return -ENOMEM;
/*
* Expand the existing vma if possible; Note that singular lists do not
* occur after forking, so the expand will only happen on new VMAs.
*/
if (vma && vma->vm_end == addr && !vma_policy(vma) &&
can_vma_merge_after(vma, flags, NULL, NULL,
addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) {
vma_iter_config(vmi, vma->vm_start, addr + len);
if (vma_iter_prealloc(vmi, vma))
goto unacct_fail;
vma_start_write(vma);
init_vma_prep(&vp, vma);
vma_prepare(&vp);
vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0);
vma->vm_end = addr + len;
vm_flags_set(vma, VM_SOFTDIRTY);
vma_iter_store(vmi, vma);
vma_complete(&vp, vmi, mm);
khugepaged_enter_vma(vma, flags);
goto out;
}
if (vma)
vma_iter_next_range(vmi);
/* create a vma struct for an anonymous mapping */
vma = vm_area_alloc(mm);
if (!vma)
goto unacct_fail;
vma_set_anonymous(vma);
vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT);
vm_flags_init(vma, flags);
vma->vm_page_prot = vm_get_page_prot(flags);
vma_start_write(vma);
if (vma_iter_store_gfp(vmi, vma, GFP_KERNEL))
goto mas_store_fail;
mm->map_count++;
validate_mm(mm);
ksm_add_vma(vma);
out:
perf_event_mmap(vma);
mm->total_vm += len >> PAGE_SHIFT;
mm->data_vm += len >> PAGE_SHIFT;
if (flags & VM_LOCKED)
mm->locked_vm += (len >> PAGE_SHIFT);
vm_flags_set(vma, VM_SOFTDIRTY);
return 0;
mas_store_fail:
vm_area_free(vma);
unacct_fail:
vm_unacct_memory(len >> PAGE_SHIFT);
return -ENOMEM;
}
int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma = NULL;
unsigned long len;
int ret;
bool populate;
LIST_HEAD(uf);
VMA_ITERATOR(vmi, mm, addr);
len = PAGE_ALIGN(request);
if (len < request)
return -ENOMEM;
if (!len)
return 0;
/* Until we need other flags, refuse anything except VM_EXEC. */
if ((flags & (~VM_EXEC)) != 0)
return -EINVAL;
if (mmap_write_lock_killable(mm))
return -EINTR;
ret = check_brk_limits(addr, len);
if (ret)
goto limits_failed;
ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0);
if (ret)
goto munmap_failed;
vma = vma_prev(&vmi);
ret = do_brk_flags(&vmi, vma, addr, len, flags);
populate = ((mm->def_flags & VM_LOCKED) != 0);
mmap_write_unlock(mm);
userfaultfd_unmap_complete(mm, &uf);
if (populate && !ret)
mm_populate(addr, len);
return ret;
munmap_failed:
limits_failed:
mmap_write_unlock(mm);
return ret;
}
EXPORT_SYMBOL(vm_brk_flags);
/* Release all mmaps. */
void exit_mmap(struct mm_struct *mm)
{
struct mmu_gather tlb;
struct vm_area_struct *vma;
unsigned long nr_accounted = 0;
MA_STATE(mas, &mm->mm_mt, 0, 0);
int count = 0;
/* mm's last user has gone, and its about to be pulled down */
mmu_notifier_release(mm);
mmap_read_lock(mm);
arch_exit_mmap(mm);
vma = mas_find(&mas, ULONG_MAX);
if (!vma || unlikely(xa_is_zero(vma))) {
/* Can happen if dup_mmap() received an OOM */
mmap_read_unlock(mm);
mmap_write_lock(mm);
goto destroy;
}
lru_add_drain();
flush_cache_mm(mm);
tlb_gather_mmu_fullmm(&tlb, mm);
/* update_hiwater_rss(mm) here? but nobody should be looking */
/* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
unmap_vmas(&tlb, &mas, vma, 0, ULONG_MAX, ULONG_MAX, false);
mmap_read_unlock(mm);
/*
* Set MMF_OOM_SKIP to hide this task from the oom killer/reaper
* because the memory has been already freed.
*/
set_bit(MMF_OOM_SKIP, &mm->flags);
mmap_write_lock(mm);
mt_clear_in_rcu(&mm->mm_mt);
mas_set(&mas, vma->vm_end);
free_pgtables(&tlb, &mas, vma, FIRST_USER_ADDRESS,
USER_PGTABLES_CEILING, true);
tlb_finish_mmu(&tlb);
/*
* Walk the list again, actually closing and freeing it, with preemption
* enabled, without holding any MM locks besides the unreachable
* mmap_write_lock.
*/
mas_set(&mas, vma->vm_end);
do {
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += vma_pages(vma);
remove_vma(vma, true);
count++;
cond_resched();
vma = mas_find(&mas, ULONG_MAX);
} while (vma && likely(!xa_is_zero(vma)));
BUG_ON(count != mm->map_count);
trace_exit_mmap(mm);
destroy:
__mt_destroy(&mm->mm_mt);
mmap_write_unlock(mm);
vm_unacct_memory(nr_accounted);
}
/* Insert vm structure into process list sorted by address
* and into the inode's i_mmap tree. If vm_file is non-NULL
* then i_mmap_rwsem is taken here.
*/
int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
{
unsigned long charged = vma_pages(vma);
if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
return -ENOMEM;
if ((vma->vm_flags & VM_ACCOUNT) &&
security_vm_enough_memory_mm(mm, charged))
return -ENOMEM;
/*
* The vm_pgoff of a purely anonymous vma should be irrelevant
* until its first write fault, when page's anon_vma and index
* are set. But now set the vm_pgoff it will almost certainly
* end up with (unless mremap moves it elsewhere before that
* first wfault), so /proc/pid/maps tells a consistent story.
*
* By setting it to reflect the virtual start address of the
* vma, merges and splits can happen in a seamless way, just
* using the existing file pgoff checks and manipulations.
* Similarly in do_mmap and in do_brk_flags.
*/
if (vma_is_anonymous(vma)) {
BUG_ON(vma->anon_vma);
vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
}
if (vma_link(mm, vma)) {
if (vma->vm_flags & VM_ACCOUNT)
vm_unacct_memory(charged);
return -ENOMEM;
}
return 0;
}
/*
* Copy the vma structure to a new location in the same mm,
* prior to moving page table entries, to effect an mremap move.
*/
struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
unsigned long addr, unsigned long len, pgoff_t pgoff,
bool *need_rmap_locks)
{
struct vm_area_struct *vma = *vmap;
unsigned long vma_start = vma->vm_start;
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *new_vma, *prev;
bool faulted_in_anon_vma = true;
VMA_ITERATOR(vmi, mm, addr);
/*
* If anonymous vma has not yet been faulted, update new pgoff
* to match new location, to increase its chance of merging.
*/
if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
pgoff = addr >> PAGE_SHIFT;
faulted_in_anon_vma = false;
}
new_vma = find_vma_prev(mm, addr, &prev);
if (new_vma && new_vma->vm_start < addr + len)
return NULL; /* should never get here */
new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff);
if (new_vma) {
/*
* Source vma may have been merged into new_vma
*/
if (unlikely(vma_start >= new_vma->vm_start &&
vma_start < new_vma->vm_end)) {
/*
* The only way we can get a vma_merge with
* self during an mremap is if the vma hasn't
* been faulted in yet and we were allowed to
* reset the dst vma->vm_pgoff to the
* destination address of the mremap to allow
* the merge to happen. mremap must change the
* vm_pgoff linearity between src and dst vmas
* (in turn preventing a vma_merge) to be
* safe. It is only safe to keep the vm_pgoff
* linear if there are no pages mapped yet.
*/
VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
*vmap = vma = new_vma;
}
*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
} else {
new_vma = vm_area_dup(vma);
if (!new_vma)
goto out;
vma_set_range(new_vma, addr, addr + len, pgoff);
if (vma_dup_policy(vma, new_vma))
goto out_free_vma;
if (anon_vma_clone(new_vma, vma))
goto out_free_mempol;
if (new_vma->vm_file)
get_file(new_vma->vm_file);
if (new_vma->vm_ops && new_vma->vm_ops->open)
new_vma->vm_ops->open(new_vma);
if (vma_link(mm, new_vma))
goto out_vma_link;
*need_rmap_locks = false;
}
return new_vma;
out_vma_link:
if (new_vma->vm_ops && new_vma->vm_ops->close)
new_vma->vm_ops->close(new_vma);
if (new_vma->vm_file)
fput(new_vma->vm_file);
unlink_anon_vmas(new_vma);
out_free_mempol:
mpol_put(vma_policy(new_vma));
out_free_vma:
vm_area_free(new_vma);
out:
return NULL;
}
/*
* Return true if the calling process may expand its vm space by the passed
* number of pages
*/
bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
{
if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
return false;
if (is_data_mapping(flags) && mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
/* Workaround for Valgrind */
if (rlimit(RLIMIT_DATA) == 0 && mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) return true; pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
current->comm, current->pid,
(mm->data_vm + npages) << PAGE_SHIFT,
rlimit(RLIMIT_DATA),
ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");
if (!ignore_rlimit_data)
return false;
}
return true;
}
void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
{
WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages);
if (is_exec_mapping(flags))
mm->exec_vm += npages; else if (is_stack_mapping(flags)) mm->stack_vm += npages; else if (is_data_mapping(flags)) mm->data_vm += npages;
}
static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
/*
* Having a close hook prevents vma merging regardless of flags.
*/
static void special_mapping_close(struct vm_area_struct *vma)
{
}
static const char *special_mapping_name(struct vm_area_struct *vma)
{
return ((struct vm_special_mapping *)vma->vm_private_data)->name;
}
static int special_mapping_mremap(struct vm_area_struct *new_vma)
{
struct vm_special_mapping *sm = new_vma->vm_private_data;
if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
return -EFAULT;
if (sm->mremap)
return sm->mremap(sm, new_vma);
return 0;
}
static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr)
{
/*
* Forbid splitting special mappings - kernel has expectations over
* the number of pages in mapping. Together with VM_DONTEXPAND
* the size of vma should stay the same over the special mapping's
* lifetime.
*/
return -EINVAL;
}
static const struct vm_operations_struct special_mapping_vmops = {
.close = special_mapping_close,
.fault = special_mapping_fault,
.mremap = special_mapping_mremap,
.name = special_mapping_name,
/* vDSO code relies that VVAR can't be accessed remotely */
.access = NULL,
.may_split = special_mapping_split,
};
static const struct vm_operations_struct legacy_special_mapping_vmops = {
.close = special_mapping_close,
.fault = special_mapping_fault,
};
static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
pgoff_t pgoff;
struct page **pages;
if (vma->vm_ops == &legacy_special_mapping_vmops) {
pages = vma->vm_private_data;
} else {
struct vm_special_mapping *sm = vma->vm_private_data;
if (sm->fault)
return sm->fault(sm, vmf->vma, vmf);
pages = sm->pages;
}
for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
pgoff--;
if (*pages) {
struct page *page = *pages;
get_page(page);
vmf->page = page;
return 0;
}
return VM_FAULT_SIGBUS;
}
static struct vm_area_struct *__install_special_mapping(
struct mm_struct *mm,
unsigned long addr, unsigned long len,
unsigned long vm_flags, void *priv,
const struct vm_operations_struct *ops)
{
int ret;
struct vm_area_struct *vma;
vma = vm_area_alloc(mm);
if (unlikely(vma == NULL))
return ERR_PTR(-ENOMEM);
vma_set_range(vma, addr, addr + len, 0);
vm_flags_init(vma, (vm_flags | mm->def_flags |
VM_DONTEXPAND | VM_SOFTDIRTY) & ~VM_LOCKED_MASK);
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
vma->vm_ops = ops;
vma->vm_private_data = priv;
ret = insert_vm_struct(mm, vma);
if (ret)
goto out;
vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
perf_event_mmap(vma);
return vma;
out:
vm_area_free(vma);
return ERR_PTR(ret);
}
bool vma_is_special_mapping(const struct vm_area_struct *vma,
const struct vm_special_mapping *sm)
{
return vma->vm_private_data == sm &&
(vma->vm_ops == &special_mapping_vmops ||
vma->vm_ops == &legacy_special_mapping_vmops);
}
/*
* Called with mm->mmap_lock held for writing.
* Insert a new vma covering the given region, with the given flags.
* Its pages are supplied by the given array of struct page *.
* The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
* The region past the last page supplied will always produce SIGBUS.
* The array pointer and the pages it points to are assumed to stay alive
* for as long as this mapping might exist.
*/
struct vm_area_struct *_install_special_mapping(
struct mm_struct *mm,
unsigned long addr, unsigned long len,
unsigned long vm_flags, const struct vm_special_mapping *spec)
{
return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
&special_mapping_vmops);
}
int install_special_mapping(struct mm_struct *mm,
unsigned long addr, unsigned long len,
unsigned long vm_flags, struct page **pages)
{
struct vm_area_struct *vma = __install_special_mapping(
mm, addr, len, vm_flags, (void *)pages,
&legacy_special_mapping_vmops);
return PTR_ERR_OR_ZERO(vma);
}
static DEFINE_MUTEX(mm_all_locks_mutex);
static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
{
if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
/*
* The LSB of head.next can't change from under us
* because we hold the mm_all_locks_mutex.
*/
down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
/*
* We can safely modify head.next after taking the
* anon_vma->root->rwsem. If some other vma in this mm shares
* the same anon_vma we won't take it again.
*
* No need of atomic instructions here, head.next
* can't change from under us thanks to the
* anon_vma->root->rwsem.
*/
if (__test_and_set_bit(0, (unsigned long *)
&anon_vma->root->rb_root.rb_root.rb_node))
BUG();
}
}
static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
{
if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
/*
* AS_MM_ALL_LOCKS can't change from under us because
* we hold the mm_all_locks_mutex.
*
* Operations on ->flags have to be atomic because
* even if AS_MM_ALL_LOCKS is stable thanks to the
* mm_all_locks_mutex, there may be other cpus
* changing other bitflags in parallel to us.
*/
if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
BUG();
down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
}
}
/*
* This operation locks against the VM for all pte/vma/mm related
* operations that could ever happen on a certain mm. This includes
* vmtruncate, try_to_unmap, and all page faults.
*
* The caller must take the mmap_lock in write mode before calling
* mm_take_all_locks(). The caller isn't allowed to release the
* mmap_lock until mm_drop_all_locks() returns.
*
* mmap_lock in write mode is required in order to block all operations
* that could modify pagetables and free pages without need of
* altering the vma layout. It's also needed in write mode to avoid new
* anon_vmas to be associated with existing vmas.
*
* A single task can't take more than one mm_take_all_locks() in a row
* or it would deadlock.
*
* The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
* mapping->flags avoid to take the same lock twice, if more than one
* vma in this mm is backed by the same anon_vma or address_space.
*
* We take locks in following order, accordingly to comment at beginning
* of mm/rmap.c:
* - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
* hugetlb mapping);
* - all vmas marked locked
* - all i_mmap_rwsem locks;
* - all anon_vma->rwseml
*
* We can take all locks within these types randomly because the VM code
* doesn't nest them and we protected from parallel mm_take_all_locks() by
* mm_all_locks_mutex.
*
* mm_take_all_locks() and mm_drop_all_locks are expensive operations
* that may have to take thousand of locks.
*
* mm_take_all_locks() can fail if it's interrupted by signals.
*/
int mm_take_all_locks(struct mm_struct *mm)
{
struct vm_area_struct *vma;
struct anon_vma_chain *avc;
MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_assert_write_locked(mm);
mutex_lock(&mm_all_locks_mutex);
/*
* vma_start_write() does not have a complement in mm_drop_all_locks()
* because vma_start_write() is always asymmetrical; it marks a VMA as
* being written to until mmap_write_unlock() or mmap_write_downgrade()
* is reached.
*/
mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
vma_start_write(vma);
}
mas_set(&mas, 0);
mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
if (vma->vm_file && vma->vm_file->f_mapping &&
is_vm_hugetlb_page(vma))
vm_lock_mapping(mm, vma->vm_file->f_mapping);
}
mas_set(&mas, 0);
mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
if (vma->vm_file && vma->vm_file->f_mapping &&
!is_vm_hugetlb_page(vma))
vm_lock_mapping(mm, vma->vm_file->f_mapping);
}
mas_set(&mas, 0);
mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
if (vma->anon_vma)
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
vm_lock_anon_vma(mm, avc->anon_vma);
}
return 0;
out_unlock:
mm_drop_all_locks(mm);
return -EINTR;
}
static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
{
if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
/*
* The LSB of head.next can't change to 0 from under
* us because we hold the mm_all_locks_mutex.
*
* We must however clear the bitflag before unlocking
* the vma so the users using the anon_vma->rb_root will
* never see our bitflag.
*
* No need of atomic instructions here, head.next
* can't change from under us until we release the
* anon_vma->root->rwsem.
*/
if (!__test_and_clear_bit(0, (unsigned long *)
&anon_vma->root->rb_root.rb_root.rb_node))
BUG();
anon_vma_unlock_write(anon_vma);
}
}
static void vm_unlock_mapping(struct address_space *mapping)
{
if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
/*
* AS_MM_ALL_LOCKS can't change to 0 from under us
* because we hold the mm_all_locks_mutex.
*/
i_mmap_unlock_write(mapping);
if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
&mapping->flags))
BUG();
}
}
/*
* The mmap_lock cannot be released by the caller until
* mm_drop_all_locks() returns.
*/
void mm_drop_all_locks(struct mm_struct *mm)
{
struct vm_area_struct *vma;
struct anon_vma_chain *avc;
MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_assert_write_locked(mm);
BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->anon_vma)
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
vm_unlock_anon_vma(avc->anon_vma);
if (vma->vm_file && vma->vm_file->f_mapping)
vm_unlock_mapping(vma->vm_file->f_mapping);
}
mutex_unlock(&mm_all_locks_mutex);
}
/*
* initialise the percpu counter for VM
*/
void __init mmap_init(void)
{
int ret;
ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
VM_BUG_ON(ret);
}
/*
* Initialise sysctl_user_reserve_kbytes.
*
* This is intended to prevent a user from starting a single memory hogging
* process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
* mode.
*
* The default value is min(3% of free memory, 128MB)
* 128MB is enough to recover with sshd/login, bash, and top/kill.
*/
static int init_user_reserve(void)
{
unsigned long free_kbytes;
free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K);
return 0;
}
subsys_initcall(init_user_reserve);
/*
* Initialise sysctl_admin_reserve_kbytes.
*
* The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
* to log in and kill a memory hogging process.
*
* Systems with more than 256MB will reserve 8MB, enough to recover
* with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
* only reserve 3% of free pages by default.
*/
static int init_admin_reserve(void)
{
unsigned long free_kbytes;
free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K);
return 0;
}
subsys_initcall(init_admin_reserve);
/*
* Reinititalise user and admin reserves if memory is added or removed.
*
* The default user reserve max is 128MB, and the default max for the
* admin reserve is 8MB. These are usually, but not always, enough to
* enable recovery from a memory hogging process using login/sshd, a shell,
* and tools like top. It may make sense to increase or even disable the
* reserve depending on the existence of swap or variations in the recovery
* tools. So, the admin may have changed them.
*
* If memory is added and the reserves have been eliminated or increased above
* the default max, then we'll trust the admin.
*
* If memory is removed and there isn't enough free memory, then we
* need to reset the reserves.
*
* Otherwise keep the reserve set by the admin.
*/
static int reserve_mem_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
unsigned long tmp, free_kbytes;
switch (action) {
case MEM_ONLINE:
/* Default max is 128MB. Leave alone if modified by operator. */
tmp = sysctl_user_reserve_kbytes;
if (tmp > 0 && tmp < SZ_128K)
init_user_reserve();
/* Default max is 8MB. Leave alone if modified by operator. */
tmp = sysctl_admin_reserve_kbytes;
if (tmp > 0 && tmp < SZ_8K)
init_admin_reserve();
break;
case MEM_OFFLINE:
free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
if (sysctl_user_reserve_kbytes > free_kbytes) {
init_user_reserve();
pr_info("vm.user_reserve_kbytes reset to %lu\n",
sysctl_user_reserve_kbytes);
}
if (sysctl_admin_reserve_kbytes > free_kbytes) {
init_admin_reserve();
pr_info("vm.admin_reserve_kbytes reset to %lu\n",
sysctl_admin_reserve_kbytes);
}
break;
default:
break;
}
return NOTIFY_OK;
}
static int __meminit init_reserve_notifier(void)
{
if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI))
pr_err("Failed registering memory add/remove notifier for admin reserve\n");
return 0;
}
subsys_initcall(init_reserve_notifier);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_RCULIST_H
#define _LINUX_RCULIST_H
#ifdef __KERNEL__
/*
* RCU-protected list version
*/
#include <linux/list.h>
#include <linux/rcupdate.h>
/*
* INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
* @list: list to be initialized
*
* You should instead use INIT_LIST_HEAD() for normal initialization and
* cleanup tasks, when readers have no access to the list being initialized.
* However, if the list being initialized is visible to readers, you
* need to keep the compiler from being too mischievous.
*/
static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
{
WRITE_ONCE(list->next, list);
WRITE_ONCE(list->prev, list);
}
/*
* return the ->next pointer of a list_head in an rcu safe
* way, we must not access it directly
*/
#define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
/**
* list_tail_rcu - returns the prev pointer of the head of the list
* @head: the head of the list
*
* Note: This should only be used with the list header, and even then
* only if list_del() and similar primitives are not also used on the
* list header.
*/
#define list_tail_rcu(head) (*((struct list_head __rcu **)(&(head)->prev)))
/*
* Check during list traversal that we are within an RCU reader
*/
#define check_arg_count_one(dummy)
#ifdef CONFIG_PROVE_RCU_LIST
#define __list_check_rcu(dummy, cond, extra...) \
({ \
check_arg_count_one(extra); \
RCU_LOCKDEP_WARN(!(cond) && !rcu_read_lock_any_held(), \
"RCU-list traversed in non-reader section!"); \
})
#define __list_check_srcu(cond) \
({ \
RCU_LOCKDEP_WARN(!(cond), \
"RCU-list traversed without holding the required lock!");\
})
#else
#define __list_check_rcu(dummy, cond, extra...) \
({ check_arg_count_one(extra); })
#define __list_check_srcu(cond) ({ })
#endif
/*
* Insert a new entry between two known consecutive entries.
*
* This is only for internal list manipulation where we know
* the prev/next entries already!
*/
static inline void __list_add_rcu(struct list_head *new,
struct list_head *prev, struct list_head *next)
{
if (!__list_add_valid(new, prev, next))
return;
new->next = next;
new->prev = prev;
rcu_assign_pointer(list_next_rcu(prev), new);
next->prev = new;
}
/**
* list_add_rcu - add a new entry to rcu-protected list
* @new: new entry to be added
* @head: list head to add it after
*
* Insert a new entry after the specified head.
* This is good for implementing stacks.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as list_add_rcu()
* or list_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* list_for_each_entry_rcu().
*/
static inline void list_add_rcu(struct list_head *new, struct list_head *head)
{
__list_add_rcu(new, head, head->next);
}
/**
* list_add_tail_rcu - add a new entry to rcu-protected list
* @new: new entry to be added
* @head: list head to add it before
*
* Insert a new entry before the specified head.
* This is useful for implementing queues.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as list_add_tail_rcu()
* or list_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* list_for_each_entry_rcu().
*/
static inline void list_add_tail_rcu(struct list_head *new,
struct list_head *head)
{
__list_add_rcu(new, head->prev, head);
}
/**
* list_del_rcu - deletes entry from list without re-initialization
* @entry: the element to delete from the list.
*
* Note: list_empty() on entry does not return true after this,
* the entry is in an undefined state. It is useful for RCU based
* lockfree traversal.
*
* In particular, it means that we can not poison the forward
* pointers that may still be used for walking the list.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as list_del_rcu()
* or list_add_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* list_for_each_entry_rcu().
*
* Note that the caller is not permitted to immediately free
* the newly deleted entry. Instead, either synchronize_rcu()
* or call_rcu() must be used to defer freeing until an RCU
* grace period has elapsed.
*/
static inline void list_del_rcu(struct list_head *entry)
{
__list_del_entry(entry); entry->prev = LIST_POISON2;
}
/**
* hlist_del_init_rcu - deletes entry from hash list with re-initialization
* @n: the element to delete from the hash list.
*
* Note: list_unhashed() on the node return true after this. It is
* useful for RCU based read lockfree traversal if the writer side
* must know if the list entry is still hashed or already unhashed.
*
* In particular, it means that we can not poison the forward pointers
* that may still be used for walking the hash list and we can only
* zero the pprev pointer so list_unhashed() will return true after
* this.
*
* The caller must take whatever precautions are necessary (such as
* holding appropriate locks) to avoid racing with another
* list-mutation primitive, such as hlist_add_head_rcu() or
* hlist_del_rcu(), running on this same list. However, it is
* perfectly legal to run concurrently with the _rcu list-traversal
* primitives, such as hlist_for_each_entry_rcu().
*/
static inline void hlist_del_init_rcu(struct hlist_node *n)
{
if (!hlist_unhashed(n)) { __hlist_del(n); WRITE_ONCE(n->pprev, NULL);
}
}
/**
* list_replace_rcu - replace old entry by new one
* @old : the element to be replaced
* @new : the new element to insert
*
* The @old entry will be replaced with the @new entry atomically.
* Note: @old should not be empty.
*/
static inline void list_replace_rcu(struct list_head *old,
struct list_head *new)
{
new->next = old->next;
new->prev = old->prev;
rcu_assign_pointer(list_next_rcu(new->prev), new);
new->next->prev = new;
old->prev = LIST_POISON2;
}
/**
* __list_splice_init_rcu - join an RCU-protected list into an existing list.
* @list: the RCU-protected list to splice
* @prev: points to the last element of the existing list
* @next: points to the first element of the existing list
* @sync: synchronize_rcu, synchronize_rcu_expedited, ...
*
* The list pointed to by @prev and @next can be RCU-read traversed
* concurrently with this function.
*
* Note that this function blocks.
*
* Important note: the caller must take whatever action is necessary to prevent
* any other updates to the existing list. In principle, it is possible to
* modify the list as soon as sync() begins execution. If this sort of thing
* becomes necessary, an alternative version based on call_rcu() could be
* created. But only if -really- needed -- there is no shortage of RCU API
* members.
*/
static inline void __list_splice_init_rcu(struct list_head *list,
struct list_head *prev,
struct list_head *next,
void (*sync)(void))
{
struct list_head *first = list->next;
struct list_head *last = list->prev;
/*
* "first" and "last" tracking list, so initialize it. RCU readers
* have access to this list, so we must use INIT_LIST_HEAD_RCU()
* instead of INIT_LIST_HEAD().
*/
INIT_LIST_HEAD_RCU(list);
/*
* At this point, the list body still points to the source list.
* Wait for any readers to finish using the list before splicing
* the list body into the new list. Any new readers will see
* an empty list.
*/
sync();
ASSERT_EXCLUSIVE_ACCESS(*first);
ASSERT_EXCLUSIVE_ACCESS(*last);
/*
* Readers are finished with the source list, so perform splice.
* The order is important if the new list is global and accessible
* to concurrent RCU readers. Note that RCU readers are not
* permitted to traverse the prev pointers without excluding
* this function.
*/
last->next = next;
rcu_assign_pointer(list_next_rcu(prev), first);
first->prev = prev;
next->prev = last;
}
/**
* list_splice_init_rcu - splice an RCU-protected list into an existing list,
* designed for stacks.
* @list: the RCU-protected list to splice
* @head: the place in the existing list to splice the first list into
* @sync: synchronize_rcu, synchronize_rcu_expedited, ...
*/
static inline void list_splice_init_rcu(struct list_head *list,
struct list_head *head,
void (*sync)(void))
{
if (!list_empty(list))
__list_splice_init_rcu(list, head, head->next, sync);
}
/**
* list_splice_tail_init_rcu - splice an RCU-protected list into an existing
* list, designed for queues.
* @list: the RCU-protected list to splice
* @head: the place in the existing list to splice the first list into
* @sync: synchronize_rcu, synchronize_rcu_expedited, ...
*/
static inline void list_splice_tail_init_rcu(struct list_head *list,
struct list_head *head,
void (*sync)(void))
{
if (!list_empty(list))
__list_splice_init_rcu(list, head->prev, head, sync);
}
/**
* list_entry_rcu - get the struct for this entry
* @ptr: the &struct list_head pointer.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* This primitive may safely run concurrently with the _rcu list-mutation
* primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
*/
#define list_entry_rcu(ptr, type, member) \
container_of(READ_ONCE(ptr), type, member)
/*
* Where are list_empty_rcu() and list_first_entry_rcu()?
*
* They do not exist because they would lead to subtle race conditions:
*
* if (!list_empty_rcu(mylist)) {
* struct foo *bar = list_first_entry_rcu(mylist, struct foo, list_member);
* do_something(bar);
* }
*
* The list might be non-empty when list_empty_rcu() checks it, but it
* might have become empty by the time that list_first_entry_rcu() rereads
* the ->next pointer, which would result in a SEGV.
*
* When not using RCU, it is OK for list_first_entry() to re-read that
* pointer because both functions should be protected by some lock that
* blocks writers.
*
* When using RCU, list_empty() uses READ_ONCE() to fetch the
* RCU-protected ->next pointer and then compares it to the address of the
* list head. However, it neither dereferences this pointer nor provides
* this pointer to its caller. Thus, READ_ONCE() suffices (that is,
* rcu_dereference() is not needed), which means that list_empty() can be
* used anywhere you would want to use list_empty_rcu(). Just don't
* expect anything useful to happen if you do a subsequent lockless
* call to list_first_entry_rcu()!!!
*
* See list_first_or_null_rcu for an alternative.
*/
/**
* list_first_or_null_rcu - get the first element from a list
* @ptr: the list head to take the element from.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* Note that if the list is empty, it returns NULL.
*
* This primitive may safely run concurrently with the _rcu list-mutation
* primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
*/
#define list_first_or_null_rcu(ptr, type, member) \
({ \
struct list_head *__ptr = (ptr); \
struct list_head *__next = READ_ONCE(__ptr->next); \
likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
})
/**
* list_next_or_null_rcu - get the next element from a list
* @head: the head for the list.
* @ptr: the list head to take the next element from.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* Note that if the ptr is at the end of the list, NULL is returned.
*
* This primitive may safely run concurrently with the _rcu list-mutation
* primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
*/
#define list_next_or_null_rcu(head, ptr, type, member) \
({ \
struct list_head *__head = (head); \
struct list_head *__ptr = (ptr); \
struct list_head *__next = READ_ONCE(__ptr->next); \
likely(__next != __head) ? list_entry_rcu(__next, type, \
member) : NULL; \
})
/**
* list_for_each_entry_rcu - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
* @cond: optional lockdep expression if called from non-RCU protection.
*
* This list-traversal primitive may safely run concurrently with
* the _rcu list-mutation primitives such as list_add_rcu()
* as long as the traversal is guarded by rcu_read_lock().
*/
#define list_for_each_entry_rcu(pos, head, member, cond...) \
for (__list_check_rcu(dummy, ## cond, 0), \
pos = list_entry_rcu((head)->next, typeof(*pos), member); \
&pos->member != (head); \
pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
/**
* list_for_each_entry_srcu - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
* @cond: lockdep expression for the lock required to traverse the list.
*
* This list-traversal primitive may safely run concurrently with
* the _rcu list-mutation primitives such as list_add_rcu()
* as long as the traversal is guarded by srcu_read_lock().
* The lockdep expression srcu_read_lock_held() can be passed as the
* cond argument from read side.
*/
#define list_for_each_entry_srcu(pos, head, member, cond) \
for (__list_check_srcu(cond), \
pos = list_entry_rcu((head)->next, typeof(*pos), member); \
&pos->member != (head); \
pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
/**
* list_entry_lockless - get the struct for this entry
* @ptr: the &struct list_head pointer.
* @type: the type of the struct this is embedded in.
* @member: the name of the list_head within the struct.
*
* This primitive may safely run concurrently with the _rcu
* list-mutation primitives such as list_add_rcu(), but requires some
* implicit RCU read-side guarding. One example is running within a special
* exception-time environment where preemption is disabled and where lockdep
* cannot be invoked. Another example is when items are added to the list,
* but never deleted.
*/
#define list_entry_lockless(ptr, type, member) \
container_of((typeof(ptr))READ_ONCE(ptr), type, member)
/**
* list_for_each_entry_lockless - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_struct within the struct.
*
* This primitive may safely run concurrently with the _rcu
* list-mutation primitives such as list_add_rcu(), but requires some
* implicit RCU read-side guarding. One example is running within a special
* exception-time environment where preemption is disabled and where lockdep
* cannot be invoked. Another example is when items are added to the list,
* but never deleted.
*/
#define list_for_each_entry_lockless(pos, head, member) \
for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
&pos->member != (head); \
pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
/**
* list_for_each_entry_continue_rcu - continue iteration over list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_head within the struct.
*
* Continue to iterate over list of given type, continuing after
* the current position which must have been in the list when the RCU read
* lock was taken.
* This would typically require either that you obtained the node from a
* previous walk of the list in the same RCU read-side critical section, or
* that you held some sort of non-RCU reference (such as a reference count)
* to keep the node alive *and* in the list.
*
* This iterator is similar to list_for_each_entry_from_rcu() except
* this starts after the given position and that one starts at the given
* position.
*/
#define list_for_each_entry_continue_rcu(pos, head, member) \
for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
&pos->member != (head); \
pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
/**
* list_for_each_entry_from_rcu - iterate over a list from current point
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the list_node within the struct.
*
* Iterate over the tail of a list starting from a given position,
* which must have been in the list when the RCU read lock was taken.
* This would typically require either that you obtained the node from a
* previous walk of the list in the same RCU read-side critical section, or
* that you held some sort of non-RCU reference (such as a reference count)
* to keep the node alive *and* in the list.
*
* This iterator is similar to list_for_each_entry_continue_rcu() except
* this starts from the given position and that one starts from the position
* after the given position.
*/
#define list_for_each_entry_from_rcu(pos, head, member) \
for (; &(pos)->member != (head); \
pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))
/**
* hlist_del_rcu - deletes entry from hash list without re-initialization
* @n: the element to delete from the hash list.
*
* Note: list_unhashed() on entry does not return true after this,
* the entry is in an undefined state. It is useful for RCU based
* lockfree traversal.
*
* In particular, it means that we can not poison the forward
* pointers that may still be used for walking the hash list.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_add_head_rcu()
* or hlist_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_for_each_entry().
*/
static inline void hlist_del_rcu(struct hlist_node *n)
{
__hlist_del(n); WRITE_ONCE(n->pprev, LIST_POISON2);
}
/**
* hlist_replace_rcu - replace old entry by new one
* @old : the element to be replaced
* @new : the new element to insert
*
* The @old entry will be replaced with the @new entry atomically.
*/
static inline void hlist_replace_rcu(struct hlist_node *old,
struct hlist_node *new)
{
struct hlist_node *next = old->next;
new->next = next;
WRITE_ONCE(new->pprev, old->pprev);
rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
if (next)
WRITE_ONCE(new->next->pprev, &new->next);
WRITE_ONCE(old->pprev, LIST_POISON2);
}
/**
* hlists_swap_heads_rcu - swap the lists the hlist heads point to
* @left: The hlist head on the left
* @right: The hlist head on the right
*
* The lists start out as [@left ][node1 ... ] and
* [@right ][node2 ... ]
* The lists end up as [@left ][node2 ... ]
* [@right ][node1 ... ]
*/
static inline void hlists_swap_heads_rcu(struct hlist_head *left, struct hlist_head *right)
{
struct hlist_node *node1 = left->first;
struct hlist_node *node2 = right->first;
rcu_assign_pointer(left->first, node2);
rcu_assign_pointer(right->first, node1);
WRITE_ONCE(node2->pprev, &left->first);
WRITE_ONCE(node1->pprev, &right->first);
}
/*
* return the first or the next element in an RCU protected hlist
*/
#define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first)))
#define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next)))
#define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
/**
* hlist_add_head_rcu
* @n: the element to add to the hash list.
* @h: the list to add to.
*
* Description:
* Adds the specified element to the specified hlist,
* while permitting racing traversals.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_add_head_rcu()
* or hlist_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_for_each_entry_rcu(), used to prevent memory-consistency
* problems on Alpha CPUs. Regardless of the type of CPU, the
* list-traversal primitive must be guarded by rcu_read_lock().
*/
static inline void hlist_add_head_rcu(struct hlist_node *n,
struct hlist_head *h)
{
struct hlist_node *first = h->first;
n->next = first;
WRITE_ONCE(n->pprev, &h->first);
rcu_assign_pointer(hlist_first_rcu(h), n);
if (first)
WRITE_ONCE(first->pprev, &n->next);
}
/**
* hlist_add_tail_rcu
* @n: the element to add to the hash list.
* @h: the list to add to.
*
* Description:
* Adds the specified element to the specified hlist,
* while permitting racing traversals.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_add_head_rcu()
* or hlist_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_for_each_entry_rcu(), used to prevent memory-consistency
* problems on Alpha CPUs. Regardless of the type of CPU, the
* list-traversal primitive must be guarded by rcu_read_lock().
*/
static inline void hlist_add_tail_rcu(struct hlist_node *n,
struct hlist_head *h)
{
struct hlist_node *i, *last = NULL;
/* Note: write side code, so rcu accessors are not needed. */
for (i = h->first; i; i = i->next)
last = i;
if (last) {
n->next = last->next;
WRITE_ONCE(n->pprev, &last->next);
rcu_assign_pointer(hlist_next_rcu(last), n);
} else {
hlist_add_head_rcu(n, h);
}
}
/**
* hlist_add_before_rcu
* @n: the new element to add to the hash list.
* @next: the existing element to add the new element before.
*
* Description:
* Adds the specified element to the specified hlist
* before the specified node while permitting racing traversals.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_add_head_rcu()
* or hlist_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_for_each_entry_rcu(), used to prevent memory-consistency
* problems on Alpha CPUs.
*/
static inline void hlist_add_before_rcu(struct hlist_node *n,
struct hlist_node *next)
{
WRITE_ONCE(n->pprev, next->pprev);
n->next = next;
rcu_assign_pointer(hlist_pprev_rcu(n), n);
WRITE_ONCE(next->pprev, &n->next);
}
/**
* hlist_add_behind_rcu
* @n: the new element to add to the hash list.
* @prev: the existing element to add the new element after.
*
* Description:
* Adds the specified element to the specified hlist
* after the specified node while permitting racing traversals.
*
* The caller must take whatever precautions are necessary
* (such as holding appropriate locks) to avoid racing
* with another list-mutation primitive, such as hlist_add_head_rcu()
* or hlist_del_rcu(), running on this same list.
* However, it is perfectly legal to run concurrently with
* the _rcu list-traversal primitives, such as
* hlist_for_each_entry_rcu(), used to prevent memory-consistency
* problems on Alpha CPUs.
*/
static inline void hlist_add_behind_rcu(struct hlist_node *n,
struct hlist_node *prev)
{
n->next = prev->next;
WRITE_ONCE(n->pprev, &prev->next);
rcu_assign_pointer(hlist_next_rcu(prev), n);
if (n->next)
WRITE_ONCE(n->next->pprev, &n->next);
}
#define __hlist_for_each_rcu(pos, head) \
for (pos = rcu_dereference(hlist_first_rcu(head)); \
pos; \
pos = rcu_dereference(hlist_next_rcu(pos)))
/**
* hlist_for_each_entry_rcu - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
* @cond: optional lockdep expression if called from non-RCU protection.
*
* This list-traversal primitive may safely run concurrently with
* the _rcu list-mutation primitives such as hlist_add_head_rcu()
* as long as the traversal is guarded by rcu_read_lock().
*/
#define hlist_for_each_entry_rcu(pos, head, member, cond...) \
for (__list_check_rcu(dummy, ## cond, 0), \
pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
typeof(*(pos)), member); \
pos; \
pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
&(pos)->member)), typeof(*(pos)), member))
/**
* hlist_for_each_entry_srcu - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
* @cond: lockdep expression for the lock required to traverse the list.
*
* This list-traversal primitive may safely run concurrently with
* the _rcu list-mutation primitives such as hlist_add_head_rcu()
* as long as the traversal is guarded by srcu_read_lock().
* The lockdep expression srcu_read_lock_held() can be passed as the
* cond argument from read side.
*/
#define hlist_for_each_entry_srcu(pos, head, member, cond) \
for (__list_check_srcu(cond), \
pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
typeof(*(pos)), member); \
pos; \
pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
&(pos)->member)), typeof(*(pos)), member))
/**
* hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*
* This list-traversal primitive may safely run concurrently with
* the _rcu list-mutation primitives such as hlist_add_head_rcu()
* as long as the traversal is guarded by rcu_read_lock().
*
* This is the same as hlist_for_each_entry_rcu() except that it does
* not do any RCU debugging or tracing.
*/
#define hlist_for_each_entry_rcu_notrace(pos, head, member) \
for (pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_first_rcu(head)),\
typeof(*(pos)), member); \
pos; \
pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_next_rcu(\
&(pos)->member)), typeof(*(pos)), member))
/**
* hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
* @pos: the type * to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*
* This list-traversal primitive may safely run concurrently with
* the _rcu list-mutation primitives such as hlist_add_head_rcu()
* as long as the traversal is guarded by rcu_read_lock().
*/
#define hlist_for_each_entry_rcu_bh(pos, head, member) \
for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_first_rcu(head)),\
typeof(*(pos)), member); \
pos; \
pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(\
&(pos)->member)), typeof(*(pos)), member))
/**
* hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
* @pos: the type * to use as a loop cursor.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry_continue_rcu(pos, member) \
for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
&(pos)->member)), typeof(*(pos)), member); \
pos; \
pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
&(pos)->member)), typeof(*(pos)), member))
/**
* hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
* @pos: the type * to use as a loop cursor.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry_continue_rcu_bh(pos, member) \
for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
&(pos)->member)), typeof(*(pos)), member); \
pos; \
pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
&(pos)->member)), typeof(*(pos)), member))
/**
* hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
* @pos: the type * to use as a loop cursor.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_for_each_entry_from_rcu(pos, member) \
for (; pos; \
pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
&(pos)->member)), typeof(*(pos)), member))
#endif /* __KERNEL__ */
#endif
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2002 Richard Henderson
* Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
* Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
*/
#define INCLUDE_VERMAGIC
#include <linux/export.h>
#include <linux/extable.h>
#include <linux/moduleloader.h>
#include <linux/module_signature.h>
#include <linux/trace_events.h>
#include <linux/init.h>
#include <linux/kallsyms.h>
#include <linux/buildid.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/kernel_read_file.h>
#include <linux/kstrtox.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/elf.h>
#include <linux/seq_file.h>
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/rcupdate.h>
#include <linux/capability.h>
#include <linux/cpu.h>
#include <linux/moduleparam.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/vermagic.h>
#include <linux/notifier.h>
#include <linux/sched.h>
#include <linux/device.h>
#include <linux/string.h>
#include <linux/mutex.h>
#include <linux/rculist.h>
#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <linux/set_memory.h>
#include <asm/mmu_context.h>
#include <linux/license.h>
#include <asm/sections.h>
#include <linux/tracepoint.h>
#include <linux/ftrace.h>
#include <linux/livepatch.h>
#include <linux/async.h>
#include <linux/percpu.h>
#include <linux/kmemleak.h>
#include <linux/jump_label.h>
#include <linux/pfn.h>
#include <linux/bsearch.h>
#include <linux/dynamic_debug.h>
#include <linux/audit.h>
#include <linux/cfi.h>
#include <linux/debugfs.h>
#include <uapi/linux/module.h>
#include "internal.h"
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>
/*
* Mutex protects:
* 1) List of modules (also safely readable with preempt_disable),
* 2) module_use links,
* 3) mod_tree.addr_min/mod_tree.addr_max.
* (delete and add uses RCU list operations).
*/
DEFINE_MUTEX(module_mutex);
LIST_HEAD(modules);
/* Work queue for freeing init sections in success case */
static void do_free_init(struct work_struct *w);
static DECLARE_WORK(init_free_wq, do_free_init);
static LLIST_HEAD(init_free_list);
struct mod_tree_root mod_tree __cacheline_aligned = {
.addr_min = -1UL,
};
struct symsearch {
const struct kernel_symbol *start, *stop;
const s32 *crcs;
enum mod_license license;
};
/*
* Bounds of module memory, for speeding up __module_address.
* Protected by module_mutex.
*/
static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,
unsigned int size, struct mod_tree_root *tree)
{
unsigned long min = (unsigned long)base;
unsigned long max = min + size;
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
if (mod_mem_type_is_core_data(type)) {
if (min < tree->data_addr_min)
tree->data_addr_min = min;
if (max > tree->data_addr_max)
tree->data_addr_max = max;
return;
}
#endif
if (min < tree->addr_min)
tree->addr_min = min;
if (max > tree->addr_max)
tree->addr_max = max;
}
static void mod_update_bounds(struct module *mod)
{
for_each_mod_mem_type(type) {
struct module_memory *mod_mem = &mod->mem[type];
if (mod_mem->size)
__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);
}
}
/* Block module loading/unloading? */
int modules_disabled;
core_param(nomodule, modules_disabled, bint, 0);
/* Waiting for a module to finish initializing? */
static DECLARE_WAIT_QUEUE_HEAD(module_wq);
static BLOCKING_NOTIFIER_HEAD(module_notify_list);
int register_module_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_register(&module_notify_list, nb);
}
EXPORT_SYMBOL(register_module_notifier);
int unregister_module_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(&module_notify_list, nb);
}
EXPORT_SYMBOL(unregister_module_notifier);
/*
* We require a truly strong try_module_get(): 0 means success.
* Otherwise an error is returned due to ongoing or failed
* initialization etc.
*/
static inline int strong_try_module_get(struct module *mod)
{
BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
if (mod && mod->state == MODULE_STATE_COMING)
return -EBUSY;
if (try_module_get(mod))
return 0;
else
return -ENOENT;
}
static inline void add_taint_module(struct module *mod, unsigned flag,
enum lockdep_ok lockdep_ok)
{
add_taint(flag, lockdep_ok);
set_bit(flag, &mod->taints);
}
/*
* A thread that wants to hold a reference to a module only while it
* is running can call this to safely exit.
*/
void __noreturn __module_put_and_kthread_exit(struct module *mod, long code)
{
module_put(mod);
kthread_exit(code);
}
EXPORT_SYMBOL(__module_put_and_kthread_exit);
/* Find a module section: 0 means not found. */
static unsigned int find_sec(const struct load_info *info, const char *name)
{
unsigned int i;
for (i = 1; i < info->hdr->e_shnum; i++) {
Elf_Shdr *shdr = &info->sechdrs[i];
/* Alloc bit cleared means "ignore it." */
if ((shdr->sh_flags & SHF_ALLOC)
&& strcmp(info->secstrings + shdr->sh_name, name) == 0)
return i;
}
return 0;
}
/* Find a module section, or NULL. */
static void *section_addr(const struct load_info *info, const char *name)
{
/* Section 0 has sh_addr 0. */
return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
}
/* Find a module section, or NULL. Fill in number of "objects" in section. */
static void *section_objs(const struct load_info *info,
const char *name,
size_t object_size,
unsigned int *num)
{
unsigned int sec = find_sec(info, name);
/* Section 0 has sh_addr 0 and sh_size 0. */
*num = info->sechdrs[sec].sh_size / object_size;
return (void *)info->sechdrs[sec].sh_addr;
}
/* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */
static unsigned int find_any_sec(const struct load_info *info, const char *name)
{
unsigned int i;
for (i = 1; i < info->hdr->e_shnum; i++) {
Elf_Shdr *shdr = &info->sechdrs[i];
if (strcmp(info->secstrings + shdr->sh_name, name) == 0)
return i;
}
return 0;
}
/*
* Find a module section, or NULL. Fill in number of "objects" in section.
* Ignores SHF_ALLOC flag.
*/
static __maybe_unused void *any_section_objs(const struct load_info *info,
const char *name,
size_t object_size,
unsigned int *num)
{
unsigned int sec = find_any_sec(info, name);
/* Section 0 has sh_addr 0 and sh_size 0. */
*num = info->sechdrs[sec].sh_size / object_size;
return (void *)info->sechdrs[sec].sh_addr;
}
#ifndef CONFIG_MODVERSIONS
#define symversion(base, idx) NULL
#else
#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
#endif
static const char *kernel_symbol_name(const struct kernel_symbol *sym)
{
#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
return offset_to_ptr(&sym->name_offset);
#else
return sym->name;
#endif
}
static const char *kernel_symbol_namespace(const struct kernel_symbol *sym)
{
#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
if (!sym->namespace_offset)
return NULL;
return offset_to_ptr(&sym->namespace_offset);
#else
return sym->namespace;
#endif
}
int cmp_name(const void *name, const void *sym)
{
return strcmp(name, kernel_symbol_name(sym));
}
static bool find_exported_symbol_in_section(const struct symsearch *syms,
struct module *owner,
struct find_symbol_arg *fsa)
{
struct kernel_symbol *sym;
if (!fsa->gplok && syms->license == GPL_ONLY)
return false;
sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
sizeof(struct kernel_symbol), cmp_name);
if (!sym)
return false;
fsa->owner = owner;
fsa->crc = symversion(syms->crcs, sym - syms->start);
fsa->sym = sym;
fsa->license = syms->license;
return true;
}
/*
* Find an exported symbol and return it, along with, (optional) crc and
* (optional) module which owns it. Needs preempt disabled or module_mutex.
*/
bool find_symbol(struct find_symbol_arg *fsa)
{
static const struct symsearch arr[] = {
{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
NOT_GPL_ONLY },
{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
__start___kcrctab_gpl,
GPL_ONLY },
};
struct module *mod;
unsigned int i;
module_assert_mutex_or_preempt();
for (i = 0; i < ARRAY_SIZE(arr); i++)
if (find_exported_symbol_in_section(&arr[i], NULL, fsa))
return true;
list_for_each_entry_rcu(mod, &modules, list,
lockdep_is_held(&module_mutex)) {
struct symsearch arr[] = {
{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
NOT_GPL_ONLY },
{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
mod->gpl_crcs,
GPL_ONLY },
};
if (mod->state == MODULE_STATE_UNFORMED)
continue;
for (i = 0; i < ARRAY_SIZE(arr); i++)
if (find_exported_symbol_in_section(&arr[i], mod, fsa))
return true;
}
pr_debug("Failed to find symbol %s\n", fsa->name);
return false;
}
/*
* Search for module by name: must hold module_mutex (or preempt disabled
* for read-only access).
*/
struct module *find_module_all(const char *name, size_t len,
bool even_unformed)
{
struct module *mod;
module_assert_mutex_or_preempt();
list_for_each_entry_rcu(mod, &modules, list,
lockdep_is_held(&module_mutex)) {
if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
continue;
if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
return mod;
}
return NULL;
}
struct module *find_module(const char *name)
{
return find_module_all(name, strlen(name), false);
}
#ifdef CONFIG_SMP
static inline void __percpu *mod_percpu(struct module *mod)
{
return mod->percpu;
}
static int percpu_modalloc(struct module *mod, struct load_info *info)
{
Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
unsigned long align = pcpusec->sh_addralign;
if (!pcpusec->sh_size)
return 0;
if (align > PAGE_SIZE) {
pr_warn("%s: per-cpu alignment %li > %li\n",
mod->name, align, PAGE_SIZE);
align = PAGE_SIZE;
}
mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
if (!mod->percpu) {
pr_warn("%s: Could not allocate %lu bytes percpu data\n",
mod->name, (unsigned long)pcpusec->sh_size);
return -ENOMEM;
}
mod->percpu_size = pcpusec->sh_size;
return 0;
}
static void percpu_modfree(struct module *mod)
{
free_percpu(mod->percpu);
}
static unsigned int find_pcpusec(struct load_info *info)
{
return find_sec(info, ".data..percpu");
}
static void percpu_modcopy(struct module *mod,
const void *from, unsigned long size)
{
int cpu;
for_each_possible_cpu(cpu)
memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
}
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
{
struct module *mod;
unsigned int cpu;
preempt_disable(); list_for_each_entry_rcu(mod, &modules, list) { if (mod->state == MODULE_STATE_UNFORMED)
continue;
if (!mod->percpu_size)
continue;
for_each_possible_cpu(cpu) { void *start = per_cpu_ptr(mod->percpu, cpu);
void *va = (void *)addr;
if (va >= start && va < start + mod->percpu_size) { if (can_addr) { *can_addr = (unsigned long) (va - start);
*can_addr += (unsigned long)
per_cpu_ptr(mod->percpu,
get_boot_cpu_id());
}
preempt_enable(); return true;
}
}
}
preempt_enable();
return false;
}
/**
* is_module_percpu_address() - test whether address is from module static percpu
* @addr: address to test
*
* Test whether @addr belongs to module static percpu area.
*
* Return: %true if @addr is from module static percpu area
*/
bool is_module_percpu_address(unsigned long addr)
{
return __is_module_percpu_address(addr, NULL);
}
#else /* ... !CONFIG_SMP */
static inline void __percpu *mod_percpu(struct module *mod)
{
return NULL;
}
static int percpu_modalloc(struct module *mod, struct load_info *info)
{
/* UP modules shouldn't have this section: ENOMEM isn't quite right */
if (info->sechdrs[info->index.pcpu].sh_size != 0)
return -ENOMEM;
return 0;
}
static inline void percpu_modfree(struct module *mod)
{
}
static unsigned int find_pcpusec(struct load_info *info)
{
return 0;
}
static inline void percpu_modcopy(struct module *mod,
const void *from, unsigned long size)
{
/* pcpusec should be 0, and size of that section should be 0. */
BUG_ON(size != 0);
}
bool is_module_percpu_address(unsigned long addr)
{
return false;
}
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
{
return false;
}
#endif /* CONFIG_SMP */
#define MODINFO_ATTR(field) \
static void setup_modinfo_##field(struct module *mod, const char *s) \
{ \
mod->field = kstrdup(s, GFP_KERNEL); \
} \
static ssize_t show_modinfo_##field(struct module_attribute *mattr, \
struct module_kobject *mk, char *buffer) \
{ \
return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
} \
static int modinfo_##field##_exists(struct module *mod) \
{ \
return mod->field != NULL; \
} \
static void free_modinfo_##field(struct module *mod) \
{ \
kfree(mod->field); \
mod->field = NULL; \
} \
static struct module_attribute modinfo_##field = { \
.attr = { .name = __stringify(field), .mode = 0444 }, \
.show = show_modinfo_##field, \
.setup = setup_modinfo_##field, \
.test = modinfo_##field##_exists, \
.free = free_modinfo_##field, \
};
MODINFO_ATTR(version);
MODINFO_ATTR(srcversion);
static struct {
char name[MODULE_NAME_LEN + 1];
char taints[MODULE_FLAGS_BUF_SIZE];
} last_unloaded_module;
#ifdef CONFIG_MODULE_UNLOAD
EXPORT_TRACEPOINT_SYMBOL(module_get);
/* MODULE_REF_BASE is the base reference count by kmodule loader. */
#define MODULE_REF_BASE 1
/* Init the unload section of the module. */
static int module_unload_init(struct module *mod)
{
/*
* Initialize reference counter to MODULE_REF_BASE.
* refcnt == 0 means module is going.
*/
atomic_set(&mod->refcnt, MODULE_REF_BASE);
INIT_LIST_HEAD(&mod->source_list);
INIT_LIST_HEAD(&mod->target_list);
/* Hold reference count during initialization. */
atomic_inc(&mod->refcnt);
return 0;
}
/* Does a already use b? */
static int already_uses(struct module *a, struct module *b)
{
struct module_use *use;
list_for_each_entry(use, &b->source_list, source_list) {
if (use->source == a)
return 1;
}
pr_debug("%s does not use %s!\n", a->name, b->name);
return 0;
}
/*
* Module a uses b
* - we add 'a' as a "source", 'b' as a "target" of module use
* - the module_use is added to the list of 'b' sources (so
* 'b' can walk the list to see who sourced them), and of 'a'
* targets (so 'a' can see what modules it targets).
*/
static int add_module_usage(struct module *a, struct module *b)
{
struct module_use *use;
pr_debug("Allocating new usage for %s.\n", a->name);
use = kmalloc(sizeof(*use), GFP_ATOMIC);
if (!use)
return -ENOMEM;
use->source = a;
use->target = b;
list_add(&use->source_list, &b->source_list);
list_add(&use->target_list, &a->target_list);
return 0;
}
/* Module a uses b: caller needs module_mutex() */
static int ref_module(struct module *a, struct module *b)
{
int err;
if (b == NULL || already_uses(a, b))
return 0;
/* If module isn't available, we fail. */
err = strong_try_module_get(b);
if (err)
return err;
err = add_module_usage(a, b);
if (err) {
module_put(b);
return err;
}
return 0;
}
/* Clear the unload stuff of the module. */
static void module_unload_free(struct module *mod)
{
struct module_use *use, *tmp;
mutex_lock(&module_mutex);
list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
struct module *i = use->target;
pr_debug("%s unusing %s\n", mod->name, i->name);
module_put(i);
list_del(&use->source_list);
list_del(&use->target_list);
kfree(use);
}
mutex_unlock(&module_mutex);
}
#ifdef CONFIG_MODULE_FORCE_UNLOAD
static inline int try_force_unload(unsigned int flags)
{
int ret = (flags & O_TRUNC);
if (ret)
add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
return ret;
}
#else
static inline int try_force_unload(unsigned int flags)
{
return 0;
}
#endif /* CONFIG_MODULE_FORCE_UNLOAD */
/* Try to release refcount of module, 0 means success. */
static int try_release_module_ref(struct module *mod)
{
int ret;
/* Try to decrement refcnt which we set at loading */
ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
BUG_ON(ret < 0);
if (ret)
/* Someone can put this right now, recover with checking */
ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
return ret;
}
static int try_stop_module(struct module *mod, int flags, int *forced)
{
/* If it's not unused, quit unless we're forcing. */
if (try_release_module_ref(mod) != 0) {
*forced = try_force_unload(flags);
if (!(*forced))
return -EWOULDBLOCK;
}
/* Mark it as dying. */
mod->state = MODULE_STATE_GOING;
return 0;
}
/**
* module_refcount() - return the refcount or -1 if unloading
* @mod: the module we're checking
*
* Return:
* -1 if the module is in the process of unloading
* otherwise the number of references in the kernel to the module
*/
int module_refcount(struct module *mod)
{
return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
}
EXPORT_SYMBOL(module_refcount);
/* This exists whether we can unload or not */
static void free_module(struct module *mod);
SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
unsigned int, flags)
{
struct module *mod;
char name[MODULE_NAME_LEN];
char buf[MODULE_FLAGS_BUF_SIZE];
int ret, forced = 0;
if (!capable(CAP_SYS_MODULE) || modules_disabled)
return -EPERM;
if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
return -EFAULT;
name[MODULE_NAME_LEN-1] = '\0';
audit_log_kern_module(name);
if (mutex_lock_interruptible(&module_mutex) != 0)
return -EINTR;
mod = find_module(name);
if (!mod) {
ret = -ENOENT;
goto out;
}
if (!list_empty(&mod->source_list)) {
/* Other modules depend on us: get rid of them first. */
ret = -EWOULDBLOCK;
goto out;
}
/* Doing init or already dying? */
if (mod->state != MODULE_STATE_LIVE) {
/* FIXME: if (force), slam module count damn the torpedoes */
pr_debug("%s already dying\n", mod->name);
ret = -EBUSY;
goto out;
}
/* If it has an init func, it must have an exit func to unload */
if (mod->init && !mod->exit) {
forced = try_force_unload(flags);
if (!forced) {
/* This module can't be removed */
ret = -EBUSY;
goto out;
}
}
ret = try_stop_module(mod, flags, &forced);
if (ret != 0)
goto out;
mutex_unlock(&module_mutex);
/* Final destruction now no one is using it. */
if (mod->exit != NULL)
mod->exit();
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_GOING, mod);
klp_module_going(mod);
ftrace_release_mod(mod);
async_synchronize_full();
/* Store the name and taints of the last unloaded module for diagnostic purposes */
strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name));
strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.taints));
free_module(mod);
/* someone could wait for the module in add_unformed_module() */
wake_up_all(&module_wq);
return 0;
out:
mutex_unlock(&module_mutex);
return ret;
}
void __symbol_put(const char *symbol)
{
struct find_symbol_arg fsa = {
.name = symbol,
.gplok = true,
};
preempt_disable();
BUG_ON(!find_symbol(&fsa));
module_put(fsa.owner);
preempt_enable();
}
EXPORT_SYMBOL(__symbol_put);
/* Note this assumes addr is a function, which it currently always is. */
void symbol_put_addr(void *addr)
{
struct module *modaddr;
unsigned long a = (unsigned long)dereference_function_descriptor(addr);
if (core_kernel_text(a))
return;
/*
* Even though we hold a reference on the module; we still need to
* disable preemption in order to safely traverse the data structure.
*/
preempt_disable(); modaddr = __module_text_address(a); BUG_ON(!modaddr); module_put(modaddr); preempt_enable();
}
EXPORT_SYMBOL_GPL(symbol_put_addr);
static ssize_t show_refcnt(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
return sprintf(buffer, "%i\n", module_refcount(mk->mod));
}
static struct module_attribute modinfo_refcnt =
__ATTR(refcnt, 0444, show_refcnt, NULL);
void __module_get(struct module *module)
{
if (module) {
atomic_inc(&module->refcnt);
trace_module_get(module, _RET_IP_);
}
}
EXPORT_SYMBOL(__module_get);
bool try_module_get(struct module *module)
{
bool ret = true;
if (module) {
/* Note: here, we can fail to get a reference */
if (likely(module_is_live(module) &&
atomic_inc_not_zero(&module->refcnt) != 0))
trace_module_get(module, _RET_IP_);
else
ret = false;
}
return ret;
}
EXPORT_SYMBOL(try_module_get);
void module_put(struct module *module)
{
int ret;
if (module) {
ret = atomic_dec_if_positive(&module->refcnt);
WARN_ON(ret < 0); /* Failed to put refcount */
trace_module_put(module, _RET_IP_);
}
}
EXPORT_SYMBOL(module_put);
#else /* !CONFIG_MODULE_UNLOAD */
static inline void module_unload_free(struct module *mod)
{
}
static int ref_module(struct module *a, struct module *b)
{
return strong_try_module_get(b);
}
static inline int module_unload_init(struct module *mod)
{
return 0;
}
#endif /* CONFIG_MODULE_UNLOAD */
size_t module_flags_taint(unsigned long taints, char *buf)
{
size_t l = 0;
int i;
for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
if (taint_flags[i].module && test_bit(i, &taints))
buf[l++] = taint_flags[i].c_true;
}
return l;
}
static ssize_t show_initstate(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
const char *state = "unknown";
switch (mk->mod->state) {
case MODULE_STATE_LIVE:
state = "live";
break;
case MODULE_STATE_COMING:
state = "coming";
break;
case MODULE_STATE_GOING:
state = "going";
break;
default:
BUG();
}
return sprintf(buffer, "%s\n", state);
}
static struct module_attribute modinfo_initstate =
__ATTR(initstate, 0444, show_initstate, NULL);
static ssize_t store_uevent(struct module_attribute *mattr,
struct module_kobject *mk,
const char *buffer, size_t count)
{
int rc;
rc = kobject_synth_uevent(&mk->kobj, buffer, count);
return rc ? rc : count;
}
struct module_attribute module_uevent =
__ATTR(uevent, 0200, NULL, store_uevent);
static ssize_t show_coresize(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
unsigned int size = mk->mod->mem[MOD_TEXT].size;
if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {
for_class_mod_mem_type(type, core_data)
size += mk->mod->mem[type].size;
}
return sprintf(buffer, "%u\n", size);
}
static struct module_attribute modinfo_coresize =
__ATTR(coresize, 0444, show_coresize, NULL);
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
static ssize_t show_datasize(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
unsigned int size = 0;
for_class_mod_mem_type(type, core_data)
size += mk->mod->mem[type].size;
return sprintf(buffer, "%u\n", size);
}
static struct module_attribute modinfo_datasize =
__ATTR(datasize, 0444, show_datasize, NULL);
#endif
static ssize_t show_initsize(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
unsigned int size = 0;
for_class_mod_mem_type(type, init)
size += mk->mod->mem[type].size;
return sprintf(buffer, "%u\n", size);
}
static struct module_attribute modinfo_initsize =
__ATTR(initsize, 0444, show_initsize, NULL);
static ssize_t show_taint(struct module_attribute *mattr,
struct module_kobject *mk, char *buffer)
{
size_t l;
l = module_flags_taint(mk->mod->taints, buffer);
buffer[l++] = '\n';
return l;
}
static struct module_attribute modinfo_taint =
__ATTR(taint, 0444, show_taint, NULL);
struct module_attribute *modinfo_attrs[] = {
&module_uevent,
&modinfo_version,
&modinfo_srcversion,
&modinfo_initstate,
&modinfo_coresize,
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
&modinfo_datasize,
#endif
&modinfo_initsize,
&modinfo_taint,
#ifdef CONFIG_MODULE_UNLOAD
&modinfo_refcnt,
#endif
NULL,
};
size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs);
static const char vermagic[] = VERMAGIC_STRING;
int try_to_force_load(struct module *mod, const char *reason)
{
#ifdef CONFIG_MODULE_FORCE_LOAD
if (!test_taint(TAINT_FORCED_MODULE))
pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
return 0;
#else
return -ENOEXEC;
#endif
}
/* Parse tag=value strings from .modinfo section */
char *module_next_tag_pair(char *string, unsigned long *secsize)
{
/* Skip non-zero chars */
while (string[0]) {
string++;
if ((*secsize)-- <= 1)
return NULL;
}
/* Skip any zero padding. */
while (!string[0]) {
string++;
if ((*secsize)-- <= 1)
return NULL;
}
return string;
}
static char *get_next_modinfo(const struct load_info *info, const char *tag,
char *prev)
{
char *p;
unsigned int taglen = strlen(tag);
Elf_Shdr *infosec = &info->sechdrs[info->index.info];
unsigned long size = infosec->sh_size;
/*
* get_modinfo() calls made before rewrite_section_headers()
* must use sh_offset, as sh_addr isn't set!
*/
char *modinfo = (char *)info->hdr + infosec->sh_offset;
if (prev) {
size -= prev - modinfo;
modinfo = module_next_tag_pair(prev, &size);
}
for (p = modinfo; p; p = module_next_tag_pair(p, &size)) {
if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
return p + taglen + 1;
}
return NULL;
}
static char *get_modinfo(const struct load_info *info, const char *tag)
{
return get_next_modinfo(info, tag, NULL);
}
static int verify_namespace_is_imported(const struct load_info *info,
const struct kernel_symbol *sym,
struct module *mod)
{
const char *namespace;
char *imported_namespace;
namespace = kernel_symbol_namespace(sym);
if (namespace && namespace[0]) {
for_each_modinfo_entry(imported_namespace, info, "import_ns") {
if (strcmp(namespace, imported_namespace) == 0)
return 0;
}
#ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
pr_warn(
#else
pr_err(
#endif
"%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
mod->name, kernel_symbol_name(sym), namespace);
#ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
return -EINVAL;
#endif
}
return 0;
}
static bool inherit_taint(struct module *mod, struct module *owner, const char *name)
{
if (!owner || !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
return true;
if (mod->using_gplonly_symbols) {
pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n",
mod->name, name, owner->name);
return false;
}
if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n",
mod->name, name, owner->name);
set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
}
return true;
}
/* Resolve a symbol for this module. I.e. if we find one, record usage. */
static const struct kernel_symbol *resolve_symbol(struct module *mod,
const struct load_info *info,
const char *name,
char ownername[])
{
struct find_symbol_arg fsa = {
.name = name,
.gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)),
.warn = true,
};
int err;
/*
* The module_mutex should not be a heavily contended lock;
* if we get the occasional sleep here, we'll go an extra iteration
* in the wait_event_interruptible(), which is harmless.
*/
sched_annotate_sleep();
mutex_lock(&module_mutex);
if (!find_symbol(&fsa))
goto unlock;
if (fsa.license == GPL_ONLY)
mod->using_gplonly_symbols = true;
if (!inherit_taint(mod, fsa.owner, name)) {
fsa.sym = NULL;
goto getname;
}
if (!check_version(info, name, mod, fsa.crc)) {
fsa.sym = ERR_PTR(-EINVAL);
goto getname;
}
err = verify_namespace_is_imported(info, fsa.sym, mod);
if (err) {
fsa.sym = ERR_PTR(err);
goto getname;
}
err = ref_module(mod, fsa.owner);
if (err) {
fsa.sym = ERR_PTR(err);
goto getname;
}
getname:
/* We must make copy under the lock if we failed to get ref. */
strncpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN);
unlock:
mutex_unlock(&module_mutex);
return fsa.sym;
}
static const struct kernel_symbol *
resolve_symbol_wait(struct module *mod,
const struct load_info *info,
const char *name)
{
const struct kernel_symbol *ksym;
char owner[MODULE_NAME_LEN];
if (wait_event_interruptible_timeout(module_wq,
!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
|| PTR_ERR(ksym) != -EBUSY,
30 * HZ) <= 0) {
pr_warn("%s: gave up waiting for init of module %s.\n",
mod->name, owner);
}
return ksym;
}
void __weak module_memfree(void *module_region)
{
/*
* This memory may be RO, and freeing RO memory in an interrupt is not
* supported by vmalloc.
*/
WARN_ON(in_interrupt());
vfree(module_region);
}
void __weak module_arch_cleanup(struct module *mod)
{
}
void __weak module_arch_freeing_init(struct module *mod)
{
}
static bool mod_mem_use_vmalloc(enum mod_mem_type type)
{
return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) &&
mod_mem_type_is_core_data(type);
}
static void *module_memory_alloc(unsigned int size, enum mod_mem_type type)
{
if (mod_mem_use_vmalloc(type))
return vzalloc(size);
return module_alloc(size);
}
static void module_memory_free(void *ptr, enum mod_mem_type type)
{
if (mod_mem_use_vmalloc(type))
vfree(ptr);
else
module_memfree(ptr);
}
static void free_mod_mem(struct module *mod)
{
for_each_mod_mem_type(type) {
struct module_memory *mod_mem = &mod->mem[type];
if (type == MOD_DATA)
continue;
/* Free lock-classes; relies on the preceding sync_rcu(). */
lockdep_free_key_range(mod_mem->base, mod_mem->size);
if (mod_mem->size)
module_memory_free(mod_mem->base, type);
}
/* MOD_DATA hosts mod, so free it at last */
lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA);
}
/* Free a module, remove from lists, etc. */
static void free_module(struct module *mod)
{
trace_module_free(mod);
mod_sysfs_teardown(mod);
/*
* We leave it in list to prevent duplicate loads, but make sure
* that noone uses it while it's being deconstructed.
*/
mutex_lock(&module_mutex);
mod->state = MODULE_STATE_UNFORMED;
mutex_unlock(&module_mutex);
/* Arch-specific cleanup. */
module_arch_cleanup(mod);
/* Module unload stuff */
module_unload_free(mod);
/* Free any allocated parameters. */
destroy_params(mod->kp, mod->num_kp);
if (is_livepatch_module(mod))
free_module_elf(mod);
/* Now we can delete it from the lists */
mutex_lock(&module_mutex);
/* Unlink carefully: kallsyms could be walking list. */
list_del_rcu(&mod->list);
mod_tree_remove(mod);
/* Remove this module from bug list, this uses list_del_rcu */
module_bug_cleanup(mod);
/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
synchronize_rcu();
if (try_add_tainted_module(mod))
pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n",
mod->name);
mutex_unlock(&module_mutex);
/* This may be empty, but that's OK */
module_arch_freeing_init(mod);
kfree(mod->args);
percpu_modfree(mod);
free_mod_mem(mod);
}
void *__symbol_get(const char *symbol)
{
struct find_symbol_arg fsa = {
.name = symbol,
.gplok = true,
.warn = true,
};
preempt_disable();
if (!find_symbol(&fsa))
goto fail;
if (fsa.license != GPL_ONLY) {
pr_warn("failing symbol_get of non-GPLONLY symbol %s.\n",
symbol);
goto fail;
}
if (strong_try_module_get(fsa.owner))
goto fail;
preempt_enable();
return (void *)kernel_symbol_value(fsa.sym);
fail:
preempt_enable();
return NULL;
}
EXPORT_SYMBOL_GPL(__symbol_get);
/*
* Ensure that an exported symbol [global namespace] does not already exist
* in the kernel or in some other module's exported symbol table.
*
* You must hold the module_mutex.
*/
static int verify_exported_symbols(struct module *mod)
{
unsigned int i;
const struct kernel_symbol *s;
struct {
const struct kernel_symbol *sym;
unsigned int num;
} arr[] = {
{ mod->syms, mod->num_syms },
{ mod->gpl_syms, mod->num_gpl_syms },
};
for (i = 0; i < ARRAY_SIZE(arr); i++) {
for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
struct find_symbol_arg fsa = {
.name = kernel_symbol_name(s),
.gplok = true,
};
if (find_symbol(&fsa)) {
pr_err("%s: exports duplicate symbol %s"
" (owned by %s)\n",
mod->name, kernel_symbol_name(s),
module_name(fsa.owner));
return -ENOEXEC;
}
}
}
return 0;
}
static bool ignore_undef_symbol(Elf_Half emachine, const char *name)
{
/*
* On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as
* before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64.
* i386 has a similar problem but may not deserve a fix.
*
* If we ever have to ignore many symbols, consider refactoring the code to
* only warn if referenced by a relocation.
*/
if (emachine == EM_386 || emachine == EM_X86_64)
return !strcmp(name, "_GLOBAL_OFFSET_TABLE_");
return false;
}
/* Change all symbols so that st_value encodes the pointer directly. */
static int simplify_symbols(struct module *mod, const struct load_info *info)
{
Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
Elf_Sym *sym = (void *)symsec->sh_addr;
unsigned long secbase;
unsigned int i;
int ret = 0;
const struct kernel_symbol *ksym;
for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
const char *name = info->strtab + sym[i].st_name;
switch (sym[i].st_shndx) {
case SHN_COMMON:
/* Ignore common symbols */
if (!strncmp(name, "__gnu_lto", 9))
break;
/*
* We compiled with -fno-common. These are not
* supposed to happen.
*/
pr_debug("Common symbol: %s\n", name);
pr_warn("%s: please compile with -fno-common\n",
mod->name);
ret = -ENOEXEC;
break;
case SHN_ABS:
/* Don't need to do anything */
pr_debug("Absolute symbol: 0x%08lx %s\n",
(long)sym[i].st_value, name);
break;
case SHN_LIVEPATCH:
/* Livepatch symbols are resolved by livepatch */
break;
case SHN_UNDEF:
ksym = resolve_symbol_wait(mod, info, name);
/* Ok if resolved. */
if (ksym && !IS_ERR(ksym)) {
sym[i].st_value = kernel_symbol_value(ksym);
break;
}
/* Ok if weak or ignored. */
if (!ksym &&
(ELF_ST_BIND(sym[i].st_info) == STB_WEAK ||
ignore_undef_symbol(info->hdr->e_machine, name)))
break;
ret = PTR_ERR(ksym) ?: -ENOENT;
pr_warn("%s: Unknown symbol %s (err %d)\n",
mod->name, name, ret);
break;
default:
/* Divert to percpu allocation if a percpu var. */
if (sym[i].st_shndx == info->index.pcpu)
secbase = (unsigned long)mod_percpu(mod);
else
secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
sym[i].st_value += secbase;
break;
}
}
return ret;
}
static int apply_relocations(struct module *mod, const struct load_info *info)
{
unsigned int i;
int err = 0;
/* Now do relocations. */
for (i = 1; i < info->hdr->e_shnum; i++) {
unsigned int infosec = info->sechdrs[i].sh_info;
/* Not a valid relocation section? */
if (infosec >= info->hdr->e_shnum)
continue;
/* Don't bother with non-allocated sections */
if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
continue;
if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
err = klp_apply_section_relocs(mod, info->sechdrs,
info->secstrings,
info->strtab,
info->index.sym, i,
NULL);
else if (info->sechdrs[i].sh_type == SHT_REL)
err = apply_relocate(info->sechdrs, info->strtab,
info->index.sym, i, mod);
else if (info->sechdrs[i].sh_type == SHT_RELA)
err = apply_relocate_add(info->sechdrs, info->strtab,
info->index.sym, i, mod);
if (err < 0)
break;
}
return err;
}
/* Additional bytes needed by arch in front of individual sections */
unsigned int __weak arch_mod_section_prepend(struct module *mod,
unsigned int section)
{
/* default implementation just returns zero */
return 0;
}
long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
Elf_Shdr *sechdr, unsigned int section)
{
long offset;
long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT;
mod->mem[type].size += arch_mod_section_prepend(mod, section);
offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1);
mod->mem[type].size = offset + sechdr->sh_size;
WARN_ON_ONCE(offset & mask);
return offset | mask;
}
bool module_init_layout_section(const char *sname)
{
#ifndef CONFIG_MODULE_UNLOAD
if (module_exit_section(sname))
return true;
#endif
return module_init_section(sname);
}
static void __layout_sections(struct module *mod, struct load_info *info, bool is_init)
{
unsigned int m, i;
static const unsigned long masks[][2] = {
/*
* NOTE: all executable code must be the first section
* in this array; otherwise modify the text_size
* finder in the two loops below
*/
{ SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
{ SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
{ SHF_RO_AFTER_INIT | SHF_ALLOC, ARCH_SHF_SMALL },
{ SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
{ ARCH_SHF_SMALL | SHF_ALLOC, 0 }
};
static const int core_m_to_mem_type[] = {
MOD_TEXT,
MOD_RODATA,
MOD_RO_AFTER_INIT,
MOD_DATA,
MOD_DATA,
};
static const int init_m_to_mem_type[] = {
MOD_INIT_TEXT,
MOD_INIT_RODATA,
MOD_INVALID,
MOD_INIT_DATA,
MOD_INIT_DATA,
};
for (m = 0; m < ARRAY_SIZE(masks); ++m) {
enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m];
for (i = 0; i < info->hdr->e_shnum; ++i) {
Elf_Shdr *s = &info->sechdrs[i];
const char *sname = info->secstrings + s->sh_name;
if ((s->sh_flags & masks[m][0]) != masks[m][0]
|| (s->sh_flags & masks[m][1])
|| s->sh_entsize != ~0UL
|| is_init != module_init_layout_section(sname))
continue;
if (WARN_ON_ONCE(type == MOD_INVALID))
continue;
s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
pr_debug("\t%s\n", sname);
}
}
}
/*
* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
* might -- code, read-only data, read-write data, small data. Tally
* sizes, and place the offsets into sh_entsize fields: high bit means it
* belongs in init.
*/
static void layout_sections(struct module *mod, struct load_info *info)
{
unsigned int i;
for (i = 0; i < info->hdr->e_shnum; i++)
info->sechdrs[i].sh_entsize = ~0UL;
pr_debug("Core section allocation order for %s:\n", mod->name);
__layout_sections(mod, info, false);
pr_debug("Init section allocation order for %s:\n", mod->name);
__layout_sections(mod, info, true);
}
static void module_license_taint_check(struct module *mod, const char *license)
{
if (!license)
license = "unspecified";
if (!license_is_gpl_compatible(license)) {
if (!test_taint(TAINT_PROPRIETARY_MODULE))
pr_warn("%s: module license '%s' taints kernel.\n",
mod->name, license);
add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
LOCKDEP_NOW_UNRELIABLE);
}
}
static void setup_modinfo(struct module *mod, struct load_info *info)
{
struct module_attribute *attr;
int i;
for (i = 0; (attr = modinfo_attrs[i]); i++) {
if (attr->setup)
attr->setup(mod, get_modinfo(info, attr->attr.name));
}
}
static void free_modinfo(struct module *mod)
{
struct module_attribute *attr;
int i;
for (i = 0; (attr = modinfo_attrs[i]); i++) {
if (attr->free)
attr->free(mod);
}
}
void * __weak module_alloc(unsigned long size)
{
return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
NUMA_NO_NODE, __builtin_return_address(0));
}
bool __weak module_init_section(const char *name)
{
return strstarts(name, ".init");
}
bool __weak module_exit_section(const char *name)
{
return strstarts(name, ".exit");
}
static int validate_section_offset(struct load_info *info, Elf_Shdr *shdr)
{
#if defined(CONFIG_64BIT)
unsigned long long secend;
#else
unsigned long secend;
#endif
/*
* Check for both overflow and offset/size being
* too large.
*/
secend = shdr->sh_offset + shdr->sh_size;
if (secend < shdr->sh_offset || secend > info->len)
return -ENOEXEC;
return 0;
}
/*
* Check userspace passed ELF module against our expectations, and cache
* useful variables for further processing as we go.
*
* This does basic validity checks against section offsets and sizes, the
* section name string table, and the indices used for it (sh_name).
*
* As a last step, since we're already checking the ELF sections we cache
* useful variables which will be used later for our convenience:
*
* o pointers to section headers
* o cache the modinfo symbol section
* o cache the string symbol section
* o cache the module section
*
* As a last step we set info->mod to the temporary copy of the module in
* info->hdr. The final one will be allocated in move_module(). Any
* modifications we make to our copy of the module will be carried over
* to the final minted module.
*/
static int elf_validity_cache_copy(struct load_info *info, int flags)
{
unsigned int i;
Elf_Shdr *shdr, *strhdr;
int err;
unsigned int num_mod_secs = 0, mod_idx;
unsigned int num_info_secs = 0, info_idx;
unsigned int num_sym_secs = 0, sym_idx;
if (info->len < sizeof(*(info->hdr))) {
pr_err("Invalid ELF header len %lu\n", info->len);
goto no_exec;
}
if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) {
pr_err("Invalid ELF header magic: != %s\n", ELFMAG);
goto no_exec;
}
if (info->hdr->e_type != ET_REL) {
pr_err("Invalid ELF header type: %u != %u\n",
info->hdr->e_type, ET_REL);
goto no_exec;
}
if (!elf_check_arch(info->hdr)) {
pr_err("Invalid architecture in ELF header: %u\n",
info->hdr->e_machine);
goto no_exec;
}
if (!module_elf_check_arch(info->hdr)) {
pr_err("Invalid module architecture in ELF header: %u\n",
info->hdr->e_machine);
goto no_exec;
}
if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) {
pr_err("Invalid ELF section header size\n");
goto no_exec;
}
/*
* e_shnum is 16 bits, and sizeof(Elf_Shdr) is
* known and small. So e_shnum * sizeof(Elf_Shdr)
* will not overflow unsigned long on any platform.
*/
if (info->hdr->e_shoff >= info->len
|| (info->hdr->e_shnum * sizeof(Elf_Shdr) >
info->len - info->hdr->e_shoff)) {
pr_err("Invalid ELF section header overflow\n");
goto no_exec;
}
info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
/*
* Verify if the section name table index is valid.
*/
if (info->hdr->e_shstrndx == SHN_UNDEF
|| info->hdr->e_shstrndx >= info->hdr->e_shnum) {
pr_err("Invalid ELF section name index: %d || e_shstrndx (%d) >= e_shnum (%d)\n",
info->hdr->e_shstrndx, info->hdr->e_shstrndx,
info->hdr->e_shnum);
goto no_exec;
}
strhdr = &info->sechdrs[info->hdr->e_shstrndx];
err = validate_section_offset(info, strhdr);
if (err < 0) {
pr_err("Invalid ELF section hdr(type %u)\n", strhdr->sh_type);
return err;
}
/*
* The section name table must be NUL-terminated, as required
* by the spec. This makes strcmp and pr_* calls that access
* strings in the section safe.
*/
info->secstrings = (void *)info->hdr + strhdr->sh_offset;
if (strhdr->sh_size == 0) {
pr_err("empty section name table\n");
goto no_exec;
}
if (info->secstrings[strhdr->sh_size - 1] != '\0') {
pr_err("ELF Spec violation: section name table isn't null terminated\n");
goto no_exec;
}
/*
* The code assumes that section 0 has a length of zero and
* an addr of zero, so check for it.
*/
if (info->sechdrs[0].sh_type != SHT_NULL
|| info->sechdrs[0].sh_size != 0
|| info->sechdrs[0].sh_addr != 0) {
pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n",
info->sechdrs[0].sh_type);
goto no_exec;
}
for (i = 1; i < info->hdr->e_shnum; i++) {
shdr = &info->sechdrs[i];
switch (shdr->sh_type) {
case SHT_NULL:
case SHT_NOBITS:
continue;
case SHT_SYMTAB:
if (shdr->sh_link == SHN_UNDEF
|| shdr->sh_link >= info->hdr->e_shnum) {
pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n",
shdr->sh_link, shdr->sh_link,
info->hdr->e_shnum);
goto no_exec;
}
num_sym_secs++;
sym_idx = i;
fallthrough;
default:
err = validate_section_offset(info, shdr);
if (err < 0) {
pr_err("Invalid ELF section in module (section %u type %u)\n",
i, shdr->sh_type);
return err;
}
if (strcmp(info->secstrings + shdr->sh_name,
".gnu.linkonce.this_module") == 0) {
num_mod_secs++;
mod_idx = i;
} else if (strcmp(info->secstrings + shdr->sh_name,
".modinfo") == 0) {
num_info_secs++;
info_idx = i;
}
if (shdr->sh_flags & SHF_ALLOC) {
if (shdr->sh_name >= strhdr->sh_size) {
pr_err("Invalid ELF section name in module (section %u type %u)\n",
i, shdr->sh_type);
return -ENOEXEC;
}
}
break;
}
}
if (num_info_secs > 1) {
pr_err("Only one .modinfo section must exist.\n");
goto no_exec;
} else if (num_info_secs == 1) {
/* Try to find a name early so we can log errors with a module name */
info->index.info = info_idx;
info->name = get_modinfo(info, "name");
}
if (num_sym_secs != 1) {
pr_warn("%s: module has no symbols (stripped?)\n",
info->name ?: "(missing .modinfo section or name field)");
goto no_exec;
}
/* Sets internal symbols and strings. */
info->index.sym = sym_idx;
shdr = &info->sechdrs[sym_idx];
info->index.str = shdr->sh_link;
info->strtab = (char *)info->hdr + info->sechdrs[info->index.str].sh_offset;
/*
* The ".gnu.linkonce.this_module" ELF section is special. It is
* what modpost uses to refer to __this_module and let's use rely
* on THIS_MODULE to point to &__this_module properly. The kernel's
* modpost declares it on each modules's *.mod.c file. If the struct
* module of the kernel changes a full kernel rebuild is required.
*
* We have a few expectaions for this special section, the following
* code validates all this for us:
*
* o Only one section must exist
* o We expect the kernel to always have to allocate it: SHF_ALLOC
* o The section size must match the kernel's run time's struct module
* size
*/
if (num_mod_secs != 1) {
pr_err("module %s: Only one .gnu.linkonce.this_module section must exist.\n",
info->name ?: "(missing .modinfo section or name field)");
goto no_exec;
}
shdr = &info->sechdrs[mod_idx];
/*
* This is already implied on the switch above, however let's be
* pedantic about it.
*/
if (shdr->sh_type == SHT_NOBITS) {
pr_err("module %s: .gnu.linkonce.this_module section must have a size set\n",
info->name ?: "(missing .modinfo section or name field)");
goto no_exec;
}
if (!(shdr->sh_flags & SHF_ALLOC)) {
pr_err("module %s: .gnu.linkonce.this_module must occupy memory during process execution\n",
info->name ?: "(missing .modinfo section or name field)");
goto no_exec;
}
if (shdr->sh_size != sizeof(struct module)) {
pr_err("module %s: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time\n",
info->name ?: "(missing .modinfo section or name field)");
goto no_exec;
}
info->index.mod = mod_idx;
/* This is temporary: point mod into copy of data. */
info->mod = (void *)info->hdr + shdr->sh_offset;
/*
* If we didn't load the .modinfo 'name' field earlier, fall back to
* on-disk struct mod 'name' field.
*/
if (!info->name)
info->name = info->mod->name;
if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
info->index.vers = 0; /* Pretend no __versions section! */
else
info->index.vers = find_sec(info, "__versions");
info->index.pcpu = find_pcpusec(info);
return 0;
no_exec:
return -ENOEXEC;
}
#define COPY_CHUNK_SIZE (16*PAGE_SIZE)
static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len)
{
do {
unsigned long n = min(len, COPY_CHUNK_SIZE);
if (copy_from_user(dst, usrc, n) != 0)
return -EFAULT;
cond_resched();
dst += n;
usrc += n;
len -= n;
} while (len);
return 0;
}
static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
{
if (!get_modinfo(info, "livepatch"))
/* Nothing more to do */
return 0;
if (set_livepatch_module(mod))
return 0;
pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
mod->name);
return -ENOEXEC;
}
static void check_modinfo_retpoline(struct module *mod, struct load_info *info)
{
if (retpoline_module_ok(get_modinfo(info, "retpoline")))
return;
pr_warn("%s: loading module not compiled with retpoline compiler.\n",
mod->name);
}
/* Sets info->hdr and info->len. */
static int copy_module_from_user(const void __user *umod, unsigned long len,
struct load_info *info)
{
int err;
info->len = len;
if (info->len < sizeof(*(info->hdr)))
return -ENOEXEC;
err = security_kernel_load_data(LOADING_MODULE, true);
if (err)
return err;
/* Suck in entire file: we'll want most of it. */
info->hdr = __vmalloc(info->len, GFP_KERNEL | __GFP_NOWARN);
if (!info->hdr)
return -ENOMEM;
if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
err = -EFAULT;
goto out;
}
err = security_kernel_post_load_data((char *)info->hdr, info->len,
LOADING_MODULE, "init_module");
out:
if (err)
vfree(info->hdr);
return err;
}
static void free_copy(struct load_info *info, int flags)
{
if (flags & MODULE_INIT_COMPRESSED_FILE)
module_decompress_cleanup(info);
else
vfree(info->hdr);
}
static int rewrite_section_headers(struct load_info *info, int flags)
{
unsigned int i;
/* This should always be true, but let's be sure. */
info->sechdrs[0].sh_addr = 0;
for (i = 1; i < info->hdr->e_shnum; i++) {
Elf_Shdr *shdr = &info->sechdrs[i];
/*
* Mark all sections sh_addr with their address in the
* temporary image.
*/
shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
}
/* Track but don't keep modinfo and version sections. */
info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
return 0;
}
/*
* These calls taint the kernel depending certain module circumstances */
static void module_augment_kernel_taints(struct module *mod, struct load_info *info)
{
int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);
if (!get_modinfo(info, "intree")) {
if (!test_taint(TAINT_OOT_MODULE))
pr_warn("%s: loading out-of-tree module taints kernel.\n",
mod->name);
add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
}
check_modinfo_retpoline(mod, info);
if (get_modinfo(info, "staging")) {
add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
pr_warn("%s: module is from the staging directory, the quality "
"is unknown, you have been warned.\n", mod->name);
}
if (is_livepatch_module(mod)) {
add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
mod->name);
}
module_license_taint_check(mod, get_modinfo(info, "license"));
if (get_modinfo(info, "test")) {
if (!test_taint(TAINT_TEST))
pr_warn("%s: loading test module taints kernel.\n",
mod->name);
add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK);
}
#ifdef CONFIG_MODULE_SIG
mod->sig_ok = info->sig_ok;
if (!mod->sig_ok) {
pr_notice_once("%s: module verification failed: signature "
"and/or required key missing - tainting "
"kernel\n", mod->name);
add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
}
#endif
/*
* ndiswrapper is under GPL by itself, but loads proprietary modules.
* Don't use add_taint_module(), as it would prevent ndiswrapper from
* using GPL-only symbols it needs.
*/
if (strcmp(mod->name, "ndiswrapper") == 0)
add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
/* driverloader was caught wrongly pretending to be under GPL */
if (strcmp(mod->name, "driverloader") == 0)
add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
LOCKDEP_NOW_UNRELIABLE);
/* lve claims to be GPL but upstream won't provide source */
if (strcmp(mod->name, "lve") == 0)
add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
LOCKDEP_NOW_UNRELIABLE);
if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
pr_warn("%s: module license taints kernel.\n", mod->name);
}
static int check_modinfo(struct module *mod, struct load_info *info, int flags)
{
const char *modmagic = get_modinfo(info, "vermagic");
int err;
if (flags & MODULE_INIT_IGNORE_VERMAGIC)
modmagic = NULL;
/* This is allowed: modprobe --force will invalidate it. */
if (!modmagic) {
err = try_to_force_load(mod, "bad vermagic");
if (err)
return err;
} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
pr_err("%s: version magic '%s' should be '%s'\n",
info->name, modmagic, vermagic);
return -ENOEXEC;
}
err = check_modinfo_livepatch(mod, info);
if (err)
return err;
return 0;
}
static int find_module_sections(struct module *mod, struct load_info *info)
{
mod->kp = section_objs(info, "__param",
sizeof(*mod->kp), &mod->num_kp);
mod->syms = section_objs(info, "__ksymtab",
sizeof(*mod->syms), &mod->num_syms);
mod->crcs = section_addr(info, "__kcrctab");
mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
sizeof(*mod->gpl_syms),
&mod->num_gpl_syms);
mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
#ifdef CONFIG_CONSTRUCTORS
mod->ctors = section_objs(info, ".ctors",
sizeof(*mod->ctors), &mod->num_ctors);
if (!mod->ctors)
mod->ctors = section_objs(info, ".init_array",
sizeof(*mod->ctors), &mod->num_ctors);
else if (find_sec(info, ".init_array")) {
/*
* This shouldn't happen with same compiler and binutils
* building all parts of the module.
*/
pr_warn("%s: has both .ctors and .init_array.\n",
mod->name);
return -EINVAL;
}
#endif
mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
&mod->noinstr_text_size);
#ifdef CONFIG_TRACEPOINTS
mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
sizeof(*mod->tracepoints_ptrs),
&mod->num_tracepoints);
#endif
#ifdef CONFIG_TREE_SRCU
mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
sizeof(*mod->srcu_struct_ptrs),
&mod->num_srcu_structs);
#endif
#ifdef CONFIG_BPF_EVENTS
mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
sizeof(*mod->bpf_raw_events),
&mod->num_bpf_raw_events);
#endif
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);
#endif
#ifdef CONFIG_JUMP_LABEL
mod->jump_entries = section_objs(info, "__jump_table",
sizeof(*mod->jump_entries),
&mod->num_jump_entries);
#endif
#ifdef CONFIG_EVENT_TRACING
mod->trace_events = section_objs(info, "_ftrace_events",
sizeof(*mod->trace_events),
&mod->num_trace_events);
mod->trace_evals = section_objs(info, "_ftrace_eval_map",
sizeof(*mod->trace_evals),
&mod->num_trace_evals);
#endif
#ifdef CONFIG_TRACING
mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
sizeof(*mod->trace_bprintk_fmt_start),
&mod->num_trace_bprintk_fmt);
#endif
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
/* sechdrs[0].sh_size is always zero */
mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
sizeof(*mod->ftrace_callsites),
&mod->num_ftrace_callsites);
#endif
#ifdef CONFIG_FUNCTION_ERROR_INJECTION
mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
sizeof(*mod->ei_funcs),
&mod->num_ei_funcs);
#endif
#ifdef CONFIG_KPROBES
mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
&mod->kprobes_text_size);
mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
sizeof(unsigned long),
&mod->num_kprobe_blacklist);
#endif
#ifdef CONFIG_PRINTK_INDEX
mod->printk_index_start = section_objs(info, ".printk_index",
sizeof(*mod->printk_index_start),
&mod->printk_index_size);
#endif
#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
mod->static_call_sites = section_objs(info, ".static_call_sites",
sizeof(*mod->static_call_sites),
&mod->num_static_call_sites);
#endif
#if IS_ENABLED(CONFIG_KUNIT)
mod->kunit_suites = section_objs(info, ".kunit_test_suites",
sizeof(*mod->kunit_suites),
&mod->num_kunit_suites);
mod->kunit_init_suites = section_objs(info, ".kunit_init_test_suites",
sizeof(*mod->kunit_init_suites),
&mod->num_kunit_init_suites);
#endif
mod->extable = section_objs(info, "__ex_table",
sizeof(*mod->extable), &mod->num_exentries);
if (section_addr(info, "__obsparm"))
pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
#ifdef CONFIG_DYNAMIC_DEBUG_CORE
mod->dyndbg_info.descs = section_objs(info, "__dyndbg",
sizeof(*mod->dyndbg_info.descs),
&mod->dyndbg_info.num_descs);
mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes",
sizeof(*mod->dyndbg_info.classes),
&mod->dyndbg_info.num_classes);
#endif
return 0;
}
static int move_module(struct module *mod, struct load_info *info)
{
int i;
void *ptr;
enum mod_mem_type t = 0;
int ret = -ENOMEM;
for_each_mod_mem_type(type) {
if (!mod->mem[type].size) {
mod->mem[type].base = NULL;
continue;
}
mod->mem[type].size = PAGE_ALIGN(mod->mem[type].size);
ptr = module_memory_alloc(mod->mem[type].size, type);
/*
* The pointer to these blocks of memory are stored on the module
* structure and we keep that around so long as the module is
* around. We only free that memory when we unload the module.
* Just mark them as not being a leak then. The .init* ELF
* sections *do* get freed after boot so we *could* treat them
* slightly differently with kmemleak_ignore() and only grey
* them out as they work as typical memory allocations which
* *do* eventually get freed, but let's just keep things simple
* and avoid *any* false positives.
*/
kmemleak_not_leak(ptr);
if (!ptr) {
t = type;
goto out_enomem;
}
memset(ptr, 0, mod->mem[type].size);
mod->mem[type].base = ptr;
}
/* Transfer each section which specifies SHF_ALLOC */
pr_debug("Final section addresses for %s:\n", mod->name);
for (i = 0; i < info->hdr->e_shnum; i++) {
void *dest;
Elf_Shdr *shdr = &info->sechdrs[i];
enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
if (!(shdr->sh_flags & SHF_ALLOC))
continue;
dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK);
if (shdr->sh_type != SHT_NOBITS) {
/*
* Our ELF checker already validated this, but let's
* be pedantic and make the goal clearer. We actually
* end up copying over all modifications made to the
* userspace copy of the entire struct module.
*/
if (i == info->index.mod &&
(WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) {
ret = -ENOEXEC;
goto out_enomem;
}
memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
}
/*
* Update the userspace copy's ELF section address to point to
* our newly allocated memory as a pure convenience so that
* users of info can keep taking advantage and using the newly
* minted official memory area.
*/
shdr->sh_addr = (unsigned long)dest;
pr_debug("\t0x%lx 0x%.8lx %s\n", (long)shdr->sh_addr,
(long)shdr->sh_size, info->secstrings + shdr->sh_name);
}
return 0;
out_enomem:
for (t--; t >= 0; t--)
module_memory_free(mod->mem[t].base, t);
return ret;
}
static int check_export_symbol_versions(struct module *mod)
{
#ifdef CONFIG_MODVERSIONS
if ((mod->num_syms && !mod->crcs) ||
(mod->num_gpl_syms && !mod->gpl_crcs)) {
return try_to_force_load(mod,
"no versions for exported symbols");
}
#endif
return 0;
}
static void flush_module_icache(const struct module *mod)
{
/*
* Flush the instruction cache, since we've played with text.
* Do it before processing of module parameters, so the module
* can provide parameter accessor functions of its own.
*/
for_each_mod_mem_type(type) {
const struct module_memory *mod_mem = &mod->mem[type];
if (mod_mem->size) {
flush_icache_range((unsigned long)mod_mem->base,
(unsigned long)mod_mem->base + mod_mem->size);
}
}
}
bool __weak module_elf_check_arch(Elf_Ehdr *hdr)
{
return true;
}
int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
Elf_Shdr *sechdrs,
char *secstrings,
struct module *mod)
{
return 0;
}
/* module_blacklist is a comma-separated list of module names */
static char *module_blacklist;
static bool blacklisted(const char *module_name)
{
const char *p;
size_t len;
if (!module_blacklist)
return false;
for (p = module_blacklist; *p; p += len) {
len = strcspn(p, ",");
if (strlen(module_name) == len && !memcmp(module_name, p, len))
return true;
if (p[len] == ',')
len++;
}
return false;
}
core_param(module_blacklist, module_blacklist, charp, 0400);
static struct module *layout_and_allocate(struct load_info *info, int flags)
{
struct module *mod;
unsigned int ndx;
int err;
/* Allow arches to frob section contents and sizes. */
err = module_frob_arch_sections(info->hdr, info->sechdrs,
info->secstrings, info->mod);
if (err < 0)
return ERR_PTR(err);
err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
info->secstrings, info->mod);
if (err < 0)
return ERR_PTR(err);
/* We will do a special allocation for per-cpu sections later. */
info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
/*
* Mark ro_after_init section with SHF_RO_AFTER_INIT so that
* layout_sections() can put it in the right place.
* Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
*/
ndx = find_sec(info, ".data..ro_after_init");
if (ndx)
info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
/*
* Mark the __jump_table section as ro_after_init as well: these data
* structures are never modified, with the exception of entries that
* refer to code in the __init section, which are annotated as such
* at module load time.
*/
ndx = find_sec(info, "__jump_table");
if (ndx)
info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
/*
* Determine total sizes, and put offsets in sh_entsize. For now
* this is done generically; there doesn't appear to be any
* special cases for the architectures.
*/
layout_sections(info->mod, info);
layout_symtab(info->mod, info);
/* Allocate and move to the final place */
err = move_module(info->mod, info);
if (err)
return ERR_PTR(err);
/* Module has been copied to its final place now: return it. */
mod = (void *)info->sechdrs[info->index.mod].sh_addr;
kmemleak_load_module(mod, info);
return mod;
}
/* mod is no longer valid after this! */
static void module_deallocate(struct module *mod, struct load_info *info)
{
percpu_modfree(mod);
module_arch_freeing_init(mod);
free_mod_mem(mod);
}
int __weak module_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs,
struct module *me)
{
return 0;
}
static int post_relocation(struct module *mod, const struct load_info *info)
{
/* Sort exception table now relocations are done. */
sort_extable(mod->extable, mod->extable + mod->num_exentries);
/* Copy relocated percpu area over. */
percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
info->sechdrs[info->index.pcpu].sh_size);
/* Setup kallsyms-specific fields. */
add_kallsyms(mod, info);
/* Arch-specific module finalizing. */
return module_finalize(info->hdr, info->sechdrs, mod);
}
/* Call module constructors. */
static void do_mod_ctors(struct module *mod)
{
#ifdef CONFIG_CONSTRUCTORS
unsigned long i;
for (i = 0; i < mod->num_ctors; i++)
mod->ctors[i]();
#endif
}
/* For freeing module_init on success, in case kallsyms traversing */
struct mod_initfree {
struct llist_node node;
void *init_text;
void *init_data;
void *init_rodata;
};
static void do_free_init(struct work_struct *w)
{
struct llist_node *pos, *n, *list;
struct mod_initfree *initfree;
list = llist_del_all(&init_free_list);
synchronize_rcu();
llist_for_each_safe(pos, n, list) {
initfree = container_of(pos, struct mod_initfree, node);
module_memfree(initfree->init_text);
module_memfree(initfree->init_data);
module_memfree(initfree->init_rodata);
kfree(initfree);
}
}
void flush_module_init_free_work(void)
{
flush_work(&init_free_wq);
}
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "module."
/* Default value for module->async_probe_requested */
static bool async_probe;
module_param(async_probe, bool, 0644);
/*
* This is where the real work happens.
*
* Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
* helper command 'lx-symbols'.
*/
static noinline int do_init_module(struct module *mod)
{
int ret = 0;
struct mod_initfree *freeinit;
#if defined(CONFIG_MODULE_STATS)
unsigned int text_size = 0, total_size = 0;
for_each_mod_mem_type(type) {
const struct module_memory *mod_mem = &mod->mem[type];
if (mod_mem->size) {
total_size += mod_mem->size;
if (type == MOD_TEXT || type == MOD_INIT_TEXT)
text_size += mod_mem->size;
}
}
#endif
freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
if (!freeinit) {
ret = -ENOMEM;
goto fail;
}
freeinit->init_text = mod->mem[MOD_INIT_TEXT].base;
freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
do_mod_ctors(mod);
/* Start the module */
if (mod->init != NULL)
ret = do_one_initcall(mod->init);
if (ret < 0) {
goto fail_free_freeinit;
}
if (ret > 0) {
pr_warn("%s: '%s'->init suspiciously returned %d, it should "
"follow 0/-E convention\n"
"%s: loading module anyway...\n",
__func__, mod->name, ret, __func__);
dump_stack();
}
/* Now it's a first class citizen! */
mod->state = MODULE_STATE_LIVE;
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_LIVE, mod);
/* Delay uevent until module has finished its init routine */
kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
/*
* We need to finish all async code before the module init sequence
* is done. This has potential to deadlock if synchronous module
* loading is requested from async (which is not allowed!).
*
* See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
* request_module() from async workers") for more details.
*/
if (!mod->async_probe_requested)
async_synchronize_full();
ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base,
mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size);
mutex_lock(&module_mutex);
/* Drop initial reference. */
module_put(mod);
trim_init_extable(mod);
#ifdef CONFIG_KALLSYMS
/* Switch to core kallsyms now init is done: kallsyms may be walking! */
rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
#endif
ret = module_enable_rodata_ro(mod, true);
if (ret)
goto fail_mutex_unlock;
mod_tree_remove_init(mod);
module_arch_freeing_init(mod);
for_class_mod_mem_type(type, init) {
mod->mem[type].base = NULL;
mod->mem[type].size = 0;
}
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */
mod->btf_data = NULL;
#endif
/*
* We want to free module_init, but be aware that kallsyms may be
* walking this with preempt disabled. In all the failure paths, we
* call synchronize_rcu(), but we don't want to slow down the success
* path. module_memfree() cannot be called in an interrupt, so do the
* work and call synchronize_rcu() in a work queue.
*
* Note that module_alloc() on most architectures creates W+X page
* mappings which won't be cleaned up until do_free_init() runs. Any
* code such as mark_rodata_ro() which depends on those mappings to
* be cleaned up needs to sync with the queued work by invoking
* flush_module_init_free_work().
*/
if (llist_add(&freeinit->node, &init_free_list))
schedule_work(&init_free_wq);
mutex_unlock(&module_mutex);
wake_up_all(&module_wq);
mod_stat_add_long(text_size, &total_text_size);
mod_stat_add_long(total_size, &total_mod_size);
mod_stat_inc(&modcount);
return 0;
fail_mutex_unlock:
mutex_unlock(&module_mutex);
fail_free_freeinit:
kfree(freeinit);
fail:
/* Try to protect us from buggy refcounters. */
mod->state = MODULE_STATE_GOING;
synchronize_rcu();
module_put(mod);
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_GOING, mod);
klp_module_going(mod);
ftrace_release_mod(mod);
free_module(mod);
wake_up_all(&module_wq);
return ret;
}
static int may_init_module(void)
{
if (!capable(CAP_SYS_MODULE) || modules_disabled)
return -EPERM;
return 0;
}
/* Is this module of this name done loading? No locks held. */
static bool finished_loading(const char *name)
{
struct module *mod;
bool ret;
/*
* The module_mutex should not be a heavily contended lock;
* if we get the occasional sleep here, we'll go an extra iteration
* in the wait_event_interruptible(), which is harmless.
*/
sched_annotate_sleep();
mutex_lock(&module_mutex);
mod = find_module_all(name, strlen(name), true);
ret = !mod || mod->state == MODULE_STATE_LIVE
|| mod->state == MODULE_STATE_GOING;
mutex_unlock(&module_mutex);
return ret;
}
/* Must be called with module_mutex held */
static int module_patient_check_exists(const char *name,
enum fail_dup_mod_reason reason)
{
struct module *old;
int err = 0;
old = find_module_all(name, strlen(name), true);
if (old == NULL)
return 0;
if (old->state == MODULE_STATE_COMING ||
old->state == MODULE_STATE_UNFORMED) {
/* Wait in case it fails to load. */
mutex_unlock(&module_mutex);
err = wait_event_interruptible(module_wq,
finished_loading(name));
mutex_lock(&module_mutex);
if (err)
return err;
/* The module might have gone in the meantime. */
old = find_module_all(name, strlen(name), true);
}
if (try_add_failed_module(name, reason))
pr_warn("Could not add fail-tracking for module: %s\n", name);
/*
* We are here only when the same module was being loaded. Do
* not try to load it again right now. It prevents long delays
* caused by serialized module load failures. It might happen
* when more devices of the same type trigger load of
* a particular module.
*/
if (old && old->state == MODULE_STATE_LIVE)
return -EEXIST;
return -EBUSY;
}
/*
* We try to place it in the list now to make sure it's unique before
* we dedicate too many resources. In particular, temporary percpu
* memory exhaustion.
*/
static int add_unformed_module(struct module *mod)
{
int err;
mod->state = MODULE_STATE_UNFORMED;
mutex_lock(&module_mutex);
err = module_patient_check_exists(mod->name, FAIL_DUP_MOD_LOAD);
if (err)
goto out;
mod_update_bounds(mod);
list_add_rcu(&mod->list, &modules);
mod_tree_insert(mod);
err = 0;
out:
mutex_unlock(&module_mutex);
return err;
}
static int complete_formation(struct module *mod, struct load_info *info)
{
int err;
mutex_lock(&module_mutex);
/* Find duplicate symbols (must be called under lock). */
err = verify_exported_symbols(mod);
if (err < 0)
goto out;
/* These rely on module_mutex for list integrity. */
module_bug_finalize(info->hdr, info->sechdrs, mod);
module_cfi_finalize(info->hdr, info->sechdrs, mod);
err = module_enable_rodata_ro(mod, false);
if (err)
goto out_strict_rwx;
err = module_enable_data_nx(mod);
if (err)
goto out_strict_rwx;
err = module_enable_text_rox(mod);
if (err)
goto out_strict_rwx;
/*
* Mark state as coming so strong_try_module_get() ignores us,
* but kallsyms etc. can see us.
*/
mod->state = MODULE_STATE_COMING;
mutex_unlock(&module_mutex);
return 0;
out_strict_rwx:
module_bug_cleanup(mod);
out:
mutex_unlock(&module_mutex);
return err;
}
static int prepare_coming_module(struct module *mod)
{
int err;
ftrace_module_enable(mod);
err = klp_module_coming(mod);
if (err)
return err;
err = blocking_notifier_call_chain_robust(&module_notify_list,
MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
err = notifier_to_errno(err);
if (err)
klp_module_going(mod);
return err;
}
static int unknown_module_param_cb(char *param, char *val, const char *modname,
void *arg)
{
struct module *mod = arg;
int ret;
if (strcmp(param, "async_probe") == 0) {
if (kstrtobool(val, &mod->async_probe_requested))
mod->async_probe_requested = true;
return 0;
}
/* Check for magic 'dyndbg' arg */
ret = ddebug_dyndbg_module_param_cb(param, val, modname);
if (ret != 0)
pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
return 0;
}
/* Module within temporary copy, this doesn't do any allocation */
static int early_mod_check(struct load_info *info, int flags)
{
int err;
/*
* Now that we know we have the correct module name, check
* if it's blacklisted.
*/
if (blacklisted(info->name)) {
pr_err("Module %s is blacklisted\n", info->name);
return -EPERM;
}
err = rewrite_section_headers(info, flags);
if (err)
return err;
/* Check module struct version now, before we try to use module. */
if (!check_modstruct_version(info, info->mod))
return -ENOEXEC;
err = check_modinfo(info->mod, info, flags);
if (err)
return err;
mutex_lock(&module_mutex);
err = module_patient_check_exists(info->mod->name, FAIL_DUP_MOD_BECOMING);
mutex_unlock(&module_mutex);
return err;
}
/*
* Allocate and load the module: note that size of section 0 is always
* zero, and we rely on this for optional sections.
*/
static int load_module(struct load_info *info, const char __user *uargs,
int flags)
{
struct module *mod;
bool module_allocated = false;
long err = 0;
char *after_dashes;
/*
* Do the signature check (if any) first. All that
* the signature check needs is info->len, it does
* not need any of the section info. That can be
* set up later. This will minimize the chances
* of a corrupt module causing problems before
* we even get to the signature check.
*
* The check will also adjust info->len by stripping
* off the sig length at the end of the module, making
* checks against info->len more correct.
*/
err = module_sig_check(info, flags);
if (err)
goto free_copy;
/*
* Do basic sanity checks against the ELF header and
* sections. Cache useful sections and set the
* info->mod to the userspace passed struct module.
*/
err = elf_validity_cache_copy(info, flags);
if (err)
goto free_copy;
err = early_mod_check(info, flags);
if (err)
goto free_copy;
/* Figure out module layout, and allocate all the memory. */
mod = layout_and_allocate(info, flags);
if (IS_ERR(mod)) {
err = PTR_ERR(mod);
goto free_copy;
}
module_allocated = true;
audit_log_kern_module(mod->name);
/* Reserve our place in the list. */
err = add_unformed_module(mod);
if (err)
goto free_module;
/*
* We are tainting your kernel if your module gets into
* the modules linked list somehow.
*/
module_augment_kernel_taints(mod, info);
/* To avoid stressing percpu allocator, do this once we're unique. */
err = percpu_modalloc(mod, info);
if (err)
goto unlink_mod;
/* Now module is in final location, initialize linked lists, etc. */
err = module_unload_init(mod);
if (err)
goto unlink_mod;
init_param_lock(mod);
/*
* Now we've got everything in the final locations, we can
* find optional sections.
*/
err = find_module_sections(mod, info);
if (err)
goto free_unload;
err = check_export_symbol_versions(mod);
if (err)
goto free_unload;
/* Set up MODINFO_ATTR fields */
setup_modinfo(mod, info);
/* Fix up syms, so that st_value is a pointer to location. */
err = simplify_symbols(mod, info);
if (err < 0)
goto free_modinfo;
err = apply_relocations(mod, info);
if (err < 0)
goto free_modinfo;
err = post_relocation(mod, info);
if (err < 0)
goto free_modinfo;
flush_module_icache(mod);
/* Now copy in args */
mod->args = strndup_user(uargs, ~0UL >> 1);
if (IS_ERR(mod->args)) {
err = PTR_ERR(mod->args);
goto free_arch_cleanup;
}
init_build_id(mod, info);
/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
ftrace_module_init(mod);
/* Finally it's fully formed, ready to start executing. */
err = complete_formation(mod, info);
if (err)
goto ddebug_cleanup;
err = prepare_coming_module(mod);
if (err)
goto bug_cleanup;
mod->async_probe_requested = async_probe;
/* Module is ready to execute: parsing args may do that. */
after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
-32768, 32767, mod,
unknown_module_param_cb);
if (IS_ERR(after_dashes)) {
err = PTR_ERR(after_dashes);
goto coming_cleanup;
} else if (after_dashes) {
pr_warn("%s: parameters '%s' after `--' ignored\n",
mod->name, after_dashes);
}
/* Link in to sysfs. */
err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
if (err < 0)
goto coming_cleanup;
if (is_livepatch_module(mod)) {
err = copy_module_elf(mod, info);
if (err < 0)
goto sysfs_cleanup;
}
/* Get rid of temporary copy. */
free_copy(info, flags);
/* Done! */
trace_module_load(mod);
return do_init_module(mod);
sysfs_cleanup:
mod_sysfs_teardown(mod);
coming_cleanup:
mod->state = MODULE_STATE_GOING;
destroy_params(mod->kp, mod->num_kp);
blocking_notifier_call_chain(&module_notify_list,
MODULE_STATE_GOING, mod);
klp_module_going(mod);
bug_cleanup:
mod->state = MODULE_STATE_GOING;
/* module_bug_cleanup needs module_mutex protection */
mutex_lock(&module_mutex);
module_bug_cleanup(mod);
mutex_unlock(&module_mutex);
ddebug_cleanup:
ftrace_release_mod(mod);
synchronize_rcu();
kfree(mod->args);
free_arch_cleanup:
module_arch_cleanup(mod);
free_modinfo:
free_modinfo(mod);
free_unload:
module_unload_free(mod);
unlink_mod:
mutex_lock(&module_mutex);
/* Unlink carefully: kallsyms could be walking list. */
list_del_rcu(&mod->list);
mod_tree_remove(mod);
wake_up_all(&module_wq);
/* Wait for RCU-sched synchronizing before releasing mod->list. */
synchronize_rcu();
mutex_unlock(&module_mutex);
free_module:
mod_stat_bump_invalid(info, flags);
/* Free lock-classes; relies on the preceding sync_rcu() */
for_class_mod_mem_type(type, core_data) {
lockdep_free_key_range(mod->mem[type].base,
mod->mem[type].size);
}
module_deallocate(mod, info);
free_copy:
/*
* The info->len is always set. We distinguish between
* failures once the proper module was allocated and
* before that.
*/
if (!module_allocated)
mod_stat_bump_becoming(info, flags);
free_copy(info, flags);
return err;
}
SYSCALL_DEFINE3(init_module, void __user *, umod,
unsigned long, len, const char __user *, uargs)
{
int err;
struct load_info info = { };
err = may_init_module();
if (err)
return err;
pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
umod, len, uargs);
err = copy_module_from_user(umod, len, &info);
if (err) {
mod_stat_inc(&failed_kreads);
mod_stat_add_long(len, &invalid_kread_bytes);
return err;
}
return load_module(&info, uargs, 0);
}
struct idempotent {
const void *cookie;
struct hlist_node entry;
struct completion complete;
int ret;
};
#define IDEM_HASH_BITS 8
static struct hlist_head idem_hash[1 << IDEM_HASH_BITS];
static DEFINE_SPINLOCK(idem_lock);
static bool idempotent(struct idempotent *u, const void *cookie)
{
int hash = hash_ptr(cookie, IDEM_HASH_BITS);
struct hlist_head *head = idem_hash + hash;
struct idempotent *existing;
bool first;
u->ret = 0;
u->cookie = cookie;
init_completion(&u->complete);
spin_lock(&idem_lock);
first = true;
hlist_for_each_entry(existing, head, entry) {
if (existing->cookie != cookie)
continue;
first = false;
break;
}
hlist_add_head(&u->entry, idem_hash + hash);
spin_unlock(&idem_lock);
return !first;
}
/*
* We were the first one with 'cookie' on the list, and we ended
* up completing the operation. We now need to walk the list,
* remove everybody - which includes ourselves - fill in the return
* value, and then complete the operation.
*/
static int idempotent_complete(struct idempotent *u, int ret)
{
const void *cookie = u->cookie;
int hash = hash_ptr(cookie, IDEM_HASH_BITS);
struct hlist_head *head = idem_hash + hash;
struct hlist_node *next;
struct idempotent *pos;
spin_lock(&idem_lock);
hlist_for_each_entry_safe(pos, next, head, entry) {
if (pos->cookie != cookie)
continue;
hlist_del(&pos->entry);
pos->ret = ret;
complete(&pos->complete);
}
spin_unlock(&idem_lock);
return ret;
}
static int init_module_from_file(struct file *f, const char __user * uargs, int flags)
{
struct load_info info = { };
void *buf = NULL;
int len;
len = kernel_read_file(f, 0, &buf, INT_MAX, NULL, READING_MODULE);
if (len < 0) {
mod_stat_inc(&failed_kreads);
return len;
}
if (flags & MODULE_INIT_COMPRESSED_FILE) {
int err = module_decompress(&info, buf, len);
vfree(buf); /* compressed data is no longer needed */
if (err) {
mod_stat_inc(&failed_decompress);
mod_stat_add_long(len, &invalid_decompress_bytes);
return err;
}
} else {
info.hdr = buf;
info.len = len;
}
return load_module(&info, uargs, flags);
}
static int idempotent_init_module(struct file *f, const char __user * uargs, int flags)
{
struct idempotent idem;
if (!f || !(f->f_mode & FMODE_READ))
return -EBADF;
/* See if somebody else is doing the operation? */
if (idempotent(&idem, file_inode(f))) {
wait_for_completion(&idem.complete);
return idem.ret;
}
/* Otherwise, we'll do it and complete others */
return idempotent_complete(&idem,
init_module_from_file(f, uargs, flags));
}
SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
{
int err;
struct fd f;
err = may_init_module();
if (err)
return err;
pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
|MODULE_INIT_IGNORE_VERMAGIC
|MODULE_INIT_COMPRESSED_FILE))
return -EINVAL;
f = fdget(fd);
err = idempotent_init_module(f.file, uargs, flags);
fdput(f);
return err;
}
/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
char *module_flags(struct module *mod, char *buf, bool show_state)
{
int bx = 0;
BUG_ON(mod->state == MODULE_STATE_UNFORMED);
if (!mod->taints && !show_state)
goto out;
if (mod->taints ||
mod->state == MODULE_STATE_GOING ||
mod->state == MODULE_STATE_COMING) {
buf[bx++] = '(';
bx += module_flags_taint(mod->taints, buf + bx);
/* Show a - for module-is-being-unloaded */
if (mod->state == MODULE_STATE_GOING && show_state)
buf[bx++] = '-';
/* Show a + for module-is-being-loaded */
if (mod->state == MODULE_STATE_COMING && show_state)
buf[bx++] = '+';
buf[bx++] = ')';
}
out:
buf[bx] = '\0';
return buf;
}
/* Given an address, look for it in the module exception tables. */
const struct exception_table_entry *search_module_extables(unsigned long addr)
{
const struct exception_table_entry *e = NULL;
struct module *mod;
preempt_disable();
mod = __module_address(addr);
if (!mod)
goto out;
if (!mod->num_exentries)
goto out;
e = search_extable(mod->extable,
mod->num_exentries,
addr);
out:
preempt_enable();
/*
* Now, if we found one, we are running inside it now, hence
* we cannot unload the module, hence no refcnt needed.
*/
return e;
}
/**
* is_module_address() - is this address inside a module?
* @addr: the address to check.
*
* See is_module_text_address() if you simply want to see if the address
* is code (not data).
*/
bool is_module_address(unsigned long addr)
{
bool ret;
preempt_disable();
ret = __module_address(addr) != NULL;
preempt_enable(); return ret;
}
/**
* __module_address() - get the module which contains an address.
* @addr: the address.
*
* Must be called with preempt disabled or module mutex held so that
* module doesn't get freed during this.
*/
struct module *__module_address(unsigned long addr)
{
struct module *mod;
if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max)
goto lookup;
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max)
goto lookup;
#endif
return NULL;
lookup:
module_assert_mutex_or_preempt(); mod = mod_find(addr, &mod_tree);
if (mod) {
BUG_ON(!within_module(addr, mod)); if (mod->state == MODULE_STATE_UNFORMED)
mod = NULL;
}
return mod;
}
/**
* is_module_text_address() - is this address inside module code?
* @addr: the address to check.
*
* See is_module_address() if you simply want to see if the address is
* anywhere in a module. See kernel_text_address() for testing if an
* address corresponds to kernel or module code.
*/
bool is_module_text_address(unsigned long addr)
{
bool ret;
preempt_disable(); ret = __module_text_address(addr) != NULL; preempt_enable(); return ret;
}
/**
* __module_text_address() - get the module whose code contains an address.
* @addr: the address.
*
* Must be called with preempt disabled or module mutex held so that
* module doesn't get freed during this.
*/
struct module *__module_text_address(unsigned long addr)
{
struct module *mod = __module_address(addr);
if (mod) {
/* Make sure it's within the text section. */
if (!within_module_mem_type(addr, mod, MOD_TEXT) && !within_module_mem_type(addr, mod, MOD_INIT_TEXT))
mod = NULL;
}
return mod;
}
/* Don't grab lock, we're oopsing. */
void print_modules(void)
{
struct module *mod;
char buf[MODULE_FLAGS_BUF_SIZE];
printk(KERN_DEFAULT "Modules linked in:");
/* Most callers should already have preempt disabled, but make sure */
preempt_disable();
list_for_each_entry_rcu(mod, &modules, list) {
if (mod->state == MODULE_STATE_UNFORMED)
continue;
pr_cont(" %s%s", mod->name, module_flags(mod, buf, true));
}
print_unloaded_tainted_modules();
preempt_enable();
if (last_unloaded_module.name[0])
pr_cont(" [last unloaded: %s%s]", last_unloaded_module.name,
last_unloaded_module.taints);
pr_cont("\n");
}
#ifdef CONFIG_MODULE_DEBUGFS
struct dentry *mod_debugfs_root;
static int module_debugfs_init(void)
{
mod_debugfs_root = debugfs_create_dir("modules", NULL);
return 0;
}
module_init(module_debugfs_init);
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 1993 Linus Torvalds
* Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
* SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
* Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
* Numa awareness, Christoph Lameter, SGI, June 2005
* Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019
*/
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/highmem.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/set_memory.h>
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/list.h>
#include <linux/notifier.h>
#include <linux/rbtree.h>
#include <linux/xarray.h>
#include <linux/io.h>
#include <linux/rcupdate.h>
#include <linux/pfn.h>
#include <linux/kmemleak.h>
#include <linux/atomic.h>
#include <linux/compiler.h>
#include <linux/memcontrol.h>
#include <linux/llist.h>
#include <linux/uio.h>
#include <linux/bitops.h>
#include <linux/rbtree_augmented.h>
#include <linux/overflow.h>
#include <linux/pgtable.h>
#include <linux/hugetlb.h>
#include <linux/sched/mm.h>
#include <asm/tlbflush.h>
#include <asm/shmparam.h>
#define CREATE_TRACE_POINTS
#include <trace/events/vmalloc.h>
#include "internal.h"
#include "pgalloc-track.h"
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1;
static int __init set_nohugeiomap(char *str)
{
ioremap_max_page_shift = PAGE_SHIFT;
return 0;
}
early_param("nohugeiomap", set_nohugeiomap);
#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */
static const unsigned int ioremap_max_page_shift = PAGE_SHIFT;
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
static bool __ro_after_init vmap_allow_huge = true;
static int __init set_nohugevmalloc(char *str)
{
vmap_allow_huge = false;
return 0;
}
early_param("nohugevmalloc", set_nohugevmalloc);
#else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */
static const bool vmap_allow_huge = false;
#endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */
bool is_vmalloc_addr(const void *x)
{
unsigned long addr = (unsigned long)kasan_reset_tag(x); return addr >= VMALLOC_START && addr < VMALLOC_END;
}
EXPORT_SYMBOL(is_vmalloc_addr);
struct vfree_deferred {
struct llist_head list;
struct work_struct wq;
};
static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
/*** Page table manipulation functions ***/
static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift, pgtbl_mod_mask *mask)
{
pte_t *pte;
u64 pfn;
unsigned long size = PAGE_SIZE;
pfn = phys_addr >> PAGE_SHIFT;
pte = pte_alloc_kernel_track(pmd, addr, mask);
if (!pte)
return -ENOMEM;
do {
BUG_ON(!pte_none(ptep_get(pte)));
#ifdef CONFIG_HUGETLB_PAGE
size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift);
if (size != PAGE_SIZE) {
pte_t entry = pfn_pte(pfn, prot);
entry = arch_make_huge_pte(entry, ilog2(size), 0);
set_huge_pte_at(&init_mm, addr, pte, entry, size);
pfn += PFN_DOWN(size);
continue;
}
#endif
set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
pfn++;
} while (pte += PFN_DOWN(size), addr += size, addr != end);
*mask |= PGTBL_PTE_MODIFIED;
return 0;
}
static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift)
{
if (max_page_shift < PMD_SHIFT)
return 0;
if (!arch_vmap_pmd_supported(prot))
return 0;
if ((end - addr) != PMD_SIZE)
return 0;
if (!IS_ALIGNED(addr, PMD_SIZE))
return 0;
if (!IS_ALIGNED(phys_addr, PMD_SIZE))
return 0;
if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))
return 0;
return pmd_set_huge(pmd, phys_addr, prot);
}
static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift, pgtbl_mod_mask *mask)
{
pmd_t *pmd;
unsigned long next;
pmd = pmd_alloc_track(&init_mm, pud, addr, mask);
if (!pmd)
return -ENOMEM;
do {
next = pmd_addr_end(addr, end);
if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot,
max_page_shift)) {
*mask |= PGTBL_PMD_MODIFIED;
continue;
}
if (vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask))
return -ENOMEM;
} while (pmd++, phys_addr += (next - addr), addr = next, addr != end);
return 0;
}
static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift)
{
if (max_page_shift < PUD_SHIFT)
return 0;
if (!arch_vmap_pud_supported(prot))
return 0;
if ((end - addr) != PUD_SIZE)
return 0;
if (!IS_ALIGNED(addr, PUD_SIZE))
return 0;
if (!IS_ALIGNED(phys_addr, PUD_SIZE))
return 0;
if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))
return 0;
return pud_set_huge(pud, phys_addr, prot);
}
static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift, pgtbl_mod_mask *mask)
{
pud_t *pud;
unsigned long next;
pud = pud_alloc_track(&init_mm, p4d, addr, mask);
if (!pud)
return -ENOMEM;
do {
next = pud_addr_end(addr, end);
if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot,
max_page_shift)) {
*mask |= PGTBL_PUD_MODIFIED;
continue;
}
if (vmap_pmd_range(pud, addr, next, phys_addr, prot,
max_page_shift, mask))
return -ENOMEM;
} while (pud++, phys_addr += (next - addr), addr = next, addr != end);
return 0;
}
static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift)
{
if (max_page_shift < P4D_SHIFT)
return 0;
if (!arch_vmap_p4d_supported(prot))
return 0;
if ((end - addr) != P4D_SIZE)
return 0;
if (!IS_ALIGNED(addr, P4D_SIZE))
return 0;
if (!IS_ALIGNED(phys_addr, P4D_SIZE))
return 0;
if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))
return 0;
return p4d_set_huge(p4d, phys_addr, prot);
}
static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift, pgtbl_mod_mask *mask)
{
p4d_t *p4d;
unsigned long next;
p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);
if (!p4d)
return -ENOMEM;
do {
next = p4d_addr_end(addr, end);
if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot,
max_page_shift)) {
*mask |= PGTBL_P4D_MODIFIED;
continue;
}
if (vmap_pud_range(p4d, addr, next, phys_addr, prot,
max_page_shift, mask))
return -ENOMEM;
} while (p4d++, phys_addr += (next - addr), addr = next, addr != end);
return 0;
}
static int vmap_range_noflush(unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift)
{
pgd_t *pgd;
unsigned long start;
unsigned long next;
int err;
pgtbl_mod_mask mask = 0;
might_sleep();
BUG_ON(addr >= end);
start = addr;
pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
err = vmap_p4d_range(pgd, addr, next, phys_addr, prot,
max_page_shift, &mask);
if (err)
break;
} while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, end);
return err;
}
int vmap_page_range(unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot)
{
int err;
err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),
ioremap_max_page_shift);
flush_cache_vmap(addr, end);
if (!err)
err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,
ioremap_max_page_shift);
return err;
}
int ioremap_page_range(unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot)
{
struct vm_struct *area;
area = find_vm_area((void *)addr);
if (!area || !(area->flags & VM_IOREMAP)) {
WARN_ONCE(1, "vm_area at addr %lx is not marked as VM_IOREMAP\n", addr);
return -EINVAL;
}
if (addr != (unsigned long)area->addr ||
(void *)end != area->addr + get_vm_area_size(area)) {
WARN_ONCE(1, "ioremap request [%lx,%lx) doesn't match vm_area [%lx, %lx)\n",
addr, end, (long)area->addr,
(long)area->addr + get_vm_area_size(area));
return -ERANGE;
}
return vmap_page_range(addr, end, phys_addr, prot);
}
static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
pgtbl_mod_mask *mask)
{
pte_t *pte;
pte = pte_offset_kernel(pmd, addr);
do {
pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); WARN_ON(!pte_none(ptent) && !pte_present(ptent)); } while (pte++, addr += PAGE_SIZE, addr != end);
*mask |= PGTBL_PTE_MODIFIED;
}
static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
pgtbl_mod_mask *mask)
{
pmd_t *pmd;
unsigned long next;
int cleared;
pmd = pmd_offset(pud, addr);
do {
next = pmd_addr_end(addr, end); cleared = pmd_clear_huge(pmd); if (cleared || pmd_bad(*pmd))
*mask |= PGTBL_PMD_MODIFIED;
if (cleared)
continue;
if (pmd_none_or_clear_bad(pmd))
continue;
vunmap_pte_range(pmd, addr, next, mask); cond_resched(); } while (pmd++, addr = next, addr != end);
}
static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
pgtbl_mod_mask *mask)
{
pud_t *pud;
unsigned long next;
int cleared;
pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end); cleared = pud_clear_huge(pud); if (cleared || pud_bad(*pud))
*mask |= PGTBL_PUD_MODIFIED;
if (cleared)
continue;
if (pud_none_or_clear_bad(pud))
continue;
vunmap_pmd_range(pud, addr, next, mask); } while (pud++, addr = next, addr != end);
}
static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
pgtbl_mod_mask *mask)
{
p4d_t *p4d;
unsigned long next;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
p4d_clear_huge(p4d);
if (p4d_bad(*p4d))
*mask |= PGTBL_P4D_MODIFIED;
if (p4d_none_or_clear_bad(p4d))
continue;
vunmap_pud_range(p4d, addr, next, mask);
} while (p4d++, addr = next, addr != end);
}
/*
* vunmap_range_noflush is similar to vunmap_range, but does not
* flush caches or TLBs.
*
* The caller is responsible for calling flush_cache_vmap() before calling
* this function, and flush_tlb_kernel_range after it has returned
* successfully (and before the addresses are expected to cause a page fault
* or be re-mapped for something else, if TLB flushes are being delayed or
* coalesced).
*
* This is an internal function only. Do not use outside mm/.
*/
void __vunmap_range_noflush(unsigned long start, unsigned long end)
{
unsigned long next;
pgd_t *pgd;
unsigned long addr = start;
pgtbl_mod_mask mask = 0;
BUG_ON(addr >= end); pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_bad(*pgd))
mask |= PGTBL_PGD_MODIFIED;
if (pgd_none_or_clear_bad(pgd))
continue;
vunmap_p4d_range(pgd, addr, next, &mask); } while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, end);
}
void vunmap_range_noflush(unsigned long start, unsigned long end)
{
kmsan_vunmap_range_noflush(start, end);
__vunmap_range_noflush(start, end);
}
/**
* vunmap_range - unmap kernel virtual addresses
* @addr: start of the VM area to unmap
* @end: end of the VM area to unmap (non-inclusive)
*
* Clears any present PTEs in the virtual address range, flushes TLBs and
* caches. Any subsequent access to the address before it has been re-mapped
* is a kernel bug.
*/
void vunmap_range(unsigned long addr, unsigned long end)
{
flush_cache_vunmap(addr, end);
vunmap_range_noflush(addr, end);
flush_tlb_kernel_range(addr, end);
}
static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
unsigned long end, pgprot_t prot, struct page **pages, int *nr,
pgtbl_mod_mask *mask)
{
pte_t *pte;
/*
* nr is a running index into the array which helps higher level
* callers keep track of where we're up to.
*/
pte = pte_alloc_kernel_track(pmd, addr, mask);
if (!pte)
return -ENOMEM;
do {
struct page *page = pages[*nr]; if (WARN_ON(!pte_none(ptep_get(pte))))
return -EBUSY;
if (WARN_ON(!page))
return -ENOMEM;
if (WARN_ON(!pfn_valid(page_to_pfn(page))))
return -EINVAL;
set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
(*nr)++;
} while (pte++, addr += PAGE_SIZE, addr != end);
*mask |= PGTBL_PTE_MODIFIED;
return 0;
}
static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr,
unsigned long end, pgprot_t prot, struct page **pages, int *nr,
pgtbl_mod_mask *mask)
{
pmd_t *pmd;
unsigned long next;
pmd = pmd_alloc_track(&init_mm, pud, addr, mask);
if (!pmd)
return -ENOMEM;
do {
next = pmd_addr_end(addr, end); if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask))
return -ENOMEM;
} while (pmd++, addr = next, addr != end);
return 0;
}
static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr,
unsigned long end, pgprot_t prot, struct page **pages, int *nr,
pgtbl_mod_mask *mask)
{
pud_t *pud;
unsigned long next;
pud = pud_alloc_track(&init_mm, p4d, addr, mask);
if (!pud)
return -ENOMEM;
do {
next = pud_addr_end(addr, end); if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask))
return -ENOMEM;
} while (pud++, addr = next, addr != end);
return 0;
}
static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr,
unsigned long end, pgprot_t prot, struct page **pages, int *nr,
pgtbl_mod_mask *mask)
{
p4d_t *p4d;
unsigned long next;
p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);
if (!p4d)
return -ENOMEM;
do {
next = p4d_addr_end(addr, end);
if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask))
return -ENOMEM;
} while (p4d++, addr = next, addr != end);
return 0;
}
static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages)
{
unsigned long start = addr;
pgd_t *pgd;
unsigned long next;
int err = 0;
int nr = 0;
pgtbl_mod_mask mask = 0;
BUG_ON(addr >= end); pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_bad(*pgd))
mask |= PGTBL_PGD_MODIFIED;
err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask);
if (err)
return err;
} while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, end);
return 0;
}
/*
* vmap_pages_range_noflush is similar to vmap_pages_range, but does not
* flush caches.
*
* The caller is responsible for calling flush_cache_vmap() after this
* function returns successfully and before the addresses are accessed.
*
* This is an internal function only. Do not use outside mm/.
*/
int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
unsigned int i, nr = (end - addr) >> PAGE_SHIFT; WARN_ON(page_shift < PAGE_SHIFT); if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) ||
page_shift == PAGE_SHIFT)
return vmap_small_pages_range_noflush(addr, end, prot, pages); for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) {
int err;
err = vmap_range_noflush(addr, addr + (1UL << page_shift), page_to_phys(pages[i]), prot,
page_shift);
if (err)
return err;
addr += 1UL << page_shift;
}
return 0;
}
int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
int ret = kmsan_vmap_pages_range_noflush(addr, end, prot, pages,
page_shift);
if (ret)
return ret;
return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
}
/**
* vmap_pages_range - map pages to a kernel virtual address
* @addr: start of the VM area to map
* @end: end of the VM area to map (non-inclusive)
* @prot: page protection flags to use
* @pages: pages to map (always PAGE_SIZE pages)
* @page_shift: maximum shift that the pages may be mapped with, @pages must
* be aligned and contiguous up to at least this shift.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
static int vmap_pages_range(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
int err;
err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
flush_cache_vmap(addr, end);
return err;
}
static int check_sparse_vm_area(struct vm_struct *area, unsigned long start,
unsigned long end)
{
might_sleep();
if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS))
return -EINVAL;
if (WARN_ON_ONCE(area->flags & VM_NO_GUARD))
return -EINVAL;
if (WARN_ON_ONCE(!(area->flags & VM_SPARSE)))
return -EINVAL;
if ((end - start) >> PAGE_SHIFT > totalram_pages())
return -E2BIG;
if (start < (unsigned long)area->addr ||
(void *)end > area->addr + get_vm_area_size(area))
return -ERANGE;
return 0;
}
/**
* vm_area_map_pages - map pages inside given sparse vm_area
* @area: vm_area
* @start: start address inside vm_area
* @end: end address inside vm_area
* @pages: pages to map (always PAGE_SIZE pages)
*/
int vm_area_map_pages(struct vm_struct *area, unsigned long start,
unsigned long end, struct page **pages)
{
int err;
err = check_sparse_vm_area(area, start, end);
if (err)
return err;
return vmap_pages_range(start, end, PAGE_KERNEL, pages, PAGE_SHIFT);
}
/**
* vm_area_unmap_pages - unmap pages inside given sparse vm_area
* @area: vm_area
* @start: start address inside vm_area
* @end: end address inside vm_area
*/
void vm_area_unmap_pages(struct vm_struct *area, unsigned long start,
unsigned long end)
{
if (check_sparse_vm_area(area, start, end))
return;
vunmap_range(start, end);
}
int is_vmalloc_or_module_addr(const void *x)
{
/*
* ARM, x86-64 and sparc64 put modules in a special place,
* and fall back on vmalloc() if that fails. Others
* just put it in the vmalloc space.
*/
#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
unsigned long addr = (unsigned long)kasan_reset_tag(x);
if (addr >= MODULES_VADDR && addr < MODULES_END)
return 1;
#endif
return is_vmalloc_addr(x);
}
EXPORT_SYMBOL_GPL(is_vmalloc_or_module_addr);
/*
* Walk a vmap address to the struct page it maps. Huge vmap mappings will
* return the tail page that corresponds to the base page address, which
* matches small vmap mappings.
*/
struct page *vmalloc_to_page(const void *vmalloc_addr)
{
unsigned long addr = (unsigned long) vmalloc_addr;
struct page *page = NULL;
pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *ptep, pte;
/*
* XXX we might need to change this if we add VIRTUAL_BUG_ON for
* architectures that do not vmalloc module space
*/
VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
if (pgd_none(*pgd))
return NULL;
if (WARN_ON_ONCE(pgd_leaf(*pgd)))
return NULL; /* XXX: no allowance for huge pgd */
if (WARN_ON_ONCE(pgd_bad(*pgd)))
return NULL;
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d))
return NULL;
if (p4d_leaf(*p4d))
return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT);
if (WARN_ON_ONCE(p4d_bad(*p4d)))
return NULL;
pud = pud_offset(p4d, addr);
if (pud_none(*pud))
return NULL;
if (pud_leaf(*pud)) return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); if (WARN_ON_ONCE(pud_bad(*pud)))
return NULL;
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd))
return NULL;
if (pmd_leaf(*pmd)) return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); if (WARN_ON_ONCE(pmd_bad(*pmd)))
return NULL;
ptep = pte_offset_kernel(pmd, addr);
pte = ptep_get(ptep);
if (pte_present(pte))
page = pte_page(pte);
return page;
}
EXPORT_SYMBOL(vmalloc_to_page);
/*
* Map a vmalloc()-space virtual address to the physical page frame number.
*/
unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
{
return page_to_pfn(vmalloc_to_page(vmalloc_addr));
}
EXPORT_SYMBOL(vmalloc_to_pfn);
/*** Global kva allocator ***/
#define DEBUG_AUGMENT_PROPAGATE_CHECK 0
#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
static DEFINE_SPINLOCK(free_vmap_area_lock);
static bool vmap_initialized __read_mostly;
/*
* This kmem_cache is used for vmap_area objects. Instead of
* allocating from slab we reuse an object from this cache to
* make things faster. Especially in "no edge" splitting of
* free block.
*/
static struct kmem_cache *vmap_area_cachep;
/*
* This linked list is used in pair with free_vmap_area_root.
* It gives O(1) access to prev/next to perform fast coalescing.
*/
static LIST_HEAD(free_vmap_area_list);
/*
* This augment red-black tree represents the free vmap space.
* All vmap_area objects in this tree are sorted by va->va_start
* address. It is used for allocation and merging when a vmap
* object is released.
*
* Each vmap_area node contains a maximum available free block
* of its sub-tree, right or left. Therefore it is possible to
* find a lowest match of free area.
*/
static struct rb_root free_vmap_area_root = RB_ROOT;
/*
* Preload a CPU with one object for "no edge" split case. The
* aim is to get rid of allocations from the atomic context, thus
* to use more permissive allocation masks.
*/
static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node);
/*
* This structure defines a single, solid model where a list and
* rb-tree are part of one entity protected by the lock. Nodes are
* sorted in ascending order, thus for O(1) access to left/right
* neighbors a list is used as well as for sequential traversal.
*/
struct rb_list {
struct rb_root root;
struct list_head head;
spinlock_t lock;
};
/*
* A fast size storage contains VAs up to 1M size. A pool consists
* of linked between each other ready to go VAs of certain sizes.
* An index in the pool-array corresponds to number of pages + 1.
*/
#define MAX_VA_SIZE_PAGES 256
struct vmap_pool {
struct list_head head;
unsigned long len;
};
/*
* An effective vmap-node logic. Users make use of nodes instead
* of a global heap. It allows to balance an access and mitigate
* contention.
*/
static struct vmap_node {
/* Simple size segregated storage. */
struct vmap_pool pool[MAX_VA_SIZE_PAGES];
spinlock_t pool_lock;
bool skip_populate;
/* Bookkeeping data of this node. */
struct rb_list busy;
struct rb_list lazy;
/*
* Ready-to-free areas.
*/
struct list_head purge_list;
struct work_struct purge_work;
unsigned long nr_purged;
} single;
/*
* Initial setup consists of one single node, i.e. a balancing
* is fully disabled. Later on, after vmap is initialized these
* parameters are updated based on a system capacity.
*/
static struct vmap_node *vmap_nodes = &single;
static __read_mostly unsigned int nr_vmap_nodes = 1;
static __read_mostly unsigned int vmap_zone_size = 1;
static inline unsigned int
addr_to_node_id(unsigned long addr)
{
return (addr / vmap_zone_size) % nr_vmap_nodes;
}
static inline struct vmap_node *
addr_to_node(unsigned long addr)
{
return &vmap_nodes[addr_to_node_id(addr)];
}
static inline struct vmap_node *
id_to_node(unsigned int id)
{
return &vmap_nodes[id % nr_vmap_nodes];
}
/*
* We use the value 0 to represent "no node", that is why
* an encoded value will be the node-id incremented by 1.
* It is always greater then 0. A valid node_id which can
* be encoded is [0:nr_vmap_nodes - 1]. If a passed node_id
* is not valid 0 is returned.
*/
static unsigned int
encode_vn_id(unsigned int node_id)
{
/* Can store U8_MAX [0:254] nodes. */
if (node_id < nr_vmap_nodes)
return (node_id + 1) << BITS_PER_BYTE;
/* Warn and no node encoded. */
WARN_ONCE(1, "Encode wrong node id (%u)\n", node_id);
return 0;
}
/*
* Returns an encoded node-id, the valid range is within
* [0:nr_vmap_nodes-1] values. Otherwise nr_vmap_nodes is
* returned if extracted data is wrong.
*/
static unsigned int
decode_vn_id(unsigned int val)
{ unsigned int node_id = (val >> BITS_PER_BYTE) - 1;
/* Can store U8_MAX [0:254] nodes. */
if (node_id < nr_vmap_nodes)
return node_id;
/* If it was _not_ zero, warn. */
WARN_ONCE(node_id != UINT_MAX,
"Decode wrong node id (%d)\n", node_id);
return nr_vmap_nodes;
}
static bool
is_vn_id_valid(unsigned int node_id)
{
if (node_id < nr_vmap_nodes)
return true;
return false;
}
static __always_inline unsigned long
va_size(struct vmap_area *va)
{
return (va->va_end - va->va_start);
}
static __always_inline unsigned long
get_subtree_max_size(struct rb_node *node)
{
struct vmap_area *va;
va = rb_entry_safe(node, struct vmap_area, rb_node); return va ? va->subtree_max_size : 0;
}
RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb,
struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size)
static void reclaim_and_purge_vmap_areas(void);
static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
static void drain_vmap_area_work(struct work_struct *work);
static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work);
static atomic_long_t nr_vmalloc_pages;
unsigned long vmalloc_nr_pages(void)
{
return atomic_long_read(&nr_vmalloc_pages);
}
static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root)
{
struct rb_node *n = root->rb_node;
addr = (unsigned long)kasan_reset_tag((void *)addr);
while (n) {
struct vmap_area *va;
va = rb_entry(n, struct vmap_area, rb_node); if (addr < va->va_start) n = n->rb_left; else if (addr >= va->va_end) n = n->rb_right;
else
return va;
}
return NULL;
}
/* Look up the first VA which satisfies addr < va_end, NULL if none. */
static struct vmap_area *
__find_vmap_area_exceed_addr(unsigned long addr, struct rb_root *root)
{
struct vmap_area *va = NULL;
struct rb_node *n = root->rb_node;
addr = (unsigned long)kasan_reset_tag((void *)addr);
while (n) {
struct vmap_area *tmp;
tmp = rb_entry(n, struct vmap_area, rb_node);
if (tmp->va_end > addr) {
va = tmp;
if (tmp->va_start <= addr)
break;
n = n->rb_left;
} else
n = n->rb_right;
}
return va;
}
/*
* Returns a node where a first VA, that satisfies addr < va_end, resides.
* If success, a node is locked. A user is responsible to unlock it when a
* VA is no longer needed to be accessed.
*
* Returns NULL if nothing found.
*/
static struct vmap_node *
find_vmap_area_exceed_addr_lock(unsigned long addr, struct vmap_area **va)
{
unsigned long va_start_lowest;
struct vmap_node *vn;
int i;
repeat:
for (i = 0, va_start_lowest = 0; i < nr_vmap_nodes; i++) {
vn = &vmap_nodes[i];
spin_lock(&vn->busy.lock);
*va = __find_vmap_area_exceed_addr(addr, &vn->busy.root);
if (*va)
if (!va_start_lowest || (*va)->va_start < va_start_lowest)
va_start_lowest = (*va)->va_start;
spin_unlock(&vn->busy.lock);
}
/*
* Check if found VA exists, it might have gone away. In this case we
* repeat the search because a VA has been removed concurrently and we
* need to proceed to the next one, which is a rare case.
*/
if (va_start_lowest) {
vn = addr_to_node(va_start_lowest);
spin_lock(&vn->busy.lock);
*va = __find_vmap_area(va_start_lowest, &vn->busy.root);
if (*va)
return vn;
spin_unlock(&vn->busy.lock);
goto repeat;
}
return NULL;
}
/*
* This function returns back addresses of parent node
* and its left or right link for further processing.
*
* Otherwise NULL is returned. In that case all further
* steps regarding inserting of conflicting overlap range
* have to be declined and actually considered as a bug.
*/
static __always_inline struct rb_node **
find_va_links(struct vmap_area *va,
struct rb_root *root, struct rb_node *from,
struct rb_node **parent)
{
struct vmap_area *tmp_va;
struct rb_node **link;
if (root) {
link = &root->rb_node;
if (unlikely(!*link)) {
*parent = NULL;
return link;
}
} else {
link = &from;
}
/*
* Go to the bottom of the tree. When we hit the last point
* we end up with parent rb_node and correct direction, i name
* it link, where the new va->rb_node will be attached to.
*/
do {
tmp_va = rb_entry(*link, struct vmap_area, rb_node);
/*
* During the traversal we also do some sanity check.
* Trigger the BUG() if there are sides(left/right)
* or full overlaps.
*/
if (va->va_end <= tmp_va->va_start) link = &(*link)->rb_left; else if (va->va_start >= tmp_va->va_end) link = &(*link)->rb_right;
else {
WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n",
va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end);
return NULL;
}
} while (*link); *parent = &tmp_va->rb_node;
return link;
}
static __always_inline struct list_head *
get_va_next_sibling(struct rb_node *parent, struct rb_node **link)
{
struct list_head *list;
if (unlikely(!parent))
/*
* The red-black tree where we try to find VA neighbors
* before merging or inserting is empty, i.e. it means
* there is no free vmap space. Normally it does not
* happen but we handle this case anyway.
*/
return NULL;
list = &rb_entry(parent, struct vmap_area, rb_node)->list;
return (&parent->rb_right == link ? list->next : list);
}
static __always_inline void
__link_va(struct vmap_area *va, struct rb_root *root,
struct rb_node *parent, struct rb_node **link,
struct list_head *head, bool augment)
{
/*
* VA is still not in the list, but we can
* identify its future previous list_head node.
*/
if (likely(parent)) {
head = &rb_entry(parent, struct vmap_area, rb_node)->list; if (&parent->rb_right != link) head = head->prev;
}
/* Insert to the rb-tree */
rb_link_node(&va->rb_node, parent, link);
if (augment) {
/*
* Some explanation here. Just perform simple insertion
* to the tree. We do not set va->subtree_max_size to
* its current size before calling rb_insert_augmented().
* It is because we populate the tree from the bottom
* to parent levels when the node _is_ in the tree.
*
* Therefore we set subtree_max_size to zero after insertion,
* to let __augment_tree_propagate_from() puts everything to
* the correct order later on.
*/
rb_insert_augmented(&va->rb_node,
root, &free_vmap_area_rb_augment_cb);
va->subtree_max_size = 0;
} else {
rb_insert_color(&va->rb_node, root);
}
/* Address-sort this list */
list_add(&va->list, head);
}
static __always_inline void
link_va(struct vmap_area *va, struct rb_root *root,
struct rb_node *parent, struct rb_node **link,
struct list_head *head)
{
__link_va(va, root, parent, link, head, false);
}
static __always_inline void
link_va_augment(struct vmap_area *va, struct rb_root *root,
struct rb_node *parent, struct rb_node **link,
struct list_head *head)
{
__link_va(va, root, parent, link, head, true);
}
static __always_inline void
__unlink_va(struct vmap_area *va, struct rb_root *root, bool augment)
{
if (WARN_ON(RB_EMPTY_NODE(&va->rb_node)))
return;
if (augment)
rb_erase_augmented(&va->rb_node,
root, &free_vmap_area_rb_augment_cb);
else
rb_erase(&va->rb_node, root);
list_del_init(&va->list);
RB_CLEAR_NODE(&va->rb_node);
}
static __always_inline void
unlink_va(struct vmap_area *va, struct rb_root *root)
{
__unlink_va(va, root, false);
}
static __always_inline void
unlink_va_augment(struct vmap_area *va, struct rb_root *root)
{
__unlink_va(va, root, true);
}
#if DEBUG_AUGMENT_PROPAGATE_CHECK
/*
* Gets called when remove the node and rotate.
*/
static __always_inline unsigned long
compute_subtree_max_size(struct vmap_area *va)
{
return max3(va_size(va),
get_subtree_max_size(va->rb_node.rb_left),
get_subtree_max_size(va->rb_node.rb_right));
}
static void
augment_tree_propagate_check(void)
{
struct vmap_area *va;
unsigned long computed_size;
list_for_each_entry(va, &free_vmap_area_list, list) {
computed_size = compute_subtree_max_size(va);
if (computed_size != va->subtree_max_size)
pr_emerg("tree is corrupted: %lu, %lu\n",
va_size(va), va->subtree_max_size);
}
}
#endif
/*
* This function populates subtree_max_size from bottom to upper
* levels starting from VA point. The propagation must be done
* when VA size is modified by changing its va_start/va_end. Or
* in case of newly inserting of VA to the tree.
*
* It means that __augment_tree_propagate_from() must be called:
* - After VA has been inserted to the tree(free path);
* - After VA has been shrunk(allocation path);
* - After VA has been increased(merging path).
*
* Please note that, it does not mean that upper parent nodes
* and their subtree_max_size are recalculated all the time up
* to the root node.
*
* 4--8
* /\
* / \
* / \
* 2--2 8--8
*
* For example if we modify the node 4, shrinking it to 2, then
* no any modification is required. If we shrink the node 2 to 1
* its subtree_max_size is updated only, and set to 1. If we shrink
* the node 8 to 6, then its subtree_max_size is set to 6 and parent
* node becomes 4--6.
*/
static __always_inline void
augment_tree_propagate_from(struct vmap_area *va)
{
/*
* Populate the tree from bottom towards the root until
* the calculated maximum available size of checked node
* is equal to its current one.
*/
free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL);
#if DEBUG_AUGMENT_PROPAGATE_CHECK
augment_tree_propagate_check();
#endif
}
static void
insert_vmap_area(struct vmap_area *va,
struct rb_root *root, struct list_head *head)
{
struct rb_node **link;
struct rb_node *parent;
link = find_va_links(va, root, NULL, &parent);
if (link)
link_va(va, root, parent, link, head);
}
static void
insert_vmap_area_augment(struct vmap_area *va,
struct rb_node *from, struct rb_root *root,
struct list_head *head)
{
struct rb_node **link;
struct rb_node *parent;
if (from)
link = find_va_links(va, NULL, from, &parent);
else
link = find_va_links(va, root, NULL, &parent);
if (link) {
link_va_augment(va, root, parent, link, head);
augment_tree_propagate_from(va);
}
}
/*
* Merge de-allocated chunk of VA memory with previous
* and next free blocks. If coalesce is not done a new
* free area is inserted. If VA has been merged, it is
* freed.
*
* Please note, it can return NULL in case of overlap
* ranges, followed by WARN() report. Despite it is a
* buggy behaviour, a system can be alive and keep
* ongoing.
*/
static __always_inline struct vmap_area *
__merge_or_add_vmap_area(struct vmap_area *va,
struct rb_root *root, struct list_head *head, bool augment)
{
struct vmap_area *sibling;
struct list_head *next;
struct rb_node **link;
struct rb_node *parent;
bool merged = false;
/*
* Find a place in the tree where VA potentially will be
* inserted, unless it is merged with its sibling/siblings.
*/
link = find_va_links(va, root, NULL, &parent);
if (!link)
return NULL;
/*
* Get next node of VA to check if merging can be done.
*/
next = get_va_next_sibling(parent, link);
if (unlikely(next == NULL))
goto insert;
/*
* start end
* | |
* |<------VA------>|<-----Next----->|
* | |
* start end
*/
if (next != head) {
sibling = list_entry(next, struct vmap_area, list);
if (sibling->va_start == va->va_end) {
sibling->va_start = va->va_start;
/* Free vmap_area object. */
kmem_cache_free(vmap_area_cachep, va);
/* Point to the new merged area. */
va = sibling;
merged = true;
}
}
/*
* start end
* | |
* |<-----Prev----->|<------VA------>|
* | |
* start end
*/
if (next->prev != head) {
sibling = list_entry(next->prev, struct vmap_area, list);
if (sibling->va_end == va->va_start) {
/*
* If both neighbors are coalesced, it is important
* to unlink the "next" node first, followed by merging
* with "previous" one. Otherwise the tree might not be
* fully populated if a sibling's augmented value is
* "normalized" because of rotation operations.
*/
if (merged)
__unlink_va(va, root, augment);
sibling->va_end = va->va_end;
/* Free vmap_area object. */
kmem_cache_free(vmap_area_cachep, va);
/* Point to the new merged area. */
va = sibling;
merged = true;
}
}
insert:
if (!merged)
__link_va(va, root, parent, link, head, augment);
return va;
}
static __always_inline struct vmap_area *
merge_or_add_vmap_area(struct vmap_area *va,
struct rb_root *root, struct list_head *head)
{
return __merge_or_add_vmap_area(va, root, head, false);
}
static __always_inline struct vmap_area *
merge_or_add_vmap_area_augment(struct vmap_area *va,
struct rb_root *root, struct list_head *head)
{
va = __merge_or_add_vmap_area(va, root, head, true);
if (va)
augment_tree_propagate_from(va);
return va;
}
static __always_inline bool
is_within_this_va(struct vmap_area *va, unsigned long size,
unsigned long align, unsigned long vstart)
{
unsigned long nva_start_addr;
if (va->va_start > vstart) nva_start_addr = ALIGN(va->va_start, align);
else
nva_start_addr = ALIGN(vstart, align);
/* Can be overflowed due to big size or alignment. */
if (nva_start_addr + size < nva_start_addr ||
nva_start_addr < vstart)
return false;
return (nva_start_addr + size <= va->va_end);
}
/*
* Find the first free block(lowest start address) in the tree,
* that will accomplish the request corresponding to passing
* parameters. Please note, with an alignment bigger than PAGE_SIZE,
* a search length is adjusted to account for worst case alignment
* overhead.
*/
static __always_inline struct vmap_area *
find_vmap_lowest_match(struct rb_root *root, unsigned long size,
unsigned long align, unsigned long vstart, bool adjust_search_size)
{
struct vmap_area *va;
struct rb_node *node;
unsigned long length;
/* Start from the root. */
node = root->rb_node;
/* Adjust the search size for alignment overhead. */
length = adjust_search_size ? size + align - 1 : size; while (node) { va = rb_entry(node, struct vmap_area, rb_node); if (get_subtree_max_size(node->rb_left) >= length &&
vstart < va->va_start) {
node = node->rb_left;
} else {
if (is_within_this_va(va, size, align, vstart))
return va;
/*
* Does not make sense to go deeper towards the right
* sub-tree if it does not have a free block that is
* equal or bigger to the requested search length.
*/
if (get_subtree_max_size(node->rb_right) >= length) {
node = node->rb_right;
continue;
}
/*
* OK. We roll back and find the first right sub-tree,
* that will satisfy the search criteria. It can happen
* due to "vstart" restriction or an alignment overhead
* that is bigger then PAGE_SIZE.
*/
while ((node = rb_parent(node))) {
va = rb_entry(node, struct vmap_area, rb_node);
if (is_within_this_va(va, size, align, vstart))
return va;
if (get_subtree_max_size(node->rb_right) >= length &&
vstart <= va->va_start) {
/*
* Shift the vstart forward. Please note, we update it with
* parent's start address adding "1" because we do not want
* to enter same sub-tree after it has already been checked
* and no suitable free block found there.
*/
vstart = va->va_start + 1;
node = node->rb_right;
break;
}
}
}
}
return NULL;
}
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
#include <linux/random.h>
static struct vmap_area *
find_vmap_lowest_linear_match(struct list_head *head, unsigned long size,
unsigned long align, unsigned long vstart)
{
struct vmap_area *va;
list_for_each_entry(va, head, list) {
if (!is_within_this_va(va, size, align, vstart))
continue;
return va;
}
return NULL;
}
static void
find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head,
unsigned long size, unsigned long align)
{
struct vmap_area *va_1, *va_2;
unsigned long vstart;
unsigned int rnd;
get_random_bytes(&rnd, sizeof(rnd));
vstart = VMALLOC_START + rnd;
va_1 = find_vmap_lowest_match(root, size, align, vstart, false);
va_2 = find_vmap_lowest_linear_match(head, size, align, vstart);
if (va_1 != va_2)
pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
va_1, va_2, vstart);
}
#endif
enum fit_type {
NOTHING_FIT = 0,
FL_FIT_TYPE = 1, /* full fit */
LE_FIT_TYPE = 2, /* left edge fit */
RE_FIT_TYPE = 3, /* right edge fit */
NE_FIT_TYPE = 4 /* no edge fit */
};
static __always_inline enum fit_type
classify_va_fit_type(struct vmap_area *va,
unsigned long nva_start_addr, unsigned long size)
{
enum fit_type type;
/* Check if it is within VA. */
if (nva_start_addr < va->va_start || nva_start_addr + size > va->va_end)
return NOTHING_FIT;
/* Now classify. */
if (va->va_start == nva_start_addr) {
if (va->va_end == nva_start_addr + size)
type = FL_FIT_TYPE;
else
type = LE_FIT_TYPE;
} else if (va->va_end == nva_start_addr + size) {
type = RE_FIT_TYPE;
} else {
type = NE_FIT_TYPE;
}
return type;
}
static __always_inline int
va_clip(struct rb_root *root, struct list_head *head,
struct vmap_area *va, unsigned long nva_start_addr,
unsigned long size)
{
struct vmap_area *lva = NULL;
enum fit_type type = classify_va_fit_type(va, nva_start_addr, size);
if (type == FL_FIT_TYPE) {
/*
* No need to split VA, it fully fits.
*
* | |
* V NVA V
* |---------------|
*/
unlink_va_augment(va, root); kmem_cache_free(vmap_area_cachep, va);
} else if (type == LE_FIT_TYPE) {
/*
* Split left edge of fit VA.
*
* | |
* V NVA V R
* |-------|-------|
*/
va->va_start += size;
} else if (type == RE_FIT_TYPE) {
/*
* Split right edge of fit VA.
*
* | |
* L V NVA V
* |-------|-------|
*/
va->va_end = nva_start_addr;
} else if (type == NE_FIT_TYPE) {
/*
* Split no edge of fit VA.
*
* | |
* L V NVA V R
* |---|-------|---|
*/
lva = __this_cpu_xchg(ne_fit_preload_node, NULL);
if (unlikely(!lva)) {
/*
* For percpu allocator we do not do any pre-allocation
* and leave it as it is. The reason is it most likely
* never ends up with NE_FIT_TYPE splitting. In case of
* percpu allocations offsets and sizes are aligned to
* fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE
* are its main fitting cases.
*
* There are a few exceptions though, as an example it is
* a first allocation (early boot up) when we have "one"
* big free space that has to be split.
*
* Also we can hit this path in case of regular "vmap"
* allocations, if "this" current CPU was not preloaded.
* See the comment in alloc_vmap_area() why. If so, then
* GFP_NOWAIT is used instead to get an extra object for
* split purpose. That is rare and most time does not
* occur.
*
* What happens if an allocation gets failed. Basically,
* an "overflow" path is triggered to purge lazily freed
* areas to free some memory, then, the "retry" path is
* triggered to repeat one more time. See more details
* in alloc_vmap_area() function.
*/
lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
if (!lva)
return -1;
}
/*
* Build the remainder.
*/
lva->va_start = va->va_start;
lva->va_end = nva_start_addr;
/*
* Shrink this VA to remaining size.
*/
va->va_start = nva_start_addr + size;
} else {
return -1;
}
if (type != FL_FIT_TYPE) {
augment_tree_propagate_from(va); if (lva) /* type == NE_FIT_TYPE */ insert_vmap_area_augment(lva, &va->rb_node, root, head);
}
return 0;
}
static unsigned long
va_alloc(struct vmap_area *va,
struct rb_root *root, struct list_head *head,
unsigned long size, unsigned long align,
unsigned long vstart, unsigned long vend)
{
unsigned long nva_start_addr;
int ret;
if (va->va_start > vstart) nva_start_addr = ALIGN(va->va_start, align);
else
nva_start_addr = ALIGN(vstart, align);
/* Check the "vend" restriction. */
if (nva_start_addr + size > vend)
return vend;
/* Update the free vmap_area. */
ret = va_clip(root, head, va, nva_start_addr, size); if (WARN_ON_ONCE(ret))
return vend;
return nva_start_addr;
}
/*
* Returns a start address of the newly allocated area, if success.
* Otherwise a vend is returned that indicates failure.
*/
static __always_inline unsigned long
__alloc_vmap_area(struct rb_root *root, struct list_head *head,
unsigned long size, unsigned long align,
unsigned long vstart, unsigned long vend)
{
bool adjust_search_size = true;
unsigned long nva_start_addr;
struct vmap_area *va;
/*
* Do not adjust when:
* a) align <= PAGE_SIZE, because it does not make any sense.
* All blocks(their start addresses) are at least PAGE_SIZE
* aligned anyway;
* b) a short range where a requested size corresponds to exactly
* specified [vstart:vend] interval and an alignment > PAGE_SIZE.
* With adjusted search length an allocation would not succeed.
*/
if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size))
adjust_search_size = false;
va = find_vmap_lowest_match(root, size, align, vstart, adjust_search_size); if (unlikely(!va))
return vend;
nva_start_addr = va_alloc(va, root, head, size, align, vstart, vend);
if (nva_start_addr == vend)
return vend;
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
find_vmap_lowest_match_check(root, head, size, align);
#endif
return nva_start_addr;
}
/*
* Free a region of KVA allocated by alloc_vmap_area
*/
static void free_vmap_area(struct vmap_area *va)
{
struct vmap_node *vn = addr_to_node(va->va_start);
/*
* Remove from the busy tree/list.
*/
spin_lock(&vn->busy.lock);
unlink_va(va, &vn->busy.root);
spin_unlock(&vn->busy.lock);
/*
* Insert/Merge it back to the free tree/list.
*/
spin_lock(&free_vmap_area_lock);
merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list);
spin_unlock(&free_vmap_area_lock);
}
static inline void
preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node)
{
struct vmap_area *va = NULL;
/*
* Preload this CPU with one extra vmap_area object. It is used
* when fit type of free area is NE_FIT_TYPE. It guarantees that
* a CPU that does an allocation is preloaded.
*
* We do it in non-atomic context, thus it allows us to use more
* permissive allocation masks to be more stable under low memory
* condition and high memory pressure.
*/
if (!this_cpu_read(ne_fit_preload_node)) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); spin_lock(lock); if (va && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, va)) kmem_cache_free(vmap_area_cachep, va);
}
static struct vmap_pool *
size_to_va_pool(struct vmap_node *vn, unsigned long size)
{
unsigned int idx = (size - 1) / PAGE_SIZE;
if (idx < MAX_VA_SIZE_PAGES)
return &vn->pool[idx];
return NULL;
}
static bool
node_pool_add_va(struct vmap_node *n, struct vmap_area *va)
{
struct vmap_pool *vp;
vp = size_to_va_pool(n, va_size(va));
if (!vp)
return false;
spin_lock(&n->pool_lock);
list_add(&va->list, &vp->head);
WRITE_ONCE(vp->len, vp->len + 1);
spin_unlock(&n->pool_lock);
return true;
}
static struct vmap_area *
node_pool_del_va(struct vmap_node *vn, unsigned long size,
unsigned long align, unsigned long vstart,
unsigned long vend)
{
struct vmap_area *va = NULL;
struct vmap_pool *vp;
int err = 0;
vp = size_to_va_pool(vn, size);
if (!vp || list_empty(&vp->head))
return NULL;
spin_lock(&vn->pool_lock);
if (!list_empty(&vp->head)) {
va = list_first_entry(&vp->head, struct vmap_area, list);
if (IS_ALIGNED(va->va_start, align)) {
/*
* Do some sanity check and emit a warning
* if one of below checks detects an error.
*/
err |= (va_size(va) != size);
err |= (va->va_start < vstart);
err |= (va->va_end > vend);
if (!WARN_ON_ONCE(err)) { list_del_init(&va->list);
WRITE_ONCE(vp->len, vp->len - 1);
} else {
va = NULL;
}
} else {
list_move_tail(&va->list, &vp->head);
va = NULL;
}
}
spin_unlock(&vn->pool_lock);
return va;
}
static struct vmap_area *
node_alloc(unsigned long size, unsigned long align,
unsigned long vstart, unsigned long vend,
unsigned long *addr, unsigned int *vn_id)
{
struct vmap_area *va;
*vn_id = 0;
*addr = vend;
/*
* Fallback to a global heap if not vmalloc or there
* is only one node.
*/
if (vstart != VMALLOC_START || vend != VMALLOC_END || nr_vmap_nodes == 1)
return NULL;
*vn_id = raw_smp_processor_id() % nr_vmap_nodes; va = node_pool_del_va(id_to_node(*vn_id), size, align, vstart, vend); *vn_id = encode_vn_id(*vn_id); if (va) *addr = va->va_start;
return va;
}
/*
* Allocate a region of KVA of the specified size and alignment, within the
* vstart and vend.
*/
static struct vmap_area *alloc_vmap_area(unsigned long size,
unsigned long align,
unsigned long vstart, unsigned long vend,
int node, gfp_t gfp_mask,
unsigned long va_flags)
{
struct vmap_node *vn;
struct vmap_area *va;
unsigned long freed;
unsigned long addr;
unsigned int vn_id;
int purged = 0;
int ret;
if (unlikely(!size || offset_in_page(size) || !is_power_of_2(align))) return ERR_PTR(-EINVAL); if (unlikely(!vmap_initialized))
return ERR_PTR(-EBUSY);
might_sleep();
/*
* If a VA is obtained from a global heap(if it fails here)
* it is anyway marked with this "vn_id" so it is returned
* to this pool's node later. Such way gives a possibility
* to populate pools based on users demand.
*
* On success a ready to go VA is returned.
*/
va = node_alloc(size, align, vstart, vend, &addr, &vn_id);
if (!va) {
gfp_mask = gfp_mask & GFP_RECLAIM_MASK;
va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
if (unlikely(!va))
return ERR_PTR(-ENOMEM);
/*
* Only scan the relevant parts containing pointers to other objects
* to avoid false negatives.
*/
kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
}
retry:
if (addr == vend) { preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); addr = __alloc_vmap_area(&free_vmap_area_root, &free_vmap_area_list,
size, align, vstart, vend);
spin_unlock(&free_vmap_area_lock);
}
trace_alloc_vmap_area(addr, size, align, vstart, vend, addr == vend);
/*
* If an allocation fails, the "vend" address is
* returned. Therefore trigger the overflow path.
*/
if (unlikely(addr == vend))
goto overflow;
va->va_start = addr;
va->va_end = addr + size;
va->vm = NULL;
va->flags = (va_flags | vn_id);
vn = addr_to_node(va->va_start);
spin_lock(&vn->busy.lock);
insert_vmap_area(va, &vn->busy.root, &vn->busy.head);
spin_unlock(&vn->busy.lock);
BUG_ON(!IS_ALIGNED(va->va_start, align)); BUG_ON(va->va_start < vstart); BUG_ON(va->va_end > vend); ret = kasan_populate_vmalloc(addr, size);
if (ret) {
free_vmap_area(va);
return ERR_PTR(ret);
}
return va;
overflow:
if (!purged) { reclaim_and_purge_vmap_areas();
purged = 1;
goto retry;
}
freed = 0;
blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
if (freed > 0) {
purged = 0;
goto retry;
}
if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
size);
kmem_cache_free(vmap_area_cachep, va);
return ERR_PTR(-EBUSY);
}
int register_vmap_purge_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_register(&vmap_notify_list, nb);
}
EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
int unregister_vmap_purge_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
}
EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
/*
* lazy_max_pages is the maximum amount of virtual address space we gather up
* before attempting to purge with a TLB flush.
*
* There is a tradeoff here: a larger number will cover more kernel page tables
* and take slightly longer to purge, but it will linearly reduce the number of
* global TLB flushes that must be performed. It would seem natural to scale
* this number up linearly with the number of CPUs (because vmapping activity
* could also scale linearly with the number of CPUs), however it is likely
* that in practice, workloads might be constrained in other ways that mean
* vmap activity will not scale linearly with CPUs. Also, I want to be
* conservative and not introduce a big latency on huge systems, so go with
* a less aggressive log scale. It will still be an improvement over the old
* code, and it will be simple to change the scale factor if we find that it
* becomes a problem on bigger systems.
*/
static unsigned long lazy_max_pages(void)
{
unsigned int log;
log = fls(num_online_cpus());
return log * (32UL * 1024 * 1024 / PAGE_SIZE);
}
static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0);
/*
* Serialize vmap purging. There is no actual critical section protected
* by this lock, but we want to avoid concurrent calls for performance
* reasons and to make the pcpu_get_vm_areas more deterministic.
*/
static DEFINE_MUTEX(vmap_purge_lock);
/* for per-CPU blocks */
static void purge_fragmented_blocks_allcpus(void);
static cpumask_t purge_nodes;
static void
reclaim_list_global(struct list_head *head)
{
struct vmap_area *va, *n;
if (list_empty(head))
return;
spin_lock(&free_vmap_area_lock);
list_for_each_entry_safe(va, n, head, list)
merge_or_add_vmap_area_augment(va,
&free_vmap_area_root, &free_vmap_area_list);
spin_unlock(&free_vmap_area_lock);
}
static void
decay_va_pool_node(struct vmap_node *vn, bool full_decay)
{
struct vmap_area *va, *nva;
struct list_head decay_list;
struct rb_root decay_root;
unsigned long n_decay;
int i;
decay_root = RB_ROOT;
INIT_LIST_HEAD(&decay_list);
for (i = 0; i < MAX_VA_SIZE_PAGES; i++) {
struct list_head tmp_list;
if (list_empty(&vn->pool[i].head))
continue;
INIT_LIST_HEAD(&tmp_list);
/* Detach the pool, so no-one can access it. */
spin_lock(&vn->pool_lock);
list_replace_init(&vn->pool[i].head, &tmp_list);
spin_unlock(&vn->pool_lock);
if (full_decay)
WRITE_ONCE(vn->pool[i].len, 0);
/* Decay a pool by ~25% out of left objects. */
n_decay = vn->pool[i].len >> 2;
list_for_each_entry_safe(va, nva, &tmp_list, list) {
list_del_init(&va->list);
merge_or_add_vmap_area(va, &decay_root, &decay_list);
if (!full_decay) {
WRITE_ONCE(vn->pool[i].len, vn->pool[i].len - 1);
if (!--n_decay)
break;
}
}
/*
* Attach the pool back if it has been partly decayed.
* Please note, it is supposed that nobody(other contexts)
* can populate the pool therefore a simple list replace
* operation takes place here.
*/
if (!full_decay && !list_empty(&tmp_list)) {
spin_lock(&vn->pool_lock);
list_replace_init(&tmp_list, &vn->pool[i].head);
spin_unlock(&vn->pool_lock);
}
}
reclaim_list_global(&decay_list);
}
static void purge_vmap_node(struct work_struct *work)
{
struct vmap_node *vn = container_of(work,
struct vmap_node, purge_work);
struct vmap_area *va, *n_va;
LIST_HEAD(local_list);
vn->nr_purged = 0;
list_for_each_entry_safe(va, n_va, &vn->purge_list, list) {
unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
unsigned long orig_start = va->va_start;
unsigned long orig_end = va->va_end;
unsigned int vn_id = decode_vn_id(va->flags);
list_del_init(&va->list);
if (is_vmalloc_or_module_addr((void *)orig_start))
kasan_release_vmalloc(orig_start, orig_end,
va->va_start, va->va_end);
atomic_long_sub(nr, &vmap_lazy_nr);
vn->nr_purged++;
if (is_vn_id_valid(vn_id) && !vn->skip_populate)
if (node_pool_add_va(vn, va))
continue;
/* Go back to global. */
list_add(&va->list, &local_list);
}
reclaim_list_global(&local_list);
}
/*
* Purges all lazily-freed vmap areas.
*/
static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
bool full_pool_decay)
{
unsigned long nr_purged_areas = 0;
unsigned int nr_purge_helpers;
unsigned int nr_purge_nodes;
struct vmap_node *vn;
int i;
lockdep_assert_held(&vmap_purge_lock);
/*
* Use cpumask to mark which node has to be processed.
*/
purge_nodes = CPU_MASK_NONE;
for (i = 0; i < nr_vmap_nodes; i++) {
vn = &vmap_nodes[i];
INIT_LIST_HEAD(&vn->purge_list);
vn->skip_populate = full_pool_decay;
decay_va_pool_node(vn, full_pool_decay);
if (RB_EMPTY_ROOT(&vn->lazy.root))
continue;
spin_lock(&vn->lazy.lock);
WRITE_ONCE(vn->lazy.root.rb_node, NULL);
list_replace_init(&vn->lazy.head, &vn->purge_list);
spin_unlock(&vn->lazy.lock);
start = min(start, list_first_entry(&vn->purge_list,
struct vmap_area, list)->va_start);
end = max(end, list_last_entry(&vn->purge_list,
struct vmap_area, list)->va_end);
cpumask_set_cpu(i, &purge_nodes);
}
nr_purge_nodes = cpumask_weight(&purge_nodes);
if (nr_purge_nodes > 0) {
flush_tlb_kernel_range(start, end);
/* One extra worker is per a lazy_max_pages() full set minus one. */
nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages();
nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1;
for_each_cpu(i, &purge_nodes) {
vn = &vmap_nodes[i];
if (nr_purge_helpers > 0) {
INIT_WORK(&vn->purge_work, purge_vmap_node);
if (cpumask_test_cpu(i, cpu_online_mask))
schedule_work_on(i, &vn->purge_work);
else
schedule_work(&vn->purge_work);
nr_purge_helpers--;
} else {
vn->purge_work.func = NULL;
purge_vmap_node(&vn->purge_work);
nr_purged_areas += vn->nr_purged;
}
}
for_each_cpu(i, &purge_nodes) {
vn = &vmap_nodes[i];
if (vn->purge_work.func) {
flush_work(&vn->purge_work);
nr_purged_areas += vn->nr_purged;
}
}
}
trace_purge_vmap_area_lazy(start, end, nr_purged_areas);
return nr_purged_areas > 0;
}
/*
* Reclaim vmap areas by purging fragmented blocks and purge_vmap_area_list.
*/
static void reclaim_and_purge_vmap_areas(void)
{
mutex_lock(&vmap_purge_lock);
purge_fragmented_blocks_allcpus();
__purge_vmap_area_lazy(ULONG_MAX, 0, true);
mutex_unlock(&vmap_purge_lock);
}
static void drain_vmap_area_work(struct work_struct *work)
{
mutex_lock(&vmap_purge_lock);
__purge_vmap_area_lazy(ULONG_MAX, 0, false);
mutex_unlock(&vmap_purge_lock);
}
/*
* Free a vmap area, caller ensuring that the area has been unmapped,
* unlinked and flush_cache_vunmap had been called for the correct
* range previously.
*/
static void free_vmap_area_noflush(struct vmap_area *va)
{
unsigned long nr_lazy_max = lazy_max_pages();
unsigned long va_start = va->va_start;
unsigned int vn_id = decode_vn_id(va->flags);
struct vmap_node *vn;
unsigned long nr_lazy;
if (WARN_ON_ONCE(!list_empty(&va->list)))
return;
nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
PAGE_SHIFT, &vmap_lazy_nr);
/*
* If it was request by a certain node we would like to
* return it to that node, i.e. its pool for later reuse.
*/
vn = is_vn_id_valid(vn_id) ?
id_to_node(vn_id):addr_to_node(va->va_start); spin_lock(&vn->lazy.lock);
insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head);
spin_unlock(&vn->lazy.lock);
trace_free_vmap_area_noflush(va_start, nr_lazy, nr_lazy_max);
/* After this point, we may free va at any time */
if (unlikely(nr_lazy > nr_lazy_max)) schedule_work(&drain_vmap_work);
}
/*
* Free and unmap a vmap area
*/
static void free_unmap_vmap_area(struct vmap_area *va)
{
flush_cache_vunmap(va->va_start, va->va_end);
vunmap_range_noflush(va->va_start, va->va_end);
if (debug_pagealloc_enabled_static())
flush_tlb_kernel_range(va->va_start, va->va_end);
free_vmap_area_noflush(va);
}
struct vmap_area *find_vmap_area(unsigned long addr)
{
struct vmap_node *vn;
struct vmap_area *va;
int i, j;
if (unlikely(!vmap_initialized))
return NULL;
/*
* An addr_to_node_id(addr) converts an address to a node index
* where a VA is located. If VA spans several zones and passed
* addr is not the same as va->va_start, what is not common, we
* may need to scan extra nodes. See an example:
*
* <----va---->
* -|-----|-----|-----|-----|-
* 1 2 0 1
*
* VA resides in node 1 whereas it spans 1, 2 an 0. If passed
* addr is within 2 or 0 nodes we should do extra work.
*/
i = j = addr_to_node_id(addr);
do {
vn = &vmap_nodes[i];
spin_lock(&vn->busy.lock);
va = __find_vmap_area(addr, &vn->busy.root);
spin_unlock(&vn->busy.lock);
if (va)
return va;
} while ((i = (i + 1) % nr_vmap_nodes) != j);
return NULL;
}
static struct vmap_area *find_unlink_vmap_area(unsigned long addr)
{
struct vmap_node *vn;
struct vmap_area *va;
int i, j;
/*
* Check the comment in the find_vmap_area() about the loop.
*/
i = j = addr_to_node_id(addr);
do {
vn = &vmap_nodes[i];
spin_lock(&vn->busy.lock);
va = __find_vmap_area(addr, &vn->busy.root); if (va) unlink_va(va, &vn->busy.root); spin_unlock(&vn->busy.lock); if (va)
return va;
} while ((i = (i + 1) % nr_vmap_nodes) != j);
return NULL;
}
/*** Per cpu kva allocator ***/
/*
* vmap space is limited especially on 32 bit architectures. Ensure there is
* room for at least 16 percpu vmap blocks per CPU.
*/
/*
* If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
* to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
* instead (we just need a rough idea)
*/
#if BITS_PER_LONG == 32
#define VMALLOC_SPACE (128UL*1024*1024)
#else
#define VMALLOC_SPACE (128UL*1024*1024*1024)
#endif
#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
#define VMAP_BBMAP_BITS \
VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
/*
* Purge threshold to prevent overeager purging of fragmented blocks for
* regular operations: Purge if vb->free is less than 1/4 of the capacity.
*/
#define VMAP_PURGE_THRESHOLD (VMAP_BBMAP_BITS / 4)
#define VMAP_RAM 0x1 /* indicates vm_map_ram area*/
#define VMAP_BLOCK 0x2 /* mark out the vmap_block sub-type*/
#define VMAP_FLAGS_MASK 0x3
struct vmap_block_queue {
spinlock_t lock;
struct list_head free;
/*
* An xarray requires an extra memory dynamically to
* be allocated. If it is an issue, we can use rb-tree
* instead.
*/
struct xarray vmap_blocks;
};
struct vmap_block {
spinlock_t lock;
struct vmap_area *va;
unsigned long free, dirty;
DECLARE_BITMAP(used_map, VMAP_BBMAP_BITS);
unsigned long dirty_min, dirty_max; /*< dirty range */
struct list_head free_list;
struct rcu_head rcu_head;
struct list_head purge;
};
/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
/*
* In order to fast access to any "vmap_block" associated with a
* specific address, we use a hash.
*
* A per-cpu vmap_block_queue is used in both ways, to serialize
* an access to free block chains among CPUs(alloc path) and it
* also acts as a vmap_block hash(alloc/free paths). It means we
* overload it, since we already have the per-cpu array which is
* used as a hash table. When used as a hash a 'cpu' passed to
* per_cpu() is not actually a CPU but rather a hash index.
*
* A hash function is addr_to_vb_xa() which hashes any address
* to a specific index(in a hash) it belongs to. This then uses a
* per_cpu() macro to access an array with generated index.
*
* An example:
*
* CPU_1 CPU_2 CPU_0
* | | |
* V V V
* 0 10 20 30 40 50 60
* |------|------|------|------|------|------|...<vmap address space>
* CPU0 CPU1 CPU2 CPU0 CPU1 CPU2
*
* - CPU_1 invokes vm_unmap_ram(6), 6 belongs to CPU0 zone, thus
* it access: CPU0/INDEX0 -> vmap_blocks -> xa_lock;
*
* - CPU_2 invokes vm_unmap_ram(11), 11 belongs to CPU1 zone, thus
* it access: CPU1/INDEX1 -> vmap_blocks -> xa_lock;
*
* - CPU_0 invokes vm_unmap_ram(20), 20 belongs to CPU2 zone, thus
* it access: CPU2/INDEX2 -> vmap_blocks -> xa_lock.
*
* This technique almost always avoids lock contention on insert/remove,
* however xarray spinlocks protect against any contention that remains.
*/
static struct xarray *
addr_to_vb_xa(unsigned long addr)
{
int index = (addr / VMAP_BLOCK_SIZE) % num_possible_cpus();
return &per_cpu(vmap_block_queue, index).vmap_blocks;
}
/*
* We should probably have a fallback mechanism to allocate virtual memory
* out of partially filled vmap blocks. However vmap block sizing should be
* fairly reasonable according to the vmalloc size, so it shouldn't be a
* big problem.
*/
static unsigned long addr_to_vb_idx(unsigned long addr)
{
addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
addr /= VMAP_BLOCK_SIZE;
return addr;
}
static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
{
unsigned long addr;
addr = va_start + (pages_off << PAGE_SHIFT);
BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
return (void *)addr;
}
/**
* new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
* block. Of course pages number can't exceed VMAP_BBMAP_BITS
* @order: how many 2^order pages should be occupied in newly allocated block
* @gfp_mask: flags for the page level allocator
*
* Return: virtual address in a newly allocated block or ERR_PTR(-errno)
*/
static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
{
struct vmap_block_queue *vbq;
struct vmap_block *vb;
struct vmap_area *va;
struct xarray *xa;
unsigned long vb_idx;
int node, err;
void *vaddr;
node = numa_node_id();
vb = kmalloc_node(sizeof(struct vmap_block),
gfp_mask & GFP_RECLAIM_MASK, node);
if (unlikely(!vb))
return ERR_PTR(-ENOMEM);
va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
VMALLOC_START, VMALLOC_END,
node, gfp_mask,
VMAP_RAM|VMAP_BLOCK);
if (IS_ERR(va)) {
kfree(vb);
return ERR_CAST(va);
}
vaddr = vmap_block_vaddr(va->va_start, 0);
spin_lock_init(&vb->lock);
vb->va = va;
/* At least something should be left free */
BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
bitmap_zero(vb->used_map, VMAP_BBMAP_BITS);
vb->free = VMAP_BBMAP_BITS - (1UL << order);
vb->dirty = 0;
vb->dirty_min = VMAP_BBMAP_BITS;
vb->dirty_max = 0;
bitmap_set(vb->used_map, 0, (1UL << order));
INIT_LIST_HEAD(&vb->free_list);
xa = addr_to_vb_xa(va->va_start);
vb_idx = addr_to_vb_idx(va->va_start);
err = xa_insert(xa, vb_idx, vb, gfp_mask);
if (err) {
kfree(vb);
free_vmap_area(va);
return ERR_PTR(err);
}
vbq = raw_cpu_ptr(&vmap_block_queue);
spin_lock(&vbq->lock);
list_add_tail_rcu(&vb->free_list, &vbq->free);
spin_unlock(&vbq->lock);
return vaddr;
}
static void free_vmap_block(struct vmap_block *vb)
{
struct vmap_node *vn;
struct vmap_block *tmp;
struct xarray *xa;
xa = addr_to_vb_xa(vb->va->va_start);
tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start));
BUG_ON(tmp != vb);
vn = addr_to_node(vb->va->va_start);
spin_lock(&vn->busy.lock);
unlink_va(vb->va, &vn->busy.root);
spin_unlock(&vn->busy.lock);
free_vmap_area_noflush(vb->va);
kfree_rcu(vb, rcu_head);
}
static bool purge_fragmented_block(struct vmap_block *vb,
struct vmap_block_queue *vbq, struct list_head *purge_list,
bool force_purge)
{
if (vb->free + vb->dirty != VMAP_BBMAP_BITS ||
vb->dirty == VMAP_BBMAP_BITS)
return false;
/* Don't overeagerly purge usable blocks unless requested */
if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD))
return false;
/* prevent further allocs after releasing lock */
WRITE_ONCE(vb->free, 0);
/* prevent purging it again */
WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS);
vb->dirty_min = 0;
vb->dirty_max = VMAP_BBMAP_BITS;
spin_lock(&vbq->lock);
list_del_rcu(&vb->free_list);
spin_unlock(&vbq->lock);
list_add_tail(&vb->purge, purge_list);
return true;
}
static void free_purged_blocks(struct list_head *purge_list)
{
struct vmap_block *vb, *n_vb;
list_for_each_entry_safe(vb, n_vb, purge_list, purge) {
list_del(&vb->purge);
free_vmap_block(vb);
}
}
static void purge_fragmented_blocks(int cpu)
{
LIST_HEAD(purge);
struct vmap_block *vb;
struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
rcu_read_lock();
list_for_each_entry_rcu(vb, &vbq->free, free_list) {
unsigned long free = READ_ONCE(vb->free);
unsigned long dirty = READ_ONCE(vb->dirty);
if (free + dirty != VMAP_BBMAP_BITS ||
dirty == VMAP_BBMAP_BITS)
continue;
spin_lock(&vb->lock);
purge_fragmented_block(vb, vbq, &purge, true);
spin_unlock(&vb->lock);
}
rcu_read_unlock();
free_purged_blocks(&purge);
}
static void purge_fragmented_blocks_allcpus(void)
{
int cpu;
for_each_possible_cpu(cpu)
purge_fragmented_blocks(cpu);
}
static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
{
struct vmap_block_queue *vbq;
struct vmap_block *vb;
void *vaddr = NULL;
unsigned int order;
BUG_ON(offset_in_page(size));
BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
if (WARN_ON(size == 0)) {
/*
* Allocating 0 bytes isn't what caller wants since
* get_order(0) returns funny result. Just warn and terminate
* early.
*/
return NULL;
}
order = get_order(size);
rcu_read_lock();
vbq = raw_cpu_ptr(&vmap_block_queue);
list_for_each_entry_rcu(vb, &vbq->free, free_list) {
unsigned long pages_off;
if (READ_ONCE(vb->free) < (1UL << order))
continue;
spin_lock(&vb->lock);
if (vb->free < (1UL << order)) {
spin_unlock(&vb->lock);
continue;
}
pages_off = VMAP_BBMAP_BITS - vb->free;
vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
WRITE_ONCE(vb->free, vb->free - (1UL << order));
bitmap_set(vb->used_map, pages_off, (1UL << order));
if (vb->free == 0) {
spin_lock(&vbq->lock);
list_del_rcu(&vb->free_list);
spin_unlock(&vbq->lock);
}
spin_unlock(&vb->lock);
break;
}
rcu_read_unlock();
/* Allocate new block if nothing was found */
if (!vaddr)
vaddr = new_vmap_block(order, gfp_mask);
return vaddr;
}
static void vb_free(unsigned long addr, unsigned long size)
{
unsigned long offset;
unsigned int order;
struct vmap_block *vb;
struct xarray *xa;
BUG_ON(offset_in_page(size));
BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
flush_cache_vunmap(addr, addr + size);
order = get_order(size);
offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT;
xa = addr_to_vb_xa(addr);
vb = xa_load(xa, addr_to_vb_idx(addr));
spin_lock(&vb->lock);
bitmap_clear(vb->used_map, offset, (1UL << order));
spin_unlock(&vb->lock);
vunmap_range_noflush(addr, addr + size);
if (debug_pagealloc_enabled_static())
flush_tlb_kernel_range(addr, addr + size);
spin_lock(&vb->lock);
/* Expand the not yet TLB flushed dirty range */
vb->dirty_min = min(vb->dirty_min, offset);
vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order));
if (vb->dirty == VMAP_BBMAP_BITS) {
BUG_ON(vb->free);
spin_unlock(&vb->lock);
free_vmap_block(vb);
} else
spin_unlock(&vb->lock);
}
static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush)
{
LIST_HEAD(purge_list);
int cpu;
if (unlikely(!vmap_initialized))
return;
mutex_lock(&vmap_purge_lock);
for_each_possible_cpu(cpu) {
struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
struct vmap_block *vb;
unsigned long idx;
rcu_read_lock();
xa_for_each(&vbq->vmap_blocks, idx, vb) {
spin_lock(&vb->lock);
/*
* Try to purge a fragmented block first. If it's
* not purgeable, check whether there is dirty
* space to be flushed.
*/
if (!purge_fragmented_block(vb, vbq, &purge_list, false) &&
vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) {
unsigned long va_start = vb->va->va_start;
unsigned long s, e;
s = va_start + (vb->dirty_min << PAGE_SHIFT);
e = va_start + (vb->dirty_max << PAGE_SHIFT);
start = min(s, start);
end = max(e, end);
/* Prevent that this is flushed again */
vb->dirty_min = VMAP_BBMAP_BITS;
vb->dirty_max = 0;
flush = 1;
}
spin_unlock(&vb->lock);
}
rcu_read_unlock();
}
free_purged_blocks(&purge_list);
if (!__purge_vmap_area_lazy(start, end, false) && flush)
flush_tlb_kernel_range(start, end);
mutex_unlock(&vmap_purge_lock);
}
/**
* vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
*
* The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
* to amortize TLB flushing overheads. What this means is that any page you
* have now, may, in a former life, have been mapped into kernel virtual
* address by the vmap layer and so there might be some CPUs with TLB entries
* still referencing that page (additional to the regular 1:1 kernel mapping).
*
* vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
* be sure that none of the pages we have control over will have any aliases
* from the vmap layer.
*/
void vm_unmap_aliases(void)
{
unsigned long start = ULONG_MAX, end = 0;
int flush = 0;
_vm_unmap_aliases(start, end, flush);
}
EXPORT_SYMBOL_GPL(vm_unmap_aliases);
/**
* vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
* @mem: the pointer returned by vm_map_ram
* @count: the count passed to that vm_map_ram call (cannot unmap partial)
*/
void vm_unmap_ram(const void *mem, unsigned int count)
{
unsigned long size = (unsigned long)count << PAGE_SHIFT;
unsigned long addr = (unsigned long)kasan_reset_tag(mem);
struct vmap_area *va;
might_sleep();
BUG_ON(!addr);
BUG_ON(addr < VMALLOC_START);
BUG_ON(addr > VMALLOC_END);
BUG_ON(!PAGE_ALIGNED(addr));
kasan_poison_vmalloc(mem, size);
if (likely(count <= VMAP_MAX_ALLOC)) {
debug_check_no_locks_freed(mem, size);
vb_free(addr, size);
return;
}
va = find_unlink_vmap_area(addr);
if (WARN_ON_ONCE(!va))
return;
debug_check_no_locks_freed((void *)va->va_start,
(va->va_end - va->va_start));
free_unmap_vmap_area(va);
}
EXPORT_SYMBOL(vm_unmap_ram);
/**
* vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
* @pages: an array of pointers to the pages to be mapped
* @count: number of pages
* @node: prefer to allocate data structures on this node
*
* If you use this function for less than VMAP_MAX_ALLOC pages, it could be
* faster than vmap so it's good. But if you mix long-life and short-life
* objects with vm_map_ram(), it could consume lots of address space through
* fragmentation (especially on a 32bit machine). You could see failures in
* the end. Please use this function for short-lived objects.
*
* Returns: a pointer to the address that has been mapped, or %NULL on failure
*/
void *vm_map_ram(struct page **pages, unsigned int count, int node)
{
unsigned long size = (unsigned long)count << PAGE_SHIFT;
unsigned long addr;
void *mem;
if (likely(count <= VMAP_MAX_ALLOC)) {
mem = vb_alloc(size, GFP_KERNEL);
if (IS_ERR(mem))
return NULL;
addr = (unsigned long)mem;
} else {
struct vmap_area *va;
va = alloc_vmap_area(size, PAGE_SIZE,
VMALLOC_START, VMALLOC_END,
node, GFP_KERNEL, VMAP_RAM);
if (IS_ERR(va))
return NULL;
addr = va->va_start;
mem = (void *)addr;
}
if (vmap_pages_range(addr, addr + size, PAGE_KERNEL,
pages, PAGE_SHIFT) < 0) {
vm_unmap_ram(mem, count);
return NULL;
}
/*
* Mark the pages as accessible, now that they are mapped.
* With hardware tag-based KASAN, marking is skipped for
* non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
*/
mem = kasan_unpoison_vmalloc(mem, size, KASAN_VMALLOC_PROT_NORMAL);
return mem;
}
EXPORT_SYMBOL(vm_map_ram);
static struct vm_struct *vmlist __initdata;
static inline unsigned int vm_area_page_order(struct vm_struct *vm)
{
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
return vm->page_order;
#else
return 0;
#endif
}
static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order)
{
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
vm->page_order = order;
#else
BUG_ON(order != 0);
#endif
}
/**
* vm_area_add_early - add vmap area early during boot
* @vm: vm_struct to add
*
* This function is used to add fixed kernel vm area to vmlist before
* vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
* should contain proper values and the other fields should be zero.
*
* DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
*/
void __init vm_area_add_early(struct vm_struct *vm)
{
struct vm_struct *tmp, **p;
BUG_ON(vmap_initialized);
for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
if (tmp->addr >= vm->addr) {
BUG_ON(tmp->addr < vm->addr + vm->size);
break;
} else
BUG_ON(tmp->addr + tmp->size > vm->addr);
}
vm->next = *p;
*p = vm;
}
/**
* vm_area_register_early - register vmap area early during boot
* @vm: vm_struct to register
* @align: requested alignment
*
* This function is used to register kernel vm area before
* vmalloc_init() is called. @vm->size and @vm->flags should contain
* proper values on entry and other fields should be zero. On return,
* vm->addr contains the allocated address.
*
* DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
*/
void __init vm_area_register_early(struct vm_struct *vm, size_t align)
{
unsigned long addr = ALIGN(VMALLOC_START, align);
struct vm_struct *cur, **p;
BUG_ON(vmap_initialized);
for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) {
if ((unsigned long)cur->addr - addr >= vm->size)
break;
addr = ALIGN((unsigned long)cur->addr + cur->size, align);
}
BUG_ON(addr > VMALLOC_END - vm->size);
vm->addr = (void *)addr;
vm->next = *p;
*p = vm;
kasan_populate_early_vm_area_shadow(vm->addr, vm->size);
}
static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
struct vmap_area *va, unsigned long flags, const void *caller)
{
vm->flags = flags;
vm->addr = (void *)va->va_start;
vm->size = va->va_end - va->va_start;
vm->caller = caller;
va->vm = vm;
}
static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
unsigned long flags, const void *caller)
{
struct vmap_node *vn = addr_to_node(va->va_start);
spin_lock(&vn->busy.lock);
setup_vmalloc_vm_locked(vm, va, flags, caller);
spin_unlock(&vn->busy.lock);
}
static void clear_vm_uninitialized_flag(struct vm_struct *vm)
{
/*
* Before removing VM_UNINITIALIZED,
* we should make sure that vm has proper values.
* Pair with smp_rmb() in show_numa_info().
*/
smp_wmb();
vm->flags &= ~VM_UNINITIALIZED;
}
static struct vm_struct *__get_vm_area_node(unsigned long size,
unsigned long align, unsigned long shift, unsigned long flags,
unsigned long start, unsigned long end, int node,
gfp_t gfp_mask, const void *caller)
{
struct vmap_area *va;
struct vm_struct *area;
unsigned long requested_size = size;
BUG_ON(in_interrupt()); size = ALIGN(size, 1ul << shift);
if (unlikely(!size))
return NULL;
if (flags & VM_IOREMAP) align = 1ul << clamp_t(int, get_count_order_long(size),
PAGE_SHIFT, IOREMAP_MAX_ORDER);
area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
if (unlikely(!area))
return NULL;
if (!(flags & VM_NO_GUARD)) size += PAGE_SIZE; va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0);
if (IS_ERR(va)) {
kfree(area);
return NULL;
}
setup_vmalloc_vm(area, va, flags, caller);
/*
* Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a
* best-effort approach, as they can be mapped outside of vmalloc code.
* For VM_ALLOC mappings, the pages are marked as accessible after
* getting mapped in __vmalloc_node_range().
* With hardware tag-based KASAN, marking is skipped for
* non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
*/
if (!(flags & VM_ALLOC)) area->addr = kasan_unpoison_vmalloc(area->addr, requested_size,
KASAN_VMALLOC_PROT_NORMAL);
return area;
}
struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
unsigned long start, unsigned long end,
const void *caller)
{
return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end,
NUMA_NO_NODE, GFP_KERNEL, caller);
}
/**
* get_vm_area - reserve a contiguous kernel virtual area
* @size: size of the area
* @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
*
* Search an area of @size in the kernel virtual mapping area,
* and reserved it for out purposes. Returns the area descriptor
* on success or %NULL on failure.
*
* Return: the area descriptor on success or %NULL on failure.
*/
struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
{
return __get_vm_area_node(size, 1, PAGE_SHIFT, flags,
VMALLOC_START, VMALLOC_END,
NUMA_NO_NODE, GFP_KERNEL,
__builtin_return_address(0));
}
struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
const void *caller)
{
return __get_vm_area_node(size, 1, PAGE_SHIFT, flags,
VMALLOC_START, VMALLOC_END,
NUMA_NO_NODE, GFP_KERNEL, caller);
}
/**
* find_vm_area - find a continuous kernel virtual area
* @addr: base address
*
* Search for the kernel VM area starting at @addr, and return it.
* It is up to the caller to do all required locking to keep the returned
* pointer valid.
*
* Return: the area descriptor on success or %NULL on failure.
*/
struct vm_struct *find_vm_area(const void *addr)
{
struct vmap_area *va;
va = find_vmap_area((unsigned long)addr);
if (!va)
return NULL;
return va->vm;
}
/**
* remove_vm_area - find and remove a continuous kernel virtual area
* @addr: base address
*
* Search for the kernel VM area starting at @addr, and remove it.
* This function returns the found VM area, but using it is NOT safe
* on SMP machines, except for its size or flags.
*
* Return: the area descriptor on success or %NULL on failure.
*/
struct vm_struct *remove_vm_area(const void *addr)
{
struct vmap_area *va;
struct vm_struct *vm;
might_sleep(); if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
addr))
return NULL;
va = find_unlink_vmap_area((unsigned long)addr); if (!va || !va->vm)
return NULL;
vm = va->vm;
debug_check_no_locks_freed(vm->addr, get_vm_area_size(vm)); debug_check_no_obj_freed(vm->addr, get_vm_area_size(vm));
kasan_free_module_shadow(vm);
kasan_poison_vmalloc(vm->addr, get_vm_area_size(vm)); free_unmap_vmap_area(va);
return vm;
}
static inline void set_area_direct_map(const struct vm_struct *area,
int (*set_direct_map)(struct page *page))
{
int i;
/* HUGE_VMALLOC passes small pages to set_direct_map */
for (i = 0; i < area->nr_pages; i++) if (page_address(area->pages[i])) set_direct_map(area->pages[i]);
}
/*
* Flush the vm mapping and reset the direct map.
*/
static void vm_reset_perms(struct vm_struct *area)
{
unsigned long start = ULONG_MAX, end = 0;
unsigned int page_order = vm_area_page_order(area);
int flush_dmap = 0;
int i;
/*
* Find the start and end range of the direct mappings to make sure that
* the vm_unmap_aliases() flush includes the direct map.
*/
for (i = 0; i < area->nr_pages; i += 1U << page_order) { unsigned long addr = (unsigned long)page_address(area->pages[i]);
if (addr) {
unsigned long page_size;
page_size = PAGE_SIZE << page_order; start = min(addr, start);
end = max(addr + page_size, end);
flush_dmap = 1;
}
}
/*
* Set direct map to something invalid so that it won't be cached if
* there are any accesses after the TLB flush, then flush the TLB and
* reset the direct map permissions to the default.
*/
set_area_direct_map(area, set_direct_map_invalid_noflush); _vm_unmap_aliases(start, end, flush_dmap); set_area_direct_map(area, set_direct_map_default_noflush);
}
static void delayed_vfree_work(struct work_struct *w)
{
struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
struct llist_node *t, *llnode;
llist_for_each_safe(llnode, t, llist_del_all(&p->list))
vfree(llnode);
}
/**
* vfree_atomic - release memory allocated by vmalloc()
* @addr: memory base address
*
* This one is just like vfree() but can be called in any atomic context
* except NMIs.
*/
void vfree_atomic(const void *addr)
{
struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
BUG_ON(in_nmi());
kmemleak_free(addr);
/*
* Use raw_cpu_ptr() because this can be called from preemptible
* context. Preemption is absolutely fine here, because the llist_add()
* implementation is lockless, so it works even if we are adding to
* another cpu's list. schedule_work() should be fine with this too.
*/
if (addr && llist_add((struct llist_node *)addr, &p->list))
schedule_work(&p->wq);
}
/**
* vfree - Release memory allocated by vmalloc()
* @addr: Memory base address
*
* Free the virtually continuous memory area starting at @addr, as obtained
* from one of the vmalloc() family of APIs. This will usually also free the
* physical memory underlying the virtual allocation, but that memory is
* reference counted, so it will not be freed until the last user goes away.
*
* If @addr is NULL, no operation is performed.
*
* Context:
* May sleep if called *not* from interrupt context.
* Must not be called in NMI context (strictly speaking, it could be
* if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
* conventions for vfree() arch-dependent would be a really bad idea).
*/
void vfree(const void *addr)
{
struct vm_struct *vm;
int i;
if (unlikely(in_interrupt())) { vfree_atomic(addr);
return;
}
BUG_ON(in_nmi());
kmemleak_free(addr);
might_sleep();
if (!addr)
return;
vm = remove_vm_area(addr);
if (unlikely(!vm)) {
WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
addr);
return;
}
if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS)) vm_reset_perms(vm); for (i = 0; i < vm->nr_pages; i++) { struct page *page = vm->pages[i]; BUG_ON(!page); mod_memcg_page_state(page, MEMCG_VMALLOC, -1);
/*
* High-order allocs for huge vmallocs are split, so
* can be freed as an array of order-0 allocations
*/
__free_page(page);
cond_resched();
}
atomic_long_sub(vm->nr_pages, &nr_vmalloc_pages);
kvfree(vm->pages);
kfree(vm);
}
EXPORT_SYMBOL(vfree);
/**
* vunmap - release virtual mapping obtained by vmap()
* @addr: memory base address
*
* Free the virtually contiguous memory area starting at @addr,
* which was created from the page array passed to vmap().
*
* Must not be called in interrupt context.
*/
void vunmap(const void *addr)
{
struct vm_struct *vm;
BUG_ON(in_interrupt());
might_sleep();
if (!addr)
return;
vm = remove_vm_area(addr);
if (unlikely(!vm)) {
WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n",
addr);
return;
}
kfree(vm);
}
EXPORT_SYMBOL(vunmap);
/**
* vmap - map an array of pages into virtually contiguous space
* @pages: array of page pointers
* @count: number of pages to map
* @flags: vm_area->flags
* @prot: page protection for the mapping
*
* Maps @count pages from @pages into contiguous kernel virtual space.
* If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself
* (which must be kmalloc or vmalloc memory) and one reference per pages in it
* are transferred from the caller to vmap(), and will be freed / dropped when
* vfree() is called on the return value.
*
* Return: the address of the area or %NULL on failure
*/
void *vmap(struct page **pages, unsigned int count,
unsigned long flags, pgprot_t prot)
{
struct vm_struct *area;
unsigned long addr;
unsigned long size; /* In bytes */
might_sleep();
if (WARN_ON_ONCE(flags & VM_FLUSH_RESET_PERMS))
return NULL;
/*
* Your top guard is someone else's bottom guard. Not having a top
* guard compromises someone else's mappings too.
*/
if (WARN_ON_ONCE(flags & VM_NO_GUARD))
flags &= ~VM_NO_GUARD;
if (count > totalram_pages())
return NULL;
size = (unsigned long)count << PAGE_SHIFT;
area = get_vm_area_caller(size, flags, __builtin_return_address(0));
if (!area)
return NULL;
addr = (unsigned long)area->addr;
if (vmap_pages_range(addr, addr + size, pgprot_nx(prot),
pages, PAGE_SHIFT) < 0) {
vunmap(area->addr);
return NULL;
}
if (flags & VM_MAP_PUT_PAGES) {
area->pages = pages;
area->nr_pages = count;
}
return area->addr;
}
EXPORT_SYMBOL(vmap);
#ifdef CONFIG_VMAP_PFN
struct vmap_pfn_data {
unsigned long *pfns;
pgprot_t prot;
unsigned int idx;
};
static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private)
{
struct vmap_pfn_data *data = private;
unsigned long pfn = data->pfns[data->idx];
pte_t ptent;
if (WARN_ON_ONCE(pfn_valid(pfn)))
return -EINVAL;
ptent = pte_mkspecial(pfn_pte(pfn, data->prot));
set_pte_at(&init_mm, addr, pte, ptent);
data->idx++;
return 0;
}
/**
* vmap_pfn - map an array of PFNs into virtually contiguous space
* @pfns: array of PFNs
* @count: number of pages to map
* @prot: page protection for the mapping
*
* Maps @count PFNs from @pfns into contiguous kernel virtual space and returns
* the start address of the mapping.
*/
void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot)
{
struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) };
struct vm_struct *area;
area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP,
__builtin_return_address(0));
if (!area)
return NULL;
if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
count * PAGE_SIZE, vmap_pfn_apply, &data)) {
free_vm_area(area);
return NULL;
}
flush_cache_vmap((unsigned long)area->addr,
(unsigned long)area->addr + count * PAGE_SIZE);
return area->addr;
}
EXPORT_SYMBOL_GPL(vmap_pfn);
#endif /* CONFIG_VMAP_PFN */
static inline unsigned int
vm_area_alloc_pages(gfp_t gfp, int nid,
unsigned int order, unsigned int nr_pages, struct page **pages)
{
unsigned int nr_allocated = 0;
gfp_t alloc_gfp = gfp;
bool nofail = false;
struct page *page;
int i;
/*
* For order-0 pages we make use of bulk allocator, if
* the page array is partly or not at all populated due
* to fails, fallback to a single page allocator that is
* more permissive.
*/
if (!order) {
/* bulk allocator doesn't support nofail req. officially */
gfp_t bulk_gfp = gfp & ~__GFP_NOFAIL;
while (nr_allocated < nr_pages) {
unsigned int nr, nr_pages_request;
/*
* A maximum allowed request is hard-coded and is 100
* pages per call. That is done in order to prevent a
* long preemption off scenario in the bulk-allocator
* so the range is [1:100].
*/
nr_pages_request = min(100U, nr_pages - nr_allocated);
/* memory allocation should consider mempolicy, we can't
* wrongly use nearest node when nid == NUMA_NO_NODE,
* otherwise memory may be allocated in only one node,
* but mempolicy wants to alloc memory by interleaving.
*/
if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE)
nr = alloc_pages_bulk_array_mempolicy(bulk_gfp,
nr_pages_request,
pages + nr_allocated);
else
nr = alloc_pages_bulk_array_node(bulk_gfp, nid,
nr_pages_request,
pages + nr_allocated);
nr_allocated += nr;
cond_resched();
/*
* If zero or pages were obtained partly,
* fallback to a single page allocator.
*/
if (nr != nr_pages_request)
break;
}
} else if (gfp & __GFP_NOFAIL) {
/*
* Higher order nofail allocations are really expensive and
* potentially dangerous (pre-mature OOM, disruptive reclaim
* and compaction etc.
*/
alloc_gfp &= ~__GFP_NOFAIL;
nofail = true;
}
/* High-order pages or fallback path if "bulk" fails. */
while (nr_allocated < nr_pages) { if (fatal_signal_pending(current))
break;
if (nid == NUMA_NO_NODE) page = alloc_pages(alloc_gfp, order);
else
page = alloc_pages_node(nid, alloc_gfp, order); if (unlikely(!page)) { if (!nofail)
break;
/* fall back to the zero order allocations */
alloc_gfp |= __GFP_NOFAIL;
order = 0;
continue;
}
/*
* Higher order allocations must be able to be treated as
* indepdenent small pages by callers (as they can with
* small-page vmallocs). Some drivers do their own refcounting
* on vmalloc_to_page() pages, some use page->mapping,
* page->lru, etc.
*/
if (order) split_page(page, order);
/*
* Careful, we allocate and map page-order pages, but
* tracking is done per PAGE_SIZE page so as to keep the
* vm_struct APIs independent of the physical/mapped size.
*/
for (i = 0; i < (1U << order); i++) pages[nr_allocated + i] = page + i; cond_resched();
nr_allocated += 1U << order;
}
return nr_allocated;
}
static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
pgprot_t prot, unsigned int page_shift,
int node)
{
const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
bool nofail = gfp_mask & __GFP_NOFAIL;
unsigned long addr = (unsigned long)area->addr; unsigned long size = get_vm_area_size(area);
unsigned long array_size;
unsigned int nr_small_pages = size >> PAGE_SHIFT;
unsigned int page_order;
unsigned int flags;
int ret;
array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
gfp_mask |= __GFP_HIGHMEM;
/* Please note that the recursion is strictly bounded. */
if (array_size > PAGE_SIZE) {
area->pages = __vmalloc_node(array_size, 1, nested_gfp, node,
area->caller);
} else {
area->pages = kmalloc_node(array_size, nested_gfp, node);
}
if (!area->pages) {
warn_alloc(gfp_mask, NULL,
"vmalloc error: size %lu, failed to allocated page array size %lu",
nr_small_pages * PAGE_SIZE, array_size);
free_vm_area(area);
return NULL;
}
set_vm_area_page_order(area, page_shift - PAGE_SHIFT);
page_order = vm_area_page_order(area);
area->nr_pages = vm_area_alloc_pages(gfp_mask | __GFP_NOWARN,
node, page_order, nr_small_pages, area->pages);
atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
if (gfp_mask & __GFP_ACCOUNT) {
int i;
for (i = 0; i < area->nr_pages; i++) mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1);
}
/*
* If not enough pages were obtained to accomplish an
* allocation request, free them via vfree() if any.
*/
if (area->nr_pages != nr_small_pages) {
/*
* vm_area_alloc_pages() can fail due to insufficient memory but
* also:-
*
* - a pending fatal signal
* - insufficient huge page-order pages
*
* Since we always retry allocations at order-0 in the huge page
* case a warning for either is spurious.
*/
if (!fatal_signal_pending(current) && page_order == 0)
warn_alloc(gfp_mask, NULL,
"vmalloc error: size %lu, failed to allocate pages",
area->nr_pages * PAGE_SIZE);
goto fail;
}
/*
* page tables allocations ignore external gfp mask, enforce it
* by the scope API
*/
if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) flags = memalloc_nofs_save(); else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) flags = memalloc_noio_save();
do {
ret = vmap_pages_range(addr, addr + size, prot, area->pages,
page_shift);
if (nofail && (ret < 0)) schedule_timeout_uninterruptible(1);
} while (nofail && (ret < 0));
if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) memalloc_nofs_restore(flags); else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) memalloc_noio_restore(flags); if (ret < 0) {
warn_alloc(gfp_mask, NULL,
"vmalloc error: size %lu, failed to map pages",
area->nr_pages * PAGE_SIZE);
goto fail;
}
return area->addr;
fail:
vfree(area->addr);
return NULL;
}
/**
* __vmalloc_node_range - allocate virtually contiguous memory
* @size: allocation size
* @align: desired alignment
* @start: vm area range start
* @end: vm area range end
* @gfp_mask: flags for the page level allocator
* @prot: protection mask for the allocated pages
* @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
* @node: node to use for allocation or NUMA_NO_NODE
* @caller: caller's return address
*
* Allocate enough pages to cover @size from the page level
* allocator with @gfp_mask flags. Please note that the full set of gfp
* flags are not supported. GFP_KERNEL, GFP_NOFS and GFP_NOIO are all
* supported.
* Zone modifiers are not supported. From the reclaim modifiers
* __GFP_DIRECT_RECLAIM is required (aka GFP_NOWAIT is not supported)
* and only __GFP_NOFAIL is supported (i.e. __GFP_NORETRY and
* __GFP_RETRY_MAYFAIL are not supported).
*
* __GFP_NOWARN can be used to suppress failures messages.
*
* Map them into contiguous kernel virtual space, using a pagetable
* protection of @prot.
*
* Return: the address of the area or %NULL on failure
*/
void *__vmalloc_node_range(unsigned long size, unsigned long align,
unsigned long start, unsigned long end, gfp_t gfp_mask,
pgprot_t prot, unsigned long vm_flags, int node,
const void *caller)
{
struct vm_struct *area;
void *ret;
kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE;
unsigned long real_size = size;
unsigned long real_align = align;
unsigned int shift = PAGE_SHIFT;
if (WARN_ON_ONCE(!size)) return NULL; if ((size >> PAGE_SHIFT) > totalram_pages()) { warn_alloc(gfp_mask, NULL,
"vmalloc error: size %lu, exceeds total pages",
real_size);
return NULL;
}
if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) {
unsigned long size_per_node;
/*
* Try huge pages. Only try for PAGE_KERNEL allocations,
* others like modules don't yet expect huge pages in
* their allocations due to apply_to_page_range not
* supporting them.
*/
size_per_node = size;
if (node == NUMA_NO_NODE) size_per_node /= num_online_nodes(); if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE)
shift = PMD_SHIFT;
else
shift = arch_vmap_pte_supported_shift(size_per_node);
align = max(real_align, 1UL << shift);
size = ALIGN(real_size, 1UL << shift);
}
again:
area = __get_vm_area_node(real_size, align, shift, VM_ALLOC |
VM_UNINITIALIZED | vm_flags, start, end, node,
gfp_mask, caller);
if (!area) {
bool nofail = gfp_mask & __GFP_NOFAIL;
warn_alloc(gfp_mask, NULL,
"vmalloc error: size %lu, vm_struct allocation failed%s",
real_size, (nofail) ? ". Retrying." : "");
if (nofail) {
schedule_timeout_uninterruptible(1);
goto again;
}
goto fail;
}
/*
* Prepare arguments for __vmalloc_area_node() and
* kasan_unpoison_vmalloc().
*/
if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) {
if (kasan_hw_tags_enabled()) {
/*
* Modify protection bits to allow tagging.
* This must be done before mapping.
*/
prot = arch_vmap_pgprot_tagged(prot);
/*
* Skip page_alloc poisoning and zeroing for physical
* pages backing VM_ALLOC mapping. Memory is instead
* poisoned and zeroed by kasan_unpoison_vmalloc().
*/
gfp_mask |= __GFP_SKIP_KASAN | __GFP_SKIP_ZERO;
}
/* Take note that the mapping is PAGE_KERNEL. */
kasan_flags |= KASAN_VMALLOC_PROT_NORMAL;
}
/* Allocate physical pages and map them into vmalloc space. */
ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node);
if (!ret)
goto fail;
/*
* Mark the pages as accessible, now that they are mapped.
* The condition for setting KASAN_VMALLOC_INIT should complement the
* one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check
* to make sure that memory is initialized under the same conditions.
* Tag-based KASAN modes only assign tags to normal non-executable
* allocations, see __kasan_unpoison_vmalloc().
*/
kasan_flags |= KASAN_VMALLOC_VM_ALLOC; if (!want_init_on_free() && want_init_on_alloc(gfp_mask) &&
(gfp_mask & __GFP_SKIP_ZERO))
kasan_flags |= KASAN_VMALLOC_INIT;
/* KASAN_VMALLOC_PROT_NORMAL already set if required. */
area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags);
/*
* In this function, newly allocated vm_struct has VM_UNINITIALIZED
* flag. It means that vm_struct is not fully initialized.
* Now, it is fully initialized, so remove this flag here.
*/
clear_vm_uninitialized_flag(area);
size = PAGE_ALIGN(size);
if (!(vm_flags & VM_DEFER_KMEMLEAK))
kmemleak_vmalloc(area, size, gfp_mask);
return area->addr;
fail:
if (shift > PAGE_SHIFT) {
shift = PAGE_SHIFT;
align = real_align;
size = real_size;
goto again;
}
return NULL;
}
/**
* __vmalloc_node - allocate virtually contiguous memory
* @size: allocation size
* @align: desired alignment
* @gfp_mask: flags for the page level allocator
* @node: node to use for allocation or NUMA_NO_NODE
* @caller: caller's return address
*
* Allocate enough pages to cover @size from the page level allocator with
* @gfp_mask flags. Map them into contiguous kernel virtual space.
*
* Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
* and __GFP_NOFAIL are not supported
*
* Any use of gfp flags outside of GFP_KERNEL should be consulted
* with mm people.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *__vmalloc_node(unsigned long size, unsigned long align,
gfp_t gfp_mask, int node, const void *caller)
{
return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
gfp_mask, PAGE_KERNEL, 0, node, caller);
}
/*
* This is only for performance analysis of vmalloc and stress purpose.
* It is required by vmalloc test module, therefore do not use it other
* than that.
*/
#ifdef CONFIG_TEST_VMALLOC_MODULE
EXPORT_SYMBOL_GPL(__vmalloc_node);
#endif
void *__vmalloc(unsigned long size, gfp_t gfp_mask)
{
return __vmalloc_node(size, 1, gfp_mask, NUMA_NO_NODE,
__builtin_return_address(0));
}
EXPORT_SYMBOL(__vmalloc);
/**
* vmalloc - allocate virtually contiguous memory
* @size: allocation size
*
* Allocate enough pages to cover @size from the page level
* allocator and map them into contiguous kernel virtual space.
*
* For tight control over page level allocator and protection flags
* use __vmalloc() instead.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vmalloc(unsigned long size)
{
return __vmalloc_node(size, 1, GFP_KERNEL, NUMA_NO_NODE,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vmalloc);
/**
* vmalloc_huge - allocate virtually contiguous memory, allow huge pages
* @size: allocation size
* @gfp_mask: flags for the page level allocator
*
* Allocate enough pages to cover @size from the page level
* allocator and map them into contiguous kernel virtual space.
* If @size is greater than or equal to PMD_SIZE, allow using
* huge pages for the memory
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vmalloc_huge(unsigned long size, gfp_t gfp_mask)
{
return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
NUMA_NO_NODE, __builtin_return_address(0));
}
EXPORT_SYMBOL_GPL(vmalloc_huge);
/**
* vzalloc - allocate virtually contiguous memory with zero fill
* @size: allocation size
*
* Allocate enough pages to cover @size from the page level
* allocator and map them into contiguous kernel virtual space.
* The memory allocated is set to zero.
*
* For tight control over page level allocator and protection flags
* use __vmalloc() instead.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vzalloc(unsigned long size)
{
return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vzalloc);
/**
* vmalloc_user - allocate zeroed virtually contiguous memory for userspace
* @size: allocation size
*
* The resulting memory area is zeroed so it can be mapped to userspace
* without leaking data.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vmalloc_user(unsigned long size)
{
return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL,
VM_USERMAP, NUMA_NO_NODE,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vmalloc_user);
/**
* vmalloc_node - allocate memory on a specific node
* @size: allocation size
* @node: numa node
*
* Allocate enough pages to cover @size from the page level
* allocator and map them into contiguous kernel virtual space.
*
* For tight control over page level allocator and protection flags
* use __vmalloc() instead.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vmalloc_node(unsigned long size, int node)
{
return __vmalloc_node(size, 1, GFP_KERNEL, node,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vmalloc_node);
/**
* vzalloc_node - allocate memory on a specific node with zero fill
* @size: allocation size
* @node: numa node
*
* Allocate enough pages to cover @size from the page level
* allocator and map them into contiguous kernel virtual space.
* The memory allocated is set to zero.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vzalloc_node(unsigned long size, int node)
{
return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_ZERO, node,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vzalloc_node);
#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
#define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)
#else
/*
* 64b systems should always have either DMA or DMA32 zones. For others
* GFP_DMA32 should do the right thing and use the normal zone.
*/
#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
#endif
/**
* vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
* @size: allocation size
*
* Allocate enough 32bit PA addressable pages to cover @size from the
* page level allocator and map them into contiguous kernel virtual space.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vmalloc_32(unsigned long size)
{
return __vmalloc_node(size, 1, GFP_VMALLOC32, NUMA_NO_NODE,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vmalloc_32);
/**
* vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
* @size: allocation size
*
* The resulting memory area is 32bit addressable and zeroed so it can be
* mapped to userspace without leaking data.
*
* Return: pointer to the allocated memory or %NULL on error
*/
void *vmalloc_32_user(unsigned long size)
{
return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END,
GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
VM_USERMAP, NUMA_NO_NODE,
__builtin_return_address(0));
}
EXPORT_SYMBOL(vmalloc_32_user);
/*
* Atomically zero bytes in the iterator.
*
* Returns the number of zeroed bytes.
*/
static size_t zero_iter(struct iov_iter *iter, size_t count)
{
size_t remains = count;
while (remains > 0) {
size_t num, copied;
num = min_t(size_t, remains, PAGE_SIZE);
copied = copy_page_to_iter_nofault(ZERO_PAGE(0), 0, num, iter);
remains -= copied;
if (copied < num)
break;
}
return count - remains;
}
/*
* small helper routine, copy contents to iter from addr.
* If the page is not present, fill zero.
*
* Returns the number of copied bytes.
*/
static size_t aligned_vread_iter(struct iov_iter *iter,
const char *addr, size_t count)
{
size_t remains = count;
struct page *page;
while (remains > 0) {
unsigned long offset, length;
size_t copied = 0;
offset = offset_in_page(addr);
length = PAGE_SIZE - offset;
if (length > remains)
length = remains;
page = vmalloc_to_page(addr);
/*
* To do safe access to this _mapped_ area, we need lock. But
* adding lock here means that we need to add overhead of
* vmalloc()/vfree() calls for this _debug_ interface, rarely
* used. Instead of that, we'll use an local mapping via
* copy_page_to_iter_nofault() and accept a small overhead in
* this access function.
*/
if (page)
copied = copy_page_to_iter_nofault(page, offset,
length, iter);
else
copied = zero_iter(iter, length);
addr += copied;
remains -= copied;
if (copied != length)
break;
}
return count - remains;
}
/*
* Read from a vm_map_ram region of memory.
*
* Returns the number of copied bytes.
*/
static size_t vmap_ram_vread_iter(struct iov_iter *iter, const char *addr,
size_t count, unsigned long flags)
{
char *start;
struct vmap_block *vb;
struct xarray *xa;
unsigned long offset;
unsigned int rs, re;
size_t remains, n;
/*
* If it's area created by vm_map_ram() interface directly, but
* not further subdividing and delegating management to vmap_block,
* handle it here.
*/
if (!(flags & VMAP_BLOCK))
return aligned_vread_iter(iter, addr, count);
remains = count;
/*
* Area is split into regions and tracked with vmap_block, read out
* each region and zero fill the hole between regions.
*/
xa = addr_to_vb_xa((unsigned long) addr);
vb = xa_load(xa, addr_to_vb_idx((unsigned long)addr));
if (!vb)
goto finished_zero;
spin_lock(&vb->lock);
if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) {
spin_unlock(&vb->lock);
goto finished_zero;
}
for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) {
size_t copied;
if (remains == 0)
goto finished;
start = vmap_block_vaddr(vb->va->va_start, rs);
if (addr < start) {
size_t to_zero = min_t(size_t, start - addr, remains);
size_t zeroed = zero_iter(iter, to_zero);
addr += zeroed;
remains -= zeroed;
if (remains == 0 || zeroed != to_zero)
goto finished;
}
/*it could start reading from the middle of used region*/
offset = offset_in_page(addr);
n = ((re - rs + 1) << PAGE_SHIFT) - offset;
if (n > remains)
n = remains;
copied = aligned_vread_iter(iter, start + offset, n);
addr += copied;
remains -= copied;
if (copied != n)
goto finished;
}
spin_unlock(&vb->lock);
finished_zero:
/* zero-fill the left dirty or free regions */
return count - remains + zero_iter(iter, remains);
finished:
/* We couldn't copy/zero everything */
spin_unlock(&vb->lock);
return count - remains;
}
/**
* vread_iter() - read vmalloc area in a safe way to an iterator.
* @iter: the iterator to which data should be written.
* @addr: vm address.
* @count: number of bytes to be read.
*
* This function checks that addr is a valid vmalloc'ed area, and
* copy data from that area to a given buffer. If the given memory range
* of [addr...addr+count) includes some valid address, data is copied to
* proper area of @buf. If there are memory holes, they'll be zero-filled.
* IOREMAP area is treated as memory hole and no copy is done.
*
* If [addr...addr+count) doesn't includes any intersects with alive
* vm_struct area, returns 0. @buf should be kernel's buffer.
*
* Note: In usual ops, vread() is never necessary because the caller
* should know vmalloc() area is valid and can use memcpy().
* This is for routines which have to access vmalloc area without
* any information, as /proc/kcore.
*
* Return: number of bytes for which addr and buf should be increased
* (same number as @count) or %0 if [addr...addr+count) doesn't
* include any intersection with valid vmalloc area
*/
long vread_iter(struct iov_iter *iter, const char *addr, size_t count)
{
struct vmap_node *vn;
struct vmap_area *va;
struct vm_struct *vm;
char *vaddr;
size_t n, size, flags, remains;
unsigned long next;
addr = kasan_reset_tag(addr);
/* Don't allow overflow */
if ((unsigned long) addr + count < count)
count = -(unsigned long) addr;
remains = count;
vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va);
if (!vn)
goto finished_zero;
/* no intersects with alive vmap_area */
if ((unsigned long)addr + remains <= va->va_start)
goto finished_zero;
do {
size_t copied;
if (remains == 0)
goto finished;
vm = va->vm;
flags = va->flags & VMAP_FLAGS_MASK;
/*
* VMAP_BLOCK indicates a sub-type of vm_map_ram area, need
* be set together with VMAP_RAM.
*/
WARN_ON(flags == VMAP_BLOCK);
if (!vm && !flags)
goto next_va;
if (vm && (vm->flags & VM_UNINITIALIZED))
goto next_va;
/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
smp_rmb();
vaddr = (char *) va->va_start;
size = vm ? get_vm_area_size(vm) : va_size(va);
if (addr >= vaddr + size)
goto next_va;
if (addr < vaddr) {
size_t to_zero = min_t(size_t, vaddr - addr, remains);
size_t zeroed = zero_iter(iter, to_zero);
addr += zeroed;
remains -= zeroed;
if (remains == 0 || zeroed != to_zero)
goto finished;
}
n = vaddr + size - addr;
if (n > remains)
n = remains;
if (flags & VMAP_RAM)
copied = vmap_ram_vread_iter(iter, addr, n, flags);
else if (!(vm && (vm->flags & (VM_IOREMAP | VM_SPARSE))))
copied = aligned_vread_iter(iter, addr, n);
else /* IOREMAP | SPARSE area is treated as memory hole */
copied = zero_iter(iter, n);
addr += copied;
remains -= copied;
if (copied != n)
goto finished;
next_va:
next = va->va_end;
spin_unlock(&vn->busy.lock);
} while ((vn = find_vmap_area_exceed_addr_lock(next, &va)));
finished_zero:
if (vn)
spin_unlock(&vn->busy.lock);
/* zero-fill memory holes */
return count - remains + zero_iter(iter, remains);
finished:
/* Nothing remains, or We couldn't copy/zero everything. */
if (vn)
spin_unlock(&vn->busy.lock);
return count - remains;
}
/**
* remap_vmalloc_range_partial - map vmalloc pages to userspace
* @vma: vma to cover
* @uaddr: target user address to start at
* @kaddr: virtual address of vmalloc kernel memory
* @pgoff: offset from @kaddr to start at
* @size: size of map area
*
* Returns: 0 for success, -Exxx on failure
*
* This function checks that @kaddr is a valid vmalloc'ed area,
* and that it is big enough to cover the range starting at
* @uaddr in @vma. Will return failure if that criteria isn't
* met.
*
* Similar to remap_pfn_range() (see mm/memory.c)
*/
int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
void *kaddr, unsigned long pgoff,
unsigned long size)
{
struct vm_struct *area;
unsigned long off;
unsigned long end_index;
if (check_shl_overflow(pgoff, PAGE_SHIFT, &off))
return -EINVAL;
size = PAGE_ALIGN(size);
if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
return -EINVAL;
area = find_vm_area(kaddr);
if (!area)
return -EINVAL;
if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))
return -EINVAL;
if (check_add_overflow(size, off, &end_index) ||
end_index > get_vm_area_size(area))
return -EINVAL;
kaddr += off;
do {
struct page *page = vmalloc_to_page(kaddr);
int ret;
ret = vm_insert_page(vma, uaddr, page);
if (ret)
return ret;
uaddr += PAGE_SIZE;
kaddr += PAGE_SIZE;
size -= PAGE_SIZE;
} while (size > 0);
vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
return 0;
}
/**
* remap_vmalloc_range - map vmalloc pages to userspace
* @vma: vma to cover (map full range of vma)
* @addr: vmalloc memory
* @pgoff: number of pages into addr before first page to map
*
* Returns: 0 for success, -Exxx on failure
*
* This function checks that addr is a valid vmalloc'ed area, and
* that it is big enough to cover the vma. Will return failure if
* that criteria isn't met.
*
* Similar to remap_pfn_range() (see mm/memory.c)
*/
int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
unsigned long pgoff)
{
return remap_vmalloc_range_partial(vma, vma->vm_start,
addr, pgoff,
vma->vm_end - vma->vm_start);
}
EXPORT_SYMBOL(remap_vmalloc_range);
void free_vm_area(struct vm_struct *area)
{
struct vm_struct *ret;
ret = remove_vm_area(area->addr);
BUG_ON(ret != area); kfree(area);
}
EXPORT_SYMBOL_GPL(free_vm_area);
#ifdef CONFIG_SMP
static struct vmap_area *node_to_va(struct rb_node *n)
{
return rb_entry_safe(n, struct vmap_area, rb_node);
}
/**
* pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
* @addr: target address
*
* Returns: vmap_area if it is found. If there is no such area
* the first highest(reverse order) vmap_area is returned
* i.e. va->va_start < addr && va->va_end < addr or NULL
* if there are no any areas before @addr.
*/
static struct vmap_area *
pvm_find_va_enclose_addr(unsigned long addr)
{
struct vmap_area *va, *tmp;
struct rb_node *n;
n = free_vmap_area_root.rb_node;
va = NULL;
while (n) {
tmp = rb_entry(n, struct vmap_area, rb_node);
if (tmp->va_start <= addr) {
va = tmp;
if (tmp->va_end >= addr)
break;
n = n->rb_right;
} else {
n = n->rb_left;
}
}
return va;
}
/**
* pvm_determine_end_from_reverse - find the highest aligned address
* of free block below VMALLOC_END
* @va:
* in - the VA we start the search(reverse order);
* out - the VA with the highest aligned end address.
* @align: alignment for required highest address
*
* Returns: determined end address within vmap_area
*/
static unsigned long
pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
{
unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
unsigned long addr;
if (likely(*va)) {
list_for_each_entry_from_reverse((*va),
&free_vmap_area_list, list) {
addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
if ((*va)->va_start < addr)
return addr;
}
}
return 0;
}
/**
* pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
* @offsets: array containing offset of each area
* @sizes: array containing size of each area
* @nr_vms: the number of areas to allocate
* @align: alignment, all entries in @offsets and @sizes must be aligned to this
*
* Returns: kmalloc'd vm_struct pointer array pointing to allocated
* vm_structs on success, %NULL on failure
*
* Percpu allocator wants to use congruent vm areas so that it can
* maintain the offsets among percpu areas. This function allocates
* congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
* be scattered pretty far, distance between two areas easily going up
* to gigabytes. To avoid interacting with regular vmallocs, these
* areas are allocated from top.
*
* Despite its complicated look, this allocator is rather simple. It
* does everything top-down and scans free blocks from the end looking
* for matching base. While scanning, if any of the areas do not fit the
* base address is pulled down to fit the area. Scanning is repeated till
* all the areas fit and then all necessary data structures are inserted
* and the result is returned.
*/
struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
const size_t *sizes, int nr_vms,
size_t align)
{
const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
struct vmap_area **vas, *va;
struct vm_struct **vms;
int area, area2, last_area, term_area;
unsigned long base, start, size, end, last_end, orig_start, orig_end;
bool purged = false;
/* verify parameters and allocate data structures */
BUG_ON(offset_in_page(align) || !is_power_of_2(align));
for (last_area = 0, area = 0; area < nr_vms; area++) {
start = offsets[area];
end = start + sizes[area];
/* is everything aligned properly? */
BUG_ON(!IS_ALIGNED(offsets[area], align));
BUG_ON(!IS_ALIGNED(sizes[area], align));
/* detect the area with the highest address */
if (start > offsets[last_area])
last_area = area;
for (area2 = area + 1; area2 < nr_vms; area2++) {
unsigned long start2 = offsets[area2];
unsigned long end2 = start2 + sizes[area2];
BUG_ON(start2 < end && start < end2);
}
}
last_end = offsets[last_area] + sizes[last_area];
if (vmalloc_end - vmalloc_start < last_end) {
WARN_ON(true);
return NULL;
}
vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
if (!vas || !vms)
goto err_free2;
for (area = 0; area < nr_vms; area++) {
vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
if (!vas[area] || !vms[area])
goto err_free;
}
retry:
spin_lock(&free_vmap_area_lock);
/* start scanning - we scan from the top, begin with the last area */
area = term_area = last_area;
start = offsets[area];
end = start + sizes[area];
va = pvm_find_va_enclose_addr(vmalloc_end);
base = pvm_determine_end_from_reverse(&va, align) - end;
while (true) {
/*
* base might have underflowed, add last_end before
* comparing.
*/
if (base + last_end < vmalloc_start + last_end)
goto overflow;
/*
* Fitting base has not been found.
*/
if (va == NULL)
goto overflow;
/*
* If required width exceeds current VA block, move
* base downwards and then recheck.
*/
if (base + end > va->va_end) {
base = pvm_determine_end_from_reverse(&va, align) - end;
term_area = area;
continue;
}
/*
* If this VA does not fit, move base downwards and recheck.
*/
if (base + start < va->va_start) {
va = node_to_va(rb_prev(&va->rb_node));
base = pvm_determine_end_from_reverse(&va, align) - end;
term_area = area;
continue;
}
/*
* This area fits, move on to the previous one. If
* the previous one is the terminal one, we're done.
*/
area = (area + nr_vms - 1) % nr_vms;
if (area == term_area)
break;
start = offsets[area];
end = start + sizes[area];
va = pvm_find_va_enclose_addr(base + end);
}
/* we've found a fitting base, insert all va's */
for (area = 0; area < nr_vms; area++) {
int ret;
start = base + offsets[area];
size = sizes[area];
va = pvm_find_va_enclose_addr(start);
if (WARN_ON_ONCE(va == NULL))
/* It is a BUG(), but trigger recovery instead. */
goto recovery;
ret = va_clip(&free_vmap_area_root,
&free_vmap_area_list, va, start, size);
if (WARN_ON_ONCE(unlikely(ret)))
/* It is a BUG(), but trigger recovery instead. */
goto recovery;
/* Allocated area. */
va = vas[area];
va->va_start = start;
va->va_end = start + size;
}
spin_unlock(&free_vmap_area_lock);
/* populate the kasan shadow space */
for (area = 0; area < nr_vms; area++) {
if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))
goto err_free_shadow;
}
/* insert all vm's */
for (area = 0; area < nr_vms; area++) {
struct vmap_node *vn = addr_to_node(vas[area]->va_start);
spin_lock(&vn->busy.lock);
insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head);
setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,
pcpu_get_vm_areas);
spin_unlock(&vn->busy.lock);
}
/*
* Mark allocated areas as accessible. Do it now as a best-effort
* approach, as they can be mapped outside of vmalloc code.
* With hardware tag-based KASAN, marking is skipped for
* non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
*/
for (area = 0; area < nr_vms; area++)
vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr,
vms[area]->size, KASAN_VMALLOC_PROT_NORMAL);
kfree(vas);
return vms;
recovery:
/*
* Remove previously allocated areas. There is no
* need in removing these areas from the busy tree,
* because they are inserted only on the final step
* and when pcpu_get_vm_areas() is success.
*/
while (area--) {
orig_start = vas[area]->va_start;
orig_end = vas[area]->va_end;
va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,
&free_vmap_area_list);
if (va)
kasan_release_vmalloc(orig_start, orig_end,
va->va_start, va->va_end);
vas[area] = NULL;
}
overflow:
spin_unlock(&free_vmap_area_lock);
if (!purged) {
reclaim_and_purge_vmap_areas();
purged = true;
/* Before "retry", check if we recover. */
for (area = 0; area < nr_vms; area++) {
if (vas[area])
continue;
vas[area] = kmem_cache_zalloc(
vmap_area_cachep, GFP_KERNEL);
if (!vas[area])
goto err_free;
}
goto retry;
}
err_free:
for (area = 0; area < nr_vms; area++) {
if (vas[area])
kmem_cache_free(vmap_area_cachep, vas[area]);
kfree(vms[area]);
}
err_free2:
kfree(vas);
kfree(vms);
return NULL;
err_free_shadow:
spin_lock(&free_vmap_area_lock);
/*
* We release all the vmalloc shadows, even the ones for regions that
* hadn't been successfully added. This relies on kasan_release_vmalloc
* being able to tolerate this case.
*/
for (area = 0; area < nr_vms; area++) {
orig_start = vas[area]->va_start;
orig_end = vas[area]->va_end;
va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,
&free_vmap_area_list);
if (va)
kasan_release_vmalloc(orig_start, orig_end,
va->va_start, va->va_end);
vas[area] = NULL;
kfree(vms[area]);
}
spin_unlock(&free_vmap_area_lock);
kfree(vas);
kfree(vms);
return NULL;
}
/**
* pcpu_free_vm_areas - free vmalloc areas for percpu allocator
* @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
* @nr_vms: the number of allocated areas
*
* Free vm_structs and the array allocated by pcpu_get_vm_areas().
*/
void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
{
int i;
for (i = 0; i < nr_vms; i++)
free_vm_area(vms[i]);
kfree(vms);
}
#endif /* CONFIG_SMP */
#ifdef CONFIG_PRINTK
bool vmalloc_dump_obj(void *object)
{
const void *caller;
struct vm_struct *vm;
struct vmap_area *va;
struct vmap_node *vn;
unsigned long addr;
unsigned int nr_pages;
addr = PAGE_ALIGN((unsigned long) object);
vn = addr_to_node(addr);
if (!spin_trylock(&vn->busy.lock))
return false;
va = __find_vmap_area(addr, &vn->busy.root);
if (!va || !va->vm) {
spin_unlock(&vn->busy.lock);
return false;
}
vm = va->vm;
addr = (unsigned long) vm->addr;
caller = vm->caller;
nr_pages = vm->nr_pages;
spin_unlock(&vn->busy.lock);
pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",
nr_pages, addr, caller);
return true;
}
#endif
#ifdef CONFIG_PROC_FS
static void show_numa_info(struct seq_file *m, struct vm_struct *v)
{
if (IS_ENABLED(CONFIG_NUMA)) {
unsigned int nr, *counters = m->private;
unsigned int step = 1U << vm_area_page_order(v);
if (!counters)
return;
if (v->flags & VM_UNINITIALIZED)
return;
/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
smp_rmb();
memset(counters, 0, nr_node_ids * sizeof(unsigned int));
for (nr = 0; nr < v->nr_pages; nr += step)
counters[page_to_nid(v->pages[nr])] += step;
for_each_node_state(nr, N_HIGH_MEMORY)
if (counters[nr])
seq_printf(m, " N%u=%u", nr, counters[nr]);
}
}
static void show_purge_info(struct seq_file *m)
{
struct vmap_node *vn;
struct vmap_area *va;
int i;
for (i = 0; i < nr_vmap_nodes; i++) {
vn = &vmap_nodes[i];
spin_lock(&vn->lazy.lock);
list_for_each_entry(va, &vn->lazy.head, list) {
seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n",
(void *)va->va_start, (void *)va->va_end,
va->va_end - va->va_start);
}
spin_unlock(&vn->lazy.lock);
}
}
static int vmalloc_info_show(struct seq_file *m, void *p)
{
struct vmap_node *vn;
struct vmap_area *va;
struct vm_struct *v;
int i;
for (i = 0; i < nr_vmap_nodes; i++) {
vn = &vmap_nodes[i];
spin_lock(&vn->busy.lock);
list_for_each_entry(va, &vn->busy.head, list) {
if (!va->vm) {
if (va->flags & VMAP_RAM)
seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
(void *)va->va_start, (void *)va->va_end,
va->va_end - va->va_start);
continue;
}
v = va->vm;
seq_printf(m, "0x%pK-0x%pK %7ld",
v->addr, v->addr + v->size, v->size);
if (v->caller)
seq_printf(m, " %pS", v->caller);
if (v->nr_pages)
seq_printf(m, " pages=%d", v->nr_pages);
if (v->phys_addr)
seq_printf(m, " phys=%pa", &v->phys_addr);
if (v->flags & VM_IOREMAP)
seq_puts(m, " ioremap");
if (v->flags & VM_SPARSE)
seq_puts(m, " sparse");
if (v->flags & VM_ALLOC)
seq_puts(m, " vmalloc");
if (v->flags & VM_MAP)
seq_puts(m, " vmap");
if (v->flags & VM_USERMAP)
seq_puts(m, " user");
if (v->flags & VM_DMA_COHERENT)
seq_puts(m, " dma-coherent");
if (is_vmalloc_addr(v->pages))
seq_puts(m, " vpages");
show_numa_info(m, v);
seq_putc(m, '\n');
}
spin_unlock(&vn->busy.lock);
}
/*
* As a final step, dump "unpurged" areas.
*/
show_purge_info(m);
return 0;
}
static int __init proc_vmalloc_init(void)
{
void *priv_data = NULL;
if (IS_ENABLED(CONFIG_NUMA))
priv_data = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL);
proc_create_single_data("vmallocinfo",
0400, NULL, vmalloc_info_show, priv_data);
return 0;
}
module_init(proc_vmalloc_init);
#endif
static void __init vmap_init_free_space(void)
{
unsigned long vmap_start = 1;
const unsigned long vmap_end = ULONG_MAX;
struct vmap_area *free;
struct vm_struct *busy;
/*
* B F B B B F
* -|-----|.....|-----|-----|-----|.....|-
* | The KVA space |
* |<--------------------------------->|
*/
for (busy = vmlist; busy; busy = busy->next) {
if ((unsigned long) busy->addr - vmap_start > 0) {
free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
if (!WARN_ON_ONCE(!free)) {
free->va_start = vmap_start;
free->va_end = (unsigned long) busy->addr;
insert_vmap_area_augment(free, NULL,
&free_vmap_area_root,
&free_vmap_area_list);
}
}
vmap_start = (unsigned long) busy->addr + busy->size;
}
if (vmap_end - vmap_start > 0) {
free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
if (!WARN_ON_ONCE(!free)) {
free->va_start = vmap_start;
free->va_end = vmap_end;
insert_vmap_area_augment(free, NULL,
&free_vmap_area_root,
&free_vmap_area_list);
}
}
}
static void vmap_init_nodes(void)
{
struct vmap_node *vn;
int i, n;
#if BITS_PER_LONG == 64
/*
* A high threshold of max nodes is fixed and bound to 128,
* thus a scale factor is 1 for systems where number of cores
* are less or equal to specified threshold.
*
* As for NUMA-aware notes. For bigger systems, for example
* NUMA with multi-sockets, where we can end-up with thousands
* of cores in total, a "sub-numa-clustering" should be added.
*
* In this case a NUMA domain is considered as a single entity
* with dedicated sub-nodes in it which describe one group or
* set of cores. Therefore a per-domain purging is supposed to
* be added as well as a per-domain balancing.
*/
n = clamp_t(unsigned int, num_possible_cpus(), 1, 128);
if (n > 1) {
vn = kmalloc_array(n, sizeof(*vn), GFP_NOWAIT | __GFP_NOWARN);
if (vn) {
/* Node partition is 16 pages. */
vmap_zone_size = (1 << 4) * PAGE_SIZE;
nr_vmap_nodes = n;
vmap_nodes = vn;
} else {
pr_err("Failed to allocate an array. Disable a node layer\n");
}
}
#endif
for (n = 0; n < nr_vmap_nodes; n++) {
vn = &vmap_nodes[n];
vn->busy.root = RB_ROOT;
INIT_LIST_HEAD(&vn->busy.head);
spin_lock_init(&vn->busy.lock);
vn->lazy.root = RB_ROOT;
INIT_LIST_HEAD(&vn->lazy.head);
spin_lock_init(&vn->lazy.lock);
for (i = 0; i < MAX_VA_SIZE_PAGES; i++) {
INIT_LIST_HEAD(&vn->pool[i].head);
WRITE_ONCE(vn->pool[i].len, 0);
}
spin_lock_init(&vn->pool_lock);
}
}
static unsigned long
vmap_node_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
unsigned long count;
struct vmap_node *vn;
int i, j;
for (count = 0, i = 0; i < nr_vmap_nodes; i++) {
vn = &vmap_nodes[i];
for (j = 0; j < MAX_VA_SIZE_PAGES; j++)
count += READ_ONCE(vn->pool[j].len);
}
return count ? count : SHRINK_EMPTY;
}
static unsigned long
vmap_node_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
{
int i;
for (i = 0; i < nr_vmap_nodes; i++)
decay_va_pool_node(&vmap_nodes[i], true);
return SHRINK_STOP;
}
void __init vmalloc_init(void)
{
struct shrinker *vmap_node_shrinker;
struct vmap_area *va;
struct vmap_node *vn;
struct vm_struct *tmp;
int i;
/*
* Create the cache for vmap_area objects.
*/
vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
for_each_possible_cpu(i) {
struct vmap_block_queue *vbq;
struct vfree_deferred *p;
vbq = &per_cpu(vmap_block_queue, i);
spin_lock_init(&vbq->lock);
INIT_LIST_HEAD(&vbq->free);
p = &per_cpu(vfree_deferred, i);
init_llist_head(&p->list);
INIT_WORK(&p->wq, delayed_vfree_work);
xa_init(&vbq->vmap_blocks);
}
/*
* Setup nodes before importing vmlist.
*/
vmap_init_nodes();
/* Import existing vmlist entries. */
for (tmp = vmlist; tmp; tmp = tmp->next) {
va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
if (WARN_ON_ONCE(!va))
continue;
va->va_start = (unsigned long)tmp->addr;
va->va_end = va->va_start + tmp->size;
va->vm = tmp;
vn = addr_to_node(va->va_start);
insert_vmap_area(va, &vn->busy.root, &vn->busy.head);
}
/*
* Now we can initialize a free vmap space.
*/
vmap_init_free_space();
vmap_initialized = true;
vmap_node_shrinker = shrinker_alloc(0, "vmap-node");
if (!vmap_node_shrinker) {
pr_err("Failed to allocate vmap-node shrinker!\n");
return;
}
vmap_node_shrinker->count_objects = vmap_node_shrink_count;
vmap_node_shrinker->scan_objects = vmap_node_shrink_scan;
shrinker_register(vmap_node_shrinker);
}
/**
* css_get - obtain a reference on the specified css
* @css: target css
*
* The caller must already have a reference.
*/
CGROUP_REF_FN_ATTRS
void css_get(struct cgroup_subsys_state *css)
{
if (!(css->flags & CSS_NO_REF))
percpu_ref_get(&css->refcnt);
}
CGROUP_REF_EXPORT(css_get)
/**
* css_get_many - obtain references on the specified css
* @css: target css
* @n: number of references to get
*
* The caller must already have a reference.
*/
CGROUP_REF_FN_ATTRS
void css_get_many(struct cgroup_subsys_state *css, unsigned int n)
{
if (!(css->flags & CSS_NO_REF))
percpu_ref_get_many(&css->refcnt, n);
}
CGROUP_REF_EXPORT(css_get_many)
/**
* css_tryget - try to obtain a reference on the specified css
* @css: target css
*
* Obtain a reference on @css unless it already has reached zero and is
* being released. This function doesn't care whether @css is on or
* offline. The caller naturally needs to ensure that @css is accessible
* but doesn't have to be holding a reference on it - IOW, RCU protected
* access is good enough for this function. Returns %true if a reference
* count was successfully obtained; %false otherwise.
*/
CGROUP_REF_FN_ATTRS
bool css_tryget(struct cgroup_subsys_state *css)
{
if (!(css->flags & CSS_NO_REF))
return percpu_ref_tryget(&css->refcnt);
return true;
}
CGROUP_REF_EXPORT(css_tryget)
/**
* css_tryget_online - try to obtain a reference on the specified css if online
* @css: target css
*
* Obtain a reference on @css if it's online. The caller naturally needs
* to ensure that @css is accessible but doesn't have to be holding a
* reference on it - IOW, RCU protected access is good enough for this
* function. Returns %true if a reference count was successfully obtained;
* %false otherwise.
*/
CGROUP_REF_FN_ATTRS
bool css_tryget_online(struct cgroup_subsys_state *css)
{
if (!(css->flags & CSS_NO_REF))
return percpu_ref_tryget_live(&css->refcnt);
return true;
}
CGROUP_REF_EXPORT(css_tryget_online)
/**
* css_put - put a css reference
* @css: target css
*
* Put a reference obtained via css_get() and css_tryget_online().
*/
CGROUP_REF_FN_ATTRS
void css_put(struct cgroup_subsys_state *css)
{
if (!(css->flags & CSS_NO_REF))
percpu_ref_put(&css->refcnt);
}
CGROUP_REF_EXPORT(css_put)
/**
* css_put_many - put css references
* @css: target css
* @n: number of references to put
*
* Put references obtained via css_get() and css_tryget_online().
*/
CGROUP_REF_FN_ATTRS
void css_put_many(struct cgroup_subsys_state *css, unsigned int n)
{
if (!(css->flags & CSS_NO_REF))
percpu_ref_put_many(&css->refcnt, n);
}
CGROUP_REF_EXPORT(css_put_many)
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __NET_CFG80211_H
#define __NET_CFG80211_H
/*
* 802.11 device and configuration interface
*
* Copyright 2006-2010 Johannes Berg <johannes@sipsolutions.net>
* Copyright 2013-2014 Intel Mobile Communications GmbH
* Copyright 2015-2017 Intel Deutschland GmbH
* Copyright (C) 2018-2024 Intel Corporation
*/
#include <linux/ethtool.h>
#include <uapi/linux/rfkill.h>
#include <linux/netdevice.h>
#include <linux/debugfs.h>
#include <linux/list.h>
#include <linux/bug.h>
#include <linux/netlink.h>
#include <linux/skbuff.h>
#include <linux/nl80211.h>
#include <linux/if_ether.h>
#include <linux/ieee80211.h>
#include <linux/net.h>
#include <linux/rfkill.h>
#include <net/regulatory.h>
/**
* DOC: Introduction
*
* cfg80211 is the configuration API for 802.11 devices in Linux. It bridges
* userspace and drivers, and offers some utility functionality associated
* with 802.11. cfg80211 must, directly or indirectly via mac80211, be used
* by all modern wireless drivers in Linux, so that they offer a consistent
* API through nl80211. For backward compatibility, cfg80211 also offers
* wireless extensions to userspace, but hides them from drivers completely.
*
* Additionally, cfg80211 contains code to help enforce regulatory spectrum
* use restrictions.
*/
/**
* DOC: Device registration
*
* In order for a driver to use cfg80211, it must register the hardware device
* with cfg80211. This happens through a number of hardware capability structs
* described below.
*
* The fundamental structure for each device is the 'wiphy', of which each
* instance describes a physical wireless device connected to the system. Each
* such wiphy can have zero, one, or many virtual interfaces associated with
* it, which need to be identified as such by pointing the network interface's
* @ieee80211_ptr pointer to a &struct wireless_dev which further describes
* the wireless part of the interface. Normally this struct is embedded in the
* network interface's private data area. Drivers can optionally allow creating
* or destroying virtual interfaces on the fly, but without at least one or the
* ability to create some the wireless device isn't useful.
*
* Each wiphy structure contains device capability information, and also has
* a pointer to the various operations the driver offers. The definitions and
* structures here describe these capabilities in detail.
*/
struct wiphy;
/*
* wireless hardware capability structures
*/
/**
* enum ieee80211_channel_flags - channel flags
*
* Channel flags set by the regulatory control code.
*
* @IEEE80211_CHAN_DISABLED: This channel is disabled.
* @IEEE80211_CHAN_NO_IR: do not initiate radiation, this includes
* sending probe requests or beaconing.
* @IEEE80211_CHAN_PSD: Power spectral density (in dBm) is set for this
* channel.
* @IEEE80211_CHAN_RADAR: Radar detection is required on this channel.
* @IEEE80211_CHAN_NO_HT40PLUS: extension channel above this channel
* is not permitted.
* @IEEE80211_CHAN_NO_HT40MINUS: extension channel below this channel
* is not permitted.
* @IEEE80211_CHAN_NO_OFDM: OFDM is not allowed on this channel.
* @IEEE80211_CHAN_NO_80MHZ: If the driver supports 80 MHz on the band,
* this flag indicates that an 80 MHz channel cannot use this
* channel as the control or any of the secondary channels.
* This may be due to the driver or due to regulatory bandwidth
* restrictions.
* @IEEE80211_CHAN_NO_160MHZ: If the driver supports 160 MHz on the band,
* this flag indicates that an 160 MHz channel cannot use this
* channel as the control or any of the secondary channels.
* This may be due to the driver or due to regulatory bandwidth
* restrictions.
* @IEEE80211_CHAN_INDOOR_ONLY: see %NL80211_FREQUENCY_ATTR_INDOOR_ONLY
* @IEEE80211_CHAN_IR_CONCURRENT: see %NL80211_FREQUENCY_ATTR_IR_CONCURRENT
* @IEEE80211_CHAN_NO_20MHZ: 20 MHz bandwidth is not permitted
* on this channel.
* @IEEE80211_CHAN_NO_10MHZ: 10 MHz bandwidth is not permitted
* on this channel.
* @IEEE80211_CHAN_NO_HE: HE operation is not permitted on this channel.
* @IEEE80211_CHAN_1MHZ: 1 MHz bandwidth is permitted
* on this channel.
* @IEEE80211_CHAN_2MHZ: 2 MHz bandwidth is permitted
* on this channel.
* @IEEE80211_CHAN_4MHZ: 4 MHz bandwidth is permitted
* on this channel.
* @IEEE80211_CHAN_8MHZ: 8 MHz bandwidth is permitted
* on this channel.
* @IEEE80211_CHAN_16MHZ: 16 MHz bandwidth is permitted
* on this channel.
* @IEEE80211_CHAN_NO_320MHZ: If the driver supports 320 MHz on the band,
* this flag indicates that a 320 MHz channel cannot use this
* channel as the control or any of the secondary channels.
* This may be due to the driver or due to regulatory bandwidth
* restrictions.
* @IEEE80211_CHAN_NO_EHT: EHT operation is not permitted on this channel.
* @IEEE80211_CHAN_DFS_CONCURRENT: See %NL80211_RRF_DFS_CONCURRENT
* @IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT: Client connection with VLP AP
* not permitted using this channel
* @IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT: Client connection with AFC AP
* not permitted using this channel
* @IEEE80211_CHAN_CAN_MONITOR: This channel can be used for monitor
* mode even in the presence of other (regulatory) restrictions,
* even if it is otherwise disabled.
*/
enum ieee80211_channel_flags {
IEEE80211_CHAN_DISABLED = 1<<0,
IEEE80211_CHAN_NO_IR = 1<<1,
IEEE80211_CHAN_PSD = 1<<2,
IEEE80211_CHAN_RADAR = 1<<3,
IEEE80211_CHAN_NO_HT40PLUS = 1<<4,
IEEE80211_CHAN_NO_HT40MINUS = 1<<5,
IEEE80211_CHAN_NO_OFDM = 1<<6,
IEEE80211_CHAN_NO_80MHZ = 1<<7,
IEEE80211_CHAN_NO_160MHZ = 1<<8,
IEEE80211_CHAN_INDOOR_ONLY = 1<<9,
IEEE80211_CHAN_IR_CONCURRENT = 1<<10,
IEEE80211_CHAN_NO_20MHZ = 1<<11,
IEEE80211_CHAN_NO_10MHZ = 1<<12,
IEEE80211_CHAN_NO_HE = 1<<13,
IEEE80211_CHAN_1MHZ = 1<<14,
IEEE80211_CHAN_2MHZ = 1<<15,
IEEE80211_CHAN_4MHZ = 1<<16,
IEEE80211_CHAN_8MHZ = 1<<17,
IEEE80211_CHAN_16MHZ = 1<<18,
IEEE80211_CHAN_NO_320MHZ = 1<<19,
IEEE80211_CHAN_NO_EHT = 1<<20,
IEEE80211_CHAN_DFS_CONCURRENT = 1<<21,
IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT = 1<<22,
IEEE80211_CHAN_NO_6GHZ_AFC_CLIENT = 1<<23,
IEEE80211_CHAN_CAN_MONITOR = 1<<24,
};
#define IEEE80211_CHAN_NO_HT40 \
(IEEE80211_CHAN_NO_HT40PLUS | IEEE80211_CHAN_NO_HT40MINUS)
#define IEEE80211_DFS_MIN_CAC_TIME_MS 60000
#define IEEE80211_DFS_MIN_NOP_TIME_MS (30 * 60 * 1000)
/**
* struct ieee80211_channel - channel definition
*
* This structure describes a single channel for use
* with cfg80211.
*
* @center_freq: center frequency in MHz
* @freq_offset: offset from @center_freq, in KHz
* @hw_value: hardware-specific value for the channel
* @flags: channel flags from &enum ieee80211_channel_flags.
* @orig_flags: channel flags at registration time, used by regulatory
* code to support devices with additional restrictions
* @band: band this channel belongs to.
* @max_antenna_gain: maximum antenna gain in dBi
* @max_power: maximum transmission power (in dBm)
* @max_reg_power: maximum regulatory transmission power (in dBm)
* @beacon_found: helper to regulatory code to indicate when a beacon
* has been found on this channel. Use regulatory_hint_found_beacon()
* to enable this, this is useful only on 5 GHz band.
* @orig_mag: internal use
* @orig_mpwr: internal use
* @dfs_state: current state of this channel. Only relevant if radar is required
* on this channel.
* @dfs_state_entered: timestamp (jiffies) when the dfs state was entered.
* @dfs_cac_ms: DFS CAC time in milliseconds, this is valid for DFS channels.
* @psd: power spectral density (in dBm)
*/
struct ieee80211_channel {
enum nl80211_band band;
u32 center_freq;
u16 freq_offset;
u16 hw_value;
u32 flags;
int max_antenna_gain;
int max_power;
int max_reg_power;
bool beacon_found;
u32 orig_flags;
int orig_mag, orig_mpwr;
enum nl80211_dfs_state dfs_state;
unsigned long dfs_state_entered;
unsigned int dfs_cac_ms;
s8 psd;
};
/**
* enum ieee80211_rate_flags - rate flags
*
* Hardware/specification flags for rates. These are structured
* in a way that allows using the same bitrate structure for
* different bands/PHY modes.
*
* @IEEE80211_RATE_SHORT_PREAMBLE: Hardware can send with short
* preamble on this bitrate; only relevant in 2.4GHz band and
* with CCK rates.
* @IEEE80211_RATE_MANDATORY_A: This bitrate is a mandatory rate
* when used with 802.11a (on the 5 GHz band); filled by the
* core code when registering the wiphy.
* @IEEE80211_RATE_MANDATORY_B: This bitrate is a mandatory rate
* when used with 802.11b (on the 2.4 GHz band); filled by the
* core code when registering the wiphy.
* @IEEE80211_RATE_MANDATORY_G: This bitrate is a mandatory rate
* when used with 802.11g (on the 2.4 GHz band); filled by the
* core code when registering the wiphy.
* @IEEE80211_RATE_ERP_G: This is an ERP rate in 802.11g mode.
* @IEEE80211_RATE_SUPPORTS_5MHZ: Rate can be used in 5 MHz mode
* @IEEE80211_RATE_SUPPORTS_10MHZ: Rate can be used in 10 MHz mode
*/
enum ieee80211_rate_flags {
IEEE80211_RATE_SHORT_PREAMBLE = 1<<0,
IEEE80211_RATE_MANDATORY_A = 1<<1,
IEEE80211_RATE_MANDATORY_B = 1<<2,
IEEE80211_RATE_MANDATORY_G = 1<<3,
IEEE80211_RATE_ERP_G = 1<<4,
IEEE80211_RATE_SUPPORTS_5MHZ = 1<<5,
IEEE80211_RATE_SUPPORTS_10MHZ = 1<<6,
};
/**
* enum ieee80211_bss_type - BSS type filter
*
* @IEEE80211_BSS_TYPE_ESS: Infrastructure BSS
* @IEEE80211_BSS_TYPE_PBSS: Personal BSS
* @IEEE80211_BSS_TYPE_IBSS: Independent BSS
* @IEEE80211_BSS_TYPE_MBSS: Mesh BSS
* @IEEE80211_BSS_TYPE_ANY: Wildcard value for matching any BSS type
*/
enum ieee80211_bss_type {
IEEE80211_BSS_TYPE_ESS,
IEEE80211_BSS_TYPE_PBSS,
IEEE80211_BSS_TYPE_IBSS,
IEEE80211_BSS_TYPE_MBSS,
IEEE80211_BSS_TYPE_ANY
};
/**
* enum ieee80211_privacy - BSS privacy filter
*
* @IEEE80211_PRIVACY_ON: privacy bit set
* @IEEE80211_PRIVACY_OFF: privacy bit clear
* @IEEE80211_PRIVACY_ANY: Wildcard value for matching any privacy setting
*/
enum ieee80211_privacy {
IEEE80211_PRIVACY_ON,
IEEE80211_PRIVACY_OFF,
IEEE80211_PRIVACY_ANY
};
#define IEEE80211_PRIVACY(x) \
((x) ? IEEE80211_PRIVACY_ON : IEEE80211_PRIVACY_OFF)
/**
* struct ieee80211_rate - bitrate definition
*
* This structure describes a bitrate that an 802.11 PHY can
* operate with. The two values @hw_value and @hw_value_short
* are only for driver use when pointers to this structure are
* passed around.
*
* @flags: rate-specific flags from &enum ieee80211_rate_flags
* @bitrate: bitrate in units of 100 Kbps
* @hw_value: driver/hardware value for this rate
* @hw_value_short: driver/hardware value for this rate when
* short preamble is used
*/
struct ieee80211_rate {
u32 flags;
u16 bitrate;
u16 hw_value, hw_value_short;
};
/**
* struct ieee80211_he_obss_pd - AP settings for spatial reuse
*
* @enable: is the feature enabled.
* @sr_ctrl: The SR Control field of SRP element.
* @non_srg_max_offset: non-SRG maximum tx power offset
* @min_offset: minimal tx power offset an associated station shall use
* @max_offset: maximum tx power offset an associated station shall use
* @bss_color_bitmap: bitmap that indicates the BSS color values used by
* members of the SRG
* @partial_bssid_bitmap: bitmap that indicates the partial BSSID values
* used by members of the SRG
*/
struct ieee80211_he_obss_pd {
bool enable;
u8 sr_ctrl;
u8 non_srg_max_offset;
u8 min_offset;
u8 max_offset;
u8 bss_color_bitmap[8];
u8 partial_bssid_bitmap[8];
};
/**
* struct cfg80211_he_bss_color - AP settings for BSS coloring
*
* @color: the current color.
* @enabled: HE BSS color is used
* @partial: define the AID equation.
*/
struct cfg80211_he_bss_color {
u8 color;
bool enabled;
bool partial;
};
/**
* struct ieee80211_sta_ht_cap - STA's HT capabilities
*
* This structure describes most essential parameters needed
* to describe 802.11n HT capabilities for an STA.
*
* @ht_supported: is HT supported by the STA
* @cap: HT capabilities map as described in 802.11n spec
* @ampdu_factor: Maximum A-MPDU length factor
* @ampdu_density: Minimum A-MPDU spacing
* @mcs: Supported MCS rates
*/
struct ieee80211_sta_ht_cap {
u16 cap; /* use IEEE80211_HT_CAP_ */
bool ht_supported;
u8 ampdu_factor;
u8 ampdu_density;
struct ieee80211_mcs_info mcs;
};
/**
* struct ieee80211_sta_vht_cap - STA's VHT capabilities
*
* This structure describes most essential parameters needed
* to describe 802.11ac VHT capabilities for an STA.
*
* @vht_supported: is VHT supported by the STA
* @cap: VHT capabilities map as described in 802.11ac spec
* @vht_mcs: Supported VHT MCS rates
*/
struct ieee80211_sta_vht_cap {
bool vht_supported;
u32 cap; /* use IEEE80211_VHT_CAP_ */
struct ieee80211_vht_mcs_info vht_mcs;
};
#define IEEE80211_HE_PPE_THRES_MAX_LEN 25
/**
* struct ieee80211_sta_he_cap - STA's HE capabilities
*
* This structure describes most essential parameters needed
* to describe 802.11ax HE capabilities for a STA.
*
* @has_he: true iff HE data is valid.
* @he_cap_elem: Fixed portion of the HE capabilities element.
* @he_mcs_nss_supp: The supported NSS/MCS combinations.
* @ppe_thres: Holds the PPE Thresholds data.
*/
struct ieee80211_sta_he_cap {
bool has_he;
struct ieee80211_he_cap_elem he_cap_elem;
struct ieee80211_he_mcs_nss_supp he_mcs_nss_supp;
u8 ppe_thres[IEEE80211_HE_PPE_THRES_MAX_LEN];
};
/**
* struct ieee80211_eht_mcs_nss_supp - EHT max supported NSS per MCS
*
* See P802.11be_D1.3 Table 9-401k - "Subfields of the Supported EHT-MCS
* and NSS Set field"
*
* @only_20mhz: MCS/NSS support for 20 MHz-only STA.
* @bw: MCS/NSS support for 80, 160 and 320 MHz
* @bw._80: MCS/NSS support for BW <= 80 MHz
* @bw._160: MCS/NSS support for BW = 160 MHz
* @bw._320: MCS/NSS support for BW = 320 MHz
*/
struct ieee80211_eht_mcs_nss_supp {
union {
struct ieee80211_eht_mcs_nss_supp_20mhz_only only_20mhz;
struct {
struct ieee80211_eht_mcs_nss_supp_bw _80;
struct ieee80211_eht_mcs_nss_supp_bw _160;
struct ieee80211_eht_mcs_nss_supp_bw _320;
} __packed bw;
} __packed;
} __packed;
#define IEEE80211_EHT_PPE_THRES_MAX_LEN 32
/**
* struct ieee80211_sta_eht_cap - STA's EHT capabilities
*
* This structure describes most essential parameters needed
* to describe 802.11be EHT capabilities for a STA.
*
* @has_eht: true iff EHT data is valid.
* @eht_cap_elem: Fixed portion of the eht capabilities element.
* @eht_mcs_nss_supp: The supported NSS/MCS combinations.
* @eht_ppe_thres: Holds the PPE Thresholds data.
*/
struct ieee80211_sta_eht_cap {
bool has_eht;
struct ieee80211_eht_cap_elem_fixed eht_cap_elem;
struct ieee80211_eht_mcs_nss_supp eht_mcs_nss_supp;
u8 eht_ppe_thres[IEEE80211_EHT_PPE_THRES_MAX_LEN];
};
/* sparse defines __CHECKER__; see Documentation/dev-tools/sparse.rst */
#ifdef __CHECKER__
/*
* This is used to mark the sband->iftype_data pointer which is supposed
* to be an array with special access semantics (per iftype), but a lot
* of code got it wrong in the past, so with this marking sparse will be
* noisy when the pointer is used directly.
*/
# define __iftd __attribute__((noderef, address_space(__iftype_data)))
#else
# define __iftd
#endif /* __CHECKER__ */
/**
* struct ieee80211_sband_iftype_data - sband data per interface type
*
* This structure encapsulates sband data that is relevant for the
* interface types defined in @types_mask. Each type in the
* @types_mask must be unique across all instances of iftype_data.
*
* @types_mask: interface types mask
* @he_cap: holds the HE capabilities
* @he_6ghz_capa: HE 6 GHz capabilities, must be filled in for a
* 6 GHz band channel (and 0 may be valid value).
* @eht_cap: STA's EHT capabilities
* @vendor_elems: vendor element(s) to advertise
* @vendor_elems.data: vendor element(s) data
* @vendor_elems.len: vendor element(s) length
*/
struct ieee80211_sband_iftype_data {
u16 types_mask;
struct ieee80211_sta_he_cap he_cap;
struct ieee80211_he_6ghz_capa he_6ghz_capa;
struct ieee80211_sta_eht_cap eht_cap;
struct {
const u8 *data;
unsigned int len;
} vendor_elems;
};
/**
* enum ieee80211_edmg_bw_config - allowed channel bandwidth configurations
*
* @IEEE80211_EDMG_BW_CONFIG_4: 2.16GHz
* @IEEE80211_EDMG_BW_CONFIG_5: 2.16GHz and 4.32GHz
* @IEEE80211_EDMG_BW_CONFIG_6: 2.16GHz, 4.32GHz and 6.48GHz
* @IEEE80211_EDMG_BW_CONFIG_7: 2.16GHz, 4.32GHz, 6.48GHz and 8.64GHz
* @IEEE80211_EDMG_BW_CONFIG_8: 2.16GHz and 2.16GHz + 2.16GHz
* @IEEE80211_EDMG_BW_CONFIG_9: 2.16GHz, 4.32GHz and 2.16GHz + 2.16GHz
* @IEEE80211_EDMG_BW_CONFIG_10: 2.16GHz, 4.32GHz, 6.48GHz and 2.16GHz+2.16GHz
* @IEEE80211_EDMG_BW_CONFIG_11: 2.16GHz, 4.32GHz, 6.48GHz, 8.64GHz and
* 2.16GHz+2.16GHz
* @IEEE80211_EDMG_BW_CONFIG_12: 2.16GHz, 2.16GHz + 2.16GHz and
* 4.32GHz + 4.32GHz
* @IEEE80211_EDMG_BW_CONFIG_13: 2.16GHz, 4.32GHz, 2.16GHz + 2.16GHz and
* 4.32GHz + 4.32GHz
* @IEEE80211_EDMG_BW_CONFIG_14: 2.16GHz, 4.32GHz, 6.48GHz, 2.16GHz + 2.16GHz
* and 4.32GHz + 4.32GHz
* @IEEE80211_EDMG_BW_CONFIG_15: 2.16GHz, 4.32GHz, 6.48GHz, 8.64GHz,
* 2.16GHz + 2.16GHz and 4.32GHz + 4.32GHz
*/
enum ieee80211_edmg_bw_config {
IEEE80211_EDMG_BW_CONFIG_4 = 4,
IEEE80211_EDMG_BW_CONFIG_5 = 5,
IEEE80211_EDMG_BW_CONFIG_6 = 6,
IEEE80211_EDMG_BW_CONFIG_7 = 7,
IEEE80211_EDMG_BW_CONFIG_8 = 8,
IEEE80211_EDMG_BW_CONFIG_9 = 9,
IEEE80211_EDMG_BW_CONFIG_10 = 10,
IEEE80211_EDMG_BW_CONFIG_11 = 11,
IEEE80211_EDMG_BW_CONFIG_12 = 12,
IEEE80211_EDMG_BW_CONFIG_13 = 13,
IEEE80211_EDMG_BW_CONFIG_14 = 14,
IEEE80211_EDMG_BW_CONFIG_15 = 15,
};
/**
* struct ieee80211_edmg - EDMG configuration
*
* This structure describes most essential parameters needed
* to describe 802.11ay EDMG configuration
*
* @channels: bitmap that indicates the 2.16 GHz channel(s)
* that are allowed to be used for transmissions.
* Bit 0 indicates channel 1, bit 1 indicates channel 2, etc.
* Set to 0 indicate EDMG not supported.
* @bw_config: Channel BW Configuration subfield encodes
* the allowed channel bandwidth configurations
*/
struct ieee80211_edmg {
u8 channels;
enum ieee80211_edmg_bw_config bw_config;
};
/**
* struct ieee80211_sta_s1g_cap - STA's S1G capabilities
*
* This structure describes most essential parameters needed
* to describe 802.11ah S1G capabilities for a STA.
*
* @s1g: is STA an S1G STA
* @cap: S1G capabilities information
* @nss_mcs: Supported NSS MCS set
*/
struct ieee80211_sta_s1g_cap {
bool s1g;
u8 cap[10]; /* use S1G_CAPAB_ */
u8 nss_mcs[5];
};
/**
* struct ieee80211_supported_band - frequency band definition
*
* This structure describes a frequency band a wiphy
* is able to operate in.
*
* @channels: Array of channels the hardware can operate with
* in this band.
* @band: the band this structure represents
* @n_channels: Number of channels in @channels
* @bitrates: Array of bitrates the hardware can operate with
* in this band. Must be sorted to give a valid "supported
* rates" IE, i.e. CCK rates first, then OFDM.
* @n_bitrates: Number of bitrates in @bitrates
* @ht_cap: HT capabilities in this band
* @vht_cap: VHT capabilities in this band
* @s1g_cap: S1G capabilities in this band
* @edmg_cap: EDMG capabilities in this band
* @s1g_cap: S1G capabilities in this band (S1B band only, of course)
* @n_iftype_data: number of iftype data entries
* @iftype_data: interface type data entries. Note that the bits in
* @types_mask inside this structure cannot overlap (i.e. only
* one occurrence of each type is allowed across all instances of
* iftype_data).
*/
struct ieee80211_supported_band {
struct ieee80211_channel *channels;
struct ieee80211_rate *bitrates;
enum nl80211_band band;
int n_channels;
int n_bitrates;
struct ieee80211_sta_ht_cap ht_cap;
struct ieee80211_sta_vht_cap vht_cap;
struct ieee80211_sta_s1g_cap s1g_cap;
struct ieee80211_edmg edmg_cap;
u16 n_iftype_data;
const struct ieee80211_sband_iftype_data __iftd *iftype_data;
};
/**
* _ieee80211_set_sband_iftype_data - set sband iftype data array
* @sband: the sband to initialize
* @iftd: the iftype data array pointer
* @n_iftd: the length of the iftype data array
*
* Set the sband iftype data array; use this where the length cannot
* be derived from the ARRAY_SIZE() of the argument, but prefer
* ieee80211_set_sband_iftype_data() where it can be used.
*/
static inline void
_ieee80211_set_sband_iftype_data(struct ieee80211_supported_band *sband,
const struct ieee80211_sband_iftype_data *iftd,
u16 n_iftd)
{
sband->iftype_data = (const void __iftd __force *)iftd;
sband->n_iftype_data = n_iftd;
}
/**
* ieee80211_set_sband_iftype_data - set sband iftype data array
* @sband: the sband to initialize
* @iftd: the iftype data array
*/
#define ieee80211_set_sband_iftype_data(sband, iftd) \
_ieee80211_set_sband_iftype_data(sband, iftd, ARRAY_SIZE(iftd))
/**
* for_each_sband_iftype_data - iterate sband iftype data entries
* @sband: the sband whose iftype_data array to iterate
* @i: iterator counter
* @iftd: iftype data pointer to set
*/
#define for_each_sband_iftype_data(sband, i, iftd) \
for (i = 0, iftd = (const void __force *)&(sband)->iftype_data[i]; \
i < (sband)->n_iftype_data; \
i++, iftd = (const void __force *)&(sband)->iftype_data[i])
/**
* ieee80211_get_sband_iftype_data - return sband data for a given iftype
* @sband: the sband to search for the STA on
* @iftype: enum nl80211_iftype
*
* Return: pointer to struct ieee80211_sband_iftype_data, or NULL is none found
*/
static inline const struct ieee80211_sband_iftype_data *
ieee80211_get_sband_iftype_data(const struct ieee80211_supported_band *sband,
u8 iftype)
{
const struct ieee80211_sband_iftype_data *data;
int i;
if (WARN_ON(iftype >= NL80211_IFTYPE_MAX))
return NULL;
if (iftype == NL80211_IFTYPE_AP_VLAN)
iftype = NL80211_IFTYPE_AP;
for_each_sband_iftype_data(sband, i, data) {
if (data->types_mask & BIT(iftype))
return data;
}
return NULL;
}
/**
* ieee80211_get_he_iftype_cap - return HE capabilities for an sband's iftype
* @sband: the sband to search for the iftype on
* @iftype: enum nl80211_iftype
*
* Return: pointer to the struct ieee80211_sta_he_cap, or NULL is none found
*/
static inline const struct ieee80211_sta_he_cap *
ieee80211_get_he_iftype_cap(const struct ieee80211_supported_band *sband,
u8 iftype)
{
const struct ieee80211_sband_iftype_data *data =
ieee80211_get_sband_iftype_data(sband, iftype);
if (data && data->he_cap.has_he)
return &data->he_cap;
return NULL;
}
/**
* ieee80211_get_he_6ghz_capa - return HE 6 GHz capabilities
* @sband: the sband to search for the STA on
* @iftype: the iftype to search for
*
* Return: the 6GHz capabilities
*/
static inline __le16
ieee80211_get_he_6ghz_capa(const struct ieee80211_supported_band *sband,
enum nl80211_iftype iftype)
{
const struct ieee80211_sband_iftype_data *data =
ieee80211_get_sband_iftype_data(sband, iftype);
if (WARN_ON(!data || !data->he_cap.has_he))
return 0;
return data->he_6ghz_capa.capa;
}
/**
* ieee80211_get_eht_iftype_cap - return ETH capabilities for an sband's iftype
* @sband: the sband to search for the iftype on
* @iftype: enum nl80211_iftype
*
* Return: pointer to the struct ieee80211_sta_eht_cap, or NULL is none found
*/
static inline const struct ieee80211_sta_eht_cap *
ieee80211_get_eht_iftype_cap(const struct ieee80211_supported_band *sband,
enum nl80211_iftype iftype)
{
const struct ieee80211_sband_iftype_data *data =
ieee80211_get_sband_iftype_data(sband, iftype);
if (data && data->eht_cap.has_eht)
return &data->eht_cap;
return NULL;
}
/**
* wiphy_read_of_freq_limits - read frequency limits from device tree
*
* @wiphy: the wireless device to get extra limits for
*
* Some devices may have extra limitations specified in DT. This may be useful
* for chipsets that normally support more bands but are limited due to board
* design (e.g. by antennas or external power amplifier).
*
* This function reads info from DT and uses it to *modify* channels (disable
* unavailable ones). It's usually a *bad* idea to use it in drivers with
* shared channel data as DT limitations are device specific. You should make
* sure to call it only if channels in wiphy are copied and can be modified
* without affecting other devices.
*
* As this function access device node it has to be called after set_wiphy_dev.
* It also modifies channels so they have to be set first.
* If using this helper, call it before wiphy_register().
*/
#ifdef CONFIG_OF
void wiphy_read_of_freq_limits(struct wiphy *wiphy);
#else /* CONFIG_OF */
static inline void wiphy_read_of_freq_limits(struct wiphy *wiphy)
{
}
#endif /* !CONFIG_OF */
/*
* Wireless hardware/device configuration structures and methods
*/
/**
* DOC: Actions and configuration
*
* Each wireless device and each virtual interface offer a set of configuration
* operations and other actions that are invoked by userspace. Each of these
* actions is described in the operations structure, and the parameters these
* operations use are described separately.
*
* Additionally, some operations are asynchronous and expect to get status
* information via some functions that drivers need to call.
*
* Scanning and BSS list handling with its associated functionality is described
* in a separate chapter.
*/
#define VHT_MUMIMO_GROUPS_DATA_LEN (WLAN_MEMBERSHIP_LEN +\
WLAN_USER_POSITION_LEN)
/**
* struct vif_params - describes virtual interface parameters
* @flags: monitor interface flags, unchanged if 0, otherwise
* %MONITOR_FLAG_CHANGED will be set
* @use_4addr: use 4-address frames
* @macaddr: address to use for this virtual interface.
* If this parameter is set to zero address the driver may
* determine the address as needed.
* This feature is only fully supported by drivers that enable the
* %NL80211_FEATURE_MAC_ON_CREATE flag. Others may support creating
** only p2p devices with specified MAC.
* @vht_mumimo_groups: MU-MIMO groupID, used for monitoring MU-MIMO packets
* belonging to that MU-MIMO groupID; %NULL if not changed
* @vht_mumimo_follow_addr: MU-MIMO follow address, used for monitoring
* MU-MIMO packets going to the specified station; %NULL if not changed
*/
struct vif_params {
u32 flags;
int use_4addr;
u8 macaddr[ETH_ALEN];
const u8 *vht_mumimo_groups;
const u8 *vht_mumimo_follow_addr;
};
/**
* struct key_params - key information
*
* Information about a key
*
* @key: key material
* @key_len: length of key material
* @cipher: cipher suite selector
* @seq: sequence counter (IV/PN) for TKIP and CCMP keys, only used
* with the get_key() callback, must be in little endian,
* length given by @seq_len.
* @seq_len: length of @seq.
* @vlan_id: vlan_id for VLAN group key (if nonzero)
* @mode: key install mode (RX_TX, NO_TX or SET_TX)
*/
struct key_params {
const u8 *key;
const u8 *seq;
int key_len;
int seq_len;
u16 vlan_id;
u32 cipher;
enum nl80211_key_mode mode;
};
/**
* struct cfg80211_chan_def - channel definition
* @chan: the (control) channel
* @width: channel width
* @center_freq1: center frequency of first segment
* @center_freq2: center frequency of second segment
* (only with 80+80 MHz)
* @edmg: define the EDMG channels configuration.
* If edmg is requested (i.e. the .channels member is non-zero),
* chan will define the primary channel and all other
* parameters are ignored.
* @freq1_offset: offset from @center_freq1, in KHz
* @punctured: mask of the punctured 20 MHz subchannels, with
* bits turned on being disabled (punctured); numbered
* from lower to higher frequency (like in the spec)
*/
struct cfg80211_chan_def {
struct ieee80211_channel *chan;
enum nl80211_chan_width width;
u32 center_freq1;
u32 center_freq2;
struct ieee80211_edmg edmg;
u16 freq1_offset;
u16 punctured;
};
/*
* cfg80211_bitrate_mask - masks for bitrate control
*/
struct cfg80211_bitrate_mask {
struct {
u32 legacy;
u8 ht_mcs[IEEE80211_HT_MCS_MASK_LEN];
u16 vht_mcs[NL80211_VHT_NSS_MAX];
u16 he_mcs[NL80211_HE_NSS_MAX];
enum nl80211_txrate_gi gi;
enum nl80211_he_gi he_gi;
enum nl80211_he_ltf he_ltf;
} control[NUM_NL80211_BANDS];
};
/**
* struct cfg80211_tid_cfg - TID specific configuration
* @config_override: Flag to notify driver to reset TID configuration
* of the peer.
* @tids: bitmap of TIDs to modify
* @mask: bitmap of attributes indicating which parameter changed,
* similar to &nl80211_tid_config_supp.
* @noack: noack configuration value for the TID
* @retry_long: retry count value
* @retry_short: retry count value
* @ampdu: Enable/Disable MPDU aggregation
* @rtscts: Enable/Disable RTS/CTS
* @amsdu: Enable/Disable MSDU aggregation
* @txrate_type: Tx bitrate mask type
* @txrate_mask: Tx bitrate to be applied for the TID
*/
struct cfg80211_tid_cfg {
bool config_override;
u8 tids;
u64 mask;
enum nl80211_tid_config noack;
u8 retry_long, retry_short;
enum nl80211_tid_config ampdu;
enum nl80211_tid_config rtscts;
enum nl80211_tid_config amsdu;
enum nl80211_tx_rate_setting txrate_type;
struct cfg80211_bitrate_mask txrate_mask;
};
/**
* struct cfg80211_tid_config - TID configuration
* @peer: Station's MAC address
* @n_tid_conf: Number of TID specific configurations to be applied
* @tid_conf: Configuration change info
*/
struct cfg80211_tid_config {
const u8 *peer;
u32 n_tid_conf;
struct cfg80211_tid_cfg tid_conf[] __counted_by(n_tid_conf);
};
/**
* struct cfg80211_fils_aad - FILS AAD data
* @macaddr: STA MAC address
* @kek: FILS KEK
* @kek_len: FILS KEK length
* @snonce: STA Nonce
* @anonce: AP Nonce
*/
struct cfg80211_fils_aad {
const u8 *macaddr;
const u8 *kek;
u8 kek_len;
const u8 *snonce;
const u8 *anonce;
};
/**
* struct cfg80211_set_hw_timestamp - enable/disable HW timestamping
* @macaddr: peer MAC address. NULL to enable/disable HW timestamping for all
* addresses.
* @enable: if set, enable HW timestamping for the specified MAC address.
* Otherwise disable HW timestamping for the specified MAC address.
*/
struct cfg80211_set_hw_timestamp {
const u8 *macaddr;
bool enable;
};
/**
* cfg80211_get_chandef_type - return old channel type from chandef
* @chandef: the channel definition
*
* Return: The old channel type (NOHT, HT20, HT40+/-) from a given
* chandef, which must have a bandwidth allowing this conversion.
*/
static inline enum nl80211_channel_type
cfg80211_get_chandef_type(const struct cfg80211_chan_def *chandef)
{
switch (chandef->width) {
case NL80211_CHAN_WIDTH_20_NOHT:
return NL80211_CHAN_NO_HT;
case NL80211_CHAN_WIDTH_20:
return NL80211_CHAN_HT20;
case NL80211_CHAN_WIDTH_40:
if (chandef->center_freq1 > chandef->chan->center_freq)
return NL80211_CHAN_HT40PLUS;
return NL80211_CHAN_HT40MINUS;
default:
WARN_ON(1);
return NL80211_CHAN_NO_HT;
}
}
/**
* cfg80211_chandef_create - create channel definition using channel type
* @chandef: the channel definition struct to fill
* @channel: the control channel
* @chantype: the channel type
*
* Given a channel type, create a channel definition.
*/
void cfg80211_chandef_create(struct cfg80211_chan_def *chandef,
struct ieee80211_channel *channel,
enum nl80211_channel_type chantype);
/**
* cfg80211_chandef_identical - check if two channel definitions are identical
* @chandef1: first channel definition
* @chandef2: second channel definition
*
* Return: %true if the channels defined by the channel definitions are
* identical, %false otherwise.
*/
static inline bool
cfg80211_chandef_identical(const struct cfg80211_chan_def *chandef1,
const struct cfg80211_chan_def *chandef2)
{
return (chandef1->chan == chandef2->chan &&
chandef1->width == chandef2->width &&
chandef1->center_freq1 == chandef2->center_freq1 &&
chandef1->freq1_offset == chandef2->freq1_offset &&
chandef1->center_freq2 == chandef2->center_freq2 &&
chandef1->punctured == chandef2->punctured);
}
/**
* cfg80211_chandef_is_edmg - check if chandef represents an EDMG channel
*
* @chandef: the channel definition
*
* Return: %true if EDMG defined, %false otherwise.
*/
static inline bool
cfg80211_chandef_is_edmg(const struct cfg80211_chan_def *chandef)
{
return chandef->edmg.channels || chandef->edmg.bw_config;
}
/**
* cfg80211_chandef_compatible - check if two channel definitions are compatible
* @chandef1: first channel definition
* @chandef2: second channel definition
*
* Return: %NULL if the given channel definitions are incompatible,
* chandef1 or chandef2 otherwise.
*/
const struct cfg80211_chan_def *
cfg80211_chandef_compatible(const struct cfg80211_chan_def *chandef1,
const struct cfg80211_chan_def *chandef2);
/**
* nl80211_chan_width_to_mhz - get the channel width in MHz
* @chan_width: the channel width from &enum nl80211_chan_width
*
* Return: channel width in MHz if the chan_width from &enum nl80211_chan_width
* is valid. -1 otherwise.
*/
int nl80211_chan_width_to_mhz(enum nl80211_chan_width chan_width);
/**
* cfg80211_chandef_valid - check if a channel definition is valid
* @chandef: the channel definition to check
* Return: %true if the channel definition is valid. %false otherwise.
*/
bool cfg80211_chandef_valid(const struct cfg80211_chan_def *chandef);
/**
* cfg80211_chandef_usable - check if secondary channels can be used
* @wiphy: the wiphy to validate against
* @chandef: the channel definition to check
* @prohibited_flags: the regulatory channel flags that must not be set
* Return: %true if secondary channels are usable. %false otherwise.
*/
bool cfg80211_chandef_usable(struct wiphy *wiphy,
const struct cfg80211_chan_def *chandef,
u32 prohibited_flags);
/**
* cfg80211_chandef_dfs_required - checks if radar detection is required
* @wiphy: the wiphy to validate against
* @chandef: the channel definition to check
* @iftype: the interface type as specified in &enum nl80211_iftype
* Returns:
* 1 if radar detection is required, 0 if it is not, < 0 on error
*/
int cfg80211_chandef_dfs_required(struct wiphy *wiphy,
const struct cfg80211_chan_def *chandef,
enum nl80211_iftype iftype);
/**
* cfg80211_chandef_dfs_usable - checks if chandef is DFS usable and we
* can/need start CAC on such channel
* @wiphy: the wiphy to validate against
* @chandef: the channel definition to check
*
* Return: true if all channels available and at least
* one channel requires CAC (NL80211_DFS_USABLE)
*/
bool cfg80211_chandef_dfs_usable(struct wiphy *wiphy,
const struct cfg80211_chan_def *chandef);
/**
* cfg80211_chandef_dfs_cac_time - get the DFS CAC time (in ms) for given
* channel definition
* @wiphy: the wiphy to validate against
* @chandef: the channel definition to check
*
* Returns: DFS CAC time (in ms) which applies for this channel definition
*/
unsigned int
cfg80211_chandef_dfs_cac_time(struct wiphy *wiphy,
const struct cfg80211_chan_def *chandef);
/**
* cfg80211_chandef_primary - calculate primary 40/80/160 MHz freq
* @chandef: chandef to calculate for
* @primary_chan_width: primary channel width to calculate center for
* @punctured: punctured sub-channel bitmap, will be recalculated
* according to the new bandwidth, can be %NULL
*
* Returns: the primary 40/80/160 MHz channel center frequency, or -1
* for errors, updating the punctured bitmap
*/
int cfg80211_chandef_primary(const struct cfg80211_chan_def *chandef,
enum nl80211_chan_width primary_chan_width,
u16 *punctured);
/**
* nl80211_send_chandef - sends the channel definition.
* @msg: the msg to send channel definition
* @chandef: the channel definition to check
*
* Returns: 0 if sent the channel definition to msg, < 0 on error
**/
int nl80211_send_chandef(struct sk_buff *msg, const struct cfg80211_chan_def *chandef);
/**
* ieee80211_chanwidth_rate_flags - return rate flags for channel width
* @width: the channel width of the channel
*
* In some channel types, not all rates may be used - for example CCK
* rates may not be used in 5/10 MHz channels.
*
* Returns: rate flags which apply for this channel width
*/
static inline enum ieee80211_rate_flags
ieee80211_chanwidth_rate_flags(enum nl80211_chan_width width)
{
switch (width) {
case NL80211_CHAN_WIDTH_5:
return IEEE80211_RATE_SUPPORTS_5MHZ;
case NL80211_CHAN_WIDTH_10:
return IEEE80211_RATE_SUPPORTS_10MHZ;
default:
break;
}
return 0;
}
/**
* ieee80211_chandef_rate_flags - returns rate flags for a channel
* @chandef: channel definition for the channel
*
* See ieee80211_chanwidth_rate_flags().
*
* Returns: rate flags which apply for this channel
*/
static inline enum ieee80211_rate_flags
ieee80211_chandef_rate_flags(struct cfg80211_chan_def *chandef)
{
return ieee80211_chanwidth_rate_flags(chandef->width);
}
/**
* ieee80211_chandef_max_power - maximum transmission power for the chandef
*
* In some regulations, the transmit power may depend on the configured channel
* bandwidth which may be defined as dBm/MHz. This function returns the actual
* max_power for non-standard (20 MHz) channels.
*
* @chandef: channel definition for the channel
*
* Returns: maximum allowed transmission power in dBm for the chandef
*/
static inline int
ieee80211_chandef_max_power(struct cfg80211_chan_def *chandef)
{
switch (chandef->width) {
case NL80211_CHAN_WIDTH_5:
return min(chandef->chan->max_reg_power - 6,
chandef->chan->max_power);
case NL80211_CHAN_WIDTH_10:
return min(chandef->chan->max_reg_power - 3,
chandef->chan->max_power);
default:
break;
}
return chandef->chan->max_power;
}
/**
* cfg80211_any_usable_channels - check for usable channels
* @wiphy: the wiphy to check for
* @band_mask: which bands to check on
* @prohibited_flags: which channels to not consider usable,
* %IEEE80211_CHAN_DISABLED is always taken into account
*/
bool cfg80211_any_usable_channels(struct wiphy *wiphy,
unsigned long band_mask,
u32 prohibited_flags);
/**
* enum survey_info_flags - survey information flags
*
* @SURVEY_INFO_NOISE_DBM: noise (in dBm) was filled in
* @SURVEY_INFO_IN_USE: channel is currently being used
* @SURVEY_INFO_TIME: active time (in ms) was filled in
* @SURVEY_INFO_TIME_BUSY: busy time was filled in
* @SURVEY_INFO_TIME_EXT_BUSY: extension channel busy time was filled in
* @SURVEY_INFO_TIME_RX: receive time was filled in
* @SURVEY_INFO_TIME_TX: transmit time was filled in
* @SURVEY_INFO_TIME_SCAN: scan time was filled in
* @SURVEY_INFO_TIME_BSS_RX: local BSS receive time was filled in
*
* Used by the driver to indicate which info in &struct survey_info
* it has filled in during the get_survey().
*/
enum survey_info_flags {
SURVEY_INFO_NOISE_DBM = BIT(0),
SURVEY_INFO_IN_USE = BIT(1),
SURVEY_INFO_TIME = BIT(2),
SURVEY_INFO_TIME_BUSY = BIT(3),
SURVEY_INFO_TIME_EXT_BUSY = BIT(4),
SURVEY_INFO_TIME_RX = BIT(5),
SURVEY_INFO_TIME_TX = BIT(6),
SURVEY_INFO_TIME_SCAN = BIT(7),
SURVEY_INFO_TIME_BSS_RX = BIT(8),
};
/**
* struct survey_info - channel survey response
*
* @channel: the channel this survey record reports, may be %NULL for a single
* record to report global statistics
* @filled: bitflag of flags from &enum survey_info_flags
* @noise: channel noise in dBm. This and all following fields are
* optional
* @time: amount of time in ms the radio was turn on (on the channel)
* @time_busy: amount of time the primary channel was sensed busy
* @time_ext_busy: amount of time the extension channel was sensed busy
* @time_rx: amount of time the radio spent receiving data
* @time_tx: amount of time the radio spent transmitting data
* @time_scan: amount of time the radio spent for scanning
* @time_bss_rx: amount of time the radio spent receiving data on a local BSS
*
* Used by dump_survey() to report back per-channel survey information.
*
* This structure can later be expanded with things like
* channel duty cycle etc.
*/
struct survey_info {
struct ieee80211_channel *channel;
u64 time;
u64 time_busy;
u64 time_ext_busy;
u64 time_rx;
u64 time_tx;
u64 time_scan;
u64 time_bss_rx;
u32 filled;
s8 noise;
};
#define CFG80211_MAX_NUM_AKM_SUITES 10
/**
* struct cfg80211_crypto_settings - Crypto settings
* @wpa_versions: indicates which, if any, WPA versions are enabled
* (from enum nl80211_wpa_versions)
* @cipher_group: group key cipher suite (or 0 if unset)
* @n_ciphers_pairwise: number of AP supported unicast ciphers
* @ciphers_pairwise: unicast key cipher suites
* @n_akm_suites: number of AKM suites
* @akm_suites: AKM suites
* @control_port: Whether user space controls IEEE 802.1X port, i.e.,
* sets/clears %NL80211_STA_FLAG_AUTHORIZED. If true, the driver is
* required to assume that the port is unauthorized until authorized by
* user space. Otherwise, port is marked authorized by default.
* @control_port_ethertype: the control port protocol that should be
* allowed through even on unauthorized ports
* @control_port_no_encrypt: TRUE to prevent encryption of control port
* protocol frames.
* @control_port_over_nl80211: TRUE if userspace expects to exchange control
* port frames over NL80211 instead of the network interface.
* @control_port_no_preauth: disables pre-auth rx over the nl80211 control
* port for mac80211
* @psk: PSK (for devices supporting 4-way-handshake offload)
* @sae_pwd: password for SAE authentication (for devices supporting SAE
* offload)
* @sae_pwd_len: length of SAE password (for devices supporting SAE offload)
* @sae_pwe: The mechanisms allowed for SAE PWE derivation:
*
* NL80211_SAE_PWE_UNSPECIFIED
* Not-specified, used to indicate userspace did not specify any
* preference. The driver should follow its internal policy in
* such a scenario.
*
* NL80211_SAE_PWE_HUNT_AND_PECK
* Allow hunting-and-pecking loop only
*
* NL80211_SAE_PWE_HASH_TO_ELEMENT
* Allow hash-to-element only
*
* NL80211_SAE_PWE_BOTH
* Allow either hunting-and-pecking loop or hash-to-element
*/
struct cfg80211_crypto_settings {
u32 wpa_versions;
u32 cipher_group;
int n_ciphers_pairwise;
u32 ciphers_pairwise[NL80211_MAX_NR_CIPHER_SUITES];
int n_akm_suites;
u32 akm_suites[CFG80211_MAX_NUM_AKM_SUITES];
bool control_port;
__be16 control_port_ethertype;
bool control_port_no_encrypt;
bool control_port_over_nl80211;
bool control_port_no_preauth;
const u8 *psk;
const u8 *sae_pwd;
u8 sae_pwd_len;
enum nl80211_sae_pwe_mechanism sae_pwe;
};
/**
* struct cfg80211_mbssid_config - AP settings for multi bssid
*
* @tx_wdev: pointer to the transmitted interface in the MBSSID set
* @index: index of this AP in the multi bssid group.
* @ema: set to true if the beacons should be sent out in EMA mode.
*/
struct cfg80211_mbssid_config {
struct wireless_dev *tx_wdev;
u8 index;
bool ema;
};
/**
* struct cfg80211_mbssid_elems - Multiple BSSID elements
*
* @cnt: Number of elements in array %elems.
*
* @elem: Array of multiple BSSID element(s) to be added into Beacon frames.
* @elem.data: Data for multiple BSSID elements.
* @elem.len: Length of data.
*/
struct cfg80211_mbssid_elems {
u8 cnt;
struct {
const u8 *data;
size_t len;
} elem[] __counted_by(cnt);
};
/**
* struct cfg80211_rnr_elems - Reduced neighbor report (RNR) elements
*
* @cnt: Number of elements in array %elems.
*
* @elem: Array of RNR element(s) to be added into Beacon frames.
* @elem.data: Data for RNR elements.
* @elem.len: Length of data.
*/
struct cfg80211_rnr_elems {
u8 cnt;
struct {
const u8 *data;
size_t len;
} elem[] __counted_by(cnt);
};
/**
* struct cfg80211_beacon_data - beacon data
* @link_id: the link ID for the AP MLD link sending this beacon
* @head: head portion of beacon (before TIM IE)
* or %NULL if not changed
* @tail: tail portion of beacon (after TIM IE)
* or %NULL if not changed
* @head_len: length of @head
* @tail_len: length of @tail
* @beacon_ies: extra information element(s) to add into Beacon frames or %NULL
* @beacon_ies_len: length of beacon_ies in octets
* @proberesp_ies: extra information element(s) to add into Probe Response
* frames or %NULL
* @proberesp_ies_len: length of proberesp_ies in octets
* @assocresp_ies: extra information element(s) to add into (Re)Association
* Response frames or %NULL
* @assocresp_ies_len: length of assocresp_ies in octets
* @probe_resp_len: length of probe response template (@probe_resp)
* @probe_resp: probe response template (AP mode only)
* @mbssid_ies: multiple BSSID elements
* @rnr_ies: reduced neighbor report elements
* @ftm_responder: enable FTM responder functionality; -1 for no change
* (which also implies no change in LCI/civic location data)
* @lci: Measurement Report element content, starting with Measurement Token
* (measurement type 8)
* @civicloc: Measurement Report element content, starting with Measurement
* Token (measurement type 11)
* @lci_len: LCI data length
* @civicloc_len: Civic location data length
* @he_bss_color: BSS Color settings
* @he_bss_color_valid: indicates whether bss color
* attribute is present in beacon data or not.
*/
struct cfg80211_beacon_data {
unsigned int link_id;
const u8 *head, *tail;
const u8 *beacon_ies;
const u8 *proberesp_ies;
const u8 *assocresp_ies;
const u8 *probe_resp;
const u8 *lci;
const u8 *civicloc;
struct cfg80211_mbssid_elems *mbssid_ies;
struct cfg80211_rnr_elems *rnr_ies;
s8 ftm_responder;
size_t head_len, tail_len;
size_t beacon_ies_len;
size_t proberesp_ies_len;
size_t assocresp_ies_len;
size_t probe_resp_len;
size_t lci_len;
size_t civicloc_len;
struct cfg80211_he_bss_color he_bss_color;
bool he_bss_color_valid;
};
struct mac_address {
u8 addr[ETH_ALEN];
};
/**
* struct cfg80211_acl_data - Access control list data
*
* @acl_policy: ACL policy to be applied on the station's
* entry specified by mac_addr
* @n_acl_entries: Number of MAC address entries passed
* @mac_addrs: List of MAC addresses of stations to be used for ACL
*/
struct cfg80211_acl_data {
enum nl80211_acl_policy acl_policy;
int n_acl_entries;
/* Keep it last */
struct mac_address mac_addrs[] __counted_by(n_acl_entries);
};
/**
* struct cfg80211_fils_discovery - FILS discovery parameters from
* IEEE Std 802.11ai-2016, Annex C.3 MIB detail.
*
* @update: Set to true if the feature configuration should be updated.
* @min_interval: Minimum packet interval in TUs (0 - 10000)
* @max_interval: Maximum packet interval in TUs (0 - 10000)
* @tmpl_len: Template length
* @tmpl: Template data for FILS discovery frame including the action
* frame headers.
*/
struct cfg80211_fils_discovery {
bool update;
u32 min_interval;
u32 max_interval;
size_t tmpl_len;
const u8 *tmpl;
};
/**
* struct cfg80211_unsol_bcast_probe_resp - Unsolicited broadcast probe
* response parameters in 6GHz.
*
* @update: Set to true if the feature configuration should be updated.
* @interval: Packet interval in TUs. Maximum allowed is 20 TU, as mentioned
* in IEEE P802.11ax/D6.0 26.17.2.3.2 - AP behavior for fast passive
* scanning
* @tmpl_len: Template length
* @tmpl: Template data for probe response
*/
struct cfg80211_unsol_bcast_probe_resp {
bool update;
u32 interval;
size_t tmpl_len;
const u8 *tmpl;
};
/**
* struct cfg80211_ap_settings - AP configuration
*
* Used to configure an AP interface.
*
* @chandef: defines the channel to use
* @beacon: beacon data
* @beacon_interval: beacon interval
* @dtim_period: DTIM period
* @ssid: SSID to be used in the BSS (note: may be %NULL if not provided from
* user space)
* @ssid_len: length of @ssid
* @hidden_ssid: whether to hide the SSID in Beacon/Probe Response frames
* @crypto: crypto settings
* @privacy: the BSS uses privacy
* @auth_type: Authentication type (algorithm)
* @smps_mode: SMPS mode
* @inactivity_timeout: time in seconds to determine station's inactivity.
* @p2p_ctwindow: P2P CT Window
* @p2p_opp_ps: P2P opportunistic PS
* @acl: ACL configuration used by the drivers which has support for
* MAC address based access control
* @pbss: If set, start as a PCP instead of AP. Relevant for DMG
* networks.
* @beacon_rate: bitrate to be used for beacons
* @ht_cap: HT capabilities (or %NULL if HT isn't enabled)
* @vht_cap: VHT capabilities (or %NULL if VHT isn't enabled)
* @he_cap: HE capabilities (or %NULL if HE isn't enabled)
* @eht_cap: EHT capabilities (or %NULL if EHT isn't enabled)
* @eht_oper: EHT operation IE (or %NULL if EHT isn't enabled)
* @ht_required: stations must support HT
* @vht_required: stations must support VHT
* @twt_responder: Enable Target Wait Time
* @he_required: stations must support HE
* @sae_h2e_required: stations must support direct H2E technique in SAE
* @flags: flags, as defined in &enum nl80211_ap_settings_flags
* @he_obss_pd: OBSS Packet Detection settings
* @he_oper: HE operation IE (or %NULL if HE isn't enabled)
* @fils_discovery: FILS discovery transmission parameters
* @unsol_bcast_probe_resp: Unsolicited broadcast probe response parameters
* @mbssid_config: AP settings for multiple bssid
*/
struct cfg80211_ap_settings {
struct cfg80211_chan_def chandef;
struct cfg80211_beacon_data beacon;
int beacon_interval, dtim_period;
const u8 *ssid;
size_t ssid_len;
enum nl80211_hidden_ssid hidden_ssid;
struct cfg80211_crypto_settings crypto;
bool privacy;
enum nl80211_auth_type auth_type;
enum nl80211_smps_mode smps_mode;
int inactivity_timeout;
u8 p2p_ctwindow;
bool p2p_opp_ps;
const struct cfg80211_acl_data *acl;
bool pbss;
struct cfg80211_bitrate_mask beacon_rate;
const struct ieee80211_ht_cap *ht_cap;
const struct ieee80211_vht_cap *vht_cap;
const struct ieee80211_he_cap_elem *he_cap;
const struct ieee80211_he_operation *he_oper;
const struct ieee80211_eht_cap_elem *eht_cap;
const struct ieee80211_eht_operation *eht_oper;
bool ht_required, vht_required, he_required, sae_h2e_required;
bool twt_responder;
u32 flags;
struct ieee80211_he_obss_pd he_obss_pd;
struct cfg80211_fils_discovery fils_discovery;
struct cfg80211_unsol_bcast_probe_resp unsol_bcast_probe_resp;
struct cfg80211_mbssid_config mbssid_config;
};
/**
* struct cfg80211_ap_update - AP configuration update
*
* Subset of &struct cfg80211_ap_settings, for updating a running AP.
*
* @beacon: beacon data
* @fils_discovery: FILS discovery transmission parameters
* @unsol_bcast_probe_resp: Unsolicited broadcast probe response parameters
*/
struct cfg80211_ap_update {
struct cfg80211_beacon_data beacon;
struct cfg80211_fils_discovery fils_discovery;
struct cfg80211_unsol_bcast_probe_resp unsol_bcast_probe_resp;
};
/**
* struct cfg80211_csa_settings - channel switch settings
*
* Used for channel switch
*
* @chandef: defines the channel to use after the switch
* @beacon_csa: beacon data while performing the switch
* @counter_offsets_beacon: offsets of the counters within the beacon (tail)
* @counter_offsets_presp: offsets of the counters within the probe response
* @n_counter_offsets_beacon: number of csa counters the beacon (tail)
* @n_counter_offsets_presp: number of csa counters in the probe response
* @beacon_after: beacon data to be used on the new channel
* @radar_required: whether radar detection is required on the new channel
* @block_tx: whether transmissions should be blocked while changing
* @count: number of beacons until switch
* @link_id: defines the link on which channel switch is expected during
* MLO. 0 in case of non-MLO.
*/
struct cfg80211_csa_settings {
struct cfg80211_chan_def chandef;
struct cfg80211_beacon_data beacon_csa;
const u16 *counter_offsets_beacon;
const u16 *counter_offsets_presp;
unsigned int n_counter_offsets_beacon;
unsigned int n_counter_offsets_presp;
struct cfg80211_beacon_data beacon_after;
bool radar_required;
bool block_tx;
u8 count;
u8 link_id;
};
/**
* struct cfg80211_color_change_settings - color change settings
*
* Used for bss color change
*
* @beacon_color_change: beacon data while performing the color countdown
* @counter_offset_beacon: offsets of the counters within the beacon (tail)
* @counter_offset_presp: offsets of the counters within the probe response
* @beacon_next: beacon data to be used after the color change
* @count: number of beacons until the color change
* @color: the color used after the change
*/
struct cfg80211_color_change_settings {
struct cfg80211_beacon_data beacon_color_change;
u16 counter_offset_beacon;
u16 counter_offset_presp;
struct cfg80211_beacon_data beacon_next;
u8 count;
u8 color;
};
/**
* struct iface_combination_params - input parameters for interface combinations
*
* Used to pass interface combination parameters
*
* @num_different_channels: the number of different channels we want
* to use for verification
* @radar_detect: a bitmap where each bit corresponds to a channel
* width where radar detection is needed, as in the definition of
* &struct ieee80211_iface_combination.@radar_detect_widths
* @iftype_num: array with the number of interfaces of each interface
* type. The index is the interface type as specified in &enum
* nl80211_iftype.
* @new_beacon_int: set this to the beacon interval of a new interface
* that's not operating yet, if such is to be checked as part of
* the verification
*/
struct iface_combination_params {
int num_different_channels;
u8 radar_detect;
int iftype_num[NUM_NL80211_IFTYPES];
u32 new_beacon_int;
};
/**
* enum station_parameters_apply_mask - station parameter values to apply
* @STATION_PARAM_APPLY_UAPSD: apply new uAPSD parameters (uapsd_queues, max_sp)
* @STATION_PARAM_APPLY_CAPABILITY: apply new capability
* @STATION_PARAM_APPLY_PLINK_STATE: apply new plink state
*
* Not all station parameters have in-band "no change" signalling,
* for those that don't these flags will are used.
*/
enum station_parameters_apply_mask {
STATION_PARAM_APPLY_UAPSD = BIT(0),
STATION_PARAM_APPLY_CAPABILITY = BIT(1),
STATION_PARAM_APPLY_PLINK_STATE = BIT(2),
};
/**
* struct sta_txpwr - station txpower configuration
*
* Used to configure txpower for station.
*
* @power: tx power (in dBm) to be used for sending data traffic. If tx power
* is not provided, the default per-interface tx power setting will be
* overriding. Driver should be picking up the lowest tx power, either tx
* power per-interface or per-station.
* @type: In particular if TPC %type is NL80211_TX_POWER_LIMITED then tx power
* will be less than or equal to specified from userspace, whereas if TPC
* %type is NL80211_TX_POWER_AUTOMATIC then it indicates default tx power.
* NL80211_TX_POWER_FIXED is not a valid configuration option for
* per peer TPC.
*/
struct sta_txpwr {
s16 power;
enum nl80211_tx_power_setting type;
};
/**
* struct link_station_parameters - link station parameters
*
* Used to change and create a new link station.
*
* @mld_mac: MAC address of the station
* @link_id: the link id (-1 for non-MLD station)
* @link_mac: MAC address of the link
* @supported_rates: supported rates in IEEE 802.11 format
* (or NULL for no change)
* @supported_rates_len: number of supported rates
* @ht_capa: HT capabilities of station
* @vht_capa: VHT capabilities of station
* @opmode_notif: operating mode field from Operating Mode Notification
* @opmode_notif_used: information if operating mode field is used
* @he_capa: HE capabilities of station
* @he_capa_len: the length of the HE capabilities
* @txpwr: transmit power for an associated station
* @txpwr_set: txpwr field is set
* @he_6ghz_capa: HE 6 GHz Band capabilities of station
* @eht_capa: EHT capabilities of station
* @eht_capa_len: the length of the EHT capabilities
*/
struct link_station_parameters {
const u8 *mld_mac;
int link_id;
const u8 *link_mac;
const u8 *supported_rates;
u8 supported_rates_len;
const struct ieee80211_ht_cap *ht_capa;
const struct ieee80211_vht_cap *vht_capa;
u8 opmode_notif;
bool opmode_notif_used;
const struct ieee80211_he_cap_elem *he_capa;
u8 he_capa_len;
struct sta_txpwr txpwr;
bool txpwr_set;
const struct ieee80211_he_6ghz_capa *he_6ghz_capa;
const struct ieee80211_eht_cap_elem *eht_capa;
u8 eht_capa_len;
};
/**
* struct link_station_del_parameters - link station deletion parameters
*
* Used to delete a link station entry (or all stations).
*
* @mld_mac: MAC address of the station
* @link_id: the link id
*/
struct link_station_del_parameters {
const u8 *mld_mac;
u32 link_id;
};
/**
* struct cfg80211_ttlm_params: TID to link mapping parameters
*
* Used for setting a TID to link mapping.
*
* @dlink: Downlink TID to link mapping, as defined in section 9.4.2.314
* (TID-To-Link Mapping element) in Draft P802.11be_D4.0.
* @ulink: Uplink TID to link mapping, as defined in section 9.4.2.314
* (TID-To-Link Mapping element) in Draft P802.11be_D4.0.
*/
struct cfg80211_ttlm_params {
u16 dlink[8];
u16 ulink[8];
};
/**
* struct station_parameters - station parameters
*
* Used to change and create a new station.
*
* @vlan: vlan interface station should belong to
* @sta_flags_mask: station flags that changed
* (bitmask of BIT(%NL80211_STA_FLAG_...))
* @sta_flags_set: station flags values
* (bitmask of BIT(%NL80211_STA_FLAG_...))
* @listen_interval: listen interval or -1 for no change
* @aid: AID or zero for no change
* @vlan_id: VLAN ID for station (if nonzero)
* @peer_aid: mesh peer AID or zero for no change
* @plink_action: plink action to take
* @plink_state: set the peer link state for a station
* @uapsd_queues: bitmap of queues configured for uapsd. same format
* as the AC bitmap in the QoS info field
* @max_sp: max Service Period. same format as the MAX_SP in the
* QoS info field (but already shifted down)
* @sta_modify_mask: bitmap indicating which parameters changed
* (for those that don't have a natural "no change" value),
* see &enum station_parameters_apply_mask
* @local_pm: local link-specific mesh power save mode (no change when set
* to unknown)
* @capability: station capability
* @ext_capab: extended capabilities of the station
* @ext_capab_len: number of extended capabilities
* @supported_channels: supported channels in IEEE 802.11 format
* @supported_channels_len: number of supported channels
* @supported_oper_classes: supported oper classes in IEEE 802.11 format
* @supported_oper_classes_len: number of supported operating classes
* @support_p2p_ps: information if station supports P2P PS mechanism
* @airtime_weight: airtime scheduler weight for this station
* @link_sta_params: link related params.
*/
struct station_parameters {
struct net_device *vlan;
u32 sta_flags_mask, sta_flags_set;
u32 sta_modify_mask;
int listen_interval;
u16 aid;
u16 vlan_id;
u16 peer_aid;
u8 plink_action;
u8 plink_state;
u8 uapsd_queues;
u8 max_sp;
enum nl80211_mesh_power_mode local_pm;
u16 capability;
const u8 *ext_capab;
u8 ext_capab_len;
const u8 *supported_channels;
u8 supported_channels_len;
const u8 *supported_oper_classes;
u8 supported_oper_classes_len;
int support_p2p_ps;
u16 airtime_weight;
struct link_station_parameters link_sta_params;
};
/**
* struct station_del_parameters - station deletion parameters
*
* Used to delete a station entry (or all stations).
*
* @mac: MAC address of the station to remove or NULL to remove all stations
* @subtype: Management frame subtype to use for indicating removal
* (10 = Disassociation, 12 = Deauthentication)
* @reason_code: Reason code for the Disassociation/Deauthentication frame
* @link_id: Link ID indicating a link that stations to be flushed must be
* using; valid only for MLO, but can also be -1 for MLO to really
* remove all stations.
*/
struct station_del_parameters {
const u8 *mac;
u8 subtype;
u16 reason_code;
int link_id;
};
/**
* enum cfg80211_station_type - the type of station being modified
* @CFG80211_STA_AP_CLIENT: client of an AP interface
* @CFG80211_STA_AP_CLIENT_UNASSOC: client of an AP interface that is still
* unassociated (update properties for this type of client is permitted)
* @CFG80211_STA_AP_MLME_CLIENT: client of an AP interface that has
* the AP MLME in the device
* @CFG80211_STA_AP_STA: AP station on managed interface
* @CFG80211_STA_IBSS: IBSS station
* @CFG80211_STA_TDLS_PEER_SETUP: TDLS peer on managed interface (dummy entry
* while TDLS setup is in progress, it moves out of this state when
* being marked authorized; use this only if TDLS with external setup is
* supported/used)
* @CFG80211_STA_TDLS_PEER_ACTIVE: TDLS peer on managed interface (active
* entry that is operating, has been marked authorized by userspace)
* @CFG80211_STA_MESH_PEER_KERNEL: peer on mesh interface (kernel managed)
* @CFG80211_STA_MESH_PEER_USER: peer on mesh interface (user managed)
*/
enum cfg80211_station_type {
CFG80211_STA_AP_CLIENT,
CFG80211_STA_AP_CLIENT_UNASSOC,
CFG80211_STA_AP_MLME_CLIENT,
CFG80211_STA_AP_STA,
CFG80211_STA_IBSS,
CFG80211_STA_TDLS_PEER_SETUP,
CFG80211_STA_TDLS_PEER_ACTIVE,
CFG80211_STA_MESH_PEER_KERNEL,
CFG80211_STA_MESH_PEER_USER,
};
/**
* cfg80211_check_station_change - validate parameter changes
* @wiphy: the wiphy this operates on
* @params: the new parameters for a station
* @statype: the type of station being modified
*
* Utility function for the @change_station driver method. Call this function
* with the appropriate station type looking up the station (and checking that
* it exists). It will verify whether the station change is acceptable, and if
* not will return an error code. Note that it may modify the parameters for
* backward compatibility reasons, so don't use them before calling this.
*/
int cfg80211_check_station_change(struct wiphy *wiphy,
struct station_parameters *params,
enum cfg80211_station_type statype);
/**
* enum rate_info_flags - bitrate info flags
*
* Used by the driver to indicate the specific rate transmission
* type for 802.11n transmissions.
*
* @RATE_INFO_FLAGS_MCS: mcs field filled with HT MCS
* @RATE_INFO_FLAGS_VHT_MCS: mcs field filled with VHT MCS
* @RATE_INFO_FLAGS_SHORT_GI: 400ns guard interval
* @RATE_INFO_FLAGS_DMG: 60GHz MCS
* @RATE_INFO_FLAGS_HE_MCS: HE MCS information
* @RATE_INFO_FLAGS_EDMG: 60GHz MCS in EDMG mode
* @RATE_INFO_FLAGS_EXTENDED_SC_DMG: 60GHz extended SC MCS
* @RATE_INFO_FLAGS_EHT_MCS: EHT MCS information
* @RATE_INFO_FLAGS_S1G_MCS: MCS field filled with S1G MCS
*/
enum rate_info_flags {
RATE_INFO_FLAGS_MCS = BIT(0),
RATE_INFO_FLAGS_VHT_MCS = BIT(1),
RATE_INFO_FLAGS_SHORT_GI = BIT(2),
RATE_INFO_FLAGS_DMG = BIT(3),
RATE_INFO_FLAGS_HE_MCS = BIT(4),
RATE_INFO_FLAGS_EDMG = BIT(5),
RATE_INFO_FLAGS_EXTENDED_SC_DMG = BIT(6),
RATE_INFO_FLAGS_EHT_MCS = BIT(7),
RATE_INFO_FLAGS_S1G_MCS = BIT(8),
};
/**
* enum rate_info_bw - rate bandwidth information
*
* Used by the driver to indicate the rate bandwidth.
*
* @RATE_INFO_BW_5: 5 MHz bandwidth
* @RATE_INFO_BW_10: 10 MHz bandwidth
* @RATE_INFO_BW_20: 20 MHz bandwidth
* @RATE_INFO_BW_40: 40 MHz bandwidth
* @RATE_INFO_BW_80: 80 MHz bandwidth
* @RATE_INFO_BW_160: 160 MHz bandwidth
* @RATE_INFO_BW_HE_RU: bandwidth determined by HE RU allocation
* @RATE_INFO_BW_320: 320 MHz bandwidth
* @RATE_INFO_BW_EHT_RU: bandwidth determined by EHT RU allocation
* @RATE_INFO_BW_1: 1 MHz bandwidth
* @RATE_INFO_BW_2: 2 MHz bandwidth
* @RATE_INFO_BW_4: 4 MHz bandwidth
* @RATE_INFO_BW_8: 8 MHz bandwidth
* @RATE_INFO_BW_16: 16 MHz bandwidth
*/
enum rate_info_bw {
RATE_INFO_BW_20 = 0,
RATE_INFO_BW_5,
RATE_INFO_BW_10,
RATE_INFO_BW_40,
RATE_INFO_BW_80,
RATE_INFO_BW_160,
RATE_INFO_BW_HE_RU,
RATE_INFO_BW_320,
RATE_INFO_BW_EHT_RU,
RATE_INFO_BW_1,
RATE_INFO_BW_2,
RATE_INFO_BW_4,
RATE_INFO_BW_8,
RATE_INFO_BW_16,
};
/**
* struct rate_info - bitrate information
*
* Information about a receiving or transmitting bitrate
*
* @flags: bitflag of flags from &enum rate_info_flags
* @legacy: bitrate in 100kbit/s for 802.11abg
* @mcs: mcs index if struct describes an HT/VHT/HE/EHT/S1G rate
* @nss: number of streams (VHT & HE only)
* @bw: bandwidth (from &enum rate_info_bw)
* @he_gi: HE guard interval (from &enum nl80211_he_gi)
* @he_dcm: HE DCM value
* @he_ru_alloc: HE RU allocation (from &enum nl80211_he_ru_alloc,
* only valid if bw is %RATE_INFO_BW_HE_RU)
* @n_bonded_ch: In case of EDMG the number of bonded channels (1-4)
* @eht_gi: EHT guard interval (from &enum nl80211_eht_gi)
* @eht_ru_alloc: EHT RU allocation (from &enum nl80211_eht_ru_alloc,
* only valid if bw is %RATE_INFO_BW_EHT_RU)
*/
struct rate_info {
u16 flags;
u16 legacy;
u8 mcs;
u8 nss;
u8 bw;
u8 he_gi;
u8 he_dcm;
u8 he_ru_alloc;
u8 n_bonded_ch;
u8 eht_gi;
u8 eht_ru_alloc;
};
/**
* enum bss_param_flags - bitrate info flags
*
* Used by the driver to indicate the specific rate transmission
* type for 802.11n transmissions.
*
* @BSS_PARAM_FLAGS_CTS_PROT: whether CTS protection is enabled
* @BSS_PARAM_FLAGS_SHORT_PREAMBLE: whether short preamble is enabled
* @BSS_PARAM_FLAGS_SHORT_SLOT_TIME: whether short slot time is enabled
*/
enum bss_param_flags {
BSS_PARAM_FLAGS_CTS_PROT = 1<<0,
BSS_PARAM_FLAGS_SHORT_PREAMBLE = 1<<1,
BSS_PARAM_FLAGS_SHORT_SLOT_TIME = 1<<2,
};
/**
* struct sta_bss_parameters - BSS parameters for the attached station
*
* Information about the currently associated BSS
*
* @flags: bitflag of flags from &enum bss_param_flags
* @dtim_period: DTIM period for the BSS
* @beacon_interval: beacon interval
*/
struct sta_bss_parameters {
u8 flags;
u8 dtim_period;
u16 beacon_interval;
};
/**
* struct cfg80211_txq_stats - TXQ statistics for this TID
* @filled: bitmap of flags using the bits of &enum nl80211_txq_stats to
* indicate the relevant values in this struct are filled
* @backlog_bytes: total number of bytes currently backlogged
* @backlog_packets: total number of packets currently backlogged
* @flows: number of new flows seen
* @drops: total number of packets dropped
* @ecn_marks: total number of packets marked with ECN CE
* @overlimit: number of drops due to queue space overflow
* @overmemory: number of drops due to memory limit overflow
* @collisions: number of hash collisions
* @tx_bytes: total number of bytes dequeued
* @tx_packets: total number of packets dequeued
* @max_flows: maximum number of flows supported
*/
struct cfg80211_txq_stats {
u32 filled;
u32 backlog_bytes;
u32 backlog_packets;
u32 flows;
u32 drops;
u32 ecn_marks;
u32 overlimit;
u32 overmemory;
u32 collisions;
u32 tx_bytes;
u32 tx_packets;
u32 max_flows;
};
/**
* struct cfg80211_tid_stats - per-TID statistics
* @filled: bitmap of flags using the bits of &enum nl80211_tid_stats to
* indicate the relevant values in this struct are filled
* @rx_msdu: number of received MSDUs
* @tx_msdu: number of (attempted) transmitted MSDUs
* @tx_msdu_retries: number of retries (not counting the first) for
* transmitted MSDUs
* @tx_msdu_failed: number of failed transmitted MSDUs
* @txq_stats: TXQ statistics
*/
struct cfg80211_tid_stats {
u32 filled;
u64 rx_msdu;
u64 tx_msdu;
u64 tx_msdu_retries;
u64 tx_msdu_failed;
struct cfg80211_txq_stats txq_stats;
};
#define IEEE80211_MAX_CHAINS 4
/**
* struct station_info - station information
*
* Station information filled by driver for get_station() and dump_station.
*
* @filled: bitflag of flags using the bits of &enum nl80211_sta_info to
* indicate the relevant values in this struct for them
* @connected_time: time(in secs) since a station is last connected
* @inactive_time: time since last station activity (tx/rx) in milliseconds
* @assoc_at: bootime (ns) of the last association
* @rx_bytes: bytes (size of MPDUs) received from this station
* @tx_bytes: bytes (size of MPDUs) transmitted to this station
* @llid: mesh local link id
* @plid: mesh peer link id
* @plink_state: mesh peer link state
* @signal: The signal strength, type depends on the wiphy's signal_type.
* For CFG80211_SIGNAL_TYPE_MBM, value is expressed in _dBm_.
* @signal_avg: Average signal strength, type depends on the wiphy's signal_type.
* For CFG80211_SIGNAL_TYPE_MBM, value is expressed in _dBm_.
* @chains: bitmask for filled values in @chain_signal, @chain_signal_avg
* @chain_signal: per-chain signal strength of last received packet in dBm
* @chain_signal_avg: per-chain signal strength average in dBm
* @txrate: current unicast bitrate from this station
* @rxrate: current unicast bitrate to this station
* @rx_packets: packets (MSDUs & MMPDUs) received from this station
* @tx_packets: packets (MSDUs & MMPDUs) transmitted to this station
* @tx_retries: cumulative retry counts (MPDUs)
* @tx_failed: number of failed transmissions (MPDUs) (retries exceeded, no ACK)
* @rx_dropped_misc: Dropped for un-specified reason.
* @bss_param: current BSS parameters
* @generation: generation number for nl80211 dumps.
* This number should increase every time the list of stations
* changes, i.e. when a station is added or removed, so that
* userspace can tell whether it got a consistent snapshot.
* @assoc_req_ies: IEs from (Re)Association Request.
* This is used only when in AP mode with drivers that do not use
* user space MLME/SME implementation. The information is provided for
* the cfg80211_new_sta() calls to notify user space of the IEs.
* @assoc_req_ies_len: Length of assoc_req_ies buffer in octets.
* @sta_flags: station flags mask & values
* @beacon_loss_count: Number of times beacon loss event has triggered.
* @t_offset: Time offset of the station relative to this host.
* @local_pm: local mesh STA power save mode
* @peer_pm: peer mesh STA power save mode
* @nonpeer_pm: non-peer mesh STA power save mode
* @expected_throughput: expected throughput in kbps (including 802.11 headers)
* towards this station.
* @rx_beacon: number of beacons received from this peer
* @rx_beacon_signal_avg: signal strength average (in dBm) for beacons received
* from this peer
* @connected_to_gate: true if mesh STA has a path to mesh gate
* @rx_duration: aggregate PPDU duration(usecs) for all the frames from a peer
* @tx_duration: aggregate PPDU duration(usecs) for all the frames to a peer
* @airtime_weight: current airtime scheduling weight
* @pertid: per-TID statistics, see &struct cfg80211_tid_stats, using the last
* (IEEE80211_NUM_TIDS) index for MSDUs not encapsulated in QoS-MPDUs.
* Note that this doesn't use the @filled bit, but is used if non-NULL.
* @ack_signal: signal strength (in dBm) of the last ACK frame.
* @avg_ack_signal: average rssi value of ack packet for the no of msdu's has
* been sent.
* @rx_mpdu_count: number of MPDUs received from this station
* @fcs_err_count: number of packets (MPDUs) received from this station with
* an FCS error. This counter should be incremented only when TA of the
* received packet with an FCS error matches the peer MAC address.
* @airtime_link_metric: mesh airtime link metric.
* @connected_to_as: true if mesh STA has a path to authentication server
* @mlo_params_valid: Indicates @assoc_link_id and @mld_addr fields are filled
* by driver. Drivers use this only in cfg80211_new_sta() calls when AP
* MLD's MLME/SME is offload to driver. Drivers won't fill this
* information in cfg80211_del_sta_sinfo(), get_station() and
* dump_station() callbacks.
* @assoc_link_id: Indicates MLO link ID of the AP, with which the station
* completed (re)association. This information filled for both MLO
* and non-MLO STA connections when the AP affiliated with an MLD.
* @mld_addr: For MLO STA connection, filled with MLD address of the station.
* For non-MLO STA connection, filled with all zeros.
* @assoc_resp_ies: IEs from (Re)Association Response.
* This is used only when in AP mode with drivers that do not use user
* space MLME/SME implementation. The information is provided only for the
* cfg80211_new_sta() calls to notify user space of the IEs. Drivers won't
* fill this information in cfg80211_del_sta_sinfo(), get_station() and
* dump_station() callbacks. User space needs this information to determine
* the accepted and rejected affiliated links of the connected station.
* @assoc_resp_ies_len: Length of @assoc_resp_ies buffer in octets.
*/
struct station_info {
u64 filled;
u32 connected_time;
u32 inactive_time;
u64 assoc_at;
u64 rx_bytes;
u64 tx_bytes;
u16 llid;
u16 plid;
u8 plink_state;
s8 signal;
s8 signal_avg;
u8 chains;
s8 chain_signal[IEEE80211_MAX_CHAINS];
s8 chain_signal_avg[IEEE80211_MAX_CHAINS];
struct rate_info txrate;
struct rate_info rxrate;
u32 rx_packets;
u32 tx_packets;
u32 tx_retries;
u32 tx_failed;
u32 rx_dropped_misc;
struct sta_bss_parameters bss_param;
struct nl80211_sta_flag_update sta_flags;
int generation;
const u8 *assoc_req_ies;
size_t assoc_req_ies_len;
u32 beacon_loss_count;
s64 t_offset;
enum nl80211_mesh_power_mode local_pm;
enum nl80211_mesh_power_mode peer_pm;
enum nl80211_mesh_power_mode nonpeer_pm;
u32 expected_throughput;
u64 tx_duration;
u64 rx_duration;
u64 rx_beacon;
u8 rx_beacon_signal_avg;
u8 connected_to_gate;
struct cfg80211_tid_stats *pertid;
s8 ack_signal;
s8 avg_ack_signal;
u16 airtime_weight;
u32 rx_mpdu_count;
u32 fcs_err_count;
u32 airtime_link_metric;
u8 connected_to_as;
bool mlo_params_valid;
u8 assoc_link_id;
u8 mld_addr[ETH_ALEN] __aligned(2);
const u8 *assoc_resp_ies;
size_t assoc_resp_ies_len;
};
/**
* struct cfg80211_sar_sub_specs - sub specs limit
* @power: power limitation in 0.25dbm
* @freq_range_index: index the power limitation applies to
*/
struct cfg80211_sar_sub_specs {
s32 power;
u32 freq_range_index;
};
/**
* struct cfg80211_sar_specs - sar limit specs
* @type: it's set with power in 0.25dbm or other types
* @num_sub_specs: number of sar sub specs
* @sub_specs: memory to hold the sar sub specs
*/
struct cfg80211_sar_specs {
enum nl80211_sar_type type;
u32 num_sub_specs;
struct cfg80211_sar_sub_specs sub_specs[];
};
/**
* struct cfg80211_sar_freq_ranges - sar frequency ranges
* @start_freq: start range edge frequency
* @end_freq: end range edge frequency
*/
struct cfg80211_sar_freq_ranges {
u32 start_freq;
u32 end_freq;
};
/**
* struct cfg80211_sar_capa - sar limit capability
* @type: it's set via power in 0.25dbm or other types
* @num_freq_ranges: number of frequency ranges
* @freq_ranges: memory to hold the freq ranges.
*
* Note: WLAN driver may append new ranges or split an existing
* range to small ones and then append them.
*/
struct cfg80211_sar_capa {
enum nl80211_sar_type type;
u32 num_freq_ranges;
const struct cfg80211_sar_freq_ranges *freq_ranges;
};
#if IS_ENABLED(CONFIG_CFG80211)
/**
* cfg80211_get_station - retrieve information about a given station
* @dev: the device where the station is supposed to be connected to
* @mac_addr: the mac address of the station of interest
* @sinfo: pointer to the structure to fill with the information
*
* Returns 0 on success and sinfo is filled with the available information
* otherwise returns a negative error code and the content of sinfo has to be
* considered undefined.
*/
int cfg80211_get_station(struct net_device *dev, const u8 *mac_addr,
struct station_info *sinfo);
#else
static inline int cfg80211_get_station(struct net_device *dev,
const u8 *mac_addr,
struct station_info *sinfo)
{
return -ENOENT;
}
#endif
/**
* enum monitor_flags - monitor flags
*
* Monitor interface configuration flags. Note that these must be the bits
* according to the nl80211 flags.
*
* @MONITOR_FLAG_CHANGED: set if the flags were changed
* @MONITOR_FLAG_FCSFAIL: pass frames with bad FCS
* @MONITOR_FLAG_PLCPFAIL: pass frames with bad PLCP
* @MONITOR_FLAG_CONTROL: pass control frames
* @MONITOR_FLAG_OTHER_BSS: disable BSSID filtering
* @MONITOR_FLAG_COOK_FRAMES: report frames after processing
* @MONITOR_FLAG_ACTIVE: active monitor, ACKs frames on its MAC address
*/
enum monitor_flags {
MONITOR_FLAG_CHANGED = 1<<__NL80211_MNTR_FLAG_INVALID,
MONITOR_FLAG_FCSFAIL = 1<<NL80211_MNTR_FLAG_FCSFAIL,
MONITOR_FLAG_PLCPFAIL = 1<<NL80211_MNTR_FLAG_PLCPFAIL,
MONITOR_FLAG_CONTROL = 1<<NL80211_MNTR_FLAG_CONTROL,
MONITOR_FLAG_OTHER_BSS = 1<<NL80211_MNTR_FLAG_OTHER_BSS,
MONITOR_FLAG_COOK_FRAMES = 1<<NL80211_MNTR_FLAG_COOK_FRAMES,
MONITOR_FLAG_ACTIVE = 1<<NL80211_MNTR_FLAG_ACTIVE,
};
/**
* enum mpath_info_flags - mesh path information flags
*
* Used by the driver to indicate which info in &struct mpath_info it has filled
* in during get_station() or dump_station().
*
* @MPATH_INFO_FRAME_QLEN: @frame_qlen filled
* @MPATH_INFO_SN: @sn filled
* @MPATH_INFO_METRIC: @metric filled
* @MPATH_INFO_EXPTIME: @exptime filled
* @MPATH_INFO_DISCOVERY_TIMEOUT: @discovery_timeout filled
* @MPATH_INFO_DISCOVERY_RETRIES: @discovery_retries filled
* @MPATH_INFO_FLAGS: @flags filled
* @MPATH_INFO_HOP_COUNT: @hop_count filled
* @MPATH_INFO_PATH_CHANGE: @path_change_count filled
*/
enum mpath_info_flags {
MPATH_INFO_FRAME_QLEN = BIT(0),
MPATH_INFO_SN = BIT(1),
MPATH_INFO_METRIC = BIT(2),
MPATH_INFO_EXPTIME = BIT(3),
MPATH_INFO_DISCOVERY_TIMEOUT = BIT(4),
MPATH_INFO_DISCOVERY_RETRIES = BIT(5),
MPATH_INFO_FLAGS = BIT(6),
MPATH_INFO_HOP_COUNT = BIT(7),
MPATH_INFO_PATH_CHANGE = BIT(8),
};
/**
* struct mpath_info - mesh path information
*
* Mesh path information filled by driver for get_mpath() and dump_mpath().
*
* @filled: bitfield of flags from &enum mpath_info_flags
* @frame_qlen: number of queued frames for this destination
* @sn: target sequence number
* @metric: metric (cost) of this mesh path
* @exptime: expiration time for the mesh path from now, in msecs
* @flags: mesh path flags from &enum mesh_path_flags
* @discovery_timeout: total mesh path discovery timeout, in msecs
* @discovery_retries: mesh path discovery retries
* @generation: generation number for nl80211 dumps.
* This number should increase every time the list of mesh paths
* changes, i.e. when a station is added or removed, so that
* userspace can tell whether it got a consistent snapshot.
* @hop_count: hops to destination
* @path_change_count: total number of path changes to destination
*/
struct mpath_info {
u32 filled;
u32 frame_qlen;
u32 sn;
u32 metric;
u32 exptime;
u32 discovery_timeout;
u8 discovery_retries;
u8 flags;
u8 hop_count;
u32 path_change_count;
int generation;
};
/**
* struct bss_parameters - BSS parameters
*
* Used to change BSS parameters (mainly for AP mode).
*
* @link_id: link_id or -1 for non-MLD
* @use_cts_prot: Whether to use CTS protection
* (0 = no, 1 = yes, -1 = do not change)
* @use_short_preamble: Whether the use of short preambles is allowed
* (0 = no, 1 = yes, -1 = do not change)
* @use_short_slot_time: Whether the use of short slot time is allowed
* (0 = no, 1 = yes, -1 = do not change)
* @basic_rates: basic rates in IEEE 802.11 format
* (or NULL for no change)
* @basic_rates_len: number of basic rates
* @ap_isolate: do not forward packets between connected stations
* (0 = no, 1 = yes, -1 = do not change)
* @ht_opmode: HT Operation mode
* (u16 = opmode, -1 = do not change)
* @p2p_ctwindow: P2P CT Window (-1 = no change)
* @p2p_opp_ps: P2P opportunistic PS (-1 = no change)
*/
struct bss_parameters {
int link_id;
int use_cts_prot;
int use_short_preamble;
int use_short_slot_time;
const u8 *basic_rates;
u8 basic_rates_len;
int ap_isolate;
int ht_opmode;
s8 p2p_ctwindow, p2p_opp_ps;
};
/**
* struct mesh_config - 802.11s mesh configuration
*
* These parameters can be changed while the mesh is active.
*
* @dot11MeshRetryTimeout: the initial retry timeout in millisecond units used
* by the Mesh Peering Open message
* @dot11MeshConfirmTimeout: the initial retry timeout in millisecond units
* used by the Mesh Peering Open message
* @dot11MeshHoldingTimeout: the confirm timeout in millisecond units used by
* the mesh peering management to close a mesh peering
* @dot11MeshMaxPeerLinks: the maximum number of peer links allowed on this
* mesh interface
* @dot11MeshMaxRetries: the maximum number of peer link open retries that can
* be sent to establish a new peer link instance in a mesh
* @dot11MeshTTL: the value of TTL field set at a source mesh STA
* @element_ttl: the value of TTL field set at a mesh STA for path selection
* elements
* @auto_open_plinks: whether we should automatically open peer links when we
* detect compatible mesh peers
* @dot11MeshNbrOffsetMaxNeighbor: the maximum number of neighbors to
* synchronize to for 11s default synchronization method
* @dot11MeshHWMPmaxPREQretries: the number of action frames containing a PREQ
* that an originator mesh STA can send to a particular path target
* @path_refresh_time: how frequently to refresh mesh paths in milliseconds
* @min_discovery_timeout: the minimum length of time to wait until giving up on
* a path discovery in milliseconds
* @dot11MeshHWMPactivePathTimeout: the time (in TUs) for which mesh STAs
* receiving a PREQ shall consider the forwarding information from the
* root to be valid. (TU = time unit)
* @dot11MeshHWMPpreqMinInterval: the minimum interval of time (in TUs) during
* which a mesh STA can send only one action frame containing a PREQ
* element
* @dot11MeshHWMPperrMinInterval: the minimum interval of time (in TUs) during
* which a mesh STA can send only one Action frame containing a PERR
* element
* @dot11MeshHWMPnetDiameterTraversalTime: the interval of time (in TUs) that
* it takes for an HWMP information element to propagate across the mesh
* @dot11MeshHWMPRootMode: the configuration of a mesh STA as root mesh STA
* @dot11MeshHWMPRannInterval: the interval of time (in TUs) between root
* announcements are transmitted
* @dot11MeshGateAnnouncementProtocol: whether to advertise that this mesh
* station has access to a broader network beyond the MBSS. (This is
* missnamed in draft 12.0: dot11MeshGateAnnouncementProtocol set to true
* only means that the station will announce others it's a mesh gate, but
* not necessarily using the gate announcement protocol. Still keeping the
* same nomenclature to be in sync with the spec)
* @dot11MeshForwarding: whether the Mesh STA is forwarding or non-forwarding
* entity (default is TRUE - forwarding entity)
* @rssi_threshold: the threshold for average signal strength of candidate
* station to establish a peer link
* @ht_opmode: mesh HT protection mode
*
* @dot11MeshHWMPactivePathToRootTimeout: The time (in TUs) for which mesh STAs
* receiving a proactive PREQ shall consider the forwarding information to
* the root mesh STA to be valid.
*
* @dot11MeshHWMProotInterval: The interval of time (in TUs) between proactive
* PREQs are transmitted.
* @dot11MeshHWMPconfirmationInterval: The minimum interval of time (in TUs)
* during which a mesh STA can send only one Action frame containing
* a PREQ element for root path confirmation.
* @power_mode: The default mesh power save mode which will be the initial
* setting for new peer links.
* @dot11MeshAwakeWindowDuration: The duration in TUs the STA will remain awake
* after transmitting its beacon.
* @plink_timeout: If no tx activity is seen from a STA we've established
* peering with for longer than this time (in seconds), then remove it
* from the STA's list of peers. Default is 30 minutes.
* @dot11MeshConnectedToAuthServer: if set to true then this mesh STA
* will advertise that it is connected to a authentication server
* in the mesh formation field.
* @dot11MeshConnectedToMeshGate: if set to true, advertise that this STA is
* connected to a mesh gate in mesh formation info. If false, the
* value in mesh formation is determined by the presence of root paths
* in the mesh path table
* @dot11MeshNolearn: Try to avoid multi-hop path discovery (e.g. PREQ/PREP
* for HWMP) if the destination is a direct neighbor. Note that this might
* not be the optimal decision as a multi-hop route might be better. So
* if using this setting you will likely also want to disable
* dot11MeshForwarding and use another mesh routing protocol on top.
*/
struct mesh_config {
u16 dot11MeshRetryTimeout;
u16 dot11MeshConfirmTimeout;
u16 dot11MeshHoldingTimeout;
u16 dot11MeshMaxPeerLinks;
u8 dot11MeshMaxRetries;
u8 dot11MeshTTL;
u8 element_ttl;
bool auto_open_plinks;
u32 dot11MeshNbrOffsetMaxNeighbor;
u8 dot11MeshHWMPmaxPREQretries;
u32 path_refresh_time;
u16 min_discovery_timeout;
u32 dot11MeshHWMPactivePathTimeout;
u16 dot11MeshHWMPpreqMinInterval;
u16 dot11MeshHWMPperrMinInterval;
u16 dot11MeshHWMPnetDiameterTraversalTime;
u8 dot11MeshHWMPRootMode;
bool dot11MeshConnectedToMeshGate;
bool dot11MeshConnectedToAuthServer;
u16 dot11MeshHWMPRannInterval;
bool dot11MeshGateAnnouncementProtocol;
bool dot11MeshForwarding;
s32 rssi_threshold;
u16 ht_opmode;
u32 dot11MeshHWMPactivePathToRootTimeout;
u16 dot11MeshHWMProotInterval;
u16 dot11MeshHWMPconfirmationInterval;
enum nl80211_mesh_power_mode power_mode;
u16 dot11MeshAwakeWindowDuration;
u32 plink_timeout;
bool dot11MeshNolearn;
};
/**
* struct mesh_setup - 802.11s mesh setup configuration
* @chandef: defines the channel to use
* @mesh_id: the mesh ID
* @mesh_id_len: length of the mesh ID, at least 1 and at most 32 bytes
* @sync_method: which synchronization method to use
* @path_sel_proto: which path selection protocol to use
* @path_metric: which metric to use
* @auth_id: which authentication method this mesh is using
* @ie: vendor information elements (optional)
* @ie_len: length of vendor information elements
* @is_authenticated: this mesh requires authentication
* @is_secure: this mesh uses security
* @user_mpm: userspace handles all MPM functions
* @dtim_period: DTIM period to use
* @beacon_interval: beacon interval to use
* @mcast_rate: multicast rate for Mesh Node [6Mbps is the default for 802.11a]
* @basic_rates: basic rates to use when creating the mesh
* @beacon_rate: bitrate to be used for beacons
* @userspace_handles_dfs: whether user space controls DFS operation, i.e.
* changes the channel when a radar is detected. This is required
* to operate on DFS channels.
* @control_port_over_nl80211: TRUE if userspace expects to exchange control
* port frames over NL80211 instead of the network interface.
*
* These parameters are fixed when the mesh is created.
*/
struct mesh_setup {
struct cfg80211_chan_def chandef;
const u8 *mesh_id;
u8 mesh_id_len;
u8 sync_method;
u8 path_sel_proto;
u8 path_metric;
u8 auth_id;
const u8 *ie;
u8 ie_len;
bool is_authenticated;
bool is_secure;
bool user_mpm;
u8 dtim_period;
u16 beacon_interval;
int mcast_rate[NUM_NL80211_BANDS];
u32 basic_rates;
struct cfg80211_bitrate_mask beacon_rate;
bool userspace_handles_dfs;
bool control_port_over_nl80211;
};
/**
* struct ocb_setup - 802.11p OCB mode setup configuration
* @chandef: defines the channel to use
*
* These parameters are fixed when connecting to the network
*/
struct ocb_setup {
struct cfg80211_chan_def chandef;
};
/**
* struct ieee80211_txq_params - TX queue parameters
* @ac: AC identifier
* @txop: Maximum burst time in units of 32 usecs, 0 meaning disabled
* @cwmin: Minimum contention window [a value of the form 2^n-1 in the range
* 1..32767]
* @cwmax: Maximum contention window [a value of the form 2^n-1 in the range
* 1..32767]
* @aifs: Arbitration interframe space [0..255]
* @link_id: link_id or -1 for non-MLD
*/
struct ieee80211_txq_params {
enum nl80211_ac ac;
u16 txop;
u16 cwmin;
u16 cwmax;
u8 aifs;
int link_id;
};
/**
* DOC: Scanning and BSS list handling
*
* The scanning process itself is fairly simple, but cfg80211 offers quite
* a bit of helper functionality. To start a scan, the scan operation will
* be invoked with a scan definition. This scan definition contains the
* channels to scan, and the SSIDs to send probe requests for (including the
* wildcard, if desired). A passive scan is indicated by having no SSIDs to
* probe. Additionally, a scan request may contain extra information elements
* that should be added to the probe request. The IEs are guaranteed to be
* well-formed, and will not exceed the maximum length the driver advertised
* in the wiphy structure.
*
* When scanning finds a BSS, cfg80211 needs to be notified of that, because
* it is responsible for maintaining the BSS list; the driver should not
* maintain a list itself. For this notification, various functions exist.
*
* Since drivers do not maintain a BSS list, there are also a number of
* functions to search for a BSS and obtain information about it from the
* BSS structure cfg80211 maintains. The BSS list is also made available
* to userspace.
*/
/**
* struct cfg80211_ssid - SSID description
* @ssid: the SSID
* @ssid_len: length of the ssid
*/
struct cfg80211_ssid {
u8 ssid[IEEE80211_MAX_SSID_LEN];
u8 ssid_len;
};
/**
* struct cfg80211_scan_info - information about completed scan
* @scan_start_tsf: scan start time in terms of the TSF of the BSS that the
* wireless device that requested the scan is connected to. If this
* information is not available, this field is left zero.
* @tsf_bssid: the BSSID according to which %scan_start_tsf is set.
* @aborted: set to true if the scan was aborted for any reason,
* userspace will be notified of that
*/
struct cfg80211_scan_info {
u64 scan_start_tsf;
u8 tsf_bssid[ETH_ALEN] __aligned(2);
bool aborted;
};
/**
* struct cfg80211_scan_6ghz_params - relevant for 6 GHz only
*
* @short_ssid: short ssid to scan for
* @bssid: bssid to scan for
* @channel_idx: idx of the channel in the channel array in the scan request
* which the above info is relevant to
* @unsolicited_probe: the AP transmits unsolicited probe response every 20 TU
* @short_ssid_valid: @short_ssid is valid and can be used
* @psc_no_listen: when set, and the channel is a PSC channel, no need to wait
* 20 TUs before starting to send probe requests.
* @psd_20: The AP's 20 MHz PSD value.
*/
struct cfg80211_scan_6ghz_params {
u32 short_ssid;
u32 channel_idx;
u8 bssid[ETH_ALEN];
bool unsolicited_probe;
bool short_ssid_valid;
bool psc_no_listen;
s8 psd_20;
};
/**
* struct cfg80211_scan_request - scan request description
*
* @ssids: SSIDs to scan for (active scan only)
* @n_ssids: number of SSIDs
* @channels: channels to scan on.
* @n_channels: total number of channels to scan
* @ie: optional information element(s) to add into Probe Request or %NULL
* @ie_len: length of ie in octets
* @duration: how long to listen on each channel, in TUs. If
* %duration_mandatory is not set, this is the maximum dwell time and
* the actual dwell time may be shorter.
* @duration_mandatory: if set, the scan duration must be as specified by the
* %duration field.
* @flags: control flags from &enum nl80211_scan_flags
* @rates: bitmap of rates to advertise for each band
* @wiphy: the wiphy this was for
* @scan_start: time (in jiffies) when the scan started
* @wdev: the wireless device to scan for
* @info: (internal) information about completed scan
* @notified: (internal) scan request was notified as done or aborted
* @no_cck: used to send probe requests at non CCK rate in 2GHz band
* @mac_addr: MAC address used with randomisation
* @mac_addr_mask: MAC address mask used with randomisation, bits that
* are 0 in the mask should be randomised, bits that are 1 should
* be taken from the @mac_addr
* @scan_6ghz: relevant for split scan request only,
* true if this is the second scan request
* @n_6ghz_params: number of 6 GHz params
* @scan_6ghz_params: 6 GHz params
* @bssid: BSSID to scan for (most commonly, the wildcard BSSID)
* @tsf_report_link_id: for MLO, indicates the link ID of the BSS that should be
* used for TSF reporting. Can be set to -1 to indicate no preference.
*/
struct cfg80211_scan_request {
struct cfg80211_ssid *ssids;
int n_ssids;
u32 n_channels;
const u8 *ie;
size_t ie_len;
u16 duration;
bool duration_mandatory;
u32 flags;
u32 rates[NUM_NL80211_BANDS];
struct wireless_dev *wdev;
u8 mac_addr[ETH_ALEN] __aligned(2);
u8 mac_addr_mask[ETH_ALEN] __aligned(2);
u8 bssid[ETH_ALEN] __aligned(2);
/* internal */
struct wiphy *wiphy;
unsigned long scan_start;
struct cfg80211_scan_info info;
bool notified;
bool no_cck;
bool scan_6ghz;
u32 n_6ghz_params;
struct cfg80211_scan_6ghz_params *scan_6ghz_params;
s8 tsf_report_link_id;
/* keep last */
struct ieee80211_channel *channels[] __counted_by(n_channels);
};
static inline void get_random_mask_addr(u8 *buf, const u8 *addr, const u8 *mask)
{
int i;
get_random_bytes(buf, ETH_ALEN);
for (i = 0; i < ETH_ALEN; i++) {
buf[i] &= ~mask[i];
buf[i] |= addr[i] & mask[i];
}
}
/**
* struct cfg80211_match_set - sets of attributes to match
*
* @ssid: SSID to be matched; may be zero-length in case of BSSID match
* or no match (RSSI only)
* @bssid: BSSID to be matched; may be all-zero BSSID in case of SSID match
* or no match (RSSI only)
* @rssi_thold: don't report scan results below this threshold (in s32 dBm)
*/
struct cfg80211_match_set {
struct cfg80211_ssid ssid;
u8 bssid[ETH_ALEN];
s32 rssi_thold;
};
/**
* struct cfg80211_sched_scan_plan - scan plan for scheduled scan
*
* @interval: interval between scheduled scan iterations. In seconds.
* @iterations: number of scan iterations in this scan plan. Zero means
* infinite loop.
* The last scan plan will always have this parameter set to zero,
* all other scan plans will have a finite number of iterations.
*/
struct cfg80211_sched_scan_plan {
u32 interval;
u32 iterations;
};
/**
* struct cfg80211_bss_select_adjust - BSS selection with RSSI adjustment.
*
* @band: band of BSS which should match for RSSI level adjustment.
* @delta: value of RSSI level adjustment.
*/
struct cfg80211_bss_select_adjust {
enum nl80211_band band;
s8 delta;
};
/**
* struct cfg80211_sched_scan_request - scheduled scan request description
*
* @reqid: identifies this request.
* @ssids: SSIDs to scan for (passed in the probe_reqs in active scans)
* @n_ssids: number of SSIDs
* @n_channels: total number of channels to scan
* @ie: optional information element(s) to add into Probe Request or %NULL
* @ie_len: length of ie in octets
* @flags: control flags from &enum nl80211_scan_flags
* @match_sets: sets of parameters to be matched for a scan result
* entry to be considered valid and to be passed to the host
* (others are filtered out).
* If omitted, all results are passed.
* @n_match_sets: number of match sets
* @report_results: indicates that results were reported for this request
* @wiphy: the wiphy this was for
* @dev: the interface
* @scan_start: start time of the scheduled scan
* @channels: channels to scan
* @min_rssi_thold: for drivers only supporting a single threshold, this
* contains the minimum over all matchsets
* @mac_addr: MAC address used with randomisation
* @mac_addr_mask: MAC address mask used with randomisation, bits that
* are 0 in the mask should be randomised, bits that are 1 should
* be taken from the @mac_addr
* @scan_plans: scan plans to be executed in this scheduled scan. Lowest
* index must be executed first.
* @n_scan_plans: number of scan plans, at least 1.
* @rcu_head: RCU callback used to free the struct
* @owner_nlportid: netlink portid of owner (if this should is a request
* owned by a particular socket)
* @nl_owner_dead: netlink owner socket was closed - this request be freed
* @list: for keeping list of requests.
* @delay: delay in seconds to use before starting the first scan
* cycle. The driver may ignore this parameter and start
* immediately (or at any other time), if this feature is not
* supported.
* @relative_rssi_set: Indicates whether @relative_rssi is set or not.
* @relative_rssi: Relative RSSI threshold in dB to restrict scan result
* reporting in connected state to cases where a matching BSS is determined
* to have better or slightly worse RSSI than the current connected BSS.
* The relative RSSI threshold values are ignored in disconnected state.
* @rssi_adjust: delta dB of RSSI preference to be given to the BSSs that belong
* to the specified band while deciding whether a better BSS is reported
* using @relative_rssi. If delta is a negative number, the BSSs that
* belong to the specified band will be penalized by delta dB in relative
* comparisons.
*/
struct cfg80211_sched_scan_request {
u64 reqid;
struct cfg80211_ssid *ssids;
int n_ssids;
u32 n_channels;
const u8 *ie;
size_t ie_len;
u32 flags;
struct cfg80211_match_set *match_sets;
int n_match_sets;
s32 min_rssi_thold;
u32 delay;
struct cfg80211_sched_scan_plan *scan_plans;
int n_scan_plans;
u8 mac_addr[ETH_ALEN] __aligned(2);
u8 mac_addr_mask[ETH_ALEN] __aligned(2);
bool relative_rssi_set;
s8 relative_rssi;
struct cfg80211_bss_select_adjust rssi_adjust;
/* internal */
struct wiphy *wiphy;
struct net_device *dev;
unsigned long scan_start;
bool report_results;
struct rcu_head rcu_head;
u32 owner_nlportid;
bool nl_owner_dead;
struct list_head list;
/* keep last */
struct ieee80211_channel *channels[];
};
/**
* enum cfg80211_signal_type - signal type
*
* @CFG80211_SIGNAL_TYPE_NONE: no signal strength information available
* @CFG80211_SIGNAL_TYPE_MBM: signal strength in mBm (100*dBm)
* @CFG80211_SIGNAL_TYPE_UNSPEC: signal strength, increasing from 0 through 100
*/
enum cfg80211_signal_type {
CFG80211_SIGNAL_TYPE_NONE,
CFG80211_SIGNAL_TYPE_MBM,
CFG80211_SIGNAL_TYPE_UNSPEC,
};
/**
* struct cfg80211_inform_bss - BSS inform data
* @chan: channel the frame was received on
* @signal: signal strength value, according to the wiphy's
* signal type
* @boottime_ns: timestamp (CLOCK_BOOTTIME) when the information was
* received; should match the time when the frame was actually
* received by the device (not just by the host, in case it was
* buffered on the device) and be accurate to about 10ms.
* If the frame isn't buffered, just passing the return value of
* ktime_get_boottime_ns() is likely appropriate.
* @parent_tsf: the time at the start of reception of the first octet of the
* timestamp field of the frame. The time is the TSF of the BSS specified
* by %parent_bssid.
* @parent_bssid: the BSS according to which %parent_tsf is set. This is set to
* the BSS that requested the scan in which the beacon/probe was received.
* @chains: bitmask for filled values in @chain_signal.
* @chain_signal: per-chain signal strength of last received BSS in dBm.
* @restrict_use: restrict usage, if not set, assume @use_for is
* %NL80211_BSS_USE_FOR_NORMAL.
* @use_for: bitmap of possible usage for this BSS, see
* &enum nl80211_bss_use_for
* @cannot_use_reasons: the reasons (bitmap) for not being able to connect,
* if @restrict_use is set and @use_for is zero (empty); may be 0 for
* unspecified reasons; see &enum nl80211_bss_cannot_use_reasons
* @drv_data: Data to be passed through to @inform_bss
*/
struct cfg80211_inform_bss {
struct ieee80211_channel *chan;
s32 signal;
u64 boottime_ns;
u64 parent_tsf;
u8 parent_bssid[ETH_ALEN] __aligned(2);
u8 chains;
s8 chain_signal[IEEE80211_MAX_CHAINS];
u8 restrict_use:1, use_for:7;
u8 cannot_use_reasons;
void *drv_data;
};
/**
* struct cfg80211_bss_ies - BSS entry IE data
* @tsf: TSF contained in the frame that carried these IEs
* @rcu_head: internal use, for freeing
* @len: length of the IEs
* @from_beacon: these IEs are known to come from a beacon
* @data: IE data
*/
struct cfg80211_bss_ies {
u64 tsf;
struct rcu_head rcu_head;
int len;
bool from_beacon;
u8 data[];
};
/**
* struct cfg80211_bss - BSS description
*
* This structure describes a BSS (which may also be a mesh network)
* for use in scan results and similar.
*
* @channel: channel this BSS is on
* @bssid: BSSID of the BSS
* @beacon_interval: the beacon interval as from the frame
* @capability: the capability field in host byte order
* @ies: the information elements (Note that there is no guarantee that these
* are well-formed!); this is a pointer to either the beacon_ies or
* proberesp_ies depending on whether Probe Response frame has been
* received. It is always non-%NULL.
* @beacon_ies: the information elements from the last Beacon frame
* (implementation note: if @hidden_beacon_bss is set this struct doesn't
* own the beacon_ies, but they're just pointers to the ones from the
* @hidden_beacon_bss struct)
* @proberesp_ies: the information elements from the last Probe Response frame
* @proberesp_ecsa_stuck: ECSA element is stuck in the Probe Response frame,
* cannot rely on it having valid data
* @hidden_beacon_bss: in case this BSS struct represents a probe response from
* a BSS that hides the SSID in its beacon, this points to the BSS struct
* that holds the beacon data. @beacon_ies is still valid, of course, and
* points to the same data as hidden_beacon_bss->beacon_ies in that case.
* @transmitted_bss: pointer to the transmitted BSS, if this is a
* non-transmitted one (multi-BSSID support)
* @nontrans_list: list of non-transmitted BSS, if this is a transmitted one
* (multi-BSSID support)
* @signal: signal strength value (type depends on the wiphy's signal_type)
* @chains: bitmask for filled values in @chain_signal.
* @chain_signal: per-chain signal strength of last received BSS in dBm.
* @bssid_index: index in the multiple BSS set
* @max_bssid_indicator: max number of members in the BSS set
* @use_for: bitmap of possible usage for this BSS, see
* &enum nl80211_bss_use_for
* @cannot_use_reasons: the reasons (bitmap) for not being able to connect,
* if @restrict_use is set and @use_for is zero (empty); may be 0 for
* unspecified reasons; see &enum nl80211_bss_cannot_use_reasons
* @priv: private area for driver use, has at least wiphy->bss_priv_size bytes
*/
struct cfg80211_bss {
struct ieee80211_channel *channel;
const struct cfg80211_bss_ies __rcu *ies;
const struct cfg80211_bss_ies __rcu *beacon_ies;
const struct cfg80211_bss_ies __rcu *proberesp_ies;
struct cfg80211_bss *hidden_beacon_bss;
struct cfg80211_bss *transmitted_bss;
struct list_head nontrans_list;
s32 signal;
u16 beacon_interval;
u16 capability;
u8 bssid[ETH_ALEN];
u8 chains;
s8 chain_signal[IEEE80211_MAX_CHAINS];
u8 proberesp_ecsa_stuck:1;
u8 bssid_index;
u8 max_bssid_indicator;
u8 use_for;
u8 cannot_use_reasons;
u8 priv[] __aligned(sizeof(void *));
};
/**
* ieee80211_bss_get_elem - find element with given ID
* @bss: the bss to search
* @id: the element ID
*
* Note that the return value is an RCU-protected pointer, so
* rcu_read_lock() must be held when calling this function.
* Return: %NULL if not found.
*/
const struct element *ieee80211_bss_get_elem(struct cfg80211_bss *bss, u8 id);
/**
* ieee80211_bss_get_ie - find IE with given ID
* @bss: the bss to search
* @id: the element ID
*
* Note that the return value is an RCU-protected pointer, so
* rcu_read_lock() must be held when calling this function.
* Return: %NULL if not found.
*/
static inline const u8 *ieee80211_bss_get_ie(struct cfg80211_bss *bss, u8 id)
{
return (const void *)ieee80211_bss_get_elem(bss, id);
}
/**
* struct cfg80211_auth_request - Authentication request data
*
* This structure provides information needed to complete IEEE 802.11
* authentication.
*
* @bss: The BSS to authenticate with, the callee must obtain a reference
* to it if it needs to keep it.
* @auth_type: Authentication type (algorithm)
* @ie: Extra IEs to add to Authentication frame or %NULL
* @ie_len: Length of ie buffer in octets
* @key_len: length of WEP key for shared key authentication
* @key_idx: index of WEP key for shared key authentication
* @key: WEP key for shared key authentication
* @auth_data: Fields and elements in Authentication frames. This contains
* the authentication frame body (non-IE and IE data), excluding the
* Authentication algorithm number, i.e., starting at the Authentication
* transaction sequence number field.
* @auth_data_len: Length of auth_data buffer in octets
* @link_id: if >= 0, indicates authentication should be done as an MLD,
* the interface address is included as the MLD address and the
* necessary link (with the given link_id) will be created (and
* given an MLD address) by the driver
* @ap_mld_addr: AP MLD address in case of authentication request with
* an AP MLD, valid iff @link_id >= 0
*/
struct cfg80211_auth_request {
struct cfg80211_bss *bss;
const u8 *ie;
size_t ie_len;
enum nl80211_auth_type auth_type;
const u8 *key;
u8 key_len;
s8 key_idx;
const u8 *auth_data;
size_t auth_data_len;
s8 link_id;
const u8 *ap_mld_addr;
};
/**
* struct cfg80211_assoc_link - per-link information for MLO association
* @bss: the BSS pointer, see also &struct cfg80211_assoc_request::bss;
* if this is %NULL for a link, that link is not requested
* @elems: extra elements for the per-STA profile for this link
* @elems_len: length of the elements
* @disabled: If set this link should be included during association etc. but it
* should not be used until enabled by the AP MLD.
* @error: per-link error code, must be <= 0. If there is an error, then the
* operation as a whole must fail.
*/
struct cfg80211_assoc_link {
struct cfg80211_bss *bss;
const u8 *elems;
size_t elems_len;
bool disabled;
int error;
};
/**
* enum cfg80211_assoc_req_flags - Over-ride default behaviour in association.
*
* @ASSOC_REQ_DISABLE_HT: Disable HT (802.11n)
* @ASSOC_REQ_DISABLE_VHT: Disable VHT
* @ASSOC_REQ_USE_RRM: Declare RRM capability in this association
* @CONNECT_REQ_EXTERNAL_AUTH_SUPPORT: User space indicates external
* authentication capability. Drivers can offload authentication to
* userspace if this flag is set. Only applicable for cfg80211_connect()
* request (connect callback).
* @ASSOC_REQ_DISABLE_HE: Disable HE
* @ASSOC_REQ_DISABLE_EHT: Disable EHT
* @CONNECT_REQ_MLO_SUPPORT: Userspace indicates support for handling MLD links.
* Drivers shall disable MLO features for the current association if this
* flag is not set.
* @ASSOC_REQ_SPP_AMSDU: SPP A-MSDUs will be used on this connection (if any)
*/
enum cfg80211_assoc_req_flags {
ASSOC_REQ_DISABLE_HT = BIT(0),
ASSOC_REQ_DISABLE_VHT = BIT(1),
ASSOC_REQ_USE_RRM = BIT(2),
CONNECT_REQ_EXTERNAL_AUTH_SUPPORT = BIT(3),
ASSOC_REQ_DISABLE_HE = BIT(4),
ASSOC_REQ_DISABLE_EHT = BIT(5),
CONNECT_REQ_MLO_SUPPORT = BIT(6),
ASSOC_REQ_SPP_AMSDU = BIT(7),
};
/**
* struct cfg80211_assoc_request - (Re)Association request data
*
* This structure provides information needed to complete IEEE 802.11
* (re)association.
* @bss: The BSS to associate with. If the call is successful the driver is
* given a reference that it must give back to cfg80211_send_rx_assoc()
* or to cfg80211_assoc_timeout(). To ensure proper refcounting, new
* association requests while already associating must be rejected.
* This also applies to the @links.bss parameter, which is used instead
* of this one (it is %NULL) for MLO associations.
* @ie: Extra IEs to add to (Re)Association Request frame or %NULL
* @ie_len: Length of ie buffer in octets
* @use_mfp: Use management frame protection (IEEE 802.11w) in this association
* @crypto: crypto settings
* @prev_bssid: previous BSSID, if not %NULL use reassociate frame. This is used
* to indicate a request to reassociate within the ESS instead of a request
* do the initial association with the ESS. When included, this is set to
* the BSSID of the current association, i.e., to the value that is
* included in the Current AP address field of the Reassociation Request
* frame.
* @flags: See &enum cfg80211_assoc_req_flags
* @ht_capa: HT Capabilities over-rides. Values set in ht_capa_mask
* will be used in ht_capa. Un-supported values will be ignored.
* @ht_capa_mask: The bits of ht_capa which are to be used.
* @vht_capa: VHT capability override
* @vht_capa_mask: VHT capability mask indicating which fields to use
* @fils_kek: FILS KEK for protecting (Re)Association Request/Response frame or
* %NULL if FILS is not used.
* @fils_kek_len: Length of fils_kek in octets
* @fils_nonces: FILS nonces (part of AAD) for protecting (Re)Association
* Request/Response frame or %NULL if FILS is not used. This field starts
* with 16 octets of STA Nonce followed by 16 octets of AP Nonce.
* @s1g_capa: S1G capability override
* @s1g_capa_mask: S1G capability override mask
* @links: per-link information for MLO connections
* @link_id: >= 0 for MLO connections, where links are given, and indicates
* the link on which the association request should be sent
* @ap_mld_addr: AP MLD address in case of MLO association request,
* valid iff @link_id >= 0
*/
struct cfg80211_assoc_request {
struct cfg80211_bss *bss;
const u8 *ie, *prev_bssid;
size_t ie_len;
struct cfg80211_crypto_settings crypto;
bool use_mfp;
u32 flags;
struct ieee80211_ht_cap ht_capa;
struct ieee80211_ht_cap ht_capa_mask;
struct ieee80211_vht_cap vht_capa, vht_capa_mask;
const u8 *fils_kek;
size_t fils_kek_len;
const u8 *fils_nonces;
struct ieee80211_s1g_cap s1g_capa, s1g_capa_mask;
struct cfg80211_assoc_link links[IEEE80211_MLD_MAX_NUM_LINKS];
const u8 *ap_mld_addr;
s8 link_id;
};
/**
* struct cfg80211_deauth_request - Deauthentication request data
*
* This structure provides information needed to complete IEEE 802.11
* deauthentication.
*
* @bssid: the BSSID or AP MLD address to deauthenticate from
* @ie: Extra IEs to add to Deauthentication frame or %NULL
* @ie_len: Length of ie buffer in octets
* @reason_code: The reason code for the deauthentication
* @local_state_change: if set, change local state only and
* do not set a deauth frame
*/
struct cfg80211_deauth_request {
const u8 *bssid;
const u8 *ie;
size_t ie_len;
u16 reason_code;
bool local_state_change;
};
/**
* struct cfg80211_disassoc_request - Disassociation request data
*
* This structure provides information needed to complete IEEE 802.11
* disassociation.
*
* @ap_addr: the BSSID or AP MLD address to disassociate from
* @ie: Extra IEs to add to Disassociation frame or %NULL
* @ie_len: Length of ie buffer in octets
* @reason_code: The reason code for the disassociation
* @local_state_change: This is a request for a local state only, i.e., no
* Disassociation frame is to be transmitted.
*/
struct cfg80211_disassoc_request {
const u8 *ap_addr;
const u8 *ie;
size_t ie_len;
u16 reason_code;
bool local_state_change;
};
/**
* struct cfg80211_ibss_params - IBSS parameters
*
* This structure defines the IBSS parameters for the join_ibss()
* method.
*
* @ssid: The SSID, will always be non-null.
* @ssid_len: The length of the SSID, will always be non-zero.
* @bssid: Fixed BSSID requested, maybe be %NULL, if set do not
* search for IBSSs with a different BSSID.
* @chandef: defines the channel to use if no other IBSS to join can be found
* @channel_fixed: The channel should be fixed -- do not search for
* IBSSs to join on other channels.
* @ie: information element(s) to include in the beacon
* @ie_len: length of that
* @beacon_interval: beacon interval to use
* @privacy: this is a protected network, keys will be configured
* after joining
* @control_port: whether user space controls IEEE 802.1X port, i.e.,
* sets/clears %NL80211_STA_FLAG_AUTHORIZED. If true, the driver is
* required to assume that the port is unauthorized until authorized by
* user space. Otherwise, port is marked authorized by default.
* @control_port_over_nl80211: TRUE if userspace expects to exchange control
* port frames over NL80211 instead of the network interface.
* @userspace_handles_dfs: whether user space controls DFS operation, i.e.
* changes the channel when a radar is detected. This is required
* to operate on DFS channels.
* @basic_rates: bitmap of basic rates to use when creating the IBSS
* @mcast_rate: per-band multicast rate index + 1 (0: disabled)
* @ht_capa: HT Capabilities over-rides. Values set in ht_capa_mask
* will be used in ht_capa. Un-supported values will be ignored.
* @ht_capa_mask: The bits of ht_capa which are to be used.
* @wep_keys: static WEP keys, if not NULL points to an array of
* CFG80211_MAX_WEP_KEYS WEP keys
* @wep_tx_key: key index (0..3) of the default TX static WEP key
*/
struct cfg80211_ibss_params {
const u8 *ssid;
const u8 *bssid;
struct cfg80211_chan_def chandef;
const u8 *ie;
u8 ssid_len, ie_len;
u16 beacon_interval;
u32 basic_rates;
bool channel_fixed;
bool privacy;
bool control_port;
bool control_port_over_nl80211;
bool userspace_handles_dfs;
int mcast_rate[NUM_NL80211_BANDS];
struct ieee80211_ht_cap ht_capa;
struct ieee80211_ht_cap ht_capa_mask;
struct key_params *wep_keys;
int wep_tx_key;
};
/**
* struct cfg80211_bss_selection - connection parameters for BSS selection.
*
* @behaviour: requested BSS selection behaviour.
* @param: parameters for requestion behaviour.
* @param.band_pref: preferred band for %NL80211_BSS_SELECT_ATTR_BAND_PREF.
* @param.adjust: parameters for %NL80211_BSS_SELECT_ATTR_RSSI_ADJUST.
*/
struct cfg80211_bss_selection {
enum nl80211_bss_select_attr behaviour;
union {
enum nl80211_band band_pref;
struct cfg80211_bss_select_adjust adjust;
} param;
};
/**
* struct cfg80211_connect_params - Connection parameters
*
* This structure provides information needed to complete IEEE 802.11
* authentication and association.
*
* @channel: The channel to use or %NULL if not specified (auto-select based
* on scan results)
* @channel_hint: The channel of the recommended BSS for initial connection or
* %NULL if not specified
* @bssid: The AP BSSID or %NULL if not specified (auto-select based on scan
* results)
* @bssid_hint: The recommended AP BSSID for initial connection to the BSS or
* %NULL if not specified. Unlike the @bssid parameter, the driver is
* allowed to ignore this @bssid_hint if it has knowledge of a better BSS
* to use.
* @ssid: SSID
* @ssid_len: Length of ssid in octets
* @auth_type: Authentication type (algorithm)
* @ie: IEs for association request
* @ie_len: Length of assoc_ie in octets
* @privacy: indicates whether privacy-enabled APs should be used
* @mfp: indicate whether management frame protection is used
* @crypto: crypto settings
* @key_len: length of WEP key for shared key authentication
* @key_idx: index of WEP key for shared key authentication
* @key: WEP key for shared key authentication
* @flags: See &enum cfg80211_assoc_req_flags
* @bg_scan_period: Background scan period in seconds
* or -1 to indicate that default value is to be used.
* @ht_capa: HT Capabilities over-rides. Values set in ht_capa_mask
* will be used in ht_capa. Un-supported values will be ignored.
* @ht_capa_mask: The bits of ht_capa which are to be used.
* @vht_capa: VHT Capability overrides
* @vht_capa_mask: The bits of vht_capa which are to be used.
* @pbss: if set, connect to a PCP instead of AP. Valid for DMG
* networks.
* @bss_select: criteria to be used for BSS selection.
* @prev_bssid: previous BSSID, if not %NULL use reassociate frame. This is used
* to indicate a request to reassociate within the ESS instead of a request
* do the initial association with the ESS. When included, this is set to
* the BSSID of the current association, i.e., to the value that is
* included in the Current AP address field of the Reassociation Request
* frame.
* @fils_erp_username: EAP re-authentication protocol (ERP) username part of the
* NAI or %NULL if not specified. This is used to construct FILS wrapped
* data IE.
* @fils_erp_username_len: Length of @fils_erp_username in octets.
* @fils_erp_realm: EAP re-authentication protocol (ERP) realm part of NAI or
* %NULL if not specified. This specifies the domain name of ER server and
* is used to construct FILS wrapped data IE.
* @fils_erp_realm_len: Length of @fils_erp_realm in octets.
* @fils_erp_next_seq_num: The next sequence number to use in the FILS ERP
* messages. This is also used to construct FILS wrapped data IE.
* @fils_erp_rrk: ERP re-authentication Root Key (rRK) used to derive additional
* keys in FILS or %NULL if not specified.
* @fils_erp_rrk_len: Length of @fils_erp_rrk in octets.
* @want_1x: indicates user-space supports and wants to use 802.1X driver
* offload of 4-way handshake.
* @edmg: define the EDMG channels.
* This may specify multiple channels and bonding options for the driver
* to choose from, based on BSS configuration.
*/
struct cfg80211_connect_params {
struct ieee80211_channel *channel;
struct ieee80211_channel *channel_hint;
const u8 *bssid;
const u8 *bssid_hint;
const u8 *ssid;
size_t ssid_len;
enum nl80211_auth_type auth_type;
const u8 *ie;
size_t ie_len;
bool privacy;
enum nl80211_mfp mfp;
struct cfg80211_crypto_settings crypto;
const u8 *key;
u8 key_len, key_idx;
u32 flags;
int bg_scan_period;
struct ieee80211_ht_cap ht_capa;
struct ieee80211_ht_cap ht_capa_mask;
struct ieee80211_vht_cap vht_capa;
struct ieee80211_vht_cap vht_capa_mask;
bool pbss;
struct cfg80211_bss_selection bss_select;
const u8 *prev_bssid;
const u8 *fils_erp_username;
size_t fils_erp_username_len;
const u8 *fils_erp_realm;
size_t fils_erp_realm_len;
u16 fils_erp_next_seq_num;
const u8 *fils_erp_rrk;
size_t fils_erp_rrk_len;
bool want_1x;
struct ieee80211_edmg edmg;
};
/**
* enum cfg80211_connect_params_changed - Connection parameters being updated
*
* This enum provides information of all connect parameters that
* have to be updated as part of update_connect_params() call.
*
* @UPDATE_ASSOC_IES: Indicates whether association request IEs are updated
* @UPDATE_FILS_ERP_INFO: Indicates that FILS connection parameters (realm,
* username, erp sequence number and rrk) are updated
* @UPDATE_AUTH_TYPE: Indicates that authentication type is updated
*/
enum cfg80211_connect_params_changed {
UPDATE_ASSOC_IES = BIT(0),
UPDATE_FILS_ERP_INFO = BIT(1),
UPDATE_AUTH_TYPE = BIT(2),
};
/**
* enum wiphy_params_flags - set_wiphy_params bitfield values
* @WIPHY_PARAM_RETRY_SHORT: wiphy->retry_short has changed
* @WIPHY_PARAM_RETRY_LONG: wiphy->retry_long has changed
* @WIPHY_PARAM_FRAG_THRESHOLD: wiphy->frag_threshold has changed
* @WIPHY_PARAM_RTS_THRESHOLD: wiphy->rts_threshold has changed
* @WIPHY_PARAM_COVERAGE_CLASS: coverage class changed
* @WIPHY_PARAM_DYN_ACK: dynack has been enabled
* @WIPHY_PARAM_TXQ_LIMIT: TXQ packet limit has been changed
* @WIPHY_PARAM_TXQ_MEMORY_LIMIT: TXQ memory limit has been changed
* @WIPHY_PARAM_TXQ_QUANTUM: TXQ scheduler quantum
*/
enum wiphy_params_flags {
WIPHY_PARAM_RETRY_SHORT = 1 << 0,
WIPHY_PARAM_RETRY_LONG = 1 << 1,
WIPHY_PARAM_FRAG_THRESHOLD = 1 << 2,
WIPHY_PARAM_RTS_THRESHOLD = 1 << 3,
WIPHY_PARAM_COVERAGE_CLASS = 1 << 4,
WIPHY_PARAM_DYN_ACK = 1 << 5,
WIPHY_PARAM_TXQ_LIMIT = 1 << 6,
WIPHY_PARAM_TXQ_MEMORY_LIMIT = 1 << 7,
WIPHY_PARAM_TXQ_QUANTUM = 1 << 8,
};
#define IEEE80211_DEFAULT_AIRTIME_WEIGHT 256
/* The per TXQ device queue limit in airtime */
#define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_L 5000
#define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_H 12000
/* The per interface airtime threshold to switch to lower queue limit */
#define IEEE80211_AQL_THRESHOLD 24000
/**
* struct cfg80211_pmksa - PMK Security Association
*
* This structure is passed to the set/del_pmksa() method for PMKSA
* caching.
*
* @bssid: The AP's BSSID (may be %NULL).
* @pmkid: The identifier to refer a PMKSA.
* @pmk: The PMK for the PMKSA identified by @pmkid. This is used for key
* derivation by a FILS STA. Otherwise, %NULL.
* @pmk_len: Length of the @pmk. The length of @pmk can differ depending on
* the hash algorithm used to generate this.
* @ssid: SSID to specify the ESS within which a PMKSA is valid when using FILS
* cache identifier (may be %NULL).
* @ssid_len: Length of the @ssid in octets.
* @cache_id: 2-octet cache identifier advertized by a FILS AP identifying the
* scope of PMKSA. This is valid only if @ssid_len is non-zero (may be
* %NULL).
* @pmk_lifetime: Maximum lifetime for PMKSA in seconds
* (dot11RSNAConfigPMKLifetime) or 0 if not specified.
* The configured PMKSA must not be used for PMKSA caching after
* expiration and any keys derived from this PMK become invalid on
* expiration, i.e., the current association must be dropped if the PMK
* used for it expires.
* @pmk_reauth_threshold: Threshold time for reauthentication (percentage of
* PMK lifetime, dot11RSNAConfigPMKReauthThreshold) or 0 if not specified.
* Drivers are expected to trigger a full authentication instead of using
* this PMKSA for caching when reassociating to a new BSS after this
* threshold to generate a new PMK before the current one expires.
*/
struct cfg80211_pmksa {
const u8 *bssid;
const u8 *pmkid;
const u8 *pmk;
size_t pmk_len;
const u8 *ssid;
size_t ssid_len;
const u8 *cache_id;
u32 pmk_lifetime;
u8 pmk_reauth_threshold;
};
/**
* struct cfg80211_pkt_pattern - packet pattern
* @mask: bitmask where to match pattern and where to ignore bytes,
* one bit per byte, in same format as nl80211
* @pattern: bytes to match where bitmask is 1
* @pattern_len: length of pattern (in bytes)
* @pkt_offset: packet offset (in bytes)
*
* Internal note: @mask and @pattern are allocated in one chunk of
* memory, free @mask only!
*/
struct cfg80211_pkt_pattern {
const u8 *mask, *pattern;
int pattern_len;
int pkt_offset;
};
/**
* struct cfg80211_wowlan_tcp - TCP connection parameters
*
* @sock: (internal) socket for source port allocation
* @src: source IP address
* @dst: destination IP address
* @dst_mac: destination MAC address
* @src_port: source port
* @dst_port: destination port
* @payload_len: data payload length
* @payload: data payload buffer
* @payload_seq: payload sequence stamping configuration
* @data_interval: interval at which to send data packets
* @wake_len: wakeup payload match length
* @wake_data: wakeup payload match data
* @wake_mask: wakeup payload match mask
* @tokens_size: length of the tokens buffer
* @payload_tok: payload token usage configuration
*/
struct cfg80211_wowlan_tcp {
struct socket *sock;
__be32 src, dst;
u16 src_port, dst_port;
u8 dst_mac[ETH_ALEN];
int payload_len;
const u8 *payload;
struct nl80211_wowlan_tcp_data_seq payload_seq;
u32 data_interval;
u32 wake_len;
const u8 *wake_data, *wake_mask;
u32 tokens_size;
/* must be last, variable member */
struct nl80211_wowlan_tcp_data_token payload_tok;
};
/**
* struct cfg80211_wowlan - Wake on Wireless-LAN support info
*
* This structure defines the enabled WoWLAN triggers for the device.
* @any: wake up on any activity -- special trigger if device continues
* operating as normal during suspend
* @disconnect: wake up if getting disconnected
* @magic_pkt: wake up on receiving magic packet
* @patterns: wake up on receiving packet matching a pattern
* @n_patterns: number of patterns
* @gtk_rekey_failure: wake up on GTK rekey failure
* @eap_identity_req: wake up on EAP identity request packet
* @four_way_handshake: wake up on 4-way handshake
* @rfkill_release: wake up when rfkill is released
* @tcp: TCP connection establishment/wakeup parameters, see nl80211.h.
* NULL if not configured.
* @nd_config: configuration for the scan to be used for net detect wake.
*/
struct cfg80211_wowlan {
bool any, disconnect, magic_pkt, gtk_rekey_failure,
eap_identity_req, four_way_handshake,
rfkill_release;
struct cfg80211_pkt_pattern *patterns;
struct cfg80211_wowlan_tcp *tcp;
int n_patterns;
struct cfg80211_sched_scan_request *nd_config;
};
/**
* struct cfg80211_coalesce_rules - Coalesce rule parameters
*
* This structure defines coalesce rule for the device.
* @delay: maximum coalescing delay in msecs.
* @condition: condition for packet coalescence.
* see &enum nl80211_coalesce_condition.
* @patterns: array of packet patterns
* @n_patterns: number of patterns
*/
struct cfg80211_coalesce_rules {
int delay;
enum nl80211_coalesce_condition condition;
struct cfg80211_pkt_pattern *patterns;
int n_patterns;
};
/**
* struct cfg80211_coalesce - Packet coalescing settings
*
* This structure defines coalescing settings.
* @rules: array of coalesce rules
* @n_rules: number of rules
*/
struct cfg80211_coalesce {
struct cfg80211_coalesce_rules *rules;
int n_rules;
};
/**
* struct cfg80211_wowlan_nd_match - information about the match
*
* @ssid: SSID of the match that triggered the wake up
* @n_channels: Number of channels where the match occurred. This
* value may be zero if the driver can't report the channels.
* @channels: center frequencies of the channels where a match
* occurred (in MHz)
*/
struct cfg80211_wowlan_nd_match {
struct cfg80211_ssid ssid;
int n_channels;
u32 channels[];
};
/**
* struct cfg80211_wowlan_nd_info - net detect wake up information
*
* @n_matches: Number of match information instances provided in
* @matches. This value may be zero if the driver can't provide
* match information.
* @matches: Array of pointers to matches containing information about
* the matches that triggered the wake up.
*/
struct cfg80211_wowlan_nd_info {
int n_matches;
struct cfg80211_wowlan_nd_match *matches[];
};
/**
* struct cfg80211_wowlan_wakeup - wakeup report
* @disconnect: woke up by getting disconnected
* @magic_pkt: woke up by receiving magic packet
* @gtk_rekey_failure: woke up by GTK rekey failure
* @eap_identity_req: woke up by EAP identity request packet
* @four_way_handshake: woke up by 4-way handshake
* @rfkill_release: woke up by rfkill being released
* @pattern_idx: pattern that caused wakeup, -1 if not due to pattern
* @packet_present_len: copied wakeup packet data
* @packet_len: original wakeup packet length
* @packet: The packet causing the wakeup, if any.
* @packet_80211: For pattern match, magic packet and other data
* frame triggers an 802.3 frame should be reported, for
* disconnect due to deauth 802.11 frame. This indicates which
* it is.
* @tcp_match: TCP wakeup packet received
* @tcp_connlost: TCP connection lost or failed to establish
* @tcp_nomoretokens: TCP data ran out of tokens
* @net_detect: if not %NULL, woke up because of net detect
* @unprot_deauth_disassoc: woke up due to unprotected deauth or
* disassoc frame (in MFP).
*/
struct cfg80211_wowlan_wakeup {
bool disconnect, magic_pkt, gtk_rekey_failure,
eap_identity_req, four_way_handshake,
rfkill_release, packet_80211,
tcp_match, tcp_connlost, tcp_nomoretokens,
unprot_deauth_disassoc;
s32 pattern_idx;
u32 packet_present_len, packet_len;
const void *packet;
struct cfg80211_wowlan_nd_info *net_detect;
};
/**
* struct cfg80211_gtk_rekey_data - rekey data
* @kek: key encryption key (@kek_len bytes)
* @kck: key confirmation key (@kck_len bytes)
* @replay_ctr: replay counter (NL80211_REPLAY_CTR_LEN bytes)
* @kek_len: length of kek
* @kck_len: length of kck
* @akm: akm (oui, id)
*/
struct cfg80211_gtk_rekey_data {
const u8 *kek, *kck, *replay_ctr;
u32 akm;
u8 kek_len, kck_len;
};
/**
* struct cfg80211_update_ft_ies_params - FT IE Information
*
* This structure provides information needed to update the fast transition IE
*
* @md: The Mobility Domain ID, 2 Octet value
* @ie: Fast Transition IEs
* @ie_len: Length of ft_ie in octets
*/
struct cfg80211_update_ft_ies_params {
u16 md;
const u8 *ie;
size_t ie_len;
};
/**
* struct cfg80211_mgmt_tx_params - mgmt tx parameters
*
* This structure provides information needed to transmit a mgmt frame
*
* @chan: channel to use
* @offchan: indicates whether off channel operation is required
* @wait: duration for ROC
* @buf: buffer to transmit
* @len: buffer length
* @no_cck: don't use cck rates for this frame
* @dont_wait_for_ack: tells the low level not to wait for an ack
* @n_csa_offsets: length of csa_offsets array
* @csa_offsets: array of all the csa offsets in the frame
* @link_id: for MLO, the link ID to transmit on, -1 if not given; note
* that the link ID isn't validated (much), it's in range but the
* link might not exist (or be used by the receiver STA)
*/
struct cfg80211_mgmt_tx_params {
struct ieee80211_channel *chan;
bool offchan;
unsigned int wait;
const u8 *buf;
size_t len;
bool no_cck;
bool dont_wait_for_ack;
int n_csa_offsets;
const u16 *csa_offsets;
int link_id;
};
/**
* struct cfg80211_dscp_exception - DSCP exception
*
* @dscp: DSCP value that does not adhere to the user priority range definition
* @up: user priority value to which the corresponding DSCP value belongs
*/
struct cfg80211_dscp_exception {
u8 dscp;
u8 up;
};
/**
* struct cfg80211_dscp_range - DSCP range definition for user priority
*
* @low: lowest DSCP value of this user priority range, inclusive
* @high: highest DSCP value of this user priority range, inclusive
*/
struct cfg80211_dscp_range {
u8 low;
u8 high;
};
/* QoS Map Set element length defined in IEEE Std 802.11-2012, 8.4.2.97 */
#define IEEE80211_QOS_MAP_MAX_EX 21
#define IEEE80211_QOS_MAP_LEN_MIN 16
#define IEEE80211_QOS_MAP_LEN_MAX \
(IEEE80211_QOS_MAP_LEN_MIN + 2 * IEEE80211_QOS_MAP_MAX_EX)
/**
* struct cfg80211_qos_map - QoS Map Information
*
* This struct defines the Interworking QoS map setting for DSCP values
*
* @num_des: number of DSCP exceptions (0..21)
* @dscp_exception: optionally up to maximum of 21 DSCP exceptions from
* the user priority DSCP range definition
* @up: DSCP range definition for a particular user priority
*/
struct cfg80211_qos_map {
u8 num_des;
struct cfg80211_dscp_exception dscp_exception[IEEE80211_QOS_MAP_MAX_EX];
struct cfg80211_dscp_range up[8];
};
/**
* struct cfg80211_nan_conf - NAN configuration
*
* This struct defines NAN configuration parameters
*
* @master_pref: master preference (1 - 255)
* @bands: operating bands, a bitmap of &enum nl80211_band values.
* For instance, for NL80211_BAND_2GHZ, bit 0 would be set
* (i.e. BIT(NL80211_BAND_2GHZ)).
*/
struct cfg80211_nan_conf {
u8 master_pref;
u8 bands;
};
/**
* enum cfg80211_nan_conf_changes - indicates changed fields in NAN
* configuration
*
* @CFG80211_NAN_CONF_CHANGED_PREF: master preference
* @CFG80211_NAN_CONF_CHANGED_BANDS: operating bands
*/
enum cfg80211_nan_conf_changes {
CFG80211_NAN_CONF_CHANGED_PREF = BIT(0),
CFG80211_NAN_CONF_CHANGED_BANDS = BIT(1),
};
/**
* struct cfg80211_nan_func_filter - a NAN function Rx / Tx filter
*
* @filter: the content of the filter
* @len: the length of the filter
*/
struct cfg80211_nan_func_filter {
const u8 *filter;
u8 len;
};
/**
* struct cfg80211_nan_func - a NAN function
*
* @type: &enum nl80211_nan_function_type
* @service_id: the service ID of the function
* @publish_type: &nl80211_nan_publish_type
* @close_range: if true, the range should be limited. Threshold is
* implementation specific.
* @publish_bcast: if true, the solicited publish should be broadcasted
* @subscribe_active: if true, the subscribe is active
* @followup_id: the instance ID for follow up
* @followup_reqid: the requester instance ID for follow up
* @followup_dest: MAC address of the recipient of the follow up
* @ttl: time to live counter in DW.
* @serv_spec_info: Service Specific Info
* @serv_spec_info_len: Service Specific Info length
* @srf_include: if true, SRF is inclusive
* @srf_bf: Bloom Filter
* @srf_bf_len: Bloom Filter length
* @srf_bf_idx: Bloom Filter index
* @srf_macs: SRF MAC addresses
* @srf_num_macs: number of MAC addresses in SRF
* @rx_filters: rx filters that are matched with corresponding peer's tx_filter
* @tx_filters: filters that should be transmitted in the SDF.
* @num_rx_filters: length of &rx_filters.
* @num_tx_filters: length of &tx_filters.
* @instance_id: driver allocated id of the function.
* @cookie: unique NAN function identifier.
*/
struct cfg80211_nan_func {
enum nl80211_nan_function_type type;
u8 service_id[NL80211_NAN_FUNC_SERVICE_ID_LEN];
u8 publish_type;
bool close_range;
bool publish_bcast;
bool subscribe_active;
u8 followup_id;
u8 followup_reqid;
struct mac_address followup_dest;
u32 ttl;
const u8 *serv_spec_info;
u8 serv_spec_info_len;
bool srf_include;
const u8 *srf_bf;
u8 srf_bf_len;
u8 srf_bf_idx;
struct mac_address *srf_macs;
int srf_num_macs;
struct cfg80211_nan_func_filter *rx_filters;
struct cfg80211_nan_func_filter *tx_filters;
u8 num_tx_filters;
u8 num_rx_filters;
u8 instance_id;
u64 cookie;
};
/**
* struct cfg80211_pmk_conf - PMK configuration
*
* @aa: authenticator address
* @pmk_len: PMK length in bytes.
* @pmk: the PMK material
* @pmk_r0_name: PMK-R0 Name. NULL if not applicable (i.e., the PMK
* is not PMK-R0). When pmk_r0_name is not NULL, the pmk field
* holds PMK-R0.
*/
struct cfg80211_pmk_conf {
const u8 *aa;
u8 pmk_len;
const u8 *pmk;
const u8 *pmk_r0_name;
};
/**
* struct cfg80211_external_auth_params - Trigger External authentication.
*
* Commonly used across the external auth request and event interfaces.
*
* @action: action type / trigger for external authentication. Only significant
* for the authentication request event interface (driver to user space).
* @bssid: BSSID of the peer with which the authentication has
* to happen. Used by both the authentication request event and
* authentication response command interface.
* @ssid: SSID of the AP. Used by both the authentication request event and
* authentication response command interface.
* @key_mgmt_suite: AKM suite of the respective authentication. Used by the
* authentication request event interface.
* @status: status code, %WLAN_STATUS_SUCCESS for successful authentication,
* use %WLAN_STATUS_UNSPECIFIED_FAILURE if user space cannot give you
* the real status code for failures. Used only for the authentication
* response command interface (user space to driver).
* @pmkid: The identifier to refer a PMKSA.
* @mld_addr: MLD address of the peer. Used by the authentication request event
* interface. Driver indicates this to enable MLO during the authentication
* offload to user space. Driver shall look at %NL80211_ATTR_MLO_SUPPORT
* flag capability in NL80211_CMD_CONNECT to know whether the user space
* supports enabling MLO during the authentication offload.
* User space should use the address of the interface (on which the
* authentication request event reported) as self MLD address. User space
* and driver should use MLD addresses in RA, TA and BSSID fields of
* authentication frames sent or received via cfg80211. The driver
* translates the MLD addresses to/from link addresses based on the link
* chosen for the authentication.
*/
struct cfg80211_external_auth_params {
enum nl80211_external_auth_action action;
u8 bssid[ETH_ALEN] __aligned(2);
struct cfg80211_ssid ssid;
unsigned int key_mgmt_suite;
u16 status;
const u8 *pmkid;
u8 mld_addr[ETH_ALEN] __aligned(2);
};
/**
* struct cfg80211_ftm_responder_stats - FTM responder statistics
*
* @filled: bitflag of flags using the bits of &enum nl80211_ftm_stats to
* indicate the relevant values in this struct for them
* @success_num: number of FTM sessions in which all frames were successfully
* answered
* @partial_num: number of FTM sessions in which part of frames were
* successfully answered
* @failed_num: number of failed FTM sessions
* @asap_num: number of ASAP FTM sessions
* @non_asap_num: number of non-ASAP FTM sessions
* @total_duration_ms: total sessions durations - gives an indication
* of how much time the responder was busy
* @unknown_triggers_num: number of unknown FTM triggers - triggers from
* initiators that didn't finish successfully the negotiation phase with
* the responder
* @reschedule_requests_num: number of FTM reschedule requests - initiator asks
* for a new scheduling although it already has scheduled FTM slot
* @out_of_window_triggers_num: total FTM triggers out of scheduled window
*/
struct cfg80211_ftm_responder_stats {
u32 filled;
u32 success_num;
u32 partial_num;
u32 failed_num;
u32 asap_num;
u32 non_asap_num;
u64 total_duration_ms;
u32 unknown_triggers_num;
u32 reschedule_requests_num;
u32 out_of_window_triggers_num;
};
/**
* struct cfg80211_pmsr_ftm_result - FTM result
* @failure_reason: if this measurement failed (PMSR status is
* %NL80211_PMSR_STATUS_FAILURE), this gives a more precise
* reason than just "failure"
* @burst_index: if reporting partial results, this is the index
* in [0 .. num_bursts-1] of the burst that's being reported
* @num_ftmr_attempts: number of FTM request frames transmitted
* @num_ftmr_successes: number of FTM request frames acked
* @busy_retry_time: if failure_reason is %NL80211_PMSR_FTM_FAILURE_PEER_BUSY,
* fill this to indicate in how many seconds a retry is deemed possible
* by the responder
* @num_bursts_exp: actual number of bursts exponent negotiated
* @burst_duration: actual burst duration negotiated
* @ftms_per_burst: actual FTMs per burst negotiated
* @lci_len: length of LCI information (if present)
* @civicloc_len: length of civic location information (if present)
* @lci: LCI data (may be %NULL)
* @civicloc: civic location data (may be %NULL)
* @rssi_avg: average RSSI over FTM action frames reported
* @rssi_spread: spread of the RSSI over FTM action frames reported
* @tx_rate: bitrate for transmitted FTM action frame response
* @rx_rate: bitrate of received FTM action frame
* @rtt_avg: average of RTTs measured (must have either this or @dist_avg)
* @rtt_variance: variance of RTTs measured (note that standard deviation is
* the square root of the variance)
* @rtt_spread: spread of the RTTs measured
* @dist_avg: average of distances (mm) measured
* (must have either this or @rtt_avg)
* @dist_variance: variance of distances measured (see also @rtt_variance)
* @dist_spread: spread of distances measured (see also @rtt_spread)
* @num_ftmr_attempts_valid: @num_ftmr_attempts is valid
* @num_ftmr_successes_valid: @num_ftmr_successes is valid
* @rssi_avg_valid: @rssi_avg is valid
* @rssi_spread_valid: @rssi_spread is valid
* @tx_rate_valid: @tx_rate is valid
* @rx_rate_valid: @rx_rate is valid
* @rtt_avg_valid: @rtt_avg is valid
* @rtt_variance_valid: @rtt_variance is valid
* @rtt_spread_valid: @rtt_spread is valid
* @dist_avg_valid: @dist_avg is valid
* @dist_variance_valid: @dist_variance is valid
* @dist_spread_valid: @dist_spread is valid
*/
struct cfg80211_pmsr_ftm_result {
const u8 *lci;
const u8 *civicloc;
unsigned int lci_len;
unsigned int civicloc_len;
enum nl80211_peer_measurement_ftm_failure_reasons failure_reason;
u32 num_ftmr_attempts, num_ftmr_successes;
s16 burst_index;
u8 busy_retry_time;
u8 num_bursts_exp;
u8 burst_duration;
u8 ftms_per_burst;
s32 rssi_avg;
s32 rssi_spread;
struct rate_info tx_rate, rx_rate;
s64 rtt_avg;
s64 rtt_variance;
s64 rtt_spread;
s64 dist_avg;
s64 dist_variance;
s64 dist_spread;
u16 num_ftmr_attempts_valid:1,
num_ftmr_successes_valid:1,
rssi_avg_valid:1,
rssi_spread_valid:1,
tx_rate_valid:1,
rx_rate_valid:1,
rtt_avg_valid:1,
rtt_variance_valid:1,
rtt_spread_valid:1,
dist_avg_valid:1,
dist_variance_valid:1,
dist_spread_valid:1;
};
/**
* struct cfg80211_pmsr_result - peer measurement result
* @addr: address of the peer
* @host_time: host time (use ktime_get_boottime() adjust to the time when the
* measurement was made)
* @ap_tsf: AP's TSF at measurement time
* @status: status of the measurement
* @final: if reporting partial results, mark this as the last one; if not
* reporting partial results always set this flag
* @ap_tsf_valid: indicates the @ap_tsf value is valid
* @type: type of the measurement reported, note that we only support reporting
* one type at a time, but you can report multiple results separately and
* they're all aggregated for userspace.
* @ftm: FTM result
*/
struct cfg80211_pmsr_result {
u64 host_time, ap_tsf;
enum nl80211_peer_measurement_status status;
u8 addr[ETH_ALEN];
u8 final:1,
ap_tsf_valid:1;
enum nl80211_peer_measurement_type type;
union {
struct cfg80211_pmsr_ftm_result ftm;
};
};
/**
* struct cfg80211_pmsr_ftm_request_peer - FTM request data
* @requested: indicates FTM is requested
* @preamble: frame preamble to use
* @burst_period: burst period to use
* @asap: indicates to use ASAP mode
* @num_bursts_exp: number of bursts exponent
* @burst_duration: burst duration
* @ftms_per_burst: number of FTMs per burst
* @ftmr_retries: number of retries for FTM request
* @request_lci: request LCI information
* @request_civicloc: request civic location information
* @trigger_based: use trigger based ranging for the measurement
* If neither @trigger_based nor @non_trigger_based is set,
* EDCA based ranging will be used.
* @non_trigger_based: use non trigger based ranging for the measurement
* If neither @trigger_based nor @non_trigger_based is set,
* EDCA based ranging will be used.
* @lmr_feedback: negotiate for I2R LMR feedback. Only valid if either
* @trigger_based or @non_trigger_based is set.
* @bss_color: the bss color of the responder. Optional. Set to zero to
* indicate the driver should set the BSS color. Only valid if
* @non_trigger_based or @trigger_based is set.
*
* See also nl80211 for the respective attribute documentation.
*/
struct cfg80211_pmsr_ftm_request_peer {
enum nl80211_preamble preamble;
u16 burst_period;
u8 requested:1,
asap:1,
request_lci:1,
request_civicloc:1,
trigger_based:1,
non_trigger_based:1,
lmr_feedback:1;
u8 num_bursts_exp;
u8 burst_duration;
u8 ftms_per_burst;
u8 ftmr_retries;
u8 bss_color;
};
/**
* struct cfg80211_pmsr_request_peer - peer data for a peer measurement request
* @addr: MAC address
* @chandef: channel to use
* @report_ap_tsf: report the associated AP's TSF
* @ftm: FTM data, see &struct cfg80211_pmsr_ftm_request_peer
*/
struct cfg80211_pmsr_request_peer {
u8 addr[ETH_ALEN];
struct cfg80211_chan_def chandef;
u8 report_ap_tsf:1;
struct cfg80211_pmsr_ftm_request_peer ftm;
};
/**
* struct cfg80211_pmsr_request - peer measurement request
* @cookie: cookie, set by cfg80211
* @nl_portid: netlink portid - used by cfg80211
* @drv_data: driver data for this request, if required for aborting,
* not otherwise freed or anything by cfg80211
* @mac_addr: MAC address used for (randomised) request
* @mac_addr_mask: MAC address mask used for randomisation, bits that
* are 0 in the mask should be randomised, bits that are 1 should
* be taken from the @mac_addr
* @list: used by cfg80211 to hold on to the request
* @timeout: timeout (in milliseconds) for the whole operation, if
* zero it means there's no timeout
* @n_peers: number of peers to do measurements with
* @peers: per-peer measurement request data
*/
struct cfg80211_pmsr_request {
u64 cookie;
void *drv_data;
u32 n_peers;
u32 nl_portid;
u32 timeout;
u8 mac_addr[ETH_ALEN] __aligned(2);
u8 mac_addr_mask[ETH_ALEN] __aligned(2);
struct list_head list;
struct cfg80211_pmsr_request_peer peers[] __counted_by(n_peers);
};
/**
* struct cfg80211_update_owe_info - OWE Information
*
* This structure provides information needed for the drivers to offload OWE
* (Opportunistic Wireless Encryption) processing to the user space.
*
* Commonly used across update_owe_info request and event interfaces.
*
* @peer: MAC address of the peer device for which the OWE processing
* has to be done.
* @status: status code, %WLAN_STATUS_SUCCESS for successful OWE info
* processing, use %WLAN_STATUS_UNSPECIFIED_FAILURE if user space
* cannot give you the real status code for failures. Used only for
* OWE update request command interface (user space to driver).
* @ie: IEs obtained from the peer or constructed by the user space. These are
* the IEs of the remote peer in the event from the host driver and
* the constructed IEs by the user space in the request interface.
* @ie_len: Length of IEs in octets.
* @assoc_link_id: MLO link ID of the AP, with which (re)association requested
* by peer. This will be filled by driver for both MLO and non-MLO station
* connections when the AP affiliated with an MLD. For non-MLD AP mode, it
* will be -1. Used only with OWE update event (driver to user space).
* @peer_mld_addr: For MLO connection, MLD address of the peer. For non-MLO
* connection, it will be all zeros. This is applicable only when
* @assoc_link_id is not -1, i.e., the AP affiliated with an MLD. Used only
* with OWE update event (driver to user space).
*/
struct cfg80211_update_owe_info {
u8 peer[ETH_ALEN] __aligned(2);
u16 status;
const u8 *ie;
size_t ie_len;
int assoc_link_id;
u8 peer_mld_addr[ETH_ALEN] __aligned(2);
};
/**
* struct mgmt_frame_regs - management frame registrations data
* @global_stypes: bitmap of management frame subtypes registered
* for the entire device
* @interface_stypes: bitmap of management frame subtypes registered
* for the given interface
* @global_mcast_stypes: mcast RX is needed globally for these subtypes
* @interface_mcast_stypes: mcast RX is needed on this interface
* for these subtypes
*/
struct mgmt_frame_regs {
u32 global_stypes, interface_stypes;
u32 global_mcast_stypes, interface_mcast_stypes;
};
/**
* struct cfg80211_ops - backend description for wireless configuration
*
* This struct is registered by fullmac card drivers and/or wireless stacks
* in order to handle configuration requests on their interfaces.
*
* All callbacks except where otherwise noted should return 0
* on success or a negative error code.
*
* All operations are invoked with the wiphy mutex held. The RTNL may be
* held in addition (due to wireless extensions) but this cannot be relied
* upon except in cases where documented below. Note that due to ordering,
* the RTNL also cannot be acquired in any handlers.
*
* @suspend: wiphy device needs to be suspended. The variable @wow will
* be %NULL or contain the enabled Wake-on-Wireless triggers that are
* configured for the device.
* @resume: wiphy device needs to be resumed
* @set_wakeup: Called when WoWLAN is enabled/disabled, use this callback
* to call device_set_wakeup_enable() to enable/disable wakeup from
* the device.
*
* @add_virtual_intf: create a new virtual interface with the given name,
* must set the struct wireless_dev's iftype. Beware: You must create
* the new netdev in the wiphy's network namespace! Returns the struct
* wireless_dev, or an ERR_PTR. For P2P device wdevs, the driver must
* also set the address member in the wdev.
* This additionally holds the RTNL to be able to do netdev changes.
*
* @del_virtual_intf: remove the virtual interface
* This additionally holds the RTNL to be able to do netdev changes.
*
* @change_virtual_intf: change type/configuration of virtual interface,
* keep the struct wireless_dev's iftype updated.
* This additionally holds the RTNL to be able to do netdev changes.
*
* @add_intf_link: Add a new MLO link to the given interface. Note that
* the wdev->link[] data structure has been updated, so the new link
* address is available.
* @del_intf_link: Remove an MLO link from the given interface.
*
* @add_key: add a key with the given parameters. @mac_addr will be %NULL
* when adding a group key. @link_id will be -1 for non-MLO connection.
* For MLO connection, @link_id will be >= 0 for group key and -1 for
* pairwise key, @mac_addr will be peer's MLD address for MLO pairwise key.
*
* @get_key: get information about the key with the given parameters.
* @mac_addr will be %NULL when requesting information for a group
* key. All pointers given to the @callback function need not be valid
* after it returns. This function should return an error if it is
* not possible to retrieve the key, -ENOENT if it doesn't exist.
* @link_id will be -1 for non-MLO connection. For MLO connection,
* @link_id will be >= 0 for group key and -1 for pairwise key, @mac_addr
* will be peer's MLD address for MLO pairwise key.
*
* @del_key: remove a key given the @mac_addr (%NULL for a group key)
* and @key_index, return -ENOENT if the key doesn't exist. @link_id will
* be -1 for non-MLO connection. For MLO connection, @link_id will be >= 0
* for group key and -1 for pairwise key, @mac_addr will be peer's MLD
* address for MLO pairwise key.
*
* @set_default_key: set the default key on an interface. @link_id will be >= 0
* for MLO connection and -1 for non-MLO connection.
*
* @set_default_mgmt_key: set the default management frame key on an interface.
* @link_id will be >= 0 for MLO connection and -1 for non-MLO connection.
*
* @set_default_beacon_key: set the default Beacon frame key on an interface.
* @link_id will be >= 0 for MLO connection and -1 for non-MLO connection.
*
* @set_rekey_data: give the data necessary for GTK rekeying to the driver
*
* @start_ap: Start acting in AP mode defined by the parameters.
* @change_beacon: Change the beacon parameters for an access point mode
* interface. This should reject the call when AP mode wasn't started.
* @stop_ap: Stop being an AP, including stopping beaconing.
*
* @add_station: Add a new station.
* @del_station: Remove a station
* @change_station: Modify a given station. Note that flags changes are not much
* validated in cfg80211, in particular the auth/assoc/authorized flags
* might come to the driver in invalid combinations -- make sure to check
* them, also against the existing state! Drivers must call
* cfg80211_check_station_change() to validate the information.
* @get_station: get station information for the station identified by @mac
* @dump_station: dump station callback -- resume dump at index @idx
*
* @add_mpath: add a fixed mesh path
* @del_mpath: delete a given mesh path
* @change_mpath: change a given mesh path
* @get_mpath: get a mesh path for the given parameters
* @dump_mpath: dump mesh path callback -- resume dump at index @idx
* @get_mpp: get a mesh proxy path for the given parameters
* @dump_mpp: dump mesh proxy path callback -- resume dump at index @idx
* @join_mesh: join the mesh network with the specified parameters
* (invoked with the wireless_dev mutex held)
* @leave_mesh: leave the current mesh network
* (invoked with the wireless_dev mutex held)
*
* @get_mesh_config: Get the current mesh configuration
*
* @update_mesh_config: Update mesh parameters on a running mesh.
* The mask is a bitfield which tells us which parameters to
* set, and which to leave alone.
*
* @change_bss: Modify parameters for a given BSS.
*
* @inform_bss: Called by cfg80211 while being informed about new BSS data
* for every BSS found within the reported data or frame. This is called
* from within the cfg8011 inform_bss handlers while holding the bss_lock.
* The data parameter is passed through from drv_data inside
* struct cfg80211_inform_bss.
* The new IE data for the BSS is explicitly passed.
*
* @set_txq_params: Set TX queue parameters
*
* @libertas_set_mesh_channel: Only for backward compatibility for libertas,
* as it doesn't implement join_mesh and needs to set the channel to
* join the mesh instead.
*
* @set_monitor_channel: Set the monitor mode channel for the device. If other
* interfaces are active this callback should reject the configuration.
* If no interfaces are active or the device is down, the channel should
* be stored for when a monitor interface becomes active.
*
* @scan: Request to do a scan. If returning zero, the scan request is given
* the driver, and will be valid until passed to cfg80211_scan_done().
* For scan results, call cfg80211_inform_bss(); you can call this outside
* the scan/scan_done bracket too.
* @abort_scan: Tell the driver to abort an ongoing scan. The driver shall
* indicate the status of the scan through cfg80211_scan_done().
*
* @auth: Request to authenticate with the specified peer
* (invoked with the wireless_dev mutex held)
* @assoc: Request to (re)associate with the specified peer
* (invoked with the wireless_dev mutex held)
* @deauth: Request to deauthenticate from the specified peer
* (invoked with the wireless_dev mutex held)
* @disassoc: Request to disassociate from the specified peer
* (invoked with the wireless_dev mutex held)
*
* @connect: Connect to the ESS with the specified parameters. When connected,
* call cfg80211_connect_result()/cfg80211_connect_bss() with status code
* %WLAN_STATUS_SUCCESS. If the connection fails for some reason, call
* cfg80211_connect_result()/cfg80211_connect_bss() with the status code
* from the AP or cfg80211_connect_timeout() if no frame with status code
* was received.
* The driver is allowed to roam to other BSSes within the ESS when the
* other BSS matches the connect parameters. When such roaming is initiated
* by the driver, the driver is expected to verify that the target matches
* the configured security parameters and to use Reassociation Request
* frame instead of Association Request frame.
* The connect function can also be used to request the driver to perform a
* specific roam when connected to an ESS. In that case, the prev_bssid
* parameter is set to the BSSID of the currently associated BSS as an
* indication of requesting reassociation.
* In both the driver-initiated and new connect() call initiated roaming
* cases, the result of roaming is indicated with a call to
* cfg80211_roamed(). (invoked with the wireless_dev mutex held)
* @update_connect_params: Update the connect parameters while connected to a
* BSS. The updated parameters can be used by driver/firmware for
* subsequent BSS selection (roaming) decisions and to form the
* Authentication/(Re)Association Request frames. This call does not
* request an immediate disassociation or reassociation with the current
* BSS, i.e., this impacts only subsequent (re)associations. The bits in
* changed are defined in &enum cfg80211_connect_params_changed.
* (invoked with the wireless_dev mutex held)
* @disconnect: Disconnect from the BSS/ESS or stop connection attempts if
* connection is in progress. Once done, call cfg80211_disconnected() in
* case connection was already established (invoked with the
* wireless_dev mutex held), otherwise call cfg80211_connect_timeout().
*
* @join_ibss: Join the specified IBSS (or create if necessary). Once done, call
* cfg80211_ibss_joined(), also call that function when changing BSSID due
* to a merge.
* (invoked with the wireless_dev mutex held)
* @leave_ibss: Leave the IBSS.
* (invoked with the wireless_dev mutex held)
*
* @set_mcast_rate: Set the specified multicast rate (only if vif is in ADHOC or
* MESH mode)
*
* @set_wiphy_params: Notify that wiphy parameters have changed;
* @changed bitfield (see &enum wiphy_params_flags) describes which values
* have changed. The actual parameter values are available in
* struct wiphy. If returning an error, no value should be changed.
*
* @set_tx_power: set the transmit power according to the parameters,
* the power passed is in mBm, to get dBm use MBM_TO_DBM(). The
* wdev may be %NULL if power was set for the wiphy, and will
* always be %NULL unless the driver supports per-vif TX power
* (as advertised by the nl80211 feature flag.)
* @get_tx_power: store the current TX power into the dbm variable;
* return 0 if successful
*
* @rfkill_poll: polls the hw rfkill line, use cfg80211 reporting
* functions to adjust rfkill hw state
*
* @dump_survey: get site survey information.
*
* @remain_on_channel: Request the driver to remain awake on the specified
* channel for the specified duration to complete an off-channel
* operation (e.g., public action frame exchange). When the driver is
* ready on the requested channel, it must indicate this with an event
* notification by calling cfg80211_ready_on_channel().
* @cancel_remain_on_channel: Cancel an on-going remain-on-channel operation.
* This allows the operation to be terminated prior to timeout based on
* the duration value.
* @mgmt_tx: Transmit a management frame.
* @mgmt_tx_cancel_wait: Cancel the wait time from transmitting a management
* frame on another channel
*
* @testmode_cmd: run a test mode command; @wdev may be %NULL
* @testmode_dump: Implement a test mode dump. The cb->args[2] and up may be
* used by the function, but 0 and 1 must not be touched. Additionally,
* return error codes other than -ENOBUFS and -ENOENT will terminate the
* dump and return to userspace with an error, so be careful. If any data
* was passed in from userspace then the data/len arguments will be present
* and point to the data contained in %NL80211_ATTR_TESTDATA.
*
* @set_bitrate_mask: set the bitrate mask configuration
*
* @set_pmksa: Cache a PMKID for a BSSID. This is mostly useful for fullmac
* devices running firmwares capable of generating the (re) association
* RSN IE. It allows for faster roaming between WPA2 BSSIDs.
* @del_pmksa: Delete a cached PMKID.
* @flush_pmksa: Flush all cached PMKIDs.
* @set_power_mgmt: Configure WLAN power management. A timeout value of -1
* allows the driver to adjust the dynamic ps timeout value.
* @set_cqm_rssi_config: Configure connection quality monitor RSSI threshold.
* After configuration, the driver should (soon) send an event indicating
* the current level is above/below the configured threshold; this may
* need some care when the configuration is changed (without first being
* disabled.)
* @set_cqm_rssi_range_config: Configure two RSSI thresholds in the
* connection quality monitor. An event is to be sent only when the
* signal level is found to be outside the two values. The driver should
* set %NL80211_EXT_FEATURE_CQM_RSSI_LIST if this method is implemented.
* If it is provided then there's no point providing @set_cqm_rssi_config.
* @set_cqm_txe_config: Configure connection quality monitor TX error
* thresholds.
* @sched_scan_start: Tell the driver to start a scheduled scan.
* @sched_scan_stop: Tell the driver to stop an ongoing scheduled scan with
* given request id. This call must stop the scheduled scan and be ready
* for starting a new one before it returns, i.e. @sched_scan_start may be
* called immediately after that again and should not fail in that case.
* The driver should not call cfg80211_sched_scan_stopped() for a requested
* stop (when this method returns 0).
*
* @update_mgmt_frame_registrations: Notify the driver that management frame
* registrations were updated. The callback is allowed to sleep.
*
* @set_antenna: Set antenna configuration (tx_ant, rx_ant) on the device.
* Parameters are bitmaps of allowed antennas to use for TX/RX. Drivers may
* reject TX/RX mask combinations they cannot support by returning -EINVAL
* (also see nl80211.h @NL80211_ATTR_WIPHY_ANTENNA_TX).
*
* @get_antenna: Get current antenna configuration from device (tx_ant, rx_ant).
*
* @tdls_mgmt: Transmit a TDLS management frame.
* @tdls_oper: Perform a high-level TDLS operation (e.g. TDLS link setup).
*
* @probe_client: probe an associated client, must return a cookie that it
* later passes to cfg80211_probe_status().
*
* @set_noack_map: Set the NoAck Map for the TIDs.
*
* @get_channel: Get the current operating channel for the virtual interface.
* For monitor interfaces, it should return %NULL unless there's a single
* current monitoring channel.
*
* @start_p2p_device: Start the given P2P device.
* @stop_p2p_device: Stop the given P2P device.
*
* @set_mac_acl: Sets MAC address control list in AP and P2P GO mode.
* Parameters include ACL policy, an array of MAC address of stations
* and the number of MAC addresses. If there is already a list in driver
* this new list replaces the existing one. Driver has to clear its ACL
* when number of MAC addresses entries is passed as 0. Drivers which
* advertise the support for MAC based ACL have to implement this callback.
*
* @start_radar_detection: Start radar detection in the driver.
*
* @end_cac: End running CAC, probably because a related CAC
* was finished on another phy.
*
* @update_ft_ies: Provide updated Fast BSS Transition information to the
* driver. If the SME is in the driver/firmware, this information can be
* used in building Authentication and Reassociation Request frames.
*
* @crit_proto_start: Indicates a critical protocol needs more link reliability
* for a given duration (milliseconds). The protocol is provided so the
* driver can take the most appropriate actions.
* @crit_proto_stop: Indicates critical protocol no longer needs increased link
* reliability. This operation can not fail.
* @set_coalesce: Set coalesce parameters.
*
* @channel_switch: initiate channel-switch procedure (with CSA). Driver is
* responsible for veryfing if the switch is possible. Since this is
* inherently tricky driver may decide to disconnect an interface later
* with cfg80211_stop_iface(). This doesn't mean driver can accept
* everything. It should do it's best to verify requests and reject them
* as soon as possible.
*
* @set_qos_map: Set QoS mapping information to the driver
*
* @set_ap_chanwidth: Set the AP (including P2P GO) mode channel width for the
* given interface This is used e.g. for dynamic HT 20/40 MHz channel width
* changes during the lifetime of the BSS.
*
* @add_tx_ts: validate (if admitted_time is 0) or add a TX TS to the device
* with the given parameters; action frame exchange has been handled by
* userspace so this just has to modify the TX path to take the TS into
* account.
* If the admitted time is 0 just validate the parameters to make sure
* the session can be created at all; it is valid to just always return
* success for that but that may result in inefficient behaviour (handshake
* with the peer followed by immediate teardown when the addition is later
* rejected)
* @del_tx_ts: remove an existing TX TS
*
* @join_ocb: join the OCB network with the specified parameters
* (invoked with the wireless_dev mutex held)
* @leave_ocb: leave the current OCB network
* (invoked with the wireless_dev mutex held)
*
* @tdls_channel_switch: Start channel-switching with a TDLS peer. The driver
* is responsible for continually initiating channel-switching operations
* and returning to the base channel for communication with the AP.
* @tdls_cancel_channel_switch: Stop channel-switching with a TDLS peer. Both
* peers must be on the base channel when the call completes.
* @start_nan: Start the NAN interface.
* @stop_nan: Stop the NAN interface.
* @add_nan_func: Add a NAN function. Returns negative value on failure.
* On success @nan_func ownership is transferred to the driver and
* it may access it outside of the scope of this function. The driver
* should free the @nan_func when no longer needed by calling
* cfg80211_free_nan_func().
* On success the driver should assign an instance_id in the
* provided @nan_func.
* @del_nan_func: Delete a NAN function.
* @nan_change_conf: changes NAN configuration. The changed parameters must
* be specified in @changes (using &enum cfg80211_nan_conf_changes);
* All other parameters must be ignored.
*
* @set_multicast_to_unicast: configure multicast to unicast conversion for BSS
*
* @get_txq_stats: Get TXQ stats for interface or phy. If wdev is %NULL, this
* function should return phy stats, and interface stats otherwise.
*
* @set_pmk: configure the PMK to be used for offloaded 802.1X 4-Way handshake.
* If not deleted through @del_pmk the PMK remains valid until disconnect
* upon which the driver should clear it.
* (invoked with the wireless_dev mutex held)
* @del_pmk: delete the previously configured PMK for the given authenticator.
* (invoked with the wireless_dev mutex held)
*
* @external_auth: indicates result of offloaded authentication processing from
* user space
*
* @tx_control_port: TX a control port frame (EAPoL). The noencrypt parameter
* tells the driver that the frame should not be encrypted.
*
* @get_ftm_responder_stats: Retrieve FTM responder statistics, if available.
* Statistics should be cumulative, currently no way to reset is provided.
* @start_pmsr: start peer measurement (e.g. FTM)
* @abort_pmsr: abort peer measurement
*
* @update_owe_info: Provide updated OWE info to driver. Driver implementing SME
* but offloading OWE processing to the user space will get the updated
* DH IE through this interface.
*
* @probe_mesh_link: Probe direct Mesh peer's link quality by sending data frame
* and overrule HWMP path selection algorithm.
* @set_tid_config: TID specific configuration, this can be peer or BSS specific
* This callback may sleep.
* @reset_tid_config: Reset TID specific configuration for the peer, for the
* given TIDs. This callback may sleep.
*
* @set_sar_specs: Update the SAR (TX power) settings.
*
* @color_change: Initiate a color change.
*
* @set_fils_aad: Set FILS AAD data to the AP driver so that the driver can use
* those to decrypt (Re)Association Request and encrypt (Re)Association
* Response frame.
*
* @set_radar_background: Configure dedicated offchannel chain available for
* radar/CAC detection on some hw. This chain can't be used to transmit
* or receive frames and it is bounded to a running wdev.
* Background radar/CAC detection allows to avoid the CAC downtime
* switching to a different channel during CAC detection on the selected
* radar channel.
* The caller is expected to set chandef pointer to NULL in order to
* disable background CAC/radar detection.
* @add_link_station: Add a link to a station.
* @mod_link_station: Modify a link of a station.
* @del_link_station: Remove a link of a station.
*
* @set_hw_timestamp: Enable/disable HW timestamping of TM/FTM frames.
* @set_ttlm: set the TID to link mapping.
*/
struct cfg80211_ops {
int (*suspend)(struct wiphy *wiphy, struct cfg80211_wowlan *wow);
int (*resume)(struct wiphy *wiphy);
void (*set_wakeup)(struct wiphy *wiphy, bool enabled);
struct wireless_dev * (*add_virtual_intf)(struct wiphy *wiphy,
const char *name,
unsigned char name_assign_type,
enum nl80211_iftype type,
struct vif_params *params);
int (*del_virtual_intf)(struct wiphy *wiphy,
struct wireless_dev *wdev);
int (*change_virtual_intf)(struct wiphy *wiphy,
struct net_device *dev,
enum nl80211_iftype type,
struct vif_params *params);
int (*add_intf_link)(struct wiphy *wiphy,
struct wireless_dev *wdev,
unsigned int link_id);
void (*del_intf_link)(struct wiphy *wiphy,
struct wireless_dev *wdev,
unsigned int link_id);
int (*add_key)(struct wiphy *wiphy, struct net_device *netdev,
int link_id, u8 key_index, bool pairwise,
const u8 *mac_addr, struct key_params *params);
int (*get_key)(struct wiphy *wiphy, struct net_device *netdev,
int link_id, u8 key_index, bool pairwise,
const u8 *mac_addr, void *cookie,
void (*callback)(void *cookie, struct key_params*));
int (*del_key)(struct wiphy *wiphy, struct net_device *netdev,
int link_id, u8 key_index, bool pairwise,
const u8 *mac_addr);
int (*set_default_key)(struct wiphy *wiphy,
struct net_device *netdev, int link_id,
u8 key_index, bool unicast, bool multicast);
int (*set_default_mgmt_key)(struct wiphy *wiphy,
struct net_device *netdev, int link_id,
u8 key_index);
int (*set_default_beacon_key)(struct wiphy *wiphy,
struct net_device *netdev,
int link_id,
u8 key_index);
int (*start_ap)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_ap_settings *settings);
int (*change_beacon)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_ap_update *info);
int (*stop_ap)(struct wiphy *wiphy, struct net_device *dev,
unsigned int link_id);
int (*add_station)(struct wiphy *wiphy, struct net_device *dev,
const u8 *mac,
struct station_parameters *params);
int (*del_station)(struct wiphy *wiphy, struct net_device *dev,
struct station_del_parameters *params);
int (*change_station)(struct wiphy *wiphy, struct net_device *dev,
const u8 *mac,
struct station_parameters *params);
int (*get_station)(struct wiphy *wiphy, struct net_device *dev,
const u8 *mac, struct station_info *sinfo);
int (*dump_station)(struct wiphy *wiphy, struct net_device *dev,
int idx, u8 *mac, struct station_info *sinfo);
int (*add_mpath)(struct wiphy *wiphy, struct net_device *dev,
const u8 *dst, const u8 *next_hop);
int (*del_mpath)(struct wiphy *wiphy, struct net_device *dev,
const u8 *dst);
int (*change_mpath)(struct wiphy *wiphy, struct net_device *dev,
const u8 *dst, const u8 *next_hop);
int (*get_mpath)(struct wiphy *wiphy, struct net_device *dev,
u8 *dst, u8 *next_hop, struct mpath_info *pinfo);
int (*dump_mpath)(struct wiphy *wiphy, struct net_device *dev,
int idx, u8 *dst, u8 *next_hop,
struct mpath_info *pinfo);
int (*get_mpp)(struct wiphy *wiphy, struct net_device *dev,
u8 *dst, u8 *mpp, struct mpath_info *pinfo);
int (*dump_mpp)(struct wiphy *wiphy, struct net_device *dev,
int idx, u8 *dst, u8 *mpp,
struct mpath_info *pinfo);
int (*get_mesh_config)(struct wiphy *wiphy,
struct net_device *dev,
struct mesh_config *conf);
int (*update_mesh_config)(struct wiphy *wiphy,
struct net_device *dev, u32 mask,
const struct mesh_config *nconf);
int (*join_mesh)(struct wiphy *wiphy, struct net_device *dev,
const struct mesh_config *conf,
const struct mesh_setup *setup);
int (*leave_mesh)(struct wiphy *wiphy, struct net_device *dev);
int (*join_ocb)(struct wiphy *wiphy, struct net_device *dev,
struct ocb_setup *setup);
int (*leave_ocb)(struct wiphy *wiphy, struct net_device *dev);
int (*change_bss)(struct wiphy *wiphy, struct net_device *dev,
struct bss_parameters *params);
void (*inform_bss)(struct wiphy *wiphy, struct cfg80211_bss *bss,
const struct cfg80211_bss_ies *ies, void *data);
int (*set_txq_params)(struct wiphy *wiphy, struct net_device *dev,
struct ieee80211_txq_params *params);
int (*libertas_set_mesh_channel)(struct wiphy *wiphy,
struct net_device *dev,
struct ieee80211_channel *chan);
int (*set_monitor_channel)(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef);
int (*scan)(struct wiphy *wiphy,
struct cfg80211_scan_request *request);
void (*abort_scan)(struct wiphy *wiphy, struct wireless_dev *wdev);
int (*auth)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_auth_request *req);
int (*assoc)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_assoc_request *req);
int (*deauth)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_deauth_request *req);
int (*disassoc)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_disassoc_request *req);
int (*connect)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_connect_params *sme);
int (*update_connect_params)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_connect_params *sme,
u32 changed);
int (*disconnect)(struct wiphy *wiphy, struct net_device *dev,
u16 reason_code);
int (*join_ibss)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_ibss_params *params);
int (*leave_ibss)(struct wiphy *wiphy, struct net_device *dev);
int (*set_mcast_rate)(struct wiphy *wiphy, struct net_device *dev,
int rate[NUM_NL80211_BANDS]);
int (*set_wiphy_params)(struct wiphy *wiphy, u32 changed);
int (*set_tx_power)(struct wiphy *wiphy, struct wireless_dev *wdev,
enum nl80211_tx_power_setting type, int mbm);
int (*get_tx_power)(struct wiphy *wiphy, struct wireless_dev *wdev,
int *dbm);
void (*rfkill_poll)(struct wiphy *wiphy);
#ifdef CONFIG_NL80211_TESTMODE
int (*testmode_cmd)(struct wiphy *wiphy, struct wireless_dev *wdev,
void *data, int len);
int (*testmode_dump)(struct wiphy *wiphy, struct sk_buff *skb,
struct netlink_callback *cb,
void *data, int len);
#endif
int (*set_bitrate_mask)(struct wiphy *wiphy,
struct net_device *dev,
unsigned int link_id,
const u8 *peer,
const struct cfg80211_bitrate_mask *mask);
int (*dump_survey)(struct wiphy *wiphy, struct net_device *netdev,
int idx, struct survey_info *info);
int (*set_pmksa)(struct wiphy *wiphy, struct net_device *netdev,
struct cfg80211_pmksa *pmksa);
int (*del_pmksa)(struct wiphy *wiphy, struct net_device *netdev,
struct cfg80211_pmksa *pmksa);
int (*flush_pmksa)(struct wiphy *wiphy, struct net_device *netdev);
int (*remain_on_channel)(struct wiphy *wiphy,
struct wireless_dev *wdev,
struct ieee80211_channel *chan,
unsigned int duration,
u64 *cookie);
int (*cancel_remain_on_channel)(struct wiphy *wiphy,
struct wireless_dev *wdev,
u64 cookie);
int (*mgmt_tx)(struct wiphy *wiphy, struct wireless_dev *wdev,
struct cfg80211_mgmt_tx_params *params,
u64 *cookie);
int (*mgmt_tx_cancel_wait)(struct wiphy *wiphy,
struct wireless_dev *wdev,
u64 cookie);
int (*set_power_mgmt)(struct wiphy *wiphy, struct net_device *dev,
bool enabled, int timeout);
int (*set_cqm_rssi_config)(struct wiphy *wiphy,
struct net_device *dev,
s32 rssi_thold, u32 rssi_hyst);
int (*set_cqm_rssi_range_config)(struct wiphy *wiphy,
struct net_device *dev,
s32 rssi_low, s32 rssi_high);
int (*set_cqm_txe_config)(struct wiphy *wiphy,
struct net_device *dev,
u32 rate, u32 pkts, u32 intvl);
void (*update_mgmt_frame_registrations)(struct wiphy *wiphy,
struct wireless_dev *wdev,
struct mgmt_frame_regs *upd);
int (*set_antenna)(struct wiphy *wiphy, u32 tx_ant, u32 rx_ant);
int (*get_antenna)(struct wiphy *wiphy, u32 *tx_ant, u32 *rx_ant);
int (*sched_scan_start)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_sched_scan_request *request);
int (*sched_scan_stop)(struct wiphy *wiphy, struct net_device *dev,
u64 reqid);
int (*set_rekey_data)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_gtk_rekey_data *data);
int (*tdls_mgmt)(struct wiphy *wiphy, struct net_device *dev,
const u8 *peer, int link_id,
u8 action_code, u8 dialog_token, u16 status_code,
u32 peer_capability, bool initiator,
const u8 *buf, size_t len);
int (*tdls_oper)(struct wiphy *wiphy, struct net_device *dev,
const u8 *peer, enum nl80211_tdls_operation oper);
int (*probe_client)(struct wiphy *wiphy, struct net_device *dev,
const u8 *peer, u64 *cookie);
int (*set_noack_map)(struct wiphy *wiphy,
struct net_device *dev,
u16 noack_map);
int (*get_channel)(struct wiphy *wiphy,
struct wireless_dev *wdev,
unsigned int link_id,
struct cfg80211_chan_def *chandef);
int (*start_p2p_device)(struct wiphy *wiphy,
struct wireless_dev *wdev);
void (*stop_p2p_device)(struct wiphy *wiphy,
struct wireless_dev *wdev);
int (*set_mac_acl)(struct wiphy *wiphy, struct net_device *dev,
const struct cfg80211_acl_data *params);
int (*start_radar_detection)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_chan_def *chandef,
u32 cac_time_ms);
void (*end_cac)(struct wiphy *wiphy,
struct net_device *dev);
int (*update_ft_ies)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_update_ft_ies_params *ftie);
int (*crit_proto_start)(struct wiphy *wiphy,
struct wireless_dev *wdev,
enum nl80211_crit_proto_id protocol,
u16 duration);
void (*crit_proto_stop)(struct wiphy *wiphy,
struct wireless_dev *wdev);
int (*set_coalesce)(struct wiphy *wiphy,
struct cfg80211_coalesce *coalesce);
int (*channel_switch)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_csa_settings *params);
int (*set_qos_map)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_qos_map *qos_map);
int (*set_ap_chanwidth)(struct wiphy *wiphy, struct net_device *dev,
unsigned int link_id,
struct cfg80211_chan_def *chandef);
int (*add_tx_ts)(struct wiphy *wiphy, struct net_device *dev,
u8 tsid, const u8 *peer, u8 user_prio,
u16 admitted_time);
int (*del_tx_ts)(struct wiphy *wiphy, struct net_device *dev,
u8 tsid, const u8 *peer);
int (*tdls_channel_switch)(struct wiphy *wiphy,
struct net_device *dev,
const u8 *addr, u8 oper_class,
struct cfg80211_chan_def *chandef);
void (*tdls_cancel_channel_switch)(struct wiphy *wiphy,
struct net_device *dev,
const u8 *addr);
int (*start_nan)(struct wiphy *wiphy, struct wireless_dev *wdev,
struct cfg80211_nan_conf *conf);
void (*stop_nan)(struct wiphy *wiphy, struct wireless_dev *wdev);
int (*add_nan_func)(struct wiphy *wiphy, struct wireless_dev *wdev,
struct cfg80211_nan_func *nan_func);
void (*del_nan_func)(struct wiphy *wiphy, struct wireless_dev *wdev,
u64 cookie);
int (*nan_change_conf)(struct wiphy *wiphy,
struct wireless_dev *wdev,
struct cfg80211_nan_conf *conf,
u32 changes);
int (*set_multicast_to_unicast)(struct wiphy *wiphy,
struct net_device *dev,
const bool enabled);
int (*get_txq_stats)(struct wiphy *wiphy,
struct wireless_dev *wdev,
struct cfg80211_txq_stats *txqstats);
int (*set_pmk)(struct wiphy *wiphy, struct net_device *dev,
const struct cfg80211_pmk_conf *conf);
int (*del_pmk)(struct wiphy *wiphy, struct net_device *dev,
const u8 *aa);
int (*external_auth)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_external_auth_params *params);
int (*tx_control_port)(struct wiphy *wiphy,
struct net_device *dev,
const u8 *buf, size_t len,
const u8 *dest, const __be16 proto,
const bool noencrypt, int link_id,
u64 *cookie);
int (*get_ftm_responder_stats)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_ftm_responder_stats *ftm_stats);
int (*start_pmsr)(struct wiphy *wiphy, struct wireless_dev *wdev,
struct cfg80211_pmsr_request *request);
void (*abort_pmsr)(struct wiphy *wiphy, struct wireless_dev *wdev,
struct cfg80211_pmsr_request *request);
int (*update_owe_info)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_update_owe_info *owe_info);
int (*probe_mesh_link)(struct wiphy *wiphy, struct net_device *dev,
const u8 *buf, size_t len);
int (*set_tid_config)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_tid_config *tid_conf);
int (*reset_tid_config)(struct wiphy *wiphy, struct net_device *dev,
const u8 *peer, u8 tids);
int (*set_sar_specs)(struct wiphy *wiphy,
struct cfg80211_sar_specs *sar);
int (*color_change)(struct wiphy *wiphy,
struct net_device *dev,
struct cfg80211_color_change_settings *params);
int (*set_fils_aad)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_fils_aad *fils_aad);
int (*set_radar_background)(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef);
int (*add_link_station)(struct wiphy *wiphy, struct net_device *dev,
struct link_station_parameters *params);
int (*mod_link_station)(struct wiphy *wiphy, struct net_device *dev,
struct link_station_parameters *params);
int (*del_link_station)(struct wiphy *wiphy, struct net_device *dev,
struct link_station_del_parameters *params);
int (*set_hw_timestamp)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_set_hw_timestamp *hwts);
int (*set_ttlm)(struct wiphy *wiphy, struct net_device *dev,
struct cfg80211_ttlm_params *params);
};
/*
* wireless hardware and networking interfaces structures
* and registration/helper functions
*/
/**
* enum wiphy_flags - wiphy capability flags
*
* @WIPHY_FLAG_SPLIT_SCAN_6GHZ: if set to true, the scan request will be split
* into two, first for legacy bands and second for 6 GHz.
* @WIPHY_FLAG_NETNS_OK: if not set, do not allow changing the netns of this
* wiphy at all
* @WIPHY_FLAG_PS_ON_BY_DEFAULT: if set to true, powersave will be enabled
* by default -- this flag will be set depending on the kernel's default
* on wiphy_new(), but can be changed by the driver if it has a good
* reason to override the default
* @WIPHY_FLAG_4ADDR_AP: supports 4addr mode even on AP (with a single station
* on a VLAN interface). This flag also serves an extra purpose of
* supporting 4ADDR AP mode on devices which do not support AP/VLAN iftype.
* @WIPHY_FLAG_4ADDR_STATION: supports 4addr mode even as a station
* @WIPHY_FLAG_CONTROL_PORT_PROTOCOL: This device supports setting the
* control port protocol ethertype. The device also honours the
* control_port_no_encrypt flag.
* @WIPHY_FLAG_IBSS_RSN: The device supports IBSS RSN.
* @WIPHY_FLAG_MESH_AUTH: The device supports mesh authentication by routing
* auth frames to userspace. See @NL80211_MESH_SETUP_USERSPACE_AUTH.
* @WIPHY_FLAG_SUPPORTS_FW_ROAM: The device supports roaming feature in the
* firmware.
* @WIPHY_FLAG_AP_UAPSD: The device supports uapsd on AP.
* @WIPHY_FLAG_SUPPORTS_TDLS: The device supports TDLS (802.11z) operation.
* @WIPHY_FLAG_TDLS_EXTERNAL_SETUP: The device does not handle TDLS (802.11z)
* link setup/discovery operations internally. Setup, discovery and
* teardown packets should be sent through the @NL80211_CMD_TDLS_MGMT
* command. When this flag is not set, @NL80211_CMD_TDLS_OPER should be
* used for asking the driver/firmware to perform a TDLS operation.
* @WIPHY_FLAG_HAVE_AP_SME: device integrates AP SME
* @WIPHY_FLAG_REPORTS_OBSS: the device will report beacons from other BSSes
* when there are virtual interfaces in AP mode by calling
* cfg80211_report_obss_beacon().
* @WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD: When operating as an AP, the device
* responds to probe-requests in hardware.
* @WIPHY_FLAG_OFFCHAN_TX: Device supports direct off-channel TX.
* @WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL: Device supports remain-on-channel call.
* @WIPHY_FLAG_SUPPORTS_5_10_MHZ: Device supports 5 MHz and 10 MHz channels.
* @WIPHY_FLAG_HAS_CHANNEL_SWITCH: Device supports channel switch in
* beaconing mode (AP, IBSS, Mesh, ...).
* @WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK: The device supports bigger kek and kck keys
* @WIPHY_FLAG_SUPPORTS_MLO: This is a temporary flag gating the MLO APIs,
* in order to not have them reachable in normal drivers, until we have
* complete feature/interface combinations/etc. advertisement. No driver
* should set this flag for now.
* @WIPHY_FLAG_SUPPORTS_EXT_KCK_32: The device supports 32-byte KCK keys.
* @WIPHY_FLAG_NOTIFY_REGDOM_BY_DRIVER: The device could handle reg notify for
* NL80211_REGDOM_SET_BY_DRIVER.
* @WIPHY_FLAG_CHANNEL_CHANGE_ON_BEACON: reg_call_notifier() is called if driver
* set this flag to update channels on beacon hints.
* @WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY: support connection to non-primary link
* of an NSTR mobile AP MLD.
* @WIPHY_FLAG_DISABLE_WEXT: disable wireless extensions for this device
*/
enum wiphy_flags {
WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK = BIT(0),
WIPHY_FLAG_SUPPORTS_MLO = BIT(1),
WIPHY_FLAG_SPLIT_SCAN_6GHZ = BIT(2),
WIPHY_FLAG_NETNS_OK = BIT(3),
WIPHY_FLAG_PS_ON_BY_DEFAULT = BIT(4),
WIPHY_FLAG_4ADDR_AP = BIT(5),
WIPHY_FLAG_4ADDR_STATION = BIT(6),
WIPHY_FLAG_CONTROL_PORT_PROTOCOL = BIT(7),
WIPHY_FLAG_IBSS_RSN = BIT(8),
WIPHY_FLAG_DISABLE_WEXT = BIT(9),
WIPHY_FLAG_MESH_AUTH = BIT(10),
WIPHY_FLAG_SUPPORTS_EXT_KCK_32 = BIT(11),
WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY = BIT(12),
WIPHY_FLAG_SUPPORTS_FW_ROAM = BIT(13),
WIPHY_FLAG_AP_UAPSD = BIT(14),
WIPHY_FLAG_SUPPORTS_TDLS = BIT(15),
WIPHY_FLAG_TDLS_EXTERNAL_SETUP = BIT(16),
WIPHY_FLAG_HAVE_AP_SME = BIT(17),
WIPHY_FLAG_REPORTS_OBSS = BIT(18),
WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD = BIT(19),
WIPHY_FLAG_OFFCHAN_TX = BIT(20),
WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL = BIT(21),
WIPHY_FLAG_SUPPORTS_5_10_MHZ = BIT(22),
WIPHY_FLAG_HAS_CHANNEL_SWITCH = BIT(23),
WIPHY_FLAG_NOTIFY_REGDOM_BY_DRIVER = BIT(24),
WIPHY_FLAG_CHANNEL_CHANGE_ON_BEACON = BIT(25),
};
/**
* struct ieee80211_iface_limit - limit on certain interface types
* @max: maximum number of interfaces of these types
* @types: interface types (bits)
*/
struct ieee80211_iface_limit {
u16 max;
u16 types;
};
/**
* struct ieee80211_iface_combination - possible interface combination
*
* With this structure the driver can describe which interface
* combinations it supports concurrently.
*
* Examples:
*
* 1. Allow #STA <= 1, #AP <= 1, matching BI, channels = 1, 2 total:
*
* .. code-block:: c
*
* struct ieee80211_iface_limit limits1[] = {
* { .max = 1, .types = BIT(NL80211_IFTYPE_STATION), },
* { .max = 1, .types = BIT(NL80211_IFTYPE_AP), },
* };
* struct ieee80211_iface_combination combination1 = {
* .limits = limits1,
* .n_limits = ARRAY_SIZE(limits1),
* .max_interfaces = 2,
* .beacon_int_infra_match = true,
* };
*
*
* 2. Allow #{AP, P2P-GO} <= 8, channels = 1, 8 total:
*
* .. code-block:: c
*
* struct ieee80211_iface_limit limits2[] = {
* { .max = 8, .types = BIT(NL80211_IFTYPE_AP) |
* BIT(NL80211_IFTYPE_P2P_GO), },
* };
* struct ieee80211_iface_combination combination2 = {
* .limits = limits2,
* .n_limits = ARRAY_SIZE(limits2),
* .max_interfaces = 8,
* .num_different_channels = 1,
* };
*
*
* 3. Allow #STA <= 1, #{P2P-client,P2P-GO} <= 3 on two channels, 4 total.
*
* This allows for an infrastructure connection and three P2P connections.
*
* .. code-block:: c
*
* struct ieee80211_iface_limit limits3[] = {
* { .max = 1, .types = BIT(NL80211_IFTYPE_STATION), },
* { .max = 3, .types = BIT(NL80211_IFTYPE_P2P_GO) |
* BIT(NL80211_IFTYPE_P2P_CLIENT), },
* };
* struct ieee80211_iface_combination combination3 = {
* .limits = limits3,
* .n_limits = ARRAY_SIZE(limits3),
* .max_interfaces = 4,
* .num_different_channels = 2,
* };
*
*/
struct ieee80211_iface_combination {
/**
* @limits:
* limits for the given interface types
*/
const struct ieee80211_iface_limit *limits;
/**
* @num_different_channels:
* can use up to this many different channels
*/
u32 num_different_channels;
/**
* @max_interfaces:
* maximum number of interfaces in total allowed in this group
*/
u16 max_interfaces;
/**
* @n_limits:
* number of limitations
*/
u8 n_limits;
/**
* @beacon_int_infra_match:
* In this combination, the beacon intervals between infrastructure
* and AP types must match. This is required only in special cases.
*/
bool beacon_int_infra_match;
/**
* @radar_detect_widths:
* bitmap of channel widths supported for radar detection
*/
u8 radar_detect_widths;
/**
* @radar_detect_regions:
* bitmap of regions supported for radar detection
*/
u8 radar_detect_regions;
/**
* @beacon_int_min_gcd:
* This interface combination supports different beacon intervals.
*
* = 0
* all beacon intervals for different interface must be same.
* > 0
* any beacon interval for the interface part of this combination AND
* GCD of all beacon intervals from beaconing interfaces of this
* combination must be greater or equal to this value.
*/
u32 beacon_int_min_gcd;
};
struct ieee80211_txrx_stypes {
u16 tx, rx;
};
/**
* enum wiphy_wowlan_support_flags - WoWLAN support flags
* @WIPHY_WOWLAN_ANY: supports wakeup for the special "any"
* trigger that keeps the device operating as-is and
* wakes up the host on any activity, for example a
* received packet that passed filtering; note that the
* packet should be preserved in that case
* @WIPHY_WOWLAN_MAGIC_PKT: supports wakeup on magic packet
* (see nl80211.h)
* @WIPHY_WOWLAN_DISCONNECT: supports wakeup on disconnect
* @WIPHY_WOWLAN_SUPPORTS_GTK_REKEY: supports GTK rekeying while asleep
* @WIPHY_WOWLAN_GTK_REKEY_FAILURE: supports wakeup on GTK rekey failure
* @WIPHY_WOWLAN_EAP_IDENTITY_REQ: supports wakeup on EAP identity request
* @WIPHY_WOWLAN_4WAY_HANDSHAKE: supports wakeup on 4-way handshake failure
* @WIPHY_WOWLAN_RFKILL_RELEASE: supports wakeup on RF-kill release
* @WIPHY_WOWLAN_NET_DETECT: supports wakeup on network detection
*/
enum wiphy_wowlan_support_flags {
WIPHY_WOWLAN_ANY = BIT(0),
WIPHY_WOWLAN_MAGIC_PKT = BIT(1),
WIPHY_WOWLAN_DISCONNECT = BIT(2),
WIPHY_WOWLAN_SUPPORTS_GTK_REKEY = BIT(3),
WIPHY_WOWLAN_GTK_REKEY_FAILURE = BIT(4),
WIPHY_WOWLAN_EAP_IDENTITY_REQ = BIT(5),
WIPHY_WOWLAN_4WAY_HANDSHAKE = BIT(6),
WIPHY_WOWLAN_RFKILL_RELEASE = BIT(7),
WIPHY_WOWLAN_NET_DETECT = BIT(8),
};
struct wiphy_wowlan_tcp_support {
const struct nl80211_wowlan_tcp_data_token_feature *tok;
u32 data_payload_max;
u32 data_interval_max;
u32 wake_payload_max;
bool seq;
};
/**
* struct wiphy_wowlan_support - WoWLAN support data
* @flags: see &enum wiphy_wowlan_support_flags
* @n_patterns: number of supported wakeup patterns
* (see nl80211.h for the pattern definition)
* @pattern_max_len: maximum length of each pattern
* @pattern_min_len: minimum length of each pattern
* @max_pkt_offset: maximum Rx packet offset
* @max_nd_match_sets: maximum number of matchsets for net-detect,
* similar, but not necessarily identical, to max_match_sets for
* scheduled scans.
* See &struct cfg80211_sched_scan_request.@match_sets for more
* details.
* @tcp: TCP wakeup support information
*/
struct wiphy_wowlan_support {
u32 flags;
int n_patterns;
int pattern_max_len;
int pattern_min_len;
int max_pkt_offset;
int max_nd_match_sets;
const struct wiphy_wowlan_tcp_support *tcp;
};
/**
* struct wiphy_coalesce_support - coalesce support data
* @n_rules: maximum number of coalesce rules
* @max_delay: maximum supported coalescing delay in msecs
* @n_patterns: number of supported patterns in a rule
* (see nl80211.h for the pattern definition)
* @pattern_max_len: maximum length of each pattern
* @pattern_min_len: minimum length of each pattern
* @max_pkt_offset: maximum Rx packet offset
*/
struct wiphy_coalesce_support {
int n_rules;
int max_delay;
int n_patterns;
int pattern_max_len;
int pattern_min_len;
int max_pkt_offset;
};
/**
* enum wiphy_vendor_command_flags - validation flags for vendor commands
* @WIPHY_VENDOR_CMD_NEED_WDEV: vendor command requires wdev
* @WIPHY_VENDOR_CMD_NEED_NETDEV: vendor command requires netdev
* @WIPHY_VENDOR_CMD_NEED_RUNNING: interface/wdev must be up & running
* (must be combined with %_WDEV or %_NETDEV)
*/
enum wiphy_vendor_command_flags {
WIPHY_VENDOR_CMD_NEED_WDEV = BIT(0),
WIPHY_VENDOR_CMD_NEED_NETDEV = BIT(1),
WIPHY_VENDOR_CMD_NEED_RUNNING = BIT(2),
};
/**
* enum wiphy_opmode_flag - Station's ht/vht operation mode information flags
*
* @STA_OPMODE_MAX_BW_CHANGED: Max Bandwidth changed
* @STA_OPMODE_SMPS_MODE_CHANGED: SMPS mode changed
* @STA_OPMODE_N_SS_CHANGED: max N_SS (number of spatial streams) changed
*
*/
enum wiphy_opmode_flag {
STA_OPMODE_MAX_BW_CHANGED = BIT(0),
STA_OPMODE_SMPS_MODE_CHANGED = BIT(1),
STA_OPMODE_N_SS_CHANGED = BIT(2),
};
/**
* struct sta_opmode_info - Station's ht/vht operation mode information
* @changed: contains value from &enum wiphy_opmode_flag
* @smps_mode: New SMPS mode value from &enum nl80211_smps_mode of a station
* @bw: new max bandwidth value from &enum nl80211_chan_width of a station
* @rx_nss: new rx_nss value of a station
*/
struct sta_opmode_info {
u32 changed;
enum nl80211_smps_mode smps_mode;
enum nl80211_chan_width bw;
u8 rx_nss;
};
#define VENDOR_CMD_RAW_DATA ((const struct nla_policy *)(long)(-ENODATA))
/**
* struct wiphy_vendor_command - vendor command definition
* @info: vendor command identifying information, as used in nl80211
* @flags: flags, see &enum wiphy_vendor_command_flags
* @doit: callback for the operation, note that wdev is %NULL if the
* flags didn't ask for a wdev and non-%NULL otherwise; the data
* pointer may be %NULL if userspace provided no data at all
* @dumpit: dump callback, for transferring bigger/multiple items. The
* @storage points to cb->args[5], ie. is preserved over the multiple
* dumpit calls.
* @policy: policy pointer for attributes within %NL80211_ATTR_VENDOR_DATA.
* Set this to %VENDOR_CMD_RAW_DATA if no policy can be given and the
* attribute is just raw data (e.g. a firmware command).
* @maxattr: highest attribute number in policy
* It's recommended to not have the same sub command with both @doit and
* @dumpit, so that userspace can assume certain ones are get and others
* are used with dump requests.
*/
struct wiphy_vendor_command {
struct nl80211_vendor_cmd_info info;
u32 flags;
int (*doit)(struct wiphy *wiphy, struct wireless_dev *wdev,
const void *data, int data_len);
int (*dumpit)(struct wiphy *wiphy, struct wireless_dev *wdev,
struct sk_buff *skb, const void *data, int data_len,
unsigned long *storage);
const struct nla_policy *policy;
unsigned int maxattr;
};
/**
* struct wiphy_iftype_ext_capab - extended capabilities per interface type
* @iftype: interface type
* @extended_capabilities: extended capabilities supported by the driver,
* additional capabilities might be supported by userspace; these are the
* 802.11 extended capabilities ("Extended Capabilities element") and are
* in the same format as in the information element. See IEEE Std
* 802.11-2012 8.4.2.29 for the defined fields.
* @extended_capabilities_mask: mask of the valid values
* @extended_capabilities_len: length of the extended capabilities
* @eml_capabilities: EML capabilities (for MLO)
* @mld_capa_and_ops: MLD capabilities and operations (for MLO)
*/
struct wiphy_iftype_ext_capab {
enum nl80211_iftype iftype;
const u8 *extended_capabilities;
const u8 *extended_capabilities_mask;
u8 extended_capabilities_len;
u16 eml_capabilities;
u16 mld_capa_and_ops;
};
/**
* cfg80211_get_iftype_ext_capa - lookup interface type extended capability
* @wiphy: the wiphy to look up from
* @type: the interface type to look up
*/
const struct wiphy_iftype_ext_capab *
cfg80211_get_iftype_ext_capa(struct wiphy *wiphy, enum nl80211_iftype type);
/**
* struct cfg80211_pmsr_capabilities - cfg80211 peer measurement capabilities
* @max_peers: maximum number of peers in a single measurement
* @report_ap_tsf: can report assoc AP's TSF for radio resource measurement
* @randomize_mac_addr: can randomize MAC address for measurement
* @ftm: FTM measurement data
* @ftm.supported: FTM measurement is supported
* @ftm.asap: ASAP-mode is supported
* @ftm.non_asap: non-ASAP-mode is supported
* @ftm.request_lci: can request LCI data
* @ftm.request_civicloc: can request civic location data
* @ftm.preambles: bitmap of preambles supported (&enum nl80211_preamble)
* @ftm.bandwidths: bitmap of bandwidths supported (&enum nl80211_chan_width)
* @ftm.max_bursts_exponent: maximum burst exponent supported
* (set to -1 if not limited; note that setting this will necessarily
* forbid using the value 15 to let the responder pick)
* @ftm.max_ftms_per_burst: maximum FTMs per burst supported (set to 0 if
* not limited)
* @ftm.trigger_based: trigger based ranging measurement is supported
* @ftm.non_trigger_based: non trigger based ranging measurement is supported
*/
struct cfg80211_pmsr_capabilities {
unsigned int max_peers;
u8 report_ap_tsf:1,
randomize_mac_addr:1;
struct {
u32 preambles;
u32 bandwidths;
s8 max_bursts_exponent;
u8 max_ftms_per_burst;
u8 supported:1,
asap:1,
non_asap:1,
request_lci:1,
request_civicloc:1,
trigger_based:1,
non_trigger_based:1;
} ftm;
};
/**
* struct wiphy_iftype_akm_suites - This structure encapsulates supported akm
* suites for interface types defined in @iftypes_mask. Each type in the
* @iftypes_mask must be unique across all instances of iftype_akm_suites.
*
* @iftypes_mask: bitmask of interfaces types
* @akm_suites: points to an array of supported akm suites
* @n_akm_suites: number of supported AKM suites
*/
struct wiphy_iftype_akm_suites {
u16 iftypes_mask;
const u32 *akm_suites;
int n_akm_suites;
};
#define CFG80211_HW_TIMESTAMP_ALL_PEERS 0xffff
/**
* struct wiphy - wireless hardware description
* @mtx: mutex for the data (structures) of this device
* @reg_notifier: the driver's regulatory notification callback,
* note that if your driver uses wiphy_apply_custom_regulatory()
* the reg_notifier's request can be passed as NULL
* @regd: the driver's regulatory domain, if one was requested via
* the regulatory_hint() API. This can be used by the driver
* on the reg_notifier() if it chooses to ignore future
* regulatory domain changes caused by other drivers.
* @signal_type: signal type reported in &struct cfg80211_bss.
* @cipher_suites: supported cipher suites
* @n_cipher_suites: number of supported cipher suites
* @akm_suites: supported AKM suites. These are the default AKMs supported if
* the supported AKMs not advertized for a specific interface type in
* iftype_akm_suites.
* @n_akm_suites: number of supported AKM suites
* @iftype_akm_suites: array of supported akm suites info per interface type.
* Note that the bits in @iftypes_mask inside this structure cannot
* overlap (i.e. only one occurrence of each type is allowed across all
* instances of iftype_akm_suites).
* @num_iftype_akm_suites: number of interface types for which supported akm
* suites are specified separately.
* @retry_short: Retry limit for short frames (dot11ShortRetryLimit)
* @retry_long: Retry limit for long frames (dot11LongRetryLimit)
* @frag_threshold: Fragmentation threshold (dot11FragmentationThreshold);
* -1 = fragmentation disabled, only odd values >= 256 used
* @rts_threshold: RTS threshold (dot11RTSThreshold); -1 = RTS/CTS disabled
* @_net: the network namespace this wiphy currently lives in
* @perm_addr: permanent MAC address of this device
* @addr_mask: If the device supports multiple MAC addresses by masking,
* set this to a mask with variable bits set to 1, e.g. if the last
* four bits are variable then set it to 00-00-00-00-00-0f. The actual
* variable bits shall be determined by the interfaces added, with
* interfaces not matching the mask being rejected to be brought up.
* @n_addresses: number of addresses in @addresses.
* @addresses: If the device has more than one address, set this pointer
* to a list of addresses (6 bytes each). The first one will be used
* by default for perm_addr. In this case, the mask should be set to
* all-zeroes. In this case it is assumed that the device can handle
* the same number of arbitrary MAC addresses.
* @registered: protects ->resume and ->suspend sysfs callbacks against
* unregister hardware
* @debugfsdir: debugfs directory used for this wiphy (ieee80211/<wiphyname>).
* It will be renamed automatically on wiphy renames
* @dev: (virtual) struct device for this wiphy. The item in
* /sys/class/ieee80211/ points to this. You need use set_wiphy_dev()
* (see below).
* @wext: wireless extension handlers
* @priv: driver private data (sized according to wiphy_new() parameter)
* @interface_modes: bitmask of interfaces types valid for this wiphy,
* must be set by driver
* @iface_combinations: Valid interface combinations array, should not
* list single interface types.
* @n_iface_combinations: number of entries in @iface_combinations array.
* @software_iftypes: bitmask of software interface types, these are not
* subject to any restrictions since they are purely managed in SW.
* @flags: wiphy flags, see &enum wiphy_flags
* @regulatory_flags: wiphy regulatory flags, see
* &enum ieee80211_regulatory_flags
* @features: features advertised to nl80211, see &enum nl80211_feature_flags.
* @ext_features: extended features advertised to nl80211, see
* &enum nl80211_ext_feature_index.
* @bss_priv_size: each BSS struct has private data allocated with it,
* this variable determines its size
* @max_scan_ssids: maximum number of SSIDs the device can scan for in
* any given scan
* @max_sched_scan_reqs: maximum number of scheduled scan requests that
* the device can run concurrently.
* @max_sched_scan_ssids: maximum number of SSIDs the device can scan
* for in any given scheduled scan
* @max_match_sets: maximum number of match sets the device can handle
* when performing a scheduled scan, 0 if filtering is not
* supported.
* @max_scan_ie_len: maximum length of user-controlled IEs device can
* add to probe request frames transmitted during a scan, must not
* include fixed IEs like supported rates
* @max_sched_scan_ie_len: same as max_scan_ie_len, but for scheduled
* scans
* @max_sched_scan_plans: maximum number of scan plans (scan interval and number
* of iterations) for scheduled scan supported by the device.
* @max_sched_scan_plan_interval: maximum interval (in seconds) for a
* single scan plan supported by the device.
* @max_sched_scan_plan_iterations: maximum number of iterations for a single
* scan plan supported by the device.
* @coverage_class: current coverage class
* @fw_version: firmware version for ethtool reporting
* @hw_version: hardware version for ethtool reporting
* @max_num_pmkids: maximum number of PMKIDs supported by device
* @privid: a pointer that drivers can use to identify if an arbitrary
* wiphy is theirs, e.g. in global notifiers
* @bands: information about bands/channels supported by this device
*
* @mgmt_stypes: bitmasks of frame subtypes that can be subscribed to or
* transmitted through nl80211, points to an array indexed by interface
* type
*
* @available_antennas_tx: bitmap of antennas which are available to be
* configured as TX antennas. Antenna configuration commands will be
* rejected unless this or @available_antennas_rx is set.
*
* @available_antennas_rx: bitmap of antennas which are available to be
* configured as RX antennas. Antenna configuration commands will be
* rejected unless this or @available_antennas_tx is set.
*
* @probe_resp_offload:
* Bitmap of supported protocols for probe response offloading.
* See &enum nl80211_probe_resp_offload_support_attr. Only valid
* when the wiphy flag @WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD is set.
*
* @max_remain_on_channel_duration: Maximum time a remain-on-channel operation
* may request, if implemented.
*
* @wowlan: WoWLAN support information
* @wowlan_config: current WoWLAN configuration; this should usually not be
* used since access to it is necessarily racy, use the parameter passed
* to the suspend() operation instead.
*
* @ap_sme_capa: AP SME capabilities, flags from &enum nl80211_ap_sme_features.
* @ht_capa_mod_mask: Specify what ht_cap values can be over-ridden.
* If null, then none can be over-ridden.
* @vht_capa_mod_mask: Specify what VHT capabilities can be over-ridden.
* If null, then none can be over-ridden.
*
* @wdev_list: the list of associated (virtual) interfaces; this list must
* not be modified by the driver, but can be read with RTNL/RCU protection.
*
* @max_acl_mac_addrs: Maximum number of MAC addresses that the device
* supports for ACL.
*
* @extended_capabilities: extended capabilities supported by the driver,
* additional capabilities might be supported by userspace; these are
* the 802.11 extended capabilities ("Extended Capabilities element")
* and are in the same format as in the information element. See
* 802.11-2012 8.4.2.29 for the defined fields. These are the default
* extended capabilities to be used if the capabilities are not specified
* for a specific interface type in iftype_ext_capab.
* @extended_capabilities_mask: mask of the valid values
* @extended_capabilities_len: length of the extended capabilities
* @iftype_ext_capab: array of extended capabilities per interface type
* @num_iftype_ext_capab: number of interface types for which extended
* capabilities are specified separately.
* @coalesce: packet coalescing support information
*
* @vendor_commands: array of vendor commands supported by the hardware
* @n_vendor_commands: number of vendor commands
* @vendor_events: array of vendor events supported by the hardware
* @n_vendor_events: number of vendor events
*
* @max_ap_assoc_sta: maximum number of associated stations supported in AP mode
* (including P2P GO) or 0 to indicate no such limit is advertised. The
* driver is allowed to advertise a theoretical limit that it can reach in
* some cases, but may not always reach.
*
* @max_num_csa_counters: Number of supported csa_counters in beacons
* and probe responses. This value should be set if the driver
* wishes to limit the number of csa counters. Default (0) means
* infinite.
* @bss_select_support: bitmask indicating the BSS selection criteria supported
* by the driver in the .connect() callback. The bit position maps to the
* attribute indices defined in &enum nl80211_bss_select_attr.
*
* @nan_supported_bands: bands supported by the device in NAN mode, a
* bitmap of &enum nl80211_band values. For instance, for
* NL80211_BAND_2GHZ, bit 0 would be set
* (i.e. BIT(NL80211_BAND_2GHZ)).
*
* @txq_limit: configuration of internal TX queue frame limit
* @txq_memory_limit: configuration internal TX queue memory limit
* @txq_quantum: configuration of internal TX queue scheduler quantum
*
* @tx_queue_len: allow setting transmit queue len for drivers not using
* wake_tx_queue
*
* @support_mbssid: can HW support association with nontransmitted AP
* @support_only_he_mbssid: don't parse MBSSID elements if it is not
* HE AP, in order to avoid compatibility issues.
* @support_mbssid must be set for this to have any effect.
*
* @pmsr_capa: peer measurement capabilities
*
* @tid_config_support: describes the per-TID config support that the
* device has
* @tid_config_support.vif: bitmap of attributes (configurations)
* supported by the driver for each vif
* @tid_config_support.peer: bitmap of attributes (configurations)
* supported by the driver for each peer
* @tid_config_support.max_retry: maximum supported retry count for
* long/short retry configuration
*
* @max_data_retry_count: maximum supported per TID retry count for
* configuration through the %NL80211_TID_CONFIG_ATTR_RETRY_SHORT and
* %NL80211_TID_CONFIG_ATTR_RETRY_LONG attributes
* @sar_capa: SAR control capabilities
* @rfkill: a pointer to the rfkill structure
*
* @mbssid_max_interfaces: maximum number of interfaces supported by the driver
* in a multiple BSSID set. This field must be set to a non-zero value
* by the driver to advertise MBSSID support.
* @ema_max_profile_periodicity: maximum profile periodicity supported by
* the driver. Setting this field to a non-zero value indicates that the
* driver supports enhanced multi-BSSID advertisements (EMA AP).
* @max_num_akm_suites: maximum number of AKM suites allowed for
* configuration through %NL80211_CMD_CONNECT, %NL80211_CMD_ASSOCIATE and
* %NL80211_CMD_START_AP. Set to NL80211_MAX_NR_AKM_SUITES if not set by
* driver. If set by driver minimum allowed value is
* NL80211_MAX_NR_AKM_SUITES in order to avoid compatibility issues with
* legacy userspace and maximum allowed value is
* CFG80211_MAX_NUM_AKM_SUITES.
*
* @hw_timestamp_max_peers: maximum number of peers that the driver supports
* enabling HW timestamping for concurrently. Setting this field to a
* non-zero value indicates that the driver supports HW timestamping.
* A value of %CFG80211_HW_TIMESTAMP_ALL_PEERS indicates the driver
* supports enabling HW timestamping for all peers (i.e. no need to
* specify a mac address).
*/
struct wiphy {
struct mutex mtx;
/* assign these fields before you register the wiphy */
u8 perm_addr[ETH_ALEN];
u8 addr_mask[ETH_ALEN];
struct mac_address *addresses;
const struct ieee80211_txrx_stypes *mgmt_stypes;
const struct ieee80211_iface_combination *iface_combinations;
int n_iface_combinations;
u16 software_iftypes;
u16 n_addresses;
/* Supported interface modes, OR together BIT(NL80211_IFTYPE_...) */
u16 interface_modes;
u16 max_acl_mac_addrs;
u32 flags, regulatory_flags, features;
u8 ext_features[DIV_ROUND_UP(NUM_NL80211_EXT_FEATURES, 8)];
u32 ap_sme_capa;
enum cfg80211_signal_type signal_type;
int bss_priv_size;
u8 max_scan_ssids;
u8 max_sched_scan_reqs;
u8 max_sched_scan_ssids;
u8 max_match_sets;
u16 max_scan_ie_len;
u16 max_sched_scan_ie_len;
u32 max_sched_scan_plans;
u32 max_sched_scan_plan_interval;
u32 max_sched_scan_plan_iterations;
int n_cipher_suites;
const u32 *cipher_suites;
int n_akm_suites;
const u32 *akm_suites;
const struct wiphy_iftype_akm_suites *iftype_akm_suites;
unsigned int num_iftype_akm_suites;
u8 retry_short;
u8 retry_long;
u32 frag_threshold;
u32 rts_threshold;
u8 coverage_class;
char fw_version[ETHTOOL_FWVERS_LEN];
u32 hw_version;
#ifdef CONFIG_PM
const struct wiphy_wowlan_support *wowlan;
struct cfg80211_wowlan *wowlan_config;
#endif
u16 max_remain_on_channel_duration;
u8 max_num_pmkids;
u32 available_antennas_tx;
u32 available_antennas_rx;
u32 probe_resp_offload;
const u8 *extended_capabilities, *extended_capabilities_mask;
u8 extended_capabilities_len;
const struct wiphy_iftype_ext_capab *iftype_ext_capab;
unsigned int num_iftype_ext_capab;
const void *privid;
struct ieee80211_supported_band *bands[NUM_NL80211_BANDS];
void (*reg_notifier)(struct wiphy *wiphy,
struct regulatory_request *request);
/* fields below are read-only, assigned by cfg80211 */
const struct ieee80211_regdomain __rcu *regd;
struct device dev;
bool registered;
struct dentry *debugfsdir;
const struct ieee80211_ht_cap *ht_capa_mod_mask;
const struct ieee80211_vht_cap *vht_capa_mod_mask;
struct list_head wdev_list;
possible_net_t _net;
#ifdef CONFIG_CFG80211_WEXT
const struct iw_handler_def *wext;
#endif
const struct wiphy_coalesce_support *coalesce;
const struct wiphy_vendor_command *vendor_commands;
const struct nl80211_vendor_cmd_info *vendor_events;
int n_vendor_commands, n_vendor_events;
u16 max_ap_assoc_sta;
u8 max_num_csa_counters;
u32 bss_select_support;
u8 nan_supported_bands;
u32 txq_limit;
u32 txq_memory_limit;
u32 txq_quantum;
unsigned long tx_queue_len;
u8 support_mbssid:1,
support_only_he_mbssid:1;
const struct cfg80211_pmsr_capabilities *pmsr_capa;
struct {
u64 peer, vif;
u8 max_retry;
} tid_config_support;
u8 max_data_retry_count;
const struct cfg80211_sar_capa *sar_capa;
struct rfkill *rfkill;
u8 mbssid_max_interfaces;
u8 ema_max_profile_periodicity;
u16 max_num_akm_suites;
u16 hw_timestamp_max_peers;
char priv[] __aligned(NETDEV_ALIGN);
};
static inline struct net *wiphy_net(struct wiphy *wiphy)
{
return read_pnet(&wiphy->_net);
}
static inline void wiphy_net_set(struct wiphy *wiphy, struct net *net)
{
write_pnet(&wiphy->_net, net);
}
/**
* wiphy_priv - return priv from wiphy
*
* @wiphy: the wiphy whose priv pointer to return
* Return: The priv of @wiphy.
*/
static inline void *wiphy_priv(struct wiphy *wiphy)
{
BUG_ON(!wiphy);
return &wiphy->priv;
}
/**
* priv_to_wiphy - return the wiphy containing the priv
*
* @priv: a pointer previously returned by wiphy_priv
* Return: The wiphy of @priv.
*/
static inline struct wiphy *priv_to_wiphy(void *priv)
{
BUG_ON(!priv);
return container_of(priv, struct wiphy, priv);
}
/**
* set_wiphy_dev - set device pointer for wiphy
*
* @wiphy: The wiphy whose device to bind
* @dev: The device to parent it to
*/
static inline void set_wiphy_dev(struct wiphy *wiphy, struct device *dev)
{
wiphy->dev.parent = dev;
}
/**
* wiphy_dev - get wiphy dev pointer
*
* @wiphy: The wiphy whose device struct to look up
* Return: The dev of @wiphy.
*/
static inline struct device *wiphy_dev(struct wiphy *wiphy)
{
return wiphy->dev.parent;
}
/**
* wiphy_name - get wiphy name
*
* @wiphy: The wiphy whose name to return
* Return: The name of @wiphy.
*/
static inline const char *wiphy_name(const struct wiphy *wiphy)
{
return dev_name(&wiphy->dev);
}
/**
* wiphy_new_nm - create a new wiphy for use with cfg80211
*
* @ops: The configuration operations for this device
* @sizeof_priv: The size of the private area to allocate
* @requested_name: Request a particular name.
* NULL is valid value, and means use the default phy%d naming.
*
* Create a new wiphy and associate the given operations with it.
* @sizeof_priv bytes are allocated for private use.
*
* Return: A pointer to the new wiphy. This pointer must be
* assigned to each netdev's ieee80211_ptr for proper operation.
*/
struct wiphy *wiphy_new_nm(const struct cfg80211_ops *ops, int sizeof_priv,
const char *requested_name);
/**
* wiphy_new - create a new wiphy for use with cfg80211
*
* @ops: The configuration operations for this device
* @sizeof_priv: The size of the private area to allocate
*
* Create a new wiphy and associate the given operations with it.
* @sizeof_priv bytes are allocated for private use.
*
* Return: A pointer to the new wiphy. This pointer must be
* assigned to each netdev's ieee80211_ptr for proper operation.
*/
static inline struct wiphy *wiphy_new(const struct cfg80211_ops *ops,
int sizeof_priv)
{
return wiphy_new_nm(ops, sizeof_priv, NULL);
}
/**
* wiphy_register - register a wiphy with cfg80211
*
* @wiphy: The wiphy to register.
*
* Return: A non-negative wiphy index or a negative error code.
*/
int wiphy_register(struct wiphy *wiphy);
/* this is a define for better error reporting (file/line) */
#define lockdep_assert_wiphy(wiphy) lockdep_assert_held(&(wiphy)->mtx)
/**
* rcu_dereference_wiphy - rcu_dereference with debug checking
* @wiphy: the wiphy to check the locking on
* @p: The pointer to read, prior to dereferencing
*
* Do an rcu_dereference(p), but check caller either holds rcu_read_lock()
* or RTNL. Note: Please prefer wiphy_dereference() or rcu_dereference().
*/
#define rcu_dereference_wiphy(wiphy, p) \
rcu_dereference_check(p, lockdep_is_held(&wiphy->mtx))
/**
* wiphy_dereference - fetch RCU pointer when updates are prevented by wiphy mtx
* @wiphy: the wiphy to check the locking on
* @p: The pointer to read, prior to dereferencing
*
* Return the value of the specified RCU-protected pointer, but omit the
* READ_ONCE(), because caller holds the wiphy mutex used for updates.
*/
#define wiphy_dereference(wiphy, p) \
rcu_dereference_protected(p, lockdep_is_held(&wiphy->mtx))
/**
* get_wiphy_regdom - get custom regdomain for the given wiphy
* @wiphy: the wiphy to get the regdomain from
*/
const struct ieee80211_regdomain *get_wiphy_regdom(struct wiphy *wiphy);
/**
* wiphy_unregister - deregister a wiphy from cfg80211
*
* @wiphy: The wiphy to unregister.
*
* After this call, no more requests can be made with this priv
* pointer, but the call may sleep to wait for an outstanding
* request that is being handled.
*/
void wiphy_unregister(struct wiphy *wiphy);
/**
* wiphy_free - free wiphy
*
* @wiphy: The wiphy to free
*/
void wiphy_free(struct wiphy *wiphy);
/* internal structs */
struct cfg80211_conn;
struct cfg80211_internal_bss;
struct cfg80211_cached_keys;
struct cfg80211_cqm_config;
/**
* wiphy_lock - lock the wiphy
* @wiphy: the wiphy to lock
*
* This is needed around registering and unregistering netdevs that
* aren't created through cfg80211 calls, since that requires locking
* in cfg80211 when the notifiers is called, but that cannot
* differentiate which way it's called.
*
* It can also be used by drivers for their own purposes.
*
* When cfg80211 ops are called, the wiphy is already locked.
*
* Note that this makes sure that no workers that have been queued
* with wiphy_queue_work() are running.
*/
static inline void wiphy_lock(struct wiphy *wiphy)
__acquires(&wiphy->mtx)
{
mutex_lock(&wiphy->mtx);
__acquire(&wiphy->mtx);
}
/**
* wiphy_unlock - unlock the wiphy again
* @wiphy: the wiphy to unlock
*/
static inline void wiphy_unlock(struct wiphy *wiphy)
__releases(&wiphy->mtx)
{
__release(&wiphy->mtx);
mutex_unlock(&wiphy->mtx);
}
struct wiphy_work;
typedef void (*wiphy_work_func_t)(struct wiphy *, struct wiphy_work *);
struct wiphy_work {
struct list_head entry;
wiphy_work_func_t func;
};
static inline void wiphy_work_init(struct wiphy_work *work,
wiphy_work_func_t func)
{
INIT_LIST_HEAD(&work->entry);
work->func = func;
}
/**
* wiphy_work_queue - queue work for the wiphy
* @wiphy: the wiphy to queue for
* @work: the work item
*
* This is useful for work that must be done asynchronously, and work
* queued here has the special property that the wiphy mutex will be
* held as if wiphy_lock() was called, and that it cannot be running
* after wiphy_lock() was called. Therefore, wiphy_cancel_work() can
* use just cancel_work() instead of cancel_work_sync(), it requires
* being in a section protected by wiphy_lock().
*/
void wiphy_work_queue(struct wiphy *wiphy, struct wiphy_work *work);
/**
* wiphy_work_cancel - cancel previously queued work
* @wiphy: the wiphy, for debug purposes
* @work: the work to cancel
*
* Cancel the work *without* waiting for it, this assumes being
* called under the wiphy mutex acquired by wiphy_lock().
*/
void wiphy_work_cancel(struct wiphy *wiphy, struct wiphy_work *work);
/**
* wiphy_work_flush - flush previously queued work
* @wiphy: the wiphy, for debug purposes
* @work: the work to flush, this can be %NULL to flush all work
*
* Flush the work (i.e. run it if pending). This must be called
* under the wiphy mutex acquired by wiphy_lock().
*/
void wiphy_work_flush(struct wiphy *wiphy, struct wiphy_work *work);
struct wiphy_delayed_work {
struct wiphy_work work;
struct wiphy *wiphy;
struct timer_list timer;
};
void wiphy_delayed_work_timer(struct timer_list *t);
static inline void wiphy_delayed_work_init(struct wiphy_delayed_work *dwork,
wiphy_work_func_t func)
{
timer_setup(&dwork->timer, wiphy_delayed_work_timer, 0);
wiphy_work_init(&dwork->work, func);
}
/**
* wiphy_delayed_work_queue - queue delayed work for the wiphy
* @wiphy: the wiphy to queue for
* @dwork: the delayable worker
* @delay: number of jiffies to wait before queueing
*
* This is useful for work that must be done asynchronously, and work
* queued here has the special property that the wiphy mutex will be
* held as if wiphy_lock() was called, and that it cannot be running
* after wiphy_lock() was called. Therefore, wiphy_cancel_work() can
* use just cancel_work() instead of cancel_work_sync(), it requires
* being in a section protected by wiphy_lock().
*/
void wiphy_delayed_work_queue(struct wiphy *wiphy,
struct wiphy_delayed_work *dwork,
unsigned long delay);
/**
* wiphy_delayed_work_cancel - cancel previously queued delayed work
* @wiphy: the wiphy, for debug purposes
* @dwork: the delayed work to cancel
*
* Cancel the work *without* waiting for it, this assumes being
* called under the wiphy mutex acquired by wiphy_lock().
*/
void wiphy_delayed_work_cancel(struct wiphy *wiphy,
struct wiphy_delayed_work *dwork);
/**
* wiphy_delayed_work_flush - flush previously queued delayed work
* @wiphy: the wiphy, for debug purposes
* @dwork: the delayed work to flush
*
* Flush the work (i.e. run it if pending). This must be called
* under the wiphy mutex acquired by wiphy_lock().
*/
void wiphy_delayed_work_flush(struct wiphy *wiphy,
struct wiphy_delayed_work *dwork);
/**
* struct wireless_dev - wireless device state
*
* For netdevs, this structure must be allocated by the driver
* that uses the ieee80211_ptr field in struct net_device (this
* is intentional so it can be allocated along with the netdev.)
* It need not be registered then as netdev registration will
* be intercepted by cfg80211 to see the new wireless device,
* however, drivers must lock the wiphy before registering or
* unregistering netdevs if they pre-create any netdevs (in ops
* called from cfg80211, the wiphy is already locked.)
*
* For non-netdev uses, it must also be allocated by the driver
* in response to the cfg80211 callbacks that require it, as
* there's no netdev registration in that case it may not be
* allocated outside of callback operations that return it.
*
* @wiphy: pointer to hardware description
* @iftype: interface type
* @registered: is this wdev already registered with cfg80211
* @registering: indicates we're doing registration under wiphy lock
* for the notifier
* @list: (private) Used to collect the interfaces
* @netdev: (private) Used to reference back to the netdev, may be %NULL
* @identifier: (private) Identifier used in nl80211 to identify this
* wireless device if it has no netdev
* @u: union containing data specific to @iftype
* @connected: indicates if connected or not (STA mode)
* @wext: (private) Used by the internal wireless extensions compat code
* @wext.ibss: (private) IBSS data part of wext handling
* @wext.connect: (private) connection handling data
* @wext.keys: (private) (WEP) key data
* @wext.ie: (private) extra elements for association
* @wext.ie_len: (private) length of extra elements
* @wext.bssid: (private) selected network BSSID
* @wext.ssid: (private) selected network SSID
* @wext.default_key: (private) selected default key index
* @wext.default_mgmt_key: (private) selected default management key index
* @wext.prev_bssid: (private) previous BSSID for reassociation
* @wext.prev_bssid_valid: (private) previous BSSID validity
* @use_4addr: indicates 4addr mode is used on this interface, must be
* set by driver (if supported) on add_interface BEFORE registering the
* netdev and may otherwise be used by driver read-only, will be update
* by cfg80211 on change_interface
* @mgmt_registrations: list of registrations for management frames
* @mgmt_registrations_need_update: mgmt registrations were updated,
* need to propagate the update to the driver
* @address: The address for this device, valid only if @netdev is %NULL
* @is_running: true if this is a non-netdev device that has been started, e.g.
* the P2P Device.
* @cac_started: true if DFS channel availability check has been started
* @cac_start_time: timestamp (jiffies) when the dfs state was entered.
* @cac_time_ms: CAC time in ms
* @ps: powersave mode is enabled
* @ps_timeout: dynamic powersave timeout
* @ap_unexpected_nlportid: (private) netlink port ID of application
* registered for unexpected class 3 frames (AP mode)
* @conn: (private) cfg80211 software SME connection state machine data
* @connect_keys: (private) keys to set after connection is established
* @conn_bss_type: connecting/connected BSS type
* @conn_owner_nlportid: (private) connection owner socket port ID
* @disconnect_wk: (private) auto-disconnect work
* @disconnect_bssid: (private) the BSSID to use for auto-disconnect
* @event_list: (private) list for internal event processing
* @event_lock: (private) lock for event list
* @owner_nlportid: (private) owner socket port ID
* @nl_owner_dead: (private) owner socket went away
* @cqm_rssi_work: (private) CQM RSSI reporting work
* @cqm_config: (private) nl80211 RSSI monitor state
* @pmsr_list: (private) peer measurement requests
* @pmsr_lock: (private) peer measurements requests/results lock
* @pmsr_free_wk: (private) peer measurements cleanup work
* @unprot_beacon_reported: (private) timestamp of last
* unprotected beacon report
* @links: array of %IEEE80211_MLD_MAX_NUM_LINKS elements containing @addr
* @ap and @client for each link
* @valid_links: bitmap describing what elements of @links are valid
*/
struct wireless_dev {
struct wiphy *wiphy;
enum nl80211_iftype iftype;
/* the remainder of this struct should be private to cfg80211 */
struct list_head list;
struct net_device *netdev;
u32 identifier;
struct list_head mgmt_registrations;
u8 mgmt_registrations_need_update:1;
bool use_4addr, is_running, registered, registering;
u8 address[ETH_ALEN] __aligned(sizeof(u16));
/* currently used for IBSS and SME - might be rearranged later */
struct cfg80211_conn *conn;
struct cfg80211_cached_keys *connect_keys;
enum ieee80211_bss_type conn_bss_type;
u32 conn_owner_nlportid;
struct work_struct disconnect_wk;
u8 disconnect_bssid[ETH_ALEN];
struct list_head event_list;
spinlock_t event_lock;
u8 connected:1;
bool ps;
int ps_timeout;
u32 ap_unexpected_nlportid;
u32 owner_nlportid;
bool nl_owner_dead;
/* FIXME: need to rework radar detection for MLO */
bool cac_started;
unsigned long cac_start_time;
unsigned int cac_time_ms;
#ifdef CONFIG_CFG80211_WEXT
/* wext data */
struct {
struct cfg80211_ibss_params ibss;
struct cfg80211_connect_params connect;
struct cfg80211_cached_keys *keys;
const u8 *ie;
size_t ie_len;
u8 bssid[ETH_ALEN];
u8 prev_bssid[ETH_ALEN];
u8 ssid[IEEE80211_MAX_SSID_LEN];
s8 default_key, default_mgmt_key;
bool prev_bssid_valid;
} wext;
#endif
struct wiphy_work cqm_rssi_work;
struct cfg80211_cqm_config __rcu *cqm_config;
struct list_head pmsr_list;
spinlock_t pmsr_lock;
struct work_struct pmsr_free_wk;
unsigned long unprot_beacon_reported;
union {
struct {
u8 connected_addr[ETH_ALEN] __aligned(2);
u8 ssid[IEEE80211_MAX_SSID_LEN];
u8 ssid_len;
} client;
struct {
int beacon_interval;
struct cfg80211_chan_def preset_chandef;
struct cfg80211_chan_def chandef;
u8 id[IEEE80211_MAX_MESH_ID_LEN];
u8 id_len, id_up_len;
} mesh;
struct {
struct cfg80211_chan_def preset_chandef;
u8 ssid[IEEE80211_MAX_SSID_LEN];
u8 ssid_len;
} ap;
struct {
struct cfg80211_internal_bss *current_bss;
struct cfg80211_chan_def chandef;
int beacon_interval;
u8 ssid[IEEE80211_MAX_SSID_LEN];
u8 ssid_len;
} ibss;
struct {
struct cfg80211_chan_def chandef;
} ocb;
} u;
struct {
u8 addr[ETH_ALEN] __aligned(2);
union {
struct {
unsigned int beacon_interval;
struct cfg80211_chan_def chandef;
} ap;
struct {
struct cfg80211_internal_bss *current_bss;
} client;
};
} links[IEEE80211_MLD_MAX_NUM_LINKS];
u16 valid_links;
};
static inline const u8 *wdev_address(struct wireless_dev *wdev)
{
if (wdev->netdev)
return wdev->netdev->dev_addr;
return wdev->address;
}
static inline bool wdev_running(struct wireless_dev *wdev)
{
if (wdev->netdev)
return netif_running(wdev->netdev);
return wdev->is_running;
}
/**
* wdev_priv - return wiphy priv from wireless_dev
*
* @wdev: The wireless device whose wiphy's priv pointer to return
* Return: The wiphy priv of @wdev.
*/
static inline void *wdev_priv(struct wireless_dev *wdev)
{
BUG_ON(!wdev);
return wiphy_priv(wdev->wiphy);
}
/**
* wdev_chandef - return chandef pointer from wireless_dev
* @wdev: the wdev
* @link_id: the link ID for MLO
*
* Return: The chandef depending on the mode, or %NULL.
*/
struct cfg80211_chan_def *wdev_chandef(struct wireless_dev *wdev,
unsigned int link_id);
static inline void WARN_INVALID_LINK_ID(struct wireless_dev *wdev,
unsigned int link_id)
{
WARN_ON(link_id && !wdev->valid_links);
WARN_ON(wdev->valid_links &&
!(wdev->valid_links & BIT(link_id)));
}
#define for_each_valid_link(link_info, link_id) \
for (link_id = 0; \
link_id < ((link_info)->valid_links ? \
ARRAY_SIZE((link_info)->links) : 1); \
link_id++) \
if (!(link_info)->valid_links || \
((link_info)->valid_links & BIT(link_id)))
/**
* DOC: Utility functions
*
* cfg80211 offers a number of utility functions that can be useful.
*/
/**
* ieee80211_channel_equal - compare two struct ieee80211_channel
*
* @a: 1st struct ieee80211_channel
* @b: 2nd struct ieee80211_channel
* Return: true if center frequency of @a == @b
*/
static inline bool
ieee80211_channel_equal(struct ieee80211_channel *a,
struct ieee80211_channel *b)
{
return (a->center_freq == b->center_freq &&
a->freq_offset == b->freq_offset);
}
/**
* ieee80211_channel_to_khz - convert ieee80211_channel to frequency in KHz
* @chan: struct ieee80211_channel to convert
* Return: The corresponding frequency (in KHz)
*/
static inline u32
ieee80211_channel_to_khz(const struct ieee80211_channel *chan)
{
return MHZ_TO_KHZ(chan->center_freq) + chan->freq_offset;
}
/**
* ieee80211_s1g_channel_width - get allowed channel width from @chan
*
* Only allowed for band NL80211_BAND_S1GHZ
* @chan: channel
* Return: The allowed channel width for this center_freq
*/
enum nl80211_chan_width
ieee80211_s1g_channel_width(const struct ieee80211_channel *chan);
/**
* ieee80211_channel_to_freq_khz - convert channel number to frequency
* @chan: channel number
* @band: band, necessary due to channel number overlap
* Return: The corresponding frequency (in KHz), or 0 if the conversion failed.
*/
u32 ieee80211_channel_to_freq_khz(int chan, enum nl80211_band band);
/**
* ieee80211_channel_to_frequency - convert channel number to frequency
* @chan: channel number
* @band: band, necessary due to channel number overlap
* Return: The corresponding frequency (in MHz), or 0 if the conversion failed.
*/
static inline int
ieee80211_channel_to_frequency(int chan, enum nl80211_band band)
{
return KHZ_TO_MHZ(ieee80211_channel_to_freq_khz(chan, band));
}
/**
* ieee80211_freq_khz_to_channel - convert frequency to channel number
* @freq: center frequency in KHz
* Return: The corresponding channel, or 0 if the conversion failed.
*/
int ieee80211_freq_khz_to_channel(u32 freq);
/**
* ieee80211_frequency_to_channel - convert frequency to channel number
* @freq: center frequency in MHz
* Return: The corresponding channel, or 0 if the conversion failed.
*/
static inline int
ieee80211_frequency_to_channel(int freq)
{
return ieee80211_freq_khz_to_channel(MHZ_TO_KHZ(freq));
}
/**
* ieee80211_get_channel_khz - get channel struct from wiphy for specified
* frequency
* @wiphy: the struct wiphy to get the channel for
* @freq: the center frequency (in KHz) of the channel
* Return: The channel struct from @wiphy at @freq.
*/
struct ieee80211_channel *
ieee80211_get_channel_khz(struct wiphy *wiphy, u32 freq);
/**
* ieee80211_get_channel - get channel struct from wiphy for specified frequency
*
* @wiphy: the struct wiphy to get the channel for
* @freq: the center frequency (in MHz) of the channel
* Return: The channel struct from @wiphy at @freq.
*/
static inline struct ieee80211_channel *
ieee80211_get_channel(struct wiphy *wiphy, int freq)
{
return ieee80211_get_channel_khz(wiphy, MHZ_TO_KHZ(freq));
}
/**
* cfg80211_channel_is_psc - Check if the channel is a 6 GHz PSC
* @chan: control channel to check
*
* The Preferred Scanning Channels (PSC) are defined in
* Draft IEEE P802.11ax/D5.0, 26.17.2.3.3
*/
static inline bool cfg80211_channel_is_psc(struct ieee80211_channel *chan)
{
if (chan->band != NL80211_BAND_6GHZ)
return false;
return ieee80211_frequency_to_channel(chan->center_freq) % 16 == 5;
}
/**
* ieee80211_get_response_rate - get basic rate for a given rate
*
* @sband: the band to look for rates in
* @basic_rates: bitmap of basic rates
* @bitrate: the bitrate for which to find the basic rate
*
* Return: The basic rate corresponding to a given bitrate, that
* is the next lower bitrate contained in the basic rate map,
* which is, for this function, given as a bitmap of indices of
* rates in the band's bitrate table.
*/
const struct ieee80211_rate *
ieee80211_get_response_rate(struct ieee80211_supported_band *sband,
u32 basic_rates, int bitrate);
/**
* ieee80211_mandatory_rates - get mandatory rates for a given band
* @sband: the band to look for rates in
*
* This function returns a bitmap of the mandatory rates for the given
* band, bits are set according to the rate position in the bitrates array.
*/
u32 ieee80211_mandatory_rates(struct ieee80211_supported_band *sband);
/*
* Radiotap parsing functions -- for controlled injection support
*
* Implemented in net/wireless/radiotap.c
* Documentation in Documentation/networking/radiotap-headers.rst
*/
struct radiotap_align_size {
uint8_t align:4, size:4;
};
struct ieee80211_radiotap_namespace {
const struct radiotap_align_size *align_size;
int n_bits;
uint32_t oui;
uint8_t subns;
};
struct ieee80211_radiotap_vendor_namespaces {
const struct ieee80211_radiotap_namespace *ns;
int n_ns;
};
/**
* struct ieee80211_radiotap_iterator - tracks walk thru present radiotap args
* @this_arg_index: index of current arg, valid after each successful call
* to ieee80211_radiotap_iterator_next()
* @this_arg: pointer to current radiotap arg; it is valid after each
* call to ieee80211_radiotap_iterator_next() but also after
* ieee80211_radiotap_iterator_init() where it will point to
* the beginning of the actual data portion
* @this_arg_size: length of the current arg, for convenience
* @current_namespace: pointer to the current namespace definition
* (or internally %NULL if the current namespace is unknown)
* @is_radiotap_ns: indicates whether the current namespace is the default
* radiotap namespace or not
*
* @_rtheader: pointer to the radiotap header we are walking through
* @_max_length: length of radiotap header in cpu byte ordering
* @_arg_index: next argument index
* @_arg: next argument pointer
* @_next_bitmap: internal pointer to next present u32
* @_bitmap_shifter: internal shifter for curr u32 bitmap, b0 set == arg present
* @_vns: vendor namespace definitions
* @_next_ns_data: beginning of the next namespace's data
* @_reset_on_ext: internal; reset the arg index to 0 when going to the
* next bitmap word
*
* Describes the radiotap parser state. Fields prefixed with an underscore
* must not be used by users of the parser, only by the parser internally.
*/
struct ieee80211_radiotap_iterator {
struct ieee80211_radiotap_header *_rtheader;
const struct ieee80211_radiotap_vendor_namespaces *_vns;
const struct ieee80211_radiotap_namespace *current_namespace;
unsigned char *_arg, *_next_ns_data;
__le32 *_next_bitmap;
unsigned char *this_arg;
int this_arg_index;
int this_arg_size;
int is_radiotap_ns;
int _max_length;
int _arg_index;
uint32_t _bitmap_shifter;
int _reset_on_ext;
};
int
ieee80211_radiotap_iterator_init(struct ieee80211_radiotap_iterator *iterator,
struct ieee80211_radiotap_header *radiotap_header,
int max_length,
const struct ieee80211_radiotap_vendor_namespaces *vns);
int
ieee80211_radiotap_iterator_next(struct ieee80211_radiotap_iterator *iterator);
extern const unsigned char rfc1042_header[6];
extern const unsigned char bridge_tunnel_header[6];
/**
* ieee80211_get_hdrlen_from_skb - get header length from data
*
* @skb: the frame
*
* Given an skb with a raw 802.11 header at the data pointer this function
* returns the 802.11 header length.
*
* Return: The 802.11 header length in bytes (not including encryption
* headers). Or 0 if the data in the sk_buff is too short to contain a valid
* 802.11 header.
*/
unsigned int ieee80211_get_hdrlen_from_skb(const struct sk_buff *skb);
/**
* ieee80211_hdrlen - get header length in bytes from frame control
* @fc: frame control field in little-endian format
* Return: The header length in bytes.
*/
unsigned int __attribute_const__ ieee80211_hdrlen(__le16 fc);
/**
* ieee80211_get_mesh_hdrlen - get mesh extension header length
* @meshhdr: the mesh extension header, only the flags field
* (first byte) will be accessed
* Return: The length of the extension header, which is always at
* least 6 bytes and at most 18 if address 5 and 6 are present.
*/
unsigned int ieee80211_get_mesh_hdrlen(struct ieee80211s_hdr *meshhdr);
/**
* DOC: Data path helpers
*
* In addition to generic utilities, cfg80211 also offers
* functions that help implement the data path for devices
* that do not do the 802.11/802.3 conversion on the device.
*/
/**
* ieee80211_data_to_8023_exthdr - convert an 802.11 data frame to 802.3
* @skb: the 802.11 data frame
* @ehdr: pointer to a &struct ethhdr that will get the header, instead
* of it being pushed into the SKB
* @addr: the device MAC address
* @iftype: the virtual interface type
* @data_offset: offset of payload after the 802.11 header
* @is_amsdu: true if the 802.11 header is A-MSDU
* Return: 0 on success. Non-zero on error.
*/
int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr,
const u8 *addr, enum nl80211_iftype iftype,
u8 data_offset, bool is_amsdu);
/**
* ieee80211_data_to_8023 - convert an 802.11 data frame to 802.3
* @skb: the 802.11 data frame
* @addr: the device MAC address
* @iftype: the virtual interface type
* Return: 0 on success. Non-zero on error.
*/
static inline int ieee80211_data_to_8023(struct sk_buff *skb, const u8 *addr,
enum nl80211_iftype iftype)
{
return ieee80211_data_to_8023_exthdr(skb, NULL, addr, iftype, 0, false);
}
/**
* ieee80211_is_valid_amsdu - check if subframe lengths of an A-MSDU are valid
*
* This is used to detect non-standard A-MSDU frames, e.g. the ones generated
* by ath10k and ath11k, where the subframe length includes the length of the
* mesh control field.
*
* @skb: The input A-MSDU frame without any headers.
* @mesh_hdr: the type of mesh header to test
* 0: non-mesh A-MSDU length field
* 1: big-endian mesh A-MSDU length field
* 2: little-endian mesh A-MSDU length field
* Returns: true if subframe header lengths are valid for the @mesh_hdr mode
*/
bool ieee80211_is_valid_amsdu(struct sk_buff *skb, u8 mesh_hdr);
/**
* ieee80211_amsdu_to_8023s - decode an IEEE 802.11n A-MSDU frame
*
* Decode an IEEE 802.11 A-MSDU and convert it to a list of 802.3 frames.
* The @list will be empty if the decode fails. The @skb must be fully
* header-less before being passed in here; it is freed in this function.
*
* @skb: The input A-MSDU frame without any headers.
* @list: The output list of 802.3 frames. It must be allocated and
* initialized by the caller.
* @addr: The device MAC address.
* @iftype: The device interface type.
* @extra_headroom: The hardware extra headroom for SKBs in the @list.
* @check_da: DA to check in the inner ethernet header, or NULL
* @check_sa: SA to check in the inner ethernet header, or NULL
* @mesh_control: see mesh_hdr in ieee80211_is_valid_amsdu
*/
void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list,
const u8 *addr, enum nl80211_iftype iftype,
const unsigned int extra_headroom,
const u8 *check_da, const u8 *check_sa,
u8 mesh_control);
/**
* ieee80211_get_8023_tunnel_proto - get RFC1042 or bridge tunnel encap protocol
*
* Check for RFC1042 or bridge tunnel header and fetch the encapsulated
* protocol.
*
* @hdr: pointer to the MSDU payload
* @proto: destination pointer to store the protocol
* Return: true if encapsulation was found
*/
bool ieee80211_get_8023_tunnel_proto(const void *hdr, __be16 *proto);
/**
* ieee80211_strip_8023_mesh_hdr - strip mesh header from converted 802.3 frames
*
* Strip the mesh header, which was left in by ieee80211_data_to_8023 as part
* of the MSDU data. Also move any source/destination addresses from the mesh
* header to the ethernet header (if present).
*
* @skb: The 802.3 frame with embedded mesh header
*/
int ieee80211_strip_8023_mesh_hdr(struct sk_buff *skb);
/**
* cfg80211_classify8021d - determine the 802.1p/1d tag for a data frame
* @skb: the data frame
* @qos_map: Interworking QoS mapping or %NULL if not in use
* Return: The 802.1p/1d tag.
*/
unsigned int cfg80211_classify8021d(struct sk_buff *skb,
struct cfg80211_qos_map *qos_map);
/**
* cfg80211_find_elem_match - match information element and byte array in data
*
* @eid: element ID
* @ies: data consisting of IEs
* @len: length of data
* @match: byte array to match
* @match_len: number of bytes in the match array
* @match_offset: offset in the IE data where the byte array should match.
* Note the difference to cfg80211_find_ie_match() which considers
* the offset to start from the element ID byte, but here we take
* the data portion instead.
*
* Return: %NULL if the element ID could not be found or if
* the element is invalid (claims to be longer than the given
* data) or if the byte array doesn't match; otherwise return the
* requested element struct.
*
* Note: There are no checks on the element length other than
* having to fit into the given data and being large enough for the
* byte array to match.
*/
const struct element *
cfg80211_find_elem_match(u8 eid, const u8 *ies, unsigned int len,
const u8 *match, unsigned int match_len,
unsigned int match_offset);
/**
* cfg80211_find_ie_match - match information element and byte array in data
*
* @eid: element ID
* @ies: data consisting of IEs
* @len: length of data
* @match: byte array to match
* @match_len: number of bytes in the match array
* @match_offset: offset in the IE where the byte array should match.
* If match_len is zero, this must also be set to zero.
* Otherwise this must be set to 2 or more, because the first
* byte is the element id, which is already compared to eid, and
* the second byte is the IE length.
*
* Return: %NULL if the element ID could not be found or if
* the element is invalid (claims to be longer than the given
* data) or if the byte array doesn't match, or a pointer to the first
* byte of the requested element, that is the byte containing the
* element ID.
*
* Note: There are no checks on the element length other than
* having to fit into the given data and being large enough for the
* byte array to match.
*/
static inline const u8 *
cfg80211_find_ie_match(u8 eid, const u8 *ies, unsigned int len,
const u8 *match, unsigned int match_len,
unsigned int match_offset)
{
/* match_offset can't be smaller than 2, unless match_len is
* zero, in which case match_offset must be zero as well.
*/
if (WARN_ON((match_len && match_offset < 2) ||
(!match_len && match_offset)))
return NULL;
return (const void *)cfg80211_find_elem_match(eid, ies, len,
match, match_len,
match_offset ?
match_offset - 2 : 0);
}
/**
* cfg80211_find_elem - find information element in data
*
* @eid: element ID
* @ies: data consisting of IEs
* @len: length of data
*
* Return: %NULL if the element ID could not be found or if
* the element is invalid (claims to be longer than the given
* data) or if the byte array doesn't match; otherwise return the
* requested element struct.
*
* Note: There are no checks on the element length other than
* having to fit into the given data.
*/
static inline const struct element *
cfg80211_find_elem(u8 eid, const u8 *ies, int len)
{
return cfg80211_find_elem_match(eid, ies, len, NULL, 0, 0);
}
/**
* cfg80211_find_ie - find information element in data
*
* @eid: element ID
* @ies: data consisting of IEs
* @len: length of data
*
* Return: %NULL if the element ID could not be found or if
* the element is invalid (claims to be longer than the given
* data), or a pointer to the first byte of the requested
* element, that is the byte containing the element ID.
*
* Note: There are no checks on the element length other than
* having to fit into the given data.
*/
static inline const u8 *cfg80211_find_ie(u8 eid, const u8 *ies, int len)
{
return cfg80211_find_ie_match(eid, ies, len, NULL, 0, 0);
}
/**
* cfg80211_find_ext_elem - find information element with EID Extension in data
*
* @ext_eid: element ID Extension
* @ies: data consisting of IEs
* @len: length of data
*
* Return: %NULL if the extended element could not be found or if
* the element is invalid (claims to be longer than the given
* data) or if the byte array doesn't match; otherwise return the
* requested element struct.
*
* Note: There are no checks on the element length other than
* having to fit into the given data.
*/
static inline const struct element *
cfg80211_find_ext_elem(u8 ext_eid, const u8 *ies, int len)
{
return cfg80211_find_elem_match(WLAN_EID_EXTENSION, ies, len,
&ext_eid, 1, 0);
}
/**
* cfg80211_find_ext_ie - find information element with EID Extension in data
*
* @ext_eid: element ID Extension
* @ies: data consisting of IEs
* @len: length of data
*
* Return: %NULL if the extended element ID could not be found or if
* the element is invalid (claims to be longer than the given
* data), or a pointer to the first byte of the requested
* element, that is the byte containing the element ID.
*
* Note: There are no checks on the element length other than
* having to fit into the given data.
*/
static inline const u8 *cfg80211_find_ext_ie(u8 ext_eid, const u8 *ies, int len)
{
return cfg80211_find_ie_match(WLAN_EID_EXTENSION, ies, len,
&ext_eid, 1, 2);
}
/**
* cfg80211_find_vendor_elem - find vendor specific information element in data
*
* @oui: vendor OUI
* @oui_type: vendor-specific OUI type (must be < 0xff), negative means any
* @ies: data consisting of IEs
* @len: length of data
*
* Return: %NULL if the vendor specific element ID could not be found or if the
* element is invalid (claims to be longer than the given data); otherwise
* return the element structure for the requested element.
*
* Note: There are no checks on the element length other than having to fit into
* the given data.
*/
const struct element *cfg80211_find_vendor_elem(unsigned int oui, int oui_type,
const u8 *ies,
unsigned int len);
/**
* cfg80211_find_vendor_ie - find vendor specific information element in data
*
* @oui: vendor OUI
* @oui_type: vendor-specific OUI type (must be < 0xff), negative means any
* @ies: data consisting of IEs
* @len: length of data
*
* Return: %NULL if the vendor specific element ID could not be found or if the
* element is invalid (claims to be longer than the given data), or a pointer to
* the first byte of the requested element, that is the byte containing the
* element ID.
*
* Note: There are no checks on the element length other than having to fit into
* the given data.
*/
static inline const u8 *
cfg80211_find_vendor_ie(unsigned int oui, int oui_type,
const u8 *ies, unsigned int len)
{
return (const void *)cfg80211_find_vendor_elem(oui, oui_type, ies, len);
}
/**
* enum cfg80211_rnr_iter_ret - reduced neighbor report iteration state
* @RNR_ITER_CONTINUE: continue iterating with the next entry
* @RNR_ITER_BREAK: break iteration and return success
* @RNR_ITER_ERROR: break iteration and return error
*/
enum cfg80211_rnr_iter_ret {
RNR_ITER_CONTINUE,
RNR_ITER_BREAK,
RNR_ITER_ERROR,
};
/**
* cfg80211_iter_rnr - iterate reduced neighbor report entries
* @elems: the frame elements to iterate RNR elements and then
* their entries in
* @elems_len: length of the elements
* @iter: iteration function, see also &enum cfg80211_rnr_iter_ret
* for the return value
* @iter_data: additional data passed to the iteration function
* Return: %true on success (after successfully iterating all entries
* or if the iteration function returned %RNR_ITER_BREAK),
* %false on error (iteration function returned %RNR_ITER_ERROR
* or elements were malformed.)
*/
bool cfg80211_iter_rnr(const u8 *elems, size_t elems_len,
enum cfg80211_rnr_iter_ret
(*iter)(void *data, u8 type,
const struct ieee80211_neighbor_ap_info *info,
const u8 *tbtt_info, u8 tbtt_info_len),
void *iter_data);
/**
* cfg80211_defragment_element - Defrag the given element data into a buffer
*
* @elem: the element to defragment
* @ies: elements where @elem is contained
* @ieslen: length of @ies
* @data: buffer to store element data, or %NULL to just determine size
* @data_len: length of @data, or 0
* @frag_id: the element ID of fragments
*
* Return: length of @data, or -EINVAL on error
*
* Copy out all data from an element that may be fragmented into @data, while
* skipping all headers.
*
* The function uses memmove() internally. It is acceptable to defragment an
* element in-place.
*/
ssize_t cfg80211_defragment_element(const struct element *elem, const u8 *ies,
size_t ieslen, u8 *data, size_t data_len,
u8 frag_id);
/**
* cfg80211_send_layer2_update - send layer 2 update frame
*
* @dev: network device
* @addr: STA MAC address
*
* Wireless drivers can use this function to update forwarding tables in bridge
* devices upon STA association.
*/
void cfg80211_send_layer2_update(struct net_device *dev, const u8 *addr);
/**
* DOC: Regulatory enforcement infrastructure
*
* TODO
*/
/**
* regulatory_hint - driver hint to the wireless core a regulatory domain
* @wiphy: the wireless device giving the hint (used only for reporting
* conflicts)
* @alpha2: the ISO/IEC 3166 alpha2 the driver claims its regulatory domain
* should be in. If @rd is set this should be NULL. Note that if you
* set this to NULL you should still set rd->alpha2 to some accepted
* alpha2.
*
* Wireless drivers can use this function to hint to the wireless core
* what it believes should be the current regulatory domain by
* giving it an ISO/IEC 3166 alpha2 country code it knows its regulatory
* domain should be in or by providing a completely build regulatory domain.
* If the driver provides an ISO/IEC 3166 alpha2 userspace will be queried
* for a regulatory domain structure for the respective country.
*
* The wiphy must have been registered to cfg80211 prior to this call.
* For cfg80211 drivers this means you must first use wiphy_register(),
* for mac80211 drivers you must first use ieee80211_register_hw().
*
* Drivers should check the return value, its possible you can get
* an -ENOMEM.
*
* Return: 0 on success. -ENOMEM.
*/
int regulatory_hint(struct wiphy *wiphy, const char *alpha2);
/**
* regulatory_set_wiphy_regd - set regdom info for self managed drivers
* @wiphy: the wireless device we want to process the regulatory domain on
* @rd: the regulatory domain information to use for this wiphy
*
* Set the regulatory domain information for self-managed wiphys, only they
* may use this function. See %REGULATORY_WIPHY_SELF_MANAGED for more
* information.
*
* Return: 0 on success. -EINVAL, -EPERM
*/
int regulatory_set_wiphy_regd(struct wiphy *wiphy,
struct ieee80211_regdomain *rd);
/**
* regulatory_set_wiphy_regd_sync - set regdom for self-managed drivers
* @wiphy: the wireless device we want to process the regulatory domain on
* @rd: the regulatory domain information to use for this wiphy
*
* This functions requires the RTNL and the wiphy mutex to be held and
* applies the new regdomain synchronously to this wiphy. For more details
* see regulatory_set_wiphy_regd().
*
* Return: 0 on success. -EINVAL, -EPERM
*/
int regulatory_set_wiphy_regd_sync(struct wiphy *wiphy,
struct ieee80211_regdomain *rd);
/**
* wiphy_apply_custom_regulatory - apply a custom driver regulatory domain
* @wiphy: the wireless device we want to process the regulatory domain on
* @regd: the custom regulatory domain to use for this wiphy
*
* Drivers can sometimes have custom regulatory domains which do not apply
* to a specific country. Drivers can use this to apply such custom regulatory
* domains. This routine must be called prior to wiphy registration. The
* custom regulatory domain will be trusted completely and as such previous
* default channel settings will be disregarded. If no rule is found for a
* channel on the regulatory domain the channel will be disabled.
* Drivers using this for a wiphy should also set the wiphy flag
* REGULATORY_CUSTOM_REG or cfg80211 will set it for the wiphy
* that called this helper.
*/
void wiphy_apply_custom_regulatory(struct wiphy *wiphy,
const struct ieee80211_regdomain *regd);
/**
* freq_reg_info - get regulatory information for the given frequency
* @wiphy: the wiphy for which we want to process this rule for
* @center_freq: Frequency in KHz for which we want regulatory information for
*
* Use this function to get the regulatory rule for a specific frequency on
* a given wireless device. If the device has a specific regulatory domain
* it wants to follow we respect that unless a country IE has been received
* and processed already.
*
* Return: A valid pointer, or, when an error occurs, for example if no rule
* can be found, the return value is encoded using ERR_PTR(). Use IS_ERR() to
* check and PTR_ERR() to obtain the numeric return value. The numeric return
* value will be -ERANGE if we determine the given center_freq does not even
* have a regulatory rule for a frequency range in the center_freq's band.
* See freq_in_rule_band() for our current definition of a band -- this is
* purely subjective and right now it's 802.11 specific.
*/
const struct ieee80211_reg_rule *freq_reg_info(struct wiphy *wiphy,
u32 center_freq);
/**
* reg_initiator_name - map regulatory request initiator enum to name
* @initiator: the regulatory request initiator
*
* You can use this to map the regulatory request initiator enum to a
* proper string representation.
*/
const char *reg_initiator_name(enum nl80211_reg_initiator initiator);
/**
* regulatory_pre_cac_allowed - check if pre-CAC allowed in the current regdom
* @wiphy: wiphy for which pre-CAC capability is checked.
*
* Pre-CAC is allowed only in some regdomains (notable ETSI).
*/
bool regulatory_pre_cac_allowed(struct wiphy *wiphy);
/**
* DOC: Internal regulatory db functions
*
*/
/**
* reg_query_regdb_wmm - Query internal regulatory db for wmm rule
* Regulatory self-managed driver can use it to proactively
*
* @alpha2: the ISO/IEC 3166 alpha2 wmm rule to be queried.
* @freq: the frequency (in MHz) to be queried.
* @rule: pointer to store the wmm rule from the regulatory db.
*
* Self-managed wireless drivers can use this function to query
* the internal regulatory database to check whether the given
* ISO/IEC 3166 alpha2 country and freq have wmm rule limitations.
*
* Drivers should check the return value, its possible you can get
* an -ENODATA.
*
* Return: 0 on success. -ENODATA.
*/
int reg_query_regdb_wmm(char *alpha2, int freq,
struct ieee80211_reg_rule *rule);
/*
* callbacks for asynchronous cfg80211 methods, notification
* functions and BSS handling helpers
*/
/**
* cfg80211_scan_done - notify that scan finished
*
* @request: the corresponding scan request
* @info: information about the completed scan
*/
void cfg80211_scan_done(struct cfg80211_scan_request *request,
struct cfg80211_scan_info *info);
/**
* cfg80211_sched_scan_results - notify that new scan results are available
*
* @wiphy: the wiphy which got scheduled scan results
* @reqid: identifier for the related scheduled scan request
*/
void cfg80211_sched_scan_results(struct wiphy *wiphy, u64 reqid);
/**
* cfg80211_sched_scan_stopped - notify that the scheduled scan has stopped
*
* @wiphy: the wiphy on which the scheduled scan stopped
* @reqid: identifier for the related scheduled scan request
*
* The driver can call this function to inform cfg80211 that the
* scheduled scan had to be stopped, for whatever reason. The driver
* is then called back via the sched_scan_stop operation when done.
*/
void cfg80211_sched_scan_stopped(struct wiphy *wiphy, u64 reqid);
/**
* cfg80211_sched_scan_stopped_locked - notify that the scheduled scan has stopped
*
* @wiphy: the wiphy on which the scheduled scan stopped
* @reqid: identifier for the related scheduled scan request
*
* The driver can call this function to inform cfg80211 that the
* scheduled scan had to be stopped, for whatever reason. The driver
* is then called back via the sched_scan_stop operation when done.
* This function should be called with the wiphy mutex held.
*/
void cfg80211_sched_scan_stopped_locked(struct wiphy *wiphy, u64 reqid);
/**
* cfg80211_inform_bss_frame_data - inform cfg80211 of a received BSS frame
* @wiphy: the wiphy reporting the BSS
* @data: the BSS metadata
* @mgmt: the management frame (probe response or beacon)
* @len: length of the management frame
* @gfp: context flags
*
* This informs cfg80211 that BSS information was found and
* the BSS should be updated/added.
*
* Return: A referenced struct, must be released with cfg80211_put_bss()!
* Or %NULL on error.
*/
struct cfg80211_bss * __must_check
cfg80211_inform_bss_frame_data(struct wiphy *wiphy,
struct cfg80211_inform_bss *data,
struct ieee80211_mgmt *mgmt, size_t len,
gfp_t gfp);
static inline struct cfg80211_bss * __must_check
cfg80211_inform_bss_frame(struct wiphy *wiphy,
struct ieee80211_channel *rx_channel,
struct ieee80211_mgmt *mgmt, size_t len,
s32 signal, gfp_t gfp)
{
struct cfg80211_inform_bss data = {
.chan = rx_channel,
.signal = signal,
};
return cfg80211_inform_bss_frame_data(wiphy, &data, mgmt, len, gfp);
}
/**
* cfg80211_gen_new_bssid - generate a nontransmitted BSSID for multi-BSSID
* @bssid: transmitter BSSID
* @max_bssid: max BSSID indicator, taken from Multiple BSSID element
* @mbssid_index: BSSID index, taken from Multiple BSSID index element
* @new_bssid: calculated nontransmitted BSSID
*/
static inline void cfg80211_gen_new_bssid(const u8 *bssid, u8 max_bssid,
u8 mbssid_index, u8 *new_bssid)
{
u64 bssid_u64 = ether_addr_to_u64(bssid);
u64 mask = GENMASK_ULL(max_bssid - 1, 0);
u64 new_bssid_u64;
new_bssid_u64 = bssid_u64 & ~mask;
new_bssid_u64 |= ((bssid_u64 & mask) + mbssid_index) & mask;
u64_to_ether_addr(new_bssid_u64, new_bssid);
}
/**
* cfg80211_is_element_inherited - returns if element ID should be inherited
* @element: element to check
* @non_inherit_element: non inheritance element
*/
bool cfg80211_is_element_inherited(const struct element *element,
const struct element *non_inherit_element);
/**
* cfg80211_merge_profile - merges a MBSSID profile if it is split between IEs
* @ie: ies
* @ielen: length of IEs
* @mbssid_elem: current MBSSID element
* @sub_elem: current MBSSID subelement (profile)
* @merged_ie: location of the merged profile
* @max_copy_len: max merged profile length
*/
size_t cfg80211_merge_profile(const u8 *ie, size_t ielen,
const struct element *mbssid_elem,
const struct element *sub_elem,
u8 *merged_ie, size_t max_copy_len);
/**
* enum cfg80211_bss_frame_type - frame type that the BSS data came from
* @CFG80211_BSS_FTYPE_UNKNOWN: driver doesn't know whether the data is
* from a beacon or probe response
* @CFG80211_BSS_FTYPE_BEACON: data comes from a beacon
* @CFG80211_BSS_FTYPE_PRESP: data comes from a probe response
* @CFG80211_BSS_FTYPE_S1G_BEACON: data comes from an S1G beacon
*/
enum cfg80211_bss_frame_type {
CFG80211_BSS_FTYPE_UNKNOWN,
CFG80211_BSS_FTYPE_BEACON,
CFG80211_BSS_FTYPE_PRESP,
CFG80211_BSS_FTYPE_S1G_BEACON,
};
/**
* cfg80211_get_ies_channel_number - returns the channel number from ies
* @ie: IEs
* @ielen: length of IEs
* @band: enum nl80211_band of the channel
*
* Returns the channel number, or -1 if none could be determined.
*/
int cfg80211_get_ies_channel_number(const u8 *ie, size_t ielen,
enum nl80211_band band);
/**
* cfg80211_ssid_eq - compare two SSIDs
* @a: first SSID
* @b: second SSID
*
* Return: %true if SSIDs are equal, %false otherwise.
*/
static inline bool
cfg80211_ssid_eq(struct cfg80211_ssid *a, struct cfg80211_ssid *b)
{
if (WARN_ON(!a || !b))
return false;
if (a->ssid_len != b->ssid_len)
return false;
return memcmp(a->ssid, b->ssid, a->ssid_len) ? false : true;
}
/**
* cfg80211_inform_bss_data - inform cfg80211 of a new BSS
*
* @wiphy: the wiphy reporting the BSS
* @data: the BSS metadata
* @ftype: frame type (if known)
* @bssid: the BSSID of the BSS
* @tsf: the TSF sent by the peer in the beacon/probe response (or 0)
* @capability: the capability field sent by the peer
* @beacon_interval: the beacon interval announced by the peer
* @ie: additional IEs sent by the peer
* @ielen: length of the additional IEs
* @gfp: context flags
*
* This informs cfg80211 that BSS information was found and
* the BSS should be updated/added.
*
* Return: A referenced struct, must be released with cfg80211_put_bss()!
* Or %NULL on error.
*/
struct cfg80211_bss * __must_check
cfg80211_inform_bss_data(struct wiphy *wiphy,
struct cfg80211_inform_bss *data,
enum cfg80211_bss_frame_type ftype,
const u8 *bssid, u64 tsf, u16 capability,
u16 beacon_interval, const u8 *ie, size_t ielen,
gfp_t gfp);
static inline struct cfg80211_bss * __must_check
cfg80211_inform_bss(struct wiphy *wiphy,
struct ieee80211_channel *rx_channel,
enum cfg80211_bss_frame_type ftype,
const u8 *bssid, u64 tsf, u16 capability,
u16 beacon_interval, const u8 *ie, size_t ielen,
s32 signal, gfp_t gfp)
{
struct cfg80211_inform_bss data = {
.chan = rx_channel,
.signal = signal,
};
return cfg80211_inform_bss_data(wiphy, &data, ftype, bssid, tsf,
capability, beacon_interval, ie, ielen,
gfp);
}
/**
* __cfg80211_get_bss - get a BSS reference
* @wiphy: the wiphy this BSS struct belongs to
* @channel: the channel to search on (or %NULL)
* @bssid: the desired BSSID (or %NULL)
* @ssid: the desired SSID (or %NULL)
* @ssid_len: length of the SSID (or 0)
* @bss_type: type of BSS, see &enum ieee80211_bss_type
* @privacy: privacy filter, see &enum ieee80211_privacy
* @use_for: indicates which use is intended
*/
struct cfg80211_bss *__cfg80211_get_bss(struct wiphy *wiphy,
struct ieee80211_channel *channel,
const u8 *bssid,
const u8 *ssid, size_t ssid_len,
enum ieee80211_bss_type bss_type,
enum ieee80211_privacy privacy,
u32 use_for);
/**
* cfg80211_get_bss - get a BSS reference
* @wiphy: the wiphy this BSS struct belongs to
* @channel: the channel to search on (or %NULL)
* @bssid: the desired BSSID (or %NULL)
* @ssid: the desired SSID (or %NULL)
* @ssid_len: length of the SSID (or 0)
* @bss_type: type of BSS, see &enum ieee80211_bss_type
* @privacy: privacy filter, see &enum ieee80211_privacy
*
* This version implies regular usage, %NL80211_BSS_USE_FOR_NORMAL.
*/
static inline struct cfg80211_bss *
cfg80211_get_bss(struct wiphy *wiphy, struct ieee80211_channel *channel,
const u8 *bssid, const u8 *ssid, size_t ssid_len,
enum ieee80211_bss_type bss_type,
enum ieee80211_privacy privacy)
{
return __cfg80211_get_bss(wiphy, channel, bssid, ssid, ssid_len,
bss_type, privacy,
NL80211_BSS_USE_FOR_NORMAL);
}
static inline struct cfg80211_bss *
cfg80211_get_ibss(struct wiphy *wiphy,
struct ieee80211_channel *channel,
const u8 *ssid, size_t ssid_len)
{
return cfg80211_get_bss(wiphy, channel, NULL, ssid, ssid_len,
IEEE80211_BSS_TYPE_IBSS,
IEEE80211_PRIVACY_ANY);
}
/**
* cfg80211_ref_bss - reference BSS struct
* @wiphy: the wiphy this BSS struct belongs to
* @bss: the BSS struct to reference
*
* Increments the refcount of the given BSS struct.
*/
void cfg80211_ref_bss(struct wiphy *wiphy, struct cfg80211_bss *bss);
/**
* cfg80211_put_bss - unref BSS struct
* @wiphy: the wiphy this BSS struct belongs to
* @bss: the BSS struct
*
* Decrements the refcount of the given BSS struct.
*/
void cfg80211_put_bss(struct wiphy *wiphy, struct cfg80211_bss *bss);
/**
* cfg80211_unlink_bss - unlink BSS from internal data structures
* @wiphy: the wiphy
* @bss: the bss to remove
*
* This function removes the given BSS from the internal data structures
* thereby making it no longer show up in scan results etc. Use this
* function when you detect a BSS is gone. Normally BSSes will also time
* out, so it is not necessary to use this function at all.
*/
void cfg80211_unlink_bss(struct wiphy *wiphy, struct cfg80211_bss *bss);
/**
* cfg80211_bss_iter - iterate all BSS entries
*
* This function iterates over the BSS entries associated with the given wiphy
* and calls the callback for the iterated BSS. The iterator function is not
* allowed to call functions that might modify the internal state of the BSS DB.
*
* @wiphy: the wiphy
* @chandef: if given, the iterator function will be called only if the channel
* of the currently iterated BSS is a subset of the given channel.
* @iter: the iterator function to call
* @iter_data: an argument to the iterator function
*/
void cfg80211_bss_iter(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef,
void (*iter)(struct wiphy *wiphy,
struct cfg80211_bss *bss,
void *data),
void *iter_data);
/**
* cfg80211_rx_mlme_mgmt - notification of processed MLME management frame
* @dev: network device
* @buf: authentication frame (header + body)
* @len: length of the frame data
*
* This function is called whenever an authentication, disassociation or
* deauthentication frame has been received and processed in station mode.
* After being asked to authenticate via cfg80211_ops::auth() the driver must
* call either this function or cfg80211_auth_timeout().
* After being asked to associate via cfg80211_ops::assoc() the driver must
* call either this function or cfg80211_auth_timeout().
* While connected, the driver must calls this for received and processed
* disassociation and deauthentication frames. If the frame couldn't be used
* because it was unprotected, the driver must call the function
* cfg80211_rx_unprot_mlme_mgmt() instead.
*
* This function may sleep. The caller must hold the corresponding wdev's mutex.
*/
void cfg80211_rx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len);
/**
* cfg80211_auth_timeout - notification of timed out authentication
* @dev: network device
* @addr: The MAC address of the device with which the authentication timed out
*
* This function may sleep. The caller must hold the corresponding wdev's
* mutex.
*/
void cfg80211_auth_timeout(struct net_device *dev, const u8 *addr);
/**
* struct cfg80211_rx_assoc_resp_data - association response data
* @buf: (Re)Association Response frame (header + body)
* @len: length of the frame data
* @uapsd_queues: bitmap of queues configured for uapsd. Same format
* as the AC bitmap in the QoS info field
* @req_ies: information elements from the (Re)Association Request frame
* @req_ies_len: length of req_ies data
* @ap_mld_addr: AP MLD address (in case of MLO)
* @links: per-link information indexed by link ID, use links[0] for
* non-MLO connections
* @links.bss: the BSS that association was requested with, ownership of the
* pointer moves to cfg80211 in the call to cfg80211_rx_assoc_resp()
* @links.status: Set this (along with a BSS pointer) for links that
* were rejected by the AP.
*/
struct cfg80211_rx_assoc_resp_data {
const u8 *buf;
size_t len;
const u8 *req_ies;
size_t req_ies_len;
int uapsd_queues;
const u8 *ap_mld_addr;
struct {
u8 addr[ETH_ALEN] __aligned(2);
struct cfg80211_bss *bss;
u16 status;
} links[IEEE80211_MLD_MAX_NUM_LINKS];
};
/**
* cfg80211_rx_assoc_resp - notification of processed association response
* @dev: network device
* @data: association response data, &struct cfg80211_rx_assoc_resp_data
*
* After being asked to associate via cfg80211_ops::assoc() the driver must
* call either this function or cfg80211_auth_timeout().
*
* This function may sleep. The caller must hold the corresponding wdev's mutex.
*/
void cfg80211_rx_assoc_resp(struct net_device *dev,
const struct cfg80211_rx_assoc_resp_data *data);
/**
* struct cfg80211_assoc_failure - association failure data
* @ap_mld_addr: AP MLD address, or %NULL
* @bss: list of BSSes, must use entry 0 for non-MLO connections
* (@ap_mld_addr is %NULL)
* @timeout: indicates the association failed due to timeout, otherwise
* the association was abandoned for a reason reported through some
* other API (e.g. deauth RX)
*/
struct cfg80211_assoc_failure {
const u8 *ap_mld_addr;
struct cfg80211_bss *bss[IEEE80211_MLD_MAX_NUM_LINKS];
bool timeout;
};
/**
* cfg80211_assoc_failure - notification of association failure
* @dev: network device
* @data: data describing the association failure
*
* This function may sleep. The caller must hold the corresponding wdev's mutex.
*/
void cfg80211_assoc_failure(struct net_device *dev,
struct cfg80211_assoc_failure *data);
/**
* cfg80211_tx_mlme_mgmt - notification of transmitted deauth/disassoc frame
* @dev: network device
* @buf: 802.11 frame (header + body)
* @len: length of the frame data
* @reconnect: immediate reconnect is desired (include the nl80211 attribute)
*
* This function is called whenever deauthentication has been processed in
* station mode. This includes both received deauthentication frames and
* locally generated ones. This function may sleep. The caller must hold the
* corresponding wdev's mutex.
*/
void cfg80211_tx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len,
bool reconnect);
/**
* cfg80211_rx_unprot_mlme_mgmt - notification of unprotected mlme mgmt frame
* @dev: network device
* @buf: received management frame (header + body)
* @len: length of the frame data
*
* This function is called whenever a received deauthentication or dissassoc
* frame has been dropped in station mode because of MFP being used but the
* frame was not protected. This is also used to notify reception of a Beacon
* frame that was dropped because it did not include a valid MME MIC while
* beacon protection was enabled (BIGTK configured in station mode).
*
* This function may sleep.
*/
void cfg80211_rx_unprot_mlme_mgmt(struct net_device *dev,
const u8 *buf, size_t len);
/**
* cfg80211_michael_mic_failure - notification of Michael MIC failure (TKIP)
* @dev: network device
* @addr: The source MAC address of the frame
* @key_type: The key type that the received frame used
* @key_id: Key identifier (0..3). Can be -1 if missing.
* @tsc: The TSC value of the frame that generated the MIC failure (6 octets)
* @gfp: allocation flags
*
* This function is called whenever the local MAC detects a MIC failure in a
* received frame. This matches with MLME-MICHAELMICFAILURE.indication()
* primitive.
*/
void cfg80211_michael_mic_failure(struct net_device *dev, const u8 *addr,
enum nl80211_key_type key_type, int key_id,
const u8 *tsc, gfp_t gfp);
/**
* cfg80211_ibss_joined - notify cfg80211 that device joined an IBSS
*
* @dev: network device
* @bssid: the BSSID of the IBSS joined
* @channel: the channel of the IBSS joined
* @gfp: allocation flags
*
* This function notifies cfg80211 that the device joined an IBSS or
* switched to a different BSSID. Before this function can be called,
* either a beacon has to have been received from the IBSS, or one of
* the cfg80211_inform_bss{,_frame} functions must have been called
* with the locally generated beacon -- this guarantees that there is
* always a scan result for this IBSS. cfg80211 will handle the rest.
*/
void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid,
struct ieee80211_channel *channel, gfp_t gfp);
/**
* cfg80211_notify_new_peer_candidate - notify cfg80211 of a new mesh peer
* candidate
*
* @dev: network device
* @macaddr: the MAC address of the new candidate
* @ie: information elements advertised by the peer candidate
* @ie_len: length of the information elements buffer
* @sig_dbm: signal level in dBm
* @gfp: allocation flags
*
* This function notifies cfg80211 that the mesh peer candidate has been
* detected, most likely via a beacon or, less likely, via a probe response.
* cfg80211 then sends a notification to userspace.
*/
void cfg80211_notify_new_peer_candidate(struct net_device *dev,
const u8 *macaddr, const u8 *ie, u8 ie_len,
int sig_dbm, gfp_t gfp);
/**
* DOC: RFkill integration
*
* RFkill integration in cfg80211 is almost invisible to drivers,
* as cfg80211 automatically registers an rfkill instance for each
* wireless device it knows about. Soft kill is also translated
* into disconnecting and turning all interfaces off. Drivers are
* expected to turn off the device when all interfaces are down.
*
* However, devices may have a hard RFkill line, in which case they
* also need to interact with the rfkill subsystem, via cfg80211.
* They can do this with a few helper functions documented here.
*/
/**
* wiphy_rfkill_set_hw_state_reason - notify cfg80211 about hw block state
* @wiphy: the wiphy
* @blocked: block status
* @reason: one of reasons in &enum rfkill_hard_block_reasons
*/
void wiphy_rfkill_set_hw_state_reason(struct wiphy *wiphy, bool blocked,
enum rfkill_hard_block_reasons reason);
static inline void wiphy_rfkill_set_hw_state(struct wiphy *wiphy, bool blocked)
{
wiphy_rfkill_set_hw_state_reason(wiphy, blocked,
RFKILL_HARD_BLOCK_SIGNAL);
}
/**
* wiphy_rfkill_start_polling - start polling rfkill
* @wiphy: the wiphy
*/
void wiphy_rfkill_start_polling(struct wiphy *wiphy);
/**
* wiphy_rfkill_stop_polling - stop polling rfkill
* @wiphy: the wiphy
*/
static inline void wiphy_rfkill_stop_polling(struct wiphy *wiphy)
{
rfkill_pause_polling(wiphy->rfkill);
}
/**
* DOC: Vendor commands
*
* Occasionally, there are special protocol or firmware features that
* can't be implemented very openly. For this and similar cases, the
* vendor command functionality allows implementing the features with
* (typically closed-source) userspace and firmware, using nl80211 as
* the configuration mechanism.
*
* A driver supporting vendor commands must register them as an array
* in struct wiphy, with handlers for each one. Each command has an
* OUI and sub command ID to identify it.
*
* Note that this feature should not be (ab)used to implement protocol
* features that could openly be shared across drivers. In particular,
* it must never be required to use vendor commands to implement any
* "normal" functionality that higher-level userspace like connection
* managers etc. need.
*/
struct sk_buff *__cfg80211_alloc_reply_skb(struct wiphy *wiphy,
enum nl80211_commands cmd,
enum nl80211_attrs attr,
int approxlen);
struct sk_buff *__cfg80211_alloc_event_skb(struct wiphy *wiphy,
struct wireless_dev *wdev,
enum nl80211_commands cmd,
enum nl80211_attrs attr,
unsigned int portid,
int vendor_event_idx,
int approxlen, gfp_t gfp);
void __cfg80211_send_event_skb(struct sk_buff *skb, gfp_t gfp);
/**
* cfg80211_vendor_cmd_alloc_reply_skb - allocate vendor command reply
* @wiphy: the wiphy
* @approxlen: an upper bound of the length of the data that will
* be put into the skb
*
* This function allocates and pre-fills an skb for a reply to
* a vendor command. Since it is intended for a reply, calling
* it outside of a vendor command's doit() operation is invalid.
*
* The returned skb is pre-filled with some identifying data in
* a way that any data that is put into the skb (with skb_put(),
* nla_put() or similar) will end up being within the
* %NL80211_ATTR_VENDOR_DATA attribute, so all that needs to be done
* with the skb is adding data for the corresponding userspace tool
* which can then read that data out of the vendor data attribute.
* You must not modify the skb in any other way.
*
* When done, call cfg80211_vendor_cmd_reply() with the skb and return
* its error code as the result of the doit() operation.
*
* Return: An allocated and pre-filled skb. %NULL if any errors happen.
*/
static inline struct sk_buff *
cfg80211_vendor_cmd_alloc_reply_skb(struct wiphy *wiphy, int approxlen)
{
return __cfg80211_alloc_reply_skb(wiphy, NL80211_CMD_VENDOR,
NL80211_ATTR_VENDOR_DATA, approxlen);
}
/**
* cfg80211_vendor_cmd_reply - send the reply skb
* @skb: The skb, must have been allocated with
* cfg80211_vendor_cmd_alloc_reply_skb()
*
* Since calling this function will usually be the last thing
* before returning from the vendor command doit() you should
* return the error code. Note that this function consumes the
* skb regardless of the return value.
*
* Return: An error code or 0 on success.
*/
int cfg80211_vendor_cmd_reply(struct sk_buff *skb);
/**
* cfg80211_vendor_cmd_get_sender - get the current sender netlink ID
* @wiphy: the wiphy
*
* Return the current netlink port ID in a vendor command handler.
* Valid to call only there.
*/
unsigned int cfg80211_vendor_cmd_get_sender(struct wiphy *wiphy);
/**
* cfg80211_vendor_event_alloc - allocate vendor-specific event skb
* @wiphy: the wiphy
* @wdev: the wireless device
* @event_idx: index of the vendor event in the wiphy's vendor_events
* @approxlen: an upper bound of the length of the data that will
* be put into the skb
* @gfp: allocation flags
*
* This function allocates and pre-fills an skb for an event on the
* vendor-specific multicast group.
*
* If wdev != NULL, both the ifindex and identifier of the specified
* wireless device are added to the event message before the vendor data
* attribute.
*
* When done filling the skb, call cfg80211_vendor_event() with the
* skb to send the event.
*
* Return: An allocated and pre-filled skb. %NULL if any errors happen.
*/
static inline struct sk_buff *
cfg80211_vendor_event_alloc(struct wiphy *wiphy, struct wireless_dev *wdev,
int approxlen, int event_idx, gfp_t gfp)
{
return __cfg80211_alloc_event_skb(wiphy, wdev, NL80211_CMD_VENDOR,
NL80211_ATTR_VENDOR_DATA,
0, event_idx, approxlen, gfp);
}
/**
* cfg80211_vendor_event_alloc_ucast - alloc unicast vendor-specific event skb
* @wiphy: the wiphy
* @wdev: the wireless device
* @event_idx: index of the vendor event in the wiphy's vendor_events
* @portid: port ID of the receiver
* @approxlen: an upper bound of the length of the data that will
* be put into the skb
* @gfp: allocation flags
*
* This function allocates and pre-fills an skb for an event to send to
* a specific (userland) socket. This socket would previously have been
* obtained by cfg80211_vendor_cmd_get_sender(), and the caller MUST take
* care to register a netlink notifier to see when the socket closes.
*
* If wdev != NULL, both the ifindex and identifier of the specified
* wireless device are added to the event message before the vendor data
* attribute.
*
* When done filling the skb, call cfg80211_vendor_event() with the
* skb to send the event.
*
* Return: An allocated and pre-filled skb. %NULL if any errors happen.
*/
static inline struct sk_buff *
cfg80211_vendor_event_alloc_ucast(struct wiphy *wiphy,
struct wireless_dev *wdev,
unsigned int portid, int approxlen,
int event_idx, gfp_t gfp)
{
return __cfg80211_alloc_event_skb(wiphy, wdev, NL80211_CMD_VENDOR,
NL80211_ATTR_VENDOR_DATA,
portid, event_idx, approxlen, gfp);
}
/**
* cfg80211_vendor_event - send the event
* @skb: The skb, must have been allocated with cfg80211_vendor_event_alloc()
* @gfp: allocation flags
*
* This function sends the given @skb, which must have been allocated
* by cfg80211_vendor_event_alloc(), as an event. It always consumes it.
*/
static inline void cfg80211_vendor_event(struct sk_buff *skb, gfp_t gfp)
{
__cfg80211_send_event_skb(skb, gfp);
}
#ifdef CONFIG_NL80211_TESTMODE
/**
* DOC: Test mode
*
* Test mode is a set of utility functions to allow drivers to
* interact with driver-specific tools to aid, for instance,
* factory programming.
*
* This chapter describes how drivers interact with it. For more
* information see the nl80211 book's chapter on it.
*/
/**
* cfg80211_testmode_alloc_reply_skb - allocate testmode reply
* @wiphy: the wiphy
* @approxlen: an upper bound of the length of the data that will
* be put into the skb
*
* This function allocates and pre-fills an skb for a reply to
* the testmode command. Since it is intended for a reply, calling
* it outside of the @testmode_cmd operation is invalid.
*
* The returned skb is pre-filled with the wiphy index and set up in
* a way that any data that is put into the skb (with skb_put(),
* nla_put() or similar) will end up being within the
* %NL80211_ATTR_TESTDATA attribute, so all that needs to be done
* with the skb is adding data for the corresponding userspace tool
* which can then read that data out of the testdata attribute. You
* must not modify the skb in any other way.
*
* When done, call cfg80211_testmode_reply() with the skb and return
* its error code as the result of the @testmode_cmd operation.
*
* Return: An allocated and pre-filled skb. %NULL if any errors happen.
*/
static inline struct sk_buff *
cfg80211_testmode_alloc_reply_skb(struct wiphy *wiphy, int approxlen)
{
return __cfg80211_alloc_reply_skb(wiphy, NL80211_CMD_TESTMODE,
NL80211_ATTR_TESTDATA, approxlen);
}
/**
* cfg80211_testmode_reply - send the reply skb
* @skb: The skb, must have been allocated with
* cfg80211_testmode_alloc_reply_skb()
*
* Since calling this function will usually be the last thing
* before returning from the @testmode_cmd you should return
* the error code. Note that this function consumes the skb
* regardless of the return value.
*
* Return: An error code or 0 on success.
*/
static inline int cfg80211_testmode_reply(struct sk_buff *skb)
{
return cfg80211_vendor_cmd_reply(skb);
}
/**
* cfg80211_testmode_alloc_event_skb - allocate testmode event
* @wiphy: the wiphy
* @approxlen: an upper bound of the length of the data that will
* be put into the skb
* @gfp: allocation flags
*
* This function allocates and pre-fills an skb for an event on the
* testmode multicast group.
*
* The returned skb is set up in the same way as with
* cfg80211_testmode_alloc_reply_skb() but prepared for an event. As
* there, you should simply add data to it that will then end up in the
* %NL80211_ATTR_TESTDATA attribute. Again, you must not modify the skb
* in any other way.
*
* When done filling the skb, call cfg80211_testmode_event() with the
* skb to send the event.
*
* Return: An allocated and pre-filled skb. %NULL if any errors happen.
*/
static inline struct sk_buff *
cfg80211_testmode_alloc_event_skb(struct wiphy *wiphy, int approxlen, gfp_t gfp)
{
return __cfg80211_alloc_event_skb(wiphy, NULL, NL80211_CMD_TESTMODE,
NL80211_ATTR_TESTDATA, 0, -1,
approxlen, gfp);
}
/**
* cfg80211_testmode_event - send the event
* @skb: The skb, must have been allocated with
* cfg80211_testmode_alloc_event_skb()
* @gfp: allocation flags
*
* This function sends the given @skb, which must have been allocated
* by cfg80211_testmode_alloc_event_skb(), as an event. It always
* consumes it.
*/
static inline void cfg80211_testmode_event(struct sk_buff *skb, gfp_t gfp)
{
__cfg80211_send_event_skb(skb, gfp);
}
#define CFG80211_TESTMODE_CMD(cmd) .testmode_cmd = (cmd),
#define CFG80211_TESTMODE_DUMP(cmd) .testmode_dump = (cmd),
#else
#define CFG80211_TESTMODE_CMD(cmd)
#define CFG80211_TESTMODE_DUMP(cmd)
#endif
/**
* struct cfg80211_fils_resp_params - FILS connection response params
* @kek: KEK derived from a successful FILS connection (may be %NULL)
* @kek_len: Length of @fils_kek in octets
* @update_erp_next_seq_num: Boolean value to specify whether the value in
* @erp_next_seq_num is valid.
* @erp_next_seq_num: The next sequence number to use in ERP message in
* FILS Authentication. This value should be specified irrespective of the
* status for a FILS connection.
* @pmk: A new PMK if derived from a successful FILS connection (may be %NULL).
* @pmk_len: Length of @pmk in octets
* @pmkid: A new PMKID if derived from a successful FILS connection or the PMKID
* used for this FILS connection (may be %NULL).
*/
struct cfg80211_fils_resp_params {
const u8 *kek;
size_t kek_len;
bool update_erp_next_seq_num;
u16 erp_next_seq_num;
const u8 *pmk;
size_t pmk_len;
const u8 *pmkid;
};
/**
* struct cfg80211_connect_resp_params - Connection response params
* @status: Status code, %WLAN_STATUS_SUCCESS for successful connection, use
* %WLAN_STATUS_UNSPECIFIED_FAILURE if your device cannot give you
* the real status code for failures. If this call is used to report a
* failure due to a timeout (e.g., not receiving an Authentication frame
* from the AP) instead of an explicit rejection by the AP, -1 is used to
* indicate that this is a failure, but without a status code.
* @timeout_reason is used to report the reason for the timeout in that
* case.
* @req_ie: Association request IEs (may be %NULL)
* @req_ie_len: Association request IEs length
* @resp_ie: Association response IEs (may be %NULL)
* @resp_ie_len: Association response IEs length
* @fils: FILS connection response parameters.
* @timeout_reason: Reason for connection timeout. This is used when the
* connection fails due to a timeout instead of an explicit rejection from
* the AP. %NL80211_TIMEOUT_UNSPECIFIED is used when the timeout reason is
* not known. This value is used only if @status < 0 to indicate that the
* failure is due to a timeout and not due to explicit rejection by the AP.
* This value is ignored in other cases (@status >= 0).
* @valid_links: For MLO connection, BIT mask of the valid link ids. Otherwise
* zero.
* @ap_mld_addr: For MLO connection, MLD address of the AP. Otherwise %NULL.
* @links : For MLO connection, contains link info for the valid links indicated
* using @valid_links. For non-MLO connection, links[0] contains the
* connected AP info.
* @links.addr: For MLO connection, MAC address of the STA link. Otherwise
* %NULL.
* @links.bssid: For MLO connection, MAC address of the AP link. For non-MLO
* connection, links[0].bssid points to the BSSID of the AP (may be %NULL).
* @links.bss: For MLO connection, entry of bss to which STA link is connected.
* For non-MLO connection, links[0].bss points to entry of bss to which STA
* is connected. It can be obtained through cfg80211_get_bss() (may be
* %NULL). It is recommended to store the bss from the connect_request and
* hold a reference to it and return through this param to avoid a warning
* if the bss is expired during the connection, esp. for those drivers
* implementing connect op. Only one parameter among @bssid and @bss needs
* to be specified.
* @links.status: per-link status code, to report a status code that's not
* %WLAN_STATUS_SUCCESS for a given link, it must also be in the
* @valid_links bitmap and may have a BSS pointer (which is then released)
*/
struct cfg80211_connect_resp_params {
int status;
const u8 *req_ie;
size_t req_ie_len;
const u8 *resp_ie;
size_t resp_ie_len;
struct cfg80211_fils_resp_params fils;
enum nl80211_timeout_reason timeout_reason;
const u8 *ap_mld_addr;
u16 valid_links;
struct {
const u8 *addr;
const u8 *bssid;
struct cfg80211_bss *bss;
u16 status;
} links[IEEE80211_MLD_MAX_NUM_LINKS];
};
/**
* cfg80211_connect_done - notify cfg80211 of connection result
*
* @dev: network device
* @params: connection response parameters
* @gfp: allocation flags
*
* It should be called by the underlying driver once execution of the connection
* request from connect() has been completed. This is similar to
* cfg80211_connect_bss(), but takes a structure pointer for connection response
* parameters. Only one of the functions among cfg80211_connect_bss(),
* cfg80211_connect_result(), cfg80211_connect_timeout(),
* and cfg80211_connect_done() should be called.
*/
void cfg80211_connect_done(struct net_device *dev,
struct cfg80211_connect_resp_params *params,
gfp_t gfp);
/**
* cfg80211_connect_bss - notify cfg80211 of connection result
*
* @dev: network device
* @bssid: the BSSID of the AP
* @bss: Entry of bss to which STA got connected to, can be obtained through
* cfg80211_get_bss() (may be %NULL). But it is recommended to store the
* bss from the connect_request and hold a reference to it and return
* through this param to avoid a warning if the bss is expired during the
* connection, esp. for those drivers implementing connect op.
* Only one parameter among @bssid and @bss needs to be specified.
* @req_ie: association request IEs (maybe be %NULL)
* @req_ie_len: association request IEs length
* @resp_ie: association response IEs (may be %NULL)
* @resp_ie_len: assoc response IEs length
* @status: status code, %WLAN_STATUS_SUCCESS for successful connection, use
* %WLAN_STATUS_UNSPECIFIED_FAILURE if your device cannot give you
* the real status code for failures. If this call is used to report a
* failure due to a timeout (e.g., not receiving an Authentication frame
* from the AP) instead of an explicit rejection by the AP, -1 is used to
* indicate that this is a failure, but without a status code.
* @timeout_reason is used to report the reason for the timeout in that
* case.
* @gfp: allocation flags
* @timeout_reason: reason for connection timeout. This is used when the
* connection fails due to a timeout instead of an explicit rejection from
* the AP. %NL80211_TIMEOUT_UNSPECIFIED is used when the timeout reason is
* not known. This value is used only if @status < 0 to indicate that the
* failure is due to a timeout and not due to explicit rejection by the AP.
* This value is ignored in other cases (@status >= 0).
*
* It should be called by the underlying driver once execution of the connection
* request from connect() has been completed. This is similar to
* cfg80211_connect_result(), but with the option of identifying the exact bss
* entry for the connection. Only one of the functions among
* cfg80211_connect_bss(), cfg80211_connect_result(),
* cfg80211_connect_timeout(), and cfg80211_connect_done() should be called.
*/
static inline void
cfg80211_connect_bss(struct net_device *dev, const u8 *bssid,
struct cfg80211_bss *bss, const u8 *req_ie,
size_t req_ie_len, const u8 *resp_ie,
size_t resp_ie_len, int status, gfp_t gfp,
enum nl80211_timeout_reason timeout_reason)
{
struct cfg80211_connect_resp_params params;
memset(¶ms, 0, sizeof(params));
params.status = status;
params.links[0].bssid = bssid;
params.links[0].bss = bss;
params.req_ie = req_ie;
params.req_ie_len = req_ie_len;
params.resp_ie = resp_ie;
params.resp_ie_len = resp_ie_len;
params.timeout_reason = timeout_reason;
cfg80211_connect_done(dev, ¶ms, gfp);
}
/**
* cfg80211_connect_result - notify cfg80211 of connection result
*
* @dev: network device
* @bssid: the BSSID of the AP
* @req_ie: association request IEs (maybe be %NULL)
* @req_ie_len: association request IEs length
* @resp_ie: association response IEs (may be %NULL)
* @resp_ie_len: assoc response IEs length
* @status: status code, %WLAN_STATUS_SUCCESS for successful connection, use
* %WLAN_STATUS_UNSPECIFIED_FAILURE if your device cannot give you
* the real status code for failures.
* @gfp: allocation flags
*
* It should be called by the underlying driver once execution of the connection
* request from connect() has been completed. This is similar to
* cfg80211_connect_bss() which allows the exact bss entry to be specified. Only
* one of the functions among cfg80211_connect_bss(), cfg80211_connect_result(),
* cfg80211_connect_timeout(), and cfg80211_connect_done() should be called.
*/
static inline void
cfg80211_connect_result(struct net_device *dev, const u8 *bssid,
const u8 *req_ie, size_t req_ie_len,
const u8 *resp_ie, size_t resp_ie_len,
u16 status, gfp_t gfp)
{
cfg80211_connect_bss(dev, bssid, NULL, req_ie, req_ie_len, resp_ie,
resp_ie_len, status, gfp,
NL80211_TIMEOUT_UNSPECIFIED);
}
/**
* cfg80211_connect_timeout - notify cfg80211 of connection timeout
*
* @dev: network device
* @bssid: the BSSID of the AP
* @req_ie: association request IEs (maybe be %NULL)
* @req_ie_len: association request IEs length
* @gfp: allocation flags
* @timeout_reason: reason for connection timeout.
*
* It should be called by the underlying driver whenever connect() has failed
* in a sequence where no explicit authentication/association rejection was
* received from the AP. This could happen, e.g., due to not being able to send
* out the Authentication or Association Request frame or timing out while
* waiting for the response. Only one of the functions among
* cfg80211_connect_bss(), cfg80211_connect_result(),
* cfg80211_connect_timeout(), and cfg80211_connect_done() should be called.
*/
static inline void
cfg80211_connect_timeout(struct net_device *dev, const u8 *bssid,
const u8 *req_ie, size_t req_ie_len, gfp_t gfp,
enum nl80211_timeout_reason timeout_reason)
{
cfg80211_connect_bss(dev, bssid, NULL, req_ie, req_ie_len, NULL, 0, -1,
gfp, timeout_reason);
}
/**
* struct cfg80211_roam_info - driver initiated roaming information
*
* @req_ie: association request IEs (maybe be %NULL)
* @req_ie_len: association request IEs length
* @resp_ie: association response IEs (may be %NULL)
* @resp_ie_len: assoc response IEs length
* @fils: FILS related roaming information.
* @valid_links: For MLO roaming, BIT mask of the new valid links is set.
* Otherwise zero.
* @ap_mld_addr: For MLO roaming, MLD address of the new AP. Otherwise %NULL.
* @links : For MLO roaming, contains new link info for the valid links set in
* @valid_links. For non-MLO roaming, links[0] contains the new AP info.
* @links.addr: For MLO roaming, MAC address of the STA link. Otherwise %NULL.
* @links.bssid: For MLO roaming, MAC address of the new AP link. For non-MLO
* roaming, links[0].bssid points to the BSSID of the new AP. May be
* %NULL if %links.bss is set.
* @links.channel: the channel of the new AP.
* @links.bss: For MLO roaming, entry of new bss to which STA link got
* roamed. For non-MLO roaming, links[0].bss points to entry of bss to
* which STA got roamed (may be %NULL if %links.bssid is set)
*/
struct cfg80211_roam_info {
const u8 *req_ie;
size_t req_ie_len;
const u8 *resp_ie;
size_t resp_ie_len;
struct cfg80211_fils_resp_params fils;
const u8 *ap_mld_addr;
u16 valid_links;
struct {
const u8 *addr;
const u8 *bssid;
struct ieee80211_channel *channel;
struct cfg80211_bss *bss;
} links[IEEE80211_MLD_MAX_NUM_LINKS];
};
/**
* cfg80211_roamed - notify cfg80211 of roaming
*
* @dev: network device
* @info: information about the new BSS. struct &cfg80211_roam_info.
* @gfp: allocation flags
*
* This function may be called with the driver passing either the BSSID of the
* new AP or passing the bss entry to avoid a race in timeout of the bss entry.
* It should be called by the underlying driver whenever it roamed from one AP
* to another while connected. Drivers which have roaming implemented in
* firmware should pass the bss entry to avoid a race in bss entry timeout where
* the bss entry of the new AP is seen in the driver, but gets timed out by the
* time it is accessed in __cfg80211_roamed() due to delay in scheduling
* rdev->event_work. In case of any failures, the reference is released
* either in cfg80211_roamed() or in __cfg80211_romed(), Otherwise, it will be
* released while disconnecting from the current bss.
*/
void cfg80211_roamed(struct net_device *dev, struct cfg80211_roam_info *info,
gfp_t gfp);
/**
* cfg80211_port_authorized - notify cfg80211 of successful security association
*
* @dev: network device
* @peer_addr: BSSID of the AP/P2P GO in case of STA/GC or STA/GC MAC address
* in case of AP/P2P GO
* @td_bitmap: transition disable policy
* @td_bitmap_len: Length of transition disable policy
* @gfp: allocation flags
*
* This function should be called by a driver that supports 4 way handshake
* offload after a security association was successfully established (i.e.,
* the 4 way handshake was completed successfully). The call to this function
* should be preceded with a call to cfg80211_connect_result(),
* cfg80211_connect_done(), cfg80211_connect_bss() or cfg80211_roamed() to
* indicate the 802.11 association.
* This function can also be called by AP/P2P GO driver that supports
* authentication offload. In this case the peer_mac passed is that of
* associated STA/GC.
*/
void cfg80211_port_authorized(struct net_device *dev, const u8 *peer_addr,
const u8* td_bitmap, u8 td_bitmap_len, gfp_t gfp);
/**
* cfg80211_disconnected - notify cfg80211 that connection was dropped
*
* @dev: network device
* @ie: information elements of the deauth/disassoc frame (may be %NULL)
* @ie_len: length of IEs
* @reason: reason code for the disconnection, set it to 0 if unknown
* @locally_generated: disconnection was requested locally
* @gfp: allocation flags
*
* After it calls this function, the driver should enter an idle state
* and not try to connect to any AP any more.
*/
void cfg80211_disconnected(struct net_device *dev, u16 reason,
const u8 *ie, size_t ie_len,
bool locally_generated, gfp_t gfp);
/**
* cfg80211_ready_on_channel - notification of remain_on_channel start
* @wdev: wireless device
* @cookie: the request cookie
* @chan: The current channel (from remain_on_channel request)
* @duration: Duration in milliseconds that the driver intents to remain on the
* channel
* @gfp: allocation flags
*/
void cfg80211_ready_on_channel(struct wireless_dev *wdev, u64 cookie,
struct ieee80211_channel *chan,
unsigned int duration, gfp_t gfp);
/**
* cfg80211_remain_on_channel_expired - remain_on_channel duration expired
* @wdev: wireless device
* @cookie: the request cookie
* @chan: The current channel (from remain_on_channel request)
* @gfp: allocation flags
*/
void cfg80211_remain_on_channel_expired(struct wireless_dev *wdev, u64 cookie,
struct ieee80211_channel *chan,
gfp_t gfp);
/**
* cfg80211_tx_mgmt_expired - tx_mgmt duration expired
* @wdev: wireless device
* @cookie: the requested cookie
* @chan: The current channel (from tx_mgmt request)
* @gfp: allocation flags
*/
void cfg80211_tx_mgmt_expired(struct wireless_dev *wdev, u64 cookie,
struct ieee80211_channel *chan, gfp_t gfp);
/**
* cfg80211_sinfo_alloc_tid_stats - allocate per-tid statistics.
*
* @sinfo: the station information
* @gfp: allocation flags
*/
int cfg80211_sinfo_alloc_tid_stats(struct station_info *sinfo, gfp_t gfp);
/**
* cfg80211_sinfo_release_content - release contents of station info
* @sinfo: the station information
*
* Releases any potentially allocated sub-information of the station
* information, but not the struct itself (since it's typically on
* the stack.)
*/
static inline void cfg80211_sinfo_release_content(struct station_info *sinfo)
{
kfree(sinfo->pertid);
}
/**
* cfg80211_new_sta - notify userspace about station
*
* @dev: the netdev
* @mac_addr: the station's address
* @sinfo: the station information
* @gfp: allocation flags
*/
void cfg80211_new_sta(struct net_device *dev, const u8 *mac_addr,
struct station_info *sinfo, gfp_t gfp);
/**
* cfg80211_del_sta_sinfo - notify userspace about deletion of a station
* @dev: the netdev
* @mac_addr: the station's address. For MLD station, MLD address is used.
* @sinfo: the station information/statistics
* @gfp: allocation flags
*/
void cfg80211_del_sta_sinfo(struct net_device *dev, const u8 *mac_addr,
struct station_info *sinfo, gfp_t gfp);
/**
* cfg80211_del_sta - notify userspace about deletion of a station
*
* @dev: the netdev
* @mac_addr: the station's address. For MLD station, MLD address is used.
* @gfp: allocation flags
*/
static inline void cfg80211_del_sta(struct net_device *dev,
const u8 *mac_addr, gfp_t gfp)
{
cfg80211_del_sta_sinfo(dev, mac_addr, NULL, gfp);
}
/**
* cfg80211_conn_failed - connection request failed notification
*
* @dev: the netdev
* @mac_addr: the station's address
* @reason: the reason for connection failure
* @gfp: allocation flags
*
* Whenever a station tries to connect to an AP and if the station
* could not connect to the AP as the AP has rejected the connection
* for some reasons, this function is called.
*
* The reason for connection failure can be any of the value from
* nl80211_connect_failed_reason enum
*/
void cfg80211_conn_failed(struct net_device *dev, const u8 *mac_addr,
enum nl80211_connect_failed_reason reason,
gfp_t gfp);
/**
* struct cfg80211_rx_info - received management frame info
*
* @freq: Frequency on which the frame was received in kHz
* @sig_dbm: signal strength in dBm, or 0 if unknown
* @have_link_id: indicates the frame was received on a link of
* an MLD, i.e. the @link_id field is valid
* @link_id: the ID of the link the frame was received on
* @buf: Management frame (header + body)
* @len: length of the frame data
* @flags: flags, as defined in &enum nl80211_rxmgmt_flags
* @rx_tstamp: Hardware timestamp of frame RX in nanoseconds
* @ack_tstamp: Hardware timestamp of ack TX in nanoseconds
*/
struct cfg80211_rx_info {
int freq;
int sig_dbm;
bool have_link_id;
u8 link_id;
const u8 *buf;
size_t len;
u32 flags;
u64 rx_tstamp;
u64 ack_tstamp;
};
/**
* cfg80211_rx_mgmt_ext - management frame notification with extended info
* @wdev: wireless device receiving the frame
* @info: RX info as defined in struct cfg80211_rx_info
*
* This function is called whenever an Action frame is received for a station
* mode interface, but is not processed in kernel.
*
* Return: %true if a user space application has registered for this frame.
* For action frames, that makes it responsible for rejecting unrecognized
* action frames; %false otherwise, in which case for action frames the
* driver is responsible for rejecting the frame.
*/
bool cfg80211_rx_mgmt_ext(struct wireless_dev *wdev,
struct cfg80211_rx_info *info);
/**
* cfg80211_rx_mgmt_khz - notification of received, unprocessed management frame
* @wdev: wireless device receiving the frame
* @freq: Frequency on which the frame was received in KHz
* @sig_dbm: signal strength in dBm, or 0 if unknown
* @buf: Management frame (header + body)
* @len: length of the frame data
* @flags: flags, as defined in enum nl80211_rxmgmt_flags
*
* This function is called whenever an Action frame is received for a station
* mode interface, but is not processed in kernel.
*
* Return: %true if a user space application has registered for this frame.
* For action frames, that makes it responsible for rejecting unrecognized
* action frames; %false otherwise, in which case for action frames the
* driver is responsible for rejecting the frame.
*/
static inline bool cfg80211_rx_mgmt_khz(struct wireless_dev *wdev, int freq,
int sig_dbm, const u8 *buf, size_t len,
u32 flags)
{
struct cfg80211_rx_info info = {
.freq = freq,
.sig_dbm = sig_dbm,
.buf = buf,
.len = len,
.flags = flags
};
return cfg80211_rx_mgmt_ext(wdev, &info);
}
/**
* cfg80211_rx_mgmt - notification of received, unprocessed management frame
* @wdev: wireless device receiving the frame
* @freq: Frequency on which the frame was received in MHz
* @sig_dbm: signal strength in dBm, or 0 if unknown
* @buf: Management frame (header + body)
* @len: length of the frame data
* @flags: flags, as defined in enum nl80211_rxmgmt_flags
*
* This function is called whenever an Action frame is received for a station
* mode interface, but is not processed in kernel.
*
* Return: %true if a user space application has registered for this frame.
* For action frames, that makes it responsible for rejecting unrecognized
* action frames; %false otherwise, in which case for action frames the
* driver is responsible for rejecting the frame.
*/
static inline bool cfg80211_rx_mgmt(struct wireless_dev *wdev, int freq,
int sig_dbm, const u8 *buf, size_t len,
u32 flags)
{
struct cfg80211_rx_info info = {
.freq = MHZ_TO_KHZ(freq),
.sig_dbm = sig_dbm,
.buf = buf,
.len = len,
.flags = flags
};
return cfg80211_rx_mgmt_ext(wdev, &info);
}
/**
* struct cfg80211_tx_status - TX status for management frame information
*
* @cookie: Cookie returned by cfg80211_ops::mgmt_tx()
* @tx_tstamp: hardware TX timestamp in nanoseconds
* @ack_tstamp: hardware ack RX timestamp in nanoseconds
* @buf: Management frame (header + body)
* @len: length of the frame data
* @ack: Whether frame was acknowledged
*/
struct cfg80211_tx_status {
u64 cookie;
u64 tx_tstamp;
u64 ack_tstamp;
const u8 *buf;
size_t len;
bool ack;
};
/**
* cfg80211_mgmt_tx_status_ext - TX status notification with extended info
* @wdev: wireless device receiving the frame
* @status: TX status data
* @gfp: context flags
*
* This function is called whenever a management frame was requested to be
* transmitted with cfg80211_ops::mgmt_tx() to report the TX status of the
* transmission attempt with extended info.
*/
void cfg80211_mgmt_tx_status_ext(struct wireless_dev *wdev,
struct cfg80211_tx_status *status, gfp_t gfp);
/**
* cfg80211_mgmt_tx_status - notification of TX status for management frame
* @wdev: wireless device receiving the frame
* @cookie: Cookie returned by cfg80211_ops::mgmt_tx()
* @buf: Management frame (header + body)
* @len: length of the frame data
* @ack: Whether frame was acknowledged
* @gfp: context flags
*
* This function is called whenever a management frame was requested to be
* transmitted with cfg80211_ops::mgmt_tx() to report the TX status of the
* transmission attempt.
*/
static inline void cfg80211_mgmt_tx_status(struct wireless_dev *wdev,
u64 cookie, const u8 *buf,
size_t len, bool ack, gfp_t gfp)
{
struct cfg80211_tx_status status = {
.cookie = cookie,
.buf = buf,
.len = len,
.ack = ack
};
cfg80211_mgmt_tx_status_ext(wdev, &status, gfp);
}
/**
* cfg80211_control_port_tx_status - notification of TX status for control
* port frames
* @wdev: wireless device receiving the frame
* @cookie: Cookie returned by cfg80211_ops::tx_control_port()
* @buf: Data frame (header + body)
* @len: length of the frame data
* @ack: Whether frame was acknowledged
* @gfp: context flags
*
* This function is called whenever a control port frame was requested to be
* transmitted with cfg80211_ops::tx_control_port() to report the TX status of
* the transmission attempt.
*/
void cfg80211_control_port_tx_status(struct wireless_dev *wdev, u64 cookie,
const u8 *buf, size_t len, bool ack,
gfp_t gfp);
/**
* cfg80211_rx_control_port - notification about a received control port frame
* @dev: The device the frame matched to
* @skb: The skbuf with the control port frame. It is assumed that the skbuf
* is 802.3 formatted (with 802.3 header). The skb can be non-linear.
* This function does not take ownership of the skb, so the caller is
* responsible for any cleanup. The caller must also ensure that
* skb->protocol is set appropriately.
* @unencrypted: Whether the frame was received unencrypted
* @link_id: the link the frame was received on, -1 if not applicable or unknown
*
* This function is used to inform userspace about a received control port
* frame. It should only be used if userspace indicated it wants to receive
* control port frames over nl80211.
*
* The frame is the data portion of the 802.3 or 802.11 data frame with all
* network layer headers removed (e.g. the raw EAPoL frame).
*
* Return: %true if the frame was passed to userspace
*/
bool cfg80211_rx_control_port(struct net_device *dev, struct sk_buff *skb,
bool unencrypted, int link_id);
/**
* cfg80211_cqm_rssi_notify - connection quality monitoring rssi event
* @dev: network device
* @rssi_event: the triggered RSSI event
* @rssi_level: new RSSI level value or 0 if not available
* @gfp: context flags
*
* This function is called when a configured connection quality monitoring
* rssi threshold reached event occurs.
*/
void cfg80211_cqm_rssi_notify(struct net_device *dev,
enum nl80211_cqm_rssi_threshold_event rssi_event,
s32 rssi_level, gfp_t gfp);
/**
* cfg80211_cqm_pktloss_notify - notify userspace about packetloss to peer
* @dev: network device
* @peer: peer's MAC address
* @num_packets: how many packets were lost -- should be a fixed threshold
* but probably no less than maybe 50, or maybe a throughput dependent
* threshold (to account for temporary interference)
* @gfp: context flags
*/
void cfg80211_cqm_pktloss_notify(struct net_device *dev,
const u8 *peer, u32 num_packets, gfp_t gfp);
/**
* cfg80211_cqm_txe_notify - TX error rate event
* @dev: network device
* @peer: peer's MAC address
* @num_packets: how many packets were lost
* @rate: % of packets which failed transmission
* @intvl: interval (in s) over which the TX failure threshold was breached.
* @gfp: context flags
*
* Notify userspace when configured % TX failures over number of packets in a
* given interval is exceeded.
*/
void cfg80211_cqm_txe_notify(struct net_device *dev, const u8 *peer,
u32 num_packets, u32 rate, u32 intvl, gfp_t gfp);
/**
* cfg80211_cqm_beacon_loss_notify - beacon loss event
* @dev: network device
* @gfp: context flags
*
* Notify userspace about beacon loss from the connected AP.
*/
void cfg80211_cqm_beacon_loss_notify(struct net_device *dev, gfp_t gfp);
/**
* __cfg80211_radar_event - radar detection event
* @wiphy: the wiphy
* @chandef: chandef for the current channel
* @offchan: the radar has been detected on the offchannel chain
* @gfp: context flags
*
* This function is called when a radar is detected on the current chanenl.
*/
void __cfg80211_radar_event(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef,
bool offchan, gfp_t gfp);
static inline void
cfg80211_radar_event(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef,
gfp_t gfp)
{
__cfg80211_radar_event(wiphy, chandef, false, gfp);
}
static inline void
cfg80211_background_radar_event(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef,
gfp_t gfp)
{
__cfg80211_radar_event(wiphy, chandef, true, gfp);
}
/**
* cfg80211_sta_opmode_change_notify - STA's ht/vht operation mode change event
* @dev: network device
* @mac: MAC address of a station which opmode got modified
* @sta_opmode: station's current opmode value
* @gfp: context flags
*
* Driver should call this function when station's opmode modified via action
* frame.
*/
void cfg80211_sta_opmode_change_notify(struct net_device *dev, const u8 *mac,
struct sta_opmode_info *sta_opmode,
gfp_t gfp);
/**
* cfg80211_cac_event - Channel availability check (CAC) event
* @netdev: network device
* @chandef: chandef for the current channel
* @event: type of event
* @gfp: context flags
*
* This function is called when a Channel availability check (CAC) is finished
* or aborted. This must be called to notify the completion of a CAC process,
* also by full-MAC drivers.
*/
void cfg80211_cac_event(struct net_device *netdev,
const struct cfg80211_chan_def *chandef,
enum nl80211_radar_event event, gfp_t gfp);
/**
* cfg80211_background_cac_abort - Channel Availability Check offchan abort event
* @wiphy: the wiphy
*
* This function is called by the driver when a Channel Availability Check
* (CAC) is aborted by a offchannel dedicated chain.
*/
void cfg80211_background_cac_abort(struct wiphy *wiphy);
/**
* cfg80211_gtk_rekey_notify - notify userspace about driver rekeying
* @dev: network device
* @bssid: BSSID of AP (to avoid races)
* @replay_ctr: new replay counter
* @gfp: allocation flags
*/
void cfg80211_gtk_rekey_notify(struct net_device *dev, const u8 *bssid,
const u8 *replay_ctr, gfp_t gfp);
/**
* cfg80211_pmksa_candidate_notify - notify about PMKSA caching candidate
* @dev: network device
* @index: candidate index (the smaller the index, the higher the priority)
* @bssid: BSSID of AP
* @preauth: Whether AP advertises support for RSN pre-authentication
* @gfp: allocation flags
*/
void cfg80211_pmksa_candidate_notify(struct net_device *dev, int index,
const u8 *bssid, bool preauth, gfp_t gfp);
/**
* cfg80211_rx_spurious_frame - inform userspace about a spurious frame
* @dev: The device the frame matched to
* @addr: the transmitter address
* @gfp: context flags
*
* This function is used in AP mode (only!) to inform userspace that
* a spurious class 3 frame was received, to be able to deauth the
* sender.
* Return: %true if the frame was passed to userspace (or this failed
* for a reason other than not having a subscription.)
*/
bool cfg80211_rx_spurious_frame(struct net_device *dev,
const u8 *addr, gfp_t gfp);
/**
* cfg80211_rx_unexpected_4addr_frame - inform about unexpected WDS frame
* @dev: The device the frame matched to
* @addr: the transmitter address
* @gfp: context flags
*
* This function is used in AP mode (only!) to inform userspace that
* an associated station sent a 4addr frame but that wasn't expected.
* It is allowed and desirable to send this event only once for each
* station to avoid event flooding.
* Return: %true if the frame was passed to userspace (or this failed
* for a reason other than not having a subscription.)
*/
bool cfg80211_rx_unexpected_4addr_frame(struct net_device *dev,
const u8 *addr, gfp_t gfp);
/**
* cfg80211_probe_status - notify userspace about probe status
* @dev: the device the probe was sent on
* @addr: the address of the peer
* @cookie: the cookie filled in @probe_client previously
* @acked: indicates whether probe was acked or not
* @ack_signal: signal strength (in dBm) of the ACK frame.
* @is_valid_ack_signal: indicates the ack_signal is valid or not.
* @gfp: allocation flags
*/
void cfg80211_probe_status(struct net_device *dev, const u8 *addr,
u64 cookie, bool acked, s32 ack_signal,
bool is_valid_ack_signal, gfp_t gfp);
/**
* cfg80211_report_obss_beacon_khz - report beacon from other APs
* @wiphy: The wiphy that received the beacon
* @frame: the frame
* @len: length of the frame
* @freq: frequency the frame was received on in KHz
* @sig_dbm: signal strength in dBm, or 0 if unknown
*
* Use this function to report to userspace when a beacon was
* received. It is not useful to call this when there is no
* netdev that is in AP/GO mode.
*/
void cfg80211_report_obss_beacon_khz(struct wiphy *wiphy, const u8 *frame,
size_t len, int freq, int sig_dbm);
/**
* cfg80211_report_obss_beacon - report beacon from other APs
* @wiphy: The wiphy that received the beacon
* @frame: the frame
* @len: length of the frame
* @freq: frequency the frame was received on
* @sig_dbm: signal strength in dBm, or 0 if unknown
*
* Use this function to report to userspace when a beacon was
* received. It is not useful to call this when there is no
* netdev that is in AP/GO mode.
*/
static inline void cfg80211_report_obss_beacon(struct wiphy *wiphy,
const u8 *frame, size_t len,
int freq, int sig_dbm)
{
cfg80211_report_obss_beacon_khz(wiphy, frame, len, MHZ_TO_KHZ(freq),
sig_dbm);
}
/**
* cfg80211_reg_can_beacon - check if beaconing is allowed
* @wiphy: the wiphy
* @chandef: the channel definition
* @iftype: interface type
*
* Return: %true if there is no secondary channel or the secondary channel(s)
* can be used for beaconing (i.e. is not a radar channel etc.)
*/
bool cfg80211_reg_can_beacon(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef,
enum nl80211_iftype iftype);
/**
* cfg80211_reg_can_beacon_relax - check if beaconing is allowed with relaxation
* @wiphy: the wiphy
* @chandef: the channel definition
* @iftype: interface type
*
* Return: %true if there is no secondary channel or the secondary channel(s)
* can be used for beaconing (i.e. is not a radar channel etc.). This version
* also checks if IR-relaxation conditions apply, to allow beaconing under
* more permissive conditions.
*
* Requires the wiphy mutex to be held.
*/
bool cfg80211_reg_can_beacon_relax(struct wiphy *wiphy,
struct cfg80211_chan_def *chandef,
enum nl80211_iftype iftype);
/*
* cfg80211_ch_switch_notify - update wdev channel and notify userspace
* @dev: the device which switched channels
* @chandef: the new channel definition
* @link_id: the link ID for MLO, must be 0 for non-MLO
*
* Caller must hold wiphy mutex, therefore must only be called from sleepable
* driver context!
*/
void cfg80211_ch_switch_notify(struct net_device *dev,
struct cfg80211_chan_def *chandef,
unsigned int link_id);
/*
* cfg80211_ch_switch_started_notify - notify channel switch start
* @dev: the device on which the channel switch started
* @chandef: the future channel definition
* @link_id: the link ID for MLO, must be 0 for non-MLO
* @count: the number of TBTTs until the channel switch happens
* @quiet: whether or not immediate quiet was requested by the AP
*
* Inform the userspace about the channel switch that has just
* started, so that it can take appropriate actions (eg. starting
* channel switch on other vifs), if necessary.
*/
void cfg80211_ch_switch_started_notify(struct net_device *dev,
struct cfg80211_chan_def *chandef,
unsigned int link_id, u8 count,
bool quiet);
/**
* ieee80211_operating_class_to_band - convert operating class to band
*
* @operating_class: the operating class to convert
* @band: band pointer to fill
*
* Returns %true if the conversion was successful, %false otherwise.
*/
bool ieee80211_operating_class_to_band(u8 operating_class,
enum nl80211_band *band);
/**
* ieee80211_operating_class_to_chandef - convert operating class to chandef
*
* @operating_class: the operating class to convert
* @chan: the ieee80211_channel to convert
* @chandef: a pointer to the resulting chandef
*
* Returns %true if the conversion was successful, %false otherwise.
*/
bool ieee80211_operating_class_to_chandef(u8 operating_class,
struct ieee80211_channel *chan,
struct cfg80211_chan_def *chandef);
/**
* ieee80211_chandef_to_operating_class - convert chandef to operation class
*
* @chandef: the chandef to convert
* @op_class: a pointer to the resulting operating class
*
* Returns %true if the conversion was successful, %false otherwise.
*/
bool ieee80211_chandef_to_operating_class(struct cfg80211_chan_def *chandef,
u8 *op_class);
/**
* ieee80211_chandef_to_khz - convert chandef to frequency in KHz
*
* @chandef: the chandef to convert
*
* Returns the center frequency of chandef (1st segment) in KHz.
*/
static inline u32
ieee80211_chandef_to_khz(const struct cfg80211_chan_def *chandef)
{
return MHZ_TO_KHZ(chandef->center_freq1) + chandef->freq1_offset;
}
/*
* cfg80211_tdls_oper_request - request userspace to perform TDLS operation
* @dev: the device on which the operation is requested
* @peer: the MAC address of the peer device
* @oper: the requested TDLS operation (NL80211_TDLS_SETUP or
* NL80211_TDLS_TEARDOWN)
* @reason_code: the reason code for teardown request
* @gfp: allocation flags
*
* This function is used to request userspace to perform TDLS operation that
* requires knowledge of keys, i.e., link setup or teardown when the AP
* connection uses encryption. This is optional mechanism for the driver to use
* if it can automatically determine when a TDLS link could be useful (e.g.,
* based on traffic and signal strength for a peer).
*/
void cfg80211_tdls_oper_request(struct net_device *dev, const u8 *peer,
enum nl80211_tdls_operation oper,
u16 reason_code, gfp_t gfp);
/*
* cfg80211_calculate_bitrate - calculate actual bitrate (in 100Kbps units)
* @rate: given rate_info to calculate bitrate from
*
* return 0 if MCS index >= 32
*/
u32 cfg80211_calculate_bitrate(struct rate_info *rate);
/**
* cfg80211_unregister_wdev - remove the given wdev
* @wdev: struct wireless_dev to remove
*
* This function removes the device so it can no longer be used. It is necessary
* to call this function even when cfg80211 requests the removal of the device
* by calling the del_virtual_intf() callback. The function must also be called
* when the driver wishes to unregister the wdev, e.g. when the hardware device
* is unbound from the driver.
*
* Requires the RTNL and wiphy mutex to be held.
*/
void cfg80211_unregister_wdev(struct wireless_dev *wdev);
/**
* cfg80211_register_netdevice - register the given netdev
* @dev: the netdev to register
*
* Note: In contexts coming from cfg80211 callbacks, you must call this rather
* than register_netdevice(), unregister_netdev() is impossible as the RTNL is
* held. Otherwise, both register_netdevice() and register_netdev() are usable
* instead as well.
*
* Requires the RTNL and wiphy mutex to be held.
*/
int cfg80211_register_netdevice(struct net_device *dev);
/**
* cfg80211_unregister_netdevice - unregister the given netdev
* @dev: the netdev to register
*
* Note: In contexts coming from cfg80211 callbacks, you must call this rather
* than unregister_netdevice(), unregister_netdev() is impossible as the RTNL
* is held. Otherwise, both unregister_netdevice() and unregister_netdev() are
* usable instead as well.
*
* Requires the RTNL and wiphy mutex to be held.
*/
static inline void cfg80211_unregister_netdevice(struct net_device *dev)
{
#if IS_ENABLED(CONFIG_CFG80211)
cfg80211_unregister_wdev(dev->ieee80211_ptr);
#endif
}
/**
* struct cfg80211_ft_event_params - FT Information Elements
* @ies: FT IEs
* @ies_len: length of the FT IE in bytes
* @target_ap: target AP's MAC address
* @ric_ies: RIC IE
* @ric_ies_len: length of the RIC IE in bytes
*/
struct cfg80211_ft_event_params {
const u8 *ies;
size_t ies_len;
const u8 *target_ap;
const u8 *ric_ies;
size_t ric_ies_len;
};
/**
* cfg80211_ft_event - notify userspace about FT IE and RIC IE
* @netdev: network device
* @ft_event: IE information
*/
void cfg80211_ft_event(struct net_device *netdev,
struct cfg80211_ft_event_params *ft_event);
/**
* cfg80211_get_p2p_attr - find and copy a P2P attribute from IE buffer
* @ies: the input IE buffer
* @len: the input length
* @attr: the attribute ID to find
* @buf: output buffer, can be %NULL if the data isn't needed, e.g.
* if the function is only called to get the needed buffer size
* @bufsize: size of the output buffer
*
* The function finds a given P2P attribute in the (vendor) IEs and
* copies its contents to the given buffer.
*
* Return: A negative error code (-%EILSEQ or -%ENOENT) if the data is
* malformed or the attribute can't be found (respectively), or the
* length of the found attribute (which can be zero).
*/
int cfg80211_get_p2p_attr(const u8 *ies, unsigned int len,
enum ieee80211_p2p_attr_id attr,
u8 *buf, unsigned int bufsize);
/**
* ieee80211_ie_split_ric - split an IE buffer according to ordering (with RIC)
* @ies: the IE buffer
* @ielen: the length of the IE buffer
* @ids: an array with element IDs that are allowed before
* the split. A WLAN_EID_EXTENSION value means that the next
* EID in the list is a sub-element of the EXTENSION IE.
* @n_ids: the size of the element ID array
* @after_ric: array IE types that come after the RIC element
* @n_after_ric: size of the @after_ric array
* @offset: offset where to start splitting in the buffer
*
* This function splits an IE buffer by updating the @offset
* variable to point to the location where the buffer should be
* split.
*
* It assumes that the given IE buffer is well-formed, this
* has to be guaranteed by the caller!
*
* It also assumes that the IEs in the buffer are ordered
* correctly, if not the result of using this function will not
* be ordered correctly either, i.e. it does no reordering.
*
* The function returns the offset where the next part of the
* buffer starts, which may be @ielen if the entire (remainder)
* of the buffer should be used.
*/
size_t ieee80211_ie_split_ric(const u8 *ies, size_t ielen,
const u8 *ids, int n_ids,
const u8 *after_ric, int n_after_ric,
size_t offset);
/**
* ieee80211_ie_split - split an IE buffer according to ordering
* @ies: the IE buffer
* @ielen: the length of the IE buffer
* @ids: an array with element IDs that are allowed before
* the split. A WLAN_EID_EXTENSION value means that the next
* EID in the list is a sub-element of the EXTENSION IE.
* @n_ids: the size of the element ID array
* @offset: offset where to start splitting in the buffer
*
* This function splits an IE buffer by updating the @offset
* variable to point to the location where the buffer should be
* split.
*
* It assumes that the given IE buffer is well-formed, this
* has to be guaranteed by the caller!
*
* It also assumes that the IEs in the buffer are ordered
* correctly, if not the result of using this function will not
* be ordered correctly either, i.e. it does no reordering.
*
* The function returns the offset where the next part of the
* buffer starts, which may be @ielen if the entire (remainder)
* of the buffer should be used.
*/
static inline size_t ieee80211_ie_split(const u8 *ies, size_t ielen,
const u8 *ids, int n_ids, size_t offset)
{
return ieee80211_ie_split_ric(ies, ielen, ids, n_ids, NULL, 0, offset);
}
/**
* ieee80211_fragment_element - fragment the last element in skb
* @skb: The skbuf that the element was added to
* @len_pos: Pointer to length of the element to fragment
* @frag_id: The element ID to use for fragments
*
* This function fragments all data after @len_pos, adding fragmentation
* elements with the given ID as appropriate. The SKB will grow in size
* accordingly.
*/
void ieee80211_fragment_element(struct sk_buff *skb, u8 *len_pos, u8 frag_id);
/**
* cfg80211_report_wowlan_wakeup - report wakeup from WoWLAN
* @wdev: the wireless device reporting the wakeup
* @wakeup: the wakeup report
* @gfp: allocation flags
*
* This function reports that the given device woke up. If it
* caused the wakeup, report the reason(s), otherwise you may
* pass %NULL as the @wakeup parameter to advertise that something
* else caused the wakeup.
*/
void cfg80211_report_wowlan_wakeup(struct wireless_dev *wdev,
struct cfg80211_wowlan_wakeup *wakeup,
gfp_t gfp);
/**
* cfg80211_crit_proto_stopped() - indicate critical protocol stopped by driver.
*
* @wdev: the wireless device for which critical protocol is stopped.
* @gfp: allocation flags
*
* This function can be called by the driver to indicate it has reverted
* operation back to normal. One reason could be that the duration given
* by .crit_proto_start() has expired.
*/
void cfg80211_crit_proto_stopped(struct wireless_dev *wdev, gfp_t gfp);
/**
* ieee80211_get_num_supported_channels - get number of channels device has
* @wiphy: the wiphy
*
* Return: the number of channels supported by the device.
*/
unsigned int ieee80211_get_num_supported_channels(struct wiphy *wiphy);
/**
* cfg80211_check_combinations - check interface combinations
*
* @wiphy: the wiphy
* @params: the interface combinations parameter
*
* This function can be called by the driver to check whether a
* combination of interfaces and their types are allowed according to
* the interface combinations.
*/
int cfg80211_check_combinations(struct wiphy *wiphy,
struct iface_combination_params *params);
/**
* cfg80211_iter_combinations - iterate over matching combinations
*
* @wiphy: the wiphy
* @params: the interface combinations parameter
* @iter: function to call for each matching combination
* @data: pointer to pass to iter function
*
* This function can be called by the driver to check what possible
* combinations it fits in at a given moment, e.g. for channel switching
* purposes.
*/
int cfg80211_iter_combinations(struct wiphy *wiphy,
struct iface_combination_params *params,
void (*iter)(const struct ieee80211_iface_combination *c,
void *data),
void *data);
/*
* cfg80211_stop_iface - trigger interface disconnection
*
* @wiphy: the wiphy
* @wdev: wireless device
* @gfp: context flags
*
* Trigger interface to be stopped as if AP was stopped, IBSS/mesh left, STA
* disconnected.
*
* Note: This doesn't need any locks and is asynchronous.
*/
void cfg80211_stop_iface(struct wiphy *wiphy, struct wireless_dev *wdev,
gfp_t gfp);
/**
* cfg80211_shutdown_all_interfaces - shut down all interfaces for a wiphy
* @wiphy: the wiphy to shut down
*
* This function shuts down all interfaces belonging to this wiphy by
* calling dev_close() (and treating non-netdev interfaces as needed).
* It shouldn't really be used unless there are some fatal device errors
* that really can't be recovered in any other way.
*
* Callers must hold the RTNL and be able to deal with callbacks into
* the driver while the function is running.
*/
void cfg80211_shutdown_all_interfaces(struct wiphy *wiphy);
/**
* wiphy_ext_feature_set - set the extended feature flag
*
* @wiphy: the wiphy to modify.
* @ftidx: extended feature bit index.
*
* The extended features are flagged in multiple bytes (see
* &struct wiphy.@ext_features)
*/
static inline void wiphy_ext_feature_set(struct wiphy *wiphy,
enum nl80211_ext_feature_index ftidx)
{
u8 *ft_byte;
ft_byte = &wiphy->ext_features[ftidx / 8];
*ft_byte |= BIT(ftidx % 8);
}
/**
* wiphy_ext_feature_isset - check the extended feature flag
*
* @wiphy: the wiphy to modify.
* @ftidx: extended feature bit index.
*
* The extended features are flagged in multiple bytes (see
* &struct wiphy.@ext_features)
*/
static inline bool
wiphy_ext_feature_isset(struct wiphy *wiphy,
enum nl80211_ext_feature_index ftidx)
{
u8 ft_byte;
ft_byte = wiphy->ext_features[ftidx / 8];
return (ft_byte & BIT(ftidx % 8)) != 0;
}
/**
* cfg80211_free_nan_func - free NAN function
* @f: NAN function that should be freed
*
* Frees all the NAN function and all it's allocated members.
*/
void cfg80211_free_nan_func(struct cfg80211_nan_func *f);
/**
* struct cfg80211_nan_match_params - NAN match parameters
* @type: the type of the function that triggered a match. If it is
* %NL80211_NAN_FUNC_SUBSCRIBE it means that we replied to a subscriber.
* If it is %NL80211_NAN_FUNC_PUBLISH, it means that we got a discovery
* result.
* If it is %NL80211_NAN_FUNC_FOLLOW_UP, we received a follow up.
* @inst_id: the local instance id
* @peer_inst_id: the instance id of the peer's function
* @addr: the MAC address of the peer
* @info_len: the length of the &info
* @info: the Service Specific Info from the peer (if any)
* @cookie: unique identifier of the corresponding function
*/
struct cfg80211_nan_match_params {
enum nl80211_nan_function_type type;
u8 inst_id;
u8 peer_inst_id;
const u8 *addr;
u8 info_len;
const u8 *info;
u64 cookie;
};
/**
* cfg80211_nan_match - report a match for a NAN function.
* @wdev: the wireless device reporting the match
* @match: match notification parameters
* @gfp: allocation flags
*
* This function reports that the a NAN function had a match. This
* can be a subscribe that had a match or a solicited publish that
* was sent. It can also be a follow up that was received.
*/
void cfg80211_nan_match(struct wireless_dev *wdev,
struct cfg80211_nan_match_params *match, gfp_t gfp);
/**
* cfg80211_nan_func_terminated - notify about NAN function termination.
*
* @wdev: the wireless device reporting the match
* @inst_id: the local instance id
* @reason: termination reason (one of the NL80211_NAN_FUNC_TERM_REASON_*)
* @cookie: unique NAN function identifier
* @gfp: allocation flags
*
* This function reports that the a NAN function is terminated.
*/
void cfg80211_nan_func_terminated(struct wireless_dev *wdev,
u8 inst_id,
enum nl80211_nan_func_term_reason reason,
u64 cookie, gfp_t gfp);
/* ethtool helper */
void cfg80211_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info);
/**
* cfg80211_external_auth_request - userspace request for authentication
* @netdev: network device
* @params: External authentication parameters
* @gfp: allocation flags
* Returns: 0 on success, < 0 on error
*/
int cfg80211_external_auth_request(struct net_device *netdev,
struct cfg80211_external_auth_params *params,
gfp_t gfp);
/**
* cfg80211_pmsr_report - report peer measurement result data
* @wdev: the wireless device reporting the measurement
* @req: the original measurement request
* @result: the result data
* @gfp: allocation flags
*/
void cfg80211_pmsr_report(struct wireless_dev *wdev,
struct cfg80211_pmsr_request *req,
struct cfg80211_pmsr_result *result,
gfp_t gfp);
/**
* cfg80211_pmsr_complete - report peer measurement completed
* @wdev: the wireless device reporting the measurement
* @req: the original measurement request
* @gfp: allocation flags
*
* Report that the entire measurement completed, after this
* the request pointer will no longer be valid.
*/
void cfg80211_pmsr_complete(struct wireless_dev *wdev,
struct cfg80211_pmsr_request *req,
gfp_t gfp);
/**
* cfg80211_iftype_allowed - check whether the interface can be allowed
* @wiphy: the wiphy
* @iftype: interface type
* @is_4addr: use_4addr flag, must be '0' when check_swif is '1'
* @check_swif: check iftype against software interfaces
*
* Check whether the interface is allowed to operate; additionally, this API
* can be used to check iftype against the software interfaces when
* check_swif is '1'.
*/
bool cfg80211_iftype_allowed(struct wiphy *wiphy, enum nl80211_iftype iftype,
bool is_4addr, u8 check_swif);
/**
* cfg80211_assoc_comeback - notification of association that was
* temporarily rejected with a comeback
* @netdev: network device
* @ap_addr: AP (MLD) address that rejected the association
* @timeout: timeout interval value TUs.
*
* this function may sleep. the caller must hold the corresponding wdev's mutex.
*/
void cfg80211_assoc_comeback(struct net_device *netdev,
const u8 *ap_addr, u32 timeout);
/* Logging, debugging and troubleshooting/diagnostic helpers. */
/* wiphy_printk helpers, similar to dev_printk */
#define wiphy_printk(level, wiphy, format, args...) \
dev_printk(level, &(wiphy)->dev, format, ##args)
#define wiphy_emerg(wiphy, format, args...) \
dev_emerg(&(wiphy)->dev, format, ##args)
#define wiphy_alert(wiphy, format, args...) \
dev_alert(&(wiphy)->dev, format, ##args)
#define wiphy_crit(wiphy, format, args...) \
dev_crit(&(wiphy)->dev, format, ##args)
#define wiphy_err(wiphy, format, args...) \
dev_err(&(wiphy)->dev, format, ##args)
#define wiphy_warn(wiphy, format, args...) \
dev_warn(&(wiphy)->dev, format, ##args)
#define wiphy_notice(wiphy, format, args...) \
dev_notice(&(wiphy)->dev, format, ##args)
#define wiphy_info(wiphy, format, args...) \
dev_info(&(wiphy)->dev, format, ##args)
#define wiphy_info_once(wiphy, format, args...) \
dev_info_once(&(wiphy)->dev, format, ##args)
#define wiphy_err_ratelimited(wiphy, format, args...) \
dev_err_ratelimited(&(wiphy)->dev, format, ##args)
#define wiphy_warn_ratelimited(wiphy, format, args...) \
dev_warn_ratelimited(&(wiphy)->dev, format, ##args)
#define wiphy_debug(wiphy, format, args...) \
wiphy_printk(KERN_DEBUG, wiphy, format, ##args)
#define wiphy_dbg(wiphy, format, args...) \
dev_dbg(&(wiphy)->dev, format, ##args)
#if defined(VERBOSE_DEBUG)
#define wiphy_vdbg wiphy_dbg
#else
#define wiphy_vdbg(wiphy, format, args...) \
({ \
if (0) \
wiphy_printk(KERN_DEBUG, wiphy, format, ##args); \
0; \
})
#endif
/*
* wiphy_WARN() acts like wiphy_printk(), but with the key difference
* of using a WARN/WARN_ON to get the message out, including the
* file/line information and a backtrace.
*/
#define wiphy_WARN(wiphy, format, args...) \
WARN(1, "wiphy: %s\n" format, wiphy_name(wiphy), ##args);
/**
* cfg80211_update_owe_info_event - Notify the peer's OWE info to user space
* @netdev: network device
* @owe_info: peer's owe info
* @gfp: allocation flags
*/
void cfg80211_update_owe_info_event(struct net_device *netdev,
struct cfg80211_update_owe_info *owe_info,
gfp_t gfp);
/**
* cfg80211_bss_flush - resets all the scan entries
* @wiphy: the wiphy
*/
void cfg80211_bss_flush(struct wiphy *wiphy);
/**
* cfg80211_bss_color_notify - notify about bss color event
* @dev: network device
* @cmd: the actual event we want to notify
* @count: the number of TBTTs until the color change happens
* @color_bitmap: representations of the colors that the local BSS is aware of
*/
int cfg80211_bss_color_notify(struct net_device *dev,
enum nl80211_commands cmd, u8 count,
u64 color_bitmap);
/**
* cfg80211_obss_color_collision_notify - notify about bss color collision
* @dev: network device
* @color_bitmap: representations of the colors that the local BSS is aware of
*/
static inline int cfg80211_obss_color_collision_notify(struct net_device *dev,
u64 color_bitmap)
{
return cfg80211_bss_color_notify(dev, NL80211_CMD_OBSS_COLOR_COLLISION,
0, color_bitmap);
}
/**
* cfg80211_color_change_started_notify - notify color change start
* @dev: the device on which the color is switched
* @count: the number of TBTTs until the color change happens
*
* Inform the userspace about the color change that has started.
*/
static inline int cfg80211_color_change_started_notify(struct net_device *dev,
u8 count)
{
return cfg80211_bss_color_notify(dev, NL80211_CMD_COLOR_CHANGE_STARTED,
count, 0);
}
/**
* cfg80211_color_change_aborted_notify - notify color change abort
* @dev: the device on which the color is switched
*
* Inform the userspace about the color change that has aborted.
*/
static inline int cfg80211_color_change_aborted_notify(struct net_device *dev)
{
return cfg80211_bss_color_notify(dev, NL80211_CMD_COLOR_CHANGE_ABORTED,
0, 0);
}
/**
* cfg80211_color_change_notify - notify color change completion
* @dev: the device on which the color was switched
*
* Inform the userspace about the color change that has completed.
*/
static inline int cfg80211_color_change_notify(struct net_device *dev)
{
return cfg80211_bss_color_notify(dev,
NL80211_CMD_COLOR_CHANGE_COMPLETED,
0, 0);
}
/**
* cfg80211_links_removed - Notify about removed STA MLD setup links.
* @dev: network device.
* @link_mask: BIT mask of removed STA MLD setup link IDs.
*
* Inform cfg80211 and the userspace about removed STA MLD setup links due to
* AP MLD removing the corresponding affiliated APs with Multi-Link
* reconfiguration. Note that it's not valid to remove all links, in this
* case disconnect instead.
* Also note that the wdev mutex must be held.
*/
void cfg80211_links_removed(struct net_device *dev, u16 link_mask);
/**
* cfg80211_schedule_channels_check - schedule regulatory check if needed
* @wdev: the wireless device to check
*
* In case the device supports NO_IR or DFS relaxations, schedule regulatory
* channels check, as previous concurrent operation conditions may not
* hold anymore.
*/
void cfg80211_schedule_channels_check(struct wireless_dev *wdev);
#ifdef CONFIG_CFG80211_DEBUGFS
/**
* wiphy_locked_debugfs_read - do a locked read in debugfs
* @wiphy: the wiphy to use
* @file: the file being read
* @buf: the buffer to fill and then read from
* @bufsize: size of the buffer
* @userbuf: the user buffer to copy to
* @count: read count
* @ppos: read position
* @handler: the read handler to call (under wiphy lock)
* @data: additional data to pass to the read handler
*/
ssize_t wiphy_locked_debugfs_read(struct wiphy *wiphy, struct file *file,
char *buf, size_t bufsize,
char __user *userbuf, size_t count,
loff_t *ppos,
ssize_t (*handler)(struct wiphy *wiphy,
struct file *file,
char *buf,
size_t bufsize,
void *data),
void *data);
/**
* wiphy_locked_debugfs_write - do a locked write in debugfs
* @wiphy: the wiphy to use
* @file: the file being written to
* @buf: the buffer to copy the user data to
* @bufsize: size of the buffer
* @userbuf: the user buffer to copy from
* @count: read count
* @handler: the write handler to call (under wiphy lock)
* @data: additional data to pass to the write handler
*/
ssize_t wiphy_locked_debugfs_write(struct wiphy *wiphy, struct file *file,
char *buf, size_t bufsize,
const char __user *userbuf, size_t count,
ssize_t (*handler)(struct wiphy *wiphy,
struct file *file,
char *buf,
size_t count,
void *data),
void *data);
#endif
#endif /* __NET_CFG80211_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Input Multitouch Library
*
* Copyright (c) 2008-2010 Henrik Rydberg
*/
#include <linux/input/mt.h>
#include <linux/export.h>
#include <linux/slab.h>
#include "input-core-private.h"
#define TRKID_SGN ((TRKID_MAX + 1) >> 1)
static void copy_abs(struct input_dev *dev, unsigned int dst, unsigned int src)
{
if (dev->absinfo && test_bit(src, dev->absbit)) {
dev->absinfo[dst] = dev->absinfo[src];
dev->absinfo[dst].fuzz = 0;
__set_bit(dst, dev->absbit);
}
}
/**
* input_mt_init_slots() - initialize MT input slots
* @dev: input device supporting MT events and finger tracking
* @num_slots: number of slots used by the device
* @flags: mt tasks to handle in core
*
* This function allocates all necessary memory for MT slot handling
* in the input device, prepares the ABS_MT_SLOT and
* ABS_MT_TRACKING_ID events for use and sets up appropriate buffers.
* Depending on the flags set, it also performs pointer emulation and
* frame synchronization.
*
* May be called repeatedly. Returns -EINVAL if attempting to
* reinitialize with a different number of slots.
*/
int input_mt_init_slots(struct input_dev *dev, unsigned int num_slots,
unsigned int flags)
{
struct input_mt *mt = dev->mt;
int i;
if (!num_slots)
return 0;
if (mt)
return mt->num_slots != num_slots ? -EINVAL : 0;
mt = kzalloc(struct_size(mt, slots, num_slots), GFP_KERNEL);
if (!mt)
goto err_mem;
mt->num_slots = num_slots;
mt->flags = flags;
input_set_abs_params(dev, ABS_MT_SLOT, 0, num_slots - 1, 0, 0);
input_set_abs_params(dev, ABS_MT_TRACKING_ID, 0, TRKID_MAX, 0, 0);
if (flags & (INPUT_MT_POINTER | INPUT_MT_DIRECT)) {
__set_bit(EV_KEY, dev->evbit);
__set_bit(BTN_TOUCH, dev->keybit);
copy_abs(dev, ABS_X, ABS_MT_POSITION_X);
copy_abs(dev, ABS_Y, ABS_MT_POSITION_Y);
copy_abs(dev, ABS_PRESSURE, ABS_MT_PRESSURE);
}
if (flags & INPUT_MT_POINTER) {
__set_bit(BTN_TOOL_FINGER, dev->keybit);
__set_bit(BTN_TOOL_DOUBLETAP, dev->keybit);
if (num_slots >= 3)
__set_bit(BTN_TOOL_TRIPLETAP, dev->keybit);
if (num_slots >= 4)
__set_bit(BTN_TOOL_QUADTAP, dev->keybit);
if (num_slots >= 5)
__set_bit(BTN_TOOL_QUINTTAP, dev->keybit);
__set_bit(INPUT_PROP_POINTER, dev->propbit);
}
if (flags & INPUT_MT_DIRECT)
__set_bit(INPUT_PROP_DIRECT, dev->propbit);
if (flags & INPUT_MT_SEMI_MT)
__set_bit(INPUT_PROP_SEMI_MT, dev->propbit);
if (flags & INPUT_MT_TRACK) {
unsigned int n2 = num_slots * num_slots;
mt->red = kcalloc(n2, sizeof(*mt->red), GFP_KERNEL);
if (!mt->red)
goto err_mem;
}
/* Mark slots as 'inactive' */
for (i = 0; i < num_slots; i++)
input_mt_set_value(&mt->slots[i], ABS_MT_TRACKING_ID, -1);
/* Mark slots as 'unused' */
mt->frame = 1;
dev->mt = mt;
return 0;
err_mem:
kfree(mt);
return -ENOMEM;
}
EXPORT_SYMBOL(input_mt_init_slots);
/**
* input_mt_destroy_slots() - frees the MT slots of the input device
* @dev: input device with allocated MT slots
*
* This function is only needed in error path as the input core will
* automatically free the MT slots when the device is destroyed.
*/
void input_mt_destroy_slots(struct input_dev *dev)
{
if (dev->mt) { kfree(dev->mt->red);
kfree(dev->mt);
}
dev->mt = NULL;
}
EXPORT_SYMBOL(input_mt_destroy_slots);
/**
* input_mt_report_slot_state() - report contact state
* @dev: input device with allocated MT slots
* @tool_type: the tool type to use in this slot
* @active: true if contact is active, false otherwise
*
* Reports a contact via ABS_MT_TRACKING_ID, and optionally
* ABS_MT_TOOL_TYPE. If active is true and the slot is currently
* inactive, or if the tool type is changed, a new tracking id is
* assigned to the slot. The tool type is only reported if the
* corresponding absbit field is set.
*
* Returns true if contact is active.
*/
bool input_mt_report_slot_state(struct input_dev *dev,
unsigned int tool_type, bool active)
{
struct input_mt *mt = dev->mt;
struct input_mt_slot *slot;
int id;
if (!mt)
return false;
slot = &mt->slots[mt->slot];
slot->frame = mt->frame;
if (!active) {
input_event(dev, EV_ABS, ABS_MT_TRACKING_ID, -1);
return false;
}
id = input_mt_get_value(slot, ABS_MT_TRACKING_ID);
if (id < 0)
id = input_mt_new_trkid(mt);
input_event(dev, EV_ABS, ABS_MT_TRACKING_ID, id);
input_event(dev, EV_ABS, ABS_MT_TOOL_TYPE, tool_type);
return true;
}
EXPORT_SYMBOL(input_mt_report_slot_state);
/**
* input_mt_report_finger_count() - report contact count
* @dev: input device with allocated MT slots
* @count: the number of contacts
*
* Reports the contact count via BTN_TOOL_FINGER, BTN_TOOL_DOUBLETAP,
* BTN_TOOL_TRIPLETAP and BTN_TOOL_QUADTAP.
*
* The input core ensures only the KEY events already setup for
* this device will produce output.
*/
void input_mt_report_finger_count(struct input_dev *dev, int count)
{
input_event(dev, EV_KEY, BTN_TOOL_FINGER, count == 1);
input_event(dev, EV_KEY, BTN_TOOL_DOUBLETAP, count == 2);
input_event(dev, EV_KEY, BTN_TOOL_TRIPLETAP, count == 3);
input_event(dev, EV_KEY, BTN_TOOL_QUADTAP, count == 4);
input_event(dev, EV_KEY, BTN_TOOL_QUINTTAP, count == 5);
}
EXPORT_SYMBOL(input_mt_report_finger_count);
/**
* input_mt_report_pointer_emulation() - common pointer emulation
* @dev: input device with allocated MT slots
* @use_count: report number of active contacts as finger count
*
* Performs legacy pointer emulation via BTN_TOUCH, ABS_X, ABS_Y and
* ABS_PRESSURE. Touchpad finger count is emulated if use_count is true.
*
* The input core ensures only the KEY and ABS axes already setup for
* this device will produce output.
*/
void input_mt_report_pointer_emulation(struct input_dev *dev, bool use_count)
{
struct input_mt *mt = dev->mt;
struct input_mt_slot *oldest;
int oldid, count, i;
if (!mt)
return;
oldest = NULL;
oldid = mt->trkid;
count = 0;
for (i = 0; i < mt->num_slots; ++i) {
struct input_mt_slot *ps = &mt->slots[i];
int id = input_mt_get_value(ps, ABS_MT_TRACKING_ID);
if (id < 0)
continue;
if ((id - oldid) & TRKID_SGN) {
oldest = ps;
oldid = id;
}
count++;
}
input_event(dev, EV_KEY, BTN_TOUCH, count > 0);
if (use_count) {
if (count == 0 &&
!test_bit(ABS_MT_DISTANCE, dev->absbit) &&
test_bit(ABS_DISTANCE, dev->absbit) &&
input_abs_get_val(dev, ABS_DISTANCE) != 0) {
/*
* Force reporting BTN_TOOL_FINGER for devices that
* only report general hover (and not per-contact
* distance) when contact is in proximity but not
* on the surface.
*/
count = 1;
}
input_mt_report_finger_count(dev, count);
}
if (oldest) {
int x = input_mt_get_value(oldest, ABS_MT_POSITION_X);
int y = input_mt_get_value(oldest, ABS_MT_POSITION_Y);
input_event(dev, EV_ABS, ABS_X, x);
input_event(dev, EV_ABS, ABS_Y, y);
if (test_bit(ABS_MT_PRESSURE, dev->absbit)) {
int p = input_mt_get_value(oldest, ABS_MT_PRESSURE);
input_event(dev, EV_ABS, ABS_PRESSURE, p);
}
} else {
if (test_bit(ABS_MT_PRESSURE, dev->absbit))
input_event(dev, EV_ABS, ABS_PRESSURE, 0);
}
}
EXPORT_SYMBOL(input_mt_report_pointer_emulation);
static void __input_mt_drop_unused(struct input_dev *dev, struct input_mt *mt)
{
int i;
lockdep_assert_held(&dev->event_lock);
for (i = 0; i < mt->num_slots; i++) {
if (input_mt_is_active(&mt->slots[i]) &&
!input_mt_is_used(mt, &mt->slots[i])) {
input_handle_event(dev, EV_ABS, ABS_MT_SLOT, i);
input_handle_event(dev, EV_ABS, ABS_MT_TRACKING_ID, -1);
}
}
}
/**
* input_mt_drop_unused() - Inactivate slots not seen in this frame
* @dev: input device with allocated MT slots
*
* Lift all slots not seen since the last call to this function.
*/
void input_mt_drop_unused(struct input_dev *dev)
{
struct input_mt *mt = dev->mt;
if (mt) {
unsigned long flags;
spin_lock_irqsave(&dev->event_lock, flags);
__input_mt_drop_unused(dev, mt);
mt->frame++;
spin_unlock_irqrestore(&dev->event_lock, flags);
}
}
EXPORT_SYMBOL(input_mt_drop_unused);
/**
* input_mt_release_slots() - Deactivate all slots
* @dev: input device with allocated MT slots
*
* Lift all active slots.
*/
void input_mt_release_slots(struct input_dev *dev)
{
struct input_mt *mt = dev->mt;
lockdep_assert_held(&dev->event_lock);
if (mt) {
/* This will effectively mark all slots unused. */
mt->frame++;
__input_mt_drop_unused(dev, mt);
if (test_bit(ABS_PRESSURE, dev->absbit))
input_handle_event(dev, EV_ABS, ABS_PRESSURE, 0);
mt->frame++;
}
}
/**
* input_mt_sync_frame() - synchronize mt frame
* @dev: input device with allocated MT slots
*
* Close the frame and prepare the internal state for a new one.
* Depending on the flags, marks unused slots as inactive and performs
* pointer emulation.
*/
void input_mt_sync_frame(struct input_dev *dev)
{
struct input_mt *mt = dev->mt;
bool use_count = false;
if (!mt)
return;
if (mt->flags & INPUT_MT_DROP_UNUSED) {
unsigned long flags;
spin_lock_irqsave(&dev->event_lock, flags);
__input_mt_drop_unused(dev, mt);
spin_unlock_irqrestore(&dev->event_lock, flags);
}
if ((mt->flags & INPUT_MT_POINTER) && !(mt->flags & INPUT_MT_SEMI_MT))
use_count = true;
input_mt_report_pointer_emulation(dev, use_count);
mt->frame++;
}
EXPORT_SYMBOL(input_mt_sync_frame);
static int adjust_dual(int *begin, int step, int *end, int eq, int mu)
{
int f, *p, s, c;
if (begin == end)
return 0;
f = *begin;
p = begin + step;
s = p == end ? f + 1 : *p;
for (; p != end; p += step) {
if (*p < f) {
s = f;
f = *p;
} else if (*p < s) {
s = *p;
}
}
c = (f + s + 1) / 2;
if (c == 0 || (c > mu && (!eq || mu > 0)))
return 0;
/* Improve convergence for positive matrices by penalizing overcovers */
if (s < 0 && mu <= 0)
c *= 2;
for (p = begin; p != end; p += step)
*p -= c;
return (c < s && s <= 0) || (f >= 0 && f < c);
}
static void find_reduced_matrix(int *w, int nr, int nc, int nrc, int mu)
{
int i, k, sum;
for (k = 0; k < nrc; k++) {
for (i = 0; i < nr; i++)
adjust_dual(w + i, nr, w + i + nrc, nr <= nc, mu);
sum = 0;
for (i = 0; i < nrc; i += nr)
sum += adjust_dual(w + i, 1, w + i + nr, nc <= nr, mu);
if (!sum)
break;
}
}
static int input_mt_set_matrix(struct input_mt *mt,
const struct input_mt_pos *pos, int num_pos,
int mu)
{
const struct input_mt_pos *p;
struct input_mt_slot *s;
int *w = mt->red;
int x, y;
for (s = mt->slots; s != mt->slots + mt->num_slots; s++) {
if (!input_mt_is_active(s))
continue;
x = input_mt_get_value(s, ABS_MT_POSITION_X);
y = input_mt_get_value(s, ABS_MT_POSITION_Y);
for (p = pos; p != pos + num_pos; p++) {
int dx = x - p->x, dy = y - p->y;
*w++ = dx * dx + dy * dy - mu;
}
}
return w - mt->red;
}
static void input_mt_set_slots(struct input_mt *mt,
int *slots, int num_pos)
{
struct input_mt_slot *s;
int *w = mt->red, j;
for (j = 0; j != num_pos; j++)
slots[j] = -1;
for (s = mt->slots; s != mt->slots + mt->num_slots; s++) {
if (!input_mt_is_active(s))
continue;
for (j = 0; j != num_pos; j++) {
if (w[j] < 0) {
slots[j] = s - mt->slots;
break;
}
}
w += num_pos;
}
for (s = mt->slots; s != mt->slots + mt->num_slots; s++) {
if (input_mt_is_active(s))
continue;
for (j = 0; j != num_pos; j++) {
if (slots[j] < 0) {
slots[j] = s - mt->slots;
break;
}
}
}
}
/**
* input_mt_assign_slots() - perform a best-match assignment
* @dev: input device with allocated MT slots
* @slots: the slot assignment to be filled
* @pos: the position array to match
* @num_pos: number of positions
* @dmax: maximum ABS_MT_POSITION displacement (zero for infinite)
*
* Performs a best match against the current contacts and returns
* the slot assignment list. New contacts are assigned to unused
* slots.
*
* The assignments are balanced so that all coordinate displacements are
* below the euclidian distance dmax. If no such assignment can be found,
* some contacts are assigned to unused slots.
*
* Returns zero on success, or negative error in case of failure.
*/
int input_mt_assign_slots(struct input_dev *dev, int *slots,
const struct input_mt_pos *pos, int num_pos,
int dmax)
{
struct input_mt *mt = dev->mt;
int mu = 2 * dmax * dmax;
int nrc;
if (!mt || !mt->red)
return -ENXIO;
if (num_pos > mt->num_slots)
return -EINVAL;
if (num_pos < 1)
return 0;
nrc = input_mt_set_matrix(mt, pos, num_pos, mu);
find_reduced_matrix(mt->red, num_pos, nrc / num_pos, nrc, mu);
input_mt_set_slots(mt, slots, num_pos);
return 0;
}
EXPORT_SYMBOL(input_mt_assign_slots);
/**
* input_mt_get_slot_by_key() - return slot matching key
* @dev: input device with allocated MT slots
* @key: the key of the sought slot
*
* Returns the slot of the given key, if it exists, otherwise
* set the key on the first unused slot and return.
*
* If no available slot can be found, -1 is returned.
* Note that for this function to work properly, input_mt_sync_frame() has
* to be called at each frame.
*/
int input_mt_get_slot_by_key(struct input_dev *dev, int key)
{
struct input_mt *mt = dev->mt;
struct input_mt_slot *s;
if (!mt)
return -1;
for (s = mt->slots; s != mt->slots + mt->num_slots; s++)
if (input_mt_is_active(s) && s->key == key)
return s - mt->slots;
for (s = mt->slots; s != mt->slots + mt->num_slots; s++)
if (!input_mt_is_active(s) && !input_mt_is_used(mt, s)) {
s->key = key;
return s - mt->slots;
}
return -1;
}
EXPORT_SYMBOL(input_mt_get_slot_by_key);
// SPDX-License-Identifier: GPL-2.0-only
/*
* kernel/workqueue.c - generic async execution with shared worker pool
*
* Copyright (C) 2002 Ingo Molnar
*
* Derived from the taskqueue/keventd code by:
* David Woodhouse <dwmw2@infradead.org>
* Andrew Morton
* Kai Petzke <wpp@marie.physik.tu-berlin.de>
* Theodore Ts'o <tytso@mit.edu>
*
* Made to use alloc_percpu by Christoph Lameter.
*
* Copyright (C) 2010 SUSE Linux Products GmbH
* Copyright (C) 2010 Tejun Heo <tj@kernel.org>
*
* This is the generic async execution mechanism. Work items as are
* executed in process context. The worker pool is shared and
* automatically managed. There are two worker pools for each CPU (one for
* normal work items and the other for high priority ones) and some extra
* pools for workqueues which are not bound to any specific CPU - the
* number of these backing pools is dynamic.
*
* Please read Documentation/core-api/workqueue.rst for details.
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/signal.h>
#include <linux/completion.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
#include <linux/hardirq.h>
#include <linux/mempolicy.h>
#include <linux/freezer.h>
#include <linux/debug_locks.h>
#include <linux/lockdep.h>
#include <linux/idr.h>
#include <linux/jhash.h>
#include <linux/hashtable.h>
#include <linux/rculist.h>
#include <linux/nodemask.h>
#include <linux/moduleparam.h>
#include <linux/uaccess.h>
#include <linux/sched/isolation.h>
#include <linux/sched/debug.h>
#include <linux/nmi.h>
#include <linux/kvm_para.h>
#include <linux/delay.h>
#include <linux/irq_work.h>
#include "workqueue_internal.h"
enum worker_pool_flags {
/*
* worker_pool flags
*
* A bound pool is either associated or disassociated with its CPU.
* While associated (!DISASSOCIATED), all workers are bound to the
* CPU and none has %WORKER_UNBOUND set and concurrency management
* is in effect.
*
* While DISASSOCIATED, the cpu may be offline and all workers have
* %WORKER_UNBOUND set and concurrency management disabled, and may
* be executing on any CPU. The pool behaves as an unbound one.
*
* Note that DISASSOCIATED should be flipped only while holding
* wq_pool_attach_mutex to avoid changing binding state while
* worker_attach_to_pool() is in progress.
*
* As there can only be one concurrent BH execution context per CPU, a
* BH pool is per-CPU and always DISASSOCIATED.
*/
POOL_BH = 1 << 0, /* is a BH pool */
POOL_MANAGER_ACTIVE = 1 << 1, /* being managed */
POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
POOL_BH_DRAINING = 1 << 3, /* draining after CPU offline */
};
enum worker_flags {
/* worker flags */
WORKER_DIE = 1 << 1, /* die die die */
WORKER_IDLE = 1 << 2, /* is idle */
WORKER_PREP = 1 << 3, /* preparing to run works */
WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
WORKER_UNBOUND = 1 << 7, /* worker is unbound */
WORKER_REBOUND = 1 << 8, /* worker was rebound */
WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
WORKER_UNBOUND | WORKER_REBOUND,
};
enum work_cancel_flags {
WORK_CANCEL_DELAYED = 1 << 0, /* canceling a delayed_work */
};
enum wq_internal_consts {
NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
/* call for help after 10ms
(min two ticks) */
MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
CREATE_COOLDOWN = HZ, /* time to breath after fail */
/*
* Rescue workers are used only on emergencies and shared by
* all cpus. Give MIN_NICE.
*/
RESCUER_NICE_LEVEL = MIN_NICE,
HIGHPRI_NICE_LEVEL = MIN_NICE,
WQ_NAME_LEN = 32,
};
/*
* We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and
* MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because
* msecs_to_jiffies() can't be an initializer.
*/
#define BH_WORKER_JIFFIES msecs_to_jiffies(2)
#define BH_WORKER_RESTARTS 10
/*
* Structure fields follow one of the following exclusion rules.
*
* I: Modifiable by initialization/destruction paths and read-only for
* everyone else.
*
* P: Preemption protected. Disabling preemption is enough and should
* only be modified and accessed from the local cpu.
*
* L: pool->lock protected. Access with pool->lock held.
*
* LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
* reads.
*
* K: Only modified by worker while holding pool->lock. Can be safely read by
* self, while holding pool->lock or from IRQ context if %current is the
* kworker.
*
* S: Only modified by worker self.
*
* A: wq_pool_attach_mutex protected.
*
* PL: wq_pool_mutex protected.
*
* PR: wq_pool_mutex protected for writes. RCU protected for reads.
*
* PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
*
* PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
* RCU for reads.
*
* WQ: wq->mutex protected.
*
* WR: wq->mutex protected for writes. RCU protected for reads.
*
* WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
* with READ_ONCE() without locking.
*
* MD: wq_mayday_lock protected.
*
* WD: Used internally by the watchdog.
*/
/* struct worker is defined in workqueue_internal.h */
struct worker_pool {
raw_spinlock_t lock; /* the pool lock */
int cpu; /* I: the associated cpu */
int node; /* I: the associated node ID */
int id; /* I: pool ID */
unsigned int flags; /* L: flags */
unsigned long watchdog_ts; /* L: watchdog timestamp */
bool cpu_stall; /* WD: stalled cpu bound pool */
/*
* The counter is incremented in a process context on the associated CPU
* w/ preemption disabled, and decremented or reset in the same context
* but w/ pool->lock held. The readers grab pool->lock and are
* guaranteed to see if the counter reached zero.
*/
int nr_running;
struct list_head worklist; /* L: list of pending works */
int nr_workers; /* L: total number of workers */
int nr_idle; /* L: currently idle workers */
struct list_head idle_list; /* L: list of idle workers */
struct timer_list idle_timer; /* L: worker idle timeout */
struct work_struct idle_cull_work; /* L: worker idle cleanup */
struct timer_list mayday_timer; /* L: SOS timer for workers */
/* a workers is either on busy_hash or idle_list, or the manager */
DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
/* L: hash of busy workers */
struct worker *manager; /* L: purely informational */
struct list_head workers; /* A: attached workers */
struct list_head dying_workers; /* A: workers about to die */
struct completion *detach_completion; /* all workers detached */
struct ida worker_ida; /* worker IDs for task name */
struct workqueue_attrs *attrs; /* I: worker attributes */
struct hlist_node hash_node; /* PL: unbound_pool_hash node */
int refcnt; /* PL: refcnt for unbound pools */
/*
* Destruction of pool is RCU protected to allow dereferences
* from get_work_pool().
*/
struct rcu_head rcu;
};
/*
* Per-pool_workqueue statistics. These can be monitored using
* tools/workqueue/wq_monitor.py.
*/
enum pool_workqueue_stats {
PWQ_STAT_STARTED, /* work items started execution */
PWQ_STAT_COMPLETED, /* work items completed execution */
PWQ_STAT_CPU_TIME, /* total CPU time consumed */
PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */
PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */
PWQ_STAT_REPATRIATED, /* unbound workers brought back into scope */
PWQ_STAT_MAYDAY, /* maydays to rescuer */
PWQ_STAT_RESCUED, /* linked work items executed by rescuer */
PWQ_NR_STATS,
};
/*
* The per-pool workqueue. While queued, bits below WORK_PWQ_SHIFT
* of work_struct->data are used for flags and the remaining high bits
* point to the pwq; thus, pwqs need to be aligned at two's power of the
* number of flag bits.
*/
struct pool_workqueue {
struct worker_pool *pool; /* I: the associated pool */
struct workqueue_struct *wq; /* I: the owning workqueue */
int work_color; /* L: current color */
int flush_color; /* L: flushing color */
int refcnt; /* L: reference count */
int nr_in_flight[WORK_NR_COLORS];
/* L: nr of in_flight works */
bool plugged; /* L: execution suspended */
/*
* nr_active management and WORK_STRUCT_INACTIVE:
*
* When pwq->nr_active >= max_active, new work item is queued to
* pwq->inactive_works instead of pool->worklist and marked with
* WORK_STRUCT_INACTIVE.
*
* All work items marked with WORK_STRUCT_INACTIVE do not participate in
* nr_active and all work items in pwq->inactive_works are marked with
* WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE work items are
* in pwq->inactive_works. Some of them are ready to run in
* pool->worklist or worker->scheduled. Those work itmes are only struct
* wq_barrier which is used for flush_work() and should not participate
* in nr_active. For non-barrier work item, it is marked with
* WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
*/
int nr_active; /* L: nr of active works */
struct list_head inactive_works; /* L: inactive works */
struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */
struct list_head pwqs_node; /* WR: node on wq->pwqs */
struct list_head mayday_node; /* MD: node on wq->maydays */
u64 stats[PWQ_NR_STATS];
/*
* Release of unbound pwq is punted to a kthread_worker. See put_pwq()
* and pwq_release_workfn() for details. pool_workqueue itself is also
* RCU protected so that the first pwq can be determined without
* grabbing wq->mutex.
*/
struct kthread_work release_work;
struct rcu_head rcu;
} __aligned(1 << WORK_STRUCT_PWQ_SHIFT);
/*
* Structure used to wait for workqueue flush.
*/
struct wq_flusher {
struct list_head list; /* WQ: list of flushers */
int flush_color; /* WQ: flush color waiting for */
struct completion done; /* flush completion */
};
struct wq_device;
/*
* Unlike in a per-cpu workqueue where max_active limits its concurrency level
* on each CPU, in an unbound workqueue, max_active applies to the whole system.
* As sharing a single nr_active across multiple sockets can be very expensive,
* the counting and enforcement is per NUMA node.
*
* The following struct is used to enforce per-node max_active. When a pwq wants
* to start executing a work item, it should increment ->nr using
* tryinc_node_nr_active(). If acquisition fails due to ->nr already being over
* ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish
* and decrement ->nr, node_activate_pending_pwq() activates the pending pwqs in
* round-robin order.
*/
struct wq_node_nr_active {
int max; /* per-node max_active */
atomic_t nr; /* per-node nr_active */
raw_spinlock_t lock; /* nests inside pool locks */
struct list_head pending_pwqs; /* LN: pwqs with inactive works */
};
/*
* The externally visible workqueue. It relays the issued work items to
* the appropriate worker_pool through its pool_workqueues.
*/
struct workqueue_struct {
struct list_head pwqs; /* WR: all pwqs of this wq */
struct list_head list; /* PR: list of all workqueues */
struct mutex mutex; /* protects this wq */
int work_color; /* WQ: current work color */
int flush_color; /* WQ: current flush color */
atomic_t nr_pwqs_to_flush; /* flush in progress */
struct wq_flusher *first_flusher; /* WQ: first flusher */
struct list_head flusher_queue; /* WQ: flush waiters */
struct list_head flusher_overflow; /* WQ: flush overflow list */
struct list_head maydays; /* MD: pwqs requesting rescue */
struct worker *rescuer; /* MD: rescue worker */
int nr_drainers; /* WQ: drain in progress */
/* See alloc_workqueue() function comment for info on min/max_active */
int max_active; /* WO: max active works */
int min_active; /* WO: min active works */
int saved_max_active; /* WQ: saved max_active */
int saved_min_active; /* WQ: saved min_active */
struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */
#ifdef CONFIG_SYSFS
struct wq_device *wq_dev; /* I: for sysfs interface */
#endif
#ifdef CONFIG_LOCKDEP
char *lock_name;
struct lock_class_key key;
struct lockdep_map lockdep_map;
#endif
char name[WQ_NAME_LEN]; /* I: workqueue name */
/*
* Destruction of workqueue_struct is RCU protected to allow walking
* the workqueues list without grabbing wq_pool_mutex.
* This is used to dump all workqueues from sysrq.
*/
struct rcu_head rcu;
/* hot fields used during command issue, aligned to cacheline */
unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
struct pool_workqueue __percpu __rcu **cpu_pwq; /* I: per-cpu pwqs */
struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
};
/*
* Each pod type describes how CPUs should be grouped for unbound workqueues.
* See the comment above workqueue_attrs->affn_scope.
*/
struct wq_pod_type {
int nr_pods; /* number of pods */
cpumask_var_t *pod_cpus; /* pod -> cpus */
int *pod_node; /* pod -> node */
int *cpu_pod; /* cpu -> pod */
};
static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
[WQ_AFFN_DFL] = "default",
[WQ_AFFN_CPU] = "cpu",
[WQ_AFFN_SMT] = "smt",
[WQ_AFFN_CACHE] = "cache",
[WQ_AFFN_NUMA] = "numa",
[WQ_AFFN_SYSTEM] = "system",
};
/*
* Per-cpu work items which run for longer than the following threshold are
* automatically considered CPU intensive and excluded from concurrency
* management to prevent them from noticeably delaying other per-cpu work items.
* ULONG_MAX indicates that the user hasn't overridden it with a boot parameter.
* The actual value is initialized in wq_cpu_intensive_thresh_init().
*/
static unsigned long wq_cpu_intensive_thresh_us = ULONG_MAX;
module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
static unsigned int wq_cpu_intensive_warning_thresh = 4;
module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh, uint, 0644);
#endif
/* see the comment above the definition of WQ_POWER_EFFICIENT */
static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
module_param_named(power_efficient, wq_power_efficient, bool, 0444);
static bool wq_online; /* can kworkers be created yet? */
static bool wq_topo_initialized __read_mostly = false;
static struct kmem_cache *pwq_cache;
static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES];
static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE;
/* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */
static struct workqueue_attrs *wq_update_pod_attrs_buf;
static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
/* wait for manager to go away */
static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
static LIST_HEAD(workqueues); /* PR: list of all workqueues */
static bool workqueue_freezing; /* PL: have wqs started freezing? */
/* PL&A: allowable cpus for unbound wqs and work items */
static cpumask_var_t wq_unbound_cpumask;
/* PL: user requested unbound cpumask via sysfs */
static cpumask_var_t wq_requested_unbound_cpumask;
/* PL: isolated cpumask to be excluded from unbound cpumask */
static cpumask_var_t wq_isolated_cpumask;
/* for further constrain wq_unbound_cpumask by cmdline parameter*/
static struct cpumask wq_cmdline_cpumask __initdata;
/* CPU where unbound work was last round robin scheduled from this CPU */
static DEFINE_PER_CPU(int, wq_rr_cpu_last);
/*
* Local execution of unbound work items is no longer guaranteed. The
* following always forces round-robin CPU selection on unbound work items
* to uncover usages which depend on it.
*/
#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
static bool wq_debug_force_rr_cpu = true;
#else
static bool wq_debug_force_rr_cpu = false;
#endif
module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
/* to raise softirq for the BH worker pools on other CPUs */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS],
bh_pool_irq_works);
/* the BH worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
bh_worker_pools);
/* the per-cpu worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
cpu_worker_pools);
static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
/* PL: hash of all unbound pools keyed by pool->attrs */
static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
/* I: attributes used when instantiating standard unbound pools on demand */
static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
/* I: attributes used when instantiating ordered pools on demand */
static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
/*
* Used to synchronize multiple cancel_sync attempts on the same work item. See
* work_grab_pending() and __cancel_work_sync().
*/
static DECLARE_WAIT_QUEUE_HEAD(wq_cancel_waitq);
/*
* I: kthread_worker to release pwq's. pwq release needs to be bounced to a
* process context while holding a pool lock. Bounce to a dedicated kthread
* worker to avoid A-A deadlocks.
*/
static struct kthread_worker *pwq_release_worker __ro_after_init;
struct workqueue_struct *system_wq __ro_after_init;
EXPORT_SYMBOL(system_wq);
struct workqueue_struct *system_highpri_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_highpri_wq);
struct workqueue_struct *system_long_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_long_wq);
struct workqueue_struct *system_unbound_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_unbound_wq);
struct workqueue_struct *system_freezable_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_freezable_wq);
struct workqueue_struct *system_power_efficient_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_power_efficient_wq);
struct workqueue_struct *system_freezable_power_efficient_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
struct workqueue_struct *system_bh_wq;
EXPORT_SYMBOL_GPL(system_bh_wq);
struct workqueue_struct *system_bh_highpri_wq;
EXPORT_SYMBOL_GPL(system_bh_highpri_wq);
static int worker_thread(void *__worker);
static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
static void show_pwq(struct pool_workqueue *pwq);
static void show_one_worker_pool(struct worker_pool *pool);
#define CREATE_TRACE_POINTS
#include <trace/events/workqueue.h>
#define assert_rcu_or_pool_mutex() \
RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() && \
!lockdep_is_held(&wq_pool_mutex), \
"RCU or wq_pool_mutex should be held")
#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() && \
!lockdep_is_held(&wq->mutex) && \
!lockdep_is_held(&wq_pool_mutex), \
"RCU, wq->mutex or wq_pool_mutex should be held")
#define for_each_bh_worker_pool(pool, cpu) \
for ((pool) = &per_cpu(bh_worker_pools, cpu)[0]; \
(pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
(pool)++)
#define for_each_cpu_worker_pool(pool, cpu) \
for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
(pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
(pool)++)
/**
* for_each_pool - iterate through all worker_pools in the system
* @pool: iteration cursor
* @pi: integer used for iteration
*
* This must be called either with wq_pool_mutex held or RCU read
* locked. If the pool needs to be used beyond the locking in effect, the
* caller is responsible for guaranteeing that the pool stays online.
*
* The if/else clause exists only for the lockdep assertion and can be
* ignored.
*/
#define for_each_pool(pool, pi) \
idr_for_each_entry(&worker_pool_idr, pool, pi) \
if (({ assert_rcu_or_pool_mutex(); false; })) { } \
else
/**
* for_each_pool_worker - iterate through all workers of a worker_pool
* @worker: iteration cursor
* @pool: worker_pool to iterate workers of
*
* This must be called with wq_pool_attach_mutex.
*
* The if/else clause exists only for the lockdep assertion and can be
* ignored.
*/
#define for_each_pool_worker(worker, pool) \
list_for_each_entry((worker), &(pool)->workers, node) \
if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
else
/**
* for_each_pwq - iterate through all pool_workqueues of the specified workqueue
* @pwq: iteration cursor
* @wq: the target workqueue
*
* This must be called either with wq->mutex held or RCU read locked.
* If the pwq needs to be used beyond the locking in effect, the caller is
* responsible for guaranteeing that the pwq stays online.
*
* The if/else clause exists only for the lockdep assertion and can be
* ignored.
*/
#define for_each_pwq(pwq, wq) \
list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
lockdep_is_held(&(wq->mutex)))
#ifdef CONFIG_DEBUG_OBJECTS_WORK
static const struct debug_obj_descr work_debug_descr;
static void *work_debug_hint(void *addr)
{
return ((struct work_struct *) addr)->func;
}
static bool work_is_static_object(void *addr)
{
struct work_struct *work = addr;
return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
}
/*
* fixup_init is called when:
* - an active object is initialized
*/
static bool work_fixup_init(void *addr, enum debug_obj_state state)
{
struct work_struct *work = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
cancel_work_sync(work);
debug_object_init(work, &work_debug_descr);
return true;
default:
return false;
}
}
/*
* fixup_free is called when:
* - an active object is freed
*/
static bool work_fixup_free(void *addr, enum debug_obj_state state)
{
struct work_struct *work = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
cancel_work_sync(work);
debug_object_free(work, &work_debug_descr);
return true;
default:
return false;
}
}
static const struct debug_obj_descr work_debug_descr = {
.name = "work_struct",
.debug_hint = work_debug_hint,
.is_static_object = work_is_static_object,
.fixup_init = work_fixup_init,
.fixup_free = work_fixup_free,
};
static inline void debug_work_activate(struct work_struct *work)
{
debug_object_activate(work, &work_debug_descr);
}
static inline void debug_work_deactivate(struct work_struct *work)
{
debug_object_deactivate(work, &work_debug_descr);
}
void __init_work(struct work_struct *work, int onstack)
{
if (onstack) debug_object_init_on_stack(work, &work_debug_descr);
else
debug_object_init(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(__init_work);
void destroy_work_on_stack(struct work_struct *work)
{
debug_object_free(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_work_on_stack);
void destroy_delayed_work_on_stack(struct delayed_work *work)
{
destroy_timer_on_stack(&work->timer);
debug_object_free(&work->work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
#else
static inline void debug_work_activate(struct work_struct *work) { }
static inline void debug_work_deactivate(struct work_struct *work) { }
#endif
/**
* worker_pool_assign_id - allocate ID and assign it to @pool
* @pool: the pool pointer of interest
*
* Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
* successfully, -errno on failure.
*/
static int worker_pool_assign_id(struct worker_pool *pool)
{
int ret;
lockdep_assert_held(&wq_pool_mutex);
ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
GFP_KERNEL);
if (ret >= 0) {
pool->id = ret;
return 0;
}
return ret;
}
static struct pool_workqueue __rcu **
unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
{
if (cpu >= 0) return per_cpu_ptr(wq->cpu_pwq, cpu);
else
return &wq->dfl_pwq;
}
/* @cpu < 0 for dfl_pwq */
static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
{
return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
lockdep_is_held(&wq_pool_mutex) ||
lockdep_is_held(&wq->mutex));
}
/**
* unbound_effective_cpumask - effective cpumask of an unbound workqueue
* @wq: workqueue of interest
*
* @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
* is masked with wq_unbound_cpumask to determine the effective cpumask. The
* default pwq is always mapped to the pool with the current effective cpumask.
*/
static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
{
return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
}
static unsigned int work_color_to_flags(int color)
{
return color << WORK_STRUCT_COLOR_SHIFT;
}
static int get_work_color(unsigned long work_data)
{
return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
((1 << WORK_STRUCT_COLOR_BITS) - 1);
}
static int work_next_color(int color)
{
return (color + 1) % WORK_NR_COLORS;
}
/*
* While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
* contain the pointer to the queued pwq. Once execution starts, the flag
* is cleared and the high bits contain OFFQ flags and pool ID.
*
* set_work_pwq(), set_work_pool_and_clear_pending() and mark_work_canceling()
* can be used to set the pwq, pool or clear work->data. These functions should
* only be called while the work is owned - ie. while the PENDING bit is set.
*
* get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
* corresponding to a work. Pool is available once the work has been
* queued anywhere after initialization until it is sync canceled. pwq is
* available only while the work item is queued.
*
* %WORK_OFFQ_CANCELING is used to mark a work item which is being
* canceled. While being canceled, a work item may have its PENDING set
* but stay off timer and worklist for arbitrarily long and nobody should
* try to steal the PENDING bit.
*/
static inline void set_work_data(struct work_struct *work, unsigned long data)
{
WARN_ON_ONCE(!work_pending(work)); atomic_long_set(&work->data, data | work_static(work));
}
static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
unsigned long flags)
{
set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING |
WORK_STRUCT_PWQ | flags);
}
static void set_work_pool_and_keep_pending(struct work_struct *work,
int pool_id, unsigned long flags)
{
set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
WORK_STRUCT_PENDING | flags);
}
static void set_work_pool_and_clear_pending(struct work_struct *work,
int pool_id, unsigned long flags)
{
/*
* The following wmb is paired with the implied mb in
* test_and_set_bit(PENDING) and ensures all updates to @work made
* here are visible to and precede any updates by the next PENDING
* owner.
*/
smp_wmb(); set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
flags);
/*
* The following mb guarantees that previous clear of a PENDING bit
* will not be reordered with any speculative LOADS or STORES from
* work->current_func, which is executed afterwards. This possible
* reordering can lead to a missed execution on attempt to queue
* the same @work. E.g. consider this case:
*
* CPU#0 CPU#1
* ---------------------------- --------------------------------
*
* 1 STORE event_indicated
* 2 queue_work_on() {
* 3 test_and_set_bit(PENDING)
* 4 } set_..._and_clear_pending() {
* 5 set_work_data() # clear bit
* 6 smp_mb()
* 7 work->current_func() {
* 8 LOAD event_indicated
* }
*
* Without an explicit full barrier speculative LOAD on line 8 can
* be executed before CPU#0 does STORE on line 1. If that happens,
* CPU#0 observes the PENDING bit is still set and new execution of
* a @work is not queued in a hope, that CPU#1 will eventually
* finish the queued @work. Meanwhile CPU#1 does not see
* event_indicated is set, because speculative LOAD was executed
* before actual STORE.
*/
smp_mb();
}
static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
{
return (struct pool_workqueue *)(data & WORK_STRUCT_PWQ_MASK);
}
static struct pool_workqueue *get_work_pwq(struct work_struct *work)
{
unsigned long data = atomic_long_read(&work->data);
if (data & WORK_STRUCT_PWQ)
return work_struct_pwq(data);
else
return NULL;
}
/**
* get_work_pool - return the worker_pool a given work was associated with
* @work: the work item of interest
*
* Pools are created and destroyed under wq_pool_mutex, and allows read
* access under RCU read lock. As such, this function should be
* called under wq_pool_mutex or inside of a rcu_read_lock() region.
*
* All fields of the returned pool are accessible as long as the above
* mentioned locking is in effect. If the returned pool needs to be used
* beyond the critical section, the caller is responsible for ensuring the
* returned pool is and stays online.
*
* Return: The worker_pool @work was last associated with. %NULL if none.
*/
static struct worker_pool *get_work_pool(struct work_struct *work)
{
unsigned long data = atomic_long_read(&work->data);
int pool_id;
assert_rcu_or_pool_mutex(); if (data & WORK_STRUCT_PWQ) return work_struct_pwq(data)->pool;
pool_id = data >> WORK_OFFQ_POOL_SHIFT;
if (pool_id == WORK_OFFQ_POOL_NONE)
return NULL;
return idr_find(&worker_pool_idr, pool_id);
}
/**
* get_work_pool_id - return the worker pool ID a given work is associated with
* @work: the work item of interest
*
* Return: The worker_pool ID @work was last associated with.
* %WORK_OFFQ_POOL_NONE if none.
*/
static int get_work_pool_id(struct work_struct *work)
{
unsigned long data = atomic_long_read(&work->data);
if (data & WORK_STRUCT_PWQ)
return work_struct_pwq(data)->pool->id;
return data >> WORK_OFFQ_POOL_SHIFT;
}
static void mark_work_canceling(struct work_struct *work)
{
unsigned long pool_id = get_work_pool_id(work);
pool_id <<= WORK_OFFQ_POOL_SHIFT;
set_work_data(work, pool_id | WORK_STRUCT_PENDING | WORK_OFFQ_CANCELING);
}
static bool work_is_canceling(struct work_struct *work)
{
unsigned long data = atomic_long_read(&work->data);
return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
}
/*
* Policy functions. These define the policies on how the global worker
* pools are managed. Unless noted otherwise, these functions assume that
* they're being called with pool->lock held.
*/
/*
* Need to wake up a worker? Called from anything but currently
* running workers.
*
* Note that, because unbound workers never contribute to nr_running, this
* function will always return %true for unbound pools as long as the
* worklist isn't empty.
*/
static bool need_more_worker(struct worker_pool *pool)
{
return !list_empty(&pool->worklist) && !pool->nr_running;
}
/* Can I start working? Called from busy but !running workers. */
static bool may_start_working(struct worker_pool *pool)
{
return pool->nr_idle;
}
/* Do I need to keep working? Called from currently running workers. */
static bool keep_working(struct worker_pool *pool)
{
return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
}
/* Do we need a new worker? Called from manager. */
static bool need_to_create_worker(struct worker_pool *pool)
{
return need_more_worker(pool) && !may_start_working(pool);
}
/* Do we have too many workers and should some go away? */
static bool too_many_workers(struct worker_pool *pool)
{
bool managing = pool->flags & POOL_MANAGER_ACTIVE;
int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
int nr_busy = pool->nr_workers - nr_idle;
return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
}
/**
* worker_set_flags - set worker flags and adjust nr_running accordingly
* @worker: self
* @flags: flags to set
*
* Set @flags in @worker->flags and adjust nr_running accordingly.
*/
static inline void worker_set_flags(struct worker *worker, unsigned int flags)
{
struct worker_pool *pool = worker->pool;
lockdep_assert_held(&pool->lock);
/* If transitioning into NOT_RUNNING, adjust nr_running. */
if ((flags & WORKER_NOT_RUNNING) &&
!(worker->flags & WORKER_NOT_RUNNING)) { pool->nr_running--;
}
worker->flags |= flags;
}
/**
* worker_clr_flags - clear worker flags and adjust nr_running accordingly
* @worker: self
* @flags: flags to clear
*
* Clear @flags in @worker->flags and adjust nr_running accordingly.
*/
static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
{
struct worker_pool *pool = worker->pool;
unsigned int oflags = worker->flags;
lockdep_assert_held(&pool->lock);
worker->flags &= ~flags;
/*
* If transitioning out of NOT_RUNNING, increment nr_running. Note
* that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
* of multiple flags, not a single flag.
*/
if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
if (!(worker->flags & WORKER_NOT_RUNNING))
pool->nr_running++;
}
/* Return the first idle worker. Called with pool->lock held. */
static struct worker *first_idle_worker(struct worker_pool *pool)
{
if (unlikely(list_empty(&pool->idle_list)))
return NULL;
return list_first_entry(&pool->idle_list, struct worker, entry);
}
/**
* worker_enter_idle - enter idle state
* @worker: worker which is entering idle state
*
* @worker is entering idle state. Update stats and idle timer if
* necessary.
*
* LOCKING:
* raw_spin_lock_irq(pool->lock).
*/
static void worker_enter_idle(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
WARN_ON_ONCE(!list_empty(&worker->entry) &&
(worker->hentry.next || worker->hentry.pprev)))
return;
/* can't use worker_set_flags(), also called from create_worker() */
worker->flags |= WORKER_IDLE;
pool->nr_idle++;
worker->last_active = jiffies;
/* idle_list is LIFO */
list_add(&worker->entry, &pool->idle_list);
if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
/* Sanity check nr_running. */
WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
}
/**
* worker_leave_idle - leave idle state
* @worker: worker which is leaving idle state
*
* @worker is leaving idle state. Update stats.
*
* LOCKING:
* raw_spin_lock_irq(pool->lock).
*/
static void worker_leave_idle(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
return;
worker_clr_flags(worker, WORKER_IDLE);
pool->nr_idle--;
list_del_init(&worker->entry);
}
/**
* find_worker_executing_work - find worker which is executing a work
* @pool: pool of interest
* @work: work to find worker for
*
* Find a worker which is executing @work on @pool by searching
* @pool->busy_hash which is keyed by the address of @work. For a worker
* to match, its current execution should match the address of @work and
* its work function. This is to avoid unwanted dependency between
* unrelated work executions through a work item being recycled while still
* being executed.
*
* This is a bit tricky. A work item may be freed once its execution
* starts and nothing prevents the freed area from being recycled for
* another work item. If the same work item address ends up being reused
* before the original execution finishes, workqueue will identify the
* recycled work item as currently executing and make it wait until the
* current execution finishes, introducing an unwanted dependency.
*
* This function checks the work item address and work function to avoid
* false positives. Note that this isn't complete as one may construct a
* work function which can introduce dependency onto itself through a
* recycled work item. Well, if somebody wants to shoot oneself in the
* foot that badly, there's only so much we can do, and if such deadlock
* actually occurs, it should be easy to locate the culprit work function.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock).
*
* Return:
* Pointer to worker which is executing @work if found, %NULL
* otherwise.
*/
static struct worker *find_worker_executing_work(struct worker_pool *pool,
struct work_struct *work)
{
struct worker *worker;
hash_for_each_possible(pool->busy_hash, worker, hentry,
(unsigned long)work)
if (worker->current_work == work && worker->current_func == work->func)
return worker;
return NULL;
}
/**
* move_linked_works - move linked works to a list
* @work: start of series of works to be scheduled
* @head: target list to append @work to
* @nextp: out parameter for nested worklist walking
*
* Schedule linked works starting from @work to @head. Work series to be
* scheduled starts at @work and includes any consecutive work with
* WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on
* @nextp.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock).
*/
static void move_linked_works(struct work_struct *work, struct list_head *head,
struct work_struct **nextp)
{
struct work_struct *n;
/*
* Linked worklist will always end before the end of the list,
* use NULL for list head.
*/
list_for_each_entry_safe_from(work, n, NULL, entry) {
list_move_tail(&work->entry, head);
if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
break;
}
/*
* If we're already inside safe list traversal and have moved
* multiple works to the scheduled queue, the next position
* needs to be updated.
*/
if (nextp)
*nextp = n;
}
/**
* assign_work - assign a work item and its linked work items to a worker
* @work: work to assign
* @worker: worker to assign to
* @nextp: out parameter for nested worklist walking
*
* Assign @work and its linked work items to @worker. If @work is already being
* executed by another worker in the same pool, it'll be punted there.
*
* If @nextp is not NULL, it's updated to point to the next work of the last
* scheduled work. This allows assign_work() to be nested inside
* list_for_each_entry_safe().
*
* Returns %true if @work was successfully assigned to @worker. %false if @work
* was punted to another worker already executing it.
*/
static bool assign_work(struct work_struct *work, struct worker *worker,
struct work_struct **nextp)
{
struct worker_pool *pool = worker->pool;
struct worker *collision;
lockdep_assert_held(&pool->lock);
/*
* A single work shouldn't be executed concurrently by multiple workers.
* __queue_work() ensures that @work doesn't jump to a different pool
* while still running in the previous pool. Here, we should ensure that
* @work is not executed concurrently by multiple workers from the same
* pool. Check whether anyone is already processing the work. If so,
* defer the work to the currently executing one.
*/
collision = find_worker_executing_work(pool, work);
if (unlikely(collision)) {
move_linked_works(work, &collision->scheduled, nextp);
return false;
}
move_linked_works(work, &worker->scheduled, nextp);
return true;
}
static struct irq_work *bh_pool_irq_work(struct worker_pool *pool)
{
int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;
return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
}
static void kick_bh_pool(struct worker_pool *pool)
{
#ifdef CONFIG_SMP
/* see drain_dead_softirq_workfn() for BH_DRAINING */
if (unlikely(pool->cpu != smp_processor_id() &&
!(pool->flags & POOL_BH_DRAINING))) {
irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
return;
}
#endif
if (pool->attrs->nice == HIGHPRI_NICE_LEVEL) raise_softirq_irqoff(HI_SOFTIRQ);
else
raise_softirq_irqoff(TASKLET_SOFTIRQ);
}
/**
* kick_pool - wake up an idle worker if necessary
* @pool: pool to kick
*
* @pool may have pending work items. Wake up worker if necessary. Returns
* whether a worker was woken up.
*/
static bool kick_pool(struct worker_pool *pool)
{
struct worker *worker = first_idle_worker(pool);
struct task_struct *p;
lockdep_assert_held(&pool->lock); if (!need_more_worker(pool) || !worker) return false; if (pool->flags & POOL_BH) { kick_bh_pool(pool);
return true;
}
p = worker->task;
#ifdef CONFIG_SMP
/*
* Idle @worker is about to execute @work and waking up provides an
* opportunity to migrate @worker at a lower cost by setting the task's
* wake_cpu field. Let's see if we want to move @worker to improve
* execution locality.
*
* We're waking the worker that went idle the latest and there's some
* chance that @worker is marked idle but hasn't gone off CPU yet. If
* so, setting the wake_cpu won't do anything. As this is a best-effort
* optimization and the race window is narrow, let's leave as-is for
* now. If this becomes pronounced, we can skip over workers which are
* still on cpu when picking an idle worker.
*
* If @pool has non-strict affinity, @worker might have ended up outside
* its affinity scope. Repatriate.
*/
if (!pool->attrs->affn_strict &&
!cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) { struct work_struct *work = list_first_entry(&pool->worklist,
struct work_struct, entry);
int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
cpu_online_mask);
if (wake_cpu < nr_cpu_ids) {
p->wake_cpu = wake_cpu;
get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;
}
}
#endif
wake_up_process(p);
return true;
}
#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
/*
* Concurrency-managed per-cpu work items that hog CPU for longer than
* wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
* which prevents them from stalling other concurrency-managed work items. If a
* work function keeps triggering this mechanism, it's likely that the work item
* should be using an unbound workqueue instead.
*
* wq_cpu_intensive_report() tracks work functions which trigger such conditions
* and report them so that they can be examined and converted to use unbound
* workqueues as appropriate. To avoid flooding the console, each violating work
* function is tracked and reported with exponential backoff.
*/
#define WCI_MAX_ENTS 128
struct wci_ent {
work_func_t func;
atomic64_t cnt;
struct hlist_node hash_node;
};
static struct wci_ent wci_ents[WCI_MAX_ENTS];
static int wci_nr_ents;
static DEFINE_RAW_SPINLOCK(wci_lock);
static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
static struct wci_ent *wci_find_ent(work_func_t func)
{
struct wci_ent *ent;
hash_for_each_possible_rcu(wci_hash, ent, hash_node,
(unsigned long)func) {
if (ent->func == func)
return ent;
}
return NULL;
}
static void wq_cpu_intensive_report(work_func_t func)
{
struct wci_ent *ent;
restart:
ent = wci_find_ent(func);
if (ent) {
u64 cnt;
/*
* Start reporting from the warning_thresh and back off
* exponentially.
*/
cnt = atomic64_inc_return_relaxed(&ent->cnt);
if (wq_cpu_intensive_warning_thresh &&
cnt >= wq_cpu_intensive_warning_thresh &&
is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh))
printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
ent->func, wq_cpu_intensive_thresh_us,
atomic64_read(&ent->cnt));
return;
}
/*
* @func is a new violation. Allocate a new entry for it. If wcn_ents[]
* is exhausted, something went really wrong and we probably made enough
* noise already.
*/
if (wci_nr_ents >= WCI_MAX_ENTS)
return;
raw_spin_lock(&wci_lock);
if (wci_nr_ents >= WCI_MAX_ENTS) {
raw_spin_unlock(&wci_lock);
return;
}
if (wci_find_ent(func)) {
raw_spin_unlock(&wci_lock);
goto restart;
}
ent = &wci_ents[wci_nr_ents++];
ent->func = func;
atomic64_set(&ent->cnt, 0);
hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
raw_spin_unlock(&wci_lock);
goto restart;
}
#else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
static void wq_cpu_intensive_report(work_func_t func) {}
#endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
/**
* wq_worker_running - a worker is running again
* @task: task waking up
*
* This function is called when a worker returns from schedule()
*/
void wq_worker_running(struct task_struct *task)
{
struct worker *worker = kthread_data(task);
if (!READ_ONCE(worker->sleeping))
return;
/*
* If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
* and the nr_running increment below, we may ruin the nr_running reset
* and leave with an unexpected pool->nr_running == 1 on the newly unbound
* pool. Protect against such race.
*/
preempt_disable();
if (!(worker->flags & WORKER_NOT_RUNNING))
worker->pool->nr_running++; preempt_enable();
/*
* CPU intensive auto-detection cares about how long a work item hogged
* CPU without sleeping. Reset the starting timestamp on wakeup.
*/
worker->current_at = worker->task->se.sum_exec_runtime;
WRITE_ONCE(worker->sleeping, 0);
}
/**
* wq_worker_sleeping - a worker is going to sleep
* @task: task going to sleep
*
* This function is called from schedule() when a busy worker is
* going to sleep.
*/
void wq_worker_sleeping(struct task_struct *task)
{
struct worker *worker = kthread_data(task);
struct worker_pool *pool;
/*
* Rescuers, which may not have all the fields set up like normal
* workers, also reach here, let's not access anything before
* checking NOT_RUNNING.
*/
if (worker->flags & WORKER_NOT_RUNNING)
return;
pool = worker->pool;
/* Return if preempted before wq_worker_running() was reached */
if (READ_ONCE(worker->sleeping))
return;
WRITE_ONCE(worker->sleeping, 1);
raw_spin_lock_irq(&pool->lock);
/*
* Recheck in case unbind_workers() preempted us. We don't
* want to decrement nr_running after the worker is unbound
* and nr_running has been reset.
*/
if (worker->flags & WORKER_NOT_RUNNING) {
raw_spin_unlock_irq(&pool->lock);
return;
}
pool->nr_running--;
if (kick_pool(pool))
worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++; raw_spin_unlock_irq(&pool->lock);
}
/**
* wq_worker_tick - a scheduler tick occurred while a kworker is running
* @task: task currently running
*
* Called from scheduler_tick(). We're in the IRQ context and the current
* worker's fields which follow the 'K' locking rule can be accessed safely.
*/
void wq_worker_tick(struct task_struct *task)
{
struct worker *worker = kthread_data(task);
struct pool_workqueue *pwq = worker->current_pwq;
struct worker_pool *pool = worker->pool;
if (!pwq)
return;
pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
if (!wq_cpu_intensive_thresh_us)
return;
/*
* If the current worker is concurrency managed and hogged the CPU for
* longer than wq_cpu_intensive_thresh_us, it's automatically marked
* CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
*
* Set @worker->sleeping means that @worker is in the process of
* switching out voluntarily and won't be contributing to
* @pool->nr_running until it wakes up. As wq_worker_sleeping() also
* decrements ->nr_running, setting CPU_INTENSIVE here can lead to
* double decrements. The task is releasing the CPU anyway. Let's skip.
* We probably want to make this prettier in the future.
*/
if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) || worker->task->se.sum_exec_runtime - worker->current_at <
wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
return;
raw_spin_lock(&pool->lock); worker_set_flags(worker, WORKER_CPU_INTENSIVE);
wq_cpu_intensive_report(worker->current_func);
pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
if (kick_pool(pool))
pwq->stats[PWQ_STAT_CM_WAKEUP]++; raw_spin_unlock(&pool->lock);
}
/**
* wq_worker_last_func - retrieve worker's last work function
* @task: Task to retrieve last work function of.
*
* Determine the last function a worker executed. This is called from
* the scheduler to get a worker's last known identity.
*
* CONTEXT:
* raw_spin_lock_irq(rq->lock)
*
* This function is called during schedule() when a kworker is going
* to sleep. It's used by psi to identify aggregation workers during
* dequeuing, to allow periodic aggregation to shut-off when that
* worker is the last task in the system or cgroup to go to sleep.
*
* As this function doesn't involve any workqueue-related locking, it
* only returns stable values when called from inside the scheduler's
* queuing and dequeuing paths, when @task, which must be a kworker,
* is guaranteed to not be processing any works.
*
* Return:
* The last work function %current executed as a worker, NULL if it
* hasn't executed any work yet.
*/
work_func_t wq_worker_last_func(struct task_struct *task)
{
struct worker *worker = kthread_data(task);
return worker->last_func;
}
/**
* wq_node_nr_active - Determine wq_node_nr_active to use
* @wq: workqueue of interest
* @node: NUMA node, can be %NUMA_NO_NODE
*
* Determine wq_node_nr_active to use for @wq on @node. Returns:
*
* - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
*
* - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
*
* - Otherwise, node_nr_active[@node].
*/
static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
int node)
{
if (!(wq->flags & WQ_UNBOUND))
return NULL;
if (node == NUMA_NO_NODE) node = nr_node_ids; return wq->node_nr_active[node];
}
/**
* wq_update_node_max_active - Update per-node max_actives to use
* @wq: workqueue to update
* @off_cpu: CPU that's going down, -1 if a CPU is not going down
*
* Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is
* distributed among nodes according to the proportions of numbers of online
* cpus. The result is always between @wq->min_active and max_active.
*/
static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu)
{
struct cpumask *effective = unbound_effective_cpumask(wq);
int min_active = READ_ONCE(wq->min_active);
int max_active = READ_ONCE(wq->max_active);
int total_cpus, node;
lockdep_assert_held(&wq->mutex);
if (!wq_topo_initialized)
return;
if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))
off_cpu = -1;
total_cpus = cpumask_weight_and(effective, cpu_online_mask);
if (off_cpu >= 0)
total_cpus--;
/* If all CPUs of the wq get offline, use the default values */
if (unlikely(!total_cpus)) {
for_each_node(node)
wq_node_nr_active(wq, node)->max = min_active;
wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
return;
}
for_each_node(node) {
int node_cpus;
node_cpus = cpumask_weight_and(effective, cpumask_of_node(node));
if (off_cpu >= 0 && cpu_to_node(off_cpu) == node)
node_cpus--;
wq_node_nr_active(wq, node)->max =
clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus),
min_active, max_active);
}
wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
}
/**
* get_pwq - get an extra reference on the specified pool_workqueue
* @pwq: pool_workqueue to get
*
* Obtain an extra reference on @pwq. The caller should guarantee that
* @pwq has positive refcnt and be holding the matching pool->lock.
*/
static void get_pwq(struct pool_workqueue *pwq)
{
lockdep_assert_held(&pwq->pool->lock); WARN_ON_ONCE(pwq->refcnt <= 0); pwq->refcnt++;
}
/**
* put_pwq - put a pool_workqueue reference
* @pwq: pool_workqueue to put
*
* Drop a reference of @pwq. If its refcnt reaches zero, schedule its
* destruction. The caller should be holding the matching pool->lock.
*/
static void put_pwq(struct pool_workqueue *pwq)
{
lockdep_assert_held(&pwq->pool->lock); if (likely(--pwq->refcnt))
return;
/*
* @pwq can't be released under pool->lock, bounce to a dedicated
* kthread_worker to avoid A-A deadlocks.
*/
kthread_queue_work(pwq_release_worker, &pwq->release_work);
}
/**
* put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
* @pwq: pool_workqueue to put (can be %NULL)
*
* put_pwq() with locking. This function also allows %NULL @pwq.
*/
static void put_pwq_unlocked(struct pool_workqueue *pwq)
{
if (pwq) {
/*
* As both pwqs and pools are RCU protected, the
* following lock operations are safe.
*/
raw_spin_lock_irq(&pwq->pool->lock);
put_pwq(pwq);
raw_spin_unlock_irq(&pwq->pool->lock);
}
}
static bool pwq_is_empty(struct pool_workqueue *pwq)
{
return !pwq->nr_active && list_empty(&pwq->inactive_works);
}
static void __pwq_activate_work(struct pool_workqueue *pwq,
struct work_struct *work)
{
unsigned long *wdb = work_data_bits(work);
WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));
trace_workqueue_activate_work(work);
if (list_empty(&pwq->pool->worklist))
pwq->pool->watchdog_ts = jiffies;
move_linked_works(work, &pwq->pool->worklist, NULL);
__clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);
}
/**
* pwq_activate_work - Activate a work item if inactive
* @pwq: pool_workqueue @work belongs to
* @work: work item to activate
*
* Returns %true if activated. %false if already active.
*/
static bool pwq_activate_work(struct pool_workqueue *pwq,
struct work_struct *work)
{
struct worker_pool *pool = pwq->pool;
struct wq_node_nr_active *nna;
lockdep_assert_held(&pool->lock); if (!(*work_data_bits(work) & WORK_STRUCT_INACTIVE))
return false;
nna = wq_node_nr_active(pwq->wq, pool->node);
if (nna)
atomic_inc(&nna->nr); pwq->nr_active++;
__pwq_activate_work(pwq, work);
return true;
}
static bool tryinc_node_nr_active(struct wq_node_nr_active *nna)
{
int max = READ_ONCE(nna->max);
while (true) {
int old, tmp;
old = atomic_read(&nna->nr);
if (old >= max)
return false; tmp = atomic_cmpxchg_relaxed(&nna->nr, old, old + 1);
if (tmp == old)
return true;
}
}
/**
* pwq_tryinc_nr_active - Try to increment nr_active for a pwq
* @pwq: pool_workqueue of interest
* @fill: max_active may have increased, try to increase concurrency level
*
* Try to increment nr_active for @pwq. Returns %true if an nr_active count is
* successfully obtained. %false otherwise.
*/
static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
{
struct workqueue_struct *wq = pwq->wq;
struct worker_pool *pool = pwq->pool;
struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
bool obtained = false;
lockdep_assert_held(&pool->lock); if (!nna) {
/* BH or per-cpu workqueue, pwq->nr_active is sufficient */
obtained = pwq->nr_active < READ_ONCE(wq->max_active);
goto out;
}
if (unlikely(pwq->plugged)) return false;
/*
* Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
* already waiting on $nna, pwq_dec_nr_active() will maintain the
* concurrency level. Don't jump the line.
*
* We need to ignore the pending test after max_active has increased as
* pwq_dec_nr_active() can only maintain the concurrency level but not
* increase it. This is indicated by @fill.
*/
if (!list_empty(&pwq->pending_node) && likely(!fill))
goto out;
obtained = tryinc_node_nr_active(nna);
if (obtained)
goto out;
/*
* Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs
* and try again. The smp_mb() is paired with the implied memory barrier
* of atomic_dec_return() in pwq_dec_nr_active() to ensure that either
* we see the decremented $nna->nr or they see non-empty
* $nna->pending_pwqs.
*/
raw_spin_lock(&nna->lock);
if (list_empty(&pwq->pending_node))
list_add_tail(&pwq->pending_node, &nna->pending_pwqs); else if (likely(!fill))
goto out_unlock;
smp_mb();
obtained = tryinc_node_nr_active(nna);
/*
* If @fill, @pwq might have already been pending. Being spuriously
* pending in cold paths doesn't affect anything. Let's leave it be.
*/
if (obtained && likely(!fill)) list_del_init(&pwq->pending_node);
out_unlock:
raw_spin_unlock(&nna->lock);
out:
if (obtained)
pwq->nr_active++;
return obtained;
}
/**
* pwq_activate_first_inactive - Activate the first inactive work item on a pwq
* @pwq: pool_workqueue of interest
* @fill: max_active may have increased, try to increase concurrency level
*
* Activate the first inactive work item of @pwq if available and allowed by
* max_active limit.
*
* Returns %true if an inactive work item has been activated. %false if no
* inactive work item is found or max_active limit is reached.
*/
static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)
{
struct work_struct *work =
list_first_entry_or_null(&pwq->inactive_works,
struct work_struct, entry);
if (work && pwq_tryinc_nr_active(pwq, fill)) {
__pwq_activate_work(pwq, work);
return true;
} else {
return false;
}
}
/**
* unplug_oldest_pwq - unplug the oldest pool_workqueue
* @wq: workqueue_struct where its oldest pwq is to be unplugged
*
* This function should only be called for ordered workqueues where only the
* oldest pwq is unplugged, the others are plugged to suspend execution to
* ensure proper work item ordering::
*
* dfl_pwq --------------+ [P] - plugged
* |
* v
* pwqs -> A -> B [P] -> C [P] (newest)
* | | |
* 1 3 5
* | | |
* 2 4 6
*
* When the oldest pwq is drained and removed, this function should be called
* to unplug the next oldest one to start its work item execution. Note that
* pwq's are linked into wq->pwqs with the oldest first, so the first one in
* the list is the oldest.
*/
static void unplug_oldest_pwq(struct workqueue_struct *wq)
{
struct pool_workqueue *pwq;
lockdep_assert_held(&wq->mutex);
/* Caller should make sure that pwqs isn't empty before calling */
pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
pwqs_node);
raw_spin_lock_irq(&pwq->pool->lock);
if (pwq->plugged) {
pwq->plugged = false;
if (pwq_activate_first_inactive(pwq, true))
kick_pool(pwq->pool);
}
raw_spin_unlock_irq(&pwq->pool->lock);
}
/**
* node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active
* @nna: wq_node_nr_active to activate a pending pwq for
* @caller_pool: worker_pool the caller is locking
*
* Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked.
* @caller_pool may be unlocked and relocked to lock other worker_pools.
*/
static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
struct worker_pool *caller_pool)
{
struct worker_pool *locked_pool = caller_pool;
struct pool_workqueue *pwq;
struct work_struct *work;
lockdep_assert_held(&caller_pool->lock);
raw_spin_lock(&nna->lock);
retry:
pwq = list_first_entry_or_null(&nna->pending_pwqs,
struct pool_workqueue, pending_node);
if (!pwq)
goto out_unlock;
/*
* If @pwq is for a different pool than @locked_pool, we need to lock
* @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
* / lock dance. For that, we also need to release @nna->lock as it's
* nested inside pool locks.
*/
if (pwq->pool != locked_pool) {
raw_spin_unlock(&locked_pool->lock);
locked_pool = pwq->pool;
if (!raw_spin_trylock(&locked_pool->lock)) {
raw_spin_unlock(&nna->lock);
raw_spin_lock(&locked_pool->lock);
raw_spin_lock(&nna->lock);
goto retry;
}
}
/*
* $pwq may not have any inactive work items due to e.g. cancellations.
* Drop it from pending_pwqs and see if there's another one.
*/
work = list_first_entry_or_null(&pwq->inactive_works,
struct work_struct, entry);
if (!work) {
list_del_init(&pwq->pending_node);
goto retry;
}
/*
* Acquire an nr_active count and activate the inactive work item. If
* $pwq still has inactive work items, rotate it to the end of the
* pending_pwqs so that we round-robin through them. This means that
* inactive work items are not activated in queueing order which is fine
* given that there has never been any ordering across different pwqs.
*/
if (likely(tryinc_node_nr_active(nna))) {
pwq->nr_active++;
__pwq_activate_work(pwq, work);
if (list_empty(&pwq->inactive_works))
list_del_init(&pwq->pending_node);
else
list_move_tail(&pwq->pending_node, &nna->pending_pwqs);
/* if activating a foreign pool, make sure it's running */
if (pwq->pool != caller_pool)
kick_pool(pwq->pool);
}
out_unlock:
raw_spin_unlock(&nna->lock);
if (locked_pool != caller_pool) {
raw_spin_unlock(&locked_pool->lock);
raw_spin_lock(&caller_pool->lock);
}
}
/**
* pwq_dec_nr_active - Retire an active count
* @pwq: pool_workqueue of interest
*
* Decrement @pwq's nr_active and try to activate the first inactive work item.
* For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
*/
static void pwq_dec_nr_active(struct pool_workqueue *pwq)
{
struct worker_pool *pool = pwq->pool;
struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
lockdep_assert_held(&pool->lock);
/*
* @pwq->nr_active should be decremented for both percpu and unbound
* workqueues.
*/
pwq->nr_active--;
/*
* For a percpu workqueue, it's simple. Just need to kick the first
* inactive work item on @pwq itself.
*/
if (!nna) {
pwq_activate_first_inactive(pwq, false);
return;
}
/*
* If @pwq is for an unbound workqueue, it's more complicated because
* multiple pwqs and pools may be sharing the nr_active count. When a
* pwq needs to wait for an nr_active count, it puts itself on
* $nna->pending_pwqs. The following atomic_dec_return()'s implied
* memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to
* guarantee that either we see non-empty pending_pwqs or they see
* decremented $nna->nr.
*
* $nna->max may change as CPUs come online/offline and @pwq->wq's
* max_active gets updated. However, it is guaranteed to be equal to or
* larger than @pwq->wq->min_active which is above zero unless freezing.
* This maintains the forward progress guarantee.
*/
if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max))
return;
if (!list_empty(&nna->pending_pwqs))
node_activate_pending_pwq(nna, pool);
}
/**
* pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
* @pwq: pwq of interest
* @work_data: work_data of work which left the queue
*
* A work either has completed or is removed from pending queue,
* decrement nr_in_flight of its pwq and handle workqueue flushing.
*
* NOTE:
* For unbound workqueues, this function may temporarily drop @pwq->pool->lock
* and thus should be called after all other state updates for the in-flight
* work item is complete.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock).
*/
static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
{
int color = get_work_color(work_data);
if (!(work_data & WORK_STRUCT_INACTIVE))
pwq_dec_nr_active(pwq);
pwq->nr_in_flight[color]--;
/* is flush in progress and are we at the flushing tip? */
if (likely(pwq->flush_color != color))
goto out_put;
/* are there still in-flight works? */
if (pwq->nr_in_flight[color])
goto out_put;
/* this pwq is done, clear flush_color */
pwq->flush_color = -1;
/*
* If this was the last pwq, wake up the first flusher. It
* will handle the rest.
*/
if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
complete(&pwq->wq->first_flusher->done);
out_put:
put_pwq(pwq);
}
/**
* try_to_grab_pending - steal work item from worklist and disable irq
* @work: work item to steal
* @cflags: %WORK_CANCEL_ flags
* @irq_flags: place to store irq state
*
* Try to grab PENDING bit of @work. This function can handle @work in any
* stable state - idle, on timer or on worklist.
*
* Return:
*
* ======== ================================================================
* 1 if @work was pending and we successfully stole PENDING
* 0 if @work was idle and we claimed PENDING
* -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
* -ENOENT if someone else is canceling @work, this state may persist
* for arbitrarily long
* ======== ================================================================
*
* Note:
* On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
* interrupted while holding PENDING and @work off queue, irq must be
* disabled on entry. This, combined with delayed_work->timer being
* irqsafe, ensures that we return -EAGAIN for finite short period of time.
*
* On successful return, >= 0, irq is disabled and the caller is
* responsible for releasing it using local_irq_restore(*@irq_flags).
*
* This function is safe to call from any context including IRQ handler.
*/
static int try_to_grab_pending(struct work_struct *work, u32 cflags,
unsigned long *irq_flags)
{
struct worker_pool *pool;
struct pool_workqueue *pwq;
local_irq_save(*irq_flags);
/* try to steal the timer if it exists */
if (cflags & WORK_CANCEL_DELAYED) {
struct delayed_work *dwork = to_delayed_work(work);
/*
* dwork->timer is irqsafe. If del_timer() fails, it's
* guaranteed that the timer is not queued anywhere and not
* running on the local CPU.
*/
if (likely(del_timer(&dwork->timer)))
return 1;
}
/* try to claim PENDING the normal way */
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) return 0; rcu_read_lock();
/*
* The queueing is in progress, or it is already queued. Try to
* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
*/
pool = get_work_pool(work);
if (!pool)
goto fail;
raw_spin_lock(&pool->lock);
/*
* work->data is guaranteed to point to pwq only while the work
* item is queued on pwq->wq, and both updating work->data to point
* to pwq on queueing and to pool on dequeueing are done under
* pwq->pool->lock. This in turn guarantees that, if work->data
* points to pwq which is associated with a locked pool, the work
* item is currently queued on that pool.
*/
pwq = get_work_pwq(work); if (pwq && pwq->pool == pool) {
unsigned long work_data;
debug_work_deactivate(work);
/*
* A cancelable inactive work item must be in the
* pwq->inactive_works since a queued barrier can't be
* canceled (see the comments in insert_wq_barrier()).
*
* An inactive work item cannot be grabbed directly because
* it might have linked barrier work items which, if left
* on the inactive_works list, will confuse pwq->nr_active
* management later on and cause stall. Make sure the work
* item is activated before grabbing.
*/
pwq_activate_work(pwq, work); list_del_init(&work->entry);
/*
* work->data points to pwq iff queued. Let's point to pool. As
* this destroys work->data needed by the next step, stash it.
*/
work_data = *work_data_bits(work);
set_work_pool_and_keep_pending(work, pool->id, 0);
/* must be the last step, see the function comment */
pwq_dec_nr_in_flight(pwq, work_data);
raw_spin_unlock(&pool->lock);
rcu_read_unlock();
return 1;
}
raw_spin_unlock(&pool->lock);
fail:
rcu_read_unlock(); local_irq_restore(*irq_flags); if (work_is_canceling(work))
return -ENOENT;
cpu_relax();
return -EAGAIN;
}
struct cwt_wait {
wait_queue_entry_t wait;
struct work_struct *work;
};
static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
{
struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
if (cwait->work != key)
return 0;
return autoremove_wake_function(wait, mode, sync, key);
}
/**
* work_grab_pending - steal work item from worklist and disable irq
* @work: work item to steal
* @cflags: %WORK_CANCEL_ flags
* @irq_flags: place to store IRQ state
*
* Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
* or on worklist.
*
* Must be called in process context. IRQ is disabled on return with IRQ state
* stored in *@irq_flags. The caller is responsible for re-enabling it using
* local_irq_restore().
*
* Returns %true if @work was pending. %false if idle.
*/
static bool work_grab_pending(struct work_struct *work, u32 cflags,
unsigned long *irq_flags)
{
struct cwt_wait cwait;
int ret;
might_sleep();
repeat:
ret = try_to_grab_pending(work, cflags, irq_flags);
if (likely(ret >= 0))
return ret; if (ret != -ENOENT)
goto repeat;
/*
* Someone is already canceling. Wait for it to finish. flush_work()
* doesn't work for PREEMPT_NONE because we may get woken up between
* @work's completion and the other canceling task resuming and clearing
* CANCELING - flush_work() will return false immediately as @work is no
* longer busy, try_to_grab_pending() will return -ENOENT as @work is
* still being canceled and the other canceling task won't be able to
* clear CANCELING as we're hogging the CPU.
*
* Let's wait for completion using a waitqueue. As this may lead to the
* thundering herd problem, use a custom wake function which matches
* @work along with exclusive wait and wakeup.
*/
init_wait(&cwait.wait);
cwait.wait.func = cwt_wakefn;
cwait.work = work;
prepare_to_wait_exclusive(&wq_cancel_waitq, &cwait.wait,
TASK_UNINTERRUPTIBLE);
if (work_is_canceling(work))
schedule(); finish_wait(&wq_cancel_waitq, &cwait.wait);
goto repeat;
}
/**
* insert_work - insert a work into a pool
* @pwq: pwq @work belongs to
* @work: work to insert
* @head: insertion point
* @extra_flags: extra WORK_STRUCT_* flags to set
*
* Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
* work_struct flags.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock).
*/
static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
struct list_head *head, unsigned int extra_flags)
{
debug_work_activate(work);
/* record the work call stack in order to print it in KASAN reports */
kasan_record_aux_stack_noalloc(work);
/* we own @work, set data and link */
set_work_pwq(work, pwq, extra_flags); list_add_tail(&work->entry, head); get_pwq(pwq);
}
/*
* Test whether @work is being queued from another work executing on the
* same workqueue.
*/
static bool is_chained_work(struct workqueue_struct *wq)
{
struct worker *worker;
worker = current_wq_worker();
/*
* Return %true iff I'm a worker executing a work item on @wq. If
* I'm @worker, it's safe to dereference it without locking.
*/
return worker && worker->current_pwq->wq == wq;
}
/*
* When queueing an unbound work item to a wq, prefer local CPU if allowed
* by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
* avoid perturbing sensitive tasks.
*/
static int wq_select_unbound_cpu(int cpu)
{
int new_cpu;
if (likely(!wq_debug_force_rr_cpu)) {
if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
return cpu;
} else {
pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
}
new_cpu = __this_cpu_read(wq_rr_cpu_last); new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask); if (unlikely(new_cpu >= nr_cpu_ids)) { new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask); if (unlikely(new_cpu >= nr_cpu_ids))
return cpu;
}
__this_cpu_write(wq_rr_cpu_last, new_cpu);
return new_cpu;
}
static void __queue_work(int cpu, struct workqueue_struct *wq,
struct work_struct *work)
{
struct pool_workqueue *pwq;
struct worker_pool *last_pool, *pool;
unsigned int work_flags;
unsigned int req_cpu = cpu;
/*
* While a work item is PENDING && off queue, a task trying to
* steal the PENDING will busy-loop waiting for it to either get
* queued or lose PENDING. Grabbing PENDING and queueing should
* happen with IRQ disabled.
*/
lockdep_assert_irqs_disabled();
/*
* For a draining wq, only works from the same workqueue are
* allowed. The __WQ_DESTROYING helps to spot the issue that
* queues a new work item to a wq after destroy_workqueue(wq).
*/
if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
WARN_ON_ONCE(!is_chained_work(wq))))
return;
rcu_read_lock();
retry:
/* pwq which will be used unless @work is executing elsewhere */
if (req_cpu == WORK_CPU_UNBOUND) { if (wq->flags & WQ_UNBOUND) cpu = wq_select_unbound_cpu(raw_smp_processor_id());
else
cpu = raw_smp_processor_id();
}
pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu)); pool = pwq->pool;
/*
* If @work was previously on a different pool, it might still be
* running there, in which case the work needs to be queued on that
* pool to guarantee non-reentrancy.
*/
last_pool = get_work_pool(work);
if (last_pool && last_pool != pool) {
struct worker *worker;
raw_spin_lock(&last_pool->lock); worker = find_worker_executing_work(last_pool, work); if (worker && worker->current_pwq->wq == wq) {
pwq = worker->current_pwq;
pool = pwq->pool; WARN_ON_ONCE(pool != last_pool);
} else {
/* meh... not running there, queue here */
raw_spin_unlock(&last_pool->lock);
raw_spin_lock(&pool->lock);
}
} else {
raw_spin_lock(&pool->lock);
}
/*
* pwq is determined and locked. For unbound pools, we could have raced
* with pwq release and it could already be dead. If its refcnt is zero,
* repeat pwq selection. Note that unbound pwqs never die without
* another pwq replacing it in cpu_pwq or while work items are executing
* on it, so the retrying is guaranteed to make forward-progress.
*/
if (unlikely(!pwq->refcnt)) { if (wq->flags & WQ_UNBOUND) { raw_spin_unlock(&pool->lock);
cpu_relax();
goto retry;
}
/* oops */
WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
wq->name, cpu);
}
/* pwq determined, queue */
trace_workqueue_queue_work(req_cpu, pwq, work); if (WARN_ON(!list_empty(&work->entry)))
goto out;
pwq->nr_in_flight[pwq->work_color]++;
work_flags = work_color_to_flags(pwq->work_color);
/*
* Limit the number of concurrently active work items to max_active.
* @work must also queue behind existing inactive work items to maintain
* ordering when max_active changes. See wq_adjust_max_active().
*/
if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) { if (list_empty(&pool->worklist)) pool->watchdog_ts = jiffies; trace_workqueue_activate_work(work);
insert_work(pwq, work, &pool->worklist, work_flags);
kick_pool(pool);
} else {
work_flags |= WORK_STRUCT_INACTIVE;
insert_work(pwq, work, &pwq->inactive_works, work_flags);
}
out:
raw_spin_unlock(&pool->lock); rcu_read_unlock();
}
/**
* queue_work_on - queue work on specific cpu
* @cpu: CPU number to execute work on
* @wq: workqueue to use
* @work: work to queue
*
* We queue the work to a specific CPU, the caller must ensure it
* can't go away. Callers that fail to ensure that the specified
* CPU cannot go away will execute on a randomly chosen CPU.
* But note well that callers specifying a CPU that never has been
* online will get a splat.
*
* Return: %false if @work was already on a queue, %true otherwise.
*/
bool queue_work_on(int cpu, struct workqueue_struct *wq,
struct work_struct *work)
{
bool ret = false;
unsigned long irq_flags;
local_irq_save(irq_flags); if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { __queue_work(cpu, wq, work);
ret = true;
}
local_irq_restore(irq_flags); return ret;
}
EXPORT_SYMBOL(queue_work_on);
/**
* select_numa_node_cpu - Select a CPU based on NUMA node
* @node: NUMA node ID that we want to select a CPU from
*
* This function will attempt to find a "random" cpu available on a given
* node. If there are no CPUs available on the given node it will return
* WORK_CPU_UNBOUND indicating that we should just schedule to any
* available CPU if we need to schedule this work.
*/
static int select_numa_node_cpu(int node)
{
int cpu;
/* Delay binding to CPU if node is not valid or online */
if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
return WORK_CPU_UNBOUND;
/* Use local node/cpu if we are already there */
cpu = raw_smp_processor_id();
if (node == cpu_to_node(cpu))
return cpu;
/* Use "random" otherwise know as "first" online CPU of node */
cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
/* If CPU is valid return that, otherwise just defer */
return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
}
/**
* queue_work_node - queue work on a "random" cpu for a given NUMA node
* @node: NUMA node that we are targeting the work for
* @wq: workqueue to use
* @work: work to queue
*
* We queue the work to a "random" CPU within a given NUMA node. The basic
* idea here is to provide a way to somehow associate work with a given
* NUMA node.
*
* This function will only make a best effort attempt at getting this onto
* the right NUMA node. If no node is requested or the requested node is
* offline then we just fall back to standard queue_work behavior.
*
* Currently the "random" CPU ends up being the first available CPU in the
* intersection of cpu_online_mask and the cpumask of the node, unless we
* are running on the node. In that case we just use the current CPU.
*
* Return: %false if @work was already on a queue, %true otherwise.
*/
bool queue_work_node(int node, struct workqueue_struct *wq,
struct work_struct *work)
{
unsigned long irq_flags;
bool ret = false;
/*
* This current implementation is specific to unbound workqueues.
* Specifically we only return the first available CPU for a given
* node instead of cycling through individual CPUs within the node.
*
* If this is used with a per-cpu workqueue then the logic in
* workqueue_select_cpu_near would need to be updated to allow for
* some round robin type logic.
*/
WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
local_irq_save(irq_flags);
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
int cpu = select_numa_node_cpu(node);
__queue_work(cpu, wq, work);
ret = true;
}
local_irq_restore(irq_flags);
return ret;
}
EXPORT_SYMBOL_GPL(queue_work_node);
void delayed_work_timer_fn(struct timer_list *t)
{
struct delayed_work *dwork = from_timer(dwork, t, timer);
/* should have been called from irqsafe timer with irq already off */
__queue_work(dwork->cpu, dwork->wq, &dwork->work);
}
EXPORT_SYMBOL(delayed_work_timer_fn);
static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
struct delayed_work *dwork, unsigned long delay)
{
struct timer_list *timer = &dwork->timer;
struct work_struct *work = &dwork->work;
WARN_ON_ONCE(!wq); WARN_ON_ONCE(timer->function != delayed_work_timer_fn); WARN_ON_ONCE(timer_pending(timer)); WARN_ON_ONCE(!list_empty(&work->entry));
/*
* If @delay is 0, queue @dwork->work immediately. This is for
* both optimization and correctness. The earliest @timer can
* expire is on the closest next tick and delayed_work users depend
* on that there's no such delay when @delay is 0.
*/
if (!delay) { __queue_work(cpu, wq, &dwork->work);
return;
}
dwork->wq = wq;
dwork->cpu = cpu;
timer->expires = jiffies + delay;
if (housekeeping_enabled(HK_TYPE_TIMER)) {
/* If the current cpu is a housekeeping cpu, use it. */
cpu = smp_processor_id();
if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER))
cpu = housekeeping_any_cpu(HK_TYPE_TIMER); add_timer_on(timer, cpu);
} else {
if (likely(cpu == WORK_CPU_UNBOUND)) add_timer_global(timer);
else
add_timer_on(timer, cpu);
}
}
/**
* queue_delayed_work_on - queue work on specific CPU after delay
* @cpu: CPU number to execute work on
* @wq: workqueue to use
* @dwork: work to queue
* @delay: number of jiffies to wait before queueing
*
* Return: %false if @work was already on a queue, %true otherwise. If
* @delay is zero and @dwork is idle, it will be scheduled for immediate
* execution.
*/
bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
struct delayed_work *dwork, unsigned long delay)
{
struct work_struct *work = &dwork->work;
bool ret = false;
unsigned long irq_flags;
/* read the comment in __queue_work() */
local_irq_save(irq_flags); if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { __queue_delayed_work(cpu, wq, dwork, delay);
ret = true;
}
local_irq_restore(irq_flags); return ret;
}
EXPORT_SYMBOL(queue_delayed_work_on);
/**
* mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
* @cpu: CPU number to execute work on
* @wq: workqueue to use
* @dwork: work to queue
* @delay: number of jiffies to wait before queueing
*
* If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
* modify @dwork's timer so that it expires after @delay. If @delay is
* zero, @work is guaranteed to be scheduled immediately regardless of its
* current state.
*
* Return: %false if @dwork was idle and queued, %true if @dwork was
* pending and its timer was modified.
*
* This function is safe to call from any context including IRQ handler.
* See try_to_grab_pending() for details.
*/
bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
struct delayed_work *dwork, unsigned long delay)
{
unsigned long irq_flags;
int ret;
do {
ret = try_to_grab_pending(&dwork->work, WORK_CANCEL_DELAYED,
&irq_flags);
} while (unlikely(ret == -EAGAIN));
if (likely(ret >= 0)) { __queue_delayed_work(cpu, wq, dwork, delay); local_irq_restore(irq_flags);
}
/* -ENOENT from try_to_grab_pending() becomes %true */
return ret;
}
EXPORT_SYMBOL_GPL(mod_delayed_work_on);
static void rcu_work_rcufn(struct rcu_head *rcu)
{
struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
/* read the comment in __queue_work() */
local_irq_disable();
__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
local_irq_enable();
}
/**
* queue_rcu_work - queue work after a RCU grace period
* @wq: workqueue to use
* @rwork: work to queue
*
* Return: %false if @rwork was already pending, %true otherwise. Note
* that a full RCU grace period is guaranteed only after a %true return.
* While @rwork is guaranteed to be executed after a %false return, the
* execution may happen before a full RCU grace period has passed.
*/
bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
{
struct work_struct *work = &rwork->work;
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
rwork->wq = wq;
call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
return true;
}
return false;
}
EXPORT_SYMBOL(queue_rcu_work);
static struct worker *alloc_worker(int node)
{
struct worker *worker;
worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
if (worker) {
INIT_LIST_HEAD(&worker->entry);
INIT_LIST_HEAD(&worker->scheduled);
INIT_LIST_HEAD(&worker->node);
/* on creation a worker is in !idle && prep state */
worker->flags = WORKER_PREP;
}
return worker;
}
static cpumask_t *pool_allowed_cpus(struct worker_pool *pool)
{
if (pool->cpu < 0 && pool->attrs->affn_strict)
return pool->attrs->__pod_cpumask;
else
return pool->attrs->cpumask;
}
/**
* worker_attach_to_pool() - attach a worker to a pool
* @worker: worker to be attached
* @pool: the target pool
*
* Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
* cpu-binding of @worker are kept coordinated with the pool across
* cpu-[un]hotplugs.
*/
static void worker_attach_to_pool(struct worker *worker,
struct worker_pool *pool)
{
mutex_lock(&wq_pool_attach_mutex);
/*
* The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable
* across this function. See the comments above the flag definition for
* details. BH workers are, while per-CPU, always DISASSOCIATED.
*/
if (pool->flags & POOL_DISASSOCIATED) {
worker->flags |= WORKER_UNBOUND;
} else {
WARN_ON_ONCE(pool->flags & POOL_BH);
kthread_set_per_cpu(worker->task, pool->cpu);
}
if (worker->rescue_wq)
set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool));
list_add_tail(&worker->node, &pool->workers);
worker->pool = pool;
mutex_unlock(&wq_pool_attach_mutex);
}
/**
* worker_detach_from_pool() - detach a worker from its pool
* @worker: worker which is attached to its pool
*
* Undo the attaching which had been done in worker_attach_to_pool(). The
* caller worker shouldn't access to the pool after detached except it has
* other reference to the pool.
*/
static void worker_detach_from_pool(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
struct completion *detach_completion = NULL;
/* there is one permanent BH worker per CPU which should never detach */
WARN_ON_ONCE(pool->flags & POOL_BH);
mutex_lock(&wq_pool_attach_mutex);
kthread_set_per_cpu(worker->task, -1);
list_del(&worker->node);
worker->pool = NULL;
if (list_empty(&pool->workers) && list_empty(&pool->dying_workers))
detach_completion = pool->detach_completion;
mutex_unlock(&wq_pool_attach_mutex);
/* clear leftover flags without pool->lock after it is detached */
worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
if (detach_completion)
complete(detach_completion);
}
/**
* create_worker - create a new workqueue worker
* @pool: pool the new worker will belong to
*
* Create and start a new worker which is attached to @pool.
*
* CONTEXT:
* Might sleep. Does GFP_KERNEL allocations.
*
* Return:
* Pointer to the newly created worker.
*/
static struct worker *create_worker(struct worker_pool *pool)
{
struct worker *worker;
int id;
char id_buf[23];
/* ID is needed to determine kthread name */
id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
if (id < 0) {
pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
ERR_PTR(id));
return NULL;
}
worker = alloc_worker(pool->node);
if (!worker) {
pr_err_once("workqueue: Failed to allocate a worker\n");
goto fail;
}
worker->id = id;
if (!(pool->flags & POOL_BH)) {
if (pool->cpu >= 0)
snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
pool->attrs->nice < 0 ? "H" : "");
else
snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
worker->task = kthread_create_on_node(worker_thread, worker,
pool->node, "kworker/%s", id_buf);
if (IS_ERR(worker->task)) {
if (PTR_ERR(worker->task) == -EINTR) {
pr_err("workqueue: Interrupted when creating a worker thread \"kworker/%s\"\n",
id_buf);
} else {
pr_err_once("workqueue: Failed to create a worker thread: %pe",
worker->task);
}
goto fail;
}
set_user_nice(worker->task, pool->attrs->nice);
kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
}
/* successful, attach the worker to the pool */
worker_attach_to_pool(worker, pool);
/* start the newly created worker */
raw_spin_lock_irq(&pool->lock);
worker->pool->nr_workers++;
worker_enter_idle(worker);
/*
* @worker is waiting on a completion in kthread() and will trigger hung
* check if not woken up soon. As kick_pool() is noop if @pool is empty,
* wake it up explicitly.
*/
if (worker->task)
wake_up_process(worker->task);
raw_spin_unlock_irq(&pool->lock);
return worker;
fail:
ida_free(&pool->worker_ida, id);
kfree(worker);
return NULL;
}
static void unbind_worker(struct worker *worker)
{
lockdep_assert_held(&wq_pool_attach_mutex);
kthread_set_per_cpu(worker->task, -1);
if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
else
WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
}
static void wake_dying_workers(struct list_head *cull_list)
{
struct worker *worker, *tmp;
list_for_each_entry_safe(worker, tmp, cull_list, entry) {
list_del_init(&worker->entry);
unbind_worker(worker);
/*
* If the worker was somehow already running, then it had to be
* in pool->idle_list when set_worker_dying() happened or we
* wouldn't have gotten here.
*
* Thus, the worker must either have observed the WORKER_DIE
* flag, or have set its state to TASK_IDLE. Either way, the
* below will be observed by the worker and is safe to do
* outside of pool->lock.
*/
wake_up_process(worker->task);
}
}
/**
* set_worker_dying - Tag a worker for destruction
* @worker: worker to be destroyed
* @list: transfer worker away from its pool->idle_list and into list
*
* Tag @worker for destruction and adjust @pool stats accordingly. The worker
* should be idle.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock).
*/
static void set_worker_dying(struct worker *worker, struct list_head *list)
{
struct worker_pool *pool = worker->pool;
lockdep_assert_held(&pool->lock);
lockdep_assert_held(&wq_pool_attach_mutex);
/* sanity check frenzy */
if (WARN_ON(worker->current_work) ||
WARN_ON(!list_empty(&worker->scheduled)) ||
WARN_ON(!(worker->flags & WORKER_IDLE)))
return;
pool->nr_workers--;
pool->nr_idle--;
worker->flags |= WORKER_DIE;
list_move(&worker->entry, list);
list_move(&worker->node, &pool->dying_workers);
}
/**
* idle_worker_timeout - check if some idle workers can now be deleted.
* @t: The pool's idle_timer that just expired
*
* The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
* worker_leave_idle(), as a worker flicking between idle and active while its
* pool is at the too_many_workers() tipping point would cause too much timer
* housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
* it expire and re-evaluate things from there.
*/
static void idle_worker_timeout(struct timer_list *t)
{
struct worker_pool *pool = from_timer(pool, t, idle_timer);
bool do_cull = false;
if (work_pending(&pool->idle_cull_work))
return;
raw_spin_lock_irq(&pool->lock);
if (too_many_workers(pool)) {
struct worker *worker;
unsigned long expires;
/* idle_list is kept in LIFO order, check the last one */
worker = list_entry(pool->idle_list.prev, struct worker, entry);
expires = worker->last_active + IDLE_WORKER_TIMEOUT;
do_cull = !time_before(jiffies, expires);
if (!do_cull)
mod_timer(&pool->idle_timer, expires);
}
raw_spin_unlock_irq(&pool->lock);
if (do_cull)
queue_work(system_unbound_wq, &pool->idle_cull_work);
}
/**
* idle_cull_fn - cull workers that have been idle for too long.
* @work: the pool's work for handling these idle workers
*
* This goes through a pool's idle workers and gets rid of those that have been
* idle for at least IDLE_WORKER_TIMEOUT seconds.
*
* We don't want to disturb isolated CPUs because of a pcpu kworker being
* culled, so this also resets worker affinity. This requires a sleepable
* context, hence the split between timer callback and work item.
*/
static void idle_cull_fn(struct work_struct *work)
{
struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
LIST_HEAD(cull_list);
/*
* Grabbing wq_pool_attach_mutex here ensures an already-running worker
* cannot proceed beyong worker_detach_from_pool() in its self-destruct
* path. This is required as a previously-preempted worker could run after
* set_worker_dying() has happened but before wake_dying_workers() did.
*/
mutex_lock(&wq_pool_attach_mutex);
raw_spin_lock_irq(&pool->lock);
while (too_many_workers(pool)) {
struct worker *worker;
unsigned long expires;
worker = list_entry(pool->idle_list.prev, struct worker, entry);
expires = worker->last_active + IDLE_WORKER_TIMEOUT;
if (time_before(jiffies, expires)) {
mod_timer(&pool->idle_timer, expires);
break;
}
set_worker_dying(worker, &cull_list);
}
raw_spin_unlock_irq(&pool->lock);
wake_dying_workers(&cull_list);
mutex_unlock(&wq_pool_attach_mutex);
}
static void send_mayday(struct work_struct *work)
{
struct pool_workqueue *pwq = get_work_pwq(work);
struct workqueue_struct *wq = pwq->wq;
lockdep_assert_held(&wq_mayday_lock);
if (!wq->rescuer)
return;
/* mayday mayday mayday */
if (list_empty(&pwq->mayday_node)) {
/*
* If @pwq is for an unbound wq, its base ref may be put at
* any time due to an attribute change. Pin @pwq until the
* rescuer is done with it.
*/
get_pwq(pwq);
list_add_tail(&pwq->mayday_node, &wq->maydays);
wake_up_process(wq->rescuer->task);
pwq->stats[PWQ_STAT_MAYDAY]++;
}
}
static void pool_mayday_timeout(struct timer_list *t)
{
struct worker_pool *pool = from_timer(pool, t, mayday_timer);
struct work_struct *work;
raw_spin_lock_irq(&pool->lock);
raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
if (need_to_create_worker(pool)) {
/*
* We've been trying to create a new worker but
* haven't been successful. We might be hitting an
* allocation deadlock. Send distress signals to
* rescuers.
*/
list_for_each_entry(work, &pool->worklist, entry)
send_mayday(work);
}
raw_spin_unlock(&wq_mayday_lock);
raw_spin_unlock_irq(&pool->lock);
mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
}
/**
* maybe_create_worker - create a new worker if necessary
* @pool: pool to create a new worker for
*
* Create a new worker for @pool if necessary. @pool is guaranteed to
* have at least one idle worker on return from this function. If
* creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
* sent to all rescuers with works scheduled on @pool to resolve
* possible allocation deadlock.
*
* On return, need_to_create_worker() is guaranteed to be %false and
* may_start_working() %true.
*
* LOCKING:
* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
* multiple times. Does GFP_KERNEL allocations. Called only from
* manager.
*/
static void maybe_create_worker(struct worker_pool *pool)
__releases(&pool->lock)
__acquires(&pool->lock)
{
restart:
raw_spin_unlock_irq(&pool->lock);
/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
while (true) {
if (create_worker(pool) || !need_to_create_worker(pool))
break;
schedule_timeout_interruptible(CREATE_COOLDOWN);
if (!need_to_create_worker(pool))
break;
}
del_timer_sync(&pool->mayday_timer);
raw_spin_lock_irq(&pool->lock);
/*
* This is necessary even after a new worker was just successfully
* created as @pool->lock was dropped and the new worker might have
* already become busy.
*/
if (need_to_create_worker(pool))
goto restart;
}
/**
* manage_workers - manage worker pool
* @worker: self
*
* Assume the manager role and manage the worker pool @worker belongs
* to. At any given time, there can be only zero or one manager per
* pool. The exclusion is handled automatically by this function.
*
* The caller can safely start processing works on false return. On
* true return, it's guaranteed that need_to_create_worker() is false
* and may_start_working() is true.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
* multiple times. Does GFP_KERNEL allocations.
*
* Return:
* %false if the pool doesn't need management and the caller can safely
* start processing works, %true if management function was performed and
* the conditions that the caller verified before calling the function may
* no longer be true.
*/
static bool manage_workers(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
if (pool->flags & POOL_MANAGER_ACTIVE)
return false;
pool->flags |= POOL_MANAGER_ACTIVE;
pool->manager = worker;
maybe_create_worker(pool);
pool->manager = NULL;
pool->flags &= ~POOL_MANAGER_ACTIVE;
rcuwait_wake_up(&manager_wait);
return true;
}
/**
* process_one_work - process single work
* @worker: self
* @work: work to process
*
* Process @work. This function contains all the logics necessary to
* process a single work including synchronization against and
* interaction with other workers on the same cpu, queueing and
* flushing. As long as context requirement is met, any worker can
* call this function to process a work.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock) which is released and regrabbed.
*/
static void process_one_work(struct worker *worker, struct work_struct *work)
__releases(&pool->lock)
__acquires(&pool->lock)
{
struct pool_workqueue *pwq = get_work_pwq(work);
struct worker_pool *pool = worker->pool;
unsigned long work_data;
int lockdep_start_depth, rcu_start_depth;
bool bh_draining = pool->flags & POOL_BH_DRAINING;
#ifdef CONFIG_LOCKDEP
/*
* It is permissible to free the struct work_struct from
* inside the function that is called from it, this we need to
* take into account for lockdep too. To avoid bogus "held
* lock freed" warnings as well as problems when looking into
* work->lockdep_map, make a copy and use that here.
*/
struct lockdep_map lockdep_map;
lockdep_copy_map(&lockdep_map, &work->lockdep_map);
#endif
/* ensure we're on the correct CPU */
WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
raw_smp_processor_id() != pool->cpu);
/* claim and dequeue */
debug_work_deactivate(work);
hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
worker->current_work = work;
worker->current_func = work->func;
worker->current_pwq = pwq;
if (worker->task)
worker->current_at = worker->task->se.sum_exec_runtime;
work_data = *work_data_bits(work);
worker->current_color = get_work_color(work_data);
/*
* Record wq name for cmdline and debug reporting, may get
* overridden through set_worker_desc().
*/
strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
list_del_init(&work->entry);
/*
* CPU intensive works don't participate in concurrency management.
* They're the scheduler's responsibility. This takes @worker out
* of concurrency management and the next code block will chain
* execution of the pending work items.
*/
if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
worker_set_flags(worker, WORKER_CPU_INTENSIVE);
/*
* Kick @pool if necessary. It's always noop for per-cpu worker pools
* since nr_running would always be >= 1 at this point. This is used to
* chain execution of the pending work items for WORKER_NOT_RUNNING
* workers such as the UNBOUND and CPU_INTENSIVE ones.
*/
kick_pool(pool);
/*
* Record the last pool and clear PENDING which should be the last
* update to @work. Also, do this inside @pool->lock so that
* PENDING and queued state changes happen together while IRQ is
* disabled.
*/
set_work_pool_and_clear_pending(work, pool->id, 0);
pwq->stats[PWQ_STAT_STARTED]++;
raw_spin_unlock_irq(&pool->lock);
rcu_start_depth = rcu_preempt_depth();
lockdep_start_depth = lockdep_depth(current);
/* see drain_dead_softirq_workfn() */
if (!bh_draining)
lock_map_acquire(&pwq->wq->lockdep_map);
lock_map_acquire(&lockdep_map);
/*
* Strictly speaking we should mark the invariant state without holding
* any locks, that is, before these two lock_map_acquire()'s.
*
* However, that would result in:
*
* A(W1)
* WFC(C)
* A(W1)
* C(C)
*
* Which would create W1->C->W1 dependencies, even though there is no
* actual deadlock possible. There are two solutions, using a
* read-recursive acquire on the work(queue) 'locks', but this will then
* hit the lockdep limitation on recursive locks, or simply discard
* these locks.
*
* AFAICT there is no possible deadlock scenario between the
* flush_work() and complete() primitives (except for single-threaded
* workqueues), so hiding them isn't a problem.
*/
lockdep_invariant_state(true);
trace_workqueue_execute_start(work);
worker->current_func(work);
/*
* While we must be careful to not use "work" after this, the trace
* point will only record its address.
*/
trace_workqueue_execute_end(work, worker->current_func);
pwq->stats[PWQ_STAT_COMPLETED]++;
lock_map_release(&lockdep_map);
if (!bh_draining)
lock_map_release(&pwq->wq->lockdep_map);
if (unlikely((worker->task && in_atomic()) ||
lockdep_depth(current) != lockdep_start_depth ||
rcu_preempt_depth() != rcu_start_depth)) {
pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n"
" preempt=0x%08x lock=%d->%d RCU=%d->%d workfn=%ps\n",
current->comm, task_pid_nr(current), preempt_count(),
lockdep_start_depth, lockdep_depth(current),
rcu_start_depth, rcu_preempt_depth(),
worker->current_func);
debug_show_held_locks(current);
dump_stack();
}
/*
* The following prevents a kworker from hogging CPU on !PREEMPTION
* kernels, where a requeueing work item waiting for something to
* happen could deadlock with stop_machine as such work item could
* indefinitely requeue itself while all other CPUs are trapped in
* stop_machine. At the same time, report a quiescent RCU state so
* the same condition doesn't freeze RCU.
*/
if (worker->task)
cond_resched();
raw_spin_lock_irq(&pool->lock);
/*
* In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
* CPU intensive by wq_worker_tick() if @work hogged CPU longer than
* wq_cpu_intensive_thresh_us. Clear it.
*/
worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
/* tag the worker for identification in schedule() */
worker->last_func = worker->current_func;
/* we're done with it, release */
hash_del(&worker->hentry);
worker->current_work = NULL;
worker->current_func = NULL;
worker->current_pwq = NULL;
worker->current_color = INT_MAX;
/* must be the last step, see the function comment */
pwq_dec_nr_in_flight(pwq, work_data);
}
/**
* process_scheduled_works - process scheduled works
* @worker: self
*
* Process all scheduled works. Please note that the scheduled list
* may change while processing a work, so this function repeatedly
* fetches a work from the top and executes it.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock) which may be released and regrabbed
* multiple times.
*/
static void process_scheduled_works(struct worker *worker)
{
struct work_struct *work;
bool first = true;
while ((work = list_first_entry_or_null(&worker->scheduled,
struct work_struct, entry))) {
if (first) {
worker->pool->watchdog_ts = jiffies;
first = false;
}
process_one_work(worker, work);
}
}
static void set_pf_worker(bool val)
{
mutex_lock(&wq_pool_attach_mutex);
if (val)
current->flags |= PF_WQ_WORKER;
else
current->flags &= ~PF_WQ_WORKER;
mutex_unlock(&wq_pool_attach_mutex);
}
/**
* worker_thread - the worker thread function
* @__worker: self
*
* The worker thread function. All workers belong to a worker_pool -
* either a per-cpu one or dynamic unbound one. These workers process all
* work items regardless of their specific target workqueue. The only
* exception is work items which belong to workqueues with a rescuer which
* will be explained in rescuer_thread().
*
* Return: 0
*/
static int worker_thread(void *__worker)
{
struct worker *worker = __worker;
struct worker_pool *pool = worker->pool;
/* tell the scheduler that this is a workqueue worker */
set_pf_worker(true);
woke_up:
raw_spin_lock_irq(&pool->lock);
/* am I supposed to die? */
if (unlikely(worker->flags & WORKER_DIE)) {
raw_spin_unlock_irq(&pool->lock);
set_pf_worker(false);
set_task_comm(worker->task, "kworker/dying");
ida_free(&pool->worker_ida, worker->id);
worker_detach_from_pool(worker);
WARN_ON_ONCE(!list_empty(&worker->entry));
kfree(worker);
return 0;
}
worker_leave_idle(worker);
recheck:
/* no more worker necessary? */
if (!need_more_worker(pool))
goto sleep;
/* do we need to manage? */
if (unlikely(!may_start_working(pool)) && manage_workers(worker))
goto recheck;
/*
* ->scheduled list can only be filled while a worker is
* preparing to process a work or actually processing it.
* Make sure nobody diddled with it while I was sleeping.
*/
WARN_ON_ONCE(!list_empty(&worker->scheduled));
/*
* Finish PREP stage. We're guaranteed to have at least one idle
* worker or that someone else has already assumed the manager
* role. This is where @worker starts participating in concurrency
* management if applicable and concurrency management is restored
* after being rebound. See rebind_workers() for details.
*/
worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
do {
struct work_struct *work =
list_first_entry(&pool->worklist,
struct work_struct, entry);
if (assign_work(work, worker, NULL))
process_scheduled_works(worker);
} while (keep_working(pool));
worker_set_flags(worker, WORKER_PREP);
sleep:
/*
* pool->lock is held and there's no work to process and no need to
* manage, sleep. Workers are woken up only while holding
* pool->lock or from local cpu, so setting the current state
* before releasing pool->lock is enough to prevent losing any
* event.
*/
worker_enter_idle(worker);
__set_current_state(TASK_IDLE);
raw_spin_unlock_irq(&pool->lock);
schedule();
goto woke_up;
}
/**
* rescuer_thread - the rescuer thread function
* @__rescuer: self
*
* Workqueue rescuer thread function. There's one rescuer for each
* workqueue which has WQ_MEM_RECLAIM set.
*
* Regular work processing on a pool may block trying to create a new
* worker which uses GFP_KERNEL allocation which has slight chance of
* developing into deadlock if some works currently on the same queue
* need to be processed to satisfy the GFP_KERNEL allocation. This is
* the problem rescuer solves.
*
* When such condition is possible, the pool summons rescuers of all
* workqueues which have works queued on the pool and let them process
* those works so that forward progress can be guaranteed.
*
* This should happen rarely.
*
* Return: 0
*/
static int rescuer_thread(void *__rescuer)
{
struct worker *rescuer = __rescuer;
struct workqueue_struct *wq = rescuer->rescue_wq;
bool should_stop;
set_user_nice(current, RESCUER_NICE_LEVEL);
/*
* Mark rescuer as worker too. As WORKER_PREP is never cleared, it
* doesn't participate in concurrency management.
*/
set_pf_worker(true);
repeat:
set_current_state(TASK_IDLE);
/*
* By the time the rescuer is requested to stop, the workqueue
* shouldn't have any work pending, but @wq->maydays may still have
* pwq(s) queued. This can happen by non-rescuer workers consuming
* all the work items before the rescuer got to them. Go through
* @wq->maydays processing before acting on should_stop so that the
* list is always empty on exit.
*/
should_stop = kthread_should_stop();
/* see whether any pwq is asking for help */
raw_spin_lock_irq(&wq_mayday_lock);
while (!list_empty(&wq->maydays)) {
struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
struct pool_workqueue, mayday_node);
struct worker_pool *pool = pwq->pool;
struct work_struct *work, *n;
__set_current_state(TASK_RUNNING);
list_del_init(&pwq->mayday_node);
raw_spin_unlock_irq(&wq_mayday_lock);
worker_attach_to_pool(rescuer, pool);
raw_spin_lock_irq(&pool->lock);
/*
* Slurp in all works issued via this workqueue and
* process'em.
*/
WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
list_for_each_entry_safe(work, n, &pool->worklist, entry) {
if (get_work_pwq(work) == pwq &&
assign_work(work, rescuer, &n))
pwq->stats[PWQ_STAT_RESCUED]++;
}
if (!list_empty(&rescuer->scheduled)) {
process_scheduled_works(rescuer);
/*
* The above execution of rescued work items could
* have created more to rescue through
* pwq_activate_first_inactive() or chained
* queueing. Let's put @pwq back on mayday list so
* that such back-to-back work items, which may be
* being used to relieve memory pressure, don't
* incur MAYDAY_INTERVAL delay inbetween.
*/
if (pwq->nr_active && need_to_create_worker(pool)) {
raw_spin_lock(&wq_mayday_lock);
/*
* Queue iff we aren't racing destruction
* and somebody else hasn't queued it already.
*/
if (wq->rescuer && list_empty(&pwq->mayday_node)) {
get_pwq(pwq);
list_add_tail(&pwq->mayday_node, &wq->maydays);
}
raw_spin_unlock(&wq_mayday_lock);
}
}
/*
* Put the reference grabbed by send_mayday(). @pool won't
* go away while we're still attached to it.
*/
put_pwq(pwq);
/*
* Leave this pool. Notify regular workers; otherwise, we end up
* with 0 concurrency and stalling the execution.
*/
kick_pool(pool);
raw_spin_unlock_irq(&pool->lock);
worker_detach_from_pool(rescuer);
raw_spin_lock_irq(&wq_mayday_lock);
}
raw_spin_unlock_irq(&wq_mayday_lock);
if (should_stop) {
__set_current_state(TASK_RUNNING);
set_pf_worker(false);
return 0;
}
/* rescuers should never participate in concurrency management */
WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
schedule();
goto repeat;
}
static void bh_worker(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
int nr_restarts = BH_WORKER_RESTARTS;
unsigned long end = jiffies + BH_WORKER_JIFFIES;
raw_spin_lock_irq(&pool->lock);
worker_leave_idle(worker);
/*
* This function follows the structure of worker_thread(). See there for
* explanations on each step.
*/
if (!need_more_worker(pool))
goto done;
WARN_ON_ONCE(!list_empty(&worker->scheduled));
worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
do {
struct work_struct *work =
list_first_entry(&pool->worklist,
struct work_struct, entry);
if (assign_work(work, worker, NULL))
process_scheduled_works(worker);
} while (keep_working(pool) &&
--nr_restarts && time_before(jiffies, end));
worker_set_flags(worker, WORKER_PREP);
done:
worker_enter_idle(worker);
kick_pool(pool);
raw_spin_unlock_irq(&pool->lock);
}
/*
* TODO: Convert all tasklet users to workqueue and use softirq directly.
*
* This is currently called from tasklet[_hi]action() and thus is also called
* whenever there are tasklets to run. Let's do an early exit if there's nothing
* queued. Once conversion from tasklet is complete, the need_more_worker() test
* can be dropped.
*
* After full conversion, we'll add worker->softirq_action, directly use the
* softirq action and obtain the worker pointer from the softirq_action pointer.
*/
void workqueue_softirq_action(bool highpri)
{
struct worker_pool *pool =
&per_cpu(bh_worker_pools, smp_processor_id())[highpri];
if (need_more_worker(pool))
bh_worker(list_first_entry(&pool->workers, struct worker, node));
}
struct wq_drain_dead_softirq_work {
struct work_struct work;
struct worker_pool *pool;
struct completion done;
};
static void drain_dead_softirq_workfn(struct work_struct *work)
{
struct wq_drain_dead_softirq_work *dead_work =
container_of(work, struct wq_drain_dead_softirq_work, work);
struct worker_pool *pool = dead_work->pool;
bool repeat;
/*
* @pool's CPU is dead and we want to execute its still pending work
* items from this BH work item which is running on a different CPU. As
* its CPU is dead, @pool can't be kicked and, as work execution path
* will be nested, a lockdep annotation needs to be suppressed. Mark
* @pool with %POOL_BH_DRAINING for the special treatments.
*/
raw_spin_lock_irq(&pool->lock);
pool->flags |= POOL_BH_DRAINING;
raw_spin_unlock_irq(&pool->lock);
bh_worker(list_first_entry(&pool->workers, struct worker, node));
raw_spin_lock_irq(&pool->lock);
pool->flags &= ~POOL_BH_DRAINING;
repeat = need_more_worker(pool);
raw_spin_unlock_irq(&pool->lock);
/*
* bh_worker() might hit consecutive execution limit and bail. If there
* still are pending work items, reschedule self and return so that we
* don't hog this CPU's BH.
*/
if (repeat) {
if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
queue_work(system_bh_highpri_wq, work);
else
queue_work(system_bh_wq, work);
} else {
complete(&dead_work->done);
}
}
/*
* @cpu is dead. Drain the remaining BH work items on the current CPU. It's
* possible to allocate dead_work per CPU and avoid flushing. However, then we
* have to worry about draining overlapping with CPU coming back online or
* nesting (one CPU's dead_work queued on another CPU which is also dead and so
* on). Let's keep it simple and drain them synchronously. These are BH work
* items which shouldn't be requeued on the same pool. Shouldn't take long.
*/
void workqueue_softirq_dead(unsigned int cpu)
{
int i;
for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i];
struct wq_drain_dead_softirq_work dead_work;
if (!need_more_worker(pool))
continue;
INIT_WORK(&dead_work.work, drain_dead_softirq_workfn);
dead_work.pool = pool;
init_completion(&dead_work.done);
if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
queue_work(system_bh_highpri_wq, &dead_work.work);
else
queue_work(system_bh_wq, &dead_work.work);
wait_for_completion(&dead_work.done);
}
}
/**
* check_flush_dependency - check for flush dependency sanity
* @target_wq: workqueue being flushed
* @target_work: work item being flushed (NULL for workqueue flushes)
*
* %current is trying to flush the whole @target_wq or @target_work on it.
* If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
* reclaiming memory or running on a workqueue which doesn't have
* %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
* a deadlock.
*/
static void check_flush_dependency(struct workqueue_struct *target_wq,
struct work_struct *target_work)
{
work_func_t target_func = target_work ? target_work->func : NULL;
struct worker *worker;
if (target_wq->flags & WQ_MEM_RECLAIM)
return;
worker = current_wq_worker(); WARN_ONCE(current->flags & PF_MEMALLOC,
"workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
current->pid, current->comm, target_wq->name, target_func);
WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
(WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
"workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
worker->current_pwq->wq->name, worker->current_func,
target_wq->name, target_func);
}
struct wq_barrier {
struct work_struct work;
struct completion done;
struct task_struct *task; /* purely informational */
};
static void wq_barrier_func(struct work_struct *work)
{
struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
complete(&barr->done);
}
/**
* insert_wq_barrier - insert a barrier work
* @pwq: pwq to insert barrier into
* @barr: wq_barrier to insert
* @target: target work to attach @barr to
* @worker: worker currently executing @target, NULL if @target is not executing
*
* @barr is linked to @target such that @barr is completed only after
* @target finishes execution. Please note that the ordering
* guarantee is observed only with respect to @target and on the local
* cpu.
*
* Currently, a queued barrier can't be canceled. This is because
* try_to_grab_pending() can't determine whether the work to be
* grabbed is at the head of the queue and thus can't clear LINKED
* flag of the previous work while there must be a valid next work
* after a work with LINKED flag set.
*
* Note that when @worker is non-NULL, @target may be modified
* underneath us, so we can't reliably determine pwq from @target.
*
* CONTEXT:
* raw_spin_lock_irq(pool->lock).
*/
static void insert_wq_barrier(struct pool_workqueue *pwq,
struct wq_barrier *barr,
struct work_struct *target, struct worker *worker)
{
static __maybe_unused struct lock_class_key bh_key, thr_key;
unsigned int work_flags = 0;
unsigned int work_color;
struct list_head *head;
/*
* debugobject calls are safe here even with pool->lock locked
* as we know for sure that this will not trigger any of the
* checks and call back into the fixup functions where we
* might deadlock.
*
* BH and threaded workqueues need separate lockdep keys to avoid
* spuriously triggering "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W}
* usage".
*/
INIT_WORK_ONSTACK_KEY(&barr->work, wq_barrier_func,
(pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
init_completion_map(&barr->done, &target->lockdep_map);
barr->task = current;
/* The barrier work item does not participate in nr_active. */
work_flags |= WORK_STRUCT_INACTIVE;
/*
* If @target is currently being executed, schedule the
* barrier to the worker; otherwise, put it after @target.
*/
if (worker) {
head = worker->scheduled.next;
work_color = worker->current_color;
} else {
unsigned long *bits = work_data_bits(target);
head = target->entry.next;
/* there can already be other linked works, inherit and set */
work_flags |= *bits & WORK_STRUCT_LINKED;
work_color = get_work_color(*bits);
__set_bit(WORK_STRUCT_LINKED_BIT, bits);
}
pwq->nr_in_flight[work_color]++;
work_flags |= work_color_to_flags(work_color);
insert_work(pwq, &barr->work, head, work_flags);
}
/**
* flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
* @wq: workqueue being flushed
* @flush_color: new flush color, < 0 for no-op
* @work_color: new work color, < 0 for no-op
*
* Prepare pwqs for workqueue flushing.
*
* If @flush_color is non-negative, flush_color on all pwqs should be
* -1. If no pwq has in-flight commands at the specified color, all
* pwq->flush_color's stay at -1 and %false is returned. If any pwq
* has in flight commands, its pwq->flush_color is set to
* @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
* wakeup logic is armed and %true is returned.
*
* The caller should have initialized @wq->first_flusher prior to
* calling this function with non-negative @flush_color. If
* @flush_color is negative, no flush color update is done and %false
* is returned.
*
* If @work_color is non-negative, all pwqs should have the same
* work_color which is previous to @work_color and all will be
* advanced to @work_color.
*
* CONTEXT:
* mutex_lock(wq->mutex).
*
* Return:
* %true if @flush_color >= 0 and there's something to flush. %false
* otherwise.
*/
static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
int flush_color, int work_color)
{
bool wait = false;
struct pool_workqueue *pwq;
if (flush_color >= 0) {
WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); atomic_set(&wq->nr_pwqs_to_flush, 1);
}
for_each_pwq(pwq, wq) { struct worker_pool *pool = pwq->pool;
raw_spin_lock_irq(&pool->lock);
if (flush_color >= 0) {
WARN_ON_ONCE(pwq->flush_color != -1); if (pwq->nr_in_flight[flush_color]) { pwq->flush_color = flush_color;
atomic_inc(&wq->nr_pwqs_to_flush);
wait = true;
}
}
if (work_color >= 0) { WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); pwq->work_color = work_color;
}
raw_spin_unlock_irq(&pool->lock);
}
if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) complete(&wq->first_flusher->done); return wait;
}
static void touch_wq_lockdep_map(struct workqueue_struct *wq)
{
#ifdef CONFIG_LOCKDEP
if (wq->flags & WQ_BH) local_bh_disable(); lock_map_acquire(&wq->lockdep_map);
lock_map_release(&wq->lockdep_map);
if (wq->flags & WQ_BH)
local_bh_enable();
#endif
}
static void touch_work_lockdep_map(struct work_struct *work,
struct workqueue_struct *wq)
{
#ifdef CONFIG_LOCKDEP
if (wq->flags & WQ_BH)
local_bh_disable(); lock_map_acquire(&work->lockdep_map);
lock_map_release(&work->lockdep_map);
if (wq->flags & WQ_BH)
local_bh_enable();
#endif
}
/**
* __flush_workqueue - ensure that any scheduled work has run to completion.
* @wq: workqueue to flush
*
* This function sleeps until all work items which were queued on entry
* have finished execution, but it is not livelocked by new incoming ones.
*/
void __flush_workqueue(struct workqueue_struct *wq)
{
struct wq_flusher this_flusher = {
.list = LIST_HEAD_INIT(this_flusher.list),
.flush_color = -1,
.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
};
int next_color;
if (WARN_ON(!wq_online))
return;
touch_wq_lockdep_map(wq);
mutex_lock(&wq->mutex);
/*
* Start-to-wait phase
*/
next_color = work_next_color(wq->work_color);
if (next_color != wq->flush_color) {
/*
* Color space is not full. The current work_color
* becomes our flush_color and work_color is advanced
* by one.
*/
WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); this_flusher.flush_color = wq->work_color;
wq->work_color = next_color;
if (!wq->first_flusher) {
/* no flush in progress, become the first flusher */
WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); wq->first_flusher = &this_flusher;
if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
wq->work_color)) {
/* nothing to flush, done */
wq->flush_color = next_color;
wq->first_flusher = NULL;
goto out_unlock;
}
} else {
/* wait in queue */
WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); list_add_tail(&this_flusher.list, &wq->flusher_queue); flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
}
} else {
/*
* Oops, color space is full, wait on overflow queue.
* The next flush completion will assign us
* flush_color and transfer to flusher_queue.
*/
list_add_tail(&this_flusher.list, &wq->flusher_overflow);
}
check_flush_dependency(wq, NULL);
mutex_unlock(&wq->mutex);
wait_for_completion(&this_flusher.done);
/*
* Wake-up-and-cascade phase
*
* First flushers are responsible for cascading flushes and
* handling overflow. Non-first flushers can simply return.
*/
if (READ_ONCE(wq->first_flusher) != &this_flusher)
return;
mutex_lock(&wq->mutex);
/* we might have raced, check again with mutex held */
if (wq->first_flusher != &this_flusher)
goto out_unlock;
WRITE_ONCE(wq->first_flusher, NULL); WARN_ON_ONCE(!list_empty(&this_flusher.list)); WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
while (true) {
struct wq_flusher *next, *tmp;
/* complete all the flushers sharing the current flush color */
list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { if (next->flush_color != wq->flush_color)
break;
list_del_init(&next->list);
complete(&next->done);
}
WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
wq->flush_color != work_next_color(wq->work_color));
/* this flush_color is finished, advance by one */
wq->flush_color = work_next_color(wq->flush_color);
/* one color has been freed, handle overflow queue */
if (!list_empty(&wq->flusher_overflow)) {
/*
* Assign the same color to all overflowed
* flushers, advance work_color and append to
* flusher_queue. This is the start-to-wait
* phase for these overflowed flushers.
*/
list_for_each_entry(tmp, &wq->flusher_overflow, list) tmp->flush_color = wq->work_color; wq->work_color = work_next_color(wq->work_color); list_splice_tail_init(&wq->flusher_overflow,
&wq->flusher_queue);
flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
}
if (list_empty(&wq->flusher_queue)) { WARN_ON_ONCE(wq->flush_color != wq->work_color);
break;
}
/*
* Need to flush more colors. Make the next flusher
* the new first flusher and arm pwqs.
*/
WARN_ON_ONCE(wq->flush_color == wq->work_color); WARN_ON_ONCE(wq->flush_color != next->flush_color); list_del_init(&next->list);
wq->first_flusher = next;
if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
break;
/*
* Meh... this color is already done, clear first
* flusher and repeat cascading.
*/
wq->first_flusher = NULL;
}
out_unlock:
mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL(__flush_workqueue);
/**
* drain_workqueue - drain a workqueue
* @wq: workqueue to drain
*
* Wait until the workqueue becomes empty. While draining is in progress,
* only chain queueing is allowed. IOW, only currently pending or running
* work items on @wq can queue further work items on it. @wq is flushed
* repeatedly until it becomes empty. The number of flushing is determined
* by the depth of chaining and should be relatively short. Whine if it
* takes too long.
*/
void drain_workqueue(struct workqueue_struct *wq)
{
unsigned int flush_cnt = 0;
struct pool_workqueue *pwq;
/*
* __queue_work() needs to test whether there are drainers, is much
* hotter than drain_workqueue() and already looks at @wq->flags.
* Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
*/
mutex_lock(&wq->mutex);
if (!wq->nr_drainers++)
wq->flags |= __WQ_DRAINING; mutex_unlock(&wq->mutex);
reflush:
__flush_workqueue(wq);
mutex_lock(&wq->mutex);
for_each_pwq(pwq, wq) {
bool drained;
raw_spin_lock_irq(&pwq->pool->lock); drained = pwq_is_empty(pwq); raw_spin_unlock_irq(&pwq->pool->lock);
if (drained)
continue;
if (++flush_cnt == 10 ||
(flush_cnt % 100 == 0 && flush_cnt <= 1000)) pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
wq->name, __func__, flush_cnt);
mutex_unlock(&wq->mutex);
goto reflush;
}
if (!--wq->nr_drainers) wq->flags &= ~__WQ_DRAINING; mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL_GPL(drain_workqueue);
static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
bool from_cancel)
{
struct worker *worker = NULL;
struct worker_pool *pool;
struct pool_workqueue *pwq;
struct workqueue_struct *wq;
might_sleep(); rcu_read_lock(); pool = get_work_pool(work);
if (!pool) {
rcu_read_unlock();
return false;
}
raw_spin_lock_irq(&pool->lock);
/* see the comment in try_to_grab_pending() with the same code */
pwq = get_work_pwq(work);
if (pwq) {
if (unlikely(pwq->pool != pool))
goto already_gone;
} else {
worker = find_worker_executing_work(pool, work); if (!worker)
goto already_gone;
pwq = worker->current_pwq;
}
wq = pwq->wq;
check_flush_dependency(wq, work);
insert_wq_barrier(pwq, barr, work, worker);
raw_spin_unlock_irq(&pool->lock);
touch_work_lockdep_map(work, wq);
/*
* Force a lock recursion deadlock when using flush_work() inside a
* single-threaded or rescuer equipped workqueue.
*
* For single threaded workqueues the deadlock happens when the work
* is after the work issuing the flush_work(). For rescuer equipped
* workqueues the deadlock happens when the rescuer stalls, blocking
* forward progress.
*/
if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer)) touch_wq_lockdep_map(wq); rcu_read_unlock();
return true;
already_gone:
raw_spin_unlock_irq(&pool->lock); rcu_read_unlock();
return false;
}
static bool __flush_work(struct work_struct *work, bool from_cancel)
{
struct wq_barrier barr; if (WARN_ON(!wq_online))
return false;
if (WARN_ON(!work->func))
return false;
if (start_flush_work(work, &barr, from_cancel)) {
wait_for_completion(&barr.done);
destroy_work_on_stack(&barr.work);
return true;
} else {
return false;
}
}
/**
* flush_work - wait for a work to finish executing the last queueing instance
* @work: the work to flush
*
* Wait until @work has finished execution. @work is guaranteed to be idle
* on return if it hasn't been requeued since flush started.
*
* Return:
* %true if flush_work() waited for the work to finish execution,
* %false if it was already idle.
*/
bool flush_work(struct work_struct *work)
{
return __flush_work(work, false);
}
EXPORT_SYMBOL_GPL(flush_work);
/**
* flush_delayed_work - wait for a dwork to finish executing the last queueing
* @dwork: the delayed work to flush
*
* Delayed timer is cancelled and the pending work is queued for
* immediate execution. Like flush_work(), this function only
* considers the last queueing instance of @dwork.
*
* Return:
* %true if flush_work() waited for the work to finish execution,
* %false if it was already idle.
*/
bool flush_delayed_work(struct delayed_work *dwork)
{
local_irq_disable(); if (del_timer_sync(&dwork->timer)) __queue_work(dwork->cpu, dwork->wq, &dwork->work); local_irq_enable();
return flush_work(&dwork->work);
}
EXPORT_SYMBOL(flush_delayed_work);
/**
* flush_rcu_work - wait for a rwork to finish executing the last queueing
* @rwork: the rcu work to flush
*
* Return:
* %true if flush_rcu_work() waited for the work to finish execution,
* %false if it was already idle.
*/
bool flush_rcu_work(struct rcu_work *rwork)
{
if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
rcu_barrier();
flush_work(&rwork->work);
return true;
} else {
return flush_work(&rwork->work);
}
}
EXPORT_SYMBOL(flush_rcu_work);
static bool __cancel_work(struct work_struct *work, u32 cflags)
{
unsigned long irq_flags;
int ret;
do {
ret = try_to_grab_pending(work, cflags, &irq_flags);
} while (unlikely(ret == -EAGAIN));
if (unlikely(ret < 0))
return false;
set_work_pool_and_clear_pending(work, get_work_pool_id(work), 0); local_irq_restore(irq_flags); return ret;
}
static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
{
unsigned long irq_flags;
bool ret;
/* claim @work and tell other tasks trying to grab @work to back off */
ret = work_grab_pending(work, cflags, &irq_flags); mark_work_canceling(work); local_irq_restore(irq_flags);
/*
* Skip __flush_work() during early boot when we know that @work isn't
* executing. This allows canceling during early boot.
*/
if (wq_online) __flush_work(work, true);
/*
* smp_mb() at the end of set_work_pool_and_clear_pending() is paired
* with prepare_to_wait() above so that either waitqueue_active() is
* visible here or !work_is_canceling() is visible there.
*/
set_work_pool_and_clear_pending(work, WORK_OFFQ_POOL_NONE, 0);
if (waitqueue_active(&wq_cancel_waitq))
__wake_up(&wq_cancel_waitq, TASK_NORMAL, 1, work); return ret;
}
/*
* See cancel_delayed_work()
*/
bool cancel_work(struct work_struct *work)
{
return __cancel_work(work, 0);
}
EXPORT_SYMBOL(cancel_work);
/**
* cancel_work_sync - cancel a work and wait for it to finish
* @work: the work to cancel
*
* Cancel @work and wait for its execution to finish. This function
* can be used even if the work re-queues itself or migrates to
* another workqueue. On return from this function, @work is
* guaranteed to be not pending or executing on any CPU.
*
* cancel_work_sync(&delayed_work->work) must not be used for
* delayed_work's. Use cancel_delayed_work_sync() instead.
*
* The caller must ensure that the workqueue on which @work was last
* queued can't be destroyed before this function returns.
*
* Return:
* %true if @work was pending, %false otherwise.
*/
bool cancel_work_sync(struct work_struct *work)
{
return __cancel_work_sync(work, 0);
}
EXPORT_SYMBOL_GPL(cancel_work_sync);
/**
* cancel_delayed_work - cancel a delayed work
* @dwork: delayed_work to cancel
*
* Kill off a pending delayed_work.
*
* Return: %true if @dwork was pending and canceled; %false if it wasn't
* pending.
*
* Note:
* The work callback function may still be running on return, unless
* it returns %true and the work doesn't re-arm itself. Explicitly flush or
* use cancel_delayed_work_sync() to wait on it.
*
* This function is safe to call from any context including IRQ handler.
*/
bool cancel_delayed_work(struct delayed_work *dwork)
{
return __cancel_work(&dwork->work, WORK_CANCEL_DELAYED);
}
EXPORT_SYMBOL(cancel_delayed_work);
/**
* cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
* @dwork: the delayed work cancel
*
* This is cancel_work_sync() for delayed works.
*
* Return:
* %true if @dwork was pending, %false otherwise.
*/
bool cancel_delayed_work_sync(struct delayed_work *dwork)
{
return __cancel_work_sync(&dwork->work, WORK_CANCEL_DELAYED);
}
EXPORT_SYMBOL(cancel_delayed_work_sync);
/**
* schedule_on_each_cpu - execute a function synchronously on each online CPU
* @func: the function to call
*
* schedule_on_each_cpu() executes @func on each online CPU using the
* system workqueue and blocks until all CPUs have completed.
* schedule_on_each_cpu() is very slow.
*
* Return:
* 0 on success, -errno on failure.
*/
int schedule_on_each_cpu(work_func_t func)
{
int cpu;
struct work_struct __percpu *works;
works = alloc_percpu(struct work_struct);
if (!works)
return -ENOMEM;
cpus_read_lock();
for_each_online_cpu(cpu) {
struct work_struct *work = per_cpu_ptr(works, cpu);
INIT_WORK(work, func);
schedule_work_on(cpu, work);
}
for_each_online_cpu(cpu)
flush_work(per_cpu_ptr(works, cpu));
cpus_read_unlock();
free_percpu(works);
return 0;
}
/**
* execute_in_process_context - reliably execute the routine with user context
* @fn: the function to execute
* @ew: guaranteed storage for the execute work structure (must
* be available when the work executes)
*
* Executes the function immediately if process context is available,
* otherwise schedules the function for delayed execution.
*
* Return: 0 - function was executed
* 1 - function was scheduled for execution
*/
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
{
if (!in_interrupt()) {
fn(&ew->work);
return 0;
}
INIT_WORK(&ew->work, fn);
schedule_work(&ew->work);
return 1;
}
EXPORT_SYMBOL_GPL(execute_in_process_context);
/**
* free_workqueue_attrs - free a workqueue_attrs
* @attrs: workqueue_attrs to free
*
* Undo alloc_workqueue_attrs().
*/
void free_workqueue_attrs(struct workqueue_attrs *attrs)
{
if (attrs) {
free_cpumask_var(attrs->cpumask);
free_cpumask_var(attrs->__pod_cpumask);
kfree(attrs);
}
}
/**
* alloc_workqueue_attrs - allocate a workqueue_attrs
*
* Allocate a new workqueue_attrs, initialize with default settings and
* return it.
*
* Return: The allocated new workqueue_attr on success. %NULL on failure.
*/
struct workqueue_attrs *alloc_workqueue_attrs(void)
{
struct workqueue_attrs *attrs;
attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
if (!attrs)
goto fail;
if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
goto fail;
if (!alloc_cpumask_var(&attrs->__pod_cpumask, GFP_KERNEL))
goto fail;
cpumask_copy(attrs->cpumask, cpu_possible_mask);
attrs->affn_scope = WQ_AFFN_DFL;
return attrs;
fail:
free_workqueue_attrs(attrs);
return NULL;
}
static void copy_workqueue_attrs(struct workqueue_attrs *to,
const struct workqueue_attrs *from)
{
to->nice = from->nice;
cpumask_copy(to->cpumask, from->cpumask);
cpumask_copy(to->__pod_cpumask, from->__pod_cpumask);
to->affn_strict = from->affn_strict;
/*
* Unlike hash and equality test, copying shouldn't ignore wq-only
* fields as copying is used for both pool and wq attrs. Instead,
* get_unbound_pool() explicitly clears the fields.
*/
to->affn_scope = from->affn_scope;
to->ordered = from->ordered;
}
/*
* Some attrs fields are workqueue-only. Clear them for worker_pool's. See the
* comments in 'struct workqueue_attrs' definition.
*/
static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
{
attrs->affn_scope = WQ_AFFN_NR_TYPES;
attrs->ordered = false;
}
/* hash value of the content of @attr */
static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
{
u32 hash = 0;
hash = jhash_1word(attrs->nice, hash);
hash = jhash(cpumask_bits(attrs->cpumask),
BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
hash = jhash(cpumask_bits(attrs->__pod_cpumask),
BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
hash = jhash_1word(attrs->affn_strict, hash);
return hash;
}
/* content equality test */
static bool wqattrs_equal(const struct workqueue_attrs *a,
const struct workqueue_attrs *b)
{
if (a->nice != b->nice)
return false;
if (!cpumask_equal(a->cpumask, b->cpumask))
return false;
if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
return false;
if (a->affn_strict != b->affn_strict)
return false;
return true;
}
/* Update @attrs with actually available CPUs */
static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs,
const cpumask_t *unbound_cpumask)
{
/*
* Calculate the effective CPU mask of @attrs given @unbound_cpumask. If
* @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to
* @unbound_cpumask.
*/
cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask);
if (unlikely(cpumask_empty(attrs->cpumask)))
cpumask_copy(attrs->cpumask, unbound_cpumask);
}
/* find wq_pod_type to use for @attrs */
static const struct wq_pod_type *
wqattrs_pod_type(const struct workqueue_attrs *attrs)
{
enum wq_affn_scope scope;
struct wq_pod_type *pt;
/* to synchronize access to wq_affn_dfl */
lockdep_assert_held(&wq_pool_mutex);
if (attrs->affn_scope == WQ_AFFN_DFL)
scope = wq_affn_dfl;
else
scope = attrs->affn_scope;
pt = &wq_pod_types[scope];
if (!WARN_ON_ONCE(attrs->affn_scope == WQ_AFFN_NR_TYPES) &&
likely(pt->nr_pods))
return pt;
/*
* Before workqueue_init_topology(), only SYSTEM is available which is
* initialized in workqueue_init_early().
*/
pt = &wq_pod_types[WQ_AFFN_SYSTEM];
BUG_ON(!pt->nr_pods);
return pt;
}
/**
* init_worker_pool - initialize a newly zalloc'd worker_pool
* @pool: worker_pool to initialize
*
* Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
*
* Return: 0 on success, -errno on failure. Even on failure, all fields
* inside @pool proper are initialized and put_unbound_pool() can be called
* on @pool safely to release it.
*/
static int init_worker_pool(struct worker_pool *pool)
{
raw_spin_lock_init(&pool->lock);
pool->id = -1;
pool->cpu = -1;
pool->node = NUMA_NO_NODE;
pool->flags |= POOL_DISASSOCIATED;
pool->watchdog_ts = jiffies;
INIT_LIST_HEAD(&pool->worklist);
INIT_LIST_HEAD(&pool->idle_list);
hash_init(pool->busy_hash);
timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
INIT_WORK(&pool->idle_cull_work, idle_cull_fn);
timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
INIT_LIST_HEAD(&pool->workers);
INIT_LIST_HEAD(&pool->dying_workers);
ida_init(&pool->worker_ida);
INIT_HLIST_NODE(&pool->hash_node);
pool->refcnt = 1;
/* shouldn't fail above this point */
pool->attrs = alloc_workqueue_attrs();
if (!pool->attrs)
return -ENOMEM;
wqattrs_clear_for_pool(pool->attrs);
return 0;
}
#ifdef CONFIG_LOCKDEP
static void wq_init_lockdep(struct workqueue_struct *wq)
{
char *lock_name;
lockdep_register_key(&wq->key);
lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
if (!lock_name)
lock_name = wq->name;
wq->lock_name = lock_name;
lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
}
static void wq_unregister_lockdep(struct workqueue_struct *wq)
{
lockdep_unregister_key(&wq->key);
}
static void wq_free_lockdep(struct workqueue_struct *wq)
{
if (wq->lock_name != wq->name)
kfree(wq->lock_name);
}
#else
static void wq_init_lockdep(struct workqueue_struct *wq)
{
}
static void wq_unregister_lockdep(struct workqueue_struct *wq)
{
}
static void wq_free_lockdep(struct workqueue_struct *wq)
{
}
#endif
static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
{
int node;
for_each_node(node) {
kfree(nna_ar[node]);
nna_ar[node] = NULL;
}
kfree(nna_ar[nr_node_ids]);
nna_ar[nr_node_ids] = NULL;
}
static void init_node_nr_active(struct wq_node_nr_active *nna)
{
nna->max = WQ_DFL_MIN_ACTIVE;
atomic_set(&nna->nr, 0);
raw_spin_lock_init(&nna->lock);
INIT_LIST_HEAD(&nna->pending_pwqs);
}
/*
* Each node's nr_active counter will be accessed mostly from its own node and
* should be allocated in the node.
*/
static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
{
struct wq_node_nr_active *nna;
int node;
for_each_node(node) {
nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
if (!nna)
goto err_free;
init_node_nr_active(nna);
nna_ar[node] = nna;
}
/* [nr_node_ids] is used as the fallback */
nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
if (!nna)
goto err_free;
init_node_nr_active(nna);
nna_ar[nr_node_ids] = nna;
return 0;
err_free:
free_node_nr_active(nna_ar);
return -ENOMEM;
}
static void rcu_free_wq(struct rcu_head *rcu)
{
struct workqueue_struct *wq =
container_of(rcu, struct workqueue_struct, rcu);
if (wq->flags & WQ_UNBOUND)
free_node_nr_active(wq->node_nr_active);
wq_free_lockdep(wq);
free_percpu(wq->cpu_pwq);
free_workqueue_attrs(wq->unbound_attrs);
kfree(wq);
}
static void rcu_free_pool(struct rcu_head *rcu)
{
struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
ida_destroy(&pool->worker_ida);
free_workqueue_attrs(pool->attrs);
kfree(pool);
}
/**
* put_unbound_pool - put a worker_pool
* @pool: worker_pool to put
*
* Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
* safe manner. get_unbound_pool() calls this function on its failure path
* and this function should be able to release pools which went through,
* successfully or not, init_worker_pool().
*
* Should be called with wq_pool_mutex held.
*/
static void put_unbound_pool(struct worker_pool *pool)
{
DECLARE_COMPLETION_ONSTACK(detach_completion);
struct worker *worker;
LIST_HEAD(cull_list);
lockdep_assert_held(&wq_pool_mutex);
if (--pool->refcnt)
return;
/* sanity checks */
if (WARN_ON(!(pool->cpu < 0)) ||
WARN_ON(!list_empty(&pool->worklist)))
return;
/* release id and unhash */
if (pool->id >= 0)
idr_remove(&worker_pool_idr, pool->id);
hash_del(&pool->hash_node);
/*
* Become the manager and destroy all workers. This prevents
* @pool's workers from blocking on attach_mutex. We're the last
* manager and @pool gets freed with the flag set.
*
* Having a concurrent manager is quite unlikely to happen as we can
* only get here with
* pwq->refcnt == pool->refcnt == 0
* which implies no work queued to the pool, which implies no worker can
* become the manager. However a worker could have taken the role of
* manager before the refcnts dropped to 0, since maybe_create_worker()
* drops pool->lock
*/
while (true) {
rcuwait_wait_event(&manager_wait,
!(pool->flags & POOL_MANAGER_ACTIVE),
TASK_UNINTERRUPTIBLE);
mutex_lock(&wq_pool_attach_mutex);
raw_spin_lock_irq(&pool->lock);
if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
pool->flags |= POOL_MANAGER_ACTIVE;
break;
}
raw_spin_unlock_irq(&pool->lock);
mutex_unlock(&wq_pool_attach_mutex);
}
while ((worker = first_idle_worker(pool)))
set_worker_dying(worker, &cull_list);
WARN_ON(pool->nr_workers || pool->nr_idle);
raw_spin_unlock_irq(&pool->lock);
wake_dying_workers(&cull_list);
if (!list_empty(&pool->workers) || !list_empty(&pool->dying_workers))
pool->detach_completion = &detach_completion;
mutex_unlock(&wq_pool_attach_mutex);
if (pool->detach_completion)
wait_for_completion(pool->detach_completion);
/* shut down the timers */
del_timer_sync(&pool->idle_timer);
cancel_work_sync(&pool->idle_cull_work);
del_timer_sync(&pool->mayday_timer);
/* RCU protected to allow dereferences from get_work_pool() */
call_rcu(&pool->rcu, rcu_free_pool);
}
/**
* get_unbound_pool - get a worker_pool with the specified attributes
* @attrs: the attributes of the worker_pool to get
*
* Obtain a worker_pool which has the same attributes as @attrs, bump the
* reference count and return it. If there already is a matching
* worker_pool, it will be used; otherwise, this function attempts to
* create a new one.
*
* Should be called with wq_pool_mutex held.
*
* Return: On success, a worker_pool with the same attributes as @attrs.
* On failure, %NULL.
*/
static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
{
struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA];
u32 hash = wqattrs_hash(attrs);
struct worker_pool *pool;
int pod, node = NUMA_NO_NODE;
lockdep_assert_held(&wq_pool_mutex);
/* do we already have a matching pool? */
hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
if (wqattrs_equal(pool->attrs, attrs)) {
pool->refcnt++;
return pool;
}
}
/* If __pod_cpumask is contained inside a NUMA pod, that's our node */
for (pod = 0; pod < pt->nr_pods; pod++) {
if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) {
node = pt->pod_node[pod];
break;
}
}
/* nope, create a new one */
pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
if (!pool || init_worker_pool(pool) < 0)
goto fail;
pool->node = node;
copy_workqueue_attrs(pool->attrs, attrs);
wqattrs_clear_for_pool(pool->attrs);
if (worker_pool_assign_id(pool) < 0)
goto fail;
/* create and start the initial worker */
if (wq_online && !create_worker(pool))
goto fail;
/* install */
hash_add(unbound_pool_hash, &pool->hash_node, hash);
return pool;
fail:
if (pool)
put_unbound_pool(pool);
return NULL;
}
static void rcu_free_pwq(struct rcu_head *rcu)
{
kmem_cache_free(pwq_cache,
container_of(rcu, struct pool_workqueue, rcu));
}
/*
* Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero
* refcnt and needs to be destroyed.
*/
static void pwq_release_workfn(struct kthread_work *work)
{
struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
release_work);
struct workqueue_struct *wq = pwq->wq;
struct worker_pool *pool = pwq->pool;
bool is_last = false;
/*
* When @pwq is not linked, it doesn't hold any reference to the
* @wq, and @wq is invalid to access.
*/
if (!list_empty(&pwq->pwqs_node)) {
mutex_lock(&wq->mutex);
list_del_rcu(&pwq->pwqs_node);
is_last = list_empty(&wq->pwqs);
/*
* For ordered workqueue with a plugged dfl_pwq, restart it now.
*/
if (!is_last && (wq->flags & __WQ_ORDERED))
unplug_oldest_pwq(wq);
mutex_unlock(&wq->mutex);
}
if (wq->flags & WQ_UNBOUND) {
mutex_lock(&wq_pool_mutex);
put_unbound_pool(pool);
mutex_unlock(&wq_pool_mutex);
}
if (!list_empty(&pwq->pending_node)) {
struct wq_node_nr_active *nna =
wq_node_nr_active(pwq->wq, pwq->pool->node);
raw_spin_lock_irq(&nna->lock);
list_del_init(&pwq->pending_node);
raw_spin_unlock_irq(&nna->lock);
}
call_rcu(&pwq->rcu, rcu_free_pwq);
/*
* If we're the last pwq going away, @wq is already dead and no one
* is gonna access it anymore. Schedule RCU free.
*/
if (is_last) {
wq_unregister_lockdep(wq);
call_rcu(&wq->rcu, rcu_free_wq);
}
}
/* initialize newly allocated @pwq which is associated with @wq and @pool */
static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
struct worker_pool *pool)
{
BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);
memset(pwq, 0, sizeof(*pwq));
pwq->pool = pool;
pwq->wq = wq;
pwq->flush_color = -1;
pwq->refcnt = 1;
INIT_LIST_HEAD(&pwq->inactive_works);
INIT_LIST_HEAD(&pwq->pending_node);
INIT_LIST_HEAD(&pwq->pwqs_node);
INIT_LIST_HEAD(&pwq->mayday_node);
kthread_init_work(&pwq->release_work, pwq_release_workfn);
}
/* sync @pwq with the current state of its associated wq and link it */
static void link_pwq(struct pool_workqueue *pwq)
{
struct workqueue_struct *wq = pwq->wq;
lockdep_assert_held(&wq->mutex);
/* may be called multiple times, ignore if already linked */
if (!list_empty(&pwq->pwqs_node))
return;
/* set the matching work_color */
pwq->work_color = wq->work_color;
/* link in @pwq */
list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
}
/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
const struct workqueue_attrs *attrs)
{
struct worker_pool *pool;
struct pool_workqueue *pwq;
lockdep_assert_held(&wq_pool_mutex);
pool = get_unbound_pool(attrs);
if (!pool)
return NULL;
pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
if (!pwq) {
put_unbound_pool(pool);
return NULL;
}
init_pwq(pwq, wq, pool);
return pwq;
}
/**
* wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
* @attrs: the wq_attrs of the default pwq of the target workqueue
* @cpu: the target CPU
* @cpu_going_down: if >= 0, the CPU to consider as offline
*
* Calculate the cpumask a workqueue with @attrs should use on @pod. If
* @cpu_going_down is >= 0, that cpu is considered offline during calculation.
* The result is stored in @attrs->__pod_cpumask.
*
* If pod affinity is not enabled, @attrs->cpumask is always used. If enabled
* and @pod has online CPUs requested by @attrs, the returned cpumask is the
* intersection of the possible CPUs of @pod and @attrs->cpumask.
*
* The caller is responsible for ensuring that the cpumask of @pod stays stable.
*/
static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu,
int cpu_going_down)
{
const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
int pod = pt->cpu_pod[cpu];
/* does @pod have any online CPUs @attrs wants? */
cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask);
cpumask_and(attrs->__pod_cpumask, attrs->__pod_cpumask, cpu_online_mask);
if (cpu_going_down >= 0)
cpumask_clear_cpu(cpu_going_down, attrs->__pod_cpumask);
if (cpumask_empty(attrs->__pod_cpumask)) {
cpumask_copy(attrs->__pod_cpumask, attrs->cpumask);
return;
}
/* yeap, return possible CPUs in @pod that @attrs wants */
cpumask_and(attrs->__pod_cpumask, attrs->cpumask, pt->pod_cpus[pod]);
if (cpumask_empty(attrs->__pod_cpumask))
pr_warn_once("WARNING: workqueue cpumask: online intersect > "
"possible intersect\n");
}
/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
int cpu, struct pool_workqueue *pwq)
{
struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
struct pool_workqueue *old_pwq;
lockdep_assert_held(&wq_pool_mutex);
lockdep_assert_held(&wq->mutex);
/* link_pwq() can handle duplicate calls */
link_pwq(pwq);
old_pwq = rcu_access_pointer(*slot);
rcu_assign_pointer(*slot, pwq);
return old_pwq;
}
/* context to store the prepared attrs & pwqs before applying */
struct apply_wqattrs_ctx {
struct workqueue_struct *wq; /* target workqueue */
struct workqueue_attrs *attrs; /* attrs to apply */
struct list_head list; /* queued for batching commit */
struct pool_workqueue *dfl_pwq;
struct pool_workqueue *pwq_tbl[];
};
/* free the resources after success or abort */
static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
{
if (ctx) {
int cpu;
for_each_possible_cpu(cpu)
put_pwq_unlocked(ctx->pwq_tbl[cpu]);
put_pwq_unlocked(ctx->dfl_pwq);
free_workqueue_attrs(ctx->attrs);
kfree(ctx);
}
}
/* allocate the attrs and pwqs for later installation */
static struct apply_wqattrs_ctx *
apply_wqattrs_prepare(struct workqueue_struct *wq,
const struct workqueue_attrs *attrs,
const cpumask_var_t unbound_cpumask)
{
struct apply_wqattrs_ctx *ctx;
struct workqueue_attrs *new_attrs;
int cpu;
lockdep_assert_held(&wq_pool_mutex);
if (WARN_ON(attrs->affn_scope < 0 ||
attrs->affn_scope >= WQ_AFFN_NR_TYPES))
return ERR_PTR(-EINVAL);
ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_cpu_ids), GFP_KERNEL);
new_attrs = alloc_workqueue_attrs();
if (!ctx || !new_attrs)
goto out_free;
/*
* If something goes wrong during CPU up/down, we'll fall back to
* the default pwq covering whole @attrs->cpumask. Always create
* it even if we don't use it immediately.
*/
copy_workqueue_attrs(new_attrs, attrs);
wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
if (!ctx->dfl_pwq)
goto out_free;
for_each_possible_cpu(cpu) {
if (new_attrs->ordered) {
ctx->dfl_pwq->refcnt++;
ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
} else {
wq_calc_pod_cpumask(new_attrs, cpu, -1);
ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
if (!ctx->pwq_tbl[cpu])
goto out_free;
}
}
/* save the user configured attrs and sanitize it. */
copy_workqueue_attrs(new_attrs, attrs);
cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
ctx->attrs = new_attrs;
/*
* For initialized ordered workqueues, there should only be one pwq
* (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution
* of newly queued work items until execution of older work items in
* the old pwq's have completed.
*/
if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))
ctx->dfl_pwq->plugged = true;
ctx->wq = wq;
return ctx;
out_free:
free_workqueue_attrs(new_attrs);
apply_wqattrs_cleanup(ctx);
return ERR_PTR(-ENOMEM);
}
/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
{
int cpu;
/* all pwqs have been created successfully, let's install'em */
mutex_lock(&ctx->wq->mutex);
copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
/* save the previous pwqs and install the new ones */
for_each_possible_cpu(cpu)
ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
ctx->pwq_tbl[cpu]);
ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
/* update node_nr_active->max */
wq_update_node_max_active(ctx->wq, -1);
/* rescuer needs to respect wq cpumask changes */
if (ctx->wq->rescuer)
set_cpus_allowed_ptr(ctx->wq->rescuer->task,
unbound_effective_cpumask(ctx->wq));
mutex_unlock(&ctx->wq->mutex);
}
static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
const struct workqueue_attrs *attrs)
{
struct apply_wqattrs_ctx *ctx;
/* only unbound workqueues can change attributes */
if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
return -EINVAL;
ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
if (IS_ERR(ctx))
return PTR_ERR(ctx);
/* the ctx has been prepared successfully, let's commit it */
apply_wqattrs_commit(ctx);
apply_wqattrs_cleanup(ctx);
return 0;
}
/**
* apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
* @wq: the target workqueue
* @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
*
* Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
* a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that
* work items are affine to the pod it was issued on. Older pwqs are released as
* in-flight work items finish. Note that a work item which repeatedly requeues
* itself back-to-back will stay on its current pwq.
*
* Performs GFP_KERNEL allocations.
*
* Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
*
* Return: 0 on success and -errno on failure.
*/
int apply_workqueue_attrs(struct workqueue_struct *wq,
const struct workqueue_attrs *attrs)
{
int ret;
lockdep_assert_cpus_held();
mutex_lock(&wq_pool_mutex);
ret = apply_workqueue_attrs_locked(wq, attrs);
mutex_unlock(&wq_pool_mutex);
return ret;
}
/**
* wq_update_pod - update pod affinity of a wq for CPU hot[un]plug
* @wq: the target workqueue
* @cpu: the CPU to update pool association for
* @hotplug_cpu: the CPU coming up or going down
* @online: whether @cpu is coming up or going down
*
* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update pod affinity of
* @wq accordingly.
*
*
* If pod affinity can't be adjusted due to memory allocation failure, it falls
* back to @wq->dfl_pwq which may not be optimal but is always correct.
*
* Note that when the last allowed CPU of a pod goes offline for a workqueue
* with a cpumask spanning multiple pods, the workers which were already
* executing the work items for the workqueue will lose their CPU affinity and
* may execute on any CPU. This is similar to how per-cpu workqueues behave on
* CPU_DOWN. If a workqueue user wants strict affinity, it's the user's
* responsibility to flush the work item from CPU_DOWN_PREPARE.
*/
static void wq_update_pod(struct workqueue_struct *wq, int cpu,
int hotplug_cpu, bool online)
{
int off_cpu = online ? -1 : hotplug_cpu;
struct pool_workqueue *old_pwq = NULL, *pwq;
struct workqueue_attrs *target_attrs;
lockdep_assert_held(&wq_pool_mutex);
if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
return;
/*
* We don't wanna alloc/free wq_attrs for each wq for each CPU.
* Let's use a preallocated one. The following buf is protected by
* CPU hotplug exclusion.
*/
target_attrs = wq_update_pod_attrs_buf;
copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask);
/* nothing to do if the target cpumask matches the current pwq */
wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);
if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
return;
/* create a new pwq */
pwq = alloc_unbound_pwq(wq, target_attrs);
if (!pwq) {
pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
wq->name);
goto use_dfl_pwq;
}
/* Install the new pwq. */
mutex_lock(&wq->mutex);
old_pwq = install_unbound_pwq(wq, cpu, pwq);
goto out_unlock;
use_dfl_pwq:
mutex_lock(&wq->mutex);
pwq = unbound_pwq(wq, -1);
raw_spin_lock_irq(&pwq->pool->lock);
get_pwq(pwq);
raw_spin_unlock_irq(&pwq->pool->lock);
old_pwq = install_unbound_pwq(wq, cpu, pwq);
out_unlock:
mutex_unlock(&wq->mutex);
put_pwq_unlocked(old_pwq);
}
static int alloc_and_link_pwqs(struct workqueue_struct *wq)
{
bool highpri = wq->flags & WQ_HIGHPRI;
int cpu, ret;
wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
if (!wq->cpu_pwq)
goto enomem;
if (!(wq->flags & WQ_UNBOUND)) {
for_each_possible_cpu(cpu) {
struct pool_workqueue **pwq_p;
struct worker_pool __percpu *pools;
struct worker_pool *pool;
if (wq->flags & WQ_BH)
pools = bh_worker_pools;
else
pools = cpu_worker_pools;
pool = &(per_cpu_ptr(pools, cpu)[highpri]);
pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);
*pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
pool->node);
if (!*pwq_p)
goto enomem;
init_pwq(*pwq_p, wq, pool);
mutex_lock(&wq->mutex);
link_pwq(*pwq_p);
mutex_unlock(&wq->mutex);
}
return 0;
}
cpus_read_lock();
if (wq->flags & __WQ_ORDERED) {
struct pool_workqueue *dfl_pwq;
ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
/* there should only be single pwq for ordering guarantee */
dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
wq->pwqs.prev != &dfl_pwq->pwqs_node),
"ordering guarantee broken for workqueue %s\n", wq->name);
} else {
ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
}
cpus_read_unlock();
/* for unbound pwq, flush the pwq_release_worker ensures that the
* pwq_release_workfn() completes before calling kfree(wq).
*/
if (ret)
kthread_flush_worker(pwq_release_worker);
return ret;
enomem:
if (wq->cpu_pwq) {
for_each_possible_cpu(cpu) {
struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
if (pwq)
kmem_cache_free(pwq_cache, pwq);
}
free_percpu(wq->cpu_pwq);
wq->cpu_pwq = NULL;
}
return -ENOMEM;
}
static int wq_clamp_max_active(int max_active, unsigned int flags,
const char *name)
{
if (max_active < 1 || max_active > WQ_MAX_ACTIVE)
pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
max_active, name, 1, WQ_MAX_ACTIVE);
return clamp_val(max_active, 1, WQ_MAX_ACTIVE);
}
/*
* Workqueues which may be used during memory reclaim should have a rescuer
* to guarantee forward progress.
*/
static int init_rescuer(struct workqueue_struct *wq)
{
struct worker *rescuer;
int ret;
if (!(wq->flags & WQ_MEM_RECLAIM))
return 0;
rescuer = alloc_worker(NUMA_NO_NODE);
if (!rescuer) {
pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
wq->name);
return -ENOMEM;
}
rescuer->rescue_wq = wq;
rescuer->task = kthread_create(rescuer_thread, rescuer, "kworker/R-%s", wq->name);
if (IS_ERR(rescuer->task)) {
ret = PTR_ERR(rescuer->task);
pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
wq->name, ERR_PTR(ret));
kfree(rescuer);
return ret;
}
wq->rescuer = rescuer;
if (wq->flags & WQ_UNBOUND)
kthread_bind_mask(rescuer->task, wq_unbound_cpumask);
else
kthread_bind_mask(rescuer->task, cpu_possible_mask);
wake_up_process(rescuer->task);
return 0;
}
/**
* wq_adjust_max_active - update a wq's max_active to the current setting
* @wq: target workqueue
*
* If @wq isn't freezing, set @wq->max_active to the saved_max_active and
* activate inactive work items accordingly. If @wq is freezing, clear
* @wq->max_active to zero.
*/
static void wq_adjust_max_active(struct workqueue_struct *wq)
{
bool activated;
int new_max, new_min;
lockdep_assert_held(&wq->mutex);
if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
new_max = 0;
new_min = 0;
} else {
new_max = wq->saved_max_active;
new_min = wq->saved_min_active;
}
if (wq->max_active == new_max && wq->min_active == new_min)
return;
/*
* Update @wq->max/min_active and then kick inactive work items if more
* active work items are allowed. This doesn't break work item ordering
* because new work items are always queued behind existing inactive
* work items if there are any.
*/
WRITE_ONCE(wq->max_active, new_max);
WRITE_ONCE(wq->min_active, new_min);
if (wq->flags & WQ_UNBOUND)
wq_update_node_max_active(wq, -1);
if (new_max == 0)
return;
/*
* Round-robin through pwq's activating the first inactive work item
* until max_active is filled.
*/
do {
struct pool_workqueue *pwq;
activated = false;
for_each_pwq(pwq, wq) {
unsigned long irq_flags;
/* can be called during early boot w/ irq disabled */
raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
if (pwq_activate_first_inactive(pwq, true)) {
activated = true;
kick_pool(pwq->pool);
}
raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
}
} while (activated);
}
__printf(1, 4)
struct workqueue_struct *alloc_workqueue(const char *fmt,
unsigned int flags,
int max_active, ...)
{
va_list args;
struct workqueue_struct *wq;
size_t wq_size;
int name_len;
if (flags & WQ_BH) {
if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))
return NULL;
if (WARN_ON_ONCE(max_active))
return NULL;
}
/* see the comment above the definition of WQ_POWER_EFFICIENT */
if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
flags |= WQ_UNBOUND;
/* allocate wq and format name */
if (flags & WQ_UNBOUND)
wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
else
wq_size = sizeof(*wq);
wq = kzalloc(wq_size, GFP_KERNEL);
if (!wq)
return NULL;
if (flags & WQ_UNBOUND) {
wq->unbound_attrs = alloc_workqueue_attrs();
if (!wq->unbound_attrs)
goto err_free_wq;
}
va_start(args, max_active);
name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
va_end(args);
if (name_len >= WQ_NAME_LEN)
pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",
wq->name);
if (flags & WQ_BH) {
/*
* BH workqueues always share a single execution context per CPU
* and don't impose any max_active limit.
*/
max_active = INT_MAX;
} else {
max_active = max_active ?: WQ_DFL_ACTIVE;
max_active = wq_clamp_max_active(max_active, flags, wq->name);
}
/* init wq */
wq->flags = flags;
wq->max_active = max_active;
wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
wq->saved_max_active = wq->max_active;
wq->saved_min_active = wq->min_active;
mutex_init(&wq->mutex);
atomic_set(&wq->nr_pwqs_to_flush, 0);
INIT_LIST_HEAD(&wq->pwqs);
INIT_LIST_HEAD(&wq->flusher_queue);
INIT_LIST_HEAD(&wq->flusher_overflow);
INIT_LIST_HEAD(&wq->maydays);
wq_init_lockdep(wq);
INIT_LIST_HEAD(&wq->list);
if (flags & WQ_UNBOUND) {
if (alloc_node_nr_active(wq->node_nr_active) < 0)
goto err_unreg_lockdep;
}
if (alloc_and_link_pwqs(wq) < 0)
goto err_free_node_nr_active;
if (wq_online && init_rescuer(wq) < 0)
goto err_destroy;
if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
goto err_destroy;
/*
* wq_pool_mutex protects global freeze state and workqueues list.
* Grab it, adjust max_active and add the new @wq to workqueues
* list.
*/
mutex_lock(&wq_pool_mutex);
mutex_lock(&wq->mutex);
wq_adjust_max_active(wq);
mutex_unlock(&wq->mutex);
list_add_tail_rcu(&wq->list, &workqueues);
mutex_unlock(&wq_pool_mutex);
return wq;
err_free_node_nr_active:
if (wq->flags & WQ_UNBOUND)
free_node_nr_active(wq->node_nr_active);
err_unreg_lockdep:
wq_unregister_lockdep(wq);
wq_free_lockdep(wq);
err_free_wq:
free_workqueue_attrs(wq->unbound_attrs);
kfree(wq);
return NULL;
err_destroy:
destroy_workqueue(wq);
return NULL;
}
EXPORT_SYMBOL_GPL(alloc_workqueue);
static bool pwq_busy(struct pool_workqueue *pwq)
{
int i;
for (i = 0; i < WORK_NR_COLORS; i++) if (pwq->nr_in_flight[i])
return true;
if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
return true;
if (!pwq_is_empty(pwq))
return true;
return false;
}
/**
* destroy_workqueue - safely terminate a workqueue
* @wq: target workqueue
*
* Safely destroy a workqueue. All work currently pending will be done first.
*/
void destroy_workqueue(struct workqueue_struct *wq)
{
struct pool_workqueue *pwq;
int cpu;
/*
* Remove it from sysfs first so that sanity check failure doesn't
* lead to sysfs name conflicts.
*/
workqueue_sysfs_unregister(wq);
/* mark the workqueue destruction is in progress */
mutex_lock(&wq->mutex);
wq->flags |= __WQ_DESTROYING;
mutex_unlock(&wq->mutex);
/* drain it before proceeding with destruction */
drain_workqueue(wq);
/* kill rescuer, if sanity checks fail, leave it w/o rescuer */
if (wq->rescuer) {
struct worker *rescuer = wq->rescuer;
/* this prevents new queueing */
raw_spin_lock_irq(&wq_mayday_lock);
wq->rescuer = NULL;
raw_spin_unlock_irq(&wq_mayday_lock);
/* rescuer will empty maydays list before exiting */
kthread_stop(rescuer->task);
kfree(rescuer);
}
/*
* Sanity checks - grab all the locks so that we wait for all
* in-flight operations which may do put_pwq().
*/
mutex_lock(&wq_pool_mutex);
mutex_lock(&wq->mutex);
for_each_pwq(pwq, wq) {
raw_spin_lock_irq(&pwq->pool->lock); if (WARN_ON(pwq_busy(pwq))) {
pr_warn("%s: %s has the following busy pwq\n",
__func__, wq->name);
show_pwq(pwq);
raw_spin_unlock_irq(&pwq->pool->lock);
mutex_unlock(&wq->mutex);
mutex_unlock(&wq_pool_mutex);
show_one_workqueue(wq);
return;
}
raw_spin_unlock_irq(&pwq->pool->lock);
}
mutex_unlock(&wq->mutex);
/*
* wq list is used to freeze wq, remove from list after
* flushing is complete in case freeze races us.
*/
list_del_rcu(&wq->list);
mutex_unlock(&wq_pool_mutex);
/*
* We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq
* to put the base refs. @wq will be auto-destroyed from the last
* pwq_put. RCU read lock prevents @wq from going away from under us.
*/
rcu_read_lock(); for_each_possible_cpu(cpu) { put_pwq_unlocked(unbound_pwq(wq, cpu)); RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
}
put_pwq_unlocked(unbound_pwq(wq, -1)); RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL); rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(destroy_workqueue);
/**
* workqueue_set_max_active - adjust max_active of a workqueue
* @wq: target workqueue
* @max_active: new max_active value.
*
* Set max_active of @wq to @max_active. See the alloc_workqueue() function
* comment.
*
* CONTEXT:
* Don't call from IRQ context.
*/
void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
{
/* max_active doesn't mean anything for BH workqueues */
if (WARN_ON(wq->flags & WQ_BH))
return;
/* disallow meddling with max_active for ordered workqueues */
if (WARN_ON(wq->flags & __WQ_ORDERED))
return;
max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
mutex_lock(&wq->mutex);
wq->saved_max_active = max_active;
if (wq->flags & WQ_UNBOUND)
wq->saved_min_active = min(wq->saved_min_active, max_active);
wq_adjust_max_active(wq);
mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL_GPL(workqueue_set_max_active);
/**
* workqueue_set_min_active - adjust min_active of an unbound workqueue
* @wq: target unbound workqueue
* @min_active: new min_active value
*
* Set min_active of an unbound workqueue. Unlike other types of workqueues, an
* unbound workqueue is not guaranteed to be able to process max_active
* interdependent work items. Instead, an unbound workqueue is guaranteed to be
* able to process min_active number of interdependent work items which is
* %WQ_DFL_MIN_ACTIVE by default.
*
* Use this function to adjust the min_active value between 0 and the current
* max_active.
*/
void workqueue_set_min_active(struct workqueue_struct *wq, int min_active)
{
/* min_active is only meaningful for non-ordered unbound workqueues */
if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) !=
WQ_UNBOUND))
return;
mutex_lock(&wq->mutex);
wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active);
wq_adjust_max_active(wq);
mutex_unlock(&wq->mutex);
}
/**
* current_work - retrieve %current task's work struct
*
* Determine if %current task is a workqueue worker and what it's working on.
* Useful to find out the context that the %current task is running in.
*
* Return: work struct if %current task is a workqueue worker, %NULL otherwise.
*/
struct work_struct *current_work(void)
{
struct worker *worker = current_wq_worker();
return worker ? worker->current_work : NULL;
}
EXPORT_SYMBOL(current_work);
/**
* current_is_workqueue_rescuer - is %current workqueue rescuer?
*
* Determine whether %current is a workqueue rescuer. Can be used from
* work functions to determine whether it's being run off the rescuer task.
*
* Return: %true if %current is a workqueue rescuer. %false otherwise.
*/
bool current_is_workqueue_rescuer(void)
{
struct worker *worker = current_wq_worker();
return worker && worker->rescue_wq;
}
/**
* workqueue_congested - test whether a workqueue is congested
* @cpu: CPU in question
* @wq: target workqueue
*
* Test whether @wq's cpu workqueue for @cpu is congested. There is
* no synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging.
*
* If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
*
* With the exception of ordered workqueues, all workqueues have per-cpu
* pool_workqueues, each with its own congested state. A workqueue being
* congested on one CPU doesn't mean that the workqueue is contested on any
* other CPUs.
*
* Return:
* %true if congested, %false otherwise.
*/
bool workqueue_congested(int cpu, struct workqueue_struct *wq)
{
struct pool_workqueue *pwq;
bool ret;
rcu_read_lock();
preempt_disable();
if (cpu == WORK_CPU_UNBOUND)
cpu = smp_processor_id();
pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
ret = !list_empty(&pwq->inactive_works);
preempt_enable();
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(workqueue_congested);
/**
* work_busy - test whether a work is currently pending or running
* @work: the work to be tested
*
* Test whether @work is currently pending or running. There is no
* synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging.
*
* Return:
* OR'd bitmask of WORK_BUSY_* bits.
*/
unsigned int work_busy(struct work_struct *work)
{
struct worker_pool *pool;
unsigned long irq_flags;
unsigned int ret = 0;
if (work_pending(work))
ret |= WORK_BUSY_PENDING;
rcu_read_lock();
pool = get_work_pool(work);
if (pool) {
raw_spin_lock_irqsave(&pool->lock, irq_flags);
if (find_worker_executing_work(pool, work))
ret |= WORK_BUSY_RUNNING;
raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
}
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(work_busy);
/**
* set_worker_desc - set description for the current work item
* @fmt: printf-style format string
* @...: arguments for the format string
*
* This function can be called by a running work function to describe what
* the work item is about. If the worker task gets dumped, this
* information will be printed out together to help debugging. The
* description can be at most WORKER_DESC_LEN including the trailing '\0'.
*/
void set_worker_desc(const char *fmt, ...)
{
struct worker *worker = current_wq_worker();
va_list args;
if (worker) {
va_start(args, fmt);
vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
va_end(args);
}
}
EXPORT_SYMBOL_GPL(set_worker_desc);
/**
* print_worker_info - print out worker information and description
* @log_lvl: the log level to use when printing
* @task: target task
*
* If @task is a worker and currently executing a work item, print out the
* name of the workqueue being serviced and worker description set with
* set_worker_desc() by the currently executing work item.
*
* This function can be safely called on any task as long as the
* task_struct itself is accessible. While safe, this function isn't
* synchronized and may print out mixups or garbages of limited length.
*/
void print_worker_info(const char *log_lvl, struct task_struct *task)
{
work_func_t *fn = NULL;
char name[WQ_NAME_LEN] = { };
char desc[WORKER_DESC_LEN] = { };
struct pool_workqueue *pwq = NULL;
struct workqueue_struct *wq = NULL;
struct worker *worker;
if (!(task->flags & PF_WQ_WORKER))
return;
/*
* This function is called without any synchronization and @task
* could be in any state. Be careful with dereferences.
*/
worker = kthread_probe_data(task);
/*
* Carefully copy the associated workqueue's workfn, name and desc.
* Keep the original last '\0' in case the original is garbage.
*/
copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
if (fn || name[0] || desc[0]) {
printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
if (strcmp(name, desc))
pr_cont(" (%s)", desc);
pr_cont("\n");
}
}
static void pr_cont_pool_info(struct worker_pool *pool)
{
pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
if (pool->node != NUMA_NO_NODE)
pr_cont(" node=%d", pool->node);
pr_cont(" flags=0x%x", pool->flags);
if (pool->flags & POOL_BH)
pr_cont(" bh%s",
pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
else
pr_cont(" nice=%d", pool->attrs->nice);
}
static void pr_cont_worker_id(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
if (pool->flags & WQ_BH)
pr_cont("bh%s",
pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
else
pr_cont("%d%s", task_pid_nr(worker->task),
worker->rescue_wq ? "(RESCUER)" : "");
}
struct pr_cont_work_struct {
bool comma;
work_func_t func;
long ctr;
};
static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
{
if (!pcwsp->ctr)
goto out_record;
if (func == pcwsp->func) {
pcwsp->ctr++;
return;
}
if (pcwsp->ctr == 1)
pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
else
pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
pcwsp->ctr = 0;
out_record:
if ((long)func == -1L)
return;
pcwsp->comma = comma;
pcwsp->func = func;
pcwsp->ctr = 1;
}
static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp)
{
if (work->func == wq_barrier_func) {
struct wq_barrier *barr;
barr = container_of(work, struct wq_barrier, work);
pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
pr_cont("%s BAR(%d)", comma ? "," : "",
task_pid_nr(barr->task));
} else {
if (!comma)
pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
pr_cont_work_flush(comma, work->func, pcwsp);
}
}
static void show_pwq(struct pool_workqueue *pwq)
{
struct pr_cont_work_struct pcws = { .ctr = 0, };
struct worker_pool *pool = pwq->pool;
struct work_struct *work;
struct worker *worker;
bool has_in_flight = false, has_pending = false;
int bkt;
pr_info(" pwq %d:", pool->id);
pr_cont_pool_info(pool);
pr_cont(" active=%d refcnt=%d%s\n",
pwq->nr_active, pwq->refcnt,
!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
hash_for_each(pool->busy_hash, bkt, worker, hentry) {
if (worker->current_pwq == pwq) {
has_in_flight = true;
break;
}
}
if (has_in_flight) {
bool comma = false;
pr_info(" in-flight:");
hash_for_each(pool->busy_hash, bkt, worker, hentry) {
if (worker->current_pwq != pwq)
continue;
pr_cont(" %s", comma ? "," : "");
pr_cont_worker_id(worker);
pr_cont(":%ps", worker->current_func);
list_for_each_entry(work, &worker->scheduled, entry)
pr_cont_work(false, work, &pcws);
pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
comma = true;
}
pr_cont("\n");
}
list_for_each_entry(work, &pool->worklist, entry) {
if (get_work_pwq(work) == pwq) {
has_pending = true;
break;
}
}
if (has_pending) {
bool comma = false;
pr_info(" pending:");
list_for_each_entry(work, &pool->worklist, entry) {
if (get_work_pwq(work) != pwq)
continue;
pr_cont_work(comma, work, &pcws);
comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
}
pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
pr_cont("\n");
}
if (!list_empty(&pwq->inactive_works)) {
bool comma = false;
pr_info(" inactive:");
list_for_each_entry(work, &pwq->inactive_works, entry) {
pr_cont_work(comma, work, &pcws);
comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
}
pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
pr_cont("\n");
}
}
/**
* show_one_workqueue - dump state of specified workqueue
* @wq: workqueue whose state will be printed
*/
void show_one_workqueue(struct workqueue_struct *wq)
{
struct pool_workqueue *pwq;
bool idle = true;
unsigned long irq_flags;
for_each_pwq(pwq, wq) {
if (!pwq_is_empty(pwq)) {
idle = false;
break;
}
}
if (idle) /* Nothing to print for idle workqueue */
return;
pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
for_each_pwq(pwq, wq) {
raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
if (!pwq_is_empty(pwq)) {
/*
* Defer printing to avoid deadlocks in console
* drivers that queue work while holding locks
* also taken in their write paths.
*/
printk_deferred_enter();
show_pwq(pwq);
printk_deferred_exit();
}
raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
/*
* We could be printing a lot from atomic context, e.g.
* sysrq-t -> show_all_workqueues(). Avoid triggering
* hard lockup.
*/
touch_nmi_watchdog();
}
}
/**
* show_one_worker_pool - dump state of specified worker pool
* @pool: worker pool whose state will be printed
*/
static void show_one_worker_pool(struct worker_pool *pool)
{
struct worker *worker;
bool first = true;
unsigned long irq_flags;
unsigned long hung = 0;
raw_spin_lock_irqsave(&pool->lock, irq_flags);
if (pool->nr_workers == pool->nr_idle)
goto next_pool;
/* How long the first pending work is waiting for a worker. */
if (!list_empty(&pool->worklist))
hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;
/*
* Defer printing to avoid deadlocks in console drivers that
* queue work while holding locks also taken in their write
* paths.
*/
printk_deferred_enter();
pr_info("pool %d:", pool->id);
pr_cont_pool_info(pool);
pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
if (pool->manager)
pr_cont(" manager: %d",
task_pid_nr(pool->manager->task));
list_for_each_entry(worker, &pool->idle_list, entry) {
pr_cont(" %s", first ? "idle: " : "");
pr_cont_worker_id(worker);
first = false;
}
pr_cont("\n");
printk_deferred_exit();
next_pool:
raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
/*
* We could be printing a lot from atomic context, e.g.
* sysrq-t -> show_all_workqueues(). Avoid triggering
* hard lockup.
*/
touch_nmi_watchdog();
}
/**
* show_all_workqueues - dump workqueue state
*
* Called from a sysrq handler and prints out all busy workqueues and pools.
*/
void show_all_workqueues(void)
{
struct workqueue_struct *wq;
struct worker_pool *pool;
int pi;
rcu_read_lock();
pr_info("Showing busy workqueues and worker pools:\n");
list_for_each_entry_rcu(wq, &workqueues, list)
show_one_workqueue(wq);
for_each_pool(pool, pi)
show_one_worker_pool(pool);
rcu_read_unlock();
}
/**
* show_freezable_workqueues - dump freezable workqueue state
*
* Called from try_to_freeze_tasks() and prints out all freezable workqueues
* still busy.
*/
void show_freezable_workqueues(void)
{
struct workqueue_struct *wq;
rcu_read_lock();
pr_info("Showing freezable workqueues that are still busy:\n");
list_for_each_entry_rcu(wq, &workqueues, list) {
if (!(wq->flags & WQ_FREEZABLE))
continue;
show_one_workqueue(wq);
}
rcu_read_unlock();
}
/* used to show worker information through /proc/PID/{comm,stat,status} */
void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
{
int off;
/* always show the actual comm */
off = strscpy(buf, task->comm, size);
if (off < 0)
return;
/* stabilize PF_WQ_WORKER and worker pool association */
mutex_lock(&wq_pool_attach_mutex);
if (task->flags & PF_WQ_WORKER) {
struct worker *worker = kthread_data(task);
struct worker_pool *pool = worker->pool;
if (pool) {
raw_spin_lock_irq(&pool->lock);
/*
* ->desc tracks information (wq name or
* set_worker_desc()) for the latest execution. If
* current, prepend '+', otherwise '-'.
*/
if (worker->desc[0] != '\0') {
if (worker->current_work)
scnprintf(buf + off, size - off, "+%s",
worker->desc);
else
scnprintf(buf + off, size - off, "-%s",
worker->desc);
}
raw_spin_unlock_irq(&pool->lock);
}
}
mutex_unlock(&wq_pool_attach_mutex);
}
#ifdef CONFIG_SMP
/*
* CPU hotplug.
*
* There are two challenges in supporting CPU hotplug. Firstly, there
* are a lot of assumptions on strong associations among work, pwq and
* pool which make migrating pending and scheduled works very
* difficult to implement without impacting hot paths. Secondly,
* worker pools serve mix of short, long and very long running works making
* blocked draining impractical.
*
* This is solved by allowing the pools to be disassociated from the CPU
* running as an unbound one and allowing it to be reattached later if the
* cpu comes back online.
*/
static void unbind_workers(int cpu)
{
struct worker_pool *pool;
struct worker *worker;
for_each_cpu_worker_pool(pool, cpu) {
mutex_lock(&wq_pool_attach_mutex);
raw_spin_lock_irq(&pool->lock);
/*
* We've blocked all attach/detach operations. Make all workers
* unbound and set DISASSOCIATED. Before this, all workers
* must be on the cpu. After this, they may become diasporas.
* And the preemption disabled section in their sched callbacks
* are guaranteed to see WORKER_UNBOUND since the code here
* is on the same cpu.
*/
for_each_pool_worker(worker, pool)
worker->flags |= WORKER_UNBOUND;
pool->flags |= POOL_DISASSOCIATED;
/*
* The handling of nr_running in sched callbacks are disabled
* now. Zap nr_running. After this, nr_running stays zero and
* need_more_worker() and keep_working() are always true as
* long as the worklist is not empty. This pool now behaves as
* an unbound (in terms of concurrency management) pool which
* are served by workers tied to the pool.
*/
pool->nr_running = 0;
/*
* With concurrency management just turned off, a busy
* worker blocking could lead to lengthy stalls. Kick off
* unbound chain execution of currently pending work items.
*/
kick_pool(pool);
raw_spin_unlock_irq(&pool->lock);
for_each_pool_worker(worker, pool)
unbind_worker(worker);
mutex_unlock(&wq_pool_attach_mutex);
}
}
/**
* rebind_workers - rebind all workers of a pool to the associated CPU
* @pool: pool of interest
*
* @pool->cpu is coming online. Rebind all workers to the CPU.
*/
static void rebind_workers(struct worker_pool *pool)
{
struct worker *worker;
lockdep_assert_held(&wq_pool_attach_mutex);
/*
* Restore CPU affinity of all workers. As all idle workers should
* be on the run-queue of the associated CPU before any local
* wake-ups for concurrency management happen, restore CPU affinity
* of all workers first and then clear UNBOUND. As we're called
* from CPU_ONLINE, the following shouldn't fail.
*/
for_each_pool_worker(worker, pool) {
kthread_set_per_cpu(worker->task, pool->cpu);
WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
pool_allowed_cpus(pool)) < 0);
}
raw_spin_lock_irq(&pool->lock);
pool->flags &= ~POOL_DISASSOCIATED;
for_each_pool_worker(worker, pool) {
unsigned int worker_flags = worker->flags;
/*
* We want to clear UNBOUND but can't directly call
* worker_clr_flags() or adjust nr_running. Atomically
* replace UNBOUND with another NOT_RUNNING flag REBOUND.
* @worker will clear REBOUND using worker_clr_flags() when
* it initiates the next execution cycle thus restoring
* concurrency management. Note that when or whether
* @worker clears REBOUND doesn't affect correctness.
*
* WRITE_ONCE() is necessary because @worker->flags may be
* tested without holding any lock in
* wq_worker_running(). Without it, NOT_RUNNING test may
* fail incorrectly leading to premature concurrency
* management operations.
*/
WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
worker_flags |= WORKER_REBOUND;
worker_flags &= ~WORKER_UNBOUND;
WRITE_ONCE(worker->flags, worker_flags);
}
raw_spin_unlock_irq(&pool->lock);
}
/**
* restore_unbound_workers_cpumask - restore cpumask of unbound workers
* @pool: unbound pool of interest
* @cpu: the CPU which is coming up
*
* An unbound pool may end up with a cpumask which doesn't have any online
* CPUs. When a worker of such pool get scheduled, the scheduler resets
* its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
* online CPU before, cpus_allowed of all its workers should be restored.
*/
static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
{
static cpumask_t cpumask;
struct worker *worker;
lockdep_assert_held(&wq_pool_attach_mutex);
/* is @cpu allowed for @pool? */
if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
return;
cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
/* as we're called from CPU_ONLINE, the following shouldn't fail */
for_each_pool_worker(worker, pool)
WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
}
int workqueue_prepare_cpu(unsigned int cpu)
{
struct worker_pool *pool;
for_each_cpu_worker_pool(pool, cpu) {
if (pool->nr_workers)
continue;
if (!create_worker(pool))
return -ENOMEM;
}
return 0;
}
int workqueue_online_cpu(unsigned int cpu)
{
struct worker_pool *pool;
struct workqueue_struct *wq;
int pi;
mutex_lock(&wq_pool_mutex);
for_each_pool(pool, pi) {
/* BH pools aren't affected by hotplug */
if (pool->flags & POOL_BH)
continue;
mutex_lock(&wq_pool_attach_mutex);
if (pool->cpu == cpu)
rebind_workers(pool);
else if (pool->cpu < 0)
restore_unbound_workers_cpumask(pool, cpu);
mutex_unlock(&wq_pool_attach_mutex);
}
/* update pod affinity of unbound workqueues */
list_for_each_entry(wq, &workqueues, list) {
struct workqueue_attrs *attrs = wq->unbound_attrs;
if (attrs) {
const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
int tcpu;
for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
wq_update_pod(wq, tcpu, cpu, true);
mutex_lock(&wq->mutex);
wq_update_node_max_active(wq, -1);
mutex_unlock(&wq->mutex);
}
}
mutex_unlock(&wq_pool_mutex);
return 0;
}
int workqueue_offline_cpu(unsigned int cpu)
{
struct workqueue_struct *wq;
/* unbinding per-cpu workers should happen on the local CPU */
if (WARN_ON(cpu != smp_processor_id()))
return -1;
unbind_workers(cpu);
/* update pod affinity of unbound workqueues */
mutex_lock(&wq_pool_mutex);
list_for_each_entry(wq, &workqueues, list) {
struct workqueue_attrs *attrs = wq->unbound_attrs;
if (attrs) {
const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
int tcpu;
for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
wq_update_pod(wq, tcpu, cpu, false);
mutex_lock(&wq->mutex);
wq_update_node_max_active(wq, cpu);
mutex_unlock(&wq->mutex);
}
}
mutex_unlock(&wq_pool_mutex);
return 0;
}
struct work_for_cpu {
struct work_struct work;
long (*fn)(void *);
void *arg;
long ret;
};
static void work_for_cpu_fn(struct work_struct *work)
{
struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
wfc->ret = wfc->fn(wfc->arg);
}
/**
* work_on_cpu_key - run a function in thread context on a particular cpu
* @cpu: the cpu to run on
* @fn: the function to run
* @arg: the function arg
* @key: The lock class key for lock debugging purposes
*
* It is up to the caller to ensure that the cpu doesn't go offline.
* The caller must not hold any locks which would prevent @fn from completing.
*
* Return: The value @fn returns.
*/
long work_on_cpu_key(int cpu, long (*fn)(void *),
void *arg, struct lock_class_key *key)
{
struct work_for_cpu wfc = { .fn = fn, .arg = arg };
INIT_WORK_ONSTACK_KEY(&wfc.work, work_for_cpu_fn, key);
schedule_work_on(cpu, &wfc.work);
flush_work(&wfc.work);
destroy_work_on_stack(&wfc.work);
return wfc.ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu_key);
/**
* work_on_cpu_safe_key - run a function in thread context on a particular cpu
* @cpu: the cpu to run on
* @fn: the function to run
* @arg: the function argument
* @key: The lock class key for lock debugging purposes
*
* Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
* any locks which would prevent @fn from completing.
*
* Return: The value @fn returns.
*/
long work_on_cpu_safe_key(int cpu, long (*fn)(void *),
void *arg, struct lock_class_key *key)
{
long ret = -ENODEV;
cpus_read_lock();
if (cpu_online(cpu))
ret = work_on_cpu_key(cpu, fn, arg, key);
cpus_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);
#endif /* CONFIG_SMP */
#ifdef CONFIG_FREEZER
/**
* freeze_workqueues_begin - begin freezing workqueues
*
* Start freezing workqueues. After this function returns, all freezable
* workqueues will queue new works to their inactive_works list instead of
* pool->worklist.
*
* CONTEXT:
* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
*/
void freeze_workqueues_begin(void)
{
struct workqueue_struct *wq;
mutex_lock(&wq_pool_mutex);
WARN_ON_ONCE(workqueue_freezing);
workqueue_freezing = true;
list_for_each_entry(wq, &workqueues, list) {
mutex_lock(&wq->mutex);
wq_adjust_max_active(wq);
mutex_unlock(&wq->mutex);
}
mutex_unlock(&wq_pool_mutex);
}
/**
* freeze_workqueues_busy - are freezable workqueues still busy?
*
* Check whether freezing is complete. This function must be called
* between freeze_workqueues_begin() and thaw_workqueues().
*
* CONTEXT:
* Grabs and releases wq_pool_mutex.
*
* Return:
* %true if some freezable workqueues are still busy. %false if freezing
* is complete.
*/
bool freeze_workqueues_busy(void)
{
bool busy = false;
struct workqueue_struct *wq;
struct pool_workqueue *pwq;
mutex_lock(&wq_pool_mutex);
WARN_ON_ONCE(!workqueue_freezing);
list_for_each_entry(wq, &workqueues, list) {
if (!(wq->flags & WQ_FREEZABLE))
continue;
/*
* nr_active is monotonically decreasing. It's safe
* to peek without lock.
*/
rcu_read_lock();
for_each_pwq(pwq, wq) {
WARN_ON_ONCE(pwq->nr_active < 0);
if (pwq->nr_active) {
busy = true;
rcu_read_unlock();
goto out_unlock;
}
}
rcu_read_unlock();
}
out_unlock:
mutex_unlock(&wq_pool_mutex);
return busy;
}
/**
* thaw_workqueues - thaw workqueues
*
* Thaw workqueues. Normal queueing is restored and all collected
* frozen works are transferred to their respective pool worklists.
*
* CONTEXT:
* Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
*/
void thaw_workqueues(void)
{
struct workqueue_struct *wq;
mutex_lock(&wq_pool_mutex);
if (!workqueue_freezing)
goto out_unlock;
workqueue_freezing = false;
/* restore max_active and repopulate worklist */
list_for_each_entry(wq, &workqueues, list) {
mutex_lock(&wq->mutex);
wq_adjust_max_active(wq);
mutex_unlock(&wq->mutex);
}
out_unlock:
mutex_unlock(&wq_pool_mutex);
}
#endif /* CONFIG_FREEZER */
static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
{
LIST_HEAD(ctxs);
int ret = 0;
struct workqueue_struct *wq;
struct apply_wqattrs_ctx *ctx, *n;
lockdep_assert_held(&wq_pool_mutex);
list_for_each_entry(wq, &workqueues, list) {
if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING))
continue;
ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
if (IS_ERR(ctx)) {
ret = PTR_ERR(ctx);
break;
}
list_add_tail(&ctx->list, &ctxs);
}
list_for_each_entry_safe(ctx, n, &ctxs, list) {
if (!ret)
apply_wqattrs_commit(ctx);
apply_wqattrs_cleanup(ctx);
}
if (!ret) {
mutex_lock(&wq_pool_attach_mutex);
cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
mutex_unlock(&wq_pool_attach_mutex);
}
return ret;
}
/**
* workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask
* @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask
*
* This function can be called from cpuset code to provide a set of isolated
* CPUs that should be excluded from wq_unbound_cpumask. The caller must hold
* either cpus_read_lock or cpus_write_lock.
*/
int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)
{
cpumask_var_t cpumask;
int ret = 0;
if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
return -ENOMEM;
lockdep_assert_cpus_held();
mutex_lock(&wq_pool_mutex);
/* Save the current isolated cpumask & export it via sysfs */
cpumask_copy(wq_isolated_cpumask, exclude_cpumask);
/*
* If the operation fails, it will fall back to
* wq_requested_unbound_cpumask which is initially set to
* (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten
* by any subsequent write to workqueue/cpumask sysfs file.
*/
if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask))
cpumask_copy(cpumask, wq_requested_unbound_cpumask);
if (!cpumask_equal(cpumask, wq_unbound_cpumask))
ret = workqueue_apply_unbound_cpumask(cpumask);
mutex_unlock(&wq_pool_mutex);
free_cpumask_var(cpumask);
return ret;
}
static int parse_affn_scope(const char *val)
{
int i;
for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) {
if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i])))
return i;
}
return -EINVAL;
}
static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp)
{
struct workqueue_struct *wq;
int affn, cpu;
affn = parse_affn_scope(val);
if (affn < 0)
return affn;
if (affn == WQ_AFFN_DFL)
return -EINVAL;
cpus_read_lock();
mutex_lock(&wq_pool_mutex);
wq_affn_dfl = affn;
list_for_each_entry(wq, &workqueues, list) {
for_each_online_cpu(cpu) {
wq_update_pod(wq, cpu, cpu, true);
}
}
mutex_unlock(&wq_pool_mutex);
cpus_read_unlock();
return 0;
}
static int wq_affn_dfl_get(char *buffer, const struct kernel_param *kp)
{
return scnprintf(buffer, PAGE_SIZE, "%s\n", wq_affn_names[wq_affn_dfl]);
}
static const struct kernel_param_ops wq_affn_dfl_ops = {
.set = wq_affn_dfl_set,
.get = wq_affn_dfl_get,
};
module_param_cb(default_affinity_scope, &wq_affn_dfl_ops, NULL, 0644);
#ifdef CONFIG_SYSFS
/*
* Workqueues with WQ_SYSFS flag set is visible to userland via
* /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
* following attributes.
*
* per_cpu RO bool : whether the workqueue is per-cpu or unbound
* max_active RW int : maximum number of in-flight work items
*
* Unbound workqueues have the following extra attributes.
*
* nice RW int : nice value of the workers
* cpumask RW mask : bitmask of allowed CPUs for the workers
* affinity_scope RW str : worker CPU affinity scope (cache, numa, none)
* affinity_strict RW bool : worker CPU affinity is strict
*/
struct wq_device {
struct workqueue_struct *wq;
struct device dev;
};
static struct workqueue_struct *dev_to_wq(struct device *dev)
{
struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
return wq_dev->wq;
}
static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct workqueue_struct *wq = dev_to_wq(dev);
return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
}
static DEVICE_ATTR_RO(per_cpu);
static ssize_t max_active_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct workqueue_struct *wq = dev_to_wq(dev);
return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
}
static ssize_t max_active_store(struct device *dev,
struct device_attribute *attr, const char *buf,
size_t count)
{
struct workqueue_struct *wq = dev_to_wq(dev);
int val;
if (sscanf(buf, "%d", &val) != 1 || val <= 0)
return -EINVAL;
workqueue_set_max_active(wq, val);
return count;
}
static DEVICE_ATTR_RW(max_active);
static struct attribute *wq_sysfs_attrs[] = {
&dev_attr_per_cpu.attr,
&dev_attr_max_active.attr,
NULL,
};
ATTRIBUTE_GROUPS(wq_sysfs);
static void apply_wqattrs_lock(void)
{
/* CPUs should stay stable across pwq creations and installations */
cpus_read_lock();
mutex_lock(&wq_pool_mutex);
}
static void apply_wqattrs_unlock(void)
{
mutex_unlock(&wq_pool_mutex);
cpus_read_unlock();
}
static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct workqueue_struct *wq = dev_to_wq(dev);
int written;
mutex_lock(&wq->mutex);
written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
mutex_unlock(&wq->mutex);
return written;
}
/* prepare workqueue_attrs for sysfs store operations */
static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
{
struct workqueue_attrs *attrs;
lockdep_assert_held(&wq_pool_mutex);
attrs = alloc_workqueue_attrs();
if (!attrs)
return NULL;
copy_workqueue_attrs(attrs, wq->unbound_attrs);
return attrs;
}
static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct workqueue_struct *wq = dev_to_wq(dev);
struct workqueue_attrs *attrs;
int ret = -ENOMEM;
apply_wqattrs_lock();
attrs = wq_sysfs_prep_attrs(wq);
if (!attrs)
goto out_unlock;
if (sscanf(buf, "%d", &attrs->nice) == 1 &&
attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
ret = apply_workqueue_attrs_locked(wq, attrs);
else
ret = -EINVAL;
out_unlock:
apply_wqattrs_unlock();
free_workqueue_attrs(attrs);
return ret ?: count;
}
static ssize_t wq_cpumask_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct workqueue_struct *wq = dev_to_wq(dev);
int written;
mutex_lock(&wq->mutex);
written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
cpumask_pr_args(wq->unbound_attrs->cpumask));
mutex_unlock(&wq->mutex);
return written;
}
static ssize_t wq_cpumask_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct workqueue_struct *wq = dev_to_wq(dev);
struct workqueue_attrs *attrs;
int ret = -ENOMEM;
apply_wqattrs_lock();
attrs = wq_sysfs_prep_attrs(wq);
if (!attrs)
goto out_unlock;
ret = cpumask_parse(buf, attrs->cpumask);
if (!ret)
ret = apply_workqueue_attrs_locked(wq, attrs);
out_unlock:
apply_wqattrs_unlock();
free_workqueue_attrs(attrs);
return ret ?: count;
}
static ssize_t wq_affn_scope_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct workqueue_struct *wq = dev_to_wq(dev);
int written;
mutex_lock(&wq->mutex);
if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n",
wq_affn_names[WQ_AFFN_DFL],
wq_affn_names[wq_affn_dfl]);
else
written = scnprintf(buf, PAGE_SIZE, "%s\n",
wq_affn_names[wq->unbound_attrs->affn_scope]);
mutex_unlock(&wq->mutex);
return written;
}
static ssize_t wq_affn_scope_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct workqueue_struct *wq = dev_to_wq(dev);
struct workqueue_attrs *attrs;
int affn, ret = -ENOMEM;
affn = parse_affn_scope(buf);
if (affn < 0)
return affn;
apply_wqattrs_lock();
attrs = wq_sysfs_prep_attrs(wq);
if (attrs) {
attrs->affn_scope = affn;
ret = apply_workqueue_attrs_locked(wq, attrs);
}
apply_wqattrs_unlock();
free_workqueue_attrs(attrs);
return ret ?: count;
}
static ssize_t wq_affinity_strict_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct workqueue_struct *wq = dev_to_wq(dev);
return scnprintf(buf, PAGE_SIZE, "%d\n",
wq->unbound_attrs->affn_strict);
}
static ssize_t wq_affinity_strict_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct workqueue_struct *wq = dev_to_wq(dev);
struct workqueue_attrs *attrs;
int v, ret = -ENOMEM;
if (sscanf(buf, "%d", &v) != 1)
return -EINVAL;
apply_wqattrs_lock();
attrs = wq_sysfs_prep_attrs(wq);
if (attrs) {
attrs->affn_strict = (bool)v;
ret = apply_workqueue_attrs_locked(wq, attrs);
}
apply_wqattrs_unlock();
free_workqueue_attrs(attrs);
return ret ?: count;
}
static struct device_attribute wq_sysfs_unbound_attrs[] = {
__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
__ATTR(affinity_scope, 0644, wq_affn_scope_show, wq_affn_scope_store),
__ATTR(affinity_strict, 0644, wq_affinity_strict_show, wq_affinity_strict_store),
__ATTR_NULL,
};
static const struct bus_type wq_subsys = {
.name = "workqueue",
.dev_groups = wq_sysfs_groups,
};
/**
* workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
* @cpumask: the cpumask to set
*
* The low-level workqueues cpumask is a global cpumask that limits
* the affinity of all unbound workqueues. This function check the @cpumask
* and apply it to all unbound workqueues and updates all pwqs of them.
*
* Return: 0 - Success
* -EINVAL - Invalid @cpumask
* -ENOMEM - Failed to allocate memory for attrs or pwqs.
*/
static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
{
int ret = -EINVAL;
/*
* Not excluding isolated cpus on purpose.
* If the user wishes to include them, we allow that.
*/
cpumask_and(cpumask, cpumask, cpu_possible_mask);
if (!cpumask_empty(cpumask)) {
apply_wqattrs_lock();
cpumask_copy(wq_requested_unbound_cpumask, cpumask);
if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
ret = 0;
goto out_unlock;
}
ret = workqueue_apply_unbound_cpumask(cpumask);
out_unlock:
apply_wqattrs_unlock();
}
return ret;
}
static ssize_t __wq_cpumask_show(struct device *dev,
struct device_attribute *attr, char *buf, cpumask_var_t mask)
{
int written;
mutex_lock(&wq_pool_mutex);
written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
mutex_unlock(&wq_pool_mutex);
return written;
}
static ssize_t wq_unbound_cpumask_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
}
static ssize_t wq_requested_cpumask_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
}
static ssize_t wq_isolated_cpumask_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
}
static ssize_t wq_unbound_cpumask_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t count)
{
cpumask_var_t cpumask;
int ret;
if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
return -ENOMEM;
ret = cpumask_parse(buf, cpumask);
if (!ret)
ret = workqueue_set_unbound_cpumask(cpumask);
free_cpumask_var(cpumask);
return ret ? ret : count;
}
static struct device_attribute wq_sysfs_cpumask_attrs[] = {
__ATTR(cpumask, 0644, wq_unbound_cpumask_show,
wq_unbound_cpumask_store),
__ATTR(cpumask_requested, 0444, wq_requested_cpumask_show, NULL),
__ATTR(cpumask_isolated, 0444, wq_isolated_cpumask_show, NULL),
__ATTR_NULL,
};
static int __init wq_sysfs_init(void)
{
struct device *dev_root;
int err;
err = subsys_virtual_register(&wq_subsys, NULL);
if (err)
return err;
dev_root = bus_get_dev_root(&wq_subsys);
if (dev_root) {
struct device_attribute *attr;
for (attr = wq_sysfs_cpumask_attrs; attr->attr.name; attr++) {
err = device_create_file(dev_root, attr);
if (err)
break;
}
put_device(dev_root);
}
return err;
}
core_initcall(wq_sysfs_init);
static void wq_device_release(struct device *dev)
{
struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
kfree(wq_dev);
}
/**
* workqueue_sysfs_register - make a workqueue visible in sysfs
* @wq: the workqueue to register
*
* Expose @wq in sysfs under /sys/bus/workqueue/devices.
* alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
* which is the preferred method.
*
* Workqueue user should use this function directly iff it wants to apply
* workqueue_attrs before making the workqueue visible in sysfs; otherwise,
* apply_workqueue_attrs() may race against userland updating the
* attributes.
*
* Return: 0 on success, -errno on failure.
*/
int workqueue_sysfs_register(struct workqueue_struct *wq)
{
struct wq_device *wq_dev;
int ret;
/*
* Adjusting max_active breaks ordering guarantee. Disallow exposing
* ordered workqueues.
*/
if (WARN_ON(wq->flags & __WQ_ORDERED))
return -EINVAL;
wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
if (!wq_dev)
return -ENOMEM;
wq_dev->wq = wq;
wq_dev->dev.bus = &wq_subsys;
wq_dev->dev.release = wq_device_release;
dev_set_name(&wq_dev->dev, "%s", wq->name);
/*
* unbound_attrs are created separately. Suppress uevent until
* everything is ready.
*/
dev_set_uevent_suppress(&wq_dev->dev, true);
ret = device_register(&wq_dev->dev);
if (ret) {
put_device(&wq_dev->dev);
wq->wq_dev = NULL;
return ret;
}
if (wq->flags & WQ_UNBOUND) {
struct device_attribute *attr;
for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
ret = device_create_file(&wq_dev->dev, attr);
if (ret) {
device_unregister(&wq_dev->dev);
wq->wq_dev = NULL;
return ret;
}
}
}
dev_set_uevent_suppress(&wq_dev->dev, false);
kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
return 0;
}
/**
* workqueue_sysfs_unregister - undo workqueue_sysfs_register()
* @wq: the workqueue to unregister
*
* If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
*/
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
{
struct wq_device *wq_dev = wq->wq_dev;
if (!wq->wq_dev)
return;
wq->wq_dev = NULL;
device_unregister(&wq_dev->dev);
}
#else /* CONFIG_SYSFS */
static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
#endif /* CONFIG_SYSFS */
/*
* Workqueue watchdog.
*
* Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
* flush dependency, a concurrency managed work item which stays RUNNING
* indefinitely. Workqueue stalls can be very difficult to debug as the
* usual warning mechanisms don't trigger and internal workqueue state is
* largely opaque.
*
* Workqueue watchdog monitors all worker pools periodically and dumps
* state if some pools failed to make forward progress for a while where
* forward progress is defined as the first item on ->worklist changing.
*
* This mechanism is controlled through the kernel parameter
* "workqueue.watchdog_thresh" which can be updated at runtime through the
* corresponding sysfs parameter file.
*/
#ifdef CONFIG_WQ_WATCHDOG
static unsigned long wq_watchdog_thresh = 30;
static struct timer_list wq_watchdog_timer;
static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
/*
* Show workers that might prevent the processing of pending work items.
* The only candidates are CPU-bound workers in the running state.
* Pending work items should be handled by another idle worker
* in all other situations.
*/
static void show_cpu_pool_hog(struct worker_pool *pool)
{
struct worker *worker;
unsigned long irq_flags;
int bkt;
raw_spin_lock_irqsave(&pool->lock, irq_flags);
hash_for_each(pool->busy_hash, bkt, worker, hentry) {
if (task_is_running(worker->task)) {
/*
* Defer printing to avoid deadlocks in console
* drivers that queue work while holding locks
* also taken in their write paths.
*/
printk_deferred_enter();
pr_info("pool %d:\n", pool->id);
sched_show_task(worker->task);
printk_deferred_exit();
}
}
raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
}
static void show_cpu_pools_hogs(void)
{
struct worker_pool *pool;
int pi;
pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");
rcu_read_lock();
for_each_pool(pool, pi) {
if (pool->cpu_stall)
show_cpu_pool_hog(pool);
}
rcu_read_unlock();
}
static void wq_watchdog_reset_touched(void)
{
int cpu;
wq_watchdog_touched = jiffies;
for_each_possible_cpu(cpu)
per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
}
static void wq_watchdog_timer_fn(struct timer_list *unused)
{
unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
bool lockup_detected = false;
bool cpu_pool_stall = false;
unsigned long now = jiffies;
struct worker_pool *pool;
int pi;
if (!thresh)
return;
rcu_read_lock();
for_each_pool(pool, pi) {
unsigned long pool_ts, touched, ts;
pool->cpu_stall = false;
if (list_empty(&pool->worklist))
continue;
/*
* If a virtual machine is stopped by the host it can look to
* the watchdog like a stall.
*/
kvm_check_and_clear_guest_paused();
/* get the latest of pool and touched timestamps */
if (pool->cpu >= 0)
touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
else
touched = READ_ONCE(wq_watchdog_touched);
pool_ts = READ_ONCE(pool->watchdog_ts);
if (time_after(pool_ts, touched))
ts = pool_ts;
else
ts = touched;
/* did we stall? */
if (time_after(now, ts + thresh)) {
lockup_detected = true;
if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
pool->cpu_stall = true;
cpu_pool_stall = true;
}
pr_emerg("BUG: workqueue lockup - pool");
pr_cont_pool_info(pool);
pr_cont(" stuck for %us!\n",
jiffies_to_msecs(now - pool_ts) / 1000);
}
}
rcu_read_unlock();
if (lockup_detected)
show_all_workqueues();
if (cpu_pool_stall)
show_cpu_pools_hogs();
wq_watchdog_reset_touched();
mod_timer(&wq_watchdog_timer, jiffies + thresh);
}
notrace void wq_watchdog_touch(int cpu)
{
if (cpu >= 0) per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies; wq_watchdog_touched = jiffies;
}
static void wq_watchdog_set_thresh(unsigned long thresh)
{
wq_watchdog_thresh = 0;
del_timer_sync(&wq_watchdog_timer);
if (thresh) {
wq_watchdog_thresh = thresh;
wq_watchdog_reset_touched();
mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
}
}
static int wq_watchdog_param_set_thresh(const char *val,
const struct kernel_param *kp)
{
unsigned long thresh;
int ret;
ret = kstrtoul(val, 0, &thresh);
if (ret)
return ret;
if (system_wq)
wq_watchdog_set_thresh(thresh);
else
wq_watchdog_thresh = thresh;
return 0;
}
static const struct kernel_param_ops wq_watchdog_thresh_ops = {
.set = wq_watchdog_param_set_thresh,
.get = param_get_ulong,
};
module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
0644);
static void wq_watchdog_init(void)
{
timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
wq_watchdog_set_thresh(wq_watchdog_thresh);
}
#else /* CONFIG_WQ_WATCHDOG */
static inline void wq_watchdog_init(void) { }
#endif /* CONFIG_WQ_WATCHDOG */
static void bh_pool_kick_normal(struct irq_work *irq_work)
{
raise_softirq_irqoff(TASKLET_SOFTIRQ);
}
static void bh_pool_kick_highpri(struct irq_work *irq_work)
{
raise_softirq_irqoff(HI_SOFTIRQ);
}
static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
{
if (!cpumask_intersects(wq_unbound_cpumask, mask)) {
pr_warn("workqueue: Restricting unbound_cpumask (%*pb) with %s (%*pb) leaves no CPU, ignoring\n",
cpumask_pr_args(wq_unbound_cpumask), name, cpumask_pr_args(mask));
return;
}
cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask);
}
static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice)
{
BUG_ON(init_worker_pool(pool));
pool->cpu = cpu;
cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
pool->attrs->nice = nice;
pool->attrs->affn_strict = true;
pool->node = cpu_to_node(cpu);
/* alloc pool ID */
mutex_lock(&wq_pool_mutex);
BUG_ON(worker_pool_assign_id(pool));
mutex_unlock(&wq_pool_mutex);
}
/**
* workqueue_init_early - early init for workqueue subsystem
*
* This is the first step of three-staged workqueue subsystem initialization and
* invoked as soon as the bare basics - memory allocation, cpumasks and idr are
* up. It sets up all the data structures and system workqueues and allows early
* boot code to create workqueues and queue/cancel work items. Actual work item
* execution starts only after kthreads can be created and scheduled right
* before early initcalls.
*/
void __init workqueue_init_early(void)
{
struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal,
bh_pool_kick_highpri };
int i, cpu;
BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL));
BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL));
cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ));
restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN));
if (!cpumask_empty(&wq_cmdline_cpumask))
restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask);
cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask);
pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
wq_update_pod_attrs_buf = alloc_workqueue_attrs();
BUG_ON(!wq_update_pod_attrs_buf);
/*
* If nohz_full is enabled, set power efficient workqueue as unbound.
* This allows workqueue items to be moved to HK CPUs.
*/
if (housekeeping_enabled(HK_TYPE_TICK))
wq_power_efficient = true;
/* initialize WQ_AFFN_SYSTEM pods */
pt->pod_cpus = kcalloc(1, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
pt->pod_node = kcalloc(1, sizeof(pt->pod_node[0]), GFP_KERNEL);
pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
BUG_ON(!pt->pod_cpus || !pt->pod_node || !pt->cpu_pod);
BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE));
pt->nr_pods = 1;
cpumask_copy(pt->pod_cpus[0], cpu_possible_mask);
pt->pod_node[0] = NUMA_NO_NODE;
pt->cpu_pod[0] = 0;
/* initialize BH and CPU pools */
for_each_possible_cpu(cpu) {
struct worker_pool *pool;
i = 0;
for_each_bh_worker_pool(pool, cpu) {
init_cpu_worker_pool(pool, cpu, std_nice[i]);
pool->flags |= POOL_BH;
init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
i++;
}
i = 0;
for_each_cpu_worker_pool(pool, cpu)
init_cpu_worker_pool(pool, cpu, std_nice[i++]);
}
/* create default unbound and ordered wq attrs */
for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
struct workqueue_attrs *attrs;
BUG_ON(!(attrs = alloc_workqueue_attrs()));
attrs->nice = std_nice[i];
unbound_std_wq_attrs[i] = attrs;
/*
* An ordered wq should have only one pwq as ordering is
* guaranteed by max_active which is enforced by pwqs.
*/
BUG_ON(!(attrs = alloc_workqueue_attrs()));
attrs->nice = std_nice[i];
attrs->ordered = true;
ordered_wq_attrs[i] = attrs;
}
system_wq = alloc_workqueue("events", 0, 0);
system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
system_long_wq = alloc_workqueue("events_long", 0, 0);
system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
WQ_MAX_ACTIVE);
system_freezable_wq = alloc_workqueue("events_freezable",
WQ_FREEZABLE, 0);
system_power_efficient_wq = alloc_workqueue("events_power_efficient",
WQ_POWER_EFFICIENT, 0);
system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",
WQ_FREEZABLE | WQ_POWER_EFFICIENT,
0);
system_bh_wq = alloc_workqueue("events_bh", WQ_BH, 0);
system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",
WQ_BH | WQ_HIGHPRI, 0);
BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
!system_unbound_wq || !system_freezable_wq ||
!system_power_efficient_wq ||
!system_freezable_power_efficient_wq ||
!system_bh_wq || !system_bh_highpri_wq);
}
static void __init wq_cpu_intensive_thresh_init(void)
{
unsigned long thresh;
unsigned long bogo;
pwq_release_worker = kthread_create_worker(0, "pool_workqueue_release");
BUG_ON(IS_ERR(pwq_release_worker));
/* if the user set it to a specific value, keep it */
if (wq_cpu_intensive_thresh_us != ULONG_MAX)
return;
/*
* The default of 10ms is derived from the fact that most modern (as of
* 2023) processors can do a lot in 10ms and that it's just below what
* most consider human-perceivable. However, the kernel also runs on a
* lot slower CPUs including microcontrollers where the threshold is way
* too low.
*
* Let's scale up the threshold upto 1 second if BogoMips is below 4000.
* This is by no means accurate but it doesn't have to be. The mechanism
* is still useful even when the threshold is fully scaled up. Also, as
* the reports would usually be applicable to everyone, some machines
* operating on longer thresholds won't significantly diminish their
* usefulness.
*/
thresh = 10 * USEC_PER_MSEC;
/* see init/calibrate.c for lpj -> BogoMIPS calculation */
bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1);
if (bogo < 4000)
thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC);
pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n",
loops_per_jiffy, bogo, thresh);
wq_cpu_intensive_thresh_us = thresh;
}
/**
* workqueue_init - bring workqueue subsystem fully online
*
* This is the second step of three-staged workqueue subsystem initialization
* and invoked as soon as kthreads can be created and scheduled. Workqueues have
* been created and work items queued on them, but there are no kworkers
* executing the work items yet. Populate the worker pools with the initial
* workers and enable future kworker creations.
*/
void __init workqueue_init(void)
{
struct workqueue_struct *wq;
struct worker_pool *pool;
int cpu, bkt;
wq_cpu_intensive_thresh_init();
mutex_lock(&wq_pool_mutex);
/*
* Per-cpu pools created earlier could be missing node hint. Fix them
* up. Also, create a rescuer for workqueues that requested it.
*/
for_each_possible_cpu(cpu) {
for_each_bh_worker_pool(pool, cpu)
pool->node = cpu_to_node(cpu);
for_each_cpu_worker_pool(pool, cpu)
pool->node = cpu_to_node(cpu);
}
list_for_each_entry(wq, &workqueues, list) {
WARN(init_rescuer(wq),
"workqueue: failed to create early rescuer for %s",
wq->name);
}
mutex_unlock(&wq_pool_mutex);
/*
* Create the initial workers. A BH pool has one pseudo worker that
* represents the shared BH execution context and thus doesn't get
* affected by hotplug events. Create the BH pseudo workers for all
* possible CPUs here.
*/
for_each_possible_cpu(cpu)
for_each_bh_worker_pool(pool, cpu)
BUG_ON(!create_worker(pool));
for_each_online_cpu(cpu) {
for_each_cpu_worker_pool(pool, cpu) {
pool->flags &= ~POOL_DISASSOCIATED;
BUG_ON(!create_worker(pool));
}
}
hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
BUG_ON(!create_worker(pool));
wq_online = true;
wq_watchdog_init();
}
/*
* Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to
* @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique
* and consecutive pod ID. The rest of @pt is initialized accordingly.
*/
static void __init init_pod_type(struct wq_pod_type *pt,
bool (*cpus_share_pod)(int, int))
{
int cur, pre, cpu, pod;
pt->nr_pods = 0;
/* init @pt->cpu_pod[] according to @cpus_share_pod() */
pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
BUG_ON(!pt->cpu_pod);
for_each_possible_cpu(cur) {
for_each_possible_cpu(pre) {
if (pre >= cur) {
pt->cpu_pod[cur] = pt->nr_pods++;
break;
}
if (cpus_share_pod(cur, pre)) {
pt->cpu_pod[cur] = pt->cpu_pod[pre];
break;
}
}
}
/* init the rest to match @pt->cpu_pod[] */
pt->pod_cpus = kcalloc(pt->nr_pods, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
pt->pod_node = kcalloc(pt->nr_pods, sizeof(pt->pod_node[0]), GFP_KERNEL);
BUG_ON(!pt->pod_cpus || !pt->pod_node);
for (pod = 0; pod < pt->nr_pods; pod++)
BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL));
for_each_possible_cpu(cpu) {
cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]);
pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu);
}
}
static bool __init cpus_dont_share(int cpu0, int cpu1)
{
return false;
}
static bool __init cpus_share_smt(int cpu0, int cpu1)
{
#ifdef CONFIG_SCHED_SMT
return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
#else
return false;
#endif
}
static bool __init cpus_share_numa(int cpu0, int cpu1)
{
return cpu_to_node(cpu0) == cpu_to_node(cpu1);
}
/**
* workqueue_init_topology - initialize CPU pods for unbound workqueues
*
* This is the third step of three-staged workqueue subsystem initialization and
* invoked after SMP and topology information are fully initialized. It
* initializes the unbound CPU pods accordingly.
*/
void __init workqueue_init_topology(void)
{
struct workqueue_struct *wq;
int cpu;
init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);
init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);
init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);
init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa);
wq_topo_initialized = true;
mutex_lock(&wq_pool_mutex);
/*
* Workqueues allocated earlier would have all CPUs sharing the default
* worker pool. Explicitly call wq_update_pod() on all workqueue and CPU
* combinations to apply per-pod sharing.
*/
list_for_each_entry(wq, &workqueues, list) {
for_each_online_cpu(cpu)
wq_update_pod(wq, cpu, cpu, true);
if (wq->flags & WQ_UNBOUND) {
mutex_lock(&wq->mutex);
wq_update_node_max_active(wq, -1);
mutex_unlock(&wq->mutex);
}
}
mutex_unlock(&wq_pool_mutex);
}
void __warn_flushing_systemwide_wq(void)
{
pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n");
dump_stack();
}
EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
static int __init workqueue_unbound_cpus_setup(char *str)
{
if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {
cpumask_clear(&wq_cmdline_cpumask);
pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");
}
return 1;
}
__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);
// SPDX-License-Identifier: GPL-2.0
/*
* linux/fs/file.c
*
* Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
*
* Manage the dynamic fd arrays in the process files_struct.
*/
#include <linux/syscalls.h>
#include <linux/export.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/bitops.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <linux/close_range.h>
#include <net/sock.h>
#include "internal.h"
unsigned int sysctl_nr_open __read_mostly = 1024*1024;
unsigned int sysctl_nr_open_min = BITS_PER_LONG;
/* our min() is unusable in constant expressions ;-/ */
#define __const_min(x, y) ((x) < (y) ? (x) : (y))
unsigned int sysctl_nr_open_max =
__const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
static void __free_fdtable(struct fdtable *fdt)
{
kvfree(fdt->fd);
kvfree(fdt->open_fds);
kfree(fdt);
}
static void free_fdtable_rcu(struct rcu_head *rcu)
{
__free_fdtable(container_of(rcu, struct fdtable, rcu));
}
#define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
#define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
/*
* Copy 'count' fd bits from the old table to the new table and clear the extra
* space if any. This does not copy the file pointers. Called with the files
* spinlock held for write.
*/
static void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
unsigned int count)
{
unsigned int cpy, set;
cpy = count / BITS_PER_BYTE;
set = (nfdt->max_fds - count) / BITS_PER_BYTE;
memcpy(nfdt->open_fds, ofdt->open_fds, cpy);
memset((char *)nfdt->open_fds + cpy, 0, set);
memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy);
memset((char *)nfdt->close_on_exec + cpy, 0, set);
cpy = BITBIT_SIZE(count);
set = BITBIT_SIZE(nfdt->max_fds) - cpy;
memcpy(nfdt->full_fds_bits, ofdt->full_fds_bits, cpy);
memset((char *)nfdt->full_fds_bits + cpy, 0, set);
}
/*
* Copy all file descriptors from the old table to the new, expanded table and
* clear the extra space. Called with the files spinlock held for write.
*/
static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
{
size_t cpy, set;
BUG_ON(nfdt->max_fds < ofdt->max_fds); cpy = ofdt->max_fds * sizeof(struct file *);
set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
memcpy(nfdt->fd, ofdt->fd, cpy);
memset((char *)nfdt->fd + cpy, 0, set);
copy_fd_bitmaps(nfdt, ofdt, ofdt->max_fds);
}
/*
* Note how the fdtable bitmap allocations very much have to be a multiple of
* BITS_PER_LONG. This is not only because we walk those things in chunks of
* 'unsigned long' in some places, but simply because that is how the Linux
* kernel bitmaps are defined to work: they are not "bits in an array of bytes",
* they are very much "bits in an array of unsigned long".
*
* The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied
* by that "1024/sizeof(ptr)" before, we already know there are sufficient
* clear low bits. Clang seems to realize that, gcc ends up being confused.
*
* On a 128-bit machine, the ALIGN() would actually matter. In the meantime,
* let's consider it documentation (and maybe a test-case for gcc to improve
* its code generation ;)
*/
static struct fdtable * alloc_fdtable(unsigned int nr)
{
struct fdtable *fdt;
void *data;
/*
* Figure out how many fds we actually want to support in this fdtable.
* Allocation steps are keyed to the size of the fdarray, since it
* grows far faster than any of the other dynamic data. We try to fit
* the fdarray into comfortable page-tuned chunks: starting at 1024B
* and growing in powers of two from there on.
*/
nr /= (1024 / sizeof(struct file *));
nr = roundup_pow_of_two(nr + 1);
nr *= (1024 / sizeof(struct file *));
nr = ALIGN(nr, BITS_PER_LONG);
/*
* Note that this can drive nr *below* what we had passed if sysctl_nr_open
* had been set lower between the check in expand_files() and here. Deal
* with that in caller, it's cheaper that way.
*
* We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
* bitmaps handling below becomes unpleasant, to put it mildly...
*/
if (unlikely(nr > sysctl_nr_open))
nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);
if (!fdt)
goto out;
fdt->max_fds = nr;
data = kvmalloc_array(nr, sizeof(struct file *), GFP_KERNEL_ACCOUNT);
if (!data)
goto out_fdt;
fdt->fd = data;
data = kvmalloc(max_t(size_t,
2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
GFP_KERNEL_ACCOUNT);
if (!data)
goto out_arr;
fdt->open_fds = data;
data += nr / BITS_PER_BYTE;
fdt->close_on_exec = data;
data += nr / BITS_PER_BYTE;
fdt->full_fds_bits = data;
return fdt;
out_arr:
kvfree(fdt->fd);
out_fdt:
kfree(fdt);
out:
return NULL;
}
/*
* Expand the file descriptor table.
* This function will allocate a new fdtable and both fd array and fdset, of
* the given size.
* Return <0 error code on error; 1 on successful completion.
* The files->file_lock should be held on entry, and will be held on exit.
*/
static int expand_fdtable(struct files_struct *files, unsigned int nr)
__releases(files->file_lock)
__acquires(files->file_lock)
{
struct fdtable *new_fdt, *cur_fdt;
spin_unlock(&files->file_lock);
new_fdt = alloc_fdtable(nr);
/* make sure all fd_install() have seen resize_in_progress
* or have finished their rcu_read_lock_sched() section.
*/
if (atomic_read(&files->count) > 1)
synchronize_rcu(); spin_lock(&files->file_lock);
if (!new_fdt)
return -ENOMEM;
/*
* extremely unlikely race - sysctl_nr_open decreased between the check in
* caller and alloc_fdtable(). Cheaper to catch it here...
*/
if (unlikely(new_fdt->max_fds <= nr)) { __free_fdtable(new_fdt);
return -EMFILE;
}
cur_fdt = files_fdtable(files); BUG_ON(nr < cur_fdt->max_fds); copy_fdtable(new_fdt, cur_fdt);
rcu_assign_pointer(files->fdt, new_fdt);
if (cur_fdt != &files->fdtab)
call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
/* coupled with smp_rmb() in fd_install() */
smp_wmb();
return 1;
}
/*
* Expand files.
* This function will expand the file structures, if the requested size exceeds
* the current capacity and there is room for expansion.
* Return <0 error code on error; 0 when nothing done; 1 when files were
* expanded and execution may have blocked.
* The files->file_lock should be held on entry, and will be held on exit.
*/
static int expand_files(struct files_struct *files, unsigned int nr)
__releases(files->file_lock)
__acquires(files->file_lock)
{
struct fdtable *fdt;
int expanded = 0;
repeat:
fdt = files_fdtable(files);
/* Do we need to expand? */
if (nr < fdt->max_fds)
return expanded;
/* Can we expand? */
if (nr >= sysctl_nr_open)
return -EMFILE;
if (unlikely(files->resize_in_progress)) { spin_unlock(&files->file_lock);
expanded = 1;
wait_event(files->resize_wait, !files->resize_in_progress); spin_lock(&files->file_lock);
goto repeat;
}
/* All good, so we try */
files->resize_in_progress = true; expanded = expand_fdtable(files, nr); files->resize_in_progress = false;
wake_up_all(&files->resize_wait);
return expanded;
}
static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)
{
__set_bit(fd, fdt->close_on_exec);
}
static inline void __clear_close_on_exec(unsigned int fd, struct fdtable *fdt)
{
if (test_bit(fd, fdt->close_on_exec)) __clear_bit(fd, fdt->close_on_exec);
}
static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)
{
__set_bit(fd, fdt->open_fds);
fd /= BITS_PER_LONG;
if (!~fdt->open_fds[fd])
__set_bit(fd, fdt->full_fds_bits);
}
static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
{
__clear_bit(fd, fdt->open_fds);
__clear_bit(fd / BITS_PER_LONG, fdt->full_fds_bits);
}
static unsigned int count_open_files(struct fdtable *fdt)
{
unsigned int size = fdt->max_fds;
unsigned int i;
/* Find the last open fd */
for (i = size / BITS_PER_LONG; i > 0; ) {
if (fdt->open_fds[--i])
break;
}
i = (i + 1) * BITS_PER_LONG;
return i;
}
/*
* Note that a sane fdtable size always has to be a multiple of
* BITS_PER_LONG, since we have bitmaps that are sized by this.
*
* 'max_fds' will normally already be properly aligned, but it
* turns out that in the close_range() -> __close_range() ->
* unshare_fd() -> dup_fd() -> sane_fdtable_size() we can end
* up having a 'max_fds' value that isn't already aligned.
*
* Rather than make close_range() have to worry about this,
* just make that BITS_PER_LONG alignment be part of a sane
* fdtable size. Becuase that's really what it is.
*/
static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
{
unsigned int count;
count = count_open_files(fdt);
if (max_fds < NR_OPEN_DEFAULT)
max_fds = NR_OPEN_DEFAULT;
return ALIGN(min(count, max_fds), BITS_PER_LONG);
}
/*
* Allocate a new files structure and copy contents from the
* passed in files structure.
* errorp will be valid only when the returned files_struct is NULL.
*/
struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, int *errorp)
{
struct files_struct *newf;
struct file **old_fds, **new_fds;
unsigned int open_files, i;
struct fdtable *old_fdt, *new_fdt;
*errorp = -ENOMEM;
newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
if (!newf)
goto out;
atomic_set(&newf->count, 1);
spin_lock_init(&newf->file_lock);
newf->resize_in_progress = false;
init_waitqueue_head(&newf->resize_wait);
newf->next_fd = 0;
new_fdt = &newf->fdtab;
new_fdt->max_fds = NR_OPEN_DEFAULT;
new_fdt->close_on_exec = newf->close_on_exec_init;
new_fdt->open_fds = newf->open_fds_init;
new_fdt->full_fds_bits = newf->full_fds_bits_init;
new_fdt->fd = &newf->fd_array[0];
spin_lock(&oldf->file_lock);
old_fdt = files_fdtable(oldf);
open_files = sane_fdtable_size(old_fdt, max_fds);
/*
* Check whether we need to allocate a larger fd array and fd set.
*/
while (unlikely(open_files > new_fdt->max_fds)) {
spin_unlock(&oldf->file_lock);
if (new_fdt != &newf->fdtab)
__free_fdtable(new_fdt);
new_fdt = alloc_fdtable(open_files - 1);
if (!new_fdt) {
*errorp = -ENOMEM;
goto out_release;
}
/* beyond sysctl_nr_open; nothing to do */
if (unlikely(new_fdt->max_fds < open_files)) {
__free_fdtable(new_fdt);
*errorp = -EMFILE;
goto out_release;
}
/*
* Reacquire the oldf lock and a pointer to its fd table
* who knows it may have a new bigger fd table. We need
* the latest pointer.
*/
spin_lock(&oldf->file_lock);
old_fdt = files_fdtable(oldf);
open_files = sane_fdtable_size(old_fdt, max_fds);
}
copy_fd_bitmaps(new_fdt, old_fdt, open_files);
old_fds = old_fdt->fd;
new_fds = new_fdt->fd;
for (i = open_files; i != 0; i--) {
struct file *f = *old_fds++;
if (f) {
get_file(f);
} else {
/*
* The fd may be claimed in the fd bitmap but not yet
* instantiated in the files array if a sibling thread
* is partway through open(). So make sure that this
* fd is available to the new process.
*/
__clear_open_fd(open_files - i, new_fdt);
}
rcu_assign_pointer(*new_fds++, f);
}
spin_unlock(&oldf->file_lock);
/* clear the remainder */
memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
rcu_assign_pointer(newf->fdt, new_fdt);
return newf;
out_release:
kmem_cache_free(files_cachep, newf);
out:
return NULL;
}
static struct fdtable *close_files(struct files_struct * files)
{
/*
* It is safe to dereference the fd table without RCU or
* ->file_lock because this is the last reference to the
* files structure.
*/
struct fdtable *fdt = rcu_dereference_raw(files->fdt);
unsigned int i, j = 0;
for (;;) {
unsigned long set;
i = j * BITS_PER_LONG;
if (i >= fdt->max_fds)
break;
set = fdt->open_fds[j++];
while (set) {
if (set & 1) {
struct file * file = xchg(&fdt->fd[i], NULL);
if (file) {
filp_close(file, files);
cond_resched();
}
}
i++;
set >>= 1;
}
}
return fdt;
}
void put_files_struct(struct files_struct *files)
{
if (atomic_dec_and_test(&files->count)) {
struct fdtable *fdt = close_files(files);
/* free the arrays if they are not embedded */
if (fdt != &files->fdtab)
__free_fdtable(fdt);
kmem_cache_free(files_cachep, files);
}
}
void exit_files(struct task_struct *tsk)
{
struct files_struct * files = tsk->files;
if (files) {
task_lock(tsk);
tsk->files = NULL;
task_unlock(tsk);
put_files_struct(files);
}
}
struct files_struct init_files = {
.count = ATOMIC_INIT(1),
.fdt = &init_files.fdtab,
.fdtab = {
.max_fds = NR_OPEN_DEFAULT,
.fd = &init_files.fd_array[0],
.close_on_exec = init_files.close_on_exec_init,
.open_fds = init_files.open_fds_init,
.full_fds_bits = init_files.full_fds_bits_init,
},
.file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
.resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
};
static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
{
unsigned int maxfd = fdt->max_fds;
unsigned int maxbit = maxfd / BITS_PER_LONG;
unsigned int bitbit = start / BITS_PER_LONG;
bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;
if (bitbit > maxfd)
return maxfd;
if (bitbit > start)
start = bitbit;
return find_next_zero_bit(fdt->open_fds, maxfd, start);
}
/*
* allocate a file descriptor, mark it busy.
*/
static int alloc_fd(unsigned start, unsigned end, unsigned flags)
{
struct files_struct *files = current->files;
unsigned int fd;
int error;
struct fdtable *fdt;
spin_lock(&files->file_lock);
repeat:
fdt = files_fdtable(files);
fd = start;
if (fd < files->next_fd)
fd = files->next_fd;
if (fd < fdt->max_fds)
fd = find_next_fd(fdt, fd);
/*
* N.B. For clone tasks sharing a files structure, this test
* will limit the total number of files that can be opened.
*/
error = -EMFILE;
if (fd >= end)
goto out;
error = expand_files(files, fd);
if (error < 0)
goto out;
/*
* If we needed to expand the fs array we
* might have blocked - try again.
*/
if (error)
goto repeat;
if (start <= files->next_fd) files->next_fd = fd + 1; __set_open_fd(fd, fdt); if (flags & O_CLOEXEC) __set_close_on_exec(fd, fdt);
else
__clear_close_on_exec(fd, fdt); error = fd;
#if 1
/* Sanity check */
if (rcu_access_pointer(fdt->fd[fd]) != NULL) {
printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd);
rcu_assign_pointer(fdt->fd[fd], NULL);
}
#endif
out:
spin_unlock(&files->file_lock);
return error;
}
int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
{
return alloc_fd(0, nofile, flags);
}
int get_unused_fd_flags(unsigned flags)
{
return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
}
EXPORT_SYMBOL(get_unused_fd_flags);
static void __put_unused_fd(struct files_struct *files, unsigned int fd)
{
struct fdtable *fdt = files_fdtable(files); __clear_open_fd(fd, fdt);
if (fd < files->next_fd)
files->next_fd = fd;
}
void put_unused_fd(unsigned int fd)
{
struct files_struct *files = current->files;
spin_lock(&files->file_lock);
__put_unused_fd(files, fd);
spin_unlock(&files->file_lock);
}
EXPORT_SYMBOL(put_unused_fd);
/*
* Install a file pointer in the fd array.
*
* The VFS is full of places where we drop the files lock between
* setting the open_fds bitmap and installing the file in the file
* array. At any such point, we are vulnerable to a dup2() race
* installing a file in the array before us. We need to detect this and
* fput() the struct file we are about to overwrite in this case.
*
* It should never happen - if we allow dup2() do it, _really_ bad things
* will follow.
*
* This consumes the "file" refcount, so callers should treat it
* as if they had called fput(file).
*/
void fd_install(unsigned int fd, struct file *file)
{
struct files_struct *files = current->files;
struct fdtable *fdt;
if (WARN_ON_ONCE(unlikely(file->f_mode & FMODE_BACKING)))
return;
rcu_read_lock_sched(); if (unlikely(files->resize_in_progress)) { rcu_read_unlock_sched(); spin_lock(&files->file_lock); fdt = files_fdtable(files); BUG_ON(fdt->fd[fd] != NULL); rcu_assign_pointer(fdt->fd[fd], file);
spin_unlock(&files->file_lock);
return;
}
/* coupled with smp_wmb() in expand_fdtable() */
smp_rmb(); fdt = rcu_dereference_sched(files->fdt); BUG_ON(fdt->fd[fd] != NULL); rcu_assign_pointer(fdt->fd[fd], file); rcu_read_unlock_sched();
}
EXPORT_SYMBOL(fd_install);
/**
* file_close_fd_locked - return file associated with fd
* @files: file struct to retrieve file from
* @fd: file descriptor to retrieve file for
*
* Doesn't take a separate reference count.
*
* Context: files_lock must be held.
*
* Returns: The file associated with @fd (NULL if @fd is not open)
*/
struct file *file_close_fd_locked(struct files_struct *files, unsigned fd)
{
struct fdtable *fdt = files_fdtable(files);
struct file *file;
lockdep_assert_held(&files->file_lock); if (fd >= fdt->max_fds)
return NULL;
fd = array_index_nospec(fd, fdt->max_fds);
file = fdt->fd[fd];
if (file) {
rcu_assign_pointer(fdt->fd[fd], NULL); __put_unused_fd(files, fd);
}
return file;
}
int close_fd(unsigned fd)
{
struct files_struct *files = current->files;
struct file *file;
spin_lock(&files->file_lock);
file = file_close_fd_locked(files, fd);
spin_unlock(&files->file_lock);
if (!file)
return -EBADF;
return filp_close(file, files);
}
EXPORT_SYMBOL(close_fd); /* for ksys_close() */
/**
* last_fd - return last valid index into fd table
* @fdt: File descriptor table.
*
* Context: Either rcu read lock or files_lock must be held.
*
* Returns: Last valid index into fdtable.
*/
static inline unsigned last_fd(struct fdtable *fdt)
{
return fdt->max_fds - 1;
}
static inline void __range_cloexec(struct files_struct *cur_fds,
unsigned int fd, unsigned int max_fd)
{
struct fdtable *fdt;
/* make sure we're using the correct maximum value */
spin_lock(&cur_fds->file_lock);
fdt = files_fdtable(cur_fds);
max_fd = min(last_fd(fdt), max_fd);
if (fd <= max_fd)
bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);
spin_unlock(&cur_fds->file_lock);
}
static inline void __range_close(struct files_struct *files, unsigned int fd,
unsigned int max_fd)
{
struct file *file;
unsigned n;
spin_lock(&files->file_lock);
n = last_fd(files_fdtable(files));
max_fd = min(max_fd, n);
for (; fd <= max_fd; fd++) {
file = file_close_fd_locked(files, fd);
if (file) {
spin_unlock(&files->file_lock);
filp_close(file, files);
cond_resched();
spin_lock(&files->file_lock);
} else if (need_resched()) {
spin_unlock(&files->file_lock);
cond_resched();
spin_lock(&files->file_lock);
}
}
spin_unlock(&files->file_lock);
}
/**
* __close_range() - Close all file descriptors in a given range.
*
* @fd: starting file descriptor to close
* @max_fd: last file descriptor to close
* @flags: CLOSE_RANGE flags.
*
* This closes a range of file descriptors. All file descriptors
* from @fd up to and including @max_fd are closed.
*/
int __close_range(unsigned fd, unsigned max_fd, unsigned int flags)
{
struct task_struct *me = current;
struct files_struct *cur_fds = me->files, *fds = NULL;
if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC))
return -EINVAL;
if (fd > max_fd)
return -EINVAL;
if (flags & CLOSE_RANGE_UNSHARE) {
int ret;
unsigned int max_unshare_fds = NR_OPEN_MAX;
/*
* If the caller requested all fds to be made cloexec we always
* copy all of the file descriptors since they still want to
* use them.
*/
if (!(flags & CLOSE_RANGE_CLOEXEC)) {
/*
* If the requested range is greater than the current
* maximum, we're closing everything so only copy all
* file descriptors beneath the lowest file descriptor.
*/
rcu_read_lock();
if (max_fd >= last_fd(files_fdtable(cur_fds)))
max_unshare_fds = fd;
rcu_read_unlock();
}
ret = unshare_fd(CLONE_FILES, max_unshare_fds, &fds);
if (ret)
return ret;
/*
* We used to share our file descriptor table, and have now
* created a private one, make sure we're using it below.
*/
if (fds)
swap(cur_fds, fds);
}
if (flags & CLOSE_RANGE_CLOEXEC)
__range_cloexec(cur_fds, fd, max_fd);
else
__range_close(cur_fds, fd, max_fd);
if (fds) {
/*
* We're done closing the files we were supposed to. Time to install
* the new file descriptor table and drop the old one.
*/
task_lock(me);
me->files = cur_fds;
task_unlock(me);
put_files_struct(fds);
}
return 0;
}
/**
* file_close_fd - return file associated with fd
* @fd: file descriptor to retrieve file for
*
* Doesn't take a separate reference count.
*
* Returns: The file associated with @fd (NULL if @fd is not open)
*/
struct file *file_close_fd(unsigned int fd)
{
struct files_struct *files = current->files;
struct file *file;
spin_lock(&files->file_lock);
file = file_close_fd_locked(files, fd);
spin_unlock(&files->file_lock);
return file;
}
void do_close_on_exec(struct files_struct *files)
{
unsigned i;
struct fdtable *fdt;
/* exec unshares first */
spin_lock(&files->file_lock);
for (i = 0; ; i++) {
unsigned long set;
unsigned fd = i * BITS_PER_LONG;
fdt = files_fdtable(files);
if (fd >= fdt->max_fds)
break;
set = fdt->close_on_exec[i];
if (!set)
continue;
fdt->close_on_exec[i] = 0;
for ( ; set ; fd++, set >>= 1) {
struct file *file;
if (!(set & 1))
continue;
file = fdt->fd[fd];
if (!file)
continue;
rcu_assign_pointer(fdt->fd[fd], NULL);
__put_unused_fd(files, fd);
spin_unlock(&files->file_lock);
filp_close(file, files);
cond_resched();
spin_lock(&files->file_lock);
}
}
spin_unlock(&files->file_lock);
}
static struct file *__get_file_rcu(struct file __rcu **f)
{
struct file __rcu *file;
struct file __rcu *file_reloaded;
struct file __rcu *file_reloaded_cmp;
file = rcu_dereference_raw(*f);
if (!file)
return NULL;
if (unlikely(!atomic_long_inc_not_zero(&file->f_count)))
return ERR_PTR(-EAGAIN);
file_reloaded = rcu_dereference_raw(*f);
/*
* Ensure that all accesses have a dependency on the load from
* rcu_dereference_raw() above so we get correct ordering
* between reuse/allocation and the pointer check below.
*/
file_reloaded_cmp = file_reloaded;
OPTIMIZER_HIDE_VAR(file_reloaded_cmp);
/*
* atomic_long_inc_not_zero() above provided a full memory
* barrier when we acquired a reference.
*
* This is paired with the write barrier from assigning to the
* __rcu protected file pointer so that if that pointer still
* matches the current file, we know we have successfully
* acquired a reference to the right file.
*
* If the pointers don't match the file has been reallocated by
* SLAB_TYPESAFE_BY_RCU.
*/
if (file == file_reloaded_cmp)
return file_reloaded;
fput(file);
return ERR_PTR(-EAGAIN);
}
/**
* get_file_rcu - try go get a reference to a file under rcu
* @f: the file to get a reference on
*
* This function tries to get a reference on @f carefully verifying that
* @f hasn't been reused.
*
* This function should rarely have to be used and only by users who
* understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
*
* Return: Returns @f with the reference count increased or NULL.
*/
struct file *get_file_rcu(struct file __rcu **f)
{
for (;;) {
struct file __rcu *file;
file = __get_file_rcu(f);
if (unlikely(!file))
return NULL;
if (unlikely(IS_ERR(file)))
continue;
return file;
}
}
EXPORT_SYMBOL_GPL(get_file_rcu);
/**
* get_file_active - try go get a reference to a file
* @f: the file to get a reference on
*
* In contast to get_file_rcu() the pointer itself isn't part of the
* reference counting.
*
* This function should rarely have to be used and only by users who
* understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
*
* Return: Returns @f with the reference count increased or NULL.
*/
struct file *get_file_active(struct file **f)
{
struct file __rcu *file;
rcu_read_lock();
file = __get_file_rcu(f);
rcu_read_unlock();
if (IS_ERR(file))
file = NULL;
return file;
}
EXPORT_SYMBOL_GPL(get_file_active);
static inline struct file *__fget_files_rcu(struct files_struct *files,
unsigned int fd, fmode_t mask)
{
for (;;) {
struct file *file;
struct fdtable *fdt = rcu_dereference_raw(files->fdt);
struct file __rcu **fdentry;
unsigned long nospec_mask;
/* Mask is a 0 for invalid fd's, ~0 for valid ones */
nospec_mask = array_index_mask_nospec(fd, fdt->max_fds);
/*
* fdentry points to the 'fd' offset, or fdt->fd[0].
* Loading from fdt->fd[0] is always safe, because the
* array always exists.
*/
fdentry = fdt->fd + (fd & nospec_mask);
/* Do the load, then mask any invalid result */
file = rcu_dereference_raw(*fdentry);
file = (void *)(nospec_mask & (unsigned long)file);
if (unlikely(!file))
return NULL;
/*
* Ok, we have a file pointer that was valid at
* some point, but it might have become stale since.
*
* We need to confirm it by incrementing the refcount
* and then check the lookup again.
*
* atomic_long_inc_not_zero() gives us a full memory
* barrier. We only really need an 'acquire' one to
* protect the loads below, but we don't have that.
*/
if (unlikely(!atomic_long_inc_not_zero(&file->f_count)))
continue;
/*
* Such a race can take two forms:
*
* (a) the file ref already went down to zero and the
* file hasn't been reused yet or the file count
* isn't zero but the file has already been reused.
*
* (b) the file table entry has changed under us.
* Note that we don't need to re-check the 'fdt->fd'
* pointer having changed, because it always goes
* hand-in-hand with 'fdt'.
*
* If so, we need to put our ref and try again.
*/
if (unlikely(file != rcu_dereference_raw(*fdentry)) || unlikely(rcu_dereference_raw(files->fdt) != fdt)) { fput(file);
continue;
}
/*
* This isn't the file we're looking for or we're not
* allowed to get a reference to it.
*/
if (unlikely(file->f_mode & mask)) { fput(file);
return NULL;
}
/*
* Ok, we have a ref to the file, and checked that it
* still exists.
*/
return file;
}
}
static struct file *__fget_files(struct files_struct *files, unsigned int fd,
fmode_t mask)
{
struct file *file;
rcu_read_lock(); file = __fget_files_rcu(files, fd, mask); rcu_read_unlock();
return file;
}
static inline struct file *__fget(unsigned int fd, fmode_t mask)
{
return __fget_files(current->files, fd, mask);
}
struct file *fget(unsigned int fd)
{
return __fget(fd, FMODE_PATH);
}
EXPORT_SYMBOL(fget);
struct file *fget_raw(unsigned int fd)
{
return __fget(fd, 0);
}
EXPORT_SYMBOL(fget_raw);
struct file *fget_task(struct task_struct *task, unsigned int fd)
{
struct file *file = NULL;
task_lock(task);
if (task->files)
file = __fget_files(task->files, fd, 0);
task_unlock(task);
return file;
}
struct file *lookup_fdget_rcu(unsigned int fd)
{
return __fget_files_rcu(current->files, fd, 0);
}
EXPORT_SYMBOL_GPL(lookup_fdget_rcu);
struct file *task_lookup_fdget_rcu(struct task_struct *task, unsigned int fd)
{
/* Must be called with rcu_read_lock held */
struct files_struct *files;
struct file *file = NULL;
task_lock(task);
files = task->files;
if (files)
file = __fget_files_rcu(files, fd, 0);
task_unlock(task);
return file;
}
struct file *task_lookup_next_fdget_rcu(struct task_struct *task, unsigned int *ret_fd)
{
/* Must be called with rcu_read_lock held */
struct files_struct *files;
unsigned int fd = *ret_fd;
struct file *file = NULL;
task_lock(task);
files = task->files;
if (files) {
for (; fd < files_fdtable(files)->max_fds; fd++) {
file = __fget_files_rcu(files, fd, 0);
if (file)
break;
}
}
task_unlock(task);
*ret_fd = fd;
return file;
}
EXPORT_SYMBOL(task_lookup_next_fdget_rcu);
/*
* Lightweight file lookup - no refcnt increment if fd table isn't shared.
*
* You can use this instead of fget if you satisfy all of the following
* conditions:
* 1) You must call fput_light before exiting the syscall and returning control
* to userspace (i.e. you cannot remember the returned struct file * after
* returning to userspace).
* 2) You must not call filp_close on the returned struct file * in between
* calls to fget_light and fput_light.
* 3) You must not clone the current task in between the calls to fget_light
* and fput_light.
*
* The fput_needed flag returned by fget_light should be passed to the
* corresponding fput_light.
*/
static unsigned long __fget_light(unsigned int fd, fmode_t mask)
{
struct files_struct *files = current->files;
struct file *file;
/*
* If another thread is concurrently calling close_fd() followed
* by put_files_struct(), we must not observe the old table
* entry combined with the new refcount - otherwise we could
* return a file that is concurrently being freed.
*
* atomic_read_acquire() pairs with atomic_dec_and_test() in
* put_files_struct().
*/
if (likely(atomic_read_acquire(&files->count) == 1)) {
file = files_lookup_fd_raw(files, fd); if (!file || unlikely(file->f_mode & mask))
return 0;
return (unsigned long)file;
} else {
file = __fget_files(files, fd, mask);
if (!file)
return 0;
return FDPUT_FPUT | (unsigned long)file;
}
}
unsigned long __fdget(unsigned int fd)
{
return __fget_light(fd, FMODE_PATH);
}
EXPORT_SYMBOL(__fdget);
unsigned long __fdget_raw(unsigned int fd)
{
return __fget_light(fd, 0);
}
/*
* Try to avoid f_pos locking. We only need it if the
* file is marked for FMODE_ATOMIC_POS, and it can be
* accessed multiple ways.
*
* Always do it for directories, because pidfd_getfd()
* can make a file accessible even if it otherwise would
* not be, and for directories this is a correctness
* issue, not a "POSIX requirement".
*/
static inline bool file_needs_f_pos_lock(struct file *file)
{
return (file->f_mode & FMODE_ATOMIC_POS) &&
(file_count(file) > 1 || file->f_op->iterate_shared);
}
unsigned long __fdget_pos(unsigned int fd)
{
unsigned long v = __fdget(fd); struct file *file = (struct file *)(v & ~3); if (file && file_needs_f_pos_lock(file)) { v |= FDPUT_POS_UNLOCK;
mutex_lock(&file->f_pos_lock);
}
return v;
}
void __f_unlock_pos(struct file *f)
{
mutex_unlock(&f->f_pos_lock);
}
/*
* We only lock f_pos if we have threads or if the file might be
* shared with another process. In both cases we'll have an elevated
* file count (done either by fdget() or by fork()).
*/
void set_close_on_exec(unsigned int fd, int flag)
{
struct files_struct *files = current->files;
struct fdtable *fdt;
spin_lock(&files->file_lock);
fdt = files_fdtable(files); if (flag) __set_close_on_exec(fd, fdt);
else
__clear_close_on_exec(fd, fdt); spin_unlock(&files->file_lock);
}
bool get_close_on_exec(unsigned int fd)
{
struct files_struct *files = current->files;
struct fdtable *fdt;
bool res;
rcu_read_lock();
fdt = files_fdtable(files);
res = close_on_exec(fd, fdt);
rcu_read_unlock();
return res;
}
static int do_dup2(struct files_struct *files,
struct file *file, unsigned fd, unsigned flags)
__releases(&files->file_lock)
{
struct file *tofree;
struct fdtable *fdt;
/*
* We need to detect attempts to do dup2() over allocated but still
* not finished descriptor. NB: OpenBSD avoids that at the price of
* extra work in their equivalent of fget() - they insert struct
* file immediately after grabbing descriptor, mark it larval if
* more work (e.g. actual opening) is needed and make sure that
* fget() treats larval files as absent. Potentially interesting,
* but while extra work in fget() is trivial, locking implications
* and amount of surgery on open()-related paths in VFS are not.
* FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
* deadlocks in rather amusing ways, AFAICS. All of that is out of
* scope of POSIX or SUS, since neither considers shared descriptor
* tables and this condition does not arise without those.
*/
fdt = files_fdtable(files);
tofree = fdt->fd[fd];
if (!tofree && fd_is_open(fd, fdt))
goto Ebusy;
get_file(file);
rcu_assign_pointer(fdt->fd[fd], file);
__set_open_fd(fd, fdt);
if (flags & O_CLOEXEC)
__set_close_on_exec(fd, fdt);
else
__clear_close_on_exec(fd, fdt);
spin_unlock(&files->file_lock);
if (tofree)
filp_close(tofree, files);
return fd;
Ebusy:
spin_unlock(&files->file_lock);
return -EBUSY;
}
int replace_fd(unsigned fd, struct file *file, unsigned flags)
{
int err;
struct files_struct *files = current->files;
if (!file)
return close_fd(fd);
if (fd >= rlimit(RLIMIT_NOFILE))
return -EBADF;
spin_lock(&files->file_lock);
err = expand_files(files, fd);
if (unlikely(err < 0))
goto out_unlock;
return do_dup2(files, file, fd, flags);
out_unlock:
spin_unlock(&files->file_lock);
return err;
}
/**
* receive_fd() - Install received file into file descriptor table
* @file: struct file that was received from another process
* @ufd: __user pointer to write new fd number to
* @o_flags: the O_* flags to apply to the new fd entry
*
* Installs a received file into the file descriptor table, with appropriate
* checks and count updates. Optionally writes the fd number to userspace, if
* @ufd is non-NULL.
*
* This helper handles its own reference counting of the incoming
* struct file.
*
* Returns newly install fd or -ve on error.
*/
int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
{
int new_fd;
int error;
error = security_file_receive(file);
if (error)
return error;
new_fd = get_unused_fd_flags(o_flags);
if (new_fd < 0)
return new_fd;
if (ufd) {
error = put_user(new_fd, ufd);
if (error) {
put_unused_fd(new_fd);
return error;
}
}
fd_install(new_fd, get_file(file));
__receive_sock(file);
return new_fd;
}
EXPORT_SYMBOL_GPL(receive_fd);
int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags)
{
int error;
error = security_file_receive(file);
if (error)
return error;
error = replace_fd(new_fd, file, o_flags);
if (error)
return error;
__receive_sock(file);
return new_fd;
}
static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
{
int err = -EBADF;
struct file *file;
struct files_struct *files = current->files;
if ((flags & ~O_CLOEXEC) != 0)
return -EINVAL;
if (unlikely(oldfd == newfd))
return -EINVAL;
if (newfd >= rlimit(RLIMIT_NOFILE))
return -EBADF;
spin_lock(&files->file_lock);
err = expand_files(files, newfd);
file = files_lookup_fd_locked(files, oldfd);
if (unlikely(!file))
goto Ebadf;
if (unlikely(err < 0)) {
if (err == -EMFILE)
goto Ebadf;
goto out_unlock;
}
return do_dup2(files, file, newfd, flags);
Ebadf:
err = -EBADF;
out_unlock:
spin_unlock(&files->file_lock);
return err;
}
SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
{
return ksys_dup3(oldfd, newfd, flags);
}
SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
{
if (unlikely(newfd == oldfd)) { /* corner case */
struct files_struct *files = current->files;
struct file *f;
int retval = oldfd;
rcu_read_lock();
f = __fget_files_rcu(files, oldfd, 0);
if (!f)
retval = -EBADF;
rcu_read_unlock();
if (f)
fput(f);
return retval;
}
return ksys_dup3(oldfd, newfd, 0);
}
SYSCALL_DEFINE1(dup, unsigned int, fildes)
{
int ret = -EBADF;
struct file *file = fget_raw(fildes);
if (file) {
ret = get_unused_fd_flags(0);
if (ret >= 0)
fd_install(ret, file);
else
fput(file);
}
return ret;
}
int f_dupfd(unsigned int from, struct file *file, unsigned flags)
{
unsigned long nofile = rlimit(RLIMIT_NOFILE);
int err;
if (from >= nofile)
return -EINVAL;
err = alloc_fd(from, nofile, flags);
if (err >= 0) {
get_file(file);
fd_install(err, file);
}
return err;
}
int iterate_fd(struct files_struct *files, unsigned n,
int (*f)(const void *, struct file *, unsigned),
const void *p)
{
struct fdtable *fdt;
int res = 0;
if (!files)
return 0;
spin_lock(&files->file_lock);
for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
struct file *file;
file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
if (!file)
continue;
res = f(p, file, n);
if (res)
break;
}
spin_unlock(&files->file_lock);
return res;
}
EXPORT_SYMBOL(iterate_fd);
/*
* Created: Fri Jan 19 10:48:35 2001 by faith@acm.org
*
* Copyright 2001 VA Linux Systems, Inc., Sunnyvale, California.
* All Rights Reserved.
*
* Author Rickard E. (Rik) Faith <faith@valinux.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <linux/debugfs.h>
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/mount.h>
#include <linux/pseudo_fs.h>
#include <linux/slab.h>
#include <linux/srcu.h>
#include <drm/drm_accel.h>
#include <drm/drm_cache.h>
#include <drm/drm_client.h>
#include <drm/drm_color_mgmt.h>
#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_managed.h>
#include <drm/drm_mode_object.h>
#include <drm/drm_print.h>
#include <drm/drm_privacy_screen_machine.h>
#include "drm_crtc_internal.h"
#include "drm_internal.h"
MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
MODULE_DESCRIPTION("DRM shared core routines");
MODULE_LICENSE("GPL and additional rights");
static DEFINE_SPINLOCK(drm_minor_lock);
static struct idr drm_minors_idr;
/*
* If the drm core fails to init for whatever reason,
* we should prevent any drivers from registering with it.
* It's best to check this at drm_dev_init(), as some drivers
* prefer to embed struct drm_device into their own device
* structure and call drm_dev_init() themselves.
*/
static bool drm_core_init_complete;
static struct dentry *drm_debugfs_root;
DEFINE_STATIC_SRCU(drm_unplug_srcu);
/*
* DRM Minors
* A DRM device can provide several char-dev interfaces on the DRM-Major. Each
* of them is represented by a drm_minor object. Depending on the capabilities
* of the device-driver, different interfaces are registered.
*
* Minors can be accessed via dev->$minor_name. This pointer is either
* NULL or a valid drm_minor pointer and stays valid as long as the device is
* valid. This means, DRM minors have the same life-time as the underlying
* device. However, this doesn't mean that the minor is active. Minors are
* registered and unregistered dynamically according to device-state.
*/
static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
enum drm_minor_type type)
{
switch (type) {
case DRM_MINOR_PRIMARY:
return &dev->primary;
case DRM_MINOR_RENDER:
return &dev->render;
case DRM_MINOR_ACCEL:
return &dev->accel;
default:
BUG();
}
}
static void drm_minor_alloc_release(struct drm_device *dev, void *data)
{
struct drm_minor *minor = data;
unsigned long flags;
WARN_ON(dev != minor->dev);
put_device(minor->kdev);
if (minor->type == DRM_MINOR_ACCEL) {
accel_minor_remove(minor->index);
} else {
spin_lock_irqsave(&drm_minor_lock, flags);
idr_remove(&drm_minors_idr, minor->index);
spin_unlock_irqrestore(&drm_minor_lock, flags);
}
}
static int drm_minor_alloc(struct drm_device *dev, enum drm_minor_type type)
{
struct drm_minor *minor;
unsigned long flags;
int r;
minor = drmm_kzalloc(dev, sizeof(*minor), GFP_KERNEL);
if (!minor)
return -ENOMEM;
minor->type = type;
minor->dev = dev;
idr_preload(GFP_KERNEL);
if (type == DRM_MINOR_ACCEL) {
r = accel_minor_alloc();
} else {
spin_lock_irqsave(&drm_minor_lock, flags);
r = idr_alloc(&drm_minors_idr,
NULL,
64 * type,
64 * (type + 1),
GFP_NOWAIT);
spin_unlock_irqrestore(&drm_minor_lock, flags);
}
idr_preload_end();
if (r < 0)
return r;
minor->index = r;
r = drmm_add_action_or_reset(dev, drm_minor_alloc_release, minor);
if (r)
return r;
minor->kdev = drm_sysfs_minor_alloc(minor);
if (IS_ERR(minor->kdev))
return PTR_ERR(minor->kdev);
*drm_minor_get_slot(dev, type) = minor;
return 0;
}
static int drm_minor_register(struct drm_device *dev, enum drm_minor_type type)
{
struct drm_minor *minor;
unsigned long flags;
int ret;
DRM_DEBUG("\n");
minor = *drm_minor_get_slot(dev, type);
if (!minor)
return 0;
if (minor->type != DRM_MINOR_ACCEL) {
ret = drm_debugfs_register(minor, minor->index,
drm_debugfs_root);
if (ret) {
DRM_ERROR("DRM: Failed to initialize /sys/kernel/debug/dri.\n");
goto err_debugfs;
}
}
ret = device_add(minor->kdev);
if (ret)
goto err_debugfs;
/* replace NULL with @minor so lookups will succeed from now on */
if (minor->type == DRM_MINOR_ACCEL) {
accel_minor_replace(minor, minor->index);
} else {
spin_lock_irqsave(&drm_minor_lock, flags);
idr_replace(&drm_minors_idr, minor, minor->index);
spin_unlock_irqrestore(&drm_minor_lock, flags);
}
DRM_DEBUG("new minor registered %d\n", minor->index);
return 0;
err_debugfs:
drm_debugfs_unregister(minor);
return ret;
}
static void drm_minor_unregister(struct drm_device *dev, enum drm_minor_type type)
{
struct drm_minor *minor;
unsigned long flags;
minor = *drm_minor_get_slot(dev, type); if (!minor || !device_is_registered(minor->kdev))
return;
/* replace @minor with NULL so lookups will fail from now on */
if (minor->type == DRM_MINOR_ACCEL) {
accel_minor_replace(NULL, minor->index);
} else {
spin_lock_irqsave(&drm_minor_lock, flags);
idr_replace(&drm_minors_idr, NULL, minor->index);
spin_unlock_irqrestore(&drm_minor_lock, flags);
}
device_del(minor->kdev);
dev_set_drvdata(minor->kdev, NULL); /* safety belt */
drm_debugfs_unregister(minor);
}
/*
* Looks up the given minor-ID and returns the respective DRM-minor object. The
* refence-count of the underlying device is increased so you must release this
* object with drm_minor_release().
*
* As long as you hold this minor, it is guaranteed that the object and the
* minor->dev pointer will stay valid! However, the device may get unplugged and
* unregistered while you hold the minor.
*/
struct drm_minor *drm_minor_acquire(unsigned int minor_id)
{
struct drm_minor *minor;
unsigned long flags;
spin_lock_irqsave(&drm_minor_lock, flags);
minor = idr_find(&drm_minors_idr, minor_id);
if (minor)
drm_dev_get(minor->dev);
spin_unlock_irqrestore(&drm_minor_lock, flags);
if (!minor) {
return ERR_PTR(-ENODEV);
} else if (drm_dev_is_unplugged(minor->dev)) {
drm_dev_put(minor->dev);
return ERR_PTR(-ENODEV);
}
return minor;
}
void drm_minor_release(struct drm_minor *minor)
{
drm_dev_put(minor->dev);
}
/**
* DOC: driver instance overview
*
* A device instance for a drm driver is represented by &struct drm_device. This
* is allocated and initialized with devm_drm_dev_alloc(), usually from
* bus-specific ->probe() callbacks implemented by the driver. The driver then
* needs to initialize all the various subsystems for the drm device like memory
* management, vblank handling, modesetting support and initial output
* configuration plus obviously initialize all the corresponding hardware bits.
* Finally when everything is up and running and ready for userspace the device
* instance can be published using drm_dev_register().
*
* There is also deprecated support for initializing device instances using
* bus-specific helpers and the &drm_driver.load callback. But due to
* backwards-compatibility needs the device instance have to be published too
* early, which requires unpretty global locking to make safe and is therefore
* only support for existing drivers not yet converted to the new scheme.
*
* When cleaning up a device instance everything needs to be done in reverse:
* First unpublish the device instance with drm_dev_unregister(). Then clean up
* any other resources allocated at device initialization and drop the driver's
* reference to &drm_device using drm_dev_put().
*
* Note that any allocation or resource which is visible to userspace must be
* released only when the final drm_dev_put() is called, and not when the
* driver is unbound from the underlying physical struct &device. Best to use
* &drm_device managed resources with drmm_add_action(), drmm_kmalloc() and
* related functions.
*
* devres managed resources like devm_kmalloc() can only be used for resources
* directly related to the underlying hardware device, and only used in code
* paths fully protected by drm_dev_enter() and drm_dev_exit().
*
* Display driver example
* ~~~~~~~~~~~~~~~~~~~~~~
*
* The following example shows a typical structure of a DRM display driver.
* The example focus on the probe() function and the other functions that is
* almost always present and serves as a demonstration of devm_drm_dev_alloc().
*
* .. code-block:: c
*
* struct driver_device {
* struct drm_device drm;
* void *userspace_facing;
* struct clk *pclk;
* };
*
* static const struct drm_driver driver_drm_driver = {
* [...]
* };
*
* static int driver_probe(struct platform_device *pdev)
* {
* struct driver_device *priv;
* struct drm_device *drm;
* int ret;
*
* priv = devm_drm_dev_alloc(&pdev->dev, &driver_drm_driver,
* struct driver_device, drm);
* if (IS_ERR(priv))
* return PTR_ERR(priv);
* drm = &priv->drm;
*
* ret = drmm_mode_config_init(drm);
* if (ret)
* return ret;
*
* priv->userspace_facing = drmm_kzalloc(..., GFP_KERNEL);
* if (!priv->userspace_facing)
* return -ENOMEM;
*
* priv->pclk = devm_clk_get(dev, "PCLK");
* if (IS_ERR(priv->pclk))
* return PTR_ERR(priv->pclk);
*
* // Further setup, display pipeline etc
*
* platform_set_drvdata(pdev, drm);
*
* drm_mode_config_reset(drm);
*
* ret = drm_dev_register(drm);
* if (ret)
* return ret;
*
* drm_fbdev_generic_setup(drm, 32);
*
* return 0;
* }
*
* // This function is called before the devm_ resources are released
* static int driver_remove(struct platform_device *pdev)
* {
* struct drm_device *drm = platform_get_drvdata(pdev);
*
* drm_dev_unregister(drm);
* drm_atomic_helper_shutdown(drm)
*
* return 0;
* }
*
* // This function is called on kernel restart and shutdown
* static void driver_shutdown(struct platform_device *pdev)
* {
* drm_atomic_helper_shutdown(platform_get_drvdata(pdev));
* }
*
* static int __maybe_unused driver_pm_suspend(struct device *dev)
* {
* return drm_mode_config_helper_suspend(dev_get_drvdata(dev));
* }
*
* static int __maybe_unused driver_pm_resume(struct device *dev)
* {
* drm_mode_config_helper_resume(dev_get_drvdata(dev));
*
* return 0;
* }
*
* static const struct dev_pm_ops driver_pm_ops = {
* SET_SYSTEM_SLEEP_PM_OPS(driver_pm_suspend, driver_pm_resume)
* };
*
* static struct platform_driver driver_driver = {
* .driver = {
* [...]
* .pm = &driver_pm_ops,
* },
* .probe = driver_probe,
* .remove = driver_remove,
* .shutdown = driver_shutdown,
* };
* module_platform_driver(driver_driver);
*
* Drivers that want to support device unplugging (USB, DT overlay unload) should
* use drm_dev_unplug() instead of drm_dev_unregister(). The driver must protect
* regions that is accessing device resources to prevent use after they're
* released. This is done using drm_dev_enter() and drm_dev_exit(). There is one
* shortcoming however, drm_dev_unplug() marks the drm_device as unplugged before
* drm_atomic_helper_shutdown() is called. This means that if the disable code
* paths are protected, they will not run on regular driver module unload,
* possibly leaving the hardware enabled.
*/
/**
* drm_put_dev - Unregister and release a DRM device
* @dev: DRM device
*
* Called at module unload time or when a PCI device is unplugged.
*
* Cleans up all DRM device, calling drm_lastclose().
*
* Note: Use of this function is deprecated. It will eventually go away
* completely. Please use drm_dev_unregister() and drm_dev_put() explicitly
* instead to make sure that the device isn't userspace accessible any more
* while teardown is in progress, ensuring that userspace can't access an
* inconsistent state.
*/
void drm_put_dev(struct drm_device *dev)
{
DRM_DEBUG("\n");
if (!dev) {
DRM_ERROR("cleanup called no dev\n");
return;
}
drm_dev_unregister(dev);
drm_dev_put(dev);
}
EXPORT_SYMBOL(drm_put_dev);
/**
* drm_dev_enter - Enter device critical section
* @dev: DRM device
* @idx: Pointer to index that will be passed to the matching drm_dev_exit()
*
* This function marks and protects the beginning of a section that should not
* be entered after the device has been unplugged. The section end is marked
* with drm_dev_exit(). Calls to this function can be nested.
*
* Returns:
* True if it is OK to enter the section, false otherwise.
*/
bool drm_dev_enter(struct drm_device *dev, int *idx)
{
*idx = srcu_read_lock(&drm_unplug_srcu);
if (dev->unplugged) {
srcu_read_unlock(&drm_unplug_srcu, *idx);
return false;
}
return true;
}
EXPORT_SYMBOL(drm_dev_enter);
/**
* drm_dev_exit - Exit device critical section
* @idx: index returned from drm_dev_enter()
*
* This function marks the end of a section that should not be entered after
* the device has been unplugged.
*/
void drm_dev_exit(int idx)
{
srcu_read_unlock(&drm_unplug_srcu, idx);
}
EXPORT_SYMBOL(drm_dev_exit);
/**
* drm_dev_unplug - unplug a DRM device
* @dev: DRM device
*
* This unplugs a hotpluggable DRM device, which makes it inaccessible to
* userspace operations. Entry-points can use drm_dev_enter() and
* drm_dev_exit() to protect device resources in a race free manner. This
* essentially unregisters the device like drm_dev_unregister(), but can be
* called while there are still open users of @dev.
*/
void drm_dev_unplug(struct drm_device *dev)
{
/*
* After synchronizing any critical read section is guaranteed to see
* the new value of ->unplugged, and any critical section which might
* still have seen the old value of ->unplugged is guaranteed to have
* finished.
*/
dev->unplugged = true;
synchronize_srcu(&drm_unplug_srcu);
drm_dev_unregister(dev);
/* Clear all CPU mappings pointing to this device */
unmap_mapping_range(dev->anon_inode->i_mapping, 0, 0, 1);
}
EXPORT_SYMBOL(drm_dev_unplug);
/*
* DRM internal mount
* We want to be able to allocate our own "struct address_space" to control
* memory-mappings in VRAM (or stolen RAM, ...). However, core MM does not allow
* stand-alone address_space objects, so we need an underlying inode. As there
* is no way to allocate an independent inode easily, we need a fake internal
* VFS mount-point.
*
* The drm_fs_inode_new() function allocates a new inode, drm_fs_inode_free()
* frees it again. You are allowed to use iget() and iput() to get references to
* the inode. But each drm_fs_inode_new() call must be paired with exactly one
* drm_fs_inode_free() call (which does not have to be the last iput()).
* We use drm_fs_inode_*() to manage our internal VFS mount-point and share it
* between multiple inode-users. You could, technically, call
* iget() + drm_fs_inode_free() directly after alloc and sometime later do an
* iput(), but this way you'd end up with a new vfsmount for each inode.
*/
static int drm_fs_cnt;
static struct vfsmount *drm_fs_mnt;
static int drm_fs_init_fs_context(struct fs_context *fc)
{
return init_pseudo(fc, 0x010203ff) ? 0 : -ENOMEM;
}
static struct file_system_type drm_fs_type = {
.name = "drm",
.owner = THIS_MODULE,
.init_fs_context = drm_fs_init_fs_context,
.kill_sb = kill_anon_super,
};
static struct inode *drm_fs_inode_new(void)
{
struct inode *inode;
int r;
r = simple_pin_fs(&drm_fs_type, &drm_fs_mnt, &drm_fs_cnt);
if (r < 0) {
DRM_ERROR("Cannot mount pseudo fs: %d\n", r);
return ERR_PTR(r);
}
inode = alloc_anon_inode(drm_fs_mnt->mnt_sb);
if (IS_ERR(inode))
simple_release_fs(&drm_fs_mnt, &drm_fs_cnt);
return inode;
}
static void drm_fs_inode_free(struct inode *inode)
{
if (inode) {
iput(inode);
simple_release_fs(&drm_fs_mnt, &drm_fs_cnt);
}
}
/**
* DOC: component helper usage recommendations
*
* DRM drivers that drive hardware where a logical device consists of a pile of
* independent hardware blocks are recommended to use the :ref:`component helper
* library<component>`. For consistency and better options for code reuse the
* following guidelines apply:
*
* - The entire device initialization procedure should be run from the
* &component_master_ops.master_bind callback, starting with
* devm_drm_dev_alloc(), then binding all components with
* component_bind_all() and finishing with drm_dev_register().
*
* - The opaque pointer passed to all components through component_bind_all()
* should point at &struct drm_device of the device instance, not some driver
* specific private structure.
*
* - The component helper fills the niche where further standardization of
* interfaces is not practical. When there already is, or will be, a
* standardized interface like &drm_bridge or &drm_panel, providing its own
* functions to find such components at driver load time, like
* drm_of_find_panel_or_bridge(), then the component helper should not be
* used.
*/
static void drm_dev_init_release(struct drm_device *dev, void *res)
{
drm_fs_inode_free(dev->anon_inode);
put_device(dev->dev);
/* Prevent use-after-free in drm_managed_release when debugging is
* enabled. Slightly awkward, but can't really be helped. */
dev->dev = NULL;
mutex_destroy(&dev->master_mutex);
mutex_destroy(&dev->clientlist_mutex);
mutex_destroy(&dev->filelist_mutex);
mutex_destroy(&dev->struct_mutex);
}
static int drm_dev_init(struct drm_device *dev,
const struct drm_driver *driver,
struct device *parent)
{
struct inode *inode;
int ret;
if (!drm_core_init_complete) {
DRM_ERROR("DRM core is not initialized\n");
return -ENODEV;
}
if (WARN_ON(!parent))
return -EINVAL;
kref_init(&dev->ref);
dev->dev = get_device(parent);
dev->driver = driver;
INIT_LIST_HEAD(&dev->managed.resources);
spin_lock_init(&dev->managed.lock);
/* no per-device feature limits by default */
dev->driver_features = ~0u;
if (drm_core_check_feature(dev, DRIVER_COMPUTE_ACCEL) &&
(drm_core_check_feature(dev, DRIVER_RENDER) ||
drm_core_check_feature(dev, DRIVER_MODESET))) {
DRM_ERROR("DRM driver can't be both a compute acceleration and graphics driver\n");
return -EINVAL;
}
INIT_LIST_HEAD(&dev->filelist);
INIT_LIST_HEAD(&dev->filelist_internal);
INIT_LIST_HEAD(&dev->clientlist);
INIT_LIST_HEAD(&dev->vblank_event_list);
spin_lock_init(&dev->event_lock);
mutex_init(&dev->struct_mutex);
mutex_init(&dev->filelist_mutex);
mutex_init(&dev->clientlist_mutex);
mutex_init(&dev->master_mutex);
ret = drmm_add_action_or_reset(dev, drm_dev_init_release, NULL);
if (ret)
return ret;
inode = drm_fs_inode_new();
if (IS_ERR(inode)) {
ret = PTR_ERR(inode);
DRM_ERROR("Cannot allocate anonymous inode: %d\n", ret);
goto err;
}
dev->anon_inode = inode;
if (drm_core_check_feature(dev, DRIVER_COMPUTE_ACCEL)) {
ret = drm_minor_alloc(dev, DRM_MINOR_ACCEL);
if (ret)
goto err;
} else {
if (drm_core_check_feature(dev, DRIVER_RENDER)) {
ret = drm_minor_alloc(dev, DRM_MINOR_RENDER);
if (ret)
goto err;
}
ret = drm_minor_alloc(dev, DRM_MINOR_PRIMARY);
if (ret)
goto err;
}
if (drm_core_check_feature(dev, DRIVER_GEM)) {
ret = drm_gem_init(dev);
if (ret) {
DRM_ERROR("Cannot initialize graphics execution manager (GEM)\n");
goto err;
}
}
dev->unique = drmm_kstrdup(dev, dev_name(parent), GFP_KERNEL);
if (!dev->unique) {
ret = -ENOMEM;
goto err;
}
if (drm_core_check_feature(dev, DRIVER_COMPUTE_ACCEL))
accel_debugfs_init(dev);
else
drm_debugfs_dev_init(dev, drm_debugfs_root);
return 0;
err:
drm_managed_release(dev);
return ret;
}
static void devm_drm_dev_init_release(void *data)
{
drm_dev_put(data);
}
static int devm_drm_dev_init(struct device *parent,
struct drm_device *dev,
const struct drm_driver *driver)
{
int ret;
ret = drm_dev_init(dev, driver, parent);
if (ret)
return ret;
return devm_add_action_or_reset(parent,
devm_drm_dev_init_release, dev);
}
void *__devm_drm_dev_alloc(struct device *parent,
const struct drm_driver *driver,
size_t size, size_t offset)
{
void *container;
struct drm_device *drm;
int ret;
container = kzalloc(size, GFP_KERNEL);
if (!container)
return ERR_PTR(-ENOMEM);
drm = container + offset;
ret = devm_drm_dev_init(parent, drm, driver);
if (ret) {
kfree(container);
return ERR_PTR(ret);
}
drmm_add_final_kfree(drm, container);
return container;
}
EXPORT_SYMBOL(__devm_drm_dev_alloc);
/**
* drm_dev_alloc - Allocate new DRM device
* @driver: DRM driver to allocate device for
* @parent: Parent device object
*
* This is the deprecated version of devm_drm_dev_alloc(), which does not support
* subclassing through embedding the struct &drm_device in a driver private
* structure, and which does not support automatic cleanup through devres.
*
* RETURNS:
* Pointer to new DRM device, or ERR_PTR on failure.
*/
struct drm_device *drm_dev_alloc(const struct drm_driver *driver,
struct device *parent)
{
struct drm_device *dev;
int ret;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return ERR_PTR(-ENOMEM);
ret = drm_dev_init(dev, driver, parent);
if (ret) {
kfree(dev);
return ERR_PTR(ret);
}
drmm_add_final_kfree(dev, dev);
return dev;
}
EXPORT_SYMBOL(drm_dev_alloc);
static void drm_dev_release(struct kref *ref)
{
struct drm_device *dev = container_of(ref, struct drm_device, ref);
/* Just in case register/unregister was never called */
drm_debugfs_dev_fini(dev);
if (dev->driver->release)
dev->driver->release(dev);
drm_managed_release(dev);
kfree(dev->managed.final_kfree);
}
/**
* drm_dev_get - Take reference of a DRM device
* @dev: device to take reference of or NULL
*
* This increases the ref-count of @dev by one. You *must* already own a
* reference when calling this. Use drm_dev_put() to drop this reference
* again.
*
* This function never fails. However, this function does not provide *any*
* guarantee whether the device is alive or running. It only provides a
* reference to the object and the memory associated with it.
*/
void drm_dev_get(struct drm_device *dev)
{
if (dev)
kref_get(&dev->ref);
}
EXPORT_SYMBOL(drm_dev_get);
/**
* drm_dev_put - Drop reference of a DRM device
* @dev: device to drop reference of or NULL
*
* This decreases the ref-count of @dev by one. The device is destroyed if the
* ref-count drops to zero.
*/
void drm_dev_put(struct drm_device *dev)
{
if (dev)
kref_put(&dev->ref, drm_dev_release);
}
EXPORT_SYMBOL(drm_dev_put);
static int create_compat_control_link(struct drm_device *dev)
{
struct drm_minor *minor;
char *name;
int ret;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return 0;
minor = *drm_minor_get_slot(dev, DRM_MINOR_PRIMARY);
if (!minor)
return 0;
/*
* Some existing userspace out there uses the existing of the controlD*
* sysfs files to figure out whether it's a modeset driver. It only does
* readdir, hence a symlink is sufficient (and the least confusing
* option). Otherwise controlD* is entirely unused.
*
* Old controlD chardev have been allocated in the range
* 64-127.
*/
name = kasprintf(GFP_KERNEL, "controlD%d", minor->index + 64);
if (!name)
return -ENOMEM;
ret = sysfs_create_link(minor->kdev->kobj.parent,
&minor->kdev->kobj,
name);
kfree(name);
return ret;
}
static void remove_compat_control_link(struct drm_device *dev)
{
struct drm_minor *minor;
char *name;
if (!drm_core_check_feature(dev, DRIVER_MODESET))
return;
minor = *drm_minor_get_slot(dev, DRM_MINOR_PRIMARY);
if (!minor)
return;
name = kasprintf(GFP_KERNEL, "controlD%d", minor->index + 64);
if (!name)
return;
sysfs_remove_link(minor->kdev->kobj.parent, name);
kfree(name);
}
/**
* drm_dev_register - Register DRM device
* @dev: Device to register
* @flags: Flags passed to the driver's .load() function
*
* Register the DRM device @dev with the system, advertise device to user-space
* and start normal device operation. @dev must be initialized via drm_dev_init()
* previously.
*
* Never call this twice on any device!
*
* NOTE: To ensure backward compatibility with existing drivers method this
* function calls the &drm_driver.load method after registering the device
* nodes, creating race conditions. Usage of the &drm_driver.load methods is
* therefore deprecated, drivers must perform all initialization before calling
* drm_dev_register().
*
* RETURNS:
* 0 on success, negative error code on failure.
*/
int drm_dev_register(struct drm_device *dev, unsigned long flags)
{
const struct drm_driver *driver = dev->driver;
int ret;
if (!driver->load)
drm_mode_config_validate(dev);
WARN_ON(!dev->managed.final_kfree);
if (drm_dev_needs_global_mutex(dev))
mutex_lock(&drm_global_mutex);
if (drm_core_check_feature(dev, DRIVER_COMPUTE_ACCEL))
accel_debugfs_register(dev);
else
drm_debugfs_dev_register(dev);
ret = drm_minor_register(dev, DRM_MINOR_RENDER);
if (ret)
goto err_minors;
ret = drm_minor_register(dev, DRM_MINOR_PRIMARY);
if (ret)
goto err_minors;
ret = drm_minor_register(dev, DRM_MINOR_ACCEL);
if (ret)
goto err_minors;
ret = create_compat_control_link(dev);
if (ret)
goto err_minors;
dev->registered = true;
if (driver->load) {
ret = driver->load(dev, flags);
if (ret)
goto err_minors;
}
if (drm_core_check_feature(dev, DRIVER_MODESET)) {
ret = drm_modeset_register_all(dev);
if (ret)
goto err_unload;
}
DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
driver->name, driver->major, driver->minor,
driver->patchlevel, driver->date,
dev->dev ? dev_name(dev->dev) : "virtual device",
dev->primary ? dev->primary->index : dev->accel->index);
goto out_unlock;
err_unload:
if (dev->driver->unload)
dev->driver->unload(dev);
err_minors:
remove_compat_control_link(dev);
drm_minor_unregister(dev, DRM_MINOR_ACCEL);
drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
drm_minor_unregister(dev, DRM_MINOR_RENDER);
out_unlock:
if (drm_dev_needs_global_mutex(dev))
mutex_unlock(&drm_global_mutex);
return ret;
}
EXPORT_SYMBOL(drm_dev_register);
/**
* drm_dev_unregister - Unregister DRM device
* @dev: Device to unregister
*
* Unregister the DRM device from the system. This does the reverse of
* drm_dev_register() but does not deallocate the device. The caller must call
* drm_dev_put() to drop their final reference, unless it is managed with devres
* (as devices allocated with devm_drm_dev_alloc() are), in which case there is
* already an unwind action registered.
*
* A special form of unregistering for hotpluggable devices is drm_dev_unplug(),
* which can be called while there are still open users of @dev.
*
* This should be called first in the device teardown code to make sure
* userspace can't access the device instance any more.
*/
void drm_dev_unregister(struct drm_device *dev)
{
dev->registered = false;
drm_client_dev_unregister(dev);
if (drm_core_check_feature(dev, DRIVER_MODESET))
drm_modeset_unregister_all(dev); if (dev->driver->unload) dev->driver->unload(dev); remove_compat_control_link(dev);
drm_minor_unregister(dev, DRM_MINOR_ACCEL);
drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
drm_minor_unregister(dev, DRM_MINOR_RENDER);
drm_debugfs_dev_fini(dev);
}
EXPORT_SYMBOL(drm_dev_unregister);
/*
* DRM Core
* The DRM core module initializes all global DRM objects and makes them
* available to drivers. Once setup, drivers can probe their respective
* devices.
* Currently, core management includes:
* - The "DRM-Global" key/value database
* - Global ID management for connectors
* - DRM major number allocation
* - DRM minor management
* - DRM sysfs class
* - DRM debugfs root
*
* Furthermore, the DRM core provides dynamic char-dev lookups. For each
* interface registered on a DRM device, you can request minor numbers from DRM
* core. DRM core takes care of major-number management and char-dev
* registration. A stub ->open() callback forwards any open() requests to the
* registered minor.
*/
static int drm_stub_open(struct inode *inode, struct file *filp)
{
const struct file_operations *new_fops;
struct drm_minor *minor;
int err;
DRM_DEBUG("\n");
minor = drm_minor_acquire(iminor(inode));
if (IS_ERR(minor))
return PTR_ERR(minor);
new_fops = fops_get(minor->dev->driver->fops);
if (!new_fops) {
err = -ENODEV;
goto out;
}
replace_fops(filp, new_fops);
if (filp->f_op->open)
err = filp->f_op->open(inode, filp);
else
err = 0;
out:
drm_minor_release(minor);
return err;
}
static const struct file_operations drm_stub_fops = {
.owner = THIS_MODULE,
.open = drm_stub_open,
.llseek = noop_llseek,
};
static void drm_core_exit(void)
{
drm_privacy_screen_lookup_exit();
accel_core_exit();
unregister_chrdev(DRM_MAJOR, "drm");
debugfs_remove(drm_debugfs_root);
drm_sysfs_destroy();
idr_destroy(&drm_minors_idr);
drm_connector_ida_destroy();
}
static int __init drm_core_init(void)
{
int ret;
drm_connector_ida_init();
idr_init(&drm_minors_idr);
drm_memcpy_init_early();
ret = drm_sysfs_init();
if (ret < 0) {
DRM_ERROR("Cannot create DRM class: %d\n", ret);
goto error;
}
drm_debugfs_root = debugfs_create_dir("dri", NULL);
ret = register_chrdev(DRM_MAJOR, "drm", &drm_stub_fops);
if (ret < 0)
goto error;
ret = accel_core_init();
if (ret < 0)
goto error;
drm_privacy_screen_lookup_init();
drm_core_init_complete = true;
DRM_DEBUG("Initialized\n");
return 0;
error:
drm_core_exit();
return ret;
}
module_init(drm_core_init);
module_exit(drm_core_exit);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Network event notifiers
*
* Authors:
* Tom Tucker <tom@opengridcomputing.com>
* Steve Wise <swise@opengridcomputing.com>
*
* Fixes:
*/
#include <linux/rtnetlink.h>
#include <linux/notifier.h>
#include <linux/export.h>
#include <net/netevent.h>
static ATOMIC_NOTIFIER_HEAD(netevent_notif_chain);
/**
* register_netevent_notifier - register a netevent notifier block
* @nb: notifier
*
* Register a notifier to be called when a netevent occurs.
* The notifier passed is linked into the kernel structures and must
* not be reused until it has been unregistered. A negative errno code
* is returned on a failure.
*/
int register_netevent_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_register(&netevent_notif_chain, nb);
}
EXPORT_SYMBOL_GPL(register_netevent_notifier);
/**
* unregister_netevent_notifier - unregister a netevent notifier block
* @nb: notifier
*
* Unregister a notifier previously registered by
* register_neigh_notifier(). The notifier is unlinked into the
* kernel structures and may then be reused. A negative errno code
* is returned on a failure.
*/
int unregister_netevent_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_unregister(&netevent_notif_chain, nb);
}
EXPORT_SYMBOL_GPL(unregister_netevent_notifier);
/**
* call_netevent_notifiers - call all netevent notifier blocks
* @val: value passed unmodified to notifier function
* @v: pointer passed unmodified to notifier function
*
* Call all neighbour notifier blocks. Parameters and return value
* are as for notifier_call_chain().
*/
int call_netevent_notifiers(unsigned long val, void *v)
{
return atomic_notifier_call_chain(&netevent_notif_chain, val, v);
}
EXPORT_SYMBOL_GPL(call_netevent_notifiers);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CPUFEATURE_H
#define _ASM_X86_CPUFEATURE_H
#include <asm/processor.h>
#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
#include <asm/asm.h>
#include <linux/bitops.h>
#include <asm/alternative.h>
enum cpuid_leafs
{
CPUID_1_EDX = 0,
CPUID_8000_0001_EDX,
CPUID_8086_0001_EDX,
CPUID_LNX_1,
CPUID_1_ECX,
CPUID_C000_0001_EDX,
CPUID_8000_0001_ECX,
CPUID_LNX_2,
CPUID_LNX_3,
CPUID_7_0_EBX,
CPUID_D_1_EAX,
CPUID_LNX_4,
CPUID_7_1_EAX,
CPUID_8000_0008_EBX,
CPUID_6_EAX,
CPUID_8000_000A_EDX,
CPUID_7_ECX,
CPUID_8000_0007_EBX,
CPUID_7_EDX,
CPUID_8000_001F_EAX,
CPUID_8000_0021_EAX,
CPUID_LNX_5,
NR_CPUID_WORDS,
};
#define X86_CAP_FMT_NUM "%d:%d"
#define x86_cap_flag_num(flag) ((flag) >> 5), ((flag) & 31)
extern const char * const x86_cap_flags[NCAPINTS*32];
extern const char * const x86_power_flags[32];
#define X86_CAP_FMT "%s"
#define x86_cap_flag(flag) x86_cap_flags[flag]
/*
* In order to save room, we index into this array by doing
* X86_BUG_<name> - NCAPINTS*32.
*/
extern const char * const x86_bug_flags[NBUGINTS*32];
#define test_cpu_cap(c, bit) \
arch_test_bit(bit, (unsigned long *)((c)->x86_capability))
/*
* There are 32 bits/features in each mask word. The high bits
* (selected with (bit>>5) give us the word number and the low 5
* bits give us the bit/feature number inside the word.
* (1UL<<((bit)&31) gives us a mask for the feature_bit so we can
* see if it is set in the mask word.
*/
#define CHECK_BIT_IN_MASK_WORD(maskname, word, bit) \
(((bit)>>5)==(word) && (1UL<<((bit)&31) & maskname##word ))
/*
* {REQUIRED,DISABLED}_MASK_CHECK below may seem duplicated with the
* following BUILD_BUG_ON_ZERO() check but when NCAPINTS gets changed, all
* header macros which use NCAPINTS need to be changed. The duplicated macro
* use causes the compiler to issue errors for all headers so that all usage
* sites can be corrected.
*/
#define REQUIRED_MASK_BIT_SET(feature_bit) \
( CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 0, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 1, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 2, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 3, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 4, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 5, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 6, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 7, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 8, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 9, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 10, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 11, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 12, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 13, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 14, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 19, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 20, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 21, feature_bit) || \
REQUIRED_MASK_CHECK || \
BUILD_BUG_ON_ZERO(NCAPINTS != 22))
#define DISABLED_MASK_BIT_SET(feature_bit) \
( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 0, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 1, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 2, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 3, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 4, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 5, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 6, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 7, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 8, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 9, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 10, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 11, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 12, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 13, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 14, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 19, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 20, feature_bit) || \
CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 21, feature_bit) || \
DISABLED_MASK_CHECK || \
BUILD_BUG_ON_ZERO(NCAPINTS != 22))
#define cpu_has(c, bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
test_cpu_cap(c, bit))
#define this_cpu_has(bit) \
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
x86_this_cpu_test_bit(bit, \
(unsigned long __percpu *)&cpu_info.x86_capability))
/*
* This macro is for detection of features which need kernel
* infrastructure to be used. It may *not* directly test the CPU
* itself. Use the cpu_has() family if you want true runtime
* testing of CPU features, like in hypervisor code where you are
* supporting a possible guest feature where host support for it
* is not relevant.
*/
#define cpu_feature_enabled(bit) \
(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))
#define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit)
#define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability))
extern void setup_clear_cpu_cap(unsigned int bit);
extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
#define setup_force_cpu_cap(bit) do { \
set_cpu_cap(&boot_cpu_data, bit); \
set_bit(bit, (unsigned long *)cpu_caps_set); \
} while (0)
#define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)
/*
* Static testing of CPU features. Used the same as boot_cpu_has(). It
* statically patches the target code for additional performance. Use
* static_cpu_has() only in fast paths, where every cycle counts. Which
* means that the boot_cpu_has() variant is already fast enough for the
* majority of cases and you should stick to using it as it is generally
* only two instructions: a RIP-relative MOV and a TEST.
*
* Do not use an "m" constraint for [cap_byte] here: gcc doesn't know
* that this is only used on a fallback path and will sometimes cause
* it to manifest the address of boot_cpu_data in a register, fouling
* the mainline (post-initialization) code.
*/
static __always_inline bool _static_cpu_has(u16 bit)
{
asm goto(
ALTERNATIVE_TERNARY("jmp 6f", %P[feature], "", "jmp %l[t_no]")
".pushsection .altinstr_aux,\"ax\"\n"
"6:\n"
" testb %[bitnum]," _ASM_RIP(%P[cap_byte]) "\n"
" jnz %l[t_yes]\n"
" jmp %l[t_no]\n"
".popsection\n"
: : [feature] "i" (bit),
[bitnum] "i" (1 << (bit & 7)),
[cap_byte] "i" (&((const char *)boot_cpu_data.x86_capability)[bit >> 3])
: : t_yes, t_no);
t_yes:
return true;
t_no:
return false;
}
#define static_cpu_has(bit) \
( \
__builtin_constant_p(boot_cpu_has(bit)) ? \
boot_cpu_has(bit) : \
_static_cpu_has(bit) \
)
#define cpu_has_bug(c, bit) cpu_has(c, (bit))
#define set_cpu_bug(c, bit) set_cpu_cap(c, (bit))
#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit))
#define static_cpu_has_bug(bit) static_cpu_has((bit))
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit))
#define boot_cpu_set_bug(bit) set_cpu_cap(&boot_cpu_data, (bit))
#define MAX_CPU_FEATURES (NCAPINTS * 32)
#define cpu_have_feature boot_cpu_has
#define CPU_FEATURE_TYPEFMT "x86,ven%04Xfam%04Xmod%04X"
#define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
boot_cpu_data.x86_model
#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
#endif /* _ASM_X86_CPUFEATURE_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/mm/filemap.c
*
* Copyright (C) 1994-1999 Linus Torvalds
*/
/*
* This file handles the generic file mmap semantics used by
* most "normal" filesystems (but you don't /have/ to use this:
* the NFS filesystem used to do this differently, for example)
*/
#include <linux/export.h>
#include <linux/compiler.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/sched/signal.h>
#include <linux/uaccess.h>
#include <linux/capability.h>
#include <linux/kernel_stat.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/syscalls.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/error-injection.h>
#include <linux/hash.h>
#include <linux/writeback.h>
#include <linux/backing-dev.h>
#include <linux/pagevec.h>
#include <linux/security.h>
#include <linux/cpuset.h>
#include <linux/hugetlb.h>
#include <linux/memcontrol.h>
#include <linux/shmem_fs.h>
#include <linux/rmap.h>
#include <linux/delayacct.h>
#include <linux/psi.h>
#include <linux/ramfs.h>
#include <linux/page_idle.h>
#include <linux/migrate.h>
#include <linux/pipe_fs_i.h>
#include <linux/splice.h>
#include <linux/rcupdate_wait.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include "internal.h"
#define CREATE_TRACE_POINTS
#include <trace/events/filemap.h>
/*
* FIXME: remove all knowledge of the buffer layer from the core VM
*/
#include <linux/buffer_head.h> /* for try_to_free_buffers */
#include <asm/mman.h>
#include "swap.h"
/*
* Shared mappings implemented 30.11.1994. It's not fully working yet,
* though.
*
* Shared mappings now work. 15.8.1995 Bruno.
*
* finished 'unifying' the page and buffer cache and SMP-threaded the
* page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
*
* SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
*/
/*
* Lock ordering:
*
* ->i_mmap_rwsem (truncate_pagecache)
* ->private_lock (__free_pte->block_dirty_folio)
* ->swap_lock (exclusive_swap_page, others)
* ->i_pages lock
*
* ->i_rwsem
* ->invalidate_lock (acquired by fs in truncate path)
* ->i_mmap_rwsem (truncate->unmap_mapping_range)
*
* ->mmap_lock
* ->i_mmap_rwsem
* ->page_table_lock or pte_lock (various, mainly in memory.c)
* ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
*
* ->mmap_lock
* ->invalidate_lock (filemap_fault)
* ->lock_page (filemap_fault, access_process_vm)
*
* ->i_rwsem (generic_perform_write)
* ->mmap_lock (fault_in_readable->do_page_fault)
*
* bdi->wb.list_lock
* sb_lock (fs/fs-writeback.c)
* ->i_pages lock (__sync_single_inode)
*
* ->i_mmap_rwsem
* ->anon_vma.lock (vma_merge)
*
* ->anon_vma.lock
* ->page_table_lock or pte_lock (anon_vma_prepare and various)
*
* ->page_table_lock or pte_lock
* ->swap_lock (try_to_unmap_one)
* ->private_lock (try_to_unmap_one)
* ->i_pages lock (try_to_unmap_one)
* ->lruvec->lru_lock (follow_page->mark_page_accessed)
* ->lruvec->lru_lock (check_pte_range->isolate_lru_page)
* ->private_lock (folio_remove_rmap_pte->set_page_dirty)
* ->i_pages lock (folio_remove_rmap_pte->set_page_dirty)
* bdi.wb->list_lock (folio_remove_rmap_pte->set_page_dirty)
* ->inode->i_lock (folio_remove_rmap_pte->set_page_dirty)
* ->memcg->move_lock (folio_remove_rmap_pte->folio_memcg_lock)
* bdi.wb->list_lock (zap_pte_range->set_page_dirty)
* ->inode->i_lock (zap_pte_range->set_page_dirty)
* ->private_lock (zap_pte_range->block_dirty_folio)
*/
static void mapping_set_update(struct xa_state *xas,
struct address_space *mapping)
{
if (dax_mapping(mapping) || shmem_mapping(mapping))
return;
xas_set_update(xas, workingset_update_node);
xas_set_lru(xas, &shadow_nodes);
}
static void page_cache_delete(struct address_space *mapping,
struct folio *folio, void *shadow)
{
XA_STATE(xas, &mapping->i_pages, folio->index);
long nr = 1;
mapping_set_update(&xas, mapping); xas_set_order(&xas, folio->index, folio_order(folio)); nr = folio_nr_pages(folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); xas_store(&xas, shadow);
xas_init_marks(&xas);
folio->mapping = NULL;
/* Leave page->index set: truncation lookup relies upon it */
mapping->nrpages -= nr;
}
static void filemap_unaccount_folio(struct address_space *mapping,
struct folio *folio)
{
long nr;
VM_BUG_ON_FOLIO(folio_mapped(folio), folio);
if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) {
pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
current->comm, folio_pfn(folio));
dump_page(&folio->page, "still mapped when deleted");
dump_stack();
add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
if (mapping_exiting(mapping) && !folio_test_large(folio)) {
int mapcount = page_mapcount(&folio->page);
if (folio_ref_count(folio) >= mapcount + 2) {
/*
* All vmas have already been torn down, so it's
* a good bet that actually the page is unmapped
* and we'd rather not leak it: if we're wrong,
* another bad page check should catch it later.
*/
page_mapcount_reset(&folio->page);
folio_ref_sub(folio, mapcount);
}
}
}
/* hugetlb folios do not participate in page cache accounting. */
if (folio_test_hugetlb(folio))
return;
nr = folio_nr_pages(folio); __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
if (folio_test_swapbacked(folio)) {
__lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
if (folio_test_pmd_mappable(folio))
__lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr);
} else if (folio_test_pmd_mappable(folio)) {
__lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr);
filemap_nr_thps_dec(mapping);
}
/*
* At this point folio must be either written or cleaned by
* truncate. Dirty folio here signals a bug and loss of
* unwritten data - on ordinary filesystems.
*
* But it's harmless on in-memory filesystems like tmpfs; and can
* occur when a driver which did get_user_pages() sets page dirty
* before putting it, while the inode is being finally evicted.
*
* Below fixes dirty accounting after removing the folio entirely
* but leaves the dirty flag set: it has no effect for truncated
* folio and anyway will be cleared before returning folio to
* buddy allocator.
*/
if (WARN_ON_ONCE(folio_test_dirty(folio) &&
mapping_can_writeback(mapping)))
folio_account_cleaned(folio, inode_to_wb(mapping->host));
}
/*
* Delete a page from the page cache and free it. Caller has to make
* sure the page is locked and that nobody else uses it - or that usage
* is safe. The caller must hold the i_pages lock.
*/
void __filemap_remove_folio(struct folio *folio, void *shadow)
{
struct address_space *mapping = folio->mapping; trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); page_cache_delete(mapping, folio, shadow);
}
void filemap_free_folio(struct address_space *mapping, struct folio *folio)
{
void (*free_folio)(struct folio *);
int refs = 1;
free_folio = mapping->a_ops->free_folio;
if (free_folio)
free_folio(folio); if (folio_test_large(folio)) refs = folio_nr_pages(folio); folio_put_refs(folio, refs);
}
/**
* filemap_remove_folio - Remove folio from page cache.
* @folio: The folio.
*
* This must be called only on folios that are locked and have been
* verified to be in the page cache. It will never put the folio into
* the free list because the caller has a reference on the page.
*/
void filemap_remove_folio(struct folio *folio)
{
struct address_space *mapping = folio->mapping; BUG_ON(!folio_test_locked(folio)); spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages);
__filemap_remove_folio(folio, NULL);
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping)) inode_add_lru(mapping->host); spin_unlock(&mapping->host->i_lock);
filemap_free_folio(mapping, folio);
}
/*
* page_cache_delete_batch - delete several folios from page cache
* @mapping: the mapping to which folios belong
* @fbatch: batch of folios to delete
*
* The function walks over mapping->i_pages and removes folios passed in
* @fbatch from the mapping. The function expects @fbatch to be sorted
* by page index and is optimised for it to be dense.
* It tolerates holes in @fbatch (mapping entries at those indices are not
* modified).
*
* The function expects the i_pages lock to be held.
*/
static void page_cache_delete_batch(struct address_space *mapping,
struct folio_batch *fbatch)
{
XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index);
long total_pages = 0;
int i = 0;
struct folio *folio;
mapping_set_update(&xas, mapping);
xas_for_each(&xas, folio, ULONG_MAX) {
if (i >= folio_batch_count(fbatch))
break;
/* A swap/dax/shadow entry got inserted? Skip it. */
if (xa_is_value(folio))
continue;
/*
* A page got inserted in our range? Skip it. We have our
* pages locked so they are protected from being removed.
* If we see a page whose index is higher than ours, it
* means our page has been removed, which shouldn't be
* possible because we're holding the PageLock.
*/
if (folio != fbatch->folios[i]) {
VM_BUG_ON_FOLIO(folio->index >
fbatch->folios[i]->index, folio);
continue;
}
WARN_ON_ONCE(!folio_test_locked(folio));
folio->mapping = NULL;
/* Leave folio->index set: truncation lookup relies on it */
i++;
xas_store(&xas, NULL);
total_pages += folio_nr_pages(folio);
}
mapping->nrpages -= total_pages;
}
void delete_from_page_cache_batch(struct address_space *mapping,
struct folio_batch *fbatch)
{
int i;
if (!folio_batch_count(fbatch))
return;
spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages);
for (i = 0; i < folio_batch_count(fbatch); i++) {
struct folio *folio = fbatch->folios[i];
trace_mm_filemap_delete_from_page_cache(folio);
filemap_unaccount_folio(mapping, folio);
}
page_cache_delete_batch(mapping, fbatch);
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping))
inode_add_lru(mapping->host);
spin_unlock(&mapping->host->i_lock);
for (i = 0; i < folio_batch_count(fbatch); i++)
filemap_free_folio(mapping, fbatch->folios[i]);
}
int filemap_check_errors(struct address_space *mapping)
{
int ret = 0;
/* Check for outstanding write errors */
if (test_bit(AS_ENOSPC, &mapping->flags) &&
test_and_clear_bit(AS_ENOSPC, &mapping->flags))
ret = -ENOSPC;
if (test_bit(AS_EIO, &mapping->flags) &&
test_and_clear_bit(AS_EIO, &mapping->flags))
ret = -EIO;
return ret;
}
EXPORT_SYMBOL(filemap_check_errors);
static int filemap_check_and_keep_errors(struct address_space *mapping)
{
/* Check for outstanding write errors */
if (test_bit(AS_EIO, &mapping->flags))
return -EIO;
if (test_bit(AS_ENOSPC, &mapping->flags))
return -ENOSPC;
return 0;
}
/**
* filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
* @mapping: address space structure to write
* @wbc: the writeback_control controlling the writeout
*
* Call writepages on the mapping using the provided wbc to control the
* writeout.
*
* Return: %0 on success, negative error code otherwise.
*/
int filemap_fdatawrite_wbc(struct address_space *mapping,
struct writeback_control *wbc)
{
int ret;
if (!mapping_can_writeback(mapping) ||
!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
return 0;
wbc_attach_fdatawrite_inode(wbc, mapping->host);
ret = do_writepages(mapping, wbc);
wbc_detach_inode(wbc);
return ret;
}
EXPORT_SYMBOL(filemap_fdatawrite_wbc);
/**
* __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
* @mapping: address space structure to write
* @start: offset in bytes where the range starts
* @end: offset in bytes where the range ends (inclusive)
* @sync_mode: enable synchronous operation
*
* Start writeback against all of a mapping's dirty pages that lie
* within the byte offsets <start, end> inclusive.
*
* If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
* opposed to a regular memory cleansing writeback. The difference between
* these two operations is that if a dirty page/buffer is encountered, it must
* be waited upon, and not just skipped over.
*
* Return: %0 on success, negative error code otherwise.
*/
int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
loff_t end, int sync_mode)
{
struct writeback_control wbc = {
.sync_mode = sync_mode,
.nr_to_write = LONG_MAX,
.range_start = start,
.range_end = end,
};
return filemap_fdatawrite_wbc(mapping, &wbc);
}
static inline int __filemap_fdatawrite(struct address_space *mapping,
int sync_mode)
{
return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
}
int filemap_fdatawrite(struct address_space *mapping)
{
return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite);
int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
loff_t end)
{
return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite_range);
/**
* filemap_flush - mostly a non-blocking flush
* @mapping: target address_space
*
* This is a mostly non-blocking flush. Not suitable for data-integrity
* purposes - I/O may not be started against all dirty pages.
*
* Return: %0 on success, negative error code otherwise.
*/
int filemap_flush(struct address_space *mapping)
{
return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
}
EXPORT_SYMBOL(filemap_flush);
/**
* filemap_range_has_page - check if a page exists in range.
* @mapping: address space within which to check
* @start_byte: offset in bytes where the range starts
* @end_byte: offset in bytes where the range ends (inclusive)
*
* Find at least one page in the range supplied, usually used to check if
* direct writing in this range will trigger a writeback.
*
* Return: %true if at least one page exists in the specified range,
* %false otherwise.
*/
bool filemap_range_has_page(struct address_space *mapping,
loff_t start_byte, loff_t end_byte)
{
struct folio *folio;
XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
pgoff_t max = end_byte >> PAGE_SHIFT;
if (end_byte < start_byte)
return false;
rcu_read_lock();
for (;;) {
folio = xas_find(&xas, max);
if (xas_retry(&xas, folio))
continue;
/* Shadow entries don't count */
if (xa_is_value(folio))
continue;
/*
* We don't need to try to pin this page; we're about to
* release the RCU lock anyway. It is enough to know that
* there was a page here recently.
*/
break;
}
rcu_read_unlock();
return folio != NULL;
}
EXPORT_SYMBOL(filemap_range_has_page);
static void __filemap_fdatawait_range(struct address_space *mapping,
loff_t start_byte, loff_t end_byte)
{
pgoff_t index = start_byte >> PAGE_SHIFT;
pgoff_t end = end_byte >> PAGE_SHIFT;
struct folio_batch fbatch;
unsigned nr_folios;
folio_batch_init(&fbatch);
while (index <= end) {
unsigned i;
nr_folios = filemap_get_folios_tag(mapping, &index, end,
PAGECACHE_TAG_WRITEBACK, &fbatch);
if (!nr_folios)
break;
for (i = 0; i < nr_folios; i++) {
struct folio *folio = fbatch.folios[i];
folio_wait_writeback(folio);
folio_clear_error(folio);
}
folio_batch_release(&fbatch);
cond_resched();
}
}
/**
* filemap_fdatawait_range - wait for writeback to complete
* @mapping: address space structure to wait for
* @start_byte: offset in bytes where the range starts
* @end_byte: offset in bytes where the range ends (inclusive)
*
* Walk the list of under-writeback pages of the given address space
* in the given range and wait for all of them. Check error status of
* the address space and return it.
*
* Since the error status of the address space is cleared by this function,
* callers are responsible for checking the return value and handling and/or
* reporting the error.
*
* Return: error status of the address space.
*/
int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
loff_t end_byte)
{
__filemap_fdatawait_range(mapping, start_byte, end_byte);
return filemap_check_errors(mapping);
}
EXPORT_SYMBOL(filemap_fdatawait_range);
/**
* filemap_fdatawait_range_keep_errors - wait for writeback to complete
* @mapping: address space structure to wait for
* @start_byte: offset in bytes where the range starts
* @end_byte: offset in bytes where the range ends (inclusive)
*
* Walk the list of under-writeback pages of the given address space in the
* given range and wait for all of them. Unlike filemap_fdatawait_range(),
* this function does not clear error status of the address space.
*
* Use this function if callers don't handle errors themselves. Expected
* call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
* fsfreeze(8)
*/
int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
loff_t start_byte, loff_t end_byte)
{
__filemap_fdatawait_range(mapping, start_byte, end_byte);
return filemap_check_and_keep_errors(mapping);
}
EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors);
/**
* file_fdatawait_range - wait for writeback to complete
* @file: file pointing to address space structure to wait for
* @start_byte: offset in bytes where the range starts
* @end_byte: offset in bytes where the range ends (inclusive)
*
* Walk the list of under-writeback pages of the address space that file
* refers to, in the given range and wait for all of them. Check error
* status of the address space vs. the file->f_wb_err cursor and return it.
*
* Since the error status of the file is advanced by this function,
* callers are responsible for checking the return value and handling and/or
* reporting the error.
*
* Return: error status of the address space vs. the file->f_wb_err cursor.
*/
int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
{
struct address_space *mapping = file->f_mapping;
__filemap_fdatawait_range(mapping, start_byte, end_byte);
return file_check_and_advance_wb_err(file);
}
EXPORT_SYMBOL(file_fdatawait_range);
/**
* filemap_fdatawait_keep_errors - wait for writeback without clearing errors
* @mapping: address space structure to wait for
*
* Walk the list of under-writeback pages of the given address space
* and wait for all of them. Unlike filemap_fdatawait(), this function
* does not clear error status of the address space.
*
* Use this function if callers don't handle errors themselves. Expected
* call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
* fsfreeze(8)
*
* Return: error status of the address space.
*/
int filemap_fdatawait_keep_errors(struct address_space *mapping)
{
__filemap_fdatawait_range(mapping, 0, LLONG_MAX);
return filemap_check_and_keep_errors(mapping);
}
EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
/* Returns true if writeback might be needed or already in progress. */
static bool mapping_needs_writeback(struct address_space *mapping)
{
return mapping->nrpages;
}
bool filemap_range_has_writeback(struct address_space *mapping,
loff_t start_byte, loff_t end_byte)
{
XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
pgoff_t max = end_byte >> PAGE_SHIFT;
struct folio *folio;
if (end_byte < start_byte)
return false;
rcu_read_lock();
xas_for_each(&xas, folio, max) {
if (xas_retry(&xas, folio))
continue;
if (xa_is_value(folio))
continue;
if (folio_test_dirty(folio) || folio_test_locked(folio) ||
folio_test_writeback(folio))
break;
}
rcu_read_unlock();
return folio != NULL;
}
EXPORT_SYMBOL_GPL(filemap_range_has_writeback);
/**
* filemap_write_and_wait_range - write out & wait on a file range
* @mapping: the address_space for the pages
* @lstart: offset in bytes where the range starts
* @lend: offset in bytes where the range ends (inclusive)
*
* Write out and wait upon file offsets lstart->lend, inclusive.
*
* Note that @lend is inclusive (describes the last byte to be written) so
* that this function can be used to write to the very end-of-file (end = -1).
*
* Return: error status of the address space.
*/
int filemap_write_and_wait_range(struct address_space *mapping,
loff_t lstart, loff_t lend)
{
int err = 0, err2;
if (lend < lstart)
return 0;
if (mapping_needs_writeback(mapping)) {
err = __filemap_fdatawrite_range(mapping, lstart, lend,
WB_SYNC_ALL);
/*
* Even if the above returned error, the pages may be
* written partially (e.g. -ENOSPC), so we wait for it.
* But the -EIO is special case, it may indicate the worst
* thing (e.g. bug) happened, so we avoid waiting for it.
*/
if (err != -EIO)
__filemap_fdatawait_range(mapping, lstart, lend);
}
err2 = filemap_check_errors(mapping);
if (!err)
err = err2;
return err;
}
EXPORT_SYMBOL(filemap_write_and_wait_range);
void __filemap_set_wb_err(struct address_space *mapping, int err)
{
errseq_t eseq = errseq_set(&mapping->wb_err, err);
trace_filemap_set_wb_err(mapping, eseq);
}
EXPORT_SYMBOL(__filemap_set_wb_err);
/**
* file_check_and_advance_wb_err - report wb error (if any) that was previously
* and advance wb_err to current one
* @file: struct file on which the error is being reported
*
* When userland calls fsync (or something like nfsd does the equivalent), we
* want to report any writeback errors that occurred since the last fsync (or
* since the file was opened if there haven't been any).
*
* Grab the wb_err from the mapping. If it matches what we have in the file,
* then just quickly return 0. The file is all caught up.
*
* If it doesn't match, then take the mapping value, set the "seen" flag in
* it and try to swap it into place. If it works, or another task beat us
* to it with the new value, then update the f_wb_err and return the error
* portion. The error at this point must be reported via proper channels
* (a'la fsync, or NFS COMMIT operation, etc.).
*
* While we handle mapping->wb_err with atomic operations, the f_wb_err
* value is protected by the f_lock since we must ensure that it reflects
* the latest value swapped in for this file descriptor.
*
* Return: %0 on success, negative error code otherwise.
*/
int file_check_and_advance_wb_err(struct file *file)
{
int err = 0;
errseq_t old = READ_ONCE(file->f_wb_err);
struct address_space *mapping = file->f_mapping;
/* Locklessly handle the common case where nothing has changed */
if (errseq_check(&mapping->wb_err, old)) {
/* Something changed, must use slow path */
spin_lock(&file->f_lock);
old = file->f_wb_err;
err = errseq_check_and_advance(&mapping->wb_err,
&file->f_wb_err);
trace_file_check_and_advance_wb_err(file, old);
spin_unlock(&file->f_lock);
}
/*
* We're mostly using this function as a drop in replacement for
* filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect
* that the legacy code would have had on these flags.
*/
clear_bit(AS_EIO, &mapping->flags);
clear_bit(AS_ENOSPC, &mapping->flags);
return err;
}
EXPORT_SYMBOL(file_check_and_advance_wb_err);
/**
* file_write_and_wait_range - write out & wait on a file range
* @file: file pointing to address_space with pages
* @lstart: offset in bytes where the range starts
* @lend: offset in bytes where the range ends (inclusive)
*
* Write out and wait upon file offsets lstart->lend, inclusive.
*
* Note that @lend is inclusive (describes the last byte to be written) so
* that this function can be used to write to the very end-of-file (end = -1).
*
* After writing out and waiting on the data, we check and advance the
* f_wb_err cursor to the latest value, and return any errors detected there.
*
* Return: %0 on success, negative error code otherwise.
*/
int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
{
int err = 0, err2;
struct address_space *mapping = file->f_mapping;
if (lend < lstart)
return 0;
if (mapping_needs_writeback(mapping)) {
err = __filemap_fdatawrite_range(mapping, lstart, lend,
WB_SYNC_ALL);
/* See comment of filemap_write_and_wait() */
if (err != -EIO)
__filemap_fdatawait_range(mapping, lstart, lend);
}
err2 = file_check_and_advance_wb_err(file);
if (!err)
err = err2;
return err;
}
EXPORT_SYMBOL(file_write_and_wait_range);
/**
* replace_page_cache_folio - replace a pagecache folio with a new one
* @old: folio to be replaced
* @new: folio to replace with
*
* This function replaces a folio in the pagecache with a new one. On
* success it acquires the pagecache reference for the new folio and
* drops it for the old folio. Both the old and new folios must be
* locked. This function does not add the new folio to the LRU, the
* caller must do that.
*
* The remove + add is atomic. This function cannot fail.
*/
void replace_page_cache_folio(struct folio *old, struct folio *new)
{
struct address_space *mapping = old->mapping;
void (*free_folio)(struct folio *) = mapping->a_ops->free_folio;
pgoff_t offset = old->index;
XA_STATE(xas, &mapping->i_pages, offset);
VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
VM_BUG_ON_FOLIO(new->mapping, new);
folio_get(new);
new->mapping = mapping;
new->index = offset;
mem_cgroup_replace_folio(old, new);
xas_lock_irq(&xas);
xas_store(&xas, new);
old->mapping = NULL;
/* hugetlb pages do not participate in page cache accounting. */
if (!folio_test_hugetlb(old))
__lruvec_stat_sub_folio(old, NR_FILE_PAGES);
if (!folio_test_hugetlb(new))
__lruvec_stat_add_folio(new, NR_FILE_PAGES);
if (folio_test_swapbacked(old))
__lruvec_stat_sub_folio(old, NR_SHMEM);
if (folio_test_swapbacked(new))
__lruvec_stat_add_folio(new, NR_SHMEM);
xas_unlock_irq(&xas);
if (free_folio)
free_folio(old);
folio_put(old);
}
EXPORT_SYMBOL_GPL(replace_page_cache_folio);
noinline int __filemap_add_folio(struct address_space *mapping,
struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp)
{
XA_STATE(xas, &mapping->i_pages, index);
bool huge = folio_test_hugetlb(folio);
bool charged = false;
long nr = 1;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio);
mapping_set_update(&xas, mapping);
if (!huge) {
int error = mem_cgroup_charge(folio, NULL, gfp);
if (error)
return error;
charged = true;
}
VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);
xas_set_order(&xas, index, folio_order(folio));
nr = folio_nr_pages(folio);
gfp &= GFP_RECLAIM_MASK;
folio_ref_add(folio, nr);
folio->mapping = mapping;
folio->index = xas.xa_index;
do {
unsigned int order = xa_get_order(xas.xa, xas.xa_index);
void *entry, *old = NULL;
if (order > folio_order(folio))
xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index),
order, gfp);
xas_lock_irq(&xas);
xas_for_each_conflict(&xas, entry) {
old = entry;
if (!xa_is_value(entry)) {
xas_set_err(&xas, -EEXIST);
goto unlock;
}
}
if (old) {
if (shadowp)
*shadowp = old;
/* entry may have been split before we acquired lock */
order = xa_get_order(xas.xa, xas.xa_index);
if (order > folio_order(folio)) {
/* How to handle large swap entries? */
BUG_ON(shmem_mapping(mapping));
xas_split(&xas, old, order);
xas_reset(&xas);
}
}
xas_store(&xas, folio);
if (xas_error(&xas))
goto unlock;
mapping->nrpages += nr;
/* hugetlb pages do not participate in page cache accounting */
if (!huge) {
__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
if (folio_test_pmd_mappable(folio))
__lruvec_stat_mod_folio(folio,
NR_FILE_THPS, nr);
}
unlock:
xas_unlock_irq(&xas);
} while (xas_nomem(&xas, gfp));
if (xas_error(&xas))
goto error;
trace_mm_filemap_add_to_page_cache(folio);
return 0;
error:
if (charged)
mem_cgroup_uncharge(folio);
folio->mapping = NULL;
/* Leave page->index set: truncation relies upon it */
folio_put_refs(folio, nr);
return xas_error(&xas);
}
ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO);
int filemap_add_folio(struct address_space *mapping, struct folio *folio,
pgoff_t index, gfp_t gfp)
{
void *shadow = NULL;
int ret;
__folio_set_locked(folio);
ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow);
if (unlikely(ret))
__folio_clear_locked(folio);
else {
/*
* The folio might have been evicted from cache only
* recently, in which case it should be activated like
* any other repeatedly accessed folio.
* The exception is folios getting rewritten; evicting other
* data from the working set, only to cache data that will
* get overwritten with something else, is a waste of memory.
*/
WARN_ON_ONCE(folio_test_active(folio));
if (!(gfp & __GFP_WRITE) && shadow)
workingset_refault(folio, shadow);
folio_add_lru(folio);
}
return ret;
}
EXPORT_SYMBOL_GPL(filemap_add_folio);
#ifdef CONFIG_NUMA
struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order)
{
int n;
struct folio *folio;
if (cpuset_do_page_mem_spread()) {
unsigned int cpuset_mems_cookie;
do {
cpuset_mems_cookie = read_mems_allowed_begin();
n = cpuset_mem_spread_node();
folio = __folio_alloc_node(gfp, order, n);
} while (!folio && read_mems_allowed_retry(cpuset_mems_cookie));
return folio;
}
return folio_alloc(gfp, order);
}
EXPORT_SYMBOL(filemap_alloc_folio);
#endif
/*
* filemap_invalidate_lock_two - lock invalidate_lock for two mappings
*
* Lock exclusively invalidate_lock of any passed mapping that is not NULL.
*
* @mapping1: the first mapping to lock
* @mapping2: the second mapping to lock
*/
void filemap_invalidate_lock_two(struct address_space *mapping1,
struct address_space *mapping2)
{
if (mapping1 > mapping2)
swap(mapping1, mapping2);
if (mapping1)
down_write(&mapping1->invalidate_lock);
if (mapping2 && mapping1 != mapping2)
down_write_nested(&mapping2->invalidate_lock, 1);
}
EXPORT_SYMBOL(filemap_invalidate_lock_two);
/*
* filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
*
* Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
*
* @mapping1: the first mapping to unlock
* @mapping2: the second mapping to unlock
*/
void filemap_invalidate_unlock_two(struct address_space *mapping1,
struct address_space *mapping2)
{
if (mapping1)
up_write(&mapping1->invalidate_lock);
if (mapping2 && mapping1 != mapping2)
up_write(&mapping2->invalidate_lock);
}
EXPORT_SYMBOL(filemap_invalidate_unlock_two);
/*
* In order to wait for pages to become available there must be
* waitqueues associated with pages. By using a hash table of
* waitqueues where the bucket discipline is to maintain all
* waiters on the same queue and wake all when any of the pages
* become available, and for the woken contexts to check to be
* sure the appropriate page became available, this saves space
* at a cost of "thundering herd" phenomena during rare hash
* collisions.
*/
#define PAGE_WAIT_TABLE_BITS 8
#define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
static wait_queue_head_t *folio_waitqueue(struct folio *folio)
{
return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)];
}
void __init pagecache_init(void)
{
int i;
for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
init_waitqueue_head(&folio_wait_table[i]);
page_writeback_init();
}
/*
* The page wait code treats the "wait->flags" somewhat unusually, because
* we have multiple different kinds of waits, not just the usual "exclusive"
* one.
*
* We have:
*
* (a) no special bits set:
*
* We're just waiting for the bit to be released, and when a waker
* calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up,
* and remove it from the wait queue.
*
* Simple and straightforward.
*
* (b) WQ_FLAG_EXCLUSIVE:
*
* The waiter is waiting to get the lock, and only one waiter should
* be woken up to avoid any thundering herd behavior. We'll set the
* WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue.
*
* This is the traditional exclusive wait.
*
* (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM:
*
* The waiter is waiting to get the bit, and additionally wants the
* lock to be transferred to it for fair lock behavior. If the lock
* cannot be taken, we stop walking the wait queue without waking
* the waiter.
*
* This is the "fair lock handoff" case, and in addition to setting
* WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see
* that it now has the lock.
*/
static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
{
unsigned int flags;
struct wait_page_key *key = arg;
struct wait_page_queue *wait_page
= container_of(wait, struct wait_page_queue, wait);
if (!wake_page_match(wait_page, key))
return 0;
/*
* If it's a lock handoff wait, we get the bit for it, and
* stop walking (and do not wake it up) if we can't.
*/
flags = wait->flags;
if (flags & WQ_FLAG_EXCLUSIVE) {
if (test_bit(key->bit_nr, &key->folio->flags))
return -1;
if (flags & WQ_FLAG_CUSTOM) {
if (test_and_set_bit(key->bit_nr, &key->folio->flags))
return -1;
flags |= WQ_FLAG_DONE;
}
}
/*
* We are holding the wait-queue lock, but the waiter that
* is waiting for this will be checking the flags without
* any locking.
*
* So update the flags atomically, and wake up the waiter
* afterwards to avoid any races. This store-release pairs
* with the load-acquire in folio_wait_bit_common().
*/
smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN);
wake_up_state(wait->private, mode);
/*
* Ok, we have successfully done what we're waiting for,
* and we can unconditionally remove the wait entry.
*
* Note that this pairs with the "finish_wait()" in the
* waiter, and has to be the absolute last thing we do.
* After this list_del_init(&wait->entry) the wait entry
* might be de-allocated and the process might even have
* exited.
*/
list_del_init_careful(&wait->entry);
return (flags & WQ_FLAG_EXCLUSIVE) != 0;
}
static void folio_wake_bit(struct folio *folio, int bit_nr)
{
wait_queue_head_t *q = folio_waitqueue(folio);
struct wait_page_key key;
unsigned long flags;
key.folio = folio;
key.bit_nr = bit_nr;
key.page_match = 0;
spin_lock_irqsave(&q->lock, flags);
__wake_up_locked_key(q, TASK_NORMAL, &key);
/*
* It's possible to miss clearing waiters here, when we woke our page
* waiters, but the hashed waitqueue has waiters for other pages on it.
* That's okay, it's a rare case. The next waker will clear it.
*
* Note that, depending on the page pool (buddy, hugetlb, ZONE_DEVICE,
* other), the flag may be cleared in the course of freeing the page;
* but that is not required for correctness.
*/
if (!waitqueue_active(q) || !key.page_match)
folio_clear_waiters(folio);
spin_unlock_irqrestore(&q->lock, flags);
}
/*
* A choice of three behaviors for folio_wait_bit_common():
*/
enum behavior {
EXCLUSIVE, /* Hold ref to page and take the bit when woken, like
* __folio_lock() waiting on then setting PG_locked.
*/
SHARED, /* Hold ref to page and check the bit when woken, like
* folio_wait_writeback() waiting on PG_writeback.
*/
DROP, /* Drop ref to page before wait, no check when woken,
* like folio_put_wait_locked() on PG_locked.
*/
};
/*
* Attempt to check (or get) the folio flag, and mark us done
* if successful.
*/
static inline bool folio_trylock_flag(struct folio *folio, int bit_nr,
struct wait_queue_entry *wait)
{
if (wait->flags & WQ_FLAG_EXCLUSIVE) {
if (test_and_set_bit(bit_nr, &folio->flags))
return false;
} else if (test_bit(bit_nr, &folio->flags))
return false;
wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
return true;
}
/* How many times do we accept lock stealing from under a waiter? */
int sysctl_page_lock_unfairness = 5;
static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
int state, enum behavior behavior)
{
wait_queue_head_t *q = folio_waitqueue(folio);
int unfairness = sysctl_page_lock_unfairness;
struct wait_page_queue wait_page;
wait_queue_entry_t *wait = &wait_page.wait;
bool thrashing = false;
unsigned long pflags;
bool in_thrashing;
if (bit_nr == PG_locked &&
!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
delayacct_thrashing_start(&in_thrashing);
psi_memstall_enter(&pflags);
thrashing = true;
}
init_wait(wait);
wait->func = wake_page_function;
wait_page.folio = folio;
wait_page.bit_nr = bit_nr;
repeat:
wait->flags = 0;
if (behavior == EXCLUSIVE) {
wait->flags = WQ_FLAG_EXCLUSIVE;
if (--unfairness < 0)
wait->flags |= WQ_FLAG_CUSTOM;
}
/*
* Do one last check whether we can get the
* page bit synchronously.
*
* Do the folio_set_waiters() marking before that
* to let any waker we _just_ missed know they
* need to wake us up (otherwise they'll never
* even go to the slow case that looks at the
* page queue), and add ourselves to the wait
* queue if we need to sleep.
*
* This part needs to be done under the queue
* lock to avoid races.
*/
spin_lock_irq(&q->lock);
folio_set_waiters(folio);
if (!folio_trylock_flag(folio, bit_nr, wait))
__add_wait_queue_entry_tail(q, wait);
spin_unlock_irq(&q->lock);
/*
* From now on, all the logic will be based on
* the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to
* see whether the page bit testing has already
* been done by the wake function.
*
* We can drop our reference to the folio.
*/
if (behavior == DROP)
folio_put(folio);
/*
* Note that until the "finish_wait()", or until
* we see the WQ_FLAG_WOKEN flag, we need to
* be very careful with the 'wait->flags', because
* we may race with a waker that sets them.
*/
for (;;) {
unsigned int flags;
set_current_state(state);
/* Loop until we've been woken or interrupted */
flags = smp_load_acquire(&wait->flags);
if (!(flags & WQ_FLAG_WOKEN)) {
if (signal_pending_state(state, current))
break;
io_schedule();
continue;
}
/* If we were non-exclusive, we're done */
if (behavior != EXCLUSIVE)
break;
/* If the waker got the lock for us, we're done */
if (flags & WQ_FLAG_DONE)
break;
/*
* Otherwise, if we're getting the lock, we need to
* try to get it ourselves.
*
* And if that fails, we'll have to retry this all.
*/
if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0))))
goto repeat;
wait->flags |= WQ_FLAG_DONE;
break;
}
/*
* If a signal happened, this 'finish_wait()' may remove the last
* waiter from the wait-queues, but the folio waiters bit will remain
* set. That's ok. The next wakeup will take care of it, and trying
* to do it here would be difficult and prone to races.
*/
finish_wait(q, wait);
if (thrashing) {
delayacct_thrashing_end(&in_thrashing);
psi_memstall_leave(&pflags);
}
/*
* NOTE! The wait->flags weren't stable until we've done the
* 'finish_wait()', and we could have exited the loop above due
* to a signal, and had a wakeup event happen after the signal
* test but before the 'finish_wait()'.
*
* So only after the finish_wait() can we reliably determine
* if we got woken up or not, so we can now figure out the final
* return value based on that state without races.
*
* Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
* waiter, but an exclusive one requires WQ_FLAG_DONE.
*/
if (behavior == EXCLUSIVE)
return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR;
return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
}
#ifdef CONFIG_MIGRATION
/**
* migration_entry_wait_on_locked - Wait for a migration entry to be removed
* @entry: migration swap entry.
* @ptl: already locked ptl. This function will drop the lock.
*
* Wait for a migration entry referencing the given page to be removed. This is
* equivalent to put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE) except
* this can be called without taking a reference on the page. Instead this
* should be called while holding the ptl for the migration entry referencing
* the page.
*
* Returns after unlocking the ptl.
*
* This follows the same logic as folio_wait_bit_common() so see the comments
* there.
*/
void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl)
__releases(ptl)
{
struct wait_page_queue wait_page;
wait_queue_entry_t *wait = &wait_page.wait;
bool thrashing = false;
unsigned long pflags;
bool in_thrashing;
wait_queue_head_t *q;
struct folio *folio = pfn_swap_entry_folio(entry);
q = folio_waitqueue(folio);
if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
delayacct_thrashing_start(&in_thrashing);
psi_memstall_enter(&pflags);
thrashing = true;
}
init_wait(wait);
wait->func = wake_page_function;
wait_page.folio = folio;
wait_page.bit_nr = PG_locked;
wait->flags = 0;
spin_lock_irq(&q->lock);
folio_set_waiters(folio);
if (!folio_trylock_flag(folio, PG_locked, wait))
__add_wait_queue_entry_tail(q, wait);
spin_unlock_irq(&q->lock);
/*
* If a migration entry exists for the page the migration path must hold
* a valid reference to the page, and it must take the ptl to remove the
* migration entry. So the page is valid until the ptl is dropped.
*/
spin_unlock(ptl);
for (;;) {
unsigned int flags;
set_current_state(TASK_UNINTERRUPTIBLE);
/* Loop until we've been woken or interrupted */
flags = smp_load_acquire(&wait->flags);
if (!(flags & WQ_FLAG_WOKEN)) {
if (signal_pending_state(TASK_UNINTERRUPTIBLE, current))
break;
io_schedule();
continue;
}
break;
}
finish_wait(q, wait);
if (thrashing) {
delayacct_thrashing_end(&in_thrashing);
psi_memstall_leave(&pflags);
}
}
#endif
void folio_wait_bit(struct folio *folio, int bit_nr)
{
folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED);
}
EXPORT_SYMBOL(folio_wait_bit);
int folio_wait_bit_killable(struct folio *folio, int bit_nr)
{
return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED);
}
EXPORT_SYMBOL(folio_wait_bit_killable);
/**
* folio_put_wait_locked - Drop a reference and wait for it to be unlocked
* @folio: The folio to wait for.
* @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc).
*
* The caller should hold a reference on @folio. They expect the page to
* become unlocked relatively soon, but do not wish to hold up migration
* (for example) by holding the reference while waiting for the folio to
* come unlocked. After this function returns, the caller should not
* dereference @folio.
*
* Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal.
*/
static int folio_put_wait_locked(struct folio *folio, int state)
{
return folio_wait_bit_common(folio, PG_locked, state, DROP);
}
/**
* folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue
* @folio: Folio defining the wait queue of interest
* @waiter: Waiter to add to the queue
*
* Add an arbitrary @waiter to the wait queue for the nominated @folio.
*/
void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter)
{
wait_queue_head_t *q = folio_waitqueue(folio);
unsigned long flags;
spin_lock_irqsave(&q->lock, flags);
__add_wait_queue_entry_tail(q, waiter);
folio_set_waiters(folio);
spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(folio_add_wait_queue);
/**
* folio_unlock - Unlock a locked folio.
* @folio: The folio.
*
* Unlocks the folio and wakes up any thread sleeping on the page lock.
*
* Context: May be called from interrupt or process context. May not be
* called from NMI context.
*/
void folio_unlock(struct folio *folio)
{
/* Bit 7 allows x86 to check the byte's sign bit */
BUILD_BUG_ON(PG_waiters != 7);
BUILD_BUG_ON(PG_locked > 7);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); if (folio_xor_flags_has_waiters(folio, 1 << PG_locked)) folio_wake_bit(folio, PG_locked);
}
EXPORT_SYMBOL(folio_unlock);
/**
* folio_end_read - End read on a folio.
* @folio: The folio.
* @success: True if all reads completed successfully.
*
* When all reads against a folio have completed, filesystems should
* call this function to let the pagecache know that no more reads
* are outstanding. This will unlock the folio and wake up any thread
* sleeping on the lock. The folio will also be marked uptodate if all
* reads succeeded.
*
* Context: May be called from interrupt or process context. May not be
* called from NMI context.
*/
void folio_end_read(struct folio *folio, bool success)
{
unsigned long mask = 1 << PG_locked;
/* Must be in bottom byte for x86 to work */
BUILD_BUG_ON(PG_uptodate > 7);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_FOLIO(folio_test_uptodate(folio), folio);
if (likely(success))
mask |= 1 << PG_uptodate;
if (folio_xor_flags_has_waiters(folio, mask))
folio_wake_bit(folio, PG_locked);
}
EXPORT_SYMBOL(folio_end_read);
/**
* folio_end_private_2 - Clear PG_private_2 and wake any waiters.
* @folio: The folio.
*
* Clear the PG_private_2 bit on a folio and wake up any sleepers waiting for
* it. The folio reference held for PG_private_2 being set is released.
*
* This is, for example, used when a netfs folio is being written to a local
* disk cache, thereby allowing writes to the cache for the same folio to be
* serialised.
*/
void folio_end_private_2(struct folio *folio)
{
VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio);
clear_bit_unlock(PG_private_2, folio_flags(folio, 0));
folio_wake_bit(folio, PG_private_2);
folio_put(folio);
}
EXPORT_SYMBOL(folio_end_private_2);
/**
* folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio.
* @folio: The folio to wait on.
*
* Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio.
*/
void folio_wait_private_2(struct folio *folio)
{
while (folio_test_private_2(folio))
folio_wait_bit(folio, PG_private_2);
}
EXPORT_SYMBOL(folio_wait_private_2);
/**
* folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio.
* @folio: The folio to wait on.
*
* Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio or until a
* fatal signal is received by the calling task.
*
* Return:
* - 0 if successful.
* - -EINTR if a fatal signal was encountered.
*/
int folio_wait_private_2_killable(struct folio *folio)
{
int ret = 0;
while (folio_test_private_2(folio)) {
ret = folio_wait_bit_killable(folio, PG_private_2);
if (ret < 0)
break;
}
return ret;
}
EXPORT_SYMBOL(folio_wait_private_2_killable);
/**
* folio_end_writeback - End writeback against a folio.
* @folio: The folio.
*
* The folio must actually be under writeback.
*
* Context: May be called from process or interrupt context.
*/
void folio_end_writeback(struct folio *folio)
{
VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio);
/*
* folio_test_clear_reclaim() could be used here but it is an
* atomic operation and overkill in this particular case. Failing
* to shuffle a folio marked for immediate reclaim is too mild
* a gain to justify taking an atomic operation penalty at the
* end of every folio writeback.
*/
if (folio_test_reclaim(folio)) {
folio_clear_reclaim(folio);
folio_rotate_reclaimable(folio);
}
/*
* Writeback does not hold a folio reference of its own, relying
* on truncation to wait for the clearing of PG_writeback.
* But here we must make sure that the folio is not freed and
* reused before the folio_wake_bit().
*/
folio_get(folio);
if (__folio_end_writeback(folio))
folio_wake_bit(folio, PG_writeback);
acct_reclaim_writeback(folio);
folio_put(folio);
}
EXPORT_SYMBOL(folio_end_writeback);
/**
* __folio_lock - Get a lock on the folio, assuming we need to sleep to get it.
* @folio: The folio to lock
*/
void __folio_lock(struct folio *folio)
{
folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE,
EXCLUSIVE);
}
EXPORT_SYMBOL(__folio_lock);
int __folio_lock_killable(struct folio *folio)
{
return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE,
EXCLUSIVE);
}
EXPORT_SYMBOL_GPL(__folio_lock_killable);
static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)
{
struct wait_queue_head *q = folio_waitqueue(folio);
int ret;
wait->folio = folio;
wait->bit_nr = PG_locked;
spin_lock_irq(&q->lock);
__add_wait_queue_entry_tail(q, &wait->wait);
folio_set_waiters(folio);
ret = !folio_trylock(folio);
/*
* If we were successful now, we know we're still on the
* waitqueue as we're still under the lock. This means it's
* safe to remove and return success, we know the callback
* isn't going to trigger.
*/
if (!ret)
__remove_wait_queue(q, &wait->wait);
else
ret = -EIOCBQUEUED;
spin_unlock_irq(&q->lock);
return ret;
}
/*
* Return values:
* 0 - folio is locked.
* non-zero - folio is not locked.
* mmap_lock or per-VMA lock has been released (mmap_read_unlock() or
* vma_end_read()), unless flags had both FAULT_FLAG_ALLOW_RETRY and
* FAULT_FLAG_RETRY_NOWAIT set, in which case the lock is still held.
*
* If neither ALLOW_RETRY nor KILLABLE are set, will always return 0
* with the folio locked and the mmap_lock/per-VMA lock is left unperturbed.
*/
vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf)
{
unsigned int flags = vmf->flags;
if (fault_flag_allow_retry_first(flags)) {
/*
* CAUTION! In this case, mmap_lock/per-VMA lock is not
* released even though returning VM_FAULT_RETRY.
*/
if (flags & FAULT_FLAG_RETRY_NOWAIT)
return VM_FAULT_RETRY;
release_fault_lock(vmf);
if (flags & FAULT_FLAG_KILLABLE)
folio_wait_locked_killable(folio);
else
folio_wait_locked(folio);
return VM_FAULT_RETRY;
}
if (flags & FAULT_FLAG_KILLABLE) {
bool ret;
ret = __folio_lock_killable(folio);
if (ret) {
release_fault_lock(vmf);
return VM_FAULT_RETRY;
}
} else {
__folio_lock(folio);
}
return 0;
}
/**
* page_cache_next_miss() - Find the next gap in the page cache.
* @mapping: Mapping.
* @index: Index.
* @max_scan: Maximum range to search.
*
* Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
* gap with the lowest index.
*
* This function may be called under the rcu_read_lock. However, this will
* not atomically search a snapshot of the cache at a single point in time.
* For example, if a gap is created at index 5, then subsequently a gap is
* created at index 10, page_cache_next_miss covering both indices may
* return 10 if called under the rcu_read_lock.
*
* Return: The index of the gap if found, otherwise an index outside the
* range specified (in which case 'return - index >= max_scan' will be true).
* In the rare case of index wrap-around, 0 will be returned.
*/
pgoff_t page_cache_next_miss(struct address_space *mapping,
pgoff_t index, unsigned long max_scan)
{
XA_STATE(xas, &mapping->i_pages, index);
while (max_scan--) {
void *entry = xas_next(&xas);
if (!entry || xa_is_value(entry))
break;
if (xas.xa_index == 0)
break;
}
return xas.xa_index;
}
EXPORT_SYMBOL(page_cache_next_miss);
/**
* page_cache_prev_miss() - Find the previous gap in the page cache.
* @mapping: Mapping.
* @index: Index.
* @max_scan: Maximum range to search.
*
* Search the range [max(index - max_scan + 1, 0), index] for the
* gap with the highest index.
*
* This function may be called under the rcu_read_lock. However, this will
* not atomically search a snapshot of the cache at a single point in time.
* For example, if a gap is created at index 10, then subsequently a gap is
* created at index 5, page_cache_prev_miss() covering both indices may
* return 5 if called under the rcu_read_lock.
*
* Return: The index of the gap if found, otherwise an index outside the
* range specified (in which case 'index - return >= max_scan' will be true).
* In the rare case of wrap-around, ULONG_MAX will be returned.
*/
pgoff_t page_cache_prev_miss(struct address_space *mapping,
pgoff_t index, unsigned long max_scan)
{
XA_STATE(xas, &mapping->i_pages, index);
while (max_scan--) {
void *entry = xas_prev(&xas);
if (!entry || xa_is_value(entry))
break;
if (xas.xa_index == ULONG_MAX)
break;
}
return xas.xa_index;
}
EXPORT_SYMBOL(page_cache_prev_miss);
/*
* Lockless page cache protocol:
* On the lookup side:
* 1. Load the folio from i_pages
* 2. Increment the refcount if it's not zero
* 3. If the folio is not found by xas_reload(), put the refcount and retry
*
* On the removal side:
* A. Freeze the page (by zeroing the refcount if nobody else has a reference)
* B. Remove the page from i_pages
* C. Return the page to the page allocator
*
* This means that any page may have its reference count temporarily
* increased by a speculative page cache (or fast GUP) lookup as it can
* be allocated by another user before the RCU grace period expires.
* Because the refcount temporarily acquired here may end up being the
* last refcount on the page, any page allocation must be freeable by
* folio_put().
*/
/*
* filemap_get_entry - Get a page cache entry.
* @mapping: the address_space to search
* @index: The page cache index.
*
* Looks up the page cache entry at @mapping & @index. If it is a folio,
* it is returned with an increased refcount. If it is a shadow entry
* of a previously evicted folio, or a swap entry from shmem/tmpfs,
* it is returned without further action.
*
* Return: The folio, swap or shadow entry, %NULL if nothing is found.
*/
void *filemap_get_entry(struct address_space *mapping, pgoff_t index)
{
XA_STATE(xas, &mapping->i_pages, index);
struct folio *folio;
rcu_read_lock();
repeat:
xas_reset(&xas);
folio = xas_load(&xas);
if (xas_retry(&xas, folio))
goto repeat;
/*
* A shadow entry of a recently evicted page, or a swap entry from
* shmem/tmpfs. Return it without attempting to raise page count.
*/
if (!folio || xa_is_value(folio))
goto out;
if (!folio_try_get_rcu(folio))
goto repeat;
if (unlikely(folio != xas_reload(&xas))) { folio_put(folio);
goto repeat;
}
out:
rcu_read_unlock();
return folio;
}
/**
* __filemap_get_folio - Find and get a reference to a folio.
* @mapping: The address_space to search.
* @index: The page index.
* @fgp_flags: %FGP flags modify how the folio is returned.
* @gfp: Memory allocation flags to use if %FGP_CREAT is specified.
*
* Looks up the page cache entry at @mapping & @index.
*
* If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even
* if the %GFP flags specified for %FGP_CREAT are atomic.
*
* If this function returns a folio, it is returned with an increased refcount.
*
* Return: The found folio or an ERR_PTR() otherwise.
*/
struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
fgf_t fgp_flags, gfp_t gfp)
{
struct folio *folio;
repeat:
folio = filemap_get_entry(mapping, index);
if (xa_is_value(folio))
folio = NULL;
if (!folio)
goto no_page;
if (fgp_flags & FGP_LOCK) {
if (fgp_flags & FGP_NOWAIT) {
if (!folio_trylock(folio)) {
folio_put(folio);
return ERR_PTR(-EAGAIN);
}
} else {
folio_lock(folio);
}
/* Has the page been truncated? */
if (unlikely(folio->mapping != mapping)) {
folio_unlock(folio);
folio_put(folio);
goto repeat;
}
VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
}
if (fgp_flags & FGP_ACCESSED)
folio_mark_accessed(folio);
else if (fgp_flags & FGP_WRITE) {
/* Clear idle flag for buffer write */
if (folio_test_idle(folio))
folio_clear_idle(folio);
}
if (fgp_flags & FGP_STABLE)
folio_wait_stable(folio);
no_page:
if (!folio && (fgp_flags & FGP_CREAT)) {
unsigned order = FGF_GET_ORDER(fgp_flags);
int err;
if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
gfp |= __GFP_WRITE;
if (fgp_flags & FGP_NOFS)
gfp &= ~__GFP_FS;
if (fgp_flags & FGP_NOWAIT) {
gfp &= ~GFP_KERNEL;
gfp |= GFP_NOWAIT | __GFP_NOWARN;
}
if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))
fgp_flags |= FGP_LOCK;
if (!mapping_large_folio_support(mapping))
order = 0;
if (order > MAX_PAGECACHE_ORDER)
order = MAX_PAGECACHE_ORDER;
/* If we're not aligned, allocate a smaller folio */
if (index & ((1UL << order) - 1))
order = __ffs(index);
do {
gfp_t alloc_gfp = gfp;
err = -ENOMEM;
if (order > 0)
alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
folio = filemap_alloc_folio(alloc_gfp, order);
if (!folio)
continue;
/* Init accessed so avoid atomic mark_page_accessed later */
if (fgp_flags & FGP_ACCESSED)
__folio_set_referenced(folio);
err = filemap_add_folio(mapping, folio, index, gfp);
if (!err)
break;
folio_put(folio);
folio = NULL;
} while (order-- > 0);
if (err == -EEXIST)
goto repeat;
if (err)
return ERR_PTR(err);
/*
* filemap_add_folio locks the page, and for mmap
* we expect an unlocked page.
*/
if (folio && (fgp_flags & FGP_FOR_MMAP))
folio_unlock(folio);
}
if (!folio)
return ERR_PTR(-ENOENT);
return folio;
}
EXPORT_SYMBOL(__filemap_get_folio);
static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
xa_mark_t mark)
{
struct folio *folio;
retry:
if (mark == XA_PRESENT)
folio = xas_find(xas, max);
else
folio = xas_find_marked(xas, max, mark);
if (xas_retry(xas, folio))
goto retry;
/*
* A shadow entry of a recently evicted page, a swap
* entry from shmem/tmpfs or a DAX entry. Return it
* without attempting to raise page count.
*/
if (!folio || xa_is_value(folio))
return folio;
if (!folio_try_get_rcu(folio))
goto reset;
if (unlikely(folio != xas_reload(xas))) { folio_put(folio);
goto reset;
}
return folio;
reset:
xas_reset(xas);
goto retry;
}
/**
* find_get_entries - gang pagecache lookup
* @mapping: The address_space to search
* @start: The starting page cache index
* @end: The final page index (inclusive).
* @fbatch: Where the resulting entries are placed.
* @indices: The cache indices corresponding to the entries in @entries
*
* find_get_entries() will search for and return a batch of entries in
* the mapping. The entries are placed in @fbatch. find_get_entries()
* takes a reference on any actual folios it returns.
*
* The entries have ascending indexes. The indices may not be consecutive
* due to not-present entries or large folios.
*
* Any shadow entries of evicted folios, or swap entries from
* shmem/tmpfs, are included in the returned array.
*
* Return: The number of entries which were found.
*/
unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
{
XA_STATE(xas, &mapping->i_pages, *start);
struct folio *folio;
rcu_read_lock(); while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { indices[fbatch->nr] = xas.xa_index;
if (!folio_batch_add(fbatch, folio))
break;
}
rcu_read_unlock();
if (folio_batch_count(fbatch)) {
unsigned long nr = 1;
int idx = folio_batch_count(fbatch) - 1;
folio = fbatch->folios[idx];
if (!xa_is_value(folio))
nr = folio_nr_pages(folio); *start = indices[idx] + nr;
}
return folio_batch_count(fbatch);
}
/**
* find_lock_entries - Find a batch of pagecache entries.
* @mapping: The address_space to search.
* @start: The starting page cache index.
* @end: The final page index (inclusive).
* @fbatch: Where the resulting entries are placed.
* @indices: The cache indices of the entries in @fbatch.
*
* find_lock_entries() will return a batch of entries from @mapping.
* Swap, shadow and DAX entries are included. Folios are returned
* locked and with an incremented refcount. Folios which are locked
* by somebody else or under writeback are skipped. Folios which are
* partially outside the range are not returned.
*
* The entries have ascending indexes. The indices may not be consecutive
* due to not-present entries, large folios, folios which could not be
* locked or folios under writeback.
*
* Return: The number of entries which were found.
*/
unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
{
XA_STATE(xas, &mapping->i_pages, *start);
struct folio *folio;
rcu_read_lock(); while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { if (!xa_is_value(folio)) { if (folio->index < *start)
goto put;
if (folio_next_index(folio) - 1 > end)
goto put;
if (!folio_trylock(folio))
goto put;
if (folio->mapping != mapping || folio_test_writeback(folio))
goto unlock;
VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
folio);
}
indices[fbatch->nr] = xas.xa_index;
if (!folio_batch_add(fbatch, folio))
break;
continue;
unlock:
folio_unlock(folio);
put:
folio_put(folio);
}
rcu_read_unlock();
if (folio_batch_count(fbatch)) {
unsigned long nr = 1;
int idx = folio_batch_count(fbatch) - 1;
folio = fbatch->folios[idx];
if (!xa_is_value(folio))
nr = folio_nr_pages(folio); *start = indices[idx] + nr;
}
return folio_batch_count(fbatch);
}
/**
* filemap_get_folios - Get a batch of folios
* @mapping: The address_space to search
* @start: The starting page index
* @end: The final page index (inclusive)
* @fbatch: The batch to fill.
*
* Search for and return a batch of folios in the mapping starting at
* index @start and up to index @end (inclusive). The folios are returned
* in @fbatch with an elevated reference count.
*
* Return: The number of folios which were found.
* We also update @start to index the next folio for the traversal.
*/
unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch)
{
return filemap_get_folios_tag(mapping, start, end, XA_PRESENT, fbatch);
}
EXPORT_SYMBOL(filemap_get_folios);
/**
* filemap_get_folios_contig - Get a batch of contiguous folios
* @mapping: The address_space to search
* @start: The starting page index
* @end: The final page index (inclusive)
* @fbatch: The batch to fill
*
* filemap_get_folios_contig() works exactly like filemap_get_folios(),
* except the returned folios are guaranteed to be contiguous. This may
* not return all contiguous folios if the batch gets filled up.
*
* Return: The number of folios found.
* Also update @start to be positioned for traversal of the next folio.
*/
unsigned filemap_get_folios_contig(struct address_space *mapping,
pgoff_t *start, pgoff_t end, struct folio_batch *fbatch)
{
XA_STATE(xas, &mapping->i_pages, *start);
unsigned long nr;
struct folio *folio;
rcu_read_lock();
for (folio = xas_load(&xas); folio && xas.xa_index <= end;
folio = xas_next(&xas)) {
if (xas_retry(&xas, folio))
continue;
/*
* If the entry has been swapped out, we can stop looking.
* No current caller is looking for DAX entries.
*/
if (xa_is_value(folio))
goto update_start;
if (!folio_try_get_rcu(folio))
goto retry;
if (unlikely(folio != xas_reload(&xas)))
goto put_folio;
if (!folio_batch_add(fbatch, folio)) {
nr = folio_nr_pages(folio);
*start = folio->index + nr;
goto out;
}
continue;
put_folio:
folio_put(folio);
retry:
xas_reset(&xas);
}
update_start:
nr = folio_batch_count(fbatch);
if (nr) {
folio = fbatch->folios[nr - 1];
*start = folio_next_index(folio);
}
out:
rcu_read_unlock();
return folio_batch_count(fbatch);
}
EXPORT_SYMBOL(filemap_get_folios_contig);
/**
* filemap_get_folios_tag - Get a batch of folios matching @tag
* @mapping: The address_space to search
* @start: The starting page index
* @end: The final page index (inclusive)
* @tag: The tag index
* @fbatch: The batch to fill
*
* The first folio may start before @start; if it does, it will contain
* @start. The final folio may extend beyond @end; if it does, it will
* contain @end. The folios have ascending indices. There may be gaps
* between the folios if there are indices which have no folio in the
* page cache. If folios are added to or removed from the page cache
* while this is running, they may or may not be found by this call.
* Only returns folios that are tagged with @tag.
*
* Return: The number of folios found.
* Also update @start to index the next folio for traversal.
*/
unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,
pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch)
{
XA_STATE(xas, &mapping->i_pages, *start);
struct folio *folio;
rcu_read_lock();
while ((folio = find_get_entry(&xas, end, tag)) != NULL) {
/*
* Shadow entries should never be tagged, but this iteration
* is lockless so there is a window for page reclaim to evict
* a page we saw tagged. Skip over it.
*/
if (xa_is_value(folio))
continue;
if (!folio_batch_add(fbatch, folio)) {
unsigned long nr = folio_nr_pages(folio);
*start = folio->index + nr;
goto out;
}
}
/*
* We come here when there is no page beyond @end. We take care to not
* overflow the index @start as it confuses some of the callers. This
* breaks the iteration when there is a page at index -1 but that is
* already broke anyway.
*/
if (end == (pgoff_t)-1)
*start = (pgoff_t)-1;
else
*start = end + 1;
out:
rcu_read_unlock();
return folio_batch_count(fbatch);
}
EXPORT_SYMBOL(filemap_get_folios_tag);
/*
* CD/DVDs are error prone. When a medium error occurs, the driver may fail
* a _large_ part of the i/o request. Imagine the worst scenario:
*
* ---R__________________________________________B__________
* ^ reading here ^ bad block(assume 4k)
*
* read(R) => miss => readahead(R...B) => media error => frustrating retries
* => failing the whole request => read(R) => read(R+1) =>
* readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
* readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
* readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
*
* It is going insane. Fix it by quickly scaling down the readahead size.
*/
static void shrink_readahead_size_eio(struct file_ra_state *ra)
{
ra->ra_pages /= 4;
}
/*
* filemap_get_read_batch - Get a batch of folios for read
*
* Get a batch of folios which represent a contiguous range of bytes in
* the file. No exceptional entries will be returned. If @index is in
* the middle of a folio, the entire folio will be returned. The last
* folio in the batch may have the readahead flag set or the uptodate flag
* clear so that the caller can take the appropriate action.
*/
static void filemap_get_read_batch(struct address_space *mapping,
pgoff_t index, pgoff_t max, struct folio_batch *fbatch)
{
XA_STATE(xas, &mapping->i_pages, index);
struct folio *folio;
rcu_read_lock();
for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
if (xas_retry(&xas, folio))
continue;
if (xas.xa_index > max || xa_is_value(folio))
break;
if (xa_is_sibling(folio))
break;
if (!folio_try_get_rcu(folio))
goto retry;
if (unlikely(folio != xas_reload(&xas)))
goto put_folio;
if (!folio_batch_add(fbatch, folio))
break;
if (!folio_test_uptodate(folio))
break;
if (folio_test_readahead(folio))
break;
xas_advance(&xas, folio_next_index(folio) - 1);
continue;
put_folio:
folio_put(folio);
retry:
xas_reset(&xas);
}
rcu_read_unlock();
}
static int filemap_read_folio(struct file *file, filler_t filler,
struct folio *folio)
{
bool workingset = folio_test_workingset(folio);
unsigned long pflags;
int error;
/*
* A previous I/O error may have been due to temporary failures,
* eg. multipath errors. PG_error will be set again if read_folio
* fails.
*/
folio_clear_error(folio);
/* Start the actual read. The read will unlock the page. */
if (unlikely(workingset))
psi_memstall_enter(&pflags);
error = filler(file, folio);
if (unlikely(workingset))
psi_memstall_leave(&pflags);
if (error)
return error;
error = folio_wait_locked_killable(folio);
if (error)
return error;
if (folio_test_uptodate(folio))
return 0;
if (file)
shrink_readahead_size_eio(&file->f_ra);
return -EIO;
}
static bool filemap_range_uptodate(struct address_space *mapping,
loff_t pos, size_t count, struct folio *folio,
bool need_uptodate)
{
if (folio_test_uptodate(folio))
return true;
/* pipes can't handle partially uptodate pages */
if (need_uptodate)
return false;
if (!mapping->a_ops->is_partially_uptodate)
return false;
if (mapping->host->i_blkbits >= folio_shift(folio))
return false;
if (folio_pos(folio) > pos) {
count -= folio_pos(folio) - pos;
pos = 0;
} else {
pos -= folio_pos(folio);
}
return mapping->a_ops->is_partially_uptodate(folio, pos, count);
}
static int filemap_update_page(struct kiocb *iocb,
struct address_space *mapping, size_t count,
struct folio *folio, bool need_uptodate)
{
int error;
if (iocb->ki_flags & IOCB_NOWAIT) {
if (!filemap_invalidate_trylock_shared(mapping))
return -EAGAIN;
} else {
filemap_invalidate_lock_shared(mapping);
}
if (!folio_trylock(folio)) {
error = -EAGAIN;
if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO))
goto unlock_mapping;
if (!(iocb->ki_flags & IOCB_WAITQ)) {
filemap_invalidate_unlock_shared(mapping);
/*
* This is where we usually end up waiting for a
* previously submitted readahead to finish.
*/
folio_put_wait_locked(folio, TASK_KILLABLE);
return AOP_TRUNCATED_PAGE;
}
error = __folio_lock_async(folio, iocb->ki_waitq);
if (error)
goto unlock_mapping;
}
error = AOP_TRUNCATED_PAGE;
if (!folio->mapping)
goto unlock;
error = 0;
if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio,
need_uptodate))
goto unlock;
error = -EAGAIN;
if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ))
goto unlock;
error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio,
folio);
goto unlock_mapping;
unlock:
folio_unlock(folio);
unlock_mapping:
filemap_invalidate_unlock_shared(mapping);
if (error == AOP_TRUNCATED_PAGE)
folio_put(folio);
return error;
}
static int filemap_create_folio(struct file *file,
struct address_space *mapping, pgoff_t index,
struct folio_batch *fbatch)
{
struct folio *folio;
int error;
folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0);
if (!folio)
return -ENOMEM;
/*
* Protect against truncate / hole punch. Grabbing invalidate_lock
* here assures we cannot instantiate and bring uptodate new
* pagecache folios after evicting page cache during truncate
* and before actually freeing blocks. Note that we could
* release invalidate_lock after inserting the folio into
* the page cache as the locked folio would then be enough to
* synchronize with hole punching. But there are code paths
* such as filemap_update_page() filling in partially uptodate
* pages or ->readahead() that need to hold invalidate_lock
* while mapping blocks for IO so let's hold the lock here as
* well to keep locking rules simple.
*/
filemap_invalidate_lock_shared(mapping);
error = filemap_add_folio(mapping, folio, index,
mapping_gfp_constraint(mapping, GFP_KERNEL));
if (error == -EEXIST)
error = AOP_TRUNCATED_PAGE;
if (error)
goto error;
error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);
if (error)
goto error;
filemap_invalidate_unlock_shared(mapping);
folio_batch_add(fbatch, folio);
return 0;
error:
filemap_invalidate_unlock_shared(mapping);
folio_put(folio);
return error;
}
static int filemap_readahead(struct kiocb *iocb, struct file *file,
struct address_space *mapping, struct folio *folio,
pgoff_t last_index)
{
DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index);
if (iocb->ki_flags & IOCB_NOIO)
return -EAGAIN;
page_cache_async_ra(&ractl, folio, last_index - folio->index);
return 0;
}
static int filemap_get_pages(struct kiocb *iocb, size_t count,
struct folio_batch *fbatch, bool need_uptodate)
{
struct file *filp = iocb->ki_filp;
struct address_space *mapping = filp->f_mapping;
struct file_ra_state *ra = &filp->f_ra;
pgoff_t index = iocb->ki_pos >> PAGE_SHIFT;
pgoff_t last_index;
struct folio *folio;
int err = 0;
/* "last_index" is the index of the page beyond the end of the read */
last_index = DIV_ROUND_UP(iocb->ki_pos + count, PAGE_SIZE);
retry:
if (fatal_signal_pending(current))
return -EINTR;
filemap_get_read_batch(mapping, index, last_index - 1, fbatch);
if (!folio_batch_count(fbatch)) {
if (iocb->ki_flags & IOCB_NOIO)
return -EAGAIN;
page_cache_sync_readahead(mapping, ra, filp, index,
last_index - index);
filemap_get_read_batch(mapping, index, last_index - 1, fbatch);
}
if (!folio_batch_count(fbatch)) {
if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))
return -EAGAIN;
err = filemap_create_folio(filp, mapping,
iocb->ki_pos >> PAGE_SHIFT, fbatch);
if (err == AOP_TRUNCATED_PAGE)
goto retry;
return err;
}
folio = fbatch->folios[folio_batch_count(fbatch) - 1];
if (folio_test_readahead(folio)) {
err = filemap_readahead(iocb, filp, mapping, folio, last_index);
if (err)
goto err;
}
if (!folio_test_uptodate(folio)) {
if ((iocb->ki_flags & IOCB_WAITQ) &&
folio_batch_count(fbatch) > 1)
iocb->ki_flags |= IOCB_NOWAIT;
err = filemap_update_page(iocb, mapping, count, folio,
need_uptodate);
if (err)
goto err;
}
return 0;
err:
if (err < 0)
folio_put(folio);
if (likely(--fbatch->nr))
return 0;
if (err == AOP_TRUNCATED_PAGE)
goto retry;
return err;
}
static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio)
{
unsigned int shift = folio_shift(folio);
return (pos1 >> shift == pos2 >> shift);
}
/**
* filemap_read - Read data from the page cache.
* @iocb: The iocb to read.
* @iter: Destination for the data.
* @already_read: Number of bytes already read by the caller.
*
* Copies data from the page cache. If the data is not currently present,
* uses the readahead and read_folio address_space operations to fetch it.
*
* Return: Total number of bytes copied, including those already read by
* the caller. If an error happens before any bytes are copied, returns
* a negative error number.
*/
ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
ssize_t already_read)
{
struct file *filp = iocb->ki_filp;
struct file_ra_state *ra = &filp->f_ra;
struct address_space *mapping = filp->f_mapping;
struct inode *inode = mapping->host;
struct folio_batch fbatch;
int i, error = 0;
bool writably_mapped;
loff_t isize, end_offset;
loff_t last_pos = ra->prev_pos;
if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes))
return 0;
if (unlikely(!iov_iter_count(iter)))
return 0;
iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
folio_batch_init(&fbatch);
do {
cond_resched();
/*
* If we've already successfully copied some data, then we
* can no longer safely return -EIOCBQUEUED. Hence mark
* an async read NOWAIT at that point.
*/
if ((iocb->ki_flags & IOCB_WAITQ) && already_read)
iocb->ki_flags |= IOCB_NOWAIT;
if (unlikely(iocb->ki_pos >= i_size_read(inode)))
break;
error = filemap_get_pages(iocb, iter->count, &fbatch, false);
if (error < 0)
break;
/*
* i_size must be checked after we know the pages are Uptodate.
*
* Checking i_size after the check allows us to calculate
* the correct value for "nr", which means the zero-filled
* part of the page is not copied back to userspace (unless
* another truncate extends the file - this is desired though).
*/
isize = i_size_read(inode);
if (unlikely(iocb->ki_pos >= isize))
goto put_folios;
end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count);
/*
* Once we start copying data, we don't want to be touching any
* cachelines that might be contended:
*/
writably_mapped = mapping_writably_mapped(mapping);
/*
* When a read accesses the same folio several times, only
* mark it as accessed the first time.
*/
if (!pos_same_folio(iocb->ki_pos, last_pos - 1,
fbatch.folios[0]))
folio_mark_accessed(fbatch.folios[0]);
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
size_t fsize = folio_size(folio);
size_t offset = iocb->ki_pos & (fsize - 1);
size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos,
fsize - offset);
size_t copied;
if (end_offset < folio_pos(folio))
break;
if (i > 0)
folio_mark_accessed(folio);
/*
* If users can be writing to this folio using arbitrary
* virtual addresses, take care of potential aliasing
* before reading the folio on the kernel side.
*/
if (writably_mapped)
flush_dcache_folio(folio);
copied = copy_folio_to_iter(folio, offset, bytes, iter);
already_read += copied;
iocb->ki_pos += copied;
last_pos = iocb->ki_pos;
if (copied < bytes) {
error = -EFAULT;
break;
}
}
put_folios:
for (i = 0; i < folio_batch_count(&fbatch); i++)
folio_put(fbatch.folios[i]);
folio_batch_init(&fbatch);
} while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);
file_accessed(filp);
ra->prev_pos = last_pos;
return already_read ? already_read : error;
}
EXPORT_SYMBOL_GPL(filemap_read);
int kiocb_write_and_wait(struct kiocb *iocb, size_t count)
{
struct address_space *mapping = iocb->ki_filp->f_mapping;
loff_t pos = iocb->ki_pos;
loff_t end = pos + count - 1;
if (iocb->ki_flags & IOCB_NOWAIT) {
if (filemap_range_needs_writeback(mapping, pos, end))
return -EAGAIN;
return 0;
}
return filemap_write_and_wait_range(mapping, pos, end);
}
EXPORT_SYMBOL_GPL(kiocb_write_and_wait);
int kiocb_invalidate_pages(struct kiocb *iocb, size_t count)
{
struct address_space *mapping = iocb->ki_filp->f_mapping;
loff_t pos = iocb->ki_pos;
loff_t end = pos + count - 1;
int ret;
if (iocb->ki_flags & IOCB_NOWAIT) {
/* we could block if there are any pages in the range */
if (filemap_range_has_page(mapping, pos, end))
return -EAGAIN;
} else {
ret = filemap_write_and_wait_range(mapping, pos, end);
if (ret)
return ret;
}
/*
* After a write we want buffered reads to be sure to go to disk to get
* the new data. We invalidate clean cached page from the region we're
* about to write. We do this *before* the write so that we can return
* without clobbering -EIOCBQUEUED from ->direct_IO().
*/
return invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT,
end >> PAGE_SHIFT);
}
EXPORT_SYMBOL_GPL(kiocb_invalidate_pages);
/**
* generic_file_read_iter - generic filesystem read routine
* @iocb: kernel I/O control block
* @iter: destination for the data read
*
* This is the "read_iter()" routine for all filesystems
* that can use the page cache directly.
*
* The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
* be returned when no data can be read without waiting for I/O requests
* to complete; it doesn't prevent readahead.
*
* The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
* requests shall be made for the read or for readahead. When no data
* can be read, -EAGAIN shall be returned. When readahead would be
* triggered, a partial, possibly empty read shall be returned.
*
* Return:
* * number of bytes copied, even for partial reads
* * negative error code (or 0 if IOCB_NOIO) if nothing was read
*/
ssize_t
generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
size_t count = iov_iter_count(iter);
ssize_t retval = 0;
if (!count)
return 0; /* skip atime */
if (iocb->ki_flags & IOCB_DIRECT) {
struct file *file = iocb->ki_filp;
struct address_space *mapping = file->f_mapping;
struct inode *inode = mapping->host;
retval = kiocb_write_and_wait(iocb, count);
if (retval < 0)
return retval;
file_accessed(file);
retval = mapping->a_ops->direct_IO(iocb, iter);
if (retval >= 0) {
iocb->ki_pos += retval;
count -= retval;
}
if (retval != -EIOCBQUEUED)
iov_iter_revert(iter, count - iov_iter_count(iter));
/*
* Btrfs can have a short DIO read if we encounter
* compressed extents, so if there was an error, or if
* we've already read everything we wanted to, or if
* there was a short read because we hit EOF, go ahead
* and return. Otherwise fallthrough to buffered io for
* the rest of the read. Buffered reads will not work for
* DAX files, so don't bother trying.
*/
if (retval < 0 || !count || IS_DAX(inode))
return retval;
if (iocb->ki_pos >= i_size_read(inode))
return retval;
}
return filemap_read(iocb, iter, retval);
}
EXPORT_SYMBOL(generic_file_read_iter);
/*
* Splice subpages from a folio into a pipe.
*/
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
struct folio *folio, loff_t fpos, size_t size)
{
struct page *page;
size_t spliced = 0, offset = offset_in_folio(folio, fpos);
page = folio_page(folio, offset / PAGE_SIZE);
size = min(size, folio_size(folio) - offset);
offset %= PAGE_SIZE;
while (spliced < size &&
!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
struct pipe_buffer *buf = pipe_head_buf(pipe);
size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced);
*buf = (struct pipe_buffer) {
.ops = &page_cache_pipe_buf_ops,
.page = page,
.offset = offset,
.len = part,
};
folio_get(folio);
pipe->head++;
page++;
spliced += part;
offset = 0;
}
return spliced;
}
/**
* filemap_splice_read - Splice data from a file's pagecache into a pipe
* @in: The file to read from
* @ppos: Pointer to the file position to read from
* @pipe: The pipe to splice into
* @len: The amount to splice
* @flags: The SPLICE_F_* flags
*
* This function gets folios from a file's pagecache and splices them into the
* pipe. Readahead will be called as necessary to fill more folios. This may
* be used for blockdevs also.
*
* Return: On success, the number of bytes read will be returned and *@ppos
* will be updated if appropriate; 0 will be returned if there is no more data
* to be read; -EAGAIN will be returned if the pipe had no space, and some
* other negative error code will be returned on error. A short read may occur
* if the pipe has insufficient space, we reach the end of the data or we hit a
* hole.
*/
ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe,
size_t len, unsigned int flags)
{
struct folio_batch fbatch;
struct kiocb iocb;
size_t total_spliced = 0, used, npages;
loff_t isize, end_offset;
bool writably_mapped;
int i, error = 0;
if (unlikely(*ppos >= in->f_mapping->host->i_sb->s_maxbytes))
return 0;
init_sync_kiocb(&iocb, in);
iocb.ki_pos = *ppos;
/* Work out how much data we can actually add into the pipe */
used = pipe_occupancy(pipe->head, pipe->tail);
npages = max_t(ssize_t, pipe->max_usage - used, 0);
len = min_t(size_t, len, npages * PAGE_SIZE);
folio_batch_init(&fbatch);
do {
cond_resched();
if (*ppos >= i_size_read(in->f_mapping->host))
break;
iocb.ki_pos = *ppos;
error = filemap_get_pages(&iocb, len, &fbatch, true);
if (error < 0)
break;
/*
* i_size must be checked after we know the pages are Uptodate.
*
* Checking i_size after the check allows us to calculate
* the correct value for "nr", which means the zero-filled
* part of the page is not copied back to userspace (unless
* another truncate extends the file - this is desired though).
*/
isize = i_size_read(in->f_mapping->host);
if (unlikely(*ppos >= isize))
break;
end_offset = min_t(loff_t, isize, *ppos + len);
/*
* Once we start copying data, we don't want to be touching any
* cachelines that might be contended:
*/
writably_mapped = mapping_writably_mapped(in->f_mapping);
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
size_t n;
if (folio_pos(folio) >= end_offset)
goto out;
folio_mark_accessed(folio);
/*
* If users can be writing to this folio using arbitrary
* virtual addresses, take care of potential aliasing
* before reading the folio on the kernel side.
*/
if (writably_mapped)
flush_dcache_folio(folio);
n = min_t(loff_t, len, isize - *ppos);
n = splice_folio_into_pipe(pipe, folio, *ppos, n);
if (!n)
goto out;
len -= n;
total_spliced += n;
*ppos += n;
in->f_ra.prev_pos = *ppos;
if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
goto out;
}
folio_batch_release(&fbatch);
} while (len);
out:
folio_batch_release(&fbatch);
file_accessed(in);
return total_spliced ? total_spliced : error;
}
EXPORT_SYMBOL(filemap_splice_read);
static inline loff_t folio_seek_hole_data(struct xa_state *xas,
struct address_space *mapping, struct folio *folio,
loff_t start, loff_t end, bool seek_data)
{
const struct address_space_operations *ops = mapping->a_ops;
size_t offset, bsz = i_blocksize(mapping->host);
if (xa_is_value(folio) || folio_test_uptodate(folio))
return seek_data ? start : end;
if (!ops->is_partially_uptodate)
return seek_data ? end : start;
xas_pause(xas);
rcu_read_unlock();
folio_lock(folio);
if (unlikely(folio->mapping != mapping))
goto unlock;
offset = offset_in_folio(folio, start) & ~(bsz - 1);
do {
if (ops->is_partially_uptodate(folio, offset, bsz) ==
seek_data)
break;
start = (start + bsz) & ~(bsz - 1);
offset += bsz;
} while (offset < folio_size(folio));
unlock:
folio_unlock(folio);
rcu_read_lock();
return start;
}
static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
{
if (xa_is_value(folio))
return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
return folio_size(folio);
}
/**
* mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
* @mapping: Address space to search.
* @start: First byte to consider.
* @end: Limit of search (exclusive).
* @whence: Either SEEK_HOLE or SEEK_DATA.
*
* If the page cache knows which blocks contain holes and which blocks
* contain data, your filesystem can use this function to implement
* SEEK_HOLE and SEEK_DATA. This is useful for filesystems which are
* entirely memory-based such as tmpfs, and filesystems which support
* unwritten extents.
*
* Return: The requested offset on success, or -ENXIO if @whence specifies
* SEEK_DATA and there is no data after @start. There is an implicit hole
* after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
* and @end contain data.
*/
loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
loff_t end, int whence)
{
XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT);
pgoff_t max = (end - 1) >> PAGE_SHIFT;
bool seek_data = (whence == SEEK_DATA);
struct folio *folio;
if (end <= start)
return -ENXIO;
rcu_read_lock();
while ((folio = find_get_entry(&xas, max, XA_PRESENT))) {
loff_t pos = (u64)xas.xa_index << PAGE_SHIFT;
size_t seek_size;
if (start < pos) {
if (!seek_data)
goto unlock;
start = pos;
}
seek_size = seek_folio_size(&xas, folio);
pos = round_up((u64)pos + 1, seek_size);
start = folio_seek_hole_data(&xas, mapping, folio, start, pos,
seek_data);
if (start < pos)
goto unlock;
if (start >= end)
break;
if (seek_size > PAGE_SIZE)
xas_set(&xas, pos >> PAGE_SHIFT);
if (!xa_is_value(folio))
folio_put(folio);
}
if (seek_data)
start = -ENXIO;
unlock:
rcu_read_unlock();
if (folio && !xa_is_value(folio))
folio_put(folio);
if (start > end)
return end;
return start;
}
#ifdef CONFIG_MMU
#define MMAP_LOTSAMISS (100)
/*
* lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
* @vmf - the vm_fault for this fault.
* @folio - the folio to lock.
* @fpin - the pointer to the file we may pin (or is already pinned).
*
* This works similar to lock_folio_or_retry in that it can drop the
* mmap_lock. It differs in that it actually returns the folio locked
* if it returns 1 and 0 if it couldn't lock the folio. If we did have
* to drop the mmap_lock then fpin will point to the pinned file and
* needs to be fput()'ed at a later point.
*/
static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio,
struct file **fpin)
{
if (folio_trylock(folio))
return 1;
/*
* NOTE! This will make us return with VM_FAULT_RETRY, but with
* the fault lock still held. That's how FAULT_FLAG_RETRY_NOWAIT
* is supposed to work. We have way too many special cases..
*/
if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
return 0;
*fpin = maybe_unlock_mmap_for_io(vmf, *fpin);
if (vmf->flags & FAULT_FLAG_KILLABLE) {
if (__folio_lock_killable(folio)) {
/*
* We didn't have the right flags to drop the
* fault lock, but all fault_handlers only check
* for fatal signals if we return VM_FAULT_RETRY,
* so we need to drop the fault lock here and
* return 0 if we don't have a fpin.
*/
if (*fpin == NULL)
release_fault_lock(vmf);
return 0;
}
} else
__folio_lock(folio);
return 1;
}
/*
* Synchronous readahead happens when we don't even find a page in the page
* cache at all. We don't want to perform IO under the mmap sem, so if we have
* to drop the mmap sem we return the file that was pinned in order for us to do
* that. If we didn't pin a file then we return NULL. The file that is
* returned needs to be fput()'ed when we're done with it.
*/
static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
{
struct file *file = vmf->vma->vm_file;
struct file_ra_state *ra = &file->f_ra;
struct address_space *mapping = file->f_mapping;
DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff);
struct file *fpin = NULL;
unsigned long vm_flags = vmf->vma->vm_flags;
unsigned int mmap_miss;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/* Use the readahead code, even if readahead is disabled */
if (vm_flags & VM_HUGEPAGE) {
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1);
ra->size = HPAGE_PMD_NR;
/*
* Fetch two PMD folios, so we get the chance to actually
* readahead, unless we've been told not to.
*/
if (!(vm_flags & VM_RAND_READ))
ra->size *= 2;
ra->async_size = HPAGE_PMD_NR;
page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER);
return fpin;
}
#endif
/* If we don't want any read-ahead, don't bother */
if (vm_flags & VM_RAND_READ)
return fpin;
if (!ra->ra_pages)
return fpin;
if (vm_flags & VM_SEQ_READ) {
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
page_cache_sync_ra(&ractl, ra->ra_pages);
return fpin;
}
/* Avoid banging the cache line if not needed */
mmap_miss = READ_ONCE(ra->mmap_miss);
if (mmap_miss < MMAP_LOTSAMISS * 10)
WRITE_ONCE(ra->mmap_miss, ++mmap_miss);
/*
* Do we miss much more than hit in this file? If so,
* stop bothering with read-ahead. It will only hurt.
*/
if (mmap_miss > MMAP_LOTSAMISS)
return fpin;
/*
* mmap read-around
*/
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
ra->size = ra->ra_pages;
ra->async_size = ra->ra_pages / 4;
ractl._index = ra->start;
page_cache_ra_order(&ractl, ra, 0);
return fpin;
}
/*
* Asynchronous readahead happens when we find the page and PG_readahead,
* so we want to possibly extend the readahead further. We return the file that
* was pinned if we have to drop the mmap_lock in order to do IO.
*/
static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
struct folio *folio)
{
struct file *file = vmf->vma->vm_file;
struct file_ra_state *ra = &file->f_ra;
DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff);
struct file *fpin = NULL;
unsigned int mmap_miss;
/* If we don't want any read-ahead, don't bother */
if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages)
return fpin;
mmap_miss = READ_ONCE(ra->mmap_miss);
if (mmap_miss)
WRITE_ONCE(ra->mmap_miss, --mmap_miss);
if (folio_test_readahead(folio)) {
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
page_cache_async_ra(&ractl, folio, ra->ra_pages);
}
return fpin;
}
static vm_fault_t filemap_fault_recheck_pte_none(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret = 0;
pte_t *ptep;
/*
* We might have COW'ed a pagecache folio and might now have an mlocked
* anon folio mapped. The original pagecache folio is not mlocked and
* might have been evicted. During a read+clear/modify/write update of
* the PTE, such as done in do_numa_page()/change_pte_range(), we
* temporarily clear the PTE under PT lock and might detect it here as
* "none" when not holding the PT lock.
*
* Not rechecking the PTE under PT lock could result in an unexpected
* major fault in an mlock'ed region. Recheck only for this special
* scenario while holding the PT lock, to not degrade non-mlocked
* scenarios. Recheck the PTE without PT lock firstly, thereby reducing
* the number of times we hold PT lock.
*/
if (!(vma->vm_flags & VM_LOCKED))
return 0;
if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
return 0;
ptep = pte_offset_map(vmf->pmd, vmf->address);
if (unlikely(!ptep))
return VM_FAULT_NOPAGE;
if (unlikely(!pte_none(ptep_get_lockless(ptep)))) {
ret = VM_FAULT_NOPAGE;
} else {
spin_lock(vmf->ptl);
if (unlikely(!pte_none(ptep_get(ptep))))
ret = VM_FAULT_NOPAGE;
spin_unlock(vmf->ptl);
}
pte_unmap(ptep);
return ret;
}
/**
* filemap_fault - read in file data for page fault handling
* @vmf: struct vm_fault containing details of the fault
*
* filemap_fault() is invoked via the vma operations vector for a
* mapped memory region to read in file data during a page fault.
*
* The goto's are kind of ugly, but this streamlines the normal case of having
* it in the page cache, and handles the special cases reasonably without
* having a lot of duplicated code.
*
* vma->vm_mm->mmap_lock must be held on entry.
*
* If our return value has VM_FAULT_RETRY set, it's because the mmap_lock
* may be dropped before doing I/O or by lock_folio_maybe_drop_mmap().
*
* If our return value does not have VM_FAULT_RETRY set, the mmap_lock
* has not been released.
*
* We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
*
* Return: bitwise-OR of %VM_FAULT_ codes.
*/
vm_fault_t filemap_fault(struct vm_fault *vmf)
{
int error;
struct file *file = vmf->vma->vm_file;
struct file *fpin = NULL;
struct address_space *mapping = file->f_mapping;
struct inode *inode = mapping->host;
pgoff_t max_idx, index = vmf->pgoff;
struct folio *folio;
vm_fault_t ret = 0;
bool mapping_locked = false;
max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
if (unlikely(index >= max_idx))
return VM_FAULT_SIGBUS;
/*
* Do we have something in the page cache already?
*/
folio = filemap_get_folio(mapping, index);
if (likely(!IS_ERR(folio))) {
/*
* We found the page, so try async readahead before waiting for
* the lock.
*/
if (!(vmf->flags & FAULT_FLAG_TRIED))
fpin = do_async_mmap_readahead(vmf, folio);
if (unlikely(!folio_test_uptodate(folio))) {
filemap_invalidate_lock_shared(mapping);
mapping_locked = true;
}
} else {
ret = filemap_fault_recheck_pte_none(vmf);
if (unlikely(ret))
return ret;
/* No page in the page cache at all */
count_vm_event(PGMAJFAULT);
count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
ret = VM_FAULT_MAJOR;
fpin = do_sync_mmap_readahead(vmf);
retry_find:
/*
* See comment in filemap_create_folio() why we need
* invalidate_lock
*/
if (!mapping_locked) {
filemap_invalidate_lock_shared(mapping);
mapping_locked = true;
}
folio = __filemap_get_folio(mapping, index,
FGP_CREAT|FGP_FOR_MMAP,
vmf->gfp_mask);
if (IS_ERR(folio)) {
if (fpin)
goto out_retry;
filemap_invalidate_unlock_shared(mapping);
return VM_FAULT_OOM;
}
}
if (!lock_folio_maybe_drop_mmap(vmf, folio, &fpin))
goto out_retry;
/* Did it get truncated? */
if (unlikely(folio->mapping != mapping)) {
folio_unlock(folio);
folio_put(folio);
goto retry_find;
}
VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
/*
* We have a locked folio in the page cache, now we need to check
* that it's up-to-date. If not, it is going to be due to an error,
* or because readahead was otherwise unable to retrieve it.
*/
if (unlikely(!folio_test_uptodate(folio))) {
/*
* If the invalidate lock is not held, the folio was in cache
* and uptodate and now it is not. Strange but possible since we
* didn't hold the page lock all the time. Let's drop
* everything, get the invalidate lock and try again.
*/
if (!mapping_locked) {
folio_unlock(folio);
folio_put(folio);
goto retry_find;
}
/*
* OK, the folio is really not uptodate. This can be because the
* VMA has the VM_RAND_READ flag set, or because an error
* arose. Let's read it in directly.
*/
goto page_not_uptodate;
}
/*
* We've made it this far and we had to drop our mmap_lock, now is the
* time to return to the upper layer and have it re-find the vma and
* redo the fault.
*/
if (fpin) {
folio_unlock(folio);
goto out_retry;
}
if (mapping_locked)
filemap_invalidate_unlock_shared(mapping);
/*
* Found the page and have a reference on it.
* We must recheck i_size under page lock.
*/
max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
if (unlikely(index >= max_idx)) {
folio_unlock(folio);
folio_put(folio);
return VM_FAULT_SIGBUS;
}
vmf->page = folio_file_page(folio, index);
return ret | VM_FAULT_LOCKED;
page_not_uptodate:
/*
* Umm, take care of errors if the page isn't up-to-date.
* Try to re-read it _once_. We do this synchronously,
* because there really aren't any performance issues here
* and we need to check for errors.
*/
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);
if (fpin)
goto out_retry;
folio_put(folio);
if (!error || error == AOP_TRUNCATED_PAGE)
goto retry_find;
filemap_invalidate_unlock_shared(mapping);
return VM_FAULT_SIGBUS;
out_retry:
/*
* We dropped the mmap_lock, we need to return to the fault handler to
* re-find the vma and come back and find our hopefully still populated
* page.
*/
if (!IS_ERR(folio))
folio_put(folio);
if (mapping_locked)
filemap_invalidate_unlock_shared(mapping);
if (fpin)
fput(fpin);
return ret | VM_FAULT_RETRY;
}
EXPORT_SYMBOL(filemap_fault);
static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio,
pgoff_t start)
{
struct mm_struct *mm = vmf->vma->vm_mm;
/* Huge page is mapped? No need to proceed. */
if (pmd_trans_huge(*vmf->pmd)) {
folio_unlock(folio);
folio_put(folio);
return true;
}
if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) {
struct page *page = folio_file_page(folio, start);
vm_fault_t ret = do_set_pmd(vmf, page);
if (!ret) {
/* The page is mapped successfully, reference consumed. */
folio_unlock(folio);
return true;
}
}
if (pmd_none(*vmf->pmd) && vmf->prealloc_pte)
pmd_install(mm, vmf->pmd, &vmf->prealloc_pte);
return false;
}
static struct folio *next_uptodate_folio(struct xa_state *xas,
struct address_space *mapping, pgoff_t end_pgoff)
{
struct folio *folio = xas_next_entry(xas, end_pgoff);
unsigned long max_idx;
do {
if (!folio) return NULL; if (xas_retry(xas, folio))
continue;
if (xa_is_value(folio))
continue;
if (folio_test_locked(folio))
continue;
if (!folio_try_get_rcu(folio))
continue;
/* Has the page moved or been split? */
if (unlikely(folio != xas_reload(xas)))
goto skip;
if (!folio_test_uptodate(folio) || folio_test_readahead(folio))
goto skip;
if (!folio_trylock(folio))
goto skip;
if (folio->mapping != mapping)
goto unlock;
if (!folio_test_uptodate(folio))
goto unlock;
max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
if (xas->xa_index >= max_idx)
goto unlock;
return folio;
unlock:
folio_unlock(folio);
skip:
folio_put(folio); } while ((folio = xas_next_entry(xas, end_pgoff)) != NULL);
return NULL;
}
/*
* Map page range [start_page, start_page + nr_pages) of folio.
* start_page is gotten from start by folio_page(folio, start)
*/
static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
struct folio *folio, unsigned long start,
unsigned long addr, unsigned int nr_pages,
unsigned int *mmap_miss)
{
vm_fault_t ret = 0;
struct page *page = folio_page(folio, start);
unsigned int count = 0;
pte_t *old_ptep = vmf->pte;
do {
if (PageHWPoison(page + count))
goto skip;
(*mmap_miss)++;
/*
* NOTE: If there're PTE markers, we'll leave them to be
* handled in the specific fault path, and it'll prohibit the
* fault-around logic.
*/
if (!pte_none(ptep_get(&vmf->pte[count])))
goto skip;
count++;
continue;
skip:
if (count) { set_pte_range(vmf, folio, page, count, addr);
folio_ref_add(folio, count);
if (in_range(vmf->address, addr, count * PAGE_SIZE))
ret = VM_FAULT_NOPAGE;
}
count++;
page += count; vmf->pte += count;
addr += count * PAGE_SIZE;
count = 0;
} while (--nr_pages > 0); if (count) { set_pte_range(vmf, folio, page, count, addr);
folio_ref_add(folio, count);
if (in_range(vmf->address, addr, count * PAGE_SIZE))
ret = VM_FAULT_NOPAGE;
}
vmf->pte = old_ptep;
return ret;
}
static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf,
struct folio *folio, unsigned long addr,
unsigned int *mmap_miss)
{
vm_fault_t ret = 0;
struct page *page = &folio->page;
if (PageHWPoison(page))
return ret;
(*mmap_miss)++;
/*
* NOTE: If there're PTE markers, we'll leave them to be
* handled in the specific fault path, and it'll prohibit
* the fault-around logic.
*/
if (!pte_none(ptep_get(vmf->pte)))
return ret;
if (vmf->address == addr)
ret = VM_FAULT_NOPAGE;
set_pte_range(vmf, folio, page, 1, addr);
folio_ref_inc(folio);
return ret;
}
vm_fault_t filemap_map_pages(struct vm_fault *vmf,
pgoff_t start_pgoff, pgoff_t end_pgoff)
{
struct vm_area_struct *vma = vmf->vma;
struct file *file = vma->vm_file;
struct address_space *mapping = file->f_mapping;
pgoff_t last_pgoff = start_pgoff;
unsigned long addr;
XA_STATE(xas, &mapping->i_pages, start_pgoff);
struct folio *folio;
vm_fault_t ret = 0;
unsigned int nr_pages = 0, mmap_miss = 0, mmap_miss_saved;
rcu_read_lock(); folio = next_uptodate_folio(&xas, mapping, end_pgoff);
if (!folio)
goto out;
if (filemap_map_pmd(vmf, folio, start_pgoff)) {
ret = VM_FAULT_NOPAGE;
goto out;
}
addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
if (!vmf->pte) {
folio_unlock(folio);
folio_put(folio);
goto out;
}
do {
unsigned long end;
addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; vmf->pte += xas.xa_index - last_pgoff;
last_pgoff = xas.xa_index;
end = folio_next_index(folio) - 1; nr_pages = min(end, end_pgoff) - xas.xa_index + 1;
if (!folio_test_large(folio))
ret |= filemap_map_order0_folio(vmf,
folio, addr, &mmap_miss);
else
ret |= filemap_map_folio_range(vmf, folio,
xas.xa_index - folio->index, addr,
nr_pages, &mmap_miss);
folio_unlock(folio); folio_put(folio); } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL); pte_unmap_unlock(vmf->pte, vmf->ptl);
out:
rcu_read_unlock();
mmap_miss_saved = READ_ONCE(file->f_ra.mmap_miss);
if (mmap_miss >= mmap_miss_saved)
WRITE_ONCE(file->f_ra.mmap_miss, 0);
else
WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss_saved - mmap_miss); return ret;
}
EXPORT_SYMBOL(filemap_map_pages);
vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
{
struct address_space *mapping = vmf->vma->vm_file->f_mapping;
struct folio *folio = page_folio(vmf->page);
vm_fault_t ret = VM_FAULT_LOCKED;
sb_start_pagefault(mapping->host->i_sb);
file_update_time(vmf->vma->vm_file);
folio_lock(folio);
if (folio->mapping != mapping) {
folio_unlock(folio);
ret = VM_FAULT_NOPAGE;
goto out;
}
/*
* We mark the folio dirty already here so that when freeze is in
* progress, we are guaranteed that writeback during freezing will
* see the dirty folio and writeprotect it again.
*/
folio_mark_dirty(folio);
folio_wait_stable(folio);
out:
sb_end_pagefault(mapping->host->i_sb);
return ret;
}
const struct vm_operations_struct generic_file_vm_ops = {
.fault = filemap_fault,
.map_pages = filemap_map_pages,
.page_mkwrite = filemap_page_mkwrite,
};
/* This is used for a general mmap of a disk file */
int generic_file_mmap(struct file *file, struct vm_area_struct *vma)
{
struct address_space *mapping = file->f_mapping;
if (!mapping->a_ops->read_folio)
return -ENOEXEC;
file_accessed(file);
vma->vm_ops = &generic_file_vm_ops;
return 0;
}
/*
* This is for filesystems which do not implement ->writepage.
*/
int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
{
if (vma_is_shared_maywrite(vma))
return -EINVAL;
return generic_file_mmap(file, vma);
}
#else
vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
{
return VM_FAULT_SIGBUS;
}
int generic_file_mmap(struct file *file, struct vm_area_struct *vma)
{
return -ENOSYS;
}
int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
{
return -ENOSYS;
}
#endif /* CONFIG_MMU */
EXPORT_SYMBOL(filemap_page_mkwrite);
EXPORT_SYMBOL(generic_file_mmap);
EXPORT_SYMBOL(generic_file_readonly_mmap);
static struct folio *do_read_cache_folio(struct address_space *mapping,
pgoff_t index, filler_t filler, struct file *file, gfp_t gfp)
{
struct folio *folio;
int err;
if (!filler)
filler = mapping->a_ops->read_folio;
repeat:
folio = filemap_get_folio(mapping, index);
if (IS_ERR(folio)) {
folio = filemap_alloc_folio(gfp, 0);
if (!folio)
return ERR_PTR(-ENOMEM);
err = filemap_add_folio(mapping, folio, index, gfp);
if (unlikely(err)) {
folio_put(folio);
if (err == -EEXIST)
goto repeat;
/* Presumably ENOMEM for xarray node */
return ERR_PTR(err);
}
goto filler;
}
if (folio_test_uptodate(folio))
goto out;
if (!folio_trylock(folio)) {
folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE);
goto repeat;
}
/* Folio was truncated from mapping */
if (!folio->mapping) {
folio_unlock(folio);
folio_put(folio);
goto repeat;
}
/* Someone else locked and filled the page in a very small window */
if (folio_test_uptodate(folio)) {
folio_unlock(folio);
goto out;
}
filler:
err = filemap_read_folio(file, filler, folio);
if (err) {
folio_put(folio);
if (err == AOP_TRUNCATED_PAGE)
goto repeat;
return ERR_PTR(err);
}
out:
folio_mark_accessed(folio);
return folio;
}
/**
* read_cache_folio - Read into page cache, fill it if needed.
* @mapping: The address_space to read from.
* @index: The index to read.
* @filler: Function to perform the read, or NULL to use aops->read_folio().
* @file: Passed to filler function, may be NULL if not required.
*
* Read one page into the page cache. If it succeeds, the folio returned
* will contain @index, but it may not be the first page of the folio.
*
* If the filler function returns an error, it will be returned to the
* caller.
*
* Context: May sleep. Expects mapping->invalidate_lock to be held.
* Return: An uptodate folio on success, ERR_PTR() on failure.
*/
struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index,
filler_t filler, struct file *file)
{
return do_read_cache_folio(mapping, index, filler, file,
mapping_gfp_mask(mapping));
}
EXPORT_SYMBOL(read_cache_folio);
/**
* mapping_read_folio_gfp - Read into page cache, using specified allocation flags.
* @mapping: The address_space for the folio.
* @index: The index that the allocated folio will contain.
* @gfp: The page allocator flags to use if allocating.
*
* This is the same as "read_cache_folio(mapping, index, NULL, NULL)", but with
* any new memory allocations done using the specified allocation flags.
*
* The most likely error from this function is EIO, but ENOMEM is
* possible and so is EINTR. If ->read_folio returns another error,
* that will be returned to the caller.
*
* The function expects mapping->invalidate_lock to be already held.
*
* Return: Uptodate folio on success, ERR_PTR() on failure.
*/
struct folio *mapping_read_folio_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp)
{
return do_read_cache_folio(mapping, index, NULL, NULL, gfp);
}
EXPORT_SYMBOL(mapping_read_folio_gfp);
static struct page *do_read_cache_page(struct address_space *mapping,
pgoff_t index, filler_t *filler, struct file *file, gfp_t gfp)
{
struct folio *folio;
folio = do_read_cache_folio(mapping, index, filler, file, gfp);
if (IS_ERR(folio))
return &folio->page;
return folio_file_page(folio, index);
}
struct page *read_cache_page(struct address_space *mapping,
pgoff_t index, filler_t *filler, struct file *file)
{
return do_read_cache_page(mapping, index, filler, file,
mapping_gfp_mask(mapping));
}
EXPORT_SYMBOL(read_cache_page);
/**
* read_cache_page_gfp - read into page cache, using specified page allocation flags.
* @mapping: the page's address_space
* @index: the page index
* @gfp: the page allocator flags to use if allocating
*
* This is the same as "read_mapping_page(mapping, index, NULL)", but with
* any new page allocations done using the specified allocation flags.
*
* If the page does not get brought uptodate, return -EIO.
*
* The function expects mapping->invalidate_lock to be already held.
*
* Return: up to date page on success, ERR_PTR() on failure.
*/
struct page *read_cache_page_gfp(struct address_space *mapping,
pgoff_t index,
gfp_t gfp)
{
return do_read_cache_page(mapping, index, NULL, NULL, gfp);
}
EXPORT_SYMBOL(read_cache_page_gfp);
/*
* Warn about a page cache invalidation failure during a direct I/O write.
*/
static void dio_warn_stale_pagecache(struct file *filp)
{
static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
char pathname[128];
char *path;
errseq_set(&filp->f_mapping->wb_err, -EIO);
if (__ratelimit(&_rs)) {
path = file_path(filp, pathname, sizeof(pathname));
if (IS_ERR(path))
path = "(unknown)";
pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
current->comm);
}
}
void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count)
{
struct address_space *mapping = iocb->ki_filp->f_mapping;
if (mapping->nrpages &&
invalidate_inode_pages2_range(mapping,
iocb->ki_pos >> PAGE_SHIFT,
(iocb->ki_pos + count - 1) >> PAGE_SHIFT))
dio_warn_stale_pagecache(iocb->ki_filp);
}
ssize_t
generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
{
struct address_space *mapping = iocb->ki_filp->f_mapping;
size_t write_len = iov_iter_count(from);
ssize_t written;
/*
* If a page can not be invalidated, return 0 to fall back
* to buffered write.
*/
written = kiocb_invalidate_pages(iocb, write_len);
if (written) {
if (written == -EBUSY)
return 0;
return written;
}
written = mapping->a_ops->direct_IO(iocb, from);
/*
* Finally, try again to invalidate clean pages which might have been
* cached by non-direct readahead, or faulted in by get_user_pages()
* if the source of the write was an mmap'ed region of the file
* we're writing. Either one is a pretty crazy thing to do,
* so we don't support it 100%. If this invalidation
* fails, tough, the write still worked...
*
* Most of the time we do not need this since dio_complete() will do
* the invalidation for us. However there are some file systems that
* do not end up with dio_complete() being called, so let's not break
* them by removing it completely.
*
* Noticeable example is a blkdev_direct_IO().
*
* Skip invalidation for async writes or if mapping has no pages.
*/
if (written > 0) {
struct inode *inode = mapping->host;
loff_t pos = iocb->ki_pos;
kiocb_invalidate_post_direct_write(iocb, written);
pos += written;
write_len -= written;
if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
i_size_write(inode, pos);
mark_inode_dirty(inode);
}
iocb->ki_pos = pos;
}
if (written != -EIOCBQUEUED)
iov_iter_revert(from, write_len - iov_iter_count(from));
return written;
}
EXPORT_SYMBOL(generic_file_direct_write);
ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i)
{
struct file *file = iocb->ki_filp;
loff_t pos = iocb->ki_pos;
struct address_space *mapping = file->f_mapping;
const struct address_space_operations *a_ops = mapping->a_ops;
long status = 0;
ssize_t written = 0;
do {
struct page *page;
unsigned long offset; /* Offset into pagecache page */
unsigned long bytes; /* Bytes to write to page */
size_t copied; /* Bytes copied from user */
void *fsdata = NULL;
offset = (pos & (PAGE_SIZE - 1));
bytes = min_t(unsigned long, PAGE_SIZE - offset,
iov_iter_count(i));
again:
/*
* Bring in the user page that we will copy from _first_.
* Otherwise there's a nasty deadlock on copying from the
* same page as we're writing to, without it being marked
* up-to-date.
*/
if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) {
status = -EFAULT;
break;
}
if (fatal_signal_pending(current)) {
status = -EINTR;
break;
}
status = a_ops->write_begin(file, mapping, pos, bytes,
&page, &fsdata);
if (unlikely(status < 0))
break;
if (mapping_writably_mapped(mapping))
flush_dcache_page(page);
copied = copy_page_from_iter_atomic(page, offset, bytes, i);
flush_dcache_page(page);
status = a_ops->write_end(file, mapping, pos, bytes, copied,
page, fsdata);
if (unlikely(status != copied)) {
iov_iter_revert(i, copied - max(status, 0L));
if (unlikely(status < 0))
break;
}
cond_resched();
if (unlikely(status == 0)) {
/*
* A short copy made ->write_end() reject the
* thing entirely. Might be memory poisoning
* halfway through, might be a race with munmap,
* might be severe memory pressure.
*/
if (copied)
bytes = copied;
goto again;
}
pos += status;
written += status;
balance_dirty_pages_ratelimited(mapping);
} while (iov_iter_count(i));
if (!written)
return status;
iocb->ki_pos += written; return written;
}
EXPORT_SYMBOL(generic_perform_write);
/**
* __generic_file_write_iter - write data to a file
* @iocb: IO state structure (file, offset, etc.)
* @from: iov_iter with data to write
*
* This function does all the work needed for actually writing data to a
* file. It does all basic checks, removes SUID from the file, updates
* modification times and calls proper subroutines depending on whether we
* do direct IO or a standard buffered write.
*
* It expects i_rwsem to be grabbed unless we work on a block device or similar
* object which does not need locking at all.
*
* This function does *not* take care of syncing data in case of O_SYNC write.
* A caller has to handle it. This is mainly due to the fact that we want to
* avoid syncing under i_rwsem.
*
* Return:
* * number of bytes written, even for truncated writes
* * negative error code if no data has been written at all
*/
ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
struct file *file = iocb->ki_filp;
struct address_space *mapping = file->f_mapping;
struct inode *inode = mapping->host;
ssize_t ret;
ret = file_remove_privs(file);
if (ret)
return ret;
ret = file_update_time(file);
if (ret)
return ret;
if (iocb->ki_flags & IOCB_DIRECT) {
ret = generic_file_direct_write(iocb, from);
/*
* If the write stopped short of completing, fall back to
* buffered writes. Some filesystems do this for writes to
* holes, for example. For DAX files, a buffered write will
* not succeed (even if it did, DAX does not handle dirty
* page-cache pages correctly).
*/
if (ret < 0 || !iov_iter_count(from) || IS_DAX(inode))
return ret;
return direct_write_fallback(iocb, from, ret,
generic_perform_write(iocb, from));
}
return generic_perform_write(iocb, from);
}
EXPORT_SYMBOL(__generic_file_write_iter);
/**
* generic_file_write_iter - write data to a file
* @iocb: IO state structure
* @from: iov_iter with data to write
*
* This is a wrapper around __generic_file_write_iter() to be used by most
* filesystems. It takes care of syncing the file in case of O_SYNC file
* and acquires i_rwsem as needed.
* Return:
* * negative error code if no data has been written at all of
* vfs_fsync_range() failed for a synchronous write
* * number of bytes written, even for truncated writes
*/
ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
struct file *file = iocb->ki_filp;
struct inode *inode = file->f_mapping->host;
ssize_t ret;
inode_lock(inode);
ret = generic_write_checks(iocb, from);
if (ret > 0)
ret = __generic_file_write_iter(iocb, from);
inode_unlock(inode);
if (ret > 0)
ret = generic_write_sync(iocb, ret);
return ret;
}
EXPORT_SYMBOL(generic_file_write_iter);
/**
* filemap_release_folio() - Release fs-specific metadata on a folio.
* @folio: The folio which the kernel is trying to free.
* @gfp: Memory allocation flags (and I/O mode).
*
* The address_space is trying to release any data attached to a folio
* (presumably at folio->private).
*
* This will also be called if the private_2 flag is set on a page,
* indicating that the folio has other metadata associated with it.
*
* The @gfp argument specifies whether I/O may be performed to release
* this page (__GFP_IO), and whether the call may block
* (__GFP_RECLAIM & __GFP_FS).
*
* Return: %true if the release was successful, otherwise %false.
*/
bool filemap_release_folio(struct folio *folio, gfp_t gfp)
{
struct address_space * const mapping = folio->mapping;
BUG_ON(!folio_test_locked(folio));
if (!folio_needs_release(folio))
return true;
if (folio_test_writeback(folio))
return false;
if (mapping && mapping->a_ops->release_folio)
return mapping->a_ops->release_folio(folio, gfp);
return try_to_free_buffers(folio);
}
EXPORT_SYMBOL(filemap_release_folio);
#ifdef CONFIG_CACHESTAT_SYSCALL
/**
* filemap_cachestat() - compute the page cache statistics of a mapping
* @mapping: The mapping to compute the statistics for.
* @first_index: The starting page cache index.
* @last_index: The final page index (inclusive).
* @cs: the cachestat struct to write the result to.
*
* This will query the page cache statistics of a mapping in the
* page range of [first_index, last_index] (inclusive). The statistics
* queried include: number of dirty pages, number of pages marked for
* writeback, and the number of (recently) evicted pages.
*/
static void filemap_cachestat(struct address_space *mapping,
pgoff_t first_index, pgoff_t last_index, struct cachestat *cs)
{
XA_STATE(xas, &mapping->i_pages, first_index);
struct folio *folio;
rcu_read_lock();
xas_for_each(&xas, folio, last_index) {
int order;
unsigned long nr_pages;
pgoff_t folio_first_index, folio_last_index;
/*
* Don't deref the folio. It is not pinned, and might
* get freed (and reused) underneath us.
*
* We *could* pin it, but that would be expensive for
* what should be a fast and lightweight syscall.
*
* Instead, derive all information of interest from
* the rcu-protected xarray.
*/
if (xas_retry(&xas, folio))
continue;
order = xa_get_order(xas.xa, xas.xa_index);
nr_pages = 1 << order;
folio_first_index = round_down(xas.xa_index, 1 << order);
folio_last_index = folio_first_index + nr_pages - 1;
/* Folios might straddle the range boundaries, only count covered pages */
if (folio_first_index < first_index)
nr_pages -= first_index - folio_first_index;
if (folio_last_index > last_index)
nr_pages -= folio_last_index - last_index;
if (xa_is_value(folio)) {
/* page is evicted */
void *shadow = (void *)folio;
bool workingset; /* not used */
cs->nr_evicted += nr_pages;
#ifdef CONFIG_SWAP /* implies CONFIG_MMU */
if (shmem_mapping(mapping)) {
/* shmem file - in swap cache */
swp_entry_t swp = radix_to_swp_entry(folio);
/* swapin error results in poisoned entry */
if (non_swap_entry(swp))
goto resched;
/*
* Getting a swap entry from the shmem
* inode means we beat
* shmem_unuse(). rcu_read_lock()
* ensures swapoff waits for us before
* freeing the swapper space. However,
* we can race with swapping and
* invalidation, so there might not be
* a shadow in the swapcache (yet).
*/
shadow = get_shadow_from_swap_cache(swp);
if (!shadow)
goto resched;
}
#endif
if (workingset_test_recent(shadow, true, &workingset))
cs->nr_recently_evicted += nr_pages;
goto resched;
}
/* page is in cache */
cs->nr_cache += nr_pages;
if (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY))
cs->nr_dirty += nr_pages;
if (xas_get_mark(&xas, PAGECACHE_TAG_WRITEBACK))
cs->nr_writeback += nr_pages;
resched:
if (need_resched()) {
xas_pause(&xas);
cond_resched_rcu();
}
}
rcu_read_unlock();
}
/*
* The cachestat(2) system call.
*
* cachestat() returns the page cache statistics of a file in the
* bytes range specified by `off` and `len`: number of cached pages,
* number of dirty pages, number of pages marked for writeback,
* number of evicted pages, and number of recently evicted pages.
*
* An evicted page is a page that is previously in the page cache
* but has been evicted since. A page is recently evicted if its last
* eviction was recent enough that its reentry to the cache would
* indicate that it is actively being used by the system, and that
* there is memory pressure on the system.
*
* `off` and `len` must be non-negative integers. If `len` > 0,
* the queried range is [`off`, `off` + `len`]. If `len` == 0,
* we will query in the range from `off` to the end of the file.
*
* The `flags` argument is unused for now, but is included for future
* extensibility. User should pass 0 (i.e no flag specified).
*
* Currently, hugetlbfs is not supported.
*
* Because the status of a page can change after cachestat() checks it
* but before it returns to the application, the returned values may
* contain stale information.
*
* return values:
* zero - success
* -EFAULT - cstat or cstat_range points to an illegal address
* -EINVAL - invalid flags
* -EBADF - invalid file descriptor
* -EOPNOTSUPP - file descriptor is of a hugetlbfs file
*/
SYSCALL_DEFINE4(cachestat, unsigned int, fd,
struct cachestat_range __user *, cstat_range,
struct cachestat __user *, cstat, unsigned int, flags)
{
struct fd f = fdget(fd);
struct address_space *mapping;
struct cachestat_range csr;
struct cachestat cs;
pgoff_t first_index, last_index;
if (!f.file)
return -EBADF;
if (copy_from_user(&csr, cstat_range,
sizeof(struct cachestat_range))) {
fdput(f);
return -EFAULT;
}
/* hugetlbfs is not supported */
if (is_file_hugepages(f.file)) {
fdput(f);
return -EOPNOTSUPP;
}
if (flags != 0) {
fdput(f);
return -EINVAL;
}
first_index = csr.off >> PAGE_SHIFT;
last_index =
csr.len == 0 ? ULONG_MAX : (csr.off + csr.len - 1) >> PAGE_SHIFT;
memset(&cs, 0, sizeof(struct cachestat));
mapping = f.file->f_mapping;
filemap_cachestat(mapping, first_index, last_index, &cs);
fdput(f);
if (copy_to_user(cstat, &cs, sizeof(struct cachestat)))
return -EFAULT;
return 0;
}
#endif /* CONFIG_CACHESTAT_SYSCALL */
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM signal
#if !defined(_TRACE_SIGNAL_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SIGNAL_H
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/tracepoint.h>
#define TP_STORE_SIGINFO(__entry, info) \
do { \
if (info == SEND_SIG_NOINFO) { \
__entry->errno = 0; \
__entry->code = SI_USER; \
} else if (info == SEND_SIG_PRIV) { \
__entry->errno = 0; \
__entry->code = SI_KERNEL; \
} else { \
__entry->errno = info->si_errno; \
__entry->code = info->si_code; \
} \
} while (0)
#ifndef TRACE_HEADER_MULTI_READ
enum {
TRACE_SIGNAL_DELIVERED,
TRACE_SIGNAL_IGNORED,
TRACE_SIGNAL_ALREADY_PENDING,
TRACE_SIGNAL_OVERFLOW_FAIL,
TRACE_SIGNAL_LOSE_INFO,
};
#endif
/**
* signal_generate - called when a signal is generated
* @sig: signal number
* @info: pointer to struct siginfo
* @task: pointer to struct task_struct
* @group: shared or private
* @result: TRACE_SIGNAL_*
*
* Current process sends a 'sig' signal to 'task' process with
* 'info' siginfo. If 'info' is SEND_SIG_NOINFO or SEND_SIG_PRIV,
* 'info' is not a pointer and you can't access its field. Instead,
* SEND_SIG_NOINFO means that si_code is SI_USER, and SEND_SIG_PRIV
* means that si_code is SI_KERNEL.
*/
TRACE_EVENT(signal_generate,
TP_PROTO(int sig, struct kernel_siginfo *info, struct task_struct *task,
int group, int result),
TP_ARGS(sig, info, task, group, result),
TP_STRUCT__entry(
__field( int, sig )
__field( int, errno )
__field( int, code )
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field( int, group )
__field( int, result )
),
TP_fast_assign(
__entry->sig = sig;
TP_STORE_SIGINFO(__entry, info);
memcpy(__entry->comm, task->comm, TASK_COMM_LEN);
__entry->pid = task->pid;
__entry->group = group;
__entry->result = result;
),
TP_printk("sig=%d errno=%d code=%d comm=%s pid=%d grp=%d res=%d",
__entry->sig, __entry->errno, __entry->code,
__entry->comm, __entry->pid, __entry->group,
__entry->result)
);
/**
* signal_deliver - called when a signal is delivered
* @sig: signal number
* @info: pointer to struct siginfo
* @ka: pointer to struct k_sigaction
*
* A 'sig' signal is delivered to current process with 'info' siginfo,
* and it will be handled by 'ka'. ka->sa.sa_handler can be SIG_IGN or
* SIG_DFL.
* Note that some signals reported by signal_generate tracepoint can be
* lost, ignored or modified (by debugger) before hitting this tracepoint.
* This means, this can show which signals are actually delivered, but
* matching generated signals and delivered signals may not be correct.
*/
TRACE_EVENT(signal_deliver,
TP_PROTO(int sig, struct kernel_siginfo *info, struct k_sigaction *ka),
TP_ARGS(sig, info, ka),
TP_STRUCT__entry(
__field( int, sig )
__field( int, errno )
__field( int, code )
__field( unsigned long, sa_handler )
__field( unsigned long, sa_flags )
),
TP_fast_assign(
__entry->sig = sig;
TP_STORE_SIGINFO(__entry, info);
__entry->sa_handler = (unsigned long)ka->sa.sa_handler;
__entry->sa_flags = ka->sa.sa_flags;
),
TP_printk("sig=%d errno=%d code=%d sa_handler=%lx sa_flags=%lx",
__entry->sig, __entry->errno, __entry->code,
__entry->sa_handler, __entry->sa_flags)
);
#endif /* _TRACE_SIGNAL_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
// SPDX-License-Identifier: GPL-2.0-only
/*
* Access kernel or user memory without faulting.
*/
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <asm/tlb.h>
bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
size_t size)
{
return true;
}
#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
while (len >= sizeof(type)) { \
__get_kernel_nofault(dst, src, type, err_label); \
dst += sizeof(type); \
src += sizeof(type); \
len -= sizeof(type); \
}
long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
{
unsigned long align = 0;
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
align = (unsigned long)dst | (unsigned long)src;
if (!copy_from_kernel_nofault_allowed(src, size))
return -ERANGE;
pagefault_disable();
if (!(align & 7))
copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
if (!(align & 3))
copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
if (!(align & 1))
copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); pagefault_enable();
return 0;
Efault:
pagefault_enable(); return -EFAULT;
}
EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
while (len >= sizeof(type)) { \
__put_kernel_nofault(dst, src, type, err_label); \
dst += sizeof(type); \
src += sizeof(type); \
len -= sizeof(type); \
}
long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
{
unsigned long align = 0;
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
align = (unsigned long)dst | (unsigned long)src;
pagefault_disable();
if (!(align & 7))
copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
if (!(align & 3))
copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
if (!(align & 1))
copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
pagefault_enable();
return 0;
Efault:
pagefault_enable();
return -EFAULT;
}
long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
{
const void *src = unsafe_addr;
if (unlikely(count <= 0))
return 0;
if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
return -ERANGE;
pagefault_disable();
do {
__get_kernel_nofault(dst, src, u8, Efault);
dst++;
src++;
} while (dst[-1] && src - unsafe_addr < count);
pagefault_enable();
dst[-1] = '\0';
return src - unsafe_addr;
Efault:
pagefault_enable();
dst[0] = '\0';
return -EFAULT;
}
/**
* copy_from_user_nofault(): safely attempt to read from a user-space location
* @dst: pointer to the buffer that shall take the data
* @src: address to read from. This must be a user address.
* @size: size of the data chunk
*
* Safely read from user address @src to the buffer at @dst. If a kernel fault
* happens, handle that and return -EFAULT.
*/
long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
{
long ret = -EFAULT;
if (!__access_ok(src, size))
return ret;
if (!nmi_uaccess_okay())
return ret;
pagefault_disable();
ret = __copy_from_user_inatomic(dst, src, size);
pagefault_enable();
if (ret)
return -EFAULT;
return 0;
}
EXPORT_SYMBOL_GPL(copy_from_user_nofault);
/**
* copy_to_user_nofault(): safely attempt to write to a user-space location
* @dst: address to write to
* @src: pointer to the data that shall be written
* @size: size of the data chunk
*
* Safely write to address @dst from the buffer at @src. If a kernel fault
* happens, handle that and return -EFAULT.
*/
long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
{
long ret = -EFAULT;
if (access_ok(dst, size)) {
pagefault_disable();
ret = __copy_to_user_inatomic(dst, src, size);
pagefault_enable();
}
if (ret)
return -EFAULT;
return 0;
}
EXPORT_SYMBOL_GPL(copy_to_user_nofault);
/**
* strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
* address.
* @dst: Destination address, in kernel space. This buffer must be at
* least @count bytes long.
* @unsafe_addr: Unsafe user address.
* @count: Maximum number of bytes to copy, including the trailing NUL.
*
* Copies a NUL-terminated string from unsafe user address to kernel buffer.
*
* On success, returns the length of the string INCLUDING the trailing NUL.
*
* If access fails, returns -EFAULT (some data may have been copied
* and the trailing NUL added).
*
* If @count is smaller than the length of the string, copies @count-1 bytes,
* sets the last byte of @dst buffer to NUL and returns @count.
*/
long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
long count)
{
long ret;
if (unlikely(count <= 0))
return 0;
pagefault_disable();
ret = strncpy_from_user(dst, unsafe_addr, count);
pagefault_enable();
if (ret >= count) {
ret = count;
dst[ret - 1] = '\0';
} else if (ret > 0) {
ret++;
}
return ret;
}
/**
* strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
* @unsafe_addr: The string to measure.
* @count: Maximum count (including NUL)
*
* Get the size of a NUL-terminated string in user space without pagefault.
*
* Returns the size of the string INCLUDING the terminating NUL.
*
* If the string is too long, returns a number larger than @count. User
* has to check the return value against "> count".
* On exception (or invalid count), returns 0.
*
* Unlike strnlen_user, this can be used from IRQ handler etc. because
* it disables pagefaults.
*/
long strnlen_user_nofault(const void __user *unsafe_addr, long count)
{
int ret;
pagefault_disable();
ret = strnlen_user(unsafe_addr, count);
pagefault_enable();
return ret;
}
void __copy_overflow(int size, unsigned long count)
{
WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
}
EXPORT_SYMBOL(__copy_overflow);
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef _LINUX_MAPLE_TREE_H
#define _LINUX_MAPLE_TREE_H
/*
* Maple Tree - An RCU-safe adaptive tree for storing ranges
* Copyright (c) 2018-2022 Oracle
* Authors: Liam R. Howlett <Liam.Howlett@Oracle.com>
* Matthew Wilcox <willy@infradead.org>
*/
#include <linux/kernel.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
/* #define CONFIG_MAPLE_RCU_DISABLED */
/*
* Allocated nodes are mutable until they have been inserted into the tree,
* at which time they cannot change their type until they have been removed
* from the tree and an RCU grace period has passed.
*
* Removed nodes have their ->parent set to point to themselves. RCU readers
* check ->parent before relying on the value that they loaded from the
* slots array. This lets us reuse the slots array for the RCU head.
*
* Nodes in the tree point to their parent unless bit 0 is set.
*/
#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64)
/* 64bit sizes */
#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */
#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */
#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */
#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1)
#else
/* 32bit sizes */
#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */
#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */
#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */
#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2)
#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */
#define MAPLE_NODE_MASK 255UL
/*
* The node->parent of the root node has bit 0 set and the rest of the pointer
* is a pointer to the tree itself. No more bits are available in this pointer
* (on m68k, the data structure may only be 2-byte aligned).
*
* Internal non-root nodes can only have maple_range_* nodes as parents. The
* parent pointer is 256B aligned like all other tree nodes. When storing a 32
* or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an
* extra bit to store the offset. This extra bit comes from a reuse of the last
* bit in the node type. This is possible by using bit 1 to indicate if bit 2
* is part of the type or the slot.
*
* Once the type is decided, the decision of an allocation range type or a range
* type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE
* flag.
*
* Node types:
* 0x??1 = Root
* 0x?00 = 16 bit nodes
* 0x010 = 32 bit nodes
* 0x110 = 64 bit nodes
*
* Slot size and location in the parent pointer:
* type : slot location
* 0x??1 : Root
* 0x?00 : 16 bit values, type in 0-1, slot in 2-6
* 0x010 : 32 bit values, type in 0-2, slot in 3-6
* 0x110 : 64 bit values, type in 0-2, slot in 3-6
*/
/*
* This metadata is used to optimize the gap updating code and in reverse
* searching for gaps or any other code that needs to find the end of the data.
*/
struct maple_metadata {
unsigned char end;
unsigned char gap;
};
/*
* Leaf nodes do not store pointers to nodes, they store user data. Users may
* store almost any bit pattern. As noted above, the optimisation of storing an
* entry at 0 in the root pointer cannot be done for data which have the bottom
* two bits set to '10'. We also reserve values with the bottom two bits set to
* '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs
* return errnos as a negative errno shifted right by two bits and the bottom
* two bits set to '10', and while choosing to store these values in the array
* is not an error, it may lead to confusion if you're testing for an error with
* mas_is_err().
*
* Non-leaf nodes store the type of the node pointed to (enum maple_type in bits
* 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now.
*
* In regular B-Tree terms, pivots are called keys. The term pivot is used to
* indicate that the tree is specifying ranges, Pivots may appear in the
* subtree with an entry attached to the value whereas keys are unique to a
* specific position of a B-tree. Pivot values are inclusive of the slot with
* the same index.
*/
struct maple_range_64 {
struct maple_pnode *parent;
unsigned long pivot[MAPLE_RANGE64_SLOTS - 1];
union {
void __rcu *slot[MAPLE_RANGE64_SLOTS];
struct {
void __rcu *pad[MAPLE_RANGE64_SLOTS - 1];
struct maple_metadata meta;
};
};
};
/*
* At tree creation time, the user can specify that they're willing to trade off
* storing fewer entries in a tree in return for storing more information in
* each node.
*
* The maple tree supports recording the largest range of NULL entries available
* in this node, also called gaps. This optimises the tree for allocating a
* range.
*/
struct maple_arange_64 {
struct maple_pnode *parent;
unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1];
void __rcu *slot[MAPLE_ARANGE64_SLOTS];
unsigned long gap[MAPLE_ARANGE64_SLOTS];
struct maple_metadata meta;
};
struct maple_alloc {
unsigned long total;
unsigned char node_count;
unsigned int request_count;
struct maple_alloc *slot[MAPLE_ALLOC_SLOTS];
};
struct maple_topiary {
struct maple_pnode *parent;
struct maple_enode *next; /* Overlaps the pivot */
};
enum maple_type {
maple_dense,
maple_leaf_64,
maple_range_64,
maple_arange_64,
};
/**
* DOC: Maple tree flags
*
* * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree
* * MT_FLAGS_USE_RCU - Operate in RCU mode
* * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags
* * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value
* * MT_FLAGS_LOCK_MASK - How the mt_lock is used
* * MT_FLAGS_LOCK_IRQ - Acquired irq-safe
* * MT_FLAGS_LOCK_BH - Acquired bh-safe
* * MT_FLAGS_LOCK_EXTERN - mt_lock is not used
*
* MAPLE_HEIGHT_MAX The largest height that can be stored
*/
#define MT_FLAGS_ALLOC_RANGE 0x01
#define MT_FLAGS_USE_RCU 0x02
#define MT_FLAGS_HEIGHT_OFFSET 0x02
#define MT_FLAGS_HEIGHT_MASK 0x7C
#define MT_FLAGS_LOCK_MASK 0x300
#define MT_FLAGS_LOCK_IRQ 0x100
#define MT_FLAGS_LOCK_BH 0x200
#define MT_FLAGS_LOCK_EXTERN 0x300
#define MT_FLAGS_ALLOC_WRAPPED 0x0800
#define MAPLE_HEIGHT_MAX 31
#define MAPLE_NODE_TYPE_MASK 0x0F
#define MAPLE_NODE_TYPE_SHIFT 0x03
#define MAPLE_RESERVED_RANGE 4096
#ifdef CONFIG_LOCKDEP
typedef struct lockdep_map *lockdep_map_p;
#define mt_lock_is_held(mt) \
(!(mt)->ma_external_lock || lock_is_held((mt)->ma_external_lock))
#define mt_write_lock_is_held(mt) \
(!(mt)->ma_external_lock || \
lock_is_held_type((mt)->ma_external_lock, 0))
#define mt_set_external_lock(mt, lock) \
(mt)->ma_external_lock = &(lock)->dep_map
#define mt_on_stack(mt) (mt).ma_external_lock = NULL
#else
typedef struct { /* nothing */ } lockdep_map_p;
#define mt_lock_is_held(mt) 1
#define mt_write_lock_is_held(mt) 1
#define mt_set_external_lock(mt, lock) do { } while (0)
#define mt_on_stack(mt) do { } while (0)
#endif
/*
* If the tree contains a single entry at index 0, it is usually stored in
* tree->ma_root. To optimise for the page cache, an entry which ends in '00',
* '01' or '11' is stored in the root, but an entry which ends in '10' will be
* stored in a node. Bits 3-6 are used to store enum maple_type.
*
* The flags are used both to store some immutable information about this tree
* (set at tree creation time) and dynamic information set under the spinlock.
*
* Another use of flags are to indicate global states of the tree. This is the
* case with the MAPLE_USE_RCU flag, which indicates the tree is currently in
* RCU mode. This mode was added to allow the tree to reuse nodes instead of
* re-allocating and RCU freeing nodes when there is a single user.
*/
struct maple_tree {
union {
spinlock_t ma_lock;
lockdep_map_p ma_external_lock;
};
unsigned int ma_flags;
void __rcu *ma_root;
};
/**
* MTREE_INIT() - Initialize a maple tree
* @name: The maple tree name
* @__flags: The maple tree flags
*
*/
#define MTREE_INIT(name, __flags) { \
.ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), \
.ma_flags = __flags, \
.ma_root = NULL, \
}
/**
* MTREE_INIT_EXT() - Initialize a maple tree with an external lock.
* @name: The tree name
* @__flags: The maple tree flags
* @__lock: The external lock
*/
#ifdef CONFIG_LOCKDEP
#define MTREE_INIT_EXT(name, __flags, __lock) { \
.ma_external_lock = &(__lock).dep_map, \
.ma_flags = (__flags), \
.ma_root = NULL, \
}
#else
#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags)
#endif
#define DEFINE_MTREE(name) \
struct maple_tree name = MTREE_INIT(name, 0)
#define mtree_lock(mt) spin_lock((&(mt)->ma_lock))
#define mtree_lock_nested(mas, subclass) \
spin_lock_nested((&(mt)->ma_lock), subclass)
#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock))
/*
* The Maple Tree squeezes various bits in at various points which aren't
* necessarily obvious. Usually, this is done by observing that pointers are
* N-byte aligned and thus the bottom log_2(N) bits are available for use. We
* don't use the high bits of pointers to store additional information because
* we don't know what bits are unused on any given architecture.
*
* Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8
* low bits for our own purposes. Nodes are currently of 4 types:
* 1. Single pointer (Range is 0-0)
* 2. Non-leaf Allocation Range nodes
* 3. Non-leaf Range nodes
* 4. Leaf Range nodes All nodes consist of a number of node slots,
* pivots, and a parent pointer.
*/
struct maple_node {
union {
struct {
struct maple_pnode *parent;
void __rcu *slot[MAPLE_NODE_SLOTS];
};
struct {
void *pad;
struct rcu_head rcu;
struct maple_enode *piv_parent;
unsigned char parent_slot;
enum maple_type type;
unsigned char slot_len;
unsigned int ma_flags;
};
struct maple_range_64 mr64;
struct maple_arange_64 ma64;
struct maple_alloc alloc;
};
};
/*
* More complicated stores can cause two nodes to become one or three and
* potentially alter the height of the tree. Either half of the tree may need
* to be rebalanced against the other. The ma_topiary struct is used to track
* which nodes have been 'cut' from the tree so that the change can be done
* safely at a later date. This is done to support RCU.
*/
struct ma_topiary {
struct maple_enode *head;
struct maple_enode *tail;
struct maple_tree *mtree;
};
void *mtree_load(struct maple_tree *mt, unsigned long index);
int mtree_insert(struct maple_tree *mt, unsigned long index,
void *entry, gfp_t gfp);
int mtree_insert_range(struct maple_tree *mt, unsigned long first,
unsigned long last, void *entry, gfp_t gfp);
int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp,
void *entry, unsigned long size, unsigned long min,
unsigned long max, gfp_t gfp);
int mtree_alloc_cyclic(struct maple_tree *mt, unsigned long *startp,
void *entry, unsigned long range_lo, unsigned long range_hi,
unsigned long *next, gfp_t gfp);
int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp,
void *entry, unsigned long size, unsigned long min,
unsigned long max, gfp_t gfp);
int mtree_store_range(struct maple_tree *mt, unsigned long first,
unsigned long last, void *entry, gfp_t gfp);
int mtree_store(struct maple_tree *mt, unsigned long index,
void *entry, gfp_t gfp);
void *mtree_erase(struct maple_tree *mt, unsigned long index);
int mtree_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp);
int __mt_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp);
void mtree_destroy(struct maple_tree *mt);
void __mt_destroy(struct maple_tree *mt);
/**
* mtree_empty() - Determine if a tree has any present entries.
* @mt: Maple Tree.
*
* Context: Any context.
* Return: %true if the tree contains only NULL pointers.
*/
static inline bool mtree_empty(const struct maple_tree *mt)
{
return mt->ma_root == NULL;
}
/* Advanced API */
/*
* Maple State Status
* ma_active means the maple state is pointing to a node and offset and can
* continue operating on the tree.
* ma_start means we have not searched the tree.
* ma_root means we have searched the tree and the entry we found lives in
* the root of the tree (ie it has index 0, length 1 and is the only entry in
* the tree).
* ma_none means we have searched the tree and there is no node in the
* tree for this entry. For example, we searched for index 1 in an empty
* tree. Or we have a tree which points to a full leaf node and we
* searched for an entry which is larger than can be contained in that
* leaf node.
* ma_pause means the data within the maple state may be stale, restart the
* operation
* ma_overflow means the search has reached the upper limit of the search
* ma_underflow means the search has reached the lower limit of the search
* ma_error means there was an error, check the node for the error number.
*/
enum maple_status {
ma_active,
ma_start,
ma_root,
ma_none,
ma_pause,
ma_overflow,
ma_underflow,
ma_error,
};
/*
* The maple state is defined in the struct ma_state and is used to keep track
* of information during operations, and even between operations when using the
* advanced API.
*
* If state->node has bit 0 set then it references a tree location which is not
* a node (eg the root). If bit 1 is set, the rest of the bits are a negative
* errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the
* node type.
*
* state->alloc either has a request number of nodes or an allocated node. If
* stat->alloc has a requested number of nodes, the first bit will be set (0x1)
* and the remaining bits are the value. If state->alloc is a node, then the
* node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for
* storing more allocated nodes, a total number of nodes allocated, and the
* node_count in this node. node_count is the number of allocated nodes in this
* node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further
* nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc
* by removing a node from the state->alloc node until state->alloc->node_count
* is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted
* to state->alloc. Nodes are pushed onto state->alloc by putting the current
* state->alloc into the pushed node's slot[0].
*
* The state also contains the implied min/max of the state->node, the depth of
* this search, and the offset. The implied min/max are either from the parent
* node or are 0-oo for the root node. The depth is incremented or decremented
* every time a node is walked down or up. The offset is the slot/pivot of
* interest in the node - either for reading or writing.
*
* When returning a value the maple state index and last respectively contain
* the start and end of the range for the entry. Ranges are inclusive in the
* Maple Tree.
*
* The status of the state is used to determine how the next action should treat
* the state. For instance, if the status is ma_start then the next action
* should start at the root of the tree and walk down. If the status is
* ma_pause then the node may be stale data and should be discarded. If the
* status is ma_overflow, then the last action hit the upper limit.
*
*/
struct ma_state {
struct maple_tree *tree; /* The tree we're operating in */
unsigned long index; /* The index we're operating on - range start */
unsigned long last; /* The last index we're operating on - range end */
struct maple_enode *node; /* The node containing this entry */
unsigned long min; /* The minimum index of this node - implied pivot min */
unsigned long max; /* The maximum index of this node - implied pivot max */
struct maple_alloc *alloc; /* Allocated nodes for this operation */
enum maple_status status; /* The status of the state (active, start, none, etc) */
unsigned char depth; /* depth of tree descent during write */
unsigned char offset;
unsigned char mas_flags;
unsigned char end; /* The end of the node */
};
struct ma_wr_state {
struct ma_state *mas;
struct maple_node *node; /* Decoded mas->node */
unsigned long r_min; /* range min */
unsigned long r_max; /* range max */
enum maple_type type; /* mas->node type */
unsigned char offset_end; /* The offset where the write ends */
unsigned long *pivots; /* mas->node->pivots pointer */
unsigned long end_piv; /* The pivot at the offset end */
void __rcu **slots; /* mas->node->slots pointer */
void *entry; /* The entry to write */
void *content; /* The existing entry that is being overwritten */
};
#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock))
#define mas_lock_nested(mas, subclass) \
spin_lock_nested(&((mas)->tree->ma_lock), subclass)
#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock))
/*
* Special values for ma_state.node.
* MA_ERROR represents an errno. After dropping the lock and attempting
* to resolve the error, the walk would have to be restarted from the
* top of the tree as the tree may have been modified.
*/
#define MA_ERROR(err) \
((struct maple_enode *)(((unsigned long)err << 2) | 2UL))
#define MA_STATE(name, mt, first, end) \
struct ma_state name = { \
.tree = mt, \
.index = first, \
.last = end, \
.node = NULL, \
.status = ma_start, \
.min = 0, \
.max = ULONG_MAX, \
.alloc = NULL, \
.mas_flags = 0, \
}
#define MA_WR_STATE(name, ma_state, wr_entry) \
struct ma_wr_state name = { \
.mas = ma_state, \
.content = NULL, \
.entry = wr_entry, \
}
#define MA_TOPIARY(name, tree) \
struct ma_topiary name = { \
.head = NULL, \
.tail = NULL, \
.mtree = tree, \
}
void *mas_walk(struct ma_state *mas);
void *mas_store(struct ma_state *mas, void *entry);
void *mas_erase(struct ma_state *mas);
int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp);
void mas_store_prealloc(struct ma_state *mas, void *entry);
void *mas_find(struct ma_state *mas, unsigned long max);
void *mas_find_range(struct ma_state *mas, unsigned long max);
void *mas_find_rev(struct ma_state *mas, unsigned long min);
void *mas_find_range_rev(struct ma_state *mas, unsigned long max);
int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp);
int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp,
void *entry, unsigned long range_lo, unsigned long range_hi,
unsigned long *next, gfp_t gfp);
bool mas_nomem(struct ma_state *mas, gfp_t gfp);
void mas_pause(struct ma_state *mas);
void maple_tree_init(void);
void mas_destroy(struct ma_state *mas);
int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries);
void *mas_prev(struct ma_state *mas, unsigned long min);
void *mas_prev_range(struct ma_state *mas, unsigned long max);
void *mas_next(struct ma_state *mas, unsigned long max);
void *mas_next_range(struct ma_state *mas, unsigned long max);
int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max,
unsigned long size);
/*
* This finds an empty area from the highest address to the lowest.
* AKA "Topdown" version,
*/
int mas_empty_area_rev(struct ma_state *mas, unsigned long min,
unsigned long max, unsigned long size);
static inline void mas_init(struct ma_state *mas, struct maple_tree *tree,
unsigned long addr)
{
memset(mas, 0, sizeof(struct ma_state));
mas->tree = tree;
mas->index = mas->last = addr;
mas->max = ULONG_MAX;
mas->status = ma_start;
mas->node = NULL;
}
static inline bool mas_is_active(struct ma_state *mas)
{
return mas->status == ma_active;
}
static inline bool mas_is_err(struct ma_state *mas)
{
return mas->status == ma_error;
}
/**
* mas_reset() - Reset a Maple Tree operation state.
* @mas: Maple Tree operation state.
*
* Resets the error or walk state of the @mas so future walks of the
* array will start from the root. Use this if you have dropped the
* lock and want to reuse the ma_state.
*
* Context: Any context.
*/
static __always_inline void mas_reset(struct ma_state *mas)
{
mas->status = ma_start;
mas->node = NULL;
}
/**
* mas_for_each() - Iterate over a range of the maple tree.
* @__mas: Maple Tree operation state (maple_state)
* @__entry: Entry retrieved from the tree
* @__max: maximum index to retrieve from the tree
*
* When returned, mas->index and mas->last will hold the entire range for the
* entry.
*
* Note: may return the zero entry.
*/
#define mas_for_each(__mas, __entry, __max) \
while (((__entry) = mas_find((__mas), (__max))) != NULL)
#ifdef CONFIG_DEBUG_MAPLE_TREE
enum mt_dump_format {
mt_dump_dec,
mt_dump_hex,
};
extern atomic_t maple_tree_tests_run;
extern atomic_t maple_tree_tests_passed;
void mt_dump(const struct maple_tree *mt, enum mt_dump_format format);
void mas_dump(const struct ma_state *mas);
void mas_wr_dump(const struct ma_wr_state *wr_mas);
void mt_validate(struct maple_tree *mt);
void mt_cache_shrink(void);
#define MT_BUG_ON(__tree, __x) do { \
atomic_inc(&maple_tree_tests_run); \
if (__x) { \
pr_info("BUG at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
mt_dump(__tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
} while (0)
#define MAS_BUG_ON(__mas, __x) do { \
atomic_inc(&maple_tree_tests_run); \
if (__x) { \
pr_info("BUG at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
mas_dump(__mas); \
mt_dump((__mas)->tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
} while (0)
#define MAS_WR_BUG_ON(__wrmas, __x) do { \
atomic_inc(&maple_tree_tests_run); \
if (__x) { \
pr_info("BUG at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
mas_wr_dump(__wrmas); \
mas_dump((__wrmas)->mas); \
mt_dump((__wrmas)->mas->tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
} while (0)
#define MT_WARN_ON(__tree, __x) ({ \
int ret = !!(__x); \
atomic_inc(&maple_tree_tests_run); \
if (ret) { \
pr_info("WARN at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
mt_dump(__tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
unlikely(ret); \
})
#define MAS_WARN_ON(__mas, __x) ({ \
int ret = !!(__x); \
atomic_inc(&maple_tree_tests_run); \
if (ret) { \
pr_info("WARN at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
mas_dump(__mas); \
mt_dump((__mas)->tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
unlikely(ret); \
})
#define MAS_WR_WARN_ON(__wrmas, __x) ({ \
int ret = !!(__x); \
atomic_inc(&maple_tree_tests_run); \
if (ret) { \
pr_info("WARN at %s:%d (%u)\n", \
__func__, __LINE__, __x); \
mas_wr_dump(__wrmas); \
mas_dump((__wrmas)->mas); \
mt_dump((__wrmas)->mas->tree, mt_dump_hex); \
pr_info("Pass: %u Run:%u\n", \
atomic_read(&maple_tree_tests_passed), \
atomic_read(&maple_tree_tests_run)); \
dump_stack(); \
} else { \
atomic_inc(&maple_tree_tests_passed); \
} \
unlikely(ret); \
})
#else
#define MT_BUG_ON(__tree, __x) BUG_ON(__x)
#define MAS_BUG_ON(__mas, __x) BUG_ON(__x)
#define MAS_WR_BUG_ON(__mas, __x) BUG_ON(__x)
#define MT_WARN_ON(__tree, __x) WARN_ON(__x)
#define MAS_WARN_ON(__mas, __x) WARN_ON(__x)
#define MAS_WR_WARN_ON(__mas, __x) WARN_ON(__x)
#endif /* CONFIG_DEBUG_MAPLE_TREE */
/**
* __mas_set_range() - Set up Maple Tree operation state to a sub-range of the
* current location.
* @mas: Maple Tree operation state.
* @start: New start of range in the Maple Tree.
* @last: New end of range in the Maple Tree.
*
* set the internal maple state values to a sub-range.
* Please use mas_set_range() if you do not know where you are in the tree.
*/
static inline void __mas_set_range(struct ma_state *mas, unsigned long start,
unsigned long last)
{
/* Ensure the range starts within the current slot */
MAS_WARN_ON(mas, mas_is_active(mas) &&
(mas->index > start || mas->last < start));
mas->index = start;
mas->last = last;
}
/**
* mas_set_range() - Set up Maple Tree operation state for a different index.
* @mas: Maple Tree operation state.
* @start: New start of range in the Maple Tree.
* @last: New end of range in the Maple Tree.
*
* Move the operation state to refer to a different range. This will
* have the effect of starting a walk from the top; see mas_next()
* to move to an adjacent index.
*/
static inline
void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last)
{
mas_reset(mas);
__mas_set_range(mas, start, last);
}
/**
* mas_set() - Set up Maple Tree operation state for a different index.
* @mas: Maple Tree operation state.
* @index: New index into the Maple Tree.
*
* Move the operation state to refer to a different index. This will
* have the effect of starting a walk from the top; see mas_next()
* to move to an adjacent index.
*/
static inline void mas_set(struct ma_state *mas, unsigned long index)
{
mas_set_range(mas, index, index);
}
static inline bool mt_external_lock(const struct maple_tree *mt)
{
return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN;
}
/**
* mt_init_flags() - Initialise an empty maple tree with flags.
* @mt: Maple Tree
* @flags: maple tree flags.
*
* If you need to initialise a Maple Tree with special flags (eg, an
* allocation tree), use this function.
*
* Context: Any context.
*/
static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags)
{
mt->ma_flags = flags;
if (!mt_external_lock(mt))
spin_lock_init(&mt->ma_lock);
rcu_assign_pointer(mt->ma_root, NULL);
}
/**
* mt_init() - Initialise an empty maple tree.
* @mt: Maple Tree
*
* An empty Maple Tree.
*
* Context: Any context.
*/
static inline void mt_init(struct maple_tree *mt)
{
mt_init_flags(mt, 0);
}
static inline bool mt_in_rcu(struct maple_tree *mt)
{
#ifdef CONFIG_MAPLE_RCU_DISABLED
return false;
#endif
return mt->ma_flags & MT_FLAGS_USE_RCU;
}
/**
* mt_clear_in_rcu() - Switch the tree to non-RCU mode.
* @mt: The Maple Tree
*/
static inline void mt_clear_in_rcu(struct maple_tree *mt)
{
if (!mt_in_rcu(mt))
return;
if (mt_external_lock(mt)) {
WARN_ON(!mt_lock_is_held(mt));
mt->ma_flags &= ~MT_FLAGS_USE_RCU;
} else {
mtree_lock(mt);
mt->ma_flags &= ~MT_FLAGS_USE_RCU;
mtree_unlock(mt);
}
}
/**
* mt_set_in_rcu() - Switch the tree to RCU safe mode.
* @mt: The Maple Tree
*/
static inline void mt_set_in_rcu(struct maple_tree *mt)
{
if (mt_in_rcu(mt))
return;
if (mt_external_lock(mt)) {
WARN_ON(!mt_lock_is_held(mt));
mt->ma_flags |= MT_FLAGS_USE_RCU;
} else {
mtree_lock(mt);
mt->ma_flags |= MT_FLAGS_USE_RCU;
mtree_unlock(mt);
}
}
static inline unsigned int mt_height(const struct maple_tree *mt)
{
return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET;
}
void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max);
void *mt_find_after(struct maple_tree *mt, unsigned long *index,
unsigned long max);
void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min);
void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max);
/**
* mt_for_each - Iterate over each entry starting at index until max.
* @__tree: The Maple Tree
* @__entry: The current entry
* @__index: The index to start the search from. Subsequently used as iterator.
* @__max: The maximum limit for @index
*
* This iterator skips all entries, which resolve to a NULL pointer,
* e.g. entries which has been reserved with XA_ZERO_ENTRY.
*/
#define mt_for_each(__tree, __entry, __index, __max) \
for (__entry = mt_find(__tree, &(__index), __max); \
__entry; __entry = mt_find_after(__tree, &(__index), __max))
#endif /*_LINUX_MAPLE_TREE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_BIT_SPINLOCK_H
#define __LINUX_BIT_SPINLOCK_H
#include <linux/kernel.h>
#include <linux/preempt.h>
#include <linux/atomic.h>
#include <linux/bug.h>
/*
* bit-based spin_lock()
*
* Don't use this unless you really need to: spin_lock() and spin_unlock()
* are significantly faster.
*/
static inline void bit_spin_lock(int bitnum, unsigned long *addr)
{
/*
* Assuming the lock is uncontended, this never enters
* the body of the outer loop. If it is contended, then
* within the inner loop a non-atomic test is used to
* busywait with less bus contention for a good time to
* attempt to acquire the lock bit.
*/
preempt_disable();
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
while (unlikely(test_and_set_bit_lock(bitnum, addr))) { preempt_enable();
do {
cpu_relax();
} while (test_bit(bitnum, addr));
preempt_disable();
}
#endif
__acquire(bitlock);
}
/*
* Return true if it was acquired
*/
static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
{
preempt_disable();
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
if (unlikely(test_and_set_bit_lock(bitnum, addr))) {
preempt_enable();
return 0;
}
#endif
__acquire(bitlock);
return 1;
}
/*
* bit-based spin_unlock()
*/
static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
{
#ifdef CONFIG_DEBUG_SPINLOCK
BUG_ON(!test_bit(bitnum, addr));
#endif
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
clear_bit_unlock(bitnum, addr);
#endif
preempt_enable();
__release(bitlock);
}
/*
* bit-based spin_unlock()
* non-atomic version, which can be used eg. if the bit lock itself is
* protecting the rest of the flags in the word.
*/
static inline void __bit_spin_unlock(int bitnum, unsigned long *addr)
{
#ifdef CONFIG_DEBUG_SPINLOCK
BUG_ON(!test_bit(bitnum, addr));
#endif
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
__clear_bit_unlock(bitnum, addr);
#endif
preempt_enable();
__release(bitlock);
}
/*
* Return true if the lock is held.
*/
static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
{
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
return test_bit(bitnum, addr);
#elif defined CONFIG_PREEMPT_COUNT
return preempt_count();
#else
return 1;
#endif
}
#endif /* __LINUX_BIT_SPINLOCK_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _SCSI_SCSI_HOST_H
#define _SCSI_SCSI_HOST_H
#include <linux/device.h>
#include <linux/list.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <linux/mutex.h>
#include <linux/seq_file.h>
#include <linux/blk-mq.h>
#include <scsi/scsi.h>
struct block_device;
struct completion;
struct module;
struct scsi_cmnd;
struct scsi_device;
struct scsi_target;
struct Scsi_Host;
struct scsi_transport_template;
#define SG_ALL SG_CHUNK_SIZE
#define MODE_UNKNOWN 0x00
#define MODE_INITIATOR 0x01
#define MODE_TARGET 0x02
/**
* enum scsi_timeout_action - How to handle a command that timed out.
* @SCSI_EH_DONE: The command has already been completed.
* @SCSI_EH_RESET_TIMER: Reset the timer and continue waiting for completion.
* @SCSI_EH_NOT_HANDLED: The command has not yet finished. Abort the command.
*/
enum scsi_timeout_action {
SCSI_EH_DONE,
SCSI_EH_RESET_TIMER,
SCSI_EH_NOT_HANDLED,
};
struct scsi_host_template {
/*
* Put fields referenced in IO submission path together in
* same cacheline
*/
/*
* Additional per-command data allocated for the driver.
*/
unsigned int cmd_size;
/*
* The queuecommand function is used to queue up a scsi
* command block to the LLDD. When the driver finished
* processing the command the done callback is invoked.
*
* If queuecommand returns 0, then the driver has accepted the
* command. It must also push it to the HBA if the scsi_cmnd
* flag SCMD_LAST is set, or if the driver does not implement
* commit_rqs. The done() function must be called on the command
* when the driver has finished with it. (you may call done on the
* command before queuecommand returns, but in this case you
* *must* return 0 from queuecommand).
*
* Queuecommand may also reject the command, in which case it may
* not touch the command and must not call done() for it.
*
* There are two possible rejection returns:
*
* SCSI_MLQUEUE_DEVICE_BUSY: Block this device temporarily, but
* allow commands to other devices serviced by this host.
*
* SCSI_MLQUEUE_HOST_BUSY: Block all devices served by this
* host temporarily.
*
* For compatibility, any other non-zero return is treated the
* same as SCSI_MLQUEUE_HOST_BUSY.
*
* NOTE: "temporarily" means either until the next command for#
* this device/host completes, or a period of time determined by
* I/O pressure in the system if there are no other outstanding
* commands.
*
* STATUS: REQUIRED
*/
int (* queuecommand)(struct Scsi_Host *, struct scsi_cmnd *);
/*
* The commit_rqs function is used to trigger a hardware
* doorbell after some requests have been queued with
* queuecommand, when an error is encountered before sending
* the request with SCMD_LAST set.
*
* STATUS: OPTIONAL
*/
void (*commit_rqs)(struct Scsi_Host *, u16);
struct module *module;
const char *name;
/*
* The info function will return whatever useful information the
* developer sees fit. If not provided, then the name field will
* be used instead.
*
* Status: OPTIONAL
*/
const char *(*info)(struct Scsi_Host *);
/*
* Ioctl interface
*
* Status: OPTIONAL
*/
int (*ioctl)(struct scsi_device *dev, unsigned int cmd,
void __user *arg);
#ifdef CONFIG_COMPAT
/*
* Compat handler. Handle 32bit ABI.
* When unknown ioctl is passed return -ENOIOCTLCMD.
*
* Status: OPTIONAL
*/
int (*compat_ioctl)(struct scsi_device *dev, unsigned int cmd,
void __user *arg);
#endif
int (*init_cmd_priv)(struct Scsi_Host *shost, struct scsi_cmnd *cmd);
int (*exit_cmd_priv)(struct Scsi_Host *shost, struct scsi_cmnd *cmd);
/*
* This is an error handling strategy routine. You don't need to
* define one of these if you don't want to - there is a default
* routine that is present that should work in most cases. For those
* driver authors that have the inclination and ability to write their
* own strategy routine, this is where it is specified. Note - the
* strategy routine is *ALWAYS* run in the context of the kernel eh
* thread. Thus you are guaranteed to *NOT* be in an interrupt
* handler when you execute this, and you are also guaranteed to
* *NOT* have any other commands being queued while you are in the
* strategy routine. When you return from this function, operations
* return to normal.
*
* See scsi_error.c scsi_unjam_host for additional comments about
* what this function should and should not be attempting to do.
*
* Status: REQUIRED (at least one of them)
*/
int (* eh_abort_handler)(struct scsi_cmnd *);
int (* eh_device_reset_handler)(struct scsi_cmnd *);
int (* eh_target_reset_handler)(struct scsi_cmnd *);
int (* eh_bus_reset_handler)(struct scsi_cmnd *);
int (* eh_host_reset_handler)(struct scsi_cmnd *);
/*
* Before the mid layer attempts to scan for a new device where none
* currently exists, it will call this entry in your driver. Should
* your driver need to allocate any structs or perform any other init
* items in order to send commands to a currently unused target/lun
* combo, then this is where you can perform those allocations. This
* is specifically so that drivers won't have to perform any kind of
* "is this a new device" checks in their queuecommand routine,
* thereby making the hot path a bit quicker.
*
* Return values: 0 on success, non-0 on failure
*
* Deallocation: If we didn't find any devices at this ID, you will
* get an immediate call to slave_destroy(). If we find something
* here then you will get a call to slave_configure(), then the
* device will be used for however long it is kept around, then when
* the device is removed from the system (or * possibly at reboot
* time), you will then get a call to slave_destroy(). This is
* assuming you implement slave_configure and slave_destroy.
* However, if you allocate memory and hang it off the device struct,
* then you must implement the slave_destroy() routine at a minimum
* in order to avoid leaking memory
* each time a device is tore down.
*
* Status: OPTIONAL
*/
int (* slave_alloc)(struct scsi_device *);
/*
* Once the device has responded to an INQUIRY and we know the
* device is online, we call into the low level driver with the
* struct scsi_device *. If the low level device driver implements
* this function, it *must* perform the task of setting the queue
* depth on the device. All other tasks are optional and depend
* on what the driver supports and various implementation details.
*
* Things currently recommended to be handled at this time include:
*
* 1. Setting the device queue depth. Proper setting of this is
* described in the comments for scsi_change_queue_depth.
* 2. Determining if the device supports the various synchronous
* negotiation protocols. The device struct will already have
* responded to INQUIRY and the results of the standard items
* will have been shoved into the various device flag bits, eg.
* device->sdtr will be true if the device supports SDTR messages.
* 3. Allocating command structs that the device will need.
* 4. Setting the default timeout on this device (if needed).
* 5. Anything else the low level driver might want to do on a device
* specific setup basis...
* 6. Return 0 on success, non-0 on error. The device will be marked
* as offline on error so that no access will occur. If you return
* non-0, your slave_destroy routine will never get called for this
* device, so don't leave any loose memory hanging around, clean
* up after yourself before returning non-0
*
* Status: OPTIONAL
*/
int (* slave_configure)(struct scsi_device *);
/*
* Immediately prior to deallocating the device and after all activity
* has ceased the mid layer calls this point so that the low level
* driver may completely detach itself from the scsi device and vice
* versa. The low level driver is responsible for freeing any memory
* it allocated in the slave_alloc or slave_configure calls.
*
* Status: OPTIONAL
*/
void (* slave_destroy)(struct scsi_device *);
/*
* Before the mid layer attempts to scan for a new device attached
* to a target where no target currently exists, it will call this
* entry in your driver. Should your driver need to allocate any
* structs or perform any other init items in order to send commands
* to a currently unused target, then this is where you can perform
* those allocations.
*
* Return values: 0 on success, non-0 on failure
*
* Status: OPTIONAL
*/
int (* target_alloc)(struct scsi_target *);
/*
* Immediately prior to deallocating the target structure, and
* after all activity to attached scsi devices has ceased, the
* midlayer calls this point so that the driver may deallocate
* and terminate any references to the target.
*
* Note: This callback is called with the host lock held and hence
* must not sleep.
*
* Status: OPTIONAL
*/
void (* target_destroy)(struct scsi_target *);
/*
* If a host has the ability to discover targets on its own instead
* of scanning the entire bus, it can fill in this function and
* call scsi_scan_host(). This function will be called periodically
* until it returns 1 with the scsi_host and the elapsed time of
* the scan in jiffies.
*
* Status: OPTIONAL
*/
int (* scan_finished)(struct Scsi_Host *, unsigned long);
/*
* If the host wants to be called before the scan starts, but
* after the midlayer has set up ready for the scan, it can fill
* in this function.
*
* Status: OPTIONAL
*/
void (* scan_start)(struct Scsi_Host *);
/*
* Fill in this function to allow the queue depth of this host
* to be changeable (on a per device basis). Returns either
* the current queue depth setting (may be different from what
* was passed in) or an error. An error should only be
* returned if the requested depth is legal but the driver was
* unable to set it. If the requested depth is illegal, the
* driver should set and return the closest legal queue depth.
*
* Status: OPTIONAL
*/
int (* change_queue_depth)(struct scsi_device *, int);
/*
* This functions lets the driver expose the queue mapping
* to the block layer.
*
* Status: OPTIONAL
*/
void (* map_queues)(struct Scsi_Host *shost);
/*
* SCSI interface of blk_poll - poll for IO completions.
* Only applicable if SCSI LLD exposes multiple h/w queues.
*
* Return value: Number of completed entries found.
*
* Status: OPTIONAL
*/
int (* mq_poll)(struct Scsi_Host *shost, unsigned int queue_num);
/*
* Check if scatterlists need to be padded for DMA draining.
*
* Status: OPTIONAL
*/
bool (* dma_need_drain)(struct request *rq);
/*
* This function determines the BIOS parameters for a given
* harddisk. These tend to be numbers that are made up by
* the host adapter. Parameters:
* size, device, list (heads, sectors, cylinders)
*
* Status: OPTIONAL
*/
int (* bios_param)(struct scsi_device *, struct block_device *,
sector_t, int []);
/*
* This function is called when one or more partitions on the
* device reach beyond the end of the device.
*
* Status: OPTIONAL
*/
void (*unlock_native_capacity)(struct scsi_device *);
/*
* Can be used to export driver statistics and other infos to the
* world outside the kernel ie. userspace and it also provides an
* interface to feed the driver with information.
*
* Status: OBSOLETE
*/
int (*show_info)(struct seq_file *, struct Scsi_Host *);
int (*write_info)(struct Scsi_Host *, char *, int);
/*
* This is an optional routine that allows the transport to become
* involved when a scsi io timer fires. The return value tells the
* timer routine how to finish the io timeout handling.
*
* Status: OPTIONAL
*/
enum scsi_timeout_action (*eh_timed_out)(struct scsi_cmnd *);
/*
* Optional routine that allows the transport to decide if a cmd
* is retryable. Return true if the transport is in a state the
* cmd should be retried on.
*/
bool (*eh_should_retry_cmd)(struct scsi_cmnd *scmd);
/* This is an optional routine that allows transport to initiate
* LLD adapter or firmware reset using sysfs attribute.
*
* Return values: 0 on success, -ve value on failure.
*
* Status: OPTIONAL
*/
int (*host_reset)(struct Scsi_Host *shost, int reset_type);
#define SCSI_ADAPTER_RESET 1
#define SCSI_FIRMWARE_RESET 2
/*
* Name of proc directory
*/
const char *proc_name;
/*
* This determines if we will use a non-interrupt driven
* or an interrupt driven scheme. It is set to the maximum number
* of simultaneous commands a single hw queue in HBA will accept.
*/
int can_queue;
/*
* In many instances, especially where disconnect / reconnect are
* supported, our host also has an ID on the SCSI bus. If this is
* the case, then it must be reserved. Please set this_id to -1 if
* your setup is in single initiator mode, and the host lacks an
* ID.
*/
int this_id;
/*
* This determines the degree to which the host adapter is capable
* of scatter-gather.
*/
unsigned short sg_tablesize;
unsigned short sg_prot_tablesize;
/*
* Set this if the host adapter has limitations beside segment count.
*/
unsigned int max_sectors;
/*
* Maximum size in bytes of a single segment.
*/
unsigned int max_segment_size;
/*
* DMA scatter gather segment boundary limit. A segment crossing this
* boundary will be split in two.
*/
unsigned long dma_boundary;
unsigned long virt_boundary_mask;
/*
* This specifies "machine infinity" for host templates which don't
* limit the transfer size. Note this limit represents an absolute
* maximum, and may be over the transfer limits allowed for
* individual devices (e.g. 256 for SCSI-1).
*/
#define SCSI_DEFAULT_MAX_SECTORS 1024
/*
* True if this host adapter can make good use of linked commands.
* This will allow more than one command to be queued to a given
* unit on a given host. Set this to the maximum number of command
* blocks to be provided for each device. Set this to 1 for one
* command block per lun, 2 for two, etc. Do not set this to 0.
* You should make sure that the host adapter will do the right thing
* before you try setting this above 1.
*/
short cmd_per_lun;
/* If use block layer to manage tags, this is tag allocation policy */
int tag_alloc_policy;
/*
* Track QUEUE_FULL events and reduce queue depth on demand.
*/
unsigned track_queue_depth:1;
/*
* This specifies the mode that a LLD supports.
*/
unsigned supported_mode:2;
/*
* True for emulated SCSI host adapters (e.g. ATAPI).
*/
unsigned emulated:1;
/*
* True if the low-level driver performs its own reset-settle delays.
*/
unsigned skip_settle_delay:1;
/* True if the controller does not support WRITE SAME */
unsigned no_write_same:1;
/* True if the host uses host-wide tagspace */
unsigned host_tagset:1;
/* The queuecommand callback may block. See also BLK_MQ_F_BLOCKING. */
unsigned queuecommand_may_block:1;
/*
* Countdown for host blocking with no commands outstanding.
*/
unsigned int max_host_blocked;
/*
* Default value for the blocking. If the queue is empty,
* host_blocked counts down in the request_fn until it restarts
* host operations as zero is reached.
*
* FIXME: This should probably be a value in the template
*/
#define SCSI_DEFAULT_HOST_BLOCKED 7
/*
* Pointer to the SCSI host sysfs attribute groups, NULL terminated.
*/
const struct attribute_group **shost_groups;
/*
* Pointer to the SCSI device attribute groups for this host,
* NULL terminated.
*/
const struct attribute_group **sdev_groups;
/*
* Vendor Identifier associated with the host
*
* Note: When specifying vendor_id, be sure to read the
* Vendor Type and ID formatting requirements specified in
* scsi_netlink.h
*/
u64 vendor_id;
};
/*
* Temporary #define for host lock push down. Can be removed when all
* drivers have been updated to take advantage of unlocked
* queuecommand.
*
*/
#define DEF_SCSI_QCMD(func_name) \
int func_name(struct Scsi_Host *shost, struct scsi_cmnd *cmd) \
{ \
unsigned long irq_flags; \
int rc; \
spin_lock_irqsave(shost->host_lock, irq_flags); \
rc = func_name##_lck(cmd); \
spin_unlock_irqrestore(shost->host_lock, irq_flags); \
return rc; \
}
/*
* shost state: If you alter this, you also need to alter scsi_sysfs.c
* (for the ascii descriptions) and the state model enforcer:
* scsi_host_set_state()
*/
enum scsi_host_state {
SHOST_CREATED = 1,
SHOST_RUNNING,
SHOST_CANCEL,
SHOST_DEL,
SHOST_RECOVERY,
SHOST_CANCEL_RECOVERY,
SHOST_DEL_RECOVERY,
};
struct Scsi_Host {
/*
* __devices is protected by the host_lock, but you should
* usually use scsi_device_lookup / shost_for_each_device
* to access it and don't care about locking yourself.
* In the rare case of being in irq context you can use
* their __ prefixed variants with the lock held. NEVER
* access this list directly from a driver.
*/
struct list_head __devices;
struct list_head __targets;
struct list_head starved_list;
spinlock_t default_lock;
spinlock_t *host_lock;
struct mutex scan_mutex;/* serialize scanning activity */
struct list_head eh_abort_list;
struct list_head eh_cmd_q;
struct task_struct * ehandler; /* Error recovery thread. */
struct completion * eh_action; /* Wait for specific actions on the
host. */
wait_queue_head_t host_wait;
const struct scsi_host_template *hostt;
struct scsi_transport_template *transportt;
struct kref tagset_refcnt;
struct completion tagset_freed;
/* Area to keep a shared tag map */
struct blk_mq_tag_set tag_set;
atomic_t host_blocked;
unsigned int host_failed; /* commands that failed.
protected by host_lock */
unsigned int host_eh_scheduled; /* EH scheduled without command */
unsigned int host_no; /* Used for IOCTL_GET_IDLUN, /proc/scsi et al. */
/* next two fields are used to bound the time spent in error handling */
int eh_deadline;
unsigned long last_reset;
/*
* These three parameters can be used to allow for wide scsi,
* and for host adapters that support multiple busses
* The last two should be set to 1 more than the actual max id
* or lun (e.g. 8 for SCSI parallel systems).
*/
unsigned int max_channel;
unsigned int max_id;
u64 max_lun;
/*
* This is a unique identifier that must be assigned so that we
* have some way of identifying each detected host adapter properly
* and uniquely. For hosts that do not support more than one card
* in the system at one time, this does not need to be set. It is
* initialized to 0 in scsi_register.
*/
unsigned int unique_id;
/*
* The maximum length of SCSI commands that this host can accept.
* Probably 12 for most host adapters, but could be 16 for others.
* or 260 if the driver supports variable length cdbs.
* For drivers that don't set this field, a value of 12 is
* assumed.
*/
unsigned short max_cmd_len;
int this_id;
int can_queue;
short cmd_per_lun;
short unsigned int sg_tablesize;
short unsigned int sg_prot_tablesize;
unsigned int max_sectors;
unsigned int opt_sectors;
unsigned int max_segment_size;
unsigned long dma_boundary;
unsigned long virt_boundary_mask;
/*
* In scsi-mq mode, the number of hardware queues supported by the LLD.
*
* Note: it is assumed that each hardware queue has a queue depth of
* can_queue. In other words, the total queue depth per host
* is nr_hw_queues * can_queue. However, for when host_tagset is set,
* the total queue depth is can_queue.
*/
unsigned nr_hw_queues;
unsigned nr_maps;
unsigned active_mode:2;
/*
* Host has requested that no further requests come through for the
* time being.
*/
unsigned host_self_blocked:1;
/*
* Host uses correct SCSI ordering not PC ordering. The bit is
* set for the minority of drivers whose authors actually read
* the spec ;).
*/
unsigned reverse_ordering:1;
/* Task mgmt function in progress */
unsigned tmf_in_progress:1;
/* Asynchronous scan in progress */
unsigned async_scan:1;
/* Don't resume host in EH */
unsigned eh_noresume:1;
/* The controller does not support WRITE SAME */
unsigned no_write_same:1;
/* True if the host uses host-wide tagspace */
unsigned host_tagset:1;
/* The queuecommand callback may block. See also BLK_MQ_F_BLOCKING. */
unsigned queuecommand_may_block:1;
/* Host responded with short (<36 bytes) INQUIRY result */
unsigned short_inquiry:1;
/* The transport requires the LUN bits NOT to be stored in CDB[1] */
unsigned no_scsi2_lun_in_cdb:1;
/*
* Optional work queue to be utilized by the transport
*/
char work_q_name[20];
struct workqueue_struct *work_q;
/*
* Task management function work queue
*/
struct workqueue_struct *tmf_work_q;
/*
* Value host_blocked counts down from
*/
unsigned int max_host_blocked;
/* Protection Information */
unsigned int prot_capabilities;
unsigned char prot_guard_type;
/* legacy crap */
unsigned long base;
unsigned long io_port;
unsigned char n_io_port;
unsigned char dma_channel;
unsigned int irq;
enum scsi_host_state shost_state;
/* ldm bits */
struct device shost_gendev, shost_dev;
/*
* Points to the transport data (if any) which is allocated
* separately
*/
void *shost_data;
/*
* Points to the physical bus device we'd use to do DMA
* Needed just in case we have virtual hosts.
*/
struct device *dma_dev;
/* Delay for runtime autosuspend */
int rpm_autosuspend_delay;
/*
* We should ensure that this is aligned, both for better performance
* and also because some compilers (m68k) don't automatically force
* alignment to a long boundary.
*/
unsigned long hostdata[] /* Used for storage of host specific stuff */
__attribute__ ((aligned (sizeof(unsigned long))));
};
#define class_to_shost(d) \
container_of(d, struct Scsi_Host, shost_dev)
#define shost_printk(prefix, shost, fmt, a...) \
dev_printk(prefix, &(shost)->shost_gendev, fmt, ##a)
static inline void *shost_priv(struct Scsi_Host *shost)
{
return (void *)shost->hostdata;
}
int scsi_is_host_device(const struct device *);
static inline struct Scsi_Host *dev_to_shost(struct device *dev)
{
while (!scsi_is_host_device(dev)) { if (!dev->parent)
return NULL;
dev = dev->parent;
}
return container_of(dev, struct Scsi_Host, shost_gendev);
}
static inline int scsi_host_in_recovery(struct Scsi_Host *shost)
{
return shost->shost_state == SHOST_RECOVERY ||
shost->shost_state == SHOST_CANCEL_RECOVERY ||
shost->shost_state == SHOST_DEL_RECOVERY ||
shost->tmf_in_progress;
}
extern int scsi_queue_work(struct Scsi_Host *, struct work_struct *);
extern void scsi_flush_work(struct Scsi_Host *);
extern struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *, int);
extern int __must_check scsi_add_host_with_dma(struct Scsi_Host *,
struct device *,
struct device *);
#if defined(CONFIG_SCSI_PROC_FS)
struct proc_dir_entry *
scsi_template_proc_dir(const struct scsi_host_template *sht);
#else
#define scsi_template_proc_dir(sht) NULL
#endif
extern void scsi_scan_host(struct Scsi_Host *);
extern int scsi_resume_device(struct scsi_device *sdev);
extern int scsi_rescan_device(struct scsi_device *sdev);
extern void scsi_remove_host(struct Scsi_Host *);
extern struct Scsi_Host *scsi_host_get(struct Scsi_Host *);
extern int scsi_host_busy(struct Scsi_Host *shost);
extern void scsi_host_put(struct Scsi_Host *t);
extern struct Scsi_Host *scsi_host_lookup(unsigned int hostnum);
extern const char *scsi_host_state_name(enum scsi_host_state);
extern void scsi_host_complete_all_commands(struct Scsi_Host *shost,
enum scsi_host_status status);
static inline int __must_check scsi_add_host(struct Scsi_Host *host,
struct device *dev)
{
return scsi_add_host_with_dma(host, dev, dev);
}
static inline struct device *scsi_get_device(struct Scsi_Host *shost)
{
return shost->shost_gendev.parent;
}
/**
* scsi_host_scan_allowed - Is scanning of this host allowed
* @shost: Pointer to Scsi_Host.
**/
static inline int scsi_host_scan_allowed(struct Scsi_Host *shost)
{
return shost->shost_state == SHOST_RUNNING ||
shost->shost_state == SHOST_RECOVERY;
}
extern void scsi_unblock_requests(struct Scsi_Host *);
extern void scsi_block_requests(struct Scsi_Host *);
extern int scsi_host_block(struct Scsi_Host *shost);
extern int scsi_host_unblock(struct Scsi_Host *shost, int new_state);
void scsi_host_busy_iter(struct Scsi_Host *,
bool (*fn)(struct scsi_cmnd *, void *), void *priv);
struct class_container;
/*
* DIF defines the exchange of protection information between
* initiator and SBC block device.
*
* DIX defines the exchange of protection information between OS and
* initiator.
*/
enum scsi_host_prot_capabilities {
SHOST_DIF_TYPE1_PROTECTION = 1 << 0, /* T10 DIF Type 1 */
SHOST_DIF_TYPE2_PROTECTION = 1 << 1, /* T10 DIF Type 2 */
SHOST_DIF_TYPE3_PROTECTION = 1 << 2, /* T10 DIF Type 3 */
SHOST_DIX_TYPE0_PROTECTION = 1 << 3, /* DIX between OS and HBA only */
SHOST_DIX_TYPE1_PROTECTION = 1 << 4, /* DIX with DIF Type 1 */
SHOST_DIX_TYPE2_PROTECTION = 1 << 5, /* DIX with DIF Type 2 */
SHOST_DIX_TYPE3_PROTECTION = 1 << 6, /* DIX with DIF Type 3 */
};
/*
* SCSI hosts which support the Data Integrity Extensions must
* indicate their capabilities by setting the prot_capabilities using
* this call.
*/
static inline void scsi_host_set_prot(struct Scsi_Host *shost, unsigned int mask)
{
shost->prot_capabilities = mask;
}
static inline unsigned int scsi_host_get_prot(struct Scsi_Host *shost)
{
return shost->prot_capabilities;
}
static inline int scsi_host_prot_dma(struct Scsi_Host *shost)
{
return shost->prot_capabilities >= SHOST_DIX_TYPE0_PROTECTION;
}
static inline unsigned int scsi_host_dif_capable(struct Scsi_Host *shost, unsigned int target_type)
{
static unsigned char cap[] = { 0,
SHOST_DIF_TYPE1_PROTECTION,
SHOST_DIF_TYPE2_PROTECTION,
SHOST_DIF_TYPE3_PROTECTION };
if (target_type >= ARRAY_SIZE(cap))
return 0;
return shost->prot_capabilities & cap[target_type] ? target_type : 0;
}
static inline unsigned int scsi_host_dix_capable(struct Scsi_Host *shost, unsigned int target_type)
{
#if defined(CONFIG_BLK_DEV_INTEGRITY)
static unsigned char cap[] = { SHOST_DIX_TYPE0_PROTECTION,
SHOST_DIX_TYPE1_PROTECTION,
SHOST_DIX_TYPE2_PROTECTION,
SHOST_DIX_TYPE3_PROTECTION };
if (target_type >= ARRAY_SIZE(cap))
return 0;
return shost->prot_capabilities & cap[target_type];
#endif
return 0;
}
/*
* All DIX-capable initiators must support the T10-mandated CRC
* checksum. Controllers can optionally implement the IP checksum
* scheme which has much lower impact on system performance. Note
* that the main rationale for the checksum is to match integrity
* metadata with data. Detecting bit errors are a job for ECC memory
* and buses.
*/
enum scsi_host_guard_type {
SHOST_DIX_GUARD_CRC = 1 << 0,
SHOST_DIX_GUARD_IP = 1 << 1,
};
static inline void scsi_host_set_guard(struct Scsi_Host *shost, unsigned char type)
{
shost->prot_guard_type = type;
}
static inline unsigned char scsi_host_get_guard(struct Scsi_Host *shost)
{
return shost->prot_guard_type;
}
extern int scsi_host_set_state(struct Scsi_Host *, enum scsi_host_state);
#endif /* _SCSI_SCSI_HOST_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_STRINGHASH_H
#define __LINUX_STRINGHASH_H
#include <linux/compiler.h> /* For __pure */
#include <linux/types.h> /* For u32, u64 */
#include <linux/hash.h>
/*
* Routines for hashing strings of bytes to a 32-bit hash value.
*
* These hash functions are NOT GUARANTEED STABLE between kernel
* versions, architectures, or even repeated boots of the same kernel.
* (E.g. they may depend on boot-time hardware detection or be
* deliberately randomized.)
*
* They are also not intended to be secure against collisions caused by
* malicious inputs; much slower hash functions are required for that.
*
* They are optimized for pathname components, meaning short strings.
* Even if a majority of files have longer names, the dynamic profile of
* pathname components skews short due to short directory names.
* (E.g. /usr/lib/libsesquipedalianism.so.3.141.)
*/
/*
* Version 1: one byte at a time. Example of use:
*
* unsigned long hash = init_name_hash;
* while (*p)
* hash = partial_name_hash(tolower(*p++), hash);
* hash = end_name_hash(hash);
*
* Although this is designed for bytes, fs/hfsplus/unicode.c
* abuses it to hash 16-bit values.
*/
/* Hash courtesy of the R5 hash in reiserfs modulo sign bits */
#define init_name_hash(salt) (unsigned long)(salt)
/* partial hash update function. Assume roughly 4 bits per character */
static inline unsigned long
partial_name_hash(unsigned long c, unsigned long prevhash)
{
return (prevhash + (c << 4) + (c >> 4)) * 11;
}
/*
* Finally: cut down the number of bits to a int value (and try to avoid
* losing bits). This also has the property (wanted by the dcache)
* that the msbits make a good hash table index.
*/
static inline unsigned int end_name_hash(unsigned long hash)
{
return hash_long(hash, 32);
}
/*
* Version 2: One word (32 or 64 bits) at a time.
* If CONFIG_DCACHE_WORD_ACCESS is defined (meaning <asm/word-at-a-time.h>
* exists, which describes major Linux platforms like x86 and ARM), then
* this computes a different hash function much faster.
*
* If not set, this falls back to a wrapper around the preceding.
*/
extern unsigned int __pure full_name_hash(const void *salt, const char *, unsigned int);
/*
* A hash_len is a u64 with the hash of a string in the low
* half and the length in the high half.
*/
#define hashlen_hash(hashlen) ((u32)(hashlen))
#define hashlen_len(hashlen) ((u32)((hashlen) >> 32))
#define hashlen_create(hash, len) ((u64)(len)<<32 | (u32)(hash))
/* Return the "hash_len" (hash and length) of a null-terminated string */
extern u64 __pure hashlen_string(const void *salt, const char *name);
#endif /* __LINUX_STRINGHASH_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/fs/buffer.c
*
* Copyright (C) 1991, 1992, 2002 Linus Torvalds
*/
/*
* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
*
* Removed a lot of unnecessary code and simplified things now that
* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
*
* Speed up hash, lru, and free list operations. Use gfp() for allocating
* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
*
* Added 32k buffer block sizes - these are required older ARM systems. - RMK
*
* async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
*/
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/iomap.h>
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/capability.h>
#include <linux/blkdev.h>
#include <linux/file.h>
#include <linux/quotaops.h>
#include <linux/highmem.h>
#include <linux/export.h>
#include <linux/backing-dev.h>
#include <linux/writeback.h>
#include <linux/hash.h>
#include <linux/suspend.h>
#include <linux/buffer_head.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/bio.h>
#include <linux/cpu.h>
#include <linux/bitops.h>
#include <linux/mpage.h>
#include <linux/bit_spinlock.h>
#include <linux/pagevec.h>
#include <linux/sched/mm.h>
#include <trace/events/block.h>
#include <linux/fscrypt.h>
#include <linux/fsverity.h>
#include <linux/sched/isolation.h>
#include "internal.h"
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
enum rw_hint hint, struct writeback_control *wbc);
#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
inline void touch_buffer(struct buffer_head *bh)
{
trace_block_touch_buffer(bh);
folio_mark_accessed(bh->b_folio);
}
EXPORT_SYMBOL(touch_buffer);
void __lock_buffer(struct buffer_head *bh)
{
wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_buffer);
void unlock_buffer(struct buffer_head *bh)
{
clear_bit_unlock(BH_Lock, &bh->b_state);
smp_mb__after_atomic();
wake_up_bit(&bh->b_state, BH_Lock);
}
EXPORT_SYMBOL(unlock_buffer);
/*
* Returns if the folio has dirty or writeback buffers. If all the buffers
* are unlocked and clean then the folio_test_dirty information is stale. If
* any of the buffers are locked, it is assumed they are locked for IO.
*/
void buffer_check_dirty_writeback(struct folio *folio,
bool *dirty, bool *writeback)
{
struct buffer_head *head, *bh;
*dirty = false;
*writeback = false;
BUG_ON(!folio_test_locked(folio));
head = folio_buffers(folio);
if (!head)
return;
if (folio_test_writeback(folio))
*writeback = true;
bh = head;
do {
if (buffer_locked(bh))
*writeback = true;
if (buffer_dirty(bh))
*dirty = true;
bh = bh->b_this_page;
} while (bh != head);
}
/*
* Block until a buffer comes unlocked. This doesn't stop it
* from becoming locked again - you have to lock it yourself
* if you want to preserve its state.
*/
void __wait_on_buffer(struct buffer_head * bh)
{
wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__wait_on_buffer);
static void buffer_io_error(struct buffer_head *bh, char *msg)
{
if (!test_bit(BH_Quiet, &bh->b_state))
printk_ratelimited(KERN_ERR
"Buffer I/O error on dev %pg, logical block %llu%s\n",
bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
}
/*
* End-of-IO handler helper function which does not touch the bh after
* unlocking it.
* Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
* a race there is benign: unlock_buffer() only use the bh's address for
* hashing after unlocking the buffer, so it doesn't actually touch the bh
* itself.
*/
static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
{
if (uptodate) {
set_buffer_uptodate(bh);
} else {
/* This happens, due to failed read-ahead attempts. */
clear_buffer_uptodate(bh);
}
unlock_buffer(bh);
}
/*
* Default synchronous end-of-IO handler.. Just mark it up-to-date and
* unlock the buffer.
*/
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
{
__end_buffer_read_notouch(bh, uptodate);
put_bh(bh);
}
EXPORT_SYMBOL(end_buffer_read_sync);
void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
{
if (uptodate) {
set_buffer_uptodate(bh);
} else {
buffer_io_error(bh, ", lost sync page write");
mark_buffer_write_io_error(bh);
clear_buffer_uptodate(bh);
}
unlock_buffer(bh);
put_bh(bh);
}
EXPORT_SYMBOL(end_buffer_write_sync);
/*
* Various filesystems appear to want __find_get_block to be non-blocking.
* But it's the page lock which protects the buffers. To get around this,
* we get exclusion from try_to_free_buffers with the blockdev mapping's
* i_private_lock.
*
* Hack idea: for the blockdev mapping, i_private_lock contention
* may be quite high. This code could TryLock the page, and if that
* succeeds, there is no need to take i_private_lock.
*/
static struct buffer_head *
__find_get_block_slow(struct block_device *bdev, sector_t block)
{
struct inode *bd_inode = bdev->bd_inode;
struct address_space *bd_mapping = bd_inode->i_mapping;
struct buffer_head *ret = NULL;
pgoff_t index;
struct buffer_head *bh;
struct buffer_head *head;
struct folio *folio;
int all_mapped = 1;
static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
index = ((loff_t)block << bd_inode->i_blkbits) / PAGE_SIZE;
folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
if (IS_ERR(folio))
goto out;
spin_lock(&bd_mapping->i_private_lock);
head = folio_buffers(folio);
if (!head)
goto out_unlock;
bh = head;
do {
if (!buffer_mapped(bh))
all_mapped = 0;
else if (bh->b_blocknr == block) {
ret = bh;
get_bh(bh);
goto out_unlock;
}
bh = bh->b_this_page;
} while (bh != head);
/* we might be here because some of the buffers on this page are
* not mapped. This is due to various races between
* file io on the block device and getblk. It gets dealt with
* elsewhere, don't buffer_error if we had some unmapped buffers
*/
ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
if (all_mapped && __ratelimit(&last_warned)) {
printk("__find_get_block_slow() failed. block=%llu, "
"b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
"device %pg blocksize: %d\n",
(unsigned long long)block,
(unsigned long long)bh->b_blocknr,
bh->b_state, bh->b_size, bdev,
1 << bd_inode->i_blkbits);
}
out_unlock:
spin_unlock(&bd_mapping->i_private_lock);
folio_put(folio);
out:
return ret;
}
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
{
unsigned long flags;
struct buffer_head *first;
struct buffer_head *tmp;
struct folio *folio;
int folio_uptodate = 1;
BUG_ON(!buffer_async_read(bh));
folio = bh->b_folio;
if (uptodate) {
set_buffer_uptodate(bh);
} else {
clear_buffer_uptodate(bh);
buffer_io_error(bh, ", async page read");
folio_set_error(folio);
}
/*
* Be _very_ careful from here on. Bad things can happen if
* two buffer heads end IO at almost the same time and both
* decide that the page is now completely done.
*/
first = folio_buffers(folio);
spin_lock_irqsave(&first->b_uptodate_lock, flags);
clear_buffer_async_read(bh);
unlock_buffer(bh);
tmp = bh;
do {
if (!buffer_uptodate(tmp))
folio_uptodate = 0;
if (buffer_async_read(tmp)) {
BUG_ON(!buffer_locked(tmp));
goto still_busy;
}
tmp = tmp->b_this_page;
} while (tmp != bh);
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
folio_end_read(folio, folio_uptodate);
return;
still_busy:
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
return;
}
struct postprocess_bh_ctx {
struct work_struct work;
struct buffer_head *bh;
};
static void verify_bh(struct work_struct *work)
{
struct postprocess_bh_ctx *ctx =
container_of(work, struct postprocess_bh_ctx, work);
struct buffer_head *bh = ctx->bh;
bool valid;
valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
end_buffer_async_read(bh, valid);
kfree(ctx);
}
static bool need_fsverity(struct buffer_head *bh)
{
struct folio *folio = bh->b_folio;
struct inode *inode = folio->mapping->host;
return fsverity_active(inode) &&
/* needed by ext4 */
folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
}
static void decrypt_bh(struct work_struct *work)
{
struct postprocess_bh_ctx *ctx =
container_of(work, struct postprocess_bh_ctx, work);
struct buffer_head *bh = ctx->bh;
int err;
err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
bh_offset(bh));
if (err == 0 && need_fsverity(bh)) {
/*
* We use different work queues for decryption and for verity
* because verity may require reading metadata pages that need
* decryption, and we shouldn't recurse to the same workqueue.
*/
INIT_WORK(&ctx->work, verify_bh);
fsverity_enqueue_verify_work(&ctx->work);
return;
}
end_buffer_async_read(bh, err == 0);
kfree(ctx);
}
/*
* I/O completion handler for block_read_full_folio() - pages
* which come unlocked at the end of I/O.
*/
static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
{
struct inode *inode = bh->b_folio->mapping->host;
bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
bool verify = need_fsverity(bh);
/* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
if (uptodate && (decrypt || verify)) {
struct postprocess_bh_ctx *ctx =
kmalloc(sizeof(*ctx), GFP_ATOMIC);
if (ctx) {
ctx->bh = bh;
if (decrypt) {
INIT_WORK(&ctx->work, decrypt_bh);
fscrypt_enqueue_decrypt_work(&ctx->work);
} else {
INIT_WORK(&ctx->work, verify_bh);
fsverity_enqueue_verify_work(&ctx->work);
}
return;
}
uptodate = 0;
}
end_buffer_async_read(bh, uptodate);
}
/*
* Completion handler for block_write_full_folio() - folios which are unlocked
* during I/O, and which have the writeback flag cleared upon I/O completion.
*/
static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
{
unsigned long flags;
struct buffer_head *first;
struct buffer_head *tmp;
struct folio *folio;
BUG_ON(!buffer_async_write(bh));
folio = bh->b_folio;
if (uptodate) {
set_buffer_uptodate(bh);
} else {
buffer_io_error(bh, ", lost async page write");
mark_buffer_write_io_error(bh);
clear_buffer_uptodate(bh);
folio_set_error(folio);
}
first = folio_buffers(folio);
spin_lock_irqsave(&first->b_uptodate_lock, flags);
clear_buffer_async_write(bh);
unlock_buffer(bh);
tmp = bh->b_this_page;
while (tmp != bh) {
if (buffer_async_write(tmp)) {
BUG_ON(!buffer_locked(tmp));
goto still_busy;
}
tmp = tmp->b_this_page;
}
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
folio_end_writeback(folio);
return;
still_busy:
spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
return;
}
/*
* If a page's buffers are under async readin (end_buffer_async_read
* completion) then there is a possibility that another thread of
* control could lock one of the buffers after it has completed
* but while some of the other buffers have not completed. This
* locked buffer would confuse end_buffer_async_read() into not unlocking
* the page. So the absence of BH_Async_Read tells end_buffer_async_read()
* that this buffer is not under async I/O.
*
* The page comes unlocked when it has no locked buffer_async buffers
* left.
*
* PageLocked prevents anyone starting new async I/O reads any of
* the buffers.
*
* PageWriteback is used to prevent simultaneous writeout of the same
* page.
*
* PageLocked prevents anyone from starting writeback of a page which is
* under read I/O (PageWriteback is only ever set against a locked page).
*/
static void mark_buffer_async_read(struct buffer_head *bh)
{
bh->b_end_io = end_buffer_async_read_io;
set_buffer_async_read(bh);
}
static void mark_buffer_async_write_endio(struct buffer_head *bh,
bh_end_io_t *handler)
{
bh->b_end_io = handler;
set_buffer_async_write(bh);
}
void mark_buffer_async_write(struct buffer_head *bh)
{
mark_buffer_async_write_endio(bh, end_buffer_async_write);
}
EXPORT_SYMBOL(mark_buffer_async_write);
/*
* fs/buffer.c contains helper functions for buffer-backed address space's
* fsync functions. A common requirement for buffer-based filesystems is
* that certain data from the backing blockdev needs to be written out for
* a successful fsync(). For example, ext2 indirect blocks need to be
* written back and waited upon before fsync() returns.
*
* The functions mark_buffer_dirty_inode(), fsync_inode_buffers(),
* inode_has_buffers() and invalidate_inode_buffers() are provided for the
* management of a list of dependent buffers at ->i_mapping->i_private_list.
*
* Locking is a little subtle: try_to_free_buffers() will remove buffers
* from their controlling inode's queue when they are being freed. But
* try_to_free_buffers() will be operating against the *blockdev* mapping
* at the time, not against the S_ISREG file which depends on those buffers.
* So the locking for i_private_list is via the i_private_lock in the address_space
* which backs the buffers. Which is different from the address_space
* against which the buffers are listed. So for a particular address_space,
* mapping->i_private_lock does *not* protect mapping->i_private_list! In fact,
* mapping->i_private_list will always be protected by the backing blockdev's
* ->i_private_lock.
*
* Which introduces a requirement: all buffers on an address_space's
* ->i_private_list must be from the same address_space: the blockdev's.
*
* address_spaces which do not place buffers at ->i_private_list via these
* utility functions are free to use i_private_lock and i_private_list for
* whatever they want. The only requirement is that list_empty(i_private_list)
* be true at clear_inode() time.
*
* FIXME: clear_inode should not call invalidate_inode_buffers(). The
* filesystems should do that. invalidate_inode_buffers() should just go
* BUG_ON(!list_empty).
*
* FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
* take an address_space, not an inode. And it should be called
* mark_buffer_dirty_fsync() to clearly define why those buffers are being
* queued up.
*
* FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
* list if it is already on a list. Because if the buffer is on a list,
* it *must* already be on the right one. If not, the filesystem is being
* silly. This will save a ton of locking. But first we have to ensure
* that buffers are taken *off* the old inode's list when they are freed
* (presumably in truncate). That requires careful auditing of all
* filesystems (do it inside bforget()). It could also be done by bringing
* b_inode back.
*/
/*
* The buffer's backing address_space's i_private_lock must be held
*/
static void __remove_assoc_queue(struct buffer_head *bh)
{
list_del_init(&bh->b_assoc_buffers);
WARN_ON(!bh->b_assoc_map);
bh->b_assoc_map = NULL;
}
int inode_has_buffers(struct inode *inode)
{
return !list_empty(&inode->i_data.i_private_list);
}
/*
* osync is designed to support O_SYNC io. It waits synchronously for
* all already-submitted IO to complete, but does not queue any new
* writes to the disk.
*
* To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
* as you dirty the buffers, and then use osync_inode_buffers to wait for
* completion. Any other dirty buffers which are not yet queued for
* write will not be flushed to disk by the osync.
*/
static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
{
struct buffer_head *bh;
struct list_head *p;
int err = 0;
spin_lock(lock);
repeat:
list_for_each_prev(p, list) {
bh = BH_ENTRY(p);
if (buffer_locked(bh)) {
get_bh(bh);
spin_unlock(lock);
wait_on_buffer(bh);
if (!buffer_uptodate(bh))
err = -EIO;
brelse(bh);
spin_lock(lock);
goto repeat;
}
}
spin_unlock(lock);
return err;
}
/**
* sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
* @mapping: the mapping which wants those buffers written
*
* Starts I/O against the buffers at mapping->i_private_list, and waits upon
* that I/O.
*
* Basically, this is a convenience function for fsync().
* @mapping is a file or directory which needs those buffers to be written for
* a successful fsync().
*/
int sync_mapping_buffers(struct address_space *mapping)
{
struct address_space *buffer_mapping = mapping->i_private_data;
if (buffer_mapping == NULL || list_empty(&mapping->i_private_list))
return 0;
return fsync_buffers_list(&buffer_mapping->i_private_lock,
&mapping->i_private_list);
}
EXPORT_SYMBOL(sync_mapping_buffers);
/**
* generic_buffers_fsync_noflush - generic buffer fsync implementation
* for simple filesystems with no inode lock
*
* @file: file to synchronize
* @start: start offset in bytes
* @end: end offset in bytes (inclusive)
* @datasync: only synchronize essential metadata if true
*
* This is a generic implementation of the fsync method for simple
* filesystems which track all non-inode metadata in the buffers list
* hanging off the address_space structure.
*/
int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
bool datasync)
{
struct inode *inode = file->f_mapping->host;
int err;
int ret;
err = file_write_and_wait_range(file, start, end);
if (err)
return err;
ret = sync_mapping_buffers(inode->i_mapping);
if (!(inode->i_state & I_DIRTY_ALL))
goto out;
if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
goto out;
err = sync_inode_metadata(inode, 1);
if (ret == 0)
ret = err;
out:
/* check and advance again to catch errors after syncing out buffers */
err = file_check_and_advance_wb_err(file);
if (ret == 0)
ret = err;
return ret;
}
EXPORT_SYMBOL(generic_buffers_fsync_noflush);
/**
* generic_buffers_fsync - generic buffer fsync implementation
* for simple filesystems with no inode lock
*
* @file: file to synchronize
* @start: start offset in bytes
* @end: end offset in bytes (inclusive)
* @datasync: only synchronize essential metadata if true
*
* This is a generic implementation of the fsync method for simple
* filesystems which track all non-inode metadata in the buffers list
* hanging off the address_space structure. This also makes sure that
* a device cache flush operation is called at the end.
*/
int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
bool datasync)
{
struct inode *inode = file->f_mapping->host;
int ret;
ret = generic_buffers_fsync_noflush(file, start, end, datasync);
if (!ret)
ret = blkdev_issue_flush(inode->i_sb->s_bdev);
return ret;
}
EXPORT_SYMBOL(generic_buffers_fsync);
/*
* Called when we've recently written block `bblock', and it is known that
* `bblock' was for a buffer_boundary() buffer. This means that the block at
* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
* dirty, schedule it for IO. So that indirects merge nicely with their data.
*/
void write_boundary_block(struct block_device *bdev,
sector_t bblock, unsigned blocksize)
{
struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
if (bh) {
if (buffer_dirty(bh))
write_dirty_buffer(bh, 0);
put_bh(bh);
}
}
void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
{
struct address_space *mapping = inode->i_mapping;
struct address_space *buffer_mapping = bh->b_folio->mapping;
mark_buffer_dirty(bh);
if (!mapping->i_private_data) {
mapping->i_private_data = buffer_mapping;
} else {
BUG_ON(mapping->i_private_data != buffer_mapping);
}
if (!bh->b_assoc_map) {
spin_lock(&buffer_mapping->i_private_lock);
list_move_tail(&bh->b_assoc_buffers,
&mapping->i_private_list);
bh->b_assoc_map = mapping;
spin_unlock(&buffer_mapping->i_private_lock);
}
}
EXPORT_SYMBOL(mark_buffer_dirty_inode);
/*
* Add a page to the dirty page list.
*
* It is a sad fact of life that this function is called from several places
* deeply under spinlocking. It may not sleep.
*
* If the page has buffers, the uptodate buffers are set dirty, to preserve
* dirty-state coherency between the page and the buffers. It the page does
* not have buffers then when they are later attached they will all be set
* dirty.
*
* The buffers are dirtied before the page is dirtied. There's a small race
* window in which a writepage caller may see the page cleanness but not the
* buffer dirtiness. That's fine. If this code were to set the page dirty
* before the buffers, a concurrent writepage caller could clear the page dirty
* bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
* page on the dirty page list.
*
* We use i_private_lock to lock against try_to_free_buffers while using the
* page's buffer list. Also use this to protect against clean buffers being
* added to the page after it was set dirty.
*
* FIXME: may need to call ->reservepage here as well. That's rather up to the
* address_space though.
*/
bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
{
struct buffer_head *head;
bool newly_dirty;
spin_lock(&mapping->i_private_lock);
head = folio_buffers(folio);
if (head) {
struct buffer_head *bh = head;
do {
set_buffer_dirty(bh);
bh = bh->b_this_page;
} while (bh != head);
}
/*
* Lock out page's memcg migration to keep PageDirty
* synchronized with per-memcg dirty page counters.
*/
folio_memcg_lock(folio);
newly_dirty = !folio_test_set_dirty(folio);
spin_unlock(&mapping->i_private_lock);
if (newly_dirty)
__folio_mark_dirty(folio, mapping, 1);
folio_memcg_unlock(folio);
if (newly_dirty)
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
return newly_dirty;
}
EXPORT_SYMBOL(block_dirty_folio);
/*
* Write out and wait upon a list of buffers.
*
* We have conflicting pressures: we want to make sure that all
* initially dirty buffers get waited on, but that any subsequently
* dirtied buffers don't. After all, we don't want fsync to last
* forever if somebody is actively writing to the file.
*
* Do this in two main stages: first we copy dirty buffers to a
* temporary inode list, queueing the writes as we go. Then we clean
* up, waiting for those writes to complete.
*
* During this second stage, any subsequent updates to the file may end
* up refiling the buffer on the original inode's dirty list again, so
* there is a chance we will end up with a buffer queued for write but
* not yet completed on that list. So, as a final cleanup we go through
* the osync code to catch these locked, dirty buffers without requeuing
* any newly dirty buffers for write.
*/
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
{
struct buffer_head *bh;
struct list_head tmp;
struct address_space *mapping;
int err = 0, err2;
struct blk_plug plug;
INIT_LIST_HEAD(&tmp);
blk_start_plug(&plug);
spin_lock(lock);
while (!list_empty(list)) {
bh = BH_ENTRY(list->next);
mapping = bh->b_assoc_map;
__remove_assoc_queue(bh);
/* Avoid race with mark_buffer_dirty_inode() which does
* a lockless check and we rely on seeing the dirty bit */
smp_mb();
if (buffer_dirty(bh) || buffer_locked(bh)) {
list_add(&bh->b_assoc_buffers, &tmp);
bh->b_assoc_map = mapping;
if (buffer_dirty(bh)) {
get_bh(bh);
spin_unlock(lock);
/*
* Ensure any pending I/O completes so that
* write_dirty_buffer() actually writes the
* current contents - it is a noop if I/O is
* still in flight on potentially older
* contents.
*/
write_dirty_buffer(bh, REQ_SYNC);
/*
* Kick off IO for the previous mapping. Note
* that we will not run the very last mapping,
* wait_on_buffer() will do that for us
* through sync_buffer().
*/
brelse(bh);
spin_lock(lock);
}
}
}
spin_unlock(lock);
blk_finish_plug(&plug);
spin_lock(lock);
while (!list_empty(&tmp)) {
bh = BH_ENTRY(tmp.prev);
get_bh(bh);
mapping = bh->b_assoc_map;
__remove_assoc_queue(bh);
/* Avoid race with mark_buffer_dirty_inode() which does
* a lockless check and we rely on seeing the dirty bit */
smp_mb();
if (buffer_dirty(bh)) {
list_add(&bh->b_assoc_buffers,
&mapping->i_private_list);
bh->b_assoc_map = mapping;
}
spin_unlock(lock);
wait_on_buffer(bh);
if (!buffer_uptodate(bh))
err = -EIO;
brelse(bh);
spin_lock(lock);
}
spin_unlock(lock);
err2 = osync_buffers_list(lock, list);
if (err)
return err;
else
return err2;
}
/*
* Invalidate any and all dirty buffers on a given inode. We are
* probably unmounting the fs, but that doesn't mean we have already
* done a sync(). Just drop the buffers from the inode list.
*
* NOTE: we take the inode's blockdev's mapping's i_private_lock. Which
* assumes that all the buffers are against the blockdev. Not true
* for reiserfs.
*/
void invalidate_inode_buffers(struct inode *inode)
{
if (inode_has_buffers(inode)) {
struct address_space *mapping = &inode->i_data;
struct list_head *list = &mapping->i_private_list;
struct address_space *buffer_mapping = mapping->i_private_data;
spin_lock(&buffer_mapping->i_private_lock);
while (!list_empty(list))
__remove_assoc_queue(BH_ENTRY(list->next));
spin_unlock(&buffer_mapping->i_private_lock);
}
}
EXPORT_SYMBOL(invalidate_inode_buffers);
/*
* Remove any clean buffers from the inode's buffer list. This is called
* when we're trying to free the inode itself. Those buffers can pin it.
*
* Returns true if all buffers were removed.
*/
int remove_inode_buffers(struct inode *inode)
{
int ret = 1;
if (inode_has_buffers(inode)) {
struct address_space *mapping = &inode->i_data;
struct list_head *list = &mapping->i_private_list;
struct address_space *buffer_mapping = mapping->i_private_data;
spin_lock(&buffer_mapping->i_private_lock);
while (!list_empty(list)) {
struct buffer_head *bh = BH_ENTRY(list->next);
if (buffer_dirty(bh)) {
ret = 0;
break;
}
__remove_assoc_queue(bh);
}
spin_unlock(&buffer_mapping->i_private_lock);
}
return ret;
}
/*
* Create the appropriate buffers when given a folio for data area and
* the size of each buffer.. Use the bh->b_this_page linked list to
* follow the buffers created. Return NULL if unable to create more
* buffers.
*
* The retry flag is used to differentiate async IO (paging, swapping)
* which may not fail from ordinary buffer allocations.
*/
struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
gfp_t gfp)
{
struct buffer_head *bh, *head;
long offset;
struct mem_cgroup *memcg, *old_memcg;
/* The folio lock pins the memcg */
memcg = folio_memcg(folio);
old_memcg = set_active_memcg(memcg);
head = NULL;
offset = folio_size(folio);
while ((offset -= size) >= 0) {
bh = alloc_buffer_head(gfp);
if (!bh)
goto no_grow;
bh->b_this_page = head;
bh->b_blocknr = -1;
head = bh;
bh->b_size = size;
/* Link the buffer to its folio */
folio_set_bh(bh, folio, offset);
}
out:
set_active_memcg(old_memcg);
return head;
/*
* In case anything failed, we just free everything we got.
*/
no_grow:
if (head) {
do {
bh = head;
head = head->b_this_page;
free_buffer_head(bh);
} while (head);
}
goto out;
}
EXPORT_SYMBOL_GPL(folio_alloc_buffers);
struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
bool retry)
{
gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
if (retry)
gfp |= __GFP_NOFAIL;
return folio_alloc_buffers(page_folio(page), size, gfp);
}
EXPORT_SYMBOL_GPL(alloc_page_buffers);
static inline void link_dev_buffers(struct folio *folio,
struct buffer_head *head)
{
struct buffer_head *bh, *tail;
bh = head;
do {
tail = bh;
bh = bh->b_this_page;
} while (bh);
tail->b_this_page = head;
folio_attach_private(folio, head);
}
static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
{
sector_t retval = ~((sector_t)0);
loff_t sz = bdev_nr_bytes(bdev);
if (sz) {
unsigned int sizebits = blksize_bits(size);
retval = (sz >> sizebits);
}
return retval;
}
/*
* Initialise the state of a blockdev folio's buffers.
*/
static sector_t folio_init_buffers(struct folio *folio,
struct block_device *bdev, unsigned size)
{
struct buffer_head *head = folio_buffers(folio);
struct buffer_head *bh = head;
bool uptodate = folio_test_uptodate(folio);
sector_t block = div_u64(folio_pos(folio), size);
sector_t end_block = blkdev_max_block(bdev, size);
do {
if (!buffer_mapped(bh)) {
bh->b_end_io = NULL;
bh->b_private = NULL;
bh->b_bdev = bdev;
bh->b_blocknr = block;
if (uptodate)
set_buffer_uptodate(bh);
if (block < end_block)
set_buffer_mapped(bh);
}
block++;
bh = bh->b_this_page;
} while (bh != head);
/*
* Caller needs to validate requested block against end of device.
*/
return end_block;
}
/*
* Create the page-cache folio that contains the requested block.
*
* This is used purely for blockdev mappings.
*
* Returns false if we have a failure which cannot be cured by retrying
* without sleeping. Returns true if we succeeded, or the caller should retry.
*/
static bool grow_dev_folio(struct block_device *bdev, sector_t block,
pgoff_t index, unsigned size, gfp_t gfp)
{
struct inode *inode = bdev->bd_inode;
struct folio *folio;
struct buffer_head *bh;
sector_t end_block = 0;
folio = __filemap_get_folio(inode->i_mapping, index,
FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
if (IS_ERR(folio))
return false;
bh = folio_buffers(folio);
if (bh) {
if (bh->b_size == size) {
end_block = folio_init_buffers(folio, bdev, size);
goto unlock;
}
/*
* Retrying may succeed; for example the folio may finish
* writeback, or buffers may be cleaned. This should not
* happen very often; maybe we have old buffers attached to
* this blockdev's page cache and we're trying to change
* the block size?
*/
if (!try_to_free_buffers(folio)) {
end_block = ~0ULL;
goto unlock;
}
}
bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT);
if (!bh)
goto unlock;
/*
* Link the folio to the buffers and initialise them. Take the
* lock to be atomic wrt __find_get_block(), which does not
* run under the folio lock.
*/
spin_lock(&inode->i_mapping->i_private_lock);
link_dev_buffers(folio, bh);
end_block = folio_init_buffers(folio, bdev, size);
spin_unlock(&inode->i_mapping->i_private_lock);
unlock:
folio_unlock(folio);
folio_put(folio);
return block < end_block;
}
/*
* Create buffers for the specified block device block's folio. If
* that folio was dirty, the buffers are set dirty also. Returns false
* if we've hit a permanent error.
*/
static bool grow_buffers(struct block_device *bdev, sector_t block,
unsigned size, gfp_t gfp)
{
loff_t pos;
/*
* Check for a block which lies outside our maximum possible
* pagecache index.
*/
if (check_mul_overflow(block, (sector_t)size, &pos) || pos > MAX_LFS_FILESIZE) {
printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
__func__, (unsigned long long)block,
bdev);
return false;
}
/* Create a folio with the proper size buffers */
return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
}
static struct buffer_head *
__getblk_slow(struct block_device *bdev, sector_t block,
unsigned size, gfp_t gfp)
{
/* Size must be multiple of hard sectorsize */
if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
(size < 512 || size > PAGE_SIZE))) {
printk(KERN_ERR "getblk(): invalid block size %d requested\n",
size);
printk(KERN_ERR "logical block size: %d\n",
bdev_logical_block_size(bdev));
dump_stack();
return NULL;
}
for (;;) {
struct buffer_head *bh;
bh = __find_get_block(bdev, block, size);
if (bh)
return bh;
if (!grow_buffers(bdev, block, size, gfp))
return NULL;
}
}
/*
* The relationship between dirty buffers and dirty pages:
*
* Whenever a page has any dirty buffers, the page's dirty bit is set, and
* the page is tagged dirty in the page cache.
*
* At all times, the dirtiness of the buffers represents the dirtiness of
* subsections of the page. If the page has buffers, the page dirty bit is
* merely a hint about the true dirty state.
*
* When a page is set dirty in its entirety, all its buffers are marked dirty
* (if the page has buffers).
*
* When a buffer is marked dirty, its page is dirtied, but the page's other
* buffers are not.
*
* Also. When blockdev buffers are explicitly read with bread(), they
* individually become uptodate. But their backing page remains not
* uptodate - even if all of its buffers are uptodate. A subsequent
* block_read_full_folio() against that folio will discover all the uptodate
* buffers, will set the folio uptodate and will perform no I/O.
*/
/**
* mark_buffer_dirty - mark a buffer_head as needing writeout
* @bh: the buffer_head to mark dirty
*
* mark_buffer_dirty() will set the dirty bit against the buffer, then set
* its backing page dirty, then tag the page as dirty in the page cache
* and then attach the address_space's inode to its superblock's dirty
* inode list.
*
* mark_buffer_dirty() is atomic. It takes bh->b_folio->mapping->i_private_lock,
* i_pages lock and mapping->host->i_lock.
*/
void mark_buffer_dirty(struct buffer_head *bh)
{
WARN_ON_ONCE(!buffer_uptodate(bh));
trace_block_dirty_buffer(bh);
/*
* Very *carefully* optimize the it-is-already-dirty case.
*
* Don't let the final "is it dirty" escape to before we
* perhaps modified the buffer.
*/
if (buffer_dirty(bh)) {
smp_mb();
if (buffer_dirty(bh))
return;
}
if (!test_set_buffer_dirty(bh)) {
struct folio *folio = bh->b_folio;
struct address_space *mapping = NULL;
folio_memcg_lock(folio);
if (!folio_test_set_dirty(folio)) {
mapping = folio->mapping;
if (mapping)
__folio_mark_dirty(folio, mapping, 0);
}
folio_memcg_unlock(folio);
if (mapping)
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
}
}
EXPORT_SYMBOL(mark_buffer_dirty);
void mark_buffer_write_io_error(struct buffer_head *bh)
{
set_buffer_write_io_error(bh);
/* FIXME: do we need to set this in both places? */
if (bh->b_folio && bh->b_folio->mapping)
mapping_set_error(bh->b_folio->mapping, -EIO);
if (bh->b_assoc_map) {
mapping_set_error(bh->b_assoc_map, -EIO);
errseq_set(&bh->b_assoc_map->host->i_sb->s_wb_err, -EIO);
}
}
EXPORT_SYMBOL(mark_buffer_write_io_error);
/*
* Decrement a buffer_head's reference count. If all buffers against a page
* have zero reference count, are clean and unlocked, and if the page is clean
* and unlocked then try_to_free_buffers() may strip the buffers from the page
* in preparation for freeing it (sometimes, rarely, buffers are removed from
* a page but it ends up not being freed, and buffers may later be reattached).
*/
void __brelse(struct buffer_head * buf)
{
if (atomic_read(&buf->b_count)) {
put_bh(buf);
return;
}
WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
}
EXPORT_SYMBOL(__brelse);
/*
* bforget() is like brelse(), except it discards any
* potentially dirty data.
*/
void __bforget(struct buffer_head *bh)
{
clear_buffer_dirty(bh);
if (bh->b_assoc_map) {
struct address_space *buffer_mapping = bh->b_folio->mapping;
spin_lock(&buffer_mapping->i_private_lock);
list_del_init(&bh->b_assoc_buffers);
bh->b_assoc_map = NULL;
spin_unlock(&buffer_mapping->i_private_lock);
}
__brelse(bh);
}
EXPORT_SYMBOL(__bforget);
static struct buffer_head *__bread_slow(struct buffer_head *bh)
{
lock_buffer(bh);
if (buffer_uptodate(bh)) {
unlock_buffer(bh);
return bh;
} else {
get_bh(bh);
bh->b_end_io = end_buffer_read_sync;
submit_bh(REQ_OP_READ, bh);
wait_on_buffer(bh);
if (buffer_uptodate(bh))
return bh;
}
brelse(bh);
return NULL;
}
/*
* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
* refcount elevated by one when they're in an LRU. A buffer can only appear
* once in a particular CPU's LRU. A single buffer can be present in multiple
* CPU's LRUs at the same time.
*
* This is a transparent caching front-end to sb_bread(), sb_getblk() and
* sb_find_get_block().
*
* The LRUs themselves only need locking against invalidate_bh_lrus. We use
* a local interrupt disable for that.
*/
#define BH_LRU_SIZE 16
struct bh_lru {
struct buffer_head *bhs[BH_LRU_SIZE];
};
static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
#ifdef CONFIG_SMP
#define bh_lru_lock() local_irq_disable()
#define bh_lru_unlock() local_irq_enable()
#else
#define bh_lru_lock() preempt_disable()
#define bh_lru_unlock() preempt_enable()
#endif
static inline void check_irqs_on(void)
{
#ifdef irqs_disabled
BUG_ON(irqs_disabled());
#endif
}
/*
* Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
* inserted at the front, and the buffer_head at the back if any is evicted.
* Or, if already in the LRU it is moved to the front.
*/
static void bh_lru_install(struct buffer_head *bh)
{
struct buffer_head *evictee = bh;
struct bh_lru *b;
int i;
check_irqs_on();
bh_lru_lock();
/*
* the refcount of buffer_head in bh_lru prevents dropping the
* attached page(i.e., try_to_free_buffers) so it could cause
* failing page migration.
* Skip putting upcoming bh into bh_lru until migration is done.
*/
if (lru_cache_disabled() || cpu_is_isolated(smp_processor_id())) {
bh_lru_unlock();
return;
}
b = this_cpu_ptr(&bh_lrus);
for (i = 0; i < BH_LRU_SIZE; i++) {
swap(evictee, b->bhs[i]);
if (evictee == bh) {
bh_lru_unlock();
return;
}
}
get_bh(bh);
bh_lru_unlock();
brelse(evictee);
}
/*
* Look up the bh in this cpu's LRU. If it's there, move it to the head.
*/
static struct buffer_head *
lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
{
struct buffer_head *ret = NULL;
unsigned int i;
check_irqs_on();
bh_lru_lock();
if (cpu_is_isolated(smp_processor_id())) {
bh_lru_unlock();
return NULL;
}
for (i = 0; i < BH_LRU_SIZE; i++) {
struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
bh->b_size == size) {
if (i) {
while (i) {
__this_cpu_write(bh_lrus.bhs[i],
__this_cpu_read(bh_lrus.bhs[i - 1]));
i--;
}
__this_cpu_write(bh_lrus.bhs[0], bh);
}
get_bh(bh);
ret = bh;
break;
}
}
bh_lru_unlock();
return ret;
}
/*
* Perform a pagecache lookup for the matching buffer. If it's there, refresh
* it in the LRU and mark it as accessed. If it is not present then return
* NULL
*/
struct buffer_head *
__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
{
struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
if (bh == NULL) {
/* __find_get_block_slow will mark the page accessed */
bh = __find_get_block_slow(bdev, block);
if (bh)
bh_lru_install(bh);
} else
touch_buffer(bh);
return bh;
}
EXPORT_SYMBOL(__find_get_block);
/**
* bdev_getblk - Get a buffer_head in a block device's buffer cache.
* @bdev: The block device.
* @block: The block number.
* @size: The size of buffer_heads for this @bdev.
* @gfp: The memory allocation flags to use.
*
* Return: The buffer head, or NULL if memory could not be allocated.
*/
struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block,
unsigned size, gfp_t gfp)
{
struct buffer_head *bh = __find_get_block(bdev, block, size);
might_alloc(gfp);
if (bh)
return bh;
return __getblk_slow(bdev, block, size, gfp);
}
EXPORT_SYMBOL(bdev_getblk);
/*
* Do async read-ahead on a buffer..
*/
void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
{
struct buffer_head *bh = bdev_getblk(bdev, block, size,
GFP_NOWAIT | __GFP_MOVABLE);
if (likely(bh)) {
bh_readahead(bh, REQ_RAHEAD);
brelse(bh);
}
}
EXPORT_SYMBOL(__breadahead);
/**
* __bread_gfp() - reads a specified block and returns the bh
* @bdev: the block_device to read from
* @block: number of block
* @size: size (in bytes) to read
* @gfp: page allocation flag
*
* Reads a specified block, and returns buffer head that contains it.
* The page cache can be allocated from non-movable area
* not to prevent page migration if you set gfp to zero.
* It returns NULL if the block was unreadable.
*/
struct buffer_head *
__bread_gfp(struct block_device *bdev, sector_t block,
unsigned size, gfp_t gfp)
{
struct buffer_head *bh;
gfp |= mapping_gfp_constraint(bdev->bd_inode->i_mapping, ~__GFP_FS);
/*
* Prefer looping in the allocator rather than here, at least that
* code knows what it's doing.
*/
gfp |= __GFP_NOFAIL;
bh = bdev_getblk(bdev, block, size, gfp);
if (likely(bh) && !buffer_uptodate(bh))
bh = __bread_slow(bh);
return bh;
}
EXPORT_SYMBOL(__bread_gfp);
static void __invalidate_bh_lrus(struct bh_lru *b)
{
int i;
for (i = 0; i < BH_LRU_SIZE; i++) {
brelse(b->bhs[i]);
b->bhs[i] = NULL;
}
}
/*
* invalidate_bh_lrus() is called rarely - but not only at unmount.
* This doesn't race because it runs in each cpu either in irq
* or with preempt disabled.
*/
static void invalidate_bh_lru(void *arg)
{
struct bh_lru *b = &get_cpu_var(bh_lrus);
__invalidate_bh_lrus(b);
put_cpu_var(bh_lrus);
}
bool has_bh_in_lru(int cpu, void *dummy)
{
struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
int i;
for (i = 0; i < BH_LRU_SIZE; i++) {
if (b->bhs[i])
return true;
}
return false;
}
void invalidate_bh_lrus(void)
{
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
}
EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
/*
* It's called from workqueue context so we need a bh_lru_lock to close
* the race with preemption/irq.
*/
void invalidate_bh_lrus_cpu(void)
{
struct bh_lru *b;
bh_lru_lock();
b = this_cpu_ptr(&bh_lrus);
__invalidate_bh_lrus(b);
bh_lru_unlock();
}
void folio_set_bh(struct buffer_head *bh, struct folio *folio,
unsigned long offset)
{
bh->b_folio = folio;
BUG_ON(offset >= folio_size(folio));
if (folio_test_highmem(folio))
/*
* This catches illegal uses and preserves the offset:
*/
bh->b_data = (char *)(0 + offset);
else
bh->b_data = folio_address(folio) + offset;
}
EXPORT_SYMBOL(folio_set_bh);
/*
* Called when truncating a buffer on a page completely.
*/
/* Bits that are cleared during an invalidate */
#define BUFFER_FLAGS_DISCARD \
(1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1 << BH_Delay | 1 << BH_Unwritten)
static void discard_buffer(struct buffer_head * bh)
{
unsigned long b_state;
lock_buffer(bh);
clear_buffer_dirty(bh);
bh->b_bdev = NULL;
b_state = READ_ONCE(bh->b_state);
do {
} while (!try_cmpxchg(&bh->b_state, &b_state,
b_state & ~BUFFER_FLAGS_DISCARD));
unlock_buffer(bh);
}
/**
* block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
* @folio: The folio which is affected.
* @offset: start of the range to invalidate
* @length: length of the range to invalidate
*
* block_invalidate_folio() is called when all or part of the folio has been
* invalidated by a truncate operation.
*
* block_invalidate_folio() does not have to release all buffers, but it must
* ensure that no dirty buffer is left outside @offset and that no I/O
* is underway against any of the blocks which are outside the truncation
* point. Because the caller is about to free (and possibly reuse) those
* blocks on-disk.
*/
void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
{
struct buffer_head *head, *bh, *next;
size_t curr_off = 0;
size_t stop = length + offset;
BUG_ON(!folio_test_locked(folio));
/*
* Check for overflow
*/
BUG_ON(stop > folio_size(folio) || stop < length);
head = folio_buffers(folio);
if (!head)
return;
bh = head;
do {
size_t next_off = curr_off + bh->b_size;
next = bh->b_this_page;
/*
* Are we still fully in range ?
*/
if (next_off > stop)
goto out;
/*
* is this block fully invalidated?
*/
if (offset <= curr_off)
discard_buffer(bh);
curr_off = next_off;
bh = next;
} while (bh != head);
/*
* We release buffers only if the entire folio is being invalidated.
* The get_block cached value has been unconditionally invalidated,
* so real IO is not possible anymore.
*/
if (length == folio_size(folio))
filemap_release_folio(folio, 0);
out:
return;
}
EXPORT_SYMBOL(block_invalidate_folio);
/*
* We attach and possibly dirty the buffers atomically wrt
* block_dirty_folio() via i_private_lock. try_to_free_buffers
* is already excluded via the folio lock.
*/
struct buffer_head *create_empty_buffers(struct folio *folio,
unsigned long blocksize, unsigned long b_state)
{
struct buffer_head *bh, *head, *tail;
gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT | __GFP_NOFAIL;
head = folio_alloc_buffers(folio, blocksize, gfp);
bh = head;
do {
bh->b_state |= b_state;
tail = bh;
bh = bh->b_this_page;
} while (bh);
tail->b_this_page = head;
spin_lock(&folio->mapping->i_private_lock);
if (folio_test_uptodate(folio) || folio_test_dirty(folio)) {
bh = head;
do {
if (folio_test_dirty(folio))
set_buffer_dirty(bh);
if (folio_test_uptodate(folio))
set_buffer_uptodate(bh);
bh = bh->b_this_page;
} while (bh != head);
}
folio_attach_private(folio, head);
spin_unlock(&folio->mapping->i_private_lock);
return head;
}
EXPORT_SYMBOL(create_empty_buffers);
/**
* clean_bdev_aliases: clean a range of buffers in block device
* @bdev: Block device to clean buffers in
* @block: Start of a range of blocks to clean
* @len: Number of blocks to clean
*
* We are taking a range of blocks for data and we don't want writeback of any
* buffer-cache aliases starting from return from this function and until the
* moment when something will explicitly mark the buffer dirty (hopefully that
* will not happen until we will free that block ;-) We don't even need to mark
* it not-uptodate - nobody can expect anything from a newly allocated buffer
* anyway. We used to use unmap_buffer() for such invalidation, but that was
* wrong. We definitely don't want to mark the alias unmapped, for example - it
* would confuse anyone who might pick it with bread() afterwards...
*
* Also.. Note that bforget() doesn't lock the buffer. So there can be
* writeout I/O going on against recently-freed buffers. We don't wait on that
* I/O in bforget() - it's more efficient to wait on the I/O only if we really
* need to. That happens here.
*/
void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
{
struct inode *bd_inode = bdev->bd_inode;
struct address_space *bd_mapping = bd_inode->i_mapping;
struct folio_batch fbatch;
pgoff_t index = ((loff_t)block << bd_inode->i_blkbits) / PAGE_SIZE;
pgoff_t end;
int i, count;
struct buffer_head *bh;
struct buffer_head *head;
end = ((loff_t)(block + len - 1) << bd_inode->i_blkbits) / PAGE_SIZE;
folio_batch_init(&fbatch);
while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
count = folio_batch_count(&fbatch);
for (i = 0; i < count; i++) {
struct folio *folio = fbatch.folios[i];
if (!folio_buffers(folio))
continue;
/*
* We use folio lock instead of bd_mapping->i_private_lock
* to pin buffers here since we can afford to sleep and
* it scales better than a global spinlock lock.
*/
folio_lock(folio);
/* Recheck when the folio is locked which pins bhs */
head = folio_buffers(folio);
if (!head)
goto unlock_page;
bh = head;
do {
if (!buffer_mapped(bh) || (bh->b_blocknr < block))
goto next;
if (bh->b_blocknr >= block + len)
break;
clear_buffer_dirty(bh);
wait_on_buffer(bh);
clear_buffer_req(bh);
next:
bh = bh->b_this_page;
} while (bh != head);
unlock_page:
folio_unlock(folio);
}
folio_batch_release(&fbatch);
cond_resched();
/* End of range already reached? */
if (index > end || !index)
break;
}
}
EXPORT_SYMBOL(clean_bdev_aliases);
static struct buffer_head *folio_create_buffers(struct folio *folio,
struct inode *inode,
unsigned int b_state)
{
struct buffer_head *bh;
BUG_ON(!folio_test_locked(folio));
bh = folio_buffers(folio);
if (!bh)
bh = create_empty_buffers(folio,
1 << READ_ONCE(inode->i_blkbits), b_state);
return bh;
}
/*
* NOTE! All mapped/uptodate combinations are valid:
*
* Mapped Uptodate Meaning
*
* No No "unknown" - must do get_block()
* No Yes "hole" - zero-filled
* Yes No "allocated" - allocated on disk, not read in
* Yes Yes "valid" - allocated and up-to-date in memory.
*
* "Dirty" is valid only with the last case (mapped+uptodate).
*/
/*
* While block_write_full_folio is writing back the dirty buffers under
* the page lock, whoever dirtied the buffers may decide to clean them
* again at any time. We handle that by only looking at the buffer
* state inside lock_buffer().
*
* If block_write_full_folio() is called for regular writeback
* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
* locked buffer. This only can happen if someone has written the buffer
* directly, with submit_bh(). At the address_space level PageWriteback
* prevents this contention from occurring.
*
* If block_write_full_folio() is called with wbc->sync_mode ==
* WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
* causes the writes to be flagged as synchronous writes.
*/
int __block_write_full_folio(struct inode *inode, struct folio *folio,
get_block_t *get_block, struct writeback_control *wbc)
{
int err;
sector_t block;
sector_t last_block;
struct buffer_head *bh, *head;
size_t blocksize;
int nr_underway = 0;
blk_opf_t write_flags = wbc_to_write_flags(wbc);
head = folio_create_buffers(folio, inode,
(1 << BH_Dirty) | (1 << BH_Uptodate));
/*
* Be very careful. We have no exclusion from block_dirty_folio
* here, and the (potentially unmapped) buffers may become dirty at
* any time. If a buffer becomes dirty here after we've inspected it
* then we just miss that fact, and the folio stays dirty.
*
* Buffers outside i_size may be dirtied by block_dirty_folio;
* handle that here by just cleaning them.
*/
bh = head;
blocksize = bh->b_size;
block = div_u64(folio_pos(folio), blocksize);
last_block = div_u64(i_size_read(inode) - 1, blocksize);
/*
* Get all the dirty buffers mapped to disk addresses and
* handle any aliases from the underlying blockdev's mapping.
*/
do {
if (block > last_block) {
/*
* mapped buffers outside i_size will occur, because
* this folio can be outside i_size when there is a
* truncate in progress.
*/
/*
* The buffer was zeroed by block_write_full_folio()
*/
clear_buffer_dirty(bh);
set_buffer_uptodate(bh);
} else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
buffer_dirty(bh)) {
WARN_ON(bh->b_size != blocksize);
err = get_block(inode, block, bh, 1);
if (err)
goto recover;
clear_buffer_delay(bh);
if (buffer_new(bh)) {
/* blockdev mappings never come here */
clear_buffer_new(bh);
clean_bdev_bh_alias(bh);
}
}
bh = bh->b_this_page;
block++;
} while (bh != head);
do {
if (!buffer_mapped(bh))
continue;
/*
* If it's a fully non-blocking write attempt and we cannot
* lock the buffer then redirty the folio. Note that this can
* potentially cause a busy-wait loop from writeback threads
* and kswapd activity, but those code paths have their own
* higher-level throttling.
*/
if (wbc->sync_mode != WB_SYNC_NONE) {
lock_buffer(bh);
} else if (!trylock_buffer(bh)) {
folio_redirty_for_writepage(wbc, folio);
continue;
}
if (test_clear_buffer_dirty(bh)) {
mark_buffer_async_write_endio(bh,
end_buffer_async_write);
} else {
unlock_buffer(bh);
}
} while ((bh = bh->b_this_page) != head);
/*
* The folio and its buffers are protected by the writeback flag,
* so we can drop the bh refcounts early.
*/
BUG_ON(folio_test_writeback(folio));
folio_start_writeback(folio);
do {
struct buffer_head *next = bh->b_this_page;
if (buffer_async_write(bh)) {
submit_bh_wbc(REQ_OP_WRITE | write_flags, bh,
inode->i_write_hint, wbc);
nr_underway++;
}
bh = next;
} while (bh != head);
folio_unlock(folio);
err = 0;
done:
if (nr_underway == 0) {
/*
* The folio was marked dirty, but the buffers were
* clean. Someone wrote them back by hand with
* write_dirty_buffer/submit_bh. A rare case.
*/
folio_end_writeback(folio);
/*
* The folio and buffer_heads can be released at any time from
* here on.
*/
}
return err;
recover:
/*
* ENOSPC, or some other error. We may already have added some
* blocks to the file, so we need to write these out to avoid
* exposing stale data.
* The folio is currently locked and not marked for writeback
*/
bh = head;
/* Recovery: lock and submit the mapped buffers */
do {
if (buffer_mapped(bh) && buffer_dirty(bh) &&
!buffer_delay(bh)) {
lock_buffer(bh);
mark_buffer_async_write_endio(bh,
end_buffer_async_write);
} else {
/*
* The buffer may have been set dirty during
* attachment to a dirty folio.
*/
clear_buffer_dirty(bh);
}
} while ((bh = bh->b_this_page) != head);
folio_set_error(folio);
BUG_ON(folio_test_writeback(folio));
mapping_set_error(folio->mapping, err);
folio_start_writeback(folio);
do {
struct buffer_head *next = bh->b_this_page;
if (buffer_async_write(bh)) {
clear_buffer_dirty(bh);
submit_bh_wbc(REQ_OP_WRITE | write_flags, bh,
inode->i_write_hint, wbc);
nr_underway++;
}
bh = next;
} while (bh != head);
folio_unlock(folio);
goto done;
}
EXPORT_SYMBOL(__block_write_full_folio);
/*
* If a folio has any new buffers, zero them out here, and mark them uptodate
* and dirty so they'll be written out (in order to prevent uninitialised
* block data from leaking). And clear the new bit.
*/
void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)
{
size_t block_start, block_end;
struct buffer_head *head, *bh;
BUG_ON(!folio_test_locked(folio));
head = folio_buffers(folio);
if (!head)
return;
bh = head;
block_start = 0;
do {
block_end = block_start + bh->b_size;
if (buffer_new(bh)) {
if (block_end > from && block_start < to) {
if (!folio_test_uptodate(folio)) {
size_t start, xend;
start = max(from, block_start);
xend = min(to, block_end);
folio_zero_segment(folio, start, xend);
set_buffer_uptodate(bh);
}
clear_buffer_new(bh);
mark_buffer_dirty(bh);
}
}
block_start = block_end;
bh = bh->b_this_page;
} while (bh != head);
}
EXPORT_SYMBOL(folio_zero_new_buffers);
static int
iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
const struct iomap *iomap)
{
loff_t offset = (loff_t)block << inode->i_blkbits;
bh->b_bdev = iomap->bdev;
/*
* Block points to offset in file we need to map, iomap contains
* the offset at which the map starts. If the map ends before the
* current block, then do not map the buffer and let the caller
* handle it.
*/
if (offset >= iomap->offset + iomap->length)
return -EIO;
switch (iomap->type) {
case IOMAP_HOLE:
/*
* If the buffer is not up to date or beyond the current EOF,
* we need to mark it as new to ensure sub-block zeroing is
* executed if necessary.
*/
if (!buffer_uptodate(bh) ||
(offset >= i_size_read(inode)))
set_buffer_new(bh);
return 0;
case IOMAP_DELALLOC:
if (!buffer_uptodate(bh) ||
(offset >= i_size_read(inode)))
set_buffer_new(bh);
set_buffer_uptodate(bh);
set_buffer_mapped(bh);
set_buffer_delay(bh);
return 0;
case IOMAP_UNWRITTEN:
/*
* For unwritten regions, we always need to ensure that regions
* in the block we are not writing to are zeroed. Mark the
* buffer as new to ensure this.
*/
set_buffer_new(bh);
set_buffer_unwritten(bh);
fallthrough;
case IOMAP_MAPPED:
if ((iomap->flags & IOMAP_F_NEW) ||
offset >= i_size_read(inode)) {
/*
* This can happen if truncating the block device races
* with the check in the caller as i_size updates on
* block devices aren't synchronized by i_rwsem for
* block devices.
*/
if (S_ISBLK(inode->i_mode))
return -EIO;
set_buffer_new(bh);
}
bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
inode->i_blkbits;
set_buffer_mapped(bh);
return 0;
default:
WARN_ON_ONCE(1);
return -EIO;
}
}
int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
get_block_t *get_block, const struct iomap *iomap)
{
size_t from = offset_in_folio(folio, pos);
size_t to = from + len;
struct inode *inode = folio->mapping->host;
size_t block_start, block_end;
sector_t block;
int err = 0;
size_t blocksize;
struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
BUG_ON(!folio_test_locked(folio));
BUG_ON(to > folio_size(folio));
BUG_ON(from > to);
head = folio_create_buffers(folio, inode, 0);
blocksize = head->b_size;
block = div_u64(folio_pos(folio), blocksize);
for (bh = head, block_start = 0; bh != head || !block_start;
block++, block_start=block_end, bh = bh->b_this_page) {
block_end = block_start + blocksize;
if (block_end <= from || block_start >= to) {
if (folio_test_uptodate(folio)) {
if (!buffer_uptodate(bh))
set_buffer_uptodate(bh);
}
continue;
}
if (buffer_new(bh))
clear_buffer_new(bh);
if (!buffer_mapped(bh)) {
WARN_ON(bh->b_size != blocksize);
if (get_block)
err = get_block(inode, block, bh, 1);
else
err = iomap_to_bh(inode, block, bh, iomap);
if (err)
break;
if (buffer_new(bh)) {
clean_bdev_bh_alias(bh);
if (folio_test_uptodate(folio)) {
clear_buffer_new(bh);
set_buffer_uptodate(bh);
mark_buffer_dirty(bh);
continue;
}
if (block_end > to || block_start < from)
folio_zero_segments(folio,
to, block_end,
block_start, from);
continue;
}
}
if (folio_test_uptodate(folio)) {
if (!buffer_uptodate(bh))
set_buffer_uptodate(bh);
continue;
}
if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
!buffer_unwritten(bh) &&
(block_start < from || block_end > to)) {
bh_read_nowait(bh, 0);
*wait_bh++=bh;
}
}
/*
* If we issued read requests - let them complete.
*/
while(wait_bh > wait) {
wait_on_buffer(*--wait_bh);
if (!buffer_uptodate(*wait_bh))
err = -EIO;
}
if (unlikely(err))
folio_zero_new_buffers(folio, from, to);
return err;
}
int __block_write_begin(struct page *page, loff_t pos, unsigned len,
get_block_t *get_block)
{
return __block_write_begin_int(page_folio(page), pos, len, get_block,
NULL);
}
EXPORT_SYMBOL(__block_write_begin);
static void __block_commit_write(struct folio *folio, size_t from, size_t to)
{
size_t block_start, block_end;
bool partial = false;
unsigned blocksize;
struct buffer_head *bh, *head;
bh = head = folio_buffers(folio);
blocksize = bh->b_size;
block_start = 0;
do {
block_end = block_start + blocksize;
if (block_end <= from || block_start >= to) {
if (!buffer_uptodate(bh))
partial = true;
} else {
set_buffer_uptodate(bh);
mark_buffer_dirty(bh);
}
if (buffer_new(bh))
clear_buffer_new(bh);
block_start = block_end;
bh = bh->b_this_page;
} while (bh != head);
/*
* If this is a partial write which happened to make all buffers
* uptodate then we can optimize away a bogus read_folio() for
* the next read(). Here we 'discover' whether the folio went
* uptodate as a result of this (potentially partial) write.
*/
if (!partial)
folio_mark_uptodate(folio);
}
/*
* block_write_begin takes care of the basic task of block allocation and
* bringing partial write blocks uptodate first.
*
* The filesystem needs to handle block truncation upon failure.
*/
int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
struct page **pagep, get_block_t *get_block)
{
pgoff_t index = pos >> PAGE_SHIFT;
struct page *page;
int status;
page = grab_cache_page_write_begin(mapping, index);
if (!page)
return -ENOMEM;
status = __block_write_begin(page, pos, len, get_block);
if (unlikely(status)) {
unlock_page(page);
put_page(page);
page = NULL;
}
*pagep = page;
return status;
}
EXPORT_SYMBOL(block_write_begin);
int block_write_end(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len, unsigned copied,
struct page *page, void *fsdata)
{
struct folio *folio = page_folio(page);
size_t start = pos - folio_pos(folio);
if (unlikely(copied < len)) {
/*
* The buffers that were written will now be uptodate, so
* we don't have to worry about a read_folio reading them
* and overwriting a partial write. However if we have
* encountered a short write and only partially written
* into a buffer, it will not be marked uptodate, so a
* read_folio might come in and destroy our partial write.
*
* Do the simplest thing, and just treat any short write to a
* non uptodate folio as a zero-length write, and force the
* caller to redo the whole thing.
*/
if (!folio_test_uptodate(folio))
copied = 0;
folio_zero_new_buffers(folio, start+copied, start+len);
}
flush_dcache_folio(folio);
/* This could be a short (even 0-length) commit */
__block_commit_write(folio, start, start + copied);
return copied;
}
EXPORT_SYMBOL(block_write_end);
int generic_write_end(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len, unsigned copied,
struct page *page, void *fsdata)
{
struct inode *inode = mapping->host;
loff_t old_size = inode->i_size;
bool i_size_changed = false;
copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
/*
* No need to use i_size_read() here, the i_size cannot change under us
* because we hold i_rwsem.
*
* But it's important to update i_size while still holding page lock:
* page writeout could otherwise come in and zero beyond i_size.
*/
if (pos + copied > inode->i_size) {
i_size_write(inode, pos + copied);
i_size_changed = true;
}
unlock_page(page);
put_page(page);
if (old_size < pos)
pagecache_isize_extended(inode, old_size, pos);
/*
* Don't mark the inode dirty under page lock. First, it unnecessarily
* makes the holding time of page lock longer. Second, it forces lock
* ordering of page lock and transaction start for journaling
* filesystems.
*/
if (i_size_changed)
mark_inode_dirty(inode);
return copied;
}
EXPORT_SYMBOL(generic_write_end);
/*
* block_is_partially_uptodate checks whether buffers within a folio are
* uptodate or not.
*
* Returns true if all buffers which correspond to the specified part
* of the folio are uptodate.
*/
bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
{
unsigned block_start, block_end, blocksize;
unsigned to;
struct buffer_head *bh, *head;
bool ret = true;
head = folio_buffers(folio);
if (!head)
return false;
blocksize = head->b_size;
to = min_t(unsigned, folio_size(folio) - from, count);
to = from + to;
if (from < blocksize && to > folio_size(folio) - blocksize)
return false;
bh = head;
block_start = 0;
do {
block_end = block_start + blocksize;
if (block_end > from && block_start < to) {
if (!buffer_uptodate(bh)) {
ret = false;
break;
}
if (block_end >= to)
break;
}
block_start = block_end;
bh = bh->b_this_page;
} while (bh != head);
return ret;
}
EXPORT_SYMBOL(block_is_partially_uptodate);
/*
* Generic "read_folio" function for block devices that have the normal
* get_block functionality. This is most of the block device filesystems.
* Reads the folio asynchronously --- the unlock_buffer() and
* set/clear_buffer_uptodate() functions propagate buffer state into the
* folio once IO has completed.
*/
int block_read_full_folio(struct folio *folio, get_block_t *get_block)
{
struct inode *inode = folio->mapping->host;
sector_t iblock, lblock;
struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
size_t blocksize;
int nr, i;
int fully_mapped = 1;
bool page_error = false;
loff_t limit = i_size_read(inode);
/* This is needed for ext4. */
if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
limit = inode->i_sb->s_maxbytes;
VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
head = folio_create_buffers(folio, inode, 0);
blocksize = head->b_size;
iblock = div_u64(folio_pos(folio), blocksize);
lblock = div_u64(limit + blocksize - 1, blocksize);
bh = head;
nr = 0;
i = 0;
do {
if (buffer_uptodate(bh))
continue;
if (!buffer_mapped(bh)) {
int err = 0;
fully_mapped = 0;
if (iblock < lblock) {
WARN_ON(bh->b_size != blocksize);
err = get_block(inode, iblock, bh, 0);
if (err) {
folio_set_error(folio);
page_error = true;
}
}
if (!buffer_mapped(bh)) {
folio_zero_range(folio, i * blocksize,
blocksize);
if (!err)
set_buffer_uptodate(bh);
continue;
}
/*
* get_block() might have updated the buffer
* synchronously
*/
if (buffer_uptodate(bh))
continue;
}
arr[nr++] = bh;
} while (i++, iblock++, (bh = bh->b_this_page) != head);
if (fully_mapped)
folio_set_mappedtodisk(folio);
if (!nr) {
/*
* All buffers are uptodate or get_block() returned an
* error when trying to map them - we can finish the read.
*/
folio_end_read(folio, !page_error);
return 0;
}
/* Stage two: lock the buffers */
for (i = 0; i < nr; i++) {
bh = arr[i];
lock_buffer(bh);
mark_buffer_async_read(bh);
}
/*
* Stage 3: start the IO. Check for uptodateness
* inside the buffer lock in case another process reading
* the underlying blockdev brought it uptodate (the sct fix).
*/
for (i = 0; i < nr; i++) {
bh = arr[i];
if (buffer_uptodate(bh))
end_buffer_async_read(bh, 1);
else
submit_bh(REQ_OP_READ, bh);
}
return 0;
}
EXPORT_SYMBOL(block_read_full_folio);
/* utility function for filesystems that need to do work on expanding
* truncates. Uses filesystem pagecache writes to allow the filesystem to
* deal with the hole.
*/
int generic_cont_expand_simple(struct inode *inode, loff_t size)
{
struct address_space *mapping = inode->i_mapping;
const struct address_space_operations *aops = mapping->a_ops;
struct page *page;
void *fsdata = NULL;
int err;
err = inode_newsize_ok(inode, size);
if (err)
goto out;
err = aops->write_begin(NULL, mapping, size, 0, &page, &fsdata);
if (err)
goto out;
err = aops->write_end(NULL, mapping, size, 0, 0, page, fsdata);
BUG_ON(err > 0);
out:
return err;
}
EXPORT_SYMBOL(generic_cont_expand_simple);
static int cont_expand_zero(struct file *file, struct address_space *mapping,
loff_t pos, loff_t *bytes)
{
struct inode *inode = mapping->host;
const struct address_space_operations *aops = mapping->a_ops;
unsigned int blocksize = i_blocksize(inode);
struct page *page;
void *fsdata = NULL;
pgoff_t index, curidx;
loff_t curpos;
unsigned zerofrom, offset, len;
int err = 0;
index = pos >> PAGE_SHIFT;
offset = pos & ~PAGE_MASK;
while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
zerofrom = curpos & ~PAGE_MASK;
if (zerofrom & (blocksize-1)) {
*bytes |= (blocksize-1);
(*bytes)++;
}
len = PAGE_SIZE - zerofrom;
err = aops->write_begin(file, mapping, curpos, len,
&page, &fsdata);
if (err)
goto out;
zero_user(page, zerofrom, len);
err = aops->write_end(file, mapping, curpos, len, len,
page, fsdata);
if (err < 0)
goto out;
BUG_ON(err != len);
err = 0;
balance_dirty_pages_ratelimited(mapping);
if (fatal_signal_pending(current)) {
err = -EINTR;
goto out;
}
}
/* page covers the boundary, find the boundary offset */
if (index == curidx) {
zerofrom = curpos & ~PAGE_MASK;
/* if we will expand the thing last block will be filled */
if (offset <= zerofrom) {
goto out;
}
if (zerofrom & (blocksize-1)) {
*bytes |= (blocksize-1);
(*bytes)++;
}
len = offset - zerofrom;
err = aops->write_begin(file, mapping, curpos, len,
&page, &fsdata);
if (err)
goto out;
zero_user(page, zerofrom, len);
err = aops->write_end(file, mapping, curpos, len, len,
page, fsdata);
if (err < 0)
goto out;
BUG_ON(err != len);
err = 0;
}
out:
return err;
}
/*
* For moronic filesystems that do not allow holes in file.
* We may have to extend the file.
*/
int cont_write_begin(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len,
struct page **pagep, void **fsdata,
get_block_t *get_block, loff_t *bytes)
{
struct inode *inode = mapping->host;
unsigned int blocksize = i_blocksize(inode);
unsigned int zerofrom;
int err;
err = cont_expand_zero(file, mapping, pos, bytes);
if (err)
return err;
zerofrom = *bytes & ~PAGE_MASK;
if (pos+len > *bytes && zerofrom & (blocksize-1)) {
*bytes |= (blocksize-1);
(*bytes)++;
}
return block_write_begin(mapping, pos, len, pagep, get_block);
}
EXPORT_SYMBOL(cont_write_begin);
void block_commit_write(struct page *page, unsigned from, unsigned to)
{
struct folio *folio = page_folio(page);
__block_commit_write(folio, from, to);
}
EXPORT_SYMBOL(block_commit_write);
/*
* block_page_mkwrite() is not allowed to change the file size as it gets
* called from a page fault handler when a page is first dirtied. Hence we must
* be careful to check for EOF conditions here. We set the page up correctly
* for a written page which means we get ENOSPC checking when writing into
* holes and correct delalloc and unwritten extent mapping on filesystems that
* support these features.
*
* We are not allowed to take the i_mutex here so we have to play games to
* protect against truncate races as the page could now be beyond EOF. Because
* truncate writes the inode size before removing pages, once we have the
* page lock we can determine safely if the page is beyond EOF. If it is not
* beyond EOF, then the page is guaranteed safe against truncation until we
* unlock the page.
*
* Direct callers of this function should protect against filesystem freezing
* using sb_start_pagefault() - sb_end_pagefault() functions.
*/
int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
get_block_t get_block)
{
struct folio *folio = page_folio(vmf->page);
struct inode *inode = file_inode(vma->vm_file);
unsigned long end;
loff_t size;
int ret;
folio_lock(folio);
size = i_size_read(inode);
if ((folio->mapping != inode->i_mapping) ||
(folio_pos(folio) >= size)) {
/* We overload EFAULT to mean page got truncated */
ret = -EFAULT;
goto out_unlock;
}
end = folio_size(folio);
/* folio is wholly or partially inside EOF */
if (folio_pos(folio) + end > size)
end = size - folio_pos(folio);
ret = __block_write_begin_int(folio, 0, end, get_block, NULL);
if (unlikely(ret))
goto out_unlock;
__block_commit_write(folio, 0, end);
folio_mark_dirty(folio);
folio_wait_stable(folio);
return 0;
out_unlock:
folio_unlock(folio);
return ret;
}
EXPORT_SYMBOL(block_page_mkwrite);
int block_truncate_page(struct address_space *mapping,
loff_t from, get_block_t *get_block)
{
pgoff_t index = from >> PAGE_SHIFT;
unsigned blocksize;
sector_t iblock;
size_t offset, length, pos;
struct inode *inode = mapping->host;
struct folio *folio;
struct buffer_head *bh;
int err = 0;
blocksize = i_blocksize(inode);
length = from & (blocksize - 1);
/* Block boundary? Nothing to do */
if (!length)
return 0;
length = blocksize - length;
iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits;
folio = filemap_grab_folio(mapping, index);
if (IS_ERR(folio))
return PTR_ERR(folio);
bh = folio_buffers(folio);
if (!bh)
bh = create_empty_buffers(folio, blocksize, 0);
/* Find the buffer that contains "offset" */
offset = offset_in_folio(folio, from);
pos = blocksize;
while (offset >= pos) {
bh = bh->b_this_page;
iblock++;
pos += blocksize;
}
if (!buffer_mapped(bh)) {
WARN_ON(bh->b_size != blocksize);
err = get_block(inode, iblock, bh, 0);
if (err)
goto unlock;
/* unmapped? It's a hole - nothing to do */
if (!buffer_mapped(bh))
goto unlock;
}
/* Ok, it's mapped. Make sure it's up-to-date */
if (folio_test_uptodate(folio))
set_buffer_uptodate(bh);
if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
err = bh_read(bh, 0);
/* Uhhuh. Read error. Complain and punt. */
if (err < 0)
goto unlock;
}
folio_zero_range(folio, offset, length);
mark_buffer_dirty(bh);
unlock:
folio_unlock(folio);
folio_put(folio);
return err;
}
EXPORT_SYMBOL(block_truncate_page);
/*
* The generic ->writepage function for buffer-backed address_spaces
*/
int block_write_full_folio(struct folio *folio, struct writeback_control *wbc,
void *get_block)
{
struct inode * const inode = folio->mapping->host;
loff_t i_size = i_size_read(inode);
/* Is the folio fully inside i_size? */
if (folio_pos(folio) + folio_size(folio) <= i_size)
return __block_write_full_folio(inode, folio, get_block, wbc);
/* Is the folio fully outside i_size? (truncate in progress) */
if (folio_pos(folio) >= i_size) {
folio_unlock(folio);
return 0; /* don't care */
}
/*
* The folio straddles i_size. It must be zeroed out on each and every
* writepage invocation because it may be mmapped. "A file is mapped
* in multiples of the page size. For a file that is not a multiple of
* the page size, the remaining memory is zeroed when mapped, and
* writes to that region are not written out to the file."
*/
folio_zero_segment(folio, offset_in_folio(folio, i_size),
folio_size(folio));
return __block_write_full_folio(inode, folio, get_block, wbc);
}
sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
get_block_t *get_block)
{
struct inode *inode = mapping->host;
struct buffer_head tmp = {
.b_size = i_blocksize(inode),
};
get_block(inode, block, &tmp, 0);
return tmp.b_blocknr;
}
EXPORT_SYMBOL(generic_block_bmap);
static void end_bio_bh_io_sync(struct bio *bio)
{
struct buffer_head *bh = bio->bi_private;
if (unlikely(bio_flagged(bio, BIO_QUIET)))
set_bit(BH_Quiet, &bh->b_state);
bh->b_end_io(bh, !bio->bi_status);
bio_put(bio);
}
static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
enum rw_hint write_hint,
struct writeback_control *wbc)
{
const enum req_op op = opf & REQ_OP_MASK;
struct bio *bio;
BUG_ON(!buffer_locked(bh));
BUG_ON(!buffer_mapped(bh));
BUG_ON(!bh->b_end_io);
BUG_ON(buffer_delay(bh));
BUG_ON(buffer_unwritten(bh));
/*
* Only clear out a write error when rewriting
*/
if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
clear_buffer_write_io_error(bh);
if (buffer_meta(bh))
opf |= REQ_META;
if (buffer_prio(bh))
opf |= REQ_PRIO;
bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
bio->bi_write_hint = write_hint;
__bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
bio->bi_end_io = end_bio_bh_io_sync;
bio->bi_private = bh;
/* Take care of bh's that straddle the end of the device */
guard_bio_eod(bio);
if (wbc) {
wbc_init_bio(wbc, bio);
wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
}
submit_bio(bio);
}
void submit_bh(blk_opf_t opf, struct buffer_head *bh)
{
submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL);
}
EXPORT_SYMBOL(submit_bh);
void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
{
lock_buffer(bh);
if (!test_clear_buffer_dirty(bh)) {
unlock_buffer(bh);
return;
}
bh->b_end_io = end_buffer_write_sync;
get_bh(bh);
submit_bh(REQ_OP_WRITE | op_flags, bh);
}
EXPORT_SYMBOL(write_dirty_buffer);
/*
* For a data-integrity writeout, we need to wait upon any in-progress I/O
* and then start new I/O and then wait upon it. The caller must have a ref on
* the buffer_head.
*/
int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
{
WARN_ON(atomic_read(&bh->b_count) < 1);
lock_buffer(bh);
if (test_clear_buffer_dirty(bh)) {
/*
* The bh should be mapped, but it might not be if the
* device was hot-removed. Not much we can do but fail the I/O.
*/
if (!buffer_mapped(bh)) {
unlock_buffer(bh);
return -EIO;
}
get_bh(bh);
bh->b_end_io = end_buffer_write_sync;
submit_bh(REQ_OP_WRITE | op_flags, bh);
wait_on_buffer(bh);
if (!buffer_uptodate(bh))
return -EIO;
} else {
unlock_buffer(bh);
}
return 0;
}
EXPORT_SYMBOL(__sync_dirty_buffer);
int sync_dirty_buffer(struct buffer_head *bh)
{
return __sync_dirty_buffer(bh, REQ_SYNC);
}
EXPORT_SYMBOL(sync_dirty_buffer);
/*
* try_to_free_buffers() checks if all the buffers on this particular folio
* are unused, and releases them if so.
*
* Exclusion against try_to_free_buffers may be obtained by either
* locking the folio or by holding its mapping's i_private_lock.
*
* If the folio is dirty but all the buffers are clean then we need to
* be sure to mark the folio clean as well. This is because the folio
* may be against a block device, and a later reattachment of buffers
* to a dirty folio will set *all* buffers dirty. Which would corrupt
* filesystem data on the same device.
*
* The same applies to regular filesystem folios: if all the buffers are
* clean then we set the folio clean and proceed. To do that, we require
* total exclusion from block_dirty_folio(). That is obtained with
* i_private_lock.
*
* try_to_free_buffers() is non-blocking.
*/
static inline int buffer_busy(struct buffer_head *bh)
{
return atomic_read(&bh->b_count) |
(bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
}
static bool
drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free)
{
struct buffer_head *head = folio_buffers(folio);
struct buffer_head *bh;
bh = head;
do {
if (buffer_busy(bh))
goto failed;
bh = bh->b_this_page;
} while (bh != head);
do {
struct buffer_head *next = bh->b_this_page;
if (bh->b_assoc_map)
__remove_assoc_queue(bh);
bh = next;
} while (bh != head);
*buffers_to_free = head;
folio_detach_private(folio);
return true;
failed:
return false;
}
bool try_to_free_buffers(struct folio *folio)
{
struct address_space * const mapping = folio->mapping;
struct buffer_head *buffers_to_free = NULL;
bool ret = 0;
BUG_ON(!folio_test_locked(folio));
if (folio_test_writeback(folio))
return false;
if (mapping == NULL) { /* can this still happen? */
ret = drop_buffers(folio, &buffers_to_free);
goto out;
}
spin_lock(&mapping->i_private_lock);
ret = drop_buffers(folio, &buffers_to_free);
/*
* If the filesystem writes its buffers by hand (eg ext3)
* then we can have clean buffers against a dirty folio. We
* clean the folio here; otherwise the VM will never notice
* that the filesystem did any IO at all.
*
* Also, during truncate, discard_buffer will have marked all
* the folio's buffers clean. We discover that here and clean
* the folio also.
*
* i_private_lock must be held over this entire operation in order
* to synchronise against block_dirty_folio and prevent the
* dirty bit from being lost.
*/
if (ret)
folio_cancel_dirty(folio);
spin_unlock(&mapping->i_private_lock);
out:
if (buffers_to_free) {
struct buffer_head *bh = buffers_to_free;
do {
struct buffer_head *next = bh->b_this_page;
free_buffer_head(bh);
bh = next;
} while (bh != buffers_to_free);
}
return ret;
}
EXPORT_SYMBOL(try_to_free_buffers);
/*
* Buffer-head allocation
*/
static struct kmem_cache *bh_cachep __ro_after_init;
/*
* Once the number of bh's in the machine exceeds this level, we start
* stripping them in writeback.
*/
static unsigned long max_buffer_heads __ro_after_init;
int buffer_heads_over_limit;
struct bh_accounting {
int nr; /* Number of live bh's */
int ratelimit; /* Limit cacheline bouncing */
};
static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
static void recalc_bh_state(void)
{
int i;
int tot = 0;
if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
return;
__this_cpu_write(bh_accounting.ratelimit, 0);
for_each_online_cpu(i)
tot += per_cpu(bh_accounting, i).nr;
buffer_heads_over_limit = (tot > max_buffer_heads);
}
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
{
struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
if (ret) {
INIT_LIST_HEAD(&ret->b_assoc_buffers);
spin_lock_init(&ret->b_uptodate_lock);
preempt_disable();
__this_cpu_inc(bh_accounting.nr);
recalc_bh_state();
preempt_enable();
}
return ret;
}
EXPORT_SYMBOL(alloc_buffer_head);
void free_buffer_head(struct buffer_head *bh)
{
BUG_ON(!list_empty(&bh->b_assoc_buffers));
kmem_cache_free(bh_cachep, bh);
preempt_disable();
__this_cpu_dec(bh_accounting.nr);
recalc_bh_state();
preempt_enable();
}
EXPORT_SYMBOL(free_buffer_head);
static int buffer_exit_cpu_dead(unsigned int cpu)
{
int i;
struct bh_lru *b = &per_cpu(bh_lrus, cpu);
for (i = 0; i < BH_LRU_SIZE; i++) {
brelse(b->bhs[i]);
b->bhs[i] = NULL;
}
this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
per_cpu(bh_accounting, cpu).nr = 0;
return 0;
}
/**
* bh_uptodate_or_lock - Test whether the buffer is uptodate
* @bh: struct buffer_head
*
* Return true if the buffer is up-to-date and false,
* with the buffer locked, if not.
*/
int bh_uptodate_or_lock(struct buffer_head *bh)
{
if (!buffer_uptodate(bh)) {
lock_buffer(bh);
if (!buffer_uptodate(bh))
return 0;
unlock_buffer(bh);
}
return 1;
}
EXPORT_SYMBOL(bh_uptodate_or_lock);
/**
* __bh_read - Submit read for a locked buffer
* @bh: struct buffer_head
* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
* @wait: wait until reading finish
*
* Returns zero on success or don't wait, and -EIO on error.
*/
int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
{
int ret = 0;
BUG_ON(!buffer_locked(bh));
get_bh(bh);
bh->b_end_io = end_buffer_read_sync;
submit_bh(REQ_OP_READ | op_flags, bh);
if (wait) {
wait_on_buffer(bh);
if (!buffer_uptodate(bh))
ret = -EIO;
}
return ret;
}
EXPORT_SYMBOL(__bh_read);
/**
* __bh_read_batch - Submit read for a batch of unlocked buffers
* @nr: entry number of the buffer batch
* @bhs: a batch of struct buffer_head
* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
* @force_lock: force to get a lock on the buffer if set, otherwise drops any
* buffer that cannot lock.
*
* Returns zero on success or don't wait, and -EIO on error.
*/
void __bh_read_batch(int nr, struct buffer_head *bhs[],
blk_opf_t op_flags, bool force_lock)
{
int i;
for (i = 0; i < nr; i++) {
struct buffer_head *bh = bhs[i];
if (buffer_uptodate(bh))
continue;
if (force_lock)
lock_buffer(bh);
else
if (!trylock_buffer(bh))
continue;
if (buffer_uptodate(bh)) {
unlock_buffer(bh);
continue;
}
bh->b_end_io = end_buffer_read_sync;
get_bh(bh);
submit_bh(REQ_OP_READ | op_flags, bh);
}
}
EXPORT_SYMBOL(__bh_read_batch);
void __init buffer_init(void)
{
unsigned long nrpages;
int ret;
bh_cachep = KMEM_CACHE(buffer_head,
SLAB_RECLAIM_ACCOUNT|SLAB_PANIC);
/*
* Limit the bh occupancy to 10% of ZONE_NORMAL
*/
nrpages = (nr_free_buffer_pages() * 10) / 100;
max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
NULL, buffer_exit_cpu_dead);
WARN_ON(ret < 0);
}
// SPDX-License-Identifier: GPL-2.0
/*
* Block multiqueue core code
*
* Copyright (C) 2013-2014 Jens Axboe
* Copyright (C) 2013-2014 Christoph Hellwig
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/blk-integrity.h>
#include <linux/kmemleak.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/llist.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/topology.h>
#include <linux/sched/signal.h>
#include <linux/delay.h>
#include <linux/crash_dump.h>
#include <linux/prefetch.h>
#include <linux/blk-crypto.h>
#include <linux/part_stat.h>
#include <trace/events/block.h>
#include <linux/t10-pi.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-pm.h"
#include "blk-stat.h"
#include "blk-mq-sched.h"
#include "blk-rq-qos.h"
static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
static void blk_mq_request_bypass_insert(struct request *rq,
blk_insert_t flags);
static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
struct list_head *list);
static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
struct io_comp_batch *iob, unsigned int flags);
/*
* Check if any of the ctx, dispatch list or elevator
* have pending work in this hardware queue.
*/
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
return !list_empty_careful(&hctx->dispatch) ||
sbitmap_any_bit_set(&hctx->ctx_map) ||
blk_mq_sched_has_work(hctx);
}
/*
* Mark this ctx as having pending work in this hardware queue
*/
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
struct blk_mq_ctx *ctx)
{
const int bit = ctx->index_hw[hctx->type];
if (!sbitmap_test_bit(&hctx->ctx_map, bit))
sbitmap_set_bit(&hctx->ctx_map, bit);
}
static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
struct blk_mq_ctx *ctx)
{
const int bit = ctx->index_hw[hctx->type];
sbitmap_clear_bit(&hctx->ctx_map, bit);
}
struct mq_inflight {
struct block_device *part;
unsigned int inflight[2];
};
static bool blk_mq_check_inflight(struct request *rq, void *priv)
{
struct mq_inflight *mi = priv;
if (rq->part && blk_do_io_stat(rq) &&
(!mi->part->bd_partno || rq->part == mi->part) &&
blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
mi->inflight[rq_data_dir(rq)]++;
return true;
}
unsigned int blk_mq_in_flight(struct request_queue *q,
struct block_device *part)
{
struct mq_inflight mi = { .part = part };
blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
return mi.inflight[0] + mi.inflight[1];
}
void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
unsigned int inflight[2])
{
struct mq_inflight mi = { .part = part };
blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
inflight[0] = mi.inflight[0];
inflight[1] = mi.inflight[1];
}
void blk_freeze_queue_start(struct request_queue *q)
{
mutex_lock(&q->mq_freeze_lock);
if (++q->mq_freeze_depth == 1) {
percpu_ref_kill(&q->q_usage_counter);
mutex_unlock(&q->mq_freeze_lock);
if (queue_is_mq(q))
blk_mq_run_hw_queues(q, false);
} else {
mutex_unlock(&q->mq_freeze_lock);
}
}
EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
void blk_mq_freeze_queue_wait(struct request_queue *q)
{
wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
unsigned long timeout)
{
return wait_event_timeout(q->mq_freeze_wq,
percpu_ref_is_zero(&q->q_usage_counter),
timeout);
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
/*
* Guarantee no request is in use, so we can change any data structure of
* the queue afterward.
*/
void blk_freeze_queue(struct request_queue *q)
{
/*
* In the !blk_mq case we are only calling this to kill the
* q_usage_counter, otherwise this increases the freeze depth
* and waits for it to return to zero. For this reason there is
* no blk_unfreeze_queue(), and blk_freeze_queue() is not
* exported to drivers as the only user for unfreeze is blk_mq.
*/
blk_freeze_queue_start(q);
blk_mq_freeze_queue_wait(q);
}
void blk_mq_freeze_queue(struct request_queue *q)
{
/*
* ...just an alias to keep freeze and unfreeze actions balanced
* in the blk_mq_* namespace
*/
blk_freeze_queue(q);
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
{
mutex_lock(&q->mq_freeze_lock);
if (force_atomic)
q->q_usage_counter.data->force_atomic = true;
q->mq_freeze_depth--;
WARN_ON_ONCE(q->mq_freeze_depth < 0);
if (!q->mq_freeze_depth) {
percpu_ref_resurrect(&q->q_usage_counter);
wake_up_all(&q->mq_freeze_wq);
}
mutex_unlock(&q->mq_freeze_lock);
}
void blk_mq_unfreeze_queue(struct request_queue *q)
{
__blk_mq_unfreeze_queue(q, false);
}
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
/*
* FIXME: replace the scsi_internal_device_*block_nowait() calls in the
* mpt3sas driver such that this function can be removed.
*/
void blk_mq_quiesce_queue_nowait(struct request_queue *q)
{
unsigned long flags;
spin_lock_irqsave(&q->queue_lock, flags);
if (!q->quiesce_depth++)
blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
spin_unlock_irqrestore(&q->queue_lock, flags);
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
/**
* blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
* @set: tag_set to wait on
*
* Note: it is driver's responsibility for making sure that quiesce has
* been started on or more of the request_queues of the tag_set. This
* function only waits for the quiesce on those request_queues that had
* the quiesce flag set using blk_mq_quiesce_queue_nowait.
*/
void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
{
if (set->flags & BLK_MQ_F_BLOCKING)
synchronize_srcu(set->srcu);
else
synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
/**
* blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
* @q: request queue.
*
* Note: this function does not prevent that the struct request end_io()
* callback function is invoked. Once this function is returned, we make
* sure no dispatch can happen until the queue is unquiesced via
* blk_mq_unquiesce_queue().
*/
void blk_mq_quiesce_queue(struct request_queue *q)
{
blk_mq_quiesce_queue_nowait(q);
/* nothing to wait for non-mq queues */
if (queue_is_mq(q))
blk_mq_wait_quiesce_done(q->tag_set);
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
/*
* blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
* @q: request queue.
*
* This function recovers queue into the state before quiescing
* which is done by blk_mq_quiesce_queue.
*/
void blk_mq_unquiesce_queue(struct request_queue *q)
{
unsigned long flags;
bool run_queue = false;
spin_lock_irqsave(&q->queue_lock, flags);
if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
;
} else if (!--q->quiesce_depth) {
blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
run_queue = true;
}
spin_unlock_irqrestore(&q->queue_lock, flags);
/* dispatch requests which are inserted during quiescing */
if (run_queue)
blk_mq_run_hw_queues(q, true);
}
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
{
struct request_queue *q;
mutex_lock(&set->tag_list_lock);
list_for_each_entry(q, &set->tag_list, tag_set_list) {
if (!blk_queue_skip_tagset_quiesce(q))
blk_mq_quiesce_queue_nowait(q);
}
blk_mq_wait_quiesce_done(set);
mutex_unlock(&set->tag_list_lock);
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
{
struct request_queue *q;
mutex_lock(&set->tag_list_lock);
list_for_each_entry(q, &set->tag_list, tag_set_list) {
if (!blk_queue_skip_tagset_quiesce(q))
blk_mq_unquiesce_queue(q);
}
mutex_unlock(&set->tag_list_lock);
}
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
void blk_mq_wake_waiters(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i)
if (blk_mq_hw_queue_mapped(hctx))
blk_mq_tag_wakeup_all(hctx->tags, true);
}
void blk_rq_init(struct request_queue *q, struct request *rq)
{
memset(rq, 0, sizeof(*rq));
INIT_LIST_HEAD(&rq->queuelist);
rq->q = q;
rq->__sector = (sector_t) -1;
INIT_HLIST_NODE(&rq->hash);
RB_CLEAR_NODE(&rq->rb_node);
rq->tag = BLK_MQ_NO_TAG;
rq->internal_tag = BLK_MQ_NO_TAG;
rq->start_time_ns = blk_time_get_ns();
rq->part = NULL;
blk_crypto_rq_set_defaults(rq);
}
EXPORT_SYMBOL(blk_rq_init);
/* Set start and alloc time when the allocated request is actually used */
static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
{
if (blk_mq_need_time_stamp(rq))
rq->start_time_ns = blk_time_get_ns();
else
rq->start_time_ns = 0;
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
if (blk_queue_rq_alloc_time(rq->q))
rq->alloc_time_ns = alloc_time_ns ?: rq->start_time_ns;
else
rq->alloc_time_ns = 0;
#endif
}
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
struct blk_mq_tags *tags, unsigned int tag)
{
struct blk_mq_ctx *ctx = data->ctx;
struct blk_mq_hw_ctx *hctx = data->hctx;
struct request_queue *q = data->q;
struct request *rq = tags->static_rqs[tag];
rq->q = q;
rq->mq_ctx = ctx;
rq->mq_hctx = hctx;
rq->cmd_flags = data->cmd_flags;
if (data->flags & BLK_MQ_REQ_PM)
data->rq_flags |= RQF_PM;
if (blk_queue_io_stat(q))
data->rq_flags |= RQF_IO_STAT;
rq->rq_flags = data->rq_flags;
if (data->rq_flags & RQF_SCHED_TAGS) {
rq->tag = BLK_MQ_NO_TAG;
rq->internal_tag = tag;
} else {
rq->tag = tag;
rq->internal_tag = BLK_MQ_NO_TAG;
}
rq->timeout = 0;
rq->part = NULL;
rq->io_start_time_ns = 0;
rq->stats_sectors = 0;
rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
rq->nr_integrity_segments = 0;
#endif
rq->end_io = NULL;
rq->end_io_data = NULL;
blk_crypto_rq_set_defaults(rq);
INIT_LIST_HEAD(&rq->queuelist);
/* tag was already set */
WRITE_ONCE(rq->deadline, 0);
req_ref_set(rq, 1);
if (rq->rq_flags & RQF_USE_SCHED) {
struct elevator_queue *e = data->q->elevator;
INIT_HLIST_NODE(&rq->hash);
RB_CLEAR_NODE(&rq->rb_node);
if (e->type->ops.prepare_request)
e->type->ops.prepare_request(rq);
}
return rq;
}
static inline struct request *
__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
{
unsigned int tag, tag_offset;
struct blk_mq_tags *tags;
struct request *rq;
unsigned long tag_mask;
int i, nr = 0;
tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset);
if (unlikely(!tag_mask))
return NULL;
tags = blk_mq_tags_from_data(data);
for (i = 0; tag_mask; i++) {
if (!(tag_mask & (1UL << i)))
continue;
tag = tag_offset + i;
prefetch(tags->static_rqs[tag]);
tag_mask &= ~(1UL << i);
rq = blk_mq_rq_ctx_init(data, tags, tag);
rq_list_add(data->cached_rq, rq);
nr++;
}
if (!(data->rq_flags & RQF_SCHED_TAGS))
blk_mq_add_active_requests(data->hctx, nr);
/* caller already holds a reference, add for remainder */
percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
data->nr_tags -= nr;
return rq_list_pop(data->cached_rq);
}
static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
{
struct request_queue *q = data->q;
u64 alloc_time_ns = 0;
struct request *rq;
unsigned int tag;
/* alloc_time includes depth and tag waits */
if (blk_queue_rq_alloc_time(q))
alloc_time_ns = blk_time_get_ns();
if (data->cmd_flags & REQ_NOWAIT)
data->flags |= BLK_MQ_REQ_NOWAIT;
if (q->elevator) {
/*
* All requests use scheduler tags when an I/O scheduler is
* enabled for the queue.
*/
data->rq_flags |= RQF_SCHED_TAGS;
/*
* Flush/passthrough requests are special and go directly to the
* dispatch list.
*/
if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH &&
!blk_op_is_passthrough(data->cmd_flags)) {
struct elevator_mq_ops *ops = &q->elevator->type->ops;
WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
data->rq_flags |= RQF_USE_SCHED;
if (ops->limit_depth)
ops->limit_depth(data->cmd_flags, data);
}
}
retry:
data->ctx = blk_mq_get_ctx(q);
data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
if (!(data->rq_flags & RQF_SCHED_TAGS))
blk_mq_tag_busy(data->hctx);
if (data->flags & BLK_MQ_REQ_RESERVED)
data->rq_flags |= RQF_RESV;
/*
* Try batched alloc if we want more than 1 tag.
*/
if (data->nr_tags > 1) {
rq = __blk_mq_alloc_requests_batch(data);
if (rq) {
blk_mq_rq_time_init(rq, alloc_time_ns);
return rq;
}
data->nr_tags = 1;
}
/*
* Waiting allocations only fail because of an inactive hctx. In that
* case just retry the hctx assignment and tag allocation as CPU hotplug
* should have migrated us to an online CPU by now.
*/
tag = blk_mq_get_tag(data);
if (tag == BLK_MQ_NO_TAG) {
if (data->flags & BLK_MQ_REQ_NOWAIT)
return NULL;
/*
* Give up the CPU and sleep for a random short time to
* ensure that thread using a realtime scheduling class
* are migrated off the CPU, and thus off the hctx that
* is going away.
*/
msleep(3);
goto retry;
}
if (!(data->rq_flags & RQF_SCHED_TAGS))
blk_mq_inc_active_requests(data->hctx);
rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
blk_mq_rq_time_init(rq, alloc_time_ns);
return rq;
}
static struct request *blk_mq_rq_cache_fill(struct request_queue *q,
struct blk_plug *plug,
blk_opf_t opf,
blk_mq_req_flags_t flags)
{
struct blk_mq_alloc_data data = {
.q = q,
.flags = flags,
.cmd_flags = opf,
.nr_tags = plug->nr_ios,
.cached_rq = &plug->cached_rq,
};
struct request *rq;
if (blk_queue_enter(q, flags))
return NULL;
plug->nr_ios = 1;
rq = __blk_mq_alloc_requests(&data);
if (unlikely(!rq))
blk_queue_exit(q);
return rq;
}
static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
blk_opf_t opf,
blk_mq_req_flags_t flags)
{
struct blk_plug *plug = current->plug;
struct request *rq;
if (!plug)
return NULL;
if (rq_list_empty(plug->cached_rq)) {
if (plug->nr_ios == 1)
return NULL;
rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
if (!rq)
return NULL;
} else {
rq = rq_list_peek(&plug->cached_rq);
if (!rq || rq->q != q)
return NULL;
if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
return NULL;
if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
return NULL;
plug->cached_rq = rq_list_next(rq);
blk_mq_rq_time_init(rq, 0);
}
rq->cmd_flags = opf;
INIT_LIST_HEAD(&rq->queuelist);
return rq;
}
struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
blk_mq_req_flags_t flags)
{
struct request *rq;
rq = blk_mq_alloc_cached_request(q, opf, flags);
if (!rq) {
struct blk_mq_alloc_data data = {
.q = q,
.flags = flags,
.cmd_flags = opf,
.nr_tags = 1,
};
int ret;
ret = blk_queue_enter(q, flags);
if (ret)
return ERR_PTR(ret);
rq = __blk_mq_alloc_requests(&data);
if (!rq)
goto out_queue_exit;
}
rq->__data_len = 0;
rq->__sector = (sector_t) -1;
rq->bio = rq->biotail = NULL;
return rq;
out_queue_exit:
blk_queue_exit(q);
return ERR_PTR(-EWOULDBLOCK);
}
EXPORT_SYMBOL(blk_mq_alloc_request);
struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
{
struct blk_mq_alloc_data data = {
.q = q,
.flags = flags,
.cmd_flags = opf,
.nr_tags = 1,
};
u64 alloc_time_ns = 0;
struct request *rq;
unsigned int cpu;
unsigned int tag;
int ret;
/* alloc_time includes depth and tag waits */
if (blk_queue_rq_alloc_time(q))
alloc_time_ns = blk_time_get_ns();
/*
* If the tag allocator sleeps we could get an allocation for a
* different hardware context. No need to complicate the low level
* allocator for this for the rare use case of a command tied to
* a specific queue.
*/
if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) ||
WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
return ERR_PTR(-EINVAL);
if (hctx_idx >= q->nr_hw_queues)
return ERR_PTR(-EIO);
ret = blk_queue_enter(q, flags);
if (ret)
return ERR_PTR(ret);
/*
* Check if the hardware context is actually mapped to anything.
* If not tell the caller that it should skip this queue.
*/
ret = -EXDEV;
data.hctx = xa_load(&q->hctx_table, hctx_idx);
if (!blk_mq_hw_queue_mapped(data.hctx))
goto out_queue_exit;
cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
if (cpu >= nr_cpu_ids)
goto out_queue_exit;
data.ctx = __blk_mq_get_ctx(q, cpu);
if (q->elevator)
data.rq_flags |= RQF_SCHED_TAGS;
else
blk_mq_tag_busy(data.hctx);
if (flags & BLK_MQ_REQ_RESERVED)
data.rq_flags |= RQF_RESV;
ret = -EWOULDBLOCK;
tag = blk_mq_get_tag(&data);
if (tag == BLK_MQ_NO_TAG)
goto out_queue_exit;
if (!(data.rq_flags & RQF_SCHED_TAGS))
blk_mq_inc_active_requests(data.hctx);
rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
blk_mq_rq_time_init(rq, alloc_time_ns);
rq->__data_len = 0;
rq->__sector = (sector_t) -1;
rq->bio = rq->biotail = NULL;
return rq;
out_queue_exit:
blk_queue_exit(q);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
static void blk_mq_finish_request(struct request *rq)
{
struct request_queue *q = rq->q;
if (rq->rq_flags & RQF_USE_SCHED) {
q->elevator->type->ops.finish_request(rq);
/*
* For postflush request that may need to be
* completed twice, we should clear this flag
* to avoid double finish_request() on the rq.
*/
rq->rq_flags &= ~RQF_USE_SCHED;
}
}
static void __blk_mq_free_request(struct request *rq)
{
struct request_queue *q = rq->q;
struct blk_mq_ctx *ctx = rq->mq_ctx;
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
const int sched_tag = rq->internal_tag;
blk_crypto_free_request(rq);
blk_pm_mark_last_busy(rq);
rq->mq_hctx = NULL;
if (rq->tag != BLK_MQ_NO_TAG) {
blk_mq_dec_active_requests(hctx);
blk_mq_put_tag(hctx->tags, ctx, rq->tag);
}
if (sched_tag != BLK_MQ_NO_TAG)
blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
blk_mq_sched_restart(hctx);
blk_queue_exit(q);
}
void blk_mq_free_request(struct request *rq)
{
struct request_queue *q = rq->q;
blk_mq_finish_request(rq);
if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
laptop_io_completion(q->disk->bdi);
rq_qos_done(q, rq);
WRITE_ONCE(rq->state, MQ_RQ_IDLE);
if (req_ref_put_and_test(rq))
__blk_mq_free_request(rq);
}
EXPORT_SYMBOL_GPL(blk_mq_free_request);
void blk_mq_free_plug_rqs(struct blk_plug *plug)
{
struct request *rq;
while ((rq = rq_list_pop(&plug->cached_rq)) != NULL)
blk_mq_free_request(rq);
}
void blk_dump_rq_flags(struct request *rq, char *msg)
{
printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
rq->q->disk ? rq->q->disk->disk_name : "?",
(__force unsigned long long) rq->cmd_flags);
printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
(unsigned long long)blk_rq_pos(rq),
blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
printk(KERN_INFO " bio %p, biotail %p, len %u\n",
rq->bio, rq->biotail, blk_rq_bytes(rq));
}
EXPORT_SYMBOL(blk_dump_rq_flags);
static void req_bio_endio(struct request *rq, struct bio *bio,
unsigned int nbytes, blk_status_t error)
{
if (unlikely(error)) {
bio->bi_status = error;
} else if (req_op(rq) == REQ_OP_ZONE_APPEND) {
/*
* Partial zone append completions cannot be supported as the
* BIO fragments may end up not being written sequentially.
*/
if (bio->bi_iter.bi_size != nbytes)
bio->bi_status = BLK_STS_IOERR;
else
bio->bi_iter.bi_sector = rq->__sector;
}
bio_advance(bio, nbytes);
if (unlikely(rq->rq_flags & RQF_QUIET))
bio_set_flag(bio, BIO_QUIET);
/* don't actually finish bio if it's part of flush sequence */
if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
bio_endio(bio);
}
static void blk_account_io_completion(struct request *req, unsigned int bytes)
{
if (req->part && blk_do_io_stat(req)) {
const int sgrp = op_stat_group(req_op(req));
part_stat_lock();
part_stat_add(req->part, sectors[sgrp], bytes >> 9);
part_stat_unlock();
}
}
static void blk_print_req_error(struct request *req, blk_status_t status)
{
printk_ratelimited(KERN_ERR
"%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
"phys_seg %u prio class %u\n",
blk_status_to_str(status),
req->q->disk ? req->q->disk->disk_name : "?",
blk_rq_pos(req), (__force u32)req_op(req),
blk_op_str(req_op(req)),
(__force u32)(req->cmd_flags & ~REQ_OP_MASK),
req->nr_phys_segments,
IOPRIO_PRIO_CLASS(req->ioprio));
}
/*
* Fully end IO on a request. Does not support partial completions, or
* errors.
*/
static void blk_complete_request(struct request *req)
{
const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
int total_bytes = blk_rq_bytes(req);
struct bio *bio = req->bio;
trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
if (!bio)
return;
#ifdef CONFIG_BLK_DEV_INTEGRITY
if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
req->q->integrity.profile->complete_fn(req, total_bytes);
#endif
/*
* Upper layers may call blk_crypto_evict_key() anytime after the last
* bio_endio(). Therefore, the keyslot must be released before that.
*/
blk_crypto_rq_put_keyslot(req);
blk_account_io_completion(req, total_bytes);
do {
struct bio *next = bio->bi_next;
/* Completion has already been traced */
bio_clear_flag(bio, BIO_TRACE_COMPLETION);
if (req_op(req) == REQ_OP_ZONE_APPEND)
bio->bi_iter.bi_sector = req->__sector;
if (!is_flush)
bio_endio(bio);
bio = next;
} while (bio);
/*
* Reset counters so that the request stacking driver
* can find how many bytes remain in the request
* later.
*/
if (!req->end_io) {
req->bio = NULL;
req->__data_len = 0;
}
}
/**
* blk_update_request - Complete multiple bytes without completing the request
* @req: the request being processed
* @error: block status code
* @nr_bytes: number of bytes to complete for @req
*
* Description:
* Ends I/O on a number of bytes attached to @req, but doesn't complete
* the request structure even if @req doesn't have leftover.
* If @req has leftover, sets it up for the next range of segments.
*
* Passing the result of blk_rq_bytes() as @nr_bytes guarantees
* %false return from this function.
*
* Note:
* The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
* except in the consistency check at the end of this function.
*
* Return:
* %false - this request doesn't have any more data
* %true - this request has more data
**/
bool blk_update_request(struct request *req, blk_status_t error,
unsigned int nr_bytes)
{
int total_bytes;
trace_block_rq_complete(req, error, nr_bytes);
if (!req->bio)
return false;
#ifdef CONFIG_BLK_DEV_INTEGRITY
if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
error == BLK_STS_OK)
req->q->integrity.profile->complete_fn(req, nr_bytes);
#endif
/*
* Upper layers may call blk_crypto_evict_key() anytime after the last
* bio_endio(). Therefore, the keyslot must be released before that.
*/
if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
__blk_crypto_rq_put_keyslot(req);
if (unlikely(error && !blk_rq_is_passthrough(req) &&
!(req->rq_flags & RQF_QUIET)) &&
!test_bit(GD_DEAD, &req->q->disk->state)) {
blk_print_req_error(req, error);
trace_block_rq_error(req, error, nr_bytes);
}
blk_account_io_completion(req, nr_bytes);
total_bytes = 0;
while (req->bio) {
struct bio *bio = req->bio;
unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
if (bio_bytes == bio->bi_iter.bi_size)
req->bio = bio->bi_next;
/* Completion has already been traced */
bio_clear_flag(bio, BIO_TRACE_COMPLETION);
req_bio_endio(req, bio, bio_bytes, error);
total_bytes += bio_bytes;
nr_bytes -= bio_bytes;
if (!nr_bytes)
break;
}
/*
* completely done
*/
if (!req->bio) {
/*
* Reset counters so that the request stacking driver
* can find how many bytes remain in the request
* later.
*/
req->__data_len = 0;
return false;
}
req->__data_len -= total_bytes;
/* update sector only for requests with clear definition of sector */
if (!blk_rq_is_passthrough(req))
req->__sector += total_bytes >> 9;
/* mixed attributes always follow the first bio */
if (req->rq_flags & RQF_MIXED_MERGE) {
req->cmd_flags &= ~REQ_FAILFAST_MASK;
req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
}
if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
/*
* If total number of sectors is less than the first segment
* size, something has gone terribly wrong.
*/
if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
blk_dump_rq_flags(req, "request botched");
req->__data_len = blk_rq_cur_bytes(req);
}
/* recalculate the number of segments */
req->nr_phys_segments = blk_recalc_rq_segments(req);
}
return true;
}
EXPORT_SYMBOL_GPL(blk_update_request);
static inline void blk_account_io_done(struct request *req, u64 now)
{
trace_block_io_done(req);
/*
* Account IO completion. flush_rq isn't accounted as a
* normal IO on queueing nor completion. Accounting the
* containing request is enough.
*/
if (blk_do_io_stat(req) && req->part &&
!(req->rq_flags & RQF_FLUSH_SEQ)) {
const int sgrp = op_stat_group(req_op(req));
part_stat_lock();
update_io_ticks(req->part, jiffies, true);
part_stat_inc(req->part, ios[sgrp]);
part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
part_stat_unlock();
}
}
static inline void blk_account_io_start(struct request *req)
{
trace_block_io_start(req);
if (blk_do_io_stat(req)) {
/*
* All non-passthrough requests are created from a bio with one
* exception: when a flush command that is part of a flush sequence
* generated by the state machine in blk-flush.c is cloned onto the
* lower device by dm-multipath we can get here without a bio.
*/
if (req->bio)
req->part = req->bio->bi_bdev;
else
req->part = req->q->disk->part0;
part_stat_lock();
update_io_ticks(req->part, jiffies, false);
part_stat_unlock();
}
}
static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
{
if (rq->rq_flags & RQF_STATS)
blk_stat_add(rq, now);
blk_mq_sched_completed_request(rq, now);
blk_account_io_done(rq, now);
}
inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
{
if (blk_mq_need_time_stamp(rq))
__blk_mq_end_request_acct(rq, blk_time_get_ns());
blk_mq_finish_request(rq);
if (rq->end_io) {
rq_qos_done(rq->q, rq);
if (rq->end_io(rq, error) == RQ_END_IO_FREE)
blk_mq_free_request(rq);
} else {
blk_mq_free_request(rq);
}
}
EXPORT_SYMBOL(__blk_mq_end_request);
void blk_mq_end_request(struct request *rq, blk_status_t error)
{
if (blk_update_request(rq, error, blk_rq_bytes(rq)))
BUG();
__blk_mq_end_request(rq, error);
}
EXPORT_SYMBOL(blk_mq_end_request);
#define TAG_COMP_BATCH 32
static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
int *tag_array, int nr_tags)
{
struct request_queue *q = hctx->queue;
blk_mq_sub_active_requests(hctx, nr_tags);
blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
percpu_ref_put_many(&q->q_usage_counter, nr_tags);
}
void blk_mq_end_request_batch(struct io_comp_batch *iob)
{
int tags[TAG_COMP_BATCH], nr_tags = 0;
struct blk_mq_hw_ctx *cur_hctx = NULL;
struct request *rq;
u64 now = 0;
if (iob->need_ts)
now = blk_time_get_ns();
while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
prefetch(rq->bio);
prefetch(rq->rq_next);
blk_complete_request(rq);
if (iob->need_ts)
__blk_mq_end_request_acct(rq, now);
blk_mq_finish_request(rq);
rq_qos_done(rq->q, rq);
/*
* If end_io handler returns NONE, then it still has
* ownership of the request.
*/
if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE)
continue;
WRITE_ONCE(rq->state, MQ_RQ_IDLE);
if (!req_ref_put_and_test(rq))
continue;
blk_crypto_free_request(rq);
blk_pm_mark_last_busy(rq);
if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) {
if (cur_hctx)
blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
nr_tags = 0;
cur_hctx = rq->mq_hctx;
}
tags[nr_tags++] = rq->tag;
}
if (nr_tags)
blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
}
EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
static void blk_complete_reqs(struct llist_head *list)
{
struct llist_node *entry = llist_reverse_order(llist_del_all(list));
struct request *rq, *next;
llist_for_each_entry_safe(rq, next, entry, ipi_list)
rq->q->mq_ops->complete(rq);
}
static __latent_entropy void blk_done_softirq(struct softirq_action *h)
{
blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
}
static int blk_softirq_cpu_dead(unsigned int cpu)
{
blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
return 0;
}
static void __blk_mq_complete_request_remote(void *data)
{
__raise_softirq_irqoff(BLOCK_SOFTIRQ);
}
static inline bool blk_mq_complete_need_ipi(struct request *rq)
{
int cpu = raw_smp_processor_id();
if (!IS_ENABLED(CONFIG_SMP) ||
!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
return false;
/*
* With force threaded interrupts enabled, raising softirq from an SMP
* function call will always result in waking the ksoftirqd thread.
* This is probably worse than completing the request on a different
* cache domain.
*/
if (force_irqthreads())
return false;
/* same CPU or cache domain and capacity? Complete locally */
if (cpu == rq->mq_ctx->cpu ||
(!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
cpus_share_cache(cpu, rq->mq_ctx->cpu) &&
cpus_equal_capacity(cpu, rq->mq_ctx->cpu)))
return false;
/* don't try to IPI to an offline CPU */
return cpu_online(rq->mq_ctx->cpu);
}
static void blk_mq_complete_send_ipi(struct request *rq)
{
unsigned int cpu;
cpu = rq->mq_ctx->cpu;
if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
}
static void blk_mq_raise_softirq(struct request *rq)
{
struct llist_head *list;
preempt_disable();
list = this_cpu_ptr(&blk_cpu_done);
if (llist_add(&rq->ipi_list, list))
raise_softirq(BLOCK_SOFTIRQ);
preempt_enable();
}
bool blk_mq_complete_request_remote(struct request *rq)
{
WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
/*
* For request which hctx has only one ctx mapping,
* or a polled request, always complete locally,
* it's pointless to redirect the completion.
*/
if ((rq->mq_hctx->nr_ctx == 1 &&
rq->mq_ctx->cpu == raw_smp_processor_id()) ||
rq->cmd_flags & REQ_POLLED)
return false;
if (blk_mq_complete_need_ipi(rq)) {
blk_mq_complete_send_ipi(rq);
return true;
}
if (rq->q->nr_hw_queues == 1) {
blk_mq_raise_softirq(rq);
return true;
}
return false;
}
EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
/**
* blk_mq_complete_request - end I/O on a request
* @rq: the request being processed
*
* Description:
* Complete a request by scheduling the ->complete_rq operation.
**/
void blk_mq_complete_request(struct request *rq)
{
if (!blk_mq_complete_request_remote(rq))
rq->q->mq_ops->complete(rq);
}
EXPORT_SYMBOL(blk_mq_complete_request);
/**
* blk_mq_start_request - Start processing a request
* @rq: Pointer to request to be started
*
* Function used by device drivers to notify the block layer that a request
* is going to be processed now, so blk layer can do proper initializations
* such as starting the timeout timer.
*/
void blk_mq_start_request(struct request *rq)
{
struct request_queue *q = rq->q;
trace_block_rq_issue(rq);
if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags) &&
!blk_rq_is_passthrough(rq)) {
rq->io_start_time_ns = blk_time_get_ns();
rq->stats_sectors = blk_rq_sectors(rq);
rq->rq_flags |= RQF_STATS;
rq_qos_issue(q, rq);
}
WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
blk_add_timer(rq);
WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
rq->mq_hctx->tags->rqs[rq->tag] = rq;
#ifdef CONFIG_BLK_DEV_INTEGRITY
if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
q->integrity.profile->prepare_fn(rq);
#endif
if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
}
EXPORT_SYMBOL(blk_mq_start_request);
/*
* Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
* queues. This is important for md arrays to benefit from merging
* requests.
*/
static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
{
if (plug->multiple_queues)
return BLK_MAX_REQUEST_COUNT * 2;
return BLK_MAX_REQUEST_COUNT;
}
static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
{
struct request *last = rq_list_peek(&plug->mq_list);
if (!plug->rq_count) {
trace_block_plug(rq->q);
} else if (plug->rq_count >= blk_plug_max_rq_count(plug) ||
(!blk_queue_nomerges(rq->q) &&
blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
blk_mq_flush_plug_list(plug, false);
last = NULL;
trace_block_plug(rq->q);
}
if (!plug->multiple_queues && last && last->q != rq->q)
plug->multiple_queues = true;
/*
* Any request allocated from sched tags can't be issued to
* ->queue_rqs() directly
*/
if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
plug->has_elevator = true;
rq->rq_next = NULL;
rq_list_add(&plug->mq_list, rq);
plug->rq_count++;
}
/**
* blk_execute_rq_nowait - insert a request to I/O scheduler for execution
* @rq: request to insert
* @at_head: insert request at head or tail of queue
*
* Description:
* Insert a fully prepared request at the back of the I/O scheduler queue
* for execution. Don't wait for completion.
*
* Note:
* This function will invoke @done directly if the queue is dead.
*/
void blk_execute_rq_nowait(struct request *rq, bool at_head)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
WARN_ON(irqs_disabled());
WARN_ON(!blk_rq_is_passthrough(rq));
blk_account_io_start(rq);
/*
* As plugging can be enabled for passthrough requests on a zoned
* device, directly accessing the plug instead of using blk_mq_plug()
* should not have any consequences.
*/
if (current->plug && !at_head) {
blk_add_rq_to_plug(current->plug, rq);
return;
}
blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
}
EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
struct blk_rq_wait {
struct completion done;
blk_status_t ret;
};
static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret)
{
struct blk_rq_wait *wait = rq->end_io_data;
wait->ret = ret;
complete(&wait->done);
return RQ_END_IO_NONE;
}
bool blk_rq_is_poll(struct request *rq)
{
if (!rq->mq_hctx)
return false;
if (rq->mq_hctx->type != HCTX_TYPE_POLL)
return false;
return true;
}
EXPORT_SYMBOL_GPL(blk_rq_is_poll);
static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
{
do {
blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0);
cond_resched();
} while (!completion_done(wait));
}
/**
* blk_execute_rq - insert a request into queue for execution
* @rq: request to insert
* @at_head: insert request at head or tail of queue
*
* Description:
* Insert a fully prepared request at the back of the I/O scheduler queue
* for execution and wait for completion.
* Return: The blk_status_t result provided to blk_mq_end_request().
*/
blk_status_t blk_execute_rq(struct request *rq, bool at_head)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
struct blk_rq_wait wait = {
.done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
};
WARN_ON(irqs_disabled());
WARN_ON(!blk_rq_is_passthrough(rq));
rq->end_io_data = &wait;
rq->end_io = blk_end_sync_rq;
blk_account_io_start(rq);
blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
blk_mq_run_hw_queue(hctx, false);
if (blk_rq_is_poll(rq))
blk_rq_poll_completion(rq, &wait.done);
else
blk_wait_io(&wait.done);
return wait.ret;
}
EXPORT_SYMBOL(blk_execute_rq);
static void __blk_mq_requeue_request(struct request *rq)
{
struct request_queue *q = rq->q;
blk_mq_put_driver_tag(rq);
trace_block_rq_requeue(rq);
rq_qos_requeue(q, rq);
if (blk_mq_request_started(rq)) {
WRITE_ONCE(rq->state, MQ_RQ_IDLE);
rq->rq_flags &= ~RQF_TIMED_OUT;
}
}
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
{
struct request_queue *q = rq->q;
unsigned long flags;
__blk_mq_requeue_request(rq);
/* this request will be re-inserted to io scheduler queue */
blk_mq_sched_requeue_request(rq);
spin_lock_irqsave(&q->requeue_lock, flags);
list_add_tail(&rq->queuelist, &q->requeue_list);
spin_unlock_irqrestore(&q->requeue_lock, flags);
if (kick_requeue_list)
blk_mq_kick_requeue_list(q);
}
EXPORT_SYMBOL(blk_mq_requeue_request);
static void blk_mq_requeue_work(struct work_struct *work)
{
struct request_queue *q =
container_of(work, struct request_queue, requeue_work.work);
LIST_HEAD(rq_list);
LIST_HEAD(flush_list);
struct request *rq;
spin_lock_irq(&q->requeue_lock);
list_splice_init(&q->requeue_list, &rq_list);
list_splice_init(&q->flush_list, &flush_list);
spin_unlock_irq(&q->requeue_lock);
while (!list_empty(&rq_list)) {
rq = list_entry(rq_list.next, struct request, queuelist);
/*
* If RQF_DONTPREP ist set, the request has been started by the
* driver already and might have driver-specific data allocated
* already. Insert it into the hctx dispatch list to avoid
* block layer merges for the request.
*/
if (rq->rq_flags & RQF_DONTPREP) {
list_del_init(&rq->queuelist);
blk_mq_request_bypass_insert(rq, 0);
} else {
list_del_init(&rq->queuelist);
blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
}
}
while (!list_empty(&flush_list)) {
rq = list_entry(flush_list.next, struct request, queuelist);
list_del_init(&rq->queuelist);
blk_mq_insert_request(rq, 0);
}
blk_mq_run_hw_queues(q, false);
}
void blk_mq_kick_requeue_list(struct request_queue *q)
{
kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
unsigned long msecs)
{
kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
msecs_to_jiffies(msecs));
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
static bool blk_is_flush_data_rq(struct request *rq)
{
return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq);
}
static bool blk_mq_rq_inflight(struct request *rq, void *priv)
{
/*
* If we find a request that isn't idle we know the queue is busy
* as it's checked in the iter.
* Return false to stop the iteration.
*
* In case of queue quiesce, if one flush data request is completed,
* don't count it as inflight given the flush sequence is suspended,
* and the original flush data request is invisible to driver, just
* like other pending requests because of quiesce
*/
if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) &&
blk_is_flush_data_rq(rq) &&
blk_mq_request_completed(rq))) {
bool *busy = priv;
*busy = true;
return false;
}
return true;
}
bool blk_mq_queue_inflight(struct request_queue *q)
{
bool busy = false;
blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
return busy;
}
EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
static void blk_mq_rq_timed_out(struct request *req)
{
req->rq_flags |= RQF_TIMED_OUT;
if (req->q->mq_ops->timeout) {
enum blk_eh_timer_return ret;
ret = req->q->mq_ops->timeout(req);
if (ret == BLK_EH_DONE)
return;
WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
}
blk_add_timer(req);
}
struct blk_expired_data {
bool has_timedout_rq;
unsigned long next;
unsigned long timeout_start;
};
static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
{
unsigned long deadline;
if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
return false;
if (rq->rq_flags & RQF_TIMED_OUT)
return false;
deadline = READ_ONCE(rq->deadline);
if (time_after_eq(expired->timeout_start, deadline))
return true;
if (expired->next == 0)
expired->next = deadline;
else if (time_after(expired->next, deadline))
expired->next = deadline;
return false;
}
void blk_mq_put_rq_ref(struct request *rq)
{
if (is_flush_rq(rq)) {
if (rq->end_io(rq, 0) == RQ_END_IO_FREE)
blk_mq_free_request(rq);
} else if (req_ref_put_and_test(rq)) {
__blk_mq_free_request(rq);
}
}
static bool blk_mq_check_expired(struct request *rq, void *priv)
{
struct blk_expired_data *expired = priv;
/*
* blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
* be reallocated underneath the timeout handler's processing, then
* the expire check is reliable. If the request is not expired, then
* it was completed and reallocated as a new request after returning
* from blk_mq_check_expired().
*/
if (blk_mq_req_expired(rq, expired)) {
expired->has_timedout_rq = true;
return false;
}
return true;
}
static bool blk_mq_handle_expired(struct request *rq, void *priv)
{
struct blk_expired_data *expired = priv;
if (blk_mq_req_expired(rq, expired))
blk_mq_rq_timed_out(rq);
return true;
}
static void blk_mq_timeout_work(struct work_struct *work)
{
struct request_queue *q =
container_of(work, struct request_queue, timeout_work);
struct blk_expired_data expired = {
.timeout_start = jiffies,
};
struct blk_mq_hw_ctx *hctx;
unsigned long i;
/* A deadlock might occur if a request is stuck requiring a
* timeout at the same time a queue freeze is waiting
* completion, since the timeout code would not be able to
* acquire the queue reference here.
*
* That's why we don't use blk_queue_enter here; instead, we use
* percpu_ref_tryget directly, because we need to be able to
* obtain a reference even in the short window between the queue
* starting to freeze, by dropping the first reference in
* blk_freeze_queue_start, and the moment the last request is
* consumed, marked by the instant q_usage_counter reaches
* zero.
*/
if (!percpu_ref_tryget(&q->q_usage_counter))
return;
/* check if there is any timed-out request */
blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
if (expired.has_timedout_rq) {
/*
* Before walking tags, we must ensure any submit started
* before the current time has finished. Since the submit
* uses srcu or rcu, wait for a synchronization point to
* ensure all running submits have finished
*/
blk_mq_wait_quiesce_done(q->tag_set);
expired.next = 0;
blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
}
if (expired.next != 0) {
mod_timer(&q->timeout, expired.next);
} else {
/*
* Request timeouts are handled as a forward rolling timer. If
* we end up here it means that no requests are pending and
* also that no request has been pending for a while. Mark
* each hctx as idle.
*/
queue_for_each_hw_ctx(q, hctx, i) {
/* the hctx may be unmapped, so check it here */
if (blk_mq_hw_queue_mapped(hctx))
blk_mq_tag_idle(hctx);
}
}
blk_queue_exit(q);
}
struct flush_busy_ctx_data {
struct blk_mq_hw_ctx *hctx;
struct list_head *list;
};
static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
struct flush_busy_ctx_data *flush_data = data;
struct blk_mq_hw_ctx *hctx = flush_data->hctx;
struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
enum hctx_type type = hctx->type;
spin_lock(&ctx->lock);
list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
sbitmap_clear_bit(sb, bitnr);
spin_unlock(&ctx->lock);
return true;
}
/*
* Process software queues that have been marked busy, splicing them
* to the for-dispatch
*/
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
{
struct flush_busy_ctx_data data = {
.hctx = hctx,
.list = list,
};
sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
}
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
struct dispatch_rq_data {
struct blk_mq_hw_ctx *hctx;
struct request *rq;
};
static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
void *data)
{
struct dispatch_rq_data *dispatch_data = data;
struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
enum hctx_type type = hctx->type;
spin_lock(&ctx->lock);
if (!list_empty(&ctx->rq_lists[type])) {
dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
list_del_init(&dispatch_data->rq->queuelist);
if (list_empty(&ctx->rq_lists[type]))
sbitmap_clear_bit(sb, bitnr);
}
spin_unlock(&ctx->lock);
return !dispatch_data->rq;
}
struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
struct blk_mq_ctx *start)
{
unsigned off = start ? start->index_hw[hctx->type] : 0;
struct dispatch_rq_data data = {
.hctx = hctx,
.rq = NULL,
};
__sbitmap_for_each_set(&hctx->ctx_map, off,
dispatch_rq_from_ctx, &data);
return data.rq;
}
bool __blk_mq_alloc_driver_tag(struct request *rq)
{
struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
int tag;
blk_mq_tag_busy(rq->mq_hctx);
if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
bt = &rq->mq_hctx->tags->breserved_tags;
tag_offset = 0;
} else {
if (!hctx_may_queue(rq->mq_hctx, bt))
return false;
}
tag = __sbitmap_queue_get(bt);
if (tag == BLK_MQ_NO_TAG)
return false;
rq->tag = tag + tag_offset;
blk_mq_inc_active_requests(rq->mq_hctx);
return true;
}
static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
int flags, void *key)
{
struct blk_mq_hw_ctx *hctx;
hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
spin_lock(&hctx->dispatch_wait_lock);
if (!list_empty(&wait->entry)) {
struct sbitmap_queue *sbq;
list_del_init(&wait->entry);
sbq = &hctx->tags->bitmap_tags;
atomic_dec(&sbq->ws_active);
}
spin_unlock(&hctx->dispatch_wait_lock);
blk_mq_run_hw_queue(hctx, true);
return 1;
}
/*
* Mark us waiting for a tag. For shared tags, this involves hooking us into
* the tag wakeups. For non-shared tags, we can simply mark us needing a
* restart. For both cases, take care to check the condition again after
* marking us as waiting.
*/
static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
struct request *rq)
{
struct sbitmap_queue *sbq;
struct wait_queue_head *wq;
wait_queue_entry_t *wait;
bool ret;
if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
!(blk_mq_is_shared_tags(hctx->flags))) {
blk_mq_sched_mark_restart_hctx(hctx);
/*
* It's possible that a tag was freed in the window between the
* allocation failure and adding the hardware queue to the wait
* queue.
*
* Don't clear RESTART here, someone else could have set it.
* At most this will cost an extra queue run.
*/
return blk_mq_get_driver_tag(rq);
}
wait = &hctx->dispatch_wait;
if (!list_empty_careful(&wait->entry))
return false;
if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
sbq = &hctx->tags->breserved_tags;
else
sbq = &hctx->tags->bitmap_tags;
wq = &bt_wait_ptr(sbq, hctx)->wait;
spin_lock_irq(&wq->lock);
spin_lock(&hctx->dispatch_wait_lock);
if (!list_empty(&wait->entry)) {
spin_unlock(&hctx->dispatch_wait_lock);
spin_unlock_irq(&wq->lock);
return false;
}
atomic_inc(&sbq->ws_active);
wait->flags &= ~WQ_FLAG_EXCLUSIVE;
__add_wait_queue(wq, wait);
/*
* Add one explicit barrier since blk_mq_get_driver_tag() may
* not imply barrier in case of failure.
*
* Order adding us to wait queue and allocating driver tag.
*
* The pair is the one implied in sbitmap_queue_wake_up() which
* orders clearing sbitmap tag bits and waitqueue_active() in
* __sbitmap_queue_wake_up(), since waitqueue_active() is lockless
*
* Otherwise, re-order of adding wait queue and getting driver tag
* may cause __sbitmap_queue_wake_up() to wake up nothing because
* the waitqueue_active() may not observe us in wait queue.
*/
smp_mb();
/*
* It's possible that a tag was freed in the window between the
* allocation failure and adding the hardware queue to the wait
* queue.
*/
ret = blk_mq_get_driver_tag(rq);
if (!ret) {
spin_unlock(&hctx->dispatch_wait_lock);
spin_unlock_irq(&wq->lock);
return false;
}
/*
* We got a tag, remove ourselves from the wait queue to ensure
* someone else gets the wakeup.
*/
list_del_init(&wait->entry);
atomic_dec(&sbq->ws_active);
spin_unlock(&hctx->dispatch_wait_lock);
spin_unlock_irq(&wq->lock);
return true;
}
#define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8
#define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4
/*
* Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
* - EWMA is one simple way to compute running average value
* - weight(7/8 and 1/8) is applied so that it can decrease exponentially
* - take 4 as factor for avoiding to get too small(0) result, and this
* factor doesn't matter because EWMA decreases exponentially
*/
static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
{
unsigned int ewma;
ewma = hctx->dispatch_busy;
if (!ewma && !busy)
return;
ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
if (busy)
ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
hctx->dispatch_busy = ewma;
}
#define BLK_MQ_RESOURCE_DELAY 3 /* ms units */
static void blk_mq_handle_dev_resource(struct request *rq,
struct list_head *list)
{
list_add(&rq->queuelist, list);
__blk_mq_requeue_request(rq);
}
static void blk_mq_handle_zone_resource(struct request *rq,
struct list_head *zone_list)
{
/*
* If we end up here it is because we cannot dispatch a request to a
* specific zone due to LLD level zone-write locking or other zone
* related resource not being available. In this case, set the request
* aside in zone_list for retrying it later.
*/
list_add(&rq->queuelist, zone_list);
__blk_mq_requeue_request(rq);
}
enum prep_dispatch {
PREP_DISPATCH_OK,
PREP_DISPATCH_NO_TAG,
PREP_DISPATCH_NO_BUDGET,
};
static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
bool need_budget)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
int budget_token = -1;
if (need_budget) {
budget_token = blk_mq_get_dispatch_budget(rq->q);
if (budget_token < 0) {
blk_mq_put_driver_tag(rq);
return PREP_DISPATCH_NO_BUDGET;
}
blk_mq_set_rq_budget_token(rq, budget_token);
}
if (!blk_mq_get_driver_tag(rq)) {
/*
* The initial allocation attempt failed, so we need to
* rerun the hardware queue when a tag is freed. The
* waitqueue takes care of that. If the queue is run
* before we add this entry back on the dispatch list,
* we'll re-run it below.
*/
if (!blk_mq_mark_tag_wait(hctx, rq)) {
/*
* All budgets not got from this function will be put
* together during handling partial dispatch
*/
if (need_budget)
blk_mq_put_dispatch_budget(rq->q, budget_token);
return PREP_DISPATCH_NO_TAG;
}
}
return PREP_DISPATCH_OK;
}
/* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
static void blk_mq_release_budgets(struct request_queue *q,
struct list_head *list)
{
struct request *rq;
list_for_each_entry(rq, list, queuelist) {
int budget_token = blk_mq_get_rq_budget_token(rq);
if (budget_token >= 0)
blk_mq_put_dispatch_budget(q, budget_token);
}
}
/*
* blk_mq_commit_rqs will notify driver using bd->last that there is no
* more requests. (See comment in struct blk_mq_ops for commit_rqs for
* details)
* Attention, we should explicitly call this in unusual cases:
* 1) did not queue everything initially scheduled to queue
* 2) the last attempt to queue a request failed
*/
static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
bool from_schedule)
{
if (hctx->queue->mq_ops->commit_rqs && queued) {
trace_block_unplug(hctx->queue, queued, !from_schedule);
hctx->queue->mq_ops->commit_rqs(hctx);
}
}
/*
* Returns true if we did some work AND can potentially do more.
*/
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
unsigned int nr_budgets)
{
enum prep_dispatch prep;
struct request_queue *q = hctx->queue;
struct request *rq;
int queued;
blk_status_t ret = BLK_STS_OK;
LIST_HEAD(zone_list);
bool needs_resource = false;
if (list_empty(list))
return false;
/*
* Now process all the entries, sending them to the driver.
*/
queued = 0;
do {
struct blk_mq_queue_data bd;
rq = list_first_entry(list, struct request, queuelist);
WARN_ON_ONCE(hctx != rq->mq_hctx);
prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
if (prep != PREP_DISPATCH_OK)
break;
list_del_init(&rq->queuelist);
bd.rq = rq;
bd.last = list_empty(list);
/*
* once the request is queued to lld, no need to cover the
* budget any more
*/
if (nr_budgets)
nr_budgets--;
ret = q->mq_ops->queue_rq(hctx, &bd);
switch (ret) {
case BLK_STS_OK:
queued++;
break;
case BLK_STS_RESOURCE:
needs_resource = true;
fallthrough;
case BLK_STS_DEV_RESOURCE:
blk_mq_handle_dev_resource(rq, list);
goto out;
case BLK_STS_ZONE_RESOURCE:
/*
* Move the request to zone_list and keep going through
* the dispatch list to find more requests the drive can
* accept.
*/
blk_mq_handle_zone_resource(rq, &zone_list);
needs_resource = true;
break;
default:
blk_mq_end_request(rq, ret);
}
} while (!list_empty(list));
out:
if (!list_empty(&zone_list))
list_splice_tail_init(&zone_list, list);
/* If we didn't flush the entire list, we could have told the driver
* there was more coming, but that turned out to be a lie.
*/
if (!list_empty(list) || ret != BLK_STS_OK)
blk_mq_commit_rqs(hctx, queued, false);
/*
* Any items that need requeuing? Stuff them into hctx->dispatch,
* that is where we will continue on next queue run.
*/
if (!list_empty(list)) {
bool needs_restart;
/* For non-shared tags, the RESTART check will suffice */
bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) ||
blk_mq_is_shared_tags(hctx->flags));
if (nr_budgets)
blk_mq_release_budgets(q, list);
spin_lock(&hctx->lock);
list_splice_tail_init(list, &hctx->dispatch);
spin_unlock(&hctx->lock);
/*
* Order adding requests to hctx->dispatch and checking
* SCHED_RESTART flag. The pair of this smp_mb() is the one
* in blk_mq_sched_restart(). Avoid restart code path to
* miss the new added requests to hctx->dispatch, meantime
* SCHED_RESTART is observed here.
*/
smp_mb();
/*
* If SCHED_RESTART was set by the caller of this function and
* it is no longer set that means that it was cleared by another
* thread and hence that a queue rerun is needed.
*
* If 'no_tag' is set, that means that we failed getting
* a driver tag with an I/O scheduler attached. If our dispatch
* waitqueue is no longer active, ensure that we run the queue
* AFTER adding our entries back to the list.
*
* If no I/O scheduler has been configured it is possible that
* the hardware queue got stopped and restarted before requests
* were pushed back onto the dispatch list. Rerun the queue to
* avoid starvation. Notes:
* - blk_mq_run_hw_queue() checks whether or not a queue has
* been stopped before rerunning a queue.
* - Some but not all block drivers stop a queue before
* returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
* and dm-rq.
*
* If driver returns BLK_STS_RESOURCE and SCHED_RESTART
* bit is set, run queue after a delay to avoid IO stalls
* that could otherwise occur if the queue is idle. We'll do
* similar if we couldn't get budget or couldn't lock a zone
* and SCHED_RESTART is set.
*/
needs_restart = blk_mq_sched_needs_restart(hctx);
if (prep == PREP_DISPATCH_NO_BUDGET)
needs_resource = true;
if (!needs_restart ||
(no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
blk_mq_run_hw_queue(hctx, true);
else if (needs_resource)
blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
blk_mq_update_dispatch_busy(hctx, true);
return false;
}
blk_mq_update_dispatch_busy(hctx, false);
return true;
}
static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
{
int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(hctx->cpumask);
return cpu;
}
/*
* It'd be great if the workqueue API had a way to pass
* in a mask and had some smarts for more clever placement.
* For now we just round-robin here, switching for every
* BLK_MQ_CPU_WORK_BATCH queued items.
*/
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
bool tried = false;
int next_cpu = hctx->next_cpu;
if (hctx->queue->nr_hw_queues == 1)
return WORK_CPU_UNBOUND;
if (--hctx->next_cpu_batch <= 0) {
select_cpu:
next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
cpu_online_mask);
if (next_cpu >= nr_cpu_ids)
next_cpu = blk_mq_first_mapped_cpu(hctx);
hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
}
/*
* Do unbound schedule if we can't find a online CPU for this hctx,
* and it should only happen in the path of handling CPU DEAD.
*/
if (!cpu_online(next_cpu)) {
if (!tried) {
tried = true;
goto select_cpu;
}
/*
* Make sure to re-select CPU next time once after CPUs
* in hctx->cpumask become online again.
*/
hctx->next_cpu = next_cpu;
hctx->next_cpu_batch = 1;
return WORK_CPU_UNBOUND;
}
hctx->next_cpu = next_cpu;
return next_cpu;
}
/**
* blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
* @hctx: Pointer to the hardware queue to run.
* @msecs: Milliseconds of delay to wait before running the queue.
*
* Run a hardware queue asynchronously with a delay of @msecs.
*/
void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
if (unlikely(blk_mq_hctx_stopped(hctx)))
return;
kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
msecs_to_jiffies(msecs));
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
/**
* blk_mq_run_hw_queue - Start to run a hardware queue.
* @hctx: Pointer to the hardware queue to run.
* @async: If we want to run the queue asynchronously.
*
* Check if the request queue is not in a quiesced state and if there are
* pending requests to be sent. If this is true, run the queue to send requests
* to hardware.
*/
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
bool need_run;
/*
* We can't run the queue inline with interrupts disabled.
*/
WARN_ON_ONCE(!async && in_interrupt());
might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
/*
* When queue is quiesced, we may be switching io scheduler, or
* updating nr_hw_queues, or other things, and we can't run queue
* any more, even __blk_mq_hctx_has_pending() can't be called safely.
*
* And queue will be rerun in blk_mq_unquiesce_queue() if it is
* quiesced.
*/
__blk_mq_run_dispatch_ops(hctx->queue, false,
need_run = !blk_queue_quiesced(hctx->queue) &&
blk_mq_hctx_has_pending(hctx));
if (!need_run)
return;
if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
blk_mq_delay_run_hw_queue(hctx, 0);
return;
}
blk_mq_run_dispatch_ops(hctx->queue,
blk_mq_sched_dispatch_requests(hctx));
}
EXPORT_SYMBOL(blk_mq_run_hw_queue);
/*
* Return prefered queue to dispatch from (if any) for non-mq aware IO
* scheduler.
*/
static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
{
struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
/*
* If the IO scheduler does not respect hardware queues when
* dispatching, we just don't bother with multiple HW queues and
* dispatch from hctx for the current CPU since running multiple queues
* just causes lock contention inside the scheduler and pointless cache
* bouncing.
*/
struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
if (!blk_mq_hctx_stopped(hctx))
return hctx;
return NULL;
}
/**
* blk_mq_run_hw_queues - Run all hardware queues in a request queue.
* @q: Pointer to the request queue to run.
* @async: If we want to run the queue asynchronously.
*/
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
{
struct blk_mq_hw_ctx *hctx, *sq_hctx;
unsigned long i;
sq_hctx = NULL;
if (blk_queue_sq_sched(q))
sq_hctx = blk_mq_get_sq_hctx(q);
queue_for_each_hw_ctx(q, hctx, i) {
if (blk_mq_hctx_stopped(hctx))
continue;
/*
* Dispatch from this hctx either if there's no hctx preferred
* by IO scheduler or if it has requests that bypass the
* scheduler.
*/
if (!sq_hctx || sq_hctx == hctx ||
!list_empty_careful(&hctx->dispatch))
blk_mq_run_hw_queue(hctx, async);
}
}
EXPORT_SYMBOL(blk_mq_run_hw_queues);
/**
* blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
* @q: Pointer to the request queue to run.
* @msecs: Milliseconds of delay to wait before running the queues.
*/
void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
{
struct blk_mq_hw_ctx *hctx, *sq_hctx;
unsigned long i;
sq_hctx = NULL;
if (blk_queue_sq_sched(q))
sq_hctx = blk_mq_get_sq_hctx(q);
queue_for_each_hw_ctx(q, hctx, i) {
if (blk_mq_hctx_stopped(hctx))
continue;
/*
* If there is already a run_work pending, leave the
* pending delay untouched. Otherwise, a hctx can stall
* if another hctx is re-delaying the other's work
* before the work executes.
*/
if (delayed_work_pending(&hctx->run_work))
continue;
/*
* Dispatch from this hctx either if there's no hctx preferred
* by IO scheduler or if it has requests that bypass the
* scheduler.
*/
if (!sq_hctx || sq_hctx == hctx ||
!list_empty_careful(&hctx->dispatch))
blk_mq_delay_run_hw_queue(hctx, msecs);
}
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
/*
* This function is often used for pausing .queue_rq() by driver when
* there isn't enough resource or some conditions aren't satisfied, and
* BLK_STS_RESOURCE is usually returned.
*
* We do not guarantee that dispatch can be drained or blocked
* after blk_mq_stop_hw_queue() returns. Please use
* blk_mq_quiesce_queue() for that requirement.
*/
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
cancel_delayed_work(&hctx->run_work);
set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);
/*
* This function is often used for pausing .queue_rq() by driver when
* there isn't enough resource or some conditions aren't satisfied, and
* BLK_STS_RESOURCE is usually returned.
*
* We do not guarantee that dispatch can be drained or blocked
* after blk_mq_stop_hw_queues() returns. Please use
* blk_mq_quiesce_queue() for that requirement.
*/
void blk_mq_stop_hw_queues(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i)
blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);
void blk_mq_start_hw_queues(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i)
blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
if (!blk_mq_hctx_stopped(hctx))
return;
clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i)
blk_mq_start_stopped_hw_queue(hctx, async ||
(hctx->flags & BLK_MQ_F_BLOCKING));
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
static void blk_mq_run_work_fn(struct work_struct *work)
{
struct blk_mq_hw_ctx *hctx =
container_of(work, struct blk_mq_hw_ctx, run_work.work);
blk_mq_run_dispatch_ops(hctx->queue,
blk_mq_sched_dispatch_requests(hctx));
}
/**
* blk_mq_request_bypass_insert - Insert a request at dispatch list.
* @rq: Pointer to request to be inserted.
* @flags: BLK_MQ_INSERT_*
*
* Should only be used carefully, when the caller knows we want to
* bypass a potential IO scheduler on the target device.
*/
static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
spin_lock(&hctx->lock);
if (flags & BLK_MQ_INSERT_AT_HEAD)
list_add(&rq->queuelist, &hctx->dispatch);
else
list_add_tail(&rq->queuelist, &hctx->dispatch);
spin_unlock(&hctx->lock);
}
static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
struct blk_mq_ctx *ctx, struct list_head *list,
bool run_queue_async)
{
struct request *rq;
enum hctx_type type = hctx->type;
/*
* Try to issue requests directly if the hw queue isn't busy to save an
* extra enqueue & dequeue to the sw queue.
*/
if (!hctx->dispatch_busy && !run_queue_async) {
blk_mq_run_dispatch_ops(hctx->queue,
blk_mq_try_issue_list_directly(hctx, list));
if (list_empty(list))
goto out;
}
/*
* preemption doesn't flush plug list, so it's possible ctx->cpu is
* offline now
*/
list_for_each_entry(rq, list, queuelist) {
BUG_ON(rq->mq_ctx != ctx);
trace_block_rq_insert(rq);
if (rq->cmd_flags & REQ_NOWAIT)
run_queue_async = true;
}
spin_lock(&ctx->lock);
list_splice_tail_init(list, &ctx->rq_lists[type]);
blk_mq_hctx_mark_pending(hctx, ctx);
spin_unlock(&ctx->lock);
out:
blk_mq_run_hw_queue(hctx, run_queue_async);
}
static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
{
struct request_queue *q = rq->q;
struct blk_mq_ctx *ctx = rq->mq_ctx;
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
if (blk_rq_is_passthrough(rq)) {
/*
* Passthrough request have to be added to hctx->dispatch
* directly. The device may be in a situation where it can't
* handle FS request, and always returns BLK_STS_RESOURCE for
* them, which gets them added to hctx->dispatch.
*
* If a passthrough request is required to unblock the queues,
* and it is added to the scheduler queue, there is no chance to
* dispatch it given we prioritize requests in hctx->dispatch.
*/
blk_mq_request_bypass_insert(rq, flags);
} else if (req_op(rq) == REQ_OP_FLUSH) {
/*
* Firstly normal IO request is inserted to scheduler queue or
* sw queue, meantime we add flush request to dispatch queue(
* hctx->dispatch) directly and there is at most one in-flight
* flush request for each hw queue, so it doesn't matter to add
* flush request to tail or front of the dispatch queue.
*
* Secondly in case of NCQ, flush request belongs to non-NCQ
* command, and queueing it will fail when there is any
* in-flight normal IO request(NCQ command). When adding flush
* rq to the front of hctx->dispatch, it is easier to introduce
* extra time to flush rq's latency because of S_SCHED_RESTART
* compared with adding to the tail of dispatch queue, then
* chance of flush merge is increased, and less flush requests
* will be issued to controller. It is observed that ~10% time
* is saved in blktests block/004 on disk attached to AHCI/NCQ
* drive when adding flush rq to the front of hctx->dispatch.
*
* Simply queue flush rq to the front of hctx->dispatch so that
* intensive flush workloads can benefit in case of NCQ HW.
*/
blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
} else if (q->elevator) {
LIST_HEAD(list);
WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
list_add(&rq->queuelist, &list);
q->elevator->type->ops.insert_requests(hctx, &list, flags);
} else {
trace_block_rq_insert(rq);
spin_lock(&ctx->lock);
if (flags & BLK_MQ_INSERT_AT_HEAD)
list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
else
list_add_tail(&rq->queuelist,
&ctx->rq_lists[hctx->type]);
blk_mq_hctx_mark_pending(hctx, ctx);
spin_unlock(&ctx->lock);
}
}
static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
unsigned int nr_segs)
{
int err;
if (bio->bi_opf & REQ_RAHEAD)
rq->cmd_flags |= REQ_FAILFAST_MASK;
rq->__sector = bio->bi_iter.bi_sector;
rq->write_hint = bio->bi_write_hint;
blk_rq_bio_prep(rq, bio, nr_segs);
/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
WARN_ON_ONCE(err);
blk_account_io_start(rq);
}
static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
struct request *rq, bool last)
{
struct request_queue *q = rq->q;
struct blk_mq_queue_data bd = {
.rq = rq,
.last = last,
};
blk_status_t ret;
/*
* For OK queue, we are done. For error, caller may kill it.
* Any other error (busy), just add it to our list as we
* previously would have done.
*/
ret = q->mq_ops->queue_rq(hctx, &bd);
switch (ret) {
case BLK_STS_OK:
blk_mq_update_dispatch_busy(hctx, false);
break;
case BLK_STS_RESOURCE:
case BLK_STS_DEV_RESOURCE:
blk_mq_update_dispatch_busy(hctx, true);
__blk_mq_requeue_request(rq);
break;
default:
blk_mq_update_dispatch_busy(hctx, false);
break;
}
return ret;
}
static bool blk_mq_get_budget_and_tag(struct request *rq)
{
int budget_token;
budget_token = blk_mq_get_dispatch_budget(rq->q);
if (budget_token < 0)
return false;
blk_mq_set_rq_budget_token(rq, budget_token);
if (!blk_mq_get_driver_tag(rq)) {
blk_mq_put_dispatch_budget(rq->q, budget_token);
return false;
}
return true;
}
/**
* blk_mq_try_issue_directly - Try to send a request directly to device driver.
* @hctx: Pointer of the associated hardware queue.
* @rq: Pointer to request to be sent.
*
* If the device has enough resources to accept a new request now, send the
* request directly to device driver. Else, insert at hctx->dispatch queue, so
* we can try send it another time in the future. Requests inserted at this
* queue have higher priority.
*/
static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
struct request *rq)
{
blk_status_t ret;
if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
blk_mq_insert_request(rq, 0);
return;
}
if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) {
blk_mq_insert_request(rq, 0);
blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
return;
}
ret = __blk_mq_issue_directly(hctx, rq, true);
switch (ret) {
case BLK_STS_OK:
break;
case BLK_STS_RESOURCE:
case BLK_STS_DEV_RESOURCE:
blk_mq_request_bypass_insert(rq, 0);
blk_mq_run_hw_queue(hctx, false);
break;
default:
blk_mq_end_request(rq, ret);
break;
}
}
static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
blk_mq_insert_request(rq, 0);
return BLK_STS_OK;
}
if (!blk_mq_get_budget_and_tag(rq))
return BLK_STS_RESOURCE;
return __blk_mq_issue_directly(hctx, rq, last);
}
static void blk_mq_plug_issue_direct(struct blk_plug *plug)
{
struct blk_mq_hw_ctx *hctx = NULL;
struct request *rq;
int queued = 0;
blk_status_t ret = BLK_STS_OK;
while ((rq = rq_list_pop(&plug->mq_list))) {
bool last = rq_list_empty(plug->mq_list);
if (hctx != rq->mq_hctx) {
if (hctx) {
blk_mq_commit_rqs(hctx, queued, false);
queued = 0;
}
hctx = rq->mq_hctx;
}
ret = blk_mq_request_issue_directly(rq, last);
switch (ret) {
case BLK_STS_OK:
queued++;
break;
case BLK_STS_RESOURCE:
case BLK_STS_DEV_RESOURCE:
blk_mq_request_bypass_insert(rq, 0);
blk_mq_run_hw_queue(hctx, false);
goto out;
default:
blk_mq_end_request(rq, ret);
break;
}
}
out:
if (ret != BLK_STS_OK)
blk_mq_commit_rqs(hctx, queued, false);
}
static void __blk_mq_flush_plug_list(struct request_queue *q,
struct blk_plug *plug)
{
if (blk_queue_quiesced(q))
return;
q->mq_ops->queue_rqs(&plug->mq_list);
}
static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)
{
struct blk_mq_hw_ctx *this_hctx = NULL;
struct blk_mq_ctx *this_ctx = NULL;
struct request *requeue_list = NULL;
struct request **requeue_lastp = &requeue_list;
unsigned int depth = 0;
bool is_passthrough = false;
LIST_HEAD(list);
do {
struct request *rq = rq_list_pop(&plug->mq_list);
if (!this_hctx) {
this_hctx = rq->mq_hctx;
this_ctx = rq->mq_ctx;
is_passthrough = blk_rq_is_passthrough(rq);
} else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
is_passthrough != blk_rq_is_passthrough(rq)) {
rq_list_add_tail(&requeue_lastp, rq);
continue;
}
list_add(&rq->queuelist, &list);
depth++;
} while (!rq_list_empty(plug->mq_list));
plug->mq_list = requeue_list;
trace_block_unplug(this_hctx->queue, depth, !from_sched);
percpu_ref_get(&this_hctx->queue->q_usage_counter);
/* passthrough requests should never be issued to the I/O scheduler */
if (is_passthrough) {
spin_lock(&this_hctx->lock);
list_splice_tail_init(&list, &this_hctx->dispatch);
spin_unlock(&this_hctx->lock);
blk_mq_run_hw_queue(this_hctx, from_sched);
} else if (this_hctx->queue->elevator) {
this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
&list, 0);
blk_mq_run_hw_queue(this_hctx, from_sched);
} else {
blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
}
percpu_ref_put(&this_hctx->queue->q_usage_counter);
}
void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
struct request *rq;
/*
* We may have been called recursively midway through handling
* plug->mq_list via a schedule() in the driver's queue_rq() callback.
* To avoid mq_list changing under our feet, clear rq_count early and
* bail out specifically if rq_count is 0 rather than checking
* whether the mq_list is empty.
*/
if (plug->rq_count == 0)
return;
plug->rq_count = 0;
if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) {
struct request_queue *q;
rq = rq_list_peek(&plug->mq_list);
q = rq->q;
/*
* Peek first request and see if we have a ->queue_rqs() hook.
* If we do, we can dispatch the whole plug list in one go. We
* already know at this point that all requests belong to the
* same queue, caller must ensure that's the case.
*/
if (q->mq_ops->queue_rqs) {
blk_mq_run_dispatch_ops(q,
__blk_mq_flush_plug_list(q, plug));
if (rq_list_empty(plug->mq_list))
return;
}
blk_mq_run_dispatch_ops(q,
blk_mq_plug_issue_direct(plug));
if (rq_list_empty(plug->mq_list))
return;
}
do {
blk_mq_dispatch_plug_list(plug, from_schedule);
} while (!rq_list_empty(plug->mq_list));
}
static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
struct list_head *list)
{
int queued = 0;
blk_status_t ret = BLK_STS_OK;
while (!list_empty(list)) {
struct request *rq = list_first_entry(list, struct request,
queuelist);
list_del_init(&rq->queuelist);
ret = blk_mq_request_issue_directly(rq, list_empty(list));
switch (ret) {
case BLK_STS_OK:
queued++;
break;
case BLK_STS_RESOURCE:
case BLK_STS_DEV_RESOURCE:
blk_mq_request_bypass_insert(rq, 0);
if (list_empty(list))
blk_mq_run_hw_queue(hctx, false);
goto out;
default:
blk_mq_end_request(rq, ret);
break;
}
}
out:
if (ret != BLK_STS_OK)
blk_mq_commit_rqs(hctx, queued, false);
}
static bool blk_mq_attempt_bio_merge(struct request_queue *q,
struct bio *bio, unsigned int nr_segs)
{
if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
if (blk_attempt_plug_merge(q, bio, nr_segs))
return true;
if (blk_mq_sched_bio_merge(q, bio, nr_segs))
return true;
}
return false;
}
static struct request *blk_mq_get_new_requests(struct request_queue *q,
struct blk_plug *plug,
struct bio *bio,
unsigned int nsegs)
{
struct blk_mq_alloc_data data = {
.q = q,
.nr_tags = 1,
.cmd_flags = bio->bi_opf,
};
struct request *rq;
rq_qos_throttle(q, bio);
if (plug) {
data.nr_tags = plug->nr_ios;
plug->nr_ios = 1;
data.cached_rq = &plug->cached_rq;
}
rq = __blk_mq_alloc_requests(&data);
if (rq)
return rq;
rq_qos_cleanup(q, bio);
if (bio->bi_opf & REQ_NOWAIT)
bio_wouldblock_error(bio);
return NULL;
}
/*
* Check if there is a suitable cached request and return it.
*/
static struct request *blk_mq_peek_cached_request(struct blk_plug *plug,
struct request_queue *q, blk_opf_t opf)
{
enum hctx_type type = blk_mq_get_hctx_type(opf);
struct request *rq;
if (!plug)
return NULL;
rq = rq_list_peek(&plug->cached_rq);
if (!rq || rq->q != q)
return NULL;
if (type != rq->mq_hctx->type &&
(type != HCTX_TYPE_READ || rq->mq_hctx->type != HCTX_TYPE_DEFAULT))
return NULL;
if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
return NULL;
return rq;
}
static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug,
struct bio *bio)
{
WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq);
/*
* If any qos ->throttle() end up blocking, we will have flushed the
* plug and hence killed the cached_rq list as well. Pop this entry
* before we throttle.
*/
plug->cached_rq = rq_list_next(rq);
rq_qos_throttle(rq->q, bio);
blk_mq_rq_time_init(rq, 0);
rq->cmd_flags = bio->bi_opf;
INIT_LIST_HEAD(&rq->queuelist);
}
/**
* blk_mq_submit_bio - Create and send a request to block device.
* @bio: Bio pointer.
*
* Builds up a request structure from @q and @bio and send to the device. The
* request may not be queued directly to hardware if:
* * This request can be merged with another one
* * We want to place request at plug queue for possible future merging
* * There is an IO scheduler active at this queue
*
* It will not queue the request if there is an error with the bio, or at the
* request creation.
*/
void blk_mq_submit_bio(struct bio *bio)
{
struct request_queue *q = bdev_get_queue(bio->bi_bdev);
struct blk_plug *plug = blk_mq_plug(bio);
const int is_sync = op_is_sync(bio->bi_opf);
struct blk_mq_hw_ctx *hctx;
unsigned int nr_segs = 1;
struct request *rq;
blk_status_t ret;
bio = blk_queue_bounce(bio, q);
/*
* If the plug has a cached request for this queue, try use it.
*
* The cached request already holds a q_usage_counter reference and we
* don't have to acquire a new one if we use it.
*/
rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf);
if (!rq) {
if (unlikely(bio_queue_enter(bio)))
return;
}
if (unlikely(bio_may_exceed_limits(bio, &q->limits))) {
bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
if (!bio)
goto queue_exit;
}
if (!bio_integrity_prep(bio))
goto queue_exit;
if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
goto queue_exit;
if (!rq) {
rq = blk_mq_get_new_requests(q, plug, bio, nr_segs);
if (unlikely(!rq))
goto queue_exit;
} else {
blk_mq_use_cached_rq(rq, plug, bio);
}
trace_block_getrq(bio);
rq_qos_track(q, rq, bio);
blk_mq_bio_to_request(rq, bio, nr_segs);
ret = blk_crypto_rq_get_keyslot(rq);
if (ret != BLK_STS_OK) {
bio->bi_status = ret;
bio_endio(bio);
blk_mq_free_request(rq);
return;
}
if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
return;
if (plug) {
blk_add_rq_to_plug(plug, rq);
return;
}
hctx = rq->mq_hctx;
if ((rq->rq_flags & RQF_USE_SCHED) ||
(hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
blk_mq_insert_request(rq, 0);
blk_mq_run_hw_queue(hctx, true);
} else {
blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
}
return;
queue_exit:
/*
* Don't drop the queue reference if we were trying to use a cached
* request and thus didn't acquire one.
*/
if (!rq)
blk_queue_exit(q);
}
#ifdef CONFIG_BLK_MQ_STACKING
/**
* blk_insert_cloned_request - Helper for stacking drivers to submit a request
* @rq: the request being queued
*/
blk_status_t blk_insert_cloned_request(struct request *rq)
{
struct request_queue *q = rq->q;
unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
unsigned int max_segments = blk_rq_get_max_segments(rq);
blk_status_t ret;
if (blk_rq_sectors(rq) > max_sectors) {
/*
* SCSI device does not have a good way to return if
* Write Same/Zero is actually supported. If a device rejects
* a non-read/write command (discard, write same,etc.) the
* low-level device driver will set the relevant queue limit to
* 0 to prevent blk-lib from issuing more of the offending
* operations. Commands queued prior to the queue limit being
* reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
* errors being propagated to upper layers.
*/
if (max_sectors == 0)
return BLK_STS_NOTSUPP;
printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
__func__, blk_rq_sectors(rq), max_sectors);
return BLK_STS_IOERR;
}
/*
* The queue settings related to segment counting may differ from the
* original queue.
*/
rq->nr_phys_segments = blk_recalc_rq_segments(rq);
if (rq->nr_phys_segments > max_segments) {
printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
__func__, rq->nr_phys_segments, max_segments);
return BLK_STS_IOERR;
}
if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
return BLK_STS_IOERR;
ret = blk_crypto_rq_get_keyslot(rq);
if (ret != BLK_STS_OK)
return ret;
blk_account_io_start(rq);
/*
* Since we have a scheduler attached on the top device,
* bypass a potential scheduler on the bottom device for
* insert.
*/
blk_mq_run_dispatch_ops(q,
ret = blk_mq_request_issue_directly(rq, true));
if (ret)
blk_account_io_done(rq, blk_time_get_ns());
return ret;
}
EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
/**
* blk_rq_unprep_clone - Helper function to free all bios in a cloned request
* @rq: the clone request to be cleaned up
*
* Description:
* Free all bios in @rq for a cloned request.
*/
void blk_rq_unprep_clone(struct request *rq)
{
struct bio *bio;
while ((bio = rq->bio) != NULL) {
rq->bio = bio->bi_next;
bio_put(bio);
}
}
EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
/**
* blk_rq_prep_clone - Helper function to setup clone request
* @rq: the request to be setup
* @rq_src: original request to be cloned
* @bs: bio_set that bios for clone are allocated from
* @gfp_mask: memory allocation mask for bio
* @bio_ctr: setup function to be called for each clone bio.
* Returns %0 for success, non %0 for failure.
* @data: private data to be passed to @bio_ctr
*
* Description:
* Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
* Also, pages which the original bios are pointing to are not copied
* and the cloned bios just point same pages.
* So cloned bios must be completed before original bios, which means
* the caller must complete @rq before @rq_src.
*/
int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
struct bio_set *bs, gfp_t gfp_mask,
int (*bio_ctr)(struct bio *, struct bio *, void *),
void *data)
{
struct bio *bio, *bio_src;
if (!bs)
bs = &fs_bio_set;
__rq_for_each_bio(bio_src, rq_src) {
bio = bio_alloc_clone(rq->q->disk->part0, bio_src, gfp_mask,
bs);
if (!bio)
goto free_and_out;
if (bio_ctr && bio_ctr(bio, bio_src, data))
goto free_and_out;
if (rq->bio) {
rq->biotail->bi_next = bio;
rq->biotail = bio;
} else {
rq->bio = rq->biotail = bio;
}
bio = NULL;
}
/* Copy attributes of the original request to the clone request. */
rq->__sector = blk_rq_pos(rq_src);
rq->__data_len = blk_rq_bytes(rq_src);
if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
rq->special_vec = rq_src->special_vec;
}
rq->nr_phys_segments = rq_src->nr_phys_segments;
rq->ioprio = rq_src->ioprio;
rq->write_hint = rq_src->write_hint;
if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
goto free_and_out;
return 0;
free_and_out:
if (bio)
bio_put(bio);
blk_rq_unprep_clone(rq);
return -ENOMEM;
}
EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
#endif /* CONFIG_BLK_MQ_STACKING */
/*
* Steal bios from a request and add them to a bio list.
* The request must not have been partially completed before.
*/
void blk_steal_bios(struct bio_list *list, struct request *rq)
{
if (rq->bio) {
if (list->tail)
list->tail->bi_next = rq->bio;
else
list->head = rq->bio;
list->tail = rq->biotail;
rq->bio = NULL;
rq->biotail = NULL;
}
rq->__data_len = 0;
}
EXPORT_SYMBOL_GPL(blk_steal_bios);
static size_t order_to_size(unsigned int order)
{
return (size_t)PAGE_SIZE << order;
}
/* called before freeing request pool in @tags */
static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
struct blk_mq_tags *tags)
{
struct page *page;
unsigned long flags;
/*
* There is no need to clear mapping if driver tags is not initialized
* or the mapping belongs to the driver tags.
*/
if (!drv_tags || drv_tags == tags)
return;
list_for_each_entry(page, &tags->page_list, lru) { unsigned long start = (unsigned long)page_address(page); unsigned long end = start + order_to_size(page->private);
int i;
for (i = 0; i < drv_tags->nr_tags; i++) { struct request *rq = drv_tags->rqs[i];
unsigned long rq_addr = (unsigned long)rq;
if (rq_addr >= start && rq_addr < end) { WARN_ON_ONCE(req_ref_read(rq) != 0); cmpxchg(&drv_tags->rqs[i], rq, NULL);
}
}
}
/*
* Wait until all pending iteration is done.
*
* Request reference is cleared and it is guaranteed to be observed
* after the ->lock is released.
*/
spin_lock_irqsave(&drv_tags->lock, flags);
spin_unlock_irqrestore(&drv_tags->lock, flags);
}
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx)
{
struct blk_mq_tags *drv_tags;
struct page *page;
if (list_empty(&tags->page_list))
return;
if (blk_mq_is_shared_tags(set->flags)) drv_tags = set->shared_tags;
else
drv_tags = set->tags[hctx_idx]; if (tags->static_rqs && set->ops->exit_request) {
int i;
for (i = 0; i < tags->nr_tags; i++) { struct request *rq = tags->static_rqs[i];
if (!rq)
continue;
set->ops->exit_request(set, rq, hctx_idx);
tags->static_rqs[i] = NULL;
}
}
blk_mq_clear_rq_mapping(drv_tags, tags); while (!list_empty(&tags->page_list)) { page = list_first_entry(&tags->page_list, struct page, lru); list_del_init(&page->lru);
/*
* Remove kmemleak object previously allocated in
* blk_mq_alloc_rqs().
*/
kmemleak_free(page_address(page));
__free_pages(page, page->private);
}
}
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
kfree(tags->rqs);
tags->rqs = NULL;
kfree(tags->static_rqs);
tags->static_rqs = NULL;
blk_mq_free_tags(tags);
}
static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
unsigned int hctx_idx)
{
int i;
for (i = 0; i < set->nr_maps; i++) {
unsigned int start = set->map[i].queue_offset;
unsigned int end = start + set->map[i].nr_queues;
if (hctx_idx >= start && hctx_idx < end)
break;
}
if (i >= set->nr_maps)
i = HCTX_TYPE_DEFAULT;
return i;
}
static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
unsigned int hctx_idx)
{
enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
}
static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
unsigned int hctx_idx,
unsigned int nr_tags,
unsigned int reserved_tags)
{
int node = blk_mq_get_hctx_node(set, hctx_idx);
struct blk_mq_tags *tags;
if (node == NUMA_NO_NODE)
node = set->numa_node;
tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
if (!tags)
return NULL;
tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
node);
if (!tags->rqs)
goto err_free_tags;
tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
node);
if (!tags->static_rqs)
goto err_free_rqs;
return tags;
err_free_rqs:
kfree(tags->rqs);
err_free_tags:
blk_mq_free_tags(tags);
return NULL;
}
static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
unsigned int hctx_idx, int node)
{
int ret;
if (set->ops->init_request) {
ret = set->ops->init_request(set, rq, hctx_idx, node);
if (ret)
return ret;
}
WRITE_ONCE(rq->state, MQ_RQ_IDLE);
return 0;
}
static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
struct blk_mq_tags *tags,
unsigned int hctx_idx, unsigned int depth)
{
unsigned int i, j, entries_per_page, max_order = 4;
int node = blk_mq_get_hctx_node(set, hctx_idx);
size_t rq_size, left;
if (node == NUMA_NO_NODE)
node = set->numa_node;
INIT_LIST_HEAD(&tags->page_list);
/*
* rq_size is the size of the request plus driver payload, rounded
* to the cacheline size
*/
rq_size = round_up(sizeof(struct request) + set->cmd_size,
cache_line_size());
left = rq_size * depth;
for (i = 0; i < depth; ) {
int this_order = max_order;
struct page *page;
int to_do;
void *p;
while (this_order && left < order_to_size(this_order - 1))
this_order--;
do {
page = alloc_pages_node(node,
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
this_order);
if (page)
break;
if (!this_order--)
break;
if (order_to_size(this_order) < rq_size)
break;
} while (1);
if (!page)
goto fail;
page->private = this_order;
list_add_tail(&page->lru, &tags->page_list);
p = page_address(page);
/*
* Allow kmemleak to scan these pages as they contain pointers
* to additional allocations like via ops->init_request().
*/
kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
entries_per_page = order_to_size(this_order) / rq_size;
to_do = min(entries_per_page, depth - i);
left -= to_do * rq_size;
for (j = 0; j < to_do; j++) {
struct request *rq = p;
tags->static_rqs[i] = rq;
if (blk_mq_init_request(set, rq, hctx_idx, node)) {
tags->static_rqs[i] = NULL;
goto fail;
}
p += rq_size;
i++;
}
}
return 0;
fail:
blk_mq_free_rqs(set, tags, hctx_idx);
return -ENOMEM;
}
struct rq_iter_data {
struct blk_mq_hw_ctx *hctx;
bool has_rq;
};
static bool blk_mq_has_request(struct request *rq, void *data)
{
struct rq_iter_data *iter_data = data;
if (rq->mq_hctx != iter_data->hctx)
return true;
iter_data->has_rq = true;
return false;
}
static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
{
struct blk_mq_tags *tags = hctx->sched_tags ?
hctx->sched_tags : hctx->tags;
struct rq_iter_data data = {
.hctx = hctx,
};
blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
return data.has_rq;
}
static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
struct blk_mq_hw_ctx *hctx)
{
if (cpumask_first_and(hctx->cpumask, cpu_online_mask) != cpu)
return false;
if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
return false;
return true;
}
static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
{
struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
struct blk_mq_hw_ctx, cpuhp_online);
if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
!blk_mq_last_cpu_in_hctx(cpu, hctx))
return 0;
/*
* Prevent new request from being allocated on the current hctx.
*
* The smp_mb__after_atomic() Pairs with the implied barrier in
* test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
* seen once we return from the tag allocator.
*/
set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
smp_mb__after_atomic();
/*
* Try to grab a reference to the queue and wait for any outstanding
* requests. If we could not grab a reference the queue has been
* frozen and there are no requests.
*/
if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
while (blk_mq_hctx_has_requests(hctx))
msleep(5);
percpu_ref_put(&hctx->queue->q_usage_counter);
}
return 0;
}
static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
{
struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
struct blk_mq_hw_ctx, cpuhp_online);
if (cpumask_test_cpu(cpu, hctx->cpumask))
clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
return 0;
}
/*
* 'cpu' is going away. splice any existing rq_list entries from this
* software queue to the hw queue dispatch list, and ensure that it
* gets run.
*/
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
{
struct blk_mq_hw_ctx *hctx;
struct blk_mq_ctx *ctx;
LIST_HEAD(tmp);
enum hctx_type type;
hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
if (!cpumask_test_cpu(cpu, hctx->cpumask))
return 0;
ctx = __blk_mq_get_ctx(hctx->queue, cpu);
type = hctx->type;
spin_lock(&ctx->lock);
if (!list_empty(&ctx->rq_lists[type])) {
list_splice_init(&ctx->rq_lists[type], &tmp);
blk_mq_hctx_clear_pending(hctx, ctx);
}
spin_unlock(&ctx->lock);
if (list_empty(&tmp))
return 0;
spin_lock(&hctx->lock);
list_splice_tail_init(&tmp, &hctx->dispatch);
spin_unlock(&hctx->lock);
blk_mq_run_hw_queue(hctx, true);
return 0;
}
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
{
if (!(hctx->flags & BLK_MQ_F_STACKING))
cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
&hctx->cpuhp_online);
cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
&hctx->cpuhp_dead);
}
/*
* Before freeing hw queue, clearing the flush request reference in
* tags->rqs[] for avoiding potential UAF.
*/
static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
unsigned int queue_depth, struct request *flush_rq)
{
int i;
unsigned long flags;
/* The hw queue may not be mapped yet */
if (!tags)
return;
WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
for (i = 0; i < queue_depth; i++)
cmpxchg(&tags->rqs[i], flush_rq, NULL);
/*
* Wait until all pending iteration is done.
*
* Request reference is cleared and it is guaranteed to be observed
* after the ->lock is released.
*/
spin_lock_irqsave(&tags->lock, flags);
spin_unlock_irqrestore(&tags->lock, flags);
}
/* hctx->ctxs will be freed in queue's release handler */
static void blk_mq_exit_hctx(struct request_queue *q,
struct blk_mq_tag_set *set,
struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
struct request *flush_rq = hctx->fq->flush_rq;
if (blk_mq_hw_queue_mapped(hctx))
blk_mq_tag_idle(hctx);
if (blk_queue_init_done(q))
blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
set->queue_depth, flush_rq);
if (set->ops->exit_request)
set->ops->exit_request(set, flush_rq, hctx_idx);
if (set->ops->exit_hctx)
set->ops->exit_hctx(hctx, hctx_idx);
blk_mq_remove_cpuhp(hctx);
xa_erase(&q->hctx_table, hctx_idx);
spin_lock(&q->unused_hctx_lock);
list_add(&hctx->hctx_list, &q->unused_hctx_list);
spin_unlock(&q->unused_hctx_lock);
}
static void blk_mq_exit_hw_queues(struct request_queue *q,
struct blk_mq_tag_set *set, int nr_queue)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i) {
if (i == nr_queue)
break;
blk_mq_exit_hctx(q, set, hctx, i);
}
}
static int blk_mq_init_hctx(struct request_queue *q,
struct blk_mq_tag_set *set,
struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
{
hctx->queue_num = hctx_idx;
if (!(hctx->flags & BLK_MQ_F_STACKING))
cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
&hctx->cpuhp_online);
cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
hctx->tags = set->tags[hctx_idx];
if (set->ops->init_hctx &&
set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
goto unregister_cpu_notifier;
if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
hctx->numa_node))
goto exit_hctx;
if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL))
goto exit_flush_rq;
return 0;
exit_flush_rq:
if (set->ops->exit_request)
set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
exit_hctx:
if (set->ops->exit_hctx)
set->ops->exit_hctx(hctx, hctx_idx);
unregister_cpu_notifier:
blk_mq_remove_cpuhp(hctx);
return -1;
}
static struct blk_mq_hw_ctx *
blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
int node)
{
struct blk_mq_hw_ctx *hctx;
gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
if (!hctx)
goto fail_alloc_hctx;
if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
goto free_hctx;
atomic_set(&hctx->nr_active, 0);
if (node == NUMA_NO_NODE)
node = set->numa_node;
hctx->numa_node = node;
INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
spin_lock_init(&hctx->lock);
INIT_LIST_HEAD(&hctx->dispatch);
hctx->queue = q;
hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
INIT_LIST_HEAD(&hctx->hctx_list);
/*
* Allocate space for all possible cpus to avoid allocation at
* runtime
*/
hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
gfp, node);
if (!hctx->ctxs)
goto free_cpumask;
if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
gfp, node, false, false))
goto free_ctxs;
hctx->nr_ctx = 0;
spin_lock_init(&hctx->dispatch_wait_lock);
init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
if (!hctx->fq)
goto free_bitmap;
blk_mq_hctx_kobj_init(hctx);
return hctx;
free_bitmap:
sbitmap_free(&hctx->ctx_map);
free_ctxs:
kfree(hctx->ctxs);
free_cpumask:
free_cpumask_var(hctx->cpumask);
free_hctx:
kfree(hctx);
fail_alloc_hctx:
return NULL;
}
static void blk_mq_init_cpu_queues(struct request_queue *q,
unsigned int nr_hw_queues)
{
struct blk_mq_tag_set *set = q->tag_set;
unsigned int i, j;
for_each_possible_cpu(i) {
struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
struct blk_mq_hw_ctx *hctx;
int k;
__ctx->cpu = i;
spin_lock_init(&__ctx->lock);
for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
INIT_LIST_HEAD(&__ctx->rq_lists[k]);
__ctx->queue = q;
/*
* Set local node, IFF we have more than one hw queue. If
* not, we remain on the home node of the device
*/
for (j = 0; j < set->nr_maps; j++) {
hctx = blk_mq_map_queue_type(q, j, i);
if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
hctx->numa_node = cpu_to_node(i);
}
}
}
struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
unsigned int hctx_idx,
unsigned int depth)
{
struct blk_mq_tags *tags;
int ret;
tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
if (!tags)
return NULL;
ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
if (ret) {
blk_mq_free_rq_map(tags);
return NULL;
}
return tags;
}
static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
int hctx_idx)
{
if (blk_mq_is_shared_tags(set->flags)) {
set->tags[hctx_idx] = set->shared_tags;
return true;
}
set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
set->queue_depth);
return set->tags[hctx_idx];
}
void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
struct blk_mq_tags *tags,
unsigned int hctx_idx)
{
if (tags) {
blk_mq_free_rqs(set, tags, hctx_idx);
blk_mq_free_rq_map(tags);
}
}
static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
unsigned int hctx_idx)
{
if (!blk_mq_is_shared_tags(set->flags)) blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx); set->tags[hctx_idx] = NULL;
}
static void blk_mq_map_swqueue(struct request_queue *q)
{
unsigned int j, hctx_idx;
unsigned long i;
struct blk_mq_hw_ctx *hctx;
struct blk_mq_ctx *ctx;
struct blk_mq_tag_set *set = q->tag_set;
queue_for_each_hw_ctx(q, hctx, i) {
cpumask_clear(hctx->cpumask);
hctx->nr_ctx = 0;
hctx->dispatch_from = NULL;
}
/*
* Map software to hardware queues.
*
* If the cpu isn't present, the cpu is mapped to first hctx.
*/
for_each_possible_cpu(i) {
ctx = per_cpu_ptr(q->queue_ctx, i);
for (j = 0; j < set->nr_maps; j++) {
if (!set->map[j].nr_queues) {
ctx->hctxs[j] = blk_mq_map_queue_type(q,
HCTX_TYPE_DEFAULT, i);
continue;
}
hctx_idx = set->map[j].mq_map[i];
/* unmapped hw queue can be remapped after CPU topo changed */
if (!set->tags[hctx_idx] &&
!__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
/*
* If tags initialization fail for some hctx,
* that hctx won't be brought online. In this
* case, remap the current ctx to hctx[0] which
* is guaranteed to always have tags allocated
*/
set->map[j].mq_map[i] = 0;
}
hctx = blk_mq_map_queue_type(q, j, i);
ctx->hctxs[j] = hctx;
/*
* If the CPU is already set in the mask, then we've
* mapped this one already. This can happen if
* devices share queues across queue maps.
*/
if (cpumask_test_cpu(i, hctx->cpumask))
continue;
cpumask_set_cpu(i, hctx->cpumask);
hctx->type = j;
ctx->index_hw[hctx->type] = hctx->nr_ctx;
hctx->ctxs[hctx->nr_ctx++] = ctx;
/*
* If the nr_ctx type overflows, we have exceeded the
* amount of sw queues we can support.
*/
BUG_ON(!hctx->nr_ctx);
}
for (; j < HCTX_MAX_TYPES; j++)
ctx->hctxs[j] = blk_mq_map_queue_type(q,
HCTX_TYPE_DEFAULT, i);
}
queue_for_each_hw_ctx(q, hctx, i) {
/*
* If no software queues are mapped to this hardware queue,
* disable it and free the request entries.
*/
if (!hctx->nr_ctx) {
/* Never unmap queue 0. We need it as a
* fallback in case of a new remap fails
* allocation
*/
if (i)
__blk_mq_free_map_and_rqs(set, i);
hctx->tags = NULL;
continue;
}
hctx->tags = set->tags[i];
WARN_ON(!hctx->tags);
/*
* Set the map size to the number of mapped software queues.
* This is more accurate and more efficient than looping
* over all possibly mapped software queues.
*/
sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
/*
* Initialize batch roundrobin counts
*/
hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
}
}
/*
* Caller needs to ensure that we're either frozen/quiesced, or that
* the queue isn't live yet.
*/
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i) {
if (shared) {
hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
} else {
blk_mq_tag_idle(hctx);
hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
}
}
}
static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
bool shared)
{
struct request_queue *q;
lockdep_assert_held(&set->tag_list_lock);
list_for_each_entry(q, &set->tag_list, tag_set_list) {
blk_mq_freeze_queue(q);
queue_set_hctx_shared(q, shared);
blk_mq_unfreeze_queue(q);
}
}
static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
struct blk_mq_tag_set *set = q->tag_set;
mutex_lock(&set->tag_list_lock);
list_del(&q->tag_set_list);
if (list_is_singular(&set->tag_list)) {
/* just transitioned to unshared */
set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
/* update existing queue */
blk_mq_update_tag_set_shared(set, false);
}
mutex_unlock(&set->tag_list_lock);
INIT_LIST_HEAD(&q->tag_set_list);
}
static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
struct request_queue *q)
{
mutex_lock(&set->tag_list_lock);
/*
* Check to see if we're transitioning to shared (from 1 to 2 queues).
*/
if (!list_empty(&set->tag_list) &&
!(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
/* update existing queue */
blk_mq_update_tag_set_shared(set, true);
}
if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
queue_set_hctx_shared(q, true);
list_add_tail(&q->tag_set_list, &set->tag_list);
mutex_unlock(&set->tag_list_lock);
}
/* All allocations will be freed in release handler of q->mq_kobj */
static int blk_mq_alloc_ctxs(struct request_queue *q)
{
struct blk_mq_ctxs *ctxs;
int cpu;
ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
if (!ctxs)
return -ENOMEM;
ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
if (!ctxs->queue_ctx)
goto fail;
for_each_possible_cpu(cpu) {
struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
ctx->ctxs = ctxs;
}
q->mq_kobj = &ctxs->kobj;
q->queue_ctx = ctxs->queue_ctx;
return 0;
fail:
kfree(ctxs);
return -ENOMEM;
}
/*
* It is the actual release handler for mq, but we do it from
* request queue's release handler for avoiding use-after-free
* and headache because q->mq_kobj shouldn't have been introduced,
* but we can't group ctx/kctx kobj without it.
*/
void blk_mq_release(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx, *next;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i)
WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
/* all hctx are in .unused_hctx_list now */
list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
list_del_init(&hctx->hctx_list);
kobject_put(&hctx->kobj);
}
xa_destroy(&q->hctx_table);
/*
* release .mq_kobj and sw queue's kobject now because
* both share lifetime with request queue.
*/
blk_mq_sysfs_deinit(q);
}
struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set,
struct queue_limits *lim, void *queuedata)
{
struct queue_limits default_lim = { };
struct request_queue *q;
int ret;
q = blk_alloc_queue(lim ? lim : &default_lim, set->numa_node);
if (IS_ERR(q))
return q;
q->queuedata = queuedata;
ret = blk_mq_init_allocated_queue(set, q);
if (ret) {
blk_put_queue(q);
return ERR_PTR(ret);
}
return q;
}
EXPORT_SYMBOL(blk_mq_alloc_queue);
/**
* blk_mq_destroy_queue - shutdown a request queue
* @q: request queue to shutdown
*
* This shuts down a request queue allocated by blk_mq_alloc_queue(). All future
* requests will be failed with -ENODEV. The caller is responsible for dropping
* the reference from blk_mq_alloc_queue() by calling blk_put_queue().
*
* Context: can sleep
*/
void blk_mq_destroy_queue(struct request_queue *q)
{
WARN_ON_ONCE(!queue_is_mq(q));
WARN_ON_ONCE(blk_queue_registered(q));
might_sleep();
blk_queue_flag_set(QUEUE_FLAG_DYING, q);
blk_queue_start_drain(q);
blk_mq_freeze_queue_wait(q);
blk_sync_queue(q);
blk_mq_cancel_work_sync(q);
blk_mq_exit_queue(q);
}
EXPORT_SYMBOL(blk_mq_destroy_queue);
struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,
struct queue_limits *lim, void *queuedata,
struct lock_class_key *lkclass)
{
struct request_queue *q;
struct gendisk *disk;
q = blk_mq_alloc_queue(set, lim, queuedata);
if (IS_ERR(q))
return ERR_CAST(q);
disk = __alloc_disk_node(q, set->numa_node, lkclass);
if (!disk) {
blk_mq_destroy_queue(q);
blk_put_queue(q);
return ERR_PTR(-ENOMEM);
}
set_bit(GD_OWNS_QUEUE, &disk->state);
return disk;
}
EXPORT_SYMBOL(__blk_mq_alloc_disk);
struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
struct lock_class_key *lkclass)
{
struct gendisk *disk;
if (!blk_get_queue(q))
return NULL;
disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
if (!disk)
blk_put_queue(q);
return disk;
}
EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
struct blk_mq_tag_set *set, struct request_queue *q,
int hctx_idx, int node)
{
struct blk_mq_hw_ctx *hctx = NULL, *tmp;
/* reuse dead hctx first */
spin_lock(&q->unused_hctx_lock);
list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
if (tmp->numa_node == node) {
hctx = tmp;
break;
}
}
if (hctx)
list_del_init(&hctx->hctx_list);
spin_unlock(&q->unused_hctx_lock);
if (!hctx)
hctx = blk_mq_alloc_hctx(q, set, node);
if (!hctx)
goto fail;
if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
goto free_hctx;
return hctx;
free_hctx:
kobject_put(&hctx->kobj);
fail:
return NULL;
}
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i, j;
/* protect against switching io scheduler */
mutex_lock(&q->sysfs_lock);
for (i = 0; i < set->nr_hw_queues; i++) {
int old_node;
int node = blk_mq_get_hctx_node(set, i);
struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);
if (old_hctx) {
old_node = old_hctx->numa_node;
blk_mq_exit_hctx(q, set, old_hctx, i);
}
if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) {
if (!old_hctx)
break;
pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
node, old_node);
hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node);
WARN_ON_ONCE(!hctx);
}
}
/*
* Increasing nr_hw_queues fails. Free the newly allocated
* hctxs and keep the previous q->nr_hw_queues.
*/
if (i != set->nr_hw_queues) {
j = q->nr_hw_queues;
} else {
j = i;
q->nr_hw_queues = set->nr_hw_queues;
}
xa_for_each_start(&q->hctx_table, j, hctx, j)
blk_mq_exit_hctx(q, set, hctx, j);
mutex_unlock(&q->sysfs_lock);
}
static void blk_mq_update_poll_flag(struct request_queue *q)
{
struct blk_mq_tag_set *set = q->tag_set;
if (set->nr_maps > HCTX_TYPE_POLL &&
set->map[HCTX_TYPE_POLL].nr_queues)
blk_queue_flag_set(QUEUE_FLAG_POLL, q);
else
blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
}
int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
struct request_queue *q)
{
/* mark the queue as mq asap */
q->mq_ops = set->ops;
if (blk_mq_alloc_ctxs(q))
goto err_exit;
/* init q->mq_kobj and sw queues' kobjects */
blk_mq_sysfs_init(q);
INIT_LIST_HEAD(&q->unused_hctx_list);
spin_lock_init(&q->unused_hctx_lock);
xa_init(&q->hctx_table);
blk_mq_realloc_hw_ctxs(set, q);
if (!q->nr_hw_queues)
goto err_hctxs;
INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
q->tag_set = set;
q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
blk_mq_update_poll_flag(q);
INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
INIT_LIST_HEAD(&q->flush_list);
INIT_LIST_HEAD(&q->requeue_list);
spin_lock_init(&q->requeue_lock);
q->nr_requests = set->queue_depth;
blk_mq_init_cpu_queues(q, set->nr_hw_queues);
blk_mq_add_queue_tag_set(set, q);
blk_mq_map_swqueue(q);
return 0;
err_hctxs:
blk_mq_release(q);
err_exit:
q->mq_ops = NULL;
return -ENOMEM;
}
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
/* tags can _not_ be used after returning from blk_mq_exit_queue */
void blk_mq_exit_queue(struct request_queue *q)
{
struct blk_mq_tag_set *set = q->tag_set;
/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
blk_mq_del_queue_tag_set(q);
}
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
int i;
if (blk_mq_is_shared_tags(set->flags)) {
set->shared_tags = blk_mq_alloc_map_and_rqs(set,
BLK_MQ_NO_HCTX_IDX,
set->queue_depth);
if (!set->shared_tags)
return -ENOMEM;
}
for (i = 0; i < set->nr_hw_queues; i++) {
if (!__blk_mq_alloc_map_and_rqs(set, i))
goto out_unwind;
cond_resched();
}
return 0;
out_unwind:
while (--i >= 0)
__blk_mq_free_map_and_rqs(set, i);
if (blk_mq_is_shared_tags(set->flags)) {
blk_mq_free_map_and_rqs(set, set->shared_tags,
BLK_MQ_NO_HCTX_IDX);
}
return -ENOMEM;
}
/*
* Allocate the request maps associated with this tag_set. Note that this
* may reduce the depth asked for, if memory is tight. set->queue_depth
* will be updated to reflect the allocated depth.
*/
static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
{
unsigned int depth;
int err;
depth = set->queue_depth;
do {
err = __blk_mq_alloc_rq_maps(set);
if (!err)
break;
set->queue_depth >>= 1;
if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
err = -ENOMEM;
break;
}
} while (set->queue_depth);
if (!set->queue_depth || err) {
pr_err("blk-mq: failed to allocate request map\n");
return -ENOMEM;
}
if (depth != set->queue_depth)
pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
depth, set->queue_depth);
return 0;
}
static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
{
/*
* blk_mq_map_queues() and multiple .map_queues() implementations
* expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
* number of hardware queues.
*/
if (set->nr_maps == 1)
set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
if (set->ops->map_queues) {
int i;
/*
* transport .map_queues is usually done in the following
* way:
*
* for (queue = 0; queue < set->nr_hw_queues; queue++) {
* mask = get_cpu_mask(queue)
* for_each_cpu(cpu, mask)
* set->map[x].mq_map[cpu] = queue;
* }
*
* When we need to remap, the table has to be cleared for
* killing stale mapping since one CPU may not be mapped
* to any hw queue.
*/
for (i = 0; i < set->nr_maps; i++)
blk_mq_clear_mq_map(&set->map[i]);
set->ops->map_queues(set);
} else {
BUG_ON(set->nr_maps > 1);
blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
}
}
static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
int new_nr_hw_queues)
{
struct blk_mq_tags **new_tags;
int i;
if (set->nr_hw_queues >= new_nr_hw_queues)
goto done;
new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
GFP_KERNEL, set->numa_node);
if (!new_tags)
return -ENOMEM;
if (set->tags)
memcpy(new_tags, set->tags, set->nr_hw_queues *
sizeof(*set->tags));
kfree(set->tags);
set->tags = new_tags;
for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
if (!__blk_mq_alloc_map_and_rqs(set, i)) {
while (--i >= set->nr_hw_queues)
__blk_mq_free_map_and_rqs(set, i);
return -ENOMEM;
}
cond_resched();
}
done:
set->nr_hw_queues = new_nr_hw_queues;
return 0;
}
/*
* Alloc a tag set to be associated with one or more request queues.
* May fail with EINVAL for various error conditions. May adjust the
* requested depth down, if it's too large. In that case, the set
* value will be stored in set->queue_depth.
*/
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
int i, ret;
BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
if (!set->nr_hw_queues)
return -EINVAL;
if (!set->queue_depth)
return -EINVAL;
if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
return -EINVAL;
if (!set->ops->queue_rq)
return -EINVAL;
if (!set->ops->get_budget ^ !set->ops->put_budget)
return -EINVAL;
if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
pr_info("blk-mq: reduced tag depth to %u\n",
BLK_MQ_MAX_DEPTH);
set->queue_depth = BLK_MQ_MAX_DEPTH;
}
if (!set->nr_maps)
set->nr_maps = 1;
else if (set->nr_maps > HCTX_MAX_TYPES)
return -EINVAL;
/*
* If a crashdump is active, then we are potentially in a very
* memory constrained environment. Limit us to 64 tags to prevent
* using too much memory.
*/
if (is_kdump_kernel())
set->queue_depth = min(64U, set->queue_depth);
/*
* There is no use for more h/w queues than cpus if we just have
* a single map
*/
if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
set->nr_hw_queues = nr_cpu_ids;
if (set->flags & BLK_MQ_F_BLOCKING) {
set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL);
if (!set->srcu)
return -ENOMEM;
ret = init_srcu_struct(set->srcu);
if (ret)
goto out_free_srcu;
}
ret = -ENOMEM;
set->tags = kcalloc_node(set->nr_hw_queues,
sizeof(struct blk_mq_tags *), GFP_KERNEL,
set->numa_node);
if (!set->tags)
goto out_cleanup_srcu;
for (i = 0; i < set->nr_maps; i++) {
set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
sizeof(set->map[i].mq_map[0]),
GFP_KERNEL, set->numa_node);
if (!set->map[i].mq_map)
goto out_free_mq_map;
set->map[i].nr_queues = set->nr_hw_queues;
}
blk_mq_update_queue_map(set);
ret = blk_mq_alloc_set_map_and_rqs(set);
if (ret)
goto out_free_mq_map;
mutex_init(&set->tag_list_lock);
INIT_LIST_HEAD(&set->tag_list);
return 0;
out_free_mq_map:
for (i = 0; i < set->nr_maps; i++) {
kfree(set->map[i].mq_map);
set->map[i].mq_map = NULL;
}
kfree(set->tags);
set->tags = NULL;
out_cleanup_srcu:
if (set->flags & BLK_MQ_F_BLOCKING)
cleanup_srcu_struct(set->srcu);
out_free_srcu:
if (set->flags & BLK_MQ_F_BLOCKING)
kfree(set->srcu);
return ret;
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);
/* allocate and initialize a tagset for a simple single-queue device */
int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
const struct blk_mq_ops *ops, unsigned int queue_depth,
unsigned int set_flags)
{
memset(set, 0, sizeof(*set));
set->ops = ops;
set->nr_hw_queues = 1;
set->nr_maps = 1;
set->queue_depth = queue_depth;
set->numa_node = NUMA_NO_NODE;
set->flags = set_flags;
return blk_mq_alloc_tag_set(set);
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
int i, j;
for (i = 0; i < set->nr_hw_queues; i++) __blk_mq_free_map_and_rqs(set, i); if (blk_mq_is_shared_tags(set->flags)) { blk_mq_free_map_and_rqs(set, set->shared_tags,
BLK_MQ_NO_HCTX_IDX);
}
for (j = 0; j < set->nr_maps; j++) { kfree(set->map[j].mq_map);
set->map[j].mq_map = NULL;
}
kfree(set->tags);
set->tags = NULL;
if (set->flags & BLK_MQ_F_BLOCKING) {
cleanup_srcu_struct(set->srcu);
kfree(set->srcu);
}
}
EXPORT_SYMBOL(blk_mq_free_tag_set);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
struct blk_mq_tag_set *set = q->tag_set;
struct blk_mq_hw_ctx *hctx;
int ret;
unsigned long i;
if (!set)
return -EINVAL;
if (q->nr_requests == nr)
return 0;
blk_mq_freeze_queue(q);
blk_mq_quiesce_queue(q);
ret = 0;
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx->tags)
continue;
/*
* If we're using an MQ scheduler, just update the scheduler
* queue depth. This is similar to what the old code would do.
*/
if (hctx->sched_tags) {
ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
nr, true);
} else {
ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
false);
}
if (ret)
break;
if (q->elevator && q->elevator->type->ops.depth_updated)
q->elevator->type->ops.depth_updated(hctx);
}
if (!ret) {
q->nr_requests = nr;
if (blk_mq_is_shared_tags(set->flags)) {
if (q->elevator)
blk_mq_tag_update_sched_shared_tags(q);
else
blk_mq_tag_resize_shared_tags(set, nr);
}
}
blk_mq_unquiesce_queue(q);
blk_mq_unfreeze_queue(q);
return ret;
}
/*
* request_queue and elevator_type pair.
* It is just used by __blk_mq_update_nr_hw_queues to cache
* the elevator_type associated with a request_queue.
*/
struct blk_mq_qe_pair {
struct list_head node;
struct request_queue *q;
struct elevator_type *type;
};
/*
* Cache the elevator_type in qe pair list and switch the
* io scheduler to 'none'
*/
static bool blk_mq_elv_switch_none(struct list_head *head,
struct request_queue *q)
{
struct blk_mq_qe_pair *qe;
qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
if (!qe)
return false;
/* q->elevator needs protection from ->sysfs_lock */
mutex_lock(&q->sysfs_lock);
/* the check has to be done with holding sysfs_lock */
if (!q->elevator) {
kfree(qe);
goto unlock;
}
INIT_LIST_HEAD(&qe->node);
qe->q = q;
qe->type = q->elevator->type;
/* keep a reference to the elevator module as we'll switch back */
__elevator_get(qe->type);
list_add(&qe->node, head);
elevator_disable(q);
unlock:
mutex_unlock(&q->sysfs_lock);
return true;
}
static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head,
struct request_queue *q)
{
struct blk_mq_qe_pair *qe;
list_for_each_entry(qe, head, node)
if (qe->q == q)
return qe;
return NULL;
}
static void blk_mq_elv_switch_back(struct list_head *head,
struct request_queue *q)
{
struct blk_mq_qe_pair *qe;
struct elevator_type *t;
qe = blk_lookup_qe_pair(head, q);
if (!qe)
return;
t = qe->type;
list_del(&qe->node);
kfree(qe);
mutex_lock(&q->sysfs_lock);
elevator_switch(q, t);
/* drop the reference acquired in blk_mq_elv_switch_none */
elevator_put(t);
mutex_unlock(&q->sysfs_lock);
}
static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
int nr_hw_queues)
{
struct request_queue *q;
LIST_HEAD(head);
int prev_nr_hw_queues = set->nr_hw_queues;
int i;
lockdep_assert_held(&set->tag_list_lock);
if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
nr_hw_queues = nr_cpu_ids;
if (nr_hw_queues < 1)
return;
if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
return;
list_for_each_entry(q, &set->tag_list, tag_set_list)
blk_mq_freeze_queue(q);
/*
* Switch IO scheduler to 'none', cleaning up the data associated
* with the previous scheduler. We will switch back once we are done
* updating the new sw to hw queue mappings.
*/
list_for_each_entry(q, &set->tag_list, tag_set_list)
if (!blk_mq_elv_switch_none(&head, q))
goto switch_back;
list_for_each_entry(q, &set->tag_list, tag_set_list) {
blk_mq_debugfs_unregister_hctxs(q);
blk_mq_sysfs_unregister_hctxs(q);
}
if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0)
goto reregister;
fallback:
blk_mq_update_queue_map(set);
list_for_each_entry(q, &set->tag_list, tag_set_list) {
blk_mq_realloc_hw_ctxs(set, q);
blk_mq_update_poll_flag(q);
if (q->nr_hw_queues != set->nr_hw_queues) {
int i = prev_nr_hw_queues;
pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
nr_hw_queues, prev_nr_hw_queues);
for (; i < set->nr_hw_queues; i++)
__blk_mq_free_map_and_rqs(set, i);
set->nr_hw_queues = prev_nr_hw_queues;
goto fallback;
}
blk_mq_map_swqueue(q);
}
reregister:
list_for_each_entry(q, &set->tag_list, tag_set_list) {
blk_mq_sysfs_register_hctxs(q);
blk_mq_debugfs_register_hctxs(q);
}
switch_back:
list_for_each_entry(q, &set->tag_list, tag_set_list)
blk_mq_elv_switch_back(&head, q);
list_for_each_entry(q, &set->tag_list, tag_set_list)
blk_mq_unfreeze_queue(q);
/* Free the excess tags when nr_hw_queues shrink. */
for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
__blk_mq_free_map_and_rqs(set, i);
}
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
mutex_lock(&set->tag_list_lock);
__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
mutex_unlock(&set->tag_list_lock);
}
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
struct io_comp_batch *iob, unsigned int flags)
{
long state = get_current_state();
int ret;
do {
ret = q->mq_ops->poll(hctx, iob);
if (ret > 0) {
__set_current_state(TASK_RUNNING);
return ret;
}
if (signal_pending_state(state, current))
__set_current_state(TASK_RUNNING);
if (task_is_running(current))
return 1;
if (ret < 0 || (flags & BLK_POLL_ONESHOT))
break;
cpu_relax();
} while (!need_resched());
__set_current_state(TASK_RUNNING);
return 0;
}
int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
struct io_comp_batch *iob, unsigned int flags)
{
struct blk_mq_hw_ctx *hctx = xa_load(&q->hctx_table, cookie);
return blk_hctx_poll(q, hctx, iob, flags);
}
int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
unsigned int poll_flags)
{
struct request_queue *q = rq->q;
int ret;
if (!blk_rq_is_poll(rq))
return 0;
if (!percpu_ref_tryget(&q->q_usage_counter))
return 0;
ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
blk_queue_exit(q);
return ret;
}
EXPORT_SYMBOL_GPL(blk_rq_poll);
unsigned int blk_mq_rq_cpu(struct request *rq)
{
return rq->mq_ctx->cpu;
}
EXPORT_SYMBOL(blk_mq_rq_cpu);
void blk_mq_cancel_work_sync(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
unsigned long i;
cancel_delayed_work_sync(&q->requeue_work);
queue_for_each_hw_ctx(q, hctx, i)
cancel_delayed_work_sync(&hctx->run_work);
}
static int __init blk_mq_init(void)
{
int i;
for_each_possible_cpu(i)
init_llist_head(&per_cpu(blk_cpu_done, i));
for_each_possible_cpu(i)
INIT_CSD(&per_cpu(blk_cpu_csd, i),
__blk_mq_complete_request_remote, NULL);
open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
"block/softirq:dead", NULL,
blk_softirq_cpu_dead);
cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
blk_mq_hctx_notify_dead);
cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
blk_mq_hctx_notify_online,
blk_mq_hctx_notify_offline);
return 0;
}
subsys_initcall(blk_mq_init);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 1994 Linus Torvalds
*
* Pentium III FXSR, SSE support
* General FPU state handling cleanups
* Gareth Hughes <gareth@valinux.com>, May 2000
* x86-64 work by Andi Kleen 2002
*/
#ifndef _ASM_X86_FPU_API_H
#define _ASM_X86_FPU_API_H
#include <linux/bottom_half.h>
#include <asm/fpu/types.h>
/*
* Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It
* disables preemption so be careful if you intend to use it for long periods
* of time.
* If you intend to use the FPU in irq/softirq you need to check first with
* irq_fpu_usable() if it is possible.
*/
/* Kernel FPU states to initialize in kernel_fpu_begin_mask() */
#define KFPU_387 _BITUL(0) /* 387 state will be initialized */
#define KFPU_MXCSR _BITUL(1) /* MXCSR will be initialized */
extern void kernel_fpu_begin_mask(unsigned int kfpu_mask);
extern void kernel_fpu_end(void);
extern bool irq_fpu_usable(void);
extern void fpregs_mark_activate(void);
/* Code that is unaware of kernel_fpu_begin_mask() can use this */
static inline void kernel_fpu_begin(void)
{
#ifdef CONFIG_X86_64
/*
* Any 64-bit code that uses 387 instructions must explicitly request
* KFPU_387.
*/
kernel_fpu_begin_mask(KFPU_MXCSR);
#else
/*
* 32-bit kernel code may use 387 operations as well as SSE2, etc,
* as long as it checks that the CPU has the required capability.
*/
kernel_fpu_begin_mask(KFPU_387 | KFPU_MXCSR);
#endif
}
/*
* Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate.
* A context switch will (and softirq might) save CPU's FPU registers to
* fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in
* a random state.
*
* local_bh_disable() protects against both preemption and soft interrupts
* on !RT kernels.
*
* On RT kernels local_bh_disable() is not sufficient because it only
* serializes soft interrupt related sections via a local lock, but stays
* preemptible. Disabling preemption is the right choice here as bottom
* half processing is always in thread context on RT kernels so it
* implicitly prevents bottom half processing as well.
*
* Disabling preemption also serializes against kernel_fpu_begin().
*/
static inline void fpregs_lock(void)
{
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
local_bh_disable();
else
preempt_disable();
}
static inline void fpregs_unlock(void)
{
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
local_bh_enable();
else
preempt_enable();
}
/*
* FPU state gets lazily restored before returning to userspace. So when in the
* kernel, the valid FPU state may be kept in the buffer. This function will force
* restore all the fpu state to the registers early if needed, and lock them from
* being automatically saved/restored. Then FPU state can be modified safely in the
* registers, before unlocking with fpregs_unlock().
*/
void fpregs_lock_and_load(void);
#ifdef CONFIG_X86_DEBUG_FPU
extern void fpregs_assert_state_consistent(void);
#else
static inline void fpregs_assert_state_consistent(void) { }
#endif
/*
* Load the task FPU state before returning to userspace.
*/
extern void switch_fpu_return(void);
/*
* Query the presence of one or more xfeatures. Works on any legacy CPU as well.
*
* If 'feature_name' is set then put a human-readable description of
* the feature there as well - this can be used to print error (or success)
* messages.
*/
extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);
/* Trap handling */
extern int fpu__exception_code(struct fpu *fpu, int trap_nr);
extern void fpu_sync_fpstate(struct fpu *fpu);
extern void fpu_reset_from_exception_fixup(void);
/* Boot, hotplug and resume */
extern void fpu__init_cpu(void);
extern void fpu__init_system(void);
extern void fpu__init_check_bugs(void);
extern void fpu__resume_cpu(void);
#ifdef CONFIG_MATH_EMULATION
extern void fpstate_init_soft(struct swregs_state *soft);
#else
static inline void fpstate_init_soft(struct swregs_state *soft) {}
#endif
/* State tracking */
DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
/* Process cleanup */
#ifdef CONFIG_X86_64
extern void fpstate_free(struct fpu *fpu);
#else
static inline void fpstate_free(struct fpu *fpu) { }
#endif
/* fpstate-related functions which are exported to KVM */
extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature);
extern u64 xstate_get_guest_group_perm(void);
/* KVM specific functions */
extern bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu);
extern void fpu_free_guest_fpstate(struct fpu_guest *gfpu);
extern int fpu_swap_kvm_fpstate(struct fpu_guest *gfpu, bool enter_guest);
extern int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures);
#ifdef CONFIG_X86_64
extern void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd);
extern void fpu_sync_guest_vmexit_xfd_state(void);
#else
static inline void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) { }
static inline void fpu_sync_guest_vmexit_xfd_state(void) { }
#endif
extern void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf,
unsigned int size, u64 xfeatures, u32 pkru);
extern int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, u64 xcr0, u32 *vpkru);
static inline void fpstate_set_confidential(struct fpu_guest *gfpu)
{
gfpu->fpstate->is_confidential = true;
}
static inline bool fpstate_is_confidential(struct fpu_guest *gfpu)
{
return gfpu->fpstate->is_confidential;
}
/* prctl */
extern long fpu_xstate_prctl(int option, unsigned long arg2);
extern void fpu_idle_fpregs(void);
#endif /* _ASM_X86_FPU_API_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* fs/kernfs/file.c - kernfs file implementation
*
* Copyright (c) 2001-3 Patrick Mochel
* Copyright (c) 2007 SUSE Linux Products GmbH
* Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
*/
#include <linux/fs.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/pagemap.h>
#include <linux/sched/mm.h>
#include <linux/fsnotify.h>
#include <linux/uio.h>
#include "kernfs-internal.h"
struct kernfs_open_node {
struct rcu_head rcu_head;
atomic_t event;
wait_queue_head_t poll;
struct list_head files; /* goes through kernfs_open_file.list */
unsigned int nr_mmapped;
unsigned int nr_to_release;
};
/*
* kernfs_notify() may be called from any context and bounces notifications
* through a work item. To minimize space overhead in kernfs_node, the
* pending queue is implemented as a singly linked list of kernfs_nodes.
* The list is terminated with the self pointer so that whether a
* kernfs_node is on the list or not can be determined by testing the next
* pointer for %NULL.
*/
#define KERNFS_NOTIFY_EOL ((void *)&kernfs_notify_list)
static DEFINE_SPINLOCK(kernfs_notify_lock);
static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL;
static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn)
{
int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS);
return &kernfs_locks->open_file_mutex[idx];
}
static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn)
{
struct mutex *lock;
lock = kernfs_open_file_mutex_ptr(kn);
mutex_lock(lock);
return lock;
}
/**
* of_on - Get the kernfs_open_node of the specified kernfs_open_file
* @of: target kernfs_open_file
*
* Return: the kernfs_open_node of the kernfs_open_file
*/
static struct kernfs_open_node *of_on(struct kernfs_open_file *of)
{
return rcu_dereference_protected(of->kn->attr.open,
!list_empty(&of->list));
}
/**
* kernfs_deref_open_node_locked - Get kernfs_open_node corresponding to @kn
*
* @kn: target kernfs_node.
*
* Fetch and return ->attr.open of @kn when caller holds the
* kernfs_open_file_mutex_ptr(kn).
*
* Update of ->attr.open happens under kernfs_open_file_mutex_ptr(kn). So when
* the caller guarantees that this mutex is being held, other updaters can't
* change ->attr.open and this means that we can safely deref ->attr.open
* outside RCU read-side critical section.
*
* The caller needs to make sure that kernfs_open_file_mutex is held.
*
* Return: @kn->attr.open when kernfs_open_file_mutex is held.
*/
static struct kernfs_open_node *
kernfs_deref_open_node_locked(struct kernfs_node *kn)
{
return rcu_dereference_protected(kn->attr.open,
lockdep_is_held(kernfs_open_file_mutex_ptr(kn)));
}
static struct kernfs_open_file *kernfs_of(struct file *file)
{
return ((struct seq_file *)file->private_data)->private;
}
/*
* Determine the kernfs_ops for the given kernfs_node. This function must
* be called while holding an active reference.
*/
static const struct kernfs_ops *kernfs_ops(struct kernfs_node *kn)
{
if (kn->flags & KERNFS_LOCKDEP)
lockdep_assert_held(kn);
return kn->attr.ops;
}
/*
* As kernfs_seq_stop() is also called after kernfs_seq_start() or
* kernfs_seq_next() failure, it needs to distinguish whether it's stopping
* a seq_file iteration which is fully initialized with an active reference
* or an aborted kernfs_seq_start() due to get_active failure. The
* position pointer is the only context for each seq_file iteration and
* thus the stop condition should be encoded in it. As the return value is
* directly visible to userland, ERR_PTR(-ENODEV) is the only acceptable
* choice to indicate get_active failure.
*
* Unfortunately, this is complicated due to the optional custom seq_file
* operations which may return ERR_PTR(-ENODEV) too. kernfs_seq_stop()
* can't distinguish whether ERR_PTR(-ENODEV) is from get_active failure or
* custom seq_file operations and thus can't decide whether put_active
* should be performed or not only on ERR_PTR(-ENODEV).
*
* This is worked around by factoring out the custom seq_stop() and
* put_active part into kernfs_seq_stop_active(), skipping it from
* kernfs_seq_stop() if ERR_PTR(-ENODEV) while invoking it directly after
* custom seq_file operations fail with ERR_PTR(-ENODEV) - this ensures
* that kernfs_seq_stop_active() is skipped only after get_active failure.
*/
static void kernfs_seq_stop_active(struct seq_file *sf, void *v)
{
struct kernfs_open_file *of = sf->private;
const struct kernfs_ops *ops = kernfs_ops(of->kn);
if (ops->seq_stop)
ops->seq_stop(sf, v);
kernfs_put_active(of->kn);
}
static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos)
{
struct kernfs_open_file *of = sf->private;
const struct kernfs_ops *ops;
/*
* @of->mutex nests outside active ref and is primarily to ensure that
* the ops aren't called concurrently for the same open file.
*/
mutex_lock(&of->mutex);
if (!kernfs_get_active(of->kn))
return ERR_PTR(-ENODEV);
ops = kernfs_ops(of->kn);
if (ops->seq_start) {
void *next = ops->seq_start(sf, ppos);
/* see the comment above kernfs_seq_stop_active() */
if (next == ERR_PTR(-ENODEV))
kernfs_seq_stop_active(sf, next);
return next;
}
return single_start(sf, ppos);
}
static void *kernfs_seq_next(struct seq_file *sf, void *v, loff_t *ppos)
{
struct kernfs_open_file *of = sf->private;
const struct kernfs_ops *ops = kernfs_ops(of->kn);
if (ops->seq_next) {
void *next = ops->seq_next(sf, v, ppos);
/* see the comment above kernfs_seq_stop_active() */
if (next == ERR_PTR(-ENODEV))
kernfs_seq_stop_active(sf, next);
return next;
} else {
/*
* The same behavior and code as single_open(), always
* terminate after the initial read.
*/
++*ppos;
return NULL;
}
}
static void kernfs_seq_stop(struct seq_file *sf, void *v)
{
struct kernfs_open_file *of = sf->private;
if (v != ERR_PTR(-ENODEV))
kernfs_seq_stop_active(sf, v);
mutex_unlock(&of->mutex);
}
static int kernfs_seq_show(struct seq_file *sf, void *v)
{
struct kernfs_open_file *of = sf->private;
of->event = atomic_read(&of_on(of)->event);
return of->kn->attr.ops->seq_show(sf, v);
}
static const struct seq_operations kernfs_seq_ops = {
.start = kernfs_seq_start,
.next = kernfs_seq_next,
.stop = kernfs_seq_stop,
.show = kernfs_seq_show,
};
/*
* As reading a bin file can have side-effects, the exact offset and bytes
* specified in read(2) call should be passed to the read callback making
* it difficult to use seq_file. Implement simplistic custom buffering for
* bin files.
*/
static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
struct kernfs_open_file *of = kernfs_of(iocb->ki_filp);
ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE);
const struct kernfs_ops *ops;
char *buf;
buf = of->prealloc_buf;
if (buf)
mutex_lock(&of->prealloc_mutex);
else
buf = kmalloc(len, GFP_KERNEL);
if (!buf)
return -ENOMEM;
/*
* @of->mutex nests outside active ref and is used both to ensure that
* the ops aren't called concurrently for the same open file.
*/
mutex_lock(&of->mutex);
if (!kernfs_get_active(of->kn)) {
len = -ENODEV;
mutex_unlock(&of->mutex);
goto out_free;
}
of->event = atomic_read(&of_on(of)->event);
ops = kernfs_ops(of->kn);
if (ops->read)
len = ops->read(of, buf, len, iocb->ki_pos);
else
len = -EINVAL;
kernfs_put_active(of->kn);
mutex_unlock(&of->mutex);
if (len < 0)
goto out_free;
if (copy_to_iter(buf, len, iter) != len) {
len = -EFAULT;
goto out_free;
}
iocb->ki_pos += len;
out_free:
if (buf == of->prealloc_buf)
mutex_unlock(&of->prealloc_mutex);
else
kfree(buf);
return len;
}
static ssize_t kernfs_fop_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
if (kernfs_of(iocb->ki_filp)->kn->flags & KERNFS_HAS_SEQ_SHOW)
return seq_read_iter(iocb, iter);
return kernfs_file_read_iter(iocb, iter);
}
/*
* Copy data in from userland and pass it to the matching kernfs write
* operation.
*
* There is no easy way for us to know if userspace is only doing a partial
* write, so we don't support them. We expect the entire buffer to come on
* the first write. Hint: if you're writing a value, first read the file,
* modify only the value you're changing, then write entire buffer
* back.
*/
static ssize_t kernfs_fop_write_iter(struct kiocb *iocb, struct iov_iter *iter)
{
struct kernfs_open_file *of = kernfs_of(iocb->ki_filp);
ssize_t len = iov_iter_count(iter);
const struct kernfs_ops *ops;
char *buf;
if (of->atomic_write_len) {
if (len > of->atomic_write_len)
return -E2BIG;
} else {
len = min_t(size_t, len, PAGE_SIZE);
}
buf = of->prealloc_buf;
if (buf)
mutex_lock(&of->prealloc_mutex);
else
buf = kmalloc(len + 1, GFP_KERNEL);
if (!buf)
return -ENOMEM;
if (copy_from_iter(buf, len, iter) != len) {
len = -EFAULT;
goto out_free;
}
buf[len] = '\0'; /* guarantee string termination */
/*
* @of->mutex nests outside active ref and is used both to ensure that
* the ops aren't called concurrently for the same open file.
*/
mutex_lock(&of->mutex);
if (!kernfs_get_active(of->kn)) {
mutex_unlock(&of->mutex);
len = -ENODEV;
goto out_free;
}
ops = kernfs_ops(of->kn);
if (ops->write)
len = ops->write(of, buf, len, iocb->ki_pos);
else
len = -EINVAL;
kernfs_put_active(of->kn);
mutex_unlock(&of->mutex);
if (len > 0)
iocb->ki_pos += len;
out_free:
if (buf == of->prealloc_buf)
mutex_unlock(&of->prealloc_mutex);
else
kfree(buf);
return len;
}
static void kernfs_vma_open(struct vm_area_struct *vma)
{
struct file *file = vma->vm_file;
struct kernfs_open_file *of = kernfs_of(file);
if (!of->vm_ops)
return;
if (!kernfs_get_active(of->kn))
return;
if (of->vm_ops->open)
of->vm_ops->open(vma);
kernfs_put_active(of->kn);
}
static vm_fault_t kernfs_vma_fault(struct vm_fault *vmf)
{
struct file *file = vmf->vma->vm_file;
struct kernfs_open_file *of = kernfs_of(file);
vm_fault_t ret;
if (!of->vm_ops)
return VM_FAULT_SIGBUS;
if (!kernfs_get_active(of->kn))
return VM_FAULT_SIGBUS;
ret = VM_FAULT_SIGBUS;
if (of->vm_ops->fault)
ret = of->vm_ops->fault(vmf);
kernfs_put_active(of->kn);
return ret;
}
static vm_fault_t kernfs_vma_page_mkwrite(struct vm_fault *vmf)
{
struct file *file = vmf->vma->vm_file;
struct kernfs_open_file *of = kernfs_of(file);
vm_fault_t ret;
if (!of->vm_ops)
return VM_FAULT_SIGBUS;
if (!kernfs_get_active(of->kn))
return VM_FAULT_SIGBUS;
ret = 0;
if (of->vm_ops->page_mkwrite)
ret = of->vm_ops->page_mkwrite(vmf);
else
file_update_time(file);
kernfs_put_active(of->kn);
return ret;
}
static int kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write)
{
struct file *file = vma->vm_file;
struct kernfs_open_file *of = kernfs_of(file);
int ret;
if (!of->vm_ops)
return -EINVAL;
if (!kernfs_get_active(of->kn))
return -EINVAL;
ret = -EINVAL;
if (of->vm_ops->access)
ret = of->vm_ops->access(vma, addr, buf, len, write);
kernfs_put_active(of->kn);
return ret;
}
static const struct vm_operations_struct kernfs_vm_ops = {
.open = kernfs_vma_open,
.fault = kernfs_vma_fault,
.page_mkwrite = kernfs_vma_page_mkwrite,
.access = kernfs_vma_access,
};
static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma)
{
struct kernfs_open_file *of = kernfs_of(file);
const struct kernfs_ops *ops;
int rc;
/*
* mmap path and of->mutex are prone to triggering spurious lockdep
* warnings and we don't want to add spurious locking dependency
* between the two. Check whether mmap is actually implemented
* without grabbing @of->mutex by testing HAS_MMAP flag. See the
* comment in kernfs_fop_open() for more details.
*/
if (!(of->kn->flags & KERNFS_HAS_MMAP))
return -ENODEV;
mutex_lock(&of->mutex);
rc = -ENODEV;
if (!kernfs_get_active(of->kn))
goto out_unlock;
ops = kernfs_ops(of->kn);
rc = ops->mmap(of, vma);
if (rc)
goto out_put;
/*
* PowerPC's pci_mmap of legacy_mem uses shmem_zero_setup()
* to satisfy versions of X which crash if the mmap fails: that
* substitutes a new vm_file, and we don't then want bin_vm_ops.
*/
if (vma->vm_file != file)
goto out_put;
rc = -EINVAL;
if (of->mmapped && of->vm_ops != vma->vm_ops)
goto out_put;
/*
* It is not possible to successfully wrap close.
* So error if someone is trying to use close.
*/
if (vma->vm_ops && vma->vm_ops->close)
goto out_put;
rc = 0;
if (!of->mmapped) {
of->mmapped = true;
of_on(of)->nr_mmapped++;
of->vm_ops = vma->vm_ops;
}
vma->vm_ops = &kernfs_vm_ops;
out_put:
kernfs_put_active(of->kn);
out_unlock:
mutex_unlock(&of->mutex);
return rc;
}
/**
* kernfs_get_open_node - get or create kernfs_open_node
* @kn: target kernfs_node
* @of: kernfs_open_file for this instance of open
*
* If @kn->attr.open exists, increment its reference count; otherwise,
* create one. @of is chained to the files list.
*
* Locking:
* Kernel thread context (may sleep).
*
* Return:
* %0 on success, -errno on failure.
*/
static int kernfs_get_open_node(struct kernfs_node *kn,
struct kernfs_open_file *of)
{
struct kernfs_open_node *on;
struct mutex *mutex;
mutex = kernfs_open_file_mutex_lock(kn);
on = kernfs_deref_open_node_locked(kn);
if (!on) {
/* not there, initialize a new one */
on = kzalloc(sizeof(*on), GFP_KERNEL);
if (!on) {
mutex_unlock(mutex);
return -ENOMEM;
}
atomic_set(&on->event, 1);
init_waitqueue_head(&on->poll);
INIT_LIST_HEAD(&on->files);
rcu_assign_pointer(kn->attr.open, on);
}
list_add_tail(&of->list, &on->files);
if (kn->flags & KERNFS_HAS_RELEASE)
on->nr_to_release++;
mutex_unlock(mutex);
return 0;
}
/**
* kernfs_unlink_open_file - Unlink @of from @kn.
*
* @kn: target kernfs_node
* @of: associated kernfs_open_file
* @open_failed: ->open() failed, cancel ->release()
*
* Unlink @of from list of @kn's associated open files. If list of
* associated open files becomes empty, disassociate and free
* kernfs_open_node.
*
* LOCKING:
* None.
*/
static void kernfs_unlink_open_file(struct kernfs_node *kn,
struct kernfs_open_file *of,
bool open_failed)
{
struct kernfs_open_node *on;
struct mutex *mutex;
mutex = kernfs_open_file_mutex_lock(kn);
on = kernfs_deref_open_node_locked(kn);
if (!on) {
mutex_unlock(mutex);
return;
}
if (of) {
if (kn->flags & KERNFS_HAS_RELEASE) {
WARN_ON_ONCE(of->released == open_failed);
if (open_failed)
on->nr_to_release--;
}
if (of->mmapped)
on->nr_mmapped--;
list_del(&of->list);
}
if (list_empty(&on->files)) {
rcu_assign_pointer(kn->attr.open, NULL);
kfree_rcu(on, rcu_head);
}
mutex_unlock(mutex);
}
static int kernfs_fop_open(struct inode *inode, struct file *file)
{
struct kernfs_node *kn = inode->i_private;
struct kernfs_root *root = kernfs_root(kn);
const struct kernfs_ops *ops;
struct kernfs_open_file *of;
bool has_read, has_write, has_mmap;
int error = -EACCES;
if (!kernfs_get_active(kn))
return -ENODEV;
ops = kernfs_ops(kn);
has_read = ops->seq_show || ops->read || ops->mmap;
has_write = ops->write || ops->mmap;
has_mmap = ops->mmap;
/* see the flag definition for details */
if (root->flags & KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK) {
if ((file->f_mode & FMODE_WRITE) &&
(!(inode->i_mode & S_IWUGO) || !has_write))
goto err_out;
if ((file->f_mode & FMODE_READ) &&
(!(inode->i_mode & S_IRUGO) || !has_read))
goto err_out;
}
/* allocate a kernfs_open_file for the file */
error = -ENOMEM;
of = kzalloc(sizeof(struct kernfs_open_file), GFP_KERNEL);
if (!of)
goto err_out;
/*
* The following is done to give a different lockdep key to
* @of->mutex for files which implement mmap. This is a rather
* crude way to avoid false positive lockdep warning around
* mm->mmap_lock - mmap nests @of->mutex under mm->mmap_lock and
* reading /sys/block/sda/trace/act_mask grabs sr_mutex, under
* which mm->mmap_lock nests, while holding @of->mutex. As each
* open file has a separate mutex, it's okay as long as those don't
* happen on the same file. At this point, we can't easily give
* each file a separate locking class. Let's differentiate on
* whether the file has mmap or not for now.
*
* For similar reasons, writable and readonly files are given different
* lockdep key, because the writable file /sys/power/resume may call vfs
* lookup helpers for arbitrary paths and readonly files can be read by
* overlayfs from vfs helpers when sysfs is a lower layer of overalyfs.
*
* All three cases look the same. They're supposed to
* look that way and give @of->mutex different static lockdep keys.
*/
if (has_mmap)
mutex_init(&of->mutex);
else if (file->f_mode & FMODE_WRITE)
mutex_init(&of->mutex);
else
mutex_init(&of->mutex);
of->kn = kn;
of->file = file;
/*
* Write path needs to atomic_write_len outside active reference.
* Cache it in open_file. See kernfs_fop_write_iter() for details.
*/
of->atomic_write_len = ops->atomic_write_len;
error = -EINVAL;
/*
* ->seq_show is incompatible with ->prealloc,
* as seq_read does its own allocation.
* ->read must be used instead.
*/
if (ops->prealloc && ops->seq_show)
goto err_free;
if (ops->prealloc) {
int len = of->atomic_write_len ?: PAGE_SIZE;
of->prealloc_buf = kmalloc(len + 1, GFP_KERNEL);
error = -ENOMEM;
if (!of->prealloc_buf)
goto err_free;
mutex_init(&of->prealloc_mutex);
}
/*
* Always instantiate seq_file even if read access doesn't use
* seq_file or is not requested. This unifies private data access
* and readable regular files are the vast majority anyway.
*/
if (ops->seq_show)
error = seq_open(file, &kernfs_seq_ops);
else
error = seq_open(file, NULL);
if (error)
goto err_free;
of->seq_file = file->private_data;
of->seq_file->private = of;
/* seq_file clears PWRITE unconditionally, restore it if WRITE */
if (file->f_mode & FMODE_WRITE)
file->f_mode |= FMODE_PWRITE;
/* make sure we have open node struct */
error = kernfs_get_open_node(kn, of);
if (error)
goto err_seq_release;
if (ops->open) {
/* nobody has access to @of yet, skip @of->mutex */
error = ops->open(of);
if (error)
goto err_put_node;
}
/* open succeeded, put active references */
kernfs_put_active(kn);
return 0;
err_put_node:
kernfs_unlink_open_file(kn, of, true);
err_seq_release:
seq_release(inode, file);
err_free:
kfree(of->prealloc_buf);
kfree(of);
err_out:
kernfs_put_active(kn);
return error;
}
/* used from release/drain to ensure that ->release() is called exactly once */
static void kernfs_release_file(struct kernfs_node *kn,
struct kernfs_open_file *of)
{
/*
* @of is guaranteed to have no other file operations in flight and
* we just want to synchronize release and drain paths.
* @kernfs_open_file_mutex_ptr(kn) is enough. @of->mutex can't be used
* here because drain path may be called from places which can
* cause circular dependency.
*/
lockdep_assert_held(kernfs_open_file_mutex_ptr(kn));
if (!of->released) {
/*
* A file is never detached without being released and we
* need to be able to release files which are deactivated
* and being drained. Don't use kernfs_ops().
*/
kn->attr.ops->release(of);
of->released = true;
of_on(of)->nr_to_release--;
}
}
static int kernfs_fop_release(struct inode *inode, struct file *filp)
{
struct kernfs_node *kn = inode->i_private;
struct kernfs_open_file *of = kernfs_of(filp);
if (kn->flags & KERNFS_HAS_RELEASE) {
struct mutex *mutex;
mutex = kernfs_open_file_mutex_lock(kn);
kernfs_release_file(kn, of);
mutex_unlock(mutex);
}
kernfs_unlink_open_file(kn, of, false);
seq_release(inode, filp);
kfree(of->prealloc_buf);
kfree(of);
return 0;
}
bool kernfs_should_drain_open_files(struct kernfs_node *kn)
{
struct kernfs_open_node *on;
bool ret;
/*
* @kn being deactivated guarantees that @kn->attr.open can't change
* beneath us making the lockless test below safe.
*/
WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); rcu_read_lock(); on = rcu_dereference(kn->attr.open); ret = on && (on->nr_mmapped || on->nr_to_release); rcu_read_unlock();
return ret;
}
void kernfs_drain_open_files(struct kernfs_node *kn)
{
struct kernfs_open_node *on;
struct kernfs_open_file *of;
struct mutex *mutex;
mutex = kernfs_open_file_mutex_lock(kn);
on = kernfs_deref_open_node_locked(kn);
if (!on) {
mutex_unlock(mutex);
return;
}
list_for_each_entry(of, &on->files, list) {
struct inode *inode = file_inode(of->file);
if (of->mmapped) {
unmap_mapping_range(inode->i_mapping, 0, 0, 1);
of->mmapped = false;
on->nr_mmapped--;
}
if (kn->flags & KERNFS_HAS_RELEASE)
kernfs_release_file(kn, of);
}
WARN_ON_ONCE(on->nr_mmapped || on->nr_to_release);
mutex_unlock(mutex);
}
/*
* Kernfs attribute files are pollable. The idea is that you read
* the content and then you use 'poll' or 'select' to wait for
* the content to change. When the content changes (assuming the
* manager for the kobject supports notification), poll will
* return EPOLLERR|EPOLLPRI, and select will return the fd whether
* it is waiting for read, write, or exceptions.
* Once poll/select indicates that the value has changed, you
* need to close and re-open the file, or seek to 0 and read again.
* Reminder: this only works for attributes which actively support
* it, and it is not possible to test an attribute from userspace
* to see if it supports poll (Neither 'poll' nor 'select' return
* an appropriate error code). When in doubt, set a suitable timeout value.
*/
__poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait)
{
struct kernfs_open_node *on = of_on(of);
poll_wait(of->file, &on->poll, wait);
if (of->event != atomic_read(&on->event))
return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI;
return DEFAULT_POLLMASK;
}
static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait)
{
struct kernfs_open_file *of = kernfs_of(filp);
struct kernfs_node *kn = kernfs_dentry_node(filp->f_path.dentry);
__poll_t ret;
if (!kernfs_get_active(kn))
return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI;
if (kn->attr.ops->poll)
ret = kn->attr.ops->poll(of, wait);
else
ret = kernfs_generic_poll(of, wait);
kernfs_put_active(kn);
return ret;
}
static loff_t kernfs_fop_llseek(struct file *file, loff_t offset, int whence)
{
struct kernfs_open_file *of = kernfs_of(file);
const struct kernfs_ops *ops;
loff_t ret;
/*
* @of->mutex nests outside active ref and is primarily to ensure that
* the ops aren't called concurrently for the same open file.
*/
mutex_lock(&of->mutex);
if (!kernfs_get_active(of->kn)) {
mutex_unlock(&of->mutex);
return -ENODEV;
}
ops = kernfs_ops(of->kn);
if (ops->llseek)
ret = ops->llseek(of, offset, whence);
else
ret = generic_file_llseek(file, offset, whence);
kernfs_put_active(of->kn);
mutex_unlock(&of->mutex);
return ret;
}
static void kernfs_notify_workfn(struct work_struct *work)
{
struct kernfs_node *kn;
struct kernfs_super_info *info;
struct kernfs_root *root;
repeat:
/* pop one off the notify_list */
spin_lock_irq(&kernfs_notify_lock);
kn = kernfs_notify_list;
if (kn == KERNFS_NOTIFY_EOL) {
spin_unlock_irq(&kernfs_notify_lock);
return;
}
kernfs_notify_list = kn->attr.notify_next;
kn->attr.notify_next = NULL;
spin_unlock_irq(&kernfs_notify_lock);
root = kernfs_root(kn);
/* kick fsnotify */
down_read(&root->kernfs_supers_rwsem);
list_for_each_entry(info, &kernfs_root(kn)->supers, node) {
struct kernfs_node *parent;
struct inode *p_inode = NULL;
struct inode *inode;
struct qstr name;
/*
* We want fsnotify_modify() on @kn but as the
* modifications aren't originating from userland don't
* have the matching @file available. Look up the inodes
* and generate the events manually.
*/
inode = ilookup(info->sb, kernfs_ino(kn));
if (!inode)
continue;
name = (struct qstr)QSTR_INIT(kn->name, strlen(kn->name));
parent = kernfs_get_parent(kn);
if (parent) {
p_inode = ilookup(info->sb, kernfs_ino(parent));
if (p_inode) {
fsnotify(FS_MODIFY | FS_EVENT_ON_CHILD,
inode, FSNOTIFY_EVENT_INODE,
p_inode, &name, inode, 0);
iput(p_inode);
}
kernfs_put(parent);
}
if (!p_inode)
fsnotify_inode(inode, FS_MODIFY);
iput(inode);
}
up_read(&root->kernfs_supers_rwsem);
kernfs_put(kn);
goto repeat;
}
/**
* kernfs_notify - notify a kernfs file
* @kn: file to notify
*
* Notify @kn such that poll(2) on @kn wakes up. Maybe be called from any
* context.
*/
void kernfs_notify(struct kernfs_node *kn)
{
static DECLARE_WORK(kernfs_notify_work, kernfs_notify_workfn);
unsigned long flags;
struct kernfs_open_node *on;
if (WARN_ON(kernfs_type(kn) != KERNFS_FILE))
return;
/* kick poll immediately */
rcu_read_lock(); on = rcu_dereference(kn->attr.open); if (on) { atomic_inc(&on->event);
wake_up_interruptible(&on->poll);
}
rcu_read_unlock();
/* schedule work to kick fsnotify */
spin_lock_irqsave(&kernfs_notify_lock, flags);
if (!kn->attr.notify_next) {
kernfs_get(kn);
kn->attr.notify_next = kernfs_notify_list;
kernfs_notify_list = kn;
schedule_work(&kernfs_notify_work);
}
spin_unlock_irqrestore(&kernfs_notify_lock, flags);
}
EXPORT_SYMBOL_GPL(kernfs_notify);
const struct file_operations kernfs_file_fops = {
.read_iter = kernfs_fop_read_iter,
.write_iter = kernfs_fop_write_iter,
.llseek = kernfs_fop_llseek,
.mmap = kernfs_fop_mmap,
.open = kernfs_fop_open,
.release = kernfs_fop_release,
.poll = kernfs_fop_poll,
.fsync = noop_fsync,
.splice_read = copy_splice_read,
.splice_write = iter_file_splice_write,
};
/**
* __kernfs_create_file - kernfs internal function to create a file
* @parent: directory to create the file in
* @name: name of the file
* @mode: mode of the file
* @uid: uid of the file
* @gid: gid of the file
* @size: size of the file
* @ops: kernfs operations for the file
* @priv: private data for the file
* @ns: optional namespace tag of the file
* @key: lockdep key for the file's active_ref, %NULL to disable lockdep
*
* Return: the created node on success, ERR_PTR() value on error.
*/
struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent,
const char *name,
umode_t mode, kuid_t uid, kgid_t gid,
loff_t size,
const struct kernfs_ops *ops,
void *priv, const void *ns,
struct lock_class_key *key)
{
struct kernfs_node *kn;
unsigned flags;
int rc;
flags = KERNFS_FILE;
kn = kernfs_new_node(parent, name, (mode & S_IALLUGO) | S_IFREG,
uid, gid, flags);
if (!kn)
return ERR_PTR(-ENOMEM);
kn->attr.ops = ops;
kn->attr.size = size;
kn->ns = ns;
kn->priv = priv;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
if (key) {
lockdep_init_map(&kn->dep_map, "kn->active", key, 0);
kn->flags |= KERNFS_LOCKDEP;
}
#endif
/*
* kn->attr.ops is accessible only while holding active ref. We
* need to know whether some ops are implemented outside active
* ref. Cache their existence in flags.
*/
if (ops->seq_show) kn->flags |= KERNFS_HAS_SEQ_SHOW; if (ops->mmap) kn->flags |= KERNFS_HAS_MMAP; if (ops->release) kn->flags |= KERNFS_HAS_RELEASE; rc = kernfs_add_one(kn); if (rc) { kernfs_put(kn);
return ERR_PTR(rc);
}
return kn;
}
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Security plug functions
*
* Copyright (C) 2001 WireX Communications, Inc <chris@wirex.com>
* Copyright (C) 2001-2002 Greg Kroah-Hartman <greg@kroah.com>
* Copyright (C) 2001 Networks Associates Technology, Inc <ssmalley@nai.com>
* Copyright (C) 2016 Mellanox Technologies
* Copyright (C) 2023 Microsoft Corporation <paul@paul-moore.com>
*/
#define pr_fmt(fmt) "LSM: " fmt
#include <linux/bpf.h>
#include <linux/capability.h>
#include <linux/dcache.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kernel_read_file.h>
#include <linux/lsm_hooks.h>
#include <linux/fsnotify.h>
#include <linux/mman.h>
#include <linux/mount.h>
#include <linux/personality.h>
#include <linux/backing-dev.h>
#include <linux/string.h>
#include <linux/xattr.h>
#include <linux/msg.h>
#include <linux/overflow.h>
#include <net/flow.h>
/* How many LSMs were built into the kernel? */
#define LSM_COUNT (__end_lsm_info - __start_lsm_info)
/*
* How many LSMs are built into the kernel as determined at
* build time. Used to determine fixed array sizes.
* The capability module is accounted for by CONFIG_SECURITY
*/
#define LSM_CONFIG_COUNT ( \
(IS_ENABLED(CONFIG_SECURITY) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_SELINUX) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_SMACK) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_TOMOYO) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_APPARMOR) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_YAMA) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_LOADPIN) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_SAFESETID) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \
(IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \
(IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0) + \
(IS_ENABLED(CONFIG_IMA) ? 1 : 0) + \
(IS_ENABLED(CONFIG_EVM) ? 1 : 0))
/*
* These are descriptions of the reasons that can be passed to the
* security_locked_down() LSM hook. Placing this array here allows
* all security modules to use the same descriptions for auditing
* purposes.
*/
const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX + 1] = {
[LOCKDOWN_NONE] = "none",
[LOCKDOWN_MODULE_SIGNATURE] = "unsigned module loading",
[LOCKDOWN_DEV_MEM] = "/dev/mem,kmem,port",
[LOCKDOWN_EFI_TEST] = "/dev/efi_test access",
[LOCKDOWN_KEXEC] = "kexec of unsigned images",
[LOCKDOWN_HIBERNATION] = "hibernation",
[LOCKDOWN_PCI_ACCESS] = "direct PCI access",
[LOCKDOWN_IOPORT] = "raw io port access",
[LOCKDOWN_MSR] = "raw MSR access",
[LOCKDOWN_ACPI_TABLES] = "modifying ACPI tables",
[LOCKDOWN_DEVICE_TREE] = "modifying device tree contents",
[LOCKDOWN_PCMCIA_CIS] = "direct PCMCIA CIS storage",
[LOCKDOWN_TIOCSSERIAL] = "reconfiguration of serial port IO",
[LOCKDOWN_MODULE_PARAMETERS] = "unsafe module parameters",
[LOCKDOWN_MMIOTRACE] = "unsafe mmio",
[LOCKDOWN_DEBUGFS] = "debugfs access",
[LOCKDOWN_XMON_WR] = "xmon write access",
[LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM",
[LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM",
[LOCKDOWN_RTAS_ERROR_INJECTION] = "RTAS error injection",
[LOCKDOWN_INTEGRITY_MAX] = "integrity",
[LOCKDOWN_KCORE] = "/proc/kcore access",
[LOCKDOWN_KPROBES] = "use of kprobes",
[LOCKDOWN_BPF_READ_KERNEL] = "use of bpf to read kernel RAM",
[LOCKDOWN_DBG_READ_KERNEL] = "use of kgdb/kdb to read kernel RAM",
[LOCKDOWN_PERF] = "unsafe use of perf",
[LOCKDOWN_TRACEFS] = "use of tracefs",
[LOCKDOWN_XMON_RW] = "xmon read and write access",
[LOCKDOWN_XFRM_SECRET] = "xfrm SA secret",
[LOCKDOWN_CONFIDENTIALITY_MAX] = "confidentiality",
};
struct security_hook_heads security_hook_heads __ro_after_init;
static BLOCKING_NOTIFIER_HEAD(blocking_lsm_notifier_chain);
static struct kmem_cache *lsm_file_cache;
static struct kmem_cache *lsm_inode_cache;
char *lsm_names;
static struct lsm_blob_sizes blob_sizes __ro_after_init;
/* Boot-time LSM user choice */
static __initdata const char *chosen_lsm_order;
static __initdata const char *chosen_major_lsm;
static __initconst const char *const builtin_lsm_order = CONFIG_LSM;
/* Ordered list of LSMs to initialize. */
static __initdata struct lsm_info **ordered_lsms;
static __initdata struct lsm_info *exclusive;
static __initdata bool debug;
#define init_debug(...) \
do { \
if (debug) \
pr_info(__VA_ARGS__); \
} while (0)
static bool __init is_enabled(struct lsm_info *lsm)
{
if (!lsm->enabled)
return false;
return *lsm->enabled;
}
/* Mark an LSM's enabled flag. */
static int lsm_enabled_true __initdata = 1;
static int lsm_enabled_false __initdata = 0;
static void __init set_enabled(struct lsm_info *lsm, bool enabled)
{
/*
* When an LSM hasn't configured an enable variable, we can use
* a hard-coded location for storing the default enabled state.
*/
if (!lsm->enabled) {
if (enabled)
lsm->enabled = &lsm_enabled_true;
else
lsm->enabled = &lsm_enabled_false;
} else if (lsm->enabled == &lsm_enabled_true) {
if (!enabled)
lsm->enabled = &lsm_enabled_false;
} else if (lsm->enabled == &lsm_enabled_false) {
if (enabled)
lsm->enabled = &lsm_enabled_true;
} else {
*lsm->enabled = enabled;
}
}
/* Is an LSM already listed in the ordered LSMs list? */
static bool __init exists_ordered_lsm(struct lsm_info *lsm)
{
struct lsm_info **check;
for (check = ordered_lsms; *check; check++)
if (*check == lsm)
return true;
return false;
}
/* Append an LSM to the list of ordered LSMs to initialize. */
static int last_lsm __initdata;
static void __init append_ordered_lsm(struct lsm_info *lsm, const char *from)
{
/* Ignore duplicate selections. */
if (exists_ordered_lsm(lsm))
return;
if (WARN(last_lsm == LSM_COUNT, "%s: out of LSM slots!?\n", from))
return;
/* Enable this LSM, if it is not already set. */
if (!lsm->enabled)
lsm->enabled = &lsm_enabled_true;
ordered_lsms[last_lsm++] = lsm;
init_debug("%s ordered: %s (%s)\n", from, lsm->name,
is_enabled(lsm) ? "enabled" : "disabled");
}
/* Is an LSM allowed to be initialized? */
static bool __init lsm_allowed(struct lsm_info *lsm)
{
/* Skip if the LSM is disabled. */
if (!is_enabled(lsm))
return false;
/* Not allowed if another exclusive LSM already initialized. */
if ((lsm->flags & LSM_FLAG_EXCLUSIVE) && exclusive) {
init_debug("exclusive disabled: %s\n", lsm->name);
return false;
}
return true;
}
static void __init lsm_set_blob_size(int *need, int *lbs)
{
int offset;
if (*need <= 0)
return;
offset = ALIGN(*lbs, sizeof(void *));
*lbs = offset + *need;
*need = offset;
}
static void __init lsm_set_blob_sizes(struct lsm_blob_sizes *needed)
{
if (!needed)
return;
lsm_set_blob_size(&needed->lbs_cred, &blob_sizes.lbs_cred);
lsm_set_blob_size(&needed->lbs_file, &blob_sizes.lbs_file);
/*
* The inode blob gets an rcu_head in addition to
* what the modules might need.
*/
if (needed->lbs_inode && blob_sizes.lbs_inode == 0)
blob_sizes.lbs_inode = sizeof(struct rcu_head);
lsm_set_blob_size(&needed->lbs_inode, &blob_sizes.lbs_inode);
lsm_set_blob_size(&needed->lbs_ipc, &blob_sizes.lbs_ipc);
lsm_set_blob_size(&needed->lbs_msg_msg, &blob_sizes.lbs_msg_msg);
lsm_set_blob_size(&needed->lbs_superblock, &blob_sizes.lbs_superblock);
lsm_set_blob_size(&needed->lbs_task, &blob_sizes.lbs_task);
lsm_set_blob_size(&needed->lbs_xattr_count,
&blob_sizes.lbs_xattr_count);
}
/* Prepare LSM for initialization. */
static void __init prepare_lsm(struct lsm_info *lsm)
{
int enabled = lsm_allowed(lsm);
/* Record enablement (to handle any following exclusive LSMs). */
set_enabled(lsm, enabled);
/* If enabled, do pre-initialization work. */
if (enabled) {
if ((lsm->flags & LSM_FLAG_EXCLUSIVE) && !exclusive) {
exclusive = lsm;
init_debug("exclusive chosen: %s\n", lsm->name);
}
lsm_set_blob_sizes(lsm->blobs);
}
}
/* Initialize a given LSM, if it is enabled. */
static void __init initialize_lsm(struct lsm_info *lsm)
{
if (is_enabled(lsm)) {
int ret;
init_debug("initializing %s\n", lsm->name);
ret = lsm->init();
WARN(ret, "%s failed to initialize: %d\n", lsm->name, ret);
}
}
/*
* Current index to use while initializing the lsm id list.
*/
u32 lsm_active_cnt __ro_after_init;
const struct lsm_id *lsm_idlist[LSM_CONFIG_COUNT];
/* Populate ordered LSMs list from comma-separated LSM name list. */
static void __init ordered_lsm_parse(const char *order, const char *origin)
{
struct lsm_info *lsm;
char *sep, *name, *next;
/* LSM_ORDER_FIRST is always first. */
for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
if (lsm->order == LSM_ORDER_FIRST)
append_ordered_lsm(lsm, " first");
}
/* Process "security=", if given. */
if (chosen_major_lsm) {
struct lsm_info *major;
/*
* To match the original "security=" behavior, this
* explicitly does NOT fallback to another Legacy Major
* if the selected one was separately disabled: disable
* all non-matching Legacy Major LSMs.
*/
for (major = __start_lsm_info; major < __end_lsm_info;
major++) {
if ((major->flags & LSM_FLAG_LEGACY_MAJOR) &&
strcmp(major->name, chosen_major_lsm) != 0) {
set_enabled(major, false);
init_debug("security=%s disabled: %s (only one legacy major LSM)\n",
chosen_major_lsm, major->name);
}
}
}
sep = kstrdup(order, GFP_KERNEL);
next = sep;
/* Walk the list, looking for matching LSMs. */
while ((name = strsep(&next, ",")) != NULL) {
bool found = false;
for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
if (strcmp(lsm->name, name) == 0) {
if (lsm->order == LSM_ORDER_MUTABLE)
append_ordered_lsm(lsm, origin);
found = true;
}
}
if (!found)
init_debug("%s ignored: %s (not built into kernel)\n",
origin, name);
}
/* Process "security=", if given. */
if (chosen_major_lsm) {
for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
if (exists_ordered_lsm(lsm))
continue;
if (strcmp(lsm->name, chosen_major_lsm) == 0)
append_ordered_lsm(lsm, "security=");
}
}
/* LSM_ORDER_LAST is always last. */
for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
if (lsm->order == LSM_ORDER_LAST)
append_ordered_lsm(lsm, " last");
}
/* Disable all LSMs not in the ordered list. */
for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
if (exists_ordered_lsm(lsm))
continue;
set_enabled(lsm, false);
init_debug("%s skipped: %s (not in requested order)\n",
origin, lsm->name);
}
kfree(sep);
}
static void __init lsm_early_cred(struct cred *cred);
static void __init lsm_early_task(struct task_struct *task);
static int lsm_append(const char *new, char **result);
static void __init report_lsm_order(void)
{
struct lsm_info **lsm, *early;
int first = 0;
pr_info("initializing lsm=");
/* Report each enabled LSM name, comma separated. */
for (early = __start_early_lsm_info;
early < __end_early_lsm_info; early++)
if (is_enabled(early))
pr_cont("%s%s", first++ == 0 ? "" : ",", early->name);
for (lsm = ordered_lsms; *lsm; lsm++)
if (is_enabled(*lsm))
pr_cont("%s%s", first++ == 0 ? "" : ",", (*lsm)->name);
pr_cont("\n");
}
static void __init ordered_lsm_init(void)
{
struct lsm_info **lsm;
ordered_lsms = kcalloc(LSM_COUNT + 1, sizeof(*ordered_lsms),
GFP_KERNEL);
if (chosen_lsm_order) {
if (chosen_major_lsm) {
pr_warn("security=%s is ignored because it is superseded by lsm=%s\n",
chosen_major_lsm, chosen_lsm_order);
chosen_major_lsm = NULL;
}
ordered_lsm_parse(chosen_lsm_order, "cmdline");
} else
ordered_lsm_parse(builtin_lsm_order, "builtin");
for (lsm = ordered_lsms; *lsm; lsm++)
prepare_lsm(*lsm);
report_lsm_order();
init_debug("cred blob size = %d\n", blob_sizes.lbs_cred);
init_debug("file blob size = %d\n", blob_sizes.lbs_file);
init_debug("inode blob size = %d\n", blob_sizes.lbs_inode);
init_debug("ipc blob size = %d\n", blob_sizes.lbs_ipc);
init_debug("msg_msg blob size = %d\n", blob_sizes.lbs_msg_msg);
init_debug("superblock blob size = %d\n", blob_sizes.lbs_superblock);
init_debug("task blob size = %d\n", blob_sizes.lbs_task);
init_debug("xattr slots = %d\n", blob_sizes.lbs_xattr_count);
/*
* Create any kmem_caches needed for blobs
*/
if (blob_sizes.lbs_file)
lsm_file_cache = kmem_cache_create("lsm_file_cache",
blob_sizes.lbs_file, 0,
SLAB_PANIC, NULL);
if (blob_sizes.lbs_inode)
lsm_inode_cache = kmem_cache_create("lsm_inode_cache",
blob_sizes.lbs_inode, 0,
SLAB_PANIC, NULL);
lsm_early_cred((struct cred *) current->cred);
lsm_early_task(current);
for (lsm = ordered_lsms; *lsm; lsm++)
initialize_lsm(*lsm);
kfree(ordered_lsms);
}
int __init early_security_init(void)
{
struct lsm_info *lsm;
#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
INIT_HLIST_HEAD(&security_hook_heads.NAME);
#include "linux/lsm_hook_defs.h"
#undef LSM_HOOK
for (lsm = __start_early_lsm_info; lsm < __end_early_lsm_info; lsm++) {
if (!lsm->enabled)
lsm->enabled = &lsm_enabled_true;
prepare_lsm(lsm);
initialize_lsm(lsm);
}
return 0;
}
/**
* security_init - initializes the security framework
*
* This should be called early in the kernel initialization sequence.
*/
int __init security_init(void)
{
struct lsm_info *lsm;
init_debug("legacy security=%s\n", chosen_major_lsm ? : " *unspecified*");
init_debug(" CONFIG_LSM=%s\n", builtin_lsm_order);
init_debug("boot arg lsm=%s\n", chosen_lsm_order ? : " *unspecified*");
/*
* Append the names of the early LSM modules now that kmalloc() is
* available
*/
for (lsm = __start_early_lsm_info; lsm < __end_early_lsm_info; lsm++) {
init_debug(" early started: %s (%s)\n", lsm->name,
is_enabled(lsm) ? "enabled" : "disabled");
if (lsm->enabled)
lsm_append(lsm->name, &lsm_names);
}
/* Load LSMs in specified order. */
ordered_lsm_init();
return 0;
}
/* Save user chosen LSM */
static int __init choose_major_lsm(char *str)
{
chosen_major_lsm = str;
return 1;
}
__setup("security=", choose_major_lsm);
/* Explicitly choose LSM initialization order. */
static int __init choose_lsm_order(char *str)
{
chosen_lsm_order = str;
return 1;
}
__setup("lsm=", choose_lsm_order);
/* Enable LSM order debugging. */
static int __init enable_debug(char *str)
{
debug = true;
return 1;
}
__setup("lsm.debug", enable_debug);
static bool match_last_lsm(const char *list, const char *lsm)
{
const char *last;
if (WARN_ON(!list || !lsm))
return false;
last = strrchr(list, ',');
if (last)
/* Pass the comma, strcmp() will check for '\0' */
last++;
else
last = list;
return !strcmp(last, lsm);
}
static int lsm_append(const char *new, char **result)
{
char *cp;
if (*result == NULL) {
*result = kstrdup(new, GFP_KERNEL);
if (*result == NULL)
return -ENOMEM;
} else {
/* Check if it is the last registered name */
if (match_last_lsm(*result, new))
return 0;
cp = kasprintf(GFP_KERNEL, "%s,%s", *result, new);
if (cp == NULL)
return -ENOMEM;
kfree(*result);
*result = cp;
}
return 0;
}
/**
* security_add_hooks - Add a modules hooks to the hook lists.
* @hooks: the hooks to add
* @count: the number of hooks to add
* @lsmid: the identification information for the security module
*
* Each LSM has to register its hooks with the infrastructure.
*/
void __init security_add_hooks(struct security_hook_list *hooks, int count,
const struct lsm_id *lsmid)
{
int i;
/*
* A security module may call security_add_hooks() more
* than once during initialization, and LSM initialization
* is serialized. Landlock is one such case.
* Look at the previous entry, if there is one, for duplication.
*/
if (lsm_active_cnt == 0 || lsm_idlist[lsm_active_cnt - 1] != lsmid) {
if (lsm_active_cnt >= LSM_CONFIG_COUNT)
panic("%s Too many LSMs registered.\n", __func__);
lsm_idlist[lsm_active_cnt++] = lsmid;
}
for (i = 0; i < count; i++) {
hooks[i].lsmid = lsmid;
hlist_add_tail_rcu(&hooks[i].list, hooks[i].head);
}
/*
* Don't try to append during early_security_init(), we'll come back
* and fix this up afterwards.
*/
if (slab_is_available()) {
if (lsm_append(lsmid->name, &lsm_names) < 0)
panic("%s - Cannot get early memory.\n", __func__);
}
}
int call_blocking_lsm_notifier(enum lsm_event event, void *data)
{
return blocking_notifier_call_chain(&blocking_lsm_notifier_chain,
event, data);
}
EXPORT_SYMBOL(call_blocking_lsm_notifier);
int register_blocking_lsm_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_register(&blocking_lsm_notifier_chain,
nb);
}
EXPORT_SYMBOL(register_blocking_lsm_notifier);
int unregister_blocking_lsm_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(&blocking_lsm_notifier_chain,
nb);
}
EXPORT_SYMBOL(unregister_blocking_lsm_notifier);
/**
* lsm_cred_alloc - allocate a composite cred blob
* @cred: the cred that needs a blob
* @gfp: allocation type
*
* Allocate the cred blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
static int lsm_cred_alloc(struct cred *cred, gfp_t gfp)
{
if (blob_sizes.lbs_cred == 0) {
cred->security = NULL;
return 0;
}
cred->security = kzalloc(blob_sizes.lbs_cred, gfp);
if (cred->security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_early_cred - during initialization allocate a composite cred blob
* @cred: the cred that needs a blob
*
* Allocate the cred blob for all the modules
*/
static void __init lsm_early_cred(struct cred *cred)
{
int rc = lsm_cred_alloc(cred, GFP_KERNEL);
if (rc)
panic("%s: Early cred alloc failed.\n", __func__);
}
/**
* lsm_file_alloc - allocate a composite file blob
* @file: the file that needs a blob
*
* Allocate the file blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
static int lsm_file_alloc(struct file *file)
{
if (!lsm_file_cache) { file->f_security = NULL;
return 0;
}
file->f_security = kmem_cache_zalloc(lsm_file_cache, GFP_KERNEL);
if (file->f_security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_inode_alloc - allocate a composite inode blob
* @inode: the inode that needs a blob
*
* Allocate the inode blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
int lsm_inode_alloc(struct inode *inode)
{
if (!lsm_inode_cache) { inode->i_security = NULL;
return 0;
}
inode->i_security = kmem_cache_zalloc(lsm_inode_cache, GFP_NOFS);
if (inode->i_security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_task_alloc - allocate a composite task blob
* @task: the task that needs a blob
*
* Allocate the task blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
static int lsm_task_alloc(struct task_struct *task)
{
if (blob_sizes.lbs_task == 0) {
task->security = NULL;
return 0;
}
task->security = kzalloc(blob_sizes.lbs_task, GFP_KERNEL);
if (task->security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_ipc_alloc - allocate a composite ipc blob
* @kip: the ipc that needs a blob
*
* Allocate the ipc blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
static int lsm_ipc_alloc(struct kern_ipc_perm *kip)
{
if (blob_sizes.lbs_ipc == 0) {
kip->security = NULL;
return 0;
}
kip->security = kzalloc(blob_sizes.lbs_ipc, GFP_KERNEL);
if (kip->security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_msg_msg_alloc - allocate a composite msg_msg blob
* @mp: the msg_msg that needs a blob
*
* Allocate the ipc blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
static int lsm_msg_msg_alloc(struct msg_msg *mp)
{
if (blob_sizes.lbs_msg_msg == 0) {
mp->security = NULL;
return 0;
}
mp->security = kzalloc(blob_sizes.lbs_msg_msg, GFP_KERNEL);
if (mp->security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_early_task - during initialization allocate a composite task blob
* @task: the task that needs a blob
*
* Allocate the task blob for all the modules
*/
static void __init lsm_early_task(struct task_struct *task)
{
int rc = lsm_task_alloc(task);
if (rc)
panic("%s: Early task alloc failed.\n", __func__);
}
/**
* lsm_superblock_alloc - allocate a composite superblock blob
* @sb: the superblock that needs a blob
*
* Allocate the superblock blob for all the modules
*
* Returns 0, or -ENOMEM if memory can't be allocated.
*/
static int lsm_superblock_alloc(struct super_block *sb)
{
if (blob_sizes.lbs_superblock == 0) {
sb->s_security = NULL;
return 0;
}
sb->s_security = kzalloc(blob_sizes.lbs_superblock, GFP_KERNEL);
if (sb->s_security == NULL)
return -ENOMEM;
return 0;
}
/**
* lsm_fill_user_ctx - Fill a user space lsm_ctx structure
* @uctx: a userspace LSM context to be filled
* @uctx_len: available uctx size (input), used uctx size (output)
* @val: the new LSM context value
* @val_len: the size of the new LSM context value
* @id: LSM id
* @flags: LSM defined flags
*
* Fill all of the fields in a userspace lsm_ctx structure. If @uctx is NULL
* simply calculate the required size to output via @utc_len and return
* success.
*
* Returns 0 on success, -E2BIG if userspace buffer is not large enough,
* -EFAULT on a copyout error, -ENOMEM if memory can't be allocated.
*/
int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, u32 *uctx_len,
void *val, size_t val_len,
u64 id, u64 flags)
{
struct lsm_ctx *nctx = NULL;
size_t nctx_len;
int rc = 0;
nctx_len = ALIGN(struct_size(nctx, ctx, val_len), sizeof(void *));
if (nctx_len > *uctx_len) {
rc = -E2BIG;
goto out;
}
/* no buffer - return success/0 and set @uctx_len to the req size */
if (!uctx)
goto out;
nctx = kzalloc(nctx_len, GFP_KERNEL);
if (nctx == NULL) {
rc = -ENOMEM;
goto out;
}
nctx->id = id;
nctx->flags = flags;
nctx->len = nctx_len;
nctx->ctx_len = val_len;
memcpy(nctx->ctx, val, val_len);
if (copy_to_user(uctx, nctx, nctx_len))
rc = -EFAULT;
out:
kfree(nctx);
*uctx_len = nctx_len;
return rc;
}
/*
* The default value of the LSM hook is defined in linux/lsm_hook_defs.h and
* can be accessed with:
*
* LSM_RET_DEFAULT(<hook_name>)
*
* The macros below define static constants for the default value of each
* LSM hook.
*/
#define LSM_RET_DEFAULT(NAME) (NAME##_default)
#define DECLARE_LSM_RET_DEFAULT_void(DEFAULT, NAME)
#define DECLARE_LSM_RET_DEFAULT_int(DEFAULT, NAME) \
static const int __maybe_unused LSM_RET_DEFAULT(NAME) = (DEFAULT);
#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
#include <linux/lsm_hook_defs.h>
#undef LSM_HOOK
/*
* Hook list operation macros.
*
* call_void_hook:
* This is a hook that does not return a value.
*
* call_int_hook:
* This is a hook that returns a value.
*/
#define call_void_hook(FUNC, ...) \
do { \
struct security_hook_list *P; \
\
hlist_for_each_entry(P, &security_hook_heads.FUNC, list) \
P->hook.FUNC(__VA_ARGS__); \
} while (0)
#define call_int_hook(FUNC, ...) ({ \
int RC = LSM_RET_DEFAULT(FUNC); \
do { \
struct security_hook_list *P; \
\
hlist_for_each_entry(P, &security_hook_heads.FUNC, list) { \
RC = P->hook.FUNC(__VA_ARGS__); \
if (RC != LSM_RET_DEFAULT(FUNC)) \
break; \
} \
} while (0); \
RC; \
})
/* Security operations */
/**
* security_binder_set_context_mgr() - Check if becoming binder ctx mgr is ok
* @mgr: task credentials of current binder process
*
* Check whether @mgr is allowed to be the binder context manager.
*
* Return: Return 0 if permission is granted.
*/
int security_binder_set_context_mgr(const struct cred *mgr)
{
return call_int_hook(binder_set_context_mgr, mgr);
}
/**
* security_binder_transaction() - Check if a binder transaction is allowed
* @from: sending process
* @to: receiving process
*
* Check whether @from is allowed to invoke a binder transaction call to @to.
*
* Return: Returns 0 if permission is granted.
*/
int security_binder_transaction(const struct cred *from,
const struct cred *to)
{
return call_int_hook(binder_transaction, from, to);
}
/**
* security_binder_transfer_binder() - Check if a binder transfer is allowed
* @from: sending process
* @to: receiving process
*
* Check whether @from is allowed to transfer a binder reference to @to.
*
* Return: Returns 0 if permission is granted.
*/
int security_binder_transfer_binder(const struct cred *from,
const struct cred *to)
{
return call_int_hook(binder_transfer_binder, from, to);
}
/**
* security_binder_transfer_file() - Check if a binder file xfer is allowed
* @from: sending process
* @to: receiving process
* @file: file being transferred
*
* Check whether @from is allowed to transfer @file to @to.
*
* Return: Returns 0 if permission is granted.
*/
int security_binder_transfer_file(const struct cred *from,
const struct cred *to, const struct file *file)
{
return call_int_hook(binder_transfer_file, from, to, file);
}
/**
* security_ptrace_access_check() - Check if tracing is allowed
* @child: target process
* @mode: PTRACE_MODE flags
*
* Check permission before allowing the current process to trace the @child
* process. Security modules may also want to perform a process tracing check
* during an execve in the set_security or apply_creds hooks of tracing check
* during an execve in the bprm_set_creds hook of binprm_security_ops if the
* process is being traced and its security attributes would be changed by the
* execve.
*
* Return: Returns 0 if permission is granted.
*/
int security_ptrace_access_check(struct task_struct *child, unsigned int mode)
{
return call_int_hook(ptrace_access_check, child, mode);
}
/**
* security_ptrace_traceme() - Check if tracing is allowed
* @parent: tracing process
*
* Check that the @parent process has sufficient permission to trace the
* current process before allowing the current process to present itself to the
* @parent process for tracing.
*
* Return: Returns 0 if permission is granted.
*/
int security_ptrace_traceme(struct task_struct *parent)
{
return call_int_hook(ptrace_traceme, parent);
}
/**
* security_capget() - Get the capability sets for a process
* @target: target process
* @effective: effective capability set
* @inheritable: inheritable capability set
* @permitted: permitted capability set
*
* Get the @effective, @inheritable, and @permitted capability sets for the
* @target process. The hook may also perform permission checking to determine
* if the current process is allowed to see the capability sets of the @target
* process.
*
* Return: Returns 0 if the capability sets were successfully obtained.
*/
int security_capget(const struct task_struct *target,
kernel_cap_t *effective,
kernel_cap_t *inheritable,
kernel_cap_t *permitted)
{
return call_int_hook(capget, target, effective, inheritable, permitted);
}
/**
* security_capset() - Set the capability sets for a process
* @new: new credentials for the target process
* @old: current credentials of the target process
* @effective: effective capability set
* @inheritable: inheritable capability set
* @permitted: permitted capability set
*
* Set the @effective, @inheritable, and @permitted capability sets for the
* current process.
*
* Return: Returns 0 and update @new if permission is granted.
*/
int security_capset(struct cred *new, const struct cred *old,
const kernel_cap_t *effective,
const kernel_cap_t *inheritable,
const kernel_cap_t *permitted)
{
return call_int_hook(capset, new, old, effective, inheritable,
permitted);
}
/**
* security_capable() - Check if a process has the necessary capability
* @cred: credentials to examine
* @ns: user namespace
* @cap: capability requested
* @opts: capability check options
*
* Check whether the @tsk process has the @cap capability in the indicated
* credentials. @cap contains the capability <include/linux/capability.h>.
* @opts contains options for the capable check <include/linux/security.h>.
*
* Return: Returns 0 if the capability is granted.
*/
int security_capable(const struct cred *cred,
struct user_namespace *ns,
int cap,
unsigned int opts)
{
return call_int_hook(capable, cred, ns, cap, opts);
}
/**
* security_quotactl() - Check if a quotactl() syscall is allowed for this fs
* @cmds: commands
* @type: type
* @id: id
* @sb: filesystem
*
* Check whether the quotactl syscall is allowed for this @sb.
*
* Return: Returns 0 if permission is granted.
*/
int security_quotactl(int cmds, int type, int id, const struct super_block *sb)
{
return call_int_hook(quotactl, cmds, type, id, sb);
}
/**
* security_quota_on() - Check if QUOTAON is allowed for a dentry
* @dentry: dentry
*
* Check whether QUOTAON is allowed for @dentry.
*
* Return: Returns 0 if permission is granted.
*/
int security_quota_on(struct dentry *dentry)
{
return call_int_hook(quota_on, dentry);
}
/**
* security_syslog() - Check if accessing the kernel message ring is allowed
* @type: SYSLOG_ACTION_* type
*
* Check permission before accessing the kernel message ring or changing
* logging to the console. See the syslog(2) manual page for an explanation of
* the @type values.
*
* Return: Return 0 if permission is granted.
*/
int security_syslog(int type)
{
return call_int_hook(syslog, type);
}
/**
* security_settime64() - Check if changing the system time is allowed
* @ts: new time
* @tz: timezone
*
* Check permission to change the system time, struct timespec64 is defined in
* <include/linux/time64.h> and timezone is defined in <include/linux/time.h>.
*
* Return: Returns 0 if permission is granted.
*/
int security_settime64(const struct timespec64 *ts, const struct timezone *tz)
{
return call_int_hook(settime, ts, tz);
}
/**
* security_vm_enough_memory_mm() - Check if allocating a new mem map is allowed
* @mm: mm struct
* @pages: number of pages
*
* Check permissions for allocating a new virtual mapping. If all LSMs return
* a positive value, __vm_enough_memory() will be called with cap_sys_admin
* set. If at least one LSM returns 0 or negative, __vm_enough_memory() will be
* called with cap_sys_admin cleared.
*
* Return: Returns 0 if permission is granted by the LSM infrastructure to the
* caller.
*/
int security_vm_enough_memory_mm(struct mm_struct *mm, long pages)
{
struct security_hook_list *hp;
int cap_sys_admin = 1;
int rc;
/*
* The module will respond with a positive value if
* it thinks the __vm_enough_memory() call should be
* made with the cap_sys_admin set. If all of the modules
* agree that it should be set it will. If any module
* thinks it should not be set it won't.
*/
hlist_for_each_entry(hp, &security_hook_heads.vm_enough_memory, list) { rc = hp->hook.vm_enough_memory(mm, pages);
if (rc <= 0) {
cap_sys_admin = 0;
break;
}
}
return __vm_enough_memory(mm, pages, cap_sys_admin);
}
/**
* security_bprm_creds_for_exec() - Prepare the credentials for exec()
* @bprm: binary program information
*
* If the setup in prepare_exec_creds did not setup @bprm->cred->security
* properly for executing @bprm->file, update the LSM's portion of
* @bprm->cred->security to be what commit_creds needs to install for the new
* program. This hook may also optionally check permissions (e.g. for
* transitions between security domains). The hook must set @bprm->secureexec
* to 1 if AT_SECURE should be set to request libc enable secure mode. @bprm
* contains the linux_binprm structure.
*
* Return: Returns 0 if the hook is successful and permission is granted.
*/
int security_bprm_creds_for_exec(struct linux_binprm *bprm)
{
return call_int_hook(bprm_creds_for_exec, bprm);
}
/**
* security_bprm_creds_from_file() - Update linux_binprm creds based on file
* @bprm: binary program information
* @file: associated file
*
* If @file is setpcap, suid, sgid or otherwise marked to change privilege upon
* exec, update @bprm->cred to reflect that change. This is called after
* finding the binary that will be executed without an interpreter. This
* ensures that the credentials will not be derived from a script that the
* binary will need to reopen, which when reopend may end up being a completely
* different file. This hook may also optionally check permissions (e.g. for
* transitions between security domains). The hook must set @bprm->secureexec
* to 1 if AT_SECURE should be set to request libc enable secure mode. The
* hook must add to @bprm->per_clear any personality flags that should be
* cleared from current->personality. @bprm contains the linux_binprm
* structure.
*
* Return: Returns 0 if the hook is successful and permission is granted.
*/
int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file)
{
return call_int_hook(bprm_creds_from_file, bprm, file);
}
/**
* security_bprm_check() - Mediate binary handler search
* @bprm: binary program information
*
* This hook mediates the point when a search for a binary handler will begin.
* It allows a check against the @bprm->cred->security value which was set in
* the preceding creds_for_exec call. The argv list and envp list are reliably
* available in @bprm. This hook may be called multiple times during a single
* execve. @bprm contains the linux_binprm structure.
*
* Return: Returns 0 if the hook is successful and permission is granted.
*/
int security_bprm_check(struct linux_binprm *bprm)
{
return call_int_hook(bprm_check_security, bprm);
}
/**
* security_bprm_committing_creds() - Install creds for a process during exec()
* @bprm: binary program information
*
* Prepare to install the new security attributes of a process being
* transformed by an execve operation, based on the old credentials pointed to
* by @current->cred and the information set in @bprm->cred by the
* bprm_creds_for_exec hook. @bprm points to the linux_binprm structure. This
* hook is a good place to perform state changes on the process such as closing
* open file descriptors to which access will no longer be granted when the
* attributes are changed. This is called immediately before commit_creds().
*/
void security_bprm_committing_creds(const struct linux_binprm *bprm)
{
call_void_hook(bprm_committing_creds, bprm);
}
/**
* security_bprm_committed_creds() - Tidy up after cred install during exec()
* @bprm: binary program information
*
* Tidy up after the installation of the new security attributes of a process
* being transformed by an execve operation. The new credentials have, by this
* point, been set to @current->cred. @bprm points to the linux_binprm
* structure. This hook is a good place to perform state changes on the
* process such as clearing out non-inheritable signal state. This is called
* immediately after commit_creds().
*/
void security_bprm_committed_creds(const struct linux_binprm *bprm)
{
call_void_hook(bprm_committed_creds, bprm);
}
/**
* security_fs_context_submount() - Initialise fc->security
* @fc: new filesystem context
* @reference: dentry reference for submount/remount
*
* Fill out the ->security field for a new fs_context.
*
* Return: Returns 0 on success or negative error code on failure.
*/
int security_fs_context_submount(struct fs_context *fc, struct super_block *reference)
{
return call_int_hook(fs_context_submount, fc, reference);
}
/**
* security_fs_context_dup() - Duplicate a fs_context LSM blob
* @fc: destination filesystem context
* @src_fc: source filesystem context
*
* Allocate and attach a security structure to sc->security. This pointer is
* initialised to NULL by the caller. @fc indicates the new filesystem context.
* @src_fc indicates the original filesystem context.
*
* Return: Returns 0 on success or a negative error code on failure.
*/
int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc)
{
return call_int_hook(fs_context_dup, fc, src_fc);
}
/**
* security_fs_context_parse_param() - Configure a filesystem context
* @fc: filesystem context
* @param: filesystem parameter
*
* Userspace provided a parameter to configure a superblock. The LSM can
* consume the parameter or return it to the caller for use elsewhere.
*
* Return: If the parameter is used by the LSM it should return 0, if it is
* returned to the caller -ENOPARAM is returned, otherwise a negative
* error code is returned.
*/
int security_fs_context_parse_param(struct fs_context *fc,
struct fs_parameter *param)
{
struct security_hook_list *hp;
int trc;
int rc = -ENOPARAM;
hlist_for_each_entry(hp, &security_hook_heads.fs_context_parse_param,
list) {
trc = hp->hook.fs_context_parse_param(fc, param);
if (trc == 0)
rc = 0;
else if (trc != -ENOPARAM)
return trc;
}
return rc;
}
/**
* security_sb_alloc() - Allocate a super_block LSM blob
* @sb: filesystem superblock
*
* Allocate and attach a security structure to the sb->s_security field. The
* s_security field is initialized to NULL when the structure is allocated.
* @sb contains the super_block structure to be modified.
*
* Return: Returns 0 if operation was successful.
*/
int security_sb_alloc(struct super_block *sb)
{
int rc = lsm_superblock_alloc(sb);
if (unlikely(rc))
return rc;
rc = call_int_hook(sb_alloc_security, sb);
if (unlikely(rc))
security_sb_free(sb);
return rc;
}
/**
* security_sb_delete() - Release super_block LSM associated objects
* @sb: filesystem superblock
*
* Release objects tied to a superblock (e.g. inodes). @sb contains the
* super_block structure being released.
*/
void security_sb_delete(struct super_block *sb)
{
call_void_hook(sb_delete, sb);
}
/**
* security_sb_free() - Free a super_block LSM blob
* @sb: filesystem superblock
*
* Deallocate and clear the sb->s_security field. @sb contains the super_block
* structure to be modified.
*/
void security_sb_free(struct super_block *sb)
{
call_void_hook(sb_free_security, sb);
kfree(sb->s_security);
sb->s_security = NULL;
}
/**
* security_free_mnt_opts() - Free memory associated with mount options
* @mnt_opts: LSM processed mount options
*
* Free memory associated with @mnt_ops.
*/
void security_free_mnt_opts(void **mnt_opts)
{
if (!*mnt_opts)
return;
call_void_hook(sb_free_mnt_opts, *mnt_opts);
*mnt_opts = NULL;
}
EXPORT_SYMBOL(security_free_mnt_opts);
/**
* security_sb_eat_lsm_opts() - Consume LSM mount options
* @options: mount options
* @mnt_opts: LSM processed mount options
*
* Eat (scan @options) and save them in @mnt_opts.
*
* Return: Returns 0 on success, negative values on failure.
*/
int security_sb_eat_lsm_opts(char *options, void **mnt_opts)
{
return call_int_hook(sb_eat_lsm_opts, options, mnt_opts);
}
EXPORT_SYMBOL(security_sb_eat_lsm_opts);
/**
* security_sb_mnt_opts_compat() - Check if new mount options are allowed
* @sb: filesystem superblock
* @mnt_opts: new mount options
*
* Determine if the new mount options in @mnt_opts are allowed given the
* existing mounted filesystem at @sb. @sb superblock being compared.
*
* Return: Returns 0 if options are compatible.
*/
int security_sb_mnt_opts_compat(struct super_block *sb,
void *mnt_opts)
{
return call_int_hook(sb_mnt_opts_compat, sb, mnt_opts);
}
EXPORT_SYMBOL(security_sb_mnt_opts_compat);
/**
* security_sb_remount() - Verify no incompatible mount changes during remount
* @sb: filesystem superblock
* @mnt_opts: (re)mount options
*
* Extracts security system specific mount options and verifies no changes are
* being made to those options.
*
* Return: Returns 0 if permission is granted.
*/
int security_sb_remount(struct super_block *sb,
void *mnt_opts)
{
return call_int_hook(sb_remount, sb, mnt_opts);
}
EXPORT_SYMBOL(security_sb_remount);
/**
* security_sb_kern_mount() - Check if a kernel mount is allowed
* @sb: filesystem superblock
*
* Mount this @sb if allowed by permissions.
*
* Return: Returns 0 if permission is granted.
*/
int security_sb_kern_mount(const struct super_block *sb)
{
return call_int_hook(sb_kern_mount, sb);
}
/**
* security_sb_show_options() - Output the mount options for a superblock
* @m: output file
* @sb: filesystem superblock
*
* Show (print on @m) mount options for this @sb.
*
* Return: Returns 0 on success, negative values on failure.
*/
int security_sb_show_options(struct seq_file *m, struct super_block *sb)
{
return call_int_hook(sb_show_options, m, sb);
}
/**
* security_sb_statfs() - Check if accessing fs stats is allowed
* @dentry: superblock handle
*
* Check permission before obtaining filesystem statistics for the @mnt
* mountpoint. @dentry is a handle on the superblock for the filesystem.
*
* Return: Returns 0 if permission is granted.
*/
int security_sb_statfs(struct dentry *dentry)
{
return call_int_hook(sb_statfs, dentry);
}
/**
* security_sb_mount() - Check permission for mounting a filesystem
* @dev_name: filesystem backing device
* @path: mount point
* @type: filesystem type
* @flags: mount flags
* @data: filesystem specific data
*
* Check permission before an object specified by @dev_name is mounted on the
* mount point named by @nd. For an ordinary mount, @dev_name identifies a
* device if the file system type requires a device. For a remount
* (@flags & MS_REMOUNT), @dev_name is irrelevant. For a loopback/bind mount
* (@flags & MS_BIND), @dev_name identifies the pathname of the object being
* mounted.
*
* Return: Returns 0 if permission is granted.
*/
int security_sb_mount(const char *dev_name, const struct path *path,
const char *type, unsigned long flags, void *data)
{
return call_int_hook(sb_mount, dev_name, path, type, flags, data);
}
/**
* security_sb_umount() - Check permission for unmounting a filesystem
* @mnt: mounted filesystem
* @flags: unmount flags
*
* Check permission before the @mnt file system is unmounted.
*
* Return: Returns 0 if permission is granted.
*/
int security_sb_umount(struct vfsmount *mnt, int flags)
{
return call_int_hook(sb_umount, mnt, flags);
}
/**
* security_sb_pivotroot() - Check permissions for pivoting the rootfs
* @old_path: new location for current rootfs
* @new_path: location of the new rootfs
*
* Check permission before pivoting the root filesystem.
*
* Return: Returns 0 if permission is granted.
*/
int security_sb_pivotroot(const struct path *old_path,
const struct path *new_path)
{
return call_int_hook(sb_pivotroot, old_path, new_path);
}
/**
* security_sb_set_mnt_opts() - Set the mount options for a filesystem
* @sb: filesystem superblock
* @mnt_opts: binary mount options
* @kern_flags: kernel flags (in)
* @set_kern_flags: kernel flags (out)
*
* Set the security relevant mount options used for a superblock.
*
* Return: Returns 0 on success, error on failure.
*/
int security_sb_set_mnt_opts(struct super_block *sb,
void *mnt_opts,
unsigned long kern_flags,
unsigned long *set_kern_flags)
{
struct security_hook_list *hp;
int rc = mnt_opts ? -EOPNOTSUPP : LSM_RET_DEFAULT(sb_set_mnt_opts);
hlist_for_each_entry(hp, &security_hook_heads.sb_set_mnt_opts,
list) {
rc = hp->hook.sb_set_mnt_opts(sb, mnt_opts, kern_flags,
set_kern_flags);
if (rc != LSM_RET_DEFAULT(sb_set_mnt_opts))
break;
}
return rc;
}
EXPORT_SYMBOL(security_sb_set_mnt_opts);
/**
* security_sb_clone_mnt_opts() - Duplicate superblock mount options
* @oldsb: source superblock
* @newsb: destination superblock
* @kern_flags: kernel flags (in)
* @set_kern_flags: kernel flags (out)
*
* Copy all security options from a given superblock to another.
*
* Return: Returns 0 on success, error on failure.
*/
int security_sb_clone_mnt_opts(const struct super_block *oldsb,
struct super_block *newsb,
unsigned long kern_flags,
unsigned long *set_kern_flags)
{
return call_int_hook(sb_clone_mnt_opts, oldsb, newsb,
kern_flags, set_kern_flags);
}
EXPORT_SYMBOL(security_sb_clone_mnt_opts);
/**
* security_move_mount() - Check permissions for moving a mount
* @from_path: source mount point
* @to_path: destination mount point
*
* Check permission before a mount is moved.
*
* Return: Returns 0 if permission is granted.
*/
int security_move_mount(const struct path *from_path,
const struct path *to_path)
{
return call_int_hook(move_mount, from_path, to_path);
}
/**
* security_path_notify() - Check if setting a watch is allowed
* @path: file path
* @mask: event mask
* @obj_type: file path type
*
* Check permissions before setting a watch on events as defined by @mask, on
* an object at @path, whose type is defined by @obj_type.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_notify(const struct path *path, u64 mask,
unsigned int obj_type)
{
return call_int_hook(path_notify, path, mask, obj_type);
}
/**
* security_inode_alloc() - Allocate an inode LSM blob
* @inode: the inode
*
* Allocate and attach a security structure to @inode->i_security. The
* i_security field is initialized to NULL when the inode structure is
* allocated.
*
* Return: Return 0 if operation was successful.
*/
int security_inode_alloc(struct inode *inode)
{
int rc = lsm_inode_alloc(inode);
if (unlikely(rc))
return rc;
rc = call_int_hook(inode_alloc_security, inode); if (unlikely(rc)) security_inode_free(inode);
return rc;
}
static void inode_free_by_rcu(struct rcu_head *head)
{
/*
* The rcu head is at the start of the inode blob
*/
kmem_cache_free(lsm_inode_cache, head);
}
/**
* security_inode_free() - Free an inode's LSM blob
* @inode: the inode
*
* Deallocate the inode security structure and set @inode->i_security to NULL.
*/
void security_inode_free(struct inode *inode)
{
call_void_hook(inode_free_security, inode);
/*
* The inode may still be referenced in a path walk and
* a call to security_inode_permission() can be made
* after inode_free_security() is called. Ideally, the VFS
* wouldn't do this, but fixing that is a much harder
* job. For now, simply free the i_security via RCU, and
* leave the current inode->i_security pointer intact.
* The inode will be freed after the RCU grace period too.
*/
if (inode->i_security) call_rcu((struct rcu_head *)inode->i_security,
inode_free_by_rcu);
}
/**
* security_dentry_init_security() - Perform dentry initialization
* @dentry: the dentry to initialize
* @mode: mode used to determine resource type
* @name: name of the last path component
* @xattr_name: name of the security/LSM xattr
* @ctx: pointer to the resulting LSM context
* @ctxlen: length of @ctx
*
* Compute a context for a dentry as the inode is not yet available since NFSv4
* has no label backed by an EA anyway. It is important to note that
* @xattr_name does not need to be free'd by the caller, it is a static string.
*
* Return: Returns 0 on success, negative values on failure.
*/
int security_dentry_init_security(struct dentry *dentry, int mode,
const struct qstr *name,
const char **xattr_name, void **ctx,
u32 *ctxlen)
{
return call_int_hook(dentry_init_security, dentry, mode, name,
xattr_name, ctx, ctxlen);
}
EXPORT_SYMBOL(security_dentry_init_security);
/**
* security_dentry_create_files_as() - Perform dentry initialization
* @dentry: the dentry to initialize
* @mode: mode used to determine resource type
* @name: name of the last path component
* @old: creds to use for LSM context calculations
* @new: creds to modify
*
* Compute a context for a dentry as the inode is not yet available and set
* that context in passed in creds so that new files are created using that
* context. Context is calculated using the passed in creds and not the creds
* of the caller.
*
* Return: Returns 0 on success, error on failure.
*/
int security_dentry_create_files_as(struct dentry *dentry, int mode,
struct qstr *name,
const struct cred *old, struct cred *new)
{
return call_int_hook(dentry_create_files_as, dentry, mode,
name, old, new);
}
EXPORT_SYMBOL(security_dentry_create_files_as);
/**
* security_inode_init_security() - Initialize an inode's LSM context
* @inode: the inode
* @dir: parent directory
* @qstr: last component of the pathname
* @initxattrs: callback function to write xattrs
* @fs_data: filesystem specific data
*
* Obtain the security attribute name suffix and value to set on a newly
* created inode and set up the incore security field for the new inode. This
* hook is called by the fs code as part of the inode creation transaction and
* provides for atomic labeling of the inode, unlike the post_create/mkdir/...
* hooks called by the VFS.
*
* The hook function is expected to populate the xattrs array, by calling
* lsm_get_xattr_slot() to retrieve the slots reserved by the security module
* with the lbs_xattr_count field of the lsm_blob_sizes structure. For each
* slot, the hook function should set ->name to the attribute name suffix
* (e.g. selinux), to allocate ->value (will be freed by the caller) and set it
* to the attribute value, to set ->value_len to the length of the value. If
* the security module does not use security attributes or does not wish to put
* a security attribute on this particular inode, then it should return
* -EOPNOTSUPP to skip this processing.
*
* Return: Returns 0 if the LSM successfully initialized all of the inode
* security attributes that are required, negative values otherwise.
*/
int security_inode_init_security(struct inode *inode, struct inode *dir,
const struct qstr *qstr,
const initxattrs initxattrs, void *fs_data)
{
struct security_hook_list *hp;
struct xattr *new_xattrs = NULL;
int ret = -EOPNOTSUPP, xattr_count = 0;
if (unlikely(IS_PRIVATE(inode)))
return 0; if (!blob_sizes.lbs_xattr_count)
return 0;
if (initxattrs) {
/* Allocate +1 as terminator. */
new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 1,
sizeof(*new_xattrs), GFP_NOFS);
if (!new_xattrs)
return -ENOMEM;
}
hlist_for_each_entry(hp, &security_hook_heads.inode_init_security,
list) {
ret = hp->hook.inode_init_security(inode, dir, qstr, new_xattrs,
&xattr_count);
if (ret && ret != -EOPNOTSUPP)
goto out;
/*
* As documented in lsm_hooks.h, -EOPNOTSUPP in this context
* means that the LSM is not willing to provide an xattr, not
* that it wants to signal an error. Thus, continue to invoke
* the remaining LSMs.
*/
}
/* If initxattrs() is NULL, xattr_count is zero, skip the call. */
if (!xattr_count)
goto out;
ret = initxattrs(inode, new_xattrs, fs_data);
out:
for (; xattr_count > 0; xattr_count--) kfree(new_xattrs[xattr_count - 1].value); kfree(new_xattrs);
return (ret == -EOPNOTSUPP) ? 0 : ret;
}
EXPORT_SYMBOL(security_inode_init_security);
/**
* security_inode_init_security_anon() - Initialize an anonymous inode
* @inode: the inode
* @name: the anonymous inode class
* @context_inode: an optional related inode
*
* Set up the incore security field for the new anonymous inode and return
* whether the inode creation is permitted by the security module or not.
*
* Return: Returns 0 on success, -EACCES if the security module denies the
* creation of this inode, or another -errno upon other errors.
*/
int security_inode_init_security_anon(struct inode *inode,
const struct qstr *name,
const struct inode *context_inode)
{
return call_int_hook(inode_init_security_anon, inode, name,
context_inode);
}
#ifdef CONFIG_SECURITY_PATH
/**
* security_path_mknod() - Check if creating a special file is allowed
* @dir: parent directory
* @dentry: new file
* @mode: new file mode
* @dev: device number
*
* Check permissions when creating a file. Note that this hook is called even
* if mknod operation is being done for a regular file.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_mknod(const struct path *dir, struct dentry *dentry,
umode_t mode, unsigned int dev)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
return 0;
return call_int_hook(path_mknod, dir, dentry, mode, dev);
}
EXPORT_SYMBOL(security_path_mknod);
/**
* security_path_post_mknod() - Update inode security after reg file creation
* @idmap: idmap of the mount
* @dentry: new file
*
* Update inode security field after a regular file has been created.
*/
void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return;
call_void_hook(path_post_mknod, idmap, dentry);
}
/**
* security_path_mkdir() - Check if creating a new directory is allowed
* @dir: parent directory
* @dentry: new directory
* @mode: new directory mode
*
* Check permissions to create a new directory in the existing directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_mkdir(const struct path *dir, struct dentry *dentry,
umode_t mode)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
return 0;
return call_int_hook(path_mkdir, dir, dentry, mode);
}
EXPORT_SYMBOL(security_path_mkdir);
/**
* security_path_rmdir() - Check if removing a directory is allowed
* @dir: parent directory
* @dentry: directory to remove
*
* Check the permission to remove a directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_rmdir(const struct path *dir, struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
return 0;
return call_int_hook(path_rmdir, dir, dentry);
}
/**
* security_path_unlink() - Check if removing a hard link is allowed
* @dir: parent directory
* @dentry: file
*
* Check the permission to remove a hard link to a file.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_unlink(const struct path *dir, struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
return 0;
return call_int_hook(path_unlink, dir, dentry);
}
EXPORT_SYMBOL(security_path_unlink);
/**
* security_path_symlink() - Check if creating a symbolic link is allowed
* @dir: parent directory
* @dentry: symbolic link
* @old_name: file pathname
*
* Check the permission to create a symbolic link to a file.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_symlink(const struct path *dir, struct dentry *dentry,
const char *old_name)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
return 0;
return call_int_hook(path_symlink, dir, dentry, old_name);
}
/**
* security_path_link - Check if creating a hard link is allowed
* @old_dentry: existing file
* @new_dir: new parent directory
* @new_dentry: new link
*
* Check permission before creating a new hard link to a file.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_link(struct dentry *old_dentry, const struct path *new_dir,
struct dentry *new_dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry))))
return 0;
return call_int_hook(path_link, old_dentry, new_dir, new_dentry);
}
/**
* security_path_rename() - Check if renaming a file is allowed
* @old_dir: parent directory of the old file
* @old_dentry: the old file
* @new_dir: parent directory of the new file
* @new_dentry: the new file
* @flags: flags
*
* Check for permission to rename a file or directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_rename(const struct path *old_dir, struct dentry *old_dentry,
const struct path *new_dir, struct dentry *new_dentry,
unsigned int flags)
{
if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)) ||
(d_is_positive(new_dentry) &&
IS_PRIVATE(d_backing_inode(new_dentry)))))
return 0;
return call_int_hook(path_rename, old_dir, old_dentry, new_dir,
new_dentry, flags);
}
EXPORT_SYMBOL(security_path_rename);
/**
* security_path_truncate() - Check if truncating a file is allowed
* @path: file
*
* Check permission before truncating the file indicated by path. Note that
* truncation permissions may also be checked based on already opened files,
* using the security_file_truncate() hook.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_truncate(const struct path *path)
{
if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
return 0;
return call_int_hook(path_truncate, path);
}
/**
* security_path_chmod() - Check if changing the file's mode is allowed
* @path: file
* @mode: new mode
*
* Check for permission to change a mode of the file @path. The new mode is
* specified in @mode which is a bitmask of constants from
* <include/uapi/linux/stat.h>.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_chmod(const struct path *path, umode_t mode)
{
if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
return 0;
return call_int_hook(path_chmod, path, mode);
}
/**
* security_path_chown() - Check if changing the file's owner/group is allowed
* @path: file
* @uid: file owner
* @gid: file group
*
* Check for permission to change owner/group of a file or directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid)
{
if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
return 0;
return call_int_hook(path_chown, path, uid, gid);
}
/**
* security_path_chroot() - Check if changing the root directory is allowed
* @path: directory
*
* Check for permission to change root directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_path_chroot(const struct path *path)
{
return call_int_hook(path_chroot, path);
}
#endif /* CONFIG_SECURITY_PATH */
/**
* security_inode_create() - Check if creating a file is allowed
* @dir: the parent directory
* @dentry: the file being created
* @mode: requested file mode
*
* Check permission to create a regular file.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_create(struct inode *dir, struct dentry *dentry,
umode_t mode)
{
if (unlikely(IS_PRIVATE(dir)))
return 0;
return call_int_hook(inode_create, dir, dentry, mode);
}
EXPORT_SYMBOL_GPL(security_inode_create);
/**
* security_inode_post_create_tmpfile() - Update inode security of new tmpfile
* @idmap: idmap of the mount
* @inode: inode of the new tmpfile
*
* Update inode security data after a tmpfile has been created.
*/
void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
struct inode *inode)
{
if (unlikely(IS_PRIVATE(inode)))
return;
call_void_hook(inode_post_create_tmpfile, idmap, inode);
}
/**
* security_inode_link() - Check if creating a hard link is allowed
* @old_dentry: existing file
* @dir: new parent directory
* @new_dentry: new link
*
* Check permission before creating a new hard link to a file.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_link(struct dentry *old_dentry, struct inode *dir,
struct dentry *new_dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry))))
return 0;
return call_int_hook(inode_link, old_dentry, dir, new_dentry);
}
/**
* security_inode_unlink() - Check if removing a hard link is allowed
* @dir: parent directory
* @dentry: file
*
* Check the permission to remove a hard link to a file.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_unlink(struct inode *dir, struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_unlink, dir, dentry);
}
/**
* security_inode_symlink() - Check if creating a symbolic link is allowed
* @dir: parent directory
* @dentry: symbolic link
* @old_name: existing filename
*
* Check the permission to create a symbolic link to a file.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_symlink(struct inode *dir, struct dentry *dentry,
const char *old_name)
{
if (unlikely(IS_PRIVATE(dir)))
return 0;
return call_int_hook(inode_symlink, dir, dentry, old_name);
}
/**
* security_inode_mkdir() - Check if creation a new director is allowed
* @dir: parent directory
* @dentry: new directory
* @mode: new directory mode
*
* Check permissions to create a new directory in the existing directory
* associated with inode structure @dir.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
{
if (unlikely(IS_PRIVATE(dir)))
return 0;
return call_int_hook(inode_mkdir, dir, dentry, mode);
}
EXPORT_SYMBOL_GPL(security_inode_mkdir);
/**
* security_inode_rmdir() - Check if removing a directory is allowed
* @dir: parent directory
* @dentry: directory to be removed
*
* Check the permission to remove a directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_rmdir(struct inode *dir, struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_rmdir, dir, dentry);
}
/**
* security_inode_mknod() - Check if creating a special file is allowed
* @dir: parent directory
* @dentry: new file
* @mode: new file mode
* @dev: device number
*
* Check permissions when creating a special file (or a socket or a fifo file
* created via the mknod system call). Note that if mknod operation is being
* done for a regular file, then the create hook will be called and not this
* hook.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_mknod(struct inode *dir, struct dentry *dentry,
umode_t mode, dev_t dev)
{
if (unlikely(IS_PRIVATE(dir)))
return 0;
return call_int_hook(inode_mknod, dir, dentry, mode, dev);
}
/**
* security_inode_rename() - Check if renaming a file is allowed
* @old_dir: parent directory of the old file
* @old_dentry: the old file
* @new_dir: parent directory of the new file
* @new_dentry: the new file
* @flags: flags
*
* Check for permission to rename a file or directory.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry,
unsigned int flags)
{
if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)) ||
(d_is_positive(new_dentry) &&
IS_PRIVATE(d_backing_inode(new_dentry)))))
return 0;
if (flags & RENAME_EXCHANGE) {
int err = call_int_hook(inode_rename, new_dir, new_dentry,
old_dir, old_dentry);
if (err)
return err;
}
return call_int_hook(inode_rename, old_dir, old_dentry,
new_dir, new_dentry);
}
/**
* security_inode_readlink() - Check if reading a symbolic link is allowed
* @dentry: link
*
* Check the permission to read the symbolic link.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_readlink(struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_readlink, dentry);
}
/**
* security_inode_follow_link() - Check if following a symbolic link is allowed
* @dentry: link dentry
* @inode: link inode
* @rcu: true if in RCU-walk mode
*
* Check permission to follow a symbolic link when looking up a pathname. If
* @rcu is true, @inode is not stable.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_follow_link(struct dentry *dentry, struct inode *inode,
bool rcu)
{
if (unlikely(IS_PRIVATE(inode)))
return 0;
return call_int_hook(inode_follow_link, dentry, inode, rcu);
}
/**
* security_inode_permission() - Check if accessing an inode is allowed
* @inode: inode
* @mask: access mask
*
* Check permission before accessing an inode. This hook is called by the
* existing Linux permission function, so a security module can use it to
* provide additional checking for existing Linux permission checks. Notice
* that this hook is called when a file is opened (as well as many other
* operations), whereas the file_security_ops permission hook is called when
* the actual read/write operations are performed.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_permission(struct inode *inode, int mask)
{
if (unlikely(IS_PRIVATE(inode)))
return 0;
return call_int_hook(inode_permission, inode, mask);}
/**
* security_inode_setattr() - Check if setting file attributes is allowed
* @idmap: idmap of the mount
* @dentry: file
* @attr: new attributes
*
* Check permission before setting file attributes. Note that the kernel call
* to notify_change is performed from several locations, whenever file
* attributes change (such as when a file is truncated, chown/chmod operations,
* transferring disk quotas, etc).
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_setattr(struct mnt_idmap *idmap,
struct dentry *dentry, struct iattr *attr)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_setattr, idmap, dentry, attr);
}
EXPORT_SYMBOL_GPL(security_inode_setattr);
/**
* security_inode_post_setattr() - Update the inode after a setattr operation
* @idmap: idmap of the mount
* @dentry: file
* @ia_valid: file attributes set
*
* Update inode security field after successful setting file attributes.
*/
void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
int ia_valid)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return;
call_void_hook(inode_post_setattr, idmap, dentry, ia_valid);
}
/**
* security_inode_getattr() - Check if getting file attributes is allowed
* @path: file
*
* Check permission before obtaining file attributes.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_getattr(const struct path *path)
{
if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
return 0;
return call_int_hook(inode_getattr, path);
}
/**
* security_inode_setxattr() - Check if setting file xattrs is allowed
* @idmap: idmap of the mount
* @dentry: file
* @name: xattr name
* @value: xattr value
* @size: size of xattr value
* @flags: flags
*
* Check permission before setting the extended attributes.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_setxattr(struct mnt_idmap *idmap,
struct dentry *dentry, const char *name,
const void *value, size_t size, int flags)
{
int ret;
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
/*
* SELinux and Smack integrate the cap call,
* so assume that all LSMs supplying this call do so.
*/
ret = call_int_hook(inode_setxattr, idmap, dentry, name, value, size,
flags);
if (ret == 1)
ret = cap_inode_setxattr(dentry, name, value, size, flags);
return ret;
}
/**
* security_inode_set_acl() - Check if setting posix acls is allowed
* @idmap: idmap of the mount
* @dentry: file
* @acl_name: acl name
* @kacl: acl struct
*
* Check permission before setting posix acls, the posix acls in @kacl are
* identified by @acl_name.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_set_acl(struct mnt_idmap *idmap,
struct dentry *dentry, const char *acl_name,
struct posix_acl *kacl)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_set_acl, idmap, dentry, acl_name, kacl);
}
/**
* security_inode_post_set_acl() - Update inode security from posix acls set
* @dentry: file
* @acl_name: acl name
* @kacl: acl struct
*
* Update inode security data after successfully setting posix acls on @dentry.
* The posix acls in @kacl are identified by @acl_name.
*/
void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name,
struct posix_acl *kacl)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return;
call_void_hook(inode_post_set_acl, dentry, acl_name, kacl);
}
/**
* security_inode_get_acl() - Check if reading posix acls is allowed
* @idmap: idmap of the mount
* @dentry: file
* @acl_name: acl name
*
* Check permission before getting osix acls, the posix acls are identified by
* @acl_name.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_get_acl(struct mnt_idmap *idmap,
struct dentry *dentry, const char *acl_name)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_get_acl, idmap, dentry, acl_name);
}
/**
* security_inode_remove_acl() - Check if removing a posix acl is allowed
* @idmap: idmap of the mount
* @dentry: file
* @acl_name: acl name
*
* Check permission before removing posix acls, the posix acls are identified
* by @acl_name.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_remove_acl(struct mnt_idmap *idmap,
struct dentry *dentry, const char *acl_name)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_remove_acl, idmap, dentry, acl_name);
}
/**
* security_inode_post_remove_acl() - Update inode security after rm posix acls
* @idmap: idmap of the mount
* @dentry: file
* @acl_name: acl name
*
* Update inode security data after successfully removing posix acls on
* @dentry in @idmap. The posix acls are identified by @acl_name.
*/
void security_inode_post_remove_acl(struct mnt_idmap *idmap,
struct dentry *dentry, const char *acl_name)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return;
call_void_hook(inode_post_remove_acl, idmap, dentry, acl_name);
}
/**
* security_inode_post_setxattr() - Update the inode after a setxattr operation
* @dentry: file
* @name: xattr name
* @value: xattr value
* @size: xattr value size
* @flags: flags
*
* Update inode security field after successful setxattr operation.
*/
void security_inode_post_setxattr(struct dentry *dentry, const char *name,
const void *value, size_t size, int flags)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return;
call_void_hook(inode_post_setxattr, dentry, name, value, size, flags);
}
/**
* security_inode_getxattr() - Check if xattr access is allowed
* @dentry: file
* @name: xattr name
*
* Check permission before obtaining the extended attributes identified by
* @name for @dentry.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_getxattr(struct dentry *dentry, const char *name)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_getxattr, dentry, name);
}
/**
* security_inode_listxattr() - Check if listing xattrs is allowed
* @dentry: file
*
* Check permission before obtaining the list of extended attribute names for
* @dentry.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_listxattr(struct dentry *dentry)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
return call_int_hook(inode_listxattr, dentry);
}
/**
* security_inode_removexattr() - Check if removing an xattr is allowed
* @idmap: idmap of the mount
* @dentry: file
* @name: xattr name
*
* Check permission before removing the extended attribute identified by @name
* for @dentry.
*
* Return: Returns 0 if permission is granted.
*/
int security_inode_removexattr(struct mnt_idmap *idmap,
struct dentry *dentry, const char *name)
{
int ret;
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return 0;
/*
* SELinux and Smack integrate the cap call,
* so assume that all LSMs supplying this call do so.
*/
ret = call_int_hook(inode_removexattr, idmap, dentry, name);
if (ret == 1)
ret = cap_inode_removexattr(idmap, dentry, name);
return ret;
}
/**
* security_inode_post_removexattr() - Update the inode after a removexattr op
* @dentry: file
* @name: xattr name
*
* Update the inode after a successful removexattr operation.
*/
void security_inode_post_removexattr(struct dentry *dentry, const char *name)
{
if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
return;
call_void_hook(inode_post_removexattr, dentry, name);
}
/**
* security_inode_need_killpriv() - Check if security_inode_killpriv() required
* @dentry: associated dentry
*
* Called when an inode has been changed to determine if
* security_inode_killpriv() should be called.
*
* Return: Return <0 on error to abort the inode change operation, return 0 if
* security_inode_killpriv() does not need to be called, return >0 if
* security_inode_killpriv() does need to be called.
*/
int security_inode_need_killpriv(struct dentry *dentry)
{
return call_int_hook(inode_need_killpriv, dentry);
}
/**
* security_inode_killpriv() - The setuid bit is removed, update LSM state
* @idmap: idmap of the mount
* @dentry: associated dentry
*
* The @dentry's setuid bit is being removed. Remove similar security labels.
* Called with the dentry->d_inode->i_mutex held.
*
* Return: Return 0 on success. If error is returned, then the operation
* causing setuid bit removal is failed.
*/
int security_inode_killpriv(struct mnt_idmap *idmap,
struct dentry *dentry)
{
return call_int_hook(inode_killpriv, idmap, dentry);
}
/**
* security_inode_getsecurity() - Get the xattr security label of an inode
* @idmap: idmap of the mount
* @inode: inode
* @name: xattr name
* @buffer: security label buffer
* @alloc: allocation flag
*
* Retrieve a copy of the extended attribute representation of the security
* label associated with @name for @inode via @buffer. Note that @name is the
* remainder of the attribute name after the security prefix has been removed.
* @alloc is used to specify if the call should return a value via the buffer
* or just the value length.
*
* Return: Returns size of buffer on success.
*/
int security_inode_getsecurity(struct mnt_idmap *idmap,
struct inode *inode, const char *name,
void **buffer, bool alloc)
{
if (unlikely(IS_PRIVATE(inode)))
return LSM_RET_DEFAULT(inode_getsecurity);
return call_int_hook(inode_getsecurity, idmap, inode, name, buffer,
alloc);
}
/**
* security_inode_setsecurity() - Set the xattr security label of an inode
* @inode: inode
* @name: xattr name
* @value: security label
* @size: length of security label
* @flags: flags
*
* Set the security label associated with @name for @inode from the extended
* attribute value @value. @size indicates the size of the @value in bytes.
* @flags may be XATTR_CREATE, XATTR_REPLACE, or 0. Note that @name is the
* remainder of the attribute name after the security. prefix has been removed.
*
* Return: Returns 0 on success.
*/
int security_inode_setsecurity(struct inode *inode, const char *name,
const void *value, size_t size, int flags)
{
if (unlikely(IS_PRIVATE(inode)))
return LSM_RET_DEFAULT(inode_setsecurity);
return call_int_hook(inode_setsecurity, inode, name, value, size,
flags);
}
/**
* security_inode_listsecurity() - List the xattr security label names
* @inode: inode
* @buffer: buffer
* @buffer_size: size of buffer
*
* Copy the extended attribute names for the security labels associated with
* @inode into @buffer. The maximum size of @buffer is specified by
* @buffer_size. @buffer may be NULL to request the size of the buffer
* required.
*
* Return: Returns number of bytes used/required on success.
*/
int security_inode_listsecurity(struct inode *inode,
char *buffer, size_t buffer_size)
{
if (unlikely(IS_PRIVATE(inode)))
return 0;
return call_int_hook(inode_listsecurity, inode, buffer, buffer_size);
}
EXPORT_SYMBOL(security_inode_listsecurity);
/**
* security_inode_getsecid() - Get an inode's secid
* @inode: inode
* @secid: secid to return
*
* Get the secid associated with the node. In case of failure, @secid will be
* set to zero.
*/
void security_inode_getsecid(struct inode *inode, u32 *secid)
{
call_void_hook(inode_getsecid, inode, secid);
}
/**
* security_inode_copy_up() - Create new creds for an overlayfs copy-up op
* @src: union dentry of copy-up file
* @new: newly created creds
*
* A file is about to be copied up from lower layer to upper layer of overlay
* filesystem. Security module can prepare a set of new creds and modify as
* need be and return new creds. Caller will switch to new creds temporarily to
* create new file and release newly allocated creds.
*
* Return: Returns 0 on success or a negative error code on error.
*/
int security_inode_copy_up(struct dentry *src, struct cred **new)
{
return call_int_hook(inode_copy_up, src, new);
}
EXPORT_SYMBOL(security_inode_copy_up);
/**
* security_inode_copy_up_xattr() - Filter xattrs in an overlayfs copy-up op
* @name: xattr name
*
* Filter the xattrs being copied up when a unioned file is copied up from a
* lower layer to the union/overlay layer. The caller is responsible for
* reading and writing the xattrs, this hook is merely a filter.
*
* Return: Returns 0 to accept the xattr, 1 to discard the xattr, -EOPNOTSUPP
* if the security module does not know about attribute, or a negative
* error code to abort the copy up.
*/
int security_inode_copy_up_xattr(const char *name)
{
int rc;
/*
* The implementation can return 0 (accept the xattr), 1 (discard the
* xattr), -EOPNOTSUPP if it does not know anything about the xattr or
* any other error code in case of an error.
*/
rc = call_int_hook(inode_copy_up_xattr, name);
if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr))
return rc;
return LSM_RET_DEFAULT(inode_copy_up_xattr);
}
EXPORT_SYMBOL(security_inode_copy_up_xattr);
/**
* security_kernfs_init_security() - Init LSM context for a kernfs node
* @kn_dir: parent kernfs node
* @kn: the kernfs node to initialize
*
* Initialize the security context of a newly created kernfs node based on its
* own and its parent's attributes.
*
* Return: Returns 0 if permission is granted.
*/
int security_kernfs_init_security(struct kernfs_node *kn_dir,
struct kernfs_node *kn)
{
return call_int_hook(kernfs_init_security, kn_dir, kn);
}
/**
* security_file_permission() - Check file permissions
* @file: file
* @mask: requested permissions
*
* Check file permissions before accessing an open file. This hook is called
* by various operations that read or write files. A security module can use
* this hook to perform additional checking on these operations, e.g. to
* revalidate permissions on use to support privilege bracketing or policy
* changes. Notice that this hook is used when the actual read/write
* operations are performed, whereas the inode_security_ops hook is called when
* a file is opened (as well as many other operations). Although this hook can
* be used to revalidate permissions for various system call operations that
* read or write files, it does not address the revalidation of permissions for
* memory-mapped files. Security modules must handle this separately if they
* need such revalidation.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_permission(struct file *file, int mask)
{
return call_int_hook(file_permission, file, mask);
}
/**
* security_file_alloc() - Allocate and init a file's LSM blob
* @file: the file
*
* Allocate and attach a security structure to the file->f_security field. The
* security field is initialized to NULL when the structure is first created.
*
* Return: Return 0 if the hook is successful and permission is granted.
*/
int security_file_alloc(struct file *file)
{
int rc = lsm_file_alloc(file);
if (rc)
return rc;
rc = call_int_hook(file_alloc_security, file); if (unlikely(rc)) security_file_free(file);
return rc;
}
/**
* security_file_release() - Perform actions before releasing the file ref
* @file: the file
*
* Perform actions before releasing the last reference to a file.
*/
void security_file_release(struct file *file)
{
call_void_hook(file_release, file);
}
/**
* security_file_free() - Free a file's LSM blob
* @file: the file
*
* Deallocate and free any security structures stored in file->f_security.
*/
void security_file_free(struct file *file)
{
void *blob;
call_void_hook(file_free_security, file); blob = file->f_security;
if (blob) {
file->f_security = NULL;
kmem_cache_free(lsm_file_cache, blob);
}
}
/**
* security_file_ioctl() - Check if an ioctl is allowed
* @file: associated file
* @cmd: ioctl cmd
* @arg: ioctl arguments
*
* Check permission for an ioctl operation on @file. Note that @arg sometimes
* represents a user space pointer; in other cases, it may be a simple integer
* value. When @arg represents a user space pointer, it should never be used
* by the security module.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
return call_int_hook(file_ioctl, file, cmd, arg);
}
EXPORT_SYMBOL_GPL(security_file_ioctl);
/**
* security_file_ioctl_compat() - Check if an ioctl is allowed in compat mode
* @file: associated file
* @cmd: ioctl cmd
* @arg: ioctl arguments
*
* Compat version of security_file_ioctl() that correctly handles 32-bit
* processes running on 64-bit kernels.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_ioctl_compat(struct file *file, unsigned int cmd,
unsigned long arg)
{
return call_int_hook(file_ioctl_compat, file, cmd, arg);
}
EXPORT_SYMBOL_GPL(security_file_ioctl_compat);
static inline unsigned long mmap_prot(struct file *file, unsigned long prot)
{
/*
* Does we have PROT_READ and does the application expect
* it to imply PROT_EXEC? If not, nothing to talk about...
*/
if ((prot & (PROT_READ | PROT_EXEC)) != PROT_READ)
return prot;
if (!(current->personality & READ_IMPLIES_EXEC))
return prot;
/*
* if that's an anonymous mapping, let it.
*/
if (!file)
return prot | PROT_EXEC;
/*
* ditto if it's not on noexec mount, except that on !MMU we need
* NOMMU_MAP_EXEC (== VM_MAYEXEC) in this case
*/
if (!path_noexec(&file->f_path)) {
#ifndef CONFIG_MMU
if (file->f_op->mmap_capabilities) {
unsigned caps = file->f_op->mmap_capabilities(file);
if (!(caps & NOMMU_MAP_EXEC))
return prot;
}
#endif
return prot | PROT_EXEC;
}
/* anything on noexec mount won't get PROT_EXEC */
return prot;
}
/**
* security_mmap_file() - Check if mmap'ing a file is allowed
* @file: file
* @prot: protection applied by the kernel
* @flags: flags
*
* Check permissions for a mmap operation. The @file may be NULL, e.g. if
* mapping anonymous memory.
*
* Return: Returns 0 if permission is granted.
*/
int security_mmap_file(struct file *file, unsigned long prot,
unsigned long flags)
{
return call_int_hook(mmap_file, file, prot, mmap_prot(file, prot),
flags);
}
/**
* security_mmap_addr() - Check if mmap'ing an address is allowed
* @addr: address
*
* Check permissions for a mmap operation at @addr.
*
* Return: Returns 0 if permission is granted.
*/
int security_mmap_addr(unsigned long addr)
{
return call_int_hook(mmap_addr, addr);
}
/**
* security_file_mprotect() - Check if changing memory protections is allowed
* @vma: memory region
* @reqprot: application requested protection
* @prot: protection applied by the kernel
*
* Check permissions before changing memory access permissions.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot,
unsigned long prot)
{
return call_int_hook(file_mprotect, vma, reqprot, prot);
}
/**
* security_file_lock() - Check if a file lock is allowed
* @file: file
* @cmd: lock operation (e.g. F_RDLCK, F_WRLCK)
*
* Check permission before performing file locking operations. Note the hook
* mediates both flock and fcntl style locks.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_lock(struct file *file, unsigned int cmd)
{
return call_int_hook(file_lock, file, cmd);
}
/**
* security_file_fcntl() - Check if fcntl() op is allowed
* @file: file
* @cmd: fcntl command
* @arg: command argument
*
* Check permission before allowing the file operation specified by @cmd from
* being performed on the file @file. Note that @arg sometimes represents a
* user space pointer; in other cases, it may be a simple integer value. When
* @arg represents a user space pointer, it should never be used by the
* security module.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
{
return call_int_hook(file_fcntl, file, cmd, arg);
}
/**
* security_file_set_fowner() - Set the file owner info in the LSM blob
* @file: the file
*
* Save owner security information (typically from current->security) in
* file->f_security for later use by the send_sigiotask hook.
*
* Return: Returns 0 on success.
*/
void security_file_set_fowner(struct file *file)
{
call_void_hook(file_set_fowner, file);
}
/**
* security_file_send_sigiotask() - Check if sending SIGIO/SIGURG is allowed
* @tsk: target task
* @fown: signal sender
* @sig: signal to be sent, SIGIO is sent if 0
*
* Check permission for the file owner @fown to send SIGIO or SIGURG to the
* process @tsk. Note that this hook is sometimes called from interrupt. Note
* that the fown_struct, @fown, is never outside the context of a struct file,
* so the file structure (and associated security information) can always be
* obtained: container_of(fown, struct file, f_owner).
*
* Return: Returns 0 if permission is granted.
*/
int security_file_send_sigiotask(struct task_struct *tsk,
struct fown_struct *fown, int sig)
{
return call_int_hook(file_send_sigiotask, tsk, fown, sig);
}
/**
* security_file_receive() - Check if receiving a file via IPC is allowed
* @file: file being received
*
* This hook allows security modules to control the ability of a process to
* receive an open file descriptor via socket IPC.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_receive(struct file *file)
{
return call_int_hook(file_receive, file);
}
/**
* security_file_open() - Save open() time state for late use by the LSM
* @file:
*
* Save open-time permission checking state for later use upon file_permission,
* and recheck access if anything has changed since inode_permission.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_open(struct file *file)
{
int ret;
ret = call_int_hook(file_open, file); if (ret)
return ret;
return fsnotify_open_perm(file);
}
/**
* security_file_post_open() - Evaluate a file after it has been opened
* @file: the file
* @mask: access mask
*
* Evaluate an opened file and the access mask requested with open(). The hook
* is useful for LSMs that require the file content to be available in order to
* make decisions.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_post_open(struct file *file, int mask)
{
return call_int_hook(file_post_open, file, mask);
}
EXPORT_SYMBOL_GPL(security_file_post_open);
/**
* security_file_truncate() - Check if truncating a file is allowed
* @file: file
*
* Check permission before truncating a file, i.e. using ftruncate. Note that
* truncation permission may also be checked based on the path, using the
* @path_truncate hook.
*
* Return: Returns 0 if permission is granted.
*/
int security_file_truncate(struct file *file)
{
return call_int_hook(file_truncate, file);
}
/**
* security_task_alloc() - Allocate a task's LSM blob
* @task: the task
* @clone_flags: flags indicating what is being shared
*
* Handle allocation of task-related resources.
*
* Return: Returns a zero on success, negative values on failure.
*/
int security_task_alloc(struct task_struct *task, unsigned long clone_flags)
{
int rc = lsm_task_alloc(task);
if (rc)
return rc;
rc = call_int_hook(task_alloc, task, clone_flags);
if (unlikely(rc))
security_task_free(task);
return rc;
}
/**
* security_task_free() - Free a task's LSM blob and related resources
* @task: task
*
* Handle release of task-related resources. Note that this can be called from
* interrupt context.
*/
void security_task_free(struct task_struct *task)
{
call_void_hook(task_free, task);
kfree(task->security);
task->security = NULL;
}
/**
* security_cred_alloc_blank() - Allocate the min memory to allow cred_transfer
* @cred: credentials
* @gfp: gfp flags
*
* Only allocate sufficient memory and attach to @cred such that
* cred_transfer() will not get ENOMEM.
*
* Return: Returns 0 on success, negative values on failure.
*/
int security_cred_alloc_blank(struct cred *cred, gfp_t gfp)
{
int rc = lsm_cred_alloc(cred, gfp);
if (rc)
return rc;
rc = call_int_hook(cred_alloc_blank, cred, gfp);
if (unlikely(rc))
security_cred_free(cred);
return rc;
}
/**
* security_cred_free() - Free the cred's LSM blob and associated resources
* @cred: credentials
*
* Deallocate and clear the cred->security field in a set of credentials.
*/
void security_cred_free(struct cred *cred)
{
/*
* There is a failure case in prepare_creds() that
* may result in a call here with ->security being NULL.
*/
if (unlikely(cred->security == NULL))
return;
call_void_hook(cred_free, cred);
kfree(cred->security);
cred->security = NULL;
}
/**
* security_prepare_creds() - Prepare a new set of credentials
* @new: new credentials
* @old: original credentials
* @gfp: gfp flags
*
* Prepare a new set of credentials by copying the data from the old set.
*
* Return: Returns 0 on success, negative values on failure.
*/
int security_prepare_creds(struct cred *new, const struct cred *old, gfp_t gfp)
{
int rc = lsm_cred_alloc(new, gfp);
if (rc)
return rc;
rc = call_int_hook(cred_prepare, new, old, gfp);
if (unlikely(rc))
security_cred_free(new);
return rc;
}
/**
* security_transfer_creds() - Transfer creds
* @new: target credentials
* @old: original credentials
*
* Transfer data from original creds to new creds.
*/
void security_transfer_creds(struct cred *new, const struct cred *old)
{
call_void_hook(cred_transfer, new, old);
}
/**
* security_cred_getsecid() - Get the secid from a set of credentials
* @c: credentials
* @secid: secid value
*
* Retrieve the security identifier of the cred structure @c. In case of
* failure, @secid will be set to zero.
*/
void security_cred_getsecid(const struct cred *c, u32 *secid)
{
*secid = 0;
call_void_hook(cred_getsecid, c, secid);
}
EXPORT_SYMBOL(security_cred_getsecid);
/**
* security_kernel_act_as() - Set the kernel credentials to act as secid
* @new: credentials
* @secid: secid
*
* Set the credentials for a kernel service to act as (subjective context).
* The current task must be the one that nominated @secid.
*
* Return: Returns 0 if successful.
*/
int security_kernel_act_as(struct cred *new, u32 secid)
{
return call_int_hook(kernel_act_as, new, secid);
}
/**
* security_kernel_create_files_as() - Set file creation context using an inode
* @new: target credentials
* @inode: reference inode
*
* Set the file creation context in a set of credentials to be the same as the
* objective context of the specified inode. The current task must be the one
* that nominated @inode.
*
* Return: Returns 0 if successful.
*/
int security_kernel_create_files_as(struct cred *new, struct inode *inode)
{
return call_int_hook(kernel_create_files_as, new, inode);
}
/**
* security_kernel_module_request() - Check if loading a module is allowed
* @kmod_name: module name
*
* Ability to trigger the kernel to automatically upcall to userspace for
* userspace to load a kernel module with the given name.
*
* Return: Returns 0 if successful.
*/
int security_kernel_module_request(char *kmod_name)
{
return call_int_hook(kernel_module_request, kmod_name);
}
/**
* security_kernel_read_file() - Read a file specified by userspace
* @file: file
* @id: file identifier
* @contents: trust if security_kernel_post_read_file() will be called
*
* Read a file specified by userspace.
*
* Return: Returns 0 if permission is granted.
*/
int security_kernel_read_file(struct file *file, enum kernel_read_file_id id,
bool contents)
{
return call_int_hook(kernel_read_file, file, id, contents);
}
EXPORT_SYMBOL_GPL(security_kernel_read_file);
/**
* security_kernel_post_read_file() - Read a file specified by userspace
* @file: file
* @buf: file contents
* @size: size of file contents
* @id: file identifier
*
* Read a file specified by userspace. This must be paired with a prior call
* to security_kernel_read_file() call that indicated this hook would also be
* called, see security_kernel_read_file() for more information.
*
* Return: Returns 0 if permission is granted.
*/
int security_kernel_post_read_file(struct file *file, char *buf, loff_t size,
enum kernel_read_file_id id)
{
return call_int_hook(kernel_post_read_file, file, buf, size, id);
}
EXPORT_SYMBOL_GPL(security_kernel_post_read_file);
/**
* security_kernel_load_data() - Load data provided by userspace
* @id: data identifier
* @contents: true if security_kernel_post_load_data() will be called
*
* Load data provided by userspace.
*
* Return: Returns 0 if permission is granted.
*/
int security_kernel_load_data(enum kernel_load_data_id id, bool contents)
{
return call_int_hook(kernel_load_data, id, contents);
}
EXPORT_SYMBOL_GPL(security_kernel_load_data);
/**
* security_kernel_post_load_data() - Load userspace data from a non-file source
* @buf: data
* @size: size of data
* @id: data identifier
* @description: text description of data, specific to the id value
*
* Load data provided by a non-file source (usually userspace buffer). This
* must be paired with a prior security_kernel_load_data() call that indicated
* this hook would also be called, see security_kernel_load_data() for more
* information.
*
* Return: Returns 0 if permission is granted.
*/
int security_kernel_post_load_data(char *buf, loff_t size,
enum kernel_load_data_id id,
char *description)
{
return call_int_hook(kernel_post_load_data, buf, size, id, description);
}
EXPORT_SYMBOL_GPL(security_kernel_post_load_data);
/**
* security_task_fix_setuid() - Update LSM with new user id attributes
* @new: updated credentials
* @old: credentials being replaced
* @flags: LSM_SETID_* flag values
*
* Update the module's state after setting one or more of the user identity
* attributes of the current process. The @flags parameter indicates which of
* the set*uid system calls invoked this hook. If @new is the set of
* credentials that will be installed. Modifications should be made to this
* rather than to @current->cred.
*
* Return: Returns 0 on success.
*/
int security_task_fix_setuid(struct cred *new, const struct cred *old,
int flags)
{
return call_int_hook(task_fix_setuid, new, old, flags);
}
/**
* security_task_fix_setgid() - Update LSM with new group id attributes
* @new: updated credentials
* @old: credentials being replaced
* @flags: LSM_SETID_* flag value
*
* Update the module's state after setting one or more of the group identity
* attributes of the current process. The @flags parameter indicates which of
* the set*gid system calls invoked this hook. @new is the set of credentials
* that will be installed. Modifications should be made to this rather than to
* @current->cred.
*
* Return: Returns 0 on success.
*/
int security_task_fix_setgid(struct cred *new, const struct cred *old,
int flags)
{
return call_int_hook(task_fix_setgid, new, old, flags);
}
/**
* security_task_fix_setgroups() - Update LSM with new supplementary groups
* @new: updated credentials
* @old: credentials being replaced
*
* Update the module's state after setting the supplementary group identity
* attributes of the current process. @new is the set of credentials that will
* be installed. Modifications should be made to this rather than to
* @current->cred.
*
* Return: Returns 0 on success.
*/
int security_task_fix_setgroups(struct cred *new, const struct cred *old)
{
return call_int_hook(task_fix_setgroups, new, old);
}
/**
* security_task_setpgid() - Check if setting the pgid is allowed
* @p: task being modified
* @pgid: new pgid
*
* Check permission before setting the process group identifier of the process
* @p to @pgid.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_setpgid(struct task_struct *p, pid_t pgid)
{
return call_int_hook(task_setpgid, p, pgid);
}
/**
* security_task_getpgid() - Check if getting the pgid is allowed
* @p: task
*
* Check permission before getting the process group identifier of the process
* @p.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_getpgid(struct task_struct *p)
{
return call_int_hook(task_getpgid, p);
}
/**
* security_task_getsid() - Check if getting the session id is allowed
* @p: task
*
* Check permission before getting the session identifier of the process @p.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_getsid(struct task_struct *p)
{
return call_int_hook(task_getsid, p);
}
/**
* security_current_getsecid_subj() - Get the current task's subjective secid
* @secid: secid value
*
* Retrieve the subjective security identifier of the current task and return
* it in @secid. In case of failure, @secid will be set to zero.
*/
void security_current_getsecid_subj(u32 *secid)
{
*secid = 0; call_void_hook(current_getsecid_subj, secid);
}
EXPORT_SYMBOL(security_current_getsecid_subj);
/**
* security_task_getsecid_obj() - Get a task's objective secid
* @p: target task
* @secid: secid value
*
* Retrieve the objective security identifier of the task_struct in @p and
* return it in @secid. In case of failure, @secid will be set to zero.
*/
void security_task_getsecid_obj(struct task_struct *p, u32 *secid)
{
*secid = 0;
call_void_hook(task_getsecid_obj, p, secid);
}
EXPORT_SYMBOL(security_task_getsecid_obj);
/**
* security_task_setnice() - Check if setting a task's nice value is allowed
* @p: target task
* @nice: nice value
*
* Check permission before setting the nice value of @p to @nice.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_setnice(struct task_struct *p, int nice)
{
return call_int_hook(task_setnice, p, nice);
}
/**
* security_task_setioprio() - Check if setting a task's ioprio is allowed
* @p: target task
* @ioprio: ioprio value
*
* Check permission before setting the ioprio value of @p to @ioprio.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_setioprio(struct task_struct *p, int ioprio)
{
return call_int_hook(task_setioprio, p, ioprio);
}
/**
* security_task_getioprio() - Check if getting a task's ioprio is allowed
* @p: task
*
* Check permission before getting the ioprio value of @p.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_getioprio(struct task_struct *p)
{
return call_int_hook(task_getioprio, p);
}
/**
* security_task_prlimit() - Check if get/setting resources limits is allowed
* @cred: current task credentials
* @tcred: target task credentials
* @flags: LSM_PRLIMIT_* flag bits indicating a get/set/both
*
* Check permission before getting and/or setting the resource limits of
* another task.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_prlimit(const struct cred *cred, const struct cred *tcred,
unsigned int flags)
{
return call_int_hook(task_prlimit, cred, tcred, flags);
}
/**
* security_task_setrlimit() - Check if setting a new rlimit value is allowed
* @p: target task's group leader
* @resource: resource whose limit is being set
* @new_rlim: new resource limit
*
* Check permission before setting the resource limits of process @p for
* @resource to @new_rlim. The old resource limit values can be examined by
* dereferencing (p->signal->rlim + resource).
*
* Return: Returns 0 if permission is granted.
*/
int security_task_setrlimit(struct task_struct *p, unsigned int resource,
struct rlimit *new_rlim)
{
return call_int_hook(task_setrlimit, p, resource, new_rlim);
}
/**
* security_task_setscheduler() - Check if setting sched policy/param is allowed
* @p: target task
*
* Check permission before setting scheduling policy and/or parameters of
* process @p.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_setscheduler(struct task_struct *p)
{
return call_int_hook(task_setscheduler, p);
}
/**
* security_task_getscheduler() - Check if getting scheduling info is allowed
* @p: target task
*
* Check permission before obtaining scheduling information for process @p.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_getscheduler(struct task_struct *p)
{
return call_int_hook(task_getscheduler, p);
}
/**
* security_task_movememory() - Check if moving memory is allowed
* @p: task
*
* Check permission before moving memory owned by process @p.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_movememory(struct task_struct *p)
{
return call_int_hook(task_movememory, p);
}
/**
* security_task_kill() - Check if sending a signal is allowed
* @p: target process
* @info: signal information
* @sig: signal value
* @cred: credentials of the signal sender, NULL if @current
*
* Check permission before sending signal @sig to @p. @info can be NULL, the
* constant 1, or a pointer to a kernel_siginfo structure. If @info is 1 or
* SI_FROMKERNEL(info) is true, then the signal should be viewed as coming from
* the kernel and should typically be permitted. SIGIO signals are handled
* separately by the send_sigiotask hook in file_security_ops.
*
* Return: Returns 0 if permission is granted.
*/
int security_task_kill(struct task_struct *p, struct kernel_siginfo *info,
int sig, const struct cred *cred)
{
return call_int_hook(task_kill, p, info, sig, cred);
}
/**
* security_task_prctl() - Check if a prctl op is allowed
* @option: operation
* @arg2: argument
* @arg3: argument
* @arg4: argument
* @arg5: argument
*
* Check permission before performing a process control operation on the
* current process.
*
* Return: Return -ENOSYS if no-one wanted to handle this op, any other value
* to cause prctl() to return immediately with that value.
*/
int security_task_prctl(int option, unsigned long arg2, unsigned long arg3,
unsigned long arg4, unsigned long arg5)
{
int thisrc;
int rc = LSM_RET_DEFAULT(task_prctl);
struct security_hook_list *hp;
hlist_for_each_entry(hp, &security_hook_heads.task_prctl, list) {
thisrc = hp->hook.task_prctl(option, arg2, arg3, arg4, arg5);
if (thisrc != LSM_RET_DEFAULT(task_prctl)) {
rc = thisrc;
if (thisrc != 0)
break;
}
}
return rc;
}
/**
* security_task_to_inode() - Set the security attributes of a task's inode
* @p: task
* @inode: inode
*
* Set the security attributes for an inode based on an associated task's
* security attributes, e.g. for /proc/pid inodes.
*/
void security_task_to_inode(struct task_struct *p, struct inode *inode)
{
call_void_hook(task_to_inode, p, inode);
}
/**
* security_create_user_ns() - Check if creating a new userns is allowed
* @cred: prepared creds
*
* Check permission prior to creating a new user namespace.
*
* Return: Returns 0 if successful, otherwise < 0 error code.
*/
int security_create_user_ns(const struct cred *cred)
{
return call_int_hook(userns_create, cred);
}
/**
* security_ipc_permission() - Check if sysv ipc access is allowed
* @ipcp: ipc permission structure
* @flag: requested permissions
*
* Check permissions for access to IPC.
*
* Return: Returns 0 if permission is granted.
*/
int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag)
{
return call_int_hook(ipc_permission, ipcp, flag);
}
/**
* security_ipc_getsecid() - Get the sysv ipc object's secid
* @ipcp: ipc permission structure
* @secid: secid pointer
*
* Get the secid associated with the ipc object. In case of failure, @secid
* will be set to zero.
*/
void security_ipc_getsecid(struct kern_ipc_perm *ipcp, u32 *secid)
{
*secid = 0;
call_void_hook(ipc_getsecid, ipcp, secid);
}
/**
* security_msg_msg_alloc() - Allocate a sysv ipc message LSM blob
* @msg: message structure
*
* Allocate and attach a security structure to the msg->security field. The
* security field is initialized to NULL when the structure is first created.
*
* Return: Return 0 if operation was successful and permission is granted.
*/
int security_msg_msg_alloc(struct msg_msg *msg)
{
int rc = lsm_msg_msg_alloc(msg);
if (unlikely(rc))
return rc;
rc = call_int_hook(msg_msg_alloc_security, msg);
if (unlikely(rc))
security_msg_msg_free(msg);
return rc;
}
/**
* security_msg_msg_free() - Free a sysv ipc message LSM blob
* @msg: message structure
*
* Deallocate the security structure for this message.
*/
void security_msg_msg_free(struct msg_msg *msg)
{
call_void_hook(msg_msg_free_security, msg);
kfree(msg->security);
msg->security = NULL;
}
/**
* security_msg_queue_alloc() - Allocate a sysv ipc msg queue LSM blob
* @msq: sysv ipc permission structure
*
* Allocate and attach a security structure to @msg. The security field is
* initialized to NULL when the structure is first created.
*
* Return: Returns 0 if operation was successful and permission is granted.
*/
int security_msg_queue_alloc(struct kern_ipc_perm *msq)
{
int rc = lsm_ipc_alloc(msq);
if (unlikely(rc))
return rc;
rc = call_int_hook(msg_queue_alloc_security, msq);
if (unlikely(rc))
security_msg_queue_free(msq);
return rc;
}
/**
* security_msg_queue_free() - Free a sysv ipc msg queue LSM blob
* @msq: sysv ipc permission structure
*
* Deallocate security field @perm->security for the message queue.
*/
void security_msg_queue_free(struct kern_ipc_perm *msq)
{
call_void_hook(msg_queue_free_security, msq);
kfree(msq->security);
msq->security = NULL;
}
/**
* security_msg_queue_associate() - Check if a msg queue operation is allowed
* @msq: sysv ipc permission structure
* @msqflg: operation flags
*
* Check permission when a message queue is requested through the msgget system
* call. This hook is only called when returning the message queue identifier
* for an existing message queue, not when a new message queue is created.
*
* Return: Return 0 if permission is granted.
*/
int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg)
{
return call_int_hook(msg_queue_associate, msq, msqflg);
}
/**
* security_msg_queue_msgctl() - Check if a msg queue operation is allowed
* @msq: sysv ipc permission structure
* @cmd: operation
*
* Check permission when a message control operation specified by @cmd is to be
* performed on the message queue with permissions.
*
* Return: Returns 0 if permission is granted.
*/
int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd)
{
return call_int_hook(msg_queue_msgctl, msq, cmd);
}
/**
* security_msg_queue_msgsnd() - Check if sending a sysv ipc message is allowed
* @msq: sysv ipc permission structure
* @msg: message
* @msqflg: operation flags
*
* Check permission before a message, @msg, is enqueued on the message queue
* with permissions specified in @msq.
*
* Return: Returns 0 if permission is granted.
*/
int security_msg_queue_msgsnd(struct kern_ipc_perm *msq,
struct msg_msg *msg, int msqflg)
{
return call_int_hook(msg_queue_msgsnd, msq, msg, msqflg);
}
/**
* security_msg_queue_msgrcv() - Check if receiving a sysv ipc msg is allowed
* @msq: sysv ipc permission structure
* @msg: message
* @target: target task
* @type: type of message requested
* @mode: operation flags
*
* Check permission before a message, @msg, is removed from the message queue.
* The @target task structure contains a pointer to the process that will be
* receiving the message (not equal to the current process when inline receives
* are being performed).
*
* Return: Returns 0 if permission is granted.
*/
int security_msg_queue_msgrcv(struct kern_ipc_perm *msq, struct msg_msg *msg,
struct task_struct *target, long type, int mode)
{
return call_int_hook(msg_queue_msgrcv, msq, msg, target, type, mode);
}
/**
* security_shm_alloc() - Allocate a sysv shm LSM blob
* @shp: sysv ipc permission structure
*
* Allocate and attach a security structure to the @shp security field. The
* security field is initialized to NULL when the structure is first created.
*
* Return: Returns 0 if operation was successful and permission is granted.
*/
int security_shm_alloc(struct kern_ipc_perm *shp)
{
int rc = lsm_ipc_alloc(shp);
if (unlikely(rc))
return rc;
rc = call_int_hook(shm_alloc_security, shp);
if (unlikely(rc))
security_shm_free(shp);
return rc;
}
/**
* security_shm_free() - Free a sysv shm LSM blob
* @shp: sysv ipc permission structure
*
* Deallocate the security structure @perm->security for the memory segment.
*/
void security_shm_free(struct kern_ipc_perm *shp)
{
call_void_hook(shm_free_security, shp);
kfree(shp->security);
shp->security = NULL;
}
/**
* security_shm_associate() - Check if a sysv shm operation is allowed
* @shp: sysv ipc permission structure
* @shmflg: operation flags
*
* Check permission when a shared memory region is requested through the shmget
* system call. This hook is only called when returning the shared memory
* region identifier for an existing region, not when a new shared memory
* region is created.
*
* Return: Returns 0 if permission is granted.
*/
int security_shm_associate(struct kern_ipc_perm *shp, int shmflg)
{
return call_int_hook(shm_associate, shp, shmflg);
}
/**
* security_shm_shmctl() - Check if a sysv shm operation is allowed
* @shp: sysv ipc permission structure
* @cmd: operation
*
* Check permission when a shared memory control operation specified by @cmd is
* to be performed on the shared memory region with permissions in @shp.
*
* Return: Return 0 if permission is granted.
*/
int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd)
{
return call_int_hook(shm_shmctl, shp, cmd);
}
/**
* security_shm_shmat() - Check if a sysv shm attach operation is allowed
* @shp: sysv ipc permission structure
* @shmaddr: address of memory region to attach
* @shmflg: operation flags
*
* Check permissions prior to allowing the shmat system call to attach the
* shared memory segment with permissions @shp to the data segment of the
* calling process. The attaching address is specified by @shmaddr.
*
* Return: Returns 0 if permission is granted.
*/
int security_shm_shmat(struct kern_ipc_perm *shp,
char __user *shmaddr, int shmflg)
{
return call_int_hook(shm_shmat, shp, shmaddr, shmflg);
}
/**
* security_sem_alloc() - Allocate a sysv semaphore LSM blob
* @sma: sysv ipc permission structure
*
* Allocate and attach a security structure to the @sma security field. The
* security field is initialized to NULL when the structure is first created.
*
* Return: Returns 0 if operation was successful and permission is granted.
*/
int security_sem_alloc(struct kern_ipc_perm *sma)
{
int rc = lsm_ipc_alloc(sma);
if (unlikely(rc))
return rc;
rc = call_int_hook(sem_alloc_security, sma);
if (unlikely(rc))
security_sem_free(sma);
return rc;
}
/**
* security_sem_free() - Free a sysv semaphore LSM blob
* @sma: sysv ipc permission structure
*
* Deallocate security structure @sma->security for the semaphore.
*/
void security_sem_free(struct kern_ipc_perm *sma)
{
call_void_hook(sem_free_security, sma);
kfree(sma->security);
sma->security = NULL;
}
/**
* security_sem_associate() - Check if a sysv semaphore operation is allowed
* @sma: sysv ipc permission structure
* @semflg: operation flags
*
* Check permission when a semaphore is requested through the semget system
* call. This hook is only called when returning the semaphore identifier for
* an existing semaphore, not when a new one must be created.
*
* Return: Returns 0 if permission is granted.
*/
int security_sem_associate(struct kern_ipc_perm *sma, int semflg)
{
return call_int_hook(sem_associate, sma, semflg);
}
/**
* security_sem_semctl() - Check if a sysv semaphore operation is allowed
* @sma: sysv ipc permission structure
* @cmd: operation
*
* Check permission when a semaphore operation specified by @cmd is to be
* performed on the semaphore.
*
* Return: Returns 0 if permission is granted.
*/
int security_sem_semctl(struct kern_ipc_perm *sma, int cmd)
{
return call_int_hook(sem_semctl, sma, cmd);
}
/**
* security_sem_semop() - Check if a sysv semaphore operation is allowed
* @sma: sysv ipc permission structure
* @sops: operations to perform
* @nsops: number of operations
* @alter: flag indicating changes will be made
*
* Check permissions before performing operations on members of the semaphore
* set. If the @alter flag is nonzero, the semaphore set may be modified.
*
* Return: Returns 0 if permission is granted.
*/
int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops,
unsigned nsops, int alter)
{
return call_int_hook(sem_semop, sma, sops, nsops, alter);
}
/**
* security_d_instantiate() - Populate an inode's LSM state based on a dentry
* @dentry: dentry
* @inode: inode
*
* Fill in @inode security information for a @dentry if allowed.
*/
void security_d_instantiate(struct dentry *dentry, struct inode *inode)
{
if (unlikely(inode && IS_PRIVATE(inode)))
return;
call_void_hook(d_instantiate, dentry, inode);
}
EXPORT_SYMBOL(security_d_instantiate);
/*
* Please keep this in sync with it's counterpart in security/lsm_syscalls.c
*/
/**
* security_getselfattr - Read an LSM attribute of the current process.
* @attr: which attribute to return
* @uctx: the user-space destination for the information, or NULL
* @size: pointer to the size of space available to receive the data
* @flags: special handling options. LSM_FLAG_SINGLE indicates that only
* attributes associated with the LSM identified in the passed @ctx be
* reported.
*
* A NULL value for @uctx can be used to get both the number of attributes
* and the size of the data.
*
* Returns the number of attributes found on success, negative value
* on error. @size is reset to the total size of the data.
* If @size is insufficient to contain the data -E2BIG is returned.
*/
int security_getselfattr(unsigned int attr, struct lsm_ctx __user *uctx,
u32 __user *size, u32 flags)
{
struct security_hook_list *hp;
struct lsm_ctx lctx = { .id = LSM_ID_UNDEF, };
u8 __user *base = (u8 __user *)uctx;
u32 entrysize;
u32 total = 0;
u32 left;
bool toobig = false;
bool single = false;
int count = 0;
int rc;
if (attr == LSM_ATTR_UNDEF)
return -EINVAL;
if (size == NULL)
return -EINVAL;
if (get_user(left, size))
return -EFAULT;
if (flags) {
/*
* Only flag supported is LSM_FLAG_SINGLE
*/
if (flags != LSM_FLAG_SINGLE || !uctx)
return -EINVAL;
if (copy_from_user(&lctx, uctx, sizeof(lctx)))
return -EFAULT;
/*
* If the LSM ID isn't specified it is an error.
*/
if (lctx.id == LSM_ID_UNDEF)
return -EINVAL;
single = true;
}
/*
* In the usual case gather all the data from the LSMs.
* In the single case only get the data from the LSM specified.
*/
hlist_for_each_entry(hp, &security_hook_heads.getselfattr, list) {
if (single && lctx.id != hp->lsmid->id)
continue;
entrysize = left;
if (base)
uctx = (struct lsm_ctx __user *)(base + total);
rc = hp->hook.getselfattr(attr, uctx, &entrysize, flags);
if (rc == -EOPNOTSUPP) {
rc = 0;
continue;
}
if (rc == -E2BIG) {
rc = 0;
left = 0;
toobig = true;
} else if (rc < 0)
return rc;
else
left -= entrysize;
total += entrysize;
count += rc;
if (single)
break;
}
if (put_user(total, size))
return -EFAULT;
if (toobig)
return -E2BIG;
if (count == 0)
return LSM_RET_DEFAULT(getselfattr);
return count;
}
/*
* Please keep this in sync with it's counterpart in security/lsm_syscalls.c
*/
/**
* security_setselfattr - Set an LSM attribute on the current process.
* @attr: which attribute to set
* @uctx: the user-space source for the information
* @size: the size of the data
* @flags: reserved for future use, must be 0
*
* Set an LSM attribute for the current process. The LSM, attribute
* and new value are included in @uctx.
*
* Returns 0 on success, -EINVAL if the input is inconsistent, -EFAULT
* if the user buffer is inaccessible, E2BIG if size is too big, or an
* LSM specific failure.
*/
int security_setselfattr(unsigned int attr, struct lsm_ctx __user *uctx,
u32 size, u32 flags)
{
struct security_hook_list *hp;
struct lsm_ctx *lctx;
int rc = LSM_RET_DEFAULT(setselfattr);
u64 required_len;
if (flags)
return -EINVAL;
if (size < sizeof(*lctx))
return -EINVAL;
if (size > PAGE_SIZE)
return -E2BIG;
lctx = memdup_user(uctx, size);
if (IS_ERR(lctx))
return PTR_ERR(lctx);
if (size < lctx->len ||
check_add_overflow(sizeof(*lctx), lctx->ctx_len, &required_len) ||
lctx->len < required_len) {
rc = -EINVAL;
goto free_out;
}
hlist_for_each_entry(hp, &security_hook_heads.setselfattr, list)
if ((hp->lsmid->id) == lctx->id) {
rc = hp->hook.setselfattr(attr, lctx, size, flags);
break;
}
free_out:
kfree(lctx);
return rc;
}
/**
* security_getprocattr() - Read an attribute for a task
* @p: the task
* @lsmid: LSM identification
* @name: attribute name
* @value: attribute value
*
* Read attribute @name for task @p and store it into @value if allowed.
*
* Return: Returns the length of @value on success, a negative value otherwise.
*/
int security_getprocattr(struct task_struct *p, int lsmid, const char *name,
char **value)
{
struct security_hook_list *hp;
hlist_for_each_entry(hp, &security_hook_heads.getprocattr, list) {
if (lsmid != 0 && lsmid != hp->lsmid->id)
continue;
return hp->hook.getprocattr(p, name, value);
}
return LSM_RET_DEFAULT(getprocattr);
}
/**
* security_setprocattr() - Set an attribute for a task
* @lsmid: LSM identification
* @name: attribute name
* @value: attribute value
* @size: attribute value size
*
* Write (set) the current task's attribute @name to @value, size @size if
* allowed.
*
* Return: Returns bytes written on success, a negative value otherwise.
*/
int security_setprocattr(int lsmid, const char *name, void *value, size_t size)
{
struct security_hook_list *hp;
hlist_for_each_entry(hp, &security_hook_heads.setprocattr, list) {
if (lsmid != 0 && lsmid != hp->lsmid->id)
continue;
return hp->hook.setprocattr(name, value, size);
}
return LSM_RET_DEFAULT(setprocattr);
}
/**
* security_netlink_send() - Save info and check if netlink sending is allowed
* @sk: sending socket
* @skb: netlink message
*
* Save security information for a netlink message so that permission checking
* can be performed when the message is processed. The security information
* can be saved using the eff_cap field of the netlink_skb_parms structure.
* Also may be used to provide fine grained control over message transmission.
*
* Return: Returns 0 if the information was successfully saved and message is
* allowed to be transmitted.
*/
int security_netlink_send(struct sock *sk, struct sk_buff *skb)
{
return call_int_hook(netlink_send, sk, skb);
}
/**
* security_ismaclabel() - Check if the named attribute is a MAC label
* @name: full extended attribute name
*
* Check if the extended attribute specified by @name represents a MAC label.
*
* Return: Returns 1 if name is a MAC attribute otherwise returns 0.
*/
int security_ismaclabel(const char *name)
{
return call_int_hook(ismaclabel, name);
}
EXPORT_SYMBOL(security_ismaclabel);
/**
* security_secid_to_secctx() - Convert a secid to a secctx
* @secid: secid
* @secdata: secctx
* @seclen: secctx length
*
* Convert secid to security context. If @secdata is NULL the length of the
* result will be returned in @seclen, but no @secdata will be returned. This
* does mean that the length could change between calls to check the length and
* the next call which actually allocates and returns the @secdata.
*
* Return: Return 0 on success, error on failure.
*/
int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen)
{
return call_int_hook(secid_to_secctx, secid, secdata, seclen);
}
EXPORT_SYMBOL(security_secid_to_secctx);
/**
* security_secctx_to_secid() - Convert a secctx to a secid
* @secdata: secctx
* @seclen: length of secctx
* @secid: secid
*
* Convert security context to secid.
*
* Return: Returns 0 on success, error on failure.
*/
int security_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid)
{
*secid = 0;
return call_int_hook(secctx_to_secid, secdata, seclen, secid);
}
EXPORT_SYMBOL(security_secctx_to_secid);
/**
* security_release_secctx() - Free a secctx buffer
* @secdata: secctx
* @seclen: length of secctx
*
* Release the security context.
*/
void security_release_secctx(char *secdata, u32 seclen)
{
call_void_hook(release_secctx, secdata, seclen);
}
EXPORT_SYMBOL(security_release_secctx);
/**
* security_inode_invalidate_secctx() - Invalidate an inode's security label
* @inode: inode
*
* Notify the security module that it must revalidate the security context of
* an inode.
*/
void security_inode_invalidate_secctx(struct inode *inode)
{
call_void_hook(inode_invalidate_secctx, inode);
}
EXPORT_SYMBOL(security_inode_invalidate_secctx);
/**
* security_inode_notifysecctx() - Notify the LSM of an inode's security label
* @inode: inode
* @ctx: secctx
* @ctxlen: length of secctx
*
* Notify the security module of what the security context of an inode should
* be. Initializes the incore security context managed by the security module
* for this inode. Example usage: NFS client invokes this hook to initialize
* the security context in its incore inode to the value provided by the server
* for the file when the server returned the file's attributes to the client.
* Must be called with inode->i_mutex locked.
*
* Return: Returns 0 on success, error on failure.
*/
int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen)
{
return call_int_hook(inode_notifysecctx, inode, ctx, ctxlen);
}
EXPORT_SYMBOL(security_inode_notifysecctx);
/**
* security_inode_setsecctx() - Change the security label of an inode
* @dentry: inode
* @ctx: secctx
* @ctxlen: length of secctx
*
* Change the security context of an inode. Updates the incore security
* context managed by the security module and invokes the fs code as needed
* (via __vfs_setxattr_noperm) to update any backing xattrs that represent the
* context. Example usage: NFS server invokes this hook to change the security
* context in its incore inode and on the backing filesystem to a value
* provided by the client on a SETATTR operation. Must be called with
* inode->i_mutex locked.
*
* Return: Returns 0 on success, error on failure.
*/
int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen)
{
return call_int_hook(inode_setsecctx, dentry, ctx, ctxlen);
}
EXPORT_SYMBOL(security_inode_setsecctx);
/**
* security_inode_getsecctx() - Get the security label of an inode
* @inode: inode
* @ctx: secctx
* @ctxlen: length of secctx
*
* On success, returns 0 and fills out @ctx and @ctxlen with the security
* context for the given @inode.
*
* Return: Returns 0 on success, error on failure.
*/
int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen)
{
return call_int_hook(inode_getsecctx, inode, ctx, ctxlen);
}
EXPORT_SYMBOL(security_inode_getsecctx);
#ifdef CONFIG_WATCH_QUEUE
/**
* security_post_notification() - Check if a watch notification can be posted
* @w_cred: credentials of the task that set the watch
* @cred: credentials of the task which triggered the watch
* @n: the notification
*
* Check to see if a watch notification can be posted to a particular queue.
*
* Return: Returns 0 if permission is granted.
*/
int security_post_notification(const struct cred *w_cred,
const struct cred *cred,
struct watch_notification *n)
{
return call_int_hook(post_notification, w_cred, cred, n);
}
#endif /* CONFIG_WATCH_QUEUE */
#ifdef CONFIG_KEY_NOTIFICATIONS
/**
* security_watch_key() - Check if a task is allowed to watch for key events
* @key: the key to watch
*
* Check to see if a process is allowed to watch for event notifications from
* a key or keyring.
*
* Return: Returns 0 if permission is granted.
*/
int security_watch_key(struct key *key)
{
return call_int_hook(watch_key, key);
}
#endif /* CONFIG_KEY_NOTIFICATIONS */
#ifdef CONFIG_SECURITY_NETWORK
/**
* security_unix_stream_connect() - Check if a AF_UNIX stream is allowed
* @sock: originating sock
* @other: peer sock
* @newsk: new sock
*
* Check permissions before establishing a Unix domain stream connection
* between @sock and @other.
*
* The @unix_stream_connect and @unix_may_send hooks were necessary because
* Linux provides an alternative to the conventional file name space for Unix
* domain sockets. Whereas binding and connecting to sockets in the file name
* space is mediated by the typical file permissions (and caught by the mknod
* and permission hooks in inode_security_ops), binding and connecting to
* sockets in the abstract name space is completely unmediated. Sufficient
* control of Unix domain sockets in the abstract name space isn't possible
* using only the socket layer hooks, since we need to know the actual target
* socket, which is not looked up until we are inside the af_unix code.
*
* Return: Returns 0 if permission is granted.
*/
int security_unix_stream_connect(struct sock *sock, struct sock *other,
struct sock *newsk)
{
return call_int_hook(unix_stream_connect, sock, other, newsk);
}
EXPORT_SYMBOL(security_unix_stream_connect);
/**
* security_unix_may_send() - Check if AF_UNIX socket can send datagrams
* @sock: originating sock
* @other: peer sock
*
* Check permissions before connecting or sending datagrams from @sock to
* @other.
*
* The @unix_stream_connect and @unix_may_send hooks were necessary because
* Linux provides an alternative to the conventional file name space for Unix
* domain sockets. Whereas binding and connecting to sockets in the file name
* space is mediated by the typical file permissions (and caught by the mknod
* and permission hooks in inode_security_ops), binding and connecting to
* sockets in the abstract name space is completely unmediated. Sufficient
* control of Unix domain sockets in the abstract name space isn't possible
* using only the socket layer hooks, since we need to know the actual target
* socket, which is not looked up until we are inside the af_unix code.
*
* Return: Returns 0 if permission is granted.
*/
int security_unix_may_send(struct socket *sock, struct socket *other)
{
return call_int_hook(unix_may_send, sock, other);
}
EXPORT_SYMBOL(security_unix_may_send);
/**
* security_socket_create() - Check if creating a new socket is allowed
* @family: protocol family
* @type: communications type
* @protocol: requested protocol
* @kern: set to 1 if a kernel socket is requested
*
* Check permissions prior to creating a new socket.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_create(int family, int type, int protocol, int kern)
{
return call_int_hook(socket_create, family, type, protocol, kern);
}
/**
* security_socket_post_create() - Initialize a newly created socket
* @sock: socket
* @family: protocol family
* @type: communications type
* @protocol: requested protocol
* @kern: set to 1 if a kernel socket is requested
*
* This hook allows a module to update or allocate a per-socket security
* structure. Note that the security field was not added directly to the socket
* structure, but rather, the socket security information is stored in the
* associated inode. Typically, the inode alloc_security hook will allocate
* and attach security information to SOCK_INODE(sock)->i_security. This hook
* may be used to update the SOCK_INODE(sock)->i_security field with additional
* information that wasn't available when the inode was allocated.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_post_create(struct socket *sock, int family,
int type, int protocol, int kern)
{
return call_int_hook(socket_post_create, sock, family, type,
protocol, kern);
}
/**
* security_socket_socketpair() - Check if creating a socketpair is allowed
* @socka: first socket
* @sockb: second socket
*
* Check permissions before creating a fresh pair of sockets.
*
* Return: Returns 0 if permission is granted and the connection was
* established.
*/
int security_socket_socketpair(struct socket *socka, struct socket *sockb)
{
return call_int_hook(socket_socketpair, socka, sockb);
}
EXPORT_SYMBOL(security_socket_socketpair);
/**
* security_socket_bind() - Check if a socket bind operation is allowed
* @sock: socket
* @address: requested bind address
* @addrlen: length of address
*
* Check permission before socket protocol layer bind operation is performed
* and the socket @sock is bound to the address specified in the @address
* parameter.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_bind(struct socket *sock,
struct sockaddr *address, int addrlen)
{
return call_int_hook(socket_bind, sock, address, addrlen);
}
/**
* security_socket_connect() - Check if a socket connect operation is allowed
* @sock: socket
* @address: address of remote connection point
* @addrlen: length of address
*
* Check permission before socket protocol layer connect operation attempts to
* connect socket @sock to a remote address, @address.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_connect(struct socket *sock,
struct sockaddr *address, int addrlen)
{
return call_int_hook(socket_connect, sock, address, addrlen);
}
/**
* security_socket_listen() - Check if a socket is allowed to listen
* @sock: socket
* @backlog: connection queue size
*
* Check permission before socket protocol layer listen operation.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_listen(struct socket *sock, int backlog)
{
return call_int_hook(socket_listen, sock, backlog);
}
/**
* security_socket_accept() - Check if a socket is allowed to accept connections
* @sock: listening socket
* @newsock: newly creation connection socket
*
* Check permission before accepting a new connection. Note that the new
* socket, @newsock, has been created and some information copied to it, but
* the accept operation has not actually been performed.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_accept(struct socket *sock, struct socket *newsock)
{
return call_int_hook(socket_accept, sock, newsock);
}
/**
* security_socket_sendmsg() - Check if sending a message is allowed
* @sock: sending socket
* @msg: message to send
* @size: size of message
*
* Check permission before transmitting a message to another socket.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size)
{
return call_int_hook(socket_sendmsg, sock, msg, size);
}
/**
* security_socket_recvmsg() - Check if receiving a message is allowed
* @sock: receiving socket
* @msg: message to receive
* @size: size of message
* @flags: operational flags
*
* Check permission before receiving a message from a socket.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_recvmsg(struct socket *sock, struct msghdr *msg,
int size, int flags)
{
return call_int_hook(socket_recvmsg, sock, msg, size, flags);
}
/**
* security_socket_getsockname() - Check if reading the socket addr is allowed
* @sock: socket
*
* Check permission before reading the local address (name) of the socket
* object.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_getsockname(struct socket *sock)
{
return call_int_hook(socket_getsockname, sock);
}
/**
* security_socket_getpeername() - Check if reading the peer's addr is allowed
* @sock: socket
*
* Check permission before the remote address (name) of a socket object.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_getpeername(struct socket *sock)
{
return call_int_hook(socket_getpeername, sock);
}
/**
* security_socket_getsockopt() - Check if reading a socket option is allowed
* @sock: socket
* @level: option's protocol level
* @optname: option name
*
* Check permissions before retrieving the options associated with socket
* @sock.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_getsockopt(struct socket *sock, int level, int optname)
{
return call_int_hook(socket_getsockopt, sock, level, optname);
}
/**
* security_socket_setsockopt() - Check if setting a socket option is allowed
* @sock: socket
* @level: option's protocol level
* @optname: option name
*
* Check permissions before setting the options associated with socket @sock.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_setsockopt(struct socket *sock, int level, int optname)
{
return call_int_hook(socket_setsockopt, sock, level, optname);
}
/**
* security_socket_shutdown() - Checks if shutting down the socket is allowed
* @sock: socket
* @how: flag indicating how sends and receives are handled
*
* Checks permission before all or part of a connection on the socket @sock is
* shut down.
*
* Return: Returns 0 if permission is granted.
*/
int security_socket_shutdown(struct socket *sock, int how)
{
return call_int_hook(socket_shutdown, sock, how);
}
/**
* security_sock_rcv_skb() - Check if an incoming network packet is allowed
* @sk: destination sock
* @skb: incoming packet
*
* Check permissions on incoming network packets. This hook is distinct from
* Netfilter's IP input hooks since it is the first time that the incoming
* sk_buff @skb has been associated with a particular socket, @sk. Must not
* sleep inside this hook because some callers hold spinlocks.
*
* Return: Returns 0 if permission is granted.
*/
int security_sock_rcv_skb(struct sock *sk, struct sk_buff *skb)
{
return call_int_hook(socket_sock_rcv_skb, sk, skb);
}
EXPORT_SYMBOL(security_sock_rcv_skb);
/**
* security_socket_getpeersec_stream() - Get the remote peer label
* @sock: socket
* @optval: destination buffer
* @optlen: size of peer label copied into the buffer
* @len: maximum size of the destination buffer
*
* This hook allows the security module to provide peer socket security state
* for unix or connected tcp sockets to userspace via getsockopt SO_GETPEERSEC.
* For tcp sockets this can be meaningful if the socket is associated with an
* ipsec SA.
*
* Return: Returns 0 if all is well, otherwise, typical getsockopt return
* values.
*/
int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval,
sockptr_t optlen, unsigned int len)
{
return call_int_hook(socket_getpeersec_stream, sock, optval, optlen,
len);
}
/**
* security_socket_getpeersec_dgram() - Get the remote peer label
* @sock: socket
* @skb: datagram packet
* @secid: remote peer label secid
*
* This hook allows the security module to provide peer socket security state
* for udp sockets on a per-packet basis to userspace via getsockopt
* SO_GETPEERSEC. The application must first have indicated the IP_PASSSEC
* option via getsockopt. It can then retrieve the security state returned by
* this hook for a packet via the SCM_SECURITY ancillary message type.
*
* Return: Returns 0 on success, error on failure.
*/
int security_socket_getpeersec_dgram(struct socket *sock,
struct sk_buff *skb, u32 *secid)
{
return call_int_hook(socket_getpeersec_dgram, sock, skb, secid);
}
EXPORT_SYMBOL(security_socket_getpeersec_dgram);
/**
* security_sk_alloc() - Allocate and initialize a sock's LSM blob
* @sk: sock
* @family: protocol family
* @priority: gfp flags
*
* Allocate and attach a security structure to the sk->sk_security field, which
* is used to copy security attributes between local stream sockets.
*
* Return: Returns 0 on success, error on failure.
*/
int security_sk_alloc(struct sock *sk, int family, gfp_t priority)
{
return call_int_hook(sk_alloc_security, sk, family, priority);
}
/**
* security_sk_free() - Free the sock's LSM blob
* @sk: sock
*
* Deallocate security structure.
*/
void security_sk_free(struct sock *sk)
{
call_void_hook(sk_free_security, sk);
}
/**
* security_sk_clone() - Clone a sock's LSM state
* @sk: original sock
* @newsk: target sock
*
* Clone/copy security structure.
*/
void security_sk_clone(const struct sock *sk, struct sock *newsk)
{
call_void_hook(sk_clone_security, sk, newsk);
}
EXPORT_SYMBOL(security_sk_clone);
/**
* security_sk_classify_flow() - Set a flow's secid based on socket
* @sk: original socket
* @flic: target flow
*
* Set the target flow's secid to socket's secid.
*/
void security_sk_classify_flow(const struct sock *sk, struct flowi_common *flic)
{
call_void_hook(sk_getsecid, sk, &flic->flowic_secid);
}
EXPORT_SYMBOL(security_sk_classify_flow);
/**
* security_req_classify_flow() - Set a flow's secid based on request_sock
* @req: request_sock
* @flic: target flow
*
* Sets @flic's secid to @req's secid.
*/
void security_req_classify_flow(const struct request_sock *req,
struct flowi_common *flic)
{
call_void_hook(req_classify_flow, req, flic);
}
EXPORT_SYMBOL(security_req_classify_flow);
/**
* security_sock_graft() - Reconcile LSM state when grafting a sock on a socket
* @sk: sock being grafted
* @parent: target parent socket
*
* Sets @parent's inode secid to @sk's secid and update @sk with any necessary
* LSM state from @parent.
*/
void security_sock_graft(struct sock *sk, struct socket *parent)
{
call_void_hook(sock_graft, sk, parent);
}
EXPORT_SYMBOL(security_sock_graft);
/**
* security_inet_conn_request() - Set request_sock state using incoming connect
* @sk: parent listening sock
* @skb: incoming connection
* @req: new request_sock
*
* Initialize the @req LSM state based on @sk and the incoming connect in @skb.
*
* Return: Returns 0 if permission is granted.
*/
int security_inet_conn_request(const struct sock *sk,
struct sk_buff *skb, struct request_sock *req)
{
return call_int_hook(inet_conn_request, sk, skb, req);
}
EXPORT_SYMBOL(security_inet_conn_request);
/**
* security_inet_csk_clone() - Set new sock LSM state based on request_sock
* @newsk: new sock
* @req: connection request_sock
*
* Set that LSM state of @sock using the LSM state from @req.
*/
void security_inet_csk_clone(struct sock *newsk,
const struct request_sock *req)
{
call_void_hook(inet_csk_clone, newsk, req);
}
/**
* security_inet_conn_established() - Update sock's LSM state with connection
* @sk: sock
* @skb: connection packet
*
* Update @sock's LSM state to represent a new connection from @skb.
*/
void security_inet_conn_established(struct sock *sk,
struct sk_buff *skb)
{
call_void_hook(inet_conn_established, sk, skb);
}
EXPORT_SYMBOL(security_inet_conn_established);
/**
* security_secmark_relabel_packet() - Check if setting a secmark is allowed
* @secid: new secmark value
*
* Check if the process should be allowed to relabel packets to @secid.
*
* Return: Returns 0 if permission is granted.
*/
int security_secmark_relabel_packet(u32 secid)
{
return call_int_hook(secmark_relabel_packet, secid);
}
EXPORT_SYMBOL(security_secmark_relabel_packet);
/**
* security_secmark_refcount_inc() - Increment the secmark labeling rule count
*
* Tells the LSM to increment the number of secmark labeling rules loaded.
*/
void security_secmark_refcount_inc(void)
{
call_void_hook(secmark_refcount_inc);
}
EXPORT_SYMBOL(security_secmark_refcount_inc);
/**
* security_secmark_refcount_dec() - Decrement the secmark labeling rule count
*
* Tells the LSM to decrement the number of secmark labeling rules loaded.
*/
void security_secmark_refcount_dec(void)
{
call_void_hook(secmark_refcount_dec);
}
EXPORT_SYMBOL(security_secmark_refcount_dec);
/**
* security_tun_dev_alloc_security() - Allocate a LSM blob for a TUN device
* @security: pointer to the LSM blob
*
* This hook allows a module to allocate a security structure for a TUN device,
* returning the pointer in @security.
*
* Return: Returns a zero on success, negative values on failure.
*/
int security_tun_dev_alloc_security(void **security)
{
return call_int_hook(tun_dev_alloc_security, security);
}
EXPORT_SYMBOL(security_tun_dev_alloc_security);
/**
* security_tun_dev_free_security() - Free a TUN device LSM blob
* @security: LSM blob
*
* This hook allows a module to free the security structure for a TUN device.
*/
void security_tun_dev_free_security(void *security)
{
call_void_hook(tun_dev_free_security, security);
}
EXPORT_SYMBOL(security_tun_dev_free_security);
/**
* security_tun_dev_create() - Check if creating a TUN device is allowed
*
* Check permissions prior to creating a new TUN device.
*
* Return: Returns 0 if permission is granted.
*/
int security_tun_dev_create(void)
{
return call_int_hook(tun_dev_create);
}
EXPORT_SYMBOL(security_tun_dev_create);
/**
* security_tun_dev_attach_queue() - Check if attaching a TUN queue is allowed
* @security: TUN device LSM blob
*
* Check permissions prior to attaching to a TUN device queue.
*
* Return: Returns 0 if permission is granted.
*/
int security_tun_dev_attach_queue(void *security)
{
return call_int_hook(tun_dev_attach_queue, security);
}
EXPORT_SYMBOL(security_tun_dev_attach_queue);
/**
* security_tun_dev_attach() - Update TUN device LSM state on attach
* @sk: associated sock
* @security: TUN device LSM blob
*
* This hook can be used by the module to update any security state associated
* with the TUN device's sock structure.
*
* Return: Returns 0 if permission is granted.
*/
int security_tun_dev_attach(struct sock *sk, void *security)
{
return call_int_hook(tun_dev_attach, sk, security);
}
EXPORT_SYMBOL(security_tun_dev_attach);
/**
* security_tun_dev_open() - Update TUN device LSM state on open
* @security: TUN device LSM blob
*
* This hook can be used by the module to update any security state associated
* with the TUN device's security structure.
*
* Return: Returns 0 if permission is granted.
*/
int security_tun_dev_open(void *security)
{
return call_int_hook(tun_dev_open, security);
}
EXPORT_SYMBOL(security_tun_dev_open);
/**
* security_sctp_assoc_request() - Update the LSM on a SCTP association req
* @asoc: SCTP association
* @skb: packet requesting the association
*
* Passes the @asoc and @chunk->skb of the association INIT packet to the LSM.
*
* Return: Returns 0 on success, error on failure.
*/
int security_sctp_assoc_request(struct sctp_association *asoc,
struct sk_buff *skb)
{
return call_int_hook(sctp_assoc_request, asoc, skb);
}
EXPORT_SYMBOL(security_sctp_assoc_request);
/**
* security_sctp_bind_connect() - Validate a list of addrs for a SCTP option
* @sk: socket
* @optname: SCTP option to validate
* @address: list of IP addresses to validate
* @addrlen: length of the address list
*
* Validiate permissions required for each address associated with sock @sk.
* Depending on @optname, the addresses will be treated as either a connect or
* bind service. The @addrlen is calculated on each IPv4 and IPv6 address using
* sizeof(struct sockaddr_in) or sizeof(struct sockaddr_in6).
*
* Return: Returns 0 on success, error on failure.
*/
int security_sctp_bind_connect(struct sock *sk, int optname,
struct sockaddr *address, int addrlen)
{
return call_int_hook(sctp_bind_connect, sk, optname, address, addrlen);
}
EXPORT_SYMBOL(security_sctp_bind_connect);
/**
* security_sctp_sk_clone() - Clone a SCTP sock's LSM state
* @asoc: SCTP association
* @sk: original sock
* @newsk: target sock
*
* Called whenever a new socket is created by accept(2) (i.e. a TCP style
* socket) or when a socket is 'peeled off' e.g userspace calls
* sctp_peeloff(3).
*/
void security_sctp_sk_clone(struct sctp_association *asoc, struct sock *sk,
struct sock *newsk)
{
call_void_hook(sctp_sk_clone, asoc, sk, newsk);
}
EXPORT_SYMBOL(security_sctp_sk_clone);
/**
* security_sctp_assoc_established() - Update LSM state when assoc established
* @asoc: SCTP association
* @skb: packet establishing the association
*
* Passes the @asoc and @chunk->skb of the association COOKIE_ACK packet to the
* security module.
*
* Return: Returns 0 if permission is granted.
*/
int security_sctp_assoc_established(struct sctp_association *asoc,
struct sk_buff *skb)
{
return call_int_hook(sctp_assoc_established, asoc, skb);
}
EXPORT_SYMBOL(security_sctp_assoc_established);
/**
* security_mptcp_add_subflow() - Inherit the LSM label from the MPTCP socket
* @sk: the owning MPTCP socket
* @ssk: the new subflow
*
* Update the labeling for the given MPTCP subflow, to match the one of the
* owning MPTCP socket. This hook has to be called after the socket creation and
* initialization via the security_socket_create() and
* security_socket_post_create() LSM hooks.
*
* Return: Returns 0 on success or a negative error code on failure.
*/
int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk)
{
return call_int_hook(mptcp_add_subflow, sk, ssk);
}
#endif /* CONFIG_SECURITY_NETWORK */
#ifdef CONFIG_SECURITY_INFINIBAND
/**
* security_ib_pkey_access() - Check if access to an IB pkey is allowed
* @sec: LSM blob
* @subnet_prefix: subnet prefix of the port
* @pkey: IB pkey
*
* Check permission to access a pkey when modifying a QP.
*
* Return: Returns 0 if permission is granted.
*/
int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey)
{
return call_int_hook(ib_pkey_access, sec, subnet_prefix, pkey);
}
EXPORT_SYMBOL(security_ib_pkey_access);
/**
* security_ib_endport_manage_subnet() - Check if SMPs traffic is allowed
* @sec: LSM blob
* @dev_name: IB device name
* @port_num: port number
*
* Check permissions to send and receive SMPs on a end port.
*
* Return: Returns 0 if permission is granted.
*/
int security_ib_endport_manage_subnet(void *sec,
const char *dev_name, u8 port_num)
{
return call_int_hook(ib_endport_manage_subnet, sec, dev_name, port_num);
}
EXPORT_SYMBOL(security_ib_endport_manage_subnet);
/**
* security_ib_alloc_security() - Allocate an Infiniband LSM blob
* @sec: LSM blob
*
* Allocate a security structure for Infiniband objects.
*
* Return: Returns 0 on success, non-zero on failure.
*/
int security_ib_alloc_security(void **sec)
{
return call_int_hook(ib_alloc_security, sec);
}
EXPORT_SYMBOL(security_ib_alloc_security);
/**
* security_ib_free_security() - Free an Infiniband LSM blob
* @sec: LSM blob
*
* Deallocate an Infiniband security structure.
*/
void security_ib_free_security(void *sec)
{
call_void_hook(ib_free_security, sec);
}
EXPORT_SYMBOL(security_ib_free_security);
#endif /* CONFIG_SECURITY_INFINIBAND */
#ifdef CONFIG_SECURITY_NETWORK_XFRM
/**
* security_xfrm_policy_alloc() - Allocate a xfrm policy LSM blob
* @ctxp: xfrm security context being added to the SPD
* @sec_ctx: security label provided by userspace
* @gfp: gfp flags
*
* Allocate a security structure to the xp->security field; the security field
* is initialized to NULL when the xfrm_policy is allocated.
*
* Return: Return 0 if operation was successful.
*/
int security_xfrm_policy_alloc(struct xfrm_sec_ctx **ctxp,
struct xfrm_user_sec_ctx *sec_ctx,
gfp_t gfp)
{
return call_int_hook(xfrm_policy_alloc_security, ctxp, sec_ctx, gfp);
}
EXPORT_SYMBOL(security_xfrm_policy_alloc);
/**
* security_xfrm_policy_clone() - Clone xfrm policy LSM state
* @old_ctx: xfrm security context
* @new_ctxp: target xfrm security context
*
* Allocate a security structure in new_ctxp that contains the information from
* the old_ctx structure.
*
* Return: Return 0 if operation was successful.
*/
int security_xfrm_policy_clone(struct xfrm_sec_ctx *old_ctx,
struct xfrm_sec_ctx **new_ctxp)
{
return call_int_hook(xfrm_policy_clone_security, old_ctx, new_ctxp);
}
/**
* security_xfrm_policy_free() - Free a xfrm security context
* @ctx: xfrm security context
*
* Free LSM resources associated with @ctx.
*/
void security_xfrm_policy_free(struct xfrm_sec_ctx *ctx)
{
call_void_hook(xfrm_policy_free_security, ctx);
}
EXPORT_SYMBOL(security_xfrm_policy_free);
/**
* security_xfrm_policy_delete() - Check if deleting a xfrm policy is allowed
* @ctx: xfrm security context
*
* Authorize deletion of a SPD entry.
*
* Return: Returns 0 if permission is granted.
*/
int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx)
{
return call_int_hook(xfrm_policy_delete_security, ctx);
}
/**
* security_xfrm_state_alloc() - Allocate a xfrm state LSM blob
* @x: xfrm state being added to the SAD
* @sec_ctx: security label provided by userspace
*
* Allocate a security structure to the @x->security field; the security field
* is initialized to NULL when the xfrm_state is allocated. Set the context to
* correspond to @sec_ctx.
*
* Return: Return 0 if operation was successful.
*/
int security_xfrm_state_alloc(struct xfrm_state *x,
struct xfrm_user_sec_ctx *sec_ctx)
{
return call_int_hook(xfrm_state_alloc, x, sec_ctx);
}
EXPORT_SYMBOL(security_xfrm_state_alloc);
/**
* security_xfrm_state_alloc_acquire() - Allocate a xfrm state LSM blob
* @x: xfrm state being added to the SAD
* @polsec: associated policy's security context
* @secid: secid from the flow
*
* Allocate a security structure to the x->security field; the security field
* is initialized to NULL when the xfrm_state is allocated. Set the context to
* correspond to secid.
*
* Return: Returns 0 if operation was successful.
*/
int security_xfrm_state_alloc_acquire(struct xfrm_state *x,
struct xfrm_sec_ctx *polsec, u32 secid)
{
return call_int_hook(xfrm_state_alloc_acquire, x, polsec, secid);
}
/**
* security_xfrm_state_delete() - Check if deleting a xfrm state is allowed
* @x: xfrm state
*
* Authorize deletion of x->security.
*
* Return: Returns 0 if permission is granted.
*/
int security_xfrm_state_delete(struct xfrm_state *x)
{
return call_int_hook(xfrm_state_delete_security, x);
}
EXPORT_SYMBOL(security_xfrm_state_delete);
/**
* security_xfrm_state_free() - Free a xfrm state
* @x: xfrm state
*
* Deallocate x->security.
*/
void security_xfrm_state_free(struct xfrm_state *x)
{
call_void_hook(xfrm_state_free_security, x);
}
/**
* security_xfrm_policy_lookup() - Check if using a xfrm policy is allowed
* @ctx: target xfrm security context
* @fl_secid: flow secid used to authorize access
*
* Check permission when a flow selects a xfrm_policy for processing XFRMs on a
* packet. The hook is called when selecting either a per-socket policy or a
* generic xfrm policy.
*
* Return: Return 0 if permission is granted, -ESRCH otherwise, or -errno on
* other errors.
*/
int security_xfrm_policy_lookup(struct xfrm_sec_ctx *ctx, u32 fl_secid)
{
return call_int_hook(xfrm_policy_lookup, ctx, fl_secid);
}
/**
* security_xfrm_state_pol_flow_match() - Check for a xfrm match
* @x: xfrm state to match
* @xp: xfrm policy to check for a match
* @flic: flow to check for a match.
*
* Check @xp and @flic for a match with @x.
*
* Return: Returns 1 if there is a match.
*/
int security_xfrm_state_pol_flow_match(struct xfrm_state *x,
struct xfrm_policy *xp,
const struct flowi_common *flic)
{
struct security_hook_list *hp;
int rc = LSM_RET_DEFAULT(xfrm_state_pol_flow_match);
/*
* Since this function is expected to return 0 or 1, the judgment
* becomes difficult if multiple LSMs supply this call. Fortunately,
* we can use the first LSM's judgment because currently only SELinux
* supplies this call.
*
* For speed optimization, we explicitly break the loop rather than
* using the macro
*/
hlist_for_each_entry(hp, &security_hook_heads.xfrm_state_pol_flow_match,
list) {
rc = hp->hook.xfrm_state_pol_flow_match(x, xp, flic);
break;
}
return rc;
}
/**
* security_xfrm_decode_session() - Determine the xfrm secid for a packet
* @skb: xfrm packet
* @secid: secid
*
* Decode the packet in @skb and return the security label in @secid.
*
* Return: Return 0 if all xfrms used have the same secid.
*/
int security_xfrm_decode_session(struct sk_buff *skb, u32 *secid)
{
return call_int_hook(xfrm_decode_session, skb, secid, 1);
}
void security_skb_classify_flow(struct sk_buff *skb, struct flowi_common *flic)
{
int rc = call_int_hook(xfrm_decode_session, skb, &flic->flowic_secid,
0);
BUG_ON(rc);
}
EXPORT_SYMBOL(security_skb_classify_flow);
#endif /* CONFIG_SECURITY_NETWORK_XFRM */
#ifdef CONFIG_KEYS
/**
* security_key_alloc() - Allocate and initialize a kernel key LSM blob
* @key: key
* @cred: credentials
* @flags: allocation flags
*
* Permit allocation of a key and assign security data. Note that key does not
* have a serial number assigned at this point.
*
* Return: Return 0 if permission is granted, -ve error otherwise.
*/
int security_key_alloc(struct key *key, const struct cred *cred,
unsigned long flags)
{
return call_int_hook(key_alloc, key, cred, flags);
}
/**
* security_key_free() - Free a kernel key LSM blob
* @key: key
*
* Notification of destruction; free security data.
*/
void security_key_free(struct key *key)
{
call_void_hook(key_free, key);
}
/**
* security_key_permission() - Check if a kernel key operation is allowed
* @key_ref: key reference
* @cred: credentials of actor requesting access
* @need_perm: requested permissions
*
* See whether a specific operational right is granted to a process on a key.
*
* Return: Return 0 if permission is granted, -ve error otherwise.
*/
int security_key_permission(key_ref_t key_ref, const struct cred *cred,
enum key_need_perm need_perm)
{
return call_int_hook(key_permission, key_ref, cred, need_perm);
}
/**
* security_key_getsecurity() - Get the key's security label
* @key: key
* @buffer: security label buffer
*
* Get a textual representation of the security context attached to a key for
* the purposes of honouring KEYCTL_GETSECURITY. This function allocates the
* storage for the NUL-terminated string and the caller should free it.
*
* Return: Returns the length of @buffer (including terminating NUL) or -ve if
* an error occurs. May also return 0 (and a NULL buffer pointer) if
* there is no security label assigned to the key.
*/
int security_key_getsecurity(struct key *key, char **buffer)
{
*buffer = NULL;
return call_int_hook(key_getsecurity, key, buffer);
}
/**
* security_key_post_create_or_update() - Notification of key create or update
* @keyring: keyring to which the key is linked to
* @key: created or updated key
* @payload: data used to instantiate or update the key
* @payload_len: length of payload
* @flags: key flags
* @create: flag indicating whether the key was created or updated
*
* Notify the caller of a key creation or update.
*/
void security_key_post_create_or_update(struct key *keyring, struct key *key,
const void *payload, size_t payload_len,
unsigned long flags, bool create)
{
call_void_hook(key_post_create_or_update, keyring, key, payload,
payload_len, flags, create);
}
#endif /* CONFIG_KEYS */
#ifdef CONFIG_AUDIT
/**
* security_audit_rule_init() - Allocate and init an LSM audit rule struct
* @field: audit action
* @op: rule operator
* @rulestr: rule context
* @lsmrule: receive buffer for audit rule struct
*
* Allocate and initialize an LSM audit rule structure.
*
* Return: Return 0 if @lsmrule has been successfully set, -EINVAL in case of
* an invalid rule.
*/
int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule)
{
return call_int_hook(audit_rule_init, field, op, rulestr, lsmrule);
}
/**
* security_audit_rule_known() - Check if an audit rule contains LSM fields
* @krule: audit rule
*
* Specifies whether given @krule contains any fields related to the current
* LSM.
*
* Return: Returns 1 in case of relation found, 0 otherwise.
*/
int security_audit_rule_known(struct audit_krule *krule)
{
return call_int_hook(audit_rule_known, krule);
}
/**
* security_audit_rule_free() - Free an LSM audit rule struct
* @lsmrule: audit rule struct
*
* Deallocate the LSM audit rule structure previously allocated by
* audit_rule_init().
*/
void security_audit_rule_free(void *lsmrule)
{
call_void_hook(audit_rule_free, lsmrule);
}
/**
* security_audit_rule_match() - Check if a label matches an audit rule
* @secid: security label
* @field: LSM audit field
* @op: matching operator
* @lsmrule: audit rule
*
* Determine if given @secid matches a rule previously approved by
* security_audit_rule_known().
*
* Return: Returns 1 if secid matches the rule, 0 if it does not, -ERRNO on
* failure.
*/
int security_audit_rule_match(u32 secid, u32 field, u32 op, void *lsmrule)
{
return call_int_hook(audit_rule_match, secid, field, op, lsmrule);
}
#endif /* CONFIG_AUDIT */
#ifdef CONFIG_BPF_SYSCALL
/**
* security_bpf() - Check if the bpf syscall operation is allowed
* @cmd: command
* @attr: bpf attribute
* @size: size
*
* Do a initial check for all bpf syscalls after the attribute is copied into
* the kernel. The actual security module can implement their own rules to
* check the specific cmd they need.
*
* Return: Returns 0 if permission is granted.
*/
int security_bpf(int cmd, union bpf_attr *attr, unsigned int size)
{
return call_int_hook(bpf, cmd, attr, size);
}
/**
* security_bpf_map() - Check if access to a bpf map is allowed
* @map: bpf map
* @fmode: mode
*
* Do a check when the kernel generates and returns a file descriptor for eBPF
* maps.
*
* Return: Returns 0 if permission is granted.
*/
int security_bpf_map(struct bpf_map *map, fmode_t fmode)
{
return call_int_hook(bpf_map, map, fmode);
}
/**
* security_bpf_prog() - Check if access to a bpf program is allowed
* @prog: bpf program
*
* Do a check when the kernel generates and returns a file descriptor for eBPF
* programs.
*
* Return: Returns 0 if permission is granted.
*/
int security_bpf_prog(struct bpf_prog *prog)
{
return call_int_hook(bpf_prog, prog);
}
/**
* security_bpf_map_create() - Check if BPF map creation is allowed
* @map: BPF map object
* @attr: BPF syscall attributes used to create BPF map
* @token: BPF token used to grant user access
*
* Do a check when the kernel creates a new BPF map. This is also the
* point where LSM blob is allocated for LSMs that need them.
*
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
struct bpf_token *token)
{
return call_int_hook(bpf_map_create, map, attr, token);
}
/**
* security_bpf_prog_load() - Check if loading of BPF program is allowed
* @prog: BPF program object
* @attr: BPF syscall attributes used to create BPF program
* @token: BPF token used to grant user access to BPF subsystem
*
* Perform an access control check when the kernel loads a BPF program and
* allocates associated BPF program object. This hook is also responsible for
* allocating any required LSM state for the BPF program.
*
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
struct bpf_token *token)
{
return call_int_hook(bpf_prog_load, prog, attr, token);
}
/**
* security_bpf_token_create() - Check if creating of BPF token is allowed
* @token: BPF token object
* @attr: BPF syscall attributes used to create BPF token
* @path: path pointing to BPF FS mount point from which BPF token is created
*
* Do a check when the kernel instantiates a new BPF token object from BPF FS
* instance. This is also the point where LSM blob can be allocated for LSMs.
*
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr,
struct path *path)
{
return call_int_hook(bpf_token_create, token, attr, path);
}
/**
* security_bpf_token_cmd() - Check if BPF token is allowed to delegate
* requested BPF syscall command
* @token: BPF token object
* @cmd: BPF syscall command requested to be delegated by BPF token
*
* Do a check when the kernel decides whether provided BPF token should allow
* delegation of requested BPF syscall command.
*
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd)
{
return call_int_hook(bpf_token_cmd, token, cmd);
}
/**
* security_bpf_token_capable() - Check if BPF token is allowed to delegate
* requested BPF-related capability
* @token: BPF token object
* @cap: capabilities requested to be delegated by BPF token
*
* Do a check when the kernel decides whether provided BPF token should allow
* delegation of requested BPF-related capabilities.
*
* Return: Returns 0 on success, error on failure.
*/
int security_bpf_token_capable(const struct bpf_token *token, int cap)
{
return call_int_hook(bpf_token_capable, token, cap);
}
/**
* security_bpf_map_free() - Free a bpf map's LSM blob
* @map: bpf map
*
* Clean up the security information stored inside bpf map.
*/
void security_bpf_map_free(struct bpf_map *map)
{
call_void_hook(bpf_map_free, map);
}
/**
* security_bpf_prog_free() - Free a BPF program's LSM blob
* @prog: BPF program struct
*
* Clean up the security information stored inside BPF program.
*/
void security_bpf_prog_free(struct bpf_prog *prog)
{
call_void_hook(bpf_prog_free, prog);
}
/**
* security_bpf_token_free() - Free a BPF token's LSM blob
* @token: BPF token struct
*
* Clean up the security information stored inside BPF token.
*/
void security_bpf_token_free(struct bpf_token *token)
{
call_void_hook(bpf_token_free, token);
}
#endif /* CONFIG_BPF_SYSCALL */
/**
* security_locked_down() - Check if a kernel feature is allowed
* @what: requested kernel feature
*
* Determine whether a kernel feature that potentially enables arbitrary code
* execution in kernel space should be permitted.
*
* Return: Returns 0 if permission is granted.
*/
int security_locked_down(enum lockdown_reason what)
{
return call_int_hook(locked_down, what);
}
EXPORT_SYMBOL(security_locked_down);
#ifdef CONFIG_PERF_EVENTS
/**
* security_perf_event_open() - Check if a perf event open is allowed
* @attr: perf event attribute
* @type: type of event
*
* Check whether the @type of perf_event_open syscall is allowed.
*
* Return: Returns 0 if permission is granted.
*/
int security_perf_event_open(struct perf_event_attr *attr, int type)
{
return call_int_hook(perf_event_open, attr, type);
}
/**
* security_perf_event_alloc() - Allocate a perf event LSM blob
* @event: perf event
*
* Allocate and save perf_event security info.
*
* Return: Returns 0 on success, error on failure.
*/
int security_perf_event_alloc(struct perf_event *event)
{
return call_int_hook(perf_event_alloc, event);
}
/**
* security_perf_event_free() - Free a perf event LSM blob
* @event: perf event
*
* Release (free) perf_event security info.
*/
void security_perf_event_free(struct perf_event *event)
{
call_void_hook(perf_event_free, event);
}
/**
* security_perf_event_read() - Check if reading a perf event label is allowed
* @event: perf event
*
* Read perf_event security info if allowed.
*
* Return: Returns 0 if permission is granted.
*/
int security_perf_event_read(struct perf_event *event)
{
return call_int_hook(perf_event_read, event);
}
/**
* security_perf_event_write() - Check if writing a perf event label is allowed
* @event: perf event
*
* Write perf_event security info if allowed.
*
* Return: Returns 0 if permission is granted.
*/
int security_perf_event_write(struct perf_event *event)
{
return call_int_hook(perf_event_write, event);
}
#endif /* CONFIG_PERF_EVENTS */
#ifdef CONFIG_IO_URING
/**
* security_uring_override_creds() - Check if overriding creds is allowed
* @new: new credentials
*
* Check if the current task, executing an io_uring operation, is allowed to
* override it's credentials with @new.
*
* Return: Returns 0 if permission is granted.
*/
int security_uring_override_creds(const struct cred *new)
{
return call_int_hook(uring_override_creds, new);
}
/**
* security_uring_sqpoll() - Check if IORING_SETUP_SQPOLL is allowed
*
* Check whether the current task is allowed to spawn a io_uring polling thread
* (IORING_SETUP_SQPOLL).
*
* Return: Returns 0 if permission is granted.
*/
int security_uring_sqpoll(void)
{
return call_int_hook(uring_sqpoll);
}
/**
* security_uring_cmd() - Check if a io_uring passthrough command is allowed
* @ioucmd: command
*
* Check whether the file_operations uring_cmd is allowed to run.
*
* Return: Returns 0 if permission is granted.
*/
int security_uring_cmd(struct io_uring_cmd *ioucmd)
{
return call_int_hook(uring_cmd, ioucmd);
}
#endif /* CONFIG_IO_URING */
// SPDX-License-Identifier: GPL-2.0
#include <linux/memblock.h>
#include <linux/mmdebug.h>
#include <linux/export.h>
#include <linux/mm.h>
#include <asm/page.h>
#include <linux/vmalloc.h>
#include "physaddr.h"
#ifdef CONFIG_X86_64
#ifdef CONFIG_DEBUG_VIRTUAL
unsigned long __phys_addr(unsigned long x)
{
unsigned long y = x - __START_KERNEL_map;
/* use the carry flag to determine if x was < __START_KERNEL_map */
if (unlikely(x > y)) {
x = y + phys_base; VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE);
} else {
x = y + (__START_KERNEL_map - PAGE_OFFSET);
/* carry flag will be set if starting x was >= PAGE_OFFSET */
VIRTUAL_BUG_ON((x > y) || !phys_addr_valid(x));
}
return x;
}
EXPORT_SYMBOL(__phys_addr);
unsigned long __phys_addr_symbol(unsigned long x)
{
unsigned long y = x - __START_KERNEL_map;
/* only check upper bounds since lower bounds will trigger carry */
VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE); return y + phys_base;
}
EXPORT_SYMBOL(__phys_addr_symbol);
#endif
bool __virt_addr_valid(unsigned long x)
{
unsigned long y = x - __START_KERNEL_map;
/* use the carry flag to determine if x was < __START_KERNEL_map */
if (unlikely(x > y)) {
x = y + phys_base;
if (y >= KERNEL_IMAGE_SIZE)
return false;
} else {
x = y + (__START_KERNEL_map - PAGE_OFFSET);
/* carry flag will be set if starting x was >= PAGE_OFFSET */
if ((x > y) || !phys_addr_valid(x))
return false;
}
return pfn_valid(x >> PAGE_SHIFT);
}
EXPORT_SYMBOL(__virt_addr_valid);
#else
#ifdef CONFIG_DEBUG_VIRTUAL
unsigned long __phys_addr(unsigned long x)
{
unsigned long phys_addr = x - PAGE_OFFSET;
/* VMALLOC_* aren't constants */
VIRTUAL_BUG_ON(x < PAGE_OFFSET);
VIRTUAL_BUG_ON(__vmalloc_start_set && is_vmalloc_addr((void *) x));
/* max_low_pfn is set early, but not _that_ early */
if (max_low_pfn) {
VIRTUAL_BUG_ON((phys_addr >> PAGE_SHIFT) > max_low_pfn);
BUG_ON(slow_virt_to_phys((void *)x) != phys_addr);
}
return phys_addr;
}
EXPORT_SYMBOL(__phys_addr);
#endif
bool __virt_addr_valid(unsigned long x)
{
if (x < PAGE_OFFSET)
return false;
if (__vmalloc_start_set && is_vmalloc_addr((void *) x))
return false;
if (x >= FIXADDR_START)
return false;
return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT);
}
EXPORT_SYMBOL(__virt_addr_valid);
#endif /* CONFIG_X86_64 */
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/syscalls.h>
#include <linux/export.h>
#include <linux/uaccess.h>
#include <linux/fs_struct.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/prefetch.h>
#include "mount.h"
#include "internal.h"
struct prepend_buffer {
char *buf;
int len;
};
#define DECLARE_BUFFER(__name, __buf, __len) \
struct prepend_buffer __name = {.buf = __buf + __len, .len = __len}
static char *extract_string(struct prepend_buffer *p)
{
if (likely(p->len >= 0)) return p->buf;
return ERR_PTR(-ENAMETOOLONG);
}
static bool prepend_char(struct prepend_buffer *p, unsigned char c)
{
if (likely(p->len > 0)) { p->len--;
*--p->buf = c;
return true;
}
p->len = -1;
return false;
}
/*
* The source of the prepend data can be an optimistic load
* of a dentry name and length. And because we don't hold any
* locks, the length and the pointer to the name may not be
* in sync if a concurrent rename happens, and the kernel
* copy might fault as a result.
*
* The end result will correct itself when we check the
* rename sequence count, but we need to be able to handle
* the fault gracefully.
*/
static bool prepend_copy(void *dst, const void *src, int len)
{
if (unlikely(copy_from_kernel_nofault(dst, src, len))) { memset(dst, 'x', len); return false;
}
return true;
}
static bool prepend(struct prepend_buffer *p, const char *str, int namelen)
{
// Already overflowed?
if (p->len < 0)
return false;
// Will overflow?
if (p->len < namelen) {
// Fill as much as possible from the end of the name
str += namelen - p->len; p->buf -= p->len;
prepend_copy(p->buf, str, p->len);
p->len = -1;
return false;
}
// Fits fully
p->len -= namelen;
p->buf -= namelen;
return prepend_copy(p->buf, str, namelen);
}
/**
* prepend_name - prepend a pathname in front of current buffer pointer
* @p: prepend buffer which contains buffer pointer and allocated length
* @name: name string and length qstr structure
*
* With RCU path tracing, it may race with d_move(). Use READ_ONCE() to
* make sure that either the old or the new name pointer and length are
* fetched. However, there may be mismatch between length and pointer.
* But since the length cannot be trusted, we need to copy the name very
* carefully when doing the prepend_copy(). It also prepends "/" at
* the beginning of the name. The sequence number check at the caller will
* retry it again when a d_move() does happen. So any garbage in the buffer
* due to mismatched pointer and length will be discarded.
*
* Load acquire is needed to make sure that we see the new name data even
* if we might get the length wrong.
*/
static bool prepend_name(struct prepend_buffer *p, const struct qstr *name)
{
const char *dname = smp_load_acquire(&name->name); /* ^^^ */
u32 dlen = READ_ONCE(name->len);
return prepend(p, dname, dlen) && prepend_char(p, '/');
}
static int __prepend_path(const struct dentry *dentry, const struct mount *mnt,
const struct path *root, struct prepend_buffer *p)
{
while (dentry != root->dentry || &mnt->mnt != root->mnt) { const struct dentry *parent = READ_ONCE(dentry->d_parent);
if (dentry == mnt->mnt.mnt_root) {
struct mount *m = READ_ONCE(mnt->mnt_parent);
struct mnt_namespace *mnt_ns;
if (likely(mnt != m)) {
dentry = READ_ONCE(mnt->mnt_mountpoint);
mnt = m;
continue;
}
/* Global root */
mnt_ns = READ_ONCE(mnt->mnt_ns);
/* open-coded is_mounted() to use local mnt_ns */
if (!IS_ERR_OR_NULL(mnt_ns) && !is_anon_ns(mnt_ns))
return 1; // absolute root
else
return 2; // detached or not attached yet
}
if (unlikely(dentry == parent))
/* Escaped? */
return 3;
prefetch(parent); if (!prepend_name(p, &dentry->d_name))
break;
dentry = parent;
}
return 0;
}
/**
* prepend_path - Prepend path string to a buffer
* @path: the dentry/vfsmount to report
* @root: root vfsmnt/dentry
* @p: prepend buffer which contains buffer pointer and allocated length
*
* The function will first try to write out the pathname without taking any
* lock other than the RCU read lock to make sure that dentries won't go away.
* It only checks the sequence number of the global rename_lock as any change
* in the dentry's d_seq will be preceded by changes in the rename_lock
* sequence number. If the sequence number had been changed, it will restart
* the whole pathname back-tracing sequence again by taking the rename_lock.
* In this case, there is no need to take the RCU read lock as the recursive
* parent pointer references will keep the dentry chain alive as long as no
* rename operation is performed.
*/
static int prepend_path(const struct path *path,
const struct path *root,
struct prepend_buffer *p)
{
unsigned seq, m_seq = 0;
struct prepend_buffer b;
int error;
rcu_read_lock();
restart_mnt:
read_seqbegin_or_lock(&mount_lock, &m_seq);
seq = 0;
rcu_read_lock();
restart:
b = *p; read_seqbegin_or_lock(&rename_lock, &seq); error = __prepend_path(path->dentry, real_mount(path->mnt), root, &b); if (!(seq & 1)) rcu_read_unlock();
if (need_seqretry(&rename_lock, seq)) {
seq = 1;
goto restart;
}
done_seqretry(&rename_lock, seq); if (!(m_seq & 1)) rcu_read_unlock();
if (need_seqretry(&mount_lock, m_seq)) {
m_seq = 1;
goto restart_mnt;
}
done_seqretry(&mount_lock, m_seq); if (unlikely(error == 3)) b = *p; if (b.len == p->len) prepend_char(&b, '/'); *p = b;
return error;
}
/**
* __d_path - return the path of a dentry
* @path: the dentry/vfsmount to report
* @root: root vfsmnt/dentry
* @buf: buffer to return value in
* @buflen: buffer length
*
* Convert a dentry into an ASCII path name.
*
* Returns a pointer into the buffer or an error code if the
* path was too long.
*
* "buflen" should be positive.
*
* If the path is not reachable from the supplied root, return %NULL.
*/
char *__d_path(const struct path *path,
const struct path *root,
char *buf, int buflen)
{
DECLARE_BUFFER(b, buf, buflen);
prepend_char(&b, 0);
if (unlikely(prepend_path(path, root, &b) > 0))
return NULL;
return extract_string(&b);
}
char *d_absolute_path(const struct path *path,
char *buf, int buflen)
{
struct path root = {};
DECLARE_BUFFER(b, buf, buflen);
prepend_char(&b, 0);
if (unlikely(prepend_path(path, &root, &b) > 1))
return ERR_PTR(-EINVAL);
return extract_string(&b);
}
static void get_fs_root_rcu(struct fs_struct *fs, struct path *root)
{
unsigned seq;
do {
seq = read_seqcount_begin(&fs->seq);
*root = fs->root;
} while (read_seqcount_retry(&fs->seq, seq));
}
/**
* d_path - return the path of a dentry
* @path: path to report
* @buf: buffer to return value in
* @buflen: buffer length
*
* Convert a dentry into an ASCII path name. If the entry has been deleted
* the string " (deleted)" is appended. Note that this is ambiguous.
*
* Returns a pointer into the buffer or an error code if the path was
* too long. Note: Callers should use the returned pointer, not the passed
* in buffer, to use the name! The implementation often starts at an offset
* into the buffer, and may leave 0 bytes at the start.
*
* "buflen" should be positive.
*/
char *d_path(const struct path *path, char *buf, int buflen)
{
DECLARE_BUFFER(b, buf, buflen);
struct path root;
/*
* We have various synthetic filesystems that never get mounted. On
* these filesystems dentries are never used for lookup purposes, and
* thus don't need to be hashed. They also don't need a name until a
* user wants to identify the object in /proc/pid/fd/. The little hack
* below allows us to generate a name for these objects on demand:
*
* Some pseudo inodes are mountable. When they are mounted
* path->dentry == path->mnt->mnt_root. In that case don't call d_dname
* and instead have d_path return the mounted path.
*/
if (path->dentry->d_op && path->dentry->d_op->d_dname &&
(!IS_ROOT(path->dentry) || path->dentry != path->mnt->mnt_root))
return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
rcu_read_lock();
get_fs_root_rcu(current->fs, &root);
if (unlikely(d_unlinked(path->dentry)))
prepend(&b, " (deleted)", 11);
else
prepend_char(&b, 0);
prepend_path(path, &root, &b);
rcu_read_unlock();
return extract_string(&b);
}
EXPORT_SYMBOL(d_path);
/*
* Helper function for dentry_operations.d_dname() members
*/
char *dynamic_dname(char *buffer, int buflen, const char *fmt, ...)
{
va_list args;
char temp[64];
int sz;
va_start(args, fmt);
sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
va_end(args);
if (sz > sizeof(temp) || sz > buflen)
return ERR_PTR(-ENAMETOOLONG);
buffer += buflen - sz;
return memcpy(buffer, temp, sz);
}
char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
{
DECLARE_BUFFER(b, buffer, buflen);
/* these dentries are never renamed, so d_lock is not needed */
prepend(&b, " (deleted)", 11);
prepend(&b, dentry->d_name.name, dentry->d_name.len);
prepend_char(&b, '/');
return extract_string(&b);
}
/*
* Write full pathname from the root of the filesystem into the buffer.
*/
static char *__dentry_path(const struct dentry *d, struct prepend_buffer *p)
{
const struct dentry *dentry;
struct prepend_buffer b;
int seq = 0;
rcu_read_lock();
restart:
dentry = d;
b = *p;
read_seqbegin_or_lock(&rename_lock, &seq);
while (!IS_ROOT(dentry)) {
const struct dentry *parent = dentry->d_parent;
prefetch(parent);
if (!prepend_name(&b, &dentry->d_name))
break;
dentry = parent;
}
if (!(seq & 1))
rcu_read_unlock();
if (need_seqretry(&rename_lock, seq)) {
seq = 1;
goto restart;
}
done_seqretry(&rename_lock, seq);
if (b.len == p->len)
prepend_char(&b, '/');
return extract_string(&b);
}
char *dentry_path_raw(const struct dentry *dentry, char *buf, int buflen)
{
DECLARE_BUFFER(b, buf, buflen);
prepend_char(&b, 0);
return __dentry_path(dentry, &b);
}
EXPORT_SYMBOL(dentry_path_raw);
char *dentry_path(const struct dentry *dentry, char *buf, int buflen)
{
DECLARE_BUFFER(b, buf, buflen);
if (unlikely(d_unlinked(dentry)))
prepend(&b, "//deleted", 10);
else
prepend_char(&b, 0);
return __dentry_path(dentry, &b);
}
static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
struct path *pwd)
{
unsigned seq;
do {
seq = read_seqcount_begin(&fs->seq);
*root = fs->root;
*pwd = fs->pwd;
} while (read_seqcount_retry(&fs->seq, seq));
}
/*
* NOTE! The user-level library version returns a
* character pointer. The kernel system call just
* returns the length of the buffer filled (which
* includes the ending '\0' character), or a negative
* error value. So libc would do something like
*
* char *getcwd(char * buf, size_t size)
* {
* int retval;
*
* retval = sys_getcwd(buf, size);
* if (retval >= 0)
* return buf;
* errno = -retval;
* return NULL;
* }
*/
SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
{
int error;
struct path pwd, root;
char *page = __getname();
if (!page)
return -ENOMEM;
rcu_read_lock();
get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
if (unlikely(d_unlinked(pwd.dentry))) {
rcu_read_unlock();
error = -ENOENT;
} else {
unsigned len;
DECLARE_BUFFER(b, page, PATH_MAX);
prepend_char(&b, 0);
if (unlikely(prepend_path(&pwd, &root, &b) > 0))
prepend(&b, "(unreachable)", 13);
rcu_read_unlock();
len = PATH_MAX - b.len;
if (unlikely(len > PATH_MAX))
error = -ENAMETOOLONG;
else if (unlikely(len > size))
error = -ERANGE;
else if (copy_to_user(buf, b.buf, len))
error = -EFAULT;
else
error = len;
}
__putname(page);
return error;
}
// SPDX-License-Identifier: GPL-2.0+
/*
* A wrapper for multiple PHYs which passes all phy_* function calls to
* multiple (actual) PHY devices. This is comes handy when initializing
* all PHYs on a HCD and to keep them all in the same state.
*
* Copyright (C) 2018 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
*/
#include <linux/device.h>
#include <linux/list.h>
#include <linux/phy/phy.h>
#include <linux/of.h>
#include "phy.h"
struct usb_phy_roothub {
struct phy *phy;
struct list_head list;
};
/* Allocate the roothub_entry by specific name of phy */
static int usb_phy_roothub_add_phy_by_name(struct device *dev, const char *name,
struct list_head *list)
{
struct usb_phy_roothub *roothub_entry;
struct phy *phy;
phy = devm_of_phy_get(dev, dev->of_node, name);
if (IS_ERR(phy))
return PTR_ERR(phy);
roothub_entry = devm_kzalloc(dev, sizeof(*roothub_entry), GFP_KERNEL);
if (!roothub_entry)
return -ENOMEM;
INIT_LIST_HEAD(&roothub_entry->list);
roothub_entry->phy = phy;
list_add_tail(&roothub_entry->list, list);
return 0;
}
static int usb_phy_roothub_add_phy(struct device *dev, int index,
struct list_head *list)
{
struct usb_phy_roothub *roothub_entry;
struct phy *phy;
phy = devm_of_phy_get_by_index(dev, dev->of_node, index);
if (IS_ERR(phy)) {
if (PTR_ERR(phy) == -ENODEV)
return 0;
else
return PTR_ERR(phy);
}
roothub_entry = devm_kzalloc(dev, sizeof(*roothub_entry), GFP_KERNEL);
if (!roothub_entry)
return -ENOMEM;
INIT_LIST_HEAD(&roothub_entry->list);
roothub_entry->phy = phy;
list_add_tail(&roothub_entry->list, list);
return 0;
}
struct usb_phy_roothub *usb_phy_roothub_alloc(struct device *dev)
{
struct usb_phy_roothub *phy_roothub;
int i, num_phys, err;
if (!IS_ENABLED(CONFIG_GENERIC_PHY))
return NULL;
num_phys = of_count_phandle_with_args(dev->of_node, "phys",
"#phy-cells");
if (num_phys <= 0)
return NULL;
phy_roothub = devm_kzalloc(dev, sizeof(*phy_roothub), GFP_KERNEL);
if (!phy_roothub)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&phy_roothub->list);
if (!usb_phy_roothub_add_phy_by_name(dev, "usb2-phy", &phy_roothub->list))
return phy_roothub;
for (i = 0; i < num_phys; i++) {
err = usb_phy_roothub_add_phy(dev, i, &phy_roothub->list);
if (err)
return ERR_PTR(err);
}
return phy_roothub;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_alloc);
/**
* usb_phy_roothub_alloc_usb3_phy - alloc the roothub
* @dev: the device of the host controller
*
* Allocate the usb phy roothub if the host use a generic usb3-phy.
*
* Return: On success, a pointer to the usb_phy_roothub. Otherwise,
* %NULL if no use usb3 phy or %-ENOMEM if out of memory.
*/
struct usb_phy_roothub *usb_phy_roothub_alloc_usb3_phy(struct device *dev)
{
struct usb_phy_roothub *phy_roothub;
int num_phys;
if (!IS_ENABLED(CONFIG_GENERIC_PHY))
return NULL;
num_phys = of_count_phandle_with_args(dev->of_node, "phys",
"#phy-cells");
if (num_phys <= 0)
return NULL;
phy_roothub = devm_kzalloc(dev, sizeof(*phy_roothub), GFP_KERNEL);
if (!phy_roothub)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&phy_roothub->list);
if (!usb_phy_roothub_add_phy_by_name(dev, "usb3-phy", &phy_roothub->list))
return phy_roothub;
return NULL;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_alloc_usb3_phy);
int usb_phy_roothub_init(struct usb_phy_roothub *phy_roothub)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) {
err = phy_init(roothub_entry->phy);
if (err)
goto err_exit_phys;
}
return 0;
err_exit_phys:
list_for_each_entry_continue_reverse(roothub_entry, head, list)
phy_exit(roothub_entry->phy);
return err;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_init);
int usb_phy_roothub_exit(struct usb_phy_roothub *phy_roothub)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err, ret = 0;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) {
err = phy_exit(roothub_entry->phy);
if (err)
ret = err;
}
return ret;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_exit);
int usb_phy_roothub_set_mode(struct usb_phy_roothub *phy_roothub,
enum phy_mode mode)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) {
err = phy_set_mode(roothub_entry->phy, mode);
if (err)
goto err_out;
}
return 0;
err_out:
list_for_each_entry_continue_reverse(roothub_entry, head, list)
phy_power_off(roothub_entry->phy);
return err;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_set_mode);
int usb_phy_roothub_calibrate(struct usb_phy_roothub *phy_roothub)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) { err = phy_calibrate(roothub_entry->phy); if (err)
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_calibrate);
/**
* usb_phy_roothub_notify_connect() - connect notification
* @phy_roothub: the phy of roothub, if the host use a generic phy.
* @port: the port index for connect
*
* If the phy needs to get connection status, the callback can be used.
* Returns: %0 if successful, a negative error code otherwise
*/
int usb_phy_roothub_notify_connect(struct usb_phy_roothub *phy_roothub, int port)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) { err = phy_notify_connect(roothub_entry->phy, port); if (err)
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_notify_connect);
/**
* usb_phy_roothub_notify_disconnect() - disconnect notification
* @phy_roothub: the phy of roothub, if the host use a generic phy.
* @port: the port index for disconnect
*
* If the phy needs to get connection status, the callback can be used.
* Returns: %0 if successful, a negative error code otherwise
*/
int usb_phy_roothub_notify_disconnect(struct usb_phy_roothub *phy_roothub, int port)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) { err = phy_notify_disconnect(roothub_entry->phy, port); if (err)
return err;
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_notify_disconnect);
int usb_phy_roothub_power_on(struct usb_phy_roothub *phy_roothub)
{
struct usb_phy_roothub *roothub_entry;
struct list_head *head;
int err;
if (!phy_roothub)
return 0;
head = &phy_roothub->list;
list_for_each_entry(roothub_entry, head, list) {
err = phy_power_on(roothub_entry->phy);
if (err)
goto err_out;
}
return 0;
err_out:
list_for_each_entry_continue_reverse(roothub_entry, head, list)
phy_power_off(roothub_entry->phy);
return err;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_power_on);
void usb_phy_roothub_power_off(struct usb_phy_roothub *phy_roothub)
{
struct usb_phy_roothub *roothub_entry;
if (!phy_roothub)
return;
list_for_each_entry_reverse(roothub_entry, &phy_roothub->list, list)
phy_power_off(roothub_entry->phy);
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_power_off);
int usb_phy_roothub_suspend(struct device *controller_dev,
struct usb_phy_roothub *phy_roothub)
{
usb_phy_roothub_power_off(phy_roothub);
/* keep the PHYs initialized so the device can wake up the system */
if (device_may_wakeup(controller_dev))
return 0;
return usb_phy_roothub_exit(phy_roothub);
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_suspend);
int usb_phy_roothub_resume(struct device *controller_dev,
struct usb_phy_roothub *phy_roothub)
{
int err;
/* if the device can't wake up the system _exit was called */
if (!device_may_wakeup(controller_dev)) {
err = usb_phy_roothub_init(phy_roothub);
if (err)
return err;
}
err = usb_phy_roothub_power_on(phy_roothub);
/* undo _init if _power_on failed */
if (err && !device_may_wakeup(controller_dev))
usb_phy_roothub_exit(phy_roothub);
return err;
}
EXPORT_SYMBOL_GPL(usb_phy_roothub_resume);
// SPDX-License-Identifier: GPL-2.0-only
/*
* lib/bitmap.c
* Helper functions for bitmap.h.
*/
#include <linux/bitmap.h>
#include <linux/bitops.h>
#include <linux/ctype.h>
#include <linux/device.h>
#include <linux/export.h>
#include <linux/slab.h>
/**
* DOC: bitmap introduction
*
* bitmaps provide an array of bits, implemented using an
* array of unsigned longs. The number of valid bits in a
* given bitmap does _not_ need to be an exact multiple of
* BITS_PER_LONG.
*
* The possible unused bits in the last, partially used word
* of a bitmap are 'don't care'. The implementation makes
* no particular effort to keep them zero. It ensures that
* their value will not affect the results of any operation.
* The bitmap operations that return Boolean (bitmap_empty,
* for example) or scalar (bitmap_weight, for example) results
* carefully filter out these unused bits from impacting their
* results.
*
* The byte ordering of bitmaps is more natural on little
* endian architectures. See the big-endian headers
* include/asm-ppc64/bitops.h and include/asm-s390/bitops.h
* for the best explanations of this ordering.
*/
bool __bitmap_equal(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k, lim = bits/BITS_PER_LONG;
for (k = 0; k < lim; ++k)
if (bitmap1[k] != bitmap2[k])
return false;
if (bits % BITS_PER_LONG)
if ((bitmap1[k] ^ bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits))
return false;
return true;
}
EXPORT_SYMBOL(__bitmap_equal);
bool __bitmap_or_equal(const unsigned long *bitmap1,
const unsigned long *bitmap2,
const unsigned long *bitmap3,
unsigned int bits)
{
unsigned int k, lim = bits / BITS_PER_LONG;
unsigned long tmp;
for (k = 0; k < lim; ++k) {
if ((bitmap1[k] | bitmap2[k]) != bitmap3[k])
return false;
}
if (!(bits % BITS_PER_LONG))
return true;
tmp = (bitmap1[k] | bitmap2[k]) ^ bitmap3[k];
return (tmp & BITMAP_LAST_WORD_MASK(bits)) == 0;
}
void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits)
{
unsigned int k, lim = BITS_TO_LONGS(bits);
for (k = 0; k < lim; ++k)
dst[k] = ~src[k];
}
EXPORT_SYMBOL(__bitmap_complement);
/**
* __bitmap_shift_right - logical right shift of the bits in a bitmap
* @dst : destination bitmap
* @src : source bitmap
* @shift : shift by this many bits
* @nbits : bitmap size, in bits
*
* Shifting right (dividing) means moving bits in the MS -> LS bit
* direction. Zeros are fed into the vacated MS positions and the
* LS bits shifted off the bottom are lost.
*/
void __bitmap_shift_right(unsigned long *dst, const unsigned long *src,
unsigned shift, unsigned nbits)
{
unsigned k, lim = BITS_TO_LONGS(nbits);
unsigned off = shift/BITS_PER_LONG, rem = shift % BITS_PER_LONG;
unsigned long mask = BITMAP_LAST_WORD_MASK(nbits);
for (k = 0; off + k < lim; ++k) {
unsigned long upper, lower;
/*
* If shift is not word aligned, take lower rem bits of
* word above and make them the top rem bits of result.
*/
if (!rem || off + k + 1 >= lim)
upper = 0;
else {
upper = src[off + k + 1];
if (off + k + 1 == lim - 1)
upper &= mask;
upper <<= (BITS_PER_LONG - rem);
}
lower = src[off + k];
if (off + k == lim - 1)
lower &= mask;
lower >>= rem;
dst[k] = lower | upper;
}
if (off)
memset(&dst[lim - off], 0, off*sizeof(unsigned long));
}
EXPORT_SYMBOL(__bitmap_shift_right);
/**
* __bitmap_shift_left - logical left shift of the bits in a bitmap
* @dst : destination bitmap
* @src : source bitmap
* @shift : shift by this many bits
* @nbits : bitmap size, in bits
*
* Shifting left (multiplying) means moving bits in the LS -> MS
* direction. Zeros are fed into the vacated LS bit positions
* and those MS bits shifted off the top are lost.
*/
void __bitmap_shift_left(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits)
{
int k;
unsigned int lim = BITS_TO_LONGS(nbits);
unsigned int off = shift/BITS_PER_LONG, rem = shift % BITS_PER_LONG;
for (k = lim - off - 1; k >= 0; --k) {
unsigned long upper, lower;
/*
* If shift is not word aligned, take upper rem bits of
* word below and make them the bottom rem bits of result.
*/
if (rem && k > 0)
lower = src[k - 1] >> (BITS_PER_LONG - rem);
else
lower = 0;
upper = src[k] << rem;
dst[k + off] = lower | upper;
}
if (off)
memset(dst, 0, off*sizeof(unsigned long));
}
EXPORT_SYMBOL(__bitmap_shift_left);
/**
* bitmap_cut() - remove bit region from bitmap and right shift remaining bits
* @dst: destination bitmap, might overlap with src
* @src: source bitmap
* @first: start bit of region to be removed
* @cut: number of bits to remove
* @nbits: bitmap size, in bits
*
* Set the n-th bit of @dst iff the n-th bit of @src is set and
* n is less than @first, or the m-th bit of @src is set for any
* m such that @first <= n < nbits, and m = n + @cut.
*
* In pictures, example for a big-endian 32-bit architecture:
*
* The @src bitmap is::
*
* 31 63
* | |
* 10000000 11000001 11110010 00010101 10000000 11000001 01110010 00010101
* | | | |
* 16 14 0 32
*
* if @cut is 3, and @first is 14, bits 14-16 in @src are cut and @dst is::
*
* 31 63
* | |
* 10110000 00011000 00110010 00010101 00010000 00011000 00101110 01000010
* | | |
* 14 (bit 17 0 32
* from @src)
*
* Note that @dst and @src might overlap partially or entirely.
*
* This is implemented in the obvious way, with a shift and carry
* step for each moved bit. Optimisation is left as an exercise
* for the compiler.
*/
void bitmap_cut(unsigned long *dst, const unsigned long *src,
unsigned int first, unsigned int cut, unsigned int nbits)
{
unsigned int len = BITS_TO_LONGS(nbits);
unsigned long keep = 0, carry;
int i;
if (first % BITS_PER_LONG) {
keep = src[first / BITS_PER_LONG] &
(~0UL >> (BITS_PER_LONG - first % BITS_PER_LONG));
}
memmove(dst, src, len * sizeof(*dst));
while (cut--) {
for (i = first / BITS_PER_LONG; i < len; i++) {
if (i < len - 1)
carry = dst[i + 1] & 1UL;
else
carry = 0;
dst[i] = (dst[i] >> 1) | (carry << (BITS_PER_LONG - 1));
}
}
dst[first / BITS_PER_LONG] &= ~0UL << (first % BITS_PER_LONG);
dst[first / BITS_PER_LONG] |= keep;
}
EXPORT_SYMBOL(bitmap_cut);
bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k;
unsigned int lim = bits/BITS_PER_LONG;
unsigned long result = 0;
for (k = 0; k < lim; k++)
result |= (dst[k] = bitmap1[k] & bitmap2[k]);
if (bits % BITS_PER_LONG)
result |= (dst[k] = bitmap1[k] & bitmap2[k] &
BITMAP_LAST_WORD_MASK(bits));
return result != 0;
}
EXPORT_SYMBOL(__bitmap_and);
void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k;
unsigned int nr = BITS_TO_LONGS(bits);
for (k = 0; k < nr; k++)
dst[k] = bitmap1[k] | bitmap2[k];
}
EXPORT_SYMBOL(__bitmap_or);
void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k;
unsigned int nr = BITS_TO_LONGS(bits);
for (k = 0; k < nr; k++)
dst[k] = bitmap1[k] ^ bitmap2[k];
}
EXPORT_SYMBOL(__bitmap_xor);
bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k;
unsigned int lim = bits/BITS_PER_LONG;
unsigned long result = 0;
for (k = 0; k < lim; k++)
result |= (dst[k] = bitmap1[k] & ~bitmap2[k]);
if (bits % BITS_PER_LONG)
result |= (dst[k] = bitmap1[k] & ~bitmap2[k] &
BITMAP_LAST_WORD_MASK(bits));
return result != 0;
}
EXPORT_SYMBOL(__bitmap_andnot);
void __bitmap_replace(unsigned long *dst,
const unsigned long *old, const unsigned long *new,
const unsigned long *mask, unsigned int nbits)
{
unsigned int k;
unsigned int nr = BITS_TO_LONGS(nbits);
for (k = 0; k < nr; k++)
dst[k] = (old[k] & ~mask[k]) | (new[k] & mask[k]);
}
EXPORT_SYMBOL(__bitmap_replace);
bool __bitmap_intersects(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k, lim = bits/BITS_PER_LONG;
for (k = 0; k < lim; ++k)
if (bitmap1[k] & bitmap2[k])
return true;
if (bits % BITS_PER_LONG)
if ((bitmap1[k] & bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits))
return true;
return false;
}
EXPORT_SYMBOL(__bitmap_intersects);
bool __bitmap_subset(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k, lim = bits/BITS_PER_LONG;
for (k = 0; k < lim; ++k)
if (bitmap1[k] & ~bitmap2[k])
return false;
if (bits % BITS_PER_LONG)
if ((bitmap1[k] & ~bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits))
return false;
return true;
}
EXPORT_SYMBOL(__bitmap_subset);
#define BITMAP_WEIGHT(FETCH, bits) \
({ \
unsigned int __bits = (bits), idx, w = 0; \
\
for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \
w += hweight_long(FETCH); \
\
if (__bits % BITS_PER_LONG) \
w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \
\
w; \
})
unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
{
return BITMAP_WEIGHT(bitmap[idx], bits);
}
EXPORT_SYMBOL(__bitmap_weight);
unsigned int __bitmap_weight_and(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits);
}
EXPORT_SYMBOL(__bitmap_weight_and);
unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
return BITMAP_WEIGHT(bitmap1[idx] & ~bitmap2[idx], bits);
}
EXPORT_SYMBOL(__bitmap_weight_andnot);
void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);
const unsigned int size = start + len;
int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
while (len - bits_to_set >= 0) {
*p |= mask_to_set;
len -= bits_to_set;
bits_to_set = BITS_PER_LONG;
mask_to_set = ~0UL;
p++;
}
if (len) {
mask_to_set &= BITMAP_LAST_WORD_MASK(size);
*p |= mask_to_set;
}
}
EXPORT_SYMBOL(__bitmap_set);
void __bitmap_clear(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start); const unsigned int size = start + len;
int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
while (len - bits_to_clear >= 0) {
*p &= ~mask_to_clear;
len -= bits_to_clear;
bits_to_clear = BITS_PER_LONG;
mask_to_clear = ~0UL;
p++;
}
if (len) {
mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
*p &= ~mask_to_clear;
}
}
EXPORT_SYMBOL(__bitmap_clear);
/**
* bitmap_find_next_zero_area_off - find a contiguous aligned zero area
* @map: The address to base the search on
* @size: The bitmap size in bits
* @start: The bitnumber to start searching at
* @nr: The number of zeroed bits we're looking for
* @align_mask: Alignment mask for zero area
* @align_offset: Alignment offset for zero area.
*
* The @align_mask should be one less than a power of 2; the effect is that
* the bit offset of all zero areas this function finds plus @align_offset
* is multiple of that power of 2.
*/
unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
unsigned long size,
unsigned long start,
unsigned int nr,
unsigned long align_mask,
unsigned long align_offset)
{
unsigned long index, end, i;
again:
index = find_next_zero_bit(map, size, start);
/* Align allocation */
index = __ALIGN_MASK(index + align_offset, align_mask) - align_offset;
end = index + nr;
if (end > size)
return end;
i = find_next_bit(map, end, index);
if (i < end) {
start = i + 1;
goto again;
}
return index;
}
EXPORT_SYMBOL(bitmap_find_next_zero_area_off);
/**
* bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap
* @buf: pointer to a bitmap
* @pos: a bit position in @buf (0 <= @pos < @nbits)
* @nbits: number of valid bit positions in @buf
*
* Map the bit at position @pos in @buf (of length @nbits) to the
* ordinal of which set bit it is. If it is not set or if @pos
* is not a valid bit position, map to -1.
*
* If for example, just bits 4 through 7 are set in @buf, then @pos
* values 4 through 7 will get mapped to 0 through 3, respectively,
* and other @pos values will get mapped to -1. When @pos value 7
* gets mapped to (returns) @ord value 3 in this example, that means
* that bit 7 is the 3rd (starting with 0th) set bit in @buf.
*
* The bit positions 0 through @bits are valid positions in @buf.
*/
static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigned int nbits)
{
if (pos >= nbits || !test_bit(pos, buf))
return -1;
return bitmap_weight(buf, pos);
}
/**
* bitmap_remap - Apply map defined by a pair of bitmaps to another bitmap
* @dst: remapped result
* @src: subset to be remapped
* @old: defines domain of map
* @new: defines range of map
* @nbits: number of bits in each of these bitmaps
*
* Let @old and @new define a mapping of bit positions, such that
* whatever position is held by the n-th set bit in @old is mapped
* to the n-th set bit in @new. In the more general case, allowing
* for the possibility that the weight 'w' of @new is less than the
* weight of @old, map the position of the n-th set bit in @old to
* the position of the m-th set bit in @new, where m == n % w.
*
* If either of the @old and @new bitmaps are empty, or if @src and
* @dst point to the same location, then this routine copies @src
* to @dst.
*
* The positions of unset bits in @old are mapped to themselves
* (the identity map).
*
* Apply the above specified mapping to @src, placing the result in
* @dst, clearing any bits previously set in @dst.
*
* For example, lets say that @old has bits 4 through 7 set, and
* @new has bits 12 through 15 set. This defines the mapping of bit
* position 4 to 12, 5 to 13, 6 to 14 and 7 to 15, and of all other
* bit positions unchanged. So if say @src comes into this routine
* with bits 1, 5 and 7 set, then @dst should leave with bits 1,
* 13 and 15 set.
*/
void bitmap_remap(unsigned long *dst, const unsigned long *src,
const unsigned long *old, const unsigned long *new,
unsigned int nbits)
{
unsigned int oldbit, w;
if (dst == src) /* following doesn't handle inplace remaps */
return;
bitmap_zero(dst, nbits);
w = bitmap_weight(new, nbits);
for_each_set_bit(oldbit, src, nbits) {
int n = bitmap_pos_to_ord(old, oldbit, nbits);
if (n < 0 || w == 0)
set_bit(oldbit, dst); /* identity map */
else
set_bit(find_nth_bit(new, nbits, n % w), dst);
}
}
EXPORT_SYMBOL(bitmap_remap);
/**
* bitmap_bitremap - Apply map defined by a pair of bitmaps to a single bit
* @oldbit: bit position to be mapped
* @old: defines domain of map
* @new: defines range of map
* @bits: number of bits in each of these bitmaps
*
* Let @old and @new define a mapping of bit positions, such that
* whatever position is held by the n-th set bit in @old is mapped
* to the n-th set bit in @new. In the more general case, allowing
* for the possibility that the weight 'w' of @new is less than the
* weight of @old, map the position of the n-th set bit in @old to
* the position of the m-th set bit in @new, where m == n % w.
*
* The positions of unset bits in @old are mapped to themselves
* (the identity map).
*
* Apply the above specified mapping to bit position @oldbit, returning
* the new bit position.
*
* For example, lets say that @old has bits 4 through 7 set, and
* @new has bits 12 through 15 set. This defines the mapping of bit
* position 4 to 12, 5 to 13, 6 to 14 and 7 to 15, and of all other
* bit positions unchanged. So if say @oldbit is 5, then this routine
* returns 13.
*/
int bitmap_bitremap(int oldbit, const unsigned long *old,
const unsigned long *new, int bits)
{
int w = bitmap_weight(new, bits);
int n = bitmap_pos_to_ord(old, oldbit, bits);
if (n < 0 || w == 0)
return oldbit;
else
return find_nth_bit(new, bits, n % w);
}
EXPORT_SYMBOL(bitmap_bitremap);
#ifdef CONFIG_NUMA
/**
* bitmap_onto - translate one bitmap relative to another
* @dst: resulting translated bitmap
* @orig: original untranslated bitmap
* @relmap: bitmap relative to which translated
* @bits: number of bits in each of these bitmaps
*
* Set the n-th bit of @dst iff there exists some m such that the
* n-th bit of @relmap is set, the m-th bit of @orig is set, and
* the n-th bit of @relmap is also the m-th _set_ bit of @relmap.
* (If you understood the previous sentence the first time your
* read it, you're overqualified for your current job.)
*
* In other words, @orig is mapped onto (surjectively) @dst,
* using the map { <n, m> | the n-th bit of @relmap is the
* m-th set bit of @relmap }.
*
* Any set bits in @orig above bit number W, where W is the
* weight of (number of set bits in) @relmap are mapped nowhere.
* In particular, if for all bits m set in @orig, m >= W, then
* @dst will end up empty. In situations where the possibility
* of such an empty result is not desired, one way to avoid it is
* to use the bitmap_fold() operator, below, to first fold the
* @orig bitmap over itself so that all its set bits x are in the
* range 0 <= x < W. The bitmap_fold() operator does this by
* setting the bit (m % W) in @dst, for each bit (m) set in @orig.
*
* Example [1] for bitmap_onto():
* Let's say @relmap has bits 30-39 set, and @orig has bits
* 1, 3, 5, 7, 9 and 11 set. Then on return from this routine,
* @dst will have bits 31, 33, 35, 37 and 39 set.
*
* When bit 0 is set in @orig, it means turn on the bit in
* @dst corresponding to whatever is the first bit (if any)
* that is turned on in @relmap. Since bit 0 was off in the
* above example, we leave off that bit (bit 30) in @dst.
*
* When bit 1 is set in @orig (as in the above example), it
* means turn on the bit in @dst corresponding to whatever
* is the second bit that is turned on in @relmap. The second
* bit in @relmap that was turned on in the above example was
* bit 31, so we turned on bit 31 in @dst.
*
* Similarly, we turned on bits 33, 35, 37 and 39 in @dst,
* because they were the 4th, 6th, 8th and 10th set bits
* set in @relmap, and the 4th, 6th, 8th and 10th bits of
* @orig (i.e. bits 3, 5, 7 and 9) were also set.
*
* When bit 11 is set in @orig, it means turn on the bit in
* @dst corresponding to whatever is the twelfth bit that is
* turned on in @relmap. In the above example, there were
* only ten bits turned on in @relmap (30..39), so that bit
* 11 was set in @orig had no affect on @dst.
*
* Example [2] for bitmap_fold() + bitmap_onto():
* Let's say @relmap has these ten bits set::
*
* 40 41 42 43 45 48 53 61 74 95
*
* (for the curious, that's 40 plus the first ten terms of the
* Fibonacci sequence.)
*
* Further lets say we use the following code, invoking
* bitmap_fold() then bitmap_onto, as suggested above to
* avoid the possibility of an empty @dst result::
*
* unsigned long *tmp; // a temporary bitmap's bits
*
* bitmap_fold(tmp, orig, bitmap_weight(relmap, bits), bits);
* bitmap_onto(dst, tmp, relmap, bits);
*
* Then this table shows what various values of @dst would be, for
* various @orig's. I list the zero-based positions of each set bit.
* The tmp column shows the intermediate result, as computed by
* using bitmap_fold() to fold the @orig bitmap modulo ten
* (the weight of @relmap):
*
* =============== ============== =================
* @orig tmp @dst
* 0 0 40
* 1 1 41
* 9 9 95
* 10 0 40 [#f1]_
* 1 3 5 7 1 3 5 7 41 43 48 61
* 0 1 2 3 4 0 1 2 3 4 40 41 42 43 45
* 0 9 18 27 0 9 8 7 40 61 74 95
* 0 10 20 30 0 40
* 0 11 22 33 0 1 2 3 40 41 42 43
* 0 12 24 36 0 2 4 6 40 42 45 53
* 78 102 211 1 2 8 41 42 74 [#f1]_
* =============== ============== =================
*
* .. [#f1]
*
* For these marked lines, if we hadn't first done bitmap_fold()
* into tmp, then the @dst result would have been empty.
*
* If either of @orig or @relmap is empty (no set bits), then @dst
* will be returned empty.
*
* If (as explained above) the only set bits in @orig are in positions
* m where m >= W, (where W is the weight of @relmap) then @dst will
* once again be returned empty.
*
* All bits in @dst not set by the above rule are cleared.
*/
void bitmap_onto(unsigned long *dst, const unsigned long *orig,
const unsigned long *relmap, unsigned int bits)
{
unsigned int n, m; /* same meaning as in above comment */
if (dst == orig) /* following doesn't handle inplace mappings */
return;
bitmap_zero(dst, bits);
/*
* The following code is a more efficient, but less
* obvious, equivalent to the loop:
* for (m = 0; m < bitmap_weight(relmap, bits); m++) {
* n = find_nth_bit(orig, bits, m);
* if (test_bit(m, orig))
* set_bit(n, dst);
* }
*/
m = 0;
for_each_set_bit(n, relmap, bits) {
/* m == bitmap_pos_to_ord(relmap, n, bits) */
if (test_bit(m, orig))
set_bit(n, dst);
m++;
}
}
/**
* bitmap_fold - fold larger bitmap into smaller, modulo specified size
* @dst: resulting smaller bitmap
* @orig: original larger bitmap
* @sz: specified size
* @nbits: number of bits in each of these bitmaps
*
* For each bit oldbit in @orig, set bit oldbit mod @sz in @dst.
* Clear all other bits in @dst. See further the comment and
* Example [2] for bitmap_onto() for why and how to use this.
*/
void bitmap_fold(unsigned long *dst, const unsigned long *orig,
unsigned int sz, unsigned int nbits)
{
unsigned int oldbit;
if (dst == orig) /* following doesn't handle inplace mappings */
return;
bitmap_zero(dst, nbits);
for_each_set_bit(oldbit, orig, nbits)
set_bit(oldbit % sz, dst);
}
#endif /* CONFIG_NUMA */
unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags)
{
return kmalloc_array(BITS_TO_LONGS(nbits), sizeof(unsigned long),
flags);
}
EXPORT_SYMBOL(bitmap_alloc);
unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags)
{
return bitmap_alloc(nbits, flags | __GFP_ZERO);
}
EXPORT_SYMBOL(bitmap_zalloc);
unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node)
{
return kmalloc_array_node(BITS_TO_LONGS(nbits), sizeof(unsigned long),
flags, node);
}
EXPORT_SYMBOL(bitmap_alloc_node);
unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node)
{
return bitmap_alloc_node(nbits, flags | __GFP_ZERO, node);
}
EXPORT_SYMBOL(bitmap_zalloc_node);
void bitmap_free(const unsigned long *bitmap)
{
kfree(bitmap);
}
EXPORT_SYMBOL(bitmap_free);
static void devm_bitmap_free(void *data)
{
unsigned long *bitmap = data;
bitmap_free(bitmap);
}
unsigned long *devm_bitmap_alloc(struct device *dev,
unsigned int nbits, gfp_t flags)
{
unsigned long *bitmap;
int ret;
bitmap = bitmap_alloc(nbits, flags);
if (!bitmap)
return NULL;
ret = devm_add_action_or_reset(dev, devm_bitmap_free, bitmap);
if (ret)
return NULL;
return bitmap;
}
EXPORT_SYMBOL_GPL(devm_bitmap_alloc);
unsigned long *devm_bitmap_zalloc(struct device *dev,
unsigned int nbits, gfp_t flags)
{
return devm_bitmap_alloc(dev, nbits, flags | __GFP_ZERO);
}
EXPORT_SYMBOL_GPL(devm_bitmap_zalloc);
#if BITS_PER_LONG == 64
/**
* bitmap_from_arr32 - copy the contents of u32 array of bits to bitmap
* @bitmap: array of unsigned longs, the destination bitmap
* @buf: array of u32 (in host byte order), the source bitmap
* @nbits: number of bits in @bitmap
*/
void bitmap_from_arr32(unsigned long *bitmap, const u32 *buf, unsigned int nbits)
{
unsigned int i, halfwords;
halfwords = DIV_ROUND_UP(nbits, 32);
for (i = 0; i < halfwords; i++) {
bitmap[i/2] = (unsigned long) buf[i];
if (++i < halfwords)
bitmap[i/2] |= ((unsigned long) buf[i]) << 32;
}
/* Clear tail bits in last word beyond nbits. */
if (nbits % BITS_PER_LONG)
bitmap[(halfwords - 1) / 2] &= BITMAP_LAST_WORD_MASK(nbits);
}
EXPORT_SYMBOL(bitmap_from_arr32);
/**
* bitmap_to_arr32 - copy the contents of bitmap to a u32 array of bits
* @buf: array of u32 (in host byte order), the dest bitmap
* @bitmap: array of unsigned longs, the source bitmap
* @nbits: number of bits in @bitmap
*/
void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits)
{
unsigned int i, halfwords;
halfwords = DIV_ROUND_UP(nbits, 32);
for (i = 0; i < halfwords; i++) {
buf[i] = (u32) (bitmap[i/2] & UINT_MAX);
if (++i < halfwords)
buf[i] = (u32) (bitmap[i/2] >> 32);
}
/* Clear tail bits in last element of array beyond nbits. */
if (nbits % BITS_PER_LONG)
buf[halfwords - 1] &= (u32) (UINT_MAX >> ((-nbits) & 31));
}
EXPORT_SYMBOL(bitmap_to_arr32);
#endif
#if BITS_PER_LONG == 32
/**
* bitmap_from_arr64 - copy the contents of u64 array of bits to bitmap
* @bitmap: array of unsigned longs, the destination bitmap
* @buf: array of u64 (in host byte order), the source bitmap
* @nbits: number of bits in @bitmap
*/
void bitmap_from_arr64(unsigned long *bitmap, const u64 *buf, unsigned int nbits)
{
int n;
for (n = nbits; n > 0; n -= 64) {
u64 val = *buf++;
*bitmap++ = val;
if (n > 32)
*bitmap++ = val >> 32;
}
/*
* Clear tail bits in the last word beyond nbits.
*
* Negative index is OK because here we point to the word next
* to the last word of the bitmap, except for nbits == 0, which
* is tested implicitly.
*/
if (nbits % BITS_PER_LONG)
bitmap[-1] &= BITMAP_LAST_WORD_MASK(nbits);
}
EXPORT_SYMBOL(bitmap_from_arr64);
/**
* bitmap_to_arr64 - copy the contents of bitmap to a u64 array of bits
* @buf: array of u64 (in host byte order), the dest bitmap
* @bitmap: array of unsigned longs, the source bitmap
* @nbits: number of bits in @bitmap
*/
void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits)
{
const unsigned long *end = bitmap + BITS_TO_LONGS(nbits);
while (bitmap < end) {
*buf = *bitmap++;
if (bitmap < end)
*buf |= (u64)(*bitmap++) << 32;
buf++;
}
/* Clear tail bits in the last element of array beyond nbits. */
if (nbits % 64)
buf[-1] &= GENMASK_ULL((nbits - 1) % 64, 0);
}
EXPORT_SYMBOL(bitmap_to_arr64);
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TIMEKEEPING_H
#define _LINUX_TIMEKEEPING_H
#include <linux/errno.h>
#include <linux/clocksource_ids.h>
#include <linux/ktime.h>
/* Included from linux/ktime.h */
void timekeeping_init(void);
extern int timekeeping_suspended;
/* Architecture timer tick functions: */
extern void legacy_timer_tick(unsigned long ticks);
/*
* Get and set timeofday
*/
extern int do_settimeofday64(const struct timespec64 *ts);
extern int do_sys_settimeofday64(const struct timespec64 *tv,
const struct timezone *tz);
/*
* ktime_get() family - read the current time in a multitude of ways.
*
* The default time reference is CLOCK_MONOTONIC, starting at
* boot time but not counting the time spent in suspend.
* For other references, use the functions with "real", "clocktai",
* "boottime" and "raw" suffixes.
*
* To get the time in a different format, use the ones with
* "ns", "ts64" and "seconds" suffix.
*
* See Documentation/core-api/timekeeping.rst for more details.
*/
/*
* timespec64 based interfaces
*/
extern void ktime_get_raw_ts64(struct timespec64 *ts);
extern void ktime_get_ts64(struct timespec64 *ts);
extern void ktime_get_real_ts64(struct timespec64 *tv);
extern void ktime_get_coarse_ts64(struct timespec64 *ts);
extern void ktime_get_coarse_real_ts64(struct timespec64 *ts);
void getboottime64(struct timespec64 *ts);
/*
* time64_t base interfaces
*/
extern time64_t ktime_get_seconds(void);
extern time64_t __ktime_get_real_seconds(void);
extern time64_t ktime_get_real_seconds(void);
/*
* ktime_t based interfaces
*/
enum tk_offsets {
TK_OFFS_REAL,
TK_OFFS_BOOT,
TK_OFFS_TAI,
TK_OFFS_MAX,
};
extern ktime_t ktime_get(void);
extern ktime_t ktime_get_with_offset(enum tk_offsets offs);
extern ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs);
extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs);
extern ktime_t ktime_get_raw(void);
extern u32 ktime_get_resolution_ns(void);
/**
* ktime_get_real - get the real (wall-) time in ktime_t format
*
* Returns: real (wall) time in ktime_t format
*/
static inline ktime_t ktime_get_real(void)
{
return ktime_get_with_offset(TK_OFFS_REAL);
}
static inline ktime_t ktime_get_coarse_real(void)
{
return ktime_get_coarse_with_offset(TK_OFFS_REAL);
}
/**
* ktime_get_boottime - Get monotonic time since boot in ktime_t format
*
* This is similar to CLOCK_MONTONIC/ktime_get, but also includes the
* time spent in suspend.
*
* Returns: monotonic time since boot in ktime_t format
*/
static inline ktime_t ktime_get_boottime(void)
{
return ktime_get_with_offset(TK_OFFS_BOOT);
}
static inline ktime_t ktime_get_coarse_boottime(void)
{
return ktime_get_coarse_with_offset(TK_OFFS_BOOT);
}
/**
* ktime_get_clocktai - Get the TAI time of day in ktime_t format
*
* Returns: the TAI time of day in ktime_t format
*/
static inline ktime_t ktime_get_clocktai(void)
{
return ktime_get_with_offset(TK_OFFS_TAI);
}
static inline ktime_t ktime_get_coarse_clocktai(void)
{
return ktime_get_coarse_with_offset(TK_OFFS_TAI);
}
static inline ktime_t ktime_get_coarse(void)
{
struct timespec64 ts;
ktime_get_coarse_ts64(&ts);
return timespec64_to_ktime(ts);
}
static inline u64 ktime_get_coarse_ns(void)
{
return ktime_to_ns(ktime_get_coarse());
}
static inline u64 ktime_get_coarse_real_ns(void)
{
return ktime_to_ns(ktime_get_coarse_real());
}
static inline u64 ktime_get_coarse_boottime_ns(void)
{
return ktime_to_ns(ktime_get_coarse_boottime());
}
static inline u64 ktime_get_coarse_clocktai_ns(void)
{
return ktime_to_ns(ktime_get_coarse_clocktai());
}
/**
* ktime_mono_to_real - Convert monotonic time to clock realtime
* @mono: monotonic time to convert
*
* Returns: time converted to realtime clock
*/
static inline ktime_t ktime_mono_to_real(ktime_t mono)
{
return ktime_mono_to_any(mono, TK_OFFS_REAL);
}
/**
* ktime_get_ns - Get the current time in nanoseconds
*
* Returns: current time converted to nanoseconds
*/
static inline u64 ktime_get_ns(void)
{
return ktime_to_ns(ktime_get());
}
/**
* ktime_get_real_ns - Get the current real/wall time in nanoseconds
*
* Returns: current real time converted to nanoseconds
*/
static inline u64 ktime_get_real_ns(void)
{
return ktime_to_ns(ktime_get_real());
}
/**
* ktime_get_boottime_ns - Get the monotonic time since boot in nanoseconds
*
* Returns: current boottime converted to nanoseconds
*/
static inline u64 ktime_get_boottime_ns(void)
{
return ktime_to_ns(ktime_get_boottime());
}
/**
* ktime_get_clocktai_ns - Get the current TAI time of day in nanoseconds
*
* Returns: current TAI time converted to nanoseconds
*/
static inline u64 ktime_get_clocktai_ns(void)
{
return ktime_to_ns(ktime_get_clocktai());
}
/**
* ktime_get_raw_ns - Get the raw monotonic time in nanoseconds
*
* Returns: current raw monotonic time converted to nanoseconds
*/
static inline u64 ktime_get_raw_ns(void)
{
return ktime_to_ns(ktime_get_raw());
}
extern u64 ktime_get_mono_fast_ns(void);
extern u64 ktime_get_raw_fast_ns(void);
extern u64 ktime_get_boot_fast_ns(void);
extern u64 ktime_get_tai_fast_ns(void);
extern u64 ktime_get_real_fast_ns(void);
/*
* timespec64/time64_t interfaces utilizing the ktime based ones
* for API completeness, these could be implemented more efficiently
* if needed.
*/
static inline void ktime_get_boottime_ts64(struct timespec64 *ts)
{
*ts = ktime_to_timespec64(ktime_get_boottime());
}
static inline void ktime_get_coarse_boottime_ts64(struct timespec64 *ts)
{
*ts = ktime_to_timespec64(ktime_get_coarse_boottime());
}
static inline time64_t ktime_get_boottime_seconds(void)
{
return ktime_divns(ktime_get_coarse_boottime(), NSEC_PER_SEC);
}
static inline void ktime_get_clocktai_ts64(struct timespec64 *ts)
{
*ts = ktime_to_timespec64(ktime_get_clocktai());
}
static inline void ktime_get_coarse_clocktai_ts64(struct timespec64 *ts)
{
*ts = ktime_to_timespec64(ktime_get_coarse_clocktai());
}
static inline time64_t ktime_get_clocktai_seconds(void)
{
return ktime_divns(ktime_get_coarse_clocktai(), NSEC_PER_SEC);
}
/*
* RTC specific
*/
extern bool timekeeping_rtc_skipsuspend(void);
extern bool timekeeping_rtc_skipresume(void);
extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);
/**
* struct ktime_timestamps - Simultaneous mono/boot/real timestamps
* @mono: Monotonic timestamp
* @boot: Boottime timestamp
* @real: Realtime timestamp
*/
struct ktime_timestamps {
u64 mono;
u64 boot;
u64 real;
};
/**
* struct system_time_snapshot - simultaneous raw/real time capture with
* counter value
* @cycles: Clocksource counter value to produce the system times
* @real: Realtime system time
* @raw: Monotonic raw system time
* @cs_id: Clocksource ID
* @clock_was_set_seq: The sequence number of clock-was-set events
* @cs_was_changed_seq: The sequence number of clocksource change events
*/
struct system_time_snapshot {
u64 cycles;
ktime_t real;
ktime_t raw;
enum clocksource_ids cs_id;
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
};
/**
* struct system_device_crosststamp - system/device cross-timestamp
* (synchronized capture)
* @device: Device time
* @sys_realtime: Realtime simultaneous with device time
* @sys_monoraw: Monotonic raw simultaneous with device time
*/
struct system_device_crosststamp {
ktime_t device;
ktime_t sys_realtime;
ktime_t sys_monoraw;
};
/**
* struct system_counterval_t - system counter value with the ID of the
* corresponding clocksource
* @cycles: System counter value
* @cs_id: Clocksource ID corresponding to system counter value. Used by
* timekeeping code to verify comparability of two cycle values.
* The default ID, CSID_GENERIC, does not identify a specific
* clocksource.
*/
struct system_counterval_t {
u64 cycles;
enum clocksource_ids cs_id;
};
/*
* Get cross timestamp between system clock and device clock
*/
extern int get_device_system_crosststamp(
int (*get_time_fn)(ktime_t *device_time,
struct system_counterval_t *system_counterval,
void *ctx),
void *ctx,
struct system_time_snapshot *history,
struct system_device_crosststamp *xtstamp);
/*
* Simultaneously snapshot realtime and monotonic raw clocks
*/
extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);
/* NMI safe mono/boot/realtime timestamps */
extern void ktime_get_fast_timestamps(struct ktime_timestamps *snap);
/*
* Persistent clock related interfaces
*/
extern int persistent_clock_is_local;
extern void read_persistent_clock64(struct timespec64 *ts);
void read_persistent_wall_and_boot_offset(struct timespec64 *wall_clock,
struct timespec64 *boot_offset);
#ifdef CONFIG_GENERIC_CMOS_UPDATE
extern int update_persistent_clock64(struct timespec64 now);
#endif
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* Common Block IO controller cgroup interface
*
* Based on ideas and code from CFQ, CFS and BFQ:
* Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
*
* Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
* Paolo Valente <paolo.valente@unimore.it>
*
* Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
* Nauman Rafique <nauman@google.com>
*
* For policy-specific per-blkcg data:
* Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
* Arianna Avanzini <avanzini.arianna@gmail.com>
*/
#include <linux/ioprio.h>
#include <linux/kdev_t.h>
#include <linux/module.h>
#include <linux/sched/signal.h>
#include <linux/err.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/atomic.h>
#include <linux/ctype.h>
#include <linux/resume_user_mode.h>
#include <linux/psi.h>
#include <linux/part_stat.h>
#include "blk.h"
#include "blk-cgroup.h"
#include "blk-ioprio.h"
#include "blk-throttle.h"
static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
/*
* blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
* blkcg_pol_register_mutex nests outside of it and synchronizes entire
* policy [un]register operations including cgroup file additions /
* removals. Putting cgroup file registration outside blkcg_pol_mutex
* allows grabbing it from cgroup callbacks.
*/
static DEFINE_MUTEX(blkcg_pol_register_mutex);
static DEFINE_MUTEX(blkcg_pol_mutex);
struct blkcg blkcg_root;
EXPORT_SYMBOL_GPL(blkcg_root);
struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
EXPORT_SYMBOL_GPL(blkcg_root_css);
static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
bool blkcg_debug_stats = false;
static DEFINE_RAW_SPINLOCK(blkg_stat_lock);
#define BLKG_DESTROY_BATCH_SIZE 64
/*
* Lockless lists for tracking IO stats update
*
* New IO stats are stored in the percpu iostat_cpu within blkcg_gq (blkg).
* There are multiple blkg's (one for each block device) attached to each
* blkcg. The rstat code keeps track of which cpu has IO stats updated,
* but it doesn't know which blkg has the updated stats. If there are many
* block devices in a system, the cost of iterating all the blkg's to flush
* out the IO stats can be high. To reduce such overhead, a set of percpu
* lockless lists (lhead) per blkcg are used to track the set of recently
* updated iostat_cpu's since the last flush. An iostat_cpu will be put
* onto the lockless list on the update side [blk_cgroup_bio_start()] if
* not there yet and then removed when being flushed [blkcg_rstat_flush()].
* References to blkg are gotten and then put back in the process to
* protect against blkg removal.
*
* Return: 0 if successful or -ENOMEM if allocation fails.
*/
static int init_blkcg_llists(struct blkcg *blkcg)
{
int cpu;
blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);
if (!blkcg->lhead)
return -ENOMEM;
for_each_possible_cpu(cpu)
init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));
return 0;
}
/**
* blkcg_css - find the current css
*
* Find the css associated with either the kthread or the current task.
* This may return a dying css, so it is up to the caller to use tryget logic
* to confirm it is alive and well.
*/
static struct cgroup_subsys_state *blkcg_css(void)
{
struct cgroup_subsys_state *css;
css = kthread_blkcg();
if (css)
return css;
return task_css(current, io_cgrp_id);
}
static bool blkcg_policy_enabled(struct request_queue *q,
const struct blkcg_policy *pol)
{
return pol && test_bit(pol->plid, q->blkcg_pols);
}
static void blkg_free_workfn(struct work_struct *work)
{
struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
free_work);
struct request_queue *q = blkg->q;
int i;
/*
* pd_free_fn() can also be called from blkcg_deactivate_policy(),
* in order to make sure pd_free_fn() is called in order, the deletion
* of the list blkg->q_node is delayed to here from blkg_destroy(), and
* blkcg_mutex is used to synchronize blkg_free_workfn() and
* blkcg_deactivate_policy().
*/
mutex_lock(&q->blkcg_mutex);
for (i = 0; i < BLKCG_MAX_POLS; i++)
if (blkg->pd[i])
blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
if (blkg->parent)
blkg_put(blkg->parent);
spin_lock_irq(&q->queue_lock);
list_del_init(&blkg->q_node);
spin_unlock_irq(&q->queue_lock);
mutex_unlock(&q->blkcg_mutex);
blk_put_queue(q);
free_percpu(blkg->iostat_cpu);
percpu_ref_exit(&blkg->refcnt);
kfree(blkg);
}
/**
* blkg_free - free a blkg
* @blkg: blkg to free
*
* Free @blkg which may be partially allocated.
*/
static void blkg_free(struct blkcg_gq *blkg)
{
if (!blkg)
return;
/*
* Both ->pd_free_fn() and request queue's release handler may
* sleep, so free us by scheduling one work func
*/
INIT_WORK(&blkg->free_work, blkg_free_workfn);
schedule_work(&blkg->free_work);
}
static void __blkg_release(struct rcu_head *rcu)
{
struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
struct blkcg *blkcg = blkg->blkcg;
int cpu;
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
WARN_ON(!bio_list_empty(&blkg->async_bios));
#endif
/*
* Flush all the non-empty percpu lockless lists before releasing
* us, given these stat belongs to us.
*
* blkg_stat_lock is for serializing blkg stat update
*/
for_each_possible_cpu(cpu)
__blkcg_rstat_flush(blkcg, cpu);
/* release the blkcg and parent blkg refs this blkg has been holding */
css_put(&blkg->blkcg->css);
blkg_free(blkg);
}
/*
* A group is RCU protected, but having an rcu lock does not mean that one
* can access all the fields of blkg and assume these are valid. For
* example, don't try to follow throtl_data and request queue links.
*
* Having a reference to blkg under an rcu allows accesses to only values
* local to groups like group stats and group rate limits.
*/
static void blkg_release(struct percpu_ref *ref)
{
struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
call_rcu(&blkg->rcu_head, __blkg_release);
}
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
static struct workqueue_struct *blkcg_punt_bio_wq;
static void blkg_async_bio_workfn(struct work_struct *work)
{
struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
async_bio_work);
struct bio_list bios = BIO_EMPTY_LIST;
struct bio *bio;
struct blk_plug plug;
bool need_plug = false;
/* as long as there are pending bios, @blkg can't go away */
spin_lock(&blkg->async_bio_lock);
bio_list_merge(&bios, &blkg->async_bios);
bio_list_init(&blkg->async_bios);
spin_unlock(&blkg->async_bio_lock);
/* start plug only when bio_list contains at least 2 bios */
if (bios.head && bios.head->bi_next) {
need_plug = true;
blk_start_plug(&plug);
}
while ((bio = bio_list_pop(&bios)))
submit_bio(bio);
if (need_plug)
blk_finish_plug(&plug);
}
/*
* When a shared kthread issues a bio for a cgroup, doing so synchronously can
* lead to priority inversions as the kthread can be trapped waiting for that
* cgroup. Use this helper instead of submit_bio to punt the actual issuing to
* a dedicated per-blkcg work item to avoid such priority inversions.
*/
void blkcg_punt_bio_submit(struct bio *bio)
{
struct blkcg_gq *blkg = bio->bi_blkg;
if (blkg->parent) {
spin_lock(&blkg->async_bio_lock);
bio_list_add(&blkg->async_bios, bio);
spin_unlock(&blkg->async_bio_lock);
queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
} else {
/* never bounce for the root cgroup */
submit_bio(bio);
}
}
EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit);
static int __init blkcg_punt_bio_init(void)
{
blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
WQ_MEM_RECLAIM | WQ_FREEZABLE |
WQ_UNBOUND | WQ_SYSFS, 0);
if (!blkcg_punt_bio_wq)
return -ENOMEM;
return 0;
}
subsys_initcall(blkcg_punt_bio_init);
#endif /* CONFIG_BLK_CGROUP_PUNT_BIO */
/**
* bio_blkcg_css - return the blkcg CSS associated with a bio
* @bio: target bio
*
* This returns the CSS for the blkcg associated with a bio, or %NULL if not
* associated. Callers are expected to either handle %NULL or know association
* has been done prior to calling this.
*/
struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)
{
if (!bio || !bio->bi_blkg)
return NULL;
return &bio->bi_blkg->blkcg->css;
}
EXPORT_SYMBOL_GPL(bio_blkcg_css);
/**
* blkcg_parent - get the parent of a blkcg
* @blkcg: blkcg of interest
*
* Return the parent blkcg of @blkcg. Can be called anytime.
*/
static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
{
return css_to_blkcg(blkcg->css.parent);
}
/**
* blkg_alloc - allocate a blkg
* @blkcg: block cgroup the new blkg is associated with
* @disk: gendisk the new blkg is associated with
* @gfp_mask: allocation mask to use
*
* Allocate a new blkg associating @blkcg and @disk.
*/
static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
gfp_t gfp_mask)
{
struct blkcg_gq *blkg;
int i, cpu;
/* alloc and init base part */
blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node);
if (!blkg)
return NULL;
if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
goto out_free_blkg;
blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
if (!blkg->iostat_cpu)
goto out_exit_refcnt;
if (!blk_get_queue(disk->queue))
goto out_free_iostat;
blkg->q = disk->queue;
INIT_LIST_HEAD(&blkg->q_node);
blkg->blkcg = blkcg;
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
spin_lock_init(&blkg->async_bio_lock);
bio_list_init(&blkg->async_bios);
INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
#endif
u64_stats_init(&blkg->iostat.sync);
for_each_possible_cpu(cpu) {
u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
}
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
struct blkg_policy_data *pd;
if (!blkcg_policy_enabled(disk->queue, pol))
continue;
/* alloc per-policy data and attach it to blkg */
pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);
if (!pd)
goto out_free_pds;
blkg->pd[i] = pd;
pd->blkg = blkg;
pd->plid = i;
pd->online = false;
}
return blkg;
out_free_pds:
while (--i >= 0)
if (blkg->pd[i])
blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
blk_put_queue(disk->queue);
out_free_iostat:
free_percpu(blkg->iostat_cpu);
out_exit_refcnt:
percpu_ref_exit(&blkg->refcnt);
out_free_blkg:
kfree(blkg);
return NULL;
}
/*
* If @new_blkg is %NULL, this function tries to allocate a new one as
* necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
*/
static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,
struct blkcg_gq *new_blkg)
{
struct blkcg_gq *blkg;
int i, ret;
lockdep_assert_held(&disk->queue->queue_lock);
/* request_queue is dying, do not create/recreate a blkg */
if (blk_queue_dying(disk->queue)) {
ret = -ENODEV;
goto err_free_blkg;
}
/* blkg holds a reference to blkcg */
if (!css_tryget_online(&blkcg->css)) {
ret = -ENODEV;
goto err_free_blkg;
}
/* allocate */
if (!new_blkg) {
new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN);
if (unlikely(!new_blkg)) {
ret = -ENOMEM;
goto err_put_css;
}
}
blkg = new_blkg;
/* link parent */
if (blkcg_parent(blkcg)) {
blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);
if (WARN_ON_ONCE(!blkg->parent)) {
ret = -ENODEV;
goto err_put_css;
}
blkg_get(blkg->parent);
}
/* invoke per-policy init */
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
if (blkg->pd[i] && pol->pd_init_fn)
pol->pd_init_fn(blkg->pd[i]);
}
/* insert */
spin_lock(&blkcg->lock);
ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);
if (likely(!ret)) {
hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
list_add(&blkg->q_node, &disk->queue->blkg_list);
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
if (blkg->pd[i]) {
if (pol->pd_online_fn)
pol->pd_online_fn(blkg->pd[i]);
blkg->pd[i]->online = true;
}
}
}
blkg->online = true;
spin_unlock(&blkcg->lock);
if (!ret)
return blkg;
/* @blkg failed fully initialized, use the usual release path */
blkg_put(blkg);
return ERR_PTR(ret);
err_put_css:
css_put(&blkcg->css);
err_free_blkg:
if (new_blkg)
blkg_free(new_blkg);
return ERR_PTR(ret);
}
/**
* blkg_lookup_create - lookup blkg, try to create one if not there
* @blkcg: blkcg of interest
* @disk: gendisk of interest
*
* Lookup blkg for the @blkcg - @disk pair. If it doesn't exist, try to
* create one. blkg creation is performed recursively from blkcg_root such
* that all non-root blkg's have access to the parent blkg. This function
* should be called under RCU read lock and takes @disk->queue->queue_lock.
*
* Returns the blkg or the closest blkg if blkg_create() fails as it walks
* down from root.
*/
static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
struct gendisk *disk)
{
struct request_queue *q = disk->queue;
struct blkcg_gq *blkg;
unsigned long flags;
WARN_ON_ONCE(!rcu_read_lock_held());
blkg = blkg_lookup(blkcg, q);
if (blkg)
return blkg;
spin_lock_irqsave(&q->queue_lock, flags);
blkg = blkg_lookup(blkcg, q);
if (blkg) {
if (blkcg != &blkcg_root &&
blkg != rcu_dereference(blkcg->blkg_hint))
rcu_assign_pointer(blkcg->blkg_hint, blkg);
goto found;
}
/*
* Create blkgs walking down from blkcg_root to @blkcg, so that all
* non-root blkgs have access to their parents. Returns the closest
* blkg to the intended blkg should blkg_create() fail.
*/
while (true) {
struct blkcg *pos = blkcg;
struct blkcg *parent = blkcg_parent(blkcg);
struct blkcg_gq *ret_blkg = q->root_blkg;
while (parent) {
blkg = blkg_lookup(parent, q);
if (blkg) {
/* remember closest blkg */
ret_blkg = blkg;
break;
}
pos = parent;
parent = blkcg_parent(parent);
}
blkg = blkg_create(pos, disk, NULL);
if (IS_ERR(blkg)) {
blkg = ret_blkg;
break;
}
if (pos == blkcg)
break;
}
found:
spin_unlock_irqrestore(&q->queue_lock, flags);
return blkg;
}
static void blkg_destroy(struct blkcg_gq *blkg)
{
struct blkcg *blkcg = blkg->blkcg;
int i;
lockdep_assert_held(&blkg->q->queue_lock);
lockdep_assert_held(&blkcg->lock);
/*
* blkg stays on the queue list until blkg_free_workfn(), see details in
* blkg_free_workfn(), hence this function can be called from
* blkcg_destroy_blkgs() first and again from blkg_destroy_all() before
* blkg_free_workfn().
*/
if (hlist_unhashed(&blkg->blkcg_node))
return;
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
if (blkg->pd[i] && blkg->pd[i]->online) {
blkg->pd[i]->online = false;
if (pol->pd_offline_fn)
pol->pd_offline_fn(blkg->pd[i]);
}
}
blkg->online = false;
radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
hlist_del_init_rcu(&blkg->blkcg_node);
/*
* Both setting lookup hint to and clearing it from @blkg are done
* under queue_lock. If it's not pointing to @blkg now, it never
* will. Hint assignment itself can race safely.
*/
if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
rcu_assign_pointer(blkcg->blkg_hint, NULL);
/*
* Put the reference taken at the time of creation so that when all
* queues are gone, group can be destroyed.
*/
percpu_ref_kill(&blkg->refcnt);
}
static void blkg_destroy_all(struct gendisk *disk)
{
struct request_queue *q = disk->queue;
struct blkcg_gq *blkg;
int count = BLKG_DESTROY_BATCH_SIZE;
int i;
restart:
spin_lock_irq(&q->queue_lock);
list_for_each_entry(blkg, &q->blkg_list, q_node) {
struct blkcg *blkcg = blkg->blkcg;
if (hlist_unhashed(&blkg->blkcg_node))
continue;
spin_lock(&blkcg->lock);
blkg_destroy(blkg);
spin_unlock(&blkcg->lock);
/*
* in order to avoid holding the spin lock for too long, release
* it when a batch of blkgs are destroyed.
*/
if (!(--count)) {
count = BLKG_DESTROY_BATCH_SIZE;
spin_unlock_irq(&q->queue_lock);
cond_resched();
goto restart;
}
}
/*
* Mark policy deactivated since policy offline has been done, and
* the free is scheduled, so future blkcg_deactivate_policy() can
* be bypassed
*/
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
if (pol)
__clear_bit(pol->plid, q->blkcg_pols);
}
q->root_blkg = NULL;
spin_unlock_irq(&q->queue_lock);
}
static int blkcg_reset_stats(struct cgroup_subsys_state *css,
struct cftype *cftype, u64 val)
{
struct blkcg *blkcg = css_to_blkcg(css);
struct blkcg_gq *blkg;
int i, cpu;
mutex_lock(&blkcg_pol_mutex);
spin_lock_irq(&blkcg->lock);
/*
* Note that stat reset is racy - it doesn't synchronize against
* stat updates. This is a debug feature which shouldn't exist
* anyway. If you get hit by a race, retry.
*/
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
for_each_possible_cpu(cpu) {
struct blkg_iostat_set *bis =
per_cpu_ptr(blkg->iostat_cpu, cpu);
memset(bis, 0, sizeof(*bis));
/* Re-initialize the cleared blkg_iostat_set */
u64_stats_init(&bis->sync);
bis->blkg = blkg;
}
memset(&blkg->iostat, 0, sizeof(blkg->iostat));
u64_stats_init(&blkg->iostat.sync);
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
if (blkg->pd[i] && pol->pd_reset_stats_fn)
pol->pd_reset_stats_fn(blkg->pd[i]);
}
}
spin_unlock_irq(&blkcg->lock);
mutex_unlock(&blkcg_pol_mutex);
return 0;
}
const char *blkg_dev_name(struct blkcg_gq *blkg)
{
if (!blkg->q->disk)
return NULL;
return bdi_dev_name(blkg->q->disk->bdi);
}
/**
* blkcg_print_blkgs - helper for printing per-blkg data
* @sf: seq_file to print to
* @blkcg: blkcg of interest
* @prfill: fill function to print out a blkg
* @pol: policy in question
* @data: data to be passed to @prfill
* @show_total: to print out sum of prfill return values or not
*
* This function invokes @prfill on each blkg of @blkcg if pd for the
* policy specified by @pol exists. @prfill is invoked with @sf, the
* policy data and @data and the matching queue lock held. If @show_total
* is %true, the sum of the return values from @prfill is printed with
* "Total" label at the end.
*
* This is to be used to construct print functions for
* cftype->read_seq_string method.
*/
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
u64 (*prfill)(struct seq_file *,
struct blkg_policy_data *, int),
const struct blkcg_policy *pol, int data,
bool show_total)
{
struct blkcg_gq *blkg;
u64 total = 0;
rcu_read_lock();
hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
spin_lock_irq(&blkg->q->queue_lock);
if (blkcg_policy_enabled(blkg->q, pol))
total += prfill(sf, blkg->pd[pol->plid], data);
spin_unlock_irq(&blkg->q->queue_lock);
}
rcu_read_unlock();
if (show_total)
seq_printf(sf, "Total %llu\n", (unsigned long long)total);
}
EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
/**
* __blkg_prfill_u64 - prfill helper for a single u64 value
* @sf: seq_file to print to
* @pd: policy private data of interest
* @v: value to print
*
* Print @v to @sf for the device associated with @pd.
*/
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
{
const char *dname = blkg_dev_name(pd->blkg);
if (!dname)
return 0;
seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
return v;
}
EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
/**
* blkg_conf_init - initialize a blkg_conf_ctx
* @ctx: blkg_conf_ctx to initialize
* @input: input string
*
* Initialize @ctx which can be used to parse blkg config input string @input.
* Once initialized, @ctx can be used with blkg_conf_open_bdev() and
* blkg_conf_prep(), and must be cleaned up with blkg_conf_exit().
*/
void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input)
{
*ctx = (struct blkg_conf_ctx){ .input = input };
}
EXPORT_SYMBOL_GPL(blkg_conf_init);
/**
* blkg_conf_open_bdev - parse and open bdev for per-blkg config update
* @ctx: blkg_conf_ctx initialized with blkg_conf_init()
*
* Parse the device node prefix part, MAJ:MIN, of per-blkg config update from
* @ctx->input and get and store the matching bdev in @ctx->bdev. @ctx->body is
* set to point past the device node prefix.
*
* This function may be called multiple times on @ctx and the extra calls become
* NOOPs. blkg_conf_prep() implicitly calls this function. Use this function
* explicitly if bdev access is needed without resolving the blkcg / policy part
* of @ctx->input. Returns -errno on error.
*/
int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx)
{
char *input = ctx->input;
unsigned int major, minor;
struct block_device *bdev;
int key_len;
if (ctx->bdev)
return 0;
if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
return -EINVAL;
input += key_len;
if (!isspace(*input))
return -EINVAL;
input = skip_spaces(input);
bdev = blkdev_get_no_open(MKDEV(major, minor));
if (!bdev)
return -ENODEV;
if (bdev_is_partition(bdev)) {
blkdev_put_no_open(bdev);
return -ENODEV;
}
mutex_lock(&bdev->bd_queue->rq_qos_mutex);
if (!disk_live(bdev->bd_disk)) {
blkdev_put_no_open(bdev);
mutex_unlock(&bdev->bd_queue->rq_qos_mutex);
return -ENODEV;
}
ctx->body = input;
ctx->bdev = bdev;
return 0;
}
/**
* blkg_conf_prep - parse and prepare for per-blkg config update
* @blkcg: target block cgroup
* @pol: target policy
* @ctx: blkg_conf_ctx initialized with blkg_conf_init()
*
* Parse per-blkg config update from @ctx->input and initialize @ctx
* accordingly. On success, @ctx->body points to the part of @ctx->input
* following MAJ:MIN, @ctx->bdev points to the target block device and
* @ctx->blkg to the blkg being configured.
*
* blkg_conf_open_bdev() may be called on @ctx beforehand. On success, this
* function returns with queue lock held and must be followed by
* blkg_conf_exit().
*/
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
struct blkg_conf_ctx *ctx)
__acquires(&bdev->bd_queue->queue_lock)
{
struct gendisk *disk;
struct request_queue *q;
struct blkcg_gq *blkg;
int ret;
ret = blkg_conf_open_bdev(ctx);
if (ret)
return ret;
disk = ctx->bdev->bd_disk;
q = disk->queue;
/*
* blkcg_deactivate_policy() requires queue to be frozen, we can grab
* q_usage_counter to prevent concurrent with blkcg_deactivate_policy().
*/
ret = blk_queue_enter(q, 0);
if (ret)
goto fail;
spin_lock_irq(&q->queue_lock);
if (!blkcg_policy_enabled(q, pol)) {
ret = -EOPNOTSUPP;
goto fail_unlock;
}
blkg = blkg_lookup(blkcg, q);
if (blkg)
goto success;
/*
* Create blkgs walking down from blkcg_root to @blkcg, so that all
* non-root blkgs have access to their parents.
*/
while (true) {
struct blkcg *pos = blkcg;
struct blkcg *parent;
struct blkcg_gq *new_blkg;
parent = blkcg_parent(blkcg);
while (parent && !blkg_lookup(parent, q)) {
pos = parent;
parent = blkcg_parent(parent);
}
/* Drop locks to do new blkg allocation with GFP_KERNEL. */
spin_unlock_irq(&q->queue_lock);
new_blkg = blkg_alloc(pos, disk, GFP_KERNEL);
if (unlikely(!new_blkg)) {
ret = -ENOMEM;
goto fail_exit_queue;
}
if (radix_tree_preload(GFP_KERNEL)) {
blkg_free(new_blkg);
ret = -ENOMEM;
goto fail_exit_queue;
}
spin_lock_irq(&q->queue_lock);
if (!blkcg_policy_enabled(q, pol)) {
blkg_free(new_blkg);
ret = -EOPNOTSUPP;
goto fail_preloaded;
}
blkg = blkg_lookup(pos, q);
if (blkg) {
blkg_free(new_blkg);
} else {
blkg = blkg_create(pos, disk, new_blkg);
if (IS_ERR(blkg)) {
ret = PTR_ERR(blkg);
goto fail_preloaded;
}
}
radix_tree_preload_end();
if (pos == blkcg)
goto success;
}
success:
blk_queue_exit(q);
ctx->blkg = blkg;
return 0;
fail_preloaded:
radix_tree_preload_end();
fail_unlock:
spin_unlock_irq(&q->queue_lock);
fail_exit_queue:
blk_queue_exit(q);
fail:
/*
* If queue was bypassing, we should retry. Do so after a
* short msleep(). It isn't strictly necessary but queue
* can be bypassing for some time and it's always nice to
* avoid busy looping.
*/
if (ret == -EBUSY) {
msleep(10);
ret = restart_syscall();
}
return ret;
}
EXPORT_SYMBOL_GPL(blkg_conf_prep);
/**
* blkg_conf_exit - clean up per-blkg config update
* @ctx: blkg_conf_ctx initialized with blkg_conf_init()
*
* Clean up after per-blkg config update. This function must be called on all
* blkg_conf_ctx's initialized with blkg_conf_init().
*/
void blkg_conf_exit(struct blkg_conf_ctx *ctx)
__releases(&ctx->bdev->bd_queue->queue_lock)
__releases(&ctx->bdev->bd_queue->rq_qos_mutex)
{
if (ctx->blkg) {
spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock);
ctx->blkg = NULL;
}
if (ctx->bdev) {
mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex);
blkdev_put_no_open(ctx->bdev);
ctx->body = NULL;
ctx->bdev = NULL;
}
}
EXPORT_SYMBOL_GPL(blkg_conf_exit);
static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
{
int i;
for (i = 0; i < BLKG_IOSTAT_NR; i++) {
dst->bytes[i] = src->bytes[i];
dst->ios[i] = src->ios[i];
}
}
static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
{
int i;
for (i = 0; i < BLKG_IOSTAT_NR; i++) {
dst->bytes[i] += src->bytes[i];
dst->ios[i] += src->ios[i];
}
}
static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
{
int i;
for (i = 0; i < BLKG_IOSTAT_NR; i++) {
dst->bytes[i] -= src->bytes[i];
dst->ios[i] -= src->ios[i];
}
}
static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
struct blkg_iostat *last)
{
struct blkg_iostat delta;
unsigned long flags;
/* propagate percpu delta to global */
flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
blkg_iostat_set(&delta, cur);
blkg_iostat_sub(&delta, last);
blkg_iostat_add(&blkg->iostat.cur, &delta);
blkg_iostat_add(last, &delta);
u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
}
static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
{
struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
struct llist_node *lnode;
struct blkg_iostat_set *bisc, *next_bisc;
unsigned long flags;
rcu_read_lock();
lnode = llist_del_all(lhead);
if (!lnode)
goto out;
/*
* For covering concurrent parent blkg update from blkg_release().
*
* When flushing from cgroup, cgroup_rstat_lock is always held, so
* this lock won't cause contention most of time.
*/
raw_spin_lock_irqsave(&blkg_stat_lock, flags);
/*
* Iterate only the iostat_cpu's queued in the lockless list.
*/
llist_for_each_entry_safe(bisc, next_bisc, lnode, lnode) {
struct blkcg_gq *blkg = bisc->blkg;
struct blkcg_gq *parent = blkg->parent;
struct blkg_iostat cur;
unsigned int seq;
WRITE_ONCE(bisc->lqueued, false);
/* fetch the current per-cpu values */
do {
seq = u64_stats_fetch_begin(&bisc->sync);
blkg_iostat_set(&cur, &bisc->cur);
} while (u64_stats_fetch_retry(&bisc->sync, seq));
blkcg_iostat_update(blkg, &cur, &bisc->last);
/* propagate global delta to parent (unless that's root) */
if (parent && parent->parent)
blkcg_iostat_update(parent, &blkg->iostat.cur,
&blkg->iostat.last);
}
raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
out:
rcu_read_unlock();
}
static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
{
/* Root-level stats are sourced from system-wide IO stats */
if (cgroup_parent(css->cgroup))
__blkcg_rstat_flush(css_to_blkcg(css), cpu);
}
/*
* We source root cgroup stats from the system-wide stats to avoid
* tracking the same information twice and incurring overhead when no
* cgroups are defined. For that reason, cgroup_rstat_flush in
* blkcg_print_stat does not actually fill out the iostat in the root
* cgroup's blkcg_gq.
*
* However, we would like to re-use the printing code between the root and
* non-root cgroups to the extent possible. For that reason, we simulate
* flushing the root cgroup's stats by explicitly filling in the iostat
* with disk level statistics.
*/
static void blkcg_fill_root_iostats(void)
{
struct class_dev_iter iter;
struct device *dev;
class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
while ((dev = class_dev_iter_next(&iter))) {
struct block_device *bdev = dev_to_bdev(dev);
struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg;
struct blkg_iostat tmp;
int cpu;
unsigned long flags;
memset(&tmp, 0, sizeof(tmp));
for_each_possible_cpu(cpu) {
struct disk_stats *cpu_dkstats;
cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
tmp.ios[BLKG_IOSTAT_READ] +=
cpu_dkstats->ios[STAT_READ];
tmp.ios[BLKG_IOSTAT_WRITE] +=
cpu_dkstats->ios[STAT_WRITE];
tmp.ios[BLKG_IOSTAT_DISCARD] +=
cpu_dkstats->ios[STAT_DISCARD];
// convert sectors to bytes
tmp.bytes[BLKG_IOSTAT_READ] +=
cpu_dkstats->sectors[STAT_READ] << 9;
tmp.bytes[BLKG_IOSTAT_WRITE] +=
cpu_dkstats->sectors[STAT_WRITE] << 9;
tmp.bytes[BLKG_IOSTAT_DISCARD] +=
cpu_dkstats->sectors[STAT_DISCARD] << 9;
}
flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
blkg_iostat_set(&blkg->iostat.cur, &tmp);
u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
}
}
static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
{
struct blkg_iostat_set *bis = &blkg->iostat;
u64 rbytes, wbytes, rios, wios, dbytes, dios;
const char *dname;
unsigned seq;
int i;
if (!blkg->online)
return;
dname = blkg_dev_name(blkg);
if (!dname)
return;
seq_printf(s, "%s ", dname);
do {
seq = u64_stats_fetch_begin(&bis->sync);
rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
rios = bis->cur.ios[BLKG_IOSTAT_READ];
wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
} while (u64_stats_fetch_retry(&bis->sync, seq));
if (rbytes || wbytes || rios || wios) {
seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
rbytes, wbytes, rios, wios,
dbytes, dios);
}
if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
seq_printf(s, " use_delay=%d delay_nsec=%llu",
atomic_read(&blkg->use_delay),
atomic64_read(&blkg->delay_nsec));
}
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
if (!blkg->pd[i] || !pol->pd_stat_fn)
continue;
pol->pd_stat_fn(blkg->pd[i], s);
}
seq_puts(s, "\n");
}
static int blkcg_print_stat(struct seq_file *sf, void *v)
{
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
struct blkcg_gq *blkg;
if (!seq_css(sf)->parent)
blkcg_fill_root_iostats();
else
cgroup_rstat_flush(blkcg->css.cgroup);
rcu_read_lock();
hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
spin_lock_irq(&blkg->q->queue_lock);
blkcg_print_one_stat(blkg, sf);
spin_unlock_irq(&blkg->q->queue_lock);
}
rcu_read_unlock();
return 0;
}
static struct cftype blkcg_files[] = {
{
.name = "stat",
.seq_show = blkcg_print_stat,
},
{ } /* terminate */
};
static struct cftype blkcg_legacy_files[] = {
{
.name = "reset_stats",
.write_u64 = blkcg_reset_stats,
},
{ } /* terminate */
};
#ifdef CONFIG_CGROUP_WRITEBACK
struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css)
{
return &css_to_blkcg(css)->cgwb_list;
}
#endif
/*
* blkcg destruction is a three-stage process.
*
* 1. Destruction starts. The blkcg_css_offline() callback is invoked
* which offlines writeback. Here we tie the next stage of blkg destruction
* to the completion of writeback associated with the blkcg. This lets us
* avoid punting potentially large amounts of outstanding writeback to root
* while maintaining any ongoing policies. The next stage is triggered when
* the nr_cgwbs count goes to zero.
*
* 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
* and handles the destruction of blkgs. Here the css reference held by
* the blkg is put back eventually allowing blkcg_css_free() to be called.
* This work may occur in cgwb_release_workfn() on the cgwb_release
* workqueue. Any submitted ios that fail to get the blkg ref will be
* punted to the root_blkg.
*
* 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
* This finally frees the blkcg.
*/
/**
* blkcg_destroy_blkgs - responsible for shooting down blkgs
* @blkcg: blkcg of interest
*
* blkgs should be removed while holding both q and blkcg locks. As blkcg lock
* is nested inside q lock, this function performs reverse double lock dancing.
* Destroying the blkgs releases the reference held on the blkcg's css allowing
* blkcg_css_free to eventually be called.
*
* This is the blkcg counterpart of ioc_release_fn().
*/
static void blkcg_destroy_blkgs(struct blkcg *blkcg)
{
might_sleep();
spin_lock_irq(&blkcg->lock);
while (!hlist_empty(&blkcg->blkg_list)) {
struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
struct blkcg_gq, blkcg_node);
struct request_queue *q = blkg->q;
if (need_resched() || !spin_trylock(&q->queue_lock)) {
/*
* Given that the system can accumulate a huge number
* of blkgs in pathological cases, check to see if we
* need to rescheduling to avoid softlockup.
*/
spin_unlock_irq(&blkcg->lock);
cond_resched();
spin_lock_irq(&blkcg->lock);
continue;
}
blkg_destroy(blkg);
spin_unlock(&q->queue_lock);
}
spin_unlock_irq(&blkcg->lock);
}
/**
* blkcg_pin_online - pin online state
* @blkcg_css: blkcg of interest
*
* While pinned, a blkcg is kept online. This is primarily used to
* impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline
* while an associated cgwb is still active.
*/
void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css)
{
refcount_inc(&css_to_blkcg(blkcg_css)->online_pin);
}
/**
* blkcg_unpin_online - unpin online state
* @blkcg_css: blkcg of interest
*
* This is primarily used to impedance-match blkg and cgwb lifetimes so
* that blkg doesn't go offline while an associated cgwb is still active.
* When this count goes to zero, all active cgwbs have finished so the
* blkcg can continue destruction by calling blkcg_destroy_blkgs().
*/
void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css)
{
struct blkcg *blkcg = css_to_blkcg(blkcg_css);
do {
if (!refcount_dec_and_test(&blkcg->online_pin))
break;
blkcg_destroy_blkgs(blkcg);
blkcg = blkcg_parent(blkcg);
} while (blkcg);
}
/**
* blkcg_css_offline - cgroup css_offline callback
* @css: css of interest
*
* This function is called when @css is about to go away. Here the cgwbs are
* offlined first and only once writeback associated with the blkcg has
* finished do we start step 2 (see above).
*/
static void blkcg_css_offline(struct cgroup_subsys_state *css)
{
/* this prevents anyone from attaching or migrating to this blkcg */
wb_blkcg_offline(css);
/* put the base online pin allowing step 2 to be triggered */
blkcg_unpin_online(css);
}
static void blkcg_css_free(struct cgroup_subsys_state *css)
{
struct blkcg *blkcg = css_to_blkcg(css);
int i;
mutex_lock(&blkcg_pol_mutex);
list_del(&blkcg->all_blkcgs_node);
for (i = 0; i < BLKCG_MAX_POLS; i++)
if (blkcg->cpd[i])
blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
mutex_unlock(&blkcg_pol_mutex);
free_percpu(blkcg->lhead);
kfree(blkcg);
}
static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
{
struct blkcg *blkcg;
int i;
mutex_lock(&blkcg_pol_mutex);
if (!parent_css) {
blkcg = &blkcg_root;
} else {
blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
if (!blkcg)
goto unlock;
}
if (init_blkcg_llists(blkcg))
goto free_blkcg;
for (i = 0; i < BLKCG_MAX_POLS ; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
struct blkcg_policy_data *cpd;
/*
* If the policy hasn't been attached yet, wait for it
* to be attached before doing anything else. Otherwise,
* check if the policy requires any specific per-cgroup
* data: if it does, allocate and initialize it.
*/
if (!pol || !pol->cpd_alloc_fn)
continue;
cpd = pol->cpd_alloc_fn(GFP_KERNEL);
if (!cpd)
goto free_pd_blkcg;
blkcg->cpd[i] = cpd;
cpd->blkcg = blkcg;
cpd->plid = i;
}
spin_lock_init(&blkcg->lock);
refcount_set(&blkcg->online_pin, 1);
INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
INIT_HLIST_HEAD(&blkcg->blkg_list);
#ifdef CONFIG_CGROUP_WRITEBACK
INIT_LIST_HEAD(&blkcg->cgwb_list);
#endif
list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
mutex_unlock(&blkcg_pol_mutex);
return &blkcg->css;
free_pd_blkcg:
for (i--; i >= 0; i--)
if (blkcg->cpd[i])
blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
free_percpu(blkcg->lhead);
free_blkcg:
if (blkcg != &blkcg_root)
kfree(blkcg);
unlock:
mutex_unlock(&blkcg_pol_mutex);
return ERR_PTR(-ENOMEM);
}
static int blkcg_css_online(struct cgroup_subsys_state *css)
{
struct blkcg *parent = blkcg_parent(css_to_blkcg(css));
/*
* blkcg_pin_online() is used to delay blkcg offline so that blkgs
* don't go offline while cgwbs are still active on them. Pin the
* parent so that offline always happens towards the root.
*/
if (parent)
blkcg_pin_online(&parent->css);
return 0;
}
void blkg_init_queue(struct request_queue *q)
{
INIT_LIST_HEAD(&q->blkg_list);
mutex_init(&q->blkcg_mutex);
}
int blkcg_init_disk(struct gendisk *disk)
{
struct request_queue *q = disk->queue;
struct blkcg_gq *new_blkg, *blkg;
bool preloaded;
int ret;
new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL);
if (!new_blkg)
return -ENOMEM;
preloaded = !radix_tree_preload(GFP_KERNEL);
/* Make sure the root blkg exists. */
/* spin_lock_irq can serve as RCU read-side critical section. */
spin_lock_irq(&q->queue_lock);
blkg = blkg_create(&blkcg_root, disk, new_blkg);
if (IS_ERR(blkg))
goto err_unlock;
q->root_blkg = blkg;
spin_unlock_irq(&q->queue_lock);
if (preloaded)
radix_tree_preload_end();
ret = blk_ioprio_init(disk);
if (ret)
goto err_destroy_all;
ret = blk_throtl_init(disk);
if (ret)
goto err_ioprio_exit;
return 0;
err_ioprio_exit:
blk_ioprio_exit(disk);
err_destroy_all:
blkg_destroy_all(disk);
return ret;
err_unlock:
spin_unlock_irq(&q->queue_lock);
if (preloaded)
radix_tree_preload_end();
return PTR_ERR(blkg);
}
void blkcg_exit_disk(struct gendisk *disk)
{
blkg_destroy_all(disk);
blk_throtl_exit(disk);
}
static void blkcg_exit(struct task_struct *tsk)
{
if (tsk->throttle_disk)
put_disk(tsk->throttle_disk);
tsk->throttle_disk = NULL;
}
struct cgroup_subsys io_cgrp_subsys = {
.css_alloc = blkcg_css_alloc,
.css_online = blkcg_css_online,
.css_offline = blkcg_css_offline,
.css_free = blkcg_css_free,
.css_rstat_flush = blkcg_rstat_flush,
.dfl_cftypes = blkcg_files,
.legacy_cftypes = blkcg_legacy_files,
.legacy_name = "blkio",
.exit = blkcg_exit,
#ifdef CONFIG_MEMCG
/*
* This ensures that, if available, memcg is automatically enabled
* together on the default hierarchy so that the owner cgroup can
* be retrieved from writeback pages.
*/
.depends_on = 1 << memory_cgrp_id,
#endif
};
EXPORT_SYMBOL_GPL(io_cgrp_subsys);
/**
* blkcg_activate_policy - activate a blkcg policy on a gendisk
* @disk: gendisk of interest
* @pol: blkcg policy to activate
*
* Activate @pol on @disk. Requires %GFP_KERNEL context. @disk goes through
* bypass mode to populate its blkgs with policy_data for @pol.
*
* Activation happens with @disk bypassed, so nobody would be accessing blkgs
* from IO path. Update of each blkg is protected by both queue and blkcg
* locks so that holding either lock and testing blkcg_policy_enabled() is
* always enough for dereferencing policy data.
*
* The caller is responsible for synchronizing [de]activations and policy
* [un]registerations. Returns 0 on success, -errno on failure.
*/
int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol)
{
struct request_queue *q = disk->queue;
struct blkg_policy_data *pd_prealloc = NULL;
struct blkcg_gq *blkg, *pinned_blkg = NULL;
int ret;
if (blkcg_policy_enabled(q, pol))
return 0;
if (queue_is_mq(q))
blk_mq_freeze_queue(q);
retry:
spin_lock_irq(&q->queue_lock);
/* blkg_list is pushed at the head, reverse walk to initialize parents first */
list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
struct blkg_policy_data *pd;
if (blkg->pd[pol->plid])
continue;
/* If prealloc matches, use it; otherwise try GFP_NOWAIT */
if (blkg == pinned_blkg) {
pd = pd_prealloc;
pd_prealloc = NULL;
} else {
pd = pol->pd_alloc_fn(disk, blkg->blkcg,
GFP_NOWAIT | __GFP_NOWARN);
}
if (!pd) {
/*
* GFP_NOWAIT failed. Free the existing one and
* prealloc for @blkg w/ GFP_KERNEL.
*/
if (pinned_blkg)
blkg_put(pinned_blkg);
blkg_get(blkg);
pinned_blkg = blkg;
spin_unlock_irq(&q->queue_lock);
if (pd_prealloc)
pol->pd_free_fn(pd_prealloc);
pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,
GFP_KERNEL);
if (pd_prealloc)
goto retry;
else
goto enomem;
}
spin_lock(&blkg->blkcg->lock);
pd->blkg = blkg;
pd->plid = pol->plid;
blkg->pd[pol->plid] = pd;
if (pol->pd_init_fn)
pol->pd_init_fn(pd);
if (pol->pd_online_fn)
pol->pd_online_fn(pd);
pd->online = true;
spin_unlock(&blkg->blkcg->lock);
}
__set_bit(pol->plid, q->blkcg_pols);
ret = 0;
spin_unlock_irq(&q->queue_lock);
out:
if (queue_is_mq(q))
blk_mq_unfreeze_queue(q);
if (pinned_blkg)
blkg_put(pinned_blkg);
if (pd_prealloc)
pol->pd_free_fn(pd_prealloc);
return ret;
enomem:
/* alloc failed, take down everything */
spin_lock_irq(&q->queue_lock);
list_for_each_entry(blkg, &q->blkg_list, q_node) {
struct blkcg *blkcg = blkg->blkcg;
struct blkg_policy_data *pd;
spin_lock(&blkcg->lock);
pd = blkg->pd[pol->plid];
if (pd) {
if (pd->online && pol->pd_offline_fn)
pol->pd_offline_fn(pd);
pd->online = false;
pol->pd_free_fn(pd);
blkg->pd[pol->plid] = NULL;
}
spin_unlock(&blkcg->lock);
}
spin_unlock_irq(&q->queue_lock);
ret = -ENOMEM;
goto out;
}
EXPORT_SYMBOL_GPL(blkcg_activate_policy);
/**
* blkcg_deactivate_policy - deactivate a blkcg policy on a gendisk
* @disk: gendisk of interest
* @pol: blkcg policy to deactivate
*
* Deactivate @pol on @disk. Follows the same synchronization rules as
* blkcg_activate_policy().
*/
void blkcg_deactivate_policy(struct gendisk *disk,
const struct blkcg_policy *pol)
{
struct request_queue *q = disk->queue;
struct blkcg_gq *blkg;
if (!blkcg_policy_enabled(q, pol))
return;
if (queue_is_mq(q))
blk_mq_freeze_queue(q);
mutex_lock(&q->blkcg_mutex);
spin_lock_irq(&q->queue_lock);
__clear_bit(pol->plid, q->blkcg_pols);
list_for_each_entry(blkg, &q->blkg_list, q_node) {
struct blkcg *blkcg = blkg->blkcg;
spin_lock(&blkcg->lock);
if (blkg->pd[pol->plid]) {
if (blkg->pd[pol->plid]->online && pol->pd_offline_fn)
pol->pd_offline_fn(blkg->pd[pol->plid]);
pol->pd_free_fn(blkg->pd[pol->plid]);
blkg->pd[pol->plid] = NULL;
}
spin_unlock(&blkcg->lock);
}
spin_unlock_irq(&q->queue_lock);
mutex_unlock(&q->blkcg_mutex);
if (queue_is_mq(q))
blk_mq_unfreeze_queue(q);
}
EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
static void blkcg_free_all_cpd(struct blkcg_policy *pol)
{
struct blkcg *blkcg;
list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
if (blkcg->cpd[pol->plid]) {
pol->cpd_free_fn(blkcg->cpd[pol->plid]);
blkcg->cpd[pol->plid] = NULL;
}
}
}
/**
* blkcg_policy_register - register a blkcg policy
* @pol: blkcg policy to register
*
* Register @pol with blkcg core. Might sleep and @pol may be modified on
* successful registration. Returns 0 on success and -errno on failure.
*/
int blkcg_policy_register(struct blkcg_policy *pol)
{
struct blkcg *blkcg;
int i, ret;
mutex_lock(&blkcg_pol_register_mutex);
mutex_lock(&blkcg_pol_mutex);
/* find an empty slot */
ret = -ENOSPC;
for (i = 0; i < BLKCG_MAX_POLS; i++)
if (!blkcg_policy[i])
break;
if (i >= BLKCG_MAX_POLS) {
pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
goto err_unlock;
}
/* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
(!pol->pd_alloc_fn ^ !pol->pd_free_fn))
goto err_unlock;
/* register @pol */
pol->plid = i;
blkcg_policy[pol->plid] = pol;
/* allocate and install cpd's */
if (pol->cpd_alloc_fn) {
list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
struct blkcg_policy_data *cpd;
cpd = pol->cpd_alloc_fn(GFP_KERNEL);
if (!cpd)
goto err_free_cpds;
blkcg->cpd[pol->plid] = cpd;
cpd->blkcg = blkcg;
cpd->plid = pol->plid;
}
}
mutex_unlock(&blkcg_pol_mutex);
/* everything is in place, add intf files for the new policy */
if (pol->dfl_cftypes)
WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
pol->dfl_cftypes));
if (pol->legacy_cftypes)
WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
pol->legacy_cftypes));
mutex_unlock(&blkcg_pol_register_mutex);
return 0;
err_free_cpds:
if (pol->cpd_free_fn)
blkcg_free_all_cpd(pol);
blkcg_policy[pol->plid] = NULL;
err_unlock:
mutex_unlock(&blkcg_pol_mutex);
mutex_unlock(&blkcg_pol_register_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(blkcg_policy_register);
/**
* blkcg_policy_unregister - unregister a blkcg policy
* @pol: blkcg policy to unregister
*
* Undo blkcg_policy_register(@pol). Might sleep.
*/
void blkcg_policy_unregister(struct blkcg_policy *pol)
{
mutex_lock(&blkcg_pol_register_mutex);
if (WARN_ON(blkcg_policy[pol->plid] != pol))
goto out_unlock;
/* kill the intf files first */
if (pol->dfl_cftypes)
cgroup_rm_cftypes(pol->dfl_cftypes);
if (pol->legacy_cftypes)
cgroup_rm_cftypes(pol->legacy_cftypes);
/* remove cpds and unregister */
mutex_lock(&blkcg_pol_mutex);
if (pol->cpd_free_fn)
blkcg_free_all_cpd(pol);
blkcg_policy[pol->plid] = NULL;
mutex_unlock(&blkcg_pol_mutex);
out_unlock:
mutex_unlock(&blkcg_pol_register_mutex);
}
EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
/*
* Scale the accumulated delay based on how long it has been since we updated
* the delay. We only call this when we are adding delay, in case it's been a
* while since we added delay, and when we are checking to see if we need to
* delay a task, to account for any delays that may have occurred.
*/
static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
{
u64 old = atomic64_read(&blkg->delay_start);
/* negative use_delay means no scaling, see blkcg_set_delay() */
if (atomic_read(&blkg->use_delay) < 0)
return;
/*
* We only want to scale down every second. The idea here is that we
* want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
* time window. We only want to throttle tasks for recent delay that
* has occurred, in 1 second time windows since that's the maximum
* things can be throttled. We save the current delay window in
* blkg->last_delay so we know what amount is still left to be charged
* to the blkg from this point onward. blkg->last_use keeps track of
* the use_delay counter. The idea is if we're unthrottling the blkg we
* are ok with whatever is happening now, and we can take away more of
* the accumulated delay as we've already throttled enough that
* everybody is happy with their IO latencies.
*/
if (time_before64(old + NSEC_PER_SEC, now) &&
atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) {
u64 cur = atomic64_read(&blkg->delay_nsec);
u64 sub = min_t(u64, blkg->last_delay, now - old);
int cur_use = atomic_read(&blkg->use_delay);
/*
* We've been unthrottled, subtract a larger chunk of our
* accumulated delay.
*/
if (cur_use < blkg->last_use)
sub = max_t(u64, sub, blkg->last_delay >> 1);
/*
* This shouldn't happen, but handle it anyway. Our delay_nsec
* should only ever be growing except here where we subtract out
* min(last_delay, 1 second), but lord knows bugs happen and I'd
* rather not end up with negative numbers.
*/
if (unlikely(cur < sub)) {
atomic64_set(&blkg->delay_nsec, 0);
blkg->last_delay = 0;
} else {
atomic64_sub(sub, &blkg->delay_nsec);
blkg->last_delay = cur - sub;
}
blkg->last_use = cur_use;
}
}
/*
* This is called when we want to actually walk up the hierarchy and check to
* see if we need to throttle, and then actually throttle if there is some
* accumulated delay. This should only be called upon return to user space so
* we're not holding some lock that would induce a priority inversion.
*/
static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
{
unsigned long pflags;
bool clamp;
u64 now = blk_time_get_ns(); u64 exp;
u64 delay_nsec = 0;
int tok;
while (blkg->parent) {
int use_delay = atomic_read(&blkg->use_delay);
if (use_delay) {
u64 this_delay;
blkcg_scale_delay(blkg, now);
this_delay = atomic64_read(&blkg->delay_nsec);
if (this_delay > delay_nsec) {
delay_nsec = this_delay;
clamp = use_delay > 0;
}
}
blkg = blkg->parent;
}
if (!delay_nsec) return;
/*
* Let's not sleep for all eternity if we've amassed a huge delay.
* Swapping or metadata IO can accumulate 10's of seconds worth of
* delay, and we want userspace to be able to do _something_ so cap the
* delays at 0.25s. If there's 10's of seconds worth of delay then the
* tasks will be delayed for 0.25 second for every syscall. If
* blkcg_set_delay() was used as indicated by negative use_delay, the
* caller is responsible for regulating the range.
*/
if (clamp) delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
if (use_memdelay)
psi_memstall_enter(&pflags);
exp = ktime_add_ns(now, delay_nsec);
tok = io_schedule_prepare();
do {
__set_current_state(TASK_KILLABLE);
if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
break;
} while (!fatal_signal_pending(current)); io_schedule_finish(tok);
if (use_memdelay)
psi_memstall_leave(&pflags);
}
/**
* blkcg_maybe_throttle_current - throttle the current task if it has been marked
*
* This is only called if we've been marked with set_notify_resume(). Obviously
* we can be set_notify_resume() for reasons other than blkcg throttling, so we
* check to see if current->throttle_disk is set and if not this doesn't do
* anything. This should only ever be called by the resume code, it's not meant
* to be called by people willy-nilly as it will actually do the work to
* throttle the task if it is setup for throttling.
*/
void blkcg_maybe_throttle_current(void)
{
struct gendisk *disk = current->throttle_disk;
struct blkcg *blkcg;
struct blkcg_gq *blkg;
bool use_memdelay = current->use_memdelay;
if (!disk)
return;
current->throttle_disk = NULL;
current->use_memdelay = false;
rcu_read_lock(); blkcg = css_to_blkcg(blkcg_css());
if (!blkcg)
goto out;
blkg = blkg_lookup(blkcg, disk->queue);
if (!blkg)
goto out;
if (!blkg_tryget(blkg))
goto out;
rcu_read_unlock(); blkcg_maybe_throttle_blkg(blkg, use_memdelay);
blkg_put(blkg);
put_disk(disk);
return;
out:
rcu_read_unlock();
}
/**
* blkcg_schedule_throttle - this task needs to check for throttling
* @disk: disk to throttle
* @use_memdelay: do we charge this to memory delay for PSI
*
* This is called by the IO controller when we know there's delay accumulated
* for the blkg for this task. We do not pass the blkg because there are places
* we call this that may not have that information, the swapping code for
* instance will only have a block_device at that point. This set's the
* notify_resume for the task to check and see if it requires throttling before
* returning to user space.
*
* We will only schedule once per syscall. You can call this over and over
* again and it will only do the check once upon return to user space, and only
* throttle once. If the task needs to be throttled again it'll need to be
* re-set at the next time we see the task.
*/
void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay)
{
if (unlikely(current->flags & PF_KTHREAD))
return;
if (current->throttle_disk != disk) {
if (test_bit(GD_DEAD, &disk->state))
return;
get_device(disk_to_dev(disk));
if (current->throttle_disk)
put_disk(current->throttle_disk);
current->throttle_disk = disk;
}
if (use_memdelay)
current->use_memdelay = use_memdelay;
set_notify_resume(current);
}
/**
* blkcg_add_delay - add delay to this blkg
* @blkg: blkg of interest
* @now: the current time in nanoseconds
* @delta: how many nanoseconds of delay to add
*
* Charge @delta to the blkg's current delay accumulation. This is used to
* throttle tasks if an IO controller thinks we need more throttling.
*/
void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
{
if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
return;
blkcg_scale_delay(blkg, now);
atomic64_add(delta, &blkg->delay_nsec);
}
/**
* blkg_tryget_closest - try and get a blkg ref on the closet blkg
* @bio: target bio
* @css: target css
*
* As the failure mode here is to walk up the blkg tree, this ensure that the
* blkg->parent pointers are always valid. This returns the blkg that it ended
* up taking a reference on or %NULL if no reference was taken.
*/
static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
struct cgroup_subsys_state *css)
{
struct blkcg_gq *blkg, *ret_blkg = NULL;
rcu_read_lock();
blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk);
while (blkg) {
if (blkg_tryget(blkg)) {
ret_blkg = blkg;
break;
}
blkg = blkg->parent;
}
rcu_read_unlock();
return ret_blkg;
}
/**
* bio_associate_blkg_from_css - associate a bio with a specified css
* @bio: target bio
* @css: target css
*
* Associate @bio with the blkg found by combining the css's blkg and the
* request_queue of the @bio. An association failure is handled by walking up
* the blkg tree. Therefore, the blkg associated can be anything between @blkg
* and q->root_blkg. This situation only happens when a cgroup is dying and
* then the remaining bios will spill to the closest alive blkg.
*
* A reference will be taken on the blkg and will be released when @bio is
* freed.
*/
void bio_associate_blkg_from_css(struct bio *bio,
struct cgroup_subsys_state *css)
{
if (bio->bi_blkg)
blkg_put(bio->bi_blkg);
if (css && css->parent) {
bio->bi_blkg = blkg_tryget_closest(bio, css);
} else {
blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);
bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;
}
}
EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
/**
* bio_associate_blkg - associate a bio with a blkg
* @bio: target bio
*
* Associate @bio with the blkg found from the bio's css and request_queue.
* If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
* already associated, the css is reused and association redone as the
* request_queue may have changed.
*/
void bio_associate_blkg(struct bio *bio)
{
struct cgroup_subsys_state *css;
if (blk_op_is_passthrough(bio->bi_opf))
return;
rcu_read_lock();
if (bio->bi_blkg)
css = bio_blkcg_css(bio);
else
css = blkcg_css();
bio_associate_blkg_from_css(bio, css);
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(bio_associate_blkg);
/**
* bio_clone_blkg_association - clone blkg association from src to dst bio
* @dst: destination bio
* @src: source bio
*/
void bio_clone_blkg_association(struct bio *dst, struct bio *src)
{
if (src->bi_blkg)
bio_associate_blkg_from_css(dst, bio_blkcg_css(src));
}
EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
static int blk_cgroup_io_type(struct bio *bio)
{
if (op_is_discard(bio->bi_opf))
return BLKG_IOSTAT_DISCARD;
if (op_is_write(bio->bi_opf))
return BLKG_IOSTAT_WRITE;
return BLKG_IOSTAT_READ;
}
void blk_cgroup_bio_start(struct bio *bio)
{
struct blkcg *blkcg = bio->bi_blkg->blkcg;
int rwd = blk_cgroup_io_type(bio), cpu;
struct blkg_iostat_set *bis;
unsigned long flags;
if (!cgroup_subsys_on_dfl(io_cgrp_subsys))
return;
/* Root-level stats are sourced from system-wide IO stats */
if (!cgroup_parent(blkcg->css.cgroup))
return;
cpu = get_cpu();
bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
flags = u64_stats_update_begin_irqsave(&bis->sync);
/*
* If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
* bio and we would have already accounted for the size of the bio.
*/
if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
bio_set_flag(bio, BIO_CGROUP_ACCT);
bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
}
bis->cur.ios[rwd]++;
/*
* If the iostat_cpu isn't in a lockless list, put it into the
* list to indicate that a stat update is pending.
*/
if (!READ_ONCE(bis->lqueued)) {
struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
llist_add(&bis->lnode, lhead);
WRITE_ONCE(bis->lqueued, true);
}
u64_stats_update_end_irqrestore(&bis->sync, flags);
cgroup_rstat_updated(blkcg->css.cgroup, cpu);
put_cpu();
}
bool blk_cgroup_congested(void)
{
struct cgroup_subsys_state *css;
bool ret = false;
rcu_read_lock(); for (css = blkcg_css(); css; css = css->parent) { if (atomic_read(&css->cgroup->congestion_count)) { ret = true;
break;
}
}
rcu_read_unlock();
return ret;
}
module_param(blkcg_debug_stats, bool, 0644);
MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");
// SPDX-License-Identifier: GPL-2.0-only
/* Common code for 32 and 64-bit NUMA */
#include <linux/acpi.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/of.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/memblock.h>
#include <linux/mmzone.h>
#include <linux/ctype.h>
#include <linux/nodemask.h>
#include <linux/sched.h>
#include <linux/topology.h>
#include <linux/sort.h>
#include <asm/e820/api.h>
#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/amd_nb.h>
#include "numa_internal.h"
int numa_off;
nodemask_t numa_nodes_parsed __initdata;
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
EXPORT_SYMBOL(node_data);
static struct numa_meminfo numa_meminfo __initdata_or_meminfo;
static struct numa_meminfo numa_reserved_meminfo __initdata_or_meminfo;
static int numa_distance_cnt;
static u8 *numa_distance;
static __init int numa_setup(char *opt)
{
if (!opt)
return -EINVAL;
if (!strncmp(opt, "off", 3))
numa_off = 1;
if (!strncmp(opt, "fake=", 5))
return numa_emu_cmdline(opt + 5);
if (!strncmp(opt, "noacpi", 6))
disable_srat();
if (!strncmp(opt, "nohmat", 6))
disable_hmat();
return 0;
}
early_param("numa", numa_setup);
/*
* apicid, cpu, node mappings
*/
s16 __apicid_to_node[MAX_LOCAL_APIC] = {
[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
};
int numa_cpu_node(int cpu)
{
u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
if (apicid != BAD_APICID)
return __apicid_to_node[apicid];
return NUMA_NO_NODE;
}
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
EXPORT_SYMBOL(node_to_cpumask_map);
/*
* Map cpu index to node index
*/
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
void numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
/* early setting, no percpu area yet */
if (cpu_to_node_map) {
cpu_to_node_map[cpu] = node;
return;
}
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
dump_stack();
return;
}
#endif
per_cpu(x86_cpu_to_node_map, cpu) = node;
set_cpu_numa_node(cpu, node);
}
void numa_clear_node(int cpu)
{
numa_set_node(cpu, NUMA_NO_NODE);
}
/*
* Allocate node_to_cpumask_map based on number of available nodes
* Requires node_possible_map to be valid.
*
* Note: cpumask_of_node() is not valid until after this is done.
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
*/
void __init setup_node_to_cpumask_map(void)
{
unsigned int node;
/* setup nr_node_ids if not done yet */
if (nr_node_ids == MAX_NUMNODES)
setup_nr_node_ids();
/* allocate the map */
for (node = 0; node < nr_node_ids; node++)
alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
/* cpumask_of_node() will now work */
pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids);
}
static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
struct numa_meminfo *mi)
{
/* ignore zero length blks */
if (start == end)
return 0;
/* whine about and ignore invalid blks */
if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
nid, start, end - 1);
return 0;
}
if (mi->nr_blks >= NR_NODE_MEMBLKS) {
pr_err("too many memblk ranges\n");
return -EINVAL;
}
mi->blk[mi->nr_blks].start = start;
mi->blk[mi->nr_blks].end = end;
mi->blk[mi->nr_blks].nid = nid;
mi->nr_blks++;
return 0;
}
/**
* numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo
* @idx: Index of memblk to remove
* @mi: numa_meminfo to remove memblk from
*
* Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and
* decrementing @mi->nr_blks.
*/
void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi)
{
mi->nr_blks--;
memmove(&mi->blk[idx], &mi->blk[idx + 1],
(mi->nr_blks - idx) * sizeof(mi->blk[0]));
}
/**
* numa_move_tail_memblk - Move a numa_memblk from one numa_meminfo to another
* @dst: numa_meminfo to append block to
* @idx: Index of memblk to remove
* @src: numa_meminfo to remove memblk from
*/
static void __init numa_move_tail_memblk(struct numa_meminfo *dst, int idx,
struct numa_meminfo *src)
{
dst->blk[dst->nr_blks++] = src->blk[idx];
numa_remove_memblk_from(idx, src);
}
/**
* numa_add_memblk - Add one numa_memblk to numa_meminfo
* @nid: NUMA node ID of the new memblk
* @start: Start address of the new memblk
* @end: End address of the new memblk
*
* Add a new memblk to the default numa_meminfo.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
int __init numa_add_memblk(int nid, u64 start, u64 end)
{
return numa_add_memblk_to(nid, start, end, &numa_meminfo);
}
/* Allocate NODE_DATA for a node on the local memory */
static void __init alloc_node_data(int nid)
{
const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
u64 nd_pa;
void *nd;
int tnid;
/*
* Allocate node data. Try node-local memory and then any node.
* Never allocate in DMA zone.
*/
nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
if (!nd_pa) {
pr_err("Cannot find %zu bytes in any node (initial node: %d)\n",
nd_size, nid);
return;
}
nd = __va(nd_pa);
/* report and initialize */
printk(KERN_INFO "NODE_DATA(%d) allocated [mem %#010Lx-%#010Lx]\n", nid,
nd_pa, nd_pa + nd_size - 1);
tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
if (tnid != nid)
printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nid, tnid);
node_data[nid] = nd;
memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
node_set_online(nid);
}
/**
* numa_cleanup_meminfo - Cleanup a numa_meminfo
* @mi: numa_meminfo to clean up
*
* Sanitize @mi by merging and removing unnecessary memblks. Also check for
* conflicts and clear unused memblks.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
{
const u64 low = 0;
const u64 high = PFN_PHYS(max_pfn);
int i, j, k;
/* first, trim all entries */
for (i = 0; i < mi->nr_blks; i++) {
struct numa_memblk *bi = &mi->blk[i];
/* move / save reserved memory ranges */
if (!memblock_overlaps_region(&memblock.memory,
bi->start, bi->end - bi->start)) {
numa_move_tail_memblk(&numa_reserved_meminfo, i--, mi);
continue;
}
/* make sure all non-reserved blocks are inside the limits */
bi->start = max(bi->start, low);
/* preserve info for non-RAM areas above 'max_pfn': */
if (bi->end > high) {
numa_add_memblk_to(bi->nid, high, bi->end,
&numa_reserved_meminfo);
bi->end = high;
}
/* and there's no empty block */
if (bi->start >= bi->end)
numa_remove_memblk_from(i--, mi);
}
/* merge neighboring / overlapping entries */
for (i = 0; i < mi->nr_blks; i++) {
struct numa_memblk *bi = &mi->blk[i];
for (j = i + 1; j < mi->nr_blks; j++) {
struct numa_memblk *bj = &mi->blk[j];
u64 start, end;
/*
* See whether there are overlapping blocks. Whine
* about but allow overlaps of the same nid. They
* will be merged below.
*/
if (bi->end > bj->start && bi->start < bj->end) {
if (bi->nid != bj->nid) {
pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n",
bi->nid, bi->start, bi->end - 1,
bj->nid, bj->start, bj->end - 1);
return -EINVAL;
}
pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n",
bi->nid, bi->start, bi->end - 1,
bj->start, bj->end - 1);
}
/*
* Join together blocks on the same node, holes
* between which don't overlap with memory on other
* nodes.
*/
if (bi->nid != bj->nid)
continue;
start = min(bi->start, bj->start);
end = max(bi->end, bj->end);
for (k = 0; k < mi->nr_blks; k++) {
struct numa_memblk *bk = &mi->blk[k];
if (bi->nid == bk->nid)
continue;
if (start < bk->end && end > bk->start)
break;
}
if (k < mi->nr_blks)
continue;
printk(KERN_INFO "NUMA: Node %d [mem %#010Lx-%#010Lx] + [mem %#010Lx-%#010Lx] -> [mem %#010Lx-%#010Lx]\n",
bi->nid, bi->start, bi->end - 1, bj->start,
bj->end - 1, start, end - 1);
bi->start = start;
bi->end = end;
numa_remove_memblk_from(j--, mi);
}
}
/* clear unused ones */
for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) {
mi->blk[i].start = mi->blk[i].end = 0;
mi->blk[i].nid = NUMA_NO_NODE;
}
return 0;
}
/*
* Set nodes, which have memory in @mi, in *@nodemask.
*/
static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
const struct numa_meminfo *mi)
{
int i;
for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
if (mi->blk[i].start != mi->blk[i].end &&
mi->blk[i].nid != NUMA_NO_NODE)
node_set(mi->blk[i].nid, *nodemask);
}
/**
* numa_reset_distance - Reset NUMA distance table
*
* The current table is freed. The next numa_set_distance() call will
* create a new one.
*/
void __init numa_reset_distance(void)
{
size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]);
/* numa_distance could be 1LU marking allocation failure, test cnt */
if (numa_distance_cnt)
memblock_free(numa_distance, size);
numa_distance_cnt = 0;
numa_distance = NULL; /* enable table creation */
}
static int __init numa_alloc_distance(void)
{
nodemask_t nodes_parsed;
size_t size;
int i, j, cnt = 0;
u64 phys;
/* size the new table and allocate it */
nodes_parsed = numa_nodes_parsed;
numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
for_each_node_mask(i, nodes_parsed)
cnt = i;
cnt++;
size = cnt * cnt * sizeof(numa_distance[0]);
phys = memblock_phys_alloc_range(size, PAGE_SIZE, 0,
PFN_PHYS(max_pfn_mapped));
if (!phys) {
pr_warn("Warning: can't allocate distance table!\n");
/* don't retry until explicitly reset */
numa_distance = (void *)1LU;
return -ENOMEM;
}
numa_distance = __va(phys);
numa_distance_cnt = cnt;
/* fill with the default distances */
for (i = 0; i < cnt; i++)
for (j = 0; j < cnt; j++)
numa_distance[i * cnt + j] = i == j ?
LOCAL_DISTANCE : REMOTE_DISTANCE;
printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt);
return 0;
}
/**
* numa_set_distance - Set NUMA distance from one NUMA to another
* @from: the 'from' node to set distance
* @to: the 'to' node to set distance
* @distance: NUMA distance
*
* Set the distance from node @from to @to to @distance. If distance table
* doesn't exist, one which is large enough to accommodate all the currently
* known nodes will be created.
*
* If such table cannot be allocated, a warning is printed and further
* calls are ignored until the distance table is reset with
* numa_reset_distance().
*
* If @from or @to is higher than the highest known node or lower than zero
* at the time of table creation or @distance doesn't make sense, the call
* is ignored.
* This is to allow simplification of specific NUMA config implementations.
*/
void __init numa_set_distance(int from, int to, int distance)
{
if (!numa_distance && numa_alloc_distance() < 0)
return;
if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
from < 0 || to < 0) {
pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
from, to, distance);
return;
}
if ((u8)distance != distance ||
(from == to && distance != LOCAL_DISTANCE)) {
pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
from, to, distance);
return;
}
numa_distance[from * numa_distance_cnt + to] = distance;
}
int __node_distance(int from, int to)
{
if (from >= numa_distance_cnt || to >= numa_distance_cnt)
return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
return numa_distance[from * numa_distance_cnt + to];
}
EXPORT_SYMBOL(__node_distance);
/*
* Mark all currently memblock-reserved physical memory (which covers the
* kernel's own memory ranges) as hot-unswappable.
*/
static void __init numa_clear_kernel_node_hotplug(void)
{
nodemask_t reserved_nodemask = NODE_MASK_NONE;
struct memblock_region *mb_region;
int i;
/*
* We have to do some preprocessing of memblock regions, to
* make them suitable for reservation.
*
* At this time, all memory regions reserved by memblock are
* used by the kernel, but those regions are not split up
* along node boundaries yet, and don't necessarily have their
* node ID set yet either.
*
* So iterate over all memory known to the x86 architecture,
* and use those ranges to set the nid in memblock.reserved.
* This will split up the memblock regions along node
* boundaries and will set the node IDs as well.
*/
for (i = 0; i < numa_meminfo.nr_blks; i++) {
struct numa_memblk *mb = numa_meminfo.blk + i;
int ret;
ret = memblock_set_node(mb->start, mb->end - mb->start, &memblock.reserved, mb->nid);
WARN_ON_ONCE(ret);
}
/*
* Now go over all reserved memblock regions, to construct a
* node mask of all kernel reserved memory areas.
*
* [ Note, when booting with mem=nn[kMG] or in a kdump kernel,
* numa_meminfo might not include all memblock.reserved
* memory ranges, because quirks such as trim_snb_memory()
* reserve specific pages for Sandy Bridge graphics. ]
*/
for_each_reserved_mem_region(mb_region) {
int nid = memblock_get_region_node(mb_region);
if (nid != MAX_NUMNODES)
node_set(nid, reserved_nodemask);
}
/*
* Finally, clear the MEMBLOCK_HOTPLUG flag for all memory
* belonging to the reserved node mask.
*
* Note that this will include memory regions that reside
* on nodes that contain kernel memory - entire nodes
* become hot-unpluggable:
*/
for (i = 0; i < numa_meminfo.nr_blks; i++) {
struct numa_memblk *mb = numa_meminfo.blk + i;
if (!node_isset(mb->nid, reserved_nodemask))
continue;
memblock_clear_hotplug(mb->start, mb->end - mb->start);
}
}
static int __init numa_register_memblks(struct numa_meminfo *mi)
{
int i, nid;
/* Account for nodes with cpus and no memory */
node_possible_map = numa_nodes_parsed;
numa_nodemask_from_meminfo(&node_possible_map, mi);
if (WARN_ON(nodes_empty(node_possible_map)))
return -EINVAL;
for (i = 0; i < mi->nr_blks; i++) {
struct numa_memblk *mb = &mi->blk[i];
memblock_set_node(mb->start, mb->end - mb->start,
&memblock.memory, mb->nid);
}
/*
* At very early time, the kernel have to use some memory such as
* loading the kernel image. We cannot prevent this anyway. So any
* node the kernel resides in should be un-hotpluggable.
*
* And when we come here, alloc node data won't fail.
*/
numa_clear_kernel_node_hotplug();
/*
* If sections array is gonna be used for pfn -> nid mapping, check
* whether its granularity is fine enough.
*/
if (IS_ENABLED(NODE_NOT_IN_PAGE_FLAGS)) {
unsigned long pfn_align = node_map_pfn_alignment();
if (pfn_align && pfn_align < PAGES_PER_SECTION) {
pr_warn("Node alignment %LuMB < min %LuMB, rejecting NUMA config\n",
PFN_PHYS(pfn_align) >> 20,
PFN_PHYS(PAGES_PER_SECTION) >> 20);
return -EINVAL;
}
}
if (!memblock_validate_numa_coverage(SZ_1M))
return -EINVAL;
/* Finally register nodes. */
for_each_node_mask(nid, node_possible_map) {
u64 start = PFN_PHYS(max_pfn);
u64 end = 0;
for (i = 0; i < mi->nr_blks; i++) {
if (nid != mi->blk[i].nid)
continue;
start = min(mi->blk[i].start, start);
end = max(mi->blk[i].end, end);
}
if (start >= end)
continue;
alloc_node_data(nid);
}
/* Dump memblock with node info and return. */
memblock_dump_all();
return 0;
}
/*
* There are unfortunately some poorly designed mainboards around that
* only connect memory to a single CPU. This breaks the 1:1 cpu->node
* mapping. To avoid this fill in the mapping for all possible CPUs,
* as the number of CPUs is not known yet. We round robin the existing
* nodes.
*/
static void __init numa_init_array(void)
{
int rr, i;
rr = first_node(node_online_map);
for (i = 0; i < nr_cpu_ids; i++) {
if (early_cpu_to_node(i) != NUMA_NO_NODE)
continue;
numa_set_node(i, rr);
rr = next_node_in(rr, node_online_map);
}
}
static int __init numa_init(int (*init_func)(void))
{
int i;
int ret;
for (i = 0; i < MAX_LOCAL_APIC; i++)
set_apicid_to_node(i, NUMA_NO_NODE);
nodes_clear(numa_nodes_parsed);
nodes_clear(node_possible_map);
nodes_clear(node_online_map);
memset(&numa_meminfo, 0, sizeof(numa_meminfo));
WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.memory,
MAX_NUMNODES));
WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.reserved,
MAX_NUMNODES));
/* In case that parsing SRAT failed. */
WARN_ON(memblock_clear_hotplug(0, ULLONG_MAX));
numa_reset_distance();
ret = init_func();
if (ret < 0)
return ret;
/*
* We reset memblock back to the top-down direction
* here because if we configured ACPI_NUMA, we have
* parsed SRAT in init_func(). It is ok to have the
* reset here even if we did't configure ACPI_NUMA
* or acpi numa init fails and fallbacks to dummy
* numa init.
*/
memblock_set_bottom_up(false);
ret = numa_cleanup_meminfo(&numa_meminfo);
if (ret < 0)
return ret;
numa_emulation(&numa_meminfo, numa_distance_cnt);
ret = numa_register_memblks(&numa_meminfo);
if (ret < 0)
return ret;
for (i = 0; i < nr_cpu_ids; i++) {
int nid = early_cpu_to_node(i);
if (nid == NUMA_NO_NODE)
continue;
if (!node_online(nid))
numa_clear_node(i);
}
numa_init_array();
return 0;
}
/**
* dummy_numa_init - Fallback dummy NUMA init
*
* Used if there's no underlying NUMA architecture, NUMA initialization
* fails, or NUMA is disabled on the command line.
*
* Must online at least one node and add memory blocks that cover all
* allowed memory. This function must not fail.
*/
static int __init dummy_numa_init(void)
{
printk(KERN_INFO "%s\n",
numa_off ? "NUMA turned off" : "No NUMA configuration found");
printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n",
0LLU, PFN_PHYS(max_pfn) - 1);
node_set(0, numa_nodes_parsed);
numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
return 0;
}
/**
* x86_numa_init - Initialize NUMA
*
* Try each configured NUMA initialization method until one succeeds. The
* last fallback is dummy single node config encompassing whole memory and
* never fails.
*/
void __init x86_numa_init(void)
{
if (!numa_off) {
#ifdef CONFIG_ACPI_NUMA
if (!numa_init(x86_acpi_numa_init))
return;
#endif
#ifdef CONFIG_AMD_NUMA
if (!numa_init(amd_numa_init))
return;
#endif
if (acpi_disabled && !numa_init(of_numa_init))
return;
}
numa_init(dummy_numa_init);
}
/*
* A node may exist which has one or more Generic Initiators but no CPUs and no
* memory.
*
* This function must be called after init_cpu_to_node(), to ensure that any
* memoryless CPU nodes have already been brought online, and before the
* node_data[nid] is needed for zone list setup in build_all_zonelists().
*
* When this function is called, any nodes containing either memory and/or CPUs
* will already be online and there is no need to do anything extra, even if
* they also contain one or more Generic Initiators.
*/
void __init init_gi_nodes(void)
{
int nid;
/*
* Exclude this node from
* bringup_nonboot_cpus
* cpu_up
* __try_online_node
* register_one_node
* because node_subsys is not initialized yet.
* TODO remove dependency on node_online
*/
for_each_node_state(nid, N_GENERIC_INITIATOR)
if (!node_online(nid))
node_set_online(nid);
}
/*
* Setup early cpu_to_node.
*
* Populate cpu_to_node[] only if x86_cpu_to_apicid[],
* and apicid_to_node[] tables have valid entries for a CPU.
* This means we skip cpu_to_node[] initialisation for NUMA
* emulation and faking node case (when running a kernel compiled
* for NUMA on a non NUMA box), which is OK as cpu_to_node[]
* is already initialized in a round robin manner at numa_init_array,
* prior to this call, and this initialization is good enough
* for the fake NUMA cases.
*
* Called before the per_cpu areas are setup.
*/
void __init init_cpu_to_node(void)
{
int cpu;
u32 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
BUG_ON(cpu_to_apicid == NULL);
for_each_possible_cpu(cpu) {
int node = numa_cpu_node(cpu);
if (node == NUMA_NO_NODE)
continue;
/*
* Exclude this node from
* bringup_nonboot_cpus
* cpu_up
* __try_online_node
* register_one_node
* because node_subsys is not initialized yet.
* TODO remove dependency on node_online
*/
if (!node_online(node))
node_set_online(node);
numa_set_node(cpu, node);
}
}
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
# ifndef CONFIG_NUMA_EMU
void numa_add_cpu(int cpu)
{
cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
void numa_remove_cpu(int cpu)
{
cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
}
# endif /* !CONFIG_NUMA_EMU */
#else /* !CONFIG_DEBUG_PER_CPU_MAPS */
int __cpu_to_node(int cpu)
{
if (early_per_cpu_ptr(x86_cpu_to_node_map)) { printk(KERN_WARNING
"cpu_to_node(%d): usage too early!\n", cpu);
dump_stack();
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
}
return per_cpu(x86_cpu_to_node_map, cpu);
}
EXPORT_SYMBOL(__cpu_to_node);
/*
* Same function as cpu_to_node() but used if called before the
* per_cpu areas are setup.
*/
int early_cpu_to_node(int cpu)
{
if (early_per_cpu_ptr(x86_cpu_to_node_map))
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
if (!cpu_possible(cpu)) {
printk(KERN_WARNING
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
dump_stack();
return NUMA_NO_NODE;
}
return per_cpu(x86_cpu_to_node_map, cpu);
}
void debug_cpumask_set_cpu(int cpu, int node, bool enable)
{
struct cpumask *mask;
if (node == NUMA_NO_NODE) {
/* early_cpu_to_node() already emits a warning and trace */
return;
}
mask = node_to_cpumask_map[node];
if (!cpumask_available(mask)) {
pr_err("node_to_cpumask_map[%i] NULL\n", node);
dump_stack();
return;
}
if (enable)
cpumask_set_cpu(cpu, mask);
else
cpumask_clear_cpu(cpu, mask);
printk(KERN_DEBUG "%s cpu %d node %d: mask now %*pbl\n",
enable ? "numa_add_cpu" : "numa_remove_cpu",
cpu, node, cpumask_pr_args(mask));
return;
}
# ifndef CONFIG_NUMA_EMU
static void numa_set_cpumask(int cpu, bool enable)
{
debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable);
}
void numa_add_cpu(int cpu)
{
numa_set_cpumask(cpu, true);
}
void numa_remove_cpu(int cpu)
{
numa_set_cpumask(cpu, false);
}
# endif /* !CONFIG_NUMA_EMU */
/*
* Returns a pointer to the bitmask of CPUs on Node 'node'.
*/
const struct cpumask *cpumask_of_node(int node)
{
if ((unsigned)node >= nr_node_ids) {
printk(KERN_WARNING
"cpumask_of_node(%d): (unsigned)node >= nr_node_ids(%u)\n",
node, nr_node_ids);
dump_stack();
return cpu_none_mask;
}
if (!cpumask_available(node_to_cpumask_map[node])) {
printk(KERN_WARNING
"cpumask_of_node(%d): no node_to_cpumask_map!\n",
node);
dump_stack();
return cpu_online_mask;
}
return node_to_cpumask_map[node];
}
EXPORT_SYMBOL(cpumask_of_node);
#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
#ifdef CONFIG_NUMA_KEEP_MEMINFO
static int meminfo_to_nid(struct numa_meminfo *mi, u64 start)
{
int i;
for (i = 0; i < mi->nr_blks; i++)
if (mi->blk[i].start <= start && mi->blk[i].end > start)
return mi->blk[i].nid;
return NUMA_NO_NODE;
}
int phys_to_target_node(phys_addr_t start)
{
int nid = meminfo_to_nid(&numa_meminfo, start);
/*
* Prefer online nodes, but if reserved memory might be
* hot-added continue the search with reserved ranges.
*/
if (nid != NUMA_NO_NODE)
return nid;
return meminfo_to_nid(&numa_reserved_meminfo, start);
}
EXPORT_SYMBOL_GPL(phys_to_target_node);
int memory_add_physaddr_to_nid(u64 start)
{
int nid = meminfo_to_nid(&numa_meminfo, start);
if (nid == NUMA_NO_NODE)
nid = numa_meminfo.blk[0].nid;
return nid;
}
EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
static int __init cmp_memblk(const void *a, const void *b)
{
const struct numa_memblk *ma = *(const struct numa_memblk **)a;
const struct numa_memblk *mb = *(const struct numa_memblk **)b;
return (ma->start > mb->start) - (ma->start < mb->start);
}
static struct numa_memblk *numa_memblk_list[NR_NODE_MEMBLKS] __initdata;
/**
* numa_fill_memblks - Fill gaps in numa_meminfo memblks
* @start: address to begin fill
* @end: address to end fill
*
* Find and extend numa_meminfo memblks to cover the physical
* address range @start-@end
*
* RETURNS:
* 0 : Success
* NUMA_NO_MEMBLK : No memblks exist in address range @start-@end
*/
int __init numa_fill_memblks(u64 start, u64 end)
{
struct numa_memblk **blk = &numa_memblk_list[0];
struct numa_meminfo *mi = &numa_meminfo;
int count = 0;
u64 prev_end;
/*
* Create a list of pointers to numa_meminfo memblks that
* overlap start, end. The list is used to make in-place
* changes that fill out the numa_meminfo memblks.
*/
for (int i = 0; i < mi->nr_blks; i++) {
struct numa_memblk *bi = &mi->blk[i];
if (memblock_addrs_overlap(start, end - start, bi->start,
bi->end - bi->start)) {
blk[count] = &mi->blk[i];
count++;
}
}
if (!count)
return NUMA_NO_MEMBLK;
/* Sort the list of pointers in memblk->start order */
sort(&blk[0], count, sizeof(blk[0]), cmp_memblk, NULL);
/* Make sure the first/last memblks include start/end */
blk[0]->start = min(blk[0]->start, start);
blk[count - 1]->end = max(blk[count - 1]->end, end);
/*
* Fill any gaps by tracking the previous memblks
* end address and backfilling to it if needed.
*/
prev_end = blk[0]->end;
for (int i = 1; i < count; i++) {
struct numa_memblk *curr = blk[i];
if (prev_end >= curr->start) {
if (prev_end < curr->end)
prev_end = curr->end;
} else {
curr->start = prev_end;
prev_end = curr->end;
}
}
return 0;
}
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* message.c - synchronous message handling
*
* Released under the GPLv2 only.
*/
#include <linux/acpi.h>
#include <linux/pci.h> /* for scatterlist macros */
#include <linux/usb.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/timer.h>
#include <linux/ctype.h>
#include <linux/nls.h>
#include <linux/device.h>
#include <linux/scatterlist.h>
#include <linux/usb/cdc.h>
#include <linux/usb/quirks.h>
#include <linux/usb/hcd.h> /* for usbcore internals */
#include <linux/usb/of.h>
#include <asm/byteorder.h>
#include "usb.h"
static void cancel_async_set_config(struct usb_device *udev);
struct api_context {
struct completion done;
int status;
};
static void usb_api_blocking_completion(struct urb *urb)
{
struct api_context *ctx = urb->context;
ctx->status = urb->status;
complete(&ctx->done);
}
/*
* Starts urb and waits for completion or timeout. Note that this call
* is NOT interruptible. Many device driver i/o requests should be
* interruptible and therefore these drivers should implement their
* own interruptible routines.
*/
static int usb_start_wait_urb(struct urb *urb, int timeout, int *actual_length)
{
struct api_context ctx;
unsigned long expire;
int retval;
init_completion(&ctx.done);
urb->context = &ctx;
urb->actual_length = 0;
retval = usb_submit_urb(urb, GFP_NOIO);
if (unlikely(retval))
goto out;
expire = timeout ? msecs_to_jiffies(timeout) : MAX_SCHEDULE_TIMEOUT; if (!wait_for_completion_timeout(&ctx.done, expire)) { usb_kill_urb(urb);
retval = (ctx.status == -ENOENT ? -ETIMEDOUT : ctx.status);
dev_dbg(&urb->dev->dev,
"%s timed out on ep%d%s len=%u/%u\n",
current->comm,
usb_endpoint_num(&urb->ep->desc),
usb_urb_dir_in(urb) ? "in" : "out",
urb->actual_length,
urb->transfer_buffer_length);
} else
retval = ctx.status;
out:
if (actual_length) *actual_length = urb->actual_length; usb_free_urb(urb);
return retval;
}
/*-------------------------------------------------------------------*/
/* returns status (negative) or length (positive) */
static int usb_internal_control_msg(struct usb_device *usb_dev,
unsigned int pipe,
struct usb_ctrlrequest *cmd,
void *data, int len, int timeout)
{
struct urb *urb;
int retv;
int length;
urb = usb_alloc_urb(0, GFP_NOIO);
if (!urb)
return -ENOMEM;
usb_fill_control_urb(urb, usb_dev, pipe, (unsigned char *)cmd, data,
len, usb_api_blocking_completion, NULL);
retv = usb_start_wait_urb(urb, timeout, &length);
if (retv < 0)
return retv;
else
return length;
}
/**
* usb_control_msg - Builds a control urb, sends it off and waits for completion
* @dev: pointer to the usb device to send the message to
* @pipe: endpoint "pipe" to send the message to
* @request: USB message request value
* @requesttype: USB message request type value
* @value: USB message value
* @index: USB message index value
* @data: pointer to the data to send
* @size: length in bytes of the data to send
* @timeout: time in msecs to wait for the message to complete before timing
* out (if 0 the wait is forever)
*
* Context: task context, might sleep.
*
* This function sends a simple control message to a specified endpoint and
* waits for the message to complete, or timeout.
*
* Don't use this function from within an interrupt context. If you need
* an asynchronous message, or need to send a message from within interrupt
* context, use usb_submit_urb(). If a thread in your driver uses this call,
* make sure your disconnect() method can wait for it to complete. Since you
* don't have a handle on the URB used, you can't cancel the request.
*
* Return: If successful, the number of bytes transferred. Otherwise, a negative
* error number.
*/
int usb_control_msg(struct usb_device *dev, unsigned int pipe, __u8 request,
__u8 requesttype, __u16 value, __u16 index, void *data,
__u16 size, int timeout)
{
struct usb_ctrlrequest *dr;
int ret;
dr = kmalloc(sizeof(struct usb_ctrlrequest), GFP_NOIO);
if (!dr)
return -ENOMEM;
dr->bRequestType = requesttype;
dr->bRequest = request;
dr->wValue = cpu_to_le16(value);
dr->wIndex = cpu_to_le16(index);
dr->wLength = cpu_to_le16(size);
ret = usb_internal_control_msg(dev, pipe, dr, data, size, timeout);
/* Linger a bit, prior to the next control message. */
if (dev->quirks & USB_QUIRK_DELAY_CTRL_MSG)
msleep(200); kfree(dr); return ret;
}
EXPORT_SYMBOL_GPL(usb_control_msg);
/**
* usb_control_msg_send - Builds a control "send" message, sends it off and waits for completion
* @dev: pointer to the usb device to send the message to
* @endpoint: endpoint to send the message to
* @request: USB message request value
* @requesttype: USB message request type value
* @value: USB message value
* @index: USB message index value
* @driver_data: pointer to the data to send
* @size: length in bytes of the data to send
* @timeout: time in msecs to wait for the message to complete before timing
* out (if 0 the wait is forever)
* @memflags: the flags for memory allocation for buffers
*
* Context: !in_interrupt ()
*
* This function sends a control message to a specified endpoint that is not
* expected to fill in a response (i.e. a "send message") and waits for the
* message to complete, or timeout.
*
* Do not use this function from within an interrupt context. If you need
* an asynchronous message, or need to send a message from within interrupt
* context, use usb_submit_urb(). If a thread in your driver uses this call,
* make sure your disconnect() method can wait for it to complete. Since you
* don't have a handle on the URB used, you can't cancel the request.
*
* The data pointer can be made to a reference on the stack, or anywhere else,
* as it will not be modified at all. This does not have the restriction that
* usb_control_msg() has where the data pointer must be to dynamically allocated
* memory (i.e. memory that can be successfully DMAed to a device).
*
* Return: If successful, 0 is returned, Otherwise, a negative error number.
*/
int usb_control_msg_send(struct usb_device *dev, __u8 endpoint, __u8 request,
__u8 requesttype, __u16 value, __u16 index,
const void *driver_data, __u16 size, int timeout,
gfp_t memflags)
{
unsigned int pipe = usb_sndctrlpipe(dev, endpoint);
int ret;
u8 *data = NULL;
if (size) {
data = kmemdup(driver_data, size, memflags);
if (!data)
return -ENOMEM;
}
ret = usb_control_msg(dev, pipe, request, requesttype, value, index,
data, size, timeout);
kfree(data);
if (ret < 0)
return ret;
return 0;
}
EXPORT_SYMBOL_GPL(usb_control_msg_send);
/**
* usb_control_msg_recv - Builds a control "receive" message, sends it off and waits for completion
* @dev: pointer to the usb device to send the message to
* @endpoint: endpoint to send the message to
* @request: USB message request value
* @requesttype: USB message request type value
* @value: USB message value
* @index: USB message index value
* @driver_data: pointer to the data to be filled in by the message
* @size: length in bytes of the data to be received
* @timeout: time in msecs to wait for the message to complete before timing
* out (if 0 the wait is forever)
* @memflags: the flags for memory allocation for buffers
*
* Context: !in_interrupt ()
*
* This function sends a control message to a specified endpoint that is
* expected to fill in a response (i.e. a "receive message") and waits for the
* message to complete, or timeout.
*
* Do not use this function from within an interrupt context. If you need
* an asynchronous message, or need to send a message from within interrupt
* context, use usb_submit_urb(). If a thread in your driver uses this call,
* make sure your disconnect() method can wait for it to complete. Since you
* don't have a handle on the URB used, you can't cancel the request.
*
* The data pointer can be made to a reference on the stack, or anywhere else
* that can be successfully written to. This function does not have the
* restriction that usb_control_msg() has where the data pointer must be to
* dynamically allocated memory (i.e. memory that can be successfully DMAed to a
* device).
*
* The "whole" message must be properly received from the device in order for
* this function to be successful. If a device returns less than the expected
* amount of data, then the function will fail. Do not use this for messages
* where a variable amount of data might be returned.
*
* Return: If successful, 0 is returned, Otherwise, a negative error number.
*/
int usb_control_msg_recv(struct usb_device *dev, __u8 endpoint, __u8 request,
__u8 requesttype, __u16 value, __u16 index,
void *driver_data, __u16 size, int timeout,
gfp_t memflags)
{
unsigned int pipe = usb_rcvctrlpipe(dev, endpoint);
int ret;
u8 *data;
if (!size || !driver_data)
return -EINVAL;
data = kmalloc(size, memflags);
if (!data)
return -ENOMEM;
ret = usb_control_msg(dev, pipe, request, requesttype, value, index,
data, size, timeout);
if (ret < 0)
goto exit;
if (ret == size) { memcpy(driver_data, data, size);
ret = 0;
} else {
ret = -EREMOTEIO;
}
exit:
kfree(data); return ret;
}
EXPORT_SYMBOL_GPL(usb_control_msg_recv);
/**
* usb_interrupt_msg - Builds an interrupt urb, sends it off and waits for completion
* @usb_dev: pointer to the usb device to send the message to
* @pipe: endpoint "pipe" to send the message to
* @data: pointer to the data to send
* @len: length in bytes of the data to send
* @actual_length: pointer to a location to put the actual length transferred
* in bytes
* @timeout: time in msecs to wait for the message to complete before
* timing out (if 0 the wait is forever)
*
* Context: task context, might sleep.
*
* This function sends a simple interrupt message to a specified endpoint and
* waits for the message to complete, or timeout.
*
* Don't use this function from within an interrupt context. If you need
* an asynchronous message, or need to send a message from within interrupt
* context, use usb_submit_urb() If a thread in your driver uses this call,
* make sure your disconnect() method can wait for it to complete. Since you
* don't have a handle on the URB used, you can't cancel the request.
*
* Return:
* If successful, 0. Otherwise a negative error number. The number of actual
* bytes transferred will be stored in the @actual_length parameter.
*/
int usb_interrupt_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length, int timeout)
{
return usb_bulk_msg(usb_dev, pipe, data, len, actual_length, timeout);
}
EXPORT_SYMBOL_GPL(usb_interrupt_msg);
/**
* usb_bulk_msg - Builds a bulk urb, sends it off and waits for completion
* @usb_dev: pointer to the usb device to send the message to
* @pipe: endpoint "pipe" to send the message to
* @data: pointer to the data to send
* @len: length in bytes of the data to send
* @actual_length: pointer to a location to put the actual length transferred
* in bytes
* @timeout: time in msecs to wait for the message to complete before
* timing out (if 0 the wait is forever)
*
* Context: task context, might sleep.
*
* This function sends a simple bulk message to a specified endpoint
* and waits for the message to complete, or timeout.
*
* Don't use this function from within an interrupt context. If you need
* an asynchronous message, or need to send a message from within interrupt
* context, use usb_submit_urb() If a thread in your driver uses this call,
* make sure your disconnect() method can wait for it to complete. Since you
* don't have a handle on the URB used, you can't cancel the request.
*
* Because there is no usb_interrupt_msg() and no USBDEVFS_INTERRUPT ioctl,
* users are forced to abuse this routine by using it to submit URBs for
* interrupt endpoints. We will take the liberty of creating an interrupt URB
* (with the default interval) if the target is an interrupt endpoint.
*
* Return:
* If successful, 0. Otherwise a negative error number. The number of actual
* bytes transferred will be stored in the @actual_length parameter.
*
*/
int usb_bulk_msg(struct usb_device *usb_dev, unsigned int pipe,
void *data, int len, int *actual_length, int timeout)
{
struct urb *urb;
struct usb_host_endpoint *ep;
ep = usb_pipe_endpoint(usb_dev, pipe); if (!ep || len < 0)
return -EINVAL;
urb = usb_alloc_urb(0, GFP_KERNEL);
if (!urb)
return -ENOMEM;
if ((ep->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) ==
USB_ENDPOINT_XFER_INT) {
pipe = (pipe & ~(3 << 30)) | (PIPE_INTERRUPT << 30); usb_fill_int_urb(urb, usb_dev, pipe, data, len,
usb_api_blocking_completion, NULL,
ep->desc.bInterval);
} else
usb_fill_bulk_urb(urb, usb_dev, pipe, data, len,
usb_api_blocking_completion, NULL);
return usb_start_wait_urb(urb, timeout, actual_length);
}
EXPORT_SYMBOL_GPL(usb_bulk_msg);
/*-------------------------------------------------------------------*/
static void sg_clean(struct usb_sg_request *io)
{
if (io->urbs) {
while (io->entries--)
usb_free_urb(io->urbs[io->entries]);
kfree(io->urbs);
io->urbs = NULL;
}
io->dev = NULL;
}
static void sg_complete(struct urb *urb)
{
unsigned long flags;
struct usb_sg_request *io = urb->context;
int status = urb->status;
spin_lock_irqsave(&io->lock, flags);
/* In 2.5 we require hcds' endpoint queues not to progress after fault
* reports, until the completion callback (this!) returns. That lets
* device driver code (like this routine) unlink queued urbs first,
* if it needs to, since the HC won't work on them at all. So it's
* not possible for page N+1 to overwrite page N, and so on.
*
* That's only for "hard" faults; "soft" faults (unlinks) sometimes
* complete before the HCD can get requests away from hardware,
* though never during cleanup after a hard fault.
*/
if (io->status
&& (io->status != -ECONNRESET
|| status != -ECONNRESET)
&& urb->actual_length) {
dev_err(io->dev->bus->controller,
"dev %s ep%d%s scatterlist error %d/%d\n",
io->dev->devpath,
usb_endpoint_num(&urb->ep->desc),
usb_urb_dir_in(urb) ? "in" : "out",
status, io->status);
/* BUG (); */
}
if (io->status == 0 && status && status != -ECONNRESET) {
int i, found, retval;
io->status = status;
/* the previous urbs, and this one, completed already.
* unlink pending urbs so they won't rx/tx bad data.
* careful: unlink can sometimes be synchronous...
*/
spin_unlock_irqrestore(&io->lock, flags);
for (i = 0, found = 0; i < io->entries; i++) {
if (!io->urbs[i])
continue;
if (found) {
usb_block_urb(io->urbs[i]);
retval = usb_unlink_urb(io->urbs[i]);
if (retval != -EINPROGRESS &&
retval != -ENODEV &&
retval != -EBUSY &&
retval != -EIDRM)
dev_err(&io->dev->dev,
"%s, unlink --> %d\n",
__func__, retval);
} else if (urb == io->urbs[i])
found = 1;
}
spin_lock_irqsave(&io->lock, flags);
}
/* on the last completion, signal usb_sg_wait() */
io->bytes += urb->actual_length;
io->count--;
if (!io->count)
complete(&io->complete);
spin_unlock_irqrestore(&io->lock, flags);
}
/**
* usb_sg_init - initializes scatterlist-based bulk/interrupt I/O request
* @io: request block being initialized. until usb_sg_wait() returns,
* treat this as a pointer to an opaque block of memory,
* @dev: the usb device that will send or receive the data
* @pipe: endpoint "pipe" used to transfer the data
* @period: polling rate for interrupt endpoints, in frames or
* (for high speed endpoints) microframes; ignored for bulk
* @sg: scatterlist entries
* @nents: how many entries in the scatterlist
* @length: how many bytes to send from the scatterlist, or zero to
* send every byte identified in the list.
* @mem_flags: SLAB_* flags affecting memory allocations in this call
*
* This initializes a scatter/gather request, allocating resources such as
* I/O mappings and urb memory (except maybe memory used by USB controller
* drivers).
*
* The request must be issued using usb_sg_wait(), which waits for the I/O to
* complete (or to be canceled) and then cleans up all resources allocated by
* usb_sg_init().
*
* The request may be canceled with usb_sg_cancel(), either before or after
* usb_sg_wait() is called.
*
* Return: Zero for success, else a negative errno value.
*/
int usb_sg_init(struct usb_sg_request *io, struct usb_device *dev,
unsigned pipe, unsigned period, struct scatterlist *sg,
int nents, size_t length, gfp_t mem_flags)
{
int i;
int urb_flags;
int use_sg;
if (!io || !dev || !sg
|| usb_pipecontrol(pipe)
|| usb_pipeisoc(pipe)
|| nents <= 0)
return -EINVAL;
spin_lock_init(&io->lock);
io->dev = dev;
io->pipe = pipe;
if (dev->bus->sg_tablesize > 0) {
use_sg = true;
io->entries = 1;
} else {
use_sg = false;
io->entries = nents;
}
/* initialize all the urbs we'll use */
io->urbs = kmalloc_array(io->entries, sizeof(*io->urbs), mem_flags);
if (!io->urbs)
goto nomem;
urb_flags = URB_NO_INTERRUPT;
if (usb_pipein(pipe))
urb_flags |= URB_SHORT_NOT_OK;
for_each_sg(sg, sg, io->entries, i) {
struct urb *urb;
unsigned len;
urb = usb_alloc_urb(0, mem_flags);
if (!urb) {
io->entries = i;
goto nomem;
}
io->urbs[i] = urb;
urb->dev = NULL;
urb->pipe = pipe;
urb->interval = period;
urb->transfer_flags = urb_flags;
urb->complete = sg_complete;
urb->context = io;
urb->sg = sg;
if (use_sg) {
/* There is no single transfer buffer */
urb->transfer_buffer = NULL;
urb->num_sgs = nents;
/* A length of zero means transfer the whole sg list */
len = length;
if (len == 0) {
struct scatterlist *sg2;
int j;
for_each_sg(sg, sg2, nents, j)
len += sg2->length;
}
} else {
/*
* Some systems can't use DMA; they use PIO instead.
* For their sakes, transfer_buffer is set whenever
* possible.
*/
if (!PageHighMem(sg_page(sg)))
urb->transfer_buffer = sg_virt(sg);
else
urb->transfer_buffer = NULL;
len = sg->length;
if (length) {
len = min_t(size_t, len, length);
length -= len;
if (length == 0)
io->entries = i + 1;
}
}
urb->transfer_buffer_length = len;
}
io->urbs[--i]->transfer_flags &= ~URB_NO_INTERRUPT;
/* transaction state */
io->count = io->entries;
io->status = 0;
io->bytes = 0;
init_completion(&io->complete);
return 0;
nomem:
sg_clean(io);
return -ENOMEM;
}
EXPORT_SYMBOL_GPL(usb_sg_init);
/**
* usb_sg_wait - synchronously execute scatter/gather request
* @io: request block handle, as initialized with usb_sg_init().
* some fields become accessible when this call returns.
*
* Context: task context, might sleep.
*
* This function blocks until the specified I/O operation completes. It
* leverages the grouping of the related I/O requests to get good transfer
* rates, by queueing the requests. At higher speeds, such queuing can
* significantly improve USB throughput.
*
* There are three kinds of completion for this function.
*
* (1) success, where io->status is zero. The number of io->bytes
* transferred is as requested.
* (2) error, where io->status is a negative errno value. The number
* of io->bytes transferred before the error is usually less
* than requested, and can be nonzero.
* (3) cancellation, a type of error with status -ECONNRESET that
* is initiated by usb_sg_cancel().
*
* When this function returns, all memory allocated through usb_sg_init() or
* this call will have been freed. The request block parameter may still be
* passed to usb_sg_cancel(), or it may be freed. It could also be
* reinitialized and then reused.
*
* Data Transfer Rates:
*
* Bulk transfers are valid for full or high speed endpoints.
* The best full speed data rate is 19 packets of 64 bytes each
* per frame, or 1216 bytes per millisecond.
* The best high speed data rate is 13 packets of 512 bytes each
* per microframe, or 52 KBytes per millisecond.
*
* The reason to use interrupt transfers through this API would most likely
* be to reserve high speed bandwidth, where up to 24 KBytes per millisecond
* could be transferred. That capability is less useful for low or full
* speed interrupt endpoints, which allow at most one packet per millisecond,
* of at most 8 or 64 bytes (respectively).
*
* It is not necessary to call this function to reserve bandwidth for devices
* under an xHCI host controller, as the bandwidth is reserved when the
* configuration or interface alt setting is selected.
*/
void usb_sg_wait(struct usb_sg_request *io)
{
int i;
int entries = io->entries;
/* queue the urbs. */
spin_lock_irq(&io->lock);
i = 0;
while (i < entries && !io->status) {
int retval;
io->urbs[i]->dev = io->dev;
spin_unlock_irq(&io->lock);
retval = usb_submit_urb(io->urbs[i], GFP_NOIO);
switch (retval) {
/* maybe we retrying will recover */
case -ENXIO: /* hc didn't queue this one */
case -EAGAIN:
case -ENOMEM:
retval = 0;
yield();
break;
/* no error? continue immediately.
*
* NOTE: to work better with UHCI (4K I/O buffer may
* need 3K of TDs) it may be good to limit how many
* URBs are queued at once; N milliseconds?
*/
case 0:
++i;
cpu_relax();
break;
/* fail any uncompleted urbs */
default:
io->urbs[i]->status = retval;
dev_dbg(&io->dev->dev, "%s, submit --> %d\n",
__func__, retval);
usb_sg_cancel(io);
}
spin_lock_irq(&io->lock);
if (retval && (io->status == 0 || io->status == -ECONNRESET))
io->status = retval;
}
io->count -= entries - i;
if (io->count == 0)
complete(&io->complete);
spin_unlock_irq(&io->lock);
/* OK, yes, this could be packaged as non-blocking.
* So could the submit loop above ... but it's easier to
* solve neither problem than to solve both!
*/
wait_for_completion(&io->complete);
sg_clean(io);
}
EXPORT_SYMBOL_GPL(usb_sg_wait);
/**
* usb_sg_cancel - stop scatter/gather i/o issued by usb_sg_wait()
* @io: request block, initialized with usb_sg_init()
*
* This stops a request after it has been started by usb_sg_wait().
* It can also prevents one initialized by usb_sg_init() from starting,
* so that call just frees resources allocated to the request.
*/
void usb_sg_cancel(struct usb_sg_request *io)
{
unsigned long flags;
int i, retval;
spin_lock_irqsave(&io->lock, flags);
if (io->status || io->count == 0) {
spin_unlock_irqrestore(&io->lock, flags);
return;
}
/* shut everything down */
io->status = -ECONNRESET;
io->count++; /* Keep the request alive until we're done */
spin_unlock_irqrestore(&io->lock, flags);
for (i = io->entries - 1; i >= 0; --i) {
usb_block_urb(io->urbs[i]);
retval = usb_unlink_urb(io->urbs[i]);
if (retval != -EINPROGRESS
&& retval != -ENODEV
&& retval != -EBUSY
&& retval != -EIDRM)
dev_warn(&io->dev->dev, "%s, unlink --> %d\n",
__func__, retval);
}
spin_lock_irqsave(&io->lock, flags);
io->count--;
if (!io->count)
complete(&io->complete);
spin_unlock_irqrestore(&io->lock, flags);
}
EXPORT_SYMBOL_GPL(usb_sg_cancel);
/*-------------------------------------------------------------------*/
/**
* usb_get_descriptor - issues a generic GET_DESCRIPTOR request
* @dev: the device whose descriptor is being retrieved
* @type: the descriptor type (USB_DT_*)
* @index: the number of the descriptor
* @buf: where to put the descriptor
* @size: how big is "buf"?
*
* Context: task context, might sleep.
*
* Gets a USB descriptor. Convenience functions exist to simplify
* getting some types of descriptors. Use
* usb_get_string() or usb_string() for USB_DT_STRING.
* Device (USB_DT_DEVICE) and configuration descriptors (USB_DT_CONFIG)
* are part of the device structure.
* In addition to a number of USB-standard descriptors, some
* devices also use class-specific or vendor-specific descriptors.
*
* This call is synchronous, and may not be used in an interrupt context.
*
* Return: The number of bytes received on success, or else the status code
* returned by the underlying usb_control_msg() call.
*/
int usb_get_descriptor(struct usb_device *dev, unsigned char type,
unsigned char index, void *buf, int size)
{
int i;
int result;
if (size <= 0) /* No point in asking for no data */
return -EINVAL;
memset(buf, 0, size); /* Make sure we parse really received data */ for (i = 0; i < 3; ++i) {
/* retry on length 0 or error; some devices are flakey */
result = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
USB_REQ_GET_DESCRIPTOR, USB_DIR_IN,
(type << 8) + index, 0, buf, size,
USB_CTRL_GET_TIMEOUT);
if (result <= 0 && result != -ETIMEDOUT)
continue;
if (result > 1 && ((u8 *)buf)[1] != type) {
result = -ENODATA;
continue;
}
break;
}
return result;
}
EXPORT_SYMBOL_GPL(usb_get_descriptor);
/**
* usb_get_string - gets a string descriptor
* @dev: the device whose string descriptor is being retrieved
* @langid: code for language chosen (from string descriptor zero)
* @index: the number of the descriptor
* @buf: where to put the string
* @size: how big is "buf"?
*
* Context: task context, might sleep.
*
* Retrieves a string, encoded using UTF-16LE (Unicode, 16 bits per character,
* in little-endian byte order).
* The usb_string() function will often be a convenient way to turn
* these strings into kernel-printable form.
*
* Strings may be referenced in device, configuration, interface, or other
* descriptors, and could also be used in vendor-specific ways.
*
* This call is synchronous, and may not be used in an interrupt context.
*
* Return: The number of bytes received on success, or else the status code
* returned by the underlying usb_control_msg() call.
*/
static int usb_get_string(struct usb_device *dev, unsigned short langid,
unsigned char index, void *buf, int size)
{
int i;
int result;
if (size <= 0) /* No point in asking for no data */
return -EINVAL;
for (i = 0; i < 3; ++i) {
/* retry on length 0 or stall; some devices are flakey */
result = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
USB_REQ_GET_DESCRIPTOR, USB_DIR_IN,
(USB_DT_STRING << 8) + index, langid, buf, size,
USB_CTRL_GET_TIMEOUT);
if (result == 0 || result == -EPIPE)
continue;
if (result > 1 && ((u8 *) buf)[1] != USB_DT_STRING) {
result = -ENODATA;
continue;
}
break;
}
return result;
}
static void usb_try_string_workarounds(unsigned char *buf, int *length)
{
int newlength, oldlength = *length;
for (newlength = 2; newlength + 1 < oldlength; newlength += 2) if (!isprint(buf[newlength]) || buf[newlength + 1])
break;
if (newlength > 2) { buf[0] = newlength;
*length = newlength;
}
}
static int usb_string_sub(struct usb_device *dev, unsigned int langid,
unsigned int index, unsigned char *buf)
{
int rc;
/* Try to read the string descriptor by asking for the maximum
* possible number of bytes */
if (dev->quirks & USB_QUIRK_STRING_FETCH_255)
rc = -EIO;
else
rc = usb_get_string(dev, langid, index, buf, 255);
/* If that failed try to read the descriptor length, then
* ask for just that many bytes */
if (rc < 2) {
rc = usb_get_string(dev, langid, index, buf, 2);
if (rc == 2)
rc = usb_get_string(dev, langid, index, buf, buf[0]);
}
if (rc >= 2) { if (!buf[0] && !buf[1]) usb_try_string_workarounds(buf, &rc);
/* There might be extra junk at the end of the descriptor */
if (buf[0] < rc)
rc = buf[0];
rc = rc - (rc & 1); /* force a multiple of two */
}
if (rc < 2)
rc = (rc < 0 ? rc : -EINVAL); return rc;
}
static int usb_get_langid(struct usb_device *dev, unsigned char *tbuf)
{
int err;
if (dev->have_langid)
return 0;
if (dev->string_langid < 0)
return -EPIPE;
err = usb_string_sub(dev, 0, 0, tbuf);
/* If the string was reported but is malformed, default to english
* (0x0409) */
if (err == -ENODATA || (err > 0 && err < 4)) { dev->string_langid = 0x0409;
dev->have_langid = 1;
dev_err(&dev->dev,
"language id specifier not provided by device, defaulting to English\n");
return 0;
}
/* In case of all other errors, we assume the device is not able to
* deal with strings at all. Set string_langid to -1 in order to
* prevent any string to be retrieved from the device */
if (err < 0) { dev_info(&dev->dev, "string descriptor 0 read error: %d\n",
err);
dev->string_langid = -1;
return -EPIPE;
}
/* always use the first langid listed */
dev->string_langid = tbuf[2] | (tbuf[3] << 8);
dev->have_langid = 1;
dev_dbg(&dev->dev, "default language 0x%04x\n",
dev->string_langid);
return 0;
}
/**
* usb_string - returns UTF-8 version of a string descriptor
* @dev: the device whose string descriptor is being retrieved
* @index: the number of the descriptor
* @buf: where to put the string
* @size: how big is "buf"?
*
* Context: task context, might sleep.
*
* This converts the UTF-16LE encoded strings returned by devices, from
* usb_get_string_descriptor(), to null-terminated UTF-8 encoded ones
* that are more usable in most kernel contexts. Note that this function
* chooses strings in the first language supported by the device.
*
* This call is synchronous, and may not be used in an interrupt context.
*
* Return: length of the string (>= 0) or usb_control_msg status (< 0).
*/
int usb_string(struct usb_device *dev, int index, char *buf, size_t size)
{
unsigned char *tbuf;
int err;
if (dev->state == USB_STATE_SUSPENDED)
return -EHOSTUNREACH;
if (size <= 0 || !buf)
return -EINVAL;
buf[0] = 0;
if (index <= 0 || index >= 256)
return -EINVAL;
tbuf = kmalloc(256, GFP_NOIO);
if (!tbuf)
return -ENOMEM;
err = usb_get_langid(dev, tbuf);
if (err < 0)
goto errout;
err = usb_string_sub(dev, dev->string_langid, index, tbuf);
if (err < 0)
goto errout;
size--; /* leave room for trailing NULL char in output buffer */
err = utf16s_to_utf8s((wchar_t *) &tbuf[2], (err - 2) / 2,
UTF16_LITTLE_ENDIAN, buf, size);
buf[err] = 0;
if (tbuf[1] != USB_DT_STRING)
dev_dbg(&dev->dev,
"wrong descriptor type %02x for string %d (\"%s\")\n",
tbuf[1], index, buf);
errout:
kfree(tbuf); return err;
}
EXPORT_SYMBOL_GPL(usb_string);
/* one UTF-8-encoded 16-bit character has at most three bytes */
#define MAX_USB_STRING_SIZE (127 * 3 + 1)
/**
* usb_cache_string - read a string descriptor and cache it for later use
* @udev: the device whose string descriptor is being read
* @index: the descriptor index
*
* Return: A pointer to a kmalloc'ed buffer containing the descriptor string,
* or %NULL if the index is 0 or the string could not be read.
*/
char *usb_cache_string(struct usb_device *udev, int index)
{
char *buf;
char *smallbuf = NULL;
int len;
if (index <= 0)
return NULL;
buf = kmalloc(MAX_USB_STRING_SIZE, GFP_NOIO);
if (buf) {
len = usb_string(udev, index, buf, MAX_USB_STRING_SIZE);
if (len > 0) {
smallbuf = kmalloc(++len, GFP_NOIO);
if (!smallbuf)
return buf;
memcpy(smallbuf, buf, len);
}
kfree(buf);
}
return smallbuf;
}
EXPORT_SYMBOL_GPL(usb_cache_string);
/*
* usb_get_device_descriptor - read the device descriptor
* @udev: the device whose device descriptor should be read
*
* Context: task context, might sleep.
*
* Not exported, only for use by the core. If drivers really want to read
* the device descriptor directly, they can call usb_get_descriptor() with
* type = USB_DT_DEVICE and index = 0.
*
* Returns: a pointer to a dynamically allocated usb_device_descriptor
* structure (which the caller must deallocate), or an ERR_PTR value.
*/
struct usb_device_descriptor *usb_get_device_descriptor(struct usb_device *udev)
{
struct usb_device_descriptor *desc;
int ret;
desc = kmalloc(sizeof(*desc), GFP_NOIO);
if (!desc)
return ERR_PTR(-ENOMEM);
ret = usb_get_descriptor(udev, USB_DT_DEVICE, 0, desc, sizeof(*desc));
if (ret == sizeof(*desc))
return desc;
if (ret >= 0)
ret = -EMSGSIZE;
kfree(desc); return ERR_PTR(ret);
}
/*
* usb_set_isoch_delay - informs the device of the packet transmit delay
* @dev: the device whose delay is to be informed
* Context: task context, might sleep
*
* Since this is an optional request, we don't bother if it fails.
*/
int usb_set_isoch_delay(struct usb_device *dev)
{
/* skip hub devices */
if (dev->descriptor.bDeviceClass == USB_CLASS_HUB)
return 0;
/* skip non-SS/non-SSP devices */
if (dev->speed < USB_SPEED_SUPER)
return 0;
return usb_control_msg_send(dev, 0,
USB_REQ_SET_ISOCH_DELAY,
USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE,
dev->hub_delay, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT,
GFP_NOIO);
}
/**
* usb_get_status - issues a GET_STATUS call
* @dev: the device whose status is being checked
* @recip: USB_RECIP_*; for device, interface, or endpoint
* @type: USB_STATUS_TYPE_*; for standard or PTM status types
* @target: zero (for device), else interface or endpoint number
* @data: pointer to two bytes of bitmap data
*
* Context: task context, might sleep.
*
* Returns device, interface, or endpoint status. Normally only of
* interest to see if the device is self powered, or has enabled the
* remote wakeup facility; or whether a bulk or interrupt endpoint
* is halted ("stalled").
*
* Bits in these status bitmaps are set using the SET_FEATURE request,
* and cleared using the CLEAR_FEATURE request. The usb_clear_halt()
* function should be used to clear halt ("stall") status.
*
* This call is synchronous, and may not be used in an interrupt context.
*
* Returns 0 and the status value in *@data (in host byte order) on success,
* or else the status code from the underlying usb_control_msg() call.
*/
int usb_get_status(struct usb_device *dev, int recip, int type, int target,
void *data)
{
int ret;
void *status;
int length;
switch (type) {
case USB_STATUS_TYPE_STANDARD:
length = 2;
break;
case USB_STATUS_TYPE_PTM:
if (recip != USB_RECIP_DEVICE)
return -EINVAL;
length = 4;
break;
default:
return -EINVAL;
}
status = kmalloc(length, GFP_KERNEL);
if (!status)
return -ENOMEM;
ret = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
USB_REQ_GET_STATUS, USB_DIR_IN | recip, USB_STATUS_TYPE_STANDARD,
target, status, length, USB_CTRL_GET_TIMEOUT);
switch (ret) {
case 4:
if (type != USB_STATUS_TYPE_PTM) { ret = -EIO;
break;
}
*(u32 *) data = le32_to_cpu(*(__le32 *) status);
ret = 0;
break;
case 2:
if (type != USB_STATUS_TYPE_STANDARD) {
ret = -EIO;
break;
}
*(u16 *) data = le16_to_cpu(*(__le16 *) status);
ret = 0;
break;
default:
ret = -EIO;
}
kfree(status);
return ret;
}
EXPORT_SYMBOL_GPL(usb_get_status);
/**
* usb_clear_halt - tells device to clear endpoint halt/stall condition
* @dev: device whose endpoint is halted
* @pipe: endpoint "pipe" being cleared
*
* Context: task context, might sleep.
*
* This is used to clear halt conditions for bulk and interrupt endpoints,
* as reported by URB completion status. Endpoints that are halted are
* sometimes referred to as being "stalled". Such endpoints are unable
* to transmit or receive data until the halt status is cleared. Any URBs
* queued for such an endpoint should normally be unlinked by the driver
* before clearing the halt condition, as described in sections 5.7.5
* and 5.8.5 of the USB 2.0 spec.
*
* Note that control and isochronous endpoints don't halt, although control
* endpoints report "protocol stall" (for unsupported requests) using the
* same status code used to report a true stall.
*
* This call is synchronous, and may not be used in an interrupt context.
* If a thread in your driver uses this call, make sure your disconnect()
* method can wait for it to complete.
*
* Return: Zero on success, or else the status code returned by the
* underlying usb_control_msg() call.
*/
int usb_clear_halt(struct usb_device *dev, int pipe)
{
int result;
int endp = usb_pipeendpoint(pipe);
if (usb_pipein(pipe))
endp |= USB_DIR_IN;
/* we don't care if it wasn't halted first. in fact some devices
* (like some ibmcam model 1 units) seem to expect hosts to make
* this request for iso endpoints, which can't halt!
*/
result = usb_control_msg_send(dev, 0,
USB_REQ_CLEAR_FEATURE, USB_RECIP_ENDPOINT,
USB_ENDPOINT_HALT, endp, NULL, 0,
USB_CTRL_SET_TIMEOUT, GFP_NOIO);
/* don't un-halt or force to DATA0 except on success */
if (result)
return result;
/* NOTE: seems like Microsoft and Apple don't bother verifying
* the clear "took", so some devices could lock up if you check...
* such as the Hagiwara FlashGate DUAL. So we won't bother.
*
* NOTE: make sure the logic here doesn't diverge much from
* the copy in usb-storage, for as long as we need two copies.
*/
usb_reset_endpoint(dev, endp);
return 0;
}
EXPORT_SYMBOL_GPL(usb_clear_halt);
static int create_intf_ep_devs(struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
struct usb_host_interface *alt = intf->cur_altsetting;
int i;
if (intf->ep_devs_created || intf->unregistering)
return 0;
for (i = 0; i < alt->desc.bNumEndpoints; ++i)
(void) usb_create_ep_devs(&intf->dev, &alt->endpoint[i], udev);
intf->ep_devs_created = 1;
return 0;
}
static void remove_intf_ep_devs(struct usb_interface *intf)
{
struct usb_host_interface *alt = intf->cur_altsetting;
int i;
if (!intf->ep_devs_created)
return;
for (i = 0; i < alt->desc.bNumEndpoints; ++i) usb_remove_ep_devs(&alt->endpoint[i]); intf->ep_devs_created = 0;
}
/**
* usb_disable_endpoint -- Disable an endpoint by address
* @dev: the device whose endpoint is being disabled
* @epaddr: the endpoint's address. Endpoint number for output,
* endpoint number + USB_DIR_IN for input
* @reset_hardware: flag to erase any endpoint state stored in the
* controller hardware
*
* Disables the endpoint for URB submission and nukes all pending URBs.
* If @reset_hardware is set then also deallocates hcd/hardware state
* for the endpoint.
*/
void usb_disable_endpoint(struct usb_device *dev, unsigned int epaddr,
bool reset_hardware)
{
unsigned int epnum = epaddr & USB_ENDPOINT_NUMBER_MASK;
struct usb_host_endpoint *ep;
if (!dev)
return;
if (usb_endpoint_out(epaddr)) { ep = dev->ep_out[epnum]; if (reset_hardware && epnum != 0) dev->ep_out[epnum] = NULL;
} else {
ep = dev->ep_in[epnum]; if (reset_hardware && epnum != 0) dev->ep_in[epnum] = NULL;
}
if (ep) { ep->enabled = 0;
usb_hcd_flush_endpoint(dev, ep);
if (reset_hardware)
usb_hcd_disable_endpoint(dev, ep);
}
}
/**
* usb_reset_endpoint - Reset an endpoint's state.
* @dev: the device whose endpoint is to be reset
* @epaddr: the endpoint's address. Endpoint number for output,
* endpoint number + USB_DIR_IN for input
*
* Resets any host-side endpoint state such as the toggle bit,
* sequence number or current window.
*/
void usb_reset_endpoint(struct usb_device *dev, unsigned int epaddr)
{
unsigned int epnum = epaddr & USB_ENDPOINT_NUMBER_MASK;
struct usb_host_endpoint *ep;
if (usb_endpoint_out(epaddr))
ep = dev->ep_out[epnum];
else
ep = dev->ep_in[epnum];
if (ep) usb_hcd_reset_endpoint(dev, ep);
}
EXPORT_SYMBOL_GPL(usb_reset_endpoint);
/**
* usb_disable_interface -- Disable all endpoints for an interface
* @dev: the device whose interface is being disabled
* @intf: pointer to the interface descriptor
* @reset_hardware: flag to erase any endpoint state stored in the
* controller hardware
*
* Disables all the endpoints for the interface's current altsetting.
*/
void usb_disable_interface(struct usb_device *dev, struct usb_interface *intf,
bool reset_hardware)
{
struct usb_host_interface *alt = intf->cur_altsetting;
int i;
for (i = 0; i < alt->desc.bNumEndpoints; ++i) { usb_disable_endpoint(dev, alt->endpoint[i].desc.bEndpointAddress,
reset_hardware);
}
}
/*
* usb_disable_device_endpoints -- Disable all endpoints for a device
* @dev: the device whose endpoints are being disabled
* @skip_ep0: 0 to disable endpoint 0, 1 to skip it.
*/
static void usb_disable_device_endpoints(struct usb_device *dev, int skip_ep0)
{
struct usb_hcd *hcd = bus_to_hcd(dev->bus);
int i;
if (hcd->driver->check_bandwidth) {
/* First pass: Cancel URBs, leave endpoint pointers intact. */
for (i = skip_ep0; i < 16; ++i) { usb_disable_endpoint(dev, i, false);
usb_disable_endpoint(dev, i + USB_DIR_IN, false);
}
/* Remove endpoints from the host controller internal state */
mutex_lock(hcd->bandwidth_mutex);
usb_hcd_alloc_bandwidth(dev, NULL, NULL, NULL);
mutex_unlock(hcd->bandwidth_mutex);
}
/* Second pass: remove endpoint pointers */
for (i = skip_ep0; i < 16; ++i) { usb_disable_endpoint(dev, i, true);
usb_disable_endpoint(dev, i + USB_DIR_IN, true);
}
}
/**
* usb_disable_device - Disable all the endpoints for a USB device
* @dev: the device whose endpoints are being disabled
* @skip_ep0: 0 to disable endpoint 0, 1 to skip it.
*
* Disables all the device's endpoints, potentially including endpoint 0.
* Deallocates hcd/hardware state for the endpoints (nuking all or most
* pending urbs) and usbcore state for the interfaces, so that usbcore
* must usb_set_configuration() before any interfaces could be used.
*/
void usb_disable_device(struct usb_device *dev, int skip_ep0)
{
int i;
/* getting rid of interfaces will disconnect
* any drivers bound to them (a key side effect)
*/
if (dev->actconfig) {
/*
* FIXME: In order to avoid self-deadlock involving the
* bandwidth_mutex, we have to mark all the interfaces
* before unregistering any of them.
*/
for (i = 0; i < dev->actconfig->desc.bNumInterfaces; i++) dev->actconfig->interface[i]->unregistering = 1; for (i = 0; i < dev->actconfig->desc.bNumInterfaces; i++) {
struct usb_interface *interface;
/* remove this interface if it has been registered */
interface = dev->actconfig->interface[i];
if (!device_is_registered(&interface->dev))
continue;
dev_dbg(&dev->dev, "unregistering interface %s\n",
dev_name(&interface->dev));
remove_intf_ep_devs(interface); device_del(&interface->dev);
}
/* Now that the interfaces are unbound, nobody should
* try to access them.
*/
for (i = 0; i < dev->actconfig->desc.bNumInterfaces; i++) { put_device(&dev->actconfig->interface[i]->dev);
dev->actconfig->interface[i] = NULL;
}
usb_disable_usb2_hardware_lpm(dev);
usb_unlocked_disable_lpm(dev);
usb_disable_ltm(dev);
dev->actconfig = NULL;
if (dev->state == USB_STATE_CONFIGURED)
usb_set_device_state(dev, USB_STATE_ADDRESS);
}
dev_dbg(&dev->dev, "%s nuking %s URBs\n", __func__,
skip_ep0 ? "non-ep0" : "all");
usb_disable_device_endpoints(dev, skip_ep0);
}
/**
* usb_enable_endpoint - Enable an endpoint for USB communications
* @dev: the device whose interface is being enabled
* @ep: the endpoint
* @reset_ep: flag to reset the endpoint state
*
* Resets the endpoint state if asked, and sets dev->ep_{in,out} pointers.
* For control endpoints, both the input and output sides are handled.
*/
void usb_enable_endpoint(struct usb_device *dev, struct usb_host_endpoint *ep,
bool reset_ep)
{
int epnum = usb_endpoint_num(&ep->desc);
int is_out = usb_endpoint_dir_out(&ep->desc);
int is_control = usb_endpoint_xfer_control(&ep->desc);
if (reset_ep)
usb_hcd_reset_endpoint(dev, ep); if (is_out || is_control) dev->ep_out[epnum] = ep; if (!is_out || is_control) dev->ep_in[epnum] = ep; ep->enabled = 1;
}
/**
* usb_enable_interface - Enable all the endpoints for an interface
* @dev: the device whose interface is being enabled
* @intf: pointer to the interface descriptor
* @reset_eps: flag to reset the endpoints' state
*
* Enables all the endpoints for the interface's current altsetting.
*/
void usb_enable_interface(struct usb_device *dev,
struct usb_interface *intf, bool reset_eps)
{
struct usb_host_interface *alt = intf->cur_altsetting;
int i;
for (i = 0; i < alt->desc.bNumEndpoints; ++i)
usb_enable_endpoint(dev, &alt->endpoint[i], reset_eps);
}
/**
* usb_set_interface - Makes a particular alternate setting be current
* @dev: the device whose interface is being updated
* @interface: the interface being updated
* @alternate: the setting being chosen.
*
* Context: task context, might sleep.
*
* This is used to enable data transfers on interfaces that may not
* be enabled by default. Not all devices support such configurability.
* Only the driver bound to an interface may change its setting.
*
* Within any given configuration, each interface may have several
* alternative settings. These are often used to control levels of
* bandwidth consumption. For example, the default setting for a high
* speed interrupt endpoint may not send more than 64 bytes per microframe,
* while interrupt transfers of up to 3KBytes per microframe are legal.
* Also, isochronous endpoints may never be part of an
* interface's default setting. To access such bandwidth, alternate
* interface settings must be made current.
*
* Note that in the Linux USB subsystem, bandwidth associated with
* an endpoint in a given alternate setting is not reserved until an URB
* is submitted that needs that bandwidth. Some other operating systems
* allocate bandwidth early, when a configuration is chosen.
*
* xHCI reserves bandwidth and configures the alternate setting in
* usb_hcd_alloc_bandwidth(). If it fails the original interface altsetting
* may be disabled. Drivers cannot rely on any particular alternate
* setting being in effect after a failure.
*
* This call is synchronous, and may not be used in an interrupt context.
* Also, drivers must not change altsettings while urbs are scheduled for
* endpoints in that interface; all such urbs must first be completed
* (perhaps forced by unlinking). If a thread in your driver uses this call,
* make sure your disconnect() method can wait for it to complete.
*
* Return: Zero on success, or else the status code returned by the
* underlying usb_control_msg() call.
*/
int usb_set_interface(struct usb_device *dev, int interface, int alternate)
{
struct usb_interface *iface;
struct usb_host_interface *alt;
struct usb_hcd *hcd = bus_to_hcd(dev->bus);
int i, ret, manual = 0;
unsigned int epaddr;
unsigned int pipe;
if (dev->state == USB_STATE_SUSPENDED)
return -EHOSTUNREACH;
iface = usb_ifnum_to_if(dev, interface);
if (!iface) {
dev_dbg(&dev->dev, "selecting invalid interface %d\n",
interface);
return -EINVAL;
}
if (iface->unregistering)
return -ENODEV;
alt = usb_altnum_to_altsetting(iface, alternate);
if (!alt) {
dev_warn(&dev->dev, "selecting invalid altsetting %d\n",
alternate);
return -EINVAL;
}
/*
* usb3 hosts configure the interface in usb_hcd_alloc_bandwidth,
* including freeing dropped endpoint ring buffers.
* Make sure the interface endpoints are flushed before that
*/
usb_disable_interface(dev, iface, false);
/* Make sure we have enough bandwidth for this alternate interface.
* Remove the current alt setting and add the new alt setting.
*/
mutex_lock(hcd->bandwidth_mutex);
/* Disable LPM, and re-enable it once the new alt setting is installed,
* so that the xHCI driver can recalculate the U1/U2 timeouts.
*/
if (usb_disable_lpm(dev)) {
dev_err(&iface->dev, "%s Failed to disable LPM\n", __func__);
mutex_unlock(hcd->bandwidth_mutex);
return -ENOMEM;
}
/* Changing alt-setting also frees any allocated streams */
for (i = 0; i < iface->cur_altsetting->desc.bNumEndpoints; i++) iface->cur_altsetting->endpoint[i].streams = 0; ret = usb_hcd_alloc_bandwidth(dev, NULL, iface->cur_altsetting, alt);
if (ret < 0) {
dev_info(&dev->dev, "Not enough bandwidth for altsetting %d\n",
alternate);
usb_enable_lpm(dev);
mutex_unlock(hcd->bandwidth_mutex);
return ret;
}
if (dev->quirks & USB_QUIRK_NO_SET_INTF)
ret = -EPIPE;
else
ret = usb_control_msg_send(dev, 0,
USB_REQ_SET_INTERFACE,
USB_RECIP_INTERFACE, alternate,
interface, NULL, 0, 5000,
GFP_NOIO);
/* 9.4.10 says devices don't need this and are free to STALL the
* request if the interface only has one alternate setting.
*/
if (ret == -EPIPE && iface->num_altsetting == 1) { dev_dbg(&dev->dev,
"manual set_interface for iface %d, alt %d\n",
interface, alternate);
manual = 1;
} else if (ret) {
/* Re-instate the old alt setting */
usb_hcd_alloc_bandwidth(dev, NULL, alt, iface->cur_altsetting);
usb_enable_lpm(dev);
mutex_unlock(hcd->bandwidth_mutex);
return ret;
}
mutex_unlock(hcd->bandwidth_mutex);
/* FIXME drivers shouldn't need to replicate/bugfix the logic here
* when they implement async or easily-killable versions of this or
* other "should-be-internal" functions (like clear_halt).
* should hcd+usbcore postprocess control requests?
*/
/* prevent submissions using previous endpoint settings */
if (iface->cur_altsetting != alt) {
remove_intf_ep_devs(iface); usb_remove_sysfs_intf_files(iface);
}
usb_disable_interface(dev, iface, true);
iface->cur_altsetting = alt;
/* Now that the interface is installed, re-enable LPM. */
usb_unlocked_enable_lpm(dev);
/* If the interface only has one altsetting and the device didn't
* accept the request, we attempt to carry out the equivalent action
* by manually clearing the HALT feature for each endpoint in the
* new altsetting.
*/
if (manual) {
for (i = 0; i < alt->desc.bNumEndpoints; i++) { epaddr = alt->endpoint[i].desc.bEndpointAddress;
pipe = __create_pipe(dev,
USB_ENDPOINT_NUMBER_MASK & epaddr) |
(usb_endpoint_out(epaddr) ?
USB_DIR_OUT : USB_DIR_IN);
usb_clear_halt(dev, pipe);
}
}
/* 9.1.1.5: reset toggles for all endpoints in the new altsetting
*
* Note:
* Despite EP0 is always present in all interfaces/AS, the list of
* endpoints from the descriptor does not contain EP0. Due to its
* omnipresence one might expect EP0 being considered "affected" by
* any SetInterface request and hence assume toggles need to be reset.
* However, EP0 toggles are re-synced for every individual transfer
* during the SETUP stage - hence EP0 toggles are "don't care" here.
* (Likewise, EP0 never "halts" on well designed devices.)
*/
usb_enable_interface(dev, iface, true); if (device_is_registered(&iface->dev)) { usb_create_sysfs_intf_files(iface);
create_intf_ep_devs(iface);
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_set_interface);
/**
* usb_reset_configuration - lightweight device reset
* @dev: the device whose configuration is being reset
*
* This issues a standard SET_CONFIGURATION request to the device using
* the current configuration. The effect is to reset most USB-related
* state in the device, including interface altsettings (reset to zero),
* endpoint halts (cleared), and endpoint state (only for bulk and interrupt
* endpoints). Other usbcore state is unchanged, including bindings of
* usb device drivers to interfaces.
*
* Because this affects multiple interfaces, avoid using this with composite
* (multi-interface) devices. Instead, the driver for each interface may
* use usb_set_interface() on the interfaces it claims. Be careful though;
* some devices don't support the SET_INTERFACE request, and others won't
* reset all the interface state (notably endpoint state). Resetting the whole
* configuration would affect other drivers' interfaces.
*
* The caller must own the device lock.
*
* Return: Zero on success, else a negative error code.
*
* If this routine fails the device will probably be in an unusable state
* with endpoints disabled, and interfaces only partially enabled.
*/
int usb_reset_configuration(struct usb_device *dev)
{
int i, retval;
struct usb_host_config *config;
struct usb_hcd *hcd = bus_to_hcd(dev->bus);
if (dev->state == USB_STATE_SUSPENDED)
return -EHOSTUNREACH;
/* caller must have locked the device and must own
* the usb bus readlock (so driver bindings are stable);
* calls during probe() are fine
*/
usb_disable_device_endpoints(dev, 1); /* skip ep0*/
config = dev->actconfig;
retval = 0;
mutex_lock(hcd->bandwidth_mutex);
/* Disable LPM, and re-enable it once the configuration is reset, so
* that the xHCI driver can recalculate the U1/U2 timeouts.
*/
if (usb_disable_lpm(dev)) {
dev_err(&dev->dev, "%s Failed to disable LPM\n", __func__);
mutex_unlock(hcd->bandwidth_mutex);
return -ENOMEM;
}
/* xHCI adds all endpoints in usb_hcd_alloc_bandwidth */
retval = usb_hcd_alloc_bandwidth(dev, config, NULL, NULL);
if (retval < 0) {
usb_enable_lpm(dev);
mutex_unlock(hcd->bandwidth_mutex);
return retval;
}
retval = usb_control_msg_send(dev, 0, USB_REQ_SET_CONFIGURATION, 0,
config->desc.bConfigurationValue, 0,
NULL, 0, USB_CTRL_SET_TIMEOUT,
GFP_NOIO);
if (retval) {
usb_hcd_alloc_bandwidth(dev, NULL, NULL, NULL);
usb_enable_lpm(dev);
mutex_unlock(hcd->bandwidth_mutex);
return retval;
}
mutex_unlock(hcd->bandwidth_mutex);
/* re-init hc/hcd interface/endpoint state */
for (i = 0; i < config->desc.bNumInterfaces; i++) {
struct usb_interface *intf = config->interface[i];
struct usb_host_interface *alt;
alt = usb_altnum_to_altsetting(intf, 0);
/* No altsetting 0? We'll assume the first altsetting.
* We could use a GetInterface call, but if a device is
* so non-compliant that it doesn't have altsetting 0
* then I wouldn't trust its reply anyway.
*/
if (!alt)
alt = &intf->altsetting[0];
if (alt != intf->cur_altsetting) {
remove_intf_ep_devs(intf);
usb_remove_sysfs_intf_files(intf);
}
intf->cur_altsetting = alt;
usb_enable_interface(dev, intf, true);
if (device_is_registered(&intf->dev)) {
usb_create_sysfs_intf_files(intf);
create_intf_ep_devs(intf);
}
}
/* Now that the interfaces are installed, re-enable LPM. */
usb_unlocked_enable_lpm(dev);
return 0;
}
EXPORT_SYMBOL_GPL(usb_reset_configuration);
static void usb_release_interface(struct device *dev)
{
struct usb_interface *intf = to_usb_interface(dev);
struct usb_interface_cache *intfc =
altsetting_to_usb_interface_cache(intf->altsetting);
kref_put(&intfc->ref, usb_release_interface_cache); usb_put_dev(interface_to_usbdev(intf));
of_node_put(dev->of_node);
kfree(intf);
}
/*
* usb_deauthorize_interface - deauthorize an USB interface
*
* @intf: USB interface structure
*/
void usb_deauthorize_interface(struct usb_interface *intf)
{
struct device *dev = &intf->dev;
device_lock(dev->parent);
if (intf->authorized) {
device_lock(dev);
intf->authorized = 0;
device_unlock(dev);
usb_forced_unbind_intf(intf);
}
device_unlock(dev->parent);
}
/*
* usb_authorize_interface - authorize an USB interface
*
* @intf: USB interface structure
*/
void usb_authorize_interface(struct usb_interface *intf)
{
struct device *dev = &intf->dev;
if (!intf->authorized) {
device_lock(dev);
intf->authorized = 1; /* authorize interface */
device_unlock(dev);
}
}
static int usb_if_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct usb_device *usb_dev;
const struct usb_interface *intf;
const struct usb_host_interface *alt;
intf = to_usb_interface(dev);
usb_dev = interface_to_usbdev(intf);
alt = intf->cur_altsetting;
if (add_uevent_var(env, "INTERFACE=%d/%d/%d",
alt->desc.bInterfaceClass,
alt->desc.bInterfaceSubClass,
alt->desc.bInterfaceProtocol))
return -ENOMEM;
if (add_uevent_var(env,
"MODALIAS=usb:"
"v%04Xp%04Xd%04Xdc%02Xdsc%02Xdp%02Xic%02Xisc%02Xip%02Xin%02X",
le16_to_cpu(usb_dev->descriptor.idVendor),
le16_to_cpu(usb_dev->descriptor.idProduct),
le16_to_cpu(usb_dev->descriptor.bcdDevice),
usb_dev->descriptor.bDeviceClass,
usb_dev->descriptor.bDeviceSubClass,
usb_dev->descriptor.bDeviceProtocol,
alt->desc.bInterfaceClass,
alt->desc.bInterfaceSubClass,
alt->desc.bInterfaceProtocol,
alt->desc.bInterfaceNumber))
return -ENOMEM;
return 0;
}
const struct device_type usb_if_device_type = {
.name = "usb_interface",
.release = usb_release_interface,
.uevent = usb_if_uevent,
};
static struct usb_interface_assoc_descriptor *find_iad(struct usb_device *dev,
struct usb_host_config *config,
u8 inum)
{
struct usb_interface_assoc_descriptor *retval = NULL;
struct usb_interface_assoc_descriptor *intf_assoc;
int first_intf;
int last_intf;
int i;
for (i = 0; (i < USB_MAXIADS && config->intf_assoc[i]); i++) { intf_assoc = config->intf_assoc[i];
if (intf_assoc->bInterfaceCount == 0)
continue;
first_intf = intf_assoc->bFirstInterface; last_intf = first_intf + (intf_assoc->bInterfaceCount - 1);
if (inum >= first_intf && inum <= last_intf) {
if (!retval)
retval = intf_assoc;
else
dev_err(&dev->dev, "Interface #%d referenced"
" by multiple IADs\n", inum);
}
}
return retval;
}
/*
* Internal function to queue a device reset
* See usb_queue_reset_device() for more details
*/
static void __usb_queue_reset_device(struct work_struct *ws)
{
int rc;
struct usb_interface *iface =
container_of(ws, struct usb_interface, reset_ws);
struct usb_device *udev = interface_to_usbdev(iface);
rc = usb_lock_device_for_reset(udev, iface);
if (rc >= 0) {
usb_reset_device(udev);
usb_unlock_device(udev);
}
usb_put_intf(iface); /* Undo _get_ in usb_queue_reset_device() */
}
/*
* Internal function to set the wireless_status sysfs attribute
* See usb_set_wireless_status() for more details
*/
static void __usb_wireless_status_intf(struct work_struct *ws)
{
struct usb_interface *iface =
container_of(ws, struct usb_interface, wireless_status_work);
device_lock(iface->dev.parent);
if (iface->sysfs_files_created)
usb_update_wireless_status_attr(iface);
device_unlock(iface->dev.parent);
usb_put_intf(iface); /* Undo _get_ in usb_set_wireless_status() */
}
/**
* usb_set_wireless_status - sets the wireless_status struct member
* @iface: the interface to modify
* @status: the new wireless status
*
* Set the wireless_status struct member to the new value, and emit
* sysfs changes as necessary.
*
* Returns: 0 on success, -EALREADY if already set.
*/
int usb_set_wireless_status(struct usb_interface *iface,
enum usb_wireless_status status)
{
if (iface->wireless_status == status)
return -EALREADY;
usb_get_intf(iface);
iface->wireless_status = status;
schedule_work(&iface->wireless_status_work);
return 0;
}
EXPORT_SYMBOL_GPL(usb_set_wireless_status);
/*
* usb_set_configuration - Makes a particular device setting be current
* @dev: the device whose configuration is being updated
* @configuration: the configuration being chosen.
*
* Context: task context, might sleep. Caller holds device lock.
*
* This is used to enable non-default device modes. Not all devices
* use this kind of configurability; many devices only have one
* configuration.
*
* @configuration is the value of the configuration to be installed.
* According to the USB spec (e.g. section 9.1.1.5), configuration values
* must be non-zero; a value of zero indicates that the device in
* unconfigured. However some devices erroneously use 0 as one of their
* configuration values. To help manage such devices, this routine will
* accept @configuration = -1 as indicating the device should be put in
* an unconfigured state.
*
* USB device configurations may affect Linux interoperability,
* power consumption and the functionality available. For example,
* the default configuration is limited to using 100mA of bus power,
* so that when certain device functionality requires more power,
* and the device is bus powered, that functionality should be in some
* non-default device configuration. Other device modes may also be
* reflected as configuration options, such as whether two ISDN
* channels are available independently; and choosing between open
* standard device protocols (like CDC) or proprietary ones.
*
* Note that a non-authorized device (dev->authorized == 0) will only
* be put in unconfigured mode.
*
* Note that USB has an additional level of device configurability,
* associated with interfaces. That configurability is accessed using
* usb_set_interface().
*
* This call is synchronous. The calling context must be able to sleep,
* must own the device lock, and must not hold the driver model's USB
* bus mutex; usb interface driver probe() methods cannot use this routine.
*
* Returns zero on success, or else the status code returned by the
* underlying call that failed. On successful completion, each interface
* in the original device configuration has been destroyed, and each one
* in the new configuration has been probed by all relevant usb device
* drivers currently known to the kernel.
*/
int usb_set_configuration(struct usb_device *dev, int configuration)
{
int i, ret;
struct usb_host_config *cp = NULL;
struct usb_interface **new_interfaces = NULL;
struct usb_hcd *hcd = bus_to_hcd(dev->bus);
int n, nintf;
if (dev->authorized == 0 || configuration == -1)
configuration = 0;
else {
for (i = 0; i < dev->descriptor.bNumConfigurations; i++) { if (dev->config[i].desc.bConfigurationValue ==
configuration) {
cp = &dev->config[i];
break;
}
}
}
if ((!cp && configuration != 0))
return -EINVAL;
/* The USB spec says configuration 0 means unconfigured.
* But if a device includes a configuration numbered 0,
* we will accept it as a correctly configured state.
* Use -1 if you really want to unconfigure the device.
*/
if (cp && configuration == 0) dev_warn(&dev->dev, "config 0 descriptor??\n");
/* Allocate memory for new interfaces before doing anything else,
* so that if we run out then nothing will have changed. */
n = nintf = 0;
if (cp) {
nintf = cp->desc.bNumInterfaces;
new_interfaces = kmalloc_array(nintf, sizeof(*new_interfaces),
GFP_NOIO);
if (!new_interfaces)
return -ENOMEM;
for (; n < nintf; ++n) { new_interfaces[n] = kzalloc(
sizeof(struct usb_interface),
GFP_NOIO);
if (!new_interfaces[n]) {
ret = -ENOMEM;
free_interfaces:
while (--n >= 0) kfree(new_interfaces[n]); kfree(new_interfaces);
return ret;
}
}
i = dev->bus_mA - usb_get_max_power(dev, cp);
if (i < 0)
dev_warn(&dev->dev, "new config #%d exceeds power "
"limit by %dmA\n",
configuration, -i);
}
/* Wake up the device so we can send it the Set-Config request */
ret = usb_autoresume_device(dev);
if (ret)
goto free_interfaces;
/* if it's already configured, clear out old state first.
* getting rid of old interfaces means unbinding their drivers.
*/
if (dev->state != USB_STATE_ADDRESS) usb_disable_device(dev, 1); /* Skip ep0 */
/* Get rid of pending async Set-Config requests for this device */
cancel_async_set_config(dev);
/* Make sure we have bandwidth (and available HCD resources) for this
* configuration. Remove endpoints from the schedule if we're dropping
* this configuration to set configuration 0. After this point, the
* host controller will not allow submissions to dropped endpoints. If
* this call fails, the device state is unchanged.
*/
mutex_lock(hcd->bandwidth_mutex);
/* Disable LPM, and re-enable it once the new configuration is
* installed, so that the xHCI driver can recalculate the U1/U2
* timeouts.
*/
if (dev->actconfig && usb_disable_lpm(dev)) { dev_err(&dev->dev, "%s Failed to disable LPM\n", __func__);
mutex_unlock(hcd->bandwidth_mutex);
ret = -ENOMEM;
goto free_interfaces;
}
ret = usb_hcd_alloc_bandwidth(dev, cp, NULL, NULL);
if (ret < 0) {
if (dev->actconfig) usb_enable_lpm(dev); mutex_unlock(hcd->bandwidth_mutex);
usb_autosuspend_device(dev);
goto free_interfaces;
}
/*
* Initialize the new interface structures and the
* hc/hcd/usbcore interface/endpoint state.
*/
for (i = 0; i < nintf; ++i) {
struct usb_interface_cache *intfc;
struct usb_interface *intf;
struct usb_host_interface *alt;
u8 ifnum;
cp->interface[i] = intf = new_interfaces[i];
intfc = cp->intf_cache[i];
intf->altsetting = intfc->altsetting;
intf->num_altsetting = intfc->num_altsetting;
intf->authorized = !!HCD_INTF_AUTHORIZED(hcd);
kref_get(&intfc->ref); alt = usb_altnum_to_altsetting(intf, 0);
/* No altsetting 0? We'll assume the first altsetting.
* We could use a GetInterface call, but if a device is
* so non-compliant that it doesn't have altsetting 0
* then I wouldn't trust its reply anyway.
*/
if (!alt)
alt = &intf->altsetting[0]; ifnum = alt->desc.bInterfaceNumber; intf->intf_assoc = find_iad(dev, cp, ifnum);
intf->cur_altsetting = alt;
usb_enable_interface(dev, intf, true);
intf->dev.parent = &dev->dev;
if (usb_of_has_combined_node(dev)) {
device_set_of_node_from_dev(&intf->dev, &dev->dev);
} else {
intf->dev.of_node = usb_of_get_interface_node(dev,
configuration, ifnum);
}
ACPI_COMPANION_SET(&intf->dev, ACPI_COMPANION(&dev->dev));
intf->dev.driver = NULL;
intf->dev.bus = &usb_bus_type;
intf->dev.type = &usb_if_device_type;
intf->dev.groups = usb_interface_groups;
INIT_WORK(&intf->reset_ws, __usb_queue_reset_device);
INIT_WORK(&intf->wireless_status_work, __usb_wireless_status_intf);
intf->minor = -1;
device_initialize(&intf->dev);
pm_runtime_no_callbacks(&intf->dev);
dev_set_name(&intf->dev, "%d-%s:%d.%d", dev->bus->busnum,
dev->devpath, configuration, ifnum);
usb_get_dev(dev);
}
kfree(new_interfaces);
ret = usb_control_msg_send(dev, 0, USB_REQ_SET_CONFIGURATION, 0,
configuration, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT, GFP_NOIO);
if (ret && cp) {
/*
* All the old state is gone, so what else can we do?
* The device is probably useless now anyway.
*/
usb_hcd_alloc_bandwidth(dev, NULL, NULL, NULL);
for (i = 0; i < nintf; ++i) {
usb_disable_interface(dev, cp->interface[i], true);
put_device(&cp->interface[i]->dev);
cp->interface[i] = NULL;
}
cp = NULL;
}
dev->actconfig = cp;
mutex_unlock(hcd->bandwidth_mutex);
if (!cp) {
usb_set_device_state(dev, USB_STATE_ADDRESS);
/* Leave LPM disabled while the device is unconfigured. */
usb_autosuspend_device(dev);
return ret;
}
usb_set_device_state(dev, USB_STATE_CONFIGURED);
if (cp->string == NULL &&
!(dev->quirks & USB_QUIRK_CONFIG_INTF_STRINGS)) cp->string = usb_cache_string(dev, cp->desc.iConfiguration);
/* Now that the interfaces are installed, re-enable LPM. */
usb_unlocked_enable_lpm(dev);
/* Enable LTM if it was turned off by usb_disable_device. */
usb_enable_ltm(dev);
/* Now that all the interfaces are set up, register them
* to trigger binding of drivers to interfaces. probe()
* routines may install different altsettings and may
* claim() any interfaces not yet bound. Many class drivers
* need that: CDC, audio, video, etc.
*/
for (i = 0; i < nintf; ++i) { struct usb_interface *intf = cp->interface[i];
if (intf->dev.of_node &&
!of_device_is_available(intf->dev.of_node)) { dev_info(&dev->dev, "skipping disabled interface %d\n",
intf->cur_altsetting->desc.bInterfaceNumber);
continue;
}
dev_dbg(&dev->dev,
"adding %s (config #%d, interface %d)\n",
dev_name(&intf->dev), configuration,
intf->cur_altsetting->desc.bInterfaceNumber);
device_enable_async_suspend(&intf->dev); ret = device_add(&intf->dev);
if (ret != 0) {
dev_err(&dev->dev, "device_add(%s) --> %d\n",
dev_name(&intf->dev), ret);
continue;
}
create_intf_ep_devs(intf);
}
usb_autosuspend_device(dev); return 0;
}
EXPORT_SYMBOL_GPL(usb_set_configuration);
static LIST_HEAD(set_config_list);
static DEFINE_SPINLOCK(set_config_lock);
struct set_config_request {
struct usb_device *udev;
int config;
struct work_struct work;
struct list_head node;
};
/* Worker routine for usb_driver_set_configuration() */
static void driver_set_config_work(struct work_struct *work)
{
struct set_config_request *req =
container_of(work, struct set_config_request, work);
struct usb_device *udev = req->udev;
usb_lock_device(udev);
spin_lock(&set_config_lock);
list_del(&req->node);
spin_unlock(&set_config_lock);
if (req->config >= -1) /* Is req still valid? */
usb_set_configuration(udev, req->config);
usb_unlock_device(udev);
usb_put_dev(udev);
kfree(req);
}
/* Cancel pending Set-Config requests for a device whose configuration
* was just changed
*/
static void cancel_async_set_config(struct usb_device *udev)
{
struct set_config_request *req;
spin_lock(&set_config_lock); list_for_each_entry(req, &set_config_list, node) { if (req->udev == udev) req->config = -999; /* Mark as cancelled */
}
spin_unlock(&set_config_lock);
}
/**
* usb_driver_set_configuration - Provide a way for drivers to change device configurations
* @udev: the device whose configuration is being updated
* @config: the configuration being chosen.
* Context: In process context, must be able to sleep
*
* Device interface drivers are not allowed to change device configurations.
* This is because changing configurations will destroy the interface the
* driver is bound to and create new ones; it would be like a floppy-disk
* driver telling the computer to replace the floppy-disk drive with a
* tape drive!
*
* Still, in certain specialized circumstances the need may arise. This
* routine gets around the normal restrictions by using a work thread to
* submit the change-config request.
*
* Return: 0 if the request was successfully queued, error code otherwise.
* The caller has no way to know whether the queued request will eventually
* succeed.
*/
int usb_driver_set_configuration(struct usb_device *udev, int config)
{
struct set_config_request *req;
req = kmalloc(sizeof(*req), GFP_KERNEL);
if (!req)
return -ENOMEM;
req->udev = udev;
req->config = config;
INIT_WORK(&req->work, driver_set_config_work);
spin_lock(&set_config_lock);
list_add(&req->node, &set_config_list);
spin_unlock(&set_config_lock);
usb_get_dev(udev);
schedule_work(&req->work);
return 0;
}
EXPORT_SYMBOL_GPL(usb_driver_set_configuration);
/**
* cdc_parse_cdc_header - parse the extra headers present in CDC devices
* @hdr: the place to put the results of the parsing
* @intf: the interface for which parsing is requested
* @buffer: pointer to the extra headers to be parsed
* @buflen: length of the extra headers
*
* This evaluates the extra headers present in CDC devices which
* bind the interfaces for data and control and provide details
* about the capabilities of the device.
*
* Return: number of descriptors parsed or -EINVAL
* if the header is contradictory beyond salvage
*/
int cdc_parse_cdc_header(struct usb_cdc_parsed_header *hdr,
struct usb_interface *intf,
u8 *buffer,
int buflen)
{
/* duplicates are ignored */
struct usb_cdc_union_desc *union_header = NULL;
/* duplicates are not tolerated */
struct usb_cdc_header_desc *header = NULL;
struct usb_cdc_ether_desc *ether = NULL;
struct usb_cdc_mdlm_detail_desc *detail = NULL;
struct usb_cdc_mdlm_desc *desc = NULL;
unsigned int elength;
int cnt = 0;
memset(hdr, 0x00, sizeof(struct usb_cdc_parsed_header));
hdr->phonet_magic_present = false;
while (buflen > 0) {
elength = buffer[0];
if (!elength) {
dev_err(&intf->dev, "skipping garbage byte\n");
elength = 1;
goto next_desc;
}
if ((buflen < elength) || (elength < 3)) {
dev_err(&intf->dev, "invalid descriptor buffer length\n");
break;
}
if (buffer[1] != USB_DT_CS_INTERFACE) {
dev_err(&intf->dev, "skipping garbage\n");
goto next_desc;
}
switch (buffer[2]) {
case USB_CDC_UNION_TYPE: /* we've found it */
if (elength < sizeof(struct usb_cdc_union_desc))
goto next_desc;
if (union_header) {
dev_err(&intf->dev, "More than one union descriptor, skipping ...\n");
goto next_desc;
}
union_header = (struct usb_cdc_union_desc *)buffer;
break;
case USB_CDC_COUNTRY_TYPE:
if (elength < sizeof(struct usb_cdc_country_functional_desc))
goto next_desc;
hdr->usb_cdc_country_functional_desc =
(struct usb_cdc_country_functional_desc *)buffer;
break;
case USB_CDC_HEADER_TYPE:
if (elength != sizeof(struct usb_cdc_header_desc))
goto next_desc;
if (header)
return -EINVAL;
header = (struct usb_cdc_header_desc *)buffer;
break;
case USB_CDC_ACM_TYPE:
if (elength < sizeof(struct usb_cdc_acm_descriptor))
goto next_desc;
hdr->usb_cdc_acm_descriptor =
(struct usb_cdc_acm_descriptor *)buffer;
break;
case USB_CDC_ETHERNET_TYPE:
if (elength != sizeof(struct usb_cdc_ether_desc))
goto next_desc;
if (ether)
return -EINVAL;
ether = (struct usb_cdc_ether_desc *)buffer;
break;
case USB_CDC_CALL_MANAGEMENT_TYPE:
if (elength < sizeof(struct usb_cdc_call_mgmt_descriptor))
goto next_desc;
hdr->usb_cdc_call_mgmt_descriptor =
(struct usb_cdc_call_mgmt_descriptor *)buffer;
break;
case USB_CDC_DMM_TYPE:
if (elength < sizeof(struct usb_cdc_dmm_desc))
goto next_desc;
hdr->usb_cdc_dmm_desc =
(struct usb_cdc_dmm_desc *)buffer;
break;
case USB_CDC_MDLM_TYPE:
if (elength < sizeof(struct usb_cdc_mdlm_desc))
goto next_desc;
if (desc)
return -EINVAL;
desc = (struct usb_cdc_mdlm_desc *)buffer;
break;
case USB_CDC_MDLM_DETAIL_TYPE:
if (elength < sizeof(struct usb_cdc_mdlm_detail_desc))
goto next_desc;
if (detail)
return -EINVAL;
detail = (struct usb_cdc_mdlm_detail_desc *)buffer;
break;
case USB_CDC_NCM_TYPE:
if (elength < sizeof(struct usb_cdc_ncm_desc))
goto next_desc;
hdr->usb_cdc_ncm_desc = (struct usb_cdc_ncm_desc *)buffer;
break;
case USB_CDC_MBIM_TYPE:
if (elength < sizeof(struct usb_cdc_mbim_desc))
goto next_desc;
hdr->usb_cdc_mbim_desc = (struct usb_cdc_mbim_desc *)buffer;
break;
case USB_CDC_MBIM_EXTENDED_TYPE:
if (elength < sizeof(struct usb_cdc_mbim_extended_desc))
break;
hdr->usb_cdc_mbim_extended_desc =
(struct usb_cdc_mbim_extended_desc *)buffer;
break;
case CDC_PHONET_MAGIC_NUMBER:
hdr->phonet_magic_present = true;
break;
default:
/*
* there are LOTS more CDC descriptors that
* could legitimately be found here.
*/
dev_dbg(&intf->dev, "Ignoring descriptor: type %02x, length %ud\n",
buffer[2], elength);
goto next_desc;
}
cnt++;
next_desc:
buflen -= elength;
buffer += elength;
}
hdr->usb_cdc_union_desc = union_header;
hdr->usb_cdc_header_desc = header;
hdr->usb_cdc_mdlm_detail_desc = detail;
hdr->usb_cdc_mdlm_desc = desc;
hdr->usb_cdc_ether_desc = ether;
return cnt;
}
EXPORT_SYMBOL(cdc_parse_cdc_header);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Prevent the compiler from merging or refetching reads or writes. The
* compiler is also forbidden from reordering successive instances of
* READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
* particular ordering. One way to make the compiler aware of ordering is to
* put the two invocations of READ_ONCE or WRITE_ONCE in different C
* statements.
*
* These two macros will also work on aggregate data types like structs or
* unions.
*
* Their two major use cases are: (1) Mediating communication between
* process-level code and irq/NMI handlers, all running on the same CPU,
* and (2) Ensuring that the compiler does not fold, spindle, or otherwise
* mutilate accesses that either do not require ordering or that interact
* with an explicit memory barrier or atomic instruction that provides the
* required ordering.
*/
#ifndef __ASM_GENERIC_RWONCE_H
#define __ASM_GENERIC_RWONCE_H
#ifndef __ASSEMBLY__
#include <linux/compiler_types.h>
#include <linux/kasan-checks.h>
#include <linux/kcsan-checks.h>
/*
* Yes, this permits 64-bit accesses on 32-bit architectures. These will
* actually be atomic in some cases (namely Armv7 + LPAE), but for others we
* rely on the access being split into 2x32-bit accesses for a 32-bit quantity
* (e.g. a virtual address) and a strong prevailing wind.
*/
#define compiletime_assert_rwonce_type(t) \
compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
"Unsupported access size for {READ,WRITE}_ONCE().")
/*
* Use __READ_ONCE() instead of READ_ONCE() if you do not require any
* atomicity. Note that this may result in tears!
*/
#ifndef __READ_ONCE
#define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
#endif
#define READ_ONCE(x) \
({ \
compiletime_assert_rwonce_type(x); \
__READ_ONCE(x); \
})
#define __WRITE_ONCE(x, val) \
do { \
*(volatile typeof(x) *)&(x) = (val); \
} while (0)
#define WRITE_ONCE(x, val) \
do { \
compiletime_assert_rwonce_type(x); \
__WRITE_ONCE(x, val); \
} while (0)
static __no_sanitize_or_inline
unsigned long __read_once_word_nocheck(const void *addr)
{
return __READ_ONCE(*(unsigned long *)addr);
}
/*
* Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a
* word from memory atomically but without telling KASAN/KCSAN. This is
* usually used by unwinding code when walking the stack of a running process.
*/
#define READ_ONCE_NOCHECK(x) \
({ \
compiletime_assert(sizeof(x) == sizeof(unsigned long), \
"Unsupported access size for READ_ONCE_NOCHECK()."); \
(typeof(x))__read_once_word_nocheck(&(x)); \
})
static __no_kasan_or_inline
unsigned long read_word_at_a_time(const void *addr)
{
kasan_check_read(addr, 1);
return *(unsigned long *)addr;
}
#endif /* __ASSEMBLY__ */
#endif /* __ASM_GENERIC_RWONCE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_PGTABLE_INVERT_H
#define _ASM_PGTABLE_INVERT_H 1
#ifndef __ASSEMBLY__
/*
* A clear pte value is special, and doesn't get inverted.
*
* Note that even users that only pass a pgprot_t (rather
* than a full pte) won't trigger the special zero case,
* because even PAGE_NONE has _PAGE_PROTNONE | _PAGE_ACCESSED
* set. So the all zero case really is limited to just the
* cleared page table entry case.
*/
static inline bool __pte_needs_invert(u64 val)
{
return val && !(val & _PAGE_PRESENT);
}
/* Get a mask to xor with the page table entry to get the correct pfn. */
static inline u64 protnone_mask(u64 val)
{
return __pte_needs_invert(val) ? ~0ull : 0;
}
static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
{
/*
* When a PTE transitions from NONE to !NONE or vice-versa
* invert the PFN part to stop speculation.
* pte_pfn undoes this when needed.
*/
if (__pte_needs_invert(oldval) != __pte_needs_invert(val))
val = (val & ~mask) | (~val & mask);
return val;
}
#endif /* __ASSEMBLY__ */
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
* Copyright (C) 2005-2006, Thomas Gleixner, Russell King
*
* This file contains the core interrupt handling code, for irq-chip based
* architectures. Detailed information is available in
* Documentation/core-api/genericirq.rst
*/
#include <linux/irq.h>
#include <linux/msi.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/irqdomain.h>
#include <trace/events/irq.h>
#include "internals.h"
static irqreturn_t bad_chained_irq(int irq, void *dev_id)
{
WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
return IRQ_NONE;
}
/*
* Chained handlers should never call action on their IRQ. This default
* action will emit warning if such thing happens.
*/
struct irqaction chained_action = {
.handler = bad_chained_irq,
};
/**
* irq_set_chip - set the irq chip for an irq
* @irq: irq number
* @chip: pointer to irq chip description structure
*/
int irq_set_chip(unsigned int irq, const struct irq_chip *chip)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
if (!desc)
return -EINVAL;
desc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip);
irq_put_desc_unlock(desc, flags);
/*
* For !CONFIG_SPARSE_IRQ make the irq show up in
* allocated_irqs.
*/
irq_mark_irq(irq);
return 0;
}
EXPORT_SYMBOL(irq_set_chip);
/**
* irq_set_irq_type - set the irq trigger type for an irq
* @irq: irq number
* @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
*/
int irq_set_irq_type(unsigned int irq, unsigned int type)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
int ret = 0;
if (!desc)
return -EINVAL;
ret = __irq_set_trigger(desc, type);
irq_put_desc_busunlock(desc, flags);
return ret;
}
EXPORT_SYMBOL(irq_set_irq_type);
/**
* irq_set_handler_data - set irq handler data for an irq
* @irq: Interrupt number
* @data: Pointer to interrupt specific data
*
* Set the hardware irq controller data for an irq
*/
int irq_set_handler_data(unsigned int irq, void *data)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
if (!desc)
return -EINVAL;
desc->irq_common_data.handler_data = data;
irq_put_desc_unlock(desc, flags);
return 0;
}
EXPORT_SYMBOL(irq_set_handler_data);
/**
* irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
* @irq_base: Interrupt number base
* @irq_offset: Interrupt number offset
* @entry: Pointer to MSI descriptor data
*
* Set the MSI descriptor entry for an irq at offset
*/
int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
struct msi_desc *entry)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
if (!desc)
return -EINVAL;
desc->irq_common_data.msi_desc = entry;
if (entry && !irq_offset)
entry->irq = irq_base;
irq_put_desc_unlock(desc, flags);
return 0;
}
/**
* irq_set_msi_desc - set MSI descriptor data for an irq
* @irq: Interrupt number
* @entry: Pointer to MSI descriptor data
*
* Set the MSI descriptor entry for an irq
*/
int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
{
return irq_set_msi_desc_off(irq, 0, entry);
}
/**
* irq_set_chip_data - set irq chip data for an irq
* @irq: Interrupt number
* @data: Pointer to chip specific data
*
* Set the hardware irq chip data for an irq
*/
int irq_set_chip_data(unsigned int irq, void *data)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
if (!desc)
return -EINVAL;
desc->irq_data.chip_data = data;
irq_put_desc_unlock(desc, flags);
return 0;
}
EXPORT_SYMBOL(irq_set_chip_data);
struct irq_data *irq_get_irq_data(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq); return desc ? &desc->irq_data : NULL;
}
EXPORT_SYMBOL_GPL(irq_get_irq_data);
static void irq_state_clr_disabled(struct irq_desc *desc)
{
irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
}
static void irq_state_clr_masked(struct irq_desc *desc)
{
irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
}
static void irq_state_clr_started(struct irq_desc *desc)
{
irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
}
static void irq_state_set_started(struct irq_desc *desc)
{
irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
}
enum {
IRQ_STARTUP_NORMAL,
IRQ_STARTUP_MANAGED,
IRQ_STARTUP_ABORT,
};
#ifdef CONFIG_SMP
static int
__irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff,
bool force)
{
struct irq_data *d = irq_desc_get_irq_data(desc);
if (!irqd_affinity_is_managed(d))
return IRQ_STARTUP_NORMAL;
irqd_clr_managed_shutdown(d);
if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
/*
* Catch code which fiddles with enable_irq() on a managed
* and potentially shutdown IRQ. Chained interrupt
* installment or irq auto probing should not happen on
* managed irqs either.
*/
if (WARN_ON_ONCE(force))
return IRQ_STARTUP_ABORT;
/*
* The interrupt was requested, but there is no online CPU
* in it's affinity mask. Put it into managed shutdown
* state and let the cpu hotplug mechanism start it up once
* a CPU in the mask becomes available.
*/
return IRQ_STARTUP_ABORT;
}
/*
* Managed interrupts have reserved resources, so this should not
* happen.
*/
if (WARN_ON(irq_domain_activate_irq(d, false)))
return IRQ_STARTUP_ABORT;
return IRQ_STARTUP_MANAGED;
}
#else
static __always_inline int
__irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff,
bool force)
{
return IRQ_STARTUP_NORMAL;
}
#endif
static int __irq_startup(struct irq_desc *desc)
{
struct irq_data *d = irq_desc_get_irq_data(desc);
int ret = 0;
/* Warn if this interrupt is not activated but try nevertheless */
WARN_ON_ONCE(!irqd_is_activated(d));
if (d->chip->irq_startup) {
ret = d->chip->irq_startup(d);
irq_state_clr_disabled(desc);
irq_state_clr_masked(desc);
} else {
irq_enable(desc);
}
irq_state_set_started(desc);
return ret;
}
int irq_startup(struct irq_desc *desc, bool resend, bool force)
{
struct irq_data *d = irq_desc_get_irq_data(desc);
const struct cpumask *aff = irq_data_get_affinity_mask(d);
int ret = 0;
desc->depth = 0;
if (irqd_is_started(d)) {
irq_enable(desc);
} else {
switch (__irq_startup_managed(desc, aff, force)) {
case IRQ_STARTUP_NORMAL:
if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
irq_setup_affinity(desc);
ret = __irq_startup(desc);
if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
irq_setup_affinity(desc);
break;
case IRQ_STARTUP_MANAGED:
irq_do_set_affinity(d, aff, false);
ret = __irq_startup(desc);
break;
case IRQ_STARTUP_ABORT:
irqd_set_managed_shutdown(d);
return 0;
}
}
if (resend)
check_irq_resend(desc, false);
return ret;
}
int irq_activate(struct irq_desc *desc)
{
struct irq_data *d = irq_desc_get_irq_data(desc);
if (!irqd_affinity_is_managed(d))
return irq_domain_activate_irq(d, false);
return 0;
}
int irq_activate_and_startup(struct irq_desc *desc, bool resend)
{
if (WARN_ON(irq_activate(desc)))
return 0;
return irq_startup(desc, resend, IRQ_START_FORCE);
}
static void __irq_disable(struct irq_desc *desc, bool mask);
void irq_shutdown(struct irq_desc *desc)
{
if (irqd_is_started(&desc->irq_data)) {
clear_irq_resend(desc);
desc->depth = 1;
if (desc->irq_data.chip->irq_shutdown) {
desc->irq_data.chip->irq_shutdown(&desc->irq_data);
irq_state_set_disabled(desc);
irq_state_set_masked(desc);
} else {
__irq_disable(desc, true);
}
irq_state_clr_started(desc);
}
}
void irq_shutdown_and_deactivate(struct irq_desc *desc)
{
irq_shutdown(desc);
/*
* This must be called even if the interrupt was never started up,
* because the activation can happen before the interrupt is
* available for request/startup. It has it's own state tracking so
* it's safe to call it unconditionally.
*/
irq_domain_deactivate_irq(&desc->irq_data);
}
void irq_enable(struct irq_desc *desc)
{
if (!irqd_irq_disabled(&desc->irq_data)) {
unmask_irq(desc);
} else {
irq_state_clr_disabled(desc);
if (desc->irq_data.chip->irq_enable) {
desc->irq_data.chip->irq_enable(&desc->irq_data);
irq_state_clr_masked(desc);
} else {
unmask_irq(desc);
}
}
}
static void __irq_disable(struct irq_desc *desc, bool mask)
{
if (irqd_irq_disabled(&desc->irq_data)) {
if (mask)
mask_irq(desc);
} else {
irq_state_set_disabled(desc);
if (desc->irq_data.chip->irq_disable) {
desc->irq_data.chip->irq_disable(&desc->irq_data);
irq_state_set_masked(desc);
} else if (mask) {
mask_irq(desc);
}
}
}
/**
* irq_disable - Mark interrupt disabled
* @desc: irq descriptor which should be disabled
*
* If the chip does not implement the irq_disable callback, we
* use a lazy disable approach. That means we mark the interrupt
* disabled, but leave the hardware unmasked. That's an
* optimization because we avoid the hardware access for the
* common case where no interrupt happens after we marked it
* disabled. If an interrupt happens, then the interrupt flow
* handler masks the line at the hardware level and marks it
* pending.
*
* If the interrupt chip does not implement the irq_disable callback,
* a driver can disable the lazy approach for a particular irq line by
* calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
* be used for devices which cannot disable the interrupt at the
* device level under certain circumstances and have to use
* disable_irq[_nosync] instead.
*/
void irq_disable(struct irq_desc *desc)
{
__irq_disable(desc, irq_settings_disable_unlazy(desc));
}
void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
{
if (desc->irq_data.chip->irq_enable)
desc->irq_data.chip->irq_enable(&desc->irq_data);
else
desc->irq_data.chip->irq_unmask(&desc->irq_data);
cpumask_set_cpu(cpu, desc->percpu_enabled);
}
void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
{
if (desc->irq_data.chip->irq_disable)
desc->irq_data.chip->irq_disable(&desc->irq_data);
else
desc->irq_data.chip->irq_mask(&desc->irq_data);
cpumask_clear_cpu(cpu, desc->percpu_enabled);
}
static inline void mask_ack_irq(struct irq_desc *desc)
{
if (desc->irq_data.chip->irq_mask_ack) {
desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
irq_state_set_masked(desc);
} else {
mask_irq(desc); if (desc->irq_data.chip->irq_ack) desc->irq_data.chip->irq_ack(&desc->irq_data);
}
}
void mask_irq(struct irq_desc *desc)
{
if (irqd_irq_masked(&desc->irq_data))
return;
if (desc->irq_data.chip->irq_mask) { desc->irq_data.chip->irq_mask(&desc->irq_data);
irq_state_set_masked(desc);
}
}
void unmask_irq(struct irq_desc *desc)
{
if (!irqd_irq_masked(&desc->irq_data))
return;
if (desc->irq_data.chip->irq_unmask) { desc->irq_data.chip->irq_unmask(&desc->irq_data);
irq_state_clr_masked(desc);
}
}
void unmask_threaded_irq(struct irq_desc *desc)
{
struct irq_chip *chip = desc->irq_data.chip;
if (chip->flags & IRQCHIP_EOI_THREADED)
chip->irq_eoi(&desc->irq_data);
unmask_irq(desc);
}
/*
* handle_nested_irq - Handle a nested irq from a irq thread
* @irq: the interrupt number
*
* Handle interrupts which are nested into a threaded interrupt
* handler. The handler function is called inside the calling
* threads context.
*/
void handle_nested_irq(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
struct irqaction *action;
irqreturn_t action_ret;
might_sleep();
raw_spin_lock_irq(&desc->lock);
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
action = desc->action;
if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
raw_spin_unlock_irq(&desc->lock);
return;
}
kstat_incr_irqs_this_cpu(desc);
atomic_inc(&desc->threads_active);
raw_spin_unlock_irq(&desc->lock);
action_ret = IRQ_NONE;
for_each_action_of_desc(desc, action)
action_ret |= action->thread_fn(action->irq, action->dev_id);
if (!irq_settings_no_debug(desc))
note_interrupt(desc, action_ret);
wake_threads_waitq(desc);
}
EXPORT_SYMBOL_GPL(handle_nested_irq);
static bool irq_check_poll(struct irq_desc *desc)
{
if (!(desc->istate & IRQS_POLL_INPROGRESS))
return false;
return irq_wait_for_poll(desc);
}
static bool irq_may_run(struct irq_desc *desc)
{
unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
/*
* If the interrupt is not in progress and is not an armed
* wakeup interrupt, proceed.
*/
if (!irqd_has_set(&desc->irq_data, mask))
return true;
/*
* If the interrupt is an armed wakeup source, mark it pending
* and suspended, disable it and notify the pm core about the
* event.
*/
if (irq_pm_check_wakeup(desc))
return false;
/*
* Handle a potential concurrent poll on a different core.
*/
return irq_check_poll(desc);
}
/**
* handle_simple_irq - Simple and software-decoded IRQs.
* @desc: the interrupt description structure for this irq
*
* Simple interrupts are either sent from a demultiplexing interrupt
* handler or come from hardware, where no interrupt hardware control
* is necessary.
*
* Note: The caller is expected to handle the ack, clear, mask and
* unmask issues if necessary.
*/
void handle_simple_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
if (!irq_may_run(desc))
goto out_unlock;
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
goto out_unlock;
}
kstat_incr_irqs_this_cpu(desc);
handle_irq_event(desc);
out_unlock:
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL_GPL(handle_simple_irq);
/**
* handle_untracked_irq - Simple and software-decoded IRQs.
* @desc: the interrupt description structure for this irq
*
* Untracked interrupts are sent from a demultiplexing interrupt
* handler when the demultiplexer does not know which device it its
* multiplexed irq domain generated the interrupt. IRQ's handled
* through here are not subjected to stats tracking, randomness, or
* spurious interrupt detection.
*
* Note: Like handle_simple_irq, the caller is expected to handle
* the ack, clear, mask and unmask issues if necessary.
*/
void handle_untracked_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
if (!irq_may_run(desc))
goto out_unlock;
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
goto out_unlock;
}
desc->istate &= ~IRQS_PENDING;
irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
raw_spin_unlock(&desc->lock);
__handle_irq_event_percpu(desc);
raw_spin_lock(&desc->lock);
irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
out_unlock:
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL_GPL(handle_untracked_irq);
/*
* Called unconditionally from handle_level_irq() and only for oneshot
* interrupts from handle_fasteoi_irq()
*/
static void cond_unmask_irq(struct irq_desc *desc)
{
/*
* We need to unmask in the following cases:
* - Standard level irq (IRQF_ONESHOT is not set)
* - Oneshot irq which did not wake the thread (caused by a
* spurious interrupt or a primary handler handling it
* completely).
*/
if (!irqd_irq_disabled(&desc->irq_data) &&
irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
unmask_irq(desc);
}
/**
* handle_level_irq - Level type irq handler
* @desc: the interrupt description structure for this irq
*
* Level type interrupts are active as long as the hardware line has
* the active level. This may require to mask the interrupt and unmask
* it after the associated handler has acknowledged the device, so the
* interrupt line is back to inactive.
*/
void handle_level_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
mask_ack_irq(desc);
if (!irq_may_run(desc))
goto out_unlock;
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
/*
* If its disabled or no action available
* keep it masked and get out of here
*/
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
goto out_unlock;
}
kstat_incr_irqs_this_cpu(desc);
handle_irq_event(desc);
cond_unmask_irq(desc);
out_unlock:
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL_GPL(handle_level_irq);
static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
{
if (!(desc->istate & IRQS_ONESHOT)) {
chip->irq_eoi(&desc->irq_data);
return;
}
/*
* We need to unmask in the following cases:
* - Oneshot irq which did not wake the thread (caused by a
* spurious interrupt or a primary handler handling it
* completely).
*/
if (!irqd_irq_disabled(&desc->irq_data) &&
irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
chip->irq_eoi(&desc->irq_data);
unmask_irq(desc);
} else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
chip->irq_eoi(&desc->irq_data);
}
}
/**
* handle_fasteoi_irq - irq handler for transparent controllers
* @desc: the interrupt description structure for this irq
*
* Only a single callback will be issued to the chip: an ->eoi()
* call when the interrupt has been serviced. This enables support
* for modern forms of interrupt handlers, which handle the flow
* details in hardware, transparently.
*/
void handle_fasteoi_irq(struct irq_desc *desc)
{
struct irq_chip *chip = desc->irq_data.chip;
raw_spin_lock(&desc->lock);
/*
* When an affinity change races with IRQ handling, the next interrupt
* can arrive on the new CPU before the original CPU has completed
* handling the previous one - it may need to be resent.
*/
if (!irq_may_run(desc)) {
if (irqd_needs_resend_when_in_progress(&desc->irq_data))
desc->istate |= IRQS_PENDING;
goto out;
}
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
/*
* If its disabled or no action available
* then mask it and get out of here:
*/
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
mask_irq(desc);
goto out;
}
kstat_incr_irqs_this_cpu(desc);
if (desc->istate & IRQS_ONESHOT)
mask_irq(desc);
handle_irq_event(desc);
cond_unmask_eoi_irq(desc, chip);
/*
* When the race described above happens this will resend the interrupt.
*/
if (unlikely(desc->istate & IRQS_PENDING))
check_irq_resend(desc, false);
raw_spin_unlock(&desc->lock);
return;
out:
if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
chip->irq_eoi(&desc->irq_data);
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
/**
* handle_fasteoi_nmi - irq handler for NMI interrupt lines
* @desc: the interrupt description structure for this irq
*
* A simple NMI-safe handler, considering the restrictions
* from request_nmi.
*
* Only a single callback will be issued to the chip: an ->eoi()
* call when the interrupt has been serviced. This enables support
* for modern forms of interrupt handlers, which handle the flow
* details in hardware, transparently.
*/
void handle_fasteoi_nmi(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct irqaction *action = desc->action;
unsigned int irq = irq_desc_get_irq(desc);
irqreturn_t res;
__kstat_incr_irqs_this_cpu(desc);
trace_irq_handler_entry(irq, action);
/*
* NMIs cannot be shared, there is only one action.
*/
res = action->handler(irq, action->dev_id);
trace_irq_handler_exit(irq, action, res);
if (chip->irq_eoi)
chip->irq_eoi(&desc->irq_data);
}
EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
/**
* handle_edge_irq - edge type IRQ handler
* @desc: the interrupt description structure for this irq
*
* Interrupt occurs on the falling and/or rising edge of a hardware
* signal. The occurrence is latched into the irq controller hardware
* and must be acked in order to be reenabled. After the ack another
* interrupt can happen on the same source even before the first one
* is handled by the associated event handler. If this happens it
* might be necessary to disable (mask) the interrupt depending on the
* controller hardware. This requires to reenable the interrupt inside
* of the loop which handles the interrupts which have arrived while
* the handler was running. If all pending interrupts are handled, the
* loop is left.
*/
void handle_edge_irq(struct irq_desc *desc)
{
raw_spin_lock(&desc->lock);
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
if (!irq_may_run(desc)) { desc->istate |= IRQS_PENDING; mask_ack_irq(desc);
goto out_unlock;
}
/*
* If its disabled or no action available then mask it and get
* out of here.
*/
if (irqd_irq_disabled(&desc->irq_data) || !desc->action) { desc->istate |= IRQS_PENDING; mask_ack_irq(desc);
goto out_unlock;
}
kstat_incr_irqs_this_cpu(desc);
/* Start handling the irq */
desc->irq_data.chip->irq_ack(&desc->irq_data);
do {
if (unlikely(!desc->action)) { mask_irq(desc);
goto out_unlock;
}
/*
* When another irq arrived while we were handling
* one, we could have masked the irq.
* Reenable it, if it was not disabled in meantime.
*/
if (unlikely(desc->istate & IRQS_PENDING)) { if (!irqd_irq_disabled(&desc->irq_data) && irqd_irq_masked(&desc->irq_data)) unmask_irq(desc);
}
handle_irq_event(desc);
} while ((desc->istate & IRQS_PENDING) &&
!irqd_irq_disabled(&desc->irq_data));
out_unlock:
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL(handle_edge_irq);
#ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
/**
* handle_edge_eoi_irq - edge eoi type IRQ handler
* @desc: the interrupt description structure for this irq
*
* Similar as the above handle_edge_irq, but using eoi and w/o the
* mask/unmask logic.
*/
void handle_edge_eoi_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
raw_spin_lock(&desc->lock);
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
if (!irq_may_run(desc)) {
desc->istate |= IRQS_PENDING;
goto out_eoi;
}
/*
* If its disabled or no action available then mask it and get
* out of here.
*/
if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
desc->istate |= IRQS_PENDING;
goto out_eoi;
}
kstat_incr_irqs_this_cpu(desc);
do {
if (unlikely(!desc->action))
goto out_eoi;
handle_irq_event(desc);
} while ((desc->istate & IRQS_PENDING) &&
!irqd_irq_disabled(&desc->irq_data));
out_eoi:
chip->irq_eoi(&desc->irq_data);
raw_spin_unlock(&desc->lock);
}
#endif
/**
* handle_percpu_irq - Per CPU local irq handler
* @desc: the interrupt description structure for this irq
*
* Per CPU interrupts on SMP machines without locking requirements
*/
void handle_percpu_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
/*
* PER CPU interrupts are not serialized. Do not touch
* desc->tot_count.
*/
__kstat_incr_irqs_this_cpu(desc);
if (chip->irq_ack)
chip->irq_ack(&desc->irq_data);
handle_irq_event_percpu(desc);
if (chip->irq_eoi)
chip->irq_eoi(&desc->irq_data);
}
/**
* handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
* @desc: the interrupt description structure for this irq
*
* Per CPU interrupts on SMP machines without locking requirements. Same as
* handle_percpu_irq() above but with the following extras:
*
* action->percpu_dev_id is a pointer to percpu variables which
* contain the real device id for the cpu on which this handler is
* called
*/
void handle_percpu_devid_irq(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct irqaction *action = desc->action;
unsigned int irq = irq_desc_get_irq(desc);
irqreturn_t res;
/*
* PER CPU interrupts are not serialized. Do not touch
* desc->tot_count.
*/
__kstat_incr_irqs_this_cpu(desc);
if (chip->irq_ack)
chip->irq_ack(&desc->irq_data);
if (likely(action)) {
trace_irq_handler_entry(irq, action);
res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
trace_irq_handler_exit(irq, action, res);
} else {
unsigned int cpu = smp_processor_id();
bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
if (enabled)
irq_percpu_disable(desc, cpu);
pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
enabled ? " and unmasked" : "", irq, cpu);
}
if (chip->irq_eoi)
chip->irq_eoi(&desc->irq_data);
}
/**
* handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
* dev ids
* @desc: the interrupt description structure for this irq
*
* Similar to handle_fasteoi_nmi, but handling the dev_id cookie
* as a percpu pointer.
*/
void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
{
struct irq_chip *chip = irq_desc_get_chip(desc);
struct irqaction *action = desc->action;
unsigned int irq = irq_desc_get_irq(desc);
irqreturn_t res;
__kstat_incr_irqs_this_cpu(desc);
trace_irq_handler_entry(irq, action);
res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
trace_irq_handler_exit(irq, action, res);
if (chip->irq_eoi)
chip->irq_eoi(&desc->irq_data);
}
static void
__irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
int is_chained, const char *name)
{
if (!handle) {
handle = handle_bad_irq;
} else {
struct irq_data *irq_data = &desc->irq_data;
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
/*
* With hierarchical domains we might run into a
* situation where the outermost chip is not yet set
* up, but the inner chips are there. Instead of
* bailing we install the handler, but obviously we
* cannot enable/startup the interrupt at this point.
*/
while (irq_data) {
if (irq_data->chip != &no_irq_chip)
break;
/*
* Bail out if the outer chip is not set up
* and the interrupt supposed to be started
* right away.
*/
if (WARN_ON(is_chained))
return;
/* Try the parent */
irq_data = irq_data->parent_data;
}
#endif
if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
return;
}
/* Uninstall? */
if (handle == handle_bad_irq) {
if (desc->irq_data.chip != &no_irq_chip)
mask_ack_irq(desc);
irq_state_set_disabled(desc);
if (is_chained) {
desc->action = NULL;
WARN_ON(irq_chip_pm_put(irq_desc_get_irq_data(desc)));
}
desc->depth = 1;
}
desc->handle_irq = handle;
desc->name = name;
if (handle != handle_bad_irq && is_chained) {
unsigned int type = irqd_get_trigger_type(&desc->irq_data);
/*
* We're about to start this interrupt immediately,
* hence the need to set the trigger configuration.
* But the .set_type callback may have overridden the
* flow handler, ignoring that we're dealing with a
* chained interrupt. Reset it immediately because we
* do know better.
*/
if (type != IRQ_TYPE_NONE) {
__irq_set_trigger(desc, type);
desc->handle_irq = handle;
}
irq_settings_set_noprobe(desc);
irq_settings_set_norequest(desc);
irq_settings_set_nothread(desc);
desc->action = &chained_action;
WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc)));
irq_activate_and_startup(desc, IRQ_RESEND);
}
}
void
__irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
const char *name)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
if (!desc)
return;
__irq_do_set_handler(desc, handle, is_chained, name);
irq_put_desc_busunlock(desc, flags);
}
EXPORT_SYMBOL_GPL(__irq_set_handler);
void
irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
void *data)
{
unsigned long flags;
struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
if (!desc)
return;
desc->irq_common_data.handler_data = data;
__irq_do_set_handler(desc, handle, 1, NULL);
irq_put_desc_busunlock(desc, flags);
}
EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
void
irq_set_chip_and_handler_name(unsigned int irq, const struct irq_chip *chip,
irq_flow_handler_t handle, const char *name)
{
irq_set_chip(irq, chip);
__irq_set_handler(irq, handle, 0, name);
}
EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
{
unsigned long flags, trigger, tmp;
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
if (!desc)
return;
/*
* Warn when a driver sets the no autoenable flag on an already
* active interrupt.
*/
WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
irq_settings_clr_and_set(desc, clr, set);
trigger = irqd_get_trigger_type(&desc->irq_data);
irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
if (irq_settings_has_no_balance_set(desc))
irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
if (irq_settings_is_per_cpu(desc))
irqd_set(&desc->irq_data, IRQD_PER_CPU);
if (irq_settings_can_move_pcntxt(desc))
irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
if (irq_settings_is_level(desc))
irqd_set(&desc->irq_data, IRQD_LEVEL);
tmp = irq_settings_get_trigger_mask(desc);
if (tmp != IRQ_TYPE_NONE)
trigger = tmp;
irqd_set(&desc->irq_data, trigger);
irq_put_desc_unlock(desc, flags);
}
EXPORT_SYMBOL_GPL(irq_modify_status);
#ifdef CONFIG_DEPRECATED_IRQ_CPU_ONOFFLINE
/**
* irq_cpu_online - Invoke all irq_cpu_online functions.
*
* Iterate through all irqs and invoke the chip.irq_cpu_online()
* for each.
*/
void irq_cpu_online(void)
{
struct irq_desc *desc;
struct irq_chip *chip;
unsigned long flags;
unsigned int irq;
for_each_active_irq(irq) {
desc = irq_to_desc(irq);
if (!desc)
continue;
raw_spin_lock_irqsave(&desc->lock, flags);
chip = irq_data_get_irq_chip(&desc->irq_data);
if (chip && chip->irq_cpu_online &&
(!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
!irqd_irq_disabled(&desc->irq_data)))
chip->irq_cpu_online(&desc->irq_data);
raw_spin_unlock_irqrestore(&desc->lock, flags);
}
}
/**
* irq_cpu_offline - Invoke all irq_cpu_offline functions.
*
* Iterate through all irqs and invoke the chip.irq_cpu_offline()
* for each.
*/
void irq_cpu_offline(void)
{
struct irq_desc *desc;
struct irq_chip *chip;
unsigned long flags;
unsigned int irq;
for_each_active_irq(irq) {
desc = irq_to_desc(irq);
if (!desc)
continue;
raw_spin_lock_irqsave(&desc->lock, flags);
chip = irq_data_get_irq_chip(&desc->irq_data);
if (chip && chip->irq_cpu_offline &&
(!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
!irqd_irq_disabled(&desc->irq_data)))
chip->irq_cpu_offline(&desc->irq_data);
raw_spin_unlock_irqrestore(&desc->lock, flags);
}
}
#endif
#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
#ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
/**
* handle_fasteoi_ack_irq - irq handler for edge hierarchy
* stacked on transparent controllers
*
* @desc: the interrupt description structure for this irq
*
* Like handle_fasteoi_irq(), but for use with hierarchy where
* the irq_chip also needs to have its ->irq_ack() function
* called.
*/
void handle_fasteoi_ack_irq(struct irq_desc *desc)
{
struct irq_chip *chip = desc->irq_data.chip;
raw_spin_lock(&desc->lock);
if (!irq_may_run(desc))
goto out;
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
/*
* If its disabled or no action available
* then mask it and get out of here:
*/
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
mask_irq(desc);
goto out;
}
kstat_incr_irqs_this_cpu(desc);
if (desc->istate & IRQS_ONESHOT)
mask_irq(desc);
/* Start handling the irq */
desc->irq_data.chip->irq_ack(&desc->irq_data);
handle_irq_event(desc);
cond_unmask_eoi_irq(desc, chip);
raw_spin_unlock(&desc->lock);
return;
out:
if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
chip->irq_eoi(&desc->irq_data);
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
/**
* handle_fasteoi_mask_irq - irq handler for level hierarchy
* stacked on transparent controllers
*
* @desc: the interrupt description structure for this irq
*
* Like handle_fasteoi_irq(), but for use with hierarchy where
* the irq_chip also needs to have its ->irq_mask_ack() function
* called.
*/
void handle_fasteoi_mask_irq(struct irq_desc *desc)
{
struct irq_chip *chip = desc->irq_data.chip;
raw_spin_lock(&desc->lock);
mask_ack_irq(desc);
if (!irq_may_run(desc))
goto out;
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
/*
* If its disabled or no action available
* then mask it and get out of here:
*/
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
desc->istate |= IRQS_PENDING;
mask_irq(desc);
goto out;
}
kstat_incr_irqs_this_cpu(desc);
if (desc->istate & IRQS_ONESHOT)
mask_irq(desc);
handle_irq_event(desc);
cond_unmask_eoi_irq(desc, chip);
raw_spin_unlock(&desc->lock);
return;
out:
if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
chip->irq_eoi(&desc->irq_data);
raw_spin_unlock(&desc->lock);
}
EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
#endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
/**
* irq_chip_set_parent_state - set the state of a parent interrupt.
*
* @data: Pointer to interrupt specific data
* @which: State to be restored (one of IRQCHIP_STATE_*)
* @val: Value corresponding to @which
*
* Conditional success, if the underlying irqchip does not implement it.
*/
int irq_chip_set_parent_state(struct irq_data *data,
enum irqchip_irq_state which,
bool val)
{
data = data->parent_data;
if (!data || !data->chip->irq_set_irqchip_state)
return 0;
return data->chip->irq_set_irqchip_state(data, which, val);
}
EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
/**
* irq_chip_get_parent_state - get the state of a parent interrupt.
*
* @data: Pointer to interrupt specific data
* @which: one of IRQCHIP_STATE_* the caller wants to know
* @state: a pointer to a boolean where the state is to be stored
*
* Conditional success, if the underlying irqchip does not implement it.
*/
int irq_chip_get_parent_state(struct irq_data *data,
enum irqchip_irq_state which,
bool *state)
{
data = data->parent_data;
if (!data || !data->chip->irq_get_irqchip_state)
return 0;
return data->chip->irq_get_irqchip_state(data, which, state);
}
EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
/**
* irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
* NULL)
* @data: Pointer to interrupt specific data
*/
void irq_chip_enable_parent(struct irq_data *data)
{
data = data->parent_data;
if (data->chip->irq_enable)
data->chip->irq_enable(data);
else
data->chip->irq_unmask(data);
}
EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
/**
* irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
* NULL)
* @data: Pointer to interrupt specific data
*/
void irq_chip_disable_parent(struct irq_data *data)
{
data = data->parent_data;
if (data->chip->irq_disable)
data->chip->irq_disable(data);
else
data->chip->irq_mask(data);
}
EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
/**
* irq_chip_ack_parent - Acknowledge the parent interrupt
* @data: Pointer to interrupt specific data
*/
void irq_chip_ack_parent(struct irq_data *data)
{
data = data->parent_data;
data->chip->irq_ack(data);
}
EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
/**
* irq_chip_mask_parent - Mask the parent interrupt
* @data: Pointer to interrupt specific data
*/
void irq_chip_mask_parent(struct irq_data *data)
{
data = data->parent_data;
data->chip->irq_mask(data);
}
EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
/**
* irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
* @data: Pointer to interrupt specific data
*/
void irq_chip_mask_ack_parent(struct irq_data *data)
{
data = data->parent_data;
data->chip->irq_mask_ack(data);
}
EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
/**
* irq_chip_unmask_parent - Unmask the parent interrupt
* @data: Pointer to interrupt specific data
*/
void irq_chip_unmask_parent(struct irq_data *data)
{
data = data->parent_data;
data->chip->irq_unmask(data);
}
EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
/**
* irq_chip_eoi_parent - Invoke EOI on the parent interrupt
* @data: Pointer to interrupt specific data
*/
void irq_chip_eoi_parent(struct irq_data *data)
{
data = data->parent_data;
data->chip->irq_eoi(data);
}
EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
/**
* irq_chip_set_affinity_parent - Set affinity on the parent interrupt
* @data: Pointer to interrupt specific data
* @dest: The affinity mask to set
* @force: Flag to enforce setting (disable online checks)
*
* Conditional, as the underlying parent chip might not implement it.
*/
int irq_chip_set_affinity_parent(struct irq_data *data,
const struct cpumask *dest, bool force)
{
data = data->parent_data;
if (data->chip->irq_set_affinity)
return data->chip->irq_set_affinity(data, dest, force);
return -ENOSYS;
}
EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
/**
* irq_chip_set_type_parent - Set IRQ type on the parent interrupt
* @data: Pointer to interrupt specific data
* @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
*
* Conditional, as the underlying parent chip might not implement it.
*/
int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
{
data = data->parent_data;
if (data->chip->irq_set_type)
return data->chip->irq_set_type(data, type);
return -ENOSYS;
}
EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
/**
* irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
* @data: Pointer to interrupt specific data
*
* Iterate through the domain hierarchy of the interrupt and check
* whether a hw retrigger function exists. If yes, invoke it.
*/
int irq_chip_retrigger_hierarchy(struct irq_data *data)
{
for (data = data->parent_data; data; data = data->parent_data)
if (data->chip && data->chip->irq_retrigger)
return data->chip->irq_retrigger(data);
return 0;
}
EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
/**
* irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
* @data: Pointer to interrupt specific data
* @vcpu_info: The vcpu affinity information
*/
int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
{
data = data->parent_data;
if (data->chip->irq_set_vcpu_affinity)
return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
return -ENOSYS;
}
EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
/**
* irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
* @data: Pointer to interrupt specific data
* @on: Whether to set or reset the wake-up capability of this irq
*
* Conditional, as the underlying parent chip might not implement it.
*/
int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
{
data = data->parent_data;
if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
return 0;
if (data->chip->irq_set_wake)
return data->chip->irq_set_wake(data, on);
return -ENOSYS;
}
EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
/**
* irq_chip_request_resources_parent - Request resources on the parent interrupt
* @data: Pointer to interrupt specific data
*/
int irq_chip_request_resources_parent(struct irq_data *data)
{
data = data->parent_data;
if (data->chip->irq_request_resources)
return data->chip->irq_request_resources(data);
/* no error on missing optional irq_chip::irq_request_resources */
return 0;
}
EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
/**
* irq_chip_release_resources_parent - Release resources on the parent interrupt
* @data: Pointer to interrupt specific data
*/
void irq_chip_release_resources_parent(struct irq_data *data)
{
data = data->parent_data;
if (data->chip->irq_release_resources)
data->chip->irq_release_resources(data);
}
EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
#endif
/**
* irq_chip_compose_msi_msg - Compose msi message for a irq chip
* @data: Pointer to interrupt specific data
* @msg: Pointer to the MSI message
*
* For hierarchical domains we find the first chip in the hierarchy
* which implements the irq_compose_msi_msg callback. For non
* hierarchical we use the top level chip.
*/
int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
{
struct irq_data *pos;
for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
if (data->chip && data->chip->irq_compose_msi_msg)
pos = data;
}
if (!pos)
return -ENOSYS;
pos->chip->irq_compose_msi_msg(pos, msg);
return 0;
}
static struct device *irq_get_pm_device(struct irq_data *data)
{
if (data->domain)
return data->domain->pm_dev;
return NULL;
}
/**
* irq_chip_pm_get - Enable power for an IRQ chip
* @data: Pointer to interrupt specific data
*
* Enable the power to the IRQ chip referenced by the interrupt data
* structure.
*/
int irq_chip_pm_get(struct irq_data *data)
{
struct device *dev = irq_get_pm_device(data);
int retval = 0;
if (IS_ENABLED(CONFIG_PM) && dev)
retval = pm_runtime_resume_and_get(dev);
return retval;
}
/**
* irq_chip_pm_put - Disable power for an IRQ chip
* @data: Pointer to interrupt specific data
*
* Disable the power to the IRQ chip referenced by the interrupt data
* structure, belongs. Note that power will only be disabled, once this
* function has been called for all IRQs that have called irq_chip_pm_get().
*/
int irq_chip_pm_put(struct irq_data *data)
{
struct device *dev = irq_get_pm_device(data);
int retval = 0;
if (IS_ENABLED(CONFIG_PM) && dev)
retval = pm_runtime_put(dev);
return (retval < 0) ? retval : 0;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_IVERSION_H
#define _LINUX_IVERSION_H
#include <linux/fs.h>
/*
* The inode->i_version field:
* ---------------------------
* The change attribute (i_version) is mandated by NFSv4 and is mostly for
* knfsd, but is also used for other purposes (e.g. IMA). The i_version must
* appear larger to observers if there was an explicit change to the inode's
* data or metadata since it was last queried.
*
* An explicit change is one that would ordinarily result in a change to the
* inode status change time (aka ctime). i_version must appear to change, even
* if the ctime does not (since the whole point is to avoid missing updates due
* to timestamp granularity). If POSIX or other relevant spec mandates that the
* ctime must change due to an operation, then the i_version counter must be
* incremented as well.
*
* Making the i_version update completely atomic with the operation itself would
* be prohibitively expensive. Traditionally the kernel has updated the times on
* directories after an operation that changes its contents. For regular files,
* the ctime is usually updated before the data is copied into the cache for a
* write. This means that there is a window of time when an observer can
* associate a new timestamp with old file contents. Since the purpose of the
* i_version is to allow for better cache coherency, the i_version must always
* be updated after the results of the operation are visible. Updating it before
* and after a change is also permitted. (Note that no filesystems currently do
* this. Fixing that is a work-in-progress).
*
* Observers see the i_version as a 64-bit number that never decreases. If it
* remains the same since it was last checked, then nothing has changed in the
* inode. If it's different then something has changed. Observers cannot infer
* anything about the nature or magnitude of the changes from the value, only
* that the inode has changed in some fashion.
*
* Not all filesystems properly implement the i_version counter. Subsystems that
* want to use i_version field on an inode should first check whether the
* filesystem sets the SB_I_VERSION flag (usually via the IS_I_VERSION macro).
*
* Those that set SB_I_VERSION will automatically have their i_version counter
* incremented on writes to normal files. If the SB_I_VERSION is not set, then
* the VFS will not touch it on writes, and the filesystem can use it how it
* wishes. Note that the filesystem is always responsible for updating the
* i_version on namespace changes in directories (mkdir, rmdir, unlink, etc.).
* We consider these sorts of filesystems to have a kernel-managed i_version.
*
* It may be impractical for filesystems to keep i_version updates atomic with
* respect to the changes that cause them. They should, however, guarantee
* that i_version updates are never visible before the changes that caused
* them. Also, i_version updates should never be delayed longer than it takes
* the original change to reach disk.
*
* This implementation uses the low bit in the i_version field as a flag to
* track when the value has been queried. If it has not been queried since it
* was last incremented, we can skip the increment in most cases.
*
* In the event that we're updating the ctime, we will usually go ahead and
* bump the i_version anyway. Since that has to go to stable storage in some
* fashion, we might as well increment it as well.
*
* With this implementation, the value should always appear to observers to
* increase over time if the file has changed. It's recommended to use
* inode_eq_iversion() helper to compare values.
*
* Note that some filesystems (e.g. NFS and AFS) just use the field to store
* a server-provided value (for the most part). For that reason, those
* filesystems do not set SB_I_VERSION. These filesystems are considered to
* have a self-managed i_version.
*
* Persistently storing the i_version
* ----------------------------------
* Queries of the i_version field are not gated on them hitting the backing
* store. It's always possible that the host could crash after allowing
* a query of the value but before it has made it to disk.
*
* To mitigate this problem, filesystems should always use
* inode_set_iversion_queried when loading an existing inode from disk. This
* ensures that the next attempted inode increment will result in the value
* changing.
*
* Storing the value to disk therefore does not count as a query, so those
* filesystems should use inode_peek_iversion to grab the value to be stored.
* There is no need to flag the value as having been queried in that case.
*/
/*
* We borrow the lowest bit in the i_version to use as a flag to tell whether
* it has been queried since we last incremented it. If it has, then we must
* increment it on the next change. After that, we can clear the flag and
* avoid incrementing it again until it has again been queried.
*/
#define I_VERSION_QUERIED_SHIFT (1)
#define I_VERSION_QUERIED (1ULL << (I_VERSION_QUERIED_SHIFT - 1))
#define I_VERSION_INCREMENT (1ULL << I_VERSION_QUERIED_SHIFT)
/**
* inode_set_iversion_raw - set i_version to the specified raw value
* @inode: inode to set
* @val: new i_version value to set
*
* Set @inode's i_version field to @val. This function is for use by
* filesystems that self-manage the i_version.
*
* For example, the NFS client stores its NFSv4 change attribute in this way,
* and the AFS client stores the data_version from the server here.
*/
static inline void
inode_set_iversion_raw(struct inode *inode, u64 val)
{
atomic64_set(&inode->i_version, val);
}
/**
* inode_peek_iversion_raw - grab a "raw" iversion value
* @inode: inode from which i_version should be read
*
* Grab a "raw" inode->i_version value and return it. The i_version is not
* flagged or converted in any way. This is mostly used to access a self-managed
* i_version.
*
* With those filesystems, we want to treat the i_version as an entirely
* opaque value.
*/
static inline u64
inode_peek_iversion_raw(const struct inode *inode)
{
return atomic64_read(&inode->i_version);
}
/**
* inode_set_max_iversion_raw - update i_version new value is larger
* @inode: inode to set
* @val: new i_version to set
*
* Some self-managed filesystems (e.g Ceph) will only update the i_version
* value if the new value is larger than the one we already have.
*/
static inline void
inode_set_max_iversion_raw(struct inode *inode, u64 val)
{
u64 cur = inode_peek_iversion_raw(inode);
do {
if (cur > val)
break;
} while (!atomic64_try_cmpxchg(&inode->i_version, &cur, val));
}
/**
* inode_set_iversion - set i_version to a particular value
* @inode: inode to set
* @val: new i_version value to set
*
* Set @inode's i_version field to @val. This function is for filesystems with
* a kernel-managed i_version, for initializing a newly-created inode from
* scratch.
*
* In this case, we do not set the QUERIED flag since we know that this value
* has never been queried.
*/
static inline void
inode_set_iversion(struct inode *inode, u64 val)
{
inode_set_iversion_raw(inode, val << I_VERSION_QUERIED_SHIFT);
}
/**
* inode_set_iversion_queried - set i_version to a particular value as quereied
* @inode: inode to set
* @val: new i_version value to set
*
* Set @inode's i_version field to @val, and flag it for increment on the next
* change.
*
* Filesystems that persistently store the i_version on disk should use this
* when loading an existing inode from disk.
*
* When loading in an i_version value from a backing store, we can't be certain
* that it wasn't previously viewed before being stored. Thus, we must assume
* that it was, to ensure that we don't end up handing out the same value for
* different versions of the same inode.
*/
static inline void
inode_set_iversion_queried(struct inode *inode, u64 val)
{
inode_set_iversion_raw(inode, (val << I_VERSION_QUERIED_SHIFT) |
I_VERSION_QUERIED);
}
bool inode_maybe_inc_iversion(struct inode *inode, bool force);
/**
* inode_inc_iversion - forcibly increment i_version
* @inode: inode that needs to be updated
*
* Forcbily increment the i_version field. This always results in a change to
* the observable value.
*/
static inline void
inode_inc_iversion(struct inode *inode)
{
inode_maybe_inc_iversion(inode, true);
}
/**
* inode_iversion_need_inc - is the i_version in need of being incremented?
* @inode: inode to check
*
* Returns whether the inode->i_version counter needs incrementing on the next
* change. Just fetch the value and check the QUERIED flag.
*/
static inline bool
inode_iversion_need_inc(struct inode *inode)
{
return inode_peek_iversion_raw(inode) & I_VERSION_QUERIED;
}
/**
* inode_inc_iversion_raw - forcibly increment raw i_version
* @inode: inode that needs to be updated
*
* Forcbily increment the raw i_version field. This always results in a change
* to the raw value.
*
* NFS will use the i_version field to store the value from the server. It
* mostly treats it as opaque, but in the case where it holds a write
* delegation, it must increment the value itself. This function does that.
*/
static inline void
inode_inc_iversion_raw(struct inode *inode)
{
atomic64_inc(&inode->i_version);
}
/**
* inode_peek_iversion - read i_version without flagging it to be incremented
* @inode: inode from which i_version should be read
*
* Read the inode i_version counter for an inode without registering it as a
* query.
*
* This is typically used by local filesystems that need to store an i_version
* on disk. In that situation, it's not necessary to flag it as having been
* viewed, as the result won't be used to gauge changes from that point.
*/
static inline u64
inode_peek_iversion(const struct inode *inode)
{
return inode_peek_iversion_raw(inode) >> I_VERSION_QUERIED_SHIFT;
}
/*
* For filesystems without any sort of change attribute, the best we can
* do is fake one up from the ctime:
*/
static inline u64 time_to_chattr(const struct timespec64 *t)
{
u64 chattr = t->tv_sec;
chattr <<= 32;
chattr += t->tv_nsec;
return chattr;
}
u64 inode_query_iversion(struct inode *inode);
/**
* inode_eq_iversion_raw - check whether the raw i_version counter has changed
* @inode: inode to check
* @old: old value to check against its i_version
*
* Compare the current raw i_version counter with a previous one. Returns true
* if they are the same or false if they are different.
*/
static inline bool
inode_eq_iversion_raw(const struct inode *inode, u64 old)
{
return inode_peek_iversion_raw(inode) == old;
}
/**
* inode_eq_iversion - check whether the i_version counter has changed
* @inode: inode to check
* @old: old value to check against its i_version
*
* Compare an i_version counter with a previous one. Returns true if they are
* the same, and false if they are different.
*
* Note that we don't need to set the QUERIED flag in this case, as the value
* in the inode is not being recorded for later use.
*/
static inline bool
inode_eq_iversion(const struct inode *inode, u64 old)
{
return inode_peek_iversion(inode) == old;
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_TASK_H
#define _LINUX_SCHED_TASK_H
/*
* Interface between the scheduler and various task lifetime (fork()/exit())
* functionality:
*/
#include <linux/rcupdate.h>
#include <linux/refcount.h>
#include <linux/sched.h>
#include <linux/uaccess.h>
struct task_struct;
struct rusage;
union thread_union;
struct css_set;
/* All the bits taken by the old clone syscall. */
#define CLONE_LEGACY_FLAGS 0xffffffffULL
struct kernel_clone_args {
u64 flags;
int __user *pidfd;
int __user *child_tid;
int __user *parent_tid;
const char *name;
int exit_signal;
u32 kthread:1;
u32 io_thread:1;
u32 user_worker:1;
u32 no_files:1;
unsigned long stack;
unsigned long stack_size;
unsigned long tls;
pid_t *set_tid;
/* Number of elements in *set_tid */
size_t set_tid_size;
int cgroup;
int idle;
int (*fn)(void *);
void *fn_arg;
struct cgroup *cgrp;
struct css_set *cset;
};
/*
* This serializes "schedule()" and also protects
* the run-queue from deletions/modifications (but
* _adding_ to the beginning of the run-queue has
* a separate lock).
*/
extern rwlock_t tasklist_lock;
extern spinlock_t mmlist_lock;
extern union thread_union init_thread_union;
extern struct task_struct init_task;
extern int lockdep_tasklist_lock_is_held(void);
extern asmlinkage void schedule_tail(struct task_struct *prev);
extern void init_idle(struct task_struct *idle, int cpu);
extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
extern void sched_post_fork(struct task_struct *p);
extern void sched_dead(struct task_struct *p);
void __noreturn do_task_dead(void);
void __noreturn make_task_dead(int signr);
extern void mm_cache_init(void);
extern void proc_caches_init(void);
extern void fork_init(void);
extern void release_task(struct task_struct * p);
extern int copy_thread(struct task_struct *, const struct kernel_clone_args *);
extern void flush_thread(void);
#ifdef CONFIG_HAVE_EXIT_THREAD
extern void exit_thread(struct task_struct *tsk);
#else
static inline void exit_thread(struct task_struct *tsk)
{
}
#endif
extern __noreturn void do_group_exit(int);
extern void exit_files(struct task_struct *);
extern void exit_itimers(struct task_struct *);
extern pid_t kernel_clone(struct kernel_clone_args *kargs);
struct task_struct *copy_process(struct pid *pid, int trace, int node,
struct kernel_clone_args *args);
struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node);
struct task_struct *fork_idle(int);
extern pid_t kernel_thread(int (*fn)(void *), void *arg, const char *name,
unsigned long flags);
extern pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags);
extern long kernel_wait4(pid_t, int __user *, int, struct rusage *);
int kernel_wait(pid_t pid, int *stat);
extern void free_task(struct task_struct *tsk);
/* sched_exec is called by processes performing an exec */
#ifdef CONFIG_SMP
extern void sched_exec(void);
#else
#define sched_exec() {}
#endif
static inline struct task_struct *get_task_struct(struct task_struct *t)
{
refcount_inc(&t->usage);
return t;
}
extern void __put_task_struct(struct task_struct *t);
extern void __put_task_struct_rcu_cb(struct rcu_head *rhp);
static inline void put_task_struct(struct task_struct *t)
{
if (!refcount_dec_and_test(&t->usage))
return;
/*
* In !RT, it is always safe to call __put_task_struct().
* Under RT, we can only call it in preemptible context.
*/
if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible()) {
static DEFINE_WAIT_OVERRIDE_MAP(put_task_map, LD_WAIT_SLEEP);
lock_map_acquire_try(&put_task_map);
__put_task_struct(t);
lock_map_release(&put_task_map);
return;
}
/*
* under PREEMPT_RT, we can't call put_task_struct
* in atomic context because it will indirectly
* acquire sleeping locks.
*
* call_rcu() will schedule delayed_put_task_struct_rcu()
* to be called in process context.
*
* __put_task_struct() is called when
* refcount_dec_and_test(&t->usage) succeeds.
*
* This means that it can't "conflict" with
* put_task_struct_rcu_user() which abuses ->rcu the same
* way; rcu_users has a reference so task->usage can't be
* zero after rcu_users 1 -> 0 transition.
*
* delayed_free_task() also uses ->rcu, but it is only called
* when it fails to fork a process. Therefore, there is no
* way it can conflict with put_task_struct().
*/
call_rcu(&t->rcu, __put_task_struct_rcu_cb);
}
DEFINE_FREE(put_task, struct task_struct *, if (_T) put_task_struct(_T))
static inline void put_task_struct_many(struct task_struct *t, int nr)
{
if (refcount_sub_and_test(nr, &t->usage))
__put_task_struct(t);
}
void put_task_struct_rcu_user(struct task_struct *task);
/* Free all architecture-specific resources held by a thread. */
void release_thread(struct task_struct *dead_task);
#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
extern int arch_task_struct_size __read_mostly;
#else
# define arch_task_struct_size (sizeof(struct task_struct))
#endif
#ifndef CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST
/*
* If an architecture has not declared a thread_struct whitelist we
* must assume something there may need to be copied to userspace.
*/
static inline void arch_thread_struct_whitelist(unsigned long *offset,
unsigned long *size)
{
*offset = 0;
/* Handle dynamically sized thread_struct. */
*size = arch_task_struct_size - offsetof(struct task_struct, thread);
}
#endif
#ifdef CONFIG_VMAP_STACK
static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
{
return t->stack_vm_area;
}
#else
static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t)
{
return NULL;
}
#endif
/*
* Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
* subscriptions and synchronises with wait4(). Also used in procfs. Also
* pins the final release of task.io_context. Also protects ->cpuset and
* ->cgroup.subsys[]. And ->vfork_done. And ->sysvshm.shm_clist.
*
* Nests both inside and outside of read_lock(&tasklist_lock).
* It must not be nested with write_lock_irq(&tasklist_lock),
* neither inside nor outside.
*/
static inline void task_lock(struct task_struct *p)
{
spin_lock(&p->alloc_lock);
}
static inline void task_unlock(struct task_struct *p)
{
spin_unlock(&p->alloc_lock);
}
DEFINE_GUARD(task_lock, struct task_struct *, task_lock(_T), task_unlock(_T))
#endif /* _LINUX_SCHED_TASK_H */
// SPDX-License-Identifier: GPL-2.0
/*
* linux/kernel/capability.c
*
* Copyright (C) 1997 Andrew Main <zefram@fysh.org>
*
* Integrated into 2.1.97+, Andrew G. Morgan <morgan@kernel.org>
* 30 May 2002: Cleanup, Robert M. Love <rml@tech9.net>
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/audit.h>
#include <linux/capability.h>
#include <linux/mm.h>
#include <linux/export.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/pid_namespace.h>
#include <linux/user_namespace.h>
#include <linux/uaccess.h>
int file_caps_enabled = 1;
static int __init file_caps_disable(char *str)
{
file_caps_enabled = 0;
return 1;
}
__setup("no_file_caps", file_caps_disable);
#ifdef CONFIG_MULTIUSER
/*
* More recent versions of libcap are available from:
*
* http://www.kernel.org/pub/linux/libs/security/linux-privs/
*/
static void warn_legacy_capability_use(void)
{
char name[sizeof(current->comm)];
pr_info_once("warning: `%s' uses 32-bit capabilities (legacy support in use)\n",
get_task_comm(name, current));
}
/*
* Version 2 capabilities worked fine, but the linux/capability.h file
* that accompanied their introduction encouraged their use without
* the necessary user-space source code changes. As such, we have
* created a version 3 with equivalent functionality to version 2, but
* with a header change to protect legacy source code from using
* version 2 when it wanted to use version 1. If your system has code
* that trips the following warning, it is using version 2 specific
* capabilities and may be doing so insecurely.
*
* The remedy is to either upgrade your version of libcap (to 2.10+,
* if the application is linked against it), or recompile your
* application with modern kernel headers and this warning will go
* away.
*/
static void warn_deprecated_v2(void)
{
char name[sizeof(current->comm)];
pr_info_once("warning: `%s' uses deprecated v2 capabilities in a way that may be insecure\n",
get_task_comm(name, current));
}
/*
* Version check. Return the number of u32s in each capability flag
* array, or a negative value on error.
*/
static int cap_validate_magic(cap_user_header_t header, unsigned *tocopy)
{
__u32 version;
if (get_user(version, &header->version))
return -EFAULT;
switch (version) {
case _LINUX_CAPABILITY_VERSION_1:
warn_legacy_capability_use();
*tocopy = _LINUX_CAPABILITY_U32S_1;
break;
case _LINUX_CAPABILITY_VERSION_2:
warn_deprecated_v2();
fallthrough; /* v3 is otherwise equivalent to v2 */
case _LINUX_CAPABILITY_VERSION_3:
*tocopy = _LINUX_CAPABILITY_U32S_3;
break;
default:
if (put_user((u32)_KERNEL_CAPABILITY_VERSION, &header->version))
return -EFAULT;
return -EINVAL;
}
return 0;
}
/*
* The only thing that can change the capabilities of the current
* process is the current process. As such, we can't be in this code
* at the same time as we are in the process of setting capabilities
* in this process. The net result is that we can limit our use of
* locks to when we are reading the caps of another process.
*/
static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp,
kernel_cap_t *pIp, kernel_cap_t *pPp)
{
int ret;
if (pid && (pid != task_pid_vnr(current))) {
const struct task_struct *target;
rcu_read_lock();
target = find_task_by_vpid(pid);
if (!target)
ret = -ESRCH;
else
ret = security_capget(target, pEp, pIp, pPp);
rcu_read_unlock();
} else
ret = security_capget(current, pEp, pIp, pPp);
return ret;
}
/**
* sys_capget - get the capabilities of a given process.
* @header: pointer to struct that contains capability version and
* target pid data
* @dataptr: pointer to struct that contains the effective, permitted,
* and inheritable capabilities that are returned
*
* Returns 0 on success and < 0 on error.
*/
SYSCALL_DEFINE2(capget, cap_user_header_t, header, cap_user_data_t, dataptr)
{
int ret = 0;
pid_t pid;
unsigned tocopy;
kernel_cap_t pE, pI, pP;
struct __user_cap_data_struct kdata[2];
ret = cap_validate_magic(header, &tocopy);
if ((dataptr == NULL) || (ret != 0))
return ((dataptr == NULL) && (ret == -EINVAL)) ? 0 : ret;
if (get_user(pid, &header->pid))
return -EFAULT;
if (pid < 0)
return -EINVAL;
ret = cap_get_target_pid(pid, &pE, &pI, &pP);
if (ret)
return ret;
/*
* Annoying legacy format with 64-bit capabilities exposed
* as two sets of 32-bit fields, so we need to split the
* capability values up.
*/
kdata[0].effective = pE.val; kdata[1].effective = pE.val >> 32;
kdata[0].permitted = pP.val; kdata[1].permitted = pP.val >> 32;
kdata[0].inheritable = pI.val; kdata[1].inheritable = pI.val >> 32;
/*
* Note, in the case, tocopy < _KERNEL_CAPABILITY_U32S,
* we silently drop the upper capabilities here. This
* has the effect of making older libcap
* implementations implicitly drop upper capability
* bits when they perform a: capget/modify/capset
* sequence.
*
* This behavior is considered fail-safe
* behavior. Upgrading the application to a newer
* version of libcap will enable access to the newer
* capabilities.
*
* An alternative would be to return an error here
* (-ERANGE), but that causes legacy applications to
* unexpectedly fail; the capget/modify/capset aborts
* before modification is attempted and the application
* fails.
*/
if (copy_to_user(dataptr, kdata, tocopy * sizeof(kdata[0])))
return -EFAULT;
return 0;
}
static kernel_cap_t mk_kernel_cap(u32 low, u32 high)
{
return (kernel_cap_t) { (low | ((u64)high << 32)) & CAP_VALID_MASK };
}
/**
* sys_capset - set capabilities for a process or (*) a group of processes
* @header: pointer to struct that contains capability version and
* target pid data
* @data: pointer to struct that contains the effective, permitted,
* and inheritable capabilities
*
* Set capabilities for the current process only. The ability to any other
* process(es) has been deprecated and removed.
*
* The restrictions on setting capabilities are specified as:
*
* I: any raised capabilities must be a subset of the old permitted
* P: any raised capabilities must be a subset of the old permitted
* E: must be set to a subset of new permitted
*
* Returns 0 on success and < 0 on error.
*/
SYSCALL_DEFINE2(capset, cap_user_header_t, header, const cap_user_data_t, data)
{
struct __user_cap_data_struct kdata[2] = { { 0, }, };
unsigned tocopy, copybytes;
kernel_cap_t inheritable, permitted, effective;
struct cred *new;
int ret;
pid_t pid;
ret = cap_validate_magic(header, &tocopy);
if (ret != 0)
return ret;
if (get_user(pid, &header->pid))
return -EFAULT;
/* may only affect current now */
if (pid != 0 && pid != task_pid_vnr(current))
return -EPERM;
copybytes = tocopy * sizeof(struct __user_cap_data_struct);
if (copybytes > sizeof(kdata))
return -EFAULT;
if (copy_from_user(&kdata, data, copybytes))
return -EFAULT;
effective = mk_kernel_cap(kdata[0].effective, kdata[1].effective);
permitted = mk_kernel_cap(kdata[0].permitted, kdata[1].permitted);
inheritable = mk_kernel_cap(kdata[0].inheritable, kdata[1].inheritable);
new = prepare_creds();
if (!new)
return -ENOMEM;
ret = security_capset(new, current_cred(),
&effective, &inheritable, &permitted);
if (ret < 0)
goto error;
audit_log_capset(new, current_cred());
return commit_creds(new);
error:
abort_creds(new);
return ret;
}
/**
* has_ns_capability - Does a task have a capability in a specific user ns
* @t: The task in question
* @ns: target user namespace
* @cap: The capability to be tested for
*
* Return true if the specified task has the given superior capability
* currently in effect to the specified user namespace, false if not.
*
* Note that this does not set PF_SUPERPRIV on the task.
*/
bool has_ns_capability(struct task_struct *t,
struct user_namespace *ns, int cap)
{
int ret;
rcu_read_lock();
ret = security_capable(__task_cred(t), ns, cap, CAP_OPT_NONE);
rcu_read_unlock();
return (ret == 0);
}
/**
* has_capability - Does a task have a capability in init_user_ns
* @t: The task in question
* @cap: The capability to be tested for
*
* Return true if the specified task has the given superior capability
* currently in effect to the initial user namespace, false if not.
*
* Note that this does not set PF_SUPERPRIV on the task.
*/
bool has_capability(struct task_struct *t, int cap)
{
return has_ns_capability(t, &init_user_ns, cap);
}
EXPORT_SYMBOL(has_capability);
/**
* has_ns_capability_noaudit - Does a task have a capability (unaudited)
* in a specific user ns.
* @t: The task in question
* @ns: target user namespace
* @cap: The capability to be tested for
*
* Return true if the specified task has the given superior capability
* currently in effect to the specified user namespace, false if not.
* Do not write an audit message for the check.
*
* Note that this does not set PF_SUPERPRIV on the task.
*/
bool has_ns_capability_noaudit(struct task_struct *t,
struct user_namespace *ns, int cap)
{
int ret;
rcu_read_lock();
ret = security_capable(__task_cred(t), ns, cap, CAP_OPT_NOAUDIT);
rcu_read_unlock();
return (ret == 0);
}
/**
* has_capability_noaudit - Does a task have a capability (unaudited) in the
* initial user ns
* @t: The task in question
* @cap: The capability to be tested for
*
* Return true if the specified task has the given superior capability
* currently in effect to init_user_ns, false if not. Don't write an
* audit message for the check.
*
* Note that this does not set PF_SUPERPRIV on the task.
*/
bool has_capability_noaudit(struct task_struct *t, int cap)
{
return has_ns_capability_noaudit(t, &init_user_ns, cap);
}
EXPORT_SYMBOL(has_capability_noaudit);
static bool ns_capable_common(struct user_namespace *ns,
int cap,
unsigned int opts)
{
int capable;
if (unlikely(!cap_valid(cap))) { pr_crit("capable() called with invalid cap=%u\n", cap);
BUG();
}
capable = security_capable(current_cred(), ns, cap, opts);
if (capable == 0) {
current->flags |= PF_SUPERPRIV;
return true;
}
return false;
}
/**
* ns_capable - Determine if the current task has a superior capability in effect
* @ns: The usernamespace we want the capability in
* @cap: The capability to be tested for
*
* Return true if the current task has the given superior capability currently
* available for use, false if not.
*
* This sets PF_SUPERPRIV on the task if the capability is available on the
* assumption that it's about to be used.
*/
bool ns_capable(struct user_namespace *ns, int cap)
{
return ns_capable_common(ns, cap, CAP_OPT_NONE);
}
EXPORT_SYMBOL(ns_capable);
/**
* ns_capable_noaudit - Determine if the current task has a superior capability
* (unaudited) in effect
* @ns: The usernamespace we want the capability in
* @cap: The capability to be tested for
*
* Return true if the current task has the given superior capability currently
* available for use, false if not.
*
* This sets PF_SUPERPRIV on the task if the capability is available on the
* assumption that it's about to be used.
*/
bool ns_capable_noaudit(struct user_namespace *ns, int cap)
{
return ns_capable_common(ns, cap, CAP_OPT_NOAUDIT);
}
EXPORT_SYMBOL(ns_capable_noaudit);
/**
* ns_capable_setid - Determine if the current task has a superior capability
* in effect, while signalling that this check is being done from within a
* setid or setgroups syscall.
* @ns: The usernamespace we want the capability in
* @cap: The capability to be tested for
*
* Return true if the current task has the given superior capability currently
* available for use, false if not.
*
* This sets PF_SUPERPRIV on the task if the capability is available on the
* assumption that it's about to be used.
*/
bool ns_capable_setid(struct user_namespace *ns, int cap)
{
return ns_capable_common(ns, cap, CAP_OPT_INSETID);
}
EXPORT_SYMBOL(ns_capable_setid);
/**
* capable - Determine if the current task has a superior capability in effect
* @cap: The capability to be tested for
*
* Return true if the current task has the given superior capability currently
* available for use, false if not.
*
* This sets PF_SUPERPRIV on the task if the capability is available on the
* assumption that it's about to be used.
*/
bool capable(int cap)
{
return ns_capable(&init_user_ns, cap);
}
EXPORT_SYMBOL(capable);
#endif /* CONFIG_MULTIUSER */
/**
* file_ns_capable - Determine if the file's opener had a capability in effect
* @file: The file we want to check
* @ns: The usernamespace we want the capability in
* @cap: The capability to be tested for
*
* Return true if task that opened the file had a capability in effect
* when the file was opened.
*
* This does not set PF_SUPERPRIV because the caller may not
* actually be privileged.
*/
bool file_ns_capable(const struct file *file, struct user_namespace *ns,
int cap)
{
if (WARN_ON_ONCE(!cap_valid(cap)))
return false;
if (security_capable(file->f_cred, ns, cap, CAP_OPT_NONE) == 0)
return true;
return false;
}
EXPORT_SYMBOL(file_ns_capable);
/**
* privileged_wrt_inode_uidgid - Do capabilities in the namespace work over the inode?
* @ns: The user namespace in question
* @idmap: idmap of the mount @inode was found from
* @inode: The inode in question
*
* Return true if the inode uid and gid are within the namespace.
*/
bool privileged_wrt_inode_uidgid(struct user_namespace *ns,
struct mnt_idmap *idmap,
const struct inode *inode)
{
return vfsuid_has_mapping(ns, i_uid_into_vfsuid(idmap, inode)) && vfsgid_has_mapping(ns, i_gid_into_vfsgid(idmap, inode));
}
/**
* capable_wrt_inode_uidgid - Check nsown_capable and uid and gid mapped
* @idmap: idmap of the mount @inode was found from
* @inode: The inode in question
* @cap: The capability in question
*
* Return true if the current task has the given capability targeted at
* its own user namespace and that the given inode's uid and gid are
* mapped into the current user namespace.
*/
bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap,
const struct inode *inode, int cap)
{
struct user_namespace *ns = current_user_ns(); return ns_capable(ns, cap) &&
privileged_wrt_inode_uidgid(ns, idmap, inode);
}
EXPORT_SYMBOL(capable_wrt_inode_uidgid);
/**
* ptracer_capable - Determine if the ptracer holds CAP_SYS_PTRACE in the namespace
* @tsk: The task that may be ptraced
* @ns: The user namespace to search for CAP_SYS_PTRACE in
*
* Return true if the task that is ptracing the current task had CAP_SYS_PTRACE
* in the specified user namespace.
*/
bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns)
{
int ret = 0; /* An absent tracer adds no restrictions */
const struct cred *cred;
rcu_read_lock();
cred = rcu_dereference(tsk->ptracer_cred);
if (cred)
ret = security_capable(cred, ns, CAP_SYS_PTRACE,
CAP_OPT_NOAUDIT);
rcu_read_unlock();
return (ret == 0);
}
// SPDX-License-Identifier: GPL-2.0
/*
* This file contains functions which manage clock event devices.
*
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
* Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
*/
#include <linux/clockchips.h>
#include <linux/hrtimer.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/device.h>
#include "tick-internal.h"
/* The registered clock event devices */
static LIST_HEAD(clockevent_devices);
static LIST_HEAD(clockevents_released);
/* Protection for the above */
static DEFINE_RAW_SPINLOCK(clockevents_lock);
/* Protection for unbind operations */
static DEFINE_MUTEX(clockevents_mutex);
struct ce_unbind {
struct clock_event_device *ce;
int res;
};
static u64 cev_delta2ns(unsigned long latch, struct clock_event_device *evt,
bool ismax)
{
u64 clc = (u64) latch << evt->shift;
u64 rnd;
if (WARN_ON(!evt->mult))
evt->mult = 1;
rnd = (u64) evt->mult - 1;
/*
* Upper bound sanity check. If the backwards conversion is
* not equal latch, we know that the above shift overflowed.
*/
if ((clc >> evt->shift) != (u64)latch)
clc = ~0ULL;
/*
* Scaled math oddities:
*
* For mult <= (1 << shift) we can safely add mult - 1 to
* prevent integer rounding loss. So the backwards conversion
* from nsec to device ticks will be correct.
*
* For mult > (1 << shift), i.e. device frequency is > 1GHz we
* need to be careful. Adding mult - 1 will result in a value
* which when converted back to device ticks can be larger
* than latch by up to (mult - 1) >> shift. For the min_delta
* calculation we still want to apply this in order to stay
* above the minimum device ticks limit. For the upper limit
* we would end up with a latch value larger than the upper
* limit of the device, so we omit the add to stay below the
* device upper boundary.
*
* Also omit the add if it would overflow the u64 boundary.
*/
if ((~0ULL - clc > rnd) &&
(!ismax || evt->mult <= (1ULL << evt->shift)))
clc += rnd;
do_div(clc, evt->mult);
/* Deltas less than 1usec are pointless noise */
return clc > 1000 ? clc : 1000;
}
/**
* clockevent_delta2ns - Convert a latch value (device ticks) to nanoseconds
* @latch: value to convert
* @evt: pointer to clock event device descriptor
*
* Math helper, returns latch value converted to nanoseconds (bound checked)
*/
u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt)
{
return cev_delta2ns(latch, evt, false);
}
EXPORT_SYMBOL_GPL(clockevent_delta2ns);
static int __clockevents_switch_state(struct clock_event_device *dev,
enum clock_event_state state)
{
if (dev->features & CLOCK_EVT_FEAT_DUMMY)
return 0;
/* Transition with new state-specific callbacks */
switch (state) {
case CLOCK_EVT_STATE_DETACHED:
/* The clockevent device is getting replaced. Shut it down. */
case CLOCK_EVT_STATE_SHUTDOWN:
if (dev->set_state_shutdown)
return dev->set_state_shutdown(dev);
return 0;
case CLOCK_EVT_STATE_PERIODIC:
/* Core internal bug */
if (!(dev->features & CLOCK_EVT_FEAT_PERIODIC))
return -ENOSYS;
if (dev->set_state_periodic)
return dev->set_state_periodic(dev);
return 0;
case CLOCK_EVT_STATE_ONESHOT:
/* Core internal bug */
if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
return -ENOSYS;
if (dev->set_state_oneshot)
return dev->set_state_oneshot(dev);
return 0;
case CLOCK_EVT_STATE_ONESHOT_STOPPED:
/* Core internal bug */
if (WARN_ONCE(!clockevent_state_oneshot(dev),
"Current state: %d\n",
clockevent_get_state(dev)))
return -EINVAL;
if (dev->set_state_oneshot_stopped)
return dev->set_state_oneshot_stopped(dev);
else
return -ENOSYS;
default:
return -ENOSYS;
}
}
/**
* clockevents_switch_state - set the operating state of a clock event device
* @dev: device to modify
* @state: new state
*
* Must be called with interrupts disabled !
*/
void clockevents_switch_state(struct clock_event_device *dev,
enum clock_event_state state)
{
if (clockevent_get_state(dev) != state) {
if (__clockevents_switch_state(dev, state))
return;
clockevent_set_state(dev, state);
/*
* A nsec2cyc multiplicator of 0 is invalid and we'd crash
* on it, so fix it up and emit a warning:
*/
if (clockevent_state_oneshot(dev)) {
if (WARN_ON(!dev->mult))
dev->mult = 1;
}
}
}
/**
* clockevents_shutdown - shutdown the device and clear next_event
* @dev: device to shutdown
*/
void clockevents_shutdown(struct clock_event_device *dev)
{
clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
dev->next_event = KTIME_MAX;
}
/**
* clockevents_tick_resume - Resume the tick device before using it again
* @dev: device to resume
*/
int clockevents_tick_resume(struct clock_event_device *dev)
{
int ret = 0;
if (dev->tick_resume)
ret = dev->tick_resume(dev);
return ret;
}
#ifdef CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST
/* Limit min_delta to a jiffie */
#define MIN_DELTA_LIMIT (NSEC_PER_SEC / HZ)
/**
* clockevents_increase_min_delta - raise minimum delta of a clock event device
* @dev: device to increase the minimum delta
*
* Returns 0 on success, -ETIME when the minimum delta reached the limit.
*/
static int clockevents_increase_min_delta(struct clock_event_device *dev)
{
/* Nothing to do if we already reached the limit */
if (dev->min_delta_ns >= MIN_DELTA_LIMIT) { printk_deferred(KERN_WARNING
"CE: Reprogramming failure. Giving up\n");
dev->next_event = KTIME_MAX;
return -ETIME;
}
if (dev->min_delta_ns < 5000)
dev->min_delta_ns = 5000;
else
dev->min_delta_ns += dev->min_delta_ns >> 1;
if (dev->min_delta_ns > MIN_DELTA_LIMIT)
dev->min_delta_ns = MIN_DELTA_LIMIT; printk_deferred(KERN_WARNING
"CE: %s increased min_delta_ns to %llu nsec\n",
dev->name ? dev->name : "?",
(unsigned long long) dev->min_delta_ns);
return 0;
}
/**
* clockevents_program_min_delta - Set clock event device to the minimum delay.
* @dev: device to program
*
* Returns 0 on success, -ETIME when the retry loop failed.
*/
static int clockevents_program_min_delta(struct clock_event_device *dev)
{
unsigned long long clc;
int64_t delta;
int i;
for (i = 0;;) {
delta = dev->min_delta_ns;
dev->next_event = ktime_add_ns(ktime_get(), delta);
if (clockevent_state_shutdown(dev))
return 0; dev->retries++; clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
if (dev->set_next_event((unsigned long) clc, dev) == 0)
return 0;
if (++i > 2) {
/*
* We tried 3 times to program the device with the
* given min_delta_ns. Try to increase the minimum
* delta, if that fails as well get out of here.
*/
if (clockevents_increase_min_delta(dev))
return -ETIME;
i = 0;
}
}
}
#else /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */
/**
* clockevents_program_min_delta - Set clock event device to the minimum delay.
* @dev: device to program
*
* Returns 0 on success, -ETIME when the retry loop failed.
*/
static int clockevents_program_min_delta(struct clock_event_device *dev)
{
unsigned long long clc;
int64_t delta = 0;
int i;
for (i = 0; i < 10; i++) {
delta += dev->min_delta_ns;
dev->next_event = ktime_add_ns(ktime_get(), delta);
if (clockevent_state_shutdown(dev))
return 0;
dev->retries++;
clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
if (dev->set_next_event((unsigned long) clc, dev) == 0)
return 0;
}
return -ETIME;
}
#endif /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */
/**
* clockevents_program_event - Reprogram the clock event device.
* @dev: device to program
* @expires: absolute expiry time (monotonic clock)
* @force: program minimum delay if expires can not be set
*
* Returns 0 on success, -ETIME when the event is in the past.
*/
int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
bool force)
{
unsigned long long clc;
int64_t delta;
int rc;
if (WARN_ON_ONCE(expires < 0))
return -ETIME;
dev->next_event = expires;
if (clockevent_state_shutdown(dev))
return 0;
/* We must be in ONESHOT state here */
WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n",
clockevent_get_state(dev));
/* Shortcut for clockevent devices that can deal with ktime. */
if (dev->features & CLOCK_EVT_FEAT_KTIME) return dev->set_next_ktime(expires, dev); delta = ktime_to_ns(ktime_sub(expires, ktime_get()));
if (delta <= 0)
return force ? clockevents_program_min_delta(dev) : -ETIME; delta = min(delta, (int64_t) dev->max_delta_ns);
delta = max(delta, (int64_t) dev->min_delta_ns);
clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
rc = dev->set_next_event((unsigned long) clc, dev);
return (rc && force) ? clockevents_program_min_delta(dev) : rc;
}
/*
* Called after a notify add to make devices available which were
* released from the notifier call.
*/
static void clockevents_notify_released(void)
{
struct clock_event_device *dev;
while (!list_empty(&clockevents_released)) {
dev = list_entry(clockevents_released.next,
struct clock_event_device, list);
list_move(&dev->list, &clockevent_devices);
tick_check_new_device(dev);
}
}
/*
* Try to install a replacement clock event device
*/
static int clockevents_replace(struct clock_event_device *ced)
{
struct clock_event_device *dev, *newdev = NULL;
list_for_each_entry(dev, &clockevent_devices, list) {
if (dev == ced || !clockevent_state_detached(dev))
continue;
if (!tick_check_replacement(newdev, dev))
continue;
if (!try_module_get(dev->owner))
continue;
if (newdev)
module_put(newdev->owner);
newdev = dev;
}
if (newdev) {
tick_install_replacement(newdev);
list_del_init(&ced->list);
}
return newdev ? 0 : -EBUSY;
}
/*
* Called with clockevents_mutex and clockevents_lock held
*/
static int __clockevents_try_unbind(struct clock_event_device *ced, int cpu)
{
/* Fast track. Device is unused */
if (clockevent_state_detached(ced)) {
list_del_init(&ced->list);
return 0;
}
return ced == per_cpu(tick_cpu_device, cpu).evtdev ? -EAGAIN : -EBUSY;
}
/*
* SMP function call to unbind a device
*/
static void __clockevents_unbind(void *arg)
{
struct ce_unbind *cu = arg;
int res;
raw_spin_lock(&clockevents_lock);
res = __clockevents_try_unbind(cu->ce, smp_processor_id());
if (res == -EAGAIN)
res = clockevents_replace(cu->ce);
cu->res = res;
raw_spin_unlock(&clockevents_lock);
}
/*
* Issues smp function call to unbind a per cpu device. Called with
* clockevents_mutex held.
*/
static int clockevents_unbind(struct clock_event_device *ced, int cpu)
{
struct ce_unbind cu = { .ce = ced, .res = -ENODEV };
smp_call_function_single(cpu, __clockevents_unbind, &cu, 1);
return cu.res;
}
/*
* Unbind a clockevents device.
*/
int clockevents_unbind_device(struct clock_event_device *ced, int cpu)
{
int ret;
mutex_lock(&clockevents_mutex);
ret = clockevents_unbind(ced, cpu);
mutex_unlock(&clockevents_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(clockevents_unbind_device);
/**
* clockevents_register_device - register a clock event device
* @dev: device to register
*/
void clockevents_register_device(struct clock_event_device *dev)
{
unsigned long flags;
/* Initialize state to DETACHED */
clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
if (!dev->cpumask) {
WARN_ON(num_possible_cpus() > 1);
dev->cpumask = cpumask_of(smp_processor_id());
}
if (dev->cpumask == cpu_all_mask) {
WARN(1, "%s cpumask == cpu_all_mask, using cpu_possible_mask instead\n",
dev->name);
dev->cpumask = cpu_possible_mask;
}
raw_spin_lock_irqsave(&clockevents_lock, flags);
list_add(&dev->list, &clockevent_devices);
tick_check_new_device(dev);
clockevents_notify_released();
raw_spin_unlock_irqrestore(&clockevents_lock, flags);
}
EXPORT_SYMBOL_GPL(clockevents_register_device);
static void clockevents_config(struct clock_event_device *dev, u32 freq)
{
u64 sec;
if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
return;
/*
* Calculate the maximum number of seconds we can sleep. Limit
* to 10 minutes for hardware which can program more than
* 32bit ticks so we still get reasonable conversion values.
*/
sec = dev->max_delta_ticks;
do_div(sec, freq);
if (!sec)
sec = 1;
else if (sec > 600 && dev->max_delta_ticks > UINT_MAX)
sec = 600;
clockevents_calc_mult_shift(dev, freq, sec);
dev->min_delta_ns = cev_delta2ns(dev->min_delta_ticks, dev, false);
dev->max_delta_ns = cev_delta2ns(dev->max_delta_ticks, dev, true);
}
/**
* clockevents_config_and_register - Configure and register a clock event device
* @dev: device to register
* @freq: The clock frequency
* @min_delta: The minimum clock ticks to program in oneshot mode
* @max_delta: The maximum clock ticks to program in oneshot mode
*
* min/max_delta can be 0 for devices which do not support oneshot mode.
*/
void clockevents_config_and_register(struct clock_event_device *dev,
u32 freq, unsigned long min_delta,
unsigned long max_delta)
{
dev->min_delta_ticks = min_delta;
dev->max_delta_ticks = max_delta;
clockevents_config(dev, freq);
clockevents_register_device(dev);
}
EXPORT_SYMBOL_GPL(clockevents_config_and_register);
int __clockevents_update_freq(struct clock_event_device *dev, u32 freq)
{
clockevents_config(dev, freq);
if (clockevent_state_oneshot(dev))
return clockevents_program_event(dev, dev->next_event, false);
if (clockevent_state_periodic(dev))
return __clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
return 0;
}
/**
* clockevents_update_freq - Update frequency and reprogram a clock event device.
* @dev: device to modify
* @freq: new device frequency
*
* Reconfigure and reprogram a clock event device in oneshot
* mode. Must be called on the cpu for which the device delivers per
* cpu timer events. If called for the broadcast device the core takes
* care of serialization.
*
* Returns 0 on success, -ETIME when the event is in the past.
*/
int clockevents_update_freq(struct clock_event_device *dev, u32 freq)
{
unsigned long flags;
int ret;
local_irq_save(flags);
ret = tick_broadcast_update_freq(dev, freq);
if (ret == -ENODEV)
ret = __clockevents_update_freq(dev, freq);
local_irq_restore(flags);
return ret;
}
/*
* Noop handler when we shut down an event device
*/
void clockevents_handle_noop(struct clock_event_device *dev)
{
}
/**
* clockevents_exchange_device - release and request clock devices
* @old: device to release (can be NULL)
* @new: device to request (can be NULL)
*
* Called from various tick functions with clockevents_lock held and
* interrupts disabled.
*/
void clockevents_exchange_device(struct clock_event_device *old,
struct clock_event_device *new)
{
/*
* Caller releases a clock event device. We queue it into the
* released list and do a notify add later.
*/
if (old) {
module_put(old->owner);
clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED);
list_move(&old->list, &clockevents_released);
}
if (new) {
BUG_ON(!clockevent_state_detached(new));
clockevents_shutdown(new);
}
}
/**
* clockevents_suspend - suspend clock devices
*/
void clockevents_suspend(void)
{
struct clock_event_device *dev;
list_for_each_entry_reverse(dev, &clockevent_devices, list)
if (dev->suspend && !clockevent_state_detached(dev))
dev->suspend(dev);
}
/**
* clockevents_resume - resume clock devices
*/
void clockevents_resume(void)
{
struct clock_event_device *dev;
list_for_each_entry(dev, &clockevent_devices, list)
if (dev->resume && !clockevent_state_detached(dev))
dev->resume(dev);
}
#ifdef CONFIG_HOTPLUG_CPU
# ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
/**
* tick_offline_cpu - Take CPU out of the broadcast mechanism
* @cpu: The outgoing CPU
*
* Called on the outgoing CPU after it took itself offline.
*/
void tick_offline_cpu(unsigned int cpu)
{
raw_spin_lock(&clockevents_lock);
tick_broadcast_offline(cpu);
raw_spin_unlock(&clockevents_lock);
}
# endif
/**
* tick_cleanup_dead_cpu - Cleanup the tick and clockevents of a dead cpu
* @cpu: The dead CPU
*/
void tick_cleanup_dead_cpu(int cpu)
{
struct clock_event_device *dev, *tmp;
unsigned long flags;
raw_spin_lock_irqsave(&clockevents_lock, flags);
tick_shutdown(cpu);
/*
* Unregister the clock event devices which were
* released from the users in the notify chain.
*/
list_for_each_entry_safe(dev, tmp, &clockevents_released, list)
list_del(&dev->list);
/*
* Now check whether the CPU has left unused per cpu devices
*/
list_for_each_entry_safe(dev, tmp, &clockevent_devices, list) {
if (cpumask_test_cpu(cpu, dev->cpumask) &&
cpumask_weight(dev->cpumask) == 1 &&
!tick_is_broadcast_device(dev)) {
BUG_ON(!clockevent_state_detached(dev));
list_del(&dev->list);
}
}
raw_spin_unlock_irqrestore(&clockevents_lock, flags);
}
#endif
#ifdef CONFIG_SYSFS
static const struct bus_type clockevents_subsys = {
.name = "clockevents",
.dev_name = "clockevent",
};
static DEFINE_PER_CPU(struct device, tick_percpu_dev);
static struct tick_device *tick_get_tick_dev(struct device *dev);
static ssize_t current_device_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct tick_device *td;
ssize_t count = 0;
raw_spin_lock_irq(&clockevents_lock);
td = tick_get_tick_dev(dev);
if (td && td->evtdev)
count = snprintf(buf, PAGE_SIZE, "%s\n", td->evtdev->name);
raw_spin_unlock_irq(&clockevents_lock);
return count;
}
static DEVICE_ATTR_RO(current_device);
/* We don't support the abomination of removable broadcast devices */
static ssize_t unbind_device_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
char name[CS_NAME_LEN];
ssize_t ret = sysfs_get_uname(buf, name, count);
struct clock_event_device *ce = NULL, *iter;
if (ret < 0)
return ret;
ret = -ENODEV;
mutex_lock(&clockevents_mutex);
raw_spin_lock_irq(&clockevents_lock);
list_for_each_entry(iter, &clockevent_devices, list) {
if (!strcmp(iter->name, name)) {
ret = __clockevents_try_unbind(iter, dev->id);
ce = iter;
break;
}
}
raw_spin_unlock_irq(&clockevents_lock);
/*
* We hold clockevents_mutex, so ce can't go away
*/
if (ret == -EAGAIN)
ret = clockevents_unbind(ce, dev->id);
mutex_unlock(&clockevents_mutex);
return ret ? ret : count;
}
static DEVICE_ATTR_WO(unbind_device);
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
static struct device tick_bc_dev = {
.init_name = "broadcast",
.id = 0,
.bus = &clockevents_subsys,
};
static struct tick_device *tick_get_tick_dev(struct device *dev)
{
return dev == &tick_bc_dev ? tick_get_broadcast_device() :
&per_cpu(tick_cpu_device, dev->id);
}
static __init int tick_broadcast_init_sysfs(void)
{
int err = device_register(&tick_bc_dev);
if (!err)
err = device_create_file(&tick_bc_dev, &dev_attr_current_device);
return err;
}
#else
static struct tick_device *tick_get_tick_dev(struct device *dev)
{
return &per_cpu(tick_cpu_device, dev->id);
}
static inline int tick_broadcast_init_sysfs(void) { return 0; }
#endif
static int __init tick_init_sysfs(void)
{
int cpu;
for_each_possible_cpu(cpu) {
struct device *dev = &per_cpu(tick_percpu_dev, cpu);
int err;
dev->id = cpu;
dev->bus = &clockevents_subsys;
err = device_register(dev);
if (!err)
err = device_create_file(dev, &dev_attr_current_device);
if (!err)
err = device_create_file(dev, &dev_attr_unbind_device);
if (err)
return err;
}
return tick_broadcast_init_sysfs();
}
static int __init clockevents_init_sysfs(void)
{
int err = subsys_system_register(&clockevents_subsys, NULL);
if (!err)
err = tick_init_sysfs();
return err;
}
device_initcall(clockevents_init_sysfs);
#endif /* SYSFS */
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
*/
#include <linux/sched/debug.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
#include <linux/kdebug.h>
#include <linux/export.h>
#include <linux/ptrace.h>
#include <linux/kexec.h>
#include <linux/sysfs.h>
#include <linux/bug.h>
#include <linux/nmi.h>
#include <asm/cpu_entry_area.h>
#include <asm/stacktrace.h>
static const char * const exception_stack_names[] = {
[ ESTACK_DF ] = "#DF",
[ ESTACK_NMI ] = "NMI",
[ ESTACK_DB ] = "#DB",
[ ESTACK_MCE ] = "#MC",
[ ESTACK_VC ] = "#VC",
[ ESTACK_VC2 ] = "#VC2",
};
const char *stack_type_name(enum stack_type type)
{
BUILD_BUG_ON(N_EXCEPTION_STACKS != 6);
if (type == STACK_TYPE_TASK)
return "TASK";
if (type == STACK_TYPE_IRQ)
return "IRQ";
if (type == STACK_TYPE_SOFTIRQ)
return "SOFTIRQ";
if (type == STACK_TYPE_ENTRY) {
/*
* On 64-bit, we have a generic entry stack that we
* use for all the kernel entry points, including
* SYSENTER.
*/
return "ENTRY_TRAMPOLINE";
}
if (type >= STACK_TYPE_EXCEPTION && type <= STACK_TYPE_EXCEPTION_LAST)
return exception_stack_names[type - STACK_TYPE_EXCEPTION];
return NULL;
}
/**
* struct estack_pages - Page descriptor for exception stacks
* @offs: Offset from the start of the exception stack area
* @size: Size of the exception stack
* @type: Type to store in the stack_info struct
*/
struct estack_pages {
u32 offs;
u16 size;
u16 type;
};
#define EPAGERANGE(st) \
[PFN_DOWN(CEA_ESTACK_OFFS(st)) ... \
PFN_DOWN(CEA_ESTACK_OFFS(st) + CEA_ESTACK_SIZE(st) - 1)] = { \
.offs = CEA_ESTACK_OFFS(st), \
.size = CEA_ESTACK_SIZE(st), \
.type = STACK_TYPE_EXCEPTION + ESTACK_ ##st, }
/*
* Array of exception stack page descriptors. If the stack is larger than
* PAGE_SIZE, all pages covering a particular stack will have the same
* info. The guard pages including the not mapped DB2 stack are zeroed
* out.
*/
static const
struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = {
EPAGERANGE(DF),
EPAGERANGE(NMI),
EPAGERANGE(DB),
EPAGERANGE(MCE),
EPAGERANGE(VC),
EPAGERANGE(VC2),
};
static __always_inline bool in_exception_stack(unsigned long *stack, struct stack_info *info)
{
unsigned long begin, end, stk = (unsigned long)stack;
const struct estack_pages *ep;
struct pt_regs *regs;
unsigned int k;
BUILD_BUG_ON(N_EXCEPTION_STACKS != 6);
begin = (unsigned long)__this_cpu_read(cea_exception_stacks);
/*
* Handle the case where stack trace is collected _before_
* cea_exception_stacks had been initialized.
*/
if (!begin)
return false;
end = begin + sizeof(struct cea_exception_stacks);
/* Bail if @stack is outside the exception stack area. */
if (stk < begin || stk >= end)
return false;
/* Calc page offset from start of exception stacks */
k = (stk - begin) >> PAGE_SHIFT;
/* Lookup the page descriptor */
ep = &estack_pages[k];
/* Guard page? */
if (!ep->size)
return false;
begin += (unsigned long)ep->offs;
end = begin + (unsigned long)ep->size;
regs = (struct pt_regs *)end - 1;
info->type = ep->type;
info->begin = (unsigned long *)begin;
info->end = (unsigned long *)end;
info->next_sp = (unsigned long *)regs->sp;
return true;
}
static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info)
{
unsigned long *end = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);
unsigned long *begin;
/*
* @end points directly to the top most stack entry to avoid a -8
* adjustment in the stack switch hotpath. Adjust it back before
* calculating @begin.
*/
end++;
begin = end - (IRQ_STACK_SIZE / sizeof(long));
/*
* Due to the switching logic RSP can never be == @end because the
* final operation is 'popq %rsp' which means after that RSP points
* to the original stack and not to @end.
*/
if (stack < begin || stack >= end)
return false;
info->type = STACK_TYPE_IRQ;
info->begin = begin;
info->end = end;
/*
* The next stack pointer is stored at the top of the irq stack
* before switching to the irq stack. Actual stack entries are all
* below that.
*/
info->next_sp = (unsigned long *)*(end - 1);
return true;
}
bool noinstr get_stack_info_noinstr(unsigned long *stack, struct task_struct *task,
struct stack_info *info)
{
if (in_task_stack(stack, task, info))
return true;
if (task != current)
return false;
if (in_exception_stack(stack, info))
return true;
if (in_irq_stack(stack, info))
return true;
if (in_entry_stack(stack, info))
return true;
return false;
}
int get_stack_info(unsigned long *stack, struct task_struct *task,
struct stack_info *info, unsigned long *visit_mask)
{
task = task ? : current; if (!stack)
goto unknown;
if (!get_stack_info_noinstr(stack, task, info))
goto unknown;
/*
* Make sure we don't iterate through any given stack more than once.
* If it comes up a second time then there's something wrong going on:
* just break out and report an unknown stack type.
*/
if (visit_mask) { if (*visit_mask & (1UL << info->type)) { if (task == current) printk_deferred_once(KERN_WARNING "WARNING: stack recursion on stack type %d\n", info->type);
goto unknown;
}
*visit_mask |= 1UL << info->type;
}
return 0;
unknown:
info->type = STACK_TYPE_UNKNOWN;
return -EINVAL;
}
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Statically sized hash table implementation
* (C) 2012 Sasha Levin <levinsasha928@gmail.com>
*/
#ifndef _LINUX_HASHTABLE_H
#define _LINUX_HASHTABLE_H
#include <linux/list.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/hash.h>
#include <linux/rculist.h>
#define DEFINE_HASHTABLE(name, bits) \
struct hlist_head name[1 << (bits)] = \
{ [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }
#define DEFINE_READ_MOSTLY_HASHTABLE(name, bits) \
struct hlist_head name[1 << (bits)] __read_mostly = \
{ [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }
#define DECLARE_HASHTABLE(name, bits) \
struct hlist_head name[1 << (bits)]
#define HASH_SIZE(name) (ARRAY_SIZE(name))
#define HASH_BITS(name) ilog2(HASH_SIZE(name))
/* Use hash_32 when possible to allow for fast 32bit hashing in 64bit kernels. */
#define hash_min(val, bits) \
(sizeof(val) <= 4 ? hash_32(val, bits) : hash_long(val, bits))
static inline void __hash_init(struct hlist_head *ht, unsigned int sz)
{
unsigned int i;
for (i = 0; i < sz; i++)
INIT_HLIST_HEAD(&ht[i]);
}
/**
* hash_init - initialize a hash table
* @hashtable: hashtable to be initialized
*
* Calculates the size of the hashtable from the given parameter, otherwise
* same as hash_init_size.
*
* This has to be a macro since HASH_BITS() will not work on pointers since
* it calculates the size during preprocessing.
*/
#define hash_init(hashtable) __hash_init(hashtable, HASH_SIZE(hashtable))
/**
* hash_add - add an object to a hashtable
* @hashtable: hashtable to add to
* @node: the &struct hlist_node of the object to be added
* @key: the key of the object to be added
*/
#define hash_add(hashtable, node, key) \
hlist_add_head(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])
/**
* hash_add_rcu - add an object to a rcu enabled hashtable
* @hashtable: hashtable to add to
* @node: the &struct hlist_node of the object to be added
* @key: the key of the object to be added
*/
#define hash_add_rcu(hashtable, node, key) \
hlist_add_head_rcu(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])
/**
* hash_hashed - check whether an object is in any hashtable
* @node: the &struct hlist_node of the object to be checked
*/
static inline bool hash_hashed(struct hlist_node *node)
{
return !hlist_unhashed(node);
}
static inline bool __hash_empty(struct hlist_head *ht, unsigned int sz)
{
unsigned int i;
for (i = 0; i < sz; i++)
if (!hlist_empty(&ht[i]))
return false;
return true;
}
/**
* hash_empty - check whether a hashtable is empty
* @hashtable: hashtable to check
*
* This has to be a macro since HASH_BITS() will not work on pointers since
* it calculates the size during preprocessing.
*/
#define hash_empty(hashtable) __hash_empty(hashtable, HASH_SIZE(hashtable))
/**
* hash_del - remove an object from a hashtable
* @node: &struct hlist_node of the object to remove
*/
static inline void hash_del(struct hlist_node *node)
{
hlist_del_init(node);
}
/**
* hash_del_rcu - remove an object from a rcu enabled hashtable
* @node: &struct hlist_node of the object to remove
*/
static inline void hash_del_rcu(struct hlist_node *node)
{
hlist_del_init_rcu(node);
}
/**
* hash_for_each - iterate over a hashtable
* @name: hashtable to iterate
* @bkt: integer to use as bucket loop cursor
* @obj: the type * to use as a loop cursor for each entry
* @member: the name of the hlist_node within the struct
*/
#define hash_for_each(name, bkt, obj, member) \
for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
(bkt)++)\
hlist_for_each_entry(obj, &name[bkt], member)
/**
* hash_for_each_rcu - iterate over a rcu enabled hashtable
* @name: hashtable to iterate
* @bkt: integer to use as bucket loop cursor
* @obj: the type * to use as a loop cursor for each entry
* @member: the name of the hlist_node within the struct
*/
#define hash_for_each_rcu(name, bkt, obj, member) \
for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
(bkt)++)\
hlist_for_each_entry_rcu(obj, &name[bkt], member)
/**
* hash_for_each_safe - iterate over a hashtable safe against removal of
* hash entry
* @name: hashtable to iterate
* @bkt: integer to use as bucket loop cursor
* @tmp: a &struct hlist_node used for temporary storage
* @obj: the type * to use as a loop cursor for each entry
* @member: the name of the hlist_node within the struct
*/
#define hash_for_each_safe(name, bkt, tmp, obj, member) \
for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
(bkt)++)\
hlist_for_each_entry_safe(obj, tmp, &name[bkt], member)
/**
* hash_for_each_possible - iterate over all possible objects hashing to the
* same bucket
* @name: hashtable to iterate
* @obj: the type * to use as a loop cursor for each entry
* @member: the name of the hlist_node within the struct
* @key: the key of the objects to iterate over
*/
#define hash_for_each_possible(name, obj, member, key) \
hlist_for_each_entry(obj, &name[hash_min(key, HASH_BITS(name))], member)
/**
* hash_for_each_possible_rcu - iterate over all possible objects hashing to the
* same bucket in an rcu enabled hashtable
* @name: hashtable to iterate
* @obj: the type * to use as a loop cursor for each entry
* @member: the name of the hlist_node within the struct
* @key: the key of the objects to iterate over
*/
#define hash_for_each_possible_rcu(name, obj, member, key, cond...) \
hlist_for_each_entry_rcu(obj, &name[hash_min(key, HASH_BITS(name))],\
member, ## cond)
/**
* hash_for_each_possible_rcu_notrace - iterate over all possible objects hashing
* to the same bucket in an rcu enabled hashtable in a rcu enabled hashtable
* @name: hashtable to iterate
* @obj: the type * to use as a loop cursor for each entry
* @member: the name of the hlist_node within the struct
* @key: the key of the objects to iterate over
*
* This is the same as hash_for_each_possible_rcu() except that it does
* not do any RCU debugging or tracing.
*/
#define hash_for_each_possible_rcu_notrace(name, obj, member, key) \
hlist_for_each_entry_rcu_notrace(obj, \
&name[hash_min(key, HASH_BITS(name))], member)
/**
* hash_for_each_possible_safe - iterate over all possible objects hashing to the
* same bucket safe against removals
* @name: hashtable to iterate
* @obj: the type * to use as a loop cursor for each entry
* @tmp: a &struct hlist_node used for temporary storage
* @member: the name of the hlist_node within the struct
* @key: the key of the objects to iterate over
*/
#define hash_for_each_possible_safe(name, obj, tmp, member, key) \
hlist_for_each_entry_safe(obj, tmp,\
&name[hash_min(key, HASH_BITS(name))], member)
#endif
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* i2c.h - definitions for the Linux i2c bus interface
* Copyright (C) 1995-2000 Simon G. Vogl
* Copyright (C) 2013-2019 Wolfram Sang <wsa@kernel.org>
*
* With some changes from Kyösti Mälkki <kmalkki@cc.hut.fi> and
* Frodo Looijaard <frodol@dds.nl>
*/
#ifndef _LINUX_I2C_H
#define _LINUX_I2C_H
#include <linux/acpi.h> /* for acpi_handle */
#include <linux/bits.h>
#include <linux/mod_devicetable.h>
#include <linux/device.h> /* for struct device */
#include <linux/sched.h> /* for completion */
#include <linux/mutex.h>
#include <linux/regulator/consumer.h>
#include <linux/rtmutex.h>
#include <linux/irqdomain.h> /* for Host Notify IRQ */
#include <linux/of.h> /* for struct device_node */
#include <linux/swab.h> /* for swab16 */
#include <uapi/linux/i2c.h>
extern const struct bus_type i2c_bus_type;
extern const struct device_type i2c_adapter_type;
extern const struct device_type i2c_client_type;
/* --- General options ------------------------------------------------ */
struct i2c_msg;
struct i2c_algorithm;
struct i2c_adapter;
struct i2c_client;
struct i2c_driver;
struct i2c_device_identity;
union i2c_smbus_data;
struct i2c_board_info;
enum i2c_slave_event;
typedef int (*i2c_slave_cb_t)(struct i2c_client *client,
enum i2c_slave_event event, u8 *val);
/* I2C Frequency Modes */
#define I2C_MAX_STANDARD_MODE_FREQ 100000
#define I2C_MAX_FAST_MODE_FREQ 400000
#define I2C_MAX_FAST_MODE_PLUS_FREQ 1000000
#define I2C_MAX_TURBO_MODE_FREQ 1400000
#define I2C_MAX_HIGH_SPEED_MODE_FREQ 3400000
#define I2C_MAX_ULTRA_FAST_MODE_FREQ 5000000
struct module;
struct property_entry;
#if IS_ENABLED(CONFIG_I2C)
/* Return the Frequency mode string based on the bus frequency */
const char *i2c_freq_mode_string(u32 bus_freq_hz);
/*
* The master routines are the ones normally used to transmit data to devices
* on a bus (or read from them). Apart from two basic transfer functions to
* transmit one message at a time, a more complex version can be used to
* transmit an arbitrary number of messages without interruption.
* @count must be less than 64k since msg.len is u16.
*/
int i2c_transfer_buffer_flags(const struct i2c_client *client,
char *buf, int count, u16 flags);
/**
* i2c_master_recv - issue a single I2C message in master receive mode
* @client: Handle to slave device
* @buf: Where to store data read from slave
* @count: How many bytes to read, must be less than 64k since msg.len is u16
*
* Returns negative errno, or else the number of bytes read.
*/
static inline int i2c_master_recv(const struct i2c_client *client,
char *buf, int count)
{
return i2c_transfer_buffer_flags(client, buf, count, I2C_M_RD);
};
/**
* i2c_master_recv_dmasafe - issue a single I2C message in master receive mode
* using a DMA safe buffer
* @client: Handle to slave device
* @buf: Where to store data read from slave, must be safe to use with DMA
* @count: How many bytes to read, must be less than 64k since msg.len is u16
*
* Returns negative errno, or else the number of bytes read.
*/
static inline int i2c_master_recv_dmasafe(const struct i2c_client *client,
char *buf, int count)
{
return i2c_transfer_buffer_flags(client, buf, count,
I2C_M_RD | I2C_M_DMA_SAFE);
};
/**
* i2c_master_send - issue a single I2C message in master transmit mode
* @client: Handle to slave device
* @buf: Data that will be written to the slave
* @count: How many bytes to write, must be less than 64k since msg.len is u16
*
* Returns negative errno, or else the number of bytes written.
*/
static inline int i2c_master_send(const struct i2c_client *client,
const char *buf, int count)
{
return i2c_transfer_buffer_flags(client, (char *)buf, count, 0);
};
/**
* i2c_master_send_dmasafe - issue a single I2C message in master transmit mode
* using a DMA safe buffer
* @client: Handle to slave device
* @buf: Data that will be written to the slave, must be safe to use with DMA
* @count: How many bytes to write, must be less than 64k since msg.len is u16
*
* Returns negative errno, or else the number of bytes written.
*/
static inline int i2c_master_send_dmasafe(const struct i2c_client *client,
const char *buf, int count)
{
return i2c_transfer_buffer_flags(client, (char *)buf, count,
I2C_M_DMA_SAFE);
};
/* Transfer num messages.
*/
int i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num);
/* Unlocked flavor */
int __i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num);
/* This is the very generalized SMBus access routine. You probably do not
want to use this, though; one of the functions below may be much easier,
and probably just as fast.
Note that we use i2c_adapter here, because you do not need a specific
smbus adapter to call this function. */
s32 i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
unsigned short flags, char read_write, u8 command,
int protocol, union i2c_smbus_data *data);
/* Unlocked flavor */
s32 __i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
unsigned short flags, char read_write, u8 command,
int protocol, union i2c_smbus_data *data);
/* Now follow the 'nice' access routines. These also document the calling
conventions of i2c_smbus_xfer. */
u8 i2c_smbus_pec(u8 crc, u8 *p, size_t count);
s32 i2c_smbus_read_byte(const struct i2c_client *client);
s32 i2c_smbus_write_byte(const struct i2c_client *client, u8 value);
s32 i2c_smbus_read_byte_data(const struct i2c_client *client, u8 command);
s32 i2c_smbus_write_byte_data(const struct i2c_client *client,
u8 command, u8 value);
s32 i2c_smbus_read_word_data(const struct i2c_client *client, u8 command);
s32 i2c_smbus_write_word_data(const struct i2c_client *client,
u8 command, u16 value);
static inline s32
i2c_smbus_read_word_swapped(const struct i2c_client *client, u8 command)
{
s32 value = i2c_smbus_read_word_data(client, command);
return (value < 0) ? value : swab16(value);
}
static inline s32
i2c_smbus_write_word_swapped(const struct i2c_client *client,
u8 command, u16 value)
{
return i2c_smbus_write_word_data(client, command, swab16(value));
}
/* Returns the number of read bytes */
s32 i2c_smbus_read_block_data(const struct i2c_client *client,
u8 command, u8 *values);
s32 i2c_smbus_write_block_data(const struct i2c_client *client,
u8 command, u8 length, const u8 *values);
/* Returns the number of read bytes */
s32 i2c_smbus_read_i2c_block_data(const struct i2c_client *client,
u8 command, u8 length, u8 *values);
s32 i2c_smbus_write_i2c_block_data(const struct i2c_client *client,
u8 command, u8 length, const u8 *values);
s32 i2c_smbus_read_i2c_block_data_or_emulated(const struct i2c_client *client,
u8 command, u8 length,
u8 *values);
int i2c_get_device_id(const struct i2c_client *client,
struct i2c_device_identity *id);
const struct i2c_device_id *i2c_client_get_device_id(const struct i2c_client *client);
#endif /* I2C */
/**
* struct i2c_device_identity - i2c client device identification
* @manufacturer_id: 0 - 4095, database maintained by NXP
* @part_id: 0 - 511, according to manufacturer
* @die_revision: 0 - 7, according to manufacturer
*/
struct i2c_device_identity {
u16 manufacturer_id;
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS 0
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS_1 1
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS_2 2
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS_3 3
#define I2C_DEVICE_ID_RAMTRON_INTERNATIONAL 4
#define I2C_DEVICE_ID_ANALOG_DEVICES 5
#define I2C_DEVICE_ID_STMICROELECTRONICS 6
#define I2C_DEVICE_ID_ON_SEMICONDUCTOR 7
#define I2C_DEVICE_ID_SPRINTEK_CORPORATION 8
#define I2C_DEVICE_ID_ESPROS_PHOTONICS_AG 9
#define I2C_DEVICE_ID_FUJITSU_SEMICONDUCTOR 10
#define I2C_DEVICE_ID_FLIR 11
#define I2C_DEVICE_ID_O2MICRO 12
#define I2C_DEVICE_ID_ATMEL 13
#define I2C_DEVICE_ID_NONE 0xffff
u16 part_id;
u8 die_revision;
};
enum i2c_alert_protocol {
I2C_PROTOCOL_SMBUS_ALERT,
I2C_PROTOCOL_SMBUS_HOST_NOTIFY,
};
/**
* enum i2c_driver_flags - Flags for an I2C device driver
*
* @I2C_DRV_ACPI_WAIVE_D0_PROBE: Don't put the device in D0 state for probe
*/
enum i2c_driver_flags {
I2C_DRV_ACPI_WAIVE_D0_PROBE = BIT(0),
};
/**
* struct i2c_driver - represent an I2C device driver
* @class: What kind of i2c device we instantiate (for detect)
* @probe: Callback for device binding
* @remove: Callback for device unbinding
* @shutdown: Callback for device shutdown
* @alert: Alert callback, for example for the SMBus alert protocol
* @command: Callback for bus-wide signaling (optional)
* @driver: Device driver model driver
* @id_table: List of I2C devices supported by this driver
* @detect: Callback for device detection
* @address_list: The I2C addresses to probe (for detect)
* @clients: List of detected clients we created (for i2c-core use only)
* @flags: A bitmask of flags defined in &enum i2c_driver_flags
*
* The driver.owner field should be set to the module owner of this driver.
* The driver.name field should be set to the name of this driver.
*
* For automatic device detection, both @detect and @address_list must
* be defined. @class should also be set, otherwise only devices forced
* with module parameters will be created. The detect function must
* fill at least the name field of the i2c_board_info structure it is
* handed upon successful detection, and possibly also the flags field.
*
* If @detect is missing, the driver will still work fine for enumerated
* devices. Detected devices simply won't be supported. This is expected
* for the many I2C/SMBus devices which can't be detected reliably, and
* the ones which can always be enumerated in practice.
*
* The i2c_client structure which is handed to the @detect callback is
* not a real i2c_client. It is initialized just enough so that you can
* call i2c_smbus_read_byte_data and friends on it. Don't do anything
* else with it. In particular, calling dev_dbg and friends on it is
* not allowed.
*/
struct i2c_driver {
unsigned int class;
/* Standard driver model interfaces */
int (*probe)(struct i2c_client *client);
void (*remove)(struct i2c_client *client);
/* driver model interfaces that don't relate to enumeration */
void (*shutdown)(struct i2c_client *client);
/* Alert callback, for example for the SMBus alert protocol.
* The format and meaning of the data value depends on the protocol.
* For the SMBus alert protocol, there is a single bit of data passed
* as the alert response's low bit ("event flag").
* For the SMBus Host Notify protocol, the data corresponds to the
* 16-bit payload data reported by the slave device acting as master.
*/
void (*alert)(struct i2c_client *client, enum i2c_alert_protocol protocol,
unsigned int data);
/* a ioctl like command that can be used to perform specific functions
* with the device.
*/
int (*command)(struct i2c_client *client, unsigned int cmd, void *arg);
struct device_driver driver;
const struct i2c_device_id *id_table;
/* Device detection callback for automatic device creation */
int (*detect)(struct i2c_client *client, struct i2c_board_info *info);
const unsigned short *address_list;
struct list_head clients;
u32 flags;
};
#define to_i2c_driver(d) container_of(d, struct i2c_driver, driver)
/**
* struct i2c_client - represent an I2C slave device
* @flags: see I2C_CLIENT_* for possible flags
* @addr: Address used on the I2C bus connected to the parent adapter.
* @name: Indicates the type of the device, usually a chip name that's
* generic enough to hide second-sourcing and compatible revisions.
* @adapter: manages the bus segment hosting this I2C device
* @dev: Driver model device node for the slave.
* @init_irq: IRQ that was set at initialization
* @irq: indicates the IRQ generated by this device (if any)
* @detected: member of an i2c_driver.clients list or i2c-core's
* userspace_devices list
* @slave_cb: Callback when I2C slave mode of an adapter is used. The adapter
* calls it to pass on slave events to the slave driver.
* @devres_group_id: id of the devres group that will be created for resources
* acquired when probing this device.
*
* An i2c_client identifies a single device (i.e. chip) connected to an
* i2c bus. The behaviour exposed to Linux is defined by the driver
* managing the device.
*/
struct i2c_client {
unsigned short flags; /* div., see below */
#define I2C_CLIENT_PEC 0x04 /* Use Packet Error Checking */
#define I2C_CLIENT_TEN 0x10 /* we have a ten bit chip address */
/* Must equal I2C_M_TEN below */
#define I2C_CLIENT_SLAVE 0x20 /* we are the slave */
#define I2C_CLIENT_HOST_NOTIFY 0x40 /* We want to use I2C host notify */
#define I2C_CLIENT_WAKE 0x80 /* for board_info; true iff can wake */
#define I2C_CLIENT_SCCB 0x9000 /* Use Omnivision SCCB protocol */
/* Must match I2C_M_STOP|IGNORE_NAK */
unsigned short addr; /* chip address - NOTE: 7bit */
/* addresses are stored in the */
/* _LOWER_ 7 bits */
char name[I2C_NAME_SIZE];
struct i2c_adapter *adapter; /* the adapter we sit on */
struct device dev; /* the device structure */
int init_irq; /* irq set at initialization */
int irq; /* irq issued by device */
struct list_head detected;
#if IS_ENABLED(CONFIG_I2C_SLAVE)
i2c_slave_cb_t slave_cb; /* callback for slave mode */
#endif
void *devres_group_id; /* ID of probe devres group */
};
#define to_i2c_client(d) container_of(d, struct i2c_client, dev)
struct i2c_adapter *i2c_verify_adapter(struct device *dev);
const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
const struct i2c_client *client);
const void *i2c_get_match_data(const struct i2c_client *client);
static inline struct i2c_client *kobj_to_i2c_client(struct kobject *kobj)
{
struct device * const dev = kobj_to_dev(kobj);
return to_i2c_client(dev);
}
static inline void *i2c_get_clientdata(const struct i2c_client *client)
{
return dev_get_drvdata(&client->dev);
}
static inline void i2c_set_clientdata(struct i2c_client *client, void *data)
{
dev_set_drvdata(&client->dev, data);
}
/* I2C slave support */
enum i2c_slave_event {
I2C_SLAVE_READ_REQUESTED,
I2C_SLAVE_WRITE_REQUESTED,
I2C_SLAVE_READ_PROCESSED,
I2C_SLAVE_WRITE_RECEIVED,
I2C_SLAVE_STOP,
};
int i2c_slave_register(struct i2c_client *client, i2c_slave_cb_t slave_cb);
int i2c_slave_unregister(struct i2c_client *client);
int i2c_slave_event(struct i2c_client *client,
enum i2c_slave_event event, u8 *val);
#if IS_ENABLED(CONFIG_I2C_SLAVE)
bool i2c_detect_slave_mode(struct device *dev);
#else
static inline bool i2c_detect_slave_mode(struct device *dev) { return false; }
#endif
/**
* struct i2c_board_info - template for device creation
* @type: chip type, to initialize i2c_client.name
* @flags: to initialize i2c_client.flags
* @addr: stored in i2c_client.addr
* @dev_name: Overrides the default <busnr>-<addr> dev_name if set
* @platform_data: stored in i2c_client.dev.platform_data
* @of_node: pointer to OpenFirmware device node
* @fwnode: device node supplied by the platform firmware
* @swnode: software node for the device
* @resources: resources associated with the device
* @num_resources: number of resources in the @resources array
* @irq: stored in i2c_client.irq
*
* I2C doesn't actually support hardware probing, although controllers and
* devices may be able to use I2C_SMBUS_QUICK to tell whether or not there's
* a device at a given address. Drivers commonly need more information than
* that, such as chip type, configuration, associated IRQ, and so on.
*
* i2c_board_info is used to build tables of information listing I2C devices
* that are present. This information is used to grow the driver model tree.
* For mainboards this is done statically using i2c_register_board_info();
* bus numbers identify adapters that aren't yet available. For add-on boards,
* i2c_new_client_device() does this dynamically with the adapter already known.
*/
struct i2c_board_info {
char type[I2C_NAME_SIZE];
unsigned short flags;
unsigned short addr;
const char *dev_name;
void *platform_data;
struct device_node *of_node;
struct fwnode_handle *fwnode;
const struct software_node *swnode;
const struct resource *resources;
unsigned int num_resources;
int irq;
};
/**
* I2C_BOARD_INFO - macro used to list an i2c device and its address
* @dev_type: identifies the device type
* @dev_addr: the device's address on the bus.
*
* This macro initializes essential fields of a struct i2c_board_info,
* declaring what has been provided on a particular board. Optional
* fields (such as associated irq, or device-specific platform_data)
* are provided using conventional syntax.
*/
#define I2C_BOARD_INFO(dev_type, dev_addr) \
.type = dev_type, .addr = (dev_addr)
#if IS_ENABLED(CONFIG_I2C)
/*
* Add-on boards should register/unregister their devices; e.g. a board
* with integrated I2C, a config eeprom, sensors, and a codec that's
* used in conjunction with the primary hardware.
*/
struct i2c_client *
i2c_new_client_device(struct i2c_adapter *adap, struct i2c_board_info const *info);
/* If you don't know the exact address of an I2C device, use this variant
* instead, which can probe for device presence in a list of possible
* addresses. The "probe" callback function is optional. If it is provided,
* it must return 1 on successful probe, 0 otherwise. If it is not provided,
* a default probing method is used.
*/
struct i2c_client *
i2c_new_scanned_device(struct i2c_adapter *adap,
struct i2c_board_info *info,
unsigned short const *addr_list,
int (*probe)(struct i2c_adapter *adap, unsigned short addr));
/* Common custom probe functions */
int i2c_probe_func_quick_read(struct i2c_adapter *adap, unsigned short addr);
struct i2c_client *
i2c_new_dummy_device(struct i2c_adapter *adapter, u16 address);
struct i2c_client *
devm_i2c_new_dummy_device(struct device *dev, struct i2c_adapter *adap, u16 address);
struct i2c_client *
i2c_new_ancillary_device(struct i2c_client *client,
const char *name,
u16 default_addr);
void i2c_unregister_device(struct i2c_client *client);
struct i2c_client *i2c_verify_client(struct device *dev);
#else
static inline struct i2c_client *i2c_verify_client(struct device *dev)
{
return NULL;
}
#endif /* I2C */
/* Mainboard arch_initcall() code should register all its I2C devices.
* This is done at arch_initcall time, before declaring any i2c adapters.
* Modules for add-on boards must use other calls.
*/
#ifdef CONFIG_I2C_BOARDINFO
int
i2c_register_board_info(int busnum, struct i2c_board_info const *info,
unsigned n);
#else
static inline int
i2c_register_board_info(int busnum, struct i2c_board_info const *info,
unsigned n)
{
return 0;
}
#endif /* I2C_BOARDINFO */
/**
* struct i2c_algorithm - represent I2C transfer method
* @master_xfer: Issue a set of i2c transactions to the given I2C adapter
* defined by the msgs array, with num messages available to transfer via
* the adapter specified by adap.
* @master_xfer_atomic: same as @master_xfer. Yet, only using atomic context
* so e.g. PMICs can be accessed very late before shutdown. Optional.
* @smbus_xfer: Issue smbus transactions to the given I2C adapter. If this
* is not present, then the bus layer will try and convert the SMBus calls
* into I2C transfers instead.
* @smbus_xfer_atomic: same as @smbus_xfer. Yet, only using atomic context
* so e.g. PMICs can be accessed very late before shutdown. Optional.
* @functionality: Return the flags that this algorithm/adapter pair supports
* from the ``I2C_FUNC_*`` flags.
* @reg_slave: Register given client to I2C slave mode of this adapter
* @unreg_slave: Unregister given client from I2C slave mode of this adapter
*
* The following structs are for those who like to implement new bus drivers:
* i2c_algorithm is the interface to a class of hardware solutions which can
* be addressed using the same bus algorithms - i.e. bit-banging or the PCF8584
* to name two of the most common.
*
* The return codes from the ``master_xfer{_atomic}`` fields should indicate the
* type of error code that occurred during the transfer, as documented in the
* Kernel Documentation file Documentation/i2c/fault-codes.rst. Otherwise, the
* number of messages executed should be returned.
*/
struct i2c_algorithm {
/*
* If an adapter algorithm can't do I2C-level access, set master_xfer
* to NULL. If an adapter algorithm can do SMBus access, set
* smbus_xfer. If set to NULL, the SMBus protocol is simulated
* using common I2C messages.
*
* master_xfer should return the number of messages successfully
* processed, or a negative value on error
*/
int (*master_xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
int num);
int (*master_xfer_atomic)(struct i2c_adapter *adap,
struct i2c_msg *msgs, int num);
int (*smbus_xfer)(struct i2c_adapter *adap, u16 addr,
unsigned short flags, char read_write,
u8 command, int size, union i2c_smbus_data *data);
int (*smbus_xfer_atomic)(struct i2c_adapter *adap, u16 addr,
unsigned short flags, char read_write,
u8 command, int size, union i2c_smbus_data *data);
/* To determine what the adapter supports */
u32 (*functionality)(struct i2c_adapter *adap);
#if IS_ENABLED(CONFIG_I2C_SLAVE)
int (*reg_slave)(struct i2c_client *client);
int (*unreg_slave)(struct i2c_client *client);
#endif
};
/**
* struct i2c_lock_operations - represent I2C locking operations
* @lock_bus: Get exclusive access to an I2C bus segment
* @trylock_bus: Try to get exclusive access to an I2C bus segment
* @unlock_bus: Release exclusive access to an I2C bus segment
*
* The main operations are wrapped by i2c_lock_bus and i2c_unlock_bus.
*/
struct i2c_lock_operations {
void (*lock_bus)(struct i2c_adapter *adapter, unsigned int flags);
int (*trylock_bus)(struct i2c_adapter *adapter, unsigned int flags);
void (*unlock_bus)(struct i2c_adapter *adapter, unsigned int flags);
};
/**
* struct i2c_timings - I2C timing information
* @bus_freq_hz: the bus frequency in Hz
* @scl_rise_ns: time SCL signal takes to rise in ns; t(r) in the I2C specification
* @scl_fall_ns: time SCL signal takes to fall in ns; t(f) in the I2C specification
* @scl_int_delay_ns: time IP core additionally needs to setup SCL in ns
* @sda_fall_ns: time SDA signal takes to fall in ns; t(f) in the I2C specification
* @sda_hold_ns: time IP core additionally needs to hold SDA in ns
* @digital_filter_width_ns: width in ns of spikes on i2c lines that the IP core
* digital filter can filter out
* @analog_filter_cutoff_freq_hz: threshold frequency for the low pass IP core
* analog filter
*/
struct i2c_timings {
u32 bus_freq_hz;
u32 scl_rise_ns;
u32 scl_fall_ns;
u32 scl_int_delay_ns;
u32 sda_fall_ns;
u32 sda_hold_ns;
u32 digital_filter_width_ns;
u32 analog_filter_cutoff_freq_hz;
};
/**
* struct i2c_bus_recovery_info - I2C bus recovery information
* @recover_bus: Recover routine. Either pass driver's recover_bus() routine, or
* i2c_generic_scl_recovery().
* @get_scl: This gets current value of SCL line. Mandatory for generic SCL
* recovery. Populated internally for generic GPIO recovery.
* @set_scl: This sets/clears the SCL line. Mandatory for generic SCL recovery.
* Populated internally for generic GPIO recovery.
* @get_sda: This gets current value of SDA line. This or set_sda() is mandatory
* for generic SCL recovery. Populated internally, if sda_gpio is a valid
* GPIO, for generic GPIO recovery.
* @set_sda: This sets/clears the SDA line. This or get_sda() is mandatory for
* generic SCL recovery. Populated internally, if sda_gpio is a valid GPIO,
* for generic GPIO recovery.
* @get_bus_free: Returns the bus free state as seen from the IP core in case it
* has a more complex internal logic than just reading SDA. Optional.
* @prepare_recovery: This will be called before starting recovery. Platform may
* configure padmux here for SDA/SCL line or something else they want.
* @unprepare_recovery: This will be called after completing recovery. Platform
* may configure padmux here for SDA/SCL line or something else they want.
* @scl_gpiod: gpiod of the SCL line. Only required for GPIO recovery.
* @sda_gpiod: gpiod of the SDA line. Only required for GPIO recovery.
* @pinctrl: pinctrl used by GPIO recovery to change the state of the I2C pins.
* Optional.
* @pins_default: default pinctrl state of SCL/SDA lines, when they are assigned
* to the I2C bus. Optional. Populated internally for GPIO recovery, if
* state with the name PINCTRL_STATE_DEFAULT is found and pinctrl is valid.
* @pins_gpio: recovery pinctrl state of SCL/SDA lines, when they are used as
* GPIOs. Optional. Populated internally for GPIO recovery, if this state
* is called "gpio" or "recovery" and pinctrl is valid.
*/
struct i2c_bus_recovery_info {
int (*recover_bus)(struct i2c_adapter *adap);
int (*get_scl)(struct i2c_adapter *adap);
void (*set_scl)(struct i2c_adapter *adap, int val);
int (*get_sda)(struct i2c_adapter *adap);
void (*set_sda)(struct i2c_adapter *adap, int val);
int (*get_bus_free)(struct i2c_adapter *adap);
void (*prepare_recovery)(struct i2c_adapter *adap);
void (*unprepare_recovery)(struct i2c_adapter *adap);
/* gpio recovery */
struct gpio_desc *scl_gpiod;
struct gpio_desc *sda_gpiod;
struct pinctrl *pinctrl;
struct pinctrl_state *pins_default;
struct pinctrl_state *pins_gpio;
};
int i2c_recover_bus(struct i2c_adapter *adap);
/* Generic recovery routines */
int i2c_generic_scl_recovery(struct i2c_adapter *adap);
/**
* struct i2c_adapter_quirks - describe flaws of an i2c adapter
* @flags: see I2C_AQ_* for possible flags and read below
* @max_num_msgs: maximum number of messages per transfer
* @max_write_len: maximum length of a write message
* @max_read_len: maximum length of a read message
* @max_comb_1st_msg_len: maximum length of the first msg in a combined message
* @max_comb_2nd_msg_len: maximum length of the second msg in a combined message
*
* Note about combined messages: Some I2C controllers can only send one message
* per transfer, plus something called combined message or write-then-read.
* This is (usually) a small write message followed by a read message and
* barely enough to access register based devices like EEPROMs. There is a flag
* to support this mode. It implies max_num_msg = 2 and does the length checks
* with max_comb_*_len because combined message mode usually has its own
* limitations. Because of HW implementations, some controllers can actually do
* write-then-anything or other variants. To support that, write-then-read has
* been broken out into smaller bits like write-first and read-second which can
* be combined as needed.
*/
struct i2c_adapter_quirks {
u64 flags;
int max_num_msgs;
u16 max_write_len;
u16 max_read_len;
u16 max_comb_1st_msg_len;
u16 max_comb_2nd_msg_len;
};
/* enforce max_num_msgs = 2 and use max_comb_*_len for length checks */
#define I2C_AQ_COMB BIT(0)
/* first combined message must be write */
#define I2C_AQ_COMB_WRITE_FIRST BIT(1)
/* second combined message must be read */
#define I2C_AQ_COMB_READ_SECOND BIT(2)
/* both combined messages must have the same target address */
#define I2C_AQ_COMB_SAME_ADDR BIT(3)
/* convenience macro for typical write-then read case */
#define I2C_AQ_COMB_WRITE_THEN_READ (I2C_AQ_COMB | I2C_AQ_COMB_WRITE_FIRST | \
I2C_AQ_COMB_READ_SECOND | I2C_AQ_COMB_SAME_ADDR)
/* clock stretching is not supported */
#define I2C_AQ_NO_CLK_STRETCH BIT(4)
/* message cannot have length of 0 */
#define I2C_AQ_NO_ZERO_LEN_READ BIT(5)
#define I2C_AQ_NO_ZERO_LEN_WRITE BIT(6)
#define I2C_AQ_NO_ZERO_LEN (I2C_AQ_NO_ZERO_LEN_READ | I2C_AQ_NO_ZERO_LEN_WRITE)
/* adapter cannot do repeated START */
#define I2C_AQ_NO_REP_START BIT(7)
/*
* i2c_adapter is the structure used to identify a physical i2c bus along
* with the access algorithms necessary to access it.
*/
struct i2c_adapter {
struct module *owner;
unsigned int class; /* classes to allow probing for */
const struct i2c_algorithm *algo; /* the algorithm to access the bus */
void *algo_data;
/* data fields that are valid for all devices */
const struct i2c_lock_operations *lock_ops;
struct rt_mutex bus_lock;
struct rt_mutex mux_lock;
int timeout; /* in jiffies */
int retries;
struct device dev; /* the adapter device */
unsigned long locked_flags; /* owned by the I2C core */
#define I2C_ALF_IS_SUSPENDED 0
#define I2C_ALF_SUSPEND_REPORTED 1
int nr;
char name[48];
struct completion dev_released;
struct mutex userspace_clients_lock;
struct list_head userspace_clients;
struct i2c_bus_recovery_info *bus_recovery_info;
const struct i2c_adapter_quirks *quirks;
struct irq_domain *host_notify_domain;
struct regulator *bus_regulator;
struct dentry *debugfs;
};
#define to_i2c_adapter(d) container_of(d, struct i2c_adapter, dev)
static inline void *i2c_get_adapdata(const struct i2c_adapter *adap)
{
return dev_get_drvdata(&adap->dev);
}
static inline void i2c_set_adapdata(struct i2c_adapter *adap, void *data)
{
dev_set_drvdata(&adap->dev, data);
}
static inline struct i2c_adapter *
i2c_parent_is_i2c_adapter(const struct i2c_adapter *adapter)
{
#if IS_ENABLED(CONFIG_I2C_MUX)
struct device *parent = adapter->dev.parent;
if (parent != NULL && parent->type == &i2c_adapter_type)
return to_i2c_adapter(parent);
else
#endif
return NULL;
}
int i2c_for_each_dev(void *data, int (*fn)(struct device *dev, void *data));
/* Adapter locking functions, exported for shared pin cases */
#define I2C_LOCK_ROOT_ADAPTER BIT(0)
#define I2C_LOCK_SEGMENT BIT(1)
/**
* i2c_lock_bus - Get exclusive access to an I2C bus segment
* @adapter: Target I2C bus segment
* @flags: I2C_LOCK_ROOT_ADAPTER locks the root i2c adapter, I2C_LOCK_SEGMENT
* locks only this branch in the adapter tree
*/
static inline void
i2c_lock_bus(struct i2c_adapter *adapter, unsigned int flags)
{
adapter->lock_ops->lock_bus(adapter, flags);
}
/**
* i2c_trylock_bus - Try to get exclusive access to an I2C bus segment
* @adapter: Target I2C bus segment
* @flags: I2C_LOCK_ROOT_ADAPTER tries to locks the root i2c adapter,
* I2C_LOCK_SEGMENT tries to lock only this branch in the adapter tree
*
* Return: true if the I2C bus segment is locked, false otherwise
*/
static inline int
i2c_trylock_bus(struct i2c_adapter *adapter, unsigned int flags)
{
return adapter->lock_ops->trylock_bus(adapter, flags);
}
/**
* i2c_unlock_bus - Release exclusive access to an I2C bus segment
* @adapter: Target I2C bus segment
* @flags: I2C_LOCK_ROOT_ADAPTER unlocks the root i2c adapter, I2C_LOCK_SEGMENT
* unlocks only this branch in the adapter tree
*/
static inline void
i2c_unlock_bus(struct i2c_adapter *adapter, unsigned int flags)
{
adapter->lock_ops->unlock_bus(adapter, flags);
}
/**
* i2c_mark_adapter_suspended - Report suspended state of the adapter to the core
* @adap: Adapter to mark as suspended
*
* When using this helper to mark an adapter as suspended, the core will reject
* further transfers to this adapter. The usage of this helper is optional but
* recommended for devices having distinct handlers for system suspend and
* runtime suspend. More complex devices are free to implement custom solutions
* to reject transfers when suspended.
*/
static inline void i2c_mark_adapter_suspended(struct i2c_adapter *adap)
{
i2c_lock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
set_bit(I2C_ALF_IS_SUSPENDED, &adap->locked_flags);
i2c_unlock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
}
/**
* i2c_mark_adapter_resumed - Report resumed state of the adapter to the core
* @adap: Adapter to mark as resumed
*
* When using this helper to mark an adapter as resumed, the core will allow
* further transfers to this adapter. See also further notes to
* @i2c_mark_adapter_suspended().
*/
static inline void i2c_mark_adapter_resumed(struct i2c_adapter *adap)
{
i2c_lock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
clear_bit(I2C_ALF_IS_SUSPENDED, &adap->locked_flags);
i2c_unlock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
}
/* i2c adapter classes (bitmask) */
#define I2C_CLASS_HWMON (1<<0) /* lm_sensors, ... */
#define I2C_CLASS_SPD (1<<7) /* Memory modules */
/* Warn users that the adapter doesn't support classes anymore */
#define I2C_CLASS_DEPRECATED (1<<8)
/* Internal numbers to terminate lists */
#define I2C_CLIENT_END 0xfffeU
/* Construct an I2C_CLIENT_END-terminated array of i2c addresses */
#define I2C_ADDRS(addr, addrs...) \
((const unsigned short []){ addr, ## addrs, I2C_CLIENT_END })
/* ----- functions exported by i2c.o */
/* administration...
*/
#if IS_ENABLED(CONFIG_I2C)
int i2c_add_adapter(struct i2c_adapter *adap);
int devm_i2c_add_adapter(struct device *dev, struct i2c_adapter *adapter);
void i2c_del_adapter(struct i2c_adapter *adap);
int i2c_add_numbered_adapter(struct i2c_adapter *adap);
int i2c_register_driver(struct module *owner, struct i2c_driver *driver);
void i2c_del_driver(struct i2c_driver *driver);
/* use a define to avoid include chaining to get THIS_MODULE */
#define i2c_add_driver(driver) \
i2c_register_driver(THIS_MODULE, driver)
static inline bool i2c_client_has_driver(struct i2c_client *client)
{
return !IS_ERR_OR_NULL(client) && client->dev.driver;
}
/* call the i2c_client->command() of all attached clients with
* the given arguments */
void i2c_clients_command(struct i2c_adapter *adap,
unsigned int cmd, void *arg);
struct i2c_adapter *i2c_get_adapter(int nr);
void i2c_put_adapter(struct i2c_adapter *adap);
unsigned int i2c_adapter_depth(struct i2c_adapter *adapter);
void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults);
/* Return the functionality mask */
static inline u32 i2c_get_functionality(struct i2c_adapter *adap)
{
return adap->algo->functionality(adap);
}
/* Return 1 if adapter supports everything we need, 0 if not. */
static inline int i2c_check_functionality(struct i2c_adapter *adap, u32 func)
{
return (func & i2c_get_functionality(adap)) == func;
}
/**
* i2c_check_quirks() - Function for checking the quirk flags in an i2c adapter
* @adap: i2c adapter
* @quirks: quirk flags
*
* Return: true if the adapter has all the specified quirk flags, false if not
*/
static inline bool i2c_check_quirks(struct i2c_adapter *adap, u64 quirks)
{
if (!adap->quirks)
return false;
return (adap->quirks->flags & quirks) == quirks;
}
/* Return the adapter number for a specific adapter */
static inline int i2c_adapter_id(struct i2c_adapter *adap)
{
return adap->nr;
}
static inline u8 i2c_8bit_addr_from_msg(const struct i2c_msg *msg)
{
return (msg->addr << 1) | (msg->flags & I2C_M_RD);
}
u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold);
void i2c_put_dma_safe_msg_buf(u8 *buf, struct i2c_msg *msg, bool xferred);
int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr);
/**
* module_i2c_driver() - Helper macro for registering a modular I2C driver
* @__i2c_driver: i2c_driver struct
*
* Helper macro for I2C drivers which do not do anything special in module
* init/exit. This eliminates a lot of boilerplate. Each module may only
* use this macro once, and calling it replaces module_init() and module_exit()
*/
#define module_i2c_driver(__i2c_driver) \
module_driver(__i2c_driver, i2c_add_driver, \
i2c_del_driver)
/**
* builtin_i2c_driver() - Helper macro for registering a builtin I2C driver
* @__i2c_driver: i2c_driver struct
*
* Helper macro for I2C drivers which do not do anything special in their
* init. This eliminates a lot of boilerplate. Each driver may only
* use this macro once, and calling it replaces device_initcall().
*/
#define builtin_i2c_driver(__i2c_driver) \
builtin_driver(__i2c_driver, i2c_add_driver)
#endif /* I2C */
/* must call put_device() when done with returned i2c_client device */
struct i2c_client *i2c_find_device_by_fwnode(struct fwnode_handle *fwnode);
/* must call put_device() when done with returned i2c_adapter device */
struct i2c_adapter *i2c_find_adapter_by_fwnode(struct fwnode_handle *fwnode);
/* must call i2c_put_adapter() when done with returned i2c_adapter device */
struct i2c_adapter *i2c_get_adapter_by_fwnode(struct fwnode_handle *fwnode);
#if IS_ENABLED(CONFIG_OF)
/* must call put_device() when done with returned i2c_client device */
static inline struct i2c_client *of_find_i2c_device_by_node(struct device_node *node)
{
return i2c_find_device_by_fwnode(of_fwnode_handle(node));
}
/* must call put_device() when done with returned i2c_adapter device */
static inline struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node)
{
return i2c_find_adapter_by_fwnode(of_fwnode_handle(node));
}
/* must call i2c_put_adapter() when done with returned i2c_adapter device */
static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node)
{
return i2c_get_adapter_by_fwnode(of_fwnode_handle(node));
}
const struct of_device_id
*i2c_of_match_device(const struct of_device_id *matches,
struct i2c_client *client);
int of_i2c_get_board_info(struct device *dev, struct device_node *node,
struct i2c_board_info *info);
#else
static inline struct i2c_client *of_find_i2c_device_by_node(struct device_node *node)
{
return NULL;
}
static inline struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node)
{
return NULL;
}
static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node)
{
return NULL;
}
static inline const struct of_device_id
*i2c_of_match_device(const struct of_device_id *matches,
struct i2c_client *client)
{
return NULL;
}
static inline int of_i2c_get_board_info(struct device *dev,
struct device_node *node,
struct i2c_board_info *info)
{
return -ENOTSUPP;
}
#endif /* CONFIG_OF */
struct acpi_resource;
struct acpi_resource_i2c_serialbus;
#if IS_ENABLED(CONFIG_ACPI)
bool i2c_acpi_get_i2c_resource(struct acpi_resource *ares,
struct acpi_resource_i2c_serialbus **i2c);
int i2c_acpi_client_count(struct acpi_device *adev);
u32 i2c_acpi_find_bus_speed(struct device *dev);
struct i2c_client *i2c_acpi_new_device_by_fwnode(struct fwnode_handle *fwnode,
int index,
struct i2c_board_info *info);
struct i2c_adapter *i2c_acpi_find_adapter_by_handle(acpi_handle handle);
bool i2c_acpi_waive_d0_probe(struct device *dev);
#else
static inline bool i2c_acpi_get_i2c_resource(struct acpi_resource *ares,
struct acpi_resource_i2c_serialbus **i2c)
{
return false;
}
static inline int i2c_acpi_client_count(struct acpi_device *adev)
{
return 0;
}
static inline u32 i2c_acpi_find_bus_speed(struct device *dev)
{
return 0;
}
static inline struct i2c_client *i2c_acpi_new_device_by_fwnode(
struct fwnode_handle *fwnode, int index,
struct i2c_board_info *info)
{
return ERR_PTR(-ENODEV);
}
static inline struct i2c_adapter *i2c_acpi_find_adapter_by_handle(acpi_handle handle)
{
return NULL;
}
static inline bool i2c_acpi_waive_d0_probe(struct device *dev)
{
return false;
}
#endif /* CONFIG_ACPI */
static inline struct i2c_client *i2c_acpi_new_device(struct device *dev,
int index,
struct i2c_board_info *info)
{
return i2c_acpi_new_device_by_fwnode(dev_fwnode(dev), index, info);
}
#endif /* _LINUX_I2C_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1995 Linus Torvalds
* Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
* Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
*/
#include <linux/sched.h> /* test_thread_flag(), ... */
#include <linux/sched/task_stack.h> /* task_stack_*(), ... */
#include <linux/kdebug.h> /* oops_begin/end, ... */
#include <linux/extable.h> /* search_exception_tables */
#include <linux/memblock.h> /* max_low_pfn */
#include <linux/kfence.h> /* kfence_handle_page_fault */
#include <linux/kprobes.h> /* NOKPROBE_SYMBOL, ... */
#include <linux/mmiotrace.h> /* kmmio_handler, ... */
#include <linux/perf_event.h> /* perf_sw_event */
#include <linux/hugetlb.h> /* hstate_index_to_shift */
#include <linux/prefetch.h> /* prefetchw */
#include <linux/context_tracking.h> /* exception_enter(), ... */
#include <linux/uaccess.h> /* faulthandler_disabled() */
#include <linux/efi.h> /* efi_crash_gracefully_on_page_fault()*/
#include <linux/mm_types.h>
#include <linux/mm.h> /* find_and_lock_vma() */
#include <asm/cpufeature.h> /* boot_cpu_has, ... */
#include <asm/traps.h> /* dotraplinkage, ... */
#include <asm/fixmap.h> /* VSYSCALL_ADDR */
#include <asm/vsyscall.h> /* emulate_vsyscall */
#include <asm/vm86.h> /* struct vm86 */
#include <asm/mmu_context.h> /* vma_pkey() */
#include <asm/efi.h> /* efi_crash_gracefully_on_page_fault()*/
#include <asm/desc.h> /* store_idt(), ... */
#include <asm/cpu_entry_area.h> /* exception stack */
#include <asm/pgtable_areas.h> /* VMALLOC_START, ... */
#include <asm/kvm_para.h> /* kvm_handle_async_pf */
#include <asm/vdso.h> /* fixup_vdso_exception() */
#include <asm/irq_stack.h>
#include <asm/fred.h>
#include <asm/sev.h> /* snp_dump_hva_rmpentry() */
#define CREATE_TRACE_POINTS
#include <asm/trace/exceptions.h>
/*
* Returns 0 if mmiotrace is disabled, or if the fault is not
* handled by mmiotrace:
*/
static nokprobe_inline int
kmmio_fault(struct pt_regs *regs, unsigned long addr)
{
if (unlikely(is_kmmio_active()))
if (kmmio_handler(regs, addr) == 1)
return -1;
return 0;
}
/*
* Prefetch quirks:
*
* 32-bit mode:
*
* Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
* Check that here and ignore it. This is AMD erratum #91.
*
* 64-bit mode:
*
* Sometimes the CPU reports invalid exceptions on prefetch.
* Check that here and ignore it.
*
* Opcode checker based on code by Richard Brunner.
*/
static inline int
check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
unsigned char opcode, int *prefetch)
{
unsigned char instr_hi = opcode & 0xf0;
unsigned char instr_lo = opcode & 0x0f;
switch (instr_hi) {
case 0x20:
case 0x30:
/*
* Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
* In X86_64 long mode, the CPU will signal invalid
* opcode if some of these prefixes are present so
* X86_64 will never get here anyway
*/
return ((instr_lo & 7) == 0x6);
#ifdef CONFIG_X86_64
case 0x40:
/*
* In 64-bit mode 0x40..0x4F are valid REX prefixes
*/
return (!user_mode(regs) || user_64bit_mode(regs));
#endif
case 0x60:
/* 0x64 thru 0x67 are valid prefixes in all modes. */
return (instr_lo & 0xC) == 0x4;
case 0xF0:
/* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
return !instr_lo || (instr_lo>>1) == 1;
case 0x00:
/* Prefetch instruction is 0x0F0D or 0x0F18 */
if (get_kernel_nofault(opcode, instr))
return 0;
*prefetch = (instr_lo == 0xF) && (opcode == 0x0D || opcode == 0x18);
return 0;
default:
return 0;
}
}
static bool is_amd_k8_pre_npt(void)
{
struct cpuinfo_x86 *c = &boot_cpu_data;
return unlikely(IS_ENABLED(CONFIG_CPU_SUP_AMD) &&
c->x86_vendor == X86_VENDOR_AMD &&
c->x86 == 0xf && c->x86_model < 0x40);
}
static int
is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
{
unsigned char *max_instr;
unsigned char *instr;
int prefetch = 0;
/* Erratum #91 affects AMD K8, pre-NPT CPUs */
if (!is_amd_k8_pre_npt()) return 0;
/*
* If it was a exec (instruction fetch) fault on NX page, then
* do not ignore the fault:
*/
if (error_code & X86_PF_INSTR)
return 0;
instr = (void *)convert_ip_to_linear(current, regs);
max_instr = instr + 15;
/*
* This code has historically always bailed out if IP points to a
* not-present page (e.g. due to a race). No one has ever
* complained about this.
*/
pagefault_disable();
while (instr < max_instr) {
unsigned char opcode;
if (user_mode(regs)) {
if (get_user(opcode, (unsigned char __user *) instr))
break;
} else {
if (get_kernel_nofault(opcode, instr))
break;
}
instr++; if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
break;
}
pagefault_enable();
return prefetch;
}
DEFINE_SPINLOCK(pgd_lock);
LIST_HEAD(pgd_list);
#ifdef CONFIG_X86_32
static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
{
unsigned index = pgd_index(address);
pgd_t *pgd_k;
p4d_t *p4d, *p4d_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pgd += index;
pgd_k = init_mm.pgd + index;
if (!pgd_present(*pgd_k))
return NULL;
/*
* set_pgd(pgd, *pgd_k); here would be useless on PAE
* and redundant with the set_pmd() on non-PAE. As would
* set_p4d/set_pud.
*/
p4d = p4d_offset(pgd, address);
p4d_k = p4d_offset(pgd_k, address);
if (!p4d_present(*p4d_k))
return NULL;
pud = pud_offset(p4d, address);
pud_k = pud_offset(p4d_k, address);
if (!pud_present(*pud_k))
return NULL;
pmd = pmd_offset(pud, address);
pmd_k = pmd_offset(pud_k, address);
if (pmd_present(*pmd) != pmd_present(*pmd_k))
set_pmd(pmd, *pmd_k);
if (!pmd_present(*pmd_k))
return NULL;
else
BUG_ON(pmd_pfn(*pmd) != pmd_pfn(*pmd_k));
return pmd_k;
}
/*
* Handle a fault on the vmalloc or module mapping area
*
* This is needed because there is a race condition between the time
* when the vmalloc mapping code updates the PMD to the point in time
* where it synchronizes this update with the other page-tables in the
* system.
*
* In this race window another thread/CPU can map an area on the same
* PMD, finds it already present and does not synchronize it with the
* rest of the system yet. As a result v[mz]alloc might return areas
* which are not mapped in every page-table in the system, causing an
* unhandled page-fault when they are accessed.
*/
static noinline int vmalloc_fault(unsigned long address)
{
unsigned long pgd_paddr;
pmd_t *pmd_k;
pte_t *pte_k;
/* Make sure we are in vmalloc area: */
if (!(address >= VMALLOC_START && address < VMALLOC_END))
return -1;
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*
* Do _not_ use "current" here. We might be inside
* an interrupt in the middle of a task switch..
*/
pgd_paddr = read_cr3_pa();
pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
if (!pmd_k)
return -1;
if (pmd_leaf(*pmd_k))
return 0;
pte_k = pte_offset_kernel(pmd_k, address);
if (!pte_present(*pte_k))
return -1;
return 0;
}
NOKPROBE_SYMBOL(vmalloc_fault);
void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
{
unsigned long addr;
for (addr = start & PMD_MASK;
addr >= TASK_SIZE_MAX && addr < VMALLOC_END;
addr += PMD_SIZE) {
struct page *page;
spin_lock(&pgd_lock);
list_for_each_entry(page, &pgd_list, lru) {
spinlock_t *pgt_lock;
/* the pgt_lock only for Xen */
pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
spin_lock(pgt_lock);
vmalloc_sync_one(page_address(page), addr);
spin_unlock(pgt_lock);
}
spin_unlock(&pgd_lock);
}
}
static bool low_pfn(unsigned long pfn)
{
return pfn < max_low_pfn;
}
static void dump_pagetable(unsigned long address)
{
pgd_t *base = __va(read_cr3_pa());
pgd_t *pgd = &base[pgd_index(address)];
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
#ifdef CONFIG_X86_PAE
pr_info("*pdpt = %016Lx ", pgd_val(*pgd));
if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
goto out;
#define pr_pde pr_cont
#else
#define pr_pde pr_info
#endif
p4d = p4d_offset(pgd, address);
pud = pud_offset(p4d, address);
pmd = pmd_offset(pud, address);
pr_pde("*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
#undef pr_pde
/*
* We must not directly access the pte in the highpte
* case if the page table is located in highmem.
* And let's rather not kmap-atomic the pte, just in case
* it's allocated already:
*/
if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_leaf(*pmd))
goto out;
pte = pte_offset_kernel(pmd, address);
pr_cont("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
out:
pr_cont("\n");
}
#else /* CONFIG_X86_64: */
#ifdef CONFIG_CPU_SUP_AMD
static const char errata93_warning[] =
KERN_ERR
"******* Your BIOS seems to not contain a fix for K8 errata #93\n"
"******* Working around it, but it may cause SEGVs or burn power.\n"
"******* Please consider a BIOS update.\n"
"******* Disabling USB legacy in the BIOS may also help.\n";
#endif
static int bad_address(void *p)
{
unsigned long dummy;
return get_kernel_nofault(dummy, (unsigned long *)p);
}
static void dump_pagetable(unsigned long address)
{
pgd_t *base = __va(read_cr3_pa());
pgd_t *pgd = base + pgd_index(address);
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
if (bad_address(pgd))
goto bad;
pr_info("PGD %lx ", pgd_val(*pgd));
if (!pgd_present(*pgd))
goto out;
p4d = p4d_offset(pgd, address);
if (bad_address(p4d))
goto bad;
pr_cont("P4D %lx ", p4d_val(*p4d));
if (!p4d_present(*p4d) || p4d_leaf(*p4d))
goto out;
pud = pud_offset(p4d, address);
if (bad_address(pud))
goto bad;
pr_cont("PUD %lx ", pud_val(*pud));
if (!pud_present(*pud) || pud_leaf(*pud))
goto out;
pmd = pmd_offset(pud, address);
if (bad_address(pmd))
goto bad;
pr_cont("PMD %lx ", pmd_val(*pmd));
if (!pmd_present(*pmd) || pmd_leaf(*pmd))
goto out;
pte = pte_offset_kernel(pmd, address);
if (bad_address(pte))
goto bad;
pr_cont("PTE %lx", pte_val(*pte));
out:
pr_cont("\n");
return;
bad:
pr_info("BAD\n");
}
#endif /* CONFIG_X86_64 */
/*
* Workaround for K8 erratum #93 & buggy BIOS.
*
* BIOS SMM functions are required to use a specific workaround
* to avoid corruption of the 64bit RIP register on C stepping K8.
*
* A lot of BIOS that didn't get tested properly miss this.
*
* The OS sees this as a page fault with the upper 32bits of RIP cleared.
* Try to work around it here.
*
* Note we only handle faults in kernel here.
* Does nothing on 32-bit.
*/
static int is_errata93(struct pt_regs *regs, unsigned long address)
{
#if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
|| boot_cpu_data.x86 != 0xf)
return 0;
if (user_mode(regs))
return 0;
if (address != regs->ip)
return 0;
if ((address >> 32) != 0)
return 0;
address |= 0xffffffffUL << 32; if ((address >= (u64)_stext && address <= (u64)_etext) || (address >= MODULES_VADDR && address <= MODULES_END)) { printk_once(errata93_warning); regs->ip = address;
return 1;
}
#endif
return 0;
}
/*
* Work around K8 erratum #100 K8 in compat mode occasionally jumps
* to illegal addresses >4GB.
*
* We catch this in the page fault handler because these addresses
* are not reachable. Just detect this case and return. Any code
* segment in LDT is compatibility mode.
*/
static int is_errata100(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_64
if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
return 1;
#endif
return 0;
}
/* Pentium F0 0F C7 C8 bug workaround: */
static int is_f00f_bug(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{
#ifdef CONFIG_X86_F00F_BUG
if (boot_cpu_has_bug(X86_BUG_F00F) && !(error_code & X86_PF_USER) &&
idt_is_f00f_address(address)) {
handle_invalid_op(regs);
return 1;
}
#endif
return 0;
}
static void show_ldttss(const struct desc_ptr *gdt, const char *name, u16 index)
{
u32 offset = (index >> 3) * sizeof(struct desc_struct);
unsigned long addr;
struct ldttss_desc desc;
if (index == 0) {
pr_alert("%s: NULL\n", name);
return;
}
if (offset + sizeof(struct ldttss_desc) >= gdt->size) {
pr_alert("%s: 0x%hx -- out of bounds\n", name, index);
return;
}
if (copy_from_kernel_nofault(&desc, (void *)(gdt->address + offset),
sizeof(struct ldttss_desc))) {
pr_alert("%s: 0x%hx -- GDT entry is not readable\n",
name, index);
return;
}
addr = desc.base0 | (desc.base1 << 16) | ((unsigned long)desc.base2 << 24);
#ifdef CONFIG_X86_64
addr |= ((u64)desc.base3 << 32);
#endif
pr_alert("%s: 0x%hx -- base=0x%lx limit=0x%x\n",
name, index, addr, (desc.limit0 | (desc.limit1 << 16)));
}
static void
show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long address)
{
if (!oops_may_print())
return;
if (error_code & X86_PF_INSTR) {
unsigned int level;
pgd_t *pgd;
pte_t *pte;
pgd = __va(read_cr3_pa());
pgd += pgd_index(address);
pte = lookup_address_in_pgd(pgd, address, &level);
if (pte && pte_present(*pte) && !pte_exec(*pte))
pr_crit("kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n",
from_kuid(&init_user_ns, current_uid()));
if (pte && pte_present(*pte) && pte_exec(*pte) &&
(pgd_flags(*pgd) & _PAGE_USER) &&
(__read_cr4() & X86_CR4_SMEP))
pr_crit("unable to execute userspace code (SMEP?) (uid: %d)\n",
from_kuid(&init_user_ns, current_uid()));
}
if (address < PAGE_SIZE && !user_mode(regs))
pr_alert("BUG: kernel NULL pointer dereference, address: %px\n",
(void *)address);
else
pr_alert("BUG: unable to handle page fault for address: %px\n",
(void *)address);
pr_alert("#PF: %s %s in %s mode\n",
(error_code & X86_PF_USER) ? "user" : "supervisor",
(error_code & X86_PF_INSTR) ? "instruction fetch" :
(error_code & X86_PF_WRITE) ? "write access" :
"read access",
user_mode(regs) ? "user" : "kernel");
pr_alert("#PF: error_code(0x%04lx) - %s\n", error_code,
!(error_code & X86_PF_PROT) ? "not-present page" :
(error_code & X86_PF_RSVD) ? "reserved bit violation" :
(error_code & X86_PF_PK) ? "protection keys violation" :
(error_code & X86_PF_RMP) ? "RMP violation" :
"permissions violation");
if (!(error_code & X86_PF_USER) && user_mode(regs)) {
struct desc_ptr idt, gdt;
u16 ldtr, tr;
/*
* This can happen for quite a few reasons. The more obvious
* ones are faults accessing the GDT, or LDT. Perhaps
* surprisingly, if the CPU tries to deliver a benign or
* contributory exception from user code and gets a page fault
* during delivery, the page fault can be delivered as though
* it originated directly from user code. This could happen
* due to wrong permissions on the IDT, GDT, LDT, TSS, or
* kernel or IST stack.
*/
store_idt(&idt);
/* Usable even on Xen PV -- it's just slow. */
native_store_gdt(&gdt);
pr_alert("IDT: 0x%lx (limit=0x%hx) GDT: 0x%lx (limit=0x%hx)\n",
idt.address, idt.size, gdt.address, gdt.size);
store_ldt(ldtr);
show_ldttss(&gdt, "LDTR", ldtr);
store_tr(tr);
show_ldttss(&gdt, "TR", tr);
}
dump_pagetable(address);
if (error_code & X86_PF_RMP)
snp_dump_hva_rmpentry(address);
}
static noinline void
pgtable_bad(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{
struct task_struct *tsk;
unsigned long flags;
int sig;
flags = oops_begin();
tsk = current;
sig = SIGKILL;
printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
tsk->comm, address);
dump_pagetable(address);
if (__die("Bad pagetable", regs, error_code))
sig = 0;
oops_end(flags, regs, sig);
}
static void sanitize_error_code(unsigned long address,
unsigned long *error_code)
{
/*
* To avoid leaking information about the kernel page
* table layout, pretend that user-mode accesses to
* kernel addresses are always protection faults.
*
* NB: This means that failed vsyscalls with vsyscall=none
* will have the PROT bit. This doesn't leak any
* information and does not appear to cause any problems.
*/
if (address >= TASK_SIZE_MAX) *error_code |= X86_PF_PROT;
}
static void set_signal_archinfo(unsigned long address,
unsigned long error_code)
{
struct task_struct *tsk = current;
tsk->thread.trap_nr = X86_TRAP_PF;
tsk->thread.error_code = error_code | X86_PF_USER;
tsk->thread.cr2 = address;
}
static noinline void
page_fault_oops(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{
#ifdef CONFIG_VMAP_STACK
struct stack_info info;
#endif
unsigned long flags;
int sig;
if (user_mode(regs)) {
/*
* Implicit kernel access from user mode? Skip the stack
* overflow and EFI special cases.
*/
goto oops;
}
#ifdef CONFIG_VMAP_STACK
/*
* Stack overflow? During boot, we can fault near the initial
* stack in the direct map, but that's not an overflow -- check
* that we're in vmalloc space to avoid this.
*/
if (is_vmalloc_addr((void *)address) &&
get_stack_guard_info((void *)address, &info)) {
/*
* We're likely to be running with very little stack space
* left. It's plausible that we'd hit this condition but
* double-fault even before we get this far, in which case
* we're fine: the double-fault handler will deal with it.
*
* We don't want to make it all the way into the oops code
* and then double-fault, though, because we're likely to
* break the console driver and lose most of the stack dump.
*/
call_on_stack(__this_cpu_ist_top_va(DF) - sizeof(void*),
handle_stack_overflow,
ASM_CALL_ARG3,
, [arg1] "r" (regs), [arg2] "r" (address), [arg3] "r" (&info));
unreachable();
}
#endif
/*
* Buggy firmware could access regions which might page fault. If
* this happens, EFI has a special OOPS path that will try to
* avoid hanging the system.
*/
if (IS_ENABLED(CONFIG_EFI))
efi_crash_gracefully_on_page_fault(address);
/* Only not-present faults should be handled by KFENCE. */
if (!(error_code & X86_PF_PROT) &&
kfence_handle_page_fault(address, error_code & X86_PF_WRITE, regs))
return;
oops:
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice:
*/
flags = oops_begin();
show_fault_oops(regs, error_code, address);
if (task_stack_end_corrupted(current))
printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
sig = SIGKILL;
if (__die("Oops", regs, error_code))
sig = 0;
/* Executive summary in case the body of the oops scrolled away */
printk(KERN_DEFAULT "CR2: %016lx\n", address);
oops_end(flags, regs, sig);
}
static noinline void
kernelmode_fixup_or_oops(struct pt_regs *regs, unsigned long error_code,
unsigned long address, int signal, int si_code,
u32 pkey)
{
WARN_ON_ONCE(user_mode(regs));
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs, X86_TRAP_PF, error_code, address))
return;
/*
* AMD erratum #91 manifests as a spurious page fault on a PREFETCH
* instruction.
*/
if (is_prefetch(regs, error_code, address))
return;
page_fault_oops(regs, error_code, address);
}
/*
* Print out info about fatal segfaults, if the show_unhandled_signals
* sysctl is set:
*/
static inline void
show_signal_msg(struct pt_regs *regs, unsigned long error_code,
unsigned long address, struct task_struct *tsk)
{
const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG;
/* This is a racy snapshot, but it's better than nothing. */
int cpu = raw_smp_processor_id();
if (!unhandled_signal(tsk, SIGSEGV))
return;
if (!printk_ratelimit())
return;
printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx",
loglvl, tsk->comm, task_pid_nr(tsk), address,
(void *)regs->ip, (void *)regs->sp, error_code);
print_vma_addr(KERN_CONT " in ", regs->ip);
/*
* Dump the likely CPU where the fatal segfault happened.
* This can help identify faulty hardware.
*/
printk(KERN_CONT " likely on CPU %d (core %d, socket %d)", cpu,
topology_core_id(cpu), topology_physical_package_id(cpu));
printk(KERN_CONT "\n");
show_opcodes(regs, loglvl);
}
static void
__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
unsigned long address, u32 pkey, int si_code)
{
struct task_struct *tsk = current;
if (!user_mode(regs)) {
kernelmode_fixup_or_oops(regs, error_code, address,
SIGSEGV, si_code, pkey);
return;
}
if (!(error_code & X86_PF_USER)) {
/* Implicit user access to kernel memory -- just oops */
page_fault_oops(regs, error_code, address);
return;
}
/*
* User mode accesses just cause a SIGSEGV.
* It's possible to have interrupts off here:
*/
local_irq_enable();
/*
* Valid to do another page fault here because this one came
* from user space:
*/
if (is_prefetch(regs, error_code, address))
return;
if (is_errata100(regs, address))
return;
sanitize_error_code(address, &error_code); if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address))
return;
if (likely(show_unhandled_signals)) show_signal_msg(regs, error_code, address, tsk); set_signal_archinfo(address, error_code);
if (si_code == SEGV_PKUERR)
force_sig_pkuerr((void __user *)address, pkey);
else
force_sig_fault(SIGSEGV, si_code, (void __user *)address); local_irq_disable();
}
static noinline void
bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{
__bad_area_nosemaphore(regs, error_code, address, 0, SEGV_MAPERR);
}
static void
__bad_area(struct pt_regs *regs, unsigned long error_code,
unsigned long address, u32 pkey, int si_code)
{
struct mm_struct *mm = current->mm;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
mmap_read_unlock(mm);
__bad_area_nosemaphore(regs, error_code, address, pkey, si_code);
}
static inline bool bad_area_access_from_pkeys(unsigned long error_code,
struct vm_area_struct *vma)
{
/* This code is always called on the current mm */
bool foreign = false;
if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
return false;
if (error_code & X86_PF_PK)
return true;
/* this checks permission keys on the VMA: */
if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE), (error_code & X86_PF_INSTR), foreign))
return true;
return false;
}
static noinline void
bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
unsigned long address, struct vm_area_struct *vma)
{
/*
* This OSPKE check is not strictly necessary at runtime.
* But, doing it this way allows compiler optimizations
* if pkeys are compiled out.
*/
if (bad_area_access_from_pkeys(error_code, vma)) {
/*
* A protection key fault means that the PKRU value did not allow
* access to some PTE. Userspace can figure out what PKRU was
* from the XSAVE state. This function captures the pkey from
* the vma and passes it to userspace so userspace can discover
* which protection key was set on the PTE.
*
* If we get here, we know that the hardware signaled a X86_PF_PK
* fault and that there was a VMA once we got in the fault
* handler. It does *not* guarantee that the VMA we find here
* was the one that we faulted on.
*
* 1. T1 : mprotect_key(foo, PAGE_SIZE, pkey=4);
* 2. T1 : set PKRU to deny access to pkey=4, touches page
* 3. T1 : faults...
* 4. T2: mprotect_key(foo, PAGE_SIZE, pkey=5);
* 5. T1 : enters fault handler, takes mmap_lock, etc...
* 6. T1 : reaches here, sees vma_pkey(vma)=5, when we really
* faulted on a pte with its pkey=4.
*/
u32 pkey = vma_pkey(vma);
__bad_area(regs, error_code, address, pkey, SEGV_PKUERR);
} else {
__bad_area(regs, error_code, address, 0, SEGV_ACCERR);
}
}
static void
do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
vm_fault_t fault)
{
/* Kernel mode? Handle exceptions or die: */
if (!user_mode(regs)) {
kernelmode_fixup_or_oops(regs, error_code, address,
SIGBUS, BUS_ADRERR, ARCH_DEFAULT_PKEY);
return;
}
/* User-space => ok to do another page fault: */
if (is_prefetch(regs, error_code, address))
return;
sanitize_error_code(address, &error_code); if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address))
return;
set_signal_archinfo(address, error_code);
#ifdef CONFIG_MEMORY_FAILURE
if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
struct task_struct *tsk = current;
unsigned lsb = 0;
pr_err(
"MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
tsk->comm, tsk->pid, address);
if (fault & VM_FAULT_HWPOISON_LARGE)
lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
if (fault & VM_FAULT_HWPOISON)
lsb = PAGE_SHIFT;
force_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb);
return;
}
#endif
force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
}
static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)
{
if ((error_code & X86_PF_WRITE) && !pte_write(*pte))
return 0;
if ((error_code & X86_PF_INSTR) && !pte_exec(*pte))
return 0;
return 1;
}
/*
* Handle a spurious fault caused by a stale TLB entry.
*
* This allows us to lazily refresh the TLB when increasing the
* permissions of a kernel page (RO -> RW or NX -> X). Doing it
* eagerly is very expensive since that implies doing a full
* cross-processor TLB flush, even if no stale TLB entries exist
* on other processors.
*
* Spurious faults may only occur if the TLB contains an entry with
* fewer permission than the page table entry. Non-present (P = 0)
* and reserved bit (R = 1) faults are never spurious.
*
* There are no security implications to leaving a stale TLB when
* increasing the permissions on a page.
*
* Returns non-zero if a spurious fault was handled, zero otherwise.
*
* See Intel Developer's Manual Vol 3 Section 4.10.4.3, bullet 3
* (Optional Invalidation).
*/
static noinline int
spurious_kernel_fault(unsigned long error_code, unsigned long address)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
int ret;
/*
* Only writes to RO or instruction fetches from NX may cause
* spurious faults.
*
* These could be from user or supervisor accesses but the TLB
* is only lazily flushed after a kernel mapping protection
* change, so user accesses are not expected to cause spurious
* faults.
*/
if (error_code != (X86_PF_WRITE | X86_PF_PROT) &&
error_code != (X86_PF_INSTR | X86_PF_PROT))
return 0; pgd = init_mm.pgd + pgd_index(address);
if (!pgd_present(*pgd))
return 0;
p4d = p4d_offset(pgd, address);
if (!p4d_present(*p4d))
return 0;
if (p4d_leaf(*p4d))
return spurious_kernel_fault_check(error_code, (pte_t *) p4d);
pud = pud_offset(p4d, address); if (!pud_present(*pud))
return 0;
if (pud_leaf(*pud))
return spurious_kernel_fault_check(error_code, (pte_t *) pud);
pmd = pmd_offset(pud, address); if (!pmd_present(*pmd))
return 0;
if (pmd_leaf(*pmd)) return spurious_kernel_fault_check(error_code, (pte_t *) pmd); pte = pte_offset_kernel(pmd, address);
if (!pte_present(*pte))
return 0;
ret = spurious_kernel_fault_check(error_code, pte);
if (!ret)
return 0;
/*
* Make sure we have permissions in PMD.
* If not, then there's a bug in the page tables:
*/
ret = spurious_kernel_fault_check(error_code, (pte_t *) pmd); WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
return ret;
}
NOKPROBE_SYMBOL(spurious_kernel_fault);
int show_unhandled_signals = 1;
static inline int
access_error(unsigned long error_code, struct vm_area_struct *vma)
{
/* This is only called for the current mm, so: */
bool foreign = false;
/*
* Read or write was blocked by protection keys. This is
* always an unconditional error and can never result in
* a follow-up action to resolve the fault, like a COW.
*/
if (error_code & X86_PF_PK)
return 1;
/*
* SGX hardware blocked the access. This usually happens
* when the enclave memory contents have been destroyed, like
* after a suspend/resume cycle. In any case, the kernel can't
* fix the cause of the fault. Handle the fault as an access
* error even in cases where no actual access violation
* occurred. This allows userspace to rebuild the enclave in
* response to the signal.
*/
if (unlikely(error_code & X86_PF_SGX))
return 1;
/*
* Make sure to check the VMA so that we do not perform
* faults just to hit a X86_PF_PK as soon as we fill in a
* page.
*/
if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE), (error_code & X86_PF_INSTR), foreign))
return 1;
/*
* Shadow stack accesses (PF_SHSTK=1) are only permitted to
* shadow stack VMAs. All other accesses result in an error.
*/
if (error_code & X86_PF_SHSTK) {
if (unlikely(!(vma->vm_flags & VM_SHADOW_STACK)))
return 1;
if (unlikely(!(vma->vm_flags & VM_WRITE)))
return 1;
return 0;
}
if (error_code & X86_PF_WRITE) {
/* write, present and write, not present: */
if (unlikely(vma->vm_flags & VM_SHADOW_STACK))
return 1;
if (unlikely(!(vma->vm_flags & VM_WRITE)))
return 1;
return 0;
}
/* read, present: */
if (unlikely(error_code & X86_PF_PROT))
return 1;
/* read, not present: */
if (unlikely(!vma_is_accessible(vma)))
return 1;
return 0;
}
bool fault_in_kernel_space(unsigned long address)
{
/*
* On 64-bit systems, the vsyscall page is at an address above
* TASK_SIZE_MAX, but is not considered part of the kernel
* address space.
*/
if (IS_ENABLED(CONFIG_X86_64) && is_vsyscall_vaddr(address))
return false;
return address >= TASK_SIZE_MAX;}
/*
* Called for all faults where 'address' is part of the kernel address
* space. Might get called for faults that originate from *code* that
* ran in userspace or the kernel.
*/
static void
do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code,
unsigned long address)
{
/*
* Protection keys exceptions only happen on user pages. We
* have no user pages in the kernel portion of the address
* space, so do not expect them here.
*/
WARN_ON_ONCE(hw_error_code & X86_PF_PK);
#ifdef CONFIG_X86_32
/*
* We can fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*
* Before doing this on-demand faulting, ensure that the
* fault is not any of the following:
* 1. A fault on a PTE with a reserved bit set.
* 2. A fault caused by a user-mode access. (Do not demand-
* fault kernel memory due to user-mode accesses).
* 3. A fault caused by a page-level protection violation.
* (A demand fault would be on a non-present page which
* would have X86_PF_PROT==0).
*
* This is only needed to close a race condition on x86-32 in
* the vmalloc mapping/unmapping code. See the comment above
* vmalloc_fault() for details. On x86-64 the race does not
* exist as the vmalloc mappings don't need to be synchronized
* there.
*/
if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
if (vmalloc_fault(address) >= 0)
return;
}
#endif
if (is_f00f_bug(regs, hw_error_code, address))
return;
/* Was the fault spurious, caused by lazy TLB invalidation? */
if (spurious_kernel_fault(hw_error_code, address))
return;
/* kprobes don't want to hook the spurious faults: */
if (WARN_ON_ONCE(kprobe_page_fault(regs, X86_TRAP_PF)))
return;
/*
* Note, despite being a "bad area", there are quite a few
* acceptable reasons to get here, such as erratum fixups
* and handling kernel code that can fault, like get_user().
*
* Don't take the mm semaphore here. If we fixup a prefetch
* fault we could otherwise deadlock:
*/
bad_area_nosemaphore(regs, hw_error_code, address);
}
NOKPROBE_SYMBOL(do_kern_addr_fault);
/*
* Handle faults in the user portion of the address space. Nothing in here
* should check X86_PF_USER without a specific justification: for almost
* all purposes, we should treat a normal kernel access to user memory
* (e.g. get_user(), put_user(), etc.) the same as the WRUSS instruction.
* The one exception is AC flag handling, which is, per the x86
* architecture, special for WRUSS.
*/
static inline
void do_user_addr_fault(struct pt_regs *regs,
unsigned long error_code,
unsigned long address)
{
struct vm_area_struct *vma;
struct task_struct *tsk;
struct mm_struct *mm;
vm_fault_t fault;
unsigned int flags = FAULT_FLAG_DEFAULT;
tsk = current;
mm = tsk->mm;
if (unlikely((error_code & (X86_PF_USER | X86_PF_INSTR)) == X86_PF_INSTR)) {
/*
* Whoops, this is kernel mode code trying to execute from
* user memory. Unless this is AMD erratum #93, which
* corrupts RIP such that it looks like a user address,
* this is unrecoverable. Don't even try to look up the
* VMA or look for extable entries.
*/
if (is_errata93(regs, address))
return;
page_fault_oops(regs, error_code, address);
return;
}
/* kprobes don't want to hook the spurious faults: */
if (WARN_ON_ONCE(kprobe_page_fault(regs, X86_TRAP_PF)))
return;
/*
* Reserved bits are never expected to be set on
* entries in the user portion of the page tables.
*/
if (unlikely(error_code & X86_PF_RSVD)) pgtable_bad(regs, error_code, address);
/*
* If SMAP is on, check for invalid kernel (supervisor) access to user
* pages in the user address space. The odd case here is WRUSS,
* which, according to the preliminary documentation, does not respect
* SMAP and will have the USER bit set so, in all cases, SMAP
* enforcement appears to be consistent with the USER bit.
*/
if (unlikely(cpu_feature_enabled(X86_FEATURE_SMAP) &&
!(error_code & X86_PF_USER) &&
!(regs->flags & X86_EFLAGS_AC))) {
/*
* No extable entry here. This was a kernel access to an
* invalid pointer. get_kernel_nofault() will not get here.
*/
page_fault_oops(regs, error_code, address);
return;
}
/*
* If we're in an interrupt, have no user context or are running
* in a region with pagefaults disabled then we must not take the fault
*/
if (unlikely(faulthandler_disabled() || !mm)) { bad_area_nosemaphore(regs, error_code, address);
return;
}
/* Legacy check - remove this after verifying that it doesn't trigger */
if (WARN_ON_ONCE(!(regs->flags & X86_EFLAGS_IF))) {
bad_area_nosemaphore(regs, error_code, address);
return;
}
local_irq_enable(); perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
/*
* Read-only permissions can not be expressed in shadow stack PTEs.
* Treat all shadow stack accesses as WRITE faults. This ensures
* that the MM will prepare everything (e.g., break COW) such that
* maybe_mkwrite() can create a proper shadow stack PTE.
*/
if (error_code & X86_PF_SHSTK)
flags |= FAULT_FLAG_WRITE;
if (error_code & X86_PF_WRITE)
flags |= FAULT_FLAG_WRITE;
if (error_code & X86_PF_INSTR) flags |= FAULT_FLAG_INSTRUCTION;
/*
* We set FAULT_FLAG_USER based on the register state, not
* based on X86_PF_USER. User space accesses that cause
* system page faults are still user accesses.
*/
if (user_mode(regs)) flags |= FAULT_FLAG_USER;
#ifdef CONFIG_X86_64
/*
* Faults in the vsyscall page might need emulation. The
* vsyscall page is at a high address (>PAGE_OFFSET), but is
* considered to be part of the user address space.
*
* The vsyscall page does not have a "real" VMA, so do this
* emulation before we go searching for VMAs.
*
* PKRU never rejects instruction fetches, so we don't need
* to consider the PF_PK bit.
*/
if (is_vsyscall_vaddr(address)) { if (emulate_vsyscall(error_code, regs, address))
return;
}
#endif
if (!(flags & FAULT_FLAG_USER))
goto lock_mmap;
vma = lock_vma_under_rcu(mm, address);
if (!vma)
goto lock_mmap;
if (unlikely(access_error(error_code, vma))) { vma_end_read(vma);
goto lock_mmap;
}
fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
vma_end_read(vma); if (!(fault & VM_FAULT_RETRY)) {
count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
goto done;
}
count_vm_vma_lock_event(VMA_LOCK_RETRY);
if (fault & VM_FAULT_MAJOR) flags |= FAULT_FLAG_TRIED;
/* Quick path to respond to signals */
if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) kernelmode_fixup_or_oops(regs, error_code, address,
SIGBUS, BUS_ADRERR,
ARCH_DEFAULT_PKEY);
return;
}
lock_mmap:
retry:
vma = lock_mm_and_find_vma(mm, address, regs);
if (unlikely(!vma)) {
bad_area_nosemaphore(regs, error_code, address);
return;
}
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
if (unlikely(access_error(error_code, vma))) { bad_area_access_error(regs, error_code, address, vma);
return;
}
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault. Since we never set FAULT_FLAG_RETRY_NOWAIT, if
* we get VM_FAULT_RETRY back, the mmap_lock has been unlocked.
*
* Note that handle_userfault() may also release and reacquire mmap_lock
* (and not return with VM_FAULT_RETRY), when returning to userland to
* repeat the page fault later with a VM_FAULT_NOPAGE retval
* (potentially after handling any pending signal during the return to
* userland). The return to userland is identified whenever
* FAULT_FLAG_USER|FAULT_FLAG_KILLABLE are both set in flags.
*/
fault = handle_mm_fault(vma, address, flags, regs);
if (fault_signal_pending(fault, regs)) {
/*
* Quick path to respond to signals. The core mm code
* has unlocked the mm for us if we get here.
*/
if (!user_mode(regs))
kernelmode_fixup_or_oops(regs, error_code, address,
SIGBUS, BUS_ADRERR,
ARCH_DEFAULT_PKEY);
return;
}
/* The fault is fully completed (including releasing mmap lock) */
if (fault & VM_FAULT_COMPLETED)
return;
/*
* If we need to retry the mmap_lock has already been released,
* and if there is a fatal signal pending there is no guarantee
* that we made any progress. Handle this case first.
*/
if (unlikely(fault & VM_FAULT_RETRY)) { flags |= FAULT_FLAG_TRIED;
goto retry;
}
mmap_read_unlock(mm);
done:
if (likely(!(fault & VM_FAULT_ERROR)))
return;
if (fatal_signal_pending(current) && !user_mode(regs)) {
kernelmode_fixup_or_oops(regs, error_code, address,
0, 0, ARCH_DEFAULT_PKEY);
return;
}
if (fault & VM_FAULT_OOM) {
/* Kernel mode? Handle exceptions or die: */
if (!user_mode(regs)) {
kernelmode_fixup_or_oops(regs, error_code, address,
SIGSEGV, SEGV_MAPERR,
ARCH_DEFAULT_PKEY);
return;
}
/*
* We ran out of memory, call the OOM killer, and return the
* userspace (which will retry the fault, or kill us if we got
* oom-killed):
*/
pagefault_out_of_memory();
} else {
if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
VM_FAULT_HWPOISON_LARGE))
do_sigbus(regs, error_code, address, fault); else if (fault & VM_FAULT_SIGSEGV)
bad_area_nosemaphore(regs, error_code, address);
else
BUG();
}
}
NOKPROBE_SYMBOL(do_user_addr_fault);
static __always_inline void
trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{
if (!trace_pagefault_enabled())
return;
if (user_mode(regs))
trace_page_fault_user(address, regs, error_code);
else
trace_page_fault_kernel(address, regs, error_code);
}
static __always_inline void
handle_page_fault(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{
trace_page_fault_entries(regs, error_code, address);
if (unlikely(kmmio_fault(regs, address)))
return;
/* Was the fault on kernel-controlled part of the address space? */
if (unlikely(fault_in_kernel_space(address))) {
do_kern_addr_fault(regs, error_code, address);
} else {
do_user_addr_fault(regs, error_code, address);
/*
* User address page fault handling might have reenabled
* interrupts. Fixing up all potential exit points of
* do_user_addr_fault() and its leaf functions is just not
* doable w/o creating an unholy mess or turning the code
* upside down.
*/
local_irq_disable();
}
}
DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)
{
irqentry_state_t state;
unsigned long address;
address = cpu_feature_enabled(X86_FEATURE_FRED) ? fred_event_data(regs) : read_cr2();
prefetchw(¤t->mm->mmap_lock);
/*
* KVM uses #PF vector to deliver 'page not present' events to guests
* (asynchronous page fault mechanism). The event happens when a
* userspace task is trying to access some valid (from guest's point of
* view) memory which is not currently mapped by the host (e.g. the
* memory is swapped out). Note, the corresponding "page ready" event
* which is injected when the memory becomes available, is delivered via
* an interrupt mechanism and not a #PF exception
* (see arch/x86/kernel/kvm.c: sysvec_kvm_asyncpf_interrupt()).
*
* We are relying on the interrupted context being sane (valid RSP,
* relevant locks not held, etc.), which is fine as long as the
* interrupted context had IF=1. We are also relying on the KVM
* async pf type field and CR2 being read consistently instead of
* getting values from real and async page faults mixed up.
*
* Fingers crossed.
*
* The async #PF handling code takes care of idtentry handling
* itself.
*/
if (kvm_handle_async_pf(regs, (u32)address))
return;
/*
* Entry handling for valid #PF from kernel mode is slightly
* different: RCU is already watching and ct_irq_enter() must not
* be invoked because a kernel fault on a user space address might
* sleep.
*
* In case the fault hit a RCU idle region the conditional entry
* code reenabled RCU to avoid subsequent wreckage which helps
* debuggability.
*/
state = irqentry_enter(regs);
instrumentation_begin();
handle_page_fault(regs, error_code, address);
instrumentation_end();
irqentry_exit(regs, state);
}
// SPDX-License-Identifier: GPL-2.0
#include <linux/compiler.h>
#include <linux/export.h>
#include <linux/fault-inject-usercopy.h>
#include <linux/kasan-checks.h>
#include <linux/thread_info.h>
#include <linux/uaccess.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <asm/byteorder.h>
#include <asm/word-at-a-time.h>
#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
#define IS_UNALIGNED(src, dst) 0
#else
#define IS_UNALIGNED(src, dst) \
(((long) dst | (long) src) & (sizeof(long) - 1))
#endif
/*
* Do a strncpy, return length of string without final '\0'.
* 'count' is the user-supplied count (return 'count' if we
* hit it), 'max' is the address space maximum (and we return
* -EFAULT if we hit it).
*/
static __always_inline long do_strncpy_from_user(char *dst, const char __user *src,
unsigned long count, unsigned long max)
{
const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
unsigned long res = 0;
if (IS_UNALIGNED(src, dst))
goto byte_at_a_time;
while (max >= sizeof(unsigned long)) {
unsigned long c, data, mask;
/* Fall back to byte-at-a-time if we get a page fault */
unsafe_get_user(c, (unsigned long __user *)(src+res), byte_at_a_time);
/*
* Note that we mask out the bytes following the NUL. This is
* important to do because string oblivious code may read past
* the NUL. For those routines, we don't want to give them
* potentially random bytes after the NUL in `src`.
*
* One example of such code is BPF map keys. BPF treats map keys
* as an opaque set of bytes. Without the post-NUL mask, any BPF
* maps keyed by strings returned from strncpy_from_user() may
* have multiple entries for semantically identical strings.
*/
if (has_zero(c, &data, &constants)) {
data = prep_zero_mask(c, data, &constants);
data = create_zero_mask(data);
mask = zero_bytemask(data);
*(unsigned long *)(dst+res) = c & mask;
return res + find_zero(data);
}
*(unsigned long *)(dst+res) = c;
res += sizeof(unsigned long);
max -= sizeof(unsigned long);
}
byte_at_a_time:
while (max) {
char c;
unsafe_get_user(c,src+res, efault);
dst[res] = c;
if (!c)
return res;
res++;
max--;
}
/*
* Uhhuh. We hit 'max'. But was that the user-specified maximum
* too? If so, that's ok - we got as much as the user asked for.
*/
if (res >= count) return res;
/*
* Nope: we hit the address space limit, and we still had more
* characters the caller would have wanted. That's an EFAULT.
*/
efault:
return -EFAULT;
}
/**
* strncpy_from_user: - Copy a NUL terminated string from userspace.
* @dst: Destination address, in kernel space. This buffer must be at
* least @count bytes long.
* @src: Source address, in user space.
* @count: Maximum number of bytes to copy, including the trailing NUL.
*
* Copies a NUL-terminated string from userspace to kernel space.
*
* On success, returns the length of the string (not including the trailing
* NUL).
*
* If access to userspace fails, returns -EFAULT (some data may have been
* copied).
*
* If @count is smaller than the length of the string, copies @count bytes
* and returns @count.
*/
long strncpy_from_user(char *dst, const char __user *src, long count)
{
unsigned long max_addr, src_addr;
might_fault();
if (should_fail_usercopy())
return -EFAULT; if (unlikely(count <= 0))
return 0;
max_addr = TASK_SIZE_MAX;
src_addr = (unsigned long)untagged_addr(src);
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
long retval;
/*
* Truncate 'max' to the user-specified limit, so that
* we only have one limit we need to check in the loop
*/
if (max > count)
max = count;
kasan_check_write(dst, count);
check_object_size(dst, count, false);
if (user_read_access_begin(src, max)) { retval = do_strncpy_from_user(dst, src, count, max); user_read_access_end();
return retval;
}
}
return -EFAULT;
}
EXPORT_SYMBOL(strncpy_from_user);
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/usb/core/file.c
*
* (C) Copyright Linus Torvalds 1999
* (C) Copyright Johannes Erdfelt 1999-2001
* (C) Copyright Andreas Gal 1999
* (C) Copyright Gregory P. Smith 1999
* (C) Copyright Deti Fliegl 1999 (new USB architecture)
* (C) Copyright Randy Dunlap 2000
* (C) Copyright David Brownell 2000-2001 (kernel hotplug, usb_device_id,
* more docs, etc)
* (C) Copyright Yggdrasil Computing, Inc. 2000
* (usb_device_id matching changes by Adam J. Richter)
* (C) Copyright Greg Kroah-Hartman 2002-2003
*
* Released under the GPLv2 only.
*/
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/usb.h>
#include "usb.h"
#define MAX_USB_MINORS 256
static const struct file_operations *usb_minors[MAX_USB_MINORS];
static DECLARE_RWSEM(minor_rwsem);
static int usb_open(struct inode *inode, struct file *file)
{
int err = -ENODEV;
const struct file_operations *new_fops;
down_read(&minor_rwsem); new_fops = fops_get(usb_minors[iminor(inode)]);
if (!new_fops)
goto done;
replace_fops(file, new_fops);
/* Curiouser and curiouser... NULL ->open() as "no device" ? */
if (file->f_op->open)
err = file->f_op->open(inode, file);
done:
up_read(&minor_rwsem);
return err;
}
static const struct file_operations usb_fops = {
.owner = THIS_MODULE,
.open = usb_open,
.llseek = noop_llseek,
};
static char *usb_devnode(const struct device *dev, umode_t *mode)
{
struct usb_class_driver *drv;
drv = dev_get_drvdata(dev); if (!drv || !drv->devnode)
return NULL;
return drv->devnode(dev, mode);}
const struct class usbmisc_class = {
.name = "usbmisc",
.devnode = usb_devnode,
};
int usb_major_init(void)
{
int error;
error = register_chrdev(USB_MAJOR, "usb", &usb_fops);
if (error)
printk(KERN_ERR "Unable to get major %d for usb devices\n",
USB_MAJOR);
return error;
}
void usb_major_cleanup(void)
{
unregister_chrdev(USB_MAJOR, "usb");
}
/**
* usb_register_dev - register a USB device, and ask for a minor number
* @intf: pointer to the usb_interface that is being registered
* @class_driver: pointer to the usb_class_driver for this device
*
* This should be called by all USB drivers that use the USB major number.
* If CONFIG_USB_DYNAMIC_MINORS is enabled, the minor number will be
* dynamically allocated out of the list of available ones. If it is not
* enabled, the minor number will be based on the next available free minor,
* starting at the class_driver->minor_base.
*
* This function also creates a usb class device in the sysfs tree.
*
* usb_deregister_dev() must be called when the driver is done with
* the minor numbers given out by this function.
*
* Return: -EINVAL if something bad happens with trying to register a
* device, and 0 on success.
*/
int usb_register_dev(struct usb_interface *intf,
struct usb_class_driver *class_driver)
{
int retval = 0;
int minor_base = class_driver->minor_base;
int minor;
char name[20];
#ifdef CONFIG_USB_DYNAMIC_MINORS
/*
* We don't care what the device tries to start at, we want to start
* at zero to pack the devices into the smallest available space with
* no holes in the minor range.
*/
minor_base = 0;
#endif
if (class_driver->fops == NULL)
return -EINVAL;
if (intf->minor >= 0)
return -EADDRINUSE;
dev_dbg(&intf->dev, "looking for a minor, starting at %d\n", minor_base);
down_write(&minor_rwsem);
for (minor = minor_base; minor < MAX_USB_MINORS; ++minor) {
if (usb_minors[minor])
continue;
usb_minors[minor] = class_driver->fops;
intf->minor = minor;
break;
}
if (intf->minor < 0) {
up_write(&minor_rwsem);
return -EXFULL;
}
/* create a usb class device for this usb interface */
snprintf(name, sizeof(name), class_driver->name, minor - minor_base);
intf->usb_dev = device_create(&usbmisc_class, &intf->dev,
MKDEV(USB_MAJOR, minor), class_driver,
"%s", kbasename(name));
if (IS_ERR(intf->usb_dev)) {
usb_minors[minor] = NULL;
intf->minor = -1;
retval = PTR_ERR(intf->usb_dev);
}
up_write(&minor_rwsem);
return retval;
}
EXPORT_SYMBOL_GPL(usb_register_dev);
/**
* usb_deregister_dev - deregister a USB device's dynamic minor.
* @intf: pointer to the usb_interface that is being deregistered
* @class_driver: pointer to the usb_class_driver for this device
*
* Used in conjunction with usb_register_dev(). This function is called
* when the USB driver is finished with the minor numbers gotten from a
* call to usb_register_dev() (usually when the device is disconnected
* from the system.)
*
* This function also removes the usb class device from the sysfs tree.
*
* This should be called by all drivers that use the USB major number.
*/
void usb_deregister_dev(struct usb_interface *intf,
struct usb_class_driver *class_driver)
{
if (intf->minor == -1)
return;
dev_dbg(&intf->dev, "removing %d minor\n", intf->minor); device_destroy(&usbmisc_class, MKDEV(USB_MAJOR, intf->minor));
down_write(&minor_rwsem);
usb_minors[intf->minor] = NULL;
up_write(&minor_rwsem);
intf->usb_dev = NULL;
intf->minor = -1;
}
EXPORT_SYMBOL_GPL(usb_deregister_dev);
// SPDX-License-Identifier: GPL-2.0-only
/*
* Simple NUMA memory policy for the Linux kernel.
*
* Copyright 2003,2004 Andi Kleen, SuSE Labs.
* (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
*
* NUMA policy allows the user to give hints in which node(s) memory should
* be allocated.
*
* Support four policies per VMA and per process:
*
* The VMA policy has priority over the process policy for a page fault.
*
* interleave Allocate memory interleaved over a set of nodes,
* with normal fallback if it fails.
* For VMA based allocations this interleaves based on the
* offset into the backing object or offset into the mapping
* for anonymous memory. For process policy an process counter
* is used.
*
* weighted interleave
* Allocate memory interleaved over a set of nodes based on
* a set of weights (per-node), with normal fallback if it
* fails. Otherwise operates the same as interleave.
* Example: nodeset(0,1) & weights (2,1) - 2 pages allocated
* on node 0 for every 1 page allocated on node 1.
*
* bind Only allocate memory on a specific set of nodes,
* no fallback.
* FIXME: memory is allocated starting with the first node
* to the last. It would be better if bind would truly restrict
* the allocation to memory nodes instead
*
* preferred Try a specific node first before normal fallback.
* As a special case NUMA_NO_NODE here means do the allocation
* on the local CPU. This is normally identical to default,
* but useful to set in a VMA when you have a non default
* process policy.
*
* preferred many Try a set of nodes first before normal fallback. This is
* similar to preferred without the special case.
*
* default Allocate on the local node first, or when on a VMA
* use the process policy. This is what Linux always did
* in a NUMA aware kernel and still does by, ahem, default.
*
* The process policy is applied for most non interrupt memory allocations
* in that process' context. Interrupts ignore the policies and always
* try to allocate on the local CPU. The VMA policy is only applied for memory
* allocations for a VMA in the VM.
*
* Currently there are a few corner cases in swapping where the policy
* is not applied, but the majority should be handled. When process policy
* is used it is not remembered over swap outs/swap ins.
*
* Only the highest zone in the zone hierarchy gets policied. Allocations
* requesting a lower zone just use default policy. This implies that
* on systems with highmem kernel lowmem allocation don't get policied.
* Same with GFP_DMA allocations.
*
* For shmem/tmpfs shared memory the policy is shared between
* all users and remembered even when nobody has memory mapped.
*/
/* Notebook:
fix mmap readahead to honour policy and enable policy for any page cache
object
statistics for bigpages
global policy for page cache? currently it uses process policy. Requires
first item above.
handle mremap for shared memory (currently ignored for the policy)
grows down?
make bind policy root only? It can trigger oom much faster and the
kernel is not always grateful with that.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/mempolicy.h>
#include <linux/pagewalk.h>
#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
#include <linux/nodemask.h>
#include <linux/cpuset.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/nsproxy.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/compat.h>
#include <linux/ptrace.h>
#include <linux/swap.h>
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <linux/migrate.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/ctype.h>
#include <linux/mm_inline.h>
#include <linux/mmu_notifier.h>
#include <linux/printk.h>
#include <linux/swapops.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <linux/uaccess.h>
#include "internal.h"
/* Internal flags */
#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
#define MPOL_MF_WRLOCK (MPOL_MF_INTERNAL << 2) /* Write-lock walked vmas */
static struct kmem_cache *policy_cache;
static struct kmem_cache *sn_cache;
/* Highest zone. An specific allocation for a zone below that is not
policied. */
enum zone_type policy_zone = 0;
/*
* run-time system-wide default policy => local allocation
*/
static struct mempolicy default_policy = {
.refcnt = ATOMIC_INIT(1), /* never free it */
.mode = MPOL_LOCAL,
};
static struct mempolicy preferred_node_policy[MAX_NUMNODES];
/*
* iw_table is the sysfs-set interleave weight table, a value of 0 denotes
* system-default value should be used. A NULL iw_table also denotes that
* system-default values should be used. Until the system-default table
* is implemented, the system-default is always 1.
*
* iw_table is RCU protected
*/
static u8 __rcu *iw_table;
static DEFINE_MUTEX(iw_table_lock);
static u8 get_il_weight(int node)
{
u8 *table;
u8 weight;
rcu_read_lock();
table = rcu_dereference(iw_table);
/* if no iw_table, use system default */
weight = table ? table[node] : 1;
/* if value in iw_table is 0, use system default */
weight = weight ? weight : 1;
rcu_read_unlock();
return weight;
}
/**
* numa_nearest_node - Find nearest node by state
* @node: Node id to start the search
* @state: State to filter the search
*
* Lookup the closest node by distance if @nid is not in state.
*
* Return: this @node if it is in state, otherwise the closest node by distance
*/
int numa_nearest_node(int node, unsigned int state)
{
int min_dist = INT_MAX, dist, n, min_node;
if (state >= NR_NODE_STATES)
return -EINVAL;
if (node == NUMA_NO_NODE || node_state(node, state))
return node;
min_node = node;
for_each_node_state(n, state) {
dist = node_distance(node, n);
if (dist < min_dist) {
min_dist = dist;
min_node = n;
}
}
return min_node;
}
EXPORT_SYMBOL_GPL(numa_nearest_node);
struct mempolicy *get_task_policy(struct task_struct *p)
{
struct mempolicy *pol = p->mempolicy;
int node;
if (pol)
return pol;
node = numa_node_id();
if (node != NUMA_NO_NODE) {
pol = &preferred_node_policy[node];
/* preferred_node_policy is not initialised early in boot */
if (pol->mode)
return pol;
}
return &default_policy;
}
static const struct mempolicy_operations {
int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
} mpol_ops[MPOL_MAX];
static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
{
return pol->flags & MPOL_MODE_FLAGS;
}
static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
const nodemask_t *rel)
{
nodemask_t tmp;
nodes_fold(tmp, *orig, nodes_weight(*rel));
nodes_onto(*ret, tmp, *rel);
}
static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
{
if (nodes_empty(*nodes))
return -EINVAL;
pol->nodes = *nodes;
return 0;
}
static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
{
if (nodes_empty(*nodes))
return -EINVAL;
nodes_clear(pol->nodes);
node_set(first_node(*nodes), pol->nodes);
return 0;
}
/*
* mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
* any, for the new policy. mpol_new() has already validated the nodes
* parameter with respect to the policy mode and flags.
*
* Must be called holding task's alloc_lock to protect task's mems_allowed
* and mempolicy. May also be called holding the mmap_lock for write.
*/
static int mpol_set_nodemask(struct mempolicy *pol,
const nodemask_t *nodes, struct nodemask_scratch *nsc)
{
int ret;
/*
* Default (pol==NULL) resp. local memory policies are not a
* subject of any remapping. They also do not need any special
* constructor.
*/
if (!pol || pol->mode == MPOL_LOCAL)
return 0;
/* Check N_MEMORY */
nodes_and(nsc->mask1,
cpuset_current_mems_allowed, node_states[N_MEMORY]);
VM_BUG_ON(!nodes);
if (pol->flags & MPOL_F_RELATIVE_NODES)
mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
else
nodes_and(nsc->mask2, *nodes, nsc->mask1);
if (mpol_store_user_nodemask(pol))
pol->w.user_nodemask = *nodes;
else
pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed;
ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
return ret;
}
/*
* This function just creates a new policy, does some check and simple
* initialization. You must invoke mpol_set_nodemask() to set nodes.
*/
static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
nodemask_t *nodes)
{
struct mempolicy *policy;
if (mode == MPOL_DEFAULT) {
if (nodes && !nodes_empty(*nodes))
return ERR_PTR(-EINVAL);
return NULL;
}
VM_BUG_ON(!nodes);
/*
* MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
* MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
* All other modes require a valid pointer to a non-empty nodemask.
*/
if (mode == MPOL_PREFERRED) {
if (nodes_empty(*nodes)) {
if (((flags & MPOL_F_STATIC_NODES) ||
(flags & MPOL_F_RELATIVE_NODES)))
return ERR_PTR(-EINVAL);
mode = MPOL_LOCAL;
}
} else if (mode == MPOL_LOCAL) {
if (!nodes_empty(*nodes) ||
(flags & MPOL_F_STATIC_NODES) ||
(flags & MPOL_F_RELATIVE_NODES))
return ERR_PTR(-EINVAL);
} else if (nodes_empty(*nodes))
return ERR_PTR(-EINVAL);
policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
if (!policy)
return ERR_PTR(-ENOMEM);
atomic_set(&policy->refcnt, 1);
policy->mode = mode;
policy->flags = flags;
policy->home_node = NUMA_NO_NODE;
return policy;
}
/* Slow path of a mpol destructor. */
void __mpol_put(struct mempolicy *pol)
{
if (!atomic_dec_and_test(&pol->refcnt))
return;
kmem_cache_free(policy_cache, pol);
}
static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
{
}
static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
{
nodemask_t tmp;
if (pol->flags & MPOL_F_STATIC_NODES)
nodes_and(tmp, pol->w.user_nodemask, *nodes);
else if (pol->flags & MPOL_F_RELATIVE_NODES)
mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
else {
nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,
*nodes);
pol->w.cpuset_mems_allowed = *nodes;
}
if (nodes_empty(tmp))
tmp = *nodes;
pol->nodes = tmp;
}
static void mpol_rebind_preferred(struct mempolicy *pol,
const nodemask_t *nodes)
{
pol->w.cpuset_mems_allowed = *nodes;
}
/*
* mpol_rebind_policy - Migrate a policy to a different set of nodes
*
* Per-vma policies are protected by mmap_lock. Allocations using per-task
* policies are protected by task->mems_allowed_seq to prevent a premature
* OOM/allocation failure due to parallel nodemask modification.
*/
static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
{
if (!pol || pol->mode == MPOL_LOCAL)
return;
if (!mpol_store_user_nodemask(pol) &&
nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
return;
mpol_ops[pol->mode].rebind(pol, newmask);
}
/*
* Wrapper for mpol_rebind_policy() that just requires task
* pointer, and updates task mempolicy.
*
* Called with task's alloc_lock held.
*/
void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
{
mpol_rebind_policy(tsk->mempolicy, new);
}
/*
* Rebind each vma in mm to new nodemask.
*
* Call holding a reference to mm. Takes mm->mmap_lock during call.
*/
void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
{
struct vm_area_struct *vma;
VMA_ITERATOR(vmi, mm, 0);
mmap_write_lock(mm);
for_each_vma(vmi, vma) {
vma_start_write(vma);
mpol_rebind_policy(vma->vm_policy, new);
}
mmap_write_unlock(mm);
}
static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
[MPOL_DEFAULT] = {
.rebind = mpol_rebind_default,
},
[MPOL_INTERLEAVE] = {
.create = mpol_new_nodemask,
.rebind = mpol_rebind_nodemask,
},
[MPOL_PREFERRED] = {
.create = mpol_new_preferred,
.rebind = mpol_rebind_preferred,
},
[MPOL_BIND] = {
.create = mpol_new_nodemask,
.rebind = mpol_rebind_nodemask,
},
[MPOL_LOCAL] = {
.rebind = mpol_rebind_default,
},
[MPOL_PREFERRED_MANY] = {
.create = mpol_new_nodemask,
.rebind = mpol_rebind_preferred,
},
[MPOL_WEIGHTED_INTERLEAVE] = {
.create = mpol_new_nodemask,
.rebind = mpol_rebind_nodemask,
},
};
static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
unsigned long flags);
static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,
pgoff_t ilx, int *nid);
static bool strictly_unmovable(unsigned long flags)
{
/*
* STRICT without MOVE flags lets do_mbind() fail immediately with -EIO
* if any misplaced page is found.
*/
return (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ==
MPOL_MF_STRICT;
}
struct migration_mpol { /* for alloc_migration_target_by_mpol() */
struct mempolicy *pol;
pgoff_t ilx;
};
struct queue_pages {
struct list_head *pagelist;
unsigned long flags;
nodemask_t *nmask;
unsigned long start;
unsigned long end;
struct vm_area_struct *first;
struct folio *large; /* note last large folio encountered */
long nr_failed; /* could not be isolated at this time */
};
/*
* Check if the folio's nid is in qp->nmask.
*
* If MPOL_MF_INVERT is set in qp->flags, check if the nid is
* in the invert of qp->nmask.
*/
static inline bool queue_folio_required(struct folio *folio,
struct queue_pages *qp)
{
int nid = folio_nid(folio);
unsigned long flags = qp->flags;
return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
}
static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
{
struct folio *folio;
struct queue_pages *qp = walk->private;
if (unlikely(is_pmd_migration_entry(*pmd))) {
qp->nr_failed++;
return;
}
folio = pfn_folio(pmd_pfn(*pmd));
if (is_huge_zero_page(&folio->page)) {
walk->action = ACTION_CONTINUE;
return;
}
if (!queue_folio_required(folio, qp))
return;
if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
!vma_migratable(walk->vma) ||
!migrate_folio_add(folio, qp->pagelist, qp->flags))
qp->nr_failed++;
}
/*
* Scan through folios, checking if they satisfy the required conditions,
* moving them from LRU to local pagelist for migration if they do (or not).
*
* queue_folios_pte_range() has two possible return values:
* 0 - continue walking to scan for more, even if an existing folio on the
* wrong node could not be isolated and queued for migration.
* -EIO - only MPOL_MF_STRICT was specified, without MPOL_MF_MOVE or ..._ALL,
* and an existing folio was on a node that does not follow the policy.
*/
static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
unsigned long end, struct mm_walk *walk)
{
struct vm_area_struct *vma = walk->vma;
struct folio *folio;
struct queue_pages *qp = walk->private;
unsigned long flags = qp->flags;
pte_t *pte, *mapped_pte;
pte_t ptent;
spinlock_t *ptl;
ptl = pmd_trans_huge_lock(pmd, vma);
if (ptl) {
queue_folios_pmd(pmd, walk);
spin_unlock(ptl);
goto out;
}
mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
if (!pte) {
walk->action = ACTION_AGAIN;
return 0;
}
for (; addr != end; pte++, addr += PAGE_SIZE) {
ptent = ptep_get(pte);
if (pte_none(ptent))
continue;
if (!pte_present(ptent)) {
if (is_migration_entry(pte_to_swp_entry(ptent)))
qp->nr_failed++;
continue;
}
folio = vm_normal_folio(vma, addr, ptent);
if (!folio || folio_is_zone_device(folio))
continue;
/*
* vm_normal_folio() filters out zero pages, but there might
* still be reserved folios to skip, perhaps in a VDSO.
*/
if (folio_test_reserved(folio))
continue;
if (!queue_folio_required(folio, qp))
continue;
if (folio_test_large(folio)) {
/*
* A large folio can only be isolated from LRU once,
* but may be mapped by many PTEs (and Copy-On-Write may
* intersperse PTEs of other, order 0, folios). This is
* a common case, so don't mistake it for failure (but
* there can be other cases of multi-mapped pages which
* this quick check does not help to filter out - and a
* search of the pagelist might grow to be prohibitive).
*
* migrate_pages(&pagelist) returns nr_failed folios, so
* check "large" now so that queue_pages_range() returns
* a comparable nr_failed folios. This does imply that
* if folio could not be isolated for some racy reason
* at its first PTE, later PTEs will not give it another
* chance of isolation; but keeps the accounting simple.
*/
if (folio == qp->large)
continue;
qp->large = folio;
}
if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
!vma_migratable(vma) ||
!migrate_folio_add(folio, qp->pagelist, flags)) {
qp->nr_failed++;
if (strictly_unmovable(flags))
break;
}
}
pte_unmap_unlock(mapped_pte, ptl);
cond_resched();
out:
if (qp->nr_failed && strictly_unmovable(flags))
return -EIO;
return 0;
}
static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,
unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
#ifdef CONFIG_HUGETLB_PAGE
struct queue_pages *qp = walk->private;
unsigned long flags = qp->flags;
struct folio *folio;
spinlock_t *ptl;
pte_t entry;
ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
entry = huge_ptep_get(pte);
if (!pte_present(entry)) {
if (unlikely(is_hugetlb_entry_migration(entry)))
qp->nr_failed++;
goto unlock;
}
folio = pfn_folio(pte_pfn(entry));
if (!queue_folio_required(folio, qp))
goto unlock;
if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
!vma_migratable(walk->vma)) {
qp->nr_failed++;
goto unlock;
}
/*
* Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.
* Choosing not to migrate a shared folio is not counted as a failure.
*
* To check if the folio is shared, ideally we want to make sure
* every page is mapped to the same process. Doing that is very
* expensive, so check the estimated sharers of the folio instead.
*/
if ((flags & MPOL_MF_MOVE_ALL) ||
(folio_estimated_sharers(folio) == 1 && !hugetlb_pmd_shared(pte)))
if (!isolate_hugetlb(folio, qp->pagelist))
qp->nr_failed++;
unlock:
spin_unlock(ptl);
if (qp->nr_failed && strictly_unmovable(flags))
return -EIO;
#endif
return 0;
}
#ifdef CONFIG_NUMA_BALANCING
/*
* This is used to mark a range of virtual addresses to be inaccessible.
* These are later cleared by a NUMA hinting fault. Depending on these
* faults, pages may be migrated for better NUMA placement.
*
* This is assuming that NUMA faults are handled using PROT_NONE. If
* an architecture makes a different choice, it will need further
* changes to the core.
*/
unsigned long change_prot_numa(struct vm_area_struct *vma,
unsigned long addr, unsigned long end)
{
struct mmu_gather tlb;
long nr_updated;
tlb_gather_mmu(&tlb, vma->vm_mm);
nr_updated = change_protection(&tlb, vma, addr, end, MM_CP_PROT_NUMA);
if (nr_updated > 0)
count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
tlb_finish_mmu(&tlb);
return nr_updated;
}
#endif /* CONFIG_NUMA_BALANCING */
static int queue_pages_test_walk(unsigned long start, unsigned long end,
struct mm_walk *walk)
{
struct vm_area_struct *next, *vma = walk->vma;
struct queue_pages *qp = walk->private;
unsigned long flags = qp->flags;
/* range check first */
VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma);
if (!qp->first) {
qp->first = vma;
if (!(flags & MPOL_MF_DISCONTIG_OK) &&
(qp->start < vma->vm_start))
/* hole at head side of range */
return -EFAULT;
}
next = find_vma(vma->vm_mm, vma->vm_end);
if (!(flags & MPOL_MF_DISCONTIG_OK) &&
((vma->vm_end < qp->end) &&
(!next || vma->vm_end < next->vm_start)))
/* hole at middle or tail of range */
return -EFAULT;
/*
* Need check MPOL_MF_STRICT to return -EIO if possible
* regardless of vma_migratable
*/
if (!vma_migratable(vma) &&
!(flags & MPOL_MF_STRICT))
return 1;
/*
* Check page nodes, and queue pages to move, in the current vma.
* But if no moving, and no strict checking, the scan can be skipped.
*/
if (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
return 0;
return 1;
}
static const struct mm_walk_ops queue_pages_walk_ops = {
.hugetlb_entry = queue_folios_hugetlb,
.pmd_entry = queue_folios_pte_range,
.test_walk = queue_pages_test_walk,
.walk_lock = PGWALK_RDLOCK,
};
static const struct mm_walk_ops queue_pages_lock_vma_walk_ops = {
.hugetlb_entry = queue_folios_hugetlb,
.pmd_entry = queue_folios_pte_range,
.test_walk = queue_pages_test_walk,
.walk_lock = PGWALK_WRLOCK,
};
/*
* Walk through page tables and collect pages to be migrated.
*
* If pages found in a given range are not on the required set of @nodes,
* and migration is allowed, they are isolated and queued to @pagelist.
*
* queue_pages_range() may return:
* 0 - all pages already on the right node, or successfully queued for moving
* (or neither strict checking nor moving requested: only range checking).
* >0 - this number of misplaced folios could not be queued for moving
* (a hugetlbfs page or a transparent huge page being counted as 1).
* -EIO - a misplaced page found, when MPOL_MF_STRICT specified without MOVEs.
* -EFAULT - a hole in the memory range, when MPOL_MF_DISCONTIG_OK unspecified.
*/
static long
queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
nodemask_t *nodes, unsigned long flags,
struct list_head *pagelist)
{
int err;
struct queue_pages qp = {
.pagelist = pagelist,
.flags = flags,
.nmask = nodes,
.start = start,
.end = end,
.first = NULL,
};
const struct mm_walk_ops *ops = (flags & MPOL_MF_WRLOCK) ?
&queue_pages_lock_vma_walk_ops : &queue_pages_walk_ops;
err = walk_page_range(mm, start, end, ops, &qp);
if (!qp.first)
/* whole range in hole */
err = -EFAULT;
return err ? : qp.nr_failed;
}
/*
* Apply policy to a single VMA
* This must be called with the mmap_lock held for writing.
*/
static int vma_replace_policy(struct vm_area_struct *vma,
struct mempolicy *pol)
{
int err;
struct mempolicy *old;
struct mempolicy *new;
vma_assert_write_locked(vma);
new = mpol_dup(pol);
if (IS_ERR(new))
return PTR_ERR(new);
if (vma->vm_ops && vma->vm_ops->set_policy) {
err = vma->vm_ops->set_policy(vma, new);
if (err)
goto err_out;
}
old = vma->vm_policy;
vma->vm_policy = new; /* protected by mmap_lock */
mpol_put(old);
return 0;
err_out:
mpol_put(new);
return err;
}
/* Split or merge the VMA (if required) and apply the new policy */
static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma,
struct vm_area_struct **prev, unsigned long start,
unsigned long end, struct mempolicy *new_pol)
{
unsigned long vmstart, vmend;
vmend = min(end, vma->vm_end);
if (start > vma->vm_start) {
*prev = vma;
vmstart = start;
} else {
vmstart = vma->vm_start;
}
if (mpol_equal(vma->vm_policy, new_pol)) {
*prev = vma;
return 0;
}
vma = vma_modify_policy(vmi, *prev, vma, vmstart, vmend, new_pol);
if (IS_ERR(vma))
return PTR_ERR(vma);
*prev = vma;
return vma_replace_policy(vma, new_pol);
}
/* Set the process memory policy */
static long do_set_mempolicy(unsigned short mode, unsigned short flags,
nodemask_t *nodes)
{
struct mempolicy *new, *old;
NODEMASK_SCRATCH(scratch);
int ret;
if (!scratch)
return -ENOMEM;
new = mpol_new(mode, flags, nodes);
if (IS_ERR(new)) {
ret = PTR_ERR(new);
goto out;
}
task_lock(current);
ret = mpol_set_nodemask(new, nodes, scratch);
if (ret) {
task_unlock(current);
mpol_put(new);
goto out;
}
old = current->mempolicy;
current->mempolicy = new;
if (new && (new->mode == MPOL_INTERLEAVE ||
new->mode == MPOL_WEIGHTED_INTERLEAVE)) {
current->il_prev = MAX_NUMNODES-1;
current->il_weight = 0;
}
task_unlock(current);
mpol_put(old);
ret = 0;
out:
NODEMASK_SCRATCH_FREE(scratch);
return ret;
}
/*
* Return nodemask for policy for get_mempolicy() query
*
* Called with task's alloc_lock held
*/
static void get_policy_nodemask(struct mempolicy *pol, nodemask_t *nodes)
{
nodes_clear(*nodes);
if (pol == &default_policy)
return;
switch (pol->mode) {
case MPOL_BIND:
case MPOL_INTERLEAVE:
case MPOL_PREFERRED:
case MPOL_PREFERRED_MANY:
case MPOL_WEIGHTED_INTERLEAVE:
*nodes = pol->nodes;
break;
case MPOL_LOCAL:
/* return empty node mask for local allocation */
break;
default:
BUG();
}
}
static int lookup_node(struct mm_struct *mm, unsigned long addr)
{
struct page *p = NULL;
int ret;
ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p);
if (ret > 0) {
ret = page_to_nid(p);
put_page(p);
}
return ret;
}
/* Retrieve NUMA policy */
static long do_get_mempolicy(int *policy, nodemask_t *nmask,
unsigned long addr, unsigned long flags)
{
int err;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma = NULL;
struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;
if (flags &
~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
return -EINVAL;
if (flags & MPOL_F_MEMS_ALLOWED) {
if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
return -EINVAL;
*policy = 0; /* just so it's initialized */
task_lock(current);
*nmask = cpuset_current_mems_allowed;
task_unlock(current);
return 0;
}
if (flags & MPOL_F_ADDR) {
pgoff_t ilx; /* ignored here */
/*
* Do NOT fall back to task policy if the
* vma/shared policy at addr is NULL. We
* want to return MPOL_DEFAULT in this case.
*/
mmap_read_lock(mm);
vma = vma_lookup(mm, addr);
if (!vma) {
mmap_read_unlock(mm);
return -EFAULT;
}
pol = __get_vma_policy(vma, addr, &ilx);
} else if (addr)
return -EINVAL;
if (!pol)
pol = &default_policy; /* indicates default behavior */
if (flags & MPOL_F_NODE) {
if (flags & MPOL_F_ADDR) {
/*
* Take a refcount on the mpol, because we are about to
* drop the mmap_lock, after which only "pol" remains
* valid, "vma" is stale.
*/
pol_refcount = pol;
vma = NULL;
mpol_get(pol);
mmap_read_unlock(mm);
err = lookup_node(mm, addr);
if (err < 0)
goto out;
*policy = err;
} else if (pol == current->mempolicy &&
pol->mode == MPOL_INTERLEAVE) {
*policy = next_node_in(current->il_prev, pol->nodes);
} else if (pol == current->mempolicy &&
pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
if (current->il_weight)
*policy = current->il_prev;
else
*policy = next_node_in(current->il_prev,
pol->nodes);
} else {
err = -EINVAL;
goto out;
}
} else {
*policy = pol == &default_policy ? MPOL_DEFAULT :
pol->mode;
/*
* Internal mempolicy flags must be masked off before exposing
* the policy to userspace.
*/
*policy |= (pol->flags & MPOL_MODE_FLAGS);
}
err = 0;
if (nmask) {
if (mpol_store_user_nodemask(pol)) {
*nmask = pol->w.user_nodemask;
} else {
task_lock(current);
get_policy_nodemask(pol, nmask);
task_unlock(current);
}
}
out:
mpol_cond_put(pol);
if (vma)
mmap_read_unlock(mm);
if (pol_refcount)
mpol_put(pol_refcount);
return err;
}
#ifdef CONFIG_MIGRATION
static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
unsigned long flags)
{
/*
* Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.
* Choosing not to migrate a shared folio is not counted as a failure.
*
* To check if the folio is shared, ideally we want to make sure
* every page is mapped to the same process. Doing that is very
* expensive, so check the estimated sharers of the folio instead.
*/
if ((flags & MPOL_MF_MOVE_ALL) || folio_estimated_sharers(folio) == 1) {
if (folio_isolate_lru(folio)) {
list_add_tail(&folio->lru, foliolist);
node_stat_mod_folio(folio,
NR_ISOLATED_ANON + folio_is_file_lru(folio),
folio_nr_pages(folio));
} else {
/*
* Non-movable folio may reach here. And, there may be
* temporary off LRU folios or non-LRU movable folios.
* Treat them as unmovable folios since they can't be
* isolated, so they can't be moved at the moment.
*/
return false;
}
}
return true;
}
/*
* Migrate pages from one node to a target node.
* Returns error or the number of pages not migrated.
*/
static long migrate_to_node(struct mm_struct *mm, int source, int dest,
int flags)
{
nodemask_t nmask;
struct vm_area_struct *vma;
LIST_HEAD(pagelist);
long nr_failed;
long err = 0;
struct migration_target_control mtc = {
.nid = dest,
.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
};
nodes_clear(nmask);
node_set(source, nmask);
VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
mmap_read_lock(mm);
vma = find_vma(mm, 0);
/*
* This does not migrate the range, but isolates all pages that
* need migration. Between passing in the full user address
* space range and MPOL_MF_DISCONTIG_OK, this call cannot fail,
* but passes back the count of pages which could not be isolated.
*/
nr_failed = queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
flags | MPOL_MF_DISCONTIG_OK, &pagelist);
mmap_read_unlock(mm);
if (!list_empty(&pagelist)) {
err = migrate_pages(&pagelist, alloc_migration_target, NULL,
(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
if (err)
putback_movable_pages(&pagelist);
}
if (err >= 0)
err += nr_failed;
return err;
}
/*
* Move pages between the two nodesets so as to preserve the physical
* layout as much as possible.
*
* Returns the number of page that could not be moved.
*/
int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
const nodemask_t *to, int flags)
{
long nr_failed = 0;
long err = 0;
nodemask_t tmp;
lru_cache_disable();
/*
* Find a 'source' bit set in 'tmp' whose corresponding 'dest'
* bit in 'to' is not also set in 'tmp'. Clear the found 'source'
* bit in 'tmp', and return that <source, dest> pair for migration.
* The pair of nodemasks 'to' and 'from' define the map.
*
* If no pair of bits is found that way, fallback to picking some
* pair of 'source' and 'dest' bits that are not the same. If the
* 'source' and 'dest' bits are the same, this represents a node
* that will be migrating to itself, so no pages need move.
*
* If no bits are left in 'tmp', or if all remaining bits left
* in 'tmp' correspond to the same bit in 'to', return false
* (nothing left to migrate).
*
* This lets us pick a pair of nodes to migrate between, such that
* if possible the dest node is not already occupied by some other
* source node, minimizing the risk of overloading the memory on a
* node that would happen if we migrated incoming memory to a node
* before migrating outgoing memory source that same node.
*
* A single scan of tmp is sufficient. As we go, we remember the
* most recent <s, d> pair that moved (s != d). If we find a pair
* that not only moved, but what's better, moved to an empty slot
* (d is not set in tmp), then we break out then, with that pair.
* Otherwise when we finish scanning from_tmp, we at least have the
* most recent <s, d> pair that moved. If we get all the way through
* the scan of tmp without finding any node that moved, much less
* moved to an empty node, then there is nothing left worth migrating.
*/
tmp = *from;
while (!nodes_empty(tmp)) {
int s, d;
int source = NUMA_NO_NODE;
int dest = 0;
for_each_node_mask(s, tmp) {
/*
* do_migrate_pages() tries to maintain the relative
* node relationship of the pages established between
* threads and memory areas.
*
* However if the number of source nodes is not equal to
* the number of destination nodes we can not preserve
* this node relative relationship. In that case, skip
* copying memory from a node that is in the destination
* mask.
*
* Example: [2,3,4] -> [3,4,5] moves everything.
* [0-7] - > [3,4,5] moves only 0,1,2,6,7.
*/
if ((nodes_weight(*from) != nodes_weight(*to)) &&
(node_isset(s, *to)))
continue;
d = node_remap(s, *from, *to);
if (s == d)
continue;
source = s; /* Node moved. Memorize */
dest = d;
/* dest not in remaining from nodes? */
if (!node_isset(dest, tmp))
break;
}
if (source == NUMA_NO_NODE)
break;
node_clear(source, tmp);
err = migrate_to_node(mm, source, dest, flags);
if (err > 0)
nr_failed += err;
if (err < 0)
break;
}
lru_cache_enable();
if (err < 0)
return err;
return (nr_failed < INT_MAX) ? nr_failed : INT_MAX;
}
/*
* Allocate a new folio for page migration, according to NUMA mempolicy.
*/
static struct folio *alloc_migration_target_by_mpol(struct folio *src,
unsigned long private)
{
struct migration_mpol *mmpol = (struct migration_mpol *)private;
struct mempolicy *pol = mmpol->pol;
pgoff_t ilx = mmpol->ilx;
struct page *page;
unsigned int order;
int nid = numa_node_id();
gfp_t gfp;
order = folio_order(src);
ilx += src->index >> order;
if (folio_test_hugetlb(src)) {
nodemask_t *nodemask;
struct hstate *h;
h = folio_hstate(src);
gfp = htlb_alloc_mask(h);
nodemask = policy_nodemask(gfp, pol, ilx, &nid);
return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp);
}
if (folio_test_large(src))
gfp = GFP_TRANSHUGE;
else
gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL | __GFP_COMP;
page = alloc_pages_mpol(gfp, order, pol, ilx, nid);
return page_rmappable_folio(page);
}
#else
static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
unsigned long flags)
{
return false;
}
int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
const nodemask_t *to, int flags)
{
return -ENOSYS;
}
static struct folio *alloc_migration_target_by_mpol(struct folio *src,
unsigned long private)
{
return NULL;
}
#endif
static long do_mbind(unsigned long start, unsigned long len,
unsigned short mode, unsigned short mode_flags,
nodemask_t *nmask, unsigned long flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma, *prev;
struct vma_iterator vmi;
struct migration_mpol mmpol;
struct mempolicy *new;
unsigned long end;
long err;
long nr_failed;
LIST_HEAD(pagelist);
if (flags & ~(unsigned long)MPOL_MF_VALID)
return -EINVAL;
if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
return -EPERM;
if (start & ~PAGE_MASK)
return -EINVAL;
if (mode == MPOL_DEFAULT)
flags &= ~MPOL_MF_STRICT;
len = PAGE_ALIGN(len);
end = start + len;
if (end < start)
return -EINVAL;
if (end == start)
return 0;
new = mpol_new(mode, mode_flags, nmask);
if (IS_ERR(new))
return PTR_ERR(new);
/*
* If we are using the default policy then operation
* on discontinuous address spaces is okay after all
*/
if (!new)
flags |= MPOL_MF_DISCONTIG_OK;
if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
lru_cache_disable();
{
NODEMASK_SCRATCH(scratch);
if (scratch) {
mmap_write_lock(mm);
err = mpol_set_nodemask(new, nmask, scratch);
if (err)
mmap_write_unlock(mm);
} else
err = -ENOMEM;
NODEMASK_SCRATCH_FREE(scratch);
}
if (err)
goto mpol_out;
/*
* Lock the VMAs before scanning for pages to migrate,
* to ensure we don't miss a concurrently inserted page.
*/
nr_failed = queue_pages_range(mm, start, end, nmask,
flags | MPOL_MF_INVERT | MPOL_MF_WRLOCK, &pagelist);
if (nr_failed < 0) {
err = nr_failed;
nr_failed = 0;
} else {
vma_iter_init(&vmi, mm, start);
prev = vma_prev(&vmi);
for_each_vma_range(vmi, vma, end) {
err = mbind_range(&vmi, vma, &prev, start, end, new);
if (err)
break;
}
}
if (!err && !list_empty(&pagelist)) {
/* Convert MPOL_DEFAULT's NULL to task or default policy */
if (!new) {
new = get_task_policy(current);
mpol_get(new);
}
mmpol.pol = new;
mmpol.ilx = 0;
/*
* In the interleaved case, attempt to allocate on exactly the
* targeted nodes, for the first VMA to be migrated; for later
* VMAs, the nodes will still be interleaved from the targeted
* nodemask, but one by one may be selected differently.
*/
if (new->mode == MPOL_INTERLEAVE ||
new->mode == MPOL_WEIGHTED_INTERLEAVE) {
struct folio *folio;
unsigned int order;
unsigned long addr = -EFAULT;
list_for_each_entry(folio, &pagelist, lru) {
if (!folio_test_ksm(folio))
break;
}
if (!list_entry_is_head(folio, &pagelist, lru)) {
vma_iter_init(&vmi, mm, start);
for_each_vma_range(vmi, vma, end) {
addr = page_address_in_vma(
folio_page(folio, 0), vma);
if (addr != -EFAULT)
break;
}
}
if (addr != -EFAULT) {
order = folio_order(folio);
/* We already know the pol, but not the ilx */
mpol_cond_put(get_vma_policy(vma, addr, order,
&mmpol.ilx));
/* Set base from which to increment by index */
mmpol.ilx -= folio->index >> order;
}
}
}
mmap_write_unlock(mm);
if (!err && !list_empty(&pagelist)) {
nr_failed |= migrate_pages(&pagelist,
alloc_migration_target_by_mpol, NULL,
(unsigned long)&mmpol, MIGRATE_SYNC,
MR_MEMPOLICY_MBIND, NULL);
}
if (nr_failed && (flags & MPOL_MF_STRICT))
err = -EIO;
if (!list_empty(&pagelist))
putback_movable_pages(&pagelist);
mpol_out:
mpol_put(new);
if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
lru_cache_enable();
return err;
}
/*
* User space interface with variable sized bitmaps for nodelists.
*/
static int get_bitmap(unsigned long *mask, const unsigned long __user *nmask,
unsigned long maxnode)
{
unsigned long nlongs = BITS_TO_LONGS(maxnode);
int ret;
if (in_compat_syscall())
ret = compat_get_bitmap(mask,
(const compat_ulong_t __user *)nmask,
maxnode);
else
ret = copy_from_user(mask, nmask,
nlongs * sizeof(unsigned long));
if (ret)
return -EFAULT;
if (maxnode % BITS_PER_LONG)
mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1;
return 0;
}
/* Copy a node mask from user space. */
static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
unsigned long maxnode)
{
--maxnode;
nodes_clear(*nodes);
if (maxnode == 0 || !nmask)
return 0;
if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
return -EINVAL;
/*
* When the user specified more nodes than supported just check
* if the non supported part is all zero, one word at a time,
* starting at the end.
*/
while (maxnode > MAX_NUMNODES) {
unsigned long bits = min_t(unsigned long, maxnode, BITS_PER_LONG);
unsigned long t;
if (get_bitmap(&t, &nmask[(maxnode - 1) / BITS_PER_LONG], bits))
return -EFAULT;
if (maxnode - bits >= MAX_NUMNODES) {
maxnode -= bits;
} else {
maxnode = MAX_NUMNODES;
t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1);
}
if (t)
return -EINVAL;
}
return get_bitmap(nodes_addr(*nodes), nmask, maxnode);
}
/* Copy a kernel node mask to user space */
static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
nodemask_t *nodes)
{
unsigned long copy = ALIGN(maxnode-1, 64) / 8;
unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
bool compat = in_compat_syscall();
if (compat)
nbytes = BITS_TO_COMPAT_LONGS(nr_node_ids) * sizeof(compat_long_t);
if (copy > nbytes) {
if (copy > PAGE_SIZE)
return -EINVAL;
if (clear_user((char __user *)mask + nbytes, copy - nbytes))
return -EFAULT;
copy = nbytes;
maxnode = nr_node_ids;
}
if (compat)
return compat_put_bitmap((compat_ulong_t __user *)mask,
nodes_addr(*nodes), maxnode);
return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
}
/* Basic parameter sanity check used by both mbind() and set_mempolicy() */
static inline int sanitize_mpol_flags(int *mode, unsigned short *flags)
{
*flags = *mode & MPOL_MODE_FLAGS;
*mode &= ~MPOL_MODE_FLAGS;
if ((unsigned int)(*mode) >= MPOL_MAX)
return -EINVAL;
if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES))
return -EINVAL;
if (*flags & MPOL_F_NUMA_BALANCING) {
if (*mode != MPOL_BIND)
return -EINVAL;
*flags |= (MPOL_F_MOF | MPOL_F_MORON);
}
return 0;
}
static long kernel_mbind(unsigned long start, unsigned long len,
unsigned long mode, const unsigned long __user *nmask,
unsigned long maxnode, unsigned int flags)
{
unsigned short mode_flags;
nodemask_t nodes;
int lmode = mode;
int err;
start = untagged_addr(start);
err = sanitize_mpol_flags(&lmode, &mode_flags);
if (err)
return err;
err = get_nodes(&nodes, nmask, maxnode);
if (err)
return err;
return do_mbind(start, len, lmode, mode_flags, &nodes, flags);
}
SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len,
unsigned long, home_node, unsigned long, flags)
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma, *prev;
struct mempolicy *new, *old;
unsigned long end;
int err = -ENOENT;
VMA_ITERATOR(vmi, mm, start);
start = untagged_addr(start);
if (start & ~PAGE_MASK)
return -EINVAL;
/*
* flags is used for future extension if any.
*/
if (flags != 0)
return -EINVAL;
/*
* Check home_node is online to avoid accessing uninitialized
* NODE_DATA.
*/
if (home_node >= MAX_NUMNODES || !node_online(home_node))
return -EINVAL;
len = PAGE_ALIGN(len);
end = start + len;
if (end < start)
return -EINVAL;
if (end == start)
return 0;
mmap_write_lock(mm);
prev = vma_prev(&vmi);
for_each_vma_range(vmi, vma, end) {
/*
* If any vma in the range got policy other than MPOL_BIND
* or MPOL_PREFERRED_MANY we return error. We don't reset
* the home node for vmas we already updated before.
*/
old = vma_policy(vma);
if (!old) {
prev = vma;
continue;
}
if (old->mode != MPOL_BIND && old->mode != MPOL_PREFERRED_MANY) {
err = -EOPNOTSUPP;
break;
}
new = mpol_dup(old);
if (IS_ERR(new)) {
err = PTR_ERR(new);
break;
}
vma_start_write(vma);
new->home_node = home_node;
err = mbind_range(&vmi, vma, &prev, start, end, new);
mpol_put(new);
if (err)
break;
}
mmap_write_unlock(mm);
return err;
}
SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
unsigned long, mode, const unsigned long __user *, nmask,
unsigned long, maxnode, unsigned int, flags)
{
return kernel_mbind(start, len, mode, nmask, maxnode, flags);
}
/* Set the process memory policy */
static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
unsigned long maxnode)
{
unsigned short mode_flags;
nodemask_t nodes;
int lmode = mode;
int err;
err = sanitize_mpol_flags(&lmode, &mode_flags);
if (err)
return err;
err = get_nodes(&nodes, nmask, maxnode);
if (err)
return err;
return do_set_mempolicy(lmode, mode_flags, &nodes);
}
SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
unsigned long, maxnode)
{
return kernel_set_mempolicy(mode, nmask, maxnode);
}
static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
const unsigned long __user *old_nodes,
const unsigned long __user *new_nodes)
{
struct mm_struct *mm = NULL;
struct task_struct *task;
nodemask_t task_nodes;
int err;
nodemask_t *old;
nodemask_t *new;
NODEMASK_SCRATCH(scratch);
if (!scratch)
return -ENOMEM;
old = &scratch->mask1;
new = &scratch->mask2;
err = get_nodes(old, old_nodes, maxnode);
if (err)
goto out;
err = get_nodes(new, new_nodes, maxnode);
if (err)
goto out;
/* Find the mm_struct */
rcu_read_lock();
task = pid ? find_task_by_vpid(pid) : current;
if (!task) {
rcu_read_unlock();
err = -ESRCH;
goto out;
}
get_task_struct(task);
err = -EINVAL;
/*
* Check if this process has the right to modify the specified process.
* Use the regular "ptrace_may_access()" checks.
*/
if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
rcu_read_unlock();
err = -EPERM;
goto out_put;
}
rcu_read_unlock();
task_nodes = cpuset_mems_allowed(task);
/* Is the user allowed to access the target nodes? */
if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
err = -EPERM;
goto out_put;
}
task_nodes = cpuset_mems_allowed(current);
nodes_and(*new, *new, task_nodes);
if (nodes_empty(*new))
goto out_put;
err = security_task_movememory(task);
if (err)
goto out_put;
mm = get_task_mm(task);
put_task_struct(task);
if (!mm) {
err = -EINVAL;
goto out;
}
err = do_migrate_pages(mm, old, new,
capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
mmput(mm);
out:
NODEMASK_SCRATCH_FREE(scratch);
return err;
out_put:
put_task_struct(task);
goto out;
}
SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
const unsigned long __user *, old_nodes,
const unsigned long __user *, new_nodes)
{
return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes);
}
/* Retrieve NUMA policy */
static int kernel_get_mempolicy(int __user *policy,
unsigned long __user *nmask,
unsigned long maxnode,
unsigned long addr,
unsigned long flags)
{
int err;
int pval;
nodemask_t nodes;
if (nmask != NULL && maxnode < nr_node_ids)
return -EINVAL;
addr = untagged_addr(addr);
err = do_get_mempolicy(&pval, &nodes, addr, flags);
if (err)
return err;
if (policy && put_user(pval, policy))
return -EFAULT;
if (nmask)
err = copy_nodes_to_user(nmask, maxnode, &nodes);
return err;
}
SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
unsigned long __user *, nmask, unsigned long, maxnode,
unsigned long, addr, unsigned long, flags)
{
return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags);
}
bool vma_migratable(struct vm_area_struct *vma)
{
if (vma->vm_flags & (VM_IO | VM_PFNMAP))
return false;
/*
* DAX device mappings require predictable access latency, so avoid
* incurring periodic faults.
*/
if (vma_is_dax(vma))
return false;
if (is_vm_hugetlb_page(vma) &&
!hugepage_migration_supported(hstate_vma(vma)))
return false;
/*
* Migration allocates pages in the highest zone. If we cannot
* do so then migration (at least from node to node) is not
* possible.
*/
if (vma->vm_file &&
gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping))
< policy_zone)
return false;
return true;
}
struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
unsigned long addr, pgoff_t *ilx)
{
*ilx = 0; return (vma->vm_ops && vma->vm_ops->get_policy) ? vma->vm_ops->get_policy(vma, addr, ilx) : vma->vm_policy;
}
/*
* get_vma_policy(@vma, @addr, @order, @ilx)
* @vma: virtual memory area whose policy is sought
* @addr: address in @vma for shared policy lookup
* @order: 0, or appropriate huge_page_order for interleaving
* @ilx: interleave index (output), for use only when MPOL_INTERLEAVE or
* MPOL_WEIGHTED_INTERLEAVE
*
* Returns effective policy for a VMA at specified address.
* Falls back to current->mempolicy or system default policy, as necessary.
* Shared policies [those marked as MPOL_F_SHARED] require an extra reference
* count--added by the get_policy() vm_op, as appropriate--to protect against
* freeing by another task. It is the caller's responsibility to free the
* extra reference for shared policies.
*/
struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
unsigned long addr, int order, pgoff_t *ilx)
{
struct mempolicy *pol;
pol = __get_vma_policy(vma, addr, ilx); if (!pol) pol = get_task_policy(current); if (pol->mode == MPOL_INTERLEAVE ||
pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
*ilx += vma->vm_pgoff >> order; *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order);
}
return pol;
}
bool vma_policy_mof(struct vm_area_struct *vma)
{
struct mempolicy *pol;
if (vma->vm_ops && vma->vm_ops->get_policy) {
bool ret = false;
pgoff_t ilx; /* ignored here */
pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx);
if (pol && (pol->flags & MPOL_F_MOF))
ret = true;
mpol_cond_put(pol);
return ret;
}
pol = vma->vm_policy;
if (!pol)
pol = get_task_policy(current);
return pol->flags & MPOL_F_MOF;
}
bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
{
enum zone_type dynamic_policy_zone = policy_zone; BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
/*
* if policy->nodes has movable memory only,
* we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
*
* policy->nodes is intersect with node_states[N_MEMORY].
* so if the following test fails, it implies
* policy->nodes has movable memory only.
*/
if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))
dynamic_policy_zone = ZONE_MOVABLE;
return zone >= dynamic_policy_zone;
}
static unsigned int weighted_interleave_nodes(struct mempolicy *policy)
{
unsigned int node;
unsigned int cpuset_mems_cookie;
retry:
/* to prevent miscount use tsk->mems_allowed_seq to detect rebind */
cpuset_mems_cookie = read_mems_allowed_begin();
node = current->il_prev;
if (!current->il_weight || !node_isset(node, policy->nodes)) {
node = next_node_in(node, policy->nodes);
if (read_mems_allowed_retry(cpuset_mems_cookie))
goto retry;
if (node == MAX_NUMNODES)
return node;
current->il_prev = node;
current->il_weight = get_il_weight(node);
}
current->il_weight--;
return node;
}
/* Do dynamic interleaving for a process */
static unsigned int interleave_nodes(struct mempolicy *policy)
{
unsigned int nid;
unsigned int cpuset_mems_cookie;
/* to prevent miscount, use tsk->mems_allowed_seq to detect rebind */
do {
cpuset_mems_cookie = read_mems_allowed_begin();
nid = next_node_in(current->il_prev, policy->nodes);
} while (read_mems_allowed_retry(cpuset_mems_cookie));
if (nid < MAX_NUMNODES)
current->il_prev = nid;
return nid;
}
/*
* Depending on the memory policy provide a node from which to allocate the
* next slab entry.
*/
unsigned int mempolicy_slab_node(void)
{
struct mempolicy *policy;
int node = numa_mem_id();
if (!in_task())
return node;
policy = current->mempolicy;
if (!policy)
return node;
switch (policy->mode) {
case MPOL_PREFERRED:
return first_node(policy->nodes);
case MPOL_INTERLEAVE:
return interleave_nodes(policy);
case MPOL_WEIGHTED_INTERLEAVE:
return weighted_interleave_nodes(policy);
case MPOL_BIND:
case MPOL_PREFERRED_MANY:
{
struct zoneref *z;
/*
* Follow bind policy behavior and start allocation at the
* first node.
*/
struct zonelist *zonelist;
enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
z = first_zones_zonelist(zonelist, highest_zoneidx,
&policy->nodes);
return z->zone ? zone_to_nid(z->zone) : node;
}
case MPOL_LOCAL:
return node;
default:
BUG();
}
}
static unsigned int read_once_policy_nodemask(struct mempolicy *pol,
nodemask_t *mask)
{
/*
* barrier stabilizes the nodemask locally so that it can be iterated
* over safely without concern for changes. Allocators validate node
* selection does not violate mems_allowed, so this is safe.
*/
barrier();
memcpy(mask, &pol->nodes, sizeof(nodemask_t));
barrier();
return nodes_weight(*mask);
}
static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx)
{
nodemask_t nodemask;
unsigned int target, nr_nodes;
u8 *table;
unsigned int weight_total = 0;
u8 weight;
int nid;
nr_nodes = read_once_policy_nodemask(pol, &nodemask);
if (!nr_nodes)
return numa_node_id();
rcu_read_lock();
table = rcu_dereference(iw_table);
/* calculate the total weight */
for_each_node_mask(nid, nodemask) {
/* detect system default usage */
weight = table ? table[nid] : 1;
weight = weight ? weight : 1;
weight_total += weight;
}
/* Calculate the node offset based on totals */
target = ilx % weight_total;
nid = first_node(nodemask);
while (target) {
/* detect system default usage */
weight = table ? table[nid] : 1;
weight = weight ? weight : 1;
if (target < weight)
break;
target -= weight;
nid = next_node_in(nid, nodemask);
}
rcu_read_unlock();
return nid;
}
/*
* Do static interleaving for interleave index @ilx. Returns the ilx'th
* node in pol->nodes (starting from ilx=0), wrapping around if ilx
* exceeds the number of present nodes.
*/
static unsigned int interleave_nid(struct mempolicy *pol, pgoff_t ilx)
{
nodemask_t nodemask;
unsigned int target, nnodes;
int i;
int nid;
nnodes = read_once_policy_nodemask(pol, &nodemask);
if (!nnodes)
return numa_node_id();
target = ilx % nnodes;
nid = first_node(nodemask);
for (i = 0; i < target; i++)
nid = next_node(nid, nodemask);
return nid;
}
/*
* Return a nodemask representing a mempolicy for filtering nodes for
* page allocation, together with preferred node id (or the input node id).
*/
static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,
pgoff_t ilx, int *nid)
{
nodemask_t *nodemask = NULL;
switch (pol->mode) {
case MPOL_PREFERRED:
/* Override input node id */
*nid = first_node(pol->nodes);
break;
case MPOL_PREFERRED_MANY:
nodemask = &pol->nodes;
if (pol->home_node != NUMA_NO_NODE)
*nid = pol->home_node;
break;
case MPOL_BIND:
/* Restrict to nodemask (but not on lower zones) */
if (apply_policy_zone(pol, gfp_zone(gfp)) && cpuset_nodemask_valid_mems_allowed(&pol->nodes))
nodemask = &pol->nodes;
if (pol->home_node != NUMA_NO_NODE) *nid = pol->home_node;
/*
* __GFP_THISNODE shouldn't even be used with the bind policy
* because we might easily break the expectation to stay on the
* requested node and not break the policy.
*/
WARN_ON_ONCE(gfp & __GFP_THISNODE);
break;
case MPOL_INTERLEAVE:
/* Override input node id */
*nid = (ilx == NO_INTERLEAVE_INDEX) ? interleave_nodes(pol) : interleave_nid(pol, ilx);
break;
case MPOL_WEIGHTED_INTERLEAVE:
*nid = (ilx == NO_INTERLEAVE_INDEX) ? weighted_interleave_nodes(pol) : weighted_interleave_nid(pol, ilx);
break;
}
return nodemask;
}
#ifdef CONFIG_HUGETLBFS
/*
* huge_node(@vma, @addr, @gfp_flags, @mpol)
* @vma: virtual memory area whose policy is sought
* @addr: address in @vma for shared policy lookup and interleave policy
* @gfp_flags: for requested zone
* @mpol: pointer to mempolicy pointer for reference counted mempolicy
* @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy
*
* Returns a nid suitable for a huge page allocation and a pointer
* to the struct mempolicy for conditional unref after allocation.
* If the effective policy is 'bind' or 'prefer-many', returns a pointer
* to the mempolicy's @nodemask for filtering the zonelist.
*/
int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
struct mempolicy **mpol, nodemask_t **nodemask)
{
pgoff_t ilx;
int nid;
nid = numa_node_id();
*mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx);
*nodemask = policy_nodemask(gfp_flags, *mpol, ilx, &nid);
return nid;
}
/*
* init_nodemask_of_mempolicy
*
* If the current task's mempolicy is "default" [NULL], return 'false'
* to indicate default policy. Otherwise, extract the policy nodemask
* for 'bind' or 'interleave' policy into the argument nodemask, or
* initialize the argument nodemask to contain the single node for
* 'preferred' or 'local' policy and return 'true' to indicate presence
* of non-default mempolicy.
*
* We don't bother with reference counting the mempolicy [mpol_get/put]
* because the current task is examining it's own mempolicy and a task's
* mempolicy is only ever changed by the task itself.
*
* N.B., it is the caller's responsibility to free a returned nodemask.
*/
bool init_nodemask_of_mempolicy(nodemask_t *mask)
{
struct mempolicy *mempolicy;
if (!(mask && current->mempolicy))
return false;
task_lock(current);
mempolicy = current->mempolicy;
switch (mempolicy->mode) {
case MPOL_PREFERRED:
case MPOL_PREFERRED_MANY:
case MPOL_BIND:
case MPOL_INTERLEAVE:
case MPOL_WEIGHTED_INTERLEAVE:
*mask = mempolicy->nodes;
break;
case MPOL_LOCAL:
init_nodemask_of_node(mask, numa_node_id());
break;
default:
BUG();
}
task_unlock(current);
return true;
}
#endif
/*
* mempolicy_in_oom_domain
*
* If tsk's mempolicy is "bind", check for intersection between mask and
* the policy nodemask. Otherwise, return true for all other policies
* including "interleave", as a tsk with "interleave" policy may have
* memory allocated from all nodes in system.
*
* Takes task_lock(tsk) to prevent freeing of its mempolicy.
*/
bool mempolicy_in_oom_domain(struct task_struct *tsk,
const nodemask_t *mask)
{
struct mempolicy *mempolicy;
bool ret = true;
if (!mask)
return ret;
task_lock(tsk);
mempolicy = tsk->mempolicy;
if (mempolicy && mempolicy->mode == MPOL_BIND)
ret = nodes_intersects(mempolicy->nodes, *mask);
task_unlock(tsk);
return ret;
}
static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
int nid, nodemask_t *nodemask)
{
struct page *page;
gfp_t preferred_gfp;
/*
* This is a two pass approach. The first pass will only try the
* preferred nodes but skip the direct reclaim and allow the
* allocation to fail, while the second pass will try all the
* nodes in system.
*/
preferred_gfp = gfp | __GFP_NOWARN;
preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
page = __alloc_pages(preferred_gfp, order, nid, nodemask);
if (!page)
page = __alloc_pages(gfp, order, nid, NULL);
return page;
}
/**
* alloc_pages_mpol - Allocate pages according to NUMA mempolicy.
* @gfp: GFP flags.
* @order: Order of the page allocation.
* @pol: Pointer to the NUMA mempolicy.
* @ilx: Index for interleave mempolicy (also distinguishes alloc_pages()).
* @nid: Preferred node (usually numa_node_id() but @mpol may override it).
*
* Return: The page on success or NULL if allocation fails.
*/
struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
struct mempolicy *pol, pgoff_t ilx, int nid)
{
nodemask_t *nodemask;
struct page *page;
nodemask = policy_nodemask(gfp, pol, ilx, &nid);
if (pol->mode == MPOL_PREFERRED_MANY)
return alloc_pages_preferred_many(gfp, order, nid, nodemask);
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
/* filter "hugepage" allocation, unless from alloc_pages() */
order == HPAGE_PMD_ORDER && ilx != NO_INTERLEAVE_INDEX) {
/*
* For hugepage allocation and non-interleave policy which
* allows the current node (or other explicitly preferred
* node) we only try to allocate from the current/preferred
* node and don't fall back to other nodes, as the cost of
* remote accesses would likely offset THP benefits.
*
* If the policy is interleave or does not allow the current
* node in its nodemask, we allocate the standard way.
*/
if (pol->mode != MPOL_INTERLEAVE &&
pol->mode != MPOL_WEIGHTED_INTERLEAVE &&
(!nodemask || node_isset(nid, *nodemask))) {
/*
* First, try to allocate THP only on local node, but
* don't reclaim unnecessarily, just compact.
*/
page = __alloc_pages_node(nid,
gfp | __GFP_THISNODE | __GFP_NORETRY, order);
if (page || !(gfp & __GFP_DIRECT_RECLAIM))
return page;
/*
* If hugepage allocations are configured to always
* synchronous compact or the vma has been madvised
* to prefer hugepage backing, retry allowing remote
* memory with both reclaim and compact as well.
*/
}
}
page = __alloc_pages(gfp, order, nid, nodemask); if (unlikely(pol->mode == MPOL_INTERLEAVE) && page) {
/* skip NUMA_INTERLEAVE_HIT update if numa stats is disabled */
if (static_branch_likely(&vm_numa_stat_key) && page_to_nid(page) == nid) { preempt_disable(); __count_numa_event(page_zone(page), NUMA_INTERLEAVE_HIT); preempt_enable();
}
}
return page;
}
/**
* vma_alloc_folio - Allocate a folio for a VMA.
* @gfp: GFP flags.
* @order: Order of the folio.
* @vma: Pointer to VMA.
* @addr: Virtual address of the allocation. Must be inside @vma.
* @hugepage: Unused (was: For hugepages try only preferred node if possible).
*
* Allocate a folio for a specific address in @vma, using the appropriate
* NUMA policy. The caller must hold the mmap_lock of the mm_struct of the
* VMA to prevent it from going away. Should be used for all allocations
* for folios that will be mapped into user space, excepting hugetlbfs, and
* excepting where direct use of alloc_pages_mpol() is more appropriate.
*
* Return: The folio on success or NULL if allocation fails.
*/
struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma,
unsigned long addr, bool hugepage)
{
struct mempolicy *pol;
pgoff_t ilx;
struct page *page;
pol = get_vma_policy(vma, addr, order, &ilx);
page = alloc_pages_mpol(gfp | __GFP_COMP, order,
pol, ilx, numa_node_id());
mpol_cond_put(pol); return page_rmappable_folio(page);
}
EXPORT_SYMBOL(vma_alloc_folio);
/**
* alloc_pages - Allocate pages.
* @gfp: GFP flags.
* @order: Power of two of number of pages to allocate.
*
* Allocate 1 << @order contiguous pages. The physical address of the
* first page is naturally aligned (eg an order-3 allocation will be aligned
* to a multiple of 8 * PAGE_SIZE bytes). The NUMA policy of the current
* process is honoured when in process context.
*
* Context: Can be called from any context, providing the appropriate GFP
* flags are used.
* Return: The page on success or NULL if allocation fails.
*/
struct page *alloc_pages(gfp_t gfp, unsigned int order)
{
struct mempolicy *pol = &default_policy;
/*
* No reference counting needed for current->mempolicy
* nor system default_policy
*/
if (!in_interrupt() && !(gfp & __GFP_THISNODE)) pol = get_task_policy(current); return alloc_pages_mpol(gfp, order,
pol, NO_INTERLEAVE_INDEX, numa_node_id());
}
EXPORT_SYMBOL(alloc_pages);
struct folio *folio_alloc(gfp_t gfp, unsigned int order)
{
return page_rmappable_folio(alloc_pages(gfp | __GFP_COMP, order));
}
EXPORT_SYMBOL(folio_alloc);
static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp,
struct mempolicy *pol, unsigned long nr_pages,
struct page **page_array)
{
int nodes;
unsigned long nr_pages_per_node;
int delta;
int i;
unsigned long nr_allocated;
unsigned long total_allocated = 0;
nodes = nodes_weight(pol->nodes);
nr_pages_per_node = nr_pages / nodes;
delta = nr_pages - nodes * nr_pages_per_node;
for (i = 0; i < nodes; i++) {
if (delta) {
nr_allocated = __alloc_pages_bulk(gfp,
interleave_nodes(pol), NULL,
nr_pages_per_node + 1, NULL,
page_array);
delta--;
} else {
nr_allocated = __alloc_pages_bulk(gfp,
interleave_nodes(pol), NULL,
nr_pages_per_node, NULL, page_array);
}
page_array += nr_allocated;
total_allocated += nr_allocated;
}
return total_allocated;
}
static unsigned long alloc_pages_bulk_array_weighted_interleave(gfp_t gfp,
struct mempolicy *pol, unsigned long nr_pages,
struct page **page_array)
{
struct task_struct *me = current;
unsigned int cpuset_mems_cookie;
unsigned long total_allocated = 0;
unsigned long nr_allocated = 0;
unsigned long rounds;
unsigned long node_pages, delta;
u8 *table, *weights, weight;
unsigned int weight_total = 0;
unsigned long rem_pages = nr_pages;
nodemask_t nodes;
int nnodes, node;
int resume_node = MAX_NUMNODES - 1;
u8 resume_weight = 0;
int prev_node;
int i;
if (!nr_pages)
return 0;
/* read the nodes onto the stack, retry if done during rebind */
do {
cpuset_mems_cookie = read_mems_allowed_begin(); nnodes = read_once_policy_nodemask(pol, &nodes); } while (read_mems_allowed_retry(cpuset_mems_cookie));
/* if the nodemask has become invalid, we cannot do anything */
if (!nnodes)
return 0;
/* Continue allocating from most recent node and adjust the nr_pages */
node = me->il_prev;
weight = me->il_weight;
if (weight && node_isset(node, nodes)) { node_pages = min(rem_pages, weight);
nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
NULL, page_array);
page_array += nr_allocated;
total_allocated += nr_allocated;
/* if that's all the pages, no need to interleave */
if (rem_pages <= weight) {
me->il_weight -= rem_pages;
return total_allocated;
}
/* Otherwise we adjust remaining pages, continue from there */
rem_pages -= weight;
}
/* clear active weight in case of an allocation failure */
me->il_weight = 0;
prev_node = node;
/* create a local copy of node weights to operate on outside rcu */
weights = kzalloc(nr_node_ids, GFP_KERNEL);
if (!weights)
return total_allocated;
rcu_read_lock(); table = rcu_dereference(iw_table); if (table) memcpy(weights, table, nr_node_ids); rcu_read_unlock();
/* calculate total, detect system default usage */
for_each_node_mask(node, nodes) { if (!weights[node]) weights[node] = 1; weight_total += weights[node];
}
/*
* Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls.
* Track which node weighted interleave should resume from.
*
* if (rounds > 0) and (delta == 0), resume_node will always be
* the node following prev_node and its weight.
*/
rounds = rem_pages / weight_total;
delta = rem_pages % weight_total;
resume_node = next_node_in(prev_node, nodes); resume_weight = weights[resume_node]; for (i = 0; i < nnodes; i++) { node = next_node_in(prev_node, nodes); weight = weights[node];
node_pages = weight * rounds;
/* If a delta exists, add this node's portion of the delta */
if (delta > weight) {
node_pages += weight;
delta -= weight;
} else if (delta) {
/* when delta is depleted, resume from that node */
node_pages += delta;
resume_node = node;
resume_weight = weight - delta;
delta = 0;
}
/* node_pages can be 0 if an allocation fails and rounds == 0 */
if (!node_pages)
break;
nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
NULL, page_array);
page_array += nr_allocated;
total_allocated += nr_allocated;
if (total_allocated == nr_pages)
break;
prev_node = node;
}
me->il_prev = resume_node;
me->il_weight = resume_weight;
kfree(weights);
return total_allocated;
}
static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid,
struct mempolicy *pol, unsigned long nr_pages,
struct page **page_array)
{
gfp_t preferred_gfp;
unsigned long nr_allocated = 0;
preferred_gfp = gfp | __GFP_NOWARN;
preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
nr_allocated = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes,
nr_pages, NULL, page_array);
if (nr_allocated < nr_pages)
nr_allocated += __alloc_pages_bulk(gfp, numa_node_id(), NULL,
nr_pages - nr_allocated, NULL,
page_array + nr_allocated);
return nr_allocated;
}
/* alloc pages bulk and mempolicy should be considered at the
* same time in some situation such as vmalloc.
*
* It can accelerate memory allocation especially interleaving
* allocate memory.
*/
unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,
unsigned long nr_pages, struct page **page_array)
{
struct mempolicy *pol = &default_policy;
nodemask_t *nodemask;
int nid;
if (!in_interrupt() && !(gfp & __GFP_THISNODE))
pol = get_task_policy(current); if (pol->mode == MPOL_INTERLEAVE) return alloc_pages_bulk_array_interleave(gfp, pol,
nr_pages, page_array);
if (pol->mode == MPOL_WEIGHTED_INTERLEAVE) return alloc_pages_bulk_array_weighted_interleave(
gfp, pol, nr_pages, page_array);
if (pol->mode == MPOL_PREFERRED_MANY)
return alloc_pages_bulk_array_preferred_many(gfp,
numa_node_id(), pol, nr_pages, page_array);
nid = numa_node_id();
nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid);
return __alloc_pages_bulk(gfp, nid, nodemask,
nr_pages, NULL, page_array);
}
int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
{
struct mempolicy *pol = mpol_dup(src->vm_policy);
if (IS_ERR(pol))
return PTR_ERR(pol);
dst->vm_policy = pol;
return 0;
}
/*
* If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
* rebinds the mempolicy its copying by calling mpol_rebind_policy()
* with the mems_allowed returned by cpuset_mems_allowed(). This
* keeps mempolicies cpuset relative after its cpuset moves. See
* further kernel/cpuset.c update_nodemask().
*
* current's mempolicy may be rebinded by the other task(the task that changes
* cpuset's mems), so we needn't do rebind work for current task.
*/
/* Slow path of a mempolicy duplicate */
struct mempolicy *__mpol_dup(struct mempolicy *old)
{
struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
if (!new)
return ERR_PTR(-ENOMEM);
/* task's mempolicy is protected by alloc_lock */
if (old == current->mempolicy) {
task_lock(current);
*new = *old;
task_unlock(current);
} else
*new = *old;
if (current_cpuset_is_being_rebound()) {
nodemask_t mems = cpuset_mems_allowed(current);
mpol_rebind_policy(new, &mems);
}
atomic_set(&new->refcnt, 1);
return new;
}
/* Slow path of a mempolicy comparison */
bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
{
if (!a || !b)
return false;
if (a->mode != b->mode)
return false;
if (a->flags != b->flags)
return false;
if (a->home_node != b->home_node)
return false;
if (mpol_store_user_nodemask(a))
if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
return false;
switch (a->mode) {
case MPOL_BIND:
case MPOL_INTERLEAVE:
case MPOL_PREFERRED:
case MPOL_PREFERRED_MANY:
case MPOL_WEIGHTED_INTERLEAVE:
return !!nodes_equal(a->nodes, b->nodes);
case MPOL_LOCAL:
return true;
default:
BUG();
return false;
}
}
/*
* Shared memory backing store policy support.
*
* Remember policies even when nobody has shared memory mapped.
* The policies are kept in Red-Black tree linked from the inode.
* They are protected by the sp->lock rwlock, which should be held
* for any accesses to the tree.
*/
/*
* lookup first element intersecting start-end. Caller holds sp->lock for
* reading or for writing
*/
static struct sp_node *sp_lookup(struct shared_policy *sp,
pgoff_t start, pgoff_t end)
{
struct rb_node *n = sp->root.rb_node;
while (n) {
struct sp_node *p = rb_entry(n, struct sp_node, nd);
if (start >= p->end)
n = n->rb_right;
else if (end <= p->start)
n = n->rb_left;
else
break;
}
if (!n)
return NULL;
for (;;) {
struct sp_node *w = NULL;
struct rb_node *prev = rb_prev(n);
if (!prev)
break;
w = rb_entry(prev, struct sp_node, nd);
if (w->end <= start)
break;
n = prev;
}
return rb_entry(n, struct sp_node, nd);
}
/*
* Insert a new shared policy into the list. Caller holds sp->lock for
* writing.
*/
static void sp_insert(struct shared_policy *sp, struct sp_node *new)
{
struct rb_node **p = &sp->root.rb_node;
struct rb_node *parent = NULL;
struct sp_node *nd;
while (*p) {
parent = *p;
nd = rb_entry(parent, struct sp_node, nd);
if (new->start < nd->start)
p = &(*p)->rb_left;
else if (new->end > nd->end)
p = &(*p)->rb_right;
else
BUG();
}
rb_link_node(&new->nd, parent, p);
rb_insert_color(&new->nd, &sp->root);
}
/* Find shared policy intersecting idx */
struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
pgoff_t idx)
{
struct mempolicy *pol = NULL;
struct sp_node *sn;
if (!sp->root.rb_node)
return NULL;
read_lock(&sp->lock);
sn = sp_lookup(sp, idx, idx+1);
if (sn) {
mpol_get(sn->policy);
pol = sn->policy;
}
read_unlock(&sp->lock); return pol;
}
static void sp_free(struct sp_node *n)
{
mpol_put(n->policy);
kmem_cache_free(sn_cache, n);
}
/**
* mpol_misplaced - check whether current folio node is valid in policy
*
* @folio: folio to be checked
* @vma: vm area where folio mapped
* @addr: virtual address in @vma for shared policy lookup and interleave policy
*
* Lookup current policy node id for vma,addr and "compare to" folio's
* node id. Policy determination "mimics" alloc_page_vma().
* Called from fault path where we know the vma and faulting address.
*
* Return: NUMA_NO_NODE if the page is in a node that is valid for this
* policy, or a suitable node ID to allocate a replacement folio from.
*/
int mpol_misplaced(struct folio *folio, struct vm_area_struct *vma,
unsigned long addr)
{
struct mempolicy *pol;
pgoff_t ilx;
struct zoneref *z;
int curnid = folio_nid(folio);
int thiscpu = raw_smp_processor_id();
int thisnid = cpu_to_node(thiscpu);
int polnid = NUMA_NO_NODE;
int ret = NUMA_NO_NODE;
pol = get_vma_policy(vma, addr, folio_order(folio), &ilx);
if (!(pol->flags & MPOL_F_MOF))
goto out;
switch (pol->mode) {
case MPOL_INTERLEAVE:
polnid = interleave_nid(pol, ilx);
break;
case MPOL_WEIGHTED_INTERLEAVE:
polnid = weighted_interleave_nid(pol, ilx);
break;
case MPOL_PREFERRED:
if (node_isset(curnid, pol->nodes))
goto out;
polnid = first_node(pol->nodes);
break;
case MPOL_LOCAL:
polnid = numa_node_id();
break;
case MPOL_BIND:
/* Optimize placement among multiple nodes via NUMA balancing */
if (pol->flags & MPOL_F_MORON) {
if (node_isset(thisnid, pol->nodes))
break;
goto out;
}
fallthrough;
case MPOL_PREFERRED_MANY:
/*
* use current page if in policy nodemask,
* else select nearest allowed node, if any.
* If no allowed nodes, use current [!misplaced].
*/
if (node_isset(curnid, pol->nodes))
goto out;
z = first_zones_zonelist(
node_zonelist(numa_node_id(), GFP_HIGHUSER),
gfp_zone(GFP_HIGHUSER),
&pol->nodes);
polnid = zone_to_nid(z->zone);
break;
default:
BUG();
}
/* Migrate the folio towards the node whose CPU is referencing it */
if (pol->flags & MPOL_F_MORON) {
polnid = thisnid;
if (!should_numa_migrate_memory(current, folio, curnid,
thiscpu))
goto out;
}
if (curnid != polnid)
ret = polnid;
out:
mpol_cond_put(pol);
return ret;
}
/*
* Drop the (possibly final) reference to task->mempolicy. It needs to be
* dropped after task->mempolicy is set to NULL so that any allocation done as
* part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
* policy.
*/
void mpol_put_task_policy(struct task_struct *task)
{
struct mempolicy *pol;
task_lock(task);
pol = task->mempolicy;
task->mempolicy = NULL;
task_unlock(task);
mpol_put(pol);
}
static void sp_delete(struct shared_policy *sp, struct sp_node *n)
{
rb_erase(&n->nd, &sp->root);
sp_free(n);
}
static void sp_node_init(struct sp_node *node, unsigned long start,
unsigned long end, struct mempolicy *pol)
{
node->start = start;
node->end = end;
node->policy = pol;
}
static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
struct mempolicy *pol)
{
struct sp_node *n;
struct mempolicy *newpol;
n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
if (!n)
return NULL;
newpol = mpol_dup(pol);
if (IS_ERR(newpol)) {
kmem_cache_free(sn_cache, n);
return NULL;
}
newpol->flags |= MPOL_F_SHARED;
sp_node_init(n, start, end, newpol);
return n;
}
/* Replace a policy range. */
static int shared_policy_replace(struct shared_policy *sp, pgoff_t start,
pgoff_t end, struct sp_node *new)
{
struct sp_node *n;
struct sp_node *n_new = NULL;
struct mempolicy *mpol_new = NULL;
int ret = 0;
restart:
write_lock(&sp->lock);
n = sp_lookup(sp, start, end);
/* Take care of old policies in the same range. */
while (n && n->start < end) {
struct rb_node *next = rb_next(&n->nd);
if (n->start >= start) {
if (n->end <= end)
sp_delete(sp, n);
else
n->start = end;
} else {
/* Old policy spanning whole new range. */
if (n->end > end) {
if (!n_new)
goto alloc_new;
*mpol_new = *n->policy;
atomic_set(&mpol_new->refcnt, 1);
sp_node_init(n_new, end, n->end, mpol_new);
n->end = start;
sp_insert(sp, n_new);
n_new = NULL;
mpol_new = NULL;
break;
} else
n->end = start;
}
if (!next)
break;
n = rb_entry(next, struct sp_node, nd);
}
if (new)
sp_insert(sp, new);
write_unlock(&sp->lock);
ret = 0;
err_out:
if (mpol_new)
mpol_put(mpol_new);
if (n_new)
kmem_cache_free(sn_cache, n_new);
return ret;
alloc_new:
write_unlock(&sp->lock);
ret = -ENOMEM;
n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
if (!n_new)
goto err_out;
mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
if (!mpol_new)
goto err_out;
atomic_set(&mpol_new->refcnt, 1);
goto restart;
}
/**
* mpol_shared_policy_init - initialize shared policy for inode
* @sp: pointer to inode shared policy
* @mpol: struct mempolicy to install
*
* Install non-NULL @mpol in inode's shared policy rb-tree.
* On entry, the current task has a reference on a non-NULL @mpol.
* This must be released on exit.
* This is called at get_inode() calls and we can use GFP_KERNEL.
*/
void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
{
int ret;
sp->root = RB_ROOT; /* empty tree == default mempolicy */
rwlock_init(&sp->lock);
if (mpol) {
struct sp_node *sn;
struct mempolicy *npol;
NODEMASK_SCRATCH(scratch);
if (!scratch)
goto put_mpol;
/* contextualize the tmpfs mount point mempolicy to this file */
npol = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
if (IS_ERR(npol))
goto free_scratch; /* no valid nodemask intersection */
task_lock(current); ret = mpol_set_nodemask(npol, &mpol->w.user_nodemask, scratch); task_unlock(current);
if (ret)
goto put_npol;
/* alloc node covering entire file; adds ref to file's npol */
sn = sp_alloc(0, MAX_LFS_FILESIZE >> PAGE_SHIFT, npol);
if (sn)
sp_insert(sp, sn);
put_npol:
mpol_put(npol); /* drop initial ref on file's npol */
free_scratch:
NODEMASK_SCRATCH_FREE(scratch);
put_mpol:
mpol_put(mpol); /* drop our incoming ref on sb mpol */
}
}
int mpol_set_shared_policy(struct shared_policy *sp,
struct vm_area_struct *vma, struct mempolicy *pol)
{
int err;
struct sp_node *new = NULL;
unsigned long sz = vma_pages(vma);
if (pol) {
new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol);
if (!new)
return -ENOMEM;
}
err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new);
if (err && new)
sp_free(new);
return err;
}
/* Free a backing policy store on inode delete. */
void mpol_free_shared_policy(struct shared_policy *sp)
{
struct sp_node *n;
struct rb_node *next;
if (!sp->root.rb_node)
return;
write_lock(&sp->lock);
next = rb_first(&sp->root);
while (next) {
n = rb_entry(next, struct sp_node, nd);
next = rb_next(&n->nd);
sp_delete(sp, n);
}
write_unlock(&sp->lock);
}
#ifdef CONFIG_NUMA_BALANCING
static int __initdata numabalancing_override;
static void __init check_numabalancing_enable(void)
{
bool numabalancing_default = false;
if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
numabalancing_default = true;
/* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
if (numabalancing_override)
set_numabalancing_state(numabalancing_override == 1);
if (num_online_nodes() > 1 && !numabalancing_override) {
pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
numabalancing_default ? "Enabling" : "Disabling");
set_numabalancing_state(numabalancing_default);
}
}
static int __init setup_numabalancing(char *str)
{
int ret = 0;
if (!str)
goto out;
if (!strcmp(str, "enable")) {
numabalancing_override = 1;
ret = 1;
} else if (!strcmp(str, "disable")) {
numabalancing_override = -1;
ret = 1;
}
out:
if (!ret)
pr_warn("Unable to parse numa_balancing=\n");
return ret;
}
__setup("numa_balancing=", setup_numabalancing);
#else
static inline void __init check_numabalancing_enable(void)
{
}
#endif /* CONFIG_NUMA_BALANCING */
void __init numa_policy_init(void)
{
nodemask_t interleave_nodes;
unsigned long largest = 0;
int nid, prefer = 0;
policy_cache = kmem_cache_create("numa_policy",
sizeof(struct mempolicy),
0, SLAB_PANIC, NULL);
sn_cache = kmem_cache_create("shared_policy_node",
sizeof(struct sp_node),
0, SLAB_PANIC, NULL);
for_each_node(nid) {
preferred_node_policy[nid] = (struct mempolicy) {
.refcnt = ATOMIC_INIT(1),
.mode = MPOL_PREFERRED,
.flags = MPOL_F_MOF | MPOL_F_MORON,
.nodes = nodemask_of_node(nid),
};
}
/*
* Set interleaving policy for system init. Interleaving is only
* enabled across suitably sized nodes (default is >= 16MB), or
* fall back to the largest node if they're all smaller.
*/
nodes_clear(interleave_nodes);
for_each_node_state(nid, N_MEMORY) {
unsigned long total_pages = node_present_pages(nid);
/* Preserve the largest node */
if (largest < total_pages) {
largest = total_pages;
prefer = nid;
}
/* Interleave this node? */
if ((total_pages << PAGE_SHIFT) >= (16 << 20))
node_set(nid, interleave_nodes);
}
/* All too small, use the largest */
if (unlikely(nodes_empty(interleave_nodes)))
node_set(prefer, interleave_nodes);
if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
pr_err("%s: interleaving failed\n", __func__);
check_numabalancing_enable();
}
/* Reset policy of current process to default */
void numa_default_policy(void)
{
do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
}
/*
* Parse and format mempolicy from/to strings
*/
static const char * const policy_modes[] =
{
[MPOL_DEFAULT] = "default",
[MPOL_PREFERRED] = "prefer",
[MPOL_BIND] = "bind",
[MPOL_INTERLEAVE] = "interleave",
[MPOL_WEIGHTED_INTERLEAVE] = "weighted interleave",
[MPOL_LOCAL] = "local",
[MPOL_PREFERRED_MANY] = "prefer (many)",
};
#ifdef CONFIG_TMPFS
/**
* mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
* @str: string containing mempolicy to parse
* @mpol: pointer to struct mempolicy pointer, returned on success.
*
* Format of input:
* <mode>[=<flags>][:<nodelist>]
*
* Return: %0 on success, else %1
*/
int mpol_parse_str(char *str, struct mempolicy **mpol)
{
struct mempolicy *new = NULL;
unsigned short mode_flags;
nodemask_t nodes;
char *nodelist = strchr(str, ':');
char *flags = strchr(str, '=');
int err = 1, mode;
if (flags)
*flags++ = '\0'; /* terminate mode string */
if (nodelist) {
/* NUL-terminate mode or flags string */
*nodelist++ = '\0';
if (nodelist_parse(nodelist, nodes))
goto out;
if (!nodes_subset(nodes, node_states[N_MEMORY]))
goto out;
} else
nodes_clear(nodes);
mode = match_string(policy_modes, MPOL_MAX, str);
if (mode < 0)
goto out;
switch (mode) {
case MPOL_PREFERRED:
/*
* Insist on a nodelist of one node only, although later
* we use first_node(nodes) to grab a single node, so here
* nodelist (or nodes) cannot be empty.
*/
if (nodelist) {
char *rest = nodelist;
while (isdigit(*rest))
rest++;
if (*rest)
goto out;
if (nodes_empty(nodes))
goto out;
}
break;
case MPOL_INTERLEAVE:
case MPOL_WEIGHTED_INTERLEAVE:
/*
* Default to online nodes with memory if no nodelist
*/
if (!nodelist)
nodes = node_states[N_MEMORY];
break;
case MPOL_LOCAL:
/*
* Don't allow a nodelist; mpol_new() checks flags
*/
if (nodelist)
goto out;
break;
case MPOL_DEFAULT:
/*
* Insist on a empty nodelist
*/
if (!nodelist)
err = 0;
goto out;
case MPOL_PREFERRED_MANY:
case MPOL_BIND:
/*
* Insist on a nodelist
*/
if (!nodelist)
goto out;
}
mode_flags = 0;
if (flags) {
/*
* Currently, we only support two mutually exclusive
* mode flags.
*/
if (!strcmp(flags, "static"))
mode_flags |= MPOL_F_STATIC_NODES;
else if (!strcmp(flags, "relative"))
mode_flags |= MPOL_F_RELATIVE_NODES;
else
goto out;
}
new = mpol_new(mode, mode_flags, &nodes);
if (IS_ERR(new))
goto out;
/*
* Save nodes for mpol_to_str() to show the tmpfs mount options
* for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
*/
if (mode != MPOL_PREFERRED) {
new->nodes = nodes;
} else if (nodelist) {
nodes_clear(new->nodes);
node_set(first_node(nodes), new->nodes);
} else {
new->mode = MPOL_LOCAL;
}
/*
* Save nodes for contextualization: this will be used to "clone"
* the mempolicy in a specific context [cpuset] at a later time.
*/
new->w.user_nodemask = nodes;
err = 0;
out:
/* Restore string for error message */
if (nodelist)
*--nodelist = ':';
if (flags)
*--flags = '=';
if (!err)
*mpol = new;
return err;
}
#endif /* CONFIG_TMPFS */
/**
* mpol_to_str - format a mempolicy structure for printing
* @buffer: to contain formatted mempolicy string
* @maxlen: length of @buffer
* @pol: pointer to mempolicy to be formatted
*
* Convert @pol into a string. If @buffer is too short, truncate the string.
* Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
* longest flag, "relative", and to display at least a few node ids.
*/
void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
{
char *p = buffer;
nodemask_t nodes = NODE_MASK_NONE;
unsigned short mode = MPOL_DEFAULT;
unsigned short flags = 0;
if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
mode = pol->mode;
flags = pol->flags;
}
switch (mode) {
case MPOL_DEFAULT:
case MPOL_LOCAL:
break;
case MPOL_PREFERRED:
case MPOL_PREFERRED_MANY:
case MPOL_BIND:
case MPOL_INTERLEAVE:
case MPOL_WEIGHTED_INTERLEAVE:
nodes = pol->nodes;
break;
default:
WARN_ON_ONCE(1);
snprintf(p, maxlen, "unknown");
return;
}
p += snprintf(p, maxlen, "%s", policy_modes[mode]);
if (flags & MPOL_MODE_FLAGS) {
p += snprintf(p, buffer + maxlen - p, "=");
/*
* Currently, the only defined flags are mutually exclusive
*/
if (flags & MPOL_F_STATIC_NODES)
p += snprintf(p, buffer + maxlen - p, "static");
else if (flags & MPOL_F_RELATIVE_NODES)
p += snprintf(p, buffer + maxlen - p, "relative");
}
if (!nodes_empty(nodes))
p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
nodemask_pr_args(&nodes));
}
#ifdef CONFIG_SYSFS
struct iw_node_attr {
struct kobj_attribute kobj_attr;
int nid;
};
static ssize_t node_show(struct kobject *kobj, struct kobj_attribute *attr,
char *buf)
{
struct iw_node_attr *node_attr;
u8 weight;
node_attr = container_of(attr, struct iw_node_attr, kobj_attr);
weight = get_il_weight(node_attr->nid);
return sysfs_emit(buf, "%d\n", weight);
}
static ssize_t node_store(struct kobject *kobj, struct kobj_attribute *attr,
const char *buf, size_t count)
{
struct iw_node_attr *node_attr;
u8 *new;
u8 *old;
u8 weight = 0;
node_attr = container_of(attr, struct iw_node_attr, kobj_attr);
if (count == 0 || sysfs_streq(buf, ""))
weight = 0;
else if (kstrtou8(buf, 0, &weight))
return -EINVAL;
new = kzalloc(nr_node_ids, GFP_KERNEL);
if (!new)
return -ENOMEM;
mutex_lock(&iw_table_lock);
old = rcu_dereference_protected(iw_table,
lockdep_is_held(&iw_table_lock));
if (old)
memcpy(new, old, nr_node_ids);
new[node_attr->nid] = weight;
rcu_assign_pointer(iw_table, new);
mutex_unlock(&iw_table_lock);
synchronize_rcu();
kfree(old);
return count;
}
static struct iw_node_attr **node_attrs;
static void sysfs_wi_node_release(struct iw_node_attr *node_attr,
struct kobject *parent)
{
if (!node_attr)
return;
sysfs_remove_file(parent, &node_attr->kobj_attr.attr);
kfree(node_attr->kobj_attr.attr.name);
kfree(node_attr);
}
static void sysfs_wi_release(struct kobject *wi_kobj)
{
int i;
for (i = 0; i < nr_node_ids; i++)
sysfs_wi_node_release(node_attrs[i], wi_kobj);
kobject_put(wi_kobj);
}
static const struct kobj_type wi_ktype = {
.sysfs_ops = &kobj_sysfs_ops,
.release = sysfs_wi_release,
};
static int add_weight_node(int nid, struct kobject *wi_kobj)
{
struct iw_node_attr *node_attr;
char *name;
node_attr = kzalloc(sizeof(*node_attr), GFP_KERNEL);
if (!node_attr)
return -ENOMEM;
name = kasprintf(GFP_KERNEL, "node%d", nid);
if (!name) {
kfree(node_attr);
return -ENOMEM;
}
sysfs_attr_init(&node_attr->kobj_attr.attr);
node_attr->kobj_attr.attr.name = name;
node_attr->kobj_attr.attr.mode = 0644;
node_attr->kobj_attr.show = node_show;
node_attr->kobj_attr.store = node_store;
node_attr->nid = nid;
if (sysfs_create_file(wi_kobj, &node_attr->kobj_attr.attr)) {
kfree(node_attr->kobj_attr.attr.name);
kfree(node_attr);
pr_err("failed to add attribute to weighted_interleave\n");
return -ENOMEM;
}
node_attrs[nid] = node_attr;
return 0;
}
static int add_weighted_interleave_group(struct kobject *root_kobj)
{
struct kobject *wi_kobj;
int nid, err;
wi_kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL);
if (!wi_kobj)
return -ENOMEM;
err = kobject_init_and_add(wi_kobj, &wi_ktype, root_kobj,
"weighted_interleave");
if (err) {
kfree(wi_kobj);
return err;
}
for_each_node_state(nid, N_POSSIBLE) {
err = add_weight_node(nid, wi_kobj);
if (err) {
pr_err("failed to add sysfs [node%d]\n", nid);
break;
}
}
if (err)
kobject_put(wi_kobj);
return 0;
}
static void mempolicy_kobj_release(struct kobject *kobj)
{
u8 *old;
mutex_lock(&iw_table_lock);
old = rcu_dereference_protected(iw_table,
lockdep_is_held(&iw_table_lock));
rcu_assign_pointer(iw_table, NULL);
mutex_unlock(&iw_table_lock);
synchronize_rcu();
kfree(old);
kfree(node_attrs);
kfree(kobj);
}
static const struct kobj_type mempolicy_ktype = {
.release = mempolicy_kobj_release
};
static int __init mempolicy_sysfs_init(void)
{
int err;
static struct kobject *mempolicy_kobj;
mempolicy_kobj = kzalloc(sizeof(*mempolicy_kobj), GFP_KERNEL);
if (!mempolicy_kobj) {
err = -ENOMEM;
goto err_out;
}
node_attrs = kcalloc(nr_node_ids, sizeof(struct iw_node_attr *),
GFP_KERNEL);
if (!node_attrs) {
err = -ENOMEM;
goto mempol_out;
}
err = kobject_init_and_add(mempolicy_kobj, &mempolicy_ktype, mm_kobj,
"mempolicy");
if (err)
goto node_out;
err = add_weighted_interleave_group(mempolicy_kobj);
if (err) {
pr_err("mempolicy sysfs structure failed to initialize\n");
kobject_put(mempolicy_kobj);
return err;
}
return err;
node_out:
kfree(node_attrs);
mempol_out:
kfree(mempolicy_kobj);
err_out:
pr_err("failed to add mempolicy kobject to the system\n");
return err;
}
late_initcall(mempolicy_sysfs_init);
#endif /* CONFIG_SYSFS */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_BITMAP_H
#define __LINUX_BITMAP_H
#ifndef __ASSEMBLY__
#include <linux/align.h>
#include <linux/bitops.h>
#include <linux/cleanup.h>
#include <linux/errno.h>
#include <linux/find.h>
#include <linux/limits.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/bitmap-str.h>
struct device;
/*
* bitmaps provide bit arrays that consume one or more unsigned
* longs. The bitmap interface and available operations are listed
* here, in bitmap.h
*
* Function implementations generic to all architectures are in
* lib/bitmap.c. Functions implementations that are architecture
* specific are in various include/asm-<arch>/bitops.h headers
* and other arch/<arch> specific files.
*
* See lib/bitmap.c for more details.
*/
/**
* DOC: bitmap overview
*
* The available bitmap operations and their rough meaning in the
* case that the bitmap is a single unsigned long are thus:
*
* The generated code is more efficient when nbits is known at
* compile-time and at most BITS_PER_LONG.
*
* ::
*
* bitmap_zero(dst, nbits) *dst = 0UL
* bitmap_fill(dst, nbits) *dst = ~0UL
* bitmap_copy(dst, src, nbits) *dst = *src
* bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2
* bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2
* bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
* bitmap_equal(src1, src2, nbits) Are *src1 and *src2 equal?
* bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap?
* bitmap_subset(src1, src2, nbits) Is *src1 a subset of *src2?
* bitmap_empty(src, nbits) Are all bits zero in *src?
* bitmap_full(src, nbits) Are all bits set in *src?
* bitmap_weight(src, nbits) Hamming Weight: number set bits
* bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap
* bitmap_weight_andnot(src1, src2, nbits) Hamming Weight of andnot'ed bitmap
* bitmap_set(dst, pos, nbits) Set specified bit area
* bitmap_clear(dst, pos, nbits) Clear specified bit area
* bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area
* bitmap_find_next_zero_area_off(buf, len, pos, n, mask, mask_off) as above
* bitmap_shift_right(dst, src, n, nbits) *dst = *src >> n
* bitmap_shift_left(dst, src, n, nbits) *dst = *src << n
* bitmap_cut(dst, src, first, n, nbits) Cut n bits from first, copy rest
* bitmap_replace(dst, old, new, mask, nbits) *dst = (*old & ~(*mask)) | (*new & *mask)
* bitmap_scatter(dst, src, mask, nbits) *dst = map(dense, sparse)(src)
* bitmap_gather(dst, src, mask, nbits) *dst = map(sparse, dense)(src)
* bitmap_remap(dst, src, old, new, nbits) *dst = map(old, new)(src)
* bitmap_bitremap(oldbit, old, new, nbits) newbit = map(old, new)(oldbit)
* bitmap_onto(dst, orig, relmap, nbits) *dst = orig relative to relmap
* bitmap_fold(dst, orig, sz, nbits) dst bits = orig bits mod sz
* bitmap_parse(buf, buflen, dst, nbits) Parse bitmap dst from kernel buf
* bitmap_parse_user(ubuf, ulen, dst, nbits) Parse bitmap dst from user buf
* bitmap_parselist(buf, dst, nbits) Parse bitmap dst from kernel buf
* bitmap_parselist_user(buf, dst, nbits) Parse bitmap dst from user buf
* bitmap_find_free_region(bitmap, bits, order) Find and allocate bit region
* bitmap_release_region(bitmap, pos, order) Free specified bit region
* bitmap_allocate_region(bitmap, pos, order) Allocate specified bit region
* bitmap_from_arr32(dst, buf, nbits) Copy nbits from u32[] buf to dst
* bitmap_from_arr64(dst, buf, nbits) Copy nbits from u64[] buf to dst
* bitmap_to_arr32(buf, src, nbits) Copy nbits from buf to u32[] dst
* bitmap_to_arr64(buf, src, nbits) Copy nbits from buf to u64[] dst
* bitmap_get_value8(map, start) Get 8bit value from map at start
* bitmap_set_value8(map, value, start) Set 8bit value to map at start
*
* Note, bitmap_zero() and bitmap_fill() operate over the region of
* unsigned longs, that is, bits behind bitmap till the unsigned long
* boundary will be zeroed or filled as well. Consider to use
* bitmap_clear() or bitmap_set() to make explicit zeroing or filling
* respectively.
*/
/**
* DOC: bitmap bitops
*
* Also the following operations in asm/bitops.h apply to bitmaps.::
*
* set_bit(bit, addr) *addr |= bit
* clear_bit(bit, addr) *addr &= ~bit
* change_bit(bit, addr) *addr ^= bit
* test_bit(bit, addr) Is bit set in *addr?
* test_and_set_bit(bit, addr) Set bit and return old value
* test_and_clear_bit(bit, addr) Clear bit and return old value
* test_and_change_bit(bit, addr) Change bit and return old value
* find_first_zero_bit(addr, nbits) Position first zero bit in *addr
* find_first_bit(addr, nbits) Position first set bit in *addr
* find_next_zero_bit(addr, nbits, bit)
* Position next zero bit in *addr >= bit
* find_next_bit(addr, nbits, bit) Position next set bit in *addr >= bit
* find_next_and_bit(addr1, addr2, nbits, bit)
* Same as find_next_bit, but in
* (*addr1 & *addr2)
*
*/
/**
* DOC: declare bitmap
* The DECLARE_BITMAP(name,bits) macro, in linux/types.h, can be used
* to declare an array named 'name' of just enough unsigned longs to
* contain all bit positions from 0 to 'bits' - 1.
*/
/*
* Allocation and deallocation of bitmap.
* Provided in lib/bitmap.c to avoid circular dependency.
*/
unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags);
unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags);
unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node);
unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node);
void bitmap_free(const unsigned long *bitmap);
DEFINE_FREE(bitmap, unsigned long *, if (_T) bitmap_free(_T))
/* Managed variants of the above. */
unsigned long *devm_bitmap_alloc(struct device *dev,
unsigned int nbits, gfp_t flags);
unsigned long *devm_bitmap_zalloc(struct device *dev,
unsigned int nbits, gfp_t flags);
/*
* lib/bitmap.c provides these functions:
*/
bool __bitmap_equal(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __pure __bitmap_or_equal(const unsigned long *src1,
const unsigned long *src2,
const unsigned long *src3,
unsigned int nbits);
void __bitmap_complement(unsigned long *dst, const unsigned long *src,
unsigned int nbits);
void __bitmap_shift_right(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits);
void __bitmap_shift_left(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits);
void bitmap_cut(unsigned long *dst, const unsigned long *src,
unsigned int first, unsigned int cut, unsigned int nbits);
bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_replace(unsigned long *dst,
const unsigned long *old, const unsigned long *new,
const unsigned long *mask, unsigned int nbits);
bool __bitmap_intersects(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __bitmap_subset(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);
unsigned int __bitmap_weight_and(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_set(unsigned long *map, unsigned int start, int len);
void __bitmap_clear(unsigned long *map, unsigned int start, int len);
unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
unsigned long size,
unsigned long start,
unsigned int nr,
unsigned long align_mask,
unsigned long align_offset);
/**
* bitmap_find_next_zero_area - find a contiguous aligned zero area
* @map: The address to base the search on
* @size: The bitmap size in bits
* @start: The bitnumber to start searching at
* @nr: The number of zeroed bits we're looking for
* @align_mask: Alignment mask for zero area
*
* The @align_mask should be one less than a power of 2; the effect is that
* the bit offset of all zero areas this function finds is multiples of that
* power of 2. A @align_mask of 0 means no alignment is required.
*/
static inline unsigned long
bitmap_find_next_zero_area(unsigned long *map,
unsigned long size,
unsigned long start,
unsigned int nr,
unsigned long align_mask)
{
return bitmap_find_next_zero_area_off(map, size, start, nr,
align_mask, 0);
}
void bitmap_remap(unsigned long *dst, const unsigned long *src,
const unsigned long *old, const unsigned long *new, unsigned int nbits);
int bitmap_bitremap(int oldbit,
const unsigned long *old, const unsigned long *new, int bits);
void bitmap_onto(unsigned long *dst, const unsigned long *orig,
const unsigned long *relmap, unsigned int bits);
void bitmap_fold(unsigned long *dst, const unsigned long *orig,
unsigned int sz, unsigned int nbits);
#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1)))
#define BITMAP_LAST_WORD_MASK(nbits) (~0UL >> (-(nbits) & (BITS_PER_LONG - 1)))
static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
{
unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
if (small_const_nbits(nbits))
*dst = 0;
else
memset(dst, 0, len);
}
static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
{
unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
if (small_const_nbits(nbits))
*dst = ~0UL;
else
memset(dst, 0xff, len);
}
static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
unsigned int nbits)
{
unsigned int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
if (small_const_nbits(nbits))
*dst = *src;
else
memcpy(dst, src, len);
}
/*
* Copy bitmap and clear tail bits in last word.
*/
static inline void bitmap_copy_clear_tail(unsigned long *dst,
const unsigned long *src, unsigned int nbits)
{
bitmap_copy(dst, src, nbits);
if (nbits % BITS_PER_LONG)
dst[nbits / BITS_PER_LONG] &= BITMAP_LAST_WORD_MASK(nbits);
}
/*
* On 32-bit systems bitmaps are represented as u32 arrays internally. On LE64
* machines the order of hi and lo parts of numbers match the bitmap structure.
* In both cases conversion is not needed when copying data from/to arrays of
* u32. But in LE64 case, typecast in bitmap_copy_clear_tail() may lead
* to out-of-bound access. To avoid that, both LE and BE variants of 64-bit
* architectures are not using bitmap_copy_clear_tail().
*/
#if BITS_PER_LONG == 64
void bitmap_from_arr32(unsigned long *bitmap, const u32 *buf,
unsigned int nbits);
void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap,
unsigned int nbits);
#else
#define bitmap_from_arr32(bitmap, buf, nbits) \
bitmap_copy_clear_tail((unsigned long *) (bitmap), \
(const unsigned long *) (buf), (nbits))
#define bitmap_to_arr32(buf, bitmap, nbits) \
bitmap_copy_clear_tail((unsigned long *) (buf), \
(const unsigned long *) (bitmap), (nbits))
#endif
/*
* On 64-bit systems bitmaps are represented as u64 arrays internally. So,
* the conversion is not needed when copying data from/to arrays of u64.
*/
#if BITS_PER_LONG == 32
void bitmap_from_arr64(unsigned long *bitmap, const u64 *buf, unsigned int nbits);
void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits);
#else
#define bitmap_from_arr64(bitmap, buf, nbits) \
bitmap_copy_clear_tail((unsigned long *)(bitmap), (const unsigned long *)(buf), (nbits))
#define bitmap_to_arr64(buf, bitmap, nbits) \
bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
#endif
static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
return __bitmap_and(dst, src1, src2, nbits);
}
static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
*dst = *src1 | *src2;
else
__bitmap_or(dst, src1, src2, nbits);
}
static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
*dst = *src1 ^ *src2;
else
__bitmap_xor(dst, src1, src2, nbits);
}
static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
return __bitmap_andnot(dst, src1, src2, nbits);
}
static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
unsigned int nbits)
{
if (small_const_nbits(nbits))
*dst = ~(*src);
else
__bitmap_complement(dst, src, nbits);
}
#ifdef __LITTLE_ENDIAN
#define BITMAP_MEM_ALIGNMENT 8
#else
#define BITMAP_MEM_ALIGNMENT (8 * sizeof(unsigned long))
#endif
#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
static inline bool bitmap_equal(const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
if (__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
IS_ALIGNED(nbits, BITMAP_MEM_ALIGNMENT))
return !memcmp(src1, src2, nbits / 8);
return __bitmap_equal(src1, src2, nbits);
}
/**
* bitmap_or_equal - Check whether the or of two bitmaps is equal to a third
* @src1: Pointer to bitmap 1
* @src2: Pointer to bitmap 2 will be or'ed with bitmap 1
* @src3: Pointer to bitmap 3. Compare to the result of *@src1 | *@src2
* @nbits: number of bits in each of these bitmaps
*
* Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
*/
static inline bool bitmap_or_equal(const unsigned long *src1,
const unsigned long *src2,
const unsigned long *src3,
unsigned int nbits)
{
if (!small_const_nbits(nbits))
return __bitmap_or_equal(src1, src2, src3, nbits);
return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
}
static inline bool bitmap_intersects(const unsigned long *src1,
const unsigned long *src2,
unsigned int nbits)
{
if (small_const_nbits(nbits))
return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
else
return __bitmap_intersects(src1, src2, nbits);
}
static inline bool bitmap_subset(const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
else
return __bitmap_subset(src1, src2, nbits);
}
static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
{
if (small_const_nbits(nbits))
return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
return find_first_bit(src, nbits) == nbits;
}
static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)
{
if (small_const_nbits(nbits))
return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
return find_first_zero_bit(src, nbits) == nbits;
}
static __always_inline
unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits)
{
if (small_const_nbits(nbits))
return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits));
return __bitmap_weight(src, nbits);
}
static __always_inline
unsigned long bitmap_weight_and(const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits));
return __bitmap_weight_and(src1, src2, nbits);
}
static __always_inline
unsigned long bitmap_weight_andnot(const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return hweight_long(*src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits));
return __bitmap_weight_andnot(src1, src2, nbits);
}
static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
unsigned int nbits)
{
if (__builtin_constant_p(nbits) && nbits == 1)
__set_bit(start, map);
else if (small_const_nbits(start + nbits))
*map |= GENMASK(start + nbits - 1, start);
else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&
IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&
__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
IS_ALIGNED(nbits, BITMAP_MEM_ALIGNMENT))
memset((char *)map + start / 8, 0xff, nbits / 8);
else
__bitmap_set(map, start, nbits);
}
static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
unsigned int nbits)
{
if (__builtin_constant_p(nbits) && nbits == 1)
__clear_bit(start, map);
else if (small_const_nbits(start + nbits))
*map &= ~GENMASK(start + nbits - 1, start);
else if (__builtin_constant_p(start & BITMAP_MEM_MASK) &&
IS_ALIGNED(start, BITMAP_MEM_ALIGNMENT) &&
__builtin_constant_p(nbits & BITMAP_MEM_MASK) &&
IS_ALIGNED(nbits, BITMAP_MEM_ALIGNMENT))
memset((char *)map + start / 8, 0, nbits / 8);
else
__bitmap_clear(map, start, nbits);
}
static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits)
{
if (small_const_nbits(nbits))
*dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
else
__bitmap_shift_right(dst, src, shift, nbits);
}
static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
unsigned int shift, unsigned int nbits)
{
if (small_const_nbits(nbits))
*dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
else
__bitmap_shift_left(dst, src, shift, nbits);
}
static inline void bitmap_replace(unsigned long *dst,
const unsigned long *old,
const unsigned long *new,
const unsigned long *mask,
unsigned int nbits)
{
if (small_const_nbits(nbits))
*dst = (*old & ~(*mask)) | (*new & *mask);
else
__bitmap_replace(dst, old, new, mask, nbits);
}
/**
* bitmap_scatter - Scatter a bitmap according to the given mask
* @dst: scattered bitmap
* @src: gathered bitmap
* @mask: mask representing bits to assign to in the scattered bitmap
* @nbits: number of bits in each of these bitmaps
*
* Scatters bitmap with sequential bits according to the given @mask.
*
* Example:
* If @src bitmap = 0x005a, with @mask = 0x1313, @dst will be 0x0302.
*
* Or in binary form
* @src @mask @dst
* 0000000001011010 0001001100010011 0000001100000010
*
* (Bits 0, 1, 2, 3, 4, 5 are copied to the bits 0, 1, 4, 8, 9, 12)
*
* A more 'visual' description of the operation::
*
* src: 0000000001011010
* ||||||
* +------+|||||
* | +----+||||
* | |+----+|||
* | || +-+||
* | || | ||
* mask: ...v..vv...v..vv
* ...0..11...0..10
* dst: 0000001100000010
*
* A relationship exists between bitmap_scatter() and bitmap_gather().
* bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
* See bitmap_scatter() for details related to this relationship.
*/
static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,
const unsigned long *mask, unsigned int nbits)
{
unsigned int n = 0;
unsigned int bit;
bitmap_zero(dst, nbits);
for_each_set_bit(bit, mask, nbits)
__assign_bit(bit, dst, test_bit(n++, src));
}
/**
* bitmap_gather - Gather a bitmap according to given mask
* @dst: gathered bitmap
* @src: scattered bitmap
* @mask: mask representing bits to extract from in the scattered bitmap
* @nbits: number of bits in each of these bitmaps
*
* Gathers bitmap with sparse bits according to the given @mask.
*
* Example:
* If @src bitmap = 0x0302, with @mask = 0x1313, @dst will be 0x001a.
*
* Or in binary form
* @src @mask @dst
* 0000001100000010 0001001100010011 0000000000011010
*
* (Bits 0, 1, 4, 8, 9, 12 are copied to the bits 0, 1, 2, 3, 4, 5)
*
* A more 'visual' description of the operation::
*
* mask: ...v..vv...v..vv
* src: 0000001100000010
* ^ ^^ ^ 0
* | || | 10
* | || > 010
* | |+--> 1010
* | +--> 11010
* +----> 011010
* dst: 0000000000011010
*
* A relationship exists between bitmap_gather() and bitmap_scatter(). See
* bitmap_scatter() for the bitmap scatter detailed operations.
* Suppose scattered computed using bitmap_scatter(scattered, src, mask, n).
* The operation bitmap_gather(result, scattered, mask, n) leads to a result
* equal or equivalent to src.
*
* The result can be 'equivalent' because bitmap_scatter() and bitmap_gather()
* are not bijective.
* The result and src values are equivalent in that sense that a call to
* bitmap_scatter(res, src, mask, n) and a call to
* bitmap_scatter(res, result, mask, n) will lead to the same res value.
*/
static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,
const unsigned long *mask, unsigned int nbits)
{
unsigned int n = 0;
unsigned int bit;
bitmap_zero(dst, nbits);
for_each_set_bit(bit, mask, nbits)
__assign_bit(n++, dst, test_bit(bit, src));
}
static inline void bitmap_next_set_region(unsigned long *bitmap,
unsigned int *rs, unsigned int *re,
unsigned int end)
{
*rs = find_next_bit(bitmap, end, *rs);
*re = find_next_zero_bit(bitmap, end, *rs + 1);
}
/**
* bitmap_release_region - release allocated bitmap region
* @bitmap: array of unsigned longs corresponding to the bitmap
* @pos: beginning of bit region to release
* @order: region size (log base 2 of number of bits) to release
*
* This is the complement to __bitmap_find_free_region() and releases
* the found region (by clearing it in the bitmap).
*/
static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
{
bitmap_clear(bitmap, pos, BIT(order));
}
/**
* bitmap_allocate_region - allocate bitmap region
* @bitmap: array of unsigned longs corresponding to the bitmap
* @pos: beginning of bit region to allocate
* @order: region size (log base 2 of number of bits) to allocate
*
* Allocate (set bits in) a specified region of a bitmap.
*
* Returns: 0 on success, or %-EBUSY if specified region wasn't
* free (not all bits were zero).
*/
static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
{
unsigned int len = BIT(order);
if (find_next_bit(bitmap, pos + len, pos) < pos + len)
return -EBUSY;
bitmap_set(bitmap, pos, len);
return 0;
}
/**
* bitmap_find_free_region - find a contiguous aligned mem region
* @bitmap: array of unsigned longs corresponding to the bitmap
* @bits: number of bits in the bitmap
* @order: region size (log base 2 of number of bits) to find
*
* Find a region of free (zero) bits in a @bitmap of @bits bits and
* allocate them (set them to one). Only consider regions of length
* a power (@order) of two, aligned to that power of two, which
* makes the search algorithm much faster.
*
* Returns: the bit offset in bitmap of the allocated region,
* or -errno on failure.
*/
static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
{
unsigned int pos, end; /* scans bitmap by regions of size order */
for (pos = 0; (end = pos + BIT(order)) <= bits; pos = end) {
if (!bitmap_allocate_region(bitmap, pos, order))
return pos;
}
return -ENOMEM;
}
/**
* BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap.
* @n: u64 value
*
* Linux bitmaps are internally arrays of unsigned longs, i.e. 32-bit
* integers in 32-bit environment, and 64-bit integers in 64-bit one.
*
* There are four combinations of endianness and length of the word in linux
* ABIs: LE64, BE64, LE32 and BE32.
*
* On 64-bit kernels 64-bit LE and BE numbers are naturally ordered in
* bitmaps and therefore don't require any special handling.
*
* On 32-bit kernels 32-bit LE ABI orders lo word of 64-bit number in memory
* prior to hi, and 32-bit BE orders hi word prior to lo. The bitmap on the
* other hand is represented as an array of 32-bit words and the position of
* bit N may therefore be calculated as: word #(N/32) and bit #(N%32) in that
* word. For example, bit #42 is located at 10th position of 2nd word.
* It matches 32-bit LE ABI, and we can simply let the compiler store 64-bit
* values in memory as it usually does. But for BE we need to swap hi and lo
* words manually.
*
* With all that, the macro BITMAP_FROM_U64() does explicit reordering of hi and
* lo parts of u64. For LE32 it does nothing, and for BE environment it swaps
* hi and lo words, as is expected by bitmap.
*/
#if __BITS_PER_LONG == 64
#define BITMAP_FROM_U64(n) (n)
#else
#define BITMAP_FROM_U64(n) ((unsigned long) ((u64)(n) & ULONG_MAX)), \
((unsigned long) ((u64)(n) >> 32))
#endif
/**
* bitmap_from_u64 - Check and swap words within u64.
* @mask: source bitmap
* @dst: destination bitmap
*
* In 32-bit Big Endian kernel, when using ``(u32 *)(&val)[*]``
* to read u64 mask, we will get the wrong word.
* That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
* but we expect the lower 32-bits of u64.
*/
static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
{
bitmap_from_arr64(dst, &mask, 64);
}
/**
* bitmap_get_value8 - get an 8-bit value within a memory region
* @map: address to the bitmap memory region
* @start: bit offset of the 8-bit value; must be a multiple of 8
*
* Returns the 8-bit value located at the @start bit offset within the @src
* memory region.
*/
static inline unsigned long bitmap_get_value8(const unsigned long *map,
unsigned long start)
{
const size_t index = BIT_WORD(start);
const unsigned long offset = start % BITS_PER_LONG;
return (map[index] >> offset) & 0xFF;
}
/**
* bitmap_set_value8 - set an 8-bit value within a memory region
* @map: address to the bitmap memory region
* @value: the 8-bit value; values wider than 8 bits may clobber bitmap
* @start: bit offset of the 8-bit value; must be a multiple of 8
*/
static inline void bitmap_set_value8(unsigned long *map, unsigned long value,
unsigned long start)
{
const size_t index = BIT_WORD(start);
const unsigned long offset = start % BITS_PER_LONG;
map[index] &= ~(0xFFUL << offset);
map[index] |= value << offset;
}
#endif /* __ASSEMBLY__ */
#endif /* __LINUX_BITMAP_H */
// SPDX-License-Identifier: GPL-2.0
/*
* <linux/usb/gadget.h>
*
* We call the USB code inside a Linux-based peripheral device a "gadget"
* driver, except for the hardware-specific bus glue. One USB host can
* talk to many USB gadgets, but the gadgets are only able to communicate
* to one host.
*
*
* (C) Copyright 2002-2004 by David Brownell
* All Rights Reserved.
*/
#ifndef __LINUX_USB_GADGET_H
#define __LINUX_USB_GADGET_H
#include <linux/configfs.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/scatterlist.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <linux/usb/ch9.h>
#define UDC_TRACE_STR_MAX 512
struct usb_ep;
/**
* struct usb_request - describes one i/o request
* @buf: Buffer used for data. Always provide this; some controllers
* only use PIO, or don't use DMA for some endpoints.
* @dma: DMA address corresponding to 'buf'. If you don't set this
* field, and the usb controller needs one, it is responsible
* for mapping and unmapping the buffer.
* @sg: a scatterlist for SG-capable controllers.
* @num_sgs: number of SG entries
* @num_mapped_sgs: number of SG entries mapped to DMA (internal)
* @length: Length of that data
* @stream_id: The stream id, when USB3.0 bulk streams are being used
* @is_last: Indicates if this is the last request of a stream_id before
* switching to a different stream (required for DWC3 controllers).
* @no_interrupt: If true, hints that no completion irq is needed.
* Helpful sometimes with deep request queues that are handled
* directly by DMA controllers.
* @zero: If true, when writing data, makes the last packet be "short"
* by adding a zero length packet as needed;
* @short_not_ok: When reading data, makes short packets be
* treated as errors (queue stops advancing till cleanup).
* @dma_mapped: Indicates if request has been mapped to DMA (internal)
* @sg_was_mapped: Set if the scatterlist has been mapped before the request
* @complete: Function called when request completes, so this request and
* its buffer may be re-used. The function will always be called with
* interrupts disabled, and it must not sleep.
* Reads terminate with a short packet, or when the buffer fills,
* whichever comes first. When writes terminate, some data bytes
* will usually still be in flight (often in a hardware fifo).
* Errors (for reads or writes) stop the queue from advancing
* until the completion function returns, so that any transfers
* invalidated by the error may first be dequeued.
* @context: For use by the completion callback
* @list: For use by the gadget driver.
* @frame_number: Reports the interval number in (micro)frame in which the
* isochronous transfer was transmitted or received.
* @status: Reports completion code, zero or a negative errno.
* Normally, faults block the transfer queue from advancing until
* the completion callback returns.
* Code "-ESHUTDOWN" indicates completion caused by device disconnect,
* or when the driver disabled the endpoint.
* @actual: Reports bytes transferred to/from the buffer. For reads (OUT
* transfers) this may be less than the requested length. If the
* short_not_ok flag is set, short reads are treated as errors
* even when status otherwise indicates successful completion.
* Note that for writes (IN transfers) some data bytes may still
* reside in a device-side FIFO when the request is reported as
* complete.
*
* These are allocated/freed through the endpoint they're used with. The
* hardware's driver can add extra per-request data to the memory it returns,
* which often avoids separate memory allocations (potential failures),
* later when the request is queued.
*
* Request flags affect request handling, such as whether a zero length
* packet is written (the "zero" flag), whether a short read should be
* treated as an error (blocking request queue advance, the "short_not_ok"
* flag), or hinting that an interrupt is not required (the "no_interrupt"
* flag, for use with deep request queues).
*
* Bulk endpoints can use any size buffers, and can also be used for interrupt
* transfers. interrupt-only endpoints can be much less functional.
*
* NOTE: this is analogous to 'struct urb' on the host side, except that
* it's thinner and promotes more pre-allocation.
*/
struct usb_request {
void *buf;
unsigned length;
dma_addr_t dma;
struct scatterlist *sg;
unsigned num_sgs;
unsigned num_mapped_sgs;
unsigned stream_id:16;
unsigned is_last:1;
unsigned no_interrupt:1;
unsigned zero:1;
unsigned short_not_ok:1;
unsigned dma_mapped:1;
unsigned sg_was_mapped:1;
void (*complete)(struct usb_ep *ep,
struct usb_request *req);
void *context;
struct list_head list;
unsigned frame_number; /* ISO ONLY */
int status;
unsigned actual;
};
/*-------------------------------------------------------------------------*/
/* endpoint-specific parts of the api to the usb controller hardware.
* unlike the urb model, (de)multiplexing layers are not required.
* (so this api could slash overhead if used on the host side...)
*
* note that device side usb controllers commonly differ in how many
* endpoints they support, as well as their capabilities.
*/
struct usb_ep_ops {
int (*enable) (struct usb_ep *ep,
const struct usb_endpoint_descriptor *desc);
int (*disable) (struct usb_ep *ep);
void (*dispose) (struct usb_ep *ep);
struct usb_request *(*alloc_request) (struct usb_ep *ep,
gfp_t gfp_flags);
void (*free_request) (struct usb_ep *ep, struct usb_request *req);
int (*queue) (struct usb_ep *ep, struct usb_request *req,
gfp_t gfp_flags);
int (*dequeue) (struct usb_ep *ep, struct usb_request *req);
int (*set_halt) (struct usb_ep *ep, int value);
int (*set_wedge) (struct usb_ep *ep);
int (*fifo_status) (struct usb_ep *ep);
void (*fifo_flush) (struct usb_ep *ep);
};
/**
* struct usb_ep_caps - endpoint capabilities description
* @type_control:Endpoint supports control type (reserved for ep0).
* @type_iso:Endpoint supports isochronous transfers.
* @type_bulk:Endpoint supports bulk transfers.
* @type_int:Endpoint supports interrupt transfers.
* @dir_in:Endpoint supports IN direction.
* @dir_out:Endpoint supports OUT direction.
*/
struct usb_ep_caps {
unsigned type_control:1;
unsigned type_iso:1;
unsigned type_bulk:1;
unsigned type_int:1;
unsigned dir_in:1;
unsigned dir_out:1;
};
#define USB_EP_CAPS_TYPE_CONTROL 0x01
#define USB_EP_CAPS_TYPE_ISO 0x02
#define USB_EP_CAPS_TYPE_BULK 0x04
#define USB_EP_CAPS_TYPE_INT 0x08
#define USB_EP_CAPS_TYPE_ALL \
(USB_EP_CAPS_TYPE_ISO | USB_EP_CAPS_TYPE_BULK | USB_EP_CAPS_TYPE_INT)
#define USB_EP_CAPS_DIR_IN 0x01
#define USB_EP_CAPS_DIR_OUT 0x02
#define USB_EP_CAPS_DIR_ALL (USB_EP_CAPS_DIR_IN | USB_EP_CAPS_DIR_OUT)
#define USB_EP_CAPS(_type, _dir) \
{ \
.type_control = !!(_type & USB_EP_CAPS_TYPE_CONTROL), \
.type_iso = !!(_type & USB_EP_CAPS_TYPE_ISO), \
.type_bulk = !!(_type & USB_EP_CAPS_TYPE_BULK), \
.type_int = !!(_type & USB_EP_CAPS_TYPE_INT), \
.dir_in = !!(_dir & USB_EP_CAPS_DIR_IN), \
.dir_out = !!(_dir & USB_EP_CAPS_DIR_OUT), \
}
/**
* struct usb_ep - device side representation of USB endpoint
* @name:identifier for the endpoint, such as "ep-a" or "ep9in-bulk"
* @ops: Function pointers used to access hardware-specific operations.
* @ep_list:the gadget's ep_list holds all of its endpoints
* @caps:The structure describing types and directions supported by endpoint.
* @enabled: The current endpoint enabled/disabled state.
* @claimed: True if this endpoint is claimed by a function.
* @maxpacket:The maximum packet size used on this endpoint. The initial
* value can sometimes be reduced (hardware allowing), according to
* the endpoint descriptor used to configure the endpoint.
* @maxpacket_limit:The maximum packet size value which can be handled by this
* endpoint. It's set once by UDC driver when endpoint is initialized, and
* should not be changed. Should not be confused with maxpacket.
* @max_streams: The maximum number of streams supported
* by this EP (0 - 16, actual number is 2^n)
* @mult: multiplier, 'mult' value for SS Isoc EPs
* @maxburst: the maximum number of bursts supported by this EP (for usb3)
* @driver_data:for use by the gadget driver.
* @address: used to identify the endpoint when finding descriptor that
* matches connection speed
* @desc: endpoint descriptor. This pointer is set before the endpoint is
* enabled and remains valid until the endpoint is disabled.
* @comp_desc: In case of SuperSpeed support, this is the endpoint companion
* descriptor that is used to configure the endpoint
*
* the bus controller driver lists all the general purpose endpoints in
* gadget->ep_list. the control endpoint (gadget->ep0) is not in that list,
* and is accessed only in response to a driver setup() callback.
*/
struct usb_ep {
void *driver_data;
const char *name;
const struct usb_ep_ops *ops;
struct list_head ep_list;
struct usb_ep_caps caps;
bool claimed;
bool enabled;
unsigned maxpacket:16;
unsigned maxpacket_limit:16;
unsigned max_streams:16;
unsigned mult:2;
unsigned maxburst:5;
u8 address;
const struct usb_endpoint_descriptor *desc;
const struct usb_ss_ep_comp_descriptor *comp_desc;
};
/*-------------------------------------------------------------------------*/
#if IS_ENABLED(CONFIG_USB_GADGET)
void usb_ep_set_maxpacket_limit(struct usb_ep *ep, unsigned maxpacket_limit);
int usb_ep_enable(struct usb_ep *ep);
int usb_ep_disable(struct usb_ep *ep);
struct usb_request *usb_ep_alloc_request(struct usb_ep *ep, gfp_t gfp_flags);
void usb_ep_free_request(struct usb_ep *ep, struct usb_request *req);
int usb_ep_queue(struct usb_ep *ep, struct usb_request *req, gfp_t gfp_flags);
int usb_ep_dequeue(struct usb_ep *ep, struct usb_request *req);
int usb_ep_set_halt(struct usb_ep *ep);
int usb_ep_clear_halt(struct usb_ep *ep);
int usb_ep_set_wedge(struct usb_ep *ep);
int usb_ep_fifo_status(struct usb_ep *ep);
void usb_ep_fifo_flush(struct usb_ep *ep);
#else
static inline void usb_ep_set_maxpacket_limit(struct usb_ep *ep,
unsigned maxpacket_limit)
{ }
static inline int usb_ep_enable(struct usb_ep *ep)
{ return 0; }
static inline int usb_ep_disable(struct usb_ep *ep)
{ return 0; }
static inline struct usb_request *usb_ep_alloc_request(struct usb_ep *ep,
gfp_t gfp_flags)
{ return NULL; }
static inline void usb_ep_free_request(struct usb_ep *ep,
struct usb_request *req)
{ }
static inline int usb_ep_queue(struct usb_ep *ep, struct usb_request *req,
gfp_t gfp_flags)
{ return 0; }
static inline int usb_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
{ return 0; }
static inline int usb_ep_set_halt(struct usb_ep *ep)
{ return 0; }
static inline int usb_ep_clear_halt(struct usb_ep *ep)
{ return 0; }
static inline int usb_ep_set_wedge(struct usb_ep *ep)
{ return 0; }
static inline int usb_ep_fifo_status(struct usb_ep *ep)
{ return 0; }
static inline void usb_ep_fifo_flush(struct usb_ep *ep)
{ }
#endif /* USB_GADGET */
/*-------------------------------------------------------------------------*/
struct usb_dcd_config_params {
__u8 bU1devExitLat; /* U1 Device exit Latency */
#define USB_DEFAULT_U1_DEV_EXIT_LAT 0x01 /* Less then 1 microsec */
__le16 bU2DevExitLat; /* U2 Device exit Latency */
#define USB_DEFAULT_U2_DEV_EXIT_LAT 0x1F4 /* Less then 500 microsec */
__u8 besl_baseline; /* Recommended baseline BESL (0-15) */
__u8 besl_deep; /* Recommended deep BESL (0-15) */
#define USB_DEFAULT_BESL_UNSPECIFIED 0xFF /* No recommended value */
};
struct usb_gadget;
struct usb_gadget_driver;
struct usb_udc;
/* the rest of the api to the controller hardware: device operations,
* which don't involve endpoints (or i/o).
*/
struct usb_gadget_ops {
int (*get_frame)(struct usb_gadget *);
int (*wakeup)(struct usb_gadget *);
int (*func_wakeup)(struct usb_gadget *gadget, int intf_id);
int (*set_remote_wakeup)(struct usb_gadget *, int set);
int (*set_selfpowered) (struct usb_gadget *, int is_selfpowered);
int (*vbus_session) (struct usb_gadget *, int is_active);
int (*vbus_draw) (struct usb_gadget *, unsigned mA);
int (*pullup) (struct usb_gadget *, int is_on);
int (*ioctl)(struct usb_gadget *,
unsigned code, unsigned long param);
void (*get_config_params)(struct usb_gadget *,
struct usb_dcd_config_params *);
int (*udc_start)(struct usb_gadget *,
struct usb_gadget_driver *);
int (*udc_stop)(struct usb_gadget *);
void (*udc_set_speed)(struct usb_gadget *, enum usb_device_speed);
void (*udc_set_ssp_rate)(struct usb_gadget *gadget,
enum usb_ssp_rate rate);
void (*udc_async_callbacks)(struct usb_gadget *gadget, bool enable);
struct usb_ep *(*match_ep)(struct usb_gadget *,
struct usb_endpoint_descriptor *,
struct usb_ss_ep_comp_descriptor *);
int (*check_config)(struct usb_gadget *gadget);
};
/**
* struct usb_gadget - represents a usb device
* @work: (internal use) Workqueue to be used for sysfs_notify()
* @udc: struct usb_udc pointer for this gadget
* @ops: Function pointers used to access hardware-specific operations.
* @ep0: Endpoint zero, used when reading or writing responses to
* driver setup() requests
* @ep_list: List of other endpoints supported by the device.
* @speed: Speed of current connection to USB host.
* @max_speed: Maximal speed the UDC can handle. UDC must support this
* and all slower speeds.
* @ssp_rate: Current connected SuperSpeed Plus signaling rate and lane count.
* @max_ssp_rate: Maximum SuperSpeed Plus signaling rate and lane count the UDC
* can handle. The UDC must support this and all slower speeds and lower
* number of lanes.
* @state: the state we are now (attached, suspended, configured, etc)
* @name: Identifies the controller hardware type. Used in diagnostics
* and sometimes configuration.
* @dev: Driver model state for this abstract device.
* @isoch_delay: value from Set Isoch Delay request. Only valid on SS/SSP
* @out_epnum: last used out ep number
* @in_epnum: last used in ep number
* @mA: last set mA value
* @otg_caps: OTG capabilities of this gadget.
* @sg_supported: true if we can handle scatter-gather
* @is_otg: True if the USB device port uses a Mini-AB jack, so that the
* gadget driver must provide a USB OTG descriptor.
* @is_a_peripheral: False unless is_otg, the "A" end of a USB cable
* is in the Mini-AB jack, and HNP has been used to switch roles
* so that the "A" device currently acts as A-Peripheral, not A-Host.
* @a_hnp_support: OTG device feature flag, indicating that the A-Host
* supports HNP at this port.
* @a_alt_hnp_support: OTG device feature flag, indicating that the A-Host
* only supports HNP on a different root port.
* @b_hnp_enable: OTG device feature flag, indicating that the A-Host
* enabled HNP support.
* @hnp_polling_support: OTG device feature flag, indicating if the OTG device
* in peripheral mode can support HNP polling.
* @host_request_flag: OTG device feature flag, indicating if A-Peripheral
* or B-Peripheral wants to take host role.
* @quirk_ep_out_aligned_size: epout requires buffer size to be aligned to
* MaxPacketSize.
* @quirk_altset_not_supp: UDC controller doesn't support alt settings.
* @quirk_stall_not_supp: UDC controller doesn't support stalling.
* @quirk_zlp_not_supp: UDC controller doesn't support ZLP.
* @quirk_avoids_skb_reserve: udc/platform wants to avoid skb_reserve() in
* u_ether.c to improve performance.
* @is_selfpowered: if the gadget is self-powered.
* @deactivated: True if gadget is deactivated - in deactivated state it cannot
* be connected.
* @connected: True if gadget is connected.
* @lpm_capable: If the gadget max_speed is FULL or HIGH, this flag
* indicates that it supports LPM as per the LPM ECN & errata.
* @wakeup_capable: True if gadget is capable of sending remote wakeup.
* @wakeup_armed: True if gadget is armed by the host for remote wakeup.
* @irq: the interrupt number for device controller.
* @id_number: a unique ID number for ensuring that gadget names are distinct
*
* Gadgets have a mostly-portable "gadget driver" implementing device
* functions, handling all usb configurations and interfaces. Gadget
* drivers talk to hardware-specific code indirectly, through ops vectors.
* That insulates the gadget driver from hardware details, and packages
* the hardware endpoints through generic i/o queues. The "usb_gadget"
* and "usb_ep" interfaces provide that insulation from the hardware.
*
* Except for the driver data, all fields in this structure are
* read-only to the gadget driver. That driver data is part of the
* "driver model" infrastructure in 2.6 (and later) kernels, and for
* earlier systems is grouped in a similar structure that's not known
* to the rest of the kernel.
*
* Values of the three OTG device feature flags are updated before the
* setup() call corresponding to USB_REQ_SET_CONFIGURATION, and before
* driver suspend() calls. They are valid only when is_otg, and when the
* device is acting as a B-Peripheral (so is_a_peripheral is false).
*/
struct usb_gadget {
struct work_struct work;
struct usb_udc *udc;
/* readonly to gadget driver */
const struct usb_gadget_ops *ops;
struct usb_ep *ep0;
struct list_head ep_list; /* of usb_ep */
enum usb_device_speed speed;
enum usb_device_speed max_speed;
/* USB SuperSpeed Plus only */
enum usb_ssp_rate ssp_rate;
enum usb_ssp_rate max_ssp_rate;
enum usb_device_state state;
const char *name;
struct device dev;
unsigned isoch_delay;
unsigned out_epnum;
unsigned in_epnum;
unsigned mA;
struct usb_otg_caps *otg_caps;
unsigned sg_supported:1;
unsigned is_otg:1;
unsigned is_a_peripheral:1;
unsigned b_hnp_enable:1;
unsigned a_hnp_support:1;
unsigned a_alt_hnp_support:1;
unsigned hnp_polling_support:1;
unsigned host_request_flag:1;
unsigned quirk_ep_out_aligned_size:1;
unsigned quirk_altset_not_supp:1;
unsigned quirk_stall_not_supp:1;
unsigned quirk_zlp_not_supp:1;
unsigned quirk_avoids_skb_reserve:1;
unsigned is_selfpowered:1;
unsigned deactivated:1;
unsigned connected:1;
unsigned lpm_capable:1;
unsigned wakeup_capable:1;
unsigned wakeup_armed:1;
int irq;
int id_number;
};
#define work_to_gadget(w) (container_of((w), struct usb_gadget, work))
/* Interface to the device model */
static inline void set_gadget_data(struct usb_gadget *gadget, void *data)
{ dev_set_drvdata(&gadget->dev, data); }
static inline void *get_gadget_data(struct usb_gadget *gadget)
{ return dev_get_drvdata(&gadget->dev); }
static inline struct usb_gadget *dev_to_usb_gadget(struct device *dev)
{
return container_of(dev, struct usb_gadget, dev);
}
static inline struct usb_gadget *usb_get_gadget(struct usb_gadget *gadget)
{
get_device(&gadget->dev);
return gadget;
}
static inline void usb_put_gadget(struct usb_gadget *gadget)
{
put_device(&gadget->dev);
}
extern void usb_initialize_gadget(struct device *parent,
struct usb_gadget *gadget, void (*release)(struct device *dev));
extern int usb_add_gadget(struct usb_gadget *gadget);
extern void usb_del_gadget(struct usb_gadget *gadget);
/* Legacy device-model interface */
extern int usb_add_gadget_udc_release(struct device *parent,
struct usb_gadget *gadget, void (*release)(struct device *dev));
extern int usb_add_gadget_udc(struct device *parent, struct usb_gadget *gadget);
extern void usb_del_gadget_udc(struct usb_gadget *gadget);
extern char *usb_get_gadget_udc_name(void);
/* iterates the non-control endpoints; 'tmp' is a struct usb_ep pointer */
#define gadget_for_each_ep(tmp, gadget) \
list_for_each_entry(tmp, &(gadget)->ep_list, ep_list)
/**
* usb_ep_align - returns @len aligned to ep's maxpacketsize.
* @ep: the endpoint whose maxpacketsize is used to align @len
* @len: buffer size's length to align to @ep's maxpacketsize
*
* This helper is used to align buffer's size to an ep's maxpacketsize.
*/
static inline size_t usb_ep_align(struct usb_ep *ep, size_t len)
{
int max_packet_size = (size_t)usb_endpoint_maxp(ep->desc);
return round_up(len, max_packet_size);
}
/**
* usb_ep_align_maybe - returns @len aligned to ep's maxpacketsize if gadget
* requires quirk_ep_out_aligned_size, otherwise returns len.
* @g: controller to check for quirk
* @ep: the endpoint whose maxpacketsize is used to align @len
* @len: buffer size's length to align to @ep's maxpacketsize
*
* This helper is used in case it's required for any reason to check and maybe
* align buffer's size to an ep's maxpacketsize.
*/
static inline size_t
usb_ep_align_maybe(struct usb_gadget *g, struct usb_ep *ep, size_t len)
{
return g->quirk_ep_out_aligned_size ? usb_ep_align(ep, len) : len;
}
/**
* gadget_is_altset_supported - return true iff the hardware supports
* altsettings
* @g: controller to check for quirk
*/
static inline int gadget_is_altset_supported(struct usb_gadget *g)
{
return !g->quirk_altset_not_supp;
}
/**
* gadget_is_stall_supported - return true iff the hardware supports stalling
* @g: controller to check for quirk
*/
static inline int gadget_is_stall_supported(struct usb_gadget *g)
{
return !g->quirk_stall_not_supp;
}
/**
* gadget_is_zlp_supported - return true iff the hardware supports zlp
* @g: controller to check for quirk
*/
static inline int gadget_is_zlp_supported(struct usb_gadget *g)
{
return !g->quirk_zlp_not_supp;
}
/**
* gadget_avoids_skb_reserve - return true iff the hardware would like to avoid
* skb_reserve to improve performance.
* @g: controller to check for quirk
*/
static inline int gadget_avoids_skb_reserve(struct usb_gadget *g)
{
return g->quirk_avoids_skb_reserve;
}
/**
* gadget_is_dualspeed - return true iff the hardware handles high speed
* @g: controller that might support both high and full speeds
*/
static inline int gadget_is_dualspeed(struct usb_gadget *g)
{
return g->max_speed >= USB_SPEED_HIGH;
}
/**
* gadget_is_superspeed() - return true if the hardware handles superspeed
* @g: controller that might support superspeed
*/
static inline int gadget_is_superspeed(struct usb_gadget *g)
{
return g->max_speed >= USB_SPEED_SUPER;
}
/**
* gadget_is_superspeed_plus() - return true if the hardware handles
* superspeed plus
* @g: controller that might support superspeed plus
*/
static inline int gadget_is_superspeed_plus(struct usb_gadget *g)
{
return g->max_speed >= USB_SPEED_SUPER_PLUS;
}
/**
* gadget_is_otg - return true iff the hardware is OTG-ready
* @g: controller that might have a Mini-AB connector
*
* This is a runtime test, since kernels with a USB-OTG stack sometimes
* run on boards which only have a Mini-B (or Mini-A) connector.
*/
static inline int gadget_is_otg(struct usb_gadget *g)
{
#ifdef CONFIG_USB_OTG
return g->is_otg;
#else
return 0;
#endif
}
/*-------------------------------------------------------------------------*/
#if IS_ENABLED(CONFIG_USB_GADGET)
int usb_gadget_frame_number(struct usb_gadget *gadget);
int usb_gadget_wakeup(struct usb_gadget *gadget);
int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set);
int usb_gadget_set_selfpowered(struct usb_gadget *gadget);
int usb_gadget_clear_selfpowered(struct usb_gadget *gadget);
int usb_gadget_vbus_connect(struct usb_gadget *gadget);
int usb_gadget_vbus_draw(struct usb_gadget *gadget, unsigned mA);
int usb_gadget_vbus_disconnect(struct usb_gadget *gadget);
int usb_gadget_connect(struct usb_gadget *gadget);
int usb_gadget_disconnect(struct usb_gadget *gadget);
int usb_gadget_deactivate(struct usb_gadget *gadget);
int usb_gadget_activate(struct usb_gadget *gadget);
int usb_gadget_check_config(struct usb_gadget *gadget);
#else
static inline int usb_gadget_frame_number(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_wakeup(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set)
{ return 0; }
static inline int usb_gadget_set_selfpowered(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_clear_selfpowered(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_vbus_connect(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_vbus_draw(struct usb_gadget *gadget, unsigned mA)
{ return 0; }
static inline int usb_gadget_vbus_disconnect(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_connect(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_disconnect(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_deactivate(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_activate(struct usb_gadget *gadget)
{ return 0; }
static inline int usb_gadget_check_config(struct usb_gadget *gadget)
{ return 0; }
#endif /* CONFIG_USB_GADGET */
/*-------------------------------------------------------------------------*/
/**
* struct usb_gadget_driver - driver for usb gadget devices
* @function: String describing the gadget's function
* @max_speed: Highest speed the driver handles.
* @setup: Invoked for ep0 control requests that aren't handled by
* the hardware level driver. Most calls must be handled by
* the gadget driver, including descriptor and configuration
* management. The 16 bit members of the setup data are in
* USB byte order. Called in_interrupt; this may not sleep. Driver
* queues a response to ep0, or returns negative to stall.
* @disconnect: Invoked after all transfers have been stopped,
* when the host is disconnected. May be called in_interrupt; this
* may not sleep. Some devices can't detect disconnect, so this might
* not be called except as part of controller shutdown.
* @bind: the driver's bind callback
* @unbind: Invoked when the driver is unbound from a gadget,
* usually from rmmod (after a disconnect is reported).
* Called in a context that permits sleeping.
* @suspend: Invoked on USB suspend. May be called in_interrupt.
* @resume: Invoked on USB resume. May be called in_interrupt.
* @reset: Invoked on USB bus reset. It is mandatory for all gadget drivers
* and should be called in_interrupt.
* @driver: Driver model state for this driver.
* @udc_name: A name of UDC this driver should be bound to. If udc_name is NULL,
* this driver will be bound to any available UDC.
* @match_existing_only: If udc is not found, return an error and fail
* the driver registration
* @is_bound: Allow a driver to be bound to only one gadget
*
* Devices are disabled till a gadget driver successfully bind()s, which
* means the driver will handle setup() requests needed to enumerate (and
* meet "chapter 9" requirements) then do some useful work.
*
* If gadget->is_otg is true, the gadget driver must provide an OTG
* descriptor during enumeration, or else fail the bind() call. In such
* cases, no USB traffic may flow until both bind() returns without
* having called usb_gadget_disconnect(), and the USB host stack has
* initialized.
*
* Drivers use hardware-specific knowledge to configure the usb hardware.
* endpoint addressing is only one of several hardware characteristics that
* are in descriptors the ep0 implementation returns from setup() calls.
*
* Except for ep0 implementation, most driver code shouldn't need change to
* run on top of different usb controllers. It'll use endpoints set up by
* that ep0 implementation.
*
* The usb controller driver handles a few standard usb requests. Those
* include set_address, and feature flags for devices, interfaces, and
* endpoints (the get_status, set_feature, and clear_feature requests).
*
* Accordingly, the driver's setup() callback must always implement all
* get_descriptor requests, returning at least a device descriptor and
* a configuration descriptor. Drivers must make sure the endpoint
* descriptors match any hardware constraints. Some hardware also constrains
* other descriptors. (The pxa250 allows only configurations 1, 2, or 3).
*
* The driver's setup() callback must also implement set_configuration,
* and should also implement set_interface, get_configuration, and
* get_interface. Setting a configuration (or interface) is where
* endpoints should be activated or (config 0) shut down.
*
* The gadget driver's setup() callback does not have to queue a response to
* ep0 within the setup() call, the driver can do it after setup() returns.
* The UDC driver must wait until such a response is queued before proceeding
* with the data/status stages of the control transfer.
*
* NOTE: Currently, a number of UDC drivers rely on USB_GADGET_DELAYED_STATUS
* being returned from the setup() callback, which is a bug. See the comment
* next to USB_GADGET_DELAYED_STATUS for details.
*
* (Note that only the default control endpoint is supported. Neither
* hosts nor devices generally support control traffic except to ep0.)
*
* Most devices will ignore USB suspend/resume operations, and so will
* not provide those callbacks. However, some may need to change modes
* when the host is not longer directing those activities. For example,
* local controls (buttons, dials, etc) may need to be re-enabled since
* the (remote) host can't do that any longer; or an error state might
* be cleared, to make the device behave identically whether or not
* power is maintained.
*/
struct usb_gadget_driver {
char *function;
enum usb_device_speed max_speed;
int (*bind)(struct usb_gadget *gadget,
struct usb_gadget_driver *driver);
void (*unbind)(struct usb_gadget *);
int (*setup)(struct usb_gadget *,
const struct usb_ctrlrequest *);
void (*disconnect)(struct usb_gadget *);
void (*suspend)(struct usb_gadget *);
void (*resume)(struct usb_gadget *);
void (*reset)(struct usb_gadget *);
/* FIXME support safe rmmod */
struct device_driver driver;
char *udc_name;
unsigned match_existing_only:1;
bool is_bound:1;
};
/*-------------------------------------------------------------------------*/
/* driver modules register and unregister, as usual.
* these calls must be made in a context that can sleep.
*
* A gadget driver can be bound to only one gadget at a time.
*/
/**
* usb_gadget_register_driver_owner - register a gadget driver
* @driver: the driver being registered
* @owner: the driver module
* @mod_name: the driver module's build name
* Context: can sleep
*
* Call this in your gadget driver's module initialization function,
* to tell the underlying UDC controller driver about your driver.
* The @bind() function will be called to bind it to a gadget before this
* registration call returns. It's expected that the @bind() function will
* be in init sections.
*
* Use the macro defined below instead of calling this directly.
*/
int usb_gadget_register_driver_owner(struct usb_gadget_driver *driver,
struct module *owner, const char *mod_name);
/* use a define to avoid include chaining to get THIS_MODULE & friends */
#define usb_gadget_register_driver(driver) \
usb_gadget_register_driver_owner(driver, THIS_MODULE, KBUILD_MODNAME)
/**
* usb_gadget_unregister_driver - unregister a gadget driver
* @driver:the driver being unregistered
* Context: can sleep
*
* Call this in your gadget driver's module cleanup function,
* to tell the underlying usb controller that your driver is
* going away. If the controller is connected to a USB host,
* it will first disconnect(). The driver is also requested
* to unbind() and clean up any device state, before this procedure
* finally returns. It's expected that the unbind() functions
* will be in exit sections, so may not be linked in some kernels.
*/
int usb_gadget_unregister_driver(struct usb_gadget_driver *driver);
/*-------------------------------------------------------------------------*/
/* utility to simplify dealing with string descriptors */
/**
* struct usb_string - wraps a C string and its USB id
* @id:the (nonzero) ID for this string
* @s:the string, in UTF-8 encoding
*
* If you're using usb_gadget_get_string(), use this to wrap a string
* together with its ID.
*/
struct usb_string {
u8 id;
const char *s;
};
/**
* struct usb_gadget_strings - a set of USB strings in a given language
* @language:identifies the strings' language (0x0409 for en-us)
* @strings:array of strings with their ids
*
* If you're using usb_gadget_get_string(), use this to wrap all the
* strings for a given language.
*/
struct usb_gadget_strings {
u16 language; /* 0x0409 for en-us */
struct usb_string *strings;
};
struct usb_gadget_string_container {
struct list_head list;
u8 *stash[];
};
/* put descriptor for string with that id into buf (buflen >= 256) */
int usb_gadget_get_string(const struct usb_gadget_strings *table, int id, u8 *buf);
/* check if the given language identifier is valid */
bool usb_validate_langid(u16 langid);
struct gadget_string {
struct config_item item;
struct list_head list;
char string[USB_MAX_STRING_LEN];
struct usb_string usb_string;
};
#define to_gadget_string(str_item)\
container_of(str_item, struct gadget_string, item)
/*-------------------------------------------------------------------------*/
/* utility to simplify managing config descriptors */
/* write vector of descriptors into buffer */
int usb_descriptor_fillbuf(void *, unsigned,
const struct usb_descriptor_header **);
/* build config descriptor from single descriptor vector */
int usb_gadget_config_buf(const struct usb_config_descriptor *config,
void *buf, unsigned buflen, const struct usb_descriptor_header **desc);
/* copy a NULL-terminated vector of descriptors */
struct usb_descriptor_header **usb_copy_descriptors(
struct usb_descriptor_header **);
/**
* usb_free_descriptors - free descriptors returned by usb_copy_descriptors()
* @v: vector of descriptors
*/
static inline void usb_free_descriptors(struct usb_descriptor_header **v)
{
kfree(v);
}
struct usb_function;
int usb_assign_descriptors(struct usb_function *f,
struct usb_descriptor_header **fs,
struct usb_descriptor_header **hs,
struct usb_descriptor_header **ss,
struct usb_descriptor_header **ssp);
void usb_free_all_descriptors(struct usb_function *f);
struct usb_descriptor_header *usb_otg_descriptor_alloc(
struct usb_gadget *gadget);
int usb_otg_descriptor_init(struct usb_gadget *gadget,
struct usb_descriptor_header *otg_desc);
/*-------------------------------------------------------------------------*/
/* utility to simplify map/unmap of usb_requests to/from DMA */
#ifdef CONFIG_HAS_DMA
extern int usb_gadget_map_request_by_dev(struct device *dev,
struct usb_request *req, int is_in);
extern int usb_gadget_map_request(struct usb_gadget *gadget,
struct usb_request *req, int is_in);
extern void usb_gadget_unmap_request_by_dev(struct device *dev,
struct usb_request *req, int is_in);
extern void usb_gadget_unmap_request(struct usb_gadget *gadget,
struct usb_request *req, int is_in);
#else /* !CONFIG_HAS_DMA */
static inline int usb_gadget_map_request_by_dev(struct device *dev,
struct usb_request *req, int is_in) { return -ENOSYS; }
static inline int usb_gadget_map_request(struct usb_gadget *gadget,
struct usb_request *req, int is_in) { return -ENOSYS; }
static inline void usb_gadget_unmap_request_by_dev(struct device *dev,
struct usb_request *req, int is_in) { }
static inline void usb_gadget_unmap_request(struct usb_gadget *gadget,
struct usb_request *req, int is_in) { }
#endif /* !CONFIG_HAS_DMA */
/*-------------------------------------------------------------------------*/
/* utility to set gadget state properly */
extern void usb_gadget_set_state(struct usb_gadget *gadget,
enum usb_device_state state);
/*-------------------------------------------------------------------------*/
/* utility to tell udc core that the bus reset occurs */
extern void usb_gadget_udc_reset(struct usb_gadget *gadget,
struct usb_gadget_driver *driver);
/*-------------------------------------------------------------------------*/
/* utility to give requests back to the gadget layer */
extern void usb_gadget_giveback_request(struct usb_ep *ep,
struct usb_request *req);
/*-------------------------------------------------------------------------*/
/* utility to find endpoint by name */
extern struct usb_ep *gadget_find_ep_by_name(struct usb_gadget *g,
const char *name);
/*-------------------------------------------------------------------------*/
/* utility to check if endpoint caps match descriptor needs */
extern int usb_gadget_ep_match_desc(struct usb_gadget *gadget,
struct usb_ep *ep, struct usb_endpoint_descriptor *desc,
struct usb_ss_ep_comp_descriptor *ep_comp);
/*-------------------------------------------------------------------------*/
/* utility to update vbus status for udc core, it may be scheduled */
extern void usb_udc_vbus_handler(struct usb_gadget *gadget, bool status);
/*-------------------------------------------------------------------------*/
/* utility wrapping a simple endpoint selection policy */
extern struct usb_ep *usb_ep_autoconfig(struct usb_gadget *,
struct usb_endpoint_descriptor *);
extern struct usb_ep *usb_ep_autoconfig_ss(struct usb_gadget *,
struct usb_endpoint_descriptor *,
struct usb_ss_ep_comp_descriptor *);
extern void usb_ep_autoconfig_release(struct usb_ep *);
extern void usb_ep_autoconfig_reset(struct usb_gadget *);
#endif /* __LINUX_USB_GADGET_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
File: fs/xattr.c
Extended attribute handling.
Copyright (C) 2001 by Andreas Gruenbacher <a.gruenbacher@computer.org>
Copyright (C) 2001 SGI - Silicon Graphics, Inc <linux-xfs@oss.sgi.com>
Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
*/
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/xattr.h>
#include <linux/mount.h>
#include <linux/namei.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/export.h>
#include <linux/fsnotify.h>
#include <linux/audit.h>
#include <linux/vmalloc.h>
#include <linux/posix_acl_xattr.h>
#include <linux/uaccess.h>
#include "internal.h"
static const char *
strcmp_prefix(const char *a, const char *a_prefix)
{
while (*a_prefix && *a == *a_prefix) { a++;
a_prefix++;
}
return *a_prefix ? NULL : a;
}
/*
* In order to implement different sets of xattr operations for each xattr
* prefix, a filesystem should create a null-terminated array of struct
* xattr_handler (one for each prefix) and hang a pointer to it off of the
* s_xattr field of the superblock.
*/
#define for_each_xattr_handler(handlers, handler) \
if (handlers) \
for ((handler) = *(handlers)++; \
(handler) != NULL; \
(handler) = *(handlers)++)
/*
* Find the xattr_handler with the matching prefix.
*/
static const struct xattr_handler *
xattr_resolve_name(struct inode *inode, const char **name)
{
const struct xattr_handler * const *handlers = inode->i_sb->s_xattr;
const struct xattr_handler *handler;
if (!(inode->i_opflags & IOP_XATTR)) {
if (unlikely(is_bad_inode(inode)))
return ERR_PTR(-EIO);
return ERR_PTR(-EOPNOTSUPP);
}
for_each_xattr_handler(handlers, handler) {
const char *n;
n = strcmp_prefix(*name, xattr_prefix(handler)); if (n) { if (!handler->prefix ^ !*n) { if (*n)
continue;
return ERR_PTR(-EINVAL);
}
*name = n;
return handler;
}
}
return ERR_PTR(-EOPNOTSUPP);
}
/**
* may_write_xattr - check whether inode allows writing xattr
* @idmap: idmap of the mount the inode was found from
* @inode: the inode on which to set an xattr
*
* Check whether the inode allows writing xattrs. Specifically, we can never
* set or remove an extended attribute on a read-only filesystem or on an
* immutable / append-only inode.
*
* We also need to ensure that the inode has a mapping in the mount to
* not risk writing back invalid i_{g,u}id values.
*
* Return: On success zero is returned. On error a negative errno is returned.
*/
int may_write_xattr(struct mnt_idmap *idmap, struct inode *inode)
{
if (IS_IMMUTABLE(inode))
return -EPERM;
if (IS_APPEND(inode))
return -EPERM;
if (HAS_UNMAPPED_ID(idmap, inode))
return -EPERM;
return 0;
}
/*
* Check permissions for extended attribute access. This is a bit complicated
* because different namespaces have very different rules.
*/
static int
xattr_permission(struct mnt_idmap *idmap, struct inode *inode,
const char *name, int mask)
{
if (mask & MAY_WRITE) {
int ret;
ret = may_write_xattr(idmap, inode);
if (ret)
return ret;
}
/*
* No restriction for security.* and system.* from the VFS. Decision
* on these is left to the underlying filesystem / security module.
*/
if (!strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) ||
!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
return 0;
/*
* The trusted.* namespace can only be accessed by privileged users.
*/
if (!strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) {
if (!capable(CAP_SYS_ADMIN))
return (mask & MAY_WRITE) ? -EPERM : -ENODATA;
return 0;
}
/*
* In the user.* namespace, only regular files and directories can have
* extended attributes. For sticky directories, only the owner and
* privileged users can write attributes.
*/
if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) {
if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
return (mask & MAY_WRITE) ? -EPERM : -ENODATA;
if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) &&
(mask & MAY_WRITE) &&
!inode_owner_or_capable(idmap, inode))
return -EPERM;
}
return inode_permission(idmap, inode, mask);
}
/*
* Look for any handler that deals with the specified namespace.
*/
int
xattr_supports_user_prefix(struct inode *inode)
{
const struct xattr_handler * const *handlers = inode->i_sb->s_xattr;
const struct xattr_handler *handler;
if (!(inode->i_opflags & IOP_XATTR)) {
if (unlikely(is_bad_inode(inode)))
return -EIO;
return -EOPNOTSUPP;
}
for_each_xattr_handler(handlers, handler) {
if (!strncmp(xattr_prefix(handler), XATTR_USER_PREFIX,
XATTR_USER_PREFIX_LEN))
return 0;
}
return -EOPNOTSUPP;
}
EXPORT_SYMBOL(xattr_supports_user_prefix);
int
__vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
struct inode *inode, const char *name, const void *value,
size_t size, int flags)
{
const struct xattr_handler *handler;
if (is_posix_acl_xattr(name))
return -EOPNOTSUPP;
handler = xattr_resolve_name(inode, &name);
if (IS_ERR(handler))
return PTR_ERR(handler);
if (!handler->set)
return -EOPNOTSUPP;
if (size == 0)
value = ""; /* empty EA, do not remove */
return handler->set(handler, idmap, dentry, inode, name, value,
size, flags);
}
EXPORT_SYMBOL(__vfs_setxattr);
/**
* __vfs_setxattr_noperm - perform setxattr operation without performing
* permission checks.
*
* @idmap: idmap of the mount the inode was found from
* @dentry: object to perform setxattr on
* @name: xattr name to set
* @value: value to set @name to
* @size: size of @value
* @flags: flags to pass into filesystem operations
*
* returns the result of the internal setxattr or setsecurity operations.
*
* This function requires the caller to lock the inode's i_mutex before it
* is executed. It also assumes that the caller will make the appropriate
* permission checks.
*/
int __vfs_setxattr_noperm(struct mnt_idmap *idmap,
struct dentry *dentry, const char *name,
const void *value, size_t size, int flags)
{
struct inode *inode = dentry->d_inode;
int error = -EAGAIN;
int issec = !strncmp(name, XATTR_SECURITY_PREFIX,
XATTR_SECURITY_PREFIX_LEN);
if (issec)
inode->i_flags &= ~S_NOSEC;
if (inode->i_opflags & IOP_XATTR) {
error = __vfs_setxattr(idmap, dentry, inode, name, value,
size, flags);
if (!error) {
fsnotify_xattr(dentry);
security_inode_post_setxattr(dentry, name, value,
size, flags);
}
} else {
if (unlikely(is_bad_inode(inode)))
return -EIO;
}
if (error == -EAGAIN) {
error = -EOPNOTSUPP;
if (issec) {
const char *suffix = name + XATTR_SECURITY_PREFIX_LEN;
error = security_inode_setsecurity(inode, suffix, value,
size, flags);
if (!error)
fsnotify_xattr(dentry);
}
}
return error;
}
/**
* __vfs_setxattr_locked - set an extended attribute while holding the inode
* lock
*
* @idmap: idmap of the mount of the target inode
* @dentry: object to perform setxattr on
* @name: xattr name to set
* @value: value to set @name to
* @size: size of @value
* @flags: flags to pass into filesystem operations
* @delegated_inode: on return, will contain an inode pointer that
* a delegation was broken on, NULL if none.
*/
int
__vfs_setxattr_locked(struct mnt_idmap *idmap, struct dentry *dentry,
const char *name, const void *value, size_t size,
int flags, struct inode **delegated_inode)
{
struct inode *inode = dentry->d_inode;
int error;
error = xattr_permission(idmap, inode, name, MAY_WRITE);
if (error)
return error;
error = security_inode_setxattr(idmap, dentry, name, value, size,
flags);
if (error)
goto out;
error = try_break_deleg(inode, delegated_inode);
if (error)
goto out;
error = __vfs_setxattr_noperm(idmap, dentry, name, value,
size, flags);
out:
return error;
}
EXPORT_SYMBOL_GPL(__vfs_setxattr_locked);
int
vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
const char *name, const void *value, size_t size, int flags)
{
struct inode *inode = dentry->d_inode;
struct inode *delegated_inode = NULL;
const void *orig_value = value;
int error;
if (size && strcmp(name, XATTR_NAME_CAPS) == 0) {
error = cap_convert_nscap(idmap, dentry, &value, size);
if (error < 0)
return error;
size = error;
}
retry_deleg:
inode_lock(inode);
error = __vfs_setxattr_locked(idmap, dentry, name, value, size,
flags, &delegated_inode);
inode_unlock(inode);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
if (value != orig_value)
kfree(value);
return error;
}
EXPORT_SYMBOL_GPL(vfs_setxattr);
static ssize_t
xattr_getsecurity(struct mnt_idmap *idmap, struct inode *inode,
const char *name, void *value, size_t size)
{
void *buffer = NULL;
ssize_t len;
if (!value || !size) {
len = security_inode_getsecurity(idmap, inode, name,
&buffer, false);
goto out_noalloc;
}
len = security_inode_getsecurity(idmap, inode, name, &buffer,
true);
if (len < 0)
return len;
if (size < len) {
len = -ERANGE;
goto out;
}
memcpy(value, buffer, len);
out:
kfree(buffer);
out_noalloc:
return len;
}
/*
* vfs_getxattr_alloc - allocate memory, if necessary, before calling getxattr
*
* Allocate memory, if not already allocated, or re-allocate correct size,
* before retrieving the extended attribute. The xattr value buffer should
* always be freed by the caller, even on error.
*
* Returns the result of alloc, if failed, or the getxattr operation.
*/
int
vfs_getxattr_alloc(struct mnt_idmap *idmap, struct dentry *dentry,
const char *name, char **xattr_value, size_t xattr_size,
gfp_t flags)
{
const struct xattr_handler *handler;
struct inode *inode = dentry->d_inode;
char *value = *xattr_value;
int error;
error = xattr_permission(idmap, inode, name, MAY_READ);
if (error)
return error;
handler = xattr_resolve_name(inode, &name);
if (IS_ERR(handler))
return PTR_ERR(handler);
if (!handler->get)
return -EOPNOTSUPP;
error = handler->get(handler, dentry, inode, name, NULL, 0);
if (error < 0)
return error;
if (!value || (error > xattr_size)) {
value = krealloc(*xattr_value, error + 1, flags);
if (!value)
return -ENOMEM;
memset(value, 0, error + 1);
}
error = handler->get(handler, dentry, inode, name, value, error);
*xattr_value = value;
return error;
}
ssize_t
__vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name,
void *value, size_t size)
{
const struct xattr_handler *handler;
if (is_posix_acl_xattr(name))
return -EOPNOTSUPP;
handler = xattr_resolve_name(inode, &name);
if (IS_ERR(handler))
return PTR_ERR(handler); if (!handler->get)
return -EOPNOTSUPP;
return handler->get(handler, dentry, inode, name, value, size);
}
EXPORT_SYMBOL(__vfs_getxattr);
ssize_t
vfs_getxattr(struct mnt_idmap *idmap, struct dentry *dentry,
const char *name, void *value, size_t size)
{
struct inode *inode = dentry->d_inode;
int error;
error = xattr_permission(idmap, inode, name, MAY_READ);
if (error)
return error;
error = security_inode_getxattr(dentry, name);
if (error)
return error;
if (!strncmp(name, XATTR_SECURITY_PREFIX,
XATTR_SECURITY_PREFIX_LEN)) {
const char *suffix = name + XATTR_SECURITY_PREFIX_LEN;
int ret = xattr_getsecurity(idmap, inode, suffix, value,
size);
/*
* Only overwrite the return value if a security module
* is actually active.
*/
if (ret == -EOPNOTSUPP)
goto nolsm;
return ret;
}
nolsm:
return __vfs_getxattr(dentry, inode, name, value, size);
}
EXPORT_SYMBOL_GPL(vfs_getxattr);
/**
* vfs_listxattr - retrieve \0 separated list of xattr names
* @dentry: the dentry from whose inode the xattr names are retrieved
* @list: buffer to store xattr names into
* @size: size of the buffer
*
* This function returns the names of all xattrs associated with the
* inode of @dentry.
*
* Note, for legacy reasons the vfs_listxattr() function lists POSIX
* ACLs as well. Since POSIX ACLs are decoupled from IOP_XATTR the
* vfs_listxattr() function doesn't check for this flag since a
* filesystem could implement POSIX ACLs without implementing any other
* xattrs.
*
* However, since all codepaths that remove IOP_XATTR also assign of
* inode operations that either don't implement or implement a stub
* ->listxattr() operation.
*
* Return: On success, the size of the buffer that was used. On error a
* negative error code.
*/
ssize_t
vfs_listxattr(struct dentry *dentry, char *list, size_t size)
{
struct inode *inode = d_inode(dentry);
ssize_t error;
error = security_inode_listxattr(dentry);
if (error)
return error;
if (inode->i_op->listxattr) {
error = inode->i_op->listxattr(dentry, list, size);
} else {
error = security_inode_listsecurity(inode, list, size);
if (size && error > size)
error = -ERANGE;
}
return error;
}
EXPORT_SYMBOL_GPL(vfs_listxattr);
int
__vfs_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
const char *name)
{
struct inode *inode = d_inode(dentry);
const struct xattr_handler *handler;
if (is_posix_acl_xattr(name))
return -EOPNOTSUPP;
handler = xattr_resolve_name(inode, &name);
if (IS_ERR(handler))
return PTR_ERR(handler);
if (!handler->set)
return -EOPNOTSUPP;
return handler->set(handler, idmap, dentry, inode, name, NULL, 0,
XATTR_REPLACE);
}
EXPORT_SYMBOL(__vfs_removexattr);
/**
* __vfs_removexattr_locked - set an extended attribute while holding the inode
* lock
*
* @idmap: idmap of the mount of the target inode
* @dentry: object to perform setxattr on
* @name: name of xattr to remove
* @delegated_inode: on return, will contain an inode pointer that
* a delegation was broken on, NULL if none.
*/
int
__vfs_removexattr_locked(struct mnt_idmap *idmap,
struct dentry *dentry, const char *name,
struct inode **delegated_inode)
{
struct inode *inode = dentry->d_inode;
int error;
error = xattr_permission(idmap, inode, name, MAY_WRITE);
if (error)
return error;
error = security_inode_removexattr(idmap, dentry, name);
if (error)
goto out;
error = try_break_deleg(inode, delegated_inode);
if (error)
goto out;
error = __vfs_removexattr(idmap, dentry, name);
if (error)
return error;
fsnotify_xattr(dentry);
security_inode_post_removexattr(dentry, name);
out:
return error;
}
EXPORT_SYMBOL_GPL(__vfs_removexattr_locked);
int
vfs_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
const char *name)
{
struct inode *inode = dentry->d_inode;
struct inode *delegated_inode = NULL;
int error;
retry_deleg:
inode_lock(inode);
error = __vfs_removexattr_locked(idmap, dentry,
name, &delegated_inode);
inode_unlock(inode);
if (delegated_inode) {
error = break_deleg_wait(&delegated_inode);
if (!error)
goto retry_deleg;
}
return error;
}
EXPORT_SYMBOL_GPL(vfs_removexattr);
/*
* Extended attribute SET operations
*/
int setxattr_copy(const char __user *name, struct xattr_ctx *ctx)
{
int error;
if (ctx->flags & ~(XATTR_CREATE|XATTR_REPLACE))
return -EINVAL;
error = strncpy_from_user(ctx->kname->name, name,
sizeof(ctx->kname->name));
if (error == 0 || error == sizeof(ctx->kname->name))
return -ERANGE;
if (error < 0)
return error;
error = 0;
if (ctx->size) {
if (ctx->size > XATTR_SIZE_MAX)
return -E2BIG;
ctx->kvalue = vmemdup_user(ctx->cvalue, ctx->size);
if (IS_ERR(ctx->kvalue)) {
error = PTR_ERR(ctx->kvalue);
ctx->kvalue = NULL;
}
}
return error;
}
int do_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
struct xattr_ctx *ctx)
{
if (is_posix_acl_xattr(ctx->kname->name))
return do_set_acl(idmap, dentry, ctx->kname->name,
ctx->kvalue, ctx->size);
return vfs_setxattr(idmap, dentry, ctx->kname->name,
ctx->kvalue, ctx->size, ctx->flags);
}
static long
setxattr(struct mnt_idmap *idmap, struct dentry *d,
const char __user *name, const void __user *value, size_t size,
int flags)
{
struct xattr_name kname;
struct xattr_ctx ctx = {
.cvalue = value,
.kvalue = NULL,
.size = size,
.kname = &kname,
.flags = flags,
};
int error;
error = setxattr_copy(name, &ctx);
if (error)
return error;
error = do_setxattr(idmap, d, &ctx);
kvfree(ctx.kvalue);
return error;
}
static int path_setxattr(const char __user *pathname,
const char __user *name, const void __user *value,
size_t size, int flags, unsigned int lookup_flags)
{
struct path path;
int error;
retry:
error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path);
if (error)
return error;
error = mnt_want_write(path.mnt);
if (!error) {
error = setxattr(mnt_idmap(path.mnt), path.dentry, name,
value, size, flags);
mnt_drop_write(path.mnt);
}
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
return error;
}
SYSCALL_DEFINE5(setxattr, const char __user *, pathname,
const char __user *, name, const void __user *, value,
size_t, size, int, flags)
{
return path_setxattr(pathname, name, value, size, flags, LOOKUP_FOLLOW);
}
SYSCALL_DEFINE5(lsetxattr, const char __user *, pathname,
const char __user *, name, const void __user *, value,
size_t, size, int, flags)
{
return path_setxattr(pathname, name, value, size, flags, 0);
}
SYSCALL_DEFINE5(fsetxattr, int, fd, const char __user *, name,
const void __user *,value, size_t, size, int, flags)
{
struct fd f = fdget(fd);
int error = -EBADF;
if (!f.file)
return error;
audit_file(f.file);
error = mnt_want_write_file(f.file);
if (!error) {
error = setxattr(file_mnt_idmap(f.file),
f.file->f_path.dentry, name,
value, size, flags);
mnt_drop_write_file(f.file);
}
fdput(f);
return error;
}
/*
* Extended attribute GET operations
*/
ssize_t
do_getxattr(struct mnt_idmap *idmap, struct dentry *d,
struct xattr_ctx *ctx)
{
ssize_t error;
char *kname = ctx->kname->name;
if (ctx->size) {
if (ctx->size > XATTR_SIZE_MAX)
ctx->size = XATTR_SIZE_MAX;
ctx->kvalue = kvzalloc(ctx->size, GFP_KERNEL);
if (!ctx->kvalue)
return -ENOMEM;
}
if (is_posix_acl_xattr(ctx->kname->name))
error = do_get_acl(idmap, d, kname, ctx->kvalue, ctx->size);
else
error = vfs_getxattr(idmap, d, kname, ctx->kvalue, ctx->size);
if (error > 0) {
if (ctx->size && copy_to_user(ctx->value, ctx->kvalue, error))
error = -EFAULT;
} else if (error == -ERANGE && ctx->size >= XATTR_SIZE_MAX) {
/* The file system tried to returned a value bigger
than XATTR_SIZE_MAX bytes. Not possible. */
error = -E2BIG;
}
return error;
}
static ssize_t
getxattr(struct mnt_idmap *idmap, struct dentry *d,
const char __user *name, void __user *value, size_t size)
{
ssize_t error;
struct xattr_name kname;
struct xattr_ctx ctx = {
.value = value,
.kvalue = NULL,
.size = size,
.kname = &kname,
.flags = 0,
};
error = strncpy_from_user(kname.name, name, sizeof(kname.name));
if (error == 0 || error == sizeof(kname.name))
error = -ERANGE;
if (error < 0)
return error;
error = do_getxattr(idmap, d, &ctx);
kvfree(ctx.kvalue);
return error;
}
static ssize_t path_getxattr(const char __user *pathname,
const char __user *name, void __user *value,
size_t size, unsigned int lookup_flags)
{
struct path path;
ssize_t error;
retry:
error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path);
if (error)
return error;
error = getxattr(mnt_idmap(path.mnt), path.dentry, name, value, size);
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
return error;
}
SYSCALL_DEFINE4(getxattr, const char __user *, pathname,
const char __user *, name, void __user *, value, size_t, size)
{
return path_getxattr(pathname, name, value, size, LOOKUP_FOLLOW);
}
SYSCALL_DEFINE4(lgetxattr, const char __user *, pathname,
const char __user *, name, void __user *, value, size_t, size)
{
return path_getxattr(pathname, name, value, size, 0);
}
SYSCALL_DEFINE4(fgetxattr, int, fd, const char __user *, name,
void __user *, value, size_t, size)
{
struct fd f = fdget(fd);
ssize_t error = -EBADF;
if (!f.file)
return error;
audit_file(f.file);
error = getxattr(file_mnt_idmap(f.file), f.file->f_path.dentry,
name, value, size);
fdput(f);
return error;
}
/*
* Extended attribute LIST operations
*/
static ssize_t
listxattr(struct dentry *d, char __user *list, size_t size)
{
ssize_t error;
char *klist = NULL;
if (size) {
if (size > XATTR_LIST_MAX)
size = XATTR_LIST_MAX;
klist = kvmalloc(size, GFP_KERNEL);
if (!klist)
return -ENOMEM;
}
error = vfs_listxattr(d, klist, size);
if (error > 0) {
if (size && copy_to_user(list, klist, error))
error = -EFAULT;
} else if (error == -ERANGE && size >= XATTR_LIST_MAX) {
/* The file system tried to returned a list bigger
than XATTR_LIST_MAX bytes. Not possible. */
error = -E2BIG;
}
kvfree(klist);
return error;
}
static ssize_t path_listxattr(const char __user *pathname, char __user *list,
size_t size, unsigned int lookup_flags)
{
struct path path;
ssize_t error;
retry:
error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path);
if (error)
return error;
error = listxattr(path.dentry, list, size);
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
return error;
}
SYSCALL_DEFINE3(listxattr, const char __user *, pathname, char __user *, list,
size_t, size)
{
return path_listxattr(pathname, list, size, LOOKUP_FOLLOW);
}
SYSCALL_DEFINE3(llistxattr, const char __user *, pathname, char __user *, list,
size_t, size)
{
return path_listxattr(pathname, list, size, 0);
}
SYSCALL_DEFINE3(flistxattr, int, fd, char __user *, list, size_t, size)
{
struct fd f = fdget(fd);
ssize_t error = -EBADF;
if (!f.file)
return error;
audit_file(f.file);
error = listxattr(f.file->f_path.dentry, list, size);
fdput(f);
return error;
}
/*
* Extended attribute REMOVE operations
*/
static long
removexattr(struct mnt_idmap *idmap, struct dentry *d,
const char __user *name)
{
int error;
char kname[XATTR_NAME_MAX + 1];
error = strncpy_from_user(kname, name, sizeof(kname));
if (error == 0 || error == sizeof(kname))
error = -ERANGE;
if (error < 0)
return error;
if (is_posix_acl_xattr(kname))
return vfs_remove_acl(idmap, d, kname);
return vfs_removexattr(idmap, d, kname);
}
static int path_removexattr(const char __user *pathname,
const char __user *name, unsigned int lookup_flags)
{
struct path path;
int error;
retry:
error = user_path_at(AT_FDCWD, pathname, lookup_flags, &path);
if (error)
return error;
error = mnt_want_write(path.mnt);
if (!error) {
error = removexattr(mnt_idmap(path.mnt), path.dentry, name);
mnt_drop_write(path.mnt);
}
path_put(&path);
if (retry_estale(error, lookup_flags)) {
lookup_flags |= LOOKUP_REVAL;
goto retry;
}
return error;
}
SYSCALL_DEFINE2(removexattr, const char __user *, pathname,
const char __user *, name)
{
return path_removexattr(pathname, name, LOOKUP_FOLLOW);
}
SYSCALL_DEFINE2(lremovexattr, const char __user *, pathname,
const char __user *, name)
{
return path_removexattr(pathname, name, 0);
}
SYSCALL_DEFINE2(fremovexattr, int, fd, const char __user *, name)
{
struct fd f = fdget(fd);
int error = -EBADF;
if (!f.file)
return error;
audit_file(f.file);
error = mnt_want_write_file(f.file);
if (!error) {
error = removexattr(file_mnt_idmap(f.file),
f.file->f_path.dentry, name);
mnt_drop_write_file(f.file);
}
fdput(f);
return error;
}
int xattr_list_one(char **buffer, ssize_t *remaining_size, const char *name)
{
size_t len;
len = strlen(name) + 1;
if (*buffer) {
if (*remaining_size < len)
return -ERANGE;
memcpy(*buffer, name, len);
*buffer += len;
}
*remaining_size -= len;
return 0;
}
/**
* generic_listxattr - run through a dentry's xattr list() operations
* @dentry: dentry to list the xattrs
* @buffer: result buffer
* @buffer_size: size of @buffer
*
* Combine the results of the list() operation from every xattr_handler in the
* xattr_handler stack.
*
* Note that this will not include the entries for POSIX ACLs.
*/
ssize_t
generic_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size)
{
const struct xattr_handler *handler, * const *handlers = dentry->d_sb->s_xattr;
ssize_t remaining_size = buffer_size;
int err = 0;
for_each_xattr_handler(handlers, handler) {
if (!handler->name || (handler->list && !handler->list(dentry)))
continue;
err = xattr_list_one(&buffer, &remaining_size, handler->name);
if (err)
return err;
}
return err ? err : buffer_size - remaining_size;
}
EXPORT_SYMBOL(generic_listxattr);
/**
* xattr_full_name - Compute full attribute name from suffix
*
* @handler: handler of the xattr_handler operation
* @name: name passed to the xattr_handler operation
*
* The get and set xattr handler operations are called with the remainder of
* the attribute name after skipping the handler's prefix: for example, "foo"
* is passed to the get operation of a handler with prefix "user." to get
* attribute "user.foo". The full name is still "there" in the name though.
*
* Note: the list xattr handler operation when called from the vfs is passed a
* NULL name; some file systems use this operation internally, with varying
* semantics.
*/
const char *xattr_full_name(const struct xattr_handler *handler,
const char *name)
{
size_t prefix_len = strlen(xattr_prefix(handler));
return name - prefix_len;
}
EXPORT_SYMBOL(xattr_full_name);
/**
* simple_xattr_space - estimate the memory used by a simple xattr
* @name: the full name of the xattr
* @size: the size of its value
*
* This takes no account of how much larger the two slab objects actually are:
* that would depend on the slab implementation, when what is required is a
* deterministic number, which grows with name length and size and quantity.
*
* Return: The approximate number of bytes of memory used by such an xattr.
*/
size_t simple_xattr_space(const char *name, size_t size)
{
/*
* Use "40" instead of sizeof(struct simple_xattr), to return the
* same result on 32-bit and 64-bit, and even if simple_xattr grows.
*/
return 40 + size + strlen(name);
}
/**
* simple_xattr_free - free an xattr object
* @xattr: the xattr object
*
* Free the xattr object. Can handle @xattr being NULL.
*/
void simple_xattr_free(struct simple_xattr *xattr)
{
if (xattr)
kfree(xattr->name);
kvfree(xattr);
}
/**
* simple_xattr_alloc - allocate new xattr object
* @value: value of the xattr object
* @size: size of @value
*
* Allocate a new xattr object and initialize respective members. The caller is
* responsible for handling the name of the xattr.
*
* Return: On success a new xattr object is returned. On failure NULL is
* returned.
*/
struct simple_xattr *simple_xattr_alloc(const void *value, size_t size)
{
struct simple_xattr *new_xattr;
size_t len;
/* wrap around? */
len = sizeof(*new_xattr) + size;
if (len < sizeof(*new_xattr))
return NULL;
new_xattr = kvmalloc(len, GFP_KERNEL_ACCOUNT);
if (!new_xattr)
return NULL;
new_xattr->size = size;
memcpy(new_xattr->value, value, size);
return new_xattr;
}
/**
* rbtree_simple_xattr_cmp - compare xattr name with current rbtree xattr entry
* @key: xattr name
* @node: current node
*
* Compare the xattr name with the xattr name attached to @node in the rbtree.
*
* Return: Negative value if continuing left, positive if continuing right, 0
* if the xattr attached to @node matches @key.
*/
static int rbtree_simple_xattr_cmp(const void *key, const struct rb_node *node)
{
const char *xattr_name = key;
const struct simple_xattr *xattr;
xattr = rb_entry(node, struct simple_xattr, rb_node);
return strcmp(xattr->name, xattr_name);
}
/**
* rbtree_simple_xattr_node_cmp - compare two xattr rbtree nodes
* @new_node: new node
* @node: current node
*
* Compare the xattr attached to @new_node with the xattr attached to @node.
*
* Return: Negative value if continuing left, positive if continuing right, 0
* if the xattr attached to @new_node matches the xattr attached to @node.
*/
static int rbtree_simple_xattr_node_cmp(struct rb_node *new_node,
const struct rb_node *node)
{
struct simple_xattr *xattr;
xattr = rb_entry(new_node, struct simple_xattr, rb_node);
return rbtree_simple_xattr_cmp(xattr->name, node);
}
/**
* simple_xattr_get - get an xattr object
* @xattrs: the header of the xattr object
* @name: the name of the xattr to retrieve
* @buffer: the buffer to store the value into
* @size: the size of @buffer
*
* Try to find and retrieve the xattr object associated with @name.
* If @buffer is provided store the value of @xattr in @buffer
* otherwise just return the length. The size of @buffer is limited
* to XATTR_SIZE_MAX which currently is 65536.
*
* Return: On success the length of the xattr value is returned. On error a
* negative error code is returned.
*/
int simple_xattr_get(struct simple_xattrs *xattrs, const char *name,
void *buffer, size_t size)
{
struct simple_xattr *xattr = NULL;
struct rb_node *rbp;
int ret = -ENODATA;
read_lock(&xattrs->lock); rbp = rb_find(name, &xattrs->rb_root, rbtree_simple_xattr_cmp); if (rbp) {
xattr = rb_entry(rbp, struct simple_xattr, rb_node);
ret = xattr->size;
if (buffer) {
if (size < xattr->size)
ret = -ERANGE;
else
memcpy(buffer, xattr->value, xattr->size);
}
}
read_unlock(&xattrs->lock);
return ret;
}
/**
* simple_xattr_set - set an xattr object
* @xattrs: the header of the xattr object
* @name: the name of the xattr to retrieve
* @value: the value to store along the xattr
* @size: the size of @value
* @flags: the flags determining how to set the xattr
*
* Set a new xattr object.
* If @value is passed a new xattr object will be allocated. If XATTR_REPLACE
* is specified in @flags a matching xattr object for @name must already exist.
* If it does it will be replaced with the new xattr object. If it doesn't we
* fail. If XATTR_CREATE is specified and a matching xattr does already exist
* we fail. If it doesn't we create a new xattr. If @flags is zero we simply
* insert the new xattr replacing any existing one.
*
* If @value is empty and a matching xattr object is found we delete it if
* XATTR_REPLACE is specified in @flags or @flags is zero.
*
* If @value is empty and no matching xattr object for @name is found we do
* nothing if XATTR_CREATE is specified in @flags or @flags is zero. For
* XATTR_REPLACE we fail as mentioned above.
*
* Return: On success, the removed or replaced xattr is returned, to be freed
* by the caller; or NULL if none. On failure a negative error code is returned.
*/
struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs,
const char *name, const void *value,
size_t size, int flags)
{
struct simple_xattr *old_xattr = NULL, *new_xattr = NULL;
struct rb_node *parent = NULL, **rbp;
int err = 0, ret;
/* value == NULL means remove */
if (value) {
new_xattr = simple_xattr_alloc(value, size);
if (!new_xattr)
return ERR_PTR(-ENOMEM);
new_xattr->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
if (!new_xattr->name) {
simple_xattr_free(new_xattr);
return ERR_PTR(-ENOMEM);
}
}
write_lock(&xattrs->lock);
rbp = &xattrs->rb_root.rb_node;
while (*rbp) {
parent = *rbp;
ret = rbtree_simple_xattr_cmp(name, *rbp);
if (ret < 0)
rbp = &(*rbp)->rb_left;
else if (ret > 0)
rbp = &(*rbp)->rb_right;
else
old_xattr = rb_entry(*rbp, struct simple_xattr, rb_node);
if (old_xattr)
break;
}
if (old_xattr) {
/* Fail if XATTR_CREATE is requested and the xattr exists. */
if (flags & XATTR_CREATE) {
err = -EEXIST;
goto out_unlock;
}
if (new_xattr)
rb_replace_node(&old_xattr->rb_node,
&new_xattr->rb_node, &xattrs->rb_root);
else
rb_erase(&old_xattr->rb_node, &xattrs->rb_root);
} else {
/* Fail if XATTR_REPLACE is requested but no xattr is found. */
if (flags & XATTR_REPLACE) {
err = -ENODATA;
goto out_unlock;
}
/*
* If XATTR_CREATE or no flags are specified together with a
* new value simply insert it.
*/
if (new_xattr) {
rb_link_node(&new_xattr->rb_node, parent, rbp);
rb_insert_color(&new_xattr->rb_node, &xattrs->rb_root);
}
/*
* If XATTR_CREATE or no flags are specified and neither an
* old or new xattr exist then we don't need to do anything.
*/
}
out_unlock:
write_unlock(&xattrs->lock);
if (!err)
return old_xattr;
simple_xattr_free(new_xattr);
return ERR_PTR(err);
}
static bool xattr_is_trusted(const char *name)
{
return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
}
/**
* simple_xattr_list - list all xattr objects
* @inode: inode from which to get the xattrs
* @xattrs: the header of the xattr object
* @buffer: the buffer to store all xattrs into
* @size: the size of @buffer
*
* List all xattrs associated with @inode. If @buffer is NULL we returned
* the required size of the buffer. If @buffer is provided we store the
* xattrs value into it provided it is big enough.
*
* Note, the number of xattr names that can be listed with listxattr(2) is
* limited to XATTR_LIST_MAX aka 65536 bytes. If a larger buffer is passed
* then vfs_listxattr() caps it to XATTR_LIST_MAX and if more xattr names
* are found it will return -E2BIG.
*
* Return: On success the required size or the size of the copied xattrs is
* returned. On error a negative error code is returned.
*/
ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs,
char *buffer, size_t size)
{
bool trusted = ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN);
struct simple_xattr *xattr;
struct rb_node *rbp;
ssize_t remaining_size = size;
int err = 0;
err = posix_acl_listxattr(inode, &buffer, &remaining_size);
if (err)
return err;
read_lock(&xattrs->lock);
for (rbp = rb_first(&xattrs->rb_root); rbp; rbp = rb_next(rbp)) {
xattr = rb_entry(rbp, struct simple_xattr, rb_node);
/* skip "trusted." attributes for unprivileged callers */
if (!trusted && xattr_is_trusted(xattr->name))
continue;
err = xattr_list_one(&buffer, &remaining_size, xattr->name);
if (err)
break;
}
read_unlock(&xattrs->lock);
return err ? err : size - remaining_size;
}
/**
* rbtree_simple_xattr_less - compare two xattr rbtree nodes
* @new_node: new node
* @node: current node
*
* Compare the xattr attached to @new_node with the xattr attached to @node.
* Note that this function technically tolerates duplicate entries.
*
* Return: True if insertion point in the rbtree is found.
*/
static bool rbtree_simple_xattr_less(struct rb_node *new_node,
const struct rb_node *node)
{
return rbtree_simple_xattr_node_cmp(new_node, node) < 0;
}
/**
* simple_xattr_add - add xattr objects
* @xattrs: the header of the xattr object
* @new_xattr: the xattr object to add
*
* Add an xattr object to @xattrs. This assumes no replacement or removal
* of matching xattrs is wanted. Should only be called during inode
* initialization when a few distinct initial xattrs are supposed to be set.
*/
void simple_xattr_add(struct simple_xattrs *xattrs,
struct simple_xattr *new_xattr)
{
write_lock(&xattrs->lock);
rb_add(&new_xattr->rb_node, &xattrs->rb_root, rbtree_simple_xattr_less);
write_unlock(&xattrs->lock);
}
/**
* simple_xattrs_init - initialize new xattr header
* @xattrs: header to initialize
*
* Initialize relevant fields of a an xattr header.
*/
void simple_xattrs_init(struct simple_xattrs *xattrs)
{
xattrs->rb_root = RB_ROOT;
rwlock_init(&xattrs->lock);
}
/**
* simple_xattrs_free - free xattrs
* @xattrs: xattr header whose xattrs to destroy
* @freed_space: approximate number of bytes of memory freed from @xattrs
*
* Destroy all xattrs in @xattr. When this is called no one can hold a
* reference to any of the xattrs anymore.
*/
void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space)
{
struct rb_node *rbp;
if (freed_space)
*freed_space = 0;
rbp = rb_first(&xattrs->rb_root);
while (rbp) {
struct simple_xattr *xattr;
struct rb_node *rbp_next;
rbp_next = rb_next(rbp);
xattr = rb_entry(rbp, struct simple_xattr, rb_node);
rb_erase(&xattr->rb_node, &xattrs->rb_root);
if (freed_space)
*freed_space += simple_xattr_space(xattr->name,
xattr->size);
simple_xattr_free(xattr);
rbp = rbp_next;
}
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_FORTIFY_STRING_H_
#define _LINUX_FORTIFY_STRING_H_
#include <linux/bitfield.h>
#include <linux/bug.h>
#include <linux/const.h>
#include <linux/limits.h>
#define __FORTIFY_INLINE extern __always_inline __gnu_inline __overloadable
#define __RENAME(x) __asm__(#x)
#define FORTIFY_REASON_DIR(r) FIELD_GET(BIT(0), r)
#define FORTIFY_REASON_FUNC(r) FIELD_GET(GENMASK(7, 1), r)
#define FORTIFY_REASON(func, write) (FIELD_PREP(BIT(0), write) | \
FIELD_PREP(GENMASK(7, 1), func))
#ifndef fortify_panic
# define fortify_panic(func, write, avail, size, retfail) \
__fortify_panic(FORTIFY_REASON(func, write), avail, size)
#endif
#define FORTIFY_READ 0
#define FORTIFY_WRITE 1
#define EACH_FORTIFY_FUNC(macro) \
macro(strncpy), \
macro(strnlen), \
macro(strlen), \
macro(strscpy), \
macro(strlcat), \
macro(strcat), \
macro(strncat), \
macro(memset), \
macro(memcpy), \
macro(memmove), \
macro(memscan), \
macro(memcmp), \
macro(memchr), \
macro(memchr_inv), \
macro(kmemdup), \
macro(strcpy), \
macro(UNKNOWN),
#define MAKE_FORTIFY_FUNC(func) FORTIFY_FUNC_##func
enum fortify_func {
EACH_FORTIFY_FUNC(MAKE_FORTIFY_FUNC)
};
void __fortify_report(const u8 reason, const size_t avail, const size_t size);
void __fortify_panic(const u8 reason, const size_t avail, const size_t size) __cold __noreturn;
void __read_overflow(void) __compiletime_error("detected read beyond size of object (1st parameter)");
void __read_overflow2(void) __compiletime_error("detected read beyond size of object (2nd parameter)");
void __read_overflow2_field(size_t avail, size_t wanted) __compiletime_warning("detected read beyond size of field (2nd parameter); maybe use struct_group()?");
void __write_overflow(void) __compiletime_error("detected write beyond size of object (1st parameter)");
void __write_overflow_field(size_t avail, size_t wanted) __compiletime_warning("detected write beyond size of field (1st parameter); maybe use struct_group()?");
#define __compiletime_strlen(p) \
({ \
char *__p = (char *)(p); \
size_t __ret = SIZE_MAX; \
const size_t __p_size = __member_size(p); \
if (__p_size != SIZE_MAX && \
__builtin_constant_p(*__p)) { \
size_t __p_len = __p_size - 1; \
if (__builtin_constant_p(__p[__p_len]) && \
__p[__p_len] == '\0') \
__ret = __builtin_strlen(__p); \
} \
__ret; \
})
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
extern void *__underlying_memchr(const void *p, int c, __kernel_size_t size) __RENAME(memchr);
extern int __underlying_memcmp(const void *p, const void *q, __kernel_size_t size) __RENAME(memcmp);
extern void *__underlying_memcpy(void *p, const void *q, __kernel_size_t size) __RENAME(memcpy);
extern void *__underlying_memmove(void *p, const void *q, __kernel_size_t size) __RENAME(memmove);
extern void *__underlying_memset(void *p, int c, __kernel_size_t size) __RENAME(memset);
extern char *__underlying_strcat(char *p, const char *q) __RENAME(strcat);
extern char *__underlying_strcpy(char *p, const char *q) __RENAME(strcpy);
extern __kernel_size_t __underlying_strlen(const char *p) __RENAME(strlen);
extern char *__underlying_strncat(char *p, const char *q, __kernel_size_t count) __RENAME(strncat);
extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) __RENAME(strncpy);
#else
#if defined(__SANITIZE_MEMORY__)
/*
* For KMSAN builds all memcpy/memset/memmove calls should be replaced by the
* corresponding __msan_XXX functions.
*/
#include <linux/kmsan_string.h>
#define __underlying_memcpy __msan_memcpy
#define __underlying_memmove __msan_memmove
#define __underlying_memset __msan_memset
#else
#define __underlying_memcpy __builtin_memcpy
#define __underlying_memmove __builtin_memmove
#define __underlying_memset __builtin_memset
#endif
#define __underlying_memchr __builtin_memchr
#define __underlying_memcmp __builtin_memcmp
#define __underlying_strcat __builtin_strcat
#define __underlying_strcpy __builtin_strcpy
#define __underlying_strlen __builtin_strlen
#define __underlying_strncat __builtin_strncat
#define __underlying_strncpy __builtin_strncpy
#endif
/**
* unsafe_memcpy - memcpy implementation with no FORTIFY bounds checking
*
* @dst: Destination memory address to write to
* @src: Source memory address to read from
* @bytes: How many bytes to write to @dst from @src
* @justification: Free-form text or comment describing why the use is needed
*
* This should be used for corner cases where the compiler cannot do the
* right thing, or during transitions between APIs, etc. It should be used
* very rarely, and includes a place for justification detailing where bounds
* checking has happened, and why existing solutions cannot be employed.
*/
#define unsafe_memcpy(dst, src, bytes, justification) \
__underlying_memcpy(dst, src, bytes)
/*
* Clang's use of __builtin_*object_size() within inlines needs hinting via
* __pass_*object_size(). The preference is to only ever use type 1 (member
* size, rather than struct size), but there remain some stragglers using
* type 0 that will be converted in the future.
*/
#if __has_builtin(__builtin_dynamic_object_size)
#define POS __pass_dynamic_object_size(1)
#define POS0 __pass_dynamic_object_size(0)
#else
#define POS __pass_object_size(1)
#define POS0 __pass_object_size(0)
#endif
#define __compiletime_lessthan(bounds, length) ( \
__builtin_constant_p((bounds) < (length)) && \
(bounds) < (length) \
)
/**
* strncpy - Copy a string to memory with non-guaranteed NUL padding
*
* @p: pointer to destination of copy
* @q: pointer to NUL-terminated source string to copy
* @size: bytes to write at @p
*
* If strlen(@q) >= @size, the copy of @q will stop after @size bytes,
* and @p will NOT be NUL-terminated
*
* If strlen(@q) < @size, following the copy of @q, trailing NUL bytes
* will be written to @p until @size total bytes have been written.
*
* Do not use this function. While FORTIFY_SOURCE tries to avoid
* over-reads of @q, it cannot defend against writing unterminated
* results to @p. Using strncpy() remains ambiguous and fragile.
* Instead, please choose an alternative, so that the expectation
* of @p's contents is unambiguous:
*
* +--------------------+--------------------+------------+
* | **p** needs to be: | padded to **size** | not padded |
* +====================+====================+============+
* | NUL-terminated | strscpy_pad() | strscpy() |
* +--------------------+--------------------+------------+
* | not NUL-terminated | strtomem_pad() | strtomem() |
* +--------------------+--------------------+------------+
*
* Note strscpy*()'s differing return values for detecting truncation,
* and strtomem*()'s expectation that the destination is marked with
* __nonstring when it is a character array.
*
*/
__FORTIFY_INLINE __diagnose_as(__builtin_strncpy, 1, 2, 3)
char *strncpy(char * const POS p, const char *q, __kernel_size_t size)
{
const size_t p_size = __member_size(p);
if (__compiletime_lessthan(p_size, size))
__write_overflow();
if (p_size < size)
fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE, p_size, size, p);
return __underlying_strncpy(p, q, size);
}
extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(strnlen);
/**
* strnlen - Return bounded count of characters in a NUL-terminated string
*
* @p: pointer to NUL-terminated string to count.
* @maxlen: maximum number of characters to count.
*
* Returns number of characters in @p (NOT including the final NUL), or
* @maxlen, if no NUL has been found up to there.
*
*/
__FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size_t maxlen)
{
const size_t p_size = __member_size(p);
const size_t p_len = __compiletime_strlen(p);
size_t ret;
/* We can take compile-time actions when maxlen is const. */
if (__builtin_constant_p(maxlen) && p_len != SIZE_MAX) {
/* If p is const, we can use its compile-time-known len. */
if (maxlen >= p_size)
return p_len;
}
/* Do not check characters beyond the end of p. */
ret = __real_strnlen(p, maxlen < p_size ? maxlen : p_size); if (p_size <= ret && maxlen != ret) fortify_panic(FORTIFY_FUNC_strnlen, FORTIFY_READ, p_size, ret + 1, ret);
return ret;
}
/*
* Defined after fortified strnlen to reuse it. However, it must still be
* possible for strlen() to be used on compile-time strings for use in
* static initializers (i.e. as a constant expression).
*/
/**
* strlen - Return count of characters in a NUL-terminated string
*
* @p: pointer to NUL-terminated string to count.
*
* Do not use this function unless the string length is known at
* compile-time. When @p is unterminated, this function may crash
* or return unexpected counts that could lead to memory content
* exposures. Prefer strnlen().
*
* Returns number of characters in @p (NOT including the final NUL).
*
*/
#define strlen(p) \
__builtin_choose_expr(__is_constexpr(__builtin_strlen(p)), \
__builtin_strlen(p), __fortify_strlen(p))
__FORTIFY_INLINE __diagnose_as(__builtin_strlen, 1)
__kernel_size_t __fortify_strlen(const char * const POS p)
{
const size_t p_size = __member_size(p);
__kernel_size_t ret;
/* Give up if we don't know how large p is. */
if (p_size == SIZE_MAX)
return __underlying_strlen(p); ret = strnlen(p, p_size);
if (p_size <= ret)
fortify_panic(FORTIFY_FUNC_strlen, FORTIFY_READ, p_size, ret + 1, ret);
return ret;
}
/* Defined after fortified strnlen() to reuse it. */
extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(sized_strscpy);
__FORTIFY_INLINE ssize_t sized_strscpy(char * const POS p, const char * const POS q, size_t size)
{
/* Use string size rather than possible enclosing struct size. */
const size_t p_size = __member_size(p);
const size_t q_size = __member_size(q);
size_t len;
/* If we cannot get size of p and q default to call strscpy. */
if (p_size == SIZE_MAX && q_size == SIZE_MAX)
return __real_strscpy(p, q, size);
/*
* If size can be known at compile time and is greater than
* p_size, generate a compile time write overflow error.
*/
if (__compiletime_lessthan(p_size, size))
__write_overflow();
/* Short-circuit for compile-time known-safe lengths. */
if (__compiletime_lessthan(p_size, SIZE_MAX)) {
len = __compiletime_strlen(q);
if (len < SIZE_MAX && __compiletime_lessthan(len, size)) {
__underlying_memcpy(p, q, len + 1);
return len;
}
}
/*
* This call protects from read overflow, because len will default to q
* length if it smaller than size.
*/
len = strnlen(q, size);
/*
* If len equals size, we will copy only size bytes which leads to
* -E2BIG being returned.
* Otherwise we will copy len + 1 because of the final '\O'.
*/
len = len == size ? size : len + 1;
/*
* Generate a runtime write overflow error if len is greater than
* p_size.
*/
if (p_size < len) fortify_panic(FORTIFY_FUNC_strscpy, FORTIFY_WRITE, p_size, len, -E2BIG);
/*
* We can now safely call vanilla strscpy because we are protected from:
* 1. Read overflow thanks to call to strnlen().
* 2. Write overflow thanks to above ifs.
*/
return __real_strscpy(p, q, len);
}
/* Defined after fortified strlen() to reuse it. */
extern size_t __real_strlcat(char *p, const char *q, size_t avail) __RENAME(strlcat);
/**
* strlcat - Append a string to an existing string
*
* @p: pointer to %NUL-terminated string to append to
* @q: pointer to %NUL-terminated string to append from
* @avail: Maximum bytes available in @p
*
* Appends %NUL-terminated string @q after the %NUL-terminated
* string at @p, but will not write beyond @avail bytes total,
* potentially truncating the copy from @q. @p will stay
* %NUL-terminated only if a %NUL already existed within
* the @avail bytes of @p. If so, the resulting number of
* bytes copied from @q will be at most "@avail - strlen(@p) - 1".
*
* Do not use this function. While FORTIFY_SOURCE tries to avoid
* read and write overflows, this is only possible when the sizes
* of @p and @q are known to the compiler. Prefer building the
* string with formatting, via scnprintf(), seq_buf, or similar.
*
* Returns total bytes that _would_ have been contained by @p
* regardless of truncation, similar to snprintf(). If return
* value is >= @avail, the string has been truncated.
*
*/
__FORTIFY_INLINE
size_t strlcat(char * const POS p, const char * const POS q, size_t avail)
{
const size_t p_size = __member_size(p);
const size_t q_size = __member_size(q);
size_t p_len, copy_len;
size_t actual, wanted;
/* Give up immediately if both buffer sizes are unknown. */
if (p_size == SIZE_MAX && q_size == SIZE_MAX)
return __real_strlcat(p, q, avail);
p_len = strnlen(p, avail); copy_len = strlen(q);
wanted = actual = p_len + copy_len;
/* Cannot append any more: report truncation. */
if (avail <= p_len)
return wanted;
/* Give up if string is already overflowed. */
if (p_size <= p_len)
fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_READ, p_size, p_len + 1, wanted);
if (actual >= avail) {
copy_len = avail - p_len - 1;
actual = p_len + copy_len;
}
/* Give up if copy will overflow. */
if (p_size <= actual)
fortify_panic(FORTIFY_FUNC_strlcat, FORTIFY_WRITE, p_size, actual + 1, wanted);
__underlying_memcpy(p + p_len, q, copy_len);
p[actual] = '\0';
return wanted;
}
/* Defined after fortified strlcat() to reuse it. */
/**
* strcat - Append a string to an existing string
*
* @p: pointer to NUL-terminated string to append to
* @q: pointer to NUL-terminated source string to append from
*
* Do not use this function. While FORTIFY_SOURCE tries to avoid
* read and write overflows, this is only possible when the
* destination buffer size is known to the compiler. Prefer
* building the string with formatting, via scnprintf() or similar.
* At the very least, use strncat().
*
* Returns @p.
*
*/
__FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2)
char *strcat(char * const POS p, const char *q)
{
const size_t p_size = __member_size(p);
const size_t wanted = strlcat(p, q, p_size);
if (p_size <= wanted)
fortify_panic(FORTIFY_FUNC_strcat, FORTIFY_WRITE, p_size, wanted + 1, p);
return p;
}
/**
* strncat - Append a string to an existing string
*
* @p: pointer to NUL-terminated string to append to
* @q: pointer to source string to append from
* @count: Maximum bytes to read from @q
*
* Appends at most @count bytes from @q (stopping at the first
* NUL byte) after the NUL-terminated string at @p. @p will be
* NUL-terminated.
*
* Do not use this function. While FORTIFY_SOURCE tries to avoid
* read and write overflows, this is only possible when the sizes
* of @p and @q are known to the compiler. Prefer building the
* string with formatting, via scnprintf() or similar.
*
* Returns @p.
*
*/
/* Defined after fortified strlen() and strnlen() to reuse them. */
__FORTIFY_INLINE __diagnose_as(__builtin_strncat, 1, 2, 3)
char *strncat(char * const POS p, const char * const POS q, __kernel_size_t count)
{
const size_t p_size = __member_size(p);
const size_t q_size = __member_size(q);
size_t p_len, copy_len, total;
if (p_size == SIZE_MAX && q_size == SIZE_MAX)
return __underlying_strncat(p, q, count);
p_len = strlen(p);
copy_len = strnlen(q, count);
total = p_len + copy_len + 1;
if (p_size < total)
fortify_panic(FORTIFY_FUNC_strncat, FORTIFY_WRITE, p_size, total, p);
__underlying_memcpy(p + p_len, q, copy_len);
p[p_len + copy_len] = '\0';
return p;
}
__FORTIFY_INLINE bool fortify_memset_chk(__kernel_size_t size,
const size_t p_size,
const size_t p_size_field)
{
if (__builtin_constant_p(size)) {
/*
* Length argument is a constant expression, so we
* can perform compile-time bounds checking where
* buffer sizes are also known at compile time.
*/
/* Error when size is larger than enclosing struct. */
if (__compiletime_lessthan(p_size_field, p_size) &&
__compiletime_lessthan(p_size, size))
__write_overflow();
/* Warn when write size is larger than dest field. */
if (__compiletime_lessthan(p_size_field, size))
__write_overflow_field(p_size_field, size);
}
/*
* At this point, length argument may not be a constant expression,
* so run-time bounds checking can be done where buffer sizes are
* known. (This is not an "else" because the above checks may only
* be compile-time warnings, and we want to still warn for run-time
* overflows.)
*/
/*
* Always stop accesses beyond the struct that contains the
* field, when the buffer's remaining size is known.
* (The SIZE_MAX test is to optimize away checks where the buffer
* lengths are unknown.)
*/
if (p_size != SIZE_MAX && p_size < size)
fortify_panic(FORTIFY_FUNC_memset, FORTIFY_WRITE, p_size, size, true);
return false;
}
#define __fortify_memset_chk(p, c, size, p_size, p_size_field) ({ \
size_t __fortify_size = (size_t)(size); \
fortify_memset_chk(__fortify_size, p_size, p_size_field), \
__underlying_memset(p, c, __fortify_size); \
})
/*
* __struct_size() vs __member_size() must be captured here to avoid
* evaluating argument side-effects further into the macro layers.
*/
#ifndef CONFIG_KMSAN
#define memset(p, c, s) __fortify_memset_chk(p, c, s, \
__struct_size(p), __member_size(p))
#endif
/*
* To make sure the compiler can enforce protection against buffer overflows,
* memcpy(), memmove(), and memset() must not be used beyond individual
* struct members. If you need to copy across multiple members, please use
* struct_group() to create a named mirror of an anonymous struct union.
* (e.g. see struct sk_buff.) Read overflow checking is currently only
* done when a write overflow is also present, or when building with W=1.
*
* Mitigation coverage matrix
* Bounds checking at:
* +-------+-------+-------+-------+
* | Compile time | Run time |
* memcpy() argument sizes: | write | read | write | read |
* dest source length +-------+-------+-------+-------+
* memcpy(known, known, constant) | y | y | n/a | n/a |
* memcpy(known, unknown, constant) | y | n | n/a | V |
* memcpy(known, known, dynamic) | n | n | B | B |
* memcpy(known, unknown, dynamic) | n | n | B | V |
* memcpy(unknown, known, constant) | n | y | V | n/a |
* memcpy(unknown, unknown, constant) | n | n | V | V |
* memcpy(unknown, known, dynamic) | n | n | V | B |
* memcpy(unknown, unknown, dynamic) | n | n | V | V |
* +-------+-------+-------+-------+
*
* y = perform deterministic compile-time bounds checking
* n = cannot perform deterministic compile-time bounds checking
* n/a = no run-time bounds checking needed since compile-time deterministic
* B = can perform run-time bounds checking (currently unimplemented)
* V = vulnerable to run-time overflow (will need refactoring to solve)
*
*/
__FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size,
const size_t p_size,
const size_t q_size,
const size_t p_size_field,
const size_t q_size_field,
const u8 func)
{
if (__builtin_constant_p(size)) {
/*
* Length argument is a constant expression, so we
* can perform compile-time bounds checking where
* buffer sizes are also known at compile time.
*/
/* Error when size is larger than enclosing struct. */
if (__compiletime_lessthan(p_size_field, p_size) &&
__compiletime_lessthan(p_size, size))
__write_overflow();
if (__compiletime_lessthan(q_size_field, q_size) &&
__compiletime_lessthan(q_size, size))
__read_overflow2();
/* Warn when write size argument larger than dest field. */
if (__compiletime_lessthan(p_size_field, size))
__write_overflow_field(p_size_field, size);
/*
* Warn for source field over-read when building with W=1
* or when an over-write happened, so both can be fixed at
* the same time.
*/
if ((IS_ENABLED(KBUILD_EXTRA_WARN1) ||
__compiletime_lessthan(p_size_field, size)) &&
__compiletime_lessthan(q_size_field, size))
__read_overflow2_field(q_size_field, size);
}
/*
* At this point, length argument may not be a constant expression,
* so run-time bounds checking can be done where buffer sizes are
* known. (This is not an "else" because the above checks may only
* be compile-time warnings, and we want to still warn for run-time
* overflows.)
*/
/*
* Always stop accesses beyond the struct that contains the
* field, when the buffer's remaining size is known.
* (The SIZE_MAX test is to optimize away checks where the buffer
* lengths are unknown.)
*/
if (p_size != SIZE_MAX && p_size < size)
fortify_panic(func, FORTIFY_WRITE, p_size, size, true); else if (q_size != SIZE_MAX && q_size < size) fortify_panic(func, FORTIFY_READ, p_size, size, true);
/*
* Warn when writing beyond destination field size.
*
* We must ignore p_size_field == 0 for existing 0-element
* fake flexible arrays, until they are all converted to
* proper flexible arrays.
*
* The implementation of __builtin_*object_size() behaves
* like sizeof() when not directly referencing a flexible
* array member, which means there will be many bounds checks
* that will appear at run-time, without a way for them to be
* detected at compile-time (as can be done when the destination
* is specifically the flexible array member).
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101832
*/
if (p_size_field != 0 && p_size_field != SIZE_MAX &&
p_size != p_size_field && p_size_field < size)
return true;
return false;
}
#define __fortify_memcpy_chk(p, q, size, p_size, q_size, \
p_size_field, q_size_field, op) ({ \
const size_t __fortify_size = (size_t)(size); \
const size_t __p_size = (p_size); \
const size_t __q_size = (q_size); \
const size_t __p_size_field = (p_size_field); \
const size_t __q_size_field = (q_size_field); \
WARN_ONCE(fortify_memcpy_chk(__fortify_size, __p_size, \
__q_size, __p_size_field, \
__q_size_field, FORTIFY_FUNC_ ##op), \
#op ": detected field-spanning write (size %zu) of single %s (size %zu)\n", \
__fortify_size, \
"field \"" #p "\" at " FILE_LINE, \
__p_size_field); \
__underlying_##op(p, q, __fortify_size); \
})
/*
* Notes about compile-time buffer size detection:
*
* With these types...
*
* struct middle {
* u16 a;
* u8 middle_buf[16];
* int b;
* };
* struct end {
* u16 a;
* u8 end_buf[16];
* };
* struct flex {
* int a;
* u8 flex_buf[];
* };
*
* void func(TYPE *ptr) { ... }
*
* Cases where destination size cannot be currently detected:
* - the size of ptr's object (seemingly by design, gcc & clang fail):
* __builtin_object_size(ptr, 1) == SIZE_MAX
* - the size of flexible arrays in ptr's obj (by design, dynamic size):
* __builtin_object_size(ptr->flex_buf, 1) == SIZE_MAX
* - the size of ANY array at the end of ptr's obj (gcc and clang bug):
* __builtin_object_size(ptr->end_buf, 1) == SIZE_MAX
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836
*
* Cases where destination size is currently detected:
* - the size of non-array members within ptr's object:
* __builtin_object_size(ptr->a, 1) == 2
* - the size of non-flexible-array in the middle of ptr's obj:
* __builtin_object_size(ptr->middle_buf, 1) == 16
*
*/
/*
* __struct_size() vs __member_size() must be captured here to avoid
* evaluating argument side-effects further into the macro layers.
*/
#define memcpy(p, q, s) __fortify_memcpy_chk(p, q, s, \
__struct_size(p), __struct_size(q), \
__member_size(p), __member_size(q), \
memcpy)
#define memmove(p, q, s) __fortify_memcpy_chk(p, q, s, \
__struct_size(p), __struct_size(q), \
__member_size(p), __member_size(q), \
memmove)
extern void *__real_memscan(void *, int, __kernel_size_t) __RENAME(memscan);
__FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size)
{
const size_t p_size = __struct_size(p);
if (__compiletime_lessthan(p_size, size))
__read_overflow();
if (p_size < size)
fortify_panic(FORTIFY_FUNC_memscan, FORTIFY_READ, p_size, size, NULL);
return __real_memscan(p, c, size);
}
__FORTIFY_INLINE __diagnose_as(__builtin_memcmp, 1, 2, 3)
int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t size)
{
const size_t p_size = __struct_size(p);
const size_t q_size = __struct_size(q);
if (__builtin_constant_p(size)) {
if (__compiletime_lessthan(p_size, size))
__read_overflow();
if (__compiletime_lessthan(q_size, size))
__read_overflow2();
}
if (p_size < size)
fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, p_size, size, INT_MIN);
else if (q_size < size)
fortify_panic(FORTIFY_FUNC_memcmp, FORTIFY_READ, q_size, size, INT_MIN);
return __underlying_memcmp(p, q, size);
}
__FORTIFY_INLINE __diagnose_as(__builtin_memchr, 1, 2, 3)
void *memchr(const void * const POS0 p, int c, __kernel_size_t size)
{
const size_t p_size = __struct_size(p);
if (__compiletime_lessthan(p_size, size))
__read_overflow();
if (p_size < size)
fortify_panic(FORTIFY_FUNC_memchr, FORTIFY_READ, p_size, size, NULL);
return __underlying_memchr(p, c, size);
}
void *__real_memchr_inv(const void *s, int c, size_t n) __RENAME(memchr_inv);
__FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size)
{
const size_t p_size = __struct_size(p);
if (__compiletime_lessthan(p_size, size))
__read_overflow();
if (p_size < size)
fortify_panic(FORTIFY_FUNC_memchr_inv, FORTIFY_READ, p_size, size, NULL);
return __real_memchr_inv(p, c, size);
}
extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup)
__realloc_size(2);
__FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp)
{
const size_t p_size = __struct_size(p);
if (__compiletime_lessthan(p_size, size))
__read_overflow();
if (p_size < size)
fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL);
return __real_kmemdup(p, size, gfp);
}
/**
* strcpy - Copy a string into another string buffer
*
* @p: pointer to destination of copy
* @q: pointer to NUL-terminated source string to copy
*
* Do not use this function. While FORTIFY_SOURCE tries to avoid
* overflows, this is only possible when the sizes of @q and @p are
* known to the compiler. Prefer strscpy(), though note its different
* return values for detecting truncation.
*
* Returns @p.
*
*/
/* Defined after fortified strlen to reuse it. */
__FORTIFY_INLINE __diagnose_as(__builtin_strcpy, 1, 2)
char *strcpy(char * const POS p, const char * const POS q)
{
const size_t p_size = __member_size(p);
const size_t q_size = __member_size(q);
size_t size;
/* If neither buffer size is known, immediately give up. */
if (__builtin_constant_p(p_size) &&
__builtin_constant_p(q_size) &&
p_size == SIZE_MAX && q_size == SIZE_MAX)
return __underlying_strcpy(p, q);
size = strlen(q) + 1;
/* Compile-time check for const size overflow. */
if (__compiletime_lessthan(p_size, size))
__write_overflow();
/* Run-time check for dynamic size overflow. */
if (p_size < size)
fortify_panic(FORTIFY_FUNC_strcpy, FORTIFY_WRITE, p_size, size, p); __underlying_memcpy(p, q, size);
return p;
}
/* Don't use these outside the FORITFY_SOURCE implementation */
#undef __underlying_memchr
#undef __underlying_memcmp
#undef __underlying_strcat
#undef __underlying_strcpy
#undef __underlying_strlen
#undef __underlying_strncat
#undef __underlying_strncpy
#undef POS
#undef POS0
#endif /* _LINUX_FORTIFY_STRING_H_ */
// SPDX-License-Identifier: GPL-2.0
/*
* scsi_scan.c
*
* Copyright (C) 2000 Eric Youngdale,
* Copyright (C) 2002 Patrick Mansfield
*
* The general scanning/probing algorithm is as follows, exceptions are
* made to it depending on device specific flags, compilation options, and
* global variable (boot or module load time) settings.
*
* A specific LUN is scanned via an INQUIRY command; if the LUN has a
* device attached, a scsi_device is allocated and setup for it.
*
* For every id of every channel on the given host:
*
* Scan LUN 0; if the target responds to LUN 0 (even if there is no
* device or storage attached to LUN 0):
*
* If LUN 0 has a device attached, allocate and setup a
* scsi_device for it.
*
* If target is SCSI-3 or up, issue a REPORT LUN, and scan
* all of the LUNs returned by the REPORT LUN; else,
* sequentially scan LUNs up until some maximum is reached,
* or a LUN is seen that cannot have a device attached to it.
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/spinlock.h>
#include <linux/async.h>
#include <linux/slab.h>
#include <asm/unaligned.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_devinfo.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_dh.h>
#include <scsi/scsi_eh.h>
#include "scsi_priv.h"
#include "scsi_logging.h"
#define ALLOC_FAILURE_MSG KERN_ERR "%s: Allocation failure during" \
" SCSI scanning, some SCSI devices might not be configured\n"
/*
* Default timeout
*/
#define SCSI_TIMEOUT (2*HZ)
#define SCSI_REPORT_LUNS_TIMEOUT (30*HZ)
/*
* Prefix values for the SCSI id's (stored in sysfs name field)
*/
#define SCSI_UID_SER_NUM 'S'
#define SCSI_UID_UNKNOWN 'Z'
/*
* Return values of some of the scanning functions.
*
* SCSI_SCAN_NO_RESPONSE: no valid response received from the target, this
* includes allocation or general failures preventing IO from being sent.
*
* SCSI_SCAN_TARGET_PRESENT: target responded, but no device is available
* on the given LUN.
*
* SCSI_SCAN_LUN_PRESENT: target responded, and a device is available on a
* given LUN.
*/
#define SCSI_SCAN_NO_RESPONSE 0
#define SCSI_SCAN_TARGET_PRESENT 1
#define SCSI_SCAN_LUN_PRESENT 2
static const char *scsi_null_device_strs = "nullnullnullnull";
#define MAX_SCSI_LUNS 512
static u64 max_scsi_luns = MAX_SCSI_LUNS;
module_param_named(max_luns, max_scsi_luns, ullong, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(max_luns,
"last scsi LUN (should be between 1 and 2^64-1)");
#ifdef CONFIG_SCSI_SCAN_ASYNC
#define SCSI_SCAN_TYPE_DEFAULT "async"
#else
#define SCSI_SCAN_TYPE_DEFAULT "sync"
#endif
static char scsi_scan_type[7] = SCSI_SCAN_TYPE_DEFAULT;
module_param_string(scan, scsi_scan_type, sizeof(scsi_scan_type),
S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(scan, "sync, async, manual, or none. "
"Setting to 'manual' disables automatic scanning, but allows "
"for manual device scan via the 'scan' sysfs attribute.");
static unsigned int scsi_inq_timeout = SCSI_TIMEOUT/HZ + 18;
module_param_named(inq_timeout, scsi_inq_timeout, uint, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(inq_timeout,
"Timeout (in seconds) waiting for devices to answer INQUIRY."
" Default is 20. Some devices may need more; most need less.");
/* This lock protects only this list */
static DEFINE_SPINLOCK(async_scan_lock);
static LIST_HEAD(scanning_hosts);
struct async_scan_data {
struct list_head list;
struct Scsi_Host *shost;
struct completion prev_finished;
};
/*
* scsi_enable_async_suspend - Enable async suspend and resume
*/
void scsi_enable_async_suspend(struct device *dev)
{
/*
* If a user has disabled async probing a likely reason is due to a
* storage enclosure that does not inject staggered spin-ups. For
* safety, make resume synchronous as well in that case.
*/
if (strncmp(scsi_scan_type, "async", 5) != 0)
return;
/* Enable asynchronous suspend and resume. */
device_enable_async_suspend(dev);
}
/**
* scsi_complete_async_scans - Wait for asynchronous scans to complete
*
* When this function returns, any host which started scanning before
* this function was called will have finished its scan. Hosts which
* started scanning after this function was called may or may not have
* finished.
*/
int scsi_complete_async_scans(void)
{
struct async_scan_data *data;
do {
if (list_empty(&scanning_hosts))
return 0;
/* If we can't get memory immediately, that's OK. Just
* sleep a little. Even if we never get memory, the async
* scans will finish eventually.
*/
data = kmalloc(sizeof(*data), GFP_KERNEL);
if (!data)
msleep(1);
} while (!data);
data->shost = NULL;
init_completion(&data->prev_finished);
spin_lock(&async_scan_lock);
/* Check that there's still somebody else on the list */
if (list_empty(&scanning_hosts))
goto done;
list_add_tail(&data->list, &scanning_hosts);
spin_unlock(&async_scan_lock);
printk(KERN_INFO "scsi: waiting for bus probes to complete ...\n");
wait_for_completion(&data->prev_finished);
spin_lock(&async_scan_lock);
list_del(&data->list);
if (!list_empty(&scanning_hosts)) {
struct async_scan_data *next = list_entry(scanning_hosts.next,
struct async_scan_data, list);
complete(&next->prev_finished);
}
done:
spin_unlock(&async_scan_lock);
kfree(data);
return 0;
}
/**
* scsi_unlock_floptical - unlock device via a special MODE SENSE command
* @sdev: scsi device to send command to
* @result: area to store the result of the MODE SENSE
*
* Description:
* Send a vendor specific MODE SENSE (not a MODE SELECT) command.
* Called for BLIST_KEY devices.
**/
static void scsi_unlock_floptical(struct scsi_device *sdev,
unsigned char *result)
{
unsigned char scsi_cmd[MAX_COMMAND_SIZE];
sdev_printk(KERN_NOTICE, sdev, "unlocking floptical drive\n");
scsi_cmd[0] = MODE_SENSE;
scsi_cmd[1] = 0;
scsi_cmd[2] = 0x2e;
scsi_cmd[3] = 0;
scsi_cmd[4] = 0x2a; /* size */
scsi_cmd[5] = 0;
scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN, result, 0x2a,
SCSI_TIMEOUT, 3, NULL);
}
static int scsi_realloc_sdev_budget_map(struct scsi_device *sdev,
unsigned int depth)
{
int new_shift = sbitmap_calculate_shift(depth);
bool need_alloc = !sdev->budget_map.map;
bool need_free = false;
int ret;
struct sbitmap sb_backup;
depth = min_t(unsigned int, depth, scsi_device_max_queue_depth(sdev));
/*
* realloc if new shift is calculated, which is caused by setting
* up one new default queue depth after calling ->slave_configure
*/
if (!need_alloc && new_shift != sdev->budget_map.shift)
need_alloc = need_free = true;
if (!need_alloc)
return 0;
/*
* Request queue has to be frozen for reallocating budget map,
* and here disk isn't added yet, so freezing is pretty fast
*/
if (need_free) {
blk_mq_freeze_queue(sdev->request_queue);
sb_backup = sdev->budget_map;
}
ret = sbitmap_init_node(&sdev->budget_map,
scsi_device_max_queue_depth(sdev),
new_shift, GFP_KERNEL,
sdev->request_queue->node, false, true);
if (!ret)
sbitmap_resize(&sdev->budget_map, depth);
if (need_free) {
if (ret)
sdev->budget_map = sb_backup;
else
sbitmap_free(&sb_backup);
ret = 0;
blk_mq_unfreeze_queue(sdev->request_queue);
}
return ret;
}
/**
* scsi_alloc_sdev - allocate and setup a scsi_Device
* @starget: which target to allocate a &scsi_device for
* @lun: which lun
* @hostdata: usually NULL and set by ->slave_alloc instead
*
* Description:
* Allocate, initialize for io, and return a pointer to a scsi_Device.
* Stores the @shost, @channel, @id, and @lun in the scsi_Device, and
* adds scsi_Device to the appropriate list.
*
* Return value:
* scsi_Device pointer, or NULL on failure.
**/
static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget,
u64 lun, void *hostdata)
{
unsigned int depth;
struct scsi_device *sdev;
struct request_queue *q;
int display_failure_msg = 1, ret;
struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
sdev = kzalloc(sizeof(*sdev) + shost->transportt->device_size,
GFP_KERNEL);
if (!sdev)
goto out;
sdev->vendor = scsi_null_device_strs;
sdev->model = scsi_null_device_strs;
sdev->rev = scsi_null_device_strs;
sdev->host = shost;
sdev->queue_ramp_up_period = SCSI_DEFAULT_RAMP_UP_PERIOD;
sdev->id = starget->id;
sdev->lun = lun;
sdev->channel = starget->channel;
mutex_init(&sdev->state_mutex);
sdev->sdev_state = SDEV_CREATED;
INIT_LIST_HEAD(&sdev->siblings);
INIT_LIST_HEAD(&sdev->same_target_siblings);
INIT_LIST_HEAD(&sdev->starved_entry);
INIT_LIST_HEAD(&sdev->event_list);
spin_lock_init(&sdev->list_lock);
mutex_init(&sdev->inquiry_mutex);
INIT_WORK(&sdev->event_work, scsi_evt_thread);
INIT_WORK(&sdev->requeue_work, scsi_requeue_run_queue);
sdev->sdev_gendev.parent = get_device(&starget->dev);
sdev->sdev_target = starget;
/* usually NULL and set by ->slave_alloc instead */
sdev->hostdata = hostdata;
/* if the device needs this changing, it may do so in the
* slave_configure function */
sdev->max_device_blocked = SCSI_DEFAULT_DEVICE_BLOCKED;
/*
* Some low level driver could use device->type
*/
sdev->type = -1;
/*
* Assume that the device will have handshaking problems,
* and then fix this field later if it turns out it
* doesn't
*/
sdev->borken = 1;
sdev->sg_reserved_size = INT_MAX;
q = blk_mq_alloc_queue(&sdev->host->tag_set, NULL, NULL);
if (IS_ERR(q)) {
/* release fn is set up in scsi_sysfs_device_initialise, so
* have to free and put manually here */
put_device(&starget->dev);
kfree(sdev);
goto out;
}
kref_get(&sdev->host->tagset_refcnt);
sdev->request_queue = q;
q->queuedata = sdev;
__scsi_init_queue(sdev->host, q);
depth = sdev->host->cmd_per_lun ?: 1;
/*
* Use .can_queue as budget map's depth because we have to
* support adjusting queue depth from sysfs. Meantime use
* default device queue depth to figure out sbitmap shift
* since we use this queue depth most of times.
*/
if (scsi_realloc_sdev_budget_map(sdev, depth)) {
put_device(&starget->dev);
kfree(sdev);
goto out;
}
scsi_change_queue_depth(sdev, depth);
scsi_sysfs_device_initialize(sdev);
if (shost->hostt->slave_alloc) {
ret = shost->hostt->slave_alloc(sdev);
if (ret) {
/*
* if LLDD reports slave not present, don't clutter
* console with alloc failure messages
*/
if (ret == -ENXIO)
display_failure_msg = 0;
goto out_device_destroy;
}
}
return sdev;
out_device_destroy:
__scsi_remove_device(sdev);
out:
if (display_failure_msg)
printk(ALLOC_FAILURE_MSG, __func__);
return NULL;
}
static void scsi_target_destroy(struct scsi_target *starget)
{
struct device *dev = &starget->dev;
struct Scsi_Host *shost = dev_to_shost(dev->parent);
unsigned long flags;
BUG_ON(starget->state == STARGET_DEL);
starget->state = STARGET_DEL;
transport_destroy_device(dev);
spin_lock_irqsave(shost->host_lock, flags);
if (shost->hostt->target_destroy)
shost->hostt->target_destroy(starget);
list_del_init(&starget->siblings);
spin_unlock_irqrestore(shost->host_lock, flags);
put_device(dev);
}
static void scsi_target_dev_release(struct device *dev)
{
struct device *parent = dev->parent;
struct scsi_target *starget = to_scsi_target(dev);
kfree(starget);
put_device(parent);
}
static const struct device_type scsi_target_type = {
.name = "scsi_target",
.release = scsi_target_dev_release,
};
int scsi_is_target_device(const struct device *dev)
{
return dev->type == &scsi_target_type;
}
EXPORT_SYMBOL(scsi_is_target_device);
static struct scsi_target *__scsi_find_target(struct device *parent,
int channel, uint id)
{
struct scsi_target *starget, *found_starget = NULL;
struct Scsi_Host *shost = dev_to_shost(parent);
/*
* Search for an existing target for this sdev.
*/
list_for_each_entry(starget, &shost->__targets, siblings) {
if (starget->id == id &&
starget->channel == channel) {
found_starget = starget;
break;
}
}
if (found_starget)
get_device(&found_starget->dev);
return found_starget;
}
/**
* scsi_target_reap_ref_release - remove target from visibility
* @kref: the reap_ref in the target being released
*
* Called on last put of reap_ref, which is the indication that no device
* under this target is visible anymore, so render the target invisible in
* sysfs. Note: we have to be in user context here because the target reaps
* should be done in places where the scsi device visibility is being removed.
*/
static void scsi_target_reap_ref_release(struct kref *kref)
{
struct scsi_target *starget
= container_of(kref, struct scsi_target, reap_ref);
/*
* if we get here and the target is still in a CREATED state that
* means it was allocated but never made visible (because a scan
* turned up no LUNs), so don't call device_del() on it.
*/
if ((starget->state != STARGET_CREATED) &&
(starget->state != STARGET_CREATED_REMOVE)) {
transport_remove_device(&starget->dev);
device_del(&starget->dev);
}
scsi_target_destroy(starget);
}
static void scsi_target_reap_ref_put(struct scsi_target *starget)
{
kref_put(&starget->reap_ref, scsi_target_reap_ref_release);
}
/**
* scsi_alloc_target - allocate a new or find an existing target
* @parent: parent of the target (need not be a scsi host)
* @channel: target channel number (zero if no channels)
* @id: target id number
*
* Return an existing target if one exists, provided it hasn't already
* gone into STARGET_DEL state, otherwise allocate a new target.
*
* The target is returned with an incremented reference, so the caller
* is responsible for both reaping and doing a last put
*/
static struct scsi_target *scsi_alloc_target(struct device *parent,
int channel, uint id)
{
struct Scsi_Host *shost = dev_to_shost(parent);
struct device *dev = NULL;
unsigned long flags;
const int size = sizeof(struct scsi_target)
+ shost->transportt->target_size;
struct scsi_target *starget;
struct scsi_target *found_target;
int error, ref_got;
starget = kzalloc(size, GFP_KERNEL);
if (!starget) {
printk(KERN_ERR "%s: allocation failure\n", __func__);
return NULL;
}
dev = &starget->dev;
device_initialize(dev);
kref_init(&starget->reap_ref);
dev->parent = get_device(parent);
dev_set_name(dev, "target%d:%d:%d", shost->host_no, channel, id);
dev->bus = &scsi_bus_type;
dev->type = &scsi_target_type;
scsi_enable_async_suspend(dev);
starget->id = id;
starget->channel = channel;
starget->can_queue = 0;
INIT_LIST_HEAD(&starget->siblings);
INIT_LIST_HEAD(&starget->devices);
starget->state = STARGET_CREATED;
starget->scsi_level = SCSI_2;
starget->max_target_blocked = SCSI_DEFAULT_TARGET_BLOCKED;
retry:
spin_lock_irqsave(shost->host_lock, flags);
found_target = __scsi_find_target(parent, channel, id);
if (found_target)
goto found;
list_add_tail(&starget->siblings, &shost->__targets);
spin_unlock_irqrestore(shost->host_lock, flags);
/* allocate and add */
transport_setup_device(dev);
if (shost->hostt->target_alloc) {
error = shost->hostt->target_alloc(starget);
if(error) {
if (error != -ENXIO)
dev_err(dev, "target allocation failed, error %d\n", error);
/* don't want scsi_target_reap to do the final
* put because it will be under the host lock */
scsi_target_destroy(starget);
return NULL;
}
}
get_device(dev);
return starget;
found:
/*
* release routine already fired if kref is zero, so if we can still
* take the reference, the target must be alive. If we can't, it must
* be dying and we need to wait for a new target
*/
ref_got = kref_get_unless_zero(&found_target->reap_ref);
spin_unlock_irqrestore(shost->host_lock, flags);
if (ref_got) {
put_device(dev);
return found_target;
}
/*
* Unfortunately, we found a dying target; need to wait until it's
* dead before we can get a new one. There is an anomaly here. We
* *should* call scsi_target_reap() to balance the kref_get() of the
* reap_ref above. However, since the target being released, it's
* already invisible and the reap_ref is irrelevant. If we call
* scsi_target_reap() we might spuriously do another device_del() on
* an already invisible target.
*/
put_device(&found_target->dev);
/*
* length of time is irrelevant here, we just want to yield the CPU
* for a tick to avoid busy waiting for the target to die.
*/
msleep(1);
goto retry;
}
/**
* scsi_target_reap - check to see if target is in use and destroy if not
* @starget: target to be checked
*
* This is used after removing a LUN or doing a last put of the target
* it checks atomically that nothing is using the target and removes
* it if so.
*/
void scsi_target_reap(struct scsi_target *starget)
{
/*
* serious problem if this triggers: STARGET_DEL is only set in the if
* the reap_ref drops to zero, so we're trying to do another final put
* on an already released kref
*/
BUG_ON(starget->state == STARGET_DEL);
scsi_target_reap_ref_put(starget);
}
/**
* scsi_sanitize_inquiry_string - remove non-graphical chars from an
* INQUIRY result string
* @s: INQUIRY result string to sanitize
* @len: length of the string
*
* Description:
* The SCSI spec says that INQUIRY vendor, product, and revision
* strings must consist entirely of graphic ASCII characters,
* padded on the right with spaces. Since not all devices obey
* this rule, we will replace non-graphic or non-ASCII characters
* with spaces. Exception: a NUL character is interpreted as a
* string terminator, so all the following characters are set to
* spaces.
**/
void scsi_sanitize_inquiry_string(unsigned char *s, int len)
{
int terminated = 0;
for (; len > 0; (--len, ++s)) {
if (*s == 0)
terminated = 1;
if (terminated || *s < 0x20 || *s > 0x7e)
*s = ' ';
}
}
EXPORT_SYMBOL(scsi_sanitize_inquiry_string);
/**
* scsi_probe_lun - probe a single LUN using a SCSI INQUIRY
* @sdev: scsi_device to probe
* @inq_result: area to store the INQUIRY result
* @result_len: len of inq_result
* @bflags: store any bflags found here
*
* Description:
* Probe the lun associated with @req using a standard SCSI INQUIRY;
*
* If the INQUIRY is successful, zero is returned and the
* INQUIRY data is in @inq_result; the scsi_level and INQUIRY length
* are copied to the scsi_device any flags value is stored in *@bflags.
**/
static int scsi_probe_lun(struct scsi_device *sdev, unsigned char *inq_result,
int result_len, blist_flags_t *bflags)
{
unsigned char scsi_cmd[MAX_COMMAND_SIZE];
int first_inquiry_len, try_inquiry_len, next_inquiry_len;
int response_len = 0;
int pass, count, result, resid;
struct scsi_failure failure_defs[] = {
/*
* not-ready to ready transition [asc/ascq=0x28/0x0] or
* power-on, reset [asc/ascq=0x29/0x0], continue. INQUIRY
* should not yield UNIT_ATTENTION but many buggy devices do
* so anyway.
*/
{
.sense = UNIT_ATTENTION,
.asc = 0x28,
.result = SAM_STAT_CHECK_CONDITION,
},
{
.sense = UNIT_ATTENTION,
.asc = 0x29,
.result = SAM_STAT_CHECK_CONDITION,
},
{
.allowed = 1,
.result = DID_TIME_OUT << 16,
},
{}
};
struct scsi_failures failures = {
.total_allowed = 3,
.failure_definitions = failure_defs,
};
const struct scsi_exec_args exec_args = {
.resid = &resid,
.failures = &failures,
};
*bflags = 0;
/* Perform up to 3 passes. The first pass uses a conservative
* transfer length of 36 unless sdev->inquiry_len specifies a
* different value. */
first_inquiry_len = sdev->inquiry_len ? sdev->inquiry_len : 36;
try_inquiry_len = first_inquiry_len;
pass = 1;
next_pass:
SCSI_LOG_SCAN_BUS(3, sdev_printk(KERN_INFO, sdev,
"scsi scan: INQUIRY pass %d length %d\n",
pass, try_inquiry_len));
/* Each pass gets up to three chances to ignore Unit Attention */
scsi_failures_reset_retries(&failures);
for (count = 0; count < 3; ++count) {
memset(scsi_cmd, 0, 6);
scsi_cmd[0] = INQUIRY;
scsi_cmd[4] = (unsigned char) try_inquiry_len;
memset(inq_result, 0, try_inquiry_len);
result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN,
inq_result, try_inquiry_len,
HZ / 2 + HZ * scsi_inq_timeout, 3,
&exec_args);
SCSI_LOG_SCAN_BUS(3, sdev_printk(KERN_INFO, sdev,
"scsi scan: INQUIRY %s with code 0x%x\n",
result ? "failed" : "successful", result));
if (result == 0) {
/*
* if nothing was transferred, we try
* again. It's a workaround for some USB
* devices.
*/
if (resid == try_inquiry_len)
continue;
}
break;
}
if (result == 0) {
scsi_sanitize_inquiry_string(&inq_result[8], 8);
scsi_sanitize_inquiry_string(&inq_result[16], 16);
scsi_sanitize_inquiry_string(&inq_result[32], 4);
response_len = inq_result[4] + 5;
if (response_len > 255)
response_len = first_inquiry_len; /* sanity */
/*
* Get any flags for this device.
*
* XXX add a bflags to scsi_device, and replace the
* corresponding bit fields in scsi_device, so bflags
* need not be passed as an argument.
*/
*bflags = scsi_get_device_flags(sdev, &inq_result[8],
&inq_result[16]);
/* When the first pass succeeds we gain information about
* what larger transfer lengths might work. */
if (pass == 1) {
if (BLIST_INQUIRY_36 & *bflags)
next_inquiry_len = 36;
/*
* LLD specified a maximum sdev->inquiry_len
* but device claims it has more data. Capping
* the length only makes sense for legacy
* devices. If a device supports SPC-4 (2014)
* or newer, assume that it is safe to ask for
* as much as the device says it supports.
*/
else if (sdev->inquiry_len &&
response_len > sdev->inquiry_len &&
(inq_result[2] & 0x7) < 6) /* SPC-4 */
next_inquiry_len = sdev->inquiry_len;
else
next_inquiry_len = response_len;
/* If more data is available perform the second pass */
if (next_inquiry_len > try_inquiry_len) {
try_inquiry_len = next_inquiry_len;
pass = 2;
goto next_pass;
}
}
} else if (pass == 2) {
sdev_printk(KERN_INFO, sdev,
"scsi scan: %d byte inquiry failed. "
"Consider BLIST_INQUIRY_36 for this device\n",
try_inquiry_len);
/* If this pass failed, the third pass goes back and transfers
* the same amount as we successfully got in the first pass. */
try_inquiry_len = first_inquiry_len;
pass = 3;
goto next_pass;
}
/* If the last transfer attempt got an error, assume the
* peripheral doesn't exist or is dead. */
if (result)
return -EIO;
/* Don't report any more data than the device says is valid */
sdev->inquiry_len = min(try_inquiry_len, response_len);
/*
* XXX Abort if the response length is less than 36? If less than
* 32, the lookup of the device flags (above) could be invalid,
* and it would be possible to take an incorrect action - we do
* not want to hang because of a short INQUIRY. On the flip side,
* if the device is spun down or becoming ready (and so it gives a
* short INQUIRY), an abort here prevents any further use of the
* device, including spin up.
*
* On the whole, the best approach seems to be to assume the first
* 36 bytes are valid no matter what the device says. That's
* better than copying < 36 bytes to the inquiry-result buffer
* and displaying garbage for the Vendor, Product, or Revision
* strings.
*/
if (sdev->inquiry_len < 36) {
if (!sdev->host->short_inquiry) {
shost_printk(KERN_INFO, sdev->host,
"scsi scan: INQUIRY result too short (%d),"
" using 36\n", sdev->inquiry_len);
sdev->host->short_inquiry = 1;
}
sdev->inquiry_len = 36;
}
/*
* Related to the above issue:
*
* XXX Devices (disk or all?) should be sent a TEST UNIT READY,
* and if not ready, sent a START_STOP to start (maybe spin up) and
* then send the INQUIRY again, since the INQUIRY can change after
* a device is initialized.
*
* Ideally, start a device if explicitly asked to do so. This
* assumes that a device is spun up on power on, spun down on
* request, and then spun up on request.
*/
/*
* The scanning code needs to know the scsi_level, even if no
* device is attached at LUN 0 (SCSI_SCAN_TARGET_PRESENT) so
* non-zero LUNs can be scanned.
*/
sdev->scsi_level = inq_result[2] & 0x0f;
if (sdev->scsi_level >= 2 ||
(sdev->scsi_level == 1 && (inq_result[3] & 0x0f) == 1))
sdev->scsi_level++;
sdev->sdev_target->scsi_level = sdev->scsi_level;
/*
* If SCSI-2 or lower, and if the transport requires it,
* store the LUN value in CDB[1].
*/
sdev->lun_in_cdb = 0;
if (sdev->scsi_level <= SCSI_2 &&
sdev->scsi_level != SCSI_UNKNOWN &&
!sdev->host->no_scsi2_lun_in_cdb)
sdev->lun_in_cdb = 1;
return 0;
}
/**
* scsi_add_lun - allocate and fully initialze a scsi_device
* @sdev: holds information to be stored in the new scsi_device
* @inq_result: holds the result of a previous INQUIRY to the LUN
* @bflags: black/white list flag
* @async: 1 if this device is being scanned asynchronously
*
* Description:
* Initialize the scsi_device @sdev. Optionally set fields based
* on values in *@bflags.
*
* Return:
* SCSI_SCAN_NO_RESPONSE: could not allocate or setup a scsi_device
* SCSI_SCAN_LUN_PRESENT: a new scsi_device was allocated and initialized
**/
static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result,
blist_flags_t *bflags, int async)
{
int ret;
/*
* XXX do not save the inquiry, since it can change underneath us,
* save just vendor/model/rev.
*
* Rather than save it and have an ioctl that retrieves the saved
* value, have an ioctl that executes the same INQUIRY code used
* in scsi_probe_lun, let user level programs doing INQUIRY
* scanning run at their own risk, or supply a user level program
* that can correctly scan.
*/
/*
* Copy at least 36 bytes of INQUIRY data, so that we don't
* dereference unallocated memory when accessing the Vendor,
* Product, and Revision strings. Badly behaved devices may set
* the INQUIRY Additional Length byte to a small value, indicating
* these strings are invalid, but often they contain plausible data
* nonetheless. It doesn't matter if the device sent < 36 bytes
* total, since scsi_probe_lun() initializes inq_result with 0s.
*/
sdev->inquiry = kmemdup(inq_result,
max_t(size_t, sdev->inquiry_len, 36),
GFP_KERNEL);
if (sdev->inquiry == NULL)
return SCSI_SCAN_NO_RESPONSE;
sdev->vendor = (char *) (sdev->inquiry + 8);
sdev->model = (char *) (sdev->inquiry + 16);
sdev->rev = (char *) (sdev->inquiry + 32);
if (strncmp(sdev->vendor, "ATA ", 8) == 0) {
/*
* sata emulation layer device. This is a hack to work around
* the SATL power management specifications which state that
* when the SATL detects the device has gone into standby
* mode, it shall respond with NOT READY.
*/
sdev->allow_restart = 1;
}
if (*bflags & BLIST_ISROM) {
sdev->type = TYPE_ROM;
sdev->removable = 1;
} else {
sdev->type = (inq_result[0] & 0x1f);
sdev->removable = (inq_result[1] & 0x80) >> 7;
/*
* some devices may respond with wrong type for
* well-known logical units. Force well-known type
* to enumerate them correctly.
*/
if (scsi_is_wlun(sdev->lun) && sdev->type != TYPE_WLUN) {
sdev_printk(KERN_WARNING, sdev,
"%s: correcting incorrect peripheral device type 0x%x for W-LUN 0x%16xhN\n",
__func__, sdev->type, (unsigned int)sdev->lun);
sdev->type = TYPE_WLUN;
}
}
if (sdev->type == TYPE_RBC || sdev->type == TYPE_ROM) {
/* RBC and MMC devices can return SCSI-3 compliance and yet
* still not support REPORT LUNS, so make them act as
* BLIST_NOREPORTLUN unless BLIST_REPORTLUN2 is
* specifically set */
if ((*bflags & BLIST_REPORTLUN2) == 0)
*bflags |= BLIST_NOREPORTLUN;
}
/*
* For a peripheral qualifier (PQ) value of 1 (001b), the SCSI
* spec says: The device server is capable of supporting the
* specified peripheral device type on this logical unit. However,
* the physical device is not currently connected to this logical
* unit.
*
* The above is vague, as it implies that we could treat 001 and
* 011 the same. Stay compatible with previous code, and create a
* scsi_device for a PQ of 1
*
* Don't set the device offline here; rather let the upper
* level drivers eval the PQ to decide whether they should
* attach. So remove ((inq_result[0] >> 5) & 7) == 1 check.
*/
sdev->inq_periph_qual = (inq_result[0] >> 5) & 7;
sdev->lockable = sdev->removable;
sdev->soft_reset = (inq_result[7] & 1) && ((inq_result[3] & 7) == 2);
if (sdev->scsi_level >= SCSI_3 ||
(sdev->inquiry_len > 56 && inq_result[56] & 0x04))
sdev->ppr = 1;
if (inq_result[7] & 0x60)
sdev->wdtr = 1;
if (inq_result[7] & 0x10)
sdev->sdtr = 1;
sdev_printk(KERN_NOTICE, sdev, "%s %.8s %.16s %.4s PQ: %d "
"ANSI: %d%s\n", scsi_device_type(sdev->type),
sdev->vendor, sdev->model, sdev->rev,
sdev->inq_periph_qual, inq_result[2] & 0x07,
(inq_result[3] & 0x0f) == 1 ? " CCS" : "");
if ((sdev->scsi_level >= SCSI_2) && (inq_result[7] & 2) &&
!(*bflags & BLIST_NOTQ)) {
sdev->tagged_supported = 1;
sdev->simple_tags = 1;
}
/*
* Some devices (Texel CD ROM drives) have handshaking problems
* when used with the Seagate controllers. borken is initialized
* to 1, and then set it to 0 here.
*/
if ((*bflags & BLIST_BORKEN) == 0)
sdev->borken = 0;
if (*bflags & BLIST_NO_ULD_ATTACH)
sdev->no_uld_attach = 1;
/*
* Apparently some really broken devices (contrary to the SCSI
* standards) need to be selected without asserting ATN
*/
if (*bflags & BLIST_SELECT_NO_ATN)
sdev->select_no_atn = 1;
/*
* Maximum 512 sector transfer length
* broken RA4x00 Compaq Disk Array
*/
if (*bflags & BLIST_MAX_512)
blk_queue_max_hw_sectors(sdev->request_queue, 512);
/*
* Max 1024 sector transfer length for targets that report incorrect
* max/optimal lengths and relied on the old block layer safe default
*/
else if (*bflags & BLIST_MAX_1024)
blk_queue_max_hw_sectors(sdev->request_queue, 1024);
/*
* Some devices may not want to have a start command automatically
* issued when a device is added.
*/
if (*bflags & BLIST_NOSTARTONADD)
sdev->no_start_on_add = 1;
if (*bflags & BLIST_SINGLELUN)
scsi_target(sdev)->single_lun = 1;
sdev->use_10_for_rw = 1;
/* some devices don't like REPORT SUPPORTED OPERATION CODES
* and will simply timeout causing sd_mod init to take a very
* very long time */
if (*bflags & BLIST_NO_RSOC)
sdev->no_report_opcodes = 1;
/* set the device running here so that slave configure
* may do I/O */
mutex_lock(&sdev->state_mutex);
ret = scsi_device_set_state(sdev, SDEV_RUNNING);
if (ret)
ret = scsi_device_set_state(sdev, SDEV_BLOCK);
mutex_unlock(&sdev->state_mutex);
if (ret) {
sdev_printk(KERN_ERR, sdev,
"in wrong state %s to complete scan\n",
scsi_device_state_name(sdev->sdev_state));
return SCSI_SCAN_NO_RESPONSE;
}
if (*bflags & BLIST_NOT_LOCKABLE)
sdev->lockable = 0;
if (*bflags & BLIST_RETRY_HWERROR)
sdev->retry_hwerror = 1;
if (*bflags & BLIST_NO_DIF)
sdev->no_dif = 1;
if (*bflags & BLIST_UNMAP_LIMIT_WS)
sdev->unmap_limit_for_ws = 1;
if (*bflags & BLIST_IGN_MEDIA_CHANGE)
sdev->ignore_media_change = 1;
sdev->eh_timeout = SCSI_DEFAULT_EH_TIMEOUT;
if (*bflags & BLIST_TRY_VPD_PAGES)
sdev->try_vpd_pages = 1;
else if (*bflags & BLIST_SKIP_VPD_PAGES)
sdev->skip_vpd_pages = 1;
if (*bflags & BLIST_NO_VPD_SIZE)
sdev->no_vpd_size = 1;
transport_configure_device(&sdev->sdev_gendev);
if (sdev->host->hostt->slave_configure) {
ret = sdev->host->hostt->slave_configure(sdev);
if (ret) {
/*
* if LLDD reports slave not present, don't clutter
* console with alloc failure messages
*/
if (ret != -ENXIO) {
sdev_printk(KERN_ERR, sdev,
"failed to configure device\n");
}
return SCSI_SCAN_NO_RESPONSE;
}
/*
* The queue_depth is often changed in ->slave_configure.
* Set up budget map again since memory consumption of
* the map depends on actual queue depth.
*/
scsi_realloc_sdev_budget_map(sdev, sdev->queue_depth);
}
if (sdev->scsi_level >= SCSI_3)
scsi_attach_vpd(sdev);
scsi_cdl_check(sdev);
sdev->max_queue_depth = sdev->queue_depth;
WARN_ON_ONCE(sdev->max_queue_depth > sdev->budget_map.depth);
sdev->sdev_bflags = *bflags;
/*
* Ok, the device is now all set up, we can
* register it and tell the rest of the kernel
* about it.
*/
if (!async && scsi_sysfs_add_sdev(sdev) != 0)
return SCSI_SCAN_NO_RESPONSE;
return SCSI_SCAN_LUN_PRESENT;
}
#ifdef CONFIG_SCSI_LOGGING
/**
* scsi_inq_str - print INQUIRY data from min to max index, strip trailing whitespace
* @buf: Output buffer with at least end-first+1 bytes of space
* @inq: Inquiry buffer (input)
* @first: Offset of string into inq
* @end: Index after last character in inq
*/
static unsigned char *scsi_inq_str(unsigned char *buf, unsigned char *inq,
unsigned first, unsigned end)
{
unsigned term = 0, idx;
for (idx = 0; idx + first < end && idx + first < inq[4] + 5; idx++) {
if (inq[idx+first] > ' ') {
buf[idx] = inq[idx+first];
term = idx+1;
} else {
buf[idx] = ' ';
}
}
buf[term] = 0;
return buf;
}
#endif
/**
* scsi_probe_and_add_lun - probe a LUN, if a LUN is found add it
* @starget: pointer to target device structure
* @lun: LUN of target device
* @bflagsp: store bflags here if not NULL
* @sdevp: probe the LUN corresponding to this scsi_device
* @rescan: if not equal to SCSI_SCAN_INITIAL skip some code only
* needed on first scan
* @hostdata: passed to scsi_alloc_sdev()
*
* Description:
* Call scsi_probe_lun, if a LUN with an attached device is found,
* allocate and set it up by calling scsi_add_lun.
*
* Return:
*
* - SCSI_SCAN_NO_RESPONSE: could not allocate or setup a scsi_device
* - SCSI_SCAN_TARGET_PRESENT: target responded, but no device is
* attached at the LUN
* - SCSI_SCAN_LUN_PRESENT: a new scsi_device was allocated and initialized
**/
static int scsi_probe_and_add_lun(struct scsi_target *starget,
u64 lun, blist_flags_t *bflagsp,
struct scsi_device **sdevp,
enum scsi_scan_mode rescan,
void *hostdata)
{
struct scsi_device *sdev;
unsigned char *result;
blist_flags_t bflags;
int res = SCSI_SCAN_NO_RESPONSE, result_len = 256;
struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
/*
* The rescan flag is used as an optimization, the first scan of a
* host adapter calls into here with rescan == 0.
*/
sdev = scsi_device_lookup_by_target(starget, lun);
if (sdev) {
if (rescan != SCSI_SCAN_INITIAL || !scsi_device_created(sdev)) {
SCSI_LOG_SCAN_BUS(3, sdev_printk(KERN_INFO, sdev,
"scsi scan: device exists on %s\n",
dev_name(&sdev->sdev_gendev)));
if (sdevp)
*sdevp = sdev;
else
scsi_device_put(sdev);
if (bflagsp)
*bflagsp = scsi_get_device_flags(sdev,
sdev->vendor,
sdev->model);
return SCSI_SCAN_LUN_PRESENT;
}
scsi_device_put(sdev);
} else
sdev = scsi_alloc_sdev(starget, lun, hostdata);
if (!sdev)
goto out;
result = kmalloc(result_len, GFP_KERNEL);
if (!result)
goto out_free_sdev;
if (scsi_probe_lun(sdev, result, result_len, &bflags))
goto out_free_result;
if (bflagsp)
*bflagsp = bflags;
/*
* result contains valid SCSI INQUIRY data.
*/
if ((result[0] >> 5) == 3) {
/*
* For a Peripheral qualifier 3 (011b), the SCSI
* spec says: The device server is not capable of
* supporting a physical device on this logical
* unit.
*
* For disks, this implies that there is no
* logical disk configured at sdev->lun, but there
* is a target id responding.
*/
SCSI_LOG_SCAN_BUS(2, sdev_printk(KERN_INFO, sdev, "scsi scan:"
" peripheral qualifier of 3, device not"
" added\n"))
if (lun == 0) {
SCSI_LOG_SCAN_BUS(1, {
unsigned char vend[9];
unsigned char mod[17];
sdev_printk(KERN_INFO, sdev,
"scsi scan: consider passing scsi_mod."
"dev_flags=%s:%s:0x240 or 0x1000240\n",
scsi_inq_str(vend, result, 8, 16),
scsi_inq_str(mod, result, 16, 32));
});
}
res = SCSI_SCAN_TARGET_PRESENT;
goto out_free_result;
}
/*
* Some targets may set slight variations of PQ and PDT to signal
* that no LUN is present, so don't add sdev in these cases.
* Two specific examples are:
* 1) NetApp targets: return PQ=1, PDT=0x1f
* 2) USB UFI: returns PDT=0x1f, with the PQ bits being "reserved"
* in the UFI 1.0 spec (we cannot rely on reserved bits).
*
* References:
* 1) SCSI SPC-3, pp. 145-146
* PQ=1: "A peripheral device having the specified peripheral
* device type is not connected to this logical unit. However, the
* device server is capable of supporting the specified peripheral
* device type on this logical unit."
* PDT=0x1f: "Unknown or no device type"
* 2) USB UFI 1.0, p. 20
* PDT=00h Direct-access device (floppy)
* PDT=1Fh none (no FDD connected to the requested logical unit)
*/
if (((result[0] >> 5) == 1 || starget->pdt_1f_for_no_lun) &&
(result[0] & 0x1f) == 0x1f &&
!scsi_is_wlun(lun)) {
SCSI_LOG_SCAN_BUS(3, sdev_printk(KERN_INFO, sdev,
"scsi scan: peripheral device type"
" of 31, no device added\n"));
res = SCSI_SCAN_TARGET_PRESENT;
goto out_free_result;
}
res = scsi_add_lun(sdev, result, &bflags, shost->async_scan);
if (res == SCSI_SCAN_LUN_PRESENT) {
if (bflags & BLIST_KEY) {
sdev->lockable = 0;
scsi_unlock_floptical(sdev, result);
}
}
out_free_result:
kfree(result);
out_free_sdev:
if (res == SCSI_SCAN_LUN_PRESENT) {
if (sdevp) {
if (scsi_device_get(sdev) == 0) {
*sdevp = sdev;
} else {
__scsi_remove_device(sdev);
res = SCSI_SCAN_NO_RESPONSE;
}
}
} else
__scsi_remove_device(sdev);
out:
return res;
}
/**
* scsi_sequential_lun_scan - sequentially scan a SCSI target
* @starget: pointer to target structure to scan
* @bflags: black/white list flag for LUN 0
* @scsi_level: Which version of the standard does this device adhere to
* @rescan: passed to scsi_probe_add_lun()
*
* Description:
* Generally, scan from LUN 1 (LUN 0 is assumed to already have been
* scanned) to some maximum lun until a LUN is found with no device
* attached. Use the bflags to figure out any oddities.
*
* Modifies sdevscan->lun.
**/
static void scsi_sequential_lun_scan(struct scsi_target *starget,
blist_flags_t bflags, int scsi_level,
enum scsi_scan_mode rescan)
{
uint max_dev_lun;
u64 sparse_lun, lun;
struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
SCSI_LOG_SCAN_BUS(3, starget_printk(KERN_INFO, starget,
"scsi scan: Sequential scan\n"));
max_dev_lun = min(max_scsi_luns, shost->max_lun);
/*
* If this device is known to support sparse multiple units,
* override the other settings, and scan all of them. Normally,
* SCSI-3 devices should be scanned via the REPORT LUNS.
*/
if (bflags & BLIST_SPARSELUN) {
max_dev_lun = shost->max_lun;
sparse_lun = 1;
} else
sparse_lun = 0;
/*
* If less than SCSI_1_CCS, and no special lun scanning, stop
* scanning; this matches 2.4 behaviour, but could just be a bug
* (to continue scanning a SCSI_1_CCS device).
*
* This test is broken. We might not have any device on lun0 for
* a sparselun device, and if that's the case then how would we
* know the real scsi_level, eh? It might make sense to just not
* scan any SCSI_1 device for non-0 luns, but that check would best
* go into scsi_alloc_sdev() and just have it return null when asked
* to alloc an sdev for lun > 0 on an already found SCSI_1 device.
*
if ((sdevscan->scsi_level < SCSI_1_CCS) &&
((bflags & (BLIST_FORCELUN | BLIST_SPARSELUN | BLIST_MAX5LUN))
== 0))
return;
*/
/*
* If this device is known to support multiple units, override
* the other settings, and scan all of them.
*/
if (bflags & BLIST_FORCELUN)
max_dev_lun = shost->max_lun;
/*
* REGAL CDC-4X: avoid hang after LUN 4
*/
if (bflags & BLIST_MAX5LUN)
max_dev_lun = min(5U, max_dev_lun);
/*
* Do not scan SCSI-2 or lower device past LUN 7, unless
* BLIST_LARGELUN.
*/
if (scsi_level < SCSI_3 && !(bflags & BLIST_LARGELUN))
max_dev_lun = min(8U, max_dev_lun);
else
max_dev_lun = min(256U, max_dev_lun);
/*
* We have already scanned LUN 0, so start at LUN 1. Keep scanning
* until we reach the max, or no LUN is found and we are not
* sparse_lun.
*/
for (lun = 1; lun < max_dev_lun; ++lun)
if ((scsi_probe_and_add_lun(starget, lun, NULL, NULL, rescan,
NULL) != SCSI_SCAN_LUN_PRESENT) &&
!sparse_lun)
return;
}
/**
* scsi_report_lun_scan - Scan using SCSI REPORT LUN results
* @starget: which target
* @bflags: Zero or a mix of BLIST_NOLUN, BLIST_REPORTLUN2, or BLIST_NOREPORTLUN
* @rescan: nonzero if we can skip code only needed on first scan
*
* Description:
* Fast scanning for modern (SCSI-3) devices by sending a REPORT LUN command.
* Scan the resulting list of LUNs by calling scsi_probe_and_add_lun.
*
* If BLINK_REPORTLUN2 is set, scan a target that supports more than 8
* LUNs even if it's older than SCSI-3.
* If BLIST_NOREPORTLUN is set, return 1 always.
* If BLIST_NOLUN is set, return 0 always.
* If starget->no_report_luns is set, return 1 always.
*
* Return:
* 0: scan completed (or no memory, so further scanning is futile)
* 1: could not scan with REPORT LUN
**/
static int scsi_report_lun_scan(struct scsi_target *starget, blist_flags_t bflags,
enum scsi_scan_mode rescan)
{
unsigned char scsi_cmd[MAX_COMMAND_SIZE];
unsigned int length;
u64 lun;
unsigned int num_luns;
int result;
struct scsi_lun *lunp, *lun_data;
struct scsi_device *sdev;
struct Scsi_Host *shost = dev_to_shost(&starget->dev);
struct scsi_failure failure_defs[] = {
{
.sense = UNIT_ATTENTION,
.asc = SCMD_FAILURE_ASC_ANY,
.ascq = SCMD_FAILURE_ASCQ_ANY,
.result = SAM_STAT_CHECK_CONDITION,
},
/* Fail all CCs except the UA above */
{
.sense = SCMD_FAILURE_SENSE_ANY,
.result = SAM_STAT_CHECK_CONDITION,
},
/* Retry any other errors not listed above */
{
.result = SCMD_FAILURE_RESULT_ANY,
},
{}
};
struct scsi_failures failures = {
.total_allowed = 3,
.failure_definitions = failure_defs,
};
const struct scsi_exec_args exec_args = {
.failures = &failures,
};
int ret = 0;
/*
* Only support SCSI-3 and up devices if BLIST_NOREPORTLUN is not set.
* Also allow SCSI-2 if BLIST_REPORTLUN2 is set and host adapter does
* support more than 8 LUNs.
* Don't attempt if the target doesn't support REPORT LUNS.
*/
if (bflags & BLIST_NOREPORTLUN)
return 1;
if (starget->scsi_level < SCSI_2 &&
starget->scsi_level != SCSI_UNKNOWN)
return 1;
if (starget->scsi_level < SCSI_3 &&
(!(bflags & BLIST_REPORTLUN2) || shost->max_lun <= 8))
return 1;
if (bflags & BLIST_NOLUN)
return 0;
if (starget->no_report_luns)
return 1;
if (!(sdev = scsi_device_lookup_by_target(starget, 0))) {
sdev = scsi_alloc_sdev(starget, 0, NULL);
if (!sdev)
return 0;
if (scsi_device_get(sdev)) {
__scsi_remove_device(sdev);
return 0;
}
}
/*
* Allocate enough to hold the header (the same size as one scsi_lun)
* plus the number of luns we are requesting. 511 was the default
* value of the now removed max_report_luns parameter.
*/
length = (511 + 1) * sizeof(struct scsi_lun);
retry:
lun_data = kmalloc(length, GFP_KERNEL);
if (!lun_data) {
printk(ALLOC_FAILURE_MSG, __func__);
goto out;
}
scsi_cmd[0] = REPORT_LUNS;
/*
* bytes 1 - 5: reserved, set to zero.
*/
memset(&scsi_cmd[1], 0, 5);
/*
* bytes 6 - 9: length of the command.
*/
put_unaligned_be32(length, &scsi_cmd[6]);
scsi_cmd[10] = 0; /* reserved */
scsi_cmd[11] = 0; /* control */
/*
* We can get a UNIT ATTENTION, for example a power on/reset, so
* retry a few times (like sd.c does for TEST UNIT READY).
* Experience shows some combinations of adapter/devices get at
* least two power on/resets.
*
* Illegal requests (for devices that do not support REPORT LUNS)
* should come through as a check condition, and will not generate
* a retry.
*/
scsi_failures_reset_retries(&failures);
SCSI_LOG_SCAN_BUS(3, sdev_printk (KERN_INFO, sdev,
"scsi scan: Sending REPORT LUNS\n"));
result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN, lun_data,
length, SCSI_REPORT_LUNS_TIMEOUT, 3,
&exec_args);
SCSI_LOG_SCAN_BUS(3, sdev_printk (KERN_INFO, sdev,
"scsi scan: REPORT LUNS %s result 0x%x\n",
result ? "failed" : "successful", result));
if (result) {
/*
* The device probably does not support a REPORT LUN command
*/
ret = 1;
goto out_err;
}
/*
* Get the length from the first four bytes of lun_data.
*/
if (get_unaligned_be32(lun_data->scsi_lun) +
sizeof(struct scsi_lun) > length) {
length = get_unaligned_be32(lun_data->scsi_lun) +
sizeof(struct scsi_lun);
kfree(lun_data);
goto retry;
}
length = get_unaligned_be32(lun_data->scsi_lun);
num_luns = (length / sizeof(struct scsi_lun));
SCSI_LOG_SCAN_BUS(3, sdev_printk (KERN_INFO, sdev,
"scsi scan: REPORT LUN scan\n"));
/*
* Scan the luns in lun_data. The entry at offset 0 is really
* the header, so start at 1 and go up to and including num_luns.
*/
for (lunp = &lun_data[1]; lunp <= &lun_data[num_luns]; lunp++) {
lun = scsilun_to_int(lunp);
if (lun > sdev->host->max_lun) {
sdev_printk(KERN_WARNING, sdev,
"lun%llu has a LUN larger than"
" allowed by the host adapter\n", lun);
} else {
int res;
res = scsi_probe_and_add_lun(starget,
lun, NULL, NULL, rescan, NULL);
if (res == SCSI_SCAN_NO_RESPONSE) {
/*
* Got some results, but now none, abort.
*/
sdev_printk(KERN_ERR, sdev,
"Unexpected response"
" from lun %llu while scanning, scan"
" aborted\n", (unsigned long long)lun);
break;
}
}
}
out_err:
kfree(lun_data);
out:
if (scsi_device_created(sdev))
/*
* the sdev we used didn't appear in the report luns scan
*/
__scsi_remove_device(sdev);
scsi_device_put(sdev);
return ret;
}
struct scsi_device *__scsi_add_device(struct Scsi_Host *shost, uint channel,
uint id, u64 lun, void *hostdata)
{
struct scsi_device *sdev = ERR_PTR(-ENODEV);
struct device *parent = &shost->shost_gendev;
struct scsi_target *starget;
if (strncmp(scsi_scan_type, "none", 4) == 0)
return ERR_PTR(-ENODEV);
starget = scsi_alloc_target(parent, channel, id);
if (!starget)
return ERR_PTR(-ENOMEM);
scsi_autopm_get_target(starget);
mutex_lock(&shost->scan_mutex);
if (!shost->async_scan)
scsi_complete_async_scans();
if (scsi_host_scan_allowed(shost) && scsi_autopm_get_host(shost) == 0) {
scsi_probe_and_add_lun(starget, lun, NULL, &sdev,
SCSI_SCAN_RESCAN, hostdata);
scsi_autopm_put_host(shost);
}
mutex_unlock(&shost->scan_mutex);
scsi_autopm_put_target(starget);
/*
* paired with scsi_alloc_target(). Target will be destroyed unless
* scsi_probe_and_add_lun made an underlying device visible
*/
scsi_target_reap(starget);
put_device(&starget->dev);
return sdev;
}
EXPORT_SYMBOL(__scsi_add_device);
int scsi_add_device(struct Scsi_Host *host, uint channel,
uint target, u64 lun)
{
struct scsi_device *sdev =
__scsi_add_device(host, channel, target, lun, NULL);
if (IS_ERR(sdev))
return PTR_ERR(sdev);
scsi_device_put(sdev);
return 0;
}
EXPORT_SYMBOL(scsi_add_device);
int scsi_resume_device(struct scsi_device *sdev)
{
struct device *dev = &sdev->sdev_gendev;
int ret = 0;
device_lock(dev);
/*
* Bail out if the device or its queue are not running. Otherwise,
* the rescan may block waiting for commands to be executed, with us
* holding the device lock. This can result in a potential deadlock
* in the power management core code when system resume is on-going.
*/
if (sdev->sdev_state != SDEV_RUNNING ||
blk_queue_pm_only(sdev->request_queue)) {
ret = -EWOULDBLOCK;
goto unlock;
}
if (dev->driver && try_module_get(dev->driver->owner)) {
struct scsi_driver *drv = to_scsi_driver(dev->driver);
if (drv->resume)
ret = drv->resume(dev);
module_put(dev->driver->owner);
}
unlock:
device_unlock(dev);
return ret;
}
EXPORT_SYMBOL(scsi_resume_device);
int scsi_rescan_device(struct scsi_device *sdev)
{
struct device *dev = &sdev->sdev_gendev;
int ret = 0;
device_lock(dev);
/*
* Bail out if the device or its queue are not running. Otherwise,
* the rescan may block waiting for commands to be executed, with us
* holding the device lock. This can result in a potential deadlock
* in the power management core code when system resume is on-going.
*/
if (sdev->sdev_state != SDEV_RUNNING ||
blk_queue_pm_only(sdev->request_queue)) {
ret = -EWOULDBLOCK;
goto unlock;
}
scsi_attach_vpd(sdev);
scsi_cdl_check(sdev);
if (sdev->handler && sdev->handler->rescan)
sdev->handler->rescan(sdev);
if (dev->driver && try_module_get(dev->driver->owner)) {
struct scsi_driver *drv = to_scsi_driver(dev->driver);
if (drv->rescan)
drv->rescan(dev);
module_put(dev->driver->owner);
}
unlock:
device_unlock(dev);
return ret;
}
EXPORT_SYMBOL(scsi_rescan_device);
static void __scsi_scan_target(struct device *parent, unsigned int channel,
unsigned int id, u64 lun, enum scsi_scan_mode rescan)
{
struct Scsi_Host *shost = dev_to_shost(parent);
blist_flags_t bflags = 0;
int res;
struct scsi_target *starget;
if (shost->this_id == id)
/*
* Don't scan the host adapter
*/
return;
starget = scsi_alloc_target(parent, channel, id);
if (!starget)
return;
scsi_autopm_get_target(starget);
if (lun != SCAN_WILD_CARD) {
/*
* Scan for a specific host/chan/id/lun.
*/
scsi_probe_and_add_lun(starget, lun, NULL, NULL, rescan, NULL);
goto out_reap;
}
/*
* Scan LUN 0, if there is some response, scan further. Ideally, we
* would not configure LUN 0 until all LUNs are scanned.
*/
res = scsi_probe_and_add_lun(starget, 0, &bflags, NULL, rescan, NULL);
if (res == SCSI_SCAN_LUN_PRESENT || res == SCSI_SCAN_TARGET_PRESENT) {
if (scsi_report_lun_scan(starget, bflags, rescan) != 0)
/*
* The REPORT LUN did not scan the target,
* do a sequential scan.
*/
scsi_sequential_lun_scan(starget, bflags,
starget->scsi_level, rescan);
}
out_reap:
scsi_autopm_put_target(starget);
/*
* paired with scsi_alloc_target(): determine if the target has
* any children at all and if not, nuke it
*/
scsi_target_reap(starget);
put_device(&starget->dev);
}
/**
* scsi_scan_target - scan a target id, possibly including all LUNs on the target.
* @parent: host to scan
* @channel: channel to scan
* @id: target id to scan
* @lun: Specific LUN to scan or SCAN_WILD_CARD
* @rescan: passed to LUN scanning routines; SCSI_SCAN_INITIAL for
* no rescan, SCSI_SCAN_RESCAN to rescan existing LUNs,
* and SCSI_SCAN_MANUAL to force scanning even if
* 'scan=manual' is set.
*
* Description:
* Scan the target id on @parent, @channel, and @id. Scan at least LUN 0,
* and possibly all LUNs on the target id.
*
* First try a REPORT LUN scan, if that does not scan the target, do a
* sequential scan of LUNs on the target id.
**/
void scsi_scan_target(struct device *parent, unsigned int channel,
unsigned int id, u64 lun, enum scsi_scan_mode rescan)
{
struct Scsi_Host *shost = dev_to_shost(parent);
if (strncmp(scsi_scan_type, "none", 4) == 0)
return;
if (rescan != SCSI_SCAN_MANUAL &&
strncmp(scsi_scan_type, "manual", 6) == 0)
return;
mutex_lock(&shost->scan_mutex);
if (!shost->async_scan)
scsi_complete_async_scans();
if (scsi_host_scan_allowed(shost) && scsi_autopm_get_host(shost) == 0) {
__scsi_scan_target(parent, channel, id, lun, rescan);
scsi_autopm_put_host(shost);
}
mutex_unlock(&shost->scan_mutex);
}
EXPORT_SYMBOL(scsi_scan_target);
static void scsi_scan_channel(struct Scsi_Host *shost, unsigned int channel,
unsigned int id, u64 lun,
enum scsi_scan_mode rescan)
{
uint order_id;
if (id == SCAN_WILD_CARD)
for (id = 0; id < shost->max_id; ++id) {
/*
* XXX adapter drivers when possible (FCP, iSCSI)
* could modify max_id to match the current max,
* not the absolute max.
*
* XXX add a shost id iterator, so for example,
* the FC ID can be the same as a target id
* without a huge overhead of sparse id's.
*/
if (shost->reverse_ordering)
/*
* Scan from high to low id.
*/
order_id = shost->max_id - id - 1;
else
order_id = id;
__scsi_scan_target(&shost->shost_gendev, channel,
order_id, lun, rescan);
}
else
__scsi_scan_target(&shost->shost_gendev, channel,
id, lun, rescan);
}
int scsi_scan_host_selected(struct Scsi_Host *shost, unsigned int channel,
unsigned int id, u64 lun,
enum scsi_scan_mode rescan)
{
SCSI_LOG_SCAN_BUS(3, shost_printk (KERN_INFO, shost,
"%s: <%u:%u:%llu>\n",
__func__, channel, id, lun));
if (((channel != SCAN_WILD_CARD) && (channel > shost->max_channel)) ||
((id != SCAN_WILD_CARD) && (id >= shost->max_id)) ||
((lun != SCAN_WILD_CARD) && (lun >= shost->max_lun)))
return -EINVAL;
mutex_lock(&shost->scan_mutex);
if (!shost->async_scan)
scsi_complete_async_scans();
if (scsi_host_scan_allowed(shost) && scsi_autopm_get_host(shost) == 0) {
if (channel == SCAN_WILD_CARD)
for (channel = 0; channel <= shost->max_channel;
channel++)
scsi_scan_channel(shost, channel, id, lun,
rescan);
else
scsi_scan_channel(shost, channel, id, lun, rescan);
scsi_autopm_put_host(shost);
}
mutex_unlock(&shost->scan_mutex);
return 0;
}
static void scsi_sysfs_add_devices(struct Scsi_Host *shost)
{
struct scsi_device *sdev;
shost_for_each_device(sdev, shost) {
/* target removed before the device could be added */
if (sdev->sdev_state == SDEV_DEL)
continue;
/* If device is already visible, skip adding it to sysfs */
if (sdev->is_visible)
continue;
if (!scsi_host_scan_allowed(shost) ||
scsi_sysfs_add_sdev(sdev) != 0)
__scsi_remove_device(sdev);
}
}
/**
* scsi_prep_async_scan - prepare for an async scan
* @shost: the host which will be scanned
* Returns: a cookie to be passed to scsi_finish_async_scan()
*
* Tells the midlayer this host is going to do an asynchronous scan.
* It reserves the host's position in the scanning list and ensures
* that other asynchronous scans started after this one won't affect the
* ordering of the discovered devices.
*/
static struct async_scan_data *scsi_prep_async_scan(struct Scsi_Host *shost)
{
struct async_scan_data *data = NULL;
unsigned long flags;
if (strncmp(scsi_scan_type, "sync", 4) == 0)
return NULL;
mutex_lock(&shost->scan_mutex);
if (shost->async_scan) {
shost_printk(KERN_DEBUG, shost, "%s called twice\n", __func__);
goto err;
}
data = kmalloc(sizeof(*data), GFP_KERNEL);
if (!data)
goto err;
data->shost = scsi_host_get(shost);
if (!data->shost)
goto err;
init_completion(&data->prev_finished);
spin_lock_irqsave(shost->host_lock, flags);
shost->async_scan = 1;
spin_unlock_irqrestore(shost->host_lock, flags);
mutex_unlock(&shost->scan_mutex);
spin_lock(&async_scan_lock);
if (list_empty(&scanning_hosts))
complete(&data->prev_finished);
list_add_tail(&data->list, &scanning_hosts);
spin_unlock(&async_scan_lock);
return data;
err:
mutex_unlock(&shost->scan_mutex);
kfree(data);
return NULL;
}
/**
* scsi_finish_async_scan - asynchronous scan has finished
* @data: cookie returned from earlier call to scsi_prep_async_scan()
*
* All the devices currently attached to this host have been found.
* This function announces all the devices it has found to the rest
* of the system.
*/
static void scsi_finish_async_scan(struct async_scan_data *data)
{
struct Scsi_Host *shost;
unsigned long flags;
if (!data)
return;
shost = data->shost;
mutex_lock(&shost->scan_mutex);
if (!shost->async_scan) {
shost_printk(KERN_INFO, shost, "%s called twice\n", __func__);
dump_stack();
mutex_unlock(&shost->scan_mutex);
return;
}
wait_for_completion(&data->prev_finished);
scsi_sysfs_add_devices(shost);
spin_lock_irqsave(shost->host_lock, flags);
shost->async_scan = 0;
spin_unlock_irqrestore(shost->host_lock, flags);
mutex_unlock(&shost->scan_mutex);
spin_lock(&async_scan_lock);
list_del(&data->list);
if (!list_empty(&scanning_hosts)) {
struct async_scan_data *next = list_entry(scanning_hosts.next,
struct async_scan_data, list);
complete(&next->prev_finished);
}
spin_unlock(&async_scan_lock);
scsi_autopm_put_host(shost);
scsi_host_put(shost);
kfree(data);
}
static void do_scsi_scan_host(struct Scsi_Host *shost)
{
if (shost->hostt->scan_finished) {
unsigned long start = jiffies;
if (shost->hostt->scan_start)
shost->hostt->scan_start(shost);
while (!shost->hostt->scan_finished(shost, jiffies - start))
msleep(10);
} else {
scsi_scan_host_selected(shost, SCAN_WILD_CARD, SCAN_WILD_CARD,
SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
}
}
static void do_scan_async(void *_data, async_cookie_t c)
{
struct async_scan_data *data = _data;
struct Scsi_Host *shost = data->shost;
do_scsi_scan_host(shost);
scsi_finish_async_scan(data);
}
/**
* scsi_scan_host - scan the given adapter
* @shost: adapter to scan
**/
void scsi_scan_host(struct Scsi_Host *shost)
{
struct async_scan_data *data;
if (strncmp(scsi_scan_type, "none", 4) == 0 ||
strncmp(scsi_scan_type, "manual", 6) == 0)
return;
if (scsi_autopm_get_host(shost) < 0)
return;
data = scsi_prep_async_scan(shost);
if (!data) {
do_scsi_scan_host(shost);
scsi_autopm_put_host(shost);
return;
}
/* register with the async subsystem so wait_for_device_probe()
* will flush this work
*/
async_schedule(do_scan_async, data);
/* scsi_autopm_put_host(shost) is called in scsi_finish_async_scan() */
}
EXPORT_SYMBOL(scsi_scan_host);
void scsi_forget_host(struct Scsi_Host *shost)
{
struct scsi_device *sdev;
unsigned long flags;
restart:
spin_lock_irqsave(shost->host_lock, flags); list_for_each_entry(sdev, &shost->__devices, siblings) { if (sdev->sdev_state == SDEV_DEL)
continue;
spin_unlock_irqrestore(shost->host_lock, flags);
__scsi_remove_device(sdev);
goto restart;
}
spin_unlock_irqrestore(shost->host_lock, flags);
}
// SPDX-License-Identifier: GPL-2.0
/*
* NETLINK Netlink attributes
*
* Authors: Thomas Graf <tgraf@suug.ch>
* Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/jiffies.h>
#include <linux/nospec.h>
#include <linux/skbuff.h>
#include <linux/string.h>
#include <linux/types.h>
#include <net/netlink.h>
/* For these data types, attribute length should be exactly the given
* size. However, to maintain compatibility with broken commands, if the
* attribute length does not match the expected size a warning is emitted
* to the user that the command is sending invalid data and needs to be fixed.
*/
static const u8 nla_attr_len[NLA_TYPE_MAX+1] = {
[NLA_U8] = sizeof(u8),
[NLA_U16] = sizeof(u16),
[NLA_U32] = sizeof(u32),
[NLA_U64] = sizeof(u64),
[NLA_S8] = sizeof(s8),
[NLA_S16] = sizeof(s16),
[NLA_S32] = sizeof(s32),
[NLA_S64] = sizeof(s64),
[NLA_BE16] = sizeof(__be16),
[NLA_BE32] = sizeof(__be32),
};
static const u8 nla_attr_minlen[NLA_TYPE_MAX+1] = {
[NLA_U8] = sizeof(u8),
[NLA_U16] = sizeof(u16),
[NLA_U32] = sizeof(u32),
[NLA_U64] = sizeof(u64),
[NLA_MSECS] = sizeof(u64),
[NLA_NESTED] = NLA_HDRLEN,
[NLA_S8] = sizeof(s8),
[NLA_S16] = sizeof(s16),
[NLA_S32] = sizeof(s32),
[NLA_S64] = sizeof(s64),
[NLA_BE16] = sizeof(__be16),
[NLA_BE32] = sizeof(__be32),
};
/*
* Nested policies might refer back to the original
* policy in some cases, and userspace could try to
* abuse that and recurse by nesting in the right
* ways. Limit recursion to avoid this problem.
*/
#define MAX_POLICY_RECURSION_DEPTH 10
static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype,
const struct nla_policy *policy,
unsigned int validate,
struct netlink_ext_ack *extack,
struct nlattr **tb, unsigned int depth);
static int validate_nla_bitfield32(const struct nlattr *nla,
const u32 valid_flags_mask)
{
const struct nla_bitfield32 *bf = nla_data(nla);
if (!valid_flags_mask)
return -EINVAL;
/*disallow invalid bit selector */
if (bf->selector & ~valid_flags_mask)
return -EINVAL;
/*disallow invalid bit values */
if (bf->value & ~valid_flags_mask)
return -EINVAL;
/*disallow valid bit values that are not selected*/
if (bf->value & ~bf->selector)
return -EINVAL;
return 0;
}
static int nla_validate_array(const struct nlattr *head, int len, int maxtype,
const struct nla_policy *policy,
struct netlink_ext_ack *extack,
unsigned int validate, unsigned int depth)
{
const struct nlattr *entry;
int rem;
nla_for_each_attr(entry, head, len, rem) {
int ret;
if (nla_len(entry) == 0)
continue;
if (nla_len(entry) < NLA_HDRLEN) {
NL_SET_ERR_MSG_ATTR_POL(extack, entry, policy,
"Array element too short");
return -ERANGE;
}
ret = __nla_validate_parse(nla_data(entry), nla_len(entry),
maxtype, policy, validate, extack,
NULL, depth + 1);
if (ret < 0)
return ret;
}
return 0;
}
void nla_get_range_unsigned(const struct nla_policy *pt,
struct netlink_range_validation *range)
{
WARN_ON_ONCE(pt->validation_type != NLA_VALIDATE_RANGE_PTR &&
(pt->min < 0 || pt->max < 0));
range->min = 0;
switch (pt->type) {
case NLA_U8:
range->max = U8_MAX;
break;
case NLA_U16:
case NLA_BE16:
case NLA_BINARY:
range->max = U16_MAX;
break;
case NLA_U32:
case NLA_BE32:
range->max = U32_MAX;
break;
case NLA_U64:
case NLA_UINT:
case NLA_MSECS:
range->max = U64_MAX;
break;
default:
WARN_ON_ONCE(1);
return;
}
switch (pt->validation_type) {
case NLA_VALIDATE_RANGE:
case NLA_VALIDATE_RANGE_WARN_TOO_LONG:
range->min = pt->min;
range->max = pt->max;
break;
case NLA_VALIDATE_RANGE_PTR:
*range = *pt->range;
break;
case NLA_VALIDATE_MIN:
range->min = pt->min;
break;
case NLA_VALIDATE_MAX:
range->max = pt->max;
break;
default:
break;
}
}
static int nla_validate_range_unsigned(const struct nla_policy *pt,
const struct nlattr *nla,
struct netlink_ext_ack *extack,
unsigned int validate)
{
struct netlink_range_validation range;
u64 value;
switch (pt->type) {
case NLA_U8:
value = nla_get_u8(nla);
break;
case NLA_U16:
value = nla_get_u16(nla);
break;
case NLA_U32:
value = nla_get_u32(nla);
break;
case NLA_U64:
value = nla_get_u64(nla);
break;
case NLA_UINT:
value = nla_get_uint(nla);
break;
case NLA_MSECS:
value = nla_get_u64(nla);
break;
case NLA_BINARY:
value = nla_len(nla);
break;
case NLA_BE16:
value = ntohs(nla_get_be16(nla));
break;
case NLA_BE32:
value = ntohl(nla_get_be32(nla));
break;
default:
return -EINVAL;
}
nla_get_range_unsigned(pt, &range);
if (pt->validation_type == NLA_VALIDATE_RANGE_WARN_TOO_LONG &&
pt->type == NLA_BINARY && value > range.max) {
pr_warn_ratelimited("netlink: '%s': attribute type %d has an invalid length.\n",
current->comm, pt->type);
if (validate & NL_VALIDATE_STRICT_ATTRS) {
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"invalid attribute length");
return -EINVAL;
}
/* this assumes min <= max (don't validate against min) */
return 0;
}
if (value < range.min || value > range.max) {
bool binary = pt->type == NLA_BINARY;
if (binary)
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"binary attribute size out of range");
else
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"integer out of range");
return -ERANGE;
}
return 0;
}
void nla_get_range_signed(const struct nla_policy *pt,
struct netlink_range_validation_signed *range)
{
switch (pt->type) {
case NLA_S8:
range->min = S8_MIN;
range->max = S8_MAX;
break;
case NLA_S16:
range->min = S16_MIN;
range->max = S16_MAX;
break;
case NLA_S32:
range->min = S32_MIN;
range->max = S32_MAX;
break;
case NLA_S64:
case NLA_SINT:
range->min = S64_MIN;
range->max = S64_MAX;
break;
default:
WARN_ON_ONCE(1);
return;
}
switch (pt->validation_type) {
case NLA_VALIDATE_RANGE:
range->min = pt->min;
range->max = pt->max;
break;
case NLA_VALIDATE_RANGE_PTR:
*range = *pt->range_signed;
break;
case NLA_VALIDATE_MIN:
range->min = pt->min;
break;
case NLA_VALIDATE_MAX:
range->max = pt->max;
break;
default:
break;
}
}
static int nla_validate_int_range_signed(const struct nla_policy *pt,
const struct nlattr *nla,
struct netlink_ext_ack *extack)
{
struct netlink_range_validation_signed range;
s64 value;
switch (pt->type) {
case NLA_S8:
value = nla_get_s8(nla);
break;
case NLA_S16:
value = nla_get_s16(nla);
break;
case NLA_S32:
value = nla_get_s32(nla);
break;
case NLA_S64:
value = nla_get_s64(nla);
break;
case NLA_SINT:
value = nla_get_sint(nla);
break;
default:
return -EINVAL;
}
nla_get_range_signed(pt, &range);
if (value < range.min || value > range.max) {
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"integer out of range");
return -ERANGE;
}
return 0;
}
static int nla_validate_int_range(const struct nla_policy *pt,
const struct nlattr *nla,
struct netlink_ext_ack *extack,
unsigned int validate)
{
switch (pt->type) {
case NLA_U8:
case NLA_U16:
case NLA_U32:
case NLA_U64:
case NLA_UINT:
case NLA_MSECS:
case NLA_BINARY:
case NLA_BE16:
case NLA_BE32:
return nla_validate_range_unsigned(pt, nla, extack, validate);
case NLA_S8:
case NLA_S16:
case NLA_S32:
case NLA_S64:
case NLA_SINT:
return nla_validate_int_range_signed(pt, nla, extack);
default:
WARN_ON(1);
return -EINVAL;
}
}
static int nla_validate_mask(const struct nla_policy *pt,
const struct nlattr *nla,
struct netlink_ext_ack *extack)
{
u64 value;
switch (pt->type) {
case NLA_U8:
value = nla_get_u8(nla);
break;
case NLA_U16:
value = nla_get_u16(nla);
break;
case NLA_U32:
value = nla_get_u32(nla);
break;
case NLA_U64:
value = nla_get_u64(nla);
break;
case NLA_UINT:
value = nla_get_uint(nla);
break;
case NLA_BE16:
value = ntohs(nla_get_be16(nla));
break;
case NLA_BE32:
value = ntohl(nla_get_be32(nla));
break;
default:
return -EINVAL;
}
if (value & ~(u64)pt->mask) {
NL_SET_ERR_MSG_ATTR(extack, nla, "reserved bit set");
return -EINVAL;
}
return 0;
}
static int validate_nla(const struct nlattr *nla, int maxtype,
const struct nla_policy *policy, unsigned int validate,
struct netlink_ext_ack *extack, unsigned int depth)
{
u16 strict_start_type = policy[0].strict_start_type;
const struct nla_policy *pt;
int minlen = 0, attrlen = nla_len(nla), type = nla_type(nla);
int err = -ERANGE;
if (strict_start_type && type >= strict_start_type)
validate |= NL_VALIDATE_STRICT;
if (type <= 0 || type > maxtype)
return 0;
type = array_index_nospec(type, maxtype + 1);
pt = &policy[type];
BUG_ON(pt->type > NLA_TYPE_MAX);
if (nla_attr_len[pt->type] && attrlen != nla_attr_len[pt->type]) {
pr_warn_ratelimited("netlink: '%s': attribute type %d has an invalid length.\n",
current->comm, type);
if (validate & NL_VALIDATE_STRICT_ATTRS) {
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"invalid attribute length");
return -EINVAL;
}
}
if (validate & NL_VALIDATE_NESTED) {
if ((pt->type == NLA_NESTED || pt->type == NLA_NESTED_ARRAY) &&
!(nla->nla_type & NLA_F_NESTED)) {
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"NLA_F_NESTED is missing");
return -EINVAL;
}
if (pt->type != NLA_NESTED && pt->type != NLA_NESTED_ARRAY &&
pt->type != NLA_UNSPEC && (nla->nla_type & NLA_F_NESTED)) {
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"NLA_F_NESTED not expected");
return -EINVAL;
}
}
switch (pt->type) {
case NLA_REJECT:
if (extack && pt->reject_message) {
NL_SET_BAD_ATTR(extack, nla);
extack->_msg = pt->reject_message;
return -EINVAL;
}
err = -EINVAL;
goto out_err;
case NLA_FLAG:
if (attrlen > 0)
goto out_err;
break;
case NLA_SINT:
case NLA_UINT:
if (attrlen != sizeof(u32) && attrlen != sizeof(u64)) {
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"invalid attribute length");
return -EINVAL;
}
break;
case NLA_BITFIELD32:
if (attrlen != sizeof(struct nla_bitfield32))
goto out_err;
err = validate_nla_bitfield32(nla, pt->bitfield32_valid);
if (err)
goto out_err;
break;
case NLA_NUL_STRING:
if (pt->len)
minlen = min_t(int, attrlen, pt->len + 1);
else
minlen = attrlen;
if (!minlen || memchr(nla_data(nla), '\0', minlen) == NULL) {
err = -EINVAL;
goto out_err;
}
fallthrough;
case NLA_STRING:
if (attrlen < 1)
goto out_err;
if (pt->len) {
char *buf = nla_data(nla);
if (buf[attrlen - 1] == '\0')
attrlen--;
if (attrlen > pt->len)
goto out_err;
}
break;
case NLA_BINARY:
if (pt->len && attrlen > pt->len)
goto out_err;
break;
case NLA_NESTED:
/* a nested attributes is allowed to be empty; if its not,
* it must have a size of at least NLA_HDRLEN.
*/
if (attrlen == 0)
break;
if (attrlen < NLA_HDRLEN)
goto out_err;
if (pt->nested_policy) {
err = __nla_validate_parse(nla_data(nla), nla_len(nla),
pt->len, pt->nested_policy,
validate, extack, NULL,
depth + 1);
if (err < 0) {
/*
* return directly to preserve the inner
* error message/attribute pointer
*/
return err;
}
}
break;
case NLA_NESTED_ARRAY:
/* a nested array attribute is allowed to be empty; if its not,
* it must have a size of at least NLA_HDRLEN.
*/
if (attrlen == 0)
break;
if (attrlen < NLA_HDRLEN)
goto out_err;
if (pt->nested_policy) {
int err;
err = nla_validate_array(nla_data(nla), nla_len(nla),
pt->len, pt->nested_policy,
extack, validate, depth);
if (err < 0) {
/*
* return directly to preserve the inner
* error message/attribute pointer
*/
return err;
}
}
break;
case NLA_UNSPEC:
if (validate & NL_VALIDATE_UNSPEC) {
NL_SET_ERR_MSG_ATTR(extack, nla,
"Unsupported attribute");
return -EINVAL;
}
if (attrlen < pt->len)
goto out_err;
break;
default:
if (pt->len)
minlen = pt->len;
else
minlen = nla_attr_minlen[pt->type];
if (attrlen < minlen)
goto out_err;
}
/* further validation */
switch (pt->validation_type) {
case NLA_VALIDATE_NONE:
/* nothing to do */
break;
case NLA_VALIDATE_RANGE_PTR:
case NLA_VALIDATE_RANGE:
case NLA_VALIDATE_RANGE_WARN_TOO_LONG:
case NLA_VALIDATE_MIN:
case NLA_VALIDATE_MAX:
err = nla_validate_int_range(pt, nla, extack, validate);
if (err)
return err;
break;
case NLA_VALIDATE_MASK:
err = nla_validate_mask(pt, nla, extack);
if (err)
return err;
break;
case NLA_VALIDATE_FUNCTION:
if (pt->validate) {
err = pt->validate(nla, extack);
if (err)
return err;
}
break;
}
return 0;
out_err:
NL_SET_ERR_MSG_ATTR_POL(extack, nla, pt,
"Attribute failed policy validation");
return err;
}
static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype,
const struct nla_policy *policy,
unsigned int validate,
struct netlink_ext_ack *extack,
struct nlattr **tb, unsigned int depth)
{
const struct nlattr *nla;
int rem;
if (depth >= MAX_POLICY_RECURSION_DEPTH) {
NL_SET_ERR_MSG(extack,
"allowed policy recursion depth exceeded");
return -EINVAL;
}
if (tb)
memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1));
nla_for_each_attr(nla, head, len, rem) {
u16 type = nla_type(nla);
if (type == 0 || type > maxtype) {
if (validate & NL_VALIDATE_MAXTYPE) {
NL_SET_ERR_MSG_ATTR(extack, nla,
"Unknown attribute type");
return -EINVAL;
}
continue;
}
type = array_index_nospec(type, maxtype + 1);
if (policy) {
int err = validate_nla(nla, maxtype, policy,
validate, extack, depth);
if (err < 0)
return err;
}
if (tb)
tb[type] = (struct nlattr *)nla;
}
if (unlikely(rem > 0)) {
pr_warn_ratelimited("netlink: %d bytes leftover after parsing attributes in process `%s'.\n",
rem, current->comm);
NL_SET_ERR_MSG(extack, "bytes leftover after parsing attributes");
if (validate & NL_VALIDATE_TRAILING)
return -EINVAL;
}
return 0;
}
/**
* __nla_validate - Validate a stream of attributes
* @head: head of attribute stream
* @len: length of attribute stream
* @maxtype: maximum attribute type to be expected
* @policy: validation policy
* @validate: validation strictness
* @extack: extended ACK report struct
*
* Validates all attributes in the specified attribute stream against the
* specified policy. Validation depends on the validate flags passed, see
* &enum netlink_validation for more details on that.
* See documentation of struct nla_policy for more details.
*
* Returns 0 on success or a negative error code.
*/
int __nla_validate(const struct nlattr *head, int len, int maxtype,
const struct nla_policy *policy, unsigned int validate,
struct netlink_ext_ack *extack)
{
return __nla_validate_parse(head, len, maxtype, policy, validate,
extack, NULL, 0);
}
EXPORT_SYMBOL(__nla_validate);
/**
* nla_policy_len - Determine the max. length of a policy
* @p: policy to use
* @n: number of policies
*
* Determines the max. length of the policy. It is currently used
* to allocated Netlink buffers roughly the size of the actual
* message.
*
* Returns 0 on success or a negative error code.
*/
int
nla_policy_len(const struct nla_policy *p, int n)
{
int i, len = 0;
for (i = 0; i < n; i++, p++) {
if (p->len)
len += nla_total_size(p->len);
else if (nla_attr_len[p->type])
len += nla_total_size(nla_attr_len[p->type]);
else if (nla_attr_minlen[p->type])
len += nla_total_size(nla_attr_minlen[p->type]);
}
return len;
}
EXPORT_SYMBOL(nla_policy_len);
/**
* __nla_parse - Parse a stream of attributes into a tb buffer
* @tb: destination array with maxtype+1 elements
* @maxtype: maximum attribute type to be expected
* @head: head of attribute stream
* @len: length of attribute stream
* @policy: validation policy
* @validate: validation strictness
* @extack: extended ACK pointer
*
* Parses a stream of attributes and stores a pointer to each attribute in
* the tb array accessible via the attribute type.
* Validation is controlled by the @validate parameter.
*
* Returns 0 on success or a negative error code.
*/
int __nla_parse(struct nlattr **tb, int maxtype,
const struct nlattr *head, int len,
const struct nla_policy *policy, unsigned int validate,
struct netlink_ext_ack *extack)
{
return __nla_validate_parse(head, len, maxtype, policy, validate,
extack, tb, 0);
}
EXPORT_SYMBOL(__nla_parse);
/**
* nla_find - Find a specific attribute in a stream of attributes
* @head: head of attribute stream
* @len: length of attribute stream
* @attrtype: type of attribute to look for
*
* Returns the first attribute in the stream matching the specified type.
*/
struct nlattr *nla_find(const struct nlattr *head, int len, int attrtype)
{
const struct nlattr *nla;
int rem;
nla_for_each_attr(nla, head, len, rem)
if (nla_type(nla) == attrtype)
return (struct nlattr *)nla;
return NULL;
}
EXPORT_SYMBOL(nla_find);
/**
* nla_strscpy - Copy string attribute payload into a sized buffer
* @dst: Where to copy the string to.
* @nla: Attribute to copy the string from.
* @dstsize: Size of destination buffer.
*
* Copies at most dstsize - 1 bytes into the destination buffer.
* Unlike strscpy() the destination buffer is always padded out.
*
* Return:
* * srclen - Returns @nla length (not including the trailing %NUL).
* * -E2BIG - If @dstsize is 0 or greater than U16_MAX or @nla length greater
* than @dstsize.
*/
ssize_t nla_strscpy(char *dst, const struct nlattr *nla, size_t dstsize)
{
size_t srclen = nla_len(nla);
char *src = nla_data(nla);
ssize_t ret;
size_t len;
if (dstsize == 0 || WARN_ON_ONCE(dstsize > U16_MAX))
return -E2BIG;
if (srclen > 0 && src[srclen - 1] == '\0')
srclen--;
if (srclen >= dstsize) {
len = dstsize - 1;
ret = -E2BIG;
} else {
len = srclen;
ret = len;
}
memcpy(dst, src, len);
/* Zero pad end of dst. */
memset(dst + len, 0, dstsize - len);
return ret;
}
EXPORT_SYMBOL(nla_strscpy);
/**
* nla_strdup - Copy string attribute payload into a newly allocated buffer
* @nla: attribute to copy the string from
* @flags: the type of memory to allocate (see kmalloc).
*
* Returns a pointer to the allocated buffer or NULL on error.
*/
char *nla_strdup(const struct nlattr *nla, gfp_t flags)
{
size_t srclen = nla_len(nla);
char *src = nla_data(nla), *dst;
if (srclen > 0 && src[srclen - 1] == '\0')
srclen--;
dst = kmalloc(srclen + 1, flags);
if (dst != NULL) {
memcpy(dst, src, srclen);
dst[srclen] = '\0';
}
return dst;
}
EXPORT_SYMBOL(nla_strdup);
/**
* nla_memcpy - Copy a netlink attribute into another memory area
* @dest: where to copy to memcpy
* @src: netlink attribute to copy from
* @count: size of the destination area
*
* Note: The number of bytes copied is limited by the length of
* attribute's payload. memcpy
*
* Returns the number of bytes copied.
*/
int nla_memcpy(void *dest, const struct nlattr *src, int count)
{
int minlen = min_t(int, count, nla_len(src));
memcpy(dest, nla_data(src), minlen);
if (count > minlen)
memset(dest + minlen, 0, count - minlen);
return minlen;
}
EXPORT_SYMBOL(nla_memcpy);
/**
* nla_memcmp - Compare an attribute with sized memory area
* @nla: netlink attribute
* @data: memory area
* @size: size of memory area
*/
int nla_memcmp(const struct nlattr *nla, const void *data,
size_t size)
{
int d = nla_len(nla) - size;
if (d == 0)
d = memcmp(nla_data(nla), data, size);
return d;
}
EXPORT_SYMBOL(nla_memcmp);
/**
* nla_strcmp - Compare a string attribute against a string
* @nla: netlink string attribute
* @str: another string
*/
int nla_strcmp(const struct nlattr *nla, const char *str)
{
int len = strlen(str);
char *buf = nla_data(nla);
int attrlen = nla_len(nla);
int d;
while (attrlen > 0 && buf[attrlen - 1] == '\0')
attrlen--;
d = attrlen - len;
if (d == 0)
d = memcmp(nla_data(nla), str, len);
return d;
}
EXPORT_SYMBOL(nla_strcmp);
#ifdef CONFIG_NET
/**
* __nla_reserve - reserve room for attribute on the skb
* @skb: socket buffer to reserve room on
* @attrtype: attribute type
* @attrlen: length of attribute payload
*
* Adds a netlink attribute header to a socket buffer and reserves
* room for the payload but does not copy it.
*
* The caller is responsible to ensure that the skb provides enough
* tailroom for the attribute header and payload.
*/
struct nlattr *__nla_reserve(struct sk_buff *skb, int attrtype, int attrlen)
{
struct nlattr *nla;
nla = skb_put(skb, nla_total_size(attrlen));
nla->nla_type = attrtype;
nla->nla_len = nla_attr_size(attrlen);
memset((unsigned char *) nla + nla->nla_len, 0, nla_padlen(attrlen));
return nla;
}
EXPORT_SYMBOL(__nla_reserve);
/**
* __nla_reserve_64bit - reserve room for attribute on the skb and align it
* @skb: socket buffer to reserve room on
* @attrtype: attribute type
* @attrlen: length of attribute payload
* @padattr: attribute type for the padding
*
* Adds a netlink attribute header to a socket buffer and reserves
* room for the payload but does not copy it. It also ensure that this
* attribute will have a 64-bit aligned nla_data() area.
*
* The caller is responsible to ensure that the skb provides enough
* tailroom for the attribute header and payload.
*/
struct nlattr *__nla_reserve_64bit(struct sk_buff *skb, int attrtype,
int attrlen, int padattr)
{
nla_align_64bit(skb, padattr);
return __nla_reserve(skb, attrtype, attrlen);
}
EXPORT_SYMBOL(__nla_reserve_64bit);
/**
* __nla_reserve_nohdr - reserve room for attribute without header
* @skb: socket buffer to reserve room on
* @attrlen: length of attribute payload
*
* Reserves room for attribute payload without a header.
*
* The caller is responsible to ensure that the skb provides enough
* tailroom for the payload.
*/
void *__nla_reserve_nohdr(struct sk_buff *skb, int attrlen)
{
return skb_put_zero(skb, NLA_ALIGN(attrlen));
}
EXPORT_SYMBOL(__nla_reserve_nohdr);
/**
* nla_reserve - reserve room for attribute on the skb
* @skb: socket buffer to reserve room on
* @attrtype: attribute type
* @attrlen: length of attribute payload
*
* Adds a netlink attribute header to a socket buffer and reserves
* room for the payload but does not copy it.
*
* Returns NULL if the tailroom of the skb is insufficient to store
* the attribute header and payload.
*/
struct nlattr *nla_reserve(struct sk_buff *skb, int attrtype, int attrlen)
{
if (unlikely(skb_tailroom(skb) < nla_total_size(attrlen)))
return NULL;
return __nla_reserve(skb, attrtype, attrlen);
}
EXPORT_SYMBOL(nla_reserve);
/**
* nla_reserve_64bit - reserve room for attribute on the skb and align it
* @skb: socket buffer to reserve room on
* @attrtype: attribute type
* @attrlen: length of attribute payload
* @padattr: attribute type for the padding
*
* Adds a netlink attribute header to a socket buffer and reserves
* room for the payload but does not copy it. It also ensure that this
* attribute will have a 64-bit aligned nla_data() area.
*
* Returns NULL if the tailroom of the skb is insufficient to store
* the attribute header and payload.
*/
struct nlattr *nla_reserve_64bit(struct sk_buff *skb, int attrtype, int attrlen,
int padattr)
{
size_t len;
if (nla_need_padding_for_64bit(skb))
len = nla_total_size_64bit(attrlen);
else
len = nla_total_size(attrlen); if (unlikely(skb_tailroom(skb) < len))
return NULL;
return __nla_reserve_64bit(skb, attrtype, attrlen, padattr);
}
EXPORT_SYMBOL(nla_reserve_64bit);
/**
* nla_reserve_nohdr - reserve room for attribute without header
* @skb: socket buffer to reserve room on
* @attrlen: length of attribute payload
*
* Reserves room for attribute payload without a header.
*
* Returns NULL if the tailroom of the skb is insufficient to store
* the attribute payload.
*/
void *nla_reserve_nohdr(struct sk_buff *skb, int attrlen)
{
if (unlikely(skb_tailroom(skb) < NLA_ALIGN(attrlen)))
return NULL;
return __nla_reserve_nohdr(skb, attrlen);
}
EXPORT_SYMBOL(nla_reserve_nohdr);
/**
* __nla_put - Add a netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @attrlen: length of attribute payload
* @data: head of attribute payload
*
* The caller is responsible to ensure that the skb provides enough
* tailroom for the attribute header and payload.
*/
void __nla_put(struct sk_buff *skb, int attrtype, int attrlen,
const void *data)
{
struct nlattr *nla;
nla = __nla_reserve(skb, attrtype, attrlen);
memcpy(nla_data(nla), data, attrlen);
}
EXPORT_SYMBOL(__nla_put);
/**
* __nla_put_64bit - Add a netlink attribute to a socket buffer and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @attrlen: length of attribute payload
* @data: head of attribute payload
* @padattr: attribute type for the padding
*
* The caller is responsible to ensure that the skb provides enough
* tailroom for the attribute header and payload.
*/
void __nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen,
const void *data, int padattr)
{
struct nlattr *nla;
nla = __nla_reserve_64bit(skb, attrtype, attrlen, padattr);
memcpy(nla_data(nla), data, attrlen);
}
EXPORT_SYMBOL(__nla_put_64bit);
/**
* __nla_put_nohdr - Add a netlink attribute without header
* @skb: socket buffer to add attribute to
* @attrlen: length of attribute payload
* @data: head of attribute payload
*
* The caller is responsible to ensure that the skb provides enough
* tailroom for the attribute payload.
*/
void __nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data)
{
void *start;
start = __nla_reserve_nohdr(skb, attrlen);
memcpy(start, data, attrlen);
}
EXPORT_SYMBOL(__nla_put_nohdr);
/**
* nla_put - Add a netlink attribute to a socket buffer
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @attrlen: length of attribute payload
* @data: head of attribute payload
*
* Returns -EMSGSIZE if the tailroom of the skb is insufficient to store
* the attribute header and payload.
*/
int nla_put(struct sk_buff *skb, int attrtype, int attrlen, const void *data)
{
if (unlikely(skb_tailroom(skb) < nla_total_size(attrlen)))
return -EMSGSIZE;
__nla_put(skb, attrtype, attrlen, data); return 0;
}
EXPORT_SYMBOL(nla_put);
/**
* nla_put_64bit - Add a netlink attribute to a socket buffer and align it
* @skb: socket buffer to add attribute to
* @attrtype: attribute type
* @attrlen: length of attribute payload
* @data: head of attribute payload
* @padattr: attribute type for the padding
*
* Returns -EMSGSIZE if the tailroom of the skb is insufficient to store
* the attribute header and payload.
*/
int nla_put_64bit(struct sk_buff *skb, int attrtype, int attrlen,
const void *data, int padattr)
{
size_t len;
if (nla_need_padding_for_64bit(skb))
len = nla_total_size_64bit(attrlen);
else
len = nla_total_size(attrlen); if (unlikely(skb_tailroom(skb) < len))
return -EMSGSIZE;
__nla_put_64bit(skb, attrtype, attrlen, data, padattr); return 0;
}
EXPORT_SYMBOL(nla_put_64bit);
/**
* nla_put_nohdr - Add a netlink attribute without header
* @skb: socket buffer to add attribute to
* @attrlen: length of attribute payload
* @data: head of attribute payload
*
* Returns -EMSGSIZE if the tailroom of the skb is insufficient to store
* the attribute payload.
*/
int nla_put_nohdr(struct sk_buff *skb, int attrlen, const void *data)
{
if (unlikely(skb_tailroom(skb) < NLA_ALIGN(attrlen)))
return -EMSGSIZE;
__nla_put_nohdr(skb, attrlen, data);
return 0;
}
EXPORT_SYMBOL(nla_put_nohdr);
/**
* nla_append - Add a netlink attribute without header or padding
* @skb: socket buffer to add attribute to
* @attrlen: length of attribute payload
* @data: head of attribute payload
*
* Returns -EMSGSIZE if the tailroom of the skb is insufficient to store
* the attribute payload.
*/
int nla_append(struct sk_buff *skb, int attrlen, const void *data)
{
if (unlikely(skb_tailroom(skb) < NLA_ALIGN(attrlen)))
return -EMSGSIZE;
skb_put_data(skb, data, attrlen);
return 0;
}
EXPORT_SYMBOL(nla_append);
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_LIST_BL_H
#define _LINUX_LIST_BL_H
#include <linux/list.h>
#include <linux/bit_spinlock.h>
/*
* Special version of lists, where head of the list has a lock in the lowest
* bit. This is useful for scalable hash tables without increasing memory
* footprint overhead.
*
* For modification operations, the 0 bit of hlist_bl_head->first
* pointer must be set.
*
* With some small modifications, this can easily be adapted to store several
* arbitrary bits (not just a single lock bit), if the need arises to store
* some fast and compact auxiliary data.
*/
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
#define LIST_BL_LOCKMASK 1UL
#else
#define LIST_BL_LOCKMASK 0UL
#endif
#ifdef CONFIG_DEBUG_LIST
#define LIST_BL_BUG_ON(x) BUG_ON(x)
#else
#define LIST_BL_BUG_ON(x)
#endif
struct hlist_bl_head {
struct hlist_bl_node *first;
};
struct hlist_bl_node {
struct hlist_bl_node *next, **pprev;
};
#define INIT_HLIST_BL_HEAD(ptr) \
((ptr)->first = NULL)
static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h)
{
h->next = NULL;
h->pprev = NULL;
}
#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)
static inline bool hlist_bl_unhashed(const struct hlist_bl_node *h)
{
return !h->pprev;
}
static inline struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h)
{
return (struct hlist_bl_node *)
((unsigned long)h->first & ~LIST_BL_LOCKMASK);
}
static inline void hlist_bl_set_first(struct hlist_bl_head *h,
struct hlist_bl_node *n)
{
LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) !=
LIST_BL_LOCKMASK);
h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK);
}
static inline bool hlist_bl_empty(const struct hlist_bl_head *h)
{
return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK);
}
static inline void hlist_bl_add_head(struct hlist_bl_node *n,
struct hlist_bl_head *h)
{
struct hlist_bl_node *first = hlist_bl_first(h);
n->next = first;
if (first)
first->pprev = &n->next; n->pprev = &h->first; hlist_bl_set_first(h, n);
}
static inline void hlist_bl_add_before(struct hlist_bl_node *n,
struct hlist_bl_node *next)
{
struct hlist_bl_node **pprev = next->pprev;
n->pprev = pprev;
n->next = next;
next->pprev = &n->next;
/* pprev may be `first`, so be careful not to lose the lock bit */
WRITE_ONCE(*pprev,
(struct hlist_bl_node *)
((uintptr_t)n | ((uintptr_t)*pprev & LIST_BL_LOCKMASK)));
}
static inline void hlist_bl_add_behind(struct hlist_bl_node *n,
struct hlist_bl_node *prev)
{
n->next = prev->next;
n->pprev = &prev->next;
prev->next = n;
if (n->next)
n->next->pprev = &n->next;
}
static inline void __hlist_bl_del(struct hlist_bl_node *n)
{
struct hlist_bl_node *next = n->next;
struct hlist_bl_node **pprev = n->pprev;
LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
/* pprev may be `first`, so be careful not to lose the lock bit */
WRITE_ONCE(*pprev,
(struct hlist_bl_node *)
((unsigned long)next |
((unsigned long)*pprev & LIST_BL_LOCKMASK)));
if (next)
next->pprev = pprev;
}
static inline void hlist_bl_del(struct hlist_bl_node *n)
{
__hlist_bl_del(n);
n->next = LIST_POISON1;
n->pprev = LIST_POISON2;
}
static inline void hlist_bl_del_init(struct hlist_bl_node *n)
{
if (!hlist_bl_unhashed(n)) {
__hlist_bl_del(n);
INIT_HLIST_BL_NODE(n);
}
}
static inline void hlist_bl_lock(struct hlist_bl_head *b)
{
bit_spin_lock(0, (unsigned long *)b);
}
static inline void hlist_bl_unlock(struct hlist_bl_head *b)
{
__bit_spin_unlock(0, (unsigned long *)b);
}
static inline bool hlist_bl_is_locked(struct hlist_bl_head *b)
{
return bit_spin_is_locked(0, (unsigned long *)b);
}
/**
* hlist_bl_for_each_entry - iterate over list of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct hlist_node to use as a loop cursor.
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*
*/
#define hlist_bl_for_each_entry(tpos, pos, head, member) \
for (pos = hlist_bl_first(head); \
pos && \
({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
pos = pos->next)
/**
* hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct hlist_node to use as a loop cursor.
* @n: another &struct hlist_node to use as temporary storage
* @head: the head for your list.
* @member: the name of the hlist_node within the struct.
*/
#define hlist_bl_for_each_entry_safe(tpos, pos, n, head, member) \
for (pos = hlist_bl_first(head); \
pos && ({ n = pos->next; 1; }) && \
({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
pos = n)
#endif
/* SPDX-License-Identifier: GPL-2.0 */
/*
* This file provides wrappers with sanitizer instrumentation for non-atomic
* bit operations.
*
* To use this functionality, an arch's bitops.h file needs to define each of
* the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
* arch___set_bit(), etc.).
*/
#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
#include <linux/instrumented.h>
/**
* ___set_bit - Set a bit in memory
* @nr: the bit to set
* @addr: the address to start counting from
*
* Unlike set_bit(), this function is non-atomic. If it is called on the same
* region of memory concurrently, the effect may be that only one operation
* succeeds.
*/
static __always_inline void
___set_bit(unsigned long nr, volatile unsigned long *addr)
{
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___set_bit(nr, addr);
}
/**
* ___clear_bit - Clears a bit in memory
* @nr: the bit to clear
* @addr: the address to start counting from
*
* Unlike clear_bit(), this function is non-atomic. If it is called on the same
* region of memory concurrently, the effect may be that only one operation
* succeeds.
*/
static __always_inline void
___clear_bit(unsigned long nr, volatile unsigned long *addr)
{
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___clear_bit(nr, addr);
}
/**
* ___change_bit - Toggle a bit in memory
* @nr: the bit to change
* @addr: the address to start counting from
*
* Unlike change_bit(), this function is non-atomic. If it is called on the same
* region of memory concurrently, the effect may be that only one operation
* succeeds.
*/
static __always_inline void
___change_bit(unsigned long nr, volatile unsigned long *addr)
{
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___change_bit(nr, addr);
}
static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
{
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
/*
* We treat non-atomic read-write bitops a little more special.
* Given the operations here only modify a single bit, assuming
* non-atomicity of the writer is sufficient may be reasonable
* for certain usage (and follows the permissible nature of the
* assume-plain-writes-atomic rule):
* 1. report read-modify-write races -> check read;
* 2. do not report races with marked readers, but do report
* races with unmarked readers -> check "atomic" write.
*/
kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
/*
* Use generic write instrumentation, in case other sanitizers
* or tools are enabled alongside KCSAN.
*/
instrument_write(addr + BIT_WORD(nr), sizeof(long));
} else {
instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
}
}
/**
* ___test_and_set_bit - Set a bit and return its old value
* @nr: Bit to set
* @addr: Address to count from
*
* This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail.
*/
static __always_inline bool
___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
{
__instrument_read_write_bitop(nr, addr);
return arch___test_and_set_bit(nr, addr);
}
/**
* ___test_and_clear_bit - Clear a bit and return its old value
* @nr: Bit to clear
* @addr: Address to count from
*
* This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail.
*/
static __always_inline bool
___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
{
__instrument_read_write_bitop(nr, addr);
return arch___test_and_clear_bit(nr, addr);
}
/**
* ___test_and_change_bit - Change a bit and return its old value
* @nr: Bit to change
* @addr: Address to count from
*
* This operation is non-atomic. If two instances of this operation race, one
* can appear to succeed but actually fail.
*/
static __always_inline bool
___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
{
__instrument_read_write_bitop(nr, addr);
return arch___test_and_change_bit(nr, addr);
}
/**
* _test_bit - Determine whether a bit is set
* @nr: bit number to test
* @addr: Address to start counting from
*/
static __always_inline bool
_test_bit(unsigned long nr, const volatile unsigned long *addr)
{
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); return arch_test_bit(nr, addr);
}
/**
* _test_bit_acquire - Determine, with acquire semantics, whether a bit is set
* @nr: bit number to test
* @addr: Address to start counting from
*/
static __always_inline bool
_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
{
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
return arch_test_bit_acquire(nr, addr);
}
#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/extable.h>
#include <linux/uaccess.h>
#include <linux/sched/debug.h>
#include <linux/bitfield.h>
#include <xen/xen.h>
#include <asm/fpu/api.h>
#include <asm/fred.h>
#include <asm/sev.h>
#include <asm/traps.h>
#include <asm/kdebug.h>
#include <asm/insn-eval.h>
#include <asm/sgx.h>
static inline unsigned long *pt_regs_nr(struct pt_regs *regs, int nr)
{
int reg_offset = pt_regs_offset(regs, nr);
static unsigned long __dummy;
if (WARN_ON_ONCE(reg_offset < 0))
return &__dummy;
return (unsigned long *)((unsigned long)regs + reg_offset);
}
static inline unsigned long
ex_fixup_addr(const struct exception_table_entry *x)
{
return (unsigned long)&x->fixup + x->fixup;
}
static bool ex_handler_default(const struct exception_table_entry *e,
struct pt_regs *regs)
{
if (e->data & EX_FLAG_CLEAR_AX) regs->ax = 0; if (e->data & EX_FLAG_CLEAR_DX) regs->dx = 0; regs->ip = ex_fixup_addr(e);
return true;
}
/*
* This is the *very* rare case where we do a "load_unaligned_zeropad()"
* and it's a page crosser into a non-existent page.
*
* This happens when we optimistically load a pathname a word-at-a-time
* and the name is less than the full word and the next page is not
* mapped. Typically that only happens for CONFIG_DEBUG_PAGEALLOC.
*
* NOTE! The faulting address is always a 'mov mem,reg' type instruction
* of size 'long', and the exception fixup must always point to right
* after the instruction.
*/
static bool ex_handler_zeropad(const struct exception_table_entry *e,
struct pt_regs *regs,
unsigned long fault_addr)
{
struct insn insn;
const unsigned long mask = sizeof(long) - 1;
unsigned long offset, addr, next_ip, len;
unsigned long *reg;
next_ip = ex_fixup_addr(e);
len = next_ip - regs->ip;
if (len > MAX_INSN_SIZE)
return false;
if (insn_decode(&insn, (void *) regs->ip, len, INSN_MODE_KERN))
return false;
if (insn.length != len)
return false;
if (insn.opcode.bytes[0] != 0x8b)
return false;
if (insn.opnd_bytes != sizeof(long))
return false;
addr = (unsigned long) insn_get_addr_ref(&insn, regs);
if (addr == ~0ul)
return false;
offset = addr & mask;
addr = addr & ~mask;
if (fault_addr != addr + sizeof(long))
return false;
reg = insn_get_modrm_reg_ptr(&insn, regs);
if (!reg)
return false;
*reg = *(unsigned long *)addr >> (offset * 8);
return ex_handler_default(e, regs);
}
static bool ex_handler_fault(const struct exception_table_entry *fixup,
struct pt_regs *regs, int trapnr)
{
regs->ax = trapnr; return ex_handler_default(fixup, regs);
}
static bool ex_handler_sgx(const struct exception_table_entry *fixup,
struct pt_regs *regs, int trapnr)
{
regs->ax = trapnr | SGX_ENCLS_FAULT_FLAG; return ex_handler_default(fixup, regs);
}
/*
* Handler for when we fail to restore a task's FPU state. We should never get
* here because the FPU state of a task using the FPU (task->thread.fpu.state)
* should always be valid. However, past bugs have allowed userspace to set
* reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn().
* These caused XRSTOR to fail when switching to the task, leaking the FPU
* registers of the task previously executing on the CPU. Mitigate this class
* of vulnerability by restoring from the initial state (essentially, zeroing
* out all the FPU registers) if we can't restore from the task's FPU state.
*/
static bool ex_handler_fprestore(const struct exception_table_entry *fixup,
struct pt_regs *regs)
{
regs->ip = ex_fixup_addr(fixup); WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.",
(void *)instruction_pointer(regs));
fpu_reset_from_exception_fixup();
return true;
}
/*
* On x86-64, we end up being imprecise with 'access_ok()', and allow
* non-canonical user addresses to make the range comparisons simpler,
* and to not have to worry about LAM being enabled.
*
* In fact, we allow up to one page of "slop" at the sign boundary,
* which means that we can do access_ok() by just checking the sign
* of the pointer for the common case of having a small access size.
*/
static bool gp_fault_address_ok(unsigned long fault_address)
{
#ifdef CONFIG_X86_64
/* Is it in the "user space" part of the non-canonical space? */
if (valid_user_address(fault_address))
return true;
/* .. or just above it? */
fault_address -= PAGE_SIZE;
if (valid_user_address(fault_address))
return true;
#endif
return false;
}
static bool ex_handler_uaccess(const struct exception_table_entry *fixup,
struct pt_regs *regs, int trapnr,
unsigned long fault_address)
{
WARN_ONCE(trapnr == X86_TRAP_GP && !gp_fault_address_ok(fault_address),
"General protection fault in user access. Non-canonical address?");
return ex_handler_default(fixup, regs);
}
static bool ex_handler_copy(const struct exception_table_entry *fixup,
struct pt_regs *regs, int trapnr)
{
WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); return ex_handler_fault(fixup, regs, trapnr);
}
static bool ex_handler_msr(const struct exception_table_entry *fixup,
struct pt_regs *regs, bool wrmsr, bool safe, int reg)
{
if (__ONCE_LITE_IF(!safe && wrmsr)) {
pr_warn("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
(unsigned int)regs->cx, (unsigned int)regs->dx,
(unsigned int)regs->ax, regs->ip, (void *)regs->ip);
show_stack_regs(regs);
}
if (__ONCE_LITE_IF(!safe && !wrmsr)) {
pr_warn("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
(unsigned int)regs->cx, regs->ip, (void *)regs->ip);
show_stack_regs(regs);
}
if (!wrmsr) {
/* Pretend that the read succeeded and returned 0. */
regs->ax = 0;
regs->dx = 0;
}
if (safe)
*pt_regs_nr(regs, reg) = -EIO;
return ex_handler_default(fixup, regs);
}
static bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
struct pt_regs *regs)
{
if (static_cpu_has(X86_BUG_NULL_SEG)) asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); asm volatile ("mov %0, %%fs" : : "rm" (0)); return ex_handler_default(fixup, regs);
}
static bool ex_handler_imm_reg(const struct exception_table_entry *fixup,
struct pt_regs *regs, int reg, int imm)
{
*pt_regs_nr(regs, reg) = (long)imm; return ex_handler_default(fixup, regs);
}
static bool ex_handler_ucopy_len(const struct exception_table_entry *fixup,
struct pt_regs *regs, int trapnr,
unsigned long fault_address,
int reg, int imm)
{
regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg);
return ex_handler_uaccess(fixup, regs, trapnr, fault_address);
}
#ifdef CONFIG_X86_FRED
static bool ex_handler_eretu(const struct exception_table_entry *fixup,
struct pt_regs *regs, unsigned long error_code)
{
struct pt_regs *uregs = (struct pt_regs *)(regs->sp - offsetof(struct pt_regs, orig_ax));
unsigned short ss = uregs->ss;
unsigned short cs = uregs->cs;
/*
* Move the NMI bit from the invalid stack frame, which caused ERETU
* to fault, to the fault handler's stack frame, thus to unblock NMI
* with the fault handler's ERETS instruction ASAP if NMI is blocked.
*/
regs->fred_ss.nmi = uregs->fred_ss.nmi;
/*
* Sync event information to uregs, i.e., the ERETU return frame, but
* is it safe to write to the ERETU return frame which is just above
* current event stack frame?
*
* The RSP used by FRED to push a stack frame is not the value in %rsp,
* it is calculated from %rsp with the following 2 steps:
* 1) RSP = %rsp - (IA32_FRED_CONFIG & 0x1c0) // Reserve N*64 bytes
* 2) RSP = RSP & ~0x3f // Align to a 64-byte cache line
* when an event delivery doesn't trigger a stack level change.
*
* Here is an example with N*64 (N=1) bytes reserved:
*
* 64-byte cache line ==> ______________
* |___Reserved___|
* |__Event_data__|
* |_____SS_______|
* |_____RSP______|
* |_____FLAGS____|
* |_____CS_______|
* |_____IP_______|
* 64-byte cache line ==> |__Error_code__| <== ERETU return frame
* |______________|
* |______________|
* |______________|
* |______________|
* |______________|
* |______________|
* |______________|
* 64-byte cache line ==> |______________| <== RSP after step 1) and 2)
* |___Reserved___|
* |__Event_data__|
* |_____SS_______|
* |_____RSP______|
* |_____FLAGS____|
* |_____CS_______|
* |_____IP_______|
* 64-byte cache line ==> |__Error_code__| <== ERETS return frame
*
* Thus a new FRED stack frame will always be pushed below a previous
* FRED stack frame ((N*64) bytes may be reserved between), and it is
* safe to write to a previous FRED stack frame as they never overlap.
*/
fred_info(uregs)->edata = fred_event_data(regs);
uregs->ssx = regs->ssx;
uregs->fred_ss.ss = ss;
/* The NMI bit was moved away above */
uregs->fred_ss.nmi = 0;
uregs->csx = regs->csx;
uregs->fred_cs.sl = 0;
uregs->fred_cs.wfe = 0;
uregs->cs = cs;
uregs->orig_ax = error_code;
return ex_handler_default(fixup, regs);
}
#endif
int ex_get_fixup_type(unsigned long ip)
{
const struct exception_table_entry *e = search_exception_tables(ip);
return e ? FIELD_GET(EX_DATA_TYPE_MASK, e->data) : EX_TYPE_NONE;
}
int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
unsigned long fault_addr)
{
const struct exception_table_entry *e;
int type, reg, imm;
#ifdef CONFIG_PNPBIOS
if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) {
extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp;
extern u32 pnp_bios_is_utter_crap;
pnp_bios_is_utter_crap = 1;
printk(KERN_CRIT "PNPBIOS fault.. attempting recovery.\n");
__asm__ volatile(
"movl %0, %%esp\n\t"
"jmp *%1\n\t"
: : "g" (pnp_bios_fault_esp), "g" (pnp_bios_fault_eip));
panic("do_trap: can't hit this");
}
#endif
e = search_exception_tables(regs->ip);
if (!e)
return 0;
type = FIELD_GET(EX_DATA_TYPE_MASK, e->data);
reg = FIELD_GET(EX_DATA_REG_MASK, e->data);
imm = FIELD_GET(EX_DATA_IMM_MASK, e->data);
switch (type) {
case EX_TYPE_DEFAULT:
case EX_TYPE_DEFAULT_MCE_SAFE:
return ex_handler_default(e, regs);
case EX_TYPE_FAULT:
case EX_TYPE_FAULT_MCE_SAFE:
return ex_handler_fault(e, regs, trapnr);
case EX_TYPE_UACCESS:
return ex_handler_uaccess(e, regs, trapnr, fault_addr);
case EX_TYPE_COPY:
return ex_handler_copy(e, regs, trapnr);
case EX_TYPE_CLEAR_FS:
return ex_handler_clear_fs(e, regs);
case EX_TYPE_FPU_RESTORE:
return ex_handler_fprestore(e, regs);
case EX_TYPE_BPF:
return ex_handler_bpf(e, regs);
case EX_TYPE_WRMSR:
return ex_handler_msr(e, regs, true, false, reg);
case EX_TYPE_RDMSR:
return ex_handler_msr(e, regs, false, false, reg);
case EX_TYPE_WRMSR_SAFE:
return ex_handler_msr(e, regs, true, true, reg);
case EX_TYPE_RDMSR_SAFE:
return ex_handler_msr(e, regs, false, true, reg);
case EX_TYPE_WRMSR_IN_MCE:
ex_handler_msr_mce(regs, true);
break;
case EX_TYPE_RDMSR_IN_MCE:
ex_handler_msr_mce(regs, false);
break;
case EX_TYPE_POP_REG:
regs->sp += sizeof(long);
fallthrough;
case EX_TYPE_IMM_REG:
return ex_handler_imm_reg(e, regs, reg, imm);
case EX_TYPE_FAULT_SGX:
return ex_handler_sgx(e, regs, trapnr);
case EX_TYPE_UCOPY_LEN:
return ex_handler_ucopy_len(e, regs, trapnr, fault_addr, reg, imm);
case EX_TYPE_ZEROPAD:
return ex_handler_zeropad(e, regs, fault_addr);
#ifdef CONFIG_X86_FRED
case EX_TYPE_ERETU:
return ex_handler_eretu(e, regs, error_code);
#endif
}
BUG();
}
extern unsigned int early_recursion_flag;
/* Restricted version used during very early boot */
void __init early_fixup_exception(struct pt_regs *regs, int trapnr)
{
/* Ignore early NMIs. */
if (trapnr == X86_TRAP_NMI)
return;
if (early_recursion_flag > 2)
goto halt_loop;
/*
* Old CPUs leave the high bits of CS on the stack
* undefined. I'm not sure which CPUs do this, but at least
* the 486 DX works this way.
* Xen pv domains are not using the default __KERNEL_CS.
*/
if (!xen_pv_domain() && regs->cs != __KERNEL_CS)
goto fail;
/*
* The full exception fixup machinery is available as soon as
* the early IDT is loaded. This means that it is the
* responsibility of extable users to either function correctly
* when handlers are invoked early or to simply avoid causing
* exceptions before they're ready to handle them.
*
* This is better than filtering which handlers can be used,
* because refusing to call a handler here is guaranteed to
* result in a hard-to-debug panic.
*
* Keep in mind that not all vectors actually get here. Early
* page faults, for example, are special.
*/
if (fixup_exception(regs, trapnr, regs->orig_ax, 0))
return;
if (trapnr == X86_TRAP_UD) {
if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {
/* Skip the ud2. */
regs->ip += LEN_UD2;
return;
}
/*
* If this was a BUG and report_bug returns or if this
* was just a normal #UD, we want to continue onward and
* crash.
*/
}
fail:
early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n",
(unsigned)trapnr, (unsigned long)regs->cs, regs->ip,
regs->orig_ax, read_cr2());
show_regs(regs);
halt_loop:
while (true)
halt();
}
// SPDX-License-Identifier: GPL-2.0
/*
* Convert integer string representation to an integer.
* If an integer doesn't fit into specified type, -E is returned.
*
* Integer starts with optional sign.
* kstrtou*() functions do not accept sign "-".
*
* Radix 0 means autodetection: leading "0x" implies radix 16,
* leading "0" implies radix 8, otherwise radix is 10.
* Autodetection hints work after optional sign, but not before.
*
* If -E is returned, result is not touched.
*/
#include <linux/ctype.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/kstrtox.h>
#include <linux/math64.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include "kstrtox.h"
noinline
const char *_parse_integer_fixup_radix(const char *s, unsigned int *base)
{
if (*base == 0) { if (s[0] == '0') { if (_tolower(s[1]) == 'x' && isxdigit(s[2])) *base = 16;
else
*base = 8;
} else
*base = 10;
}
if (*base == 16 && s[0] == '0' && _tolower(s[1]) == 'x') s += 2; return s;
}
/*
* Convert non-negative integer string representation in explicitly given radix
* to an integer. A maximum of max_chars characters will be converted.
*
* Return number of characters consumed maybe or-ed with overflow bit.
* If overflow occurs, result integer (incorrect) is still returned.
*
* Don't you dare use this function.
*/
noinline
unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p,
size_t max_chars)
{
unsigned long long res;
unsigned int rv;
res = 0;
rv = 0;
while (max_chars--) { unsigned int c = *s; unsigned int lc = _tolower(c);
unsigned int val;
if ('0' <= c && c <= '9')
val = c - '0';
else if ('a' <= lc && lc <= 'f')
val = lc - 'a' + 10;
else
break;
if (val >= base)
break;
/*
* Check for overflow only if we are within range of
* it in the max base we support (16)
*/
if (unlikely(res & (~0ull << 60))) { if (res > div_u64(ULLONG_MAX - val, base)) rv |= KSTRTOX_OVERFLOW;
}
res = res * base + val;
rv++;
s++;
}
*p = res;
return rv;
}
noinline
unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p)
{
return _parse_integer_limit(s, base, p, INT_MAX);
}
static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
{
unsigned long long _res;
unsigned int rv;
s = _parse_integer_fixup_radix(s, &base);
rv = _parse_integer(s, base, &_res);
if (rv & KSTRTOX_OVERFLOW)
return -ERANGE;
if (rv == 0)
return -EINVAL;
s += rv;
if (*s == '\n')
s++;
if (*s)
return -EINVAL;
*res = _res;
return 0;
}
/**
* kstrtoull - convert a string to an unsigned long long
* @s: The start of the string. The string must be null-terminated, and may also
* include a single newline before its terminating null. The first character
* may also be a plus sign, but not a minus sign.
* @base: The number base to use. The maximum supported base is 16. If base is
* given as 0, then the base of the string is automatically detected with the
* conventional semantics - If it begins with 0x the number will be parsed as a
* hexadecimal (case insensitive), if it otherwise begins with 0, it will be
* parsed as an octal number. Otherwise it will be parsed as a decimal.
* @res: Where to write the result of the conversion on success.
*
* Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
* Preferred over simple_strtoull(). Return code must be checked.
*/
noinline
int kstrtoull(const char *s, unsigned int base, unsigned long long *res)
{
if (s[0] == '+')
s++;
return _kstrtoull(s, base, res);
}
EXPORT_SYMBOL(kstrtoull);
/**
* kstrtoll - convert a string to a long long
* @s: The start of the string. The string must be null-terminated, and may also
* include a single newline before its terminating null. The first character
* may also be a plus sign or a minus sign.
* @base: The number base to use. The maximum supported base is 16. If base is
* given as 0, then the base of the string is automatically detected with the
* conventional semantics - If it begins with 0x the number will be parsed as a
* hexadecimal (case insensitive), if it otherwise begins with 0, it will be
* parsed as an octal number. Otherwise it will be parsed as a decimal.
* @res: Where to write the result of the conversion on success.
*
* Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
* Preferred over simple_strtoll(). Return code must be checked.
*/
noinline
int kstrtoll(const char *s, unsigned int base, long long *res)
{
unsigned long long tmp;
int rv;
if (s[0] == '-') {
rv = _kstrtoull(s + 1, base, &tmp);
if (rv < 0)
return rv;
if ((long long)-tmp > 0)
return -ERANGE;
*res = -tmp;
} else {
rv = kstrtoull(s, base, &tmp);
if (rv < 0)
return rv;
if ((long long)tmp < 0)
return -ERANGE;
*res = tmp;
}
return 0;
}
EXPORT_SYMBOL(kstrtoll);
/* Internal, do not use. */
int _kstrtoul(const char *s, unsigned int base, unsigned long *res)
{
unsigned long long tmp;
int rv;
rv = kstrtoull(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (unsigned long)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(_kstrtoul);
/* Internal, do not use. */
int _kstrtol(const char *s, unsigned int base, long *res)
{
long long tmp;
int rv;
rv = kstrtoll(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (long)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(_kstrtol);
/**
* kstrtouint - convert a string to an unsigned int
* @s: The start of the string. The string must be null-terminated, and may also
* include a single newline before its terminating null. The first character
* may also be a plus sign, but not a minus sign.
* @base: The number base to use. The maximum supported base is 16. If base is
* given as 0, then the base of the string is automatically detected with the
* conventional semantics - If it begins with 0x the number will be parsed as a
* hexadecimal (case insensitive), if it otherwise begins with 0, it will be
* parsed as an octal number. Otherwise it will be parsed as a decimal.
* @res: Where to write the result of the conversion on success.
*
* Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
* Preferred over simple_strtoul(). Return code must be checked.
*/
noinline
int kstrtouint(const char *s, unsigned int base, unsigned int *res)
{
unsigned long long tmp;
int rv;
rv = kstrtoull(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (unsigned int)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(kstrtouint);
/**
* kstrtoint - convert a string to an int
* @s: The start of the string. The string must be null-terminated, and may also
* include a single newline before its terminating null. The first character
* may also be a plus sign or a minus sign.
* @base: The number base to use. The maximum supported base is 16. If base is
* given as 0, then the base of the string is automatically detected with the
* conventional semantics - If it begins with 0x the number will be parsed as a
* hexadecimal (case insensitive), if it otherwise begins with 0, it will be
* parsed as an octal number. Otherwise it will be parsed as a decimal.
* @res: Where to write the result of the conversion on success.
*
* Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
* Preferred over simple_strtol(). Return code must be checked.
*/
noinline
int kstrtoint(const char *s, unsigned int base, int *res)
{
long long tmp;
int rv;
rv = kstrtoll(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (int)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(kstrtoint);
noinline
int kstrtou16(const char *s, unsigned int base, u16 *res)
{
unsigned long long tmp;
int rv;
rv = kstrtoull(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (u16)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(kstrtou16);
noinline
int kstrtos16(const char *s, unsigned int base, s16 *res)
{
long long tmp;
int rv;
rv = kstrtoll(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (s16)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(kstrtos16);
noinline
int kstrtou8(const char *s, unsigned int base, u8 *res)
{
unsigned long long tmp;
int rv;
rv = kstrtoull(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (u8)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(kstrtou8);
noinline
int kstrtos8(const char *s, unsigned int base, s8 *res)
{
long long tmp;
int rv;
rv = kstrtoll(s, base, &tmp);
if (rv < 0)
return rv;
if (tmp != (s8)tmp)
return -ERANGE;
*res = tmp;
return 0;
}
EXPORT_SYMBOL(kstrtos8);
/**
* kstrtobool - convert common user inputs into boolean values
* @s: input string
* @res: result
*
* This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or
* [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value
* pointed to by res is updated upon finding a match.
*/
noinline
int kstrtobool(const char *s, bool *res)
{
if (!s)
return -EINVAL;
switch (s[0]) {
case 'y':
case 'Y':
case 't':
case 'T':
case '1':
*res = true;
return 0;
case 'n':
case 'N':
case 'f':
case 'F':
case '0':
*res = false;
return 0;
case 'o':
case 'O':
switch (s[1]) {
case 'n':
case 'N':
*res = true;
return 0;
case 'f':
case 'F':
*res = false;
return 0;
default:
break;
}
break;
default:
break;
}
return -EINVAL;
}
EXPORT_SYMBOL(kstrtobool);
/*
* Since "base" would be a nonsense argument, this open-codes the
* _from_user helper instead of using the helper macro below.
*/
int kstrtobool_from_user(const char __user *s, size_t count, bool *res)
{
/* Longest string needed to differentiate, newline, terminator */
char buf[4];
count = min(count, sizeof(buf) - 1);
if (copy_from_user(buf, s, count))
return -EFAULT;
buf[count] = '\0';
return kstrtobool(buf, res);
}
EXPORT_SYMBOL(kstrtobool_from_user);
#define kstrto_from_user(f, g, type) \
int f(const char __user *s, size_t count, unsigned int base, type *res) \
{ \
/* sign, base 2 representation, newline, terminator */ \
char buf[1 + sizeof(type) * 8 + 1 + 1]; \
\
count = min(count, sizeof(buf) - 1); \
if (copy_from_user(buf, s, count)) \
return -EFAULT; \
buf[count] = '\0'; \
return g(buf, base, res); \
} \
EXPORT_SYMBOL(f)
kstrto_from_user(kstrtoull_from_user, kstrtoull, unsigned long long);
kstrto_from_user(kstrtoll_from_user, kstrtoll, long long);
kstrto_from_user(kstrtoul_from_user, kstrtoul, unsigned long);
kstrto_from_user(kstrtol_from_user, kstrtol, long);
kstrto_from_user(kstrtouint_from_user, kstrtouint, unsigned int);
kstrto_from_user(kstrtoint_from_user, kstrtoint, int);
kstrto_from_user(kstrtou16_from_user, kstrtou16, u16);
kstrto_from_user(kstrtos16_from_user, kstrtos16, s16);
kstrto_from_user(kstrtou8_from_user, kstrtou8, u8);
kstrto_from_user(kstrtos8_from_user, kstrtos8, s8);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Macros for manipulating and testing page->flags
*/
#ifndef PAGE_FLAGS_H
#define PAGE_FLAGS_H
#include <linux/types.h>
#include <linux/bug.h>
#include <linux/mmdebug.h>
#ifndef __GENERATING_BOUNDS_H
#include <linux/mm_types.h>
#include <generated/bounds.h>
#endif /* !__GENERATING_BOUNDS_H */
/*
* Various page->flags bits:
*
* PG_reserved is set for special pages. The "struct page" of such a page
* should in general not be touched (e.g. set dirty) except by its owner.
* Pages marked as PG_reserved include:
* - Pages part of the kernel image (including vDSO) and similar (e.g. BIOS,
* initrd, HW tables)
* - Pages reserved or allocated early during boot (before the page allocator
* was initialized). This includes (depending on the architecture) the
* initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
* much more. Once (if ever) freed, PG_reserved is cleared and they will
* be given to the page allocator.
* - Pages falling into physical memory gaps - not IORESOURCE_SYSRAM. Trying
* to read/write these pages might end badly. Don't touch!
* - The zero page(s)
* - Pages not added to the page allocator when onlining a section because
* they were excluded via the online_page_callback() or because they are
* PG_hwpoison.
* - Pages allocated in the context of kexec/kdump (loaded kernel image,
* control pages, vmcoreinfo)
* - MMIO/DMA pages. Some architectures don't allow to ioremap pages that are
* not marked PG_reserved (as they might be in use by somebody else who does
* not respect the caching strategy).
* - Pages part of an offline section (struct pages of offline sections should
* not be trusted as they will be initialized when first onlined).
* - MCA pages on ia64
* - Pages holding CPU notes for POWER Firmware Assisted Dump
* - Device memory (e.g. PMEM, DAX, HMM)
* Some PG_reserved pages will be excluded from the hibernation image.
* PG_reserved does in general not hinder anybody from dumping or swapping
* and is no longer required for remap_pfn_range(). ioremap might require it.
* Consequently, PG_reserved for a page mapped into user space can indicate
* the zero page, the vDSO, MMIO pages or device memory.
*
* The PG_private bitflag is set on pagecache pages if they contain filesystem
* specific data (which is normally at page->private). It can be used by
* private allocations for its own usage.
*
* During initiation of disk I/O, PG_locked is set. This bit is set before I/O
* and cleared when writeback _starts_ or when read _completes_. PG_writeback
* is set before writeback starts and cleared when it finishes.
*
* PG_locked also pins a page in pagecache, and blocks truncation of the file
* while it is held.
*
* page_waitqueue(page) is a wait queue of all tasks waiting for the page
* to become unlocked.
*
* PG_swapbacked is set when a page uses swap as a backing storage. This are
* usually PageAnon or shmem pages but please note that even anonymous pages
* might lose their PG_swapbacked flag when they simply can be dropped (e.g. as
* a result of MADV_FREE).
*
* PG_referenced, PG_reclaim are used for page reclaim for anonymous and
* file-backed pagecache (see mm/vmscan.c).
*
* PG_error is set to indicate that an I/O error occurred on this page.
*
* PG_arch_1 is an architecture specific page state bit. The generic code
* guarantees that this bit is cleared for a page when it first is entered into
* the page cache.
*
* PG_hwpoison indicates that a page got corrupted in hardware and contains
* data with incorrect ECC bits that triggered a machine check. Accessing is
* not safe since it may cause another machine check. Don't touch!
*/
/*
* Don't use the pageflags directly. Use the PageFoo macros.
*
* The page flags field is split into two parts, the main flags area
* which extends from the low bits upwards, and the fields area which
* extends from the high bits downwards.
*
* | FIELD | ... | FLAGS |
* N-1 ^ 0
* (NR_PAGEFLAGS)
*
* The fields area is reserved for fields mapping zone, node (for NUMA) and
* SPARSEMEM section (for variants of SPARSEMEM that require section ids like
* SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
*/
enum pageflags {
PG_locked, /* Page is locked. Don't touch. */
PG_writeback, /* Page is under writeback */
PG_referenced,
PG_uptodate,
PG_dirty,
PG_lru,
PG_head, /* Must be in bit 6 */
PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_locked" */
PG_active,
PG_workingset,
PG_error,
PG_slab,
PG_owner_priv_1, /* Owner use. If pagecache, fs may use*/
PG_arch_1,
PG_reserved,
PG_private, /* If pagecache, has fs-private data */
PG_private_2, /* If pagecache, has fs aux data */
PG_mappedtodisk, /* Has blocks allocated on-disk */
PG_reclaim, /* To be reclaimed asap */
PG_swapbacked, /* Page is backed by RAM/swap */
PG_unevictable, /* Page is "unevictable" */
#ifdef CONFIG_MMU
PG_mlocked, /* Page is vma mlocked */
#endif
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
PG_uncached, /* Page has been mapped as uncached */
#endif
#ifdef CONFIG_MEMORY_FAILURE
PG_hwpoison, /* hardware poisoned page. Don't touch */
#endif
#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
PG_young,
PG_idle,
#endif
#ifdef CONFIG_ARCH_USES_PG_ARCH_X
PG_arch_2,
PG_arch_3,
#endif
__NR_PAGEFLAGS,
PG_readahead = PG_reclaim,
/*
* Depending on the way an anonymous folio can be mapped into a page
* table (e.g., single PMD/PUD/CONT of the head page vs. PTE-mapped
* THP), PG_anon_exclusive may be set only for the head page or for
* tail pages of an anonymous folio. For now, we only expect it to be
* set on tail pages for PTE-mapped THP.
*/
PG_anon_exclusive = PG_mappedtodisk,
/* Filesystems */
PG_checked = PG_owner_priv_1,
/* SwapBacked */
PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */
/* Two page bits are conscripted by FS-Cache to maintain local caching
* state. These bits are set on pages belonging to the netfs's inodes
* when those inodes are being locally cached.
*/
PG_fscache = PG_private_2, /* page backed by cache */
/* XEN */
/* Pinned in Xen as a read-only pagetable page. */
PG_pinned = PG_owner_priv_1,
/* Pinned as part of domain save (see xen_mm_pin_all()). */
PG_savepinned = PG_dirty,
/* Has a grant mapping of another (foreign) domain's page. */
PG_foreign = PG_owner_priv_1,
/* Remapped by swiotlb-xen. */
PG_xen_remapped = PG_owner_priv_1,
/* non-lru isolated movable page */
PG_isolated = PG_reclaim,
/* Only valid for buddy pages. Used to track pages that are reported */
PG_reported = PG_uptodate,
#ifdef CONFIG_MEMORY_HOTPLUG
/* For self-hosted memmap pages */
PG_vmemmap_self_hosted = PG_owner_priv_1,
#endif
/*
* Flags only valid for compound pages. Stored in first tail page's
* flags word. Cannot use the first 8 flags or any flag marked as
* PF_ANY.
*/
/* At least one page in this folio has the hwpoison flag set */
PG_has_hwpoisoned = PG_error,
PG_large_rmappable = PG_workingset, /* anon or file-backed */
};
#define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1)
#ifndef __GENERATING_BOUNDS_H
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
/*
* Return the real head page struct iff the @page is a fake head page, otherwise
* return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
*/
static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
{
if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
return page;
/*
* Only addresses aligned with PAGE_SIZE of struct page may be fake head
* struct page. The alignment check aims to avoid access the fields (
* e.g. compound_head) of the @page[1]. It can avoid touch a (possibly)
* cold cacheline in some cases.
*/
if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) && test_bit(PG_head, &page->flags)) {
/*
* We can safely access the field of the @page[1] with PG_head
* because the @page is a compound page composed with at least
* two contiguous pages.
*/
unsigned long head = READ_ONCE(page[1].compound_head);
if (likely(head & 1))
return (const struct page *)(head - 1);
}
return page;
}
#else
static inline const struct page *page_fixed_fake_head(const struct page *page)
{
return page;
}
#endif
static __always_inline int page_is_fake_head(const struct page *page)
{
return page_fixed_fake_head(page) != page;
}
static inline unsigned long _compound_head(const struct page *page)
{
unsigned long head = READ_ONCE(page->compound_head);
if (unlikely(head & 1))
return head - 1; return (unsigned long)page_fixed_fake_head(page);
}
#define compound_head(page) ((typeof(page))_compound_head(page))
/**
* page_folio - Converts from page to folio.
* @p: The page.
*
* Every page is part of a folio. This function cannot be called on a
* NULL pointer.
*
* Context: No reference, nor lock is required on @page. If the caller
* does not hold a reference, this call may race with a folio split, so
* it should re-check the folio still contains this page after gaining
* a reference on the folio.
* Return: The folio which contains this page.
*/
#define page_folio(p) (_Generic((p), \
const struct page *: (const struct folio *)_compound_head(p), \
struct page *: (struct folio *)_compound_head(p)))
/**
* folio_page - Return a page from a folio.
* @folio: The folio.
* @n: The page number to return.
*
* @n is relative to the start of the folio. This function does not
* check that the page number lies within @folio; the caller is presumed
* to have a reference to the page.
*/
#define folio_page(folio, n) nth_page(&(folio)->page, n)
static __always_inline int PageTail(const struct page *page)
{
return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);
}
static __always_inline int PageCompound(const struct page *page)
{
return test_bit(PG_head, &page->flags) ||
READ_ONCE(page->compound_head) & 1;
}
#define PAGE_POISON_PATTERN -1l
static inline int PagePoisoned(const struct page *page)
{
return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;
}
#ifdef CONFIG_DEBUG_VM
void page_init_poison(struct page *page, size_t size);
#else
static inline void page_init_poison(struct page *page, size_t size)
{
}
#endif
static const unsigned long *const_folio_flags(const struct folio *folio,
unsigned n)
{
const struct page *page = &folio->page; VM_BUG_ON_PGFLAGS(PageTail(page), page);
VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);
return &page[n].flags;
}
static unsigned long *folio_flags(struct folio *folio, unsigned n)
{
struct page *page = &folio->page; VM_BUG_ON_PGFLAGS(PageTail(page), page);
VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);
return &page[n].flags;
}
/*
* Page flags policies wrt compound pages
*
* PF_POISONED_CHECK
* check if this struct page poisoned/uninitialized
*
* PF_ANY:
* the page flag is relevant for small, head and tail pages.
*
* PF_HEAD:
* for compound page all operations related to the page flag applied to
* head page.
*
* PF_NO_TAIL:
* modifications of the page flag must be done on small or head pages,
* checks can be done on tail pages too.
*
* PF_NO_COMPOUND:
* the page flag is not relevant for compound pages.
*
* PF_SECOND:
* the page flag is stored in the first tail page.
*/
#define PF_POISONED_CHECK(page) ({ \
VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
page; })
#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
#define PF_NO_TAIL(page, enforce) ({ \
VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
PF_POISONED_CHECK(compound_head(page)); })
#define PF_NO_COMPOUND(page, enforce) ({ \
VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
PF_POISONED_CHECK(page); })
#define PF_SECOND(page, enforce) ({ \
VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
PF_POISONED_CHECK(&page[1]); })
/* Which page is the flag stored in */
#define FOLIO_PF_ANY 0
#define FOLIO_PF_HEAD 0
#define FOLIO_PF_NO_TAIL 0
#define FOLIO_PF_NO_COMPOUND 0
#define FOLIO_PF_SECOND 1
#define FOLIO_HEAD_PAGE 0
#define FOLIO_SECOND_PAGE 1
/*
* Macros to create function definitions for page flags
*/
#define FOLIO_TEST_FLAG(name, page) \
static __always_inline bool folio_test_##name(const struct folio *folio) \
{ return test_bit(PG_##name, const_folio_flags(folio, page)); }
#define FOLIO_SET_FLAG(name, page) \
static __always_inline void folio_set_##name(struct folio *folio) \
{ set_bit(PG_##name, folio_flags(folio, page)); }
#define FOLIO_CLEAR_FLAG(name, page) \
static __always_inline void folio_clear_##name(struct folio *folio) \
{ clear_bit(PG_##name, folio_flags(folio, page)); }
#define __FOLIO_SET_FLAG(name, page) \
static __always_inline void __folio_set_##name(struct folio *folio) \
{ __set_bit(PG_##name, folio_flags(folio, page)); }
#define __FOLIO_CLEAR_FLAG(name, page) \
static __always_inline void __folio_clear_##name(struct folio *folio) \
{ __clear_bit(PG_##name, folio_flags(folio, page)); }
#define FOLIO_TEST_SET_FLAG(name, page) \
static __always_inline bool folio_test_set_##name(struct folio *folio) \
{ return test_and_set_bit(PG_##name, folio_flags(folio, page)); }
#define FOLIO_TEST_CLEAR_FLAG(name, page) \
static __always_inline bool folio_test_clear_##name(struct folio *folio) \
{ return test_and_clear_bit(PG_##name, folio_flags(folio, page)); }
#define FOLIO_FLAG(name, page) \
FOLIO_TEST_FLAG(name, page) \
FOLIO_SET_FLAG(name, page) \
FOLIO_CLEAR_FLAG(name, page)
#define TESTPAGEFLAG(uname, lname, policy) \
FOLIO_TEST_FLAG(lname, FOLIO_##policy) \
static __always_inline int Page##uname(const struct page *page) \
{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
#define SETPAGEFLAG(uname, lname, policy) \
FOLIO_SET_FLAG(lname, FOLIO_##policy) \
static __always_inline void SetPage##uname(struct page *page) \
{ set_bit(PG_##lname, &policy(page, 1)->flags); }
#define CLEARPAGEFLAG(uname, lname, policy) \
FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \
static __always_inline void ClearPage##uname(struct page *page) \
{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
#define __SETPAGEFLAG(uname, lname, policy) \
__FOLIO_SET_FLAG(lname, FOLIO_##policy) \
static __always_inline void __SetPage##uname(struct page *page) \
{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
#define __CLEARPAGEFLAG(uname, lname, policy) \
__FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \
static __always_inline void __ClearPage##uname(struct page *page) \
{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
#define TESTSETFLAG(uname, lname, policy) \
FOLIO_TEST_SET_FLAG(lname, FOLIO_##policy) \
static __always_inline int TestSetPage##uname(struct page *page) \
{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
#define TESTCLEARFLAG(uname, lname, policy) \
FOLIO_TEST_CLEAR_FLAG(lname, FOLIO_##policy) \
static __always_inline int TestClearPage##uname(struct page *page) \
{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
#define PAGEFLAG(uname, lname, policy) \
TESTPAGEFLAG(uname, lname, policy) \
SETPAGEFLAG(uname, lname, policy) \
CLEARPAGEFLAG(uname, lname, policy)
#define __PAGEFLAG(uname, lname, policy) \
TESTPAGEFLAG(uname, lname, policy) \
__SETPAGEFLAG(uname, lname, policy) \
__CLEARPAGEFLAG(uname, lname, policy)
#define TESTSCFLAG(uname, lname, policy) \
TESTSETFLAG(uname, lname, policy) \
TESTCLEARFLAG(uname, lname, policy)
#define FOLIO_TEST_FLAG_FALSE(name) \
static inline bool folio_test_##name(const struct folio *folio) \
{ return false; }
#define FOLIO_SET_FLAG_NOOP(name) \
static inline void folio_set_##name(struct folio *folio) { }
#define FOLIO_CLEAR_FLAG_NOOP(name) \
static inline void folio_clear_##name(struct folio *folio) { }
#define __FOLIO_SET_FLAG_NOOP(name) \
static inline void __folio_set_##name(struct folio *folio) { }
#define __FOLIO_CLEAR_FLAG_NOOP(name) \
static inline void __folio_clear_##name(struct folio *folio) { }
#define FOLIO_TEST_SET_FLAG_FALSE(name) \
static inline bool folio_test_set_##name(struct folio *folio) \
{ return false; }
#define FOLIO_TEST_CLEAR_FLAG_FALSE(name) \
static inline bool folio_test_clear_##name(struct folio *folio) \
{ return false; }
#define FOLIO_FLAG_FALSE(name) \
FOLIO_TEST_FLAG_FALSE(name) \
FOLIO_SET_FLAG_NOOP(name) \
FOLIO_CLEAR_FLAG_NOOP(name)
#define TESTPAGEFLAG_FALSE(uname, lname) \
FOLIO_TEST_FLAG_FALSE(lname) \
static inline int Page##uname(const struct page *page) { return 0; }
#define SETPAGEFLAG_NOOP(uname, lname) \
FOLIO_SET_FLAG_NOOP(lname) \
static inline void SetPage##uname(struct page *page) { }
#define CLEARPAGEFLAG_NOOP(uname, lname) \
FOLIO_CLEAR_FLAG_NOOP(lname) \
static inline void ClearPage##uname(struct page *page) { }
#define __CLEARPAGEFLAG_NOOP(uname, lname) \
__FOLIO_CLEAR_FLAG_NOOP(lname) \
static inline void __ClearPage##uname(struct page *page) { }
#define TESTSETFLAG_FALSE(uname, lname) \
FOLIO_TEST_SET_FLAG_FALSE(lname) \
static inline int TestSetPage##uname(struct page *page) { return 0; }
#define TESTCLEARFLAG_FALSE(uname, lname) \
FOLIO_TEST_CLEAR_FLAG_FALSE(lname) \
static inline int TestClearPage##uname(struct page *page) { return 0; }
#define PAGEFLAG_FALSE(uname, lname) TESTPAGEFLAG_FALSE(uname, lname) \
SETPAGEFLAG_NOOP(uname, lname) CLEARPAGEFLAG_NOOP(uname, lname)
#define TESTSCFLAG_FALSE(uname, lname) \
TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname)
__PAGEFLAG(Locked, locked, PF_NO_TAIL)
FOLIO_FLAG(waiters, FOLIO_HEAD_PAGE)
PAGEFLAG(Error, error, PF_NO_TAIL) TESTCLEARFLAG(Error, error, PF_NO_TAIL)
PAGEFLAG(Referenced, referenced, PF_HEAD)
TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
__SETPAGEFLAG(Referenced, referenced, PF_HEAD)
PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
__CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD) TESTCLEARFLAG(LRU, lru, PF_HEAD)PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
TESTCLEARFLAG(Active, active, PF_HEAD)
PAGEFLAG(Workingset, workingset, PF_HEAD)
TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
__PAGEFLAG(Slab, slab, PF_NO_TAIL)
PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */
/* Xen */
PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
__CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
__SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
__CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
__SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
/*
* Private page markings that may be used by the filesystem that owns the page
* for its own purposes.
* - PG_private and PG_private_2 cause release_folio() and co to be invoked
*/
PAGEFLAG(Private, private, PF_ANY)
PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
/*
* Only test-and-set exist for PG_writeback. The unconditional operators are
* risky: they bypass page accounting.
*/
TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
/* PG_readahead is only used for reads; PG_reclaim is only for writes */
PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
PAGEFLAG(Readahead, readahead, PF_NO_COMPOUND)
TESTCLEARFLAG(Readahead, readahead, PF_NO_COMPOUND)
#ifdef CONFIG_HIGHMEM
/*
* Must use a macro here due to header dependency issues. page_zone() is not
* available at this point.
*/
#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
#define folio_test_highmem(__f) is_highmem_idx(folio_zonenum(__f))
#else
PAGEFLAG_FALSE(HighMem, highmem)
#endif
#ifdef CONFIG_SWAP
static __always_inline bool folio_test_swapcache(const struct folio *folio)
{
return folio_test_swapbacked(folio) && test_bit(PG_swapcache, const_folio_flags(folio, 0));
}
static __always_inline bool PageSwapCache(const struct page *page)
{
return folio_test_swapcache(page_folio(page));
}
SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
#else
PAGEFLAG_FALSE(SwapCache, swapcache)
#endif
PAGEFLAG(Unevictable, unevictable, PF_HEAD)
__CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
#ifdef CONFIG_MMU
PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL) __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
#else
PAGEFLAG_FALSE(Mlocked, mlocked) __CLEARPAGEFLAG_NOOP(Mlocked, mlocked)
TESTSCFLAG_FALSE(Mlocked, mlocked)
#endif
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
#else
PAGEFLAG_FALSE(Uncached, uncached)
#endif
#ifdef CONFIG_MEMORY_FAILURE
PAGEFLAG(HWPoison, hwpoison, PF_ANY)
TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
#define __PG_HWPOISON (1UL << PG_hwpoison)
#define MAGIC_HWPOISON 0x48575053U /* HWPS */
extern void SetPageHWPoisonTakenOff(struct page *page);
extern void ClearPageHWPoisonTakenOff(struct page *page);
extern bool take_page_off_buddy(struct page *page);
extern bool put_page_back_buddy(struct page *page);
#else
PAGEFLAG_FALSE(HWPoison, hwpoison)
#define __PG_HWPOISON 0
#endif
#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
FOLIO_TEST_FLAG(young, FOLIO_HEAD_PAGE)
FOLIO_SET_FLAG(young, FOLIO_HEAD_PAGE)
FOLIO_TEST_CLEAR_FLAG(young, FOLIO_HEAD_PAGE)
FOLIO_FLAG(idle, FOLIO_HEAD_PAGE)
#endif
/*
* PageReported() is used to track reported free pages within the Buddy
* allocator. We can use the non-atomic version of the test and set
* operations as both should be shielded with the zone lock to prevent
* any possible races on the setting or clearing of the bit.
*/
__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
#ifdef CONFIG_MEMORY_HOTPLUG
PAGEFLAG(VmemmapSelfHosted, vmemmap_self_hosted, PF_ANY)
#else
PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted)
#endif
/*
* On an anonymous page mapped into a user virtual memory area,
* page->mapping points to its anon_vma, not to a struct address_space;
* with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
*
* On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
* the PAGE_MAPPING_MOVABLE bit may be set along with the PAGE_MAPPING_ANON
* bit; and then page->mapping points, not to an anon_vma, but to a private
* structure which KSM associates with that merged page. See ksm.h.
*
* PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is used for non-lru movable
* page and then page->mapping points to a struct movable_operations.
*
* Please note that, confusingly, "page_mapping" refers to the inode
* address_space which maps the page from disk; whereas "page_mapped"
* refers to user virtual address space into which the page is mapped.
*
* For slab pages, since slab reuses the bits in struct page to store its
* internal states, the page->mapping does not exist as such, nor do these
* flags below. So in order to avoid testing non-existent bits, please
* make sure that PageSlab(page) actually evaluates to false before calling
* the following functions (e.g., PageAnon). See mm/slab.h.
*/
#define PAGE_MAPPING_ANON 0x1
#define PAGE_MAPPING_MOVABLE 0x2
#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
/*
* Different with flags above, this flag is used only for fsdax mode. It
* indicates that this page->mapping is now under reflink case.
*/
#define PAGE_MAPPING_DAX_SHARED ((void *)0x1)
static __always_inline bool folio_mapping_flags(const struct folio *folio)
{
return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) != 0;
}
static __always_inline int PageMappingFlags(const struct page *page)
{
return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
}
static __always_inline bool folio_test_anon(const struct folio *folio)
{
return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0;
}
static __always_inline bool PageAnon(const struct page *page)
{
return folio_test_anon(page_folio(page));
}
static __always_inline bool __folio_test_movable(const struct folio *folio)
{
return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==
PAGE_MAPPING_MOVABLE;
}
static __always_inline int __PageMovable(const struct page *page)
{
return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
PAGE_MAPPING_MOVABLE;
}
#ifdef CONFIG_KSM
/*
* A KSM page is one of those write-protected "shared pages" or "merged pages"
* which KSM maps into multiple mms, wherever identical anonymous page content
* is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
* anon_vma, but to that page's node of the stable tree.
*/
static __always_inline bool folio_test_ksm(const struct folio *folio)
{
return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==
PAGE_MAPPING_KSM;
}
static __always_inline bool PageKsm(const struct page *page)
{
return folio_test_ksm(page_folio(page));
}
#else
TESTPAGEFLAG_FALSE(Ksm, ksm)
#endif
u64 stable_page_flags(struct page *page);
/**
* folio_xor_flags_has_waiters - Change some folio flags.
* @folio: The folio.
* @mask: Bits set in this word will be changed.
*
* This must only be used for flags which are changed with the folio
* lock held. For example, it is unsafe to use for PG_dirty as that
* can be set without the folio lock held. It can also only be used
* on flags which are in the range 0-6 as some of the implementations
* only affect those bits.
*
* Return: Whether there are tasks waiting on the folio.
*/
static inline bool folio_xor_flags_has_waiters(struct folio *folio,
unsigned long mask)
{
return xor_unlock_is_negative_byte(mask, folio_flags(folio, 0));
}
/**
* folio_test_uptodate - Is this folio up to date?
* @folio: The folio.
*
* The uptodate flag is set on a folio when every byte in the folio is
* at least as new as the corresponding bytes on storage. Anonymous
* and CoW folios are always uptodate. If the folio is not uptodate,
* some of the bytes in it may be; see the is_partially_uptodate()
* address_space operation.
*/
static inline bool folio_test_uptodate(const struct folio *folio)
{
bool ret = test_bit(PG_uptodate, const_folio_flags(folio, 0));
/*
* Must ensure that the data we read out of the folio is loaded
* _after_ we've loaded folio->flags to check the uptodate bit.
* We can skip the barrier if the folio is not uptodate, because
* we wouldn't be reading anything from it.
*
* See folio_mark_uptodate() for the other side of the story.
*/
if (ret)
smp_rmb();
return ret;
}
static inline int PageUptodate(const struct page *page)
{
return folio_test_uptodate(page_folio(page));
}
static __always_inline void __folio_mark_uptodate(struct folio *folio)
{
smp_wmb();
__set_bit(PG_uptodate, folio_flags(folio, 0));
}
static __always_inline void folio_mark_uptodate(struct folio *folio)
{
/*
* Memory barrier must be issued before setting the PG_uptodate bit,
* so that all previous stores issued in order to bring the folio
* uptodate are actually visible before folio_test_uptodate becomes true.
*/
smp_wmb();
set_bit(PG_uptodate, folio_flags(folio, 0));
}
static __always_inline void __SetPageUptodate(struct page *page)
{
__folio_mark_uptodate((struct folio *)page);
}
static __always_inline void SetPageUptodate(struct page *page)
{
folio_mark_uptodate((struct folio *)page);
}
CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
void __folio_start_writeback(struct folio *folio, bool keep_write);
void set_page_writeback(struct page *page);
#define folio_start_writeback(folio) \
__folio_start_writeback(folio, false)
#define folio_start_writeback_keepwrite(folio) \
__folio_start_writeback(folio, true)
static __always_inline bool folio_test_head(const struct folio *folio)
{
return test_bit(PG_head, const_folio_flags(folio, FOLIO_PF_ANY));
}
static __always_inline int PageHead(const struct page *page)
{
PF_POISONED_CHECK(page); return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);
}
__SETPAGEFLAG(Head, head, PF_ANY)
__CLEARPAGEFLAG(Head, head, PF_ANY)
CLEARPAGEFLAG(Head, head, PF_ANY)
/**
* folio_test_large() - Does this folio contain more than one page?
* @folio: The folio to test.
*
* Return: True if the folio is larger than one page.
*/
static inline bool folio_test_large(const struct folio *folio)
{
return folio_test_head(folio);
}
static __always_inline void set_compound_head(struct page *page, struct page *head)
{
WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
}
static __always_inline void clear_compound_head(struct page *page)
{
WRITE_ONCE(page->compound_head, 0);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline void ClearPageCompound(struct page *page)
{
BUG_ON(!PageHead(page));
ClearPageHead(page);
}
PAGEFLAG(LargeRmappable, large_rmappable, PF_SECOND)
#else
TESTPAGEFLAG_FALSE(LargeRmappable, large_rmappable)
#endif
#define PG_head_mask ((1UL << PG_head))
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
* PageHuge() only returns true for hugetlbfs pages, but not for
* normal or transparent huge pages.
*
* PageTransHuge() returns true for both transparent huge and
* hugetlbfs pages, but not normal pages. PageTransHuge() can only be
* called only in the core VM paths where hugetlbfs pages can't exist.
*/
static inline int PageTransHuge(const struct page *page)
{
VM_BUG_ON_PAGE(PageTail(page), page);
return PageHead(page);
}
/*
* PageTransCompound returns true for both transparent huge pages
* and hugetlbfs pages, so it should only be called when it's known
* that hugetlbfs pages aren't involved.
*/
static inline int PageTransCompound(const struct page *page)
{
return PageCompound(page);
}
/*
* PageTransTail returns true for both transparent huge pages
* and hugetlbfs pages, so it should only be called when it's known
* that hugetlbfs pages aren't involved.
*/
static inline int PageTransTail(const struct page *page)
{
return PageTail(page);
}
#else
TESTPAGEFLAG_FALSE(TransHuge, transhuge)
TESTPAGEFLAG_FALSE(TransCompound, transcompound)
TESTPAGEFLAG_FALSE(TransCompoundMap, transcompoundmap)
TESTPAGEFLAG_FALSE(TransTail, transtail)
#endif
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
/*
* PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the
* compound page.
*
* This flag is set by hwpoison handler. Cleared by THP split or free page.
*/
PAGEFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
TESTSCFLAG(HasHWPoisoned, has_hwpoisoned, PF_SECOND)
#else
PAGEFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
TESTSCFLAG_FALSE(HasHWPoisoned, has_hwpoisoned)
#endif
/*
* For pages that are never mapped to userspace (and aren't PageSlab),
* page_type may be used. Because it is initialised to -1, we invert the
* sense of the bit, so __SetPageFoo *clears* the bit used for PageFoo, and
* __ClearPageFoo *sets* the bit used for PageFoo. We reserve a few high and
* low bits so that an underflow or overflow of _mapcount won't be
* mistaken for a page type value.
*/
#define PAGE_TYPE_BASE 0xf0000000
/* Reserve 0x0000007f to catch underflows of _mapcount */
#define PAGE_MAPCOUNT_RESERVE -128
#define PG_buddy 0x00000080
#define PG_offline 0x00000100
#define PG_table 0x00000200
#define PG_guard 0x00000400
#define PG_hugetlb 0x00000800
#define PageType(page, flag) \
((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
#define folio_test_type(folio, flag) \
((folio->page.page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
static inline int page_type_has_type(unsigned int page_type)
{
return (int)page_type < PAGE_MAPCOUNT_RESERVE;
}
static inline int page_has_type(const struct page *page)
{
return page_type_has_type(page->page_type);
}
#define FOLIO_TYPE_OPS(lname, fname) \
static __always_inline bool folio_test_##fname(const struct folio *folio)\
{ \
return folio_test_type(folio, PG_##lname); \
} \
static __always_inline void __folio_set_##fname(struct folio *folio) \
{ \
VM_BUG_ON_FOLIO(!folio_test_type(folio, 0), folio); \
folio->page.page_type &= ~PG_##lname; \
} \
static __always_inline void __folio_clear_##fname(struct folio *folio) \
{ \
VM_BUG_ON_FOLIO(!folio_test_##fname(folio), folio); \
folio->page.page_type |= PG_##lname; \
}
#define PAGE_TYPE_OPS(uname, lname, fname) \
FOLIO_TYPE_OPS(lname, fname) \
static __always_inline int Page##uname(const struct page *page) \
{ \
return PageType(page, PG_##lname); \
} \
static __always_inline void __SetPage##uname(struct page *page) \
{ \
VM_BUG_ON_PAGE(!PageType(page, 0), page); \
page->page_type &= ~PG_##lname; \
} \
static __always_inline void __ClearPage##uname(struct page *page) \
{ \
VM_BUG_ON_PAGE(!Page##uname(page), page); \
page->page_type |= PG_##lname; \
}
/*
* PageBuddy() indicates that the page is free and in the buddy system
* (see mm/page_alloc.c).
*/
PAGE_TYPE_OPS(Buddy, buddy, buddy)
/*
* PageOffline() indicates that the page is logically offline although the
* containing section is online. (e.g. inflated in a balloon driver or
* not onlined when onlining the section).
* The content of these pages is effectively stale. Such pages should not
* be touched (read/write/dump/save) except by their owner.
*
* If a driver wants to allow to offline unmovable PageOffline() pages without
* putting them back to the buddy, it can do so via the memory notifier by
* decrementing the reference count in MEM_GOING_OFFLINE and incrementing the
* reference count in MEM_CANCEL_OFFLINE. When offlining, the PageOffline()
* pages (now with a reference count of zero) are treated like free pages,
* allowing the containing memory block to get offlined. A driver that
* relies on this feature is aware that re-onlining the memory block will
* require to re-set the pages PageOffline() and not giving them to the
* buddy via online_page_callback_t.
*
* There are drivers that mark a page PageOffline() and expect there won't be
* any further access to page content. PFN walkers that read content of random
* pages should check PageOffline() and synchronize with such drivers using
* page_offline_freeze()/page_offline_thaw().
*/
PAGE_TYPE_OPS(Offline, offline, offline)
extern void page_offline_freeze(void);
extern void page_offline_thaw(void);
extern void page_offline_begin(void);
extern void page_offline_end(void);
/*
* Marks pages in use as page tables.
*/
PAGE_TYPE_OPS(Table, table, pgtable)
/*
* Marks guardpages used with debug_pagealloc.
*/
PAGE_TYPE_OPS(Guard, guard, guard)
#ifdef CONFIG_HUGETLB_PAGE
FOLIO_TYPE_OPS(hugetlb, hugetlb)
#else
FOLIO_TEST_FLAG_FALSE(hugetlb)
#endif
/**
* PageHuge - Determine if the page belongs to hugetlbfs
* @page: The page to test.
*
* Context: Any context.
* Return: True for hugetlbfs pages, false for anon pages or pages
* belonging to other filesystems.
*/
static inline bool PageHuge(const struct page *page)
{
return folio_test_hugetlb(page_folio(page));
}
/*
* Check if a page is currently marked HWPoisoned. Note that this check is
* best effort only and inherently racy: there is no way to synchronize with
* failing hardware.
*/
static inline bool is_page_hwpoison(struct page *page)
{
if (PageHWPoison(page))
return true;
return PageHuge(page) && PageHWPoison(compound_head(page));
}
extern bool is_free_buddy_page(struct page *page);
PAGEFLAG(Isolated, isolated, PF_ANY);
static __always_inline int PageAnonExclusive(const struct page *page)
{
VM_BUG_ON_PGFLAGS(!PageAnon(page), page); VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
}
static __always_inline void SetPageAnonExclusive(struct page *page)
{
VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page); VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page); set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
}
static __always_inline void ClearPageAnonExclusive(struct page *page)
{
VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);
VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
}
static __always_inline void __ClearPageAnonExclusive(struct page *page)
{
VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);
}
#ifdef CONFIG_MMU
#define __PG_MLOCKED (1UL << PG_mlocked)
#else
#define __PG_MLOCKED 0
#endif
/*
* Flags checked when a page is freed. Pages being freed should not have
* these flags set. If they are, there is a problem.
*/
#define PAGE_FLAGS_CHECK_AT_FREE \
(1UL << PG_lru | 1UL << PG_locked | \
1UL << PG_private | 1UL << PG_private_2 | \
1UL << PG_writeback | 1UL << PG_reserved | \
1UL << PG_slab | 1UL << PG_active | \
1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK)
/*
* Flags checked when a page is prepped for return by the page allocator.
* Pages being prepped should not have these flags set. If they are set,
* there has been a kernel bug or struct page corruption.
*
* __PG_HWPOISON is exceptional because it needs to be kept beyond page's
* alloc-free cycle to prevent from reusing the page.
*/
#define PAGE_FLAGS_CHECK_AT_PREP \
((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK)
/*
* Flags stored in the second page of a compound page. They may overlap
* the CHECK_AT_FREE flags above, so need to be cleared.
*/
#define PAGE_FLAGS_SECOND \
(0xffUL /* order */ | 1UL << PG_has_hwpoisoned | \
1UL << PG_large_rmappable)
#define PAGE_FLAGS_PRIVATE \
(1UL << PG_private | 1UL << PG_private_2)
/**
* page_has_private - Determine if page has private stuff
* @page: The page to be checked
*
* Determine if a page has private stuff, indicating that release routines
* should be invoked upon it.
*/
static inline int page_has_private(const struct page *page)
{
return !!(page->flags & PAGE_FLAGS_PRIVATE);
}
static inline bool folio_has_private(const struct folio *folio)
{
return page_has_private(&folio->page);
}
#undef PF_ANY
#undef PF_HEAD
#undef PF_NO_TAIL
#undef PF_NO_COMPOUND
#undef PF_SECOND
#endif /* !__GENERATING_BOUNDS_H */
#endif /* PAGE_FLAGS_H */
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/export.h>
#include <linux/bvec.h>
#include <linux/fault-inject-usercopy.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/splice.h>
#include <linux/compat.h>
#include <linux/scatterlist.h>
#include <linux/instrumented.h>
#include <linux/iov_iter.h>
static __always_inline
size_t copy_to_user_iter(void __user *iter_to, size_t progress,
size_t len, void *from, void *priv2)
{
if (should_fail_usercopy())
return len;
if (access_ok(iter_to, len)) { from += progress; instrument_copy_to_user(iter_to, from, len);
len = raw_copy_to_user(iter_to, from, len);
}
return len;
}
static __always_inline
size_t copy_to_user_iter_nofault(void __user *iter_to, size_t progress,
size_t len, void *from, void *priv2)
{
ssize_t res;
if (should_fail_usercopy())
return len;
from += progress;
res = copy_to_user_nofault(iter_to, from, len);
return res < 0 ? len : res;
}
static __always_inline
size_t copy_from_user_iter(void __user *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
size_t res = len;
if (should_fail_usercopy())
return len;
if (access_ok(iter_from, len)) { to += progress; instrument_copy_from_user_before(to, iter_from, len);
res = raw_copy_from_user(to, iter_from, len);
instrument_copy_from_user_after(to, iter_from, len, res);
}
return res;
}
static __always_inline
size_t memcpy_to_iter(void *iter_to, size_t progress,
size_t len, void *from, void *priv2)
{
memcpy(iter_to, from + progress, len);
return 0;
}
static __always_inline
size_t memcpy_from_iter(void *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
memcpy(to + progress, iter_from, len);
return 0;
}
/*
* fault_in_iov_iter_readable - fault in iov iterator for reading
* @i: iterator
* @size: maximum length
*
* Fault in one or more iovecs of the given iov_iter, to a maximum length of
* @size. For each iovec, fault in each page that constitutes the iovec.
*
* Returns the number of bytes not faulted in (like copy_to_user() and
* copy_from_user()).
*
* Always returns 0 for non-userspace iterators.
*/
size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
{
if (iter_is_ubuf(i)) { size_t n = min(size, iov_iter_count(i));
n -= fault_in_readable(i->ubuf + i->iov_offset, n);
return size - n;
} else if (iter_is_iovec(i)) { size_t count = min(size, iov_iter_count(i));
const struct iovec *p;
size_t skip;
size -= count;
for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) { size_t len = min(count, p->iov_len - skip);
size_t ret;
if (unlikely(!len))
continue;
ret = fault_in_readable(p->iov_base + skip, len);
count -= len - ret;
if (ret)
break;
}
return count + size;
}
return 0;
}
EXPORT_SYMBOL(fault_in_iov_iter_readable);
/*
* fault_in_iov_iter_writeable - fault in iov iterator for writing
* @i: iterator
* @size: maximum length
*
* Faults in the iterator using get_user_pages(), i.e., without triggering
* hardware page faults. This is primarily useful when we already know that
* some or all of the pages in @i aren't in memory.
*
* Returns the number of bytes not faulted in, like copy_to_user() and
* copy_from_user().
*
* Always returns 0 for non-user-space iterators.
*/
size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
{
if (iter_is_ubuf(i)) {
size_t n = min(size, iov_iter_count(i));
n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);
return size - n;
} else if (iter_is_iovec(i)) {
size_t count = min(size, iov_iter_count(i));
const struct iovec *p;
size_t skip;
size -= count;
for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
size_t len = min(count, p->iov_len - skip);
size_t ret;
if (unlikely(!len))
continue;
ret = fault_in_safe_writeable(p->iov_base + skip, len);
count -= len - ret;
if (ret)
break;
}
return count + size;
}
return 0;
}
EXPORT_SYMBOL(fault_in_iov_iter_writeable);
void iov_iter_init(struct iov_iter *i, unsigned int direction,
const struct iovec *iov, unsigned long nr_segs,
size_t count)
{
WARN_ON(direction & ~(READ | WRITE));
*i = (struct iov_iter) {
.iter_type = ITER_IOVEC,
.nofault = false,
.data_source = direction,
.__iov = iov,
.nr_segs = nr_segs,
.iov_offset = 0,
.count = count
};
}
EXPORT_SYMBOL(iov_iter_init);
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
if (WARN_ON_ONCE(i->data_source)) return 0; if (user_backed_iter(i)) might_fault(); return iterate_and_advance(i, bytes, (void *)addr,
copy_to_user_iter, memcpy_to_iter);
}
EXPORT_SYMBOL(_copy_to_iter);
#ifdef CONFIG_ARCH_HAS_COPY_MC
static __always_inline
size_t copy_to_user_iter_mc(void __user *iter_to, size_t progress,
size_t len, void *from, void *priv2)
{
if (access_ok(iter_to, len)) {
from += progress;
instrument_copy_to_user(iter_to, from, len);
len = copy_mc_to_user(iter_to, from, len);
}
return len;
}
static __always_inline
size_t memcpy_to_iter_mc(void *iter_to, size_t progress,
size_t len, void *from, void *priv2)
{
return copy_mc_to_kernel(iter_to, from + progress, len);
}
/**
* _copy_mc_to_iter - copy to iter with source memory error exception handling
* @addr: source kernel address
* @bytes: total transfer length
* @i: destination iterator
*
* The pmem driver deploys this for the dax operation
* (dax_copy_to_iter()) for dax reads (bypass page-cache and the
* block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
* successfully copied.
*
* The main differences between this and typical _copy_to_iter().
*
* * Typical tail/residue handling after a fault retries the copy
* byte-by-byte until the fault happens again. Re-triggering machine
* checks is potentially fatal so the implementation uses source
* alignment and poison alignment assumptions to avoid re-triggering
* hardware exceptions.
*
* * ITER_KVEC and ITER_BVEC can return short copies. Compare to
* copy_to_iter() where only ITER_IOVEC attempts might return a short copy.
*
* Return: number of bytes copied (may be %0)
*/
size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
if (WARN_ON_ONCE(i->data_source))
return 0;
if (user_backed_iter(i))
might_fault();
return iterate_and_advance(i, bytes, (void *)addr,
copy_to_user_iter_mc, memcpy_to_iter_mc);
}
EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
#endif /* CONFIG_ARCH_HAS_COPY_MC */
static __always_inline
size_t __copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
return iterate_and_advance(i, bytes, addr,
copy_from_user_iter, memcpy_from_iter);
}
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
if (WARN_ON_ONCE(!i->data_source))
return 0;
if (user_backed_iter(i))
might_fault();
return __copy_from_iter(addr, bytes, i);
}
EXPORT_SYMBOL(_copy_from_iter);
static __always_inline
size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
return __copy_from_user_inatomic_nocache(to + progress, iter_from, len);
}
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
if (WARN_ON_ONCE(!i->data_source))
return 0;
return iterate_and_advance(i, bytes, addr,
copy_from_user_iter_nocache,
memcpy_from_iter);
}
EXPORT_SYMBOL(_copy_from_iter_nocache);
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
static __always_inline
size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
return __copy_from_user_flushcache(to + progress, iter_from, len);
}
static __always_inline
size_t memcpy_from_iter_flushcache(void *iter_from, size_t progress,
size_t len, void *to, void *priv2)
{
memcpy_flushcache(to + progress, iter_from, len);
return 0;
}
/**
* _copy_from_iter_flushcache - write destination through cpu cache
* @addr: destination kernel address
* @bytes: total transfer length
* @i: source iterator
*
* The pmem driver arranges for filesystem-dax to use this facility via
* dax_copy_from_iter() for ensuring that writes to persistent memory
* are flushed through the CPU cache. It is differentiated from
* _copy_from_iter_nocache() in that guarantees all data is flushed for
* all iterator types. The _copy_from_iter_nocache() only attempts to
* bypass the cache for the ITER_IOVEC case, and on some archs may use
* instructions that strand dirty-data in the cache.
*
* Return: number of bytes copied (may be %0)
*/
size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
{
if (WARN_ON_ONCE(!i->data_source))
return 0;
return iterate_and_advance(i, bytes, addr,
copy_from_user_iter_flushcache,
memcpy_from_iter_flushcache);
}
EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
#endif
static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
{
struct page *head;
size_t v = n + offset;
/*
* The general case needs to access the page order in order
* to compute the page size.
* However, we mostly deal with order-0 pages and thus can
* avoid a possible cache line miss for requests that fit all
* page orders.
*/
if (n <= v && v <= PAGE_SIZE)
return true;
head = compound_head(page);
v += (page - head) << PAGE_SHIFT;
if (WARN_ON(n > v || v > page_size(head))) return false;
return true;
}
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
size_t res = 0;
if (!page_copy_sane(page, offset, bytes)) return 0; if (WARN_ON_ONCE(i->data_source))
return 0;
page += offset / PAGE_SIZE; // first subpage
offset %= PAGE_SIZE;
while (1) {
void *kaddr = kmap_local_page(page);
size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
n = _copy_to_iter(kaddr + offset, n, i);
kunmap_local(kaddr);
res += n;
bytes -= n;
if (!bytes || !n)
break;
offset += n;
if (offset == PAGE_SIZE) {
page++;
offset = 0;
}
}
return res;
}
EXPORT_SYMBOL(copy_page_to_iter);
size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes,
struct iov_iter *i)
{
size_t res = 0;
if (!page_copy_sane(page, offset, bytes))
return 0;
if (WARN_ON_ONCE(i->data_source))
return 0;
page += offset / PAGE_SIZE; // first subpage
offset %= PAGE_SIZE;
while (1) {
void *kaddr = kmap_local_page(page);
size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
n = iterate_and_advance(i, n, kaddr + offset,
copy_to_user_iter_nofault,
memcpy_to_iter);
kunmap_local(kaddr);
res += n;
bytes -= n;
if (!bytes || !n)
break;
offset += n;
if (offset == PAGE_SIZE) {
page++;
offset = 0;
}
}
return res;
}
EXPORT_SYMBOL(copy_page_to_iter_nofault);
size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
size_t res = 0;
if (!page_copy_sane(page, offset, bytes))
return 0;
page += offset / PAGE_SIZE; // first subpage
offset %= PAGE_SIZE;
while (1) {
void *kaddr = kmap_local_page(page);
size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
n = _copy_from_iter(kaddr + offset, n, i);
kunmap_local(kaddr);
res += n;
bytes -= n;
if (!bytes || !n)
break;
offset += n;
if (offset == PAGE_SIZE) {
page++;
offset = 0;
}
}
return res;
}
EXPORT_SYMBOL(copy_page_from_iter);
static __always_inline
size_t zero_to_user_iter(void __user *iter_to, size_t progress,
size_t len, void *priv, void *priv2)
{
return clear_user(iter_to, len);
}
static __always_inline
size_t zero_to_iter(void *iter_to, size_t progress,
size_t len, void *priv, void *priv2)
{
memset(iter_to, 0, len);
return 0;
}
size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
{
return iterate_and_advance(i, bytes, NULL,
zero_to_user_iter, zero_to_iter);
}
EXPORT_SYMBOL(iov_iter_zero);
size_t copy_page_from_iter_atomic(struct page *page, size_t offset,
size_t bytes, struct iov_iter *i)
{
size_t n, copied = 0;
if (!page_copy_sane(page, offset, bytes))
return 0;
if (WARN_ON_ONCE(!i->data_source))
return 0;
do {
char *p;
n = bytes - copied;
if (PageHighMem(page)) {
page += offset / PAGE_SIZE;
offset %= PAGE_SIZE;
n = min_t(size_t, n, PAGE_SIZE - offset);
}
p = kmap_atomic(page) + offset; n = __copy_from_iter(p, n, i); kunmap_atomic(p);
copied += n;
offset += n;
} while (PageHighMem(page) && copied != bytes && n > 0);
return copied;
}
EXPORT_SYMBOL(copy_page_from_iter_atomic);
static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
{
const struct bio_vec *bvec, *end;
if (!i->count)
return;
i->count -= size;
size += i->iov_offset;
for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) {
if (likely(size < bvec->bv_len))
break;
size -= bvec->bv_len;
}
i->iov_offset = size;
i->nr_segs -= bvec - i->bvec;
i->bvec = bvec;
}
static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)
{
const struct iovec *iov, *end;
if (!i->count)
return;
i->count -= size;
size += i->iov_offset; // from beginning of current segment
for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) {
if (likely(size < iov->iov_len))
break;
size -= iov->iov_len;
}
i->iov_offset = size;
i->nr_segs -= iov - iter_iov(i);
i->__iov = iov;
}
void iov_iter_advance(struct iov_iter *i, size_t size)
{
if (unlikely(i->count < size))
size = i->count;
if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
i->iov_offset += size;
i->count -= size;
} else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) {
/* iovec and kvec have identical layouts */
iov_iter_iovec_advance(i, size);
} else if (iov_iter_is_bvec(i)) {
iov_iter_bvec_advance(i, size);
} else if (iov_iter_is_discard(i)) {
i->count -= size;
}
}
EXPORT_SYMBOL(iov_iter_advance);
void iov_iter_revert(struct iov_iter *i, size_t unroll)
{
if (!unroll)
return;
if (WARN_ON(unroll > MAX_RW_COUNT))
return;
i->count += unroll;
if (unlikely(iov_iter_is_discard(i)))
return;
if (unroll <= i->iov_offset) {
i->iov_offset -= unroll;
return;
}
unroll -= i->iov_offset;
if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) {
BUG(); /* We should never go beyond the start of the specified
* range since we might then be straying into pages that
* aren't pinned.
*/
} else if (iov_iter_is_bvec(i)) {
const struct bio_vec *bvec = i->bvec;
while (1) {
size_t n = (--bvec)->bv_len;
i->nr_segs++;
if (unroll <= n) {
i->bvec = bvec;
i->iov_offset = n - unroll;
return;
}
unroll -= n;
}
} else { /* same logics for iovec and kvec */
const struct iovec *iov = iter_iov(i);
while (1) {
size_t n = (--iov)->iov_len;
i->nr_segs++;
if (unroll <= n) {
i->__iov = iov;
i->iov_offset = n - unroll;
return;
}
unroll -= n;
}
}
}
EXPORT_SYMBOL(iov_iter_revert);
/*
* Return the count of just the current iov_iter segment.
*/
size_t iov_iter_single_seg_count(const struct iov_iter *i)
{
if (i->nr_segs > 1) {
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
return min(i->count, iter_iov(i)->iov_len - i->iov_offset);
if (iov_iter_is_bvec(i))
return min(i->count, i->bvec->bv_len - i->iov_offset);
}
return i->count;
}
EXPORT_SYMBOL(iov_iter_single_seg_count);
void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
const struct kvec *kvec, unsigned long nr_segs,
size_t count)
{
WARN_ON(direction & ~(READ | WRITE));
*i = (struct iov_iter){
.iter_type = ITER_KVEC,
.data_source = direction,
.kvec = kvec,
.nr_segs = nr_segs,
.iov_offset = 0,
.count = count
};
}
EXPORT_SYMBOL(iov_iter_kvec);
void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
const struct bio_vec *bvec, unsigned long nr_segs,
size_t count)
{
WARN_ON(direction & ~(READ | WRITE));
*i = (struct iov_iter){
.iter_type = ITER_BVEC,
.data_source = direction,
.bvec = bvec,
.nr_segs = nr_segs,
.iov_offset = 0,
.count = count
};
}
EXPORT_SYMBOL(iov_iter_bvec);
/**
* iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
* @i: The iterator to initialise.
* @direction: The direction of the transfer.
* @xarray: The xarray to access.
* @start: The start file position.
* @count: The size of the I/O buffer in bytes.
*
* Set up an I/O iterator to either draw data out of the pages attached to an
* inode or to inject data into those pages. The pages *must* be prevented
* from evaporation, either by taking a ref on them or locking them by the
* caller.
*/
void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
struct xarray *xarray, loff_t start, size_t count)
{
BUG_ON(direction & ~1);
*i = (struct iov_iter) {
.iter_type = ITER_XARRAY,
.data_source = direction,
.xarray = xarray,
.xarray_start = start,
.count = count,
.iov_offset = 0
};
}
EXPORT_SYMBOL(iov_iter_xarray);
/**
* iov_iter_discard - Initialise an I/O iterator that discards data
* @i: The iterator to initialise.
* @direction: The direction of the transfer.
* @count: The size of the I/O buffer in bytes.
*
* Set up an I/O iterator that just discards everything that's written to it.
* It's only available as a READ iterator.
*/
void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
{
BUG_ON(direction != READ);
*i = (struct iov_iter){
.iter_type = ITER_DISCARD,
.data_source = false,
.count = count,
.iov_offset = 0
};
}
EXPORT_SYMBOL(iov_iter_discard);
static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
unsigned len_mask)
{
const struct iovec *iov = iter_iov(i);
size_t size = i->count;
size_t skip = i->iov_offset;
do {
size_t len = iov->iov_len - skip;
if (len > size)
len = size;
if (len & len_mask)
return false;
if ((unsigned long)(iov->iov_base + skip) & addr_mask)
return false;
iov++;
size -= len;
skip = 0;
} while (size);
return true;
}
static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask,
unsigned len_mask)
{
const struct bio_vec *bvec = i->bvec;
unsigned skip = i->iov_offset;
size_t size = i->count;
do {
size_t len = bvec->bv_len;
if (len > size)
len = size;
if (len & len_mask)
return false;
if ((unsigned long)(bvec->bv_offset + skip) & addr_mask)
return false;
bvec++;
size -= len;
skip = 0;
} while (size);
return true;
}
/**
* iov_iter_is_aligned() - Check if the addresses and lengths of each segments
* are aligned to the parameters.
*
* @i: &struct iov_iter to restore
* @addr_mask: bit mask to check against the iov element's addresses
* @len_mask: bit mask to check against the iov element's lengths
*
* Return: false if any addresses or lengths intersect with the provided masks
*/
bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask,
unsigned len_mask)
{
if (likely(iter_is_ubuf(i))) {
if (i->count & len_mask)
return false;
if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask)
return false;
return true;
}
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
return iov_iter_aligned_iovec(i, addr_mask, len_mask);
if (iov_iter_is_bvec(i))
return iov_iter_aligned_bvec(i, addr_mask, len_mask);
if (iov_iter_is_xarray(i)) {
if (i->count & len_mask)
return false;
if ((i->xarray_start + i->iov_offset) & addr_mask)
return false;
}
return true;
}
EXPORT_SYMBOL_GPL(iov_iter_is_aligned);
static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
{
const struct iovec *iov = iter_iov(i);
unsigned long res = 0;
size_t size = i->count;
size_t skip = i->iov_offset;
do {
size_t len = iov->iov_len - skip;
if (len) {
res |= (unsigned long)iov->iov_base + skip;
if (len > size)
len = size;
res |= len;
size -= len;
}
iov++;
skip = 0;
} while (size);
return res;
}
static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i)
{
const struct bio_vec *bvec = i->bvec;
unsigned res = 0;
size_t size = i->count;
unsigned skip = i->iov_offset;
do {
size_t len = bvec->bv_len - skip;
res |= (unsigned long)bvec->bv_offset + skip;
if (len > size)
len = size;
res |= len;
bvec++;
size -= len;
skip = 0;
} while (size);
return res;
}
unsigned long iov_iter_alignment(const struct iov_iter *i)
{
if (likely(iter_is_ubuf(i))) {
size_t size = i->count;
if (size)
return ((unsigned long)i->ubuf + i->iov_offset) | size;
return 0;
}
/* iovec and kvec have identical layouts */
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
return iov_iter_alignment_iovec(i);
if (iov_iter_is_bvec(i))
return iov_iter_alignment_bvec(i);
if (iov_iter_is_xarray(i))
return (i->xarray_start + i->iov_offset) | i->count;
return 0;
}
EXPORT_SYMBOL(iov_iter_alignment);
unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
{
unsigned long res = 0;
unsigned long v = 0;
size_t size = i->count;
unsigned k;
if (iter_is_ubuf(i))
return 0;
if (WARN_ON(!iter_is_iovec(i)))
return ~0U;
for (k = 0; k < i->nr_segs; k++) {
const struct iovec *iov = iter_iov(i) + k;
if (iov->iov_len) {
unsigned long base = (unsigned long)iov->iov_base;
if (v) // if not the first one
res |= base | v; // this start | previous end
v = base + iov->iov_len;
if (size <= iov->iov_len)
break;
size -= iov->iov_len;
}
}
return res;
}
EXPORT_SYMBOL(iov_iter_gap_alignment);
static int want_pages_array(struct page ***res, size_t size,
size_t start, unsigned int maxpages)
{
unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE);
if (count > maxpages)
count = maxpages;
WARN_ON(!count); // caller should've prevented that
if (!*res) {
*res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
if (!*res)
return 0;
}
return count;
}
static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,
pgoff_t index, unsigned int nr_pages)
{
XA_STATE(xas, xa, index);
struct page *page;
unsigned int ret = 0;
rcu_read_lock();
for (page = xas_load(&xas); page; page = xas_next(&xas)) {
if (xas_retry(&xas, page))
continue;
/* Has the page moved or been split? */
if (unlikely(page != xas_reload(&xas))) {
xas_reset(&xas);
continue;
}
pages[ret] = find_subpage(page, xas.xa_index);
get_page(pages[ret]);
if (++ret == nr_pages)
break;
}
rcu_read_unlock();
return ret;
}
static ssize_t iter_xarray_get_pages(struct iov_iter *i,
struct page ***pages, size_t maxsize,
unsigned maxpages, size_t *_start_offset)
{
unsigned nr, offset, count;
pgoff_t index;
loff_t pos;
pos = i->xarray_start + i->iov_offset;
index = pos >> PAGE_SHIFT;
offset = pos & ~PAGE_MASK;
*_start_offset = offset;
count = want_pages_array(pages, maxsize, offset, maxpages);
if (!count)
return -ENOMEM;
nr = iter_xarray_populate_pages(*pages, i->xarray, index, count);
if (nr == 0)
return 0;
maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
i->iov_offset += maxsize;
i->count -= maxsize;
return maxsize;
}
/* must be done on non-empty ITER_UBUF or ITER_IOVEC one */
static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)
{
size_t skip;
long k;
if (iter_is_ubuf(i))
return (unsigned long)i->ubuf + i->iov_offset;
for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
const struct iovec *iov = iter_iov(i) + k;
size_t len = iov->iov_len - skip;
if (unlikely(!len))
continue;
if (*size > len)
*size = len;
return (unsigned long)iov->iov_base + skip;
}
BUG(); // if it had been empty, we wouldn't get called
}
/* must be done on non-empty ITER_BVEC one */
static struct page *first_bvec_segment(const struct iov_iter *i,
size_t *size, size_t *start)
{
struct page *page;
size_t skip = i->iov_offset, len;
len = i->bvec->bv_len - skip;
if (*size > len)
*size = len;
skip += i->bvec->bv_offset;
page = i->bvec->bv_page + skip / PAGE_SIZE;
*start = skip % PAGE_SIZE;
return page;
}
static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
struct page ***pages, size_t maxsize,
unsigned int maxpages, size_t *start)
{
unsigned int n, gup_flags = 0;
if (maxsize > i->count)
maxsize = i->count;
if (!maxsize)
return 0;
if (maxsize > MAX_RW_COUNT)
maxsize = MAX_RW_COUNT;
if (likely(user_backed_iter(i))) {
unsigned long addr;
int res;
if (iov_iter_rw(i) != WRITE)
gup_flags |= FOLL_WRITE;
if (i->nofault)
gup_flags |= FOLL_NOFAULT;
addr = first_iovec_segment(i, &maxsize);
*start = addr % PAGE_SIZE;
addr &= PAGE_MASK;
n = want_pages_array(pages, maxsize, *start, maxpages);
if (!n)
return -ENOMEM;
res = get_user_pages_fast(addr, n, gup_flags, *pages);
if (unlikely(res <= 0))
return res;
maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);
iov_iter_advance(i, maxsize);
return maxsize;
}
if (iov_iter_is_bvec(i)) {
struct page **p;
struct page *page;
page = first_bvec_segment(i, &maxsize, start);
n = want_pages_array(pages, maxsize, *start, maxpages);
if (!n)
return -ENOMEM;
p = *pages;
for (int k = 0; k < n; k++)
get_page(p[k] = page + k);
maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);
i->count -= maxsize;
i->iov_offset += maxsize;
if (i->iov_offset == i->bvec->bv_len) {
i->iov_offset = 0;
i->bvec++;
i->nr_segs--;
}
return maxsize;
}
if (iov_iter_is_xarray(i))
return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
return -EFAULT;
}
ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
size_t maxsize, unsigned maxpages, size_t *start)
{
if (!maxpages)
return 0;
BUG_ON(!pages);
return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages, start);
}
EXPORT_SYMBOL(iov_iter_get_pages2);
ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i,
struct page ***pages, size_t maxsize, size_t *start)
{
ssize_t len;
*pages = NULL;
len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start);
if (len <= 0) {
kvfree(*pages);
*pages = NULL;
}
return len;
}
EXPORT_SYMBOL(iov_iter_get_pages_alloc2);
static int iov_npages(const struct iov_iter *i, int maxpages)
{
size_t skip = i->iov_offset, size = i->count;
const struct iovec *p;
int npages = 0;
for (p = iter_iov(i); size; skip = 0, p++) {
unsigned offs = offset_in_page(p->iov_base + skip);
size_t len = min(p->iov_len - skip, size);
if (len) {
size -= len;
npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
if (unlikely(npages > maxpages))
return maxpages;
}
}
return npages;
}
static int bvec_npages(const struct iov_iter *i, int maxpages)
{
size_t skip = i->iov_offset, size = i->count;
const struct bio_vec *p;
int npages = 0;
for (p = i->bvec; size; skip = 0, p++) {
unsigned offs = (p->bv_offset + skip) % PAGE_SIZE;
size_t len = min(p->bv_len - skip, size);
size -= len;
npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
if (unlikely(npages > maxpages))
return maxpages;
}
return npages;
}
int iov_iter_npages(const struct iov_iter *i, int maxpages)
{
if (unlikely(!i->count))
return 0;
if (likely(iter_is_ubuf(i))) {
unsigned offs = offset_in_page(i->ubuf + i->iov_offset);
int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE);
return min(npages, maxpages);
}
/* iovec and kvec have identical layouts */
if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
return iov_npages(i, maxpages);
if (iov_iter_is_bvec(i))
return bvec_npages(i, maxpages);
if (iov_iter_is_xarray(i)) {
unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;
int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);
return min(npages, maxpages);
}
return 0;
}
EXPORT_SYMBOL(iov_iter_npages);
const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
{
*new = *old;
if (iov_iter_is_bvec(new))
return new->bvec = kmemdup(new->bvec,
new->nr_segs * sizeof(struct bio_vec),
flags);
else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
/* iovec and kvec have identical layout */
return new->__iov = kmemdup(new->__iov,
new->nr_segs * sizeof(struct iovec),
flags);
return NULL;
}
EXPORT_SYMBOL(dup_iter);
static __noclone int copy_compat_iovec_from_user(struct iovec *iov,
const struct iovec __user *uvec, u32 nr_segs)
{
const struct compat_iovec __user *uiov =
(const struct compat_iovec __user *)uvec;
int ret = -EFAULT;
u32 i;
if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
return -EFAULT;
for (i = 0; i < nr_segs; i++) {
compat_uptr_t buf;
compat_ssize_t len;
unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
/* check for compat_size_t not fitting in compat_ssize_t .. */
if (len < 0) {
ret = -EINVAL;
goto uaccess_end;
}
iov[i].iov_base = compat_ptr(buf);
iov[i].iov_len = len;
}
ret = 0;
uaccess_end:
user_access_end();
return ret;
}
static __noclone int copy_iovec_from_user(struct iovec *iov,
const struct iovec __user *uiov, unsigned long nr_segs)
{
int ret = -EFAULT;
if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
return -EFAULT;
do {
void __user *buf;
ssize_t len;
unsafe_get_user(len, &uiov->iov_len, uaccess_end);
unsafe_get_user(buf, &uiov->iov_base, uaccess_end);
/* check for size_t not fitting in ssize_t .. */
if (unlikely(len < 0)) {
ret = -EINVAL;
goto uaccess_end;
}
iov->iov_base = buf;
iov->iov_len = len;
uiov++; iov++;
} while (--nr_segs);
ret = 0;
uaccess_end:
user_access_end();
return ret;
}
struct iovec *iovec_from_user(const struct iovec __user *uvec,
unsigned long nr_segs, unsigned long fast_segs,
struct iovec *fast_iov, bool compat)
{
struct iovec *iov = fast_iov;
int ret;
/*
* SuS says "The readv() function *may* fail if the iovcnt argument was
* less than or equal to 0, or greater than {IOV_MAX}. Linux has
* traditionally returned zero for zero segments, so...
*/
if (nr_segs == 0)
return iov;
if (nr_segs > UIO_MAXIOV)
return ERR_PTR(-EINVAL);
if (nr_segs > fast_segs) {
iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
if (!iov)
return ERR_PTR(-ENOMEM);
}
if (unlikely(compat))
ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
else
ret = copy_iovec_from_user(iov, uvec, nr_segs);
if (ret) {
if (iov != fast_iov)
kfree(iov);
return ERR_PTR(ret);
}
return iov;
}
/*
* Single segment iovec supplied by the user, import it as ITER_UBUF.
*/
static ssize_t __import_iovec_ubuf(int type, const struct iovec __user *uvec,
struct iovec **iovp, struct iov_iter *i,
bool compat)
{
struct iovec *iov = *iovp;
ssize_t ret;
if (compat)
ret = copy_compat_iovec_from_user(iov, uvec, 1);
else
ret = copy_iovec_from_user(iov, uvec, 1);
if (unlikely(ret))
return ret;
ret = import_ubuf(type, iov->iov_base, iov->iov_len, i);
if (unlikely(ret))
return ret;
*iovp = NULL;
return i->count;
}
ssize_t __import_iovec(int type, const struct iovec __user *uvec,
unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
struct iov_iter *i, bool compat)
{
ssize_t total_len = 0;
unsigned long seg;
struct iovec *iov;
if (nr_segs == 1)
return __import_iovec_ubuf(type, uvec, iovp, i, compat);
iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
if (IS_ERR(iov)) {
*iovp = NULL;
return PTR_ERR(iov);
}
/*
* According to the Single Unix Specification we should return EINVAL if
* an element length is < 0 when cast to ssize_t or if the total length
* would overflow the ssize_t return value of the system call.
*
* Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
* overflow case.
*/
for (seg = 0; seg < nr_segs; seg++) {
ssize_t len = (ssize_t)iov[seg].iov_len;
if (!access_ok(iov[seg].iov_base, len)) {
if (iov != *iovp)
kfree(iov);
*iovp = NULL;
return -EFAULT;
}
if (len > MAX_RW_COUNT - total_len) {
len = MAX_RW_COUNT - total_len;
iov[seg].iov_len = len;
}
total_len += len;
}
iov_iter_init(i, type, iov, nr_segs, total_len);
if (iov == *iovp)
*iovp = NULL;
else
*iovp = iov;
return total_len;
}
/**
* import_iovec() - Copy an array of &struct iovec from userspace
* into the kernel, check that it is valid, and initialize a new
* &struct iov_iter iterator to access it.
*
* @type: One of %READ or %WRITE.
* @uvec: Pointer to the userspace array.
* @nr_segs: Number of elements in userspace array.
* @fast_segs: Number of elements in @iov.
* @iovp: (input and output parameter) Pointer to pointer to (usually small
* on-stack) kernel array.
* @i: Pointer to iterator that will be initialized on success.
*
* If the array pointed to by *@iov is large enough to hold all @nr_segs,
* then this function places %NULL in *@iov on return. Otherwise, a new
* array will be allocated and the result placed in *@iov. This means that
* the caller may call kfree() on *@iov regardless of whether the small
* on-stack array was used or not (and regardless of whether this function
* returns an error or not).
*
* Return: Negative error code on error, bytes imported on success
*/
ssize_t import_iovec(int type, const struct iovec __user *uvec,
unsigned nr_segs, unsigned fast_segs,
struct iovec **iovp, struct iov_iter *i)
{
return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
in_compat_syscall());
}
EXPORT_SYMBOL(import_iovec);
int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i)
{
if (len > MAX_RW_COUNT)
len = MAX_RW_COUNT;
if (unlikely(!access_ok(buf, len)))
return -EFAULT;
iov_iter_ubuf(i, rw, buf, len);
return 0;
}
EXPORT_SYMBOL_GPL(import_ubuf);
/**
* iov_iter_restore() - Restore a &struct iov_iter to the same state as when
* iov_iter_save_state() was called.
*
* @i: &struct iov_iter to restore
* @state: state to restore from
*
* Used after iov_iter_save_state() to bring restore @i, if operations may
* have advanced it.
*
* Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC
*/
void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state)
{
if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) &&
!iter_is_ubuf(i)) && !iov_iter_is_kvec(i))
return;
i->iov_offset = state->iov_offset;
i->count = state->count;
if (iter_is_ubuf(i))
return;
/*
* For the *vec iters, nr_segs + iov is constant - if we increment
* the vec, then we also decrement the nr_segs count. Hence we don't
* need to track both of these, just one is enough and we can deduct
* the other from that. ITER_KVEC and ITER_IOVEC are the same struct
* size, so we can just increment the iov pointer as they are unionzed.
* ITER_BVEC _may_ be the same size on some archs, but on others it is
* not. Be safe and handle it separately.
*/
BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
if (iov_iter_is_bvec(i))
i->bvec -= state->nr_segs - i->nr_segs;
else
i->__iov -= state->nr_segs - i->nr_segs;
i->nr_segs = state->nr_segs;
}
/*
* Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not
* get references on the pages, nor does it get a pin on them.
*/
static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i,
struct page ***pages, size_t maxsize,
unsigned int maxpages,
iov_iter_extraction_t extraction_flags,
size_t *offset0)
{
struct page *page, **p;
unsigned int nr = 0, offset;
loff_t pos = i->xarray_start + i->iov_offset;
pgoff_t index = pos >> PAGE_SHIFT;
XA_STATE(xas, i->xarray, index);
offset = pos & ~PAGE_MASK;
*offset0 = offset;
maxpages = want_pages_array(pages, maxsize, offset, maxpages);
if (!maxpages)
return -ENOMEM;
p = *pages;
rcu_read_lock();
for (page = xas_load(&xas); page; page = xas_next(&xas)) {
if (xas_retry(&xas, page))
continue;
/* Has the page moved or been split? */
if (unlikely(page != xas_reload(&xas))) {
xas_reset(&xas);
continue;
}
p[nr++] = find_subpage(page, xas.xa_index);
if (nr == maxpages)
break;
}
rcu_read_unlock();
maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);
iov_iter_advance(i, maxsize);
return maxsize;
}
/*
* Extract a list of contiguous pages from an ITER_BVEC iterator. This does
* not get references on the pages, nor does it get a pin on them.
*/
static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i,
struct page ***pages, size_t maxsize,
unsigned int maxpages,
iov_iter_extraction_t extraction_flags,
size_t *offset0)
{
struct page **p, *page;
size_t skip = i->iov_offset, offset, size;
int k;
for (;;) {
if (i->nr_segs == 0)
return 0;
size = min(maxsize, i->bvec->bv_len - skip);
if (size)
break;
i->iov_offset = 0;
i->nr_segs--;
i->bvec++;
skip = 0;
}
skip += i->bvec->bv_offset;
page = i->bvec->bv_page + skip / PAGE_SIZE;
offset = skip % PAGE_SIZE;
*offset0 = offset;
maxpages = want_pages_array(pages, size, offset, maxpages);
if (!maxpages)
return -ENOMEM;
p = *pages;
for (k = 0; k < maxpages; k++)
p[k] = page + k;
size = min_t(size_t, size, maxpages * PAGE_SIZE - offset);
iov_iter_advance(i, size);
return size;
}
/*
* Extract a list of virtually contiguous pages from an ITER_KVEC iterator.
* This does not get references on the pages, nor does it get a pin on them.
*/
static ssize_t iov_iter_extract_kvec_pages(struct iov_iter *i,
struct page ***pages, size_t maxsize,
unsigned int maxpages,
iov_iter_extraction_t extraction_flags,
size_t *offset0)
{
struct page **p, *page;
const void *kaddr;
size_t skip = i->iov_offset, offset, len, size;
int k;
for (;;) {
if (i->nr_segs == 0)
return 0;
size = min(maxsize, i->kvec->iov_len - skip);
if (size)
break;
i->iov_offset = 0;
i->nr_segs--;
i->kvec++;
skip = 0;
}
kaddr = i->kvec->iov_base + skip;
offset = (unsigned long)kaddr & ~PAGE_MASK;
*offset0 = offset;
maxpages = want_pages_array(pages, size, offset, maxpages);
if (!maxpages)
return -ENOMEM;
p = *pages;
kaddr -= offset;
len = offset + size;
for (k = 0; k < maxpages; k++) {
size_t seg = min_t(size_t, len, PAGE_SIZE);
if (is_vmalloc_or_module_addr(kaddr))
page = vmalloc_to_page(kaddr);
else
page = virt_to_page(kaddr);
p[k] = page;
len -= seg;
kaddr += PAGE_SIZE;
}
size = min_t(size_t, size, maxpages * PAGE_SIZE - offset);
iov_iter_advance(i, size);
return size;
}
/*
* Extract a list of contiguous pages from a user iterator and get a pin on
* each of them. This should only be used if the iterator is user-backed
* (IOBUF/UBUF).
*
* It does not get refs on the pages, but the pages must be unpinned by the
* caller once the transfer is complete.
*
* This is safe to be used where background IO/DMA *is* going to be modifying
* the buffer; using a pin rather than a ref makes forces fork() to give the
* child a copy of the page.
*/
static ssize_t iov_iter_extract_user_pages(struct iov_iter *i,
struct page ***pages,
size_t maxsize,
unsigned int maxpages,
iov_iter_extraction_t extraction_flags,
size_t *offset0)
{
unsigned long addr;
unsigned int gup_flags = 0;
size_t offset;
int res;
if (i->data_source == ITER_DEST)
gup_flags |= FOLL_WRITE;
if (extraction_flags & ITER_ALLOW_P2PDMA)
gup_flags |= FOLL_PCI_P2PDMA;
if (i->nofault)
gup_flags |= FOLL_NOFAULT;
addr = first_iovec_segment(i, &maxsize);
*offset0 = offset = addr % PAGE_SIZE;
addr &= PAGE_MASK;
maxpages = want_pages_array(pages, maxsize, offset, maxpages);
if (!maxpages)
return -ENOMEM;
res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages);
if (unlikely(res <= 0))
return res;
maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset);
iov_iter_advance(i, maxsize);
return maxsize;
}
/**
* iov_iter_extract_pages - Extract a list of contiguous pages from an iterator
* @i: The iterator to extract from
* @pages: Where to return the list of pages
* @maxsize: The maximum amount of iterator to extract
* @maxpages: The maximum size of the list of pages
* @extraction_flags: Flags to qualify request
* @offset0: Where to return the starting offset into (*@pages)[0]
*
* Extract a list of contiguous pages from the current point of the iterator,
* advancing the iterator. The maximum number of pages and the maximum amount
* of page contents can be set.
*
* If *@pages is NULL, a page list will be allocated to the required size and
* *@pages will be set to its base. If *@pages is not NULL, it will be assumed
* that the caller allocated a page list at least @maxpages in size and this
* will be filled in.
*
* @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA
* be allowed on the pages extracted.
*
* The iov_iter_extract_will_pin() function can be used to query how cleanup
* should be performed.
*
* Extra refs or pins on the pages may be obtained as follows:
*
* (*) If the iterator is user-backed (ITER_IOVEC/ITER_UBUF), pins will be
* added to the pages, but refs will not be taken.
* iov_iter_extract_will_pin() will return true.
*
* (*) If the iterator is ITER_KVEC, ITER_BVEC or ITER_XARRAY, the pages are
* merely listed; no extra refs or pins are obtained.
* iov_iter_extract_will_pin() will return 0.
*
* Note also:
*
* (*) Use with ITER_DISCARD is not supported as that has no content.
*
* On success, the function sets *@pages to the new pagelist, if allocated, and
* sets *offset0 to the offset into the first page.
*
* It may also return -ENOMEM and -EFAULT.
*/
ssize_t iov_iter_extract_pages(struct iov_iter *i,
struct page ***pages,
size_t maxsize,
unsigned int maxpages,
iov_iter_extraction_t extraction_flags,
size_t *offset0)
{
maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);
if (!maxsize)
return 0;
if (likely(user_backed_iter(i)))
return iov_iter_extract_user_pages(i, pages, maxsize,
maxpages, extraction_flags,
offset0);
if (iov_iter_is_kvec(i))
return iov_iter_extract_kvec_pages(i, pages, maxsize,
maxpages, extraction_flags,
offset0);
if (iov_iter_is_bvec(i))
return iov_iter_extract_bvec_pages(i, pages, maxsize,
maxpages, extraction_flags,
offset0);
if (iov_iter_is_xarray(i))
return iov_iter_extract_xarray_pages(i, pages, maxsize,
maxpages, extraction_flags,
offset0);
return -EFAULT;
}
EXPORT_SYMBOL_GPL(iov_iter_extract_pages);
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/mm/swapfile.c
*
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
* Swap reorganised 29.12.95, Stephen Tweedie
*/
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/slab.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/namei.h>
#include <linux/shmem_fs.h>
#include <linux/blk-cgroup.h>
#include <linux/random.h>
#include <linux/writeback.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/init.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/security.h>
#include <linux/backing-dev.h>
#include <linux/mutex.h>
#include <linux/capability.h>
#include <linux/syscalls.h>
#include <linux/memcontrol.h>
#include <linux/poll.h>
#include <linux/oom.h>
#include <linux/swapfile.h>
#include <linux/export.h>
#include <linux/swap_slots.h>
#include <linux/sort.h>
#include <linux/completion.h>
#include <linux/suspend.h>
#include <linux/zswap.h>
#include <linux/plist.h>
#include <asm/tlbflush.h>
#include <linux/swapops.h>
#include <linux/swap_cgroup.h>
#include "internal.h"
#include "swap.h"
static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
unsigned char);
static void free_swap_count_continuations(struct swap_info_struct *);
static DEFINE_SPINLOCK(swap_lock);
static unsigned int nr_swapfiles;
atomic_long_t nr_swap_pages;
/*
* Some modules use swappable objects and may try to swap them out under
* memory pressure (via the shrinker). Before doing so, they may wish to
* check to see if any swap space is available.
*/
EXPORT_SYMBOL_GPL(nr_swap_pages);
/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
long total_swap_pages;
static int least_priority = -1;
unsigned long swapfile_maximum_size;
#ifdef CONFIG_MIGRATION
bool swap_migration_ad_supported;
#endif /* CONFIG_MIGRATION */
static const char Bad_file[] = "Bad swap file entry ";
static const char Unused_file[] = "Unused swap file entry ";
static const char Bad_offset[] = "Bad swap offset entry ";
static const char Unused_offset[] = "Unused swap offset entry ";
/*
* all active swap_info_structs
* protected with swap_lock, and ordered by priority.
*/
static PLIST_HEAD(swap_active_head);
/*
* all available (active, not full) swap_info_structs
* protected with swap_avail_lock, ordered by priority.
* This is used by folio_alloc_swap() instead of swap_active_head
* because swap_active_head includes all swap_info_structs,
* but folio_alloc_swap() doesn't need to look at full ones.
* This uses its own lock instead of swap_lock because when a
* swap_info_struct changes between not-full/full, it needs to
* add/remove itself to/from this list, but the swap_info_struct->lock
* is held and the locking order requires swap_lock to be taken
* before any swap_info_struct->lock.
*/
static struct plist_head *swap_avail_heads;
static DEFINE_SPINLOCK(swap_avail_lock);
static struct swap_info_struct *swap_info[MAX_SWAPFILES];
static DEFINE_MUTEX(swapon_mutex);
static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
/* Activity counter to indicate that a swapon or swapoff has occurred */
static atomic_t proc_poll_event = ATOMIC_INIT(0);
atomic_t nr_rotate_swap = ATOMIC_INIT(0);
static struct swap_info_struct *swap_type_to_swap_info(int type)
{
if (type >= MAX_SWAPFILES)
return NULL;
return READ_ONCE(swap_info[type]); /* rcu_dereference() */
}
static inline unsigned char swap_count(unsigned char ent)
{
return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */
}
/* Reclaim the swap entry anyway if possible */
#define TTRS_ANYWAY 0x1
/*
* Reclaim the swap entry if there are no more mappings of the
* corresponding page
*/
#define TTRS_UNMAPPED 0x2
/* Reclaim the swap entry if swap is getting full*/
#define TTRS_FULL 0x4
/* returns 1 if swap entry is freed */
static int __try_to_reclaim_swap(struct swap_info_struct *si,
unsigned long offset, unsigned long flags)
{
swp_entry_t entry = swp_entry(si->type, offset);
struct folio *folio;
int ret = 0;
folio = filemap_get_folio(swap_address_space(entry), offset);
if (IS_ERR(folio))
return 0;
/*
* When this function is called from scan_swap_map_slots() and it's
* called by vmscan.c at reclaiming folios. So we hold a folio lock
* here. We have to use trylock for avoiding deadlock. This is a special
* case and you should use folio_free_swap() with explicit folio_lock()
* in usual operations.
*/
if (folio_trylock(folio)) {
if ((flags & TTRS_ANYWAY) ||
((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) ||
((flags & TTRS_FULL) && mem_cgroup_swap_full(folio)))
ret = folio_free_swap(folio);
folio_unlock(folio);
}
folio_put(folio);
return ret;
}
static inline struct swap_extent *first_se(struct swap_info_struct *sis)
{
struct rb_node *rb = rb_first(&sis->swap_extent_root);
return rb_entry(rb, struct swap_extent, rb_node);
}
static inline struct swap_extent *next_se(struct swap_extent *se)
{
struct rb_node *rb = rb_next(&se->rb_node);
return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL;
}
/*
* swapon tell device that all the old swap contents can be discarded,
* to allow the swap device to optimize its wear-levelling.
*/
static int discard_swap(struct swap_info_struct *si)
{
struct swap_extent *se;
sector_t start_block;
sector_t nr_blocks;
int err = 0;
/* Do not discard the swap header page! */
se = first_se(si);
start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
if (nr_blocks) {
err = blkdev_issue_discard(si->bdev, start_block,
nr_blocks, GFP_KERNEL);
if (err)
return err;
cond_resched();
}
for (se = next_se(se); se; se = next_se(se)) {
start_block = se->start_block << (PAGE_SHIFT - 9);
nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
err = blkdev_issue_discard(si->bdev, start_block,
nr_blocks, GFP_KERNEL);
if (err)
break;
cond_resched();
}
return err; /* That will often be -EOPNOTSUPP */
}
static struct swap_extent *
offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)
{
struct swap_extent *se;
struct rb_node *rb;
rb = sis->swap_extent_root.rb_node;
while (rb) {
se = rb_entry(rb, struct swap_extent, rb_node);
if (offset < se->start_page)
rb = rb->rb_left;
else if (offset >= se->start_page + se->nr_pages)
rb = rb->rb_right;
else
return se;
}
/* It *must* be present */
BUG();
}
sector_t swap_folio_sector(struct folio *folio)
{
struct swap_info_struct *sis = swp_swap_info(folio->swap);
struct swap_extent *se;
sector_t sector;
pgoff_t offset;
offset = swp_offset(folio->swap);
se = offset_to_swap_extent(sis, offset);
sector = se->start_block + (offset - se->start_page);
return sector << (PAGE_SHIFT - 9);
}
/*
* swap allocation tell device that a cluster of swap can now be discarded,
* to allow the swap device to optimize its wear-levelling.
*/
static void discard_swap_cluster(struct swap_info_struct *si,
pgoff_t start_page, pgoff_t nr_pages)
{
struct swap_extent *se = offset_to_swap_extent(si, start_page);
while (nr_pages) {
pgoff_t offset = start_page - se->start_page;
sector_t start_block = se->start_block + offset;
sector_t nr_blocks = se->nr_pages - offset;
if (nr_blocks > nr_pages)
nr_blocks = nr_pages;
start_page += nr_blocks;
nr_pages -= nr_blocks;
start_block <<= PAGE_SHIFT - 9;
nr_blocks <<= PAGE_SHIFT - 9;
if (blkdev_issue_discard(si->bdev, start_block,
nr_blocks, GFP_NOIO))
break;
se = next_se(se);
}
}
#ifdef CONFIG_THP_SWAP
#define SWAPFILE_CLUSTER HPAGE_PMD_NR
#define swap_entry_size(size) (size)
#else
#define SWAPFILE_CLUSTER 256
/*
* Define swap_entry_size() as constant to let compiler to optimize
* out some code if !CONFIG_THP_SWAP
*/
#define swap_entry_size(size) 1
#endif
#define LATENCY_LIMIT 256
static inline void cluster_set_flag(struct swap_cluster_info *info,
unsigned int flag)
{
info->flags = flag;
}
static inline unsigned int cluster_count(struct swap_cluster_info *info)
{
return info->data;
}
static inline void cluster_set_count(struct swap_cluster_info *info,
unsigned int c)
{
info->data = c;
}
static inline void cluster_set_count_flag(struct swap_cluster_info *info,
unsigned int c, unsigned int f)
{
info->flags = f;
info->data = c;
}
static inline unsigned int cluster_next(struct swap_cluster_info *info)
{
return info->data;
}
static inline void cluster_set_next(struct swap_cluster_info *info,
unsigned int n)
{
info->data = n;
}
static inline void cluster_set_next_flag(struct swap_cluster_info *info,
unsigned int n, unsigned int f)
{
info->flags = f;
info->data = n;
}
static inline bool cluster_is_free(struct swap_cluster_info *info)
{
return info->flags & CLUSTER_FLAG_FREE;
}
static inline bool cluster_is_null(struct swap_cluster_info *info)
{
return info->flags & CLUSTER_FLAG_NEXT_NULL;
}
static inline void cluster_set_null(struct swap_cluster_info *info)
{
info->flags = CLUSTER_FLAG_NEXT_NULL;
info->data = 0;
}
static inline bool cluster_is_huge(struct swap_cluster_info *info)
{
if (IS_ENABLED(CONFIG_THP_SWAP))
return info->flags & CLUSTER_FLAG_HUGE;
return false;
}
static inline void cluster_clear_huge(struct swap_cluster_info *info)
{
info->flags &= ~CLUSTER_FLAG_HUGE;
}
static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si,
unsigned long offset)
{
struct swap_cluster_info *ci;
ci = si->cluster_info;
if (ci) {
ci += offset / SWAPFILE_CLUSTER;
spin_lock(&ci->lock);
}
return ci;
}
static inline void unlock_cluster(struct swap_cluster_info *ci)
{
if (ci)
spin_unlock(&ci->lock);
}
/*
* Determine the locking method in use for this device. Return
* swap_cluster_info if SSD-style cluster-based locking is in place.
*/
static inline struct swap_cluster_info *lock_cluster_or_swap_info(
struct swap_info_struct *si, unsigned long offset)
{
struct swap_cluster_info *ci;
/* Try to use fine-grained SSD-style locking if available: */
ci = lock_cluster(si, offset);
/* Otherwise, fall back to traditional, coarse locking: */
if (!ci)
spin_lock(&si->lock);
return ci;
}
static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si,
struct swap_cluster_info *ci)
{
if (ci)
unlock_cluster(ci);
else
spin_unlock(&si->lock);
}
static inline bool cluster_list_empty(struct swap_cluster_list *list)
{
return cluster_is_null(&list->head);
}
static inline unsigned int cluster_list_first(struct swap_cluster_list *list)
{
return cluster_next(&list->head);
}
static void cluster_list_init(struct swap_cluster_list *list)
{
cluster_set_null(&list->head);
cluster_set_null(&list->tail);
}
static void cluster_list_add_tail(struct swap_cluster_list *list,
struct swap_cluster_info *ci,
unsigned int idx)
{
if (cluster_list_empty(list)) {
cluster_set_next_flag(&list->head, idx, 0);
cluster_set_next_flag(&list->tail, idx, 0);
} else {
struct swap_cluster_info *ci_tail;
unsigned int tail = cluster_next(&list->tail);
/*
* Nested cluster lock, but both cluster locks are
* only acquired when we held swap_info_struct->lock
*/
ci_tail = ci + tail;
spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING);
cluster_set_next(ci_tail, idx);
spin_unlock(&ci_tail->lock);
cluster_set_next_flag(&list->tail, idx, 0);
}
}
static unsigned int cluster_list_del_first(struct swap_cluster_list *list,
struct swap_cluster_info *ci)
{
unsigned int idx;
idx = cluster_next(&list->head);
if (cluster_next(&list->tail) == idx) {
cluster_set_null(&list->head);
cluster_set_null(&list->tail);
} else
cluster_set_next_flag(&list->head,
cluster_next(&ci[idx]), 0);
return idx;
}
/* Add a cluster to discard list and schedule it to do discard */
static void swap_cluster_schedule_discard(struct swap_info_struct *si,
unsigned int idx)
{
/*
* If scan_swap_map_slots() can't find a free cluster, it will check
* si->swap_map directly. To make sure the discarding cluster isn't
* taken by scan_swap_map_slots(), mark the swap entries bad (occupied).
* It will be cleared after discard
*/
memset(si->swap_map + idx * SWAPFILE_CLUSTER,
SWAP_MAP_BAD, SWAPFILE_CLUSTER);
cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx);
schedule_work(&si->discard_work);
}
static void __free_cluster(struct swap_info_struct *si, unsigned long idx)
{
struct swap_cluster_info *ci = si->cluster_info;
cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE);
cluster_list_add_tail(&si->free_clusters, ci, idx);
}
/*
* Doing discard actually. After a cluster discard is finished, the cluster
* will be added to free cluster list. caller should hold si->lock.
*/
static void swap_do_scheduled_discard(struct swap_info_struct *si)
{
struct swap_cluster_info *info, *ci;
unsigned int idx;
info = si->cluster_info;
while (!cluster_list_empty(&si->discard_clusters)) {
idx = cluster_list_del_first(&si->discard_clusters, info);
spin_unlock(&si->lock);
discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
SWAPFILE_CLUSTER);
spin_lock(&si->lock);
ci = lock_cluster(si, idx * SWAPFILE_CLUSTER);
__free_cluster(si, idx);
memset(si->swap_map + idx * SWAPFILE_CLUSTER,
0, SWAPFILE_CLUSTER);
unlock_cluster(ci);
}
}
static void swap_discard_work(struct work_struct *work)
{
struct swap_info_struct *si;
si = container_of(work, struct swap_info_struct, discard_work);
spin_lock(&si->lock);
swap_do_scheduled_discard(si);
spin_unlock(&si->lock);
}
static void swap_users_ref_free(struct percpu_ref *ref)
{
struct swap_info_struct *si;
si = container_of(ref, struct swap_info_struct, users);
complete(&si->comp);
}
static void alloc_cluster(struct swap_info_struct *si, unsigned long idx)
{
struct swap_cluster_info *ci = si->cluster_info;
VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx);
cluster_list_del_first(&si->free_clusters, ci);
cluster_set_count_flag(ci + idx, 0, 0);
}
static void free_cluster(struct swap_info_struct *si, unsigned long idx)
{
struct swap_cluster_info *ci = si->cluster_info + idx;
VM_BUG_ON(cluster_count(ci) != 0);
/*
* If the swap is discardable, prepare discard the cluster
* instead of free it immediately. The cluster will be freed
* after discard.
*/
if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
(SWP_WRITEOK | SWP_PAGE_DISCARD)) {
swap_cluster_schedule_discard(si, idx);
return;
}
__free_cluster(si, idx);
}
/*
* The cluster corresponding to page_nr will be used. The cluster will be
* removed from free cluster list and its usage counter will be increased.
*/
static void inc_cluster_info_page(struct swap_info_struct *p,
struct swap_cluster_info *cluster_info, unsigned long page_nr)
{
unsigned long idx = page_nr / SWAPFILE_CLUSTER;
if (!cluster_info)
return;
if (cluster_is_free(&cluster_info[idx]))
alloc_cluster(p, idx);
VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
cluster_set_count(&cluster_info[idx],
cluster_count(&cluster_info[idx]) + 1);
}
/*
* The cluster corresponding to page_nr decreases one usage. If the usage
* counter becomes 0, which means no page in the cluster is in using, we can
* optionally discard the cluster and add it to free cluster list.
*/
static void dec_cluster_info_page(struct swap_info_struct *p,
struct swap_cluster_info *cluster_info, unsigned long page_nr)
{
unsigned long idx = page_nr / SWAPFILE_CLUSTER;
if (!cluster_info)
return;
VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
cluster_set_count(&cluster_info[idx],
cluster_count(&cluster_info[idx]) - 1);
if (cluster_count(&cluster_info[idx]) == 0)
free_cluster(p, idx);
}
/*
* It's possible scan_swap_map_slots() uses a free cluster in the middle of free
* cluster list. Avoiding such abuse to avoid list corruption.
*/
static bool
scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
unsigned long offset)
{
struct percpu_cluster *percpu_cluster;
bool conflict;
offset /= SWAPFILE_CLUSTER;
conflict = !cluster_list_empty(&si->free_clusters) &&
offset != cluster_list_first(&si->free_clusters) &&
cluster_is_free(&si->cluster_info[offset]);
if (!conflict)
return false;
percpu_cluster = this_cpu_ptr(si->percpu_cluster);
cluster_set_null(&percpu_cluster->index);
return true;
}
/*
* Try to get a swap entry from current cpu's swap entry pool (a cluster). This
* might involve allocating a new cluster for current CPU too.
*/
static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
unsigned long *offset, unsigned long *scan_base)
{
struct percpu_cluster *cluster;
struct swap_cluster_info *ci;
unsigned long tmp, max;
new_cluster:
cluster = this_cpu_ptr(si->percpu_cluster);
if (cluster_is_null(&cluster->index)) {
if (!cluster_list_empty(&si->free_clusters)) {
cluster->index = si->free_clusters.head;
cluster->next = cluster_next(&cluster->index) *
SWAPFILE_CLUSTER;
} else if (!cluster_list_empty(&si->discard_clusters)) {
/*
* we don't have free cluster but have some clusters in
* discarding, do discard now and reclaim them, then
* reread cluster_next_cpu since we dropped si->lock
*/
swap_do_scheduled_discard(si);
*scan_base = this_cpu_read(*si->cluster_next_cpu);
*offset = *scan_base;
goto new_cluster;
} else
return false;
}
/*
* Other CPUs can use our cluster if they can't find a free cluster,
* check if there is still free entry in the cluster
*/
tmp = cluster->next;
max = min_t(unsigned long, si->max,
(cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER);
if (tmp < max) {
ci = lock_cluster(si, tmp);
while (tmp < max) {
if (!si->swap_map[tmp])
break;
tmp++;
}
unlock_cluster(ci);
}
if (tmp >= max) {
cluster_set_null(&cluster->index);
goto new_cluster;
}
cluster->next = tmp + 1;
*offset = tmp;
*scan_base = tmp;
return true;
}
static void __del_from_avail_list(struct swap_info_struct *p)
{
int nid;
assert_spin_locked(&p->lock);
for_each_node(nid)
plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]);
}
static void del_from_avail_list(struct swap_info_struct *p)
{
spin_lock(&swap_avail_lock);
__del_from_avail_list(p);
spin_unlock(&swap_avail_lock);
}
static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset,
unsigned int nr_entries)
{
unsigned int end = offset + nr_entries - 1;
if (offset == si->lowest_bit)
si->lowest_bit += nr_entries;
if (end == si->highest_bit)
WRITE_ONCE(si->highest_bit, si->highest_bit - nr_entries);
WRITE_ONCE(si->inuse_pages, si->inuse_pages + nr_entries);
if (si->inuse_pages == si->pages) {
si->lowest_bit = si->max;
si->highest_bit = 0;
del_from_avail_list(si);
}
}
static void add_to_avail_list(struct swap_info_struct *p)
{
int nid;
spin_lock(&swap_avail_lock);
for_each_node(nid)
plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]);
spin_unlock(&swap_avail_lock);
}
static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
unsigned int nr_entries)
{
unsigned long begin = offset;
unsigned long end = offset + nr_entries - 1;
void (*swap_slot_free_notify)(struct block_device *, unsigned long);
if (offset < si->lowest_bit)
si->lowest_bit = offset;
if (end > si->highest_bit) {
bool was_full = !si->highest_bit;
WRITE_ONCE(si->highest_bit, end);
if (was_full && (si->flags & SWP_WRITEOK))
add_to_avail_list(si);
}
if (si->flags & SWP_BLKDEV)
swap_slot_free_notify =
si->bdev->bd_disk->fops->swap_slot_free_notify;
else
swap_slot_free_notify = NULL;
while (offset <= end) {
arch_swap_invalidate_page(si->type, offset);
if (swap_slot_free_notify)
swap_slot_free_notify(si->bdev, offset);
offset++;
}
clear_shadow_from_swap_cache(si->type, begin, end);
/*
* Make sure that try_to_unuse() observes si->inuse_pages reaching 0
* only after the above cleanups are done.
*/
smp_wmb();
atomic_long_add(nr_entries, &nr_swap_pages);
WRITE_ONCE(si->inuse_pages, si->inuse_pages - nr_entries);
}
static void set_cluster_next(struct swap_info_struct *si, unsigned long next)
{
unsigned long prev;
if (!(si->flags & SWP_SOLIDSTATE)) {
si->cluster_next = next;
return;
}
prev = this_cpu_read(*si->cluster_next_cpu);
/*
* Cross the swap address space size aligned trunk, choose
* another trunk randomly to avoid lock contention on swap
* address space if possible.
*/
if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) !=
(next >> SWAP_ADDRESS_SPACE_SHIFT)) {
/* No free swap slots available */
if (si->highest_bit <= si->lowest_bit)
return;
next = get_random_u32_inclusive(si->lowest_bit, si->highest_bit);
next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES);
next = max_t(unsigned int, next, si->lowest_bit);
}
this_cpu_write(*si->cluster_next_cpu, next);
}
static bool swap_offset_available_and_locked(struct swap_info_struct *si,
unsigned long offset)
{
if (data_race(!si->swap_map[offset])) {
spin_lock(&si->lock);
return true;
}
if (vm_swap_full() && READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) {
spin_lock(&si->lock);
return true;
}
return false;
}
static int scan_swap_map_slots(struct swap_info_struct *si,
unsigned char usage, int nr,
swp_entry_t slots[])
{
struct swap_cluster_info *ci;
unsigned long offset;
unsigned long scan_base;
unsigned long last_in_cluster = 0;
int latency_ration = LATENCY_LIMIT;
int n_ret = 0;
bool scanned_many = false;
/*
* We try to cluster swap pages by allocating them sequentially
* in swap. Once we've allocated SWAPFILE_CLUSTER pages this
* way, however, we resort to first-free allocation, starting
* a new cluster. This prevents us from scattering swap pages
* all over the entire swap partition, so that we reduce
* overall disk seek times between swap pages. -- sct
* But we do now try to find an empty cluster. -Andrea
* And we let swap pages go all over an SSD partition. Hugh
*/
si->flags += SWP_SCANNING;
/*
* Use percpu scan base for SSD to reduce lock contention on
* cluster and swap cache. For HDD, sequential access is more
* important.
*/
if (si->flags & SWP_SOLIDSTATE)
scan_base = this_cpu_read(*si->cluster_next_cpu);
else
scan_base = si->cluster_next;
offset = scan_base;
/* SSD algorithm */
if (si->cluster_info) {
if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
goto scan;
} else if (unlikely(!si->cluster_nr--)) {
if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
si->cluster_nr = SWAPFILE_CLUSTER - 1;
goto checks;
}
spin_unlock(&si->lock);
/*
* If seek is expensive, start searching for new cluster from
* start of partition, to minimize the span of allocated swap.
* If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
* case, just handled by scan_swap_map_try_ssd_cluster() above.
*/
scan_base = offset = si->lowest_bit;
last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
/* Locate the first empty (unaligned) cluster */
for (; last_in_cluster <= si->highest_bit; offset++) {
if (si->swap_map[offset])
last_in_cluster = offset + SWAPFILE_CLUSTER;
else if (offset == last_in_cluster) {
spin_lock(&si->lock);
offset -= SWAPFILE_CLUSTER - 1;
si->cluster_next = offset;
si->cluster_nr = SWAPFILE_CLUSTER - 1;
goto checks;
}
if (unlikely(--latency_ration < 0)) {
cond_resched();
latency_ration = LATENCY_LIMIT;
}
}
offset = scan_base;
spin_lock(&si->lock);
si->cluster_nr = SWAPFILE_CLUSTER - 1;
}
checks:
if (si->cluster_info) {
while (scan_swap_map_ssd_cluster_conflict(si, offset)) {
/* take a break if we already got some slots */
if (n_ret)
goto done;
if (!scan_swap_map_try_ssd_cluster(si, &offset,
&scan_base))
goto scan;
}
}
if (!(si->flags & SWP_WRITEOK))
goto no_page;
if (!si->highest_bit)
goto no_page;
if (offset > si->highest_bit)
scan_base = offset = si->lowest_bit;
ci = lock_cluster(si, offset);
/* reuse swap entry of cache-only swap if not busy. */
if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
int swap_was_freed;
unlock_cluster(ci);
spin_unlock(&si->lock);
swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
spin_lock(&si->lock);
/* entry was freed successfully, try to use this again */
if (swap_was_freed)
goto checks;
goto scan; /* check next one */
}
if (si->swap_map[offset]) {
unlock_cluster(ci);
if (!n_ret)
goto scan;
else
goto done;
}
WRITE_ONCE(si->swap_map[offset], usage);
inc_cluster_info_page(si, si->cluster_info, offset);
unlock_cluster(ci);
swap_range_alloc(si, offset, 1);
slots[n_ret++] = swp_entry(si->type, offset);
/* got enough slots or reach max slots? */
if ((n_ret == nr) || (offset >= si->highest_bit))
goto done;
/* search for next available slot */
/* time to take a break? */
if (unlikely(--latency_ration < 0)) {
if (n_ret)
goto done;
spin_unlock(&si->lock);
cond_resched();
spin_lock(&si->lock);
latency_ration = LATENCY_LIMIT;
}
/* try to get more slots in cluster */
if (si->cluster_info) {
if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
goto checks;
} else if (si->cluster_nr && !si->swap_map[++offset]) {
/* non-ssd case, still more slots in cluster? */
--si->cluster_nr;
goto checks;
}
/*
* Even if there's no free clusters available (fragmented),
* try to scan a little more quickly with lock held unless we
* have scanned too many slots already.
*/
if (!scanned_many) {
unsigned long scan_limit;
if (offset < scan_base)
scan_limit = scan_base;
else
scan_limit = si->highest_bit;
for (; offset <= scan_limit && --latency_ration > 0;
offset++) {
if (!si->swap_map[offset])
goto checks;
}
}
done:
set_cluster_next(si, offset + 1);
si->flags -= SWP_SCANNING;
return n_ret;
scan:
spin_unlock(&si->lock);
while (++offset <= READ_ONCE(si->highest_bit)) {
if (unlikely(--latency_ration < 0)) {
cond_resched();
latency_ration = LATENCY_LIMIT;
scanned_many = true;
}
if (swap_offset_available_and_locked(si, offset))
goto checks;
}
offset = si->lowest_bit;
while (offset < scan_base) {
if (unlikely(--latency_ration < 0)) {
cond_resched();
latency_ration = LATENCY_LIMIT;
scanned_many = true;
}
if (swap_offset_available_and_locked(si, offset))
goto checks;
offset++;
}
spin_lock(&si->lock);
no_page:
si->flags -= SWP_SCANNING;
return n_ret;
}
static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot)
{
unsigned long idx;
struct swap_cluster_info *ci;
unsigned long offset;
/*
* Should not even be attempting cluster allocations when huge
* page swap is disabled. Warn and fail the allocation.
*/
if (!IS_ENABLED(CONFIG_THP_SWAP)) {
VM_WARN_ON_ONCE(1);
return 0;
}
if (cluster_list_empty(&si->free_clusters))
return 0;
idx = cluster_list_first(&si->free_clusters);
offset = idx * SWAPFILE_CLUSTER;
ci = lock_cluster(si, offset);
alloc_cluster(si, idx);
cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE);
memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER);
unlock_cluster(ci);
swap_range_alloc(si, offset, SWAPFILE_CLUSTER);
*slot = swp_entry(si->type, offset);
return 1;
}
static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx)
{
unsigned long offset = idx * SWAPFILE_CLUSTER;
struct swap_cluster_info *ci;
ci = lock_cluster(si, offset);
memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER);
cluster_set_count_flag(ci, 0, 0);
free_cluster(si, idx);
unlock_cluster(ci);
swap_range_free(si, offset, SWAPFILE_CLUSTER);
}
int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size)
{
unsigned long size = swap_entry_size(entry_size);
struct swap_info_struct *si, *next;
long avail_pgs;
int n_ret = 0;
int node;
/* Only single cluster request supported */
WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER);
spin_lock(&swap_avail_lock);
avail_pgs = atomic_long_read(&nr_swap_pages) / size;
if (avail_pgs <= 0) {
spin_unlock(&swap_avail_lock);
goto noswap;
}
n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs);
atomic_long_sub(n_goal * size, &nr_swap_pages);
start_over:
node = numa_node_id();
plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) {
/* requeue si to after same-priority siblings */
plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]);
spin_unlock(&swap_avail_lock);
spin_lock(&si->lock);
if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
spin_lock(&swap_avail_lock);
if (plist_node_empty(&si->avail_lists[node])) {
spin_unlock(&si->lock);
goto nextsi;
}
WARN(!si->highest_bit,
"swap_info %d in list but !highest_bit\n",
si->type);
WARN(!(si->flags & SWP_WRITEOK),
"swap_info %d in list but !SWP_WRITEOK\n",
si->type);
__del_from_avail_list(si);
spin_unlock(&si->lock);
goto nextsi;
}
if (size == SWAPFILE_CLUSTER) {
if (si->flags & SWP_BLKDEV)
n_ret = swap_alloc_cluster(si, swp_entries);
} else
n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE,
n_goal, swp_entries);
spin_unlock(&si->lock);
if (n_ret || size == SWAPFILE_CLUSTER)
goto check_out;
cond_resched();
spin_lock(&swap_avail_lock);
nextsi:
/*
* if we got here, it's likely that si was almost full before,
* and since scan_swap_map_slots() can drop the si->lock,
* multiple callers probably all tried to get a page from the
* same si and it filled up before we could get one; or, the si
* filled up between us dropping swap_avail_lock and taking
* si->lock. Since we dropped the swap_avail_lock, the
* swap_avail_head list may have been modified; so if next is
* still in the swap_avail_head list then try it, otherwise
* start over if we have not gotten any slots.
*/
if (plist_node_empty(&next->avail_lists[node]))
goto start_over;
}
spin_unlock(&swap_avail_lock);
check_out:
if (n_ret < n_goal)
atomic_long_add((long)(n_goal - n_ret) * size,
&nr_swap_pages);
noswap:
return n_ret;
}
static struct swap_info_struct *_swap_info_get(swp_entry_t entry)
{
struct swap_info_struct *p;
unsigned long offset;
if (!entry.val)
goto out;
p = swp_swap_info(entry);
if (!p)
goto bad_nofile;
if (data_race(!(p->flags & SWP_USED)))
goto bad_device;
offset = swp_offset(entry);
if (offset >= p->max)
goto bad_offset;
if (data_race(!p->swap_map[swp_offset(entry)]))
goto bad_free;
return p;
bad_free:
pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val);
goto out;
bad_offset:
pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
goto out;
bad_device:
pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val);
goto out;
bad_nofile:
pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
out:
return NULL;
}
static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry,
struct swap_info_struct *q)
{
struct swap_info_struct *p;
p = _swap_info_get(entry);
if (p != q) {
if (q != NULL)
spin_unlock(&q->lock);
if (p != NULL)
spin_lock(&p->lock);
}
return p;
}
static unsigned char __swap_entry_free_locked(struct swap_info_struct *p,
unsigned long offset,
unsigned char usage)
{
unsigned char count;
unsigned char has_cache;
count = p->swap_map[offset];
has_cache = count & SWAP_HAS_CACHE;
count &= ~SWAP_HAS_CACHE;
if (usage == SWAP_HAS_CACHE) {
VM_BUG_ON(!has_cache);
has_cache = 0;
} else if (count == SWAP_MAP_SHMEM) {
/*
* Or we could insist on shmem.c using a special
* swap_shmem_free() and free_shmem_swap_and_cache()...
*/
count = 0;
} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
if (count == COUNT_CONTINUED) {
if (swap_count_continued(p, offset, count))
count = SWAP_MAP_MAX | COUNT_CONTINUED;
else
count = SWAP_MAP_MAX;
} else
count--;
}
usage = count | has_cache;
if (usage)
WRITE_ONCE(p->swap_map[offset], usage);
else
WRITE_ONCE(p->swap_map[offset], SWAP_HAS_CACHE);
return usage;
}
/*
* When we get a swap entry, if there aren't some other ways to
* prevent swapoff, such as the folio in swap cache is locked, page
* table lock is held, etc., the swap entry may become invalid because
* of swapoff. Then, we need to enclose all swap related functions
* with get_swap_device() and put_swap_device(), unless the swap
* functions call get/put_swap_device() by themselves.
*
* Note that when only holding the PTL, swapoff might succeed immediately
* after freeing a swap entry. Therefore, immediately after
* __swap_entry_free(), the swap info might become stale and should not
* be touched without a prior get_swap_device().
*
* Check whether swap entry is valid in the swap device. If so,
* return pointer to swap_info_struct, and keep the swap entry valid
* via preventing the swap device from being swapoff, until
* put_swap_device() is called. Otherwise return NULL.
*
* Notice that swapoff or swapoff+swapon can still happen before the
* percpu_ref_tryget_live() in get_swap_device() or after the
* percpu_ref_put() in put_swap_device() if there isn't any other way
* to prevent swapoff. The caller must be prepared for that. For
* example, the following situation is possible.
*
* CPU1 CPU2
* do_swap_page()
* ... swapoff+swapon
* __read_swap_cache_async()
* swapcache_prepare()
* __swap_duplicate()
* // check swap_map
* // verify PTE not changed
*
* In __swap_duplicate(), the swap_map need to be checked before
* changing partly because the specified swap entry may be for another
* swap device which has been swapoff. And in do_swap_page(), after
* the page is read from the swap device, the PTE is verified not
* changed with the page table locked to check whether the swap device
* has been swapoff or swapoff+swapon.
*/
struct swap_info_struct *get_swap_device(swp_entry_t entry)
{
struct swap_info_struct *si;
unsigned long offset;
if (!entry.val)
goto out;
si = swp_swap_info(entry);
if (!si)
goto bad_nofile;
if (!percpu_ref_tryget_live(&si->users))
goto out;
/*
* Guarantee the si->users are checked before accessing other
* fields of swap_info_struct.
*
* Paired with the spin_unlock() after setup_swap_info() in
* enable_swap_info().
*/
smp_rmb();
offset = swp_offset(entry);
if (offset >= si->max)
goto put_out;
return si;
bad_nofile:
pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
out:
return NULL;
put_out:
pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
percpu_ref_put(&si->users);
return NULL;
}
static unsigned char __swap_entry_free(struct swap_info_struct *p,
swp_entry_t entry)
{
struct swap_cluster_info *ci;
unsigned long offset = swp_offset(entry);
unsigned char usage;
ci = lock_cluster_or_swap_info(p, offset);
usage = __swap_entry_free_locked(p, offset, 1);
unlock_cluster_or_swap_info(p, ci);
if (!usage)
free_swap_slot(entry);
return usage;
}
static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry)
{
struct swap_cluster_info *ci;
unsigned long offset = swp_offset(entry);
unsigned char count;
ci = lock_cluster(p, offset);
count = p->swap_map[offset];
VM_BUG_ON(count != SWAP_HAS_CACHE);
p->swap_map[offset] = 0;
dec_cluster_info_page(p, p->cluster_info, offset);
unlock_cluster(ci);
mem_cgroup_uncharge_swap(entry, 1);
swap_range_free(p, offset, 1);
}
/*
* Caller has made sure that the swap device corresponding to entry
* is still around or has not been recycled.
*/
void swap_free(swp_entry_t entry)
{
struct swap_info_struct *p;
p = _swap_info_get(entry);
if (p)
__swap_entry_free(p, entry);
}
/*
* Called after dropping swapcache to decrease refcnt to swap entries.
*/
void put_swap_folio(struct folio *folio, swp_entry_t entry)
{
unsigned long offset = swp_offset(entry);
unsigned long idx = offset / SWAPFILE_CLUSTER;
struct swap_cluster_info *ci;
struct swap_info_struct *si;
unsigned char *map;
unsigned int i, free_entries = 0;
unsigned char val;
int size = swap_entry_size(folio_nr_pages(folio));
si = _swap_info_get(entry);
if (!si)
return;
ci = lock_cluster_or_swap_info(si, offset);
if (size == SWAPFILE_CLUSTER) {
VM_BUG_ON(!cluster_is_huge(ci));
map = si->swap_map + offset;
for (i = 0; i < SWAPFILE_CLUSTER; i++) {
val = map[i];
VM_BUG_ON(!(val & SWAP_HAS_CACHE));
if (val == SWAP_HAS_CACHE)
free_entries++;
}
cluster_clear_huge(ci);
if (free_entries == SWAPFILE_CLUSTER) {
unlock_cluster_or_swap_info(si, ci);
spin_lock(&si->lock);
mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER);
swap_free_cluster(si, idx);
spin_unlock(&si->lock);
return;
}
}
for (i = 0; i < size; i++, entry.val++) {
if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) {
unlock_cluster_or_swap_info(si, ci);
free_swap_slot(entry);
if (i == size - 1)
return;
lock_cluster_or_swap_info(si, offset);
}
}
unlock_cluster_or_swap_info(si, ci);
}
#ifdef CONFIG_THP_SWAP
int split_swap_cluster(swp_entry_t entry)
{
struct swap_info_struct *si;
struct swap_cluster_info *ci;
unsigned long offset = swp_offset(entry);
si = _swap_info_get(entry);
if (!si)
return -EBUSY;
ci = lock_cluster(si, offset);
cluster_clear_huge(ci);
unlock_cluster(ci);
return 0;
}
#endif
static int swp_entry_cmp(const void *ent1, const void *ent2)
{
const swp_entry_t *e1 = ent1, *e2 = ent2;
return (int)swp_type(*e1) - (int)swp_type(*e2);
}
void swapcache_free_entries(swp_entry_t *entries, int n)
{
struct swap_info_struct *p, *prev;
int i;
if (n <= 0)
return;
prev = NULL;
p = NULL;
/*
* Sort swap entries by swap device, so each lock is only taken once.
* nr_swapfiles isn't absolutely correct, but the overhead of sort() is
* so low that it isn't necessary to optimize further.
*/
if (nr_swapfiles > 1)
sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL);
for (i = 0; i < n; ++i) {
p = swap_info_get_cont(entries[i], prev);
if (p)
swap_entry_free(p, entries[i]);
prev = p;
}
if (p)
spin_unlock(&p->lock);
}
int __swap_count(swp_entry_t entry)
{
struct swap_info_struct *si = swp_swap_info(entry);
pgoff_t offset = swp_offset(entry);
return swap_count(si->swap_map[offset]);
}
/*
* How many references to @entry are currently swapped out?
* This does not give an exact answer when swap count is continued,
* but does include the high COUNT_CONTINUED flag to allow for that.
*/
int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
{
pgoff_t offset = swp_offset(entry);
struct swap_cluster_info *ci;
int count;
ci = lock_cluster_or_swap_info(si, offset);
count = swap_count(si->swap_map[offset]);
unlock_cluster_or_swap_info(si, ci);
return count;
}
/*
* How many references to @entry are currently swapped out?
* This considers COUNT_CONTINUED so it returns exact answer.
*/
int swp_swapcount(swp_entry_t entry)
{
int count, tmp_count, n;
struct swap_info_struct *p;
struct swap_cluster_info *ci;
struct page *page;
pgoff_t offset;
unsigned char *map;
p = _swap_info_get(entry);
if (!p)
return 0;
offset = swp_offset(entry);
ci = lock_cluster_or_swap_info(p, offset);
count = swap_count(p->swap_map[offset]);
if (!(count & COUNT_CONTINUED))
goto out;
count &= ~COUNT_CONTINUED;
n = SWAP_MAP_MAX + 1;
page = vmalloc_to_page(p->swap_map + offset);
offset &= ~PAGE_MASK;
VM_BUG_ON(page_private(page) != SWP_CONTINUED);
do {
page = list_next_entry(page, lru);
map = kmap_local_page(page);
tmp_count = map[offset];
kunmap_local(map);
count += (tmp_count & ~COUNT_CONTINUED) * n;
n *= (SWAP_CONT_MAX + 1);
} while (tmp_count & COUNT_CONTINUED);
out:
unlock_cluster_or_swap_info(p, ci);
return count;
}
static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
swp_entry_t entry)
{
struct swap_cluster_info *ci;
unsigned char *map = si->swap_map;
unsigned long roffset = swp_offset(entry);
unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER);
int i;
bool ret = false;
ci = lock_cluster_or_swap_info(si, offset);
if (!ci || !cluster_is_huge(ci)) {
if (swap_count(map[roffset]))
ret = true;
goto unlock_out;
}
for (i = 0; i < SWAPFILE_CLUSTER; i++) {
if (swap_count(map[offset + i])) {
ret = true;
break;
}
}
unlock_out:
unlock_cluster_or_swap_info(si, ci);
return ret;
}
static bool folio_swapped(struct folio *folio)
{
swp_entry_t entry = folio->swap;
struct swap_info_struct *si = _swap_info_get(entry);
if (!si)
return false;
if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio)))
return swap_swapcount(si, entry) != 0;
return swap_page_trans_huge_swapped(si, entry);
}
/**
* folio_free_swap() - Free the swap space used for this folio.
* @folio: The folio to remove.
*
* If swap is getting full, or if there are no more mappings of this folio,
* then call folio_free_swap to free its swap space.
*
* Return: true if we were able to release the swap space.
*/
bool folio_free_swap(struct folio *folio)
{
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (!folio_test_swapcache(folio))
return false;
if (folio_test_writeback(folio))
return false;
if (folio_swapped(folio))
return false;
/*
* Once hibernation has begun to create its image of memory,
* there's a danger that one of the calls to folio_free_swap()
* - most probably a call from __try_to_reclaim_swap() while
* hibernation is allocating its own swap pages for the image,
* but conceivably even a call from memory reclaim - will free
* the swap from a folio which has already been recorded in the
* image as a clean swapcache folio, and then reuse its swap for
* another page of the image. On waking from hibernation, the
* original folio might be freed under memory pressure, then
* later read back in from swap, now with the wrong data.
*
* Hibernation suspends storage while it is writing the image
* to disk so check that here.
*/
if (pm_suspended_storage())
return false;
delete_from_swap_cache(folio);
folio_set_dirty(folio);
return true;
}
/*
* Free the swap entry like above, but also try to
* free the page cache entry if it is the last user.
*/
int free_swap_and_cache(swp_entry_t entry)
{
struct swap_info_struct *p;
unsigned char count;
if (non_swap_entry(entry))
return 1;
p = get_swap_device(entry);
if (p) {
if (WARN_ON(data_race(!p->swap_map[swp_offset(entry)]))) {
put_swap_device(p);
return 0;
}
count = __swap_entry_free(p, entry);
if (count == SWAP_HAS_CACHE &&
!swap_page_trans_huge_swapped(p, entry))
__try_to_reclaim_swap(p, swp_offset(entry),
TTRS_UNMAPPED | TTRS_FULL);
put_swap_device(p);
}
return p != NULL;
}
#ifdef CONFIG_HIBERNATION
swp_entry_t get_swap_page_of_type(int type)
{
struct swap_info_struct *si = swap_type_to_swap_info(type);
swp_entry_t entry = {0};
if (!si)
goto fail;
/* This is called for allocating swap entry, not cache */
spin_lock(&si->lock);
if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry))
atomic_long_dec(&nr_swap_pages);
spin_unlock(&si->lock);
fail:
return entry;
}
/*
* Find the swap type that corresponds to given device (if any).
*
* @offset - number of the PAGE_SIZE-sized block of the device, starting
* from 0, in which the swap header is expected to be located.
*
* This is needed for the suspend to disk (aka swsusp).
*/
int swap_type_of(dev_t device, sector_t offset)
{
int type;
if (!device)
return -1;
spin_lock(&swap_lock);
for (type = 0; type < nr_swapfiles; type++) {
struct swap_info_struct *sis = swap_info[type];
if (!(sis->flags & SWP_WRITEOK))
continue;
if (device == sis->bdev->bd_dev) {
struct swap_extent *se = first_se(sis);
if (se->start_block == offset) {
spin_unlock(&swap_lock);
return type;
}
}
}
spin_unlock(&swap_lock);
return -ENODEV;
}
int find_first_swap(dev_t *device)
{
int type;
spin_lock(&swap_lock);
for (type = 0; type < nr_swapfiles; type++) {
struct swap_info_struct *sis = swap_info[type];
if (!(sis->flags & SWP_WRITEOK))
continue;
*device = sis->bdev->bd_dev;
spin_unlock(&swap_lock);
return type;
}
spin_unlock(&swap_lock);
return -ENODEV;
}
/*
* Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
* corresponding to given index in swap_info (swap type).
*/
sector_t swapdev_block(int type, pgoff_t offset)
{
struct swap_info_struct *si = swap_type_to_swap_info(type);
struct swap_extent *se;
if (!si || !(si->flags & SWP_WRITEOK))
return 0;
se = offset_to_swap_extent(si, offset);
return se->start_block + (offset - se->start_page);
}
/*
* Return either the total number of swap pages of given type, or the number
* of free pages of that type (depending on @free)
*
* This is needed for software suspend
*/
unsigned int count_swap_pages(int type, int free)
{
unsigned int n = 0;
spin_lock(&swap_lock);
if ((unsigned int)type < nr_swapfiles) {
struct swap_info_struct *sis = swap_info[type];
spin_lock(&sis->lock);
if (sis->flags & SWP_WRITEOK) {
n = sis->pages;
if (free)
n -= sis->inuse_pages;
}
spin_unlock(&sis->lock);
}
spin_unlock(&swap_lock);
return n;
}
#endif /* CONFIG_HIBERNATION */
static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
{
return pte_same(pte_swp_clear_flags(pte), swp_pte);
}
/*
* No need to decide whether this PTE shares the swap entry with others,
* just let do_wp_page work it out if a write is requested later - to
* force COW, vm_page_prot omits write permission from any private vma.
*/
static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, swp_entry_t entry, struct folio *folio)
{
struct page *page;
struct folio *swapcache;
spinlock_t *ptl;
pte_t *pte, new_pte, old_pte;
bool hwpoisoned = false;
int ret = 1;
swapcache = folio;
folio = ksm_might_need_to_copy(folio, vma, addr);
if (unlikely(!folio))
return -ENOMEM;
else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
hwpoisoned = true;
folio = swapcache;
}
page = folio_file_page(folio, swp_offset(entry));
if (PageHWPoison(page))
hwpoisoned = true;
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
if (unlikely(!pte || !pte_same_as_swp(ptep_get(pte),
swp_entry_to_pte(entry)))) {
ret = 0;
goto out;
}
old_pte = ptep_get(pte);
if (unlikely(hwpoisoned || !folio_test_uptodate(folio))) {
swp_entry_t swp_entry;
dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
if (hwpoisoned) {
swp_entry = make_hwpoison_entry(page);
} else {
swp_entry = make_poisoned_swp_entry();
}
new_pte = swp_entry_to_pte(swp_entry);
ret = 0;
goto setpte;
}
/*
* Some architectures may have to restore extra metadata to the page
* when reading from swap. This metadata may be indexed by swap entry
* so this must be called before swap_free().
*/
arch_swap_restore(entry, folio);
dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
folio_get(folio);
if (folio == swapcache) {
rmap_t rmap_flags = RMAP_NONE;
/*
* See do_swap_page(): writeback would be problematic.
* However, we do a folio_wait_writeback() just before this
* call and have the folio locked.
*/
VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
if (pte_swp_exclusive(old_pte))
rmap_flags |= RMAP_EXCLUSIVE;
folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags);
} else { /* ksm created a completely new copy */
folio_add_new_anon_rmap(folio, vma, addr);
folio_add_lru_vma(folio, vma);
}
new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot));
if (pte_swp_soft_dirty(old_pte))
new_pte = pte_mksoft_dirty(new_pte);
if (pte_swp_uffd_wp(old_pte))
new_pte = pte_mkuffd_wp(new_pte);
setpte:
set_pte_at(vma->vm_mm, addr, pte, new_pte);
swap_free(entry);
out:
if (pte)
pte_unmap_unlock(pte, ptl);
if (folio != swapcache) {
folio_unlock(folio);
folio_put(folio);
}
return ret;
}
static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end,
unsigned int type)
{
pte_t *pte = NULL;
struct swap_info_struct *si;
si = swap_info[type];
do {
struct folio *folio;
unsigned long offset;
unsigned char swp_count;
swp_entry_t entry;
int ret;
pte_t ptent;
if (!pte++) {
pte = pte_offset_map(pmd, addr);
if (!pte)
break;
}
ptent = ptep_get_lockless(pte);
if (!is_swap_pte(ptent))
continue;
entry = pte_to_swp_entry(ptent);
if (swp_type(entry) != type)
continue;
offset = swp_offset(entry);
pte_unmap(pte);
pte = NULL;
folio = swap_cache_get_folio(entry, vma, addr);
if (!folio) {
struct page *page;
struct vm_fault vmf = {
.vma = vma,
.address = addr,
.real_address = addr,
.pmd = pmd,
};
page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
&vmf);
if (page)
folio = page_folio(page);
}
if (!folio) {
swp_count = READ_ONCE(si->swap_map[offset]);
if (swp_count == 0 || swp_count == SWAP_MAP_BAD)
continue;
return -ENOMEM;
}
folio_lock(folio);
folio_wait_writeback(folio);
ret = unuse_pte(vma, pmd, addr, entry, folio);
if (ret < 0) {
folio_unlock(folio);
folio_put(folio);
return ret;
}
folio_free_swap(folio);
folio_unlock(folio);
folio_put(folio);
} while (addr += PAGE_SIZE, addr != end);
if (pte)
pte_unmap(pte);
return 0;
}
static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
unsigned long addr, unsigned long end,
unsigned int type)
{
pmd_t *pmd;
unsigned long next;
int ret;
pmd = pmd_offset(pud, addr);
do {
cond_resched();
next = pmd_addr_end(addr, end);
ret = unuse_pte_range(vma, pmd, addr, next, type);
if (ret)
return ret;
} while (pmd++, addr = next, addr != end);
return 0;
}
static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned int type)
{
pud_t *pud;
unsigned long next;
int ret;
pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
continue;
ret = unuse_pmd_range(vma, pud, addr, next, type);
if (ret)
return ret;
} while (pud++, addr = next, addr != end);
return 0;
}
static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
unsigned long addr, unsigned long end,
unsigned int type)
{
p4d_t *p4d;
unsigned long next;
int ret;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (p4d_none_or_clear_bad(p4d))
continue;
ret = unuse_pud_range(vma, p4d, addr, next, type);
if (ret)
return ret;
} while (p4d++, addr = next, addr != end);
return 0;
}
static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
{
pgd_t *pgd;
unsigned long addr, end, next;
int ret;
addr = vma->vm_start;
end = vma->vm_end;
pgd = pgd_offset(vma->vm_mm, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
ret = unuse_p4d_range(vma, pgd, addr, next, type);
if (ret)
return ret;
} while (pgd++, addr = next, addr != end);
return 0;
}
static int unuse_mm(struct mm_struct *mm, unsigned int type)
{
struct vm_area_struct *vma;
int ret = 0;
VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
for_each_vma(vmi, vma) {
if (vma->anon_vma) {
ret = unuse_vma(vma, type);
if (ret)
break;
}
cond_resched();
}
mmap_read_unlock(mm);
return ret;
}
/*
* Scan swap_map from current position to next entry still in use.
* Return 0 if there are no inuse entries after prev till end of
* the map.
*/
static unsigned int find_next_to_unuse(struct swap_info_struct *si,
unsigned int prev)
{
unsigned int i;
unsigned char count;
/*
* No need for swap_lock here: we're just looking
* for whether an entry is in use, not modifying it; false
* hits are okay, and sys_swapoff() has already prevented new
* allocations from this area (while holding swap_lock).
*/
for (i = prev + 1; i < si->max; i++) {
count = READ_ONCE(si->swap_map[i]);
if (count && swap_count(count) != SWAP_MAP_BAD)
break;
if ((i % LATENCY_LIMIT) == 0)
cond_resched();
}
if (i == si->max)
i = 0;
return i;
}
static int try_to_unuse(unsigned int type)
{
struct mm_struct *prev_mm;
struct mm_struct *mm;
struct list_head *p;
int retval = 0;
struct swap_info_struct *si = swap_info[type];
struct folio *folio;
swp_entry_t entry;
unsigned int i;
if (!READ_ONCE(si->inuse_pages))
goto success;
retry:
retval = shmem_unuse(type);
if (retval)
return retval;
prev_mm = &init_mm;
mmget(prev_mm);
spin_lock(&mmlist_lock);
p = &init_mm.mmlist;
while (READ_ONCE(si->inuse_pages) &&
!signal_pending(current) &&
(p = p->next) != &init_mm.mmlist) {
mm = list_entry(p, struct mm_struct, mmlist);
if (!mmget_not_zero(mm))
continue;
spin_unlock(&mmlist_lock);
mmput(prev_mm);
prev_mm = mm;
retval = unuse_mm(mm, type);
if (retval) {
mmput(prev_mm);
return retval;
}
/*
* Make sure that we aren't completely killing
* interactive performance.
*/
cond_resched();
spin_lock(&mmlist_lock);
}
spin_unlock(&mmlist_lock);
mmput(prev_mm);
i = 0;
while (READ_ONCE(si->inuse_pages) &&
!signal_pending(current) &&
(i = find_next_to_unuse(si, i)) != 0) {
entry = swp_entry(type, i);
folio = filemap_get_folio(swap_address_space(entry), i);
if (IS_ERR(folio))
continue;
/*
* It is conceivable that a racing task removed this folio from
* swap cache just before we acquired the page lock. The folio
* might even be back in swap cache on another swap area. But
* that is okay, folio_free_swap() only removes stale folios.
*/
folio_lock(folio);
folio_wait_writeback(folio);
folio_free_swap(folio);
folio_unlock(folio);
folio_put(folio);
}
/*
* Lets check again to see if there are still swap entries in the map.
* If yes, we would need to do retry the unuse logic again.
* Under global memory pressure, swap entries can be reinserted back
* into process space after the mmlist loop above passes over them.
*
* Limit the number of retries? No: when mmget_not_zero()
* above fails, that mm is likely to be freeing swap from
* exit_mmap(), which proceeds at its own independent pace;
* and even shmem_writepage() could have been preempted after
* folio_alloc_swap(), temporarily hiding that swap. It's easy
* and robust (though cpu-intensive) just to keep retrying.
*/
if (READ_ONCE(si->inuse_pages)) {
if (!signal_pending(current))
goto retry;
return -EINTR;
}
success:
/*
* Make sure that further cleanups after try_to_unuse() returns happen
* after swap_range_free() reduces si->inuse_pages to 0.
*/
smp_mb();
return 0;
}
/*
* After a successful try_to_unuse, if no swap is now in use, we know
* we can empty the mmlist. swap_lock must be held on entry and exit.
* Note that mmlist_lock nests inside swap_lock, and an mm must be
* added to the mmlist just after page_duplicate - before would be racy.
*/
static void drain_mmlist(void)
{
struct list_head *p, *next;
unsigned int type;
for (type = 0; type < nr_swapfiles; type++)
if (swap_info[type]->inuse_pages)
return;
spin_lock(&mmlist_lock);
list_for_each_safe(p, next, &init_mm.mmlist)
list_del_init(p);
spin_unlock(&mmlist_lock);
}
/*
* Free all of a swapdev's extent information
*/
static void destroy_swap_extents(struct swap_info_struct *sis)
{
while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) {
struct rb_node *rb = sis->swap_extent_root.rb_node;
struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node);
rb_erase(rb, &sis->swap_extent_root);
kfree(se);
}
if (sis->flags & SWP_ACTIVATED) {
struct file *swap_file = sis->swap_file;
struct address_space *mapping = swap_file->f_mapping;
sis->flags &= ~SWP_ACTIVATED;
if (mapping->a_ops->swap_deactivate)
mapping->a_ops->swap_deactivate(swap_file);
}
}
/*
* Add a block range (and the corresponding page range) into this swapdev's
* extent tree.
*
* This function rather assumes that it is called in ascending page order.
*/
int
add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
unsigned long nr_pages, sector_t start_block)
{
struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL;
struct swap_extent *se;
struct swap_extent *new_se;
/*
* place the new node at the right most since the
* function is called in ascending page order.
*/
while (*link) {
parent = *link;
link = &parent->rb_right;
}
if (parent) {
se = rb_entry(parent, struct swap_extent, rb_node);
BUG_ON(se->start_page + se->nr_pages != start_page);
if (se->start_block + se->nr_pages == start_block) {
/* Merge it */
se->nr_pages += nr_pages;
return 0;
}
}
/* No merge, insert a new extent. */
new_se = kmalloc(sizeof(*se), GFP_KERNEL);
if (new_se == NULL)
return -ENOMEM;
new_se->start_page = start_page;
new_se->nr_pages = nr_pages;
new_se->start_block = start_block;
rb_link_node(&new_se->rb_node, parent, link);
rb_insert_color(&new_se->rb_node, &sis->swap_extent_root);
return 1;
}
EXPORT_SYMBOL_GPL(add_swap_extent);
/*
* A `swap extent' is a simple thing which maps a contiguous range of pages
* onto a contiguous range of disk blocks. A rbtree of swap extents is
* built at swapon time and is then used at swap_writepage/swap_read_folio
* time for locating where on disk a page belongs.
*
* If the swapfile is an S_ISBLK block device, a single extent is installed.
* This is done so that the main operating code can treat S_ISBLK and S_ISREG
* swap files identically.
*
* Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
* extent rbtree operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
* swapfiles are handled *identically* after swapon time.
*
* For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
* and will parse them into a rbtree, in PAGE_SIZE chunks. If some stray
* blocks are found which do not fall within the PAGE_SIZE alignment
* requirements, they are simply tossed out - we will never use those blocks
* for swapping.
*
* For all swap devices we set S_SWAPFILE across the life of the swapon. This
* prevents users from writing to the swap device, which will corrupt memory.
*
* The amount of disk space which a single swap extent represents varies.
* Typically it is in the 1-4 megabyte range. So we can have hundreds of
* extents in the rbtree. - akpm.
*/
static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
{
struct file *swap_file = sis->swap_file;
struct address_space *mapping = swap_file->f_mapping;
struct inode *inode = mapping->host;
int ret;
if (S_ISBLK(inode->i_mode)) {
ret = add_swap_extent(sis, 0, sis->max, 0);
*span = sis->pages;
return ret;
}
if (mapping->a_ops->swap_activate) {
ret = mapping->a_ops->swap_activate(sis, swap_file, span);
if (ret < 0)
return ret;
sis->flags |= SWP_ACTIVATED;
if ((sis->flags & SWP_FS_OPS) &&
sio_pool_init() != 0) {
destroy_swap_extents(sis);
return -ENOMEM;
}
return ret;
}
return generic_swapfile_activate(sis, swap_file, span);
}
static int swap_node(struct swap_info_struct *p)
{
struct block_device *bdev;
if (p->bdev)
bdev = p->bdev;
else
bdev = p->swap_file->f_inode->i_sb->s_bdev;
return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE;
}
static void setup_swap_info(struct swap_info_struct *p, int prio,
unsigned char *swap_map,
struct swap_cluster_info *cluster_info)
{
int i;
if (prio >= 0)
p->prio = prio;
else
p->prio = --least_priority;
/*
* the plist prio is negated because plist ordering is
* low-to-high, while swap ordering is high-to-low
*/
p->list.prio = -p->prio;
for_each_node(i) {
if (p->prio >= 0)
p->avail_lists[i].prio = -p->prio;
else {
if (swap_node(p) == i)
p->avail_lists[i].prio = 1;
else
p->avail_lists[i].prio = -p->prio;
}
}
p->swap_map = swap_map;
p->cluster_info = cluster_info;
}
static void _enable_swap_info(struct swap_info_struct *p)
{
p->flags |= SWP_WRITEOK;
atomic_long_add(p->pages, &nr_swap_pages);
total_swap_pages += p->pages;
assert_spin_locked(&swap_lock);
/*
* both lists are plists, and thus priority ordered.
* swap_active_head needs to be priority ordered for swapoff(),
* which on removal of any swap_info_struct with an auto-assigned
* (i.e. negative) priority increments the auto-assigned priority
* of any lower-priority swap_info_structs.
* swap_avail_head needs to be priority ordered for folio_alloc_swap(),
* which allocates swap pages from the highest available priority
* swap_info_struct.
*/
plist_add(&p->list, &swap_active_head);
/* add to available list iff swap device is not full */
if (p->highest_bit)
add_to_avail_list(p);
}
static void enable_swap_info(struct swap_info_struct *p, int prio,
unsigned char *swap_map,
struct swap_cluster_info *cluster_info)
{
spin_lock(&swap_lock);
spin_lock(&p->lock);
setup_swap_info(p, prio, swap_map, cluster_info);
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
/*
* Finished initializing swap device, now it's safe to reference it.
*/
percpu_ref_resurrect(&p->users);
spin_lock(&swap_lock);
spin_lock(&p->lock);
_enable_swap_info(p);
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
}
static void reinsert_swap_info(struct swap_info_struct *p)
{
spin_lock(&swap_lock);
spin_lock(&p->lock);
setup_swap_info(p, p->prio, p->swap_map, p->cluster_info);
_enable_swap_info(p);
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
}
bool has_usable_swap(void)
{
bool ret = true;
spin_lock(&swap_lock);
if (plist_head_empty(&swap_active_head))
ret = false;
spin_unlock(&swap_lock);
return ret;
}
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
{
struct swap_info_struct *p = NULL;
unsigned char *swap_map;
struct swap_cluster_info *cluster_info;
struct file *swap_file, *victim;
struct address_space *mapping;
struct inode *inode;
struct filename *pathname;
int err, found = 0;
unsigned int old_block_size;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
BUG_ON(!current->mm);
pathname = getname(specialfile);
if (IS_ERR(pathname))
return PTR_ERR(pathname);
victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
err = PTR_ERR(victim);
if (IS_ERR(victim))
goto out;
mapping = victim->f_mapping;
spin_lock(&swap_lock);
plist_for_each_entry(p, &swap_active_head, list) {
if (p->flags & SWP_WRITEOK) {
if (p->swap_file->f_mapping == mapping) {
found = 1;
break;
}
}
}
if (!found) {
err = -EINVAL;
spin_unlock(&swap_lock);
goto out_dput;
}
if (!security_vm_enough_memory_mm(current->mm, p->pages))
vm_unacct_memory(p->pages);
else {
err = -ENOMEM;
spin_unlock(&swap_lock);
goto out_dput;
}
spin_lock(&p->lock);
del_from_avail_list(p);
if (p->prio < 0) {
struct swap_info_struct *si = p;
int nid;
plist_for_each_entry_continue(si, &swap_active_head, list) {
si->prio++;
si->list.prio--;
for_each_node(nid) {
if (si->avail_lists[nid].prio != 1)
si->avail_lists[nid].prio--;
}
}
least_priority++;
}
plist_del(&p->list, &swap_active_head);
atomic_long_sub(p->pages, &nr_swap_pages);
total_swap_pages -= p->pages;
p->flags &= ~SWP_WRITEOK;
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
disable_swap_slots_cache_lock();
set_current_oom_origin();
err = try_to_unuse(p->type);
clear_current_oom_origin();
if (err) {
/* re-insert swap space back into swap_list */
reinsert_swap_info(p);
reenable_swap_slots_cache_unlock();
goto out_dput;
}
reenable_swap_slots_cache_unlock();
/*
* Wait for swap operations protected by get/put_swap_device()
* to complete.
*
* We need synchronize_rcu() here to protect the accessing to
* the swap cache data structure.
*/
percpu_ref_kill(&p->users);
synchronize_rcu();
wait_for_completion(&p->comp);
flush_work(&p->discard_work);
destroy_swap_extents(p);
if (p->flags & SWP_CONTINUED)
free_swap_count_continuations(p);
if (!p->bdev || !bdev_nonrot(p->bdev))
atomic_dec(&nr_rotate_swap);
mutex_lock(&swapon_mutex);
spin_lock(&swap_lock);
spin_lock(&p->lock);
drain_mmlist();
/* wait for anyone still in scan_swap_map_slots */
p->highest_bit = 0; /* cuts scans short */
while (p->flags >= SWP_SCANNING) {
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
schedule_timeout_uninterruptible(1);
spin_lock(&swap_lock);
spin_lock(&p->lock);
}
swap_file = p->swap_file;
old_block_size = p->old_block_size;
p->swap_file = NULL;
p->max = 0;
swap_map = p->swap_map;
p->swap_map = NULL;
cluster_info = p->cluster_info;
p->cluster_info = NULL;
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
arch_swap_invalidate_area(p->type);
zswap_swapoff(p->type);
mutex_unlock(&swapon_mutex);
free_percpu(p->percpu_cluster);
p->percpu_cluster = NULL;
free_percpu(p->cluster_next_cpu);
p->cluster_next_cpu = NULL;
vfree(swap_map);
kvfree(cluster_info);
/* Destroy swap account information */
swap_cgroup_swapoff(p->type);
exit_swap_address_space(p->type);
inode = mapping->host;
if (p->bdev_file) {
set_blocksize(p->bdev, old_block_size);
fput(p->bdev_file);
p->bdev_file = NULL;
}
inode_lock(inode);
inode->i_flags &= ~S_SWAPFILE;
inode_unlock(inode);
filp_close(swap_file, NULL);
/*
* Clear the SWP_USED flag after all resources are freed so that swapon
* can reuse this swap_info in alloc_swap_info() safely. It is ok to
* not hold p->lock after we cleared its SWP_WRITEOK.
*/
spin_lock(&swap_lock);
p->flags = 0;
spin_unlock(&swap_lock);
err = 0;
atomic_inc(&proc_poll_event);
wake_up_interruptible(&proc_poll_wait);
out_dput:
filp_close(victim, NULL);
out:
putname(pathname);
return err;
}
#ifdef CONFIG_PROC_FS
static __poll_t swaps_poll(struct file *file, poll_table *wait)
{
struct seq_file *seq = file->private_data;
poll_wait(file, &proc_poll_wait, wait);
if (seq->poll_event != atomic_read(&proc_poll_event)) {
seq->poll_event = atomic_read(&proc_poll_event);
return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI;
}
return EPOLLIN | EPOLLRDNORM;
}
/* iterator */
static void *swap_start(struct seq_file *swap, loff_t *pos)
{
struct swap_info_struct *si;
int type;
loff_t l = *pos;
mutex_lock(&swapon_mutex);
if (!l)
return SEQ_START_TOKEN;
for (type = 0; (si = swap_type_to_swap_info(type)); type++) {
if (!(si->flags & SWP_USED) || !si->swap_map)
continue;
if (!--l)
return si;
}
return NULL;
}
static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
{
struct swap_info_struct *si = v;
int type;
if (v == SEQ_START_TOKEN)
type = 0;
else
type = si->type + 1;
++(*pos);
for (; (si = swap_type_to_swap_info(type)); type++) {
if (!(si->flags & SWP_USED) || !si->swap_map)
continue;
return si;
}
return NULL;
}
static void swap_stop(struct seq_file *swap, void *v)
{
mutex_unlock(&swapon_mutex);
}
static int swap_show(struct seq_file *swap, void *v)
{
struct swap_info_struct *si = v;
struct file *file;
int len;
unsigned long bytes, inuse;
if (si == SEQ_START_TOKEN) {
seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");
return 0;
}
bytes = K(si->pages);
inuse = K(READ_ONCE(si->inuse_pages));
file = si->swap_file;
len = seq_file_path(swap, file, " \t\n\\");
seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n",
len < 40 ? 40 - len : 1, " ",
S_ISBLK(file_inode(file)->i_mode) ?
"partition" : "file\t",
bytes, bytes < 10000000 ? "\t" : "",
inuse, inuse < 10000000 ? "\t" : "",
si->prio);
return 0;
}
static const struct seq_operations swaps_op = {
.start = swap_start,
.next = swap_next,
.stop = swap_stop,
.show = swap_show
};
static int swaps_open(struct inode *inode, struct file *file)
{
struct seq_file *seq;
int ret;
ret = seq_open(file, &swaps_op);
if (ret)
return ret;
seq = file->private_data;
seq->poll_event = atomic_read(&proc_poll_event);
return 0;
}
static const struct proc_ops swaps_proc_ops = {
.proc_flags = PROC_ENTRY_PERMANENT,
.proc_open = swaps_open,
.proc_read = seq_read,
.proc_lseek = seq_lseek,
.proc_release = seq_release,
.proc_poll = swaps_poll,
};
static int __init procswaps_init(void)
{
proc_create("swaps", 0, NULL, &swaps_proc_ops);
return 0;
}
__initcall(procswaps_init);
#endif /* CONFIG_PROC_FS */
#ifdef MAX_SWAPFILES_CHECK
static int __init max_swapfiles_check(void)
{
MAX_SWAPFILES_CHECK();
return 0;
}
late_initcall(max_swapfiles_check);
#endif
static struct swap_info_struct *alloc_swap_info(void)
{
struct swap_info_struct *p;
struct swap_info_struct *defer = NULL;
unsigned int type;
int i;
p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL);
if (!p)
return ERR_PTR(-ENOMEM);
if (percpu_ref_init(&p->users, swap_users_ref_free,
PERCPU_REF_INIT_DEAD, GFP_KERNEL)) {
kvfree(p);
return ERR_PTR(-ENOMEM);
}
spin_lock(&swap_lock);
for (type = 0; type < nr_swapfiles; type++) {
if (!(swap_info[type]->flags & SWP_USED))
break;
}
if (type >= MAX_SWAPFILES) {
spin_unlock(&swap_lock);
percpu_ref_exit(&p->users);
kvfree(p);
return ERR_PTR(-EPERM);
}
if (type >= nr_swapfiles) {
p->type = type;
/*
* Publish the swap_info_struct after initializing it.
* Note that kvzalloc() above zeroes all its fields.
*/
smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */
nr_swapfiles++;
} else {
defer = p;
p = swap_info[type];
/*
* Do not memset this entry: a racing procfs swap_next()
* would be relying on p->type to remain valid.
*/
}
p->swap_extent_root = RB_ROOT;
plist_node_init(&p->list, 0);
for_each_node(i)
plist_node_init(&p->avail_lists[i], 0);
p->flags = SWP_USED;
spin_unlock(&swap_lock);
if (defer) {
percpu_ref_exit(&defer->users);
kvfree(defer);
}
spin_lock_init(&p->lock);
spin_lock_init(&p->cont_lock);
init_completion(&p->comp);
return p;
}
static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
{
int error;
if (S_ISBLK(inode->i_mode)) {
p->bdev_file = bdev_file_open_by_dev(inode->i_rdev,
BLK_OPEN_READ | BLK_OPEN_WRITE, p, NULL);
if (IS_ERR(p->bdev_file)) {
error = PTR_ERR(p->bdev_file);
p->bdev_file = NULL;
return error;
}
p->bdev = file_bdev(p->bdev_file);
p->old_block_size = block_size(p->bdev);
error = set_blocksize(p->bdev, PAGE_SIZE);
if (error < 0)
return error;
/*
* Zoned block devices contain zones that have a sequential
* write only restriction. Hence zoned block devices are not
* suitable for swapping. Disallow them here.
*/
if (bdev_is_zoned(p->bdev))
return -EINVAL;
p->flags |= SWP_BLKDEV;
} else if (S_ISREG(inode->i_mode)) {
p->bdev = inode->i_sb->s_bdev;
}
return 0;
}
/*
* Find out how many pages are allowed for a single swap device. There
* are two limiting factors:
* 1) the number of bits for the swap offset in the swp_entry_t type, and
* 2) the number of bits in the swap pte, as defined by the different
* architectures.
*
* In order to find the largest possible bit mask, a swap entry with
* swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
* decoded to a swp_entry_t again, and finally the swap offset is
* extracted.
*
* This will mask all the bits from the initial ~0UL mask that can't
* be encoded in either the swp_entry_t or the architecture definition
* of a swap pte.
*/
unsigned long generic_max_swapfile_size(void)
{
return swp_offset(pte_to_swp_entry(
swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
}
/* Can be overridden by an architecture for additional checks. */
__weak unsigned long arch_max_swapfile_size(void)
{
return generic_max_swapfile_size();
}
static unsigned long read_swap_header(struct swap_info_struct *p,
union swap_header *swap_header,
struct inode *inode)
{
int i;
unsigned long maxpages;
unsigned long swapfilepages;
unsigned long last_page;
if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
pr_err("Unable to find swap-space signature\n");
return 0;
}
/* swap partition endianness hack... */
if (swab32(swap_header->info.version) == 1) {
swab32s(&swap_header->info.version);
swab32s(&swap_header->info.last_page);
swab32s(&swap_header->info.nr_badpages);
if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
return 0;
for (i = 0; i < swap_header->info.nr_badpages; i++)
swab32s(&swap_header->info.badpages[i]);
}
/* Check the swap header's sub-version */
if (swap_header->info.version != 1) {
pr_warn("Unable to handle swap header version %d\n",
swap_header->info.version);
return 0;
}
p->lowest_bit = 1;
p->cluster_next = 1;
p->cluster_nr = 0;
maxpages = swapfile_maximum_size;
last_page = swap_header->info.last_page;
if (!last_page) {
pr_warn("Empty swap-file\n");
return 0;
}
if (last_page > maxpages) {
pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
K(maxpages), K(last_page));
}
if (maxpages > last_page) {
maxpages = last_page + 1;
/* p->max is an unsigned int: don't overflow it */
if ((unsigned int)maxpages == 0)
maxpages = UINT_MAX;
}
p->highest_bit = maxpages - 1;
if (!maxpages)
return 0;
swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
if (swapfilepages && maxpages > swapfilepages) {
pr_warn("Swap area shorter than signature indicates\n");
return 0;
}
if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
return 0;
if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
return 0;
return maxpages;
}
#define SWAP_CLUSTER_INFO_COLS \
DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info))
#define SWAP_CLUSTER_SPACE_COLS \
DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER)
#define SWAP_CLUSTER_COLS \
max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS)
static int setup_swap_map_and_extents(struct swap_info_struct *p,
union swap_header *swap_header,
unsigned char *swap_map,
struct swap_cluster_info *cluster_info,
unsigned long maxpages,
sector_t *span)
{
unsigned int j, k;
unsigned int nr_good_pages;
int nr_extents;
unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS;
unsigned long i, idx;
nr_good_pages = maxpages - 1; /* omit header page */
cluster_list_init(&p->free_clusters);
cluster_list_init(&p->discard_clusters);
for (i = 0; i < swap_header->info.nr_badpages; i++) {
unsigned int page_nr = swap_header->info.badpages[i];
if (page_nr == 0 || page_nr > swap_header->info.last_page)
return -EINVAL;
if (page_nr < maxpages) {
swap_map[page_nr] = SWAP_MAP_BAD;
nr_good_pages--;
/*
* Haven't marked the cluster free yet, no list
* operation involved
*/
inc_cluster_info_page(p, cluster_info, page_nr);
}
}
/* Haven't marked the cluster free yet, no list operation involved */
for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
inc_cluster_info_page(p, cluster_info, i);
if (nr_good_pages) {
swap_map[0] = SWAP_MAP_BAD;
/*
* Not mark the cluster free yet, no list
* operation involved
*/
inc_cluster_info_page(p, cluster_info, 0);
p->max = maxpages;
p->pages = nr_good_pages;
nr_extents = setup_swap_extents(p, span);
if (nr_extents < 0)
return nr_extents;
nr_good_pages = p->pages;
}
if (!nr_good_pages) {
pr_warn("Empty swap-file\n");
return -EINVAL;
}
if (!cluster_info)
return nr_extents;
/*
* Reduce false cache line sharing between cluster_info and
* sharing same address space.
*/
for (k = 0; k < SWAP_CLUSTER_COLS; k++) {
j = (k + col) % SWAP_CLUSTER_COLS;
for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) {
idx = i * SWAP_CLUSTER_COLS + j;
if (idx >= nr_clusters)
continue;
if (cluster_count(&cluster_info[idx]))
continue;
cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
cluster_list_add_tail(&p->free_clusters, cluster_info,
idx);
}
}
return nr_extents;
}
SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
{
struct swap_info_struct *p;
struct filename *name;
struct file *swap_file = NULL;
struct address_space *mapping;
struct dentry *dentry;
int prio;
int error;
union swap_header *swap_header;
int nr_extents;
sector_t span;
unsigned long maxpages;
unsigned char *swap_map = NULL;
struct swap_cluster_info *cluster_info = NULL;
struct page *page = NULL;
struct inode *inode = NULL;
bool inced_nr_rotate_swap = false;
if (swap_flags & ~SWAP_FLAGS_VALID)
return -EINVAL;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (!swap_avail_heads)
return -ENOMEM;
p = alloc_swap_info();
if (IS_ERR(p))
return PTR_ERR(p);
INIT_WORK(&p->discard_work, swap_discard_work);
name = getname(specialfile);
if (IS_ERR(name)) {
error = PTR_ERR(name);
name = NULL;
goto bad_swap;
}
swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
if (IS_ERR(swap_file)) {
error = PTR_ERR(swap_file);
swap_file = NULL;
goto bad_swap;
}
p->swap_file = swap_file;
mapping = swap_file->f_mapping;
dentry = swap_file->f_path.dentry;
inode = mapping->host;
error = claim_swapfile(p, inode);
if (unlikely(error))
goto bad_swap;
inode_lock(inode);
if (d_unlinked(dentry) || cant_mount(dentry)) {
error = -ENOENT;
goto bad_swap_unlock_inode;
}
if (IS_SWAPFILE(inode)) {
error = -EBUSY;
goto bad_swap_unlock_inode;
}
/*
* Read the swap header.
*/
if (!mapping->a_ops->read_folio) {
error = -EINVAL;
goto bad_swap_unlock_inode;
}
page = read_mapping_page(mapping, 0, swap_file);
if (IS_ERR(page)) {
error = PTR_ERR(page);
goto bad_swap_unlock_inode;
}
swap_header = kmap(page);
maxpages = read_swap_header(p, swap_header, inode);
if (unlikely(!maxpages)) {
error = -EINVAL;
goto bad_swap_unlock_inode;
}
/* OK, set up the swap map and apply the bad block list */
swap_map = vzalloc(maxpages);
if (!swap_map) {
error = -ENOMEM;
goto bad_swap_unlock_inode;
}
if (p->bdev && bdev_stable_writes(p->bdev))
p->flags |= SWP_STABLE_WRITES;
if (p->bdev && bdev_synchronous(p->bdev))
p->flags |= SWP_SYNCHRONOUS_IO;
if (p->bdev && bdev_nonrot(p->bdev)) {
int cpu;
unsigned long ci, nr_cluster;
p->flags |= SWP_SOLIDSTATE;
p->cluster_next_cpu = alloc_percpu(unsigned int);
if (!p->cluster_next_cpu) {
error = -ENOMEM;
goto bad_swap_unlock_inode;
}
/*
* select a random position to start with to help wear leveling
* SSD
*/
for_each_possible_cpu(cpu) {
per_cpu(*p->cluster_next_cpu, cpu) =
get_random_u32_inclusive(1, p->highest_bit);
}
nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info),
GFP_KERNEL);
if (!cluster_info) {
error = -ENOMEM;
goto bad_swap_unlock_inode;
}
for (ci = 0; ci < nr_cluster; ci++)
spin_lock_init(&((cluster_info + ci)->lock));
p->percpu_cluster = alloc_percpu(struct percpu_cluster);
if (!p->percpu_cluster) {
error = -ENOMEM;
goto bad_swap_unlock_inode;
}
for_each_possible_cpu(cpu) {
struct percpu_cluster *cluster;
cluster = per_cpu_ptr(p->percpu_cluster, cpu);
cluster_set_null(&cluster->index);
}
} else {
atomic_inc(&nr_rotate_swap);
inced_nr_rotate_swap = true;
}
error = swap_cgroup_swapon(p->type, maxpages);
if (error)
goto bad_swap_unlock_inode;
nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
cluster_info, maxpages, &span);
if (unlikely(nr_extents < 0)) {
error = nr_extents;
goto bad_swap_unlock_inode;
}
if ((swap_flags & SWAP_FLAG_DISCARD) &&
p->bdev && bdev_max_discard_sectors(p->bdev)) {
/*
* When discard is enabled for swap with no particular
* policy flagged, we set all swap discard flags here in
* order to sustain backward compatibility with older
* swapon(8) releases.
*/
p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
SWP_PAGE_DISCARD);
/*
* By flagging sys_swapon, a sysadmin can tell us to
* either do single-time area discards only, or to just
* perform discards for released swap page-clusters.
* Now it's time to adjust the p->flags accordingly.
*/
if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
p->flags &= ~SWP_PAGE_DISCARD;
else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
p->flags &= ~SWP_AREA_DISCARD;
/* issue a swapon-time discard if it's still required */
if (p->flags & SWP_AREA_DISCARD) {
int err = discard_swap(p);
if (unlikely(err))
pr_err("swapon: discard_swap(%p): %d\n",
p, err);
}
}
error = init_swap_address_space(p->type, maxpages);
if (error)
goto bad_swap_unlock_inode;
error = zswap_swapon(p->type, maxpages);
if (error)
goto free_swap_address_space;
/*
* Flush any pending IO and dirty mappings before we start using this
* swap device.
*/
inode->i_flags |= S_SWAPFILE;
error = inode_drain_writes(inode);
if (error) {
inode->i_flags &= ~S_SWAPFILE;
goto free_swap_zswap;
}
mutex_lock(&swapon_mutex);
prio = -1;
if (swap_flags & SWAP_FLAG_PREFER)
prio =
(swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
enable_swap_info(p, prio, swap_map, cluster_info);
pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s\n",
K(p->pages), name->name, p->prio, nr_extents,
K((unsigned long long)span),
(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
(p->flags & SWP_DISCARDABLE) ? "D" : "",
(p->flags & SWP_AREA_DISCARD) ? "s" : "",
(p->flags & SWP_PAGE_DISCARD) ? "c" : "");
mutex_unlock(&swapon_mutex);
atomic_inc(&proc_poll_event);
wake_up_interruptible(&proc_poll_wait);
error = 0;
goto out;
free_swap_zswap:
zswap_swapoff(p->type);
free_swap_address_space:
exit_swap_address_space(p->type);
bad_swap_unlock_inode:
inode_unlock(inode);
bad_swap:
free_percpu(p->percpu_cluster);
p->percpu_cluster = NULL;
free_percpu(p->cluster_next_cpu);
p->cluster_next_cpu = NULL;
if (p->bdev_file) {
set_blocksize(p->bdev, p->old_block_size);
fput(p->bdev_file);
p->bdev_file = NULL;
}
inode = NULL;
destroy_swap_extents(p);
swap_cgroup_swapoff(p->type);
spin_lock(&swap_lock);
p->swap_file = NULL;
p->flags = 0;
spin_unlock(&swap_lock);
vfree(swap_map);
kvfree(cluster_info);
if (inced_nr_rotate_swap)
atomic_dec(&nr_rotate_swap);
if (swap_file)
filp_close(swap_file, NULL);
out:
if (page && !IS_ERR(page)) {
kunmap(page);
put_page(page);
}
if (name)
putname(name);
if (inode)
inode_unlock(inode);
if (!error)
enable_swap_slots_cache();
return error;
}
void si_swapinfo(struct sysinfo *val)
{
unsigned int type;
unsigned long nr_to_be_unused = 0;
spin_lock(&swap_lock);
for (type = 0; type < nr_swapfiles; type++) {
struct swap_info_struct *si = swap_info[type];
if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
nr_to_be_unused += READ_ONCE(si->inuse_pages);
}
val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
val->totalswap = total_swap_pages + nr_to_be_unused;
spin_unlock(&swap_lock);
}
/*
* Verify that a swap entry is valid and increment its swap map count.
*
* Returns error code in following case.
* - success -> 0
* - swp_entry is invalid -> EINVAL
* - swp_entry is migration entry -> EINVAL
* - swap-cache reference is requested but there is already one. -> EEXIST
* - swap-cache reference is requested but the entry is not used. -> ENOENT
* - swap-mapped reference requested but needs continued swap count. -> ENOMEM
*/
static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
{
struct swap_info_struct *p;
struct swap_cluster_info *ci;
unsigned long offset;
unsigned char count;
unsigned char has_cache;
int err;
p = swp_swap_info(entry);
offset = swp_offset(entry);
ci = lock_cluster_or_swap_info(p, offset);
count = p->swap_map[offset];
/*
* swapin_readahead() doesn't check if a swap entry is valid, so the
* swap entry could be SWAP_MAP_BAD. Check here with lock held.
*/
if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
err = -ENOENT;
goto unlock_out;
}
has_cache = count & SWAP_HAS_CACHE;
count &= ~SWAP_HAS_CACHE;
err = 0;
if (usage == SWAP_HAS_CACHE) {
/* set SWAP_HAS_CACHE if there is no cache and entry is used */
if (!has_cache && count)
has_cache = SWAP_HAS_CACHE;
else if (has_cache) /* someone else added cache */
err = -EEXIST;
else /* no users remaining */
err = -ENOENT;
} else if (count || has_cache) {
if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
count += usage;
else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
err = -EINVAL;
else if (swap_count_continued(p, offset, count))
count = COUNT_CONTINUED;
else
err = -ENOMEM;
} else
err = -ENOENT; /* unused swap entry */
if (!err)
WRITE_ONCE(p->swap_map[offset], count | has_cache);
unlock_out:
unlock_cluster_or_swap_info(p, ci);
return err;
}
/*
* Help swapoff by noting that swap entry belongs to shmem/tmpfs
* (in which case its reference count is never incremented).
*/
void swap_shmem_alloc(swp_entry_t entry)
{
__swap_duplicate(entry, SWAP_MAP_SHMEM);
}
/*
* Increase reference count of swap entry by 1.
* Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
* but could not be atomically allocated. Returns 0, just as if it succeeded,
* if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
* might occur if a page table entry has got corrupted.
*/
int swap_duplicate(swp_entry_t entry)
{
int err = 0;
while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
err = add_swap_count_continuation(entry, GFP_ATOMIC);
return err;
}
/*
* @entry: swap entry for which we allocate swap cache.
*
* Called when allocating swap cache for existing swap entry,
* This can return error codes. Returns 0 at success.
* -EEXIST means there is a swap cache.
* Note: return code is different from swap_duplicate().
*/
int swapcache_prepare(swp_entry_t entry)
{
return __swap_duplicate(entry, SWAP_HAS_CACHE);
}
void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry)
{
struct swap_cluster_info *ci;
unsigned long offset = swp_offset(entry);
unsigned char usage;
ci = lock_cluster_or_swap_info(si, offset);
usage = __swap_entry_free_locked(si, offset, SWAP_HAS_CACHE);
unlock_cluster_or_swap_info(si, ci);
if (!usage)
free_swap_slot(entry);
}
struct swap_info_struct *swp_swap_info(swp_entry_t entry)
{
return swap_type_to_swap_info(swp_type(entry));
}
/*
* out-of-line methods to avoid include hell.
*/
struct address_space *swapcache_mapping(struct folio *folio)
{
return swp_swap_info(folio->swap)->swap_file->f_mapping;
}
EXPORT_SYMBOL_GPL(swapcache_mapping);
pgoff_t __page_file_index(struct page *page)
{
swp_entry_t swap = page_swap_entry(page);
return swp_offset(swap);
}
EXPORT_SYMBOL_GPL(__page_file_index);
/*
* add_swap_count_continuation - called when a swap count is duplicated
* beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
* page of the original vmalloc'ed swap_map, to hold the continuation count
* (for that entry and for its neighbouring PAGE_SIZE swap entries). Called
* again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
*
* These continuation pages are seldom referenced: the common paths all work
* on the original swap_map, only referring to a continuation page when the
* low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
*
* add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
* page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
* can be called after dropping locks.
*/
int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
{
struct swap_info_struct *si;
struct swap_cluster_info *ci;
struct page *head;
struct page *page;
struct page *list_page;
pgoff_t offset;
unsigned char count;
int ret = 0;
/*
* When debugging, it's easier to use __GFP_ZERO here; but it's better
* for latency not to zero a page while GFP_ATOMIC and holding locks.
*/
page = alloc_page(gfp_mask | __GFP_HIGHMEM);
si = get_swap_device(entry);
if (!si) {
/*
* An acceptable race has occurred since the failing
* __swap_duplicate(): the swap device may be swapoff
*/
goto outer;
}
spin_lock(&si->lock);
offset = swp_offset(entry);
ci = lock_cluster(si, offset);
count = swap_count(si->swap_map[offset]);
if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
/*
* The higher the swap count, the more likely it is that tasks
* will race to add swap count continuation: we need to avoid
* over-provisioning.
*/
goto out;
}
if (!page) {
ret = -ENOMEM;
goto out;
}
head = vmalloc_to_page(si->swap_map + offset);
offset &= ~PAGE_MASK;
spin_lock(&si->cont_lock);
/*
* Page allocation does not initialize the page's lru field,
* but it does always reset its private field.
*/
if (!page_private(head)) {
BUG_ON(count & COUNT_CONTINUED);
INIT_LIST_HEAD(&head->lru);
set_page_private(head, SWP_CONTINUED);
si->flags |= SWP_CONTINUED;
}
list_for_each_entry(list_page, &head->lru, lru) {
unsigned char *map;
/*
* If the previous map said no continuation, but we've found
* a continuation page, free our allocation and use this one.
*/
if (!(count & COUNT_CONTINUED))
goto out_unlock_cont;
map = kmap_local_page(list_page) + offset;
count = *map;
kunmap_local(map);
/*
* If this continuation count now has some space in it,
* free our allocation and use this one.
*/
if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
goto out_unlock_cont;
}
list_add_tail(&page->lru, &head->lru);
page = NULL; /* now it's attached, don't free it */
out_unlock_cont:
spin_unlock(&si->cont_lock);
out:
unlock_cluster(ci);
spin_unlock(&si->lock);
put_swap_device(si);
outer:
if (page)
__free_page(page);
return ret;
}
/*
* swap_count_continued - when the original swap_map count is incremented
* from SWAP_MAP_MAX, check if there is already a continuation page to carry
* into, carry if so, or else fail until a new continuation page is allocated;
* when the original swap_map count is decremented from 0 with continuation,
* borrow from the continuation and report whether it still holds more.
* Called while __swap_duplicate() or swap_entry_free() holds swap or cluster
* lock.
*/
static bool swap_count_continued(struct swap_info_struct *si,
pgoff_t offset, unsigned char count)
{
struct page *head;
struct page *page;
unsigned char *map;
bool ret;
head = vmalloc_to_page(si->swap_map + offset);
if (page_private(head) != SWP_CONTINUED) {
BUG_ON(count & COUNT_CONTINUED);
return false; /* need to add count continuation */
}
spin_lock(&si->cont_lock);
offset &= ~PAGE_MASK;
page = list_next_entry(head, lru);
map = kmap_local_page(page) + offset;
if (count == SWAP_MAP_MAX) /* initial increment from swap_map */
goto init_map; /* jump over SWAP_CONT_MAX checks */
if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
/*
* Think of how you add 1 to 999
*/
while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
kunmap_local(map);
page = list_next_entry(page, lru);
BUG_ON(page == head);
map = kmap_local_page(page) + offset;
}
if (*map == SWAP_CONT_MAX) {
kunmap_local(map);
page = list_next_entry(page, lru);
if (page == head) {
ret = false; /* add count continuation */
goto out;
}
map = kmap_local_page(page) + offset;
init_map: *map = 0; /* we didn't zero the page */
}
*map += 1;
kunmap_local(map);
while ((page = list_prev_entry(page, lru)) != head) {
map = kmap_local_page(page) + offset;
*map = COUNT_CONTINUED;
kunmap_local(map);
}
ret = true; /* incremented */
} else { /* decrementing */
/*
* Think of how you subtract 1 from 1000
*/
BUG_ON(count != COUNT_CONTINUED);
while (*map == COUNT_CONTINUED) {
kunmap_local(map);
page = list_next_entry(page, lru);
BUG_ON(page == head);
map = kmap_local_page(page) + offset;
}
BUG_ON(*map == 0);
*map -= 1;
if (*map == 0)
count = 0;
kunmap_local(map);
while ((page = list_prev_entry(page, lru)) != head) {
map = kmap_local_page(page) + offset;
*map = SWAP_CONT_MAX | count;
count = COUNT_CONTINUED;
kunmap_local(map);
}
ret = count == COUNT_CONTINUED;
}
out:
spin_unlock(&si->cont_lock);
return ret;
}
/*
* free_swap_count_continuations - swapoff free all the continuation pages
* appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
*/
static void free_swap_count_continuations(struct swap_info_struct *si)
{
pgoff_t offset;
for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
struct page *head;
head = vmalloc_to_page(si->swap_map + offset);
if (page_private(head)) {
struct page *page, *next;
list_for_each_entry_safe(page, next, &head->lru, lru) {
list_del(&page->lru);
__free_page(page);
}
}
}
}
#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
{
struct swap_info_struct *si, *next;
int nid = folio_nid(folio);
if (!(gfp & __GFP_IO))
return;
if (!blk_cgroup_congested())
return;
/*
* We've already scheduled a throttle, avoid taking the global swap
* lock.
*/
if (current->throttle_disk)
return;
spin_lock(&swap_avail_lock);
plist_for_each_entry_safe(si, next, &swap_avail_heads[nid],
avail_lists[nid]) {
if (si->bdev) { blkcg_schedule_throttle(si->bdev->bd_disk, true);
break;
}
}
spin_unlock(&swap_avail_lock);
}
#endif
static int __init swapfile_init(void)
{
int nid;
swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head),
GFP_KERNEL);
if (!swap_avail_heads) {
pr_emerg("Not enough memory for swap heads, swap is disabled\n");
return -ENOMEM;
}
for_each_node(nid)
plist_head_init(&swap_avail_heads[nid]);
swapfile_maximum_size = arch_max_swapfile_size();
#ifdef CONFIG_MIGRATION
if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
swap_migration_ad_supported = true;
#endif /* CONFIG_MIGRATION */
return 0;
}
subsys_initcall(swapfile_init);
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
*
* High-resolution kernel timers
*
* In contrast to the low-resolution timeout API, aka timer wheel,
* hrtimers provide finer resolution and accuracy depending on system
* configuration and capabilities.
*
* Started by: Thomas Gleixner and Ingo Molnar
*
* Credits:
* Based on the original timer wheel code
*
* Help, testing, suggestions, bugfixes, improvements were
* provided by:
*
* George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
* et. al.
*/
#include <linux/cpu.h>
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
#include <linux/notifier.h>
#include <linux/syscalls.h>
#include <linux/interrupt.h>
#include <linux/tick.h>
#include <linux/err.h>
#include <linux/debugobjects.h>
#include <linux/sched/signal.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
#include <linux/sched/nohz.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>
#include <linux/timer.h>
#include <linux/freezer.h>
#include <linux/compat.h>
#include <linux/uaccess.h>
#include <trace/events/timer.h>
#include "tick-internal.h"
/*
* Masks for selecting the soft and hard context timers from
* cpu_base->active
*/
#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)
#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
/*
* The timer bases:
*
* There are more clockids than hrtimer bases. Thus, we index
* into the timer bases by the hrtimer_base_type enum. When trying
* to reach a base using a clockid, hrtimer_clockid_to_base()
* is used to convert from clockid to the proper hrtimer_base_type.
*/
DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
{
.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
.clock_base =
{
{
.index = HRTIMER_BASE_MONOTONIC,
.clockid = CLOCK_MONOTONIC,
.get_time = &ktime_get,
},
{
.index = HRTIMER_BASE_REALTIME,
.clockid = CLOCK_REALTIME,
.get_time = &ktime_get_real,
},
{
.index = HRTIMER_BASE_BOOTTIME,
.clockid = CLOCK_BOOTTIME,
.get_time = &ktime_get_boottime,
},
{
.index = HRTIMER_BASE_TAI,
.clockid = CLOCK_TAI,
.get_time = &ktime_get_clocktai,
},
{
.index = HRTIMER_BASE_MONOTONIC_SOFT,
.clockid = CLOCK_MONOTONIC,
.get_time = &ktime_get,
},
{
.index = HRTIMER_BASE_REALTIME_SOFT,
.clockid = CLOCK_REALTIME,
.get_time = &ktime_get_real,
},
{
.index = HRTIMER_BASE_BOOTTIME_SOFT,
.clockid = CLOCK_BOOTTIME,
.get_time = &ktime_get_boottime,
},
{
.index = HRTIMER_BASE_TAI_SOFT,
.clockid = CLOCK_TAI,
.get_time = &ktime_get_clocktai,
},
}
};
static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
/* Make sure we catch unsupported clockids */
[0 ... MAX_CLOCKS - 1] = HRTIMER_MAX_CLOCK_BASES,
[CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,
[CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,
[CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,
[CLOCK_TAI] = HRTIMER_BASE_TAI,
};
/*
* Functions and macros which are different for UP/SMP systems are kept in a
* single place
*/
#ifdef CONFIG_SMP
/*
* We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
* such that hrtimer_callback_running() can unconditionally dereference
* timer->base->cpu_base
*/
static struct hrtimer_cpu_base migration_cpu_base = {
.clock_base = { {
.cpu_base = &migration_cpu_base,
.seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
&migration_cpu_base.lock),
}, },
};
#define migration_base migration_cpu_base.clock_base[0]
static inline bool is_migration_base(struct hrtimer_clock_base *base)
{
return base == &migration_base;
}
/*
* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
* means that all timers which are tied to this base via timer->base are
* locked, and the base itself is locked too.
*
* So __run_timers/migrate_timers can safely modify all timers which could
* be found on the lists/queues.
*
* When the timer's base is locked, and the timer removed from list, it is
* possible to set timer->base = &migration_base and drop the lock: the timer
* remains locked.
*/
static
struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
unsigned long *flags)
__acquires(&timer->base->lock)
{
struct hrtimer_clock_base *base;
for (;;) {
base = READ_ONCE(timer->base);
if (likely(base != &migration_base)) {
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
if (likely(base == timer->base))
return base;
/* The timer has migrated to another CPU: */
raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
}
cpu_relax();
}
}
/*
* We do not migrate the timer when it is expiring before the next
* event on the target cpu. When high resolution is enabled, we cannot
* reprogram the target cpu hardware and we would cause it to fire
* late. To keep it simple, we handle the high resolution enabled and
* disabled case similar.
*
* Called with cpu_base->lock of target cpu held.
*/
static int
hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
{
ktime_t expires;
expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
return expires < new_base->cpu_base->expires_next;
}
static inline
struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
int pinned)
{
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
if (static_branch_likely(&timers_migration_enabled) && !pinned) return &per_cpu(hrtimer_bases, get_nohz_timer_target());
#endif
return base;
}
/*
* We switch the timer base to a power-optimized selected CPU target,
* if:
* - NO_HZ_COMMON is enabled
* - timer migration is enabled
* - the timer callback is not running
* - the timer is not the first expiring timer on the new target
*
* If one of the above requirements is not fulfilled we move the timer
* to the current CPU or leave it on the previously assigned CPU if
* the timer callback is currently running.
*/
static inline struct hrtimer_clock_base *
switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
int pinned)
{
struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
struct hrtimer_clock_base *new_base;
int basenum = base->index;
this_cpu_base = this_cpu_ptr(&hrtimer_bases); new_cpu_base = get_target_base(this_cpu_base, pinned);
again:
new_base = &new_cpu_base->clock_base[basenum];
if (base != new_base) {
/*
* We are trying to move timer to new_base.
* However we can't change timer's base while it is running,
* so we keep it on the same CPU. No hassle vs. reprogramming
* the event source in the high resolution case. The softirq
* code will take care of this when the timer function has
* completed. There is no conflict as we hold the lock until
* the timer is enqueued.
*/
if (unlikely(hrtimer_callback_running(timer)))
return base;
/* See the comment in lock_hrtimer_base() */
WRITE_ONCE(timer->base, &migration_base);
raw_spin_unlock(&base->cpu_base->lock);
raw_spin_lock(&new_base->cpu_base->lock);
if (new_cpu_base != this_cpu_base &&
hrtimer_check_target(timer, new_base)) { raw_spin_unlock(&new_base->cpu_base->lock);
raw_spin_lock(&base->cpu_base->lock);
new_cpu_base = this_cpu_base;
WRITE_ONCE(timer->base, base);
goto again;
}
WRITE_ONCE(timer->base, new_base);
} else {
if (new_cpu_base != this_cpu_base && hrtimer_check_target(timer, new_base)) {
new_cpu_base = this_cpu_base;
goto again;
}
}
return new_base;
}
#else /* CONFIG_SMP */
static inline bool is_migration_base(struct hrtimer_clock_base *base)
{
return false;
}
static inline struct hrtimer_clock_base *
lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
__acquires(&timer->base->cpu_base->lock)
{
struct hrtimer_clock_base *base = timer->base;
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
return base;
}
# define switch_hrtimer_base(t, b, p) (b)
#endif /* !CONFIG_SMP */
/*
* Functions for the union type storage format of ktime_t which are
* too large for inlining:
*/
#if BITS_PER_LONG < 64
/*
* Divide a ktime value by a nanosecond value
*/
s64 __ktime_divns(const ktime_t kt, s64 div)
{
int sft = 0;
s64 dclc;
u64 tmp;
dclc = ktime_to_ns(kt);
tmp = dclc < 0 ? -dclc : dclc;
/* Make sure the divisor is less than 2^32: */
while (div >> 32) {
sft++;
div >>= 1;
}
tmp >>= sft;
do_div(tmp, (u32) div);
return dclc < 0 ? -tmp : tmp;
}
EXPORT_SYMBOL_GPL(__ktime_divns);
#endif /* BITS_PER_LONG >= 64 */
/*
* Add two ktime values and do a safety check for overflow:
*/
ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
{
ktime_t res = ktime_add_unsafe(lhs, rhs);
/*
* We use KTIME_SEC_MAX here, the maximum timeout which we can
* return to user space in a timespec:
*/
if (res < 0 || res < lhs || res < rhs)
res = ktime_set(KTIME_SEC_MAX, 0);
return res;
}
EXPORT_SYMBOL_GPL(ktime_add_safe);
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
static const struct debug_obj_descr hrtimer_debug_descr;
static void *hrtimer_debug_hint(void *addr)
{
return ((struct hrtimer *) addr)->function;
}
/*
* fixup_init is called when:
* - an active object is initialized
*/
static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
{
struct hrtimer *timer = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
hrtimer_cancel(timer);
debug_object_init(timer, &hrtimer_debug_descr);
return true;
default:
return false;
}
}
/*
* fixup_activate is called when:
* - an active object is activated
* - an unknown non-static object is activated
*/
static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
{
switch (state) {
case ODEBUG_STATE_ACTIVE:
WARN_ON(1);
fallthrough;
default:
return false;
}
}
/*
* fixup_free is called when:
* - an active object is freed
*/
static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
{
struct hrtimer *timer = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
hrtimer_cancel(timer);
debug_object_free(timer, &hrtimer_debug_descr);
return true;
default:
return false;
}
}
static const struct debug_obj_descr hrtimer_debug_descr = {
.name = "hrtimer",
.debug_hint = hrtimer_debug_hint,
.fixup_init = hrtimer_fixup_init,
.fixup_activate = hrtimer_fixup_activate,
.fixup_free = hrtimer_fixup_free,
};
static inline void debug_hrtimer_init(struct hrtimer *timer)
{
debug_object_init(timer, &hrtimer_debug_descr);
}
static inline void debug_hrtimer_activate(struct hrtimer *timer,
enum hrtimer_mode mode)
{
debug_object_activate(timer, &hrtimer_debug_descr);
}
static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
{
debug_object_deactivate(timer, &hrtimer_debug_descr);
}
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode);
void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode)
{
debug_object_init_on_stack(timer, &hrtimer_debug_descr);
__hrtimer_init(timer, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
clockid_t clock_id, enum hrtimer_mode mode);
void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
clockid_t clock_id, enum hrtimer_mode mode)
{
debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
__hrtimer_init_sleeper(sl, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
void destroy_hrtimer_on_stack(struct hrtimer *timer)
{
debug_object_free(timer, &hrtimer_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
#else
static inline void debug_hrtimer_init(struct hrtimer *timer) { }
static inline void debug_hrtimer_activate(struct hrtimer *timer,
enum hrtimer_mode mode) { }
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
#endif
static inline void
debug_init(struct hrtimer *timer, clockid_t clockid,
enum hrtimer_mode mode)
{
debug_hrtimer_init(timer); trace_hrtimer_init(timer, clockid, mode);
}
static inline void debug_activate(struct hrtimer *timer,
enum hrtimer_mode mode)
{
debug_hrtimer_activate(timer, mode); trace_hrtimer_start(timer, mode);
}
static inline void debug_deactivate(struct hrtimer *timer)
{
debug_hrtimer_deactivate(timer); trace_hrtimer_cancel(timer);
}
static struct hrtimer_clock_base *
__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
{
unsigned int idx;
if (!*active)
return NULL;
idx = __ffs(*active); *active &= ~(1U << idx);
return &cpu_base->clock_base[idx];
}
#define for_each_active_base(base, cpu_base, active) \
while ((base = __next_base((cpu_base), &(active))))
static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
const struct hrtimer *exclude,
unsigned int active,
ktime_t expires_next)
{
struct hrtimer_clock_base *base;
ktime_t expires;
for_each_active_base(base, cpu_base, active) {
struct timerqueue_node *next;
struct hrtimer *timer;
next = timerqueue_getnext(&base->active);
timer = container_of(next, struct hrtimer, node);
if (timer == exclude) {
/* Get to the next timer in the queue. */
next = timerqueue_iterate_next(next);
if (!next)
continue;
timer = container_of(next, struct hrtimer, node);
}
expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
if (expires < expires_next) {
expires_next = expires;
/* Skip cpu_base update if a timer is being excluded. */
if (exclude)
continue;
if (timer->is_soft) cpu_base->softirq_next_timer = timer;
else
cpu_base->next_timer = timer;
}
}
/*
* clock_was_set() might have changed base->offset of any of
* the clock bases so the result might be negative. Fix it up
* to prevent a false positive in clockevents_program_event().
*/
if (expires_next < 0)
expires_next = 0;
return expires_next;
}
/*
* Recomputes cpu_base::*next_timer and returns the earliest expires_next
* but does not set cpu_base::*expires_next, that is done by
* hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
* cpu_base::*expires_next right away, reprogramming logic would no longer
* work.
*
* When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
* those timers will get run whenever the softirq gets handled, at the end of
* hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
*
* Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
* The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
* softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
*
* @active_mask must be one of:
* - HRTIMER_ACTIVE_ALL,
* - HRTIMER_ACTIVE_SOFT, or
* - HRTIMER_ACTIVE_HARD.
*/
static ktime_t
__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
{
unsigned int active;
struct hrtimer *next_timer = NULL;
ktime_t expires_next = KTIME_MAX;
if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
cpu_base->softirq_next_timer = NULL;
expires_next = __hrtimer_next_event_base(cpu_base, NULL,
active, KTIME_MAX);
next_timer = cpu_base->softirq_next_timer;
}
if (active_mask & HRTIMER_ACTIVE_HARD) {
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
cpu_base->next_timer = next_timer;
expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
expires_next);
}
return expires_next;
}
static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
{
ktime_t expires_next, soft = KTIME_MAX;
/*
* If the soft interrupt has already been activated, ignore the
* soft bases. They will be handled in the already raised soft
* interrupt.
*/
if (!cpu_base->softirq_activated) { soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
/*
* Update the soft expiry time. clock_settime() might have
* affected it.
*/
cpu_base->softirq_expires_next = soft;
}
expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
/*
* If a softirq timer is expiring first, update cpu_base->next_timer
* and program the hardware with the soft expiry time.
*/
if (expires_next > soft) {
cpu_base->next_timer = cpu_base->softirq_next_timer;
expires_next = soft;
}
return expires_next;
}
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
{
ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
offs_real, offs_boot, offs_tai);
base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
return now;
}
/*
* Is the high resolution mode active ?
*/
static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
{
return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
cpu_base->hres_active : 0;
}
static inline int hrtimer_hres_active(void)
{
return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
}
static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
struct hrtimer *next_timer,
ktime_t expires_next)
{
cpu_base->expires_next = expires_next;
/*
* If hres is not active, hardware does not have to be
* reprogrammed yet.
*
* If a hang was detected in the last timer interrupt then we
* leave the hang delay active in the hardware. We want the
* system to make progress. That also prevents the following
* scenario:
* T1 expires 50ms from now
* T2 expires 5s from now
*
* T1 is removed, so this code is called and would reprogram
* the hardware to 5s from now. Any hrtimer_start after that
* will not reprogram the hardware due to hang_detected being
* set. So we'd effectively block all timers until the T2 event
* fires.
*/
if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
return;
tick_program_event(expires_next, 1);
}
/*
* Reprogram the event source with checking both queues for the
* next event
* Called with interrupts disabled and base->lock held
*/
static void
hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
{
ktime_t expires_next;
expires_next = hrtimer_update_next_event(cpu_base);
if (skip_equal && expires_next == cpu_base->expires_next)
return;
__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
}
/* High resolution timer related functions */
#ifdef CONFIG_HIGH_RES_TIMERS
/*
* High resolution timer enabled ?
*/
static bool hrtimer_hres_enabled __read_mostly = true;
unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
EXPORT_SYMBOL_GPL(hrtimer_resolution);
/*
* Enable / Disable high resolution mode
*/
static int __init setup_hrtimer_hres(char *str)
{
return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
}
__setup("highres=", setup_hrtimer_hres);
/*
* hrtimer_high_res_enabled - query, if the highres mode is enabled
*/
static inline int hrtimer_is_hres_enabled(void)
{
return hrtimer_hres_enabled;
}
static void retrigger_next_event(void *arg);
/*
* Switch to high resolution mode
*/
static void hrtimer_switch_to_hres(void)
{
struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
if (tick_init_highres()) {
pr_warn("Could not switch to high resolution mode on CPU %u\n",
base->cpu);
return;
}
base->hres_active = 1;
hrtimer_resolution = HIGH_RES_NSEC;
tick_setup_sched_timer(true);
/* "Retrigger" the interrupt to get things going */
retrigger_next_event(NULL);
}
#else
static inline int hrtimer_is_hres_enabled(void) { return 0; }
static inline void hrtimer_switch_to_hres(void) { }
#endif /* CONFIG_HIGH_RES_TIMERS */
/*
* Retrigger next event is called after clock was set with interrupts
* disabled through an SMP function call or directly from low level
* resume code.
*
* This is only invoked when:
* - CONFIG_HIGH_RES_TIMERS is enabled.
* - CONFIG_NOHZ_COMMON is enabled
*
* For the other cases this function is empty and because the call sites
* are optimized out it vanishes as well, i.e. no need for lots of
* #ifdeffery.
*/
static void retrigger_next_event(void *arg)
{
struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
/*
* When high resolution mode or nohz is active, then the offsets of
* CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the
* next tick will take care of that.
*
* If high resolution mode is active then the next expiring timer
* must be reevaluated and the clock event device reprogrammed if
* necessary.
*
* In the NOHZ case the update of the offset and the reevaluation
* of the next expiring timer is enough. The return from the SMP
* function call will take care of the reprogramming in case the
* CPU was in a NOHZ idle sleep.
*/
if (!__hrtimer_hres_active(base) && !tick_nohz_active)
return;
raw_spin_lock(&base->lock);
hrtimer_update_base(base);
if (__hrtimer_hres_active(base))
hrtimer_force_reprogram(base, 0);
else
hrtimer_update_next_event(base);
raw_spin_unlock(&base->lock);
}
/*
* When a timer is enqueued and expires earlier than the already enqueued
* timers, we have to check, whether it expires earlier than the timer for
* which the clock event device was armed.
*
* Called with interrupts disabled and base->cpu_base.lock held
*/
static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
struct hrtimer_clock_base *base = timer->base;
ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
/*
* CLOCK_REALTIME timer might be requested with an absolute
* expiry time which is less than base->offset. Set it to 0.
*/
if (expires < 0)
expires = 0;
if (timer->is_soft) {
/*
* soft hrtimer could be started on a remote CPU. In this
* case softirq_expires_next needs to be updated on the
* remote CPU. The soft hrtimer will not expire before the
* first hard hrtimer on the remote CPU -
* hrtimer_check_target() prevents this case.
*/
struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
if (timer_cpu_base->softirq_activated)
return;
if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
return;
timer_cpu_base->softirq_next_timer = timer;
timer_cpu_base->softirq_expires_next = expires;
if (!ktime_before(expires, timer_cpu_base->expires_next) ||
!reprogram)
return;
}
/*
* If the timer is not on the current cpu, we cannot reprogram
* the other cpus clock event device.
*/
if (base->cpu_base != cpu_base)
return;
if (expires >= cpu_base->expires_next)
return;
/*
* If the hrtimer interrupt is running, then it will reevaluate the
* clock bases and reprogram the clock event device.
*/
if (cpu_base->in_hrtirq)
return;
cpu_base->next_timer = timer; __hrtimer_reprogram(cpu_base, timer, expires);
}
static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
unsigned int active)
{
struct hrtimer_clock_base *base;
unsigned int seq;
ktime_t expires;
/*
* Update the base offsets unconditionally so the following
* checks whether the SMP function call is required works.
*
* The update is safe even when the remote CPU is in the hrtimer
* interrupt or the hrtimer soft interrupt and expiring affected
* bases. Either it will see the update before handling a base or
* it will see it when it finishes the processing and reevaluates
* the next expiring timer.
*/
seq = cpu_base->clock_was_set_seq;
hrtimer_update_base(cpu_base);
/*
* If the sequence did not change over the update then the
* remote CPU already handled it.
*/
if (seq == cpu_base->clock_was_set_seq)
return false;
/*
* If the remote CPU is currently handling an hrtimer interrupt, it
* will reevaluate the first expiring timer of all clock bases
* before reprogramming. Nothing to do here.
*/
if (cpu_base->in_hrtirq)
return false;
/*
* Walk the affected clock bases and check whether the first expiring
* timer in a clock base is moving ahead of the first expiring timer of
* @cpu_base. If so, the IPI must be invoked because per CPU clock
* event devices cannot be remotely reprogrammed.
*/
active &= cpu_base->active_bases;
for_each_active_base(base, cpu_base, active) {
struct timerqueue_node *next;
next = timerqueue_getnext(&base->active);
expires = ktime_sub(next->expires, base->offset);
if (expires < cpu_base->expires_next)
return true;
/* Extra check for softirq clock bases */
if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT)
continue;
if (cpu_base->softirq_activated)
continue;
if (expires < cpu_base->softirq_expires_next)
return true;
}
return false;
}
/*
* Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and
* CLOCK_BOOTTIME (for late sleep time injection).
*
* This requires to update the offsets for these clocks
* vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this
* also requires to eventually reprogram the per CPU clock event devices
* when the change moves an affected timer ahead of the first expiring
* timer on that CPU. Obviously remote per CPU clock event devices cannot
* be reprogrammed. The other reason why an IPI has to be sent is when the
* system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets
* in the tick, which obviously might be stopped, so this has to bring out
* the remote CPU which might sleep in idle to get this sorted.
*/
void clock_was_set(unsigned int bases)
{
struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);
cpumask_var_t mask;
int cpu;
if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)
goto out_timerfd;
if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
on_each_cpu(retrigger_next_event, NULL, 1);
goto out_timerfd;
}
/* Avoid interrupting CPUs if possible */
cpus_read_lock();
for_each_online_cpu(cpu) {
unsigned long flags;
cpu_base = &per_cpu(hrtimer_bases, cpu);
raw_spin_lock_irqsave(&cpu_base->lock, flags);
if (update_needs_ipi(cpu_base, bases))
cpumask_set_cpu(cpu, mask);
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
}
preempt_disable();
smp_call_function_many(mask, retrigger_next_event, NULL, 1);
preempt_enable();
cpus_read_unlock();
free_cpumask_var(mask);
out_timerfd:
timerfd_clock_was_set();
}
static void clock_was_set_work(struct work_struct *work)
{
clock_was_set(CLOCK_SET_WALL);
}
static DECLARE_WORK(hrtimer_work, clock_was_set_work);
/*
* Called from timekeeping code to reprogram the hrtimer interrupt device
* on all cpus and to notify timerfd.
*/
void clock_was_set_delayed(void)
{
schedule_work(&hrtimer_work);
}
/*
* Called during resume either directly from via timekeeping_resume()
* or in the case of s2idle from tick_unfreeze() to ensure that the
* hrtimers are up to date.
*/
void hrtimers_resume_local(void)
{
lockdep_assert_irqs_disabled();
/* Retrigger on the local CPU */
retrigger_next_event(NULL);
}
/*
* Counterpart to lock_hrtimer_base above:
*/
static inline
void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
__releases(&timer->base->cpu_base->lock)
{
raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
}
/**
* hrtimer_forward() - forward the timer expiry
* @timer: hrtimer to forward
* @now: forward past this time
* @interval: the interval to forward
*
* Forward the timer expiry so it will expire in the future.
*
* .. note::
* This only updates the timer expiry value and does not requeue the timer.
*
* There is also a variant of the function hrtimer_forward_now().
*
* Context: Can be safely called from the callback function of @timer. If called
* from other contexts @timer must neither be enqueued nor running the
* callback and the caller needs to take care of serialization.
*
* Return: The number of overruns are returned.
*/
u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
{
u64 orun = 1;
ktime_t delta;
delta = ktime_sub(now, hrtimer_get_expires(timer));
if (delta < 0)
return 0;
if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
return 0;
if (interval < hrtimer_resolution)
interval = hrtimer_resolution;
if (unlikely(delta >= interval)) {
s64 incr = ktime_to_ns(interval);
orun = ktime_divns(delta, incr);
hrtimer_add_expires_ns(timer, incr * orun);
if (hrtimer_get_expires_tv64(timer) > now)
return orun;
/*
* This (and the ktime_add() below) is the
* correction for exact:
*/
orun++;
}
hrtimer_add_expires(timer, interval);
return orun;
}
EXPORT_SYMBOL_GPL(hrtimer_forward);
/*
* enqueue_hrtimer - internal function to (re)start a timer
*
* The timer is inserted in expiry order. Insertion into the
* red black tree is O(log(n)). Must hold the base lock.
*
* Returns 1 when the new timer is the leftmost timer in the tree.
*/
static int enqueue_hrtimer(struct hrtimer *timer,
struct hrtimer_clock_base *base,
enum hrtimer_mode mode)
{
debug_activate(timer, mode); WARN_ON_ONCE(!base->cpu_base->online); base->cpu_base->active_bases |= 1 << base->index;
/* Pairs with the lockless read in hrtimer_is_queued() */
WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);
return timerqueue_add(&base->active, &timer->node);
}
/*
* __remove_hrtimer - internal function to remove a timer
*
* Caller must hold the base lock.
*
* High resolution timer mode reprograms the clock event device when the
* timer is the one which expires next. The caller can disable this by setting
* reprogram to zero. This is useful, when the context does a reprogramming
* anyway (e.g. timer interrupt)
*/
static void __remove_hrtimer(struct hrtimer *timer,
struct hrtimer_clock_base *base,
u8 newstate, int reprogram)
{
struct hrtimer_cpu_base *cpu_base = base->cpu_base;
u8 state = timer->state;
/* Pairs with the lockless read in hrtimer_is_queued() */
WRITE_ONCE(timer->state, newstate);
if (!(state & HRTIMER_STATE_ENQUEUED))
return;
if (!timerqueue_del(&base->active, &timer->node)) cpu_base->active_bases &= ~(1 << base->index);
/*
* Note: If reprogram is false we do not update
* cpu_base->next_timer. This happens when we remove the first
* timer on a remote cpu. No harm as we never dereference
* cpu_base->next_timer. So the worst thing what can happen is
* an superfluous call to hrtimer_force_reprogram() on the
* remote cpu later on if the same timer gets enqueued again.
*/
if (reprogram && timer == cpu_base->next_timer) hrtimer_force_reprogram(cpu_base, 1);
}
/*
* remove hrtimer, called with base lock held
*/
static inline int
remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
bool restart, bool keep_local)
{
u8 state = timer->state;
if (state & HRTIMER_STATE_ENQUEUED) {
bool reprogram;
/*
* Remove the timer and force reprogramming when high
* resolution mode is active and the timer is on the current
* CPU. If we remove a timer on another CPU, reprogramming is
* skipped. The interrupt event on this CPU is fired and
* reprogramming happens in the interrupt handler. This is a
* rare case and less expensive than a smp call.
*/
debug_deactivate(timer); reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
/*
* If the timer is not restarted then reprogramming is
* required if the timer is local. If it is local and about
* to be restarted, avoid programming it twice (on removal
* and a moment later when it's requeued).
*/
if (!restart)
state = HRTIMER_STATE_INACTIVE;
else
reprogram &= !keep_local;
__remove_hrtimer(timer, base, state, reprogram);
return 1;
}
return 0;
}
static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
const enum hrtimer_mode mode)
{
#ifdef CONFIG_TIME_LOW_RES
/*
* CONFIG_TIME_LOW_RES indicates that the system has no way to return
* granular time values. For relative timers we add hrtimer_resolution
* (i.e. one jiffie) to prevent short timeouts.
*/
timer->is_rel = mode & HRTIMER_MODE_REL;
if (timer->is_rel)
tim = ktime_add_safe(tim, hrtimer_resolution);
#endif
return tim;
}
static void
hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
{
ktime_t expires;
/*
* Find the next SOFT expiration.
*/
expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
/*
* reprogramming needs to be triggered, even if the next soft
* hrtimer expires at the same time than the next hard
* hrtimer. cpu_base->softirq_expires_next needs to be updated!
*/
if (expires == KTIME_MAX)
return;
/*
* cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
* cpu_base->*expires_next is only set by hrtimer_reprogram()
*/
hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
}
static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
u64 delta_ns, const enum hrtimer_mode mode,
struct hrtimer_clock_base *base)
{
struct hrtimer_clock_base *new_base;
bool force_local, first;
/*
* If the timer is on the local cpu base and is the first expiring
* timer then this might end up reprogramming the hardware twice
* (on removal and on enqueue). To avoid that by prevent the
* reprogram on removal, keep the timer local to the current CPU
* and enforce reprogramming after it is queued no matter whether
* it is the new first expiring timer again or not.
*/
force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
force_local &= base->cpu_base->next_timer == timer;
/*
* Remove an active timer from the queue. In case it is not queued
* on the current CPU, make sure that remove_hrtimer() updates the
* remote data correctly.
*
* If it's on the current CPU and the first expiring timer, then
* skip reprogramming, keep the timer local and enforce
* reprogramming later if it was the first expiring timer. This
* avoids programming the underlying clock event twice (once at
* removal and once after enqueue).
*/
remove_hrtimer(timer, base, true, force_local); if (mode & HRTIMER_MODE_REL) tim = ktime_add_safe(tim, base->get_time());
tim = hrtimer_update_lowres(timer, tim, mode);
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
/* Switch the timer base, if necessary: */
if (!force_local) {
new_base = switch_hrtimer_base(timer, base,
mode & HRTIMER_MODE_PINNED);
} else {
new_base = base;
}
first = enqueue_hrtimer(timer, new_base, mode);
if (!force_local)
return first;
/*
* Timer was forced to stay on the current CPU to avoid
* reprogramming on removal and enqueue. Force reprogram the
* hardware by evaluating the new first expiring timer.
*/
hrtimer_force_reprogram(new_base->cpu_base, 1);
return 0;
}
/**
* hrtimer_start_range_ns - (re)start an hrtimer
* @timer: the timer to be added
* @tim: expiry time
* @delta_ns: "slack" range for the timer
* @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
* relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
* softirq based mode is considered for debug purpose only!
*/
void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
u64 delta_ns, const enum hrtimer_mode mode)
{
struct hrtimer_clock_base *base;
unsigned long flags;
/*
* Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
* match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
* expiry mode because unmarked timers are moved to softirq expiry.
*/
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
else
WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
base = lock_hrtimer_base(timer, &flags); if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) hrtimer_reprogram(timer, true); unlock_hrtimer_base(timer, &flags);
}
EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
/**
* hrtimer_try_to_cancel - try to deactivate a timer
* @timer: hrtimer to stop
*
* Returns:
*
* * 0 when the timer was not active
* * 1 when the timer was active
* * -1 when the timer is currently executing the callback function and
* cannot be stopped
*/
int hrtimer_try_to_cancel(struct hrtimer *timer)
{
struct hrtimer_clock_base *base;
unsigned long flags;
int ret = -1;
/*
* Check lockless first. If the timer is not active (neither
* enqueued nor running the callback, nothing to do here. The
* base lock does not serialize against a concurrent enqueue,
* so we can avoid taking it.
*/
if (!hrtimer_active(timer))
return 0;
base = lock_hrtimer_base(timer, &flags); if (!hrtimer_callback_running(timer)) ret = remove_hrtimer(timer, base, false, false); unlock_hrtimer_base(timer, &flags);
return ret;
}
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
#ifdef CONFIG_PREEMPT_RT
static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
{
spin_lock_init(&base->softirq_expiry_lock);
}
static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
{
spin_lock(&base->softirq_expiry_lock);
}
static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
{
spin_unlock(&base->softirq_expiry_lock);
}
/*
* The counterpart to hrtimer_cancel_wait_running().
*
* If there is a waiter for cpu_base->expiry_lock, then it was waiting for
* the timer callback to finish. Drop expiry_lock and reacquire it. That
* allows the waiter to acquire the lock and make progress.
*/
static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
unsigned long flags)
{
if (atomic_read(&cpu_base->timer_waiters)) {
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
spin_unlock(&cpu_base->softirq_expiry_lock);
spin_lock(&cpu_base->softirq_expiry_lock);
raw_spin_lock_irq(&cpu_base->lock);
}
}
/*
* This function is called on PREEMPT_RT kernels when the fast path
* deletion of a timer failed because the timer callback function was
* running.
*
* This prevents priority inversion: if the soft irq thread is preempted
* in the middle of a timer callback, then calling del_timer_sync() can
* lead to two issues:
*
* - If the caller is on a remote CPU then it has to spin wait for the timer
* handler to complete. This can result in unbound priority inversion.
*
* - If the caller originates from the task which preempted the timer
* handler on the same CPU, then spin waiting for the timer handler to
* complete is never going to end.
*/
void hrtimer_cancel_wait_running(const struct hrtimer *timer)
{
/* Lockless read. Prevent the compiler from reloading it below */
struct hrtimer_clock_base *base = READ_ONCE(timer->base);
/*
* Just relax if the timer expires in hard interrupt context or if
* it is currently on the migration base.
*/
if (!timer->is_soft || is_migration_base(base)) {
cpu_relax();
return;
}
/*
* Mark the base as contended and grab the expiry lock, which is
* held by the softirq across the timer callback. Drop the lock
* immediately so the softirq can expire the next timer. In theory
* the timer could already be running again, but that's more than
* unlikely and just causes another wait loop.
*/
atomic_inc(&base->cpu_base->timer_waiters);
spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
atomic_dec(&base->cpu_base->timer_waiters);
spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
}
#else
static inline void
hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
static inline void
hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
static inline void
hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
unsigned long flags) { }
#endif
/**
* hrtimer_cancel - cancel a timer and wait for the handler to finish.
* @timer: the timer to be cancelled
*
* Returns:
* 0 when the timer was not active
* 1 when the timer was active
*/
int hrtimer_cancel(struct hrtimer *timer)
{
int ret;
do {
ret = hrtimer_try_to_cancel(timer);
if (ret < 0)
hrtimer_cancel_wait_running(timer);
} while (ret < 0);
return ret;
}
EXPORT_SYMBOL_GPL(hrtimer_cancel);
/**
* __hrtimer_get_remaining - get remaining time for the timer
* @timer: the timer to read
* @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y
*/
ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
{
unsigned long flags;
ktime_t rem;
lock_hrtimer_base(timer, &flags);
if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
rem = hrtimer_expires_remaining_adjusted(timer);
else
rem = hrtimer_expires_remaining(timer);
unlock_hrtimer_base(timer, &flags);
return rem;
}
EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
#ifdef CONFIG_NO_HZ_COMMON
/**
* hrtimer_get_next_event - get the time until next expiry event
*
* Returns the next expiry time or KTIME_MAX if no timer is pending.
*/
u64 hrtimer_get_next_event(void)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
u64 expires = KTIME_MAX;
unsigned long flags;
raw_spin_lock_irqsave(&cpu_base->lock, flags);
if (!__hrtimer_hres_active(cpu_base))
expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
return expires;
}
/**
* hrtimer_next_event_without - time until next expiry event w/o one timer
* @exclude: timer to exclude
*
* Returns the next expiry time over all timers except for the @exclude one or
* KTIME_MAX if none of them is pending.
*/
u64 hrtimer_next_event_without(const struct hrtimer *exclude)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
u64 expires = KTIME_MAX;
unsigned long flags;
raw_spin_lock_irqsave(&cpu_base->lock, flags);
if (__hrtimer_hres_active(cpu_base)) {
unsigned int active;
if (!cpu_base->softirq_activated) {
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
expires = __hrtimer_next_event_base(cpu_base, exclude,
active, KTIME_MAX);
}
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
expires = __hrtimer_next_event_base(cpu_base, exclude, active,
expires);
}
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
return expires;
}
#endif
static inline int hrtimer_clockid_to_base(clockid_t clock_id)
{
if (likely(clock_id < MAX_CLOCKS)) { int base = hrtimer_clock_to_base_table[clock_id];
if (likely(base != HRTIMER_MAX_CLOCK_BASES))
return base;
}
WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
return HRTIMER_BASE_MONOTONIC;
}
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode)
{
bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
struct hrtimer_cpu_base *cpu_base;
int base;
/*
* On PREEMPT_RT enabled kernels hrtimers which are not explicitly
* marked for hard interrupt expiry mode are moved into soft
* interrupt context for latency reasons and because the callbacks
* can invoke functions which might sleep on RT, e.g. spin_lock().
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
softtimer = true;
memset(timer, 0, sizeof(struct hrtimer));
cpu_base = raw_cpu_ptr(&hrtimer_bases);
/*
* POSIX magic: Relative CLOCK_REALTIME timers are not affected by
* clock modifications, so they needs to become CLOCK_MONOTONIC to
* ensure POSIX compliance.
*/
if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
clock_id = CLOCK_MONOTONIC;
base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
base += hrtimer_clockid_to_base(clock_id); timer->is_soft = softtimer;
timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
timer->base = &cpu_base->clock_base[base];
timerqueue_init(&timer->node);
}
/**
* hrtimer_init - initialize a timer to the given clock
* @timer: the timer to be initialized
* @clock_id: the clock to be used
* @mode: The modes which are relevant for initialization:
* HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
* HRTIMER_MODE_REL_SOFT
*
* The PINNED variants of the above can be handed in,
* but the PINNED bit is ignored as pinning happens
* when the hrtimer is started
*/
void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode)
{
debug_init(timer, clock_id, mode); __hrtimer_init(timer, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_init);
/*
* A timer is active, when it is enqueued into the rbtree or the
* callback function is running or it's in the state of being migrated
* to another cpu.
*
* It is important for this function to not return a false negative.
*/
bool hrtimer_active(const struct hrtimer *timer)
{
struct hrtimer_clock_base *base;
unsigned int seq;
do {
base = READ_ONCE(timer->base); seq = raw_read_seqcount_begin(&base->seq);
if (timer->state != HRTIMER_STATE_INACTIVE ||
base->running == timer) return true; } while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base));
return false;
}
EXPORT_SYMBOL_GPL(hrtimer_active);
/*
* The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
* distinct sections:
*
* - queued: the timer is queued
* - callback: the timer is being ran
* - post: the timer is inactive or (re)queued
*
* On the read side we ensure we observe timer->state and cpu_base->running
* from the same section, if anything changed while we looked at it, we retry.
* This includes timer->base changing because sequence numbers alone are
* insufficient for that.
*
* The sequence numbers are required because otherwise we could still observe
* a false negative if the read side got smeared over multiple consecutive
* __run_hrtimer() invocations.
*/
static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
struct hrtimer_clock_base *base,
struct hrtimer *timer, ktime_t *now,
unsigned long flags) __must_hold(&cpu_base->lock)
{
enum hrtimer_restart (*fn)(struct hrtimer *);
bool expires_in_hardirq;
int restart;
lockdep_assert_held(&cpu_base->lock); debug_deactivate(timer); base->running = timer;
/*
* Separate the ->running assignment from the ->state assignment.
*
* As with a regular write barrier, this ensures the read side in
* hrtimer_active() cannot observe base->running == NULL &&
* timer->state == INACTIVE.
*/
raw_write_seqcount_barrier(&base->seq);
__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
fn = timer->function;
/*
* Clear the 'is relative' flag for the TIME_LOW_RES case. If the
* timer is restarted with a period then it becomes an absolute
* timer. If its not restarted it does not matter.
*/
if (IS_ENABLED(CONFIG_TIME_LOW_RES))
timer->is_rel = false;
/*
* The timer is marked as running in the CPU base, so it is
* protected against migration to a different CPU even if the lock
* is dropped.
*/
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
trace_hrtimer_expire_entry(timer, now); expires_in_hardirq = lockdep_hrtimer_enter(timer); restart = fn(timer);
lockdep_hrtimer_exit(expires_in_hardirq);
trace_hrtimer_expire_exit(timer); raw_spin_lock_irq(&cpu_base->lock);
/*
* Note: We clear the running state after enqueue_hrtimer and
* we do not reprogram the event hardware. Happens either in
* hrtimer_start_range_ns() or in hrtimer_interrupt()
*
* Note: Because we dropped the cpu_base->lock above,
* hrtimer_start_range_ns() can have popped in and enqueued the timer
* for us already.
*/
if (restart != HRTIMER_NORESTART &&
!(timer->state & HRTIMER_STATE_ENQUEUED)) enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
/*
* Separate the ->running assignment from the ->state assignment.
*
* As with a regular write barrier, this ensures the read side in
* hrtimer_active() cannot observe base->running.timer == NULL &&
* timer->state == INACTIVE.
*/
raw_write_seqcount_barrier(&base->seq); WARN_ON_ONCE(base->running != timer); base->running = NULL;
}
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
unsigned long flags, unsigned int active_mask)
{
struct hrtimer_clock_base *base;
unsigned int active = cpu_base->active_bases & active_mask; for_each_active_base(base, cpu_base, active) {
struct timerqueue_node *node;
ktime_t basenow;
basenow = ktime_add(now, base->offset);
while ((node = timerqueue_getnext(&base->active))) {
struct hrtimer *timer;
timer = container_of(node, struct hrtimer, node);
/*
* The immediate goal for using the softexpires is
* minimizing wakeups, not running timers at the
* earliest interrupt after their soft expiration.
* This allows us to avoid using a Priority Search
* Tree, which can answer a stabbing query for
* overlapping intervals and instead use the simple
* BST we already have.
* We don't add extra wakeups by delaying timers that
* are right-of a not yet expired timer, because that
* timer will have to trigger a wakeup anyway.
*/
if (basenow < hrtimer_get_softexpires_tv64(timer))
break;
__run_hrtimer(cpu_base, base, timer, &basenow, flags);
if (active_mask == HRTIMER_ACTIVE_SOFT)
hrtimer_sync_wait_running(cpu_base, flags);
}
}
}
static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
unsigned long flags;
ktime_t now;
hrtimer_cpu_base_lock_expiry(cpu_base);
raw_spin_lock_irqsave(&cpu_base->lock, flags);
now = hrtimer_update_base(cpu_base);
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
cpu_base->softirq_activated = 0;
hrtimer_update_softirq_timer(cpu_base, true);
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
hrtimer_cpu_base_unlock_expiry(cpu_base);
}
#ifdef CONFIG_HIGH_RES_TIMERS
/*
* High resolution timer interrupt
* Called with interrupts disabled
*/
void hrtimer_interrupt(struct clock_event_device *dev)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
ktime_t expires_next, now, entry_time, delta;
unsigned long flags;
int retries = 0;
BUG_ON(!cpu_base->hres_active); cpu_base->nr_events++;
dev->next_event = KTIME_MAX;
raw_spin_lock_irqsave(&cpu_base->lock, flags);
entry_time = now = hrtimer_update_base(cpu_base);
retry:
cpu_base->in_hrtirq = 1;
/*
* We set expires_next to KTIME_MAX here with cpu_base->lock
* held to prevent that a timer is enqueued in our queue via
* the migration code. This does not affect enqueueing of
* timers which run their callback and need to be requeued on
* this CPU.
*/
cpu_base->expires_next = KTIME_MAX;
if (!ktime_before(now, cpu_base->softirq_expires_next)) {
cpu_base->softirq_expires_next = KTIME_MAX;
cpu_base->softirq_activated = 1;
raise_softirq_irqoff(HRTIMER_SOFTIRQ);
}
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
/* Reevaluate the clock bases for the [soft] next expiry */
expires_next = hrtimer_update_next_event(cpu_base);
/*
* Store the new expiry value so the migration code can verify
* against it.
*/
cpu_base->expires_next = expires_next;
cpu_base->in_hrtirq = 0;
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
/* Reprogramming necessary ? */
if (!tick_program_event(expires_next, 0)) {
cpu_base->hang_detected = 0;
return;
}
/*
* The next timer was already expired due to:
* - tracing
* - long lasting callbacks
* - being scheduled away when running in a VM
*
* We need to prevent that we loop forever in the hrtimer
* interrupt routine. We give it 3 attempts to avoid
* overreacting on some spurious event.
*
* Acquire base lock for updating the offsets and retrieving
* the current time.
*/
raw_spin_lock_irqsave(&cpu_base->lock, flags);
now = hrtimer_update_base(cpu_base);
cpu_base->nr_retries++;
if (++retries < 3)
goto retry;
/*
* Give the system a chance to do something else than looping
* here. We stored the entry time, so we know exactly how long
* we spent here. We schedule the next event this amount of
* time away.
*/
cpu_base->nr_hangs++;
cpu_base->hang_detected = 1;
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
delta = ktime_sub(now, entry_time);
if ((unsigned int)delta > cpu_base->max_hang_time)
cpu_base->max_hang_time = (unsigned int) delta;
/*
* Limit it to a sensible value as we enforce a longer
* delay. Give the CPU at least 100ms to catch up.
*/
if (delta > 100 * NSEC_PER_MSEC) expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
else
expires_next = ktime_add(now, delta); tick_program_event(expires_next, 1); pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
}
/* called with interrupts disabled */
static inline void __hrtimer_peek_ahead_timers(void)
{
struct tick_device *td;
if (!hrtimer_hres_active())
return;
td = this_cpu_ptr(&tick_cpu_device);
if (td && td->evtdev)
hrtimer_interrupt(td->evtdev);
}
#else /* CONFIG_HIGH_RES_TIMERS */
static inline void __hrtimer_peek_ahead_timers(void) { }
#endif /* !CONFIG_HIGH_RES_TIMERS */
/*
* Called from run_local_timers in hardirq context every jiffy
*/
void hrtimer_run_queues(void)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
unsigned long flags;
ktime_t now;
if (__hrtimer_hres_active(cpu_base))
return;
/*
* This _is_ ugly: We have to check periodically, whether we
* can switch to highres and / or nohz mode. The clocksource
* switch happens with xtime_lock held. Notification from
* there only sets the check bit in the tick_oneshot code,
* otherwise we might deadlock vs. xtime_lock.
*/
if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) { hrtimer_switch_to_hres();
return;
}
raw_spin_lock_irqsave(&cpu_base->lock, flags);
now = hrtimer_update_base(cpu_base);
if (!ktime_before(now, cpu_base->softirq_expires_next)) {
cpu_base->softirq_expires_next = KTIME_MAX;
cpu_base->softirq_activated = 1;
raise_softirq_irqoff(HRTIMER_SOFTIRQ);
}
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
}
/*
* Sleep related functions:
*/
static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
{
struct hrtimer_sleeper *t =
container_of(timer, struct hrtimer_sleeper, timer);
struct task_struct *task = t->task;
t->task = NULL;
if (task)
wake_up_process(task); return HRTIMER_NORESTART;
}
/**
* hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
* @sl: sleeper to be started
* @mode: timer mode abs/rel
*
* Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
* to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
*/
void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
enum hrtimer_mode mode)
{
/*
* Make the enqueue delivery mode check work on RT. If the sleeper
* was initialized for hard interrupt delivery, force the mode bit.
* This is a special case for hrtimer_sleepers because
* hrtimer_init_sleeper() determines the delivery mode on RT so the
* fiddling with this decision is avoided at the call sites.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
mode |= HRTIMER_MODE_HARD;
hrtimer_start_expires(&sl->timer, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
clockid_t clock_id, enum hrtimer_mode mode)
{
/*
* On PREEMPT_RT enabled kernels hrtimers which are not explicitly
* marked for hard interrupt expiry mode are moved into soft
* interrupt context either for latency reasons or because the
* hrtimer callback takes regular spinlocks or invokes other
* functions which are not suitable for hard interrupt context on
* PREEMPT_RT.
*
* The hrtimer_sleeper callback is RT compatible in hard interrupt
* context, but there is a latency concern: Untrusted userspace can
* spawn many threads which arm timers for the same expiry time on
* the same CPU. That causes a latency spike due to the wakeup of
* a gazillion threads.
*
* OTOH, privileged real-time user space applications rely on the
* low latency of hard interrupt wakeups. If the current task is in
* a real-time scheduling class, mark the mode for hard interrupt
* expiry.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
mode |= HRTIMER_MODE_HARD;
}
__hrtimer_init(&sl->timer, clock_id, mode);
sl->timer.function = hrtimer_wakeup;
sl->task = current;
}
/**
* hrtimer_init_sleeper - initialize sleeper to the given clock
* @sl: sleeper to be initialized
* @clock_id: the clock to be used
* @mode: timer mode abs/rel
*/
void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
enum hrtimer_mode mode)
{
debug_init(&sl->timer, clock_id, mode);
__hrtimer_init_sleeper(sl, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
{
switch(restart->nanosleep.type) {
#ifdef CONFIG_COMPAT_32BIT_TIME
case TT_COMPAT:
if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
return -EFAULT;
break;
#endif
case TT_NATIVE:
if (put_timespec64(ts, restart->nanosleep.rmtp))
return -EFAULT;
break;
default:
BUG();
}
return -ERESTART_RESTARTBLOCK;
}
static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
{
struct restart_block *restart;
do {
set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); hrtimer_sleeper_start_expires(t, mode);
if (likely(t->task))
schedule(); hrtimer_cancel(&t->timer);
mode = HRTIMER_MODE_ABS;
} while (t->task && !signal_pending(current)); __set_current_state(TASK_RUNNING);
if (!t->task)
return 0;
restart = ¤t->restart_block;
if (restart->nanosleep.type != TT_NONE) {
ktime_t rem = hrtimer_expires_remaining(&t->timer);
struct timespec64 rmt;
if (rem <= 0)
return 0;
rmt = ktime_to_timespec64(rem);
return nanosleep_copyout(restart, &rmt);
}
return -ERESTART_RESTARTBLOCK;
}
static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
{
struct hrtimer_sleeper t;
int ret;
hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
HRTIMER_MODE_ABS);
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
destroy_hrtimer_on_stack(&t.timer);
return ret;
}
long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
const clockid_t clockid)
{
struct restart_block *restart;
struct hrtimer_sleeper t;
int ret = 0;
u64 slack;
slack = current->timer_slack_ns;
if (rt_task(current))
slack = 0;
hrtimer_init_sleeper_on_stack(&t, clockid, mode); hrtimer_set_expires_range_ns(&t.timer, rqtp, slack);
ret = do_nanosleep(&t, mode);
if (ret != -ERESTART_RESTARTBLOCK)
goto out;
/* Absolute timers do not update the rmtp value and restart: */
if (mode == HRTIMER_MODE_ABS) {
ret = -ERESTARTNOHAND;
goto out;
}
restart = ¤t->restart_block;
restart->nanosleep.clockid = t.timer.base->clockid;
restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
set_restart_fn(restart, hrtimer_nanosleep_restart);
out:
destroy_hrtimer_on_stack(&t.timer); return ret;
}
#ifdef CONFIG_64BIT
SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
struct __kernel_timespec __user *, rmtp)
{
struct timespec64 tu;
if (get_timespec64(&tu, rqtp))
return -EFAULT;
if (!timespec64_valid(&tu))
return -EINVAL;
current->restart_block.fn = do_no_restart_syscall;
current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
current->restart_block.nanosleep.rmtp = rmtp;
return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
CLOCK_MONOTONIC);
}
#endif
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
struct old_timespec32 __user *, rmtp)
{
struct timespec64 tu;
if (get_old_timespec32(&tu, rqtp))
return -EFAULT;
if (!timespec64_valid(&tu))
return -EINVAL;
current->restart_block.fn = do_no_restart_syscall;
current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
current->restart_block.nanosleep.compat_rmtp = rmtp;
return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
CLOCK_MONOTONIC);
}
#endif
/*
* Functions related to boot-time initialization:
*/
int hrtimers_prepare_cpu(unsigned int cpu)
{
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
int i;
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
clock_b->cpu_base = cpu_base;
seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
timerqueue_init_head(&clock_b->active);
}
cpu_base->cpu = cpu;
cpu_base->active_bases = 0;
cpu_base->hres_active = 0;
cpu_base->hang_detected = 0;
cpu_base->next_timer = NULL;
cpu_base->softirq_next_timer = NULL;
cpu_base->expires_next = KTIME_MAX;
cpu_base->softirq_expires_next = KTIME_MAX;
cpu_base->online = 1;
hrtimer_cpu_base_init_expiry_lock(cpu_base);
return 0;
}
#ifdef CONFIG_HOTPLUG_CPU
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
struct hrtimer_clock_base *new_base)
{
struct hrtimer *timer;
struct timerqueue_node *node;
while ((node = timerqueue_getnext(&old_base->active))) {
timer = container_of(node, struct hrtimer, node);
BUG_ON(hrtimer_callback_running(timer));
debug_deactivate(timer);
/*
* Mark it as ENQUEUED not INACTIVE otherwise the
* timer could be seen as !active and just vanish away
* under us on another CPU
*/
__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
timer->base = new_base;
/*
* Enqueue the timers on the new cpu. This does not
* reprogram the event device in case the timer
* expires before the earliest on this CPU, but we run
* hrtimer_interrupt after we migrated everything to
* sort out already expired timers and reprogram the
* event device.
*/
enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
}
}
int hrtimers_cpu_dying(unsigned int dying_cpu)
{
int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
struct hrtimer_cpu_base *old_base, *new_base;
old_base = this_cpu_ptr(&hrtimer_bases);
new_base = &per_cpu(hrtimer_bases, ncpu);
/*
* The caller is globally serialized and nobody else
* takes two locks at once, deadlock is not possible.
*/
raw_spin_lock(&old_base->lock);
raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
migrate_hrtimer_list(&old_base->clock_base[i],
&new_base->clock_base[i]);
}
/*
* The migration might have changed the first expiring softirq
* timer on this CPU. Update it.
*/
__hrtimer_get_next_event(new_base, HRTIMER_ACTIVE_SOFT);
/* Tell the other CPU to retrigger the next event */
smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);
raw_spin_unlock(&new_base->lock);
old_base->online = 0;
raw_spin_unlock(&old_base->lock);
return 0;
}
#endif /* CONFIG_HOTPLUG_CPU */
void __init hrtimers_init(void)
{
hrtimers_prepare_cpu(smp_processor_id());
open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
}
/**
* schedule_hrtimeout_range_clock - sleep until timeout
* @expires: timeout value (ktime_t)
* @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks
* @mode: timer mode
* @clock_id: timer clock to be used
*/
int __sched
schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
const enum hrtimer_mode mode, clockid_t clock_id)
{
struct hrtimer_sleeper t;
/*
* Optimize when a zero timeout value is given. It does not
* matter whether this is an absolute or a relative time.
*/
if (expires && *expires == 0) {
__set_current_state(TASK_RUNNING);
return 0;
}
/*
* A NULL parameter means "infinite"
*/
if (!expires) {
schedule();
return -EINTR;
}
/*
* Override any slack passed by the user if under
* rt contraints.
*/
if (rt_task(current))
delta = 0;
hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
hrtimer_sleeper_start_expires(&t, mode);
if (likely(t.task))
schedule();
hrtimer_cancel(&t.timer);
destroy_hrtimer_on_stack(&t.timer);
__set_current_state(TASK_RUNNING);
return !t.task ? 0 : -EINTR;
}
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
/**
* schedule_hrtimeout_range - sleep until timeout
* @expires: timeout value (ktime_t)
* @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks
* @mode: timer mode
*
* Make the current task sleep until the given expiry time has
* elapsed. The routine will return immediately unless
* the current task state has been set (see set_current_state()).
*
* The @delta argument gives the kernel the freedom to schedule the
* actual wakeup to a time that is both power and performance friendly
* for regular (non RT/DL) tasks.
* The kernel give the normal best effort behavior for "@expires+@delta",
* but may decide to fire the timer earlier, but no earlier than @expires.
*
* You can set the task state as follows -
*
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
* pass before the routine returns unless the current task is explicitly
* woken up, (e.g. by wake_up_process()).
*
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
* delivered to the current task or the current task is explicitly woken
* up.
*
* The current task state is guaranteed to be TASK_RUNNING when this
* routine returns.
*
* Returns 0 when the timer has expired. If the task was woken before the
* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
* by an explicit wakeup, it returns -EINTR.
*/
int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
const enum hrtimer_mode mode)
{
return schedule_hrtimeout_range_clock(expires, delta, mode,
CLOCK_MONOTONIC);
}
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
/**
* schedule_hrtimeout - sleep until timeout
* @expires: timeout value (ktime_t)
* @mode: timer mode
*
* Make the current task sleep until the given expiry time has
* elapsed. The routine will return immediately unless
* the current task state has been set (see set_current_state()).
*
* You can set the task state as follows -
*
* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
* pass before the routine returns unless the current task is explicitly
* woken up, (e.g. by wake_up_process()).
*
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
* delivered to the current task or the current task is explicitly woken
* up.
*
* The current task state is guaranteed to be TASK_RUNNING when this
* routine returns.
*
* Returns 0 when the timer has expired. If the task was woken before the
* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
* by an explicit wakeup, it returns -EINTR.
*/
int __sched schedule_hrtimeout(ktime_t *expires,
const enum hrtimer_mode mode)
{
return schedule_hrtimeout_range(expires, 0, mode);
}
EXPORT_SYMBOL_GPL(schedule_hrtimeout);
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/usb/core/driver.c - most of the driver model stuff for usb
*
* (C) Copyright 2005 Greg Kroah-Hartman <gregkh@suse.de>
*
* based on drivers/usb/usb.c which had the following copyrights:
* (C) Copyright Linus Torvalds 1999
* (C) Copyright Johannes Erdfelt 1999-2001
* (C) Copyright Andreas Gal 1999
* (C) Copyright Gregory P. Smith 1999
* (C) Copyright Deti Fliegl 1999 (new USB architecture)
* (C) Copyright Randy Dunlap 2000
* (C) Copyright David Brownell 2000-2004
* (C) Copyright Yggdrasil Computing, Inc. 2000
* (usb_device_id matching changes by Adam J. Richter)
* (C) Copyright Greg Kroah-Hartman 2002-2003
*
* Released under the GPLv2 only.
*
* NOTE! This is not actually a driver at all, rather this is
* just a collection of helper routines that implement the
* matching, probing, releasing, suspending and resuming for
* real drivers.
*
*/
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/usb.h>
#include <linux/usb/quirks.h>
#include <linux/usb/hcd.h>
#include "usb.h"
/*
* Adds a new dynamic USBdevice ID to this driver,
* and cause the driver to probe for all devices again.
*/
ssize_t usb_store_new_id(struct usb_dynids *dynids,
const struct usb_device_id *id_table,
struct device_driver *driver,
const char *buf, size_t count)
{
struct usb_dynid *dynid;
u32 idVendor = 0;
u32 idProduct = 0;
unsigned int bInterfaceClass = 0;
u32 refVendor, refProduct;
int fields = 0;
int retval = 0;
fields = sscanf(buf, "%x %x %x %x %x", &idVendor, &idProduct,
&bInterfaceClass, &refVendor, &refProduct);
if (fields < 2)
return -EINVAL;
dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
if (!dynid)
return -ENOMEM;
INIT_LIST_HEAD(&dynid->node);
dynid->id.idVendor = idVendor;
dynid->id.idProduct = idProduct;
dynid->id.match_flags = USB_DEVICE_ID_MATCH_DEVICE;
if (fields > 2 && bInterfaceClass) {
if (bInterfaceClass > 255) {
retval = -EINVAL;
goto fail;
}
dynid->id.bInterfaceClass = (u8)bInterfaceClass;
dynid->id.match_flags |= USB_DEVICE_ID_MATCH_INT_CLASS;
}
if (fields > 4) {
const struct usb_device_id *id = id_table;
if (!id) {
retval = -ENODEV;
goto fail;
}
for (; id->match_flags; id++)
if (id->idVendor == refVendor && id->idProduct == refProduct)
break;
if (id->match_flags) {
dynid->id.driver_info = id->driver_info;
} else {
retval = -ENODEV;
goto fail;
}
}
spin_lock(&dynids->lock);
list_add_tail(&dynid->node, &dynids->list);
spin_unlock(&dynids->lock);
retval = driver_attach(driver);
if (retval)
return retval;
return count;
fail:
kfree(dynid);
return retval;
}
EXPORT_SYMBOL_GPL(usb_store_new_id);
ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf)
{
struct usb_dynid *dynid;
size_t count = 0;
list_for_each_entry(dynid, &dynids->list, node)
if (dynid->id.bInterfaceClass != 0)
count += scnprintf(&buf[count], PAGE_SIZE - count, "%04x %04x %02x\n",
dynid->id.idVendor, dynid->id.idProduct,
dynid->id.bInterfaceClass);
else
count += scnprintf(&buf[count], PAGE_SIZE - count, "%04x %04x\n",
dynid->id.idVendor, dynid->id.idProduct);
return count;
}
EXPORT_SYMBOL_GPL(usb_show_dynids);
static ssize_t new_id_show(struct device_driver *driver, char *buf)
{
struct usb_driver *usb_drv = to_usb_driver(driver);
return usb_show_dynids(&usb_drv->dynids, buf);
}
static ssize_t new_id_store(struct device_driver *driver,
const char *buf, size_t count)
{
struct usb_driver *usb_drv = to_usb_driver(driver);
return usb_store_new_id(&usb_drv->dynids, usb_drv->id_table, driver, buf, count);
}
static DRIVER_ATTR_RW(new_id);
/*
* Remove a USB device ID from this driver
*/
static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
size_t count)
{
struct usb_dynid *dynid, *n;
struct usb_driver *usb_driver = to_usb_driver(driver);
u32 idVendor;
u32 idProduct;
int fields;
fields = sscanf(buf, "%x %x", &idVendor, &idProduct);
if (fields < 2)
return -EINVAL;
spin_lock(&usb_driver->dynids.lock);
list_for_each_entry_safe(dynid, n, &usb_driver->dynids.list, node) {
struct usb_device_id *id = &dynid->id;
if ((id->idVendor == idVendor) &&
(id->idProduct == idProduct)) {
list_del(&dynid->node);
kfree(dynid);
break;
}
}
spin_unlock(&usb_driver->dynids.lock);
return count;
}
static ssize_t remove_id_show(struct device_driver *driver, char *buf)
{
return new_id_show(driver, buf);
}
static DRIVER_ATTR_RW(remove_id);
static int usb_create_newid_files(struct usb_driver *usb_drv)
{
int error = 0;
if (usb_drv->no_dynamic_id)
goto exit;
if (usb_drv->probe != NULL) {
error = driver_create_file(&usb_drv->driver,
&driver_attr_new_id);
if (error == 0) {
error = driver_create_file(&usb_drv->driver,
&driver_attr_remove_id);
if (error)
driver_remove_file(&usb_drv->driver,
&driver_attr_new_id);
}
}
exit:
return error;
}
static void usb_remove_newid_files(struct usb_driver *usb_drv)
{
if (usb_drv->no_dynamic_id)
return;
if (usb_drv->probe != NULL) {
driver_remove_file(&usb_drv->driver,
&driver_attr_remove_id);
driver_remove_file(&usb_drv->driver,
&driver_attr_new_id);
}
}
static void usb_free_dynids(struct usb_driver *usb_drv)
{
struct usb_dynid *dynid, *n;
spin_lock(&usb_drv->dynids.lock);
list_for_each_entry_safe(dynid, n, &usb_drv->dynids.list, node) {
list_del(&dynid->node);
kfree(dynid);
}
spin_unlock(&usb_drv->dynids.lock);
}
static const struct usb_device_id *usb_match_dynamic_id(struct usb_interface *intf,
struct usb_driver *drv)
{
struct usb_dynid *dynid;
spin_lock(&drv->dynids.lock);
list_for_each_entry(dynid, &drv->dynids.list, node) {
if (usb_match_one_id(intf, &dynid->id)) {
spin_unlock(&drv->dynids.lock);
return &dynid->id;
}
}
spin_unlock(&drv->dynids.lock);
return NULL;
}
/* called from driver core with dev locked */
static int usb_probe_device(struct device *dev)
{
struct usb_device_driver *udriver = to_usb_device_driver(dev->driver);
struct usb_device *udev = to_usb_device(dev);
int error = 0;
dev_dbg(dev, "%s\n", __func__);
/* TODO: Add real matching code */
/* The device should always appear to be in use
* unless the driver supports autosuspend.
*/
if (!udriver->supports_autosuspend) error = usb_autoresume_device(udev);
if (error)
return error;
if (udriver->generic_subclass) error = usb_generic_driver_probe(udev);
if (error)
return error;
/* Probe the USB device with the driver in hand, but only
* defer to a generic driver in case the current USB
* device driver has an id_table or a match function; i.e.,
* when the device driver was explicitly matched against
* a device.
*
* If the device driver does not have either of these,
* then we assume that it can bind to any device and is
* not truly a more specialized/non-generic driver, so a
* return value of -ENODEV should not force the device
* to be handled by the generic USB driver, as there
* can still be another, more specialized, device driver.
*
* This accommodates the usbip driver.
*
* TODO: What if, in the future, there are multiple
* specialized USB device drivers for a particular device?
* In such cases, there is a need to try all matching
* specialised device drivers prior to setting the
* use_generic_driver bit.
*/
if (udriver->probe) error = udriver->probe(udev); else if (!udriver->generic_subclass)
error = -EINVAL;
if (error == -ENODEV && udriver != &usb_generic_driver &&
(udriver->id_table || udriver->match)) { udev->use_generic_driver = 1; return -EPROBE_DEFER;
}
return error;
}
/* called from driver core with dev locked */
static int usb_unbind_device(struct device *dev)
{
struct usb_device *udev = to_usb_device(dev);
struct usb_device_driver *udriver = to_usb_device_driver(dev->driver);
if (udriver->disconnect)
udriver->disconnect(udev); if (udriver->generic_subclass) usb_generic_driver_disconnect(udev); if (!udriver->supports_autosuspend) usb_autosuspend_device(udev); return 0;
}
/* called from driver core with dev locked */
static int usb_probe_interface(struct device *dev)
{
struct usb_driver *driver = to_usb_driver(dev->driver);
struct usb_interface *intf = to_usb_interface(dev);
struct usb_device *udev = interface_to_usbdev(intf);
const struct usb_device_id *id;
int error = -ENODEV;
int lpm_disable_error = -ENODEV;
dev_dbg(dev, "%s\n", __func__);
intf->needs_binding = 0;
if (usb_device_is_owned(udev))
return error;
if (udev->authorized == 0) {
dev_err(&intf->dev, "Device is not authorized for usage\n");
return error;
} else if (intf->authorized == 0) {
dev_err(&intf->dev, "Interface %d is not authorized for usage\n",
intf->altsetting->desc.bInterfaceNumber);
return error;
}
id = usb_match_dynamic_id(intf, driver);
if (!id)
id = usb_match_id(intf, driver->id_table);
if (!id)
return error;
dev_dbg(dev, "%s - got id\n", __func__);
error = usb_autoresume_device(udev);
if (error)
return error;
intf->condition = USB_INTERFACE_BINDING;
/* Probed interfaces are initially active. They are
* runtime-PM-enabled only if the driver has autosuspend support.
* They are sensitive to their children's power states.
*/
pm_runtime_set_active(dev);
pm_suspend_ignore_children(dev, false);
if (driver->supports_autosuspend)
pm_runtime_enable(dev);
/* If the new driver doesn't allow hub-initiated LPM, and we can't
* disable hub-initiated LPM, then fail the probe.
*
* Otherwise, leaving LPM enabled should be harmless, because the
* endpoint intervals should remain the same, and the U1/U2 timeouts
* should remain the same.
*
* If we need to install alt setting 0 before probe, or another alt
* setting during probe, that should also be fine. usb_set_interface()
* will attempt to disable LPM, and fail if it can't disable it.
*/
if (driver->disable_hub_initiated_lpm) {
lpm_disable_error = usb_unlocked_disable_lpm(udev);
if (lpm_disable_error) {
dev_err(&intf->dev, "%s Failed to disable LPM for driver %s\n",
__func__, driver->name);
error = lpm_disable_error;
goto err;
}
}
/* Carry out a deferred switch to altsetting 0 */
if (intf->needs_altsetting0) {
error = usb_set_interface(udev, intf->altsetting[0].
desc.bInterfaceNumber, 0);
if (error < 0)
goto err;
intf->needs_altsetting0 = 0;
}
error = driver->probe(intf, id);
if (error)
goto err;
intf->condition = USB_INTERFACE_BOUND;
/* If the LPM disable succeeded, balance the ref counts. */
if (!lpm_disable_error)
usb_unlocked_enable_lpm(udev);
usb_autosuspend_device(udev);
return error;
err:
usb_set_intfdata(intf, NULL);
intf->needs_remote_wakeup = 0;
intf->condition = USB_INTERFACE_UNBOUND;
/* If the LPM disable succeeded, balance the ref counts. */
if (!lpm_disable_error)
usb_unlocked_enable_lpm(udev);
/* Unbound interfaces are always runtime-PM-disabled and -suspended */
if (driver->supports_autosuspend)
pm_runtime_disable(dev);
pm_runtime_set_suspended(dev);
usb_autosuspend_device(udev);
return error;
}
/* called from driver core with dev locked */
static int usb_unbind_interface(struct device *dev)
{
struct usb_driver *driver = to_usb_driver(dev->driver);
struct usb_interface *intf = to_usb_interface(dev);
struct usb_host_endpoint *ep, **eps = NULL;
struct usb_device *udev;
int i, j, error, r;
int lpm_disable_error = -ENODEV;
intf->condition = USB_INTERFACE_UNBINDING;
/* Autoresume for set_interface call below */
udev = interface_to_usbdev(intf);
error = usb_autoresume_device(udev);
/* If hub-initiated LPM policy may change, attempt to disable LPM until
* the driver is unbound. If LPM isn't disabled, that's fine because it
* wouldn't be enabled unless all the bound interfaces supported
* hub-initiated LPM.
*/
if (driver->disable_hub_initiated_lpm) lpm_disable_error = usb_unlocked_disable_lpm(udev);
/*
* Terminate all URBs for this interface unless the driver
* supports "soft" unbinding and the device is still present.
*/
if (!driver->soft_unbind || udev->state == USB_STATE_NOTATTACHED) usb_disable_interface(udev, intf, false); driver->disconnect(intf);
/* Free streams */
for (i = 0, j = 0; i < intf->cur_altsetting->desc.bNumEndpoints; i++) { ep = &intf->cur_altsetting->endpoint[i];
if (ep->streams == 0)
continue;
if (j == 0) { eps = kmalloc_array(USB_MAXENDPOINTS, sizeof(void *),
GFP_KERNEL);
if (!eps)
break;
}
eps[j++] = ep;
}
if (j) { usb_free_streams(intf, eps, j, GFP_KERNEL);
kfree(eps);
}
/* Reset other interface state.
* We cannot do a Set-Interface if the device is suspended or
* if it is prepared for a system sleep (since installing a new
* altsetting means creating new endpoint device entries).
* When either of these happens, defer the Set-Interface.
*/
if (intf->cur_altsetting->desc.bAlternateSetting == 0) {
/* Already in altsetting 0 so skip Set-Interface.
* Just re-enable it without affecting the endpoint toggles.
*/
usb_enable_interface(udev, intf, false); } else if (!error && !intf->dev.power.is_prepared) { r = usb_set_interface(udev, intf->altsetting[0].
desc.bInterfaceNumber, 0);
if (r < 0)
intf->needs_altsetting0 = 1;
} else {
intf->needs_altsetting0 = 1;
}
usb_set_intfdata(intf, NULL);
intf->condition = USB_INTERFACE_UNBOUND;
intf->needs_remote_wakeup = 0;
/* Attempt to re-enable USB3 LPM, if the disable succeeded. */
if (!lpm_disable_error)
usb_unlocked_enable_lpm(udev);
/* Unbound interfaces are always runtime-PM-disabled and -suspended */
if (driver->supports_autosuspend) pm_runtime_disable(dev); pm_runtime_set_suspended(dev);
if (!error)
usb_autosuspend_device(udev); return 0;
}
/**
* usb_driver_claim_interface - bind a driver to an interface
* @driver: the driver to be bound
* @iface: the interface to which it will be bound; must be in the
* usb device's active configuration
* @data: driver data associated with that interface
*
* This is used by usb device drivers that need to claim more than one
* interface on a device when probing (audio and acm are current examples).
* No device driver should directly modify internal usb_interface or
* usb_device structure members.
*
* Callers must own the device lock, so driver probe() entries don't need
* extra locking, but other call contexts may need to explicitly claim that
* lock.
*
* Return: 0 on success.
*/
int usb_driver_claim_interface(struct usb_driver *driver,
struct usb_interface *iface, void *data)
{
struct device *dev;
int retval = 0;
if (!iface)
return -ENODEV;
dev = &iface->dev;
if (dev->driver)
return -EBUSY;
/* reject claim if interface is not authorized */
if (!iface->authorized)
return -ENODEV;
dev->driver = &driver->driver;
usb_set_intfdata(iface, data);
iface->needs_binding = 0;
iface->condition = USB_INTERFACE_BOUND;
/* Claimed interfaces are initially inactive (suspended) and
* runtime-PM-enabled, but only if the driver has autosuspend
* support. Otherwise they are marked active, to prevent the
* device from being autosuspended, but left disabled. In either
* case they are sensitive to their children's power states.
*/
pm_suspend_ignore_children(dev, false);
if (driver->supports_autosuspend)
pm_runtime_enable(dev);
else
pm_runtime_set_active(dev);
/* if interface was already added, bind now; else let
* the future device_add() bind it, bypassing probe()
*/
if (device_is_registered(dev))
retval = device_bind_driver(dev);
if (retval) {
dev->driver = NULL;
usb_set_intfdata(iface, NULL);
iface->needs_remote_wakeup = 0;
iface->condition = USB_INTERFACE_UNBOUND;
/*
* Unbound interfaces are always runtime-PM-disabled
* and runtime-PM-suspended
*/
if (driver->supports_autosuspend)
pm_runtime_disable(dev);
pm_runtime_set_suspended(dev);
}
return retval;
}
EXPORT_SYMBOL_GPL(usb_driver_claim_interface);
/**
* usb_driver_release_interface - unbind a driver from an interface
* @driver: the driver to be unbound
* @iface: the interface from which it will be unbound
*
* This can be used by drivers to release an interface without waiting
* for their disconnect() methods to be called. In typical cases this
* also causes the driver disconnect() method to be called.
*
* This call is synchronous, and may not be used in an interrupt context.
* Callers must own the device lock, so driver disconnect() entries don't
* need extra locking, but other call contexts may need to explicitly claim
* that lock.
*/
void usb_driver_release_interface(struct usb_driver *driver,
struct usb_interface *iface)
{
struct device *dev = &iface->dev;
/* this should never happen, don't release something that's not ours */
if (!dev->driver || dev->driver != &driver->driver)
return;
/* don't release from within disconnect() */
if (iface->condition != USB_INTERFACE_BOUND)
return;
iface->condition = USB_INTERFACE_UNBINDING;
/* Release via the driver core only if the interface
* has already been registered
*/
if (device_is_registered(dev)) {
device_release_driver(dev);
} else {
device_lock(dev);
usb_unbind_interface(dev);
dev->driver = NULL;
device_unlock(dev);
}
}
EXPORT_SYMBOL_GPL(usb_driver_release_interface);
/* returns 0 if no match, 1 if match */
int usb_match_device(struct usb_device *dev, const struct usb_device_id *id)
{
if ((id->match_flags & USB_DEVICE_ID_MATCH_VENDOR) && id->idVendor != le16_to_cpu(dev->descriptor.idVendor))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_PRODUCT) && id->idProduct != le16_to_cpu(dev->descriptor.idProduct))
return 0;
/* No need to test id->bcdDevice_lo != 0, since 0 is never
greater than any unsigned number. */
if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_LO) && (id->bcdDevice_lo > le16_to_cpu(dev->descriptor.bcdDevice)))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_HI) && (id->bcdDevice_hi < le16_to_cpu(dev->descriptor.bcdDevice)))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_CLASS) && (id->bDeviceClass != dev->descriptor.bDeviceClass))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_SUBCLASS) && (id->bDeviceSubClass != dev->descriptor.bDeviceSubClass))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_PROTOCOL) && (id->bDeviceProtocol != dev->descriptor.bDeviceProtocol))
return 0;
return 1;
}
/* returns 0 if no match, 1 if match */
int usb_match_one_id_intf(struct usb_device *dev,
struct usb_host_interface *intf,
const struct usb_device_id *id)
{
/* The interface class, subclass, protocol and number should never be
* checked for a match if the device class is Vendor Specific,
* unless the match record specifies the Vendor ID. */
if (dev->descriptor.bDeviceClass == USB_CLASS_VENDOR_SPEC &&
!(id->match_flags & USB_DEVICE_ID_MATCH_VENDOR) &&
(id->match_flags & (USB_DEVICE_ID_MATCH_INT_CLASS |
USB_DEVICE_ID_MATCH_INT_SUBCLASS |
USB_DEVICE_ID_MATCH_INT_PROTOCOL |
USB_DEVICE_ID_MATCH_INT_NUMBER)))
return 0; if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_CLASS) && (id->bInterfaceClass != intf->desc.bInterfaceClass))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_SUBCLASS) && (id->bInterfaceSubClass != intf->desc.bInterfaceSubClass))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_PROTOCOL) && (id->bInterfaceProtocol != intf->desc.bInterfaceProtocol))
return 0;
if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_NUMBER) && (id->bInterfaceNumber != intf->desc.bInterfaceNumber))
return 0;
return 1;
}
/* returns 0 if no match, 1 if match */
int usb_match_one_id(struct usb_interface *interface,
const struct usb_device_id *id)
{
struct usb_host_interface *intf;
struct usb_device *dev;
/* proc_connectinfo in devio.c may call us with id == NULL. */
if (id == NULL)
return 0;
intf = interface->cur_altsetting;
dev = interface_to_usbdev(interface);
if (!usb_match_device(dev, id))
return 0;
return usb_match_one_id_intf(dev, intf, id);
}
EXPORT_SYMBOL_GPL(usb_match_one_id);
/**
* usb_match_id - find first usb_device_id matching device or interface
* @interface: the interface of interest
* @id: array of usb_device_id structures, terminated by zero entry
*
* usb_match_id searches an array of usb_device_id's and returns
* the first one matching the device or interface, or null.
* This is used when binding (or rebinding) a driver to an interface.
* Most USB device drivers will use this indirectly, through the usb core,
* but some layered driver frameworks use it directly.
* These device tables are exported with MODULE_DEVICE_TABLE, through
* modutils, to support the driver loading functionality of USB hotplugging.
*
* Return: The first matching usb_device_id, or %NULL.
*
* What Matches:
*
* The "match_flags" element in a usb_device_id controls which
* members are used. If the corresponding bit is set, the
* value in the device_id must match its corresponding member
* in the device or interface descriptor, or else the device_id
* does not match.
*
* "driver_info" is normally used only by device drivers,
* but you can create a wildcard "matches anything" usb_device_id
* as a driver's "modules.usbmap" entry if you provide an id with
* only a nonzero "driver_info" field. If you do this, the USB device
* driver's probe() routine should use additional intelligence to
* decide whether to bind to the specified interface.
*
* What Makes Good usb_device_id Tables:
*
* The match algorithm is very simple, so that intelligence in
* driver selection must come from smart driver id records.
* Unless you have good reasons to use another selection policy,
* provide match elements only in related groups, and order match
* specifiers from specific to general. Use the macros provided
* for that purpose if you can.
*
* The most specific match specifiers use device descriptor
* data. These are commonly used with product-specific matches;
* the USB_DEVICE macro lets you provide vendor and product IDs,
* and you can also match against ranges of product revisions.
* These are widely used for devices with application or vendor
* specific bDeviceClass values.
*
* Matches based on device class/subclass/protocol specifications
* are slightly more general; use the USB_DEVICE_INFO macro, or
* its siblings. These are used with single-function devices
* where bDeviceClass doesn't specify that each interface has
* its own class.
*
* Matches based on interface class/subclass/protocol are the
* most general; they let drivers bind to any interface on a
* multiple-function device. Use the USB_INTERFACE_INFO
* macro, or its siblings, to match class-per-interface style
* devices (as recorded in bInterfaceClass).
*
* Note that an entry created by USB_INTERFACE_INFO won't match
* any interface if the device class is set to Vendor-Specific.
* This is deliberate; according to the USB spec the meanings of
* the interface class/subclass/protocol for these devices are also
* vendor-specific, and hence matching against a standard product
* class wouldn't work anyway. If you really want to use an
* interface-based match for such a device, create a match record
* that also specifies the vendor ID. (Unforunately there isn't a
* standard macro for creating records like this.)
*
* Within those groups, remember that not all combinations are
* meaningful. For example, don't give a product version range
* without vendor and product IDs; or specify a protocol without
* its associated class and subclass.
*/
const struct usb_device_id *usb_match_id(struct usb_interface *interface,
const struct usb_device_id *id)
{
/* proc_connectinfo in devio.c may call us with id == NULL. */
if (id == NULL)
return NULL;
/* It is important to check that id->driver_info is nonzero,
since an entry that is all zeroes except for a nonzero
id->driver_info is the way to create an entry that
indicates that the driver want to examine every
device and interface. */
for (; id->idVendor || id->idProduct || id->bDeviceClass ||
id->bInterfaceClass || id->driver_info; id++) {
if (usb_match_one_id(interface, id))
return id;
}
return NULL;
}
EXPORT_SYMBOL_GPL(usb_match_id);
const struct usb_device_id *usb_device_match_id(struct usb_device *udev,
const struct usb_device_id *id)
{
if (!id)
return NULL;
for (; id->idVendor || id->idProduct ; id++) { if (usb_match_device(udev, id))
return id;
}
return NULL;
}
EXPORT_SYMBOL_GPL(usb_device_match_id);
bool usb_driver_applicable(struct usb_device *udev,
struct usb_device_driver *udrv)
{
if (udrv->id_table && udrv->match) return usb_device_match_id(udev, udrv->id_table) != NULL &&
udrv->match(udev);
if (udrv->id_table)
return usb_device_match_id(udev, udrv->id_table) != NULL; if (udrv->match) return udrv->match(udev);
return false;
}
static int usb_device_match(struct device *dev, struct device_driver *drv)
{
/* devices and interfaces are handled separately */
if (is_usb_device(dev)) {
struct usb_device *udev;
struct usb_device_driver *udrv;
/* interface drivers never match devices */
if (!is_usb_device_driver(drv)) return 0; udev = to_usb_device(dev);
udrv = to_usb_device_driver(drv);
/* If the device driver under consideration does not have a
* id_table or a match function, then let the driver's probe
* function decide.
*/
if (!udrv->id_table && !udrv->match)
return 1;
return usb_driver_applicable(udev, udrv); } else if (is_usb_interface(dev)) {
struct usb_interface *intf;
struct usb_driver *usb_drv;
const struct usb_device_id *id;
/* device drivers never match interfaces */
if (is_usb_device_driver(drv))
return 0;
intf = to_usb_interface(dev); usb_drv = to_usb_driver(drv);
id = usb_match_id(intf, usb_drv->id_table);
if (id)
return 1;
id = usb_match_dynamic_id(intf, usb_drv);
if (id)
return 1;
}
return 0;
}
static int usb_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct usb_device *usb_dev;
if (is_usb_device(dev)) { usb_dev = to_usb_device(dev); } else if (is_usb_interface(dev)) {
const struct usb_interface *intf = to_usb_interface(dev);
usb_dev = interface_to_usbdev(intf);
} else {
return 0;
}
if (usb_dev->devnum < 0) {
/* driver is often null here; dev_dbg() would oops */
pr_debug("usb %s: already deleted?\n", dev_name(dev));
return -ENODEV;
}
if (!usb_dev->bus) { pr_debug("usb %s: bus removed?\n", dev_name(dev));
return -ENODEV;
}
/* per-device configurations are common */
if (add_uevent_var(env, "PRODUCT=%x/%x/%x",
le16_to_cpu(usb_dev->descriptor.idVendor),
le16_to_cpu(usb_dev->descriptor.idProduct),
le16_to_cpu(usb_dev->descriptor.bcdDevice)))
return -ENOMEM;
/* class-based driver binding models */
if (add_uevent_var(env, "TYPE=%d/%d/%d",
usb_dev->descriptor.bDeviceClass,
usb_dev->descriptor.bDeviceSubClass,
usb_dev->descriptor.bDeviceProtocol))
return -ENOMEM;
return 0;
}
static int __usb_bus_reprobe_drivers(struct device *dev, void *data)
{
struct usb_device_driver *new_udriver = data;
struct usb_device *udev;
int ret;
/* Don't reprobe if current driver isn't usb_generic_driver */
if (dev->driver != &usb_generic_driver.driver)
return 0;
udev = to_usb_device(dev);
if (!usb_driver_applicable(udev, new_udriver))
return 0;
ret = device_reprobe(dev);
if (ret && ret != -EPROBE_DEFER)
dev_err(dev, "Failed to reprobe device (error %d)\n", ret);
return 0;
}
bool is_usb_device_driver(const struct device_driver *drv)
{
return drv->probe == usb_probe_device;
}
/**
* usb_register_device_driver - register a USB device (not interface) driver
* @new_udriver: USB operations for the device driver
* @owner: module owner of this driver.
*
* Registers a USB device driver with the USB core. The list of
* unattached devices will be rescanned whenever a new driver is
* added, allowing the new driver to attach to any recognized devices.
*
* Return: A negative error code on failure and 0 on success.
*/
int usb_register_device_driver(struct usb_device_driver *new_udriver,
struct module *owner)
{
int retval = 0;
if (usb_disabled())
return -ENODEV;
new_udriver->driver.name = new_udriver->name;
new_udriver->driver.bus = &usb_bus_type;
new_udriver->driver.probe = usb_probe_device;
new_udriver->driver.remove = usb_unbind_device;
new_udriver->driver.owner = owner;
new_udriver->driver.dev_groups = new_udriver->dev_groups;
retval = driver_register(&new_udriver->driver);
if (!retval) {
pr_info("%s: registered new device driver %s\n",
usbcore_name, new_udriver->name);
/*
* Check whether any device could be better served with
* this new driver
*/
bus_for_each_dev(&usb_bus_type, NULL, new_udriver,
__usb_bus_reprobe_drivers);
} else {
pr_err("%s: error %d registering device driver %s\n",
usbcore_name, retval, new_udriver->name);
}
return retval;
}
EXPORT_SYMBOL_GPL(usb_register_device_driver);
/**
* usb_deregister_device_driver - unregister a USB device (not interface) driver
* @udriver: USB operations of the device driver to unregister
* Context: must be able to sleep
*
* Unlinks the specified driver from the internal USB driver list.
*/
void usb_deregister_device_driver(struct usb_device_driver *udriver)
{
pr_info("%s: deregistering device driver %s\n",
usbcore_name, udriver->name);
driver_unregister(&udriver->driver);
}
EXPORT_SYMBOL_GPL(usb_deregister_device_driver);
/**
* usb_register_driver - register a USB interface driver
* @new_driver: USB operations for the interface driver
* @owner: module owner of this driver.
* @mod_name: module name string
*
* Registers a USB interface driver with the USB core. The list of
* unattached interfaces will be rescanned whenever a new driver is
* added, allowing the new driver to attach to any recognized interfaces.
*
* Return: A negative error code on failure and 0 on success.
*
* NOTE: if you want your driver to use the USB major number, you must call
* usb_register_dev() to enable that functionality. This function no longer
* takes care of that.
*/
int usb_register_driver(struct usb_driver *new_driver, struct module *owner,
const char *mod_name)
{
int retval = 0;
if (usb_disabled())
return -ENODEV;
new_driver->driver.name = new_driver->name;
new_driver->driver.bus = &usb_bus_type;
new_driver->driver.probe = usb_probe_interface;
new_driver->driver.remove = usb_unbind_interface;
new_driver->driver.owner = owner;
new_driver->driver.mod_name = mod_name;
new_driver->driver.dev_groups = new_driver->dev_groups;
spin_lock_init(&new_driver->dynids.lock);
INIT_LIST_HEAD(&new_driver->dynids.list);
retval = driver_register(&new_driver->driver);
if (retval)
goto out;
retval = usb_create_newid_files(new_driver);
if (retval)
goto out_newid;
pr_info("%s: registered new interface driver %s\n",
usbcore_name, new_driver->name);
out:
return retval;
out_newid:
driver_unregister(&new_driver->driver);
pr_err("%s: error %d registering interface driver %s\n",
usbcore_name, retval, new_driver->name);
goto out;
}
EXPORT_SYMBOL_GPL(usb_register_driver);
/**
* usb_deregister - unregister a USB interface driver
* @driver: USB operations of the interface driver to unregister
* Context: must be able to sleep
*
* Unlinks the specified driver from the internal USB driver list.
*
* NOTE: If you called usb_register_dev(), you still need to call
* usb_deregister_dev() to clean up your driver's allocated minor numbers,
* this * call will no longer do it for you.
*/
void usb_deregister(struct usb_driver *driver)
{
pr_info("%s: deregistering interface driver %s\n",
usbcore_name, driver->name);
usb_remove_newid_files(driver);
driver_unregister(&driver->driver);
usb_free_dynids(driver);
}
EXPORT_SYMBOL_GPL(usb_deregister);
/* Forced unbinding of a USB interface driver, either because
* it doesn't support pre_reset/post_reset/reset_resume or
* because it doesn't support suspend/resume.
*
* The caller must hold @intf's device's lock, but not @intf's lock.
*/
void usb_forced_unbind_intf(struct usb_interface *intf)
{
struct usb_driver *driver = to_usb_driver(intf->dev.driver);
dev_dbg(&intf->dev, "forced unbind\n");
usb_driver_release_interface(driver, intf);
/* Mark the interface for later rebinding */
intf->needs_binding = 1;
}
/*
* Unbind drivers for @udev's marked interfaces. These interfaces have
* the needs_binding flag set, for example by usb_resume_interface().
*
* The caller must hold @udev's device lock.
*/
static void unbind_marked_interfaces(struct usb_device *udev)
{
struct usb_host_config *config;
int i;
struct usb_interface *intf;
config = udev->actconfig;
if (config) {
for (i = 0; i < config->desc.bNumInterfaces; ++i) {
intf = config->interface[i];
if (intf->dev.driver && intf->needs_binding)
usb_forced_unbind_intf(intf);
}
}
}
/* Delayed forced unbinding of a USB interface driver and scan
* for rebinding.
*
* The caller must hold @intf's device's lock, but not @intf's lock.
*
* Note: Rebinds will be skipped if a system sleep transition is in
* progress and the PM "complete" callback hasn't occurred yet.
*/
static void usb_rebind_intf(struct usb_interface *intf)
{
int rc;
/* Delayed unbind of an existing driver */
if (intf->dev.driver)
usb_forced_unbind_intf(intf);
/* Try to rebind the interface */
if (!intf->dev.power.is_prepared) {
intf->needs_binding = 0;
rc = device_attach(&intf->dev);
if (rc < 0 && rc != -EPROBE_DEFER)
dev_warn(&intf->dev, "rebind failed: %d\n", rc);
}
}
/*
* Rebind drivers to @udev's marked interfaces. These interfaces have
* the needs_binding flag set.
*
* The caller must hold @udev's device lock.
*/
static void rebind_marked_interfaces(struct usb_device *udev)
{
struct usb_host_config *config;
int i;
struct usb_interface *intf;
config = udev->actconfig;
if (config) {
for (i = 0; i < config->desc.bNumInterfaces; ++i) {
intf = config->interface[i];
if (intf->needs_binding)
usb_rebind_intf(intf);
}
}
}
/*
* Unbind all of @udev's marked interfaces and then rebind all of them.
* This ordering is necessary because some drivers claim several interfaces
* when they are first probed.
*
* The caller must hold @udev's device lock.
*/
void usb_unbind_and_rebind_marked_interfaces(struct usb_device *udev)
{
unbind_marked_interfaces(udev);
rebind_marked_interfaces(udev);
}
#ifdef CONFIG_PM
/* Unbind drivers for @udev's interfaces that don't support suspend/resume
* There is no check for reset_resume here because it can be determined
* only during resume whether reset_resume is needed.
*
* The caller must hold @udev's device lock.
*/
static void unbind_no_pm_drivers_interfaces(struct usb_device *udev)
{
struct usb_host_config *config;
int i;
struct usb_interface *intf;
struct usb_driver *drv;
config = udev->actconfig;
if (config) {
for (i = 0; i < config->desc.bNumInterfaces; ++i) {
intf = config->interface[i];
if (intf->dev.driver) {
drv = to_usb_driver(intf->dev.driver);
if (!drv->suspend || !drv->resume)
usb_forced_unbind_intf(intf);
}
}
}
}
static int usb_suspend_device(struct usb_device *udev, pm_message_t msg)
{
struct usb_device_driver *udriver;
int status = 0;
if (udev->state == USB_STATE_NOTATTACHED ||
udev->state == USB_STATE_SUSPENDED)
goto done;
/* For devices that don't have a driver, we do a generic suspend. */
if (udev->dev.driver)
udriver = to_usb_device_driver(udev->dev.driver);
else {
udev->do_remote_wakeup = 0;
udriver = &usb_generic_driver;
}
if (udriver->suspend)
status = udriver->suspend(udev, msg);
if (status == 0 && udriver->generic_subclass)
status = usb_generic_driver_suspend(udev, msg);
done:
dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
return status;
}
static int usb_resume_device(struct usb_device *udev, pm_message_t msg)
{
struct usb_device_driver *udriver;
int status = 0;
if (udev->state == USB_STATE_NOTATTACHED)
goto done;
/* Can't resume it if it doesn't have a driver. */
if (udev->dev.driver == NULL) {
status = -ENOTCONN;
goto done;
}
/* Non-root devices on a full/low-speed bus must wait for their
* companion high-speed root hub, in case a handoff is needed.
*/
if (!PMSG_IS_AUTO(msg) && udev->parent && udev->bus->hs_companion)
device_pm_wait_for_dev(&udev->dev,
&udev->bus->hs_companion->root_hub->dev); if (udev->quirks & USB_QUIRK_RESET_RESUME) udev->reset_resume = 1;
udriver = to_usb_device_driver(udev->dev.driver);
if (udriver->generic_subclass) status = usb_generic_driver_resume(udev, msg); if (status == 0 && udriver->resume) status = udriver->resume(udev, msg);
done:
dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
return status;
}
static int usb_suspend_interface(struct usb_device *udev,
struct usb_interface *intf, pm_message_t msg)
{
struct usb_driver *driver;
int status = 0;
if (udev->state == USB_STATE_NOTATTACHED ||
intf->condition == USB_INTERFACE_UNBOUND)
goto done;
driver = to_usb_driver(intf->dev.driver);
/* at this time we know the driver supports suspend */
status = driver->suspend(intf, msg);
if (status && !PMSG_IS_AUTO(msg))
dev_err(&intf->dev, "suspend error %d\n", status);
done:
dev_vdbg(&intf->dev, "%s: status %d\n", __func__, status);
return status;
}
static int usb_resume_interface(struct usb_device *udev,
struct usb_interface *intf, pm_message_t msg, int reset_resume)
{
struct usb_driver *driver;
int status = 0;
if (udev->state == USB_STATE_NOTATTACHED)
goto done;
/* Don't let autoresume interfere with unbinding */
if (intf->condition == USB_INTERFACE_UNBINDING)
goto done;
/* Can't resume it if it doesn't have a driver. */
if (intf->condition == USB_INTERFACE_UNBOUND) {
/* Carry out a deferred switch to altsetting 0 */
if (intf->needs_altsetting0 && !intf->dev.power.is_prepared) { usb_set_interface(udev, intf->altsetting[0].
desc.bInterfaceNumber, 0);
intf->needs_altsetting0 = 0;
}
goto done;
}
/* Don't resume if the interface is marked for rebinding */
if (intf->needs_binding)
goto done;
driver = to_usb_driver(intf->dev.driver);
if (reset_resume) {
if (driver->reset_resume) { status = driver->reset_resume(intf);
if (status)
dev_err(&intf->dev, "%s error %d\n",
"reset_resume", status);
} else {
intf->needs_binding = 1; dev_dbg(&intf->dev, "no reset_resume for driver %s?\n",
driver->name);
}
} else {
status = driver->resume(intf);
if (status)
dev_err(&intf->dev, "resume error %d\n", status);
}
done:
dev_vdbg(&intf->dev, "%s: status %d\n", __func__, status);
/* Later we will unbind the driver and/or reprobe, if necessary */
return status;
}
/**
* usb_suspend_both - suspend a USB device and its interfaces
* @udev: the usb_device to suspend
* @msg: Power Management message describing this state transition
*
* This is the central routine for suspending USB devices. It calls the
* suspend methods for all the interface drivers in @udev and then calls
* the suspend method for @udev itself. When the routine is called in
* autosuspend, if an error occurs at any stage, all the interfaces
* which were suspended are resumed so that they remain in the same
* state as the device, but when called from system sleep, all error
* from suspend methods of interfaces and the non-root-hub device itself
* are simply ignored, so all suspended interfaces are only resumed
* to the device's state when @udev is root-hub and its suspend method
* returns failure.
*
* Autosuspend requests originating from a child device or an interface
* driver may be made without the protection of @udev's device lock, but
* all other suspend calls will hold the lock. Usbcore will insure that
* method calls do not arrive during bind, unbind, or reset operations.
* However drivers must be prepared to handle suspend calls arriving at
* unpredictable times.
*
* This routine can run only in process context.
*
* Return: 0 if the suspend succeeded.
*/
static int usb_suspend_both(struct usb_device *udev, pm_message_t msg)
{
int status = 0;
int i = 0, n = 0;
struct usb_interface *intf;
if (udev->state == USB_STATE_NOTATTACHED ||
udev->state == USB_STATE_SUSPENDED)
goto done;
/* Suspend all the interfaces and then udev itself */
if (udev->actconfig) {
n = udev->actconfig->desc.bNumInterfaces;
for (i = n - 1; i >= 0; --i) {
intf = udev->actconfig->interface[i];
status = usb_suspend_interface(udev, intf, msg);
/* Ignore errors during system sleep transitions */
if (!PMSG_IS_AUTO(msg))
status = 0;
if (status != 0)
break;
}
}
if (status == 0) {
status = usb_suspend_device(udev, msg);
/*
* Ignore errors from non-root-hub devices during
* system sleep transitions. For the most part,
* these devices should go to low power anyway when
* the entire bus is suspended.
*/
if (udev->parent && !PMSG_IS_AUTO(msg))
status = 0;
/*
* If the device is inaccessible, don't try to resume
* suspended interfaces and just return the error.
*/
if (status && status != -EBUSY) {
int err;
u16 devstat;
err = usb_get_std_status(udev, USB_RECIP_DEVICE, 0,
&devstat);
if (err) {
dev_err(&udev->dev,
"Failed to suspend device, error %d\n",
status);
goto done;
}
}
}
/* If the suspend failed, resume interfaces that did get suspended */
if (status != 0) {
if (udev->actconfig) {
msg.event ^= (PM_EVENT_SUSPEND | PM_EVENT_RESUME);
while (++i < n) {
intf = udev->actconfig->interface[i];
usb_resume_interface(udev, intf, msg, 0);
}
}
/* If the suspend succeeded then prevent any more URB submissions
* and flush any outstanding URBs.
*/
} else {
udev->can_submit = 0;
for (i = 0; i < 16; ++i) {
usb_hcd_flush_endpoint(udev, udev->ep_out[i]);
usb_hcd_flush_endpoint(udev, udev->ep_in[i]);
}
}
done:
dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
return status;
}
/**
* usb_resume_both - resume a USB device and its interfaces
* @udev: the usb_device to resume
* @msg: Power Management message describing this state transition
*
* This is the central routine for resuming USB devices. It calls the
* resume method for @udev and then calls the resume methods for all
* the interface drivers in @udev.
*
* Autoresume requests originating from a child device or an interface
* driver may be made without the protection of @udev's device lock, but
* all other resume calls will hold the lock. Usbcore will insure that
* method calls do not arrive during bind, unbind, or reset operations.
* However drivers must be prepared to handle resume calls arriving at
* unpredictable times.
*
* This routine can run only in process context.
*
* Return: 0 on success.
*/
static int usb_resume_both(struct usb_device *udev, pm_message_t msg)
{
int status = 0;
int i;
struct usb_interface *intf;
if (udev->state == USB_STATE_NOTATTACHED) {
status = -ENODEV;
goto done;
}
udev->can_submit = 1;
/* Resume the device */
if (udev->state == USB_STATE_SUSPENDED || udev->reset_resume) status = usb_resume_device(udev, msg);
/* Resume the interfaces */
if (status == 0 && udev->actconfig) { for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) { intf = udev->actconfig->interface[i];
usb_resume_interface(udev, intf, msg,
udev->reset_resume);
}
}
usb_mark_last_busy(udev);
done:
dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
if (!status)
udev->reset_resume = 0; return status;
}
static void choose_wakeup(struct usb_device *udev, pm_message_t msg)
{
int w;
/*
* For FREEZE/QUIESCE, disable remote wakeups so no interrupts get
* generated.
*/
if (msg.event == PM_EVENT_FREEZE || msg.event == PM_EVENT_QUIESCE) {
w = 0;
} else {
/*
* Enable remote wakeup if it is allowed, even if no interface
* drivers actually want it.
*/
w = device_may_wakeup(&udev->dev);
}
/*
* If the device is autosuspended with the wrong wakeup setting,
* autoresume now so the setting can be changed.
*/
if (udev->state == USB_STATE_SUSPENDED && w != udev->do_remote_wakeup)
pm_runtime_resume(&udev->dev);
udev->do_remote_wakeup = w;
}
/* The device lock is held by the PM core */
int usb_suspend(struct device *dev, pm_message_t msg)
{
struct usb_device *udev = to_usb_device(dev);
int r;
unbind_no_pm_drivers_interfaces(udev);
/* From now on we are sure all drivers support suspend/resume
* but not necessarily reset_resume()
* so we may still need to unbind and rebind upon resume
*/
choose_wakeup(udev, msg);
r = usb_suspend_both(udev, msg);
if (r)
return r;
if (udev->quirks & USB_QUIRK_DISCONNECT_SUSPEND)
usb_port_disable(udev);
return 0;
}
/* The device lock is held by the PM core */
int usb_resume_complete(struct device *dev)
{
struct usb_device *udev = to_usb_device(dev);
/* For PM complete calls, all we do is rebind interfaces
* whose needs_binding flag is set
*/
if (udev->state != USB_STATE_NOTATTACHED)
rebind_marked_interfaces(udev);
return 0;
}
/* The device lock is held by the PM core */
int usb_resume(struct device *dev, pm_message_t msg)
{
struct usb_device *udev = to_usb_device(dev);
int status;
/* For all calls, take the device back to full power and
* tell the PM core in case it was autosuspended previously.
* Unbind the interfaces that will need rebinding later,
* because they fail to support reset_resume.
* (This can't be done in usb_resume_interface()
* above because it doesn't own the right set of locks.)
*/
status = usb_resume_both(udev, msg);
if (status == 0) {
pm_runtime_disable(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
unbind_marked_interfaces(udev);
}
/* Avoid PM error messages for devices disconnected while suspended
* as we'll display regular disconnect messages just a bit later.
*/
if (status == -ENODEV || status == -ESHUTDOWN)
status = 0;
return status;
}
/**
* usb_enable_autosuspend - allow a USB device to be autosuspended
* @udev: the USB device which may be autosuspended
*
* This routine allows @udev to be autosuspended. An autosuspend won't
* take place until the autosuspend_delay has elapsed and all the other
* necessary conditions are satisfied.
*
* The caller must hold @udev's device lock.
*/
void usb_enable_autosuspend(struct usb_device *udev)
{
pm_runtime_allow(&udev->dev);
}
EXPORT_SYMBOL_GPL(usb_enable_autosuspend);
/**
* usb_disable_autosuspend - prevent a USB device from being autosuspended
* @udev: the USB device which may not be autosuspended
*
* This routine prevents @udev from being autosuspended and wakes it up
* if it is already autosuspended.
*
* The caller must hold @udev's device lock.
*/
void usb_disable_autosuspend(struct usb_device *udev)
{
pm_runtime_forbid(&udev->dev);
}
EXPORT_SYMBOL_GPL(usb_disable_autosuspend);
/**
* usb_autosuspend_device - delayed autosuspend of a USB device and its interfaces
* @udev: the usb_device to autosuspend
*
* This routine should be called when a core subsystem is finished using
* @udev and wants to allow it to autosuspend. Examples would be when
* @udev's device file in usbfs is closed or after a configuration change.
*
* @udev's usage counter is decremented; if it drops to 0 and all the
* interfaces are inactive then a delayed autosuspend will be attempted.
* The attempt may fail (see autosuspend_check()).
*
* The caller must hold @udev's device lock.
*
* This routine can run only in process context.
*/
void usb_autosuspend_device(struct usb_device *udev)
{
int status;
usb_mark_last_busy(udev);
status = pm_runtime_put_sync_autosuspend(&udev->dev);
dev_vdbg(&udev->dev, "%s: cnt %d -> %d\n",
__func__, atomic_read(&udev->dev.power.usage_count),
status);
}
/**
* usb_autoresume_device - immediately autoresume a USB device and its interfaces
* @udev: the usb_device to autoresume
*
* This routine should be called when a core subsystem wants to use @udev
* and needs to guarantee that it is not suspended. No autosuspend will
* occur until usb_autosuspend_device() is called. (Note that this will
* not prevent suspend events originating in the PM core.) Examples would
* be when @udev's device file in usbfs is opened or when a remote-wakeup
* request is received.
*
* @udev's usage counter is incremented to prevent subsequent autosuspends.
* However if the autoresume fails then the usage counter is re-decremented.
*
* The caller must hold @udev's device lock.
*
* This routine can run only in process context.
*
* Return: 0 on success. A negative error code otherwise.
*/
int usb_autoresume_device(struct usb_device *udev)
{
int status;
status = pm_runtime_resume_and_get(&udev->dev);
dev_vdbg(&udev->dev, "%s: cnt %d -> %d\n",
__func__, atomic_read(&udev->dev.power.usage_count),
status);
if (status > 0)
status = 0;
return status;
}
/**
* usb_autopm_put_interface - decrement a USB interface's PM-usage counter
* @intf: the usb_interface whose counter should be decremented
*
* This routine should be called by an interface driver when it is
* finished using @intf and wants to allow it to autosuspend. A typical
* example would be a character-device driver when its device file is
* closed.
*
* The routine decrements @intf's usage counter. When the counter reaches
* 0, a delayed autosuspend request for @intf's device is attempted. The
* attempt may fail (see autosuspend_check()).
*
* This routine can run only in process context.
*/
void usb_autopm_put_interface(struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
int status;
usb_mark_last_busy(udev);
status = pm_runtime_put_sync(&intf->dev);
dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
__func__, atomic_read(&intf->dev.power.usage_count),
status);
}
EXPORT_SYMBOL_GPL(usb_autopm_put_interface);
/**
* usb_autopm_put_interface_async - decrement a USB interface's PM-usage counter
* @intf: the usb_interface whose counter should be decremented
*
* This routine does much the same thing as usb_autopm_put_interface():
* It decrements @intf's usage counter and schedules a delayed
* autosuspend request if the counter is <= 0. The difference is that it
* does not perform any synchronization; callers should hold a private
* lock and handle all synchronization issues themselves.
*
* Typically a driver would call this routine during an URB's completion
* handler, if no more URBs were pending.
*
* This routine can run in atomic context.
*/
void usb_autopm_put_interface_async(struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
int status;
usb_mark_last_busy(udev);
status = pm_runtime_put(&intf->dev);
dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
__func__, atomic_read(&intf->dev.power.usage_count),
status);
}
EXPORT_SYMBOL_GPL(usb_autopm_put_interface_async);
/**
* usb_autopm_put_interface_no_suspend - decrement a USB interface's PM-usage counter
* @intf: the usb_interface whose counter should be decremented
*
* This routine decrements @intf's usage counter but does not carry out an
* autosuspend.
*
* This routine can run in atomic context.
*/
void usb_autopm_put_interface_no_suspend(struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
usb_mark_last_busy(udev);
pm_runtime_put_noidle(&intf->dev);
}
EXPORT_SYMBOL_GPL(usb_autopm_put_interface_no_suspend);
/**
* usb_autopm_get_interface - increment a USB interface's PM-usage counter
* @intf: the usb_interface whose counter should be incremented
*
* This routine should be called by an interface driver when it wants to
* use @intf and needs to guarantee that it is not suspended. In addition,
* the routine prevents @intf from being autosuspended subsequently. (Note
* that this will not prevent suspend events originating in the PM core.)
* This prevention will persist until usb_autopm_put_interface() is called
* or @intf is unbound. A typical example would be a character-device
* driver when its device file is opened.
*
* @intf's usage counter is incremented to prevent subsequent autosuspends.
* However if the autoresume fails then the counter is re-decremented.
*
* This routine can run only in process context.
*
* Return: 0 on success.
*/
int usb_autopm_get_interface(struct usb_interface *intf)
{
int status;
status = pm_runtime_resume_and_get(&intf->dev);
dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
__func__, atomic_read(&intf->dev.power.usage_count),
status);
if (status > 0)
status = 0;
return status;
}
EXPORT_SYMBOL_GPL(usb_autopm_get_interface);
/**
* usb_autopm_get_interface_async - increment a USB interface's PM-usage counter
* @intf: the usb_interface whose counter should be incremented
*
* This routine does much the same thing as
* usb_autopm_get_interface(): It increments @intf's usage counter and
* queues an autoresume request if the device is suspended. The
* differences are that it does not perform any synchronization (callers
* should hold a private lock and handle all synchronization issues
* themselves), and it does not autoresume the device directly (it only
* queues a request). After a successful call, the device may not yet be
* resumed.
*
* This routine can run in atomic context.
*
* Return: 0 on success. A negative error code otherwise.
*/
int usb_autopm_get_interface_async(struct usb_interface *intf)
{
int status;
status = pm_runtime_get(&intf->dev); if (status < 0 && status != -EINPROGRESS) pm_runtime_put_noidle(&intf->dev);
dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
__func__, atomic_read(&intf->dev.power.usage_count),
status);
if (status > 0 || status == -EINPROGRESS)
status = 0;
return status;
}
EXPORT_SYMBOL_GPL(usb_autopm_get_interface_async);
/**
* usb_autopm_get_interface_no_resume - increment a USB interface's PM-usage counter
* @intf: the usb_interface whose counter should be incremented
*
* This routine increments @intf's usage counter but does not carry out an
* autoresume.
*
* This routine can run in atomic context.
*/
void usb_autopm_get_interface_no_resume(struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
usb_mark_last_busy(udev);
pm_runtime_get_noresume(&intf->dev);
}
EXPORT_SYMBOL_GPL(usb_autopm_get_interface_no_resume);
/* Internal routine to check whether we may autosuspend a device. */
static int autosuspend_check(struct usb_device *udev)
{
int w, i;
struct usb_interface *intf;
if (udev->state == USB_STATE_NOTATTACHED)
return -ENODEV;
/* Fail if autosuspend is disabled, or any interfaces are in use, or
* any interface drivers require remote wakeup but it isn't available.
*/
w = 0;
if (udev->actconfig) { for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) { intf = udev->actconfig->interface[i];
/* We don't need to check interfaces that are
* disabled for runtime PM. Either they are unbound
* or else their drivers don't support autosuspend
* and so they are permanently active.
*/
if (intf->dev.power.disable_depth)
continue;
if (atomic_read(&intf->dev.power.usage_count) > 0)
return -EBUSY;
w |= intf->needs_remote_wakeup;
/* Don't allow autosuspend if the device will need
* a reset-resume and any of its interface drivers
* doesn't include support or needs remote wakeup.
*/
if (udev->quirks & USB_QUIRK_RESET_RESUME) {
struct usb_driver *driver;
driver = to_usb_driver(intf->dev.driver); if (!driver->reset_resume ||
intf->needs_remote_wakeup)
return -EOPNOTSUPP;
}
}
}
if (w && !device_can_wakeup(&udev->dev)) { dev_dbg(&udev->dev, "remote wakeup needed for autosuspend\n");
return -EOPNOTSUPP;
}
/*
* If the device is a direct child of the root hub and the HCD
* doesn't handle wakeup requests, don't allow autosuspend when
* wakeup is needed.
*/
if (w && udev->parent == udev->bus->root_hub &&
bus_to_hcd(udev->bus)->cant_recv_wakeups) {
dev_dbg(&udev->dev, "HCD doesn't handle wakeup requests\n");
return -EOPNOTSUPP;
}
udev->do_remote_wakeup = w; return 0;
}
int usb_runtime_suspend(struct device *dev)
{
struct usb_device *udev = to_usb_device(dev);
int status;
/* A USB device can be suspended if it passes the various autosuspend
* checks. Runtime suspend for a USB device means suspending all the
* interfaces and then the device itself.
*/
if (autosuspend_check(udev) != 0)
return -EAGAIN;
status = usb_suspend_both(udev, PMSG_AUTO_SUSPEND);
/* Allow a retry if autosuspend failed temporarily */
if (status == -EAGAIN || status == -EBUSY) usb_mark_last_busy(udev);
/*
* The PM core reacts badly unless the return code is 0,
* -EAGAIN, or -EBUSY, so always return -EBUSY on an error
* (except for root hubs, because they don't suspend through
* an upstream port like other USB devices).
*/
if (status != 0 && udev->parent) return -EBUSY;
return status;
}
int usb_runtime_resume(struct device *dev)
{
struct usb_device *udev = to_usb_device(dev);
int status;
/* Runtime resume for a USB device means resuming both the device
* and all its interfaces.
*/
status = usb_resume_both(udev, PMSG_AUTO_RESUME);
return status;
}
int usb_runtime_idle(struct device *dev)
{
struct usb_device *udev = to_usb_device(dev);
/* An idle USB device can be suspended if it passes the various
* autosuspend checks.
*/
if (autosuspend_check(udev) == 0)
pm_runtime_autosuspend(dev);
/* Tell the core not to suspend it, though. */
return -EBUSY;
}
static int usb_set_usb2_hardware_lpm(struct usb_device *udev, int enable)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
int ret = -EPERM;
if (hcd->driver->set_usb2_hw_lpm) {
ret = hcd->driver->set_usb2_hw_lpm(hcd, udev, enable);
if (!ret)
udev->usb2_hw_lpm_enabled = enable;
}
return ret;
}
int usb_enable_usb2_hardware_lpm(struct usb_device *udev)
{
if (!udev->usb2_hw_lpm_capable ||
!udev->usb2_hw_lpm_allowed ||
udev->usb2_hw_lpm_enabled)
return 0; return usb_set_usb2_hardware_lpm(udev, 1);
}
int usb_disable_usb2_hardware_lpm(struct usb_device *udev)
{
if (!udev->usb2_hw_lpm_enabled) return 0; return usb_set_usb2_hardware_lpm(udev, 0);
}
#endif /* CONFIG_PM */
const struct bus_type usb_bus_type = {
.name = "usb",
.match = usb_device_match,
.uevent = usb_uevent,
.need_parent_lock = true,
};
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Percpu refcounts:
* (C) 2012 Google, Inc.
* Author: Kent Overstreet <koverstreet@google.com>
*
* This implements a refcount with similar semantics to atomic_t - atomic_inc(),
* atomic_dec_and_test() - but percpu.
*
* There's one important difference between percpu refs and normal atomic_t
* refcounts; you have to keep track of your initial refcount, and then when you
* start shutting down you call percpu_ref_kill() _before_ dropping the initial
* refcount.
*
* The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
* than an atomic_t - this is because of the way shutdown works, see
* percpu_ref_kill()/PERCPU_COUNT_BIAS.
*
* Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
* refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
* puts the ref back in single atomic_t mode, collecting the per cpu refs and
* issuing the appropriate barriers, and then marks the ref as shutting down so
* that percpu_ref_put() will check for the ref hitting 0. After it returns,
* it's safe to drop the initial ref.
*
* USAGE:
*
* See fs/aio.c for some example usage; it's used there for struct kioctx, which
* is created when userspaces calls io_setup(), and destroyed when userspace
* calls io_destroy() or the process exits.
*
* In the aio code, kill_ioctx() is called when we wish to destroy a kioctx; it
* removes the kioctx from the proccess's table of kioctxs and kills percpu_ref.
* After that, there can't be any new users of the kioctx (from lookup_ioctx())
* and it's then safe to drop the initial ref with percpu_ref_put().
*
* Note that the free path, free_ioctx(), needs to go through explicit call_rcu()
* to synchronize with RCU protected lookup_ioctx(). percpu_ref operations don't
* imply RCU grace periods of any kind and if a user wants to combine percpu_ref
* with RCU protection, it must be done explicitly.
*
* Code that does a two stage shutdown like this often needs some kind of
* explicit synchronization to ensure the initial refcount can only be dropped
* once - percpu_ref_kill() does this for you, it returns true once and false if
* someone else already called it. The aio code uses it this way, but it's not
* necessary if the code has some other mechanism to synchronize teardown.
* around.
*/
#ifndef _LINUX_PERCPU_REFCOUNT_H
#define _LINUX_PERCPU_REFCOUNT_H
#include <linux/atomic.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/types.h>
#include <linux/gfp.h>
struct percpu_ref;
typedef void (percpu_ref_func_t)(struct percpu_ref *);
/* flags set in the lower bits of percpu_ref->percpu_count_ptr */
enum {
__PERCPU_REF_ATOMIC = 1LU << 0, /* operating in atomic mode */
__PERCPU_REF_DEAD = 1LU << 1, /* (being) killed */
__PERCPU_REF_ATOMIC_DEAD = __PERCPU_REF_ATOMIC | __PERCPU_REF_DEAD,
__PERCPU_REF_FLAG_BITS = 2,
};
/* @flags for percpu_ref_init() */
enum {
/*
* Start w/ ref == 1 in atomic mode. Can be switched to percpu
* operation using percpu_ref_switch_to_percpu(). If initialized
* with this flag, the ref will stay in atomic mode until
* percpu_ref_switch_to_percpu() is invoked on it.
* Implies ALLOW_REINIT.
*/
PERCPU_REF_INIT_ATOMIC = 1 << 0,
/*
* Start dead w/ ref == 0 in atomic mode. Must be revived with
* percpu_ref_reinit() before used. Implies INIT_ATOMIC and
* ALLOW_REINIT.
*/
PERCPU_REF_INIT_DEAD = 1 << 1,
/*
* Allow switching from atomic mode to percpu mode.
*/
PERCPU_REF_ALLOW_REINIT = 1 << 2,
};
struct percpu_ref_data {
atomic_long_t count;
percpu_ref_func_t *release;
percpu_ref_func_t *confirm_switch;
bool force_atomic:1;
bool allow_reinit:1;
struct rcu_head rcu;
struct percpu_ref *ref;
};
struct percpu_ref {
/*
* The low bit of the pointer indicates whether the ref is in percpu
* mode; if set, then get/put will manipulate the atomic_t.
*/
unsigned long percpu_count_ptr;
/*
* 'percpu_ref' is often embedded into user structure, and only
* 'percpu_count_ptr' is required in fast path, move other fields
* into 'percpu_ref_data', so we can reduce memory footprint in
* fast path.
*/
struct percpu_ref_data *data;
};
int __must_check percpu_ref_init(struct percpu_ref *ref,
percpu_ref_func_t *release, unsigned int flags,
gfp_t gfp);
void percpu_ref_exit(struct percpu_ref *ref);
void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
percpu_ref_func_t *confirm_switch);
void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref);
void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
percpu_ref_func_t *confirm_kill);
void percpu_ref_resurrect(struct percpu_ref *ref);
void percpu_ref_reinit(struct percpu_ref *ref);
bool percpu_ref_is_zero(struct percpu_ref *ref);
/**
* percpu_ref_kill - drop the initial ref
* @ref: percpu_ref to kill
*
* Must be used to drop the initial ref on a percpu refcount; must be called
* precisely once before shutdown.
*
* Switches @ref into atomic mode before gathering up the percpu counters
* and dropping the initial ref.
*
* There are no implied RCU grace periods between kill and release.
*/
static inline void percpu_ref_kill(struct percpu_ref *ref)
{
percpu_ref_kill_and_confirm(ref, NULL);
}
/*
* Internal helper. Don't use outside percpu-refcount proper. The
* function doesn't return the pointer and let the caller test it for NULL
* because doing so forces the compiler to generate two conditional
* branches as it can't assume that @ref->percpu_count is not NULL.
*/
static inline bool __ref_is_percpu(struct percpu_ref *ref,
unsigned long __percpu **percpu_countp)
{
unsigned long percpu_ptr;
/*
* The value of @ref->percpu_count_ptr is tested for
* !__PERCPU_REF_ATOMIC, which may be set asynchronously, and then
* used as a pointer. If the compiler generates a separate fetch
* when using it as a pointer, __PERCPU_REF_ATOMIC may be set in
* between contaminating the pointer value, meaning that
* READ_ONCE() is required when fetching it.
*
* The dependency ordering from the READ_ONCE() pairs
* with smp_store_release() in __percpu_ref_switch_to_percpu().
*/
percpu_ptr = READ_ONCE(ref->percpu_count_ptr);
/*
* Theoretically, the following could test just ATOMIC; however,
* then we'd have to mask off DEAD separately as DEAD may be
* visible without ATOMIC if we race with percpu_ref_kill(). DEAD
* implies ATOMIC anyway. Test them together.
*/
if (unlikely(percpu_ptr & __PERCPU_REF_ATOMIC_DEAD))
return false;
*percpu_countp = (unsigned long __percpu *)percpu_ptr;
return true;
}
/**
* percpu_ref_get_many - increment a percpu refcount
* @ref: percpu_ref to get
* @nr: number of references to get
*
* Analogous to atomic_long_add().
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)
{
unsigned long __percpu *percpu_count;
rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) this_cpu_add(*percpu_count, nr);
else
atomic_long_add(nr, &ref->data->count); rcu_read_unlock();
}
/**
* percpu_ref_get - increment a percpu refcount
* @ref: percpu_ref to get
*
* Analogous to atomic_long_inc().
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline void percpu_ref_get(struct percpu_ref *ref)
{
percpu_ref_get_many(ref, 1);
}
/**
* percpu_ref_tryget_many - try to increment a percpu refcount
* @ref: percpu_ref to try-get
* @nr: number of references to get
*
* Increment a percpu refcount by @nr unless its count already reached zero.
* Returns %true on success; %false on failure.
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline bool percpu_ref_tryget_many(struct percpu_ref *ref,
unsigned long nr)
{
unsigned long __percpu *percpu_count;
bool ret;
rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) { this_cpu_add(*percpu_count, nr);
ret = true;
} else {
ret = atomic_long_add_unless(&ref->data->count, nr, 0);
}
rcu_read_unlock();
return ret;
}
/**
* percpu_ref_tryget - try to increment a percpu refcount
* @ref: percpu_ref to try-get
*
* Increment a percpu refcount unless its count already reached zero.
* Returns %true on success; %false on failure.
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline bool percpu_ref_tryget(struct percpu_ref *ref)
{
return percpu_ref_tryget_many(ref, 1);
}
/**
* percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the
* caller is responsible for taking RCU.
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline bool percpu_ref_tryget_live_rcu(struct percpu_ref *ref)
{
unsigned long __percpu *percpu_count;
bool ret = false;
WARN_ON_ONCE(!rcu_read_lock_held());
if (likely(__ref_is_percpu(ref, &percpu_count))) {
this_cpu_inc(*percpu_count);
ret = true;
} else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) {
ret = atomic_long_inc_not_zero(&ref->data->count);
}
return ret;
}
/**
* percpu_ref_tryget_live - try to increment a live percpu refcount
* @ref: percpu_ref to try-get
*
* Increment a percpu refcount unless it has already been killed. Returns
* %true on success; %false on failure.
*
* Completion of percpu_ref_kill() in itself doesn't guarantee that this
* function will fail. For such guarantee, percpu_ref_kill_and_confirm()
* should be used. After the confirm_kill callback is invoked, it's
* guaranteed that no new reference will be given out by
* percpu_ref_tryget_live().
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
{
bool ret = false;
rcu_read_lock();
ret = percpu_ref_tryget_live_rcu(ref);
rcu_read_unlock();
return ret;
}
/**
* percpu_ref_put_many - decrement a percpu refcount
* @ref: percpu_ref to put
* @nr: number of references to put
*
* Decrement the refcount, and if 0, call the release function (which was passed
* to percpu_ref_init())
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)
{
unsigned long __percpu *percpu_count;
rcu_read_lock(); if (__ref_is_percpu(ref, &percpu_count)) this_cpu_sub(*percpu_count, nr); else if (unlikely(atomic_long_sub_and_test(nr, &ref->data->count))) ref->data->release(ref); rcu_read_unlock();
}
/**
* percpu_ref_put - decrement a percpu refcount
* @ref: percpu_ref to put
*
* Decrement the refcount, and if 0, call the release function (which was passed
* to percpu_ref_init())
*
* This function is safe to call as long as @ref is between init and exit.
*/
static inline void percpu_ref_put(struct percpu_ref *ref)
{
percpu_ref_put_many(ref, 1);
}
/**
* percpu_ref_is_dying - test whether a percpu refcount is dying or dead
* @ref: percpu_ref to test
*
* Returns %true if @ref is dying or dead.
*
* This function is safe to call as long as @ref is between init and exit
* and the caller is responsible for synchronizing against state changes.
*/
static inline bool percpu_ref_is_dying(struct percpu_ref *ref)
{
return ref->percpu_count_ptr & __PERCPU_REF_DEAD;
}
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/base/power/main.c - Where the driver meets power management.
*
* Copyright (c) 2003 Patrick Mochel
* Copyright (c) 2003 Open Source Development Lab
*
* The driver model core calls device_pm_add() when a device is registered.
* This will initialize the embedded device_pm_info object in the device
* and add it to the list of power-controlled devices. sysfs entries for
* controlling device power management will also be added.
*
* A separate list is used for keeping track of power info, because the power
* domain dependencies may differ from the ancestral dependencies that the
* subsystem list maintains.
*/
#define pr_fmt(fmt) "PM: " fmt
#define dev_fmt pr_fmt
#include <linux/device.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include <linux/pm.h>
#include <linux/pm_runtime.h>
#include <linux/pm-trace.h>
#include <linux/pm_wakeirq.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/async.h>
#include <linux/suspend.h>
#include <trace/events/power.h>
#include <linux/cpufreq.h>
#include <linux/devfreq.h>
#include <linux/timer.h>
#include "../base.h"
#include "power.h"
typedef int (*pm_callback_t)(struct device *);
#define list_for_each_entry_rcu_locked(pos, head, member) \
list_for_each_entry_rcu(pos, head, member, \
device_links_read_lock_held())
/*
* The entries in the dpm_list list are in a depth first order, simply
* because children are guaranteed to be discovered after parents, and
* are inserted at the back of the list on discovery.
*
* Since device_pm_add() may be called with a device lock held,
* we must never try to acquire a device lock while holding
* dpm_list_mutex.
*/
LIST_HEAD(dpm_list);
static LIST_HEAD(dpm_prepared_list);
static LIST_HEAD(dpm_suspended_list);
static LIST_HEAD(dpm_late_early_list);
static LIST_HEAD(dpm_noirq_list);
static DEFINE_MUTEX(dpm_list_mtx);
static pm_message_t pm_transition;
static int async_error;
static const char *pm_verb(int event)
{
switch (event) {
case PM_EVENT_SUSPEND:
return "suspend";
case PM_EVENT_RESUME:
return "resume";
case PM_EVENT_FREEZE:
return "freeze";
case PM_EVENT_QUIESCE:
return "quiesce";
case PM_EVENT_HIBERNATE:
return "hibernate";
case PM_EVENT_THAW:
return "thaw";
case PM_EVENT_RESTORE:
return "restore";
case PM_EVENT_RECOVER:
return "recover";
default:
return "(unknown PM event)";
}
}
/**
* device_pm_sleep_init - Initialize system suspend-related device fields.
* @dev: Device object being initialized.
*/
void device_pm_sleep_init(struct device *dev)
{
dev->power.is_prepared = false;
dev->power.is_suspended = false;
dev->power.is_noirq_suspended = false;
dev->power.is_late_suspended = false;
init_completion(&dev->power.completion);
complete_all(&dev->power.completion);
dev->power.wakeup = NULL;
INIT_LIST_HEAD(&dev->power.entry);
}
/**
* device_pm_lock - Lock the list of active devices used by the PM core.
*/
void device_pm_lock(void)
{
mutex_lock(&dpm_list_mtx);
}
/**
* device_pm_unlock - Unlock the list of active devices used by the PM core.
*/
void device_pm_unlock(void)
{
mutex_unlock(&dpm_list_mtx);
}
/**
* device_pm_add - Add a device to the PM core's list of active devices.
* @dev: Device to add to the list.
*/
void device_pm_add(struct device *dev)
{
/* Skip PM setup/initialization. */
if (device_pm_not_required(dev))
return;
pr_debug("Adding info for %s:%s\n",
dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
device_pm_check_callbacks(dev);
mutex_lock(&dpm_list_mtx);
if (dev->parent && dev->parent->power.is_prepared) dev_warn(dev, "parent %s should not be sleeping\n",
dev_name(dev->parent));
list_add_tail(&dev->power.entry, &dpm_list); dev->power.in_dpm_list = true;
mutex_unlock(&dpm_list_mtx);
}
/**
* device_pm_remove - Remove a device from the PM core's list of active devices.
* @dev: Device to be removed from the list.
*/
void device_pm_remove(struct device *dev)
{
if (device_pm_not_required(dev))
return;
pr_debug("Removing info for %s:%s\n",
dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
complete_all(&dev->power.completion);
mutex_lock(&dpm_list_mtx);
list_del_init(&dev->power.entry);
dev->power.in_dpm_list = false;
mutex_unlock(&dpm_list_mtx);
device_wakeup_disable(dev);
pm_runtime_remove(dev);
device_pm_check_callbacks(dev);
}
/**
* device_pm_move_before - Move device in the PM core's list of active devices.
* @deva: Device to move in dpm_list.
* @devb: Device @deva should come before.
*/
void device_pm_move_before(struct device *deva, struct device *devb)
{
pr_debug("Moving %s:%s before %s:%s\n",
deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
/* Delete deva from dpm_list and reinsert before devb. */
list_move_tail(&deva->power.entry, &devb->power.entry);
}
/**
* device_pm_move_after - Move device in the PM core's list of active devices.
* @deva: Device to move in dpm_list.
* @devb: Device @deva should come after.
*/
void device_pm_move_after(struct device *deva, struct device *devb)
{
pr_debug("Moving %s:%s after %s:%s\n",
deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
/* Delete deva from dpm_list and reinsert after devb. */
list_move(&deva->power.entry, &devb->power.entry);
}
/**
* device_pm_move_last - Move device to end of the PM core's list of devices.
* @dev: Device to move in dpm_list.
*/
void device_pm_move_last(struct device *dev)
{
pr_debug("Moving %s:%s to end of list\n",
dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
list_move_tail(&dev->power.entry, &dpm_list);
}
static ktime_t initcall_debug_start(struct device *dev, void *cb)
{
if (!pm_print_times_enabled)
return 0;
dev_info(dev, "calling %pS @ %i, parent: %s\n", cb,
task_pid_nr(current),
dev->parent ? dev_name(dev->parent) : "none");
return ktime_get();
}
static void initcall_debug_report(struct device *dev, ktime_t calltime,
void *cb, int error)
{
ktime_t rettime;
if (!pm_print_times_enabled)
return;
rettime = ktime_get();
dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error,
(unsigned long long)ktime_us_delta(rettime, calltime));
}
/**
* dpm_wait - Wait for a PM operation to complete.
* @dev: Device to wait for.
* @async: If unset, wait only if the device's power.async_suspend flag is set.
*/
static void dpm_wait(struct device *dev, bool async)
{
if (!dev)
return;
if (async || (pm_async_enabled && dev->power.async_suspend))
wait_for_completion(&dev->power.completion);
}
static int dpm_wait_fn(struct device *dev, void *async_ptr)
{
dpm_wait(dev, *((bool *)async_ptr));
return 0;
}
static void dpm_wait_for_children(struct device *dev, bool async)
{
device_for_each_child(dev, &async, dpm_wait_fn);
}
static void dpm_wait_for_suppliers(struct device *dev, bool async)
{
struct device_link *link;
int idx;
idx = device_links_read_lock();
/*
* If the supplier goes away right after we've checked the link to it,
* we'll wait for its completion to change the state, but that's fine,
* because the only things that will block as a result are the SRCU
* callbacks freeing the link objects for the links in the list we're
* walking.
*/
list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
if (READ_ONCE(link->status) != DL_STATE_DORMANT)
dpm_wait(link->supplier, async);
device_links_read_unlock(idx);
}
static bool dpm_wait_for_superior(struct device *dev, bool async)
{
struct device *parent;
/*
* If the device is resumed asynchronously and the parent's callback
* deletes both the device and the parent itself, the parent object may
* be freed while this function is running, so avoid that by reference
* counting the parent once more unless the device has been deleted
* already (in which case return right away).
*/
mutex_lock(&dpm_list_mtx);
if (!device_pm_initialized(dev)) {
mutex_unlock(&dpm_list_mtx);
return false;
}
parent = get_device(dev->parent);
mutex_unlock(&dpm_list_mtx);
dpm_wait(parent, async);
put_device(parent);
dpm_wait_for_suppliers(dev, async);
/*
* If the parent's callback has deleted the device, attempting to resume
* it would be invalid, so avoid doing that then.
*/
return device_pm_initialized(dev);
}
static void dpm_wait_for_consumers(struct device *dev, bool async)
{
struct device_link *link;
int idx;
idx = device_links_read_lock();
/*
* The status of a device link can only be changed from "dormant" by a
* probe, but that cannot happen during system suspend/resume. In
* theory it can change to "dormant" at that time, but then it is
* reasonable to wait for the target device anyway (eg. if it goes
* away, it's better to wait for it to go away completely and then
* continue instead of trying to continue in parallel with its
* unregistration).
*/
list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node)
if (READ_ONCE(link->status) != DL_STATE_DORMANT)
dpm_wait(link->consumer, async);
device_links_read_unlock(idx);
}
static void dpm_wait_for_subordinate(struct device *dev, bool async)
{
dpm_wait_for_children(dev, async);
dpm_wait_for_consumers(dev, async);
}
/**
* pm_op - Return the PM operation appropriate for given PM event.
* @ops: PM operations to choose from.
* @state: PM transition of the system being carried out.
*/
static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
{
switch (state.event) {
#ifdef CONFIG_SUSPEND
case PM_EVENT_SUSPEND:
return ops->suspend;
case PM_EVENT_RESUME:
return ops->resume;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
case PM_EVENT_FREEZE:
case PM_EVENT_QUIESCE:
return ops->freeze;
case PM_EVENT_HIBERNATE:
return ops->poweroff;
case PM_EVENT_THAW:
case PM_EVENT_RECOVER:
return ops->thaw;
case PM_EVENT_RESTORE:
return ops->restore;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
}
return NULL;
}
/**
* pm_late_early_op - Return the PM operation appropriate for given PM event.
* @ops: PM operations to choose from.
* @state: PM transition of the system being carried out.
*
* Runtime PM is disabled for @dev while this function is being executed.
*/
static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
pm_message_t state)
{
switch (state.event) {
#ifdef CONFIG_SUSPEND
case PM_EVENT_SUSPEND:
return ops->suspend_late;
case PM_EVENT_RESUME:
return ops->resume_early;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
case PM_EVENT_FREEZE:
case PM_EVENT_QUIESCE:
return ops->freeze_late;
case PM_EVENT_HIBERNATE:
return ops->poweroff_late;
case PM_EVENT_THAW:
case PM_EVENT_RECOVER:
return ops->thaw_early;
case PM_EVENT_RESTORE:
return ops->restore_early;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
}
return NULL;
}
/**
* pm_noirq_op - Return the PM operation appropriate for given PM event.
* @ops: PM operations to choose from.
* @state: PM transition of the system being carried out.
*
* The driver of @dev will not receive interrupts while this function is being
* executed.
*/
static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
{
switch (state.event) {
#ifdef CONFIG_SUSPEND
case PM_EVENT_SUSPEND:
return ops->suspend_noirq;
case PM_EVENT_RESUME:
return ops->resume_noirq;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
case PM_EVENT_FREEZE:
case PM_EVENT_QUIESCE:
return ops->freeze_noirq;
case PM_EVENT_HIBERNATE:
return ops->poweroff_noirq;
case PM_EVENT_THAW:
case PM_EVENT_RECOVER:
return ops->thaw_noirq;
case PM_EVENT_RESTORE:
return ops->restore_noirq;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
}
return NULL;
}
static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
{
dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event),
((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
", may wakeup" : "", dev->power.driver_flags);
}
static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
int error)
{
dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info,
error);
}
static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
const char *info)
{
ktime_t calltime;
u64 usecs64;
int usecs;
calltime = ktime_get();
usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
do_div(usecs64, NSEC_PER_USEC);
usecs = usecs64;
if (usecs == 0)
usecs = 1;
pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
info ?: "", info ? " " : "", pm_verb(state.event),
error ? "aborted" : "complete",
usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
}
static int dpm_run_callback(pm_callback_t cb, struct device *dev,
pm_message_t state, const char *info)
{
ktime_t calltime;
int error;
if (!cb)
return 0;
calltime = initcall_debug_start(dev, cb);
pm_dev_dbg(dev, state, info);
trace_device_pm_callback_start(dev, info, state.event);
error = cb(dev);
trace_device_pm_callback_end(dev, error);
suspend_report_result(dev, cb, error);
initcall_debug_report(dev, calltime, cb, error);
return error;
}
#ifdef CONFIG_DPM_WATCHDOG
struct dpm_watchdog {
struct device *dev;
struct task_struct *tsk;
struct timer_list timer;
};
#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
struct dpm_watchdog wd
/**
* dpm_watchdog_handler - Driver suspend / resume watchdog handler.
* @t: The timer that PM watchdog depends on.
*
* Called when a driver has timed out suspending or resuming.
* There's not much we can do here to recover so panic() to
* capture a crash-dump in pstore.
*/
static void dpm_watchdog_handler(struct timer_list *t)
{
struct dpm_watchdog *wd = from_timer(wd, t, timer);
dev_emerg(wd->dev, "**** DPM device timeout ****\n");
show_stack(wd->tsk, NULL, KERN_EMERG);
panic("%s %s: unrecoverable failure\n",
dev_driver_string(wd->dev), dev_name(wd->dev));
}
/**
* dpm_watchdog_set - Enable pm watchdog for given device.
* @wd: Watchdog. Must be allocated on the stack.
* @dev: Device to handle.
*/
static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
{
struct timer_list *timer = &wd->timer;
wd->dev = dev;
wd->tsk = current;
timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
/* use same timeout value for both suspend and resume */
timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
add_timer(timer);
}
/**
* dpm_watchdog_clear - Disable suspend/resume watchdog.
* @wd: Watchdog to disable.
*/
static void dpm_watchdog_clear(struct dpm_watchdog *wd)
{
struct timer_list *timer = &wd->timer;
del_timer_sync(timer);
destroy_timer_on_stack(timer);
}
#else
#define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
#define dpm_watchdog_set(x, y)
#define dpm_watchdog_clear(x)
#endif
/*------------------------- Resume routines -------------------------*/
/**
* dev_pm_skip_resume - System-wide device resume optimization check.
* @dev: Target device.
*
* Return:
* - %false if the transition under way is RESTORE.
* - Return value of dev_pm_skip_suspend() if the transition under way is THAW.
* - The logical negation of %power.must_resume otherwise (that is, when the
* transition under way is RESUME).
*/
bool dev_pm_skip_resume(struct device *dev)
{
if (pm_transition.event == PM_EVENT_RESTORE)
return false;
if (pm_transition.event == PM_EVENT_THAW)
return dev_pm_skip_suspend(dev);
return !dev->power.must_resume;
}
static bool is_async(struct device *dev)
{
return dev->power.async_suspend && pm_async_enabled
&& !pm_trace_is_enabled();
}
static bool dpm_async_fn(struct device *dev, async_func_t func)
{
reinit_completion(&dev->power.completion);
if (is_async(dev)) {
dev->power.async_in_progress = true;
get_device(dev);
if (async_schedule_dev_nocall(func, dev))
return true;
put_device(dev);
}
/*
* Because async_schedule_dev_nocall() above has returned false or it
* has not been called at all, func() is not running and it is safe to
* update the async_in_progress flag without extra synchronization.
*/
dev->power.async_in_progress = false;
return false;
}
/**
* device_resume_noirq - Execute a "noirq resume" callback for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
* @async: If true, the device is being resumed asynchronously.
*
* The driver of @dev will not receive interrupts while this function is being
* executed.
*/
static void device_resume_noirq(struct device *dev, pm_message_t state, bool async)
{
pm_callback_t callback = NULL;
const char *info = NULL;
bool skip_resume;
int error = 0;
TRACE_DEVICE(dev);
TRACE_RESUME(0);
if (dev->power.syscore || dev->power.direct_complete)
goto Out;
if (!dev->power.is_noirq_suspended)
goto Out;
if (!dpm_wait_for_superior(dev, async))
goto Out;
skip_resume = dev_pm_skip_resume(dev);
/*
* If the driver callback is skipped below or by the middle layer
* callback and device_resume_early() also skips the driver callback for
* this device later, it needs to appear as "suspended" to PM-runtime,
* so change its status accordingly.
*
* Otherwise, the device is going to be resumed, so set its PM-runtime
* status to "active", but do that only if DPM_FLAG_SMART_SUSPEND is set
* to avoid confusing drivers that don't use it.
*/
if (skip_resume)
pm_runtime_set_suspended(dev);
else if (dev_pm_skip_suspend(dev))
pm_runtime_set_active(dev);
if (dev->pm_domain) {
info = "noirq power domain ";
callback = pm_noirq_op(&dev->pm_domain->ops, state);
} else if (dev->type && dev->type->pm) {
info = "noirq type ";
callback = pm_noirq_op(dev->type->pm, state);
} else if (dev->class && dev->class->pm) {
info = "noirq class ";
callback = pm_noirq_op(dev->class->pm, state);
} else if (dev->bus && dev->bus->pm) {
info = "noirq bus ";
callback = pm_noirq_op(dev->bus->pm, state);
}
if (callback)
goto Run;
if (skip_resume)
goto Skip;
if (dev->driver && dev->driver->pm) {
info = "noirq driver ";
callback = pm_noirq_op(dev->driver->pm, state);
}
Run:
error = dpm_run_callback(callback, dev, state, info);
Skip:
dev->power.is_noirq_suspended = false;
Out:
complete_all(&dev->power.completion);
TRACE_RESUME(error);
if (error) {
async_error = error;
dpm_save_failed_dev(dev_name(dev));
pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
}
}
static void async_resume_noirq(void *data, async_cookie_t cookie)
{
struct device *dev = data;
device_resume_noirq(dev, pm_transition, true);
put_device(dev);
}
static void dpm_noirq_resume_devices(pm_message_t state)
{
struct device *dev;
ktime_t starttime = ktime_get();
trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
async_error = 0;
pm_transition = state;
mutex_lock(&dpm_list_mtx);
/*
* Trigger the resume of "async" devices upfront so they don't have to
* wait for the "non-async" ones they don't depend on.
*/
list_for_each_entry(dev, &dpm_noirq_list, power.entry)
dpm_async_fn(dev, async_resume_noirq);
while (!list_empty(&dpm_noirq_list)) {
dev = to_device(dpm_noirq_list.next);
list_move_tail(&dev->power.entry, &dpm_late_early_list);
if (!dev->power.async_in_progress) {
get_device(dev);
mutex_unlock(&dpm_list_mtx);
device_resume_noirq(dev, state, false);
put_device(dev);
mutex_lock(&dpm_list_mtx);
}
}
mutex_unlock(&dpm_list_mtx);
async_synchronize_full();
dpm_show_time(starttime, state, 0, "noirq");
if (async_error)
dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
}
/**
* dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
* @state: PM transition of the system being carried out.
*
* Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
* allow device drivers' interrupt handlers to be called.
*/
void dpm_resume_noirq(pm_message_t state)
{
dpm_noirq_resume_devices(state);
resume_device_irqs();
device_wakeup_disarm_wake_irqs();
}
/**
* device_resume_early - Execute an "early resume" callback for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
* @async: If true, the device is being resumed asynchronously.
*
* Runtime PM is disabled for @dev while this function is being executed.
*/
static void device_resume_early(struct device *dev, pm_message_t state, bool async)
{
pm_callback_t callback = NULL;
const char *info = NULL;
int error = 0;
TRACE_DEVICE(dev);
TRACE_RESUME(0);
if (dev->power.syscore || dev->power.direct_complete)
goto Out;
if (!dev->power.is_late_suspended)
goto Out;
if (!dpm_wait_for_superior(dev, async))
goto Out;
if (dev->pm_domain) {
info = "early power domain ";
callback = pm_late_early_op(&dev->pm_domain->ops, state);
} else if (dev->type && dev->type->pm) {
info = "early type ";
callback = pm_late_early_op(dev->type->pm, state);
} else if (dev->class && dev->class->pm) {
info = "early class ";
callback = pm_late_early_op(dev->class->pm, state);
} else if (dev->bus && dev->bus->pm) {
info = "early bus ";
callback = pm_late_early_op(dev->bus->pm, state);
}
if (callback)
goto Run;
if (dev_pm_skip_resume(dev))
goto Skip;
if (dev->driver && dev->driver->pm) {
info = "early driver ";
callback = pm_late_early_op(dev->driver->pm, state);
}
Run:
error = dpm_run_callback(callback, dev, state, info);
Skip:
dev->power.is_late_suspended = false;
Out:
TRACE_RESUME(error);
pm_runtime_enable(dev);
complete_all(&dev->power.completion);
if (error) {
async_error = error;
dpm_save_failed_dev(dev_name(dev));
pm_dev_err(dev, state, async ? " async early" : " early", error);
}
}
static void async_resume_early(void *data, async_cookie_t cookie)
{
struct device *dev = data;
device_resume_early(dev, pm_transition, true);
put_device(dev);
}
/**
* dpm_resume_early - Execute "early resume" callbacks for all devices.
* @state: PM transition of the system being carried out.
*/
void dpm_resume_early(pm_message_t state)
{
struct device *dev;
ktime_t starttime = ktime_get();
trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
async_error = 0;
pm_transition = state;
mutex_lock(&dpm_list_mtx);
/*
* Trigger the resume of "async" devices upfront so they don't have to
* wait for the "non-async" ones they don't depend on.
*/
list_for_each_entry(dev, &dpm_late_early_list, power.entry)
dpm_async_fn(dev, async_resume_early);
while (!list_empty(&dpm_late_early_list)) {
dev = to_device(dpm_late_early_list.next);
list_move_tail(&dev->power.entry, &dpm_suspended_list);
if (!dev->power.async_in_progress) {
get_device(dev);
mutex_unlock(&dpm_list_mtx);
device_resume_early(dev, state, false);
put_device(dev);
mutex_lock(&dpm_list_mtx);
}
}
mutex_unlock(&dpm_list_mtx);
async_synchronize_full();
dpm_show_time(starttime, state, 0, "early");
if (async_error)
dpm_save_failed_step(SUSPEND_RESUME_EARLY);
trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
}
/**
* dpm_resume_start - Execute "noirq" and "early" device callbacks.
* @state: PM transition of the system being carried out.
*/
void dpm_resume_start(pm_message_t state)
{
dpm_resume_noirq(state);
dpm_resume_early(state);
}
EXPORT_SYMBOL_GPL(dpm_resume_start);
/**
* device_resume - Execute "resume" callbacks for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
* @async: If true, the device is being resumed asynchronously.
*/
static void device_resume(struct device *dev, pm_message_t state, bool async)
{
pm_callback_t callback = NULL;
const char *info = NULL;
int error = 0;
DECLARE_DPM_WATCHDOG_ON_STACK(wd);
TRACE_DEVICE(dev);
TRACE_RESUME(0);
if (dev->power.syscore)
goto Complete;
if (dev->power.direct_complete) {
/* Match the pm_runtime_disable() in __device_suspend(). */
pm_runtime_enable(dev);
goto Complete;
}
if (!dpm_wait_for_superior(dev, async))
goto Complete;
dpm_watchdog_set(&wd, dev);
device_lock(dev);
/*
* This is a fib. But we'll allow new children to be added below
* a resumed device, even if the device hasn't been completed yet.
*/
dev->power.is_prepared = false;
if (!dev->power.is_suspended)
goto Unlock;
if (dev->pm_domain) {
info = "power domain ";
callback = pm_op(&dev->pm_domain->ops, state);
goto Driver;
}
if (dev->type && dev->type->pm) {
info = "type ";
callback = pm_op(dev->type->pm, state);
goto Driver;
}
if (dev->class && dev->class->pm) {
info = "class ";
callback = pm_op(dev->class->pm, state);
goto Driver;
}
if (dev->bus) {
if (dev->bus->pm) {
info = "bus ";
callback = pm_op(dev->bus->pm, state);
} else if (dev->bus->resume) {
info = "legacy bus ";
callback = dev->bus->resume;
goto End;
}
}
Driver:
if (!callback && dev->driver && dev->driver->pm) {
info = "driver ";
callback = pm_op(dev->driver->pm, state);
}
End:
error = dpm_run_callback(callback, dev, state, info);
dev->power.is_suspended = false;
Unlock:
device_unlock(dev);
dpm_watchdog_clear(&wd);
Complete:
complete_all(&dev->power.completion);
TRACE_RESUME(error);
if (error) {
async_error = error;
dpm_save_failed_dev(dev_name(dev));
pm_dev_err(dev, state, async ? " async" : "", error);
}
}
static void async_resume(void *data, async_cookie_t cookie)
{
struct device *dev = data;
device_resume(dev, pm_transition, true);
put_device(dev);
}
/**
* dpm_resume - Execute "resume" callbacks for non-sysdev devices.
* @state: PM transition of the system being carried out.
*
* Execute the appropriate "resume" callback for all devices whose status
* indicates that they are suspended.
*/
void dpm_resume(pm_message_t state)
{
struct device *dev;
ktime_t starttime = ktime_get();
trace_suspend_resume(TPS("dpm_resume"), state.event, true);
might_sleep();
pm_transition = state;
async_error = 0;
mutex_lock(&dpm_list_mtx);
/*
* Trigger the resume of "async" devices upfront so they don't have to
* wait for the "non-async" ones they don't depend on.
*/
list_for_each_entry(dev, &dpm_suspended_list, power.entry)
dpm_async_fn(dev, async_resume);
while (!list_empty(&dpm_suspended_list)) {
dev = to_device(dpm_suspended_list.next);
list_move_tail(&dev->power.entry, &dpm_prepared_list);
if (!dev->power.async_in_progress) {
get_device(dev);
mutex_unlock(&dpm_list_mtx);
device_resume(dev, state, false);
put_device(dev);
mutex_lock(&dpm_list_mtx);
}
}
mutex_unlock(&dpm_list_mtx);
async_synchronize_full();
dpm_show_time(starttime, state, 0, NULL);
if (async_error)
dpm_save_failed_step(SUSPEND_RESUME);
cpufreq_resume();
devfreq_resume();
trace_suspend_resume(TPS("dpm_resume"), state.event, false);
}
/**
* device_complete - Complete a PM transition for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
*/
static void device_complete(struct device *dev, pm_message_t state)
{
void (*callback)(struct device *) = NULL;
const char *info = NULL;
if (dev->power.syscore)
goto out;
device_lock(dev);
if (dev->pm_domain) {
info = "completing power domain ";
callback = dev->pm_domain->ops.complete;
} else if (dev->type && dev->type->pm) {
info = "completing type ";
callback = dev->type->pm->complete;
} else if (dev->class && dev->class->pm) {
info = "completing class ";
callback = dev->class->pm->complete;
} else if (dev->bus && dev->bus->pm) {
info = "completing bus ";
callback = dev->bus->pm->complete;
}
if (!callback && dev->driver && dev->driver->pm) {
info = "completing driver ";
callback = dev->driver->pm->complete;
}
if (callback) {
pm_dev_dbg(dev, state, info);
callback(dev);
}
device_unlock(dev);
out:
pm_runtime_put(dev);
}
/**
* dpm_complete - Complete a PM transition for all non-sysdev devices.
* @state: PM transition of the system being carried out.
*
* Execute the ->complete() callbacks for all devices whose PM status is not
* DPM_ON (this allows new devices to be registered).
*/
void dpm_complete(pm_message_t state)
{
struct list_head list;
trace_suspend_resume(TPS("dpm_complete"), state.event, true);
might_sleep();
INIT_LIST_HEAD(&list);
mutex_lock(&dpm_list_mtx);
while (!list_empty(&dpm_prepared_list)) {
struct device *dev = to_device(dpm_prepared_list.prev);
get_device(dev);
dev->power.is_prepared = false;
list_move(&dev->power.entry, &list);
mutex_unlock(&dpm_list_mtx);
trace_device_pm_callback_start(dev, "", state.event);
device_complete(dev, state);
trace_device_pm_callback_end(dev, 0);
put_device(dev);
mutex_lock(&dpm_list_mtx);
}
list_splice(&list, &dpm_list);
mutex_unlock(&dpm_list_mtx);
/* Allow device probing and trigger re-probing of deferred devices */
device_unblock_probing();
trace_suspend_resume(TPS("dpm_complete"), state.event, false);
}
/**
* dpm_resume_end - Execute "resume" callbacks and complete system transition.
* @state: PM transition of the system being carried out.
*
* Execute "resume" callbacks for all devices and complete the PM transition of
* the system.
*/
void dpm_resume_end(pm_message_t state)
{
dpm_resume(state);
dpm_complete(state);
}
EXPORT_SYMBOL_GPL(dpm_resume_end);
/*------------------------- Suspend routines -------------------------*/
/**
* resume_event - Return a "resume" message for given "suspend" sleep state.
* @sleep_state: PM message representing a sleep state.
*
* Return a PM message representing the resume event corresponding to given
* sleep state.
*/
static pm_message_t resume_event(pm_message_t sleep_state)
{
switch (sleep_state.event) {
case PM_EVENT_SUSPEND:
return PMSG_RESUME;
case PM_EVENT_FREEZE:
case PM_EVENT_QUIESCE:
return PMSG_RECOVER;
case PM_EVENT_HIBERNATE:
return PMSG_RESTORE;
}
return PMSG_ON;
}
static void dpm_superior_set_must_resume(struct device *dev)
{
struct device_link *link;
int idx;
if (dev->parent)
dev->parent->power.must_resume = true;
idx = device_links_read_lock();
list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
link->supplier->power.must_resume = true;
device_links_read_unlock(idx);
}
/**
* device_suspend_noirq - Execute a "noirq suspend" callback for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
* @async: If true, the device is being suspended asynchronously.
*
* The driver of @dev will not receive interrupts while this function is being
* executed.
*/
static int device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
{
pm_callback_t callback = NULL;
const char *info = NULL;
int error = 0;
TRACE_DEVICE(dev);
TRACE_SUSPEND(0);
dpm_wait_for_subordinate(dev, async);
if (async_error)
goto Complete;
if (dev->power.syscore || dev->power.direct_complete)
goto Complete;
if (dev->pm_domain) {
info = "noirq power domain ";
callback = pm_noirq_op(&dev->pm_domain->ops, state);
} else if (dev->type && dev->type->pm) {
info = "noirq type ";
callback = pm_noirq_op(dev->type->pm, state);
} else if (dev->class && dev->class->pm) {
info = "noirq class ";
callback = pm_noirq_op(dev->class->pm, state);
} else if (dev->bus && dev->bus->pm) {
info = "noirq bus ";
callback = pm_noirq_op(dev->bus->pm, state);
}
if (callback)
goto Run;
if (dev_pm_skip_suspend(dev))
goto Skip;
if (dev->driver && dev->driver->pm) {
info = "noirq driver ";
callback = pm_noirq_op(dev->driver->pm, state);
}
Run:
error = dpm_run_callback(callback, dev, state, info);
if (error) {
async_error = error;
dpm_save_failed_dev(dev_name(dev));
pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
goto Complete;
}
Skip:
dev->power.is_noirq_suspended = true;
/*
* Skipping the resume of devices that were in use right before the
* system suspend (as indicated by their PM-runtime usage counters)
* would be suboptimal. Also resume them if doing that is not allowed
* to be skipped.
*/
if (atomic_read(&dev->power.usage_count) > 1 ||
!(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) &&
dev->power.may_skip_resume))
dev->power.must_resume = true;
if (dev->power.must_resume)
dpm_superior_set_must_resume(dev);
Complete:
complete_all(&dev->power.completion);
TRACE_SUSPEND(error);
return error;
}
static void async_suspend_noirq(void *data, async_cookie_t cookie)
{
struct device *dev = data;
device_suspend_noirq(dev, pm_transition, true);
put_device(dev);
}
static int dpm_noirq_suspend_devices(pm_message_t state)
{
ktime_t starttime = ktime_get();
int error = 0;
trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
pm_transition = state;
async_error = 0;
mutex_lock(&dpm_list_mtx);
while (!list_empty(&dpm_late_early_list)) {
struct device *dev = to_device(dpm_late_early_list.prev);
list_move(&dev->power.entry, &dpm_noirq_list);
if (dpm_async_fn(dev, async_suspend_noirq))
continue;
get_device(dev);
mutex_unlock(&dpm_list_mtx);
error = device_suspend_noirq(dev, state, false);
put_device(dev);
mutex_lock(&dpm_list_mtx);
if (error || async_error)
break;
}
mutex_unlock(&dpm_list_mtx);
async_synchronize_full();
if (!error)
error = async_error;
if (error)
dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
dpm_show_time(starttime, state, error, "noirq");
trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
return error;
}
/**
* dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
* @state: PM transition of the system being carried out.
*
* Prevent device drivers' interrupt handlers from being called and invoke
* "noirq" suspend callbacks for all non-sysdev devices.
*/
int dpm_suspend_noirq(pm_message_t state)
{
int ret;
device_wakeup_arm_wake_irqs();
suspend_device_irqs();
ret = dpm_noirq_suspend_devices(state);
if (ret)
dpm_resume_noirq(resume_event(state));
return ret;
}
static void dpm_propagate_wakeup_to_parent(struct device *dev)
{
struct device *parent = dev->parent;
if (!parent)
return;
spin_lock_irq(&parent->power.lock);
if (device_wakeup_path(dev) && !parent->power.ignore_children)
parent->power.wakeup_path = true;
spin_unlock_irq(&parent->power.lock);
}
/**
* device_suspend_late - Execute a "late suspend" callback for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
* @async: If true, the device is being suspended asynchronously.
*
* Runtime PM is disabled for @dev while this function is being executed.
*/
static int device_suspend_late(struct device *dev, pm_message_t state, bool async)
{
pm_callback_t callback = NULL;
const char *info = NULL;
int error = 0;
TRACE_DEVICE(dev);
TRACE_SUSPEND(0);
__pm_runtime_disable(dev, false);
dpm_wait_for_subordinate(dev, async);
if (async_error)
goto Complete;
if (pm_wakeup_pending()) {
async_error = -EBUSY;
goto Complete;
}
if (dev->power.syscore || dev->power.direct_complete)
goto Complete;
if (dev->pm_domain) {
info = "late power domain ";
callback = pm_late_early_op(&dev->pm_domain->ops, state);
} else if (dev->type && dev->type->pm) {
info = "late type ";
callback = pm_late_early_op(dev->type->pm, state);
} else if (dev->class && dev->class->pm) {
info = "late class ";
callback = pm_late_early_op(dev->class->pm, state);
} else if (dev->bus && dev->bus->pm) {
info = "late bus ";
callback = pm_late_early_op(dev->bus->pm, state);
}
if (callback)
goto Run;
if (dev_pm_skip_suspend(dev))
goto Skip;
if (dev->driver && dev->driver->pm) {
info = "late driver ";
callback = pm_late_early_op(dev->driver->pm, state);
}
Run:
error = dpm_run_callback(callback, dev, state, info);
if (error) {
async_error = error;
dpm_save_failed_dev(dev_name(dev));
pm_dev_err(dev, state, async ? " async late" : " late", error);
goto Complete;
}
dpm_propagate_wakeup_to_parent(dev);
Skip:
dev->power.is_late_suspended = true;
Complete:
TRACE_SUSPEND(error);
complete_all(&dev->power.completion);
return error;
}
static void async_suspend_late(void *data, async_cookie_t cookie)
{
struct device *dev = data;
device_suspend_late(dev, pm_transition, true);
put_device(dev);
}
/**
* dpm_suspend_late - Execute "late suspend" callbacks for all devices.
* @state: PM transition of the system being carried out.
*/
int dpm_suspend_late(pm_message_t state)
{
ktime_t starttime = ktime_get();
int error = 0;
trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
pm_transition = state;
async_error = 0;
wake_up_all_idle_cpus();
mutex_lock(&dpm_list_mtx);
while (!list_empty(&dpm_suspended_list)) {
struct device *dev = to_device(dpm_suspended_list.prev);
list_move(&dev->power.entry, &dpm_late_early_list);
if (dpm_async_fn(dev, async_suspend_late))
continue;
get_device(dev);
mutex_unlock(&dpm_list_mtx);
error = device_suspend_late(dev, state, false);
put_device(dev);
mutex_lock(&dpm_list_mtx);
if (error || async_error)
break;
}
mutex_unlock(&dpm_list_mtx);
async_synchronize_full();
if (!error)
error = async_error;
if (error) {
dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
dpm_resume_early(resume_event(state));
}
dpm_show_time(starttime, state, error, "late");
trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
return error;
}
/**
* dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
* @state: PM transition of the system being carried out.
*/
int dpm_suspend_end(pm_message_t state)
{
ktime_t starttime = ktime_get();
int error;
error = dpm_suspend_late(state);
if (error)
goto out;
error = dpm_suspend_noirq(state);
if (error)
dpm_resume_early(resume_event(state));
out:
dpm_show_time(starttime, state, error, "end");
return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_end);
/**
* legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
* @dev: Device to suspend.
* @state: PM transition of the system being carried out.
* @cb: Suspend callback to execute.
* @info: string description of caller.
*/
static int legacy_suspend(struct device *dev, pm_message_t state,
int (*cb)(struct device *dev, pm_message_t state),
const char *info)
{
int error;
ktime_t calltime;
calltime = initcall_debug_start(dev, cb);
trace_device_pm_callback_start(dev, info, state.event);
error = cb(dev, state);
trace_device_pm_callback_end(dev, error);
suspend_report_result(dev, cb, error);
initcall_debug_report(dev, calltime, cb, error);
return error;
}
static void dpm_clear_superiors_direct_complete(struct device *dev)
{
struct device_link *link;
int idx;
if (dev->parent) {
spin_lock_irq(&dev->parent->power.lock);
dev->parent->power.direct_complete = false;
spin_unlock_irq(&dev->parent->power.lock);
}
idx = device_links_read_lock();
list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) {
spin_lock_irq(&link->supplier->power.lock);
link->supplier->power.direct_complete = false;
spin_unlock_irq(&link->supplier->power.lock);
}
device_links_read_unlock(idx);
}
/**
* device_suspend - Execute "suspend" callbacks for given device.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
* @async: If true, the device is being suspended asynchronously.
*/
static int device_suspend(struct device *dev, pm_message_t state, bool async)
{
pm_callback_t callback = NULL;
const char *info = NULL;
int error = 0;
DECLARE_DPM_WATCHDOG_ON_STACK(wd);
TRACE_DEVICE(dev);
TRACE_SUSPEND(0);
dpm_wait_for_subordinate(dev, async);
if (async_error) {
dev->power.direct_complete = false;
goto Complete;
}
/*
* Wait for possible runtime PM transitions of the device in progress
* to complete and if there's a runtime resume request pending for it,
* resume it before proceeding with invoking the system-wide suspend
* callbacks for it.
*
* If the system-wide suspend callbacks below change the configuration
* of the device, they must disable runtime PM for it or otherwise
* ensure that its runtime-resume callbacks will not be confused by that
* change in case they are invoked going forward.
*/
pm_runtime_barrier(dev);
if (pm_wakeup_pending()) {
dev->power.direct_complete = false;
async_error = -EBUSY;
goto Complete;
}
if (dev->power.syscore)
goto Complete;
/* Avoid direct_complete to let wakeup_path propagate. */
if (device_may_wakeup(dev) || device_wakeup_path(dev))
dev->power.direct_complete = false;
if (dev->power.direct_complete) {
if (pm_runtime_status_suspended(dev)) {
pm_runtime_disable(dev);
if (pm_runtime_status_suspended(dev)) {
pm_dev_dbg(dev, state, "direct-complete ");
goto Complete;
}
pm_runtime_enable(dev);
}
dev->power.direct_complete = false;
}
dev->power.may_skip_resume = true;
dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME);
dpm_watchdog_set(&wd, dev);
device_lock(dev);
if (dev->pm_domain) {
info = "power domain ";
callback = pm_op(&dev->pm_domain->ops, state);
goto Run;
}
if (dev->type && dev->type->pm) {
info = "type ";
callback = pm_op(dev->type->pm, state);
goto Run;
}
if (dev->class && dev->class->pm) {
info = "class ";
callback = pm_op(dev->class->pm, state);
goto Run;
}
if (dev->bus) {
if (dev->bus->pm) {
info = "bus ";
callback = pm_op(dev->bus->pm, state);
} else if (dev->bus->suspend) {
pm_dev_dbg(dev, state, "legacy bus ");
error = legacy_suspend(dev, state, dev->bus->suspend,
"legacy bus ");
goto End;
}
}
Run:
if (!callback && dev->driver && dev->driver->pm) {
info = "driver ";
callback = pm_op(dev->driver->pm, state);
}
error = dpm_run_callback(callback, dev, state, info);
End:
if (!error) {
dev->power.is_suspended = true;
if (device_may_wakeup(dev))
dev->power.wakeup_path = true;
dpm_propagate_wakeup_to_parent(dev);
dpm_clear_superiors_direct_complete(dev);
}
device_unlock(dev);
dpm_watchdog_clear(&wd);
Complete:
if (error) {
async_error = error;
dpm_save_failed_dev(dev_name(dev));
pm_dev_err(dev, state, async ? " async" : "", error);
}
complete_all(&dev->power.completion);
TRACE_SUSPEND(error);
return error;
}
static void async_suspend(void *data, async_cookie_t cookie)
{
struct device *dev = data;
device_suspend(dev, pm_transition, true);
put_device(dev);
}
/**
* dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
* @state: PM transition of the system being carried out.
*/
int dpm_suspend(pm_message_t state)
{
ktime_t starttime = ktime_get();
int error = 0;
trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
might_sleep();
devfreq_suspend();
cpufreq_suspend();
pm_transition = state;
async_error = 0;
mutex_lock(&dpm_list_mtx);
while (!list_empty(&dpm_prepared_list)) {
struct device *dev = to_device(dpm_prepared_list.prev);
list_move(&dev->power.entry, &dpm_suspended_list);
if (dpm_async_fn(dev, async_suspend))
continue;
get_device(dev);
mutex_unlock(&dpm_list_mtx);
error = device_suspend(dev, state, false);
put_device(dev);
mutex_lock(&dpm_list_mtx);
if (error || async_error)
break;
}
mutex_unlock(&dpm_list_mtx);
async_synchronize_full();
if (!error)
error = async_error;
if (error)
dpm_save_failed_step(SUSPEND_SUSPEND);
dpm_show_time(starttime, state, error, NULL);
trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
return error;
}
/**
* device_prepare - Prepare a device for system power transition.
* @dev: Device to handle.
* @state: PM transition of the system being carried out.
*
* Execute the ->prepare() callback(s) for given device. No new children of the
* device may be registered after this function has returned.
*/
static int device_prepare(struct device *dev, pm_message_t state)
{
int (*callback)(struct device *) = NULL;
int ret = 0;
/*
* If a device's parent goes into runtime suspend at the wrong time,
* it won't be possible to resume the device. To prevent this we
* block runtime suspend here, during the prepare phase, and allow
* it again during the complete phase.
*/
pm_runtime_get_noresume(dev);
if (dev->power.syscore)
return 0;
device_lock(dev);
dev->power.wakeup_path = false;
if (dev->power.no_pm_callbacks)
goto unlock;
if (dev->pm_domain)
callback = dev->pm_domain->ops.prepare;
else if (dev->type && dev->type->pm)
callback = dev->type->pm->prepare;
else if (dev->class && dev->class->pm)
callback = dev->class->pm->prepare;
else if (dev->bus && dev->bus->pm)
callback = dev->bus->pm->prepare;
if (!callback && dev->driver && dev->driver->pm)
callback = dev->driver->pm->prepare;
if (callback)
ret = callback(dev);
unlock:
device_unlock(dev);
if (ret < 0) {
suspend_report_result(dev, callback, ret);
pm_runtime_put(dev);
return ret;
}
/*
* A positive return value from ->prepare() means "this device appears
* to be runtime-suspended and its state is fine, so if it really is
* runtime-suspended, you can leave it in that state provided that you
* will do the same thing with all of its descendants". This only
* applies to suspend transitions, however.
*/
spin_lock_irq(&dev->power.lock);
dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
(ret > 0 || dev->power.no_pm_callbacks) &&
!dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
spin_unlock_irq(&dev->power.lock);
return 0;
}
/**
* dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
* @state: PM transition of the system being carried out.
*
* Execute the ->prepare() callback(s) for all devices.
*/
int dpm_prepare(pm_message_t state)
{
int error = 0;
trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
might_sleep();
/*
* Give a chance for the known devices to complete their probes, before
* disable probing of devices. This sync point is important at least
* at boot time + hibernation restore.
*/
wait_for_device_probe();
/*
* It is unsafe if probing of devices will happen during suspend or
* hibernation and system behavior will be unpredictable in this case.
* So, let's prohibit device's probing here and defer their probes
* instead. The normal behavior will be restored in dpm_complete().
*/
device_block_probing();
mutex_lock(&dpm_list_mtx);
while (!list_empty(&dpm_list) && !error) {
struct device *dev = to_device(dpm_list.next);
get_device(dev);
mutex_unlock(&dpm_list_mtx);
trace_device_pm_callback_start(dev, "", state.event);
error = device_prepare(dev, state);
trace_device_pm_callback_end(dev, error);
mutex_lock(&dpm_list_mtx);
if (!error) {
dev->power.is_prepared = true;
if (!list_empty(&dev->power.entry))
list_move_tail(&dev->power.entry, &dpm_prepared_list);
} else if (error == -EAGAIN) {
error = 0;
} else {
dev_info(dev, "not prepared for power transition: code %d\n",
error);
}
mutex_unlock(&dpm_list_mtx);
put_device(dev);
mutex_lock(&dpm_list_mtx);
}
mutex_unlock(&dpm_list_mtx);
trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
return error;
}
/**
* dpm_suspend_start - Prepare devices for PM transition and suspend them.
* @state: PM transition of the system being carried out.
*
* Prepare all non-sysdev devices for system PM transition and execute "suspend"
* callbacks for them.
*/
int dpm_suspend_start(pm_message_t state)
{
ktime_t starttime = ktime_get();
int error;
error = dpm_prepare(state);
if (error)
dpm_save_failed_step(SUSPEND_PREPARE);
else
error = dpm_suspend(state);
dpm_show_time(starttime, state, error, "start");
return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_start);
void __suspend_report_result(const char *function, struct device *dev, void *fn, int ret)
{
if (ret)
dev_err(dev, "%s(): %pS returns %d\n", function, fn, ret);
}
EXPORT_SYMBOL_GPL(__suspend_report_result);
/**
* device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
* @subordinate: Device that needs to wait for @dev.
* @dev: Device to wait for.
*/
int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
{
dpm_wait(dev, subordinate->power.async_suspend);
return async_error;
}
EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
/**
* dpm_for_each_dev - device iterator.
* @data: data for the callback.
* @fn: function to be called for each device.
*
* Iterate over devices in dpm_list, and call @fn for each device,
* passing it @data.
*/
void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
{
struct device *dev;
if (!fn)
return;
device_pm_lock();
list_for_each_entry(dev, &dpm_list, power.entry)
fn(dev, data);
device_pm_unlock();
}
EXPORT_SYMBOL_GPL(dpm_for_each_dev);
static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
{ if (!ops)
return true;
return !ops->prepare && !ops->suspend && !ops->suspend_late && !ops->suspend_noirq && !ops->resume_noirq && !ops->resume_early && !ops->resume && !ops->complete;
}
void device_pm_check_callbacks(struct device *dev)
{
unsigned long flags;
spin_lock_irqsave(&dev->power.lock, flags); dev->power.no_pm_callbacks = (!dev->bus || (pm_ops_is_empty(dev->bus->pm) && !dev->bus->suspend && !dev->bus->resume)) && (!dev->class || pm_ops_is_empty(dev->class->pm)) && (!dev->type || pm_ops_is_empty(dev->type->pm)) && (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) && (!dev->driver || (pm_ops_is_empty(dev->driver->pm) && !dev->driver->suspend && !dev->driver->resume));
spin_unlock_irqrestore(&dev->power.lock, flags);
}
bool dev_pm_skip_suspend(struct device *dev)
{
return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
pm_runtime_status_suspended(dev);
}
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/export.h>
#include <linux/nsproxy.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/user_namespace.h>
#include <linux/proc_ns.h>
#include <linux/highuid.h>
#include <linux/cred.h>
#include <linux/securebits.h>
#include <linux/security.h>
#include <linux/keyctl.h>
#include <linux/key-type.h>
#include <keys/user-type.h>
#include <linux/seq_file.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <linux/ctype.h>
#include <linux/projid.h>
#include <linux/fs_struct.h>
#include <linux/bsearch.h>
#include <linux/sort.h>
static struct kmem_cache *user_ns_cachep __ro_after_init;
static DEFINE_MUTEX(userns_state_mutex);
static bool new_idmap_permitted(const struct file *file,
struct user_namespace *ns, int cap_setid,
struct uid_gid_map *map);
static void free_user_ns(struct work_struct *work);
static struct ucounts *inc_user_namespaces(struct user_namespace *ns, kuid_t uid)
{
return inc_ucount(ns, uid, UCOUNT_USER_NAMESPACES);
}
static void dec_user_namespaces(struct ucounts *ucounts)
{
return dec_ucount(ucounts, UCOUNT_USER_NAMESPACES);
}
static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns)
{
/* Start with the same capabilities as init but useless for doing
* anything as the capabilities are bound to the new user namespace.
*/
cred->securebits = SECUREBITS_DEFAULT;
cred->cap_inheritable = CAP_EMPTY_SET;
cred->cap_permitted = CAP_FULL_SET;
cred->cap_effective = CAP_FULL_SET;
cred->cap_ambient = CAP_EMPTY_SET;
cred->cap_bset = CAP_FULL_SET;
#ifdef CONFIG_KEYS
key_put(cred->request_key_auth);
cred->request_key_auth = NULL;
#endif
/* tgcred will be cleared in our caller bc CLONE_THREAD won't be set */
cred->user_ns = user_ns;
}
static unsigned long enforced_nproc_rlimit(void)
{
unsigned long limit = RLIM_INFINITY;
/* Is RLIMIT_NPROC currently enforced? */
if (!uid_eq(current_uid(), GLOBAL_ROOT_UID) ||
(current_user_ns() != &init_user_ns))
limit = rlimit(RLIMIT_NPROC);
return limit;
}
/*
* Create a new user namespace, deriving the creator from the user in the
* passed credentials, and replacing that user with the new root user for the
* new namespace.
*
* This is called by copy_creds(), which will finish setting the target task's
* credentials.
*/
int create_user_ns(struct cred *new)
{
struct user_namespace *ns, *parent_ns = new->user_ns;
kuid_t owner = new->euid;
kgid_t group = new->egid;
struct ucounts *ucounts;
int ret, i;
ret = -ENOSPC;
if (parent_ns->level > 32)
goto fail;
ucounts = inc_user_namespaces(parent_ns, owner);
if (!ucounts)
goto fail;
/*
* Verify that we can not violate the policy of which files
* may be accessed that is specified by the root directory,
* by verifying that the root directory is at the root of the
* mount namespace which allows all files to be accessed.
*/
ret = -EPERM;
if (current_chrooted())
goto fail_dec;
/* The creator needs a mapping in the parent user namespace
* or else we won't be able to reasonably tell userspace who
* created a user_namespace.
*/
ret = -EPERM;
if (!kuid_has_mapping(parent_ns, owner) ||
!kgid_has_mapping(parent_ns, group))
goto fail_dec;
ret = security_create_user_ns(new);
if (ret < 0)
goto fail_dec;
ret = -ENOMEM;
ns = kmem_cache_zalloc(user_ns_cachep, GFP_KERNEL);
if (!ns)
goto fail_dec;
ns->parent_could_setfcap = cap_raised(new->cap_effective, CAP_SETFCAP);
ret = ns_alloc_inum(&ns->ns);
if (ret)
goto fail_free;
ns->ns.ops = &userns_operations;
refcount_set(&ns->ns.count, 1);
/* Leave the new->user_ns reference with the new user namespace. */
ns->parent = parent_ns;
ns->level = parent_ns->level + 1;
ns->owner = owner;
ns->group = group;
INIT_WORK(&ns->work, free_user_ns);
for (i = 0; i < UCOUNT_COUNTS; i++) {
ns->ucount_max[i] = INT_MAX;
}
set_userns_rlimit_max(ns, UCOUNT_RLIMIT_NPROC, enforced_nproc_rlimit());
set_userns_rlimit_max(ns, UCOUNT_RLIMIT_MSGQUEUE, rlimit(RLIMIT_MSGQUEUE));
set_userns_rlimit_max(ns, UCOUNT_RLIMIT_SIGPENDING, rlimit(RLIMIT_SIGPENDING));
set_userns_rlimit_max(ns, UCOUNT_RLIMIT_MEMLOCK, rlimit(RLIMIT_MEMLOCK));
ns->ucounts = ucounts;
/* Inherit USERNS_SETGROUPS_ALLOWED from our parent */
mutex_lock(&userns_state_mutex);
ns->flags = parent_ns->flags;
mutex_unlock(&userns_state_mutex);
#ifdef CONFIG_KEYS
INIT_LIST_HEAD(&ns->keyring_name_list);
init_rwsem(&ns->keyring_sem);
#endif
ret = -ENOMEM;
if (!setup_userns_sysctls(ns))
goto fail_keyring;
set_cred_user_ns(new, ns);
return 0;
fail_keyring:
#ifdef CONFIG_PERSISTENT_KEYRINGS
key_put(ns->persistent_keyring_register);
#endif
ns_free_inum(&ns->ns);
fail_free:
kmem_cache_free(user_ns_cachep, ns);
fail_dec:
dec_user_namespaces(ucounts);
fail:
return ret;
}
int unshare_userns(unsigned long unshare_flags, struct cred **new_cred)
{
struct cred *cred;
int err = -ENOMEM;
if (!(unshare_flags & CLONE_NEWUSER))
return 0;
cred = prepare_creds();
if (cred) {
err = create_user_ns(cred);
if (err)
put_cred(cred);
else
*new_cred = cred;
}
return err;
}
static void free_user_ns(struct work_struct *work)
{
struct user_namespace *parent, *ns =
container_of(work, struct user_namespace, work);
do {
struct ucounts *ucounts = ns->ucounts;
parent = ns->parent;
if (ns->gid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) {
kfree(ns->gid_map.forward);
kfree(ns->gid_map.reverse);
}
if (ns->uid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) {
kfree(ns->uid_map.forward);
kfree(ns->uid_map.reverse);
}
if (ns->projid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) {
kfree(ns->projid_map.forward);
kfree(ns->projid_map.reverse);
}
#if IS_ENABLED(CONFIG_BINFMT_MISC)
kfree(ns->binfmt_misc);
#endif
retire_userns_sysctls(ns);
key_free_user_ns(ns);
ns_free_inum(&ns->ns);
kmem_cache_free(user_ns_cachep, ns);
dec_user_namespaces(ucounts);
ns = parent;
} while (refcount_dec_and_test(&parent->ns.count));
}
void __put_user_ns(struct user_namespace *ns)
{
schedule_work(&ns->work);
}
EXPORT_SYMBOL(__put_user_ns);
/*
* struct idmap_key - holds the information necessary to find an idmapping in a
* sorted idmap array. It is passed to cmp_map_id() as first argument.
*/
struct idmap_key {
bool map_up; /* true -> id from kid; false -> kid from id */
u32 id; /* id to find */
u32 count; /* == 0 unless used with map_id_range_down() */
};
/*
* cmp_map_id - Function to be passed to bsearch() to find the requested
* idmapping. Expects struct idmap_key to be passed via @k.
*/
static int cmp_map_id(const void *k, const void *e)
{
u32 first, last, id2;
const struct idmap_key *key = k;
const struct uid_gid_extent *el = e;
id2 = key->id + key->count - 1;
/* handle map_id_{down,up}() */
if (key->map_up)
first = el->lower_first;
else
first = el->first;
last = first + el->count - 1;
if (key->id >= first && key->id <= last &&
(id2 >= first && id2 <= last))
return 0;
if (key->id < first || id2 < first)
return -1;
return 1;
}
/*
* map_id_range_down_max - Find idmap via binary search in ordered idmap array.
* Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
*/
static struct uid_gid_extent *
map_id_range_down_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count)
{
struct idmap_key key;
key.map_up = false;
key.count = count;
key.id = id;
return bsearch(&key, map->forward, extents,
sizeof(struct uid_gid_extent), cmp_map_id);
}
/*
* map_id_range_down_base - Find idmap via binary search in static extent array.
* Can only be called if number of mappings is equal or less than
* UID_GID_MAP_MAX_BASE_EXTENTS.
*/
static struct uid_gid_extent *
map_id_range_down_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 count)
{
unsigned idx;
u32 first, last, id2;
id2 = id + count - 1;
/* Find the matching extent */
for (idx = 0; idx < extents; idx++) { first = map->extent[idx].first; last = first + map->extent[idx].count - 1; if (id >= first && id <= last && (id2 >= first && id2 <= last)) return &map->extent[idx];
}
return NULL;
}
static u32 map_id_range_down(struct uid_gid_map *map, u32 id, u32 count)
{
struct uid_gid_extent *extent;
unsigned extents = map->nr_extents;
smp_rmb();
if (extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
extent = map_id_range_down_base(extents, map, id, count);
else
extent = map_id_range_down_max(extents, map, id, count);
/* Map the id or note failure */
if (extent) id = (id - extent->first) + extent->lower_first;
else
id = (u32) -1;
return id;
}
u32 map_id_down(struct uid_gid_map *map, u32 id)
{
return map_id_range_down(map, id, 1);
}
/*
* map_id_up_base - Find idmap via binary search in static extent array.
* Can only be called if number of mappings is equal or less than
* UID_GID_MAP_MAX_BASE_EXTENTS.
*/
static struct uid_gid_extent *
map_id_up_base(unsigned extents, struct uid_gid_map *map, u32 id)
{
unsigned idx;
u32 first, last;
/* Find the matching extent */
for (idx = 0; idx < extents; idx++) { first = map->extent[idx].lower_first; last = first + map->extent[idx].count - 1;
if (id >= first && id <= last)
return &map->extent[idx];
}
return NULL;
}
/*
* map_id_up_max - Find idmap via binary search in ordered idmap array.
* Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
*/
static struct uid_gid_extent *
map_id_up_max(unsigned extents, struct uid_gid_map *map, u32 id)
{
struct idmap_key key;
key.map_up = true;
key.count = 1;
key.id = id;
return bsearch(&key, map->reverse, extents,
sizeof(struct uid_gid_extent), cmp_map_id);
}
u32 map_id_up(struct uid_gid_map *map, u32 id)
{
struct uid_gid_extent *extent;
unsigned extents = map->nr_extents;
smp_rmb();
if (extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
extent = map_id_up_base(extents, map, id);
else
extent = map_id_up_max(extents, map, id);
/* Map the id or note failure */
if (extent) id = (id - extent->lower_first) + extent->first;
else
id = (u32) -1;
return id;
}
/**
* make_kuid - Map a user-namespace uid pair into a kuid.
* @ns: User namespace that the uid is in
* @uid: User identifier
*
* Maps a user-namespace uid pair into a kernel internal kuid,
* and returns that kuid.
*
* When there is no mapping defined for the user-namespace uid
* pair INVALID_UID is returned. Callers are expected to test
* for and handle INVALID_UID being returned. INVALID_UID
* may be tested for using uid_valid().
*/
kuid_t make_kuid(struct user_namespace *ns, uid_t uid)
{
/* Map the uid to a global kernel uid */
return KUIDT_INIT(map_id_down(&ns->uid_map, uid));
}
EXPORT_SYMBOL(make_kuid);
/**
* from_kuid - Create a uid from a kuid user-namespace pair.
* @targ: The user namespace we want a uid in.
* @kuid: The kernel internal uid to start with.
*
* Map @kuid into the user-namespace specified by @targ and
* return the resulting uid.
*
* There is always a mapping into the initial user_namespace.
*
* If @kuid has no mapping in @targ (uid_t)-1 is returned.
*/
uid_t from_kuid(struct user_namespace *targ, kuid_t kuid)
{
/* Map the uid from a global kernel uid */
return map_id_up(&targ->uid_map, __kuid_val(kuid));
}
EXPORT_SYMBOL(from_kuid);
/**
* from_kuid_munged - Create a uid from a kuid user-namespace pair.
* @targ: The user namespace we want a uid in.
* @kuid: The kernel internal uid to start with.
*
* Map @kuid into the user-namespace specified by @targ and
* return the resulting uid.
*
* There is always a mapping into the initial user_namespace.
*
* Unlike from_kuid from_kuid_munged never fails and always
* returns a valid uid. This makes from_kuid_munged appropriate
* for use in syscalls like stat and getuid where failing the
* system call and failing to provide a valid uid are not an
* options.
*
* If @kuid has no mapping in @targ overflowuid is returned.
*/
uid_t from_kuid_munged(struct user_namespace *targ, kuid_t kuid)
{
uid_t uid;
uid = from_kuid(targ, kuid);
if (uid == (uid_t) -1)
uid = overflowuid;
return uid;
}
EXPORT_SYMBOL(from_kuid_munged);
/**
* make_kgid - Map a user-namespace gid pair into a kgid.
* @ns: User namespace that the gid is in
* @gid: group identifier
*
* Maps a user-namespace gid pair into a kernel internal kgid,
* and returns that kgid.
*
* When there is no mapping defined for the user-namespace gid
* pair INVALID_GID is returned. Callers are expected to test
* for and handle INVALID_GID being returned. INVALID_GID may be
* tested for using gid_valid().
*/
kgid_t make_kgid(struct user_namespace *ns, gid_t gid)
{
/* Map the gid to a global kernel gid */
return KGIDT_INIT(map_id_down(&ns->gid_map, gid));
}
EXPORT_SYMBOL(make_kgid);
/**
* from_kgid - Create a gid from a kgid user-namespace pair.
* @targ: The user namespace we want a gid in.
* @kgid: The kernel internal gid to start with.
*
* Map @kgid into the user-namespace specified by @targ and
* return the resulting gid.
*
* There is always a mapping into the initial user_namespace.
*
* If @kgid has no mapping in @targ (gid_t)-1 is returned.
*/
gid_t from_kgid(struct user_namespace *targ, kgid_t kgid)
{
/* Map the gid from a global kernel gid */
return map_id_up(&targ->gid_map, __kgid_val(kgid));
}
EXPORT_SYMBOL(from_kgid);
/**
* from_kgid_munged - Create a gid from a kgid user-namespace pair.
* @targ: The user namespace we want a gid in.
* @kgid: The kernel internal gid to start with.
*
* Map @kgid into the user-namespace specified by @targ and
* return the resulting gid.
*
* There is always a mapping into the initial user_namespace.
*
* Unlike from_kgid from_kgid_munged never fails and always
* returns a valid gid. This makes from_kgid_munged appropriate
* for use in syscalls like stat and getgid where failing the
* system call and failing to provide a valid gid are not options.
*
* If @kgid has no mapping in @targ overflowgid is returned.
*/
gid_t from_kgid_munged(struct user_namespace *targ, kgid_t kgid)
{
gid_t gid;
gid = from_kgid(targ, kgid);
if (gid == (gid_t) -1)
gid = overflowgid;
return gid;
}
EXPORT_SYMBOL(from_kgid_munged);
/**
* make_kprojid - Map a user-namespace projid pair into a kprojid.
* @ns: User namespace that the projid is in
* @projid: Project identifier
*
* Maps a user-namespace uid pair into a kernel internal kuid,
* and returns that kuid.
*
* When there is no mapping defined for the user-namespace projid
* pair INVALID_PROJID is returned. Callers are expected to test
* for and handle INVALID_PROJID being returned. INVALID_PROJID
* may be tested for using projid_valid().
*/
kprojid_t make_kprojid(struct user_namespace *ns, projid_t projid)
{
/* Map the uid to a global kernel uid */
return KPROJIDT_INIT(map_id_down(&ns->projid_map, projid));
}
EXPORT_SYMBOL(make_kprojid);
/**
* from_kprojid - Create a projid from a kprojid user-namespace pair.
* @targ: The user namespace we want a projid in.
* @kprojid: The kernel internal project identifier to start with.
*
* Map @kprojid into the user-namespace specified by @targ and
* return the resulting projid.
*
* There is always a mapping into the initial user_namespace.
*
* If @kprojid has no mapping in @targ (projid_t)-1 is returned.
*/
projid_t from_kprojid(struct user_namespace *targ, kprojid_t kprojid)
{
/* Map the uid from a global kernel uid */
return map_id_up(&targ->projid_map, __kprojid_val(kprojid));
}
EXPORT_SYMBOL(from_kprojid);
/**
* from_kprojid_munged - Create a projiid from a kprojid user-namespace pair.
* @targ: The user namespace we want a projid in.
* @kprojid: The kernel internal projid to start with.
*
* Map @kprojid into the user-namespace specified by @targ and
* return the resulting projid.
*
* There is always a mapping into the initial user_namespace.
*
* Unlike from_kprojid from_kprojid_munged never fails and always
* returns a valid projid. This makes from_kprojid_munged
* appropriate for use in syscalls like stat and where
* failing the system call and failing to provide a valid projid are
* not an options.
*
* If @kprojid has no mapping in @targ OVERFLOW_PROJID is returned.
*/
projid_t from_kprojid_munged(struct user_namespace *targ, kprojid_t kprojid)
{
projid_t projid;
projid = from_kprojid(targ, kprojid);
if (projid == (projid_t) -1)
projid = OVERFLOW_PROJID;
return projid;
}
EXPORT_SYMBOL(from_kprojid_munged);
static int uid_m_show(struct seq_file *seq, void *v)
{
struct user_namespace *ns = seq->private;
struct uid_gid_extent *extent = v;
struct user_namespace *lower_ns;
uid_t lower;
lower_ns = seq_user_ns(seq);
if ((lower_ns == ns) && lower_ns->parent)
lower_ns = lower_ns->parent;
lower = from_kuid(lower_ns, KUIDT_INIT(extent->lower_first));
seq_printf(seq, "%10u %10u %10u\n",
extent->first,
lower,
extent->count);
return 0;
}
static int gid_m_show(struct seq_file *seq, void *v)
{
struct user_namespace *ns = seq->private;
struct uid_gid_extent *extent = v;
struct user_namespace *lower_ns;
gid_t lower;
lower_ns = seq_user_ns(seq);
if ((lower_ns == ns) && lower_ns->parent)
lower_ns = lower_ns->parent;
lower = from_kgid(lower_ns, KGIDT_INIT(extent->lower_first));
seq_printf(seq, "%10u %10u %10u\n",
extent->first,
lower,
extent->count);
return 0;
}
static int projid_m_show(struct seq_file *seq, void *v)
{
struct user_namespace *ns = seq->private;
struct uid_gid_extent *extent = v;
struct user_namespace *lower_ns;
projid_t lower;
lower_ns = seq_user_ns(seq);
if ((lower_ns == ns) && lower_ns->parent)
lower_ns = lower_ns->parent;
lower = from_kprojid(lower_ns, KPROJIDT_INIT(extent->lower_first));
seq_printf(seq, "%10u %10u %10u\n",
extent->first,
lower,
extent->count);
return 0;
}
static void *m_start(struct seq_file *seq, loff_t *ppos,
struct uid_gid_map *map)
{
loff_t pos = *ppos;
unsigned extents = map->nr_extents;
smp_rmb();
if (pos >= extents)
return NULL;
if (extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
return &map->extent[pos];
return &map->forward[pos];
}
static void *uid_m_start(struct seq_file *seq, loff_t *ppos)
{
struct user_namespace *ns = seq->private;
return m_start(seq, ppos, &ns->uid_map);
}
static void *gid_m_start(struct seq_file *seq, loff_t *ppos)
{
struct user_namespace *ns = seq->private;
return m_start(seq, ppos, &ns->gid_map);
}
static void *projid_m_start(struct seq_file *seq, loff_t *ppos)
{
struct user_namespace *ns = seq->private;
return m_start(seq, ppos, &ns->projid_map);
}
static void *m_next(struct seq_file *seq, void *v, loff_t *pos)
{
(*pos)++;
return seq->op->start(seq, pos);
}
static void m_stop(struct seq_file *seq, void *v)
{
return;
}
const struct seq_operations proc_uid_seq_operations = {
.start = uid_m_start,
.stop = m_stop,
.next = m_next,
.show = uid_m_show,
};
const struct seq_operations proc_gid_seq_operations = {
.start = gid_m_start,
.stop = m_stop,
.next = m_next,
.show = gid_m_show,
};
const struct seq_operations proc_projid_seq_operations = {
.start = projid_m_start,
.stop = m_stop,
.next = m_next,
.show = projid_m_show,
};
static bool mappings_overlap(struct uid_gid_map *new_map,
struct uid_gid_extent *extent)
{
u32 upper_first, lower_first, upper_last, lower_last;
unsigned idx;
upper_first = extent->first;
lower_first = extent->lower_first;
upper_last = upper_first + extent->count - 1;
lower_last = lower_first + extent->count - 1;
for (idx = 0; idx < new_map->nr_extents; idx++) {
u32 prev_upper_first, prev_lower_first;
u32 prev_upper_last, prev_lower_last;
struct uid_gid_extent *prev;
if (new_map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
prev = &new_map->extent[idx];
else
prev = &new_map->forward[idx];
prev_upper_first = prev->first;
prev_lower_first = prev->lower_first;
prev_upper_last = prev_upper_first + prev->count - 1;
prev_lower_last = prev_lower_first + prev->count - 1;
/* Does the upper range intersect a previous extent? */
if ((prev_upper_first <= upper_last) &&
(prev_upper_last >= upper_first))
return true;
/* Does the lower range intersect a previous extent? */
if ((prev_lower_first <= lower_last) &&
(prev_lower_last >= lower_first))
return true;
}
return false;
}
/*
* insert_extent - Safely insert a new idmap extent into struct uid_gid_map.
* Takes care to allocate a 4K block of memory if the number of mappings exceeds
* UID_GID_MAP_MAX_BASE_EXTENTS.
*/
static int insert_extent(struct uid_gid_map *map, struct uid_gid_extent *extent)
{
struct uid_gid_extent *dest;
if (map->nr_extents == UID_GID_MAP_MAX_BASE_EXTENTS) {
struct uid_gid_extent *forward;
/* Allocate memory for 340 mappings. */
forward = kmalloc_array(UID_GID_MAP_MAX_EXTENTS,
sizeof(struct uid_gid_extent),
GFP_KERNEL);
if (!forward)
return -ENOMEM;
/* Copy over memory. Only set up memory for the forward pointer.
* Defer the memory setup for the reverse pointer.
*/
memcpy(forward, map->extent,
map->nr_extents * sizeof(map->extent[0]));
map->forward = forward;
map->reverse = NULL;
}
if (map->nr_extents < UID_GID_MAP_MAX_BASE_EXTENTS)
dest = &map->extent[map->nr_extents];
else
dest = &map->forward[map->nr_extents];
*dest = *extent;
map->nr_extents++;
return 0;
}
/* cmp function to sort() forward mappings */
static int cmp_extents_forward(const void *a, const void *b)
{
const struct uid_gid_extent *e1 = a;
const struct uid_gid_extent *e2 = b;
if (e1->first < e2->first)
return -1;
if (e1->first > e2->first)
return 1;
return 0;
}
/* cmp function to sort() reverse mappings */
static int cmp_extents_reverse(const void *a, const void *b)
{
const struct uid_gid_extent *e1 = a;
const struct uid_gid_extent *e2 = b;
if (e1->lower_first < e2->lower_first)
return -1;
if (e1->lower_first > e2->lower_first)
return 1;
return 0;
}
/*
* sort_idmaps - Sorts an array of idmap entries.
* Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
*/
static int sort_idmaps(struct uid_gid_map *map)
{
if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
return 0;
/* Sort forward array. */
sort(map->forward, map->nr_extents, sizeof(struct uid_gid_extent),
cmp_extents_forward, NULL);
/* Only copy the memory from forward we actually need. */
map->reverse = kmemdup(map->forward,
map->nr_extents * sizeof(struct uid_gid_extent),
GFP_KERNEL);
if (!map->reverse)
return -ENOMEM;
/* Sort reverse array. */
sort(map->reverse, map->nr_extents, sizeof(struct uid_gid_extent),
cmp_extents_reverse, NULL);
return 0;
}
/**
* verify_root_map() - check the uid 0 mapping
* @file: idmapping file
* @map_ns: user namespace of the target process
* @new_map: requested idmap
*
* If a process requests mapping parent uid 0 into the new ns, verify that the
* process writing the map had the CAP_SETFCAP capability as the target process
* will be able to write fscaps that are valid in ancestor user namespaces.
*
* Return: true if the mapping is allowed, false if not.
*/
static bool verify_root_map(const struct file *file,
struct user_namespace *map_ns,
struct uid_gid_map *new_map)
{
int idx;
const struct user_namespace *file_ns = file->f_cred->user_ns;
struct uid_gid_extent *extent0 = NULL;
for (idx = 0; idx < new_map->nr_extents; idx++) {
if (new_map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
extent0 = &new_map->extent[idx];
else
extent0 = &new_map->forward[idx];
if (extent0->lower_first == 0)
break;
extent0 = NULL;
}
if (!extent0)
return true;
if (map_ns == file_ns) {
/* The process unshared its ns and is writing to its own
* /proc/self/uid_map. User already has full capabilites in
* the new namespace. Verify that the parent had CAP_SETFCAP
* when it unshared.
* */
if (!file_ns->parent_could_setfcap)
return false;
} else {
/* Process p1 is writing to uid_map of p2, who is in a child
* user namespace to p1's. Verify that the opener of the map
* file has CAP_SETFCAP against the parent of the new map
* namespace */
if (!file_ns_capable(file, map_ns->parent, CAP_SETFCAP))
return false;
}
return true;
}
static ssize_t map_write(struct file *file, const char __user *buf,
size_t count, loff_t *ppos,
int cap_setid,
struct uid_gid_map *map,
struct uid_gid_map *parent_map)
{
struct seq_file *seq = file->private_data;
struct user_namespace *map_ns = seq->private;
struct uid_gid_map new_map;
unsigned idx;
struct uid_gid_extent extent;
char *kbuf, *pos, *next_line;
ssize_t ret;
/* Only allow < page size writes at the beginning of the file */
if ((*ppos != 0) || (count >= PAGE_SIZE))
return -EINVAL;
/* Slurp in the user data */
kbuf = memdup_user_nul(buf, count);
if (IS_ERR(kbuf))
return PTR_ERR(kbuf);
/*
* The userns_state_mutex serializes all writes to any given map.
*
* Any map is only ever written once.
*
* An id map fits within 1 cache line on most architectures.
*
* On read nothing needs to be done unless you are on an
* architecture with a crazy cache coherency model like alpha.
*
* There is a one time data dependency between reading the
* count of the extents and the values of the extents. The
* desired behavior is to see the values of the extents that
* were written before the count of the extents.
*
* To achieve this smp_wmb() is used on guarantee the write
* order and smp_rmb() is guaranteed that we don't have crazy
* architectures returning stale data.
*/
mutex_lock(&userns_state_mutex);
memset(&new_map, 0, sizeof(struct uid_gid_map));
ret = -EPERM;
/* Only allow one successful write to the map */
if (map->nr_extents != 0)
goto out;
/*
* Adjusting namespace settings requires capabilities on the target.
*/
if (cap_valid(cap_setid) && !file_ns_capable(file, map_ns, CAP_SYS_ADMIN))
goto out;
/* Parse the user data */
ret = -EINVAL;
pos = kbuf;
for (; pos; pos = next_line) {
/* Find the end of line and ensure I don't look past it */
next_line = strchr(pos, '\n');
if (next_line) {
*next_line = '\0';
next_line++;
if (*next_line == '\0')
next_line = NULL;
}
pos = skip_spaces(pos);
extent.first = simple_strtoul(pos, &pos, 10);
if (!isspace(*pos))
goto out;
pos = skip_spaces(pos);
extent.lower_first = simple_strtoul(pos, &pos, 10);
if (!isspace(*pos))
goto out;
pos = skip_spaces(pos);
extent.count = simple_strtoul(pos, &pos, 10);
if (*pos && !isspace(*pos))
goto out;
/* Verify there is not trailing junk on the line */
pos = skip_spaces(pos);
if (*pos != '\0')
goto out;
/* Verify we have been given valid starting values */
if ((extent.first == (u32) -1) ||
(extent.lower_first == (u32) -1))
goto out;
/* Verify count is not zero and does not cause the
* extent to wrap
*/
if ((extent.first + extent.count) <= extent.first)
goto out;
if ((extent.lower_first + extent.count) <=
extent.lower_first)
goto out;
/* Do the ranges in extent overlap any previous extents? */
if (mappings_overlap(&new_map, &extent))
goto out;
if ((new_map.nr_extents + 1) == UID_GID_MAP_MAX_EXTENTS &&
(next_line != NULL))
goto out;
ret = insert_extent(&new_map, &extent);
if (ret < 0)
goto out;
ret = -EINVAL;
}
/* Be very certain the new map actually exists */
if (new_map.nr_extents == 0)
goto out;
ret = -EPERM;
/* Validate the user is allowed to use user id's mapped to. */
if (!new_idmap_permitted(file, map_ns, cap_setid, &new_map))
goto out;
ret = -EPERM;
/* Map the lower ids from the parent user namespace to the
* kernel global id space.
*/
for (idx = 0; idx < new_map.nr_extents; idx++) {
struct uid_gid_extent *e;
u32 lower_first;
if (new_map.nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
e = &new_map.extent[idx];
else
e = &new_map.forward[idx];
lower_first = map_id_range_down(parent_map,
e->lower_first,
e->count);
/* Fail if we can not map the specified extent to
* the kernel global id space.
*/
if (lower_first == (u32) -1)
goto out;
e->lower_first = lower_first;
}
/*
* If we want to use binary search for lookup, this clones the extent
* array and sorts both copies.
*/
ret = sort_idmaps(&new_map);
if (ret < 0)
goto out;
/* Install the map */
if (new_map.nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS) {
memcpy(map->extent, new_map.extent,
new_map.nr_extents * sizeof(new_map.extent[0]));
} else {
map->forward = new_map.forward;
map->reverse = new_map.reverse;
}
smp_wmb();
map->nr_extents = new_map.nr_extents;
*ppos = count;
ret = count;
out:
if (ret < 0 && new_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) {
kfree(new_map.forward);
kfree(new_map.reverse);
map->forward = NULL;
map->reverse = NULL;
map->nr_extents = 0;
}
mutex_unlock(&userns_state_mutex);
kfree(kbuf);
return ret;
}
ssize_t proc_uid_map_write(struct file *file, const char __user *buf,
size_t size, loff_t *ppos)
{
struct seq_file *seq = file->private_data;
struct user_namespace *ns = seq->private;
struct user_namespace *seq_ns = seq_user_ns(seq);
if (!ns->parent)
return -EPERM;
if ((seq_ns != ns) && (seq_ns != ns->parent))
return -EPERM;
return map_write(file, buf, size, ppos, CAP_SETUID,
&ns->uid_map, &ns->parent->uid_map);
}
ssize_t proc_gid_map_write(struct file *file, const char __user *buf,
size_t size, loff_t *ppos)
{
struct seq_file *seq = file->private_data;
struct user_namespace *ns = seq->private;
struct user_namespace *seq_ns = seq_user_ns(seq);
if (!ns->parent)
return -EPERM;
if ((seq_ns != ns) && (seq_ns != ns->parent))
return -EPERM;
return map_write(file, buf, size, ppos, CAP_SETGID,
&ns->gid_map, &ns->parent->gid_map);
}
ssize_t proc_projid_map_write(struct file *file, const char __user *buf,
size_t size, loff_t *ppos)
{
struct seq_file *seq = file->private_data;
struct user_namespace *ns = seq->private;
struct user_namespace *seq_ns = seq_user_ns(seq);
if (!ns->parent)
return -EPERM;
if ((seq_ns != ns) && (seq_ns != ns->parent))
return -EPERM;
/* Anyone can set any valid project id no capability needed */
return map_write(file, buf, size, ppos, -1,
&ns->projid_map, &ns->parent->projid_map);
}
static bool new_idmap_permitted(const struct file *file,
struct user_namespace *ns, int cap_setid,
struct uid_gid_map *new_map)
{
const struct cred *cred = file->f_cred;
if (cap_setid == CAP_SETUID && !verify_root_map(file, ns, new_map))
return false;
/* Don't allow mappings that would allow anything that wouldn't
* be allowed without the establishment of unprivileged mappings.
*/
if ((new_map->nr_extents == 1) && (new_map->extent[0].count == 1) &&
uid_eq(ns->owner, cred->euid)) {
u32 id = new_map->extent[0].lower_first;
if (cap_setid == CAP_SETUID) {
kuid_t uid = make_kuid(ns->parent, id);
if (uid_eq(uid, cred->euid))
return true;
} else if (cap_setid == CAP_SETGID) {
kgid_t gid = make_kgid(ns->parent, id);
if (!(ns->flags & USERNS_SETGROUPS_ALLOWED) &&
gid_eq(gid, cred->egid))
return true;
}
}
/* Allow anyone to set a mapping that doesn't require privilege */
if (!cap_valid(cap_setid))
return true;
/* Allow the specified ids if we have the appropriate capability
* (CAP_SETUID or CAP_SETGID) over the parent user namespace.
* And the opener of the id file also has the appropriate capability.
*/
if (ns_capable(ns->parent, cap_setid) &&
file_ns_capable(file, ns->parent, cap_setid))
return true;
return false;
}
int proc_setgroups_show(struct seq_file *seq, void *v)
{
struct user_namespace *ns = seq->private;
unsigned long userns_flags = READ_ONCE(ns->flags);
seq_printf(seq, "%s\n",
(userns_flags & USERNS_SETGROUPS_ALLOWED) ?
"allow" : "deny");
return 0;
}
ssize_t proc_setgroups_write(struct file *file, const char __user *buf,
size_t count, loff_t *ppos)
{
struct seq_file *seq = file->private_data;
struct user_namespace *ns = seq->private;
char kbuf[8], *pos;
bool setgroups_allowed;
ssize_t ret;
/* Only allow a very narrow range of strings to be written */
ret = -EINVAL;
if ((*ppos != 0) || (count >= sizeof(kbuf)))
goto out;
/* What was written? */
ret = -EFAULT;
if (copy_from_user(kbuf, buf, count))
goto out;
kbuf[count] = '\0';
pos = kbuf;
/* What is being requested? */
ret = -EINVAL;
if (strncmp(pos, "allow", 5) == 0) {
pos += 5;
setgroups_allowed = true;
}
else if (strncmp(pos, "deny", 4) == 0) {
pos += 4;
setgroups_allowed = false;
}
else
goto out;
/* Verify there is not trailing junk on the line */
pos = skip_spaces(pos);
if (*pos != '\0')
goto out;
ret = -EPERM;
mutex_lock(&userns_state_mutex);
if (setgroups_allowed) {
/* Enabling setgroups after setgroups has been disabled
* is not allowed.
*/
if (!(ns->flags & USERNS_SETGROUPS_ALLOWED))
goto out_unlock;
} else {
/* Permanently disabling setgroups after setgroups has
* been enabled by writing the gid_map is not allowed.
*/
if (ns->gid_map.nr_extents != 0)
goto out_unlock;
ns->flags &= ~USERNS_SETGROUPS_ALLOWED;
}
mutex_unlock(&userns_state_mutex);
/* Report a successful write */
*ppos = count;
ret = count;
out:
return ret;
out_unlock:
mutex_unlock(&userns_state_mutex);
goto out;
}
bool userns_may_setgroups(const struct user_namespace *ns)
{
bool allowed;
mutex_lock(&userns_state_mutex);
/* It is not safe to use setgroups until a gid mapping in
* the user namespace has been established.
*/
allowed = ns->gid_map.nr_extents != 0;
/* Is setgroups allowed? */
allowed = allowed && (ns->flags & USERNS_SETGROUPS_ALLOWED);
mutex_unlock(&userns_state_mutex);
return allowed;
}
/*
* Returns true if @child is the same namespace or a descendant of
* @ancestor.
*/
bool in_userns(const struct user_namespace *ancestor,
const struct user_namespace *child)
{
const struct user_namespace *ns;
for (ns = child; ns->level > ancestor->level; ns = ns->parent)
;
return (ns == ancestor);
}
bool current_in_userns(const struct user_namespace *target_ns)
{
return in_userns(target_ns, current_user_ns());
}
EXPORT_SYMBOL(current_in_userns);
static inline struct user_namespace *to_user_ns(struct ns_common *ns)
{
return container_of(ns, struct user_namespace, ns);
}
static struct ns_common *userns_get(struct task_struct *task)
{
struct user_namespace *user_ns;
rcu_read_lock();
user_ns = get_user_ns(__task_cred(task)->user_ns);
rcu_read_unlock();
return user_ns ? &user_ns->ns : NULL;
}
static void userns_put(struct ns_common *ns)
{
put_user_ns(to_user_ns(ns));
}
static int userns_install(struct nsset *nsset, struct ns_common *ns)
{
struct user_namespace *user_ns = to_user_ns(ns);
struct cred *cred;
/* Don't allow gaining capabilities by reentering
* the same user namespace.
*/
if (user_ns == current_user_ns())
return -EINVAL;
/* Tasks that share a thread group must share a user namespace */
if (!thread_group_empty(current))
return -EINVAL;
if (current->fs->users != 1)
return -EINVAL;
if (!ns_capable(user_ns, CAP_SYS_ADMIN))
return -EPERM;
cred = nsset_cred(nsset);
if (!cred)
return -EINVAL;
put_user_ns(cred->user_ns);
set_cred_user_ns(cred, get_user_ns(user_ns));
if (set_cred_ucounts(cred) < 0)
return -EINVAL;
return 0;
}
struct ns_common *ns_get_owner(struct ns_common *ns)
{
struct user_namespace *my_user_ns = current_user_ns();
struct user_namespace *owner, *p;
/* See if the owner is in the current user namespace */
owner = p = ns->ops->owner(ns);
for (;;) {
if (!p)
return ERR_PTR(-EPERM);
if (p == my_user_ns)
break;
p = p->parent;
}
return &get_user_ns(owner)->ns;
}
static struct user_namespace *userns_owner(struct ns_common *ns)
{
return to_user_ns(ns)->parent;
}
const struct proc_ns_operations userns_operations = {
.name = "user",
.type = CLONE_NEWUSER,
.get = userns_get,
.put = userns_put,
.install = userns_install,
.owner = userns_owner,
.get_parent = ns_get_owner,
};
static __init int user_namespaces_init(void)
{
user_ns_cachep = KMEM_CACHE(user_namespace, SLAB_PANIC | SLAB_ACCOUNT);
return 0;
}
subsys_initcall(user_namespaces_init);
// SPDX-License-Identifier: GPL-2.0
/*
* linux/fs/super.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*
* super.c contains code to handle: - mount structures
* - super-block tables
* - filesystem drivers list
* - mount system call
* - umount system call
* - ustat system call
*
* GK 2/5/95 - Changed to support mounting the root fs via NFS
*
* Added kerneld support: Jacques Gelinas and Bjorn Ekwall
* Added change_root: Werner Almesberger & Hans Lermen, Feb '96
* Added options to /proc/mounts:
* Torbjörn Lindh (torbjorn.lindh@gopta.se), April 14, 1996.
* Added devfs support: Richard Gooch <rgooch@atnf.csiro.au>, 13-JAN-1998
* Heavily rewritten for 'one fs - one tree' dcache architecture. AV, Mar 2000
*/
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/mount.h>
#include <linux/security.h>
#include <linux/writeback.h> /* for the emergency remount stuff */
#include <linux/idr.h>
#include <linux/mutex.h>
#include <linux/backing-dev.h>
#include <linux/rculist_bl.h>
#include <linux/fscrypt.h>
#include <linux/fsnotify.h>
#include <linux/lockdep.h>
#include <linux/user_namespace.h>
#include <linux/fs_context.h>
#include <uapi/linux/mount.h>
#include "internal.h"
static int thaw_super_locked(struct super_block *sb, enum freeze_holder who);
static LIST_HEAD(super_blocks);
static DEFINE_SPINLOCK(sb_lock);
static char *sb_writers_name[SB_FREEZE_LEVELS] = {
"sb_writers",
"sb_pagefaults",
"sb_internal",
};
static inline void __super_lock(struct super_block *sb, bool excl)
{
if (excl)
down_write(&sb->s_umount);
else
down_read(&sb->s_umount);
}
static inline void super_unlock(struct super_block *sb, bool excl)
{
if (excl)
up_write(&sb->s_umount);
else
up_read(&sb->s_umount);
}
static inline void __super_lock_excl(struct super_block *sb)
{
__super_lock(sb, true);
}
static inline void super_unlock_excl(struct super_block *sb)
{
super_unlock(sb, true);
}
static inline void super_unlock_shared(struct super_block *sb)
{
super_unlock(sb, false);
}
static bool super_flags(const struct super_block *sb, unsigned int flags)
{
/*
* Pairs with smp_store_release() in super_wake() and ensures
* that we see @flags after we're woken.
*/
return smp_load_acquire(&sb->s_flags) & flags;
}
/**
* super_lock - wait for superblock to become ready and lock it
* @sb: superblock to wait for
* @excl: whether exclusive access is required
*
* If the superblock has neither passed through vfs_get_tree() or
* generic_shutdown_super() yet wait for it to happen. Either superblock
* creation will succeed and SB_BORN is set by vfs_get_tree() or we're
* woken and we'll see SB_DYING.
*
* The caller must have acquired a temporary reference on @sb->s_count.
*
* Return: The function returns true if SB_BORN was set and with
* s_umount held. The function returns false if SB_DYING was
* set and without s_umount held.
*/
static __must_check bool super_lock(struct super_block *sb, bool excl)
{
lockdep_assert_not_held(&sb->s_umount);
/* wait until the superblock is ready or dying */
wait_var_event(&sb->s_flags, super_flags(sb, SB_BORN | SB_DYING));
/* Don't pointlessly acquire s_umount. */
if (super_flags(sb, SB_DYING))
return false;
__super_lock(sb, excl);
/*
* Has gone through generic_shutdown_super() in the meantime.
* @sb->s_root is NULL and @sb->s_active is 0. No one needs to
* grab a reference to this. Tell them so.
*/
if (sb->s_flags & SB_DYING) {
super_unlock(sb, excl);
return false;
}
WARN_ON_ONCE(!(sb->s_flags & SB_BORN)); return true;
}
/* wait and try to acquire read-side of @sb->s_umount */
static inline bool super_lock_shared(struct super_block *sb)
{
return super_lock(sb, false);
}
/* wait and try to acquire write-side of @sb->s_umount */
static inline bool super_lock_excl(struct super_block *sb)
{
return super_lock(sb, true);
}
/* wake waiters */
#define SUPER_WAKE_FLAGS (SB_BORN | SB_DYING | SB_DEAD)
static void super_wake(struct super_block *sb, unsigned int flag)
{
WARN_ON_ONCE((flag & ~SUPER_WAKE_FLAGS));
WARN_ON_ONCE(hweight32(flag & SUPER_WAKE_FLAGS) > 1);
/*
* Pairs with smp_load_acquire() in super_lock() to make sure
* all initializations in the superblock are seen by the user
* seeing SB_BORN sent.
*/
smp_store_release(&sb->s_flags, sb->s_flags | flag);
/*
* Pairs with the barrier in prepare_to_wait_event() to make sure
* ___wait_var_event() either sees SB_BORN set or
* waitqueue_active() check in wake_up_var() sees the waiter.
*/
smp_mb();
wake_up_var(&sb->s_flags);
}
/*
* One thing we have to be careful of with a per-sb shrinker is that we don't
* drop the last active reference to the superblock from within the shrinker.
* If that happens we could trigger unregistering the shrinker from within the
* shrinker path and that leads to deadlock on the shrinker_mutex. Hence we
* take a passive reference to the superblock to avoid this from occurring.
*/
static unsigned long super_cache_scan(struct shrinker *shrink,
struct shrink_control *sc)
{
struct super_block *sb;
long fs_objects = 0;
long total_objects;
long freed = 0;
long dentries;
long inodes;
sb = shrink->private_data;
/*
* Deadlock avoidance. We may hold various FS locks, and we don't want
* to recurse into the FS that called us in clear_inode() and friends..
*/
if (!(sc->gfp_mask & __GFP_FS))
return SHRINK_STOP;
if (!super_trylock_shared(sb))
return SHRINK_STOP;
if (sb->s_op->nr_cached_objects)
fs_objects = sb->s_op->nr_cached_objects(sb, sc);
inodes = list_lru_shrink_count(&sb->s_inode_lru, sc);
dentries = list_lru_shrink_count(&sb->s_dentry_lru, sc);
total_objects = dentries + inodes + fs_objects + 1;
if (!total_objects)
total_objects = 1;
/* proportion the scan between the caches */
dentries = mult_frac(sc->nr_to_scan, dentries, total_objects);
inodes = mult_frac(sc->nr_to_scan, inodes, total_objects);
fs_objects = mult_frac(sc->nr_to_scan, fs_objects, total_objects);
/*
* prune the dcache first as the icache is pinned by it, then
* prune the icache, followed by the filesystem specific caches
*
* Ensure that we always scan at least one object - memcg kmem
* accounting uses this to fully empty the caches.
*/
sc->nr_to_scan = dentries + 1;
freed = prune_dcache_sb(sb, sc);
sc->nr_to_scan = inodes + 1;
freed += prune_icache_sb(sb, sc);
if (fs_objects) {
sc->nr_to_scan = fs_objects + 1;
freed += sb->s_op->free_cached_objects(sb, sc);
}
super_unlock_shared(sb);
return freed;
}
static unsigned long super_cache_count(struct shrinker *shrink,
struct shrink_control *sc)
{
struct super_block *sb;
long total_objects = 0;
sb = shrink->private_data;
/*
* We don't call super_trylock_shared() here as it is a scalability
* bottleneck, so we're exposed to partial setup state. The shrinker
* rwsem does not protect filesystem operations backing
* list_lru_shrink_count() or s_op->nr_cached_objects(). Counts can
* change between super_cache_count and super_cache_scan, so we really
* don't need locks here.
*
* However, if we are currently mounting the superblock, the underlying
* filesystem might be in a state of partial construction and hence it
* is dangerous to access it. super_trylock_shared() uses a SB_BORN check
* to avoid this situation, so do the same here. The memory barrier is
* matched with the one in mount_fs() as we don't hold locks here.
*/
if (!(sb->s_flags & SB_BORN))
return 0;
smp_rmb();
if (sb->s_op && sb->s_op->nr_cached_objects)
total_objects = sb->s_op->nr_cached_objects(sb, sc);
total_objects += list_lru_shrink_count(&sb->s_dentry_lru, sc);
total_objects += list_lru_shrink_count(&sb->s_inode_lru, sc);
if (!total_objects)
return SHRINK_EMPTY;
total_objects = vfs_pressure_ratio(total_objects);
return total_objects;
}
static void destroy_super_work(struct work_struct *work)
{
struct super_block *s = container_of(work, struct super_block,
destroy_work);
security_sb_free(s);
put_user_ns(s->s_user_ns);
kfree(s->s_subtype);
for (int i = 0; i < SB_FREEZE_LEVELS; i++)
percpu_free_rwsem(&s->s_writers.rw_sem[i]);
kfree(s);
}
static void destroy_super_rcu(struct rcu_head *head)
{
struct super_block *s = container_of(head, struct super_block, rcu);
INIT_WORK(&s->destroy_work, destroy_super_work);
schedule_work(&s->destroy_work);
}
/* Free a superblock that has never been seen by anyone */
static void destroy_unused_super(struct super_block *s)
{
if (!s)
return;
super_unlock_excl(s);
list_lru_destroy(&s->s_dentry_lru);
list_lru_destroy(&s->s_inode_lru);
shrinker_free(s->s_shrink);
/* no delays needed */
destroy_super_work(&s->destroy_work);
}
/**
* alloc_super - create new superblock
* @type: filesystem type superblock should belong to
* @flags: the mount flags
* @user_ns: User namespace for the super_block
*
* Allocates and initializes a new &struct super_block. alloc_super()
* returns a pointer new superblock or %NULL if allocation had failed.
*/
static struct super_block *alloc_super(struct file_system_type *type, int flags,
struct user_namespace *user_ns)
{
struct super_block *s = kzalloc(sizeof(struct super_block), GFP_KERNEL);
static const struct super_operations default_op;
int i;
if (!s)
return NULL;
INIT_LIST_HEAD(&s->s_mounts);
s->s_user_ns = get_user_ns(user_ns);
init_rwsem(&s->s_umount);
lockdep_set_class(&s->s_umount, &type->s_umount_key);
/*
* sget() can have s_umount recursion.
*
* When it cannot find a suitable sb, it allocates a new
* one (this one), and tries again to find a suitable old
* one.
*
* In case that succeeds, it will acquire the s_umount
* lock of the old one. Since these are clearly distrinct
* locks, and this object isn't exposed yet, there's no
* risk of deadlocks.
*
* Annotate this by putting this lock in a different
* subclass.
*/
down_write_nested(&s->s_umount, SINGLE_DEPTH_NESTING);
if (security_sb_alloc(s))
goto fail;
for (i = 0; i < SB_FREEZE_LEVELS; i++) {
if (__percpu_init_rwsem(&s->s_writers.rw_sem[i],
sb_writers_name[i],
&type->s_writers_key[i]))
goto fail;
}
s->s_bdi = &noop_backing_dev_info;
s->s_flags = flags;
if (s->s_user_ns != &init_user_ns)
s->s_iflags |= SB_I_NODEV;
INIT_HLIST_NODE(&s->s_instances);
INIT_HLIST_BL_HEAD(&s->s_roots);
mutex_init(&s->s_sync_lock);
INIT_LIST_HEAD(&s->s_inodes);
spin_lock_init(&s->s_inode_list_lock);
INIT_LIST_HEAD(&s->s_inodes_wb);
spin_lock_init(&s->s_inode_wblist_lock);
s->s_count = 1;
atomic_set(&s->s_active, 1);
mutex_init(&s->s_vfs_rename_mutex);
lockdep_set_class(&s->s_vfs_rename_mutex, &type->s_vfs_rename_key);
init_rwsem(&s->s_dquot.dqio_sem);
s->s_maxbytes = MAX_NON_LFS;
s->s_op = &default_op;
s->s_time_gran = 1000000000;
s->s_time_min = TIME64_MIN;
s->s_time_max = TIME64_MAX;
s->s_shrink = shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE,
"sb-%s", type->name);
if (!s->s_shrink)
goto fail;
s->s_shrink->scan_objects = super_cache_scan;
s->s_shrink->count_objects = super_cache_count;
s->s_shrink->batch = 1024;
s->s_shrink->private_data = s;
if (list_lru_init_memcg(&s->s_dentry_lru, s->s_shrink))
goto fail;
if (list_lru_init_memcg(&s->s_inode_lru, s->s_shrink))
goto fail;
return s;
fail:
destroy_unused_super(s);
return NULL;
}
/* Superblock refcounting */
/*
* Drop a superblock's refcount. The caller must hold sb_lock.
*/
static void __put_super(struct super_block *s)
{
if (!--s->s_count) {
list_del_init(&s->s_list);
WARN_ON(s->s_dentry_lru.node);
WARN_ON(s->s_inode_lru.node);
WARN_ON(!list_empty(&s->s_mounts));
call_rcu(&s->rcu, destroy_super_rcu);
}
}
/**
* put_super - drop a temporary reference to superblock
* @sb: superblock in question
*
* Drops a temporary reference, frees superblock if there's no
* references left.
*/
void put_super(struct super_block *sb)
{
spin_lock(&sb_lock);
__put_super(sb);
spin_unlock(&sb_lock);
}
static void kill_super_notify(struct super_block *sb)
{
lockdep_assert_not_held(&sb->s_umount);
/* already notified earlier */
if (sb->s_flags & SB_DEAD)
return;
/*
* Remove it from @fs_supers so it isn't found by new
* sget{_fc}() walkers anymore. Any concurrent mounter still
* managing to grab a temporary reference is guaranteed to
* already see SB_DYING and will wait until we notify them about
* SB_DEAD.
*/
spin_lock(&sb_lock);
hlist_del_init(&sb->s_instances);
spin_unlock(&sb_lock);
/*
* Let concurrent mounts know that this thing is really dead.
* We don't need @sb->s_umount here as every concurrent caller
* will see SB_DYING and either discard the superblock or wait
* for SB_DEAD.
*/
super_wake(sb, SB_DEAD);
}
/**
* deactivate_locked_super - drop an active reference to superblock
* @s: superblock to deactivate
*
* Drops an active reference to superblock, converting it into a temporary
* one if there is no other active references left. In that case we
* tell fs driver to shut it down and drop the temporary reference we
* had just acquired.
*
* Caller holds exclusive lock on superblock; that lock is released.
*/
void deactivate_locked_super(struct super_block *s)
{
struct file_system_type *fs = s->s_type;
if (atomic_dec_and_test(&s->s_active)) {
shrinker_free(s->s_shrink);
fs->kill_sb(s);
kill_super_notify(s);
/*
* Since list_lru_destroy() may sleep, we cannot call it from
* put_super(), where we hold the sb_lock. Therefore we destroy
* the lru lists right now.
*/
list_lru_destroy(&s->s_dentry_lru);
list_lru_destroy(&s->s_inode_lru);
put_filesystem(fs);
put_super(s);
} else {
super_unlock_excl(s);
}
}
EXPORT_SYMBOL(deactivate_locked_super);
/**
* deactivate_super - drop an active reference to superblock
* @s: superblock to deactivate
*
* Variant of deactivate_locked_super(), except that superblock is *not*
* locked by caller. If we are going to drop the final active reference,
* lock will be acquired prior to that.
*/
void deactivate_super(struct super_block *s)
{
if (!atomic_add_unless(&s->s_active, -1, 1)) { __super_lock_excl(s);
deactivate_locked_super(s);
}
}
EXPORT_SYMBOL(deactivate_super);
/**
* grab_super - acquire an active reference to a superblock
* @sb: superblock to acquire
*
* Acquire a temporary reference on a superblock and try to trade it for
* an active reference. This is used in sget{_fc}() to wait for a
* superblock to either become SB_BORN or for it to pass through
* sb->kill() and be marked as SB_DEAD.
*
* Return: This returns true if an active reference could be acquired,
* false if not.
*/
static bool grab_super(struct super_block *sb)
{
bool locked;
sb->s_count++;
spin_unlock(&sb_lock);
locked = super_lock_excl(sb);
if (locked) {
if (atomic_inc_not_zero(&sb->s_active)) {
put_super(sb);
return true;
}
super_unlock_excl(sb);
}
wait_var_event(&sb->s_flags, super_flags(sb, SB_DEAD));
put_super(sb);
return false;
}
/*
* super_trylock_shared - try to grab ->s_umount shared
* @sb: reference we are trying to grab
*
* Try to prevent fs shutdown. This is used in places where we
* cannot take an active reference but we need to ensure that the
* filesystem is not shut down while we are working on it. It returns
* false if we cannot acquire s_umount or if we lose the race and
* filesystem already got into shutdown, and returns true with the s_umount
* lock held in read mode in case of success. On successful return,
* the caller must drop the s_umount lock when done.
*
* Note that unlike get_super() et.al. this one does *not* bump ->s_count.
* The reason why it's safe is that we are OK with doing trylock instead
* of down_read(). There's a couple of places that are OK with that, but
* it's very much not a general-purpose interface.
*/
bool super_trylock_shared(struct super_block *sb)
{
if (down_read_trylock(&sb->s_umount)) {
if (!(sb->s_flags & SB_DYING) && sb->s_root &&
(sb->s_flags & SB_BORN))
return true;
super_unlock_shared(sb);
}
return false;
}
/**
* retire_super - prevents superblock from being reused
* @sb: superblock to retire
*
* The function marks superblock to be ignored in superblock test, which
* prevents it from being reused for any new mounts. If the superblock has
* a private bdi, it also unregisters it, but doesn't reduce the refcount
* of the superblock to prevent potential races. The refcount is reduced
* by generic_shutdown_super(). The function can not be called
* concurrently with generic_shutdown_super(). It is safe to call the
* function multiple times, subsequent calls have no effect.
*
* The marker will affect the re-use only for block-device-based
* superblocks. Other superblocks will still get marked if this function
* is used, but that will not affect their reusability.
*/
void retire_super(struct super_block *sb)
{
WARN_ON(!sb->s_bdev);
__super_lock_excl(sb);
if (sb->s_iflags & SB_I_PERSB_BDI) {
bdi_unregister(sb->s_bdi);
sb->s_iflags &= ~SB_I_PERSB_BDI;
}
sb->s_iflags |= SB_I_RETIRED;
super_unlock_excl(sb);
}
EXPORT_SYMBOL(retire_super);
/**
* generic_shutdown_super - common helper for ->kill_sb()
* @sb: superblock to kill
*
* generic_shutdown_super() does all fs-independent work on superblock
* shutdown. Typical ->kill_sb() should pick all fs-specific objects
* that need destruction out of superblock, call generic_shutdown_super()
* and release aforementioned objects. Note: dentries and inodes _are_
* taken care of and do not need specific handling.
*
* Upon calling this function, the filesystem may no longer alter or
* rearrange the set of dentries belonging to this super_block, nor may it
* change the attachments of dentries to inodes.
*/
void generic_shutdown_super(struct super_block *sb)
{
const struct super_operations *sop = sb->s_op;
if (sb->s_root) {
shrink_dcache_for_umount(sb);
sync_filesystem(sb);
sb->s_flags &= ~SB_ACTIVE;
cgroup_writeback_umount();
/* Evict all inodes with zero refcount. */
evict_inodes(sb);
/*
* Clean up and evict any inodes that still have references due
* to fsnotify or the security policy.
*/
fsnotify_sb_delete(sb);
security_sb_delete(sb);
if (sb->s_dio_done_wq) {
destroy_workqueue(sb->s_dio_done_wq);
sb->s_dio_done_wq = NULL;
}
if (sop->put_super)
sop->put_super(sb);
/*
* Now that all potentially-encrypted inodes have been evicted,
* the fscrypt keyring can be destroyed.
*/
fscrypt_destroy_keyring(sb);
if (CHECK_DATA_CORRUPTION(!list_empty(&sb->s_inodes),
"VFS: Busy inodes after unmount of %s (%s)",
sb->s_id, sb->s_type->name)) {
/*
* Adding a proper bailout path here would be hard, but
* we can at least make it more likely that a later
* iput_final() or such crashes cleanly.
*/
struct inode *inode;
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
inode->i_op = VFS_PTR_POISON;
inode->i_sb = VFS_PTR_POISON;
inode->i_mapping = VFS_PTR_POISON;
}
spin_unlock(&sb->s_inode_list_lock);
}
}
/*
* Broadcast to everyone that grabbed a temporary reference to this
* superblock before we removed it from @fs_supers that the superblock
* is dying. Every walker of @fs_supers outside of sget{_fc}() will now
* discard this superblock and treat it as dead.
*
* We leave the superblock on @fs_supers so it can be found by
* sget{_fc}() until we passed sb->kill_sb().
*/
super_wake(sb, SB_DYING);
super_unlock_excl(sb);
if (sb->s_bdi != &noop_backing_dev_info) {
if (sb->s_iflags & SB_I_PERSB_BDI)
bdi_unregister(sb->s_bdi);
bdi_put(sb->s_bdi);
sb->s_bdi = &noop_backing_dev_info;
}
}
EXPORT_SYMBOL(generic_shutdown_super);
bool mount_capable(struct fs_context *fc)
{
if (!(fc->fs_type->fs_flags & FS_USERNS_MOUNT))
return capable(CAP_SYS_ADMIN);
else
return ns_capable(fc->user_ns, CAP_SYS_ADMIN);
}
/**
* sget_fc - Find or create a superblock
* @fc: Filesystem context.
* @test: Comparison callback
* @set: Setup callback
*
* Create a new superblock or find an existing one.
*
* The @test callback is used to find a matching existing superblock.
* Whether or not the requested parameters in @fc are taken into account
* is specific to the @test callback that is used. They may even be
* completely ignored.
*
* If an extant superblock is matched, it will be returned unless:
*
* (1) the namespace the filesystem context @fc and the extant
* superblock's namespace differ
*
* (2) the filesystem context @fc has requested that reusing an extant
* superblock is not allowed
*
* In both cases EBUSY will be returned.
*
* If no match is made, a new superblock will be allocated and basic
* initialisation will be performed (s_type, s_fs_info and s_id will be
* set and the @set callback will be invoked), the superblock will be
* published and it will be returned in a partially constructed state
* with SB_BORN and SB_ACTIVE as yet unset.
*
* Return: On success, an extant or newly created superblock is
* returned. On failure an error pointer is returned.
*/
struct super_block *sget_fc(struct fs_context *fc,
int (*test)(struct super_block *, struct fs_context *),
int (*set)(struct super_block *, struct fs_context *))
{
struct super_block *s = NULL;
struct super_block *old;
struct user_namespace *user_ns = fc->global ? &init_user_ns : fc->user_ns;
int err;
retry:
spin_lock(&sb_lock);
if (test) {
hlist_for_each_entry(old, &fc->fs_type->fs_supers, s_instances) {
if (test(old, fc))
goto share_extant_sb;
}
}
if (!s) {
spin_unlock(&sb_lock);
s = alloc_super(fc->fs_type, fc->sb_flags, user_ns);
if (!s)
return ERR_PTR(-ENOMEM);
goto retry;
}
s->s_fs_info = fc->s_fs_info;
err = set(s, fc);
if (err) {
s->s_fs_info = NULL;
spin_unlock(&sb_lock);
destroy_unused_super(s);
return ERR_PTR(err);
}
fc->s_fs_info = NULL;
s->s_type = fc->fs_type;
s->s_iflags |= fc->s_iflags;
strscpy(s->s_id, s->s_type->name, sizeof(s->s_id));
/*
* Make the superblock visible on @super_blocks and @fs_supers.
* It's in a nascent state and users should wait on SB_BORN or
* SB_DYING to be set.
*/
list_add_tail(&s->s_list, &super_blocks);
hlist_add_head(&s->s_instances, &s->s_type->fs_supers);
spin_unlock(&sb_lock);
get_filesystem(s->s_type);
shrinker_register(s->s_shrink);
return s;
share_extant_sb:
if (user_ns != old->s_user_ns || fc->exclusive) {
spin_unlock(&sb_lock);
destroy_unused_super(s);
if (fc->exclusive)
warnfc(fc, "reusing existing filesystem not allowed");
else
warnfc(fc, "reusing existing filesystem in another namespace not allowed");
return ERR_PTR(-EBUSY);
}
if (!grab_super(old))
goto retry;
destroy_unused_super(s);
return old;
}
EXPORT_SYMBOL(sget_fc);
/**
* sget - find or create a superblock
* @type: filesystem type superblock should belong to
* @test: comparison callback
* @set: setup callback
* @flags: mount flags
* @data: argument to each of them
*/
struct super_block *sget(struct file_system_type *type,
int (*test)(struct super_block *,void *),
int (*set)(struct super_block *,void *),
int flags,
void *data)
{
struct user_namespace *user_ns = current_user_ns();
struct super_block *s = NULL;
struct super_block *old;
int err;
/* We don't yet pass the user namespace of the parent
* mount through to here so always use &init_user_ns
* until that changes.
*/
if (flags & SB_SUBMOUNT)
user_ns = &init_user_ns;
retry:
spin_lock(&sb_lock);
if (test) {
hlist_for_each_entry(old, &type->fs_supers, s_instances) {
if (!test(old, data))
continue;
if (user_ns != old->s_user_ns) {
spin_unlock(&sb_lock);
destroy_unused_super(s);
return ERR_PTR(-EBUSY);
}
if (!grab_super(old))
goto retry;
destroy_unused_super(s);
return old;
}
}
if (!s) {
spin_unlock(&sb_lock);
s = alloc_super(type, (flags & ~SB_SUBMOUNT), user_ns);
if (!s)
return ERR_PTR(-ENOMEM);
goto retry;
}
err = set(s, data);
if (err) {
spin_unlock(&sb_lock);
destroy_unused_super(s);
return ERR_PTR(err);
}
s->s_type = type;
strscpy(s->s_id, type->name, sizeof(s->s_id));
list_add_tail(&s->s_list, &super_blocks);
hlist_add_head(&s->s_instances, &type->fs_supers);
spin_unlock(&sb_lock);
get_filesystem(type);
shrinker_register(s->s_shrink);
return s;
}
EXPORT_SYMBOL(sget);
void drop_super(struct super_block *sb)
{
super_unlock_shared(sb);
put_super(sb);
}
EXPORT_SYMBOL(drop_super);
void drop_super_exclusive(struct super_block *sb)
{
super_unlock_excl(sb);
put_super(sb);
}
EXPORT_SYMBOL(drop_super_exclusive);
static void __iterate_supers(void (*f)(struct super_block *))
{
struct super_block *sb, *p = NULL;
spin_lock(&sb_lock);
list_for_each_entry(sb, &super_blocks, s_list) {
if (super_flags(sb, SB_DYING))
continue;
sb->s_count++;
spin_unlock(&sb_lock);
f(sb);
spin_lock(&sb_lock);
if (p)
__put_super(p);
p = sb;
}
if (p)
__put_super(p);
spin_unlock(&sb_lock);
}
/**
* iterate_supers - call function for all active superblocks
* @f: function to call
* @arg: argument to pass to it
*
* Scans the superblock list and calls given function, passing it
* locked superblock and given argument.
*/
void iterate_supers(void (*f)(struct super_block *, void *), void *arg)
{
struct super_block *sb, *p = NULL;
spin_lock(&sb_lock);
list_for_each_entry(sb, &super_blocks, s_list) {
bool locked;
sb->s_count++;
spin_unlock(&sb_lock);
locked = super_lock_shared(sb);
if (locked) {
if (sb->s_root)
f(sb, arg);
super_unlock_shared(sb);
}
spin_lock(&sb_lock);
if (p)
__put_super(p);
p = sb;
}
if (p)
__put_super(p);
spin_unlock(&sb_lock);
}
/**
* iterate_supers_type - call function for superblocks of given type
* @type: fs type
* @f: function to call
* @arg: argument to pass to it
*
* Scans the superblock list and calls given function, passing it
* locked superblock and given argument.
*/
void iterate_supers_type(struct file_system_type *type,
void (*f)(struct super_block *, void *), void *arg)
{
struct super_block *sb, *p = NULL;
spin_lock(&sb_lock);
hlist_for_each_entry(sb, &type->fs_supers, s_instances) {
bool locked;
sb->s_count++;
spin_unlock(&sb_lock);
locked = super_lock_shared(sb);
if (locked) {
if (sb->s_root)
f(sb, arg);
super_unlock_shared(sb);
}
spin_lock(&sb_lock);
if (p)
__put_super(p);
p = sb;
}
if (p)
__put_super(p);
spin_unlock(&sb_lock);
}
EXPORT_SYMBOL(iterate_supers_type);
struct super_block *user_get_super(dev_t dev, bool excl)
{
struct super_block *sb;
spin_lock(&sb_lock);
list_for_each_entry(sb, &super_blocks, s_list) {
if (sb->s_dev == dev) {
bool locked;
sb->s_count++;
spin_unlock(&sb_lock);
/* still alive? */
locked = super_lock(sb, excl);
if (locked) {
if (sb->s_root)
return sb;
super_unlock(sb, excl);
}
/* nope, got unmounted */
spin_lock(&sb_lock);
__put_super(sb);
break;
}
}
spin_unlock(&sb_lock);
return NULL;
}
/**
* reconfigure_super - asks filesystem to change superblock parameters
* @fc: The superblock and configuration
*
* Alters the configuration parameters of a live superblock.
*/
int reconfigure_super(struct fs_context *fc)
{
struct super_block *sb = fc->root->d_sb;
int retval;
bool remount_ro = false;
bool remount_rw = false;
bool force = fc->sb_flags & SB_FORCE;
if (fc->sb_flags_mask & ~MS_RMT_MASK)
return -EINVAL;
if (sb->s_writers.frozen != SB_UNFROZEN)
return -EBUSY;
retval = security_sb_remount(sb, fc->security);
if (retval)
return retval;
if (fc->sb_flags_mask & SB_RDONLY) {
#ifdef CONFIG_BLOCK
if (!(fc->sb_flags & SB_RDONLY) && sb->s_bdev &&
bdev_read_only(sb->s_bdev))
return -EACCES;
#endif
remount_rw = !(fc->sb_flags & SB_RDONLY) && sb_rdonly(sb);
remount_ro = (fc->sb_flags & SB_RDONLY) && !sb_rdonly(sb);
}
if (remount_ro) {
if (!hlist_empty(&sb->s_pins)) {
super_unlock_excl(sb);
group_pin_kill(&sb->s_pins);
__super_lock_excl(sb);
if (!sb->s_root)
return 0;
if (sb->s_writers.frozen != SB_UNFROZEN)
return -EBUSY;
remount_ro = !sb_rdonly(sb);
}
}
shrink_dcache_sb(sb);
/* If we are reconfiguring to RDONLY and current sb is read/write,
* make sure there are no files open for writing.
*/
if (remount_ro) {
if (force) {
sb_start_ro_state_change(sb);
} else {
retval = sb_prepare_remount_readonly(sb);
if (retval)
return retval;
}
} else if (remount_rw) {
/*
* Protect filesystem's reconfigure code from writes from
* userspace until reconfigure finishes.
*/
sb_start_ro_state_change(sb);
}
if (fc->ops->reconfigure) {
retval = fc->ops->reconfigure(fc);
if (retval) {
if (!force)
goto cancel_readonly;
/* If forced remount, go ahead despite any errors */
WARN(1, "forced remount of a %s fs returned %i\n",
sb->s_type->name, retval);
}
}
WRITE_ONCE(sb->s_flags, ((sb->s_flags & ~fc->sb_flags_mask) |
(fc->sb_flags & fc->sb_flags_mask)));
sb_end_ro_state_change(sb);
/*
* Some filesystems modify their metadata via some other path than the
* bdev buffer cache (eg. use a private mapping, or directories in
* pagecache, etc). Also file data modifications go via their own
* mappings. So If we try to mount readonly then copy the filesystem
* from bdev, we could get stale data, so invalidate it to give a best
* effort at coherency.
*/
if (remount_ro && sb->s_bdev)
invalidate_bdev(sb->s_bdev);
return 0;
cancel_readonly:
sb_end_ro_state_change(sb);
return retval;
}
static void do_emergency_remount_callback(struct super_block *sb)
{
bool locked = super_lock_excl(sb);
if (locked && sb->s_root && sb->s_bdev && !sb_rdonly(sb)) {
struct fs_context *fc;
fc = fs_context_for_reconfigure(sb->s_root,
SB_RDONLY | SB_FORCE, SB_RDONLY);
if (!IS_ERR(fc)) {
if (parse_monolithic_mount_data(fc, NULL) == 0)
(void)reconfigure_super(fc);
put_fs_context(fc);
}
}
if (locked)
super_unlock_excl(sb);
}
static void do_emergency_remount(struct work_struct *work)
{
__iterate_supers(do_emergency_remount_callback);
kfree(work);
printk("Emergency Remount complete\n");
}
void emergency_remount(void)
{
struct work_struct *work;
work = kmalloc(sizeof(*work), GFP_ATOMIC);
if (work) {
INIT_WORK(work, do_emergency_remount);
schedule_work(work);
}
}
static void do_thaw_all_callback(struct super_block *sb)
{
bool locked = super_lock_excl(sb);
if (locked && sb->s_root) {
if (IS_ENABLED(CONFIG_BLOCK))
while (sb->s_bdev && !bdev_thaw(sb->s_bdev))
pr_warn("Emergency Thaw on %pg\n", sb->s_bdev);
thaw_super_locked(sb, FREEZE_HOLDER_USERSPACE);
return;
}
if (locked)
super_unlock_excl(sb);
}
static void do_thaw_all(struct work_struct *work)
{
__iterate_supers(do_thaw_all_callback);
kfree(work);
printk(KERN_WARNING "Emergency Thaw complete\n");
}
/**
* emergency_thaw_all -- forcibly thaw every frozen filesystem
*
* Used for emergency unfreeze of all filesystems via SysRq
*/
void emergency_thaw_all(void)
{
struct work_struct *work;
work = kmalloc(sizeof(*work), GFP_ATOMIC);
if (work) {
INIT_WORK(work, do_thaw_all);
schedule_work(work);
}
}
static DEFINE_IDA(unnamed_dev_ida);
/**
* get_anon_bdev - Allocate a block device for filesystems which don't have one.
* @p: Pointer to a dev_t.
*
* Filesystems which don't use real block devices can call this function
* to allocate a virtual block device.
*
* Context: Any context. Frequently called while holding sb_lock.
* Return: 0 on success, -EMFILE if there are no anonymous bdevs left
* or -ENOMEM if memory allocation failed.
*/
int get_anon_bdev(dev_t *p)
{
int dev;
/*
* Many userspace utilities consider an FSID of 0 invalid.
* Always return at least 1 from get_anon_bdev.
*/
dev = ida_alloc_range(&unnamed_dev_ida, 1, (1 << MINORBITS) - 1,
GFP_ATOMIC);
if (dev == -ENOSPC)
dev = -EMFILE;
if (dev < 0)
return dev;
*p = MKDEV(0, dev);
return 0;
}
EXPORT_SYMBOL(get_anon_bdev);
void free_anon_bdev(dev_t dev)
{
ida_free(&unnamed_dev_ida, MINOR(dev));
}
EXPORT_SYMBOL(free_anon_bdev);
int set_anon_super(struct super_block *s, void *data)
{
return get_anon_bdev(&s->s_dev);
}
EXPORT_SYMBOL(set_anon_super);
void kill_anon_super(struct super_block *sb)
{
dev_t dev = sb->s_dev;
generic_shutdown_super(sb);
kill_super_notify(sb);
free_anon_bdev(dev);
}
EXPORT_SYMBOL(kill_anon_super);
void kill_litter_super(struct super_block *sb)
{
if (sb->s_root)
d_genocide(sb->s_root);
kill_anon_super(sb);
}
EXPORT_SYMBOL(kill_litter_super);
int set_anon_super_fc(struct super_block *sb, struct fs_context *fc)
{
return set_anon_super(sb, NULL);
}
EXPORT_SYMBOL(set_anon_super_fc);
static int test_keyed_super(struct super_block *sb, struct fs_context *fc)
{
return sb->s_fs_info == fc->s_fs_info;
}
static int test_single_super(struct super_block *s, struct fs_context *fc)
{
return 1;
}
static int vfs_get_super(struct fs_context *fc,
int (*test)(struct super_block *, struct fs_context *),
int (*fill_super)(struct super_block *sb,
struct fs_context *fc))
{
struct super_block *sb;
int err;
sb = sget_fc(fc, test, set_anon_super_fc);
if (IS_ERR(sb))
return PTR_ERR(sb);
if (!sb->s_root) {
err = fill_super(sb, fc);
if (err)
goto error;
sb->s_flags |= SB_ACTIVE;
}
fc->root = dget(sb->s_root);
return 0;
error:
deactivate_locked_super(sb);
return err;
}
int get_tree_nodev(struct fs_context *fc,
int (*fill_super)(struct super_block *sb,
struct fs_context *fc))
{
return vfs_get_super(fc, NULL, fill_super);
}
EXPORT_SYMBOL(get_tree_nodev);
int get_tree_single(struct fs_context *fc,
int (*fill_super)(struct super_block *sb,
struct fs_context *fc))
{
return vfs_get_super(fc, test_single_super, fill_super);
}
EXPORT_SYMBOL(get_tree_single);
int get_tree_keyed(struct fs_context *fc,
int (*fill_super)(struct super_block *sb,
struct fs_context *fc),
void *key)
{
fc->s_fs_info = key;
return vfs_get_super(fc, test_keyed_super, fill_super);
}
EXPORT_SYMBOL(get_tree_keyed);
static int set_bdev_super(struct super_block *s, void *data)
{
s->s_dev = *(dev_t *)data;
return 0;
}
static int super_s_dev_set(struct super_block *s, struct fs_context *fc)
{
return set_bdev_super(s, fc->sget_key);
}
static int super_s_dev_test(struct super_block *s, struct fs_context *fc)
{
return !(s->s_iflags & SB_I_RETIRED) &&
s->s_dev == *(dev_t *)fc->sget_key;
}
/**
* sget_dev - Find or create a superblock by device number
* @fc: Filesystem context.
* @dev: device number
*
* Find or create a superblock using the provided device number that
* will be stored in fc->sget_key.
*
* If an extant superblock is matched, then that will be returned with
* an elevated reference count that the caller must transfer or discard.
*
* If no match is made, a new superblock will be allocated and basic
* initialisation will be performed (s_type, s_fs_info, s_id, s_dev will
* be set). The superblock will be published and it will be returned in
* a partially constructed state with SB_BORN and SB_ACTIVE as yet
* unset.
*
* Return: an existing or newly created superblock on success, an error
* pointer on failure.
*/
struct super_block *sget_dev(struct fs_context *fc, dev_t dev)
{
fc->sget_key = &dev;
return sget_fc(fc, super_s_dev_test, super_s_dev_set);
}
EXPORT_SYMBOL(sget_dev);
#ifdef CONFIG_BLOCK
/*
* Lock the superblock that is holder of the bdev. Returns the superblock
* pointer if we successfully locked the superblock and it is alive. Otherwise
* we return NULL and just unlock bdev->bd_holder_lock.
*
* The function must be called with bdev->bd_holder_lock and releases it.
*/
static struct super_block *bdev_super_lock(struct block_device *bdev, bool excl)
__releases(&bdev->bd_holder_lock)
{
struct super_block *sb = bdev->bd_holder;
bool locked;
lockdep_assert_held(&bdev->bd_holder_lock);
lockdep_assert_not_held(&sb->s_umount);
lockdep_assert_not_held(&bdev->bd_disk->open_mutex);
/* Make sure sb doesn't go away from under us */
spin_lock(&sb_lock);
sb->s_count++;
spin_unlock(&sb_lock);
mutex_unlock(&bdev->bd_holder_lock);
locked = super_lock(sb, excl);
/*
* If the superblock wasn't already SB_DYING then we hold
* s_umount and can safely drop our temporary reference.
*/
put_super(sb);
if (!locked)
return NULL;
if (!sb->s_root || !(sb->s_flags & SB_ACTIVE)) {
super_unlock(sb, excl);
return NULL;
}
return sb;
}
static void fs_bdev_mark_dead(struct block_device *bdev, bool surprise)
{
struct super_block *sb;
sb = bdev_super_lock(bdev, false);
if (!sb)
return;
if (!surprise)
sync_filesystem(sb);
shrink_dcache_sb(sb);
invalidate_inodes(sb);
if (sb->s_op->shutdown)
sb->s_op->shutdown(sb);
super_unlock_shared(sb);
}
static void fs_bdev_sync(struct block_device *bdev)
{
struct super_block *sb;
sb = bdev_super_lock(bdev, false);
if (!sb)
return;
sync_filesystem(sb);
super_unlock_shared(sb);
}
static struct super_block *get_bdev_super(struct block_device *bdev)
{
bool active = false;
struct super_block *sb;
sb = bdev_super_lock(bdev, true);
if (sb) {
active = atomic_inc_not_zero(&sb->s_active);
super_unlock_excl(sb);
}
if (!active)
return NULL;
return sb;
}
/**
* fs_bdev_freeze - freeze owning filesystem of block device
* @bdev: block device
*
* Freeze the filesystem that owns this block device if it is still
* active.
*
* A filesystem that owns multiple block devices may be frozen from each
* block device and won't be unfrozen until all block devices are
* unfrozen. Each block device can only freeze the filesystem once as we
* nest freezes for block devices in the block layer.
*
* Return: If the freeze was successful zero is returned. If the freeze
* failed a negative error code is returned.
*/
static int fs_bdev_freeze(struct block_device *bdev)
{
struct super_block *sb;
int error = 0;
lockdep_assert_held(&bdev->bd_fsfreeze_mutex);
sb = get_bdev_super(bdev);
if (!sb)
return -EINVAL;
if (sb->s_op->freeze_super)
error = sb->s_op->freeze_super(sb,
FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
else
error = freeze_super(sb,
FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
if (!error)
error = sync_blockdev(bdev);
deactivate_super(sb);
return error;
}
/**
* fs_bdev_thaw - thaw owning filesystem of block device
* @bdev: block device
*
* Thaw the filesystem that owns this block device.
*
* A filesystem that owns multiple block devices may be frozen from each
* block device and won't be unfrozen until all block devices are
* unfrozen. Each block device can only freeze the filesystem once as we
* nest freezes for block devices in the block layer.
*
* Return: If the thaw was successful zero is returned. If the thaw
* failed a negative error code is returned. If this function
* returns zero it doesn't mean that the filesystem is unfrozen
* as it may have been frozen multiple times (kernel may hold a
* freeze or might be frozen from other block devices).
*/
static int fs_bdev_thaw(struct block_device *bdev)
{
struct super_block *sb;
int error;
lockdep_assert_held(&bdev->bd_fsfreeze_mutex);
sb = get_bdev_super(bdev);
if (WARN_ON_ONCE(!sb))
return -EINVAL;
if (sb->s_op->thaw_super)
error = sb->s_op->thaw_super(sb,
FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
else
error = thaw_super(sb,
FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
deactivate_super(sb);
return error;
}
const struct blk_holder_ops fs_holder_ops = {
.mark_dead = fs_bdev_mark_dead,
.sync = fs_bdev_sync,
.freeze = fs_bdev_freeze,
.thaw = fs_bdev_thaw,
};
EXPORT_SYMBOL_GPL(fs_holder_ops);
int setup_bdev_super(struct super_block *sb, int sb_flags,
struct fs_context *fc)
{
blk_mode_t mode = sb_open_mode(sb_flags);
struct file *bdev_file;
struct block_device *bdev;
bdev_file = bdev_file_open_by_dev(sb->s_dev, mode, sb, &fs_holder_ops);
if (IS_ERR(bdev_file)) {
if (fc)
errorf(fc, "%s: Can't open blockdev", fc->source);
return PTR_ERR(bdev_file);
}
bdev = file_bdev(bdev_file);
/*
* This really should be in blkdev_get_by_dev, but right now can't due
* to legacy issues that require us to allow opening a block device node
* writable from userspace even for a read-only block device.
*/
if ((mode & BLK_OPEN_WRITE) && bdev_read_only(bdev)) {
bdev_fput(bdev_file);
return -EACCES;
}
/*
* It is enough to check bdev was not frozen before we set
* s_bdev as freezing will wait until SB_BORN is set.
*/
if (atomic_read(&bdev->bd_fsfreeze_count) > 0) {
if (fc)
warnf(fc, "%pg: Can't mount, blockdev is frozen", bdev);
bdev_fput(bdev_file);
return -EBUSY;
}
spin_lock(&sb_lock);
sb->s_bdev_file = bdev_file;
sb->s_bdev = bdev;
sb->s_bdi = bdi_get(bdev->bd_disk->bdi);
if (bdev_stable_writes(bdev))
sb->s_iflags |= SB_I_STABLE_WRITES;
spin_unlock(&sb_lock);
snprintf(sb->s_id, sizeof(sb->s_id), "%pg", bdev);
shrinker_debugfs_rename(sb->s_shrink, "sb-%s:%s", sb->s_type->name,
sb->s_id);
sb_set_blocksize(sb, block_size(bdev));
return 0;
}
EXPORT_SYMBOL_GPL(setup_bdev_super);
/**
* get_tree_bdev - Get a superblock based on a single block device
* @fc: The filesystem context holding the parameters
* @fill_super: Helper to initialise a new superblock
*/
int get_tree_bdev(struct fs_context *fc,
int (*fill_super)(struct super_block *,
struct fs_context *))
{
struct super_block *s;
int error = 0;
dev_t dev;
if (!fc->source)
return invalf(fc, "No source specified");
error = lookup_bdev(fc->source, &dev);
if (error) {
errorf(fc, "%s: Can't lookup blockdev", fc->source);
return error;
}
fc->sb_flags |= SB_NOSEC;
s = sget_dev(fc, dev);
if (IS_ERR(s))
return PTR_ERR(s);
if (s->s_root) {
/* Don't summarily change the RO/RW state. */
if ((fc->sb_flags ^ s->s_flags) & SB_RDONLY) {
warnf(fc, "%pg: Can't mount, would change RO state", s->s_bdev);
deactivate_locked_super(s);
return -EBUSY;
}
} else {
error = setup_bdev_super(s, fc->sb_flags, fc);
if (!error)
error = fill_super(s, fc);
if (error) {
deactivate_locked_super(s);
return error;
}
s->s_flags |= SB_ACTIVE;
}
BUG_ON(fc->root);
fc->root = dget(s->s_root);
return 0;
}
EXPORT_SYMBOL(get_tree_bdev);
static int test_bdev_super(struct super_block *s, void *data)
{
return !(s->s_iflags & SB_I_RETIRED) && s->s_dev == *(dev_t *)data;
}
struct dentry *mount_bdev(struct file_system_type *fs_type,
int flags, const char *dev_name, void *data,
int (*fill_super)(struct super_block *, void *, int))
{
struct super_block *s;
int error;
dev_t dev;
error = lookup_bdev(dev_name, &dev);
if (error)
return ERR_PTR(error);
flags |= SB_NOSEC;
s = sget(fs_type, test_bdev_super, set_bdev_super, flags, &dev);
if (IS_ERR(s))
return ERR_CAST(s);
if (s->s_root) {
if ((flags ^ s->s_flags) & SB_RDONLY) {
deactivate_locked_super(s);
return ERR_PTR(-EBUSY);
}
} else {
error = setup_bdev_super(s, flags, NULL);
if (!error)
error = fill_super(s, data, flags & SB_SILENT ? 1 : 0);
if (error) {
deactivate_locked_super(s);
return ERR_PTR(error);
}
s->s_flags |= SB_ACTIVE;
}
return dget(s->s_root);
}
EXPORT_SYMBOL(mount_bdev);
void kill_block_super(struct super_block *sb)
{
struct block_device *bdev = sb->s_bdev;
generic_shutdown_super(sb);
if (bdev) {
sync_blockdev(bdev);
bdev_fput(sb->s_bdev_file);
}
}
EXPORT_SYMBOL(kill_block_super);
#endif
struct dentry *mount_nodev(struct file_system_type *fs_type,
int flags, void *data,
int (*fill_super)(struct super_block *, void *, int))
{
int error;
struct super_block *s = sget(fs_type, NULL, set_anon_super, flags, NULL);
if (IS_ERR(s))
return ERR_CAST(s);
error = fill_super(s, data, flags & SB_SILENT ? 1 : 0);
if (error) {
deactivate_locked_super(s);
return ERR_PTR(error);
}
s->s_flags |= SB_ACTIVE;
return dget(s->s_root);
}
EXPORT_SYMBOL(mount_nodev);
int reconfigure_single(struct super_block *s,
int flags, void *data)
{
struct fs_context *fc;
int ret;
/* The caller really need to be passing fc down into mount_single(),
* then a chunk of this can be removed. [Bollocks -- AV]
* Better yet, reconfiguration shouldn't happen, but rather the second
* mount should be rejected if the parameters are not compatible.
*/
fc = fs_context_for_reconfigure(s->s_root, flags, MS_RMT_MASK);
if (IS_ERR(fc))
return PTR_ERR(fc);
ret = parse_monolithic_mount_data(fc, data);
if (ret < 0)
goto out;
ret = reconfigure_super(fc);
out:
put_fs_context(fc);
return ret;
}
static int compare_single(struct super_block *s, void *p)
{
return 1;
}
struct dentry *mount_single(struct file_system_type *fs_type,
int flags, void *data,
int (*fill_super)(struct super_block *, void *, int))
{
struct super_block *s;
int error;
s = sget(fs_type, compare_single, set_anon_super, flags, NULL);
if (IS_ERR(s))
return ERR_CAST(s);
if (!s->s_root) {
error = fill_super(s, data, flags & SB_SILENT ? 1 : 0);
if (!error)
s->s_flags |= SB_ACTIVE;
} else {
error = reconfigure_single(s, flags, data);
}
if (unlikely(error)) {
deactivate_locked_super(s);
return ERR_PTR(error);
}
return dget(s->s_root);
}
EXPORT_SYMBOL(mount_single);
/**
* vfs_get_tree - Get the mountable root
* @fc: The superblock configuration context.
*
* The filesystem is invoked to get or create a superblock which can then later
* be used for mounting. The filesystem places a pointer to the root to be
* used for mounting in @fc->root.
*/
int vfs_get_tree(struct fs_context *fc)
{
struct super_block *sb;
int error;
if (fc->root)
return -EBUSY;
/* Get the mountable root in fc->root, with a ref on the root and a ref
* on the superblock.
*/
error = fc->ops->get_tree(fc);
if (error < 0)
return error;
if (!fc->root) {
pr_err("Filesystem %s get_tree() didn't set fc->root\n",
fc->fs_type->name);
/* We don't know what the locking state of the superblock is -
* if there is a superblock.
*/
BUG();
}
sb = fc->root->d_sb;
WARN_ON(!sb->s_bdi);
/*
* super_wake() contains a memory barrier which also care of
* ordering for super_cache_count(). We place it before setting
* SB_BORN as the data dependency between the two functions is
* the superblock structure contents that we just set up, not
* the SB_BORN flag.
*/
super_wake(sb, SB_BORN);
error = security_sb_set_mnt_opts(sb, fc->security, 0, NULL);
if (unlikely(error)) {
fc_drop_locked(fc);
return error;
}
/*
* filesystems should never set s_maxbytes larger than MAX_LFS_FILESIZE
* but s_maxbytes was an unsigned long long for many releases. Throw
* this warning for a little while to try and catch filesystems that
* violate this rule.
*/
WARN((sb->s_maxbytes < 0), "%s set sb->s_maxbytes to "
"negative value (%lld)\n", fc->fs_type->name, sb->s_maxbytes);
return 0;
}
EXPORT_SYMBOL(vfs_get_tree);
/*
* Setup private BDI for given superblock. It gets automatically cleaned up
* in generic_shutdown_super().
*/
int super_setup_bdi_name(struct super_block *sb, char *fmt, ...)
{
struct backing_dev_info *bdi;
int err;
va_list args;
bdi = bdi_alloc(NUMA_NO_NODE);
if (!bdi)
return -ENOMEM;
va_start(args, fmt);
err = bdi_register_va(bdi, fmt, args);
va_end(args);
if (err) {
bdi_put(bdi);
return err;
}
WARN_ON(sb->s_bdi != &noop_backing_dev_info);
sb->s_bdi = bdi;
sb->s_iflags |= SB_I_PERSB_BDI;
return 0;
}
EXPORT_SYMBOL(super_setup_bdi_name);
/*
* Setup private BDI for given superblock. I gets automatically cleaned up
* in generic_shutdown_super().
*/
int super_setup_bdi(struct super_block *sb)
{
static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
return super_setup_bdi_name(sb, "%.28s-%ld", sb->s_type->name,
atomic_long_inc_return(&bdi_seq));
}
EXPORT_SYMBOL(super_setup_bdi);
/**
* sb_wait_write - wait until all writers to given file system finish
* @sb: the super for which we wait
* @level: type of writers we wait for (normal vs page fault)
*
* This function waits until there are no writers of given type to given file
* system.
*/
static void sb_wait_write(struct super_block *sb, int level)
{
percpu_down_write(sb->s_writers.rw_sem + level-1);
}
/*
* We are going to return to userspace and forget about these locks, the
* ownership goes to the caller of thaw_super() which does unlock().
*/
static void lockdep_sb_freeze_release(struct super_block *sb)
{
int level;
for (level = SB_FREEZE_LEVELS - 1; level >= 0; level--)
percpu_rwsem_release(sb->s_writers.rw_sem + level, 0, _THIS_IP_);
}
/*
* Tell lockdep we are holding these locks before we call ->unfreeze_fs(sb).
*/
static void lockdep_sb_freeze_acquire(struct super_block *sb)
{
int level;
for (level = 0; level < SB_FREEZE_LEVELS; ++level)
percpu_rwsem_acquire(sb->s_writers.rw_sem + level, 0, _THIS_IP_);
}
static void sb_freeze_unlock(struct super_block *sb, int level)
{
for (level--; level >= 0; level--)
percpu_up_write(sb->s_writers.rw_sem + level);
}
static int wait_for_partially_frozen(struct super_block *sb)
{
int ret = 0;
do {
unsigned short old = sb->s_writers.frozen;
up_write(&sb->s_umount);
ret = wait_var_event_killable(&sb->s_writers.frozen,
sb->s_writers.frozen != old);
down_write(&sb->s_umount);
} while (ret == 0 &&
sb->s_writers.frozen != SB_UNFROZEN &&
sb->s_writers.frozen != SB_FREEZE_COMPLETE);
return ret;
}
#define FREEZE_HOLDERS (FREEZE_HOLDER_KERNEL | FREEZE_HOLDER_USERSPACE)
#define FREEZE_FLAGS (FREEZE_HOLDERS | FREEZE_MAY_NEST)
static inline int freeze_inc(struct super_block *sb, enum freeze_holder who)
{
WARN_ON_ONCE((who & ~FREEZE_FLAGS)); WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1); if (who & FREEZE_HOLDER_KERNEL) ++sb->s_writers.freeze_kcount; if (who & FREEZE_HOLDER_USERSPACE) ++sb->s_writers.freeze_ucount;
return sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount;
}
static inline int freeze_dec(struct super_block *sb, enum freeze_holder who)
{
WARN_ON_ONCE((who & ~FREEZE_FLAGS)); WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1); if ((who & FREEZE_HOLDER_KERNEL) && sb->s_writers.freeze_kcount) --sb->s_writers.freeze_kcount; if ((who & FREEZE_HOLDER_USERSPACE) && sb->s_writers.freeze_ucount) --sb->s_writers.freeze_ucount; return sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount;
}
static inline bool may_freeze(struct super_block *sb, enum freeze_holder who)
{
WARN_ON_ONCE((who & ~FREEZE_FLAGS));
WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1);
if (who & FREEZE_HOLDER_KERNEL)
return (who & FREEZE_MAY_NEST) ||
sb->s_writers.freeze_kcount == 0;
if (who & FREEZE_HOLDER_USERSPACE)
return (who & FREEZE_MAY_NEST) ||
sb->s_writers.freeze_ucount == 0;
return false;
}
/**
* freeze_super - lock the filesystem and force it into a consistent state
* @sb: the super to lock
* @who: context that wants to freeze
*
* Syncs the super to make sure the filesystem is consistent and calls the fs's
* freeze_fs. Subsequent calls to this without first thawing the fs may return
* -EBUSY.
*
* @who should be:
* * %FREEZE_HOLDER_USERSPACE if userspace wants to freeze the fs;
* * %FREEZE_HOLDER_KERNEL if the kernel wants to freeze the fs.
* * %FREEZE_MAY_NEST whether nesting freeze and thaw requests is allowed.
*
* The @who argument distinguishes between the kernel and userspace trying to
* freeze the filesystem. Although there cannot be multiple kernel freezes or
* multiple userspace freezes in effect at any given time, the kernel and
* userspace can both hold a filesystem frozen. The filesystem remains frozen
* until there are no kernel or userspace freezes in effect.
*
* A filesystem may hold multiple devices and thus a filesystems may be
* frozen through the block layer via multiple block devices. In this
* case the request is marked as being allowed to nest by passing
* FREEZE_MAY_NEST. The filesystem remains frozen until all block
* devices are unfrozen. If multiple freezes are attempted without
* FREEZE_MAY_NEST -EBUSY will be returned.
*
* During this function, sb->s_writers.frozen goes through these values:
*
* SB_UNFROZEN: File system is normal, all writes progress as usual.
*
* SB_FREEZE_WRITE: The file system is in the process of being frozen. New
* writes should be blocked, though page faults are still allowed. We wait for
* all writes to complete and then proceed to the next stage.
*
* SB_FREEZE_PAGEFAULT: Freezing continues. Now also page faults are blocked
* but internal fs threads can still modify the filesystem (although they
* should not dirty new pages or inodes), writeback can run etc. After waiting
* for all running page faults we sync the filesystem which will clean all
* dirty pages and inodes (no new dirty pages or inodes can be created when
* sync is running).
*
* SB_FREEZE_FS: The file system is frozen. Now all internal sources of fs
* modification are blocked (e.g. XFS preallocation truncation on inode
* reclaim). This is usually implemented by blocking new transactions for
* filesystems that have them and need this additional guard. After all
* internal writers are finished we call ->freeze_fs() to finish filesystem
* freezing. Then we transition to SB_FREEZE_COMPLETE state. This state is
* mostly auxiliary for filesystems to verify they do not modify frozen fs.
*
* sb->s_writers.frozen is protected by sb->s_umount.
*
* Return: If the freeze was successful zero is returned. If the freeze
* failed a negative error code is returned.
*/
int freeze_super(struct super_block *sb, enum freeze_holder who)
{
int ret;
if (!super_lock_excl(sb)) {
WARN_ON_ONCE("Dying superblock while freezing!");
return -EINVAL;
}
atomic_inc(&sb->s_active);
retry:
if (sb->s_writers.frozen == SB_FREEZE_COMPLETE) {
if (may_freeze(sb, who))
ret = !!WARN_ON_ONCE(freeze_inc(sb, who) == 1);
else
ret = -EBUSY;
/* All freezers share a single active reference. */
deactivate_locked_super(sb);
return ret;
}
if (sb->s_writers.frozen != SB_UNFROZEN) {
ret = wait_for_partially_frozen(sb);
if (ret) {
deactivate_locked_super(sb);
return ret;
}
goto retry;
}
if (sb_rdonly(sb)) {
/* Nothing to do really... */
WARN_ON_ONCE(freeze_inc(sb, who) > 1);
sb->s_writers.frozen = SB_FREEZE_COMPLETE;
wake_up_var(&sb->s_writers.frozen);
super_unlock_excl(sb);
return 0;
}
sb->s_writers.frozen = SB_FREEZE_WRITE;
/* Release s_umount to preserve sb_start_write -> s_umount ordering */
super_unlock_excl(sb);
sb_wait_write(sb, SB_FREEZE_WRITE);
__super_lock_excl(sb);
/* Now we go and block page faults... */
sb->s_writers.frozen = SB_FREEZE_PAGEFAULT;
sb_wait_write(sb, SB_FREEZE_PAGEFAULT);
/* All writers are done so after syncing there won't be dirty data */
ret = sync_filesystem(sb);
if (ret) {
sb->s_writers.frozen = SB_UNFROZEN;
sb_freeze_unlock(sb, SB_FREEZE_PAGEFAULT);
wake_up_var(&sb->s_writers.frozen);
deactivate_locked_super(sb);
return ret;
}
/* Now wait for internal filesystem counter */
sb->s_writers.frozen = SB_FREEZE_FS;
sb_wait_write(sb, SB_FREEZE_FS);
if (sb->s_op->freeze_fs) {
ret = sb->s_op->freeze_fs(sb);
if (ret) {
printk(KERN_ERR
"VFS:Filesystem freeze failed\n");
sb->s_writers.frozen = SB_UNFROZEN;
sb_freeze_unlock(sb, SB_FREEZE_FS);
wake_up_var(&sb->s_writers.frozen);
deactivate_locked_super(sb);
return ret;
}
}
/*
* For debugging purposes so that fs can warn if it sees write activity
* when frozen is set to SB_FREEZE_COMPLETE, and for thaw_super().
*/
WARN_ON_ONCE(freeze_inc(sb, who) > 1);
sb->s_writers.frozen = SB_FREEZE_COMPLETE;
wake_up_var(&sb->s_writers.frozen);
lockdep_sb_freeze_release(sb);
super_unlock_excl(sb);
return 0;
}
EXPORT_SYMBOL(freeze_super);
/*
* Undoes the effect of a freeze_super_locked call. If the filesystem is
* frozen both by userspace and the kernel, a thaw call from either source
* removes that state without releasing the other state or unlocking the
* filesystem.
*/
static int thaw_super_locked(struct super_block *sb, enum freeze_holder who)
{
int error = -EINVAL;
if (sb->s_writers.frozen != SB_FREEZE_COMPLETE)
goto out_unlock;
/*
* All freezers share a single active reference.
* So just unlock in case there are any left.
*/
if (freeze_dec(sb, who))
goto out_unlock;
if (sb_rdonly(sb)) { sb->s_writers.frozen = SB_UNFROZEN;
wake_up_var(&sb->s_writers.frozen);
goto out_deactivate;
}
lockdep_sb_freeze_acquire(sb); if (sb->s_op->unfreeze_fs) { error = sb->s_op->unfreeze_fs(sb);
if (error) {
pr_err("VFS: Filesystem thaw failed\n"); freeze_inc(sb, who); lockdep_sb_freeze_release(sb);
goto out_unlock;
}
}
sb->s_writers.frozen = SB_UNFROZEN;
wake_up_var(&sb->s_writers.frozen);
sb_freeze_unlock(sb, SB_FREEZE_FS);
out_deactivate:
deactivate_locked_super(sb);
return 0;
out_unlock:
super_unlock_excl(sb); return error;
}
/**
* thaw_super -- unlock filesystem
* @sb: the super to thaw
* @who: context that wants to freeze
*
* Unlocks the filesystem and marks it writeable again after freeze_super()
* if there are no remaining freezes on the filesystem.
*
* @who should be:
* * %FREEZE_HOLDER_USERSPACE if userspace wants to thaw the fs;
* * %FREEZE_HOLDER_KERNEL if the kernel wants to thaw the fs.
* * %FREEZE_MAY_NEST whether nesting freeze and thaw requests is allowed
*
* A filesystem may hold multiple devices and thus a filesystems may
* have been frozen through the block layer via multiple block devices.
* The filesystem remains frozen until all block devices are unfrozen.
*/
int thaw_super(struct super_block *sb, enum freeze_holder who)
{
if (!super_lock_excl(sb)) { WARN_ON_ONCE("Dying superblock while thawing!");
return -EINVAL;
}
return thaw_super_locked(sb, who);
}
EXPORT_SYMBOL(thaw_super);
/*
* Create workqueue for deferred direct IO completions. We allocate the
* workqueue when it's first needed. This avoids creating workqueue for
* filesystems that don't need it and also allows us to create the workqueue
* late enough so the we can include s_id in the name of the workqueue.
*/
int sb_init_dio_done_wq(struct super_block *sb)
{
struct workqueue_struct *old;
struct workqueue_struct *wq = alloc_workqueue("dio/%s",
WQ_MEM_RECLAIM, 0,
sb->s_id);
if (!wq)
return -ENOMEM;
/*
* This has to be atomic as more DIOs can race to create the workqueue
*/
old = cmpxchg(&sb->s_dio_done_wq, NULL, wq);
/* Someone created workqueue before us? Free ours... */
if (old)
destroy_workqueue(wq);
return 0;
}
EXPORT_SYMBOL_GPL(sb_init_dio_done_wq);
// SPDX-License-Identifier: GPL-2.0+
/*
* Universal/legacy driver for 8250/16550-type serial ports
*
* Based on drivers/char/serial.c, by Linus Torvalds, Theodore Ts'o.
*
* Copyright (C) 2001 Russell King.
*
* Supports: ISA-compatible 8250/16550 ports
* PNP 8250/16550 ports
* early_serial_setup() ports
* userspace-configurable "phantom" ports
* "serial8250" platform devices
* serial8250_register_8250_port() ports
*/
#include <linux/acpi.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/ioport.h>
#include <linux/init.h>
#include <linux/console.h>
#include <linux/sysrq.h>
#include <linux/delay.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/tty.h>
#include <linux/ratelimit.h>
#include <linux/tty_flip.h>
#include <linux/serial.h>
#include <linux/serial_8250.h>
#include <linux/nmi.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/string_helpers.h>
#include <linux/uaccess.h>
#include <linux/io.h>
#ifdef CONFIG_SPARC
#include <linux/sunserialcore.h>
#endif
#include <asm/irq.h>
#include "8250.h"
/*
* Configuration:
* share_irqs - whether we pass IRQF_SHARED to request_irq(). This option
* is unsafe when used on edge-triggered interrupts.
*/
static unsigned int share_irqs = SERIAL8250_SHARE_IRQS;
static unsigned int nr_uarts = CONFIG_SERIAL_8250_RUNTIME_UARTS;
static struct uart_driver serial8250_reg;
static unsigned int skip_txen_test; /* force skip of txen test at init time */
#define PASS_LIMIT 512
#include <asm/serial.h>
/*
* SERIAL_PORT_DFNS tells us about built-in ports that have no
* standard enumeration mechanism. Platforms that can find all
* serial ports via mechanisms like ACPI or PCI need not supply it.
*/
#ifndef SERIAL_PORT_DFNS
#define SERIAL_PORT_DFNS
#endif
static const struct old_serial_port old_serial_port[] = {
SERIAL_PORT_DFNS /* defined in asm/serial.h */
};
#define UART_NR CONFIG_SERIAL_8250_NR_UARTS
#ifdef CONFIG_SERIAL_8250_RSA
#define PORT_RSA_MAX 4
static unsigned long probe_rsa[PORT_RSA_MAX];
static unsigned int probe_rsa_count;
#endif /* CONFIG_SERIAL_8250_RSA */
struct irq_info {
struct hlist_node node;
int irq;
spinlock_t lock; /* Protects list not the hash */
struct list_head *head;
};
#define NR_IRQ_HASH 32 /* Can be adjusted later */
static struct hlist_head irq_lists[NR_IRQ_HASH];
static DEFINE_MUTEX(hash_mutex); /* Used to walk the hash */
/*
* This is the serial driver's interrupt routine.
*
* Arjan thinks the old way was overly complex, so it got simplified.
* Alan disagrees, saying that need the complexity to handle the weird
* nature of ISA shared interrupts. (This is a special exception.)
*
* In order to handle ISA shared interrupts properly, we need to check
* that all ports have been serviced, and therefore the ISA interrupt
* line has been de-asserted.
*
* This means we need to loop through all ports. checking that they
* don't have an interrupt pending.
*/
static irqreturn_t serial8250_interrupt(int irq, void *dev_id)
{
struct irq_info *i = dev_id;
struct list_head *l, *end = NULL;
int pass_counter = 0, handled = 0;
pr_debug("%s(%d): start\n", __func__, irq);
spin_lock(&i->lock);
l = i->head;
do {
struct uart_8250_port *up;
struct uart_port *port;
up = list_entry(l, struct uart_8250_port, list);
port = &up->port;
if (port->handle_irq(port)) {
handled = 1;
end = NULL;
} else if (end == NULL)
end = l;
l = l->next;
if (l == i->head && pass_counter++ > PASS_LIMIT)
break;
} while (l != end);
spin_unlock(&i->lock);
pr_debug("%s(%d): end\n", __func__, irq);
return IRQ_RETVAL(handled);
}
/*
* To support ISA shared interrupts, we need to have one interrupt
* handler that ensures that the IRQ line has been deasserted
* before returning. Failing to do this will result in the IRQ
* line being stuck active, and, since ISA irqs are edge triggered,
* no more IRQs will be seen.
*/
static void serial_do_unlink(struct irq_info *i, struct uart_8250_port *up)
{
spin_lock_irq(&i->lock);
if (!list_empty(i->head)) {
if (i->head == &up->list)
i->head = i->head->next;
list_del(&up->list);
} else {
BUG_ON(i->head != &up->list);
i->head = NULL;
}
spin_unlock_irq(&i->lock);
/* List empty so throw away the hash node */
if (i->head == NULL) {
hlist_del(&i->node);
kfree(i);
}
}
static int serial_link_irq_chain(struct uart_8250_port *up)
{
struct hlist_head *h;
struct irq_info *i;
int ret;
mutex_lock(&hash_mutex);
h = &irq_lists[up->port.irq % NR_IRQ_HASH];
hlist_for_each_entry(i, h, node)
if (i->irq == up->port.irq)
break;
if (i == NULL) {
i = kzalloc(sizeof(struct irq_info), GFP_KERNEL);
if (i == NULL) {
mutex_unlock(&hash_mutex);
return -ENOMEM;
}
spin_lock_init(&i->lock);
i->irq = up->port.irq;
hlist_add_head(&i->node, h);
}
mutex_unlock(&hash_mutex);
spin_lock_irq(&i->lock);
if (i->head) {
list_add(&up->list, i->head);
spin_unlock_irq(&i->lock);
ret = 0;
} else {
INIT_LIST_HEAD(&up->list);
i->head = &up->list;
spin_unlock_irq(&i->lock);
ret = request_irq(up->port.irq, serial8250_interrupt,
up->port.irqflags, up->port.name, i);
if (ret < 0)
serial_do_unlink(i, up);
}
return ret;
}
static void serial_unlink_irq_chain(struct uart_8250_port *up)
{
struct irq_info *i;
struct hlist_head *h;
mutex_lock(&hash_mutex);
h = &irq_lists[up->port.irq % NR_IRQ_HASH];
hlist_for_each_entry(i, h, node)
if (i->irq == up->port.irq)
break;
BUG_ON(i == NULL);
BUG_ON(i->head == NULL);
if (list_empty(i->head))
free_irq(up->port.irq, i);
serial_do_unlink(i, up);
mutex_unlock(&hash_mutex);
}
/*
* This function is used to handle ports that do not have an
* interrupt. This doesn't work very well for 16450's, but gives
* barely passable results for a 16550A. (Although at the expense
* of much CPU overhead).
*/
static void serial8250_timeout(struct timer_list *t)
{
struct uart_8250_port *up = from_timer(up, t, timer);
up->port.handle_irq(&up->port);
mod_timer(&up->timer, jiffies + uart_poll_timeout(&up->port));
}
static void serial8250_backup_timeout(struct timer_list *t)
{
struct uart_8250_port *up = from_timer(up, t, timer);
unsigned int iir, ier = 0, lsr;
unsigned long flags;
uart_port_lock_irqsave(&up->port, &flags);
/*
* Must disable interrupts or else we risk racing with the interrupt
* based handler.
*/
if (up->port.irq) {
ier = serial_in(up, UART_IER);
serial_out(up, UART_IER, 0);
}
iir = serial_in(up, UART_IIR);
/*
* This should be a safe test for anyone who doesn't trust the
* IIR bits on their UART, but it's specifically designed for
* the "Diva" UART used on the management processor on many HP
* ia64 and parisc boxes.
*/
lsr = serial_lsr_in(up);
if ((iir & UART_IIR_NO_INT) && (up->ier & UART_IER_THRI) &&
(!uart_circ_empty(&up->port.state->xmit) || up->port.x_char) &&
(lsr & UART_LSR_THRE)) {
iir &= ~(UART_IIR_ID | UART_IIR_NO_INT);
iir |= UART_IIR_THRI;
}
if (!(iir & UART_IIR_NO_INT))
serial8250_tx_chars(up);
if (up->port.irq)
serial_out(up, UART_IER, ier);
uart_port_unlock_irqrestore(&up->port, flags);
/* Standard timer interval plus 0.2s to keep the port running */
mod_timer(&up->timer,
jiffies + uart_poll_timeout(&up->port) + HZ / 5);
}
static void univ8250_setup_timer(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
/*
* The above check will only give an accurate result the first time
* the port is opened so this value needs to be preserved.
*/
if (up->bugs & UART_BUG_THRE) {
pr_debug("%s - using backup timer\n", port->name);
up->timer.function = serial8250_backup_timeout;
mod_timer(&up->timer, jiffies +
uart_poll_timeout(port) + HZ / 5);
}
/*
* If the "interrupt" for this port doesn't correspond with any
* hardware interrupt, we use a timer-based system. The original
* driver used to do this with IRQ0.
*/
if (!port->irq)
mod_timer(&up->timer, jiffies + uart_poll_timeout(port));
}
static int univ8250_setup_irq(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
if (port->irq)
return serial_link_irq_chain(up);
return 0;
}
static void univ8250_release_irq(struct uart_8250_port *up)
{
struct uart_port *port = &up->port;
del_timer_sync(&up->timer);
up->timer.function = serial8250_timeout;
if (port->irq)
serial_unlink_irq_chain(up);
}
#ifdef CONFIG_SERIAL_8250_RSA
static int serial8250_request_rsa_resource(struct uart_8250_port *up)
{
unsigned long start = UART_RSA_BASE << up->port.regshift;
unsigned int size = 8 << up->port.regshift;
struct uart_port *port = &up->port;
int ret = -EINVAL;
switch (port->iotype) {
case UPIO_HUB6:
case UPIO_PORT:
start += port->iobase;
if (request_region(start, size, "serial-rsa"))
ret = 0;
else
ret = -EBUSY;
break;
}
return ret;
}
static void serial8250_release_rsa_resource(struct uart_8250_port *up)
{
unsigned long offset = UART_RSA_BASE << up->port.regshift;
unsigned int size = 8 << up->port.regshift;
struct uart_port *port = &up->port;
switch (port->iotype) {
case UPIO_HUB6:
case UPIO_PORT:
release_region(port->iobase + offset, size);
break;
}
}
#endif
static const struct uart_ops *base_ops;
static struct uart_ops univ8250_port_ops;
static const struct uart_8250_ops univ8250_driver_ops = {
.setup_irq = univ8250_setup_irq,
.release_irq = univ8250_release_irq,
.setup_timer = univ8250_setup_timer,
};
static struct uart_8250_port serial8250_ports[UART_NR];
/**
* serial8250_get_port - retrieve struct uart_8250_port
* @line: serial line number
*
* This function retrieves struct uart_8250_port for the specific line.
* This struct *must* *not* be used to perform a 8250 or serial core operation
* which is not accessible otherwise. Its only purpose is to make the struct
* accessible to the runtime-pm callbacks for context suspend/restore.
* The lock assumption made here is none because runtime-pm suspend/resume
* callbacks should not be invoked if there is any operation performed on the
* port.
*/
struct uart_8250_port *serial8250_get_port(int line)
{
return &serial8250_ports[line];
}
EXPORT_SYMBOL_GPL(serial8250_get_port);
static void (*serial8250_isa_config)(int port, struct uart_port *up,
u32 *capabilities);
void serial8250_set_isa_configurator(
void (*v)(int port, struct uart_port *up, u32 *capabilities))
{
serial8250_isa_config = v;
}
EXPORT_SYMBOL(serial8250_set_isa_configurator);
#ifdef CONFIG_SERIAL_8250_RSA
static void univ8250_config_port(struct uart_port *port, int flags)
{
struct uart_8250_port *up = up_to_u8250p(port);
up->probe &= ~UART_PROBE_RSA;
if (port->type == PORT_RSA) {
if (serial8250_request_rsa_resource(up) == 0)
up->probe |= UART_PROBE_RSA;
} else if (flags & UART_CONFIG_TYPE) {
int i;
for (i = 0; i < probe_rsa_count; i++) {
if (probe_rsa[i] == up->port.iobase) {
if (serial8250_request_rsa_resource(up) == 0)
up->probe |= UART_PROBE_RSA;
break;
}
}
}
base_ops->config_port(port, flags);
if (port->type != PORT_RSA && up->probe & UART_PROBE_RSA)
serial8250_release_rsa_resource(up);
}
static int univ8250_request_port(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
int ret;
ret = base_ops->request_port(port);
if (ret == 0 && port->type == PORT_RSA) {
ret = serial8250_request_rsa_resource(up);
if (ret < 0)
base_ops->release_port(port);
}
return ret;
}
static void univ8250_release_port(struct uart_port *port)
{
struct uart_8250_port *up = up_to_u8250p(port);
if (port->type == PORT_RSA)
serial8250_release_rsa_resource(up);
base_ops->release_port(port);
}
static void univ8250_rsa_support(struct uart_ops *ops)
{
ops->config_port = univ8250_config_port;
ops->request_port = univ8250_request_port;
ops->release_port = univ8250_release_port;
}
#else
#define univ8250_rsa_support(x) do { } while (0)
#endif /* CONFIG_SERIAL_8250_RSA */
static inline void serial8250_apply_quirks(struct uart_8250_port *up)
{
up->port.quirks |= skip_txen_test ? UPQ_NO_TXEN_TEST : 0;
}
static struct uart_8250_port *serial8250_setup_port(int index)
{
struct uart_8250_port *up;
if (index >= UART_NR)
return NULL;
up = &serial8250_ports[index];
up->port.line = index;
up->port.port_id = index;
serial8250_init_port(up);
if (!base_ops)
base_ops = up->port.ops;
up->port.ops = &univ8250_port_ops;
timer_setup(&up->timer, serial8250_timeout, 0);
up->ops = &univ8250_driver_ops;
if (IS_ENABLED(CONFIG_ALPHA_JENSEN) ||
(IS_ENABLED(CONFIG_ALPHA_GENERIC) && alpha_jensen()))
up->port.set_mctrl = alpha_jensen_set_mctrl;
serial8250_set_defaults(up);
return up;
}
static void __init serial8250_isa_init_ports(void)
{
struct uart_8250_port *up;
static int first = 1;
int i, irqflag = 0;
if (!first)
return;
first = 0;
if (nr_uarts > UART_NR)
nr_uarts = UART_NR;
/*
* Set up initial isa ports based on nr_uart module param, or else
* default to CONFIG_SERIAL_8250_RUNTIME_UARTS. Note that we do not
* need to increase nr_uarts when setting up the initial isa ports.
*/
for (i = 0; i < nr_uarts; i++)
serial8250_setup_port(i);
/* chain base port ops to support Remote Supervisor Adapter */
univ8250_port_ops = *base_ops;
univ8250_rsa_support(&univ8250_port_ops);
if (share_irqs)
irqflag = IRQF_SHARED;
for (i = 0, up = serial8250_ports;
i < ARRAY_SIZE(old_serial_port) && i < nr_uarts;
i++, up++) {
struct uart_port *port = &up->port;
port->iobase = old_serial_port[i].port;
port->irq = irq_canonicalize(old_serial_port[i].irq);
port->irqflags = 0;
port->uartclk = old_serial_port[i].baud_base * 16;
port->flags = old_serial_port[i].flags;
port->hub6 = 0;
port->membase = old_serial_port[i].iomem_base;
port->iotype = old_serial_port[i].io_type;
port->regshift = old_serial_port[i].iomem_reg_shift;
port->irqflags |= irqflag;
if (serial8250_isa_config != NULL)
serial8250_isa_config(i, &up->port, &up->capabilities);
}
}
static void __init
serial8250_register_ports(struct uart_driver *drv, struct device *dev)
{
int i;
for (i = 0; i < nr_uarts; i++) {
struct uart_8250_port *up = &serial8250_ports[i];
if (up->port.type == PORT_8250_CIR)
continue;
if (up->port.dev)
continue;
up->port.dev = dev;
if (uart_console_registered(&up->port))
pm_runtime_get_sync(up->port.dev);
serial8250_apply_quirks(up);
uart_add_one_port(drv, &up->port);
}
}
#ifdef CONFIG_SERIAL_8250_CONSOLE
static void univ8250_console_write(struct console *co, const char *s,
unsigned int count)
{
struct uart_8250_port *up = &serial8250_ports[co->index];
serial8250_console_write(up, s, count);
}
static int univ8250_console_setup(struct console *co, char *options)
{
struct uart_8250_port *up;
struct uart_port *port;
int retval, i;
/*
* Check whether an invalid uart number has been specified, and
* if so, search for the first available port that does have
* console support.
*/
if (co->index < 0 || co->index >= UART_NR)
co->index = 0;
/*
* If the console is past the initial isa ports, init more ports up to
* co->index as needed and increment nr_uarts accordingly.
*/
for (i = nr_uarts; i <= co->index; i++) {
up = serial8250_setup_port(i);
if (!up)
return -ENODEV;
nr_uarts++;
}
port = &serial8250_ports[co->index].port;
/* link port to console */
port->cons = co;
retval = serial8250_console_setup(port, options, false);
if (retval != 0)
port->cons = NULL;
return retval;
}
static int univ8250_console_exit(struct console *co)
{
struct uart_port *port;
port = &serial8250_ports[co->index].port;
return serial8250_console_exit(port);
}
/**
* univ8250_console_match - non-standard console matching
* @co: registering console
* @name: name from console command line
* @idx: index from console command line
* @options: ptr to option string from console command line
*
* Only attempts to match console command lines of the form:
* console=uart[8250],io|mmio|mmio16|mmio32,<addr>[,<options>]
* console=uart[8250],0x<addr>[,<options>]
* This form is used to register an initial earlycon boot console and
* replace it with the serial8250_console at 8250 driver init.
*
* Performs console setup for a match (as required by interface)
* If no <options> are specified, then assume the h/w is already setup.
*
* Returns 0 if console matches; otherwise non-zero to use default matching
*/
static int univ8250_console_match(struct console *co, char *name, int idx,
char *options)
{
char match[] = "uart"; /* 8250-specific earlycon name */
unsigned char iotype;
resource_size_t addr;
int i;
if (strncmp(name, match, 4) != 0)
return -ENODEV;
if (uart_parse_earlycon(options, &iotype, &addr, &options))
return -ENODEV;
/* try to match the port specified on the command line */
for (i = 0; i < nr_uarts; i++) {
struct uart_port *port = &serial8250_ports[i].port;
if (port->iotype != iotype)
continue;
if ((iotype == UPIO_MEM || iotype == UPIO_MEM16 ||
iotype == UPIO_MEM32 || iotype == UPIO_MEM32BE)
&& (port->mapbase != addr))
continue;
if (iotype == UPIO_PORT && port->iobase != addr)
continue;
co->index = i;
port->cons = co;
return serial8250_console_setup(port, options, true);
}
return -ENODEV;
}
static struct console univ8250_console = {
.name = "ttyS",
.write = univ8250_console_write,
.device = uart_console_device,
.setup = univ8250_console_setup,
.exit = univ8250_console_exit,
.match = univ8250_console_match,
.flags = CON_PRINTBUFFER | CON_ANYTIME,
.index = -1,
.data = &serial8250_reg,
};
static int __init univ8250_console_init(void)
{
if (nr_uarts == 0)
return -ENODEV;
serial8250_isa_init_ports();
register_console(&univ8250_console);
return 0;
}
console_initcall(univ8250_console_init);
#define SERIAL8250_CONSOLE (&univ8250_console)
#else
#define SERIAL8250_CONSOLE NULL
#endif
static struct uart_driver serial8250_reg = {
.owner = THIS_MODULE,
.driver_name = "serial",
.dev_name = "ttyS",
.major = TTY_MAJOR,
.minor = 64,
.cons = SERIAL8250_CONSOLE,
};
/*
* early_serial_setup - early registration for 8250 ports
*
* Setup an 8250 port structure prior to console initialisation. Use
* after console initialisation will cause undefined behaviour.
*/
int __init early_serial_setup(struct uart_port *port)
{
struct uart_port *p;
if (port->line >= ARRAY_SIZE(serial8250_ports) || nr_uarts == 0)
return -ENODEV;
serial8250_isa_init_ports();
p = &serial8250_ports[port->line].port;
p->iobase = port->iobase;
p->membase = port->membase;
p->irq = port->irq;
p->irqflags = port->irqflags;
p->uartclk = port->uartclk;
p->fifosize = port->fifosize;
p->regshift = port->regshift;
p->iotype = port->iotype;
p->flags = port->flags;
p->mapbase = port->mapbase;
p->mapsize = port->mapsize;
p->private_data = port->private_data;
p->type = port->type;
p->line = port->line;
serial8250_set_defaults(up_to_u8250p(p));
if (port->serial_in)
p->serial_in = port->serial_in;
if (port->serial_out)
p->serial_out = port->serial_out;
if (port->handle_irq)
p->handle_irq = port->handle_irq;
return 0;
}
/**
* serial8250_suspend_port - suspend one serial port
* @line: serial line number
*
* Suspend one serial port.
*/
void serial8250_suspend_port(int line)
{
struct uart_8250_port *up = &serial8250_ports[line];
struct uart_port *port = &up->port;
if (!console_suspend_enabled && uart_console(port) &&
port->type != PORT_8250) {
unsigned char canary = 0xa5;
serial_out(up, UART_SCR, canary);
if (serial_in(up, UART_SCR) == canary)
up->canary = canary;
}
uart_suspend_port(&serial8250_reg, port);
}
EXPORT_SYMBOL(serial8250_suspend_port);
/**
* serial8250_resume_port - resume one serial port
* @line: serial line number
*
* Resume one serial port.
*/
void serial8250_resume_port(int line)
{
struct uart_8250_port *up = &serial8250_ports[line];
struct uart_port *port = &up->port;
up->canary = 0;
if (up->capabilities & UART_NATSEMI) {
/* Ensure it's still in high speed mode */
serial_port_out(port, UART_LCR, 0xE0);
ns16550a_goto_highspeed(up);
serial_port_out(port, UART_LCR, 0);
port->uartclk = 921600*16;
}
uart_resume_port(&serial8250_reg, port);
}
EXPORT_SYMBOL(serial8250_resume_port);
/*
* Register a set of serial devices attached to a platform device. The
* list is terminated with a zero flags entry, which means we expect
* all entries to have at least UPF_BOOT_AUTOCONF set.
*/
static int serial8250_probe(struct platform_device *dev)
{
struct plat_serial8250_port *p = dev_get_platdata(&dev->dev);
struct uart_8250_port uart;
int ret, i, irqflag = 0;
memset(&uart, 0, sizeof(uart));
if (share_irqs)
irqflag = IRQF_SHARED;
for (i = 0; p && p->flags != 0; p++, i++) {
uart.port.iobase = p->iobase;
uart.port.membase = p->membase;
uart.port.irq = p->irq;
uart.port.irqflags = p->irqflags;
uart.port.uartclk = p->uartclk;
uart.port.regshift = p->regshift;
uart.port.iotype = p->iotype;
uart.port.flags = p->flags;
uart.port.mapbase = p->mapbase;
uart.port.mapsize = p->mapsize;
uart.port.hub6 = p->hub6;
uart.port.has_sysrq = p->has_sysrq;
uart.port.private_data = p->private_data;
uart.port.type = p->type;
uart.bugs = p->bugs;
uart.port.serial_in = p->serial_in;
uart.port.serial_out = p->serial_out;
uart.dl_read = p->dl_read;
uart.dl_write = p->dl_write;
uart.port.handle_irq = p->handle_irq;
uart.port.handle_break = p->handle_break;
uart.port.set_termios = p->set_termios;
uart.port.set_ldisc = p->set_ldisc;
uart.port.get_mctrl = p->get_mctrl;
uart.port.pm = p->pm;
uart.port.dev = &dev->dev;
uart.port.irqflags |= irqflag;
ret = serial8250_register_8250_port(&uart);
if (ret < 0) {
dev_err(&dev->dev, "unable to register port at index %d "
"(IO%lx MEM%llx IRQ%d): %d\n", i,
p->iobase, (unsigned long long)p->mapbase,
p->irq, ret);
}
}
return 0;
}
/*
* Remove serial ports registered against a platform device.
*/
static void serial8250_remove(struct platform_device *dev)
{
int i;
for (i = 0; i < nr_uarts; i++) {
struct uart_8250_port *up = &serial8250_ports[i];
if (up->port.dev == &dev->dev)
serial8250_unregister_port(i);
}
}
static int serial8250_suspend(struct platform_device *dev, pm_message_t state)
{
int i;
for (i = 0; i < UART_NR; i++) {
struct uart_8250_port *up = &serial8250_ports[i];
if (up->port.type != PORT_UNKNOWN && up->port.dev == &dev->dev)
uart_suspend_port(&serial8250_reg, &up->port);
}
return 0;
}
static int serial8250_resume(struct platform_device *dev)
{
int i;
for (i = 0; i < UART_NR; i++) {
struct uart_8250_port *up = &serial8250_ports[i];
if (up->port.type != PORT_UNKNOWN && up->port.dev == &dev->dev)
serial8250_resume_port(i);
}
return 0;
}
static struct platform_driver serial8250_isa_driver = {
.probe = serial8250_probe,
.remove_new = serial8250_remove,
.suspend = serial8250_suspend,
.resume = serial8250_resume,
.driver = {
.name = "serial8250",
},
};
/*
* This "device" covers _all_ ISA 8250-compatible serial devices listed
* in the table in include/asm/serial.h
*/
static struct platform_device *serial8250_isa_devs;
/*
* serial8250_register_8250_port and serial8250_unregister_port allows for
* 16x50 serial ports to be configured at run-time, to support PCMCIA
* modems and PCI multiport cards.
*/
static DEFINE_MUTEX(serial_mutex);
static struct uart_8250_port *serial8250_find_match_or_unused(const struct uart_port *port)
{
int i;
/*
* First, find a port entry which matches.
*/
for (i = 0; i < nr_uarts; i++)
if (uart_match_port(&serial8250_ports[i].port, port))
return &serial8250_ports[i];
/* try line number first if still available */
i = port->line;
if (i < nr_uarts && serial8250_ports[i].port.type == PORT_UNKNOWN &&
serial8250_ports[i].port.iobase == 0)
return &serial8250_ports[i];
/*
* We didn't find a matching entry, so look for the first
* free entry. We look for one which hasn't been previously
* used (indicated by zero iobase).
*/
for (i = 0; i < nr_uarts; i++)
if (serial8250_ports[i].port.type == PORT_UNKNOWN &&
serial8250_ports[i].port.iobase == 0)
return &serial8250_ports[i];
/*
* That also failed. Last resort is to find any entry which
* doesn't have a real port associated with it.
*/
for (i = 0; i < nr_uarts; i++)
if (serial8250_ports[i].port.type == PORT_UNKNOWN)
return &serial8250_ports[i];
return NULL;
}
static void serial_8250_overrun_backoff_work(struct work_struct *work)
{
struct uart_8250_port *up =
container_of(to_delayed_work(work), struct uart_8250_port,
overrun_backoff);
struct uart_port *port = &up->port;
unsigned long flags;
uart_port_lock_irqsave(port, &flags);
up->ier |= UART_IER_RLSI | UART_IER_RDI;
up->port.read_status_mask |= UART_LSR_DR;
serial_out(up, UART_IER, up->ier);
uart_port_unlock_irqrestore(port, flags);
}
/**
* serial8250_register_8250_port - register a serial port
* @up: serial port template
*
* Configure the serial port specified by the request. If the
* port exists and is in use, it is hung up and unregistered
* first.
*
* The port is then probed and if necessary the IRQ is autodetected
* If this fails an error is returned.
*
* On success the port is ready to use and the line number is returned.
*/
int serial8250_register_8250_port(const struct uart_8250_port *up)
{
struct uart_8250_port *uart;
int ret = -ENOSPC;
if (up->port.uartclk == 0)
return -EINVAL;
mutex_lock(&serial_mutex);
uart = serial8250_find_match_or_unused(&up->port);
if (!uart) {
/*
* If the port is past the initial isa ports, initialize a new
* port and increment nr_uarts accordingly.
*/
uart = serial8250_setup_port(nr_uarts);
if (!uart)
goto unlock;
nr_uarts++;
}
if (uart->port.type != PORT_8250_CIR) {
struct mctrl_gpios *gpios;
if (uart->port.dev)
uart_remove_one_port(&serial8250_reg, &uart->port);
uart->port.ctrl_id = up->port.ctrl_id;
uart->port.port_id = up->port.port_id;
uart->port.iobase = up->port.iobase;
uart->port.membase = up->port.membase;
uart->port.irq = up->port.irq;
uart->port.irqflags = up->port.irqflags;
uart->port.uartclk = up->port.uartclk;
uart->port.fifosize = up->port.fifosize;
uart->port.regshift = up->port.regshift;
uart->port.iotype = up->port.iotype;
uart->port.flags = up->port.flags | UPF_BOOT_AUTOCONF;
uart->bugs = up->bugs;
uart->port.mapbase = up->port.mapbase;
uart->port.mapsize = up->port.mapsize;
uart->port.private_data = up->port.private_data;
uart->tx_loadsz = up->tx_loadsz;
uart->capabilities = up->capabilities;
uart->port.throttle = up->port.throttle;
uart->port.unthrottle = up->port.unthrottle;
uart->port.rs485_config = up->port.rs485_config;
uart->port.rs485_supported = up->port.rs485_supported;
uart->port.rs485 = up->port.rs485;
uart->rs485_start_tx = up->rs485_start_tx;
uart->rs485_stop_tx = up->rs485_stop_tx;
uart->lsr_save_mask = up->lsr_save_mask;
uart->dma = up->dma;
/* Take tx_loadsz from fifosize if it wasn't set separately */
if (uart->port.fifosize && !uart->tx_loadsz)
uart->tx_loadsz = uart->port.fifosize;
if (up->port.dev) {
uart->port.dev = up->port.dev;
ret = uart_get_rs485_mode(&uart->port);
if (ret)
goto err;
}
if (up->port.flags & UPF_FIXED_TYPE)
uart->port.type = up->port.type;
/*
* Only call mctrl_gpio_init(), if the device has no ACPI
* companion device
*/
if (!has_acpi_companion(uart->port.dev)) {
gpios = mctrl_gpio_init(&uart->port, 0);
if (IS_ERR(gpios)) {
ret = PTR_ERR(gpios);
goto err;
} else {
uart->gpios = gpios;
}
}
serial8250_set_defaults(uart);
/* Possibly override default I/O functions. */
if (up->port.serial_in)
uart->port.serial_in = up->port.serial_in;
if (up->port.serial_out)
uart->port.serial_out = up->port.serial_out;
if (up->port.handle_irq)
uart->port.handle_irq = up->port.handle_irq;
/* Possibly override set_termios call */
if (up->port.set_termios)
uart->port.set_termios = up->port.set_termios;
if (up->port.set_ldisc)
uart->port.set_ldisc = up->port.set_ldisc;
if (up->port.get_mctrl)
uart->port.get_mctrl = up->port.get_mctrl;
if (up->port.set_mctrl)
uart->port.set_mctrl = up->port.set_mctrl;
if (up->port.get_divisor)
uart->port.get_divisor = up->port.get_divisor;
if (up->port.set_divisor)
uart->port.set_divisor = up->port.set_divisor;
if (up->port.startup)
uart->port.startup = up->port.startup;
if (up->port.shutdown)
uart->port.shutdown = up->port.shutdown;
if (up->port.pm)
uart->port.pm = up->port.pm;
if (up->port.handle_break)
uart->port.handle_break = up->port.handle_break;
if (up->dl_read)
uart->dl_read = up->dl_read;
if (up->dl_write)
uart->dl_write = up->dl_write;
if (uart->port.type != PORT_8250_CIR) {
if (serial8250_isa_config != NULL)
serial8250_isa_config(0, &uart->port,
&uart->capabilities);
serial8250_apply_quirks(uart);
ret = uart_add_one_port(&serial8250_reg,
&uart->port);
if (ret)
goto err;
ret = uart->port.line;
} else {
dev_info(uart->port.dev,
"skipping CIR port at 0x%lx / 0x%llx, IRQ %d\n",
uart->port.iobase,
(unsigned long long)uart->port.mapbase,
uart->port.irq);
ret = 0;
}
if (!uart->lsr_save_mask)
uart->lsr_save_mask = LSR_SAVE_FLAGS; /* Use default LSR mask */
/* Initialise interrupt backoff work if required */
if (up->overrun_backoff_time_ms > 0) {
uart->overrun_backoff_time_ms =
up->overrun_backoff_time_ms;
INIT_DELAYED_WORK(&uart->overrun_backoff,
serial_8250_overrun_backoff_work);
} else {
uart->overrun_backoff_time_ms = 0;
}
}
unlock:
mutex_unlock(&serial_mutex);
return ret;
err:
uart->port.dev = NULL;
mutex_unlock(&serial_mutex);
return ret;
}
EXPORT_SYMBOL(serial8250_register_8250_port);
/**
* serial8250_unregister_port - remove a 16x50 serial port at runtime
* @line: serial line number
*
* Remove one serial port. This may not be called from interrupt
* context. We hand the port back to the our control.
*/
void serial8250_unregister_port(int line)
{
struct uart_8250_port *uart = &serial8250_ports[line];
mutex_lock(&serial_mutex);
if (uart->em485) {
unsigned long flags;
uart_port_lock_irqsave(&uart->port, &flags);
serial8250_em485_destroy(uart);
uart_port_unlock_irqrestore(&uart->port, flags);
}
uart_remove_one_port(&serial8250_reg, &uart->port);
if (serial8250_isa_devs) {
uart->port.flags &= ~UPF_BOOT_AUTOCONF;
uart->port.type = PORT_UNKNOWN;
uart->port.dev = &serial8250_isa_devs->dev;
uart->port.port_id = line;
uart->capabilities = 0;
serial8250_init_port(uart);
serial8250_apply_quirks(uart);
uart_add_one_port(&serial8250_reg, &uart->port);
} else {
uart->port.dev = NULL;
}
mutex_unlock(&serial_mutex);
}
EXPORT_SYMBOL(serial8250_unregister_port);
static int __init serial8250_init(void)
{
int ret;
if (nr_uarts == 0)
return -ENODEV;
serial8250_isa_init_ports();
pr_info("Serial: 8250/16550 driver, %d ports, IRQ sharing %s\n",
nr_uarts, str_enabled_disabled(share_irqs));
#ifdef CONFIG_SPARC
ret = sunserial_register_minors(&serial8250_reg, UART_NR);
#else
serial8250_reg.nr = UART_NR;
ret = uart_register_driver(&serial8250_reg);
#endif
if (ret)
goto out;
ret = serial8250_pnp_init();
if (ret)
goto unreg_uart_drv;
serial8250_isa_devs = platform_device_alloc("serial8250",
PLAT8250_DEV_LEGACY);
if (!serial8250_isa_devs) {
ret = -ENOMEM;
goto unreg_pnp;
}
ret = platform_device_add(serial8250_isa_devs);
if (ret)
goto put_dev;
serial8250_register_ports(&serial8250_reg, &serial8250_isa_devs->dev);
ret = platform_driver_register(&serial8250_isa_driver);
if (ret == 0)
goto out;
platform_device_del(serial8250_isa_devs);
put_dev:
platform_device_put(serial8250_isa_devs);
unreg_pnp:
serial8250_pnp_exit();
unreg_uart_drv:
#ifdef CONFIG_SPARC
sunserial_unregister_minors(&serial8250_reg, UART_NR);
#else
uart_unregister_driver(&serial8250_reg);
#endif
out:
return ret;
}
static void __exit serial8250_exit(void)
{
struct platform_device *isa_dev = serial8250_isa_devs;
/*
* This tells serial8250_unregister_port() not to re-register
* the ports (thereby making serial8250_isa_driver permanently
* in use.)
*/
serial8250_isa_devs = NULL;
platform_driver_unregister(&serial8250_isa_driver);
platform_device_unregister(isa_dev);
serial8250_pnp_exit();
#ifdef CONFIG_SPARC
sunserial_unregister_minors(&serial8250_reg, UART_NR);
#else
uart_unregister_driver(&serial8250_reg);
#endif
}
module_init(serial8250_init);
module_exit(serial8250_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Generic 8250/16x50 serial driver");
module_param_hw(share_irqs, uint, other, 0644);
MODULE_PARM_DESC(share_irqs, "Share IRQs with other non-8250/16x50 devices (unsafe)");
module_param(nr_uarts, uint, 0644);
MODULE_PARM_DESC(nr_uarts, "Maximum number of UARTs supported. (1-" __MODULE_STRING(CONFIG_SERIAL_8250_NR_UARTS) ")");
module_param(skip_txen_test, uint, 0644);
MODULE_PARM_DESC(skip_txen_test, "Skip checking for the TXEN bug at init time");
#ifdef CONFIG_SERIAL_8250_RSA
module_param_hw_array(probe_rsa, ulong, ioport, &probe_rsa_count, 0444);
MODULE_PARM_DESC(probe_rsa, "Probe I/O ports for RSA");
#endif
MODULE_ALIAS_CHARDEV_MAJOR(TTY_MAJOR);
#ifdef CONFIG_SERIAL_8250_DEPRECATED_OPTIONS
#ifndef MODULE
/* This module was renamed to 8250_core in 3.7. Keep the old "8250" name
* working as well for the module options so we don't break people. We
* need to keep the names identical and the convenient macros will happily
* refuse to let us do that by failing the build with redefinition errors
* of global variables. So we stick them inside a dummy function to avoid
* those conflicts. The options still get parsed, and the redefined
* MODULE_PARAM_PREFIX lets us keep the "8250." syntax alive.
*
* This is hacky. I'm sorry.
*/
static void __used s8250_options(void)
{
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "8250_core."
module_param_cb(share_irqs, ¶m_ops_uint, &share_irqs, 0644);
module_param_cb(nr_uarts, ¶m_ops_uint, &nr_uarts, 0644);
module_param_cb(skip_txen_test, ¶m_ops_uint, &skip_txen_test, 0644);
#ifdef CONFIG_SERIAL_8250_RSA
__module_param_call(MODULE_PARAM_PREFIX, probe_rsa,
¶m_array_ops, .arr = &__param_arr_probe_rsa,
0444, -1, 0);
#endif
}
#else
MODULE_ALIAS("8250_core");
#endif
#endif
/* SPDX-License-Identifier: GPL-2.0+ */
/*
* Read-Copy Update mechanism for mutual exclusion
*
* Copyright IBM Corporation, 2001
*
* Author: Dipankar Sarma <dipankar@in.ibm.com>
*
* Based on the original work by Paul McKenney <paulmck@vnet.ibm.com>
* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
* Papers:
* http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
* http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
*
* For detailed explanation of Read-Copy Update mechanism see -
* http://lse.sourceforge.net/locking/rcupdate.html
*
*/
#ifndef __LINUX_RCUPDATE_H
#define __LINUX_RCUPDATE_H
#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/atomic.h>
#include <linux/irqflags.h>
#include <linux/preempt.h>
#include <linux/bottom_half.h>
#include <linux/lockdep.h>
#include <linux/cleanup.h>
#include <asm/processor.h>
#include <linux/cpumask.h>
#include <linux/context_tracking_irq.h>
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
/* Exported common interfaces */
void call_rcu(struct rcu_head *head, rcu_callback_t func);
void rcu_barrier_tasks(void);
void rcu_barrier_tasks_rude(void);
void synchronize_rcu(void);
struct rcu_gp_oldstate;
unsigned long get_completed_synchronize_rcu(void);
void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
// Maximum number of unsigned long values corresponding to
// not-yet-completed RCU grace periods.
#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2
/**
* same_state_synchronize_rcu - Are two old-state values identical?
* @oldstate1: First old-state value.
* @oldstate2: Second old-state value.
*
* The two old-state values must have been obtained from either
* get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or
* get_completed_synchronize_rcu(). Returns @true if the two values are
* identical and @false otherwise. This allows structures whose lifetimes
* are tracked by old-state values to push these values to a list header,
* allowing those structures to be slightly smaller.
*/
static inline bool same_state_synchronize_rcu(unsigned long oldstate1, unsigned long oldstate2)
{
return oldstate1 == oldstate2;
}
#ifdef CONFIG_PREEMPT_RCU
void __rcu_read_lock(void);
void __rcu_read_unlock(void);
/*
* Defined as a macro as it is a very low level header included from
* areas that don't even know about current. This gives the rcu_read_lock()
* nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
* types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
*/
#define rcu_preempt_depth() READ_ONCE(current->rcu_read_lock_nesting)
#else /* #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_TINY_RCU
#define rcu_read_unlock_strict() do { } while (0)
#else
void rcu_read_unlock_strict(void);
#endif
static inline void __rcu_read_lock(void)
{
preempt_disable();
}
static inline void __rcu_read_unlock(void)
{
preempt_enable();
if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
rcu_read_unlock_strict();
}
static inline int rcu_preempt_depth(void)
{
return 0;
}
#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_RCU_LAZY
void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func);
#else
static inline void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func)
{
call_rcu(head, func);
}
#endif
/* Internal to kernel */
void rcu_init(void);
extern int rcu_scheduler_active;
void rcu_sched_clock_irq(int user);
#ifdef CONFIG_TASKS_RCU_GENERIC
void rcu_init_tasks_generic(void);
#else
static inline void rcu_init_tasks_generic(void) { }
#endif
#ifdef CONFIG_RCU_STALL_COMMON
void rcu_sysrq_start(void);
void rcu_sysrq_end(void);
#else /* #ifdef CONFIG_RCU_STALL_COMMON */
static inline void rcu_sysrq_start(void) { }
static inline void rcu_sysrq_end(void) { }
#endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */
#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK))
void rcu_irq_work_resched(void);
#else
static inline void rcu_irq_work_resched(void) { }
#endif
#ifdef CONFIG_RCU_NOCB_CPU
void rcu_init_nohz(void);
int rcu_nocb_cpu_offload(int cpu);
int rcu_nocb_cpu_deoffload(int cpu);
void rcu_nocb_flush_deferred_wakeup(void);
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
static inline void rcu_init_nohz(void) { }
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
/*
* Note a quasi-voluntary context switch for RCU-tasks's benefit.
* This is a macro rather than an inline function to avoid #include hell.
*/
#ifdef CONFIG_TASKS_RCU_GENERIC
# ifdef CONFIG_TASKS_RCU
# define rcu_tasks_classic_qs(t, preempt) \
do { \
if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
WRITE_ONCE((t)->rcu_tasks_holdout, false); \
} while (0)
void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks(void);
# else
# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
# define call_rcu_tasks call_rcu
# define synchronize_rcu_tasks synchronize_rcu
# endif
# ifdef CONFIG_TASKS_TRACE_RCU
// Bits for ->trc_reader_special.b.need_qs field.
#define TRC_NEED_QS 0x1 // Task needs a quiescent state.
#define TRC_NEED_QS_CHECKED 0x2 // Task has been checked for needing quiescent state.
u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new);
void rcu_tasks_trace_qs_blkd(struct task_struct *t);
# define rcu_tasks_trace_qs(t) \
do { \
int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting); \
\
if (unlikely(READ_ONCE((t)->trc_reader_special.b.need_qs) == TRC_NEED_QS) && \
likely(!___rttq_nesting)) { \
rcu_trc_cmpxchg_need_qs((t), TRC_NEED_QS, TRC_NEED_QS_CHECKED); \
} else if (___rttq_nesting && ___rttq_nesting != INT_MIN && \
!READ_ONCE((t)->trc_reader_special.b.blocked)) { \
rcu_tasks_trace_qs_blkd(t); \
} \
} while (0)
# else
# define rcu_tasks_trace_qs(t) do { } while (0)
# endif
#define rcu_tasks_qs(t, preempt) \
do { \
rcu_tasks_classic_qs((t), (preempt)); \
rcu_tasks_trace_qs(t); \
} while (0)
# ifdef CONFIG_TASKS_RUDE_RCU
void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks_rude(void);
# endif
#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
void exit_tasks_rcu_start(void);
void exit_tasks_rcu_stop(void);
void exit_tasks_rcu_finish(void);
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
#define rcu_tasks_classic_qs(t, preempt) do { } while (0)
#define rcu_tasks_qs(t, preempt) do { } while (0)
#define rcu_note_voluntary_context_switch(t) do { } while (0)
#define call_rcu_tasks call_rcu
#define synchronize_rcu_tasks synchronize_rcu
static inline void exit_tasks_rcu_start(void) { }
static inline void exit_tasks_rcu_stop(void) { }
static inline void exit_tasks_rcu_finish(void) { }
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
/**
* rcu_trace_implies_rcu_gp - does an RCU Tasks Trace grace period imply an RCU grace period?
*
* As an accident of implementation, an RCU Tasks Trace grace period also
* acts as an RCU grace period. However, this could change at any time.
* Code relying on this accident must call this function to verify that
* this accident is still happening.
*
* You have been warned!
*/
static inline bool rcu_trace_implies_rcu_gp(void) { return true; }
/**
* cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
*
* This macro resembles cond_resched(), except that it is defined to
* report potential quiescent states to RCU-tasks even if the cond_resched()
* machinery were to be shut off, as some advocate for PREEMPTION kernels.
*/
#define cond_resched_tasks_rcu_qs() \
do { \
rcu_tasks_qs(current, false); \
cond_resched(); \
} while (0)
/**
* rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states
* @old_ts: jiffies at start of processing.
*
* This helper is for long-running softirq handlers, such as NAPI threads in
* networking. The caller should initialize the variable passed in as @old_ts
* at the beginning of the softirq handler. When invoked frequently, this macro
* will invoke rcu_softirq_qs() every 100 milliseconds thereafter, which will
* provide both RCU and RCU-Tasks quiescent states. Note that this macro
* modifies its old_ts argument.
*
* Because regions of code that have disabled softirq act as RCU read-side
* critical sections, this macro should be invoked with softirq (and
* preemption) enabled.
*
* The macro is not needed when CONFIG_PREEMPT_RT is defined. RT kernels would
* have more chance to invoke schedule() calls and provide necessary quiescent
* states. As a contrast, calling cond_resched() only won't achieve the same
* effect because cond_resched() does not provide RCU-Tasks quiescent states.
*/
#define rcu_softirq_qs_periodic(old_ts) \
do { \
if (!IS_ENABLED(CONFIG_PREEMPT_RT) && \
time_after(jiffies, (old_ts) + HZ / 10)) { \
preempt_disable(); \
rcu_softirq_qs(); \
preempt_enable(); \
(old_ts) = jiffies; \
} \
} while (0)
/*
* Infrastructure to implement the synchronize_() primitives in
* TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
*/
#if defined(CONFIG_TREE_RCU)
#include <linux/rcutree.h>
#elif defined(CONFIG_TINY_RCU)
#include <linux/rcutiny.h>
#else
#error "Unknown RCU implementation specified to kernel configuration"
#endif
/*
* The init_rcu_head_on_stack() and destroy_rcu_head_on_stack() calls
* are needed for dynamic initialization and destruction of rcu_head
* on the stack, and init_rcu_head()/destroy_rcu_head() are needed for
* dynamic initialization and destruction of statically allocated rcu_head
* structures. However, rcu_head structures allocated dynamically in the
* heap don't need any initialization.
*/
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
void init_rcu_head(struct rcu_head *head);
void destroy_rcu_head(struct rcu_head *head);
void init_rcu_head_on_stack(struct rcu_head *head);
void destroy_rcu_head_on_stack(struct rcu_head *head);
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
static inline void init_rcu_head(struct rcu_head *head) { }
static inline void destroy_rcu_head(struct rcu_head *head) { }
static inline void init_rcu_head_on_stack(struct rcu_head *head) { }
static inline void destroy_rcu_head_on_stack(struct rcu_head *head) { }
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
bool rcu_lockdep_current_cpu_online(void);
#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
static inline bool rcu_lockdep_current_cpu_online(void) { return true; }
#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
extern struct lockdep_map rcu_lock_map;
extern struct lockdep_map rcu_bh_lock_map;
extern struct lockdep_map rcu_sched_lock_map;
extern struct lockdep_map rcu_callback_map;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
static inline void rcu_lock_acquire(struct lockdep_map *map)
{
lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
}
static inline void rcu_try_lock_acquire(struct lockdep_map *map)
{
lock_acquire(map, 0, 1, 2, 0, NULL, _THIS_IP_);
}
static inline void rcu_lock_release(struct lockdep_map *map)
{
lock_release(map, _THIS_IP_);
}
int debug_lockdep_rcu_enabled(void);
int rcu_read_lock_held(void);
int rcu_read_lock_bh_held(void);
int rcu_read_lock_sched_held(void);
int rcu_read_lock_any_held(void);
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
# define rcu_lock_acquire(a) do { } while (0)
# define rcu_try_lock_acquire(a) do { } while (0)
# define rcu_lock_release(a) do { } while (0)
static inline int rcu_read_lock_held(void)
{
return 1;
}
static inline int rcu_read_lock_bh_held(void)
{
return 1;
}
static inline int rcu_read_lock_sched_held(void)
{
return !preemptible();
}
static inline int rcu_read_lock_any_held(void)
{
return !preemptible();
}
static inline int debug_lockdep_rcu_enabled(void)
{
return 0;
}
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
#ifdef CONFIG_PROVE_RCU
/**
* RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
* @c: condition to check
* @s: informative message
*
* This checks debug_lockdep_rcu_enabled() before checking (c) to
* prevent early boot splats due to lockdep not yet being initialized,
* and rechecks it after checking (c) to prevent false-positive splats
* due to races with lockdep being disabled. See commit 3066820034b5dd
* ("rcu: Reject RCU_LOCKDEP_WARN() false positives") for more detail.
*/
#define RCU_LOCKDEP_WARN(c, s) \
do { \
static bool __section(".data.unlikely") __warned; \
if (debug_lockdep_rcu_enabled() && (c) && \
debug_lockdep_rcu_enabled() && !__warned) { \
__warned = true; \
lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
} \
} while (0)
#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
static inline void rcu_preempt_sleep_check(void)
{
RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
"Illegal context switch in RCU read-side critical section");
}
#else /* #ifdef CONFIG_PROVE_RCU */
static inline void rcu_preempt_sleep_check(void) { }
#endif /* #else #ifdef CONFIG_PROVE_RCU */
#define rcu_sleep_check() \
do { \
rcu_preempt_sleep_check(); \
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \
RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map), \
"Illegal context switch in RCU-bh read-side critical section"); \
RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map), \
"Illegal context switch in RCU-sched read-side critical section"); \
} while (0)
#else /* #ifdef CONFIG_PROVE_RCU */
#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))
#define rcu_sleep_check() do { } while (0)
#endif /* #else #ifdef CONFIG_PROVE_RCU */
/*
* Helper functions for rcu_dereference_check(), rcu_dereference_protected()
* and rcu_assign_pointer(). Some of these could be folded into their
* callers, but they are left separate in order to ease introduction of
* multiple pointers markings to match different RCU implementations
* (e.g., __srcu), should this make sense in the future.
*/
#ifdef __CHECKER__
#define rcu_check_sparse(p, space) \
((void)(((typeof(*p) space *)p) == p))
#else /* #ifdef __CHECKER__ */
#define rcu_check_sparse(p, space)
#endif /* #else #ifdef __CHECKER__ */
#define __unrcu_pointer(p, local) \
({ \
typeof(*p) *local = (typeof(*p) *__force)(p); \
rcu_check_sparse(p, __rcu); \
((typeof(*p) __force __kernel *)(local)); \
})
/**
* unrcu_pointer - mark a pointer as not being RCU protected
* @p: pointer needing to lose its __rcu property
*
* Converts @p from an __rcu pointer to a __kernel pointer.
* This allows an __rcu pointer to be used with xchg() and friends.
*/
#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu))
#define __rcu_access_pointer(p, local, space) \
({ \
typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
rcu_check_sparse(p, space); \
((typeof(*p) __force __kernel *)(local)); \
})
#define __rcu_dereference_check(p, local, c, space) \
({ \
/* Dependency order vs. p above. */ \
typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
rcu_check_sparse(p, space); \
((typeof(*p) __force __kernel *)(local)); \
})
#define __rcu_dereference_protected(p, local, c, space) \
({ \
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
rcu_check_sparse(p, space); \
((typeof(*p) __force __kernel *)(p)); \
})
#define __rcu_dereference_raw(p, local) \
({ \
/* Dependency order vs. p above. */ \
typeof(p) local = READ_ONCE(p); \
((typeof(*p) __force __kernel *)(local)); \
})
#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu))
/**
* RCU_INITIALIZER() - statically initialize an RCU-protected global variable
* @v: The value to statically initialize with.
*/
#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
/**
* rcu_assign_pointer() - assign to RCU-protected pointer
* @p: pointer to assign to
* @v: value to assign (publish)
*
* Assigns the specified value to the specified RCU-protected
* pointer, ensuring that any concurrent RCU readers will see
* any prior initialization.
*
* Inserts memory barriers on architectures that require them
* (which is most of them), and also prevents the compiler from
* reordering the code that initializes the structure after the pointer
* assignment. More importantly, this call documents which pointers
* will be dereferenced by RCU read-side code.
*
* In some special cases, you may use RCU_INIT_POINTER() instead
* of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
* to the fact that it does not constrain either the CPU or the compiler.
* That said, using RCU_INIT_POINTER() when you should have used
* rcu_assign_pointer() is a very bad thing that results in
* impossible-to-diagnose memory corruption. So please be careful.
* See the RCU_INIT_POINTER() comment header for details.
*
* Note that rcu_assign_pointer() evaluates each of its arguments only
* once, appearances notwithstanding. One of the "extra" evaluations
* is in typeof() and the other visible only to sparse (__CHECKER__),
* neither of which actually execute the argument. As with most cpp
* macros, this execute-arguments-only-once property is important, so
* please be careful when making changes to rcu_assign_pointer() and the
* other macros that it invokes.
*/
#define rcu_assign_pointer(p, v) \
do { \
uintptr_t _r_a_p__v = (uintptr_t)(v); \
rcu_check_sparse(p, __rcu); \
\
if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \
WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \
else \
smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
} while (0)
/**
* rcu_replace_pointer() - replace an RCU pointer, returning its old value
* @rcu_ptr: RCU pointer, whose old value is returned
* @ptr: regular pointer
* @c: the lockdep conditions under which the dereference will take place
*
* Perform a replacement, where @rcu_ptr is an RCU-annotated
* pointer and @c is the lockdep argument that is passed to the
* rcu_dereference_protected() call used to read that pointer. The old
* value of @rcu_ptr is returned, and @rcu_ptr is set to @ptr.
*/
#define rcu_replace_pointer(rcu_ptr, ptr, c) \
({ \
typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c)); \
rcu_assign_pointer((rcu_ptr), (ptr)); \
__tmp; \
})
/**
* rcu_access_pointer() - fetch RCU pointer with no dereferencing
* @p: The pointer to read
*
* Return the value of the specified RCU-protected pointer, but omit the
* lockdep checks for being in an RCU read-side critical section. This is
* useful when the value of this pointer is accessed, but the pointer is
* not dereferenced, for example, when testing an RCU-protected pointer
* against NULL. Although rcu_access_pointer() may also be used in cases
* where update-side locks prevent the value of the pointer from changing,
* you should instead use rcu_dereference_protected() for this use case.
* Within an RCU read-side critical section, there is little reason to
* use rcu_access_pointer().
*
* It is usually best to test the rcu_access_pointer() return value
* directly in order to avoid accidental dereferences being introduced
* by later inattentive changes. In other words, assigning the
* rcu_access_pointer() return value to a local variable results in an
* accident waiting to happen.
*
* It is also permissible to use rcu_access_pointer() when read-side
* access to the pointer was removed at least one grace period ago, as is
* the case in the context of the RCU callback that is freeing up the data,
* or after a synchronize_rcu() returns. This can be useful when tearing
* down multi-linked structures after a grace period has elapsed. However,
* rcu_dereference_protected() is normally preferred for this use case.
*/
#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)
/**
* rcu_dereference_check() - rcu_dereference with debug checking
* @p: The pointer to read, prior to dereferencing
* @c: The conditions under which the dereference will take place
*
* Do an rcu_dereference(), but check that the conditions under which the
* dereference will take place are correct. Typically the conditions
* indicate the various locking conditions that should be held at that
* point. The check should return true if the conditions are satisfied.
* An implicit check for being in an RCU read-side critical section
* (rcu_read_lock()) is included.
*
* For example:
*
* bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
*
* could be used to indicate to lockdep that foo->bar may only be dereferenced
* if either rcu_read_lock() is held, or that the lock required to replace
* the bar struct at foo->bar is held.
*
* Note that the list of conditions may also include indications of when a lock
* need not be held, for example during initialisation or destruction of the
* target struct:
*
* bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
* atomic_read(&foo->usage) == 0);
*
* Inserts memory barriers on architectures that require them
* (currently only the Alpha), prevents the compiler from refetching
* (and from merging fetches), and, more importantly, documents exactly
* which pointers are protected by RCU and checks that the pointer is
* annotated as __rcu.
*/
#define rcu_dereference_check(p, c) \
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
(c) || rcu_read_lock_held(), __rcu)
/**
* rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
* @p: The pointer to read, prior to dereferencing
* @c: The conditions under which the dereference will take place
*
* This is the RCU-bh counterpart to rcu_dereference_check(). However,
* please note that starting in v5.0 kernels, vanilla RCU grace periods
* wait for local_bh_disable() regions of code in addition to regions of
* code demarked by rcu_read_lock() and rcu_read_unlock(). This means
* that synchronize_rcu(), call_rcu, and friends all take not only
* rcu_read_lock() but also rcu_read_lock_bh() into account.
*/
#define rcu_dereference_bh_check(p, c) \
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
(c) || rcu_read_lock_bh_held(), __rcu)
/**
* rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
* @p: The pointer to read, prior to dereferencing
* @c: The conditions under which the dereference will take place
*
* This is the RCU-sched counterpart to rcu_dereference_check().
* However, please note that starting in v5.0 kernels, vanilla RCU grace
* periods wait for preempt_disable() regions of code in addition to
* regions of code demarked by rcu_read_lock() and rcu_read_unlock().
* This means that synchronize_rcu(), call_rcu, and friends all take not
* only rcu_read_lock() but also rcu_read_lock_sched() into account.
*/
#define rcu_dereference_sched_check(p, c) \
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
(c) || rcu_read_lock_sched_held(), \
__rcu)
/*
* The tracing infrastructure traces RCU (we want that), but unfortunately
* some of the RCU checks causes tracing to lock up the system.
*
* The no-tracing version of rcu_dereference_raw() must not call
* rcu_read_lock_held().
*/
#define rcu_dereference_raw_check(p) \
__rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu)
/**
* rcu_dereference_protected() - fetch RCU pointer when updates prevented
* @p: The pointer to read, prior to dereferencing
* @c: The conditions under which the dereference will take place
*
* Return the value of the specified RCU-protected pointer, but omit
* the READ_ONCE(). This is useful in cases where update-side locks
* prevent the value of the pointer from changing. Please note that this
* primitive does *not* prevent the compiler from repeating this reference
* or combining it with other references, so it should not be used without
* protection of appropriate locks.
*
* This function is only for update-side use. Using this function
* when protected only by rcu_read_lock() will result in infrequent
* but very ugly failures.
*/
#define rcu_dereference_protected(p, c) \
__rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu)
/**
* rcu_dereference() - fetch RCU-protected pointer for dereferencing
* @p: The pointer to read, prior to dereferencing
*
* This is a simple wrapper around rcu_dereference_check().
*/
#define rcu_dereference(p) rcu_dereference_check(p, 0)
/**
* rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
* @p: The pointer to read, prior to dereferencing
*
* Makes rcu_dereference_check() do the dirty work.
*/
#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
/**
* rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
* @p: The pointer to read, prior to dereferencing
*
* Makes rcu_dereference_check() do the dirty work.
*/
#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
/**
* rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
* @p: The pointer to hand off
*
* This is simply an identity function, but it documents where a pointer
* is handed off from RCU to some other synchronization mechanism, for
* example, reference counting or locking. In C11, it would map to
* kill_dependency(). It could be used as follows::
*
* rcu_read_lock();
* p = rcu_dereference(gp);
* long_lived = is_long_lived(p);
* if (long_lived) {
* if (!atomic_inc_not_zero(p->refcnt))
* long_lived = false;
* else
* p = rcu_pointer_handoff(p);
* }
* rcu_read_unlock();
*/
#define rcu_pointer_handoff(p) (p)
/**
* rcu_read_lock() - mark the beginning of an RCU read-side critical section
*
* When synchronize_rcu() is invoked on one CPU while other CPUs
* are within RCU read-side critical sections, then the
* synchronize_rcu() is guaranteed to block until after all the other
* CPUs exit their critical sections. Similarly, if call_rcu() is invoked
* on one CPU while other CPUs are within RCU read-side critical
* sections, invocation of the corresponding RCU callback is deferred
* until after the all the other CPUs exit their critical sections.
*
* In v5.0 and later kernels, synchronize_rcu() and call_rcu() also
* wait for regions of code with preemption disabled, including regions of
* code with interrupts or softirqs disabled. In pre-v5.0 kernels, which
* define synchronize_sched(), only code enclosed within rcu_read_lock()
* and rcu_read_unlock() are guaranteed to be waited for.
*
* Note, however, that RCU callbacks are permitted to run concurrently
* with new RCU read-side critical sections. One way that this can happen
* is via the following sequence of events: (1) CPU 0 enters an RCU
* read-side critical section, (2) CPU 1 invokes call_rcu() to register
* an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
* (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
* callback is invoked. This is legal, because the RCU read-side critical
* section that was running concurrently with the call_rcu() (and which
* therefore might be referencing something that the corresponding RCU
* callback would free up) has completed before the corresponding
* RCU callback is invoked.
*
* RCU read-side critical sections may be nested. Any deferred actions
* will be deferred until the outermost RCU read-side critical section
* completes.
*
* You can avoid reading and understanding the next paragraph by
* following this rule: don't put anything in an rcu_read_lock() RCU
* read-side critical section that would block in a !PREEMPTION kernel.
* But if you want the full story, read on!
*
* In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
* it is illegal to block while in an RCU read-side critical section.
* In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION
* kernel builds, RCU read-side critical sections may be preempted,
* but explicit blocking is illegal. Finally, in preemptible RCU
* implementations in real-time (with -rt patchset) kernel builds, RCU
* read-side critical sections may be preempted and they may also block, but
* only when acquiring spinlocks that are subject to priority inheritance.
*/
static __always_inline void rcu_read_lock(void)
{
__rcu_read_lock();
__acquire(RCU);
rcu_lock_acquire(&rcu_lock_map);
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_lock() used illegally while idle");
}
/*
* So where is rcu_write_lock()? It does not exist, as there is no
* way for writers to lock out RCU readers. This is a feature, not
* a bug -- this property is what provides RCU's performance benefits.
* Of course, writers must coordinate with each other. The normal
* spinlock primitives work well for this, but any other technique may be
* used as well. RCU does not care how the writers keep out of each
* others' way, as long as they do so.
*/
/**
* rcu_read_unlock() - marks the end of an RCU read-side critical section.
*
* In almost all situations, rcu_read_unlock() is immune from deadlock.
* In recent kernels that have consolidated synchronize_sched() and
* synchronize_rcu_bh() into synchronize_rcu(), this deadlock immunity
* also extends to the scheduler's runqueue and priority-inheritance
* spinlocks, courtesy of the quiescent-state deferral that is carried
* out when rcu_read_unlock() is invoked with interrupts disabled.
*
* See rcu_read_lock() for more information.
*/
static inline void rcu_read_unlock(void)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_unlock() used illegally while idle");
__release(RCU);
__rcu_read_unlock(); rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
}
/**
* rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
*
* This is equivalent to rcu_read_lock(), but also disables softirqs.
* Note that anything else that disables softirqs can also serve as an RCU
* read-side critical section. However, please note that this equivalence
* applies only to v5.0 and later. Before v5.0, rcu_read_lock() and
* rcu_read_lock_bh() were unrelated.
*
* Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
* must occur in the same context, for example, it is illegal to invoke
* rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
* was invoked from some other task.
*/
static inline void rcu_read_lock_bh(void)
{
local_bh_disable();
__acquire(RCU_BH);
rcu_lock_acquire(&rcu_bh_lock_map);
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_lock_bh() used illegally while idle");
}
/**
* rcu_read_unlock_bh() - marks the end of a softirq-only RCU critical section
*
* See rcu_read_lock_bh() for more information.
*/
static inline void rcu_read_unlock_bh(void)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_unlock_bh() used illegally while idle");
rcu_lock_release(&rcu_bh_lock_map);
__release(RCU_BH);
local_bh_enable();
}
/**
* rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
*
* This is equivalent to rcu_read_lock(), but also disables preemption.
* Read-side critical sections can also be introduced by anything else that
* disables preemption, including local_irq_disable() and friends. However,
* please note that the equivalence to rcu_read_lock() applies only to
* v5.0 and later. Before v5.0, rcu_read_lock() and rcu_read_lock_sched()
* were unrelated.
*
* Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
* must occur in the same context, for example, it is illegal to invoke
* rcu_read_unlock_sched() from process context if the matching
* rcu_read_lock_sched() was invoked from an NMI handler.
*/
static inline void rcu_read_lock_sched(void)
{
preempt_disable();
__acquire(RCU_SCHED);
rcu_lock_acquire(&rcu_sched_lock_map);
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_lock_sched() used illegally while idle");
}
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_lock_sched_notrace(void)
{
preempt_disable_notrace();
__acquire(RCU_SCHED);
}
/**
* rcu_read_unlock_sched() - marks the end of a RCU-classic critical section
*
* See rcu_read_lock_sched() for more information.
*/
static inline void rcu_read_unlock_sched(void)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(),
"rcu_read_unlock_sched() used illegally while idle");
rcu_lock_release(&rcu_sched_lock_map);
__release(RCU_SCHED);
preempt_enable();
}
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_unlock_sched_notrace(void)
{
__release(RCU_SCHED);
preempt_enable_notrace();
}
/**
* RCU_INIT_POINTER() - initialize an RCU protected pointer
* @p: The pointer to be initialized.
* @v: The value to initialized the pointer to.
*
* Initialize an RCU-protected pointer in special cases where readers
* do not need ordering constraints on the CPU or the compiler. These
* special cases are:
*
* 1. This use of RCU_INIT_POINTER() is NULLing out the pointer *or*
* 2. The caller has taken whatever steps are required to prevent
* RCU readers from concurrently accessing this pointer *or*
* 3. The referenced data structure has already been exposed to
* readers either at compile time or via rcu_assign_pointer() *and*
*
* a. You have not made *any* reader-visible changes to
* this structure since then *or*
* b. It is OK for readers accessing this structure from its
* new location to see the old state of the structure. (For
* example, the changes were to statistical counters or to
* other state where exact synchronization is not required.)
*
* Failure to follow these rules governing use of RCU_INIT_POINTER() will
* result in impossible-to-diagnose memory corruption. As in the structures
* will look OK in crash dumps, but any concurrent RCU readers might
* see pre-initialized values of the referenced data structure. So
* please be very careful how you use RCU_INIT_POINTER()!!!
*
* If you are creating an RCU-protected linked structure that is accessed
* by a single external-to-structure RCU-protected pointer, then you may
* use RCU_INIT_POINTER() to initialize the internal RCU-protected
* pointers, but you must use rcu_assign_pointer() to initialize the
* external-to-structure pointer *after* you have completely initialized
* the reader-accessible portions of the linked structure.
*
* Note that unlike rcu_assign_pointer(), RCU_INIT_POINTER() provides no
* ordering guarantees for either the CPU or the compiler.
*/
#define RCU_INIT_POINTER(p, v) \
do { \
rcu_check_sparse(p, __rcu); \
WRITE_ONCE(p, RCU_INITIALIZER(v)); \
} while (0)
/**
* RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
* @p: The pointer to be initialized.
* @v: The value to initialized the pointer to.
*
* GCC-style initialization for an RCU-protected pointer in a structure field.
*/
#define RCU_POINTER_INITIALIZER(p, v) \
.p = RCU_INITIALIZER(v)
/*
* Does the specified offset indicate that the corresponding rcu_head
* structure can be handled by kvfree_rcu()?
*/
#define __is_kvfree_rcu_offset(offset) ((offset) < 4096)
/**
* kfree_rcu() - kfree an object after a grace period.
* @ptr: pointer to kfree for double-argument invocations.
* @rhf: the name of the struct rcu_head within the type of @ptr.
*
* Many rcu callbacks functions just call kfree() on the base structure.
* These functions are trivial, but their size adds up, and furthermore
* when they are used in a kernel module, that module must invoke the
* high-latency rcu_barrier() function at module-unload time.
*
* The kfree_rcu() function handles this issue. Rather than encoding a
* function address in the embedded rcu_head structure, kfree_rcu() instead
* encodes the offset of the rcu_head structure within the base structure.
* Because the functions are not allowed in the low-order 4096 bytes of
* kernel virtual memory, offsets up to 4095 bytes can be accommodated.
* If the offset is larger than 4095 bytes, a compile-time error will
* be generated in kvfree_rcu_arg_2(). If this error is triggered, you can
* either fall back to use of call_rcu() or rearrange the structure to
* position the rcu_head structure into the first 4096 bytes.
*
* The object to be freed can be allocated either by kmalloc() or
* kmem_cache_alloc().
*
* Note that the allowable offset might decrease in the future.
*
* The BUILD_BUG_ON check must not involve any function calls, hence the
* checks are done in macros here.
*/
#define kfree_rcu(ptr, rhf) kvfree_rcu_arg_2(ptr, rhf)
#define kvfree_rcu(ptr, rhf) kvfree_rcu_arg_2(ptr, rhf)
/**
* kfree_rcu_mightsleep() - kfree an object after a grace period.
* @ptr: pointer to kfree for single-argument invocations.
*
* When it comes to head-less variant, only one argument
* is passed and that is just a pointer which has to be
* freed after a grace period. Therefore the semantic is
*
* kfree_rcu_mightsleep(ptr);
*
* where @ptr is the pointer to be freed by kvfree().
*
* Please note, head-less way of freeing is permitted to
* use from a context that has to follow might_sleep()
* annotation. Otherwise, please switch and embed the
* rcu_head structure within the type of @ptr.
*/
#define kfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr)
#define kvfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr)
#define kvfree_rcu_arg_2(ptr, rhf) \
do { \
typeof (ptr) ___p = (ptr); \
\
if (___p) { \
BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf))); \
kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \
} \
} while (0)
#define kvfree_rcu_arg_1(ptr) \
do { \
typeof(ptr) ___p = (ptr); \
\
if (___p) \
kvfree_call_rcu(NULL, (void *) (___p)); \
} while (0)
/*
* Place this after a lock-acquisition primitive to guarantee that
* an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies
* if the UNLOCK and LOCK are executed by the same CPU or if the
* UNLOCK and LOCK operate on the same lock variable.
*/
#ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE
#define smp_mb__after_unlock_lock() smp_mb() /* Full ordering for lock. */
#else /* #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
#define smp_mb__after_unlock_lock() do { } while (0)
#endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
/* Has the specified rcu_head structure been handed to call_rcu()? */
/**
* rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
* @rhp: The rcu_head structure to initialize.
*
* If you intend to invoke rcu_head_after_call_rcu() to test whether a
* given rcu_head structure has already been passed to call_rcu(), then
* you must also invoke this rcu_head_init() function on it just after
* allocating that structure. Calls to this function must not race with
* calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.
*/
static inline void rcu_head_init(struct rcu_head *rhp)
{
rhp->func = (rcu_callback_t)~0L;
}
/**
* rcu_head_after_call_rcu() - Has this rcu_head been passed to call_rcu()?
* @rhp: The rcu_head structure to test.
* @f: The function passed to call_rcu() along with @rhp.
*
* Returns @true if the @rhp has been passed to call_rcu() with @func,
* and @false otherwise. Emits a warning in any other case, including
* the case where @rhp has already been invoked after a grace period.
* Calls to this function must not race with callback invocation. One way
* to avoid such races is to enclose the call to rcu_head_after_call_rcu()
* in an RCU read-side critical section that includes a read-side fetch
* of the pointer to the structure containing @rhp.
*/
static inline bool
rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
{
rcu_callback_t func = READ_ONCE(rhp->func);
if (func == f)
return true;
WARN_ON_ONCE(func != (rcu_callback_t)~0L);
return false;
}
/* kernel/ksysfs.c definitions */
extern int rcu_expedited;
extern int rcu_normal;
DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock())
#endif /* __LINUX_RCUPDATE_H */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Resizable, Scalable, Concurrent Hash Table
*
* Copyright (c) 2015-2016 Herbert Xu <herbert@gondor.apana.org.au>
* Copyright (c) 2014-2015 Thomas Graf <tgraf@suug.ch>
* Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
*
* Code partially derived from nft_hash
* Rewritten with rehash code from br_multicast plus single list
* pointer as suggested by Josh Triplett
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef _LINUX_RHASHTABLE_H
#define _LINUX_RHASHTABLE_H
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/jhash.h>
#include <linux/list_nulls.h>
#include <linux/workqueue.h>
#include <linux/rculist.h>
#include <linux/bit_spinlock.h>
#include <linux/rhashtable-types.h>
/*
* Objects in an rhashtable have an embedded struct rhash_head
* which is linked into as hash chain from the hash table - or one
* of two or more hash tables when the rhashtable is being resized.
* The end of the chain is marked with a special nulls marks which has
* the least significant bit set but otherwise stores the address of
* the hash bucket. This allows us to be sure we've found the end
* of the right list.
* The value stored in the hash bucket has BIT(0) used as a lock bit.
* This bit must be atomically set before any changes are made to
* the chain. To avoid dereferencing this pointer without clearing
* the bit first, we use an opaque 'struct rhash_lock_head *' for the
* pointer stored in the bucket. This struct needs to be defined so
* that rcu_dereference() works on it, but it has no content so a
* cast is needed for it to be useful. This ensures it isn't
* used by mistake with clearing the lock bit first.
*/
struct rhash_lock_head {};
/* Maximum chain length before rehash
*
* The maximum (not average) chain length grows with the size of the hash
* table, at a rate of (log N)/(log log N).
*
* The value of 16 is selected so that even if the hash table grew to
* 2^32 you would not expect the maximum chain length to exceed it
* unless we are under attack (or extremely unlucky).
*
* As this limit is only to detect attacks, we don't need to set it to a
* lower value as you'd need the chain length to vastly exceed 16 to have
* any real effect on the system.
*/
#define RHT_ELASTICITY 16u
/**
* struct bucket_table - Table of hash buckets
* @size: Number of hash buckets
* @nest: Number of bits of first-level nested table.
* @rehash: Current bucket being rehashed
* @hash_rnd: Random seed to fold into hash
* @walkers: List of active walkers
* @rcu: RCU structure for freeing the table
* @future_tbl: Table under construction during rehashing
* @ntbl: Nested table used when out of memory.
* @buckets: size * hash buckets
*/
struct bucket_table {
unsigned int size;
unsigned int nest;
u32 hash_rnd;
struct list_head walkers;
struct rcu_head rcu;
struct bucket_table __rcu *future_tbl;
struct lockdep_map dep_map;
struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp;
};
/*
* NULLS_MARKER() expects a hash value with the low
* bits mostly likely to be significant, and it discards
* the msb.
* We give it an address, in which the bottom bit is
* always 0, and the msb might be significant.
* So we shift the address down one bit to align with
* expectations and avoid losing a significant bit.
*
* We never store the NULLS_MARKER in the hash table
* itself as we need the lsb for locking.
* Instead we store a NULL
*/
#define RHT_NULLS_MARKER(ptr) \
((void *)NULLS_MARKER(((unsigned long) (ptr)) >> 1))
#define INIT_RHT_NULLS_HEAD(ptr) \
((ptr) = NULL)
static inline bool rht_is_a_nulls(const struct rhash_head *ptr)
{
return ((unsigned long) ptr & 1);
}
static inline void *rht_obj(const struct rhashtable *ht,
const struct rhash_head *he)
{
return (char *)he - ht->p.head_offset;
}
static inline unsigned int rht_bucket_index(const struct bucket_table *tbl,
unsigned int hash)
{
return hash & (tbl->size - 1);
}
static inline unsigned int rht_key_get_hash(struct rhashtable *ht,
const void *key, const struct rhashtable_params params,
unsigned int hash_rnd)
{
unsigned int hash;
/* params must be equal to ht->p if it isn't constant. */
if (!__builtin_constant_p(params.key_len))
hash = ht->p.hashfn(key, ht->key_len, hash_rnd);
else if (params.key_len) {
unsigned int key_len = params.key_len;
if (params.hashfn)
hash = params.hashfn(key, key_len, hash_rnd);
else if (key_len & (sizeof(u32) - 1))
hash = jhash(key, key_len, hash_rnd);
else
hash = jhash2(key, key_len / sizeof(u32), hash_rnd);
} else {
unsigned int key_len = ht->p.key_len;
if (params.hashfn)
hash = params.hashfn(key, key_len, hash_rnd);
else
hash = jhash(key, key_len, hash_rnd);
}
return hash;
}
static inline unsigned int rht_key_hashfn(
struct rhashtable *ht, const struct bucket_table *tbl,
const void *key, const struct rhashtable_params params)
{
unsigned int hash = rht_key_get_hash(ht, key, params, tbl->hash_rnd);
return rht_bucket_index(tbl, hash);
}
static inline unsigned int rht_head_hashfn(
struct rhashtable *ht, const struct bucket_table *tbl,
const struct rhash_head *he, const struct rhashtable_params params)
{
const char *ptr = rht_obj(ht, he);
return likely(params.obj_hashfn) ?
rht_bucket_index(tbl, params.obj_hashfn(ptr, params.key_len ?:
ht->p.key_len,
tbl->hash_rnd)) :
rht_key_hashfn(ht, tbl, ptr + params.key_offset, params);
}
/**
* rht_grow_above_75 - returns true if nelems > 0.75 * table-size
* @ht: hash table
* @tbl: current table
*/
static inline bool rht_grow_above_75(const struct rhashtable *ht,
const struct bucket_table *tbl)
{
/* Expand table when exceeding 75% load */
return atomic_read(&ht->nelems) > (tbl->size / 4 * 3) &&
(!ht->p.max_size || tbl->size < ht->p.max_size);
}
/**
* rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
* @ht: hash table
* @tbl: current table
*/
static inline bool rht_shrink_below_30(const struct rhashtable *ht,
const struct bucket_table *tbl)
{
/* Shrink table beneath 30% load */
return atomic_read(&ht->nelems) < (tbl->size * 3 / 10) &&
tbl->size > ht->p.min_size;
}
/**
* rht_grow_above_100 - returns true if nelems > table-size
* @ht: hash table
* @tbl: current table
*/
static inline bool rht_grow_above_100(const struct rhashtable *ht,
const struct bucket_table *tbl)
{
return atomic_read(&ht->nelems) > tbl->size &&
(!ht->p.max_size || tbl->size < ht->p.max_size);
}
/**
* rht_grow_above_max - returns true if table is above maximum
* @ht: hash table
* @tbl: current table
*/
static inline bool rht_grow_above_max(const struct rhashtable *ht,
const struct bucket_table *tbl)
{
return atomic_read(&ht->nelems) >= ht->max_elems;
}
#ifdef CONFIG_PROVE_LOCKING
int lockdep_rht_mutex_is_held(struct rhashtable *ht);
int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);
#else
static inline int lockdep_rht_mutex_is_held(struct rhashtable *ht)
{
return 1;
}
static inline int lockdep_rht_bucket_is_held(const struct bucket_table *tbl,
u32 hash)
{
return 1;
}
#endif /* CONFIG_PROVE_LOCKING */
void *rhashtable_insert_slow(struct rhashtable *ht, const void *key,
struct rhash_head *obj);
void rhashtable_walk_enter(struct rhashtable *ht,
struct rhashtable_iter *iter);
void rhashtable_walk_exit(struct rhashtable_iter *iter);
int rhashtable_walk_start_check(struct rhashtable_iter *iter) __acquires(RCU);
static inline void rhashtable_walk_start(struct rhashtable_iter *iter)
{
(void)rhashtable_walk_start_check(iter);
}
void *rhashtable_walk_next(struct rhashtable_iter *iter);
void *rhashtable_walk_peek(struct rhashtable_iter *iter);
void rhashtable_walk_stop(struct rhashtable_iter *iter) __releases(RCU);
void rhashtable_free_and_destroy(struct rhashtable *ht,
void (*free_fn)(void *ptr, void *arg),
void *arg);
void rhashtable_destroy(struct rhashtable *ht);
struct rhash_lock_head __rcu **rht_bucket_nested(
const struct bucket_table *tbl, unsigned int hash);
struct rhash_lock_head __rcu **__rht_bucket_nested(
const struct bucket_table *tbl, unsigned int hash);
struct rhash_lock_head __rcu **rht_bucket_nested_insert(
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash);
#define rht_dereference(p, ht) \
rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
#define rht_dereference_rcu(p, ht) \
rcu_dereference_check(p, lockdep_rht_mutex_is_held(ht))
#define rht_dereference_bucket(p, tbl, hash) \
rcu_dereference_protected(p, lockdep_rht_bucket_is_held(tbl, hash))
#define rht_dereference_bucket_rcu(p, tbl, hash) \
rcu_dereference_check(p, lockdep_rht_bucket_is_held(tbl, hash))
#define rht_entry(tpos, pos, member) \
({ tpos = container_of(pos, typeof(*tpos), member); 1; })
static inline struct rhash_lock_head __rcu *const *rht_bucket(
const struct bucket_table *tbl, unsigned int hash)
{
return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
&tbl->buckets[hash];
}
static inline struct rhash_lock_head __rcu **rht_bucket_var(
struct bucket_table *tbl, unsigned int hash)
{
return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :
&tbl->buckets[hash];
}
static inline struct rhash_lock_head __rcu **rht_bucket_insert(
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
{
return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
&tbl->buckets[hash];
}
/*
* We lock a bucket by setting BIT(0) in the pointer - this is always
* zero in real pointers. The NULLS mark is never stored in the bucket,
* rather we store NULL if the bucket is empty.
* bit_spin_locks do not handle contention well, but the whole point
* of the hashtable design is to achieve minimum per-bucket contention.
* A nested hash table might not have a bucket pointer. In that case
* we cannot get a lock. For remove and replace the bucket cannot be
* interesting and doesn't need locking.
* For insert we allocate the bucket if this is the last bucket_table,
* and then take the lock.
* Sometimes we unlock a bucket by writing a new pointer there. In that
* case we don't need to unlock, but we do need to reset state such as
* local_bh. For that we have rht_assign_unlock(). As rcu_assign_pointer()
* provides the same release semantics that bit_spin_unlock() provides,
* this is safe.
* When we write to a bucket without unlocking, we use rht_assign_locked().
*/
static inline unsigned long rht_lock(struct bucket_table *tbl,
struct rhash_lock_head __rcu **bkt)
{
unsigned long flags;
local_irq_save(flags);
bit_spin_lock(0, (unsigned long *)bkt);
lock_map_acquire(&tbl->dep_map);
return flags;
}
static inline unsigned long rht_lock_nested(struct bucket_table *tbl,
struct rhash_lock_head __rcu **bucket,
unsigned int subclass)
{
unsigned long flags;
local_irq_save(flags);
bit_spin_lock(0, (unsigned long *)bucket);
lock_acquire_exclusive(&tbl->dep_map, subclass, 0, NULL, _THIS_IP_);
return flags;
}
static inline void rht_unlock(struct bucket_table *tbl,
struct rhash_lock_head __rcu **bkt,
unsigned long flags)
{
lock_map_release(&tbl->dep_map);
bit_spin_unlock(0, (unsigned long *)bkt);
local_irq_restore(flags);
}
static inline struct rhash_head *__rht_ptr(
struct rhash_lock_head *p, struct rhash_lock_head __rcu *const *bkt)
{
return (struct rhash_head *) ((unsigned long)p & ~BIT(0) ?: (unsigned long)RHT_NULLS_MARKER(bkt));
}
/*
* Where 'bkt' is a bucket and might be locked:
* rht_ptr_rcu() dereferences that pointer and clears the lock bit.
* rht_ptr() dereferences in a context where the bucket is locked.
* rht_ptr_exclusive() dereferences in a context where exclusive
* access is guaranteed, such as when destroying the table.
*/
static inline struct rhash_head *rht_ptr_rcu(
struct rhash_lock_head __rcu *const *bkt)
{
return __rht_ptr(rcu_dereference(*bkt), bkt);
}
static inline struct rhash_head *rht_ptr(
struct rhash_lock_head __rcu *const *bkt,
struct bucket_table *tbl,
unsigned int hash)
{
return __rht_ptr(rht_dereference_bucket(*bkt, tbl, hash), bkt);
}
static inline struct rhash_head *rht_ptr_exclusive(
struct rhash_lock_head __rcu *const *bkt)
{
return __rht_ptr(rcu_dereference_protected(*bkt, 1), bkt);
}
static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
struct rhash_head *obj)
{
if (rht_is_a_nulls(obj))
obj = NULL;
rcu_assign_pointer(*bkt, (void *)((unsigned long)obj | BIT(0)));
}
static inline void rht_assign_unlock(struct bucket_table *tbl,
struct rhash_lock_head __rcu **bkt,
struct rhash_head *obj,
unsigned long flags)
{
if (rht_is_a_nulls(obj))
obj = NULL;
lock_map_release(&tbl->dep_map);
rcu_assign_pointer(*bkt, (void *)obj);
preempt_enable();
__release(bitlock);
local_irq_restore(flags);
}
/**
* rht_for_each_from - iterate over hash chain from given head
* @pos: the &struct rhash_head to use as a loop cursor.
* @head: the &struct rhash_head to start from
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
*/
#define rht_for_each_from(pos, head, tbl, hash) \
for (pos = head; \
!rht_is_a_nulls(pos); \
pos = rht_dereference_bucket((pos)->next, tbl, hash))
/**
* rht_for_each - iterate over hash chain
* @pos: the &struct rhash_head to use as a loop cursor.
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
*/
#define rht_for_each(pos, tbl, hash) \
rht_for_each_from(pos, rht_ptr(rht_bucket(tbl, hash), tbl, hash), \
tbl, hash)
/**
* rht_for_each_entry_from - iterate over hash chain from given head
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct rhash_head to use as a loop cursor.
* @head: the &struct rhash_head to start from
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
* @member: name of the &struct rhash_head within the hashable struct.
*/
#define rht_for_each_entry_from(tpos, pos, head, tbl, hash, member) \
for (pos = head; \
(!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
pos = rht_dereference_bucket((pos)->next, tbl, hash))
/**
* rht_for_each_entry - iterate over hash chain of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct rhash_head to use as a loop cursor.
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
* @member: name of the &struct rhash_head within the hashable struct.
*/
#define rht_for_each_entry(tpos, pos, tbl, hash, member) \
rht_for_each_entry_from(tpos, pos, \
rht_ptr(rht_bucket(tbl, hash), tbl, hash), \
tbl, hash, member)
/**
* rht_for_each_entry_safe - safely iterate over hash chain of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct rhash_head to use as a loop cursor.
* @next: the &struct rhash_head to use as next in loop cursor.
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
* @member: name of the &struct rhash_head within the hashable struct.
*
* This hash chain list-traversal primitive allows for the looped code to
* remove the loop cursor from the list.
*/
#define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \
for (pos = rht_ptr(rht_bucket(tbl, hash), tbl, hash), \
next = !rht_is_a_nulls(pos) ? \
rht_dereference_bucket(pos->next, tbl, hash) : NULL; \
(!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
pos = next, \
next = !rht_is_a_nulls(pos) ? \
rht_dereference_bucket(pos->next, tbl, hash) : NULL)
/**
* rht_for_each_rcu_from - iterate over rcu hash chain from given head
* @pos: the &struct rhash_head to use as a loop cursor.
* @head: the &struct rhash_head to start from
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
*
* This hash chain list-traversal primitive may safely run concurrently with
* the _rcu mutation primitives such as rhashtable_insert() as long as the
* traversal is guarded by rcu_read_lock().
*/
#define rht_for_each_rcu_from(pos, head, tbl, hash) \
for (({barrier(); }), \
pos = head; \
!rht_is_a_nulls(pos); \
pos = rcu_dereference_raw(pos->next))
/**
* rht_for_each_rcu - iterate over rcu hash chain
* @pos: the &struct rhash_head to use as a loop cursor.
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
*
* This hash chain list-traversal primitive may safely run concurrently with
* the _rcu mutation primitives such as rhashtable_insert() as long as the
* traversal is guarded by rcu_read_lock().
*/
#define rht_for_each_rcu(pos, tbl, hash) \
for (({barrier(); }), \
pos = rht_ptr_rcu(rht_bucket(tbl, hash)); \
!rht_is_a_nulls(pos); \
pos = rcu_dereference_raw(pos->next))
/**
* rht_for_each_entry_rcu_from - iterated over rcu hash chain from given head
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct rhash_head to use as a loop cursor.
* @head: the &struct rhash_head to start from
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
* @member: name of the &struct rhash_head within the hashable struct.
*
* This hash chain list-traversal primitive may safely run concurrently with
* the _rcu mutation primitives such as rhashtable_insert() as long as the
* traversal is guarded by rcu_read_lock().
*/
#define rht_for_each_entry_rcu_from(tpos, pos, head, tbl, hash, member) \
for (({barrier(); }), \
pos = head; \
(!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
pos = rht_dereference_bucket_rcu(pos->next, tbl, hash))
/**
* rht_for_each_entry_rcu - iterate over rcu hash chain of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct rhash_head to use as a loop cursor.
* @tbl: the &struct bucket_table
* @hash: the hash value / bucket index
* @member: name of the &struct rhash_head within the hashable struct.
*
* This hash chain list-traversal primitive may safely run concurrently with
* the _rcu mutation primitives such as rhashtable_insert() as long as the
* traversal is guarded by rcu_read_lock().
*/
#define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \
rht_for_each_entry_rcu_from(tpos, pos, \
rht_ptr_rcu(rht_bucket(tbl, hash)), \
tbl, hash, member)
/**
* rhl_for_each_rcu - iterate over rcu hash table list
* @pos: the &struct rlist_head to use as a loop cursor.
* @list: the head of the list
*
* This hash chain list-traversal primitive should be used on the
* list returned by rhltable_lookup.
*/
#define rhl_for_each_rcu(pos, list) \
for (pos = list; pos; pos = rcu_dereference_raw(pos->next))
/**
* rhl_for_each_entry_rcu - iterate over rcu hash table list of given type
* @tpos: the type * to use as a loop cursor.
* @pos: the &struct rlist_head to use as a loop cursor.
* @list: the head of the list
* @member: name of the &struct rlist_head within the hashable struct.
*
* This hash chain list-traversal primitive should be used on the
* list returned by rhltable_lookup.
*/
#define rhl_for_each_entry_rcu(tpos, pos, list, member) \
for (pos = list; pos && rht_entry(tpos, pos, member); \
pos = rcu_dereference_raw(pos->next))
static inline int rhashtable_compare(struct rhashtable_compare_arg *arg,
const void *obj)
{
struct rhashtable *ht = arg->ht;
const char *ptr = obj;
return memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len);
}
/* Internal function, do not use. */
static inline struct rhash_head *__rhashtable_lookup(
struct rhashtable *ht, const void *key,
const struct rhashtable_params params)
{
struct rhashtable_compare_arg arg = {
.ht = ht,
.key = key,
};
struct rhash_lock_head __rcu *const *bkt;
struct bucket_table *tbl;
struct rhash_head *he;
unsigned int hash;
tbl = rht_dereference_rcu(ht->tbl, ht);
restart:
hash = rht_key_hashfn(ht, tbl, key, params); bkt = rht_bucket(tbl, hash);
do {
rht_for_each_rcu_from(he, rht_ptr_rcu(bkt), tbl, hash) { if (params.obj_cmpfn ?
params.obj_cmpfn(&arg, rht_obj(ht, he)) :
rhashtable_compare(&arg, rht_obj(ht, he)))
continue;
return he;
}
/* An object might have been moved to a different hash chain,
* while we walk along it - better check and retry.
*/
} while (he != RHT_NULLS_MARKER(bkt));
/* Ensure we see any new tables. */
smp_rmb(); tbl = rht_dereference_rcu(tbl->future_tbl, ht); if (unlikely(tbl))
goto restart;
return NULL;
}
/**
* rhashtable_lookup - search hash table
* @ht: hash table
* @key: the pointer to the key
* @params: hash table parameters
*
* Computes the hash value for the key and traverses the bucket chain looking
* for a entry with an identical key. The first matching entry is returned.
*
* This must only be called under the RCU read lock.
*
* Returns the first entry on which the compare function returned true.
*/
static inline void *rhashtable_lookup(
struct rhashtable *ht, const void *key,
const struct rhashtable_params params)
{
struct rhash_head *he = __rhashtable_lookup(ht, key, params); return he ? rht_obj(ht, he) : NULL;
}
/**
* rhashtable_lookup_fast - search hash table, without RCU read lock
* @ht: hash table
* @key: the pointer to the key
* @params: hash table parameters
*
* Computes the hash value for the key and traverses the bucket chain looking
* for a entry with an identical key. The first matching entry is returned.
*
* Only use this function when you have other mechanisms guaranteeing
* that the object won't go away after the RCU read lock is released.
*
* Returns the first entry on which the compare function returned true.
*/
static inline void *rhashtable_lookup_fast(
struct rhashtable *ht, const void *key,
const struct rhashtable_params params)
{
void *obj;
rcu_read_lock(); obj = rhashtable_lookup(ht, key, params); rcu_read_unlock();
return obj;
}
/**
* rhltable_lookup - search hash list table
* @hlt: hash table
* @key: the pointer to the key
* @params: hash table parameters
*
* Computes the hash value for the key and traverses the bucket chain looking
* for a entry with an identical key. All matching entries are returned
* in a list.
*
* This must only be called under the RCU read lock.
*
* Returns the list of entries that match the given key.
*/
static inline struct rhlist_head *rhltable_lookup(
struct rhltable *hlt, const void *key,
const struct rhashtable_params params)
{
struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params);
return he ? container_of(he, struct rhlist_head, rhead) : NULL;
}
/* Internal function, please use rhashtable_insert_fast() instead. This
* function returns the existing element already in hashes in there is a clash,
* otherwise it returns an error via ERR_PTR().
*/
static inline void *__rhashtable_insert_fast(
struct rhashtable *ht, const void *key, struct rhash_head *obj,
const struct rhashtable_params params, bool rhlist)
{
struct rhashtable_compare_arg arg = {
.ht = ht,
.key = key,
};
struct rhash_lock_head __rcu **bkt;
struct rhash_head __rcu **pprev;
struct bucket_table *tbl;
struct rhash_head *head;
unsigned long flags;
unsigned int hash;
int elasticity;
void *data;
rcu_read_lock();
tbl = rht_dereference_rcu(ht->tbl, ht);
hash = rht_head_hashfn(ht, tbl, obj, params);
elasticity = RHT_ELASTICITY;
bkt = rht_bucket_insert(ht, tbl, hash);
data = ERR_PTR(-ENOMEM);
if (!bkt)
goto out;
pprev = NULL;
flags = rht_lock(tbl, bkt);
if (unlikely(rcu_access_pointer(tbl->future_tbl))) {
slow_path:
rht_unlock(tbl, bkt, flags);
rcu_read_unlock();
return rhashtable_insert_slow(ht, key, obj);
}
rht_for_each_from(head, rht_ptr(bkt, tbl, hash), tbl, hash) {
struct rhlist_head *plist;
struct rhlist_head *list;
elasticity--;
if (!key ||
(params.obj_cmpfn ?
params.obj_cmpfn(&arg, rht_obj(ht, head)) :
rhashtable_compare(&arg, rht_obj(ht, head)))) {
pprev = &head->next;
continue;
}
data = rht_obj(ht, head);
if (!rhlist)
goto out_unlock;
list = container_of(obj, struct rhlist_head, rhead);
plist = container_of(head, struct rhlist_head, rhead);
RCU_INIT_POINTER(list->next, plist);
head = rht_dereference_bucket(head->next, tbl, hash);
RCU_INIT_POINTER(list->rhead.next, head);
if (pprev) {
rcu_assign_pointer(*pprev, obj);
rht_unlock(tbl, bkt, flags);
} else
rht_assign_unlock(tbl, bkt, obj, flags);
data = NULL;
goto out;
}
if (elasticity <= 0)
goto slow_path;
data = ERR_PTR(-E2BIG);
if (unlikely(rht_grow_above_max(ht, tbl)))
goto out_unlock;
if (unlikely(rht_grow_above_100(ht, tbl)))
goto slow_path;
/* Inserting at head of list makes unlocking free. */
head = rht_ptr(bkt, tbl, hash);
RCU_INIT_POINTER(obj->next, head);
if (rhlist) {
struct rhlist_head *list;
list = container_of(obj, struct rhlist_head, rhead);
RCU_INIT_POINTER(list->next, NULL);
}
atomic_inc(&ht->nelems);
rht_assign_unlock(tbl, bkt, obj, flags);
if (rht_grow_above_75(ht, tbl))
schedule_work(&ht->run_work);
data = NULL;
out:
rcu_read_unlock();
return data;
out_unlock:
rht_unlock(tbl, bkt, flags);
goto out;
}
/**
* rhashtable_insert_fast - insert object into hash table
* @ht: hash table
* @obj: pointer to hash head inside object
* @params: hash table parameters
*
* Will take the per bucket bitlock to protect against mutual mutations
* on the same bucket. Multiple insertions may occur in parallel unless
* they map to the same bucket.
*
* It is safe to call this function from atomic context.
*
* Will trigger an automatic deferred table resizing if residency in the
* table grows beyond 70%.
*/
static inline int rhashtable_insert_fast(
struct rhashtable *ht, struct rhash_head *obj,
const struct rhashtable_params params)
{
void *ret;
ret = __rhashtable_insert_fast(ht, NULL, obj, params, false);
if (IS_ERR(ret))
return PTR_ERR(ret);
return ret == NULL ? 0 : -EEXIST;
}
/**
* rhltable_insert_key - insert object into hash list table
* @hlt: hash list table
* @key: the pointer to the key
* @list: pointer to hash list head inside object
* @params: hash table parameters
*
* Will take the per bucket bitlock to protect against mutual mutations
* on the same bucket. Multiple insertions may occur in parallel unless
* they map to the same bucket.
*
* It is safe to call this function from atomic context.
*
* Will trigger an automatic deferred table resizing if residency in the
* table grows beyond 70%.
*/
static inline int rhltable_insert_key(
struct rhltable *hlt, const void *key, struct rhlist_head *list,
const struct rhashtable_params params)
{
return PTR_ERR(__rhashtable_insert_fast(&hlt->ht, key, &list->rhead,
params, true));
}
/**
* rhltable_insert - insert object into hash list table
* @hlt: hash list table
* @list: pointer to hash list head inside object
* @params: hash table parameters
*
* Will take the per bucket bitlock to protect against mutual mutations
* on the same bucket. Multiple insertions may occur in parallel unless
* they map to the same bucket.
*
* It is safe to call this function from atomic context.
*
* Will trigger an automatic deferred table resizing if residency in the
* table grows beyond 70%.
*/
static inline int rhltable_insert(
struct rhltable *hlt, struct rhlist_head *list,
const struct rhashtable_params params)
{
const char *key = rht_obj(&hlt->ht, &list->rhead);
key += params.key_offset;
return rhltable_insert_key(hlt, key, list, params);
}
/**
* rhashtable_lookup_insert_fast - lookup and insert object into hash table
* @ht: hash table
* @obj: pointer to hash head inside object
* @params: hash table parameters
*
* This lookup function may only be used for fixed key hash table (key_len
* parameter set). It will BUG() if used inappropriately.
*
* It is safe to call this function from atomic context.
*
* Will trigger an automatic deferred table resizing if residency in the
* table grows beyond 70%.
*/
static inline int rhashtable_lookup_insert_fast(
struct rhashtable *ht, struct rhash_head *obj,
const struct rhashtable_params params)
{
const char *key = rht_obj(ht, obj);
void *ret;
BUG_ON(ht->p.obj_hashfn);
ret = __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj, params,
false);
if (IS_ERR(ret))
return PTR_ERR(ret);
return ret == NULL ? 0 : -EEXIST;
}
/**
* rhashtable_lookup_get_insert_fast - lookup and insert object into hash table
* @ht: hash table
* @obj: pointer to hash head inside object
* @params: hash table parameters
*
* Just like rhashtable_lookup_insert_fast(), but this function returns the
* object if it exists, NULL if it did not and the insertion was successful,
* and an ERR_PTR otherwise.
*/
static inline void *rhashtable_lookup_get_insert_fast(
struct rhashtable *ht, struct rhash_head *obj,
const struct rhashtable_params params)
{
const char *key = rht_obj(ht, obj);
BUG_ON(ht->p.obj_hashfn);
return __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj, params,
false);
}
/**
* rhashtable_lookup_insert_key - search and insert object to hash table
* with explicit key
* @ht: hash table
* @key: key
* @obj: pointer to hash head inside object
* @params: hash table parameters
*
* Lookups may occur in parallel with hashtable mutations and resizing.
*
* Will trigger an automatic deferred table resizing if residency in the
* table grows beyond 70%.
*
* Returns zero on success.
*/
static inline int rhashtable_lookup_insert_key(
struct rhashtable *ht, const void *key, struct rhash_head *obj,
const struct rhashtable_params params)
{
void *ret;
BUG_ON(!ht->p.obj_hashfn || !key);
ret = __rhashtable_insert_fast(ht, key, obj, params, false);
if (IS_ERR(ret))
return PTR_ERR(ret);
return ret == NULL ? 0 : -EEXIST;
}
/**
* rhashtable_lookup_get_insert_key - lookup and insert object into hash table
* @ht: hash table
* @key: key
* @obj: pointer to hash head inside object
* @params: hash table parameters
*
* Just like rhashtable_lookup_insert_key(), but this function returns the
* object if it exists, NULL if it does not and the insertion was successful,
* and an ERR_PTR otherwise.
*/
static inline void *rhashtable_lookup_get_insert_key(
struct rhashtable *ht, const void *key, struct rhash_head *obj,
const struct rhashtable_params params)
{
BUG_ON(!ht->p.obj_hashfn || !key);
return __rhashtable_insert_fast(ht, key, obj, params, false);
}
/* Internal function, please use rhashtable_remove_fast() instead */
static inline int __rhashtable_remove_fast_one(
struct rhashtable *ht, struct bucket_table *tbl,
struct rhash_head *obj, const struct rhashtable_params params,
bool rhlist)
{
struct rhash_lock_head __rcu **bkt;
struct rhash_head __rcu **pprev;
struct rhash_head *he;
unsigned long flags;
unsigned int hash;
int err = -ENOENT;
hash = rht_head_hashfn(ht, tbl, obj, params);
bkt = rht_bucket_var(tbl, hash);
if (!bkt)
return -ENOENT;
pprev = NULL;
flags = rht_lock(tbl, bkt);
rht_for_each_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) {
struct rhlist_head *list;
list = container_of(he, struct rhlist_head, rhead);
if (he != obj) {
struct rhlist_head __rcu **lpprev;
pprev = &he->next;
if (!rhlist)
continue;
do {
lpprev = &list->next;
list = rht_dereference_bucket(list->next,
tbl, hash);
} while (list && obj != &list->rhead);
if (!list)
continue;
list = rht_dereference_bucket(list->next, tbl, hash);
RCU_INIT_POINTER(*lpprev, list);
err = 0;
break;
}
obj = rht_dereference_bucket(obj->next, tbl, hash);
err = 1;
if (rhlist) {
list = rht_dereference_bucket(list->next, tbl, hash);
if (list) {
RCU_INIT_POINTER(list->rhead.next, obj);
obj = &list->rhead;
err = 0;
}
}
if (pprev) {
rcu_assign_pointer(*pprev, obj);
rht_unlock(tbl, bkt, flags);
} else {
rht_assign_unlock(tbl, bkt, obj, flags);
}
goto unlocked;
}
rht_unlock(tbl, bkt, flags);
unlocked:
if (err > 0) {
atomic_dec(&ht->nelems);
if (unlikely(ht->p.automatic_shrinking &&
rht_shrink_below_30(ht, tbl)))
schedule_work(&ht->run_work);
err = 0;
}
return err;
}
/* Internal function, please use rhashtable_remove_fast() instead */
static inline int __rhashtable_remove_fast(
struct rhashtable *ht, struct rhash_head *obj,
const struct rhashtable_params params, bool rhlist)
{
struct bucket_table *tbl;
int err;
rcu_read_lock();
tbl = rht_dereference_rcu(ht->tbl, ht);
/* Because we have already taken (and released) the bucket
* lock in old_tbl, if we find that future_tbl is not yet
* visible then that guarantees the entry to still be in
* the old tbl if it exists.
*/
while ((err = __rhashtable_remove_fast_one(ht, tbl, obj, params,
rhlist)) &&
(tbl = rht_dereference_rcu(tbl->future_tbl, ht)))
;
rcu_read_unlock();
return err;
}
/**
* rhashtable_remove_fast - remove object from hash table
* @ht: hash table
* @obj: pointer to hash head inside object
* @params: hash table parameters
*
* Since the hash chain is single linked, the removal operation needs to
* walk the bucket chain upon removal. The removal operation is thus
* considerable slow if the hash table is not correctly sized.
*
* Will automatically shrink the table if permitted when residency drops
* below 30%.
*
* Returns zero on success, -ENOENT if the entry could not be found.
*/
static inline int rhashtable_remove_fast(
struct rhashtable *ht, struct rhash_head *obj,
const struct rhashtable_params params)
{
return __rhashtable_remove_fast(ht, obj, params, false);
}
/**
* rhltable_remove - remove object from hash list table
* @hlt: hash list table
* @list: pointer to hash list head inside object
* @params: hash table parameters
*
* Since the hash chain is single linked, the removal operation needs to
* walk the bucket chain upon removal. The removal operation is thus
* considerable slow if the hash table is not correctly sized.
*
* Will automatically shrink the table if permitted when residency drops
* below 30%
*
* Returns zero on success, -ENOENT if the entry could not be found.
*/
static inline int rhltable_remove(
struct rhltable *hlt, struct rhlist_head *list,
const struct rhashtable_params params)
{
return __rhashtable_remove_fast(&hlt->ht, &list->rhead, params, true);
}
/* Internal function, please use rhashtable_replace_fast() instead */
static inline int __rhashtable_replace_fast(
struct rhashtable *ht, struct bucket_table *tbl,
struct rhash_head *obj_old, struct rhash_head *obj_new,
const struct rhashtable_params params)
{
struct rhash_lock_head __rcu **bkt;
struct rhash_head __rcu **pprev;
struct rhash_head *he;
unsigned long flags;
unsigned int hash;
int err = -ENOENT;
/* Minimally, the old and new objects must have same hash
* (which should mean identifiers are the same).
*/
hash = rht_head_hashfn(ht, tbl, obj_old, params);
if (hash != rht_head_hashfn(ht, tbl, obj_new, params))
return -EINVAL;
bkt = rht_bucket_var(tbl, hash);
if (!bkt)
return -ENOENT;
pprev = NULL;
flags = rht_lock(tbl, bkt);
rht_for_each_from(he, rht_ptr(bkt, tbl, hash), tbl, hash) {
if (he != obj_old) {
pprev = &he->next;
continue;
}
rcu_assign_pointer(obj_new->next, obj_old->next);
if (pprev) {
rcu_assign_pointer(*pprev, obj_new);
rht_unlock(tbl, bkt, flags);
} else {
rht_assign_unlock(tbl, bkt, obj_new, flags);
}
err = 0;
goto unlocked;
}
rht_unlock(tbl, bkt, flags);
unlocked:
return err;
}
/**
* rhashtable_replace_fast - replace an object in hash table
* @ht: hash table
* @obj_old: pointer to hash head inside object being replaced
* @obj_new: pointer to hash head inside object which is new
* @params: hash table parameters
*
* Replacing an object doesn't affect the number of elements in the hash table
* or bucket, so we don't need to worry about shrinking or expanding the
* table here.
*
* Returns zero on success, -ENOENT if the entry could not be found,
* -EINVAL if hash is not the same for the old and new objects.
*/
static inline int rhashtable_replace_fast(
struct rhashtable *ht, struct rhash_head *obj_old,
struct rhash_head *obj_new,
const struct rhashtable_params params)
{
struct bucket_table *tbl;
int err;
rcu_read_lock();
tbl = rht_dereference_rcu(ht->tbl, ht);
/* Because we have already taken (and released) the bucket
* lock in old_tbl, if we find that future_tbl is not yet
* visible then that guarantees the entry to still be in
* the old tbl if it exists.
*/
while ((err = __rhashtable_replace_fast(ht, tbl, obj_old,
obj_new, params)) &&
(tbl = rht_dereference_rcu(tbl->future_tbl, ht)))
;
rcu_read_unlock();
return err;
}
/**
* rhltable_walk_enter - Initialise an iterator
* @hlt: Table to walk over
* @iter: Hash table Iterator
*
* This function prepares a hash table walk.
*
* Note that if you restart a walk after rhashtable_walk_stop you
* may see the same object twice. Also, you may miss objects if
* there are removals in between rhashtable_walk_stop and the next
* call to rhashtable_walk_start.
*
* For a completely stable walk you should construct your own data
* structure outside the hash table.
*
* This function may be called from any process context, including
* non-preemptable context, but cannot be called from softirq or
* hardirq context.
*
* You must call rhashtable_walk_exit after this function returns.
*/
static inline void rhltable_walk_enter(struct rhltable *hlt,
struct rhashtable_iter *iter)
{
return rhashtable_walk_enter(&hlt->ht, iter);
}
/**
* rhltable_free_and_destroy - free elements and destroy hash list table
* @hlt: the hash list table to destroy
* @free_fn: callback to release resources of element
* @arg: pointer passed to free_fn
*
* See documentation for rhashtable_free_and_destroy.
*/
static inline void rhltable_free_and_destroy(struct rhltable *hlt,
void (*free_fn)(void *ptr,
void *arg),
void *arg)
{
return rhashtable_free_and_destroy(&hlt->ht, free_fn, arg);
}
static inline void rhltable_destroy(struct rhltable *hlt)
{
return rhltable_free_and_destroy(hlt, NULL, NULL);
}
#endif /* _LINUX_RHASHTABLE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TIMER_H
#define _LINUX_TIMER_H
#include <linux/list.h>
#include <linux/ktime.h>
#include <linux/stddef.h>
#include <linux/debugobjects.h>
#include <linux/stringify.h>
#include <linux/timer_types.h>
#ifdef CONFIG_LOCKDEP
/*
* NB: because we have to copy the lockdep_map, setting the lockdep_map key
* (second argument) here is required, otherwise it could be initialised to
* the copy of the lockdep_map later! We use the pointer to and the string
* "<file>:<line>" as the key resp. the name of the lockdep_map.
*/
#define __TIMER_LOCKDEP_MAP_INITIALIZER(_kn) \
.lockdep_map = STATIC_LOCKDEP_MAP_INIT(_kn, &_kn),
#else
#define __TIMER_LOCKDEP_MAP_INITIALIZER(_kn)
#endif
/*
* @TIMER_DEFERRABLE: A deferrable timer will work normally when the
* system is busy, but will not cause a CPU to come out of idle just
* to service it; instead, the timer will be serviced when the CPU
* eventually wakes up with a subsequent non-deferrable timer.
*
* @TIMER_IRQSAFE: An irqsafe timer is executed with IRQ disabled and
* it's safe to wait for the completion of the running instance from
* IRQ handlers, for example, by calling del_timer_sync().
*
* Note: The irq disabled callback execution is a special case for
* workqueue locking issues. It's not meant for executing random crap
* with interrupts disabled. Abuse is monitored!
*
* @TIMER_PINNED: A pinned timer will always expire on the CPU on which the
* timer was enqueued. When a particular CPU is required, add_timer_on()
* has to be used. Enqueue via mod_timer() and add_timer() is always done
* on the local CPU.
*/
#define TIMER_CPUMASK 0x0003FFFF
#define TIMER_MIGRATING 0x00040000
#define TIMER_BASEMASK (TIMER_CPUMASK | TIMER_MIGRATING)
#define TIMER_DEFERRABLE 0x00080000
#define TIMER_PINNED 0x00100000
#define TIMER_IRQSAFE 0x00200000
#define TIMER_INIT_FLAGS (TIMER_DEFERRABLE | TIMER_PINNED | TIMER_IRQSAFE)
#define TIMER_ARRAYSHIFT 22
#define TIMER_ARRAYMASK 0xFFC00000
#define TIMER_TRACE_FLAGMASK (TIMER_MIGRATING | TIMER_DEFERRABLE | TIMER_PINNED | TIMER_IRQSAFE)
#define __TIMER_INITIALIZER(_function, _flags) { \
.entry = { .next = TIMER_ENTRY_STATIC }, \
.function = (_function), \
.flags = (_flags), \
__TIMER_LOCKDEP_MAP_INITIALIZER(FILE_LINE) \
}
#define DEFINE_TIMER(_name, _function) \
struct timer_list _name = \
__TIMER_INITIALIZER(_function, 0)
/*
* LOCKDEP and DEBUG timer interfaces.
*/
void init_timer_key(struct timer_list *timer,
void (*func)(struct timer_list *), unsigned int flags,
const char *name, struct lock_class_key *key);
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
extern void init_timer_on_stack_key(struct timer_list *timer,
void (*func)(struct timer_list *),
unsigned int flags, const char *name,
struct lock_class_key *key);
#else
static inline void init_timer_on_stack_key(struct timer_list *timer,
void (*func)(struct timer_list *),
unsigned int flags,
const char *name,
struct lock_class_key *key)
{
init_timer_key(timer, func, flags, name, key);
}
#endif
#ifdef CONFIG_LOCKDEP
#define __init_timer(_timer, _fn, _flags) \
do { \
static struct lock_class_key __key; \
init_timer_key((_timer), (_fn), (_flags), #_timer, &__key);\
} while (0)
#define __init_timer_on_stack(_timer, _fn, _flags) \
do { \
static struct lock_class_key __key; \
init_timer_on_stack_key((_timer), (_fn), (_flags), \
#_timer, &__key); \
} while (0)
#else
#define __init_timer(_timer, _fn, _flags) \
init_timer_key((_timer), (_fn), (_flags), NULL, NULL)
#define __init_timer_on_stack(_timer, _fn, _flags) \
init_timer_on_stack_key((_timer), (_fn), (_flags), NULL, NULL)
#endif
/**
* timer_setup - prepare a timer for first use
* @timer: the timer in question
* @callback: the function to call when timer expires
* @flags: any TIMER_* flags
*
* Regular timer initialization should use either DEFINE_TIMER() above,
* or timer_setup(). For timers on the stack, timer_setup_on_stack() must
* be used and must be balanced with a call to destroy_timer_on_stack().
*/
#define timer_setup(timer, callback, flags) \
__init_timer((timer), (callback), (flags))
#define timer_setup_on_stack(timer, callback, flags) \
__init_timer_on_stack((timer), (callback), (flags))
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
extern void destroy_timer_on_stack(struct timer_list *timer);
#else
static inline void destroy_timer_on_stack(struct timer_list *timer) { }
#endif
#define from_timer(var, callback_timer, timer_fieldname) \
container_of(callback_timer, typeof(*var), timer_fieldname)
/**
* timer_pending - is a timer pending?
* @timer: the timer in question
*
* timer_pending will tell whether a given timer is currently pending,
* or not. Callers must ensure serialization wrt. other operations done
* to this timer, eg. interrupt contexts, or other CPUs on SMP.
*
* Returns: 1 if the timer is pending, 0 if not.
*/
static inline int timer_pending(const struct timer_list * timer)
{
return !hlist_unhashed_lockless(&timer->entry);
}
extern void add_timer_on(struct timer_list *timer, int cpu);
extern int mod_timer(struct timer_list *timer, unsigned long expires);
extern int mod_timer_pending(struct timer_list *timer, unsigned long expires);
extern int timer_reduce(struct timer_list *timer, unsigned long expires);
/*
* The jiffies value which is added to now, when there is no timer
* in the timer wheel:
*/
#define NEXT_TIMER_MAX_DELTA ((1UL << 30) - 1)
extern void add_timer(struct timer_list *timer);
extern void add_timer_local(struct timer_list *timer);
extern void add_timer_global(struct timer_list *timer);
extern int try_to_del_timer_sync(struct timer_list *timer);
extern int timer_delete_sync(struct timer_list *timer);
extern int timer_delete(struct timer_list *timer);
extern int timer_shutdown_sync(struct timer_list *timer);
extern int timer_shutdown(struct timer_list *timer);
/**
* del_timer_sync - Delete a pending timer and wait for a running callback
* @timer: The timer to be deleted
*
* See timer_delete_sync() for detailed explanation.
*
* Do not use in new code. Use timer_delete_sync() instead.
*
* Returns:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
*/
static inline int del_timer_sync(struct timer_list *timer)
{
return timer_delete_sync(timer);
}
/**
* del_timer - Delete a pending timer
* @timer: The timer to be deleted
*
* See timer_delete() for detailed explanation.
*
* Do not use in new code. Use timer_delete() instead.
*
* Returns:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
*/
static inline int del_timer(struct timer_list *timer)
{
return timer_delete(timer);
}
extern void init_timers(void);
struct hrtimer;
extern enum hrtimer_restart it_real_fn(struct hrtimer *);
unsigned long __round_jiffies(unsigned long j, int cpu);
unsigned long __round_jiffies_relative(unsigned long j, int cpu);
unsigned long round_jiffies(unsigned long j);
unsigned long round_jiffies_relative(unsigned long j);
unsigned long __round_jiffies_up(unsigned long j, int cpu);
unsigned long __round_jiffies_up_relative(unsigned long j, int cpu);
unsigned long round_jiffies_up(unsigned long j);
unsigned long round_jiffies_up_relative(unsigned long j);
#ifdef CONFIG_HOTPLUG_CPU
int timers_prepare_cpu(unsigned int cpu);
int timers_dead_cpu(unsigned int cpu);
#else
#define timers_prepare_cpu NULL
#define timers_dead_cpu NULL
#endif
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MMU_NOTIFIER_H
#define _LINUX_MMU_NOTIFIER_H
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/mm_types.h>
#include <linux/mmap_lock.h>
#include <linux/srcu.h>
#include <linux/interval_tree.h>
struct mmu_notifier_subscriptions;
struct mmu_notifier;
struct mmu_notifier_range;
struct mmu_interval_notifier;
/**
* enum mmu_notifier_event - reason for the mmu notifier callback
* @MMU_NOTIFY_UNMAP: either munmap() that unmap the range or a mremap() that
* move the range
*
* @MMU_NOTIFY_CLEAR: clear page table entry (many reasons for this like
* madvise() or replacing a page by another one, ...).
*
* @MMU_NOTIFY_PROTECTION_VMA: update is due to protection change for the range
* ie using the vma access permission (vm_page_prot) to update the whole range
* is enough no need to inspect changes to the CPU page table (mprotect()
* syscall)
*
* @MMU_NOTIFY_PROTECTION_PAGE: update is due to change in read/write flag for
* pages in the range so to mirror those changes the user must inspect the CPU
* page table (from the end callback).
*
* @MMU_NOTIFY_SOFT_DIRTY: soft dirty accounting (still same page and same
* access flags). User should soft dirty the page in the end callback to make
* sure that anyone relying on soft dirtiness catch pages that might be written
* through non CPU mappings.
*
* @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
* that the mm refcount is zero and the range is no longer accessible.
*
* @MMU_NOTIFY_MIGRATE: used during migrate_vma_collect() invalidate to signal
* a device driver to possibly ignore the invalidation if the
* owner field matches the driver's device private pgmap owner.
*
* @MMU_NOTIFY_EXCLUSIVE: to signal a device driver that the device will no
* longer have exclusive access to the page. When sent during creation of an
* exclusive range the owner will be initialised to the value provided by the
* caller of make_device_exclusive_range(), otherwise the owner will be NULL.
*/
enum mmu_notifier_event {
MMU_NOTIFY_UNMAP = 0,
MMU_NOTIFY_CLEAR,
MMU_NOTIFY_PROTECTION_VMA,
MMU_NOTIFY_PROTECTION_PAGE,
MMU_NOTIFY_SOFT_DIRTY,
MMU_NOTIFY_RELEASE,
MMU_NOTIFY_MIGRATE,
MMU_NOTIFY_EXCLUSIVE,
};
#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
struct mmu_notifier_ops {
/*
* Called either by mmu_notifier_unregister or when the mm is
* being destroyed by exit_mmap, always before all pages are
* freed. This can run concurrently with other mmu notifier
* methods (the ones invoked outside the mm context) and it
* should tear down all secondary mmu mappings and freeze the
* secondary mmu. If this method isn't implemented you've to
* be sure that nothing could possibly write to the pages
* through the secondary mmu by the time the last thread with
* tsk->mm == mm exits.
*
* As side note: the pages freed after ->release returns could
* be immediately reallocated by the gart at an alias physical
* address with a different cache model, so if ->release isn't
* implemented because all _software_ driven memory accesses
* through the secondary mmu are terminated by the time the
* last thread of this mm quits, you've also to be sure that
* speculative _hardware_ operations can't allocate dirty
* cachelines in the cpu that could not be snooped and made
* coherent with the other read and write operations happening
* through the gart alias address, so leading to memory
* corruption.
*/
void (*release)(struct mmu_notifier *subscription,
struct mm_struct *mm);
/*
* clear_flush_young is called after the VM is
* test-and-clearing the young/accessed bitflag in the
* pte. This way the VM will provide proper aging to the
* accesses to the page through the secondary MMUs and not
* only to the ones through the Linux pte.
* Start-end is necessary in case the secondary MMU is mapping the page
* at a smaller granularity than the primary MMU.
*/
int (*clear_flush_young)(struct mmu_notifier *subscription,
struct mm_struct *mm,
unsigned long start,
unsigned long end);
/*
* clear_young is a lightweight version of clear_flush_young. Like the
* latter, it is supposed to test-and-clear the young/accessed bitflag
* in the secondary pte, but it may omit flushing the secondary tlb.
*/
int (*clear_young)(struct mmu_notifier *subscription,
struct mm_struct *mm,
unsigned long start,
unsigned long end);
/*
* test_young is called to check the young/accessed bitflag in
* the secondary pte. This is used to know if the page is
* frequently used without actually clearing the flag or tearing
* down the secondary mapping on the page.
*/
int (*test_young)(struct mmu_notifier *subscription,
struct mm_struct *mm,
unsigned long address);
/*
* change_pte is called in cases that pte mapping to page is changed:
* for example, when ksm remaps pte to point to a new shared page.
*/
void (*change_pte)(struct mmu_notifier *subscription,
struct mm_struct *mm,
unsigned long address,
pte_t pte);
/*
* invalidate_range_start() and invalidate_range_end() must be
* paired and are called only when the mmap_lock and/or the
* locks protecting the reverse maps are held. If the subsystem
* can't guarantee that no additional references are taken to
* the pages in the range, it has to implement the
* invalidate_range() notifier to remove any references taken
* after invalidate_range_start().
*
* Invalidation of multiple concurrent ranges may be
* optionally permitted by the driver. Either way the
* establishment of sptes is forbidden in the range passed to
* invalidate_range_begin/end for the whole duration of the
* invalidate_range_begin/end critical section.
*
* invalidate_range_start() is called when all pages in the
* range are still mapped and have at least a refcount of one.
*
* invalidate_range_end() is called when all pages in the
* range have been unmapped and the pages have been freed by
* the VM.
*
* The VM will remove the page table entries and potentially
* the page between invalidate_range_start() and
* invalidate_range_end(). If the page must not be freed
* because of pending I/O or other circumstances then the
* invalidate_range_start() callback (or the initial mapping
* by the driver) must make sure that the refcount is kept
* elevated.
*
* If the driver increases the refcount when the pages are
* initially mapped into an address space then either
* invalidate_range_start() or invalidate_range_end() may
* decrease the refcount. If the refcount is decreased on
* invalidate_range_start() then the VM can free pages as page
* table entries are removed. If the refcount is only
* dropped on invalidate_range_end() then the driver itself
* will drop the last refcount but it must take care to flush
* any secondary tlb before doing the final free on the
* page. Pages will no longer be referenced by the linux
* address space but may still be referenced by sptes until
* the last refcount is dropped.
*
* If blockable argument is set to false then the callback cannot
* sleep and has to return with -EAGAIN if sleeping would be required.
* 0 should be returned otherwise. Please note that notifiers that can
* fail invalidate_range_start are not allowed to implement
* invalidate_range_end, as there is no mechanism for informing the
* notifier that its start failed.
*/
int (*invalidate_range_start)(struct mmu_notifier *subscription,
const struct mmu_notifier_range *range);
void (*invalidate_range_end)(struct mmu_notifier *subscription,
const struct mmu_notifier_range *range);
/*
* arch_invalidate_secondary_tlbs() is used to manage a non-CPU TLB
* which shares page-tables with the CPU. The
* invalidate_range_start()/end() callbacks should not be implemented as
* invalidate_secondary_tlbs() already catches the points in time when
* an external TLB needs to be flushed.
*
* This requires arch_invalidate_secondary_tlbs() to be called while
* holding the ptl spin-lock and therefore this callback is not allowed
* to sleep.
*
* This is called by architecture code whenever invalidating a TLB
* entry. It is assumed that any secondary TLB has the same rules for
* when invalidations are required. If this is not the case architecture
* code will need to call this explicitly when required for secondary
* TLB invalidation.
*/
void (*arch_invalidate_secondary_tlbs)(
struct mmu_notifier *subscription,
struct mm_struct *mm,
unsigned long start,
unsigned long end);
/*
* These callbacks are used with the get/put interface to manage the
* lifetime of the mmu_notifier memory. alloc_notifier() returns a new
* notifier for use with the mm.
*
* free_notifier() is only called after the mmu_notifier has been
* fully put, calls to any ops callback are prevented and no ops
* callbacks are currently running. It is called from a SRCU callback
* and cannot sleep.
*/
struct mmu_notifier *(*alloc_notifier)(struct mm_struct *mm);
void (*free_notifier)(struct mmu_notifier *subscription);
};
/*
* The notifier chains are protected by mmap_lock and/or the reverse map
* semaphores. Notifier chains are only changed when all reverse maps and
* the mmap_lock locks are taken.
*
* Therefore notifier chains can only be traversed when either
*
* 1. mmap_lock is held.
* 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem).
* 3. No other concurrent thread can access the list (release)
*/
struct mmu_notifier {
struct hlist_node hlist;
const struct mmu_notifier_ops *ops;
struct mm_struct *mm;
struct rcu_head rcu;
unsigned int users;
};
/**
* struct mmu_interval_notifier_ops
* @invalidate: Upon return the caller must stop using any SPTEs within this
* range. This function can sleep. Return false only if sleeping
* was required but mmu_notifier_range_blockable(range) is false.
*/
struct mmu_interval_notifier_ops {
bool (*invalidate)(struct mmu_interval_notifier *interval_sub,
const struct mmu_notifier_range *range,
unsigned long cur_seq);
};
struct mmu_interval_notifier {
struct interval_tree_node interval_tree;
const struct mmu_interval_notifier_ops *ops;
struct mm_struct *mm;
struct hlist_node deferred_item;
unsigned long invalidate_seq;
};
#ifdef CONFIG_MMU_NOTIFIER
#ifdef CONFIG_LOCKDEP
extern struct lockdep_map __mmu_notifier_invalidate_range_start_map;
#endif
struct mmu_notifier_range {
struct mm_struct *mm;
unsigned long start;
unsigned long end;
unsigned flags;
enum mmu_notifier_event event;
void *owner;
};
static inline int mm_has_notifiers(struct mm_struct *mm)
{
return unlikely(mm->notifier_subscriptions);
}
struct mmu_notifier *mmu_notifier_get_locked(const struct mmu_notifier_ops *ops,
struct mm_struct *mm);
static inline struct mmu_notifier *
mmu_notifier_get(const struct mmu_notifier_ops *ops, struct mm_struct *mm)
{
struct mmu_notifier *ret;
mmap_write_lock(mm);
ret = mmu_notifier_get_locked(ops, mm);
mmap_write_unlock(mm);
return ret;
}
void mmu_notifier_put(struct mmu_notifier *subscription);
void mmu_notifier_synchronize(void);
extern int mmu_notifier_register(struct mmu_notifier *subscription,
struct mm_struct *mm);
extern int __mmu_notifier_register(struct mmu_notifier *subscription,
struct mm_struct *mm);
extern void mmu_notifier_unregister(struct mmu_notifier *subscription,
struct mm_struct *mm);
unsigned long
mmu_interval_read_begin(struct mmu_interval_notifier *interval_sub);
int mmu_interval_notifier_insert(struct mmu_interval_notifier *interval_sub,
struct mm_struct *mm, unsigned long start,
unsigned long length,
const struct mmu_interval_notifier_ops *ops);
int mmu_interval_notifier_insert_locked(
struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,
unsigned long start, unsigned long length,
const struct mmu_interval_notifier_ops *ops);
void mmu_interval_notifier_remove(struct mmu_interval_notifier *interval_sub);
/**
* mmu_interval_set_seq - Save the invalidation sequence
* @interval_sub - The subscription passed to invalidate
* @cur_seq - The cur_seq passed to the invalidate() callback
*
* This must be called unconditionally from the invalidate callback of a
* struct mmu_interval_notifier_ops under the same lock that is used to call
* mmu_interval_read_retry(). It updates the sequence number for later use by
* mmu_interval_read_retry(). The provided cur_seq will always be odd.
*
* If the caller does not call mmu_interval_read_begin() or
* mmu_interval_read_retry() then this call is not required.
*/
static inline void
mmu_interval_set_seq(struct mmu_interval_notifier *interval_sub,
unsigned long cur_seq)
{
WRITE_ONCE(interval_sub->invalidate_seq, cur_seq);
}
/**
* mmu_interval_read_retry - End a read side critical section against a VA range
* interval_sub: The subscription
* seq: The return of the paired mmu_interval_read_begin()
*
* This MUST be called under a user provided lock that is also held
* unconditionally by op->invalidate() when it calls mmu_interval_set_seq().
*
* Each call should be paired with a single mmu_interval_read_begin() and
* should be used to conclude the read side.
*
* Returns true if an invalidation collided with this critical section, and
* the caller should retry.
*/
static inline bool
mmu_interval_read_retry(struct mmu_interval_notifier *interval_sub,
unsigned long seq)
{
return interval_sub->invalidate_seq != seq;
}
/**
* mmu_interval_check_retry - Test if a collision has occurred
* interval_sub: The subscription
* seq: The return of the matching mmu_interval_read_begin()
*
* This can be used in the critical section between mmu_interval_read_begin()
* and mmu_interval_read_retry(). A return of true indicates an invalidation
* has collided with this critical region and a future
* mmu_interval_read_retry() will return true.
*
* False is not reliable and only suggests a collision may not have
* occurred. It can be called many times and does not have to hold the user
* provided lock.
*
* This call can be used as part of loops and other expensive operations to
* expedite a retry.
*/
static inline bool
mmu_interval_check_retry(struct mmu_interval_notifier *interval_sub,
unsigned long seq)
{
/* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
return READ_ONCE(interval_sub->invalidate_seq) != seq;
}
extern void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm);
extern void __mmu_notifier_release(struct mm_struct *mm);
extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
unsigned long start,
unsigned long end);
extern int __mmu_notifier_clear_young(struct mm_struct *mm,
unsigned long start,
unsigned long end);
extern int __mmu_notifier_test_young(struct mm_struct *mm,
unsigned long address);
extern void __mmu_notifier_change_pte(struct mm_struct *mm,
unsigned long address, pte_t pte);
extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r);
extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r);
extern void __mmu_notifier_arch_invalidate_secondary_tlbs(struct mm_struct *mm,
unsigned long start, unsigned long end);
extern bool
mmu_notifier_range_update_to_read_only(const struct mmu_notifier_range *range);
static inline bool
mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
{
return (range->flags & MMU_NOTIFIER_RANGE_BLOCKABLE);
}
static inline void mmu_notifier_release(struct mm_struct *mm)
{
if (mm_has_notifiers(mm))
__mmu_notifier_release(mm);
}
static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
unsigned long start,
unsigned long end)
{
if (mm_has_notifiers(mm))
return __mmu_notifier_clear_flush_young(mm, start, end);
return 0;
}
static inline int mmu_notifier_clear_young(struct mm_struct *mm,
unsigned long start,
unsigned long end)
{
if (mm_has_notifiers(mm))
return __mmu_notifier_clear_young(mm, start, end);
return 0;
}
static inline int mmu_notifier_test_young(struct mm_struct *mm,
unsigned long address)
{
if (mm_has_notifiers(mm))
return __mmu_notifier_test_young(mm, address);
return 0;
}
static inline void mmu_notifier_change_pte(struct mm_struct *mm,
unsigned long address, pte_t pte)
{
if (mm_has_notifiers(mm))
__mmu_notifier_change_pte(mm, address, pte);
}
static inline void
mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
might_sleep();
lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
if (mm_has_notifiers(range->mm)) {
range->flags |= MMU_NOTIFIER_RANGE_BLOCKABLE;
__mmu_notifier_invalidate_range_start(range);
}
lock_map_release(&__mmu_notifier_invalidate_range_start_map);
}
/*
* This version of mmu_notifier_invalidate_range_start() avoids blocking, but it
* can return an error if a notifier can't proceed without blocking, in which
* case you're not allowed to modify PTEs in the specified range.
*
* This is mainly intended for OOM handling.
*/
static inline int __must_check
mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
{
int ret = 0;
lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
if (mm_has_notifiers(range->mm)) {
range->flags &= ~MMU_NOTIFIER_RANGE_BLOCKABLE;
ret = __mmu_notifier_invalidate_range_start(range);
}
lock_map_release(&__mmu_notifier_invalidate_range_start_map);
return ret;
}
static inline void
mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
{
if (mmu_notifier_range_blockable(range)) might_sleep(); if (mm_has_notifiers(range->mm)) __mmu_notifier_invalidate_range_end(range);
}
static inline void mmu_notifier_arch_invalidate_secondary_tlbs(struct mm_struct *mm,
unsigned long start, unsigned long end)
{
if (mm_has_notifiers(mm))
__mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
}
static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm)
{
mm->notifier_subscriptions = NULL;
}
static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
{
if (mm_has_notifiers(mm))
__mmu_notifier_subscriptions_destroy(mm);
}
static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
enum mmu_notifier_event event,
unsigned flags,
struct mm_struct *mm,
unsigned long start,
unsigned long end)
{
range->event = event;
range->mm = mm;
range->start = start;
range->end = end;
range->flags = flags;
}
static inline void mmu_notifier_range_init_owner(
struct mmu_notifier_range *range,
enum mmu_notifier_event event, unsigned int flags,
struct mm_struct *mm, unsigned long start,
unsigned long end, void *owner)
{
mmu_notifier_range_init(range, event, flags, mm, start, end);
range->owner = owner;
}
#define ptep_clear_flush_young_notify(__vma, __address, __ptep) \
({ \
int __young; \
struct vm_area_struct *___vma = __vma; \
unsigned long ___address = __address; \
__young = ptep_clear_flush_young(___vma, ___address, __ptep); \
__young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
___address, \
___address + \
PAGE_SIZE); \
__young; \
})
#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \
({ \
int __young; \
struct vm_area_struct *___vma = __vma; \
unsigned long ___address = __address; \
__young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \
__young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
___address, \
___address + \
PMD_SIZE); \
__young; \
})
#define ptep_clear_young_notify(__vma, __address, __ptep) \
({ \
int __young; \
struct vm_area_struct *___vma = __vma; \
unsigned long ___address = __address; \
__young = ptep_test_and_clear_young(___vma, ___address, __ptep);\
__young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
___address + PAGE_SIZE); \
__young; \
})
#define pmdp_clear_young_notify(__vma, __address, __pmdp) \
({ \
int __young; \
struct vm_area_struct *___vma = __vma; \
unsigned long ___address = __address; \
__young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\
__young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \
___address + PMD_SIZE); \
__young; \
})
/*
* set_pte_at_notify() sets the pte _after_ running the notifier.
* This is safe to start by updating the secondary MMUs, because the primary MMU
* pte invalidate must have already happened with a ptep_clear_flush() before
* set_pte_at_notify() has been invoked. Updating the secondary MMUs first is
* required when we change both the protection of the mapping from read-only to
* read-write and the pfn (like during copy on write page faults). Otherwise the
* old page would remain mapped readonly in the secondary MMUs after the new
* page is already writable by some CPU through the primary MMU.
*/
#define set_pte_at_notify(__mm, __address, __ptep, __pte) \
({ \
struct mm_struct *___mm = __mm; \
unsigned long ___address = __address; \
pte_t ___pte = __pte; \
\
mmu_notifier_change_pte(___mm, ___address, ___pte); \
set_pte_at(___mm, ___address, __ptep, ___pte); \
})
#else /* CONFIG_MMU_NOTIFIER */
struct mmu_notifier_range {
unsigned long start;
unsigned long end;
};
static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
unsigned long start,
unsigned long end)
{
range->start = start;
range->end = end;
}
#define mmu_notifier_range_init(range,event,flags,mm,start,end) \
_mmu_notifier_range_init(range, start, end)
#define mmu_notifier_range_init_owner(range, event, flags, mm, start, \
end, owner) \
_mmu_notifier_range_init(range, start, end)
static inline bool
mmu_notifier_range_blockable(const struct mmu_notifier_range *range)
{
return true;
}
static inline int mm_has_notifiers(struct mm_struct *mm)
{
return 0;
}
static inline void mmu_notifier_release(struct mm_struct *mm)
{
}
static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
unsigned long start,
unsigned long end)
{
return 0;
}
static inline int mmu_notifier_test_young(struct mm_struct *mm,
unsigned long address)
{
return 0;
}
static inline void mmu_notifier_change_pte(struct mm_struct *mm,
unsigned long address, pte_t pte)
{
}
static inline void
mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
{
}
static inline int
mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
{
return 0;
}
static inline
void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
{
}
static inline void mmu_notifier_arch_invalidate_secondary_tlbs(struct mm_struct *mm,
unsigned long start, unsigned long end)
{
}
static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm)
{
}
static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
{
}
#define mmu_notifier_range_update_to_read_only(r) false
#define ptep_clear_flush_young_notify ptep_clear_flush_young
#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
#define ptep_clear_young_notify ptep_test_and_clear_young
#define pmdp_clear_young_notify pmdp_test_and_clear_young
#define ptep_clear_flush_notify ptep_clear_flush
#define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
#define pudp_huge_clear_flush_notify pudp_huge_clear_flush
#define set_pte_at_notify set_pte_at
static inline void mmu_notifier_synchronize(void)
{
}
#endif /* CONFIG_MMU_NOTIFIER */
#endif /* _LINUX_MMU_NOTIFIER_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Generic helpers for smp ipi calls
*
* (C) Jens Axboe <jens.axboe@oracle.com> 2008
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/irq_work.h>
#include <linux/rcupdate.h>
#include <linux/rculist.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/gfp.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/sched.h>
#include <linux/sched/idle.h>
#include <linux/hypervisor.h>
#include <linux/sched/clock.h>
#include <linux/nmi.h>
#include <linux/sched/debug.h>
#include <linux/jump_label.h>
#include <trace/events/ipi.h>
#define CREATE_TRACE_POINTS
#include <trace/events/csd.h>
#undef CREATE_TRACE_POINTS
#include "smpboot.h"
#include "sched/smp.h"
#define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
struct call_function_data {
call_single_data_t __percpu *csd;
cpumask_var_t cpumask;
cpumask_var_t cpumask_ipi;
};
static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);
static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1);
static void __flush_smp_call_function_queue(bool warn_cpu_offline);
int smpcfd_prepare_cpu(unsigned int cpu)
{
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
cpu_to_node(cpu)))
return -ENOMEM;
if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
cpu_to_node(cpu))) {
free_cpumask_var(cfd->cpumask);
return -ENOMEM;
}
cfd->csd = alloc_percpu(call_single_data_t);
if (!cfd->csd) {
free_cpumask_var(cfd->cpumask);
free_cpumask_var(cfd->cpumask_ipi);
return -ENOMEM;
}
return 0;
}
int smpcfd_dead_cpu(unsigned int cpu)
{
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
free_cpumask_var(cfd->cpumask);
free_cpumask_var(cfd->cpumask_ipi);
free_percpu(cfd->csd);
return 0;
}
int smpcfd_dying_cpu(unsigned int cpu)
{
/*
* The IPIs for the smp-call-function callbacks queued by other
* CPUs might arrive late, either due to hardware latencies or
* because this CPU disabled interrupts (inside stop-machine)
* before the IPIs were sent. So flush out any pending callbacks
* explicitly (without waiting for the IPIs to arrive), to
* ensure that the outgoing CPU doesn't go offline with work
* still pending.
*/
__flush_smp_call_function_queue(false);
irq_work_run();
return 0;
}
void __init call_function_init(void)
{
int i;
for_each_possible_cpu(i)
init_llist_head(&per_cpu(call_single_queue, i));
smpcfd_prepare_cpu(smp_processor_id());
}
static __always_inline void
send_call_function_single_ipi(int cpu)
{
if (call_function_single_prep_ipi(cpu)) { trace_ipi_send_cpu(cpu, _RET_IP_,
generic_smp_call_function_single_interrupt);
arch_send_call_function_single_ipi(cpu);
}
}
static __always_inline void
send_call_function_ipi_mask(struct cpumask *mask)
{
trace_ipi_send_cpumask(mask, _RET_IP_,
generic_smp_call_function_single_interrupt);
arch_send_call_function_ipi_mask(mask);
}
static __always_inline void
csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd)
{
trace_csd_function_entry(func, csd);
func(info);
trace_csd_function_exit(func, csd);
}
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
/*
* Parse the csdlock_debug= kernel boot parameter.
*
* If you need to restore the old "ext" value that once provided
* additional debugging information, reapply the following commits:
*
* de7b09ef658d ("locking/csd_lock: Prepare more CSD lock debugging")
* a5aabace5fb8 ("locking/csd_lock: Add more data to CSD lock debugging")
*/
static int __init csdlock_debug(char *str)
{
int ret;
unsigned int val = 0;
ret = get_option(&str, &val);
if (ret) {
if (val)
static_branch_enable(&csdlock_debug_enabled);
else
static_branch_disable(&csdlock_debug_enabled);
}
return 1;
}
__setup("csdlock_debug=", csdlock_debug);
static DEFINE_PER_CPU(call_single_data_t *, cur_csd);
static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func);
static DEFINE_PER_CPU(void *, cur_csd_info);
static ulong csd_lock_timeout = 5000; /* CSD lock timeout in milliseconds. */
module_param(csd_lock_timeout, ulong, 0444);
static int panic_on_ipistall; /* CSD panic timeout in milliseconds, 300000 for five minutes. */
module_param(panic_on_ipistall, int, 0444);
static atomic_t csd_bug_count = ATOMIC_INIT(0);
/* Record current CSD work for current CPU, NULL to erase. */
static void __csd_lock_record(call_single_data_t *csd)
{
if (!csd) {
smp_mb(); /* NULL cur_csd after unlock. */
__this_cpu_write(cur_csd, NULL);
return;
}
__this_cpu_write(cur_csd_func, csd->func);
__this_cpu_write(cur_csd_info, csd->info);
smp_wmb(); /* func and info before csd. */
__this_cpu_write(cur_csd, csd);
smp_mb(); /* Update cur_csd before function call. */
/* Or before unlock, as the case may be. */
}
static __always_inline void csd_lock_record(call_single_data_t *csd)
{
if (static_branch_unlikely(&csdlock_debug_enabled)) __csd_lock_record(csd);
}
static int csd_lock_wait_getcpu(call_single_data_t *csd)
{
unsigned int csd_type;
csd_type = CSD_TYPE(csd);
if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC)
return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */
return -1;
}
/*
* Complain if too much time spent waiting. Note that only
* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
* so waiting on other types gets much less information.
*/
static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id)
{
int cpu = -1;
int cpux;
bool firsttime;
u64 ts2, ts_delta;
call_single_data_t *cpu_cur_csd;
unsigned int flags = READ_ONCE(csd->node.u_flags);
unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC;
if (!(flags & CSD_FLAG_LOCK)) {
if (!unlikely(*bug_id))
return true;
cpu = csd_lock_wait_getcpu(csd);
pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
*bug_id, raw_smp_processor_id(), cpu);
return true;
}
ts2 = sched_clock();
/* How long since we last checked for a stuck CSD lock.*/
ts_delta = ts2 - *ts1;
if (likely(ts_delta <= csd_lock_timeout_ns || csd_lock_timeout_ns == 0))
return false;
firsttime = !*bug_id;
if (firsttime)
*bug_id = atomic_inc_return(&csd_bug_count);
cpu = csd_lock_wait_getcpu(csd);
if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu))
cpux = 0;
else
cpux = cpu;
cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
/* How long since this CSD lock was stuck. */
ts_delta = ts2 - ts0;
pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %llu ns for CPU#%02d %pS(%ps).\n",
firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), ts_delta,
cpu, csd->func, csd->info);
/*
* If the CSD lock is still stuck after 5 minutes, it is unlikely
* to become unstuck. Use a signed comparison to avoid triggering
* on underflows when the TSC is out of sync between sockets.
*/
BUG_ON(panic_on_ipistall > 0 && (s64)ts_delta > ((s64)panic_on_ipistall * NSEC_PER_MSEC));
if (cpu_cur_csd && csd != cpu_cur_csd) {
pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n",
*bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
READ_ONCE(per_cpu(cur_csd_info, cpux)));
} else {
pr_alert("\tcsd: CSD lock (#%d) %s.\n",
*bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
}
if (cpu >= 0) {
if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0))
dump_cpu_task(cpu);
if (!cpu_cur_csd) {
pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
arch_send_call_function_single_ipi(cpu);
}
}
if (firsttime)
dump_stack();
*ts1 = ts2;
return false;
}
/*
* csd_lock/csd_unlock used to serialize access to per-cpu csd resources
*
* For non-synchronous ipi calls the csd can still be in use by the
* previous function call. For multi-cpu calls its even more interesting
* as we'll have to ensure no other cpu is observing our csd.
*/
static void __csd_lock_wait(call_single_data_t *csd)
{
int bug_id = 0;
u64 ts0, ts1;
ts1 = ts0 = sched_clock();
for (;;) {
if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id))
break;
cpu_relax();
}
smp_acquire__after_ctrl_dep();
}
static __always_inline void csd_lock_wait(call_single_data_t *csd)
{
if (static_branch_unlikely(&csdlock_debug_enabled)) { __csd_lock_wait(csd);
return;
}
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
}
#else
static void csd_lock_record(call_single_data_t *csd)
{
}
static __always_inline void csd_lock_wait(call_single_data_t *csd)
{
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
}
#endif
static __always_inline void csd_lock(call_single_data_t *csd)
{
csd_lock_wait(csd); csd->node.u_flags |= CSD_FLAG_LOCK;
/*
* prevent CPU from reordering the above assignment
* to ->flags with any subsequent assignments to other
* fields of the specified call_single_data_t structure:
*/
smp_wmb();
}
static __always_inline void csd_unlock(call_single_data_t *csd)
{
WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
/*
* ensure we're all done before releasing data:
*/
smp_store_release(&csd->node.u_flags, 0);
}
static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
void __smp_call_single_queue(int cpu, struct llist_node *node)
{
/*
* We have to check the type of the CSD before queueing it, because
* once queued it can have its flags cleared by
* flush_smp_call_function_queue()
* even if we haven't sent the smp_call IPI yet (e.g. the stopper
* executes migration_cpu_stop() on the remote CPU).
*/
if (trace_csd_queue_cpu_enabled()) {
call_single_data_t *csd;
smp_call_func_t func;
csd = container_of(node, call_single_data_t, node.llist);
func = CSD_TYPE(csd) == CSD_TYPE_TTWU ? sched_ttwu_pending : csd->func; trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);
}
/*
* The list addition should be visible to the target CPU when it pops
* the head of the list to pull the entry off it in the IPI handler
* because of normal cache coherency rules implied by the underlying
* llist ops.
*
* If IPIs can go out of order to the cache coherency protocol
* in an architecture, sufficient synchronisation should be added
* to arch code to make it appear to obey cache coherency WRT
* locking and barrier primitives. Generic code isn't really
* equipped to do the right thing...
*/
if (llist_add(node, &per_cpu(call_single_queue, cpu))) send_call_function_single_ipi(cpu);
}
/*
* Insert a previously allocated call_single_data_t element
* for execution on the given CPU. data must already have
* ->func, ->info, and ->flags set.
*/
static int generic_exec_single(int cpu, call_single_data_t *csd)
{
if (cpu == smp_processor_id()) { smp_call_func_t func = csd->func;
void *info = csd->info;
unsigned long flags;
/*
* We can unlock early even for the synchronous on-stack case,
* since we're doing this from the same CPU..
*/
csd_lock_record(csd); csd_unlock(csd); local_irq_save(flags); csd_do_func(func, info, NULL); csd_lock_record(NULL); local_irq_restore(flags); return 0;
}
if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) { csd_unlock(csd);
return -ENXIO;
}
__smp_call_single_queue(cpu, &csd->node.llist);
return 0;
}
/**
* generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
*
* Invoked by arch to handle an IPI for call function single.
* Must be called with interrupts disabled.
*/
void generic_smp_call_function_single_interrupt(void)
{
__flush_smp_call_function_queue(true);
}
/**
* __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
*
* @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
* offline CPU. Skip this check if set to 'false'.
*
* Flush any pending smp-call-function callbacks queued on this CPU. This is
* invoked by the generic IPI handler, as well as by a CPU about to go offline,
* to ensure that all pending IPI callbacks are run before it goes completely
* offline.
*
* Loop through the call_single_queue and run all the queued callbacks.
* Must be called with interrupts disabled.
*/
static void __flush_smp_call_function_queue(bool warn_cpu_offline)
{
call_single_data_t *csd, *csd_next;
struct llist_node *entry, *prev;
struct llist_head *head;
static bool warned;
atomic_t *tbt;
lockdep_assert_irqs_disabled();
/* Allow waiters to send backtrace NMI from here onwards */
tbt = this_cpu_ptr(&trigger_backtrace);
atomic_set_release(tbt, 1);
head = this_cpu_ptr(&call_single_queue);
entry = llist_del_all(head);
entry = llist_reverse_order(entry);
/* There shouldn't be any pending callbacks on an offline CPU. */
if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
!warned && entry != NULL)) {
warned = true;
WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
/*
* We don't have to use the _safe() variant here
* because we are not invoking the IPI handlers yet.
*/
llist_for_each_entry(csd, entry, node.llist) { switch (CSD_TYPE(csd)) {
case CSD_TYPE_ASYNC:
case CSD_TYPE_SYNC:
case CSD_TYPE_IRQ_WORK:
pr_warn("IPI callback %pS sent to offline CPU\n",
csd->func);
break;
case CSD_TYPE_TTWU:
pr_warn("IPI task-wakeup sent to offline CPU\n");
break;
default:
pr_warn("IPI callback, unknown type %d, sent to offline CPU\n",
CSD_TYPE(csd));
break;
}
}
}
/*
* First; run all SYNC callbacks, people are waiting for us.
*/
prev = NULL;
llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
/* Do we wait until *after* callback? */
if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
smp_call_func_t func = csd->func;
void *info = csd->info;
if (prev) {
prev->next = &csd_next->node.llist;
} else {
entry = &csd_next->node.llist;
}
csd_lock_record(csd); csd_do_func(func, info, csd); csd_unlock(csd); csd_lock_record(NULL);
} else {
prev = &csd->node.llist;
}
}
if (!entry)
return;
/*
* Second; run all !SYNC callbacks.
*/
prev = NULL;
llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {
int type = CSD_TYPE(csd);
if (type != CSD_TYPE_TTWU) {
if (prev) {
prev->next = &csd_next->node.llist;
} else {
entry = &csd_next->node.llist;
}
if (type == CSD_TYPE_ASYNC) { smp_call_func_t func = csd->func;
void *info = csd->info;
csd_lock_record(csd); csd_unlock(csd);
csd_do_func(func, info, csd);
csd_lock_record(NULL); } else if (type == CSD_TYPE_IRQ_WORK) { irq_work_single(csd);
}
} else {
prev = &csd->node.llist;
}
}
/*
* Third; only CSD_TYPE_TTWU is left, issue those.
*/
if (entry) {
csd = llist_entry(entry, typeof(*csd), node.llist);
csd_do_func(sched_ttwu_pending, entry, csd);
}
}
/**
* flush_smp_call_function_queue - Flush pending smp-call-function callbacks
* from task context (idle, migration thread)
*
* When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
* set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
* setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
* handle queued SMP function calls before scheduling.
*
* The migration thread has to ensure that an eventually pending wakeup has
* been handled before it migrates a task.
*/
void flush_smp_call_function_queue(void)
{
unsigned int was_pending;
unsigned long flags;
if (llist_empty(this_cpu_ptr(&call_single_queue)))
return;
local_irq_save(flags);
/* Get the already pending soft interrupts for RT enabled kernels */
was_pending = local_softirq_pending();
__flush_smp_call_function_queue(true);
if (local_softirq_pending())
do_softirq_post_smp_call_flush(was_pending);
local_irq_restore(flags);
}
/*
* smp_call_function_single - Run a function on a specific CPU
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @wait: If true, wait until function has completed on other CPUs.
*
* Returns 0 on success, else a negative status code.
*/
int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
int wait)
{
call_single_data_t *csd;
call_single_data_t csd_stack = {
.node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, },
};
int this_cpu;
int err;
/*
* prevent preemption and reschedule on another processor,
* as well as CPU removal
*/
this_cpu = get_cpu();
/*
* Can deadlock when called with interrupts disabled.
* We allow cpu's that are not yet online though, as no one else can
* send smp call function interrupt to this cpu and as such deadlocks
* can't happen.
*/
WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
&& !oops_in_progress);
/*
* When @wait we can deadlock when we interrupt between llist_add() and
* arch_send_call_function_ipi*(); when !@wait we can deadlock due to
* csd_lock() on because the interrupt context uses the same csd
* storage.
*/
WARN_ON_ONCE(!in_task());
csd = &csd_stack;
if (!wait) {
csd = this_cpu_ptr(&csd_data); csd_lock(csd);
}
csd->func = func;
csd->info = info;
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
csd->node.src = smp_processor_id();
csd->node.dst = cpu;
#endif
err = generic_exec_single(cpu, csd);
if (wait)
csd_lock_wait(csd); put_cpu(); return err;
}
EXPORT_SYMBOL(smp_call_function_single);
/**
* smp_call_function_single_async() - Run an asynchronous function on a
* specific CPU.
* @cpu: The CPU to run on.
* @csd: Pre-allocated and setup data structure
*
* Like smp_call_function_single(), but the call is asynchonous and
* can thus be done from contexts with disabled interrupts.
*
* The caller passes his own pre-allocated data structure
* (ie: embedded in an object) and is responsible for synchronizing it
* such that the IPIs performed on the @csd are strictly serialized.
*
* If the function is called with one csd which has not yet been
* processed by previous call to smp_call_function_single_async(), the
* function will return immediately with -EBUSY showing that the csd
* object is still in progress.
*
* NOTE: Be careful, there is unfortunately no current debugging facility to
* validate the correctness of this serialization.
*
* Return: %0 on success or negative errno value on error
*/
int smp_call_function_single_async(int cpu, call_single_data_t *csd)
{
int err = 0;
preempt_disable();
if (csd->node.u_flags & CSD_FLAG_LOCK) {
err = -EBUSY;
goto out;
}
csd->node.u_flags = CSD_FLAG_LOCK;
smp_wmb();
err = generic_exec_single(cpu, csd);
out:
preempt_enable();
return err;
}
EXPORT_SYMBOL_GPL(smp_call_function_single_async);
/*
* smp_call_function_any - Run a function on any of the given cpus
* @mask: The mask of cpus it can run on.
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @wait: If true, wait until function has completed.
*
* Returns 0 on success, else a negative status code (if no cpus were online).
*
* Selection preference:
* 1) current cpu if in @mask
* 2) any cpu of current node if in @mask
* 3) any other online cpu in @mask
*/
int smp_call_function_any(const struct cpumask *mask,
smp_call_func_t func, void *info, int wait)
{
unsigned int cpu;
const struct cpumask *nodemask;
int ret;
/* Try for same CPU (cheapest) */
cpu = get_cpu();
if (cpumask_test_cpu(cpu, mask))
goto call;
/* Try for same node. */
nodemask = cpumask_of_node(cpu_to_node(cpu));
for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
cpu = cpumask_next_and(cpu, nodemask, mask)) {
if (cpu_online(cpu))
goto call;
}
/* Any online will do: smp_call_function_single handles nr_cpu_ids. */
cpu = cpumask_any_and(mask, cpu_online_mask);
call:
ret = smp_call_function_single(cpu, func, info, wait);
put_cpu();
return ret;
}
EXPORT_SYMBOL_GPL(smp_call_function_any);
/*
* Flags to be used as scf_flags argument of smp_call_function_many_cond().
*
* %SCF_WAIT: Wait until function execution is completed
* %SCF_RUN_LOCAL: Run also locally if local cpu is set in cpumask
*/
#define SCF_WAIT (1U << 0)
#define SCF_RUN_LOCAL (1U << 1)
static void smp_call_function_many_cond(const struct cpumask *mask,
smp_call_func_t func, void *info,
unsigned int scf_flags,
smp_cond_func_t cond_func)
{
int cpu, last_cpu, this_cpu = smp_processor_id(); struct call_function_data *cfd;
bool wait = scf_flags & SCF_WAIT;
int nr_cpus = 0;
bool run_remote = false;
bool run_local = false;
lockdep_assert_preemption_disabled();
/*
* Can deadlock when called with interrupts disabled.
* We allow cpu's that are not yet online though, as no one else can
* send smp call function interrupt to this cpu and as such deadlocks
* can't happen.
*/
if (cpu_online(this_cpu) && !oops_in_progress && !early_boot_irqs_disabled) lockdep_assert_irqs_enabled();
/*
* When @wait we can deadlock when we interrupt between llist_add() and
* arch_send_call_function_ipi*(); when !@wait we can deadlock due to
* csd_lock() on because the interrupt context uses the same csd
* storage.
*/
WARN_ON_ONCE(!in_task());
/* Check if we need local execution. */
if ((scf_flags & SCF_RUN_LOCAL) && cpumask_test_cpu(this_cpu, mask))
run_local = true;
/* Check if we need remote execution, i.e., any CPU excluding this one. */
cpu = cpumask_first_and(mask, cpu_online_mask); if (cpu == this_cpu) cpu = cpumask_next_and(cpu, mask, cpu_online_mask); if (cpu < nr_cpu_ids)
run_remote = true;
if (run_remote) {
cfd = this_cpu_ptr(&cfd_data);
cpumask_and(cfd->cpumask, mask, cpu_online_mask);
__cpumask_clear_cpu(this_cpu, cfd->cpumask);
cpumask_clear(cfd->cpumask_ipi);
for_each_cpu(cpu, cfd->cpumask) { call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu); if (cond_func && !cond_func(cpu, info)) { __cpumask_clear_cpu(cpu, cfd->cpumask);
continue;
}
csd_lock(csd);
if (wait)
csd->node.u_flags |= CSD_TYPE_SYNC; csd->func = func;
csd->info = info;
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
csd->node.src = smp_processor_id();
csd->node.dst = cpu;
#endif
trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);
if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {
__cpumask_set_cpu(cpu, cfd->cpumask_ipi);
nr_cpus++;
last_cpu = cpu;
}
}
/*
* Choose the most efficient way to send an IPI. Note that the
* number of CPUs might be zero due to concurrent changes to the
* provided mask.
*/
if (nr_cpus == 1) send_call_function_single_ipi(last_cpu); else if (likely(nr_cpus > 1)) send_call_function_ipi_mask(cfd->cpumask_ipi);
}
if (run_local && (!cond_func || cond_func(this_cpu, info))) {
unsigned long flags;
local_irq_save(flags); csd_do_func(func, info, NULL); local_irq_restore(flags);
}
if (run_remote && wait) { for_each_cpu(cpu, cfd->cpumask) {
call_single_data_t *csd;
csd = per_cpu_ptr(cfd->csd, cpu); csd_lock_wait(csd);
}
}
}
/**
* smp_call_function_many(): Run a function on a set of CPUs.
* @mask: The set of cpus to run on (only runs on online subset).
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @wait: Bitmask that controls the operation. If %SCF_WAIT is set, wait
* (atomically) until function has completed on other CPUs. If
* %SCF_RUN_LOCAL is set, the function will also be run locally
* if the local CPU is set in the @cpumask.
*
* If @wait is true, then returns once @func has returned.
*
* You must not call this function with disabled interrupts or from a
* hardware interrupt handler or from a bottom half handler. Preemption
* must be disabled when calling this function.
*/
void smp_call_function_many(const struct cpumask *mask,
smp_call_func_t func, void *info, bool wait)
{
smp_call_function_many_cond(mask, func, info, wait * SCF_WAIT, NULL);
}
EXPORT_SYMBOL(smp_call_function_many);
/**
* smp_call_function(): Run a function on all other CPUs.
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @wait: If true, wait (atomically) until function has completed
* on other CPUs.
*
* Returns 0.
*
* If @wait is true, then returns once @func has returned; otherwise
* it returns just before the target cpu calls @func.
*
* You must not call this function with disabled interrupts or from a
* hardware interrupt handler or from a bottom half handler.
*/
void smp_call_function(smp_call_func_t func, void *info, int wait)
{
preempt_disable();
smp_call_function_many(cpu_online_mask, func, info, wait);
preempt_enable();
}
EXPORT_SYMBOL(smp_call_function);
/* Setup configured maximum number of CPUs to activate */
unsigned int setup_max_cpus = NR_CPUS;
EXPORT_SYMBOL(setup_max_cpus);
/*
* Setup routine for controlling SMP activation
*
* Command-line option of "nosmp" or "maxcpus=0" will disable SMP
* activation entirely (the MPS table probe still happens, though).
*
* Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
* greater than 0, limits the maximum number of CPUs activated in
* SMP mode to <NUM>.
*/
void __weak __init arch_disable_smp_support(void) { }
static int __init nosmp(char *str)
{
setup_max_cpus = 0;
arch_disable_smp_support();
return 0;
}
early_param("nosmp", nosmp);
/* this is hard limit */
static int __init nrcpus(char *str)
{
int nr_cpus;
if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids)
set_nr_cpu_ids(nr_cpus);
return 0;
}
early_param("nr_cpus", nrcpus);
static int __init maxcpus(char *str)
{
get_option(&str, &setup_max_cpus);
if (setup_max_cpus == 0)
arch_disable_smp_support();
return 0;
}
early_param("maxcpus", maxcpus);
#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS)
/* Setup number of possible processor ids */
unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);
#endif
/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
void __init setup_nr_cpu_ids(void)
{
set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
}
/* Called by boot processor to activate the rest. */
void __init smp_init(void)
{
int num_nodes, num_cpus;
idle_threads_init();
cpuhp_threads_init();
pr_info("Bringing up secondary CPUs ...\n");
bringup_nonboot_cpus(setup_max_cpus);
num_nodes = num_online_nodes();
num_cpus = num_online_cpus();
pr_info("Brought up %d node%s, %d CPU%s\n",
num_nodes, (num_nodes > 1 ? "s" : ""),
num_cpus, (num_cpus > 1 ? "s" : ""));
/* Any cleanup work */
smp_cpus_done(setup_max_cpus);
}
/*
* on_each_cpu_cond(): Call a function on each processor for which
* the supplied function cond_func returns true, optionally waiting
* for all the required CPUs to finish. This may include the local
* processor.
* @cond_func: A callback function that is passed a cpu id and
* the info parameter. The function is called
* with preemption disabled. The function should
* return a blooean value indicating whether to IPI
* the specified CPU.
* @func: The function to run on all applicable CPUs.
* This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to both functions.
* @wait: If true, wait (atomically) until function has
* completed on other CPUs.
*
* Preemption is disabled to protect against CPUs going offline but not online.
* CPUs going online during the call will not be seen or sent an IPI.
*
* You must not call this function with disabled interrupts or
* from a hardware interrupt handler or from a bottom half handler.
*/
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
void *info, bool wait, const struct cpumask *mask)
{
unsigned int scf_flags = SCF_RUN_LOCAL;
if (wait)
scf_flags |= SCF_WAIT;
preempt_disable();
smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
preempt_enable();
}
EXPORT_SYMBOL(on_each_cpu_cond_mask);
static void do_nothing(void *unused)
{
}
/**
* kick_all_cpus_sync - Force all cpus out of idle
*
* Used to synchronize the update of pm_idle function pointer. It's
* called after the pointer is updated and returns after the dummy
* callback function has been executed on all cpus. The execution of
* the function can only happen on the remote cpus after they have
* left the idle function which had been called via pm_idle function
* pointer. So it's guaranteed that nothing uses the previous pointer
* anymore.
*/
void kick_all_cpus_sync(void)
{
/* Make sure the change is visible before we kick the cpus */
smp_mb();
smp_call_function(do_nothing, NULL, 1);
}
EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
/**
* wake_up_all_idle_cpus - break all cpus out of idle
* wake_up_all_idle_cpus try to break all cpus which is in idle state even
* including idle polling cpus, for non-idle cpus, we will do nothing
* for them.
*/
void wake_up_all_idle_cpus(void)
{
int cpu;
for_each_possible_cpu(cpu) {
preempt_disable();
if (cpu != smp_processor_id() && cpu_online(cpu))
wake_up_if_idle(cpu);
preempt_enable();
}
}
EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
/**
* struct smp_call_on_cpu_struct - Call a function on a specific CPU
* @work: &work_struct
* @done: &completion to signal
* @func: function to call
* @data: function's data argument
* @ret: return value from @func
* @cpu: target CPU (%-1 for any CPU)
*
* Used to call a function on a specific cpu and wait for it to return.
* Optionally make sure the call is done on a specified physical cpu via vcpu
* pinning in order to support virtualized environments.
*/
struct smp_call_on_cpu_struct {
struct work_struct work;
struct completion done;
int (*func)(void *);
void *data;
int ret;
int cpu;
};
static void smp_call_on_cpu_callback(struct work_struct *work)
{
struct smp_call_on_cpu_struct *sscs;
sscs = container_of(work, struct smp_call_on_cpu_struct, work);
if (sscs->cpu >= 0)
hypervisor_pin_vcpu(sscs->cpu);
sscs->ret = sscs->func(sscs->data);
if (sscs->cpu >= 0)
hypervisor_pin_vcpu(-1);
complete(&sscs->done);
}
int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
{
struct smp_call_on_cpu_struct sscs = {
.done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
.func = func,
.data = par,
.cpu = phys ? cpu : -1,
};
INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
if (cpu >= nr_cpu_ids || !cpu_online(cpu))
return -ENXIO;
queue_work_on(cpu, system_wq, &sscs.work);
wait_for_completion(&sscs.done);
return sscs.ret;
}
EXPORT_SYMBOL_GPL(smp_call_on_cpu);
// SPDX-License-Identifier: GPL-2.0
/*
* DMA memory management for framework level HCD code (hc_driver)
*
* This implementation plugs in through generic "usb_bus" level methods,
* and should work with all USB controllers, regardless of bus type.
*
* Released under the GPLv2 only.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/device.h>
#include <linux/mm.h>
#include <linux/io.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/genalloc.h>
#include <linux/usb.h>
#include <linux/usb/hcd.h>
/*
* DMA-Coherent Buffers
*/
/* FIXME tune these based on pool statistics ... */
static size_t pool_max[HCD_BUFFER_POOLS] = {
32, 128, 512, 2048,
};
void __init usb_init_pool_max(void)
{
/*
* The pool_max values must never be smaller than
* ARCH_DMA_MINALIGN.
*/
if (ARCH_DMA_MINALIGN <= 32)
; /* Original value is okay */
else if (ARCH_DMA_MINALIGN <= 64)
pool_max[0] = 64;
else if (ARCH_DMA_MINALIGN <= 128)
pool_max[0] = 0; /* Don't use this pool */
else
BUILD_BUG(); /* We don't allow this */
}
/* SETUP primitives */
/**
* hcd_buffer_create - initialize buffer pools
* @hcd: the bus whose buffer pools are to be initialized
*
* Context: task context, might sleep
*
* Call this as part of initializing a host controller that uses the dma
* memory allocators. It initializes some pools of dma-coherent memory that
* will be shared by all drivers using that controller.
*
* Call hcd_buffer_destroy() to clean up after using those pools.
*
* Return: 0 if successful. A negative errno value otherwise.
*/
int hcd_buffer_create(struct usb_hcd *hcd)
{
char name[16];
int i, size;
if (hcd->localmem_pool || !hcd_uses_dma(hcd))
return 0;
for (i = 0; i < HCD_BUFFER_POOLS; i++) {
size = pool_max[i];
if (!size)
continue;
snprintf(name, sizeof(name), "buffer-%d", size);
hcd->pool[i] = dma_pool_create(name, hcd->self.sysdev,
size, size, 0);
if (!hcd->pool[i]) {
hcd_buffer_destroy(hcd);
return -ENOMEM;
}
}
return 0;
}
/**
* hcd_buffer_destroy - deallocate buffer pools
* @hcd: the bus whose buffer pools are to be destroyed
*
* Context: task context, might sleep
*
* This frees the buffer pools created by hcd_buffer_create().
*/
void hcd_buffer_destroy(struct usb_hcd *hcd)
{
int i;
if (!IS_ENABLED(CONFIG_HAS_DMA))
return;
for (i = 0; i < HCD_BUFFER_POOLS; i++) {
dma_pool_destroy(hcd->pool[i]);
hcd->pool[i] = NULL;
}
}
/* sometimes alloc/free could use kmalloc with GFP_DMA, for
* better sharing and to leverage mm/slab.c intelligence.
*/
void *hcd_buffer_alloc(
struct usb_bus *bus,
size_t size,
gfp_t mem_flags,
dma_addr_t *dma
)
{
struct usb_hcd *hcd = bus_to_hcd(bus);
int i;
if (size == 0)
return NULL;
if (hcd->localmem_pool)
return gen_pool_dma_alloc(hcd->localmem_pool, size, dma);
/* some USB hosts just use PIO */
if (!hcd_uses_dma(hcd)) {
*dma = ~(dma_addr_t) 0;
return kmalloc(size, mem_flags);
}
for (i = 0; i < HCD_BUFFER_POOLS; i++) {
if (size <= pool_max[i])
return dma_pool_alloc(hcd->pool[i], mem_flags, dma);
}
return dma_alloc_coherent(hcd->self.sysdev, size, dma, mem_flags);
}
void hcd_buffer_free(
struct usb_bus *bus,
size_t size,
void *addr,
dma_addr_t dma
)
{
struct usb_hcd *hcd = bus_to_hcd(bus);
int i;
if (!addr)
return;
if (hcd->localmem_pool) { gen_pool_free(hcd->localmem_pool, (unsigned long)addr, size);
return;
}
if (!hcd_uses_dma(hcd)) { kfree(addr);
return;
}
for (i = 0; i < HCD_BUFFER_POOLS; i++) {
if (size <= pool_max[i]) { dma_pool_free(hcd->pool[i], addr, dma);
return;
}
}
dma_free_coherent(hcd->self.sysdev, size, addr, dma);
}
void *hcd_buffer_alloc_pages(struct usb_hcd *hcd,
size_t size, gfp_t mem_flags, dma_addr_t *dma)
{
if (size == 0)
return NULL;
if (hcd->localmem_pool)
return gen_pool_dma_alloc_align(hcd->localmem_pool,
size, dma, PAGE_SIZE);
/* some USB hosts just use PIO */
if (!hcd_uses_dma(hcd)) {
*dma = DMA_MAPPING_ERROR;
return (void *)__get_free_pages(mem_flags,
get_order(size));
}
return dma_alloc_coherent(hcd->self.sysdev,
size, dma, mem_flags);
}
void hcd_buffer_free_pages(struct usb_hcd *hcd,
size_t size, void *addr, dma_addr_t dma)
{
if (!addr)
return;
if (hcd->localmem_pool) {
gen_pool_free(hcd->localmem_pool,
(unsigned long)addr, size);
return;
}
if (!hcd_uses_dma(hcd)) {
free_pages((unsigned long)addr, get_order(size));
return;
}
dma_free_coherent(hcd->self.sysdev, size, addr, dma);
}
/*
* include/linux/topology.h
*
* Written by: Matthew Dobson, IBM Corporation
*
* Copyright (C) 2002, IBM Corp.
*
* All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
* NON INFRINGEMENT. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Send feedback to <colpatch@us.ibm.com>
*/
#ifndef _LINUX_TOPOLOGY_H
#define _LINUX_TOPOLOGY_H
#include <linux/arch_topology.h>
#include <linux/cpumask.h>
#include <linux/bitops.h>
#include <linux/mmzone.h>
#include <linux/smp.h>
#include <linux/percpu.h>
#include <asm/topology.h>
#ifndef nr_cpus_node
#define nr_cpus_node(node) cpumask_weight(cpumask_of_node(node))
#endif
#define for_each_node_with_cpus(node) \
for_each_online_node(node) \
if (nr_cpus_node(node))
int arch_update_cpu_topology(void);
/* Conform to ACPI 2.0 SLIT distance definitions */
#define LOCAL_DISTANCE 10
#define REMOTE_DISTANCE 20
#define DISTANCE_BITS 8
#ifndef node_distance
#define node_distance(from,to) ((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
#endif
#ifndef RECLAIM_DISTANCE
/*
* If the distance between nodes in a system is larger than RECLAIM_DISTANCE
* (in whatever arch specific measurement units returned by node_distance())
* and node_reclaim_mode is enabled then the VM will only call node_reclaim()
* on nodes within this distance.
*/
#define RECLAIM_DISTANCE 30
#endif
/*
* The following tunable allows platforms to override the default node
* reclaim distance (RECLAIM_DISTANCE) if remote memory accesses are
* sufficiently fast that the default value actually hurts
* performance.
*
* AMD EPYC machines use this because even though the 2-hop distance
* is 32 (3.2x slower than a local memory access) performance actually
* *improves* if allowed to reclaim memory and load balance tasks
* between NUMA nodes 2-hops apart.
*/
extern int __read_mostly node_reclaim_distance;
#ifndef PENALTY_FOR_NODE_WITH_CPUS
#define PENALTY_FOR_NODE_WITH_CPUS (1)
#endif
#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DECLARE_PER_CPU(int, numa_node);
#ifndef numa_node_id
/* Returns the number of the current Node. */
static inline int numa_node_id(void)
{
return raw_cpu_read(numa_node);
}
#endif
#ifndef cpu_to_node
static inline int cpu_to_node(int cpu)
{
return per_cpu(numa_node, cpu);
}
#endif
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
this_cpu_write(numa_node, node);
}
#endif
#ifndef set_cpu_numa_node
static inline void set_cpu_numa_node(int cpu, int node)
{
per_cpu(numa_node, cpu) = node;
}
#endif
#else /* !CONFIG_USE_PERCPU_NUMA_NODE_ID */
/* Returns the number of the current Node. */
#ifndef numa_node_id
static inline int numa_node_id(void)
{
return cpu_to_node(raw_smp_processor_id());
}
#endif
#endif /* [!]CONFIG_USE_PERCPU_NUMA_NODE_ID */
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
* N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
* It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
* Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem().
*/
DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
this_cpu_write(_numa_mem_, node);
}
#endif
#ifndef numa_mem_id
/* Returns the number of the nearest Node with memory */
static inline int numa_mem_id(void)
{
return raw_cpu_read(_numa_mem_);
}
#endif
#ifndef cpu_to_mem
static inline int cpu_to_mem(int cpu)
{
return per_cpu(_numa_mem_, cpu);
}
#endif
#ifndef set_cpu_numa_mem
static inline void set_cpu_numa_mem(int cpu, int node)
{
per_cpu(_numa_mem_, cpu) = node;
}
#endif
#else /* !CONFIG_HAVE_MEMORYLESS_NODES */
#ifndef numa_mem_id
/* Returns the number of the nearest Node with memory */
static inline int numa_mem_id(void)
{
return numa_node_id();
}
#endif
#ifndef cpu_to_mem
static inline int cpu_to_mem(int cpu)
{
return cpu_to_node(cpu);
}
#endif
#endif /* [!]CONFIG_HAVE_MEMORYLESS_NODES */
#if defined(topology_die_id) && defined(topology_die_cpumask)
#define TOPOLOGY_DIE_SYSFS
#endif
#if defined(topology_cluster_id) && defined(topology_cluster_cpumask)
#define TOPOLOGY_CLUSTER_SYSFS
#endif
#if defined(topology_book_id) && defined(topology_book_cpumask)
#define TOPOLOGY_BOOK_SYSFS
#endif
#if defined(topology_drawer_id) && defined(topology_drawer_cpumask)
#define TOPOLOGY_DRAWER_SYSFS
#endif
#ifndef topology_physical_package_id
#define topology_physical_package_id(cpu) ((void)(cpu), -1)
#endif
#ifndef topology_die_id
#define topology_die_id(cpu) ((void)(cpu), -1)
#endif
#ifndef topology_cluster_id
#define topology_cluster_id(cpu) ((void)(cpu), -1)
#endif
#ifndef topology_core_id
#define topology_core_id(cpu) ((void)(cpu), 0)
#endif
#ifndef topology_book_id
#define topology_book_id(cpu) ((void)(cpu), -1)
#endif
#ifndef topology_drawer_id
#define topology_drawer_id(cpu) ((void)(cpu), -1)
#endif
#ifndef topology_ppin
#define topology_ppin(cpu) ((void)(cpu), 0ull)
#endif
#ifndef topology_sibling_cpumask
#define topology_sibling_cpumask(cpu) cpumask_of(cpu)
#endif
#ifndef topology_core_cpumask
#define topology_core_cpumask(cpu) cpumask_of(cpu)
#endif
#ifndef topology_cluster_cpumask
#define topology_cluster_cpumask(cpu) cpumask_of(cpu)
#endif
#ifndef topology_die_cpumask
#define topology_die_cpumask(cpu) cpumask_of(cpu)
#endif
#ifndef topology_book_cpumask
#define topology_book_cpumask(cpu) cpumask_of(cpu)
#endif
#ifndef topology_drawer_cpumask
#define topology_drawer_cpumask(cpu) cpumask_of(cpu)
#endif
#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
static inline const struct cpumask *cpu_smt_mask(int cpu)
{
return topology_sibling_cpumask(cpu);
}
#endif
static inline const struct cpumask *cpu_cpu_mask(int cpu)
{
return cpumask_of_node(cpu_to_node(cpu));
}
#ifdef CONFIG_NUMA
int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node);
extern const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops);
#else
static __always_inline int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node)
{
return cpumask_nth_and(cpu, cpus, cpu_online_mask);
}
static inline const struct cpumask *
sched_numa_hop_mask(unsigned int node, unsigned int hops)
{
return ERR_PTR(-EOPNOTSUPP);
}
#endif /* CONFIG_NUMA */
/**
* for_each_numa_hop_mask - iterate over cpumasks of increasing NUMA distance
* from a given node.
* @mask: the iteration variable.
* @node: the NUMA node to start the search from.
*
* Requires rcu_lock to be held.
*
* Yields cpu_online_mask for @node == NUMA_NO_NODE.
*/
#define for_each_numa_hop_mask(mask, node) \
for (unsigned int __hops = 0; \
mask = (node != NUMA_NO_NODE || __hops) ? \
sched_numa_hop_mask(node, __hops) : \
cpu_online_mask, \
!IS_ERR_OR_NULL(mask); \
__hops++)
#endif /* _LINUX_TOPOLOGY_H */
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
#ifndef _LINUX_KCOV_IOCTLS_H
#define _LINUX_KCOV_IOCTLS_H
#include <linux/types.h>
/*
* Argument for KCOV_REMOTE_ENABLE ioctl, see Documentation/dev-tools/kcov.rst
* and the comment before kcov_remote_start() for usage details.
*/
struct kcov_remote_arg {
__u32 trace_mode; /* KCOV_TRACE_PC or KCOV_TRACE_CMP */
__u32 area_size; /* Length of coverage buffer in words */
__u32 num_handles; /* Size of handles array */
__aligned_u64 common_handle;
__aligned_u64 handles[];
};
#define KCOV_REMOTE_MAX_HANDLES 0x100
#define KCOV_INIT_TRACE _IOR('c', 1, unsigned long)
#define KCOV_ENABLE _IO('c', 100)
#define KCOV_DISABLE _IO('c', 101)
#define KCOV_REMOTE_ENABLE _IOW('c', 102, struct kcov_remote_arg)
enum {
/*
* Tracing coverage collection mode.
* Covered PCs are collected in a per-task buffer.
* In new KCOV version the mode is chosen by calling
* ioctl(fd, KCOV_ENABLE, mode). In older versions the mode argument
* was supposed to be 0 in such a call. So, for reasons of backward
* compatibility, we have chosen the value KCOV_TRACE_PC to be 0.
*/
KCOV_TRACE_PC = 0,
/* Collecting comparison operands mode. */
KCOV_TRACE_CMP = 1,
};
/*
* The format for the types of collected comparisons.
*
* Bit 0 shows whether one of the arguments is a compile-time constant.
* Bits 1 & 2 contain log2 of the argument size, up to 8 bytes.
*/
#define KCOV_CMP_CONST (1 << 0)
#define KCOV_CMP_SIZE(n) ((n) << 1)
#define KCOV_CMP_MASK KCOV_CMP_SIZE(3)
#define KCOV_SUBSYSTEM_COMMON (0x00ull << 56)
#define KCOV_SUBSYSTEM_USB (0x01ull << 56)
#define KCOV_SUBSYSTEM_MASK (0xffull << 56)
#define KCOV_INSTANCE_MASK (0xffffffffull)
static inline __u64 kcov_remote_handle(__u64 subsys, __u64 inst)
{
if (subsys & ~KCOV_SUBSYSTEM_MASK || inst & ~KCOV_INSTANCE_MASK)
return 0;
return subsys | inst;
}
#endif /* _LINUX_KCOV_IOCTLS_H */
// SPDX-License-Identifier: GPL-2.0+
/*
* Driver for USB Mass Storage compliant devices
*
* Current development and maintenance by:
* (c) 1999-2003 Matthew Dharm (mdharm-usb@one-eyed-alien.net)
*
* Developed with the assistance of:
* (c) 2000 David L. Brown, Jr. (usb-storage@davidb.org)
* (c) 2003-2009 Alan Stern (stern@rowland.harvard.edu)
*
* Initial work by:
* (c) 1999 Michael Gee (michael@linuxspecific.com)
*
* usb_device_id support by Adam J. Richter (adam@yggdrasil.com):
* (c) 2000 Yggdrasil Computing, Inc.
*
* This driver is based on the 'USB Mass Storage Class' document. This
* describes in detail the protocol used to communicate with such
* devices. Clearly, the designers had SCSI and ATAPI commands in
* mind when they created this document. The commands are all very
* similar to commands in the SCSI-II and ATAPI specifications.
*
* It is important to note that in a number of cases this class
* exhibits class-specific exemptions from the USB specification.
* Notably the usage of NAK, STALL and ACK differs from the norm, in
* that they are used to communicate wait, failed and OK on commands.
*
* Also, for certain devices, the interrupt endpoint is used to convey
* status of a command.
*/
#ifdef CONFIG_USB_STORAGE_DEBUG
#define DEBUG
#endif
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/kthread.h>
#include <linux/mutex.h>
#include <linux/utsname.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include "usb.h"
#include "scsiglue.h"
#include "transport.h"
#include "protocol.h"
#include "debug.h"
#include "initializers.h"
#include "sierra_ms.h"
#include "option_ms.h"
#if IS_ENABLED(CONFIG_USB_UAS)
#include "uas-detect.h"
#endif
#define DRV_NAME "usb-storage"
/* Some informational data */
MODULE_AUTHOR("Matthew Dharm <mdharm-usb@one-eyed-alien.net>");
MODULE_DESCRIPTION("USB Mass Storage driver for Linux");
MODULE_LICENSE("GPL");
static unsigned int delay_use = 1;
module_param(delay_use, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(delay_use, "seconds to delay before using a new device");
static char quirks[128];
module_param_string(quirks, quirks, sizeof(quirks), S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(quirks, "supplemental list of device IDs and their quirks");
/*
* The entries in this table correspond, line for line,
* with the entries in usb_storage_usb_ids[], defined in usual-tables.c.
*/
/*
*The vendor name should be kept at eight characters or less, and
* the product name should be kept at 16 characters or less. If a device
* has the US_FL_FIX_INQUIRY flag, then the vendor and product names
* normally generated by a device through the INQUIRY response will be
* taken from this list, and this is the reason for the above size
* restriction. However, if the flag is not present, then you
* are free to use as many characters as you like.
*/
#define UNUSUAL_DEV(idVendor, idProduct, bcdDeviceMin, bcdDeviceMax, \
vendor_name, product_name, use_protocol, use_transport, \
init_function, Flags) \
{ \
.vendorName = vendor_name, \
.productName = product_name, \
.useProtocol = use_protocol, \
.useTransport = use_transport, \
.initFunction = init_function, \
}
#define COMPLIANT_DEV UNUSUAL_DEV
#define USUAL_DEV(use_protocol, use_transport) \
{ \
.useProtocol = use_protocol, \
.useTransport = use_transport, \
}
static const struct us_unusual_dev us_unusual_dev_list[] = {
# include "unusual_devs.h"
{ } /* Terminating entry */
};
static const struct us_unusual_dev for_dynamic_ids =
USUAL_DEV(USB_SC_SCSI, USB_PR_BULK);
#undef UNUSUAL_DEV
#undef COMPLIANT_DEV
#undef USUAL_DEV
#ifdef CONFIG_LOCKDEP
static struct lock_class_key us_interface_key[USB_MAXINTERFACES];
static void us_set_lock_class(struct mutex *mutex,
struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
struct usb_host_config *config = udev->actconfig;
int i;
for (i = 0; i < config->desc.bNumInterfaces; i++) {
if (config->interface[i] == intf)
break;
}
BUG_ON(i == config->desc.bNumInterfaces);
lockdep_set_class(mutex, &us_interface_key[i]);
}
#else
static void us_set_lock_class(struct mutex *mutex,
struct usb_interface *intf)
{
}
#endif
#ifdef CONFIG_PM /* Minimal support for suspend and resume */
int usb_stor_suspend(struct usb_interface *iface, pm_message_t message)
{
struct us_data *us = usb_get_intfdata(iface);
/* Wait until no command is running */
mutex_lock(&us->dev_mutex);
if (us->suspend_resume_hook)
(us->suspend_resume_hook)(us, US_SUSPEND);
/*
* When runtime PM is working, we'll set a flag to indicate
* whether we should autoresume when a SCSI request arrives.
*/
mutex_unlock(&us->dev_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(usb_stor_suspend);
int usb_stor_resume(struct usb_interface *iface)
{
struct us_data *us = usb_get_intfdata(iface);
mutex_lock(&us->dev_mutex);
if (us->suspend_resume_hook)
(us->suspend_resume_hook)(us, US_RESUME);
mutex_unlock(&us->dev_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(usb_stor_resume);
int usb_stor_reset_resume(struct usb_interface *iface)
{
struct us_data *us = usb_get_intfdata(iface);
/* Report the reset to the SCSI core */
usb_stor_report_bus_reset(us);
/*
* If any of the subdrivers implemented a reinitialization scheme,
* this is where the callback would be invoked.
*/
return 0;
}
EXPORT_SYMBOL_GPL(usb_stor_reset_resume);
#endif /* CONFIG_PM */
/*
* The next two routines get called just before and just after
* a USB port reset, whether from this driver or a different one.
*/
int usb_stor_pre_reset(struct usb_interface *iface)
{
struct us_data *us = usb_get_intfdata(iface);
/* Make sure no command runs during the reset */
mutex_lock(&us->dev_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(usb_stor_pre_reset);
int usb_stor_post_reset(struct usb_interface *iface)
{
struct us_data *us = usb_get_intfdata(iface);
/* Report the reset to the SCSI core */
usb_stor_report_bus_reset(us);
/*
* If any of the subdrivers implemented a reinitialization scheme,
* this is where the callback would be invoked.
*/
mutex_unlock(&us->dev_mutex);
return 0;
}
EXPORT_SYMBOL_GPL(usb_stor_post_reset);
/*
* fill_inquiry_response takes an unsigned char array (which must
* be at least 36 characters) and populates the vendor name,
* product name, and revision fields. Then the array is copied
* into the SCSI command's response buffer (oddly enough
* called request_buffer). data_len contains the length of the
* data array, which again must be at least 36.
*/
void fill_inquiry_response(struct us_data *us, unsigned char *data,
unsigned int data_len)
{
if (data_len < 36) /* You lose. */
return;
memset(data+8, ' ', 28);
if (data[0]&0x20) { /*
* USB device currently not connected. Return
* peripheral qualifier 001b ("...however, the
* physical device is not currently connected
* to this logical unit") and leave vendor and
* product identification empty. ("If the target
* does store some of the INQUIRY data on the
* device, it may return zeros or ASCII spaces
* (20h) in those fields until the data is
* available from the device.").
*/
} else {
u16 bcdDevice = le16_to_cpu(us->pusb_dev->descriptor.bcdDevice);
int n;
n = strlen(us->unusual_dev->vendorName);
memcpy(data+8, us->unusual_dev->vendorName, min(8, n));
n = strlen(us->unusual_dev->productName);
memcpy(data+16, us->unusual_dev->productName, min(16, n));
data[32] = 0x30 + ((bcdDevice>>12) & 0x0F);
data[33] = 0x30 + ((bcdDevice>>8) & 0x0F);
data[34] = 0x30 + ((bcdDevice>>4) & 0x0F);
data[35] = 0x30 + ((bcdDevice) & 0x0F);
}
usb_stor_set_xfer_buf(data, data_len, us->srb);
}
EXPORT_SYMBOL_GPL(fill_inquiry_response);
static int usb_stor_control_thread(void * __us)
{
struct us_data *us = (struct us_data *)__us;
struct Scsi_Host *host = us_to_host(us);
struct scsi_cmnd *srb;
for (;;) {
usb_stor_dbg(us, "*** thread sleeping\n");
if (wait_for_completion_interruptible(&us->cmnd_ready))
break;
usb_stor_dbg(us, "*** thread awakened\n");
/* lock the device pointers */
mutex_lock(&(us->dev_mutex));
/* lock access to the state */
scsi_lock(host);
/* When we are called with no command pending, we're done */
srb = us->srb;
if (srb == NULL) {
scsi_unlock(host);
mutex_unlock(&us->dev_mutex);
usb_stor_dbg(us, "-- exiting\n");
break;
}
/* has the command timed out *already* ? */
if (test_bit(US_FLIDX_TIMED_OUT, &us->dflags)) {
srb->result = DID_ABORT << 16;
goto SkipForAbort;
}
scsi_unlock(host);
/*
* reject the command if the direction indicator
* is UNKNOWN
*/
if (srb->sc_data_direction == DMA_BIDIRECTIONAL) {
usb_stor_dbg(us, "UNKNOWN data direction\n");
srb->result = DID_ERROR << 16;
}
/*
* reject if target != 0 or if LUN is higher than
* the maximum known LUN
*/
else if (srb->device->id &&
!(us->fflags & US_FL_SCM_MULT_TARG)) {
usb_stor_dbg(us, "Bad target number (%d:%llu)\n",
srb->device->id,
srb->device->lun);
srb->result = DID_BAD_TARGET << 16;
}
else if (srb->device->lun > us->max_lun) {
usb_stor_dbg(us, "Bad LUN (%d:%llu)\n",
srb->device->id,
srb->device->lun);
srb->result = DID_BAD_TARGET << 16;
}
/*
* Handle those devices which need us to fake
* their inquiry data
*/
else if ((srb->cmnd[0] == INQUIRY) &&
(us->fflags & US_FL_FIX_INQUIRY)) {
unsigned char data_ptr[36] = {
0x00, 0x80, 0x02, 0x02,
0x1F, 0x00, 0x00, 0x00};
usb_stor_dbg(us, "Faking INQUIRY command\n");
fill_inquiry_response(us, data_ptr, 36);
srb->result = SAM_STAT_GOOD;
}
/* we've got a command, let's do it! */
else {
US_DEBUG(usb_stor_show_command(us, srb));
us->proto_handler(srb, us);
usb_mark_last_busy(us->pusb_dev);
}
/* lock access to the state */
scsi_lock(host);
/* was the command aborted? */
if (srb->result == DID_ABORT << 16) {
SkipForAbort:
usb_stor_dbg(us, "scsi command aborted\n");
srb = NULL; /* Don't call scsi_done() */
}
/*
* If an abort request was received we need to signal that
* the abort has finished. The proper test for this is
* the TIMED_OUT flag, not srb->result == DID_ABORT, because
* the timeout might have occurred after the command had
* already completed with a different result code.
*/
if (test_bit(US_FLIDX_TIMED_OUT, &us->dflags)) {
complete(&(us->notify));
/* Allow USB transfers to resume */
clear_bit(US_FLIDX_ABORTING, &us->dflags);
clear_bit(US_FLIDX_TIMED_OUT, &us->dflags);
}
/* finished working on this command */
us->srb = NULL;
scsi_unlock(host);
/* unlock the device pointers */
mutex_unlock(&us->dev_mutex);
/* now that the locks are released, notify the SCSI core */
if (srb) {
usb_stor_dbg(us, "scsi cmd done, result=0x%x\n",
srb->result);
scsi_done_direct(srb);
}
} /* for (;;) */
/* Wait until we are told to stop */
for (;;) {
set_current_state(TASK_INTERRUPTIBLE);
if (kthread_should_stop())
break;
schedule();
}
__set_current_state(TASK_RUNNING);
return 0;
}
/***********************************************************************
* Device probing and disconnecting
***********************************************************************/
/* Associate our private data with the USB device */
static int associate_dev(struct us_data *us, struct usb_interface *intf)
{
/* Fill in the device-related fields */
us->pusb_dev = interface_to_usbdev(intf);
us->pusb_intf = intf;
us->ifnum = intf->cur_altsetting->desc.bInterfaceNumber;
usb_stor_dbg(us, "Vendor: 0x%04x, Product: 0x%04x, Revision: 0x%04x\n",
le16_to_cpu(us->pusb_dev->descriptor.idVendor),
le16_to_cpu(us->pusb_dev->descriptor.idProduct),
le16_to_cpu(us->pusb_dev->descriptor.bcdDevice));
usb_stor_dbg(us, "Interface Subclass: 0x%02x, Protocol: 0x%02x\n",
intf->cur_altsetting->desc.bInterfaceSubClass,
intf->cur_altsetting->desc.bInterfaceProtocol);
/* Store our private data in the interface */
usb_set_intfdata(intf, us);
/* Allocate the control/setup and DMA-mapped buffers */
us->cr = kmalloc(sizeof(*us->cr), GFP_KERNEL);
if (!us->cr)
return -ENOMEM;
us->iobuf = usb_alloc_coherent(us->pusb_dev, US_IOBUF_SIZE,
GFP_KERNEL, &us->iobuf_dma);
if (!us->iobuf) {
usb_stor_dbg(us, "I/O buffer allocation failed\n");
return -ENOMEM;
}
return 0;
}
/* Works only for digits and letters, but small and fast */
#define TOLOWER(x) ((x) | 0x20)
/* Adjust device flags based on the "quirks=" module parameter */
void usb_stor_adjust_quirks(struct usb_device *udev, u64 *fflags)
{
char *p;
u16 vid = le16_to_cpu(udev->descriptor.idVendor);
u16 pid = le16_to_cpu(udev->descriptor.idProduct);
u64 f = 0;
u64 mask = (US_FL_SANE_SENSE | US_FL_BAD_SENSE |
US_FL_FIX_CAPACITY | US_FL_IGNORE_UAS |
US_FL_CAPACITY_HEURISTICS | US_FL_IGNORE_DEVICE |
US_FL_NOT_LOCKABLE | US_FL_MAX_SECTORS_64 |
US_FL_CAPACITY_OK | US_FL_IGNORE_RESIDUE |
US_FL_SINGLE_LUN | US_FL_NO_WP_DETECT |
US_FL_NO_READ_DISC_INFO | US_FL_NO_READ_CAPACITY_16 |
US_FL_INITIAL_READ10 | US_FL_WRITE_CACHE |
US_FL_NO_ATA_1X | US_FL_NO_REPORT_OPCODES |
US_FL_MAX_SECTORS_240 | US_FL_NO_REPORT_LUNS |
US_FL_ALWAYS_SYNC);
p = quirks;
while (*p) {
/* Each entry consists of VID:PID:flags */
if (vid == simple_strtoul(p, &p, 16) &&
*p == ':' &&
pid == simple_strtoul(p+1, &p, 16) &&
*p == ':')
break;
/* Move forward to the next entry */
while (*p) {
if (*p++ == ',')
break;
}
}
if (!*p) /* No match */
return;
/* Collect the flags */
while (*++p && *p != ',') {
switch (TOLOWER(*p)) {
case 'a':
f |= US_FL_SANE_SENSE;
break;
case 'b':
f |= US_FL_BAD_SENSE;
break;
case 'c':
f |= US_FL_FIX_CAPACITY;
break;
case 'd':
f |= US_FL_NO_READ_DISC_INFO;
break;
case 'e':
f |= US_FL_NO_READ_CAPACITY_16;
break;
case 'f':
f |= US_FL_NO_REPORT_OPCODES;
break;
case 'g':
f |= US_FL_MAX_SECTORS_240;
break;
case 'h':
f |= US_FL_CAPACITY_HEURISTICS;
break;
case 'i':
f |= US_FL_IGNORE_DEVICE;
break;
case 'j':
f |= US_FL_NO_REPORT_LUNS;
break;
case 'k':
f |= US_FL_NO_SAME;
break;
case 'l':
f |= US_FL_NOT_LOCKABLE;
break;
case 'm':
f |= US_FL_MAX_SECTORS_64;
break;
case 'n':
f |= US_FL_INITIAL_READ10;
break;
case 'o':
f |= US_FL_CAPACITY_OK;
break;
case 'p':
f |= US_FL_WRITE_CACHE;
break;
case 'r':
f |= US_FL_IGNORE_RESIDUE;
break;
case 's':
f |= US_FL_SINGLE_LUN;
break;
case 't':
f |= US_FL_NO_ATA_1X;
break;
case 'u':
f |= US_FL_IGNORE_UAS;
break;
case 'w':
f |= US_FL_NO_WP_DETECT;
break;
case 'y':
f |= US_FL_ALWAYS_SYNC;
break;
/* Ignore unrecognized flag characters */
}
}
*fflags = (*fflags & ~mask) | f;
}
EXPORT_SYMBOL_GPL(usb_stor_adjust_quirks);
/* Get the unusual_devs entries and the string descriptors */
static int get_device_info(struct us_data *us, const struct usb_device_id *id,
const struct us_unusual_dev *unusual_dev)
{
struct usb_device *dev = us->pusb_dev;
struct usb_interface_descriptor *idesc =
&us->pusb_intf->cur_altsetting->desc;
struct device *pdev = &us->pusb_intf->dev;
/* Store the entries */
us->unusual_dev = unusual_dev;
us->subclass = (unusual_dev->useProtocol == USB_SC_DEVICE) ?
idesc->bInterfaceSubClass :
unusual_dev->useProtocol;
us->protocol = (unusual_dev->useTransport == USB_PR_DEVICE) ?
idesc->bInterfaceProtocol :
unusual_dev->useTransport;
us->fflags = id->driver_info;
usb_stor_adjust_quirks(us->pusb_dev, &us->fflags);
if (us->fflags & US_FL_IGNORE_DEVICE) {
dev_info(pdev, "device ignored\n");
return -ENODEV;
}
/*
* This flag is only needed when we're in high-speed, so let's
* disable it if we're in full-speed
*/
if (dev->speed != USB_SPEED_HIGH)
us->fflags &= ~US_FL_GO_SLOW;
if (us->fflags)
dev_info(pdev, "Quirks match for vid %04x pid %04x: %llx\n",
le16_to_cpu(dev->descriptor.idVendor),
le16_to_cpu(dev->descriptor.idProduct),
us->fflags);
/*
* Log a message if a non-generic unusual_dev entry contains an
* unnecessary subclass or protocol override. This may stimulate
* reports from users that will help us remove unneeded entries
* from the unusual_devs.h table.
*/
if (id->idVendor || id->idProduct) {
static const char *msgs[3] = {
"an unneeded SubClass entry",
"an unneeded Protocol entry",
"unneeded SubClass and Protocol entries"};
struct usb_device_descriptor *ddesc = &dev->descriptor;
int msg = -1;
if (unusual_dev->useProtocol != USB_SC_DEVICE &&
us->subclass == idesc->bInterfaceSubClass)
msg += 1;
if (unusual_dev->useTransport != USB_PR_DEVICE &&
us->protocol == idesc->bInterfaceProtocol)
msg += 2;
if (msg >= 0 && !(us->fflags & US_FL_NEED_OVERRIDE))
dev_notice(pdev, "This device "
"(%04x,%04x,%04x S %02x P %02x)"
" has %s in unusual_devs.h (kernel"
" %s)\n"
" Please send a copy of this message to "
"<linux-usb@vger.kernel.org> and "
"<usb-storage@lists.one-eyed-alien.net>\n",
le16_to_cpu(ddesc->idVendor),
le16_to_cpu(ddesc->idProduct),
le16_to_cpu(ddesc->bcdDevice),
idesc->bInterfaceSubClass,
idesc->bInterfaceProtocol,
msgs[msg],
utsname()->release);
}
return 0;
}
/* Get the transport settings */
static void get_transport(struct us_data *us)
{
switch (us->protocol) {
case USB_PR_CB:
us->transport_name = "Control/Bulk";
us->transport = usb_stor_CB_transport;
us->transport_reset = usb_stor_CB_reset;
us->max_lun = 7;
break;
case USB_PR_CBI:
us->transport_name = "Control/Bulk/Interrupt";
us->transport = usb_stor_CB_transport;
us->transport_reset = usb_stor_CB_reset;
us->max_lun = 7;
break;
case USB_PR_BULK:
us->transport_name = "Bulk";
us->transport = usb_stor_Bulk_transport;
us->transport_reset = usb_stor_Bulk_reset;
break;
}
}
/* Get the protocol settings */
static void get_protocol(struct us_data *us)
{
switch (us->subclass) {
case USB_SC_RBC:
us->protocol_name = "Reduced Block Commands (RBC)";
us->proto_handler = usb_stor_transparent_scsi_command;
break;
case USB_SC_8020:
us->protocol_name = "8020i";
us->proto_handler = usb_stor_pad12_command;
us->max_lun = 0;
break;
case USB_SC_QIC:
us->protocol_name = "QIC-157";
us->proto_handler = usb_stor_pad12_command;
us->max_lun = 0;
break;
case USB_SC_8070:
us->protocol_name = "8070i";
us->proto_handler = usb_stor_pad12_command;
us->max_lun = 0;
break;
case USB_SC_SCSI:
us->protocol_name = "Transparent SCSI";
us->proto_handler = usb_stor_transparent_scsi_command;
break;
case USB_SC_UFI:
us->protocol_name = "Uniform Floppy Interface (UFI)";
us->proto_handler = usb_stor_ufi_command;
break;
}
}
/* Get the pipe settings */
static int get_pipes(struct us_data *us)
{
struct usb_host_interface *alt = us->pusb_intf->cur_altsetting;
struct usb_endpoint_descriptor *ep_in;
struct usb_endpoint_descriptor *ep_out;
struct usb_endpoint_descriptor *ep_int;
int res;
/*
* Find the first endpoint of each type we need.
* We are expecting a minimum of 2 endpoints - in and out (bulk).
* An optional interrupt-in is OK (necessary for CBI protocol).
* We will ignore any others.
*/
res = usb_find_common_endpoints(alt, &ep_in, &ep_out, NULL, NULL);
if (res) {
usb_stor_dbg(us, "bulk endpoints not found\n");
return res;
}
res = usb_find_int_in_endpoint(alt, &ep_int);
if (res && us->protocol == USB_PR_CBI) {
usb_stor_dbg(us, "interrupt endpoint not found\n");
return res;
}
/* Calculate and store the pipe values */
us->send_ctrl_pipe = usb_sndctrlpipe(us->pusb_dev, 0);
us->recv_ctrl_pipe = usb_rcvctrlpipe(us->pusb_dev, 0);
us->send_bulk_pipe = usb_sndbulkpipe(us->pusb_dev,
usb_endpoint_num(ep_out));
us->recv_bulk_pipe = usb_rcvbulkpipe(us->pusb_dev,
usb_endpoint_num(ep_in));
if (ep_int) {
us->recv_intr_pipe = usb_rcvintpipe(us->pusb_dev,
usb_endpoint_num(ep_int));
us->ep_bInterval = ep_int->bInterval;
}
return 0;
}
/* Initialize all the dynamic resources we need */
static int usb_stor_acquire_resources(struct us_data *us)
{
int p;
struct task_struct *th;
us->current_urb = usb_alloc_urb(0, GFP_KERNEL);
if (!us->current_urb)
return -ENOMEM;
/*
* Just before we start our control thread, initialize
* the device if it needs initialization
*/
if (us->unusual_dev->initFunction) {
p = us->unusual_dev->initFunction(us);
if (p)
return p;
}
/* Start up our control thread */
th = kthread_run(usb_stor_control_thread, us, "usb-storage");
if (IS_ERR(th)) {
dev_warn(&us->pusb_intf->dev,
"Unable to start control thread\n");
return PTR_ERR(th);
}
us->ctl_thread = th;
return 0;
}
/* Release all our dynamic resources */
static void usb_stor_release_resources(struct us_data *us)
{
/*
* Tell the control thread to exit. The SCSI host must
* already have been removed and the DISCONNECTING flag set
* so that we won't accept any more commands.
*/
usb_stor_dbg(us, "-- sending exit command to thread\n");
complete(&us->cmnd_ready);
if (us->ctl_thread)
kthread_stop(us->ctl_thread);
/* Call the destructor routine, if it exists */
if (us->extra_destructor) {
usb_stor_dbg(us, "-- calling extra_destructor()\n");
us->extra_destructor(us->extra);
}
/* Free the extra data and the URB */
kfree(us->extra);
usb_free_urb(us->current_urb);
}
/* Dissociate from the USB device */
static void dissociate_dev(struct us_data *us)
{
/* Free the buffers */
kfree(us->cr);
usb_free_coherent(us->pusb_dev, US_IOBUF_SIZE, us->iobuf, us->iobuf_dma);
/* Remove our private data from the interface */
usb_set_intfdata(us->pusb_intf, NULL);
}
/*
* First stage of disconnect processing: stop SCSI scanning,
* remove the host, and stop accepting new commands
*/
static void quiesce_and_remove_host(struct us_data *us)
{
struct Scsi_Host *host = us_to_host(us);
/* If the device is really gone, cut short reset delays */
if (us->pusb_dev->state == USB_STATE_NOTATTACHED) {
set_bit(US_FLIDX_DISCONNECTING, &us->dflags);
wake_up(&us->delay_wait);
}
/*
* Prevent SCSI scanning (if it hasn't started yet)
* or wait for the SCSI-scanning routine to stop.
*/
cancel_delayed_work_sync(&us->scan_dwork);
/* Balance autopm calls if scanning was cancelled */
if (test_bit(US_FLIDX_SCAN_PENDING, &us->dflags))
usb_autopm_put_interface_no_suspend(us->pusb_intf);
/*
* Removing the host will perform an orderly shutdown: caches
* synchronized, disks spun down, etc.
*/
scsi_remove_host(host);
/*
* Prevent any new commands from being accepted and cut short
* reset delays.
*/
scsi_lock(host);
set_bit(US_FLIDX_DISCONNECTING, &us->dflags);
scsi_unlock(host);
wake_up(&us->delay_wait);
}
/* Second stage of disconnect processing: deallocate all resources */
static void release_everything(struct us_data *us)
{
usb_stor_release_resources(us);
dissociate_dev(us);
/*
* Drop our reference to the host; the SCSI core will free it
* (and "us" along with it) when the refcount becomes 0.
*/
scsi_host_put(us_to_host(us));
}
/* Delayed-work routine to carry out SCSI-device scanning */
static void usb_stor_scan_dwork(struct work_struct *work)
{
struct us_data *us = container_of(work, struct us_data,
scan_dwork.work);
struct device *dev = &us->pusb_intf->dev;
dev_dbg(dev, "starting scan\n");
/* For bulk-only devices, determine the max LUN value */
if (us->protocol == USB_PR_BULK &&
!(us->fflags & US_FL_SINGLE_LUN) &&
!(us->fflags & US_FL_SCM_MULT_TARG)) {
mutex_lock(&us->dev_mutex);
us->max_lun = usb_stor_Bulk_max_lun(us);
/*
* Allow proper scanning of devices that present more than 8 LUNs
* While not affecting other devices that may need the previous
* behavior
*/
if (us->max_lun >= 8)
us_to_host(us)->max_lun = us->max_lun+1;
mutex_unlock(&us->dev_mutex);
}
scsi_scan_host(us_to_host(us));
dev_dbg(dev, "scan complete\n");
/* Should we unbind if no devices were detected? */
usb_autopm_put_interface(us->pusb_intf);
clear_bit(US_FLIDX_SCAN_PENDING, &us->dflags);
}
static unsigned int usb_stor_sg_tablesize(struct usb_interface *intf)
{
struct usb_device *usb_dev = interface_to_usbdev(intf);
if (usb_dev->bus->sg_tablesize) {
return usb_dev->bus->sg_tablesize;
}
return SG_ALL;
}
/* First part of general USB mass-storage probing */
int usb_stor_probe1(struct us_data **pus,
struct usb_interface *intf,
const struct usb_device_id *id,
const struct us_unusual_dev *unusual_dev,
const struct scsi_host_template *sht)
{
struct Scsi_Host *host;
struct us_data *us;
int result;
dev_info(&intf->dev, "USB Mass Storage device detected\n");
/*
* Ask the SCSI layer to allocate a host structure, with extra
* space at the end for our private us_data structure.
*/
host = scsi_host_alloc(sht, sizeof(*us));
if (!host) {
dev_warn(&intf->dev, "Unable to allocate the scsi host\n");
return -ENOMEM;
}
/*
* Allow 16-byte CDBs and thus > 2TB
*/
host->max_cmd_len = 16;
host->sg_tablesize = usb_stor_sg_tablesize(intf);
*pus = us = host_to_us(host);
mutex_init(&(us->dev_mutex));
us_set_lock_class(&us->dev_mutex, intf);
init_completion(&us->cmnd_ready);
init_completion(&(us->notify));
init_waitqueue_head(&us->delay_wait);
INIT_DELAYED_WORK(&us->scan_dwork, usb_stor_scan_dwork);
/* Associate the us_data structure with the USB device */
result = associate_dev(us, intf);
if (result)
goto BadDevice;
/* Get the unusual_devs entries and the descriptors */
result = get_device_info(us, id, unusual_dev);
if (result)
goto BadDevice;
/* Get standard transport and protocol settings */
get_transport(us);
get_protocol(us);
/*
* Give the caller a chance to fill in specialized transport
* or protocol settings.
*/
return 0;
BadDevice:
usb_stor_dbg(us, "storage_probe() failed\n");
release_everything(us);
return result;
}
EXPORT_SYMBOL_GPL(usb_stor_probe1);
/* Second part of general USB mass-storage probing */
int usb_stor_probe2(struct us_data *us)
{
int result;
struct device *dev = &us->pusb_intf->dev;
/* Make sure the transport and protocol have both been set */
if (!us->transport || !us->proto_handler) {
result = -ENXIO;
goto BadDevice;
}
usb_stor_dbg(us, "Transport: %s\n", us->transport_name);
usb_stor_dbg(us, "Protocol: %s\n", us->protocol_name);
if (us->fflags & US_FL_SCM_MULT_TARG) {
/*
* SCM eUSCSI bridge devices can have different numbers
* of LUNs on different targets; allow all to be probed.
*/
us->max_lun = 7;
/* The eUSCSI itself has ID 7, so avoid scanning that */
us_to_host(us)->this_id = 7;
/* max_id is 8 initially, so no need to set it here */
} else {
/* In the normal case there is only a single target */
us_to_host(us)->max_id = 1;
/*
* Like Windows, we won't store the LUN bits in CDB[1] for
* SCSI-2 devices using the Bulk-Only transport (even though
* this violates the SCSI spec).
*/
if (us->transport == usb_stor_Bulk_transport)
us_to_host(us)->no_scsi2_lun_in_cdb = 1;
}
/* fix for single-lun devices */
if (us->fflags & US_FL_SINGLE_LUN)
us->max_lun = 0;
/* Find the endpoints and calculate pipe values */
result = get_pipes(us);
if (result)
goto BadDevice;
/*
* If the device returns invalid data for the first READ(10)
* command, indicate the command should be retried.
*/
if (us->fflags & US_FL_INITIAL_READ10)
set_bit(US_FLIDX_REDO_READ10, &us->dflags);
/* Acquire all the other resources and add the host */
result = usb_stor_acquire_resources(us);
if (result)
goto BadDevice;
usb_autopm_get_interface_no_resume(us->pusb_intf);
snprintf(us->scsi_name, sizeof(us->scsi_name), "usb-storage %s",
dev_name(&us->pusb_intf->dev));
result = scsi_add_host(us_to_host(us), dev);
if (result) {
dev_warn(dev,
"Unable to add the scsi host\n");
goto HostAddErr;
}
/* Submit the delayed_work for SCSI-device scanning */
set_bit(US_FLIDX_SCAN_PENDING, &us->dflags);
if (delay_use > 0)
dev_dbg(dev, "waiting for device to settle before scanning\n");
queue_delayed_work(system_freezable_wq, &us->scan_dwork,
delay_use * HZ);
return 0;
/* We come here if there are any problems */
HostAddErr:
usb_autopm_put_interface_no_suspend(us->pusb_intf);
BadDevice:
usb_stor_dbg(us, "storage_probe() failed\n");
release_everything(us);
return result;
}
EXPORT_SYMBOL_GPL(usb_stor_probe2);
/* Handle a USB mass-storage disconnect */
void usb_stor_disconnect(struct usb_interface *intf)
{
struct us_data *us = usb_get_intfdata(intf); quiesce_and_remove_host(us);
release_everything(us);
}
EXPORT_SYMBOL_GPL(usb_stor_disconnect);
static struct scsi_host_template usb_stor_host_template;
/* The main probe routine for standard devices */
static int storage_probe(struct usb_interface *intf,
const struct usb_device_id *id)
{
const struct us_unusual_dev *unusual_dev;
struct us_data *us;
int result;
int size;
/* If uas is enabled and this device can do uas then ignore it. */
#if IS_ENABLED(CONFIG_USB_UAS)
if (uas_use_uas_driver(intf, id, NULL))
return -ENXIO;
#endif
/*
* If the device isn't standard (is handled by a subdriver
* module) then don't accept it.
*/
if (usb_usual_ignore_device(intf))
return -ENXIO;
/*
* Call the general probe procedures.
*
* The unusual_dev_list array is parallel to the usb_storage_usb_ids
* table, so we use the index of the id entry to find the
* corresponding unusual_devs entry.
*/
size = ARRAY_SIZE(us_unusual_dev_list);
if (id >= usb_storage_usb_ids && id < usb_storage_usb_ids + size) {
unusual_dev = (id - usb_storage_usb_ids) + us_unusual_dev_list;
} else {
unusual_dev = &for_dynamic_ids;
dev_dbg(&intf->dev, "Use Bulk-Only transport with the Transparent SCSI protocol for dynamic id: 0x%04x 0x%04x\n",
id->idVendor, id->idProduct);
}
result = usb_stor_probe1(&us, intf, id, unusual_dev,
&usb_stor_host_template);
if (result)
return result;
/* No special transport or protocol settings in the main module */
result = usb_stor_probe2(us);
return result;
}
static struct usb_driver usb_storage_driver = {
.name = DRV_NAME,
.probe = storage_probe,
.disconnect = usb_stor_disconnect,
.suspend = usb_stor_suspend,
.resume = usb_stor_resume,
.reset_resume = usb_stor_reset_resume,
.pre_reset = usb_stor_pre_reset,
.post_reset = usb_stor_post_reset,
.id_table = usb_storage_usb_ids,
.supports_autosuspend = 1,
.soft_unbind = 1,
};
module_usb_stor_driver(usb_storage_driver, usb_stor_host_template, DRV_NAME);
// SPDX-License-Identifier: GPL-2.0
/*
* USB Serial Converter stuff
*
* Copyright (C) 1999 - 2012
* Greg Kroah-Hartman (greg@kroah.com)
*/
#ifndef __LINUX_USB_SERIAL_H
#define __LINUX_USB_SERIAL_H
#include <linux/kref.h>
#include <linux/mutex.h>
#include <linux/serial.h>
#include <linux/kfifo.h>
/* The maximum number of ports one device can grab at once */
#define MAX_NUM_PORTS 16
/* USB serial flags */
#define USB_SERIAL_WRITE_BUSY 0
#define USB_SERIAL_THROTTLED 1
/**
* usb_serial_port: structure for the specific ports of a device.
* @serial: pointer back to the struct usb_serial owner of this port.
* @port: pointer to the corresponding tty_port for this port.
* @lock: spinlock to grab when updating portions of this structure.
* @minor: the minor number of the port
* @port_number: the struct usb_serial port number of this port (starts at 0)
* @interrupt_in_buffer: pointer to the interrupt in buffer for this port.
* @interrupt_in_urb: pointer to the interrupt in struct urb for this port.
* @interrupt_in_endpointAddress: endpoint address for the interrupt in pipe
* for this port.
* @interrupt_out_buffer: pointer to the interrupt out buffer for this port.
* @interrupt_out_size: the size of the interrupt_out_buffer, in bytes.
* @interrupt_out_urb: pointer to the interrupt out struct urb for this port.
* @interrupt_out_endpointAddress: endpoint address for the interrupt out pipe
* for this port.
* @bulk_in_buffer: pointer to the bulk in buffer for this port.
* @bulk_in_size: the size of the bulk_in_buffer, in bytes.
* @read_urb: pointer to the bulk in struct urb for this port.
* @bulk_in_endpointAddress: endpoint address for the bulk in pipe for this
* port.
* @bulk_in_buffers: pointers to the bulk in buffers for this port
* @read_urbs: pointers to the bulk in urbs for this port
* @read_urbs_free: status bitmap the for bulk in urbs
* @bulk_out_buffer: pointer to the bulk out buffer for this port.
* @bulk_out_size: the size of the bulk_out_buffer, in bytes.
* @write_urb: pointer to the bulk out struct urb for this port.
* @write_fifo: kfifo used to buffer outgoing data
* @bulk_out_buffers: pointers to the bulk out buffers for this port
* @write_urbs: pointers to the bulk out urbs for this port
* @write_urbs_free: status bitmap the for bulk out urbs
* @icount: interrupt counters
* @tx_bytes: number of bytes currently in host stack queues
* @bulk_out_endpointAddress: endpoint address for the bulk out pipe for this
* port.
* @flags: usb serial port flags
* @work: work queue entry for the line discipline waking up.
* @dev: pointer to the serial device
*
* This structure is used by the usb-serial core and drivers for the specific
* ports of a device.
*/
struct usb_serial_port {
struct usb_serial *serial;
struct tty_port port;
spinlock_t lock;
u32 minor;
u8 port_number;
unsigned char *interrupt_in_buffer;
struct urb *interrupt_in_urb;
__u8 interrupt_in_endpointAddress;
unsigned char *interrupt_out_buffer;
int interrupt_out_size;
struct urb *interrupt_out_urb;
__u8 interrupt_out_endpointAddress;
unsigned char *bulk_in_buffer;
int bulk_in_size;
struct urb *read_urb;
__u8 bulk_in_endpointAddress;
unsigned char *bulk_in_buffers[2];
struct urb *read_urbs[2];
unsigned long read_urbs_free;
unsigned char *bulk_out_buffer;
int bulk_out_size;
struct urb *write_urb;
struct kfifo write_fifo;
unsigned char *bulk_out_buffers[2];
struct urb *write_urbs[2];
unsigned long write_urbs_free;
__u8 bulk_out_endpointAddress;
struct async_icount icount;
int tx_bytes;
unsigned long flags;
struct work_struct work;
unsigned long sysrq; /* sysrq timeout */
struct device dev;
};
#define to_usb_serial_port(d) container_of(d, struct usb_serial_port, dev)
/* get and set the port private data pointer helper functions */
static inline void *usb_get_serial_port_data(struct usb_serial_port *port)
{
return dev_get_drvdata(&port->dev);
}
static inline void usb_set_serial_port_data(struct usb_serial_port *port,
void *data)
{
dev_set_drvdata(&port->dev, data);
}
/**
* usb_serial - structure used by the usb-serial core for a device
* @dev: pointer to the struct usb_device for this device
* @type: pointer to the struct usb_serial_driver for this device
* @interface: pointer to the struct usb_interface for this device
* @sibling: pointer to the struct usb_interface of any sibling interface
* @suspend_count: number of suspended (sibling) interfaces
* @num_ports: the number of ports this device has
* @num_interrupt_in: number of interrupt in endpoints we have
* @num_interrupt_out: number of interrupt out endpoints we have
* @num_bulk_in: number of bulk in endpoints we have
* @num_bulk_out: number of bulk out endpoints we have
* @port: array of struct usb_serial_port structures for the different ports.
* @private: place to put any driver specific information that is needed. The
* usb-serial driver is required to manage this data, the usb-serial core
* will not touch this. Use usb_get_serial_data() and
* usb_set_serial_data() to access this.
*/
struct usb_serial {
struct usb_device *dev;
struct usb_serial_driver *type;
struct usb_interface *interface;
struct usb_interface *sibling;
unsigned int suspend_count;
unsigned char disconnected:1;
unsigned char attached:1;
unsigned char minors_reserved:1;
unsigned char num_ports;
unsigned char num_port_pointers;
unsigned char num_interrupt_in;
unsigned char num_interrupt_out;
unsigned char num_bulk_in;
unsigned char num_bulk_out;
struct usb_serial_port *port[MAX_NUM_PORTS];
struct kref kref;
struct mutex disc_mutex;
void *private;
};
#define to_usb_serial(d) container_of(d, struct usb_serial, kref)
/* get and set the serial private data pointer helper functions */
static inline void *usb_get_serial_data(struct usb_serial *serial)
{
return serial->private;
}
static inline void usb_set_serial_data(struct usb_serial *serial, void *data)
{
serial->private = data;
}
struct usb_serial_endpoints {
unsigned char num_bulk_in;
unsigned char num_bulk_out;
unsigned char num_interrupt_in;
unsigned char num_interrupt_out;
struct usb_endpoint_descriptor *bulk_in[MAX_NUM_PORTS];
struct usb_endpoint_descriptor *bulk_out[MAX_NUM_PORTS];
struct usb_endpoint_descriptor *interrupt_in[MAX_NUM_PORTS];
struct usb_endpoint_descriptor *interrupt_out[MAX_NUM_PORTS];
};
/**
* usb_serial_driver - describes a usb serial driver
* @description: pointer to a string that describes this driver. This string
* used in the syslog messages when a device is inserted or removed.
* @id_table: pointer to a list of usb_device_id structures that define all
* of the devices this structure can support.
* @num_ports: the number of different ports this device will have.
* @num_bulk_in: minimum number of bulk-in endpoints
* @num_bulk_out: minimum number of bulk-out endpoints
* @num_interrupt_in: minimum number of interrupt-in endpoints
* @num_interrupt_out: minimum number of interrupt-out endpoints
* @bulk_in_size: minimum number of bytes to allocate for bulk-in buffer
* (0 = end-point size)
* @bulk_out_size: bytes to allocate for bulk-out buffer (0 = end-point size)
* @calc_num_ports: pointer to a function to determine how many ports this
* device has dynamically. It can also be used to verify the number of
* endpoints or to modify the port-endpoint mapping. It will be called
* after the probe() callback is called, but before attach().
* @probe: pointer to the driver's probe function.
* This will be called when the device is inserted into the system,
* but before the device has been fully initialized by the usb_serial
* subsystem. Use this function to download any firmware to the device,
* or any other early initialization that might be needed.
* Return 0 to continue on with the initialization sequence. Anything
* else will abort it.
* @attach: pointer to the driver's attach function.
* This will be called when the struct usb_serial structure is fully
* set up. Do any local initialization of the device, or any private
* memory structure allocation at this point in time.
* @disconnect: pointer to the driver's disconnect function. This will be
* called when the device is unplugged or unbound from the driver.
* @release: pointer to the driver's release function. This will be called
* when the usb_serial data structure is about to be destroyed.
* @usb_driver: pointer to the struct usb_driver that controls this
* device. This is necessary to allow dynamic ids to be added to
* the driver from sysfs.
*
* This structure is defines a USB Serial driver. It provides all of
* the information that the USB serial core code needs. If the function
* pointers are defined, then the USB serial core code will call them when
* the corresponding tty port functions are called. If they are not
* called, the generic serial function will be used instead.
*
* The driver.owner field should be set to the module owner of this driver.
* The driver.name field should be set to the name of this driver (remember
* it will show up in sysfs, so it needs to be short and to the point.
* Using the module name is a good idea.)
*/
struct usb_serial_driver {
const char *description;
const struct usb_device_id *id_table;
struct list_head driver_list;
struct device_driver driver;
struct usb_driver *usb_driver;
struct usb_dynids dynids;
unsigned char num_ports;
unsigned char num_bulk_in;
unsigned char num_bulk_out;
unsigned char num_interrupt_in;
unsigned char num_interrupt_out;
size_t bulk_in_size;
size_t bulk_out_size;
int (*probe)(struct usb_serial *serial, const struct usb_device_id *id);
int (*attach)(struct usb_serial *serial);
int (*calc_num_ports)(struct usb_serial *serial,
struct usb_serial_endpoints *epds);
void (*disconnect)(struct usb_serial *serial);
void (*release)(struct usb_serial *serial);
int (*port_probe)(struct usb_serial_port *port);
void (*port_remove)(struct usb_serial_port *port);
int (*suspend)(struct usb_serial *serial, pm_message_t message);
int (*resume)(struct usb_serial *serial);
int (*reset_resume)(struct usb_serial *serial);
/* serial function calls */
/* Called by console and by the tty layer */
int (*open)(struct tty_struct *tty, struct usb_serial_port *port);
void (*close)(struct usb_serial_port *port);
int (*write)(struct tty_struct *tty, struct usb_serial_port *port,
const unsigned char *buf, int count);
/* Called only by the tty layer */
unsigned int (*write_room)(struct tty_struct *tty);
int (*ioctl)(struct tty_struct *tty,
unsigned int cmd, unsigned long arg);
void (*get_serial)(struct tty_struct *tty, struct serial_struct *ss);
int (*set_serial)(struct tty_struct *tty, struct serial_struct *ss);
void (*set_termios)(struct tty_struct *tty, struct usb_serial_port *port,
const struct ktermios *old);
int (*break_ctl)(struct tty_struct *tty, int break_state);
unsigned int (*chars_in_buffer)(struct tty_struct *tty);
void (*wait_until_sent)(struct tty_struct *tty, long timeout);
bool (*tx_empty)(struct usb_serial_port *port);
void (*throttle)(struct tty_struct *tty);
void (*unthrottle)(struct tty_struct *tty);
int (*tiocmget)(struct tty_struct *tty);
int (*tiocmset)(struct tty_struct *tty,
unsigned int set, unsigned int clear);
int (*tiocmiwait)(struct tty_struct *tty, unsigned long arg);
int (*get_icount)(struct tty_struct *tty,
struct serial_icounter_struct *icount);
/* Called by the tty layer for port level work. There may or may not
be an attached tty at this point */
void (*dtr_rts)(struct usb_serial_port *port, int on);
int (*carrier_raised)(struct usb_serial_port *port);
/* Called by the usb serial hooks to allow the user to rework the
termios state */
void (*init_termios)(struct tty_struct *tty);
/* USB events */
void (*read_int_callback)(struct urb *urb);
void (*write_int_callback)(struct urb *urb);
void (*read_bulk_callback)(struct urb *urb);
void (*write_bulk_callback)(struct urb *urb);
/* Called by the generic read bulk callback */
void (*process_read_urb)(struct urb *urb);
/* Called by the generic write implementation */
int (*prepare_write_buffer)(struct usb_serial_port *port,
void *dest, size_t size);
};
#define to_usb_serial_driver(d) \
container_of(d, struct usb_serial_driver, driver)
int usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers[],
const char *name, const struct usb_device_id *id_table);
void usb_serial_deregister_drivers(struct usb_serial_driver *const serial_drivers[]);
void usb_serial_port_softint(struct usb_serial_port *port);
int usb_serial_suspend(struct usb_interface *intf, pm_message_t message);
int usb_serial_resume(struct usb_interface *intf);
/* USB Serial console functions */
#ifdef CONFIG_USB_SERIAL_CONSOLE
void usb_serial_console_init(int minor);
void usb_serial_console_exit(void);
void usb_serial_console_disconnect(struct usb_serial *serial);
#else
static inline void usb_serial_console_init(int minor) { }
static inline void usb_serial_console_exit(void) { }
static inline void usb_serial_console_disconnect(struct usb_serial *serial) {}
#endif
/* Functions needed by other parts of the usbserial core */
struct usb_serial_port *usb_serial_port_get_by_minor(unsigned int minor);
void usb_serial_put(struct usb_serial *serial);
int usb_serial_claim_interface(struct usb_serial *serial, struct usb_interface *intf);
int usb_serial_generic_open(struct tty_struct *tty, struct usb_serial_port *port);
int usb_serial_generic_write_start(struct usb_serial_port *port, gfp_t mem_flags);
int usb_serial_generic_write(struct tty_struct *tty, struct usb_serial_port *port,
const unsigned char *buf, int count);
void usb_serial_generic_close(struct usb_serial_port *port);
int usb_serial_generic_resume(struct usb_serial *serial);
unsigned int usb_serial_generic_write_room(struct tty_struct *tty);
unsigned int usb_serial_generic_chars_in_buffer(struct tty_struct *tty);
void usb_serial_generic_wait_until_sent(struct tty_struct *tty, long timeout);
void usb_serial_generic_read_bulk_callback(struct urb *urb);
void usb_serial_generic_write_bulk_callback(struct urb *urb);
void usb_serial_generic_throttle(struct tty_struct *tty);
void usb_serial_generic_unthrottle(struct tty_struct *tty);
int usb_serial_generic_tiocmiwait(struct tty_struct *tty, unsigned long arg);
int usb_serial_generic_get_icount(struct tty_struct *tty, struct serial_icounter_struct *icount);
int usb_serial_generic_register(void);
void usb_serial_generic_deregister(void);
int usb_serial_generic_submit_read_urbs(struct usb_serial_port *port, gfp_t mem_flags);
void usb_serial_generic_process_read_urb(struct urb *urb);
int usb_serial_generic_prepare_write_buffer(struct usb_serial_port *port, void *dest, size_t size);
#if defined(CONFIG_USB_SERIAL_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
int usb_serial_handle_sysrq_char(struct usb_serial_port *port, unsigned int ch);
int usb_serial_handle_break(struct usb_serial_port *port);
#else
static inline int usb_serial_handle_sysrq_char(struct usb_serial_port *port, unsigned int ch)
{
return 0;
}
static inline int usb_serial_handle_break(struct usb_serial_port *port)
{
return 0;
}
#endif
void usb_serial_handle_dcd_change(struct usb_serial_port *usb_port,
struct tty_struct *tty, unsigned int status);
int usb_serial_bus_register(struct usb_serial_driver *device);
void usb_serial_bus_deregister(struct usb_serial_driver *device);
extern const struct bus_type usb_serial_bus_type;
extern struct tty_driver *usb_serial_tty_driver;
static inline void usb_serial_debug_data(struct device *dev,
const char *function, int size,
const unsigned char *data)
{
dev_dbg(dev, "%s - length = %d, data = %*ph\n",
function, size, size, data);
}
/*
* Macro for reporting errors in write path to avoid infinite loop
* when port is used as a console.
*/
#define dev_err_console(usport, fmt, ...) \
do { \
static bool __print_once; \
struct usb_serial_port *__port = (usport); \
\
if (!__port->port.console || !__print_once) { \
__print_once = true; \
dev_err(&__port->dev, fmt, ##__VA_ARGS__); \
} \
} while (0)
/*
* module_usb_serial_driver() - Helper macro for registering a USB Serial driver
* @__serial_drivers: list of usb_serial drivers to register
* @__ids: all device ids that @__serial_drivers bind to
*
* Helper macro for USB serial drivers which do not do anything special
* in module init/exit. This eliminates a lot of boilerplate. Each
* module may only use this macro once, and calling it replaces
* module_init() and module_exit()
*
*/
#define usb_serial_module_driver(__name, __serial_drivers, __ids) \
static int __init usb_serial_module_init(void) \
{ \
return usb_serial_register_drivers(__serial_drivers, \
__name, __ids); \
} \
module_init(usb_serial_module_init); \
static void __exit usb_serial_module_exit(void) \
{ \
usb_serial_deregister_drivers(__serial_drivers); \
} \
module_exit(usb_serial_module_exit);
#define module_usb_serial_driver(__serial_drivers, __ids) \
usb_serial_module_driver(KBUILD_MODNAME, __serial_drivers, __ids)
#endif /* __LINUX_USB_SERIAL_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_RMAP_H
#define _LINUX_RMAP_H
/*
* Declarations for Reverse Mapping functions in mm/rmap.c
*/
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/rwsem.h>
#include <linux/memcontrol.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
/*
* The anon_vma heads a list of private "related" vmas, to scan if
* an anonymous page pointing to this anon_vma needs to be unmapped:
* the vmas on the list will be related by forking, or by splitting.
*
* Since vmas come and go as they are split and merged (particularly
* in mprotect), the mapping field of an anonymous page cannot point
* directly to a vma: instead it points to an anon_vma, on whose list
* the related vmas can be easily linked or unlinked.
*
* After unlinking the last vma on the list, we must garbage collect
* the anon_vma object itself: we're guaranteed no page can be
* pointing to this anon_vma once its vma list is empty.
*/
struct anon_vma {
struct anon_vma *root; /* Root of this anon_vma tree */
struct rw_semaphore rwsem; /* W: modification, R: walking the list */
/*
* The refcount is taken on an anon_vma when there is no
* guarantee that the vma of page tables will exist for
* the duration of the operation. A caller that takes
* the reference is responsible for clearing up the
* anon_vma if they are the last user on release
*/
atomic_t refcount;
/*
* Count of child anon_vmas. Equals to the count of all anon_vmas that
* have ->parent pointing to this one, including itself.
*
* This counter is used for making decision about reusing anon_vma
* instead of forking new one. See comments in function anon_vma_clone.
*/
unsigned long num_children;
/* Count of VMAs whose ->anon_vma pointer points to this object. */
unsigned long num_active_vmas;
struct anon_vma *parent; /* Parent of this anon_vma */
/*
* NOTE: the LSB of the rb_root.rb_node is set by
* mm_take_all_locks() _after_ taking the above lock. So the
* rb_root must only be read/written after taking the above lock
* to be sure to see a valid next pointer. The LSB bit itself
* is serialized by a system wide lock only visible to
* mm_take_all_locks() (mm_all_locks_mutex).
*/
/* Interval tree of private "related" vmas */
struct rb_root_cached rb_root;
};
/*
* The copy-on-write semantics of fork mean that an anon_vma
* can become associated with multiple processes. Furthermore,
* each child process will have its own anon_vma, where new
* pages for that process are instantiated.
*
* This structure allows us to find the anon_vmas associated
* with a VMA, or the VMAs associated with an anon_vma.
* The "same_vma" list contains the anon_vma_chains linking
* all the anon_vmas associated with this VMA.
* The "rb" field indexes on an interval tree the anon_vma_chains
* which link all the VMAs associated with this anon_vma.
*/
struct anon_vma_chain {
struct vm_area_struct *vma;
struct anon_vma *anon_vma;
struct list_head same_vma; /* locked by mmap_lock & page_table_lock */
struct rb_node rb; /* locked by anon_vma->rwsem */
unsigned long rb_subtree_last;
#ifdef CONFIG_DEBUG_VM_RB
unsigned long cached_vma_start, cached_vma_last;
#endif
};
enum ttu_flags {
TTU_SPLIT_HUGE_PMD = 0x4, /* split huge PMD if any */
TTU_IGNORE_MLOCK = 0x8, /* ignore mlock */
TTU_SYNC = 0x10, /* avoid racy checks with PVMW_SYNC */
TTU_HWPOISON = 0x20, /* do convert pte to hwpoison entry */
TTU_BATCH_FLUSH = 0x40, /* Batch TLB flushes where possible
* and caller guarantees they will
* do a final flush if necessary */
TTU_RMAP_LOCKED = 0x80, /* do not grab rmap lock:
* caller holds it */
};
#ifdef CONFIG_MMU
static inline void get_anon_vma(struct anon_vma *anon_vma)
{
atomic_inc(&anon_vma->refcount);
}
void __put_anon_vma(struct anon_vma *anon_vma);
static inline void put_anon_vma(struct anon_vma *anon_vma)
{
if (atomic_dec_and_test(&anon_vma->refcount))
__put_anon_vma(anon_vma);
}
static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
{
down_write(&anon_vma->root->rwsem);
}
static inline int anon_vma_trylock_write(struct anon_vma *anon_vma)
{
return down_write_trylock(&anon_vma->root->rwsem);
}
static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
{
up_write(&anon_vma->root->rwsem);
}
static inline void anon_vma_lock_read(struct anon_vma *anon_vma)
{
down_read(&anon_vma->root->rwsem);
}
static inline int anon_vma_trylock_read(struct anon_vma *anon_vma)
{
return down_read_trylock(&anon_vma->root->rwsem);
}
static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
{
up_read(&anon_vma->root->rwsem);
}
/*
* anon_vma helper functions.
*/
void anon_vma_init(void); /* create anon_vma_cachep */
int __anon_vma_prepare(struct vm_area_struct *);
void unlink_anon_vmas(struct vm_area_struct *);
int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
static inline int anon_vma_prepare(struct vm_area_struct *vma)
{
if (likely(vma->anon_vma))
return 0;
return __anon_vma_prepare(vma);
}
static inline void anon_vma_merge(struct vm_area_struct *vma,
struct vm_area_struct *next)
{
VM_BUG_ON_VMA(vma->anon_vma != next->anon_vma, vma);
unlink_anon_vmas(next);
}
struct anon_vma *folio_get_anon_vma(struct folio *folio);
/* RMAP flags, currently only relevant for some anon rmap operations. */
typedef int __bitwise rmap_t;
/*
* No special request: A mapped anonymous (sub)page is possibly shared between
* processes.
*/
#define RMAP_NONE ((__force rmap_t)0)
/* The anonymous (sub)page is exclusive to a single process. */
#define RMAP_EXCLUSIVE ((__force rmap_t)BIT(0))
/*
* Internally, we're using an enum to specify the granularity. We make the
* compiler emit specialized code for each granularity.
*/
enum rmap_level {
RMAP_LEVEL_PTE = 0,
RMAP_LEVEL_PMD,
};
static inline void __folio_rmap_sanity_checks(struct folio *folio,
struct page *page, int nr_pages, enum rmap_level level)
{
/* hugetlb folios are handled separately. */
VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
/*
* TODO: we get driver-allocated folios that have nothing to do with
* the rmap using vm_insert_page(); therefore, we cannot assume that
* folio_test_large_rmappable() holds for large folios. We should
* handle any desired mapcount+stats accounting for these folios in
* VM_MIXEDMAP VMAs separately, and then sanity-check here that
* we really only get rmappable folios.
*/
VM_WARN_ON_ONCE(nr_pages <= 0); VM_WARN_ON_FOLIO(page_folio(page) != folio, folio); VM_WARN_ON_FOLIO(page_folio(page + nr_pages - 1) != folio, folio);
switch (level) {
case RMAP_LEVEL_PTE:
break;
case RMAP_LEVEL_PMD:
/*
* We don't support folios larger than a single PMD yet. So
* when RMAP_LEVEL_PMD is set, we assume that we are creating
* a single "entire" mapping of the folio.
*/
VM_WARN_ON_FOLIO(folio_nr_pages(folio) != HPAGE_PMD_NR, folio);
VM_WARN_ON_FOLIO(nr_pages != HPAGE_PMD_NR, folio);
break;
default:
VM_WARN_ON_ONCE(true);
}
}
/*
* rmap interfaces called when adding or removing pte of page
*/
void folio_move_anon_rmap(struct folio *, struct vm_area_struct *);
void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,
struct vm_area_struct *, unsigned long address, rmap_t flags);
#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \
folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags)
void folio_add_anon_rmap_pmd(struct folio *, struct page *,
struct vm_area_struct *, unsigned long address, rmap_t flags);
void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *,
unsigned long address);
void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages,
struct vm_area_struct *);
#define folio_add_file_rmap_pte(folio, page, vma) \
folio_add_file_rmap_ptes(folio, page, 1, vma)
void folio_add_file_rmap_pmd(struct folio *, struct page *,
struct vm_area_struct *);
void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages,
struct vm_area_struct *);
#define folio_remove_rmap_pte(folio, page, vma) \
folio_remove_rmap_ptes(folio, page, 1, vma)
void folio_remove_rmap_pmd(struct folio *, struct page *,
struct vm_area_struct *);
void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,
unsigned long address, rmap_t flags);
void hugetlb_add_new_anon_rmap(struct folio *, struct vm_area_struct *,
unsigned long address);
/* See folio_try_dup_anon_rmap_*() */
static inline int hugetlb_try_dup_anon_rmap(struct folio *folio,
struct vm_area_struct *vma)
{
VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
if (PageAnonExclusive(&folio->page)) {
if (unlikely(folio_needs_cow_for_dma(vma, folio)))
return -EBUSY;
ClearPageAnonExclusive(&folio->page);
}
atomic_inc(&folio->_entire_mapcount);
return 0;
}
/* See folio_try_share_anon_rmap_*() */
static inline int hugetlb_try_share_anon_rmap(struct folio *folio)
{
VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
VM_WARN_ON_FOLIO(!PageAnonExclusive(&folio->page), folio);
/* Paired with the memory barrier in try_grab_folio(). */
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
smp_mb();
if (unlikely(folio_maybe_dma_pinned(folio)))
return -EBUSY;
ClearPageAnonExclusive(&folio->page);
/*
* This is conceptually a smp_wmb() paired with the smp_rmb() in
* gup_must_unshare().
*/
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
smp_mb__after_atomic();
return 0;
}
static inline void hugetlb_add_file_rmap(struct folio *folio)
{
VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);
atomic_inc(&folio->_entire_mapcount);
}
static inline void hugetlb_remove_rmap(struct folio *folio)
{
VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
atomic_dec(&folio->_entire_mapcount);
}
static __always_inline void __folio_dup_file_rmap(struct folio *folio,
struct page *page, int nr_pages, enum rmap_level level)
{
__folio_rmap_sanity_checks(folio, page, nr_pages, level);
switch (level) {
case RMAP_LEVEL_PTE:
do {
atomic_inc(&page->_mapcount);
} while (page++, --nr_pages > 0);
break;
case RMAP_LEVEL_PMD:
atomic_inc(&folio->_entire_mapcount);
break;
}
}
/**
* folio_dup_file_rmap_ptes - duplicate PTE mappings of a page range of a folio
* @folio: The folio to duplicate the mappings of
* @page: The first page to duplicate the mappings of
* @nr_pages: The number of pages of which the mapping will be duplicated
*
* The page range of the folio is defined by [page, page + nr_pages)
*
* The caller needs to hold the page table lock.
*/
static inline void folio_dup_file_rmap_ptes(struct folio *folio,
struct page *page, int nr_pages)
{
__folio_dup_file_rmap(folio, page, nr_pages, RMAP_LEVEL_PTE);
}
#define folio_dup_file_rmap_pte(folio, page) \
folio_dup_file_rmap_ptes(folio, page, 1)
/**
* folio_dup_file_rmap_pmd - duplicate a PMD mapping of a page range of a folio
* @folio: The folio to duplicate the mapping of
* @page: The first page to duplicate the mapping of
*
* The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
*
* The caller needs to hold the page table lock.
*/
static inline void folio_dup_file_rmap_pmd(struct folio *folio,
struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
__folio_dup_file_rmap(folio, page, HPAGE_PMD_NR, RMAP_LEVEL_PTE);
#else
WARN_ON_ONCE(true);
#endif
}
static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
struct page *page, int nr_pages, struct vm_area_struct *src_vma,
enum rmap_level level)
{
bool maybe_pinned;
int i;
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
__folio_rmap_sanity_checks(folio, page, nr_pages, level);
/*
* If this folio may have been pinned by the parent process,
* don't allow to duplicate the mappings but instead require to e.g.,
* copy the subpage immediately for the child so that we'll always
* guarantee the pinned folio won't be randomly replaced in the
* future on write faults.
*/
maybe_pinned = likely(!folio_is_device_private(folio)) &&
unlikely(folio_needs_cow_for_dma(src_vma, folio));
/*
* No need to check+clear for already shared PTEs/PMDs of the
* folio. But if any page is PageAnonExclusive, we must fallback to
* copying if the folio maybe pinned.
*/
switch (level) {
case RMAP_LEVEL_PTE:
if (unlikely(maybe_pinned)) {
for (i = 0; i < nr_pages; i++)
if (PageAnonExclusive(page + i))
return -EBUSY;
}
do {
if (PageAnonExclusive(page))
ClearPageAnonExclusive(page);
atomic_inc(&page->_mapcount);
} while (page++, --nr_pages > 0);
break;
case RMAP_LEVEL_PMD:
if (PageAnonExclusive(page)) {
if (unlikely(maybe_pinned))
return -EBUSY;
ClearPageAnonExclusive(page);
}
atomic_inc(&folio->_entire_mapcount);
break;
}
return 0;
}
/**
* folio_try_dup_anon_rmap_ptes - try duplicating PTE mappings of a page range
* of a folio
* @folio: The folio to duplicate the mappings of
* @page: The first page to duplicate the mappings of
* @nr_pages: The number of pages of which the mapping will be duplicated
* @src_vma: The vm area from which the mappings are duplicated
*
* The page range of the folio is defined by [page, page + nr_pages)
*
* The caller needs to hold the page table lock and the
* vma->vma_mm->write_protect_seq.
*
* Duplicating the mappings can only fail if the folio may be pinned; device
* private folios cannot get pinned and consequently this function cannot fail
* for them.
*
* If duplicating the mappings succeeded, the duplicated PTEs have to be R/O in
* the parent and the child. They must *not* be writable after this call
* succeeded.
*
* Returns 0 if duplicating the mappings succeeded. Returns -EBUSY otherwise.
*/
static inline int folio_try_dup_anon_rmap_ptes(struct folio *folio,
struct page *page, int nr_pages, struct vm_area_struct *src_vma)
{
return __folio_try_dup_anon_rmap(folio, page, nr_pages, src_vma,
RMAP_LEVEL_PTE);
}
#define folio_try_dup_anon_rmap_pte(folio, page, vma) \
folio_try_dup_anon_rmap_ptes(folio, page, 1, vma)
/**
* folio_try_dup_anon_rmap_pmd - try duplicating a PMD mapping of a page range
* of a folio
* @folio: The folio to duplicate the mapping of
* @page: The first page to duplicate the mapping of
* @src_vma: The vm area from which the mapping is duplicated
*
* The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
*
* The caller needs to hold the page table lock and the
* vma->vma_mm->write_protect_seq.
*
* Duplicating the mapping can only fail if the folio may be pinned; device
* private folios cannot get pinned and consequently this function cannot fail
* for them.
*
* If duplicating the mapping succeeds, the duplicated PMD has to be R/O in
* the parent and the child. They must *not* be writable after this call
* succeeded.
*
* Returns 0 if duplicating the mapping succeeded. Returns -EBUSY otherwise.
*/
static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio,
struct page *page, struct vm_area_struct *src_vma)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
return __folio_try_dup_anon_rmap(folio, page, HPAGE_PMD_NR, src_vma,
RMAP_LEVEL_PMD);
#else
WARN_ON_ONCE(true);
return -EBUSY;
#endif
}
static __always_inline int __folio_try_share_anon_rmap(struct folio *folio,
struct page *page, int nr_pages, enum rmap_level level)
{
VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio);
__folio_rmap_sanity_checks(folio, page, nr_pages, level);
/* device private folios cannot get pinned via GUP. */
if (unlikely(folio_is_device_private(folio))) {
ClearPageAnonExclusive(page);
return 0;
}
/*
* We have to make sure that when we clear PageAnonExclusive, that
* the page is not pinned and that concurrent GUP-fast won't succeed in
* concurrently pinning the page.
*
* Conceptually, PageAnonExclusive clearing consists of:
* (A1) Clear PTE
* (A2) Check if the page is pinned; back off if so.
* (A3) Clear PageAnonExclusive
* (A4) Restore PTE (optional, but certainly not writable)
*
* When clearing PageAnonExclusive, we cannot possibly map the page
* writable again, because anon pages that may be shared must never
* be writable. So in any case, if the PTE was writable it cannot
* be writable anymore afterwards and there would be a PTE change. Only
* if the PTE wasn't writable, there might not be a PTE change.
*
* Conceptually, GUP-fast pinning of an anon page consists of:
* (B1) Read the PTE
* (B2) FOLL_WRITE: check if the PTE is not writable; back off if so.
* (B3) Pin the mapped page
* (B4) Check if the PTE changed by re-reading it; back off if so.
* (B5) If the original PTE is not writable, check if
* PageAnonExclusive is not set; back off if so.
*
* If the PTE was writable, we only have to make sure that GUP-fast
* observes a PTE change and properly backs off.
*
* If the PTE was not writable, we have to make sure that GUP-fast either
* detects a (temporary) PTE change or that PageAnonExclusive is cleared
* and properly backs off.
*
* Consequently, when clearing PageAnonExclusive(), we have to make
* sure that (A1), (A2)/(A3) and (A4) happen in the right memory
* order. In GUP-fast pinning code, we have to make sure that (B3),(B4)
* and (B5) happen in the right memory order.
*
* We assume that there might not be a memory barrier after
* clearing/invalidating the PTE (A1) and before restoring the PTE (A4),
* so we use explicit ones here.
*/
/* Paired with the memory barrier in try_grab_folio(). */
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
smp_mb();
if (unlikely(folio_maybe_dma_pinned(folio)))
return -EBUSY;
ClearPageAnonExclusive(page);
/*
* This is conceptually a smp_wmb() paired with the smp_rmb() in
* gup_must_unshare().
*/
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
smp_mb__after_atomic();
return 0;
}
/**
* folio_try_share_anon_rmap_pte - try marking an exclusive anonymous page
* mapped by a PTE possibly shared to prepare
* for KSM or temporary unmapping
* @folio: The folio to share a mapping of
* @page: The mapped exclusive page
*
* The caller needs to hold the page table lock and has to have the page table
* entries cleared/invalidated.
*
* This is similar to folio_try_dup_anon_rmap_pte(), however, not used during
* fork() to duplicate mappings, but instead to prepare for KSM or temporarily
* unmapping parts of a folio (swap, migration) via folio_remove_rmap_pte().
*
* Marking the mapped page shared can only fail if the folio maybe pinned;
* device private folios cannot get pinned and consequently this function cannot
* fail.
*
* Returns 0 if marking the mapped page possibly shared succeeded. Returns
* -EBUSY otherwise.
*/
static inline int folio_try_share_anon_rmap_pte(struct folio *folio,
struct page *page)
{
return __folio_try_share_anon_rmap(folio, page, 1, RMAP_LEVEL_PTE);
}
/**
* folio_try_share_anon_rmap_pmd - try marking an exclusive anonymous page
* range mapped by a PMD possibly shared to
* prepare for temporary unmapping
* @folio: The folio to share the mapping of
* @page: The first page to share the mapping of
*
* The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
*
* The caller needs to hold the page table lock and has to have the page table
* entries cleared/invalidated.
*
* This is similar to folio_try_dup_anon_rmap_pmd(), however, not used during
* fork() to duplicate a mapping, but instead to prepare for temporarily
* unmapping parts of a folio (swap, migration) via folio_remove_rmap_pmd().
*
* Marking the mapped pages shared can only fail if the folio maybe pinned;
* device private folios cannot get pinned and consequently this function cannot
* fail.
*
* Returns 0 if marking the mapped pages possibly shared succeeded. Returns
* -EBUSY otherwise.
*/
static inline int folio_try_share_anon_rmap_pmd(struct folio *folio,
struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
return __folio_try_share_anon_rmap(folio, page, HPAGE_PMD_NR,
RMAP_LEVEL_PMD);
#else
WARN_ON_ONCE(true);
return -EBUSY;
#endif
}
/*
* Called from mm/vmscan.c to handle paging out
*/
int folio_referenced(struct folio *, int is_locked,
struct mem_cgroup *memcg, unsigned long *vm_flags);
void try_to_migrate(struct folio *folio, enum ttu_flags flags);
void try_to_unmap(struct folio *, enum ttu_flags flags);
int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
unsigned long end, struct page **pages,
void *arg);
/* Avoid racy checks */
#define PVMW_SYNC (1 << 0)
/* Look for migration entries rather than present PTEs */
#define PVMW_MIGRATION (1 << 1)
struct page_vma_mapped_walk {
unsigned long pfn;
unsigned long nr_pages;
pgoff_t pgoff;
struct vm_area_struct *vma;
unsigned long address;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
unsigned int flags;
};
#define DEFINE_PAGE_VMA_WALK(name, _page, _vma, _address, _flags) \
struct page_vma_mapped_walk name = { \
.pfn = page_to_pfn(_page), \
.nr_pages = compound_nr(_page), \
.pgoff = page_to_pgoff(_page), \
.vma = _vma, \
.address = _address, \
.flags = _flags, \
}
#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
struct page_vma_mapped_walk name = { \
.pfn = folio_pfn(_folio), \
.nr_pages = folio_nr_pages(_folio), \
.pgoff = folio_pgoff(_folio), \
.vma = _vma, \
.address = _address, \
.flags = _flags, \
}
static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
{
/* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
pte_unmap(pvmw->pte);
if (pvmw->ptl)
spin_unlock(pvmw->ptl);
}
bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw);
/*
* Used by swapoff to help locate where page is expected in vma.
*/
unsigned long page_address_in_vma(struct page *, struct vm_area_struct *);
/*
* Cleans the PTEs of shared mappings.
* (and since clean PTEs should also be readonly, write protects them too)
*
* returns the number of cleaned PTEs.
*/
int folio_mkclean(struct folio *);
int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
struct vm_area_struct *vma);
void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked);
int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
/*
* rmap_walk_control: To control rmap traversing for specific needs
*
* arg: passed to rmap_one() and invalid_vma()
* try_lock: bail out if the rmap lock is contended
* contended: indicate the rmap traversal bailed out due to lock contention
* rmap_one: executed on each vma where page is mapped
* done: for checking traversing termination condition
* anon_lock: for getting anon_lock by optimized way rather than default
* invalid_vma: for skipping uninterested vma
*/
struct rmap_walk_control {
void *arg;
bool try_lock;
bool contended;
/*
* Return false if page table scanning in rmap_walk should be stopped.
* Otherwise, return true.
*/
bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma,
unsigned long addr, void *arg);
int (*done)(struct folio *folio);
struct anon_vma *(*anon_lock)(struct folio *folio,
struct rmap_walk_control *rwc);
bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
};
void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc);
void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc);
struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,
struct rmap_walk_control *rwc);
#else /* !CONFIG_MMU */
#define anon_vma_init() do {} while (0)
#define anon_vma_prepare(vma) (0)
static inline int folio_referenced(struct folio *folio, int is_locked,
struct mem_cgroup *memcg,
unsigned long *vm_flags)
{
*vm_flags = 0;
return 0;
}
static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
}
static inline int folio_mkclean(struct folio *folio)
{
return 0;
}
#endif /* CONFIG_MMU */
static inline int page_mkclean(struct page *page)
{
return folio_mkclean(page_folio(page));
}
#endif /* _LINUX_RMAP_H */
// SPDX-License-Identifier: GPL-2.0+
/*
* Procedures for creating, accessing and interpreting the device tree.
*
* Paul Mackerras August 1996.
* Copyright (C) 1996-2005 Paul Mackerras.
*
* Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner.
* {engebret|bergner}@us.ibm.com
*
* Adapted for sparc and sparc64 by David S. Miller davem@davemloft.net
*
* Reconsolidated from arch/x/kernel/prom.c by Stephen Rothwell and
* Grant Likely.
*/
#define pr_fmt(fmt) "OF: " fmt
#include <linux/console.h>
#include <linux/ctype.h>
#include <linux/cpu.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_graph.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/proc_fs.h>
#include "of_private.h"
LIST_HEAD(aliases_lookup);
struct device_node *of_root;
EXPORT_SYMBOL(of_root);
struct device_node *of_chosen;
EXPORT_SYMBOL(of_chosen);
struct device_node *of_aliases;
struct device_node *of_stdout;
static const char *of_stdout_options;
struct kset *of_kset;
/*
* Used to protect the of_aliases, to hold off addition of nodes to sysfs.
* This mutex must be held whenever modifications are being made to the
* device tree. The of_{attach,detach}_node() and
* of_{add,remove,update}_property() helpers make sure this happens.
*/
DEFINE_MUTEX(of_mutex);
/* use when traversing tree through the child, sibling,
* or parent members of struct device_node.
*/
DEFINE_RAW_SPINLOCK(devtree_lock);
bool of_node_name_eq(const struct device_node *np, const char *name)
{
const char *node_name;
size_t len;
if (!np)
return false;
node_name = kbasename(np->full_name);
len = strchrnul(node_name, '@') - node_name;
return (strlen(name) == len) && (strncmp(node_name, name, len) == 0);
}
EXPORT_SYMBOL(of_node_name_eq);
bool of_node_name_prefix(const struct device_node *np, const char *prefix)
{
if (!np)
return false;
return strncmp(kbasename(np->full_name), prefix, strlen(prefix)) == 0;
}
EXPORT_SYMBOL(of_node_name_prefix);
static bool __of_node_is_type(const struct device_node *np, const char *type)
{
const char *match = __of_get_property(np, "device_type", NULL);
return np && match && type && !strcmp(match, type);
}
int of_bus_n_addr_cells(struct device_node *np)
{
u32 cells;
for (; np; np = np->parent)
if (!of_property_read_u32(np, "#address-cells", &cells))
return cells;
/* No #address-cells property for the root node */
return OF_ROOT_NODE_ADDR_CELLS_DEFAULT;
}
int of_n_addr_cells(struct device_node *np)
{
if (np->parent)
np = np->parent;
return of_bus_n_addr_cells(np);
}
EXPORT_SYMBOL(of_n_addr_cells);
int of_bus_n_size_cells(struct device_node *np)
{
u32 cells;
for (; np; np = np->parent)
if (!of_property_read_u32(np, "#size-cells", &cells))
return cells;
/* No #size-cells property for the root node */
return OF_ROOT_NODE_SIZE_CELLS_DEFAULT;
}
int of_n_size_cells(struct device_node *np)
{
if (np->parent)
np = np->parent;
return of_bus_n_size_cells(np);
}
EXPORT_SYMBOL(of_n_size_cells);
#ifdef CONFIG_NUMA
int __weak of_node_to_nid(struct device_node *np)
{
return NUMA_NO_NODE;
}
#endif
#define OF_PHANDLE_CACHE_BITS 7
#define OF_PHANDLE_CACHE_SZ BIT(OF_PHANDLE_CACHE_BITS)
static struct device_node *phandle_cache[OF_PHANDLE_CACHE_SZ];
static u32 of_phandle_cache_hash(phandle handle)
{
return hash_32(handle, OF_PHANDLE_CACHE_BITS);
}
/*
* Caller must hold devtree_lock.
*/
void __of_phandle_cache_inv_entry(phandle handle)
{
u32 handle_hash;
struct device_node *np;
if (!handle)
return;
handle_hash = of_phandle_cache_hash(handle);
np = phandle_cache[handle_hash];
if (np && handle == np->phandle)
phandle_cache[handle_hash] = NULL;
}
void __init of_core_init(void)
{
struct device_node *np;
of_platform_register_reconfig_notifier();
/* Create the kset, and register existing nodes */
mutex_lock(&of_mutex);
of_kset = kset_create_and_add("devicetree", NULL, firmware_kobj);
if (!of_kset) {
mutex_unlock(&of_mutex);
pr_err("failed to register existing nodes\n");
return;
}
for_each_of_allnodes(np) {
__of_attach_node_sysfs(np);
if (np->phandle && !phandle_cache[of_phandle_cache_hash(np->phandle)])
phandle_cache[of_phandle_cache_hash(np->phandle)] = np;
}
mutex_unlock(&of_mutex);
/* Symlink in /proc as required by userspace ABI */
if (of_root)
proc_symlink("device-tree", NULL, "/sys/firmware/devicetree/base");
}
static struct property *__of_find_property(const struct device_node *np,
const char *name, int *lenp)
{
struct property *pp;
if (!np)
return NULL;
for (pp = np->properties; pp; pp = pp->next) {
if (of_prop_cmp(pp->name, name) == 0) {
if (lenp)
*lenp = pp->length;
break;
}
}
return pp;
}
struct property *of_find_property(const struct device_node *np,
const char *name,
int *lenp)
{
struct property *pp;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
pp = __of_find_property(np, name, lenp);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return pp;
}
EXPORT_SYMBOL(of_find_property);
struct device_node *__of_find_all_nodes(struct device_node *prev)
{
struct device_node *np;
if (!prev) {
np = of_root;
} else if (prev->child) {
np = prev->child;
} else {
/* Walk back up looking for a sibling, or the end of the structure */
np = prev;
while (np->parent && !np->sibling)
np = np->parent;
np = np->sibling; /* Might be null at the end of the tree */
}
return np;
}
/**
* of_find_all_nodes - Get next node in global list
* @prev: Previous node or NULL to start iteration
* of_node_put() will be called on it
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_all_nodes(struct device_node *prev)
{
struct device_node *np;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
np = __of_find_all_nodes(prev);
of_node_get(np);
of_node_put(prev);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_all_nodes);
/*
* Find a property with a given name for a given node
* and return the value.
*/
const void *__of_get_property(const struct device_node *np,
const char *name, int *lenp)
{
struct property *pp = __of_find_property(np, name, lenp);
return pp ? pp->value : NULL;
}
/*
* Find a property with a given name for a given node
* and return the value.
*/
const void *of_get_property(const struct device_node *np, const char *name,
int *lenp)
{
struct property *pp = of_find_property(np, name, lenp);
return pp ? pp->value : NULL;
}
EXPORT_SYMBOL(of_get_property);
/**
* __of_device_is_compatible() - Check if the node matches given constraints
* @device: pointer to node
* @compat: required compatible string, NULL or "" for any match
* @type: required device_type value, NULL or "" for any match
* @name: required node name, NULL or "" for any match
*
* Checks if the given @compat, @type and @name strings match the
* properties of the given @device. A constraints can be skipped by
* passing NULL or an empty string as the constraint.
*
* Returns 0 for no match, and a positive integer on match. The return
* value is a relative score with larger values indicating better
* matches. The score is weighted for the most specific compatible value
* to get the highest score. Matching type is next, followed by matching
* name. Practically speaking, this results in the following priority
* order for matches:
*
* 1. specific compatible && type && name
* 2. specific compatible && type
* 3. specific compatible && name
* 4. specific compatible
* 5. general compatible && type && name
* 6. general compatible && type
* 7. general compatible && name
* 8. general compatible
* 9. type && name
* 10. type
* 11. name
*/
static int __of_device_is_compatible(const struct device_node *device,
const char *compat, const char *type, const char *name)
{
struct property *prop;
const char *cp;
int index = 0, score = 0;
/* Compatible match has highest priority */
if (compat && compat[0]) {
prop = __of_find_property(device, "compatible", NULL);
for (cp = of_prop_next_string(prop, NULL); cp;
cp = of_prop_next_string(prop, cp), index++) {
if (of_compat_cmp(cp, compat, strlen(compat)) == 0) {
score = INT_MAX/2 - (index << 2);
break;
}
}
if (!score)
return 0;
}
/* Matching type is better than matching name */
if (type && type[0]) {
if (!__of_node_is_type(device, type))
return 0;
score += 2;
}
/* Matching name is a bit better than not */
if (name && name[0]) {
if (!of_node_name_eq(device, name))
return 0;
score++;
}
return score;
}
/** Checks if the given "compat" string matches one of the strings in
* the device's "compatible" property
*/
int of_device_is_compatible(const struct device_node *device,
const char *compat)
{
unsigned long flags;
int res;
raw_spin_lock_irqsave(&devtree_lock, flags);
res = __of_device_is_compatible(device, compat, NULL, NULL);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return res;
}
EXPORT_SYMBOL(of_device_is_compatible);
/** Checks if the device is compatible with any of the entries in
* a NULL terminated array of strings. Returns the best match
* score or 0.
*/
int of_device_compatible_match(const struct device_node *device,
const char *const *compat)
{
unsigned int tmp, score = 0;
if (!compat)
return 0;
while (*compat) {
tmp = of_device_is_compatible(device, *compat);
if (tmp > score)
score = tmp;
compat++;
}
return score;
}
EXPORT_SYMBOL_GPL(of_device_compatible_match);
/**
* of_machine_compatible_match - Test root of device tree against a compatible array
* @compats: NULL terminated array of compatible strings to look for in root node's compatible property.
*
* Returns true if the root node has any of the given compatible values in its
* compatible property.
*/
bool of_machine_compatible_match(const char *const *compats)
{
struct device_node *root;
int rc = 0;
root = of_find_node_by_path("/");
if (root) {
rc = of_device_compatible_match(root, compats);
of_node_put(root);
}
return rc != 0;
}
EXPORT_SYMBOL(of_machine_compatible_match);
static bool __of_device_is_status(const struct device_node *device,
const char * const*strings)
{
const char *status;
int statlen;
if (!device)
return false;
status = __of_get_property(device, "status", &statlen);
if (status == NULL)
return false;
if (statlen > 0) {
while (*strings) {
unsigned int len = strlen(*strings);
if ((*strings)[len - 1] == '-') {
if (!strncmp(status, *strings, len))
return true;
} else {
if (!strcmp(status, *strings))
return true;
}
strings++;
}
}
return false;
}
/**
* __of_device_is_available - check if a device is available for use
*
* @device: Node to check for availability, with locks already held
*
* Return: True if the status property is absent or set to "okay" or "ok",
* false otherwise
*/
static bool __of_device_is_available(const struct device_node *device)
{
static const char * const ok[] = {"okay", "ok", NULL};
if (!device)
return false;
return !__of_get_property(device, "status", NULL) ||
__of_device_is_status(device, ok);
}
/**
* __of_device_is_reserved - check if a device is reserved
*
* @device: Node to check for availability, with locks already held
*
* Return: True if the status property is set to "reserved", false otherwise
*/
static bool __of_device_is_reserved(const struct device_node *device)
{
static const char * const reserved[] = {"reserved", NULL};
return __of_device_is_status(device, reserved);
}
/**
* of_device_is_available - check if a device is available for use
*
* @device: Node to check for availability
*
* Return: True if the status property is absent or set to "okay" or "ok",
* false otherwise
*/
bool of_device_is_available(const struct device_node *device)
{
unsigned long flags;
bool res;
raw_spin_lock_irqsave(&devtree_lock, flags);
res = __of_device_is_available(device);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return res;
}
EXPORT_SYMBOL(of_device_is_available);
/**
* __of_device_is_fail - check if a device has status "fail" or "fail-..."
*
* @device: Node to check status for, with locks already held
*
* Return: True if the status property is set to "fail" or "fail-..." (for any
* error code suffix), false otherwise
*/
static bool __of_device_is_fail(const struct device_node *device)
{
static const char * const fail[] = {"fail", "fail-", NULL};
return __of_device_is_status(device, fail);
}
/**
* of_device_is_big_endian - check if a device has BE registers
*
* @device: Node to check for endianness
*
* Return: True if the device has a "big-endian" property, or if the kernel
* was compiled for BE *and* the device has a "native-endian" property.
* Returns false otherwise.
*
* Callers would nominally use ioread32be/iowrite32be if
* of_device_is_big_endian() == true, or readl/writel otherwise.
*/
bool of_device_is_big_endian(const struct device_node *device)
{
if (of_property_read_bool(device, "big-endian"))
return true;
if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) &&
of_property_read_bool(device, "native-endian"))
return true;
return false;
}
EXPORT_SYMBOL(of_device_is_big_endian);
/**
* of_get_parent - Get a node's parent if any
* @node: Node to get parent
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_get_parent(const struct device_node *node)
{
struct device_node *np;
unsigned long flags;
if (!node)
return NULL;
raw_spin_lock_irqsave(&devtree_lock, flags);
np = of_node_get(node->parent);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_get_parent);
/**
* of_get_next_parent - Iterate to a node's parent
* @node: Node to get parent of
*
* This is like of_get_parent() except that it drops the
* refcount on the passed node, making it suitable for iterating
* through a node's parents.
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_get_next_parent(struct device_node *node)
{
struct device_node *parent;
unsigned long flags;
if (!node)
return NULL;
raw_spin_lock_irqsave(&devtree_lock, flags);
parent = of_node_get(node->parent);
of_node_put(node);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return parent;
}
EXPORT_SYMBOL(of_get_next_parent);
static struct device_node *__of_get_next_child(const struct device_node *node,
struct device_node *prev)
{
struct device_node *next;
if (!node)
return NULL;
next = prev ? prev->sibling : node->child;
of_node_get(next);
of_node_put(prev);
return next;
}
#define __for_each_child_of_node(parent, child) \
for (child = __of_get_next_child(parent, NULL); child != NULL; \
child = __of_get_next_child(parent, child))
/**
* of_get_next_child - Iterate a node childs
* @node: parent node
* @prev: previous child of the parent node, or NULL to get first
*
* Return: A node pointer with refcount incremented, use of_node_put() on
* it when done. Returns NULL when prev is the last child. Decrements the
* refcount of prev.
*/
struct device_node *of_get_next_child(const struct device_node *node,
struct device_node *prev)
{
struct device_node *next;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags); next = __of_get_next_child(node, prev); raw_spin_unlock_irqrestore(&devtree_lock, flags);
return next;
}
EXPORT_SYMBOL(of_get_next_child);
static struct device_node *of_get_next_status_child(const struct device_node *node,
struct device_node *prev,
bool (*checker)(const struct device_node *))
{
struct device_node *next;
unsigned long flags;
if (!node)
return NULL;
raw_spin_lock_irqsave(&devtree_lock, flags);
next = prev ? prev->sibling : node->child;
for (; next; next = next->sibling) {
if (!checker(next))
continue;
if (of_node_get(next))
break;
}
of_node_put(prev);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return next;
}
/**
* of_get_next_available_child - Find the next available child node
* @node: parent node
* @prev: previous child of the parent node, or NULL to get first
*
* This function is like of_get_next_child(), except that it
* automatically skips any disabled nodes (i.e. status = "disabled").
*/
struct device_node *of_get_next_available_child(const struct device_node *node,
struct device_node *prev)
{
return of_get_next_status_child(node, prev, __of_device_is_available);
}
EXPORT_SYMBOL(of_get_next_available_child);
/**
* of_get_next_reserved_child - Find the next reserved child node
* @node: parent node
* @prev: previous child of the parent node, or NULL to get first
*
* This function is like of_get_next_child(), except that it
* automatically skips any disabled nodes (i.e. status = "disabled").
*/
struct device_node *of_get_next_reserved_child(const struct device_node *node,
struct device_node *prev)
{
return of_get_next_status_child(node, prev, __of_device_is_reserved);
}
EXPORT_SYMBOL(of_get_next_reserved_child);
/**
* of_get_next_cpu_node - Iterate on cpu nodes
* @prev: previous child of the /cpus node, or NULL to get first
*
* Unusable CPUs (those with the status property set to "fail" or "fail-...")
* will be skipped.
*
* Return: A cpu node pointer with refcount incremented, use of_node_put()
* on it when done. Returns NULL when prev is the last child. Decrements
* the refcount of prev.
*/
struct device_node *of_get_next_cpu_node(struct device_node *prev)
{
struct device_node *next = NULL;
unsigned long flags;
struct device_node *node;
if (!prev)
node = of_find_node_by_path("/cpus");
raw_spin_lock_irqsave(&devtree_lock, flags);
if (prev)
next = prev->sibling;
else if (node) {
next = node->child;
of_node_put(node);
}
for (; next; next = next->sibling) {
if (__of_device_is_fail(next))
continue;
if (!(of_node_name_eq(next, "cpu") ||
__of_node_is_type(next, "cpu")))
continue;
if (of_node_get(next))
break;
}
of_node_put(prev);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return next;
}
EXPORT_SYMBOL(of_get_next_cpu_node);
/**
* of_get_compatible_child - Find compatible child node
* @parent: parent node
* @compatible: compatible string
*
* Lookup child node whose compatible property contains the given compatible
* string.
*
* Return: a node pointer with refcount incremented, use of_node_put() on it
* when done; or NULL if not found.
*/
struct device_node *of_get_compatible_child(const struct device_node *parent,
const char *compatible)
{
struct device_node *child;
for_each_child_of_node(parent, child) {
if (of_device_is_compatible(child, compatible))
break;
}
return child;
}
EXPORT_SYMBOL(of_get_compatible_child);
/**
* of_get_child_by_name - Find the child node by name for a given parent
* @node: parent node
* @name: child name to look for.
*
* This function looks for child node for given matching name
*
* Return: A node pointer if found, with refcount incremented, use
* of_node_put() on it when done.
* Returns NULL if node is not found.
*/
struct device_node *of_get_child_by_name(const struct device_node *node,
const char *name)
{
struct device_node *child;
for_each_child_of_node(node, child)
if (of_node_name_eq(child, name))
break;
return child;
}
EXPORT_SYMBOL(of_get_child_by_name);
struct device_node *__of_find_node_by_path(struct device_node *parent,
const char *path)
{
struct device_node *child;
int len;
len = strcspn(path, "/:");
if (!len)
return NULL;
__for_each_child_of_node(parent, child) {
const char *name = kbasename(child->full_name);
if (strncmp(path, name, len) == 0 && (strlen(name) == len))
return child;
}
return NULL;
}
struct device_node *__of_find_node_by_full_path(struct device_node *node,
const char *path)
{
const char *separator = strchr(path, ':');
while (node && *path == '/') {
struct device_node *tmp = node;
path++; /* Increment past '/' delimiter */
node = __of_find_node_by_path(node, path);
of_node_put(tmp);
path = strchrnul(path, '/');
if (separator && separator < path)
break;
}
return node;
}
/**
* of_find_node_opts_by_path - Find a node matching a full OF path
* @path: Either the full path to match, or if the path does not
* start with '/', the name of a property of the /aliases
* node (an alias). In the case of an alias, the node
* matching the alias' value will be returned.
* @opts: Address of a pointer into which to store the start of
* an options string appended to the end of the path with
* a ':' separator.
*
* Valid paths:
* * /foo/bar Full path
* * foo Valid alias
* * foo/bar Valid alias + relative path
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_node_opts_by_path(const char *path, const char **opts)
{
struct device_node *np = NULL;
struct property *pp;
unsigned long flags;
const char *separator = strchr(path, ':');
if (opts)
*opts = separator ? separator + 1 : NULL;
if (strcmp(path, "/") == 0)
return of_node_get(of_root);
/* The path could begin with an alias */
if (*path != '/') {
int len;
const char *p = separator;
if (!p)
p = strchrnul(path, '/');
len = p - path;
/* of_aliases must not be NULL */
if (!of_aliases)
return NULL;
for_each_property_of_node(of_aliases, pp) {
if (strlen(pp->name) == len && !strncmp(pp->name, path, len)) {
np = of_find_node_by_path(pp->value);
break;
}
}
if (!np)
return NULL;
path = p;
}
/* Step down the tree matching path components */
raw_spin_lock_irqsave(&devtree_lock, flags);
if (!np)
np = of_node_get(of_root);
np = __of_find_node_by_full_path(np, path);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_node_opts_by_path);
/**
* of_find_node_by_name - Find a node by its "name" property
* @from: The node to start searching from or NULL; the node
* you pass will not be searched, only the next one
* will. Typically, you pass what the previous call
* returned. of_node_put() will be called on @from.
* @name: The name string to match against
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_node_by_name(struct device_node *from,
const char *name)
{
struct device_node *np;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
for_each_of_allnodes_from(from, np)
if (of_node_name_eq(np, name) && of_node_get(np))
break;
of_node_put(from);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_node_by_name);
/**
* of_find_node_by_type - Find a node by its "device_type" property
* @from: The node to start searching from, or NULL to start searching
* the entire device tree. The node you pass will not be
* searched, only the next one will; typically, you pass
* what the previous call returned. of_node_put() will be
* called on from for you.
* @type: The type string to match against
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_node_by_type(struct device_node *from,
const char *type)
{
struct device_node *np;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
for_each_of_allnodes_from(from, np)
if (__of_node_is_type(np, type) && of_node_get(np))
break;
of_node_put(from);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_node_by_type);
/**
* of_find_compatible_node - Find a node based on type and one of the
* tokens in its "compatible" property
* @from: The node to start searching from or NULL, the node
* you pass will not be searched, only the next one
* will; typically, you pass what the previous call
* returned. of_node_put() will be called on it
* @type: The type string to match "device_type" or NULL to ignore
* @compatible: The string to match to one of the tokens in the device
* "compatible" list.
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_compatible_node(struct device_node *from,
const char *type, const char *compatible)
{
struct device_node *np;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
for_each_of_allnodes_from(from, np)
if (__of_device_is_compatible(np, compatible, type, NULL) &&
of_node_get(np))
break;
of_node_put(from);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_compatible_node);
/**
* of_find_node_with_property - Find a node which has a property with
* the given name.
* @from: The node to start searching from or NULL, the node
* you pass will not be searched, only the next one
* will; typically, you pass what the previous call
* returned. of_node_put() will be called on it
* @prop_name: The name of the property to look for.
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_node_with_property(struct device_node *from,
const char *prop_name)
{
struct device_node *np;
struct property *pp;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
for_each_of_allnodes_from(from, np) {
for (pp = np->properties; pp; pp = pp->next) {
if (of_prop_cmp(pp->name, prop_name) == 0) {
of_node_get(np);
goto out;
}
}
}
out:
of_node_put(from);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_node_with_property);
static
const struct of_device_id *__of_match_node(const struct of_device_id *matches,
const struct device_node *node)
{
const struct of_device_id *best_match = NULL;
int score, best_score = 0;
if (!matches)
return NULL;
for (; matches->name[0] || matches->type[0] || matches->compatible[0]; matches++) {
score = __of_device_is_compatible(node, matches->compatible,
matches->type, matches->name);
if (score > best_score) {
best_match = matches;
best_score = score;
}
}
return best_match;
}
/**
* of_match_node - Tell if a device_node has a matching of_match structure
* @matches: array of of device match structures to search in
* @node: the of device structure to match against
*
* Low level utility function used by device matching.
*/
const struct of_device_id *of_match_node(const struct of_device_id *matches,
const struct device_node *node)
{
const struct of_device_id *match;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
match = __of_match_node(matches, node);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return match;
}
EXPORT_SYMBOL(of_match_node);
/**
* of_find_matching_node_and_match - Find a node based on an of_device_id
* match table.
* @from: The node to start searching from or NULL, the node
* you pass will not be searched, only the next one
* will; typically, you pass what the previous call
* returned. of_node_put() will be called on it
* @matches: array of of device match structures to search in
* @match: Updated to point at the matches entry which matched
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_matching_node_and_match(struct device_node *from,
const struct of_device_id *matches,
const struct of_device_id **match)
{
struct device_node *np;
const struct of_device_id *m;
unsigned long flags;
if (match)
*match = NULL;
raw_spin_lock_irqsave(&devtree_lock, flags);
for_each_of_allnodes_from(from, np) {
m = __of_match_node(matches, np);
if (m && of_node_get(np)) {
if (match)
*match = m;
break;
}
}
of_node_put(from);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_matching_node_and_match);
/**
* of_alias_from_compatible - Lookup appropriate alias for a device node
* depending on compatible
* @node: pointer to a device tree node
* @alias: Pointer to buffer that alias value will be copied into
* @len: Length of alias value
*
* Based on the value of the compatible property, this routine will attempt
* to choose an appropriate alias value for a particular device tree node.
* It does this by stripping the manufacturer prefix (as delimited by a ',')
* from the first entry in the compatible list property.
*
* Note: The matching on just the "product" side of the compatible is a relic
* from I2C and SPI. Please do not add any new user.
*
* Return: This routine returns 0 on success, <0 on failure.
*/
int of_alias_from_compatible(const struct device_node *node, char *alias, int len)
{
const char *compatible, *p;
int cplen;
compatible = of_get_property(node, "compatible", &cplen);
if (!compatible || strlen(compatible) > cplen)
return -ENODEV;
p = strchr(compatible, ',');
strscpy(alias, p ? p + 1 : compatible, len);
return 0;
}
EXPORT_SYMBOL_GPL(of_alias_from_compatible);
/**
* of_find_node_by_phandle - Find a node given a phandle
* @handle: phandle of the node to find
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done.
*/
struct device_node *of_find_node_by_phandle(phandle handle)
{
struct device_node *np = NULL;
unsigned long flags;
u32 handle_hash;
if (!handle)
return NULL;
handle_hash = of_phandle_cache_hash(handle);
raw_spin_lock_irqsave(&devtree_lock, flags);
if (phandle_cache[handle_hash] &&
handle == phandle_cache[handle_hash]->phandle)
np = phandle_cache[handle_hash];
if (!np) {
for_each_of_allnodes(np)
if (np->phandle == handle &&
!of_node_check_flag(np, OF_DETACHED)) {
phandle_cache[handle_hash] = np;
break;
}
}
of_node_get(np);
raw_spin_unlock_irqrestore(&devtree_lock, flags);
return np;
}
EXPORT_SYMBOL(of_find_node_by_phandle);
void of_print_phandle_args(const char *msg, const struct of_phandle_args *args)
{
int i;
printk("%s %pOF", msg, args->np);
for (i = 0; i < args->args_count; i++) {
const char delim = i ? ',' : ':';
pr_cont("%c%08x", delim, args->args[i]);
}
pr_cont("\n");
}
int of_phandle_iterator_init(struct of_phandle_iterator *it,
const struct device_node *np,
const char *list_name,
const char *cells_name,
int cell_count)
{
const __be32 *list;
int size;
memset(it, 0, sizeof(*it));
/*
* one of cell_count or cells_name must be provided to determine the
* argument length.
*/
if (cell_count < 0 && !cells_name)
return -EINVAL;
list = of_get_property(np, list_name, &size);
if (!list)
return -ENOENT;
it->cells_name = cells_name;
it->cell_count = cell_count;
it->parent = np;
it->list_end = list + size / sizeof(*list);
it->phandle_end = list;
it->cur = list;
return 0;
}
EXPORT_SYMBOL_GPL(of_phandle_iterator_init);
int of_phandle_iterator_next(struct of_phandle_iterator *it)
{
uint32_t count = 0;
if (it->node) {
of_node_put(it->node);
it->node = NULL;
}
if (!it->cur || it->phandle_end >= it->list_end)
return -ENOENT;
it->cur = it->phandle_end;
/* If phandle is 0, then it is an empty entry with no arguments. */
it->phandle = be32_to_cpup(it->cur++);
if (it->phandle) {
/*
* Find the provider node and parse the #*-cells property to
* determine the argument length.
*/
it->node = of_find_node_by_phandle(it->phandle);
if (it->cells_name) {
if (!it->node) {
pr_err("%pOF: could not find phandle %d\n",
it->parent, it->phandle);
goto err;
}
if (of_property_read_u32(it->node, it->cells_name,
&count)) {
/*
* If both cell_count and cells_name is given,
* fall back to cell_count in absence
* of the cells_name property
*/
if (it->cell_count >= 0) {
count = it->cell_count;
} else {
pr_err("%pOF: could not get %s for %pOF\n",
it->parent,
it->cells_name,
it->node);
goto err;
}
}
} else {
count = it->cell_count;
}
/*
* Make sure that the arguments actually fit in the remaining
* property data length
*/
if (it->cur + count > it->list_end) {
if (it->cells_name)
pr_err("%pOF: %s = %d found %td\n",
it->parent, it->cells_name,
count, it->list_end - it->cur);
else
pr_err("%pOF: phandle %s needs %d, found %td\n",
it->parent, of_node_full_name(it->node),
count, it->list_end - it->cur);
goto err;
}
}
it->phandle_end = it->cur + count;
it->cur_count = count;
return 0;
err:
if (it->node) {
of_node_put(it->node);
it->node = NULL;
}
return -EINVAL;
}
EXPORT_SYMBOL_GPL(of_phandle_iterator_next);
int of_phandle_iterator_args(struct of_phandle_iterator *it,
uint32_t *args,
int size)
{
int i, count;
count = it->cur_count;
if (WARN_ON(size < count))
count = size;
for (i = 0; i < count; i++)
args[i] = be32_to_cpup(it->cur++);
return count;
}
int __of_parse_phandle_with_args(const struct device_node *np,
const char *list_name,
const char *cells_name,
int cell_count, int index,
struct of_phandle_args *out_args)
{
struct of_phandle_iterator it;
int rc, cur_index = 0;
if (index < 0)
return -EINVAL;
/* Loop over the phandles until all the requested entry is found */
of_for_each_phandle(&it, rc, np, list_name, cells_name, cell_count) {
/*
* All of the error cases bail out of the loop, so at
* this point, the parsing is successful. If the requested
* index matches, then fill the out_args structure and return,
* or return -ENOENT for an empty entry.
*/
rc = -ENOENT;
if (cur_index == index) {
if (!it.phandle)
goto err;
if (out_args) {
int c;
c = of_phandle_iterator_args(&it,
out_args->args,
MAX_PHANDLE_ARGS);
out_args->np = it.node;
out_args->args_count = c;
} else {
of_node_put(it.node);
}
/* Found it! return success */
return 0;
}
cur_index++;
}
/*
* Unlock node before returning result; will be one of:
* -ENOENT : index is for empty phandle
* -EINVAL : parsing error on data
*/
err:
of_node_put(it.node);
return rc;
}
EXPORT_SYMBOL(__of_parse_phandle_with_args);
/**
* of_parse_phandle_with_args_map() - Find a node pointed by phandle in a list and remap it
* @np: pointer to a device tree node containing a list
* @list_name: property name that contains a list
* @stem_name: stem of property names that specify phandles' arguments count
* @index: index of a phandle to parse out
* @out_args: optional pointer to output arguments structure (will be filled)
*
* This function is useful to parse lists of phandles and their arguments.
* Returns 0 on success and fills out_args, on error returns appropriate errno
* value. The difference between this function and of_parse_phandle_with_args()
* is that this API remaps a phandle if the node the phandle points to has
* a <@stem_name>-map property.
*
* Caller is responsible to call of_node_put() on the returned out_args->np
* pointer.
*
* Example::
*
* phandle1: node1 {
* #list-cells = <2>;
* };
*
* phandle2: node2 {
* #list-cells = <1>;
* };
*
* phandle3: node3 {
* #list-cells = <1>;
* list-map = <0 &phandle2 3>,
* <1 &phandle2 2>,
* <2 &phandle1 5 1>;
* list-map-mask = <0x3>;
* };
*
* node4 {
* list = <&phandle1 1 2 &phandle3 0>;
* };
*
* To get a device_node of the ``node2`` node you may call this:
* of_parse_phandle_with_args(node4, "list", "list", 1, &args);
*/
int of_parse_phandle_with_args_map(const struct device_node *np,
const char *list_name,
const char *stem_name,
int index, struct of_phandle_args *out_args)
{
char *cells_name, *map_name = NULL, *mask_name = NULL;
char *pass_name = NULL;
struct device_node *cur, *new = NULL;
const __be32 *map, *mask, *pass;
static const __be32 dummy_mask[] = { [0 ... MAX_PHANDLE_ARGS] = cpu_to_be32(~0) };
static const __be32 dummy_pass[] = { [0 ... MAX_PHANDLE_ARGS] = cpu_to_be32(0) };
__be32 initial_match_array[MAX_PHANDLE_ARGS];
const __be32 *match_array = initial_match_array;
int i, ret, map_len, match;
u32 list_size, new_size;
if (index < 0)
return -EINVAL;
cells_name = kasprintf(GFP_KERNEL, "#%s-cells", stem_name);
if (!cells_name)
return -ENOMEM;
ret = -ENOMEM;
map_name = kasprintf(GFP_KERNEL, "%s-map", stem_name);
if (!map_name)
goto free;
mask_name = kasprintf(GFP_KERNEL, "%s-map-mask", stem_name);
if (!mask_name)
goto free;
pass_name = kasprintf(GFP_KERNEL, "%s-map-pass-thru", stem_name);
if (!pass_name)
goto free;
ret = __of_parse_phandle_with_args(np, list_name, cells_name, -1, index,
out_args);
if (ret)
goto free;
/* Get the #<list>-cells property */
cur = out_args->np;
ret = of_property_read_u32(cur, cells_name, &list_size);
if (ret < 0)
goto put;
/* Precalculate the match array - this simplifies match loop */
for (i = 0; i < list_size; i++)
initial_match_array[i] = cpu_to_be32(out_args->args[i]);
ret = -EINVAL;
while (cur) {
/* Get the <list>-map property */
map = of_get_property(cur, map_name, &map_len);
if (!map) {
ret = 0;
goto free;
}
map_len /= sizeof(u32);
/* Get the <list>-map-mask property (optional) */
mask = of_get_property(cur, mask_name, NULL);
if (!mask)
mask = dummy_mask;
/* Iterate through <list>-map property */
match = 0;
while (map_len > (list_size + 1) && !match) {
/* Compare specifiers */
match = 1;
for (i = 0; i < list_size; i++, map_len--)
match &= !((match_array[i] ^ *map++) & mask[i]);
of_node_put(new);
new = of_find_node_by_phandle(be32_to_cpup(map));
map++;
map_len--;
/* Check if not found */
if (!new)
goto put;
if (!of_device_is_available(new))
match = 0;
ret = of_property_read_u32(new, cells_name, &new_size);
if (ret)
goto put;
/* Check for malformed properties */
if (WARN_ON(new_size > MAX_PHANDLE_ARGS))
goto put;
if (map_len < new_size)
goto put;
/* Move forward by new node's #<list>-cells amount */
map += new_size;
map_len -= new_size;
}
if (!match)
goto put;
/* Get the <list>-map-pass-thru property (optional) */
pass = of_get_property(cur, pass_name, NULL);
if (!pass)
pass = dummy_pass;
/*
* Successfully parsed a <list>-map translation; copy new
* specifier into the out_args structure, keeping the
* bits specified in <list>-map-pass-thru.
*/
match_array = map - new_size;
for (i = 0; i < new_size; i++) {
__be32 val = *(map - new_size + i);
if (i < list_size) {
val &= ~pass[i];
val |= cpu_to_be32(out_args->args[i]) & pass[i];
}
out_args->args[i] = be32_to_cpu(val);
}
out_args->args_count = list_size = new_size;
/* Iterate again with new provider */
out_args->np = new;
of_node_put(cur);
cur = new;
new = NULL;
}
put:
of_node_put(cur);
of_node_put(new);
free:
kfree(mask_name);
kfree(map_name);
kfree(cells_name);
kfree(pass_name);
return ret;
}
EXPORT_SYMBOL(of_parse_phandle_with_args_map);
/**
* of_count_phandle_with_args() - Find the number of phandles references in a property
* @np: pointer to a device tree node containing a list
* @list_name: property name that contains a list
* @cells_name: property name that specifies phandles' arguments count
*
* Return: The number of phandle + argument tuples within a property. It
* is a typical pattern to encode a list of phandle and variable
* arguments into a single property. The number of arguments is encoded
* by a property in the phandle-target node. For example, a gpios
* property would contain a list of GPIO specifies consisting of a
* phandle and 1 or more arguments. The number of arguments are
* determined by the #gpio-cells property in the node pointed to by the
* phandle.
*/
int of_count_phandle_with_args(const struct device_node *np, const char *list_name,
const char *cells_name)
{
struct of_phandle_iterator it;
int rc, cur_index = 0;
/*
* If cells_name is NULL we assume a cell count of 0. This makes
* counting the phandles trivial as each 32bit word in the list is a
* phandle and no arguments are to consider. So we don't iterate through
* the list but just use the length to determine the phandle count.
*/
if (!cells_name) {
const __be32 *list;
int size;
list = of_get_property(np, list_name, &size);
if (!list)
return -ENOENT;
return size / sizeof(*list);
}
rc = of_phandle_iterator_init(&it, np, list_name, cells_name, -1);
if (rc)
return rc;
while ((rc = of_phandle_iterator_next(&it)) == 0)
cur_index += 1;
if (rc != -ENOENT)
return rc;
return cur_index;
}
EXPORT_SYMBOL(of_count_phandle_with_args);
static struct property *__of_remove_property_from_list(struct property **list, struct property *prop)
{
struct property **next;
for (next = list; *next; next = &(*next)->next) {
if (*next == prop) {
*next = prop->next;
prop->next = NULL;
return prop;
}
}
return NULL;
}
/**
* __of_add_property - Add a property to a node without lock operations
* @np: Caller's Device Node
* @prop: Property to add
*/
int __of_add_property(struct device_node *np, struct property *prop)
{
int rc = 0;
unsigned long flags;
struct property **next;
raw_spin_lock_irqsave(&devtree_lock, flags);
__of_remove_property_from_list(&np->deadprops, prop);
prop->next = NULL;
next = &np->properties;
while (*next) {
if (strcmp(prop->name, (*next)->name) == 0) {
/* duplicate ! don't insert it */
rc = -EEXIST;
goto out_unlock;
}
next = &(*next)->next;
}
*next = prop;
out_unlock:
raw_spin_unlock_irqrestore(&devtree_lock, flags);
if (rc)
return rc;
__of_add_property_sysfs(np, prop);
return 0;
}
/**
* of_add_property - Add a property to a node
* @np: Caller's Device Node
* @prop: Property to add
*/
int of_add_property(struct device_node *np, struct property *prop)
{
int rc;
mutex_lock(&of_mutex);
rc = __of_add_property(np, prop);
mutex_unlock(&of_mutex);
if (!rc)
of_property_notify(OF_RECONFIG_ADD_PROPERTY, np, prop, NULL);
return rc;
}
EXPORT_SYMBOL_GPL(of_add_property);
int __of_remove_property(struct device_node *np, struct property *prop)
{
unsigned long flags;
int rc = -ENODEV;
raw_spin_lock_irqsave(&devtree_lock, flags);
if (__of_remove_property_from_list(&np->properties, prop)) {
/* Found the property, add it to deadprops list */
prop->next = np->deadprops;
np->deadprops = prop;
rc = 0;
}
raw_spin_unlock_irqrestore(&devtree_lock, flags);
if (rc)
return rc;
__of_remove_property_sysfs(np, prop);
return 0;
}
/**
* of_remove_property - Remove a property from a node.
* @np: Caller's Device Node
* @prop: Property to remove
*
* Note that we don't actually remove it, since we have given out
* who-knows-how-many pointers to the data using get-property.
* Instead we just move the property to the "dead properties"
* list, so it won't be found any more.
*/
int of_remove_property(struct device_node *np, struct property *prop)
{
int rc;
if (!prop)
return -ENODEV;
mutex_lock(&of_mutex);
rc = __of_remove_property(np, prop);
mutex_unlock(&of_mutex);
if (!rc)
of_property_notify(OF_RECONFIG_REMOVE_PROPERTY, np, prop, NULL);
return rc;
}
EXPORT_SYMBOL_GPL(of_remove_property);
int __of_update_property(struct device_node *np, struct property *newprop,
struct property **oldpropp)
{
struct property **next, *oldprop;
unsigned long flags;
raw_spin_lock_irqsave(&devtree_lock, flags);
__of_remove_property_from_list(&np->deadprops, newprop);
for (next = &np->properties; *next; next = &(*next)->next) {
if (of_prop_cmp((*next)->name, newprop->name) == 0)
break;
}
*oldpropp = oldprop = *next;
if (oldprop) {
/* replace the node */
newprop->next = oldprop->next;
*next = newprop;
oldprop->next = np->deadprops;
np->deadprops = oldprop;
} else {
/* new node */
newprop->next = NULL;
*next = newprop;
}
raw_spin_unlock_irqrestore(&devtree_lock, flags);
__of_update_property_sysfs(np, newprop, oldprop);
return 0;
}
/*
* of_update_property - Update a property in a node, if the property does
* not exist, add it.
*
* Note that we don't actually remove it, since we have given out
* who-knows-how-many pointers to the data using get-property.
* Instead we just move the property to the "dead properties" list,
* and add the new property to the property list
*/
int of_update_property(struct device_node *np, struct property *newprop)
{
struct property *oldprop;
int rc;
if (!newprop->name)
return -EINVAL;
mutex_lock(&of_mutex);
rc = __of_update_property(np, newprop, &oldprop);
mutex_unlock(&of_mutex);
if (!rc)
of_property_notify(OF_RECONFIG_UPDATE_PROPERTY, np, newprop, oldprop);
return rc;
}
static void of_alias_add(struct alias_prop *ap, struct device_node *np,
int id, const char *stem, int stem_len)
{
ap->np = np;
ap->id = id;
strscpy(ap->stem, stem, stem_len + 1);
list_add_tail(&ap->link, &aliases_lookup);
pr_debug("adding DT alias:%s: stem=%s id=%i node=%pOF\n",
ap->alias, ap->stem, ap->id, np);
}
/**
* of_alias_scan - Scan all properties of the 'aliases' node
* @dt_alloc: An allocator that provides a virtual address to memory
* for storing the resulting tree
*
* The function scans all the properties of the 'aliases' node and populates
* the global lookup table with the properties. It returns the
* number of alias properties found, or an error code in case of failure.
*/
void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align))
{
struct property *pp;
of_aliases = of_find_node_by_path("/aliases");
of_chosen = of_find_node_by_path("/chosen");
if (of_chosen == NULL)
of_chosen = of_find_node_by_path("/chosen@0");
if (of_chosen) {
/* linux,stdout-path and /aliases/stdout are for legacy compatibility */
const char *name = NULL;
if (of_property_read_string(of_chosen, "stdout-path", &name))
of_property_read_string(of_chosen, "linux,stdout-path",
&name);
if (IS_ENABLED(CONFIG_PPC) && !name)
of_property_read_string(of_aliases, "stdout", &name);
if (name)
of_stdout = of_find_node_opts_by_path(name, &of_stdout_options);
if (of_stdout)
of_stdout->fwnode.flags |= FWNODE_FLAG_BEST_EFFORT;
}
if (!of_aliases)
return;
for_each_property_of_node(of_aliases, pp) {
const char *start = pp->name;
const char *end = start + strlen(start);
struct device_node *np;
struct alias_prop *ap;
int id, len;
/* Skip those we do not want to proceed */
if (!strcmp(pp->name, "name") ||
!strcmp(pp->name, "phandle") ||
!strcmp(pp->name, "linux,phandle"))
continue;
np = of_find_node_by_path(pp->value);
if (!np)
continue;
/* walk the alias backwards to extract the id and work out
* the 'stem' string */
while (isdigit(*(end-1)) && end > start)
end--;
len = end - start;
if (kstrtoint(end, 10, &id) < 0)
continue;
/* Allocate an alias_prop with enough space for the stem */
ap = dt_alloc(sizeof(*ap) + len + 1, __alignof__(*ap));
if (!ap)
continue;
memset(ap, 0, sizeof(*ap) + len + 1);
ap->alias = start;
of_alias_add(ap, np, id, start, len);
}
}
/**
* of_alias_get_id - Get alias id for the given device_node
* @np: Pointer to the given device_node
* @stem: Alias stem of the given device_node
*
* The function travels the lookup table to get the alias id for the given
* device_node and alias stem.
*
* Return: The alias id if found.
*/
int of_alias_get_id(struct device_node *np, const char *stem)
{
struct alias_prop *app;
int id = -ENODEV;
mutex_lock(&of_mutex);
list_for_each_entry(app, &aliases_lookup, link) {
if (strcmp(app->stem, stem) != 0)
continue;
if (np == app->np) {
id = app->id;
break;
}
}
mutex_unlock(&of_mutex);
return id;
}
EXPORT_SYMBOL_GPL(of_alias_get_id);
/**
* of_alias_get_highest_id - Get highest alias id for the given stem
* @stem: Alias stem to be examined
*
* The function travels the lookup table to get the highest alias id for the
* given alias stem. It returns the alias id if found.
*/
int of_alias_get_highest_id(const char *stem)
{
struct alias_prop *app;
int id = -ENODEV;
mutex_lock(&of_mutex);
list_for_each_entry(app, &aliases_lookup, link) {
if (strcmp(app->stem, stem) != 0)
continue;
if (app->id > id)
id = app->id;
}
mutex_unlock(&of_mutex);
return id;
}
EXPORT_SYMBOL_GPL(of_alias_get_highest_id);
/**
* of_console_check() - Test and setup console for DT setup
* @dn: Pointer to device node
* @name: Name to use for preferred console without index. ex. "ttyS"
* @index: Index to use for preferred console.
*
* Check if the given device node matches the stdout-path property in the
* /chosen node. If it does then register it as the preferred console.
*
* Return: TRUE if console successfully setup. Otherwise return FALSE.
*/
bool of_console_check(struct device_node *dn, char *name, int index)
{
if (!dn || dn != of_stdout || console_set_on_cmdline)
return false;
/*
* XXX: cast `options' to char pointer to suppress complication
* warnings: printk, UART and console drivers expect char pointer.
*/
return !add_preferred_console(name, index, (char *)of_stdout_options);
}
EXPORT_SYMBOL_GPL(of_console_check);
/**
* of_find_next_cache_node - Find a node's subsidiary cache
* @np: node of type "cpu" or "cache"
*
* Return: A node pointer with refcount incremented, use
* of_node_put() on it when done. Caller should hold a reference
* to np.
*/
struct device_node *of_find_next_cache_node(const struct device_node *np)
{
struct device_node *child, *cache_node;
cache_node = of_parse_phandle(np, "l2-cache", 0);
if (!cache_node)
cache_node = of_parse_phandle(np, "next-level-cache", 0);
if (cache_node)
return cache_node;
/* OF on pmac has nodes instead of properties named "l2-cache"
* beneath CPU nodes.
*/
if (IS_ENABLED(CONFIG_PPC_PMAC) && of_node_is_type(np, "cpu"))
for_each_child_of_node(np, child)
if (of_node_is_type(child, "cache"))
return child;
return NULL;
}
/**
* of_find_last_cache_level - Find the level at which the last cache is
* present for the given logical cpu
*
* @cpu: cpu number(logical index) for which the last cache level is needed
*
* Return: The level at which the last cache is present. It is exactly
* same as the total number of cache levels for the given logical cpu.
*/
int of_find_last_cache_level(unsigned int cpu)
{
u32 cache_level = 0;
struct device_node *prev = NULL, *np = of_cpu_device_node_get(cpu);
while (np) {
of_node_put(prev);
prev = np;
np = of_find_next_cache_node(np);
}
of_property_read_u32(prev, "cache-level", &cache_level);
of_node_put(prev);
return cache_level;
}
/**
* of_map_id - Translate an ID through a downstream mapping.
* @np: root complex device node.
* @id: device ID to map.
* @map_name: property name of the map to use.
* @map_mask_name: optional property name of the mask to use.
* @target: optional pointer to a target device node.
* @id_out: optional pointer to receive the translated ID.
*
* Given a device ID, look up the appropriate implementation-defined
* platform ID and/or the target device which receives transactions on that
* ID, as per the "iommu-map" and "msi-map" bindings. Either of @target or
* @id_out may be NULL if only the other is required. If @target points to
* a non-NULL device node pointer, only entries targeting that node will be
* matched; if it points to a NULL value, it will receive the device node of
* the first matching target phandle, with a reference held.
*
* Return: 0 on success or a standard error code on failure.
*/
int of_map_id(struct device_node *np, u32 id,
const char *map_name, const char *map_mask_name,
struct device_node **target, u32 *id_out)
{
u32 map_mask, masked_id;
int map_len;
const __be32 *map = NULL;
if (!np || !map_name || (!target && !id_out))
return -EINVAL;
map = of_get_property(np, map_name, &map_len);
if (!map) {
if (target)
return -ENODEV;
/* Otherwise, no map implies no translation */
*id_out = id;
return 0;
}
if (!map_len || map_len % (4 * sizeof(*map))) {
pr_err("%pOF: Error: Bad %s length: %d\n", np,
map_name, map_len);
return -EINVAL;
}
/* The default is to select all bits. */
map_mask = 0xffffffff;
/*
* Can be overridden by "{iommu,msi}-map-mask" property.
* If of_property_read_u32() fails, the default is used.
*/
if (map_mask_name)
of_property_read_u32(np, map_mask_name, &map_mask);
masked_id = map_mask & id;
for ( ; map_len > 0; map_len -= 4 * sizeof(*map), map += 4) {
struct device_node *phandle_node;
u32 id_base = be32_to_cpup(map + 0);
u32 phandle = be32_to_cpup(map + 1);
u32 out_base = be32_to_cpup(map + 2);
u32 id_len = be32_to_cpup(map + 3);
if (id_base & ~map_mask) {
pr_err("%pOF: Invalid %s translation - %s-mask (0x%x) ignores id-base (0x%x)\n",
np, map_name, map_name,
map_mask, id_base);
return -EFAULT;
}
if (masked_id < id_base || masked_id >= id_base + id_len)
continue;
phandle_node = of_find_node_by_phandle(phandle);
if (!phandle_node)
return -ENODEV;
if (target) {
if (*target)
of_node_put(phandle_node);
else
*target = phandle_node;
if (*target != phandle_node)
continue;
}
if (id_out)
*id_out = masked_id - id_base + out_base;
pr_debug("%pOF: %s, using mask %08x, id-base: %08x, out-base: %08x, length: %08x, id: %08x -> %08x\n",
np, map_name, map_mask, id_base, out_base,
id_len, id, masked_id - id_base + out_base);
return 0;
}
pr_info("%pOF: no %s translation for id 0x%x on %pOF\n", np, map_name,
id, target && *target ? *target : NULL);
/* Bypasses translation */
if (id_out)
*id_out = id;
return 0;
}
EXPORT_SYMBOL_GPL(of_map_id);
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1991, 1992 Linus Torvalds
*
* This file contains the interface functions for the various time related
* system calls: time, stime, gettimeofday, settimeofday, adjtime
*
* Modification history:
*
* 1993-09-02 Philip Gladstone
* Created file with time related functions from sched/core.c and adjtimex()
* 1993-10-08 Torsten Duwe
* adjtime interface update and CMOS clock write code
* 1995-08-13 Torsten Duwe
* kernel PLL updated to 1994-12-13 specs (rfc-1589)
* 1999-01-16 Ulrich Windl
* Introduced error checking for many cases in adjtimex().
* Updated NTP code according to technical memorandum Jan '96
* "A Kernel Model for Precision Timekeeping" by Dave Mills
* Allow time_constant larger than MAXTC(6) for NTP v4 (MAXTC == 10)
* (Even though the technical memorandum forbids it)
* 2004-07-14 Christoph Lameter
* Added getnstimeofday to allow the posix timer functions to return
* with nanosecond accuracy
*/
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/timex.h>
#include <linux/capability.h>
#include <linux/timekeeper_internal.h>
#include <linux/errno.h>
#include <linux/syscalls.h>
#include <linux/security.h>
#include <linux/fs.h>
#include <linux/math64.h>
#include <linux/ptrace.h>
#include <linux/uaccess.h>
#include <linux/compat.h>
#include <asm/unistd.h>
#include <generated/timeconst.h>
#include "timekeeping.h"
/*
* The timezone where the local system is located. Used as a default by some
* programs who obtain this value by using gettimeofday.
*/
struct timezone sys_tz;
EXPORT_SYMBOL(sys_tz);
#ifdef __ARCH_WANT_SYS_TIME
/*
* sys_time() can be implemented in user-level using
* sys_gettimeofday(). Is this for backwards compatibility? If so,
* why not move it into the appropriate arch directory (for those
* architectures that need it).
*/
SYSCALL_DEFINE1(time, __kernel_old_time_t __user *, tloc)
{
__kernel_old_time_t i = (__kernel_old_time_t)ktime_get_real_seconds();
if (tloc) {
if (put_user(i,tloc))
return -EFAULT;
}
force_successful_syscall_return();
return i;
}
/*
* sys_stime() can be implemented in user-level using
* sys_settimeofday(). Is this for backwards compatibility? If so,
* why not move it into the appropriate arch directory (for those
* architectures that need it).
*/
SYSCALL_DEFINE1(stime, __kernel_old_time_t __user *, tptr)
{
struct timespec64 tv;
int err;
if (get_user(tv.tv_sec, tptr))
return -EFAULT;
tv.tv_nsec = 0;
err = security_settime64(&tv, NULL);
if (err)
return err;
do_settimeofday64(&tv);
return 0;
}
#endif /* __ARCH_WANT_SYS_TIME */
#ifdef CONFIG_COMPAT_32BIT_TIME
#ifdef __ARCH_WANT_SYS_TIME32
/* old_time32_t is a 32 bit "long" and needs to get converted. */
SYSCALL_DEFINE1(time32, old_time32_t __user *, tloc)
{
old_time32_t i;
i = (old_time32_t)ktime_get_real_seconds();
if (tloc) {
if (put_user(i,tloc))
return -EFAULT;
}
force_successful_syscall_return();
return i;
}
SYSCALL_DEFINE1(stime32, old_time32_t __user *, tptr)
{
struct timespec64 tv;
int err;
if (get_user(tv.tv_sec, tptr))
return -EFAULT;
tv.tv_nsec = 0;
err = security_settime64(&tv, NULL);
if (err)
return err;
do_settimeofday64(&tv);
return 0;
}
#endif /* __ARCH_WANT_SYS_TIME32 */
#endif
SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv,
struct timezone __user *, tz)
{
if (likely(tv != NULL)) {
struct timespec64 ts;
ktime_get_real_ts64(&ts);
if (put_user(ts.tv_sec, &tv->tv_sec) ||
put_user(ts.tv_nsec / 1000, &tv->tv_usec))
return -EFAULT;
}
if (unlikely(tz != NULL)) {
if (copy_to_user(tz, &sys_tz, sizeof(sys_tz)))
return -EFAULT;
}
return 0;
}
/*
* In case for some reason the CMOS clock has not already been running
* in UTC, but in some local time: The first time we set the timezone,
* we will warp the clock so that it is ticking UTC time instead of
* local time. Presumably, if someone is setting the timezone then we
* are running in an environment where the programs understand about
* timezones. This should be done at boot time in the /etc/rc script,
* as soon as possible, so that the clock can be set right. Otherwise,
* various programs will get confused when the clock gets warped.
*/
int do_sys_settimeofday64(const struct timespec64 *tv, const struct timezone *tz)
{
static int firsttime = 1;
int error = 0;
if (tv && !timespec64_valid_settod(tv))
return -EINVAL;
error = security_settime64(tv, tz);
if (error)
return error;
if (tz) {
/* Verify we're within the +-15 hrs range */
if (tz->tz_minuteswest > 15*60 || tz->tz_minuteswest < -15*60)
return -EINVAL;
sys_tz = *tz;
update_vsyscall_tz();
if (firsttime) {
firsttime = 0;
if (!tv)
timekeeping_warp_clock();
}
}
if (tv)
return do_settimeofday64(tv);
return 0;
}
SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv,
struct timezone __user *, tz)
{
struct timespec64 new_ts;
struct timezone new_tz;
if (tv) {
if (get_user(new_ts.tv_sec, &tv->tv_sec) ||
get_user(new_ts.tv_nsec, &tv->tv_usec))
return -EFAULT;
if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0)
return -EINVAL;
new_ts.tv_nsec *= NSEC_PER_USEC;
}
if (tz) {
if (copy_from_user(&new_tz, tz, sizeof(*tz)))
return -EFAULT;
}
return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv,
struct timezone __user *, tz)
{
if (tv) {
struct timespec64 ts;
ktime_get_real_ts64(&ts);
if (put_user(ts.tv_sec, &tv->tv_sec) ||
put_user(ts.tv_nsec / 1000, &tv->tv_usec))
return -EFAULT;
}
if (tz) {
if (copy_to_user(tz, &sys_tz, sizeof(sys_tz)))
return -EFAULT;
}
return 0;
}
COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
struct timezone __user *, tz)
{
struct timespec64 new_ts;
struct timezone new_tz;
if (tv) {
if (get_user(new_ts.tv_sec, &tv->tv_sec) ||
get_user(new_ts.tv_nsec, &tv->tv_usec))
return -EFAULT;
if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0)
return -EINVAL;
new_ts.tv_nsec *= NSEC_PER_USEC;
}
if (tz) {
if (copy_from_user(&new_tz, tz, sizeof(*tz)))
return -EFAULT;
}
return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL);
}
#endif
#ifdef CONFIG_64BIT
SYSCALL_DEFINE1(adjtimex, struct __kernel_timex __user *, txc_p)
{
struct __kernel_timex txc; /* Local copy of parameter */
int ret;
/* Copy the user data space into the kernel copy
* structure. But bear in mind that the structures
* may change
*/
if (copy_from_user(&txc, txc_p, sizeof(struct __kernel_timex)))
return -EFAULT;
ret = do_adjtimex(&txc);
return copy_to_user(txc_p, &txc, sizeof(struct __kernel_timex)) ? -EFAULT : ret;
}
#endif
#ifdef CONFIG_COMPAT_32BIT_TIME
int get_old_timex32(struct __kernel_timex *txc, const struct old_timex32 __user *utp)
{
struct old_timex32 tx32;
memset(txc, 0, sizeof(struct __kernel_timex));
if (copy_from_user(&tx32, utp, sizeof(struct old_timex32)))
return -EFAULT;
txc->modes = tx32.modes;
txc->offset = tx32.offset;
txc->freq = tx32.freq;
txc->maxerror = tx32.maxerror;
txc->esterror = tx32.esterror;
txc->status = tx32.status;
txc->constant = tx32.constant;
txc->precision = tx32.precision;
txc->tolerance = tx32.tolerance;
txc->time.tv_sec = tx32.time.tv_sec;
txc->time.tv_usec = tx32.time.tv_usec;
txc->tick = tx32.tick;
txc->ppsfreq = tx32.ppsfreq;
txc->jitter = tx32.jitter;
txc->shift = tx32.shift;
txc->stabil = tx32.stabil;
txc->jitcnt = tx32.jitcnt;
txc->calcnt = tx32.calcnt;
txc->errcnt = tx32.errcnt;
txc->stbcnt = tx32.stbcnt;
return 0;
}
int put_old_timex32(struct old_timex32 __user *utp, const struct __kernel_timex *txc)
{
struct old_timex32 tx32;
memset(&tx32, 0, sizeof(struct old_timex32));
tx32.modes = txc->modes;
tx32.offset = txc->offset;
tx32.freq = txc->freq;
tx32.maxerror = txc->maxerror;
tx32.esterror = txc->esterror;
tx32.status = txc->status;
tx32.constant = txc->constant;
tx32.precision = txc->precision;
tx32.tolerance = txc->tolerance;
tx32.time.tv_sec = txc->time.tv_sec;
tx32.time.tv_usec = txc->time.tv_usec;
tx32.tick = txc->tick;
tx32.ppsfreq = txc->ppsfreq;
tx32.jitter = txc->jitter;
tx32.shift = txc->shift;
tx32.stabil = txc->stabil;
tx32.jitcnt = txc->jitcnt;
tx32.calcnt = txc->calcnt;
tx32.errcnt = txc->errcnt;
tx32.stbcnt = txc->stbcnt;
tx32.tai = txc->tai;
if (copy_to_user(utp, &tx32, sizeof(struct old_timex32)))
return -EFAULT;
return 0;
}
SYSCALL_DEFINE1(adjtimex_time32, struct old_timex32 __user *, utp)
{
struct __kernel_timex txc;
int err, ret;
err = get_old_timex32(&txc, utp);
if (err)
return err;
ret = do_adjtimex(&txc);
err = put_old_timex32(utp, &txc);
if (err)
return err;
return ret;
}
#endif
/**
* jiffies_to_msecs - Convert jiffies to milliseconds
* @j: jiffies value
*
* Avoid unnecessary multiplications/divisions in the
* two most common HZ cases.
*
* Return: milliseconds value
*/
unsigned int jiffies_to_msecs(const unsigned long j)
{
#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ)
return (MSEC_PER_SEC / HZ) * j;
#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC)
return (j + (HZ / MSEC_PER_SEC) - 1)/(HZ / MSEC_PER_SEC);
#else
# if BITS_PER_LONG == 32
return (HZ_TO_MSEC_MUL32 * j + (1ULL << HZ_TO_MSEC_SHR32) - 1) >>
HZ_TO_MSEC_SHR32;
# else
return DIV_ROUND_UP(j * HZ_TO_MSEC_NUM, HZ_TO_MSEC_DEN);
# endif
#endif
}
EXPORT_SYMBOL(jiffies_to_msecs);
/**
* jiffies_to_usecs - Convert jiffies to microseconds
* @j: jiffies value
*
* Return: microseconds value
*/
unsigned int jiffies_to_usecs(const unsigned long j)
{
/*
* Hz usually doesn't go much further MSEC_PER_SEC.
* jiffies_to_usecs() and usecs_to_jiffies() depend on that.
*/
BUILD_BUG_ON(HZ > USEC_PER_SEC);
#if !(USEC_PER_SEC % HZ)
return (USEC_PER_SEC / HZ) * j;
#else
# if BITS_PER_LONG == 32
return (HZ_TO_USEC_MUL32 * j) >> HZ_TO_USEC_SHR32;
# else
return (j * HZ_TO_USEC_NUM) / HZ_TO_USEC_DEN;
# endif
#endif
}
EXPORT_SYMBOL(jiffies_to_usecs);
/**
* mktime64 - Converts date to seconds.
* @year0: year to convert
* @mon0: month to convert
* @day: day to convert
* @hour: hour to convert
* @min: minute to convert
* @sec: second to convert
*
* Converts Gregorian date to seconds since 1970-01-01 00:00:00.
* Assumes input in normal date format, i.e. 1980-12-31 23:59:59
* => year=1980, mon=12, day=31, hour=23, min=59, sec=59.
*
* [For the Julian calendar (which was used in Russia before 1917,
* Britain & colonies before 1752, anywhere else before 1582,
* and is still in use by some communities) leave out the
* -year/100+year/400 terms, and add 10.]
*
* This algorithm was first published by Gauss (I think).
*
* A leap second can be indicated by calling this function with sec as
* 60 (allowable under ISO 8601). The leap second is treated the same
* as the following second since they don't exist in UNIX time.
*
* An encoding of midnight at the end of the day as 24:00:00 - ie. midnight
* tomorrow - (allowable under ISO 8601) is supported.
*
* Return: seconds since the epoch time for the given input date
*/
time64_t mktime64(const unsigned int year0, const unsigned int mon0,
const unsigned int day, const unsigned int hour,
const unsigned int min, const unsigned int sec)
{
unsigned int mon = mon0, year = year0;
/* 1..12 -> 11,12,1..10 */
if (0 >= (int) (mon -= 2)) {
mon += 12; /* Puts Feb last since it has leap day */
year -= 1;
}
return ((((time64_t)
(year/4 - year/100 + year/400 + 367*mon/12 + day) +
year*365 - 719499
)*24 + hour /* now have hours - midnight tomorrow handled here */
)*60 + min /* now have minutes */
)*60 + sec; /* finally seconds */
}
EXPORT_SYMBOL(mktime64);
struct __kernel_old_timeval ns_to_kernel_old_timeval(s64 nsec)
{
struct timespec64 ts = ns_to_timespec64(nsec);
struct __kernel_old_timeval tv;
tv.tv_sec = ts.tv_sec;
tv.tv_usec = (suseconds_t)ts.tv_nsec / 1000;
return tv;
}
EXPORT_SYMBOL(ns_to_kernel_old_timeval);
/**
* set_normalized_timespec64 - set timespec sec and nsec parts and normalize
*
* @ts: pointer to timespec variable to be set
* @sec: seconds to set
* @nsec: nanoseconds to set
*
* Set seconds and nanoseconds field of a timespec variable and
* normalize to the timespec storage format
*
* Note: The tv_nsec part is always in the range of 0 <= tv_nsec < NSEC_PER_SEC.
* For negative values only the tv_sec field is negative !
*/
void set_normalized_timespec64(struct timespec64 *ts, time64_t sec, s64 nsec)
{
while (nsec >= NSEC_PER_SEC) {
/*
* The following asm() prevents the compiler from
* optimising this loop into a modulo operation. See
* also __iter_div_u64_rem() in include/linux/time.h
*/
asm("" : "+rm"(nsec));
nsec -= NSEC_PER_SEC;
++sec;
}
while (nsec < 0) {
asm("" : "+rm"(nsec));
nsec += NSEC_PER_SEC;
--sec;
}
ts->tv_sec = sec;
ts->tv_nsec = nsec;
}
EXPORT_SYMBOL(set_normalized_timespec64);
/**
* ns_to_timespec64 - Convert nanoseconds to timespec64
* @nsec: the nanoseconds value to be converted
*
* Return: the timespec64 representation of the nsec parameter.
*/
struct timespec64 ns_to_timespec64(s64 nsec)
{
struct timespec64 ts = { 0, 0 };
s32 rem;
if (likely(nsec > 0)) { ts.tv_sec = div_u64_rem(nsec, NSEC_PER_SEC, &rem);
ts.tv_nsec = rem;
} else if (nsec < 0) {
/*
* With negative times, tv_sec points to the earlier
* second, and tv_nsec counts the nanoseconds since
* then, so tv_nsec is always a positive number.
*/
ts.tv_sec = -div_u64_rem(-nsec - 1, NSEC_PER_SEC, &rem) - 1;
ts.tv_nsec = NSEC_PER_SEC - rem - 1;
}
return ts;
}
EXPORT_SYMBOL(ns_to_timespec64);
/**
* __msecs_to_jiffies: - convert milliseconds to jiffies
* @m: time in milliseconds
*
* conversion is done as follows:
*
* - negative values mean 'infinite timeout' (MAX_JIFFY_OFFSET)
*
* - 'too large' values [that would result in larger than
* MAX_JIFFY_OFFSET values] mean 'infinite timeout' too.
*
* - all other values are converted to jiffies by either multiplying
* the input value by a factor or dividing it with a factor and
* handling any 32-bit overflows.
* for the details see __msecs_to_jiffies()
*
* __msecs_to_jiffies() checks for the passed in value being a constant
* via __builtin_constant_p() allowing gcc to eliminate most of the
* code, __msecs_to_jiffies() is called if the value passed does not
* allow constant folding and the actual conversion must be done at
* runtime.
* The _msecs_to_jiffies helpers are the HZ dependent conversion
* routines found in include/linux/jiffies.h
*
* Return: jiffies value
*/
unsigned long __msecs_to_jiffies(const unsigned int m)
{
/*
* Negative value, means infinite timeout:
*/
if ((int)m < 0)
return MAX_JIFFY_OFFSET;
return _msecs_to_jiffies(m);
}
EXPORT_SYMBOL(__msecs_to_jiffies);
/**
* __usecs_to_jiffies: - convert microseconds to jiffies
* @u: time in milliseconds
*
* Return: jiffies value
*/
unsigned long __usecs_to_jiffies(const unsigned int u)
{
if (u > jiffies_to_usecs(MAX_JIFFY_OFFSET))
return MAX_JIFFY_OFFSET;
return _usecs_to_jiffies(u);
}
EXPORT_SYMBOL(__usecs_to_jiffies);
/**
* timespec64_to_jiffies - convert a timespec64 value to jiffies
* @value: pointer to &struct timespec64
*
* The TICK_NSEC - 1 rounds up the value to the next resolution. Note
* that a remainder subtract here would not do the right thing as the
* resolution values don't fall on second boundaries. I.e. the line:
* nsec -= nsec % TICK_NSEC; is NOT a correct resolution rounding.
* Note that due to the small error in the multiplier here, this
* rounding is incorrect for sufficiently large values of tv_nsec, but
* well formed timespecs should have tv_nsec < NSEC_PER_SEC, so we're
* OK.
*
* Rather, we just shift the bits off the right.
*
* The >> (NSEC_JIFFIE_SC - SEC_JIFFIE_SC) converts the scaled nsec
* value to a scaled second value.
*
* Return: jiffies value
*/
unsigned long
timespec64_to_jiffies(const struct timespec64 *value)
{
u64 sec = value->tv_sec;
long nsec = value->tv_nsec + TICK_NSEC - 1;
if (sec >= MAX_SEC_IN_JIFFIES){
sec = MAX_SEC_IN_JIFFIES;
nsec = 0;
}
return ((sec * SEC_CONVERSION) +
(((u64)nsec * NSEC_CONVERSION) >>
(NSEC_JIFFIE_SC - SEC_JIFFIE_SC))) >> SEC_JIFFIE_SC;
}
EXPORT_SYMBOL(timespec64_to_jiffies);
/**
* jiffies_to_timespec64 - convert jiffies value to &struct timespec64
* @jiffies: jiffies value
* @value: pointer to &struct timespec64
*/
void
jiffies_to_timespec64(const unsigned long jiffies, struct timespec64 *value)
{
/*
* Convert jiffies to nanoseconds and separate with
* one divide.
*/
u32 rem;
value->tv_sec = div_u64_rem((u64)jiffies * TICK_NSEC,
NSEC_PER_SEC, &rem);
value->tv_nsec = rem;
}
EXPORT_SYMBOL(jiffies_to_timespec64);
/*
* Convert jiffies/jiffies_64 to clock_t and back.
*/
/**
* jiffies_to_clock_t - Convert jiffies to clock_t
* @x: jiffies value
*
* Return: jiffies converted to clock_t (CLOCKS_PER_SEC)
*/
clock_t jiffies_to_clock_t(unsigned long x)
{
#if (TICK_NSEC % (NSEC_PER_SEC / USER_HZ)) == 0
# if HZ < USER_HZ
return x * (USER_HZ / HZ);
# else
return x / (HZ / USER_HZ);
# endif
#else
return div_u64((u64)x * TICK_NSEC, NSEC_PER_SEC / USER_HZ);
#endif
}
EXPORT_SYMBOL(jiffies_to_clock_t);
/**
* clock_t_to_jiffies - Convert clock_t to jiffies
* @x: clock_t value
*
* Return: clock_t value converted to jiffies
*/
unsigned long clock_t_to_jiffies(unsigned long x)
{
#if (HZ % USER_HZ)==0
if (x >= ~0UL / (HZ / USER_HZ))
return ~0UL;
return x * (HZ / USER_HZ);
#else
/* Don't worry about loss of precision here .. */
if (x >= ~0UL / HZ * USER_HZ)
return ~0UL;
/* .. but do try to contain it here */
return div_u64((u64)x * HZ, USER_HZ);
#endif
}
EXPORT_SYMBOL(clock_t_to_jiffies);
/**
* jiffies_64_to_clock_t - Convert jiffies_64 to clock_t
* @x: jiffies_64 value
*
* Return: jiffies_64 value converted to 64-bit "clock_t" (CLOCKS_PER_SEC)
*/
u64 jiffies_64_to_clock_t(u64 x)
{
#if (TICK_NSEC % (NSEC_PER_SEC / USER_HZ)) == 0
# if HZ < USER_HZ
x = div_u64(x * USER_HZ, HZ);
# elif HZ > USER_HZ
x = div_u64(x, HZ / USER_HZ);
# else
/* Nothing to do */
# endif
#else
/*
* There are better ways that don't overflow early,
* but even this doesn't overflow in hundreds of years
* in 64 bits, so..
*/
x = div_u64(x * TICK_NSEC, (NSEC_PER_SEC / USER_HZ));
#endif
return x;
}
EXPORT_SYMBOL(jiffies_64_to_clock_t);
/**
* nsec_to_clock_t - Convert nsec value to clock_t
* @x: nsec value
*
* Return: nsec value converted to 64-bit "clock_t" (CLOCKS_PER_SEC)
*/
u64 nsec_to_clock_t(u64 x)
{
#if (NSEC_PER_SEC % USER_HZ) == 0
return div_u64(x, NSEC_PER_SEC / USER_HZ);
#elif (USER_HZ % 512) == 0
return div_u64(x * USER_HZ / 512, NSEC_PER_SEC / 512);
#else
/*
* max relative error 5.7e-8 (1.8s per year) for USER_HZ <= 1024,
* overflow after 64.99 years.
* exact for HZ=60, 72, 90, 120, 144, 180, 300, 600, 900, ...
*/
return div_u64(x * 9, (9ull * NSEC_PER_SEC + (USER_HZ / 2)) / USER_HZ);
#endif
}
/**
* jiffies64_to_nsecs - Convert jiffies64 to nanoseconds
* @j: jiffies64 value
*
* Return: nanoseconds value
*/
u64 jiffies64_to_nsecs(u64 j)
{
#if !(NSEC_PER_SEC % HZ)
return (NSEC_PER_SEC / HZ) * j;
# else
return div_u64(j * HZ_TO_NSEC_NUM, HZ_TO_NSEC_DEN);
#endif
}
EXPORT_SYMBOL(jiffies64_to_nsecs);
/**
* jiffies64_to_msecs - Convert jiffies64 to milliseconds
* @j: jiffies64 value
*
* Return: milliseconds value
*/
u64 jiffies64_to_msecs(const u64 j)
{
#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ)
return (MSEC_PER_SEC / HZ) * j;
#else
return div_u64(j * HZ_TO_MSEC_NUM, HZ_TO_MSEC_DEN);
#endif
}
EXPORT_SYMBOL(jiffies64_to_msecs);
/**
* nsecs_to_jiffies64 - Convert nsecs in u64 to jiffies64
*
* @n: nsecs in u64
*
* Unlike {m,u}secs_to_jiffies, type of input is not unsigned int but u64.
* And this doesn't return MAX_JIFFY_OFFSET since this function is designed
* for scheduler, not for use in device drivers to calculate timeout value.
*
* note:
* NSEC_PER_SEC = 10^9 = (5^9 * 2^9) = (1953125 * 512)
* ULLONG_MAX ns = 18446744073.709551615 secs = about 584 years
*
* Return: nsecs converted to jiffies64 value
*/
u64 nsecs_to_jiffies64(u64 n)
{
#if (NSEC_PER_SEC % HZ) == 0
/* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */
return div_u64(n, NSEC_PER_SEC / HZ);
#elif (HZ % 512) == 0
/* overflow after 292 years if HZ = 1024 */
return div_u64(n * HZ / 512, NSEC_PER_SEC / 512);
#else
/*
* Generic case - optimized for cases where HZ is a multiple of 3.
* overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc.
*/
return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ);
#endif
}
EXPORT_SYMBOL(nsecs_to_jiffies64);
/**
* nsecs_to_jiffies - Convert nsecs in u64 to jiffies
*
* @n: nsecs in u64
*
* Unlike {m,u}secs_to_jiffies, type of input is not unsigned int but u64.
* And this doesn't return MAX_JIFFY_OFFSET since this function is designed
* for scheduler, not for use in device drivers to calculate timeout value.
*
* note:
* NSEC_PER_SEC = 10^9 = (5^9 * 2^9) = (1953125 * 512)
* ULLONG_MAX ns = 18446744073.709551615 secs = about 584 years
*
* Return: nsecs converted to jiffies value
*/
unsigned long nsecs_to_jiffies(u64 n)
{
return (unsigned long)nsecs_to_jiffies64(n);
}
EXPORT_SYMBOL_GPL(nsecs_to_jiffies);
/**
* timespec64_add_safe - Add two timespec64 values and do a safety check
* for overflow.
* @lhs: first (left) timespec64 to add
* @rhs: second (right) timespec64 to add
*
* It's assumed that both values are valid (>= 0).
* And, each timespec64 is in normalized form.
*
* Return: sum of @lhs + @rhs
*/
struct timespec64 timespec64_add_safe(const struct timespec64 lhs,
const struct timespec64 rhs)
{
struct timespec64 res;
set_normalized_timespec64(&res, (timeu64_t) lhs.tv_sec + rhs.tv_sec,
lhs.tv_nsec + rhs.tv_nsec);
if (unlikely(res.tv_sec < lhs.tv_sec || res.tv_sec < rhs.tv_sec)) {
res.tv_sec = TIME64_MAX;
res.tv_nsec = 0;
}
return res;
}
/**
* get_timespec64 - get user's time value into kernel space
* @ts: destination &struct timespec64
* @uts: user's time value as &struct __kernel_timespec
*
* Handles compat or 32-bit modes.
*
* Return: %0 on success or negative errno on error
*/
int get_timespec64(struct timespec64 *ts,
const struct __kernel_timespec __user *uts)
{
struct __kernel_timespec kts;
int ret;
ret = copy_from_user(&kts, uts, sizeof(kts));
if (ret)
return -EFAULT;
ts->tv_sec = kts.tv_sec;
/* Zero out the padding in compat mode */
if (in_compat_syscall()) kts.tv_nsec &= 0xFFFFFFFFUL;
/* In 32-bit mode, this drops the padding */
ts->tv_nsec = kts.tv_nsec; return 0;
}
EXPORT_SYMBOL_GPL(get_timespec64);
/**
* put_timespec64 - convert timespec64 value to __kernel_timespec format and
* copy the latter to userspace
* @ts: input &struct timespec64
* @uts: user's &struct __kernel_timespec
*
* Return: %0 on success or negative errno on error
*/
int put_timespec64(const struct timespec64 *ts,
struct __kernel_timespec __user *uts)
{
struct __kernel_timespec kts = {
.tv_sec = ts->tv_sec,
.tv_nsec = ts->tv_nsec
};
return copy_to_user(uts, &kts, sizeof(kts)) ? -EFAULT : 0;
}
EXPORT_SYMBOL_GPL(put_timespec64);
static int __get_old_timespec32(struct timespec64 *ts64,
const struct old_timespec32 __user *cts)
{
struct old_timespec32 ts;
int ret;
ret = copy_from_user(&ts, cts, sizeof(ts));
if (ret)
return -EFAULT;
ts64->tv_sec = ts.tv_sec;
ts64->tv_nsec = ts.tv_nsec;
return 0;
}
static int __put_old_timespec32(const struct timespec64 *ts64,
struct old_timespec32 __user *cts)
{
struct old_timespec32 ts = {
.tv_sec = ts64->tv_sec,
.tv_nsec = ts64->tv_nsec
};
return copy_to_user(cts, &ts, sizeof(ts)) ? -EFAULT : 0;
}
/**
* get_old_timespec32 - get user's old-format time value into kernel space
* @ts: destination &struct timespec64
* @uts: user's old-format time value (&struct old_timespec32)
*
* Handles X86_X32_ABI compatibility conversion.
*
* Return: %0 on success or negative errno on error
*/
int get_old_timespec32(struct timespec64 *ts, const void __user *uts)
{
if (COMPAT_USE_64BIT_TIME)
return copy_from_user(ts, uts, sizeof(*ts)) ? -EFAULT : 0;
else
return __get_old_timespec32(ts, uts);
}
EXPORT_SYMBOL_GPL(get_old_timespec32);
/**
* put_old_timespec32 - convert timespec64 value to &struct old_timespec32 and
* copy the latter to userspace
* @ts: input &struct timespec64
* @uts: user's &struct old_timespec32
*
* Handles X86_X32_ABI compatibility conversion.
*
* Return: %0 on success or negative errno on error
*/
int put_old_timespec32(const struct timespec64 *ts, void __user *uts)
{
if (COMPAT_USE_64BIT_TIME)
return copy_to_user(uts, ts, sizeof(*ts)) ? -EFAULT : 0;
else
return __put_old_timespec32(ts, uts);
}
EXPORT_SYMBOL_GPL(put_old_timespec32);
/**
* get_itimerspec64 - get user's &struct __kernel_itimerspec into kernel space
* @it: destination &struct itimerspec64
* @uit: user's &struct __kernel_itimerspec
*
* Return: %0 on success or negative errno on error
*/
int get_itimerspec64(struct itimerspec64 *it,
const struct __kernel_itimerspec __user *uit)
{
int ret;
ret = get_timespec64(&it->it_interval, &uit->it_interval);
if (ret)
return ret;
ret = get_timespec64(&it->it_value, &uit->it_value);
return ret;
}
EXPORT_SYMBOL_GPL(get_itimerspec64);
/**
* put_itimerspec64 - convert &struct itimerspec64 to __kernel_itimerspec format
* and copy the latter to userspace
* @it: input &struct itimerspec64
* @uit: user's &struct __kernel_itimerspec
*
* Return: %0 on success or negative errno on error
*/
int put_itimerspec64(const struct itimerspec64 *it,
struct __kernel_itimerspec __user *uit)
{
int ret;
ret = put_timespec64(&it->it_interval, &uit->it_interval);
if (ret)
return ret;
ret = put_timespec64(&it->it_value, &uit->it_value);
return ret;
}
EXPORT_SYMBOL_GPL(put_itimerspec64);
/**
* get_old_itimerspec32 - get user's &struct old_itimerspec32 into kernel space
* @its: destination &struct itimerspec64
* @uits: user's &struct old_itimerspec32
*
* Return: %0 on success or negative errno on error
*/
int get_old_itimerspec32(struct itimerspec64 *its,
const struct old_itimerspec32 __user *uits)
{
if (__get_old_timespec32(&its->it_interval, &uits->it_interval) ||
__get_old_timespec32(&its->it_value, &uits->it_value))
return -EFAULT;
return 0;
}
EXPORT_SYMBOL_GPL(get_old_itimerspec32);
/**
* put_old_itimerspec32 - convert &struct itimerspec64 to &struct
* old_itimerspec32 and copy the latter to userspace
* @its: input &struct itimerspec64
* @uits: user's &struct old_itimerspec32
*
* Return: %0 on success or negative errno on error
*/
int put_old_itimerspec32(const struct itimerspec64 *its,
struct old_itimerspec32 __user *uits)
{
if (__put_old_timespec32(&its->it_interval, &uits->it_interval) ||
__put_old_timespec32(&its->it_value, &uits->it_value))
return -EFAULT;
return 0;
}
EXPORT_SYMBOL_GPL(put_old_itimerspec32);
// SPDX-License-Identifier: GPL-2.0
/*
* Detect hard and soft lockups on a system
*
* started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
*
* Note: Most of this code is borrowed heavily from the original softlockup
* detector, so thanks to Ingo for the initial implementation.
* Some chunks also taken from the old x86-specific nmi watchdog code, thanks
* to those contributors as well.
*/
#define pr_fmt(fmt) "watchdog: " fmt
#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/nmi.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/sysctl.h>
#include <linux/tick.h>
#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>
#include <linux/stop_machine.h>
#include <asm/irq_regs.h>
#include <linux/kvm_para.h>
static DEFINE_MUTEX(watchdog_mutex);
#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64)
# define WATCHDOG_HARDLOCKUP_DEFAULT 1
#else
# define WATCHDOG_HARDLOCKUP_DEFAULT 0
#endif
unsigned long __read_mostly watchdog_enabled;
int __read_mostly watchdog_user_enabled = 1;
static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT;
static int __read_mostly watchdog_softlockup_user_enabled = 1;
int __read_mostly watchdog_thresh = 10;
static int __read_mostly watchdog_hardlockup_available;
struct cpumask watchdog_cpumask __read_mostly;
unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
#ifdef CONFIG_HARDLOCKUP_DETECTOR
# ifdef CONFIG_SMP
int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
# endif /* CONFIG_SMP */
/*
* Should we panic when a soft-lockup or hard-lockup occurs:
*/
unsigned int __read_mostly hardlockup_panic =
IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
/*
* We may not want to enable hard lockup detection by default in all cases,
* for example when running the kernel as a guest on a hypervisor. In these
* cases this function can be called to disable hard lockup detection. This
* function should only be executed once by the boot processor before the
* kernel command line parameters are parsed, because otherwise it is not
* possible to override this in hardlockup_panic_setup().
*/
void __init hardlockup_detector_disable(void)
{
watchdog_hardlockup_user_enabled = 0;
}
static int __init hardlockup_panic_setup(char *str)
{
if (!strncmp(str, "panic", 5))
hardlockup_panic = 1;
else if (!strncmp(str, "nopanic", 7))
hardlockup_panic = 0;
else if (!strncmp(str, "0", 1))
watchdog_hardlockup_user_enabled = 0;
else if (!strncmp(str, "1", 1))
watchdog_hardlockup_user_enabled = 1;
return 1;
}
__setup("nmi_watchdog=", hardlockup_panic_setup);
#endif /* CONFIG_HARDLOCKUP_DETECTOR */
#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)
static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);
static unsigned long hard_lockup_nmi_warn;
notrace void arch_touch_nmi_watchdog(void)
{
/*
* Using __raw here because some code paths have
* preemption enabled. If preemption is enabled
* then interrupts should be enabled too, in which
* case we shouldn't have to worry about the watchdog
* going off.
*/
raw_cpu_write(watchdog_hardlockup_touched, true);
}
EXPORT_SYMBOL(arch_touch_nmi_watchdog);
void watchdog_hardlockup_touch_cpu(unsigned int cpu)
{
per_cpu(watchdog_hardlockup_touched, cpu) = true;
}
static bool is_hardlockup(unsigned int cpu)
{
int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));
if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
return true;
/*
* NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE
* for hrtimer_interrupts_saved. hrtimer_interrupts_saved is
* written/read by a single CPU.
*/
per_cpu(hrtimer_interrupts_saved, cpu) = hrint;
return false;
}
static void watchdog_hardlockup_kick(void)
{
int new_interrupts;
new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts));
watchdog_buddy_check_hardlockup(new_interrupts);
}
void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
{
if (per_cpu(watchdog_hardlockup_touched, cpu)) {
per_cpu(watchdog_hardlockup_touched, cpu) = false;
return;
}
/*
* Check for a hardlockup by making sure the CPU's timer
* interrupt is incrementing. The timer interrupt should have
* fired multiple times before we overflow'd. If it hasn't
* then this is a good indication the cpu is stuck
*/
if (is_hardlockup(cpu)) {
unsigned int this_cpu = smp_processor_id();
unsigned long flags;
/* Only print hardlockups once. */
if (per_cpu(watchdog_hardlockup_warned, cpu))
return;
/*
* Prevent multiple hard-lockup reports if one cpu is already
* engaged in dumping all cpu back traces.
*/
if (sysctl_hardlockup_all_cpu_backtrace) {
if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
return;
}
/*
* NOTE: we call printk_cpu_sync_get_irqsave() after printing
* the lockup message. While it would be nice to serialize
* that printout, we really want to make sure that if some
* other CPU somehow locked up while holding the lock associated
* with printk_cpu_sync_get_irqsave() that we can still at least
* get the message about the lockup out.
*/
pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu);
printk_cpu_sync_get_irqsave(flags);
print_modules();
print_irqtrace_events(current);
if (cpu == this_cpu) {
if (regs)
show_regs(regs);
else
dump_stack();
printk_cpu_sync_put_irqrestore(flags);
} else {
printk_cpu_sync_put_irqrestore(flags);
trigger_single_cpu_backtrace(cpu);
}
if (sysctl_hardlockup_all_cpu_backtrace) {
trigger_allbutcpu_cpu_backtrace(cpu);
if (!hardlockup_panic)
clear_bit_unlock(0, &hard_lockup_nmi_warn);
}
if (hardlockup_panic)
nmi_panic(regs, "Hard LOCKUP");
per_cpu(watchdog_hardlockup_warned, cpu) = true;
} else {
per_cpu(watchdog_hardlockup_warned, cpu) = false;
}
}
#else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */
static inline void watchdog_hardlockup_kick(void) { }
#endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */
/*
* These functions can be overridden based on the configured hardlockdup detector.
*
* watchdog_hardlockup_enable/disable can be implemented to start and stop when
* softlockup watchdog start and stop. The detector must select the
* SOFTLOCKUP_DETECTOR Kconfig.
*/
void __weak watchdog_hardlockup_enable(unsigned int cpu) { }
void __weak watchdog_hardlockup_disable(unsigned int cpu) { }
/*
* Watchdog-detector specific API.
*
* Return 0 when hardlockup watchdog is available, negative value otherwise.
* Note that the negative value means that a delayed probe might
* succeed later.
*/
int __weak __init watchdog_hardlockup_probe(void)
{
return -ENODEV;
}
/**
* watchdog_hardlockup_stop - Stop the watchdog for reconfiguration
*
* The reconfiguration steps are:
* watchdog_hardlockup_stop();
* update_variables();
* watchdog_hardlockup_start();
*/
void __weak watchdog_hardlockup_stop(void) { }
/**
* watchdog_hardlockup_start - Start the watchdog after reconfiguration
*
* Counterpart to watchdog_hardlockup_stop().
*
* The following variables have been updated in update_variables() and
* contain the currently valid configuration:
* - watchdog_enabled
* - watchdog_thresh
* - watchdog_cpumask
*/
void __weak watchdog_hardlockup_start(void) { }
/**
* lockup_detector_update_enable - Update the sysctl enable bit
*
* Caller needs to make sure that the hard watchdogs are off, so this
* can't race with watchdog_hardlockup_disable().
*/
static void lockup_detector_update_enable(void)
{
watchdog_enabled = 0;
if (!watchdog_user_enabled)
return;
if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)
watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;
if (watchdog_softlockup_user_enabled)
watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;
}
#ifdef CONFIG_SOFTLOCKUP_DETECTOR
/*
* Delay the soflockup report when running a known slow code.
* It does _not_ affect the timestamp of the last successdul reschedule.
*/
#define SOFTLOCKUP_DELAY_REPORT ULONG_MAX
#ifdef CONFIG_SMP
int __read_mostly sysctl_softlockup_all_cpu_backtrace;
#endif
static struct cpumask watchdog_allowed_mask __read_mostly;
/* Global variables, exported for sysctl */
unsigned int __read_mostly softlockup_panic =
IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);
static bool softlockup_initialized __read_mostly;
static u64 __read_mostly sample_period;
/* Timestamp taken after the last successful reschedule. */
static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
/* Timestamp of the last softlockup report. */
static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
static DEFINE_PER_CPU(bool, softlockup_touch_sync);
static unsigned long soft_lockup_nmi_warn;
static int __init softlockup_panic_setup(char *str)
{
softlockup_panic = simple_strtoul(str, NULL, 0);
return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);
static int __init nowatchdog_setup(char *str)
{
watchdog_user_enabled = 0;
return 1;
}
__setup("nowatchdog", nowatchdog_setup);
static int __init nosoftlockup_setup(char *str)
{
watchdog_softlockup_user_enabled = 0;
return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);
static int __init watchdog_thresh_setup(char *str)
{
get_option(&str, &watchdog_thresh);
return 1;
}
__setup("watchdog_thresh=", watchdog_thresh_setup);
static void __lockup_detector_cleanup(void);
/*
* Hard-lockup warnings should be triggered after just a few seconds. Soft-
* lockups can have false positives under extreme conditions. So we generally
* want a higher threshold for soft lockups than for hard lockups. So we couple
* the thresholds with a factor: we make the soft threshold twice the amount of
* time the hard threshold is.
*/
static int get_softlockup_thresh(void)
{
return watchdog_thresh * 2;
}
/*
* Returns seconds, approximately. We don't need nanosecond
* resolution, and we don't need to waste time with a big divide when
* 2^30ns == 1.074s.
*/
static unsigned long get_timestamp(void)
{
return running_clock() >> 30LL; /* 2^30 ~= 10^9 */
}
static void set_sample_period(void)
{
/*
* convert watchdog_thresh from seconds to ns
* the divide by 5 is to give hrtimer several chances (two
* or three with the current relation between the soft
* and hard thresholds) to increment before the
* hardlockup detector generates a warning
*/
sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
watchdog_update_hrtimer_threshold(sample_period);
}
static void update_report_ts(void)
{
__this_cpu_write(watchdog_report_ts, get_timestamp());
}
/* Commands for resetting the watchdog */
static void update_touch_ts(void)
{
__this_cpu_write(watchdog_touch_ts, get_timestamp());
update_report_ts();
}
/**
* touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
*
* Call when the scheduler may have stalled for legitimate reasons
* preventing the watchdog task from executing - e.g. the scheduler
* entering idle state. This should only be used for scheduler events.
* Use touch_softlockup_watchdog() for everything else.
*/
notrace void touch_softlockup_watchdog_sched(void)
{
/*
* Preemption can be enabled. It doesn't matter which CPU's watchdog
* report period gets restarted here, so use the raw_ operation.
*/
raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}
notrace void touch_softlockup_watchdog(void)
{
touch_softlockup_watchdog_sched();
wq_watchdog_touch(raw_smp_processor_id());
}
EXPORT_SYMBOL(touch_softlockup_watchdog);
void touch_all_softlockup_watchdogs(void)
{
int cpu;
/*
* watchdog_mutex cannpt be taken here, as this might be called
* from (soft)interrupt context, so the access to
* watchdog_allowed_cpumask might race with a concurrent update.
*
* The watchdog time stamp can race against a concurrent real
* update as well, the only side effect might be a cycle delay for
* the softlockup check.
*/
for_each_cpu(cpu, &watchdog_allowed_mask) {
per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
wq_watchdog_touch(cpu);
}
}
void touch_softlockup_watchdog_sync(void)
{
__this_cpu_write(softlockup_touch_sync, true);
__this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}
static int is_softlockup(unsigned long touch_ts,
unsigned long period_ts,
unsigned long now)
{
if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) {
/* Warn about unreasonable delays. */
if (time_after(now, period_ts + get_softlockup_thresh()))
return now - touch_ts;
}
return 0;
}
/* watchdog detector functions */
static DEFINE_PER_CPU(struct completion, softlockup_completion);
static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);
/*
* The watchdog feed function - touches the timestamp.
*
* It only runs once every sample_period seconds (4 seconds by
* default) to reset the softlockup timestamp. If this gets delayed
* for more than 2*watchdog_thresh seconds then the debug-printout
* triggers in watchdog_timer_fn().
*/
static int softlockup_fn(void *data)
{
update_touch_ts();
complete(this_cpu_ptr(&softlockup_completion));
return 0;
}
/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
unsigned long touch_ts, period_ts, now;
struct pt_regs *regs = get_irq_regs();
int duration;
int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
unsigned long flags;
if (!watchdog_enabled)
return HRTIMER_NORESTART;
watchdog_hardlockup_kick();
/* kick the softlockup detector */
if (completion_done(this_cpu_ptr(&softlockup_completion))) {
reinit_completion(this_cpu_ptr(&softlockup_completion));
stop_one_cpu_nowait(smp_processor_id(),
softlockup_fn, NULL,
this_cpu_ptr(&softlockup_stop_work));
}
/* .. and repeat */
hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
/*
* Read the current timestamp first. It might become invalid anytime
* when a virtual machine is stopped by the host or when the watchog
* is touched from NMI.
*/
now = get_timestamp();
/*
* If a virtual machine is stopped by the host it can look to
* the watchdog like a soft lockup. This function touches the watchdog.
*/
kvm_check_and_clear_guest_paused();
/*
* The stored timestamp is comparable with @now only when not touched.
* It might get touched anytime from NMI. Make sure that is_softlockup()
* uses the same (valid) value.
*/
period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));
/* Reset the interval when touched by known problematic code. */
if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
/*
* If the time stamp was touched atomically
* make sure the scheduler tick is up to date.
*/
__this_cpu_write(softlockup_touch_sync, false);
sched_clock_tick();
}
update_report_ts();
return HRTIMER_RESTART;
}
/* Check for a softlockup. */
touch_ts = __this_cpu_read(watchdog_touch_ts);
duration = is_softlockup(touch_ts, period_ts, now);
if (unlikely(duration)) {
/*
* Prevent multiple soft-lockup reports if one cpu is already
* engaged in dumping all cpu back traces.
*/
if (softlockup_all_cpu_backtrace) {
if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
return HRTIMER_RESTART;
}
/* Start period for the next softlockup warning. */
update_report_ts();
printk_cpu_sync_get_irqsave(flags);
pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
smp_processor_id(), duration,
current->comm, task_pid_nr(current));
print_modules();
print_irqtrace_events(current);
if (regs)
show_regs(regs);
else
dump_stack();
printk_cpu_sync_put_irqrestore(flags);
if (softlockup_all_cpu_backtrace) {
trigger_allbutcpu_cpu_backtrace(smp_processor_id());
if (!softlockup_panic)
clear_bit_unlock(0, &soft_lockup_nmi_warn);
}
add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
if (softlockup_panic)
panic("softlockup: hung tasks");
}
return HRTIMER_RESTART;
}
static void watchdog_enable(unsigned int cpu)
{
struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
struct completion *done = this_cpu_ptr(&softlockup_completion);
WARN_ON_ONCE(cpu != smp_processor_id());
init_completion(done);
complete(done);
/*
* Start the timer first to prevent the hardlockup watchdog triggering
* before the timer has a chance to fire.
*/
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
hrtimer->function = watchdog_timer_fn;
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
HRTIMER_MODE_REL_PINNED_HARD);
/* Initialize timestamp */
update_touch_ts();
/* Enable the hardlockup detector */
if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)
watchdog_hardlockup_enable(cpu);
}
static void watchdog_disable(unsigned int cpu)
{
struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
WARN_ON_ONCE(cpu != smp_processor_id());
/*
* Disable the hardlockup detector first. That prevents that a large
* delay between disabling the timer and disabling the hardlockup
* detector causes a false positive.
*/
watchdog_hardlockup_disable(cpu);
hrtimer_cancel(hrtimer);
wait_for_completion(this_cpu_ptr(&softlockup_completion));
}
static int softlockup_stop_fn(void *data)
{
watchdog_disable(smp_processor_id());
return 0;
}
static void softlockup_stop_all(void)
{
int cpu;
if (!softlockup_initialized)
return;
for_each_cpu(cpu, &watchdog_allowed_mask)
smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);
cpumask_clear(&watchdog_allowed_mask);
}
static int softlockup_start_fn(void *data)
{
watchdog_enable(smp_processor_id());
return 0;
}
static void softlockup_start_all(void)
{
int cpu;
cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
for_each_cpu(cpu, &watchdog_allowed_mask)
smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
}
int lockup_detector_online_cpu(unsigned int cpu)
{
if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
watchdog_enable(cpu);
return 0;
}
int lockup_detector_offline_cpu(unsigned int cpu)
{
if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
watchdog_disable(cpu);
return 0;
}
static void __lockup_detector_reconfigure(void)
{
cpus_read_lock();
watchdog_hardlockup_stop();
softlockup_stop_all();
set_sample_period();
lockup_detector_update_enable();
if (watchdog_enabled && watchdog_thresh)
softlockup_start_all();
watchdog_hardlockup_start();
cpus_read_unlock();
/*
* Must be called outside the cpus locked section to prevent
* recursive locking in the perf code.
*/
__lockup_detector_cleanup();
}
void lockup_detector_reconfigure(void)
{
mutex_lock(&watchdog_mutex);
__lockup_detector_reconfigure();
mutex_unlock(&watchdog_mutex);
}
/*
* Create the watchdog infrastructure and configure the detector(s).
*/
static __init void lockup_detector_setup(void)
{
/*
* If sysctl is off and watchdog got disabled on the command line,
* nothing to do here.
*/
lockup_detector_update_enable();
if (!IS_ENABLED(CONFIG_SYSCTL) &&
!(watchdog_enabled && watchdog_thresh))
return;
mutex_lock(&watchdog_mutex);
__lockup_detector_reconfigure();
softlockup_initialized = true;
mutex_unlock(&watchdog_mutex);
}
#else /* CONFIG_SOFTLOCKUP_DETECTOR */
static void __lockup_detector_reconfigure(void)
{
cpus_read_lock();
watchdog_hardlockup_stop();
lockup_detector_update_enable();
watchdog_hardlockup_start();
cpus_read_unlock();
}
void lockup_detector_reconfigure(void)
{
__lockup_detector_reconfigure();
}
static inline void lockup_detector_setup(void)
{
__lockup_detector_reconfigure();
}
#endif /* !CONFIG_SOFTLOCKUP_DETECTOR */
static void __lockup_detector_cleanup(void)
{
lockdep_assert_held(&watchdog_mutex);
hardlockup_detector_perf_cleanup();
}
/**
* lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
*
* Caller must not hold the cpu hotplug rwsem.
*/
void lockup_detector_cleanup(void)
{
mutex_lock(&watchdog_mutex);
__lockup_detector_cleanup();
mutex_unlock(&watchdog_mutex);
}
/**
* lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
*
* Special interface for parisc. It prevents lockup detector warnings from
* the default pm_poweroff() function which busy loops forever.
*/
void lockup_detector_soft_poweroff(void)
{
watchdog_enabled = 0;
}
#ifdef CONFIG_SYSCTL
/* Propagate any changes to the watchdog infrastructure */
static void proc_watchdog_update(void)
{
/* Remove impossible cpus to keep sysctl output clean. */
cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
__lockup_detector_reconfigure();
}
/*
* common function for watchdog, nmi_watchdog and soft_watchdog parameter
*
* caller | table->data points to | 'which'
* -------------------|----------------------------------|-------------------------------
* proc_watchdog | watchdog_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED |
* | | WATCHDOG_SOFTOCKUP_ENABLED
* -------------------|----------------------------------|-------------------------------
* proc_nmi_watchdog | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED
* -------------------|----------------------------------|-------------------------------
* proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED
*/
static int proc_watchdog_common(int which, struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int err, old, *param = table->data;
mutex_lock(&watchdog_mutex);
if (!write) {
/*
* On read synchronize the userspace interface. This is a
* racy snapshot.
*/
*param = (watchdog_enabled & which) != 0;
err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
} else {
old = READ_ONCE(*param);
err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (!err && old != READ_ONCE(*param))
proc_watchdog_update();
}
mutex_unlock(&watchdog_mutex);
return err;
}
/*
* /proc/sys/kernel/watchdog
*/
static int proc_watchdog(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED |
WATCHDOG_SOFTOCKUP_ENABLED,
table, write, buffer, lenp, ppos);
}
/*
* /proc/sys/kernel/nmi_watchdog
*/
static int proc_nmi_watchdog(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
if (!watchdog_hardlockup_available && write)
return -ENOTSUPP;
return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED,
table, write, buffer, lenp, ppos);
}
#ifdef CONFIG_SOFTLOCKUP_DETECTOR
/*
* /proc/sys/kernel/soft_watchdog
*/
static int proc_soft_watchdog(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED,
table, write, buffer, lenp, ppos);
}
#endif
/*
* /proc/sys/kernel/watchdog_thresh
*/
static int proc_watchdog_thresh(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int err, old;
mutex_lock(&watchdog_mutex);
old = READ_ONCE(watchdog_thresh);
err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (!err && write && old != READ_ONCE(watchdog_thresh))
proc_watchdog_update();
mutex_unlock(&watchdog_mutex);
return err;
}
/*
* The cpumask is the mask of possible cpus that the watchdog can run
* on, not the mask of cpus it is actually running on. This allows the
* user to specify a mask that will include cpus that have not yet
* been brought online, if desired.
*/
static int proc_watchdog_cpumask(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int err;
mutex_lock(&watchdog_mutex);
err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
if (!err && write)
proc_watchdog_update();
mutex_unlock(&watchdog_mutex);
return err;
}
static const int sixty = 60;
static struct ctl_table watchdog_sysctls[] = {
{
.procname = "watchdog",
.data = &watchdog_user_enabled,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_watchdog,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{
.procname = "watchdog_thresh",
.data = &watchdog_thresh,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_watchdog_thresh,
.extra1 = SYSCTL_ZERO,
.extra2 = (void *)&sixty,
},
{
.procname = "watchdog_cpumask",
.data = &watchdog_cpumask_bits,
.maxlen = NR_CPUS,
.mode = 0644,
.proc_handler = proc_watchdog_cpumask,
},
#ifdef CONFIG_SOFTLOCKUP_DETECTOR
{
.procname = "soft_watchdog",
.data = &watchdog_softlockup_user_enabled,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_soft_watchdog,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{
.procname = "softlockup_panic",
.data = &softlockup_panic,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
#ifdef CONFIG_SMP
{
.procname = "softlockup_all_cpu_backtrace",
.data = &sysctl_softlockup_all_cpu_backtrace,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
#endif /* CONFIG_SMP */
#endif
#ifdef CONFIG_HARDLOCKUP_DETECTOR
{
.procname = "hardlockup_panic",
.data = &hardlockup_panic,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
#ifdef CONFIG_SMP
{
.procname = "hardlockup_all_cpu_backtrace",
.data = &sysctl_hardlockup_all_cpu_backtrace,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
#endif /* CONFIG_SMP */
#endif
{}
};
static struct ctl_table watchdog_hardlockup_sysctl[] = {
{
.procname = "nmi_watchdog",
.data = &watchdog_hardlockup_user_enabled,
.maxlen = sizeof(int),
.mode = 0444,
.proc_handler = proc_nmi_watchdog,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{}
};
static void __init watchdog_sysctl_init(void)
{
register_sysctl_init("kernel", watchdog_sysctls);
if (watchdog_hardlockup_available)
watchdog_hardlockup_sysctl[0].mode = 0644;
register_sysctl_init("kernel", watchdog_hardlockup_sysctl);
}
#else
#define watchdog_sysctl_init() do { } while (0)
#endif /* CONFIG_SYSCTL */
static void __init lockup_detector_delay_init(struct work_struct *work);
static bool allow_lockup_detector_init_retry __initdata;
static struct work_struct detector_work __initdata =
__WORK_INITIALIZER(detector_work, lockup_detector_delay_init);
static void __init lockup_detector_delay_init(struct work_struct *work)
{
int ret;
ret = watchdog_hardlockup_probe();
if (ret) {
pr_info("Delayed init of the lockup detector failed: %d\n", ret);
pr_info("Hard watchdog permanently disabled\n");
return;
}
allow_lockup_detector_init_retry = false;
watchdog_hardlockup_available = true;
lockup_detector_setup();
}
/*
* lockup_detector_retry_init - retry init lockup detector if possible.
*
* Retry hardlockup detector init. It is useful when it requires some
* functionality that has to be initialized later on a particular
* platform.
*/
void __init lockup_detector_retry_init(void)
{
/* Must be called before late init calls */
if (!allow_lockup_detector_init_retry)
return;
schedule_work(&detector_work);
}
/*
* Ensure that optional delayed hardlockup init is proceed before
* the init code and memory is freed.
*/
static int __init lockup_detector_check(void)
{
/* Prevent any later retry. */
allow_lockup_detector_init_retry = false;
/* Make sure no work is pending. */
flush_work(&detector_work);
watchdog_sysctl_init();
return 0;
}
late_initcall_sync(lockup_detector_check);
void __init lockup_detector_init(void)
{
if (tick_nohz_full_enabled())
pr_info("Disabling watchdog on nohz_full cores by default\n");
cpumask_copy(&watchdog_cpumask,
housekeeping_cpumask(HK_TYPE_TIMER));
if (!watchdog_hardlockup_probe())
watchdog_hardlockup_available = true;
else
allow_lockup_detector_init_retry = true;
lockup_detector_setup();
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* Integrity Measurement Architecture
*
* Copyright (C) 2005,2006,2007,2008 IBM Corporation
*
* Authors:
* Reiner Sailer <sailer@watson.ibm.com>
* Serge Hallyn <serue@us.ibm.com>
* Kylene Hall <kylene@us.ibm.com>
* Mimi Zohar <zohar@us.ibm.com>
*
* File: ima_main.c
* implements the IMA hooks: ima_bprm_check, ima_file_mmap,
* and ima_file_check.
*/
#include <linux/module.h>
#include <linux/file.h>
#include <linux/binfmts.h>
#include <linux/kernel_read_file.h>
#include <linux/mount.h>
#include <linux/mman.h>
#include <linux/slab.h>
#include <linux/xattr.h>
#include <linux/ima.h>
#include <linux/fs.h>
#include <linux/iversion.h>
#include "ima.h"
#ifdef CONFIG_IMA_APPRAISE
int ima_appraise = IMA_APPRAISE_ENFORCE;
#else
int ima_appraise;
#endif
int __ro_after_init ima_hash_algo = HASH_ALGO_SHA1;
static int hash_setup_done;
static struct notifier_block ima_lsm_policy_notifier = {
.notifier_call = ima_lsm_policy_change,
};
static int __init hash_setup(char *str)
{
struct ima_template_desc *template_desc = ima_template_desc_current();
int i;
if (hash_setup_done)
return 1;
if (strcmp(template_desc->name, IMA_TEMPLATE_IMA_NAME) == 0) {
if (strncmp(str, "sha1", 4) == 0) {
ima_hash_algo = HASH_ALGO_SHA1;
} else if (strncmp(str, "md5", 3) == 0) {
ima_hash_algo = HASH_ALGO_MD5;
} else {
pr_err("invalid hash algorithm \"%s\" for template \"%s\"",
str, IMA_TEMPLATE_IMA_NAME);
return 1;
}
goto out;
}
i = match_string(hash_algo_name, HASH_ALGO__LAST, str);
if (i < 0) {
pr_err("invalid hash algorithm \"%s\"", str);
return 1;
}
ima_hash_algo = i;
out:
hash_setup_done = 1;
return 1;
}
__setup("ima_hash=", hash_setup);
enum hash_algo ima_get_current_hash_algo(void)
{
return ima_hash_algo;
}
/* Prevent mmap'ing a file execute that is already mmap'ed write */
static int mmap_violation_check(enum ima_hooks func, struct file *file,
char **pathbuf, const char **pathname,
char *filename)
{
struct inode *inode;
int rc = 0;
if ((func == MMAP_CHECK || func == MMAP_CHECK_REQPROT) &&
mapping_writably_mapped(file->f_mapping)) {
rc = -ETXTBSY;
inode = file_inode(file);
if (!*pathbuf) /* ima_rdwr_violation possibly pre-fetched */
*pathname = ima_d_path(&file->f_path, pathbuf,
filename);
integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode, *pathname,
"mmap_file", "mmapped_writers", rc, 0);
}
return rc;
}
/*
* ima_rdwr_violation_check
*
* Only invalidate the PCR for measured files:
* - Opening a file for write when already open for read,
* results in a time of measure, time of use (ToMToU) error.
* - Opening a file for read when already open for write,
* could result in a file measurement error.
*
*/
static void ima_rdwr_violation_check(struct file *file,
struct ima_iint_cache *iint,
int must_measure,
char **pathbuf,
const char **pathname,
char *filename)
{
struct inode *inode = file_inode(file);
fmode_t mode = file->f_mode;
bool send_tomtou = false, send_writers = false;
if (mode & FMODE_WRITE) {
if (atomic_read(&inode->i_readcount) && IS_IMA(inode)) { if (!iint) iint = ima_iint_find(inode);
/* IMA_MEASURE is set from reader side */
if (iint && test_bit(IMA_MUST_MEASURE,
&iint->atomic_flags))
send_tomtou = true;
}
} else {
if (must_measure)
set_bit(IMA_MUST_MEASURE, &iint->atomic_flags); if (inode_is_open_for_write(inode) && must_measure)
send_writers = true;
}
if (!send_tomtou && !send_writers)
return;
*pathname = ima_d_path(&file->f_path, pathbuf, filename);
if (send_tomtou)
ima_add_violation(file, *pathname, iint,
"invalid_pcr", "ToMToU");
if (send_writers)
ima_add_violation(file, *pathname, iint,
"invalid_pcr", "open_writers");
}
static void ima_check_last_writer(struct ima_iint_cache *iint,
struct inode *inode, struct file *file)
{
fmode_t mode = file->f_mode;
bool update;
if (!(mode & FMODE_WRITE))
return;
mutex_lock(&iint->mutex);
if (atomic_read(&inode->i_writecount) == 1) {
struct kstat stat;
update = test_and_clear_bit(IMA_UPDATE_XATTR,
&iint->atomic_flags);
if ((iint->flags & IMA_NEW_FILE) ||
vfs_getattr_nosec(&file->f_path, &stat,
STATX_CHANGE_COOKIE,
AT_STATX_SYNC_AS_STAT) ||
!(stat.result_mask & STATX_CHANGE_COOKIE) ||
stat.change_cookie != iint->version) {
iint->flags &= ~(IMA_DONE_MASK | IMA_NEW_FILE);
iint->measured_pcrs = 0;
if (update)
ima_update_xattr(iint, file);
}
}
mutex_unlock(&iint->mutex);
}
/**
* ima_file_free - called on __fput()
* @file: pointer to file structure being freed
*
* Flag files that changed, based on i_version
*/
static void ima_file_free(struct file *file)
{
struct inode *inode = file_inode(file);
struct ima_iint_cache *iint;
if (!ima_policy_flag || !S_ISREG(inode->i_mode))
return;
iint = ima_iint_find(inode);
if (!iint)
return;
ima_check_last_writer(iint, inode, file);
}
static int process_measurement(struct file *file, const struct cred *cred,
u32 secid, char *buf, loff_t size, int mask,
enum ima_hooks func)
{
struct inode *backing_inode, *inode = file_inode(file);
struct ima_iint_cache *iint = NULL;
struct ima_template_desc *template_desc = NULL;
char *pathbuf = NULL;
char filename[NAME_MAX];
const char *pathname = NULL;
int rc = 0, action, must_appraise = 0;
int pcr = CONFIG_IMA_MEASURE_PCR_IDX;
struct evm_ima_xattr_data *xattr_value = NULL;
struct modsig *modsig = NULL;
int xattr_len = 0;
bool violation_check;
enum hash_algo hash_algo;
unsigned int allowed_algos = 0;
if (!ima_policy_flag || !S_ISREG(inode->i_mode)) return 0;
/* Return an IMA_MEASURE, IMA_APPRAISE, IMA_AUDIT action
* bitmask based on the appraise/audit/measurement policy.
* Included is the appraise submask.
*/
action = ima_get_action(file_mnt_idmap(file), inode, cred, secid,
mask, func, &pcr, &template_desc, NULL,
&allowed_algos);
violation_check = ((func == FILE_CHECK || func == MMAP_CHECK ||
func == MMAP_CHECK_REQPROT) &&
(ima_policy_flag & IMA_MEASURE)); if (!action && !violation_check)
return 0;
must_appraise = action & IMA_APPRAISE;
/* Is the appraise rule hook specific? */
if (action & IMA_FILE_APPRAISE)
func = FILE_CHECK;
inode_lock(inode);
if (action) {
iint = ima_inode_get(inode);
if (!iint)
rc = -ENOMEM;
}
if (!rc && violation_check) ima_rdwr_violation_check(file, iint, action & IMA_MEASURE,
&pathbuf, &pathname, filename);
inode_unlock(inode);
if (rc)
goto out;
if (!action)
goto out;
mutex_lock(&iint->mutex); if (test_and_clear_bit(IMA_CHANGE_ATTR, &iint->atomic_flags))
/* reset appraisal flags if ima_inode_post_setattr was called */
iint->flags &= ~(IMA_APPRAISE | IMA_APPRAISED |
IMA_APPRAISE_SUBMASK | IMA_APPRAISED_SUBMASK |
IMA_NONACTION_FLAGS);
/*
* Re-evaulate the file if either the xattr has changed or the
* kernel has no way of detecting file change on the filesystem.
* (Limited to privileged mounted filesystems.)
*/
if (test_and_clear_bit(IMA_CHANGE_XATTR, &iint->atomic_flags) || ((inode->i_sb->s_iflags & SB_I_IMA_UNVERIFIABLE_SIGNATURE) &&
!(inode->i_sb->s_iflags & SB_I_UNTRUSTED_MOUNTER) &&
!(action & IMA_FAIL_UNVERIFIABLE_SIGS))) { iint->flags &= ~IMA_DONE_MASK;
iint->measured_pcrs = 0;
}
/* Detect and re-evaluate changes made to the backing file. */
backing_inode = d_real_inode(file_dentry(file));
if (backing_inode != inode &&
(action & IMA_DO_MASK) && (iint->flags & IMA_DONE_MASK)) { if (!IS_I_VERSION(backing_inode) || backing_inode->i_sb->s_dev != iint->real_dev || backing_inode->i_ino != iint->real_ino || !inode_eq_iversion(backing_inode, iint->version)) { iint->flags &= ~IMA_DONE_MASK;
iint->measured_pcrs = 0;
}
}
/* Determine if already appraised/measured based on bitmask
* (IMA_MEASURE, IMA_MEASURED, IMA_XXXX_APPRAISE, IMA_XXXX_APPRAISED,
* IMA_AUDIT, IMA_AUDITED)
*/
iint->flags |= action;
action &= IMA_DO_MASK;
action &= ~((iint->flags & (IMA_DONE_MASK ^ IMA_MEASURED)) >> 1);
/* If target pcr is already measured, unset IMA_MEASURE action */
if ((action & IMA_MEASURE) && (iint->measured_pcrs & (0x1 << pcr))) action ^= IMA_MEASURE;
/* HASH sets the digital signature and update flags, nothing else */
if ((action & IMA_HASH) && !(test_bit(IMA_DIGSIG, &iint->atomic_flags))) { xattr_len = ima_read_xattr(file_dentry(file),
&xattr_value, xattr_len);
if ((xattr_value && xattr_len > 2) && (xattr_value->type == EVM_IMA_XATTR_DIGSIG)) set_bit(IMA_DIGSIG, &iint->atomic_flags); iint->flags |= IMA_HASHED;
action ^= IMA_HASH;
set_bit(IMA_UPDATE_XATTR, &iint->atomic_flags);
}
/* Nothing to do, just return existing appraised status */
if (!action) { if (must_appraise) { rc = mmap_violation_check(func, file, &pathbuf,
&pathname, filename);
if (!rc)
rc = ima_get_cache_status(iint, func);
}
goto out_locked;
}
if ((action & IMA_APPRAISE_SUBMASK) || strcmp(template_desc->name, IMA_TEMPLATE_IMA_NAME) != 0) {
/* read 'security.ima' */
xattr_len = ima_read_xattr(file_dentry(file),
&xattr_value, xattr_len);
/*
* Read the appended modsig if allowed by the policy, and allow
* an additional measurement list entry, if needed, based on the
* template format and whether the file was already measured.
*/
if (iint->flags & IMA_MODSIG_ALLOWED) {
rc = ima_read_modsig(func, buf, size, &modsig); if (!rc && ima_template_has_modsig(template_desc) && iint->flags & IMA_MEASURED) action |= IMA_MEASURE;
}
}
hash_algo = ima_get_hash_algo(xattr_value, xattr_len);
rc = ima_collect_measurement(iint, file, buf, size, hash_algo, modsig);
if (rc != 0 && rc != -EBADF && rc != -EINVAL)
goto out_locked;
if (!pathbuf) /* ima_rdwr_violation possibly pre-fetched */ pathname = ima_d_path(&file->f_path, &pathbuf, filename); if (action & IMA_MEASURE) ima_store_measurement(iint, file, pathname,
xattr_value, xattr_len, modsig, pcr,
template_desc);
if (rc == 0 && (action & IMA_APPRAISE_SUBMASK)) { rc = ima_check_blacklist(iint, modsig, pcr);
if (rc != -EPERM) {
inode_lock(inode);
rc = ima_appraise_measurement(func, iint, file,
pathname, xattr_value,
xattr_len, modsig);
inode_unlock(inode);
}
if (!rc)
rc = mmap_violation_check(func, file, &pathbuf,
&pathname, filename);
}
if (action & IMA_AUDIT) ima_audit_measurement(iint, pathname); if ((file->f_flags & O_DIRECT) && (iint->flags & IMA_PERMIT_DIRECTIO))
rc = 0;
/* Ensure the digest was generated using an allowed algorithm */
if (rc == 0 && must_appraise && allowed_algos != 0 && (allowed_algos & (1U << hash_algo)) == 0) {
rc = -EACCES;
integrity_audit_msg(AUDIT_INTEGRITY_DATA, file_inode(file),
pathname, "collect_data",
"denied-hash-algorithm", rc, 0);
}
out_locked:
if ((mask & MAY_WRITE) && test_bit(IMA_DIGSIG, &iint->atomic_flags) && !(iint->flags & IMA_NEW_FILE))
rc = -EACCES;
mutex_unlock(&iint->mutex);
kfree(xattr_value);
ima_free_modsig(modsig);
out:
if (pathbuf) __putname(pathbuf); if (must_appraise) { if (rc && (ima_appraise & IMA_APPRAISE_ENFORCE))
return -EACCES;
if (file->f_mode & FMODE_WRITE) set_bit(IMA_UPDATE_XATTR, &iint->atomic_flags);
}
return 0;
}
/**
* ima_file_mmap - based on policy, collect/store measurement.
* @file: pointer to the file to be measured (May be NULL)
* @reqprot: protection requested by the application
* @prot: protection that will be applied by the kernel
* @flags: operational flags
*
* Measure files being mmapped executable based on the ima_must_measure()
* policy decision.
*
* On success return 0. On integrity appraisal error, assuming the file
* is in policy and IMA-appraisal is in enforcing mode, return -EACCES.
*/
static int ima_file_mmap(struct file *file, unsigned long reqprot,
unsigned long prot, unsigned long flags)
{
u32 secid;
int ret;
if (!file)
return 0;
security_current_getsecid_subj(&secid);
if (reqprot & PROT_EXEC) {
ret = process_measurement(file, current_cred(), secid, NULL,
0, MAY_EXEC, MMAP_CHECK_REQPROT);
if (ret)
return ret;
}
if (prot & PROT_EXEC)
return process_measurement(file, current_cred(), secid, NULL,
0, MAY_EXEC, MMAP_CHECK);
return 0;
}
/**
* ima_file_mprotect - based on policy, limit mprotect change
* @vma: vm_area_struct protection is set to
* @reqprot: protection requested by the application
* @prot: protection that will be applied by the kernel
*
* Files can be mmap'ed read/write and later changed to execute to circumvent
* IMA's mmap appraisal policy rules. Due to locking issues (mmap semaphore
* would be taken before i_mutex), files can not be measured or appraised at
* this point. Eliminate this integrity gap by denying the mprotect
* PROT_EXECUTE change, if an mmap appraise policy rule exists.
*
* On mprotect change success, return 0. On failure, return -EACESS.
*/
static int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot,
unsigned long prot)
{
struct ima_template_desc *template = NULL;
struct file *file;
char filename[NAME_MAX];
char *pathbuf = NULL;
const char *pathname = NULL;
struct inode *inode;
int result = 0;
int action;
u32 secid;
int pcr;
/* Is mprotect making an mmap'ed file executable? */
if (!(ima_policy_flag & IMA_APPRAISE) || !vma->vm_file ||
!(prot & PROT_EXEC) || (vma->vm_flags & VM_EXEC))
return 0;
security_current_getsecid_subj(&secid);
inode = file_inode(vma->vm_file);
action = ima_get_action(file_mnt_idmap(vma->vm_file), inode,
current_cred(), secid, MAY_EXEC, MMAP_CHECK,
&pcr, &template, NULL, NULL);
action |= ima_get_action(file_mnt_idmap(vma->vm_file), inode,
current_cred(), secid, MAY_EXEC,
MMAP_CHECK_REQPROT, &pcr, &template, NULL,
NULL);
/* Is the mmap'ed file in policy? */
if (!(action & (IMA_MEASURE | IMA_APPRAISE_SUBMASK)))
return 0;
if (action & IMA_APPRAISE_SUBMASK)
result = -EPERM;
file = vma->vm_file;
pathname = ima_d_path(&file->f_path, &pathbuf, filename);
integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode, pathname,
"collect_data", "failed-mprotect", result, 0);
if (pathbuf)
__putname(pathbuf);
return result;
}
/**
* ima_bprm_check - based on policy, collect/store measurement.
* @bprm: contains the linux_binprm structure
*
* The OS protects against an executable file, already open for write,
* from being executed in deny_write_access() and an executable file,
* already open for execute, from being modified in get_write_access().
* So we can be certain that what we verify and measure here is actually
* what is being executed.
*
* On success return 0. On integrity appraisal error, assuming the file
* is in policy and IMA-appraisal is in enforcing mode, return -EACCES.
*/
static int ima_bprm_check(struct linux_binprm *bprm)
{
int ret;
u32 secid;
security_current_getsecid_subj(&secid);
ret = process_measurement(bprm->file, current_cred(), secid, NULL, 0,
MAY_EXEC, BPRM_CHECK);
if (ret)
return ret;
security_cred_getsecid(bprm->cred, &secid);
return process_measurement(bprm->file, bprm->cred, secid, NULL, 0,
MAY_EXEC, CREDS_CHECK);
}
/**
* ima_file_check - based on policy, collect/store measurement.
* @file: pointer to the file to be measured
* @mask: contains MAY_READ, MAY_WRITE, MAY_EXEC or MAY_APPEND
*
* Measure files based on the ima_must_measure() policy decision.
*
* On success return 0. On integrity appraisal error, assuming the file
* is in policy and IMA-appraisal is in enforcing mode, return -EACCES.
*/
static int ima_file_check(struct file *file, int mask)
{
u32 secid;
security_current_getsecid_subj(&secid);
return process_measurement(file, current_cred(), secid, NULL, 0,
mask & (MAY_READ | MAY_WRITE | MAY_EXEC |
MAY_APPEND), FILE_CHECK);
}
static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf,
size_t buf_size)
{
struct ima_iint_cache *iint = NULL, tmp_iint;
int rc, hash_algo;
if (ima_policy_flag) {
iint = ima_iint_find(inode);
if (iint)
mutex_lock(&iint->mutex);
}
if ((!iint || !(iint->flags & IMA_COLLECTED)) && file) {
if (iint)
mutex_unlock(&iint->mutex);
memset(&tmp_iint, 0, sizeof(tmp_iint));
mutex_init(&tmp_iint.mutex);
rc = ima_collect_measurement(&tmp_iint, file, NULL, 0,
ima_hash_algo, NULL);
if (rc < 0) {
/* ima_hash could be allocated in case of failure. */
if (rc != -ENOMEM)
kfree(tmp_iint.ima_hash);
return -EOPNOTSUPP;
}
iint = &tmp_iint;
mutex_lock(&iint->mutex);
}
if (!iint)
return -EOPNOTSUPP;
/*
* ima_file_hash can be called when ima_collect_measurement has still
* not been called, we might not always have a hash.
*/
if (!iint->ima_hash || !(iint->flags & IMA_COLLECTED)) {
mutex_unlock(&iint->mutex);
return -EOPNOTSUPP;
}
if (buf) {
size_t copied_size;
copied_size = min_t(size_t, iint->ima_hash->length, buf_size);
memcpy(buf, iint->ima_hash->digest, copied_size);
}
hash_algo = iint->ima_hash->algo;
mutex_unlock(&iint->mutex);
if (iint == &tmp_iint)
kfree(iint->ima_hash);
return hash_algo;
}
/**
* ima_file_hash - return a measurement of the file
* @file: pointer to the file
* @buf: buffer in which to store the hash
* @buf_size: length of the buffer
*
* On success, return the hash algorithm (as defined in the enum hash_algo).
* If buf is not NULL, this function also outputs the hash into buf.
* If the hash is larger than buf_size, then only buf_size bytes will be copied.
* It generally just makes sense to pass a buffer capable of holding the largest
* possible hash: IMA_MAX_DIGEST_SIZE.
* The file hash returned is based on the entire file, including the appended
* signature.
*
* If the measurement cannot be performed, return -EOPNOTSUPP.
* If the parameters are incorrect, return -EINVAL.
*/
int ima_file_hash(struct file *file, char *buf, size_t buf_size)
{
if (!file)
return -EINVAL;
return __ima_inode_hash(file_inode(file), file, buf, buf_size);
}
EXPORT_SYMBOL_GPL(ima_file_hash);
/**
* ima_inode_hash - return the stored measurement if the inode has been hashed
* and is in the iint cache.
* @inode: pointer to the inode
* @buf: buffer in which to store the hash
* @buf_size: length of the buffer
*
* On success, return the hash algorithm (as defined in the enum hash_algo).
* If buf is not NULL, this function also outputs the hash into buf.
* If the hash is larger than buf_size, then only buf_size bytes will be copied.
* It generally just makes sense to pass a buffer capable of holding the largest
* possible hash: IMA_MAX_DIGEST_SIZE.
* The hash returned is based on the entire contents, including the appended
* signature.
*
* If IMA is disabled or if no measurement is available, return -EOPNOTSUPP.
* If the parameters are incorrect, return -EINVAL.
*/
int ima_inode_hash(struct inode *inode, char *buf, size_t buf_size)
{
if (!inode)
return -EINVAL;
return __ima_inode_hash(inode, NULL, buf, buf_size);
}
EXPORT_SYMBOL_GPL(ima_inode_hash);
/**
* ima_post_create_tmpfile - mark newly created tmpfile as new
* @idmap: idmap of the mount the inode was found from
* @inode: inode of the newly created tmpfile
*
* No measuring, appraising or auditing of newly created tmpfiles is needed.
* Skip calling process_measurement(), but indicate which newly, created
* tmpfiles are in policy.
*/
static void ima_post_create_tmpfile(struct mnt_idmap *idmap,
struct inode *inode)
{
struct ima_iint_cache *iint;
int must_appraise;
if (!ima_policy_flag || !S_ISREG(inode->i_mode))
return;
must_appraise = ima_must_appraise(idmap, inode, MAY_ACCESS,
FILE_CHECK);
if (!must_appraise)
return;
/* Nothing to do if we can't allocate memory */
iint = ima_inode_get(inode);
if (!iint)
return;
/* needed for writing the security xattrs */
set_bit(IMA_UPDATE_XATTR, &iint->atomic_flags);
iint->ima_file_status = INTEGRITY_PASS;
}
/**
* ima_post_path_mknod - mark as a new inode
* @idmap: idmap of the mount the inode was found from
* @dentry: newly created dentry
*
* Mark files created via the mknodat syscall as new, so that the
* file data can be written later.
*/
static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry)
{
struct ima_iint_cache *iint;
struct inode *inode = dentry->d_inode;
int must_appraise;
if (!ima_policy_flag || !S_ISREG(inode->i_mode))
return;
must_appraise = ima_must_appraise(idmap, inode, MAY_ACCESS,
FILE_CHECK);
if (!must_appraise)
return;
/* Nothing to do if we can't allocate memory */
iint = ima_inode_get(inode);
if (!iint)
return;
/* needed for re-opening empty files */
iint->flags |= IMA_NEW_FILE;
}
/**
* ima_read_file - pre-measure/appraise hook decision based on policy
* @file: pointer to the file to be measured/appraised/audit
* @read_id: caller identifier
* @contents: whether a subsequent call will be made to ima_post_read_file()
*
* Permit reading a file based on policy. The policy rules are written
* in terms of the policy identifier. Appraising the integrity of
* a file requires a file descriptor.
*
* For permission return 0, otherwise return -EACCES.
*/
static int ima_read_file(struct file *file, enum kernel_read_file_id read_id,
bool contents)
{
enum ima_hooks func;
u32 secid;
/*
* Do devices using pre-allocated memory run the risk of the
* firmware being accessible to the device prior to the completion
* of IMA's signature verification any more than when using two
* buffers? It may be desirable to include the buffer address
* in this API and walk all the dma_map_single() mappings to check.
*/
/*
* There will be a call made to ima_post_read_file() with
* a filled buffer, so we don't need to perform an extra
* read early here.
*/
if (contents)
return 0;
/* Read entire file for all partial reads. */
func = read_idmap[read_id] ?: FILE_CHECK;
security_current_getsecid_subj(&secid);
return process_measurement(file, current_cred(), secid, NULL,
0, MAY_READ, func);
}
const int read_idmap[READING_MAX_ID] = {
[READING_FIRMWARE] = FIRMWARE_CHECK,
[READING_MODULE] = MODULE_CHECK,
[READING_KEXEC_IMAGE] = KEXEC_KERNEL_CHECK,
[READING_KEXEC_INITRAMFS] = KEXEC_INITRAMFS_CHECK,
[READING_POLICY] = POLICY_CHECK
};
/**
* ima_post_read_file - in memory collect/appraise/audit measurement
* @file: pointer to the file to be measured/appraised/audit
* @buf: pointer to in memory file contents
* @size: size of in memory file contents
* @read_id: caller identifier
*
* Measure/appraise/audit in memory file based on policy. Policy rules
* are written in terms of a policy identifier.
*
* On success return 0. On integrity appraisal error, assuming the file
* is in policy and IMA-appraisal is in enforcing mode, return -EACCES.
*/
static int ima_post_read_file(struct file *file, char *buf, loff_t size,
enum kernel_read_file_id read_id)
{
enum ima_hooks func;
u32 secid;
/* permit signed certs */
if (!file && read_id == READING_X509_CERTIFICATE)
return 0;
if (!file || !buf || size == 0) { /* should never happen */
if (ima_appraise & IMA_APPRAISE_ENFORCE)
return -EACCES;
return 0;
}
func = read_idmap[read_id] ?: FILE_CHECK;
security_current_getsecid_subj(&secid);
return process_measurement(file, current_cred(), secid, buf, size,
MAY_READ, func);
}
/**
* ima_load_data - appraise decision based on policy
* @id: kernel load data caller identifier
* @contents: whether the full contents will be available in a later
* call to ima_post_load_data().
*
* Callers of this LSM hook can not measure, appraise, or audit the
* data provided by userspace. Enforce policy rules requiring a file
* signature (eg. kexec'ed kernel image).
*
* For permission return 0, otherwise return -EACCES.
*/
static int ima_load_data(enum kernel_load_data_id id, bool contents)
{
bool ima_enforce, sig_enforce;
ima_enforce =
(ima_appraise & IMA_APPRAISE_ENFORCE) == IMA_APPRAISE_ENFORCE;
switch (id) {
case LOADING_KEXEC_IMAGE:
if (IS_ENABLED(CONFIG_KEXEC_SIG)
&& arch_ima_get_secureboot()) {
pr_err("impossible to appraise a kernel image without a file descriptor; try using kexec_file_load syscall.\n");
return -EACCES;
}
if (ima_enforce && (ima_appraise & IMA_APPRAISE_KEXEC)) {
pr_err("impossible to appraise a kernel image without a file descriptor; try using kexec_file_load syscall.\n");
return -EACCES; /* INTEGRITY_UNKNOWN */
}
break;
case LOADING_FIRMWARE:
if (ima_enforce && (ima_appraise & IMA_APPRAISE_FIRMWARE) && !contents) {
pr_err("Prevent firmware sysfs fallback loading.\n");
return -EACCES; /* INTEGRITY_UNKNOWN */
}
break;
case LOADING_MODULE:
sig_enforce = is_module_sig_enforced();
if (ima_enforce && (!sig_enforce
&& (ima_appraise & IMA_APPRAISE_MODULES))) {
pr_err("impossible to appraise a module without a file descriptor. sig_enforce kernel parameter might help\n");
return -EACCES; /* INTEGRITY_UNKNOWN */
}
break;
default:
break;
}
return 0;
}
/**
* ima_post_load_data - appraise decision based on policy
* @buf: pointer to in memory file contents
* @size: size of in memory file contents
* @load_id: kernel load data caller identifier
* @description: @load_id-specific description of contents
*
* Measure/appraise/audit in memory buffer based on policy. Policy rules
* are written in terms of a policy identifier.
*
* On success return 0. On integrity appraisal error, assuming the file
* is in policy and IMA-appraisal is in enforcing mode, return -EACCES.
*/
static int ima_post_load_data(char *buf, loff_t size,
enum kernel_load_data_id load_id,
char *description)
{
if (load_id == LOADING_FIRMWARE) {
if ((ima_appraise & IMA_APPRAISE_FIRMWARE) &&
(ima_appraise & IMA_APPRAISE_ENFORCE)) {
pr_err("Prevent firmware loading_store.\n");
return -EACCES; /* INTEGRITY_UNKNOWN */
}
return 0;
}
return 0;
}
/**
* process_buffer_measurement - Measure the buffer or the buffer data hash
* @idmap: idmap of the mount the inode was found from
* @inode: inode associated with the object being measured (NULL for KEY_CHECK)
* @buf: pointer to the buffer that needs to be added to the log.
* @size: size of buffer(in bytes).
* @eventname: event name to be used for the buffer entry.
* @func: IMA hook
* @pcr: pcr to extend the measurement
* @func_data: func specific data, may be NULL
* @buf_hash: measure buffer data hash
* @digest: buffer digest will be written to
* @digest_len: buffer length
*
* Based on policy, either the buffer data or buffer data hash is measured
*
* Return: 0 if the buffer has been successfully measured, 1 if the digest
* has been written to the passed location but not added to a measurement entry,
* a negative value otherwise.
*/
int process_buffer_measurement(struct mnt_idmap *idmap,
struct inode *inode, const void *buf, int size,
const char *eventname, enum ima_hooks func,
int pcr, const char *func_data,
bool buf_hash, u8 *digest, size_t digest_len)
{
int ret = 0;
const char *audit_cause = "ENOMEM";
struct ima_template_entry *entry = NULL;
struct ima_iint_cache iint = {};
struct ima_event_data event_data = {.iint = &iint,
.filename = eventname,
.buf = buf,
.buf_len = size};
struct ima_template_desc *template;
struct ima_max_digest_data hash;
char digest_hash[IMA_MAX_DIGEST_SIZE];
int digest_hash_len = hash_digest_size[ima_hash_algo];
int violation = 0;
int action = 0;
u32 secid;
if (digest && digest_len < digest_hash_len)
return -EINVAL;
if (!ima_policy_flag && !digest)
return -ENOENT;
template = ima_template_desc_buf();
if (!template) {
ret = -EINVAL;
audit_cause = "ima_template_desc_buf";
goto out;
}
/*
* Both LSM hooks and auxilary based buffer measurements are
* based on policy. To avoid code duplication, differentiate
* between the LSM hooks and auxilary buffer measurements,
* retrieving the policy rule information only for the LSM hook
* buffer measurements.
*/
if (func) {
security_current_getsecid_subj(&secid);
action = ima_get_action(idmap, inode, current_cred(),
secid, 0, func, &pcr, &template,
func_data, NULL);
if (!(action & IMA_MEASURE) && !digest)
return -ENOENT;
}
if (!pcr)
pcr = CONFIG_IMA_MEASURE_PCR_IDX;
iint.ima_hash = &hash.hdr;
iint.ima_hash->algo = ima_hash_algo;
iint.ima_hash->length = hash_digest_size[ima_hash_algo];
ret = ima_calc_buffer_hash(buf, size, iint.ima_hash);
if (ret < 0) {
audit_cause = "hashing_error";
goto out;
}
if (buf_hash) {
memcpy(digest_hash, hash.hdr.digest, digest_hash_len);
ret = ima_calc_buffer_hash(digest_hash, digest_hash_len,
iint.ima_hash);
if (ret < 0) {
audit_cause = "hashing_error";
goto out;
}
event_data.buf = digest_hash;
event_data.buf_len = digest_hash_len;
}
if (digest)
memcpy(digest, iint.ima_hash->digest, digest_hash_len);
if (!ima_policy_flag || (func && !(action & IMA_MEASURE)))
return 1;
ret = ima_alloc_init_template(&event_data, &entry, template);
if (ret < 0) {
audit_cause = "alloc_entry";
goto out;
}
ret = ima_store_template(entry, violation, NULL, event_data.buf, pcr);
if (ret < 0) {
audit_cause = "store_entry";
ima_free_template_entry(entry);
}
out:
if (ret < 0)
integrity_audit_message(AUDIT_INTEGRITY_PCR, NULL, eventname,
func_measure_str(func),
audit_cause, ret, 0, ret);
return ret;
}
/**
* ima_kexec_cmdline - measure kexec cmdline boot args
* @kernel_fd: file descriptor of the kexec kernel being loaded
* @buf: pointer to buffer
* @size: size of buffer
*
* Buffers can only be measured, not appraised.
*/
void ima_kexec_cmdline(int kernel_fd, const void *buf, int size)
{
struct fd f;
if (!buf || !size)
return;
f = fdget(kernel_fd);
if (!f.file)
return;
process_buffer_measurement(file_mnt_idmap(f.file), file_inode(f.file),
buf, size, "kexec-cmdline", KEXEC_CMDLINE, 0,
NULL, false, NULL, 0);
fdput(f);
}
/**
* ima_measure_critical_data - measure kernel integrity critical data
* @event_label: unique event label for grouping and limiting critical data
* @event_name: event name for the record in the IMA measurement list
* @buf: pointer to buffer data
* @buf_len: length of buffer data (in bytes)
* @hash: measure buffer data hash
* @digest: buffer digest will be written to
* @digest_len: buffer length
*
* Measure data critical to the integrity of the kernel into the IMA log
* and extend the pcr. Examples of critical data could be various data
* structures, policies, and states stored in kernel memory that can
* impact the integrity of the system.
*
* Return: 0 if the buffer has been successfully measured, 1 if the digest
* has been written to the passed location but not added to a measurement entry,
* a negative value otherwise.
*/
int ima_measure_critical_data(const char *event_label,
const char *event_name,
const void *buf, size_t buf_len,
bool hash, u8 *digest, size_t digest_len)
{
if (!event_name || !event_label || !buf || !buf_len)
return -ENOPARAM;
return process_buffer_measurement(&nop_mnt_idmap, NULL, buf, buf_len,
event_name, CRITICAL_DATA, 0,
event_label, hash, digest,
digest_len);
}
EXPORT_SYMBOL_GPL(ima_measure_critical_data);
#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS
/**
* ima_kernel_module_request - Prevent crypto-pkcs1pad(rsa,*) requests
* @kmod_name: kernel module name
*
* Avoid a verification loop where verifying the signature of the modprobe
* binary requires executing modprobe itself. Since the modprobe iint->mutex
* is already held when the signature verification is performed, a deadlock
* occurs as soon as modprobe is executed within the critical region, since
* the same lock cannot be taken again.
*
* This happens when public_key_verify_signature(), in case of RSA algorithm,
* use alg_name to store internal information in order to construct an
* algorithm on the fly, but crypto_larval_lookup() will try to use alg_name
* in order to load a kernel module with same name.
*
* Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules,
* we are safe to fail such module request from crypto_larval_lookup(), and
* avoid the verification loop.
*
* Return: Zero if it is safe to load the kernel module, -EINVAL otherwise.
*/
static int ima_kernel_module_request(char *kmod_name)
{
if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0)
return -EINVAL;
return 0;
}
#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */
static int __init init_ima(void)
{
int error;
ima_appraise_parse_cmdline();
ima_init_template_list();
hash_setup(CONFIG_IMA_DEFAULT_HASH);
error = ima_init();
if (error && strcmp(hash_algo_name[ima_hash_algo],
CONFIG_IMA_DEFAULT_HASH) != 0) {
pr_info("Allocating %s failed, going to use default hash algorithm %s\n",
hash_algo_name[ima_hash_algo], CONFIG_IMA_DEFAULT_HASH);
hash_setup_done = 0;
hash_setup(CONFIG_IMA_DEFAULT_HASH);
error = ima_init();
}
if (error)
return error;
error = register_blocking_lsm_notifier(&ima_lsm_policy_notifier);
if (error)
pr_warn("Couldn't register LSM notifier, error %d\n", error);
if (!error)
ima_update_policy_flags();
return error;
}
static struct security_hook_list ima_hooks[] __ro_after_init = {
LSM_HOOK_INIT(bprm_check_security, ima_bprm_check),
LSM_HOOK_INIT(file_post_open, ima_file_check),
LSM_HOOK_INIT(inode_post_create_tmpfile, ima_post_create_tmpfile),
LSM_HOOK_INIT(file_release, ima_file_free),
LSM_HOOK_INIT(mmap_file, ima_file_mmap),
LSM_HOOK_INIT(file_mprotect, ima_file_mprotect),
LSM_HOOK_INIT(kernel_load_data, ima_load_data),
LSM_HOOK_INIT(kernel_post_load_data, ima_post_load_data),
LSM_HOOK_INIT(kernel_read_file, ima_read_file),
LSM_HOOK_INIT(kernel_post_read_file, ima_post_read_file),
LSM_HOOK_INIT(path_post_mknod, ima_post_path_mknod),
#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS
LSM_HOOK_INIT(key_post_create_or_update, ima_post_key_create_or_update),
#endif
#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS
LSM_HOOK_INIT(kernel_module_request, ima_kernel_module_request),
#endif
LSM_HOOK_INIT(inode_free_security, ima_inode_free),
};
static const struct lsm_id ima_lsmid = {
.name = "ima",
.id = LSM_ID_IMA,
};
static int __init init_ima_lsm(void)
{
ima_iintcache_init();
security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid);
init_ima_appraise_lsm(&ima_lsmid);
return 0;
}
struct lsm_blob_sizes ima_blob_sizes __ro_after_init = {
.lbs_inode = sizeof(struct ima_iint_cache *),
};
DEFINE_LSM(ima) = {
.name = "ima",
.init = init_ima_lsm,
.order = LSM_ORDER_LAST,
.blobs = &ima_blob_sizes,
};
late_initcall(init_ima); /* Start IMA after the TPM is available */
// SPDX-License-Identifier: GPL-2.0
/*
* linux/drivers/char/misc.c
*
* Generic misc open routine by Johan Myreen
*
* Based on code from Linus
*
* Teemu Rantanen's Microsoft Busmouse support and Derrick Cole's
* changes incorporated into 0.97pl4
* by Peter Cervasio (pete%q106fm.uucp@wupost.wustl.edu) (08SEP92)
* See busmouse.c for particulars.
*
* Made things a lot mode modular - easy to compile in just one or two
* of the misc drivers, as they are now completely independent. Linus.
*
* Support for loadable modules. 8-Sep-95 Philip Blundell <pjb27@cam.ac.uk>
*
* Fixed a failing symbol register to free the device registration
* Alan Cox <alan@lxorguk.ukuu.org.uk> 21-Jan-96
*
* Dynamic minors and /proc/mice by Alessandro Rubini. 26-Mar-96
*
* Renamed to misc and miscdevice to be more accurate. Alan Cox 26-Mar-96
*
* Handling of mouse minor numbers for kerneld:
* Idea by Jacques Gelinas <jack@solucorp.qc.ca>,
* adapted by Bjorn Ekwall <bj0rn@blox.se>
* corrected by Alan Cox <alan@lxorguk.ukuu.org.uk>
*
* Changes for kmod (from kerneld):
* Cyrus Durgin <cider@speakeasy.org>
*
* Added devfs support. Richard Gooch <rgooch@atnf.csiro.au> 10-Jan-1998
*/
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/miscdevice.h>
#include <linux/kernel.h>
#include <linux/major.h>
#include <linux/mutex.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/stat.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/tty.h>
#include <linux/kmod.h>
#include <linux/gfp.h>
/*
* Head entry for the doubly linked miscdevice list
*/
static LIST_HEAD(misc_list);
static DEFINE_MUTEX(misc_mtx);
/*
* Assigned numbers, used for dynamic minors
*/
#define DYNAMIC_MINORS 128 /* like dynamic majors */
static DEFINE_IDA(misc_minors_ida);
static int misc_minor_alloc(void)
{
int ret;
ret = ida_alloc_max(&misc_minors_ida, DYNAMIC_MINORS - 1, GFP_KERNEL);
if (ret >= 0) {
ret = DYNAMIC_MINORS - ret - 1;
} else {
ret = ida_alloc_range(&misc_minors_ida, MISC_DYNAMIC_MINOR + 1,
MINORMASK, GFP_KERNEL);
}
return ret;
}
static void misc_minor_free(int minor)
{
if (minor < DYNAMIC_MINORS)
ida_free(&misc_minors_ida, DYNAMIC_MINORS - minor - 1);
else if (minor > MISC_DYNAMIC_MINOR)
ida_free(&misc_minors_ida, minor);
}
#ifdef CONFIG_PROC_FS
static void *misc_seq_start(struct seq_file *seq, loff_t *pos)
{
mutex_lock(&misc_mtx);
return seq_list_start(&misc_list, *pos);
}
static void *misc_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
return seq_list_next(v, &misc_list, pos);
}
static void misc_seq_stop(struct seq_file *seq, void *v)
{
mutex_unlock(&misc_mtx);
}
static int misc_seq_show(struct seq_file *seq, void *v)
{
const struct miscdevice *p = list_entry(v, struct miscdevice, list);
seq_printf(seq, "%3i %s\n", p->minor, p->name ? p->name : "");
return 0;
}
static const struct seq_operations misc_seq_ops = {
.start = misc_seq_start,
.next = misc_seq_next,
.stop = misc_seq_stop,
.show = misc_seq_show,
};
#endif
static int misc_open(struct inode *inode, struct file *file)
{
int minor = iminor(inode);
struct miscdevice *c = NULL, *iter;
int err = -ENODEV;
const struct file_operations *new_fops = NULL;
mutex_lock(&misc_mtx);
list_for_each_entry(iter, &misc_list, list) { if (iter->minor != minor)
continue;
c = iter;
new_fops = fops_get(iter->fops);
break;
}
if (!new_fops) { mutex_unlock(&misc_mtx);
request_module("char-major-%d-%d", MISC_MAJOR, minor);
mutex_lock(&misc_mtx);
list_for_each_entry(iter, &misc_list, list) { if (iter->minor != minor)
continue;
c = iter;
new_fops = fops_get(iter->fops);
break;
}
if (!new_fops)
goto fail;
}
/*
* Place the miscdevice in the file's
* private_data so it can be used by the
* file operations, including f_op->open below
*/
file->private_data = c;
err = 0;
replace_fops(file, new_fops);
if (file->f_op->open)
err = file->f_op->open(inode, file);
fail:
mutex_unlock(&misc_mtx);
return err;
}
static char *misc_devnode(const struct device *dev, umode_t *mode)
{
const struct miscdevice *c = dev_get_drvdata(dev);
if (mode && c->mode)
*mode = c->mode;
if (c->nodename)
return kstrdup(c->nodename, GFP_KERNEL);
return NULL;
}
static const struct class misc_class = {
.name = "misc",
.devnode = misc_devnode,
};
static const struct file_operations misc_fops = {
.owner = THIS_MODULE,
.open = misc_open,
.llseek = noop_llseek,
};
/**
* misc_register - register a miscellaneous device
* @misc: device structure
*
* Register a miscellaneous device with the kernel. If the minor
* number is set to %MISC_DYNAMIC_MINOR a minor number is assigned
* and placed in the minor field of the structure. For other cases
* the minor number requested is used.
*
* The structure passed is linked into the kernel and may not be
* destroyed until it has been unregistered. By default, an open()
* syscall to the device sets file->private_data to point to the
* structure. Drivers don't need open in fops for this.
*
* A zero is returned on success and a negative errno code for
* failure.
*/
int misc_register(struct miscdevice *misc)
{
dev_t dev;
int err = 0;
bool is_dynamic = (misc->minor == MISC_DYNAMIC_MINOR);
INIT_LIST_HEAD(&misc->list);
mutex_lock(&misc_mtx);
if (is_dynamic) {
int i = misc_minor_alloc();
if (i < 0) {
err = -EBUSY;
goto out;
}
misc->minor = i;
} else {
struct miscdevice *c;
list_for_each_entry(c, &misc_list, list) {
if (c->minor == misc->minor) {
err = -EBUSY;
goto out;
}
}
}
dev = MKDEV(MISC_MAJOR, misc->minor);
misc->this_device =
device_create_with_groups(&misc_class, misc->parent, dev,
misc, misc->groups, "%s", misc->name);
if (IS_ERR(misc->this_device)) {
if (is_dynamic) {
misc_minor_free(misc->minor);
misc->minor = MISC_DYNAMIC_MINOR;
}
err = PTR_ERR(misc->this_device);
goto out;
}
/*
* Add it to the front, so that later devices can "override"
* earlier defaults
*/
list_add(&misc->list, &misc_list);
out:
mutex_unlock(&misc_mtx);
return err;
}
EXPORT_SYMBOL(misc_register);
/**
* misc_deregister - unregister a miscellaneous device
* @misc: device to unregister
*
* Unregister a miscellaneous device that was previously
* successfully registered with misc_register().
*/
void misc_deregister(struct miscdevice *misc)
{
if (WARN_ON(list_empty(&misc->list)))
return;
mutex_lock(&misc_mtx);
list_del(&misc->list);
device_destroy(&misc_class, MKDEV(MISC_MAJOR, misc->minor));
misc_minor_free(misc->minor);
mutex_unlock(&misc_mtx);
}
EXPORT_SYMBOL(misc_deregister);
static int __init misc_init(void)
{
int err;
struct proc_dir_entry *ret;
ret = proc_create_seq("misc", 0, NULL, &misc_seq_ops);
err = class_register(&misc_class);
if (err)
goto fail_remove;
err = -EIO;
if (register_chrdev(MISC_MAJOR, "misc", &misc_fops))
goto fail_printk;
return 0;
fail_printk:
pr_err("unable to get major %d for misc devices\n", MISC_MAJOR);
class_unregister(&misc_class);
fail_remove:
if (ret)
remove_proc_entry("misc", NULL);
return err;
}
subsys_initcall(misc_init);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Linux I2C core
*
* Copyright (C) 1995-99 Simon G. Vogl
* With some changes from Kyösti Mälkki <kmalkki@cc.hut.fi>
* Mux support by Rodolfo Giometti <giometti@enneenne.com> and
* Michael Lawnick <michael.lawnick.ext@nsn.com>
*
* Copyright (C) 2013-2017 Wolfram Sang <wsa@kernel.org>
*/
#define pr_fmt(fmt) "i2c-core: " fmt
#include <dt-bindings/i2c/i2c.h>
#include <linux/acpi.h>
#include <linux/clk/clk-conf.h>
#include <linux/completion.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/gpio/consumer.h>
#include <linux/i2c.h>
#include <linux/i2c-smbus.h>
#include <linux/idr.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/irqflags.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/of_device.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/pinctrl/consumer.h>
#include <linux/pinctrl/devinfo.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/pm_wakeirq.h>
#include <linux/property.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
#include "i2c-core.h"
#define CREATE_TRACE_POINTS
#include <trace/events/i2c.h>
#define I2C_ADDR_OFFSET_TEN_BIT 0xa000
#define I2C_ADDR_OFFSET_SLAVE 0x1000
#define I2C_ADDR_7BITS_MAX 0x77
#define I2C_ADDR_7BITS_COUNT (I2C_ADDR_7BITS_MAX + 1)
#define I2C_ADDR_DEVICE_ID 0x7c
/*
* core_lock protects i2c_adapter_idr, and guarantees that device detection,
* deletion of detected devices are serialized
*/
static DEFINE_MUTEX(core_lock);
static DEFINE_IDR(i2c_adapter_idr);
static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver);
static DEFINE_STATIC_KEY_FALSE(i2c_trace_msg_key);
static bool is_registered;
static struct dentry *i2c_debugfs_root;
int i2c_transfer_trace_reg(void)
{
static_branch_inc(&i2c_trace_msg_key);
return 0;
}
void i2c_transfer_trace_unreg(void)
{
static_branch_dec(&i2c_trace_msg_key);
}
const char *i2c_freq_mode_string(u32 bus_freq_hz)
{
switch (bus_freq_hz) {
case I2C_MAX_STANDARD_MODE_FREQ:
return "Standard Mode (100 kHz)";
case I2C_MAX_FAST_MODE_FREQ:
return "Fast Mode (400 kHz)";
case I2C_MAX_FAST_MODE_PLUS_FREQ:
return "Fast Mode Plus (1.0 MHz)";
case I2C_MAX_TURBO_MODE_FREQ:
return "Turbo Mode (1.4 MHz)";
case I2C_MAX_HIGH_SPEED_MODE_FREQ:
return "High Speed Mode (3.4 MHz)";
case I2C_MAX_ULTRA_FAST_MODE_FREQ:
return "Ultra Fast Mode (5.0 MHz)";
default:
return "Unknown Mode";
}
}
EXPORT_SYMBOL_GPL(i2c_freq_mode_string);
const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
const struct i2c_client *client)
{
if (!(id && client))
return NULL;
while (id->name[0]) {
if (strcmp(client->name, id->name) == 0)
return id;
id++;
}
return NULL;
}
EXPORT_SYMBOL_GPL(i2c_match_id);
const void *i2c_get_match_data(const struct i2c_client *client)
{
struct i2c_driver *driver = to_i2c_driver(client->dev.driver);
const struct i2c_device_id *match;
const void *data;
data = device_get_match_data(&client->dev);
if (!data) {
match = i2c_match_id(driver->id_table, client);
if (!match)
return NULL;
data = (const void *)match->driver_data;
}
return data;
}
EXPORT_SYMBOL(i2c_get_match_data);
static int i2c_device_match(struct device *dev, struct device_driver *drv)
{
struct i2c_client *client = i2c_verify_client(dev);
struct i2c_driver *driver;
/* Attempt an OF style match */
if (i2c_of_match_device(drv->of_match_table, client))
return 1;
/* Then ACPI style match */
if (acpi_driver_match_device(dev, drv))
return 1;
driver = to_i2c_driver(drv);
/* Finally an I2C match */
if (i2c_match_id(driver->id_table, client))
return 1;
return 0;
}
static int i2c_device_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct i2c_client *client = to_i2c_client(dev);
int rc;
rc = of_device_uevent_modalias(dev, env); if (rc != -ENODEV)
return rc;
rc = acpi_device_uevent_modalias(dev, env);
if (rc != -ENODEV)
return rc;
return add_uevent_var(env, "MODALIAS=%s%s", I2C_MODULE_PREFIX, client->name);
}
/* i2c bus recovery routines */
static int get_scl_gpio_value(struct i2c_adapter *adap)
{
return gpiod_get_value_cansleep(adap->bus_recovery_info->scl_gpiod);
}
static void set_scl_gpio_value(struct i2c_adapter *adap, int val)
{
gpiod_set_value_cansleep(adap->bus_recovery_info->scl_gpiod, val);
}
static int get_sda_gpio_value(struct i2c_adapter *adap)
{
return gpiod_get_value_cansleep(adap->bus_recovery_info->sda_gpiod);
}
static void set_sda_gpio_value(struct i2c_adapter *adap, int val)
{
gpiod_set_value_cansleep(adap->bus_recovery_info->sda_gpiod, val);
}
static int i2c_generic_bus_free(struct i2c_adapter *adap)
{
struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
int ret = -EOPNOTSUPP;
if (bri->get_bus_free)
ret = bri->get_bus_free(adap);
else if (bri->get_sda)
ret = bri->get_sda(adap);
if (ret < 0)
return ret;
return ret ? 0 : -EBUSY;
}
/*
* We are generating clock pulses. ndelay() determines durating of clk pulses.
* We will generate clock with rate 100 KHz and so duration of both clock levels
* is: delay in ns = (10^6 / 100) / 2
*/
#define RECOVERY_NDELAY 5000
#define RECOVERY_CLK_CNT 9
int i2c_generic_scl_recovery(struct i2c_adapter *adap)
{
struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
int i = 0, scl = 1, ret = 0;
if (bri->prepare_recovery)
bri->prepare_recovery(adap);
if (bri->pinctrl)
pinctrl_select_state(bri->pinctrl, bri->pins_gpio);
/*
* If we can set SDA, we will always create a STOP to ensure additional
* pulses will do no harm. This is achieved by letting SDA follow SCL
* half a cycle later. Check the 'incomplete_write_byte' fault injector
* for details. Note that we must honour tsu:sto, 4us, but lets use 5us
* here for simplicity.
*/
bri->set_scl(adap, scl);
ndelay(RECOVERY_NDELAY);
if (bri->set_sda)
bri->set_sda(adap, scl);
ndelay(RECOVERY_NDELAY / 2);
/*
* By this time SCL is high, as we need to give 9 falling-rising edges
*/
while (i++ < RECOVERY_CLK_CNT * 2) {
if (scl) {
/* SCL shouldn't be low here */
if (!bri->get_scl(adap)) {
dev_err(&adap->dev,
"SCL is stuck low, exit recovery\n");
ret = -EBUSY;
break;
}
}
scl = !scl;
bri->set_scl(adap, scl);
/* Creating STOP again, see above */
if (scl) {
/* Honour minimum tsu:sto */
ndelay(RECOVERY_NDELAY);
} else {
/* Honour minimum tf and thd:dat */
ndelay(RECOVERY_NDELAY / 2);
}
if (bri->set_sda)
bri->set_sda(adap, scl);
ndelay(RECOVERY_NDELAY / 2);
if (scl) {
ret = i2c_generic_bus_free(adap);
if (ret == 0)
break;
}
}
/* If we can't check bus status, assume recovery worked */
if (ret == -EOPNOTSUPP)
ret = 0;
if (bri->unprepare_recovery)
bri->unprepare_recovery(adap);
if (bri->pinctrl)
pinctrl_select_state(bri->pinctrl, bri->pins_default);
return ret;
}
EXPORT_SYMBOL_GPL(i2c_generic_scl_recovery);
int i2c_recover_bus(struct i2c_adapter *adap)
{
if (!adap->bus_recovery_info)
return -EBUSY;
dev_dbg(&adap->dev, "Trying i2c bus recovery\n");
return adap->bus_recovery_info->recover_bus(adap);
}
EXPORT_SYMBOL_GPL(i2c_recover_bus);
static void i2c_gpio_init_pinctrl_recovery(struct i2c_adapter *adap)
{
struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
struct device *dev = &adap->dev;
struct pinctrl *p = bri->pinctrl ?: dev_pinctrl(dev->parent);
bri->pinctrl = p;
/*
* we can't change states without pinctrl, so remove the states if
* populated
*/
if (!p) {
bri->pins_default = NULL;
bri->pins_gpio = NULL;
return;
}
if (!bri->pins_default) {
bri->pins_default = pinctrl_lookup_state(p,
PINCTRL_STATE_DEFAULT);
if (IS_ERR(bri->pins_default)) {
dev_dbg(dev, PINCTRL_STATE_DEFAULT " state not found for GPIO recovery\n");
bri->pins_default = NULL;
}
}
if (!bri->pins_gpio) {
bri->pins_gpio = pinctrl_lookup_state(p, "gpio");
if (IS_ERR(bri->pins_gpio))
bri->pins_gpio = pinctrl_lookup_state(p, "recovery");
if (IS_ERR(bri->pins_gpio)) {
dev_dbg(dev, "no gpio or recovery state found for GPIO recovery\n");
bri->pins_gpio = NULL;
}
}
/* for pinctrl state changes, we need all the information */
if (bri->pins_default && bri->pins_gpio) {
dev_info(dev, "using pinctrl states for GPIO recovery");
} else {
bri->pinctrl = NULL;
bri->pins_default = NULL;
bri->pins_gpio = NULL;
}
}
static int i2c_gpio_init_generic_recovery(struct i2c_adapter *adap)
{
struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
struct device *dev = &adap->dev;
struct gpio_desc *gpiod;
int ret = 0;
/*
* don't touch the recovery information if the driver is not using
* generic SCL recovery
*/
if (bri->recover_bus && bri->recover_bus != i2c_generic_scl_recovery)
return 0;
/*
* pins might be taken as GPIO, so we should inform pinctrl about
* this and move the state to GPIO
*/
if (bri->pinctrl)
pinctrl_select_state(bri->pinctrl, bri->pins_gpio);
/*
* if there is incomplete or no recovery information, see if generic
* GPIO recovery is available
*/
if (!bri->scl_gpiod) {
gpiod = devm_gpiod_get(dev, "scl", GPIOD_OUT_HIGH_OPEN_DRAIN);
if (PTR_ERR(gpiod) == -EPROBE_DEFER) {
ret = -EPROBE_DEFER;
goto cleanup_pinctrl_state;
}
if (!IS_ERR(gpiod)) {
bri->scl_gpiod = gpiod;
bri->recover_bus = i2c_generic_scl_recovery;
dev_info(dev, "using generic GPIOs for recovery\n");
}
}
/* SDA GPIOD line is optional, so we care about DEFER only */
if (!bri->sda_gpiod) {
/*
* We have SCL. Pull SCL low and wait a bit so that SDA glitches
* have no effect.
*/
gpiod_direction_output(bri->scl_gpiod, 0);
udelay(10);
gpiod = devm_gpiod_get(dev, "sda", GPIOD_IN);
/* Wait a bit in case of a SDA glitch, and then release SCL. */
udelay(10);
gpiod_direction_output(bri->scl_gpiod, 1);
if (PTR_ERR(gpiod) == -EPROBE_DEFER) {
ret = -EPROBE_DEFER;
goto cleanup_pinctrl_state;
}
if (!IS_ERR(gpiod))
bri->sda_gpiod = gpiod;
}
cleanup_pinctrl_state:
/* change the state of the pins back to their default state */
if (bri->pinctrl)
pinctrl_select_state(bri->pinctrl, bri->pins_default);
return ret;
}
static int i2c_gpio_init_recovery(struct i2c_adapter *adap)
{
i2c_gpio_init_pinctrl_recovery(adap);
return i2c_gpio_init_generic_recovery(adap);
}
static int i2c_init_recovery(struct i2c_adapter *adap)
{
struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
bool is_error_level = true;
char *err_str;
if (!bri)
return 0;
if (i2c_gpio_init_recovery(adap) == -EPROBE_DEFER)
return -EPROBE_DEFER;
if (!bri->recover_bus) {
err_str = "no suitable method provided";
is_error_level = false;
goto err;
}
if (bri->scl_gpiod && bri->recover_bus == i2c_generic_scl_recovery) {
bri->get_scl = get_scl_gpio_value;
bri->set_scl = set_scl_gpio_value;
if (bri->sda_gpiod) {
bri->get_sda = get_sda_gpio_value;
/* FIXME: add proper flag instead of '0' once available */
if (gpiod_get_direction(bri->sda_gpiod) == 0)
bri->set_sda = set_sda_gpio_value;
}
} else if (bri->recover_bus == i2c_generic_scl_recovery) {
/* Generic SCL recovery */
if (!bri->set_scl || !bri->get_scl) {
err_str = "no {get|set}_scl() found";
goto err;
}
if (!bri->set_sda && !bri->get_sda) {
err_str = "either get_sda() or set_sda() needed";
goto err;
}
}
return 0;
err:
if (is_error_level)
dev_err(&adap->dev, "Not using recovery: %s\n", err_str);
else
dev_dbg(&adap->dev, "Not using recovery: %s\n", err_str);
adap->bus_recovery_info = NULL;
return -EINVAL;
}
static int i2c_smbus_host_notify_to_irq(const struct i2c_client *client)
{
struct i2c_adapter *adap = client->adapter;
unsigned int irq;
if (!adap->host_notify_domain)
return -ENXIO;
if (client->flags & I2C_CLIENT_TEN)
return -EINVAL;
irq = irq_create_mapping(adap->host_notify_domain, client->addr);
return irq > 0 ? irq : -ENXIO;
}
static int i2c_device_probe(struct device *dev)
{
struct i2c_client *client = i2c_verify_client(dev);
struct i2c_driver *driver;
bool do_power_on;
int status;
if (!client)
return 0;
client->irq = client->init_irq;
if (!client->irq) {
int irq = -ENOENT;
if (client->flags & I2C_CLIENT_HOST_NOTIFY) {
dev_dbg(dev, "Using Host Notify IRQ\n");
/* Keep adapter active when Host Notify is required */
pm_runtime_get_sync(&client->adapter->dev);
irq = i2c_smbus_host_notify_to_irq(client);
} else if (dev->of_node) {
irq = of_irq_get_byname(dev->of_node, "irq");
if (irq == -EINVAL || irq == -ENODATA)
irq = of_irq_get(dev->of_node, 0);
} else if (ACPI_COMPANION(dev)) {
bool wake_capable;
irq = i2c_acpi_get_irq(client, &wake_capable);
if (irq > 0 && wake_capable)
client->flags |= I2C_CLIENT_WAKE;
}
if (irq == -EPROBE_DEFER) {
status = irq;
goto put_sync_adapter;
}
if (irq < 0)
irq = 0;
client->irq = irq;
}
driver = to_i2c_driver(dev->driver);
/*
* An I2C ID table is not mandatory, if and only if, a suitable OF
* or ACPI ID table is supplied for the probing device.
*/
if (!driver->id_table &&
!acpi_driver_match_device(dev, dev->driver) &&
!i2c_of_match_device(dev->driver->of_match_table, client)) {
status = -ENODEV;
goto put_sync_adapter;
}
if (client->flags & I2C_CLIENT_WAKE) {
int wakeirq;
wakeirq = of_irq_get_byname(dev->of_node, "wakeup");
if (wakeirq == -EPROBE_DEFER) {
status = wakeirq;
goto put_sync_adapter;
}
device_init_wakeup(&client->dev, true);
if (wakeirq > 0 && wakeirq != client->irq)
status = dev_pm_set_dedicated_wake_irq(dev, wakeirq);
else if (client->irq > 0)
status = dev_pm_set_wake_irq(dev, client->irq);
else
status = 0;
if (status)
dev_warn(&client->dev, "failed to set up wakeup irq\n");
}
dev_dbg(dev, "probe\n");
status = of_clk_set_defaults(dev->of_node, false);
if (status < 0)
goto err_clear_wakeup_irq;
do_power_on = !i2c_acpi_waive_d0_probe(dev);
status = dev_pm_domain_attach(&client->dev, do_power_on);
if (status)
goto err_clear_wakeup_irq;
client->devres_group_id = devres_open_group(&client->dev, NULL,
GFP_KERNEL);
if (!client->devres_group_id) {
status = -ENOMEM;
goto err_detach_pm_domain;
}
if (driver->probe)
status = driver->probe(client);
else
status = -EINVAL;
/*
* Note that we are not closing the devres group opened above so
* even resources that were attached to the device after probe is
* run are released when i2c_device_remove() is executed. This is
* needed as some drivers would allocate additional resources,
* for example when updating firmware.
*/
if (status)
goto err_release_driver_resources;
return 0;
err_release_driver_resources:
devres_release_group(&client->dev, client->devres_group_id);
err_detach_pm_domain:
dev_pm_domain_detach(&client->dev, do_power_on);
err_clear_wakeup_irq:
dev_pm_clear_wake_irq(&client->dev);
device_init_wakeup(&client->dev, false);
put_sync_adapter:
if (client->flags & I2C_CLIENT_HOST_NOTIFY)
pm_runtime_put_sync(&client->adapter->dev);
return status;
}
static void i2c_device_remove(struct device *dev)
{
struct i2c_client *client = to_i2c_client(dev);
struct i2c_driver *driver;
driver = to_i2c_driver(dev->driver);
if (driver->remove) {
dev_dbg(dev, "remove\n");
driver->remove(client);
}
devres_release_group(&client->dev, client->devres_group_id);
dev_pm_domain_detach(&client->dev, true);
dev_pm_clear_wake_irq(&client->dev);
device_init_wakeup(&client->dev, false);
client->irq = 0;
if (client->flags & I2C_CLIENT_HOST_NOTIFY)
pm_runtime_put(&client->adapter->dev);
}
static void i2c_device_shutdown(struct device *dev)
{
struct i2c_client *client = i2c_verify_client(dev);
struct i2c_driver *driver;
if (!client || !dev->driver)
return;
driver = to_i2c_driver(dev->driver);
if (driver->shutdown)
driver->shutdown(client);
else if (client->irq > 0)
disable_irq(client->irq);
}
static void i2c_client_dev_release(struct device *dev)
{
kfree(to_i2c_client(dev));
}
static ssize_t
name_show(struct device *dev, struct device_attribute *attr, char *buf)
{
return sprintf(buf, "%s\n", dev->type == &i2c_client_type ?
to_i2c_client(dev)->name : to_i2c_adapter(dev)->name);
}
static DEVICE_ATTR_RO(name);
static ssize_t
modalias_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct i2c_client *client = to_i2c_client(dev);
int len;
len = of_device_modalias(dev, buf, PAGE_SIZE);
if (len != -ENODEV)
return len;
len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1);
if (len != -ENODEV)
return len;
return sprintf(buf, "%s%s\n", I2C_MODULE_PREFIX, client->name);
}
static DEVICE_ATTR_RO(modalias);
static struct attribute *i2c_dev_attrs[] = {
&dev_attr_name.attr,
/* modalias helps coldplug: modprobe $(cat .../modalias) */
&dev_attr_modalias.attr,
NULL
};
ATTRIBUTE_GROUPS(i2c_dev);
const struct bus_type i2c_bus_type = {
.name = "i2c",
.match = i2c_device_match,
.probe = i2c_device_probe,
.remove = i2c_device_remove,
.shutdown = i2c_device_shutdown,
};
EXPORT_SYMBOL_GPL(i2c_bus_type);
const struct device_type i2c_client_type = {
.groups = i2c_dev_groups,
.uevent = i2c_device_uevent,
.release = i2c_client_dev_release,
};
EXPORT_SYMBOL_GPL(i2c_client_type);
/**
* i2c_verify_client - return parameter as i2c_client, or NULL
* @dev: device, probably from some driver model iterator
*
* When traversing the driver model tree, perhaps using driver model
* iterators like @device_for_each_child(), you can't assume very much
* about the nodes you find. Use this function to avoid oopses caused
* by wrongly treating some non-I2C device as an i2c_client.
*/
struct i2c_client *i2c_verify_client(struct device *dev)
{
return (dev->type == &i2c_client_type) ? to_i2c_client(dev)
: NULL;
}
EXPORT_SYMBOL(i2c_verify_client);
/* Return a unique address which takes the flags of the client into account */
static unsigned short i2c_encode_flags_to_addr(struct i2c_client *client)
{
unsigned short addr = client->addr;
/* For some client flags, add an arbitrary offset to avoid collisions */
if (client->flags & I2C_CLIENT_TEN)
addr |= I2C_ADDR_OFFSET_TEN_BIT;
if (client->flags & I2C_CLIENT_SLAVE)
addr |= I2C_ADDR_OFFSET_SLAVE;
return addr;
}
/* This is a permissive address validity check, I2C address map constraints
* are purposely not enforced, except for the general call address. */
static int i2c_check_addr_validity(unsigned int addr, unsigned short flags)
{
if (flags & I2C_CLIENT_TEN) {
/* 10-bit address, all values are valid */
if (addr > 0x3ff)
return -EINVAL;
} else {
/* 7-bit address, reject the general call address */
if (addr == 0x00 || addr > 0x7f)
return -EINVAL;
}
return 0;
}
/* And this is a strict address validity check, used when probing. If a
* device uses a reserved address, then it shouldn't be probed. 7-bit
* addressing is assumed, 10-bit address devices are rare and should be
* explicitly enumerated. */
int i2c_check_7bit_addr_validity_strict(unsigned short addr)
{
/*
* Reserved addresses per I2C specification:
* 0x00 General call address / START byte
* 0x01 CBUS address
* 0x02 Reserved for different bus format
* 0x03 Reserved for future purposes
* 0x04-0x07 Hs-mode master code
* 0x78-0x7b 10-bit slave addressing
* 0x7c-0x7f Reserved for future purposes
*/
if (addr < 0x08 || addr > 0x77)
return -EINVAL;
return 0;
}
static int __i2c_check_addr_busy(struct device *dev, void *addrp)
{
struct i2c_client *client = i2c_verify_client(dev);
int addr = *(int *)addrp;
if (client && i2c_encode_flags_to_addr(client) == addr)
return -EBUSY;
return 0;
}
/* walk up mux tree */
static int i2c_check_mux_parents(struct i2c_adapter *adapter, int addr)
{
struct i2c_adapter *parent = i2c_parent_is_i2c_adapter(adapter);
int result;
result = device_for_each_child(&adapter->dev, &addr,
__i2c_check_addr_busy);
if (!result && parent)
result = i2c_check_mux_parents(parent, addr);
return result;
}
/* recurse down mux tree */
static int i2c_check_mux_children(struct device *dev, void *addrp)
{
int result;
if (dev->type == &i2c_adapter_type)
result = device_for_each_child(dev, addrp,
i2c_check_mux_children);
else
result = __i2c_check_addr_busy(dev, addrp);
return result;
}
static int i2c_check_addr_busy(struct i2c_adapter *adapter, int addr)
{
struct i2c_adapter *parent = i2c_parent_is_i2c_adapter(adapter);
int result = 0;
if (parent)
result = i2c_check_mux_parents(parent, addr);
if (!result)
result = device_for_each_child(&adapter->dev, &addr,
i2c_check_mux_children);
return result;
}
/**
* i2c_adapter_lock_bus - Get exclusive access to an I2C bus segment
* @adapter: Target I2C bus segment
* @flags: I2C_LOCK_ROOT_ADAPTER locks the root i2c adapter, I2C_LOCK_SEGMENT
* locks only this branch in the adapter tree
*/
static void i2c_adapter_lock_bus(struct i2c_adapter *adapter,
unsigned int flags)
{
rt_mutex_lock_nested(&adapter->bus_lock, i2c_adapter_depth(adapter));
}
/**
* i2c_adapter_trylock_bus - Try to get exclusive access to an I2C bus segment
* @adapter: Target I2C bus segment
* @flags: I2C_LOCK_ROOT_ADAPTER trylocks the root i2c adapter, I2C_LOCK_SEGMENT
* trylocks only this branch in the adapter tree
*/
static int i2c_adapter_trylock_bus(struct i2c_adapter *adapter,
unsigned int flags)
{
return rt_mutex_trylock(&adapter->bus_lock);
}
/**
* i2c_adapter_unlock_bus - Release exclusive access to an I2C bus segment
* @adapter: Target I2C bus segment
* @flags: I2C_LOCK_ROOT_ADAPTER unlocks the root i2c adapter, I2C_LOCK_SEGMENT
* unlocks only this branch in the adapter tree
*/
static void i2c_adapter_unlock_bus(struct i2c_adapter *adapter,
unsigned int flags)
{
rt_mutex_unlock(&adapter->bus_lock);
}
static void i2c_dev_set_name(struct i2c_adapter *adap,
struct i2c_client *client,
struct i2c_board_info const *info)
{
struct acpi_device *adev = ACPI_COMPANION(&client->dev);
if (info && info->dev_name) {
dev_set_name(&client->dev, "i2c-%s", info->dev_name);
return;
}
if (adev) {
dev_set_name(&client->dev, "i2c-%s", acpi_dev_name(adev));
return;
}
dev_set_name(&client->dev, "%d-%04x", i2c_adapter_id(adap),
i2c_encode_flags_to_addr(client));
}
int i2c_dev_irq_from_resources(const struct resource *resources,
unsigned int num_resources)
{
struct irq_data *irqd;
int i;
for (i = 0; i < num_resources; i++) {
const struct resource *r = &resources[i];
if (resource_type(r) != IORESOURCE_IRQ)
continue;
if (r->flags & IORESOURCE_BITS) {
irqd = irq_get_irq_data(r->start);
if (!irqd)
break;
irqd_set_trigger_type(irqd, r->flags & IORESOURCE_BITS);
}
return r->start;
}
return 0;
}
/**
* i2c_new_client_device - instantiate an i2c device
* @adap: the adapter managing the device
* @info: describes one I2C device; bus_num is ignored
* Context: can sleep
*
* Create an i2c device. Binding is handled through driver model
* probe()/remove() methods. A driver may be bound to this device when we
* return from this function, or any later moment (e.g. maybe hotplugging will
* load the driver module). This call is not appropriate for use by mainboard
* initialization logic, which usually runs during an arch_initcall() long
* before any i2c_adapter could exist.
*
* This returns the new i2c client, which may be saved for later use with
* i2c_unregister_device(); or an ERR_PTR to describe the error.
*/
struct i2c_client *
i2c_new_client_device(struct i2c_adapter *adap, struct i2c_board_info const *info)
{
struct i2c_client *client;
bool need_put = false;
int status;
client = kzalloc(sizeof *client, GFP_KERNEL);
if (!client)
return ERR_PTR(-ENOMEM);
client->adapter = adap;
client->dev.platform_data = info->platform_data;
client->flags = info->flags;
client->addr = info->addr;
client->init_irq = info->irq;
if (!client->init_irq)
client->init_irq = i2c_dev_irq_from_resources(info->resources,
info->num_resources);
strscpy(client->name, info->type, sizeof(client->name));
status = i2c_check_addr_validity(client->addr, client->flags);
if (status) {
dev_err(&adap->dev, "Invalid %d-bit I2C address 0x%02hx\n",
client->flags & I2C_CLIENT_TEN ? 10 : 7, client->addr);
goto out_err_silent;
}
/* Check for address business */
status = i2c_check_addr_busy(adap, i2c_encode_flags_to_addr(client));
if (status)
goto out_err;
client->dev.parent = &client->adapter->dev;
client->dev.bus = &i2c_bus_type;
client->dev.type = &i2c_client_type;
client->dev.of_node = of_node_get(info->of_node);
client->dev.fwnode = info->fwnode;
device_enable_async_suspend(&client->dev);
if (info->swnode) {
status = device_add_software_node(&client->dev, info->swnode);
if (status) {
dev_err(&adap->dev,
"Failed to add software node to client %s: %d\n",
client->name, status);
goto out_err_put_of_node;
}
}
i2c_dev_set_name(adap, client, info);
status = device_register(&client->dev);
if (status)
goto out_remove_swnode;
dev_dbg(&adap->dev, "client [%s] registered with bus id %s\n",
client->name, dev_name(&client->dev));
return client;
out_remove_swnode:
device_remove_software_node(&client->dev);
need_put = true;
out_err_put_of_node:
of_node_put(info->of_node);
out_err:
dev_err(&adap->dev,
"Failed to register i2c client %s at 0x%02x (%d)\n",
client->name, client->addr, status);
out_err_silent:
if (need_put)
put_device(&client->dev);
else
kfree(client);
return ERR_PTR(status);
}
EXPORT_SYMBOL_GPL(i2c_new_client_device);
/**
* i2c_unregister_device - reverse effect of i2c_new_*_device()
* @client: value returned from i2c_new_*_device()
* Context: can sleep
*/
void i2c_unregister_device(struct i2c_client *client)
{
if (IS_ERR_OR_NULL(client))
return;
if (client->dev.of_node) { of_node_clear_flag(client->dev.of_node, OF_POPULATED);
of_node_put(client->dev.of_node);
}
if (ACPI_COMPANION(&client->dev)) acpi_device_clear_enumerated(ACPI_COMPANION(&client->dev)); device_remove_software_node(&client->dev);
device_unregister(&client->dev);
}
EXPORT_SYMBOL_GPL(i2c_unregister_device);
/**
* i2c_find_device_by_fwnode() - find an i2c_client for the fwnode
* @fwnode: &struct fwnode_handle corresponding to the &struct i2c_client
*
* Look up and return the &struct i2c_client corresponding to the @fwnode.
* If no client can be found, or @fwnode is NULL, this returns NULL.
*
* The user must call put_device(&client->dev) once done with the i2c client.
*/
struct i2c_client *i2c_find_device_by_fwnode(struct fwnode_handle *fwnode)
{
struct i2c_client *client;
struct device *dev;
if (!fwnode)
return NULL;
dev = bus_find_device_by_fwnode(&i2c_bus_type, fwnode);
if (!dev)
return NULL;
client = i2c_verify_client(dev);
if (!client)
put_device(dev);
return client;
}
EXPORT_SYMBOL(i2c_find_device_by_fwnode);
static const struct i2c_device_id dummy_id[] = {
{ "dummy", 0 },
{ },
};
static int dummy_probe(struct i2c_client *client)
{
return 0;
}
static struct i2c_driver dummy_driver = {
.driver.name = "dummy",
.probe = dummy_probe,
.id_table = dummy_id,
};
/**
* i2c_new_dummy_device - return a new i2c device bound to a dummy driver
* @adapter: the adapter managing the device
* @address: seven bit address to be used
* Context: can sleep
*
* This returns an I2C client bound to the "dummy" driver, intended for use
* with devices that consume multiple addresses. Examples of such chips
* include various EEPROMS (like 24c04 and 24c08 models).
*
* These dummy devices have two main uses. First, most I2C and SMBus calls
* except i2c_transfer() need a client handle; the dummy will be that handle.
* And second, this prevents the specified address from being bound to a
* different driver.
*
* This returns the new i2c client, which should be saved for later use with
* i2c_unregister_device(); or an ERR_PTR to describe the error.
*/
struct i2c_client *i2c_new_dummy_device(struct i2c_adapter *adapter, u16 address)
{
struct i2c_board_info info = {
I2C_BOARD_INFO("dummy", address),
};
return i2c_new_client_device(adapter, &info);
}
EXPORT_SYMBOL_GPL(i2c_new_dummy_device);
static void devm_i2c_release_dummy(void *client)
{
i2c_unregister_device(client);
}
/**
* devm_i2c_new_dummy_device - return a new i2c device bound to a dummy driver
* @dev: device the managed resource is bound to
* @adapter: the adapter managing the device
* @address: seven bit address to be used
* Context: can sleep
*
* This is the device-managed version of @i2c_new_dummy_device. It returns the
* new i2c client or an ERR_PTR in case of an error.
*/
struct i2c_client *devm_i2c_new_dummy_device(struct device *dev,
struct i2c_adapter *adapter,
u16 address)
{
struct i2c_client *client;
int ret;
client = i2c_new_dummy_device(adapter, address);
if (IS_ERR(client))
return client;
ret = devm_add_action_or_reset(dev, devm_i2c_release_dummy, client);
if (ret)
return ERR_PTR(ret);
return client;
}
EXPORT_SYMBOL_GPL(devm_i2c_new_dummy_device);
/**
* i2c_new_ancillary_device - Helper to get the instantiated secondary address
* and create the associated device
* @client: Handle to the primary client
* @name: Handle to specify which secondary address to get
* @default_addr: Used as a fallback if no secondary address was specified
* Context: can sleep
*
* I2C clients can be composed of multiple I2C slaves bound together in a single
* component. The I2C client driver then binds to the master I2C slave and needs
* to create I2C dummy clients to communicate with all the other slaves.
*
* This function creates and returns an I2C dummy client whose I2C address is
* retrieved from the platform firmware based on the given slave name. If no
* address is specified by the firmware default_addr is used.
*
* On DT-based platforms the address is retrieved from the "reg" property entry
* cell whose "reg-names" value matches the slave name.
*
* This returns the new i2c client, which should be saved for later use with
* i2c_unregister_device(); or an ERR_PTR to describe the error.
*/
struct i2c_client *i2c_new_ancillary_device(struct i2c_client *client,
const char *name,
u16 default_addr)
{
struct device_node *np = client->dev.of_node;
u32 addr = default_addr;
int i;
if (np) {
i = of_property_match_string(np, "reg-names", name);
if (i >= 0)
of_property_read_u32_index(np, "reg", i, &addr);
}
dev_dbg(&client->adapter->dev, "Address for %s : 0x%x\n", name, addr);
return i2c_new_dummy_device(client->adapter, addr);
}
EXPORT_SYMBOL_GPL(i2c_new_ancillary_device);
/* ------------------------------------------------------------------------- */
/* I2C bus adapters -- one roots each I2C or SMBUS segment */
static void i2c_adapter_dev_release(struct device *dev)
{
struct i2c_adapter *adap = to_i2c_adapter(dev);
complete(&adap->dev_released);
}
unsigned int i2c_adapter_depth(struct i2c_adapter *adapter)
{
unsigned int depth = 0;
struct device *parent;
for (parent = adapter->dev.parent; parent; parent = parent->parent) if (parent->type == &i2c_adapter_type) depth++; WARN_ONCE(depth >= MAX_LOCKDEP_SUBCLASSES,
"adapter depth exceeds lockdep subclass limit\n");
return depth;
}
EXPORT_SYMBOL_GPL(i2c_adapter_depth);
/*
* Let users instantiate I2C devices through sysfs. This can be used when
* platform initialization code doesn't contain the proper data for
* whatever reason. Also useful for drivers that do device detection and
* detection fails, either because the device uses an unexpected address,
* or this is a compatible device with different ID register values.
*
* Parameter checking may look overzealous, but we really don't want
* the user to provide incorrect parameters.
*/
static ssize_t
new_device_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct i2c_adapter *adap = to_i2c_adapter(dev);
struct i2c_board_info info;
struct i2c_client *client;
char *blank, end;
int res;
memset(&info, 0, sizeof(struct i2c_board_info));
blank = strchr(buf, ' ');
if (!blank) {
dev_err(dev, "%s: Missing parameters\n", "new_device");
return -EINVAL;
}
if (blank - buf > I2C_NAME_SIZE - 1) {
dev_err(dev, "%s: Invalid device name\n", "new_device");
return -EINVAL;
}
memcpy(info.type, buf, blank - buf);
/* Parse remaining parameters, reject extra parameters */
res = sscanf(++blank, "%hi%c", &info.addr, &end);
if (res < 1) {
dev_err(dev, "%s: Can't parse I2C address\n", "new_device");
return -EINVAL;
}
if (res > 1 && end != '\n') {
dev_err(dev, "%s: Extra parameters\n", "new_device");
return -EINVAL;
}
if ((info.addr & I2C_ADDR_OFFSET_TEN_BIT) == I2C_ADDR_OFFSET_TEN_BIT) {
info.addr &= ~I2C_ADDR_OFFSET_TEN_BIT;
info.flags |= I2C_CLIENT_TEN;
}
if (info.addr & I2C_ADDR_OFFSET_SLAVE) {
info.addr &= ~I2C_ADDR_OFFSET_SLAVE;
info.flags |= I2C_CLIENT_SLAVE;
}
client = i2c_new_client_device(adap, &info);
if (IS_ERR(client))
return PTR_ERR(client);
/* Keep track of the added device */
mutex_lock(&adap->userspace_clients_lock);
list_add_tail(&client->detected, &adap->userspace_clients);
mutex_unlock(&adap->userspace_clients_lock);
dev_info(dev, "%s: Instantiated device %s at 0x%02hx\n", "new_device",
info.type, info.addr);
return count;
}
static DEVICE_ATTR_WO(new_device);
/*
* And of course let the users delete the devices they instantiated, if
* they got it wrong. This interface can only be used to delete devices
* instantiated by i2c_sysfs_new_device above. This guarantees that we
* don't delete devices to which some kernel code still has references.
*
* Parameter checking may look overzealous, but we really don't want
* the user to delete the wrong device.
*/
static ssize_t
delete_device_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct i2c_adapter *adap = to_i2c_adapter(dev);
struct i2c_client *client, *next;
unsigned short addr;
char end;
int res;
/* Parse parameters, reject extra parameters */
res = sscanf(buf, "%hi%c", &addr, &end);
if (res < 1) {
dev_err(dev, "%s: Can't parse I2C address\n", "delete_device");
return -EINVAL;
}
if (res > 1 && end != '\n') {
dev_err(dev, "%s: Extra parameters\n", "delete_device");
return -EINVAL;
}
/* Make sure the device was added through sysfs */
res = -ENOENT;
mutex_lock_nested(&adap->userspace_clients_lock,
i2c_adapter_depth(adap));
list_for_each_entry_safe(client, next, &adap->userspace_clients,
detected) {
if (i2c_encode_flags_to_addr(client) == addr) {
dev_info(dev, "%s: Deleting device %s at 0x%02hx\n",
"delete_device", client->name, client->addr);
list_del(&client->detected);
i2c_unregister_device(client);
res = count;
break;
}
}
mutex_unlock(&adap->userspace_clients_lock);
if (res < 0)
dev_err(dev, "%s: Can't find device in list\n",
"delete_device");
return res;
}
static DEVICE_ATTR_IGNORE_LOCKDEP(delete_device, S_IWUSR, NULL,
delete_device_store);
static struct attribute *i2c_adapter_attrs[] = {
&dev_attr_name.attr,
&dev_attr_new_device.attr,
&dev_attr_delete_device.attr,
NULL
};
ATTRIBUTE_GROUPS(i2c_adapter);
const struct device_type i2c_adapter_type = {
.groups = i2c_adapter_groups,
.release = i2c_adapter_dev_release,
};
EXPORT_SYMBOL_GPL(i2c_adapter_type);
/**
* i2c_verify_adapter - return parameter as i2c_adapter or NULL
* @dev: device, probably from some driver model iterator
*
* When traversing the driver model tree, perhaps using driver model
* iterators like @device_for_each_child(), you can't assume very much
* about the nodes you find. Use this function to avoid oopses caused
* by wrongly treating some non-I2C device as an i2c_adapter.
*/
struct i2c_adapter *i2c_verify_adapter(struct device *dev)
{
return (dev->type == &i2c_adapter_type)
? to_i2c_adapter(dev)
: NULL;
}
EXPORT_SYMBOL(i2c_verify_adapter);
#ifdef CONFIG_I2C_COMPAT
static struct class_compat *i2c_adapter_compat_class;
#endif
static void i2c_scan_static_board_info(struct i2c_adapter *adapter)
{
struct i2c_devinfo *devinfo;
down_read(&__i2c_board_lock);
list_for_each_entry(devinfo, &__i2c_board_list, list) {
if (devinfo->busnum == adapter->nr &&
IS_ERR(i2c_new_client_device(adapter, &devinfo->board_info)))
dev_err(&adapter->dev,
"Can't create device at 0x%02x\n",
devinfo->board_info.addr);
}
up_read(&__i2c_board_lock);
}
static int i2c_do_add_adapter(struct i2c_driver *driver,
struct i2c_adapter *adap)
{
/* Detect supported devices on that bus, and instantiate them */
i2c_detect(adap, driver);
return 0;
}
static int __process_new_adapter(struct device_driver *d, void *data)
{
return i2c_do_add_adapter(to_i2c_driver(d), data);
}
static const struct i2c_lock_operations i2c_adapter_lock_ops = {
.lock_bus = i2c_adapter_lock_bus,
.trylock_bus = i2c_adapter_trylock_bus,
.unlock_bus = i2c_adapter_unlock_bus,
};
static void i2c_host_notify_irq_teardown(struct i2c_adapter *adap)
{
struct irq_domain *domain = adap->host_notify_domain;
irq_hw_number_t hwirq;
if (!domain)
return;
for (hwirq = 0 ; hwirq < I2C_ADDR_7BITS_COUNT ; hwirq++)
irq_dispose_mapping(irq_find_mapping(domain, hwirq)); irq_domain_remove(domain);
adap->host_notify_domain = NULL;
}
static int i2c_host_notify_irq_map(struct irq_domain *h,
unsigned int virq,
irq_hw_number_t hw_irq_num)
{
irq_set_chip_and_handler(virq, &dummy_irq_chip, handle_simple_irq);
return 0;
}
static const struct irq_domain_ops i2c_host_notify_irq_ops = {
.map = i2c_host_notify_irq_map,
};
static int i2c_setup_host_notify_irq_domain(struct i2c_adapter *adap)
{
struct irq_domain *domain;
if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_HOST_NOTIFY))
return 0;
domain = irq_domain_create_linear(adap->dev.parent->fwnode,
I2C_ADDR_7BITS_COUNT,
&i2c_host_notify_irq_ops, adap);
if (!domain)
return -ENOMEM;
adap->host_notify_domain = domain;
return 0;
}
/**
* i2c_handle_smbus_host_notify - Forward a Host Notify event to the correct
* I2C client.
* @adap: the adapter
* @addr: the I2C address of the notifying device
* Context: can't sleep
*
* Helper function to be called from an I2C bus driver's interrupt
* handler. It will schedule the Host Notify IRQ.
*/
int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr)
{
int irq;
if (!adap)
return -EINVAL;
irq = irq_find_mapping(adap->host_notify_domain, addr);
if (irq <= 0)
return -ENXIO;
generic_handle_irq_safe(irq);
return 0;
}
EXPORT_SYMBOL_GPL(i2c_handle_smbus_host_notify);
static int i2c_register_adapter(struct i2c_adapter *adap)
{
int res = -EINVAL;
/* Can't register until after driver model init */
if (WARN_ON(!is_registered)) {
res = -EAGAIN;
goto out_list;
}
/* Sanity checks */
if (WARN(!adap->name[0], "i2c adapter has no name"))
goto out_list;
if (!adap->algo) {
pr_err("adapter '%s': no algo supplied!\n", adap->name);
goto out_list;
}
if (!adap->lock_ops)
adap->lock_ops = &i2c_adapter_lock_ops;
adap->locked_flags = 0;
rt_mutex_init(&adap->bus_lock);
rt_mutex_init(&adap->mux_lock);
mutex_init(&adap->userspace_clients_lock);
INIT_LIST_HEAD(&adap->userspace_clients);
/* Set default timeout to 1 second if not already set */
if (adap->timeout == 0)
adap->timeout = HZ;
/* register soft irqs for Host Notify */
res = i2c_setup_host_notify_irq_domain(adap);
if (res) {
pr_err("adapter '%s': can't create Host Notify IRQs (%d)\n",
adap->name, res);
goto out_list;
}
dev_set_name(&adap->dev, "i2c-%d", adap->nr);
adap->dev.bus = &i2c_bus_type;
adap->dev.type = &i2c_adapter_type;
res = device_register(&adap->dev);
if (res) {
pr_err("adapter '%s': can't register device (%d)\n", adap->name, res);
goto out_list;
}
adap->debugfs = debugfs_create_dir(dev_name(&adap->dev), i2c_debugfs_root);
res = i2c_setup_smbus_alert(adap);
if (res)
goto out_reg;
device_enable_async_suspend(&adap->dev);
pm_runtime_no_callbacks(&adap->dev);
pm_suspend_ignore_children(&adap->dev, true);
pm_runtime_enable(&adap->dev);
res = i2c_init_recovery(adap);
if (res == -EPROBE_DEFER)
goto out_reg;
dev_dbg(&adap->dev, "adapter [%s] registered\n", adap->name);
#ifdef CONFIG_I2C_COMPAT
res = class_compat_create_link(i2c_adapter_compat_class, &adap->dev,
adap->dev.parent);
if (res)
dev_warn(&adap->dev,
"Failed to create compatibility class link\n");
#endif
/* create pre-declared device nodes */
of_i2c_register_devices(adap);
i2c_acpi_install_space_handler(adap);
i2c_acpi_register_devices(adap);
if (adap->nr < __i2c_first_dynamic_bus_num)
i2c_scan_static_board_info(adap);
/* Notify drivers */
mutex_lock(&core_lock);
bus_for_each_drv(&i2c_bus_type, NULL, adap, __process_new_adapter);
mutex_unlock(&core_lock);
return 0;
out_reg:
debugfs_remove_recursive(adap->debugfs);
init_completion(&adap->dev_released);
device_unregister(&adap->dev);
wait_for_completion(&adap->dev_released);
out_list:
mutex_lock(&core_lock);
idr_remove(&i2c_adapter_idr, adap->nr);
mutex_unlock(&core_lock);
return res;
}
/**
* __i2c_add_numbered_adapter - i2c_add_numbered_adapter where nr is never -1
* @adap: the adapter to register (with adap->nr initialized)
* Context: can sleep
*
* See i2c_add_numbered_adapter() for details.
*/
static int __i2c_add_numbered_adapter(struct i2c_adapter *adap)
{
int id;
mutex_lock(&core_lock);
id = idr_alloc(&i2c_adapter_idr, adap, adap->nr, adap->nr + 1, GFP_KERNEL);
mutex_unlock(&core_lock);
if (WARN(id < 0, "couldn't get idr"))
return id == -ENOSPC ? -EBUSY : id;
return i2c_register_adapter(adap);
}
/**
* i2c_add_adapter - declare i2c adapter, use dynamic bus number
* @adapter: the adapter to add
* Context: can sleep
*
* This routine is used to declare an I2C adapter when its bus number
* doesn't matter or when its bus number is specified by an dt alias.
* Examples of bases when the bus number doesn't matter: I2C adapters
* dynamically added by USB links or PCI plugin cards.
*
* When this returns zero, a new bus number was allocated and stored
* in adap->nr, and the specified adapter became available for clients.
* Otherwise, a negative errno value is returned.
*/
int i2c_add_adapter(struct i2c_adapter *adapter)
{
struct device *dev = &adapter->dev;
int id;
if (dev->of_node) {
id = of_alias_get_id(dev->of_node, "i2c");
if (id >= 0) {
adapter->nr = id;
return __i2c_add_numbered_adapter(adapter);
}
}
mutex_lock(&core_lock);
id = idr_alloc(&i2c_adapter_idr, adapter,
__i2c_first_dynamic_bus_num, 0, GFP_KERNEL);
mutex_unlock(&core_lock);
if (WARN(id < 0, "couldn't get idr"))
return id;
adapter->nr = id;
return i2c_register_adapter(adapter);
}
EXPORT_SYMBOL(i2c_add_adapter);
/**
* i2c_add_numbered_adapter - declare i2c adapter, use static bus number
* @adap: the adapter to register (with adap->nr initialized)
* Context: can sleep
*
* This routine is used to declare an I2C adapter when its bus number
* matters. For example, use it for I2C adapters from system-on-chip CPUs,
* or otherwise built in to the system's mainboard, and where i2c_board_info
* is used to properly configure I2C devices.
*
* If the requested bus number is set to -1, then this function will behave
* identically to i2c_add_adapter, and will dynamically assign a bus number.
*
* If no devices have pre-been declared for this bus, then be sure to
* register the adapter before any dynamically allocated ones. Otherwise
* the required bus ID may not be available.
*
* When this returns zero, the specified adapter became available for
* clients using the bus number provided in adap->nr. Also, the table
* of I2C devices pre-declared using i2c_register_board_info() is scanned,
* and the appropriate driver model device nodes are created. Otherwise, a
* negative errno value is returned.
*/
int i2c_add_numbered_adapter(struct i2c_adapter *adap)
{
if (adap->nr == -1) /* -1 means dynamically assign bus id */
return i2c_add_adapter(adap);
return __i2c_add_numbered_adapter(adap);
}
EXPORT_SYMBOL_GPL(i2c_add_numbered_adapter);
static void i2c_do_del_adapter(struct i2c_driver *driver,
struct i2c_adapter *adapter)
{
struct i2c_client *client, *_n;
/* Remove the devices we created ourselves as the result of hardware
* probing (using a driver's detect method) */
list_for_each_entry_safe(client, _n, &driver->clients, detected) { if (client->adapter == adapter) { dev_dbg(&adapter->dev, "Removing %s at 0x%x\n",
client->name, client->addr);
list_del(&client->detected);
i2c_unregister_device(client);
}
}
}
static int __unregister_client(struct device *dev, void *dummy)
{
struct i2c_client *client = i2c_verify_client(dev); if (client && strcmp(client->name, "dummy")) i2c_unregister_device(client); return 0;
}
static int __unregister_dummy(struct device *dev, void *dummy)
{
struct i2c_client *client = i2c_verify_client(dev);
i2c_unregister_device(client);
return 0;
}
static int __process_removed_adapter(struct device_driver *d, void *data)
{
i2c_do_del_adapter(to_i2c_driver(d), data);
return 0;
}
/**
* i2c_del_adapter - unregister I2C adapter
* @adap: the adapter being unregistered
* Context: can sleep
*
* This unregisters an I2C adapter which was previously registered
* by @i2c_add_adapter or @i2c_add_numbered_adapter.
*/
void i2c_del_adapter(struct i2c_adapter *adap)
{
struct i2c_adapter *found;
struct i2c_client *client, *next;
/* First make sure that this adapter was ever added */
mutex_lock(&core_lock);
found = idr_find(&i2c_adapter_idr, adap->nr);
mutex_unlock(&core_lock);
if (found != adap) {
pr_debug("attempting to delete unregistered adapter [%s]\n", adap->name);
return;
}
i2c_acpi_remove_space_handler(adap);
/* Tell drivers about this removal */
mutex_lock(&core_lock);
bus_for_each_drv(&i2c_bus_type, NULL, adap,
__process_removed_adapter);
mutex_unlock(&core_lock);
/* Remove devices instantiated from sysfs */
mutex_lock_nested(&adap->userspace_clients_lock,
i2c_adapter_depth(adap));
list_for_each_entry_safe(client, next, &adap->userspace_clients,
detected) {
dev_dbg(&adap->dev, "Removing %s at 0x%x\n", client->name,
client->addr);
list_del(&client->detected);
i2c_unregister_device(client);
}
mutex_unlock(&adap->userspace_clients_lock);
/* Detach any active clients. This can't fail, thus we do not
* check the returned value. This is a two-pass process, because
* we can't remove the dummy devices during the first pass: they
* could have been instantiated by real devices wishing to clean
* them up properly, so we give them a chance to do that first. */
device_for_each_child(&adap->dev, NULL, __unregister_client);
device_for_each_child(&adap->dev, NULL, __unregister_dummy);
#ifdef CONFIG_I2C_COMPAT
class_compat_remove_link(i2c_adapter_compat_class, &adap->dev,
adap->dev.parent);
#endif
/* device name is gone after device_unregister */
dev_dbg(&adap->dev, "adapter [%s] unregistered\n", adap->name); pm_runtime_disable(&adap->dev); i2c_host_notify_irq_teardown(adap); debugfs_remove_recursive(adap->debugfs);
/* wait until all references to the device are gone
*
* FIXME: This is old code and should ideally be replaced by an
* alternative which results in decoupling the lifetime of the struct
* device from the i2c_adapter, like spi or netdev do. Any solution
* should be thoroughly tested with DEBUG_KOBJECT_RELEASE enabled!
*/
init_completion(&adap->dev_released);
device_unregister(&adap->dev);
wait_for_completion(&adap->dev_released);
/* free bus id */
mutex_lock(&core_lock);
idr_remove(&i2c_adapter_idr, adap->nr);
mutex_unlock(&core_lock);
/* Clear the device structure in case this adapter is ever going to be
added again */
memset(&adap->dev, 0, sizeof(adap->dev));
}
EXPORT_SYMBOL(i2c_del_adapter);
static void devm_i2c_del_adapter(void *adapter)
{
i2c_del_adapter(adapter);
}
/**
* devm_i2c_add_adapter - device-managed variant of i2c_add_adapter()
* @dev: managing device for adding this I2C adapter
* @adapter: the adapter to add
* Context: can sleep
*
* Add adapter with dynamic bus number, same with i2c_add_adapter()
* but the adapter will be auto deleted on driver detach.
*/
int devm_i2c_add_adapter(struct device *dev, struct i2c_adapter *adapter)
{
int ret;
ret = i2c_add_adapter(adapter);
if (ret)
return ret;
return devm_add_action_or_reset(dev, devm_i2c_del_adapter, adapter);
}
EXPORT_SYMBOL_GPL(devm_i2c_add_adapter);
static int i2c_dev_or_parent_fwnode_match(struct device *dev, const void *data)
{
if (dev_fwnode(dev) == data)
return 1;
if (dev->parent && dev_fwnode(dev->parent) == data)
return 1;
return 0;
}
/**
* i2c_find_adapter_by_fwnode() - find an i2c_adapter for the fwnode
* @fwnode: &struct fwnode_handle corresponding to the &struct i2c_adapter
*
* Look up and return the &struct i2c_adapter corresponding to the @fwnode.
* If no adapter can be found, or @fwnode is NULL, this returns NULL.
*
* The user must call put_device(&adapter->dev) once done with the i2c adapter.
*/
struct i2c_adapter *i2c_find_adapter_by_fwnode(struct fwnode_handle *fwnode)
{
struct i2c_adapter *adapter;
struct device *dev;
if (!fwnode)
return NULL;
dev = bus_find_device(&i2c_bus_type, NULL, fwnode,
i2c_dev_or_parent_fwnode_match);
if (!dev)
return NULL;
adapter = i2c_verify_adapter(dev);
if (!adapter)
put_device(dev);
return adapter;
}
EXPORT_SYMBOL(i2c_find_adapter_by_fwnode);
/**
* i2c_get_adapter_by_fwnode() - find an i2c_adapter for the fwnode
* @fwnode: &struct fwnode_handle corresponding to the &struct i2c_adapter
*
* Look up and return the &struct i2c_adapter corresponding to the @fwnode,
* and increment the adapter module's use count. If no adapter can be found,
* or @fwnode is NULL, this returns NULL.
*
* The user must call i2c_put_adapter(adapter) once done with the i2c adapter.
* Note that this is different from i2c_find_adapter_by_node().
*/
struct i2c_adapter *i2c_get_adapter_by_fwnode(struct fwnode_handle *fwnode)
{
struct i2c_adapter *adapter;
adapter = i2c_find_adapter_by_fwnode(fwnode);
if (!adapter)
return NULL;
if (!try_module_get(adapter->owner)) {
put_device(&adapter->dev);
adapter = NULL;
}
return adapter;
}
EXPORT_SYMBOL(i2c_get_adapter_by_fwnode);
static void i2c_parse_timing(struct device *dev, char *prop_name, u32 *cur_val_p,
u32 def_val, bool use_def)
{
int ret;
ret = device_property_read_u32(dev, prop_name, cur_val_p);
if (ret && use_def)
*cur_val_p = def_val;
dev_dbg(dev, "%s: %u\n", prop_name, *cur_val_p);
}
/**
* i2c_parse_fw_timings - get I2C related timing parameters from firmware
* @dev: The device to scan for I2C timing properties
* @t: the i2c_timings struct to be filled with values
* @use_defaults: bool to use sane defaults derived from the I2C specification
* when properties are not found, otherwise don't update
*
* Scan the device for the generic I2C properties describing timing parameters
* for the signal and fill the given struct with the results. If a property was
* not found and use_defaults was true, then maximum timings are assumed which
* are derived from the I2C specification. If use_defaults is not used, the
* results will be as before, so drivers can apply their own defaults before
* calling this helper. The latter is mainly intended for avoiding regressions
* of existing drivers which want to switch to this function. New drivers
* almost always should use the defaults.
*/
void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults)
{
bool u = use_defaults;
u32 d;
i2c_parse_timing(dev, "clock-frequency", &t->bus_freq_hz,
I2C_MAX_STANDARD_MODE_FREQ, u);
d = t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ ? 1000 :
t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ ? 300 : 120;
i2c_parse_timing(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns, d, u);
d = t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ ? 300 : 120;
i2c_parse_timing(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns, d, u);
i2c_parse_timing(dev, "i2c-scl-internal-delay-ns",
&t->scl_int_delay_ns, 0, u);
i2c_parse_timing(dev, "i2c-sda-falling-time-ns", &t->sda_fall_ns,
t->scl_fall_ns, u);
i2c_parse_timing(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns, 0, u);
i2c_parse_timing(dev, "i2c-digital-filter-width-ns",
&t->digital_filter_width_ns, 0, u);
i2c_parse_timing(dev, "i2c-analog-filter-cutoff-frequency",
&t->analog_filter_cutoff_freq_hz, 0, u);
}
EXPORT_SYMBOL_GPL(i2c_parse_fw_timings);
/* ------------------------------------------------------------------------- */
int i2c_for_each_dev(void *data, int (*fn)(struct device *dev, void *data))
{
int res;
mutex_lock(&core_lock);
res = bus_for_each_dev(&i2c_bus_type, NULL, data, fn);
mutex_unlock(&core_lock);
return res;
}
EXPORT_SYMBOL_GPL(i2c_for_each_dev);
static int __process_new_driver(struct device *dev, void *data)
{
if (dev->type != &i2c_adapter_type)
return 0;
return i2c_do_add_adapter(data, to_i2c_adapter(dev));
}
/*
* An i2c_driver is used with one or more i2c_client (device) nodes to access
* i2c slave chips, on a bus instance associated with some i2c_adapter.
*/
int i2c_register_driver(struct module *owner, struct i2c_driver *driver)
{
int res;
/* Can't register until after driver model init */
if (WARN_ON(!is_registered))
return -EAGAIN;
/* add the driver to the list of i2c drivers in the driver core */
driver->driver.owner = owner;
driver->driver.bus = &i2c_bus_type;
INIT_LIST_HEAD(&driver->clients);
/* When registration returns, the driver core
* will have called probe() for all matching-but-unbound devices.
*/
res = driver_register(&driver->driver);
if (res)
return res;
pr_debug("driver [%s] registered\n", driver->driver.name);
/* Walk the adapters that are already present */
i2c_for_each_dev(driver, __process_new_driver);
return 0;
}
EXPORT_SYMBOL(i2c_register_driver);
static int __process_removed_driver(struct device *dev, void *data)
{
if (dev->type == &i2c_adapter_type)
i2c_do_del_adapter(data, to_i2c_adapter(dev));
return 0;
}
/**
* i2c_del_driver - unregister I2C driver
* @driver: the driver being unregistered
* Context: can sleep
*/
void i2c_del_driver(struct i2c_driver *driver)
{
i2c_for_each_dev(driver, __process_removed_driver);
driver_unregister(&driver->driver);
pr_debug("driver [%s] unregistered\n", driver->driver.name);
}
EXPORT_SYMBOL(i2c_del_driver);
/* ------------------------------------------------------------------------- */
struct i2c_cmd_arg {
unsigned cmd;
void *arg;
};
static int i2c_cmd(struct device *dev, void *_arg)
{
struct i2c_client *client = i2c_verify_client(dev);
struct i2c_cmd_arg *arg = _arg;
struct i2c_driver *driver;
if (!client || !client->dev.driver)
return 0;
driver = to_i2c_driver(client->dev.driver);
if (driver->command)
driver->command(client, arg->cmd, arg->arg);
return 0;
}
void i2c_clients_command(struct i2c_adapter *adap, unsigned int cmd, void *arg)
{
struct i2c_cmd_arg cmd_arg;
cmd_arg.cmd = cmd;
cmd_arg.arg = arg;
device_for_each_child(&adap->dev, &cmd_arg, i2c_cmd);
}
EXPORT_SYMBOL(i2c_clients_command);
static int __init i2c_init(void)
{
int retval;
retval = of_alias_get_highest_id("i2c");
down_write(&__i2c_board_lock);
if (retval >= __i2c_first_dynamic_bus_num)
__i2c_first_dynamic_bus_num = retval + 1;
up_write(&__i2c_board_lock);
retval = bus_register(&i2c_bus_type);
if (retval)
return retval;
is_registered = true;
i2c_debugfs_root = debugfs_create_dir("i2c", NULL);
#ifdef CONFIG_I2C_COMPAT
i2c_adapter_compat_class = class_compat_register("i2c-adapter");
if (!i2c_adapter_compat_class) {
retval = -ENOMEM;
goto bus_err;
}
#endif
retval = i2c_add_driver(&dummy_driver);
if (retval)
goto class_err;
if (IS_ENABLED(CONFIG_OF_DYNAMIC))
WARN_ON(of_reconfig_notifier_register(&i2c_of_notifier));
if (IS_ENABLED(CONFIG_ACPI))
WARN_ON(acpi_reconfig_notifier_register(&i2c_acpi_notifier));
return 0;
class_err:
#ifdef CONFIG_I2C_COMPAT
class_compat_unregister(i2c_adapter_compat_class);
bus_err:
#endif
is_registered = false;
bus_unregister(&i2c_bus_type);
return retval;
}
static void __exit i2c_exit(void)
{
if (IS_ENABLED(CONFIG_ACPI))
WARN_ON(acpi_reconfig_notifier_unregister(&i2c_acpi_notifier));
if (IS_ENABLED(CONFIG_OF_DYNAMIC))
WARN_ON(of_reconfig_notifier_unregister(&i2c_of_notifier));
i2c_del_driver(&dummy_driver);
#ifdef CONFIG_I2C_COMPAT
class_compat_unregister(i2c_adapter_compat_class);
#endif
debugfs_remove_recursive(i2c_debugfs_root);
bus_unregister(&i2c_bus_type);
tracepoint_synchronize_unregister();
}
/* We must initialize early, because some subsystems register i2c drivers
* in subsys_initcall() code, but are linked (and initialized) before i2c.
*/
postcore_initcall(i2c_init);
module_exit(i2c_exit);
/* ----------------------------------------------------
* the functional interface to the i2c busses.
* ----------------------------------------------------
*/
/* Check if val is exceeding the quirk IFF quirk is non 0 */
#define i2c_quirk_exceeded(val, quirk) ((quirk) && ((val) > (quirk)))
static int i2c_quirk_error(struct i2c_adapter *adap, struct i2c_msg *msg, char *err_msg)
{
dev_err_ratelimited(&adap->dev, "adapter quirk: %s (addr 0x%04x, size %u, %s)\n",
err_msg, msg->addr, msg->len,
msg->flags & I2C_M_RD ? "read" : "write");
return -EOPNOTSUPP;
}
static int i2c_check_for_quirks(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
{
const struct i2c_adapter_quirks *q = adap->quirks;
int max_num = q->max_num_msgs, i;
bool do_len_check = true;
if (q->flags & I2C_AQ_COMB) {
max_num = 2;
/* special checks for combined messages */
if (num == 2) { if (q->flags & I2C_AQ_COMB_WRITE_FIRST && msgs[0].flags & I2C_M_RD) return i2c_quirk_error(adap, &msgs[0], "1st comb msg must be write"); if (q->flags & I2C_AQ_COMB_READ_SECOND && !(msgs[1].flags & I2C_M_RD)) return i2c_quirk_error(adap, &msgs[1], "2nd comb msg must be read"); if (q->flags & I2C_AQ_COMB_SAME_ADDR && msgs[0].addr != msgs[1].addr) return i2c_quirk_error(adap, &msgs[0], "comb msg only to same addr"); if (i2c_quirk_exceeded(msgs[0].len, q->max_comb_1st_msg_len)) return i2c_quirk_error(adap, &msgs[0], "msg too long"); if (i2c_quirk_exceeded(msgs[1].len, q->max_comb_2nd_msg_len)) return i2c_quirk_error(adap, &msgs[1], "msg too long");
do_len_check = false;
}
}
if (i2c_quirk_exceeded(num, max_num)) return i2c_quirk_error(adap, &msgs[0], "too many messages"); for (i = 0; i < num; i++) { u16 len = msgs[i].len;
if (msgs[i].flags & I2C_M_RD) {
if (do_len_check && i2c_quirk_exceeded(len, q->max_read_len)) return i2c_quirk_error(adap, &msgs[i], "msg too long"); if (q->flags & I2C_AQ_NO_ZERO_LEN_READ && len == 0) return i2c_quirk_error(adap, &msgs[i], "no zero length");
} else {
if (do_len_check && i2c_quirk_exceeded(len, q->max_write_len)) return i2c_quirk_error(adap, &msgs[i], "msg too long"); if (q->flags & I2C_AQ_NO_ZERO_LEN_WRITE && len == 0) return i2c_quirk_error(adap, &msgs[i], "no zero length");
}
}
return 0;
}
/**
* __i2c_transfer - unlocked flavor of i2c_transfer
* @adap: Handle to I2C bus
* @msgs: One or more messages to execute before STOP is issued to
* terminate the operation; each message begins with a START.
* @num: Number of messages to be executed.
*
* Returns negative errno, else the number of messages executed.
*
* Adapter lock must be held when calling this function. No debug logging
* takes place.
*/
int __i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
{
unsigned long orig_jiffies;
int ret, try;
if (!adap->algo->master_xfer) { dev_dbg(&adap->dev, "I2C level transfers not supported\n");
return -EOPNOTSUPP;
}
if (WARN_ON(!msgs || num < 1)) return -EINVAL; ret = __i2c_check_suspended(adap);
if (ret)
return ret;
if (adap->quirks && i2c_check_for_quirks(adap, msgs, num))
return -EOPNOTSUPP;
/*
* i2c_trace_msg_key gets enabled when tracepoint i2c_transfer gets
* enabled. This is an efficient way of keeping the for-loop from
* being executed when not needed.
*/
if (static_branch_unlikely(&i2c_trace_msg_key)) {
int i;
for (i = 0; i < num; i++) if (msgs[i].flags & I2C_M_RD) trace_i2c_read(adap, &msgs[i], i);
else
trace_i2c_write(adap, &msgs[i], i);
}
/* Retry automatically on arbitration loss */
orig_jiffies = jiffies; for (ret = 0, try = 0; try <= adap->retries; try++) { if (i2c_in_atomic_xfer_mode() && adap->algo->master_xfer_atomic) ret = adap->algo->master_xfer_atomic(adap, msgs, num);
else
ret = adap->algo->master_xfer(adap, msgs, num); if (ret != -EAGAIN)
break;
if (time_after(jiffies, orig_jiffies + adap->timeout))
break;
}
if (static_branch_unlikely(&i2c_trace_msg_key)) {
int i;
for (i = 0; i < ret; i++) if (msgs[i].flags & I2C_M_RD) trace_i2c_reply(adap, &msgs[i], i); trace_i2c_result(adap, num, ret);
}
return ret;
}
EXPORT_SYMBOL(__i2c_transfer);
/**
* i2c_transfer - execute a single or combined I2C message
* @adap: Handle to I2C bus
* @msgs: One or more messages to execute before STOP is issued to
* terminate the operation; each message begins with a START.
* @num: Number of messages to be executed.
*
* Returns negative errno, else the number of messages executed.
*
* Note that there is no requirement that each message be sent to
* the same slave address, although that is the most common model.
*/
int i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
{
int ret;
/* REVISIT the fault reporting model here is weak:
*
* - When we get an error after receiving N bytes from a slave,
* there is no way to report "N".
*
* - When we get a NAK after transmitting N bytes to a slave,
* there is no way to report "N" ... or to let the master
* continue executing the rest of this combined message, if
* that's the appropriate response.
*
* - When for example "num" is two and we successfully complete
* the first message but get an error part way through the
* second, it's unclear whether that should be reported as
* one (discarding status on the second message) or errno
* (discarding status on the first one).
*/
ret = __i2c_lock_bus_helper(adap);
if (ret)
return ret;
ret = __i2c_transfer(adap, msgs, num);
i2c_unlock_bus(adap, I2C_LOCK_SEGMENT);
return ret;
}
EXPORT_SYMBOL(i2c_transfer);
/**
* i2c_transfer_buffer_flags - issue a single I2C message transferring data
* to/from a buffer
* @client: Handle to slave device
* @buf: Where the data is stored
* @count: How many bytes to transfer, must be less than 64k since msg.len is u16
* @flags: The flags to be used for the message, e.g. I2C_M_RD for reads
*
* Returns negative errno, or else the number of bytes transferred.
*/
int i2c_transfer_buffer_flags(const struct i2c_client *client, char *buf,
int count, u16 flags)
{
int ret;
struct i2c_msg msg = {
.addr = client->addr,
.flags = flags | (client->flags & I2C_M_TEN),
.len = count,
.buf = buf,
};
ret = i2c_transfer(client->adapter, &msg, 1);
/*
* If everything went ok (i.e. 1 msg transferred), return #bytes
* transferred, else error code.
*/
return (ret == 1) ? count : ret;
}
EXPORT_SYMBOL(i2c_transfer_buffer_flags);
/**
* i2c_get_device_id - get manufacturer, part id and die revision of a device
* @client: The device to query
* @id: The queried information
*
* Returns negative errno on error, zero on success.
*/
int i2c_get_device_id(const struct i2c_client *client,
struct i2c_device_identity *id)
{
struct i2c_adapter *adap = client->adapter;
union i2c_smbus_data raw_id;
int ret;
if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_READ_I2C_BLOCK))
return -EOPNOTSUPP;
raw_id.block[0] = 3;
ret = i2c_smbus_xfer(adap, I2C_ADDR_DEVICE_ID, 0,
I2C_SMBUS_READ, client->addr << 1,
I2C_SMBUS_I2C_BLOCK_DATA, &raw_id);
if (ret)
return ret;
id->manufacturer_id = (raw_id.block[1] << 4) | (raw_id.block[2] >> 4);
id->part_id = ((raw_id.block[2] & 0xf) << 5) | (raw_id.block[3] >> 3);
id->die_revision = raw_id.block[3] & 0x7;
return 0;
}
EXPORT_SYMBOL_GPL(i2c_get_device_id);
/**
* i2c_client_get_device_id - get the driver match table entry of a device
* @client: the device to query. The device must be bound to a driver
*
* Returns a pointer to the matching entry if found, NULL otherwise.
*/
const struct i2c_device_id *i2c_client_get_device_id(const struct i2c_client *client)
{
const struct i2c_driver *drv = to_i2c_driver(client->dev.driver);
return i2c_match_id(drv->id_table, client);
}
EXPORT_SYMBOL_GPL(i2c_client_get_device_id);
/* ----------------------------------------------------
* the i2c address scanning function
* Will not work for 10-bit addresses!
* ----------------------------------------------------
*/
/*
* Legacy default probe function, mostly relevant for SMBus. The default
* probe method is a quick write, but it is known to corrupt the 24RF08
* EEPROMs due to a state machine bug, and could also irreversibly
* write-protect some EEPROMs, so for address ranges 0x30-0x37 and 0x50-0x5f,
* we use a short byte read instead. Also, some bus drivers don't implement
* quick write, so we fallback to a byte read in that case too.
* On x86, there is another special case for FSC hardware monitoring chips,
* which want regular byte reads (address 0x73.) Fortunately, these are the
* only known chips using this I2C address on PC hardware.
* Returns 1 if probe succeeded, 0 if not.
*/
static int i2c_default_probe(struct i2c_adapter *adap, unsigned short addr)
{
int err;
union i2c_smbus_data dummy;
#ifdef CONFIG_X86
if (addr == 0x73 && (adap->class & I2C_CLASS_HWMON)
&& i2c_check_functionality(adap, I2C_FUNC_SMBUS_READ_BYTE_DATA))
err = i2c_smbus_xfer(adap, addr, 0, I2C_SMBUS_READ, 0,
I2C_SMBUS_BYTE_DATA, &dummy);
else
#endif
if (!((addr & ~0x07) == 0x30 || (addr & ~0x0f) == 0x50)
&& i2c_check_functionality(adap, I2C_FUNC_SMBUS_QUICK))
err = i2c_smbus_xfer(adap, addr, 0, I2C_SMBUS_WRITE, 0,
I2C_SMBUS_QUICK, NULL);
else if (i2c_check_functionality(adap, I2C_FUNC_SMBUS_READ_BYTE))
err = i2c_smbus_xfer(adap, addr, 0, I2C_SMBUS_READ, 0,
I2C_SMBUS_BYTE, &dummy);
else {
dev_warn(&adap->dev, "No suitable probing method supported for address 0x%02X\n",
addr);
err = -EOPNOTSUPP;
}
return err >= 0;
}
static int i2c_detect_address(struct i2c_client *temp_client,
struct i2c_driver *driver)
{
struct i2c_board_info info;
struct i2c_adapter *adapter = temp_client->adapter;
int addr = temp_client->addr;
int err;
/* Make sure the address is valid */
err = i2c_check_7bit_addr_validity_strict(addr);
if (err) {
dev_warn(&adapter->dev, "Invalid probe address 0x%02x\n",
addr);
return err;
}
/* Skip if already in use (7 bit, no need to encode flags) */
if (i2c_check_addr_busy(adapter, addr))
return 0;
/* Make sure there is something at this address */
if (!i2c_default_probe(adapter, addr))
return 0;
/* Finally call the custom detection function */
memset(&info, 0, sizeof(struct i2c_board_info));
info.addr = addr;
err = driver->detect(temp_client, &info);
if (err) {
/* -ENODEV is returned if the detection fails. We catch it
here as this isn't an error. */
return err == -ENODEV ? 0 : err;
}
/* Consistency check */
if (info.type[0] == '\0') {
dev_err(&adapter->dev,
"%s detection function provided no name for 0x%x\n",
driver->driver.name, addr);
} else {
struct i2c_client *client;
/* Detection succeeded, instantiate the device */
if (adapter->class & I2C_CLASS_DEPRECATED)
dev_warn(&adapter->dev,
"This adapter will soon drop class based instantiation of devices. "
"Please make sure client 0x%02x gets instantiated by other means. "
"Check 'Documentation/i2c/instantiating-devices.rst' for details.\n",
info.addr);
dev_dbg(&adapter->dev, "Creating %s at 0x%02x\n",
info.type, info.addr);
client = i2c_new_client_device(adapter, &info);
if (!IS_ERR(client))
list_add_tail(&client->detected, &driver->clients);
else
dev_err(&adapter->dev, "Failed creating %s at 0x%02x\n",
info.type, info.addr);
}
return 0;
}
static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver)
{
const unsigned short *address_list;
struct i2c_client *temp_client;
int i, err = 0;
address_list = driver->address_list;
if (!driver->detect || !address_list)
return 0;
/* Warn that the adapter lost class based instantiation */
if (adapter->class == I2C_CLASS_DEPRECATED) {
dev_dbg(&adapter->dev,
"This adapter dropped support for I2C classes and won't auto-detect %s devices anymore. "
"If you need it, check 'Documentation/i2c/instantiating-devices.rst' for alternatives.\n",
driver->driver.name);
return 0;
}
/* Stop here if the classes do not match */
if (!(adapter->class & driver->class))
return 0;
/* Set up a temporary client to help detect callback */
temp_client = kzalloc(sizeof(struct i2c_client), GFP_KERNEL);
if (!temp_client)
return -ENOMEM;
temp_client->adapter = adapter;
for (i = 0; address_list[i] != I2C_CLIENT_END; i += 1) {
dev_dbg(&adapter->dev,
"found normal entry for adapter %d, addr 0x%02x\n",
i2c_adapter_id(adapter), address_list[i]);
temp_client->addr = address_list[i];
err = i2c_detect_address(temp_client, driver);
if (unlikely(err))
break;
}
kfree(temp_client);
return err;
}
int i2c_probe_func_quick_read(struct i2c_adapter *adap, unsigned short addr)
{
return i2c_smbus_xfer(adap, addr, 0, I2C_SMBUS_READ, 0,
I2C_SMBUS_QUICK, NULL) >= 0;
}
EXPORT_SYMBOL_GPL(i2c_probe_func_quick_read);
struct i2c_client *
i2c_new_scanned_device(struct i2c_adapter *adap,
struct i2c_board_info *info,
unsigned short const *addr_list,
int (*probe)(struct i2c_adapter *adap, unsigned short addr))
{
int i;
if (!probe)
probe = i2c_default_probe;
for (i = 0; addr_list[i] != I2C_CLIENT_END; i++) {
/* Check address validity */
if (i2c_check_7bit_addr_validity_strict(addr_list[i]) < 0) {
dev_warn(&adap->dev, "Invalid 7-bit address 0x%02x\n",
addr_list[i]);
continue;
}
/* Check address availability (7 bit, no need to encode flags) */
if (i2c_check_addr_busy(adap, addr_list[i])) {
dev_dbg(&adap->dev,
"Address 0x%02x already in use, not probing\n",
addr_list[i]);
continue;
}
/* Test address responsiveness */
if (probe(adap, addr_list[i]))
break;
}
if (addr_list[i] == I2C_CLIENT_END) {
dev_dbg(&adap->dev, "Probing failed, no device found\n");
return ERR_PTR(-ENODEV);
}
info->addr = addr_list[i];
return i2c_new_client_device(adap, info);
}
EXPORT_SYMBOL_GPL(i2c_new_scanned_device);
struct i2c_adapter *i2c_get_adapter(int nr)
{
struct i2c_adapter *adapter;
mutex_lock(&core_lock);
adapter = idr_find(&i2c_adapter_idr, nr);
if (!adapter)
goto exit;
if (try_module_get(adapter->owner))
get_device(&adapter->dev);
else
adapter = NULL;
exit:
mutex_unlock(&core_lock);
return adapter;
}
EXPORT_SYMBOL(i2c_get_adapter);
void i2c_put_adapter(struct i2c_adapter *adap)
{
if (!adap)
return;
module_put(adap->owner);
/* Should be last, otherwise we risk use-after-free with 'adap' */
put_device(&adap->dev);
}
EXPORT_SYMBOL(i2c_put_adapter);
/**
* i2c_get_dma_safe_msg_buf() - get a DMA safe buffer for the given i2c_msg
* @msg: the message to be checked
* @threshold: the minimum number of bytes for which using DMA makes sense.
* Should at least be 1.
*
* Return: NULL if a DMA safe buffer was not obtained. Use msg->buf with PIO.
* Or a valid pointer to be used with DMA. After use, release it by
* calling i2c_put_dma_safe_msg_buf().
*
* This function must only be called from process context!
*/
u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold)
{
/* also skip 0-length msgs for bogus thresholds of 0 */
if (!threshold)
pr_debug("DMA buffer for addr=0x%02x with length 0 is bogus\n",
msg->addr);
if (msg->len < threshold || msg->len == 0)
return NULL;
if (msg->flags & I2C_M_DMA_SAFE)
return msg->buf;
pr_debug("using bounce buffer for addr=0x%02x, len=%d\n",
msg->addr, msg->len);
if (msg->flags & I2C_M_RD)
return kzalloc(msg->len, GFP_KERNEL);
else
return kmemdup(msg->buf, msg->len, GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(i2c_get_dma_safe_msg_buf);
/**
* i2c_put_dma_safe_msg_buf - release DMA safe buffer and sync with i2c_msg
* @buf: the buffer obtained from i2c_get_dma_safe_msg_buf(). May be NULL.
* @msg: the message which the buffer corresponds to
* @xferred: bool saying if the message was transferred
*/
void i2c_put_dma_safe_msg_buf(u8 *buf, struct i2c_msg *msg, bool xferred)
{
if (!buf || buf == msg->buf)
return;
if (xferred && msg->flags & I2C_M_RD)
memcpy(msg->buf, buf, msg->len);
kfree(buf);
}
EXPORT_SYMBOL_GPL(i2c_put_dma_safe_msg_buf);
MODULE_AUTHOR("Simon G. Vogl <simon@tk.uni-linz.ac.at>");
MODULE_DESCRIPTION("I2C-Bus main module");
MODULE_LICENSE("GPL");
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MM_H
#define _LINUX_MM_H
#include <linux/errno.h>
#include <linux/mmdebug.h>
#include <linux/gfp.h>
#include <linux/bug.h>
#include <linux/list.h>
#include <linux/mmzone.h>
#include <linux/rbtree.h>
#include <linux/atomic.h>
#include <linux/debug_locks.h>
#include <linux/mm_types.h>
#include <linux/mmap_lock.h>
#include <linux/range.h>
#include <linux/pfn.h>
#include <linux/percpu-refcount.h>
#include <linux/bit_spinlock.h>
#include <linux/shrinker.h>
#include <linux/resource.h>
#include <linux/page_ext.h>
#include <linux/err.h>
#include <linux/page-flags.h>
#include <linux/page_ref.h>
#include <linux/overflow.h>
#include <linux/sizes.h>
#include <linux/sched.h>
#include <linux/pgtable.h>
#include <linux/kasan.h>
#include <linux/memremap.h>
#include <linux/slab.h>
struct mempolicy;
struct anon_vma;
struct anon_vma_chain;
struct user_struct;
struct pt_regs;
struct folio_batch;
extern int sysctl_page_lock_unfairness;
void mm_core_init(void);
void init_mm_internals(void);
#ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */
extern unsigned long max_mapnr;
static inline void set_max_mapnr(unsigned long limit)
{
max_mapnr = limit;
}
#else
static inline void set_max_mapnr(unsigned long limit) { }
#endif
extern atomic_long_t _totalram_pages;
static inline unsigned long totalram_pages(void)
{
return (unsigned long)atomic_long_read(&_totalram_pages);
}
static inline void totalram_pages_inc(void)
{
atomic_long_inc(&_totalram_pages);
}
static inline void totalram_pages_dec(void)
{
atomic_long_dec(&_totalram_pages);
}
static inline void totalram_pages_add(long count)
{
atomic_long_add(count, &_totalram_pages);
}
extern void * high_memory;
extern int page_cluster;
extern const int page_cluster_max;
#ifdef CONFIG_SYSCTL
extern int sysctl_legacy_va_layout;
#else
#define sysctl_legacy_va_layout 0
#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
extern const int mmap_rnd_bits_min;
extern int mmap_rnd_bits_max __ro_after_init;
extern int mmap_rnd_bits __read_mostly;
#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
extern const int mmap_rnd_compat_bits_min;
extern const int mmap_rnd_compat_bits_max;
extern int mmap_rnd_compat_bits __read_mostly;
#endif
#include <asm/page.h>
#include <asm/processor.h>
#ifndef __pa_symbol
#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
#endif
#ifndef page_to_virt
#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
#endif
#ifndef lm_alias
#define lm_alias(x) __va(__pa_symbol(x))
#endif
/*
* To prevent common memory management code establishing
* a zero page mapping on a read fault.
* This macro should be defined within <asm/pgtable.h>.
* s390 does this to prevent multiplexing of hardware bits
* related to the physical page in case of virtualization.
*/
#ifndef mm_forbids_zeropage
#define mm_forbids_zeropage(X) (0)
#endif
/*
* On some architectures it is expensive to call memset() for small sizes.
* If an architecture decides to implement their own version of
* mm_zero_struct_page they should wrap the defines below in a #ifndef and
* define their own version of this macro in <asm/pgtable.h>
*/
#if BITS_PER_LONG == 64
/* This function must be updated when the size of struct page grows above 96
* or reduces below 56. The idea that compiler optimizes out switch()
* statement, and only leaves move/store instructions. Also the compiler can
* combine write statements if they are both assignments and can be reordered,
* this can result in several of the writes here being dropped.
*/
#define mm_zero_struct_page(pp) __mm_zero_struct_page(pp)
static inline void __mm_zero_struct_page(struct page *page)
{
unsigned long *_pp = (void *)page;
/* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */
BUILD_BUG_ON(sizeof(struct page) & 7);
BUILD_BUG_ON(sizeof(struct page) < 56);
BUILD_BUG_ON(sizeof(struct page) > 96);
switch (sizeof(struct page)) {
case 96:
_pp[11] = 0;
fallthrough;
case 88:
_pp[10] = 0;
fallthrough;
case 80:
_pp[9] = 0;
fallthrough;
case 72:
_pp[8] = 0;
fallthrough;
case 64:
_pp[7] = 0;
fallthrough;
case 56:
_pp[6] = 0;
_pp[5] = 0;
_pp[4] = 0;
_pp[3] = 0;
_pp[2] = 0;
_pp[1] = 0;
_pp[0] = 0;
}
}
#else
#define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
#endif
/*
* Default maximum number of active map areas, this limits the number of vmas
* per mm struct. Users can overwrite this number by sysctl but there is a
* problem.
*
* When a program's coredump is generated as ELF format, a section is created
* per a vma. In ELF, the number of sections is represented in unsigned short.
* This means the number of sections should be smaller than 65535 at coredump.
* Because the kernel adds some informative sections to a image of program at
* generating coredump, we need some margin. The number of extra sections is
* 1-3 now and depends on arch. We use "5" as safe margin, here.
*
* ELF extended numbering allows more than 65535 sections, so 16-bit bound is
* not a hard limit any more. Although some userspace tools can be surprised by
* that.
*/
#define MAPCOUNT_ELF_CORE_MARGIN (5)
#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
extern int sysctl_max_map_count;
extern unsigned long sysctl_user_reserve_kbytes;
extern unsigned long sysctl_admin_reserve_kbytes;
extern int sysctl_overcommit_memory;
extern int sysctl_overcommit_ratio;
extern unsigned long sysctl_overcommit_kbytes;
int overcommit_ratio_handler(struct ctl_table *, int, void *, size_t *,
loff_t *);
int overcommit_kbytes_handler(struct ctl_table *, int, void *, size_t *,
loff_t *);
int overcommit_policy_handler(struct ctl_table *, int, void *, size_t *,
loff_t *);
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
#define folio_page_idx(folio, p) (page_to_pfn(p) - folio_pfn(folio))
#else
#define nth_page(page,n) ((page) + (n))
#define folio_page_idx(folio, p) ((p) - &(folio)->page)
#endif
/* to align the pointer to the (next) page boundary */
#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
/* to align the pointer to the (prev) page boundary */
#define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE)
/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
static inline struct folio *lru_to_folio(struct list_head *head)
{
return list_entry((head)->prev, struct folio, lru);
}
void setup_initial_init_mm(void *start_code, void *end_code,
void *end_data, void *brk);
/*
* Linux kernel virtual memory manager primitives.
* The idea being to have a "virtual" mm in the same way
* we have a virtual fs - giving a cleaner interface to the
* mm details, and allowing different kinds of memory mappings
* (from shared memory to executable loading to arbitrary
* mmap() functions).
*/
struct vm_area_struct *vm_area_alloc(struct mm_struct *);
struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
void vm_area_free(struct vm_area_struct *);
/* Use only if VMA has no other users */
void __vm_area_free(struct vm_area_struct *vma);
#ifndef CONFIG_MMU
extern struct rb_root nommu_region_tree;
extern struct rw_semaphore nommu_region_sem;
extern unsigned int kobjsize(const void *objp);
#endif
/*
* vm_flags in vm_area_struct, see mm_types.h.
* When changing, update also include/trace/events/mmflags.h
*/
#define VM_NONE 0x00000000
#define VM_READ 0x00000001 /* currently active flags */
#define VM_WRITE 0x00000002
#define VM_EXEC 0x00000004
#define VM_SHARED 0x00000008
/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
#define VM_MAYWRITE 0x00000020
#define VM_MAYEXEC 0x00000040
#define VM_MAYSHARE 0x00000080
#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
#ifdef CONFIG_MMU
#define VM_UFFD_MISSING 0x00000200 /* missing pages tracking */
#else /* CONFIG_MMU */
#define VM_MAYOVERLAY 0x00000200 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */
#define VM_UFFD_MISSING 0
#endif /* CONFIG_MMU */
#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
#define VM_UFFD_WP 0x00001000 /* wrprotect pages tracking */
#define VM_LOCKED 0x00002000
#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
/* Used by sys_madvise() */
#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
#define VM_LOCKONFAULT 0x00080000 /* Lock the pages covered when they are faulted in */
#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
#define VM_SYNC 0x00800000 /* Synchronous page faults */
#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
#define VM_WIPEONFORK 0x02000000 /* Wipe VMA contents in child. */
#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
#ifdef CONFIG_MEM_SOFT_DIRTY
# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
#else
# define VM_SOFTDIRTY 0
#endif
#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
#ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
#define VM_HIGH_ARCH_BIT_0 32 /* bit only usable on 64-bit architectures */
#define VM_HIGH_ARCH_BIT_1 33 /* bit only usable on 64-bit architectures */
#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
#define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */
#define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */
#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
#define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)
#define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5)
#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
#ifdef CONFIG_ARCH_HAS_PKEYS
# define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0
# define VM_PKEY_BIT0 VM_HIGH_ARCH_0 /* A protection key is a 4-bit value */
# define VM_PKEY_BIT1 VM_HIGH_ARCH_1 /* on x86 and 5-bit value on ppc64 */
# define VM_PKEY_BIT2 VM_HIGH_ARCH_2
# define VM_PKEY_BIT3 VM_HIGH_ARCH_3
#ifdef CONFIG_PPC
# define VM_PKEY_BIT4 VM_HIGH_ARCH_4
#else
# define VM_PKEY_BIT4 0
#endif
#endif /* CONFIG_ARCH_HAS_PKEYS */
#ifdef CONFIG_X86_USER_SHADOW_STACK
/*
* VM_SHADOW_STACK should not be set with VM_SHARED because of lack of
* support core mm.
*
* These VMAs will get a single end guard page. This helps userspace protect
* itself from attacks. A single page is enough for current shadow stack archs
* (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c
* for more details on the guard size.
*/
# define VM_SHADOW_STACK VM_HIGH_ARCH_5
#else
# define VM_SHADOW_STACK VM_NONE
#endif
#if defined(CONFIG_X86)
# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
#elif defined(CONFIG_PPC)
# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
#elif defined(CONFIG_PARISC)
# define VM_GROWSUP VM_ARCH_1
#elif defined(CONFIG_SPARC64)
# define VM_SPARC_ADI VM_ARCH_1 /* Uses ADI tag for access control */
# define VM_ARCH_CLEAR VM_SPARC_ADI
#elif defined(CONFIG_ARM64)
# define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
# define VM_ARCH_CLEAR VM_ARM64_BTI
#elif !defined(CONFIG_MMU)
# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
#endif
#if defined(CONFIG_ARM64_MTE)
# define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */
# define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */
#else
# define VM_MTE VM_NONE
# define VM_MTE_ALLOWED VM_NONE
#endif
#ifndef VM_GROWSUP
# define VM_GROWSUP VM_NONE
#endif
#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
# define VM_UFFD_MINOR_BIT 38
# define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */
#else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
# define VM_UFFD_MINOR VM_NONE
#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
/*
* This flag is used to connect VFIO to arch specific KVM code. It
* indicates that the memory under this VMA is safe for use with any
* non-cachable memory type inside KVM. Some VFIO devices, on some
* platforms, are thought to be unsafe and can cause machine crashes
* if KVM does not lock down the memory type.
*/
#ifdef CONFIG_64BIT
#define VM_ALLOW_ANY_UNCACHED_BIT 39
#define VM_ALLOW_ANY_UNCACHED BIT(VM_ALLOW_ANY_UNCACHED_BIT)
#else
#define VM_ALLOW_ANY_UNCACHED VM_NONE
#endif
/* Bits set in the VMA until the stack is in its final location */
#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
#define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0)
/* Common data flag combinations */
#define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \
VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
#define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \
VM_MAYWRITE | VM_MAYEXEC)
#define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \
VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
#ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */
#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC
#endif
#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
#endif
#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
#ifdef CONFIG_STACK_GROWSUP
#define VM_STACK VM_GROWSUP
#define VM_STACK_EARLY VM_GROWSDOWN
#else
#define VM_STACK VM_GROWSDOWN
#define VM_STACK_EARLY 0
#endif
#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
/* VMA basic access permission flags */
#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
/*
* Special vmas that are non-mergable, non-mlock()able.
*/
#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
/* This mask prevents VMA from being scanned with khugepaged */
#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
/* This mask defines which mm->def_flags a process can inherit its parent */
#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
/* This mask represents all the VMA flag bits used by mlock */
#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
/* Arch-specific flags to clear when updating VM flags on protection change */
#ifndef VM_ARCH_CLEAR
# define VM_ARCH_CLEAR VM_NONE
#endif
#define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR)
/*
* mapping from the currently active vm_flags protection bits (the
* low four bits) to a page protection mask..
*/
/*
* The default fault flags that should be used by most of the
* arch-specific page fault handlers.
*/
#define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \
FAULT_FLAG_KILLABLE | \
FAULT_FLAG_INTERRUPTIBLE)
/**
* fault_flag_allow_retry_first - check ALLOW_RETRY the first time
* @flags: Fault flags.
*
* This is mostly used for places where we want to try to avoid taking
* the mmap_lock for too long a time when waiting for another condition
* to change, in which case we can try to be polite to release the
* mmap_lock in the first round to avoid potential starvation of other
* processes that would also want the mmap_lock.
*
* Return: true if the page fault allows retry and this is the first
* attempt of the fault handling; false otherwise.
*/
static inline bool fault_flag_allow_retry_first(enum fault_flag flags)
{
return (flags & FAULT_FLAG_ALLOW_RETRY) &&
(!(flags & FAULT_FLAG_TRIED));
}
#define FAULT_FLAG_TRACE \
{ FAULT_FLAG_WRITE, "WRITE" }, \
{ FAULT_FLAG_MKWRITE, "MKWRITE" }, \
{ FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \
{ FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \
{ FAULT_FLAG_KILLABLE, "KILLABLE" }, \
{ FAULT_FLAG_TRIED, "TRIED" }, \
{ FAULT_FLAG_USER, "USER" }, \
{ FAULT_FLAG_REMOTE, "REMOTE" }, \
{ FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \
{ FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \
{ FAULT_FLAG_VMA_LOCK, "VMA_LOCK" }
/*
* vm_fault is filled by the pagefault handler and passed to the vma's
* ->fault function. The vma's ->fault is responsible for returning a bitmask
* of VM_FAULT_xxx flags that give details about how the fault was handled.
*
* MM layer fills up gfp_mask for page allocations but fault handler might
* alter it if its implementation requires a different allocation context.
*
* pgoff should be used in favour of virtual_address, if possible.
*/
struct vm_fault {
const struct {
struct vm_area_struct *vma; /* Target VMA */
gfp_t gfp_mask; /* gfp mask to be used for allocations */
pgoff_t pgoff; /* Logical page offset based on vma */
unsigned long address; /* Faulting virtual address - masked */
unsigned long real_address; /* Faulting virtual address - unmasked */
};
enum fault_flag flags; /* FAULT_FLAG_xxx flags
* XXX: should really be 'const' */
pmd_t *pmd; /* Pointer to pmd entry matching
* the 'address' */
pud_t *pud; /* Pointer to pud entry matching
* the 'address'
*/
union {
pte_t orig_pte; /* Value of PTE at the time of fault */
pmd_t orig_pmd; /* Value of PMD at the time of fault,
* used by PMD fault only.
*/
};
struct page *cow_page; /* Page handler may use for COW fault */
struct page *page; /* ->fault handlers should return a
* page here, unless VM_FAULT_NOPAGE
* is set (which is also implied by
* VM_FAULT_ERROR).
*/
/* These three entries are valid only while holding ptl lock */
pte_t *pte; /* Pointer to pte entry matching
* the 'address'. NULL if the page
* table hasn't been allocated.
*/
spinlock_t *ptl; /* Page table lock.
* Protects pte page table if 'pte'
* is not NULL, otherwise pmd.
*/
pgtable_t prealloc_pte; /* Pre-allocated pte page table.
* vm_ops->map_pages() sets up a page
* table from atomic context.
* do_fault_around() pre-allocates
* page table to avoid allocation from
* atomic context.
*/
};
/*
* These are the virtual MM functions - opening of an area, closing and
* unmapping it (needed to keep files on disk up-to-date etc), pointer
* to the functions called when a no-page or a wp-page exception occurs.
*/
struct vm_operations_struct {
void (*open)(struct vm_area_struct * area);
/**
* @close: Called when the VMA is being removed from the MM.
* Context: User context. May sleep. Caller holds mmap_lock.
*/
void (*close)(struct vm_area_struct * area);
/* Called any time before splitting to check if it's allowed */
int (*may_split)(struct vm_area_struct *area, unsigned long addr);
int (*mremap)(struct vm_area_struct *area);
/*
* Called by mprotect() to make driver-specific permission
* checks before mprotect() is finalised. The VMA must not
* be modified. Returns 0 if mprotect() can proceed.
*/
int (*mprotect)(struct vm_area_struct *vma, unsigned long start,
unsigned long end, unsigned long newflags);
vm_fault_t (*fault)(struct vm_fault *vmf);
vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order);
vm_fault_t (*map_pages)(struct vm_fault *vmf,
pgoff_t start_pgoff, pgoff_t end_pgoff);
unsigned long (*pagesize)(struct vm_area_struct * area);
/* notification that a previously read-only page is about to become
* writable, if an error is returned it will cause a SIGBUS */
vm_fault_t (*page_mkwrite)(struct vm_fault *vmf);
/* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
/* called by access_process_vm when get_user_pages() fails, typically
* for use by special VMAs. See also generic_access_phys() for a generic
* implementation useful for any iomem mapping.
*/
int (*access)(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write);
/* Called by the /proc/PID/maps code to ask the vma whether it
* has a special name. Returning non-NULL will also cause this
* vma to be dumped unconditionally. */
const char *(*name)(struct vm_area_struct *vma);
#ifdef CONFIG_NUMA
/*
* set_policy() op must add a reference to any non-NULL @new mempolicy
* to hold the policy upon return. Caller should pass NULL @new to
* remove a policy and fall back to surrounding context--i.e. do not
* install a MPOL_DEFAULT policy, nor the task or system default
* mempolicy.
*/
int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
/*
* get_policy() op must add reference [mpol_get()] to any policy at
* (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
* in mm/mempolicy.c will do this automatically.
* get_policy() must NOT add a ref if the policy at (vma,addr) is not
* marked as MPOL_SHARED. vma policies are protected by the mmap_lock.
* If no [shared/vma] mempolicy exists at the addr, get_policy() op
* must return NULL--i.e., do not "fallback" to task or system default
* policy.
*/
struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
unsigned long addr, pgoff_t *ilx);
#endif
/*
* Called by vm_normal_page() for special PTEs to find the
* page for @addr. This is useful if the default behavior
* (using pte_page()) would not find the correct page.
*/
struct page *(*find_special_page)(struct vm_area_struct *vma,
unsigned long addr);
};
#ifdef CONFIG_NUMA_BALANCING
static inline void vma_numab_state_init(struct vm_area_struct *vma)
{
vma->numab_state = NULL;
}
static inline void vma_numab_state_free(struct vm_area_struct *vma)
{
kfree(vma->numab_state);
}
#else
static inline void vma_numab_state_init(struct vm_area_struct *vma) {}
static inline void vma_numab_state_free(struct vm_area_struct *vma) {}
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_PER_VMA_LOCK
/*
* Try to read-lock a vma. The function is allowed to occasionally yield false
* locked result to avoid performance overhead, in which case we fall back to
* using mmap_lock. The function should never yield false unlocked result.
*/
static inline bool vma_start_read(struct vm_area_struct *vma)
{
/*
* Check before locking. A race might cause false locked result.
* We can use READ_ONCE() for the mm_lock_seq here, and don't need
* ACQUIRE semantics, because this is just a lockless check whose result
* we don't rely on for anything - the mm_lock_seq read against which we
* need ordering is below.
*/
if (READ_ONCE(vma->vm_lock_seq) == READ_ONCE(vma->vm_mm->mm_lock_seq))
return false;
if (unlikely(down_read_trylock(&vma->vm_lock->lock) == 0))
return false;
/*
* Overflow might produce false locked result.
* False unlocked result is impossible because we modify and check
* vma->vm_lock_seq under vma->vm_lock protection and mm->mm_lock_seq
* modification invalidates all existing locks.
*
* We must use ACQUIRE semantics for the mm_lock_seq so that if we are
* racing with vma_end_write_all(), we only start reading from the VMA
* after it has been unlocked.
* This pairs with RELEASE semantics in vma_end_write_all().
*/
if (unlikely(vma->vm_lock_seq == smp_load_acquire(&vma->vm_mm->mm_lock_seq))) { up_read(&vma->vm_lock->lock);
return false;
}
return true;
}
static inline void vma_end_read(struct vm_area_struct *vma)
{
rcu_read_lock(); /* keeps vma alive till the end of up_read */ up_read(&vma->vm_lock->lock); rcu_read_unlock();
}
/* WARNING! Can only be used if mmap_lock is expected to be write-locked */
static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)
{
mmap_assert_write_locked(vma->vm_mm);
/*
* current task is holding mmap_write_lock, both vma->vm_lock_seq and
* mm->mm_lock_seq can't be concurrently modified.
*/
*mm_lock_seq = vma->vm_mm->mm_lock_seq;
return (vma->vm_lock_seq == *mm_lock_seq);
}
/*
* Begin writing to a VMA.
* Exclude concurrent readers under the per-VMA lock until the currently
* write-locked mmap_lock is dropped or downgraded.
*/
static inline void vma_start_write(struct vm_area_struct *vma)
{
int mm_lock_seq;
if (__is_vma_write_locked(vma, &mm_lock_seq))
return;
down_write(&vma->vm_lock->lock);
/*
* We should use WRITE_ONCE() here because we can have concurrent reads
* from the early lockless pessimistic check in vma_start_read().
* We don't really care about the correctness of that early check, but
* we should use WRITE_ONCE() for cleanliness and to keep KCSAN happy.
*/
WRITE_ONCE(vma->vm_lock_seq, mm_lock_seq);
up_write(&vma->vm_lock->lock);
}
static inline void vma_assert_write_locked(struct vm_area_struct *vma)
{
int mm_lock_seq;
VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma);
}
static inline void vma_assert_locked(struct vm_area_struct *vma)
{
if (!rwsem_is_locked(&vma->vm_lock->lock))
vma_assert_write_locked(vma);
}
static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached)
{
/* When detaching vma should be write-locked */
if (detached)
vma_assert_write_locked(vma);
vma->detached = detached;
}
static inline void release_fault_lock(struct vm_fault *vmf)
{
if (vmf->flags & FAULT_FLAG_VMA_LOCK)
vma_end_read(vmf->vma);
else
mmap_read_unlock(vmf->vma->vm_mm);
}
static inline void assert_fault_locked(struct vm_fault *vmf)
{
if (vmf->flags & FAULT_FLAG_VMA_LOCK)
vma_assert_locked(vmf->vma);
else
mmap_assert_locked(vmf->vma->vm_mm);
}
struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
unsigned long address);
#else /* CONFIG_PER_VMA_LOCK */
static inline bool vma_start_read(struct vm_area_struct *vma)
{ return false; }
static inline void vma_end_read(struct vm_area_struct *vma) {}
static inline void vma_start_write(struct vm_area_struct *vma) {}
static inline void vma_assert_write_locked(struct vm_area_struct *vma)
{ mmap_assert_write_locked(vma->vm_mm); }
static inline void vma_mark_detached(struct vm_area_struct *vma,
bool detached) {}
static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
unsigned long address)
{
return NULL;
}
static inline void vma_assert_locked(struct vm_area_struct *vma)
{
mmap_assert_locked(vma->vm_mm);
}
static inline void release_fault_lock(struct vm_fault *vmf)
{
mmap_read_unlock(vmf->vma->vm_mm);
}
static inline void assert_fault_locked(struct vm_fault *vmf)
{
mmap_assert_locked(vmf->vma->vm_mm);
}
#endif /* CONFIG_PER_VMA_LOCK */
extern const struct vm_operations_struct vma_dummy_vm_ops;
/*
* WARNING: vma_init does not initialize vma->vm_lock.
* Use vm_area_alloc()/vm_area_free() if vma needs locking.
*/
static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
{
memset(vma, 0, sizeof(*vma));
vma->vm_mm = mm;
vma->vm_ops = &vma_dummy_vm_ops;
INIT_LIST_HEAD(&vma->anon_vma_chain);
vma_mark_detached(vma, false);
vma_numab_state_init(vma);
}
/* Use when VMA is not part of the VMA tree and needs no locking */
static inline void vm_flags_init(struct vm_area_struct *vma,
vm_flags_t flags)
{
ACCESS_PRIVATE(vma, __vm_flags) = flags;
}
/*
* Use when VMA is part of the VMA tree and modifications need coordination
* Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and
* it should be locked explicitly beforehand.
*/
static inline void vm_flags_reset(struct vm_area_struct *vma,
vm_flags_t flags)
{
vma_assert_write_locked(vma);
vm_flags_init(vma, flags);
}
static inline void vm_flags_reset_once(struct vm_area_struct *vma,
vm_flags_t flags)
{
vma_assert_write_locked(vma);
WRITE_ONCE(ACCESS_PRIVATE(vma, __vm_flags), flags);
}
static inline void vm_flags_set(struct vm_area_struct *vma,
vm_flags_t flags)
{
vma_start_write(vma);
ACCESS_PRIVATE(vma, __vm_flags) |= flags;
}
static inline void vm_flags_clear(struct vm_area_struct *vma,
vm_flags_t flags)
{
vma_start_write(vma);
ACCESS_PRIVATE(vma, __vm_flags) &= ~flags;
}
/*
* Use only if VMA is not part of the VMA tree or has no other users and
* therefore needs no locking.
*/
static inline void __vm_flags_mod(struct vm_area_struct *vma,
vm_flags_t set, vm_flags_t clear)
{
vm_flags_init(vma, (vma->vm_flags | set) & ~clear);
}
/*
* Use only when the order of set/clear operations is unimportant, otherwise
* use vm_flags_{set|clear} explicitly.
*/
static inline void vm_flags_mod(struct vm_area_struct *vma,
vm_flags_t set, vm_flags_t clear)
{
vma_start_write(vma);
__vm_flags_mod(vma, set, clear);
}
static inline void vma_set_anonymous(struct vm_area_struct *vma)
{
vma->vm_ops = NULL;
}
static inline bool vma_is_anonymous(struct vm_area_struct *vma)
{
return !vma->vm_ops;
}
/*
* Indicate if the VMA is a heap for the given task; for
* /proc/PID/maps that is the heap of the main task.
*/
static inline bool vma_is_initial_heap(const struct vm_area_struct *vma)
{
return vma->vm_start < vma->vm_mm->brk &&
vma->vm_end > vma->vm_mm->start_brk;
}
/*
* Indicate if the VMA is a stack for the given task; for
* /proc/PID/maps that is the stack of the main task.
*/
static inline bool vma_is_initial_stack(const struct vm_area_struct *vma)
{
/*
* We make no effort to guess what a given thread considers to be
* its "stack". It's not even well-defined for programs written
* languages like Go.
*/
return vma->vm_start <= vma->vm_mm->start_stack &&
vma->vm_end >= vma->vm_mm->start_stack;
}
static inline bool vma_is_temporary_stack(struct vm_area_struct *vma)
{
int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
if (!maybe_stack)
return false;
if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
VM_STACK_INCOMPLETE_SETUP)
return true;
return false;
}
static inline bool vma_is_foreign(struct vm_area_struct *vma)
{
if (!current->mm)
return true;
if (current->mm != vma->vm_mm)
return true;
return false;
}
static inline bool vma_is_accessible(struct vm_area_struct *vma)
{
return vma->vm_flags & VM_ACCESS_FLAGS;
}
static inline bool is_shared_maywrite(vm_flags_t vm_flags)
{
return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
(VM_SHARED | VM_MAYWRITE);
}
static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma)
{
return is_shared_maywrite(vma->vm_flags);
}
static inline
struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max)
{
return mas_find(&vmi->mas, max - 1);
}
static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi)
{
/*
* Uses mas_find() to get the first VMA when the iterator starts.
* Calling mas_next() could skip the first entry.
*/
return mas_find(&vmi->mas, ULONG_MAX);
}
static inline
struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi)
{
return mas_next_range(&vmi->mas, ULONG_MAX);
}
static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi)
{
return mas_prev(&vmi->mas, 0);
}
static inline
struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
{
return mas_prev_range(&vmi->mas, 0);
}
static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
{
return vmi->mas.index;
}
static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
{
return vmi->mas.last + 1;
}
static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
unsigned long count)
{
return mas_expected_entries(&vmi->mas, count);
}
static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
unsigned long start, unsigned long end, gfp_t gfp)
{
__mas_set_range(&vmi->mas, start, end - 1);
mas_store_gfp(&vmi->mas, NULL, gfp);
if (unlikely(mas_is_err(&vmi->mas)))
return -ENOMEM;
return 0;
}
/* Free any unused preallocations */
static inline void vma_iter_free(struct vma_iterator *vmi)
{
mas_destroy(&vmi->mas);
}
static inline int vma_iter_bulk_store(struct vma_iterator *vmi,
struct vm_area_struct *vma)
{
vmi->mas.index = vma->vm_start;
vmi->mas.last = vma->vm_end - 1;
mas_store(&vmi->mas, vma);
if (unlikely(mas_is_err(&vmi->mas)))
return -ENOMEM;
return 0;
}
static inline void vma_iter_invalidate(struct vma_iterator *vmi)
{
mas_pause(&vmi->mas);
}
static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr)
{
mas_set(&vmi->mas, addr);
}
#define for_each_vma(__vmi, __vma) \
while (((__vma) = vma_next(&(__vmi))) != NULL)
/* The MM code likes to work with exclusive end addresses */
#define for_each_vma_range(__vmi, __vma, __end) \
while (((__vma) = vma_find(&(__vmi), (__end))) != NULL)
#ifdef CONFIG_SHMEM
/*
* The vma_is_shmem is not inline because it is used only by slow
* paths in userfault.
*/
bool vma_is_shmem(struct vm_area_struct *vma);
bool vma_is_anon_shmem(struct vm_area_struct *vma);
#else
static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; }
static inline bool vma_is_anon_shmem(struct vm_area_struct *vma) { return false; }
#endif
int vma_is_stack_for_current(struct vm_area_struct *vma);
/* flush_tlb_range() takes a vma, not a mm, and can care about flags */
#define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) }
struct mmu_gather;
struct inode;
/*
* compound_order() can be called without holding a reference, which means
* that niceties like page_folio() don't work. These callers should be
* prepared to handle wild return values. For example, PG_head may be
* set before the order is initialised, or this may be a tail page.
* See compaction.c for some good examples.
*/
static inline unsigned int compound_order(struct page *page)
{
struct folio *folio = (struct folio *)page;
if (!test_bit(PG_head, &folio->flags))
return 0;
return folio->_flags_1 & 0xff;
}
/**
* folio_order - The allocation order of a folio.
* @folio: The folio.
*
* A folio is composed of 2^order pages. See get_order() for the definition
* of order.
*
* Return: The order of the folio.
*/
static inline unsigned int folio_order(struct folio *folio)
{
if (!folio_test_large(folio))
return 0;
return folio->_flags_1 & 0xff;
}
#include <linux/huge_mm.h>
/*
* Methods to modify the page usage count.
*
* What counts for a page usage:
* - cache mapping (page->mapping)
* - private data (page->private)
* - page mapped in a task's page tables, each mapping
* is counted separately
*
* Also, many kernel routines increase the page count before a critical
* routine so they can be sure the page doesn't go away from under them.
*/
/*
* Drop a ref, return true if the refcount fell to zero (the page has no users)
*/
static inline int put_page_testzero(struct page *page)
{
VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); return page_ref_dec_and_test(page);
}
static inline int folio_put_testzero(struct folio *folio)
{
return put_page_testzero(&folio->page);
}
/*
* Try to grab a ref unless the page has a refcount of zero, return false if
* that is the case.
* This can be called when MMU is off so it must not access
* any of the virtual mappings.
*/
static inline bool get_page_unless_zero(struct page *page)
{
return page_ref_add_unless(page, 1, 0);
}
static inline struct folio *folio_get_nontail_page(struct page *page)
{
if (unlikely(!get_page_unless_zero(page)))
return NULL;
return (struct folio *)page;
}
extern int page_is_ram(unsigned long pfn);
enum {
REGION_INTERSECTS,
REGION_DISJOINT,
REGION_MIXED,
};
int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
unsigned long desc);
/* Support for virtually mapped pages */
struct page *vmalloc_to_page(const void *addr);
unsigned long vmalloc_to_pfn(const void *addr);
/*
* Determine if an address is within the vmalloc range
*
* On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
* is no special casing required.
*/
#ifdef CONFIG_MMU
extern bool is_vmalloc_addr(const void *x);
extern int is_vmalloc_or_module_addr(const void *x);
#else
static inline bool is_vmalloc_addr(const void *x)
{
return false;
}
static inline int is_vmalloc_or_module_addr(const void *x)
{
return 0;
}
#endif
/*
* How many times the entire folio is mapped as a single unit (eg by a
* PMD or PUD entry). This is probably not what you want, except for
* debugging purposes - it does not include PTE-mapped sub-pages; look
* at folio_mapcount() or page_mapcount() instead.
*/
static inline int folio_entire_mapcount(struct folio *folio)
{
VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
return atomic_read(&folio->_entire_mapcount) + 1;
}
/*
* The atomic page->_mapcount, starts from -1: so that transitions
* both from it and to it can be tracked, using atomic_inc_and_test
* and atomic_add_negative(-1).
*/
static inline void page_mapcount_reset(struct page *page)
{
atomic_set(&(page)->_mapcount, -1);
}
/**
* page_mapcount() - Number of times this precise page is mapped.
* @page: The page.
*
* The number of times this page is mapped. If this page is part of
* a large folio, it includes the number of times this page is mapped
* as part of that folio.
*
* Will report 0 for pages which cannot be mapped into userspace, eg
* slab, page tables and similar.
*/
static inline int page_mapcount(struct page *page)
{
int mapcount = atomic_read(&page->_mapcount) + 1;
/* Handle page_has_type() pages */
if (mapcount < 0)
mapcount = 0;
if (unlikely(PageCompound(page)))
mapcount += folio_entire_mapcount(page_folio(page));
return mapcount;
}
int folio_total_mapcount(struct folio *folio);
/**
* folio_mapcount() - Calculate the number of mappings of this folio.
* @folio: The folio.
*
* A large folio tracks both how many times the entire folio is mapped,
* and how many times each individual page in the folio is mapped.
* This function calculates the total number of times the folio is
* mapped.
*
* Return: The number of times this folio is mapped.
*/
static inline int folio_mapcount(struct folio *folio)
{
if (likely(!folio_test_large(folio)))
return atomic_read(&folio->_mapcount) + 1;
return folio_total_mapcount(folio);
}
static inline bool folio_large_is_mapped(struct folio *folio)
{
/*
* Reading _entire_mapcount below could be omitted if hugetlb
* participated in incrementing nr_pages_mapped when compound mapped.
*/
return atomic_read(&folio->_nr_pages_mapped) > 0 || atomic_read(&folio->_entire_mapcount) >= 0;
}
/**
* folio_mapped - Is this folio mapped into userspace?
* @folio: The folio.
*
* Return: True if any page in this folio is referenced by user page tables.
*/
static inline bool folio_mapped(struct folio *folio)
{
if (likely(!folio_test_large(folio))) return atomic_read(&folio->_mapcount) >= 0; return folio_large_is_mapped(folio);
}
/*
* Return true if this page is mapped into pagetables.
* For compound page it returns true if any sub-page of compound page is mapped,
* even if this particular sub-page is not itself mapped by any PTE or PMD.
*/
static inline bool page_mapped(struct page *page)
{
if (likely(!PageCompound(page)))
return atomic_read(&page->_mapcount) >= 0;
return folio_large_is_mapped(page_folio(page));
}
static inline struct page *virt_to_head_page(const void *x)
{
struct page *page = virt_to_page(x);
return compound_head(page);
}
static inline struct folio *virt_to_folio(const void *x)
{
struct page *page = virt_to_page(x); return page_folio(page);
}
void __folio_put(struct folio *folio);
void put_pages_list(struct list_head *pages);
void split_page(struct page *page, unsigned int order);
void folio_copy(struct folio *dst, struct folio *src);
unsigned long nr_free_buffer_pages(void);
void destroy_large_folio(struct folio *folio);
/* Returns the number of bytes in this potentially compound page. */
static inline unsigned long page_size(struct page *page)
{
return PAGE_SIZE << compound_order(page);
}
/* Returns the number of bits needed for the number of bytes in a page */
static inline unsigned int page_shift(struct page *page)
{
return PAGE_SHIFT + compound_order(page);
}
/**
* thp_order - Order of a transparent huge page.
* @page: Head page of a transparent huge page.
*/
static inline unsigned int thp_order(struct page *page)
{
VM_BUG_ON_PGFLAGS(PageTail(page), page);
return compound_order(page);
}
/**
* thp_size - Size of a transparent huge page.
* @page: Head page of a transparent huge page.
*
* Return: Number of bytes in this page.
*/
static inline unsigned long thp_size(struct page *page)
{
return PAGE_SIZE << thp_order(page);
}
#ifdef CONFIG_MMU
/*
* Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
* servicing faults for write access. In the normal case, do always want
* pte_mkwrite. But get_user_pages can cause write faults for mappings
* that do not have writing enabled, when used by access_process_vm.
*/
static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
{
if (likely(vma->vm_flags & VM_WRITE))
pte = pte_mkwrite(pte, vma);
return pte;
}
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
struct page *page, unsigned int nr, unsigned long addr);
vm_fault_t finish_fault(struct vm_fault *vmf);
#endif
/*
* Multiple processes may "see" the same page. E.g. for untouched
* mappings of /dev/null, all processes see the same page full of
* zeroes, and text pages of executables and shared libraries have
* only one copy in memory, at most, normally.
*
* For the non-reserved pages, page_count(page) denotes a reference count.
* page_count() == 0 means the page is free. page->lru is then used for
* freelist management in the buddy allocator.
* page_count() > 0 means the page has been allocated.
*
* Pages are allocated by the slab allocator in order to provide memory
* to kmalloc and kmem_cache_alloc. In this case, the management of the
* page, and the fields in 'struct page' are the responsibility of mm/slab.c
* unless a particular usage is carefully commented. (the responsibility of
* freeing the kmalloc memory is the caller's, of course).
*
* A page may be used by anyone else who does a __get_free_page().
* In this case, page_count still tracks the references, and should only
* be used through the normal accessor functions. The top bits of page->flags
* and page->virtual store page management information, but all other fields
* are unused and could be used privately, carefully. The management of this
* page is the responsibility of the one who allocated it, and those who have
* subsequently been given references to it.
*
* The other pages (we may call them "pagecache pages") are completely
* managed by the Linux memory manager: I/O, buffers, swapping etc.
* The following discussion applies only to them.
*
* A pagecache page contains an opaque `private' member, which belongs to the
* page's address_space. Usually, this is the address of a circular list of
* the page's disk buffers. PG_private must be set to tell the VM to call
* into the filesystem to release these pages.
*
* A page may belong to an inode's memory mapping. In this case, page->mapping
* is the pointer to the inode, and page->index is the file offset of the page,
* in units of PAGE_SIZE.
*
* If pagecache pages are not associated with an inode, they are said to be
* anonymous pages. These may become associated with the swapcache, and in that
* case PG_swapcache is set, and page->private is an offset into the swapcache.
*
* In either case (swapcache or inode backed), the pagecache itself holds one
* reference to the page. Setting PG_private should also increment the
* refcount. The each user mapping also has a reference to the page.
*
* The pagecache pages are stored in a per-mapping radix tree, which is
* rooted at mapping->i_pages, and indexed by offset.
* Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
* lists, we instead now tag pages as dirty/writeback in the radix tree.
*
* All pagecache pages may be subject to I/O:
* - inode pages may need to be read from disk,
* - inode pages which have been modified and are MAP_SHARED may need
* to be written back to the inode on disk,
* - anonymous pages (including MAP_PRIVATE file mappings) which have been
* modified may need to be swapped out to swap space and (later) to be read
* back into memory.
*/
#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_FS_DAX)
DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
bool __put_devmap_managed_page_refs(struct page *page, int refs);
static inline bool put_devmap_managed_page_refs(struct page *page, int refs)
{
if (!static_branch_unlikely(&devmap_managed_key))
return false;
if (!is_zone_device_page(page))
return false;
return __put_devmap_managed_page_refs(page, refs);
}
#else /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
static inline bool put_devmap_managed_page_refs(struct page *page, int refs)
{
return false;
}
#endif /* CONFIG_ZONE_DEVICE && CONFIG_FS_DAX */
static inline bool put_devmap_managed_page(struct page *page)
{
return put_devmap_managed_page_refs(page, 1);
}
/* 127: arbitrary random number, small enough to assemble well */
#define folio_ref_zero_or_close_to_overflow(folio) \
((unsigned int) folio_ref_count(folio) + 127u <= 127u)
/**
* folio_get - Increment the reference count on a folio.
* @folio: The folio.
*
* Context: May be called in any context, as long as you know that
* you have a refcount on the folio. If you do not already have one,
* folio_try_get() may be the right interface for you to use.
*/
static inline void folio_get(struct folio *folio)
{
VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); folio_ref_inc(folio);
}
static inline void get_page(struct page *page)
{
folio_get(page_folio(page));
}
static inline __must_check bool try_get_page(struct page *page)
{
page = compound_head(page);
if (WARN_ON_ONCE(page_ref_count(page) <= 0))
return false;
page_ref_inc(page);
return true;
}
/**
* folio_put - Decrement the reference count on a folio.
* @folio: The folio.
*
* If the folio's reference count reaches zero, the memory will be
* released back to the page allocator and may be used by another
* allocation immediately. Do not access the memory or the struct folio
* after calling folio_put() unless you can be sure that it wasn't the
* last reference.
*
* Context: May be called in process or interrupt context, but not in NMI
* context. May be called while holding a spinlock.
*/
static inline void folio_put(struct folio *folio)
{
if (folio_put_testzero(folio)) __folio_put(folio);
}
/**
* folio_put_refs - Reduce the reference count on a folio.
* @folio: The folio.
* @refs: The amount to subtract from the folio's reference count.
*
* If the folio's reference count reaches zero, the memory will be
* released back to the page allocator and may be used by another
* allocation immediately. Do not access the memory or the struct folio
* after calling folio_put_refs() unless you can be sure that these weren't
* the last references.
*
* Context: May be called in process or interrupt context, but not in NMI
* context. May be called while holding a spinlock.
*/
static inline void folio_put_refs(struct folio *folio, int refs)
{
if (folio_ref_sub_and_test(folio, refs)) __folio_put(folio);
}
void folios_put_refs(struct folio_batch *folios, unsigned int *refs);
/*
* union release_pages_arg - an array of pages or folios
*
* release_pages() releases a simple array of multiple pages, and
* accepts various different forms of said page array: either
* a regular old boring array of pages, an array of folios, or
* an array of encoded page pointers.
*
* The transparent union syntax for this kind of "any of these
* argument types" is all kinds of ugly, so look away.
*/
typedef union {
struct page **pages;
struct folio **folios;
struct encoded_page **encoded_pages;
} release_pages_arg __attribute__ ((__transparent_union__));
void release_pages(release_pages_arg, int nr);
/**
* folios_put - Decrement the reference count on an array of folios.
* @folios: The folios.
*
* Like folio_put(), but for a batch of folios. This is more efficient
* than writing the loop yourself as it will optimise the locks which need
* to be taken if the folios are freed. The folios batch is returned
* empty and ready to be reused for another batch; there is no need to
* reinitialise it.
*
* Context: May be called in process or interrupt context, but not in NMI
* context. May be called while holding a spinlock.
*/
static inline void folios_put(struct folio_batch *folios)
{
folios_put_refs(folios, NULL);
}
static inline void put_page(struct page *page)
{
struct folio *folio = page_folio(page);
/*
* For some devmap managed pages we need to catch refcount transition
* from 2 to 1:
*/
if (put_devmap_managed_page(&folio->page))
return;
folio_put(folio);
}
/*
* GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload
* the page's refcount so that two separate items are tracked: the original page
* reference count, and also a new count of how many pin_user_pages() calls were
* made against the page. ("gup-pinned" is another term for the latter).
*
* With this scheme, pin_user_pages() becomes special: such pages are marked as
* distinct from normal pages. As such, the unpin_user_page() call (and its
* variants) must be used in order to release gup-pinned pages.
*
* Choice of value:
*
* By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
* counts with respect to pin_user_pages() and unpin_user_page() becomes
* simpler, due to the fact that adding an even power of two to the page
* refcount has the effect of using only the upper N bits, for the code that
* counts up using the bias value. This means that the lower bits are left for
* the exclusive use of the original code that increments and decrements by one
* (or at least, by much smaller values than the bias value).
*
* Of course, once the lower bits overflow into the upper bits (and this is
* OK, because subtraction recovers the original values), then visual inspection
* no longer suffices to directly view the separate counts. However, for normal
* applications that don't have huge page reference counts, this won't be an
* issue.
*
* Locking: the lockless algorithm described in folio_try_get_rcu()
* provides safe operation for get_user_pages(), page_mkclean() and
* other calls that race to set up page table entries.
*/
#define GUP_PIN_COUNTING_BIAS (1U << 10)
void unpin_user_page(struct page *page);
void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
bool make_dirty);
void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
bool make_dirty);
void unpin_user_pages(struct page **pages, unsigned long npages);
static inline bool is_cow_mapping(vm_flags_t flags)
{
return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
}
#ifndef CONFIG_MMU
static inline bool is_nommu_shared_mapping(vm_flags_t flags)
{
/*
* NOMMU shared mappings are ordinary MAP_SHARED mappings and selected
* R/O MAP_PRIVATE file mappings that are an effective R/O overlay of
* a file mapping. R/O MAP_PRIVATE mappings might still modify
* underlying memory if ptrace is active, so this is only possible if
* ptrace does not apply. Note that there is no mprotect() to upgrade
* write permissions later.
*/
return flags & (VM_MAYSHARE | VM_MAYOVERLAY);
}
#endif
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
#define SECTION_IN_PAGE_FLAGS
#endif
/*
* The identification function is mainly used by the buddy allocator for
* determining if two pages could be buddies. We are not really identifying
* the zone since we could be using the section number id if we do not have
* node id available in page flags.
* We only guarantee that it will return the same value for two combinable
* pages in a zone.
*/
static inline int page_zone_id(struct page *page)
{
return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
}
#ifdef NODE_NOT_IN_PAGE_FLAGS
int page_to_nid(const struct page *page);
#else
static inline int page_to_nid(const struct page *page)
{
return (PF_POISONED_CHECK(page)->flags >> NODES_PGSHIFT) & NODES_MASK;
}
#endif
static inline int folio_nid(const struct folio *folio)
{
return page_to_nid(&folio->page);
}
#ifdef CONFIG_NUMA_BALANCING
/* page access time bits needs to hold at least 4 seconds */
#define PAGE_ACCESS_TIME_MIN_BITS 12
#if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS
#define PAGE_ACCESS_TIME_BUCKETS \
(PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT)
#else
#define PAGE_ACCESS_TIME_BUCKETS 0
#endif
#define PAGE_ACCESS_TIME_MASK \
(LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS)
static inline int cpu_pid_to_cpupid(int cpu, int pid)
{
return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
}
static inline int cpupid_to_pid(int cpupid)
{
return cpupid & LAST__PID_MASK;
}
static inline int cpupid_to_cpu(int cpupid)
{
return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
}
static inline int cpupid_to_nid(int cpupid)
{
return cpu_to_node(cpupid_to_cpu(cpupid));
}
static inline bool cpupid_pid_unset(int cpupid)
{
return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
}
static inline bool cpupid_cpu_unset(int cpupid)
{
return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
}
static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
{
return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
}
#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
{
return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK);
}
static inline int folio_last_cpupid(struct folio *folio)
{
return folio->_last_cpupid;
}
static inline void page_cpupid_reset_last(struct page *page)
{
page->_last_cpupid = -1 & LAST_CPUPID_MASK;
}
#else
static inline int folio_last_cpupid(struct folio *folio)
{
return (folio->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
}
int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
static inline void page_cpupid_reset_last(struct page *page)
{
page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
}
#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
static inline int folio_xchg_access_time(struct folio *folio, int time)
{
int last_time;
last_time = folio_xchg_last_cpupid(folio,
time >> PAGE_ACCESS_TIME_BUCKETS);
return last_time << PAGE_ACCESS_TIME_BUCKETS;
}
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
{
unsigned int pid_bit;
pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG));
if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) {
__set_bit(pid_bit, &vma->numab_state->pids_active[1]);
}
}
#else /* !CONFIG_NUMA_BALANCING */
static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
{
return folio_nid(folio); /* XXX */
}
static inline int folio_xchg_access_time(struct folio *folio, int time)
{
return 0;
}
static inline int folio_last_cpupid(struct folio *folio)
{
return folio_nid(folio); /* XXX */
}
static inline int cpupid_to_nid(int cpupid)
{
return -1;
}
static inline int cpupid_to_pid(int cpupid)
{
return -1;
}
static inline int cpupid_to_cpu(int cpupid)
{
return -1;
}
static inline int cpu_pid_to_cpupid(int nid, int pid)
{
return -1;
}
static inline bool cpupid_pid_unset(int cpupid)
{
return true;
}
static inline void page_cpupid_reset_last(struct page *page)
{
}
static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
{
return false;
}
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
{
}
#endif /* CONFIG_NUMA_BALANCING */
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
/*
* KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid
* setting tags for all pages to native kernel tag value 0xff, as the default
* value 0x00 maps to 0xff.
*/
static inline u8 page_kasan_tag(const struct page *page)
{
u8 tag = KASAN_TAG_KERNEL;
if (kasan_enabled()) {
tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;
tag ^= 0xff;
}
return tag;
}
static inline void page_kasan_tag_set(struct page *page, u8 tag)
{
unsigned long old_flags, flags;
if (!kasan_enabled())
return;
tag ^= 0xff;
old_flags = READ_ONCE(page->flags);
do {
flags = old_flags;
flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
} while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));
}
static inline void page_kasan_tag_reset(struct page *page)
{
if (kasan_enabled())
page_kasan_tag_set(page, KASAN_TAG_KERNEL);
}
#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline u8 page_kasan_tag(const struct page *page)
{
return 0xff;
}
static inline void page_kasan_tag_set(struct page *page, u8 tag) { }
static inline void page_kasan_tag_reset(struct page *page) { }
#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline struct zone *page_zone(const struct page *page)
{
return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
}
static inline pg_data_t *page_pgdat(const struct page *page)
{
return NODE_DATA(page_to_nid(page));
}
static inline struct zone *folio_zone(const struct folio *folio)
{
return page_zone(&folio->page);
}
static inline pg_data_t *folio_pgdat(const struct folio *folio)
{
return page_pgdat(&folio->page);
}
#ifdef SECTION_IN_PAGE_FLAGS
static inline void set_page_section(struct page *page, unsigned long section)
{
page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
}
static inline unsigned long page_to_section(const struct page *page)
{
return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
}
#endif
/**
* folio_pfn - Return the Page Frame Number of a folio.
* @folio: The folio.
*
* A folio may contain multiple pages. The pages have consecutive
* Page Frame Numbers.
*
* Return: The Page Frame Number of the first page in the folio.
*/
static inline unsigned long folio_pfn(struct folio *folio)
{
return page_to_pfn(&folio->page);
}
static inline struct folio *pfn_folio(unsigned long pfn)
{
return page_folio(pfn_to_page(pfn));
}
/**
* folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
* @folio: The folio.
*
* This function checks if a folio has been pinned via a call to
* a function in the pin_user_pages() family.
*
* For small folios, the return value is partially fuzzy: false is not fuzzy,
* because it means "definitely not pinned for DMA", but true means "probably
* pinned for DMA, but possibly a false positive due to having at least
* GUP_PIN_COUNTING_BIAS worth of normal folio references".
*
* False positives are OK, because: a) it's unlikely for a folio to
* get that many refcounts, and b) all the callers of this routine are
* expected to be able to deal gracefully with a false positive.
*
* For large folios, the result will be exactly correct. That's because
* we have more tracking data available: the _pincount field is used
* instead of the GUP_PIN_COUNTING_BIAS scheme.
*
* For more information, please see Documentation/core-api/pin_user_pages.rst.
*
* Return: True, if it is likely that the page has been "dma-pinned".
* False, if the page is definitely not dma-pinned.
*/
static inline bool folio_maybe_dma_pinned(struct folio *folio)
{
if (folio_test_large(folio))
return atomic_read(&folio->_pincount) > 0;
/*
* folio_ref_count() is signed. If that refcount overflows, then
* folio_ref_count() returns a negative value, and callers will avoid
* further incrementing the refcount.
*
* Here, for that overflow case, use the sign bit to count a little
* bit higher via unsigned math, and thus still get an accurate result.
*/
return ((unsigned int)folio_ref_count(folio)) >=
GUP_PIN_COUNTING_BIAS;
}
static inline bool page_maybe_dma_pinned(struct page *page)
{
return folio_maybe_dma_pinned(page_folio(page));
}
/*
* This should most likely only be called during fork() to see whether we
* should break the cow immediately for an anon page on the src mm.
*
* The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq.
*/
static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma,
struct folio *folio)
{
VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1));
if (!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags))
return false;
return folio_maybe_dma_pinned(folio);
}
/**
* is_zero_page - Query if a page is a zero page
* @page: The page to query
*
* This returns true if @page is one of the permanent zero pages.
*/
static inline bool is_zero_page(const struct page *page)
{
return is_zero_pfn(page_to_pfn(page));
}
/**
* is_zero_folio - Query if a folio is a zero page
* @folio: The folio to query
*
* This returns true if @folio is one of the permanent zero pages.
*/
static inline bool is_zero_folio(const struct folio *folio)
{
return is_zero_page(&folio->page);
}
/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */
#ifdef CONFIG_MIGRATION
static inline bool folio_is_longterm_pinnable(struct folio *folio)
{
#ifdef CONFIG_CMA
int mt = folio_migratetype(folio);
if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE)
return false;
#endif
/* The zero page can be "pinned" but gets special handling. */
if (is_zero_folio(folio))
return true;
/* Coherent device memory must always allow eviction. */
if (folio_is_device_coherent(folio))
return false;
/* Otherwise, non-movable zone folios can be pinned. */
return !folio_is_zone_movable(folio);
}
#else
static inline bool folio_is_longterm_pinnable(struct folio *folio)
{
return true;
}
#endif
static inline void set_page_zone(struct page *page, enum zone_type zone)
{
page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
}
static inline void set_page_node(struct page *page, unsigned long node)
{
page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
}
static inline void set_page_links(struct page *page, enum zone_type zone,
unsigned long node, unsigned long pfn)
{
set_page_zone(page, zone);
set_page_node(page, node);
#ifdef SECTION_IN_PAGE_FLAGS
set_page_section(page, pfn_to_section_nr(pfn));
#endif
}
/**
* folio_nr_pages - The number of pages in the folio.
* @folio: The folio.
*
* Return: A positive power of two.
*/
static inline long folio_nr_pages(struct folio *folio)
{
if (!folio_test_large(folio))
return 1;
#ifdef CONFIG_64BIT
return folio->_folio_nr_pages;
#else
return 1L << (folio->_flags_1 & 0xff);
#endif
}
/* Only hugetlbfs can allocate folios larger than MAX_ORDER */
#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
#define MAX_FOLIO_NR_PAGES (1UL << PUD_ORDER)
#else
#define MAX_FOLIO_NR_PAGES MAX_ORDER_NR_PAGES
#endif
/*
* compound_nr() returns the number of pages in this potentially compound
* page. compound_nr() can be called on a tail page, and is defined to
* return 1 in that case.
*/
static inline unsigned long compound_nr(struct page *page)
{
struct folio *folio = (struct folio *)page;
if (!test_bit(PG_head, &folio->flags))
return 1;
#ifdef CONFIG_64BIT
return folio->_folio_nr_pages;
#else
return 1L << (folio->_flags_1 & 0xff);
#endif
}
/**
* thp_nr_pages - The number of regular pages in this huge page.
* @page: The head page of a huge page.
*/
static inline int thp_nr_pages(struct page *page)
{
return folio_nr_pages((struct folio *)page);
}
/**
* folio_next - Move to the next physical folio.
* @folio: The folio we're currently operating on.
*
* If you have physically contiguous memory which may span more than
* one folio (eg a &struct bio_vec), use this function to move from one
* folio to the next. Do not use it if the memory is only virtually
* contiguous as the folios are almost certainly not adjacent to each
* other. This is the folio equivalent to writing ``page++``.
*
* Context: We assume that the folios are refcounted and/or locked at a
* higher level and do not adjust the reference counts.
* Return: The next struct folio.
*/
static inline struct folio *folio_next(struct folio *folio)
{
return (struct folio *)folio_page(folio, folio_nr_pages(folio));
}
/**
* folio_shift - The size of the memory described by this folio.
* @folio: The folio.
*
* A folio represents a number of bytes which is a power-of-two in size.
* This function tells you which power-of-two the folio is. See also
* folio_size() and folio_order().
*
* Context: The caller should have a reference on the folio to prevent
* it from being split. It is not necessary for the folio to be locked.
* Return: The base-2 logarithm of the size of this folio.
*/
static inline unsigned int folio_shift(struct folio *folio)
{
return PAGE_SHIFT + folio_order(folio);
}
/**
* folio_size - The number of bytes in a folio.
* @folio: The folio.
*
* Context: The caller should have a reference on the folio to prevent
* it from being split. It is not necessary for the folio to be locked.
* Return: The number of bytes in this folio.
*/
static inline size_t folio_size(struct folio *folio)
{
return PAGE_SIZE << folio_order(folio);
}
/**
* folio_estimated_sharers - Estimate the number of sharers of a folio.
* @folio: The folio.
*
* folio_estimated_sharers() aims to serve as a function to efficiently
* estimate the number of processes sharing a folio. This is done by
* looking at the precise mapcount of the first subpage in the folio, and
* assuming the other subpages are the same. This may not be true for large
* folios. If you want exact mapcounts for exact calculations, look at
* page_mapcount() or folio_total_mapcount().
*
* Return: The estimated number of processes sharing a folio.
*/
static inline int folio_estimated_sharers(struct folio *folio)
{
return page_mapcount(folio_page(folio, 0));
}
#ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE
static inline int arch_make_page_accessible(struct page *page)
{
return 0;
}
#endif
#ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE
static inline int arch_make_folio_accessible(struct folio *folio)
{
int ret;
long i, nr = folio_nr_pages(folio);
for (i = 0; i < nr; i++) {
ret = arch_make_page_accessible(folio_page(folio, i));
if (ret)
break;
}
return ret;
}
#endif
/*
* Some inline functions in vmstat.h depend on page_zone()
*/
#include <linux/vmstat.h>
#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
#define HASHED_PAGE_VIRTUAL
#endif
#if defined(WANT_PAGE_VIRTUAL)
static inline void *page_address(const struct page *page)
{
return page->virtual;
}
static inline void set_page_address(struct page *page, void *address)
{
page->virtual = address;
}
#define page_address_init() do { } while(0)
#endif
#if defined(HASHED_PAGE_VIRTUAL)
void *page_address(const struct page *page);
void set_page_address(struct page *page, void *virtual);
void page_address_init(void);
#endif
static __always_inline void *lowmem_page_address(const struct page *page)
{
return page_to_virt(page);
}
#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
#define page_address(page) lowmem_page_address(page)
#define set_page_address(page, address) do { } while(0)
#define page_address_init() do { } while(0)
#endif
static inline void *folio_address(const struct folio *folio)
{
return page_address(&folio->page);
}
extern pgoff_t __page_file_index(struct page *page);
/*
* Return the pagecache index of the passed page. Regular pagecache pages
* use ->index whereas swapcache pages use swp_offset(->private)
*/
static inline pgoff_t page_index(struct page *page)
{
if (unlikely(PageSwapCache(page)))
return __page_file_index(page);
return page->index;
}
/*
* Return true only if the page has been allocated with
* ALLOC_NO_WATERMARKS and the low watermark was not
* met implying that the system is under some pressure.
*/
static inline bool page_is_pfmemalloc(const struct page *page)
{
/*
* lru.next has bit 1 set if the page is allocated from the
* pfmemalloc reserves. Callers may simply overwrite it if
* they do not need to preserve that information.
*/
return (uintptr_t)page->lru.next & BIT(1);
}
/*
* Return true only if the folio has been allocated with
* ALLOC_NO_WATERMARKS and the low watermark was not
* met implying that the system is under some pressure.
*/
static inline bool folio_is_pfmemalloc(const struct folio *folio)
{
/*
* lru.next has bit 1 set if the page is allocated from the
* pfmemalloc reserves. Callers may simply overwrite it if
* they do not need to preserve that information.
*/
return (uintptr_t)folio->lru.next & BIT(1);
}
/*
* Only to be called by the page allocator on a freshly allocated
* page.
*/
static inline void set_page_pfmemalloc(struct page *page)
{
page->lru.next = (void *)BIT(1);
}
static inline void clear_page_pfmemalloc(struct page *page)
{
page->lru.next = NULL;
}
/*
* Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
*/
extern void pagefault_out_of_memory(void);
#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
#define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1))
#define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1))
/*
* Parameter block passed down to zap_pte_range in exceptional cases.
*/
struct zap_details {
struct folio *single_folio; /* Locked folio to be unmapped */
bool even_cows; /* Zap COWed private pages too? */
zap_flags_t zap_flags; /* Extra flags for zapping */
};
/*
* Whether to drop the pte markers, for example, the uffd-wp information for
* file-backed memory. This should only be specified when we will completely
* drop the page in the mm, either by truncation or unmapping of the vma. By
* default, the flag is not set.
*/
#define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0))
/* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */
#define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1))
#ifdef CONFIG_SCHED_MM_CID
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit_signals(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid;
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
* disabled. This provides functional per-cpu data structure accesses
* in user-space, althrough it won't provide the memory usage benefits.
*/
return raw_smp_processor_id();
}
#endif
#ifdef CONFIG_MMU
extern bool can_do_mlock(void);
#else
static inline bool can_do_mlock(void) { return false; }
#endif
extern int user_shm_lock(size_t, struct ucounts *);
extern void user_shm_unlock(size_t, struct ucounts *);
struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
pte_t pte);
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
pte_t pte);
struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
unsigned long addr, pmd_t pmd);
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
pmd_t pmd);
void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
unsigned long size);
void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
unsigned long size, struct zap_details *details);
static inline void zap_vma_pages(struct vm_area_struct *vma)
{
zap_page_range_single(vma, vma->vm_start,
vma->vm_end - vma->vm_start, NULL);
}
void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
struct vm_area_struct *start_vma, unsigned long start,
unsigned long end, unsigned long tree_end, bool mm_wr_locked);
struct mmu_notifier_range;
void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
unsigned long end, unsigned long floor, unsigned long ceiling);
int
copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
int follow_pte(struct mm_struct *mm, unsigned long address,
pte_t **ptepp, spinlock_t **ptlp);
int follow_pfn(struct vm_area_struct *vma, unsigned long address,
unsigned long *pfn);
int follow_phys(struct vm_area_struct *vma, unsigned long address,
unsigned int flags, unsigned long *prot, resource_size_t *phys);
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write);
extern void truncate_pagecache(struct inode *inode, loff_t new);
extern void truncate_setsize(struct inode *inode, loff_t newsize);
void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
int generic_error_remove_folio(struct address_space *mapping,
struct folio *folio);
struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
unsigned long address, struct pt_regs *regs);
#ifdef CONFIG_MMU
extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
unsigned long address, unsigned int flags,
struct pt_regs *regs);
extern int fixup_user_fault(struct mm_struct *mm,
unsigned long address, unsigned int fault_flags,
bool *unlocked);
void unmap_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t nr, bool even_cows);
void unmap_mapping_range(struct address_space *mapping,
loff_t const holebegin, loff_t const holelen, int even_cows);
#else
static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
unsigned long address, unsigned int flags,
struct pt_regs *regs)
{
/* should never happen if there's no MMU */
BUG();
return VM_FAULT_SIGBUS;
}
static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address,
unsigned int fault_flags, bool *unlocked)
{
/* should never happen if there's no MMU */
BUG();
return -EFAULT;
}
static inline void unmap_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t nr, bool even_cows) { }
static inline void unmap_mapping_range(struct address_space *mapping,
loff_t const holebegin, loff_t const holelen, int even_cows) { }
#endif
static inline void unmap_shared_mapping_range(struct address_space *mapping,
loff_t const holebegin, loff_t const holelen)
{
unmap_mapping_range(mapping, holebegin, holelen, 0);
}
static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm,
unsigned long addr);
extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
void *buf, int len, unsigned int gup_flags);
extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
void *buf, int len, unsigned int gup_flags);
long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
int *locked);
long pin_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
int *locked);
/*
* Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT.
*/
static inline struct page *get_user_page_vma_remote(struct mm_struct *mm,
unsigned long addr,
int gup_flags,
struct vm_area_struct **vmap)
{
struct page *page;
struct vm_area_struct *vma;
int got;
if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT)))
return ERR_PTR(-EINVAL);
got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL);
if (got < 0)
return ERR_PTR(got);
vma = vma_lookup(mm, addr);
if (WARN_ON_ONCE(!vma)) {
put_page(page);
return ERR_PTR(-EINVAL);
}
*vmap = vma;
return page;
}
long get_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages);
long pin_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages);
long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
struct page **pages, unsigned int gup_flags);
long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
struct page **pages, unsigned int gup_flags);
int get_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages);
int pin_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages);
void folio_add_pin(struct folio *folio);
int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
struct task_struct *task, bool bypass_rlim);
struct kvec;
struct page *get_dump_page(unsigned long addr);
bool folio_mark_dirty(struct folio *folio);
bool set_page_dirty(struct page *page);
int set_page_dirty_lock(struct page *page);
int get_cmdline(struct task_struct *task, char *buffer, int buflen);
extern unsigned long move_page_tables(struct vm_area_struct *vma,
unsigned long old_addr, struct vm_area_struct *new_vma,
unsigned long new_addr, unsigned long len,
bool need_rmap_locks, bool for_stack);
/*
* Flags used by change_protection(). For now we make it a bitmap so
* that we can pass in multiple flags just like parameters. However
* for now all the callers are only use one of the flags at the same
* time.
*/
/*
* Whether we should manually check if we can map individual PTEs writable,
* because something (e.g., COW, uffd-wp) blocks that from happening for all
* PTEs automatically in a writable mapping.
*/
#define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0)
/* Whether this protection change is for NUMA hints */
#define MM_CP_PROT_NUMA (1UL << 1)
/* Whether this change is for write protecting */
#define MM_CP_UFFD_WP (1UL << 2) /* do wp */
#define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */
#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \
MM_CP_UFFD_WP_RESOLVE)
bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
{
/*
* We want to check manually if we can change individual PTEs writable
* if we can't do that automatically for all PTEs in a mapping. For
* private mappings, that's always the case when we have write
* permissions as we properly have to handle COW.
*/
if (vma->vm_flags & VM_SHARED)
return vma_wants_writenotify(vma, vma->vm_page_prot);
return !!(vma->vm_flags & VM_WRITE);
}
bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
pte_t pte);
extern long change_protection(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long start,
unsigned long end, unsigned long cp_flags);
extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
struct vm_area_struct *vma, struct vm_area_struct **pprev,
unsigned long start, unsigned long end, unsigned long newflags);
/*
* doesn't attempt to fault and will return short.
*/
int get_user_pages_fast_only(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages);
static inline bool get_user_page_fast_only(unsigned long addr,
unsigned int gup_flags, struct page **pagep)
{
return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;
}
/*
* per-process(per-mm_struct) statistics.
*/
static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
{
return percpu_counter_read_positive(&mm->rss_stat[member]);
}
void mm_trace_rss_stat(struct mm_struct *mm, int member);
static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
{
percpu_counter_add(&mm->rss_stat[member], value);
mm_trace_rss_stat(mm, member);
}
static inline void inc_mm_counter(struct mm_struct *mm, int member)
{
percpu_counter_inc(&mm->rss_stat[member]);
mm_trace_rss_stat(mm, member);
}
static inline void dec_mm_counter(struct mm_struct *mm, int member)
{
percpu_counter_dec(&mm->rss_stat[member]);
mm_trace_rss_stat(mm, member);
}
/* Optimized variant when folio is already known not to be anon */
static inline int mm_counter_file(struct folio *folio)
{
if (folio_test_swapbacked(folio))
return MM_SHMEMPAGES;
return MM_FILEPAGES;
}
static inline int mm_counter(struct folio *folio)
{
if (folio_test_anon(folio))
return MM_ANONPAGES;
return mm_counter_file(folio);
}
static inline unsigned long get_mm_rss(struct mm_struct *mm)
{
return get_mm_counter(mm, MM_FILEPAGES) +
get_mm_counter(mm, MM_ANONPAGES) +
get_mm_counter(mm, MM_SHMEMPAGES);
}
static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
{
return max(mm->hiwater_rss, get_mm_rss(mm));
}
static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
{
return max(mm->hiwater_vm, mm->total_vm);
}
static inline void update_hiwater_rss(struct mm_struct *mm)
{
unsigned long _rss = get_mm_rss(mm);
if ((mm)->hiwater_rss < _rss)
(mm)->hiwater_rss = _rss;
}
static inline void update_hiwater_vm(struct mm_struct *mm)
{
if (mm->hiwater_vm < mm->total_vm)
mm->hiwater_vm = mm->total_vm;
}
static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
{
mm->hiwater_rss = get_mm_rss(mm);
}
static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
struct mm_struct *mm)
{
unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
if (*maxrss < hiwater_rss)
*maxrss = hiwater_rss;
}
#ifndef CONFIG_ARCH_HAS_PTE_SPECIAL
static inline int pte_special(pte_t pte)
{
return 0;
}
static inline pte_t pte_mkspecial(pte_t pte)
{
return pte;
}
#endif
#ifndef CONFIG_ARCH_HAS_PTE_DEVMAP
static inline int pte_devmap(pte_t pte)
{
return 0;
}
#endif
extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
spinlock_t **ptl);
static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
spinlock_t **ptl)
{
pte_t *ptep;
__cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
return ptep;
}
#ifdef __PAGETABLE_P4D_FOLDED
static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
unsigned long address)
{
return 0;
}
#else
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
#endif
#if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)
static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
unsigned long address)
{
return 0;
}
static inline void mm_inc_nr_puds(struct mm_struct *mm) {}
static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
#else
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
static inline void mm_inc_nr_puds(struct mm_struct *mm)
{
if (mm_pud_folded(mm))
return;
atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
}
static inline void mm_dec_nr_puds(struct mm_struct *mm)
{
if (mm_pud_folded(mm))
return;
atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
}
#endif
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
unsigned long address)
{
return 0;
}
static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
#else
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
static inline void mm_inc_nr_pmds(struct mm_struct *mm)
{
if (mm_pmd_folded(mm))
return;
atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
}
static inline void mm_dec_nr_pmds(struct mm_struct *mm)
{
if (mm_pmd_folded(mm))
return;
atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
}
#endif
#ifdef CONFIG_MMU
static inline void mm_pgtables_bytes_init(struct mm_struct *mm)
{
atomic_long_set(&mm->pgtables_bytes, 0);
}
static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
{
return atomic_long_read(&mm->pgtables_bytes);
}
static inline void mm_inc_nr_ptes(struct mm_struct *mm)
{
atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
}
static inline void mm_dec_nr_ptes(struct mm_struct *mm)
{
atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
}
#else
static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}
static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
{
return 0;
}
static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}
static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
#endif
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);
int __pte_alloc_kernel(pmd_t *pmd);
#if defined(CONFIG_MMU)
static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
unsigned long address)
{
return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
NULL : p4d_offset(pgd, address);
}
static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
unsigned long address)
{
return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? NULL : pud_offset(p4d, address);
}
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? NULL: pmd_offset(pud, address);
}
#endif /* CONFIG_MMU */
static inline struct ptdesc *virt_to_ptdesc(const void *x)
{
return page_ptdesc(virt_to_page(x));
}
static inline void *ptdesc_to_virt(const struct ptdesc *pt)
{
return page_to_virt(ptdesc_page(pt));
}
static inline void *ptdesc_address(const struct ptdesc *pt)
{
return folio_address(ptdesc_folio(pt));
}
static inline bool pagetable_is_reserved(struct ptdesc *pt)
{
return folio_test_reserved(ptdesc_folio(pt));
}
/**
* pagetable_alloc - Allocate pagetables
* @gfp: GFP flags
* @order: desired pagetable order
*
* pagetable_alloc allocates memory for page tables as well as a page table
* descriptor to describe that memory.
*
* Return: The ptdesc describing the allocated page tables.
*/
static inline struct ptdesc *pagetable_alloc(gfp_t gfp, unsigned int order)
{
struct page *page = alloc_pages(gfp | __GFP_COMP, order);
return page_ptdesc(page);
}
/**
* pagetable_free - Free pagetables
* @pt: The page table descriptor
*
* pagetable_free frees the memory of all page tables described by a page
* table descriptor and the memory for the descriptor itself.
*/
static inline void pagetable_free(struct ptdesc *pt)
{
struct page *page = ptdesc_page(pt);
__free_pages(page, compound_order(page));
}
#if USE_SPLIT_PTE_PTLOCKS
#if ALLOC_SPLIT_PTLOCKS
void __init ptlock_cache_init(void);
bool ptlock_alloc(struct ptdesc *ptdesc);
void ptlock_free(struct ptdesc *ptdesc);
static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
{
return ptdesc->ptl;
}
#else /* ALLOC_SPLIT_PTLOCKS */
static inline void ptlock_cache_init(void)
{
}
static inline bool ptlock_alloc(struct ptdesc *ptdesc)
{
return true;
}
static inline void ptlock_free(struct ptdesc *ptdesc)
{
}
static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
{
return &ptdesc->ptl;
}
#endif /* ALLOC_SPLIT_PTLOCKS */
static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
return ptlock_ptr(page_ptdesc(pmd_page(*pmd)));
}
static inline bool ptlock_init(struct ptdesc *ptdesc)
{
/*
* prep_new_page() initialize page->private (and therefore page->ptl)
* with 0. Make sure nobody took it in use in between.
*
* It can happen if arch try to use slab for page table allocation:
* slab code uses page->slab_cache, which share storage with page->ptl.
*/
VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); if (!ptlock_alloc(ptdesc))
return false;
spin_lock_init(ptlock_ptr(ptdesc));
return true;
}
#else /* !USE_SPLIT_PTE_PTLOCKS */
/*
* We use mm->page_table_lock to guard all pagetable pages of the mm.
*/
static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
return &mm->page_table_lock;
}
static inline void ptlock_cache_init(void) {}
static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; }
static inline void ptlock_free(struct ptdesc *ptdesc) {}
#endif /* USE_SPLIT_PTE_PTLOCKS */
static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
if (!ptlock_init(ptdesc))
return false;
__folio_set_pgtable(folio); lruvec_stat_add_folio(folio, NR_PAGETABLE);
return true;
}
static inline void pagetable_pte_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
ptlock_free(ptdesc); __folio_clear_pgtable(folio); lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr)
{
return __pte_offset_map(pmd, addr, NULL);
}
pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp);
static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp)
{
pte_t *pte;
__cond_lock(*ptlp, pte = __pte_offset_map_lock(mm, pmd, addr, ptlp));
return pte;
}
pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp);
#define pte_unmap_unlock(pte, ptl) do { \
spin_unlock(ptl); \
pte_unmap(pte); \
} while (0)
#define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd))
#define pte_alloc_map(mm, pmd, address) \
(pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address))
#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
(pte_alloc(mm, pmd) ? \
NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
#define pte_alloc_kernel(pmd, address) \
((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \
NULL: pte_offset_kernel(pmd, address))
#if USE_SPLIT_PMD_PTLOCKS
static inline struct page *pmd_pgtable_page(pmd_t *pmd)
{
unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
return virt_to_page((void *)((unsigned long) pmd & mask));
}
static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd)
{
return page_ptdesc(pmd_pgtable_page(pmd));
}
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
return ptlock_ptr(pmd_ptdesc(pmd));
}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
ptdesc->pmd_huge_pte = NULL;
#endif
return ptlock_init(ptdesc);
}
static inline void pmd_ptlock_free(struct ptdesc *ptdesc)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc));
#endif
ptlock_free(ptdesc);
}
#define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
#else
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
return &mm->page_table_lock;
}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {}
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
#endif
static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
{
spinlock_t *ptl = pmd_lockptr(mm, pmd);
spin_lock(ptl);
return ptl;
}
static inline bool pagetable_pmd_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
if (!pmd_ptlock_init(ptdesc))
return false;
__folio_set_pgtable(folio); lruvec_stat_add_folio(folio, NR_PAGETABLE);
return true;
}
static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
pmd_ptlock_free(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
/*
* No scalability reason to split PUD locks yet, but follow the same pattern
* as the PMD locks to make it easier if we decide to. The VM should not be
* considered ready to switch to split PUD locks yet; there may be places
* which need to be converted from page_table_lock.
*/
static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud)
{
return &mm->page_table_lock;
}
static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
{
spinlock_t *ptl = pud_lockptr(mm, pud);
spin_lock(ptl);
return ptl;
}
static inline void pagetable_pud_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
__folio_set_pgtable(folio);
lruvec_stat_add_folio(folio, NR_PAGETABLE);
}
static inline void pagetable_pud_dtor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
__folio_clear_pgtable(folio);
lruvec_stat_sub_folio(folio, NR_PAGETABLE);
}
extern void __init pagecache_init(void);
extern void free_initmem(void);
/*
* Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
* into the buddy system. The freed pages will be poisoned with pattern
* "poison" if it's within range [0, UCHAR_MAX].
* Return pages freed into the buddy system.
*/
extern unsigned long free_reserved_area(void *start, void *end,
int poison, const char *s);
extern void adjust_managed_page_count(struct page *page, long count);
extern void reserve_bootmem_region(phys_addr_t start,
phys_addr_t end, int nid);
/* Free the reserved page into the buddy system, so it gets managed. */
static inline void free_reserved_page(struct page *page)
{
ClearPageReserved(page);
init_page_count(page);
__free_page(page);
adjust_managed_page_count(page, 1);
}
#define free_highmem_page(page) free_reserved_page(page)
static inline void mark_page_reserved(struct page *page)
{
SetPageReserved(page);
adjust_managed_page_count(page, -1);
}
static inline void free_reserved_ptdesc(struct ptdesc *pt)
{
free_reserved_page(ptdesc_page(pt));
}
/*
* Default method to free all the __init memory into the buddy system.
* The freed pages will be poisoned with pattern "poison" if it's within
* range [0, UCHAR_MAX].
* Return pages freed into the buddy system.
*/
static inline unsigned long free_initmem_default(int poison)
{
extern char __init_begin[], __init_end[];
return free_reserved_area(&__init_begin, &__init_end,
poison, "unused kernel image (initmem)");
}
static inline unsigned long get_num_physpages(void)
{
int nid;
unsigned long phys_pages = 0;
for_each_online_node(nid)
phys_pages += node_present_pages(nid);
return phys_pages;
}
/*
* Using memblock node mappings, an architecture may initialise its
* zones, allocate the backing mem_map and account for memory holes in an
* architecture independent manner.
*
* An architecture is expected to register range of page frames backed by
* physical memory with memblock_add[_node]() before calling
* free_area_init() passing in the PFN each zone ends at. At a basic
* usage, an architecture is expected to do something like
*
* unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
* max_highmem_pfn};
* for_each_valid_physical_page_range()
* memblock_add_node(base, size, nid, MEMBLOCK_NONE)
* free_area_init(max_zone_pfns);
*/
void free_area_init(unsigned long *max_zone_pfn);
unsigned long node_map_pfn_alignment(void);
unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
unsigned long end_pfn);
extern unsigned long absent_pages_in_range(unsigned long start_pfn,
unsigned long end_pfn);
extern void get_pfn_range_for_nid(unsigned int nid,
unsigned long *start_pfn, unsigned long *end_pfn);
#ifndef CONFIG_NUMA
static inline int early_pfn_to_nid(unsigned long pfn)
{
return 0;
}
#else
/* please see mm/page_alloc.c */
extern int __meminit early_pfn_to_nid(unsigned long pfn);
#endif
extern void set_dma_reserve(unsigned long new_dma_reserve);
extern void mem_init(void);
extern void __init mmap_init(void);
extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);
static inline void show_mem(void)
{
__show_mem(0, NULL, MAX_NR_ZONES - 1);
}
extern long si_mem_available(void);
extern void si_meminfo(struct sysinfo * val);
extern void si_meminfo_node(struct sysinfo *val, int nid);
#ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES
extern unsigned long arch_reserved_kernel_pages(void);
#endif
extern __printf(3, 4)
void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...);
extern void setup_per_cpu_pageset(void);
/* nommu.c */
extern atomic_long_t mmap_pages_allocated;
extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
/* interval_tree.c */
void vma_interval_tree_insert(struct vm_area_struct *node,
struct rb_root_cached *root);
void vma_interval_tree_insert_after(struct vm_area_struct *node,
struct vm_area_struct *prev,
struct rb_root_cached *root);
void vma_interval_tree_remove(struct vm_area_struct *node,
struct rb_root_cached *root);
struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root,
unsigned long start, unsigned long last);
struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
unsigned long start, unsigned long last);
#define vma_interval_tree_foreach(vma, root, start, last) \
for (vma = vma_interval_tree_iter_first(root, start, last); \
vma; vma = vma_interval_tree_iter_next(vma, start, last))
void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
struct rb_root_cached *root);
void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
struct rb_root_cached *root);
struct anon_vma_chain *
anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
unsigned long start, unsigned long last);
struct anon_vma_chain *anon_vma_interval_tree_iter_next(
struct anon_vma_chain *node, unsigned long start, unsigned long last);
#ifdef CONFIG_DEBUG_VM_RB
void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
#endif
#define anon_vma_interval_tree_foreach(avc, root, start, last) \
for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
/* mmap.c */
extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
extern int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long start, unsigned long end, pgoff_t pgoff,
struct vm_area_struct *next);
extern int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long start, unsigned long end, pgoff_t pgoff);
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
extern void unlink_file_vma(struct vm_area_struct *);
extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
unsigned long addr, unsigned long len, pgoff_t pgoff,
bool *need_rmap_locks);
extern void exit_mmap(struct mm_struct *);
struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long vm_flags,
struct mempolicy *policy,
struct vm_userfaultfd_ctx uffd_ctx,
struct anon_vma_name *anon_name);
/* We are about to modify the VMA's flags. */
static inline struct vm_area_struct
*vma_modify_flags(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long new_flags)
{
return vma_modify(vmi, prev, vma, start, end, new_flags,
vma_policy(vma), vma->vm_userfaultfd_ctx,
anon_vma_name(vma));
}
/* We are about to modify the VMA's flags and/or anon_name. */
static inline struct vm_area_struct
*vma_modify_flags_name(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start,
unsigned long end,
unsigned long new_flags,
struct anon_vma_name *new_name)
{
return vma_modify(vmi, prev, vma, start, end, new_flags,
vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
}
/* We are about to modify the VMA's memory policy. */
static inline struct vm_area_struct
*vma_modify_policy(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct mempolicy *new_pol)
{
return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
}
/* We are about to modify the VMA's flags and/or uffd context. */
static inline struct vm_area_struct
*vma_modify_flags_uffd(struct vma_iterator *vmi,
struct vm_area_struct *prev,
struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long new_flags,
struct vm_userfaultfd_ctx new_ctx)
{
return vma_modify(vmi, prev, vma, start, end, new_flags,
vma_policy(vma), new_ctx, anon_vma_name(vma));
}
static inline int check_data_rlimit(unsigned long rlim,
unsigned long new,
unsigned long start,
unsigned long end_data,
unsigned long start_data)
{
if (rlim < RLIM_INFINITY) {
if (((new - start) + (end_data - start_data)) > rlim)
return -ENOSPC;
}
return 0;
}
extern int mm_take_all_locks(struct mm_struct *mm);
extern void mm_drop_all_locks(struct mm_struct *mm);
extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
extern struct file *get_mm_exe_file(struct mm_struct *mm);
extern struct file *get_task_exe_file(struct task_struct *task);
extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
const struct vm_special_mapping *sm);
extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
unsigned long addr, unsigned long len,
unsigned long flags,
const struct vm_special_mapping *spec);
/* This is an obsolete alternative to _install_special_mapping. */
extern int install_special_mapping(struct mm_struct *mm,
unsigned long addr, unsigned long len,
unsigned long flags, struct page **pages);
unsigned long randomize_stack_top(unsigned long stack_top);
unsigned long randomize_page(unsigned long start, unsigned long range);
extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
extern unsigned long mmap_region(struct file *file, unsigned long addr,
unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
struct list_head *uf);
extern unsigned long do_mmap(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot, unsigned long flags,
vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
struct list_head *uf);
extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
unsigned long start, size_t len, struct list_head *uf,
bool unlock);
extern int do_munmap(struct mm_struct *, unsigned long, size_t,
struct list_head *uf);
extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);
#ifdef CONFIG_MMU
extern int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct list_head *uf, bool unlock);
extern int __mm_populate(unsigned long addr, unsigned long len,
int ignore_errors);
static inline void mm_populate(unsigned long addr, unsigned long len)
{
/* Ignore errors */
(void) __mm_populate(addr, len, 1);
}
#else
static inline void mm_populate(unsigned long addr, unsigned long len) {}
#endif
/* This takes the mm semaphore itself */
extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long);
extern int vm_munmap(unsigned long, size_t);
extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
unsigned long, unsigned long,
unsigned long, unsigned long);
struct vm_unmapped_area_info {
#define VM_UNMAPPED_AREA_TOPDOWN 1
unsigned long flags;
unsigned long length;
unsigned long low_limit;
unsigned long high_limit;
unsigned long align_mask;
unsigned long align_offset;
};
extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info);
/* truncate.c */
extern void truncate_inode_pages(struct address_space *, loff_t);
extern void truncate_inode_pages_range(struct address_space *,
loff_t lstart, loff_t lend);
extern void truncate_inode_pages_final(struct address_space *);
/* generic vm_area_ops exported for stackable file systems */
extern vm_fault_t filemap_fault(struct vm_fault *vmf);
extern vm_fault_t filemap_map_pages(struct vm_fault *vmf,
pgoff_t start_pgoff, pgoff_t end_pgoff);
extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
extern unsigned long stack_guard_gap;
/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
int expand_stack_locked(struct vm_area_struct *vma, unsigned long address);
struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr);
/* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */
int expand_downwards(struct vm_area_struct *vma, unsigned long address);
/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
struct vm_area_struct **pprev);
/*
* Look up the first VMA which intersects the interval [start_addr, end_addr)
* NULL if none. Assume start_addr < end_addr.
*/
struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
unsigned long start_addr, unsigned long end_addr);
/**
* vma_lookup() - Find a VMA at a specific address
* @mm: The process address space.
* @addr: The user address.
*
* Return: The vm_area_struct at the given address, %NULL otherwise.
*/
static inline
struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr)
{
return mtree_load(&mm->mm_mt, addr);
}
static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma)
{
if (vma->vm_flags & VM_GROWSDOWN)
return stack_guard_gap;
/* See reasoning around the VM_SHADOW_STACK definition */
if (vma->vm_flags & VM_SHADOW_STACK)
return PAGE_SIZE;
return 0;
}
static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
{
unsigned long gap = stack_guard_start_gap(vma);
unsigned long vm_start = vma->vm_start;
vm_start -= gap;
if (vm_start > vma->vm_start)
vm_start = 0;
return vm_start;
}
static inline unsigned long vm_end_gap(struct vm_area_struct *vma)
{
unsigned long vm_end = vma->vm_end;
if (vma->vm_flags & VM_GROWSUP) {
vm_end += stack_guard_gap;
if (vm_end < vma->vm_end)
vm_end = -PAGE_SIZE;
}
return vm_end;
}
static inline unsigned long vma_pages(struct vm_area_struct *vma)
{
return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
}
/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
unsigned long vm_start, unsigned long vm_end)
{
struct vm_area_struct *vma = vma_lookup(mm, vm_start);
if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
vma = NULL;
return vma;
}
static inline bool range_in_vma(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
return (vma && vma->vm_start <= start && end <= vma->vm_end);
}
#ifdef CONFIG_MMU
pgprot_t vm_get_page_prot(unsigned long vm_flags);
void vma_set_page_prot(struct vm_area_struct *vma);
#else
static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
{
return __pgprot(0);
}
static inline void vma_set_page_prot(struct vm_area_struct *vma)
{
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
}
#endif
void vma_set_file(struct vm_area_struct *vma, struct file *file);
#ifdef CONFIG_NUMA_BALANCING
unsigned long change_prot_numa(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
#endif
struct vm_area_struct *find_extend_vma_locked(struct mm_struct *,
unsigned long addr);
int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t);
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t prot);
int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
struct page **pages, unsigned long *num);
int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
unsigned long num);
int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
unsigned long num);
vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn);
vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t pgprot);
vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
pfn_t pfn);
vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
unsigned long addr, pfn_t pfn);
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma,
unsigned long addr, struct page *page)
{
int err = vm_insert_page(vma, addr, page);
if (err == -ENOMEM)
return VM_FAULT_OOM;
if (err < 0 && err != -EBUSY)
return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;
}
#ifndef io_remap_pfn_range
static inline int io_remap_pfn_range(struct vm_area_struct *vma,
unsigned long addr, unsigned long pfn,
unsigned long size, pgprot_t prot)
{
return remap_pfn_range(vma, addr, pfn, size, pgprot_decrypted(prot));
}
#endif
static inline vm_fault_t vmf_error(int err)
{
if (err == -ENOMEM)
return VM_FAULT_OOM;
else if (err == -EHWPOISON)
return VM_FAULT_HWPOISON;
return VM_FAULT_SIGBUS;
}
/*
* Convert errno to return value for ->page_mkwrite() calls.
*
* This should eventually be merged with vmf_error() above, but will need a
* careful audit of all vmf_error() callers.
*/
static inline vm_fault_t vmf_fs_error(int err)
{
if (err == 0)
return VM_FAULT_LOCKED;
if (err == -EFAULT || err == -EAGAIN)
return VM_FAULT_NOPAGE;
if (err == -ENOMEM)
return VM_FAULT_OOM;
/* -ENOSPC, -EDQUOT, -EIO ... */
return VM_FAULT_SIGBUS;
}
struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
unsigned int foll_flags);
static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
{
if (vm_fault & VM_FAULT_OOM)
return -ENOMEM;
if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT;
if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV))
return -EFAULT;
return 0;
}
/*
* Indicates whether GUP can follow a PROT_NONE mapped page, or whether
* a (NUMA hinting) fault is required.
*/
static inline bool gup_can_follow_protnone(struct vm_area_struct *vma,
unsigned int flags)
{
/*
* If callers don't want to honor NUMA hinting faults, no need to
* determine if we would actually have to trigger a NUMA hinting fault.
*/
if (!(flags & FOLL_HONOR_NUMA_FAULT))
return true;
/*
* NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs.
*
* Requiring a fault here even for inaccessible VMAs would mean that
* FOLL_FORCE cannot make any progress, because handle_mm_fault()
* refuses to process NUMA hinting faults in inaccessible VMAs.
*/
return !vma_is_accessible(vma);
}
typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);
extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
unsigned long size, pte_fn_t fn, void *data);
extern int apply_to_existing_page_range(struct mm_struct *mm,
unsigned long address, unsigned long size,
pte_fn_t fn, void *data);
#ifdef CONFIG_PAGE_POISONING
extern void __kernel_poison_pages(struct page *page, int numpages);
extern void __kernel_unpoison_pages(struct page *page, int numpages);
extern bool _page_poisoning_enabled_early;
DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled);
static inline bool page_poisoning_enabled(void)
{
return _page_poisoning_enabled_early;
}
/*
* For use in fast paths after init_mem_debugging() has run, or when a
* false negative result is not harmful when called too early.
*/
static inline bool page_poisoning_enabled_static(void)
{
return static_branch_unlikely(&_page_poisoning_enabled);
}
static inline void kernel_poison_pages(struct page *page, int numpages)
{
if (page_poisoning_enabled_static())
__kernel_poison_pages(page, numpages);
}
static inline void kernel_unpoison_pages(struct page *page, int numpages)
{
if (page_poisoning_enabled_static())
__kernel_unpoison_pages(page, numpages);
}
#else
static inline bool page_poisoning_enabled(void) { return false; }
static inline bool page_poisoning_enabled_static(void) { return false; }
static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }
static inline void kernel_poison_pages(struct page *page, int numpages) { }
static inline void kernel_unpoison_pages(struct page *page, int numpages) { }
#endif
DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
static inline bool want_init_on_alloc(gfp_t flags)
{
if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
&init_on_alloc))
return true;
return flags & __GFP_ZERO;
}
DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
static inline bool want_init_on_free(void)
{
return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
&init_on_free);
}
extern bool _debug_pagealloc_enabled_early;
DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
static inline bool debug_pagealloc_enabled(void)
{
return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
_debug_pagealloc_enabled_early;
}
/*
* For use in fast paths after mem_debugging_and_hardening_init() has run,
* or when a false negative result is not harmful when called too early.
*/
static inline bool debug_pagealloc_enabled_static(void)
{
if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC))
return false;
return static_branch_unlikely(&_debug_pagealloc_enabled);
}
/*
* To support DEBUG_PAGEALLOC architecture must ensure that
* __kernel_map_pages() never fails
*/
extern void __kernel_map_pages(struct page *page, int numpages, int enable);
#ifdef CONFIG_DEBUG_PAGEALLOC
static inline void debug_pagealloc_map_pages(struct page *page, int numpages)
{
if (debug_pagealloc_enabled_static())
__kernel_map_pages(page, numpages, 1);
}
static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)
{
if (debug_pagealloc_enabled_static())
__kernel_map_pages(page, numpages, 0);
}
extern unsigned int _debug_guardpage_minorder;
DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
static inline unsigned int debug_guardpage_minorder(void)
{
return _debug_guardpage_minorder;
}
static inline bool debug_guardpage_enabled(void)
{
return static_branch_unlikely(&_debug_guardpage_enabled);
}
static inline bool page_is_guard(struct page *page)
{
if (!debug_guardpage_enabled())
return false;
return PageGuard(page);
}
bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order,
int migratetype);
static inline bool set_page_guard(struct zone *zone, struct page *page,
unsigned int order, int migratetype)
{
if (!debug_guardpage_enabled())
return false;
return __set_page_guard(zone, page, order, migratetype);
}
void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order,
int migratetype);
static inline void clear_page_guard(struct zone *zone, struct page *page,
unsigned int order, int migratetype)
{
if (!debug_guardpage_enabled())
return;
__clear_page_guard(zone, page, order, migratetype);
}
#else /* CONFIG_DEBUG_PAGEALLOC */
static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}
static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}
static inline unsigned int debug_guardpage_minorder(void) { return 0; }
static inline bool debug_guardpage_enabled(void) { return false; }
static inline bool page_is_guard(struct page *page) { return false; }
static inline bool set_page_guard(struct zone *zone, struct page *page,
unsigned int order, int migratetype) { return false; }
static inline void clear_page_guard(struct zone *zone, struct page *page,
unsigned int order, int migratetype) {}
#endif /* CONFIG_DEBUG_PAGEALLOC */
#ifdef __HAVE_ARCH_GATE_AREA
extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
extern int in_gate_area_no_mm(unsigned long addr);
extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
#else
static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
return NULL;
}
static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
return 0;
}
#endif /* __HAVE_ARCH_GATE_AREA */
extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm);
#ifdef CONFIG_SYSCTL
extern int sysctl_drop_caches;
int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *,
loff_t *);
#endif
void drop_slab(void);
#ifndef CONFIG_MMU
#define randomize_va_space 0
#else
extern int randomize_va_space;
#endif
const char * arch_vma_name(struct vm_area_struct *vma);
#ifdef CONFIG_MMU
void print_vma_addr(char *prefix, unsigned long rip);
#else
static inline void print_vma_addr(char *prefix, unsigned long rip)
{
}
#endif
void *sparse_buffer_alloc(unsigned long size);
struct page * __populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap);
void pmd_init(void *addr);
void pud_init(void *addr);
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
struct vmem_altmap *altmap, struct page *reuse);
void *vmemmap_alloc_block(unsigned long size, int node);
struct vmem_altmap;
void *vmemmap_alloc_block_buf(unsigned long size, int node,
struct vmem_altmap *altmap);
void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
void vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
unsigned long addr, unsigned long next);
int vmemmap_check_pmd(pmd_t *pmd, int node,
unsigned long addr, unsigned long next);
int vmemmap_populate_basepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap);
int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap);
int vmemmap_populate(unsigned long start, unsigned long end, int node,
struct vmem_altmap *altmap);
void vmemmap_populate_print_last(void);
#ifdef CONFIG_MEMORY_HOTPLUG
void vmemmap_free(unsigned long start, unsigned long end,
struct vmem_altmap *altmap);
#endif
#ifdef CONFIG_SPARSEMEM_VMEMMAP
static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
{
/* number of pfns from base where pfn_to_page() is valid */
if (altmap)
return altmap->reserve + altmap->free;
return 0;
}
static inline void vmem_altmap_free(struct vmem_altmap *altmap,
unsigned long nr_pfns)
{
altmap->alloc -= nr_pfns;
}
#else
static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
{
return 0;
}
static inline void vmem_altmap_free(struct vmem_altmap *altmap,
unsigned long nr_pfns)
{
}
#endif
#define VMEMMAP_RESERVE_NR 2
#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
unsigned long nr_pages;
unsigned long nr_vmemmap_pages;
if (!pgmap || !is_power_of_2(sizeof(struct page)))
return false;
nr_pages = pgmap_vmemmap_nr(pgmap);
nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT);
/*
* For vmemmap optimization with DAX we need minimum 2 vmemmap
* pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst
*/
return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR);
}
/*
* If we don't have an architecture override, use the generic rule
*/
#ifndef vmemmap_can_optimize
#define vmemmap_can_optimize __vmemmap_can_optimize
#endif
#else
static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
return false;
}
#endif
void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
unsigned long nr_pages);
enum mf_flags {
MF_COUNT_INCREASED = 1 << 0,
MF_ACTION_REQUIRED = 1 << 1,
MF_MUST_KILL = 1 << 2,
MF_SOFT_OFFLINE = 1 << 3,
MF_UNPOISON = 1 << 4,
MF_SW_SIMULATED = 1 << 5,
MF_NO_RETRY = 1 << 6,
MF_MEM_PRE_REMOVE = 1 << 7,
};
int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index,
unsigned long count, int mf_flags);
extern int memory_failure(unsigned long pfn, int flags);
extern void memory_failure_queue_kick(int cpu);
extern int unpoison_memory(unsigned long pfn);
extern void shake_page(struct page *p);
extern atomic_long_t num_poisoned_pages __read_mostly;
extern int soft_offline_page(unsigned long pfn, int flags);
#ifdef CONFIG_MEMORY_FAILURE
/*
* Sysfs entries for memory failure handling statistics.
*/
extern const struct attribute_group memory_failure_attr_group;
extern void memory_failure_queue(unsigned long pfn, int flags);
extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
bool *migratable_cleared);
void num_poisoned_pages_inc(unsigned long pfn);
void num_poisoned_pages_sub(unsigned long pfn, long i);
struct task_struct *task_early_kill(struct task_struct *tsk, int force_early);
#else
static inline void memory_failure_queue(unsigned long pfn, int flags)
{
}
static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
bool *migratable_cleared)
{
return 0;
}
static inline void num_poisoned_pages_inc(unsigned long pfn)
{
}
static inline void num_poisoned_pages_sub(unsigned long pfn, long i)
{
}
#endif
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_KSM)
void add_to_kill_ksm(struct task_struct *tsk, struct page *p,
struct vm_area_struct *vma, struct list_head *to_kill,
unsigned long ksm_addr);
#endif
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)
extern void memblk_nr_poison_inc(unsigned long pfn);
extern void memblk_nr_poison_sub(unsigned long pfn, long i);
#else
static inline void memblk_nr_poison_inc(unsigned long pfn)
{
}
static inline void memblk_nr_poison_sub(unsigned long pfn, long i)
{
}
#endif
#ifndef arch_memory_failure
static inline int arch_memory_failure(unsigned long pfn, int flags)
{
return -ENXIO;
}
#endif
#ifndef arch_is_platform_page
static inline bool arch_is_platform_page(u64 paddr)
{
return false;
}
#endif
/*
* Error handlers for various types of pages.
*/
enum mf_result {
MF_IGNORED, /* Error: cannot be handled */
MF_FAILED, /* Error: handling failed */
MF_DELAYED, /* Will be handled later */
MF_RECOVERED, /* Successfully recovered */
};
enum mf_action_page_type {
MF_MSG_KERNEL,
MF_MSG_KERNEL_HIGH_ORDER,
MF_MSG_SLAB,
MF_MSG_DIFFERENT_COMPOUND,
MF_MSG_HUGE,
MF_MSG_FREE_HUGE,
MF_MSG_UNMAP_FAILED,
MF_MSG_DIRTY_SWAPCACHE,
MF_MSG_CLEAN_SWAPCACHE,
MF_MSG_DIRTY_MLOCKED_LRU,
MF_MSG_CLEAN_MLOCKED_LRU,
MF_MSG_DIRTY_UNEVICTABLE_LRU,
MF_MSG_CLEAN_UNEVICTABLE_LRU,
MF_MSG_DIRTY_LRU,
MF_MSG_CLEAN_LRU,
MF_MSG_TRUNCATED_LRU,
MF_MSG_BUDDY,
MF_MSG_DAX,
MF_MSG_UNSPLIT_THP,
MF_MSG_UNKNOWN,
};
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
extern void clear_huge_page(struct page *page,
unsigned long addr_hint,
unsigned int pages_per_huge_page);
int copy_user_large_folio(struct folio *dst, struct folio *src,
unsigned long addr_hint,
struct vm_area_struct *vma);
long copy_folio_from_user(struct folio *dst_folio,
const void __user *usr_src,
bool allow_pagefault);
/**
* vma_is_special_huge - Are transhuge page-table entries considered special?
* @vma: Pointer to the struct vm_area_struct to consider
*
* Whether transhuge page-table entries are considered "special" following
* the definition in vm_normal_page().
*
* Return: true if transhuge page-table entries should be considered special,
* false otherwise.
*/
static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
{
return vma_is_dax(vma) || (vma->vm_file &&
(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
#if MAX_NUMNODES > 1
void __init setup_nr_node_ids(void);
#else
static inline void setup_nr_node_ids(void) {}
#endif
extern int memcmp_pages(struct page *page1, struct page *page2);
static inline int pages_identical(struct page *page1, struct page *page2)
{
return !memcmp_pages(page1, page2);
}
#ifdef CONFIG_MAPPING_DIRTY_HELPERS
unsigned long clean_record_shared_mapping_range(struct address_space *mapping,
pgoff_t first_index, pgoff_t nr,
pgoff_t bitmap_pgoff,
unsigned long *bitmap,
pgoff_t *start,
pgoff_t *end);
unsigned long wp_shared_mapping_range(struct address_space *mapping,
pgoff_t first_index, pgoff_t nr);
#endif
extern int sysctl_nr_trim_pages;
#ifdef CONFIG_PRINTK
void mem_dump_obj(void *object);
#else
static inline void mem_dump_obj(void *object) {}
#endif
/**
* seal_check_write - Check for F_SEAL_WRITE or F_SEAL_FUTURE_WRITE flags and
* handle them.
* @seals: the seals to check
* @vma: the vma to operate on
*
* Check whether F_SEAL_WRITE or F_SEAL_FUTURE_WRITE are set; if so, do proper
* check/handling on the vma flags. Return 0 if check pass, or <0 for errors.
*/
static inline int seal_check_write(int seals, struct vm_area_struct *vma)
{
if (seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
/*
* New PROT_WRITE and MAP_SHARED mmaps are not allowed when
* write seals are active.
*/
if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
return -EPERM;
/*
* Since an F_SEAL_[FUTURE_]WRITE sealed memfd can be mapped as
* MAP_SHARED and read-only, take care to not allow mprotect to
* revert protections on such mappings. Do this only for shared
* mappings. For private mappings, don't need to mask
* VM_MAYWRITE as we still want them to be COW-writable.
*/
if (vma->vm_flags & VM_SHARED)
vm_flags_clear(vma, VM_MAYWRITE);
}
return 0;
}
#ifdef CONFIG_ANON_VMA_NAME
int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
unsigned long len_in,
struct anon_vma_name *anon_name);
#else
static inline int
madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
unsigned long len_in, struct anon_vma_name *anon_name) {
return 0;
}
#endif
#ifdef CONFIG_UNACCEPTED_MEMORY
bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end);
void accept_memory(phys_addr_t start, phys_addr_t end);
#else
static inline bool range_contains_unaccepted_memory(phys_addr_t start,
phys_addr_t end)
{
return false;
}
static inline void accept_memory(phys_addr_t start, phys_addr_t end)
{
}
#endif
static inline bool pfn_is_unaccepted_memory(unsigned long pfn)
{
phys_addr_t paddr = pfn << PAGE_SHIFT;
return range_contains_unaccepted_memory(paddr, paddr + PAGE_SIZE);
}
#endif /* _LINUX_MM_H */
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Definitions for the 'struct sk_buff' memory handlers.
*
* Authors:
* Alan Cox, <gw4pts@gw4pts.ampr.org>
* Florian La Roche, <rzsfl@rz.uni-sb.de>
*/
#ifndef _LINUX_SKBUFF_H
#define _LINUX_SKBUFF_H
#include <linux/kernel.h>
#include <linux/compiler.h>
#include <linux/time.h>
#include <linux/bug.h>
#include <linux/bvec.h>
#include <linux/cache.h>
#include <linux/rbtree.h>
#include <linux/socket.h>
#include <linux/refcount.h>
#include <linux/atomic.h>
#include <asm/types.h>
#include <linux/spinlock.h>
#include <net/checksum.h>
#include <linux/rcupdate.h>
#include <linux/dma-mapping.h>
#include <linux/netdev_features.h>
#include <net/flow_dissector.h>
#include <linux/in6.h>
#include <linux/if_packet.h>
#include <linux/llist.h>
#include <net/flow.h>
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <linux/netfilter/nf_conntrack_common.h>
#endif
#include <net/net_debug.h>
#include <net/dropreason-core.h>
#include <net/netmem.h>
/**
* DOC: skb checksums
*
* The interface for checksum offload between the stack and networking drivers
* is as follows...
*
* IP checksum related features
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* Drivers advertise checksum offload capabilities in the features of a device.
* From the stack's point of view these are capabilities offered by the driver.
* A driver typically only advertises features that it is capable of offloading
* to its device.
*
* .. flat-table:: Checksum related device features
* :widths: 1 10
*
* * - %NETIF_F_HW_CSUM
* - The driver (or its device) is able to compute one
* IP (one's complement) checksum for any combination
* of protocols or protocol layering. The checksum is
* computed and set in a packet per the CHECKSUM_PARTIAL
* interface (see below).
*
* * - %NETIF_F_IP_CSUM
* - Driver (device) is only able to checksum plain
* TCP or UDP packets over IPv4. These are specifically
* unencapsulated packets of the form IPv4|TCP or
* IPv4|UDP where the Protocol field in the IPv4 header
* is TCP or UDP. The IPv4 header may contain IP options.
* This feature cannot be set in features for a device
* with NETIF_F_HW_CSUM also set. This feature is being
* DEPRECATED (see below).
*
* * - %NETIF_F_IPV6_CSUM
* - Driver (device) is only able to checksum plain
* TCP or UDP packets over IPv6. These are specifically
* unencapsulated packets of the form IPv6|TCP or
* IPv6|UDP where the Next Header field in the IPv6
* header is either TCP or UDP. IPv6 extension headers
* are not supported with this feature. This feature
* cannot be set in features for a device with
* NETIF_F_HW_CSUM also set. This feature is being
* DEPRECATED (see below).
*
* * - %NETIF_F_RXCSUM
* - Driver (device) performs receive checksum offload.
* This flag is only used to disable the RX checksum
* feature for a device. The stack will accept receive
* checksum indication in packets received on a device
* regardless of whether NETIF_F_RXCSUM is set.
*
* Checksumming of received packets by device
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* Indication of checksum verification is set in &sk_buff.ip_summed.
* Possible values are:
*
* - %CHECKSUM_NONE
*
* Device did not checksum this packet e.g. due to lack of capabilities.
* The packet contains full (though not verified) checksum in packet but
* not in skb->csum. Thus, skb->csum is undefined in this case.
*
* - %CHECKSUM_UNNECESSARY
*
* The hardware you're dealing with doesn't calculate the full checksum
* (as in %CHECKSUM_COMPLETE), but it does parse headers and verify checksums
* for specific protocols. For such packets it will set %CHECKSUM_UNNECESSARY
* if their checksums are okay. &sk_buff.csum is still undefined in this case
* though. A driver or device must never modify the checksum field in the
* packet even if checksum is verified.
*
* %CHECKSUM_UNNECESSARY is applicable to following protocols:
*
* - TCP: IPv6 and IPv4.
* - UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
* zero UDP checksum for either IPv4 or IPv6, the networking stack
* may perform further validation in this case.
* - GRE: only if the checksum is present in the header.
* - SCTP: indicates the CRC in SCTP header has been validated.
* - FCOE: indicates the CRC in FC frame has been validated.
*
* &sk_buff.csum_level indicates the number of consecutive checksums found in
* the packet minus one that have been verified as %CHECKSUM_UNNECESSARY.
* For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
* and a device is able to verify the checksums for UDP (possibly zero),
* GRE (checksum flag is set) and TCP, &sk_buff.csum_level would be set to
* two. If the device were only able to verify the UDP checksum and not
* GRE, either because it doesn't support GRE checksum or because GRE
* checksum is bad, skb->csum_level would be set to zero (TCP checksum is
* not considered in this case).
*
* - %CHECKSUM_COMPLETE
*
* This is the most generic way. The device supplied checksum of the _whole_
* packet as seen by netif_rx() and fills in &sk_buff.csum. This means the
* hardware doesn't need to parse L3/L4 headers to implement this.
*
* Notes:
*
* - Even if device supports only some protocols, but is able to produce
* skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
* - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
*
* - %CHECKSUM_PARTIAL
*
* A checksum is set up to be offloaded to a device as described in the
* output description for CHECKSUM_PARTIAL. This may occur on a packet
* received directly from another Linux OS, e.g., a virtualized Linux kernel
* on the same host, or it may be set in the input path in GRO or remote
* checksum offload. For the purposes of checksum verification, the checksum
* referred to by skb->csum_start + skb->csum_offset and any preceding
* checksums in the packet are considered verified. Any checksums in the
* packet that are after the checksum being offloaded are not considered to
* be verified.
*
* Checksumming on transmit for non-GSO
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* The stack requests checksum offload in the &sk_buff.ip_summed for a packet.
* Values are:
*
* - %CHECKSUM_PARTIAL
*
* The driver is required to checksum the packet as seen by hard_start_xmit()
* from &sk_buff.csum_start up to the end, and to record/write the checksum at
* offset &sk_buff.csum_start + &sk_buff.csum_offset.
* A driver may verify that the
* csum_start and csum_offset values are valid values given the length and
* offset of the packet, but it should not attempt to validate that the
* checksum refers to a legitimate transport layer checksum -- it is the
* purview of the stack to validate that csum_start and csum_offset are set
* correctly.
*
* When the stack requests checksum offload for a packet, the driver MUST
* ensure that the checksum is set correctly. A driver can either offload the
* checksum calculation to the device, or call skb_checksum_help (in the case
* that the device does not support offload for a particular checksum).
*
* %NETIF_F_IP_CSUM and %NETIF_F_IPV6_CSUM are being deprecated in favor of
* %NETIF_F_HW_CSUM. New devices should use %NETIF_F_HW_CSUM to indicate
* checksum offload capability.
* skb_csum_hwoffload_help() can be called to resolve %CHECKSUM_PARTIAL based
* on network device checksumming capabilities: if a packet does not match
* them, skb_checksum_help() or skb_crc32c_help() (depending on the value of
* &sk_buff.csum_not_inet, see :ref:`crc`)
* is called to resolve the checksum.
*
* - %CHECKSUM_NONE
*
* The skb was already checksummed by the protocol, or a checksum is not
* required.
*
* - %CHECKSUM_UNNECESSARY
*
* This has the same meaning as CHECKSUM_NONE for checksum offload on
* output.
*
* - %CHECKSUM_COMPLETE
*
* Not used in checksum output. If a driver observes a packet with this value
* set in skbuff, it should treat the packet as if %CHECKSUM_NONE were set.
*
* .. _crc:
*
* Non-IP checksum (CRC) offloads
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* .. flat-table::
* :widths: 1 10
*
* * - %NETIF_F_SCTP_CRC
* - This feature indicates that a device is capable of
* offloading the SCTP CRC in a packet. To perform this offload the stack
* will set csum_start and csum_offset accordingly, set ip_summed to
* %CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication
* in the skbuff that the %CHECKSUM_PARTIAL refers to CRC32c.
* A driver that supports both IP checksum offload and SCTP CRC32c offload
* must verify which offload is configured for a packet by testing the
* value of &sk_buff.csum_not_inet; skb_crc32c_csum_help() is provided to
* resolve %CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
*
* * - %NETIF_F_FCOE_CRC
* - This feature indicates that a device is capable of offloading the FCOE
* CRC in a packet. To perform this offload the stack will set ip_summed
* to %CHECKSUM_PARTIAL and set csum_start and csum_offset
* accordingly. Note that there is no indication in the skbuff that the
* %CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports
* both IP checksum offload and FCOE CRC offload must verify which offload
* is configured for a packet, presumably by inspecting packet headers.
*
* Checksumming on output with GSO
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* In the case of a GSO packet (skb_is_gso() is true), checksum offload
* is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
* gso_type is %SKB_GSO_TCPV4 or %SKB_GSO_TCPV6, TCP checksum offload as
* part of the GSO operation is implied. If a checksum is being offloaded
* with GSO then ip_summed is %CHECKSUM_PARTIAL, and both csum_start and
* csum_offset are set to refer to the outermost checksum being offloaded
* (two offloaded checksums are possible with UDP encapsulation).
*/
/* Don't change this without changing skb_csum_unnecessary! */
#define CHECKSUM_NONE 0
#define CHECKSUM_UNNECESSARY 1
#define CHECKSUM_COMPLETE 2
#define CHECKSUM_PARTIAL 3
/* Maximum value in skb->csum_level */
#define SKB_MAX_CSUM_LEVEL 3
#define SKB_DATA_ALIGN(X) ALIGN(X, SMP_CACHE_BYTES)
#define SKB_WITH_OVERHEAD(X) \
((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
/* For X bytes available in skb->head, what is the minimal
* allocation needed, knowing struct skb_shared_info needs
* to be aligned.
*/
#define SKB_HEAD_ALIGN(X) (SKB_DATA_ALIGN(X) + \
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
#define SKB_MAX_ORDER(X, ORDER) \
SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
#define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0))
#define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2))
/* return minimum truesize of one skb containing X bytes of data */
#define SKB_TRUESIZE(X) ((X) + \
SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
struct ahash_request;
struct net_device;
struct scatterlist;
struct pipe_inode_info;
struct iov_iter;
struct napi_struct;
struct bpf_prog;
union bpf_attr;
struct skb_ext;
struct ts_config;
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
struct nf_bridge_info {
enum {
BRNF_PROTO_UNCHANGED,
BRNF_PROTO_8021Q,
BRNF_PROTO_PPPOE
} orig_proto:8;
u8 pkt_otherhost:1;
u8 in_prerouting:1;
u8 bridged_dnat:1;
u8 sabotage_in_done:1;
__u16 frag_max_size;
int physinif;
/* always valid & non-NULL from FORWARD on, for physdev match */
struct net_device *physoutdev;
union {
/* prerouting: detect dnat in orig/reply direction */
__be32 ipv4_daddr;
struct in6_addr ipv6_daddr;
/* after prerouting + nat detected: store original source
* mac since neigh resolution overwrites it, only used while
* skb is out in neigh layer.
*/
char neigh_header[8];
};
};
#endif
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
/* Chain in tc_skb_ext will be used to share the tc chain with
* ovs recirc_id. It will be set to the current chain by tc
* and read by ovs to recirc_id.
*/
struct tc_skb_ext {
union {
u64 act_miss_cookie;
__u32 chain;
};
__u16 mru;
__u16 zone;
u8 post_ct:1;
u8 post_ct_snat:1;
u8 post_ct_dnat:1;
u8 act_miss:1; /* Set if act_miss_cookie is used */
u8 l2_miss:1; /* Set by bridge upon FDB or MDB miss */
};
#endif
struct sk_buff_head {
/* These two members must be first to match sk_buff. */
struct_group_tagged(sk_buff_list, list,
struct sk_buff *next;
struct sk_buff *prev;
);
__u32 qlen;
spinlock_t lock;
};
struct sk_buff;
#ifndef CONFIG_MAX_SKB_FRAGS
# define CONFIG_MAX_SKB_FRAGS 17
#endif
#define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS
extern int sysctl_max_skb_frags;
/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
* segment using its current segmentation instead.
*/
#define GSO_BY_FRAGS 0xFFFF
typedef struct skb_frag {
netmem_ref netmem;
unsigned int len;
unsigned int offset;
} skb_frag_t;
/**
* skb_frag_size() - Returns the size of a skb fragment
* @frag: skb fragment
*/
static inline unsigned int skb_frag_size(const skb_frag_t *frag)
{
return frag->len;
}
/**
* skb_frag_size_set() - Sets the size of a skb fragment
* @frag: skb fragment
* @size: size of fragment
*/
static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
{
frag->len = size;
}
/**
* skb_frag_size_add() - Increments the size of a skb fragment by @delta
* @frag: skb fragment
* @delta: value to add
*/
static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
{
frag->len += delta;
}
/**
* skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
* @frag: skb fragment
* @delta: value to subtract
*/
static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
{
frag->len -= delta;
}
/**
* skb_frag_must_loop - Test if %p is a high memory page
* @p: fragment's page
*/
static inline bool skb_frag_must_loop(struct page *p)
{
#if defined(CONFIG_HIGHMEM)
if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) || PageHighMem(p))
return true;
#endif
return false;
}
/**
* skb_frag_foreach_page - loop over pages in a fragment
*
* @f: skb frag to operate on
* @f_off: offset from start of f->netmem
* @f_len: length from f_off to loop over
* @p: (temp var) current page
* @p_off: (temp var) offset from start of current page,
* non-zero only on first page.
* @p_len: (temp var) length in current page,
* < PAGE_SIZE only on first and last page.
* @copied: (temp var) length so far, excluding current p_len.
*
* A fragment can hold a compound page, in which case per-page
* operations, notably kmap_atomic, must be called for each
* regular page.
*/
#define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied) \
for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT), \
p_off = (f_off) & (PAGE_SIZE - 1), \
p_len = skb_frag_must_loop(p) ? \
min_t(u32, f_len, PAGE_SIZE - p_off) : f_len, \
copied = 0; \
copied < f_len; \
copied += p_len, p++, p_off = 0, \
p_len = min_t(u32, f_len - copied, PAGE_SIZE)) \
/**
* struct skb_shared_hwtstamps - hardware time stamps
* @hwtstamp: hardware time stamp transformed into duration
* since arbitrary point in time
* @netdev_data: address/cookie of network device driver used as
* reference to actual hardware time stamp
*
* Software time stamps generated by ktime_get_real() are stored in
* skb->tstamp.
*
* hwtstamps can only be compared against other hwtstamps from
* the same device.
*
* This structure is attached to packets as part of the
* &skb_shared_info. Use skb_hwtstamps() to get a pointer.
*/
struct skb_shared_hwtstamps {
union {
ktime_t hwtstamp;
void *netdev_data;
};
};
/* Definitions for tx_flags in struct skb_shared_info */
enum {
/* generate hardware time stamp */
SKBTX_HW_TSTAMP = 1 << 0,
/* generate software time stamp when queueing packet to NIC */
SKBTX_SW_TSTAMP = 1 << 1,
/* device driver is going to provide hardware time stamp */
SKBTX_IN_PROGRESS = 1 << 2,
/* generate hardware time stamp based on cycles if supported */
SKBTX_HW_TSTAMP_USE_CYCLES = 1 << 3,
/* generate wifi status information (where possible) */
SKBTX_WIFI_STATUS = 1 << 4,
/* determine hardware time stamp based on time or cycles */
SKBTX_HW_TSTAMP_NETDEV = 1 << 5,
/* generate software time stamp when entering packet scheduling */
SKBTX_SCHED_TSTAMP = 1 << 6,
};
#define SKBTX_ANY_SW_TSTAMP (SKBTX_SW_TSTAMP | \
SKBTX_SCHED_TSTAMP)
#define SKBTX_ANY_TSTAMP (SKBTX_HW_TSTAMP | \
SKBTX_HW_TSTAMP_USE_CYCLES | \
SKBTX_ANY_SW_TSTAMP)
/* Definitions for flags in struct skb_shared_info */
enum {
/* use zcopy routines */
SKBFL_ZEROCOPY_ENABLE = BIT(0),
/* This indicates at least one fragment might be overwritten
* (as in vmsplice(), sendfile() ...)
* If we need to compute a TX checksum, we'll need to copy
* all frags to avoid possible bad checksum
*/
SKBFL_SHARED_FRAG = BIT(1),
/* segment contains only zerocopy data and should not be
* charged to the kernel memory.
*/
SKBFL_PURE_ZEROCOPY = BIT(2),
SKBFL_DONT_ORPHAN = BIT(3),
/* page references are managed by the ubuf_info, so it's safe to
* use frags only up until ubuf_info is released
*/
SKBFL_MANAGED_FRAG_REFS = BIT(4),
};
#define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG)
#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
/*
* The callback notifies userspace to release buffers when skb DMA is done in
* lower device, the skb last reference should be 0 when calling this.
* The zerocopy_success argument is true if zero copy transmit occurred,
* false on data copy or out of memory error caused by data copy attempt.
* The ctx field is used to track device context.
* The desc field is used to track userspace buffer index.
*/
struct ubuf_info {
void (*callback)(struct sk_buff *, struct ubuf_info *,
bool zerocopy_success);
refcount_t refcnt;
u8 flags;
};
struct ubuf_info_msgzc {
struct ubuf_info ubuf;
union {
struct {
unsigned long desc;
void *ctx;
};
struct {
u32 id;
u16 len;
u16 zerocopy:1;
u32 bytelen;
};
};
struct mmpin {
struct user_struct *user;
unsigned int num_pg;
} mmp;
};
#define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
#define uarg_to_msgzc(ubuf_ptr) container_of((ubuf_ptr), struct ubuf_info_msgzc, \
ubuf)
int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
void mm_unaccount_pinned_pages(struct mmpin *mmp);
/* Preserve some data across TX submission and completion.
*
* Note, this state is stored in the driver. Extending the layout
* might need some special care.
*/
struct xsk_tx_metadata_compl {
__u64 *tx_timestamp;
};
/* This data is invariant across clones and lives at
* the end of the header data, ie. at skb->end.
*/
struct skb_shared_info {
__u8 flags;
__u8 meta_len;
__u8 nr_frags;
__u8 tx_flags;
unsigned short gso_size;
/* Warning: this field is not always filled in (UFO)! */
unsigned short gso_segs;
struct sk_buff *frag_list;
union {
struct skb_shared_hwtstamps hwtstamps;
struct xsk_tx_metadata_compl xsk_meta;
};
unsigned int gso_type;
u32 tskey;
/*
* Warning : all fields before dataref are cleared in __alloc_skb()
*/
atomic_t dataref;
unsigned int xdp_frags_size;
/* Intermediate layers must ensure that destructor_arg
* remains valid until skb destructor */
void * destructor_arg;
/* must be last field, see pskb_expand_head() */
skb_frag_t frags[MAX_SKB_FRAGS];
};
/**
* DOC: dataref and headerless skbs
*
* Transport layers send out clones of payload skbs they hold for
* retransmissions. To allow lower layers of the stack to prepend their headers
* we split &skb_shared_info.dataref into two halves.
* The lower 16 bits count the overall number of references.
* The higher 16 bits indicate how many of the references are payload-only.
* skb_header_cloned() checks if skb is allowed to add / write the headers.
*
* The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr
* (via __skb_header_release()). Any clone created from marked skb will get
* &sk_buff.hdr_len populated with the available headroom.
* If there's the only clone in existence it's able to modify the headroom
* at will. The sequence of calls inside the transport layer is::
*
* <alloc skb>
* skb_reserve()
* __skb_header_release()
* skb_clone()
* // send the clone down the stack
*
* This is not a very generic construct and it depends on the transport layers
* doing the right thing. In practice there's usually only one payload-only skb.
* Having multiple payload-only skbs with different lengths of hdr_len is not
* possible. The payload-only skbs should never leave their owner.
*/
#define SKB_DATAREF_SHIFT 16
#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
enum {
SKB_FCLONE_UNAVAILABLE, /* skb has no fclone (from head_cache) */
SKB_FCLONE_ORIG, /* orig skb (from fclone_cache) */
SKB_FCLONE_CLONE, /* companion fclone skb (from fclone_cache) */
};
enum {
SKB_GSO_TCPV4 = 1 << 0,
/* This indicates the skb is from an untrusted source. */
SKB_GSO_DODGY = 1 << 1,
/* This indicates the tcp segment has CWR set. */
SKB_GSO_TCP_ECN = 1 << 2,
SKB_GSO_TCP_FIXEDID = 1 << 3,
SKB_GSO_TCPV6 = 1 << 4,
SKB_GSO_FCOE = 1 << 5,
SKB_GSO_GRE = 1 << 6,
SKB_GSO_GRE_CSUM = 1 << 7,
SKB_GSO_IPXIP4 = 1 << 8,
SKB_GSO_IPXIP6 = 1 << 9,
SKB_GSO_UDP_TUNNEL = 1 << 10,
SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
SKB_GSO_PARTIAL = 1 << 12,
SKB_GSO_TUNNEL_REMCSUM = 1 << 13,
SKB_GSO_SCTP = 1 << 14,
SKB_GSO_ESP = 1 << 15,
SKB_GSO_UDP = 1 << 16,
SKB_GSO_UDP_L4 = 1 << 17,
SKB_GSO_FRAGLIST = 1 << 18,
};
#if BITS_PER_LONG > 32
#define NET_SKBUFF_DATA_USES_OFFSET 1
#endif
#ifdef NET_SKBUFF_DATA_USES_OFFSET
typedef unsigned int sk_buff_data_t;
#else
typedef unsigned char *sk_buff_data_t;
#endif
/**
* DOC: Basic sk_buff geometry
*
* struct sk_buff itself is a metadata structure and does not hold any packet
* data. All the data is held in associated buffers.
*
* &sk_buff.head points to the main "head" buffer. The head buffer is divided
* into two parts:
*
* - data buffer, containing headers and sometimes payload;
* this is the part of the skb operated on by the common helpers
* such as skb_put() or skb_pull();
* - shared info (struct skb_shared_info) which holds an array of pointers
* to read-only data in the (page, offset, length) format.
*
* Optionally &skb_shared_info.frag_list may point to another skb.
*
* Basic diagram may look like this::
*
* ---------------
* | sk_buff |
* ---------------
* ,--------------------------- + head
* / ,----------------- + data
* / / ,----------- + tail
* | | | , + end
* | | | |
* v v v v
* -----------------------------------------------
* | headroom | data | tailroom | skb_shared_info |
* -----------------------------------------------
* + [page frag]
* + [page frag]
* + [page frag]
* + [page frag] ---------
* + frag_list --> | sk_buff |
* ---------
*
*/
/**
* struct sk_buff - socket buffer
* @next: Next buffer in list
* @prev: Previous buffer in list
* @tstamp: Time we arrived/left
* @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
* for retransmit timer
* @rbnode: RB tree node, alternative to next/prev for netem/tcp
* @list: queue head
* @ll_node: anchor in an llist (eg socket defer_list)
* @sk: Socket we are owned by
* @dev: Device we arrived on/are leaving by
* @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
* @cb: Control buffer. Free for use by every layer. Put private vars here
* @_skb_refdst: destination entry (with norefcount bit)
* @len: Length of actual data
* @data_len: Data length
* @mac_len: Length of link layer header
* @hdr_len: writable header length of cloned skb
* @csum: Checksum (must include start/offset pair)
* @csum_start: Offset from skb->head where checksumming should start
* @csum_offset: Offset from csum_start where checksum should be stored
* @priority: Packet queueing priority
* @ignore_df: allow local fragmentation
* @cloned: Head may be cloned (check refcnt to be sure)
* @ip_summed: Driver fed us an IP checksum
* @nohdr: Payload reference only, must not modify header
* @pkt_type: Packet class
* @fclone: skbuff clone status
* @ipvs_property: skbuff is owned by ipvs
* @inner_protocol_type: whether the inner protocol is
* ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO
* @remcsum_offload: remote checksum offload is enabled
* @offload_fwd_mark: Packet was L2-forwarded in hardware
* @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
* @tc_skip_classify: do not classify packet. set by IFB device
* @tc_at_ingress: used within tc_classify to distinguish in/egress
* @redirected: packet was redirected by packet classifier
* @from_ingress: packet was redirected from the ingress path
* @nf_skip_egress: packet shall skip nf egress - see netfilter_netdev.h
* @peeked: this packet has been seen already, so stats have been
* done for it, don't do them again
* @nf_trace: netfilter packet trace flag
* @protocol: Packet protocol from driver
* @destructor: Destruct function
* @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
* @_sk_redir: socket redirection information for skmsg
* @_nfct: Associated connection, if any (with nfctinfo bits)
* @skb_iif: ifindex of device we arrived on
* @tc_index: Traffic control index
* @hash: the packet hash
* @queue_mapping: Queue mapping for multiqueue devices
* @head_frag: skb was allocated from page fragments,
* not allocated by kmalloc() or vmalloc().
* @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
* @pp_recycle: mark the packet for recycling instead of freeing (implies
* page_pool support on driver)
* @active_extensions: active extensions (skb_ext_id types)
* @ndisc_nodetype: router type (from link layer)
* @ooo_okay: allow the mapping of a socket to a queue to be changed
* @l4_hash: indicate hash is a canonical 4-tuple hash over transport
* ports.
* @sw_hash: indicates hash was computed in software stack
* @wifi_acked_valid: wifi_acked was set
* @wifi_acked: whether frame was acked on wifi or not
* @no_fcs: Request NIC to treat last 4 bytes as Ethernet FCS
* @encapsulation: indicates the inner headers in the skbuff are valid
* @encap_hdr_csum: software checksum is needed
* @csum_valid: checksum is already valid
* @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
* @csum_complete_sw: checksum was completed by software
* @csum_level: indicates the number of consecutive checksums found in
* the packet minus one that have been verified as
* CHECKSUM_UNNECESSARY (max 3)
* @dst_pending_confirm: need to confirm neighbour
* @decrypted: Decrypted SKB
* @slow_gro: state present at GRO time, slower prepare step required
* @mono_delivery_time: When set, skb->tstamp has the
* delivery_time in mono clock base (i.e. EDT). Otherwise, the
* skb->tstamp has the (rcv) timestamp at ingress and
* delivery_time at egress.
* @napi_id: id of the NAPI struct this skb came from
* @sender_cpu: (aka @napi_id) source CPU in XPS
* @alloc_cpu: CPU which did the skb allocation.
* @secmark: security marking
* @mark: Generic packet mark
* @reserved_tailroom: (aka @mark) number of bytes of free space available
* at the tail of an sk_buff
* @vlan_all: vlan fields (proto & tci)
* @vlan_proto: vlan encapsulation protocol
* @vlan_tci: vlan tag control information
* @inner_protocol: Protocol (encapsulation)
* @inner_ipproto: (aka @inner_protocol) stores ipproto when
* skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
* @inner_transport_header: Inner transport layer header (encapsulation)
* @inner_network_header: Network layer header (encapsulation)
* @inner_mac_header: Link layer header (encapsulation)
* @transport_header: Transport layer header
* @network_header: Network layer header
* @mac_header: Link layer header
* @kcov_handle: KCOV remote handle for remote coverage collection
* @tail: Tail pointer
* @end: End pointer
* @head: Head of buffer
* @data: Data head pointer
* @truesize: Buffer size
* @users: User count - see {datagram,tcp}.c
* @extensions: allocated extensions, valid if active_extensions is nonzero
*/
struct sk_buff {
union {
struct {
/* These two members must be first to match sk_buff_head. */
struct sk_buff *next;
struct sk_buff *prev;
union {
struct net_device *dev;
/* Some protocols might use this space to store information,
* while device pointer would be NULL.
* UDP receive path is one user.
*/
unsigned long dev_scratch;
};
};
struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */
struct list_head list;
struct llist_node ll_node;
};
struct sock *sk;
union {
ktime_t tstamp;
u64 skb_mstamp_ns; /* earliest departure time */
};
/*
* This is the control buffer. It is free to use for every
* layer. Please put your private variables there. If you
* want to keep them across layers you have to do a skb_clone()
* first. This is owned by whoever has the skb queued ATM.
*/
char cb[48] __aligned(8);
union {
struct {
unsigned long _skb_refdst;
void (*destructor)(struct sk_buff *skb);
};
struct list_head tcp_tsorted_anchor;
#ifdef CONFIG_NET_SOCK_MSG
unsigned long _sk_redir;
#endif
};
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
unsigned long _nfct;
#endif
unsigned int len,
data_len;
__u16 mac_len,
hdr_len;
/* Following fields are _not_ copied in __copy_skb_header()
* Note that queue_mapping is here mostly to fill a hole.
*/
__u16 queue_mapping;
/* if you move cloned around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define CLONED_MASK (1 << 7)
#else
#define CLONED_MASK 1
#endif
#define CLONED_OFFSET offsetof(struct sk_buff, __cloned_offset)
/* private: */
__u8 __cloned_offset[0];
/* public: */
__u8 cloned:1,
nohdr:1,
fclone:2,
peeked:1,
head_frag:1,
pfmemalloc:1,
pp_recycle:1; /* page_pool recycle indicator */
#ifdef CONFIG_SKB_EXTENSIONS
__u8 active_extensions;
#endif
/* Fields enclosed in headers group are copied
* using a single memcpy() in __copy_skb_header()
*/
struct_group(headers,
/* private: */
__u8 __pkt_type_offset[0];
/* public: */
__u8 pkt_type:3; /* see PKT_TYPE_MAX */
__u8 ignore_df:1;
__u8 dst_pending_confirm:1;
__u8 ip_summed:2;
__u8 ooo_okay:1;
/* private: */
__u8 __mono_tc_offset[0];
/* public: */
__u8 mono_delivery_time:1; /* See SKB_MONO_DELIVERY_TIME_MASK */
#ifdef CONFIG_NET_XGRESS
__u8 tc_at_ingress:1; /* See TC_AT_INGRESS_MASK */
__u8 tc_skip_classify:1;
#endif
__u8 remcsum_offload:1;
__u8 csum_complete_sw:1;
__u8 csum_level:2;
__u8 inner_protocol_type:1;
__u8 l4_hash:1;
__u8 sw_hash:1;
#ifdef CONFIG_WIRELESS
__u8 wifi_acked_valid:1;
__u8 wifi_acked:1;
#endif
__u8 no_fcs:1;
/* Indicates the inner headers are valid in the skbuff. */
__u8 encapsulation:1;
__u8 encap_hdr_csum:1;
__u8 csum_valid:1;
#ifdef CONFIG_IPV6_NDISC_NODETYPE
__u8 ndisc_nodetype:2;
#endif
#if IS_ENABLED(CONFIG_IP_VS)
__u8 ipvs_property:1;
#endif
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
__u8 nf_trace:1;
#endif
#ifdef CONFIG_NET_SWITCHDEV
__u8 offload_fwd_mark:1;
__u8 offload_l3_fwd_mark:1;
#endif
__u8 redirected:1;
#ifdef CONFIG_NET_REDIRECT
__u8 from_ingress:1;
#endif
#ifdef CONFIG_NETFILTER_SKIP_EGRESS
__u8 nf_skip_egress:1;
#endif
#ifdef CONFIG_TLS_DEVICE
__u8 decrypted:1;
#endif
__u8 slow_gro:1;
#if IS_ENABLED(CONFIG_IP_SCTP)
__u8 csum_not_inet:1;
#endif
#if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
__u16 tc_index; /* traffic control index */
#endif
u16 alloc_cpu;
union {
__wsum csum;
struct {
__u16 csum_start;
__u16 csum_offset;
};
};
__u32 priority;
int skb_iif;
__u32 hash;
union {
u32 vlan_all;
struct {
__be16 vlan_proto;
__u16 vlan_tci;
};
};
#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
union {
unsigned int napi_id;
unsigned int sender_cpu;
};
#endif
#ifdef CONFIG_NETWORK_SECMARK
__u32 secmark;
#endif
union {
__u32 mark;
__u32 reserved_tailroom;
};
union {
__be16 inner_protocol;
__u8 inner_ipproto;
};
__u16 inner_transport_header;
__u16 inner_network_header;
__u16 inner_mac_header;
__be16 protocol;
__u16 transport_header;
__u16 network_header;
__u16 mac_header;
#ifdef CONFIG_KCOV
u64 kcov_handle;
#endif
); /* end headers group */
/* These elements must be at the end, see alloc_skb() for details. */
sk_buff_data_t tail;
sk_buff_data_t end;
unsigned char *head,
*data;
unsigned int truesize;
refcount_t users;
#ifdef CONFIG_SKB_EXTENSIONS
/* only usable after checking ->active_extensions != 0 */
struct skb_ext *extensions;
#endif
};
/* if you move pkt_type around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define PKT_TYPE_MAX (7 << 5)
#else
#define PKT_TYPE_MAX 7
#endif
#define PKT_TYPE_OFFSET offsetof(struct sk_buff, __pkt_type_offset)
/* if you move tc_at_ingress or mono_delivery_time
* around, you also must adapt these constants.
*/
#ifdef __BIG_ENDIAN_BITFIELD
#define SKB_MONO_DELIVERY_TIME_MASK (1 << 7)
#define TC_AT_INGRESS_MASK (1 << 6)
#else
#define SKB_MONO_DELIVERY_TIME_MASK (1 << 0)
#define TC_AT_INGRESS_MASK (1 << 1)
#endif
#define SKB_BF_MONO_TC_OFFSET offsetof(struct sk_buff, __mono_tc_offset)
#ifdef __KERNEL__
/*
* Handling routines are only of interest to the kernel
*/
#define SKB_ALLOC_FCLONE 0x01
#define SKB_ALLOC_RX 0x02
#define SKB_ALLOC_NAPI 0x04
/**
* skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
* @skb: buffer
*/
static inline bool skb_pfmemalloc(const struct sk_buff *skb)
{
return unlikely(skb->pfmemalloc);
}
/*
* skb might have a dst pointer attached, refcounted or not.
* _skb_refdst low order bit is set if refcount was _not_ taken
*/
#define SKB_DST_NOREF 1UL
#define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
/**
* skb_dst - returns skb dst_entry
* @skb: buffer
*
* Returns skb dst_entry, regardless of reference taken or not.
*/
static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
{
/* If refdst was not refcounted, check we still are in a
* rcu_read_lock section
*/
WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
!rcu_read_lock_held() &&
!rcu_read_lock_bh_held());
return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
}
/**
* skb_dst_set - sets skb dst
* @skb: buffer
* @dst: dst entry
*
* Sets skb dst, assuming a reference was taken on dst and should
* be released by skb_dst_drop()
*/
static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
{
skb->slow_gro |= !!dst;
skb->_skb_refdst = (unsigned long)dst;
}
/**
* skb_dst_set_noref - sets skb dst, hopefully, without taking reference
* @skb: buffer
* @dst: dst entry
*
* Sets skb dst, assuming a reference was not taken on dst.
* If dst entry is cached, we do not take reference and dst_release
* will be avoided by refdst_drop. If dst entry is not cached, we take
* reference, so that last dst_release can destroy the dst immediately.
*/
static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
{
WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
skb->slow_gro |= !!dst;
skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
}
/**
* skb_dst_is_noref - Test if skb dst isn't refcounted
* @skb: buffer
*/
static inline bool skb_dst_is_noref(const struct sk_buff *skb)
{
return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
}
/**
* skb_rtable - Returns the skb &rtable
* @skb: buffer
*/
static inline struct rtable *skb_rtable(const struct sk_buff *skb)
{
return (struct rtable *)skb_dst(skb);
}
/* For mangling skb->pkt_type from user space side from applications
* such as nft, tc, etc, we only allow a conservative subset of
* possible pkt_types to be set.
*/
static inline bool skb_pkt_type_ok(u32 ptype)
{
return ptype <= PACKET_OTHERHOST;
}
/**
* skb_napi_id - Returns the skb's NAPI id
* @skb: buffer
*/
static inline unsigned int skb_napi_id(const struct sk_buff *skb)
{
#ifdef CONFIG_NET_RX_BUSY_POLL
return skb->napi_id;
#else
return 0;
#endif
}
static inline bool skb_wifi_acked_valid(const struct sk_buff *skb)
{
#ifdef CONFIG_WIRELESS
return skb->wifi_acked_valid;
#else
return 0;
#endif
}
/**
* skb_unref - decrement the skb's reference count
* @skb: buffer
*
* Returns true if we can free the skb.
*/
static inline bool skb_unref(struct sk_buff *skb)
{
if (unlikely(!skb))
return false;
if (likely(refcount_read(&skb->users) == 1))
smp_rmb();
else if (likely(!refcount_dec_and_test(&skb->users)))
return false;
return true;
}
static inline bool skb_data_unref(const struct sk_buff *skb,
struct skb_shared_info *shinfo)
{
int bias;
if (!skb->cloned)
return true;
bias = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1; if (atomic_read(&shinfo->dataref) == bias) smp_rmb(); else if (atomic_sub_return(bias, &shinfo->dataref))
return false;
return true;
}
void __fix_address
kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason);
/**
* kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
* @skb: buffer to free
*/
static inline void kfree_skb(struct sk_buff *skb)
{
kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
}
void skb_release_head_state(struct sk_buff *skb);
void kfree_skb_list_reason(struct sk_buff *segs,
enum skb_drop_reason reason);
void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
void skb_tx_error(struct sk_buff *skb);
static inline void kfree_skb_list(struct sk_buff *segs)
{
kfree_skb_list_reason(segs, SKB_DROP_REASON_NOT_SPECIFIED);
}
#ifdef CONFIG_TRACEPOINTS
void consume_skb(struct sk_buff *skb);
#else
static inline void consume_skb(struct sk_buff *skb)
{
return kfree_skb(skb);
}
#endif
void __consume_stateless_skb(struct sk_buff *skb);
void __kfree_skb(struct sk_buff *skb);
void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
bool *fragstolen, int *delta_truesize);
struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
int node);
struct sk_buff *__build_skb(void *data, unsigned int frag_size);
struct sk_buff *build_skb(void *data, unsigned int frag_size);
struct sk_buff *build_skb_around(struct sk_buff *skb,
void *data, unsigned int frag_size);
void skb_attempt_defer_free(struct sk_buff *skb);
struct sk_buff *napi_build_skb(void *data, unsigned int frag_size);
struct sk_buff *slab_build_skb(void *data);
/**
* alloc_skb - allocate a network buffer
* @size: size to allocate
* @priority: allocation mask
*
* This function is a convenient wrapper around __alloc_skb().
*/
static inline struct sk_buff *alloc_skb(unsigned int size,
gfp_t priority)
{
return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
}
struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
unsigned long data_len,
int max_page_order,
int *errcode,
gfp_t gfp_mask);
struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);
/* Layout of fast clones : [skb1][skb2][fclone_ref] */
struct sk_buff_fclones {
struct sk_buff skb1;
struct sk_buff skb2;
refcount_t fclone_ref;
};
/**
* skb_fclone_busy - check if fclone is busy
* @sk: socket
* @skb: buffer
*
* Returns true if skb is a fast clone, and its clone is not freed.
* Some drivers call skb_orphan() in their ndo_start_xmit(),
* so we also check that didn't happen.
*/
static inline bool skb_fclone_busy(const struct sock *sk,
const struct sk_buff *skb)
{
const struct sk_buff_fclones *fclones;
fclones = container_of(skb, struct sk_buff_fclones, skb1);
return skb->fclone == SKB_FCLONE_ORIG &&
refcount_read(&fclones->fclone_ref) > 1 &&
READ_ONCE(fclones->skb2.sk) == sk;
}
/**
* alloc_skb_fclone - allocate a network buffer from fclone cache
* @size: size to allocate
* @priority: allocation mask
*
* This function is a convenient wrapper around __alloc_skb().
*/
static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
gfp_t priority)
{
return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
}
struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
void skb_headers_offset_update(struct sk_buff *skb, int off);
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
gfp_t gfp_mask, bool fclone);
static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
gfp_t gfp_mask)
{
return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
}
int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
unsigned int headroom);
struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom);
struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
int newtailroom, gfp_t priority);
int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
int offset, int len);
int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
int offset, int len);
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
/**
* skb_pad - zero pad the tail of an skb
* @skb: buffer to pad
* @pad: space to pad
*
* Ensure that a buffer is followed by a padding area that is zero
* filled. Used by network drivers which may DMA or transfer data
* beyond the buffer end onto the wire.
*
* May return error in out of memory cases. The skb is freed on error.
*/
static inline int skb_pad(struct sk_buff *skb, int pad)
{
return __skb_pad(skb, pad, true);
}
#define dev_kfree_skb(a) consume_skb(a)
int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
int offset, size_t size, size_t max_frags);
struct skb_seq_state {
__u32 lower_offset;
__u32 upper_offset;
__u32 frag_idx;
__u32 stepped_offset;
struct sk_buff *root_skb;
struct sk_buff *cur_skb;
__u8 *frag_data;
__u32 frag_off;
};
void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
unsigned int to, struct skb_seq_state *st);
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
struct skb_seq_state *st);
void skb_abort_seq_read(struct skb_seq_state *st);
unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
unsigned int to, struct ts_config *config);
/*
* Packet hash types specify the type of hash in skb_set_hash.
*
* Hash types refer to the protocol layer addresses which are used to
* construct a packet's hash. The hashes are used to differentiate or identify
* flows of the protocol layer for the hash type. Hash types are either
* layer-2 (L2), layer-3 (L3), or layer-4 (L4).
*
* Properties of hashes:
*
* 1) Two packets in different flows have different hash values
* 2) Two packets in the same flow should have the same hash value
*
* A hash at a higher layer is considered to be more specific. A driver should
* set the most specific hash possible.
*
* A driver cannot indicate a more specific hash than the layer at which a hash
* was computed. For instance an L3 hash cannot be set as an L4 hash.
*
* A driver may indicate a hash level which is less specific than the
* actual layer the hash was computed on. For instance, a hash computed
* at L4 may be considered an L3 hash. This should only be done if the
* driver can't unambiguously determine that the HW computed the hash at
* the higher layer. Note that the "should" in the second property above
* permits this.
*/
enum pkt_hash_types {
PKT_HASH_TYPE_NONE, /* Undefined type */
PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */
PKT_HASH_TYPE_L3, /* Input: src_IP, dst_IP */
PKT_HASH_TYPE_L4, /* Input: src_IP, dst_IP, src_port, dst_port */
};
static inline void skb_clear_hash(struct sk_buff *skb)
{
skb->hash = 0;
skb->sw_hash = 0;
skb->l4_hash = 0;
}
static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
{
if (!skb->l4_hash)
skb_clear_hash(skb);
}
static inline void
__skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
{
skb->l4_hash = is_l4;
skb->sw_hash = is_sw;
skb->hash = hash;
}
static inline void
skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
{
/* Used by drivers to set hash from HW */
__skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
}
static inline void
__skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
{
__skb_set_hash(skb, hash, true, is_l4);
}
void __skb_get_hash(struct sk_buff *skb);
u32 __skb_get_hash_symmetric(const struct sk_buff *skb);
u32 skb_get_poff(const struct sk_buff *skb);
u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
const struct flow_keys_basic *keys, int hlen);
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
const void *data, int hlen_proto);
static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
int thoff, u8 ip_proto)
{
return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
}
void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
const struct flow_dissector_key *key,
unsigned int key_count);
struct bpf_flow_dissector;
u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
__be16 proto, int nhoff, int hlen, unsigned int flags);
bool __skb_flow_dissect(const struct net *net,
const struct sk_buff *skb,
struct flow_dissector *flow_dissector,
void *target_container, const void *data,
__be16 proto, int nhoff, int hlen, unsigned int flags);
static inline bool skb_flow_dissect(const struct sk_buff *skb,
struct flow_dissector *flow_dissector,
void *target_container, unsigned int flags)
{
return __skb_flow_dissect(NULL, skb, flow_dissector,
target_container, NULL, 0, 0, 0, flags);
}
static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
struct flow_keys *flow,
unsigned int flags)
{
memset(flow, 0, sizeof(*flow));
return __skb_flow_dissect(NULL, skb, &flow_keys_dissector,
flow, NULL, 0, 0, 0, flags);
}
static inline bool
skb_flow_dissect_flow_keys_basic(const struct net *net,
const struct sk_buff *skb,
struct flow_keys_basic *flow,
const void *data, __be16 proto,
int nhoff, int hlen, unsigned int flags)
{
memset(flow, 0, sizeof(*flow));
return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
data, proto, nhoff, hlen, flags);
}
void skb_flow_dissect_meta(const struct sk_buff *skb,
struct flow_dissector *flow_dissector,
void *target_container);
/* Gets a skb connection tracking info, ctinfo map should be a
* map of mapsize to translate enum ip_conntrack_info states
* to user states.
*/
void
skb_flow_dissect_ct(const struct sk_buff *skb,
struct flow_dissector *flow_dissector,
void *target_container,
u16 *ctinfo_map, size_t mapsize,
bool post_ct, u16 zone);
void
skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
struct flow_dissector *flow_dissector,
void *target_container);
void skb_flow_dissect_hash(const struct sk_buff *skb,
struct flow_dissector *flow_dissector,
void *target_container);
static inline __u32 skb_get_hash(struct sk_buff *skb)
{
if (!skb->l4_hash && !skb->sw_hash)
__skb_get_hash(skb);
return skb->hash;
}
static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
{
if (!skb->l4_hash && !skb->sw_hash) {
struct flow_keys keys;
__u32 hash = __get_hash_from_flowi6(fl6, &keys);
__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
}
return skb->hash;
}
__u32 skb_get_hash_perturb(const struct sk_buff *skb,
const siphash_key_t *perturb);
static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
{
return skb->hash;
}
static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
{
to->hash = from->hash;
to->sw_hash = from->sw_hash;
to->l4_hash = from->l4_hash;
};
static inline int skb_cmp_decrypted(const struct sk_buff *skb1,
const struct sk_buff *skb2)
{
#ifdef CONFIG_TLS_DEVICE
return skb2->decrypted - skb1->decrypted;
#else
return 0;
#endif
}
static inline void skb_copy_decrypted(struct sk_buff *to,
const struct sk_buff *from)
{
#ifdef CONFIG_TLS_DEVICE
to->decrypted = from->decrypted;
#endif
}
#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
return skb->head + skb->end;
}
static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
return skb->end;
}
static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
{
skb->end = offset;
}
#else
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
return skb->end;
}
static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
return skb->end - skb->head;
}
static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
{
skb->end = skb->head + offset;
}
#endif
struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
struct ubuf_info *uarg);
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
bool success);
int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
struct sk_buff *skb, struct iov_iter *from,
size_t length);
static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
struct msghdr *msg, int len)
{
return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len);
}
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
struct msghdr *msg, int len,
struct ubuf_info *uarg);
/* Internal */
#define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
{
return &skb_shinfo(skb)->hwtstamps;
}
static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
{
bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE; return is_zcopy ? skb_uarg(skb) : NULL;
}
static inline bool skb_zcopy_pure(const struct sk_buff *skb)
{
return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY;
}
static inline bool skb_zcopy_managed(const struct sk_buff *skb)
{
return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS;
}
static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1,
const struct sk_buff *skb2)
{
return skb_zcopy_pure(skb1) == skb_zcopy_pure(skb2);
}
static inline void net_zcopy_get(struct ubuf_info *uarg)
{
refcount_inc(&uarg->refcnt);
}
static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg)
{
skb_shinfo(skb)->destructor_arg = uarg;
skb_shinfo(skb)->flags |= uarg->flags;
}
static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
bool *have_ref)
{
if (skb && uarg && !skb_zcopy(skb)) {
if (unlikely(have_ref && *have_ref))
*have_ref = false;
else
net_zcopy_get(uarg);
skb_zcopy_init(skb, uarg);
}
}
static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
{
skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG;
}
static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
{
return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
}
static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
{
return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
}
static inline void net_zcopy_put(struct ubuf_info *uarg)
{
if (uarg)
uarg->callback(NULL, uarg, true);
}
static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{
if (uarg) {
if (uarg->callback == msg_zerocopy_callback)
msg_zerocopy_put_abort(uarg, have_uref);
else if (have_uref)
net_zcopy_put(uarg);
}
}
/* Release a reference on a zerocopy structure */
static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
{
struct ubuf_info *uarg = skb_zcopy(skb);
if (uarg) {
if (!skb_zcopy_is_nouarg(skb))
uarg->callback(skb, uarg, zerocopy_success); skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
}
}
void __skb_zcopy_downgrade_managed(struct sk_buff *skb);
static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb)
{
if (unlikely(skb_zcopy_managed(skb))) __skb_zcopy_downgrade_managed(skb);
}
static inline void skb_mark_not_on_list(struct sk_buff *skb)
{
skb->next = NULL;
}
static inline void skb_poison_list(struct sk_buff *skb)
{
#ifdef CONFIG_DEBUG_NET
skb->next = SKB_LIST_POISON_NEXT;
#endif
}
/* Iterate through singly-linked GSO fragments of an skb. */
#define skb_list_walk_safe(first, skb, next_skb) \
for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \
(skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)
static inline void skb_list_del_init(struct sk_buff *skb)
{
__list_del_entry(&skb->list);
skb_mark_not_on_list(skb);
}
/**
* skb_queue_empty - check if a queue is empty
* @list: queue head
*
* Returns true if the queue is empty, false otherwise.
*/
static inline int skb_queue_empty(const struct sk_buff_head *list)
{
return list->next == (const struct sk_buff *) list;
}
/**
* skb_queue_empty_lockless - check if a queue is empty
* @list: queue head
*
* Returns true if the queue is empty, false otherwise.
* This variant can be used in lockless contexts.
*/
static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
{
return READ_ONCE(list->next) == (const struct sk_buff *) list;
}
/**
* skb_queue_is_last - check if skb is the last entry in the queue
* @list: queue head
* @skb: buffer
*
* Returns true if @skb is the last buffer on the list.
*/
static inline bool skb_queue_is_last(const struct sk_buff_head *list,
const struct sk_buff *skb)
{
return skb->next == (const struct sk_buff *) list;
}
/**
* skb_queue_is_first - check if skb is the first entry in the queue
* @list: queue head
* @skb: buffer
*
* Returns true if @skb is the first buffer on the list.
*/
static inline bool skb_queue_is_first(const struct sk_buff_head *list,
const struct sk_buff *skb)
{
return skb->prev == (const struct sk_buff *) list;
}
/**
* skb_queue_next - return the next packet in the queue
* @list: queue head
* @skb: current buffer
*
* Return the next packet in @list after @skb. It is only valid to
* call this if skb_queue_is_last() evaluates to false.
*/
static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
const struct sk_buff *skb)
{
/* This BUG_ON may seem severe, but if we just return then we
* are going to dereference garbage.
*/
BUG_ON(skb_queue_is_last(list, skb));
return skb->next;
}
/**
* skb_queue_prev - return the prev packet in the queue
* @list: queue head
* @skb: current buffer
*
* Return the prev packet in @list before @skb. It is only valid to
* call this if skb_queue_is_first() evaluates to false.
*/
static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
const struct sk_buff *skb)
{
/* This BUG_ON may seem severe, but if we just return then we
* are going to dereference garbage.
*/
BUG_ON(skb_queue_is_first(list, skb));
return skb->prev;
}
/**
* skb_get - reference buffer
* @skb: buffer to reference
*
* Makes another reference to a socket buffer and returns a pointer
* to the buffer.
*/
static inline struct sk_buff *skb_get(struct sk_buff *skb)
{
refcount_inc(&skb->users);
return skb;
}
/*
* If users == 1, we are the only owner and can avoid redundant atomic changes.
*/
/**
* skb_cloned - is the buffer a clone
* @skb: buffer to check
*
* Returns true if the buffer was generated with skb_clone() and is
* one of multiple shared copies of the buffer. Cloned buffers are
* shared data so must not be written to under normal circumstances.
*/
static inline int skb_cloned(const struct sk_buff *skb)
{
return skb->cloned &&
(atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
}
static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
{
might_sleep_if(gfpflags_allow_blocking(pri));
if (skb_cloned(skb))
return pskb_expand_head(skb, 0, 0, pri);
return 0;
}
/* This variant of skb_unclone() makes sure skb->truesize
* and skb_end_offset() are not changed, whenever a new skb->head is needed.
*
* Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X))
* when various debugging features are in place.
*/
int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
{
might_sleep_if(gfpflags_allow_blocking(pri));
if (skb_cloned(skb))
return __skb_unclone_keeptruesize(skb, pri);
return 0;
}
/**
* skb_header_cloned - is the header a clone
* @skb: buffer to check
*
* Returns true if modifying the header part of the buffer requires
* the data to be copied.
*/
static inline int skb_header_cloned(const struct sk_buff *skb)
{
int dataref;
if (!skb->cloned)
return 0;
dataref = atomic_read(&skb_shinfo(skb)->dataref);
dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
return dataref != 1;
}
static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
{
might_sleep_if(gfpflags_allow_blocking(pri));
if (skb_header_cloned(skb))
return pskb_expand_head(skb, 0, 0, pri);
return 0;
}
/**
* __skb_header_release() - allow clones to use the headroom
* @skb: buffer to operate on
*
* See "DOC: dataref and headerless skbs".
*/
static inline void __skb_header_release(struct sk_buff *skb)
{
skb->nohdr = 1;
atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
}
/**
* skb_shared - is the buffer shared
* @skb: buffer to check
*
* Returns true if more than one person has a reference to this
* buffer.
*/
static inline int skb_shared(const struct sk_buff *skb)
{
return refcount_read(&skb->users) != 1;
}
/**
* skb_share_check - check if buffer is shared and if so clone it
* @skb: buffer to check
* @pri: priority for memory allocation
*
* If the buffer is shared the buffer is cloned and the old copy
* drops a reference. A new clone with a single reference is returned.
* If the buffer is not shared the original buffer is returned. When
* being called from interrupt status or with spinlocks held pri must
* be GFP_ATOMIC.
*
* NULL is returned on a memory allocation failure.
*/
static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
{
might_sleep_if(gfpflags_allow_blocking(pri));
if (skb_shared(skb)) {
struct sk_buff *nskb = skb_clone(skb, pri);
if (likely(nskb))
consume_skb(skb);
else
kfree_skb(skb);
skb = nskb;
}
return skb;
}
/*
* Copy shared buffers into a new sk_buff. We effectively do COW on
* packets to handle cases where we have a local reader and forward
* and a couple of other messy ones. The normal one is tcpdumping
* a packet that's being forwarded.
*/
/**
* skb_unshare - make a copy of a shared buffer
* @skb: buffer to check
* @pri: priority for memory allocation
*
* If the socket buffer is a clone then this function creates a new
* copy of the data, drops a reference count on the old copy and returns
* the new copy with the reference count at 1. If the buffer is not a clone
* the original buffer is returned. When called with a spinlock held or
* from interrupt state @pri must be %GFP_ATOMIC
*
* %NULL is returned on a memory allocation failure.
*/
static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
gfp_t pri)
{
might_sleep_if(gfpflags_allow_blocking(pri));
if (skb_cloned(skb)) {
struct sk_buff *nskb = skb_copy(skb, pri);
/* Free our shared copy */
if (likely(nskb))
consume_skb(skb);
else
kfree_skb(skb);
skb = nskb;
}
return skb;
}
/**
* skb_peek - peek at the head of an &sk_buff_head
* @list_: list to peek at
*
* Peek an &sk_buff. Unlike most other operations you _MUST_
* be careful with this one. A peek leaves the buffer on the
* list and someone else may run off with it. You must hold
* the appropriate locks or have a private queue to do this.
*
* Returns %NULL for an empty list or a pointer to the head element.
* The reference count is not incremented and the reference is therefore
* volatile. Use with caution.
*/
static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
{
struct sk_buff *skb = list_->next;
if (skb == (struct sk_buff *)list_)
skb = NULL;
return skb;
}
/**
* __skb_peek - peek at the head of a non-empty &sk_buff_head
* @list_: list to peek at
*
* Like skb_peek(), but the caller knows that the list is not empty.
*/
static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
{
return list_->next;
}
/**
* skb_peek_next - peek skb following the given one from a queue
* @skb: skb to start from
* @list_: list to peek at
*
* Returns %NULL when the end of the list is met or a pointer to the
* next element. The reference count is not incremented and the
* reference is therefore volatile. Use with caution.
*/
static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
const struct sk_buff_head *list_)
{
struct sk_buff *next = skb->next;
if (next == (struct sk_buff *)list_)
next = NULL;
return next;
}
/**
* skb_peek_tail - peek at the tail of an &sk_buff_head
* @list_: list to peek at
*
* Peek an &sk_buff. Unlike most other operations you _MUST_
* be careful with this one. A peek leaves the buffer on the
* list and someone else may run off with it. You must hold
* the appropriate locks or have a private queue to do this.
*
* Returns %NULL for an empty list or a pointer to the tail element.
* The reference count is not incremented and the reference is therefore
* volatile. Use with caution.
*/
static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
{
struct sk_buff *skb = READ_ONCE(list_->prev);
if (skb == (struct sk_buff *)list_)
skb = NULL;
return skb;
}
/**
* skb_queue_len - get queue length
* @list_: list to measure
*
* Return the length of an &sk_buff queue.
*/
static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
{
return list_->qlen;
}
/**
* skb_queue_len_lockless - get queue length
* @list_: list to measure
*
* Return the length of an &sk_buff queue.
* This variant can be used in lockless contexts.
*/
static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_)
{
return READ_ONCE(list_->qlen);
}
/**
* __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
* @list: queue to initialize
*
* This initializes only the list and queue length aspects of
* an sk_buff_head object. This allows to initialize the list
* aspects of an sk_buff_head without reinitializing things like
* the spinlock. It can also be used for on-stack sk_buff_head
* objects where the spinlock is known to not be used.
*/
static inline void __skb_queue_head_init(struct sk_buff_head *list)
{
list->prev = list->next = (struct sk_buff *)list;
list->qlen = 0;
}
/*
* This function creates a split out lock class for each invocation;
* this is needed for now since a whole lot of users of the skb-queue
* infrastructure in drivers have different locking usage (in hardirq)
* than the networking core (in softirq only). In the long run either the
* network layer or drivers should need annotation to consolidate the
* main types of usage into 3 classes.
*/
static inline void skb_queue_head_init(struct sk_buff_head *list)
{
spin_lock_init(&list->lock);
__skb_queue_head_init(list);
}
static inline void skb_queue_head_init_class(struct sk_buff_head *list,
struct lock_class_key *class)
{
skb_queue_head_init(list);
lockdep_set_class(&list->lock, class);
}
/*
* Insert an sk_buff on a list.
*
* The "__skb_xxxx()" functions are the non-atomic ones that
* can only be called with interrupts disabled.
*/
static inline void __skb_insert(struct sk_buff *newsk,
struct sk_buff *prev, struct sk_buff *next,
struct sk_buff_head *list)
{
/* See skb_queue_empty_lockless() and skb_peek_tail()
* for the opposite READ_ONCE()
*/
WRITE_ONCE(newsk->next, next);
WRITE_ONCE(newsk->prev, prev);
WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk);
WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk);
WRITE_ONCE(list->qlen, list->qlen + 1);
}
static inline void __skb_queue_splice(const struct sk_buff_head *list,
struct sk_buff *prev,
struct sk_buff *next)
{
struct sk_buff *first = list->next;
struct sk_buff *last = list->prev;
WRITE_ONCE(first->prev, prev);
WRITE_ONCE(prev->next, first);
WRITE_ONCE(last->next, next);
WRITE_ONCE(next->prev, last);
}
/**
* skb_queue_splice - join two skb lists, this is designed for stacks
* @list: the new list to add
* @head: the place to add it in the first list
*/
static inline void skb_queue_splice(const struct sk_buff_head *list,
struct sk_buff_head *head)
{
if (!skb_queue_empty(list)) {
__skb_queue_splice(list, (struct sk_buff *) head, head->next);
head->qlen += list->qlen;
}
}
/**
* skb_queue_splice_init - join two skb lists and reinitialise the emptied list
* @list: the new list to add
* @head: the place to add it in the first list
*
* The list at @list is reinitialised
*/
static inline void skb_queue_splice_init(struct sk_buff_head *list,
struct sk_buff_head *head)
{
if (!skb_queue_empty(list)) {
__skb_queue_splice(list, (struct sk_buff *) head, head->next);
head->qlen += list->qlen;
__skb_queue_head_init(list);
}
}
/**
* skb_queue_splice_tail - join two skb lists, each list being a queue
* @list: the new list to add
* @head: the place to add it in the first list
*/
static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
struct sk_buff_head *head)
{
if (!skb_queue_empty(list)) {
__skb_queue_splice(list, head->prev, (struct sk_buff *) head);
head->qlen += list->qlen;
}
}
/**
* skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
* @list: the new list to add
* @head: the place to add it in the first list
*
* Each of the lists is a queue.
* The list at @list is reinitialised
*/
static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
struct sk_buff_head *head)
{
if (!skb_queue_empty(list)) {
__skb_queue_splice(list, head->prev, (struct sk_buff *) head);
head->qlen += list->qlen;
__skb_queue_head_init(list);
}
}
/**
* __skb_queue_after - queue a buffer at the list head
* @list: list to use
* @prev: place after this buffer
* @newsk: buffer to queue
*
* Queue a buffer int the middle of a list. This function takes no locks
* and you must therefore hold required locks before calling it.
*
* A buffer cannot be placed on two lists at the same time.
*/
static inline void __skb_queue_after(struct sk_buff_head *list,
struct sk_buff *prev,
struct sk_buff *newsk)
{
__skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list);
}
void skb_append(struct sk_buff *old, struct sk_buff *newsk,
struct sk_buff_head *list);
static inline void __skb_queue_before(struct sk_buff_head *list,
struct sk_buff *next,
struct sk_buff *newsk)
{
__skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list);
}
/**
* __skb_queue_head - queue a buffer at the list head
* @list: list to use
* @newsk: buffer to queue
*
* Queue a buffer at the start of a list. This function takes no locks
* and you must therefore hold required locks before calling it.
*
* A buffer cannot be placed on two lists at the same time.
*/
static inline void __skb_queue_head(struct sk_buff_head *list,
struct sk_buff *newsk)
{
__skb_queue_after(list, (struct sk_buff *)list, newsk);
}
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
/**
* __skb_queue_tail - queue a buffer at the list tail
* @list: list to use
* @newsk: buffer to queue
*
* Queue a buffer at the end of a list. This function takes no locks
* and you must therefore hold required locks before calling it.
*
* A buffer cannot be placed on two lists at the same time.
*/
static inline void __skb_queue_tail(struct sk_buff_head *list,
struct sk_buff *newsk)
{
__skb_queue_before(list, (struct sk_buff *)list, newsk);
}
void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
/*
* remove sk_buff from list. _Must_ be called atomically, and with
* the list known..
*/
void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
{
struct sk_buff *next, *prev;
WRITE_ONCE(list->qlen, list->qlen - 1);
next = skb->next;
prev = skb->prev;
skb->next = skb->prev = NULL;
WRITE_ONCE(next->prev, prev);
WRITE_ONCE(prev->next, next);
}
/**
* __skb_dequeue - remove from the head of the queue
* @list: list to dequeue from
*
* Remove the head of the list. This function does not take any locks
* so must be used with appropriate locks held only. The head item is
* returned or %NULL if the list is empty.
*/
static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
{
struct sk_buff *skb = skb_peek(list);
if (skb) __skb_unlink(skb, list);
return skb;
}
struct sk_buff *skb_dequeue(struct sk_buff_head *list);
/**
* __skb_dequeue_tail - remove from the tail of the queue
* @list: list to dequeue from
*
* Remove the tail of the list. This function does not take any locks
* so must be used with appropriate locks held only. The tail item is
* returned or %NULL if the list is empty.
*/
static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
{
struct sk_buff *skb = skb_peek_tail(list);
if (skb)
__skb_unlink(skb, list);
return skb;
}
struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
static inline bool skb_is_nonlinear(const struct sk_buff *skb)
{
return skb->data_len;
}
static inline unsigned int skb_headlen(const struct sk_buff *skb)
{
return skb->len - skb->data_len;
}
static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
{
unsigned int i, len = 0;
for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
return len;
}
static inline unsigned int skb_pagelen(const struct sk_buff *skb)
{
return skb_headlen(skb) + __skb_pagelen(skb);
}
static inline void skb_frag_fill_netmem_desc(skb_frag_t *frag,
netmem_ref netmem, int off,
int size)
{
frag->netmem = netmem;
frag->offset = off;
skb_frag_size_set(frag, size);
}
static inline void skb_frag_fill_page_desc(skb_frag_t *frag,
struct page *page,
int off, int size)
{
skb_frag_fill_netmem_desc(frag, page_to_netmem(page), off, size);
}
static inline void __skb_fill_netmem_desc_noacc(struct skb_shared_info *shinfo,
int i, netmem_ref netmem,
int off, int size)
{
skb_frag_t *frag = &shinfo->frags[i];
skb_frag_fill_netmem_desc(frag, netmem, off, size);
}
static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo,
int i, struct page *page,
int off, int size)
{
__skb_fill_netmem_desc_noacc(shinfo, i, page_to_netmem(page), off,
size);
}
/**
* skb_len_add - adds a number to len fields of skb
* @skb: buffer to add len to
* @delta: number of bytes to add
*/
static inline void skb_len_add(struct sk_buff *skb, int delta)
{
skb->len += delta;
skb->data_len += delta;
skb->truesize += delta;
}
/**
* __skb_fill_netmem_desc - initialise a fragment in an skb
* @skb: buffer containing fragment to be initialised
* @i: fragment index to initialise
* @netmem: the netmem to use for this fragment
* @off: the offset to the data with @page
* @size: the length of the data
*
* Initialises the @i'th fragment of @skb to point to &size bytes at
* offset @off within @page.
*
* Does not take any additional reference on the fragment.
*/
static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i,
netmem_ref netmem, int off, int size)
{
struct page *page = netmem_to_page(netmem);
__skb_fill_netmem_desc_noacc(skb_shinfo(skb), i, netmem, off, size);
/* Propagate page pfmemalloc to the skb if we can. The problem is
* that not all callers have unique ownership of the page but rely
* on page_is_pfmemalloc doing the right thing(tm).
*/
page = compound_head(page);
if (page_is_pfmemalloc(page))
skb->pfmemalloc = true;
}
static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
struct page *page, int off, int size)
{
__skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
}
static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i,
netmem_ref netmem, int off, int size)
{
__skb_fill_netmem_desc(skb, i, netmem, off, size);
skb_shinfo(skb)->nr_frags = i + 1;
}
/**
* skb_fill_page_desc - initialise a paged fragment in an skb
* @skb: buffer containing fragment to be initialised
* @i: paged fragment index to initialise
* @page: the page to use for this fragment
* @off: the offset to the data with @page
* @size: the length of the data
*
* As per __skb_fill_page_desc() -- initialises the @i'th fragment of
* @skb to point to @size bytes at offset @off within @page. In
* addition updates @skb such that @i is the last fragment.
*
* Does not take any additional reference on the fragment.
*/
static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
struct page *page, int off, int size)
{
skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
}
/**
* skb_fill_page_desc_noacc - initialise a paged fragment in an skb
* @skb: buffer containing fragment to be initialised
* @i: paged fragment index to initialise
* @page: the page to use for this fragment
* @off: the offset to the data with @page
* @size: the length of the data
*
* Variant of skb_fill_page_desc() which does not deal with
* pfmemalloc, if page is not owned by us.
*/
static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i,
struct page *page, int off,
int size)
{
struct skb_shared_info *shinfo = skb_shinfo(skb);
__skb_fill_page_desc_noacc(shinfo, i, page, off, size);
shinfo->nr_frags = i + 1;
}
void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
int off, int size, unsigned int truesize);
static inline void skb_add_rx_frag(struct sk_buff *skb, int i,
struct page *page, int off, int size,
unsigned int truesize)
{
skb_add_rx_frag_netmem(skb, i, page_to_netmem(page), off, size,
truesize);
}
void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
unsigned int truesize);
#define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
{
return skb->head + skb->tail;
}
static inline void skb_reset_tail_pointer(struct sk_buff *skb)
{
skb->tail = skb->data - skb->head;
}
static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
{
skb_reset_tail_pointer(skb);
skb->tail += offset;
}
#else /* NET_SKBUFF_DATA_USES_OFFSET */
static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
{
return skb->tail;
}
static inline void skb_reset_tail_pointer(struct sk_buff *skb)
{
skb->tail = skb->data;
}
static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
{
skb->tail = skb->data + offset;
}
#endif /* NET_SKBUFF_DATA_USES_OFFSET */
static inline void skb_assert_len(struct sk_buff *skb)
{
#ifdef CONFIG_DEBUG_NET
if (WARN_ONCE(!skb->len, "%s\n", __func__))
DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
#endif /* CONFIG_DEBUG_NET */
}
/*
* Add data to an sk_buff
*/
void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
void *skb_put(struct sk_buff *skb, unsigned int len);
static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
{
void *tmp = skb_tail_pointer(skb);
SKB_LINEAR_ASSERT(skb);
skb->tail += len;
skb->len += len;
return tmp;
}
static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
{
void *tmp = __skb_put(skb, len);
memset(tmp, 0, len);
return tmp;
}
static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
unsigned int len)
{
void *tmp = __skb_put(skb, len);
memcpy(tmp, data, len);
return tmp;
}
static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
{
*(u8 *)__skb_put(skb, 1) = val;
}
static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
{
void *tmp = skb_put(skb, len);
memset(tmp, 0, len);
return tmp;
}
static inline void *skb_put_data(struct sk_buff *skb, const void *data,
unsigned int len)
{
void *tmp = skb_put(skb, len);
memcpy(tmp, data, len);
return tmp;
}
static inline void skb_put_u8(struct sk_buff *skb, u8 val)
{
*(u8 *)skb_put(skb, 1) = val;
}
void *skb_push(struct sk_buff *skb, unsigned int len);
static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
{
DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
skb->data -= len;
skb->len += len;
return skb->data;
}
void *skb_pull(struct sk_buff *skb, unsigned int len);
static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
{
DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
skb->len -= len;
if (unlikely(skb->len < skb->data_len)) {
#if defined(CONFIG_DEBUG_NET)
skb->len += len;
pr_err("__skb_pull(len=%u)\n", len);
skb_dump(KERN_ERR, skb, false);
#endif
BUG();
}
return skb->data += len;
}
static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
{
return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
}
void *skb_pull_data(struct sk_buff *skb, size_t len);
void *__pskb_pull_tail(struct sk_buff *skb, int delta);
static inline enum skb_drop_reason
pskb_may_pull_reason(struct sk_buff *skb, unsigned int len)
{
DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
if (likely(len <= skb_headlen(skb)))
return SKB_NOT_DROPPED_YET;
if (unlikely(len > skb->len))
return SKB_DROP_REASON_PKT_TOO_SMALL;
if (unlikely(!__pskb_pull_tail(skb, len - skb_headlen(skb))))
return SKB_DROP_REASON_NOMEM;
return SKB_NOT_DROPPED_YET;
}
static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
{
return pskb_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
}
static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
{
if (!pskb_may_pull(skb, len))
return NULL;
skb->len -= len;
return skb->data += len;
}
void skb_condense(struct sk_buff *skb);
/**
* skb_headroom - bytes at buffer head
* @skb: buffer to check
*
* Return the number of bytes of free space at the head of an &sk_buff.
*/
static inline unsigned int skb_headroom(const struct sk_buff *skb)
{
return skb->data - skb->head;
}
/**
* skb_tailroom - bytes at buffer end
* @skb: buffer to check
*
* Return the number of bytes of free space at the tail of an sk_buff
*/
static inline int skb_tailroom(const struct sk_buff *skb)
{
return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
}
/**
* skb_availroom - bytes at buffer end
* @skb: buffer to check
*
* Return the number of bytes of free space at the tail of an sk_buff
* allocated by sk_stream_alloc()
*/
static inline int skb_availroom(const struct sk_buff *skb)
{
if (skb_is_nonlinear(skb))
return 0;
return skb->end - skb->tail - skb->reserved_tailroom;
}
/**
* skb_reserve - adjust headroom
* @skb: buffer to alter
* @len: bytes to move
*
* Increase the headroom of an empty &sk_buff by reducing the tail
* room. This is only allowed for an empty buffer.
*/
static inline void skb_reserve(struct sk_buff *skb, int len)
{
skb->data += len;
skb->tail += len;
}
/**
* skb_tailroom_reserve - adjust reserved_tailroom
* @skb: buffer to alter
* @mtu: maximum amount of headlen permitted
* @needed_tailroom: minimum amount of reserved_tailroom
*
* Set reserved_tailroom so that headlen can be as large as possible but
* not larger than mtu and tailroom cannot be smaller than
* needed_tailroom.
* The required headroom should already have been reserved before using
* this function.
*/
static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
unsigned int needed_tailroom)
{
SKB_LINEAR_ASSERT(skb);
if (mtu < skb_tailroom(skb) - needed_tailroom)
/* use at most mtu */
skb->reserved_tailroom = skb_tailroom(skb) - mtu;
else
/* use up to all available space */
skb->reserved_tailroom = needed_tailroom;
}
#define ENCAP_TYPE_ETHER 0
#define ENCAP_TYPE_IPPROTO 1
static inline void skb_set_inner_protocol(struct sk_buff *skb,
__be16 protocol)
{
skb->inner_protocol = protocol;
skb->inner_protocol_type = ENCAP_TYPE_ETHER;
}
static inline void skb_set_inner_ipproto(struct sk_buff *skb,
__u8 ipproto)
{
skb->inner_ipproto = ipproto;
skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
}
static inline void skb_reset_inner_headers(struct sk_buff *skb)
{
skb->inner_mac_header = skb->mac_header;
skb->inner_network_header = skb->network_header;
skb->inner_transport_header = skb->transport_header;
}
static inline void skb_reset_mac_len(struct sk_buff *skb)
{
skb->mac_len = skb->network_header - skb->mac_header;
}
static inline unsigned char *skb_inner_transport_header(const struct sk_buff
*skb)
{
return skb->head + skb->inner_transport_header;
}
static inline int skb_inner_transport_offset(const struct sk_buff *skb)
{
return skb_inner_transport_header(skb) - skb->data;
}
static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
{
skb->inner_transport_header = skb->data - skb->head;
}
static inline void skb_set_inner_transport_header(struct sk_buff *skb,
const int offset)
{
skb_reset_inner_transport_header(skb);
skb->inner_transport_header += offset;
}
static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
{
return skb->head + skb->inner_network_header;
}
static inline void skb_reset_inner_network_header(struct sk_buff *skb)
{
skb->inner_network_header = skb->data - skb->head;
}
static inline void skb_set_inner_network_header(struct sk_buff *skb,
const int offset)
{
skb_reset_inner_network_header(skb);
skb->inner_network_header += offset;
}
static inline bool skb_inner_network_header_was_set(const struct sk_buff *skb)
{
return skb->inner_network_header > 0;
}
static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
{
return skb->head + skb->inner_mac_header;
}
static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
{
skb->inner_mac_header = skb->data - skb->head;
}
static inline void skb_set_inner_mac_header(struct sk_buff *skb,
const int offset)
{
skb_reset_inner_mac_header(skb);
skb->inner_mac_header += offset;
}
static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
{
return skb->transport_header != (typeof(skb->transport_header))~0U;
}
static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
{
DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
return skb->head + skb->transport_header;
}
static inline void skb_reset_transport_header(struct sk_buff *skb)
{
skb->transport_header = skb->data - skb->head;
}
static inline void skb_set_transport_header(struct sk_buff *skb,
const int offset)
{
skb_reset_transport_header(skb);
skb->transport_header += offset;
}
static inline unsigned char *skb_network_header(const struct sk_buff *skb)
{
return skb->head + skb->network_header;
}
static inline void skb_reset_network_header(struct sk_buff *skb)
{
skb->network_header = skb->data - skb->head;
}
static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
{
skb_reset_network_header(skb);
skb->network_header += offset;
}
static inline int skb_mac_header_was_set(const struct sk_buff *skb)
{
return skb->mac_header != (typeof(skb->mac_header))~0U;
}
static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
{
DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
return skb->head + skb->mac_header;
}
static inline int skb_mac_offset(const struct sk_buff *skb)
{
return skb_mac_header(skb) - skb->data;
}
static inline u32 skb_mac_header_len(const struct sk_buff *skb)
{
DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
return skb->network_header - skb->mac_header;
}
static inline void skb_unset_mac_header(struct sk_buff *skb)
{
skb->mac_header = (typeof(skb->mac_header))~0U;
}
static inline void skb_reset_mac_header(struct sk_buff *skb)
{
skb->mac_header = skb->data - skb->head;
}
static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
{
skb_reset_mac_header(skb);
skb->mac_header += offset;
}
static inline void skb_pop_mac_header(struct sk_buff *skb)
{
skb->mac_header = skb->network_header;
}
static inline void skb_probe_transport_header(struct sk_buff *skb)
{
struct flow_keys_basic keys;
if (skb_transport_header_was_set(skb))
return;
if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
NULL, 0, 0, 0, 0))
skb_set_transport_header(skb, keys.control.thoff);
}
static inline void skb_mac_header_rebuild(struct sk_buff *skb)
{
if (skb_mac_header_was_set(skb)) {
const unsigned char *old_mac = skb_mac_header(skb);
skb_set_mac_header(skb, -skb->mac_len);
memmove(skb_mac_header(skb), old_mac, skb->mac_len);
}
}
static inline int skb_checksum_start_offset(const struct sk_buff *skb)
{
return skb->csum_start - skb_headroom(skb);
}
static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
{
return skb->head + skb->csum_start;
}
static inline int skb_transport_offset(const struct sk_buff *skb)
{
return skb_transport_header(skb) - skb->data;
}
static inline u32 skb_network_header_len(const struct sk_buff *skb)
{
DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
return skb->transport_header - skb->network_header;
}
static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
{
return skb->inner_transport_header - skb->inner_network_header;
}
static inline int skb_network_offset(const struct sk_buff *skb)
{
return skb_network_header(skb) - skb->data;
}
static inline int skb_inner_network_offset(const struct sk_buff *skb)
{
return skb_inner_network_header(skb) - skb->data;
}
static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
{
return pskb_may_pull(skb, skb_network_offset(skb) + len);
}
/*
* CPUs often take a performance hit when accessing unaligned memory
* locations. The actual performance hit varies, it can be small if the
* hardware handles it or large if we have to take an exception and fix it
* in software.
*
* Since an ethernet header is 14 bytes network drivers often end up with
* the IP header at an unaligned offset. The IP header can be aligned by
* shifting the start of the packet by 2 bytes. Drivers should do this
* with:
*
* skb_reserve(skb, NET_IP_ALIGN);
*
* The downside to this alignment of the IP header is that the DMA is now
* unaligned. On some architectures the cost of an unaligned DMA is high
* and this cost outweighs the gains made by aligning the IP header.
*
* Since this trade off varies between architectures, we allow NET_IP_ALIGN
* to be overridden.
*/
#ifndef NET_IP_ALIGN
#define NET_IP_ALIGN 2
#endif
/*
* The networking layer reserves some headroom in skb data (via
* dev_alloc_skb). This is used to avoid having to reallocate skb data when
* the header has to grow. In the default case, if the header has to grow
* 32 bytes or less we avoid the reallocation.
*
* Unfortunately this headroom changes the DMA alignment of the resulting
* network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
* on some architectures. An architecture can override this value,
* perhaps setting it to a cacheline in size (since that will maintain
* cacheline alignment of the DMA). It must be a power of 2.
*
* Various parts of the networking layer expect at least 32 bytes of
* headroom, you should not reduce this.
*
* Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
* to reduce average number of cache lines per packet.
* get_rps_cpu() for example only access one 64 bytes aligned block :
* NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
*/
#ifndef NET_SKB_PAD
#define NET_SKB_PAD max(32, L1_CACHE_BYTES)
#endif
int ___pskb_trim(struct sk_buff *skb, unsigned int len);
static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
{
if (WARN_ON(skb_is_nonlinear(skb)))
return;
skb->len = len;
skb_set_tail_pointer(skb, len);
}
static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
{
__skb_set_length(skb, len);
}
void skb_trim(struct sk_buff *skb, unsigned int len);
static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
{
if (skb->data_len) return ___pskb_trim(skb, len); __skb_trim(skb, len);
return 0;
}
static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
{
return (len < skb->len) ? __pskb_trim(skb, len) : 0;
}
/**
* pskb_trim_unique - remove end from a paged unique (not cloned) buffer
* @skb: buffer to alter
* @len: new length
*
* This is identical to pskb_trim except that the caller knows that
* the skb is not cloned so we should never get an error due to out-
* of-memory.
*/
static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
{
int err = pskb_trim(skb, len);
BUG_ON(err);
}
static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
{
unsigned int diff = len - skb->len;
if (skb_tailroom(skb) < diff) {
int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb),
GFP_ATOMIC);
if (ret)
return ret;
}
__skb_set_length(skb, len);
return 0;
}
/**
* skb_orphan - orphan a buffer
* @skb: buffer to orphan
*
* If a buffer currently has an owner then we call the owner's
* destructor function and make the @skb unowned. The buffer continues
* to exist but is no longer charged to its former owner.
*/
static inline void skb_orphan(struct sk_buff *skb)
{
if (skb->destructor) { skb->destructor(skb);
skb->destructor = NULL;
skb->sk = NULL;
} else {
BUG_ON(skb->sk);
}
}
/**
* skb_orphan_frags - orphan the frags contained in a buffer
* @skb: buffer to orphan frags from
* @gfp_mask: allocation mask for replacement pages
*
* For each frag in the SKB which needs a destructor (i.e. has an
* owner) create a copy of that frag and release the original
* page by calling the destructor.
*/
static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
{
if (likely(!skb_zcopy(skb)))
return 0;
if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN)
return 0;
return skb_copy_ubufs(skb, gfp_mask);
}
/* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
{
if (likely(!skb_zcopy(skb)))
return 0;
return skb_copy_ubufs(skb, gfp_mask);
}
/**
* __skb_queue_purge_reason - empty a list
* @list: list to empty
* @reason: drop reason
*
* Delete all buffers on an &sk_buff list. Each buffer is removed from
* the list and one reference dropped. This function does not take the
* list lock and the caller must hold the relevant locks to use it.
*/
static inline void __skb_queue_purge_reason(struct sk_buff_head *list,
enum skb_drop_reason reason)
{
struct sk_buff *skb;
while ((skb = __skb_dequeue(list)) != NULL)
kfree_skb_reason(skb, reason);
}
static inline void __skb_queue_purge(struct sk_buff_head *list)
{
__skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
}
void skb_queue_purge_reason(struct sk_buff_head *list,
enum skb_drop_reason reason);
static inline void skb_queue_purge(struct sk_buff_head *list)
{
skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
}
unsigned int skb_rbtree_purge(struct rb_root *root);
void skb_errqueue_purge(struct sk_buff_head *list);
void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);
/**
* netdev_alloc_frag - allocate a page fragment
* @fragsz: fragment size
*
* Allocates a frag from a page for receive buffer.
* Uses GFP_ATOMIC allocations.
*/
static inline void *netdev_alloc_frag(unsigned int fragsz)
{
return __netdev_alloc_frag_align(fragsz, ~0u);
}
static inline void *netdev_alloc_frag_align(unsigned int fragsz,
unsigned int align)
{
WARN_ON_ONCE(!is_power_of_2(align));
return __netdev_alloc_frag_align(fragsz, -align);
}
struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
gfp_t gfp_mask);
/**
* netdev_alloc_skb - allocate an skbuff for rx on a specific device
* @dev: network device to receive on
* @length: length to allocate
*
* Allocate a new &sk_buff and assign it a usage count of one. The
* buffer has unspecified headroom built in. Users should allocate
* the headroom they think they need without accounting for the
* built in space. The built in space is used for optimisations.
*
* %NULL is returned if there is no free memory. Although this function
* allocates memory it can be called from an interrupt.
*/
static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
unsigned int length)
{
return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
}
/* legacy helper around __netdev_alloc_skb() */
static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
gfp_t gfp_mask)
{
return __netdev_alloc_skb(NULL, length, gfp_mask);
}
/* legacy helper around netdev_alloc_skb() */
static inline struct sk_buff *dev_alloc_skb(unsigned int length)
{
return netdev_alloc_skb(NULL, length);
}
static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
unsigned int length, gfp_t gfp)
{
struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
if (NET_IP_ALIGN && skb)
skb_reserve(skb, NET_IP_ALIGN);
return skb;
}
static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
unsigned int length)
{
return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
}
static inline void skb_free_frag(void *addr)
{
page_frag_free(addr);
}
void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);
static inline void *napi_alloc_frag(unsigned int fragsz)
{
return __napi_alloc_frag_align(fragsz, ~0u);
}
static inline void *napi_alloc_frag_align(unsigned int fragsz,
unsigned int align)
{
WARN_ON_ONCE(!is_power_of_2(align));
return __napi_alloc_frag_align(fragsz, -align);
}
struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
unsigned int length, gfp_t gfp_mask);
static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,
unsigned int length)
{
return __napi_alloc_skb(napi, length, GFP_ATOMIC);
}
void napi_consume_skb(struct sk_buff *skb, int budget);
void napi_skb_free_stolen_head(struct sk_buff *skb);
void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason);
/**
* __dev_alloc_pages - allocate page for network Rx
* @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
* @order: size of the allocation
*
* Allocate a new page.
*
* %NULL is returned if there is no free memory.
*/
static inline struct page *__dev_alloc_pages(gfp_t gfp_mask,
unsigned int order)
{
/* This piece of code contains several assumptions.
* 1. This is for device Rx, therefore a cold page is preferred.
* 2. The expectation is the user wants a compound page.
* 3. If requesting a order 0 page it will not be compound
* due to the check to see if order has a value in prep_new_page
* 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
* code in gfp_to_alloc_flags that should be enforcing this.
*/
gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;
return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
}
static inline struct page *dev_alloc_pages(unsigned int order)
{
return __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, order);
}
/**
* __dev_alloc_page - allocate a page for network Rx
* @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
*
* Allocate a new page.
*
* %NULL is returned if there is no free memory.
*/
static inline struct page *__dev_alloc_page(gfp_t gfp_mask)
{
return __dev_alloc_pages(gfp_mask, 0);
}
static inline struct page *dev_alloc_page(void)
{
return dev_alloc_pages(0);
}
/**
* dev_page_is_reusable - check whether a page can be reused for network Rx
* @page: the page to test
*
* A page shouldn't be considered for reusing/recycling if it was allocated
* under memory pressure or at a distant memory node.
*
* Returns false if this page should be returned to page allocator, true
* otherwise.
*/
static inline bool dev_page_is_reusable(const struct page *page)
{
return likely(page_to_nid(page) == numa_mem_id() &&
!page_is_pfmemalloc(page));
}
/**
* skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
* @page: The page that was allocated from skb_alloc_page
* @skb: The skb that may need pfmemalloc set
*/
static inline void skb_propagate_pfmemalloc(const struct page *page,
struct sk_buff *skb)
{
if (page_is_pfmemalloc(page))
skb->pfmemalloc = true;
}
/**
* skb_frag_off() - Returns the offset of a skb fragment
* @frag: the paged fragment
*/
static inline unsigned int skb_frag_off(const skb_frag_t *frag)
{
return frag->offset;
}
/**
* skb_frag_off_add() - Increments the offset of a skb fragment by @delta
* @frag: skb fragment
* @delta: value to add
*/
static inline void skb_frag_off_add(skb_frag_t *frag, int delta)
{
frag->offset += delta;
}
/**
* skb_frag_off_set() - Sets the offset of a skb fragment
* @frag: skb fragment
* @offset: offset of fragment
*/
static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset)
{
frag->offset = offset;
}
/**
* skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
* @fragto: skb fragment where offset is set
* @fragfrom: skb fragment offset is copied from
*/
static inline void skb_frag_off_copy(skb_frag_t *fragto,
const skb_frag_t *fragfrom)
{
fragto->offset = fragfrom->offset;
}
/**
* skb_frag_page - retrieve the page referred to by a paged fragment
* @frag: the paged fragment
*
* Returns the &struct page associated with @frag.
*/
static inline struct page *skb_frag_page(const skb_frag_t *frag)
{
return netmem_to_page(frag->netmem);
}
/**
* __skb_frag_ref - take an addition reference on a paged fragment.
* @frag: the paged fragment
*
* Takes an additional reference on the paged fragment @frag.
*/
static inline void __skb_frag_ref(skb_frag_t *frag)
{
get_page(skb_frag_page(frag));
}
/**
* skb_frag_ref - take an addition reference on a paged fragment of an skb.
* @skb: the buffer
* @f: the fragment offset.
*
* Takes an additional reference on the @f'th paged fragment of @skb.
*/
static inline void skb_frag_ref(struct sk_buff *skb, int f)
{
__skb_frag_ref(&skb_shinfo(skb)->frags[f]);
}
int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
unsigned int headroom);
int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
struct bpf_prog *prog);
bool napi_pp_put_page(struct page *page, bool napi_safe);
static inline void
skb_page_unref(const struct sk_buff *skb, struct page *page, bool napi_safe)
{
#ifdef CONFIG_PAGE_POOL
if (skb->pp_recycle && napi_pp_put_page(page, napi_safe))
return;
#endif
put_page(page);
}
static inline void
napi_frag_unref(skb_frag_t *frag, bool recycle, bool napi_safe)
{
struct page *page = skb_frag_page(frag);
#ifdef CONFIG_PAGE_POOL
if (recycle && napi_pp_put_page(page, napi_safe))
return;
#endif
put_page(page);
}
/**
* __skb_frag_unref - release a reference on a paged fragment.
* @frag: the paged fragment
* @recycle: recycle the page if allocated via page_pool
*
* Releases a reference on the paged fragment @frag
* or recycles the page via the page_pool API.
*/
static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
{
napi_frag_unref(frag, recycle, false);
}
/**
* skb_frag_unref - release a reference on a paged fragment of an skb.
* @skb: the buffer
* @f: the fragment offset
*
* Releases a reference on the @f'th paged fragment of @skb.
*/
static inline void skb_frag_unref(struct sk_buff *skb, int f)
{
struct skb_shared_info *shinfo = skb_shinfo(skb);
if (!skb_zcopy_managed(skb))
__skb_frag_unref(&shinfo->frags[f], skb->pp_recycle);
}
/**
* skb_frag_address - gets the address of the data contained in a paged fragment
* @frag: the paged fragment buffer
*
* Returns the address of the data within @frag. The page must already
* be mapped.
*/
static inline void *skb_frag_address(const skb_frag_t *frag)
{
return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
}
/**
* skb_frag_address_safe - gets the address of the data contained in a paged fragment
* @frag: the paged fragment buffer
*
* Returns the address of the data within @frag. Checks that the page
* is mapped and returns %NULL otherwise.
*/
static inline void *skb_frag_address_safe(const skb_frag_t *frag)
{
void *ptr = page_address(skb_frag_page(frag));
if (unlikely(!ptr))
return NULL;
return ptr + skb_frag_off(frag);
}
/**
* skb_frag_page_copy() - sets the page in a fragment from another fragment
* @fragto: skb fragment where page is set
* @fragfrom: skb fragment page is copied from
*/
static inline void skb_frag_page_copy(skb_frag_t *fragto,
const skb_frag_t *fragfrom)
{
fragto->netmem = fragfrom->netmem;
}
bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
/**
* skb_frag_dma_map - maps a paged fragment via the DMA API
* @dev: the device to map the fragment to
* @frag: the paged fragment to map
* @offset: the offset within the fragment (starting at the
* fragment's own offset)
* @size: the number of bytes to map
* @dir: the direction of the mapping (``PCI_DMA_*``)
*
* Maps the page associated with @frag to @device.
*/
static inline dma_addr_t skb_frag_dma_map(struct device *dev,
const skb_frag_t *frag,
size_t offset, size_t size,
enum dma_data_direction dir)
{
return dma_map_page(dev, skb_frag_page(frag),
skb_frag_off(frag) + offset, size, dir);
}
static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
gfp_t gfp_mask)
{
return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
}
static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
gfp_t gfp_mask)
{
return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
}
/**
* skb_clone_writable - is the header of a clone writable
* @skb: buffer to check
* @len: length up to which to write
*
* Returns true if modifying the header part of the cloned buffer
* does not requires the data to be copied.
*/
static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
{
return !skb_header_cloned(skb) &&
skb_headroom(skb) + len <= skb->hdr_len;
}
static inline int skb_try_make_writable(struct sk_buff *skb,
unsigned int write_len)
{
return skb_cloned(skb) && !skb_clone_writable(skb, write_len) &&
pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
}
static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
int cloned)
{
int delta = 0;
if (headroom > skb_headroom(skb))
delta = headroom - skb_headroom(skb);
if (delta || cloned)
return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
GFP_ATOMIC);
return 0;
}
/**
* skb_cow - copy header of skb when it is required
* @skb: buffer to cow
* @headroom: needed headroom
*
* If the skb passed lacks sufficient headroom or its data part
* is shared, data is reallocated. If reallocation fails, an error
* is returned and original skb is not changed.
*
* The result is skb with writable area skb->head...skb->tail
* and at least @headroom of space at head.
*/
static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
{
return __skb_cow(skb, headroom, skb_cloned(skb));
}
/**
* skb_cow_head - skb_cow but only making the head writable
* @skb: buffer to cow
* @headroom: needed headroom
*
* This function is identical to skb_cow except that we replace the
* skb_cloned check by skb_header_cloned. It should be used when
* you only need to push on some header and do not need to modify
* the data.
*/
static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
{
return __skb_cow(skb, headroom, skb_header_cloned(skb));
}
/**
* skb_padto - pad an skbuff up to a minimal size
* @skb: buffer to pad
* @len: minimal length
*
* Pads up a buffer to ensure the trailing bytes exist and are
* blanked. If the buffer already contains sufficient data it
* is untouched. Otherwise it is extended. Returns zero on
* success. The skb is freed on error.
*/
static inline int skb_padto(struct sk_buff *skb, unsigned int len)
{
unsigned int size = skb->len;
if (likely(size >= len))
return 0;
return skb_pad(skb, len - size);
}
/**
* __skb_put_padto - increase size and pad an skbuff up to a minimal size
* @skb: buffer to pad
* @len: minimal length
* @free_on_error: free buffer on error
*
* Pads up a buffer to ensure the trailing bytes exist and are
* blanked. If the buffer already contains sufficient data it
* is untouched. Otherwise it is extended. Returns zero on
* success. The skb is freed on error if @free_on_error is true.
*/
static inline int __must_check __skb_put_padto(struct sk_buff *skb,
unsigned int len,
bool free_on_error)
{
unsigned int size = skb->len;
if (unlikely(size < len)) {
len -= size;
if (__skb_pad(skb, len, free_on_error))
return -ENOMEM;
__skb_put(skb, len);
}
return 0;
}
/**
* skb_put_padto - increase size and pad an skbuff up to a minimal size
* @skb: buffer to pad
* @len: minimal length
*
* Pads up a buffer to ensure the trailing bytes exist and are
* blanked. If the buffer already contains sufficient data it
* is untouched. Otherwise it is extended. Returns zero on
* success. The skb is freed on error.
*/
static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len)
{
return __skb_put_padto(skb, len, true);
}
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i)
__must_check;
static inline int skb_add_data(struct sk_buff *skb,
struct iov_iter *from, int copy)
{
const int off = skb->len;
if (skb->ip_summed == CHECKSUM_NONE) {
__wsum csum = 0;
if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy,
&csum, from)) {
skb->csum = csum_block_add(skb->csum, csum, off);
return 0;
}
} else if (copy_from_iter_full(skb_put(skb, copy), copy, from))
return 0;
__skb_trim(skb, off);
return -EFAULT;
}
static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
const struct page *page, int off)
{
if (skb_zcopy(skb))
return false;
if (i) {
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
return page == skb_frag_page(frag) &&
off == skb_frag_off(frag) + skb_frag_size(frag);
}
return false;
}
static inline int __skb_linearize(struct sk_buff *skb)
{
return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
}
/**
* skb_linearize - convert paged skb to linear one
* @skb: buffer to linarize
*
* If there is no free memory -ENOMEM is returned, otherwise zero
* is returned and the old skb data released.
*/
static inline int skb_linearize(struct sk_buff *skb)
{
return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
}
/**
* skb_has_shared_frag - can any frag be overwritten
* @skb: buffer to test
*
* Return true if the skb has at least one frag that might be modified
* by an external entity (as in vmsplice()/sendfile())
*/
static inline bool skb_has_shared_frag(const struct sk_buff *skb)
{
return skb_is_nonlinear(skb) &&
skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
}
/**
* skb_linearize_cow - make sure skb is linear and writable
* @skb: buffer to process
*
* If there is no free memory -ENOMEM is returned, otherwise zero
* is returned and the old skb data released.
*/
static inline int skb_linearize_cow(struct sk_buff *skb)
{
return skb_is_nonlinear(skb) || skb_cloned(skb) ?
__skb_linearize(skb) : 0;
}
static __always_inline void
__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
unsigned int off)
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->csum = csum_block_sub(skb->csum,
csum_partial(start, len, 0), off);
else if (skb->ip_summed == CHECKSUM_PARTIAL &&
skb_checksum_start_offset(skb) < 0)
skb->ip_summed = CHECKSUM_NONE;
}
/**
* skb_postpull_rcsum - update checksum for received skb after pull
* @skb: buffer to update
* @start: start of data before pull
* @len: length of data pulled
*
* After doing a pull on a received packet, you need to call this to
* update the CHECKSUM_COMPLETE checksum, or set ip_summed to
* CHECKSUM_NONE so that it can be recomputed from scratch.
*/
static inline void skb_postpull_rcsum(struct sk_buff *skb,
const void *start, unsigned int len)
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->csum = wsum_negate(csum_partial(start, len,
wsum_negate(skb->csum)));
else if (skb->ip_summed == CHECKSUM_PARTIAL &&
skb_checksum_start_offset(skb) < 0)
skb->ip_summed = CHECKSUM_NONE;
}
static __always_inline void
__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
unsigned int off)
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->csum = csum_block_add(skb->csum,
csum_partial(start, len, 0), off);
}
/**
* skb_postpush_rcsum - update checksum for received skb after push
* @skb: buffer to update
* @start: start of data after push
* @len: length of data pushed
*
* After doing a push on a received packet, you need to call this to
* update the CHECKSUM_COMPLETE checksum.
*/
static inline void skb_postpush_rcsum(struct sk_buff *skb,
const void *start, unsigned int len)
{
__skb_postpush_rcsum(skb, start, len, 0);
}
void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
/**
* skb_push_rcsum - push skb and update receive checksum
* @skb: buffer to update
* @len: length of data pulled
*
* This function performs an skb_push on the packet and updates
* the CHECKSUM_COMPLETE checksum. It should be used on
* receive path processing instead of skb_push unless you know
* that the checksum difference is zero (e.g., a valid IP header)
* or you are setting ip_summed to CHECKSUM_NONE.
*/
static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
{
skb_push(skb, len);
skb_postpush_rcsum(skb, skb->data, len);
return skb->data;
}
int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
/**
* pskb_trim_rcsum - trim received skb and update checksum
* @skb: buffer to trim
* @len: new length
*
* This is exactly the same as pskb_trim except that it ensures the
* checksum of received packets are still valid after the operation.
* It can change skb pointers.
*/
static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
{
if (likely(len >= skb->len))
return 0;
return pskb_trim_rcsum_slow(skb, len);
}
static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->ip_summed = CHECKSUM_NONE;
__skb_trim(skb, len);
return 0;
}
static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->ip_summed = CHECKSUM_NONE;
return __skb_grow(skb, len);
}
#define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
#define skb_rb_first(root) rb_to_skb(rb_first(root))
#define skb_rb_last(root) rb_to_skb(rb_last(root))
#define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode))
#define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode))
#define skb_queue_walk(queue, skb) \
for (skb = (queue)->next; \
skb != (struct sk_buff *)(queue); \
skb = skb->next)
#define skb_queue_walk_safe(queue, skb, tmp) \
for (skb = (queue)->next, tmp = skb->next; \
skb != (struct sk_buff *)(queue); \
skb = tmp, tmp = skb->next)
#define skb_queue_walk_from(queue, skb) \
for (; skb != (struct sk_buff *)(queue); \
skb = skb->next)
#define skb_rbtree_walk(skb, root) \
for (skb = skb_rb_first(root); skb != NULL; \
skb = skb_rb_next(skb))
#define skb_rbtree_walk_from(skb) \
for (; skb != NULL; \
skb = skb_rb_next(skb))
#define skb_rbtree_walk_from_safe(skb, tmp) \
for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL); \
skb = tmp)
#define skb_queue_walk_from_safe(queue, skb, tmp) \
for (tmp = skb->next; \
skb != (struct sk_buff *)(queue); \
skb = tmp, tmp = skb->next)
#define skb_queue_reverse_walk(queue, skb) \
for (skb = (queue)->prev; \
skb != (struct sk_buff *)(queue); \
skb = skb->prev)
#define skb_queue_reverse_walk_safe(queue, skb, tmp) \
for (skb = (queue)->prev, tmp = skb->prev; \
skb != (struct sk_buff *)(queue); \
skb = tmp, tmp = skb->prev)
#define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \
for (tmp = skb->prev; \
skb != (struct sk_buff *)(queue); \
skb = tmp, tmp = skb->prev)
static inline bool skb_has_frag_list(const struct sk_buff *skb)
{
return skb_shinfo(skb)->frag_list != NULL;
}
static inline void skb_frag_list_init(struct sk_buff *skb)
{
skb_shinfo(skb)->frag_list = NULL;
}
#define skb_walk_frags(skb, iter) \
for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue,
int *err, long *timeo_p,
const struct sk_buff *skb);
struct sk_buff *__skb_try_recv_from_queue(struct sock *sk,
struct sk_buff_head *queue,
unsigned int flags,
int *off, int *err,
struct sk_buff **last);
struct sk_buff *__skb_try_recv_datagram(struct sock *sk,
struct sk_buff_head *queue,
unsigned int flags, int *off, int *err,
struct sk_buff **last);
struct sk_buff *__skb_recv_datagram(struct sock *sk,
struct sk_buff_head *sk_queue,
unsigned int flags, int *off, int *err);
struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err);
__poll_t datagram_poll(struct file *file, struct socket *sock,
struct poll_table_struct *wait);
int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
struct iov_iter *to, int size);
static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
struct msghdr *msg, int size)
{
return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
}
int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
struct msghdr *msg);
int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
struct iov_iter *to, int len,
struct ahash_request *hash);
int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
struct iov_iter *from, int len);
int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len);
static inline void skb_free_datagram_locked(struct sock *sk,
struct sk_buff *skb)
{
__skb_free_datagram_locked(sk, skb, 0);
}
int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
int len);
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
struct pipe_inode_info *pipe, unsigned int len,
unsigned int flags);
int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
int len);
int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
int len, int hlen);
void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
void skb_scrub_packet(struct sk_buff *skb, bool xnet);
struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
unsigned int offset);
struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len);
int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev);
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
int skb_vlan_pop(struct sk_buff *skb);
int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
int skb_eth_pop(struct sk_buff *skb);
int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
const unsigned char *src);
int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
int mac_len, bool ethernet);
int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
bool ethernet);
int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
int skb_mpls_dec_ttl(struct sk_buff *skb);
struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
gfp_t gfp);
static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
{
return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT;
}
static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
{
return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
}
struct skb_checksum_ops {
__wsum (*update)(const void *mem, int len, __wsum wsum);
__wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
};
extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly;
__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
__wsum csum, const struct skb_checksum_ops *ops);
__wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
__wsum csum);
static inline void * __must_check
__skb_header_pointer(const struct sk_buff *skb, int offset, int len,
const void *data, int hlen, void *buffer)
{
if (likely(hlen - offset >= len))
return (void *)data + offset;
if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))
return NULL;
return buffer;
}
static inline void * __must_check
skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
{
return __skb_header_pointer(skb, offset, len, skb->data,
skb_headlen(skb), buffer);
}
static inline void * __must_check
skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len)
{
if (likely(skb_headlen(skb) - offset >= len))
return skb->data + offset;
return NULL;
}
/**
* skb_needs_linearize - check if we need to linearize a given skb
* depending on the given device features.
* @skb: socket buffer to check
* @features: net device features
*
* Returns true if either:
* 1. skb has frag_list and the device doesn't support FRAGLIST, or
* 2. skb is fragmented and the device does not support SG.
*/
static inline bool skb_needs_linearize(struct sk_buff *skb,
netdev_features_t features)
{
return skb_is_nonlinear(skb) &&
((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
(skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
}
static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
void *to,
const unsigned int len)
{
memcpy(to, skb->data, len);
}
static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
const int offset, void *to,
const unsigned int len)
{
memcpy(to, skb->data + offset, len);
}
static inline void skb_copy_to_linear_data(struct sk_buff *skb,
const void *from,
const unsigned int len)
{
memcpy(skb->data, from, len);
}
static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
const int offset,
const void *from,
const unsigned int len)
{
memcpy(skb->data + offset, from, len);
}
void skb_init(void);
static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
{
return skb->tstamp;
}
/**
* skb_get_timestamp - get timestamp from a skb
* @skb: skb to get stamp from
* @stamp: pointer to struct __kernel_old_timeval to store stamp in
*
* Timestamps are stored in the skb as offsets to a base timestamp.
* This function converts the offset back to a struct timeval and stores
* it in stamp.
*/
static inline void skb_get_timestamp(const struct sk_buff *skb,
struct __kernel_old_timeval *stamp)
{
*stamp = ns_to_kernel_old_timeval(skb->tstamp);
}
static inline void skb_get_new_timestamp(const struct sk_buff *skb,
struct __kernel_sock_timeval *stamp)
{
struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
stamp->tv_sec = ts.tv_sec;
stamp->tv_usec = ts.tv_nsec / 1000;
}
static inline void skb_get_timestampns(const struct sk_buff *skb,
struct __kernel_old_timespec *stamp)
{
struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
stamp->tv_sec = ts.tv_sec;
stamp->tv_nsec = ts.tv_nsec;
}
static inline void skb_get_new_timestampns(const struct sk_buff *skb,
struct __kernel_timespec *stamp)
{
struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
stamp->tv_sec = ts.tv_sec;
stamp->tv_nsec = ts.tv_nsec;
}
static inline void __net_timestamp(struct sk_buff *skb)
{
skb->tstamp = ktime_get_real();
skb->mono_delivery_time = 0;
}
static inline ktime_t net_timedelta(ktime_t t)
{
return ktime_sub(ktime_get_real(), t);
}
static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt,
bool mono)
{
skb->tstamp = kt;
skb->mono_delivery_time = kt && mono;
}
DECLARE_STATIC_KEY_FALSE(netstamp_needed_key);
/* It is used in the ingress path to clear the delivery_time.
* If needed, set the skb->tstamp to the (rcv) timestamp.
*/
static inline void skb_clear_delivery_time(struct sk_buff *skb)
{
if (skb->mono_delivery_time) {
skb->mono_delivery_time = 0;
if (static_branch_unlikely(&netstamp_needed_key))
skb->tstamp = ktime_get_real();
else
skb->tstamp = 0;
}
}
static inline void skb_clear_tstamp(struct sk_buff *skb)
{
if (skb->mono_delivery_time)
return;
skb->tstamp = 0;
}
static inline ktime_t skb_tstamp(const struct sk_buff *skb)
{
if (skb->mono_delivery_time)
return 0;
return skb->tstamp;
}
static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond)
{
if (!skb->mono_delivery_time && skb->tstamp)
return skb->tstamp;
if (static_branch_unlikely(&netstamp_needed_key) || cond)
return ktime_get_real();
return 0;
}
static inline u8 skb_metadata_len(const struct sk_buff *skb)
{
return skb_shinfo(skb)->meta_len;
}
static inline void *skb_metadata_end(const struct sk_buff *skb)
{
return skb_mac_header(skb);
}
static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
const struct sk_buff *skb_b,
u8 meta_len)
{
const void *a = skb_metadata_end(skb_a);
const void *b = skb_metadata_end(skb_b);
u64 diffs = 0;
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
BITS_PER_LONG != 64)
goto slow;
/* Using more efficient variant than plain call to memcmp(). */
switch (meta_len) {
#define __it(x, op) (x -= sizeof(u##op))
#define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
case 32: diffs |= __it_diff(a, b, 64);
fallthrough;
case 24: diffs |= __it_diff(a, b, 64);
fallthrough;
case 16: diffs |= __it_diff(a, b, 64);
fallthrough;
case 8: diffs |= __it_diff(a, b, 64);
break;
case 28: diffs |= __it_diff(a, b, 64);
fallthrough;
case 20: diffs |= __it_diff(a, b, 64);
fallthrough;
case 12: diffs |= __it_diff(a, b, 64);
fallthrough;
case 4: diffs |= __it_diff(a, b, 32);
break;
default:
slow:
return memcmp(a - meta_len, b - meta_len, meta_len);
}
return diffs;
}
static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
const struct sk_buff *skb_b)
{
u8 len_a = skb_metadata_len(skb_a);
u8 len_b = skb_metadata_len(skb_b);
if (!(len_a | len_b))
return false;
return len_a != len_b ?
true : __skb_metadata_differs(skb_a, skb_b, len_a);
}
static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
{
skb_shinfo(skb)->meta_len = meta_len;
}
static inline void skb_metadata_clear(struct sk_buff *skb)
{
skb_metadata_set(skb, 0);
}
struct sk_buff *skb_clone_sk(struct sk_buff *skb);
#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
void skb_clone_tx_timestamp(struct sk_buff *skb);
bool skb_defer_rx_timestamp(struct sk_buff *skb);
#else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
{
}
static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
{
return false;
}
#endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
/**
* skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
*
* PHY drivers may accept clones of transmitted packets for
* timestamping via their phy_driver.txtstamp method. These drivers
* must call this function to return the skb back to the stack with a
* timestamp.
*
* @skb: clone of the original outgoing packet
* @hwtstamps: hardware time stamps
*
*/
void skb_complete_tx_timestamp(struct sk_buff *skb,
struct skb_shared_hwtstamps *hwtstamps);
void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb,
struct skb_shared_hwtstamps *hwtstamps,
struct sock *sk, int tstype);
/**
* skb_tstamp_tx - queue clone of skb with send time stamps
* @orig_skb: the original outgoing packet
* @hwtstamps: hardware time stamps, may be NULL if not available
*
* If the skb has a socket associated, then this function clones the
* skb (thus sharing the actual data and optional structures), stores
* the optional hardware time stamping information (if non NULL) or
* generates a software time stamp (otherwise), then queues the clone
* to the error queue of the socket. Errors are silently ignored.
*/
void skb_tstamp_tx(struct sk_buff *orig_skb,
struct skb_shared_hwtstamps *hwtstamps);
/**
* skb_tx_timestamp() - Driver hook for transmit timestamping
*
* Ethernet MAC Drivers should call this function in their hard_xmit()
* function immediately before giving the sk_buff to the MAC hardware.
*
* Specifically, one should make absolutely sure that this function is
* called before TX completion of this packet can trigger. Otherwise
* the packet could potentially already be freed.
*
* @skb: A socket buffer.
*/
static inline void skb_tx_timestamp(struct sk_buff *skb)
{
skb_clone_tx_timestamp(skb);
if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
skb_tstamp_tx(skb, NULL);
}
/**
* skb_complete_wifi_ack - deliver skb with wifi status
*
* @skb: the original outgoing packet
* @acked: ack status
*
*/
void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
__sum16 __skb_checksum_complete(struct sk_buff *skb);
static inline int skb_csum_unnecessary(const struct sk_buff *skb)
{
return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
skb->csum_valid ||
(skb->ip_summed == CHECKSUM_PARTIAL &&
skb_checksum_start_offset(skb) >= 0));
}
/**
* skb_checksum_complete - Calculate checksum of an entire packet
* @skb: packet to process
*
* This function calculates the checksum over the entire packet plus
* the value of skb->csum. The latter can be used to supply the
* checksum of a pseudo header as used by TCP/UDP. It returns the
* checksum.
*
* For protocols that contain complete checksums such as ICMP/TCP/UDP,
* this function can be used to verify that checksum on received
* packets. In that case the function should return zero if the
* checksum is correct. In particular, this function will return zero
* if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
* hardware has already verified the correctness of the checksum.
*/
static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
{
return skb_csum_unnecessary(skb) ?
0 : __skb_checksum_complete(skb);
}
static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
{
if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
if (skb->csum_level == 0)
skb->ip_summed = CHECKSUM_NONE;
else
skb->csum_level--;
}
}
static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
{
if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
skb->csum_level++;
} else if (skb->ip_summed == CHECKSUM_NONE) {
skb->ip_summed = CHECKSUM_UNNECESSARY;
skb->csum_level = 0;
}
}
static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb)
{
if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
skb->ip_summed = CHECKSUM_NONE;
skb->csum_level = 0;
}
}
/* Check if we need to perform checksum complete validation.
*
* Returns true if checksum complete is needed, false otherwise
* (either checksum is unnecessary or zero checksum is allowed).
*/
static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
bool zero_okay,
__sum16 check)
{
if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
skb->csum_valid = 1;
__skb_decr_checksum_unnecessary(skb);
return false;
}
return true;
}
/* For small packets <= CHECKSUM_BREAK perform checksum complete directly
* in checksum_init.
*/
#define CHECKSUM_BREAK 76
/* Unset checksum-complete
*
* Unset checksum complete can be done when packet is being modified
* (uncompressed for instance) and checksum-complete value is
* invalidated.
*/
static inline void skb_checksum_complete_unset(struct sk_buff *skb)
{
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->ip_summed = CHECKSUM_NONE;
}
/* Validate (init) checksum based on checksum complete.
*
* Return values:
* 0: checksum is validated or try to in skb_checksum_complete. In the latter
* case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
* checksum is stored in skb->csum for use in __skb_checksum_complete
* non-zero: value of invalid checksum
*
*/
static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
bool complete,
__wsum psum)
{
if (skb->ip_summed == CHECKSUM_COMPLETE) {
if (!csum_fold(csum_add(psum, skb->csum))) {
skb->csum_valid = 1;
return 0;
}
}
skb->csum = psum;
if (complete || skb->len <= CHECKSUM_BREAK) {
__sum16 csum;
csum = __skb_checksum_complete(skb);
skb->csum_valid = !csum;
return csum;
}
return 0;
}
static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
{
return 0;
}
/* Perform checksum validate (init). Note that this is a macro since we only
* want to calculate the pseudo header which is an input function if necessary.
* First we try to validate without any computation (checksum unnecessary) and
* then calculate based on checksum complete calling the function to compute
* pseudo header.
*
* Return values:
* 0: checksum is validated or try to in skb_checksum_complete
* non-zero: value of invalid checksum
*/
#define __skb_checksum_validate(skb, proto, complete, \
zero_okay, check, compute_pseudo) \
({ \
__sum16 __ret = 0; \
skb->csum_valid = 0; \
if (__skb_checksum_validate_needed(skb, zero_okay, check)) \
__ret = __skb_checksum_validate_complete(skb, \
complete, compute_pseudo(skb, proto)); \
__ret; \
})
#define skb_checksum_init(skb, proto, compute_pseudo) \
__skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
#define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \
__skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
#define skb_checksum_validate(skb, proto, compute_pseudo) \
__skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
#define skb_checksum_validate_zero_check(skb, proto, check, \
compute_pseudo) \
__skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
#define skb_checksum_simple_validate(skb) \
__skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
{
return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
}
static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
{
skb->csum = ~pseudo;
skb->ip_summed = CHECKSUM_COMPLETE;
}
#define skb_checksum_try_convert(skb, proto, compute_pseudo) \
do { \
if (__skb_checksum_convert_check(skb)) \
__skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
} while (0)
static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
u16 start, u16 offset)
{
skb->ip_summed = CHECKSUM_PARTIAL;
skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
skb->csum_offset = offset - start;
}
/* Update skbuf and packet to reflect the remote checksum offload operation.
* When called, ptr indicates the starting point for skb->csum when
* ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
* here, skb_postpull_rcsum is done so skb->csum start is ptr.
*/
static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
int start, int offset, bool nopartial)
{
__wsum delta;
if (!nopartial) {
skb_remcsum_adjust_partial(skb, ptr, start, offset);
return;
}
if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
__skb_checksum_complete(skb);
skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
}
delta = remcsum_adjust(ptr, skb->csum, start, offset);
/* Adjust skb->csum since we changed the packet */
skb->csum = csum_add(skb->csum, delta);
}
static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
return (void *)(skb->_nfct & NFCT_PTRMASK);
#else
return NULL;
#endif
}
static inline unsigned long skb_get_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
return skb->_nfct;
#else
return 0UL;
#endif
}
static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
skb->slow_gro |= !!nfct;
skb->_nfct = nfct;
#endif
}
#ifdef CONFIG_SKB_EXTENSIONS
enum skb_ext_id {
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
SKB_EXT_BRIDGE_NF,
#endif
#ifdef CONFIG_XFRM
SKB_EXT_SEC_PATH,
#endif
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
TC_SKB_EXT,
#endif
#if IS_ENABLED(CONFIG_MPTCP)
SKB_EXT_MPTCP,
#endif
#if IS_ENABLED(CONFIG_MCTP_FLOWS)
SKB_EXT_MCTP,
#endif
SKB_EXT_NUM, /* must be last */
};
/**
* struct skb_ext - sk_buff extensions
* @refcnt: 1 on allocation, deallocated on 0
* @offset: offset to add to @data to obtain extension address
* @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
* @data: start of extension data, variable sized
*
* Note: offsets/lengths are stored in chunks of 8 bytes, this allows
* to use 'u8' types while allowing up to 2kb worth of extension data.
*/
struct skb_ext {
refcount_t refcnt;
u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
u8 chunks; /* same */
char data[] __aligned(8);
};
struct skb_ext *__skb_ext_alloc(gfp_t flags);
void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
struct skb_ext *ext);
void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
void __skb_ext_put(struct skb_ext *ext);
static inline void skb_ext_put(struct sk_buff *skb)
{
if (skb->active_extensions)
__skb_ext_put(skb->extensions);
}
static inline void __skb_ext_copy(struct sk_buff *dst,
const struct sk_buff *src)
{
dst->active_extensions = src->active_extensions;
if (src->active_extensions) {
struct skb_ext *ext = src->extensions; refcount_inc(&ext->refcnt); dst->extensions = ext;
}
}
static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
{
skb_ext_put(dst);
__skb_ext_copy(dst, src);
}
static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
{
return !!ext->offset[i];
}
static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
{
return skb->active_extensions & (1 << id);
}
static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
{
if (skb_ext_exist(skb, id))
__skb_ext_del(skb, id);
}
static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
{
if (skb_ext_exist(skb, id)) {
struct skb_ext *ext = skb->extensions;
return (void *)ext + (ext->offset[id] << 3);
}
return NULL;
}
static inline void skb_ext_reset(struct sk_buff *skb)
{
if (unlikely(skb->active_extensions)) {
__skb_ext_put(skb->extensions);
skb->active_extensions = 0;
}
}
static inline bool skb_has_extensions(struct sk_buff *skb)
{
return unlikely(skb->active_extensions);
}
#else
static inline void skb_ext_put(struct sk_buff *skb) {}
static inline void skb_ext_reset(struct sk_buff *skb) {}
static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
static inline bool skb_has_extensions(struct sk_buff *skb) { return false; }
#endif /* CONFIG_SKB_EXTENSIONS */
static inline void nf_reset_ct(struct sk_buff *skb)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
nf_conntrack_put(skb_nfct(skb));
skb->_nfct = 0;
#endif
}
static inline void nf_reset_trace(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
skb->nf_trace = 0;
#endif
}
static inline void ipvs_reset(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IP_VS)
skb->ipvs_property = 0;
#endif
}
/* Note: This doesn't put any conntrack info in dst. */
static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
bool copy)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
dst->_nfct = src->_nfct; nf_conntrack_get(skb_nfct(src));
#endif
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
if (copy)
dst->nf_trace = src->nf_trace;
#endif
}
static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
nf_conntrack_put(skb_nfct(dst));
#endif
dst->slow_gro = src->slow_gro;
__nf_copy(dst, src, true);
}
#ifdef CONFIG_NETWORK_SECMARK
static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
{
to->secmark = from->secmark;
}
static inline void skb_init_secmark(struct sk_buff *skb)
{
skb->secmark = 0;
}
#else
static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
{ }
static inline void skb_init_secmark(struct sk_buff *skb)
{ }
#endif
static inline int secpath_exists(const struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
return skb_ext_exist(skb, SKB_EXT_SEC_PATH);
#else
return 0;
#endif
}
static inline bool skb_irq_freeable(const struct sk_buff *skb)
{
return !skb->destructor &&
!secpath_exists(skb) &&
!skb_nfct(skb) &&
!skb->_skb_refdst &&
!skb_has_frag_list(skb);
}
static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
{
skb->queue_mapping = queue_mapping;
}
static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
{
return skb->queue_mapping;
}
static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
{
to->queue_mapping = from->queue_mapping;
}
static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
{
skb->queue_mapping = rx_queue + 1;
}
static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
{
return skb->queue_mapping - 1;
}
static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
{
return skb->queue_mapping != 0;
}
static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
{
skb->dst_pending_confirm = val;
}
static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
{
return skb->dst_pending_confirm != 0;
}
static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
return skb_ext_find(skb, SKB_EXT_SEC_PATH);
#else
return NULL;
#endif
}
static inline bool skb_is_gso(const struct sk_buff *skb)
{
return skb_shinfo(skb)->gso_size;
}
/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_v6(const struct sk_buff *skb)
{
return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
}
/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
{
return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
}
/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
{
return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
}
static inline void skb_gso_reset(struct sk_buff *skb)
{
skb_shinfo(skb)->gso_size = 0;
skb_shinfo(skb)->gso_segs = 0;
skb_shinfo(skb)->gso_type = 0;
}
static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
u16 increment)
{
if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
return;
shinfo->gso_size += increment;
}
static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
u16 decrement)
{
if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
return;
shinfo->gso_size -= decrement;
}
void __skb_warn_lro_forwarding(const struct sk_buff *skb);
static inline bool skb_warn_if_lro(const struct sk_buff *skb)
{
/* LRO sets gso_size but not gso_type, whereas if GSO is really
* wanted then gso_type will be set. */
const struct skb_shared_info *shinfo = skb_shinfo(skb);
if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
unlikely(shinfo->gso_type == 0)) {
__skb_warn_lro_forwarding(skb);
return true;
}
return false;
}
static inline void skb_forward_csum(struct sk_buff *skb)
{
/* Unfortunately we don't support this one. Any brave souls? */
if (skb->ip_summed == CHECKSUM_COMPLETE)
skb->ip_summed = CHECKSUM_NONE;
}
/**
* skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
* @skb: skb to check
*
* fresh skbs have their ip_summed set to CHECKSUM_NONE.
* Instead of forcing ip_summed to CHECKSUM_NONE, we can
* use this helper, to document places where we make this assertion.
*/
static inline void skb_checksum_none_assert(const struct sk_buff *skb)
{
DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE);
}
bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
unsigned int transport_len,
__sum16(*skb_chkf)(struct sk_buff *skb));
/**
* skb_head_is_locked - Determine if the skb->head is locked down
* @skb: skb to check
*
* The head on skbs build around a head frag can be removed if they are
* not cloned. This function returns true if the skb head is locked down
* due to either being allocated via kmalloc, or by being a clone with
* multiple references to the head.
*/
static inline bool skb_head_is_locked(const struct sk_buff *skb)
{
return !skb->head_frag || skb_cloned(skb);
}
/* Local Checksum Offload.
* Compute outer checksum based on the assumption that the
* inner checksum will be offloaded later.
* See Documentation/networking/checksum-offloads.rst for
* explanation of how this works.
* Fill in outer checksum adjustment (e.g. with sum of outer
* pseudo-header) before calling.
* Also ensure that inner checksum is in linear data area.
*/
static inline __wsum lco_csum(struct sk_buff *skb)
{
unsigned char *csum_start = skb_checksum_start(skb);
unsigned char *l4_hdr = skb_transport_header(skb);
__wsum partial;
/* Start with complement of inner checksum adjustment */
partial = ~csum_unfold(*(__force __sum16 *)(csum_start +
skb->csum_offset));
/* Add in checksum of our headers (incl. outer checksum
* adjustment filled in by caller) and return result.
*/
return csum_partial(l4_hdr, csum_start - l4_hdr, partial);
}
static inline bool skb_is_redirected(const struct sk_buff *skb)
{
return skb->redirected;
}
static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)
{
skb->redirected = 1;
#ifdef CONFIG_NET_REDIRECT
skb->from_ingress = from_ingress;
if (skb->from_ingress)
skb_clear_tstamp(skb);
#endif
}
static inline void skb_reset_redirect(struct sk_buff *skb)
{
skb->redirected = 0;
}
static inline void skb_set_redirected_noclear(struct sk_buff *skb,
bool from_ingress)
{
skb->redirected = 1;
#ifdef CONFIG_NET_REDIRECT
skb->from_ingress = from_ingress;
#endif
}
static inline bool skb_csum_is_sctp(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IP_SCTP)
return skb->csum_not_inet;
#else
return 0;
#endif
}
static inline void skb_reset_csum_not_inet(struct sk_buff *skb)
{
skb->ip_summed = CHECKSUM_NONE;
#if IS_ENABLED(CONFIG_IP_SCTP)
skb->csum_not_inet = 0;
#endif
}
static inline void skb_set_kcov_handle(struct sk_buff *skb,
const u64 kcov_handle)
{
#ifdef CONFIG_KCOV
skb->kcov_handle = kcov_handle;
#endif
}
static inline u64 skb_get_kcov_handle(struct sk_buff *skb)
{
#ifdef CONFIG_KCOV
return skb->kcov_handle;
#else
return 0;
#endif
}
static inline void skb_mark_for_recycle(struct sk_buff *skb)
{
#ifdef CONFIG_PAGE_POOL
skb->pp_recycle = 1;
#endif
}
ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
ssize_t maxsize, gfp_t gfp);
#endif /* __KERNEL__ */
#endif /* _LINUX_SKBUFF_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H
/*
* Define 'struct task_struct' and provide the main scheduler
* APIs (schedule(), wakeup variants, etc.)
*/
#include <uapi/linux/sched.h>
#include <asm/current.h>
#include <asm/processor.h>
#include <linux/thread_info.h>
#include <linux/preempt.h>
#include <linux/cpumask.h>
#include <linux/cache.h>
#include <linux/irqflags_types.h>
#include <linux/smp_types.h>
#include <linux/pid_types.h>
#include <linux/sem_types.h>
#include <linux/shm.h>
#include <linux/kmsan_types.h>
#include <linux/mutex_types.h>
#include <linux/plist_types.h>
#include <linux/hrtimer_types.h>
#include <linux/timer_types.h>
#include <linux/seccomp_types.h>
#include <linux/nodemask_types.h>
#include <linux/refcount_types.h>
#include <linux/resource.h>
#include <linux/latencytop.h>
#include <linux/sched/prio.h>
#include <linux/sched/types.h>
#include <linux/signal_types.h>
#include <linux/syscall_user_dispatch_types.h>
#include <linux/mm_types_task.h>
#include <linux/task_io_accounting.h>
#include <linux/posix-timers_types.h>
#include <linux/restart_block.h>
#include <uapi/linux/rseq.h>
#include <linux/seqlock_types.h>
#include <linux/kcsan.h>
#include <linux/rv.h>
#include <linux/livepatch_sched.h>
#include <linux/uidgid_types.h>
#include <asm/kmap_size.h>
/* task_struct member predeclarations (sorted alphabetically): */
struct audit_context;
struct bio_list;
struct blk_plug;
struct bpf_local_storage;
struct bpf_run_ctx;
struct capture_control;
struct cfs_rq;
struct fs_struct;
struct futex_pi_state;
struct io_context;
struct io_uring_task;
struct mempolicy;
struct nameidata;
struct nsproxy;
struct perf_event_context;
struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
struct robust_list_head;
struct root_domain;
struct rq;
struct sched_attr;
struct sched_dl_entity;
struct seq_file;
struct sighand_struct;
struct signal_struct;
struct task_delay_info;
struct task_group;
struct task_struct;
struct user_event_mm;
/*
* Task state bitmask. NOTE! These bits are also
* encoded in fs/proc/array.c: get_task_state().
*
* We have two separate sets of flags: task->__state
* is about runnability, while task->exit_state are
* about the task exiting. Confusing, but this way
* modifying one set can't modify the other one by
* mistake.
*/
/* Used in tsk->__state: */
#define TASK_RUNNING 0x00000000
#define TASK_INTERRUPTIBLE 0x00000001
#define TASK_UNINTERRUPTIBLE 0x00000002
#define __TASK_STOPPED 0x00000004
#define __TASK_TRACED 0x00000008
/* Used in tsk->exit_state: */
#define EXIT_DEAD 0x00000010
#define EXIT_ZOMBIE 0x00000020
#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
/* Used in tsk->__state again: */
#define TASK_PARKED 0x00000040
#define TASK_DEAD 0x00000080
#define TASK_WAKEKILL 0x00000100
#define TASK_WAKING 0x00000200
#define TASK_NOLOAD 0x00000400
#define TASK_NEW 0x00000800
#define TASK_RTLOCK_WAIT 0x00001000
#define TASK_FREEZABLE 0x00002000
#define __TASK_FREEZABLE_UNSAFE (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP))
#define TASK_FROZEN 0x00008000
#define TASK_STATE_MAX 0x00010000
#define TASK_ANY (TASK_STATE_MAX-1)
/*
* DO NOT ADD ANY NEW USERS !
*/
#define TASK_FREEZABLE_UNSAFE (TASK_FREEZABLE | __TASK_FREEZABLE_UNSAFE)
/* Convenience macros for the sake of set_current_state: */
#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED __TASK_TRACED
#define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
/* Convenience macros for the sake of wake_up(): */
#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
/* get_task_state(): */
#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
__TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
TASK_PARKED)
#define task_is_running(task) (READ_ONCE((task)->__state) == TASK_RUNNING)
#define task_is_traced(task) ((READ_ONCE(task->jobctl) & JOBCTL_TRACED) != 0)
#define task_is_stopped(task) ((READ_ONCE(task->jobctl) & JOBCTL_STOPPED) != 0)
#define task_is_stopped_or_traced(task) ((READ_ONCE(task->jobctl) & (JOBCTL_STOPPED | JOBCTL_TRACED)) != 0)
/*
* Special states are those that do not use the normal wait-loop pattern. See
* the comment with set_special_state().
*/
#define is_special_task_state(state) \
((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
# define debug_normal_state_change(state_value) \
do { \
WARN_ON_ONCE(is_special_task_state(state_value)); \
current->task_state_change = _THIS_IP_; \
} while (0)
# define debug_special_state_change(state_value) \
do { \
WARN_ON_ONCE(!is_special_task_state(state_value)); \
current->task_state_change = _THIS_IP_; \
} while (0)
# define debug_rtlock_wait_set_state() \
do { \
current->saved_state_change = current->task_state_change;\
current->task_state_change = _THIS_IP_; \
} while (0)
# define debug_rtlock_wait_restore_state() \
do { \
current->task_state_change = current->saved_state_change;\
} while (0)
#else
# define debug_normal_state_change(cond) do { } while (0)
# define debug_special_state_change(cond) do { } while (0)
# define debug_rtlock_wait_set_state() do { } while (0)
# define debug_rtlock_wait_restore_state() do { } while (0)
#endif
/*
* set_current_state() includes a barrier so that the write of current->__state
* is correctly serialised wrt the caller's subsequent test of whether to
* actually sleep:
*
* for (;;) {
* set_current_state(TASK_UNINTERRUPTIBLE);
* if (CONDITION)
* break;
*
* schedule();
* }
* __set_current_state(TASK_RUNNING);
*
* If the caller does not need such serialisation (because, for instance, the
* CONDITION test and condition change and wakeup are under the same lock) then
* use __set_current_state().
*
* The above is typically ordered against the wakeup, which does:
*
* CONDITION = 1;
* wake_up_state(p, TASK_UNINTERRUPTIBLE);
*
* where wake_up_state()/try_to_wake_up() executes a full memory barrier before
* accessing p->__state.
*
* Wakeup will do: if (@state & p->__state) p->__state = TASK_RUNNING, that is,
* once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
* TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
*
* However, with slightly different timing the wakeup TASK_RUNNING store can
* also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
* a problem either because that will result in one extra go around the loop
* and our @cond test will save the day.
*
* Also see the comments of try_to_wake_up().
*/
#define __set_current_state(state_value) \
do { \
debug_normal_state_change((state_value)); \
WRITE_ONCE(current->__state, (state_value)); \
} while (0)
#define set_current_state(state_value) \
do { \
debug_normal_state_change((state_value)); \
smp_store_mb(current->__state, (state_value)); \
} while (0)
/*
* set_special_state() should be used for those states when the blocking task
* can not use the regular condition based wait-loop. In that case we must
* serialize against wakeups such that any possible in-flight TASK_RUNNING
* stores will not collide with our state change.
*/
#define set_special_state(state_value) \
do { \
unsigned long flags; /* may shadow */ \
\
raw_spin_lock_irqsave(¤t->pi_lock, flags); \
debug_special_state_change((state_value)); \
WRITE_ONCE(current->__state, (state_value)); \
raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \
} while (0)
/*
* PREEMPT_RT specific variants for "sleeping" spin/rwlocks
*
* RT's spin/rwlock substitutions are state preserving. The state of the
* task when blocking on the lock is saved in task_struct::saved_state and
* restored after the lock has been acquired. These operations are
* serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT
* lock related wakeups while the task is blocked on the lock are
* redirected to operate on task_struct::saved_state to ensure that these
* are not dropped. On restore task_struct::saved_state is set to
* TASK_RUNNING so any wakeup attempt redirected to saved_state will fail.
*
* The lock operation looks like this:
*
* current_save_and_set_rtlock_wait_state();
* for (;;) {
* if (try_lock())
* break;
* raw_spin_unlock_irq(&lock->wait_lock);
* schedule_rtlock();
* raw_spin_lock_irq(&lock->wait_lock);
* set_current_state(TASK_RTLOCK_WAIT);
* }
* current_restore_rtlock_saved_state();
*/
#define current_save_and_set_rtlock_wait_state() \
do { \
lockdep_assert_irqs_disabled(); \
raw_spin_lock(¤t->pi_lock); \
current->saved_state = current->__state; \
debug_rtlock_wait_set_state(); \
WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT); \
raw_spin_unlock(¤t->pi_lock); \
} while (0);
#define current_restore_rtlock_saved_state() \
do { \
lockdep_assert_irqs_disabled(); \
raw_spin_lock(¤t->pi_lock); \
debug_rtlock_wait_restore_state(); \
WRITE_ONCE(current->__state, current->saved_state); \
current->saved_state = TASK_RUNNING; \
raw_spin_unlock(¤t->pi_lock); \
} while (0);
#define get_current_state() READ_ONCE(current->__state)
/*
* Define the task command name length as enum, then it can be visible to
* BPF programs.
*/
enum {
TASK_COMM_LEN = 16,
};
extern void scheduler_tick(void);
#define MAX_SCHEDULE_TIMEOUT LONG_MAX
extern long schedule_timeout(long timeout);
extern long schedule_timeout_interruptible(long timeout);
extern long schedule_timeout_killable(long timeout);
extern long schedule_timeout_uninterruptible(long timeout);
extern long schedule_timeout_idle(long timeout);
asmlinkage void schedule(void);
extern void schedule_preempt_disabled(void);
asmlinkage void preempt_schedule_irq(void);
#ifdef CONFIG_PREEMPT_RT
extern void schedule_rtlock(void);
#endif
extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
extern long io_schedule_timeout(long timeout);
extern void io_schedule(void);
/**
* struct prev_cputime - snapshot of system and user cputime
* @utime: time spent in user mode
* @stime: time spent in system mode
* @lock: protects the above two fields
*
* Stores previous user/system time values such that we can guarantee
* monotonicity.
*/
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
u64 utime;
u64 stime;
raw_spinlock_t lock;
#endif
};
enum vtime_state {
/* Task is sleeping or running in a CPU with VTIME inactive: */
VTIME_INACTIVE = 0,
/* Task is idle */
VTIME_IDLE,
/* Task runs in kernelspace in a CPU with VTIME active: */
VTIME_SYS,
/* Task runs in userspace in a CPU with VTIME active: */
VTIME_USER,
/* Task runs as guests in a CPU with VTIME active: */
VTIME_GUEST,
};
struct vtime {
seqcount_t seqcount;
unsigned long long starttime;
enum vtime_state state;
unsigned int cpu;
u64 utime;
u64 stime;
u64 gtime;
};
/*
* Utilization clamp constraints.
* @UCLAMP_MIN: Minimum utilization
* @UCLAMP_MAX: Maximum utilization
* @UCLAMP_CNT: Utilization clamp constraints count
*/
enum uclamp_id {
UCLAMP_MIN = 0,
UCLAMP_MAX,
UCLAMP_CNT
};
#ifdef CONFIG_SMP
extern struct root_domain def_root_domain;
extern struct mutex sched_domains_mutex;
#endif
struct sched_param {
int sched_priority;
};
struct sched_info {
#ifdef CONFIG_SCHED_INFO
/* Cumulative counters: */
/* # of times we have run on this CPU: */
unsigned long pcount;
/* Time spent waiting on a runqueue: */
unsigned long long run_delay;
/* Timestamps: */
/* When did we last run on a CPU? */
unsigned long long last_arrival;
/* When were we last queued to run? */
unsigned long long last_queued;
#endif /* CONFIG_SCHED_INFO */
};
/*
* Integer metrics need fixed point arithmetic, e.g., sched/fair
* has a few: load, load_avg, util_avg, freq, and capacity.
*
* We define a basic fixed point arithmetic range, and then formalize
* all these metrics based on that basic range.
*/
# define SCHED_FIXEDPOINT_SHIFT 10
# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
/* Increase resolution of cpu_capacity calculations */
# define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT
# define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT)
struct load_weight {
unsigned long weight;
u32 inv_weight;
};
/*
* The load/runnable/util_avg accumulates an infinite geometric series
* (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
*
* [load_avg definition]
*
* load_avg = runnable% * scale_load_down(load)
*
* [runnable_avg definition]
*
* runnable_avg = runnable% * SCHED_CAPACITY_SCALE
*
* [util_avg definition]
*
* util_avg = running% * SCHED_CAPACITY_SCALE
*
* where runnable% is the time ratio that a sched_entity is runnable and
* running% the time ratio that a sched_entity is running.
*
* For cfs_rq, they are the aggregated values of all runnable and blocked
* sched_entities.
*
* The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
* capacity scaling. The scaling is done through the rq_clock_pelt that is used
* for computing those signals (see update_rq_clock_pelt())
*
* N.B., the above ratios (runnable% and running%) themselves are in the
* range of [0, 1]. To do fixed point arithmetics, we therefore scale them
* to as large a range as necessary. This is for example reflected by
* util_avg's SCHED_CAPACITY_SCALE.
*
* [Overflow issue]
*
* The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
* with the highest load (=88761), always runnable on a single cfs_rq,
* and should not overflow as the number already hits PID_MAX_LIMIT.
*
* For all other cases (including 32-bit kernels), struct load_weight's
* weight will overflow first before we do, because:
*
* Max(load_avg) <= Max(load.weight)
*
* Then it is the load_weight's responsibility to consider overflow
* issues.
*/
struct sched_avg {
u64 last_update_time;
u64 load_sum;
u64 runnable_sum;
u32 util_sum;
u32 period_contrib;
unsigned long load_avg;
unsigned long runnable_avg;
unsigned long util_avg;
unsigned int util_est;
} ____cacheline_aligned;
/*
* The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
* updates. When a task is dequeued, its util_est should not be updated if its
* util_avg has not been updated in the meantime.
* This information is mapped into the MSB bit of util_est at dequeue time.
* Since max value of util_est for a task is 1024 (PELT util_avg for a task)
* it is safe to use MSB.
*/
#define UTIL_EST_WEIGHT_SHIFT 2
#define UTIL_AVG_UNCHANGED 0x80000000
struct sched_statistics {
#ifdef CONFIG_SCHEDSTATS
u64 wait_start;
u64 wait_max;
u64 wait_count;
u64 wait_sum;
u64 iowait_count;
u64 iowait_sum;
u64 sleep_start;
u64 sleep_max;
s64 sum_sleep_runtime;
u64 block_start;
u64 block_max;
s64 sum_block_runtime;
s64 exec_max;
u64 slice_max;
u64 nr_migrations_cold;
u64 nr_failed_migrations_affine;
u64 nr_failed_migrations_running;
u64 nr_failed_migrations_hot;
u64 nr_forced_migrations;
u64 nr_wakeups;
u64 nr_wakeups_sync;
u64 nr_wakeups_migrate;
u64 nr_wakeups_local;
u64 nr_wakeups_remote;
u64 nr_wakeups_affine;
u64 nr_wakeups_affine_attempts;
u64 nr_wakeups_passive;
u64 nr_wakeups_idle;
#ifdef CONFIG_SCHED_CORE
u64 core_forceidle_sum;
#endif
#endif /* CONFIG_SCHEDSTATS */
} ____cacheline_aligned;
struct sched_entity {
/* For load-balancing: */
struct load_weight load;
struct rb_node run_node;
u64 deadline;
u64 min_vruntime;
struct list_head group_node;
unsigned int on_rq;
u64 exec_start;
u64 sum_exec_runtime;
u64 prev_sum_exec_runtime;
u64 vruntime;
s64 vlag;
u64 slice;
u64 nr_migrations;
#ifdef CONFIG_FAIR_GROUP_SCHED
int depth;
struct sched_entity *parent;
/* rq on which this entity is (to be) queued: */
struct cfs_rq *cfs_rq;
/* rq "owned" by this entity/group: */
struct cfs_rq *my_q;
/* cached value of my_q->h_nr_running */
unsigned long runnable_weight;
#endif
#ifdef CONFIG_SMP
/*
* Per entity load average tracking.
*
* Put into separate cache line so it does not
* collide with read-mostly values above.
*/
struct sched_avg avg;
#endif
};
struct sched_rt_entity {
struct list_head run_list;
unsigned long timeout;
unsigned long watchdog_stamp;
unsigned int time_slice;
unsigned short on_rq;
unsigned short on_list;
struct sched_rt_entity *back;
#ifdef CONFIG_RT_GROUP_SCHED
struct sched_rt_entity *parent;
/* rq on which this entity is (to be) queued: */
struct rt_rq *rt_rq;
/* rq "owned" by this entity/group: */
struct rt_rq *my_q;
#endif
} __randomize_layout;
typedef bool (*dl_server_has_tasks_f)(struct sched_dl_entity *);
typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *);
struct sched_dl_entity {
struct rb_node rb_node;
/*
* Original scheduling parameters. Copied here from sched_attr
* during sched_setattr(), they will remain the same until
* the next sched_setattr().
*/
u64 dl_runtime; /* Maximum runtime for each instance */
u64 dl_deadline; /* Relative deadline of each instance */
u64 dl_period; /* Separation of two instances (period) */
u64 dl_bw; /* dl_runtime / dl_period */
u64 dl_density; /* dl_runtime / dl_deadline */
/*
* Actual scheduling parameters. Initialized with the values above,
* they are continuously updated during task execution. Note that
* the remaining runtime could be < 0 in case we are in overrun.
*/
s64 runtime; /* Remaining runtime for this instance */
u64 deadline; /* Absolute deadline for this instance */
unsigned int flags; /* Specifying the scheduler behaviour */
/*
* Some bool flags:
*
* @dl_throttled tells if we exhausted the runtime. If so, the
* task has to wait for a replenishment to be performed at the
* next firing of dl_timer.
*
* @dl_yielded tells if task gave up the CPU before consuming
* all its available runtime during the last job.
*
* @dl_non_contending tells if the task is inactive while still
* contributing to the active utilization. In other words, it
* indicates if the inactive timer has been armed and its handler
* has not been executed yet. This flag is useful to avoid race
* conditions between the inactive timer handler and the wakeup
* code.
*
* @dl_overrun tells if the task asked to be informed about runtime
* overruns.
*/
unsigned int dl_throttled : 1;
unsigned int dl_yielded : 1;
unsigned int dl_non_contending : 1;
unsigned int dl_overrun : 1;
unsigned int dl_server : 1;
/*
* Bandwidth enforcement timer. Each -deadline task has its
* own bandwidth to be enforced, thus we need one timer per task.
*/
struct hrtimer dl_timer;
/*
* Inactive timer, responsible for decreasing the active utilization
* at the "0-lag time". When a -deadline task blocks, it contributes
* to GRUB's active utilization until the "0-lag time", hence a
* timer is needed to decrease the active utilization at the correct
* time.
*/
struct hrtimer inactive_timer;
/*
* Bits for DL-server functionality. Also see the comment near
* dl_server_update().
*
* @rq the runqueue this server is for
*
* @server_has_tasks() returns true if @server_pick return a
* runnable task.
*/
struct rq *rq;
dl_server_has_tasks_f server_has_tasks;
dl_server_pick_f server_pick;
#ifdef CONFIG_RT_MUTEXES
/*
* Priority Inheritance. When a DEADLINE scheduling entity is boosted
* pi_se points to the donor, otherwise points to the dl_se it belongs
* to (the original one/itself).
*/
struct sched_dl_entity *pi_se;
#endif
};
#ifdef CONFIG_UCLAMP_TASK
/* Number of utilization clamp buckets (shorter alias) */
#define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
/*
* Utilization clamp for a scheduling entity
* @value: clamp value "assigned" to a se
* @bucket_id: bucket index corresponding to the "assigned" value
* @active: the se is currently refcounted in a rq's bucket
* @user_defined: the requested clamp value comes from user-space
*
* The bucket_id is the index of the clamp bucket matching the clamp value
* which is pre-computed and stored to avoid expensive integer divisions from
* the fast path.
*
* The active bit is set whenever a task has got an "effective" value assigned,
* which can be different from the clamp value "requested" from user-space.
* This allows to know a task is refcounted in the rq's bucket corresponding
* to the "effective" bucket_id.
*
* The user_defined bit is set whenever a task has got a task-specific clamp
* value requested from userspace, i.e. the system defaults apply to this task
* just as a restriction. This allows to relax default clamps when a less
* restrictive task-specific value has been requested, thus allowing to
* implement a "nice" semantic. For example, a task running with a 20%
* default boost can still drop its own boosting to 0%.
*/
struct uclamp_se {
unsigned int value : bits_per(SCHED_CAPACITY_SCALE);
unsigned int bucket_id : bits_per(UCLAMP_BUCKETS);
unsigned int active : 1;
unsigned int user_defined : 1;
};
#endif /* CONFIG_UCLAMP_TASK */
union rcu_special {
struct {
u8 blocked;
u8 need_qs;
u8 exp_hint; /* Hint for performance. */
u8 need_mb; /* Readers need smp_mb(). */
} b; /* Bits. */
u32 s; /* Set of bits. */
};
enum perf_event_task_context {
perf_invalid_context = -1,
perf_hw_context = 0,
perf_sw_context,
perf_nr_task_contexts,
};
struct wake_q_node {
struct wake_q_node *next;
};
struct kmap_ctrl {
#ifdef CONFIG_KMAP_LOCAL
int idx;
pte_t pteval[KM_MAX_IDX];
#endif
};
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
* For reasons of header soup (see current_thread_info()), this
* must be the first element of task_struct.
*/
struct thread_info thread_info;
#endif
unsigned int __state;
/* saved state for "spinlock sleepers" */
unsigned int saved_state;
/*
* This begins the randomizable portion of task_struct. Only
* scheduling-critical items should be added above here.
*/
randomized_struct_fields_start
void *stack;
refcount_t usage;
/* Per task flags (PF_*), defined further below: */
unsigned int flags;
unsigned int ptrace;
#ifdef CONFIG_SMP
int on_cpu;
struct __call_single_node wake_entry;
unsigned int wakee_flips;
unsigned long wakee_flip_decay_ts;
struct task_struct *last_wakee;
/*
* recent_used_cpu is initially set as the last CPU used by a task
* that wakes affine another task. Waker/wakee relationships can
* push tasks around a CPU where each wakeup moves to the next one.
* Tracking a recently used CPU allows a quick search for a recently
* used CPU that may be idle.
*/
int recent_used_cpu;
int wake_cpu;
#endif
int on_rq;
int prio;
int static_prio;
int normal_prio;
unsigned int rt_priority;
struct sched_entity se;
struct sched_rt_entity rt;
struct sched_dl_entity dl;
struct sched_dl_entity *dl_server;
const struct sched_class *sched_class;
#ifdef CONFIG_SCHED_CORE
struct rb_node core_node;
unsigned long core_cookie;
unsigned int core_occupation;
#endif
#ifdef CONFIG_CGROUP_SCHED
struct task_group *sched_task_group;
#endif
#ifdef CONFIG_UCLAMP_TASK
/*
* Clamp values requested for a scheduling entity.
* Must be updated with task_rq_lock() held.
*/
struct uclamp_se uclamp_req[UCLAMP_CNT];
/*
* Effective clamp values used for a scheduling entity.
* Must be updated with task_rq_lock() held.
*/
struct uclamp_se uclamp[UCLAMP_CNT];
#endif
struct sched_statistics stats;
#ifdef CONFIG_PREEMPT_NOTIFIERS
/* List of struct preempt_notifier: */
struct hlist_head preempt_notifiers;
#endif
#ifdef CONFIG_BLK_DEV_IO_TRACE
unsigned int btrace_seq;
#endif
unsigned int policy;
int nr_cpus_allowed;
const cpumask_t *cpus_ptr;
cpumask_t *user_cpus_ptr;
cpumask_t cpus_mask;
void *migration_pending;
#ifdef CONFIG_SMP
unsigned short migration_disabled;
#endif
unsigned short migration_flags;
#ifdef CONFIG_PREEMPT_RCU
int rcu_read_lock_nesting;
union rcu_special rcu_read_unlock_special;
struct list_head rcu_node_entry;
struct rcu_node *rcu_blocked_node;
#endif /* #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_TASKS_RCU
unsigned long rcu_tasks_nvcsw;
u8 rcu_tasks_holdout;
u8 rcu_tasks_idx;
int rcu_tasks_idle_cpu;
struct list_head rcu_tasks_holdout_list;
int rcu_tasks_exit_cpu;
struct list_head rcu_tasks_exit_list;
#endif /* #ifdef CONFIG_TASKS_RCU */
#ifdef CONFIG_TASKS_TRACE_RCU
int trc_reader_nesting;
int trc_ipi_to_cpu;
union rcu_special trc_reader_special;
struct list_head trc_holdout_list;
struct list_head trc_blkd_node;
int trc_blkd_cpu;
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
struct sched_info sched_info;
struct list_head tasks;
#ifdef CONFIG_SMP
struct plist_node pushable_tasks;
struct rb_node pushable_dl_tasks;
#endif
struct mm_struct *mm;
struct mm_struct *active_mm;
struct address_space *faults_disabled_mapping;
int exit_state;
int exit_code;
int exit_signal;
/* The signal sent when the parent dies: */
int pdeath_signal;
/* JOBCTL_*, siglock protected: */
unsigned long jobctl;
/* Used for emulating ABI behavior of previous Linux versions: */
unsigned int personality;
/* Scheduler bits, serialized by scheduler locks: */
unsigned sched_reset_on_fork:1;
unsigned sched_contributes_to_load:1;
unsigned sched_migrated:1;
/* Force alignment to the next boundary: */
unsigned :0;
/* Unserialized, strictly 'current' */
/*
* This field must not be in the scheduler word above due to wakelist
* queueing no longer being serialized by p->on_cpu. However:
*
* p->XXX = X; ttwu()
* schedule() if (p->on_rq && ..) // false
* smp_mb__after_spinlock(); if (smp_load_acquire(&p->on_cpu) && //true
* deactivate_task() ttwu_queue_wakelist())
* p->on_rq = 0; p->sched_remote_wakeup = Y;
*
* guarantees all stores of 'current' are visible before
* ->sched_remote_wakeup gets used, so it can be in this word.
*/
unsigned sched_remote_wakeup:1;
#ifdef CONFIG_RT_MUTEXES
unsigned sched_rt_mutex:1;
#endif
/* Bit to tell TOMOYO we're in execve(): */
unsigned in_execve:1;
unsigned in_iowait:1;
#ifndef TIF_RESTORE_SIGMASK
unsigned restore_sigmask:1;
#endif
#ifdef CONFIG_MEMCG
unsigned in_user_fault:1;
#endif
#ifdef CONFIG_LRU_GEN
/* whether the LRU algorithm may apply to this access */
unsigned in_lru_fault:1;
#endif
#ifdef CONFIG_COMPAT_BRK
unsigned brk_randomized:1;
#endif
#ifdef CONFIG_CGROUPS
/* disallow userland-initiated cgroup migration */
unsigned no_cgroup_migration:1;
/* task is frozen/stopped (used by the cgroup freezer) */
unsigned frozen:1;
#endif
#ifdef CONFIG_BLK_CGROUP
unsigned use_memdelay:1;
#endif
#ifdef CONFIG_PSI
/* Stalled due to lack of memory */
unsigned in_memstall:1;
#endif
#ifdef CONFIG_PAGE_OWNER
/* Used by page_owner=on to detect recursion in page tracking. */
unsigned in_page_owner:1;
#endif
#ifdef CONFIG_EVENTFD
/* Recursion prevention for eventfd_signal() */
unsigned in_eventfd:1;
#endif
#ifdef CONFIG_ARCH_HAS_CPU_PASID
unsigned pasid_activated:1;
#endif
#ifdef CONFIG_CPU_SUP_INTEL
unsigned reported_split_lock:1;
#endif
#ifdef CONFIG_TASK_DELAY_ACCT
/* delay due to memory thrashing */
unsigned in_thrashing:1;
#endif
unsigned long atomic_flags; /* Flags requiring atomic access. */
struct restart_block restart_block;
pid_t pid;
pid_t tgid;
#ifdef CONFIG_STACKPROTECTOR
/* Canary value for the -fstack-protector GCC feature: */
unsigned long stack_canary;
#endif
/*
* Pointers to the (original) parent process, youngest child, younger sibling,
* older sibling, respectively. (p->father can be replaced with
* p->real_parent->pid)
*/
/* Real parent process: */
struct task_struct __rcu *real_parent;
/* Recipient of SIGCHLD, wait4() reports: */
struct task_struct __rcu *parent;
/*
* Children/sibling form the list of natural children:
*/
struct list_head children;
struct list_head sibling;
struct task_struct *group_leader;
/*
* 'ptraced' is the list of tasks this task is using ptrace() on.
*
* This includes both natural children and PTRACE_ATTACH targets.
* 'ptrace_entry' is this task's link on the p->parent->ptraced list.
*/
struct list_head ptraced;
struct list_head ptrace_entry;
/* PID/PID hash table linkage. */
struct pid *thread_pid;
struct hlist_node pid_links[PIDTYPE_MAX];
struct list_head thread_node;
struct completion *vfork_done;
/* CLONE_CHILD_SETTID: */
int __user *set_child_tid;
/* CLONE_CHILD_CLEARTID: */
int __user *clear_child_tid;
/* PF_KTHREAD | PF_IO_WORKER */
void *worker_private;
u64 utime;
u64 stime;
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
u64 utimescaled;
u64 stimescaled;
#endif
u64 gtime;
struct prev_cputime prev_cputime;
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
struct vtime vtime;
#endif
#ifdef CONFIG_NO_HZ_FULL
atomic_t tick_dep_mask;
#endif
/* Context switch counts: */
unsigned long nvcsw;
unsigned long nivcsw;
/* Monotonic time in nsecs: */
u64 start_time;
/* Boot based time in nsecs: */
u64 start_boottime;
/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
unsigned long min_flt;
unsigned long maj_flt;
/* Empty if CONFIG_POSIX_CPUTIMERS=n */
struct posix_cputimers posix_cputimers;
#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
struct posix_cputimers_work posix_cputimers_work;
#endif
/* Process credentials: */
/* Tracer's credentials at attach: */
const struct cred __rcu *ptracer_cred;
/* Objective and real subjective task credentials (COW): */
const struct cred __rcu *real_cred;
/* Effective (overridable) subjective task credentials (COW): */
const struct cred __rcu *cred;
#ifdef CONFIG_KEYS
/* Cached requested key. */
struct key *cached_requested_key;
#endif
/*
* executable name, excluding path.
*
* - normally initialized setup_new_exec()
* - access it with [gs]et_task_comm()
* - lock it with task_lock()
*/
char comm[TASK_COMM_LEN];
struct nameidata *nameidata;
#ifdef CONFIG_SYSVIPC
struct sysv_sem sysvsem;
struct sysv_shm sysvshm;
#endif
#ifdef CONFIG_DETECT_HUNG_TASK
unsigned long last_switch_count;
unsigned long last_switch_time;
#endif
/* Filesystem information: */
struct fs_struct *fs;
/* Open file information: */
struct files_struct *files;
#ifdef CONFIG_IO_URING
struct io_uring_task *io_uring;
#endif
/* Namespaces: */
struct nsproxy *nsproxy;
/* Signal handlers: */
struct signal_struct *signal;
struct sighand_struct __rcu *sighand;
sigset_t blocked;
sigset_t real_blocked;
/* Restored if set_restore_sigmask() was used: */
sigset_t saved_sigmask;
struct sigpending pending;
unsigned long sas_ss_sp;
size_t sas_ss_size;
unsigned int sas_ss_flags;
struct callback_head *task_works;
#ifdef CONFIG_AUDIT
#ifdef CONFIG_AUDITSYSCALL
struct audit_context *audit_context;
#endif
kuid_t loginuid;
unsigned int sessionid;
#endif
struct seccomp seccomp;
struct syscall_user_dispatch syscall_dispatch;
/* Thread group tracking: */
u64 parent_exec_id;
u64 self_exec_id;
/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
spinlock_t alloc_lock;
/* Protection of the PI data structures: */
raw_spinlock_t pi_lock;
struct wake_q_node wake_q;
#ifdef CONFIG_RT_MUTEXES
/* PI waiters blocked on a rt_mutex held by this task: */
struct rb_root_cached pi_waiters;
/* Updated under owner's pi_lock and rq lock */
struct task_struct *pi_top_task;
/* Deadlock detection and priority inheritance handling: */
struct rt_mutex_waiter *pi_blocked_on;
#endif
#ifdef CONFIG_DEBUG_MUTEXES
/* Mutex deadlock detection: */
struct mutex_waiter *blocked_on;
#endif
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
int non_block_count;
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
struct irqtrace_events irqtrace;
unsigned int hardirq_threaded;
u64 hardirq_chain_key;
int softirqs_enabled;
int softirq_context;
int irq_config;
#endif
#ifdef CONFIG_PREEMPT_RT
int softirq_disable_cnt;
#endif
#ifdef CONFIG_LOCKDEP
# define MAX_LOCK_DEPTH 48UL
u64 curr_chain_key;
int lockdep_depth;
unsigned int lockdep_recursion;
struct held_lock held_locks[MAX_LOCK_DEPTH];
#endif
#if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
unsigned int in_ubsan;
#endif
/* Journalling filesystem info: */
void *journal_info;
/* Stacked block device info: */
struct bio_list *bio_list;
/* Stack plugging: */
struct blk_plug *plug;
/* VM state: */
struct reclaim_state *reclaim_state;
struct io_context *io_context;
#ifdef CONFIG_COMPACTION
struct capture_control *capture_control;
#endif
/* Ptrace state: */
unsigned long ptrace_message;
kernel_siginfo_t *last_siginfo;
struct task_io_accounting ioac;
#ifdef CONFIG_PSI
/* Pressure stall state */
unsigned int psi_flags;
#endif
#ifdef CONFIG_TASK_XACCT
/* Accumulated RSS usage: */
u64 acct_rss_mem1;
/* Accumulated virtual memory usage: */
u64 acct_vm_mem1;
/* stime + utime since last update: */
u64 acct_timexpd;
#endif
#ifdef CONFIG_CPUSETS
/* Protected by ->alloc_lock: */
nodemask_t mems_allowed;
/* Sequence number to catch updates: */
seqcount_spinlock_t mems_allowed_seq;
int cpuset_mem_spread_rotor;
int cpuset_slab_spread_rotor;
#endif
#ifdef CONFIG_CGROUPS
/* Control Group info protected by css_set_lock: */
struct css_set __rcu *cgroups;
/* cg_list protected by css_set_lock and tsk->alloc_lock: */
struct list_head cg_list;
#endif
#ifdef CONFIG_X86_CPU_RESCTRL
u32 closid;
u32 rmid;
#endif
#ifdef CONFIG_FUTEX
struct robust_list_head __user *robust_list;
#ifdef CONFIG_COMPAT
struct compat_robust_list_head __user *compat_robust_list;
#endif
struct list_head pi_state_list;
struct futex_pi_state *pi_state_cache;
struct mutex futex_exit_mutex;
unsigned int futex_state;
#endif
#ifdef CONFIG_PERF_EVENTS
struct perf_event_context *perf_event_ctxp;
struct mutex perf_event_mutex;
struct list_head perf_event_list;
#endif
#ifdef CONFIG_DEBUG_PREEMPT
unsigned long preempt_disable_ip;
#endif
#ifdef CONFIG_NUMA
/* Protected by alloc_lock: */
struct mempolicy *mempolicy;
short il_prev;
u8 il_weight;
short pref_node_fork;
#endif
#ifdef CONFIG_NUMA_BALANCING
int numa_scan_seq;
unsigned int numa_scan_period;
unsigned int numa_scan_period_max;
int numa_preferred_nid;
unsigned long numa_migrate_retry;
/* Migration stamp: */
u64 node_stamp;
u64 last_task_numa_placement;
u64 last_sum_exec_runtime;
struct callback_head numa_work;
/*
* This pointer is only modified for current in syscall and
* pagefault context (and for tasks being destroyed), so it can be read
* from any of the following contexts:
* - RCU read-side critical section
* - current->numa_group from everywhere
* - task's runqueue locked, task not running
*/
struct numa_group __rcu *numa_group;
/*
* numa_faults is an array split into four regions:
* faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
* in this precise order.
*
* faults_memory: Exponential decaying average of faults on a per-node
* basis. Scheduling placement decisions are made based on these
* counts. The values remain static for the duration of a PTE scan.
* faults_cpu: Track the nodes the process was running on when a NUMA
* hinting fault was incurred.
* faults_memory_buffer and faults_cpu_buffer: Record faults per node
* during the current scan window. When the scan completes, the counts
* in faults_memory and faults_cpu decay and these values are copied.
*/
unsigned long *numa_faults;
unsigned long total_numa_faults;
/*
* numa_faults_locality tracks if faults recorded during the last
* scan window were remote/local or failed to migrate. The task scan
* period is adapted based on the locality of the faults with different
* weights depending on whether they were shared or private faults
*/
unsigned long numa_faults_locality[3];
unsigned long numa_pages_migrated;
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_RSEQ
struct rseq __user *rseq;
u32 rseq_len;
u32 rseq_sig;
/*
* RmW on rseq_event_mask must be performed atomically
* with respect to preemption.
*/
unsigned long rseq_event_mask;
#endif
#ifdef CONFIG_SCHED_MM_CID
int mm_cid; /* Current cid in mm */
int last_mm_cid; /* Most recent cid in mm */
int migrate_from_cpu;
int mm_cid_active; /* Whether cid bitmap is active */
struct callback_head cid_work;
#endif
struct tlbflush_unmap_batch tlb_ubc;
/* Cache last used pipe for splice(): */
struct pipe_inode_info *splice_pipe;
struct page_frag task_frag;
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
#ifdef CONFIG_FAULT_INJECTION
int make_it_fail;
unsigned int fail_nth;
#endif
/*
* When (nr_dirtied >= nr_dirtied_pause), it's time to call
* balance_dirty_pages() for a dirty throttling pause:
*/
int nr_dirtied;
int nr_dirtied_pause;
/* Start of a write-and-pause period: */
unsigned long dirty_paused_when;
#ifdef CONFIG_LATENCYTOP
int latency_record_count;
struct latency_record latency_record[LT_SAVECOUNT];
#endif
/*
* Time slack values; these are used to round up poll() and
* select() etc timeout values. These are in nanoseconds.
*/
u64 timer_slack_ns;
u64 default_timer_slack_ns;
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
unsigned int kasan_depth;
#endif
#ifdef CONFIG_KCSAN
struct kcsan_ctx kcsan_ctx;
#ifdef CONFIG_TRACE_IRQFLAGS
struct irqtrace_events kcsan_save_irqtrace;
#endif
#ifdef CONFIG_KCSAN_WEAK_MEMORY
int kcsan_stack_depth;
#endif
#endif
#ifdef CONFIG_KMSAN
struct kmsan_ctx kmsan_ctx;
#endif
#if IS_ENABLED(CONFIG_KUNIT)
struct kunit *kunit_test;
#endif
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* Index of current stored address in ret_stack: */
int curr_ret_stack;
int curr_ret_depth;
/* Stack of return addresses for return function tracing: */
struct ftrace_ret_stack *ret_stack;
/* Timestamp for last schedule: */
unsigned long long ftrace_timestamp;
/*
* Number of functions that haven't been traced
* because of depth overrun:
*/
atomic_t trace_overrun;
/* Pause tracing: */
atomic_t tracing_graph_pause;
#endif
#ifdef CONFIG_TRACING
/* Bitmask and counter of trace recursion: */
unsigned long trace_recursion;
#endif /* CONFIG_TRACING */
#ifdef CONFIG_KCOV
/* See kernel/kcov.c for more details. */
/* Coverage collection mode enabled for this task (0 if disabled): */
unsigned int kcov_mode;
/* Size of the kcov_area: */
unsigned int kcov_size;
/* Buffer for coverage collection: */
void *kcov_area;
/* KCOV descriptor wired with this task or NULL: */
struct kcov *kcov;
/* KCOV common handle for remote coverage collection: */
u64 kcov_handle;
/* KCOV sequence number: */
int kcov_sequence;
/* Collect coverage from softirq context: */
unsigned int kcov_softirq;
#endif
#ifdef CONFIG_MEMCG
struct mem_cgroup *memcg_in_oom;
gfp_t memcg_oom_gfp_mask;
int memcg_oom_order;
/* Number of pages to reclaim on returning to userland: */
unsigned int memcg_nr_pages_over_high;
/* Used by memcontrol for targeted memcg charge: */
struct mem_cgroup *active_memcg;
#endif
#ifdef CONFIG_MEMCG_KMEM
struct obj_cgroup *objcg;
#endif
#ifdef CONFIG_BLK_CGROUP
struct gendisk *throttle_disk;
#endif
#ifdef CONFIG_UPROBES
struct uprobe_task *utask;
#endif
#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
unsigned int sequential_io;
unsigned int sequential_io_avg;
#endif
struct kmap_ctrl kmap_ctrl;
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
unsigned long task_state_change;
# ifdef CONFIG_PREEMPT_RT
unsigned long saved_state_change;
# endif
#endif
struct rcu_head rcu;
refcount_t rcu_users;
int pagefault_disabled;
#ifdef CONFIG_MMU
struct task_struct *oom_reaper_list;
struct timer_list oom_reaper_timer;
#endif
#ifdef CONFIG_VMAP_STACK
struct vm_struct *stack_vm_area;
#endif
#ifdef CONFIG_THREAD_INFO_IN_TASK
/* A live task holds one reference: */
refcount_t stack_refcount;
#endif
#ifdef CONFIG_LIVEPATCH
int patch_state;
#endif
#ifdef CONFIG_SECURITY
/* Used by LSM modules for access restriction: */
void *security;
#endif
#ifdef CONFIG_BPF_SYSCALL
/* Used by BPF task local storage */
struct bpf_local_storage __rcu *bpf_storage;
/* Used for BPF run context */
struct bpf_run_ctx *bpf_ctx;
#endif
#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
unsigned long lowest_stack;
unsigned long prev_lowest_stack;
#endif
#ifdef CONFIG_X86_MCE
void __user *mce_vaddr;
__u64 mce_kflags;
u64 mce_addr;
__u64 mce_ripv : 1,
mce_whole_page : 1,
__mce_reserved : 62;
struct callback_head mce_kill_me;
int mce_count;
#endif
#ifdef CONFIG_KRETPROBES
struct llist_head kretprobe_instances;
#endif
#ifdef CONFIG_RETHOOK
struct llist_head rethooks;
#endif
#ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH
/*
* If L1D flush is supported on mm context switch
* then we use this callback head to queue kill work
* to kill tasks that are not running on SMT disabled
* cores
*/
struct callback_head l1d_flush_kill;
#endif
#ifdef CONFIG_RV
/*
* Per-task RV monitor. Nowadays fixed in RV_PER_TASK_MONITORS.
* If we find justification for more monitors, we can think
* about adding more or developing a dynamic method. So far,
* none of these are justified.
*/
union rv_task_monitor rv[RV_PER_TASK_MONITORS];
#endif
#ifdef CONFIG_USER_EVENTS
struct user_event_mm *user_event_mm;
#endif
/*
* New fields for task_struct should be added above here, so that
* they are included in the randomized portion of task_struct.
*/
randomized_struct_fields_end
/* CPU-specific state of this task: */
struct thread_struct thread;
/*
* WARNING: on x86, 'thread_struct' contains a variable-sized
* structure. It *MUST* be at the end of 'task_struct'.
*
* Do not put anything below here!
*/
};
#define TASK_REPORT_IDLE (TASK_REPORT + 1)
#define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1)
static inline unsigned int __task_state_index(unsigned int tsk_state,
unsigned int tsk_exit_state)
{
unsigned int state = (tsk_state | tsk_exit_state) & TASK_REPORT;
BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
if ((tsk_state & TASK_IDLE) == TASK_IDLE)
state = TASK_REPORT_IDLE;
/*
* We're lying here, but rather than expose a completely new task state
* to userspace, we can make this appear as if the task has gone through
* a regular rt_mutex_lock() call.
*/
if (tsk_state & TASK_RTLOCK_WAIT)
state = TASK_UNINTERRUPTIBLE;
return fls(state);
}
static inline unsigned int task_state_index(struct task_struct *tsk)
{
return __task_state_index(READ_ONCE(tsk->__state), tsk->exit_state);
}
static inline char task_index_to_char(unsigned int state)
{
static const char state_char[] = "RSDTtXZPI";
BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
return state_char[state];
}
static inline char task_state_to_char(struct task_struct *tsk)
{
return task_index_to_char(task_state_index(tsk));
}
extern struct pid *cad_pid;
/*
* Per process flags
*/
#define PF_VCPU 0x00000001 /* I'm a virtual CPU */
#define PF_IDLE 0x00000002 /* I am an IDLE thread */
#define PF_EXITING 0x00000004 /* Getting shut down */
#define PF_POSTCOREDUMP 0x00000008 /* Coredumps should ignore this task */
#define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
#define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */
#define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */
#define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */
#define PF_DUMPCORE 0x00000200 /* Dumped core */
#define PF_SIGNALED 0x00000400 /* Killed by a signal */
#define PF_MEMALLOC 0x00000800 /* Allocating memory to free memory. See memalloc_noreclaim_save() */
#define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */
#define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */
#define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */
#define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */
#define PF__HOLE__00010000 0x00010000
#define PF_KSWAPD 0x00020000 /* I am kswapd */
#define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */
#define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */
#define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to,
* I am cleaning dirty pages from some other bdi. */
#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
#define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */
#define PF_MEMALLOC_NORECLAIM 0x00800000 /* All allocation requests will clear __GFP_DIRECT_RECLAIM */
#define PF_MEMALLOC_NOWARN 0x01000000 /* All allocation requests will inherit __GFP_NOWARN */
#define PF__HOLE__02000000 0x02000000
#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */
#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
#define PF_MEMALLOC_PIN 0x10000000 /* Allocations constrained to zones which allow long term pinning.
* See memalloc_pin_save() */
#define PF_BLOCK_TS 0x20000000 /* plug has ts that needs updating */
#define PF__HOLE__40000000 0x40000000
#define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */
/*
* Only the _current_ task can read/write to tsk->flags, but other
* tasks can access tsk->flags in readonly mode for example
* with tsk_used_math (like during threaded core dumping).
* There is however an exception to this rule during ptrace
* or during fork: the ptracer task is allowed to write to the
* child->flags of its traced child (same goes for fork, the parent
* can write to the child->flags), because we're guaranteed the
* child is not running and in turn not changing child->flags
* at the same time the parent does it.
*/
#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
#define clear_used_math() clear_stopped_child_used_math(current)
#define set_used_math() set_stopped_child_used_math(current)
#define conditional_stopped_child_used_math(condition, child) \
do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
#define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current)
#define copy_to_stopped_child_used_math(child) \
do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
#define used_math() tsk_used_math(current)
static __always_inline bool is_percpu_thread(void)
{
#ifdef CONFIG_SMP
return (current->flags & PF_NO_SETAFFINITY) &&
(current->nr_cpus_allowed == 1);
#else
return true;
#endif
}
/* Per-process atomic flags. */
#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */
#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */
#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */
#define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */
#define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/
#define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */
#define PFA_SPEC_IB_FORCE_DISABLE 6 /* Indirect branch speculation permanently restricted */
#define PFA_SPEC_SSB_NOEXEC 7 /* Speculative Store Bypass clear on execve() */
#define TASK_PFA_TEST(name, func) \
static inline bool task_##func(struct task_struct *p) \
{ return test_bit(PFA_##name, &p->atomic_flags); }
#define TASK_PFA_SET(name, func) \
static inline void task_set_##func(struct task_struct *p) \
{ set_bit(PFA_##name, &p->atomic_flags); }
#define TASK_PFA_CLEAR(name, func) \
static inline void task_clear_##func(struct task_struct *p) \
{ clear_bit(PFA_##name, &p->atomic_flags); }
TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
TASK_PFA_TEST(SPREAD_PAGE, spread_page)
TASK_PFA_SET(SPREAD_PAGE, spread_page)
TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
TASK_PFA_SET(SPREAD_SLAB, spread_slab)
TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
static inline void
current_restore_flags(unsigned long orig_flags, unsigned long flags)
{
current->flags &= ~flags;
current->flags |= orig_flags & flags;
}
extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
extern int task_can_attach(struct task_struct *p);
extern int dl_bw_alloc(int cpu, u64 dl_bw);
extern void dl_bw_free(int cpu, u64 dl_bw);
#ifdef CONFIG_SMP
/* do_set_cpus_allowed() - consider using set_cpus_allowed_ptr() instead */
extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
/**
* set_cpus_allowed_ptr - set CPU affinity mask of a task
* @p: the task
* @new_mask: CPU affinity mask
*
* Return: zero if successful, or a negative error code
*/
extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node);
extern void release_user_cpus_ptr(struct task_struct *p);
extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask);
extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p);
#else
static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
}
static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
{
if (!cpumask_test_cpu(0, new_mask))
return -EINVAL;
return 0;
}
static inline int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node)
{
if (src->user_cpus_ptr)
return -EINVAL;
return 0;
}
static inline void release_user_cpus_ptr(struct task_struct *p)
{
WARN_ON(p->user_cpus_ptr);
}
static inline int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
{
return 0;
}
#endif
extern int yield_to(struct task_struct *p, bool preempt);
extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p);
/**
* task_nice - return the nice value of a given task.
* @p: the task in question.
*
* Return: The nice value [ -20 ... 0 ... 19 ].
*/
static inline int task_nice(const struct task_struct *p)
{
return PRIO_TO_NICE((p)->static_prio);
}
extern int can_nice(const struct task_struct *p, const int nice);
extern int task_curr(const struct task_struct *p);
extern int idle_cpu(int cpu);
extern int available_idle_cpu(int cpu);
extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
extern void sched_set_fifo(struct task_struct *p);
extern void sched_set_fifo_low(struct task_struct *p);
extern void sched_set_normal(struct task_struct *p, int nice);
extern int sched_setattr(struct task_struct *, const struct sched_attr *);
extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
extern struct task_struct *idle_task(int cpu);
/**
* is_idle_task - is the specified task an idle task?
* @p: the task in question.
*
* Return: 1 if @p is an idle task. 0 otherwise.
*/
static __always_inline bool is_idle_task(const struct task_struct *p)
{
return !!(p->flags & PF_IDLE);
}
extern struct task_struct *curr_task(int cpu);
extern void ia64_set_curr_task(int cpu, struct task_struct *p);
void yield(void);
union thread_union {
struct task_struct task;
#ifndef CONFIG_THREAD_INFO_IN_TASK
struct thread_info thread_info;
#endif
unsigned long stack[THREAD_SIZE/sizeof(long)];
};
#ifndef CONFIG_THREAD_INFO_IN_TASK
extern struct thread_info init_thread_info;
#endif
extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
#ifdef CONFIG_THREAD_INFO_IN_TASK
# define task_thread_info(task) (&(task)->thread_info)
#elif !defined(__HAVE_THREAD_FUNCTIONS)
# define task_thread_info(task) ((struct thread_info *)(task)->stack)
#endif
/*
* find a task by one of its numerical ids
*
* find_task_by_pid_ns():
* finds a task by its pid in the specified namespace
* find_task_by_vpid():
* finds a task by its virtual pid
*
* see also find_vpid() etc in include/linux/pid.h
*/
extern struct task_struct *find_task_by_vpid(pid_t nr);
extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
/*
* find a task by its virtual pid and get the task struct
*/
extern struct task_struct *find_get_task_by_vpid(pid_t nr);
extern int wake_up_state(struct task_struct *tsk, unsigned int state);
extern int wake_up_process(struct task_struct *tsk);
extern void wake_up_new_task(struct task_struct *tsk);
#ifdef CONFIG_SMP
extern void kick_process(struct task_struct *tsk);
#else
static inline void kick_process(struct task_struct *tsk) { }
#endif
extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
static inline void set_task_comm(struct task_struct *tsk, const char *from)
{
__set_task_comm(tsk, from, false);
}
extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
#define get_task_comm(buf, tsk) ({ \
BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \
__get_task_comm(buf, sizeof(buf), tsk); \
})
#ifdef CONFIG_SMP
static __always_inline void scheduler_ipi(void)
{
/*
* Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
* TIF_NEED_RESCHED remotely (for the first time) will also send
* this IPI.
*/
preempt_fold_need_resched();
}
#else
static inline void scheduler_ipi(void) { }
#endif
extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state);
/*
* Set thread flags in other task's structures.
* See asm/thread_info.h for TIF_xxxx flags available:
*/
static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
set_ti_thread_flag(task_thread_info(tsk), flag);
}
static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
clear_ti_thread_flag(task_thread_info(tsk), flag);
}
static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
bool value)
{
update_ti_thread_flag(task_thread_info(tsk), flag, value);
}
static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
{
return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
}
static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
{
return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
}
static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
{
return test_ti_thread_flag(task_thread_info(tsk), flag);
}
static inline void set_tsk_need_resched(struct task_struct *tsk)
{
set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
}
static inline void clear_tsk_need_resched(struct task_struct *tsk)
{
clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
}
static inline int test_tsk_need_resched(struct task_struct *tsk)
{
return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
}
/*
* cond_resched() and cond_resched_lock(): latency reduction via
* explicit rescheduling in places that are safe. The return
* value indicates whether a reschedule was done in fact.
* cond_resched_lock() will drop the spinlock before scheduling,
*/
#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
extern int __cond_resched(void);
#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
void sched_dynamic_klp_enable(void);
void sched_dynamic_klp_disable(void);
DECLARE_STATIC_CALL(cond_resched, __cond_resched);
static __always_inline int _cond_resched(void)
{
return static_call_mod(cond_resched)();
}
#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
extern int dynamic_cond_resched(void);
static __always_inline int _cond_resched(void)
{
return dynamic_cond_resched();
}
#else /* !CONFIG_PREEMPTION */
static inline int _cond_resched(void)
{
klp_sched_try_switch();
return __cond_resched();
}
#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */
#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */
static inline int _cond_resched(void)
{
klp_sched_try_switch();
return 0;
}
#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */
#define cond_resched() ({ \
__might_resched(__FILE__, __LINE__, 0); \
_cond_resched(); \
})
extern int __cond_resched_lock(spinlock_t *lock);
extern int __cond_resched_rwlock_read(rwlock_t *lock);
extern int __cond_resched_rwlock_write(rwlock_t *lock);
#define MIGHT_RESCHED_RCU_SHIFT 8
#define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1)
#ifndef CONFIG_PREEMPT_RT
/*
* Non RT kernels have an elevated preempt count due to the held lock,
* but are not allowed to be inside a RCU read side critical section
*/
# define PREEMPT_LOCK_RESCHED_OFFSETS PREEMPT_LOCK_OFFSET
#else
/*
* spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in
* cond_resched*lock() has to take that into account because it checks for
* preempt_count() and rcu_preempt_depth().
*/
# define PREEMPT_LOCK_RESCHED_OFFSETS \
(PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT))
#endif
#define cond_resched_lock(lock) ({ \
__might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
__cond_resched_lock(lock); \
})
#define cond_resched_rwlock_read(lock) ({ \
__might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
__cond_resched_rwlock_read(lock); \
})
#define cond_resched_rwlock_write(lock) ({ \
__might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
__cond_resched_rwlock_write(lock); \
})
#ifdef CONFIG_PREEMPT_DYNAMIC
extern bool preempt_model_none(void);
extern bool preempt_model_voluntary(void);
extern bool preempt_model_full(void);
#else
static inline bool preempt_model_none(void)
{
return IS_ENABLED(CONFIG_PREEMPT_NONE);
}
static inline bool preempt_model_voluntary(void)
{
return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
}
static inline bool preempt_model_full(void)
{
return IS_ENABLED(CONFIG_PREEMPT);
}
#endif
static inline bool preempt_model_rt(void)
{
return IS_ENABLED(CONFIG_PREEMPT_RT);
}
/*
* Does the preemption model allow non-cooperative preemption?
*
* For !CONFIG_PREEMPT_DYNAMIC kernels this is an exact match with
* CONFIG_PREEMPTION; for CONFIG_PREEMPT_DYNAMIC this doesn't work as the
* kernel is *built* with CONFIG_PREEMPTION=y but may run with e.g. the
* PREEMPT_NONE model.
*/
static inline bool preempt_model_preemptible(void)
{
return preempt_model_full() || preempt_model_rt();
}
static __always_inline bool need_resched(void)
{
return unlikely(tif_need_resched());
}
/*
* Wrappers for p->thread_info->cpu access. No-op on UP.
*/
#ifdef CONFIG_SMP
static inline unsigned int task_cpu(const struct task_struct *p)
{
return READ_ONCE(task_thread_info(p)->cpu);
}
extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
#else
static inline unsigned int task_cpu(const struct task_struct *p)
{
return 0;
}
static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
{
}
#endif /* CONFIG_SMP */
extern bool sched_task_on_rq(struct task_struct *p);
extern unsigned long get_wchan(struct task_struct *p);
extern struct task_struct *cpu_curr_snapshot(int cpu);
#include <linux/spinlock.h>
/*
* In order to reduce various lock holder preemption latencies provide an
* interface to see if a vCPU is currently running or not.
*
* This allows us to terminate optimistic spin loops and block, analogous to
* the native optimistic spin heuristic of testing if the lock owner task is
* running or not.
*/
#ifndef vcpu_is_preempted
static inline bool vcpu_is_preempted(int cpu)
{
return false;
}
#endif
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
#ifndef TASK_SIZE_OF
#define TASK_SIZE_OF(tsk) TASK_SIZE
#endif
#ifdef CONFIG_SMP
static inline bool owner_on_cpu(struct task_struct *owner)
{
/*
* As lock holder preemption issue, we both skip spinning if
* task is not on cpu or its cpu is preempted
*/
return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(task_cpu(owner));
}
/* Returns effective CPU energy utilization, as seen by the scheduler */
unsigned long sched_cpu_util(int cpu);
#endif /* CONFIG_SMP */
#ifdef CONFIG_SCHED_CORE
extern void sched_core_free(struct task_struct *tsk);
extern void sched_core_fork(struct task_struct *p);
extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
unsigned long uaddr);
extern int sched_core_idle_cpu(int cpu);
#else
static inline void sched_core_free(struct task_struct *tsk) { }
static inline void sched_core_fork(struct task_struct *p) { }
static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); }
#endif
extern void sched_set_stop_task(int cpu, struct task_struct *stop);
#endif
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* pm_runtime.h - Device run-time power management helper functions.
*
* Copyright (C) 2009 Rafael J. Wysocki <rjw@sisk.pl>
*/
#ifndef _LINUX_PM_RUNTIME_H
#define _LINUX_PM_RUNTIME_H
#include <linux/device.h>
#include <linux/notifier.h>
#include <linux/pm.h>
#include <linux/jiffies.h>
/* Runtime PM flag argument bits */
#define RPM_ASYNC 0x01 /* Request is asynchronous */
#define RPM_NOWAIT 0x02 /* Don't wait for concurrent
state change */
#define RPM_GET_PUT 0x04 /* Increment/decrement the
usage_count */
#define RPM_AUTO 0x08 /* Use autosuspend_delay */
/*
* Use this for defining a set of PM operations to be used in all situations
* (system suspend, hibernation or runtime PM).
*
* Note that the behaviour differs from the deprecated UNIVERSAL_DEV_PM_OPS()
* macro, which uses the provided callbacks for both runtime PM and system
* sleep, while DEFINE_RUNTIME_DEV_PM_OPS() uses pm_runtime_force_suspend()
* and pm_runtime_force_resume() for its system sleep callbacks.
*
* If the underlying dev_pm_ops struct symbol has to be exported, use
* EXPORT_RUNTIME_DEV_PM_OPS() or EXPORT_GPL_RUNTIME_DEV_PM_OPS() instead.
*/
#define DEFINE_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
_DEFINE_DEV_PM_OPS(name, pm_runtime_force_suspend, \
pm_runtime_force_resume, suspend_fn, \
resume_fn, idle_fn)
#define EXPORT_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
EXPORT_DEV_PM_OPS(name) = { \
RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
}
#define EXPORT_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
EXPORT_GPL_DEV_PM_OPS(name) = { \
RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
}
#define EXPORT_NS_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \
EXPORT_NS_DEV_PM_OPS(name, ns) = { \
RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
}
#define EXPORT_NS_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \
EXPORT_NS_GPL_DEV_PM_OPS(name, ns) = { \
RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
}
#ifdef CONFIG_PM
extern struct workqueue_struct *pm_wq;
static inline bool queue_pm_work(struct work_struct *work)
{
return queue_work(pm_wq, work);
}
extern int pm_generic_runtime_suspend(struct device *dev);
extern int pm_generic_runtime_resume(struct device *dev);
extern int pm_runtime_force_suspend(struct device *dev);
extern int pm_runtime_force_resume(struct device *dev);
extern int __pm_runtime_idle(struct device *dev, int rpmflags);
extern int __pm_runtime_suspend(struct device *dev, int rpmflags);
extern int __pm_runtime_resume(struct device *dev, int rpmflags);
extern int pm_runtime_get_if_active(struct device *dev);
extern int pm_runtime_get_if_in_use(struct device *dev);
extern int pm_schedule_suspend(struct device *dev, unsigned int delay);
extern int __pm_runtime_set_status(struct device *dev, unsigned int status);
extern int pm_runtime_barrier(struct device *dev);
extern void pm_runtime_enable(struct device *dev);
extern void __pm_runtime_disable(struct device *dev, bool check_resume);
extern void pm_runtime_allow(struct device *dev);
extern void pm_runtime_forbid(struct device *dev);
extern void pm_runtime_no_callbacks(struct device *dev);
extern void pm_runtime_irq_safe(struct device *dev);
extern void __pm_runtime_use_autosuspend(struct device *dev, bool use);
extern void pm_runtime_set_autosuspend_delay(struct device *dev, int delay);
extern u64 pm_runtime_autosuspend_expiration(struct device *dev);
extern void pm_runtime_set_memalloc_noio(struct device *dev, bool enable);
extern void pm_runtime_get_suppliers(struct device *dev);
extern void pm_runtime_put_suppliers(struct device *dev);
extern void pm_runtime_new_link(struct device *dev);
extern void pm_runtime_drop_link(struct device_link *link);
extern void pm_runtime_release_supplier(struct device_link *link);
extern int devm_pm_runtime_enable(struct device *dev);
/**
* pm_suspend_ignore_children - Set runtime PM behavior regarding children.
* @dev: Target device.
* @enable: Whether or not to ignore possible dependencies on children.
*
* The dependencies of @dev on its children will not be taken into account by
* the runtime PM framework going forward if @enable is %true, or they will
* be taken into account otherwise.
*/
static inline void pm_suspend_ignore_children(struct device *dev, bool enable)
{
dev->power.ignore_children = enable;
}
/**
* pm_runtime_get_noresume - Bump up runtime PM usage counter of a device.
* @dev: Target device.
*/
static inline void pm_runtime_get_noresume(struct device *dev)
{
atomic_inc(&dev->power.usage_count);
}
/**
* pm_runtime_put_noidle - Drop runtime PM usage counter of a device.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev unless it is 0 already.
*/
static inline void pm_runtime_put_noidle(struct device *dev)
{
atomic_add_unless(&dev->power.usage_count, -1, 0);
}
/**
* pm_runtime_suspended - Check whether or not a device is runtime-suspended.
* @dev: Target device.
*
* Return %true if runtime PM is enabled for @dev and its runtime PM status is
* %RPM_SUSPENDED, or %false otherwise.
*
* Note that the return value of this function can only be trusted if it is
* called under the runtime PM lock of @dev or under conditions in which
* runtime PM cannot be either disabled or enabled for @dev and its runtime PM
* status cannot change.
*/
static inline bool pm_runtime_suspended(struct device *dev)
{
return dev->power.runtime_status == RPM_SUSPENDED
&& !dev->power.disable_depth;
}
/**
* pm_runtime_active - Check whether or not a device is runtime-active.
* @dev: Target device.
*
* Return %true if runtime PM is disabled for @dev or its runtime PM status is
* %RPM_ACTIVE, or %false otherwise.
*
* Note that the return value of this function can only be trusted if it is
* called under the runtime PM lock of @dev or under conditions in which
* runtime PM cannot be either disabled or enabled for @dev and its runtime PM
* status cannot change.
*/
static inline bool pm_runtime_active(struct device *dev)
{
return dev->power.runtime_status == RPM_ACTIVE || dev->power.disable_depth;
}
/**
* pm_runtime_status_suspended - Check if runtime PM status is "suspended".
* @dev: Target device.
*
* Return %true if the runtime PM status of @dev is %RPM_SUSPENDED, or %false
* otherwise, regardless of whether or not runtime PM has been enabled for @dev.
*
* Note that the return value of this function can only be trusted if it is
* called under the runtime PM lock of @dev or under conditions in which the
* runtime PM status of @dev cannot change.
*/
static inline bool pm_runtime_status_suspended(struct device *dev)
{
return dev->power.runtime_status == RPM_SUSPENDED;
}
/**
* pm_runtime_enabled - Check if runtime PM is enabled.
* @dev: Target device.
*
* Return %true if runtime PM is enabled for @dev or %false otherwise.
*
* Note that the return value of this function can only be trusted if it is
* called under the runtime PM lock of @dev or under conditions in which
* runtime PM cannot be either disabled or enabled for @dev.
*/
static inline bool pm_runtime_enabled(struct device *dev)
{
return !dev->power.disable_depth;
}
/**
* pm_runtime_has_no_callbacks - Check if runtime PM callbacks may be present.
* @dev: Target device.
*
* Return %true if @dev is a special device without runtime PM callbacks or
* %false otherwise.
*/
static inline bool pm_runtime_has_no_callbacks(struct device *dev)
{
return dev->power.no_callbacks;
}
/**
* pm_runtime_mark_last_busy - Update the last access time of a device.
* @dev: Target device.
*
* Update the last access time of @dev used by the runtime PM autosuspend
* mechanism to the current time as returned by ktime_get_mono_fast_ns().
*/
static inline void pm_runtime_mark_last_busy(struct device *dev)
{
WRITE_ONCE(dev->power.last_busy, ktime_get_mono_fast_ns());
}
/**
* pm_runtime_is_irq_safe - Check if runtime PM can work in interrupt context.
* @dev: Target device.
*
* Return %true if @dev has been marked as an "IRQ-safe" device (with respect
* to runtime PM), in which case its runtime PM callabcks can be expected to
* work correctly when invoked from interrupt handlers.
*/
static inline bool pm_runtime_is_irq_safe(struct device *dev)
{
return dev->power.irq_safe;
}
extern u64 pm_runtime_suspended_time(struct device *dev);
#else /* !CONFIG_PM */
static inline bool queue_pm_work(struct work_struct *work) { return false; }
static inline int pm_generic_runtime_suspend(struct device *dev) { return 0; }
static inline int pm_generic_runtime_resume(struct device *dev) { return 0; }
static inline int pm_runtime_force_suspend(struct device *dev) { return 0; }
static inline int pm_runtime_force_resume(struct device *dev) { return 0; }
static inline int __pm_runtime_idle(struct device *dev, int rpmflags)
{
return -ENOSYS;
}
static inline int __pm_runtime_suspend(struct device *dev, int rpmflags)
{
return -ENOSYS;
}
static inline int __pm_runtime_resume(struct device *dev, int rpmflags)
{
return 1;
}
static inline int pm_schedule_suspend(struct device *dev, unsigned int delay)
{
return -ENOSYS;
}
static inline int pm_runtime_get_if_in_use(struct device *dev)
{
return -EINVAL;
}
static inline int pm_runtime_get_if_active(struct device *dev)
{
return -EINVAL;
}
static inline int __pm_runtime_set_status(struct device *dev,
unsigned int status) { return 0; }
static inline int pm_runtime_barrier(struct device *dev) { return 0; }
static inline void pm_runtime_enable(struct device *dev) {}
static inline void __pm_runtime_disable(struct device *dev, bool c) {}
static inline void pm_runtime_allow(struct device *dev) {}
static inline void pm_runtime_forbid(struct device *dev) {}
static inline int devm_pm_runtime_enable(struct device *dev) { return 0; }
static inline void pm_suspend_ignore_children(struct device *dev, bool enable) {}
static inline void pm_runtime_get_noresume(struct device *dev) {}
static inline void pm_runtime_put_noidle(struct device *dev) {}
static inline bool pm_runtime_suspended(struct device *dev) { return false; }
static inline bool pm_runtime_active(struct device *dev) { return true; }
static inline bool pm_runtime_status_suspended(struct device *dev) { return false; }
static inline bool pm_runtime_enabled(struct device *dev) { return false; }
static inline void pm_runtime_no_callbacks(struct device *dev) {}
static inline void pm_runtime_irq_safe(struct device *dev) {}
static inline bool pm_runtime_is_irq_safe(struct device *dev) { return false; }
static inline bool pm_runtime_has_no_callbacks(struct device *dev) { return false; }
static inline void pm_runtime_mark_last_busy(struct device *dev) {}
static inline void __pm_runtime_use_autosuspend(struct device *dev,
bool use) {}
static inline void pm_runtime_set_autosuspend_delay(struct device *dev,
int delay) {}
static inline u64 pm_runtime_autosuspend_expiration(
struct device *dev) { return 0; }
static inline void pm_runtime_set_memalloc_noio(struct device *dev,
bool enable){}
static inline void pm_runtime_get_suppliers(struct device *dev) {}
static inline void pm_runtime_put_suppliers(struct device *dev) {}
static inline void pm_runtime_new_link(struct device *dev) {}
static inline void pm_runtime_drop_link(struct device_link *link) {}
static inline void pm_runtime_release_supplier(struct device_link *link) {}
#endif /* !CONFIG_PM */
/**
* pm_runtime_idle - Conditionally set up autosuspend of a device or suspend it.
* @dev: Target device.
*
* Invoke the "idle check" callback of @dev and, depending on its return value,
* set up autosuspend of @dev or suspend it (depending on whether or not
* autosuspend has been enabled for it).
*/
static inline int pm_runtime_idle(struct device *dev)
{
return __pm_runtime_idle(dev, 0);
}
/**
* pm_runtime_suspend - Suspend a device synchronously.
* @dev: Target device.
*/
static inline int pm_runtime_suspend(struct device *dev)
{
return __pm_runtime_suspend(dev, 0);
}
/**
* pm_runtime_autosuspend - Set up autosuspend of a device or suspend it.
* @dev: Target device.
*
* Set up autosuspend of @dev or suspend it (depending on whether or not
* autosuspend is enabled for it) without engaging its "idle check" callback.
*/
static inline int pm_runtime_autosuspend(struct device *dev)
{
return __pm_runtime_suspend(dev, RPM_AUTO);
}
/**
* pm_runtime_resume - Resume a device synchronously.
* @dev: Target device.
*/
static inline int pm_runtime_resume(struct device *dev)
{
return __pm_runtime_resume(dev, 0);
}
/**
* pm_request_idle - Queue up "idle check" execution for a device.
* @dev: Target device.
*
* Queue up a work item to run an equivalent of pm_runtime_idle() for @dev
* asynchronously.
*/
static inline int pm_request_idle(struct device *dev)
{
return __pm_runtime_idle(dev, RPM_ASYNC);
}
/**
* pm_request_resume - Queue up runtime-resume of a device.
* @dev: Target device.
*/
static inline int pm_request_resume(struct device *dev)
{
return __pm_runtime_resume(dev, RPM_ASYNC);
}
/**
* pm_request_autosuspend - Queue up autosuspend of a device.
* @dev: Target device.
*
* Queue up a work item to run an equivalent pm_runtime_autosuspend() for @dev
* asynchronously.
*/
static inline int pm_request_autosuspend(struct device *dev)
{
return __pm_runtime_suspend(dev, RPM_ASYNC | RPM_AUTO);
}
/**
* pm_runtime_get - Bump up usage counter and queue up resume of a device.
* @dev: Target device.
*
* Bump up the runtime PM usage counter of @dev and queue up a work item to
* carry out runtime-resume of it.
*/
static inline int pm_runtime_get(struct device *dev)
{
return __pm_runtime_resume(dev, RPM_GET_PUT | RPM_ASYNC);
}
/**
* pm_runtime_get_sync - Bump up usage counter of a device and resume it.
* @dev: Target device.
*
* Bump up the runtime PM usage counter of @dev and carry out runtime-resume of
* it synchronously.
*
* The possible return values of this function are the same as for
* pm_runtime_resume() and the runtime PM usage counter of @dev remains
* incremented in all cases, even if it returns an error code.
* Consider using pm_runtime_resume_and_get() instead of it, especially
* if its return value is checked by the caller, as this is likely to result
* in cleaner code.
*/
static inline int pm_runtime_get_sync(struct device *dev)
{
return __pm_runtime_resume(dev, RPM_GET_PUT);
}
/**
* pm_runtime_resume_and_get - Bump up usage counter of a device and resume it.
* @dev: Target device.
*
* Resume @dev synchronously and if that is successful, increment its runtime
* PM usage counter. Return 0 if the runtime PM usage counter of @dev has been
* incremented or a negative error code otherwise.
*/
static inline int pm_runtime_resume_and_get(struct device *dev)
{
int ret;
ret = __pm_runtime_resume(dev, RPM_GET_PUT);
if (ret < 0) {
pm_runtime_put_noidle(dev);
return ret;
}
return 0;
}
/**
* pm_runtime_put - Drop device usage counter and queue up "idle check" if 0.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev and if it turns out to be
* equal to 0, queue up a work item for @dev like in pm_request_idle().
*/
static inline int pm_runtime_put(struct device *dev)
{
return __pm_runtime_idle(dev, RPM_GET_PUT | RPM_ASYNC);
}
/**
* __pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev and if it turns out to be
* equal to 0, queue up a work item for @dev like in pm_request_autosuspend().
*/
static inline int __pm_runtime_put_autosuspend(struct device *dev)
{
return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_ASYNC | RPM_AUTO);
}
/**
* pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev and if it turns out to be
* equal to 0, queue up a work item for @dev like in pm_request_autosuspend().
*/
static inline int pm_runtime_put_autosuspend(struct device *dev)
{
return __pm_runtime_suspend(dev,
RPM_GET_PUT | RPM_ASYNC | RPM_AUTO);
}
/**
* pm_runtime_put_sync - Drop device usage counter and run "idle check" if 0.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev and if it turns out to be
* equal to 0, invoke the "idle check" callback of @dev and, depending on its
* return value, set up autosuspend of @dev or suspend it (depending on whether
* or not autosuspend has been enabled for it).
*
* The possible return values of this function are the same as for
* pm_runtime_idle() and the runtime PM usage counter of @dev remains
* decremented in all cases, even if it returns an error code.
*/
static inline int pm_runtime_put_sync(struct device *dev)
{
return __pm_runtime_idle(dev, RPM_GET_PUT);
}
/**
* pm_runtime_put_sync_suspend - Drop device usage counter and suspend if 0.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev and if it turns out to be
* equal to 0, carry out runtime-suspend of @dev synchronously.
*
* The possible return values of this function are the same as for
* pm_runtime_suspend() and the runtime PM usage counter of @dev remains
* decremented in all cases, even if it returns an error code.
*/
static inline int pm_runtime_put_sync_suspend(struct device *dev)
{
return __pm_runtime_suspend(dev, RPM_GET_PUT);
}
/**
* pm_runtime_put_sync_autosuspend - Drop device usage counter and autosuspend if 0.
* @dev: Target device.
*
* Decrement the runtime PM usage counter of @dev and if it turns out to be
* equal to 0, set up autosuspend of @dev or suspend it synchronously (depending
* on whether or not autosuspend has been enabled for it).
*
* The possible return values of this function are the same as for
* pm_runtime_autosuspend() and the runtime PM usage counter of @dev remains
* decremented in all cases, even if it returns an error code.
*/
static inline int pm_runtime_put_sync_autosuspend(struct device *dev)
{
return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_AUTO);
}
/**
* pm_runtime_set_active - Set runtime PM status to "active".
* @dev: Target device.
*
* Set the runtime PM status of @dev to %RPM_ACTIVE and ensure that dependencies
* of it will be taken into account.
*
* It is not valid to call this function for devices with runtime PM enabled.
*/
static inline int pm_runtime_set_active(struct device *dev)
{
return __pm_runtime_set_status(dev, RPM_ACTIVE);
}
/**
* pm_runtime_set_suspended - Set runtime PM status to "suspended".
* @dev: Target device.
*
* Set the runtime PM status of @dev to %RPM_SUSPENDED and ensure that
* dependencies of it will be taken into account.
*
* It is not valid to call this function for devices with runtime PM enabled.
*/
static inline int pm_runtime_set_suspended(struct device *dev)
{
return __pm_runtime_set_status(dev, RPM_SUSPENDED);
}
/**
* pm_runtime_disable - Disable runtime PM for a device.
* @dev: Target device.
*
* Prevent the runtime PM framework from working with @dev (by incrementing its
* "blocking" counter).
*
* For each invocation of this function for @dev there must be a matching
* pm_runtime_enable() call in order for runtime PM to be enabled for it.
*/
static inline void pm_runtime_disable(struct device *dev)
{
__pm_runtime_disable(dev, true);
}
/**
* pm_runtime_use_autosuspend - Allow autosuspend to be used for a device.
* @dev: Target device.
*
* Allow the runtime PM autosuspend mechanism to be used for @dev whenever
* requested (or "autosuspend" will be handled as direct runtime-suspend for
* it).
*
* NOTE: It's important to undo this with pm_runtime_dont_use_autosuspend()
* at driver exit time unless your driver initially enabled pm_runtime
* with devm_pm_runtime_enable() (which handles it for you).
*/
static inline void pm_runtime_use_autosuspend(struct device *dev)
{
__pm_runtime_use_autosuspend(dev, true);
}
/**
* pm_runtime_dont_use_autosuspend - Prevent autosuspend from being used.
* @dev: Target device.
*
* Prevent the runtime PM autosuspend mechanism from being used for @dev which
* means that "autosuspend" will be handled as direct runtime-suspend for it
* going forward.
*/
static inline void pm_runtime_dont_use_autosuspend(struct device *dev)
{
__pm_runtime_use_autosuspend(dev, false);
}
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* fs/kernfs/dir.c - kernfs directory implementation
*
* Copyright (c) 2001-3 Patrick Mochel
* Copyright (c) 2007 SUSE Linux Products GmbH
* Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
*/
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/idr.h>
#include <linux/slab.h>
#include <linux/security.h>
#include <linux/hash.h>
#include "kernfs-internal.h"
static DEFINE_RWLOCK(kernfs_rename_lock); /* kn->parent and ->name */
/*
* Don't use rename_lock to piggy back on pr_cont_buf. We don't want to
* call pr_cont() while holding rename_lock. Because sometimes pr_cont()
* will perform wakeups when releasing console_sem. Holding rename_lock
* will introduce deadlock if the scheduler reads the kernfs_name in the
* wakeup path.
*/
static DEFINE_SPINLOCK(kernfs_pr_cont_lock);
static char kernfs_pr_cont_buf[PATH_MAX]; /* protected by pr_cont_lock */
static DEFINE_SPINLOCK(kernfs_idr_lock); /* root->ino_idr */
#define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb)
static bool __kernfs_active(struct kernfs_node *kn)
{
return atomic_read(&kn->active) >= 0;
}
static bool kernfs_active(struct kernfs_node *kn)
{
lockdep_assert_held(&kernfs_root(kn)->kernfs_rwsem); return __kernfs_active(kn);
}
static bool kernfs_lockdep(struct kernfs_node *kn)
{
#ifdef CONFIG_DEBUG_LOCK_ALLOC
return kn->flags & KERNFS_LOCKDEP;
#else
return false;
#endif
}
static int kernfs_name_locked(struct kernfs_node *kn, char *buf, size_t buflen)
{
if (!kn)
return strscpy(buf, "(null)", buflen);
return strscpy(buf, kn->parent ? kn->name : "/", buflen);
}
/* kernfs_node_depth - compute depth from @from to @to */
static size_t kernfs_depth(struct kernfs_node *from, struct kernfs_node *to)
{
size_t depth = 0;
while (to->parent && to != from) {
depth++;
to = to->parent;
}
return depth;
}
static struct kernfs_node *kernfs_common_ancestor(struct kernfs_node *a,
struct kernfs_node *b)
{
size_t da, db;
struct kernfs_root *ra = kernfs_root(a), *rb = kernfs_root(b);
if (ra != rb)
return NULL;
da = kernfs_depth(ra->kn, a);
db = kernfs_depth(rb->kn, b);
while (da > db) {
a = a->parent;
da--;
}
while (db > da) {
b = b->parent;
db--;
}
/* worst case b and a will be the same at root */
while (b != a) {
b = b->parent;
a = a->parent;
}
return a;
}
/**
* kernfs_path_from_node_locked - find a pseudo-absolute path to @kn_to,
* where kn_from is treated as root of the path.
* @kn_from: kernfs node which should be treated as root for the path
* @kn_to: kernfs node to which path is needed
* @buf: buffer to copy the path into
* @buflen: size of @buf
*
* We need to handle couple of scenarios here:
* [1] when @kn_from is an ancestor of @kn_to at some level
* kn_from: /n1/n2/n3
* kn_to: /n1/n2/n3/n4/n5
* result: /n4/n5
*
* [2] when @kn_from is on a different hierarchy and we need to find common
* ancestor between @kn_from and @kn_to.
* kn_from: /n1/n2/n3/n4
* kn_to: /n1/n2/n5
* result: /../../n5
* OR
* kn_from: /n1/n2/n3/n4/n5 [depth=5]
* kn_to: /n1/n2/n3 [depth=3]
* result: /../..
*
* [3] when @kn_to is %NULL result will be "(null)"
*
* Return: the length of the constructed path. If the path would have been
* greater than @buflen, @buf contains the truncated path with the trailing
* '\0'. On error, -errno is returned.
*/
static int kernfs_path_from_node_locked(struct kernfs_node *kn_to,
struct kernfs_node *kn_from,
char *buf, size_t buflen)
{
struct kernfs_node *kn, *common;
const char parent_str[] = "/..";
size_t depth_from, depth_to, len = 0;
ssize_t copied;
int i, j;
if (!kn_to)
return strscpy(buf, "(null)", buflen);
if (!kn_from)
kn_from = kernfs_root(kn_to)->kn;
if (kn_from == kn_to)
return strscpy(buf, "/", buflen);
common = kernfs_common_ancestor(kn_from, kn_to);
if (WARN_ON(!common))
return -EINVAL;
depth_to = kernfs_depth(common, kn_to);
depth_from = kernfs_depth(common, kn_from);
buf[0] = '\0';
for (i = 0; i < depth_from; i++) {
copied = strscpy(buf + len, parent_str, buflen - len);
if (copied < 0)
return copied;
len += copied;
}
/* Calculate how many bytes we need for the rest */
for (i = depth_to - 1; i >= 0; i--) {
for (kn = kn_to, j = 0; j < i; j++)
kn = kn->parent;
len += scnprintf(buf + len, buflen - len, "/%s", kn->name);
}
return len;
}
/**
* kernfs_name - obtain the name of a given node
* @kn: kernfs_node of interest
* @buf: buffer to copy @kn's name into
* @buflen: size of @buf
*
* Copies the name of @kn into @buf of @buflen bytes. The behavior is
* similar to strscpy().
*
* Fills buffer with "(null)" if @kn is %NULL.
*
* Return: the resulting length of @buf. If @buf isn't long enough,
* it's filled up to @buflen-1 and nul terminated, and returns -E2BIG.
*
* This function can be called from any context.
*/
int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)
{
unsigned long flags;
int ret;
read_lock_irqsave(&kernfs_rename_lock, flags);
ret = kernfs_name_locked(kn, buf, buflen);
read_unlock_irqrestore(&kernfs_rename_lock, flags);
return ret;
}
/**
* kernfs_path_from_node - build path of node @to relative to @from.
* @from: parent kernfs_node relative to which we need to build the path
* @to: kernfs_node of interest
* @buf: buffer to copy @to's path into
* @buflen: size of @buf
*
* Builds @to's path relative to @from in @buf. @from and @to must
* be on the same kernfs-root. If @from is not parent of @to, then a relative
* path (which includes '..'s) as needed to reach from @from to @to is
* returned.
*
* Return: the length of the constructed path. If the path would have been
* greater than @buflen, @buf contains the truncated path with the trailing
* '\0'. On error, -errno is returned.
*/
int kernfs_path_from_node(struct kernfs_node *to, struct kernfs_node *from,
char *buf, size_t buflen)
{
unsigned long flags;
int ret;
read_lock_irqsave(&kernfs_rename_lock, flags);
ret = kernfs_path_from_node_locked(to, from, buf, buflen);
read_unlock_irqrestore(&kernfs_rename_lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(kernfs_path_from_node);
/**
* pr_cont_kernfs_name - pr_cont name of a kernfs_node
* @kn: kernfs_node of interest
*
* This function can be called from any context.
*/
void pr_cont_kernfs_name(struct kernfs_node *kn)
{
unsigned long flags;
spin_lock_irqsave(&kernfs_pr_cont_lock, flags);
kernfs_name(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf));
pr_cont("%s", kernfs_pr_cont_buf);
spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags);
}
/**
* pr_cont_kernfs_path - pr_cont path of a kernfs_node
* @kn: kernfs_node of interest
*
* This function can be called from any context.
*/
void pr_cont_kernfs_path(struct kernfs_node *kn)
{
unsigned long flags;
int sz;
spin_lock_irqsave(&kernfs_pr_cont_lock, flags);
sz = kernfs_path_from_node(kn, NULL, kernfs_pr_cont_buf,
sizeof(kernfs_pr_cont_buf));
if (sz < 0) {
if (sz == -E2BIG)
pr_cont("(name too long)");
else
pr_cont("(error)");
goto out;
}
pr_cont("%s", kernfs_pr_cont_buf);
out:
spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags);
}
/**
* kernfs_get_parent - determine the parent node and pin it
* @kn: kernfs_node of interest
*
* Determines @kn's parent, pins and returns it. This function can be
* called from any context.
*
* Return: parent node of @kn
*/
struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)
{
struct kernfs_node *parent;
unsigned long flags;
read_lock_irqsave(&kernfs_rename_lock, flags);
parent = kn->parent;
kernfs_get(parent);
read_unlock_irqrestore(&kernfs_rename_lock, flags);
return parent;
}
/**
* kernfs_name_hash - calculate hash of @ns + @name
* @name: Null terminated string to hash
* @ns: Namespace tag to hash
*
* Return: 31-bit hash of ns + name (so it fits in an off_t)
*/
static unsigned int kernfs_name_hash(const char *name, const void *ns)
{
unsigned long hash = init_name_hash(ns);
unsigned int len = strlen(name);
while (len--)
hash = partial_name_hash(*name++, hash); hash = end_name_hash(hash);
hash &= 0x7fffffffU;
/* Reserve hash numbers 0, 1 and INT_MAX for magic directory entries */
if (hash < 2)
hash += 2; if (hash >= INT_MAX)
hash = INT_MAX - 1;
return hash;
}
static int kernfs_name_compare(unsigned int hash, const char *name,
const void *ns, const struct kernfs_node *kn)
{
if (hash < kn->hash)
return -1;
if (hash > kn->hash)
return 1;
if (ns < kn->ns)
return -1;
if (ns > kn->ns)
return 1;
return strcmp(name, kn->name);
}
static int kernfs_sd_compare(const struct kernfs_node *left,
const struct kernfs_node *right)
{
return kernfs_name_compare(left->hash, left->name, left->ns, right);
}
/**
* kernfs_link_sibling - link kernfs_node into sibling rbtree
* @kn: kernfs_node of interest
*
* Link @kn into its sibling rbtree which starts from
* @kn->parent->dir.children.
*
* Locking:
* kernfs_rwsem held exclusive
*
* Return:
* %0 on success, -EEXIST on failure.
*/
static int kernfs_link_sibling(struct kernfs_node *kn)
{
struct rb_node **node = &kn->parent->dir.children.rb_node;
struct rb_node *parent = NULL;
while (*node) {
struct kernfs_node *pos;
int result;
pos = rb_to_kn(*node);
parent = *node;
result = kernfs_sd_compare(kn, pos);
if (result < 0)
node = &pos->rb.rb_left; else if (result > 0) node = &pos->rb.rb_right;
else
return -EEXIST;
}
/* add new node and rebalance the tree */
rb_link_node(&kn->rb, parent, node);
rb_insert_color(&kn->rb, &kn->parent->dir.children);
/* successfully added, account subdir number */
down_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
if (kernfs_type(kn) == KERNFS_DIR)
kn->parent->dir.subdirs++; kernfs_inc_rev(kn->parent); up_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
return 0;
}
/**
* kernfs_unlink_sibling - unlink kernfs_node from sibling rbtree
* @kn: kernfs_node of interest
*
* Try to unlink @kn from its sibling rbtree which starts from
* kn->parent->dir.children.
*
* Return: %true if @kn was actually removed,
* %false if @kn wasn't on the rbtree.
*
* Locking:
* kernfs_rwsem held exclusive
*/
static bool kernfs_unlink_sibling(struct kernfs_node *kn)
{
if (RB_EMPTY_NODE(&kn->rb))
return false;
down_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
if (kernfs_type(kn) == KERNFS_DIR)
kn->parent->dir.subdirs--; kernfs_inc_rev(kn->parent); up_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
rb_erase(&kn->rb, &kn->parent->dir.children);
RB_CLEAR_NODE(&kn->rb);
return true;
}
/**
* kernfs_get_active - get an active reference to kernfs_node
* @kn: kernfs_node to get an active reference to
*
* Get an active reference of @kn. This function is noop if @kn
* is %NULL.
*
* Return:
* Pointer to @kn on success, %NULL on failure.
*/
struct kernfs_node *kernfs_get_active(struct kernfs_node *kn)
{
if (unlikely(!kn))
return NULL;
if (!atomic_inc_unless_negative(&kn->active))
return NULL;
if (kernfs_lockdep(kn))
rwsem_acquire_read(&kn->dep_map, 0, 1, _RET_IP_);
return kn;
}
/**
* kernfs_put_active - put an active reference to kernfs_node
* @kn: kernfs_node to put an active reference to
*
* Put an active reference to @kn. This function is noop if @kn
* is %NULL.
*/
void kernfs_put_active(struct kernfs_node *kn)
{
int v;
if (unlikely(!kn))
return;
if (kernfs_lockdep(kn))
rwsem_release(&kn->dep_map, _RET_IP_);
v = atomic_dec_return(&kn->active);
if (likely(v != KN_DEACTIVATED_BIAS))
return;
wake_up_all(&kernfs_root(kn)->deactivate_waitq);
}
/**
* kernfs_drain - drain kernfs_node
* @kn: kernfs_node to drain
*
* Drain existing usages and nuke all existing mmaps of @kn. Multiple
* removers may invoke this function concurrently on @kn and all will
* return after draining is complete.
*/
static void kernfs_drain(struct kernfs_node *kn)
__releases(&kernfs_root(kn)->kernfs_rwsem)
__acquires(&kernfs_root(kn)->kernfs_rwsem)
{
struct kernfs_root *root = kernfs_root(kn); lockdep_assert_held_write(&root->kernfs_rwsem); WARN_ON_ONCE(kernfs_active(kn));
/*
* Skip draining if already fully drained. This avoids draining and its
* lockdep annotations for nodes which have never been activated
* allowing embedding kernfs_remove() in create error paths without
* worrying about draining.
*/
if (atomic_read(&kn->active) == KN_DEACTIVATED_BIAS && !kernfs_should_drain_open_files(kn))
return;
up_write(&root->kernfs_rwsem);
if (kernfs_lockdep(kn)) {
rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_);
if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS)
lock_contended(&kn->dep_map, _RET_IP_);
}
wait_event(root->deactivate_waitq,
atomic_read(&kn->active) == KN_DEACTIVATED_BIAS);
if (kernfs_lockdep(kn)) {
lock_acquired(&kn->dep_map, _RET_IP_);
rwsem_release(&kn->dep_map, _RET_IP_);
}
if (kernfs_should_drain_open_files(kn)) kernfs_drain_open_files(kn); down_write(&root->kernfs_rwsem);
}
/**
* kernfs_get - get a reference count on a kernfs_node
* @kn: the target kernfs_node
*/
void kernfs_get(struct kernfs_node *kn)
{
if (kn) { WARN_ON(!atomic_read(&kn->count)); atomic_inc(&kn->count);
}
}
EXPORT_SYMBOL_GPL(kernfs_get);
static void kernfs_free_rcu(struct rcu_head *rcu)
{
struct kernfs_node *kn = container_of(rcu, struct kernfs_node, rcu);
kfree_const(kn->name);
if (kn->iattr) {
simple_xattrs_free(&kn->iattr->xattrs, NULL);
kmem_cache_free(kernfs_iattrs_cache, kn->iattr);
}
kmem_cache_free(kernfs_node_cache, kn);
}
/**
* kernfs_put - put a reference count on a kernfs_node
* @kn: the target kernfs_node
*
* Put a reference count of @kn and destroy it if it reached zero.
*/
void kernfs_put(struct kernfs_node *kn)
{
struct kernfs_node *parent;
struct kernfs_root *root;
if (!kn || !atomic_dec_and_test(&kn->count))
return;
root = kernfs_root(kn);
repeat:
/*
* Moving/renaming is always done while holding reference.
* kn->parent won't change beneath us.
*/
parent = kn->parent; WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS,
"kernfs_put: %s/%s: released with incorrect active_ref %d\n",
parent ? parent->name : "", kn->name, atomic_read(&kn->active));
if (kernfs_type(kn) == KERNFS_LINK) kernfs_put(kn->symlink.target_kn); spin_lock(&kernfs_idr_lock);
idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));
spin_unlock(&kernfs_idr_lock);
call_rcu(&kn->rcu, kernfs_free_rcu);
kn = parent;
if (kn) {
if (atomic_dec_and_test(&kn->count))
goto repeat;
} else {
/* just released the root kn, free @root too */
idr_destroy(&root->ino_idr); kfree_rcu(root, rcu);
}
}
EXPORT_SYMBOL_GPL(kernfs_put);
/**
* kernfs_node_from_dentry - determine kernfs_node associated with a dentry
* @dentry: the dentry in question
*
* Return: the kernfs_node associated with @dentry. If @dentry is not a
* kernfs one, %NULL is returned.
*
* While the returned kernfs_node will stay accessible as long as @dentry
* is accessible, the returned node can be in any state and the caller is
* fully responsible for determining what's accessible.
*/
struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry)
{
if (dentry->d_sb->s_op == &kernfs_sops)
return kernfs_dentry_node(dentry);
return NULL;
}
static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root,
struct kernfs_node *parent,
const char *name, umode_t mode,
kuid_t uid, kgid_t gid,
unsigned flags)
{
struct kernfs_node *kn;
u32 id_highbits;
int ret;
name = kstrdup_const(name, GFP_KERNEL);
if (!name)
return NULL;
kn = kmem_cache_zalloc(kernfs_node_cache, GFP_KERNEL);
if (!kn)
goto err_out1;
idr_preload(GFP_KERNEL);
spin_lock(&kernfs_idr_lock);
ret = idr_alloc_cyclic(&root->ino_idr, kn, 1, 0, GFP_ATOMIC);
if (ret >= 0 && ret < root->last_id_lowbits) root->id_highbits++;
id_highbits = root->id_highbits;
root->last_id_lowbits = ret;
spin_unlock(&kernfs_idr_lock);
idr_preload_end(); if (ret < 0)
goto err_out2;
kn->id = (u64)id_highbits << 32 | ret;
atomic_set(&kn->count, 1);
atomic_set(&kn->active, KN_DEACTIVATED_BIAS);
RB_CLEAR_NODE(&kn->rb);
kn->name = name;
kn->mode = mode;
kn->flags = flags;
if (!uid_eq(uid, GLOBAL_ROOT_UID) || !gid_eq(gid, GLOBAL_ROOT_GID)) {
struct iattr iattr = {
.ia_valid = ATTR_UID | ATTR_GID,
.ia_uid = uid,
.ia_gid = gid,
};
ret = __kernfs_setattr(kn, &iattr);
if (ret < 0) goto err_out3;
}
if (parent) { ret = security_kernfs_init_security(parent, kn); if (ret)
goto err_out3;
}
return kn;
err_out3:
spin_lock(&kernfs_idr_lock);
idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));
spin_unlock(&kernfs_idr_lock);
err_out2:
kmem_cache_free(kernfs_node_cache, kn);
err_out1:
kfree_const(name);
return NULL;
}
struct kernfs_node *kernfs_new_node(struct kernfs_node *parent,
const char *name, umode_t mode,
kuid_t uid, kgid_t gid,
unsigned flags)
{
struct kernfs_node *kn;
if (parent->mode & S_ISGID) {
/* this code block imitates inode_init_owner() for
* kernfs
*/
if (parent->iattr) gid = parent->iattr->ia_gid; if (flags & KERNFS_DIR) mode |= S_ISGID;
}
kn = __kernfs_new_node(kernfs_root(parent), parent,
name, mode, uid, gid, flags);
if (kn) {
kernfs_get(parent); kn->parent = parent;
}
return kn;
}
/*
* kernfs_find_and_get_node_by_id - get kernfs_node from node id
* @root: the kernfs root
* @id: the target node id
*
* @id's lower 32bits encode ino and upper gen. If the gen portion is
* zero, all generations are matched.
*
* Return: %NULL on failure,
* otherwise a kernfs node with reference counter incremented.
*/
struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root,
u64 id)
{
struct kernfs_node *kn;
ino_t ino = kernfs_id_ino(id);
u32 gen = kernfs_id_gen(id);
rcu_read_lock();
kn = idr_find(&root->ino_idr, (u32)ino);
if (!kn)
goto err_unlock;
if (sizeof(ino_t) >= sizeof(u64)) {
/* we looked up with the low 32bits, compare the whole */
if (kernfs_ino(kn) != ino)
goto err_unlock;
} else {
/* 0 matches all generations */
if (unlikely(gen && kernfs_gen(kn) != gen))
goto err_unlock;
}
/*
* We should fail if @kn has never been activated and guarantee success
* if the caller knows that @kn is active. Both can be achieved by
* __kernfs_active() which tests @kn->active without kernfs_rwsem.
*/
if (unlikely(!__kernfs_active(kn) || !atomic_inc_not_zero(&kn->count)))
goto err_unlock;
rcu_read_unlock();
return kn;
err_unlock:
rcu_read_unlock();
return NULL;
}
/**
* kernfs_add_one - add kernfs_node to parent without warning
* @kn: kernfs_node to be added
*
* The caller must already have initialized @kn->parent. This
* function increments nlink of the parent's inode if @kn is a
* directory and link into the children list of the parent.
*
* Return:
* %0 on success, -EEXIST if entry with the given name already
* exists.
*/
int kernfs_add_one(struct kernfs_node *kn)
{
struct kernfs_node *parent = kn->parent; struct kernfs_root *root = kernfs_root(parent);
struct kernfs_iattrs *ps_iattr;
bool has_ns;
int ret;
down_write(&root->kernfs_rwsem);
ret = -EINVAL;
has_ns = kernfs_ns_enabled(parent);
if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",
has_ns ? "required" : "invalid", parent->name, kn->name))
goto out_unlock;
if (kernfs_type(parent) != KERNFS_DIR)
goto out_unlock;
ret = -ENOENT;
if (parent->flags & (KERNFS_REMOVING | KERNFS_EMPTY_DIR))
goto out_unlock;
kn->hash = kernfs_name_hash(kn->name, kn->ns);
ret = kernfs_link_sibling(kn);
if (ret)
goto out_unlock;
/* Update timestamps on the parent */
down_write(&root->kernfs_iattr_rwsem);
ps_iattr = parent->iattr;
if (ps_iattr) {
ktime_get_real_ts64(&ps_iattr->ia_ctime);
ps_iattr->ia_mtime = ps_iattr->ia_ctime;
}
up_write(&root->kernfs_iattr_rwsem);
up_write(&root->kernfs_rwsem);
/*
* Activate the new node unless CREATE_DEACTIVATED is requested.
* If not activated here, the kernfs user is responsible for
* activating the node with kernfs_activate(). A node which hasn't
* been activated is not visible to userland and its removal won't
* trigger deactivation.
*/
if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED)) kernfs_activate(kn);
return 0;
out_unlock:
up_write(&root->kernfs_rwsem); return ret;
}
/**
* kernfs_find_ns - find kernfs_node with the given name
* @parent: kernfs_node to search under
* @name: name to look for
* @ns: the namespace tag to use
*
* Look for kernfs_node with name @name under @parent.
*
* Return: pointer to the found kernfs_node on success, %NULL on failure.
*/
static struct kernfs_node *kernfs_find_ns(struct kernfs_node *parent,
const unsigned char *name,
const void *ns)
{
struct rb_node *node = parent->dir.children.rb_node;
bool has_ns = kernfs_ns_enabled(parent);
unsigned int hash;
lockdep_assert_held(&kernfs_root(parent)->kernfs_rwsem); if (has_ns != (bool)ns) { WARN(1, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",
has_ns ? "required" : "invalid", parent->name, name);
return NULL;
}
hash = kernfs_name_hash(name, ns); while (node) {
struct kernfs_node *kn;
int result;
kn = rb_to_kn(node); result = kernfs_name_compare(hash, name, ns, kn);
if (result < 0)
node = node->rb_left; else if (result > 0) node = node->rb_right;
else
return kn;
}
return NULL;
}
static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent,
const unsigned char *path,
const void *ns)
{
ssize_t len;
char *p, *name;
lockdep_assert_held_read(&kernfs_root(parent)->kernfs_rwsem);
spin_lock_irq(&kernfs_pr_cont_lock);
len = strscpy(kernfs_pr_cont_buf, path, sizeof(kernfs_pr_cont_buf));
if (len < 0) {
spin_unlock_irq(&kernfs_pr_cont_lock);
return NULL;
}
p = kernfs_pr_cont_buf;
while ((name = strsep(&p, "/")) && parent) {
if (*name == '\0')
continue;
parent = kernfs_find_ns(parent, name, ns);
}
spin_unlock_irq(&kernfs_pr_cont_lock);
return parent;
}
/**
* kernfs_find_and_get_ns - find and get kernfs_node with the given name
* @parent: kernfs_node to search under
* @name: name to look for
* @ns: the namespace tag to use
*
* Look for kernfs_node with name @name under @parent and get a reference
* if found. This function may sleep.
*
* Return: pointer to the found kernfs_node on success, %NULL on failure.
*/
struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent,
const char *name, const void *ns)
{
struct kernfs_node *kn;
struct kernfs_root *root = kernfs_root(parent);
down_read(&root->kernfs_rwsem);
kn = kernfs_find_ns(parent, name, ns);
kernfs_get(kn); up_read(&root->kernfs_rwsem);
return kn;
}
EXPORT_SYMBOL_GPL(kernfs_find_and_get_ns);
/**
* kernfs_walk_and_get_ns - find and get kernfs_node with the given path
* @parent: kernfs_node to search under
* @path: path to look for
* @ns: the namespace tag to use
*
* Look for kernfs_node with path @path under @parent and get a reference
* if found. This function may sleep.
*
* Return: pointer to the found kernfs_node on success, %NULL on failure.
*/
struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent,
const char *path, const void *ns)
{
struct kernfs_node *kn;
struct kernfs_root *root = kernfs_root(parent);
down_read(&root->kernfs_rwsem);
kn = kernfs_walk_ns(parent, path, ns);
kernfs_get(kn);
up_read(&root->kernfs_rwsem);
return kn;
}
/**
* kernfs_create_root - create a new kernfs hierarchy
* @scops: optional syscall operations for the hierarchy
* @flags: KERNFS_ROOT_* flags
* @priv: opaque data associated with the new directory
*
* Return: the root of the new hierarchy on success, ERR_PTR() value on
* failure.
*/
struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
unsigned int flags, void *priv)
{
struct kernfs_root *root;
struct kernfs_node *kn;
root = kzalloc(sizeof(*root), GFP_KERNEL);
if (!root)
return ERR_PTR(-ENOMEM);
idr_init(&root->ino_idr);
init_rwsem(&root->kernfs_rwsem);
init_rwsem(&root->kernfs_iattr_rwsem);
init_rwsem(&root->kernfs_supers_rwsem);
INIT_LIST_HEAD(&root->supers);
/*
* On 64bit ino setups, id is ino. On 32bit, low 32bits are ino.
* High bits generation. The starting value for both ino and
* genenration is 1. Initialize upper 32bit allocation
* accordingly.
*/
if (sizeof(ino_t) >= sizeof(u64))
root->id_highbits = 0;
else
root->id_highbits = 1;
kn = __kernfs_new_node(root, NULL, "", S_IFDIR | S_IRUGO | S_IXUGO,
GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
KERNFS_DIR);
if (!kn) {
idr_destroy(&root->ino_idr);
kfree(root);
return ERR_PTR(-ENOMEM);
}
kn->priv = priv;
kn->dir.root = root;
root->syscall_ops = scops;
root->flags = flags;
root->kn = kn;
init_waitqueue_head(&root->deactivate_waitq);
if (!(root->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
kernfs_activate(kn);
return root;
}
/**
* kernfs_destroy_root - destroy a kernfs hierarchy
* @root: root of the hierarchy to destroy
*
* Destroy the hierarchy anchored at @root by removing all existing
* directories and destroying @root.
*/
void kernfs_destroy_root(struct kernfs_root *root)
{
/*
* kernfs_remove holds kernfs_rwsem from the root so the root
* shouldn't be freed during the operation.
*/
kernfs_get(root->kn);
kernfs_remove(root->kn);
kernfs_put(root->kn); /* will also free @root */
}
/**
* kernfs_root_to_node - return the kernfs_node associated with a kernfs_root
* @root: root to use to lookup
*
* Return: @root's kernfs_node
*/
struct kernfs_node *kernfs_root_to_node(struct kernfs_root *root)
{
return root->kn;
}
/**
* kernfs_create_dir_ns - create a directory
* @parent: parent in which to create a new directory
* @name: name of the new directory
* @mode: mode of the new directory
* @uid: uid of the new directory
* @gid: gid of the new directory
* @priv: opaque data associated with the new directory
* @ns: optional namespace tag of the directory
*
* Return: the created node on success, ERR_PTR() value on failure.
*/
struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent,
const char *name, umode_t mode,
kuid_t uid, kgid_t gid,
void *priv, const void *ns)
{
struct kernfs_node *kn;
int rc;
/* allocate */
kn = kernfs_new_node(parent, name, mode | S_IFDIR,
uid, gid, KERNFS_DIR);
if (!kn)
return ERR_PTR(-ENOMEM);
kn->dir.root = parent->dir.root;
kn->ns = ns;
kn->priv = priv;
/* link in */
rc = kernfs_add_one(kn);
if (!rc)
return kn;
kernfs_put(kn);
return ERR_PTR(rc);
}
/**
* kernfs_create_empty_dir - create an always empty directory
* @parent: parent in which to create a new directory
* @name: name of the new directory
*
* Return: the created node on success, ERR_PTR() value on failure.
*/
struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
const char *name)
{
struct kernfs_node *kn;
int rc;
/* allocate */
kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR,
GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, KERNFS_DIR);
if (!kn)
return ERR_PTR(-ENOMEM);
kn->flags |= KERNFS_EMPTY_DIR;
kn->dir.root = parent->dir.root;
kn->ns = NULL;
kn->priv = NULL;
/* link in */
rc = kernfs_add_one(kn);
if (!rc)
return kn;
kernfs_put(kn);
return ERR_PTR(rc);
}
static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags)
{
struct kernfs_node *kn;
struct kernfs_root *root;
if (flags & LOOKUP_RCU)
return -ECHILD;
/* Negative hashed dentry? */
if (d_really_is_negative(dentry)) {
struct kernfs_node *parent;
/* If the kernfs parent node has changed discard and
* proceed to ->lookup.
*
* There's nothing special needed here when getting the
* dentry parent, even if a concurrent rename is in
* progress. That's because the dentry is negative so
* it can only be the target of the rename and it will
* be doing a d_move() not a replace. Consequently the
* dentry d_parent won't change over the d_move().
*
* Also kernfs negative dentries transitioning from
* negative to positive during revalidate won't happen
* because they are invalidated on containing directory
* changes and the lookup re-done so that a new positive
* dentry can be properly created.
*/
root = kernfs_root_from_sb(dentry->d_sb);
down_read(&root->kernfs_rwsem);
parent = kernfs_dentry_node(dentry->d_parent);
if (parent) {
if (kernfs_dir_changed(parent, dentry)) {
up_read(&root->kernfs_rwsem);
return 0;
}
}
up_read(&root->kernfs_rwsem);
/* The kernfs parent node hasn't changed, leave the
* dentry negative and return success.
*/
return 1;
}
kn = kernfs_dentry_node(dentry);
root = kernfs_root(kn);
down_read(&root->kernfs_rwsem);
/* The kernfs node has been deactivated */
if (!kernfs_active(kn))
goto out_bad;
/* The kernfs node has been moved? */
if (kernfs_dentry_node(dentry->d_parent) != kn->parent)
goto out_bad;
/* The kernfs node has been renamed */
if (strcmp(dentry->d_name.name, kn->name) != 0)
goto out_bad;
/* The kernfs node has been moved to a different namespace */
if (kn->parent && kernfs_ns_enabled(kn->parent) &&
kernfs_info(dentry->d_sb)->ns != kn->ns)
goto out_bad;
up_read(&root->kernfs_rwsem);
return 1;
out_bad:
up_read(&root->kernfs_rwsem);
return 0;
}
const struct dentry_operations kernfs_dops = {
.d_revalidate = kernfs_dop_revalidate,
};
static struct dentry *kernfs_iop_lookup(struct inode *dir,
struct dentry *dentry,
unsigned int flags)
{
struct kernfs_node *parent = dir->i_private;
struct kernfs_node *kn;
struct kernfs_root *root;
struct inode *inode = NULL;
const void *ns = NULL;
root = kernfs_root(parent);
down_read(&root->kernfs_rwsem);
if (kernfs_ns_enabled(parent))
ns = kernfs_info(dir->i_sb)->ns;
kn = kernfs_find_ns(parent, dentry->d_name.name, ns);
/* attach dentry and inode */
if (kn) {
/* Inactive nodes are invisible to the VFS so don't
* create a negative.
*/
if (!kernfs_active(kn)) {
up_read(&root->kernfs_rwsem);
return NULL;
}
inode = kernfs_get_inode(dir->i_sb, kn);
if (!inode)
inode = ERR_PTR(-ENOMEM);
}
/*
* Needed for negative dentry validation.
* The negative dentry can be created in kernfs_iop_lookup()
* or transforms from positive dentry in dentry_unlink_inode()
* called from vfs_rmdir().
*/
if (!IS_ERR(inode))
kernfs_set_rev(parent, dentry);
up_read(&root->kernfs_rwsem);
/* instantiate and hash (possibly negative) dentry */
return d_splice_alias(inode, dentry);
}
static int kernfs_iop_mkdir(struct mnt_idmap *idmap,
struct inode *dir, struct dentry *dentry,
umode_t mode)
{
struct kernfs_node *parent = dir->i_private;
struct kernfs_syscall_ops *scops = kernfs_root(parent)->syscall_ops;
int ret;
if (!scops || !scops->mkdir)
return -EPERM;
if (!kernfs_get_active(parent))
return -ENODEV;
ret = scops->mkdir(parent, dentry->d_name.name, mode);
kernfs_put_active(parent);
return ret;
}
static int kernfs_iop_rmdir(struct inode *dir, struct dentry *dentry)
{
struct kernfs_node *kn = kernfs_dentry_node(dentry);
struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;
int ret;
if (!scops || !scops->rmdir)
return -EPERM;
if (!kernfs_get_active(kn))
return -ENODEV;
ret = scops->rmdir(kn);
kernfs_put_active(kn);
return ret;
}
static int kernfs_iop_rename(struct mnt_idmap *idmap,
struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry,
unsigned int flags)
{
struct kernfs_node *kn = kernfs_dentry_node(old_dentry);
struct kernfs_node *new_parent = new_dir->i_private;
struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;
int ret;
if (flags)
return -EINVAL;
if (!scops || !scops->rename)
return -EPERM;
if (!kernfs_get_active(kn))
return -ENODEV;
if (!kernfs_get_active(new_parent)) {
kernfs_put_active(kn);
return -ENODEV;
}
ret = scops->rename(kn, new_parent, new_dentry->d_name.name);
kernfs_put_active(new_parent);
kernfs_put_active(kn);
return ret;
}
const struct inode_operations kernfs_dir_iops = {
.lookup = kernfs_iop_lookup,
.permission = kernfs_iop_permission,
.setattr = kernfs_iop_setattr,
.getattr = kernfs_iop_getattr,
.listxattr = kernfs_iop_listxattr,
.mkdir = kernfs_iop_mkdir,
.rmdir = kernfs_iop_rmdir,
.rename = kernfs_iop_rename,
};
static struct kernfs_node *kernfs_leftmost_descendant(struct kernfs_node *pos)
{
struct kernfs_node *last;
while (true) {
struct rb_node *rbn;
last = pos;
if (kernfs_type(pos) != KERNFS_DIR)
break;
rbn = rb_first(&pos->dir.children);
if (!rbn)
break;
pos = rb_to_kn(rbn);
}
return last;
}
/**
* kernfs_next_descendant_post - find the next descendant for post-order walk
* @pos: the current position (%NULL to initiate traversal)
* @root: kernfs_node whose descendants to walk
*
* Find the next descendant to visit for post-order traversal of @root's
* descendants. @root is included in the iteration and the last node to be
* visited.
*
* Return: the next descendant to visit or %NULL when done.
*/
static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos,
struct kernfs_node *root)
{
struct rb_node *rbn;
lockdep_assert_held_write(&kernfs_root(root)->kernfs_rwsem);
/* if first iteration, visit leftmost descendant which may be root */
if (!pos) return kernfs_leftmost_descendant(root);
/* if we visited @root, we're done */
if (pos == root)
return NULL;
/* if there's an unvisited sibling, visit its leftmost descendant */
rbn = rb_next(&pos->rb);
if (rbn)
return kernfs_leftmost_descendant(rb_to_kn(rbn));
/* no sibling left, visit parent */
return pos->parent;
}
static void kernfs_activate_one(struct kernfs_node *kn)
{
lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem); kn->flags |= KERNFS_ACTIVATED; if (kernfs_active(kn) || (kn->flags & (KERNFS_HIDDEN | KERNFS_REMOVING)))
return;
WARN_ON_ONCE(kn->parent && RB_EMPTY_NODE(&kn->rb)); WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); atomic_sub(KN_DEACTIVATED_BIAS, &kn->active);
}
/**
* kernfs_activate - activate a node which started deactivated
* @kn: kernfs_node whose subtree is to be activated
*
* If the root has KERNFS_ROOT_CREATE_DEACTIVATED set, a newly created node
* needs to be explicitly activated. A node which hasn't been activated
* isn't visible to userland and deactivation is skipped during its
* removal. This is useful to construct atomic init sequences where
* creation of multiple nodes should either succeed or fail atomically.
*
* The caller is responsible for ensuring that this function is not called
* after kernfs_remove*() is invoked on @kn.
*/
void kernfs_activate(struct kernfs_node *kn)
{
struct kernfs_node *pos;
struct kernfs_root *root = kernfs_root(kn);
down_write(&root->kernfs_rwsem);
pos = NULL;
while ((pos = kernfs_next_descendant_post(pos, kn))) kernfs_activate_one(pos); up_write(&root->kernfs_rwsem);
}
/**
* kernfs_show - show or hide a node
* @kn: kernfs_node to show or hide
* @show: whether to show or hide
*
* If @show is %false, @kn is marked hidden and deactivated. A hidden node is
* ignored in future activaitons. If %true, the mark is removed and activation
* state is restored. This function won't implicitly activate a new node in a
* %KERNFS_ROOT_CREATE_DEACTIVATED root which hasn't been activated yet.
*
* To avoid recursion complexities, directories aren't supported for now.
*/
void kernfs_show(struct kernfs_node *kn, bool show)
{
struct kernfs_root *root = kernfs_root(kn);
if (WARN_ON_ONCE(kernfs_type(kn) == KERNFS_DIR))
return;
down_write(&root->kernfs_rwsem);
if (show) {
kn->flags &= ~KERNFS_HIDDEN;
if (kn->flags & KERNFS_ACTIVATED)
kernfs_activate_one(kn);
} else {
kn->flags |= KERNFS_HIDDEN;
if (kernfs_active(kn))
atomic_add(KN_DEACTIVATED_BIAS, &kn->active);
kernfs_drain(kn);
}
up_write(&root->kernfs_rwsem);
}
static void __kernfs_remove(struct kernfs_node *kn)
{
struct kernfs_node *pos;
/* Short-circuit if non-root @kn has already finished removal. */
if (!kn)
return;
lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem);
/*
* This is for kernfs_remove_self() which plays with active ref
* after removal.
*/
if (kn->parent && RB_EMPTY_NODE(&kn->rb))
return;
pr_debug("kernfs %s: removing\n", kn->name);
/* prevent new usage by marking all nodes removing and deactivating */
pos = NULL;
while ((pos = kernfs_next_descendant_post(pos, kn))) { pos->flags |= KERNFS_REMOVING;
if (kernfs_active(pos))
atomic_add(KN_DEACTIVATED_BIAS, &pos->active);
}
/* deactivate and unlink the subtree node-by-node */
do {
pos = kernfs_leftmost_descendant(kn);
/*
* kernfs_drain() may drop kernfs_rwsem temporarily and @pos's
* base ref could have been put by someone else by the time
* the function returns. Make sure it doesn't go away
* underneath us.
*/
kernfs_get(pos); kernfs_drain(pos);
/*
* kernfs_unlink_sibling() succeeds once per node. Use it
* to decide who's responsible for cleanups.
*/
if (!pos->parent || kernfs_unlink_sibling(pos)) {
struct kernfs_iattrs *ps_iattr =
pos->parent ? pos->parent->iattr : NULL;
/* update timestamps on the parent */
down_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
if (ps_iattr) {
ktime_get_real_ts64(&ps_iattr->ia_ctime);
ps_iattr->ia_mtime = ps_iattr->ia_ctime;
}
up_write(&kernfs_root(kn)->kernfs_iattr_rwsem);
kernfs_put(pos);
}
kernfs_put(pos);
} while (pos != kn);
}
/**
* kernfs_remove - remove a kernfs_node recursively
* @kn: the kernfs_node to remove
*
* Remove @kn along with all its subdirectories and files.
*/
void kernfs_remove(struct kernfs_node *kn)
{
struct kernfs_root *root;
if (!kn)
return;
root = kernfs_root(kn);
down_write(&root->kernfs_rwsem);
__kernfs_remove(kn);
up_write(&root->kernfs_rwsem);
}
/**
* kernfs_break_active_protection - break out of active protection
* @kn: the self kernfs_node
*
* The caller must be running off of a kernfs operation which is invoked
* with an active reference - e.g. one of kernfs_ops. Each invocation of
* this function must also be matched with an invocation of
* kernfs_unbreak_active_protection().
*
* This function releases the active reference of @kn the caller is
* holding. Once this function is called, @kn may be removed at any point
* and the caller is solely responsible for ensuring that the objects it
* dereferences are accessible.
*/
void kernfs_break_active_protection(struct kernfs_node *kn)
{
/*
* Take out ourself out of the active ref dependency chain. If
* we're called without an active ref, lockdep will complain.
*/
kernfs_put_active(kn);
}
/**
* kernfs_unbreak_active_protection - undo kernfs_break_active_protection()
* @kn: the self kernfs_node
*
* If kernfs_break_active_protection() was called, this function must be
* invoked before finishing the kernfs operation. Note that while this
* function restores the active reference, it doesn't and can't actually
* restore the active protection - @kn may already or be in the process of
* being removed. Once kernfs_break_active_protection() is invoked, that
* protection is irreversibly gone for the kernfs operation instance.
*
* While this function may be called at any point after
* kernfs_break_active_protection() is invoked, its most useful location
* would be right before the enclosing kernfs operation returns.
*/
void kernfs_unbreak_active_protection(struct kernfs_node *kn)
{
/*
* @kn->active could be in any state; however, the increment we do
* here will be undone as soon as the enclosing kernfs operation
* finishes and this temporary bump can't break anything. If @kn
* is alive, nothing changes. If @kn is being deactivated, the
* soon-to-follow put will either finish deactivation or restore
* deactivated state. If @kn is already removed, the temporary
* bump is guaranteed to be gone before @kn is released.
*/
atomic_inc(&kn->active);
if (kernfs_lockdep(kn))
rwsem_acquire(&kn->dep_map, 0, 1, _RET_IP_);
}
/**
* kernfs_remove_self - remove a kernfs_node from its own method
* @kn: the self kernfs_node to remove
*
* The caller must be running off of a kernfs operation which is invoked
* with an active reference - e.g. one of kernfs_ops. This can be used to
* implement a file operation which deletes itself.
*
* For example, the "delete" file for a sysfs device directory can be
* implemented by invoking kernfs_remove_self() on the "delete" file
* itself. This function breaks the circular dependency of trying to
* deactivate self while holding an active ref itself. It isn't necessary
* to modify the usual removal path to use kernfs_remove_self(). The
* "delete" implementation can simply invoke kernfs_remove_self() on self
* before proceeding with the usual removal path. kernfs will ignore later
* kernfs_remove() on self.
*
* kernfs_remove_self() can be called multiple times concurrently on the
* same kernfs_node. Only the first one actually performs removal and
* returns %true. All others will wait until the kernfs operation which
* won self-removal finishes and return %false. Note that the losers wait
* for the completion of not only the winning kernfs_remove_self() but also
* the whole kernfs_ops which won the arbitration. This can be used to
* guarantee, for example, all concurrent writes to a "delete" file to
* finish only after the whole operation is complete.
*
* Return: %true if @kn is removed by this call, otherwise %false.
*/
bool kernfs_remove_self(struct kernfs_node *kn)
{
bool ret;
struct kernfs_root *root = kernfs_root(kn);
down_write(&root->kernfs_rwsem);
kernfs_break_active_protection(kn);
/*
* SUICIDAL is used to arbitrate among competing invocations. Only
* the first one will actually perform removal. When the removal
* is complete, SUICIDED is set and the active ref is restored
* while kernfs_rwsem for held exclusive. The ones which lost
* arbitration waits for SUICIDED && drained which can happen only
* after the enclosing kernfs operation which executed the winning
* instance of kernfs_remove_self() finished.
*/
if (!(kn->flags & KERNFS_SUICIDAL)) {
kn->flags |= KERNFS_SUICIDAL;
__kernfs_remove(kn);
kn->flags |= KERNFS_SUICIDED;
ret = true;
} else {
wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq;
DEFINE_WAIT(wait);
while (true) {
prepare_to_wait(waitq, &wait, TASK_UNINTERRUPTIBLE);
if ((kn->flags & KERNFS_SUICIDED) &&
atomic_read(&kn->active) == KN_DEACTIVATED_BIAS)
break;
up_write(&root->kernfs_rwsem);
schedule();
down_write(&root->kernfs_rwsem);
}
finish_wait(waitq, &wait);
WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb));
ret = false;
}
/*
* This must be done while kernfs_rwsem held exclusive; otherwise,
* waiting for SUICIDED && deactivated could finish prematurely.
*/
kernfs_unbreak_active_protection(kn);
up_write(&root->kernfs_rwsem);
return ret;
}
/**
* kernfs_remove_by_name_ns - find a kernfs_node by name and remove it
* @parent: parent of the target
* @name: name of the kernfs_node to remove
* @ns: namespace tag of the kernfs_node to remove
*
* Look for the kernfs_node with @name and @ns under @parent and remove it.
*
* Return: %0 on success, -ENOENT if such entry doesn't exist.
*/
int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
const void *ns)
{
struct kernfs_node *kn;
struct kernfs_root *root;
if (!parent) { WARN(1, KERN_WARNING "kernfs: can not remove '%s', no directory\n",
name);
return -ENOENT;
}
root = kernfs_root(parent);
down_write(&root->kernfs_rwsem);
kn = kernfs_find_ns(parent, name, ns);
if (kn) {
kernfs_get(kn);
__kernfs_remove(kn);
kernfs_put(kn);
}
up_write(&root->kernfs_rwsem); if (kn)
return 0;
else
return -ENOENT;
}
/**
* kernfs_rename_ns - move and rename a kernfs_node
* @kn: target node
* @new_parent: new parent to put @sd under
* @new_name: new name
* @new_ns: new namespace tag
*
* Return: %0 on success, -errno on failure.
*/
int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
const char *new_name, const void *new_ns)
{
struct kernfs_node *old_parent;
struct kernfs_root *root;
const char *old_name = NULL;
int error;
/* can't move or rename root */
if (!kn->parent)
return -EINVAL;
root = kernfs_root(kn);
down_write(&root->kernfs_rwsem);
error = -ENOENT;
if (!kernfs_active(kn) || !kernfs_active(new_parent) ||
(new_parent->flags & KERNFS_EMPTY_DIR))
goto out;
error = 0;
if ((kn->parent == new_parent) && (kn->ns == new_ns) &&
(strcmp(kn->name, new_name) == 0))
goto out; /* nothing to rename */
error = -EEXIST;
if (kernfs_find_ns(new_parent, new_name, new_ns))
goto out;
/* rename kernfs_node */
if (strcmp(kn->name, new_name) != 0) {
error = -ENOMEM;
new_name = kstrdup_const(new_name, GFP_KERNEL);
if (!new_name)
goto out;
} else {
new_name = NULL;
}
/*
* Move to the appropriate place in the appropriate directories rbtree.
*/
kernfs_unlink_sibling(kn);
kernfs_get(new_parent);
/* rename_lock protects ->parent and ->name accessors */
write_lock_irq(&kernfs_rename_lock);
old_parent = kn->parent;
kn->parent = new_parent;
kn->ns = new_ns;
if (new_name) {
old_name = kn->name;
kn->name = new_name;
}
write_unlock_irq(&kernfs_rename_lock);
kn->hash = kernfs_name_hash(kn->name, kn->ns);
kernfs_link_sibling(kn);
kernfs_put(old_parent);
kfree_const(old_name);
error = 0;
out:
up_write(&root->kernfs_rwsem);
return error;
}
static int kernfs_dir_fop_release(struct inode *inode, struct file *filp)
{
kernfs_put(filp->private_data);
return 0;
}
static struct kernfs_node *kernfs_dir_pos(const void *ns,
struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos)
{
if (pos) {
int valid = kernfs_active(pos) &&
pos->parent == parent && hash == pos->hash;
kernfs_put(pos);
if (!valid)
pos = NULL;
}
if (!pos && (hash > 1) && (hash < INT_MAX)) {
struct rb_node *node = parent->dir.children.rb_node;
while (node) {
pos = rb_to_kn(node);
if (hash < pos->hash)
node = node->rb_left;
else if (hash > pos->hash)
node = node->rb_right;
else
break;
}
}
/* Skip over entries which are dying/dead or in the wrong namespace */
while (pos && (!kernfs_active(pos) || pos->ns != ns)) {
struct rb_node *node = rb_next(&pos->rb);
if (!node)
pos = NULL;
else
pos = rb_to_kn(node);
}
return pos;
}
static struct kernfs_node *kernfs_dir_next_pos(const void *ns,
struct kernfs_node *parent, ino_t ino, struct kernfs_node *pos)
{
pos = kernfs_dir_pos(ns, parent, ino, pos);
if (pos) {
do {
struct rb_node *node = rb_next(&pos->rb);
if (!node)
pos = NULL;
else
pos = rb_to_kn(node);
} while (pos && (!kernfs_active(pos) || pos->ns != ns));
}
return pos;
}
static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx)
{
struct dentry *dentry = file->f_path.dentry;
struct kernfs_node *parent = kernfs_dentry_node(dentry);
struct kernfs_node *pos = file->private_data;
struct kernfs_root *root;
const void *ns = NULL;
if (!dir_emit_dots(file, ctx))
return 0;
root = kernfs_root(parent);
down_read(&root->kernfs_rwsem);
if (kernfs_ns_enabled(parent))
ns = kernfs_info(dentry->d_sb)->ns;
for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos);
pos;
pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos)) {
const char *name = pos->name;
unsigned int type = fs_umode_to_dtype(pos->mode);
int len = strlen(name);
ino_t ino = kernfs_ino(pos);
ctx->pos = pos->hash;
file->private_data = pos;
kernfs_get(pos);
up_read(&root->kernfs_rwsem);
if (!dir_emit(ctx, name, len, ino, type))
return 0;
down_read(&root->kernfs_rwsem);
}
up_read(&root->kernfs_rwsem);
file->private_data = NULL;
ctx->pos = INT_MAX;
return 0;
}
const struct file_operations kernfs_dir_fops = {
.read = generic_read_dir,
.iterate_shared = kernfs_fop_readdir,
.release = kernfs_dir_fop_release,
.llseek = generic_file_llseek,
};
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/kernel/signal.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*
* 1997-11-02 Modified for POSIX.1b signals by Richard Henderson
*
* 2003-06-02 Jim Houston - Concurrent Computer Corp.
* Changes to use preallocated sigqueue structures
* to allow signals to be sent reliably.
*/
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/user.h>
#include <linux/sched/debug.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/tty.h>
#include <linux/binfmts.h>
#include <linux/coredump.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/signalfd.h>
#include <linux/ratelimit.h>
#include <linux/task_work.h>
#include <linux/capability.h>
#include <linux/freezer.h>
#include <linux/pid_namespace.h>
#include <linux/nsproxy.h>
#include <linux/user_namespace.h>
#include <linux/uprobes.h>
#include <linux/compat.h>
#include <linux/cn_proc.h>
#include <linux/compiler.h>
#include <linux/posix-timers.h>
#include <linux/cgroup.h>
#include <linux/audit.h>
#include <linux/sysctl.h>
#include <uapi/linux/pidfd.h>
#define CREATE_TRACE_POINTS
#include <trace/events/signal.h>
#include <asm/param.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
#include <asm/siginfo.h>
#include <asm/cacheflush.h>
#include <asm/syscall.h> /* for syscall_get_* */
/*
* SLAB caches for signal bits.
*/
static struct kmem_cache *sigqueue_cachep;
int print_fatal_signals __read_mostly;
static void __user *sig_handler(struct task_struct *t, int sig)
{
return t->sighand->action[sig - 1].sa.sa_handler;
}
static inline bool sig_handler_ignored(void __user *handler, int sig)
{
/* Is it explicitly or implicitly ignored? */
return handler == SIG_IGN || (handler == SIG_DFL && sig_kernel_ignore(sig));
}
static bool sig_task_ignored(struct task_struct *t, int sig, bool force)
{
void __user *handler;
handler = sig_handler(t, sig);
/* SIGKILL and SIGSTOP may not be sent to the global init */
if (unlikely(is_global_init(t) && sig_kernel_only(sig)))
return true;
if (unlikely(t->signal->flags & SIGNAL_UNKILLABLE) && handler == SIG_DFL && !(force && sig_kernel_only(sig)))
return true;
/* Only allow kernel generated signals to this kthread */
if (unlikely((t->flags & PF_KTHREAD) &&
(handler == SIG_KTHREAD_KERNEL) && !force))
return true;
return sig_handler_ignored(handler, sig);
}
static bool sig_ignored(struct task_struct *t, int sig, bool force)
{
/*
* Blocked signals are never ignored, since the
* signal handler may change by the time it is
* unblocked.
*/
if (sigismember(&t->blocked, sig) || sigismember(&t->real_blocked, sig))
return false;
/*
* Tracers may want to know about even ignored signal unless it
* is SIGKILL which can't be reported anyway but can be ignored
* by SIGNAL_UNKILLABLE task.
*/
if (t->ptrace && sig != SIGKILL)
return false;
return sig_task_ignored(t, sig, force);
}
/*
* Re-calculate pending state from the set of locally pending
* signals, globally pending signals, and blocked signals.
*/
static inline bool has_pending_signals(sigset_t *signal, sigset_t *blocked)
{
unsigned long ready;
long i;
switch (_NSIG_WORDS) {
default:
for (i = _NSIG_WORDS, ready = 0; --i >= 0 ;)
ready |= signal->sig[i] &~ blocked->sig[i];
break;
case 4: ready = signal->sig[3] &~ blocked->sig[3];
ready |= signal->sig[2] &~ blocked->sig[2];
ready |= signal->sig[1] &~ blocked->sig[1];
ready |= signal->sig[0] &~ blocked->sig[0];
break;
case 2: ready = signal->sig[1] &~ blocked->sig[1];
ready |= signal->sig[0] &~ blocked->sig[0];
break;
case 1: ready = signal->sig[0] &~ blocked->sig[0];
}
return ready != 0;
}
#define PENDING(p,b) has_pending_signals(&(p)->signal, (b))
static bool recalc_sigpending_tsk(struct task_struct *t)
{
if ((t->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) ||
PENDING(&t->pending, &t->blocked) || PENDING(&t->signal->shared_pending, &t->blocked) || cgroup_task_frozen(t)) { set_tsk_thread_flag(t, TIF_SIGPENDING);
return true;
}
/*
* We must never clear the flag in another thread, or in current
* when it's possible the current syscall is returning -ERESTART*.
* So we don't clear it here, and only callers who know they should do.
*/
return false;
}
void recalc_sigpending(void)
{
if (!recalc_sigpending_tsk(current) && !freezing(current)) clear_thread_flag(TIF_SIGPENDING);
}
EXPORT_SYMBOL(recalc_sigpending);
void calculate_sigpending(void)
{
/* Have any signals or users of TIF_SIGPENDING been delayed
* until after fork?
*/
spin_lock_irq(¤t->sighand->siglock);
set_tsk_thread_flag(current, TIF_SIGPENDING);
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
}
/* Given the mask, find the first available signal that should be serviced. */
#define SYNCHRONOUS_MASK \
(sigmask(SIGSEGV) | sigmask(SIGBUS) | sigmask(SIGILL) | \
sigmask(SIGTRAP) | sigmask(SIGFPE) | sigmask(SIGSYS))
int next_signal(struct sigpending *pending, sigset_t *mask)
{
unsigned long i, *s, *m, x;
int sig = 0;
s = pending->signal.sig;
m = mask->sig;
/*
* Handle the first word specially: it contains the
* synchronous signals that need to be dequeued first.
*/
x = *s &~ *m;
if (x) {
if (x & SYNCHRONOUS_MASK)
x &= SYNCHRONOUS_MASK;
sig = ffz(~x) + 1;
return sig;
}
switch (_NSIG_WORDS) {
default:
for (i = 1; i < _NSIG_WORDS; ++i) {
x = *++s &~ *++m;
if (!x)
continue;
sig = ffz(~x) + i*_NSIG_BPW + 1;
break;
}
break;
case 2:
x = s[1] &~ m[1];
if (!x)
break;
sig = ffz(~x) + _NSIG_BPW + 1;
break;
case 1:
/* Nothing to do */
break;
}
return sig;
}
static inline void print_dropped_signal(int sig)
{
static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
if (!print_fatal_signals)
return;
if (!__ratelimit(&ratelimit_state))
return;
pr_info("%s/%d: reached RLIMIT_SIGPENDING, dropped signal %d\n",
current->comm, current->pid, sig);
}
/**
* task_set_jobctl_pending - set jobctl pending bits
* @task: target task
* @mask: pending bits to set
*
* Clear @mask from @task->jobctl. @mask must be subset of
* %JOBCTL_PENDING_MASK | %JOBCTL_STOP_CONSUME | %JOBCTL_STOP_SIGMASK |
* %JOBCTL_TRAPPING. If stop signo is being set, the existing signo is
* cleared. If @task is already being killed or exiting, this function
* becomes noop.
*
* CONTEXT:
* Must be called with @task->sighand->siglock held.
*
* RETURNS:
* %true if @mask is set, %false if made noop because @task was dying.
*/
bool task_set_jobctl_pending(struct task_struct *task, unsigned long mask)
{
BUG_ON(mask & ~(JOBCTL_PENDING_MASK | JOBCTL_STOP_CONSUME |
JOBCTL_STOP_SIGMASK | JOBCTL_TRAPPING));
BUG_ON((mask & JOBCTL_TRAPPING) && !(mask & JOBCTL_PENDING_MASK));
if (unlikely(fatal_signal_pending(task) || (task->flags & PF_EXITING)))
return false;
if (mask & JOBCTL_STOP_SIGMASK)
task->jobctl &= ~JOBCTL_STOP_SIGMASK;
task->jobctl |= mask;
return true;
}
/**
* task_clear_jobctl_trapping - clear jobctl trapping bit
* @task: target task
*
* If JOBCTL_TRAPPING is set, a ptracer is waiting for us to enter TRACED.
* Clear it and wake up the ptracer. Note that we don't need any further
* locking. @task->siglock guarantees that @task->parent points to the
* ptracer.
*
* CONTEXT:
* Must be called with @task->sighand->siglock held.
*/
void task_clear_jobctl_trapping(struct task_struct *task)
{
if (unlikely(task->jobctl & JOBCTL_TRAPPING)) {
task->jobctl &= ~JOBCTL_TRAPPING;
smp_mb(); /* advised by wake_up_bit() */
wake_up_bit(&task->jobctl, JOBCTL_TRAPPING_BIT);
}
}
/**
* task_clear_jobctl_pending - clear jobctl pending bits
* @task: target task
* @mask: pending bits to clear
*
* Clear @mask from @task->jobctl. @mask must be subset of
* %JOBCTL_PENDING_MASK. If %JOBCTL_STOP_PENDING is being cleared, other
* STOP bits are cleared together.
*
* If clearing of @mask leaves no stop or trap pending, this function calls
* task_clear_jobctl_trapping().
*
* CONTEXT:
* Must be called with @task->sighand->siglock held.
*/
void task_clear_jobctl_pending(struct task_struct *task, unsigned long mask)
{
BUG_ON(mask & ~JOBCTL_PENDING_MASK);
if (mask & JOBCTL_STOP_PENDING)
mask |= JOBCTL_STOP_CONSUME | JOBCTL_STOP_DEQUEUED;
task->jobctl &= ~mask;
if (!(task->jobctl & JOBCTL_PENDING_MASK))
task_clear_jobctl_trapping(task);
}
/**
* task_participate_group_stop - participate in a group stop
* @task: task participating in a group stop
*
* @task has %JOBCTL_STOP_PENDING set and is participating in a group stop.
* Group stop states are cleared and the group stop count is consumed if
* %JOBCTL_STOP_CONSUME was set. If the consumption completes the group
* stop, the appropriate `SIGNAL_*` flags are set.
*
* CONTEXT:
* Must be called with @task->sighand->siglock held.
*
* RETURNS:
* %true if group stop completion should be notified to the parent, %false
* otherwise.
*/
static bool task_participate_group_stop(struct task_struct *task)
{
struct signal_struct *sig = task->signal;
bool consume = task->jobctl & JOBCTL_STOP_CONSUME;
WARN_ON_ONCE(!(task->jobctl & JOBCTL_STOP_PENDING));
task_clear_jobctl_pending(task, JOBCTL_STOP_PENDING);
if (!consume)
return false;
if (!WARN_ON_ONCE(sig->group_stop_count == 0))
sig->group_stop_count--;
/*
* Tell the caller to notify completion iff we are entering into a
* fresh group stop. Read comment in do_signal_stop() for details.
*/
if (!sig->group_stop_count && !(sig->flags & SIGNAL_STOP_STOPPED)) {
signal_set_stop_flags(sig, SIGNAL_STOP_STOPPED);
return true;
}
return false;
}
void task_join_group_stop(struct task_struct *task)
{
unsigned long mask = current->jobctl & JOBCTL_STOP_SIGMASK;
struct signal_struct *sig = current->signal;
if (sig->group_stop_count) {
sig->group_stop_count++;
mask |= JOBCTL_STOP_CONSUME;
} else if (!(sig->flags & SIGNAL_STOP_STOPPED))
return;
/* Have the new thread join an on-going signal group stop */
task_set_jobctl_pending(task, mask | JOBCTL_STOP_PENDING);
}
/*
* allocate a new signal queue record
* - this may be called without locks if and only if t == current, otherwise an
* appropriate lock must be held to stop the target task from exiting
*/
static struct sigqueue *
__sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags,
int override_rlimit, const unsigned int sigqueue_flags)
{
struct sigqueue *q = NULL;
struct ucounts *ucounts;
long sigpending;
/*
* Protect access to @t credentials. This can go away when all
* callers hold rcu read lock.
*
* NOTE! A pending signal will hold on to the user refcount,
* and we get/put the refcount only when the sigpending count
* changes from/to zero.
*/
rcu_read_lock(); ucounts = task_ucounts(t);
sigpending = inc_rlimit_get_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
rcu_read_unlock();
if (!sigpending)
return NULL;
if (override_rlimit || likely(sigpending <= task_rlimit(t, RLIMIT_SIGPENDING))) { q = kmem_cache_alloc(sigqueue_cachep, gfp_flags);
} else {
print_dropped_signal(sig);
}
if (unlikely(q == NULL)) {
dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
} else {
INIT_LIST_HEAD(&q->list);
q->flags = sigqueue_flags;
q->ucounts = ucounts;
}
return q;
}
static void __sigqueue_free(struct sigqueue *q)
{
if (q->flags & SIGQUEUE_PREALLOC)
return;
if (q->ucounts) { dec_rlimit_put_ucounts(q->ucounts, UCOUNT_RLIMIT_SIGPENDING);
q->ucounts = NULL;
}
kmem_cache_free(sigqueue_cachep, q);
}
void flush_sigqueue(struct sigpending *queue)
{
struct sigqueue *q;
sigemptyset(&queue->signal);
while (!list_empty(&queue->list)) {
q = list_entry(queue->list.next, struct sigqueue , list);
list_del_init(&q->list);
__sigqueue_free(q);
}
}
/*
* Flush all pending signals for this kthread.
*/
void flush_signals(struct task_struct *t)
{
unsigned long flags;
spin_lock_irqsave(&t->sighand->siglock, flags);
clear_tsk_thread_flag(t, TIF_SIGPENDING);
flush_sigqueue(&t->pending);
flush_sigqueue(&t->signal->shared_pending);
spin_unlock_irqrestore(&t->sighand->siglock, flags);
}
EXPORT_SYMBOL(flush_signals);
#ifdef CONFIG_POSIX_TIMERS
static void __flush_itimer_signals(struct sigpending *pending)
{
sigset_t signal, retain;
struct sigqueue *q, *n;
signal = pending->signal;
sigemptyset(&retain);
list_for_each_entry_safe(q, n, &pending->list, list) {
int sig = q->info.si_signo;
if (likely(q->info.si_code != SI_TIMER)) {
sigaddset(&retain, sig);
} else {
sigdelset(&signal, sig);
list_del_init(&q->list);
__sigqueue_free(q);
}
}
sigorsets(&pending->signal, &signal, &retain);
}
void flush_itimer_signals(void)
{
struct task_struct *tsk = current;
unsigned long flags;
spin_lock_irqsave(&tsk->sighand->siglock, flags);
__flush_itimer_signals(&tsk->pending);
__flush_itimer_signals(&tsk->signal->shared_pending);
spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
}
#endif
void ignore_signals(struct task_struct *t)
{
int i;
for (i = 0; i < _NSIG; ++i)
t->sighand->action[i].sa.sa_handler = SIG_IGN;
flush_signals(t);
}
/*
* Flush all handlers for a task.
*/
void
flush_signal_handlers(struct task_struct *t, int force_default)
{
int i;
struct k_sigaction *ka = &t->sighand->action[0];
for (i = _NSIG ; i != 0 ; i--) {
if (force_default || ka->sa.sa_handler != SIG_IGN)
ka->sa.sa_handler = SIG_DFL;
ka->sa.sa_flags = 0;
#ifdef __ARCH_HAS_SA_RESTORER
ka->sa.sa_restorer = NULL;
#endif
sigemptyset(&ka->sa.sa_mask);
ka++;
}
}
bool unhandled_signal(struct task_struct *tsk, int sig)
{
void __user *handler = tsk->sighand->action[sig-1].sa.sa_handler;
if (is_global_init(tsk))
return true;
if (handler != SIG_IGN && handler != SIG_DFL)
return false;
/* If dying, we handle all new signals by ignoring them */
if (fatal_signal_pending(tsk))
return false;
/* if ptraced, let the tracer determine */
return !tsk->ptrace;
}
static void collect_signal(int sig, struct sigpending *list, kernel_siginfo_t *info,
bool *resched_timer)
{
struct sigqueue *q, *first = NULL;
/*
* Collect the siginfo appropriate to this signal. Check if
* there is another siginfo for the same signal.
*/
list_for_each_entry(q, &list->list, list) {
if (q->info.si_signo == sig) {
if (first)
goto still_pending;
first = q;
}
}
sigdelset(&list->signal, sig);
if (first) {
still_pending:
list_del_init(&first->list);
copy_siginfo(info, &first->info);
*resched_timer =
(first->flags & SIGQUEUE_PREALLOC) &&
(info->si_code == SI_TIMER) &&
(info->si_sys_private);
__sigqueue_free(first);
} else {
/*
* Ok, it wasn't in the queue. This must be
* a fast-pathed signal or we must have been
* out of queue space. So zero out the info.
*/
clear_siginfo(info);
info->si_signo = sig;
info->si_errno = 0;
info->si_code = SI_USER;
info->si_pid = 0;
info->si_uid = 0;
}
}
static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
kernel_siginfo_t *info, bool *resched_timer)
{
int sig = next_signal(pending, mask);
if (sig)
collect_signal(sig, pending, info, resched_timer);
return sig;
}
/*
* Dequeue a signal and return the element to the caller, which is
* expected to free it.
*
* All callers have to hold the siglock.
*/
int dequeue_signal(struct task_struct *tsk, sigset_t *mask,
kernel_siginfo_t *info, enum pid_type *type)
{
bool resched_timer = false;
int signr;
/* We only dequeue private signals from ourselves, we don't let
* signalfd steal them
*/
*type = PIDTYPE_PID;
signr = __dequeue_signal(&tsk->pending, mask, info, &resched_timer);
if (!signr) {
*type = PIDTYPE_TGID;
signr = __dequeue_signal(&tsk->signal->shared_pending,
mask, info, &resched_timer);
#ifdef CONFIG_POSIX_TIMERS
/*
* itimer signal ?
*
* itimers are process shared and we restart periodic
* itimers in the signal delivery path to prevent DoS
* attacks in the high resolution timer case. This is
* compliant with the old way of self-restarting
* itimers, as the SIGALRM is a legacy signal and only
* queued once. Changing the restart behaviour to
* restart the timer in the signal dequeue path is
* reducing the timer noise on heavy loaded !highres
* systems too.
*/
if (unlikely(signr == SIGALRM)) {
struct hrtimer *tmr = &tsk->signal->real_timer;
if (!hrtimer_is_queued(tmr) &&
tsk->signal->it_real_incr != 0) {
hrtimer_forward(tmr, tmr->base->get_time(),
tsk->signal->it_real_incr);
hrtimer_restart(tmr);
}
}
#endif
}
recalc_sigpending();
if (!signr)
return 0;
if (unlikely(sig_kernel_stop(signr))) {
/*
* Set a marker that we have dequeued a stop signal. Our
* caller might release the siglock and then the pending
* stop signal it is about to process is no longer in the
* pending bitmasks, but must still be cleared by a SIGCONT
* (and overruled by a SIGKILL). So those cases clear this
* shared flag after we've set it. Note that this flag may
* remain set after the signal we return is ignored or
* handled. That doesn't matter because its only purpose
* is to alert stop-signal processing code when another
* processor has come along and cleared the flag.
*/
current->jobctl |= JOBCTL_STOP_DEQUEUED;
}
#ifdef CONFIG_POSIX_TIMERS
if (resched_timer) {
/*
* Release the siglock to ensure proper locking order
* of timer locks outside of siglocks. Note, we leave
* irqs disabled here, since the posix-timers code is
* about to disable them again anyway.
*/
spin_unlock(&tsk->sighand->siglock);
posixtimer_rearm(info);
spin_lock(&tsk->sighand->siglock);
/* Don't expose the si_sys_private value to userspace */
info->si_sys_private = 0;
}
#endif
return signr;
}
EXPORT_SYMBOL_GPL(dequeue_signal);
static int dequeue_synchronous_signal(kernel_siginfo_t *info)
{
struct task_struct *tsk = current;
struct sigpending *pending = &tsk->pending;
struct sigqueue *q, *sync = NULL;
/*
* Might a synchronous signal be in the queue?
*/
if (!((pending->signal.sig[0] & ~tsk->blocked.sig[0]) & SYNCHRONOUS_MASK))
return 0;
/*
* Return the first synchronous signal in the queue.
*/
list_for_each_entry(q, &pending->list, list) {
/* Synchronous signals have a positive si_code */
if ((q->info.si_code > SI_USER) && (sigmask(q->info.si_signo) & SYNCHRONOUS_MASK)) {
sync = q;
goto next;
}
}
return 0;
next:
/*
* Check if there is another siginfo for the same signal.
*/
list_for_each_entry_continue(q, &pending->list, list) { if (q->info.si_signo == sync->info.si_signo)
goto still_pending;
}
sigdelset(&pending->signal, sync->info.si_signo);
recalc_sigpending();
still_pending:
list_del_init(&sync->list);
copy_siginfo(info, &sync->info);
__sigqueue_free(sync); return info->si_signo;
}
/*
* Tell a process that it has a new active signal..
*
* NOTE! we rely on the previous spin_lock to
* lock interrupts for us! We can only be called with
* "siglock" held, and the local interrupt must
* have been disabled when that got acquired!
*
* No need to set need_resched since signal event passing
* goes through ->blocked
*/
void signal_wake_up_state(struct task_struct *t, unsigned int state)
{
lockdep_assert_held(&t->sighand->siglock); set_tsk_thread_flag(t, TIF_SIGPENDING);
/*
* TASK_WAKEKILL also means wake it up in the stopped/traced/killable
* case. We don't check t->state here because there is a race with it
* executing another processor and just now entering stopped state.
* By using wake_up_state, we ensure the process will wake up and
* handle its death signal.
*/
if (!wake_up_state(t, state | TASK_INTERRUPTIBLE))
kick_process(t);
}
/*
* Remove signals in mask from the pending set and queue.
* Returns 1 if any signals were found.
*
* All callers must be holding the siglock.
*/
static void flush_sigqueue_mask(sigset_t *mask, struct sigpending *s)
{
struct sigqueue *q, *n;
sigset_t m;
sigandsets(&m, mask, &s->signal);
if (sigisemptyset(&m))
return;
sigandnsets(&s->signal, &s->signal, mask);
list_for_each_entry_safe(q, n, &s->list, list) {
if (sigismember(mask, q->info.si_signo)) {
list_del_init(&q->list);
__sigqueue_free(q);
}
}
}
static inline int is_si_special(const struct kernel_siginfo *info)
{
return info <= SEND_SIG_PRIV;
}
static inline bool si_fromuser(const struct kernel_siginfo *info)
{
return info == SEND_SIG_NOINFO ||
(!is_si_special(info) && SI_FROMUSER(info));
}
/*
* called with RCU read lock from check_kill_permission()
*/
static bool kill_ok_by_cred(struct task_struct *t)
{
const struct cred *cred = current_cred();
const struct cred *tcred = __task_cred(t);
return uid_eq(cred->euid, tcred->suid) ||
uid_eq(cred->euid, tcred->uid) ||
uid_eq(cred->uid, tcred->suid) ||
uid_eq(cred->uid, tcred->uid) ||
ns_capable(tcred->user_ns, CAP_KILL);
}
/*
* Bad permissions for sending the signal
* - the caller must hold the RCU read lock
*/
static int check_kill_permission(int sig, struct kernel_siginfo *info,
struct task_struct *t)
{
struct pid *sid;
int error;
if (!valid_signal(sig))
return -EINVAL;
if (!si_fromuser(info))
return 0;
error = audit_signal_info(sig, t); /* Let audit system see the signal */
if (error)
return error;
if (!same_thread_group(current, t) &&
!kill_ok_by_cred(t)) {
switch (sig) {
case SIGCONT:
sid = task_session(t);
/*
* We don't return the error if sid == NULL. The
* task was unhashed, the caller must notice this.
*/
if (!sid || sid == task_session(current))
break;
fallthrough;
default:
return -EPERM;
}
}
return security_task_kill(t, info, sig, NULL);
}
/**
* ptrace_trap_notify - schedule trap to notify ptracer
* @t: tracee wanting to notify tracer
*
* This function schedules sticky ptrace trap which is cleared on the next
* TRAP_STOP to notify ptracer of an event. @t must have been seized by
* ptracer.
*
* If @t is running, STOP trap will be taken. If trapped for STOP and
* ptracer is listening for events, tracee is woken up so that it can
* re-trap for the new event. If trapped otherwise, STOP trap will be
* eventually taken without returning to userland after the existing traps
* are finished by PTRACE_CONT.
*
* CONTEXT:
* Must be called with @task->sighand->siglock held.
*/
static void ptrace_trap_notify(struct task_struct *t)
{
WARN_ON_ONCE(!(t->ptrace & PT_SEIZED));
lockdep_assert_held(&t->sighand->siglock);
task_set_jobctl_pending(t, JOBCTL_TRAP_NOTIFY);
ptrace_signal_wake_up(t, t->jobctl & JOBCTL_LISTENING);
}
/*
* Handle magic process-wide effects of stop/continue signals. Unlike
* the signal actions, these happen immediately at signal-generation
* time regardless of blocking, ignoring, or handling. This does the
* actual continuing for SIGCONT, but not the actual stopping for stop
* signals. The process stop is done as a signal action for SIG_DFL.
*
* Returns true if the signal should be actually delivered, otherwise
* it should be dropped.
*/
static bool prepare_signal(int sig, struct task_struct *p, bool force)
{
struct signal_struct *signal = p->signal;
struct task_struct *t;
sigset_t flush;
if (signal->flags & SIGNAL_GROUP_EXIT) {
if (signal->core_state) return sig == SIGKILL;
/*
* The process is in the middle of dying, drop the signal.
*/
return false;
} else if (sig_kernel_stop(sig)) {
/*
* This is a stop signal. Remove SIGCONT from all queues.
*/
siginitset(&flush, sigmask(SIGCONT));
flush_sigqueue_mask(&flush, &signal->shared_pending);
for_each_thread(p, t)
flush_sigqueue_mask(&flush, &t->pending); } else if (sig == SIGCONT) {
unsigned int why;
/*
* Remove all stop signals from all queues, wake all threads.
*/
siginitset(&flush, SIG_KERNEL_STOP_MASK);
flush_sigqueue_mask(&flush, &signal->shared_pending);
for_each_thread(p, t) { flush_sigqueue_mask(&flush, &t->pending);
task_clear_jobctl_pending(t, JOBCTL_STOP_PENDING);
if (likely(!(t->ptrace & PT_SEIZED))) {
t->jobctl &= ~JOBCTL_STOPPED;
wake_up_state(t, __TASK_STOPPED);
} else
ptrace_trap_notify(t);
}
/*
* Notify the parent with CLD_CONTINUED if we were stopped.
*
* If we were in the middle of a group stop, we pretend it
* was already finished, and then continued. Since SIGCHLD
* doesn't queue we report only CLD_STOPPED, as if the next
* CLD_CONTINUED was dropped.
*/
why = 0;
if (signal->flags & SIGNAL_STOP_STOPPED)
why |= SIGNAL_CLD_CONTINUED;
else if (signal->group_stop_count)
why |= SIGNAL_CLD_STOPPED;
if (why) {
/*
* The first thread which returns from do_signal_stop()
* will take ->siglock, notice SIGNAL_CLD_MASK, and
* notify its parent. See get_signal().
*/
signal_set_stop_flags(signal, why | SIGNAL_STOP_CONTINUED);
signal->group_stop_count = 0;
signal->group_exit_code = 0;
}
}
return !sig_ignored(p, sig, force);
}
/*
* Test if P wants to take SIG. After we've checked all threads with this,
* it's equivalent to finding no threads not blocking SIG. Any threads not
* blocking SIG were ruled out because they are not running and already
* have pending signals. Such threads will dequeue from the shared queue
* as soon as they're available, so putting the signal on the shared queue
* will be equivalent to sending it to one such thread.
*/
static inline bool wants_signal(int sig, struct task_struct *p)
{
if (sigismember(&p->blocked, sig))
return false;
if (p->flags & PF_EXITING)
return false;
if (sig == SIGKILL)
return true;
if (task_is_stopped_or_traced(p))
return false;
return task_curr(p) || !task_sigpending(p);
}
static void complete_signal(int sig, struct task_struct *p, enum pid_type type)
{
struct signal_struct *signal = p->signal;
struct task_struct *t;
/*
* Now find a thread we can wake up to take the signal off the queue.
*
* Try the suggested task first (may or may not be the main thread).
*/
if (wants_signal(sig, p))
t = p;
else if ((type == PIDTYPE_PID) || thread_group_empty(p))
/*
* There is just one thread and it does not need to be woken.
* It will dequeue unblocked signals before it runs again.
*/
return;
else {
/*
* Otherwise try to find a suitable thread.
*/
t = signal->curr_target; while (!wants_signal(sig, t)) { t = next_thread(t); if (t == signal->curr_target)
/*
* No thread needs to be woken.
* Any eligible threads will see
* the signal in the queue soon.
*/
return;
}
signal->curr_target = t;
}
/*
* Found a killable thread. If the signal will be fatal,
* then start taking the whole group down immediately.
*/
if (sig_fatal(p, sig) && (signal->core_state || !(signal->flags & SIGNAL_GROUP_EXIT)) && !sigismember(&t->real_blocked, sig) && (sig == SIGKILL || !p->ptrace)) {
/*
* This signal will be fatal to the whole group.
*/
if (!sig_kernel_coredump(sig)) {
/*
* Start a group exit and wake everybody up.
* This way we don't have other threads
* running and doing things after a slower
* thread has the fatal signal pending.
*/
signal->flags = SIGNAL_GROUP_EXIT;
signal->group_exit_code = sig;
signal->group_stop_count = 0;
__for_each_thread(signal, t) {
task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
sigaddset(&t->pending.signal, SIGKILL);
signal_wake_up(t, 1);
}
return;
}
}
/*
* The signal is already in the shared-pending queue.
* Tell the chosen thread to wake up and dequeue it.
*/
signal_wake_up(t, sig == SIGKILL);
return;
}
static inline bool legacy_queue(struct sigpending *signals, int sig)
{
return (sig < SIGRTMIN) && sigismember(&signals->signal, sig);
}
static int __send_signal_locked(int sig, struct kernel_siginfo *info,
struct task_struct *t, enum pid_type type, bool force)
{
struct sigpending *pending;
struct sigqueue *q;
int override_rlimit;
int ret = 0, result;
lockdep_assert_held(&t->sighand->siglock);
result = TRACE_SIGNAL_IGNORED;
if (!prepare_signal(sig, t, force))
goto ret;
pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
/*
* Short-circuit ignored signals and support queuing
* exactly one non-rt signal, so that we can get more
* detailed information about the cause of the signal.
*/
result = TRACE_SIGNAL_ALREADY_PENDING;
if (legacy_queue(pending, sig))
goto ret;
result = TRACE_SIGNAL_DELIVERED;
/*
* Skip useless siginfo allocation for SIGKILL and kernel threads.
*/
if ((sig == SIGKILL) || (t->flags & PF_KTHREAD))
goto out_set;
/*
* Real-time signals must be queued if sent by sigqueue, or
* some other real-time mechanism. It is implementation
* defined whether kill() does so. We attempt to do so, on
* the principle of least surprise, but since kill is not
* allowed to fail with EAGAIN when low on memory we just
* make sure at least one signal gets delivered and don't
* pass on the info struct.
*/
if (sig < SIGRTMIN)
override_rlimit = (is_si_special(info) || info->si_code >= 0);
else
override_rlimit = 0;
q = __sigqueue_alloc(sig, t, GFP_ATOMIC, override_rlimit, 0);
if (q) {
list_add_tail(&q->list, &pending->list); switch ((unsigned long) info) {
case (unsigned long) SEND_SIG_NOINFO:
clear_siginfo(&q->info);
q->info.si_signo = sig;
q->info.si_errno = 0;
q->info.si_code = SI_USER;
q->info.si_pid = task_tgid_nr_ns(current,
task_active_pid_ns(t));
rcu_read_lock();
q->info.si_uid =
from_kuid_munged(task_cred_xxx(t, user_ns), current_uid()); rcu_read_unlock();
break;
case (unsigned long) SEND_SIG_PRIV:
clear_siginfo(&q->info);
q->info.si_signo = sig;
q->info.si_errno = 0;
q->info.si_code = SI_KERNEL;
q->info.si_pid = 0;
q->info.si_uid = 0;
break;
default:
copy_siginfo(&q->info, info);
break;
}
} else if (!is_si_special(info) && sig >= SIGRTMIN && info->si_code != SI_USER) {
/*
* Queue overflow, abort. We may abort if the
* signal was rt and sent by user using something
* other than kill().
*/
result = TRACE_SIGNAL_OVERFLOW_FAIL;
ret = -EAGAIN;
goto ret;
} else {
/*
* This is a silent loss of information. We still
* send the signal, but the *info bits are lost.
*/
result = TRACE_SIGNAL_LOSE_INFO;
}
out_set:
signalfd_notify(t, sig); sigaddset(&pending->signal, sig);
/* Let multiprocess signals appear after on-going forks */
if (type > PIDTYPE_TGID) {
struct multiprocess_signals *delayed;
hlist_for_each_entry(delayed, &t->signal->multiprocess, node) {
sigset_t *signal = &delayed->signal;
/* Can't queue both a stop and a continue signal */
if (sig == SIGCONT) sigdelsetmask(signal, SIG_KERNEL_STOP_MASK); else if (sig_kernel_stop(sig)) sigdelset(signal, SIGCONT); sigaddset(signal, sig);
}
}
complete_signal(sig, t, type);
ret:
trace_signal_generate(sig, info, t, type != PIDTYPE_PID, result);
return ret;
}
static inline bool has_si_pid_and_uid(struct kernel_siginfo *info)
{
bool ret = false;
switch (siginfo_layout(info->si_signo, info->si_code)) {
case SIL_KILL:
case SIL_CHLD:
case SIL_RT:
ret = true;
break;
case SIL_TIMER:
case SIL_POLL:
case SIL_FAULT:
case SIL_FAULT_TRAPNO:
case SIL_FAULT_MCEERR:
case SIL_FAULT_BNDERR:
case SIL_FAULT_PKUERR:
case SIL_FAULT_PERF_EVENT:
case SIL_SYS:
ret = false;
break;
}
return ret;
}
int send_signal_locked(int sig, struct kernel_siginfo *info,
struct task_struct *t, enum pid_type type)
{
/* Should SIGKILL or SIGSTOP be received by a pid namespace init? */
bool force = false;
if (info == SEND_SIG_NOINFO) {
/* Force if sent from an ancestor pid namespace */
force = !task_pid_nr_ns(current, task_active_pid_ns(t)); } else if (info == SEND_SIG_PRIV) {
/* Don't ignore kernel generated signals */
force = true;
} else if (has_si_pid_and_uid(info)) {
/* SIGKILL and SIGSTOP is special or has ids */
struct user_namespace *t_user_ns;
rcu_read_lock(); t_user_ns = task_cred_xxx(t, user_ns);
if (current_user_ns() != t_user_ns) {
kuid_t uid = make_kuid(current_user_ns(), info->si_uid);
info->si_uid = from_kuid_munged(t_user_ns, uid);
}
rcu_read_unlock();
/* A kernel generated signal? */
force = (info->si_code == SI_KERNEL);
/* From an ancestor pid namespace? */
if (!task_pid_nr_ns(current, task_active_pid_ns(t))) {
info->si_pid = 0;
force = true;
}
}
return __send_signal_locked(sig, info, t, type, force);
}
static void print_fatal_signal(int signr)
{
struct pt_regs *regs = task_pt_regs(current);
struct file *exe_file;
exe_file = get_task_exe_file(current);
if (exe_file) {
pr_info("%pD: %s: potentially unexpected fatal signal %d.\n",
exe_file, current->comm, signr);
fput(exe_file);
} else {
pr_info("%s: potentially unexpected fatal signal %d.\n",
current->comm, signr);
}
#if defined(__i386__) && !defined(__arch_um__)
pr_info("code at %08lx: ", regs->ip);
{
int i;
for (i = 0; i < 16; i++) {
unsigned char insn;
if (get_user(insn, (unsigned char *)(regs->ip + i)))
break;
pr_cont("%02x ", insn);
}
}
pr_cont("\n");
#endif
preempt_disable();
show_regs(regs);
preempt_enable();
}
static int __init setup_print_fatal_signals(char *str)
{
get_option (&str, &print_fatal_signals);
return 1;
}
__setup("print-fatal-signals=", setup_print_fatal_signals);
int do_send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p,
enum pid_type type)
{
unsigned long flags;
int ret = -ESRCH;
if (lock_task_sighand(p, &flags)) {
ret = send_signal_locked(sig, info, p, type);
unlock_task_sighand(p, &flags);
}
return ret;
}
enum sig_handler {
HANDLER_CURRENT, /* If reachable use the current handler */
HANDLER_SIG_DFL, /* Always use SIG_DFL handler semantics */
HANDLER_EXIT, /* Only visible as the process exit code */
};
/*
* Force a signal that the process can't ignore: if necessary
* we unblock the signal and change any SIG_IGN to SIG_DFL.
*
* Note: If we unblock the signal, we always reset it to SIG_DFL,
* since we do not want to have a signal handler that was blocked
* be invoked when user space had explicitly blocked it.
*
* We don't want to have recursive SIGSEGV's etc, for example,
* that is why we also clear SIGNAL_UNKILLABLE.
*/
static int
force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
enum sig_handler handler)
{
unsigned long int flags;
int ret, blocked, ignored;
struct k_sigaction *action;
int sig = info->si_signo;
spin_lock_irqsave(&t->sighand->siglock, flags);
action = &t->sighand->action[sig-1];
ignored = action->sa.sa_handler == SIG_IGN;
blocked = sigismember(&t->blocked, sig); if (blocked || ignored || (handler != HANDLER_CURRENT)) { action->sa.sa_handler = SIG_DFL;
if (handler == HANDLER_EXIT)
action->sa.sa_flags |= SA_IMMUTABLE; if (blocked) sigdelset(&t->blocked, sig);
}
/*
* Don't clear SIGNAL_UNKILLABLE for traced tasks, users won't expect
* debugging to leave init killable. But HANDLER_EXIT is always fatal.
*/
if (action->sa.sa_handler == SIG_DFL && (!t->ptrace || (handler == HANDLER_EXIT))) t->signal->flags &= ~SIGNAL_UNKILLABLE; ret = send_signal_locked(sig, info, t, PIDTYPE_PID);
/* This can happen if the signal was already pending and blocked */
if (!task_sigpending(t))
signal_wake_up(t, 0); spin_unlock_irqrestore(&t->sighand->siglock, flags);
return ret;
}
int force_sig_info(struct kernel_siginfo *info)
{
return force_sig_info_to_task(info, current, HANDLER_CURRENT);
}
/*
* Nuke all other threads in the group.
*/
int zap_other_threads(struct task_struct *p)
{
struct task_struct *t;
int count = 0;
p->signal->group_stop_count = 0;
for_other_threads(p, t) {
task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
/* Don't require de_thread to wait for the vhost_worker */
if ((t->flags & (PF_IO_WORKER | PF_USER_WORKER)) != PF_USER_WORKER)
count++;
/* Don't bother with already dead threads */
if (t->exit_state)
continue;
sigaddset(&t->pending.signal, SIGKILL);
signal_wake_up(t, 1);
}
return count;
}
struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
unsigned long *flags)
{
struct sighand_struct *sighand;
rcu_read_lock();
for (;;) {
sighand = rcu_dereference(tsk->sighand);
if (unlikely(sighand == NULL))
break;
/*
* This sighand can be already freed and even reused, but
* we rely on SLAB_TYPESAFE_BY_RCU and sighand_ctor() which
* initializes ->siglock: this slab can't go away, it has
* the same object type, ->siglock can't be reinitialized.
*
* We need to ensure that tsk->sighand is still the same
* after we take the lock, we can race with de_thread() or
* __exit_signal(). In the latter case the next iteration
* must see ->sighand == NULL.
*/
spin_lock_irqsave(&sighand->siglock, *flags);
if (likely(sighand == rcu_access_pointer(tsk->sighand)))
break;
spin_unlock_irqrestore(&sighand->siglock, *flags);
}
rcu_read_unlock();
return sighand;
}
#ifdef CONFIG_LOCKDEP
void lockdep_assert_task_sighand_held(struct task_struct *task)
{
struct sighand_struct *sighand;
rcu_read_lock();
sighand = rcu_dereference(task->sighand);
if (sighand)
lockdep_assert_held(&sighand->siglock);
else
WARN_ON_ONCE(1);
rcu_read_unlock();
}
#endif
/*
* send signal info to all the members of a thread group or to the
* individual thread if type == PIDTYPE_PID.
*/
int group_send_sig_info(int sig, struct kernel_siginfo *info,
struct task_struct *p, enum pid_type type)
{
int ret;
rcu_read_lock();
ret = check_kill_permission(sig, info, p);
rcu_read_unlock();
if (!ret && sig)
ret = do_send_sig_info(sig, info, p, type);
return ret;
}
/*
* __kill_pgrp_info() sends a signal to a process group: this is what the tty
* control characters do (^C, ^Z etc)
* - the caller must hold at least a readlock on tasklist_lock
*/
int __kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp)
{
struct task_struct *p = NULL;
int ret = -ESRCH;
do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
int err = group_send_sig_info(sig, info, p, PIDTYPE_PGID);
/*
* If group_send_sig_info() succeeds at least once ret
* becomes 0 and after that the code below has no effect.
* Otherwise we return the last err or -ESRCH if this
* process group is empty.
*/
if (ret)
ret = err;
} while_each_pid_task(pgrp, PIDTYPE_PGID, p);
return ret;
}
static int kill_pid_info_type(int sig, struct kernel_siginfo *info,
struct pid *pid, enum pid_type type)
{
int error = -ESRCH;
struct task_struct *p;
for (;;) {
rcu_read_lock();
p = pid_task(pid, PIDTYPE_PID);
if (p)
error = group_send_sig_info(sig, info, p, type);
rcu_read_unlock();
if (likely(!p || error != -ESRCH))
return error;
/*
* The task was unhashed in between, try again. If it
* is dead, pid_task() will return NULL, if we race with
* de_thread() it will find the new leader.
*/
}
}
int kill_pid_info(int sig, struct kernel_siginfo *info, struct pid *pid)
{
return kill_pid_info_type(sig, info, pid, PIDTYPE_TGID);
}
static int kill_proc_info(int sig, struct kernel_siginfo *info, pid_t pid)
{
int error;
rcu_read_lock();
error = kill_pid_info(sig, info, find_vpid(pid));
rcu_read_unlock();
return error;
}
static inline bool kill_as_cred_perm(const struct cred *cred,
struct task_struct *target)
{
const struct cred *pcred = __task_cred(target);
return uid_eq(cred->euid, pcred->suid) ||
uid_eq(cred->euid, pcred->uid) ||
uid_eq(cred->uid, pcred->suid) ||
uid_eq(cred->uid, pcred->uid);
}
/*
* The usb asyncio usage of siginfo is wrong. The glibc support
* for asyncio which uses SI_ASYNCIO assumes the layout is SIL_RT.
* AKA after the generic fields:
* kernel_pid_t si_pid;
* kernel_uid32_t si_uid;
* sigval_t si_value;
*
* Unfortunately when usb generates SI_ASYNCIO it assumes the layout
* after the generic fields is:
* void __user *si_addr;
*
* This is a practical problem when there is a 64bit big endian kernel
* and a 32bit userspace. As the 32bit address will encoded in the low
* 32bits of the pointer. Those low 32bits will be stored at higher
* address than appear in a 32 bit pointer. So userspace will not
* see the address it was expecting for it's completions.
*
* There is nothing in the encoding that can allow
* copy_siginfo_to_user32 to detect this confusion of formats, so
* handle this by requiring the caller of kill_pid_usb_asyncio to
* notice when this situration takes place and to store the 32bit
* pointer in sival_int, instead of sival_addr of the sigval_t addr
* parameter.
*/
int kill_pid_usb_asyncio(int sig, int errno, sigval_t addr,
struct pid *pid, const struct cred *cred)
{
struct kernel_siginfo info;
struct task_struct *p;
unsigned long flags;
int ret = -EINVAL;
if (!valid_signal(sig))
return ret;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = errno;
info.si_code = SI_ASYNCIO;
*((sigval_t *)&info.si_pid) = addr;
rcu_read_lock();
p = pid_task(pid, PIDTYPE_PID);
if (!p) {
ret = -ESRCH;
goto out_unlock;
}
if (!kill_as_cred_perm(cred, p)) {
ret = -EPERM;
goto out_unlock;
}
ret = security_task_kill(p, &info, sig, cred);
if (ret)
goto out_unlock;
if (sig) {
if (lock_task_sighand(p, &flags)) {
ret = __send_signal_locked(sig, &info, p, PIDTYPE_TGID, false);
unlock_task_sighand(p, &flags);
} else
ret = -ESRCH;
}
out_unlock:
rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(kill_pid_usb_asyncio);
/*
* kill_something_info() interprets pid in interesting ways just like kill(2).
*
* POSIX specifies that kill(-1,sig) is unspecified, but what we have
* is probably wrong. Should make it like BSD or SYSV.
*/
static int kill_something_info(int sig, struct kernel_siginfo *info, pid_t pid)
{
int ret;
if (pid > 0)
return kill_proc_info(sig, info, pid);
/* -INT_MIN is undefined. Exclude this case to avoid a UBSAN warning */
if (pid == INT_MIN)
return -ESRCH;
read_lock(&tasklist_lock);
if (pid != -1) {
ret = __kill_pgrp_info(sig, info,
pid ? find_vpid(-pid) : task_pgrp(current));
} else {
int retval = 0, count = 0;
struct task_struct * p;
for_each_process(p) {
if (task_pid_vnr(p) > 1 &&
!same_thread_group(p, current)) {
int err = group_send_sig_info(sig, info, p,
PIDTYPE_MAX);
++count;
if (err != -EPERM)
retval = err;
}
}
ret = count ? retval : -ESRCH;
}
read_unlock(&tasklist_lock);
return ret;
}
/*
* These are for backward compatibility with the rest of the kernel source.
*/
int send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p)
{
/*
* Make sure legacy kernel users don't send in bad values
* (normal paths check this in check_kill_permission).
*/
if (!valid_signal(sig))
return -EINVAL;
return do_send_sig_info(sig, info, p, PIDTYPE_PID);
}
EXPORT_SYMBOL(send_sig_info);
#define __si_special(priv) \
((priv) ? SEND_SIG_PRIV : SEND_SIG_NOINFO)
int
send_sig(int sig, struct task_struct *p, int priv)
{
return send_sig_info(sig, __si_special(priv), p);
}
EXPORT_SYMBOL(send_sig);
void force_sig(int sig)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = SI_KERNEL;
info.si_pid = 0;
info.si_uid = 0;
force_sig_info(&info);
}
EXPORT_SYMBOL(force_sig);
void force_fatal_sig(int sig)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = SI_KERNEL;
info.si_pid = 0;
info.si_uid = 0;
force_sig_info_to_task(&info, current, HANDLER_SIG_DFL);
}
void force_exit_sig(int sig)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = SI_KERNEL;
info.si_pid = 0;
info.si_uid = 0;
force_sig_info_to_task(&info, current, HANDLER_EXIT);
}
/*
* When things go south during signal handling, we
* will force a SIGSEGV. And if the signal that caused
* the problem was already a SIGSEGV, we'll want to
* make sure we don't even try to deliver the signal..
*/
void force_sigsegv(int sig)
{
if (sig == SIGSEGV)
force_fatal_sig(SIGSEGV);
else
force_sig(SIGSEGV);
}
int force_sig_fault_to_task(int sig, int code, void __user *addr,
struct task_struct *t)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = code;
info.si_addr = addr;
return force_sig_info_to_task(&info, t, HANDLER_CURRENT);
}
int force_sig_fault(int sig, int code, void __user *addr)
{
return force_sig_fault_to_task(sig, code, addr, current);
}
int send_sig_fault(int sig, int code, void __user *addr, struct task_struct *t)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = code;
info.si_addr = addr;
return send_sig_info(info.si_signo, &info, t);
}
int force_sig_mceerr(int code, void __user *addr, short lsb)
{
struct kernel_siginfo info;
WARN_ON((code != BUS_MCEERR_AO) && (code != BUS_MCEERR_AR));
clear_siginfo(&info);
info.si_signo = SIGBUS;
info.si_errno = 0;
info.si_code = code;
info.si_addr = addr;
info.si_addr_lsb = lsb;
return force_sig_info(&info);
}
int send_sig_mceerr(int code, void __user *addr, short lsb, struct task_struct *t)
{
struct kernel_siginfo info;
WARN_ON((code != BUS_MCEERR_AO) && (code != BUS_MCEERR_AR));
clear_siginfo(&info);
info.si_signo = SIGBUS;
info.si_errno = 0;
info.si_code = code;
info.si_addr = addr;
info.si_addr_lsb = lsb;
return send_sig_info(info.si_signo, &info, t);
}
EXPORT_SYMBOL(send_sig_mceerr);
int force_sig_bnderr(void __user *addr, void __user *lower, void __user *upper)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = SIGSEGV;
info.si_errno = 0;
info.si_code = SEGV_BNDERR;
info.si_addr = addr;
info.si_lower = lower;
info.si_upper = upper;
return force_sig_info(&info);
}
#ifdef SEGV_PKUERR
int force_sig_pkuerr(void __user *addr, u32 pkey)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = SIGSEGV;
info.si_errno = 0;
info.si_code = SEGV_PKUERR;
info.si_addr = addr;
info.si_pkey = pkey;
return force_sig_info(&info);
}
#endif
int send_sig_perf(void __user *addr, u32 type, u64 sig_data)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = SIGTRAP;
info.si_errno = 0;
info.si_code = TRAP_PERF;
info.si_addr = addr;
info.si_perf_data = sig_data;
info.si_perf_type = type;
/*
* Signals generated by perf events should not terminate the whole
* process if SIGTRAP is blocked, however, delivering the signal
* asynchronously is better than not delivering at all. But tell user
* space if the signal was asynchronous, so it can clearly be
* distinguished from normal synchronous ones.
*/
info.si_perf_flags = sigismember(¤t->blocked, info.si_signo) ?
TRAP_PERF_FLAG_ASYNC :
0;
return send_sig_info(info.si_signo, &info, current);
}
/**
* force_sig_seccomp - signals the task to allow in-process syscall emulation
* @syscall: syscall number to send to userland
* @reason: filter-supplied reason code to send to userland (via si_errno)
* @force_coredump: true to trigger a coredump
*
* Forces a SIGSYS with a code of SYS_SECCOMP and related sigsys info.
*/
int force_sig_seccomp(int syscall, int reason, bool force_coredump)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = SIGSYS;
info.si_code = SYS_SECCOMP;
info.si_call_addr = (void __user *)KSTK_EIP(current);
info.si_errno = reason;
info.si_arch = syscall_get_arch(current);
info.si_syscall = syscall;
return force_sig_info_to_task(&info, current,
force_coredump ? HANDLER_EXIT : HANDLER_CURRENT);
}
/* For the crazy architectures that include trap information in
* the errno field, instead of an actual errno value.
*/
int force_sig_ptrace_errno_trap(int errno, void __user *addr)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = SIGTRAP;
info.si_errno = errno;
info.si_code = TRAP_HWBKPT;
info.si_addr = addr;
return force_sig_info(&info);
}
/* For the rare architectures that include trap information using
* si_trapno.
*/
int force_sig_fault_trapno(int sig, int code, void __user *addr, int trapno)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = code;
info.si_addr = addr;
info.si_trapno = trapno;
return force_sig_info(&info);
}
/* For the rare architectures that include trap information using
* si_trapno.
*/
int send_sig_fault_trapno(int sig, int code, void __user *addr, int trapno,
struct task_struct *t)
{
struct kernel_siginfo info;
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
info.si_code = code;
info.si_addr = addr;
info.si_trapno = trapno;
return send_sig_info(info.si_signo, &info, t);
}
static int kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp)
{
int ret;
read_lock(&tasklist_lock);
ret = __kill_pgrp_info(sig, info, pgrp);
read_unlock(&tasklist_lock);
return ret;
}
int kill_pgrp(struct pid *pid, int sig, int priv)
{
return kill_pgrp_info(sig, __si_special(priv), pid);
}
EXPORT_SYMBOL(kill_pgrp);
int kill_pid(struct pid *pid, int sig, int priv)
{
return kill_pid_info(sig, __si_special(priv), pid);
}
EXPORT_SYMBOL(kill_pid);
/*
* These functions support sending signals using preallocated sigqueue
* structures. This is needed "because realtime applications cannot
* afford to lose notifications of asynchronous events, like timer
* expirations or I/O completions". In the case of POSIX Timers
* we allocate the sigqueue structure from the timer_create. If this
* allocation fails we are able to report the failure to the application
* with an EAGAIN error.
*/
struct sigqueue *sigqueue_alloc(void)
{
return __sigqueue_alloc(-1, current, GFP_KERNEL, 0, SIGQUEUE_PREALLOC);
}
void sigqueue_free(struct sigqueue *q)
{
unsigned long flags;
spinlock_t *lock = ¤t->sighand->siglock;
BUG_ON(!(q->flags & SIGQUEUE_PREALLOC));
/*
* We must hold ->siglock while testing q->list
* to serialize with collect_signal() or with
* __exit_signal()->flush_sigqueue().
*/
spin_lock_irqsave(lock, flags);
q->flags &= ~SIGQUEUE_PREALLOC;
/*
* If it is queued it will be freed when dequeued,
* like the "regular" sigqueue.
*/
if (!list_empty(&q->list))
q = NULL;
spin_unlock_irqrestore(lock, flags);
if (q)
__sigqueue_free(q);
}
int send_sigqueue(struct sigqueue *q, struct pid *pid, enum pid_type type)
{
int sig = q->info.si_signo;
struct sigpending *pending;
struct task_struct *t;
unsigned long flags;
int ret, result;
BUG_ON(!(q->flags & SIGQUEUE_PREALLOC));
ret = -1;
rcu_read_lock();
/*
* This function is used by POSIX timers to deliver a timer signal.
* Where type is PIDTYPE_PID (such as for timers with SIGEV_THREAD_ID
* set), the signal must be delivered to the specific thread (queues
* into t->pending).
*
* Where type is not PIDTYPE_PID, signals must be delivered to the
* process. In this case, prefer to deliver to current if it is in
* the same thread group as the target process, which avoids
* unnecessarily waking up a potentially idle task.
*/
t = pid_task(pid, type);
if (!t)
goto ret;
if (type != PIDTYPE_PID && same_thread_group(t, current))
t = current;
if (!likely(lock_task_sighand(t, &flags)))
goto ret;
ret = 1; /* the signal is ignored */
result = TRACE_SIGNAL_IGNORED;
if (!prepare_signal(sig, t, false))
goto out;
ret = 0;
if (unlikely(!list_empty(&q->list))) {
/*
* If an SI_TIMER entry is already queue just increment
* the overrun count.
*/
BUG_ON(q->info.si_code != SI_TIMER);
q->info.si_overrun++;
result = TRACE_SIGNAL_ALREADY_PENDING;
goto out;
}
q->info.si_overrun = 0;
signalfd_notify(t, sig);
pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
list_add_tail(&q->list, &pending->list);
sigaddset(&pending->signal, sig);
complete_signal(sig, t, type);
result = TRACE_SIGNAL_DELIVERED;
out:
trace_signal_generate(sig, &q->info, t, type != PIDTYPE_PID, result);
unlock_task_sighand(t, &flags);
ret:
rcu_read_unlock();
return ret;
}
void do_notify_pidfd(struct task_struct *task)
{
struct pid *pid = task_pid(task);
WARN_ON(task->exit_state == 0);
__wake_up(&pid->wait_pidfd, TASK_NORMAL, 0,
poll_to_key(EPOLLIN | EPOLLRDNORM));
}
/*
* Let a parent know about the death of a child.
* For a stopped/continued status change, use do_notify_parent_cldstop instead.
*
* Returns true if our parent ignored us and so we've switched to
* self-reaping.
*/
bool do_notify_parent(struct task_struct *tsk, int sig)
{
struct kernel_siginfo info;
unsigned long flags;
struct sighand_struct *psig;
bool autoreap = false;
u64 utime, stime;
WARN_ON_ONCE(sig == -1);
/* do_notify_parent_cldstop should have been called instead. */
WARN_ON_ONCE(task_is_stopped_or_traced(tsk));
WARN_ON_ONCE(!tsk->ptrace &&
(tsk->group_leader != tsk || !thread_group_empty(tsk)));
/*
* tsk is a group leader and has no threads, wake up the
* non-PIDFD_THREAD waiters.
*/
if (thread_group_empty(tsk))
do_notify_pidfd(tsk);
if (sig != SIGCHLD) {
/*
* This is only possible if parent == real_parent.
* Check if it has changed security domain.
*/
if (tsk->parent_exec_id != READ_ONCE(tsk->parent->self_exec_id))
sig = SIGCHLD;
}
clear_siginfo(&info);
info.si_signo = sig;
info.si_errno = 0;
/*
* We are under tasklist_lock here so our parent is tied to
* us and cannot change.
*
* task_active_pid_ns will always return the same pid namespace
* until a task passes through release_task.
*
* write_lock() currently calls preempt_disable() which is the
* same as rcu_read_lock(), but according to Oleg, this is not
* correct to rely on this
*/
rcu_read_lock();
info.si_pid = task_pid_nr_ns(tsk, task_active_pid_ns(tsk->parent));
info.si_uid = from_kuid_munged(task_cred_xxx(tsk->parent, user_ns),
task_uid(tsk));
rcu_read_unlock();
task_cputime(tsk, &utime, &stime);
info.si_utime = nsec_to_clock_t(utime + tsk->signal->utime);
info.si_stime = nsec_to_clock_t(stime + tsk->signal->stime);
info.si_status = tsk->exit_code & 0x7f;
if (tsk->exit_code & 0x80)
info.si_code = CLD_DUMPED;
else if (tsk->exit_code & 0x7f)
info.si_code = CLD_KILLED;
else {
info.si_code = CLD_EXITED;
info.si_status = tsk->exit_code >> 8;
}
psig = tsk->parent->sighand;
spin_lock_irqsave(&psig->siglock, flags);
if (!tsk->ptrace && sig == SIGCHLD &&
(psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN ||
(psig->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDWAIT))) {
/*
* We are exiting and our parent doesn't care. POSIX.1
* defines special semantics for setting SIGCHLD to SIG_IGN
* or setting the SA_NOCLDWAIT flag: we should be reaped
* automatically and not left for our parent's wait4 call.
* Rather than having the parent do it as a magic kind of
* signal handler, we just set this to tell do_exit that we
* can be cleaned up without becoming a zombie. Note that
* we still call __wake_up_parent in this case, because a
* blocked sys_wait4 might now return -ECHILD.
*
* Whether we send SIGCHLD or not for SA_NOCLDWAIT
* is implementation-defined: we do (if you don't want
* it, just use SIG_IGN instead).
*/
autoreap = true;
if (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN)
sig = 0;
}
/*
* Send with __send_signal as si_pid and si_uid are in the
* parent's namespaces.
*/
if (valid_signal(sig) && sig)
__send_signal_locked(sig, &info, tsk->parent, PIDTYPE_TGID, false);
__wake_up_parent(tsk, tsk->parent);
spin_unlock_irqrestore(&psig->siglock, flags);
return autoreap;
}
/**
* do_notify_parent_cldstop - notify parent of stopped/continued state change
* @tsk: task reporting the state change
* @for_ptracer: the notification is for ptracer
* @why: CLD_{CONTINUED|STOPPED|TRAPPED} to report
*
* Notify @tsk's parent that the stopped/continued state has changed. If
* @for_ptracer is %false, @tsk's group leader notifies to its real parent.
* If %true, @tsk reports to @tsk->parent which should be the ptracer.
*
* CONTEXT:
* Must be called with tasklist_lock at least read locked.
*/
static void do_notify_parent_cldstop(struct task_struct *tsk,
bool for_ptracer, int why)
{
struct kernel_siginfo info;
unsigned long flags;
struct task_struct *parent;
struct sighand_struct *sighand;
u64 utime, stime;
if (for_ptracer) {
parent = tsk->parent;
} else {
tsk = tsk->group_leader;
parent = tsk->real_parent;
}
clear_siginfo(&info);
info.si_signo = SIGCHLD;
info.si_errno = 0;
/*
* see comment in do_notify_parent() about the following 4 lines
*/
rcu_read_lock();
info.si_pid = task_pid_nr_ns(tsk, task_active_pid_ns(parent));
info.si_uid = from_kuid_munged(task_cred_xxx(parent, user_ns), task_uid(tsk));
rcu_read_unlock();
task_cputime(tsk, &utime, &stime);
info.si_utime = nsec_to_clock_t(utime);
info.si_stime = nsec_to_clock_t(stime);
info.si_code = why;
switch (why) {
case CLD_CONTINUED:
info.si_status = SIGCONT;
break;
case CLD_STOPPED:
info.si_status = tsk->signal->group_exit_code & 0x7f;
break;
case CLD_TRAPPED:
info.si_status = tsk->exit_code & 0x7f;
break;
default:
BUG();
}
sighand = parent->sighand;
spin_lock_irqsave(&sighand->siglock, flags);
if (sighand->action[SIGCHLD-1].sa.sa_handler != SIG_IGN &&
!(sighand->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDSTOP))
send_signal_locked(SIGCHLD, &info, parent, PIDTYPE_TGID);
/*
* Even if SIGCHLD is not generated, we must wake up wait4 calls.
*/
__wake_up_parent(tsk, parent);
spin_unlock_irqrestore(&sighand->siglock, flags);
}
/*
* This must be called with current->sighand->siglock held.
*
* This should be the path for all ptrace stops.
* We always set current->last_siginfo while stopped here.
* That makes it a way to test a stopped process for
* being ptrace-stopped vs being job-control-stopped.
*
* Returns the signal the ptracer requested the code resume
* with. If the code did not stop because the tracer is gone,
* the stop signal remains unchanged unless clear_code.
*/
static int ptrace_stop(int exit_code, int why, unsigned long message,
kernel_siginfo_t *info)
__releases(¤t->sighand->siglock)
__acquires(¤t->sighand->siglock)
{
bool gstop_done = false;
if (arch_ptrace_stop_needed()) {
/*
* The arch code has something special to do before a
* ptrace stop. This is allowed to block, e.g. for faults
* on user stack pages. We can't keep the siglock while
* calling arch_ptrace_stop, so we must release it now.
* To preserve proper semantics, we must do this before
* any signal bookkeeping like checking group_stop_count.
*/
spin_unlock_irq(¤t->sighand->siglock);
arch_ptrace_stop();
spin_lock_irq(¤t->sighand->siglock);
}
/*
* After this point ptrace_signal_wake_up or signal_wake_up
* will clear TASK_TRACED if ptrace_unlink happens or a fatal
* signal comes in. Handle previous ptrace_unlinks and fatal
* signals here to prevent ptrace_stop sleeping in schedule.
*/
if (!current->ptrace || __fatal_signal_pending(current))
return exit_code;
set_special_state(TASK_TRACED);
current->jobctl |= JOBCTL_TRACED;
/*
* We're committing to trapping. TRACED should be visible before
* TRAPPING is cleared; otherwise, the tracer might fail do_wait().
* Also, transition to TRACED and updates to ->jobctl should be
* atomic with respect to siglock and should be done after the arch
* hook as siglock is released and regrabbed across it.
*
* TRACER TRACEE
*
* ptrace_attach()
* [L] wait_on_bit(JOBCTL_TRAPPING) [S] set_special_state(TRACED)
* do_wait()
* set_current_state() smp_wmb();
* ptrace_do_wait()
* wait_task_stopped()
* task_stopped_code()
* [L] task_is_traced() [S] task_clear_jobctl_trapping();
*/
smp_wmb();
current->ptrace_message = message;
current->last_siginfo = info;
current->exit_code = exit_code;
/*
* If @why is CLD_STOPPED, we're trapping to participate in a group
* stop. Do the bookkeeping. Note that if SIGCONT was delievered
* across siglock relocks since INTERRUPT was scheduled, PENDING
* could be clear now. We act as if SIGCONT is received after
* TASK_TRACED is entered - ignore it.
*/
if (why == CLD_STOPPED && (current->jobctl & JOBCTL_STOP_PENDING))
gstop_done = task_participate_group_stop(current);
/* any trap clears pending STOP trap, STOP trap clears NOTIFY */
task_clear_jobctl_pending(current, JOBCTL_TRAP_STOP);
if (info && info->si_code >> 8 == PTRACE_EVENT_STOP)
task_clear_jobctl_pending(current, JOBCTL_TRAP_NOTIFY);
/* entering a trap, clear TRAPPING */
task_clear_jobctl_trapping(current);
spin_unlock_irq(¤t->sighand->siglock);
read_lock(&tasklist_lock);
/*
* Notify parents of the stop.
*
* While ptraced, there are two parents - the ptracer and
* the real_parent of the group_leader. The ptracer should
* know about every stop while the real parent is only
* interested in the completion of group stop. The states
* for the two don't interact with each other. Notify
* separately unless they're gonna be duplicates.
*/
if (current->ptrace)
do_notify_parent_cldstop(current, true, why);
if (gstop_done && (!current->ptrace || ptrace_reparented(current)))
do_notify_parent_cldstop(current, false, why);
/*
* The previous do_notify_parent_cldstop() invocation woke ptracer.
* One a PREEMPTION kernel this can result in preemption requirement
* which will be fulfilled after read_unlock() and the ptracer will be
* put on the CPU.
* The ptracer is in wait_task_inactive(, __TASK_TRACED) waiting for
* this task wait in schedule(). If this task gets preempted then it
* remains enqueued on the runqueue. The ptracer will observe this and
* then sleep for a delay of one HZ tick. In the meantime this task
* gets scheduled, enters schedule() and will wait for the ptracer.
*
* This preemption point is not bad from a correctness point of
* view but extends the runtime by one HZ tick time due to the
* ptracer's sleep. The preempt-disable section ensures that there
* will be no preemption between unlock and schedule() and so
* improving the performance since the ptracer will observe that
* the tracee is scheduled out once it gets on the CPU.
*
* On PREEMPT_RT locking tasklist_lock does not disable preemption.
* Therefore the task can be preempted after do_notify_parent_cldstop()
* before unlocking tasklist_lock so there is no benefit in doing this.
*
* In fact disabling preemption is harmful on PREEMPT_RT because
* the spinlock_t in cgroup_enter_frozen() must not be acquired
* with preemption disabled due to the 'sleeping' spinlock
* substitution of RT.
*/
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_disable();
read_unlock(&tasklist_lock);
cgroup_enter_frozen();
if (!IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_enable_no_resched();
schedule();
cgroup_leave_frozen(true);
/*
* We are back. Now reacquire the siglock before touching
* last_siginfo, so that we are sure to have synchronized with
* any signal-sending on another CPU that wants to examine it.
*/
spin_lock_irq(¤t->sighand->siglock);
exit_code = current->exit_code;
current->last_siginfo = NULL;
current->ptrace_message = 0;
current->exit_code = 0;
/* LISTENING can be set only during STOP traps, clear it */
current->jobctl &= ~(JOBCTL_LISTENING | JOBCTL_PTRACE_FROZEN);
/*
* Queued signals ignored us while we were stopped for tracing.
* So check for any that we should take before resuming user mode.
* This sets TIF_SIGPENDING, but never clears it.
*/
recalc_sigpending_tsk(current);
return exit_code;
}
static int ptrace_do_notify(int signr, int exit_code, int why, unsigned long message)
{
kernel_siginfo_t info;
clear_siginfo(&info);
info.si_signo = signr;
info.si_code = exit_code;
info.si_pid = task_pid_vnr(current);
info.si_uid = from_kuid_munged(current_user_ns(), current_uid());
/* Let the debugger run. */
return ptrace_stop(exit_code, why, message, &info);
}
int ptrace_notify(int exit_code, unsigned long message)
{
int signr;
BUG_ON((exit_code & (0x7f | ~0xffff)) != SIGTRAP);
if (unlikely(task_work_pending(current)))
task_work_run();
spin_lock_irq(¤t->sighand->siglock);
signr = ptrace_do_notify(SIGTRAP, exit_code, CLD_TRAPPED, message);
spin_unlock_irq(¤t->sighand->siglock);
return signr;
}
/**
* do_signal_stop - handle group stop for SIGSTOP and other stop signals
* @signr: signr causing group stop if initiating
*
* If %JOBCTL_STOP_PENDING is not set yet, initiate group stop with @signr
* and participate in it. If already set, participate in the existing
* group stop. If participated in a group stop (and thus slept), %true is
* returned with siglock released.
*
* If ptraced, this function doesn't handle stop itself. Instead,
* %JOBCTL_TRAP_STOP is scheduled and %false is returned with siglock
* untouched. The caller must ensure that INTERRUPT trap handling takes
* places afterwards.
*
* CONTEXT:
* Must be called with @current->sighand->siglock held, which is released
* on %true return.
*
* RETURNS:
* %false if group stop is already cancelled or ptrace trap is scheduled.
* %true if participated in group stop.
*/
static bool do_signal_stop(int signr)
__releases(¤t->sighand->siglock)
{
struct signal_struct *sig = current->signal;
if (!(current->jobctl & JOBCTL_STOP_PENDING)) {
unsigned long gstop = JOBCTL_STOP_PENDING | JOBCTL_STOP_CONSUME;
struct task_struct *t;
/* signr will be recorded in task->jobctl for retries */
WARN_ON_ONCE(signr & ~JOBCTL_STOP_SIGMASK);
if (!likely(current->jobctl & JOBCTL_STOP_DEQUEUED) ||
unlikely(sig->flags & SIGNAL_GROUP_EXIT) ||
unlikely(sig->group_exec_task))
return false;
/*
* There is no group stop already in progress. We must
* initiate one now.
*
* While ptraced, a task may be resumed while group stop is
* still in effect and then receive a stop signal and
* initiate another group stop. This deviates from the
* usual behavior as two consecutive stop signals can't
* cause two group stops when !ptraced. That is why we
* also check !task_is_stopped(t) below.
*
* The condition can be distinguished by testing whether
* SIGNAL_STOP_STOPPED is already set. Don't generate
* group_exit_code in such case.
*
* This is not necessary for SIGNAL_STOP_CONTINUED because
* an intervening stop signal is required to cause two
* continued events regardless of ptrace.
*/
if (!(sig->flags & SIGNAL_STOP_STOPPED))
sig->group_exit_code = signr;
sig->group_stop_count = 0;
if (task_set_jobctl_pending(current, signr | gstop))
sig->group_stop_count++;
for_other_threads(current, t) {
/*
* Setting state to TASK_STOPPED for a group
* stop is always done with the siglock held,
* so this check has no races.
*/
if (!task_is_stopped(t) &&
task_set_jobctl_pending(t, signr | gstop)) {
sig->group_stop_count++;
if (likely(!(t->ptrace & PT_SEIZED)))
signal_wake_up(t, 0);
else
ptrace_trap_notify(t);
}
}
}
if (likely(!current->ptrace)) {
int notify = 0;
/*
* If there are no other threads in the group, or if there
* is a group stop in progress and we are the last to stop,
* report to the parent.
*/
if (task_participate_group_stop(current))
notify = CLD_STOPPED;
current->jobctl |= JOBCTL_STOPPED;
set_special_state(TASK_STOPPED);
spin_unlock_irq(¤t->sighand->siglock);
/*
* Notify the parent of the group stop completion. Because
* we're not holding either the siglock or tasklist_lock
* here, ptracer may attach inbetween; however, this is for
* group stop and should always be delivered to the real
* parent of the group leader. The new ptracer will get
* its notification when this task transitions into
* TASK_TRACED.
*/
if (notify) {
read_lock(&tasklist_lock);
do_notify_parent_cldstop(current, false, notify);
read_unlock(&tasklist_lock);
}
/* Now we don't run again until woken by SIGCONT or SIGKILL */
cgroup_enter_frozen();
schedule();
return true;
} else {
/*
* While ptraced, group stop is handled by STOP trap.
* Schedule it and let the caller deal with it.
*/
task_set_jobctl_pending(current, JOBCTL_TRAP_STOP);
return false;
}
}
/**
* do_jobctl_trap - take care of ptrace jobctl traps
*
* When PT_SEIZED, it's used for both group stop and explicit
* SEIZE/INTERRUPT traps. Both generate PTRACE_EVENT_STOP trap with
* accompanying siginfo. If stopped, lower eight bits of exit_code contain
* the stop signal; otherwise, %SIGTRAP.
*
* When !PT_SEIZED, it's used only for group stop trap with stop signal
* number as exit_code and no siginfo.
*
* CONTEXT:
* Must be called with @current->sighand->siglock held, which may be
* released and re-acquired before returning with intervening sleep.
*/
static void do_jobctl_trap(void)
{
struct signal_struct *signal = current->signal;
int signr = current->jobctl & JOBCTL_STOP_SIGMASK;
if (current->ptrace & PT_SEIZED) {
if (!signal->group_stop_count &&
!(signal->flags & SIGNAL_STOP_STOPPED))
signr = SIGTRAP; WARN_ON_ONCE(!signr); ptrace_do_notify(signr, signr | (PTRACE_EVENT_STOP << 8),
CLD_STOPPED, 0);
} else {
WARN_ON_ONCE(!signr); ptrace_stop(signr, CLD_STOPPED, 0, NULL);
}
}
/**
* do_freezer_trap - handle the freezer jobctl trap
*
* Puts the task into frozen state, if only the task is not about to quit.
* In this case it drops JOBCTL_TRAP_FREEZE.
*
* CONTEXT:
* Must be called with @current->sighand->siglock held,
* which is always released before returning.
*/
static void do_freezer_trap(void)
__releases(¤t->sighand->siglock)
{
/*
* If there are other trap bits pending except JOBCTL_TRAP_FREEZE,
* let's make another loop to give it a chance to be handled.
* In any case, we'll return back.
*/
if ((current->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) !=
JOBCTL_TRAP_FREEZE) {
spin_unlock_irq(¤t->sighand->siglock);
return;
}
/*
* Now we're sure that there is no pending fatal signal and no
* pending traps. Clear TIF_SIGPENDING to not get out of schedule()
* immediately (if there is a non-fatal signal pending), and
* put the task into sleep.
*/
__set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
clear_thread_flag(TIF_SIGPENDING);
spin_unlock_irq(¤t->sighand->siglock);
cgroup_enter_frozen();
schedule();
}
static int ptrace_signal(int signr, kernel_siginfo_t *info, enum pid_type type)
{
/*
* We do not check sig_kernel_stop(signr) but set this marker
* unconditionally because we do not know whether debugger will
* change signr. This flag has no meaning unless we are going
* to stop after return from ptrace_stop(). In this case it will
* be checked in do_signal_stop(), we should only stop if it was
* not cleared by SIGCONT while we were sleeping. See also the
* comment in dequeue_signal().
*/
current->jobctl |= JOBCTL_STOP_DEQUEUED;
signr = ptrace_stop(signr, CLD_TRAPPED, 0, info);
/* We're back. Did the debugger cancel the sig? */
if (signr == 0)
return signr;
/*
* Update the siginfo structure if the signal has
* changed. If the debugger wanted something
* specific in the siginfo structure then it should
* have updated *info via PTRACE_SETSIGINFO.
*/
if (signr != info->si_signo) { clear_siginfo(info);
info->si_signo = signr;
info->si_errno = 0;
info->si_code = SI_USER;
rcu_read_lock(); info->si_pid = task_pid_vnr(current->parent);
info->si_uid = from_kuid_munged(current_user_ns(),
task_uid(current->parent)); rcu_read_unlock();
}
/* If the (new) signal is now blocked, requeue it. */
if (sigismember(¤t->blocked, signr) || fatal_signal_pending(current)) { send_signal_locked(signr, info, current, type);
signr = 0;
}
return signr;
}
static void hide_si_addr_tag_bits(struct ksignal *ksig)
{
switch (siginfo_layout(ksig->sig, ksig->info.si_code)) {
case SIL_FAULT:
case SIL_FAULT_TRAPNO:
case SIL_FAULT_MCEERR:
case SIL_FAULT_BNDERR:
case SIL_FAULT_PKUERR:
case SIL_FAULT_PERF_EVENT:
ksig->info.si_addr = arch_untagged_si_addr(
ksig->info.si_addr, ksig->sig, ksig->info.si_code);
break;
case SIL_KILL:
case SIL_TIMER:
case SIL_POLL:
case SIL_CHLD:
case SIL_RT:
case SIL_SYS:
break;
}
}
bool get_signal(struct ksignal *ksig)
{
struct sighand_struct *sighand = current->sighand;
struct signal_struct *signal = current->signal;
int signr;
clear_notify_signal();
if (unlikely(task_work_pending(current)))
task_work_run(); if (!task_sigpending(current)) return false; if (unlikely(uprobe_deny_signal()))
return false;
/*
* Do this once, we can't return to user-mode if freezing() == T.
* do_signal_stop() and ptrace_stop() do freezable_schedule() and
* thus do not need another check after return.
*/
try_to_freeze();
relock:
spin_lock_irq(&sighand->siglock);
/*
* Every stopped thread goes here after wakeup. Check to see if
* we should notify the parent, prepare_signal(SIGCONT) encodes
* the CLD_ si_code into SIGNAL_CLD_MASK bits.
*/
if (unlikely(signal->flags & SIGNAL_CLD_MASK)) {
int why;
if (signal->flags & SIGNAL_CLD_CONTINUED)
why = CLD_CONTINUED;
else
why = CLD_STOPPED;
signal->flags &= ~SIGNAL_CLD_MASK;
spin_unlock_irq(&sighand->siglock);
/*
* Notify the parent that we're continuing. This event is
* always per-process and doesn't make whole lot of sense
* for ptracers, who shouldn't consume the state via
* wait(2) either, but, for backward compatibility, notify
* the ptracer of the group leader too unless it's gonna be
* a duplicate.
*/
read_lock(&tasklist_lock);
do_notify_parent_cldstop(current, false, why);
if (ptrace_reparented(current->group_leader))
do_notify_parent_cldstop(current->group_leader,
true, why);
read_unlock(&tasklist_lock);
goto relock;
}
for (;;) {
struct k_sigaction *ka;
enum pid_type type;
/* Has this task already been marked for death? */
if ((signal->flags & SIGNAL_GROUP_EXIT) || signal->group_exec_task) {
signr = SIGKILL;
sigdelset(¤t->pending.signal, SIGKILL);
trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO,
&sighand->action[SIGKILL-1]);
recalc_sigpending();
/*
* implies do_group_exit() or return to PF_USER_WORKER,
* no need to initialize ksig->info/etc.
*/
goto fatal;
}
if (unlikely(current->jobctl & JOBCTL_STOP_PENDING) && do_signal_stop(0)) goto relock; if (unlikely(current->jobctl &
(JOBCTL_TRAP_MASK | JOBCTL_TRAP_FREEZE))) {
if (current->jobctl & JOBCTL_TRAP_MASK) { do_jobctl_trap(); spin_unlock_irq(&sighand->siglock); } else if (current->jobctl & JOBCTL_TRAP_FREEZE) do_freezer_trap();
goto relock;
}
/*
* If the task is leaving the frozen state, let's update
* cgroup counters and reset the frozen bit.
*/
if (unlikely(cgroup_task_frozen(current))) { spin_unlock_irq(&sighand->siglock);
cgroup_leave_frozen(false);
goto relock;
}
/*
* Signals generated by the execution of an instruction
* need to be delivered before any other pending signals
* so that the instruction pointer in the signal stack
* frame points to the faulting instruction.
*/
type = PIDTYPE_PID; signr = dequeue_synchronous_signal(&ksig->info); if (!signr) signr = dequeue_signal(current, ¤t->blocked,
&ksig->info, &type);
if (!signr)
break; /* will return 0 */
if (unlikely(current->ptrace) && (signr != SIGKILL) && !(sighand->action[signr -1].sa.sa_flags & SA_IMMUTABLE)) { signr = ptrace_signal(signr, &ksig->info, type);
if (!signr)
continue;
}
ka = &sighand->action[signr-1];
/* Trace actually delivered signals. */
trace_signal_deliver(signr, &ksig->info, ka);
if (ka->sa.sa_handler == SIG_IGN) /* Do nothing. */
continue;
if (ka->sa.sa_handler != SIG_DFL) {
/* Run the handler. */
ksig->ka = *ka;
if (ka->sa.sa_flags & SA_ONESHOT)
ka->sa.sa_handler = SIG_DFL;
break; /* will return non-zero "signr" value */
}
/*
* Now we are doing the default action for this signal.
*/
if (sig_kernel_ignore(signr)) /* Default is nothing. */
continue;
/*
* Global init gets no signals it doesn't want.
* Container-init gets no signals it doesn't want from same
* container.
*
* Note that if global/container-init sees a sig_kernel_only()
* signal here, the signal must have been generated internally
* or must have come from an ancestor namespace. In either
* case, the signal cannot be dropped.
*/
if (unlikely(signal->flags & SIGNAL_UNKILLABLE) && !sig_kernel_only(signr))
continue;
if (sig_kernel_stop(signr)) {
/*
* The default action is to stop all threads in
* the thread group. The job control signals
* do nothing in an orphaned pgrp, but SIGSTOP
* always works. Note that siglock needs to be
* dropped during the call to is_orphaned_pgrp()
* because of lock ordering with tasklist_lock.
* This allows an intervening SIGCONT to be posted.
* We need to check for that and bail out if necessary.
*/
if (signr != SIGSTOP) { spin_unlock_irq(&sighand->siglock);
/* signals can be posted during this window */
if (is_current_pgrp_orphaned())
goto relock;
spin_lock_irq(&sighand->siglock);
}
if (likely(do_signal_stop(signr))) {
/* It released the siglock. */
goto relock;
}
/*
* We didn't actually stop, due to a race
* with SIGCONT or something like that.
*/
continue;
}
fatal:
spin_unlock_irq(&sighand->siglock);
if (unlikely(cgroup_task_frozen(current)))
cgroup_leave_frozen(true);
/*
* Anything else is fatal, maybe with a core dump.
*/
current->flags |= PF_SIGNALED; if (sig_kernel_coredump(signr)) { if (print_fatal_signals) print_fatal_signal(signr); proc_coredump_connector(current);
/*
* If it was able to dump core, this kills all
* other threads in the group and synchronizes with
* their demise. If we lost the race with another
* thread getting here, it set group_exit_code
* first and our do_group_exit call below will use
* that value and ignore the one we pass it.
*/
do_coredump(&ksig->info);
}
/*
* PF_USER_WORKER threads will catch and exit on fatal signals
* themselves. They have cleanup that must be performed, so we
* cannot call do_exit() on their behalf. Note that ksig won't
* be properly initialized, PF_USER_WORKER's shouldn't use it.
*/
if (current->flags & PF_USER_WORKER) goto out;
/*
* Death signals, no core dump.
*/
do_group_exit(signr);
/* NOTREACHED */
}
spin_unlock_irq(&sighand->siglock);
ksig->sig = signr; if (signr && !(ksig->ka.sa.sa_flags & SA_EXPOSE_TAGBITS)) hide_si_addr_tag_bits(ksig);
out:
return signr > 0;
}
/**
* signal_delivered - called after signal delivery to update blocked signals
* @ksig: kernel signal struct
* @stepping: nonzero if debugger single-step or block-step in use
*
* This function should be called when a signal has successfully been
* delivered. It updates the blocked signals accordingly (@ksig->ka.sa.sa_mask
* is always blocked), and the signal itself is blocked unless %SA_NODEFER
* is set in @ksig->ka.sa.sa_flags. Tracing is notified.
*/
static void signal_delivered(struct ksignal *ksig, int stepping)
{
sigset_t blocked;
/* A signal was successfully delivered, and the
saved sigmask was stored on the signal frame,
and will be restored by sigreturn. So we can
simply clear the restore sigmask flag. */
clear_restore_sigmask();
sigorsets(&blocked, ¤t->blocked, &ksig->ka.sa.sa_mask);
if (!(ksig->ka.sa.sa_flags & SA_NODEFER))
sigaddset(&blocked, ksig->sig); set_current_blocked(&blocked); if (current->sas_ss_flags & SS_AUTODISARM) sas_ss_reset(current); if (stepping) ptrace_notify(SIGTRAP, 0);
}
void signal_setup_done(int failed, struct ksignal *ksig, int stepping)
{
if (failed) force_sigsegv(ksig->sig);
else
signal_delivered(ksig, stepping);}
/*
* It could be that complete_signal() picked us to notify about the
* group-wide signal. Other threads should be notified now to take
* the shared signals in @which since we will not.
*/
static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
{
sigset_t retarget;
struct task_struct *t;
sigandsets(&retarget, &tsk->signal->shared_pending.signal, which);
if (sigisemptyset(&retarget))
return;
for_other_threads(tsk, t) {
if (t->flags & PF_EXITING)
continue;
if (!has_pending_signals(&retarget, &t->blocked))
continue;
/* Remove the signals this thread can handle. */
sigandsets(&retarget, &retarget, &t->blocked);
if (!task_sigpending(t))
signal_wake_up(t, 0);
if (sigisemptyset(&retarget))
break;
}
}
void exit_signals(struct task_struct *tsk)
{
int group_stop = 0;
sigset_t unblocked;
/*
* @tsk is about to have PF_EXITING set - lock out users which
* expect stable threadgroup.
*/
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
}
spin_lock_irq(&tsk->sighand->siglock);
/*
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
if (!task_sigpending(tsk))
goto out;
unblocked = tsk->blocked;
signotset(&unblocked);
retarget_shared_pending(tsk, &unblocked);
if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) &&
task_participate_group_stop(tsk))
group_stop = CLD_STOPPED;
out:
spin_unlock_irq(&tsk->sighand->siglock);
/*
* If group stop has completed, deliver the notification. This
* should always go to the real parent of the group leader.
*/
if (unlikely(group_stop)) {
read_lock(&tasklist_lock);
do_notify_parent_cldstop(tsk, false, group_stop);
read_unlock(&tasklist_lock);
}
}
/*
* System call entry points.
*/
/**
* sys_restart_syscall - restart a system call
*/
SYSCALL_DEFINE0(restart_syscall)
{
struct restart_block *restart = ¤t->restart_block;
return restart->fn(restart);
}
long do_no_restart_syscall(struct restart_block *param)
{
return -EINTR;
}
static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset)
{
if (task_sigpending(tsk) && !thread_group_empty(tsk)) {
sigset_t newblocked;
/* A set of now blocked but previously unblocked signals. */
sigandnsets(&newblocked, newset, ¤t->blocked);
retarget_shared_pending(tsk, &newblocked);
}
tsk->blocked = *newset;
recalc_sigpending();
}
/**
* set_current_blocked - change current->blocked mask
* @newset: new mask
*
* It is wrong to change ->blocked directly, this helper should be used
* to ensure the process can't miss a shared signal we are going to block.
*/
void set_current_blocked(sigset_t *newset)
{
sigdelsetmask(newset, sigmask(SIGKILL) | sigmask(SIGSTOP)); __set_current_blocked(newset);
}
void __set_current_blocked(const sigset_t *newset)
{
struct task_struct *tsk = current;
/*
* In case the signal mask hasn't changed, there is nothing we need
* to do. The current->blocked shouldn't be modified by other task.
*/
if (sigequalsets(&tsk->blocked, newset))
return;
spin_lock_irq(&tsk->sighand->siglock);
__set_task_blocked(tsk, newset);
spin_unlock_irq(&tsk->sighand->siglock);
}
/*
* This is also useful for kernel threads that want to temporarily
* (or permanently) block certain signals.
*
* NOTE! Unlike the user-mode sys_sigprocmask(), the kernel
* interface happily blocks "unblockable" signals like SIGKILL
* and friends.
*/
int sigprocmask(int how, sigset_t *set, sigset_t *oldset)
{
struct task_struct *tsk = current;
sigset_t newset;
/* Lockless, only current can change ->blocked, never from irq */
if (oldset)
*oldset = tsk->blocked;
switch (how) {
case SIG_BLOCK:
sigorsets(&newset, &tsk->blocked, set);
break;
case SIG_UNBLOCK:
sigandnsets(&newset, &tsk->blocked, set);
break;
case SIG_SETMASK:
newset = *set;
break;
default:
return -EINVAL;
}
__set_current_blocked(&newset);
return 0;
}
EXPORT_SYMBOL(sigprocmask);
/*
* The api helps set app-provided sigmasks.
*
* This is useful for syscalls such as ppoll, pselect, io_pgetevents and
* epoll_pwait where a new sigmask is passed from userland for the syscalls.
*
* Note that it does set_restore_sigmask() in advance, so it must be always
* paired with restore_saved_sigmask_unless() before return from syscall.
*/
int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize)
{
sigset_t kmask;
if (!umask)
return 0;
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (copy_from_user(&kmask, umask, sizeof(sigset_t)))
return -EFAULT;
set_restore_sigmask();
current->saved_sigmask = current->blocked;
set_current_blocked(&kmask);
return 0;
}
#ifdef CONFIG_COMPAT
int set_compat_user_sigmask(const compat_sigset_t __user *umask,
size_t sigsetsize)
{
sigset_t kmask;
if (!umask)
return 0;
if (sigsetsize != sizeof(compat_sigset_t))
return -EINVAL;
if (get_compat_sigset(&kmask, umask))
return -EFAULT;
set_restore_sigmask();
current->saved_sigmask = current->blocked;
set_current_blocked(&kmask);
return 0;
}
#endif
/**
* sys_rt_sigprocmask - change the list of currently blocked signals
* @how: whether to add, remove, or set signals
* @nset: stores pending signals
* @oset: previous value of signal mask if non-null
* @sigsetsize: size of sigset_t type
*/
SYSCALL_DEFINE4(rt_sigprocmask, int, how, sigset_t __user *, nset,
sigset_t __user *, oset, size_t, sigsetsize)
{
sigset_t old_set, new_set;
int error;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
old_set = current->blocked;
if (nset) {
if (copy_from_user(&new_set, nset, sizeof(sigset_t)))
return -EFAULT;
sigdelsetmask(&new_set, sigmask(SIGKILL)|sigmask(SIGSTOP));
error = sigprocmask(how, &new_set, NULL);
if (error)
return error;
}
if (oset) {
if (copy_to_user(oset, &old_set, sizeof(sigset_t)))
return -EFAULT;
}
return 0;
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigprocmask, int, how, compat_sigset_t __user *, nset,
compat_sigset_t __user *, oset, compat_size_t, sigsetsize)
{
sigset_t old_set = current->blocked;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (nset) {
sigset_t new_set;
int error;
if (get_compat_sigset(&new_set, nset))
return -EFAULT;
sigdelsetmask(&new_set, sigmask(SIGKILL)|sigmask(SIGSTOP));
error = sigprocmask(how, &new_set, NULL);
if (error)
return error;
}
return oset ? put_compat_sigset(oset, &old_set, sizeof(*oset)) : 0;
}
#endif
static void do_sigpending(sigset_t *set)
{
spin_lock_irq(¤t->sighand->siglock);
sigorsets(set, ¤t->pending.signal,
¤t->signal->shared_pending.signal);
spin_unlock_irq(¤t->sighand->siglock);
/* Outside the lock because only this thread touches it. */
sigandsets(set, ¤t->blocked, set);
}
/**
* sys_rt_sigpending - examine a pending signal that has been raised
* while blocked
* @uset: stores pending signals
* @sigsetsize: size of sigset_t type or larger
*/
SYSCALL_DEFINE2(rt_sigpending, sigset_t __user *, uset, size_t, sigsetsize)
{
sigset_t set;
if (sigsetsize > sizeof(*uset))
return -EINVAL;
do_sigpending(&set);
if (copy_to_user(uset, &set, sigsetsize))
return -EFAULT;
return 0;
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(rt_sigpending, compat_sigset_t __user *, uset,
compat_size_t, sigsetsize)
{
sigset_t set;
if (sigsetsize > sizeof(*uset))
return -EINVAL;
do_sigpending(&set);
return put_compat_sigset(uset, &set, sigsetsize);
}
#endif
static const struct {
unsigned char limit, layout;
} sig_sicodes[] = {
[SIGILL] = { NSIGILL, SIL_FAULT },
[SIGFPE] = { NSIGFPE, SIL_FAULT },
[SIGSEGV] = { NSIGSEGV, SIL_FAULT },
[SIGBUS] = { NSIGBUS, SIL_FAULT },
[SIGTRAP] = { NSIGTRAP, SIL_FAULT },
#if defined(SIGEMT)
[SIGEMT] = { NSIGEMT, SIL_FAULT },
#endif
[SIGCHLD] = { NSIGCHLD, SIL_CHLD },
[SIGPOLL] = { NSIGPOLL, SIL_POLL },
[SIGSYS] = { NSIGSYS, SIL_SYS },
};
static bool known_siginfo_layout(unsigned sig, int si_code)
{
if (si_code == SI_KERNEL)
return true;
else if ((si_code > SI_USER)) {
if (sig_specific_sicodes(sig)) {
if (si_code <= sig_sicodes[sig].limit)
return true;
}
else if (si_code <= NSIGPOLL)
return true;
}
else if (si_code >= SI_DETHREAD)
return true;
else if (si_code == SI_ASYNCNL)
return true;
return false;
}
enum siginfo_layout siginfo_layout(unsigned sig, int si_code)
{
enum siginfo_layout layout = SIL_KILL;
if ((si_code > SI_USER) && (si_code < SI_KERNEL)) { if ((sig < ARRAY_SIZE(sig_sicodes)) && (si_code <= sig_sicodes[sig].limit)) { layout = sig_sicodes[sig].layout;
/* Handle the exceptions */
if ((sig == SIGBUS) &&
(si_code >= BUS_MCEERR_AR) && (si_code <= BUS_MCEERR_AO))
layout = SIL_FAULT_MCEERR;
else if ((sig == SIGSEGV) && (si_code == SEGV_BNDERR))
layout = SIL_FAULT_BNDERR;
#ifdef SEGV_PKUERR
else if ((sig == SIGSEGV) && (si_code == SEGV_PKUERR))
layout = SIL_FAULT_PKUERR;
#endif
else if ((sig == SIGTRAP) && (si_code == TRAP_PERF))
layout = SIL_FAULT_PERF_EVENT;
else if (IS_ENABLED(CONFIG_SPARC) &&
(sig == SIGILL) && (si_code == ILL_ILLTRP))
layout = SIL_FAULT_TRAPNO;
else if (IS_ENABLED(CONFIG_ALPHA) &&
((sig == SIGFPE) ||
((sig == SIGTRAP) && (si_code == TRAP_UNK))))
layout = SIL_FAULT_TRAPNO;
}
else if (si_code <= NSIGPOLL)
layout = SIL_POLL;
} else {
if (si_code == SI_TIMER)
layout = SIL_TIMER;
else if (si_code == SI_SIGIO)
layout = SIL_POLL;
else if (si_code < 0)
layout = SIL_RT;
}
return layout;
}
static inline char __user *si_expansion(const siginfo_t __user *info)
{
return ((char __user *)info) + sizeof(struct kernel_siginfo);
}
int copy_siginfo_to_user(siginfo_t __user *to, const kernel_siginfo_t *from)
{
char __user *expansion = si_expansion(to);
if (copy_to_user(to, from , sizeof(struct kernel_siginfo)))
return -EFAULT;
if (clear_user(expansion, SI_EXPANSION_SIZE))
return -EFAULT;
return 0;
}
static int post_copy_siginfo_from_user(kernel_siginfo_t *info,
const siginfo_t __user *from)
{
if (unlikely(!known_siginfo_layout(info->si_signo, info->si_code))) {
char __user *expansion = si_expansion(from);
char buf[SI_EXPANSION_SIZE];
int i;
/*
* An unknown si_code might need more than
* sizeof(struct kernel_siginfo) bytes. Verify all of the
* extra bytes are 0. This guarantees copy_siginfo_to_user
* will return this data to userspace exactly.
*/
if (copy_from_user(&buf, expansion, SI_EXPANSION_SIZE))
return -EFAULT;
for (i = 0; i < SI_EXPANSION_SIZE; i++) {
if (buf[i] != 0)
return -E2BIG;
}
}
return 0;
}
static int __copy_siginfo_from_user(int signo, kernel_siginfo_t *to,
const siginfo_t __user *from)
{
if (copy_from_user(to, from, sizeof(struct kernel_siginfo)))
return -EFAULT;
to->si_signo = signo;
return post_copy_siginfo_from_user(to, from);
}
int copy_siginfo_from_user(kernel_siginfo_t *to, const siginfo_t __user *from)
{
if (copy_from_user(to, from, sizeof(struct kernel_siginfo)))
return -EFAULT;
return post_copy_siginfo_from_user(to, from);
}
#ifdef CONFIG_COMPAT
/**
* copy_siginfo_to_external32 - copy a kernel siginfo into a compat user siginfo
* @to: compat siginfo destination
* @from: kernel siginfo source
*
* Note: This function does not work properly for the SIGCHLD on x32, but
* fortunately it doesn't have to. The only valid callers for this function are
* copy_siginfo_to_user32, which is overriden for x32 and the coredump code.
* The latter does not care because SIGCHLD will never cause a coredump.
*/
void copy_siginfo_to_external32(struct compat_siginfo *to,
const struct kernel_siginfo *from)
{
memset(to, 0, sizeof(*to));
to->si_signo = from->si_signo;
to->si_errno = from->si_errno;
to->si_code = from->si_code;
switch(siginfo_layout(from->si_signo, from->si_code)) {
case SIL_KILL:
to->si_pid = from->si_pid;
to->si_uid = from->si_uid;
break;
case SIL_TIMER:
to->si_tid = from->si_tid;
to->si_overrun = from->si_overrun;
to->si_int = from->si_int;
break;
case SIL_POLL:
to->si_band = from->si_band;
to->si_fd = from->si_fd;
break;
case SIL_FAULT:
to->si_addr = ptr_to_compat(from->si_addr);
break;
case SIL_FAULT_TRAPNO:
to->si_addr = ptr_to_compat(from->si_addr);
to->si_trapno = from->si_trapno;
break;
case SIL_FAULT_MCEERR:
to->si_addr = ptr_to_compat(from->si_addr);
to->si_addr_lsb = from->si_addr_lsb;
break;
case SIL_FAULT_BNDERR:
to->si_addr = ptr_to_compat(from->si_addr);
to->si_lower = ptr_to_compat(from->si_lower);
to->si_upper = ptr_to_compat(from->si_upper);
break;
case SIL_FAULT_PKUERR:
to->si_addr = ptr_to_compat(from->si_addr);
to->si_pkey = from->si_pkey;
break;
case SIL_FAULT_PERF_EVENT:
to->si_addr = ptr_to_compat(from->si_addr);
to->si_perf_data = from->si_perf_data;
to->si_perf_type = from->si_perf_type;
to->si_perf_flags = from->si_perf_flags;
break;
case SIL_CHLD:
to->si_pid = from->si_pid;
to->si_uid = from->si_uid;
to->si_status = from->si_status;
to->si_utime = from->si_utime;
to->si_stime = from->si_stime;
break;
case SIL_RT:
to->si_pid = from->si_pid;
to->si_uid = from->si_uid;
to->si_int = from->si_int;
break;
case SIL_SYS:
to->si_call_addr = ptr_to_compat(from->si_call_addr);
to->si_syscall = from->si_syscall;
to->si_arch = from->si_arch;
break;
}
}
int __copy_siginfo_to_user32(struct compat_siginfo __user *to,
const struct kernel_siginfo *from)
{
struct compat_siginfo new;
copy_siginfo_to_external32(&new, from);
if (copy_to_user(to, &new, sizeof(struct compat_siginfo)))
return -EFAULT;
return 0;
}
static int post_copy_siginfo_from_user32(kernel_siginfo_t *to,
const struct compat_siginfo *from)
{
clear_siginfo(to);
to->si_signo = from->si_signo;
to->si_errno = from->si_errno;
to->si_code = from->si_code;
switch(siginfo_layout(from->si_signo, from->si_code)) {
case SIL_KILL:
to->si_pid = from->si_pid;
to->si_uid = from->si_uid;
break;
case SIL_TIMER:
to->si_tid = from->si_tid;
to->si_overrun = from->si_overrun;
to->si_int = from->si_int;
break;
case SIL_POLL:
to->si_band = from->si_band;
to->si_fd = from->si_fd;
break;
case SIL_FAULT:
to->si_addr = compat_ptr(from->si_addr);
break;
case SIL_FAULT_TRAPNO:
to->si_addr = compat_ptr(from->si_addr);
to->si_trapno = from->si_trapno;
break;
case SIL_FAULT_MCEERR:
to->si_addr = compat_ptr(from->si_addr);
to->si_addr_lsb = from->si_addr_lsb;
break;
case SIL_FAULT_BNDERR:
to->si_addr = compat_ptr(from->si_addr);
to->si_lower = compat_ptr(from->si_lower);
to->si_upper = compat_ptr(from->si_upper);
break;
case SIL_FAULT_PKUERR:
to->si_addr = compat_ptr(from->si_addr);
to->si_pkey = from->si_pkey;
break;
case SIL_FAULT_PERF_EVENT:
to->si_addr = compat_ptr(from->si_addr);
to->si_perf_data = from->si_perf_data;
to->si_perf_type = from->si_perf_type;
to->si_perf_flags = from->si_perf_flags;
break;
case SIL_CHLD:
to->si_pid = from->si_pid;
to->si_uid = from->si_uid;
to->si_status = from->si_status;
#ifdef CONFIG_X86_X32_ABI
if (in_x32_syscall()) {
to->si_utime = from->_sifields._sigchld_x32._utime;
to->si_stime = from->_sifields._sigchld_x32._stime;
} else
#endif
{
to->si_utime = from->si_utime;
to->si_stime = from->si_stime;
}
break;
case SIL_RT:
to->si_pid = from->si_pid;
to->si_uid = from->si_uid;
to->si_int = from->si_int;
break;
case SIL_SYS:
to->si_call_addr = compat_ptr(from->si_call_addr);
to->si_syscall = from->si_syscall;
to->si_arch = from->si_arch;
break;
}
return 0;
}
static int __copy_siginfo_from_user32(int signo, struct kernel_siginfo *to,
const struct compat_siginfo __user *ufrom)
{
struct compat_siginfo from;
if (copy_from_user(&from, ufrom, sizeof(struct compat_siginfo)))
return -EFAULT;
from.si_signo = signo;
return post_copy_siginfo_from_user32(to, &from);
}
int copy_siginfo_from_user32(struct kernel_siginfo *to,
const struct compat_siginfo __user *ufrom)
{
struct compat_siginfo from;
if (copy_from_user(&from, ufrom, sizeof(struct compat_siginfo)))
return -EFAULT;
return post_copy_siginfo_from_user32(to, &from);
}
#endif /* CONFIG_COMPAT */
/**
* do_sigtimedwait - wait for queued signals specified in @which
* @which: queued signals to wait for
* @info: if non-null, the signal's siginfo is returned here
* @ts: upper bound on process time suspension
*/
static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info,
const struct timespec64 *ts)
{
ktime_t *to = NULL, timeout = KTIME_MAX;
struct task_struct *tsk = current;
sigset_t mask = *which;
enum pid_type type;
int sig, ret = 0;
if (ts) {
if (!timespec64_valid(ts))
return -EINVAL;
timeout = timespec64_to_ktime(*ts);
to = &timeout;
}
/*
* Invert the set of allowed signals to get those we want to block.
*/
sigdelsetmask(&mask, sigmask(SIGKILL) | sigmask(SIGSTOP));
signotset(&mask);
spin_lock_irq(&tsk->sighand->siglock);
sig = dequeue_signal(tsk, &mask, info, &type);
if (!sig && timeout) {
/*
* None ready, temporarily unblock those we're interested
* while we are sleeping in so that we'll be awakened when
* they arrive. Unblocking is always fine, we can avoid
* set_current_blocked().
*/
tsk->real_blocked = tsk->blocked;
sigandsets(&tsk->blocked, &tsk->blocked, &mask);
recalc_sigpending();
spin_unlock_irq(&tsk->sighand->siglock);
__set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
ret = schedule_hrtimeout_range(to, tsk->timer_slack_ns,
HRTIMER_MODE_REL);
spin_lock_irq(&tsk->sighand->siglock);
__set_task_blocked(tsk, &tsk->real_blocked);
sigemptyset(&tsk->real_blocked);
sig = dequeue_signal(tsk, &mask, info, &type);
}
spin_unlock_irq(&tsk->sighand->siglock);
if (sig)
return sig;
return ret ? -EINTR : -EAGAIN;
}
/**
* sys_rt_sigtimedwait - synchronously wait for queued signals specified
* in @uthese
* @uthese: queued signals to wait for
* @uinfo: if non-null, the signal's siginfo is returned here
* @uts: upper bound on process time suspension
* @sigsetsize: size of sigset_t type
*/
SYSCALL_DEFINE4(rt_sigtimedwait, const sigset_t __user *, uthese,
siginfo_t __user *, uinfo,
const struct __kernel_timespec __user *, uts,
size_t, sigsetsize)
{
sigset_t these;
struct timespec64 ts;
kernel_siginfo_t info;
int ret;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (copy_from_user(&these, uthese, sizeof(these)))
return -EFAULT;
if (uts) {
if (get_timespec64(&ts, uts))
return -EFAULT;
}
ret = do_sigtimedwait(&these, &info, uts ? &ts : NULL);
if (ret > 0 && uinfo) {
if (copy_siginfo_to_user(uinfo, &info))
ret = -EFAULT;
}
return ret;
}
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE4(rt_sigtimedwait_time32, const sigset_t __user *, uthese,
siginfo_t __user *, uinfo,
const struct old_timespec32 __user *, uts,
size_t, sigsetsize)
{
sigset_t these;
struct timespec64 ts;
kernel_siginfo_t info;
int ret;
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (copy_from_user(&these, uthese, sizeof(these)))
return -EFAULT;
if (uts) {
if (get_old_timespec32(&ts, uts))
return -EFAULT;
}
ret = do_sigtimedwait(&these, &info, uts ? &ts : NULL);
if (ret > 0 && uinfo) {
if (copy_siginfo_to_user(uinfo, &info))
ret = -EFAULT;
}
return ret;
}
#endif
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time64, compat_sigset_t __user *, uthese,
struct compat_siginfo __user *, uinfo,
struct __kernel_timespec __user *, uts, compat_size_t, sigsetsize)
{
sigset_t s;
struct timespec64 t;
kernel_siginfo_t info;
long ret;
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (get_compat_sigset(&s, uthese))
return -EFAULT;
if (uts) {
if (get_timespec64(&t, uts))
return -EFAULT;
}
ret = do_sigtimedwait(&s, &info, uts ? &t : NULL);
if (ret > 0 && uinfo) {
if (copy_siginfo_to_user32(uinfo, &info))
ret = -EFAULT;
}
return ret;
}
#ifdef CONFIG_COMPAT_32BIT_TIME
COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time32, compat_sigset_t __user *, uthese,
struct compat_siginfo __user *, uinfo,
struct old_timespec32 __user *, uts, compat_size_t, sigsetsize)
{
sigset_t s;
struct timespec64 t;
kernel_siginfo_t info;
long ret;
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (get_compat_sigset(&s, uthese))
return -EFAULT;
if (uts) {
if (get_old_timespec32(&t, uts))
return -EFAULT;
}
ret = do_sigtimedwait(&s, &info, uts ? &t : NULL);
if (ret > 0 && uinfo) {
if (copy_siginfo_to_user32(uinfo, &info))
ret = -EFAULT;
}
return ret;
}
#endif
#endif
static void prepare_kill_siginfo(int sig, struct kernel_siginfo *info,
enum pid_type type)
{
clear_siginfo(info);
info->si_signo = sig;
info->si_errno = 0;
info->si_code = (type == PIDTYPE_PID) ? SI_TKILL : SI_USER;
info->si_pid = task_tgid_vnr(current);
info->si_uid = from_kuid_munged(current_user_ns(), current_uid());
}
/**
* sys_kill - send a signal to a process
* @pid: the PID of the process
* @sig: signal to be sent
*/
SYSCALL_DEFINE2(kill, pid_t, pid, int, sig)
{
struct kernel_siginfo info;
prepare_kill_siginfo(sig, &info, PIDTYPE_TGID);
return kill_something_info(sig, &info, pid);
}
/*
* Verify that the signaler and signalee either are in the same pid namespace
* or that the signaler's pid namespace is an ancestor of the signalee's pid
* namespace.
*/
static bool access_pidfd_pidns(struct pid *pid)
{
struct pid_namespace *active = task_active_pid_ns(current);
struct pid_namespace *p = ns_of_pid(pid);
for (;;) {
if (!p)
return false;
if (p == active)
break;
p = p->parent;
}
return true;
}
static int copy_siginfo_from_user_any(kernel_siginfo_t *kinfo,
siginfo_t __user *info)
{
#ifdef CONFIG_COMPAT
/*
* Avoid hooking up compat syscalls and instead handle necessary
* conversions here. Note, this is a stop-gap measure and should not be
* considered a generic solution.
*/
if (in_compat_syscall())
return copy_siginfo_from_user32(
kinfo, (struct compat_siginfo __user *)info);
#endif
return copy_siginfo_from_user(kinfo, info);
}
static struct pid *pidfd_to_pid(const struct file *file)
{
struct pid *pid;
pid = pidfd_pid(file);
if (!IS_ERR(pid))
return pid;
return tgid_pidfd_to_pid(file);
}
#define PIDFD_SEND_SIGNAL_FLAGS \
(PIDFD_SIGNAL_THREAD | PIDFD_SIGNAL_THREAD_GROUP | \
PIDFD_SIGNAL_PROCESS_GROUP)
/**
* sys_pidfd_send_signal - Signal a process through a pidfd
* @pidfd: file descriptor of the process
* @sig: signal to send
* @info: signal info
* @flags: future flags
*
* Send the signal to the thread group or to the individual thread depending
* on PIDFD_THREAD.
* In the future extension to @flags may be used to override the default scope
* of @pidfd.
*
* Return: 0 on success, negative errno on failure
*/
SYSCALL_DEFINE4(pidfd_send_signal, int, pidfd, int, sig,
siginfo_t __user *, info, unsigned int, flags)
{
int ret;
struct fd f;
struct pid *pid;
kernel_siginfo_t kinfo;
enum pid_type type;
/* Enforce flags be set to 0 until we add an extension. */
if (flags & ~PIDFD_SEND_SIGNAL_FLAGS)
return -EINVAL;
/* Ensure that only a single signal scope determining flag is set. */
if (hweight32(flags & PIDFD_SEND_SIGNAL_FLAGS) > 1)
return -EINVAL;
f = fdget(pidfd);
if (!f.file)
return -EBADF;
/* Is this a pidfd? */
pid = pidfd_to_pid(f.file);
if (IS_ERR(pid)) {
ret = PTR_ERR(pid);
goto err;
}
ret = -EINVAL;
if (!access_pidfd_pidns(pid))
goto err;
switch (flags) {
case 0:
/* Infer scope from the type of pidfd. */
if (f.file->f_flags & PIDFD_THREAD)
type = PIDTYPE_PID;
else
type = PIDTYPE_TGID;
break;
case PIDFD_SIGNAL_THREAD:
type = PIDTYPE_PID;
break;
case PIDFD_SIGNAL_THREAD_GROUP:
type = PIDTYPE_TGID;
break;
case PIDFD_SIGNAL_PROCESS_GROUP:
type = PIDTYPE_PGID;
break;
}
if (info) {
ret = copy_siginfo_from_user_any(&kinfo, info);
if (unlikely(ret))
goto err;
ret = -EINVAL;
if (unlikely(sig != kinfo.si_signo))
goto err;
/* Only allow sending arbitrary signals to yourself. */
ret = -EPERM;
if ((task_pid(current) != pid || type > PIDTYPE_TGID) &&
(kinfo.si_code >= 0 || kinfo.si_code == SI_TKILL))
goto err;
} else {
prepare_kill_siginfo(sig, &kinfo, type);
}
if (type == PIDTYPE_PGID)
ret = kill_pgrp_info(sig, &kinfo, pid);
else
ret = kill_pid_info_type(sig, &kinfo, pid, type);
err:
fdput(f);
return ret;
}
static int
do_send_specific(pid_t tgid, pid_t pid, int sig, struct kernel_siginfo *info)
{
struct task_struct *p;
int error = -ESRCH;
rcu_read_lock();
p = find_task_by_vpid(pid);
if (p && (tgid <= 0 || task_tgid_vnr(p) == tgid)) {
error = check_kill_permission(sig, info, p);
/*
* The null signal is a permissions and process existence
* probe. No signal is actually delivered.
*/
if (!error && sig) {
error = do_send_sig_info(sig, info, p, PIDTYPE_PID);
/*
* If lock_task_sighand() failed we pretend the task
* dies after receiving the signal. The window is tiny,
* and the signal is private anyway.
*/
if (unlikely(error == -ESRCH))
error = 0;
}
}
rcu_read_unlock();
return error;
}
static int do_tkill(pid_t tgid, pid_t pid, int sig)
{
struct kernel_siginfo info;
prepare_kill_siginfo(sig, &info, PIDTYPE_PID);
return do_send_specific(tgid, pid, sig, &info);
}
/**
* sys_tgkill - send signal to one specific thread
* @tgid: the thread group ID of the thread
* @pid: the PID of the thread
* @sig: signal to be sent
*
* This syscall also checks the @tgid and returns -ESRCH even if the PID
* exists but it's not belonging to the target process anymore. This
* method solves the problem of threads exiting and PIDs getting reused.
*/
SYSCALL_DEFINE3(tgkill, pid_t, tgid, pid_t, pid, int, sig)
{
/* This is only valid for single tasks */
if (pid <= 0 || tgid <= 0)
return -EINVAL;
return do_tkill(tgid, pid, sig);
}
/**
* sys_tkill - send signal to one specific task
* @pid: the PID of the task
* @sig: signal to be sent
*
* Send a signal to only one task, even if it's a CLONE_THREAD task.
*/
SYSCALL_DEFINE2(tkill, pid_t, pid, int, sig)
{
/* This is only valid for single tasks */
if (pid <= 0)
return -EINVAL;
return do_tkill(0, pid, sig);
}
static int do_rt_sigqueueinfo(pid_t pid, int sig, kernel_siginfo_t *info)
{
/* Not even root can pretend to send signals from the kernel.
* Nor can they impersonate a kill()/tgkill(), which adds source info.
*/
if ((info->si_code >= 0 || info->si_code == SI_TKILL) &&
(task_pid_vnr(current) != pid))
return -EPERM;
/* POSIX.1b doesn't mention process groups. */
return kill_proc_info(sig, info, pid);
}
/**
* sys_rt_sigqueueinfo - send signal information to a signal
* @pid: the PID of the thread
* @sig: signal to be sent
* @uinfo: signal info to be sent
*/
SYSCALL_DEFINE3(rt_sigqueueinfo, pid_t, pid, int, sig,
siginfo_t __user *, uinfo)
{
kernel_siginfo_t info;
int ret = __copy_siginfo_from_user(sig, &info, uinfo);
if (unlikely(ret))
return ret;
return do_rt_sigqueueinfo(pid, sig, &info);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(rt_sigqueueinfo,
compat_pid_t, pid,
int, sig,
struct compat_siginfo __user *, uinfo)
{
kernel_siginfo_t info;
int ret = __copy_siginfo_from_user32(sig, &info, uinfo);
if (unlikely(ret))
return ret;
return do_rt_sigqueueinfo(pid, sig, &info);
}
#endif
static int do_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, kernel_siginfo_t *info)
{
/* This is only valid for single tasks */
if (pid <= 0 || tgid <= 0)
return -EINVAL;
/* Not even root can pretend to send signals from the kernel.
* Nor can they impersonate a kill()/tgkill(), which adds source info.
*/
if ((info->si_code >= 0 || info->si_code == SI_TKILL) &&
(task_pid_vnr(current) != pid))
return -EPERM;
return do_send_specific(tgid, pid, sig, info);
}
SYSCALL_DEFINE4(rt_tgsigqueueinfo, pid_t, tgid, pid_t, pid, int, sig,
siginfo_t __user *, uinfo)
{
kernel_siginfo_t info;
int ret = __copy_siginfo_from_user(sig, &info, uinfo);
if (unlikely(ret))
return ret;
return do_rt_tgsigqueueinfo(tgid, pid, sig, &info);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_tgsigqueueinfo,
compat_pid_t, tgid,
compat_pid_t, pid,
int, sig,
struct compat_siginfo __user *, uinfo)
{
kernel_siginfo_t info;
int ret = __copy_siginfo_from_user32(sig, &info, uinfo);
if (unlikely(ret))
return ret;
return do_rt_tgsigqueueinfo(tgid, pid, sig, &info);
}
#endif
/*
* For kthreads only, must not be used if cloned with CLONE_SIGHAND
*/
void kernel_sigaction(int sig, __sighandler_t action)
{
spin_lock_irq(¤t->sighand->siglock);
current->sighand->action[sig - 1].sa.sa_handler = action;
if (action == SIG_IGN) {
sigset_t mask;
sigemptyset(&mask);
sigaddset(&mask, sig);
flush_sigqueue_mask(&mask, ¤t->signal->shared_pending);
flush_sigqueue_mask(&mask, ¤t->pending);
recalc_sigpending();
}
spin_unlock_irq(¤t->sighand->siglock);
}
EXPORT_SYMBOL(kernel_sigaction);
void __weak sigaction_compat_abi(struct k_sigaction *act,
struct k_sigaction *oact)
{
}
int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
{
struct task_struct *p = current, *t;
struct k_sigaction *k;
sigset_t mask;
if (!valid_signal(sig) || sig < 1 || (act && sig_kernel_only(sig)))
return -EINVAL;
k = &p->sighand->action[sig-1];
spin_lock_irq(&p->sighand->siglock);
if (k->sa.sa_flags & SA_IMMUTABLE) {
spin_unlock_irq(&p->sighand->siglock);
return -EINVAL;
}
if (oact)
*oact = *k;
/*
* Make sure that we never accidentally claim to support SA_UNSUPPORTED,
* e.g. by having an architecture use the bit in their uapi.
*/
BUILD_BUG_ON(UAPI_SA_FLAGS & SA_UNSUPPORTED);
/*
* Clear unknown flag bits in order to allow userspace to detect missing
* support for flag bits and to allow the kernel to use non-uapi bits
* internally.
*/
if (act)
act->sa.sa_flags &= UAPI_SA_FLAGS;
if (oact)
oact->sa.sa_flags &= UAPI_SA_FLAGS;
sigaction_compat_abi(act, oact);
if (act) {
sigdelsetmask(&act->sa.sa_mask,
sigmask(SIGKILL) | sigmask(SIGSTOP));
*k = *act;
/*
* POSIX 3.3.1.3:
* "Setting a signal action to SIG_IGN for a signal that is
* pending shall cause the pending signal to be discarded,
* whether or not it is blocked."
*
* "Setting a signal action to SIG_DFL for a signal that is
* pending and whose default action is to ignore the signal
* (for example, SIGCHLD), shall cause the pending signal to
* be discarded, whether or not it is blocked"
*/
if (sig_handler_ignored(sig_handler(p, sig), sig)) {
sigemptyset(&mask);
sigaddset(&mask, sig);
flush_sigqueue_mask(&mask, &p->signal->shared_pending);
for_each_thread(p, t)
flush_sigqueue_mask(&mask, &t->pending);
}
}
spin_unlock_irq(&p->sighand->siglock);
return 0;
}
#ifdef CONFIG_DYNAMIC_SIGFRAME
static inline void sigaltstack_lock(void)
__acquires(¤t->sighand->siglock)
{
spin_lock_irq(¤t->sighand->siglock);
}
static inline void sigaltstack_unlock(void)
__releases(¤t->sighand->siglock)
{
spin_unlock_irq(¤t->sighand->siglock);
}
#else
static inline void sigaltstack_lock(void) { }
static inline void sigaltstack_unlock(void) { }
#endif
static int
do_sigaltstack (const stack_t *ss, stack_t *oss, unsigned long sp,
size_t min_ss_size)
{
struct task_struct *t = current;
int ret = 0;
if (oss) {
memset(oss, 0, sizeof(stack_t));
oss->ss_sp = (void __user *) t->sas_ss_sp;
oss->ss_size = t->sas_ss_size;
oss->ss_flags = sas_ss_flags(sp) |
(current->sas_ss_flags & SS_FLAG_BITS);
}
if (ss) {
void __user *ss_sp = ss->ss_sp;
size_t ss_size = ss->ss_size;
unsigned ss_flags = ss->ss_flags;
int ss_mode;
if (unlikely(on_sig_stack(sp)))
return -EPERM;
ss_mode = ss_flags & ~SS_FLAG_BITS;
if (unlikely(ss_mode != SS_DISABLE && ss_mode != SS_ONSTACK &&
ss_mode != 0))
return -EINVAL;
/*
* Return before taking any locks if no actual
* sigaltstack changes were requested.
*/
if (t->sas_ss_sp == (unsigned long)ss_sp &&
t->sas_ss_size == ss_size &&
t->sas_ss_flags == ss_flags)
return 0;
sigaltstack_lock();
if (ss_mode == SS_DISABLE) {
ss_size = 0;
ss_sp = NULL;
} else {
if (unlikely(ss_size < min_ss_size))
ret = -ENOMEM;
if (!sigaltstack_size_valid(ss_size))
ret = -ENOMEM;
}
if (!ret) {
t->sas_ss_sp = (unsigned long) ss_sp;
t->sas_ss_size = ss_size;
t->sas_ss_flags = ss_flags;
}
sigaltstack_unlock();
}
return ret;
}
SYSCALL_DEFINE2(sigaltstack,const stack_t __user *,uss, stack_t __user *,uoss)
{
stack_t new, old;
int err;
if (uss && copy_from_user(&new, uss, sizeof(stack_t)))
return -EFAULT;
err = do_sigaltstack(uss ? &new : NULL, uoss ? &old : NULL,
current_user_stack_pointer(),
MINSIGSTKSZ);
if (!err && uoss && copy_to_user(uoss, &old, sizeof(stack_t)))
err = -EFAULT;
return err;
}
int restore_altstack(const stack_t __user *uss)
{
stack_t new;
if (copy_from_user(&new, uss, sizeof(stack_t)))
return -EFAULT;
(void)do_sigaltstack(&new, NULL, current_user_stack_pointer(),
MINSIGSTKSZ);
/* squash all but EFAULT for now */
return 0;
}
int __save_altstack(stack_t __user *uss, unsigned long sp)
{
struct task_struct *t = current;
int err = __put_user((void __user *)t->sas_ss_sp, &uss->ss_sp) |
__put_user(t->sas_ss_flags, &uss->ss_flags) |
__put_user(t->sas_ss_size, &uss->ss_size);
return err;
}
#ifdef CONFIG_COMPAT
static int do_compat_sigaltstack(const compat_stack_t __user *uss_ptr,
compat_stack_t __user *uoss_ptr)
{
stack_t uss, uoss;
int ret;
if (uss_ptr) {
compat_stack_t uss32;
if (copy_from_user(&uss32, uss_ptr, sizeof(compat_stack_t)))
return -EFAULT;
uss.ss_sp = compat_ptr(uss32.ss_sp);
uss.ss_flags = uss32.ss_flags;
uss.ss_size = uss32.ss_size;
}
ret = do_sigaltstack(uss_ptr ? &uss : NULL, &uoss,
compat_user_stack_pointer(),
COMPAT_MINSIGSTKSZ);
if (ret >= 0 && uoss_ptr) {
compat_stack_t old;
memset(&old, 0, sizeof(old));
old.ss_sp = ptr_to_compat(uoss.ss_sp);
old.ss_flags = uoss.ss_flags;
old.ss_size = uoss.ss_size;
if (copy_to_user(uoss_ptr, &old, sizeof(compat_stack_t)))
ret = -EFAULT;
}
return ret;
}
COMPAT_SYSCALL_DEFINE2(sigaltstack,
const compat_stack_t __user *, uss_ptr,
compat_stack_t __user *, uoss_ptr)
{
return do_compat_sigaltstack(uss_ptr, uoss_ptr);
}
int compat_restore_altstack(const compat_stack_t __user *uss)
{
int err = do_compat_sigaltstack(uss, NULL);
/* squash all but -EFAULT for now */
return err == -EFAULT ? err : 0;
}
int __compat_save_altstack(compat_stack_t __user *uss, unsigned long sp)
{
int err;
struct task_struct *t = current;
err = __put_user(ptr_to_compat((void __user *)t->sas_ss_sp),
&uss->ss_sp) |
__put_user(t->sas_ss_flags, &uss->ss_flags) |
__put_user(t->sas_ss_size, &uss->ss_size);
return err;
}
#endif
#ifdef __ARCH_WANT_SYS_SIGPENDING
/**
* sys_sigpending - examine pending signals
* @uset: where mask of pending signal is returned
*/
SYSCALL_DEFINE1(sigpending, old_sigset_t __user *, uset)
{
sigset_t set;
if (sizeof(old_sigset_t) > sizeof(*uset))
return -EINVAL;
do_sigpending(&set);
if (copy_to_user(uset, &set, sizeof(old_sigset_t)))
return -EFAULT;
return 0;
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE1(sigpending, compat_old_sigset_t __user *, set32)
{
sigset_t set;
do_sigpending(&set);
return put_user(set.sig[0], set32);
}
#endif
#endif
#ifdef __ARCH_WANT_SYS_SIGPROCMASK
/**
* sys_sigprocmask - examine and change blocked signals
* @how: whether to add, remove, or set signals
* @nset: signals to add or remove (if non-null)
* @oset: previous value of signal mask if non-null
*
* Some platforms have their own version with special arguments;
* others support only sys_rt_sigprocmask.
*/
SYSCALL_DEFINE3(sigprocmask, int, how, old_sigset_t __user *, nset,
old_sigset_t __user *, oset)
{
old_sigset_t old_set, new_set;
sigset_t new_blocked;
old_set = current->blocked.sig[0];
if (nset) {
if (copy_from_user(&new_set, nset, sizeof(*nset)))
return -EFAULT;
new_blocked = current->blocked;
switch (how) {
case SIG_BLOCK:
sigaddsetmask(&new_blocked, new_set);
break;
case SIG_UNBLOCK:
sigdelsetmask(&new_blocked, new_set);
break;
case SIG_SETMASK:
new_blocked.sig[0] = new_set;
break;
default:
return -EINVAL;
}
set_current_blocked(&new_blocked);
}
if (oset) {
if (copy_to_user(oset, &old_set, sizeof(*oset)))
return -EFAULT;
}
return 0;
}
#endif /* __ARCH_WANT_SYS_SIGPROCMASK */
#ifndef CONFIG_ODD_RT_SIGACTION
/**
* sys_rt_sigaction - alter an action taken by a process
* @sig: signal to be sent
* @act: new sigaction
* @oact: used to save the previous sigaction
* @sigsetsize: size of sigset_t type
*/
SYSCALL_DEFINE4(rt_sigaction, int, sig,
const struct sigaction __user *, act,
struct sigaction __user *, oact,
size_t, sigsetsize)
{
struct k_sigaction new_sa, old_sa;
int ret;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (act && copy_from_user(&new_sa.sa, act, sizeof(new_sa.sa)))
return -EFAULT;
ret = do_sigaction(sig, act ? &new_sa : NULL, oact ? &old_sa : NULL);
if (ret)
return ret;
if (oact && copy_to_user(oact, &old_sa.sa, sizeof(old_sa.sa)))
return -EFAULT;
return 0;
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigaction, int, sig,
const struct compat_sigaction __user *, act,
struct compat_sigaction __user *, oact,
compat_size_t, sigsetsize)
{
struct k_sigaction new_ka, old_ka;
#ifdef __ARCH_HAS_SA_RESTORER
compat_uptr_t restorer;
#endif
int ret;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(compat_sigset_t))
return -EINVAL;
if (act) {
compat_uptr_t handler;
ret = get_user(handler, &act->sa_handler);
new_ka.sa.sa_handler = compat_ptr(handler);
#ifdef __ARCH_HAS_SA_RESTORER
ret |= get_user(restorer, &act->sa_restorer);
new_ka.sa.sa_restorer = compat_ptr(restorer);
#endif
ret |= get_compat_sigset(&new_ka.sa.sa_mask, &act->sa_mask);
ret |= get_user(new_ka.sa.sa_flags, &act->sa_flags);
if (ret)
return -EFAULT;
}
ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
if (!ret && oact) {
ret = put_user(ptr_to_compat(old_ka.sa.sa_handler),
&oact->sa_handler);
ret |= put_compat_sigset(&oact->sa_mask, &old_ka.sa.sa_mask,
sizeof(oact->sa_mask));
ret |= put_user(old_ka.sa.sa_flags, &oact->sa_flags);
#ifdef __ARCH_HAS_SA_RESTORER
ret |= put_user(ptr_to_compat(old_ka.sa.sa_restorer),
&oact->sa_restorer);
#endif
}
return ret;
}
#endif
#endif /* !CONFIG_ODD_RT_SIGACTION */
#ifdef CONFIG_OLD_SIGACTION
SYSCALL_DEFINE3(sigaction, int, sig,
const struct old_sigaction __user *, act,
struct old_sigaction __user *, oact)
{
struct k_sigaction new_ka, old_ka;
int ret;
if (act) {
old_sigset_t mask;
if (!access_ok(act, sizeof(*act)) ||
__get_user(new_ka.sa.sa_handler, &act->sa_handler) ||
__get_user(new_ka.sa.sa_restorer, &act->sa_restorer) ||
__get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
__get_user(mask, &act->sa_mask))
return -EFAULT;
#ifdef __ARCH_HAS_KA_RESTORER
new_ka.ka_restorer = NULL;
#endif
siginitset(&new_ka.sa.sa_mask, mask);
}
ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
if (!ret && oact) {
if (!access_ok(oact, sizeof(*oact)) ||
__put_user(old_ka.sa.sa_handler, &oact->sa_handler) ||
__put_user(old_ka.sa.sa_restorer, &oact->sa_restorer) ||
__put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
__put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
return -EFAULT;
}
return ret;
}
#endif
#ifdef CONFIG_COMPAT_OLD_SIGACTION
COMPAT_SYSCALL_DEFINE3(sigaction, int, sig,
const struct compat_old_sigaction __user *, act,
struct compat_old_sigaction __user *, oact)
{
struct k_sigaction new_ka, old_ka;
int ret;
compat_old_sigset_t mask;
compat_uptr_t handler, restorer;
if (act) {
if (!access_ok(act, sizeof(*act)) ||
__get_user(handler, &act->sa_handler) ||
__get_user(restorer, &act->sa_restorer) ||
__get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
__get_user(mask, &act->sa_mask))
return -EFAULT;
#ifdef __ARCH_HAS_KA_RESTORER
new_ka.ka_restorer = NULL;
#endif
new_ka.sa.sa_handler = compat_ptr(handler);
new_ka.sa.sa_restorer = compat_ptr(restorer);
siginitset(&new_ka.sa.sa_mask, mask);
}
ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
if (!ret && oact) {
if (!access_ok(oact, sizeof(*oact)) ||
__put_user(ptr_to_compat(old_ka.sa.sa_handler),
&oact->sa_handler) ||
__put_user(ptr_to_compat(old_ka.sa.sa_restorer),
&oact->sa_restorer) ||
__put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
__put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
return -EFAULT;
}
return ret;
}
#endif
#ifdef CONFIG_SGETMASK_SYSCALL
/*
* For backwards compatibility. Functionality superseded by sigprocmask.
*/
SYSCALL_DEFINE0(sgetmask)
{
/* SMP safe */
return current->blocked.sig[0];
}
SYSCALL_DEFINE1(ssetmask, int, newmask)
{
int old = current->blocked.sig[0];
sigset_t newset;
siginitset(&newset, newmask);
set_current_blocked(&newset);
return old;
}
#endif /* CONFIG_SGETMASK_SYSCALL */
#ifdef __ARCH_WANT_SYS_SIGNAL
/*
* For backwards compatibility. Functionality superseded by sigaction.
*/
SYSCALL_DEFINE2(signal, int, sig, __sighandler_t, handler)
{
struct k_sigaction new_sa, old_sa;
int ret;
new_sa.sa.sa_handler = handler;
new_sa.sa.sa_flags = SA_ONESHOT | SA_NOMASK;
sigemptyset(&new_sa.sa.sa_mask);
ret = do_sigaction(sig, &new_sa, &old_sa);
return ret ? ret : (unsigned long)old_sa.sa.sa_handler;
}
#endif /* __ARCH_WANT_SYS_SIGNAL */
#ifdef __ARCH_WANT_SYS_PAUSE
SYSCALL_DEFINE0(pause)
{
while (!signal_pending(current)) {
__set_current_state(TASK_INTERRUPTIBLE);
schedule();
}
return -ERESTARTNOHAND;
}
#endif
static int sigsuspend(sigset_t *set)
{
current->saved_sigmask = current->blocked;
set_current_blocked(set);
while (!signal_pending(current)) {
__set_current_state(TASK_INTERRUPTIBLE);
schedule();
}
set_restore_sigmask();
return -ERESTARTNOHAND;
}
/**
* sys_rt_sigsuspend - replace the signal mask for a value with the
* @unewset value until a signal is received
* @unewset: new signal mask value
* @sigsetsize: size of sigset_t type
*/
SYSCALL_DEFINE2(rt_sigsuspend, sigset_t __user *, unewset, size_t, sigsetsize)
{
sigset_t newset;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (copy_from_user(&newset, unewset, sizeof(newset)))
return -EFAULT;
return sigsuspend(&newset);
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(rt_sigsuspend, compat_sigset_t __user *, unewset, compat_size_t, sigsetsize)
{
sigset_t newset;
/* XXX: Don't preclude handling different sized sigset_t's. */
if (sigsetsize != sizeof(sigset_t))
return -EINVAL;
if (get_compat_sigset(&newset, unewset))
return -EFAULT;
return sigsuspend(&newset);
}
#endif
#ifdef CONFIG_OLD_SIGSUSPEND
SYSCALL_DEFINE1(sigsuspend, old_sigset_t, mask)
{
sigset_t blocked;
siginitset(&blocked, mask);
return sigsuspend(&blocked);
}
#endif
#ifdef CONFIG_OLD_SIGSUSPEND3
SYSCALL_DEFINE3(sigsuspend, int, unused1, int, unused2, old_sigset_t, mask)
{
sigset_t blocked;
siginitset(&blocked, mask);
return sigsuspend(&blocked);
}
#endif
__weak const char *arch_vma_name(struct vm_area_struct *vma)
{
return NULL;
}
static inline void siginfo_buildtime_checks(void)
{
BUILD_BUG_ON(sizeof(struct siginfo) != SI_MAX_SIZE);
/* Verify the offsets in the two siginfos match */
#define CHECK_OFFSET(field) \
BUILD_BUG_ON(offsetof(siginfo_t, field) != offsetof(kernel_siginfo_t, field))
/* kill */
CHECK_OFFSET(si_pid);
CHECK_OFFSET(si_uid);
/* timer */
CHECK_OFFSET(si_tid);
CHECK_OFFSET(si_overrun);
CHECK_OFFSET(si_value);
/* rt */
CHECK_OFFSET(si_pid);
CHECK_OFFSET(si_uid);
CHECK_OFFSET(si_value);
/* sigchld */
CHECK_OFFSET(si_pid);
CHECK_OFFSET(si_uid);
CHECK_OFFSET(si_status);
CHECK_OFFSET(si_utime);
CHECK_OFFSET(si_stime);
/* sigfault */
CHECK_OFFSET(si_addr);
CHECK_OFFSET(si_trapno);
CHECK_OFFSET(si_addr_lsb);
CHECK_OFFSET(si_lower);
CHECK_OFFSET(si_upper);
CHECK_OFFSET(si_pkey);
CHECK_OFFSET(si_perf_data);
CHECK_OFFSET(si_perf_type);
CHECK_OFFSET(si_perf_flags);
/* sigpoll */
CHECK_OFFSET(si_band);
CHECK_OFFSET(si_fd);
/* sigsys */
CHECK_OFFSET(si_call_addr);
CHECK_OFFSET(si_syscall);
CHECK_OFFSET(si_arch);
#undef CHECK_OFFSET
/* usb asyncio */
BUILD_BUG_ON(offsetof(struct siginfo, si_pid) !=
offsetof(struct siginfo, si_addr));
if (sizeof(int) == sizeof(void __user *)) {
BUILD_BUG_ON(sizeof_field(struct siginfo, si_pid) !=
sizeof(void __user *));
} else {
BUILD_BUG_ON((sizeof_field(struct siginfo, si_pid) +
sizeof_field(struct siginfo, si_uid)) !=
sizeof(void __user *));
BUILD_BUG_ON(offsetofend(struct siginfo, si_pid) !=
offsetof(struct siginfo, si_uid));
}
#ifdef CONFIG_COMPAT
BUILD_BUG_ON(offsetof(struct compat_siginfo, si_pid) !=
offsetof(struct compat_siginfo, si_addr));
BUILD_BUG_ON(sizeof_field(struct compat_siginfo, si_pid) !=
sizeof(compat_uptr_t));
BUILD_BUG_ON(sizeof_field(struct compat_siginfo, si_pid) !=
sizeof_field(struct siginfo, si_pid));
#endif
}
#if defined(CONFIG_SYSCTL)
static struct ctl_table signal_debug_table[] = {
#ifdef CONFIG_SYSCTL_EXCEPTION_TRACE
{
.procname = "exception-trace",
.data = &show_unhandled_signals,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec
},
#endif
{ }
};
static int __init init_signal_sysctls(void)
{
register_sysctl_init("debug", signal_debug_table);
return 0;
}
early_initcall(init_signal_sysctls);
#endif /* CONFIG_SYSCTL */
void __init signals_init(void)
{
siginfo_buildtime_checks();
sigqueue_cachep = KMEM_CACHE(sigqueue, SLAB_PANIC | SLAB_ACCOUNT);
}
#ifdef CONFIG_KGDB_KDB
#include <linux/kdb.h>
/*
* kdb_send_sig - Allows kdb to send signals without exposing
* signal internals. This function checks if the required locks are
* available before calling the main signal code, to avoid kdb
* deadlocks.
*/
void kdb_send_sig(struct task_struct *t, int sig)
{
static struct task_struct *kdb_prev_t;
int new_t, ret;
if (!spin_trylock(&t->sighand->siglock)) {
kdb_printf("Can't do kill command now.\n"
"The sigmask lock is held somewhere else in "
"kernel, try again later\n");
return;
}
new_t = kdb_prev_t != t;
kdb_prev_t = t;
if (!task_is_running(t) && new_t) {
spin_unlock(&t->sighand->siglock);
kdb_printf("Process is not RUNNING, sending a signal from "
"kdb risks deadlock\n"
"on the run queue locks. "
"The signal has _not_ been sent.\n"
"Reissue the kill command if you want to risk "
"the deadlock.\n");
return;
}
ret = send_signal_locked(sig, SEND_SIG_PRIV, t, PIDTYPE_PID);
spin_unlock(&t->sighand->siglock);
if (ret)
kdb_printf("Fail to deliver Signal %d to process %d.\n",
sig, t->pid);
else
kdb_printf("Signal %d is sent to process %d.\n", sig, t->pid);
}
#endif /* CONFIG_KGDB_KDB */
// SPDX-License-Identifier: GPL-2.0-only
/*
* ACPI device specific properties support.
*
* Copyright (C) 2014 - 2023, Intel Corporation
* All rights reserved.
*
* Authors: Mika Westerberg <mika.westerberg@linux.intel.com>
* Darren Hart <dvhart@linux.intel.com>
* Rafael J. Wysocki <rafael.j.wysocki@intel.com>
* Sakari Ailus <sakari.ailus@linux.intel.com>
*/
#define pr_fmt(fmt) "ACPI: " fmt
#include <linux/acpi.h>
#include <linux/device.h>
#include <linux/export.h>
#include "internal.h"
static int acpi_data_get_property_array(const struct acpi_device_data *data,
const char *name,
acpi_object_type type,
const union acpi_object **obj);
/*
* The GUIDs here are made equivalent to each other in order to avoid extra
* complexity in the properties handling code, with the caveat that the
* kernel will accept certain combinations of GUID and properties that are
* not defined without a warning. For instance if any of the properties
* from different GUID appear in a property list of another, it will be
* accepted by the kernel. Firmware validation tools should catch these.
*/
static const guid_t prp_guids[] = {
/* ACPI _DSD device properties GUID: daffd814-6eba-4d8c-8a91-bc9bbf4aa301 */
GUID_INIT(0xdaffd814, 0x6eba, 0x4d8c,
0x8a, 0x91, 0xbc, 0x9b, 0xbf, 0x4a, 0xa3, 0x01),
/* Hotplug in D3 GUID: 6211e2c0-58a3-4af3-90e1-927a4e0c55a4 */
GUID_INIT(0x6211e2c0, 0x58a3, 0x4af3,
0x90, 0xe1, 0x92, 0x7a, 0x4e, 0x0c, 0x55, 0xa4),
/* External facing port GUID: efcc06cc-73ac-4bc3-bff0-76143807c389 */
GUID_INIT(0xefcc06cc, 0x73ac, 0x4bc3,
0xbf, 0xf0, 0x76, 0x14, 0x38, 0x07, 0xc3, 0x89),
/* Thunderbolt GUID for IMR_VALID: c44d002f-69f9-4e7d-a904-a7baabdf43f7 */
GUID_INIT(0xc44d002f, 0x69f9, 0x4e7d,
0xa9, 0x04, 0xa7, 0xba, 0xab, 0xdf, 0x43, 0xf7),
/* Thunderbolt GUID for WAKE_SUPPORTED: 6c501103-c189-4296-ba72-9bf5a26ebe5d */
GUID_INIT(0x6c501103, 0xc189, 0x4296,
0xba, 0x72, 0x9b, 0xf5, 0xa2, 0x6e, 0xbe, 0x5d),
/* Storage device needs D3 GUID: 5025030f-842f-4ab4-a561-99a5189762d0 */
GUID_INIT(0x5025030f, 0x842f, 0x4ab4,
0xa5, 0x61, 0x99, 0xa5, 0x18, 0x97, 0x62, 0xd0),
};
/* ACPI _DSD data subnodes GUID: dbb8e3e6-5886-4ba6-8795-1319f52a966b */
static const guid_t ads_guid =
GUID_INIT(0xdbb8e3e6, 0x5886, 0x4ba6,
0x87, 0x95, 0x13, 0x19, 0xf5, 0x2a, 0x96, 0x6b);
/* ACPI _DSD data buffer GUID: edb12dd0-363d-4085-a3d2-49522ca160c4 */
static const guid_t buffer_prop_guid =
GUID_INIT(0xedb12dd0, 0x363d, 0x4085,
0xa3, 0xd2, 0x49, 0x52, 0x2c, 0xa1, 0x60, 0xc4);
static bool acpi_enumerate_nondev_subnodes(acpi_handle scope,
union acpi_object *desc,
struct acpi_device_data *data,
struct fwnode_handle *parent);
static bool acpi_extract_properties(acpi_handle handle,
union acpi_object *desc,
struct acpi_device_data *data);
static bool acpi_nondev_subnode_extract(union acpi_object *desc,
acpi_handle handle,
const union acpi_object *link,
struct list_head *list,
struct fwnode_handle *parent)
{
struct acpi_data_node *dn;
bool result;
if (acpi_graph_ignore_port(handle))
return false;
dn = kzalloc(sizeof(*dn), GFP_KERNEL);
if (!dn)
return false;
dn->name = link->package.elements[0].string.pointer;
fwnode_init(&dn->fwnode, &acpi_data_fwnode_ops);
dn->parent = parent;
INIT_LIST_HEAD(&dn->data.properties);
INIT_LIST_HEAD(&dn->data.subnodes);
result = acpi_extract_properties(handle, desc, &dn->data);
if (handle) {
acpi_handle scope;
acpi_status status;
/*
* The scope for the subnode object lookup is the one of the
* namespace node (device) containing the object that has
* returned the package. That is, it's the scope of that
* object's parent.
*/
status = acpi_get_parent(handle, &scope);
if (ACPI_SUCCESS(status)
&& acpi_enumerate_nondev_subnodes(scope, desc, &dn->data,
&dn->fwnode))
result = true;
} else if (acpi_enumerate_nondev_subnodes(NULL, desc, &dn->data,
&dn->fwnode)) {
result = true;
}
if (result) {
dn->handle = handle;
dn->data.pointer = desc;
list_add_tail(&dn->sibling, list);
return true;
}
kfree(dn);
acpi_handle_debug(handle, "Invalid properties/subnodes data, skipping\n");
return false;
}
static bool acpi_nondev_subnode_data_ok(acpi_handle handle,
const union acpi_object *link,
struct list_head *list,
struct fwnode_handle *parent)
{
struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
acpi_status status;
status = acpi_evaluate_object_typed(handle, NULL, NULL, &buf,
ACPI_TYPE_PACKAGE);
if (ACPI_FAILURE(status))
return false;
if (acpi_nondev_subnode_extract(buf.pointer, handle, link, list,
parent))
return true;
ACPI_FREE(buf.pointer);
return false;
}
static bool acpi_nondev_subnode_ok(acpi_handle scope,
const union acpi_object *link,
struct list_head *list,
struct fwnode_handle *parent)
{
acpi_handle handle;
acpi_status status;
if (!scope)
return false;
status = acpi_get_handle(scope, link->package.elements[1].string.pointer,
&handle);
if (ACPI_FAILURE(status))
return false;
return acpi_nondev_subnode_data_ok(handle, link, list, parent);
}
static bool acpi_add_nondev_subnodes(acpi_handle scope,
union acpi_object *links,
struct list_head *list,
struct fwnode_handle *parent)
{
bool ret = false;
int i;
for (i = 0; i < links->package.count; i++) {
union acpi_object *link, *desc;
acpi_handle handle;
bool result;
link = &links->package.elements[i];
/* Only two elements allowed. */
if (link->package.count != 2)
continue;
/* The first one must be a string. */
if (link->package.elements[0].type != ACPI_TYPE_STRING)
continue;
/* The second one may be a string, a reference or a package. */
switch (link->package.elements[1].type) {
case ACPI_TYPE_STRING:
result = acpi_nondev_subnode_ok(scope, link, list,
parent);
break;
case ACPI_TYPE_LOCAL_REFERENCE:
handle = link->package.elements[1].reference.handle;
result = acpi_nondev_subnode_data_ok(handle, link, list,
parent);
break;
case ACPI_TYPE_PACKAGE:
desc = &link->package.elements[1];
result = acpi_nondev_subnode_extract(desc, NULL, link,
list, parent);
break;
default:
result = false;
break;
}
ret = ret || result;
}
return ret;
}
static bool acpi_enumerate_nondev_subnodes(acpi_handle scope,
union acpi_object *desc,
struct acpi_device_data *data,
struct fwnode_handle *parent)
{
int i;
/* Look for the ACPI data subnodes GUID. */
for (i = 0; i < desc->package.count; i += 2) {
const union acpi_object *guid;
union acpi_object *links;
guid = &desc->package.elements[i];
links = &desc->package.elements[i + 1];
/*
* The first element must be a GUID and the second one must be
* a package.
*/
if (guid->type != ACPI_TYPE_BUFFER ||
guid->buffer.length != 16 ||
links->type != ACPI_TYPE_PACKAGE)
break;
if (!guid_equal((guid_t *)guid->buffer.pointer, &ads_guid))
continue;
return acpi_add_nondev_subnodes(scope, links, &data->subnodes,
parent);
}
return false;
}
static bool acpi_property_value_ok(const union acpi_object *value)
{
int j;
/*
* The value must be an integer, a string, a reference, or a package
* whose every element must be an integer, a string, or a reference.
*/
switch (value->type) {
case ACPI_TYPE_INTEGER:
case ACPI_TYPE_STRING:
case ACPI_TYPE_LOCAL_REFERENCE:
return true;
case ACPI_TYPE_PACKAGE:
for (j = 0; j < value->package.count; j++)
switch (value->package.elements[j].type) {
case ACPI_TYPE_INTEGER:
case ACPI_TYPE_STRING:
case ACPI_TYPE_LOCAL_REFERENCE:
continue;
default:
return false;
}
return true;
}
return false;
}
static bool acpi_properties_format_valid(const union acpi_object *properties)
{
int i;
for (i = 0; i < properties->package.count; i++) {
const union acpi_object *property;
property = &properties->package.elements[i];
/*
* Only two elements allowed, the first one must be a string and
* the second one has to satisfy certain conditions.
*/
if (property->package.count != 2
|| property->package.elements[0].type != ACPI_TYPE_STRING
|| !acpi_property_value_ok(&property->package.elements[1]))
return false;
}
return true;
}
static void acpi_init_of_compatible(struct acpi_device *adev)
{
const union acpi_object *of_compatible;
int ret;
ret = acpi_data_get_property_array(&adev->data, "compatible",
ACPI_TYPE_STRING, &of_compatible);
if (ret) {
ret = acpi_dev_get_property(adev, "compatible",
ACPI_TYPE_STRING, &of_compatible);
if (ret) {
struct acpi_device *parent;
parent = acpi_dev_parent(adev);
if (parent && parent->flags.of_compatible_ok)
goto out;
return;
}
}
adev->data.of_compatible = of_compatible;
out:
adev->flags.of_compatible_ok = 1;
}
static bool acpi_is_property_guid(const guid_t *guid)
{
int i;
for (i = 0; i < ARRAY_SIZE(prp_guids); i++) {
if (guid_equal(guid, &prp_guids[i]))
return true;
}
return false;
}
struct acpi_device_properties *
acpi_data_add_props(struct acpi_device_data *data, const guid_t *guid,
union acpi_object *properties)
{
struct acpi_device_properties *props;
props = kzalloc(sizeof(*props), GFP_KERNEL);
if (props) {
INIT_LIST_HEAD(&props->list);
props->guid = guid;
props->properties = properties;
list_add_tail(&props->list, &data->properties);
}
return props;
}
static void acpi_nondev_subnode_tag(acpi_handle handle, void *context)
{
}
static void acpi_untie_nondev_subnodes(struct acpi_device_data *data)
{
struct acpi_data_node *dn;
list_for_each_entry(dn, &data->subnodes, sibling) {
acpi_detach_data(dn->handle, acpi_nondev_subnode_tag);
acpi_untie_nondev_subnodes(&dn->data);
}
}
static bool acpi_tie_nondev_subnodes(struct acpi_device_data *data)
{
struct acpi_data_node *dn;
list_for_each_entry(dn, &data->subnodes, sibling) {
acpi_status status;
bool ret;
status = acpi_attach_data(dn->handle, acpi_nondev_subnode_tag, dn);
if (ACPI_FAILURE(status) && status != AE_ALREADY_EXISTS) {
acpi_handle_err(dn->handle, "Can't tag data node\n");
return false;
}
ret = acpi_tie_nondev_subnodes(&dn->data);
if (!ret)
return ret;
}
return true;
}
static void acpi_data_add_buffer_props(acpi_handle handle,
struct acpi_device_data *data,
union acpi_object *properties)
{
struct acpi_device_properties *props;
union acpi_object *package;
size_t alloc_size;
unsigned int i;
u32 *count;
if (check_mul_overflow((size_t)properties->package.count,
sizeof(*package) + sizeof(void *),
&alloc_size) ||
check_add_overflow(sizeof(*props) + sizeof(*package), alloc_size,
&alloc_size)) {
acpi_handle_warn(handle,
"can't allocate memory for %u buffer props",
properties->package.count);
return;
}
props = kvzalloc(alloc_size, GFP_KERNEL);
if (!props)
return;
props->guid = &buffer_prop_guid;
props->bufs = (void *)(props + 1);
props->properties = (void *)(props->bufs + properties->package.count);
/* Outer package */
package = props->properties;
package->type = ACPI_TYPE_PACKAGE;
package->package.elements = package + 1;
count = &package->package.count;
*count = 0;
/* Inner packages */
package++;
for (i = 0; i < properties->package.count; i++) {
struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
union acpi_object *property = &properties->package.elements[i];
union acpi_object *prop, *obj, *buf_obj;
acpi_status status;
if (property->type != ACPI_TYPE_PACKAGE ||
property->package.count != 2) {
acpi_handle_warn(handle,
"buffer property %u has %u entries\n",
i, property->package.count);
continue;
}
prop = &property->package.elements[0];
obj = &property->package.elements[1];
if (prop->type != ACPI_TYPE_STRING ||
obj->type != ACPI_TYPE_STRING) {
acpi_handle_warn(handle,
"wrong object types %u and %u\n",
prop->type, obj->type);
continue;
}
status = acpi_evaluate_object_typed(handle, obj->string.pointer,
NULL, &buf,
ACPI_TYPE_BUFFER);
if (ACPI_FAILURE(status)) {
acpi_handle_warn(handle,
"can't evaluate \"%*pE\" as buffer\n",
obj->string.length,
obj->string.pointer);
continue;
}
package->type = ACPI_TYPE_PACKAGE;
package->package.elements = prop;
package->package.count = 2;
buf_obj = buf.pointer;
/* Replace the string object with a buffer object */
obj->type = ACPI_TYPE_BUFFER;
obj->buffer.length = buf_obj->buffer.length;
obj->buffer.pointer = buf_obj->buffer.pointer;
props->bufs[i] = buf.pointer;
package++;
(*count)++;
}
if (*count)
list_add(&props->list, &data->properties);
else
kvfree(props);
}
static bool acpi_extract_properties(acpi_handle scope, union acpi_object *desc,
struct acpi_device_data *data)
{
int i;
if (desc->package.count % 2)
return false;
/* Look for the device properties GUID. */
for (i = 0; i < desc->package.count; i += 2) {
const union acpi_object *guid;
union acpi_object *properties;
guid = &desc->package.elements[i];
properties = &desc->package.elements[i + 1];
/*
* The first element must be a GUID and the second one must be
* a package.
*/
if (guid->type != ACPI_TYPE_BUFFER ||
guid->buffer.length != 16 ||
properties->type != ACPI_TYPE_PACKAGE)
break;
if (guid_equal((guid_t *)guid->buffer.pointer,
&buffer_prop_guid)) {
acpi_data_add_buffer_props(scope, data, properties);
continue;
}
if (!acpi_is_property_guid((guid_t *)guid->buffer.pointer))
continue;
/*
* We found the matching GUID. Now validate the format of the
* package immediately following it.
*/
if (!acpi_properties_format_valid(properties))
continue;
acpi_data_add_props(data, (const guid_t *)guid->buffer.pointer,
properties);
}
return !list_empty(&data->properties);
}
void acpi_init_properties(struct acpi_device *adev)
{
struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
struct acpi_hardware_id *hwid;
acpi_status status;
bool acpi_of = false;
INIT_LIST_HEAD(&adev->data.properties);
INIT_LIST_HEAD(&adev->data.subnodes);
if (!adev->handle)
return;
/*
* Check if ACPI_DT_NAMESPACE_HID is present and inthat case we fill in
* Device Tree compatible properties for this device.
*/
list_for_each_entry(hwid, &adev->pnp.ids, list) {
if (!strcmp(hwid->id, ACPI_DT_NAMESPACE_HID)) {
acpi_of = true;
break;
}
}
status = acpi_evaluate_object_typed(adev->handle, "_DSD", NULL, &buf,
ACPI_TYPE_PACKAGE);
if (ACPI_FAILURE(status))
goto out;
if (acpi_extract_properties(adev->handle, buf.pointer, &adev->data)) {
adev->data.pointer = buf.pointer;
if (acpi_of)
acpi_init_of_compatible(adev);
}
if (acpi_enumerate_nondev_subnodes(adev->handle, buf.pointer,
&adev->data, acpi_fwnode_handle(adev)))
adev->data.pointer = buf.pointer;
if (!adev->data.pointer) {
acpi_handle_debug(adev->handle, "Invalid _DSD data, skipping\n");
ACPI_FREE(buf.pointer);
} else {
if (!acpi_tie_nondev_subnodes(&adev->data))
acpi_untie_nondev_subnodes(&adev->data);
}
out:
if (acpi_of && !adev->flags.of_compatible_ok)
acpi_handle_info(adev->handle,
ACPI_DT_NAMESPACE_HID " requires 'compatible' property\n");
if (!adev->data.pointer)
acpi_extract_apple_properties(adev);
}
static void acpi_free_device_properties(struct list_head *list)
{
struct acpi_device_properties *props, *tmp;
list_for_each_entry_safe(props, tmp, list, list) {
u32 i;
list_del(&props->list);
/* Buffer data properties were separately allocated */
if (props->bufs)
for (i = 0; i < props->properties->package.count; i++)
ACPI_FREE(props->bufs[i]);
kvfree(props);
}
}
static void acpi_destroy_nondev_subnodes(struct list_head *list)
{
struct acpi_data_node *dn, *next;
if (list_empty(list))
return;
list_for_each_entry_safe_reverse(dn, next, list, sibling) {
acpi_destroy_nondev_subnodes(&dn->data.subnodes);
wait_for_completion(&dn->kobj_done);
list_del(&dn->sibling);
ACPI_FREE((void *)dn->data.pointer);
acpi_free_device_properties(&dn->data.properties);
kfree(dn);
}
}
void acpi_free_properties(struct acpi_device *adev)
{
acpi_untie_nondev_subnodes(&adev->data);
acpi_destroy_nondev_subnodes(&adev->data.subnodes);
ACPI_FREE((void *)adev->data.pointer);
adev->data.of_compatible = NULL;
adev->data.pointer = NULL;
acpi_free_device_properties(&adev->data.properties);
}
/**
* acpi_data_get_property - return an ACPI property with given name
* @data: ACPI device deta object to get the property from
* @name: Name of the property
* @type: Expected property type
* @obj: Location to store the property value (if not %NULL)
*
* Look up a property with @name and store a pointer to the resulting ACPI
* object at the location pointed to by @obj if found.
*
* Callers must not attempt to free the returned objects. These objects will be
* freed by the ACPI core automatically during the removal of @data.
*
* Return: %0 if property with @name has been found (success),
* %-EINVAL if the arguments are invalid,
* %-EINVAL if the property doesn't exist,
* %-EPROTO if the property value type doesn't match @type.
*/
static int acpi_data_get_property(const struct acpi_device_data *data,
const char *name, acpi_object_type type,
const union acpi_object **obj)
{
const struct acpi_device_properties *props;
if (!data || !name)
return -EINVAL;
if (!data->pointer || list_empty(&data->properties))
return -EINVAL;
list_for_each_entry(props, &data->properties, list) {
const union acpi_object *properties;
unsigned int i;
properties = props->properties;
for (i = 0; i < properties->package.count; i++) {
const union acpi_object *propname, *propvalue;
const union acpi_object *property;
property = &properties->package.elements[i];
propname = &property->package.elements[0];
propvalue = &property->package.elements[1];
if (!strcmp(name, propname->string.pointer)) {
if (type != ACPI_TYPE_ANY &&
propvalue->type != type)
return -EPROTO;
if (obj)
*obj = propvalue;
return 0;
}
}
}
return -EINVAL;
}
/**
* acpi_dev_get_property - return an ACPI property with given name.
* @adev: ACPI device to get the property from.
* @name: Name of the property.
* @type: Expected property type.
* @obj: Location to store the property value (if not %NULL).
*/
int acpi_dev_get_property(const struct acpi_device *adev, const char *name,
acpi_object_type type, const union acpi_object **obj)
{
return adev ? acpi_data_get_property(&adev->data, name, type, obj) : -EINVAL;
}
EXPORT_SYMBOL_GPL(acpi_dev_get_property);
static const struct acpi_device_data *
acpi_device_data_of_node(const struct fwnode_handle *fwnode)
{
if (is_acpi_device_node(fwnode)) {
const struct acpi_device *adev = to_acpi_device_node(fwnode);
return &adev->data;
}
if (is_acpi_data_node(fwnode)) {
const struct acpi_data_node *dn = to_acpi_data_node(fwnode);
return &dn->data;
}
return NULL;
}
/**
* acpi_node_prop_get - return an ACPI property with given name.
* @fwnode: Firmware node to get the property from.
* @propname: Name of the property.
* @valptr: Location to store a pointer to the property value (if not %NULL).
*/
int acpi_node_prop_get(const struct fwnode_handle *fwnode,
const char *propname, void **valptr)
{
return acpi_data_get_property(acpi_device_data_of_node(fwnode),
propname, ACPI_TYPE_ANY,
(const union acpi_object **)valptr);
}
/**
* acpi_data_get_property_array - return an ACPI array property with given name
* @data: ACPI data object to get the property from
* @name: Name of the property
* @type: Expected type of array elements
* @obj: Location to store a pointer to the property value (if not NULL)
*
* Look up an array property with @name and store a pointer to the resulting
* ACPI object at the location pointed to by @obj if found.
*
* Callers must not attempt to free the returned objects. Those objects will be
* freed by the ACPI core automatically during the removal of @data.
*
* Return: %0 if array property (package) with @name has been found (success),
* %-EINVAL if the arguments are invalid,
* %-EINVAL if the property doesn't exist,
* %-EPROTO if the property is not a package or the type of its elements
* doesn't match @type.
*/
static int acpi_data_get_property_array(const struct acpi_device_data *data,
const char *name,
acpi_object_type type,
const union acpi_object **obj)
{
const union acpi_object *prop;
int ret, i;
ret = acpi_data_get_property(data, name, ACPI_TYPE_PACKAGE, &prop);
if (ret)
return ret;
if (type != ACPI_TYPE_ANY) {
/* Check that all elements are of correct type. */
for (i = 0; i < prop->package.count; i++)
if (prop->package.elements[i].type != type)
return -EPROTO;
}
if (obj)
*obj = prop;
return 0;
}
static struct fwnode_handle *
acpi_fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
const char *childname)
{
struct fwnode_handle *child;
fwnode_for_each_child_node(fwnode, child) {
if (is_acpi_data_node(child)) {
if (acpi_data_node_match(child, childname))
return child;
continue;
}
if (!strncmp(acpi_device_bid(to_acpi_device_node(child)),
childname, ACPI_NAMESEG_SIZE))
return child;
}
return NULL;
}
static int acpi_get_ref_args(struct fwnode_reference_args *args,
struct fwnode_handle *ref_fwnode,
const union acpi_object **element,
const union acpi_object *end, size_t num_args)
{
u32 nargs = 0, i;
/*
* Assume the following integer elements are all args. Stop counting on
* the first reference (possibly represented as a string) or end of the
* package arguments. In case of neither reference, nor integer, return
* an error, we can't parse it.
*/
for (i = 0; (*element) + i < end && i < num_args; i++) {
acpi_object_type type = (*element)[i].type;
if (type == ACPI_TYPE_LOCAL_REFERENCE || type == ACPI_TYPE_STRING)
break;
if (type == ACPI_TYPE_INTEGER)
nargs++;
else
return -EINVAL;
}
if (nargs > NR_FWNODE_REFERENCE_ARGS)
return -EINVAL;
if (args) {
args->fwnode = ref_fwnode;
args->nargs = nargs;
for (i = 0; i < nargs; i++)
args->args[i] = (*element)[i].integer.value;
}
(*element) += nargs;
return 0;
}
static struct fwnode_handle *acpi_parse_string_ref(const struct fwnode_handle *fwnode,
const char *refstring)
{
acpi_handle scope, handle;
struct acpi_data_node *dn;
struct acpi_device *device;
acpi_status status;
if (is_acpi_device_node(fwnode)) {
scope = to_acpi_device_node(fwnode)->handle;
} else if (is_acpi_data_node(fwnode)) {
scope = to_acpi_data_node(fwnode)->handle;
} else {
pr_debug("Bad node type for node %pfw\n", fwnode);
return NULL;
}
status = acpi_get_handle(scope, refstring, &handle);
if (ACPI_FAILURE(status)) {
acpi_handle_debug(scope, "Unable to get an ACPI handle for %s\n",
refstring);
return NULL;
}
device = acpi_fetch_acpi_dev(handle);
if (device)
return acpi_fwnode_handle(device);
status = acpi_get_data_full(handle, acpi_nondev_subnode_tag,
(void **)&dn, NULL);
if (ACPI_FAILURE(status) || !dn) {
acpi_handle_debug(handle, "Subnode not found\n");
return NULL;
}
return &dn->fwnode;
}
/**
* __acpi_node_get_property_reference - returns handle to the referenced object
* @fwnode: Firmware node to get the property from
* @propname: Name of the property
* @index: Index of the reference to return
* @num_args: Maximum number of arguments after each reference
* @args: Location to store the returned reference with optional arguments
* (may be NULL)
*
* Find property with @name, verifify that it is a package containing at least
* one object reference and if so, store the ACPI device object pointer to the
* target object in @args->adev. If the reference includes arguments, store
* them in the @args->args[] array.
*
* If there's more than one reference in the property value package, @index is
* used to select the one to return.
*
* It is possible to leave holes in the property value set like in the
* example below:
*
* Package () {
* "cs-gpios",
* Package () {
* ^GPIO, 19, 0, 0,
* ^GPIO, 20, 0, 0,
* 0,
* ^GPIO, 21, 0, 0,
* }
* }
*
* Calling this function with index %2 or index %3 return %-ENOENT. If the
* property does not contain any more values %-ENOENT is returned. The NULL
* entry must be single integer and preferably contain value %0.
*
* Return: %0 on success, negative error code on failure.
*/
int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode,
const char *propname, size_t index, size_t num_args,
struct fwnode_reference_args *args)
{
const union acpi_object *element, *end;
const union acpi_object *obj;
const struct acpi_device_data *data;
struct fwnode_handle *ref_fwnode;
struct acpi_device *device;
int ret, idx = 0;
data = acpi_device_data_of_node(fwnode);
if (!data)
return -ENOENT;
ret = acpi_data_get_property(data, propname, ACPI_TYPE_ANY, &obj);
if (ret)
return ret == -EINVAL ? -ENOENT : -EINVAL;
switch (obj->type) {
case ACPI_TYPE_LOCAL_REFERENCE:
/* Plain single reference without arguments. */
if (index)
return -ENOENT;
device = acpi_fetch_acpi_dev(obj->reference.handle);
if (!device)
return -EINVAL;
if (!args)
return 0;
args->fwnode = acpi_fwnode_handle(device);
args->nargs = 0;
return 0;
case ACPI_TYPE_STRING:
if (index)
return -ENOENT;
ref_fwnode = acpi_parse_string_ref(fwnode, obj->string.pointer);
if (!ref_fwnode)
return -EINVAL;
args->fwnode = ref_fwnode;
args->nargs = 0;
return 0;
case ACPI_TYPE_PACKAGE:
/*
* If it is not a single reference, then it is a package of
* references, followed by number of ints as follows:
*
* Package () { REF, INT, REF, INT, INT }
*
* Here, REF may be either a local reference or a string. The
* index argument is then used to determine which reference the
* caller wants (along with the arguments).
*/
break;
default:
return -EINVAL;
}
if (index >= obj->package.count)
return -ENOENT;
element = obj->package.elements;
end = element + obj->package.count;
while (element < end) {
switch (element->type) {
case ACPI_TYPE_LOCAL_REFERENCE:
device = acpi_fetch_acpi_dev(element->reference.handle);
if (!device)
return -EINVAL;
element++;
ret = acpi_get_ref_args(idx == index ? args : NULL,
acpi_fwnode_handle(device),
&element, end, num_args);
if (ret < 0)
return ret;
if (idx == index)
return 0;
break;
case ACPI_TYPE_STRING:
ref_fwnode = acpi_parse_string_ref(fwnode,
element->string.pointer);
if (!ref_fwnode)
return -EINVAL;
element++;
ret = acpi_get_ref_args(idx == index ? args : NULL,
ref_fwnode, &element, end,
num_args);
if (ret < 0)
return ret;
if (idx == index)
return 0;
break;
case ACPI_TYPE_INTEGER:
if (idx == index)
return -ENOENT;
element++;
break;
default:
return -EINVAL;
}
idx++;
}
return -ENOENT;
}
EXPORT_SYMBOL_GPL(__acpi_node_get_property_reference);
static int acpi_data_prop_read_single(const struct acpi_device_data *data,
const char *propname,
enum dev_prop_type proptype, void *val)
{
const union acpi_object *obj;
int ret = 0;
if (proptype >= DEV_PROP_U8 && proptype <= DEV_PROP_U64)
ret = acpi_data_get_property(data, propname, ACPI_TYPE_INTEGER, &obj);
else if (proptype == DEV_PROP_STRING)
ret = acpi_data_get_property(data, propname, ACPI_TYPE_STRING, &obj);
if (ret)
return ret;
switch (proptype) {
case DEV_PROP_U8:
if (obj->integer.value > U8_MAX)
return -EOVERFLOW;
if (val)
*(u8 *)val = obj->integer.value;
break;
case DEV_PROP_U16:
if (obj->integer.value > U16_MAX)
return -EOVERFLOW;
if (val)
*(u16 *)val = obj->integer.value;
break;
case DEV_PROP_U32:
if (obj->integer.value > U32_MAX)
return -EOVERFLOW;
if (val)
*(u32 *)val = obj->integer.value;
break;
case DEV_PROP_U64:
if (val)
*(u64 *)val = obj->integer.value;
break;
case DEV_PROP_STRING:
if (val)
*(char **)val = obj->string.pointer;
return 1;
default:
return -EINVAL;
}
/* When no storage provided return number of available values */
return val ? 0 : 1;
}
#define acpi_copy_property_array_uint(items, val, nval) \
({ \
typeof(items) __items = items; \
typeof(val) __val = val; \
typeof(nval) __nval = nval; \
size_t i; \
int ret = 0; \
\
for (i = 0; i < __nval; i++) { \
if (__items->type == ACPI_TYPE_BUFFER) { \
__val[i] = __items->buffer.pointer[i]; \
continue; \
} \
if (__items[i].type != ACPI_TYPE_INTEGER) { \
ret = -EPROTO; \
break; \
} \
if (__items[i].integer.value > _Generic(__val, \
u8 *: U8_MAX, \
u16 *: U16_MAX, \
u32 *: U32_MAX, \
u64 *: U64_MAX)) { \
ret = -EOVERFLOW; \
break; \
} \
\
__val[i] = __items[i].integer.value; \
} \
ret; \
})
static int acpi_copy_property_array_string(const union acpi_object *items,
char **val, size_t nval)
{
int i;
for (i = 0; i < nval; i++) {
if (items[i].type != ACPI_TYPE_STRING)
return -EPROTO;
val[i] = items[i].string.pointer;
}
return nval;
}
static int acpi_data_prop_read(const struct acpi_device_data *data,
const char *propname,
enum dev_prop_type proptype,
void *val, size_t nval)
{
const union acpi_object *obj;
const union acpi_object *items;
int ret;
if (nval == 1 || !val) {
ret = acpi_data_prop_read_single(data, propname, proptype, val);
/*
* The overflow error means that the property is there and it is
* single-value, but its type does not match, so return.
*/
if (ret >= 0 || ret == -EOVERFLOW)
return ret;
/*
* Reading this property as a single-value one failed, but its
* value may still be represented as one-element array, so
* continue.
*/
}
ret = acpi_data_get_property_array(data, propname, ACPI_TYPE_ANY, &obj);
if (ret && proptype >= DEV_PROP_U8 && proptype <= DEV_PROP_U64)
ret = acpi_data_get_property(data, propname, ACPI_TYPE_BUFFER,
&obj);
if (ret)
return ret;
if (!val) {
if (obj->type == ACPI_TYPE_BUFFER)
return obj->buffer.length;
return obj->package.count;
}
switch (proptype) {
case DEV_PROP_STRING:
break;
default:
if (obj->type == ACPI_TYPE_BUFFER) {
if (nval > obj->buffer.length)
return -EOVERFLOW;
} else {
if (nval > obj->package.count)
return -EOVERFLOW;
}
break;
}
if (nval == 0)
return -EINVAL;
if (obj->type == ACPI_TYPE_BUFFER) {
if (proptype != DEV_PROP_U8)
return -EPROTO;
items = obj;
} else {
items = obj->package.elements;
}
switch (proptype) {
case DEV_PROP_U8:
ret = acpi_copy_property_array_uint(items, (u8 *)val, nval);
break;
case DEV_PROP_U16:
ret = acpi_copy_property_array_uint(items, (u16 *)val, nval);
break;
case DEV_PROP_U32:
ret = acpi_copy_property_array_uint(items, (u32 *)val, nval);
break;
case DEV_PROP_U64:
ret = acpi_copy_property_array_uint(items, (u64 *)val, nval);
break;
case DEV_PROP_STRING:
ret = acpi_copy_property_array_string(
items, (char **)val,
min_t(u32, nval, obj->package.count));
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
/**
* acpi_node_prop_read - retrieve the value of an ACPI property with given name.
* @fwnode: Firmware node to get the property from.
* @propname: Name of the property.
* @proptype: Expected property type.
* @val: Location to store the property value (if not %NULL).
* @nval: Size of the array pointed to by @val.
*
* If @val is %NULL, return the number of array elements comprising the value
* of the property. Otherwise, read at most @nval values to the array at the
* location pointed to by @val.
*/
static int acpi_node_prop_read(const struct fwnode_handle *fwnode,
const char *propname, enum dev_prop_type proptype,
void *val, size_t nval)
{
return acpi_data_prop_read(acpi_device_data_of_node(fwnode),
propname, proptype, val, nval);
}
static int stop_on_next(struct acpi_device *adev, void *data)
{
struct acpi_device **ret_p = data;
if (!*ret_p) {
*ret_p = adev;
return 1;
}
/* Skip until the "previous" object is found. */
if (*ret_p == adev)
*ret_p = NULL;
return 0;
}
/**
* acpi_get_next_subnode - Return the next child node handle for a fwnode
* @fwnode: Firmware node to find the next child node for.
* @child: Handle to one of the device's child nodes or a null handle.
*/
struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode,
struct fwnode_handle *child)
{
struct acpi_device *adev = to_acpi_device_node(fwnode);
if ((!child || is_acpi_device_node(child)) && adev) {
struct acpi_device *child_adev = to_acpi_device_node(child);
acpi_dev_for_each_child(adev, stop_on_next, &child_adev);
if (child_adev)
return acpi_fwnode_handle(child_adev);
child = NULL;
}
if (!child || is_acpi_data_node(child)) {
const struct acpi_data_node *data = to_acpi_data_node(fwnode);
const struct list_head *head;
struct list_head *next;
struct acpi_data_node *dn;
/*
* We can have a combination of device and data nodes, e.g. with
* hierarchical _DSD properties. Make sure the adev pointer is
* restored before going through data nodes, otherwise we will
* be looking for data_nodes below the last device found instead
* of the common fwnode shared by device_nodes and data_nodes.
*/
adev = to_acpi_device_node(fwnode);
if (adev)
head = &adev->data.subnodes;
else if (data)
head = &data->data.subnodes;
else
return NULL;
if (list_empty(head))
return NULL;
if (child) {
dn = to_acpi_data_node(child);
next = dn->sibling.next;
if (next == head)
return NULL;
dn = list_entry(next, struct acpi_data_node, sibling);
} else {
dn = list_first_entry(head, struct acpi_data_node, sibling);
}
return &dn->fwnode;
}
return NULL;
}
/**
* acpi_node_get_parent - Return parent fwnode of this fwnode
* @fwnode: Firmware node whose parent to get
*
* Returns parent node of an ACPI device or data firmware node or %NULL if
* not available.
*/
static struct fwnode_handle *
acpi_node_get_parent(const struct fwnode_handle *fwnode)
{
if (is_acpi_data_node(fwnode)) {
/* All data nodes have parent pointer so just return that */
return to_acpi_data_node(fwnode)->parent;
}
if (is_acpi_device_node(fwnode)) {
struct acpi_device *parent;
parent = acpi_dev_parent(to_acpi_device_node(fwnode));
if (parent)
return acpi_fwnode_handle(parent);
}
return NULL;
}
/*
* Return true if the node is an ACPI graph node. Called on either ports
* or endpoints.
*/
static bool is_acpi_graph_node(struct fwnode_handle *fwnode,
const char *str)
{
unsigned int len = strlen(str);
const char *name;
if (!len || !is_acpi_data_node(fwnode))
return false;
name = to_acpi_data_node(fwnode)->name;
return (fwnode_property_present(fwnode, "reg") &&
!strncmp(name, str, len) && name[len] == '@') ||
fwnode_property_present(fwnode, str);
}
/**
* acpi_graph_get_next_endpoint - Get next endpoint ACPI firmware node
* @fwnode: Pointer to the parent firmware node
* @prev: Previous endpoint node or %NULL to get the first
*
* Looks up next endpoint ACPI firmware node below a given @fwnode. Returns
* %NULL if there is no next endpoint or in case of error. In case of success
* the next endpoint is returned.
*/
static struct fwnode_handle *acpi_graph_get_next_endpoint(
const struct fwnode_handle *fwnode, struct fwnode_handle *prev)
{
struct fwnode_handle *port = NULL;
struct fwnode_handle *endpoint;
if (!prev) {
do {
port = fwnode_get_next_child_node(fwnode, port);
/*
* The names of the port nodes begin with "port@"
* followed by the number of the port node and they also
* have a "reg" property that also has the number of the
* port node. For compatibility reasons a node is also
* recognised as a port node from the "port" property.
*/
if (is_acpi_graph_node(port, "port"))
break;
} while (port);
} else {
port = fwnode_get_parent(prev);
}
if (!port)
return NULL;
endpoint = fwnode_get_next_child_node(port, prev);
while (!endpoint) {
port = fwnode_get_next_child_node(fwnode, port);
if (!port)
break;
if (is_acpi_graph_node(port, "port"))
endpoint = fwnode_get_next_child_node(port, NULL);
}
/*
* The names of the endpoint nodes begin with "endpoint@" followed by
* the number of the endpoint node and they also have a "reg" property
* that also has the number of the endpoint node. For compatibility
* reasons a node is also recognised as an endpoint node from the
* "endpoint" property.
*/
if (!is_acpi_graph_node(endpoint, "endpoint"))
return NULL;
return endpoint;
}
/**
* acpi_graph_get_child_prop_value - Return a child with a given property value
* @fwnode: device fwnode
* @prop_name: The name of the property to look for
* @val: the desired property value
*
* Return the port node corresponding to a given port number. Returns
* the child node on success, NULL otherwise.
*/
static struct fwnode_handle *acpi_graph_get_child_prop_value(
const struct fwnode_handle *fwnode, const char *prop_name,
unsigned int val)
{
struct fwnode_handle *child;
fwnode_for_each_child_node(fwnode, child) {
u32 nr;
if (fwnode_property_read_u32(child, prop_name, &nr))
continue;
if (val == nr)
return child;
}
return NULL;
}
/**
* acpi_graph_get_remote_endpoint - Parses and returns remote end of an endpoint
* @__fwnode: Endpoint firmware node pointing to a remote device
*
* Returns the remote endpoint corresponding to @__fwnode. NULL on error.
*/
static struct fwnode_handle *
acpi_graph_get_remote_endpoint(const struct fwnode_handle *__fwnode)
{
struct fwnode_handle *fwnode;
unsigned int port_nr, endpoint_nr;
struct fwnode_reference_args args;
int ret;
memset(&args, 0, sizeof(args));
ret = acpi_node_get_property_reference(__fwnode, "remote-endpoint", 0,
&args);
if (ret)
return NULL;
/* Direct endpoint reference? */
if (!is_acpi_device_node(args.fwnode))
return args.nargs ? NULL : args.fwnode;
/*
* Always require two arguments with the reference: port and
* endpoint indices.
*/
if (args.nargs != 2)
return NULL;
fwnode = args.fwnode;
port_nr = args.args[0];
endpoint_nr = args.args[1];
fwnode = acpi_graph_get_child_prop_value(fwnode, "port", port_nr);
return acpi_graph_get_child_prop_value(fwnode, "endpoint", endpoint_nr);
}
static bool acpi_fwnode_device_is_available(const struct fwnode_handle *fwnode)
{
if (!is_acpi_device_node(fwnode))
return false;
return acpi_device_is_present(to_acpi_device_node(fwnode));
}
static const void *
acpi_fwnode_device_get_match_data(const struct fwnode_handle *fwnode,
const struct device *dev)
{
return acpi_device_get_match_data(dev);
}
static bool acpi_fwnode_device_dma_supported(const struct fwnode_handle *fwnode)
{
return acpi_dma_supported(to_acpi_device_node(fwnode));
}
static enum dev_dma_attr
acpi_fwnode_device_get_dma_attr(const struct fwnode_handle *fwnode)
{
return acpi_get_dma_attr(to_acpi_device_node(fwnode));
}
static bool acpi_fwnode_property_present(const struct fwnode_handle *fwnode,
const char *propname)
{
return !acpi_node_prop_get(fwnode, propname, NULL);
}
static int
acpi_fwnode_property_read_int_array(const struct fwnode_handle *fwnode,
const char *propname,
unsigned int elem_size, void *val,
size_t nval)
{
enum dev_prop_type type;
switch (elem_size) {
case sizeof(u8):
type = DEV_PROP_U8;
break;
case sizeof(u16):
type = DEV_PROP_U16;
break;
case sizeof(u32):
type = DEV_PROP_U32;
break;
case sizeof(u64):
type = DEV_PROP_U64;
break;
default:
return -ENXIO;
}
return acpi_node_prop_read(fwnode, propname, type, val, nval);
}
static int
acpi_fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
const char *propname, const char **val,
size_t nval)
{
return acpi_node_prop_read(fwnode, propname, DEV_PROP_STRING,
val, nval);
}
static int
acpi_fwnode_get_reference_args(const struct fwnode_handle *fwnode,
const char *prop, const char *nargs_prop,
unsigned int args_count, unsigned int index,
struct fwnode_reference_args *args)
{
return __acpi_node_get_property_reference(fwnode, prop, index,
args_count, args);
}
static const char *acpi_fwnode_get_name(const struct fwnode_handle *fwnode)
{
const struct acpi_device *adev;
struct fwnode_handle *parent;
/* Is this the root node? */
parent = fwnode_get_parent(fwnode);
if (!parent)
return "\\";
fwnode_handle_put(parent);
if (is_acpi_data_node(fwnode)) {
const struct acpi_data_node *dn = to_acpi_data_node(fwnode);
return dn->name;
}
adev = to_acpi_device_node(fwnode);
if (WARN_ON(!adev))
return NULL;
return acpi_device_bid(adev);
}
static const char *
acpi_fwnode_get_name_prefix(const struct fwnode_handle *fwnode)
{
struct fwnode_handle *parent;
/* Is this the root node? */
parent = fwnode_get_parent(fwnode);
if (!parent)
return "";
/* Is this 2nd node from the root? */
parent = fwnode_get_next_parent(parent);
if (!parent)
return "";
fwnode_handle_put(parent);
/* ACPI device or data node. */
return ".";
}
static struct fwnode_handle *
acpi_fwnode_get_parent(struct fwnode_handle *fwnode)
{
return acpi_node_get_parent(fwnode);
}
static int acpi_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
struct fwnode_endpoint *endpoint)
{
struct fwnode_handle *port_fwnode = fwnode_get_parent(fwnode);
endpoint->local_fwnode = fwnode;
if (fwnode_property_read_u32(port_fwnode, "reg", &endpoint->port))
fwnode_property_read_u32(port_fwnode, "port", &endpoint->port);
if (fwnode_property_read_u32(fwnode, "reg", &endpoint->id))
fwnode_property_read_u32(fwnode, "endpoint", &endpoint->id);
return 0;
}
static int acpi_fwnode_irq_get(const struct fwnode_handle *fwnode,
unsigned int index)
{
struct resource res;
int ret;
ret = acpi_irq_get(ACPI_HANDLE_FWNODE(fwnode), index, &res);
if (ret)
return ret;
return res.start;
}
#define DECLARE_ACPI_FWNODE_OPS(ops) \
const struct fwnode_operations ops = { \
.device_is_available = acpi_fwnode_device_is_available, \
.device_get_match_data = acpi_fwnode_device_get_match_data, \
.device_dma_supported = \
acpi_fwnode_device_dma_supported, \
.device_get_dma_attr = acpi_fwnode_device_get_dma_attr, \
.property_present = acpi_fwnode_property_present, \
.property_read_int_array = \
acpi_fwnode_property_read_int_array, \
.property_read_string_array = \
acpi_fwnode_property_read_string_array, \
.get_parent = acpi_node_get_parent, \
.get_next_child_node = acpi_get_next_subnode, \
.get_named_child_node = acpi_fwnode_get_named_child_node, \
.get_name = acpi_fwnode_get_name, \
.get_name_prefix = acpi_fwnode_get_name_prefix, \
.get_reference_args = acpi_fwnode_get_reference_args, \
.graph_get_next_endpoint = \
acpi_graph_get_next_endpoint, \
.graph_get_remote_endpoint = \
acpi_graph_get_remote_endpoint, \
.graph_get_port_parent = acpi_fwnode_get_parent, \
.graph_parse_endpoint = acpi_fwnode_graph_parse_endpoint, \
.irq_get = acpi_fwnode_irq_get, \
}; \
EXPORT_SYMBOL_GPL(ops)
DECLARE_ACPI_FWNODE_OPS(acpi_device_fwnode_ops);
DECLARE_ACPI_FWNODE_OPS(acpi_data_fwnode_ops);
const struct fwnode_operations acpi_static_fwnode_ops;
bool is_acpi_device_node(const struct fwnode_handle *fwnode)
{
return !IS_ERR_OR_NULL(fwnode) && fwnode->ops == &acpi_device_fwnode_ops;
}
EXPORT_SYMBOL(is_acpi_device_node);
bool is_acpi_data_node(const struct fwnode_handle *fwnode)
{
return !IS_ERR_OR_NULL(fwnode) && fwnode->ops == &acpi_data_fwnode_ops;
}
EXPORT_SYMBOL(is_acpi_data_node);
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/irqflags.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/bug.h>
#include "printk_ringbuffer.h"
#include "internal.h"
/**
* DOC: printk_ringbuffer overview
*
* Data Structure
* --------------
* The printk_ringbuffer is made up of 3 internal ringbuffers:
*
* desc_ring
* A ring of descriptors and their meta data (such as sequence number,
* timestamp, loglevel, etc.) as well as internal state information about
* the record and logical positions specifying where in the other
* ringbuffer the text strings are located.
*
* text_data_ring
* A ring of data blocks. A data block consists of an unsigned long
* integer (ID) that maps to a desc_ring index followed by the text
* string of the record.
*
* The internal state information of a descriptor is the key element to allow
* readers and writers to locklessly synchronize access to the data.
*
* Implementation
* --------------
*
* Descriptor Ring
* ~~~~~~~~~~~~~~~
* The descriptor ring is an array of descriptors. A descriptor contains
* essential meta data to track the data of a printk record using
* blk_lpos structs pointing to associated text data blocks (see
* "Data Rings" below). Each descriptor is assigned an ID that maps
* directly to index values of the descriptor array and has a state. The ID
* and the state are bitwise combined into a single descriptor field named
* @state_var, allowing ID and state to be synchronously and atomically
* updated.
*
* Descriptors have four states:
*
* reserved
* A writer is modifying the record.
*
* committed
* The record and all its data are written. A writer can reopen the
* descriptor (transitioning it back to reserved), but in the committed
* state the data is consistent.
*
* finalized
* The record and all its data are complete and available for reading. A
* writer cannot reopen the descriptor.
*
* reusable
* The record exists, but its text and/or meta data may no longer be
* available.
*
* Querying the @state_var of a record requires providing the ID of the
* descriptor to query. This can yield a possible fifth (pseudo) state:
*
* miss
* The descriptor being queried has an unexpected ID.
*
* The descriptor ring has a @tail_id that contains the ID of the oldest
* descriptor and @head_id that contains the ID of the newest descriptor.
*
* When a new descriptor should be created (and the ring is full), the tail
* descriptor is invalidated by first transitioning to the reusable state and
* then invalidating all tail data blocks up to and including the data blocks
* associated with the tail descriptor (for the text ring). Then
* @tail_id is advanced, followed by advancing @head_id. And finally the
* @state_var of the new descriptor is initialized to the new ID and reserved
* state.
*
* The @tail_id can only be advanced if the new @tail_id would be in the
* committed or reusable queried state. This makes it possible that a valid
* sequence number of the tail is always available.
*
* Descriptor Finalization
* ~~~~~~~~~~~~~~~~~~~~~~~
* When a writer calls the commit function prb_commit(), record data is
* fully stored and is consistent within the ringbuffer. However, a writer can
* reopen that record, claiming exclusive access (as with prb_reserve()), and
* modify that record. When finished, the writer must again commit the record.
*
* In order for a record to be made available to readers (and also become
* recyclable for writers), it must be finalized. A finalized record cannot be
* reopened and can never become "unfinalized". Record finalization can occur
* in three different scenarios:
*
* 1) A writer can simultaneously commit and finalize its record by calling
* prb_final_commit() instead of prb_commit().
*
* 2) When a new record is reserved and the previous record has been
* committed via prb_commit(), that previous record is automatically
* finalized.
*
* 3) When a record is committed via prb_commit() and a newer record
* already exists, the record being committed is automatically finalized.
*
* Data Ring
* ~~~~~~~~~
* The text data ring is a byte array composed of data blocks. Data blocks are
* referenced by blk_lpos structs that point to the logical position of the
* beginning of a data block and the beginning of the next adjacent data
* block. Logical positions are mapped directly to index values of the byte
* array ringbuffer.
*
* Each data block consists of an ID followed by the writer data. The ID is
* the identifier of a descriptor that is associated with the data block. A
* given data block is considered valid if all of the following conditions
* are met:
*
* 1) The descriptor associated with the data block is in the committed
* or finalized queried state.
*
* 2) The blk_lpos struct within the descriptor associated with the data
* block references back to the same data block.
*
* 3) The data block is within the head/tail logical position range.
*
* If the writer data of a data block would extend beyond the end of the
* byte array, only the ID of the data block is stored at the logical
* position and the full data block (ID and writer data) is stored at the
* beginning of the byte array. The referencing blk_lpos will point to the
* ID before the wrap and the next data block will be at the logical
* position adjacent the full data block after the wrap.
*
* Data rings have a @tail_lpos that points to the beginning of the oldest
* data block and a @head_lpos that points to the logical position of the
* next (not yet existing) data block.
*
* When a new data block should be created (and the ring is full), tail data
* blocks will first be invalidated by putting their associated descriptors
* into the reusable state and then pushing the @tail_lpos forward beyond
* them. Then the @head_lpos is pushed forward and is associated with a new
* descriptor. If a data block is not valid, the @tail_lpos cannot be
* advanced beyond it.
*
* Info Array
* ~~~~~~~~~~
* The general meta data of printk records are stored in printk_info structs,
* stored in an array with the same number of elements as the descriptor ring.
* Each info corresponds to the descriptor of the same index in the
* descriptor ring. Info validity is confirmed by evaluating the corresponding
* descriptor before and after loading the info.
*
* Usage
* -----
* Here are some simple examples demonstrating writers and readers. For the
* examples a global ringbuffer (test_rb) is available (which is not the
* actual ringbuffer used by printk)::
*
* DEFINE_PRINTKRB(test_rb, 15, 5);
*
* This ringbuffer allows up to 32768 records (2 ^ 15) and has a size of
* 1 MiB (2 ^ (15 + 5)) for text data.
*
* Sample writer code::
*
* const char *textstr = "message text";
* struct prb_reserved_entry e;
* struct printk_record r;
*
* // specify how much to allocate
* prb_rec_init_wr(&r, strlen(textstr) + 1);
*
* if (prb_reserve(&e, &test_rb, &r)) {
* snprintf(r.text_buf, r.text_buf_size, "%s", textstr);
*
* r.info->text_len = strlen(textstr);
* r.info->ts_nsec = local_clock();
* r.info->caller_id = printk_caller_id();
*
* // commit and finalize the record
* prb_final_commit(&e);
* }
*
* Note that additional writer functions are available to extend a record
* after it has been committed but not yet finalized. This can be done as
* long as no new records have been reserved and the caller is the same.
*
* Sample writer code (record extending)::
*
* // alternate rest of previous example
*
* r.info->text_len = strlen(textstr);
* r.info->ts_nsec = local_clock();
* r.info->caller_id = printk_caller_id();
*
* // commit the record (but do not finalize yet)
* prb_commit(&e);
* }
*
* ...
*
* // specify additional 5 bytes text space to extend
* prb_rec_init_wr(&r, 5);
*
* // try to extend, but only if it does not exceed 32 bytes
* if (prb_reserve_in_last(&e, &test_rb, &r, printk_caller_id(), 32)) {
* snprintf(&r.text_buf[r.info->text_len],
* r.text_buf_size - r.info->text_len, "hello");
*
* r.info->text_len += 5;
*
* // commit and finalize the record
* prb_final_commit(&e);
* }
*
* Sample reader code::
*
* struct printk_info info;
* struct printk_record r;
* char text_buf[32];
* u64 seq;
*
* prb_rec_init_rd(&r, &info, &text_buf[0], sizeof(text_buf));
*
* prb_for_each_record(0, &test_rb, &seq, &r) {
* if (info.seq != seq)
* pr_warn("lost %llu records\n", info.seq - seq);
*
* if (info.text_len > r.text_buf_size) {
* pr_warn("record %llu text truncated\n", info.seq);
* text_buf[r.text_buf_size - 1] = 0;
* }
*
* pr_info("%llu: %llu: %s\n", info.seq, info.ts_nsec,
* &text_buf[0]);
* }
*
* Note that additional less convenient reader functions are available to
* allow complex record access.
*
* ABA Issues
* ~~~~~~~~~~
* To help avoid ABA issues, descriptors are referenced by IDs (array index
* values combined with tagged bits counting array wraps) and data blocks are
* referenced by logical positions (array index values combined with tagged
* bits counting array wraps). However, on 32-bit systems the number of
* tagged bits is relatively small such that an ABA incident is (at least
* theoretically) possible. For example, if 4 million maximally sized (1KiB)
* printk messages were to occur in NMI context on a 32-bit system, the
* interrupted context would not be able to recognize that the 32-bit integer
* completely wrapped and thus represents a different data block than the one
* the interrupted context expects.
*
* To help combat this possibility, additional state checking is performed
* (such as using cmpxchg() even though set() would suffice). These extra
* checks are commented as such and will hopefully catch any ABA issue that
* a 32-bit system might experience.
*
* Memory Barriers
* ~~~~~~~~~~~~~~~
* Multiple memory barriers are used. To simplify proving correctness and
* generating litmus tests, lines of code related to memory barriers
* (loads, stores, and the associated memory barriers) are labeled::
*
* LMM(function:letter)
*
* Comments reference the labels using only the "function:letter" part.
*
* The memory barrier pairs and their ordering are:
*
* desc_reserve:D / desc_reserve:B
* push descriptor tail (id), then push descriptor head (id)
*
* desc_reserve:D / data_push_tail:B
* push data tail (lpos), then set new descriptor reserved (state)
*
* desc_reserve:D / desc_push_tail:C
* push descriptor tail (id), then set new descriptor reserved (state)
*
* desc_reserve:D / prb_first_seq:C
* push descriptor tail (id), then set new descriptor reserved (state)
*
* desc_reserve:F / desc_read:D
* set new descriptor id and reserved (state), then allow writer changes
*
* data_alloc:A (or data_realloc:A) / desc_read:D
* set old descriptor reusable (state), then modify new data block area
*
* data_alloc:A (or data_realloc:A) / data_push_tail:B
* push data tail (lpos), then modify new data block area
*
* _prb_commit:B / desc_read:B
* store writer changes, then set new descriptor committed (state)
*
* desc_reopen_last:A / _prb_commit:B
* set descriptor reserved (state), then read descriptor data
*
* _prb_commit:B / desc_reserve:D
* set new descriptor committed (state), then check descriptor head (id)
*
* data_push_tail:D / data_push_tail:A
* set descriptor reusable (state), then push data tail (lpos)
*
* desc_push_tail:B / desc_reserve:D
* set descriptor reusable (state), then push descriptor tail (id)
*
* desc_update_last_finalized:A / desc_last_finalized_seq:A
* store finalized record, then set new highest finalized sequence number
*/
#define DATA_SIZE(data_ring) _DATA_SIZE((data_ring)->size_bits)
#define DATA_SIZE_MASK(data_ring) (DATA_SIZE(data_ring) - 1)
#define DESCS_COUNT(desc_ring) _DESCS_COUNT((desc_ring)->count_bits)
#define DESCS_COUNT_MASK(desc_ring) (DESCS_COUNT(desc_ring) - 1)
/* Determine the data array index from a logical position. */
#define DATA_INDEX(data_ring, lpos) ((lpos) & DATA_SIZE_MASK(data_ring))
/* Determine the desc array index from an ID or sequence number. */
#define DESC_INDEX(desc_ring, n) ((n) & DESCS_COUNT_MASK(desc_ring))
/* Determine how many times the data array has wrapped. */
#define DATA_WRAPS(data_ring, lpos) ((lpos) >> (data_ring)->size_bits)
/* Determine if a logical position refers to a data-less block. */
#define LPOS_DATALESS(lpos) ((lpos) & 1UL)
#define BLK_DATALESS(blk) (LPOS_DATALESS((blk)->begin) && \
LPOS_DATALESS((blk)->next))
/* Get the logical position at index 0 of the current wrap. */
#define DATA_THIS_WRAP_START_LPOS(data_ring, lpos) \
((lpos) & ~DATA_SIZE_MASK(data_ring))
/* Get the ID for the same index of the previous wrap as the given ID. */
#define DESC_ID_PREV_WRAP(desc_ring, id) \
DESC_ID((id) - DESCS_COUNT(desc_ring))
/*
* A data block: mapped directly to the beginning of the data block area
* specified as a logical position within the data ring.
*
* @id: the ID of the associated descriptor
* @data: the writer data
*
* Note that the size of a data block is only known by its associated
* descriptor.
*/
struct prb_data_block {
unsigned long id;
char data[];
};
/*
* Return the descriptor associated with @n. @n can be either a
* descriptor ID or a sequence number.
*/
static struct prb_desc *to_desc(struct prb_desc_ring *desc_ring, u64 n)
{
return &desc_ring->descs[DESC_INDEX(desc_ring, n)];
}
/*
* Return the printk_info associated with @n. @n can be either a
* descriptor ID or a sequence number.
*/
static struct printk_info *to_info(struct prb_desc_ring *desc_ring, u64 n)
{
return &desc_ring->infos[DESC_INDEX(desc_ring, n)];
}
static struct prb_data_block *to_block(struct prb_data_ring *data_ring,
unsigned long begin_lpos)
{
return (void *)&data_ring->data[DATA_INDEX(data_ring, begin_lpos)];
}
/*
* Increase the data size to account for data block meta data plus any
* padding so that the adjacent data block is aligned on the ID size.
*/
static unsigned int to_blk_size(unsigned int size)
{
struct prb_data_block *db = NULL;
size += sizeof(*db);
size = ALIGN(size, sizeof(db->id));
return size;
}
/*
* Sanity checker for reserve size. The ringbuffer code assumes that a data
* block does not exceed the maximum possible size that could fit within the
* ringbuffer. This function provides that basic size check so that the
* assumption is safe.
*/
static bool data_check_size(struct prb_data_ring *data_ring, unsigned int size)
{
struct prb_data_block *db = NULL;
if (size == 0)
return true;
/*
* Ensure the alignment padded size could possibly fit in the data
* array. The largest possible data block must still leave room for
* at least the ID of the next block.
*/
size = to_blk_size(size); if (size > DATA_SIZE(data_ring) - sizeof(db->id))
return false;
return true;
}
/* Query the state of a descriptor. */
static enum desc_state get_desc_state(unsigned long id,
unsigned long state_val)
{
if (id != DESC_ID(state_val))
return desc_miss;
return DESC_STATE(state_val);
}
/*
* Get a copy of a specified descriptor and return its queried state. If the
* descriptor is in an inconsistent state (miss or reserved), the caller can
* only expect the descriptor's @state_var field to be valid.
*
* The sequence number and caller_id can be optionally retrieved. Like all
* non-state_var data, they are only valid if the descriptor is in a
* consistent state.
*/
static enum desc_state desc_read(struct prb_desc_ring *desc_ring,
unsigned long id, struct prb_desc *desc_out,
u64 *seq_out, u32 *caller_id_out)
{
struct printk_info *info = to_info(desc_ring, id); struct prb_desc *desc = to_desc(desc_ring, id);
atomic_long_t *state_var = &desc->state_var;
enum desc_state d_state;
unsigned long state_val;
/* Check the descriptor state. */
state_val = atomic_long_read(state_var); /* LMM(desc_read:A) */
d_state = get_desc_state(id, state_val);
if (d_state == desc_miss || d_state == desc_reserved) {
/*
* The descriptor is in an inconsistent state. Set at least
* @state_var so that the caller can see the details of
* the inconsistent state.
*/
goto out;
}
/*
* Guarantee the state is loaded before copying the descriptor
* content. This avoids copying obsolete descriptor content that might
* not apply to the descriptor state. This pairs with _prb_commit:B.
*
* Memory barrier involvement:
*
* If desc_read:A reads from _prb_commit:B, then desc_read:C reads
* from _prb_commit:A.
*
* Relies on:
*
* WMB from _prb_commit:A to _prb_commit:B
* matching
* RMB from desc_read:A to desc_read:C
*/
smp_rmb(); /* LMM(desc_read:B) */
/*
* Copy the descriptor data. The data is not valid until the
* state has been re-checked. A memcpy() for all of @desc
* cannot be used because of the atomic_t @state_var field.
*/
if (desc_out) {
memcpy(&desc_out->text_blk_lpos, &desc->text_blk_lpos,
sizeof(desc_out->text_blk_lpos)); /* LMM(desc_read:C) */
}
if (seq_out)
*seq_out = info->seq; /* also part of desc_read:C */ if (caller_id_out) *caller_id_out = info->caller_id; /* also part of desc_read:C */
/*
* 1. Guarantee the descriptor content is loaded before re-checking
* the state. This avoids reading an obsolete descriptor state
* that may not apply to the copied content. This pairs with
* desc_reserve:F.
*
* Memory barrier involvement:
*
* If desc_read:C reads from desc_reserve:G, then desc_read:E
* reads from desc_reserve:F.
*
* Relies on:
*
* WMB from desc_reserve:F to desc_reserve:G
* matching
* RMB from desc_read:C to desc_read:E
*
* 2. Guarantee the record data is loaded before re-checking the
* state. This avoids reading an obsolete descriptor state that may
* not apply to the copied data. This pairs with data_alloc:A and
* data_realloc:A.
*
* Memory barrier involvement:
*
* If copy_data:A reads from data_alloc:B, then desc_read:E
* reads from desc_make_reusable:A.
*
* Relies on:
*
* MB from desc_make_reusable:A to data_alloc:B
* matching
* RMB from desc_read:C to desc_read:E
*
* Note: desc_make_reusable:A and data_alloc:B can be different
* CPUs. However, the data_alloc:B CPU (which performs the
* full memory barrier) must have previously seen
* desc_make_reusable:A.
*/
smp_rmb(); /* LMM(desc_read:D) */
/*
* The data has been copied. Return the current descriptor state,
* which may have changed since the load above.
*/
state_val = atomic_long_read(state_var); /* LMM(desc_read:E) */
d_state = get_desc_state(id, state_val);
out:
if (desc_out)
atomic_long_set(&desc_out->state_var, state_val);
return d_state;
}
/*
* Take a specified descriptor out of the finalized state by attempting
* the transition from finalized to reusable. Either this context or some
* other context will have been successful.
*/
static void desc_make_reusable(struct prb_desc_ring *desc_ring,
unsigned long id)
{
unsigned long val_finalized = DESC_SV(id, desc_finalized);
unsigned long val_reusable = DESC_SV(id, desc_reusable);
struct prb_desc *desc = to_desc(desc_ring, id);
atomic_long_t *state_var = &desc->state_var;
atomic_long_cmpxchg_relaxed(state_var, val_finalized,
val_reusable); /* LMM(desc_make_reusable:A) */
}
/*
* Given the text data ring, put the associated descriptor of each
* data block from @lpos_begin until @lpos_end into the reusable state.
*
* If there is any problem making the associated descriptor reusable, either
* the descriptor has not yet been finalized or another writer context has
* already pushed the tail lpos past the problematic data block. Regardless,
* on error the caller can re-load the tail lpos to determine the situation.
*/
static bool data_make_reusable(struct printk_ringbuffer *rb,
unsigned long lpos_begin,
unsigned long lpos_end,
unsigned long *lpos_out)
{
struct prb_data_ring *data_ring = &rb->text_data_ring;
struct prb_desc_ring *desc_ring = &rb->desc_ring;
struct prb_data_block *blk;
enum desc_state d_state;
struct prb_desc desc;
struct prb_data_blk_lpos *blk_lpos = &desc.text_blk_lpos;
unsigned long id;
/* Loop until @lpos_begin has advanced to or beyond @lpos_end. */
while ((lpos_end - lpos_begin) - 1 < DATA_SIZE(data_ring)) { blk = to_block(data_ring, lpos_begin);
/*
* Load the block ID from the data block. This is a data race
* against a writer that may have newly reserved this data
* area. If the loaded value matches a valid descriptor ID,
* the blk_lpos of that descriptor will be checked to make
* sure it points back to this data block. If the check fails,
* the data area has been recycled by another writer.
*/
id = blk->id; /* LMM(data_make_reusable:A) */
d_state = desc_read(desc_ring, id, &desc,
NULL, NULL); /* LMM(data_make_reusable:B) */
switch (d_state) {
case desc_miss:
case desc_reserved:
case desc_committed:
return false;
case desc_finalized:
/*
* This data block is invalid if the descriptor
* does not point back to it.
*/
if (blk_lpos->begin != lpos_begin)
return false;
desc_make_reusable(desc_ring, id);
break;
case desc_reusable:
/*
* This data block is invalid if the descriptor
* does not point back to it.
*/
if (blk_lpos->begin != lpos_begin)
return false;
break;
}
/* Advance @lpos_begin to the next data block. */
lpos_begin = blk_lpos->next;
}
*lpos_out = lpos_begin;
return true;
}
/*
* Advance the data ring tail to at least @lpos. This function puts
* descriptors into the reusable state if the tail is pushed beyond
* their associated data block.
*/
static bool data_push_tail(struct printk_ringbuffer *rb, unsigned long lpos)
{
struct prb_data_ring *data_ring = &rb->text_data_ring;
unsigned long tail_lpos_new;
unsigned long tail_lpos; unsigned long next_lpos;
/* If @lpos is from a data-less block, there is nothing to do. */
if (LPOS_DATALESS(lpos))
return true;
/*
* Any descriptor states that have transitioned to reusable due to the
* data tail being pushed to this loaded value will be visible to this
* CPU. This pairs with data_push_tail:D.
*
* Memory barrier involvement:
*
* If data_push_tail:A reads from data_push_tail:D, then this CPU can
* see desc_make_reusable:A.
*
* Relies on:
*
* MB from desc_make_reusable:A to data_push_tail:D
* matches
* READFROM from data_push_tail:D to data_push_tail:A
* thus
* READFROM from desc_make_reusable:A to this CPU
*/
tail_lpos = atomic_long_read(&data_ring->tail_lpos); /* LMM(data_push_tail:A) */
/*
* Loop until the tail lpos is at or beyond @lpos. This condition
* may already be satisfied, resulting in no full memory barrier
* from data_push_tail:D being performed. However, since this CPU
* sees the new tail lpos, any descriptor states that transitioned to
* the reusable state must already be visible.
*/
while ((lpos - tail_lpos) - 1 < DATA_SIZE(data_ring)) {
/*
* Make all descriptors reusable that are associated with
* data blocks before @lpos.
*/
if (!data_make_reusable(rb, tail_lpos, lpos, &next_lpos)) {
/*
* 1. Guarantee the block ID loaded in
* data_make_reusable() is performed before
* reloading the tail lpos. The failed
* data_make_reusable() may be due to a newly
* recycled data area causing the tail lpos to
* have been previously pushed. This pairs with
* data_alloc:A and data_realloc:A.
*
* Memory barrier involvement:
*
* If data_make_reusable:A reads from data_alloc:B,
* then data_push_tail:C reads from
* data_push_tail:D.
*
* Relies on:
*
* MB from data_push_tail:D to data_alloc:B
* matching
* RMB from data_make_reusable:A to
* data_push_tail:C
*
* Note: data_push_tail:D and data_alloc:B can be
* different CPUs. However, the data_alloc:B
* CPU (which performs the full memory
* barrier) must have previously seen
* data_push_tail:D.
*
* 2. Guarantee the descriptor state loaded in
* data_make_reusable() is performed before
* reloading the tail lpos. The failed
* data_make_reusable() may be due to a newly
* recycled descriptor causing the tail lpos to
* have been previously pushed. This pairs with
* desc_reserve:D.
*
* Memory barrier involvement:
*
* If data_make_reusable:B reads from
* desc_reserve:F, then data_push_tail:C reads
* from data_push_tail:D.
*
* Relies on:
*
* MB from data_push_tail:D to desc_reserve:F
* matching
* RMB from data_make_reusable:B to
* data_push_tail:C
*
* Note: data_push_tail:D and desc_reserve:F can
* be different CPUs. However, the
* desc_reserve:F CPU (which performs the
* full memory barrier) must have previously
* seen data_push_tail:D.
*/
smp_rmb(); /* LMM(data_push_tail:B) */
tail_lpos_new = atomic_long_read(&data_ring->tail_lpos
); /* LMM(data_push_tail:C) */
if (tail_lpos_new == tail_lpos)
return false;
/* Another CPU pushed the tail. Try again. */
tail_lpos = tail_lpos_new;
continue;
}
/*
* Guarantee any descriptor states that have transitioned to
* reusable are stored before pushing the tail lpos. A full
* memory barrier is needed since other CPUs may have made
* the descriptor states reusable. This pairs with
* data_push_tail:A.
*/
if (atomic_long_try_cmpxchg(&data_ring->tail_lpos, &tail_lpos,
next_lpos)) { /* LMM(data_push_tail:D) */
break;
}
}
return true;
}
/*
* Advance the desc ring tail. This function advances the tail by one
* descriptor, thus invalidating the oldest descriptor. Before advancing
* the tail, the tail descriptor is made reusable and all data blocks up to
* and including the descriptor's data block are invalidated (i.e. the data
* ring tail is pushed past the data block of the descriptor being made
* reusable).
*/
static bool desc_push_tail(struct printk_ringbuffer *rb,
unsigned long tail_id)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
enum desc_state d_state;
struct prb_desc desc;
d_state = desc_read(desc_ring, tail_id, &desc, NULL, NULL);
switch (d_state) {
case desc_miss:
/*
* If the ID is exactly 1 wrap behind the expected, it is
* in the process of being reserved by another writer and
* must be considered reserved.
*/
if (DESC_ID(atomic_long_read(&desc.state_var)) == DESC_ID_PREV_WRAP(desc_ring, tail_id)) {
return false;
}
/*
* The ID has changed. Another writer must have pushed the
* tail and recycled the descriptor already. Success is
* returned because the caller is only interested in the
* specified tail being pushed, which it was.
*/
return true;
case desc_reserved:
case desc_committed:
return false;
case desc_finalized:
desc_make_reusable(desc_ring, tail_id);
break;
case desc_reusable:
break;
}
/*
* Data blocks must be invalidated before their associated
* descriptor can be made available for recycling. Invalidating
* them later is not possible because there is no way to trust
* data blocks once their associated descriptor is gone.
*/
if (!data_push_tail(rb, desc.text_blk_lpos.next))
return false;
/*
* Check the next descriptor after @tail_id before pushing the tail
* to it because the tail must always be in a finalized or reusable
* state. The implementation of prb_first_seq() relies on this.
*
* A successful read implies that the next descriptor is less than or
* equal to @head_id so there is no risk of pushing the tail past the
* head.
*/
d_state = desc_read(desc_ring, DESC_ID(tail_id + 1), &desc,
NULL, NULL); /* LMM(desc_push_tail:A) */
if (d_state == desc_finalized || d_state == desc_reusable) {
/*
* Guarantee any descriptor states that have transitioned to
* reusable are stored before pushing the tail ID. This allows
* verifying the recycled descriptor state. A full memory
* barrier is needed since other CPUs may have made the
* descriptor states reusable. This pairs with desc_reserve:D.
*/
atomic_long_cmpxchg(&desc_ring->tail_id, tail_id,
DESC_ID(tail_id + 1)); /* LMM(desc_push_tail:B) */
} else {
/*
* Guarantee the last state load from desc_read() is before
* reloading @tail_id in order to see a new tail ID in the
* case that the descriptor has been recycled. This pairs
* with desc_reserve:D.
*
* Memory barrier involvement:
*
* If desc_push_tail:A reads from desc_reserve:F, then
* desc_push_tail:D reads from desc_push_tail:B.
*
* Relies on:
*
* MB from desc_push_tail:B to desc_reserve:F
* matching
* RMB from desc_push_tail:A to desc_push_tail:D
*
* Note: desc_push_tail:B and desc_reserve:F can be different
* CPUs. However, the desc_reserve:F CPU (which performs
* the full memory barrier) must have previously seen
* desc_push_tail:B.
*/
smp_rmb(); /* LMM(desc_push_tail:C) */
/*
* Re-check the tail ID. The descriptor following @tail_id is
* not in an allowed tail state. But if the tail has since
* been moved by another CPU, then it does not matter.
*/
if (atomic_long_read(&desc_ring->tail_id) == tail_id) /* LMM(desc_push_tail:D) */
return false;
}
return true;
}
/* Reserve a new descriptor, invalidating the oldest if necessary. */
static bool desc_reserve(struct printk_ringbuffer *rb, unsigned long *id_out)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
unsigned long prev_state_val;
unsigned long id_prev_wrap;
struct prb_desc *desc;
unsigned long head_id;
unsigned long id;
head_id = atomic_long_read(&desc_ring->head_id); /* LMM(desc_reserve:A) */
do {
id = DESC_ID(head_id + 1); id_prev_wrap = DESC_ID_PREV_WRAP(desc_ring, id);
/*
* Guarantee the head ID is read before reading the tail ID.
* Since the tail ID is updated before the head ID, this
* guarantees that @id_prev_wrap is never ahead of the tail
* ID. This pairs with desc_reserve:D.
*
* Memory barrier involvement:
*
* If desc_reserve:A reads from desc_reserve:D, then
* desc_reserve:C reads from desc_push_tail:B.
*
* Relies on:
*
* MB from desc_push_tail:B to desc_reserve:D
* matching
* RMB from desc_reserve:A to desc_reserve:C
*
* Note: desc_push_tail:B and desc_reserve:D can be different
* CPUs. However, the desc_reserve:D CPU (which performs
* the full memory barrier) must have previously seen
* desc_push_tail:B.
*/
smp_rmb(); /* LMM(desc_reserve:B) */
if (id_prev_wrap == atomic_long_read(&desc_ring->tail_id
)) { /* LMM(desc_reserve:C) */
/*
* Make space for the new descriptor by
* advancing the tail.
*/
if (!desc_push_tail(rb, id_prev_wrap))
return false;
}
/*
* 1. Guarantee the tail ID is read before validating the
* recycled descriptor state. A read memory barrier is
* sufficient for this. This pairs with desc_push_tail:B.
*
* Memory barrier involvement:
*
* If desc_reserve:C reads from desc_push_tail:B, then
* desc_reserve:E reads from desc_make_reusable:A.
*
* Relies on:
*
* MB from desc_make_reusable:A to desc_push_tail:B
* matching
* RMB from desc_reserve:C to desc_reserve:E
*
* Note: desc_make_reusable:A and desc_push_tail:B can be
* different CPUs. However, the desc_push_tail:B CPU
* (which performs the full memory barrier) must have
* previously seen desc_make_reusable:A.
*
* 2. Guarantee the tail ID is stored before storing the head
* ID. This pairs with desc_reserve:B.
*
* 3. Guarantee any data ring tail changes are stored before
* recycling the descriptor. Data ring tail changes can
* happen via desc_push_tail()->data_push_tail(). A full
* memory barrier is needed since another CPU may have
* pushed the data ring tails. This pairs with
* data_push_tail:B.
*
* 4. Guarantee a new tail ID is stored before recycling the
* descriptor. A full memory barrier is needed since
* another CPU may have pushed the tail ID. This pairs
* with desc_push_tail:C and this also pairs with
* prb_first_seq:C.
*
* 5. Guarantee the head ID is stored before trying to
* finalize the previous descriptor. This pairs with
* _prb_commit:B.
*/
} while (!atomic_long_try_cmpxchg(&desc_ring->head_id, &head_id,
id)); /* LMM(desc_reserve:D) */
desc = to_desc(desc_ring, id);
/*
* If the descriptor has been recycled, verify the old state val.
* See "ABA Issues" about why this verification is performed.
*/
prev_state_val = atomic_long_read(&desc->state_var); /* LMM(desc_reserve:E) */
if (prev_state_val &&
get_desc_state(id_prev_wrap, prev_state_val) != desc_reusable) { WARN_ON_ONCE(1); return false;
}
/*
* Assign the descriptor a new ID and set its state to reserved.
* See "ABA Issues" about why cmpxchg() instead of set() is used.
*
* Guarantee the new descriptor ID and state is stored before making
* any other changes. A write memory barrier is sufficient for this.
* This pairs with desc_read:D.
*/
if (!atomic_long_try_cmpxchg(&desc->state_var, &prev_state_val,
DESC_SV(id, desc_reserved))) { /* LMM(desc_reserve:F) */
WARN_ON_ONCE(1);
return false;
}
/* Now data in @desc can be modified: LMM(desc_reserve:G) */
*id_out = id;
return true;
}
/* Determine the end of a data block. */
static unsigned long get_next_lpos(struct prb_data_ring *data_ring,
unsigned long lpos, unsigned int size)
{
unsigned long begin_lpos;
unsigned long next_lpos;
begin_lpos = lpos;
next_lpos = lpos + size;
/* First check if the data block does not wrap. */
if (DATA_WRAPS(data_ring, begin_lpos) == DATA_WRAPS(data_ring, next_lpos))
return next_lpos;
/* Wrapping data blocks store their data at the beginning. */
return (DATA_THIS_WRAP_START_LPOS(data_ring, next_lpos) + size);
}
/*
* Allocate a new data block, invalidating the oldest data block(s)
* if necessary. This function also associates the data block with
* a specified descriptor.
*/
static char *data_alloc(struct printk_ringbuffer *rb, unsigned int size,
struct prb_data_blk_lpos *blk_lpos, unsigned long id)
{
struct prb_data_ring *data_ring = &rb->text_data_ring;
struct prb_data_block *blk;
unsigned long begin_lpos;
unsigned long next_lpos;
if (size == 0) {
/*
* Data blocks are not created for empty lines. Instead, the
* reader will recognize these special lpos values and handle
* it appropriately.
*/
blk_lpos->begin = EMPTY_LINE_LPOS;
blk_lpos->next = EMPTY_LINE_LPOS;
return NULL;
}
size = to_blk_size(size);
begin_lpos = atomic_long_read(&data_ring->head_lpos);
do {
next_lpos = get_next_lpos(data_ring, begin_lpos, size); if (!data_push_tail(rb, next_lpos - DATA_SIZE(data_ring))) {
/* Failed to allocate, specify a data-less block. */
blk_lpos->begin = FAILED_LPOS;
blk_lpos->next = FAILED_LPOS;
return NULL;
}
/*
* 1. Guarantee any descriptor states that have transitioned
* to reusable are stored before modifying the newly
* allocated data area. A full memory barrier is needed
* since other CPUs may have made the descriptor states
* reusable. See data_push_tail:A about why the reusable
* states are visible. This pairs with desc_read:D.
*
* 2. Guarantee any updated tail lpos is stored before
* modifying the newly allocated data area. Another CPU may
* be in data_make_reusable() and is reading a block ID
* from this area. data_make_reusable() can handle reading
* a garbage block ID value, but then it must be able to
* load a new tail lpos. A full memory barrier is needed
* since other CPUs may have updated the tail lpos. This
* pairs with data_push_tail:B.
*/
} while (!atomic_long_try_cmpxchg(&data_ring->head_lpos, &begin_lpos,
next_lpos)); /* LMM(data_alloc:A) */
blk = to_block(data_ring, begin_lpos);
blk->id = id; /* LMM(data_alloc:B) */
if (DATA_WRAPS(data_ring, begin_lpos) != DATA_WRAPS(data_ring, next_lpos)) {
/* Wrapping data blocks store their data at the beginning. */
blk = to_block(data_ring, 0);
/*
* Store the ID on the wrapped block for consistency.
* The printk_ringbuffer does not actually use it.
*/
blk->id = id;
}
blk_lpos->begin = begin_lpos;
blk_lpos->next = next_lpos;
return &blk->data[0];
}
/*
* Try to resize an existing data block associated with the descriptor
* specified by @id. If the resized data block should become wrapped, it
* copies the old data to the new data block. If @size yields a data block
* with the same or less size, the data block is left as is.
*
* Fail if this is not the last allocated data block or if there is not
* enough space or it is not possible make enough space.
*
* Return a pointer to the beginning of the entire data buffer or NULL on
* failure.
*/
static char *data_realloc(struct printk_ringbuffer *rb, unsigned int size,
struct prb_data_blk_lpos *blk_lpos, unsigned long id)
{
struct prb_data_ring *data_ring = &rb->text_data_ring;
struct prb_data_block *blk;
unsigned long head_lpos;
unsigned long next_lpos;
bool wrapped;
/* Reallocation only works if @blk_lpos is the newest data block. */
head_lpos = atomic_long_read(&data_ring->head_lpos);
if (head_lpos != blk_lpos->next)
return NULL;
/* Keep track if @blk_lpos was a wrapping data block. */
wrapped = (DATA_WRAPS(data_ring, blk_lpos->begin) != DATA_WRAPS(data_ring, blk_lpos->next));
size = to_blk_size(size);
next_lpos = get_next_lpos(data_ring, blk_lpos->begin, size);
/* If the data block does not increase, there is nothing to do. */
if (head_lpos - next_lpos < DATA_SIZE(data_ring)) {
if (wrapped)
blk = to_block(data_ring, 0);
else
blk = to_block(data_ring, blk_lpos->begin);
return &blk->data[0];
}
if (!data_push_tail(rb, next_lpos - DATA_SIZE(data_ring)))
return NULL;
/* The memory barrier involvement is the same as data_alloc:A. */
if (!atomic_long_try_cmpxchg(&data_ring->head_lpos, &head_lpos,
next_lpos)) { /* LMM(data_realloc:A) */
return NULL;
}
blk = to_block(data_ring, blk_lpos->begin);
if (DATA_WRAPS(data_ring, blk_lpos->begin) != DATA_WRAPS(data_ring, next_lpos)) {
struct prb_data_block *old_blk = blk;
/* Wrapping data blocks store their data at the beginning. */
blk = to_block(data_ring, 0);
/*
* Store the ID on the wrapped block for consistency.
* The printk_ringbuffer does not actually use it.
*/
blk->id = id;
if (!wrapped) {
/*
* Since the allocated space is now in the newly
* created wrapping data block, copy the content
* from the old data block.
*/
memcpy(&blk->data[0], &old_blk->data[0],
(blk_lpos->next - blk_lpos->begin) - sizeof(blk->id));
}
}
blk_lpos->next = next_lpos;
return &blk->data[0];
}
/* Return the number of bytes used by a data block. */
static unsigned int space_used(struct prb_data_ring *data_ring,
struct prb_data_blk_lpos *blk_lpos)
{
/* Data-less blocks take no space. */
if (BLK_DATALESS(blk_lpos))
return 0;
if (DATA_WRAPS(data_ring, blk_lpos->begin) == DATA_WRAPS(data_ring, blk_lpos->next)) {
/* Data block does not wrap. */
return (DATA_INDEX(data_ring, blk_lpos->next) -
DATA_INDEX(data_ring, blk_lpos->begin));
}
/*
* For wrapping data blocks, the trailing (wasted) space is
* also counted.
*/
return (DATA_INDEX(data_ring, blk_lpos->next) + DATA_SIZE(data_ring) - DATA_INDEX(data_ring, blk_lpos->begin));
}
/*
* Given @blk_lpos, return a pointer to the writer data from the data block
* and calculate the size of the data part. A NULL pointer is returned if
* @blk_lpos specifies values that could never be legal.
*
* This function (used by readers) performs strict validation on the lpos
* values to possibly detect bugs in the writer code. A WARN_ON_ONCE() is
* triggered if an internal error is detected.
*/
static const char *get_data(struct prb_data_ring *data_ring,
struct prb_data_blk_lpos *blk_lpos,
unsigned int *data_size)
{
struct prb_data_block *db;
/* Data-less data block description. */
if (BLK_DATALESS(blk_lpos)) {
/*
* Records that are just empty lines are also valid, even
* though they do not have a data block. For such records
* explicitly return empty string data to signify success.
*/
if (blk_lpos->begin == EMPTY_LINE_LPOS &&
blk_lpos->next == EMPTY_LINE_LPOS) {
*data_size = 0;
return "";
}
/* Data lost, invalid, or otherwise unavailable. */
return NULL;
}
/* Regular data block: @begin less than @next and in same wrap. */
if (DATA_WRAPS(data_ring, blk_lpos->begin) == DATA_WRAPS(data_ring, blk_lpos->next) &&
blk_lpos->begin < blk_lpos->next) {
db = to_block(data_ring, blk_lpos->begin); *data_size = blk_lpos->next - blk_lpos->begin;
/* Wrapping data block: @begin is one wrap behind @next. */
} else if (DATA_WRAPS(data_ring, blk_lpos->begin + DATA_SIZE(data_ring)) ==
DATA_WRAPS(data_ring, blk_lpos->next)) {
db = to_block(data_ring, 0); *data_size = DATA_INDEX(data_ring, blk_lpos->next);
/* Illegal block description. */
} else {
WARN_ON_ONCE(1);
return NULL;
}
/* A valid data block will always be aligned to the ID size. */
if (WARN_ON_ONCE(blk_lpos->begin != ALIGN(blk_lpos->begin, sizeof(db->id))) || WARN_ON_ONCE(blk_lpos->next != ALIGN(blk_lpos->next, sizeof(db->id)))) {
return NULL;
}
/* A valid data block will always have at least an ID. */
if (WARN_ON_ONCE(*data_size < sizeof(db->id)))
return NULL;
/* Subtract block ID space from size to reflect data size. */
*data_size -= sizeof(db->id);
return &db->data[0];
}
/*
* Attempt to transition the newest descriptor from committed back to reserved
* so that the record can be modified by a writer again. This is only possible
* if the descriptor is not yet finalized and the provided @caller_id matches.
*/
static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring,
u32 caller_id, unsigned long *id_out)
{
unsigned long prev_state_val;
enum desc_state d_state;
struct prb_desc desc;
struct prb_desc *d;
unsigned long id;
u32 cid;
id = atomic_long_read(&desc_ring->head_id);
/*
* To reduce unnecessarily reopening, first check if the descriptor
* state and caller ID are correct.
*/
d_state = desc_read(desc_ring, id, &desc, NULL, &cid);
if (d_state != desc_committed || cid != caller_id)
return NULL;
d = to_desc(desc_ring, id);
prev_state_val = DESC_SV(id, desc_committed);
/*
* Guarantee the reserved state is stored before reading any
* record data. A full memory barrier is needed because @state_var
* modification is followed by reading. This pairs with _prb_commit:B.
*
* Memory barrier involvement:
*
* If desc_reopen_last:A reads from _prb_commit:B, then
* prb_reserve_in_last:A reads from _prb_commit:A.
*
* Relies on:
*
* WMB from _prb_commit:A to _prb_commit:B
* matching
* MB If desc_reopen_last:A to prb_reserve_in_last:A
*/
if (!atomic_long_try_cmpxchg(&d->state_var, &prev_state_val,
DESC_SV(id, desc_reserved))) { /* LMM(desc_reopen_last:A) */
return NULL;
}
*id_out = id;
return d;
}
/**
* prb_reserve_in_last() - Re-reserve and extend the space in the ringbuffer
* used by the newest record.
*
* @e: The entry structure to setup.
* @rb: The ringbuffer to re-reserve and extend data in.
* @r: The record structure to allocate buffers for.
* @caller_id: The caller ID of the caller (reserving writer).
* @max_size: Fail if the extended size would be greater than this.
*
* This is the public function available to writers to re-reserve and extend
* data.
*
* The writer specifies the text size to extend (not the new total size) by
* setting the @text_buf_size field of @r. To ensure proper initialization
* of @r, prb_rec_init_wr() should be used.
*
* This function will fail if @caller_id does not match the caller ID of the
* newest record. In that case the caller must reserve new data using
* prb_reserve().
*
* Context: Any context. Disables local interrupts on success.
* Return: true if text data could be extended, otherwise false.
*
* On success:
*
* - @r->text_buf points to the beginning of the entire text buffer.
*
* - @r->text_buf_size is set to the new total size of the buffer.
*
* - @r->info is not touched so that @r->info->text_len could be used
* to append the text.
*
* - prb_record_text_space() can be used on @e to query the new
* actually used space.
*
* Important: All @r->info fields will already be set with the current values
* for the record. I.e. @r->info->text_len will be less than
* @text_buf_size. Writers can use @r->info->text_len to know
* where concatenation begins and writers should update
* @r->info->text_len after concatenating.
*/
bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
struct printk_record *r, u32 caller_id, unsigned int max_size)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
struct printk_info *info;
unsigned int data_size;
struct prb_desc *d;
unsigned long id;
local_irq_save(e->irqflags);
/* Transition the newest descriptor back to the reserved state. */
d = desc_reopen_last(desc_ring, caller_id, &id);
if (!d) {
local_irq_restore(e->irqflags);
goto fail_reopen;
}
/* Now the writer has exclusive access: LMM(prb_reserve_in_last:A) */
info = to_info(desc_ring, id);
/*
* Set the @e fields here so that prb_commit() can be used if
* anything fails from now on.
*/
e->rb = rb;
e->id = id;
/*
* desc_reopen_last() checked the caller_id, but there was no
* exclusive access at that point. The descriptor may have
* changed since then.
*/
if (caller_id != info->caller_id)
goto fail;
if (BLK_DATALESS(&d->text_blk_lpos)) {
if (WARN_ON_ONCE(info->text_len != 0)) {
pr_warn_once("wrong text_len value (%hu, expecting 0)\n",
info->text_len);
info->text_len = 0;
}
if (!data_check_size(&rb->text_data_ring, r->text_buf_size))
goto fail;
if (r->text_buf_size > max_size)
goto fail;
r->text_buf = data_alloc(rb, r->text_buf_size,
&d->text_blk_lpos, id);
} else {
if (!get_data(&rb->text_data_ring, &d->text_blk_lpos, &data_size))
goto fail;
/*
* Increase the buffer size to include the original size. If
* the meta data (@text_len) is not sane, use the full data
* block size.
*/
if (WARN_ON_ONCE(info->text_len > data_size)) {
pr_warn_once("wrong text_len value (%hu, expecting <=%u)\n",
info->text_len, data_size);
info->text_len = data_size;
}
r->text_buf_size += info->text_len;
if (!data_check_size(&rb->text_data_ring, r->text_buf_size))
goto fail;
if (r->text_buf_size > max_size)
goto fail;
r->text_buf = data_realloc(rb, r->text_buf_size,
&d->text_blk_lpos, id);
}
if (r->text_buf_size && !r->text_buf)
goto fail;
r->info = info;
e->text_space = space_used(&rb->text_data_ring, &d->text_blk_lpos);
return true;
fail:
prb_commit(e);
/* prb_commit() re-enabled interrupts. */
fail_reopen:
/* Make it clear to the caller that the re-reserve failed. */
memset(r, 0, sizeof(*r));
return false;
}
/*
* @last_finalized_seq value guarantees that all records up to and including
* this sequence number are finalized and can be read. The only exception are
* too old records which have already been overwritten.
*
* It is also guaranteed that @last_finalized_seq only increases.
*
* Be aware that finalized records following non-finalized records are not
* reported because they are not yet available to the reader. For example,
* a new record stored via printk() will not be available to a printer if
* it follows a record that has not been finalized yet. However, once that
* non-finalized record becomes finalized, @last_finalized_seq will be
* appropriately updated and the full set of finalized records will be
* available to the printer. And since each printk() caller will either
* directly print or trigger deferred printing of all available unprinted
* records, all printk() messages will get printed.
*/
static u64 desc_last_finalized_seq(struct printk_ringbuffer *rb)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
unsigned long ulseq;
/*
* Guarantee the sequence number is loaded before loading the
* associated record in order to guarantee that the record can be
* seen by this CPU. This pairs with desc_update_last_finalized:A.
*/
ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq
); /* LMM(desc_last_finalized_seq:A) */
return __ulseq_to_u64seq(rb, ulseq);
}
static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq,
struct printk_record *r, unsigned int *line_count);
/*
* Check if there are records directly following @last_finalized_seq that are
* finalized. If so, update @last_finalized_seq to the latest of these
* records. It is not allowed to skip over records that are not yet finalized.
*/
static void desc_update_last_finalized(struct printk_ringbuffer *rb)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
u64 old_seq = desc_last_finalized_seq(rb);
unsigned long oldval;
unsigned long newval;
u64 finalized_seq;
u64 try_seq;
try_again:
finalized_seq = old_seq;
try_seq = finalized_seq + 1;
/* Try to find later finalized records. */
while (_prb_read_valid(rb, &try_seq, NULL, NULL)) {
finalized_seq = try_seq;
try_seq++;
}
/* No update needed if no later finalized record was found. */
if (finalized_seq == old_seq) return; oldval = __u64seq_to_ulseq(old_seq);
newval = __u64seq_to_ulseq(finalized_seq);
/*
* Set the sequence number of a later finalized record that has been
* seen.
*
* Guarantee the record data is visible to other CPUs before storing
* its sequence number. This pairs with desc_last_finalized_seq:A.
*
* Memory barrier involvement:
*
* If desc_last_finalized_seq:A reads from
* desc_update_last_finalized:A, then desc_read:A reads from
* _prb_commit:B.
*
* Relies on:
*
* RELEASE from _prb_commit:B to desc_update_last_finalized:A
* matching
* ACQUIRE from desc_last_finalized_seq:A to desc_read:A
*
* Note: _prb_commit:B and desc_update_last_finalized:A can be
* different CPUs. However, the desc_update_last_finalized:A
* CPU (which performs the release) must have previously seen
* _prb_commit:B.
*/
if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq,
&oldval, newval)) { /* LMM(desc_update_last_finalized:A) */
old_seq = __ulseq_to_u64seq(rb, oldval);
goto try_again;
}
}
/*
* Attempt to finalize a specified descriptor. If this fails, the descriptor
* is either already final or it will finalize itself when the writer commits.
*/
static void desc_make_final(struct printk_ringbuffer *rb, unsigned long id)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
unsigned long prev_state_val = DESC_SV(id, desc_committed); struct prb_desc *d = to_desc(desc_ring, id);
if (atomic_long_try_cmpxchg_relaxed(&d->state_var, &prev_state_val,
DESC_SV(id, desc_finalized))) { /* LMM(desc_make_final:A) */
desc_update_last_finalized(rb);
}
}
/**
* prb_reserve() - Reserve space in the ringbuffer.
*
* @e: The entry structure to setup.
* @rb: The ringbuffer to reserve data in.
* @r: The record structure to allocate buffers for.
*
* This is the public function available to writers to reserve data.
*
* The writer specifies the text size to reserve by setting the
* @text_buf_size field of @r. To ensure proper initialization of @r,
* prb_rec_init_wr() should be used.
*
* Context: Any context. Disables local interrupts on success.
* Return: true if at least text data could be allocated, otherwise false.
*
* On success, the fields @info and @text_buf of @r will be set by this
* function and should be filled in by the writer before committing. Also
* on success, prb_record_text_space() can be used on @e to query the actual
* space used for the text data block.
*
* Important: @info->text_len needs to be set correctly by the writer in
* order for data to be readable and/or extended. Its value
* is initialized to 0.
*/
bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
struct printk_record *r)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
struct printk_info *info;
struct prb_desc *d;
unsigned long id;
u64 seq;
if (!data_check_size(&rb->text_data_ring, r->text_buf_size))
goto fail;
/*
* Descriptors in the reserved state act as blockers to all further
* reservations once the desc_ring has fully wrapped. Disable
* interrupts during the reserve/commit window in order to minimize
* the likelihood of this happening.
*/
local_irq_save(e->irqflags); if (!desc_reserve(rb, &id)) {
/* Descriptor reservation failures are tracked. */
atomic_long_inc(&rb->fail);
local_irq_restore(e->irqflags);
goto fail;
}
d = to_desc(desc_ring, id); info = to_info(desc_ring, id);
/*
* All @info fields (except @seq) are cleared and must be filled in
* by the writer. Save @seq before clearing because it is used to
* determine the new sequence number.
*/
seq = info->seq;
memset(info, 0, sizeof(*info));
/*
* Set the @e fields here so that prb_commit() can be used if
* text data allocation fails.
*/
e->rb = rb;
e->id = id;
/*
* Initialize the sequence number if it has "never been set".
* Otherwise just increment it by a full wrap.
*
* @seq is considered "never been set" if it has a value of 0,
* _except_ for @infos[0], which was specially setup by the ringbuffer
* initializer and therefore is always considered as set.
*
* See the "Bootstrap" comment block in printk_ringbuffer.h for
* details about how the initializer bootstraps the descriptors.
*/
if (seq == 0 && DESC_INDEX(desc_ring, id) != 0) info->seq = DESC_INDEX(desc_ring, id);
else
info->seq = seq + DESCS_COUNT(desc_ring);
/*
* New data is about to be reserved. Once that happens, previous
* descriptors are no longer able to be extended. Finalize the
* previous descriptor now so that it can be made available to
* readers. (For seq==0 there is no previous descriptor.)
*/
if (info->seq > 0)
desc_make_final(rb, DESC_ID(id - 1)); r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id);
/* If text data allocation fails, a data-less record is committed. */
if (r->text_buf_size && !r->text_buf) { prb_commit(e);
/* prb_commit() re-enabled interrupts. */
goto fail;
}
r->info = info;
/* Record full text space used by record. */
e->text_space = space_used(&rb->text_data_ring, &d->text_blk_lpos);
return true;
fail:
/* Make it clear to the caller that the reserve failed. */
memset(r, 0, sizeof(*r)); return false;
}
/* Commit the data (possibly finalizing it) and restore interrupts. */
static void _prb_commit(struct prb_reserved_entry *e, unsigned long state_val)
{
struct prb_desc_ring *desc_ring = &e->rb->desc_ring; struct prb_desc *d = to_desc(desc_ring, e->id);
unsigned long prev_state_val = DESC_SV(e->id, desc_reserved);
/* Now the writer has finished all writing: LMM(_prb_commit:A) */
/*
* Set the descriptor as committed. See "ABA Issues" about why
* cmpxchg() instead of set() is used.
*
* 1 Guarantee all record data is stored before the descriptor state
* is stored as committed. A write memory barrier is sufficient
* for this. This pairs with desc_read:B and desc_reopen_last:A.
*
* 2. Guarantee the descriptor state is stored as committed before
* re-checking the head ID in order to possibly finalize this
* descriptor. This pairs with desc_reserve:D.
*
* Memory barrier involvement:
*
* If prb_commit:A reads from desc_reserve:D, then
* desc_make_final:A reads from _prb_commit:B.
*
* Relies on:
*
* MB _prb_commit:B to prb_commit:A
* matching
* MB desc_reserve:D to desc_make_final:A
*/
if (!atomic_long_try_cmpxchg(&d->state_var, &prev_state_val,
DESC_SV(e->id, state_val))) { /* LMM(_prb_commit:B) */
WARN_ON_ONCE(1);
}
/* Restore interrupts, the reserve/commit window is finished. */
local_irq_restore(e->irqflags);
}
/**
* prb_commit() - Commit (previously reserved) data to the ringbuffer.
*
* @e: The entry containing the reserved data information.
*
* This is the public function available to writers to commit data.
*
* Note that the data is not yet available to readers until it is finalized.
* Finalizing happens automatically when space for the next record is
* reserved.
*
* See prb_final_commit() for a version of this function that finalizes
* immediately.
*
* Context: Any context. Enables local interrupts.
*/
void prb_commit(struct prb_reserved_entry *e)
{
struct prb_desc_ring *desc_ring = &e->rb->desc_ring;
unsigned long head_id;
_prb_commit(e, desc_committed);
/*
* If this descriptor is no longer the head (i.e. a new record has
* been allocated), extending the data for this record is no longer
* allowed and therefore it must be finalized.
*/
head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */
if (head_id != e->id)
desc_make_final(e->rb, e->id);
}
/**
* prb_final_commit() - Commit and finalize (previously reserved) data to
* the ringbuffer.
*
* @e: The entry containing the reserved data information.
*
* This is the public function available to writers to commit+finalize data.
*
* By finalizing, the data is made immediately available to readers.
*
* This function should only be used if there are no intentions of extending
* this data using prb_reserve_in_last().
*
* Context: Any context. Enables local interrupts.
*/
void prb_final_commit(struct prb_reserved_entry *e)
{
_prb_commit(e, desc_finalized);
desc_update_last_finalized(e->rb);
}
/*
* Count the number of lines in provided text. All text has at least 1 line
* (even if @text_size is 0). Each '\n' processed is counted as an additional
* line.
*/
static unsigned int count_lines(const char *text, unsigned int text_size)
{
unsigned int next_size = text_size;
unsigned int line_count = 1;
const char *next = text;
while (next_size) { next = memchr(next, '\n', next_size);
if (!next)
break;
line_count++;
next++;
next_size = text_size - (next - text);
}
return line_count;
}
/*
* Given @blk_lpos, copy an expected @len of data into the provided buffer.
* If @line_count is provided, count the number of lines in the data.
*
* This function (used by readers) performs strict validation on the data
* size to possibly detect bugs in the writer code. A WARN_ON_ONCE() is
* triggered if an internal error is detected.
*/
static bool copy_data(struct prb_data_ring *data_ring,
struct prb_data_blk_lpos *blk_lpos, u16 len, char *buf,
unsigned int buf_size, unsigned int *line_count)
{
unsigned int data_size;
const char *data;
/* Caller might not want any data. */
if ((!buf || !buf_size) && !line_count)
return true;
data = get_data(data_ring, blk_lpos, &data_size);
if (!data)
return false;
/*
* Actual cannot be less than expected. It can be more than expected
* because of the trailing alignment padding.
*
* Note that invalid @len values can occur because the caller loads
* the value during an allowed data race.
*/
if (data_size < (unsigned int)len)
return false;
/* Caller interested in the line count? */
if (line_count) *line_count = count_lines(data, len);
/* Caller interested in the data content? */
if (!buf || !buf_size)
return true;
data_size = min_t(unsigned int, buf_size, len);
memcpy(&buf[0], data, data_size); /* LMM(copy_data:A) */
return true;
}
/*
* This is an extended version of desc_read(). It gets a copy of a specified
* descriptor. However, it also verifies that the record is finalized and has
* the sequence number @seq. On success, 0 is returned.
*
* Error return values:
* -EINVAL: A finalized record with sequence number @seq does not exist.
* -ENOENT: A finalized record with sequence number @seq exists, but its data
* is not available. This is a valid record, so readers should
* continue with the next record.
*/
static int desc_read_finalized_seq(struct prb_desc_ring *desc_ring,
unsigned long id, u64 seq,
struct prb_desc *desc_out)
{
struct prb_data_blk_lpos *blk_lpos = &desc_out->text_blk_lpos;
enum desc_state d_state;
u64 s;
d_state = desc_read(desc_ring, id, desc_out, &s, NULL);
/*
* An unexpected @id (desc_miss) or @seq mismatch means the record
* does not exist. A descriptor in the reserved or committed state
* means the record does not yet exist for the reader.
*/
if (d_state == desc_miss ||
d_state == desc_reserved ||
d_state == desc_committed ||
s != seq) {
return -EINVAL;
}
/*
* A descriptor in the reusable state may no longer have its data
* available; report it as existing but with lost data. Or the record
* may actually be a record with lost data.
*/
if (d_state == desc_reusable || (blk_lpos->begin == FAILED_LPOS && blk_lpos->next == FAILED_LPOS)) {
return -ENOENT;
}
return 0;
}
/*
* Copy the ringbuffer data from the record with @seq to the provided
* @r buffer. On success, 0 is returned.
*
* See desc_read_finalized_seq() for error return values.
*/
static int prb_read(struct printk_ringbuffer *rb, u64 seq,
struct printk_record *r, unsigned int *line_count)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring; struct printk_info *info = to_info(desc_ring, seq); struct prb_desc *rdesc = to_desc(desc_ring, seq);
atomic_long_t *state_var = &rdesc->state_var;
struct prb_desc desc;
unsigned long id;
int err;
/* Extract the ID, used to specify the descriptor to read. */
id = DESC_ID(atomic_long_read(state_var));
/* Get a local copy of the correct descriptor (if available). */
err = desc_read_finalized_seq(desc_ring, id, seq, &desc);
/*
* If @r is NULL, the caller is only interested in the availability
* of the record.
*/
if (err || !r)
return err;
/* If requested, copy meta data. */
if (r->info) memcpy(r->info, info, sizeof(*(r->info)));
/* Copy text data. If it fails, this is a data-less record. */
if (!copy_data(&rb->text_data_ring, &desc.text_blk_lpos, info->text_len,
r->text_buf, r->text_buf_size, line_count)) {
return -ENOENT;
}
/* Ensure the record is still finalized and has the same @seq. */
return desc_read_finalized_seq(desc_ring, id, seq, &desc);
}
/* Get the sequence number of the tail descriptor. */
u64 prb_first_seq(struct printk_ringbuffer *rb)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
enum desc_state d_state;
struct prb_desc desc;
unsigned long id;
u64 seq;
for (;;) {
id = atomic_long_read(&rb->desc_ring.tail_id); /* LMM(prb_first_seq:A) */
d_state = desc_read(desc_ring, id, &desc, &seq, NULL); /* LMM(prb_first_seq:B) */
/*
* This loop will not be infinite because the tail is
* _always_ in the finalized or reusable state.
*/
if (d_state == desc_finalized || d_state == desc_reusable)
break;
/*
* Guarantee the last state load from desc_read() is before
* reloading @tail_id in order to see a new tail in the case
* that the descriptor has been recycled. This pairs with
* desc_reserve:D.
*
* Memory barrier involvement:
*
* If prb_first_seq:B reads from desc_reserve:F, then
* prb_first_seq:A reads from desc_push_tail:B.
*
* Relies on:
*
* MB from desc_push_tail:B to desc_reserve:F
* matching
* RMB prb_first_seq:B to prb_first_seq:A
*/
smp_rmb(); /* LMM(prb_first_seq:C) */
}
return seq;
}
/**
* prb_next_reserve_seq() - Get the sequence number after the most recently
* reserved record.
*
* @rb: The ringbuffer to get the sequence number from.
*
* This is the public function available to readers to see what sequence
* number will be assigned to the next reserved record.
*
* Note that depending on the situation, this value can be equal to or
* higher than the sequence number returned by prb_next_seq().
*
* Context: Any context.
* Return: The sequence number that will be assigned to the next record
* reserved.
*/
u64 prb_next_reserve_seq(struct printk_ringbuffer *rb)
{
struct prb_desc_ring *desc_ring = &rb->desc_ring;
unsigned long last_finalized_id;
atomic_long_t *state_var;
u64 last_finalized_seq;
unsigned long head_id;
struct prb_desc desc;
unsigned long diff;
struct prb_desc *d;
int err;
/*
* It may not be possible to read a sequence number for @head_id.
* So the ID of @last_finailzed_seq is used to calculate what the
* sequence number of @head_id will be.
*/
try_again:
last_finalized_seq = desc_last_finalized_seq(rb);
/*
* @head_id is loaded after @last_finalized_seq to ensure that
* it points to the record with @last_finalized_seq or newer.
*
* Memory barrier involvement:
*
* If desc_last_finalized_seq:A reads from
* desc_update_last_finalized:A, then
* prb_next_reserve_seq:A reads from desc_reserve:D.
*
* Relies on:
*
* RELEASE from desc_reserve:D to desc_update_last_finalized:A
* matching
* ACQUIRE from desc_last_finalized_seq:A to prb_next_reserve_seq:A
*
* Note: desc_reserve:D and desc_update_last_finalized:A can be
* different CPUs. However, the desc_update_last_finalized:A CPU
* (which performs the release) must have previously seen
* desc_read:C, which implies desc_reserve:D can be seen.
*/
head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_next_reserve_seq:A) */
d = to_desc(desc_ring, last_finalized_seq);
state_var = &d->state_var;
/* Extract the ID, used to specify the descriptor to read. */
last_finalized_id = DESC_ID(atomic_long_read(state_var));
/* Ensure @last_finalized_id is correct. */
err = desc_read_finalized_seq(desc_ring, last_finalized_id, last_finalized_seq, &desc);
if (err == -EINVAL) {
if (last_finalized_seq == 0) {
/*
* No record has been finalized or even reserved yet.
*
* The @head_id is initialized such that the first
* increment will yield the first record (seq=0).
* Handle it separately to avoid a negative @diff
* below.
*/
if (head_id == DESC0_ID(desc_ring->count_bits))
return 0;
/*
* One or more descriptors are already reserved. Use
* the descriptor ID of the first one (@seq=0) for
* the @diff below.
*/
last_finalized_id = DESC0_ID(desc_ring->count_bits) + 1;
} else {
/* Record must have been overwritten. Try again. */
goto try_again;
}
}
/* Diff of known descriptor IDs to compute related sequence numbers. */
diff = head_id - last_finalized_id;
/*
* @head_id points to the most recently reserved record, but this
* function returns the sequence number that will be assigned to the
* next (not yet reserved) record. Thus +1 is needed.
*/
return (last_finalized_seq + diff + 1);
}
/*
* Non-blocking read of a record.
*
* On success @seq is updated to the record that was read and (if provided)
* @r and @line_count will contain the read/calculated data.
*
* On failure @seq is updated to a record that is not yet available to the
* reader, but it will be the next record available to the reader.
*
* Note: When the current CPU is in panic, this function will skip over any
* non-existent/non-finalized records in order to allow the panic CPU
* to print any and all records that have been finalized.
*/
static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq,
struct printk_record *r, unsigned int *line_count)
{
u64 tail_seq;
int err;
while ((err = prb_read(rb, *seq, r, line_count))) { tail_seq = prb_first_seq(rb);
if (*seq < tail_seq) {
/*
* Behind the tail. Catch up and try again. This
* can happen for -ENOENT and -EINVAL cases.
*/
*seq = tail_seq;
} else if (err == -ENOENT) {
/* Record exists, but the data was lost. Skip. */
(*seq)++;
} else {
/*
* Non-existent/non-finalized record. Must stop.
*
* For panic situations it cannot be expected that
* non-finalized records will become finalized. But
* there may be other finalized records beyond that
* need to be printed for a panic situation. If this
* is the panic CPU, skip this
* non-existent/non-finalized record unless it is
* at or beyond the head, in which case it is not
* possible to continue.
*
* Note that new messages printed on panic CPU are
* finalized when we are here. The only exception
* might be the last message without trailing newline.
* But it would have the sequence number returned
* by "prb_next_reserve_seq() - 1".
*/
if (this_cpu_in_panic() && ((*seq + 1) < prb_next_reserve_seq(rb))) (*seq)++;
else
return false;
}
}
return true;
}
/**
* prb_read_valid() - Non-blocking read of a requested record or (if gone)
* the next available record.
*
* @rb: The ringbuffer to read from.
* @seq: The sequence number of the record to read.
* @r: A record data buffer to store the read record to.
*
* This is the public function available to readers to read a record.
*
* The reader provides the @info and @text_buf buffers of @r to be
* filled in. Any of the buffer pointers can be set to NULL if the reader
* is not interested in that data. To ensure proper initialization of @r,
* prb_rec_init_rd() should be used.
*
* Context: Any context.
* Return: true if a record was read, otherwise false.
*
* On success, the reader must check r->info.seq to see which record was
* actually read. This allows the reader to detect dropped records.
*
* Failure means @seq refers to a record not yet available to the reader.
*/
bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
struct printk_record *r)
{
return _prb_read_valid(rb, &seq, r, NULL);
}
/**
* prb_read_valid_info() - Non-blocking read of meta data for a requested
* record or (if gone) the next available record.
*
* @rb: The ringbuffer to read from.
* @seq: The sequence number of the record to read.
* @info: A buffer to store the read record meta data to.
* @line_count: A buffer to store the number of lines in the record text.
*
* This is the public function available to readers to read only the
* meta data of a record.
*
* The reader provides the @info, @line_count buffers to be filled in.
* Either of the buffer pointers can be set to NULL if the reader is not
* interested in that data.
*
* Context: Any context.
* Return: true if a record's meta data was read, otherwise false.
*
* On success, the reader must check info->seq to see which record meta data
* was actually read. This allows the reader to detect dropped records.
*
* Failure means @seq refers to a record not yet available to the reader.
*/
bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
struct printk_info *info, unsigned int *line_count)
{
struct printk_record r;
prb_rec_init_rd(&r, info, NULL, 0);
return _prb_read_valid(rb, &seq, &r, line_count);
}
/**
* prb_first_valid_seq() - Get the sequence number of the oldest available
* record.
*
* @rb: The ringbuffer to get the sequence number from.
*
* This is the public function available to readers to see what the
* first/oldest valid sequence number is.
*
* This provides readers a starting point to begin iterating the ringbuffer.
*
* Context: Any context.
* Return: The sequence number of the first/oldest record or, if the
* ringbuffer is empty, 0 is returned.
*/
u64 prb_first_valid_seq(struct printk_ringbuffer *rb)
{
u64 seq = 0;
if (!_prb_read_valid(rb, &seq, NULL, NULL))
return 0;
return seq;
}
/**
* prb_next_seq() - Get the sequence number after the last available record.
*
* @rb: The ringbuffer to get the sequence number from.
*
* This is the public function available to readers to see what the next
* newest sequence number available to readers will be.
*
* This provides readers a sequence number to jump to if all currently
* available records should be skipped. It is guaranteed that all records
* previous to the returned value have been finalized and are (or were)
* available to the reader.
*
* Context: Any context.
* Return: The sequence number of the next newest (not yet available) record
* for readers.
*/
u64 prb_next_seq(struct printk_ringbuffer *rb)
{
u64 seq;
seq = desc_last_finalized_seq(rb);
/*
* Begin searching after the last finalized record.
*
* On 0, the search must begin at 0 because of hack#2
* of the bootstrapping phase it is not known if a
* record at index 0 exists.
*/
if (seq != 0)
seq++;
/*
* The information about the last finalized @seq might be inaccurate.
* Search forward to find the current one.
*/
while (_prb_read_valid(rb, &seq, NULL, NULL))
seq++;
return seq;
}
/**
* prb_init() - Initialize a ringbuffer to use provided external buffers.
*
* @rb: The ringbuffer to initialize.
* @text_buf: The data buffer for text data.
* @textbits: The size of @text_buf as a power-of-2 value.
* @descs: The descriptor buffer for ringbuffer records.
* @descbits: The count of @descs items as a power-of-2 value.
* @infos: The printk_info buffer for ringbuffer records.
*
* This is the public function available to writers to setup a ringbuffer
* during runtime using provided buffers.
*
* This must match the initialization of DEFINE_PRINTKRB().
*
* Context: Any context.
*/
void prb_init(struct printk_ringbuffer *rb,
char *text_buf, unsigned int textbits,
struct prb_desc *descs, unsigned int descbits,
struct printk_info *infos)
{
memset(descs, 0, _DESCS_COUNT(descbits) * sizeof(descs[0]));
memset(infos, 0, _DESCS_COUNT(descbits) * sizeof(infos[0]));
rb->desc_ring.count_bits = descbits;
rb->desc_ring.descs = descs;
rb->desc_ring.infos = infos;
atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits));
atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits));
atomic_long_set(&rb->desc_ring.last_finalized_seq, 0);
rb->text_data_ring.size_bits = textbits;
rb->text_data_ring.data = text_buf;
atomic_long_set(&rb->text_data_ring.head_lpos, BLK0_LPOS(textbits));
atomic_long_set(&rb->text_data_ring.tail_lpos, BLK0_LPOS(textbits));
atomic_long_set(&rb->fail, 0);
atomic_long_set(&(descs[_DESCS_COUNT(descbits) - 1].state_var), DESC0_SV(descbits));
descs[_DESCS_COUNT(descbits) - 1].text_blk_lpos.begin = FAILED_LPOS;
descs[_DESCS_COUNT(descbits) - 1].text_blk_lpos.next = FAILED_LPOS;
infos[0].seq = -(u64)_DESCS_COUNT(descbits);
infos[_DESCS_COUNT(descbits) - 1].seq = 0;
}
/**
* prb_record_text_space() - Query the full actual used ringbuffer space for
* the text data of a reserved entry.
*
* @e: The successfully reserved entry to query.
*
* This is the public function available to writers to see how much actual
* space is used in the ringbuffer to store the text data of the specified
* entry.
*
* This function is only valid if @e has been successfully reserved using
* prb_reserve().
*
* Context: Any context.
* Return: The size in bytes used by the text data of the associated record.
*/
unsigned int prb_record_text_space(struct prb_reserved_entry *e)
{
return e->text_space;
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* fs/libfs.c
* Library for filesystems writers.
*/
#include <linux/blkdev.h>
#include <linux/export.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/cred.h>
#include <linux/mount.h>
#include <linux/vfs.h>
#include <linux/quotaops.h>
#include <linux/mutex.h>
#include <linux/namei.h>
#include <linux/exportfs.h>
#include <linux/iversion.h>
#include <linux/writeback.h>
#include <linux/buffer_head.h> /* sync_mapping_buffers */
#include <linux/fs_context.h>
#include <linux/pseudo_fs.h>
#include <linux/fsnotify.h>
#include <linux/unicode.h>
#include <linux/fscrypt.h>
#include <linux/pidfs.h>
#include <linux/uaccess.h>
#include "internal.h"
int simple_getattr(struct mnt_idmap *idmap, const struct path *path,
struct kstat *stat, u32 request_mask,
unsigned int query_flags)
{
struct inode *inode = d_inode(path->dentry);
generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);
stat->blocks = inode->i_mapping->nrpages << (PAGE_SHIFT - 9);
return 0;
}
EXPORT_SYMBOL(simple_getattr);
int simple_statfs(struct dentry *dentry, struct kstatfs *buf)
{
u64 id = huge_encode_dev(dentry->d_sb->s_dev);
buf->f_fsid = u64_to_fsid(id);
buf->f_type = dentry->d_sb->s_magic;
buf->f_bsize = PAGE_SIZE;
buf->f_namelen = NAME_MAX;
return 0;
}
EXPORT_SYMBOL(simple_statfs);
/*
* Retaining negative dentries for an in-memory filesystem just wastes
* memory and lookup time: arrange for them to be deleted immediately.
*/
int always_delete_dentry(const struct dentry *dentry)
{
return 1;
}
EXPORT_SYMBOL(always_delete_dentry);
const struct dentry_operations simple_dentry_operations = {
.d_delete = always_delete_dentry,
};
EXPORT_SYMBOL(simple_dentry_operations);
/*
* Lookup the data. This is trivial - if the dentry didn't already
* exist, we know it is negative. Set d_op to delete negative dentries.
*/
struct dentry *simple_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
{
if (dentry->d_name.len > NAME_MAX)
return ERR_PTR(-ENAMETOOLONG);
if (!dentry->d_sb->s_d_op) d_set_d_op(dentry, &simple_dentry_operations); d_add(dentry, NULL); return NULL;
}
EXPORT_SYMBOL(simple_lookup);
int dcache_dir_open(struct inode *inode, struct file *file)
{
file->private_data = d_alloc_cursor(file->f_path.dentry);
return file->private_data ? 0 : -ENOMEM;
}
EXPORT_SYMBOL(dcache_dir_open);
int dcache_dir_close(struct inode *inode, struct file *file)
{
dput(file->private_data);
return 0;
}
EXPORT_SYMBOL(dcache_dir_close);
/* parent is locked at least shared */
/*
* Returns an element of siblings' list.
* We are looking for <count>th positive after <p>; if
* found, dentry is grabbed and returned to caller.
* If no such element exists, NULL is returned.
*/
static struct dentry *scan_positives(struct dentry *cursor,
struct hlist_node **p,
loff_t count,
struct dentry *last)
{
struct dentry *dentry = cursor->d_parent, *found = NULL;
spin_lock(&dentry->d_lock);
while (*p) {
struct dentry *d = hlist_entry(*p, struct dentry, d_sib);
p = &d->d_sib.next;
// we must at least skip cursors, to avoid livelocks
if (d->d_flags & DCACHE_DENTRY_CURSOR)
continue;
if (simple_positive(d) && !--count) {
spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED);
if (simple_positive(d))
found = dget_dlock(d);
spin_unlock(&d->d_lock);
if (likely(found))
break;
count = 1;
}
if (need_resched()) {
if (!hlist_unhashed(&cursor->d_sib))
__hlist_del(&cursor->d_sib);
hlist_add_behind(&cursor->d_sib, &d->d_sib);
p = &cursor->d_sib.next;
spin_unlock(&dentry->d_lock);
cond_resched();
spin_lock(&dentry->d_lock);
}
}
spin_unlock(&dentry->d_lock);
dput(last);
return found;
}
loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence)
{
struct dentry *dentry = file->f_path.dentry;
switch (whence) {
case 1:
offset += file->f_pos;
fallthrough;
case 0:
if (offset >= 0)
break;
fallthrough;
default:
return -EINVAL;
}
if (offset != file->f_pos) {
struct dentry *cursor = file->private_data;
struct dentry *to = NULL;
inode_lock_shared(dentry->d_inode);
if (offset > 2)
to = scan_positives(cursor, &dentry->d_children.first,
offset - 2, NULL);
spin_lock(&dentry->d_lock);
hlist_del_init(&cursor->d_sib);
if (to)
hlist_add_behind(&cursor->d_sib, &to->d_sib);
spin_unlock(&dentry->d_lock);
dput(to);
file->f_pos = offset;
inode_unlock_shared(dentry->d_inode);
}
return offset;
}
EXPORT_SYMBOL(dcache_dir_lseek);
/*
* Directory is locked and all positive dentries in it are safe, since
* for ramfs-type trees they can't go away without unlink() or rmdir(),
* both impossible due to the lock on directory.
*/
int dcache_readdir(struct file *file, struct dir_context *ctx)
{
struct dentry *dentry = file->f_path.dentry;
struct dentry *cursor = file->private_data;
struct dentry *next = NULL;
struct hlist_node **p;
if (!dir_emit_dots(file, ctx))
return 0;
if (ctx->pos == 2)
p = &dentry->d_children.first;
else
p = &cursor->d_sib.next;
while ((next = scan_positives(cursor, p, 1, next)) != NULL) {
if (!dir_emit(ctx, next->d_name.name, next->d_name.len,
d_inode(next)->i_ino,
fs_umode_to_dtype(d_inode(next)->i_mode)))
break;
ctx->pos++;
p = &next->d_sib.next;
}
spin_lock(&dentry->d_lock);
hlist_del_init(&cursor->d_sib);
if (next)
hlist_add_before(&cursor->d_sib, &next->d_sib);
spin_unlock(&dentry->d_lock);
dput(next);
return 0;
}
EXPORT_SYMBOL(dcache_readdir);
ssize_t generic_read_dir(struct file *filp, char __user *buf, size_t siz, loff_t *ppos)
{
return -EISDIR;
}
EXPORT_SYMBOL(generic_read_dir);
const struct file_operations simple_dir_operations = {
.open = dcache_dir_open,
.release = dcache_dir_close,
.llseek = dcache_dir_lseek,
.read = generic_read_dir,
.iterate_shared = dcache_readdir,
.fsync = noop_fsync,
};
EXPORT_SYMBOL(simple_dir_operations);
const struct inode_operations simple_dir_inode_operations = {
.lookup = simple_lookup,
};
EXPORT_SYMBOL(simple_dir_inode_operations);
/* 0 is '.', 1 is '..', so always start with offset 2 or more */
enum {
DIR_OFFSET_MIN = 2,
};
static void offset_set(struct dentry *dentry, long offset)
{
dentry->d_fsdata = (void *)offset;
}
static long dentry2offset(struct dentry *dentry)
{
return (long)dentry->d_fsdata;
}
static struct lock_class_key simple_offset_lock_class;
/**
* simple_offset_init - initialize an offset_ctx
* @octx: directory offset map to be initialized
*
*/
void simple_offset_init(struct offset_ctx *octx)
{
mt_init_flags(&octx->mt, MT_FLAGS_ALLOC_RANGE);
lockdep_set_class(&octx->mt.ma_lock, &simple_offset_lock_class);
octx->next_offset = DIR_OFFSET_MIN;
}
/**
* simple_offset_add - Add an entry to a directory's offset map
* @octx: directory offset ctx to be updated
* @dentry: new dentry being added
*
* Returns zero on success. @octx and the dentry's offset are updated.
* Otherwise, a negative errno value is returned.
*/
int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry)
{
unsigned long offset;
int ret;
if (dentry2offset(dentry) != 0)
return -EBUSY;
ret = mtree_alloc_cyclic(&octx->mt, &offset, dentry, DIR_OFFSET_MIN,
LONG_MAX, &octx->next_offset, GFP_KERNEL);
if (ret < 0)
return ret;
offset_set(dentry, offset); return 0;
}
/**
* simple_offset_remove - Remove an entry to a directory's offset map
* @octx: directory offset ctx to be updated
* @dentry: dentry being removed
*
*/
void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry)
{
long offset;
offset = dentry2offset(dentry);
if (offset == 0)
return;
mtree_erase(&octx->mt, offset);
offset_set(dentry, 0);
}
/**
* simple_offset_empty - Check if a dentry can be unlinked
* @dentry: dentry to be tested
*
* Returns 0 if @dentry is a non-empty directory; otherwise returns 1.
*/
int simple_offset_empty(struct dentry *dentry)
{
struct inode *inode = d_inode(dentry);
struct offset_ctx *octx;
struct dentry *child;
unsigned long index;
int ret = 1;
if (!inode || !S_ISDIR(inode->i_mode))
return ret;
index = DIR_OFFSET_MIN;
octx = inode->i_op->get_offset_ctx(inode);
mt_for_each(&octx->mt, child, index, LONG_MAX) {
spin_lock(&child->d_lock);
if (simple_positive(child)) {
spin_unlock(&child->d_lock);
ret = 0;
break;
}
spin_unlock(&child->d_lock);
}
return ret;
}
/**
* simple_offset_rename_exchange - exchange rename with directory offsets
* @old_dir: parent of dentry being moved
* @old_dentry: dentry being moved
* @new_dir: destination parent
* @new_dentry: destination dentry
*
* Returns zero on success. Otherwise a negative errno is returned and the
* rename is rolled back.
*/
int simple_offset_rename_exchange(struct inode *old_dir,
struct dentry *old_dentry,
struct inode *new_dir,
struct dentry *new_dentry)
{
struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir);
struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir);
long old_index = dentry2offset(old_dentry);
long new_index = dentry2offset(new_dentry);
int ret;
simple_offset_remove(old_ctx, old_dentry);
simple_offset_remove(new_ctx, new_dentry);
ret = simple_offset_add(new_ctx, old_dentry);
if (ret)
goto out_restore;
ret = simple_offset_add(old_ctx, new_dentry);
if (ret) {
simple_offset_remove(new_ctx, old_dentry);
goto out_restore;
}
ret = simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
if (ret) {
simple_offset_remove(new_ctx, old_dentry);
simple_offset_remove(old_ctx, new_dentry);
goto out_restore;
}
return 0;
out_restore:
offset_set(old_dentry, old_index);
mtree_store(&old_ctx->mt, old_index, old_dentry, GFP_KERNEL);
offset_set(new_dentry, new_index);
mtree_store(&new_ctx->mt, new_index, new_dentry, GFP_KERNEL);
return ret;
}
/**
* simple_offset_destroy - Release offset map
* @octx: directory offset ctx that is about to be destroyed
*
* During fs teardown (eg. umount), a directory's offset map might still
* contain entries. xa_destroy() cleans out anything that remains.
*/
void simple_offset_destroy(struct offset_ctx *octx)
{
mtree_destroy(&octx->mt);
}
/**
* offset_dir_llseek - Advance the read position of a directory descriptor
* @file: an open directory whose position is to be updated
* @offset: a byte offset
* @whence: enumerator describing the starting position for this update
*
* SEEK_END, SEEK_DATA, and SEEK_HOLE are not supported for directories.
*
* Returns the updated read position if successful; otherwise a
* negative errno is returned and the read position remains unchanged.
*/
static loff_t offset_dir_llseek(struct file *file, loff_t offset, int whence)
{
switch (whence) {
case SEEK_CUR:
offset += file->f_pos;
fallthrough;
case SEEK_SET:
if (offset >= 0)
break;
fallthrough;
default:
return -EINVAL;
}
/* In this case, ->private_data is protected by f_pos_lock */
file->private_data = NULL;
return vfs_setpos(file, offset, LONG_MAX);
}
static struct dentry *offset_find_next(struct offset_ctx *octx, loff_t offset)
{
MA_STATE(mas, &octx->mt, offset, offset);
struct dentry *child, *found = NULL;
rcu_read_lock();
child = mas_find(&mas, LONG_MAX);
if (!child)
goto out;
spin_lock(&child->d_lock);
if (simple_positive(child))
found = dget_dlock(child);
spin_unlock(&child->d_lock);
out:
rcu_read_unlock();
return found;
}
static bool offset_dir_emit(struct dir_context *ctx, struct dentry *dentry)
{
struct inode *inode = d_inode(dentry);
long offset = dentry2offset(dentry);
return ctx->actor(ctx, dentry->d_name.name, dentry->d_name.len, offset,
inode->i_ino, fs_umode_to_dtype(inode->i_mode));
}
static void *offset_iterate_dir(struct inode *inode, struct dir_context *ctx)
{
struct offset_ctx *octx = inode->i_op->get_offset_ctx(inode);
struct dentry *dentry;
while (true) {
dentry = offset_find_next(octx, ctx->pos);
if (!dentry)
return ERR_PTR(-ENOENT);
if (!offset_dir_emit(ctx, dentry)) {
dput(dentry);
break;
}
ctx->pos = dentry2offset(dentry) + 1;
dput(dentry);
}
return NULL;
}
/**
* offset_readdir - Emit entries starting at offset @ctx->pos
* @file: an open directory to iterate over
* @ctx: directory iteration context
*
* Caller must hold @file's i_rwsem to prevent insertion or removal of
* entries during this call.
*
* On entry, @ctx->pos contains an offset that represents the first entry
* to be read from the directory.
*
* The operation continues until there are no more entries to read, or
* until the ctx->actor indicates there is no more space in the caller's
* output buffer.
*
* On return, @ctx->pos contains an offset that will read the next entry
* in this directory when offset_readdir() is called again with @ctx.
*
* Return values:
* %0 - Complete
*/
static int offset_readdir(struct file *file, struct dir_context *ctx)
{
struct dentry *dir = file->f_path.dentry;
lockdep_assert_held(&d_inode(dir)->i_rwsem);
if (!dir_emit_dots(file, ctx))
return 0;
/* In this case, ->private_data is protected by f_pos_lock */
if (ctx->pos == DIR_OFFSET_MIN)
file->private_data = NULL;
else if (file->private_data == ERR_PTR(-ENOENT))
return 0;
file->private_data = offset_iterate_dir(d_inode(dir), ctx);
return 0;
}
const struct file_operations simple_offset_dir_operations = {
.llseek = offset_dir_llseek,
.iterate_shared = offset_readdir,
.read = generic_read_dir,
.fsync = noop_fsync,
};
static struct dentry *find_next_child(struct dentry *parent, struct dentry *prev)
{
struct dentry *child = NULL, *d;
spin_lock(&parent->d_lock); d = prev ? d_next_sibling(prev) : d_first_child(parent); hlist_for_each_entry_from(d, d_sib) { if (simple_positive(d)) { spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED); if (simple_positive(d)) child = dget_dlock(d); spin_unlock(&d->d_lock);
if (likely(child))
break;
}
}
spin_unlock(&parent->d_lock);
dput(prev);
return child;
}
void simple_recursive_removal(struct dentry *dentry,
void (*callback)(struct dentry *))
{
struct dentry *this = dget(dentry);
while (true) {
struct dentry *victim = NULL, *child;
struct inode *inode = this->d_inode;
inode_lock(inode);
if (d_is_dir(this))
inode->i_flags |= S_DEAD; while ((child = find_next_child(this, victim)) == NULL) {
// kill and ascend
// update metadata while it's still locked
inode_set_ctime_current(inode);
clear_nlink(inode);
inode_unlock(inode);
victim = this;
this = this->d_parent;
inode = this->d_inode;
inode_lock(inode);
if (simple_positive(victim)) { d_invalidate(victim); // avoid lost mounts if (d_is_dir(victim)) fsnotify_rmdir(inode, victim);
else
fsnotify_unlink(inode, victim); if (callback) callback(victim); dput(victim); // unpin it
}
if (victim == dentry) { inode_set_mtime_to_ts(inode,
inode_set_ctime_current(inode));
if (d_is_dir(dentry))
drop_nlink(inode); inode_unlock(inode);
dput(dentry);
return;
}
}
inode_unlock(inode);
this = child;
}
}
EXPORT_SYMBOL(simple_recursive_removal);
static const struct super_operations simple_super_operations = {
.statfs = simple_statfs,
};
static int pseudo_fs_fill_super(struct super_block *s, struct fs_context *fc)
{
struct pseudo_fs_context *ctx = fc->fs_private;
struct inode *root;
s->s_maxbytes = MAX_LFS_FILESIZE;
s->s_blocksize = PAGE_SIZE;
s->s_blocksize_bits = PAGE_SHIFT;
s->s_magic = ctx->magic;
s->s_op = ctx->ops ?: &simple_super_operations;
s->s_xattr = ctx->xattr;
s->s_time_gran = 1;
root = new_inode(s);
if (!root)
return -ENOMEM;
/*
* since this is the first inode, make it number 1. New inodes created
* after this must take care not to collide with it (by passing
* max_reserved of 1 to iunique).
*/
root->i_ino = 1;
root->i_mode = S_IFDIR | S_IRUSR | S_IWUSR;
simple_inode_init_ts(root);
s->s_root = d_make_root(root);
if (!s->s_root)
return -ENOMEM;
s->s_d_op = ctx->dops;
return 0;
}
static int pseudo_fs_get_tree(struct fs_context *fc)
{
return get_tree_nodev(fc, pseudo_fs_fill_super);
}
static void pseudo_fs_free(struct fs_context *fc)
{
kfree(fc->fs_private);
}
static const struct fs_context_operations pseudo_fs_context_ops = {
.free = pseudo_fs_free,
.get_tree = pseudo_fs_get_tree,
};
/*
* Common helper for pseudo-filesystems (sockfs, pipefs, bdev - stuff that
* will never be mountable)
*/
struct pseudo_fs_context *init_pseudo(struct fs_context *fc,
unsigned long magic)
{
struct pseudo_fs_context *ctx;
ctx = kzalloc(sizeof(struct pseudo_fs_context), GFP_KERNEL);
if (likely(ctx)) {
ctx->magic = magic;
fc->fs_private = ctx;
fc->ops = &pseudo_fs_context_ops;
fc->sb_flags |= SB_NOUSER;
fc->global = true;
}
return ctx;
}
EXPORT_SYMBOL(init_pseudo);
int simple_open(struct inode *inode, struct file *file)
{
if (inode->i_private)
file->private_data = inode->i_private;
return 0;
}
EXPORT_SYMBOL(simple_open);
int simple_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
{
struct inode *inode = d_inode(old_dentry);
inode_set_mtime_to_ts(dir,
inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
inc_nlink(inode);
ihold(inode);
dget(dentry);
d_instantiate(dentry, inode);
return 0;
}
EXPORT_SYMBOL(simple_link);
int simple_empty(struct dentry *dentry)
{
struct dentry *child;
int ret = 0;
spin_lock(&dentry->d_lock);
hlist_for_each_entry(child, &dentry->d_children, d_sib) {
spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED);
if (simple_positive(child)) {
spin_unlock(&child->d_lock);
goto out;
}
spin_unlock(&child->d_lock);
}
ret = 1;
out:
spin_unlock(&dentry->d_lock);
return ret;
}
EXPORT_SYMBOL(simple_empty);
int simple_unlink(struct inode *dir, struct dentry *dentry)
{
struct inode *inode = d_inode(dentry);
inode_set_mtime_to_ts(dir,
inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)));
drop_nlink(inode);
dput(dentry);
return 0;
}
EXPORT_SYMBOL(simple_unlink);
int simple_rmdir(struct inode *dir, struct dentry *dentry)
{
if (!simple_empty(dentry))
return -ENOTEMPTY;
drop_nlink(d_inode(dentry));
simple_unlink(dir, dentry);
drop_nlink(dir);
return 0;
}
EXPORT_SYMBOL(simple_rmdir);
/**
* simple_rename_timestamp - update the various inode timestamps for rename
* @old_dir: old parent directory
* @old_dentry: dentry that is being renamed
* @new_dir: new parent directory
* @new_dentry: target for rename
*
* POSIX mandates that the old and new parent directories have their ctime and
* mtime updated, and that inodes of @old_dentry and @new_dentry (if any), have
* their ctime updated.
*/
void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry)
{
struct inode *newino = d_inode(new_dentry);
inode_set_mtime_to_ts(old_dir, inode_set_ctime_current(old_dir));
if (new_dir != old_dir)
inode_set_mtime_to_ts(new_dir,
inode_set_ctime_current(new_dir));
inode_set_ctime_current(d_inode(old_dentry));
if (newino)
inode_set_ctime_current(newino);
}
EXPORT_SYMBOL_GPL(simple_rename_timestamp);
int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
struct inode *new_dir, struct dentry *new_dentry)
{
bool old_is_dir = d_is_dir(old_dentry);
bool new_is_dir = d_is_dir(new_dentry);
if (old_dir != new_dir && old_is_dir != new_is_dir) {
if (old_is_dir) {
drop_nlink(old_dir);
inc_nlink(new_dir);
} else {
drop_nlink(new_dir);
inc_nlink(old_dir);
}
}
simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
return 0;
}
EXPORT_SYMBOL_GPL(simple_rename_exchange);
int simple_rename(struct mnt_idmap *idmap, struct inode *old_dir,
struct dentry *old_dentry, struct inode *new_dir,
struct dentry *new_dentry, unsigned int flags)
{
int they_are_dirs = d_is_dir(old_dentry);
if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE))
return -EINVAL;
if (flags & RENAME_EXCHANGE)
return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
if (!simple_empty(new_dentry))
return -ENOTEMPTY;
if (d_really_is_positive(new_dentry)) {
simple_unlink(new_dir, new_dentry);
if (they_are_dirs) {
drop_nlink(d_inode(new_dentry));
drop_nlink(old_dir);
}
} else if (they_are_dirs) {
drop_nlink(old_dir);
inc_nlink(new_dir);
}
simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
return 0;
}
EXPORT_SYMBOL(simple_rename);
/**
* simple_setattr - setattr for simple filesystem
* @idmap: idmap of the target mount
* @dentry: dentry
* @iattr: iattr structure
*
* Returns 0 on success, -error on failure.
*
* simple_setattr is a simple ->setattr implementation without a proper
* implementation of size changes.
*
* It can either be used for in-memory filesystems or special files
* on simple regular filesystems. Anything that needs to change on-disk
* or wire state on size changes needs its own setattr method.
*/
int simple_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
struct iattr *iattr)
{
struct inode *inode = d_inode(dentry);
int error;
error = setattr_prepare(idmap, dentry, iattr);
if (error)
return error;
if (iattr->ia_valid & ATTR_SIZE)
truncate_setsize(inode, iattr->ia_size);
setattr_copy(idmap, inode, iattr);
mark_inode_dirty(inode);
return 0;
}
EXPORT_SYMBOL(simple_setattr);
static int simple_read_folio(struct file *file, struct folio *folio)
{
folio_zero_range(folio, 0, folio_size(folio));
flush_dcache_folio(folio);
folio_mark_uptodate(folio);
folio_unlock(folio);
return 0;
}
int simple_write_begin(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len,
struct page **pagep, void **fsdata)
{
struct folio *folio;
folio = __filemap_get_folio(mapping, pos / PAGE_SIZE, FGP_WRITEBEGIN,
mapping_gfp_mask(mapping));
if (IS_ERR(folio))
return PTR_ERR(folio);
*pagep = &folio->page;
if (!folio_test_uptodate(folio) && (len != folio_size(folio))) {
size_t from = offset_in_folio(folio, pos);
folio_zero_segments(folio, 0, from,
from + len, folio_size(folio));
}
return 0;
}
EXPORT_SYMBOL(simple_write_begin);
/**
* simple_write_end - .write_end helper for non-block-device FSes
* @file: See .write_end of address_space_operations
* @mapping: "
* @pos: "
* @len: "
* @copied: "
* @page: "
* @fsdata: "
*
* simple_write_end does the minimum needed for updating a page after writing is
* done. It has the same API signature as the .write_end of
* address_space_operations vector. So it can just be set onto .write_end for
* FSes that don't need any other processing. i_mutex is assumed to be held.
* Block based filesystems should use generic_write_end().
* NOTE: Even though i_size might get updated by this function, mark_inode_dirty
* is not called, so a filesystem that actually does store data in .write_inode
* should extend on what's done here with a call to mark_inode_dirty() in the
* case that i_size has changed.
*
* Use *ONLY* with simple_read_folio()
*/
static int simple_write_end(struct file *file, struct address_space *mapping,
loff_t pos, unsigned len, unsigned copied,
struct page *page, void *fsdata)
{
struct folio *folio = page_folio(page);
struct inode *inode = folio->mapping->host;
loff_t last_pos = pos + copied;
/* zero the stale part of the folio if we did a short copy */
if (!folio_test_uptodate(folio)) {
if (copied < len) {
size_t from = offset_in_folio(folio, pos);
folio_zero_range(folio, from + copied, len - copied);
}
folio_mark_uptodate(folio);
}
/*
* No need to use i_size_read() here, the i_size
* cannot change under us because we hold the i_mutex.
*/
if (last_pos > inode->i_size)
i_size_write(inode, last_pos);
folio_mark_dirty(folio);
folio_unlock(folio);
folio_put(folio);
return copied;
}
/*
* Provides ramfs-style behavior: data in the pagecache, but no writeback.
*/
const struct address_space_operations ram_aops = {
.read_folio = simple_read_folio,
.write_begin = simple_write_begin,
.write_end = simple_write_end,
.dirty_folio = noop_dirty_folio,
};
EXPORT_SYMBOL(ram_aops);
/*
* the inodes created here are not hashed. If you use iunique to generate
* unique inode values later for this filesystem, then you must take care
* to pass it an appropriate max_reserved value to avoid collisions.
*/
int simple_fill_super(struct super_block *s, unsigned long magic,
const struct tree_descr *files)
{
struct inode *inode;
struct dentry *dentry;
int i;
s->s_blocksize = PAGE_SIZE;
s->s_blocksize_bits = PAGE_SHIFT;
s->s_magic = magic;
s->s_op = &simple_super_operations;
s->s_time_gran = 1;
inode = new_inode(s);
if (!inode)
return -ENOMEM;
/*
* because the root inode is 1, the files array must not contain an
* entry at index 1
*/
inode->i_ino = 1;
inode->i_mode = S_IFDIR | 0755;
simple_inode_init_ts(inode);
inode->i_op = &simple_dir_inode_operations;
inode->i_fop = &simple_dir_operations;
set_nlink(inode, 2);
s->s_root = d_make_root(inode);
if (!s->s_root)
return -ENOMEM;
for (i = 0; !files->name || files->name[0]; i++, files++) {
if (!files->name)
continue;
/* warn if it tries to conflict with the root inode */
if (unlikely(i == 1))
printk(KERN_WARNING "%s: %s passed in a files array"
"with an index of 1!\n", __func__,
s->s_type->name);
dentry = d_alloc_name(s->s_root, files->name);
if (!dentry)
return -ENOMEM;
inode = new_inode(s);
if (!inode) {
dput(dentry);
return -ENOMEM;
}
inode->i_mode = S_IFREG | files->mode;
simple_inode_init_ts(inode);
inode->i_fop = files->ops;
inode->i_ino = i;
d_add(dentry, inode);
}
return 0;
}
EXPORT_SYMBOL(simple_fill_super);
static DEFINE_SPINLOCK(pin_fs_lock);
int simple_pin_fs(struct file_system_type *type, struct vfsmount **mount, int *count)
{
struct vfsmount *mnt = NULL;
spin_lock(&pin_fs_lock);
if (unlikely(!*mount)) {
spin_unlock(&pin_fs_lock);
mnt = vfs_kern_mount(type, SB_KERNMOUNT, type->name, NULL);
if (IS_ERR(mnt))
return PTR_ERR(mnt); spin_lock(&pin_fs_lock);
if (!*mount)
*mount = mnt;
}
mntget(*mount);
++*count;
spin_unlock(&pin_fs_lock);
mntput(mnt);
return 0;
}
EXPORT_SYMBOL(simple_pin_fs);
void simple_release_fs(struct vfsmount **mount, int *count)
{
struct vfsmount *mnt;
spin_lock(&pin_fs_lock);
mnt = *mount;
if (!--*count)
*mount = NULL; spin_unlock(&pin_fs_lock);
mntput(mnt);
}
EXPORT_SYMBOL(simple_release_fs);
/**
* simple_read_from_buffer - copy data from the buffer to user space
* @to: the user space buffer to read to
* @count: the maximum number of bytes to read
* @ppos: the current position in the buffer
* @from: the buffer to read from
* @available: the size of the buffer
*
* The simple_read_from_buffer() function reads up to @count bytes from the
* buffer @from at offset @ppos into the user space address starting at @to.
*
* On success, the number of bytes read is returned and the offset @ppos is
* advanced by this number, or negative value is returned on error.
**/
ssize_t simple_read_from_buffer(void __user *to, size_t count, loff_t *ppos,
const void *from, size_t available)
{
loff_t pos = *ppos;
size_t ret;
if (pos < 0)
return -EINVAL;
if (pos >= available || !count) return 0; if (count > available - pos)
count = available - pos;
ret = copy_to_user(to, from + pos, count);
if (ret == count)
return -EFAULT;
count -= ret;
*ppos = pos + count;
return count;
}
EXPORT_SYMBOL(simple_read_from_buffer);
/**
* simple_write_to_buffer - copy data from user space to the buffer
* @to: the buffer to write to
* @available: the size of the buffer
* @ppos: the current position in the buffer
* @from: the user space buffer to read from
* @count: the maximum number of bytes to read
*
* The simple_write_to_buffer() function reads up to @count bytes from the user
* space address starting at @from into the buffer @to at offset @ppos.
*
* On success, the number of bytes written is returned and the offset @ppos is
* advanced by this number, or negative value is returned on error.
**/
ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos,
const void __user *from, size_t count)
{
loff_t pos = *ppos;
size_t res;
if (pos < 0)
return -EINVAL;
if (pos >= available || !count)
return 0;
if (count > available - pos)
count = available - pos;
res = copy_from_user(to + pos, from, count);
if (res == count)
return -EFAULT;
count -= res;
*ppos = pos + count;
return count;
}
EXPORT_SYMBOL(simple_write_to_buffer);
/**
* memory_read_from_buffer - copy data from the buffer
* @to: the kernel space buffer to read to
* @count: the maximum number of bytes to read
* @ppos: the current position in the buffer
* @from: the buffer to read from
* @available: the size of the buffer
*
* The memory_read_from_buffer() function reads up to @count bytes from the
* buffer @from at offset @ppos into the kernel space address starting at @to.
*
* On success, the number of bytes read is returned and the offset @ppos is
* advanced by this number, or negative value is returned on error.
**/
ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos,
const void *from, size_t available)
{
loff_t pos = *ppos;
if (pos < 0)
return -EINVAL;
if (pos >= available)
return 0;
if (count > available - pos)
count = available - pos;
memcpy(to, from + pos, count);
*ppos = pos + count;
return count;
}
EXPORT_SYMBOL(memory_read_from_buffer);
/*
* Transaction based IO.
* The file expects a single write which triggers the transaction, and then
* possibly a read which collects the result - which is stored in a
* file-local buffer.
*/
void simple_transaction_set(struct file *file, size_t n)
{
struct simple_transaction_argresp *ar = file->private_data;
BUG_ON(n > SIMPLE_TRANSACTION_LIMIT);
/*
* The barrier ensures that ar->size will really remain zero until
* ar->data is ready for reading.
*/
smp_mb();
ar->size = n;
}
EXPORT_SYMBOL(simple_transaction_set);
char *simple_transaction_get(struct file *file, const char __user *buf, size_t size)
{
struct simple_transaction_argresp *ar;
static DEFINE_SPINLOCK(simple_transaction_lock);
if (size > SIMPLE_TRANSACTION_LIMIT - 1)
return ERR_PTR(-EFBIG);
ar = (struct simple_transaction_argresp *)get_zeroed_page(GFP_KERNEL);
if (!ar)
return ERR_PTR(-ENOMEM);
spin_lock(&simple_transaction_lock);
/* only one write allowed per open */
if (file->private_data) {
spin_unlock(&simple_transaction_lock);
free_page((unsigned long)ar);
return ERR_PTR(-EBUSY);
}
file->private_data = ar;
spin_unlock(&simple_transaction_lock);
if (copy_from_user(ar->data, buf, size))
return ERR_PTR(-EFAULT);
return ar->data;
}
EXPORT_SYMBOL(simple_transaction_get);
ssize_t simple_transaction_read(struct file *file, char __user *buf, size_t size, loff_t *pos)
{
struct simple_transaction_argresp *ar = file->private_data;
if (!ar)
return 0;
return simple_read_from_buffer(buf, size, pos, ar->data, ar->size);
}
EXPORT_SYMBOL(simple_transaction_read);
int simple_transaction_release(struct inode *inode, struct file *file)
{
free_page((unsigned long)file->private_data);
return 0;
}
EXPORT_SYMBOL(simple_transaction_release);
/* Simple attribute files */
struct simple_attr {
int (*get)(void *, u64 *);
int (*set)(void *, u64);
char get_buf[24]; /* enough to store a u64 and "\n\0" */
char set_buf[24];
void *data;
const char *fmt; /* format for read operation */
struct mutex mutex; /* protects access to these buffers */
};
/* simple_attr_open is called by an actual attribute open file operation
* to set the attribute specific access operations. */
int simple_attr_open(struct inode *inode, struct file *file,
int (*get)(void *, u64 *), int (*set)(void *, u64),
const char *fmt)
{
struct simple_attr *attr;
attr = kzalloc(sizeof(*attr), GFP_KERNEL);
if (!attr)
return -ENOMEM;
attr->get = get;
attr->set = set;
attr->data = inode->i_private;
attr->fmt = fmt;
mutex_init(&attr->mutex);
file->private_data = attr;
return nonseekable_open(inode, file);
}
EXPORT_SYMBOL_GPL(simple_attr_open);
int simple_attr_release(struct inode *inode, struct file *file)
{
kfree(file->private_data);
return 0;
}
EXPORT_SYMBOL_GPL(simple_attr_release); /* GPL-only? This? Really? */
/* read from the buffer that is filled with the get function */
ssize_t simple_attr_read(struct file *file, char __user *buf,
size_t len, loff_t *ppos)
{
struct simple_attr *attr;
size_t size;
ssize_t ret;
attr = file->private_data;
if (!attr->get)
return -EACCES;
ret = mutex_lock_interruptible(&attr->mutex);
if (ret)
return ret;
if (*ppos && attr->get_buf[0]) {
/* continued read */
size = strlen(attr->get_buf);
} else {
/* first read */
u64 val;
ret = attr->get(attr->data, &val);
if (ret)
goto out;
size = scnprintf(attr->get_buf, sizeof(attr->get_buf),
attr->fmt, (unsigned long long)val);
}
ret = simple_read_from_buffer(buf, len, ppos, attr->get_buf, size);
out:
mutex_unlock(&attr->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(simple_attr_read);
/* interpret the buffer as a number to call the set function with */
static ssize_t simple_attr_write_xsigned(struct file *file, const char __user *buf,
size_t len, loff_t *ppos, bool is_signed)
{
struct simple_attr *attr;
unsigned long long val;
size_t size;
ssize_t ret;
attr = file->private_data;
if (!attr->set)
return -EACCES;
ret = mutex_lock_interruptible(&attr->mutex);
if (ret)
return ret;
ret = -EFAULT;
size = min(sizeof(attr->set_buf) - 1, len);
if (copy_from_user(attr->set_buf, buf, size))
goto out;
attr->set_buf[size] = '\0';
if (is_signed)
ret = kstrtoll(attr->set_buf, 0, &val);
else
ret = kstrtoull(attr->set_buf, 0, &val);
if (ret)
goto out;
ret = attr->set(attr->data, val);
if (ret == 0)
ret = len; /* on success, claim we got the whole input */
out:
mutex_unlock(&attr->mutex);
return ret;
}
ssize_t simple_attr_write(struct file *file, const char __user *buf,
size_t len, loff_t *ppos)
{
return simple_attr_write_xsigned(file, buf, len, ppos, false);
}
EXPORT_SYMBOL_GPL(simple_attr_write);
ssize_t simple_attr_write_signed(struct file *file, const char __user *buf,
size_t len, loff_t *ppos)
{
return simple_attr_write_xsigned(file, buf, len, ppos, true);
}
EXPORT_SYMBOL_GPL(simple_attr_write_signed);
/**
* generic_encode_ino32_fh - generic export_operations->encode_fh function
* @inode: the object to encode
* @fh: where to store the file handle fragment
* @max_len: maximum length to store there (in 4 byte units)
* @parent: parent directory inode, if wanted
*
* This generic encode_fh function assumes that the 32 inode number
* is suitable for locating an inode, and that the generation number
* can be used to check that it is still valid. It places them in the
* filehandle fragment where export_decode_fh expects to find them.
*/
int generic_encode_ino32_fh(struct inode *inode, __u32 *fh, int *max_len,
struct inode *parent)
{
struct fid *fid = (void *)fh;
int len = *max_len;
int type = FILEID_INO32_GEN;
if (parent && (len < 4)) {
*max_len = 4;
return FILEID_INVALID;
} else if (len < 2) {
*max_len = 2;
return FILEID_INVALID;
}
len = 2;
fid->i32.ino = inode->i_ino;
fid->i32.gen = inode->i_generation;
if (parent) {
fid->i32.parent_ino = parent->i_ino;
fid->i32.parent_gen = parent->i_generation;
len = 4;
type = FILEID_INO32_GEN_PARENT;
}
*max_len = len;
return type;
}
EXPORT_SYMBOL_GPL(generic_encode_ino32_fh);
/**
* generic_fh_to_dentry - generic helper for the fh_to_dentry export operation
* @sb: filesystem to do the file handle conversion on
* @fid: file handle to convert
* @fh_len: length of the file handle in bytes
* @fh_type: type of file handle
* @get_inode: filesystem callback to retrieve inode
*
* This function decodes @fid as long as it has one of the well-known
* Linux filehandle types and calls @get_inode on it to retrieve the
* inode for the object specified in the file handle.
*/
struct dentry *generic_fh_to_dentry(struct super_block *sb, struct fid *fid,
int fh_len, int fh_type, struct inode *(*get_inode)
(struct super_block *sb, u64 ino, u32 gen))
{
struct inode *inode = NULL;
if (fh_len < 2)
return NULL;
switch (fh_type) {
case FILEID_INO32_GEN:
case FILEID_INO32_GEN_PARENT:
inode = get_inode(sb, fid->i32.ino, fid->i32.gen);
break;
}
return d_obtain_alias(inode);
}
EXPORT_SYMBOL_GPL(generic_fh_to_dentry);
/**
* generic_fh_to_parent - generic helper for the fh_to_parent export operation
* @sb: filesystem to do the file handle conversion on
* @fid: file handle to convert
* @fh_len: length of the file handle in bytes
* @fh_type: type of file handle
* @get_inode: filesystem callback to retrieve inode
*
* This function decodes @fid as long as it has one of the well-known
* Linux filehandle types and calls @get_inode on it to retrieve the
* inode for the _parent_ object specified in the file handle if it
* is specified in the file handle, or NULL otherwise.
*/
struct dentry *generic_fh_to_parent(struct super_block *sb, struct fid *fid,
int fh_len, int fh_type, struct inode *(*get_inode)
(struct super_block *sb, u64 ino, u32 gen))
{
struct inode *inode = NULL;
if (fh_len <= 2)
return NULL;
switch (fh_type) {
case FILEID_INO32_GEN_PARENT:
inode = get_inode(sb, fid->i32.parent_ino,
(fh_len > 3 ? fid->i32.parent_gen : 0));
break;
}
return d_obtain_alias(inode);
}
EXPORT_SYMBOL_GPL(generic_fh_to_parent);
/**
* __generic_file_fsync - generic fsync implementation for simple filesystems
*
* @file: file to synchronize
* @start: start offset in bytes
* @end: end offset in bytes (inclusive)
* @datasync: only synchronize essential metadata if true
*
* This is a generic implementation of the fsync method for simple
* filesystems which track all non-inode metadata in the buffers list
* hanging off the address_space structure.
*/
int __generic_file_fsync(struct file *file, loff_t start, loff_t end,
int datasync)
{
struct inode *inode = file->f_mapping->host;
int err;
int ret;
err = file_write_and_wait_range(file, start, end);
if (err)
return err;
inode_lock(inode);
ret = sync_mapping_buffers(inode->i_mapping);
if (!(inode->i_state & I_DIRTY_ALL))
goto out;
if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
goto out;
err = sync_inode_metadata(inode, 1);
if (ret == 0)
ret = err;
out:
inode_unlock(inode);
/* check and advance again to catch errors after syncing out buffers */
err = file_check_and_advance_wb_err(file);
if (ret == 0)
ret = err;
return ret;
}
EXPORT_SYMBOL(__generic_file_fsync);
/**
* generic_file_fsync - generic fsync implementation for simple filesystems
* with flush
* @file: file to synchronize
* @start: start offset in bytes
* @end: end offset in bytes (inclusive)
* @datasync: only synchronize essential metadata if true
*
*/
int generic_file_fsync(struct file *file, loff_t start, loff_t end,
int datasync)
{
struct inode *inode = file->f_mapping->host;
int err;
err = __generic_file_fsync(file, start, end, datasync);
if (err)
return err;
return blkdev_issue_flush(inode->i_sb->s_bdev);
}
EXPORT_SYMBOL(generic_file_fsync);
/**
* generic_check_addressable - Check addressability of file system
* @blocksize_bits: log of file system block size
* @num_blocks: number of blocks in file system
*
* Determine whether a file system with @num_blocks blocks (and a
* block size of 2**@blocksize_bits) is addressable by the sector_t
* and page cache of the system. Return 0 if so and -EFBIG otherwise.
*/
int generic_check_addressable(unsigned blocksize_bits, u64 num_blocks)
{
u64 last_fs_block = num_blocks - 1;
u64 last_fs_page =
last_fs_block >> (PAGE_SHIFT - blocksize_bits);
if (unlikely(num_blocks == 0))
return 0;
if ((blocksize_bits < 9) || (blocksize_bits > PAGE_SHIFT))
return -EINVAL;
if ((last_fs_block > (sector_t)(~0ULL) >> (blocksize_bits - 9)) ||
(last_fs_page > (pgoff_t)(~0ULL))) {
return -EFBIG;
}
return 0;
}
EXPORT_SYMBOL(generic_check_addressable);
/*
* No-op implementation of ->fsync for in-memory filesystems.
*/
int noop_fsync(struct file *file, loff_t start, loff_t end, int datasync)
{
return 0;
}
EXPORT_SYMBOL(noop_fsync);
ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
{
/*
* iomap based filesystems support direct I/O without need for
* this callback. However, it still needs to be set in
* inode->a_ops so that open/fcntl know that direct I/O is
* generally supported.
*/
return -EINVAL;
}
EXPORT_SYMBOL_GPL(noop_direct_IO);
/* Because kfree isn't assignment-compatible with void(void*) ;-/ */
void kfree_link(void *p)
{
kfree(p);
}
EXPORT_SYMBOL(kfree_link);
struct inode *alloc_anon_inode(struct super_block *s)
{
static const struct address_space_operations anon_aops = {
.dirty_folio = noop_dirty_folio,
};
struct inode *inode = new_inode_pseudo(s);
if (!inode)
return ERR_PTR(-ENOMEM);
inode->i_ino = get_next_ino();
inode->i_mapping->a_ops = &anon_aops;
/*
* Mark the inode dirty from the very beginning,
* that way it will never be moved to the dirty
* list because mark_inode_dirty() will think
* that it already _is_ on the dirty list.
*/
inode->i_state = I_DIRTY;
inode->i_mode = S_IRUSR | S_IWUSR;
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
inode->i_flags |= S_PRIVATE;
simple_inode_init_ts(inode);
return inode;
}
EXPORT_SYMBOL(alloc_anon_inode);
/**
* simple_nosetlease - generic helper for prohibiting leases
* @filp: file pointer
* @arg: type of lease to obtain
* @flp: new lease supplied for insertion
* @priv: private data for lm_setup operation
*
* Generic helper for filesystems that do not wish to allow leases to be set.
* All arguments are ignored and it just returns -EINVAL.
*/
int
simple_nosetlease(struct file *filp, int arg, struct file_lease **flp,
void **priv)
{
return -EINVAL;
}
EXPORT_SYMBOL(simple_nosetlease);
/**
* simple_get_link - generic helper to get the target of "fast" symlinks
* @dentry: not used here
* @inode: the symlink inode
* @done: not used here
*
* Generic helper for filesystems to use for symlink inodes where a pointer to
* the symlink target is stored in ->i_link. NOTE: this isn't normally called,
* since as an optimization the path lookup code uses any non-NULL ->i_link
* directly, without calling ->get_link(). But ->get_link() still must be set,
* to mark the inode_operations as being for a symlink.
*
* Return: the symlink target
*/
const char *simple_get_link(struct dentry *dentry, struct inode *inode,
struct delayed_call *done)
{
return inode->i_link;
}
EXPORT_SYMBOL(simple_get_link);
const struct inode_operations simple_symlink_inode_operations = {
.get_link = simple_get_link,
};
EXPORT_SYMBOL(simple_symlink_inode_operations);
/*
* Operations for a permanently empty directory.
*/
static struct dentry *empty_dir_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
{
return ERR_PTR(-ENOENT);
}
static int empty_dir_getattr(struct mnt_idmap *idmap,
const struct path *path, struct kstat *stat,
u32 request_mask, unsigned int query_flags)
{
struct inode *inode = d_inode(path->dentry);
generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);
return 0;
}
static int empty_dir_setattr(struct mnt_idmap *idmap,
struct dentry *dentry, struct iattr *attr)
{
return -EPERM;
}
static ssize_t empty_dir_listxattr(struct dentry *dentry, char *list, size_t size)
{
return -EOPNOTSUPP;
}
static const struct inode_operations empty_dir_inode_operations = {
.lookup = empty_dir_lookup,
.permission = generic_permission,
.setattr = empty_dir_setattr,
.getattr = empty_dir_getattr,
.listxattr = empty_dir_listxattr,
};
static loff_t empty_dir_llseek(struct file *file, loff_t offset, int whence)
{
/* An empty directory has two entries . and .. at offsets 0 and 1 */
return generic_file_llseek_size(file, offset, whence, 2, 2);
}
static int empty_dir_readdir(struct file *file, struct dir_context *ctx)
{
dir_emit_dots(file, ctx);
return 0;
}
static const struct file_operations empty_dir_operations = {
.llseek = empty_dir_llseek,
.read = generic_read_dir,
.iterate_shared = empty_dir_readdir,
.fsync = noop_fsync,
};
void make_empty_dir_inode(struct inode *inode)
{
set_nlink(inode, 2);
inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO;
inode->i_uid = GLOBAL_ROOT_UID;
inode->i_gid = GLOBAL_ROOT_GID;
inode->i_rdev = 0;
inode->i_size = 0;
inode->i_blkbits = PAGE_SHIFT;
inode->i_blocks = 0;
inode->i_op = &empty_dir_inode_operations;
inode->i_opflags &= ~IOP_XATTR;
inode->i_fop = &empty_dir_operations;
}
bool is_empty_dir_inode(struct inode *inode)
{
return (inode->i_fop == &empty_dir_operations) &&
(inode->i_op == &empty_dir_inode_operations);
}
#if IS_ENABLED(CONFIG_UNICODE)
/**
* generic_ci_d_compare - generic d_compare implementation for casefolding filesystems
* @dentry: dentry whose name we are checking against
* @len: len of name of dentry
* @str: str pointer to name of dentry
* @name: Name to compare against
*
* Return: 0 if names match, 1 if mismatch, or -ERRNO
*/
static int generic_ci_d_compare(const struct dentry *dentry, unsigned int len,
const char *str, const struct qstr *name)
{
const struct dentry *parent;
const struct inode *dir;
char strbuf[DNAME_INLINE_LEN];
struct qstr qstr;
/*
* Attempt a case-sensitive match first. It is cheaper and
* should cover most lookups, including all the sane
* applications that expect a case-sensitive filesystem.
*
* This comparison is safe under RCU because the caller
* guarantees the consistency between str and len. See
* __d_lookup_rcu_op_compare() for details.
*/
if (len == name->len && !memcmp(str, name->name, len))
return 0;
parent = READ_ONCE(dentry->d_parent);
dir = READ_ONCE(parent->d_inode);
if (!dir || !IS_CASEFOLDED(dir))
return 1;
/*
* If the dentry name is stored in-line, then it may be concurrently
* modified by a rename. If this happens, the VFS will eventually retry
* the lookup, so it doesn't matter what ->d_compare() returns.
* However, it's unsafe to call utf8_strncasecmp() with an unstable
* string. Therefore, we have to copy the name into a temporary buffer.
*/
if (len <= DNAME_INLINE_LEN - 1) {
memcpy(strbuf, str, len);
strbuf[len] = 0;
str = strbuf;
/* prevent compiler from optimizing out the temporary buffer */
barrier();
}
qstr.len = len;
qstr.name = str;
return utf8_strncasecmp(dentry->d_sb->s_encoding, name, &qstr);
}
/**
* generic_ci_d_hash - generic d_hash implementation for casefolding filesystems
* @dentry: dentry of the parent directory
* @str: qstr of name whose hash we should fill in
*
* Return: 0 if hash was successful or unchanged, and -EINVAL on error
*/
static int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str)
{
const struct inode *dir = READ_ONCE(dentry->d_inode);
struct super_block *sb = dentry->d_sb;
const struct unicode_map *um = sb->s_encoding;
int ret;
if (!dir || !IS_CASEFOLDED(dir))
return 0;
ret = utf8_casefold_hash(um, dentry, str);
if (ret < 0 && sb_has_strict_encoding(sb))
return -EINVAL;
return 0;
}
static const struct dentry_operations generic_ci_dentry_ops = {
.d_hash = generic_ci_d_hash,
.d_compare = generic_ci_d_compare,
#ifdef CONFIG_FS_ENCRYPTION
.d_revalidate = fscrypt_d_revalidate,
#endif
};
#endif
#ifdef CONFIG_FS_ENCRYPTION
static const struct dentry_operations generic_encrypted_dentry_ops = {
.d_revalidate = fscrypt_d_revalidate,
};
#endif
/**
* generic_set_sb_d_ops - helper for choosing the set of
* filesystem-wide dentry operations for the enabled features
* @sb: superblock to be configured
*
* Filesystems supporting casefolding and/or fscrypt can call this
* helper at mount-time to configure sb->s_d_op to best set of dentry
* operations required for the enabled features. The helper must be
* called after these have been configured, but before the root dentry
* is created.
*/
void generic_set_sb_d_ops(struct super_block *sb)
{
#if IS_ENABLED(CONFIG_UNICODE)
if (sb->s_encoding) {
sb->s_d_op = &generic_ci_dentry_ops;
return;
}
#endif
#ifdef CONFIG_FS_ENCRYPTION
if (sb->s_cop) {
sb->s_d_op = &generic_encrypted_dentry_ops;
return;
}
#endif
}
EXPORT_SYMBOL(generic_set_sb_d_ops);
/**
* inode_maybe_inc_iversion - increments i_version
* @inode: inode with the i_version that should be updated
* @force: increment the counter even if it's not necessary?
*
* Every time the inode is modified, the i_version field must be seen to have
* changed by any observer.
*
* If "force" is set or the QUERIED flag is set, then ensure that we increment
* the value, and clear the queried flag.
*
* In the common case where neither is set, then we can return "false" without
* updating i_version.
*
* If this function returns false, and no other metadata has changed, then we
* can avoid logging the metadata.
*/
bool inode_maybe_inc_iversion(struct inode *inode, bool force)
{
u64 cur, new;
/*
* The i_version field is not strictly ordered with any other inode
* information, but the legacy inode_inc_iversion code used a spinlock
* to serialize increments.
*
* Here, we add full memory barriers to ensure that any de-facto
* ordering with other info is preserved.
*
* This barrier pairs with the barrier in inode_query_iversion()
*/
smp_mb();
cur = inode_peek_iversion_raw(inode);
do {
/* If flag is clear then we needn't do anything */
if (!force && !(cur & I_VERSION_QUERIED))
return false;
/* Since lowest bit is flag, add 2 to avoid it */
new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT; } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new));
return true;
}
EXPORT_SYMBOL(inode_maybe_inc_iversion);
/**
* inode_query_iversion - read i_version for later use
* @inode: inode from which i_version should be read
*
* Read the inode i_version counter. This should be used by callers that wish
* to store the returned i_version for later comparison. This will guarantee
* that a later query of the i_version will result in a different value if
* anything has changed.
*
* In this implementation, we fetch the current value, set the QUERIED flag and
* then try to swap it into place with a cmpxchg, if it wasn't already set. If
* that fails, we try again with the newly fetched value from the cmpxchg.
*/
u64 inode_query_iversion(struct inode *inode)
{
u64 cur, new;
cur = inode_peek_iversion_raw(inode);
do {
/* If flag is already set, then no need to swap */
if (cur & I_VERSION_QUERIED) {
/*
* This barrier (and the implicit barrier in the
* cmpxchg below) pairs with the barrier in
* inode_maybe_inc_iversion().
*/
smp_mb();
break;
}
new = cur | I_VERSION_QUERIED;
} while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new));
return cur >> I_VERSION_QUERIED_SHIFT;
}
EXPORT_SYMBOL(inode_query_iversion);
ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter,
ssize_t direct_written, ssize_t buffered_written)
{
struct address_space *mapping = iocb->ki_filp->f_mapping;
loff_t pos = iocb->ki_pos - buffered_written;
loff_t end = iocb->ki_pos - 1;
int err;
/*
* If the buffered write fallback returned an error, we want to return
* the number of bytes which were written by direct I/O, or the error
* code if that was zero.
*
* Note that this differs from normal direct-io semantics, which will
* return -EFOO even if some bytes were written.
*/
if (unlikely(buffered_written < 0)) {
if (direct_written)
return direct_written;
return buffered_written;
}
/*
* We need to ensure that the page cache pages are written to disk and
* invalidated to preserve the expected O_DIRECT semantics.
*/
err = filemap_write_and_wait_range(mapping, pos, end);
if (err < 0) {
/*
* We don't know how much we wrote, so just return the number of
* bytes which were direct-written
*/
iocb->ki_pos -= buffered_written;
if (direct_written)
return direct_written;
return err;
}
invalidate_mapping_pages(mapping, pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
return direct_written + buffered_written;
}
EXPORT_SYMBOL_GPL(direct_write_fallback);
/**
* simple_inode_init_ts - initialize the timestamps for a new inode
* @inode: inode to be initialized
*
* When a new inode is created, most filesystems set the timestamps to the
* current time. Add a helper to do this.
*/
struct timespec64 simple_inode_init_ts(struct inode *inode)
{
struct timespec64 ts = inode_set_ctime_current(inode);
inode_set_atime_to_ts(inode, ts);
inode_set_mtime_to_ts(inode, ts);
return ts;
}
EXPORT_SYMBOL(simple_inode_init_ts);
static inline struct dentry *get_stashed_dentry(struct dentry *stashed)
{
struct dentry *dentry;
guard(rcu)();
dentry = READ_ONCE(stashed);
if (!dentry)
return NULL;
if (!lockref_get_not_dead(&dentry->d_lockref))
return NULL;
return dentry;
}
static struct dentry *prepare_anon_dentry(struct dentry **stashed,
struct super_block *sb,
void *data)
{
struct dentry *dentry;
struct inode *inode;
const struct stashed_operations *sops = sb->s_fs_info;
int ret;
inode = new_inode_pseudo(sb);
if (!inode) {
sops->put_data(data);
return ERR_PTR(-ENOMEM);
}
inode->i_flags |= S_IMMUTABLE;
inode->i_mode = S_IFREG;
simple_inode_init_ts(inode);
ret = sops->init_inode(inode, data);
if (ret < 0) {
iput(inode);
return ERR_PTR(ret);
}
/* Notice when this is changed. */
WARN_ON_ONCE(!S_ISREG(inode->i_mode));
WARN_ON_ONCE(!IS_IMMUTABLE(inode));
dentry = d_alloc_anon(sb);
if (!dentry) {
iput(inode);
return ERR_PTR(-ENOMEM);
}
/* Store address of location where dentry's supposed to be stashed. */
dentry->d_fsdata = stashed;
/* @data is now owned by the fs */
d_instantiate(dentry, inode);
return dentry;
}
static struct dentry *stash_dentry(struct dentry **stashed,
struct dentry *dentry)
{
guard(rcu)();
for (;;) {
struct dentry *old;
/* Assume any old dentry was cleared out. */
old = cmpxchg(stashed, NULL, dentry);
if (likely(!old))
return dentry;
/* Check if somebody else installed a reusable dentry. */
if (lockref_get_not_dead(&old->d_lockref))
return old;
/* There's an old dead dentry there, try to take it over. */
if (likely(try_cmpxchg(stashed, &old, dentry)))
return dentry;
}
}
/**
* path_from_stashed - create path from stashed or new dentry
* @stashed: where to retrieve or stash dentry
* @mnt: mnt of the filesystems to use
* @data: data to store in inode->i_private
* @path: path to create
*
* The function tries to retrieve a stashed dentry from @stashed. If the dentry
* is still valid then it will be reused. If the dentry isn't able the function
* will allocate a new dentry and inode. It will then check again whether it
* can reuse an existing dentry in case one has been added in the meantime or
* update @stashed with the newly added dentry.
*
* Special-purpose helper for nsfs and pidfs.
*
* Return: On success zero and on failure a negative error is returned.
*/
int path_from_stashed(struct dentry **stashed, struct vfsmount *mnt, void *data,
struct path *path)
{
struct dentry *dentry;
const struct stashed_operations *sops = mnt->mnt_sb->s_fs_info;
/* See if dentry can be reused. */
path->dentry = get_stashed_dentry(*stashed);
if (path->dentry) {
sops->put_data(data);
goto out_path;
}
/* Allocate a new dentry. */
dentry = prepare_anon_dentry(stashed, mnt->mnt_sb, data);
if (IS_ERR(dentry))
return PTR_ERR(dentry);
/* Added a new dentry. @data is now owned by the filesystem. */
path->dentry = stash_dentry(stashed, dentry);
if (path->dentry != dentry)
dput(dentry);
out_path:
WARN_ON_ONCE(path->dentry->d_fsdata != stashed);
WARN_ON_ONCE(d_inode(path->dentry)->i_private != data);
path->mnt = mntget(mnt);
return 0;
}
void stashed_dentry_prune(struct dentry *dentry)
{
struct dentry **stashed = dentry->d_fsdata;
struct inode *inode = d_inode(dentry);
if (WARN_ON_ONCE(!stashed))
return;
if (!inode)
return;
/*
* Only replace our own @dentry as someone else might've
* already cleared out @dentry and stashed their own
* dentry in there.
*/
cmpxchg(stashed, dentry, NULL);
}
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Initialization routines
* Copyright (c) by Jaroslav Kysela <perex@perex.cz>
*/
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/ctype.h>
#include <linux/pm.h>
#include <linux/debugfs.h>
#include <linux/completion.h>
#include <linux/interrupt.h>
#include <sound/core.h>
#include <sound/control.h>
#include <sound/info.h>
/* monitor files for graceful shutdown (hotplug) */
struct snd_monitor_file {
struct file *file;
const struct file_operations *disconnected_f_op;
struct list_head shutdown_list; /* still need to shutdown */
struct list_head list; /* link of monitor files */
};
static DEFINE_SPINLOCK(shutdown_lock);
static LIST_HEAD(shutdown_files);
static const struct file_operations snd_shutdown_f_ops;
/* locked for registering/using */
static DECLARE_BITMAP(snd_cards_lock, SNDRV_CARDS);
static struct snd_card *snd_cards[SNDRV_CARDS];
static DEFINE_MUTEX(snd_card_mutex);
static char *slots[SNDRV_CARDS];
module_param_array(slots, charp, NULL, 0444);
MODULE_PARM_DESC(slots, "Module names assigned to the slots.");
/* return non-zero if the given index is reserved for the given
* module via slots option
*/
static int module_slot_match(struct module *module, int idx)
{
int match = 1;
#ifdef MODULE
const char *s1, *s2;
if (!module || !*module->name || !slots[idx])
return 0;
s1 = module->name;
s2 = slots[idx];
if (*s2 == '!') {
match = 0; /* negative match */
s2++;
}
/* compare module name strings
* hyphens are handled as equivalent with underscore
*/
for (;;) {
char c1 = *s1++;
char c2 = *s2++;
if (c1 == '-')
c1 = '_';
if (c2 == '-')
c2 = '_';
if (c1 != c2)
return !match;
if (!c1)
break;
}
#endif /* MODULE */
return match;
}
#if IS_ENABLED(CONFIG_SND_MIXER_OSS)
int (*snd_mixer_oss_notify_callback)(struct snd_card *card, int free_flag);
EXPORT_SYMBOL(snd_mixer_oss_notify_callback);
#endif
static int check_empty_slot(struct module *module, int slot)
{
return !slots[slot] || !*slots[slot];
}
/* return an empty slot number (>= 0) found in the given bitmask @mask.
* @mask == -1 == 0xffffffff means: take any free slot up to 32
* when no slot is available, return the original @mask as is.
*/
static int get_slot_from_bitmask(int mask, int (*check)(struct module *, int),
struct module *module)
{
int slot;
for (slot = 0; slot < SNDRV_CARDS; slot++) {
if (slot < 32 && !(mask & (1U << slot)))
continue;
if (!test_bit(slot, snd_cards_lock)) {
if (check(module, slot))
return slot; /* found */
}
}
return mask; /* unchanged */
}
/* the default release callback set in snd_device_alloc() */
static void default_release_alloc(struct device *dev)
{
kfree(dev);
}
/**
* snd_device_alloc - Allocate and initialize struct device for sound devices
* @dev_p: pointer to store the allocated device
* @card: card to assign, optional
*
* For releasing the allocated device, call put_device().
*/
int snd_device_alloc(struct device **dev_p, struct snd_card *card)
{
struct device *dev;
*dev_p = NULL;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return -ENOMEM;
device_initialize(dev);
if (card)
dev->parent = &card->card_dev;
dev->class = &sound_class;
dev->release = default_release_alloc;
*dev_p = dev;
return 0;
}
EXPORT_SYMBOL_GPL(snd_device_alloc);
static int snd_card_init(struct snd_card *card, struct device *parent,
int idx, const char *xid, struct module *module,
size_t extra_size);
static int snd_card_do_free(struct snd_card *card);
static const struct attribute_group card_dev_attr_group;
static void release_card_device(struct device *dev)
{
snd_card_do_free(dev_to_snd_card(dev));
}
/**
* snd_card_new - create and initialize a soundcard structure
* @parent: the parent device object
* @idx: card index (address) [0 ... (SNDRV_CARDS-1)]
* @xid: card identification (ASCII string)
* @module: top level module for locking
* @extra_size: allocate this extra size after the main soundcard structure
* @card_ret: the pointer to store the created card instance
*
* The function allocates snd_card instance via kzalloc with the given
* space for the driver to use freely. The allocated struct is stored
* in the given card_ret pointer.
*
* Return: Zero if successful or a negative error code.
*/
int snd_card_new(struct device *parent, int idx, const char *xid,
struct module *module, int extra_size,
struct snd_card **card_ret)
{
struct snd_card *card;
int err;
if (snd_BUG_ON(!card_ret))
return -EINVAL;
*card_ret = NULL;
if (extra_size < 0)
extra_size = 0;
card = kzalloc(sizeof(*card) + extra_size, GFP_KERNEL);
if (!card)
return -ENOMEM;
err = snd_card_init(card, parent, idx, xid, module, extra_size);
if (err < 0)
return err; /* card is freed by error handler */
*card_ret = card;
return 0;
}
EXPORT_SYMBOL(snd_card_new);
static void __snd_card_release(struct device *dev, void *data)
{
snd_card_free(data);
}
/**
* snd_devm_card_new - managed snd_card object creation
* @parent: the parent device object
* @idx: card index (address) [0 ... (SNDRV_CARDS-1)]
* @xid: card identification (ASCII string)
* @module: top level module for locking
* @extra_size: allocate this extra size after the main soundcard structure
* @card_ret: the pointer to store the created card instance
*
* This function works like snd_card_new() but manages the allocated resource
* via devres, i.e. you don't need to free explicitly.
*
* When a snd_card object is created with this function and registered via
* snd_card_register(), the very first devres action to call snd_card_free()
* is added automatically. In that way, the resource disconnection is assured
* at first, then released in the expected order.
*
* If an error happens at the probe before snd_card_register() is called and
* there have been other devres resources, you'd need to free the card manually
* via snd_card_free() call in the error; otherwise it may lead to UAF due to
* devres call orders. You can use snd_card_free_on_error() helper for
* handling it more easily.
*
* Return: zero if successful, or a negative error code
*/
int snd_devm_card_new(struct device *parent, int idx, const char *xid,
struct module *module, size_t extra_size,
struct snd_card **card_ret)
{
struct snd_card *card;
int err;
*card_ret = NULL;
card = devres_alloc(__snd_card_release, sizeof(*card) + extra_size,
GFP_KERNEL);
if (!card)
return -ENOMEM;
card->managed = true;
err = snd_card_init(card, parent, idx, xid, module, extra_size);
if (err < 0) {
devres_free(card); /* in managed mode, we need to free manually */
return err;
}
devres_add(parent, card);
*card_ret = card;
return 0;
}
EXPORT_SYMBOL_GPL(snd_devm_card_new);
/**
* snd_card_free_on_error - a small helper for handling devm probe errors
* @dev: the managed device object
* @ret: the return code from the probe callback
*
* This function handles the explicit snd_card_free() call at the error from
* the probe callback. It's just a small helper for simplifying the error
* handling with the managed devices.
*
* Return: zero if successful, or a negative error code
*/
int snd_card_free_on_error(struct device *dev, int ret)
{
struct snd_card *card;
if (!ret)
return 0;
card = devres_find(dev, __snd_card_release, NULL, NULL);
if (card)
snd_card_free(card);
return ret;
}
EXPORT_SYMBOL_GPL(snd_card_free_on_error);
static int snd_card_init(struct snd_card *card, struct device *parent,
int idx, const char *xid, struct module *module,
size_t extra_size)
{
int err;
if (extra_size > 0)
card->private_data = (char *)card + sizeof(struct snd_card);
if (xid)
strscpy(card->id, xid, sizeof(card->id));
err = 0;
scoped_guard(mutex, &snd_card_mutex) {
if (idx < 0) /* first check the matching module-name slot */
idx = get_slot_from_bitmask(idx, module_slot_match, module);
if (idx < 0) /* if not matched, assign an empty slot */
idx = get_slot_from_bitmask(idx, check_empty_slot, module);
if (idx < 0)
err = -ENODEV;
else if (idx < snd_ecards_limit) {
if (test_bit(idx, snd_cards_lock))
err = -EBUSY; /* invalid */
} else if (idx >= SNDRV_CARDS)
err = -ENODEV;
if (!err) {
set_bit(idx, snd_cards_lock); /* lock it */
if (idx >= snd_ecards_limit)
snd_ecards_limit = idx + 1; /* increase the limit */
}
}
if (err < 0) {
dev_err(parent, "cannot find the slot for index %d (range 0-%i), error: %d\n",
idx, snd_ecards_limit - 1, err);
if (!card->managed)
kfree(card); /* manually free here, as no destructor called */
return err;
}
card->dev = parent;
card->number = idx;
#ifdef MODULE
WARN_ON(!module);
card->module = module;
#endif
INIT_LIST_HEAD(&card->devices);
init_rwsem(&card->controls_rwsem);
rwlock_init(&card->ctl_files_rwlock);
INIT_LIST_HEAD(&card->controls);
INIT_LIST_HEAD(&card->ctl_files);
#ifdef CONFIG_SND_CTL_FAST_LOOKUP
xa_init(&card->ctl_numids);
xa_init(&card->ctl_hash);
#endif
spin_lock_init(&card->files_lock);
INIT_LIST_HEAD(&card->files_list);
mutex_init(&card->memory_mutex);
#ifdef CONFIG_PM
init_waitqueue_head(&card->power_sleep);
init_waitqueue_head(&card->power_ref_sleep);
atomic_set(&card->power_ref, 0);
#endif
init_waitqueue_head(&card->remove_sleep);
card->sync_irq = -1;
device_initialize(&card->card_dev);
card->card_dev.parent = parent;
card->card_dev.class = &sound_class;
card->card_dev.release = release_card_device;
card->card_dev.groups = card->dev_groups;
card->dev_groups[0] = &card_dev_attr_group;
err = kobject_set_name(&card->card_dev.kobj, "card%d", idx);
if (err < 0)
goto __error;
snprintf(card->irq_descr, sizeof(card->irq_descr), "%s:%s",
dev_driver_string(card->dev), dev_name(&card->card_dev));
/* the control interface cannot be accessed from the user space until */
/* snd_cards_bitmask and snd_cards are set with snd_card_register */
err = snd_ctl_create(card);
if (err < 0) {
dev_err(parent, "unable to register control minors\n");
goto __error;
}
err = snd_info_card_create(card);
if (err < 0) {
dev_err(parent, "unable to create card info\n");
goto __error_ctl;
}
#ifdef CONFIG_SND_DEBUG
card->debugfs_root = debugfs_create_dir(dev_name(&card->card_dev),
sound_debugfs_root);
#endif
return 0;
__error_ctl:
snd_device_free_all(card);
__error:
put_device(&card->card_dev);
return err;
}
/**
* snd_card_ref - Get the card object from the index
* @idx: the card index
*
* Returns a card object corresponding to the given index or NULL if not found.
* Release the object via snd_card_unref().
*
* Return: a card object or NULL
*/
struct snd_card *snd_card_ref(int idx)
{
struct snd_card *card;
guard(mutex)(&snd_card_mutex);
card = snd_cards[idx];
if (card)
get_device(&card->card_dev);
return card;
}
EXPORT_SYMBOL_GPL(snd_card_ref);
/* return non-zero if a card is already locked */
int snd_card_locked(int card)
{
guard(mutex)(&snd_card_mutex);
return test_bit(card, snd_cards_lock);
}
static loff_t snd_disconnect_llseek(struct file *file, loff_t offset, int orig)
{
return -ENODEV;
}
static ssize_t snd_disconnect_read(struct file *file, char __user *buf,
size_t count, loff_t *offset)
{
return -ENODEV;
}
static ssize_t snd_disconnect_write(struct file *file, const char __user *buf,
size_t count, loff_t *offset)
{
return -ENODEV;
}
static int snd_disconnect_release(struct inode *inode, struct file *file)
{
struct snd_monitor_file *df = NULL, *_df;
scoped_guard(spinlock, &shutdown_lock) {
list_for_each_entry(_df, &shutdown_files, shutdown_list) {
if (_df->file == file) {
df = _df;
list_del_init(&df->shutdown_list);
break;
}
}
}
if (likely(df)) {
if ((file->f_flags & FASYNC) && df->disconnected_f_op->fasync)
df->disconnected_f_op->fasync(-1, file, 0);
return df->disconnected_f_op->release(inode, file);
}
panic("%s(%p, %p) failed!", __func__, inode, file);
}
static __poll_t snd_disconnect_poll(struct file * file, poll_table * wait)
{
return EPOLLERR | EPOLLNVAL;
}
static long snd_disconnect_ioctl(struct file *file,
unsigned int cmd, unsigned long arg)
{
return -ENODEV;
}
static int snd_disconnect_mmap(struct file *file, struct vm_area_struct *vma)
{
return -ENODEV;
}
static int snd_disconnect_fasync(int fd, struct file *file, int on)
{
return -ENODEV;
}
static const struct file_operations snd_shutdown_f_ops =
{
.owner = THIS_MODULE,
.llseek = snd_disconnect_llseek,
.read = snd_disconnect_read,
.write = snd_disconnect_write,
.release = snd_disconnect_release,
.poll = snd_disconnect_poll,
.unlocked_ioctl = snd_disconnect_ioctl,
#ifdef CONFIG_COMPAT
.compat_ioctl = snd_disconnect_ioctl,
#endif
.mmap = snd_disconnect_mmap,
.fasync = snd_disconnect_fasync
};
/**
* snd_card_disconnect - disconnect all APIs from the file-operations (user space)
* @card: soundcard structure
*
* Disconnects all APIs from the file-operations (user space).
*
* Return: Zero, otherwise a negative error code.
*
* Note: The current implementation replaces all active file->f_op with special
* dummy file operations (they do nothing except release).
*/
void snd_card_disconnect(struct snd_card *card)
{
struct snd_monitor_file *mfile;
if (!card)
return;
scoped_guard(spinlock, &card->files_lock) { if (card->shutdown)
return;
card->shutdown = 1;
/* replace file->f_op with special dummy operations */
list_for_each_entry(mfile, &card->files_list, list) {
/* it's critical part, use endless loop */
/* we have no room to fail */
mfile->disconnected_f_op = mfile->file->f_op; scoped_guard(spinlock, &shutdown_lock) list_add(&mfile->shutdown_list, &shutdown_files);
mfile->file->f_op = &snd_shutdown_f_ops;
fops_get(mfile->file->f_op);
}
}
/* notify all connected devices about disconnection */
/* at this point, they cannot respond to any calls except release() */
#if IS_ENABLED(CONFIG_SND_MIXER_OSS)
if (snd_mixer_oss_notify_callback)
snd_mixer_oss_notify_callback(card, SND_MIXER_OSS_NOTIFY_DISCONNECT);
#endif
/* notify all devices that we are disconnected */
snd_device_disconnect_all(card);
if (card->sync_irq > 0)
synchronize_irq(card->sync_irq); snd_info_card_disconnect(card);
if (card->registered) {
device_del(&card->card_dev);
card->registered = false;
}
/* disable fops (user space) operations for ALSA API */
scoped_guard(mutex, &snd_card_mutex) {
snd_cards[card->number] = NULL;
clear_bit(card->number, snd_cards_lock);
}
#ifdef CONFIG_PM
wake_up(&card->power_sleep);
snd_power_sync_ref(card);
#endif
}
EXPORT_SYMBOL(snd_card_disconnect);
/**
* snd_card_disconnect_sync - disconnect card and wait until files get closed
* @card: card object to disconnect
*
* This calls snd_card_disconnect() for disconnecting all belonging components
* and waits until all pending files get closed.
* It assures that all accesses from user-space finished so that the driver
* can release its resources gracefully.
*/
void snd_card_disconnect_sync(struct snd_card *card)
{
snd_card_disconnect(card);
guard(spinlock_irq)(&card->files_lock);
wait_event_lock_irq(card->remove_sleep,
list_empty(&card->files_list),
card->files_lock);
}
EXPORT_SYMBOL_GPL(snd_card_disconnect_sync);
static int snd_card_do_free(struct snd_card *card)
{
card->releasing = true;
#if IS_ENABLED(CONFIG_SND_MIXER_OSS)
if (snd_mixer_oss_notify_callback)
snd_mixer_oss_notify_callback(card, SND_MIXER_OSS_NOTIFY_FREE);
#endif
snd_device_free_all(card);
if (card->private_free)
card->private_free(card); if (snd_info_card_free(card) < 0) { dev_warn(card->dev, "unable to free card info\n");
/* Not fatal error */
}
#ifdef CONFIG_SND_DEBUG
debugfs_remove(card->debugfs_root);
card->debugfs_root = NULL;
#endif
if (card->release_completion)
complete(card->release_completion); if (!card->managed) kfree(card);
return 0;
}
/**
* snd_card_free_when_closed - Disconnect the card, free it later eventually
* @card: soundcard structure
*
* Unlike snd_card_free(), this function doesn't try to release the card
* resource immediately, but tries to disconnect at first. When the card
* is still in use, the function returns before freeing the resources.
* The card resources will be freed when the refcount gets to zero.
*
* Return: zero if successful, or a negative error code
*/
void snd_card_free_when_closed(struct snd_card *card)
{
if (!card)
return;
snd_card_disconnect(card);
put_device(&card->card_dev);
return;
}
EXPORT_SYMBOL(snd_card_free_when_closed);
/**
* snd_card_free - frees given soundcard structure
* @card: soundcard structure
*
* This function releases the soundcard structure and the all assigned
* devices automatically. That is, you don't have to release the devices
* by yourself.
*
* This function waits until the all resources are properly released.
*
* Return: Zero. Frees all associated devices and frees the control
* interface associated to given soundcard.
*/
void snd_card_free(struct snd_card *card)
{
DECLARE_COMPLETION_ONSTACK(released);
/* The call of snd_card_free() is allowed from various code paths;
* a manual call from the driver and the call via devres_free, and
* we need to avoid double-free. Moreover, the release via devres
* may call snd_card_free() twice due to its nature, we need to have
* the check here at the beginning.
*/
if (card->releasing)
return;
card->release_completion = &released; snd_card_free_when_closed(card);
/* wait, until all devices are ready for the free operation */
wait_for_completion(&released);
}
EXPORT_SYMBOL(snd_card_free);
/* retrieve the last word of shortname or longname */
static const char *retrieve_id_from_card_name(const char *name)
{
const char *spos = name;
while (*name) {
if (isspace(*name) && isalnum(name[1]))
spos = name + 1;
name++;
}
return spos;
}
/* return true if the given id string doesn't conflict any other card ids */
static bool card_id_ok(struct snd_card *card, const char *id)
{
int i;
if (!snd_info_check_reserved_words(id))
return false;
for (i = 0; i < snd_ecards_limit; i++) {
if (snd_cards[i] && snd_cards[i] != card &&
!strcmp(snd_cards[i]->id, id))
return false;
}
return true;
}
/* copy to card->id only with valid letters from nid */
static void copy_valid_id_string(struct snd_card *card, const char *src,
const char *nid)
{
char *id = card->id;
while (*nid && !isalnum(*nid))
nid++;
if (isdigit(*nid))
*id++ = isalpha(*src) ? *src : 'D';
while (*nid && (size_t)(id - card->id) < sizeof(card->id) - 1) {
if (isalnum(*nid))
*id++ = *nid;
nid++;
}
*id = 0;
}
/* Set card->id from the given string
* If the string conflicts with other ids, add a suffix to make it unique.
*/
static void snd_card_set_id_no_lock(struct snd_card *card, const char *src,
const char *nid)
{
int len, loops;
bool is_default = false;
char *id;
copy_valid_id_string(card, src, nid);
id = card->id;
again:
/* use "Default" for obviously invalid strings
* ("card" conflicts with proc directories)
*/
if (!*id || !strncmp(id, "card", 4)) {
strcpy(id, "Default");
is_default = true;
}
len = strlen(id);
for (loops = 0; loops < SNDRV_CARDS; loops++) {
char *spos;
char sfxstr[5]; /* "_012" */
int sfxlen;
if (card_id_ok(card, id))
return; /* OK */
/* Add _XYZ suffix */
sprintf(sfxstr, "_%X", loops + 1);
sfxlen = strlen(sfxstr);
if (len + sfxlen >= sizeof(card->id))
spos = id + sizeof(card->id) - sfxlen - 1;
else
spos = id + len;
strcpy(spos, sfxstr);
}
/* fallback to the default id */
if (!is_default) {
*id = 0;
goto again;
}
/* last resort... */
dev_err(card->dev, "unable to set card id (%s)\n", id);
if (card->proc_root->name)
strscpy(card->id, card->proc_root->name, sizeof(card->id));
}
/**
* snd_card_set_id - set card identification name
* @card: soundcard structure
* @nid: new identification string
*
* This function sets the card identification and checks for name
* collisions.
*/
void snd_card_set_id(struct snd_card *card, const char *nid)
{
/* check if user specified own card->id */
if (card->id[0] != '\0')
return;
guard(mutex)(&snd_card_mutex);
snd_card_set_id_no_lock(card, nid, nid);
}
EXPORT_SYMBOL(snd_card_set_id);
static ssize_t id_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct snd_card *card = container_of(dev, struct snd_card, card_dev);
return sysfs_emit(buf, "%s\n", card->id);
}
static ssize_t id_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
struct snd_card *card = container_of(dev, struct snd_card, card_dev);
char buf1[sizeof(card->id)];
size_t copy = count > sizeof(card->id) - 1 ?
sizeof(card->id) - 1 : count;
size_t idx;
int c;
for (idx = 0; idx < copy; idx++) {
c = buf[idx];
if (!isalnum(c) && c != '_' && c != '-')
return -EINVAL;
}
memcpy(buf1, buf, copy);
buf1[copy] = '\0';
guard(mutex)(&snd_card_mutex);
if (!card_id_ok(NULL, buf1))
return -EEXIST;
strcpy(card->id, buf1);
snd_info_card_id_change(card);
return count;
}
static DEVICE_ATTR_RW(id);
static ssize_t number_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct snd_card *card = container_of(dev, struct snd_card, card_dev);
return sysfs_emit(buf, "%i\n", card->number);
}
static DEVICE_ATTR_RO(number);
static struct attribute *card_dev_attrs[] = {
&dev_attr_id.attr,
&dev_attr_number.attr,
NULL
};
static const struct attribute_group card_dev_attr_group = {
.attrs = card_dev_attrs,
};
/**
* snd_card_add_dev_attr - Append a new sysfs attribute group to card
* @card: card instance
* @group: attribute group to append
*
* Return: zero if successful, or a negative error code
*/
int snd_card_add_dev_attr(struct snd_card *card,
const struct attribute_group *group)
{
int i;
/* loop for (arraysize-1) here to keep NULL at the last entry */
for (i = 0; i < ARRAY_SIZE(card->dev_groups) - 1; i++) {
if (!card->dev_groups[i]) {
card->dev_groups[i] = group;
return 0;
}
}
dev_err(card->dev, "Too many groups assigned\n");
return -ENOSPC;
}
EXPORT_SYMBOL_GPL(snd_card_add_dev_attr);
static void trigger_card_free(void *data)
{
snd_card_free(data);
}
/**
* snd_card_register - register the soundcard
* @card: soundcard structure
*
* This function registers all the devices assigned to the soundcard.
* Until calling this, the ALSA control interface is blocked from the
* external accesses. Thus, you should call this function at the end
* of the initialization of the card.
*
* Return: Zero otherwise a negative error code if the registration failed.
*/
int snd_card_register(struct snd_card *card)
{
int err;
if (snd_BUG_ON(!card))
return -EINVAL;
if (!card->registered) {
err = device_add(&card->card_dev);
if (err < 0)
return err;
card->registered = true;
} else {
if (card->managed)
devm_remove_action(card->dev, trigger_card_free, card);
}
if (card->managed) {
err = devm_add_action(card->dev, trigger_card_free, card);
if (err < 0)
return err;
}
err = snd_device_register_all(card);
if (err < 0)
return err;
scoped_guard(mutex, &snd_card_mutex) {
if (snd_cards[card->number]) {
/* already registered */
return snd_info_card_register(card); /* register pending info */
}
if (*card->id) {
/* make a unique id name from the given string */
char tmpid[sizeof(card->id)];
memcpy(tmpid, card->id, sizeof(card->id));
snd_card_set_id_no_lock(card, tmpid, tmpid);
} else {
/* create an id from either shortname or longname */
const char *src;
src = *card->shortname ? card->shortname : card->longname;
snd_card_set_id_no_lock(card, src,
retrieve_id_from_card_name(src));
}
snd_cards[card->number] = card;
}
err = snd_info_card_register(card);
if (err < 0)
return err;
#if IS_ENABLED(CONFIG_SND_MIXER_OSS)
if (snd_mixer_oss_notify_callback)
snd_mixer_oss_notify_callback(card, SND_MIXER_OSS_NOTIFY_REGISTER);
#endif
return 0;
}
EXPORT_SYMBOL(snd_card_register);
#ifdef CONFIG_SND_PROC_FS
static void snd_card_info_read(struct snd_info_entry *entry,
struct snd_info_buffer *buffer)
{
int idx, count;
struct snd_card *card;
for (idx = count = 0; idx < SNDRV_CARDS; idx++) {
guard(mutex)(&snd_card_mutex);
card = snd_cards[idx];
if (card) {
count++;
snd_iprintf(buffer, "%2i [%-15s]: %s - %s\n",
idx,
card->id,
card->driver,
card->shortname);
snd_iprintf(buffer, " %s\n",
card->longname);
}
}
if (!count)
snd_iprintf(buffer, "--- no soundcards ---\n");
}
#ifdef CONFIG_SND_OSSEMUL
void snd_card_info_read_oss(struct snd_info_buffer *buffer)
{
int idx, count;
struct snd_card *card;
for (idx = count = 0; idx < SNDRV_CARDS; idx++) {
guard(mutex)(&snd_card_mutex);
card = snd_cards[idx];
if (card) {
count++;
snd_iprintf(buffer, "%s\n", card->longname);
}
}
if (!count) {
snd_iprintf(buffer, "--- no soundcards ---\n");
}
}
#endif
#ifdef MODULE
static void snd_card_module_info_read(struct snd_info_entry *entry,
struct snd_info_buffer *buffer)
{
int idx;
struct snd_card *card;
for (idx = 0; idx < SNDRV_CARDS; idx++) {
guard(mutex)(&snd_card_mutex);
card = snd_cards[idx];
if (card)
snd_iprintf(buffer, "%2i %s\n",
idx, card->module->name);
}
}
#endif
int __init snd_card_info_init(void)
{
struct snd_info_entry *entry;
entry = snd_info_create_module_entry(THIS_MODULE, "cards", NULL);
if (! entry)
return -ENOMEM;
entry->c.text.read = snd_card_info_read;
if (snd_info_register(entry) < 0)
return -ENOMEM; /* freed in error path */
#ifdef MODULE
entry = snd_info_create_module_entry(THIS_MODULE, "modules", NULL);
if (!entry)
return -ENOMEM;
entry->c.text.read = snd_card_module_info_read;
if (snd_info_register(entry) < 0)
return -ENOMEM; /* freed in error path */
#endif
return 0;
}
#endif /* CONFIG_SND_PROC_FS */
/**
* snd_component_add - add a component string
* @card: soundcard structure
* @component: the component id string
*
* This function adds the component id string to the supported list.
* The component can be referred from the alsa-lib.
*
* Return: Zero otherwise a negative error code.
*/
int snd_component_add(struct snd_card *card, const char *component)
{
char *ptr;
int len = strlen(component);
ptr = strstr(card->components, component);
if (ptr != NULL) {
if (ptr[len] == '\0' || ptr[len] == ' ') /* already there */
return 1;
}
if (strlen(card->components) + 1 + len + 1 > sizeof(card->components)) {
snd_BUG();
return -ENOMEM;
}
if (card->components[0] != '\0')
strcat(card->components, " ");
strcat(card->components, component);
return 0;
}
EXPORT_SYMBOL(snd_component_add);
/**
* snd_card_file_add - add the file to the file list of the card
* @card: soundcard structure
* @file: file pointer
*
* This function adds the file to the file linked-list of the card.
* This linked-list is used to keep tracking the connection state,
* and to avoid the release of busy resources by hotplug.
*
* Return: zero or a negative error code.
*/
int snd_card_file_add(struct snd_card *card, struct file *file)
{
struct snd_monitor_file *mfile;
mfile = kmalloc(sizeof(*mfile), GFP_KERNEL);
if (mfile == NULL)
return -ENOMEM;
mfile->file = file;
mfile->disconnected_f_op = NULL;
INIT_LIST_HEAD(&mfile->shutdown_list);
guard(spinlock)(&card->files_lock);
if (card->shutdown) {
kfree(mfile);
return -ENODEV;
}
list_add(&mfile->list, &card->files_list);
get_device(&card->card_dev);
return 0;
}
EXPORT_SYMBOL(snd_card_file_add);
/**
* snd_card_file_remove - remove the file from the file list
* @card: soundcard structure
* @file: file pointer
*
* This function removes the file formerly added to the card via
* snd_card_file_add() function.
* If all files are removed and snd_card_free_when_closed() was
* called beforehand, it processes the pending release of
* resources.
*
* Return: Zero or a negative error code.
*/
int snd_card_file_remove(struct snd_card *card, struct file *file)
{
struct snd_monitor_file *mfile, *found = NULL;
scoped_guard(spinlock, &card->files_lock) {
list_for_each_entry(mfile, &card->files_list, list) {
if (mfile->file == file) {
list_del(&mfile->list);
scoped_guard(spinlock, &shutdown_lock)
list_del(&mfile->shutdown_list);
if (mfile->disconnected_f_op)
fops_put(mfile->disconnected_f_op);
found = mfile;
break;
}
}
if (list_empty(&card->files_list))
wake_up_all(&card->remove_sleep);
}
if (!found) {
dev_err(card->dev, "card file remove problem (%p)\n", file);
return -ENOENT;
}
kfree(found);
put_device(&card->card_dev);
return 0;
}
EXPORT_SYMBOL(snd_card_file_remove);
#ifdef CONFIG_PM
/**
* snd_power_ref_and_wait - wait until the card gets powered up
* @card: soundcard structure
*
* Take the power_ref reference count of the given card, and
* wait until the card gets powered up to SNDRV_CTL_POWER_D0 state.
* The refcount is down again while sleeping until power-up, hence this
* function can be used for syncing the floating control ops accesses,
* typically around calling control ops.
*
* The caller needs to pull down the refcount via snd_power_unref() later
* no matter whether the error is returned from this function or not.
*
* Return: Zero if successful, or a negative error code.
*/
int snd_power_ref_and_wait(struct snd_card *card)
{
snd_power_ref(card);
if (snd_power_get_state(card) == SNDRV_CTL_POWER_D0)
return 0;
wait_event_cmd(card->power_sleep,
card->shutdown ||
snd_power_get_state(card) == SNDRV_CTL_POWER_D0,
snd_power_unref(card), snd_power_ref(card));
return card->shutdown ? -ENODEV : 0;
}
EXPORT_SYMBOL_GPL(snd_power_ref_and_wait);
/**
* snd_power_wait - wait until the card gets powered up (old form)
* @card: soundcard structure
*
* Wait until the card gets powered up to SNDRV_CTL_POWER_D0 state.
*
* Return: Zero if successful, or a negative error code.
*/
int snd_power_wait(struct snd_card *card)
{
int ret;
ret = snd_power_ref_and_wait(card);
snd_power_unref(card);
return ret;
}
EXPORT_SYMBOL(snd_power_wait);
#endif /* CONFIG_PM */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2008 IBM Corporation
*
* Author: Mimi Zohar <zohar@us.ibm.com>
*
* File: ima_api.c
* Implements must_appraise_or_measure, collect_measurement,
* appraise_measurement, store_measurement and store_template.
*/
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/xattr.h>
#include <linux/evm.h>
#include <linux/fsverity.h>
#include "ima.h"
/*
* ima_free_template_entry - free an existing template entry
*/
void ima_free_template_entry(struct ima_template_entry *entry)
{
int i;
for (i = 0; i < entry->template_desc->num_fields; i++)
kfree(entry->template_data[i].data);
kfree(entry->digests);
kfree(entry);
}
/*
* ima_alloc_init_template - create and initialize a new template entry
*/
int ima_alloc_init_template(struct ima_event_data *event_data,
struct ima_template_entry **entry,
struct ima_template_desc *desc)
{
struct ima_template_desc *template_desc;
struct tpm_digest *digests;
int i, result = 0;
if (desc)
template_desc = desc;
else
template_desc = ima_template_desc_current();
*entry = kzalloc(struct_size(*entry, template_data,
template_desc->num_fields), GFP_NOFS);
if (!*entry)
return -ENOMEM;
digests = kcalloc(NR_BANKS(ima_tpm_chip) + ima_extra_slots,
sizeof(*digests), GFP_NOFS);
if (!digests) {
kfree(*entry);
*entry = NULL;
return -ENOMEM;
}
(*entry)->digests = digests;
(*entry)->template_desc = template_desc;
for (i = 0; i < template_desc->num_fields; i++) {
const struct ima_template_field *field =
template_desc->fields[i];
u32 len;
result = field->field_init(event_data,
&((*entry)->template_data[i]));
if (result != 0)
goto out;
len = (*entry)->template_data[i].len;
(*entry)->template_data_len += sizeof(len);
(*entry)->template_data_len += len;
}
return 0;
out:
ima_free_template_entry(*entry);
*entry = NULL;
return result;
}
/*
* ima_store_template - store ima template measurements
*
* Calculate the hash of a template entry, add the template entry
* to an ordered list of measurement entries maintained inside the kernel,
* and also update the aggregate integrity value (maintained inside the
* configured TPM PCR) over the hashes of the current list of measurement
* entries.
*
* Applications retrieve the current kernel-held measurement list through
* the securityfs entries in /sys/kernel/security/ima. The signed aggregate
* TPM PCR (called quote) can be retrieved using a TPM user space library
* and is used to validate the measurement list.
*
* Returns 0 on success, error code otherwise
*/
int ima_store_template(struct ima_template_entry *entry,
int violation, struct inode *inode,
const unsigned char *filename, int pcr)
{
static const char op[] = "add_template_measure";
static const char audit_cause[] = "hashing_error";
char *template_name = entry->template_desc->name;
int result;
if (!violation) {
result = ima_calc_field_array_hash(&entry->template_data[0],
entry);
if (result < 0) {
integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode,
template_name, op,
audit_cause, result, 0);
return result;
}
}
entry->pcr = pcr;
result = ima_add_template_entry(entry, violation, op, inode, filename);
return result;
}
/*
* ima_add_violation - add violation to measurement list.
*
* Violations are flagged in the measurement list with zero hash values.
* By extending the PCR with 0xFF's instead of with zeroes, the PCR
* value is invalidated.
*/
void ima_add_violation(struct file *file, const unsigned char *filename,
struct ima_iint_cache *iint, const char *op,
const char *cause)
{
struct ima_template_entry *entry;
struct inode *inode = file_inode(file);
struct ima_event_data event_data = { .iint = iint,
.file = file,
.filename = filename,
.violation = cause };
int violation = 1;
int result;
/* can overflow, only indicator */
atomic_long_inc(&ima_htable.violations);
result = ima_alloc_init_template(&event_data, &entry, NULL);
if (result < 0) {
result = -ENOMEM;
goto err_out;
}
result = ima_store_template(entry, violation, inode,
filename, CONFIG_IMA_MEASURE_PCR_IDX);
if (result < 0)
ima_free_template_entry(entry);
err_out:
integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
op, cause, result, 0);
}
/**
* ima_get_action - appraise & measure decision based on policy.
* @idmap: idmap of the mount the inode was found from
* @inode: pointer to the inode associated with the object being validated
* @cred: pointer to credentials structure to validate
* @secid: secid of the task being validated
* @mask: contains the permission mask (MAY_READ, MAY_WRITE, MAY_EXEC,
* MAY_APPEND)
* @func: caller identifier
* @pcr: pointer filled in if matched measure policy sets pcr=
* @template_desc: pointer filled in if matched measure policy sets template=
* @func_data: func specific data, may be NULL
* @allowed_algos: allowlist of hash algorithms for the IMA xattr
*
* The policy is defined in terms of keypairs:
* subj=, obj=, type=, func=, mask=, fsmagic=
* subj,obj, and type: are LSM specific.
* func: FILE_CHECK | BPRM_CHECK | CREDS_CHECK | MMAP_CHECK | MODULE_CHECK
* | KEXEC_CMDLINE | KEY_CHECK | CRITICAL_DATA | SETXATTR_CHECK
* | MMAP_CHECK_REQPROT
* mask: contains the permission mask
* fsmagic: hex value
*
* Returns IMA_MEASURE, IMA_APPRAISE mask.
*
*/
int ima_get_action(struct mnt_idmap *idmap, struct inode *inode,
const struct cred *cred, u32 secid, int mask,
enum ima_hooks func, int *pcr,
struct ima_template_desc **template_desc,
const char *func_data, unsigned int *allowed_algos)
{
int flags = IMA_MEASURE | IMA_AUDIT | IMA_APPRAISE | IMA_HASH;
flags &= ima_policy_flag;
return ima_match_policy(idmap, inode, cred, secid, func, mask,
flags, pcr, template_desc, func_data,
allowed_algos);
}
static bool ima_get_verity_digest(struct ima_iint_cache *iint,
struct inode *inode,
struct ima_max_digest_data *hash)
{
enum hash_algo alg;
int digest_len;
/*
* On failure, 'measure' policy rules will result in a file data
* hash containing 0's.
*/
digest_len = fsverity_get_digest(inode, hash->digest, NULL, &alg);
if (digest_len == 0)
return false;
/*
* Unlike in the case of actually calculating the file hash, in
* the fsverity case regardless of the hash algorithm, return
* the verity digest to be included in the measurement list. A
* mismatch between the verity algorithm and the xattr signature
* algorithm, if one exists, will be detected later.
*/
hash->hdr.algo = alg;
hash->hdr.length = digest_len;
return true;
}
/*
* ima_collect_measurement - collect file measurement
*
* Calculate the file hash, if it doesn't already exist,
* storing the measurement and i_version in the iint.
*
* Must be called with iint->mutex held.
*
* Return 0 on success, error code otherwise
*/
int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file,
void *buf, loff_t size, enum hash_algo algo,
struct modsig *modsig)
{
const char *audit_cause = "failed";
struct inode *inode = file_inode(file);
struct inode *real_inode = d_real_inode(file_dentry(file));
const char *filename = file->f_path.dentry->d_name.name;
struct ima_max_digest_data hash;
struct kstat stat;
int result = 0;
int length;
void *tmpbuf;
u64 i_version = 0;
/*
* Always collect the modsig, because IMA might have already collected
* the file digest without collecting the modsig in a previous
* measurement rule.
*/
if (modsig)
ima_collect_modsig(modsig, buf, size);
if (iint->flags & IMA_COLLECTED)
goto out;
/*
* Detecting file change is based on i_version. On filesystems
* which do not support i_version, support was originally limited
* to an initial measurement/appraisal/audit, but was modified to
* assume the file changed.
*/
result = vfs_getattr_nosec(&file->f_path, &stat, STATX_CHANGE_COOKIE,
AT_STATX_SYNC_AS_STAT);
if (!result && (stat.result_mask & STATX_CHANGE_COOKIE))
i_version = stat.change_cookie;
hash.hdr.algo = algo;
hash.hdr.length = hash_digest_size[algo];
/* Initialize hash digest to 0's in case of failure */
memset(&hash.digest, 0, sizeof(hash.digest));
if (iint->flags & IMA_VERITY_REQUIRED) {
if (!ima_get_verity_digest(iint, inode, &hash)) {
audit_cause = "no-verity-digest";
result = -ENODATA;
}
} else if (buf) {
result = ima_calc_buffer_hash(buf, size, &hash.hdr);
} else {
result = ima_calc_file_hash(file, &hash.hdr);
}
if (result && result != -EBADF && result != -EINVAL)
goto out;
length = sizeof(hash.hdr) + hash.hdr.length;
tmpbuf = krealloc(iint->ima_hash, length, GFP_NOFS);
if (!tmpbuf) {
result = -ENOMEM;
goto out;
}
iint->ima_hash = tmpbuf;
memcpy(iint->ima_hash, &hash, length);
iint->version = i_version;
if (real_inode != inode) {
iint->real_ino = real_inode->i_ino;
iint->real_dev = real_inode->i_sb->s_dev;
}
/* Possibly temporary failure due to type of read (eg. O_DIRECT) */
if (!result)
iint->flags |= IMA_COLLECTED;
out:
if (result) {
if (file->f_flags & O_DIRECT)
audit_cause = "failed(directio)";
integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode,
filename, "collect_data", audit_cause,
result, 0);
}
return result;
}
/*
* ima_store_measurement - store file measurement
*
* Create an "ima" template and then store the template by calling
* ima_store_template.
*
* We only get here if the inode has not already been measured,
* but the measurement could already exist:
* - multiple copies of the same file on either the same or
* different filesystems.
* - the inode was previously flushed as well as the iint info,
* containing the hashing info.
*
* Must be called with iint->mutex held.
*/
void ima_store_measurement(struct ima_iint_cache *iint, struct file *file,
const unsigned char *filename,
struct evm_ima_xattr_data *xattr_value,
int xattr_len, const struct modsig *modsig, int pcr,
struct ima_template_desc *template_desc)
{
static const char op[] = "add_template_measure";
static const char audit_cause[] = "ENOMEM";
int result = -ENOMEM;
struct inode *inode = file_inode(file);
struct ima_template_entry *entry;
struct ima_event_data event_data = { .iint = iint,
.file = file,
.filename = filename,
.xattr_value = xattr_value,
.xattr_len = xattr_len,
.modsig = modsig };
int violation = 0;
/*
* We still need to store the measurement in the case of MODSIG because
* we only have its contents to put in the list at the time of
* appraisal, but a file measurement from earlier might already exist in
* the measurement list.
*/
if (iint->measured_pcrs & (0x1 << pcr) && !modsig)
return;
result = ima_alloc_init_template(&event_data, &entry, template_desc);
if (result < 0) {
integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
op, audit_cause, result, 0);
return;
}
result = ima_store_template(entry, violation, inode, filename, pcr);
if ((!result || result == -EEXIST) && !(file->f_flags & O_DIRECT)) {
iint->flags |= IMA_MEASURED;
iint->measured_pcrs |= (0x1 << pcr);
}
if (result < 0)
ima_free_template_entry(entry);
}
void ima_audit_measurement(struct ima_iint_cache *iint,
const unsigned char *filename)
{
struct audit_buffer *ab;
char *hash;
const char *algo_name = hash_algo_name[iint->ima_hash->algo];
int i;
if (iint->flags & IMA_AUDITED)
return;
hash = kzalloc((iint->ima_hash->length * 2) + 1, GFP_KERNEL);
if (!hash)
return;
for (i = 0; i < iint->ima_hash->length; i++)
hex_byte_pack(hash + (i * 2), iint->ima_hash->digest[i]);
hash[i * 2] = '\0';
ab = audit_log_start(audit_context(), GFP_KERNEL,
AUDIT_INTEGRITY_RULE);
if (!ab)
goto out;
audit_log_format(ab, "file=");
audit_log_untrustedstring(ab, filename);
audit_log_format(ab, " hash=\"%s:%s\"", algo_name, hash);
audit_log_task_info(ab);
audit_log_end(ab);
iint->flags |= IMA_AUDITED;
out:
kfree(hash);
return;
}
/*
* ima_d_path - return a pointer to the full pathname
*
* Attempt to return a pointer to the full pathname for use in the
* IMA measurement list, IMA audit records, and auditing logs.
*
* On failure, return a pointer to a copy of the filename, not dname.
* Returning a pointer to dname, could result in using the pointer
* after the memory has been freed.
*/
const char *ima_d_path(const struct path *path, char **pathbuf, char *namebuf)
{
char *pathname = NULL;
*pathbuf = __getname();
if (*pathbuf) {
pathname = d_absolute_path(path, *pathbuf, PATH_MAX);
if (IS_ERR(pathname)) {
__putname(*pathbuf);
*pathbuf = NULL;
pathname = NULL;
}
}
if (!pathname) {
strscpy(namebuf, path->dentry->d_name.name, NAME_MAX);
pathname = namebuf;
}
return pathname;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PAGEMAP_H
#define _LINUX_PAGEMAP_H
/*
* Copyright 1995 Linus Torvalds
*/
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/list.h>
#include <linux/highmem.h>
#include <linux/compiler.h>
#include <linux/uaccess.h>
#include <linux/gfp.h>
#include <linux/bitops.h>
#include <linux/hardirq.h> /* for in_interrupt() */
#include <linux/hugetlb_inline.h>
struct folio_batch;
unsigned long invalidate_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t end);
static inline void invalidate_remote_inode(struct inode *inode)
{
if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
S_ISLNK(inode->i_mode))
invalidate_mapping_pages(inode->i_mapping, 0, -1);
}
int invalidate_inode_pages2(struct address_space *mapping);
int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end);
int kiocb_invalidate_pages(struct kiocb *iocb, size_t count);
void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count);
int write_inode_now(struct inode *, int sync);
int filemap_fdatawrite(struct address_space *);
int filemap_flush(struct address_space *);
int filemap_fdatawait_keep_errors(struct address_space *mapping);
int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
loff_t start_byte, loff_t end_byte);
static inline int filemap_fdatawait(struct address_space *mapping)
{
return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
}
bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
int filemap_write_and_wait_range(struct address_space *mapping,
loff_t lstart, loff_t lend);
int __filemap_fdatawrite_range(struct address_space *mapping,
loff_t start, loff_t end, int sync_mode);
int filemap_fdatawrite_range(struct address_space *mapping,
loff_t start, loff_t end);
int filemap_check_errors(struct address_space *mapping);
void __filemap_set_wb_err(struct address_space *mapping, int err);
int filemap_fdatawrite_wbc(struct address_space *mapping,
struct writeback_control *wbc);
int kiocb_write_and_wait(struct kiocb *iocb, size_t count);
static inline int filemap_write_and_wait(struct address_space *mapping)
{
return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
}
/**
* filemap_set_wb_err - set a writeback error on an address_space
* @mapping: mapping in which to set writeback error
* @err: error to be set in mapping
*
* When writeback fails in some way, we must record that error so that
* userspace can be informed when fsync and the like are called. We endeavor
* to report errors on any file that was open at the time of the error. Some
* internal callers also need to know when writeback errors have occurred.
*
* When a writeback error occurs, most filesystems will want to call
* filemap_set_wb_err to record the error in the mapping so that it will be
* automatically reported whenever fsync is called on the file.
*/
static inline void filemap_set_wb_err(struct address_space *mapping, int err)
{
/* Fastpath for common case of no error */
if (unlikely(err))
__filemap_set_wb_err(mapping, err);
}
/**
* filemap_check_wb_err - has an error occurred since the mark was sampled?
* @mapping: mapping to check for writeback errors
* @since: previously-sampled errseq_t
*
* Grab the errseq_t value from the mapping, and see if it has changed "since"
* the given value was sampled.
*
* If it has then report the latest error set, otherwise return 0.
*/
static inline int filemap_check_wb_err(struct address_space *mapping,
errseq_t since)
{
return errseq_check(&mapping->wb_err, since);
}
/**
* filemap_sample_wb_err - sample the current errseq_t to test for later errors
* @mapping: mapping to be sampled
*
* Writeback errors are always reported relative to a particular sample point
* in the past. This function provides those sample points.
*/
static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
{
return errseq_sample(&mapping->wb_err);
}
/**
* file_sample_sb_err - sample the current errseq_t to test for later errors
* @file: file pointer to be sampled
*
* Grab the most current superblock-level errseq_t value for the given
* struct file.
*/
static inline errseq_t file_sample_sb_err(struct file *file)
{
return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
}
/*
* Flush file data before changing attributes. Caller must hold any locks
* required to prevent further writes to this file until we're done setting
* flags.
*/
static inline int inode_drain_writes(struct inode *inode)
{
inode_dio_wait(inode);
return filemap_write_and_wait(inode->i_mapping);
}
static inline bool mapping_empty(struct address_space *mapping)
{
return xa_empty(&mapping->i_pages);
}
/*
* mapping_shrinkable - test if page cache state allows inode reclaim
* @mapping: the page cache mapping
*
* This checks the mapping's cache state for the pupose of inode
* reclaim and LRU management.
*
* The caller is expected to hold the i_lock, but is not required to
* hold the i_pages lock, which usually protects cache state. That's
* because the i_lock and the list_lru lock that protect the inode and
* its LRU state don't nest inside the irq-safe i_pages lock.
*
* Cache deletions are performed under the i_lock, which ensures that
* when an inode goes empty, it will reliably get queued on the LRU.
*
* Cache additions do not acquire the i_lock and may race with this
* check, in which case we'll report the inode as shrinkable when it
* has cache pages. This is okay: the shrinker also checks the
* refcount and the referenced bit, which will be elevated or set in
* the process of adding new cache pages to an inode.
*/
static inline bool mapping_shrinkable(struct address_space *mapping)
{
void *head;
/*
* On highmem systems, there could be lowmem pressure from the
* inodes before there is highmem pressure from the page
* cache. Make inodes shrinkable regardless of cache state.
*/
if (IS_ENABLED(CONFIG_HIGHMEM))
return true;
/* Cache completely empty? Shrink away. */
head = rcu_access_pointer(mapping->i_pages.xa_head);
if (!head)
return true;
/*
* The xarray stores single offset-0 entries directly in the
* head pointer, which allows non-resident page cache entries
* to escape the shadow shrinker's list of xarray nodes. The
* inode shrinker needs to pick them up under memory pressure.
*/
if (!xa_is_node(head) && xa_is_value(head))
return true;
return false;
}
/*
* Bits in mapping->flags.
*/
enum mapping_flags {
AS_EIO = 0, /* IO error on async write */
AS_ENOSPC = 1, /* ENOSPC on async write */
AS_MM_ALL_LOCKS = 2, /* under mm_take_all_locks() */
AS_UNEVICTABLE = 3, /* e.g., ramdisk, SHM_LOCK */
AS_EXITING = 4, /* final truncate in progress */
/* writeback related tags are not used */
AS_NO_WRITEBACK_TAGS = 5,
AS_LARGE_FOLIO_SUPPORT = 6,
AS_RELEASE_ALWAYS, /* Call ->release_folio(), even if no private data */
AS_STABLE_WRITES, /* must wait for writeback before modifying
folio contents */
AS_UNMOVABLE, /* The mapping cannot be moved, ever */
};
/**
* mapping_set_error - record a writeback error in the address_space
* @mapping: the mapping in which an error should be set
* @error: the error to set in the mapping
*
* When writeback fails in some way, we must record that error so that
* userspace can be informed when fsync and the like are called. We endeavor
* to report errors on any file that was open at the time of the error. Some
* internal callers also need to know when writeback errors have occurred.
*
* When a writeback error occurs, most filesystems will want to call
* mapping_set_error to record the error in the mapping so that it can be
* reported when the application calls fsync(2).
*/
static inline void mapping_set_error(struct address_space *mapping, int error)
{
if (likely(!error))
return;
/* Record in wb_err for checkers using errseq_t based tracking */
__filemap_set_wb_err(mapping, error);
/* Record it in superblock */
if (mapping->host)
errseq_set(&mapping->host->i_sb->s_wb_err, error);
/* Record it in flags for now, for legacy callers */
if (error == -ENOSPC)
set_bit(AS_ENOSPC, &mapping->flags);
else
set_bit(AS_EIO, &mapping->flags);
}
static inline void mapping_set_unevictable(struct address_space *mapping)
{
set_bit(AS_UNEVICTABLE, &mapping->flags);
}
static inline void mapping_clear_unevictable(struct address_space *mapping)
{
clear_bit(AS_UNEVICTABLE, &mapping->flags);
}
static inline bool mapping_unevictable(struct address_space *mapping)
{
return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);
}
static inline void mapping_set_exiting(struct address_space *mapping)
{
set_bit(AS_EXITING, &mapping->flags);
}
static inline int mapping_exiting(struct address_space *mapping)
{
return test_bit(AS_EXITING, &mapping->flags);
}
static inline void mapping_set_no_writeback_tags(struct address_space *mapping)
{
set_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);
}
static inline int mapping_use_writeback_tags(struct address_space *mapping)
{
return !test_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);
}
static inline bool mapping_release_always(const struct address_space *mapping)
{
return test_bit(AS_RELEASE_ALWAYS, &mapping->flags);
}
static inline void mapping_set_release_always(struct address_space *mapping)
{
set_bit(AS_RELEASE_ALWAYS, &mapping->flags);
}
static inline void mapping_clear_release_always(struct address_space *mapping)
{
clear_bit(AS_RELEASE_ALWAYS, &mapping->flags);
}
static inline bool mapping_stable_writes(const struct address_space *mapping)
{
return test_bit(AS_STABLE_WRITES, &mapping->flags);
}
static inline void mapping_set_stable_writes(struct address_space *mapping)
{
set_bit(AS_STABLE_WRITES, &mapping->flags);
}
static inline void mapping_clear_stable_writes(struct address_space *mapping)
{
clear_bit(AS_STABLE_WRITES, &mapping->flags);
}
static inline void mapping_set_unmovable(struct address_space *mapping)
{
/*
* It's expected unmovable mappings are also unevictable. Compaction
* migrate scanner (isolate_migratepages_block()) relies on this to
* reduce page locking.
*/
set_bit(AS_UNEVICTABLE, &mapping->flags);
set_bit(AS_UNMOVABLE, &mapping->flags);
}
static inline bool mapping_unmovable(struct address_space *mapping)
{
return test_bit(AS_UNMOVABLE, &mapping->flags);
}
static inline gfp_t mapping_gfp_mask(struct address_space * mapping)
{
return mapping->gfp_mask;
}
/* Restricts the given gfp_mask to what the mapping allows. */
static inline gfp_t mapping_gfp_constraint(struct address_space *mapping,
gfp_t gfp_mask)
{
return mapping_gfp_mask(mapping) & gfp_mask;
}
/*
* This is non-atomic. Only to be used before the mapping is activated.
* Probably needs a barrier...
*/
static inline void mapping_set_gfp_mask(struct address_space *m, gfp_t mask)
{
m->gfp_mask = mask;
}
/**
* mapping_set_large_folios() - Indicate the file supports large folios.
* @mapping: The file.
*
* The filesystem should call this function in its inode constructor to
* indicate that the VFS can use large folios to cache the contents of
* the file.
*
* Context: This should not be called while the inode is active as it
* is non-atomic.
*/
static inline void mapping_set_large_folios(struct address_space *mapping)
{
__set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
}
/*
* Large folio support currently depends on THP. These dependencies are
* being worked on but are not yet fixed.
*/
static inline bool mapping_large_folio_support(struct address_space *mapping)
{
return IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);
}
static inline int filemap_nr_thps(struct address_space *mapping)
{
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
return atomic_read(&mapping->nr_thps);
#else
return 0;
#endif
}
static inline void filemap_nr_thps_inc(struct address_space *mapping)
{
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
if (!mapping_large_folio_support(mapping))
atomic_inc(&mapping->nr_thps);
#else
WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
#endif
}
static inline void filemap_nr_thps_dec(struct address_space *mapping)
{
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
if (!mapping_large_folio_support(mapping))
atomic_dec(&mapping->nr_thps);
#else
WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
#endif
}
struct address_space *page_mapping(struct page *);
struct address_space *folio_mapping(struct folio *);
struct address_space *swapcache_mapping(struct folio *);
/**
* folio_file_mapping - Find the mapping this folio belongs to.
* @folio: The folio.
*
* For folios which are in the page cache, return the mapping that this
* page belongs to. Folios in the swap cache return the mapping of the
* swap file or swap device where the data is stored. This is different
* from the mapping returned by folio_mapping(). The only reason to
* use it is if, like NFS, you return 0 from ->activate_swapfile.
*
* Do not call this for folios which aren't in the page cache or swap cache.
*/
static inline struct address_space *folio_file_mapping(struct folio *folio)
{
if (unlikely(folio_test_swapcache(folio)))
return swapcache_mapping(folio);
return folio->mapping;
}
/**
* folio_flush_mapping - Find the file mapping this folio belongs to.
* @folio: The folio.
*
* For folios which are in the page cache, return the mapping that this
* page belongs to. Anonymous folios return NULL, even if they're in
* the swap cache. Other kinds of folio also return NULL.
*
* This is ONLY used by architecture cache flushing code. If you aren't
* writing cache flushing code, you want either folio_mapping() or
* folio_file_mapping().
*/
static inline struct address_space *folio_flush_mapping(struct folio *folio)
{
if (unlikely(folio_test_swapcache(folio)))
return NULL;
return folio_mapping(folio);
}
static inline struct address_space *page_file_mapping(struct page *page)
{
return folio_file_mapping(page_folio(page));
}
/**
* folio_inode - Get the host inode for this folio.
* @folio: The folio.
*
* For folios which are in the page cache, return the inode that this folio
* belongs to.
*
* Do not call this for folios which aren't in the page cache.
*/
static inline struct inode *folio_inode(struct folio *folio)
{
return folio->mapping->host;
}
/**
* folio_attach_private - Attach private data to a folio.
* @folio: Folio to attach data to.
* @data: Data to attach to folio.
*
* Attaching private data to a folio increments the page's reference count.
* The data must be detached before the folio will be freed.
*/
static inline void folio_attach_private(struct folio *folio, void *data)
{
folio_get(folio);
folio->private = data;
folio_set_private(folio);
}
/**
* folio_change_private - Change private data on a folio.
* @folio: Folio to change the data on.
* @data: Data to set on the folio.
*
* Change the private data attached to a folio and return the old
* data. The page must previously have had data attached and the data
* must be detached before the folio will be freed.
*
* Return: Data that was previously attached to the folio.
*/
static inline void *folio_change_private(struct folio *folio, void *data)
{
void *old = folio_get_private(folio);
folio->private = data;
return old;
}
/**
* folio_detach_private - Detach private data from a folio.
* @folio: Folio to detach data from.
*
* Removes the data that was previously attached to the folio and decrements
* the refcount on the page.
*
* Return: Data that was attached to the folio.
*/
static inline void *folio_detach_private(struct folio *folio)
{
void *data = folio_get_private(folio);
if (!folio_test_private(folio))
return NULL;
folio_clear_private(folio);
folio->private = NULL;
folio_put(folio);
return data;
}
static inline void attach_page_private(struct page *page, void *data)
{
folio_attach_private(page_folio(page), data);
}
static inline void *detach_page_private(struct page *page)
{
return folio_detach_private(page_folio(page));
}
/*
* There are some parts of the kernel which assume that PMD entries
* are exactly HPAGE_PMD_ORDER. Those should be fixed, but until then,
* limit the maximum allocation order to PMD size. I'm not aware of any
* assumptions about maximum order if THP are disabled, but 8 seems like
* a good order (that's 1MB if you're using 4kB pages)
*/
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define MAX_PAGECACHE_ORDER HPAGE_PMD_ORDER
#else
#define MAX_PAGECACHE_ORDER 8
#endif
#ifdef CONFIG_NUMA
struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order);
#else
static inline struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order)
{
return folio_alloc(gfp, order);
}
#endif
static inline struct page *__page_cache_alloc(gfp_t gfp)
{
return &filemap_alloc_folio(gfp, 0)->page;
}
static inline struct page *page_cache_alloc(struct address_space *x)
{
return __page_cache_alloc(mapping_gfp_mask(x));
}
static inline gfp_t readahead_gfp_mask(struct address_space *x)
{
return mapping_gfp_mask(x) | __GFP_NORETRY | __GFP_NOWARN;
}
typedef int filler_t(struct file *, struct folio *);
pgoff_t page_cache_next_miss(struct address_space *mapping,
pgoff_t index, unsigned long max_scan);
pgoff_t page_cache_prev_miss(struct address_space *mapping,
pgoff_t index, unsigned long max_scan);
/**
* typedef fgf_t - Flags for getting folios from the page cache.
*
* Most users of the page cache will not need to use these flags;
* there are convenience functions such as filemap_get_folio() and
* filemap_lock_folio(). For users which need more control over exactly
* what is done with the folios, these flags to __filemap_get_folio()
* are available.
*
* * %FGP_ACCESSED - The folio will be marked accessed.
* * %FGP_LOCK - The folio is returned locked.
* * %FGP_CREAT - If no folio is present then a new folio is allocated,
* added to the page cache and the VM's LRU list. The folio is
* returned locked.
* * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
* folio is already in cache. If the folio was allocated, unlock it
* before returning so the caller can do the same dance.
* * %FGP_WRITE - The folio will be written to by the caller.
* * %FGP_NOFS - __GFP_FS will get cleared in gfp.
* * %FGP_NOWAIT - Don't block on the folio lock.
* * %FGP_STABLE - Wait for the folio to be stable (finished writeback)
* * %FGP_WRITEBEGIN - The flags to use in a filesystem write_begin()
* implementation.
*/
typedef unsigned int __bitwise fgf_t;
#define FGP_ACCESSED ((__force fgf_t)0x00000001)
#define FGP_LOCK ((__force fgf_t)0x00000002)
#define FGP_CREAT ((__force fgf_t)0x00000004)
#define FGP_WRITE ((__force fgf_t)0x00000008)
#define FGP_NOFS ((__force fgf_t)0x00000010)
#define FGP_NOWAIT ((__force fgf_t)0x00000020)
#define FGP_FOR_MMAP ((__force fgf_t)0x00000040)
#define FGP_STABLE ((__force fgf_t)0x00000080)
#define FGF_GET_ORDER(fgf) (((__force unsigned)fgf) >> 26) /* top 6 bits */
#define FGP_WRITEBEGIN (FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE)
/**
* fgf_set_order - Encode a length in the fgf_t flags.
* @size: The suggested size of the folio to create.
*
* The caller of __filemap_get_folio() can use this to suggest a preferred
* size for the folio that is created. If there is already a folio at
* the index, it will be returned, no matter what its size. If a folio
* is freshly created, it may be of a different size than requested
* due to alignment constraints, memory pressure, or the presence of
* other folios at nearby indices.
*/
static inline fgf_t fgf_set_order(size_t size)
{
unsigned int shift = ilog2(size);
if (shift <= PAGE_SHIFT)
return 0;
return (__force fgf_t)((shift - PAGE_SHIFT) << 26);
}
void *filemap_get_entry(struct address_space *mapping, pgoff_t index);
struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
fgf_t fgp_flags, gfp_t gfp);
struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
fgf_t fgp_flags, gfp_t gfp);
/**
* filemap_get_folio - Find and get a folio.
* @mapping: The address_space to search.
* @index: The page index.
*
* Looks up the page cache entry at @mapping & @index. If a folio is
* present, it is returned with an increased refcount.
*
* Return: A folio or ERR_PTR(-ENOENT) if there is no folio in the cache for
* this index. Will not return a shadow, swap or DAX entry.
*/
static inline struct folio *filemap_get_folio(struct address_space *mapping,
pgoff_t index)
{
return __filemap_get_folio(mapping, index, 0, 0);
}
/**
* filemap_lock_folio - Find and lock a folio.
* @mapping: The address_space to search.
* @index: The page index.
*
* Looks up the page cache entry at @mapping & @index. If a folio is
* present, it is returned locked with an increased refcount.
*
* Context: May sleep.
* Return: A folio or ERR_PTR(-ENOENT) if there is no folio in the cache for
* this index. Will not return a shadow, swap or DAX entry.
*/
static inline struct folio *filemap_lock_folio(struct address_space *mapping,
pgoff_t index)
{
return __filemap_get_folio(mapping, index, FGP_LOCK, 0);
}
/**
* filemap_grab_folio - grab a folio from the page cache
* @mapping: The address space to search
* @index: The page index
*
* Looks up the page cache entry at @mapping & @index. If no folio is found,
* a new folio is created. The folio is locked, marked as accessed, and
* returned.
*
* Return: A found or created folio. ERR_PTR(-ENOMEM) if no folio is found
* and failed to create a folio.
*/
static inline struct folio *filemap_grab_folio(struct address_space *mapping,
pgoff_t index)
{
return __filemap_get_folio(mapping, index,
FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
mapping_gfp_mask(mapping));
}
/**
* find_get_page - find and get a page reference
* @mapping: the address_space to search
* @offset: the page index
*
* Looks up the page cache slot at @mapping & @offset. If there is a
* page cache page, it is returned with an increased refcount.
*
* Otherwise, %NULL is returned.
*/
static inline struct page *find_get_page(struct address_space *mapping,
pgoff_t offset)
{
return pagecache_get_page(mapping, offset, 0, 0);
}
static inline struct page *find_get_page_flags(struct address_space *mapping,
pgoff_t offset, fgf_t fgp_flags)
{
return pagecache_get_page(mapping, offset, fgp_flags, 0);
}
/**
* find_lock_page - locate, pin and lock a pagecache page
* @mapping: the address_space to search
* @index: the page index
*
* Looks up the page cache entry at @mapping & @index. If there is a
* page cache page, it is returned locked and with an increased
* refcount.
*
* Context: May sleep.
* Return: A struct page or %NULL if there is no page in the cache for this
* index.
*/
static inline struct page *find_lock_page(struct address_space *mapping,
pgoff_t index)
{
return pagecache_get_page(mapping, index, FGP_LOCK, 0);
}
/**
* find_or_create_page - locate or add a pagecache page
* @mapping: the page's address_space
* @index: the page's index into the mapping
* @gfp_mask: page allocation mode
*
* Looks up the page cache slot at @mapping & @offset. If there is a
* page cache page, it is returned locked and with an increased
* refcount.
*
* If the page is not present, a new page is allocated using @gfp_mask
* and added to the page cache and the VM's LRU list. The page is
* returned locked and with an increased refcount.
*
* On memory exhaustion, %NULL is returned.
*
* find_or_create_page() may sleep, even if @gfp_flags specifies an
* atomic allocation!
*/
static inline struct page *find_or_create_page(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask)
{
return pagecache_get_page(mapping, index,
FGP_LOCK|FGP_ACCESSED|FGP_CREAT,
gfp_mask);
}
/**
* grab_cache_page_nowait - returns locked page at given index in given cache
* @mapping: target address_space
* @index: the page index
*
* Same as grab_cache_page(), but do not wait if the page is unavailable.
* This is intended for speculative data generators, where the data can
* be regenerated if the page couldn't be grabbed. This routine should
* be safe to call while holding the lock for another page.
*
* Clear __GFP_FS when allocating the page to avoid recursion into the fs
* and deadlock against the caller's locked page.
*/
static inline struct page *grab_cache_page_nowait(struct address_space *mapping,
pgoff_t index)
{
return pagecache_get_page(mapping, index,
FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
mapping_gfp_mask(mapping));
}
#define swapcache_index(folio) __page_file_index(&(folio)->page)
/**
* folio_index - File index of a folio.
* @folio: The folio.
*
* For a folio which is either in the page cache or the swap cache,
* return its index within the address_space it belongs to. If you know
* the page is definitely in the page cache, you can look at the folio's
* index directly.
*
* Return: The index (offset in units of pages) of a folio in its file.
*/
static inline pgoff_t folio_index(struct folio *folio)
{
if (unlikely(folio_test_swapcache(folio))) return swapcache_index(folio); return folio->index;
}
/**
* folio_next_index - Get the index of the next folio.
* @folio: The current folio.
*
* Return: The index of the folio which follows this folio in the file.
*/
static inline pgoff_t folio_next_index(struct folio *folio)
{
return folio->index + folio_nr_pages(folio);
}
/**
* folio_file_page - The page for a particular index.
* @folio: The folio which contains this index.
* @index: The index we want to look up.
*
* Sometimes after looking up a folio in the page cache, we need to
* obtain the specific page for an index (eg a page fault).
*
* Return: The page containing the file data for this index.
*/
static inline struct page *folio_file_page(struct folio *folio, pgoff_t index)
{
return folio_page(folio, index & (folio_nr_pages(folio) - 1));
}
/**
* folio_contains - Does this folio contain this index?
* @folio: The folio.
* @index: The page index within the file.
*
* Context: The caller should have the page locked in order to prevent
* (eg) shmem from moving the page between the page cache and swap cache
* and changing its index in the middle of the operation.
* Return: true or false.
*/
static inline bool folio_contains(struct folio *folio, pgoff_t index)
{
return index - folio_index(folio) < folio_nr_pages(folio);
}
/*
* Given the page we found in the page cache, return the page corresponding
* to this index in the file
*/
static inline struct page *find_subpage(struct page *head, pgoff_t index)
{
/* HugeTLBfs wants the head page regardless */
if (PageHuge(head))
return head;
return head + (index & (thp_nr_pages(head) - 1));
}
unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch);
unsigned filemap_get_folios_contig(struct address_space *mapping,
pgoff_t *start, pgoff_t end, struct folio_batch *fbatch);
unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,
pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch);
struct page *grab_cache_page_write_begin(struct address_space *mapping,
pgoff_t index);
/*
* Returns locked page at given index in given cache, creating it if needed.
*/
static inline struct page *grab_cache_page(struct address_space *mapping,
pgoff_t index)
{
return find_or_create_page(mapping, index, mapping_gfp_mask(mapping));
}
struct folio *read_cache_folio(struct address_space *, pgoff_t index,
filler_t *filler, struct file *file);
struct folio *mapping_read_folio_gfp(struct address_space *, pgoff_t index,
gfp_t flags);
struct page *read_cache_page(struct address_space *, pgoff_t index,
filler_t *filler, struct file *file);
extern struct page * read_cache_page_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask);
static inline struct page *read_mapping_page(struct address_space *mapping,
pgoff_t index, struct file *file)
{
return read_cache_page(mapping, index, NULL, file);
}
static inline struct folio *read_mapping_folio(struct address_space *mapping,
pgoff_t index, struct file *file)
{
return read_cache_folio(mapping, index, NULL, file);
}
/*
* Get the offset in PAGE_SIZE (even for hugetlb pages).
*/
static inline pgoff_t page_to_pgoff(struct page *page)
{
struct page *head;
if (likely(!PageTransTail(page)))
return page->index;
head = compound_head(page);
/*
* We don't initialize ->index for tail pages: calculate based on
* head page
*/
return head->index + page - head;
}
/*
* Return byte-offset into filesystem object for page.
*/
static inline loff_t page_offset(struct page *page)
{
return ((loff_t)page->index) << PAGE_SHIFT;
}
static inline loff_t page_file_offset(struct page *page)
{
return ((loff_t)page_index(page)) << PAGE_SHIFT;
}
/**
* folio_pos - Returns the byte position of this folio in its file.
* @folio: The folio.
*/
static inline loff_t folio_pos(struct folio *folio)
{
return page_offset(&folio->page);
}
/**
* folio_file_pos - Returns the byte position of this folio in its file.
* @folio: The folio.
*
* This differs from folio_pos() for folios which belong to a swap file.
* NFS is the only filesystem today which needs to use folio_file_pos().
*/
static inline loff_t folio_file_pos(struct folio *folio)
{
return page_file_offset(&folio->page);
}
/*
* Get the offset in PAGE_SIZE (even for hugetlb folios).
*/
static inline pgoff_t folio_pgoff(struct folio *folio)
{
return folio->index;
}
static inline pgoff_t linear_page_index(struct vm_area_struct *vma,
unsigned long address)
{
pgoff_t pgoff;
pgoff = (address - vma->vm_start) >> PAGE_SHIFT;
pgoff += vma->vm_pgoff;
return pgoff;
}
struct wait_page_key {
struct folio *folio;
int bit_nr;
int page_match;
};
struct wait_page_queue {
struct folio *folio;
int bit_nr;
wait_queue_entry_t wait;
};
static inline bool wake_page_match(struct wait_page_queue *wait_page,
struct wait_page_key *key)
{
if (wait_page->folio != key->folio)
return false;
key->page_match = 1;
if (wait_page->bit_nr != key->bit_nr)
return false;
return true;
}
void __folio_lock(struct folio *folio);
int __folio_lock_killable(struct folio *folio);
vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf);
void unlock_page(struct page *page);
void folio_unlock(struct folio *folio);
/**
* folio_trylock() - Attempt to lock a folio.
* @folio: The folio to attempt to lock.
*
* Sometimes it is undesirable to wait for a folio to be unlocked (eg
* when the locks are being taken in the wrong order, or if making
* progress through a batch of folios is more important than processing
* them in order). Usually folio_lock() is the correct function to call.
*
* Context: Any context.
* Return: Whether the lock was successfully acquired.
*/
static inline bool folio_trylock(struct folio *folio)
{
return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0)));
}
/*
* Return true if the page was successfully locked
*/
static inline int trylock_page(struct page *page)
{
return folio_trylock(page_folio(page));
}
/**
* folio_lock() - Lock this folio.
* @folio: The folio to lock.
*
* The folio lock protects against many things, probably more than it
* should. It is primarily held while a folio is being brought uptodate,
* either from its backing file or from swap. It is also held while a
* folio is being truncated from its address_space, so holding the lock
* is sufficient to keep folio->mapping stable.
*
* The folio lock is also held while write() is modifying the page to
* provide POSIX atomicity guarantees (as long as the write does not
* cross a page boundary). Other modifications to the data in the folio
* do not hold the folio lock and can race with writes, eg DMA and stores
* to mapped pages.
*
* Context: May sleep. If you need to acquire the locks of two or
* more folios, they must be in order of ascending index, if they are
* in the same address_space. If they are in different address_spaces,
* acquire the lock of the folio which belongs to the address_space which
* has the lowest address in memory first.
*/
static inline void folio_lock(struct folio *folio)
{
might_sleep();
if (!folio_trylock(folio))
__folio_lock(folio);
}
/**
* lock_page() - Lock the folio containing this page.
* @page: The page to lock.
*
* See folio_lock() for a description of what the lock protects.
* This is a legacy function and new code should probably use folio_lock()
* instead.
*
* Context: May sleep. Pages in the same folio share a lock, so do not
* attempt to lock two pages which share a folio.
*/
static inline void lock_page(struct page *page)
{
struct folio *folio;
might_sleep();
folio = page_folio(page);
if (!folio_trylock(folio))
__folio_lock(folio);
}
/**
* folio_lock_killable() - Lock this folio, interruptible by a fatal signal.
* @folio: The folio to lock.
*
* Attempts to lock the folio, like folio_lock(), except that the sleep
* to acquire the lock is interruptible by a fatal signal.
*
* Context: May sleep; see folio_lock().
* Return: 0 if the lock was acquired; -EINTR if a fatal signal was received.
*/
static inline int folio_lock_killable(struct folio *folio)
{
might_sleep();
if (!folio_trylock(folio))
return __folio_lock_killable(folio);
return 0;
}
/*
* folio_lock_or_retry - Lock the folio, unless this would block and the
* caller indicated that it can handle a retry.
*
* Return value and mmap_lock implications depend on flags; see
* __folio_lock_or_retry().
*/
static inline vm_fault_t folio_lock_or_retry(struct folio *folio,
struct vm_fault *vmf)
{
might_sleep();
if (!folio_trylock(folio))
return __folio_lock_or_retry(folio, vmf);
return 0;
}
/*
* This is exported only for folio_wait_locked/folio_wait_writeback, etc.,
* and should not be used directly.
*/
void folio_wait_bit(struct folio *folio, int bit_nr);
int folio_wait_bit_killable(struct folio *folio, int bit_nr);
/*
* Wait for a folio to be unlocked.
*
* This must be called with the caller "holding" the folio,
* ie with increased folio reference count so that the folio won't
* go away during the wait.
*/
static inline void folio_wait_locked(struct folio *folio)
{
if (folio_test_locked(folio))
folio_wait_bit(folio, PG_locked);
}
static inline int folio_wait_locked_killable(struct folio *folio)
{
if (!folio_test_locked(folio))
return 0;
return folio_wait_bit_killable(folio, PG_locked);
}
static inline void wait_on_page_locked(struct page *page)
{
folio_wait_locked(page_folio(page));
}
void folio_end_read(struct folio *folio, bool success);
void wait_on_page_writeback(struct page *page);
void folio_wait_writeback(struct folio *folio);
int folio_wait_writeback_killable(struct folio *folio);
void end_page_writeback(struct page *page);
void folio_end_writeback(struct folio *folio);
void wait_for_stable_page(struct page *page);
void folio_wait_stable(struct folio *folio);
void __folio_mark_dirty(struct folio *folio, struct address_space *, int warn);
static inline void __set_page_dirty(struct page *page,
struct address_space *mapping, int warn)
{
__folio_mark_dirty(page_folio(page), mapping, warn);
}
void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb);
void __folio_cancel_dirty(struct folio *folio);
static inline void folio_cancel_dirty(struct folio *folio)
{
/* Avoid atomic ops, locking, etc. when not actually needed. */
if (folio_test_dirty(folio)) __folio_cancel_dirty(folio);
}
bool folio_clear_dirty_for_io(struct folio *folio);
bool clear_page_dirty_for_io(struct page *page);
void folio_invalidate(struct folio *folio, size_t offset, size_t length);
int __set_page_dirty_nobuffers(struct page *page);
bool noop_dirty_folio(struct address_space *mapping, struct folio *folio);
#ifdef CONFIG_MIGRATION
int filemap_migrate_folio(struct address_space *mapping, struct folio *dst,
struct folio *src, enum migrate_mode mode);
#else
#define filemap_migrate_folio NULL
#endif
void folio_end_private_2(struct folio *folio);
void folio_wait_private_2(struct folio *folio);
int folio_wait_private_2_killable(struct folio *folio);
/*
* Add an arbitrary waiter to a page's wait queue
*/
void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter);
/*
* Fault in userspace address range.
*/
size_t fault_in_writeable(char __user *uaddr, size_t size);
size_t fault_in_subpage_writeable(char __user *uaddr, size_t size);
size_t fault_in_safe_writeable(const char __user *uaddr, size_t size);
size_t fault_in_readable(const char __user *uaddr, size_t size);
int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
pgoff_t index, gfp_t gfp);
int filemap_add_folio(struct address_space *mapping, struct folio *folio,
pgoff_t index, gfp_t gfp);
void filemap_remove_folio(struct folio *folio);
void __filemap_remove_folio(struct folio *folio, void *shadow);
void replace_page_cache_folio(struct folio *old, struct folio *new);
void delete_from_page_cache_batch(struct address_space *mapping,
struct folio_batch *fbatch);
bool filemap_release_folio(struct folio *folio, gfp_t gfp);
loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end,
int whence);
/* Must be non-static for BPF error injection */
int __filemap_add_folio(struct address_space *mapping, struct folio *folio,
pgoff_t index, gfp_t gfp, void **shadowp);
bool filemap_range_has_writeback(struct address_space *mapping,
loff_t start_byte, loff_t end_byte);
/**
* filemap_range_needs_writeback - check if range potentially needs writeback
* @mapping: address space within which to check
* @start_byte: offset in bytes where the range starts
* @end_byte: offset in bytes where the range ends (inclusive)
*
* Find at least one page in the range supplied, usually used to check if
* direct writing in this range will trigger a writeback. Used by O_DIRECT
* read/write with IOCB_NOWAIT, to see if the caller needs to do
* filemap_write_and_wait_range() before proceeding.
*
* Return: %true if the caller should do filemap_write_and_wait_range() before
* doing O_DIRECT to a page in this range, %false otherwise.
*/
static inline bool filemap_range_needs_writeback(struct address_space *mapping,
loff_t start_byte,
loff_t end_byte)
{
if (!mapping->nrpages)
return false;
if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
return false;
return filemap_range_has_writeback(mapping, start_byte, end_byte);
}
/**
* struct readahead_control - Describes a readahead request.
*
* A readahead request is for consecutive pages. Filesystems which
* implement the ->readahead method should call readahead_page() or
* readahead_page_batch() in a loop and attempt to start I/O against
* each page in the request.
*
* Most of the fields in this struct are private and should be accessed
* by the functions below.
*
* @file: The file, used primarily by network filesystems for authentication.
* May be NULL if invoked internally by the filesystem.
* @mapping: Readahead this filesystem object.
* @ra: File readahead state. May be NULL.
*/
struct readahead_control {
struct file *file;
struct address_space *mapping;
struct file_ra_state *ra;
/* private: use the readahead_* accessors instead */
pgoff_t _index;
unsigned int _nr_pages;
unsigned int _batch_count;
bool _workingset;
unsigned long _pflags;
};
#define DEFINE_READAHEAD(ractl, f, r, m, i) \
struct readahead_control ractl = { \
.file = f, \
.mapping = m, \
.ra = r, \
._index = i, \
}
#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE)
void page_cache_ra_unbounded(struct readahead_control *,
unsigned long nr_to_read, unsigned long lookahead_count);
void page_cache_sync_ra(struct readahead_control *, unsigned long req_count);
void page_cache_async_ra(struct readahead_control *, struct folio *,
unsigned long req_count);
void readahead_expand(struct readahead_control *ractl,
loff_t new_start, size_t new_len);
/**
* page_cache_sync_readahead - generic file readahead
* @mapping: address_space which holds the pagecache and I/O vectors
* @ra: file_ra_state which holds the readahead state
* @file: Used by the filesystem for authentication.
* @index: Index of first page to be read.
* @req_count: Total number of pages being read by the caller.
*
* page_cache_sync_readahead() should be called when a cache miss happened:
* it will submit the read. The readahead logic may decide to piggyback more
* pages onto the read request if access patterns suggest it will improve
* performance.
*/
static inline
void page_cache_sync_readahead(struct address_space *mapping,
struct file_ra_state *ra, struct file *file, pgoff_t index,
unsigned long req_count)
{
DEFINE_READAHEAD(ractl, file, ra, mapping, index);
page_cache_sync_ra(&ractl, req_count);
}
/**
* page_cache_async_readahead - file readahead for marked pages
* @mapping: address_space which holds the pagecache and I/O vectors
* @ra: file_ra_state which holds the readahead state
* @file: Used by the filesystem for authentication.
* @folio: The folio at @index which triggered the readahead call.
* @index: Index of first page to be read.
* @req_count: Total number of pages being read by the caller.
*
* page_cache_async_readahead() should be called when a page is used which
* is marked as PageReadahead; this is a marker to suggest that the application
* has used up enough of the readahead window that we should start pulling in
* more pages.
*/
static inline
void page_cache_async_readahead(struct address_space *mapping,
struct file_ra_state *ra, struct file *file,
struct folio *folio, pgoff_t index, unsigned long req_count)
{
DEFINE_READAHEAD(ractl, file, ra, mapping, index);
page_cache_async_ra(&ractl, folio, req_count);
}
static inline struct folio *__readahead_folio(struct readahead_control *ractl)
{
struct folio *folio;
BUG_ON(ractl->_batch_count > ractl->_nr_pages);
ractl->_nr_pages -= ractl->_batch_count;
ractl->_index += ractl->_batch_count;
if (!ractl->_nr_pages) {
ractl->_batch_count = 0;
return NULL;
}
folio = xa_load(&ractl->mapping->i_pages, ractl->_index);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
ractl->_batch_count = folio_nr_pages(folio);
return folio;
}
/**
* readahead_page - Get the next page to read.
* @ractl: The current readahead request.
*
* Context: The page is locked and has an elevated refcount. The caller
* should decreases the refcount once the page has been submitted for I/O
* and unlock the page once all I/O to that page has completed.
* Return: A pointer to the next page, or %NULL if we are done.
*/
static inline struct page *readahead_page(struct readahead_control *ractl)
{
struct folio *folio = __readahead_folio(ractl);
return &folio->page;
}
/**
* readahead_folio - Get the next folio to read.
* @ractl: The current readahead request.
*
* Context: The folio is locked. The caller should unlock the folio once
* all I/O to that folio has completed.
* Return: A pointer to the next folio, or %NULL if we are done.
*/
static inline struct folio *readahead_folio(struct readahead_control *ractl)
{
struct folio *folio = __readahead_folio(ractl);
if (folio)
folio_put(folio);
return folio;
}
static inline unsigned int __readahead_batch(struct readahead_control *rac,
struct page **array, unsigned int array_sz)
{
unsigned int i = 0;
XA_STATE(xas, &rac->mapping->i_pages, 0);
struct page *page;
BUG_ON(rac->_batch_count > rac->_nr_pages);
rac->_nr_pages -= rac->_batch_count;
rac->_index += rac->_batch_count;
rac->_batch_count = 0;
xas_set(&xas, rac->_index);
rcu_read_lock();
xas_for_each(&xas, page, rac->_index + rac->_nr_pages - 1) {
if (xas_retry(&xas, page))
continue;
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(PageTail(page), page);
array[i++] = page;
rac->_batch_count += thp_nr_pages(page);
if (i == array_sz)
break;
}
rcu_read_unlock();
return i;
}
/**
* readahead_page_batch - Get a batch of pages to read.
* @rac: The current readahead request.
* @array: An array of pointers to struct page.
*
* Context: The pages are locked and have an elevated refcount. The caller
* should decreases the refcount once the page has been submitted for I/O
* and unlock the page once all I/O to that page has completed.
* Return: The number of pages placed in the array. 0 indicates the request
* is complete.
*/
#define readahead_page_batch(rac, array) \
__readahead_batch(rac, array, ARRAY_SIZE(array))
/**
* readahead_pos - The byte offset into the file of this readahead request.
* @rac: The readahead request.
*/
static inline loff_t readahead_pos(struct readahead_control *rac)
{
return (loff_t)rac->_index * PAGE_SIZE;
}
/**
* readahead_length - The number of bytes in this readahead request.
* @rac: The readahead request.
*/
static inline size_t readahead_length(struct readahead_control *rac)
{
return rac->_nr_pages * PAGE_SIZE;
}
/**
* readahead_index - The index of the first page in this readahead request.
* @rac: The readahead request.
*/
static inline pgoff_t readahead_index(struct readahead_control *rac)
{
return rac->_index;
}
/**
* readahead_count - The number of pages in this readahead request.
* @rac: The readahead request.
*/
static inline unsigned int readahead_count(struct readahead_control *rac)
{
return rac->_nr_pages;
}
/**
* readahead_batch_length - The number of bytes in the current batch.
* @rac: The readahead request.
*/
static inline size_t readahead_batch_length(struct readahead_control *rac)
{
return rac->_batch_count * PAGE_SIZE;
}
static inline unsigned long dir_pages(struct inode *inode)
{
return (unsigned long)(inode->i_size + PAGE_SIZE - 1) >>
PAGE_SHIFT;
}
/**
* folio_mkwrite_check_truncate - check if folio was truncated
* @folio: the folio to check
* @inode: the inode to check the folio against
*
* Return: the number of bytes in the folio up to EOF,
* or -EFAULT if the folio was truncated.
*/
static inline ssize_t folio_mkwrite_check_truncate(struct folio *folio,
struct inode *inode)
{
loff_t size = i_size_read(inode);
pgoff_t index = size >> PAGE_SHIFT;
size_t offset = offset_in_folio(folio, size);
if (!folio->mapping)
return -EFAULT;
/* folio is wholly inside EOF */
if (folio_next_index(folio) - 1 < index)
return folio_size(folio);
/* folio is wholly past EOF */
if (folio->index > index || !offset)
return -EFAULT;
/* folio is partially inside EOF */
return offset;
}
/**
* page_mkwrite_check_truncate - check if page was truncated
* @page: the page to check
* @inode: the inode to check the page against
*
* Returns the number of bytes in the page up to EOF,
* or -EFAULT if the page was truncated.
*/
static inline int page_mkwrite_check_truncate(struct page *page,
struct inode *inode)
{
loff_t size = i_size_read(inode);
pgoff_t index = size >> PAGE_SHIFT;
int offset = offset_in_page(size);
if (page->mapping != inode->i_mapping)
return -EFAULT;
/* page is wholly inside EOF */
if (page->index < index)
return PAGE_SIZE;
/* page is wholly past EOF */
if (page->index > index || !offset)
return -EFAULT;
/* page is partially inside EOF */
return offset;
}
/**
* i_blocks_per_folio - How many blocks fit in this folio.
* @inode: The inode which contains the blocks.
* @folio: The folio.
*
* If the block size is larger than the size of this folio, return zero.
*
* Context: The caller should hold a refcount on the folio to prevent it
* from being split.
* Return: The number of filesystem blocks covered by this folio.
*/
static inline
unsigned int i_blocks_per_folio(struct inode *inode, struct folio *folio)
{
return folio_size(folio) >> inode->i_blkbits;
}
static inline
unsigned int i_blocks_per_page(struct inode *inode, struct page *page)
{
return i_blocks_per_folio(inode, page_folio(page));
}
#endif /* _LINUX_PAGEMAP_H */
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
* Copyright (C) 2005-2006, Thomas Gleixner, Russell King
*
* This file contains the core interrupt handling code. Detailed
* information is available in Documentation/core-api/genericirq.rst
*
*/
#include <linux/irq.h>
#include <linux/random.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <asm/irq_regs.h>
#include <trace/events/irq.h>
#include "internals.h"
#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
void (*handle_arch_irq)(struct pt_regs *) __ro_after_init;
#endif
/**
* handle_bad_irq - handle spurious and unhandled irqs
* @desc: description of the interrupt
*
* Handles spurious and unhandled IRQ's. It also prints a debugmessage.
*/
void handle_bad_irq(struct irq_desc *desc)
{
unsigned int irq = irq_desc_get_irq(desc);
print_irq_desc(irq, desc);
kstat_incr_irqs_this_cpu(desc);
ack_bad_irq(irq);
}
EXPORT_SYMBOL_GPL(handle_bad_irq);
/*
* Special, empty irq handler:
*/
irqreturn_t no_action(int cpl, void *dev_id)
{
return IRQ_NONE;
}
EXPORT_SYMBOL_GPL(no_action);
static void warn_no_thread(unsigned int irq, struct irqaction *action)
{
if (test_and_set_bit(IRQTF_WARNED, &action->thread_flags))
return;
printk(KERN_WARNING "IRQ %d device %s returned IRQ_WAKE_THREAD "
"but no thread function available.", irq, action->name);
}
void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action)
{
/*
* In case the thread crashed and was killed we just pretend that
* we handled the interrupt. The hardirq handler has disabled the
* device interrupt, so no irq storm is lurking.
*/
if (action->thread->flags & PF_EXITING)
return;
/*
* Wake up the handler thread for this action. If the
* RUNTHREAD bit is already set, nothing to do.
*/
if (test_and_set_bit(IRQTF_RUNTHREAD, &action->thread_flags))
return;
/*
* It's safe to OR the mask lockless here. We have only two
* places which write to threads_oneshot: This code and the
* irq thread.
*
* This code is the hard irq context and can never run on two
* cpus in parallel. If it ever does we have more serious
* problems than this bitmask.
*
* The irq threads of this irq which clear their "running" bit
* in threads_oneshot are serialized via desc->lock against
* each other and they are serialized against this code by
* IRQS_INPROGRESS.
*
* Hard irq handler:
*
* spin_lock(desc->lock);
* desc->state |= IRQS_INPROGRESS;
* spin_unlock(desc->lock);
* set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
* desc->threads_oneshot |= mask;
* spin_lock(desc->lock);
* desc->state &= ~IRQS_INPROGRESS;
* spin_unlock(desc->lock);
*
* irq thread:
*
* again:
* spin_lock(desc->lock);
* if (desc->state & IRQS_INPROGRESS) {
* spin_unlock(desc->lock);
* while(desc->state & IRQS_INPROGRESS)
* cpu_relax();
* goto again;
* }
* if (!test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
* desc->threads_oneshot &= ~mask;
* spin_unlock(desc->lock);
*
* So either the thread waits for us to clear IRQS_INPROGRESS
* or we are waiting in the flow handler for desc->lock to be
* released before we reach this point. The thread also checks
* IRQTF_RUNTHREAD under desc->lock. If set it leaves
* threads_oneshot untouched and runs the thread another time.
*/
desc->threads_oneshot |= action->thread_mask;
/*
* We increment the threads_active counter in case we wake up
* the irq thread. The irq thread decrements the counter when
* it returns from the handler or in the exit path and wakes
* up waiters which are stuck in synchronize_irq() when the
* active count becomes zero. synchronize_irq() is serialized
* against this code (hard irq handler) via IRQS_INPROGRESS
* like the finalize_oneshot() code. See comment above.
*/
atomic_inc(&desc->threads_active);
wake_up_process(action->thread);
}
irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc)
{
irqreturn_t retval = IRQ_NONE;
unsigned int irq = desc->irq_data.irq;
struct irqaction *action;
record_irq_time(desc);
for_each_action_of_desc(desc, action) {
irqreturn_t res;
/*
* If this IRQ would be threaded under force_irqthreads, mark it so.
*/
if (irq_settings_can_thread(desc) && !(action->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))) lockdep_hardirq_threaded(); trace_irq_handler_entry(irq, action); res = action->handler(irq, action->dev_id); trace_irq_handler_exit(irq, action, res); if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n",
irq, action->handler))
local_irq_disable(); switch (res) {
case IRQ_WAKE_THREAD:
/*
* Catch drivers which return WAKE_THREAD but
* did not set up a thread function
*/
if (unlikely(!action->thread_fn)) { warn_no_thread(irq, action);
break;
}
__irq_wake_thread(desc, action);
break;
default:
break;
}
retval |= res;
}
return retval;
}
irqreturn_t handle_irq_event_percpu(struct irq_desc *desc)
{
irqreturn_t retval;
retval = __handle_irq_event_percpu(desc);
add_interrupt_randomness(desc->irq_data.irq);
if (!irq_settings_no_debug(desc))
note_interrupt(desc, retval);
return retval;
}
irqreturn_t handle_irq_event(struct irq_desc *desc)
{
irqreturn_t ret;
desc->istate &= ~IRQS_PENDING;
irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
raw_spin_unlock(&desc->lock);
ret = handle_irq_event_percpu(desc); raw_spin_lock(&desc->lock);
irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
return ret;
}
#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
int __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
{
if (handle_arch_irq)
return -EBUSY;
handle_arch_irq = handle_irq;
return 0;
}
/**
* generic_handle_arch_irq - root irq handler for architectures which do no
* entry accounting themselves
* @regs: Register file coming from the low-level handling code
*/
asmlinkage void noinstr generic_handle_arch_irq(struct pt_regs *regs)
{
struct pt_regs *old_regs;
irq_enter();
old_regs = set_irq_regs(regs);
handle_arch_irq(regs);
set_irq_regs(old_regs);
irq_exit();
}
#endif
// SPDX-License-Identifier: GPL-2.0-only
/*
* fs/fs-writeback.c
*
* Copyright (C) 2002, Linus Torvalds.
*
* Contains all the functions related to writing back and waiting
* upon dirty inodes against superblocks, and writing back dirty
* pages against inodes. ie: data writeback. Writeout of the
* inode itself is not handled here.
*
* 10Apr2002 Andrew Morton
* Split out of fs/inode.c
* Additions for address_space-based writeback
*/
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/kthread.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/tracepoint.h>
#include <linux/device.h>
#include <linux/memcontrol.h>
#include "internal.h"
/*
* 4MB minimal write chunk size
*/
#define MIN_WRITEBACK_PAGES (4096UL >> (PAGE_SHIFT - 10))
/*
* Passed into wb_writeback(), essentially a subset of writeback_control
*/
struct wb_writeback_work {
long nr_pages;
struct super_block *sb;
enum writeback_sync_modes sync_mode;
unsigned int tagged_writepages:1;
unsigned int for_kupdate:1;
unsigned int range_cyclic:1;
unsigned int for_background:1;
unsigned int for_sync:1; /* sync(2) WB_SYNC_ALL writeback */
unsigned int auto_free:1; /* free on completion */
enum wb_reason reason; /* why was writeback initiated? */
struct list_head list; /* pending work list */
struct wb_completion *done; /* set if the caller waits */
};
/*
* If an inode is constantly having its pages dirtied, but then the
* updates stop dirtytime_expire_interval seconds in the past, it's
* possible for the worst case time between when an inode has its
* timestamps updated and when they finally get written out to be two
* dirtytime_expire_intervals. We set the default to 12 hours (in
* seconds), which means most of the time inodes will have their
* timestamps written to disk after 12 hours, but in the worst case a
* few inodes might not their timestamps updated for 24 hours.
*/
unsigned int dirtytime_expire_interval = 12 * 60 * 60;
static inline struct inode *wb_inode(struct list_head *head)
{
return list_entry(head, struct inode, i_io_list);
}
/*
* Include the creation of the trace points after defining the
* wb_writeback_work structure and inline functions so that the definition
* remains local to this file.
*/
#define CREATE_TRACE_POINTS
#include <trace/events/writeback.h>
EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage);
static bool wb_io_lists_populated(struct bdi_writeback *wb)
{
if (wb_has_dirty_io(wb)) {
return false;
} else {
set_bit(WB_has_dirty_io, &wb->state);
WARN_ON_ONCE(!wb->avg_write_bandwidth);
atomic_long_add(wb->avg_write_bandwidth,
&wb->bdi->tot_write_bandwidth);
return true;
}
}
static void wb_io_lists_depopulated(struct bdi_writeback *wb)
{
if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&
list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {
clear_bit(WB_has_dirty_io, &wb->state);
WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,
&wb->bdi->tot_write_bandwidth) < 0);
}
}
/**
* inode_io_list_move_locked - move an inode onto a bdi_writeback IO list
* @inode: inode to be moved
* @wb: target bdi_writeback
* @head: one of @wb->b_{dirty|io|more_io|dirty_time}
*
* Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io.
* Returns %true if @inode is the first occupant of the !dirty_time IO
* lists; otherwise, %false.
*/
static bool inode_io_list_move_locked(struct inode *inode,
struct bdi_writeback *wb,
struct list_head *head)
{
assert_spin_locked(&wb->list_lock);
assert_spin_locked(&inode->i_lock);
WARN_ON_ONCE(inode->i_state & I_FREEING);
list_move(&inode->i_io_list, head);
/* dirty_time doesn't count as dirty_io until expiration */
if (head != &wb->b_dirty_time)
return wb_io_lists_populated(wb);
wb_io_lists_depopulated(wb);
return false;
}
static void wb_wakeup(struct bdi_writeback *wb)
{
spin_lock_irq(&wb->work_lock);
if (test_bit(WB_registered, &wb->state))
mod_delayed_work(bdi_wq, &wb->dwork, 0);
spin_unlock_irq(&wb->work_lock);
}
/*
* This function is used when the first inode for this wb is marked dirty. It
* wakes-up the corresponding bdi thread which should then take care of the
* periodic background write-out of dirty inodes. Since the write-out would
* starts only 'dirty_writeback_interval' centisecs from now anyway, we just
* set up a timer which wakes the bdi thread up later.
*
* Note, we wouldn't bother setting up the timer, but this function is on the
* fast-path (used by '__mark_inode_dirty()'), so we save few context switches
* by delaying the wake-up.
*
* We have to be careful not to postpone flush work if it is scheduled for
* earlier. Thus we use queue_delayed_work().
*/
static void wb_wakeup_delayed(struct bdi_writeback *wb)
{
unsigned long timeout;
timeout = msecs_to_jiffies(dirty_writeback_interval * 10);
spin_lock_irq(&wb->work_lock);
if (test_bit(WB_registered, &wb->state))
queue_delayed_work(bdi_wq, &wb->dwork, timeout);
spin_unlock_irq(&wb->work_lock);
}
static void finish_writeback_work(struct bdi_writeback *wb,
struct wb_writeback_work *work)
{
struct wb_completion *done = work->done;
if (work->auto_free)
kfree(work);
if (done) {
wait_queue_head_t *waitq = done->waitq;
/* @done can't be accessed after the following dec */
if (atomic_dec_and_test(&done->cnt))
wake_up_all(waitq);
}
}
static void wb_queue_work(struct bdi_writeback *wb,
struct wb_writeback_work *work)
{
trace_writeback_queue(wb, work);
if (work->done)
atomic_inc(&work->done->cnt);
spin_lock_irq(&wb->work_lock);
if (test_bit(WB_registered, &wb->state)) {
list_add_tail(&work->list, &wb->work_list);
mod_delayed_work(bdi_wq, &wb->dwork, 0);
} else
finish_writeback_work(wb, work);
spin_unlock_irq(&wb->work_lock);
}
/**
* wb_wait_for_completion - wait for completion of bdi_writeback_works
* @done: target wb_completion
*
* Wait for one or more work items issued to @bdi with their ->done field
* set to @done, which should have been initialized with
* DEFINE_WB_COMPLETION(). This function returns after all such work items
* are completed. Work items which are waited upon aren't freed
* automatically on completion.
*/
void wb_wait_for_completion(struct wb_completion *done)
{
atomic_dec(&done->cnt); /* put down the initial count */
wait_event(*done->waitq, !atomic_read(&done->cnt));
}
#ifdef CONFIG_CGROUP_WRITEBACK
/*
* Parameters for foreign inode detection, see wbc_detach_inode() to see
* how they're used.
*
* These paramters are inherently heuristical as the detection target
* itself is fuzzy. All we want to do is detaching an inode from the
* current owner if it's being written to by some other cgroups too much.
*
* The current cgroup writeback is built on the assumption that multiple
* cgroups writing to the same inode concurrently is very rare and a mode
* of operation which isn't well supported. As such, the goal is not
* taking too long when a different cgroup takes over an inode while
* avoiding too aggressive flip-flops from occasional foreign writes.
*
* We record, very roughly, 2s worth of IO time history and if more than
* half of that is foreign, trigger the switch. The recording is quantized
* to 16 slots. To avoid tiny writes from swinging the decision too much,
* writes smaller than 1/8 of avg size are ignored.
*/
#define WB_FRN_TIME_SHIFT 13 /* 1s = 2^13, upto 8 secs w/ 16bit */
#define WB_FRN_TIME_AVG_SHIFT 3 /* avg = avg * 7/8 + new * 1/8 */
#define WB_FRN_TIME_CUT_DIV 8 /* ignore rounds < avg / 8 */
#define WB_FRN_TIME_PERIOD (2 * (1 << WB_FRN_TIME_SHIFT)) /* 2s */
#define WB_FRN_HIST_SLOTS 16 /* inode->i_wb_frn_history is 16bit */
#define WB_FRN_HIST_UNIT (WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS)
/* each slot's duration is 2s / 16 */
#define WB_FRN_HIST_THR_SLOTS (WB_FRN_HIST_SLOTS / 2)
/* if foreign slots >= 8, switch */
#define WB_FRN_HIST_MAX_SLOTS (WB_FRN_HIST_THR_SLOTS / 2 + 1)
/* one round can affect upto 5 slots */
#define WB_FRN_MAX_IN_FLIGHT 1024 /* don't queue too many concurrently */
/*
* Maximum inodes per isw. A specific value has been chosen to make
* struct inode_switch_wbs_context fit into 1024 bytes kmalloc.
*/
#define WB_MAX_INODES_PER_ISW ((1024UL - sizeof(struct inode_switch_wbs_context)) \
/ sizeof(struct inode *))
static atomic_t isw_nr_in_flight = ATOMIC_INIT(0);
static struct workqueue_struct *isw_wq;
void __inode_attach_wb(struct inode *inode, struct folio *folio)
{
struct backing_dev_info *bdi = inode_to_bdi(inode);
struct bdi_writeback *wb = NULL;
if (inode_cgwb_enabled(inode)) {
struct cgroup_subsys_state *memcg_css;
if (folio) { memcg_css = mem_cgroup_css_from_folio(folio);
wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
} else {
/* must pin memcg_css, see wb_get_create() */
memcg_css = task_get_css(current, memory_cgrp_id);
wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
css_put(memcg_css);
}
}
if (!wb) wb = &bdi->wb;
/*
* There may be multiple instances of this function racing to
* update the same inode. Use cmpxchg() to tell the winner.
*/
if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) wb_put(wb);
}
EXPORT_SYMBOL_GPL(__inode_attach_wb);
/**
* inode_cgwb_move_to_attached - put the inode onto wb->b_attached list
* @inode: inode of interest with i_lock held
* @wb: target bdi_writeback
*
* Remove the inode from wb's io lists and if necessarily put onto b_attached
* list. Only inodes attached to cgwb's are kept on this list.
*/
static void inode_cgwb_move_to_attached(struct inode *inode,
struct bdi_writeback *wb)
{
assert_spin_locked(&wb->list_lock);
assert_spin_locked(&inode->i_lock);
WARN_ON_ONCE(inode->i_state & I_FREEING);
inode->i_state &= ~I_SYNC_QUEUED;
if (wb != &wb->bdi->wb)
list_move(&inode->i_io_list, &wb->b_attached);
else
list_del_init(&inode->i_io_list);
wb_io_lists_depopulated(wb);
}
/**
* locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it
* @inode: inode of interest with i_lock held
*
* Returns @inode's wb with its list_lock held. @inode->i_lock must be
* held on entry and is released on return. The returned wb is guaranteed
* to stay @inode's associated wb until its list_lock is released.
*/
static struct bdi_writeback *
locked_inode_to_wb_and_lock_list(struct inode *inode)
__releases(&inode->i_lock)
__acquires(&wb->list_lock)
{
while (true) {
struct bdi_writeback *wb = inode_to_wb(inode);
/*
* inode_to_wb() association is protected by both
* @inode->i_lock and @wb->list_lock but list_lock nests
* outside i_lock. Drop i_lock and verify that the
* association hasn't changed after acquiring list_lock.
*/
wb_get(wb); spin_unlock(&inode->i_lock);
spin_lock(&wb->list_lock);
/* i_wb may have changed inbetween, can't use inode_to_wb() */
if (likely(wb == inode->i_wb)) {
wb_put(wb); /* @inode already has ref */ return wb;
}
spin_unlock(&wb->list_lock); wb_put(wb); cpu_relax();
spin_lock(&inode->i_lock);
}
}
/**
* inode_to_wb_and_lock_list - determine an inode's wb and lock it
* @inode: inode of interest
*
* Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held
* on entry.
*/
static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
__acquires(&wb->list_lock)
{
spin_lock(&inode->i_lock);
return locked_inode_to_wb_and_lock_list(inode);
}
struct inode_switch_wbs_context {
struct rcu_work work;
/*
* Multiple inodes can be switched at once. The switching procedure
* consists of two parts, separated by a RCU grace period. To make
* sure that the second part is executed for each inode gone through
* the first part, all inode pointers are placed into a NULL-terminated
* array embedded into struct inode_switch_wbs_context. Otherwise
* an inode could be left in a non-consistent state.
*/
struct bdi_writeback *new_wb;
struct inode *inodes[];
};
static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi)
{
down_write(&bdi->wb_switch_rwsem);
}
static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi)
{
up_write(&bdi->wb_switch_rwsem);
}
static bool inode_do_switch_wbs(struct inode *inode,
struct bdi_writeback *old_wb,
struct bdi_writeback *new_wb)
{
struct address_space *mapping = inode->i_mapping;
XA_STATE(xas, &mapping->i_pages, 0);
struct folio *folio;
bool switched = false;
spin_lock(&inode->i_lock);
xa_lock_irq(&mapping->i_pages);
/*
* Once I_FREEING or I_WILL_FREE are visible under i_lock, the eviction
* path owns the inode and we shouldn't modify ->i_io_list.
*/
if (unlikely(inode->i_state & (I_FREEING | I_WILL_FREE)))
goto skip_switch;
trace_inode_switch_wbs(inode, old_wb, new_wb);
/*
* Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points
* to possibly dirty folios while PAGECACHE_TAG_WRITEBACK points to
* folios actually under writeback.
*/
xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_DIRTY) {
if (folio_test_dirty(folio)) {
long nr = folio_nr_pages(folio);
wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr);
wb_stat_mod(new_wb, WB_RECLAIMABLE, nr);
}
}
xas_set(&xas, 0);
xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) {
long nr = folio_nr_pages(folio);
WARN_ON_ONCE(!folio_test_writeback(folio));
wb_stat_mod(old_wb, WB_WRITEBACK, -nr);
wb_stat_mod(new_wb, WB_WRITEBACK, nr);
}
if (mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) {
atomic_dec(&old_wb->writeback_inodes);
atomic_inc(&new_wb->writeback_inodes);
}
wb_get(new_wb);
/*
* Transfer to @new_wb's IO list if necessary. If the @inode is dirty,
* the specific list @inode was on is ignored and the @inode is put on
* ->b_dirty which is always correct including from ->b_dirty_time.
* The transfer preserves @inode->dirtied_when ordering. If the @inode
* was clean, it means it was on the b_attached list, so move it onto
* the b_attached list of @new_wb.
*/
if (!list_empty(&inode->i_io_list)) {
inode->i_wb = new_wb;
if (inode->i_state & I_DIRTY_ALL) {
struct inode *pos;
list_for_each_entry(pos, &new_wb->b_dirty, i_io_list)
if (time_after_eq(inode->dirtied_when,
pos->dirtied_when))
break;
inode_io_list_move_locked(inode, new_wb,
pos->i_io_list.prev);
} else {
inode_cgwb_move_to_attached(inode, new_wb);
}
} else {
inode->i_wb = new_wb;
}
/* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */
inode->i_wb_frn_winner = 0;
inode->i_wb_frn_avg_time = 0;
inode->i_wb_frn_history = 0;
switched = true;
skip_switch:
/*
* Paired with load_acquire in unlocked_inode_to_wb_begin() and
* ensures that the new wb is visible if they see !I_WB_SWITCH.
*/
smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH);
xa_unlock_irq(&mapping->i_pages);
spin_unlock(&inode->i_lock);
return switched;
}
static void inode_switch_wbs_work_fn(struct work_struct *work)
{
struct inode_switch_wbs_context *isw =
container_of(to_rcu_work(work), struct inode_switch_wbs_context, work);
struct backing_dev_info *bdi = inode_to_bdi(isw->inodes[0]);
struct bdi_writeback *old_wb = isw->inodes[0]->i_wb;
struct bdi_writeback *new_wb = isw->new_wb;
unsigned long nr_switched = 0;
struct inode **inodep;
/*
* If @inode switches cgwb membership while sync_inodes_sb() is
* being issued, sync_inodes_sb() might miss it. Synchronize.
*/
down_read(&bdi->wb_switch_rwsem);
/*
* By the time control reaches here, RCU grace period has passed
* since I_WB_SWITCH assertion and all wb stat update transactions
* between unlocked_inode_to_wb_begin/end() are guaranteed to be
* synchronizing against the i_pages lock.
*
* Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock
* gives us exclusion against all wb related operations on @inode
* including IO list manipulations and stat updates.
*/
if (old_wb < new_wb) {
spin_lock(&old_wb->list_lock);
spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING);
} else {
spin_lock(&new_wb->list_lock);
spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING);
}
for (inodep = isw->inodes; *inodep; inodep++) {
WARN_ON_ONCE((*inodep)->i_wb != old_wb);
if (inode_do_switch_wbs(*inodep, old_wb, new_wb))
nr_switched++;
}
spin_unlock(&new_wb->list_lock);
spin_unlock(&old_wb->list_lock);
up_read(&bdi->wb_switch_rwsem);
if (nr_switched) {
wb_wakeup(new_wb);
wb_put_many(old_wb, nr_switched);
}
for (inodep = isw->inodes; *inodep; inodep++)
iput(*inodep);
wb_put(new_wb);
kfree(isw);
atomic_dec(&isw_nr_in_flight);
}
static bool inode_prepare_wbs_switch(struct inode *inode,
struct bdi_writeback *new_wb)
{
/*
* Paired with smp_mb() in cgroup_writeback_umount().
* isw_nr_in_flight must be increased before checking SB_ACTIVE and
* grabbing an inode, otherwise isw_nr_in_flight can be observed as 0
* in cgroup_writeback_umount() and the isw_wq will be not flushed.
*/
smp_mb();
if (IS_DAX(inode))
return false;
/* while holding I_WB_SWITCH, no one else can update the association */
spin_lock(&inode->i_lock);
if (!(inode->i_sb->s_flags & SB_ACTIVE) ||
inode->i_state & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) ||
inode_to_wb(inode) == new_wb) {
spin_unlock(&inode->i_lock);
return false;
}
inode->i_state |= I_WB_SWITCH;
__iget(inode);
spin_unlock(&inode->i_lock);
return true;
}
/**
* inode_switch_wbs - change the wb association of an inode
* @inode: target inode
* @new_wb_id: ID of the new wb
*
* Switch @inode's wb association to the wb identified by @new_wb_id. The
* switching is performed asynchronously and may fail silently.
*/
static void inode_switch_wbs(struct inode *inode, int new_wb_id)
{
struct backing_dev_info *bdi = inode_to_bdi(inode);
struct cgroup_subsys_state *memcg_css;
struct inode_switch_wbs_context *isw;
/* noop if seems to be already in progress */
if (inode->i_state & I_WB_SWITCH)
return;
/* avoid queueing a new switch if too many are already in flight */
if (atomic_read(&isw_nr_in_flight) > WB_FRN_MAX_IN_FLIGHT)
return;
isw = kzalloc(struct_size(isw, inodes, 2), GFP_ATOMIC);
if (!isw)
return;
atomic_inc(&isw_nr_in_flight);
/* find and pin the new wb */
rcu_read_lock();
memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys);
if (memcg_css && !css_tryget(memcg_css))
memcg_css = NULL;
rcu_read_unlock();
if (!memcg_css)
goto out_free;
isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
css_put(memcg_css);
if (!isw->new_wb)
goto out_free;
if (!inode_prepare_wbs_switch(inode, isw->new_wb))
goto out_free;
isw->inodes[0] = inode;
/*
* In addition to synchronizing among switchers, I_WB_SWITCH tells
* the RCU protected stat update paths to grab the i_page
* lock so that stat transfer can synchronize against them.
* Let's continue after I_WB_SWITCH is guaranteed to be visible.
*/
INIT_RCU_WORK(&isw->work, inode_switch_wbs_work_fn);
queue_rcu_work(isw_wq, &isw->work);
return;
out_free:
atomic_dec(&isw_nr_in_flight);
if (isw->new_wb)
wb_put(isw->new_wb);
kfree(isw);
}
static bool isw_prepare_wbs_switch(struct inode_switch_wbs_context *isw,
struct list_head *list, int *nr)
{
struct inode *inode;
list_for_each_entry(inode, list, i_io_list) {
if (!inode_prepare_wbs_switch(inode, isw->new_wb))
continue;
isw->inodes[*nr] = inode;
(*nr)++;
if (*nr >= WB_MAX_INODES_PER_ISW - 1)
return true;
}
return false;
}
/**
* cleanup_offline_cgwb - detach associated inodes
* @wb: target wb
*
* Switch all inodes attached to @wb to a nearest living ancestor's wb in order
* to eventually release the dying @wb. Returns %true if not all inodes were
* switched and the function has to be restarted.
*/
bool cleanup_offline_cgwb(struct bdi_writeback *wb)
{
struct cgroup_subsys_state *memcg_css;
struct inode_switch_wbs_context *isw;
int nr;
bool restart = false;
isw = kzalloc(struct_size(isw, inodes, WB_MAX_INODES_PER_ISW),
GFP_KERNEL);
if (!isw)
return restart;
atomic_inc(&isw_nr_in_flight);
for (memcg_css = wb->memcg_css->parent; memcg_css;
memcg_css = memcg_css->parent) {
isw->new_wb = wb_get_create(wb->bdi, memcg_css, GFP_KERNEL);
if (isw->new_wb)
break;
}
if (unlikely(!isw->new_wb))
isw->new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */
nr = 0;
spin_lock(&wb->list_lock);
/*
* In addition to the inodes that have completed writeback, also switch
* cgwbs for those inodes only with dirty timestamps. Otherwise, those
* inodes won't be written back for a long time when lazytime is
* enabled, and thus pinning the dying cgwbs. It won't break the
* bandwidth restrictions, as writeback of inode metadata is not
* accounted for.
*/
restart = isw_prepare_wbs_switch(isw, &wb->b_attached, &nr);
if (!restart)
restart = isw_prepare_wbs_switch(isw, &wb->b_dirty_time, &nr);
spin_unlock(&wb->list_lock);
/* no attached inodes? bail out */
if (nr == 0) {
atomic_dec(&isw_nr_in_flight);
wb_put(isw->new_wb);
kfree(isw);
return restart;
}
/*
* In addition to synchronizing among switchers, I_WB_SWITCH tells
* the RCU protected stat update paths to grab the i_page
* lock so that stat transfer can synchronize against them.
* Let's continue after I_WB_SWITCH is guaranteed to be visible.
*/
INIT_RCU_WORK(&isw->work, inode_switch_wbs_work_fn);
queue_rcu_work(isw_wq, &isw->work);
return restart;
}
/**
* wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it
* @wbc: writeback_control of interest
* @inode: target inode
*
* @inode is locked and about to be written back under the control of @wbc.
* Record @inode's writeback context into @wbc and unlock the i_lock. On
* writeback completion, wbc_detach_inode() should be called. This is used
* to track the cgroup writeback context.
*/
void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
struct inode *inode)
{
if (!inode_cgwb_enabled(inode)) {
spin_unlock(&inode->i_lock);
return;
}
wbc->wb = inode_to_wb(inode);
wbc->inode = inode;
wbc->wb_id = wbc->wb->memcg_css->id;
wbc->wb_lcand_id = inode->i_wb_frn_winner;
wbc->wb_tcand_id = 0;
wbc->wb_bytes = 0;
wbc->wb_lcand_bytes = 0;
wbc->wb_tcand_bytes = 0;
wb_get(wbc->wb);
spin_unlock(&inode->i_lock);
/*
* A dying wb indicates that either the blkcg associated with the
* memcg changed or the associated memcg is dying. In the first
* case, a replacement wb should already be available and we should
* refresh the wb immediately. In the second case, trying to
* refresh will keep failing.
*/
if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css)))
inode_switch_wbs(inode, wbc->wb_id);
}
EXPORT_SYMBOL_GPL(wbc_attach_and_unlock_inode);
/**
* wbc_detach_inode - disassociate wbc from inode and perform foreign detection
* @wbc: writeback_control of the just finished writeback
*
* To be called after a writeback attempt of an inode finishes and undoes
* wbc_attach_and_unlock_inode(). Can be called under any context.
*
* As concurrent write sharing of an inode is expected to be very rare and
* memcg only tracks page ownership on first-use basis severely confining
* the usefulness of such sharing, cgroup writeback tracks ownership
* per-inode. While the support for concurrent write sharing of an inode
* is deemed unnecessary, an inode being written to by different cgroups at
* different points in time is a lot more common, and, more importantly,
* charging only by first-use can too readily lead to grossly incorrect
* behaviors (single foreign page can lead to gigabytes of writeback to be
* incorrectly attributed).
*
* To resolve this issue, cgroup writeback detects the majority dirtier of
* an inode and transfers the ownership to it. To avoid unnecessary
* oscillation, the detection mechanism keeps track of history and gives
* out the switch verdict only if the foreign usage pattern is stable over
* a certain amount of time and/or writeback attempts.
*
* On each writeback attempt, @wbc tries to detect the majority writer
* using Boyer-Moore majority vote algorithm. In addition to the byte
* count from the majority voting, it also counts the bytes written for the
* current wb and the last round's winner wb (max of last round's current
* wb, the winner from two rounds ago, and the last round's majority
* candidate). Keeping track of the historical winner helps the algorithm
* to semi-reliably detect the most active writer even when it's not the
* absolute majority.
*
* Once the winner of the round is determined, whether the winner is
* foreign or not and how much IO time the round consumed is recorded in
* inode->i_wb_frn_history. If the amount of recorded foreign IO time is
* over a certain threshold, the switch verdict is given.
*/
void wbc_detach_inode(struct writeback_control *wbc)
{
struct bdi_writeback *wb = wbc->wb;
struct inode *inode = wbc->inode;
unsigned long avg_time, max_bytes, max_time;
u16 history;
int max_id;
if (!wb)
return;
history = inode->i_wb_frn_history;
avg_time = inode->i_wb_frn_avg_time;
/* pick the winner of this round */
if (wbc->wb_bytes >= wbc->wb_lcand_bytes &&
wbc->wb_bytes >= wbc->wb_tcand_bytes) {
max_id = wbc->wb_id;
max_bytes = wbc->wb_bytes;
} else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) {
max_id = wbc->wb_lcand_id;
max_bytes = wbc->wb_lcand_bytes;
} else {
max_id = wbc->wb_tcand_id;
max_bytes = wbc->wb_tcand_bytes;
}
/*
* Calculate the amount of IO time the winner consumed and fold it
* into the running average kept per inode. If the consumed IO
* time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for
* deciding whether to switch or not. This is to prevent one-off
* small dirtiers from skewing the verdict.
*/
max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT,
wb->avg_write_bandwidth);
if (avg_time)
avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) -
(avg_time >> WB_FRN_TIME_AVG_SHIFT);
else
avg_time = max_time; /* immediate catch up on first run */
if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) {
int slots;
/*
* The switch verdict is reached if foreign wb's consume
* more than a certain proportion of IO time in a
* WB_FRN_TIME_PERIOD. This is loosely tracked by 16 slot
* history mask where each bit represents one sixteenth of
* the period. Determine the number of slots to shift into
* history from @max_time.
*/
slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT),
(unsigned long)WB_FRN_HIST_MAX_SLOTS);
history <<= slots;
if (wbc->wb_id != max_id)
history |= (1U << slots) - 1;
if (history)
trace_inode_foreign_history(inode, wbc, history);
/*
* Switch if the current wb isn't the consistent winner.
* If there are multiple closely competing dirtiers, the
* inode may switch across them repeatedly over time, which
* is okay. The main goal is avoiding keeping an inode on
* the wrong wb for an extended period of time.
*/
if (hweight16(history) > WB_FRN_HIST_THR_SLOTS)
inode_switch_wbs(inode, max_id);
}
/*
* Multiple instances of this function may race to update the
* following fields but we don't mind occassional inaccuracies.
*/
inode->i_wb_frn_winner = max_id;
inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX);
inode->i_wb_frn_history = history;
wb_put(wbc->wb);
wbc->wb = NULL;
}
EXPORT_SYMBOL_GPL(wbc_detach_inode);
/**
* wbc_account_cgroup_owner - account writeback to update inode cgroup ownership
* @wbc: writeback_control of the writeback in progress
* @page: page being written out
* @bytes: number of bytes being written out
*
* @bytes from @page are about to written out during the writeback
* controlled by @wbc. Keep the book for foreign inode detection. See
* wbc_detach_inode().
*/
void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page,
size_t bytes)
{
struct folio *folio;
struct cgroup_subsys_state *css;
int id;
/*
* pageout() path doesn't attach @wbc to the inode being written
* out. This is intentional as we don't want the function to block
* behind a slow cgroup. Ultimately, we want pageout() to kick off
* regular writeback instead of writing things out itself.
*/
if (!wbc->wb || wbc->no_cgroup_owner)
return;
folio = page_folio(page);
css = mem_cgroup_css_from_folio(folio);
/* dead cgroups shouldn't contribute to inode ownership arbitration */
if (!(css->flags & CSS_ONLINE))
return;
id = css->id;
if (id == wbc->wb_id) {
wbc->wb_bytes += bytes;
return;
}
if (id == wbc->wb_lcand_id)
wbc->wb_lcand_bytes += bytes;
/* Boyer-Moore majority vote algorithm */
if (!wbc->wb_tcand_bytes)
wbc->wb_tcand_id = id;
if (id == wbc->wb_tcand_id)
wbc->wb_tcand_bytes += bytes;
else
wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes);
}
EXPORT_SYMBOL_GPL(wbc_account_cgroup_owner);
/**
* wb_split_bdi_pages - split nr_pages to write according to bandwidth
* @wb: target bdi_writeback to split @nr_pages to
* @nr_pages: number of pages to write for the whole bdi
*
* Split @wb's portion of @nr_pages according to @wb's write bandwidth in
* relation to the total write bandwidth of all wb's w/ dirty inodes on
* @wb->bdi.
*/
static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
{
unsigned long this_bw = wb->avg_write_bandwidth;
unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
if (nr_pages == LONG_MAX)
return LONG_MAX;
/*
* This may be called on clean wb's and proportional distribution
* may not make sense, just use the original @nr_pages in those
* cases. In general, we wanna err on the side of writing more.
*/
if (!tot_bw || this_bw >= tot_bw)
return nr_pages;
else
return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw);
}
/**
* bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi
* @bdi: target backing_dev_info
* @base_work: wb_writeback_work to issue
* @skip_if_busy: skip wb's which already have writeback in progress
*
* Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which
* have dirty inodes. If @base_work->nr_page isn't %LONG_MAX, it's
* distributed to the busy wbs according to each wb's proportion in the
* total active write bandwidth of @bdi.
*/
static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
struct wb_writeback_work *base_work,
bool skip_if_busy)
{
struct bdi_writeback *last_wb = NULL;
struct bdi_writeback *wb = list_entry(&bdi->wb_list,
struct bdi_writeback, bdi_node);
might_sleep();
restart:
rcu_read_lock();
list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) {
DEFINE_WB_COMPLETION(fallback_work_done, bdi);
struct wb_writeback_work fallback_work;
struct wb_writeback_work *work;
long nr_pages;
if (last_wb) {
wb_put(last_wb);
last_wb = NULL;
}
/* SYNC_ALL writes out I_DIRTY_TIME too */
if (!wb_has_dirty_io(wb) &&
(base_work->sync_mode == WB_SYNC_NONE ||
list_empty(&wb->b_dirty_time)))
continue;
if (skip_if_busy && writeback_in_progress(wb))
continue;
nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);
work = kmalloc(sizeof(*work), GFP_ATOMIC);
if (work) {
*work = *base_work;
work->nr_pages = nr_pages;
work->auto_free = 1;
wb_queue_work(wb, work);
continue;
}
/*
* If wb_tryget fails, the wb has been shutdown, skip it.
*
* Pin @wb so that it stays on @bdi->wb_list. This allows
* continuing iteration from @wb after dropping and
* regrabbing rcu read lock.
*/
if (!wb_tryget(wb))
continue;
/* alloc failed, execute synchronously using on-stack fallback */
work = &fallback_work;
*work = *base_work;
work->nr_pages = nr_pages;
work->auto_free = 0;
work->done = &fallback_work_done;
wb_queue_work(wb, work);
last_wb = wb;
rcu_read_unlock();
wb_wait_for_completion(&fallback_work_done);
goto restart;
}
rcu_read_unlock();
if (last_wb)
wb_put(last_wb);
}
/**
* cgroup_writeback_by_id - initiate cgroup writeback from bdi and memcg IDs
* @bdi_id: target bdi id
* @memcg_id: target memcg css id
* @reason: reason why some writeback work initiated
* @done: target wb_completion
*
* Initiate flush of the bdi_writeback identified by @bdi_id and @memcg_id
* with the specified parameters.
*/
int cgroup_writeback_by_id(u64 bdi_id, int memcg_id,
enum wb_reason reason, struct wb_completion *done)
{
struct backing_dev_info *bdi;
struct cgroup_subsys_state *memcg_css;
struct bdi_writeback *wb;
struct wb_writeback_work *work;
unsigned long dirty;
int ret;
/* lookup bdi and memcg */
bdi = bdi_get_by_id(bdi_id);
if (!bdi)
return -ENOENT;
rcu_read_lock();
memcg_css = css_from_id(memcg_id, &memory_cgrp_subsys);
if (memcg_css && !css_tryget(memcg_css))
memcg_css = NULL;
rcu_read_unlock();
if (!memcg_css) {
ret = -ENOENT;
goto out_bdi_put;
}
/*
* And find the associated wb. If the wb isn't there already
* there's nothing to flush, don't create one.
*/
wb = wb_get_lookup(bdi, memcg_css);
if (!wb) {
ret = -ENOENT;
goto out_css_put;
}
/*
* The caller is attempting to write out most of
* the currently dirty pages. Let's take the current dirty page
* count and inflate it by 25% which should be large enough to
* flush out most dirty pages while avoiding getting livelocked by
* concurrent dirtiers.
*
* BTW the memcg stats are flushed periodically and this is best-effort
* estimation, so some potential error is ok.
*/
dirty = memcg_page_state(mem_cgroup_from_css(memcg_css), NR_FILE_DIRTY);
dirty = dirty * 10 / 8;
/* issue the writeback work */
work = kzalloc(sizeof(*work), GFP_NOWAIT | __GFP_NOWARN);
if (work) {
work->nr_pages = dirty;
work->sync_mode = WB_SYNC_NONE;
work->range_cyclic = 1;
work->reason = reason;
work->done = done;
work->auto_free = 1;
wb_queue_work(wb, work);
ret = 0;
} else {
ret = -ENOMEM;
}
wb_put(wb);
out_css_put:
css_put(memcg_css);
out_bdi_put:
bdi_put(bdi);
return ret;
}
/**
* cgroup_writeback_umount - flush inode wb switches for umount
*
* This function is called when a super_block is about to be destroyed and
* flushes in-flight inode wb switches. An inode wb switch goes through
* RCU and then workqueue, so the two need to be flushed in order to ensure
* that all previously scheduled switches are finished. As wb switches are
* rare occurrences and synchronize_rcu() can take a while, perform
* flushing iff wb switches are in flight.
*/
void cgroup_writeback_umount(void)
{
/*
* SB_ACTIVE should be reliably cleared before checking
* isw_nr_in_flight, see generic_shutdown_super().
*/
smp_mb();
if (atomic_read(&isw_nr_in_flight)) {
/*
* Use rcu_barrier() to wait for all pending callbacks to
* ensure that all in-flight wb switches are in the workqueue.
*/
rcu_barrier();
flush_workqueue(isw_wq);
}
}
static int __init cgroup_writeback_init(void)
{
isw_wq = alloc_workqueue("inode_switch_wbs", 0, 0);
if (!isw_wq)
return -ENOMEM;
return 0;
}
fs_initcall(cgroup_writeback_init);
#else /* CONFIG_CGROUP_WRITEBACK */
static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }
static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }
static void inode_cgwb_move_to_attached(struct inode *inode,
struct bdi_writeback *wb)
{
assert_spin_locked(&wb->list_lock);
assert_spin_locked(&inode->i_lock);
WARN_ON_ONCE(inode->i_state & I_FREEING);
inode->i_state &= ~I_SYNC_QUEUED;
list_del_init(&inode->i_io_list);
wb_io_lists_depopulated(wb);
}
static struct bdi_writeback *
locked_inode_to_wb_and_lock_list(struct inode *inode)
__releases(&inode->i_lock)
__acquires(&wb->list_lock)
{
struct bdi_writeback *wb = inode_to_wb(inode);
spin_unlock(&inode->i_lock);
spin_lock(&wb->list_lock);
return wb;
}
static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
__acquires(&wb->list_lock)
{
struct bdi_writeback *wb = inode_to_wb(inode);
spin_lock(&wb->list_lock);
return wb;
}
static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
{
return nr_pages;
}
static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
struct wb_writeback_work *base_work,
bool skip_if_busy)
{
might_sleep();
if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {
base_work->auto_free = 0;
wb_queue_work(&bdi->wb, base_work);
}
}
#endif /* CONFIG_CGROUP_WRITEBACK */
/*
* Add in the number of potentially dirty inodes, because each inode
* write can dirty pagecache in the underlying blockdev.
*/
static unsigned long get_nr_dirty_pages(void)
{
return global_node_page_state(NR_FILE_DIRTY) +
get_nr_dirty_inodes();
}
static void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason)
{
if (!wb_has_dirty_io(wb))
return;
/*
* All callers of this function want to start writeback of all
* dirty pages. Places like vmscan can call this at a very
* high frequency, causing pointless allocations of tons of
* work items and keeping the flusher threads busy retrieving
* that work. Ensure that we only allow one of them pending and
* inflight at the time.
*/
if (test_bit(WB_start_all, &wb->state) ||
test_and_set_bit(WB_start_all, &wb->state))
return;
wb->start_all_reason = reason;
wb_wakeup(wb);
}
/**
* wb_start_background_writeback - start background writeback
* @wb: bdi_writback to write from
*
* Description:
* This makes sure WB_SYNC_NONE background writeback happens. When
* this function returns, it is only guaranteed that for given wb
* some IO is happening if we are over background dirty threshold.
* Caller need not hold sb s_umount semaphore.
*/
void wb_start_background_writeback(struct bdi_writeback *wb)
{
/*
* We just wake up the flusher thread. It will perform background
* writeback as soon as there is no other work to do.
*/
trace_writeback_wake_background(wb);
wb_wakeup(wb);
}
/*
* Remove the inode from the writeback list it is on.
*/
void inode_io_list_del(struct inode *inode)
{
struct bdi_writeback *wb;
wb = inode_to_wb_and_lock_list(inode);
spin_lock(&inode->i_lock);
inode->i_state &= ~I_SYNC_QUEUED;
list_del_init(&inode->i_io_list);
wb_io_lists_depopulated(wb);
spin_unlock(&inode->i_lock);
spin_unlock(&wb->list_lock);
}
EXPORT_SYMBOL(inode_io_list_del);
/*
* mark an inode as under writeback on the sb
*/
void sb_mark_inode_writeback(struct inode *inode)
{
struct super_block *sb = inode->i_sb;
unsigned long flags;
if (list_empty(&inode->i_wb_list)) {
spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
if (list_empty(&inode->i_wb_list)) {
list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb);
trace_sb_mark_inode_writeback(inode);
}
spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
}
}
/*
* clear an inode as under writeback on the sb
*/
void sb_clear_inode_writeback(struct inode *inode)
{
struct super_block *sb = inode->i_sb;
unsigned long flags;
if (!list_empty(&inode->i_wb_list)) {
spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
if (!list_empty(&inode->i_wb_list)) {
list_del_init(&inode->i_wb_list);
trace_sb_clear_inode_writeback(inode);
}
spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
}
}
/*
* Redirty an inode: set its when-it-was dirtied timestamp and move it to the
* furthest end of its superblock's dirty-inode list.
*
* Before stamping the inode's ->dirtied_when, we check to see whether it is
* already the most-recently-dirtied inode on the b_dirty list. If that is
* the case then the inode must have been redirtied while it was being written
* out and we don't reset its dirtied_when.
*/
static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb)
{
assert_spin_locked(&inode->i_lock);
inode->i_state &= ~I_SYNC_QUEUED;
/*
* When the inode is being freed just don't bother with dirty list
* tracking. Flush worker will ignore this inode anyway and it will
* trigger assertions in inode_io_list_move_locked().
*/
if (inode->i_state & I_FREEING) {
list_del_init(&inode->i_io_list);
wb_io_lists_depopulated(wb);
return;
}
if (!list_empty(&wb->b_dirty)) {
struct inode *tail;
tail = wb_inode(wb->b_dirty.next);
if (time_before(inode->dirtied_when, tail->dirtied_when))
inode->dirtied_when = jiffies;
}
inode_io_list_move_locked(inode, wb, &wb->b_dirty);
}
static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
{
spin_lock(&inode->i_lock);
redirty_tail_locked(inode, wb);
spin_unlock(&inode->i_lock);
}
/*
* requeue inode for re-scanning after bdi->b_io list is exhausted.
*/
static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
{
inode_io_list_move_locked(inode, wb, &wb->b_more_io);
}
static void inode_sync_complete(struct inode *inode)
{
inode->i_state &= ~I_SYNC;
/* If inode is clean an unused, put it into LRU now... */
inode_add_lru(inode);
/* Waiters must see I_SYNC cleared before being woken up */
smp_mb();
wake_up_bit(&inode->i_state, __I_SYNC);
}
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
{
bool ret = time_after(inode->dirtied_when, t);
#ifndef CONFIG_64BIT
/*
* For inodes being constantly redirtied, dirtied_when can get stuck.
* It _appears_ to be in the future, but is actually in distant past.
* This test is necessary to prevent such wrapped-around relative times
* from permanently stopping the whole bdi writeback.
*/
ret = ret && time_before_eq(inode->dirtied_when, jiffies);
#endif
return ret;
}
/*
* Move expired (dirtied before dirtied_before) dirty inodes from
* @delaying_queue to @dispatch_queue.
*/
static int move_expired_inodes(struct list_head *delaying_queue,
struct list_head *dispatch_queue,
unsigned long dirtied_before)
{
LIST_HEAD(tmp);
struct list_head *pos, *node;
struct super_block *sb = NULL;
struct inode *inode;
int do_sb_sort = 0;
int moved = 0;
while (!list_empty(delaying_queue)) {
inode = wb_inode(delaying_queue->prev);
if (inode_dirtied_after(inode, dirtied_before))
break;
spin_lock(&inode->i_lock);
list_move(&inode->i_io_list, &tmp);
moved++;
inode->i_state |= I_SYNC_QUEUED;
spin_unlock(&inode->i_lock);
if (sb_is_blkdev_sb(inode->i_sb))
continue;
if (sb && sb != inode->i_sb)
do_sb_sort = 1;
sb = inode->i_sb;
}
/* just one sb in list, splice to dispatch_queue and we're done */
if (!do_sb_sort) {
list_splice(&tmp, dispatch_queue);
goto out;
}
/*
* Although inode's i_io_list is moved from 'tmp' to 'dispatch_queue',
* we don't take inode->i_lock here because it is just a pointless overhead.
* Inode is already marked as I_SYNC_QUEUED so writeback list handling is
* fully under our control.
*/
while (!list_empty(&tmp)) {
sb = wb_inode(tmp.prev)->i_sb;
list_for_each_prev_safe(pos, node, &tmp) {
inode = wb_inode(pos);
if (inode->i_sb == sb)
list_move(&inode->i_io_list, dispatch_queue);
}
}
out:
return moved;
}
/*
* Queue all expired dirty inodes for io, eldest first.
* Before
* newly dirtied b_dirty b_io b_more_io
* =============> gf edc BA
* After
* newly dirtied b_dirty b_io b_more_io
* =============> g fBAedc
* |
* +--> dequeue for IO
*/
static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work,
unsigned long dirtied_before)
{
int moved;
unsigned long time_expire_jif = dirtied_before;
assert_spin_locked(&wb->list_lock);
list_splice_init(&wb->b_more_io, &wb->b_io);
moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before);
if (!work->for_sync)
time_expire_jif = jiffies - dirtytime_expire_interval * HZ;
moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,
time_expire_jif);
if (moved)
wb_io_lists_populated(wb);
trace_writeback_queue_io(wb, work, dirtied_before, moved);
}
static int write_inode(struct inode *inode, struct writeback_control *wbc)
{
int ret;
if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
trace_writeback_write_inode_start(inode, wbc);
ret = inode->i_sb->s_op->write_inode(inode, wbc);
trace_writeback_write_inode(inode, wbc);
return ret;
}
return 0;
}
/*
* Wait for writeback on an inode to complete. Called with i_lock held.
* Caller must make sure inode cannot go away when we drop i_lock.
*/
static void __inode_wait_for_writeback(struct inode *inode)
__releases(inode->i_lock)
__acquires(inode->i_lock)
{
DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
wait_queue_head_t *wqh;
wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
while (inode->i_state & I_SYNC) { spin_unlock(&inode->i_lock);
__wait_on_bit(wqh, &wq, bit_wait,
TASK_UNINTERRUPTIBLE);
spin_lock(&inode->i_lock);
}
}
/*
* Wait for writeback on an inode to complete. Caller must have inode pinned.
*/
void inode_wait_for_writeback(struct inode *inode)
{
spin_lock(&inode->i_lock);
__inode_wait_for_writeback(inode);
spin_unlock(&inode->i_lock);
}
/*
* Sleep until I_SYNC is cleared. This function must be called with i_lock
* held and drops it. It is aimed for callers not holding any inode reference
* so once i_lock is dropped, inode can go away.
*/
static void inode_sleep_on_writeback(struct inode *inode)
__releases(inode->i_lock)
{
DEFINE_WAIT(wait);
wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
int sleep;
prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
sleep = inode->i_state & I_SYNC;
spin_unlock(&inode->i_lock);
if (sleep)
schedule();
finish_wait(wqh, &wait);
}
/*
* Find proper writeback list for the inode depending on its current state and
* possibly also change of its state while we were doing writeback. Here we
* handle things such as livelock prevention or fairness of writeback among
* inodes. This function can be called only by flusher thread - noone else
* processes all inodes in writeback lists and requeueing inodes behind flusher
* thread's back can have unexpected consequences.
*/
static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
struct writeback_control *wbc)
{
if (inode->i_state & I_FREEING)
return;
/*
* Sync livelock prevention. Each inode is tagged and synced in one
* shot. If still dirty, it will be redirty_tail()'ed below. Update
* the dirty time to prevent enqueue and sync it again.
*/
if ((inode->i_state & I_DIRTY) &&
(wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
inode->dirtied_when = jiffies;
if (wbc->pages_skipped) {
/*
* Writeback is not making progress due to locked buffers.
* Skip this inode for now. Although having skipped pages
* is odd for clean inodes, it can happen for some
* filesystems so handle that gracefully.
*/
if (inode->i_state & I_DIRTY_ALL)
redirty_tail_locked(inode, wb);
else
inode_cgwb_move_to_attached(inode, wb);
return;
}
if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
/*
* We didn't write back all the pages. nfs_writepages()
* sometimes bales out without doing anything.
*/
if (wbc->nr_to_write <= 0) {
/* Slice used up. Queue for next turn. */
requeue_io(inode, wb);
} else {
/*
* Writeback blocked by something other than
* congestion. Delay the inode for some time to
* avoid spinning on the CPU (100% iowait)
* retrying writeback of the dirty page/inode
* that cannot be performed immediately.
*/
redirty_tail_locked(inode, wb);
}
} else if (inode->i_state & I_DIRTY) {
/*
* Filesystems can dirty the inode during writeback operations,
* such as delayed allocation during submission or metadata
* updates after data IO completion.
*/
redirty_tail_locked(inode, wb);
} else if (inode->i_state & I_DIRTY_TIME) {
inode->dirtied_when = jiffies;
inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);
inode->i_state &= ~I_SYNC_QUEUED;
} else {
/* The inode is clean. Remove from writeback lists. */
inode_cgwb_move_to_attached(inode, wb);
}
}
/*
* Write out an inode and its dirty pages (or some of its dirty pages, depending
* on @wbc->nr_to_write), and clear the relevant dirty flags from i_state.
*
* This doesn't remove the inode from the writeback list it is on, except
* potentially to move it from b_dirty_time to b_dirty due to timestamp
* expiration. The caller is otherwise responsible for writeback list handling.
*
* The caller is also responsible for setting the I_SYNC flag beforehand and
* calling inode_sync_complete() to clear it afterwards.
*/
static int
__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
{
struct address_space *mapping = inode->i_mapping;
long nr_to_write = wbc->nr_to_write;
unsigned dirty;
int ret;
WARN_ON(!(inode->i_state & I_SYNC));
trace_writeback_single_inode_start(inode, wbc, nr_to_write);
ret = do_writepages(mapping, wbc);
/*
* Make sure to wait on the data before writing out the metadata.
* This is important for filesystems that modify metadata on data
* I/O completion. We don't do it for sync(2) writeback because it has a
* separate, external IO completion path and ->sync_fs for guaranteeing
* inode metadata is written back correctly.
*/
if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {
int err = filemap_fdatawait(mapping);
if (ret == 0)
ret = err;
}
/*
* If the inode has dirty timestamps and we need to write them, call
* mark_inode_dirty_sync() to notify the filesystem about it and to
* change I_DIRTY_TIME into I_DIRTY_SYNC.
*/
if ((inode->i_state & I_DIRTY_TIME) &&
(wbc->sync_mode == WB_SYNC_ALL ||
time_after(jiffies, inode->dirtied_time_when +
dirtytime_expire_interval * HZ))) {
trace_writeback_lazytime(inode);
mark_inode_dirty_sync(inode);
}
/*
* Get and clear the dirty flags from i_state. This needs to be done
* after calling writepages because some filesystems may redirty the
* inode during writepages due to delalloc. It also needs to be done
* after handling timestamp expiration, as that may dirty the inode too.
*/
spin_lock(&inode->i_lock);
dirty = inode->i_state & I_DIRTY;
inode->i_state &= ~dirty;
/*
* Paired with smp_mb() in __mark_inode_dirty(). This allows
* __mark_inode_dirty() to test i_state without grabbing i_lock -
* either they see the I_DIRTY bits cleared or we see the dirtied
* inode.
*
* I_DIRTY_PAGES is always cleared together above even if @mapping
* still has dirty pages. The flag is reinstated after smp_mb() if
* necessary. This guarantees that either __mark_inode_dirty()
* sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.
*/
smp_mb();
if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
inode->i_state |= I_DIRTY_PAGES;
else if (unlikely(inode->i_state & I_PINNING_NETFS_WB)) {
if (!(inode->i_state & I_DIRTY_PAGES)) {
inode->i_state &= ~I_PINNING_NETFS_WB;
wbc->unpinned_netfs_wb = true;
dirty |= I_PINNING_NETFS_WB; /* Cause write_inode */
}
}
spin_unlock(&inode->i_lock);
/* Don't write the inode if only I_DIRTY_PAGES was set */
if (dirty & ~I_DIRTY_PAGES) {
int err = write_inode(inode, wbc);
if (ret == 0)
ret = err;
}
wbc->unpinned_netfs_wb = false;
trace_writeback_single_inode(inode, wbc, nr_to_write);
return ret;
}
/*
* Write out an inode's dirty data and metadata on-demand, i.e. separately from
* the regular batched writeback done by the flusher threads in
* writeback_sb_inodes(). @wbc controls various aspects of the write, such as
* whether it is a data-integrity sync (%WB_SYNC_ALL) or not (%WB_SYNC_NONE).
*
* To prevent the inode from going away, either the caller must have a reference
* to the inode, or the inode must have I_WILL_FREE or I_FREEING set.
*/
static int writeback_single_inode(struct inode *inode,
struct writeback_control *wbc)
{
struct bdi_writeback *wb;
int ret = 0;
spin_lock(&inode->i_lock);
if (!atomic_read(&inode->i_count))
WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
else
WARN_ON(inode->i_state & I_WILL_FREE);
if (inode->i_state & I_SYNC) {
/*
* Writeback is already running on the inode. For WB_SYNC_NONE,
* that's enough and we can just return. For WB_SYNC_ALL, we
* must wait for the existing writeback to complete, then do
* writeback again if there's anything left.
*/
if (wbc->sync_mode != WB_SYNC_ALL)
goto out;
__inode_wait_for_writeback(inode);
}
WARN_ON(inode->i_state & I_SYNC);
/*
* If the inode is already fully clean, then there's nothing to do.
*
* For data-integrity syncs we also need to check whether any pages are
* still under writeback, e.g. due to prior WB_SYNC_NONE writeback. If
* there are any such pages, we'll need to wait for them.
*/
if (!(inode->i_state & I_DIRTY_ALL) &&
(wbc->sync_mode != WB_SYNC_ALL ||
!mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
goto out;
inode->i_state |= I_SYNC;
wbc_attach_and_unlock_inode(wbc, inode);
ret = __writeback_single_inode(inode, wbc);
wbc_detach_inode(wbc);
wb = inode_to_wb_and_lock_list(inode);
spin_lock(&inode->i_lock);
/*
* If the inode is freeing, its i_io_list shoudn't be updated
* as it can be finally deleted at this moment.
*/
if (!(inode->i_state & I_FREEING)) {
/*
* If the inode is now fully clean, then it can be safely
* removed from its writeback list (if any). Otherwise the
* flusher threads are responsible for the writeback lists.
*/
if (!(inode->i_state & I_DIRTY_ALL))
inode_cgwb_move_to_attached(inode, wb);
else if (!(inode->i_state & I_SYNC_QUEUED)) {
if ((inode->i_state & I_DIRTY))
redirty_tail_locked(inode, wb);
else if (inode->i_state & I_DIRTY_TIME) {
inode->dirtied_when = jiffies;
inode_io_list_move_locked(inode,
wb,
&wb->b_dirty_time);
}
}
}
spin_unlock(&wb->list_lock);
inode_sync_complete(inode);
out:
spin_unlock(&inode->i_lock);
return ret;
}
static long writeback_chunk_size(struct bdi_writeback *wb,
struct wb_writeback_work *work)
{
long pages;
/*
* WB_SYNC_ALL mode does livelock avoidance by syncing dirty
* inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
* here avoids calling into writeback_inodes_wb() more than once.
*
* The intended call sequence for WB_SYNC_ALL writeback is:
*
* wb_writeback()
* writeback_sb_inodes() <== called only once
* write_cache_pages() <== called once for each inode
* (quickly) tag currently dirty pages
* (maybe slowly) sync all tagged pages
*/
if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
pages = LONG_MAX;
else {
pages = min(wb->avg_write_bandwidth / 2,
global_wb_domain.dirty_limit / DIRTY_SCOPE);
pages = min(pages, work->nr_pages);
pages = round_down(pages + MIN_WRITEBACK_PAGES,
MIN_WRITEBACK_PAGES);
}
return pages;
}
/*
* Write a portion of b_io inodes which belong to @sb.
*
* Return the number of pages and/or inodes written.
*
* NOTE! This is called with wb->list_lock held, and will
* unlock and relock that for each inode it ends up doing
* IO for.
*/
static long writeback_sb_inodes(struct super_block *sb,
struct bdi_writeback *wb,
struct wb_writeback_work *work)
{
struct writeback_control wbc = {
.sync_mode = work->sync_mode,
.tagged_writepages = work->tagged_writepages,
.for_kupdate = work->for_kupdate,
.for_background = work->for_background,
.for_sync = work->for_sync,
.range_cyclic = work->range_cyclic,
.range_start = 0,
.range_end = LLONG_MAX,
};
unsigned long start_time = jiffies;
long write_chunk;
long total_wrote = 0; /* count both pages and inodes */
while (!list_empty(&wb->b_io)) {
struct inode *inode = wb_inode(wb->b_io.prev);
struct bdi_writeback *tmp_wb;
long wrote;
if (inode->i_sb != sb) {
if (work->sb) {
/*
* We only want to write back data for this
* superblock, move all inodes not belonging
* to it back onto the dirty list.
*/
redirty_tail(inode, wb);
continue;
}
/*
* The inode belongs to a different superblock.
* Bounce back to the caller to unpin this and
* pin the next superblock.
*/
break;
}
/*
* Don't bother with new inodes or inodes being freed, first
* kind does not need periodic writeout yet, and for the latter
* kind writeout is handled by the freer.
*/
spin_lock(&inode->i_lock);
if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
redirty_tail_locked(inode, wb);
spin_unlock(&inode->i_lock);
continue;
}
if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
/*
* If this inode is locked for writeback and we are not
* doing writeback-for-data-integrity, move it to
* b_more_io so that writeback can proceed with the
* other inodes on s_io.
*
* We'll have another go at writing back this inode
* when we completed a full scan of b_io.
*/
requeue_io(inode, wb);
spin_unlock(&inode->i_lock);
trace_writeback_sb_inodes_requeue(inode);
continue;
}
spin_unlock(&wb->list_lock);
/*
* We already requeued the inode if it had I_SYNC set and we
* are doing WB_SYNC_NONE writeback. So this catches only the
* WB_SYNC_ALL case.
*/
if (inode->i_state & I_SYNC) {
/* Wait for I_SYNC. This function drops i_lock... */
inode_sleep_on_writeback(inode);
/* Inode may be gone, start again */
spin_lock(&wb->list_lock);
continue;
}
inode->i_state |= I_SYNC;
wbc_attach_and_unlock_inode(&wbc, inode);
write_chunk = writeback_chunk_size(wb, work);
wbc.nr_to_write = write_chunk;
wbc.pages_skipped = 0;
/*
* We use I_SYNC to pin the inode in memory. While it is set
* evict_inode() will wait so the inode cannot be freed.
*/
__writeback_single_inode(inode, &wbc);
wbc_detach_inode(&wbc);
work->nr_pages -= write_chunk - wbc.nr_to_write;
wrote = write_chunk - wbc.nr_to_write - wbc.pages_skipped;
wrote = wrote < 0 ? 0 : wrote;
total_wrote += wrote;
if (need_resched()) {
/*
* We're trying to balance between building up a nice
* long list of IOs to improve our merge rate, and
* getting those IOs out quickly for anyone throttling
* in balance_dirty_pages(). cond_resched() doesn't
* unplug, so get our IOs out the door before we
* give up the CPU.
*/
blk_flush_plug(current->plug, false);
cond_resched();
}
/*
* Requeue @inode if still dirty. Be careful as @inode may
* have been switched to another wb in the meantime.
*/
tmp_wb = inode_to_wb_and_lock_list(inode);
spin_lock(&inode->i_lock);
if (!(inode->i_state & I_DIRTY_ALL))
total_wrote++;
requeue_inode(inode, tmp_wb, &wbc);
inode_sync_complete(inode);
spin_unlock(&inode->i_lock);
if (unlikely(tmp_wb != wb)) {
spin_unlock(&tmp_wb->list_lock);
spin_lock(&wb->list_lock);
}
/*
* bail out to wb_writeback() often enough to check
* background threshold and other termination conditions.
*/
if (total_wrote) {
if (time_is_before_jiffies(start_time + HZ / 10UL))
break;
if (work->nr_pages <= 0)
break;
}
}
return total_wrote;
}
static long __writeback_inodes_wb(struct bdi_writeback *wb,
struct wb_writeback_work *work)
{
unsigned long start_time = jiffies;
long wrote = 0;
while (!list_empty(&wb->b_io)) {
struct inode *inode = wb_inode(wb->b_io.prev);
struct super_block *sb = inode->i_sb;
if (!super_trylock_shared(sb)) {
/*
* super_trylock_shared() may fail consistently due to
* s_umount being grabbed by someone else. Don't use
* requeue_io() to avoid busy retrying the inode/sb.
*/
redirty_tail(inode, wb);
continue;
}
wrote += writeback_sb_inodes(sb, wb, work);
up_read(&sb->s_umount);
/* refer to the same tests at the end of writeback_sb_inodes */
if (wrote) {
if (time_is_before_jiffies(start_time + HZ / 10UL))
break;
if (work->nr_pages <= 0)
break;
}
}
/* Leave any unwritten inodes on b_io */
return wrote;
}
static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
enum wb_reason reason)
{
struct wb_writeback_work work = {
.nr_pages = nr_pages,
.sync_mode = WB_SYNC_NONE,
.range_cyclic = 1,
.reason = reason,
};
struct blk_plug plug;
blk_start_plug(&plug);
spin_lock(&wb->list_lock);
if (list_empty(&wb->b_io))
queue_io(wb, &work, jiffies);
__writeback_inodes_wb(wb, &work);
spin_unlock(&wb->list_lock);
blk_finish_plug(&plug);
return nr_pages - work.nr_pages;
}
/*
* Explicit flushing or periodic writeback of "old" data.
*
* Define "old": the first time one of an inode's pages is dirtied, we mark the
* dirtying-time in the inode's address_space. So this periodic writeback code
* just walks the superblock inode list, writing back any inodes which are
* older than a specific point in time.
*
* Try to run once per dirty_writeback_interval. But if a writeback event
* takes longer than a dirty_writeback_interval interval, then leave a
* one-second gap.
*
* dirtied_before takes precedence over nr_to_write. So we'll only write back
* all dirty pages if they are all attached to "old" mappings.
*/
static long wb_writeback(struct bdi_writeback *wb,
struct wb_writeback_work *work)
{
long nr_pages = work->nr_pages;
unsigned long dirtied_before = jiffies;
struct inode *inode;
long progress;
struct blk_plug plug;
blk_start_plug(&plug);
for (;;) {
/*
* Stop writeback when nr_pages has been consumed
*/
if (work->nr_pages <= 0)
break;
/*
* Background writeout and kupdate-style writeback may
* run forever. Stop them if there is other work to do
* so that e.g. sync can proceed. They'll be restarted
* after the other works are all done.
*/
if ((work->for_background || work->for_kupdate) &&
!list_empty(&wb->work_list))
break;
/*
* For background writeout, stop when we are below the
* background dirty threshold
*/
if (work->for_background && !wb_over_bg_thresh(wb))
break;
spin_lock(&wb->list_lock);
/*
* Kupdate and background works are special and we want to
* include all inodes that need writing. Livelock avoidance is
* handled by these works yielding to any other work so we are
* safe.
*/
if (work->for_kupdate) {
dirtied_before = jiffies -
msecs_to_jiffies(dirty_expire_interval * 10);
} else if (work->for_background)
dirtied_before = jiffies;
trace_writeback_start(wb, work);
if (list_empty(&wb->b_io))
queue_io(wb, work, dirtied_before);
if (work->sb)
progress = writeback_sb_inodes(work->sb, wb, work);
else
progress = __writeback_inodes_wb(wb, work);
trace_writeback_written(wb, work);
/*
* Did we write something? Try for more
*
* Dirty inodes are moved to b_io for writeback in batches.
* The completion of the current batch does not necessarily
* mean the overall work is done. So we keep looping as long
* as made some progress on cleaning pages or inodes.
*/
if (progress) {
spin_unlock(&wb->list_lock);
continue;
}
/*
* No more inodes for IO, bail
*/
if (list_empty(&wb->b_more_io)) {
spin_unlock(&wb->list_lock);
break;
}
/*
* Nothing written. Wait for some inode to
* become available for writeback. Otherwise
* we'll just busyloop.
*/
trace_writeback_wait(wb, work);
inode = wb_inode(wb->b_more_io.prev);
spin_lock(&inode->i_lock);
spin_unlock(&wb->list_lock);
/* This function drops i_lock... */
inode_sleep_on_writeback(inode);
}
blk_finish_plug(&plug);
return nr_pages - work->nr_pages;
}
/*
* Return the next wb_writeback_work struct that hasn't been processed yet.
*/
static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)
{
struct wb_writeback_work *work = NULL;
spin_lock_irq(&wb->work_lock);
if (!list_empty(&wb->work_list)) {
work = list_entry(wb->work_list.next,
struct wb_writeback_work, list);
list_del_init(&work->list);
}
spin_unlock_irq(&wb->work_lock);
return work;
}
static long wb_check_background_flush(struct bdi_writeback *wb)
{
if (wb_over_bg_thresh(wb)) {
struct wb_writeback_work work = {
.nr_pages = LONG_MAX,
.sync_mode = WB_SYNC_NONE,
.for_background = 1,
.range_cyclic = 1,
.reason = WB_REASON_BACKGROUND,
};
return wb_writeback(wb, &work);
}
return 0;
}
static long wb_check_old_data_flush(struct bdi_writeback *wb)
{
unsigned long expired;
long nr_pages;
/*
* When set to zero, disable periodic writeback
*/
if (!dirty_writeback_interval)
return 0;
expired = wb->last_old_flush +
msecs_to_jiffies(dirty_writeback_interval * 10);
if (time_before(jiffies, expired))
return 0;
wb->last_old_flush = jiffies;
nr_pages = get_nr_dirty_pages();
if (nr_pages) {
struct wb_writeback_work work = {
.nr_pages = nr_pages,
.sync_mode = WB_SYNC_NONE,
.for_kupdate = 1,
.range_cyclic = 1,
.reason = WB_REASON_PERIODIC,
};
return wb_writeback(wb, &work);
}
return 0;
}
static long wb_check_start_all(struct bdi_writeback *wb)
{
long nr_pages;
if (!test_bit(WB_start_all, &wb->state))
return 0;
nr_pages = get_nr_dirty_pages();
if (nr_pages) {
struct wb_writeback_work work = {
.nr_pages = wb_split_bdi_pages(wb, nr_pages),
.sync_mode = WB_SYNC_NONE,
.range_cyclic = 1,
.reason = wb->start_all_reason,
};
nr_pages = wb_writeback(wb, &work);
}
clear_bit(WB_start_all, &wb->state);
return nr_pages;
}
/*
* Retrieve work items and do the writeback they describe
*/
static long wb_do_writeback(struct bdi_writeback *wb)
{
struct wb_writeback_work *work;
long wrote = 0;
set_bit(WB_writeback_running, &wb->state);
while ((work = get_next_work_item(wb)) != NULL) {
trace_writeback_exec(wb, work);
wrote += wb_writeback(wb, work);
finish_writeback_work(wb, work);
}
/*
* Check for a flush-everything request
*/
wrote += wb_check_start_all(wb);
/*
* Check for periodic writeback, kupdated() style
*/
wrote += wb_check_old_data_flush(wb);
wrote += wb_check_background_flush(wb);
clear_bit(WB_writeback_running, &wb->state);
return wrote;
}
/*
* Handle writeback of dirty data for the device backed by this bdi. Also
* reschedules periodically and does kupdated style flushing.
*/
void wb_workfn(struct work_struct *work)
{
struct bdi_writeback *wb = container_of(to_delayed_work(work),
struct bdi_writeback, dwork);
long pages_written;
set_worker_desc("flush-%s", bdi_dev_name(wb->bdi));
if (likely(!current_is_workqueue_rescuer() ||
!test_bit(WB_registered, &wb->state))) {
/*
* The normal path. Keep writing back @wb until its
* work_list is empty. Note that this path is also taken
* if @wb is shutting down even when we're running off the
* rescuer as work_list needs to be drained.
*/
do {
pages_written = wb_do_writeback(wb);
trace_writeback_pages_written(pages_written);
} while (!list_empty(&wb->work_list));
} else {
/*
* bdi_wq can't get enough workers and we're running off
* the emergency worker. Don't hog it. Hopefully, 1024 is
* enough for efficient IO.
*/
pages_written = writeback_inodes_wb(wb, 1024,
WB_REASON_FORKER_THREAD);
trace_writeback_pages_written(pages_written);
}
if (!list_empty(&wb->work_list))
wb_wakeup(wb);
else if (wb_has_dirty_io(wb) && dirty_writeback_interval)
wb_wakeup_delayed(wb);
}
/*
* Start writeback of `nr_pages' pages on this bdi. If `nr_pages' is zero,
* write back the whole world.
*/
static void __wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
enum wb_reason reason)
{
struct bdi_writeback *wb;
if (!bdi_has_dirty_io(bdi))
return;
list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
wb_start_writeback(wb, reason);
}
void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
enum wb_reason reason)
{
rcu_read_lock();
__wakeup_flusher_threads_bdi(bdi, reason);
rcu_read_unlock();
}
/*
* Wakeup the flusher threads to start writeback of all currently dirty pages
*/
void wakeup_flusher_threads(enum wb_reason reason)
{
struct backing_dev_info *bdi;
/*
* If we are expecting writeback progress we must submit plugged IO.
*/
blk_flush_plug(current->plug, true);
rcu_read_lock();
list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
__wakeup_flusher_threads_bdi(bdi, reason);
rcu_read_unlock();
}
/*
* Wake up bdi's periodically to make sure dirtytime inodes gets
* written back periodically. We deliberately do *not* check the
* b_dirtytime list in wb_has_dirty_io(), since this would cause the
* kernel to be constantly waking up once there are any dirtytime
* inodes on the system. So instead we define a separate delayed work
* function which gets called much more rarely. (By default, only
* once every 12 hours.)
*
* If there is any other write activity going on in the file system,
* this function won't be necessary. But if the only thing that has
* happened on the file system is a dirtytime inode caused by an atime
* update, we need this infrastructure below to make sure that inode
* eventually gets pushed out to disk.
*/
static void wakeup_dirtytime_writeback(struct work_struct *w);
static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);
static void wakeup_dirtytime_writeback(struct work_struct *w)
{
struct backing_dev_info *bdi;
rcu_read_lock();
list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
struct bdi_writeback *wb;
list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
if (!list_empty(&wb->b_dirty_time))
wb_wakeup(wb);
}
rcu_read_unlock();
schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
}
static int __init start_dirtytime_writeback(void)
{
schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
return 0;
}
__initcall(start_dirtytime_writeback);
int dirtytime_interval_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int ret;
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (ret == 0 && write)
mod_delayed_work(system_wq, &dirtytime_work, 0);
return ret;
}
/**
* __mark_inode_dirty - internal function to mark an inode dirty
*
* @inode: inode to mark
* @flags: what kind of dirty, e.g. I_DIRTY_SYNC. This can be a combination of
* multiple I_DIRTY_* flags, except that I_DIRTY_TIME can't be combined
* with I_DIRTY_PAGES.
*
* Mark an inode as dirty. We notify the filesystem, then update the inode's
* dirty flags. Then, if needed we add the inode to the appropriate dirty list.
*
* Most callers should use mark_inode_dirty() or mark_inode_dirty_sync()
* instead of calling this directly.
*
* CAREFUL! We only add the inode to the dirty list if it is hashed or if it
* refers to a blockdev. Unhashed inodes will never be added to the dirty list
* even if they are later hashed, as they will have been marked dirty already.
*
* In short, ensure you hash any inodes _before_ you start marking them dirty.
*
* Note that for blockdevs, inode->dirtied_when represents the dirtying time of
* the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of
* the kernel-internal blockdev inode represents the dirtying time of the
* blockdev's pages. This is why for I_DIRTY_PAGES we always use
* page->mapping->host, so the page-dirtying time is recorded in the internal
* blockdev inode.
*/
void __mark_inode_dirty(struct inode *inode, int flags)
{
struct super_block *sb = inode->i_sb;
int dirtytime = 0;
struct bdi_writeback *wb = NULL;
trace_writeback_mark_inode_dirty(inode, flags); if (flags & I_DIRTY_INODE) {
/*
* Inode timestamp update will piggback on this dirtying.
* We tell ->dirty_inode callback that timestamps need to
* be updated by setting I_DIRTY_TIME in flags.
*/
if (inode->i_state & I_DIRTY_TIME) { spin_lock(&inode->i_lock);
if (inode->i_state & I_DIRTY_TIME) {
inode->i_state &= ~I_DIRTY_TIME;
flags |= I_DIRTY_TIME;
}
spin_unlock(&inode->i_lock);
}
/*
* Notify the filesystem about the inode being dirtied, so that
* (if needed) it can update on-disk fields and journal the
* inode. This is only needed when the inode itself is being
* dirtied now. I.e. it's only needed for I_DIRTY_INODE, not
* for just I_DIRTY_PAGES or I_DIRTY_TIME.
*/
trace_writeback_dirty_inode_start(inode, flags); if (sb->s_op->dirty_inode) sb->s_op->dirty_inode(inode,
flags & (I_DIRTY_INODE | I_DIRTY_TIME));
trace_writeback_dirty_inode(inode, flags);
/* I_DIRTY_INODE supersedes I_DIRTY_TIME. */
flags &= ~I_DIRTY_TIME;
} else {
/*
* Else it's either I_DIRTY_PAGES, I_DIRTY_TIME, or nothing.
* (We don't support setting both I_DIRTY_PAGES and I_DIRTY_TIME
* in one call to __mark_inode_dirty().)
*/
dirtytime = flags & I_DIRTY_TIME; WARN_ON_ONCE(dirtytime && flags != I_DIRTY_TIME);
}
/*
* Paired with smp_mb() in __writeback_single_inode() for the
* following lockless i_state test. See there for details.
*/
smp_mb();
if ((inode->i_state & flags) == flags)
return;
spin_lock(&inode->i_lock);
if ((inode->i_state & flags) != flags) {
const int was_dirty = inode->i_state & I_DIRTY; inode_attach_wb(inode, NULL); inode->i_state |= flags;
/*
* Grab inode's wb early because it requires dropping i_lock and we
* need to make sure following checks happen atomically with dirty
* list handling so that we don't move inodes under flush worker's
* hands.
*/
if (!was_dirty) {
wb = locked_inode_to_wb_and_lock_list(inode);
spin_lock(&inode->i_lock);
}
/*
* If the inode is queued for writeback by flush worker, just
* update its dirty state. Once the flush worker is done with
* the inode it will place it on the appropriate superblock
* list, based upon its state.
*/
if (inode->i_state & I_SYNC_QUEUED)
goto out_unlock;
/*
* Only add valid (hashed) inodes to the superblock's
* dirty list. Add blockdev inodes as well.
*/
if (!S_ISBLK(inode->i_mode)) { if (inode_unhashed(inode))
goto out_unlock;
}
if (inode->i_state & I_FREEING)
goto out_unlock;
/*
* If the inode was already on b_dirty/b_io/b_more_io, don't
* reposition it (that would break b_dirty time-ordering).
*/
if (!was_dirty) {
struct list_head *dirty_list;
bool wakeup_bdi = false;
inode->dirtied_when = jiffies;
if (dirtytime)
inode->dirtied_time_when = jiffies; if (inode->i_state & I_DIRTY) dirty_list = &wb->b_dirty;
else
dirty_list = &wb->b_dirty_time; wakeup_bdi = inode_io_list_move_locked(inode, wb,
dirty_list);
spin_unlock(&wb->list_lock);
spin_unlock(&inode->i_lock);
trace_writeback_dirty_inode_enqueue(inode);
/*
* If this is the first dirty inode for this bdi,
* we have to wake-up the corresponding bdi thread
* to make sure background write-back happens
* later.
*/
if (wakeup_bdi && (wb->bdi->capabilities & BDI_CAP_WRITEBACK)) wb_wakeup_delayed(wb);
return;
}
}
out_unlock:
if (wb) spin_unlock(&wb->list_lock); spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL(__mark_inode_dirty);
/*
* The @s_sync_lock is used to serialise concurrent sync operations
* to avoid lock contention problems with concurrent wait_sb_inodes() calls.
* Concurrent callers will block on the s_sync_lock rather than doing contending
* walks. The queueing maintains sync(2) required behaviour as all the IO that
* has been issued up to the time this function is enter is guaranteed to be
* completed by the time we have gained the lock and waited for all IO that is
* in progress regardless of the order callers are granted the lock.
*/
static void wait_sb_inodes(struct super_block *sb)
{
LIST_HEAD(sync_list);
/*
* We need to be protected against the filesystem going from
* r/o to r/w or vice versa.
*/
WARN_ON(!rwsem_is_locked(&sb->s_umount));
mutex_lock(&sb->s_sync_lock);
/*
* Splice the writeback list onto a temporary list to avoid waiting on
* inodes that have started writeback after this point.
*
* Use rcu_read_lock() to keep the inodes around until we have a
* reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as
* the local list because inodes can be dropped from either by writeback
* completion.
*/
rcu_read_lock();
spin_lock_irq(&sb->s_inode_wblist_lock);
list_splice_init(&sb->s_inodes_wb, &sync_list);
/*
* Data integrity sync. Must wait for all pages under writeback, because
* there may have been pages dirtied before our sync call, but which had
* writeout started before we write it out. In which case, the inode
* may not be on the dirty list, but we still have to wait for that
* writeout.
*/
while (!list_empty(&sync_list)) {
struct inode *inode = list_first_entry(&sync_list, struct inode,
i_wb_list);
struct address_space *mapping = inode->i_mapping;
/*
* Move each inode back to the wb list before we drop the lock
* to preserve consistency between i_wb_list and the mapping
* writeback tag. Writeback completion is responsible to remove
* the inode from either list once the writeback tag is cleared.
*/
list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb);
/*
* The mapping can appear untagged while still on-list since we
* do not have the mapping lock. Skip it here, wb completion
* will remove it.
*/
if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
continue;
spin_unlock_irq(&sb->s_inode_wblist_lock);
spin_lock(&inode->i_lock);
if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) {
spin_unlock(&inode->i_lock);
spin_lock_irq(&sb->s_inode_wblist_lock);
continue;
}
__iget(inode);
spin_unlock(&inode->i_lock);
rcu_read_unlock();
/*
* We keep the error status of individual mapping so that
* applications can catch the writeback error using fsync(2).
* See filemap_fdatawait_keep_errors() for details.
*/
filemap_fdatawait_keep_errors(mapping);
cond_resched();
iput(inode);
rcu_read_lock();
spin_lock_irq(&sb->s_inode_wblist_lock);
}
spin_unlock_irq(&sb->s_inode_wblist_lock);
rcu_read_unlock();
mutex_unlock(&sb->s_sync_lock);
}
static void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr,
enum wb_reason reason, bool skip_if_busy)
{
struct backing_dev_info *bdi = sb->s_bdi;
DEFINE_WB_COMPLETION(done, bdi);
struct wb_writeback_work work = {
.sb = sb,
.sync_mode = WB_SYNC_NONE,
.tagged_writepages = 1,
.done = &done,
.nr_pages = nr,
.reason = reason,
};
if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)
return;
WARN_ON(!rwsem_is_locked(&sb->s_umount));
bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy);
wb_wait_for_completion(&done);
}
/**
* writeback_inodes_sb_nr - writeback dirty inodes from given super_block
* @sb: the superblock
* @nr: the number of pages to write
* @reason: reason why some writeback work initiated
*
* Start writeback on some inodes on this super_block. No guarantees are made
* on how many (if any) will be written, and this function does not wait
* for IO completion of submitted IO.
*/
void writeback_inodes_sb_nr(struct super_block *sb,
unsigned long nr,
enum wb_reason reason)
{
__writeback_inodes_sb_nr(sb, nr, reason, false);
}
EXPORT_SYMBOL(writeback_inodes_sb_nr);
/**
* writeback_inodes_sb - writeback dirty inodes from given super_block
* @sb: the superblock
* @reason: reason why some writeback work was initiated
*
* Start writeback on some inodes on this super_block. No guarantees are made
* on how many (if any) will be written, and this function does not wait
* for IO completion of submitted IO.
*/
void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
{
return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
}
EXPORT_SYMBOL(writeback_inodes_sb);
/**
* try_to_writeback_inodes_sb - try to start writeback if none underway
* @sb: the superblock
* @reason: reason why some writeback work was initiated
*
* Invoke __writeback_inodes_sb_nr if no writeback is currently underway.
*/
void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
{
if (!down_read_trylock(&sb->s_umount))
return;
__writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason, true);
up_read(&sb->s_umount);
}
EXPORT_SYMBOL(try_to_writeback_inodes_sb);
/**
* sync_inodes_sb - sync sb inode pages
* @sb: the superblock
*
* This function writes and waits on any dirty inode belonging to this
* super_block.
*/
void sync_inodes_sb(struct super_block *sb)
{
struct backing_dev_info *bdi = sb->s_bdi;
DEFINE_WB_COMPLETION(done, bdi);
struct wb_writeback_work work = {
.sb = sb,
.sync_mode = WB_SYNC_ALL,
.nr_pages = LONG_MAX,
.range_cyclic = 0,
.done = &done,
.reason = WB_REASON_SYNC,
.for_sync = 1,
};
/*
* Can't skip on !bdi_has_dirty() because we should wait for !dirty
* inodes under writeback and I_DIRTY_TIME inodes ignored by
* bdi_has_dirty() need to be written out too.
*/
if (bdi == &noop_backing_dev_info)
return;
WARN_ON(!rwsem_is_locked(&sb->s_umount));
/* protect against inode wb switch, see inode_switch_wbs_work_fn() */
bdi_down_write_wb_switch_rwsem(bdi);
bdi_split_work_to_wbs(bdi, &work, false);
wb_wait_for_completion(&done);
bdi_up_write_wb_switch_rwsem(bdi);
wait_sb_inodes(sb);
}
EXPORT_SYMBOL(sync_inodes_sb);
/**
* write_inode_now - write an inode to disk
* @inode: inode to write to disk
* @sync: whether the write should be synchronous or not
*
* This function commits an inode to disk immediately if it is dirty. This is
* primarily needed by knfsd.
*
* The caller must either have a ref on the inode or must have set I_WILL_FREE.
*/
int write_inode_now(struct inode *inode, int sync)
{
struct writeback_control wbc = {
.nr_to_write = LONG_MAX,
.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
.range_start = 0,
.range_end = LLONG_MAX,
};
if (!mapping_can_writeback(inode->i_mapping))
wbc.nr_to_write = 0;
might_sleep();
return writeback_single_inode(inode, &wbc);
}
EXPORT_SYMBOL(write_inode_now);
/**
* sync_inode_metadata - write an inode to disk
* @inode: the inode to sync
* @wait: wait for I/O to complete.
*
* Write an inode to disk and adjust its dirty state after completion.
*
* Note: only writes the actual inode, no associated data or other metadata.
*/
int sync_inode_metadata(struct inode *inode, int wait)
{
struct writeback_control wbc = {
.sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
.nr_to_write = 0, /* metadata-only */
};
return writeback_single_inode(inode, &wbc);
}
EXPORT_SYMBOL(sync_inode_metadata);
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef _LINUX_XARRAY_H
#define _LINUX_XARRAY_H
/*
* eXtensible Arrays
* Copyright (c) 2017 Microsoft Corporation
* Author: Matthew Wilcox <willy@infradead.org>
*
* See Documentation/core-api/xarray.rst for how to use the XArray.
*/
#include <linux/bitmap.h>
#include <linux/bug.h>
#include <linux/compiler.h>
#include <linux/gfp.h>
#include <linux/kconfig.h>
#include <linux/kernel.h>
#include <linux/rcupdate.h>
#include <linux/sched/mm.h>
#include <linux/spinlock.h>
#include <linux/types.h>
/*
* The bottom two bits of the entry determine how the XArray interprets
* the contents:
*
* 00: Pointer entry
* 10: Internal entry
* x1: Value entry or tagged pointer
*
* Attempting to store internal entries in the XArray is a bug.
*
* Most internal entries are pointers to the next node in the tree.
* The following internal entries have a special meaning:
*
* 0-62: Sibling entries
* 256: Retry entry
* 257: Zero entry
*
* Errors are also represented as internal entries, but use the negative
* space (-4094 to -2). They're never stored in the slots array; only
* returned by the normal API.
*/
#define BITS_PER_XA_VALUE (BITS_PER_LONG - 1)
/**
* xa_mk_value() - Create an XArray entry from an integer.
* @v: Value to store in XArray.
*
* Context: Any context.
* Return: An entry suitable for storing in the XArray.
*/
static inline void *xa_mk_value(unsigned long v)
{
WARN_ON((long)v < 0);
return (void *)((v << 1) | 1);
}
/**
* xa_to_value() - Get value stored in an XArray entry.
* @entry: XArray entry.
*
* Context: Any context.
* Return: The value stored in the XArray entry.
*/
static inline unsigned long xa_to_value(const void *entry)
{
return (unsigned long)entry >> 1;
}
/**
* xa_is_value() - Determine if an entry is a value.
* @entry: XArray entry.
*
* Context: Any context.
* Return: True if the entry is a value, false if it is a pointer.
*/
static inline bool xa_is_value(const void *entry)
{
return (unsigned long)entry & 1;
}
/**
* xa_tag_pointer() - Create an XArray entry for a tagged pointer.
* @p: Plain pointer.
* @tag: Tag value (0, 1 or 3).
*
* If the user of the XArray prefers, they can tag their pointers instead
* of storing value entries. Three tags are available (0, 1 and 3).
* These are distinct from the xa_mark_t as they are not replicated up
* through the array and cannot be searched for.
*
* Context: Any context.
* Return: An XArray entry.
*/
static inline void *xa_tag_pointer(void *p, unsigned long tag)
{
return (void *)((unsigned long)p | tag);
}
/**
* xa_untag_pointer() - Turn an XArray entry into a plain pointer.
* @entry: XArray entry.
*
* If you have stored a tagged pointer in the XArray, call this function
* to get the untagged version of the pointer.
*
* Context: Any context.
* Return: A pointer.
*/
static inline void *xa_untag_pointer(void *entry)
{
return (void *)((unsigned long)entry & ~3UL);
}
/**
* xa_pointer_tag() - Get the tag stored in an XArray entry.
* @entry: XArray entry.
*
* If you have stored a tagged pointer in the XArray, call this function
* to get the tag of that pointer.
*
* Context: Any context.
* Return: A tag.
*/
static inline unsigned int xa_pointer_tag(void *entry)
{
return (unsigned long)entry & 3UL;
}
/*
* xa_mk_internal() - Create an internal entry.
* @v: Value to turn into an internal entry.
*
* Internal entries are used for a number of purposes. Entries 0-255 are
* used for sibling entries (only 0-62 are used by the current code). 256
* is used for the retry entry. 257 is used for the reserved / zero entry.
* Negative internal entries are used to represent errnos. Node pointers
* are also tagged as internal entries in some situations.
*
* Context: Any context.
* Return: An XArray internal entry corresponding to this value.
*/
static inline void *xa_mk_internal(unsigned long v)
{
return (void *)((v << 2) | 2);
}
/*
* xa_to_internal() - Extract the value from an internal entry.
* @entry: XArray entry.
*
* Context: Any context.
* Return: The value which was stored in the internal entry.
*/
static inline unsigned long xa_to_internal(const void *entry)
{
return (unsigned long)entry >> 2;
}
/*
* xa_is_internal() - Is the entry an internal entry?
* @entry: XArray entry.
*
* Context: Any context.
* Return: %true if the entry is an internal entry.
*/
static inline bool xa_is_internal(const void *entry)
{
return ((unsigned long)entry & 3) == 2;
}
#define XA_ZERO_ENTRY xa_mk_internal(257)
/**
* xa_is_zero() - Is the entry a zero entry?
* @entry: Entry retrieved from the XArray
*
* The normal API will return NULL as the contents of a slot containing
* a zero entry. You can only see zero entries by using the advanced API.
*
* Return: %true if the entry is a zero entry.
*/
static inline bool xa_is_zero(const void *entry)
{
return unlikely(entry == XA_ZERO_ENTRY);
}
/**
* xa_is_err() - Report whether an XArray operation returned an error
* @entry: Result from calling an XArray function
*
* If an XArray operation cannot complete an operation, it will return
* a special value indicating an error. This function tells you
* whether an error occurred; xa_err() tells you which error occurred.
*
* Context: Any context.
* Return: %true if the entry indicates an error.
*/
static inline bool xa_is_err(const void *entry)
{
return unlikely(xa_is_internal(entry) &&
entry >= xa_mk_internal(-MAX_ERRNO));
}
/**
* xa_err() - Turn an XArray result into an errno.
* @entry: Result from calling an XArray function.
*
* If an XArray operation cannot complete an operation, it will return
* a special pointer value which encodes an errno. This function extracts
* the errno from the pointer value, or returns 0 if the pointer does not
* represent an errno.
*
* Context: Any context.
* Return: A negative errno or 0.
*/
static inline int xa_err(void *entry)
{
/* xa_to_internal() would not do sign extension. */
if (xa_is_err(entry)) return (long)entry >> 2;
return 0;
}
/**
* struct xa_limit - Represents a range of IDs.
* @min: The lowest ID to allocate (inclusive).
* @max: The maximum ID to allocate (inclusive).
*
* This structure is used either directly or via the XA_LIMIT() macro
* to communicate the range of IDs that are valid for allocation.
* Three common ranges are predefined for you:
* * xa_limit_32b - [0 - UINT_MAX]
* * xa_limit_31b - [0 - INT_MAX]
* * xa_limit_16b - [0 - USHRT_MAX]
*/
struct xa_limit {
u32 max;
u32 min;
};
#define XA_LIMIT(_min, _max) (struct xa_limit) { .min = _min, .max = _max }
#define xa_limit_32b XA_LIMIT(0, UINT_MAX)
#define xa_limit_31b XA_LIMIT(0, INT_MAX)
#define xa_limit_16b XA_LIMIT(0, USHRT_MAX)
typedef unsigned __bitwise xa_mark_t;
#define XA_MARK_0 ((__force xa_mark_t)0U)
#define XA_MARK_1 ((__force xa_mark_t)1U)
#define XA_MARK_2 ((__force xa_mark_t)2U)
#define XA_PRESENT ((__force xa_mark_t)8U)
#define XA_MARK_MAX XA_MARK_2
#define XA_FREE_MARK XA_MARK_0
enum xa_lock_type {
XA_LOCK_IRQ = 1,
XA_LOCK_BH = 2,
};
/*
* Values for xa_flags. The radix tree stores its GFP flags in the xa_flags,
* and we remain compatible with that.
*/
#define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ)
#define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH)
#define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U)
#define XA_FLAGS_ZERO_BUSY ((__force gfp_t)8U)
#define XA_FLAGS_ALLOC_WRAPPED ((__force gfp_t)16U)
#define XA_FLAGS_ACCOUNT ((__force gfp_t)32U)
#define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \
(__force unsigned)(mark)))
/* ALLOC is for a normal 0-based alloc. ALLOC1 is for an 1-based alloc */
#define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK))
#define XA_FLAGS_ALLOC1 (XA_FLAGS_TRACK_FREE | XA_FLAGS_ZERO_BUSY)
/**
* struct xarray - The anchor of the XArray.
* @xa_lock: Lock that protects the contents of the XArray.
*
* To use the xarray, define it statically or embed it in your data structure.
* It is a very small data structure, so it does not usually make sense to
* allocate it separately and keep a pointer to it in your data structure.
*
* You may use the xa_lock to protect your own data structures as well.
*/
/*
* If all of the entries in the array are NULL, @xa_head is a NULL pointer.
* If the only non-NULL entry in the array is at index 0, @xa_head is that
* entry. If any other entry in the array is non-NULL, @xa_head points
* to an @xa_node.
*/
struct xarray {
spinlock_t xa_lock;
/* private: The rest of the data structure is not to be used directly. */
gfp_t xa_flags;
void __rcu * xa_head;
};
#define XARRAY_INIT(name, flags) { \
.xa_lock = __SPIN_LOCK_UNLOCKED(name.xa_lock), \
.xa_flags = flags, \
.xa_head = NULL, \
}
/**
* DEFINE_XARRAY_FLAGS() - Define an XArray with custom flags.
* @name: A string that names your XArray.
* @flags: XA_FLAG values.
*
* This is intended for file scope definitions of XArrays. It declares
* and initialises an empty XArray with the chosen name and flags. It is
* equivalent to calling xa_init_flags() on the array, but it does the
* initialisation at compiletime instead of runtime.
*/
#define DEFINE_XARRAY_FLAGS(name, flags) \
struct xarray name = XARRAY_INIT(name, flags)
/**
* DEFINE_XARRAY() - Define an XArray.
* @name: A string that names your XArray.
*
* This is intended for file scope definitions of XArrays. It declares
* and initialises an empty XArray with the chosen name. It is equivalent
* to calling xa_init() on the array, but it does the initialisation at
* compiletime instead of runtime.
*/
#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)
/**
* DEFINE_XARRAY_ALLOC() - Define an XArray which allocates IDs starting at 0.
* @name: A string that names your XArray.
*
* This is intended for file scope definitions of allocating XArrays.
* See also DEFINE_XARRAY().
*/
#define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC)
/**
* DEFINE_XARRAY_ALLOC1() - Define an XArray which allocates IDs starting at 1.
* @name: A string that names your XArray.
*
* This is intended for file scope definitions of allocating XArrays.
* See also DEFINE_XARRAY().
*/
#define DEFINE_XARRAY_ALLOC1(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC1)
void *xa_load(struct xarray *, unsigned long index);
void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
void *xa_erase(struct xarray *, unsigned long index);
void *xa_store_range(struct xarray *, unsigned long first, unsigned long last,
void *entry, gfp_t);
bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t);
void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
void *xa_find(struct xarray *xa, unsigned long *index,
unsigned long max, xa_mark_t) __attribute__((nonnull(2)));
void *xa_find_after(struct xarray *xa, unsigned long *index,
unsigned long max, xa_mark_t) __attribute__((nonnull(2)));
unsigned int xa_extract(struct xarray *, void **dst, unsigned long start,
unsigned long max, unsigned int n, xa_mark_t);
void xa_destroy(struct xarray *);
/**
* xa_init_flags() - Initialise an empty XArray with flags.
* @xa: XArray.
* @flags: XA_FLAG values.
*
* If you need to initialise an XArray with special flags (eg you need
* to take the lock from interrupt context), use this function instead
* of xa_init().
*
* Context: Any context.
*/
static inline void xa_init_flags(struct xarray *xa, gfp_t flags)
{
spin_lock_init(&xa->xa_lock);
xa->xa_flags = flags;
xa->xa_head = NULL;
}
/**
* xa_init() - Initialise an empty XArray.
* @xa: XArray.
*
* An empty XArray is full of NULL entries.
*
* Context: Any context.
*/
static inline void xa_init(struct xarray *xa)
{
xa_init_flags(xa, 0);
}
/**
* xa_empty() - Determine if an array has any present entries.
* @xa: XArray.
*
* Context: Any context.
* Return: %true if the array contains only NULL pointers.
*/
static inline bool xa_empty(const struct xarray *xa)
{
return xa->xa_head == NULL;
}
/**
* xa_marked() - Inquire whether any entry in this array has a mark set
* @xa: Array
* @mark: Mark value
*
* Context: Any context.
* Return: %true if any entry has this mark set.
*/
static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark)
{
return xa->xa_flags & XA_FLAGS_MARK(mark);
}
/**
* xa_for_each_range() - Iterate over a portion of an XArray.
* @xa: XArray.
* @index: Index of @entry.
* @entry: Entry retrieved from array.
* @start: First index to retrieve from array.
* @last: Last index to retrieve from array.
*
* During the iteration, @entry will have the value of the entry stored
* in @xa at @index. You may modify @index during the iteration if you
* want to skip or reprocess indices. It is safe to modify the array
* during the iteration. At the end of the iteration, @entry will be set
* to NULL and @index will have a value less than or equal to max.
*
* xa_for_each_range() is O(n.log(n)) while xas_for_each() is O(n). You have
* to handle your own locking with xas_for_each(), and if you have to unlock
* after each iteration, it will also end up being O(n.log(n)).
* xa_for_each_range() will spin if it hits a retry entry; if you intend to
* see retry entries, you should use the xas_for_each() iterator instead.
* The xas_for_each() iterator will expand into more inline code than
* xa_for_each_range().
*
* Context: Any context. Takes and releases the RCU lock.
*/
#define xa_for_each_range(xa, index, entry, start, last) \
for (index = start, \
entry = xa_find(xa, &index, last, XA_PRESENT); \
entry; \
entry = xa_find_after(xa, &index, last, XA_PRESENT))
/**
* xa_for_each_start() - Iterate over a portion of an XArray.
* @xa: XArray.
* @index: Index of @entry.
* @entry: Entry retrieved from array.
* @start: First index to retrieve from array.
*
* During the iteration, @entry will have the value of the entry stored
* in @xa at @index. You may modify @index during the iteration if you
* want to skip or reprocess indices. It is safe to modify the array
* during the iteration. At the end of the iteration, @entry will be set
* to NULL and @index will have a value less than or equal to max.
*
* xa_for_each_start() is O(n.log(n)) while xas_for_each() is O(n). You have
* to handle your own locking with xas_for_each(), and if you have to unlock
* after each iteration, it will also end up being O(n.log(n)).
* xa_for_each_start() will spin if it hits a retry entry; if you intend to
* see retry entries, you should use the xas_for_each() iterator instead.
* The xas_for_each() iterator will expand into more inline code than
* xa_for_each_start().
*
* Context: Any context. Takes and releases the RCU lock.
*/
#define xa_for_each_start(xa, index, entry, start) \
xa_for_each_range(xa, index, entry, start, ULONG_MAX)
/**
* xa_for_each() - Iterate over present entries in an XArray.
* @xa: XArray.
* @index: Index of @entry.
* @entry: Entry retrieved from array.
*
* During the iteration, @entry will have the value of the entry stored
* in @xa at @index. You may modify @index during the iteration if you want
* to skip or reprocess indices. It is safe to modify the array during the
* iteration. At the end of the iteration, @entry will be set to NULL and
* @index will have a value less than or equal to max.
*
* xa_for_each() is O(n.log(n)) while xas_for_each() is O(n). You have
* to handle your own locking with xas_for_each(), and if you have to unlock
* after each iteration, it will also end up being O(n.log(n)). xa_for_each()
* will spin if it hits a retry entry; if you intend to see retry entries,
* you should use the xas_for_each() iterator instead. The xas_for_each()
* iterator will expand into more inline code than xa_for_each().
*
* Context: Any context. Takes and releases the RCU lock.
*/
#define xa_for_each(xa, index, entry) \
xa_for_each_start(xa, index, entry, 0)
/**
* xa_for_each_marked() - Iterate over marked entries in an XArray.
* @xa: XArray.
* @index: Index of @entry.
* @entry: Entry retrieved from array.
* @filter: Selection criterion.
*
* During the iteration, @entry will have the value of the entry stored
* in @xa at @index. The iteration will skip all entries in the array
* which do not match @filter. You may modify @index during the iteration
* if you want to skip or reprocess indices. It is safe to modify the array
* during the iteration. At the end of the iteration, @entry will be set to
* NULL and @index will have a value less than or equal to max.
*
* xa_for_each_marked() is O(n.log(n)) while xas_for_each_marked() is O(n).
* You have to handle your own locking with xas_for_each(), and if you have
* to unlock after each iteration, it will also end up being O(n.log(n)).
* xa_for_each_marked() will spin if it hits a retry entry; if you intend to
* see retry entries, you should use the xas_for_each_marked() iterator
* instead. The xas_for_each_marked() iterator will expand into more inline
* code than xa_for_each_marked().
*
* Context: Any context. Takes and releases the RCU lock.
*/
#define xa_for_each_marked(xa, index, entry, filter) \
for (index = 0, entry = xa_find(xa, &index, ULONG_MAX, filter); \
entry; entry = xa_find_after(xa, &index, ULONG_MAX, filter))
#define xa_trylock(xa) spin_trylock(&(xa)->xa_lock)
#define xa_lock(xa) spin_lock(&(xa)->xa_lock)
#define xa_unlock(xa) spin_unlock(&(xa)->xa_lock)
#define xa_lock_bh(xa) spin_lock_bh(&(xa)->xa_lock)
#define xa_unlock_bh(xa) spin_unlock_bh(&(xa)->xa_lock)
#define xa_lock_irq(xa) spin_lock_irq(&(xa)->xa_lock)
#define xa_unlock_irq(xa) spin_unlock_irq(&(xa)->xa_lock)
#define xa_lock_irqsave(xa, flags) \
spin_lock_irqsave(&(xa)->xa_lock, flags)
#define xa_unlock_irqrestore(xa, flags) \
spin_unlock_irqrestore(&(xa)->xa_lock, flags)
#define xa_lock_nested(xa, subclass) \
spin_lock_nested(&(xa)->xa_lock, subclass)
#define xa_lock_bh_nested(xa, subclass) \
spin_lock_bh_nested(&(xa)->xa_lock, subclass)
#define xa_lock_irq_nested(xa, subclass) \
spin_lock_irq_nested(&(xa)->xa_lock, subclass)
#define xa_lock_irqsave_nested(xa, flags, subclass) \
spin_lock_irqsave_nested(&(xa)->xa_lock, flags, subclass)
/*
* Versions of the normal API which require the caller to hold the
* xa_lock. If the GFP flags allow it, they will drop the lock to
* allocate memory, then reacquire it afterwards. These functions
* may also re-enable interrupts if the XArray flags indicate the
* locking should be interrupt safe.
*/
void *__xa_erase(struct xarray *, unsigned long index);
void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old,
void *entry, gfp_t);
int __must_check __xa_insert(struct xarray *, unsigned long index,
void *entry, gfp_t);
int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry,
struct xa_limit, gfp_t);
int __must_check __xa_alloc_cyclic(struct xarray *, u32 *id, void *entry,
struct xa_limit, u32 *next, gfp_t);
void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
/**
* xa_store_bh() - Store this entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* This function is like calling xa_store() except it disables softirqs
* while holding the array lock.
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs.
* Return: The old entry at this index or xa_err() if an error happened.
*/
static inline void *xa_store_bh(struct xarray *xa, unsigned long index,
void *entry, gfp_t gfp)
{
void *curr;
might_alloc(gfp);
xa_lock_bh(xa);
curr = __xa_store(xa, index, entry, gfp);
xa_unlock_bh(xa);
return curr;
}
/**
* xa_store_irq() - Store this entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* This function is like calling xa_store() except it disables interrupts
* while holding the array lock.
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts.
* Return: The old entry at this index or xa_err() if an error happened.
*/
static inline void *xa_store_irq(struct xarray *xa, unsigned long index,
void *entry, gfp_t gfp)
{
void *curr;
might_alloc(gfp);
xa_lock_irq(xa);
curr = __xa_store(xa, index, entry, gfp);
xa_unlock_irq(xa);
return curr;
}
/**
* xa_erase_bh() - Erase this entry from the XArray.
* @xa: XArray.
* @index: Index of entry.
*
* After this function returns, loading from @index will return %NULL.
* If the index is part of a multi-index entry, all indices will be erased
* and none of the entries will be part of a multi-index entry.
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs.
* Return: The entry which used to be at this index.
*/
static inline void *xa_erase_bh(struct xarray *xa, unsigned long index)
{
void *entry;
xa_lock_bh(xa);
entry = __xa_erase(xa, index);
xa_unlock_bh(xa);
return entry;
}
/**
* xa_erase_irq() - Erase this entry from the XArray.
* @xa: XArray.
* @index: Index of entry.
*
* After this function returns, loading from @index will return %NULL.
* If the index is part of a multi-index entry, all indices will be erased
* and none of the entries will be part of a multi-index entry.
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts.
* Return: The entry which used to be at this index.
*/
static inline void *xa_erase_irq(struct xarray *xa, unsigned long index)
{
void *entry;
xa_lock_irq(xa);
entry = __xa_erase(xa, index);
xa_unlock_irq(xa);
return entry;
}
/**
* xa_cmpxchg() - Conditionally replace an entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @old: Old value to test against.
* @entry: New value to place in array.
* @gfp: Memory allocation flags.
*
* If the entry at @index is the same as @old, replace it with @entry.
* If the return value is equal to @old, then the exchange was successful.
*
* Context: Any context. Takes and releases the xa_lock. May sleep
* if the @gfp flags permit.
* Return: The old value at this index or xa_err() if an error happened.
*/
static inline void *xa_cmpxchg(struct xarray *xa, unsigned long index,
void *old, void *entry, gfp_t gfp)
{
void *curr;
might_alloc(gfp);
xa_lock(xa);
curr = __xa_cmpxchg(xa, index, old, entry, gfp);
xa_unlock(xa);
return curr;
}
/**
* xa_cmpxchg_bh() - Conditionally replace an entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @old: Old value to test against.
* @entry: New value to place in array.
* @gfp: Memory allocation flags.
*
* This function is like calling xa_cmpxchg() except it disables softirqs
* while holding the array lock.
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs. May sleep if the @gfp flags permit.
* Return: The old value at this index or xa_err() if an error happened.
*/
static inline void *xa_cmpxchg_bh(struct xarray *xa, unsigned long index,
void *old, void *entry, gfp_t gfp)
{
void *curr;
might_alloc(gfp);
xa_lock_bh(xa);
curr = __xa_cmpxchg(xa, index, old, entry, gfp);
xa_unlock_bh(xa);
return curr;
}
/**
* xa_cmpxchg_irq() - Conditionally replace an entry in the XArray.
* @xa: XArray.
* @index: Index into array.
* @old: Old value to test against.
* @entry: New value to place in array.
* @gfp: Memory allocation flags.
*
* This function is like calling xa_cmpxchg() except it disables interrupts
* while holding the array lock.
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts. May sleep if the @gfp flags permit.
* Return: The old value at this index or xa_err() if an error happened.
*/
static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index,
void *old, void *entry, gfp_t gfp)
{
void *curr;
might_alloc(gfp);
xa_lock_irq(xa);
curr = __xa_cmpxchg(xa, index, old, entry, gfp);
xa_unlock_irq(xa);
return curr;
}
/**
* xa_insert() - Store this entry in the XArray unless another entry is
* already present.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* Inserting a NULL entry will store a reserved entry (like xa_reserve())
* if no entry is present. Inserting will fail if a reserved entry is
* present, even though loading from this index will return NULL.
*
* Context: Any context. Takes and releases the xa_lock. May sleep if
* the @gfp flags permit.
* Return: 0 if the store succeeded. -EBUSY if another entry was present.
* -ENOMEM if memory could not be allocated.
*/
static inline int __must_check xa_insert(struct xarray *xa,
unsigned long index, void *entry, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock(xa);
err = __xa_insert(xa, index, entry, gfp);
xa_unlock(xa);
return err;
}
/**
* xa_insert_bh() - Store this entry in the XArray unless another entry is
* already present.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* Inserting a NULL entry will store a reserved entry (like xa_reserve())
* if no entry is present. Inserting will fail if a reserved entry is
* present, even though loading from this index will return NULL.
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs. May sleep if the @gfp flags permit.
* Return: 0 if the store succeeded. -EBUSY if another entry was present.
* -ENOMEM if memory could not be allocated.
*/
static inline int __must_check xa_insert_bh(struct xarray *xa,
unsigned long index, void *entry, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock_bh(xa);
err = __xa_insert(xa, index, entry, gfp);
xa_unlock_bh(xa);
return err;
}
/**
* xa_insert_irq() - Store this entry in the XArray unless another entry is
* already present.
* @xa: XArray.
* @index: Index into array.
* @entry: New entry.
* @gfp: Memory allocation flags.
*
* Inserting a NULL entry will store a reserved entry (like xa_reserve())
* if no entry is present. Inserting will fail if a reserved entry is
* present, even though loading from this index will return NULL.
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts. May sleep if the @gfp flags permit.
* Return: 0 if the store succeeded. -EBUSY if another entry was present.
* -ENOMEM if memory could not be allocated.
*/
static inline int __must_check xa_insert_irq(struct xarray *xa,
unsigned long index, void *entry, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock_irq(xa);
err = __xa_insert(xa, index, entry, gfp);
xa_unlock_irq(xa);
return err;
}
/**
* xa_alloc() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Any context. Takes and releases the xa_lock. May sleep if
* the @gfp flags permit.
* Return: 0 on success, -ENOMEM if memory could not be allocated or
* -EBUSY if there are no free entries in @limit.
*/
static inline __must_check int xa_alloc(struct xarray *xa, u32 *id,
void *entry, struct xa_limit limit, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock(xa);
err = __xa_alloc(xa, id, entry, limit, gfp);
xa_unlock(xa);
return err;
}
/**
* xa_alloc_bh() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs. May sleep if the @gfp flags permit.
* Return: 0 on success, -ENOMEM if memory could not be allocated or
* -EBUSY if there are no free entries in @limit.
*/
static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id,
void *entry, struct xa_limit limit, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock_bh(xa);
err = __xa_alloc(xa, id, entry, limit, gfp);
xa_unlock_bh(xa);
return err;
}
/**
* xa_alloc_irq() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts. May sleep if the @gfp flags permit.
* Return: 0 on success, -ENOMEM if memory could not be allocated or
* -EBUSY if there are no free entries in @limit.
*/
static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id,
void *entry, struct xa_limit limit, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock_irq(xa);
err = __xa_alloc(xa, id, entry, limit, gfp);
xa_unlock_irq(xa);
return err;
}
/**
* xa_alloc_cyclic() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of allocated ID.
* @next: Pointer to next ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
* The search for an empty entry will start at @next and will wrap
* around if necessary.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Any context. Takes and releases the xa_lock. May sleep if
* the @gfp flags permit.
* Return: 0 if the allocation succeeded without wrapping. 1 if the
* allocation succeeded after wrapping, -ENOMEM if memory could not be
* allocated or -EBUSY if there are no free entries in @limit.
*/
static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
struct xa_limit limit, u32 *next, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock(xa);
err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
xa_unlock(xa);
return err;
}
/**
* xa_alloc_cyclic_bh() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of allocated ID.
* @next: Pointer to next ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
* The search for an empty entry will start at @next and will wrap
* around if necessary.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs. May sleep if the @gfp flags permit.
* Return: 0 if the allocation succeeded without wrapping. 1 if the
* allocation succeeded after wrapping, -ENOMEM if memory could not be
* allocated or -EBUSY if there are no free entries in @limit.
*/
static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry,
struct xa_limit limit, u32 *next, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock_bh(xa);
err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
xa_unlock_bh(xa);
return err;
}
/**
* xa_alloc_cyclic_irq() - Find somewhere to store this entry in the XArray.
* @xa: XArray.
* @id: Pointer to ID.
* @entry: New entry.
* @limit: Range of allocated ID.
* @next: Pointer to next ID to allocate.
* @gfp: Memory allocation flags.
*
* Finds an empty entry in @xa between @limit.min and @limit.max,
* stores the index into the @id pointer, then stores the entry at
* that index. A concurrent lookup will not see an uninitialised @id.
* The search for an empty entry will start at @next and will wrap
* around if necessary.
*
* Must only be operated on an xarray initialized with flag XA_FLAGS_ALLOC set
* in xa_init_flags().
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts. May sleep if the @gfp flags permit.
* Return: 0 if the allocation succeeded without wrapping. 1 if the
* allocation succeeded after wrapping, -ENOMEM if memory could not be
* allocated or -EBUSY if there are no free entries in @limit.
*/
static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry,
struct xa_limit limit, u32 *next, gfp_t gfp)
{
int err;
might_alloc(gfp);
xa_lock_irq(xa);
err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
xa_unlock_irq(xa);
return err;
}
/**
* xa_reserve() - Reserve this index in the XArray.
* @xa: XArray.
* @index: Index into array.
* @gfp: Memory allocation flags.
*
* Ensures there is somewhere to store an entry at @index in the array.
* If there is already something stored at @index, this function does
* nothing. If there was nothing there, the entry is marked as reserved.
* Loading from a reserved entry returns a %NULL pointer.
*
* If you do not use the entry that you have reserved, call xa_release()
* or xa_erase() to free any unnecessary memory.
*
* Context: Any context. Takes and releases the xa_lock.
* May sleep if the @gfp flags permit.
* Return: 0 if the reservation succeeded or -ENOMEM if it failed.
*/
static inline __must_check
int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
{
return xa_err(xa_cmpxchg(xa, index, NULL, XA_ZERO_ENTRY, gfp));
}
/**
* xa_reserve_bh() - Reserve this index in the XArray.
* @xa: XArray.
* @index: Index into array.
* @gfp: Memory allocation flags.
*
* A softirq-disabling version of xa_reserve().
*
* Context: Any context. Takes and releases the xa_lock while
* disabling softirqs.
* Return: 0 if the reservation succeeded or -ENOMEM if it failed.
*/
static inline __must_check
int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp)
{
return xa_err(xa_cmpxchg_bh(xa, index, NULL, XA_ZERO_ENTRY, gfp));
}
/**
* xa_reserve_irq() - Reserve this index in the XArray.
* @xa: XArray.
* @index: Index into array.
* @gfp: Memory allocation flags.
*
* An interrupt-disabling version of xa_reserve().
*
* Context: Process context. Takes and releases the xa_lock while
* disabling interrupts.
* Return: 0 if the reservation succeeded or -ENOMEM if it failed.
*/
static inline __must_check
int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp)
{
return xa_err(xa_cmpxchg_irq(xa, index, NULL, XA_ZERO_ENTRY, gfp));
}
/**
* xa_release() - Release a reserved entry.
* @xa: XArray.
* @index: Index of entry.
*
* After calling xa_reserve(), you can call this function to release the
* reservation. If the entry at @index has been stored to, this function
* will do nothing.
*/
static inline void xa_release(struct xarray *xa, unsigned long index)
{
xa_cmpxchg(xa, index, XA_ZERO_ENTRY, NULL, 0);
}
/* Everything below here is the Advanced API. Proceed with caution. */
/*
* The xarray is constructed out of a set of 'chunks' of pointers. Choosing
* the best chunk size requires some tradeoffs. A power of two recommends
* itself so that we can walk the tree based purely on shifts and masks.
* Generally, the larger the better; as the number of slots per level of the
* tree increases, the less tall the tree needs to be. But that needs to be
* balanced against the memory consumption of each node. On a 64-bit system,
* xa_node is currently 576 bytes, and we get 7 of them per 4kB page. If we
* doubled the number of slots per node, we'd get only 3 nodes per 4kB page.
*/
#ifndef XA_CHUNK_SHIFT
#define XA_CHUNK_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
#endif
#define XA_CHUNK_SIZE (1UL << XA_CHUNK_SHIFT)
#define XA_CHUNK_MASK (XA_CHUNK_SIZE - 1)
#define XA_MAX_MARKS 3
#define XA_MARK_LONGS DIV_ROUND_UP(XA_CHUNK_SIZE, BITS_PER_LONG)
/*
* @count is the count of every non-NULL element in the ->slots array
* whether that is a value entry, a retry entry, a user pointer,
* a sibling entry or a pointer to the next level of the tree.
* @nr_values is the count of every element in ->slots which is
* either a value entry or a sibling of a value entry.
*/
struct xa_node {
unsigned char shift; /* Bits remaining in each slot */
unsigned char offset; /* Slot offset in parent */
unsigned char count; /* Total entry count */
unsigned char nr_values; /* Value entry count */
struct xa_node __rcu *parent; /* NULL at top of tree */
struct xarray *array; /* The array we belong to */
union {
struct list_head private_list; /* For tree user */
struct rcu_head rcu_head; /* Used when freeing node */
};
void __rcu *slots[XA_CHUNK_SIZE];
union {
unsigned long tags[XA_MAX_MARKS][XA_MARK_LONGS];
unsigned long marks[XA_MAX_MARKS][XA_MARK_LONGS];
};
};
void xa_dump(const struct xarray *);
void xa_dump_node(const struct xa_node *);
#ifdef XA_DEBUG
#define XA_BUG_ON(xa, x) do { \
if (x) { \
xa_dump(xa); \
BUG(); \
} \
} while (0)
#define XA_NODE_BUG_ON(node, x) do { \
if (x) { \
if (node) xa_dump_node(node); \
BUG(); \
} \
} while (0)
#else
#define XA_BUG_ON(xa, x) do { } while (0)
#define XA_NODE_BUG_ON(node, x) do { } while (0)
#endif
/* Private */
static inline void *xa_head(const struct xarray *xa)
{
return rcu_dereference_check(xa->xa_head,
lockdep_is_held(&xa->xa_lock));
}
/* Private */
static inline void *xa_head_locked(const struct xarray *xa)
{
return rcu_dereference_protected(xa->xa_head,
lockdep_is_held(&xa->xa_lock));
}
/* Private */
static inline void *xa_entry(const struct xarray *xa,
const struct xa_node *node, unsigned int offset)
{
XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE);
return rcu_dereference_check(node->slots[offset],
lockdep_is_held(&xa->xa_lock));
}
/* Private */
static inline void *xa_entry_locked(const struct xarray *xa,
const struct xa_node *node, unsigned int offset)
{
XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE);
return rcu_dereference_protected(node->slots[offset],
lockdep_is_held(&xa->xa_lock));
}
/* Private */
static inline struct xa_node *xa_parent(const struct xarray *xa,
const struct xa_node *node)
{
return rcu_dereference_check(node->parent,
lockdep_is_held(&xa->xa_lock));
}
/* Private */
static inline struct xa_node *xa_parent_locked(const struct xarray *xa,
const struct xa_node *node)
{
return rcu_dereference_protected(node->parent,
lockdep_is_held(&xa->xa_lock));
}
/* Private */
static inline void *xa_mk_node(const struct xa_node *node)
{
return (void *)((unsigned long)node | 2);
}
/* Private */
static inline struct xa_node *xa_to_node(const void *entry)
{
return (struct xa_node *)((unsigned long)entry - 2);
}
/* Private */
static inline bool xa_is_node(const void *entry)
{
return xa_is_internal(entry) && (unsigned long)entry > 4096;
}
/* Private */
static inline void *xa_mk_sibling(unsigned int offset)
{
return xa_mk_internal(offset);
}
/* Private */
static inline unsigned long xa_to_sibling(const void *entry)
{
return xa_to_internal(entry);
}
/**
* xa_is_sibling() - Is the entry a sibling entry?
* @entry: Entry retrieved from the XArray
*
* Return: %true if the entry is a sibling entry.
*/
static inline bool xa_is_sibling(const void *entry)
{
return IS_ENABLED(CONFIG_XARRAY_MULTI) && xa_is_internal(entry) &&
(entry < xa_mk_sibling(XA_CHUNK_SIZE - 1));
}
#define XA_RETRY_ENTRY xa_mk_internal(256)
/**
* xa_is_retry() - Is the entry a retry entry?
* @entry: Entry retrieved from the XArray
*
* Return: %true if the entry is a retry entry.
*/
static inline bool xa_is_retry(const void *entry)
{
return unlikely(entry == XA_RETRY_ENTRY);
}
/**
* xa_is_advanced() - Is the entry only permitted for the advanced API?
* @entry: Entry to be stored in the XArray.
*
* Return: %true if the entry cannot be stored by the normal API.
*/
static inline bool xa_is_advanced(const void *entry)
{
return xa_is_internal(entry) && (entry <= XA_RETRY_ENTRY);
}
/**
* typedef xa_update_node_t - A callback function from the XArray.
* @node: The node which is being processed
*
* This function is called every time the XArray updates the count of
* present and value entries in a node. It allows advanced users to
* maintain the private_list in the node.
*
* Context: The xa_lock is held and interrupts may be disabled.
* Implementations should not drop the xa_lock, nor re-enable
* interrupts.
*/
typedef void (*xa_update_node_t)(struct xa_node *node);
void xa_delete_node(struct xa_node *, xa_update_node_t);
/*
* The xa_state is opaque to its users. It contains various different pieces
* of state involved in the current operation on the XArray. It should be
* declared on the stack and passed between the various internal routines.
* The various elements in it should not be accessed directly, but only
* through the provided accessor functions. The below documentation is for
* the benefit of those working on the code, not for users of the XArray.
*
* @xa_node usually points to the xa_node containing the slot we're operating
* on (and @xa_offset is the offset in the slots array). If there is a
* single entry in the array at index 0, there are no allocated xa_nodes to
* point to, and so we store %NULL in @xa_node. @xa_node is set to
* the value %XAS_RESTART if the xa_state is not walked to the correct
* position in the tree of nodes for this operation. If an error occurs
* during an operation, it is set to an %XAS_ERROR value. If we run off the
* end of the allocated nodes, it is set to %XAS_BOUNDS.
*/
struct xa_state {
struct xarray *xa;
unsigned long xa_index;
unsigned char xa_shift;
unsigned char xa_sibs;
unsigned char xa_offset;
unsigned char xa_pad; /* Helps gcc generate better code */
struct xa_node *xa_node;
struct xa_node *xa_alloc;
xa_update_node_t xa_update;
struct list_lru *xa_lru;
};
/*
* We encode errnos in the xas->xa_node. If an error has happened, we need to
* drop the lock to fix it, and once we've done so the xa_state is invalid.
*/
#define XA_ERROR(errno) ((struct xa_node *)(((unsigned long)errno << 2) | 2UL))
#define XAS_BOUNDS ((struct xa_node *)1UL)
#define XAS_RESTART ((struct xa_node *)3UL)
#define __XA_STATE(array, index, shift, sibs) { \
.xa = array, \
.xa_index = index, \
.xa_shift = shift, \
.xa_sibs = sibs, \
.xa_offset = 0, \
.xa_pad = 0, \
.xa_node = XAS_RESTART, \
.xa_alloc = NULL, \
.xa_update = NULL, \
.xa_lru = NULL, \
}
/**
* XA_STATE() - Declare an XArray operation state.
* @name: Name of this operation state (usually xas).
* @array: Array to operate on.
* @index: Initial index of interest.
*
* Declare and initialise an xa_state on the stack.
*/
#define XA_STATE(name, array, index) \
struct xa_state name = __XA_STATE(array, index, 0, 0)
/**
* XA_STATE_ORDER() - Declare an XArray operation state.
* @name: Name of this operation state (usually xas).
* @array: Array to operate on.
* @index: Initial index of interest.
* @order: Order of entry.
*
* Declare and initialise an xa_state on the stack. This variant of
* XA_STATE() allows you to specify the 'order' of the element you
* want to operate on.`
*/
#define XA_STATE_ORDER(name, array, index, order) \
struct xa_state name = __XA_STATE(array, \
(index >> order) << order, \
order - (order % XA_CHUNK_SHIFT), \
(1U << (order % XA_CHUNK_SHIFT)) - 1)
#define xas_marked(xas, mark) xa_marked((xas)->xa, (mark))
#define xas_trylock(xas) xa_trylock((xas)->xa)
#define xas_lock(xas) xa_lock((xas)->xa)
#define xas_unlock(xas) xa_unlock((xas)->xa)
#define xas_lock_bh(xas) xa_lock_bh((xas)->xa)
#define xas_unlock_bh(xas) xa_unlock_bh((xas)->xa)
#define xas_lock_irq(xas) xa_lock_irq((xas)->xa)
#define xas_unlock_irq(xas) xa_unlock_irq((xas)->xa)
#define xas_lock_irqsave(xas, flags) \
xa_lock_irqsave((xas)->xa, flags)
#define xas_unlock_irqrestore(xas, flags) \
xa_unlock_irqrestore((xas)->xa, flags)
/**
* xas_error() - Return an errno stored in the xa_state.
* @xas: XArray operation state.
*
* Return: 0 if no error has been noted. A negative errno if one has.
*/
static inline int xas_error(const struct xa_state *xas)
{
return xa_err(xas->xa_node);
}
/**
* xas_set_err() - Note an error in the xa_state.
* @xas: XArray operation state.
* @err: Negative error number.
*
* Only call this function with a negative @err; zero or positive errors
* will probably not behave the way you think they should. If you want
* to clear the error from an xa_state, use xas_reset().
*/
static inline void xas_set_err(struct xa_state *xas, long err)
{
xas->xa_node = XA_ERROR(err);
}
/**
* xas_invalid() - Is the xas in a retry or error state?
* @xas: XArray operation state.
*
* Return: %true if the xas cannot be used for operations.
*/
static inline bool xas_invalid(const struct xa_state *xas)
{
return (unsigned long)xas->xa_node & 3;
}
/**
* xas_valid() - Is the xas a valid cursor into the array?
* @xas: XArray operation state.
*
* Return: %true if the xas can be used for operations.
*/
static inline bool xas_valid(const struct xa_state *xas)
{
return !xas_invalid(xas);
}
/**
* xas_is_node() - Does the xas point to a node?
* @xas: XArray operation state.
*
* Return: %true if the xas currently references a node.
*/
static inline bool xas_is_node(const struct xa_state *xas)
{
return xas_valid(xas) && xas->xa_node;
}
/* True if the pointer is something other than a node */
static inline bool xas_not_node(struct xa_node *node)
{
return ((unsigned long)node & 3) || !node;
}
/* True if the node represents RESTART or an error */
static inline bool xas_frozen(struct xa_node *node)
{
return (unsigned long)node & 2;
}
/* True if the node represents head-of-tree, RESTART or BOUNDS */
static inline bool xas_top(struct xa_node *node)
{
return node <= XAS_RESTART;
}
/**
* xas_reset() - Reset an XArray operation state.
* @xas: XArray operation state.
*
* Resets the error or walk state of the @xas so future walks of the
* array will start from the root. Use this if you have dropped the
* xarray lock and want to reuse the xa_state.
*
* Context: Any context.
*/
static inline void xas_reset(struct xa_state *xas)
{
xas->xa_node = XAS_RESTART;
}
/**
* xas_retry() - Retry the operation if appropriate.
* @xas: XArray operation state.
* @entry: Entry from xarray.
*
* The advanced functions may sometimes return an internal entry, such as
* a retry entry or a zero entry. This function sets up the @xas to restart
* the walk from the head of the array if needed.
*
* Context: Any context.
* Return: true if the operation needs to be retried.
*/
static inline bool xas_retry(struct xa_state *xas, const void *entry)
{
if (xa_is_zero(entry))
return true;
if (!xa_is_retry(entry))
return false;
xas_reset(xas);
return true;
}
void *xas_load(struct xa_state *);
void *xas_store(struct xa_state *, void *entry);
void *xas_find(struct xa_state *, unsigned long max);
void *xas_find_conflict(struct xa_state *);
bool xas_get_mark(const struct xa_state *, xa_mark_t);
void xas_set_mark(const struct xa_state *, xa_mark_t);
void xas_clear_mark(const struct xa_state *, xa_mark_t);
void *xas_find_marked(struct xa_state *, unsigned long max, xa_mark_t);
void xas_init_marks(const struct xa_state *);
bool xas_nomem(struct xa_state *, gfp_t);
void xas_destroy(struct xa_state *);
void xas_pause(struct xa_state *);
void xas_create_range(struct xa_state *);
#ifdef CONFIG_XARRAY_MULTI
int xa_get_order(struct xarray *, unsigned long index);
void xas_split(struct xa_state *, void *entry, unsigned int order);
void xas_split_alloc(struct xa_state *, void *entry, unsigned int order, gfp_t);
#else
static inline int xa_get_order(struct xarray *xa, unsigned long index)
{
return 0;
}
static inline void xas_split(struct xa_state *xas, void *entry,
unsigned int order)
{
xas_store(xas, entry);
}
static inline void xas_split_alloc(struct xa_state *xas, void *entry,
unsigned int order, gfp_t gfp)
{
}
#endif
/**
* xas_reload() - Refetch an entry from the xarray.
* @xas: XArray operation state.
*
* Use this function to check that a previously loaded entry still has
* the same value. This is useful for the lockless pagecache lookup where
* we walk the array with only the RCU lock to protect us, lock the page,
* then check that the page hasn't moved since we looked it up.
*
* The caller guarantees that @xas is still valid. If it may be in an
* error or restart state, call xas_load() instead.
*
* Return: The entry at this location in the xarray.
*/
static inline void *xas_reload(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;
void *entry;
char offset;
if (!node) return xa_head(xas->xa);
if (IS_ENABLED(CONFIG_XARRAY_MULTI)) {
offset = (xas->xa_index >> node->shift) & XA_CHUNK_MASK; entry = xa_entry(xas->xa, node, offset); if (!xa_is_sibling(entry))
return entry;
offset = xa_to_sibling(entry);
} else {
offset = xas->xa_offset;
}
return xa_entry(xas->xa, node, offset);
}
/**
* xas_set() - Set up XArray operation state for a different index.
* @xas: XArray operation state.
* @index: New index into the XArray.
*
* Move the operation state to refer to a different index. This will
* have the effect of starting a walk from the top; see xas_next()
* to move to an adjacent index.
*/
static inline void xas_set(struct xa_state *xas, unsigned long index)
{
xas->xa_index = index;
xas->xa_node = XAS_RESTART;
}
/**
* xas_advance() - Skip over sibling entries.
* @xas: XArray operation state.
* @index: Index of last sibling entry.
*
* Move the operation state to refer to the last sibling entry.
* This is useful for loops that normally want to see sibling
* entries but sometimes want to skip them. Use xas_set() if you
* want to move to an index which is not part of this entry.
*/
static inline void xas_advance(struct xa_state *xas, unsigned long index)
{
unsigned char shift = xas_is_node(xas) ? xas->xa_node->shift : 0;
xas->xa_index = index;
xas->xa_offset = (index >> shift) & XA_CHUNK_MASK;
}
/**
* xas_set_order() - Set up XArray operation state for a multislot entry.
* @xas: XArray operation state.
* @index: Target of the operation.
* @order: Entry occupies 2^@order indices.
*/
static inline void xas_set_order(struct xa_state *xas, unsigned long index,
unsigned int order)
{
#ifdef CONFIG_XARRAY_MULTI
xas->xa_index = order < BITS_PER_LONG ? (index >> order) << order : 0;
xas->xa_shift = order - (order % XA_CHUNK_SHIFT);
xas->xa_sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1;
xas->xa_node = XAS_RESTART;
#else
BUG_ON(order > 0);
xas_set(xas, index);
#endif
}
/**
* xas_set_update() - Set up XArray operation state for a callback.
* @xas: XArray operation state.
* @update: Function to call when updating a node.
*
* The XArray can notify a caller after it has updated an xa_node.
* This is advanced functionality and is only needed by the page
* cache and swap cache.
*/
static inline void xas_set_update(struct xa_state *xas, xa_update_node_t update)
{
xas->xa_update = update;
}
static inline void xas_set_lru(struct xa_state *xas, struct list_lru *lru)
{
xas->xa_lru = lru;
}
/**
* xas_next_entry() - Advance iterator to next present entry.
* @xas: XArray operation state.
* @max: Highest index to return.
*
* xas_next_entry() is an inline function to optimise xarray traversal for
* speed. It is equivalent to calling xas_find(), and will call xas_find()
* for all the hard cases.
*
* Return: The next present entry after the one currently referred to by @xas.
*/
static inline void *xas_next_entry(struct xa_state *xas, unsigned long max)
{
struct xa_node *node = xas->xa_node;
void *entry;
if (unlikely(xas_not_node(node) || node->shift ||
xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)))
return xas_find(xas, max);
do {
if (unlikely(xas->xa_index >= max))
return xas_find(xas, max);
if (unlikely(xas->xa_offset == XA_CHUNK_MASK))
return xas_find(xas, max);
entry = xa_entry(xas->xa, node, xas->xa_offset + 1); if (unlikely(xa_is_internal(entry))) return xas_find(xas, max); xas->xa_offset++;
xas->xa_index++;
} while (!entry);
return entry;
}
/* Private */
static inline unsigned int xas_find_chunk(struct xa_state *xas, bool advance,
xa_mark_t mark)
{
unsigned long *addr = xas->xa_node->marks[(__force unsigned)mark];
unsigned int offset = xas->xa_offset;
if (advance)
offset++;
if (XA_CHUNK_SIZE == BITS_PER_LONG) {
if (offset < XA_CHUNK_SIZE) { unsigned long data = *addr & (~0UL << offset);
if (data)
return __ffs(data);
}
return XA_CHUNK_SIZE;
}
return find_next_bit(addr, XA_CHUNK_SIZE, offset);
}
/**
* xas_next_marked() - Advance iterator to next marked entry.
* @xas: XArray operation state.
* @max: Highest index to return.
* @mark: Mark to search for.
*
* xas_next_marked() is an inline function to optimise xarray traversal for
* speed. It is equivalent to calling xas_find_marked(), and will call
* xas_find_marked() for all the hard cases.
*
* Return: The next marked entry after the one currently referred to by @xas.
*/
static inline void *xas_next_marked(struct xa_state *xas, unsigned long max,
xa_mark_t mark)
{
struct xa_node *node = xas->xa_node;
void *entry;
unsigned int offset;
if (unlikely(xas_not_node(node) || node->shift))
return xas_find_marked(xas, max, mark);
offset = xas_find_chunk(xas, true, mark);
xas->xa_offset = offset;
xas->xa_index = (xas->xa_index & ~XA_CHUNK_MASK) + offset;
if (xas->xa_index > max)
return NULL;
if (offset == XA_CHUNK_SIZE)
return xas_find_marked(xas, max, mark);
entry = xa_entry(xas->xa, node, offset);
if (!entry)
return xas_find_marked(xas, max, mark);
return entry;
}
/*
* If iterating while holding a lock, drop the lock and reschedule
* every %XA_CHECK_SCHED loops.
*/
enum {
XA_CHECK_SCHED = 4096,
};
/**
* xas_for_each() - Iterate over a range of an XArray.
* @xas: XArray operation state.
* @entry: Entry retrieved from the array.
* @max: Maximum index to retrieve from array.
*
* The loop body will be executed for each entry present in the xarray
* between the current xas position and @max. @entry will be set to
* the entry retrieved from the xarray. It is safe to delete entries
* from the array in the loop body. You should hold either the RCU lock
* or the xa_lock while iterating. If you need to drop the lock, call
* xas_pause() first.
*/
#define xas_for_each(xas, entry, max) \
for (entry = xas_find(xas, max); entry; \
entry = xas_next_entry(xas, max))
/**
* xas_for_each_marked() - Iterate over a range of an XArray.
* @xas: XArray operation state.
* @entry: Entry retrieved from the array.
* @max: Maximum index to retrieve from array.
* @mark: Mark to search for.
*
* The loop body will be executed for each marked entry in the xarray
* between the current xas position and @max. @entry will be set to
* the entry retrieved from the xarray. It is safe to delete entries
* from the array in the loop body. You should hold either the RCU lock
* or the xa_lock while iterating. If you need to drop the lock, call
* xas_pause() first.
*/
#define xas_for_each_marked(xas, entry, max, mark) \
for (entry = xas_find_marked(xas, max, mark); entry; \
entry = xas_next_marked(xas, max, mark))
/**
* xas_for_each_conflict() - Iterate over a range of an XArray.
* @xas: XArray operation state.
* @entry: Entry retrieved from the array.
*
* The loop body will be executed for each entry in the XArray that
* lies within the range specified by @xas. If the loop terminates
* normally, @entry will be %NULL. The user may break out of the loop,
* which will leave @entry set to the conflicting entry. The caller
* may also call xa_set_err() to exit the loop while setting an error
* to record the reason.
*/
#define xas_for_each_conflict(xas, entry) \
while ((entry = xas_find_conflict(xas)))
void *__xas_next(struct xa_state *);
void *__xas_prev(struct xa_state *);
/**
* xas_prev() - Move iterator to previous index.
* @xas: XArray operation state.
*
* If the @xas was in an error state, it will remain in an error state
* and this function will return %NULL. If the @xas has never been walked,
* it will have the effect of calling xas_load(). Otherwise one will be
* subtracted from the index and the state will be walked to the correct
* location in the array for the next operation.
*
* If the iterator was referencing index 0, this function wraps
* around to %ULONG_MAX.
*
* Return: The entry at the new index. This may be %NULL or an internal
* entry.
*/
static inline void *xas_prev(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;
if (unlikely(xas_not_node(node) || node->shift ||
xas->xa_offset == 0))
return __xas_prev(xas);
xas->xa_index--;
xas->xa_offset--;
return xa_entry(xas->xa, node, xas->xa_offset);
}
/**
* xas_next() - Move state to next index.
* @xas: XArray operation state.
*
* If the @xas was in an error state, it will remain in an error state
* and this function will return %NULL. If the @xas has never been walked,
* it will have the effect of calling xas_load(). Otherwise one will be
* added to the index and the state will be walked to the correct
* location in the array for the next operation.
*
* If the iterator was referencing index %ULONG_MAX, this function wraps
* around to 0.
*
* Return: The entry at the new index. This may be %NULL or an internal
* entry.
*/
static inline void *xas_next(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;
if (unlikely(xas_not_node(node) || node->shift ||
xas->xa_offset == XA_CHUNK_MASK))
return __xas_next(xas);
xas->xa_index++;
xas->xa_offset++;
return xa_entry(xas->xa, node, xas->xa_offset);
}
#endif /* _LINUX_XARRAY_H */
// SPDX-License-Identifier: GPL-2.0-only
/*
* Generic pidhash and scalable, time-bounded PID allocator
*
* (C) 2002-2003 Nadia Yvette Chambers, IBM
* (C) 2004 Nadia Yvette Chambers, Oracle
* (C) 2002-2004 Ingo Molnar, Red Hat
*
* pid-structures are backing objects for tasks sharing a given ID to chain
* against. There is very little to them aside from hashing them and
* parking tasks using given ID's on a list.
*
* The hash is always changed with the tasklist_lock write-acquired,
* and the hash is only accessed with the tasklist_lock at least
* read-acquired, so there's no additional SMP locking needed here.
*
* We have a list of bitmap pages, which bitmaps represent the PID space.
* Allocating and freeing PIDs is completely lockless. The worst-case
* allocation scenario when all but one out of 1 million PIDs possible are
* allocated already: the scanning of 32 list entries and at most PAGE_SIZE
* bytes. The typical fastpath is a single successful setbit. Freeing is O(1).
*
* Pid namespaces:
* (C) 2007 Pavel Emelyanov <xemul@openvz.org>, OpenVZ, SWsoft Inc.
* (C) 2007 Sukadev Bhattiprolu <sukadev@us.ibm.com>, IBM
* Many thanks to Oleg Nesterov for comments and help
*
*/
#include <linux/mm.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/rculist.h>
#include <linux/memblock.h>
#include <linux/pid_namespace.h>
#include <linux/init_task.h>
#include <linux/syscalls.h>
#include <linux/proc_ns.h>
#include <linux/refcount.h>
#include <linux/anon_inodes.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/idr.h>
#include <linux/pidfs.h>
#include <net/sock.h>
#include <uapi/linux/pidfd.h>
struct pid init_struct_pid = {
.count = REFCOUNT_INIT(1),
.tasks = {
{ .first = NULL },
{ .first = NULL },
{ .first = NULL },
},
.level = 0,
.numbers = { {
.nr = 0,
.ns = &init_pid_ns,
}, }
};
int pid_max = PID_MAX_DEFAULT;
int pid_max_min = RESERVED_PIDS + 1;
int pid_max_max = PID_MAX_LIMIT;
/*
* Pseudo filesystems start inode numbering after one. We use Reserved
* PIDs as a natural offset.
*/
static u64 pidfs_ino = RESERVED_PIDS;
/*
* PID-map pages start out as NULL, they get allocated upon
* first use and are never deallocated. This way a low pid_max
* value does not cause lots of bitmaps to be allocated, but
* the scheme scales to up to 4 million PIDs, runtime.
*/
struct pid_namespace init_pid_ns = {
.ns.count = REFCOUNT_INIT(2),
.idr = IDR_INIT(init_pid_ns.idr),
.pid_allocated = PIDNS_ADDING,
.level = 0,
.child_reaper = &init_task,
.user_ns = &init_user_ns,
.ns.inum = PROC_PID_INIT_INO,
#ifdef CONFIG_PID_NS
.ns.ops = &pidns_operations,
#endif
#if defined(CONFIG_SYSCTL) && defined(CONFIG_MEMFD_CREATE)
.memfd_noexec_scope = MEMFD_NOEXEC_SCOPE_EXEC,
#endif
};
EXPORT_SYMBOL_GPL(init_pid_ns);
/*
* Note: disable interrupts while the pidmap_lock is held as an
* interrupt might come in and do read_lock(&tasklist_lock).
*
* If we don't disable interrupts there is a nasty deadlock between
* detach_pid()->free_pid() and another cpu that does
* spin_lock(&pidmap_lock) followed by an interrupt routine that does
* read_lock(&tasklist_lock);
*
* After we clean up the tasklist_lock and know there are no
* irq handlers that take it we can leave the interrupts enabled.
* For now it is easier to be safe than to prove it can't happen.
*/
static __cacheline_aligned_in_smp DEFINE_SPINLOCK(pidmap_lock);
void put_pid(struct pid *pid)
{
struct pid_namespace *ns;
if (!pid)
return;
ns = pid->numbers[pid->level].ns;
if (refcount_dec_and_test(&pid->count)) {
kmem_cache_free(ns->pid_cachep, pid);
put_pid_ns(ns);
}
}
EXPORT_SYMBOL_GPL(put_pid);
static void delayed_put_pid(struct rcu_head *rhp)
{
struct pid *pid = container_of(rhp, struct pid, rcu);
put_pid(pid);
}
void free_pid(struct pid *pid)
{
/* We can be called with write_lock_irq(&tasklist_lock) held */
int i;
unsigned long flags;
spin_lock_irqsave(&pidmap_lock, flags);
for (i = 0; i <= pid->level; i++) {
struct upid *upid = pid->numbers + i;
struct pid_namespace *ns = upid->ns;
switch (--ns->pid_allocated) {
case 2:
case 1:
/* When all that is left in the pid namespace
* is the reaper wake up the reaper. The reaper
* may be sleeping in zap_pid_ns_processes().
*/
wake_up_process(ns->child_reaper);
break;
case PIDNS_ADDING:
/* Handle a fork failure of the first process */
WARN_ON(ns->child_reaper);
ns->pid_allocated = 0;
break;
}
idr_remove(&ns->idr, upid->nr);
}
spin_unlock_irqrestore(&pidmap_lock, flags);
call_rcu(&pid->rcu, delayed_put_pid);
}
struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
size_t set_tid_size)
{
struct pid *pid;
enum pid_type type;
int i, nr;
struct pid_namespace *tmp;
struct upid *upid;
int retval = -ENOMEM;
/*
* set_tid_size contains the size of the set_tid array. Starting at
* the most nested currently active PID namespace it tells alloc_pid()
* which PID to set for a process in that most nested PID namespace
* up to set_tid_size PID namespaces. It does not have to set the PID
* for a process in all nested PID namespaces but set_tid_size must
* never be greater than the current ns->level + 1.
*/
if (set_tid_size > ns->level + 1)
return ERR_PTR(-EINVAL);
pid = kmem_cache_alloc(ns->pid_cachep, GFP_KERNEL);
if (!pid)
return ERR_PTR(retval);
tmp = ns;
pid->level = ns->level;
for (i = ns->level; i >= 0; i--) {
int tid = 0;
if (set_tid_size) {
tid = set_tid[ns->level - i];
retval = -EINVAL;
if (tid < 1 || tid >= pid_max)
goto out_free;
/*
* Also fail if a PID != 1 is requested and
* no PID 1 exists.
*/
if (tid != 1 && !tmp->child_reaper)
goto out_free;
retval = -EPERM;
if (!checkpoint_restore_ns_capable(tmp->user_ns))
goto out_free;
set_tid_size--;
}
idr_preload(GFP_KERNEL);
spin_lock_irq(&pidmap_lock);
if (tid) {
nr = idr_alloc(&tmp->idr, NULL, tid,
tid + 1, GFP_ATOMIC);
/*
* If ENOSPC is returned it means that the PID is
* alreay in use. Return EEXIST in that case.
*/
if (nr == -ENOSPC)
nr = -EEXIST;
} else {
int pid_min = 1;
/*
* init really needs pid 1, but after reaching the
* maximum wrap back to RESERVED_PIDS
*/
if (idr_get_cursor(&tmp->idr) > RESERVED_PIDS)
pid_min = RESERVED_PIDS;
/*
* Store a null pointer so find_pid_ns does not find
* a partially initialized PID (see below).
*/
nr = idr_alloc_cyclic(&tmp->idr, NULL, pid_min,
pid_max, GFP_ATOMIC);
}
spin_unlock_irq(&pidmap_lock);
idr_preload_end();
if (nr < 0) {
retval = (nr == -ENOSPC) ? -EAGAIN : nr;
goto out_free;
}
pid->numbers[i].nr = nr;
pid->numbers[i].ns = tmp;
tmp = tmp->parent;
}
/*
* ENOMEM is not the most obvious choice especially for the case
* where the child subreaper has already exited and the pid
* namespace denies the creation of any new processes. But ENOMEM
* is what we have exposed to userspace for a long time and it is
* documented behavior for pid namespaces. So we can't easily
* change it even if there were an error code better suited.
*/
retval = -ENOMEM;
get_pid_ns(ns);
refcount_set(&pid->count, 1);
spin_lock_init(&pid->lock);
for (type = 0; type < PIDTYPE_MAX; ++type)
INIT_HLIST_HEAD(&pid->tasks[type]);
init_waitqueue_head(&pid->wait_pidfd);
INIT_HLIST_HEAD(&pid->inodes);
upid = pid->numbers + ns->level;
spin_lock_irq(&pidmap_lock);
if (!(ns->pid_allocated & PIDNS_ADDING))
goto out_unlock;
pid->stashed = NULL;
pid->ino = ++pidfs_ino;
for ( ; upid >= pid->numbers; --upid) {
/* Make the PID visible to find_pid_ns. */
idr_replace(&upid->ns->idr, pid, upid->nr);
upid->ns->pid_allocated++;
}
spin_unlock_irq(&pidmap_lock);
return pid;
out_unlock:
spin_unlock_irq(&pidmap_lock);
put_pid_ns(ns);
out_free:
spin_lock_irq(&pidmap_lock);
while (++i <= ns->level) {
upid = pid->numbers + i;
idr_remove(&upid->ns->idr, upid->nr);
}
/* On failure to allocate the first pid, reset the state */
if (ns->pid_allocated == PIDNS_ADDING)
idr_set_cursor(&ns->idr, 0);
spin_unlock_irq(&pidmap_lock);
kmem_cache_free(ns->pid_cachep, pid);
return ERR_PTR(retval);
}
void disable_pid_allocation(struct pid_namespace *ns)
{
spin_lock_irq(&pidmap_lock);
ns->pid_allocated &= ~PIDNS_ADDING;
spin_unlock_irq(&pidmap_lock);
}
struct pid *find_pid_ns(int nr, struct pid_namespace *ns)
{
return idr_find(&ns->idr, nr);
}
EXPORT_SYMBOL_GPL(find_pid_ns);
struct pid *find_vpid(int nr)
{
return find_pid_ns(nr, task_active_pid_ns(current));
}
EXPORT_SYMBOL_GPL(find_vpid);
static struct pid **task_pid_ptr(struct task_struct *task, enum pid_type type)
{
return (type == PIDTYPE_PID) ?
&task->thread_pid : &task->signal->pids[type];
}
/*
* attach_pid() must be called with the tasklist_lock write-held.
*/
void attach_pid(struct task_struct *task, enum pid_type type)
{
struct pid *pid = *task_pid_ptr(task, type);
hlist_add_head_rcu(&task->pid_links[type], &pid->tasks[type]);
}
static void __change_pid(struct task_struct *task, enum pid_type type,
struct pid *new)
{
struct pid **pid_ptr = task_pid_ptr(task, type);
struct pid *pid;
int tmp;
pid = *pid_ptr;
hlist_del_rcu(&task->pid_links[type]);
*pid_ptr = new;
if (type == PIDTYPE_PID) {
WARN_ON_ONCE(pid_has_task(pid, PIDTYPE_PID));
wake_up_all(&pid->wait_pidfd);
}
for (tmp = PIDTYPE_MAX; --tmp >= 0; )
if (pid_has_task(pid, tmp))
return;
free_pid(pid);
}
void detach_pid(struct task_struct *task, enum pid_type type)
{
__change_pid(task, type, NULL);
}
void change_pid(struct task_struct *task, enum pid_type type,
struct pid *pid)
{
__change_pid(task, type, pid);
attach_pid(task, type);
}
void exchange_tids(struct task_struct *left, struct task_struct *right)
{
struct pid *pid1 = left->thread_pid;
struct pid *pid2 = right->thread_pid;
struct hlist_head *head1 = &pid1->tasks[PIDTYPE_PID];
struct hlist_head *head2 = &pid2->tasks[PIDTYPE_PID];
/* Swap the single entry tid lists */
hlists_swap_heads_rcu(head1, head2);
/* Swap the per task_struct pid */
rcu_assign_pointer(left->thread_pid, pid2);
rcu_assign_pointer(right->thread_pid, pid1);
/* Swap the cached value */
WRITE_ONCE(left->pid, pid_nr(pid2));
WRITE_ONCE(right->pid, pid_nr(pid1));
}
/* transfer_pid is an optimization of attach_pid(new), detach_pid(old) */
void transfer_pid(struct task_struct *old, struct task_struct *new,
enum pid_type type)
{
WARN_ON_ONCE(type == PIDTYPE_PID);
hlist_replace_rcu(&old->pid_links[type], &new->pid_links[type]);
}
struct task_struct *pid_task(struct pid *pid, enum pid_type type)
{
struct task_struct *result = NULL;
if (pid) {
struct hlist_node *first;
first = rcu_dereference_check(hlist_first_rcu(&pid->tasks[type]),
lockdep_tasklist_lock_is_held());
if (first)
result = hlist_entry(first, struct task_struct, pid_links[(type)]);
}
return result;
}
EXPORT_SYMBOL(pid_task);
/*
* Must be called under rcu_read_lock().
*/
struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns)
{
RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
"find_task_by_pid_ns() needs rcu_read_lock() protection");
return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID);
}
struct task_struct *find_task_by_vpid(pid_t vnr)
{
return find_task_by_pid_ns(vnr, task_active_pid_ns(current));
}
struct task_struct *find_get_task_by_vpid(pid_t nr)
{
struct task_struct *task;
rcu_read_lock();
task = find_task_by_vpid(nr);
if (task)
get_task_struct(task);
rcu_read_unlock();
return task;
}
struct pid *get_task_pid(struct task_struct *task, enum pid_type type)
{
struct pid *pid;
rcu_read_lock();
pid = get_pid(rcu_dereference(*task_pid_ptr(task, type)));
rcu_read_unlock();
return pid;
}
EXPORT_SYMBOL_GPL(get_task_pid);
struct task_struct *get_pid_task(struct pid *pid, enum pid_type type)
{
struct task_struct *result;
rcu_read_lock();
result = pid_task(pid, type);
if (result)
get_task_struct(result);
rcu_read_unlock();
return result;
}
EXPORT_SYMBOL_GPL(get_pid_task);
struct pid *find_get_pid(pid_t nr)
{
struct pid *pid;
rcu_read_lock();
pid = get_pid(find_vpid(nr));
rcu_read_unlock();
return pid;
}
EXPORT_SYMBOL_GPL(find_get_pid);
pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns)
{
struct upid *upid;
pid_t nr = 0;
if (pid && ns->level <= pid->level) {
upid = &pid->numbers[ns->level];
if (upid->ns == ns) nr = upid->nr;
}
return nr;
}
EXPORT_SYMBOL_GPL(pid_nr_ns);
pid_t pid_vnr(struct pid *pid)
{
return pid_nr_ns(pid, task_active_pid_ns(current));
}
EXPORT_SYMBOL_GPL(pid_vnr);
pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type,
struct pid_namespace *ns)
{
pid_t nr = 0;
rcu_read_lock(); if (!ns) ns = task_active_pid_ns(current); nr = pid_nr_ns(rcu_dereference(*task_pid_ptr(task, type)), ns); rcu_read_unlock();
return nr;
}
EXPORT_SYMBOL(__task_pid_nr_ns);
struct pid_namespace *task_active_pid_ns(struct task_struct *tsk)
{
return ns_of_pid(task_pid(tsk));
}
EXPORT_SYMBOL_GPL(task_active_pid_ns);
/*
* Used by proc to find the first pid that is greater than or equal to nr.
*
* If there is a pid at nr this function is exactly the same as find_pid_ns.
*/
struct pid *find_ge_pid(int nr, struct pid_namespace *ns)
{
return idr_get_next(&ns->idr, &nr);
}
EXPORT_SYMBOL_GPL(find_ge_pid);
struct pid *pidfd_get_pid(unsigned int fd, unsigned int *flags)
{
struct fd f;
struct pid *pid;
f = fdget(fd);
if (!f.file)
return ERR_PTR(-EBADF);
pid = pidfd_pid(f.file);
if (!IS_ERR(pid)) {
get_pid(pid);
*flags = f.file->f_flags;
}
fdput(f);
return pid;
}
/**
* pidfd_get_task() - Get the task associated with a pidfd
*
* @pidfd: pidfd for which to get the task
* @flags: flags associated with this pidfd
*
* Return the task associated with @pidfd. The function takes a reference on
* the returned task. The caller is responsible for releasing that reference.
*
* Return: On success, the task_struct associated with the pidfd.
* On error, a negative errno number will be returned.
*/
struct task_struct *pidfd_get_task(int pidfd, unsigned int *flags)
{
unsigned int f_flags;
struct pid *pid;
struct task_struct *task;
pid = pidfd_get_pid(pidfd, &f_flags);
if (IS_ERR(pid))
return ERR_CAST(pid);
task = get_pid_task(pid, PIDTYPE_TGID);
put_pid(pid);
if (!task)
return ERR_PTR(-ESRCH);
*flags = f_flags;
return task;
}
/**
* pidfd_create() - Create a new pid file descriptor.
*
* @pid: struct pid that the pidfd will reference
* @flags: flags to pass
*
* This creates a new pid file descriptor with the O_CLOEXEC flag set.
*
* Note, that this function can only be called after the fd table has
* been unshared to avoid leaking the pidfd to the new process.
*
* This symbol should not be explicitly exported to loadable modules.
*
* Return: On success, a cloexec pidfd is returned.
* On error, a negative errno number will be returned.
*/
static int pidfd_create(struct pid *pid, unsigned int flags)
{
int pidfd;
struct file *pidfd_file;
pidfd = pidfd_prepare(pid, flags, &pidfd_file);
if (pidfd < 0)
return pidfd;
fd_install(pidfd, pidfd_file);
return pidfd;
}
/**
* sys_pidfd_open() - Open new pid file descriptor.
*
* @pid: pid for which to retrieve a pidfd
* @flags: flags to pass
*
* This creates a new pid file descriptor with the O_CLOEXEC flag set for
* the task identified by @pid. Without PIDFD_THREAD flag the target task
* must be a thread-group leader.
*
* Return: On success, a cloexec pidfd is returned.
* On error, a negative errno number will be returned.
*/
SYSCALL_DEFINE2(pidfd_open, pid_t, pid, unsigned int, flags)
{
int fd;
struct pid *p;
if (flags & ~(PIDFD_NONBLOCK | PIDFD_THREAD))
return -EINVAL;
if (pid <= 0)
return -EINVAL;
p = find_get_pid(pid);
if (!p)
return -ESRCH;
fd = pidfd_create(p, flags);
put_pid(p);
return fd;
}
void __init pid_idr_init(void)
{
/* Verify no one has done anything silly: */
BUILD_BUG_ON(PID_MAX_LIMIT >= PIDNS_ADDING);
/* bump default and minimum pid_max based on number of cpus */
pid_max = min(pid_max_max, max_t(int, pid_max,
PIDS_PER_CPU_DEFAULT * num_possible_cpus()));
pid_max_min = max_t(int, pid_max_min,
PIDS_PER_CPU_MIN * num_possible_cpus());
pr_info("pid_max: default: %u minimum: %u\n", pid_max, pid_max_min);
idr_init(&init_pid_ns.idr);
init_pid_ns.pid_cachep = kmem_cache_create("pid",
struct_size_t(struct pid, numbers, 1),
__alignof__(struct pid),
SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT,
NULL);
}
static struct file *__pidfd_fget(struct task_struct *task, int fd)
{
struct file *file;
int ret;
ret = down_read_killable(&task->signal->exec_update_lock);
if (ret)
return ERR_PTR(ret);
if (ptrace_may_access(task, PTRACE_MODE_ATTACH_REALCREDS))
file = fget_task(task, fd);
else
file = ERR_PTR(-EPERM);
up_read(&task->signal->exec_update_lock);
if (!file) {
/*
* It is possible that the target thread is exiting; it can be
* either:
* 1. before exit_signals(), which gives a real fd
* 2. before exit_files() takes the task_lock() gives a real fd
* 3. after exit_files() releases task_lock(), ->files is NULL;
* this has PF_EXITING, since it was set in exit_signals(),
* __pidfd_fget() returns EBADF.
* In case 3 we get EBADF, but that really means ESRCH, since
* the task is currently exiting and has freed its files
* struct, so we fix it up.
*/
if (task->flags & PF_EXITING)
file = ERR_PTR(-ESRCH);
else
file = ERR_PTR(-EBADF);
}
return file;
}
static int pidfd_getfd(struct pid *pid, int fd)
{
struct task_struct *task;
struct file *file;
int ret;
task = get_pid_task(pid, PIDTYPE_PID);
if (!task)
return -ESRCH;
file = __pidfd_fget(task, fd);
put_task_struct(task);
if (IS_ERR(file))
return PTR_ERR(file);
ret = receive_fd(file, NULL, O_CLOEXEC);
fput(file);
return ret;
}
/**
* sys_pidfd_getfd() - Get a file descriptor from another process
*
* @pidfd: the pidfd file descriptor of the process
* @fd: the file descriptor number to get
* @flags: flags on how to get the fd (reserved)
*
* This syscall gets a copy of a file descriptor from another process
* based on the pidfd, and file descriptor number. It requires that
* the calling process has the ability to ptrace the process represented
* by the pidfd. The process which is having its file descriptor copied
* is otherwise unaffected.
*
* Return: On success, a cloexec file descriptor is returned.
* On error, a negative errno number will be returned.
*/
SYSCALL_DEFINE3(pidfd_getfd, int, pidfd, int, fd,
unsigned int, flags)
{
struct pid *pid;
struct fd f;
int ret;
/* flags is currently unused - make sure it's unset */
if (flags)
return -EINVAL;
f = fdget(pidfd);
if (!f.file)
return -EBADF;
pid = pidfd_pid(f.file);
if (IS_ERR(pid))
ret = PTR_ERR(pid);
else
ret = pidfd_getfd(pid, fd);
fdput(f);
return ret;
}
// SPDX-License-Identifier: GPL-2.0
/*
* Fast batching percpu counters.
*/
#include <linux/percpu_counter.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/module.h>
#include <linux/debugobjects.h>
#ifdef CONFIG_HOTPLUG_CPU
static LIST_HEAD(percpu_counters);
static DEFINE_SPINLOCK(percpu_counters_lock);
#endif
#ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
static const struct debug_obj_descr percpu_counter_debug_descr;
static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
{
struct percpu_counter *fbc = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
percpu_counter_destroy(fbc);
debug_object_free(fbc, &percpu_counter_debug_descr);
return true;
default:
return false;
}
}
static const struct debug_obj_descr percpu_counter_debug_descr = {
.name = "percpu_counter",
.fixup_free = percpu_counter_fixup_free,
};
static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
{
debug_object_init(fbc, &percpu_counter_debug_descr);
debug_object_activate(fbc, &percpu_counter_debug_descr);
}
static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
{
debug_object_deactivate(fbc, &percpu_counter_debug_descr);
debug_object_free(fbc, &percpu_counter_debug_descr);
}
#else /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
{ }
static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
{ }
#endif /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
{
int cpu;
unsigned long flags;
raw_spin_lock_irqsave(&fbc->lock, flags);
for_each_possible_cpu(cpu) {
s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
*pcount = 0;
}
fbc->count = amount;
raw_spin_unlock_irqrestore(&fbc->lock, flags);
}
EXPORT_SYMBOL(percpu_counter_set);
/*
* local_irq_save() is needed to make the function irq safe:
* - The slow path would be ok as protected by an irq-safe spinlock.
* - this_cpu_add would be ok as it is irq-safe by definition.
* But:
* The decision slow path/fast path and the actual update must be atomic, too.
* Otherwise a call in process context could check the current values and
* decide that the fast path can be used. If now an interrupt occurs before
* the this_cpu_add(), and the interrupt updates this_cpu(*fbc->counters),
* then the this_cpu_add() that is executed after the interrupt has completed
* can produce values larger than "batch" or even overflows.
*/
void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
{
s64 count;
unsigned long flags;
local_irq_save(flags); count = __this_cpu_read(*fbc->counters) + amount;
if (abs(count) >= batch) {
raw_spin_lock(&fbc->lock);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count - amount);
raw_spin_unlock(&fbc->lock);
} else {
this_cpu_add(*fbc->counters, amount);
}
local_irq_restore(flags);
}
EXPORT_SYMBOL(percpu_counter_add_batch);
/*
* For percpu_counter with a big batch, the devication of its count could
* be big, and there is requirement to reduce the deviation, like when the
* counter's batch could be runtime decreased to get a better accuracy,
* which can be achieved by running this sync function on each CPU.
*/
void percpu_counter_sync(struct percpu_counter *fbc)
{
unsigned long flags;
s64 count;
raw_spin_lock_irqsave(&fbc->lock, flags);
count = __this_cpu_read(*fbc->counters);
fbc->count += count;
__this_cpu_sub(*fbc->counters, count);
raw_spin_unlock_irqrestore(&fbc->lock, flags);
}
EXPORT_SYMBOL(percpu_counter_sync);
/*
* Add up all the per-cpu counts, return the result. This is a more accurate
* but much slower version of percpu_counter_read_positive().
*
* We use the cpu mask of (cpu_online_mask | cpu_dying_mask) to capture sums
* from CPUs that are in the process of being taken offline. Dying cpus have
* been removed from the online mask, but may not have had the hotplug dead
* notifier called to fold the percpu count back into the global counter sum.
* By including dying CPUs in the iteration mask, we avoid this race condition
* so __percpu_counter_sum() just does the right thing when CPUs are being taken
* offline.
*/
s64 __percpu_counter_sum(struct percpu_counter *fbc)
{
s64 ret;
int cpu;
unsigned long flags;
raw_spin_lock_irqsave(&fbc->lock, flags);
ret = fbc->count;
for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) { s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
ret += *pcount;
}
raw_spin_unlock_irqrestore(&fbc->lock, flags);
return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);
int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
gfp_t gfp, u32 nr_counters,
struct lock_class_key *key)
{
unsigned long flags __maybe_unused;
size_t counter_size;
s32 __percpu *counters;
u32 i;
counter_size = ALIGN(sizeof(*counters), __alignof__(*counters));
counters = __alloc_percpu_gfp(nr_counters * counter_size,
__alignof__(*counters), gfp);
if (!counters) {
fbc[0].counters = NULL;
return -ENOMEM;
}
for (i = 0; i < nr_counters; i++) {
raw_spin_lock_init(&fbc[i].lock);
lockdep_set_class(&fbc[i].lock, key);
#ifdef CONFIG_HOTPLUG_CPU
INIT_LIST_HEAD(&fbc[i].list);
#endif
fbc[i].count = amount;
fbc[i].counters = (void *)counters + (i * counter_size);
debug_percpu_counter_activate(&fbc[i]);
}
#ifdef CONFIG_HOTPLUG_CPU
spin_lock_irqsave(&percpu_counters_lock, flags);
for (i = 0; i < nr_counters; i++)
list_add(&fbc[i].list, &percpu_counters);
spin_unlock_irqrestore(&percpu_counters_lock, flags);
#endif
return 0;
}
EXPORT_SYMBOL(__percpu_counter_init_many);
void percpu_counter_destroy_many(struct percpu_counter *fbc, u32 nr_counters)
{
unsigned long flags __maybe_unused;
u32 i;
if (WARN_ON_ONCE(!fbc))
return;
if (!fbc[0].counters)
return;
for (i = 0; i < nr_counters; i++) debug_percpu_counter_deactivate(&fbc[i]);
#ifdef CONFIG_HOTPLUG_CPU
spin_lock_irqsave(&percpu_counters_lock, flags);
for (i = 0; i < nr_counters; i++)
list_del(&fbc[i].list); spin_unlock_irqrestore(&percpu_counters_lock, flags);
#endif
free_percpu(fbc[0].counters);
for (i = 0; i < nr_counters; i++)
fbc[i].counters = NULL;
}
EXPORT_SYMBOL(percpu_counter_destroy_many);
int percpu_counter_batch __read_mostly = 32;
EXPORT_SYMBOL(percpu_counter_batch);
static int compute_batch_value(unsigned int cpu)
{
int nr = num_online_cpus();
percpu_counter_batch = max(32, nr*2);
return 0;
}
static int percpu_counter_cpu_dead(unsigned int cpu)
{
#ifdef CONFIG_HOTPLUG_CPU
struct percpu_counter *fbc;
compute_batch_value(cpu);
spin_lock_irq(&percpu_counters_lock);
list_for_each_entry(fbc, &percpu_counters, list) {
s32 *pcount;
raw_spin_lock(&fbc->lock);
pcount = per_cpu_ptr(fbc->counters, cpu);
fbc->count += *pcount;
*pcount = 0;
raw_spin_unlock(&fbc->lock);
}
spin_unlock_irq(&percpu_counters_lock);
#endif
return 0;
}
/*
* Compare counter against given value.
* Return 1 if greater, 0 if equal and -1 if less
*/
int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
{
s64 count;
count = percpu_counter_read(fbc);
/* Check to see if rough count will be sufficient for comparison */
if (abs(count - rhs) > (batch * num_online_cpus())) {
if (count > rhs)
return 1;
else
return -1;
}
/* Need to use precise count */
count = percpu_counter_sum(fbc);
if (count > rhs)
return 1;
else if (count < rhs)
return -1;
else
return 0;
}
EXPORT_SYMBOL(__percpu_counter_compare);
/*
* Compare counter, and add amount if total is: less than or equal to limit if
* amount is positive, or greater than or equal to limit if amount is negative.
* Return true if amount is added, or false if total would be beyond the limit.
*
* Negative limit is allowed, but unusual.
* When negative amounts (subs) are given to percpu_counter_limited_add(),
* the limit would most naturally be 0 - but other limits are also allowed.
*
* Overflow beyond S64_MAX is not allowed for: counter, limit and amount
* are all assumed to be sane (far from S64_MIN and S64_MAX).
*/
bool __percpu_counter_limited_add(struct percpu_counter *fbc,
s64 limit, s64 amount, s32 batch)
{
s64 count;
s64 unknown;
unsigned long flags;
bool good = false;
if (amount == 0)
return true;
local_irq_save(flags); unknown = batch * num_online_cpus();
count = __this_cpu_read(*fbc->counters);
/* Skip taking the lock when safe */
if (abs(count + amount) <= batch &&
((amount > 0 && fbc->count + unknown <= limit) || (amount < 0 && fbc->count - unknown >= limit))) { this_cpu_add(*fbc->counters, amount); local_irq_restore(flags);
return true;
}
raw_spin_lock(&fbc->lock);
count = fbc->count + amount;
/* Skip percpu_counter_sum() when safe */
if (amount > 0) {
if (count - unknown > limit)
goto out;
if (count + unknown <= limit)
good = true;
} else {
if (count + unknown < limit)
goto out;
if (count - unknown >= limit)
good = true;
}
if (!good) {
s32 *pcount;
int cpu;
for_each_cpu_or(cpu, cpu_online_mask, cpu_dying_mask) { pcount = per_cpu_ptr(fbc->counters, cpu);
count += *pcount;
}
if (amount > 0) { if (count > limit)
goto out;
} else {
if (count < limit)
goto out;
}
good = true;
}
count = __this_cpu_read(*fbc->counters);
fbc->count += count + amount;
__this_cpu_sub(*fbc->counters, count);
out:
raw_spin_unlock(&fbc->lock); local_irq_restore(flags);
return good;
}
static int __init percpu_counter_startup(void)
{
int ret;
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
compute_batch_value, NULL);
WARN_ON(ret < 0);
ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
"lib/percpu_cnt:dead", NULL,
percpu_counter_cpu_dead);
WARN_ON(ret < 0);
return 0;
}
module_init(percpu_counter_startup);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Routines for driver control interface
* Copyright (c) by Jaroslav Kysela <perex@perex.cz>
*/
#include <linux/threads.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/time.h>
#include <linux/mm.h>
#include <linux/math64.h>
#include <linux/sched/signal.h>
#include <sound/core.h>
#include <sound/minors.h>
#include <sound/info.h>
#include <sound/control.h>
// Max allocation size for user controls.
static int max_user_ctl_alloc_size = 8 * 1024 * 1024;
module_param_named(max_user_ctl_alloc_size, max_user_ctl_alloc_size, int, 0444);
MODULE_PARM_DESC(max_user_ctl_alloc_size, "Max allocation size for user controls");
#define MAX_CONTROL_COUNT 1028
struct snd_kctl_ioctl {
struct list_head list; /* list of all ioctls */
snd_kctl_ioctl_func_t fioctl;
};
static DECLARE_RWSEM(snd_ioctl_rwsem);
static DECLARE_RWSEM(snd_ctl_layer_rwsem);
static LIST_HEAD(snd_control_ioctls);
#ifdef CONFIG_COMPAT
static LIST_HEAD(snd_control_compat_ioctls);
#endif
static struct snd_ctl_layer_ops *snd_ctl_layer;
static int snd_ctl_remove_locked(struct snd_card *card,
struct snd_kcontrol *kcontrol);
static int snd_ctl_open(struct inode *inode, struct file *file)
{
struct snd_card *card;
struct snd_ctl_file *ctl;
int i, err;
err = stream_open(inode, file);
if (err < 0)
return err;
card = snd_lookup_minor_data(iminor(inode), SNDRV_DEVICE_TYPE_CONTROL);
if (!card) {
err = -ENODEV;
goto __error1;
}
err = snd_card_file_add(card, file);
if (err < 0) {
err = -ENODEV;
goto __error1;
}
if (!try_module_get(card->module)) {
err = -EFAULT;
goto __error2;
}
ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
if (ctl == NULL) {
err = -ENOMEM;
goto __error;
}
INIT_LIST_HEAD(&ctl->events);
init_waitqueue_head(&ctl->change_sleep);
spin_lock_init(&ctl->read_lock);
ctl->card = card;
for (i = 0; i < SND_CTL_SUBDEV_ITEMS; i++)
ctl->preferred_subdevice[i] = -1;
ctl->pid = get_pid(task_pid(current));
file->private_data = ctl;
scoped_guard(write_lock_irqsave, &card->ctl_files_rwlock)
list_add_tail(&ctl->list, &card->ctl_files);
snd_card_unref(card);
return 0;
__error:
module_put(card->module);
__error2:
snd_card_file_remove(card, file);
__error1:
if (card)
snd_card_unref(card);
return err;
}
static void snd_ctl_empty_read_queue(struct snd_ctl_file * ctl)
{
struct snd_kctl_event *cread;
guard(spinlock_irqsave)(&ctl->read_lock);
while (!list_empty(&ctl->events)) {
cread = snd_kctl_event(ctl->events.next);
list_del(&cread->list);
kfree(cread);
}
}
static int snd_ctl_release(struct inode *inode, struct file *file)
{
struct snd_card *card;
struct snd_ctl_file *ctl;
struct snd_kcontrol *control;
unsigned int idx;
ctl = file->private_data;
file->private_data = NULL;
card = ctl->card;
scoped_guard(write_lock_irqsave, &card->ctl_files_rwlock)
list_del(&ctl->list);
scoped_guard(rwsem_write, &card->controls_rwsem) {
list_for_each_entry(control, &card->controls, list)
for (idx = 0; idx < control->count; idx++)
if (control->vd[idx].owner == ctl)
control->vd[idx].owner = NULL;
}
snd_fasync_free(ctl->fasync);
snd_ctl_empty_read_queue(ctl);
put_pid(ctl->pid);
kfree(ctl);
module_put(card->module);
snd_card_file_remove(card, file);
return 0;
}
/**
* snd_ctl_notify - Send notification to user-space for a control change
* @card: the card to send notification
* @mask: the event mask, SNDRV_CTL_EVENT_*
* @id: the ctl element id to send notification
*
* This function adds an event record with the given id and mask, appends
* to the list and wakes up the user-space for notification. This can be
* called in the atomic context.
*/
void snd_ctl_notify(struct snd_card *card, unsigned int mask,
struct snd_ctl_elem_id *id)
{
struct snd_ctl_file *ctl;
struct snd_kctl_event *ev;
if (snd_BUG_ON(!card || !id))
return;
if (card->shutdown)
return;
guard(read_lock_irqsave)(&card->ctl_files_rwlock);
#if IS_ENABLED(CONFIG_SND_MIXER_OSS)
card->mixer_oss_change_count++;
#endif
list_for_each_entry(ctl, &card->ctl_files, list) { if (!ctl->subscribed)
continue;
scoped_guard(spinlock, &ctl->read_lock) { list_for_each_entry(ev, &ctl->events, list) { if (ev->id.numid == id->numid) { ev->mask |= mask;
goto _found;
}
}
ev = kzalloc(sizeof(*ev), GFP_ATOMIC);
if (ev) {
ev->id = *id;
ev->mask = mask;
list_add_tail(&ev->list, &ctl->events);
} else {
dev_err(card->dev, "No memory available to allocate event\n");
}
_found:
wake_up(&ctl->change_sleep);
}
snd_kill_fasync(ctl->fasync, SIGIO, POLL_IN);
}
}
EXPORT_SYMBOL(snd_ctl_notify);
/**
* snd_ctl_notify_one - Send notification to user-space for a control change
* @card: the card to send notification
* @mask: the event mask, SNDRV_CTL_EVENT_*
* @kctl: the pointer with the control instance
* @ioff: the additional offset to the control index
*
* This function calls snd_ctl_notify() and does additional jobs
* like LED state changes.
*/
void snd_ctl_notify_one(struct snd_card *card, unsigned int mask,
struct snd_kcontrol *kctl, unsigned int ioff)
{
struct snd_ctl_elem_id id = kctl->id;
struct snd_ctl_layer_ops *lops;
id.index += ioff;
id.numid += ioff;
snd_ctl_notify(card, mask, &id);
guard(rwsem_read)(&snd_ctl_layer_rwsem);
for (lops = snd_ctl_layer; lops; lops = lops->next)
lops->lnotify(card, mask, kctl, ioff);
}
EXPORT_SYMBOL(snd_ctl_notify_one);
/**
* snd_ctl_new - create a new control instance with some elements
* @kctl: the pointer to store new control instance
* @count: the number of elements in this control
* @access: the default access flags for elements in this control
* @file: given when locking these elements
*
* Allocates a memory object for a new control instance. The instance has
* elements as many as the given number (@count). Each element has given
* access permissions (@access). Each element is locked when @file is given.
*
* Return: 0 on success, error code on failure
*/
static int snd_ctl_new(struct snd_kcontrol **kctl, unsigned int count,
unsigned int access, struct snd_ctl_file *file)
{
unsigned int idx;
if (count == 0 || count > MAX_CONTROL_COUNT)
return -EINVAL;
*kctl = kzalloc(struct_size(*kctl, vd, count), GFP_KERNEL);
if (!*kctl)
return -ENOMEM;
for (idx = 0; idx < count; idx++) {
(*kctl)->vd[idx].access = access;
(*kctl)->vd[idx].owner = file;
}
(*kctl)->count = count;
return 0;
}
/**
* snd_ctl_new1 - create a control instance from the template
* @ncontrol: the initialization record
* @private_data: the private data to set
*
* Allocates a new struct snd_kcontrol instance and initialize from the given
* template. When the access field of ncontrol is 0, it's assumed as
* READWRITE access. When the count field is 0, it's assumes as one.
*
* Return: The pointer of the newly generated instance, or %NULL on failure.
*/
struct snd_kcontrol *snd_ctl_new1(const struct snd_kcontrol_new *ncontrol,
void *private_data)
{
struct snd_kcontrol *kctl;
unsigned int count;
unsigned int access;
int err;
if (snd_BUG_ON(!ncontrol || !ncontrol->info))
return NULL;
count = ncontrol->count;
if (count == 0)
count = 1;
access = ncontrol->access;
if (access == 0)
access = SNDRV_CTL_ELEM_ACCESS_READWRITE;
access &= (SNDRV_CTL_ELEM_ACCESS_READWRITE |
SNDRV_CTL_ELEM_ACCESS_VOLATILE |
SNDRV_CTL_ELEM_ACCESS_INACTIVE |
SNDRV_CTL_ELEM_ACCESS_TLV_READWRITE |
SNDRV_CTL_ELEM_ACCESS_TLV_COMMAND |
SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK |
SNDRV_CTL_ELEM_ACCESS_LED_MASK |
SNDRV_CTL_ELEM_ACCESS_SKIP_CHECK);
err = snd_ctl_new(&kctl, count, access, NULL);
if (err < 0)
return NULL;
/* The 'numid' member is decided when calling snd_ctl_add(). */
kctl->id.iface = ncontrol->iface;
kctl->id.device = ncontrol->device;
kctl->id.subdevice = ncontrol->subdevice;
if (ncontrol->name) {
strscpy(kctl->id.name, ncontrol->name, sizeof(kctl->id.name));
if (strcmp(ncontrol->name, kctl->id.name) != 0)
pr_warn("ALSA: Control name '%s' truncated to '%s'\n",
ncontrol->name, kctl->id.name);
}
kctl->id.index = ncontrol->index;
kctl->info = ncontrol->info;
kctl->get = ncontrol->get;
kctl->put = ncontrol->put;
kctl->tlv.p = ncontrol->tlv.p;
kctl->private_value = ncontrol->private_value;
kctl->private_data = private_data;
return kctl;
}
EXPORT_SYMBOL(snd_ctl_new1);
/**
* snd_ctl_free_one - release the control instance
* @kcontrol: the control instance
*
* Releases the control instance created via snd_ctl_new()
* or snd_ctl_new1().
* Don't call this after the control was added to the card.
*/
void snd_ctl_free_one(struct snd_kcontrol *kcontrol)
{
if (kcontrol) {
if (kcontrol->private_free) kcontrol->private_free(kcontrol); kfree(kcontrol);
}
}
EXPORT_SYMBOL(snd_ctl_free_one);
static bool snd_ctl_remove_numid_conflict(struct snd_card *card,
unsigned int count)
{
struct snd_kcontrol *kctl;
/* Make sure that the ids assigned to the control do not wrap around */
if (card->last_numid >= UINT_MAX - count)
card->last_numid = 0;
list_for_each_entry(kctl, &card->controls, list) {
if (kctl->id.numid < card->last_numid + 1 + count &&
kctl->id.numid + kctl->count > card->last_numid + 1) {
card->last_numid = kctl->id.numid + kctl->count - 1;
return true;
}
}
return false;
}
static int snd_ctl_find_hole(struct snd_card *card, unsigned int count)
{
unsigned int iter = 100000;
while (snd_ctl_remove_numid_conflict(card, count)) {
if (--iter == 0) {
/* this situation is very unlikely */
dev_err(card->dev, "unable to allocate new control numid\n");
return -ENOMEM;
}
}
return 0;
}
/* check whether the given id is contained in the given kctl */
static bool elem_id_matches(const struct snd_kcontrol *kctl,
const struct snd_ctl_elem_id *id)
{
return kctl->id.iface == id->iface &&
kctl->id.device == id->device &&
kctl->id.subdevice == id->subdevice &&
!strncmp(kctl->id.name, id->name, sizeof(kctl->id.name)) &&
kctl->id.index <= id->index &&
kctl->id.index + kctl->count > id->index;
}
#ifdef CONFIG_SND_CTL_FAST_LOOKUP
/* Compute a hash key for the corresponding ctl id
* It's for the name lookup, hence the numid is excluded.
* The hash key is bound in LONG_MAX to be used for Xarray key.
*/
#define MULTIPLIER 37
static unsigned long get_ctl_id_hash(const struct snd_ctl_elem_id *id)
{
int i;
unsigned long h;
h = id->iface;
h = MULTIPLIER * h + id->device;
h = MULTIPLIER * h + id->subdevice;
for (i = 0; i < SNDRV_CTL_ELEM_ID_NAME_MAXLEN && id->name[i]; i++) h = MULTIPLIER * h + id->name[i]; h = MULTIPLIER * h + id->index;
h &= LONG_MAX;
return h;
}
/* add hash entries to numid and ctl xarray tables */
static void add_hash_entries(struct snd_card *card,
struct snd_kcontrol *kcontrol)
{
struct snd_ctl_elem_id id = kcontrol->id;
int i;
xa_store_range(&card->ctl_numids, kcontrol->id.numid,
kcontrol->id.numid + kcontrol->count - 1,
kcontrol, GFP_KERNEL);
for (i = 0; i < kcontrol->count; i++) {
id.index = kcontrol->id.index + i;
if (xa_insert(&card->ctl_hash, get_ctl_id_hash(&id),
kcontrol, GFP_KERNEL)) {
/* skip hash for this entry, noting we had collision */
card->ctl_hash_collision = true;
dev_dbg(card->dev, "ctl_hash collision %d:%s:%d\n",
id.iface, id.name, id.index);
}
}
}
/* remove hash entries that have been added */
static void remove_hash_entries(struct snd_card *card,
struct snd_kcontrol *kcontrol)
{
struct snd_ctl_elem_id id = kcontrol->id;
struct snd_kcontrol *matched;
unsigned long h;
int i;
for (i = 0; i < kcontrol->count; i++) { xa_erase(&card->ctl_numids, id.numid);
h = get_ctl_id_hash(&id);
matched = xa_load(&card->ctl_hash, h);
if (matched && (matched == kcontrol || elem_id_matches(matched, &id))) xa_erase(&card->ctl_hash, h); id.index++;
id.numid++;
}
}
#else /* CONFIG_SND_CTL_FAST_LOOKUP */
static inline void add_hash_entries(struct snd_card *card,
struct snd_kcontrol *kcontrol)
{
}
static inline void remove_hash_entries(struct snd_card *card,
struct snd_kcontrol *kcontrol)
{
}
#endif /* CONFIG_SND_CTL_FAST_LOOKUP */
enum snd_ctl_add_mode {
CTL_ADD_EXCLUSIVE, CTL_REPLACE, CTL_ADD_ON_REPLACE,
};
/* add/replace a new kcontrol object; call with card->controls_rwsem locked */
static int __snd_ctl_add_replace(struct snd_card *card,
struct snd_kcontrol *kcontrol,
enum snd_ctl_add_mode mode)
{
struct snd_ctl_elem_id id;
unsigned int idx;
struct snd_kcontrol *old;
int err;
lockdep_assert_held_write(&card->controls_rwsem);
id = kcontrol->id;
if (id.index > UINT_MAX - kcontrol->count)
return -EINVAL;
old = snd_ctl_find_id_locked(card, &id);
if (!old) {
if (mode == CTL_REPLACE)
return -EINVAL;
} else {
if (mode == CTL_ADD_EXCLUSIVE) {
dev_err(card->dev,
"control %i:%i:%i:%s:%i is already present\n",
id.iface, id.device, id.subdevice, id.name,
id.index);
return -EBUSY;
}
err = snd_ctl_remove_locked(card, old);
if (err < 0)
return err;
}
if (snd_ctl_find_hole(card, kcontrol->count) < 0)
return -ENOMEM;
list_add_tail(&kcontrol->list, &card->controls);
card->controls_count += kcontrol->count;
kcontrol->id.numid = card->last_numid + 1;
card->last_numid += kcontrol->count;
add_hash_entries(card, kcontrol);
for (idx = 0; idx < kcontrol->count; idx++)
snd_ctl_notify_one(card, SNDRV_CTL_EVENT_MASK_ADD, kcontrol, idx);
return 0;
}
static int snd_ctl_add_replace(struct snd_card *card,
struct snd_kcontrol *kcontrol,
enum snd_ctl_add_mode mode)
{
int err = -EINVAL;
if (! kcontrol)
return err;
if (snd_BUG_ON(!card || !kcontrol->info))
goto error;
scoped_guard(rwsem_write, &card->controls_rwsem)
err = __snd_ctl_add_replace(card, kcontrol, mode);
if (err < 0)
goto error;
return 0;
error:
snd_ctl_free_one(kcontrol);
return err;
}
/**
* snd_ctl_add - add the control instance to the card
* @card: the card instance
* @kcontrol: the control instance to add
*
* Adds the control instance created via snd_ctl_new() or
* snd_ctl_new1() to the given card. Assigns also an unique
* numid used for fast search.
*
* It frees automatically the control which cannot be added.
*
* Return: Zero if successful, or a negative error code on failure.
*
*/
int snd_ctl_add(struct snd_card *card, struct snd_kcontrol *kcontrol)
{
return snd_ctl_add_replace(card, kcontrol, CTL_ADD_EXCLUSIVE);
}
EXPORT_SYMBOL(snd_ctl_add);
/**
* snd_ctl_replace - replace the control instance of the card
* @card: the card instance
* @kcontrol: the control instance to replace
* @add_on_replace: add the control if not already added
*
* Replaces the given control. If the given control does not exist
* and the add_on_replace flag is set, the control is added. If the
* control exists, it is destroyed first.
*
* It frees automatically the control which cannot be added or replaced.
*
* Return: Zero if successful, or a negative error code on failure.
*/
int snd_ctl_replace(struct snd_card *card, struct snd_kcontrol *kcontrol,
bool add_on_replace)
{
return snd_ctl_add_replace(card, kcontrol,
add_on_replace ? CTL_ADD_ON_REPLACE : CTL_REPLACE);
}
EXPORT_SYMBOL(snd_ctl_replace);
static int __snd_ctl_remove(struct snd_card *card,
struct snd_kcontrol *kcontrol,
bool remove_hash)
{
unsigned int idx;
lockdep_assert_held_write(&card->controls_rwsem); if (snd_BUG_ON(!card || !kcontrol))
return -EINVAL;
list_del(&kcontrol->list);
if (remove_hash)
remove_hash_entries(card, kcontrol); card->controls_count -= kcontrol->count;
for (idx = 0; idx < kcontrol->count; idx++)
snd_ctl_notify_one(card, SNDRV_CTL_EVENT_MASK_REMOVE, kcontrol, idx); snd_ctl_free_one(kcontrol);
return 0;
}
static inline int snd_ctl_remove_locked(struct snd_card *card,
struct snd_kcontrol *kcontrol)
{
return __snd_ctl_remove(card, kcontrol, true);
}
/**
* snd_ctl_remove - remove the control from the card and release it
* @card: the card instance
* @kcontrol: the control instance to remove
*
* Removes the control from the card and then releases the instance.
* You don't need to call snd_ctl_free_one().
*
* Return: 0 if successful, or a negative error code on failure.
*
* Note that this function takes card->controls_rwsem lock internally.
*/
int snd_ctl_remove(struct snd_card *card, struct snd_kcontrol *kcontrol)
{
guard(rwsem_write)(&card->controls_rwsem); return snd_ctl_remove_locked(card, kcontrol);
}
EXPORT_SYMBOL(snd_ctl_remove);
/**
* snd_ctl_remove_id - remove the control of the given id and release it
* @card: the card instance
* @id: the control id to remove
*
* Finds the control instance with the given id, removes it from the
* card list and releases it.
*
* Return: 0 if successful, or a negative error code on failure.
*/
int snd_ctl_remove_id(struct snd_card *card, struct snd_ctl_elem_id *id)
{
struct snd_kcontrol *kctl;
guard(rwsem_write)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, id);
if (kctl == NULL)
return -ENOENT;
return snd_ctl_remove_locked(card, kctl);
}
EXPORT_SYMBOL(snd_ctl_remove_id);
/**
* snd_ctl_remove_user_ctl - remove and release the unlocked user control
* @file: active control handle
* @id: the control id to remove
*
* Finds the control instance with the given id, removes it from the
* card list and releases it.
*
* Return: 0 if successful, or a negative error code on failure.
*/
static int snd_ctl_remove_user_ctl(struct snd_ctl_file * file,
struct snd_ctl_elem_id *id)
{
struct snd_card *card = file->card;
struct snd_kcontrol *kctl;
int idx;
guard(rwsem_write)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, id);
if (kctl == NULL)
return -ENOENT;
if (!(kctl->vd[0].access & SNDRV_CTL_ELEM_ACCESS_USER))
return -EINVAL;
for (idx = 0; idx < kctl->count; idx++)
if (kctl->vd[idx].owner != NULL && kctl->vd[idx].owner != file)
return -EBUSY;
return snd_ctl_remove_locked(card, kctl);
}
/**
* snd_ctl_activate_id - activate/inactivate the control of the given id
* @card: the card instance
* @id: the control id to activate/inactivate
* @active: non-zero to activate
*
* Finds the control instance with the given id, and activate or
* inactivate the control together with notification, if changed.
* The given ID data is filled with full information.
*
* Return: 0 if unchanged, 1 if changed, or a negative error code on failure.
*/
int snd_ctl_activate_id(struct snd_card *card, struct snd_ctl_elem_id *id,
int active)
{
struct snd_kcontrol *kctl;
struct snd_kcontrol_volatile *vd;
unsigned int index_offset;
int ret;
down_write(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, id);
if (kctl == NULL) {
ret = -ENOENT;
goto unlock;
}
index_offset = snd_ctl_get_ioff(kctl, id);
vd = &kctl->vd[index_offset];
ret = 0;
if (active) {
if (!(vd->access & SNDRV_CTL_ELEM_ACCESS_INACTIVE))
goto unlock;
vd->access &= ~SNDRV_CTL_ELEM_ACCESS_INACTIVE;
} else {
if (vd->access & SNDRV_CTL_ELEM_ACCESS_INACTIVE)
goto unlock;
vd->access |= SNDRV_CTL_ELEM_ACCESS_INACTIVE;
}
snd_ctl_build_ioff(id, kctl, index_offset);
downgrade_write(&card->controls_rwsem);
snd_ctl_notify_one(card, SNDRV_CTL_EVENT_MASK_INFO, kctl, index_offset);
up_read(&card->controls_rwsem);
return 1;
unlock:
up_write(&card->controls_rwsem);
return ret;
}
EXPORT_SYMBOL_GPL(snd_ctl_activate_id);
/**
* snd_ctl_rename_id - replace the id of a control on the card
* @card: the card instance
* @src_id: the old id
* @dst_id: the new id
*
* Finds the control with the old id from the card, and replaces the
* id with the new one.
*
* The function tries to keep the already assigned numid while replacing
* the rest.
*
* Note that this function should be used only in the card initialization
* phase. Calling after the card instantiation may cause issues with
* user-space expecting persistent numids.
*
* Return: Zero if successful, or a negative error code on failure.
*/
int snd_ctl_rename_id(struct snd_card *card, struct snd_ctl_elem_id *src_id,
struct snd_ctl_elem_id *dst_id)
{
struct snd_kcontrol *kctl;
int saved_numid;
guard(rwsem_write)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, src_id);
if (kctl == NULL)
return -ENOENT;
saved_numid = kctl->id.numid;
remove_hash_entries(card, kctl);
kctl->id = *dst_id;
kctl->id.numid = saved_numid;
add_hash_entries(card, kctl);
return 0;
}
EXPORT_SYMBOL(snd_ctl_rename_id);
/**
* snd_ctl_rename - rename the control on the card
* @card: the card instance
* @kctl: the control to rename
* @name: the new name
*
* Renames the specified control on the card to the new name.
*
* Note that this function takes card->controls_rwsem lock internally.
*/
void snd_ctl_rename(struct snd_card *card, struct snd_kcontrol *kctl,
const char *name)
{
guard(rwsem_write)(&card->controls_rwsem);
remove_hash_entries(card, kctl);
if (strscpy(kctl->id.name, name, sizeof(kctl->id.name)) < 0)
pr_warn("ALSA: Renamed control new name '%s' truncated to '%s'\n",
name, kctl->id.name);
add_hash_entries(card, kctl);
}
EXPORT_SYMBOL(snd_ctl_rename);
#ifndef CONFIG_SND_CTL_FAST_LOOKUP
static struct snd_kcontrol *
snd_ctl_find_numid_slow(struct snd_card *card, unsigned int numid)
{
struct snd_kcontrol *kctl;
list_for_each_entry(kctl, &card->controls, list) {
if (kctl->id.numid <= numid && kctl->id.numid + kctl->count > numid)
return kctl;
}
return NULL;
}
#endif /* !CONFIG_SND_CTL_FAST_LOOKUP */
/**
* snd_ctl_find_numid_locked - find the control instance with the given number-id
* @card: the card instance
* @numid: the number-id to search
*
* Finds the control instance with the given number-id from the card.
*
* The caller must down card->controls_rwsem before calling this function
* (if the race condition can happen).
*
* Return: The pointer of the instance if found, or %NULL if not.
*/
struct snd_kcontrol *
snd_ctl_find_numid_locked(struct snd_card *card, unsigned int numid)
{
if (snd_BUG_ON(!card || !numid))
return NULL;
lockdep_assert_held(&card->controls_rwsem);
#ifdef CONFIG_SND_CTL_FAST_LOOKUP
return xa_load(&card->ctl_numids, numid);
#else
return snd_ctl_find_numid_slow(card, numid);
#endif
}
EXPORT_SYMBOL(snd_ctl_find_numid_locked);
/**
* snd_ctl_find_numid - find the control instance with the given number-id
* @card: the card instance
* @numid: the number-id to search
*
* Finds the control instance with the given number-id from the card.
*
* Return: The pointer of the instance if found, or %NULL if not.
*
* Note that this function takes card->controls_rwsem lock internally.
*/
struct snd_kcontrol *snd_ctl_find_numid(struct snd_card *card,
unsigned int numid)
{
guard(rwsem_read)(&card->controls_rwsem);
return snd_ctl_find_numid_locked(card, numid);
}
EXPORT_SYMBOL(snd_ctl_find_numid);
/**
* snd_ctl_find_id_locked - find the control instance with the given id
* @card: the card instance
* @id: the id to search
*
* Finds the control instance with the given id from the card.
*
* The caller must down card->controls_rwsem before calling this function
* (if the race condition can happen).
*
* Return: The pointer of the instance if found, or %NULL if not.
*/
struct snd_kcontrol *snd_ctl_find_id_locked(struct snd_card *card,
const struct snd_ctl_elem_id *id)
{
struct snd_kcontrol *kctl;
if (snd_BUG_ON(!card || !id))
return NULL;
lockdep_assert_held(&card->controls_rwsem);
if (id->numid != 0)
return snd_ctl_find_numid_locked(card, id->numid);
#ifdef CONFIG_SND_CTL_FAST_LOOKUP
kctl = xa_load(&card->ctl_hash, get_ctl_id_hash(id));
if (kctl && elem_id_matches(kctl, id))
return kctl;
if (!card->ctl_hash_collision)
return NULL; /* we can rely on only hash table */
#endif
/* no matching in hash table - try all as the last resort */
list_for_each_entry(kctl, &card->controls, list)
if (elem_id_matches(kctl, id))
return kctl;
return NULL;
}
EXPORT_SYMBOL(snd_ctl_find_id_locked);
/**
* snd_ctl_find_id - find the control instance with the given id
* @card: the card instance
* @id: the id to search
*
* Finds the control instance with the given id from the card.
*
* Return: The pointer of the instance if found, or %NULL if not.
*
* Note that this function takes card->controls_rwsem lock internally.
*/
struct snd_kcontrol *snd_ctl_find_id(struct snd_card *card,
const struct snd_ctl_elem_id *id)
{
guard(rwsem_read)(&card->controls_rwsem);
return snd_ctl_find_id_locked(card, id);
}
EXPORT_SYMBOL(snd_ctl_find_id);
static int snd_ctl_card_info(struct snd_card *card, struct snd_ctl_file * ctl,
unsigned int cmd, void __user *arg)
{
struct snd_ctl_card_info *info __free(kfree) = NULL;
info = kzalloc(sizeof(*info), GFP_KERNEL);
if (! info)
return -ENOMEM;
scoped_guard(rwsem_read, &snd_ioctl_rwsem) {
info->card = card->number;
strscpy(info->id, card->id, sizeof(info->id));
strscpy(info->driver, card->driver, sizeof(info->driver));
strscpy(info->name, card->shortname, sizeof(info->name));
strscpy(info->longname, card->longname, sizeof(info->longname));
strscpy(info->mixername, card->mixername, sizeof(info->mixername));
strscpy(info->components, card->components, sizeof(info->components));
}
if (copy_to_user(arg, info, sizeof(struct snd_ctl_card_info)))
return -EFAULT;
return 0;
}
static int snd_ctl_elem_list(struct snd_card *card,
struct snd_ctl_elem_list *list)
{
struct snd_kcontrol *kctl;
struct snd_ctl_elem_id id;
unsigned int offset, space, jidx;
offset = list->offset;
space = list->space;
guard(rwsem_read)(&card->controls_rwsem);
list->count = card->controls_count;
list->used = 0;
if (!space)
return 0;
list_for_each_entry(kctl, &card->controls, list) {
if (offset >= kctl->count) {
offset -= kctl->count;
continue;
}
for (jidx = offset; jidx < kctl->count; jidx++) {
snd_ctl_build_ioff(&id, kctl, jidx);
if (copy_to_user(list->pids + list->used, &id, sizeof(id)))
return -EFAULT;
list->used++;
if (!--space)
return 0;
}
offset = 0;
}
return 0;
}
static int snd_ctl_elem_list_user(struct snd_card *card,
struct snd_ctl_elem_list __user *_list)
{
struct snd_ctl_elem_list list;
int err;
if (copy_from_user(&list, _list, sizeof(list)))
return -EFAULT;
err = snd_ctl_elem_list(card, &list);
if (err)
return err;
if (copy_to_user(_list, &list, sizeof(list)))
return -EFAULT;
return 0;
}
/* Check whether the given kctl info is valid */
static int snd_ctl_check_elem_info(struct snd_card *card,
const struct snd_ctl_elem_info *info)
{
static const unsigned int max_value_counts[] = {
[SNDRV_CTL_ELEM_TYPE_BOOLEAN] = 128,
[SNDRV_CTL_ELEM_TYPE_INTEGER] = 128,
[SNDRV_CTL_ELEM_TYPE_ENUMERATED] = 128,
[SNDRV_CTL_ELEM_TYPE_BYTES] = 512,
[SNDRV_CTL_ELEM_TYPE_IEC958] = 1,
[SNDRV_CTL_ELEM_TYPE_INTEGER64] = 64,
};
if (info->type < SNDRV_CTL_ELEM_TYPE_BOOLEAN ||
info->type > SNDRV_CTL_ELEM_TYPE_INTEGER64) {
if (card)
dev_err(card->dev,
"control %i:%i:%i:%s:%i: invalid type %d\n",
info->id.iface, info->id.device,
info->id.subdevice, info->id.name,
info->id.index, info->type);
return -EINVAL;
}
if (info->type == SNDRV_CTL_ELEM_TYPE_ENUMERATED &&
info->value.enumerated.items == 0) {
if (card)
dev_err(card->dev,
"control %i:%i:%i:%s:%i: zero enum items\n",
info->id.iface, info->id.device,
info->id.subdevice, info->id.name,
info->id.index);
return -EINVAL;
}
if (info->count > max_value_counts[info->type]) {
if (card)
dev_err(card->dev,
"control %i:%i:%i:%s:%i: invalid count %d\n",
info->id.iface, info->id.device,
info->id.subdevice, info->id.name,
info->id.index, info->count);
return -EINVAL;
}
return 0;
}
/* The capacity of struct snd_ctl_elem_value.value.*/
static const unsigned int value_sizes[] = {
[SNDRV_CTL_ELEM_TYPE_BOOLEAN] = sizeof(long),
[SNDRV_CTL_ELEM_TYPE_INTEGER] = sizeof(long),
[SNDRV_CTL_ELEM_TYPE_ENUMERATED] = sizeof(unsigned int),
[SNDRV_CTL_ELEM_TYPE_BYTES] = sizeof(unsigned char),
[SNDRV_CTL_ELEM_TYPE_IEC958] = sizeof(struct snd_aes_iec958),
[SNDRV_CTL_ELEM_TYPE_INTEGER64] = sizeof(long long),
};
/* fill the remaining snd_ctl_elem_value data with the given pattern */
static void fill_remaining_elem_value(struct snd_ctl_elem_value *control,
struct snd_ctl_elem_info *info,
u32 pattern)
{
size_t offset = value_sizes[info->type] * info->count;
offset = DIV_ROUND_UP(offset, sizeof(u32));
memset32((u32 *)control->value.bytes.data + offset, pattern,
sizeof(control->value) / sizeof(u32) - offset);
}
/* check whether the given integer ctl value is valid */
static int sanity_check_int_value(struct snd_card *card,
const struct snd_ctl_elem_value *control,
const struct snd_ctl_elem_info *info,
int i, bool print_error)
{
long long lval, lmin, lmax, lstep;
u64 rem;
switch (info->type) {
default:
case SNDRV_CTL_ELEM_TYPE_BOOLEAN:
lval = control->value.integer.value[i];
lmin = 0;
lmax = 1;
lstep = 0;
break;
case SNDRV_CTL_ELEM_TYPE_INTEGER:
lval = control->value.integer.value[i];
lmin = info->value.integer.min;
lmax = info->value.integer.max;
lstep = info->value.integer.step;
break;
case SNDRV_CTL_ELEM_TYPE_INTEGER64:
lval = control->value.integer64.value[i];
lmin = info->value.integer64.min;
lmax = info->value.integer64.max;
lstep = info->value.integer64.step;
break;
case SNDRV_CTL_ELEM_TYPE_ENUMERATED:
lval = control->value.enumerated.item[i];
lmin = 0;
lmax = info->value.enumerated.items - 1;
lstep = 0;
break;
}
if (lval < lmin || lval > lmax) {
if (print_error)
dev_err(card->dev,
"control %i:%i:%i:%s:%i: value out of range %lld (%lld/%lld) at count %i\n",
control->id.iface, control->id.device,
control->id.subdevice, control->id.name,
control->id.index, lval, lmin, lmax, i);
return -EINVAL;
}
if (lstep) {
div64_u64_rem(lval, lstep, &rem);
if (rem) {
if (print_error)
dev_err(card->dev,
"control %i:%i:%i:%s:%i: unaligned value %lld (step %lld) at count %i\n",
control->id.iface, control->id.device,
control->id.subdevice, control->id.name,
control->id.index, lval, lstep, i);
return -EINVAL;
}
}
return 0;
}
/* check whether the all input values are valid for the given elem value */
static int sanity_check_input_values(struct snd_card *card,
const struct snd_ctl_elem_value *control,
const struct snd_ctl_elem_info *info,
bool print_error)
{
int i, ret;
switch (info->type) {
case SNDRV_CTL_ELEM_TYPE_BOOLEAN:
case SNDRV_CTL_ELEM_TYPE_INTEGER:
case SNDRV_CTL_ELEM_TYPE_INTEGER64:
case SNDRV_CTL_ELEM_TYPE_ENUMERATED:
for (i = 0; i < info->count; i++) {
ret = sanity_check_int_value(card, control, info, i,
print_error);
if (ret < 0)
return ret;
}
break;
default:
break;
}
return 0;
}
/* perform sanity checks to the given snd_ctl_elem_value object */
static int sanity_check_elem_value(struct snd_card *card,
const struct snd_ctl_elem_value *control,
const struct snd_ctl_elem_info *info,
u32 pattern)
{
size_t offset;
int ret;
u32 *p;
ret = sanity_check_input_values(card, control, info, true);
if (ret < 0)
return ret;
/* check whether the remaining area kept untouched */
offset = value_sizes[info->type] * info->count;
offset = DIV_ROUND_UP(offset, sizeof(u32));
p = (u32 *)control->value.bytes.data + offset;
for (; offset < sizeof(control->value) / sizeof(u32); offset++, p++) {
if (*p != pattern) {
ret = -EINVAL;
break;
}
*p = 0; /* clear the checked area */
}
return ret;
}
static int __snd_ctl_elem_info(struct snd_card *card,
struct snd_kcontrol *kctl,
struct snd_ctl_elem_info *info,
struct snd_ctl_file *ctl)
{
struct snd_kcontrol_volatile *vd;
unsigned int index_offset;
int result;
#ifdef CONFIG_SND_DEBUG
info->access = 0;
#endif
result = snd_power_ref_and_wait(card);
if (!result)
result = kctl->info(kctl, info);
snd_power_unref(card);
if (result >= 0) {
snd_BUG_ON(info->access);
index_offset = snd_ctl_get_ioff(kctl, &info->id);
vd = &kctl->vd[index_offset];
snd_ctl_build_ioff(&info->id, kctl, index_offset);
info->access = vd->access;
if (vd->owner) {
info->access |= SNDRV_CTL_ELEM_ACCESS_LOCK;
if (vd->owner == ctl)
info->access |= SNDRV_CTL_ELEM_ACCESS_OWNER;
info->owner = pid_vnr(vd->owner->pid);
} else {
info->owner = -1;
}
if (!snd_ctl_skip_validation(info) &&
snd_ctl_check_elem_info(card, info) < 0)
result = -EINVAL;
}
return result;
}
static int snd_ctl_elem_info(struct snd_ctl_file *ctl,
struct snd_ctl_elem_info *info)
{
struct snd_card *card = ctl->card;
struct snd_kcontrol *kctl;
guard(rwsem_read)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, &info->id);
if (!kctl)
return -ENOENT;
return __snd_ctl_elem_info(card, kctl, info, ctl);
}
static int snd_ctl_elem_info_user(struct snd_ctl_file *ctl,
struct snd_ctl_elem_info __user *_info)
{
struct snd_ctl_elem_info info;
int result;
if (copy_from_user(&info, _info, sizeof(info)))
return -EFAULT;
result = snd_ctl_elem_info(ctl, &info);
if (result < 0)
return result;
/* drop internal access flags */
info.access &= ~(SNDRV_CTL_ELEM_ACCESS_SKIP_CHECK|
SNDRV_CTL_ELEM_ACCESS_LED_MASK);
if (copy_to_user(_info, &info, sizeof(info)))
return -EFAULT;
return result;
}
static int snd_ctl_elem_read(struct snd_card *card,
struct snd_ctl_elem_value *control)
{
struct snd_kcontrol *kctl;
struct snd_kcontrol_volatile *vd;
unsigned int index_offset;
struct snd_ctl_elem_info info;
const u32 pattern = 0xdeadbeef;
int ret;
guard(rwsem_read)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, &control->id);
if (!kctl)
return -ENOENT;
index_offset = snd_ctl_get_ioff(kctl, &control->id);
vd = &kctl->vd[index_offset];
if (!(vd->access & SNDRV_CTL_ELEM_ACCESS_READ) || !kctl->get)
return -EPERM;
snd_ctl_build_ioff(&control->id, kctl, index_offset);
#ifdef CONFIG_SND_CTL_DEBUG
/* info is needed only for validation */
memset(&info, 0, sizeof(info));
info.id = control->id;
ret = __snd_ctl_elem_info(card, kctl, &info, NULL);
if (ret < 0)
return ret;
#endif
if (!snd_ctl_skip_validation(&info))
fill_remaining_elem_value(control, &info, pattern);
ret = snd_power_ref_and_wait(card);
if (!ret)
ret = kctl->get(kctl, control);
snd_power_unref(card);
if (ret < 0)
return ret;
if (!snd_ctl_skip_validation(&info) &&
sanity_check_elem_value(card, control, &info, pattern) < 0) {
dev_err(card->dev,
"control %i:%i:%i:%s:%i: access overflow\n",
control->id.iface, control->id.device,
control->id.subdevice, control->id.name,
control->id.index);
return -EINVAL;
}
return 0;
}
static int snd_ctl_elem_read_user(struct snd_card *card,
struct snd_ctl_elem_value __user *_control)
{
struct snd_ctl_elem_value *control __free(kfree) = NULL;
int result;
control = memdup_user(_control, sizeof(*control));
if (IS_ERR(control))
return PTR_ERR(no_free_ptr(control));
result = snd_ctl_elem_read(card, control);
if (result < 0)
return result;
if (copy_to_user(_control, control, sizeof(*control)))
return -EFAULT;
return result;
}
static int snd_ctl_elem_write(struct snd_card *card, struct snd_ctl_file *file,
struct snd_ctl_elem_value *control)
{
struct snd_kcontrol *kctl;
struct snd_kcontrol_volatile *vd;
unsigned int index_offset;
int result;
down_write(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, &control->id);
if (kctl == NULL) {
up_write(&card->controls_rwsem);
return -ENOENT;
}
index_offset = snd_ctl_get_ioff(kctl, &control->id);
vd = &kctl->vd[index_offset];
if (!(vd->access & SNDRV_CTL_ELEM_ACCESS_WRITE) || kctl->put == NULL ||
(file && vd->owner && vd->owner != file)) {
up_write(&card->controls_rwsem);
return -EPERM;
}
snd_ctl_build_ioff(&control->id, kctl, index_offset);
result = snd_power_ref_and_wait(card);
/* validate input values */
if (IS_ENABLED(CONFIG_SND_CTL_INPUT_VALIDATION) && !result) {
struct snd_ctl_elem_info info;
memset(&info, 0, sizeof(info));
info.id = control->id;
result = __snd_ctl_elem_info(card, kctl, &info, NULL);
if (!result)
result = sanity_check_input_values(card, control, &info,
false);
}
if (!result)
result = kctl->put(kctl, control);
snd_power_unref(card);
if (result < 0) {
up_write(&card->controls_rwsem);
return result;
}
if (result > 0) {
downgrade_write(&card->controls_rwsem);
snd_ctl_notify_one(card, SNDRV_CTL_EVENT_MASK_VALUE, kctl, index_offset);
up_read(&card->controls_rwsem);
} else {
up_write(&card->controls_rwsem);
}
return 0;
}
static int snd_ctl_elem_write_user(struct snd_ctl_file *file,
struct snd_ctl_elem_value __user *_control)
{
struct snd_ctl_elem_value *control __free(kfree) = NULL;
struct snd_card *card;
int result;
control = memdup_user(_control, sizeof(*control));
if (IS_ERR(control))
return PTR_ERR(no_free_ptr(control));
card = file->card;
result = snd_ctl_elem_write(card, file, control);
if (result < 0)
return result;
if (copy_to_user(_control, control, sizeof(*control)))
return -EFAULT;
return result;
}
static int snd_ctl_elem_lock(struct snd_ctl_file *file,
struct snd_ctl_elem_id __user *_id)
{
struct snd_card *card = file->card;
struct snd_ctl_elem_id id;
struct snd_kcontrol *kctl;
struct snd_kcontrol_volatile *vd;
if (copy_from_user(&id, _id, sizeof(id)))
return -EFAULT;
guard(rwsem_write)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, &id);
if (!kctl)
return -ENOENT;
vd = &kctl->vd[snd_ctl_get_ioff(kctl, &id)];
if (vd->owner)
return -EBUSY;
vd->owner = file;
return 0;
}
static int snd_ctl_elem_unlock(struct snd_ctl_file *file,
struct snd_ctl_elem_id __user *_id)
{
struct snd_card *card = file->card;
struct snd_ctl_elem_id id;
struct snd_kcontrol *kctl;
struct snd_kcontrol_volatile *vd;
if (copy_from_user(&id, _id, sizeof(id)))
return -EFAULT;
guard(rwsem_write)(&card->controls_rwsem);
kctl = snd_ctl_find_id_locked(card, &id);
if (!kctl)
return -ENOENT;
vd = &kctl->vd[snd_ctl_get_ioff(kctl, &id)];
if (!vd->owner)
return -EINVAL;
if (vd->owner != file)
return -EPERM;
vd->owner = NULL;
return 0;
}
struct user_element {
struct snd_ctl_elem_info info;
struct snd_card *card;
char *elem_data; /* element data */
unsigned long elem_data_size; /* size of element data in bytes */
void *tlv_data; /* TLV data */
unsigned long tlv_data_size; /* TLV data size */
void *priv_data; /* private data (like strings for enumerated type) */
};
// check whether the addition (in bytes) of user ctl element may overflow the limit.
static bool check_user_elem_overflow(struct snd_card *card, ssize_t add)
{
return (ssize_t)card->user_ctl_alloc_size + add > max_user_ctl_alloc_size;
}
static int snd_ctl_elem_user_info(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_info *uinfo)
{
struct user_element *ue = kcontrol->private_data;
unsigned int offset;
offset = snd_ctl_get_ioff(kcontrol, &uinfo->id);
*uinfo = ue->info;
snd_ctl_build_ioff(&uinfo->id, kcontrol, offset);
return 0;
}
static int snd_ctl_elem_user_enum_info(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_info *uinfo)
{
struct user_element *ue = kcontrol->private_data;
const char *names;
unsigned int item;
unsigned int offset;
item = uinfo->value.enumerated.item;
offset = snd_ctl_get_ioff(kcontrol, &uinfo->id);
*uinfo = ue->info;
snd_ctl_build_ioff(&uinfo->id, kcontrol, offset);
item = min(item, uinfo->value.enumerated.items - 1);
uinfo->value.enumerated.item = item;
names = ue->priv_data;
for (; item > 0; --item)
names += strlen(names) + 1;
strcpy(uinfo->value.enumerated.name, names);
return 0;
}
static int snd_ctl_elem_user_get(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol)
{
struct user_element *ue = kcontrol->private_data;
unsigned int size = ue->elem_data_size;
char *src = ue->elem_data +
snd_ctl_get_ioff(kcontrol, &ucontrol->id) * size;
memcpy(&ucontrol->value, src, size);
return 0;
}
static int snd_ctl_elem_user_put(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_value *ucontrol)
{
int change;
struct user_element *ue = kcontrol->private_data;
unsigned int size = ue->elem_data_size;
char *dst = ue->elem_data +
snd_ctl_get_ioff(kcontrol, &ucontrol->id) * size;
change = memcmp(&ucontrol->value, dst, size) != 0;
if (change)
memcpy(dst, &ucontrol->value, size);
return change;
}
/* called in controls_rwsem write lock */
static int replace_user_tlv(struct snd_kcontrol *kctl, unsigned int __user *buf,
unsigned int size)
{
struct user_element *ue = kctl->private_data;
unsigned int *container;
unsigned int mask = 0;
int i;
int change;
lockdep_assert_held_write(&ue->card->controls_rwsem);
if (size > 1024 * 128) /* sane value */
return -EINVAL;
// does the TLV size change cause overflow?
if (check_user_elem_overflow(ue->card, (ssize_t)(size - ue->tlv_data_size)))
return -ENOMEM;
container = vmemdup_user(buf, size);
if (IS_ERR(container))
return PTR_ERR(container);
change = ue->tlv_data_size != size;
if (!change)
change = memcmp(ue->tlv_data, container, size) != 0;
if (!change) {
kvfree(container);
return 0;
}
if (ue->tlv_data == NULL) {
/* Now TLV data is available. */
for (i = 0; i < kctl->count; ++i)
kctl->vd[i].access |= SNDRV_CTL_ELEM_ACCESS_TLV_READ;
mask = SNDRV_CTL_EVENT_MASK_INFO;
} else {
ue->card->user_ctl_alloc_size -= ue->tlv_data_size;
ue->tlv_data_size = 0;
kvfree(ue->tlv_data);
}
ue->tlv_data = container;
ue->tlv_data_size = size;
// decremented at private_free.
ue->card->user_ctl_alloc_size += size;
mask |= SNDRV_CTL_EVENT_MASK_TLV;
for (i = 0; i < kctl->count; ++i)
snd_ctl_notify_one(ue->card, mask, kctl, i);
return change;
}
static int read_user_tlv(struct snd_kcontrol *kctl, unsigned int __user *buf,
unsigned int size)
{
struct user_element *ue = kctl->private_data;
if (ue->tlv_data_size == 0 || ue->tlv_data == NULL)
return -ENXIO;
if (size < ue->tlv_data_size)
return -ENOSPC;
if (copy_to_user(buf, ue->tlv_data, ue->tlv_data_size))
return -EFAULT;
return 0;
}
static int snd_ctl_elem_user_tlv(struct snd_kcontrol *kctl, int op_flag,
unsigned int size, unsigned int __user *buf)
{
if (op_flag == SNDRV_CTL_TLV_OP_WRITE)
return replace_user_tlv(kctl, buf, size);
else
return read_user_tlv(kctl, buf, size);
}
/* called in controls_rwsem write lock */
static int snd_ctl_elem_init_enum_names(struct user_element *ue)
{
char *names, *p;
size_t buf_len, name_len;
unsigned int i;
const uintptr_t user_ptrval = ue->info.value.enumerated.names_ptr;
lockdep_assert_held_write(&ue->card->controls_rwsem);
buf_len = ue->info.value.enumerated.names_length;
if (buf_len > 64 * 1024)
return -EINVAL;
if (check_user_elem_overflow(ue->card, buf_len))
return -ENOMEM;
names = vmemdup_user((const void __user *)user_ptrval, buf_len);
if (IS_ERR(names))
return PTR_ERR(names);
/* check that there are enough valid names */
p = names;
for (i = 0; i < ue->info.value.enumerated.items; ++i) {
name_len = strnlen(p, buf_len);
if (name_len == 0 || name_len >= 64 || name_len == buf_len) {
kvfree(names);
return -EINVAL;
}
p += name_len + 1;
buf_len -= name_len + 1;
}
ue->priv_data = names;
ue->info.value.enumerated.names_ptr = 0;
// increment the allocation size; decremented again at private_free.
ue->card->user_ctl_alloc_size += ue->info.value.enumerated.names_length;
return 0;
}
static size_t compute_user_elem_size(size_t size, unsigned int count)
{
return sizeof(struct user_element) + size * count;
}
static void snd_ctl_elem_user_free(struct snd_kcontrol *kcontrol)
{
struct user_element *ue = kcontrol->private_data;
// decrement the allocation size.
ue->card->user_ctl_alloc_size -= compute_user_elem_size(ue->elem_data_size, kcontrol->count);
ue->card->user_ctl_alloc_size -= ue->tlv_data_size;
if (ue->priv_data)
ue->card->user_ctl_alloc_size -= ue->info.value.enumerated.names_length;
kvfree(ue->tlv_data);
kvfree(ue->priv_data);
kfree(ue);
}
static int snd_ctl_elem_add(struct snd_ctl_file *file,
struct snd_ctl_elem_info *info, int replace)
{
struct snd_card *card = file->card;
struct snd_kcontrol *kctl;
unsigned int count;
unsigned int access;
long private_size;
size_t alloc_size;
struct user_element *ue;
unsigned int offset;
int err;
if (!*info->id.name)
return -EINVAL;
if (strnlen(info->id.name, sizeof(info->id.name)) >= sizeof(info->id.name))
return -EINVAL;
/* Delete a control to replace them if needed. */
if (replace) {
info->id.numid = 0;
err = snd_ctl_remove_user_ctl(file, &info->id);
if (err)
return err;
}
/* Check the number of elements for this userspace control. */
count = info->owner;
if (count == 0)
count = 1;
/* Arrange access permissions if needed. */
access = info->access;
if (access == 0)
access = SNDRV_CTL_ELEM_ACCESS_READWRITE;
access &= (SNDRV_CTL_ELEM_ACCESS_READWRITE |
SNDRV_CTL_ELEM_ACCESS_INACTIVE |
SNDRV_CTL_ELEM_ACCESS_TLV_WRITE);
/* In initial state, nothing is available as TLV container. */
if (access & SNDRV_CTL_ELEM_ACCESS_TLV_WRITE)
access |= SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK;
access |= SNDRV_CTL_ELEM_ACCESS_USER;
/*
* Check information and calculate the size of data specific to
* this userspace control.
*/
/* pass NULL to card for suppressing error messages */
err = snd_ctl_check_elem_info(NULL, info);
if (err < 0)
return err;
/* user-space control doesn't allow zero-size data */
if (info->count < 1)
return -EINVAL;
private_size = value_sizes[info->type] * info->count;
alloc_size = compute_user_elem_size(private_size, count);
guard(rwsem_write)(&card->controls_rwsem);
if (check_user_elem_overflow(card, alloc_size))
return -ENOMEM;
/*
* Keep memory object for this userspace control. After passing this
* code block, the instance should be freed by snd_ctl_free_one().
*
* Note that these elements in this control are locked.
*/
err = snd_ctl_new(&kctl, count, access, file);
if (err < 0)
return err;
memcpy(&kctl->id, &info->id, sizeof(kctl->id));
ue = kzalloc(alloc_size, GFP_KERNEL);
if (!ue) {
kfree(kctl);
return -ENOMEM;
}
kctl->private_data = ue;
kctl->private_free = snd_ctl_elem_user_free;
// increment the allocated size; decremented again at private_free.
card->user_ctl_alloc_size += alloc_size;
/* Set private data for this userspace control. */
ue->card = card;
ue->info = *info;
ue->info.access = 0;
ue->elem_data = (char *)ue + sizeof(*ue);
ue->elem_data_size = private_size;
if (ue->info.type == SNDRV_CTL_ELEM_TYPE_ENUMERATED) {
err = snd_ctl_elem_init_enum_names(ue);
if (err < 0) {
snd_ctl_free_one(kctl);
return err;
}
}
/* Set callback functions. */
if (info->type == SNDRV_CTL_ELEM_TYPE_ENUMERATED)
kctl->info = snd_ctl_elem_user_enum_info;
else
kctl->info = snd_ctl_elem_user_info;
if (access & SNDRV_CTL_ELEM_ACCESS_READ)
kctl->get = snd_ctl_elem_user_get;
if (access & SNDRV_CTL_ELEM_ACCESS_WRITE)
kctl->put = snd_ctl_elem_user_put;
if (access & SNDRV_CTL_ELEM_ACCESS_TLV_WRITE)
kctl->tlv.c = snd_ctl_elem_user_tlv;
/* This function manage to free the instance on failure. */
err = __snd_ctl_add_replace(card, kctl, CTL_ADD_EXCLUSIVE);
if (err < 0) {
snd_ctl_free_one(kctl);
return err;
}
offset = snd_ctl_get_ioff(kctl, &info->id);
snd_ctl_build_ioff(&info->id, kctl, offset);
/*
* Here we cannot fill any field for the number of elements added by
* this operation because there're no specific fields. The usage of
* 'owner' field for this purpose may cause any bugs to userspace
* applications because the field originally means PID of a process
* which locks the element.
*/
return 0;
}
static int snd_ctl_elem_add_user(struct snd_ctl_file *file,
struct snd_ctl_elem_info __user *_info, int replace)
{
struct snd_ctl_elem_info info;
int err;
if (copy_from_user(&info, _info, sizeof(info)))
return -EFAULT;
err = snd_ctl_elem_add(file, &info, replace);
if (err < 0)
return err;
if (copy_to_user(_info, &info, sizeof(info))) {
snd_ctl_remove_user_ctl(file, &info.id);
return -EFAULT;
}
return 0;
}
static int snd_ctl_elem_remove(struct snd_ctl_file *file,
struct snd_ctl_elem_id __user *_id)
{
struct snd_ctl_elem_id id;
if (copy_from_user(&id, _id, sizeof(id)))
return -EFAULT;
return snd_ctl_remove_user_ctl(file, &id);
}
static int snd_ctl_subscribe_events(struct snd_ctl_file *file, int __user *ptr)
{
int subscribe;
if (get_user(subscribe, ptr))
return -EFAULT;
if (subscribe < 0) {
subscribe = file->subscribed;
if (put_user(subscribe, ptr))
return -EFAULT;
return 0;
}
if (subscribe) {
file->subscribed = 1;
return 0;
} else if (file->subscribed) {
snd_ctl_empty_read_queue(file);
file->subscribed = 0;
}
return 0;
}
static int call_tlv_handler(struct snd_ctl_file *file, int op_flag,
struct snd_kcontrol *kctl,
struct snd_ctl_elem_id *id,
unsigned int __user *buf, unsigned int size)
{
static const struct {
int op;
int perm;
} pairs[] = {
{SNDRV_CTL_TLV_OP_READ, SNDRV_CTL_ELEM_ACCESS_TLV_READ},
{SNDRV_CTL_TLV_OP_WRITE, SNDRV_CTL_ELEM_ACCESS_TLV_WRITE},
{SNDRV_CTL_TLV_OP_CMD, SNDRV_CTL_ELEM_ACCESS_TLV_COMMAND},
};
struct snd_kcontrol_volatile *vd = &kctl->vd[snd_ctl_get_ioff(kctl, id)];
int i, ret;
/* Check support of the request for this element. */
for (i = 0; i < ARRAY_SIZE(pairs); ++i) {
if (op_flag == pairs[i].op && (vd->access & pairs[i].perm))
break;
}
if (i == ARRAY_SIZE(pairs))
return -ENXIO;
if (kctl->tlv.c == NULL)
return -ENXIO;
/* Write and command operations are not allowed for locked element. */
if (op_flag != SNDRV_CTL_TLV_OP_READ &&
vd->owner != NULL && vd->owner != file)
return -EPERM;
ret = snd_power_ref_and_wait(file->card);
if (!ret)
ret = kctl->tlv.c(kctl, op_flag, size, buf);
snd_power_unref(file->card);
return ret;
}
static int read_tlv_buf(struct snd_kcontrol *kctl, struct snd_ctl_elem_id *id,
unsigned int __user *buf, unsigned int size)
{
struct snd_kcontrol_volatile *vd = &kctl->vd[snd_ctl_get_ioff(kctl, id)];
unsigned int len;
if (!(vd->access & SNDRV_CTL_ELEM_ACCESS_TLV_READ))
return -ENXIO;
if (kctl->tlv.p == NULL)
return -ENXIO;
len = sizeof(unsigned int) * 2 + kctl->tlv.p[1];
if (size < len)
return -ENOMEM;
if (copy_to_user(buf, kctl->tlv.p, len))
return -EFAULT;
return 0;
}
static int snd_ctl_tlv_ioctl(struct snd_ctl_file *file,
struct snd_ctl_tlv __user *buf,
int op_flag)
{
struct snd_ctl_tlv header;
unsigned int __user *container;
unsigned int container_size;
struct snd_kcontrol *kctl;
struct snd_ctl_elem_id id;
struct snd_kcontrol_volatile *vd;
lockdep_assert_held(&file->card->controls_rwsem);
if (copy_from_user(&header, buf, sizeof(header)))
return -EFAULT;
/* In design of control core, numerical ID starts at 1. */
if (header.numid == 0)
return -EINVAL;
/* At least, container should include type and length fields. */
if (header.length < sizeof(unsigned int) * 2)
return -EINVAL;
container_size = header.length;
container = buf->tlv;
kctl = snd_ctl_find_numid_locked(file->card, header.numid);
if (kctl == NULL)
return -ENOENT;
/* Calculate index of the element in this set. */
id = kctl->id;
snd_ctl_build_ioff(&id, kctl, header.numid - id.numid);
vd = &kctl->vd[snd_ctl_get_ioff(kctl, &id)];
if (vd->access & SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK) {
return call_tlv_handler(file, op_flag, kctl, &id, container,
container_size);
} else {
if (op_flag == SNDRV_CTL_TLV_OP_READ) {
return read_tlv_buf(kctl, &id, container,
container_size);
}
}
/* Not supported. */
return -ENXIO;
}
static long snd_ctl_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
struct snd_ctl_file *ctl;
struct snd_card *card;
struct snd_kctl_ioctl *p;
void __user *argp = (void __user *)arg;
int __user *ip = argp;
int err;
ctl = file->private_data;
card = ctl->card;
if (snd_BUG_ON(!card))
return -ENXIO;
switch (cmd) {
case SNDRV_CTL_IOCTL_PVERSION:
return put_user(SNDRV_CTL_VERSION, ip) ? -EFAULT : 0;
case SNDRV_CTL_IOCTL_CARD_INFO:
return snd_ctl_card_info(card, ctl, cmd, argp);
case SNDRV_CTL_IOCTL_ELEM_LIST:
return snd_ctl_elem_list_user(card, argp);
case SNDRV_CTL_IOCTL_ELEM_INFO:
return snd_ctl_elem_info_user(ctl, argp);
case SNDRV_CTL_IOCTL_ELEM_READ:
return snd_ctl_elem_read_user(card, argp);
case SNDRV_CTL_IOCTL_ELEM_WRITE:
return snd_ctl_elem_write_user(ctl, argp);
case SNDRV_CTL_IOCTL_ELEM_LOCK:
return snd_ctl_elem_lock(ctl, argp);
case SNDRV_CTL_IOCTL_ELEM_UNLOCK:
return snd_ctl_elem_unlock(ctl, argp);
case SNDRV_CTL_IOCTL_ELEM_ADD:
return snd_ctl_elem_add_user(ctl, argp, 0);
case SNDRV_CTL_IOCTL_ELEM_REPLACE:
return snd_ctl_elem_add_user(ctl, argp, 1);
case SNDRV_CTL_IOCTL_ELEM_REMOVE:
return snd_ctl_elem_remove(ctl, argp);
case SNDRV_CTL_IOCTL_SUBSCRIBE_EVENTS:
return snd_ctl_subscribe_events(ctl, ip);
case SNDRV_CTL_IOCTL_TLV_READ:
scoped_guard(rwsem_read, &ctl->card->controls_rwsem)
err = snd_ctl_tlv_ioctl(ctl, argp, SNDRV_CTL_TLV_OP_READ);
return err;
case SNDRV_CTL_IOCTL_TLV_WRITE:
scoped_guard(rwsem_write, &ctl->card->controls_rwsem)
err = snd_ctl_tlv_ioctl(ctl, argp, SNDRV_CTL_TLV_OP_WRITE);
return err;
case SNDRV_CTL_IOCTL_TLV_COMMAND:
scoped_guard(rwsem_write, &ctl->card->controls_rwsem)
err = snd_ctl_tlv_ioctl(ctl, argp, SNDRV_CTL_TLV_OP_CMD);
return err;
case SNDRV_CTL_IOCTL_POWER:
return -ENOPROTOOPT;
case SNDRV_CTL_IOCTL_POWER_STATE:
return put_user(SNDRV_CTL_POWER_D0, ip) ? -EFAULT : 0;
}
guard(rwsem_read)(&snd_ioctl_rwsem);
list_for_each_entry(p, &snd_control_ioctls, list) {
err = p->fioctl(card, ctl, cmd, arg);
if (err != -ENOIOCTLCMD)
return err;
}
dev_dbg(card->dev, "unknown ioctl = 0x%x\n", cmd);
return -ENOTTY;
}
static ssize_t snd_ctl_read(struct file *file, char __user *buffer,
size_t count, loff_t * offset)
{
struct snd_ctl_file *ctl;
int err = 0;
ssize_t result = 0;
ctl = file->private_data;
if (snd_BUG_ON(!ctl || !ctl->card))
return -ENXIO;
if (!ctl->subscribed)
return -EBADFD;
if (count < sizeof(struct snd_ctl_event))
return -EINVAL;
spin_lock_irq(&ctl->read_lock);
while (count >= sizeof(struct snd_ctl_event)) {
struct snd_ctl_event ev;
struct snd_kctl_event *kev;
while (list_empty(&ctl->events)) {
wait_queue_entry_t wait;
if ((file->f_flags & O_NONBLOCK) != 0 || result > 0) {
err = -EAGAIN;
goto __end_lock;
}
init_waitqueue_entry(&wait, current);
add_wait_queue(&ctl->change_sleep, &wait);
set_current_state(TASK_INTERRUPTIBLE);
spin_unlock_irq(&ctl->read_lock);
schedule();
remove_wait_queue(&ctl->change_sleep, &wait);
if (ctl->card->shutdown)
return -ENODEV;
if (signal_pending(current))
return -ERESTARTSYS;
spin_lock_irq(&ctl->read_lock);
}
kev = snd_kctl_event(ctl->events.next);
ev.type = SNDRV_CTL_EVENT_ELEM;
ev.data.elem.mask = kev->mask;
ev.data.elem.id = kev->id;
list_del(&kev->list);
spin_unlock_irq(&ctl->read_lock);
kfree(kev);
if (copy_to_user(buffer, &ev, sizeof(struct snd_ctl_event))) {
err = -EFAULT;
goto __end;
}
spin_lock_irq(&ctl->read_lock);
buffer += sizeof(struct snd_ctl_event);
count -= sizeof(struct snd_ctl_event);
result += sizeof(struct snd_ctl_event);
}
__end_lock:
spin_unlock_irq(&ctl->read_lock);
__end:
return result > 0 ? result : err;
}
static __poll_t snd_ctl_poll(struct file *file, poll_table * wait)
{
__poll_t mask;
struct snd_ctl_file *ctl;
ctl = file->private_data;
if (!ctl->subscribed)
return 0;
poll_wait(file, &ctl->change_sleep, wait);
mask = 0;
if (!list_empty(&ctl->events))
mask |= EPOLLIN | EPOLLRDNORM;
return mask;
}
/*
* register the device-specific control-ioctls.
* called from each device manager like pcm.c, hwdep.c, etc.
*/
static int _snd_ctl_register_ioctl(snd_kctl_ioctl_func_t fcn, struct list_head *lists)
{
struct snd_kctl_ioctl *pn;
pn = kzalloc(sizeof(struct snd_kctl_ioctl), GFP_KERNEL);
if (pn == NULL)
return -ENOMEM;
pn->fioctl = fcn;
guard(rwsem_write)(&snd_ioctl_rwsem);
list_add_tail(&pn->list, lists);
return 0;
}
/**
* snd_ctl_register_ioctl - register the device-specific control-ioctls
* @fcn: ioctl callback function
*
* called from each device manager like pcm.c, hwdep.c, etc.
*
* Return: zero if successful, or a negative error code
*/
int snd_ctl_register_ioctl(snd_kctl_ioctl_func_t fcn)
{
return _snd_ctl_register_ioctl(fcn, &snd_control_ioctls);
}
EXPORT_SYMBOL(snd_ctl_register_ioctl);
#ifdef CONFIG_COMPAT
/**
* snd_ctl_register_ioctl_compat - register the device-specific 32bit compat
* control-ioctls
* @fcn: ioctl callback function
*
* Return: zero if successful, or a negative error code
*/
int snd_ctl_register_ioctl_compat(snd_kctl_ioctl_func_t fcn)
{
return _snd_ctl_register_ioctl(fcn, &snd_control_compat_ioctls);
}
EXPORT_SYMBOL(snd_ctl_register_ioctl_compat);
#endif
/*
* de-register the device-specific control-ioctls.
*/
static int _snd_ctl_unregister_ioctl(snd_kctl_ioctl_func_t fcn,
struct list_head *lists)
{
struct snd_kctl_ioctl *p;
if (snd_BUG_ON(!fcn))
return -EINVAL;
guard(rwsem_write)(&snd_ioctl_rwsem);
list_for_each_entry(p, lists, list) {
if (p->fioctl == fcn) {
list_del(&p->list);
kfree(p);
return 0;
}
}
snd_BUG();
return -EINVAL;
}
/**
* snd_ctl_unregister_ioctl - de-register the device-specific control-ioctls
* @fcn: ioctl callback function to unregister
*
* Return: zero if successful, or a negative error code
*/
int snd_ctl_unregister_ioctl(snd_kctl_ioctl_func_t fcn)
{
return _snd_ctl_unregister_ioctl(fcn, &snd_control_ioctls);
}
EXPORT_SYMBOL(snd_ctl_unregister_ioctl);
#ifdef CONFIG_COMPAT
/**
* snd_ctl_unregister_ioctl_compat - de-register the device-specific compat
* 32bit control-ioctls
* @fcn: ioctl callback function to unregister
*
* Return: zero if successful, or a negative error code
*/
int snd_ctl_unregister_ioctl_compat(snd_kctl_ioctl_func_t fcn)
{
return _snd_ctl_unregister_ioctl(fcn, &snd_control_compat_ioctls);
}
EXPORT_SYMBOL(snd_ctl_unregister_ioctl_compat);
#endif
static int snd_ctl_fasync(int fd, struct file * file, int on)
{
struct snd_ctl_file *ctl;
ctl = file->private_data;
return snd_fasync_helper(fd, file, on, &ctl->fasync);
}
/* return the preferred subdevice number if already assigned;
* otherwise return -1
*/
int snd_ctl_get_preferred_subdevice(struct snd_card *card, int type)
{
struct snd_ctl_file *kctl;
int subdevice = -1;
guard(read_lock_irqsave)(&card->ctl_files_rwlock);
list_for_each_entry(kctl, &card->ctl_files, list) {
if (kctl->pid == task_pid(current)) {
subdevice = kctl->preferred_subdevice[type];
if (subdevice != -1)
break;
}
}
return subdevice;
}
EXPORT_SYMBOL_GPL(snd_ctl_get_preferred_subdevice);
/*
* ioctl32 compat
*/
#ifdef CONFIG_COMPAT
#include "control_compat.c"
#else
#define snd_ctl_ioctl_compat NULL
#endif
/*
* control layers (audio LED etc.)
*/
/**
* snd_ctl_request_layer - request to use the layer
* @module_name: Name of the kernel module (NULL == build-in)
*
* Return: zero if successful, or an error code when the module cannot be loaded
*/
int snd_ctl_request_layer(const char *module_name)
{
struct snd_ctl_layer_ops *lops;
if (module_name == NULL)
return 0;
scoped_guard(rwsem_read, &snd_ctl_layer_rwsem) {
for (lops = snd_ctl_layer; lops; lops = lops->next)
if (strcmp(lops->module_name, module_name) == 0)
return 0;
}
return request_module(module_name);
}
EXPORT_SYMBOL_GPL(snd_ctl_request_layer);
/**
* snd_ctl_register_layer - register new control layer
* @lops: operation structure
*
* The new layer can track all control elements and do additional
* operations on top (like audio LED handling).
*/
void snd_ctl_register_layer(struct snd_ctl_layer_ops *lops)
{
struct snd_card *card;
int card_number;
scoped_guard(rwsem_write, &snd_ctl_layer_rwsem) {
lops->next = snd_ctl_layer;
snd_ctl_layer = lops;
}
for (card_number = 0; card_number < SNDRV_CARDS; card_number++) {
card = snd_card_ref(card_number);
if (card) {
scoped_guard(rwsem_read, &card->controls_rwsem)
lops->lregister(card);
snd_card_unref(card);
}
}
}
EXPORT_SYMBOL_GPL(snd_ctl_register_layer);
/**
* snd_ctl_disconnect_layer - disconnect control layer
* @lops: operation structure
*
* It is expected that the information about tracked cards
* is freed before this call (the disconnect callback is
* not called here).
*/
void snd_ctl_disconnect_layer(struct snd_ctl_layer_ops *lops)
{
struct snd_ctl_layer_ops *lops2, *prev_lops2;
guard(rwsem_write)(&snd_ctl_layer_rwsem);
for (lops2 = snd_ctl_layer, prev_lops2 = NULL; lops2; lops2 = lops2->next) {
if (lops2 == lops) {
if (!prev_lops2)
snd_ctl_layer = lops->next;
else
prev_lops2->next = lops->next;
break;
}
prev_lops2 = lops2;
}
}
EXPORT_SYMBOL_GPL(snd_ctl_disconnect_layer);
/*
* INIT PART
*/
static const struct file_operations snd_ctl_f_ops =
{
.owner = THIS_MODULE,
.read = snd_ctl_read,
.open = snd_ctl_open,
.release = snd_ctl_release,
.llseek = no_llseek,
.poll = snd_ctl_poll,
.unlocked_ioctl = snd_ctl_ioctl,
.compat_ioctl = snd_ctl_ioctl_compat,
.fasync = snd_ctl_fasync,
};
/* call lops under rwsems; called from snd_ctl_dev_*() below() */
#define call_snd_ctl_lops(_card, _op) \
do { \
struct snd_ctl_layer_ops *lops; \
guard(rwsem_read)(&(_card)->controls_rwsem); \
guard(rwsem_read)(&snd_ctl_layer_rwsem); \
for (lops = snd_ctl_layer; lops; lops = lops->next) \
lops->_op(_card); \
} while (0)
/*
* registration of the control device
*/
static int snd_ctl_dev_register(struct snd_device *device)
{
struct snd_card *card = device->device_data;
int err;
err = snd_register_device(SNDRV_DEVICE_TYPE_CONTROL, card, -1,
&snd_ctl_f_ops, card, card->ctl_dev);
if (err < 0)
return err;
call_snd_ctl_lops(card, lregister);
return 0;
}
/*
* disconnection of the control device
*/
static int snd_ctl_dev_disconnect(struct snd_device *device)
{
struct snd_card *card = device->device_data;
struct snd_ctl_file *ctl;
scoped_guard(read_lock_irqsave, &card->ctl_files_rwlock) { list_for_each_entry(ctl, &card->ctl_files, list) { wake_up(&ctl->change_sleep);
snd_kill_fasync(ctl->fasync, SIGIO, POLL_ERR);
}
}
call_snd_ctl_lops(card, ldisconnect); return snd_unregister_device(card->ctl_dev);
}
/*
* free all controls
*/
static int snd_ctl_dev_free(struct snd_device *device)
{
struct snd_card *card = device->device_data;
struct snd_kcontrol *control;
scoped_guard(rwsem_write, &card->controls_rwsem) { while (!list_empty(&card->controls)) { control = snd_kcontrol(card->controls.next);
__snd_ctl_remove(card, control, false);
}
#ifdef CONFIG_SND_CTL_FAST_LOOKUP
xa_destroy(&card->ctl_numids);
xa_destroy(&card->ctl_hash);
#endif
}
put_device(card->ctl_dev);
return 0;
}
/*
* create control core:
* called from init.c
*/
int snd_ctl_create(struct snd_card *card)
{
static const struct snd_device_ops ops = {
.dev_free = snd_ctl_dev_free,
.dev_register = snd_ctl_dev_register,
.dev_disconnect = snd_ctl_dev_disconnect,
};
int err;
if (snd_BUG_ON(!card))
return -ENXIO;
if (snd_BUG_ON(card->number < 0 || card->number >= SNDRV_CARDS))
return -ENXIO;
err = snd_device_alloc(&card->ctl_dev, card);
if (err < 0)
return err;
dev_set_name(card->ctl_dev, "controlC%d", card->number);
err = snd_device_new(card, SNDRV_DEV_CONTROL, card, &ops);
if (err < 0)
put_device(card->ctl_dev);
return err;
}
/*
* Frequently used control callbacks/helpers
*/
/**
* snd_ctl_boolean_mono_info - Helper function for a standard boolean info
* callback with a mono channel
* @kcontrol: the kcontrol instance
* @uinfo: info to store
*
* This is a function that can be used as info callback for a standard
* boolean control with a single mono channel.
*
* Return: Zero (always successful)
*/
int snd_ctl_boolean_mono_info(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_info *uinfo)
{
uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
uinfo->count = 1;
uinfo->value.integer.min = 0;
uinfo->value.integer.max = 1;
return 0;
}
EXPORT_SYMBOL(snd_ctl_boolean_mono_info);
/**
* snd_ctl_boolean_stereo_info - Helper function for a standard boolean info
* callback with stereo two channels
* @kcontrol: the kcontrol instance
* @uinfo: info to store
*
* This is a function that can be used as info callback for a standard
* boolean control with stereo two channels.
*
* Return: Zero (always successful)
*/
int snd_ctl_boolean_stereo_info(struct snd_kcontrol *kcontrol,
struct snd_ctl_elem_info *uinfo)
{
uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
uinfo->count = 2;
uinfo->value.integer.min = 0;
uinfo->value.integer.max = 1;
return 0;
}
EXPORT_SYMBOL(snd_ctl_boolean_stereo_info);
/**
* snd_ctl_enum_info - fills the info structure for an enumerated control
* @info: the structure to be filled
* @channels: the number of the control's channels; often one
* @items: the number of control values; also the size of @names
* @names: an array containing the names of all control values
*
* Sets all required fields in @info to their appropriate values.
* If the control's accessibility is not the default (readable and writable),
* the caller has to fill @info->access.
*
* Return: Zero (always successful)
*/
int snd_ctl_enum_info(struct snd_ctl_elem_info *info, unsigned int channels,
unsigned int items, const char *const names[])
{
info->type = SNDRV_CTL_ELEM_TYPE_ENUMERATED;
info->count = channels;
info->value.enumerated.items = items;
if (!items)
return 0;
if (info->value.enumerated.item >= items)
info->value.enumerated.item = items - 1;
WARN(strlen(names[info->value.enumerated.item]) >= sizeof(info->value.enumerated.name),
"ALSA: too long item name '%s'\n",
names[info->value.enumerated.item]);
strscpy(info->value.enumerated.name,
names[info->value.enumerated.item],
sizeof(info->value.enumerated.name));
return 0;
}
EXPORT_SYMBOL(snd_ctl_enum_info);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BLK_CGROUP_PRIVATE_H
#define _BLK_CGROUP_PRIVATE_H
/*
* block cgroup private header
*
* Based on ideas and code from CFQ, CFS and BFQ:
* Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
*
* Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
* Paolo Valente <paolo.valente@unimore.it>
*
* Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
* Nauman Rafique <nauman@google.com>
*/
#include <linux/blk-cgroup.h>
#include <linux/cgroup.h>
#include <linux/kthread.h>
#include <linux/blk-mq.h>
#include <linux/llist.h>
#include "blk.h"
struct blkcg_gq;
struct blkg_policy_data;
/* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
#define BLKG_STAT_CPU_BATCH (INT_MAX / 2)
#ifdef CONFIG_BLK_CGROUP
enum blkg_iostat_type {
BLKG_IOSTAT_READ,
BLKG_IOSTAT_WRITE,
BLKG_IOSTAT_DISCARD,
BLKG_IOSTAT_NR,
};
struct blkg_iostat {
u64 bytes[BLKG_IOSTAT_NR];
u64 ios[BLKG_IOSTAT_NR];
};
struct blkg_iostat_set {
struct u64_stats_sync sync;
struct blkcg_gq *blkg;
struct llist_node lnode;
int lqueued; /* queued in llist */
struct blkg_iostat cur;
struct blkg_iostat last;
};
/* association between a blk cgroup and a request queue */
struct blkcg_gq {
/* Pointer to the associated request_queue */
struct request_queue *q;
struct list_head q_node;
struct hlist_node blkcg_node;
struct blkcg *blkcg;
/* all non-root blkcg_gq's are guaranteed to have access to parent */
struct blkcg_gq *parent;
/* reference count */
struct percpu_ref refcnt;
/* is this blkg online? protected by both blkcg and q locks */
bool online;
struct blkg_iostat_set __percpu *iostat_cpu;
struct blkg_iostat_set iostat;
struct blkg_policy_data *pd[BLKCG_MAX_POLS];
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
spinlock_t async_bio_lock;
struct bio_list async_bios;
#endif
union {
struct work_struct async_bio_work;
struct work_struct free_work;
};
atomic_t use_delay;
atomic64_t delay_nsec;
atomic64_t delay_start;
u64 last_delay;
int last_use;
struct rcu_head rcu_head;
};
struct blkcg {
struct cgroup_subsys_state css;
spinlock_t lock;
refcount_t online_pin;
struct radix_tree_root blkg_tree;
struct blkcg_gq __rcu *blkg_hint;
struct hlist_head blkg_list;
struct blkcg_policy_data *cpd[BLKCG_MAX_POLS];
struct list_head all_blkcgs_node;
/*
* List of updated percpu blkg_iostat_set's since the last flush.
*/
struct llist_head __percpu *lhead;
#ifdef CONFIG_BLK_CGROUP_FC_APPID
char fc_app_id[FC_APPID_LEN];
#endif
#ifdef CONFIG_CGROUP_WRITEBACK
struct list_head cgwb_list;
#endif
};
static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
{
return css ? container_of(css, struct blkcg, css) : NULL;
}
/*
* A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
* request_queue (q). This is used by blkcg policies which need to track
* information per blkcg - q pair.
*
* There can be multiple active blkcg policies and each blkg:policy pair is
* represented by a blkg_policy_data which is allocated and freed by each
* policy's pd_alloc/free_fn() methods. A policy can allocate private data
* area by allocating larger data structure which embeds blkg_policy_data
* at the beginning.
*/
struct blkg_policy_data {
/* the blkg and policy id this per-policy data belongs to */
struct blkcg_gq *blkg;
int plid;
bool online;
};
/*
* Policies that need to keep per-blkcg data which is independent from any
* request_queue associated to it should implement cpd_alloc/free_fn()
* methods. A policy can allocate private data area by allocating larger
* data structure which embeds blkcg_policy_data at the beginning.
* cpd_init() is invoked to let each policy handle per-blkcg data.
*/
struct blkcg_policy_data {
/* the blkcg and policy id this per-policy data belongs to */
struct blkcg *blkcg;
int plid;
};
typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp);
typedef void (blkcg_pol_init_cpd_fn)(struct blkcg_policy_data *cpd);
typedef void (blkcg_pol_free_cpd_fn)(struct blkcg_policy_data *cpd);
typedef void (blkcg_pol_bind_cpd_fn)(struct blkcg_policy_data *cpd);
typedef struct blkg_policy_data *(blkcg_pol_alloc_pd_fn)(struct gendisk *disk,
struct blkcg *blkcg, gfp_t gfp);
typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_stat_pd_fn)(struct blkg_policy_data *pd,
struct seq_file *s);
struct blkcg_policy {
int plid;
/* cgroup files for the policy */
struct cftype *dfl_cftypes;
struct cftype *legacy_cftypes;
/* operations */
blkcg_pol_alloc_cpd_fn *cpd_alloc_fn;
blkcg_pol_free_cpd_fn *cpd_free_fn;
blkcg_pol_alloc_pd_fn *pd_alloc_fn;
blkcg_pol_init_pd_fn *pd_init_fn;
blkcg_pol_online_pd_fn *pd_online_fn;
blkcg_pol_offline_pd_fn *pd_offline_fn;
blkcg_pol_free_pd_fn *pd_free_fn;
blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn;
blkcg_pol_stat_pd_fn *pd_stat_fn;
};
extern struct blkcg blkcg_root;
extern bool blkcg_debug_stats;
void blkg_init_queue(struct request_queue *q);
int blkcg_init_disk(struct gendisk *disk);
void blkcg_exit_disk(struct gendisk *disk);
/* Blkio controller policy registration */
int blkcg_policy_register(struct blkcg_policy *pol);
void blkcg_policy_unregister(struct blkcg_policy *pol);
int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol);
void blkcg_deactivate_policy(struct gendisk *disk,
const struct blkcg_policy *pol);
const char *blkg_dev_name(struct blkcg_gq *blkg);
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
u64 (*prfill)(struct seq_file *,
struct blkg_policy_data *, int),
const struct blkcg_policy *pol, int data,
bool show_total);
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
struct blkg_conf_ctx {
char *input;
char *body;
struct block_device *bdev;
struct blkcg_gq *blkg;
};
void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input);
int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx);
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
struct blkg_conf_ctx *ctx);
void blkg_conf_exit(struct blkg_conf_ctx *ctx);
/**
* bio_issue_as_root_blkg - see if this bio needs to be issued as root blkg
* @return: true if this bio needs to be submitted with the root blkg context.
*
* In order to avoid priority inversions we sometimes need to issue a bio as if
* it were attached to the root blkg, and then backcharge to the actual owning
* blkg. The idea is we do bio_blkcg_css() to look up the actual context for
* the bio and attach the appropriate blkg to the bio. Then we call this helper
* and if it is true run with the root blkg for that queue and then do any
* backcharging to the originating cgroup once the io is complete.
*/
static inline bool bio_issue_as_root_blkg(struct bio *bio)
{
return (bio->bi_opf & (REQ_META | REQ_SWAP)) != 0;
}
/**
* blkg_lookup - lookup blkg for the specified blkcg - q pair
* @blkcg: blkcg of interest
* @q: request_queue of interest
*
* Lookup blkg for the @blkcg - @q pair.
* Must be called in a RCU critical section.
*/
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
struct request_queue *q)
{
struct blkcg_gq *blkg;
if (blkcg == &blkcg_root)
return q->root_blkg; blkg = rcu_dereference_check(blkcg->blkg_hint,
lockdep_is_held(&q->queue_lock));
if (blkg && blkg->q == q)
return blkg;
blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); if (blkg && blkg->q != q)
blkg = NULL;
return blkg;
}
/**
* blkg_to_pdata - get policy private data
* @blkg: blkg of interest
* @pol: policy of interest
*
* Return pointer to private data associated with the @blkg-@pol pair.
*/
static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
struct blkcg_policy *pol)
{
return blkg ? blkg->pd[pol->plid] : NULL;
}
static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
struct blkcg_policy *pol)
{
return blkcg ? blkcg->cpd[pol->plid] : NULL;
}
/**
* pdata_to_blkg - get blkg associated with policy private data
* @pd: policy private data of interest
*
* @pd is policy private data. Determine the blkg it's associated with.
*/
static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
{
return pd ? pd->blkg : NULL;
}
static inline struct blkcg *cpd_to_blkcg(struct blkcg_policy_data *cpd)
{
return cpd ? cpd->blkcg : NULL;
}
/**
* blkg_path - format cgroup path of blkg
* @blkg: blkg of interest
* @buf: target buffer
* @buflen: target buffer length
*
* Format the path of the cgroup of @blkg into @buf.
*/
static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
{
return cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
}
/**
* blkg_get - get a blkg reference
* @blkg: blkg to get
*
* The caller should be holding an existing reference.
*/
static inline void blkg_get(struct blkcg_gq *blkg)
{
percpu_ref_get(&blkg->refcnt);
}
/**
* blkg_tryget - try and get a blkg reference
* @blkg: blkg to get
*
* This is for use when doing an RCU lookup of the blkg. We may be in the midst
* of freeing this blkg, so we can only use it if the refcnt is not zero.
*/
static inline bool blkg_tryget(struct blkcg_gq *blkg)
{
return blkg && percpu_ref_tryget(&blkg->refcnt);
}
/**
* blkg_put - put a blkg reference
* @blkg: blkg to put
*/
static inline void blkg_put(struct blkcg_gq *blkg)
{
percpu_ref_put(&blkg->refcnt);
}
/**
* blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
* @d_blkg: loop cursor pointing to the current descendant
* @pos_css: used for iteration
* @p_blkg: target blkg to walk descendants of
*
* Walk @c_blkg through the descendants of @p_blkg. Must be used with RCU
* read locked. If called under either blkcg or queue lock, the iteration
* is guaranteed to include all and only online blkgs. The caller may
* update @pos_css by calling css_rightmost_descendant() to skip subtree.
* @p_blkg is included in the iteration and the first node to be visited.
*/
#define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg) \
css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css) \
if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \
(p_blkg)->q)))
/**
* blkg_for_each_descendant_post - post-order walk of a blkg's descendants
* @d_blkg: loop cursor pointing to the current descendant
* @pos_css: used for iteration
* @p_blkg: target blkg to walk descendants of
*
* Similar to blkg_for_each_descendant_pre() but performs post-order
* traversal instead. Synchronization rules are the same. @p_blkg is
* included in the iteration and the last node to be visited.
*/
#define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg) \
css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css) \
if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \
(p_blkg)->q)))
static inline void blkcg_bio_issue_init(struct bio *bio)
{
bio_issue_init(&bio->bi_issue, bio_sectors(bio));
}
static inline void blkcg_use_delay(struct blkcg_gq *blkg)
{
if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
return;
if (atomic_add_return(1, &blkg->use_delay) == 1)
atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);
}
static inline int blkcg_unuse_delay(struct blkcg_gq *blkg)
{
int old = atomic_read(&blkg->use_delay);
if (WARN_ON_ONCE(old < 0))
return 0;
if (old == 0)
return 0;
/*
* We do this song and dance because we can race with somebody else
* adding or removing delay. If we just did an atomic_dec we'd end up
* negative and we'd already be in trouble. We need to subtract 1 and
* then check to see if we were the last delay so we can drop the
* congestion count on the cgroup.
*/
while (old && !atomic_try_cmpxchg(&blkg->use_delay, &old, old - 1))
;
if (old == 0)
return 0;
if (old == 1)
atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
return 1;
}
/**
* blkcg_set_delay - Enable allocator delay mechanism with the specified delay amount
* @blkg: target blkg
* @delay: delay duration in nsecs
*
* When enabled with this function, the delay is not decayed and must be
* explicitly cleared with blkcg_clear_delay(). Must not be mixed with
* blkcg_[un]use_delay() and blkcg_add_delay() usages.
*/
static inline void blkcg_set_delay(struct blkcg_gq *blkg, u64 delay)
{
int old = atomic_read(&blkg->use_delay);
/* We only want 1 person setting the congestion count for this blkg. */
if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))
atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);
atomic64_set(&blkg->delay_nsec, delay);
}
/**
* blkcg_clear_delay - Disable allocator delay mechanism
* @blkg: target blkg
*
* Disable use_delay mechanism. See blkcg_set_delay().
*/
static inline void blkcg_clear_delay(struct blkcg_gq *blkg)
{
int old = atomic_read(&blkg->use_delay);
/* We only want 1 person clearing the congestion count for this blkg. */
if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))
atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);
}
/**
* blk_cgroup_mergeable - Determine whether to allow or disallow merges
* @rq: request to merge into
* @bio: bio to merge
*
* @bio and @rq should belong to the same cgroup and their issue_as_root should
* match. The latter is necessary as we don't want to throttle e.g. a metadata
* update because it happens to be next to a regular IO.
*/
static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio)
{
return rq->bio->bi_blkg == bio->bi_blkg &&
bio_issue_as_root_blkg(rq->bio) == bio_issue_as_root_blkg(bio);
}
void blk_cgroup_bio_start(struct bio *bio);
void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta);
#else /* CONFIG_BLK_CGROUP */
struct blkg_policy_data {
};
struct blkcg_policy_data {
};
struct blkcg_policy {
};
struct blkcg {
};
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
static inline void blkg_init_queue(struct request_queue *q) { }
static inline int blkcg_init_disk(struct gendisk *disk) { return 0; }
static inline void blkcg_exit_disk(struct gendisk *disk) { }
static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
static inline int blkcg_activate_policy(struct gendisk *disk,
const struct blkcg_policy *pol) { return 0; }
static inline void blkcg_deactivate_policy(struct gendisk *disk,
const struct blkcg_policy *pol) { }
static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
struct blkcg_policy *pol) { return NULL; }
static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
static inline void blkg_get(struct blkcg_gq *blkg) { }
static inline void blkg_put(struct blkcg_gq *blkg) { }
static inline void blkcg_bio_issue_init(struct bio *bio) { }
static inline void blk_cgroup_bio_start(struct bio *bio) { }
static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { return true; }
#define blk_queue_for_each_rl(rl, q) \
for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
#endif /* CONFIG_BLK_CGROUP */
#endif /* _BLK_CGROUP_PRIVATE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_UACCESS_H__
#define __LINUX_UACCESS_H__
#include <linux/fault-inject-usercopy.h>
#include <linux/instrumented.h>
#include <linux/minmax.h>
#include <linux/sched.h>
#include <linux/thread_info.h>
#include <asm/uaccess.h>
/*
* Architectures that support memory tagging (assigning tags to memory regions,
* embedding these tags into addresses that point to these memory regions, and
* checking that the memory and the pointer tags match on memory accesses)
* redefine this macro to strip tags from pointers.
*
* Passing down mm_struct allows to define untagging rules on per-process
* basis.
*
* It's defined as noop for architectures that don't support memory tagging.
*/
#ifndef untagged_addr
#define untagged_addr(addr) (addr)
#endif
#ifndef untagged_addr_remote
#define untagged_addr_remote(mm, addr) ({ \
mmap_assert_locked(mm); \
untagged_addr(addr); \
})
#endif
/*
* Architectures should provide two primitives (raw_copy_{to,from}_user())
* and get rid of their private instances of copy_{to,from}_user() and
* __copy_{to,from}_user{,_inatomic}().
*
* raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and
* return the amount left to copy. They should assume that access_ok() has
* already been checked (and succeeded); they should *not* zero-pad anything.
* No KASAN or object size checks either - those belong here.
*
* Both of these functions should attempt to copy size bytes starting at from
* into the area starting at to. They must not fetch or store anything
* outside of those areas. Return value must be between 0 (everything
* copied successfully) and size (nothing copied).
*
* If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting
* at to must become equal to the bytes fetched from the corresponding area
* starting at from. All data past to + size - N must be left unmodified.
*
* If copying succeeds, the return value must be 0. If some data cannot be
* fetched, it is permitted to copy less than had been fetched; the only
* hard requirement is that not storing anything at all (i.e. returning size)
* should happen only when nothing could be copied. In other words, you don't
* have to squeeze as much as possible - it is allowed, but not necessary.
*
* For raw_copy_from_user() to always points to kernel memory and no faults
* on store should happen. Interpretation of from is affected by set_fs().
* For raw_copy_to_user() it's the other way round.
*
* Both can be inlined - it's up to architectures whether it wants to bother
* with that. They should not be used directly; they are used to implement
* the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic())
* that are used instead. Out of those, __... ones are inlined. Plain
* copy_{to,from}_user() might or might not be inlined. If you want them
* inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER.
*
* NOTE: only copy_from_user() zero-pads the destination in case of short copy.
* Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything
* at all; their callers absolutely must check the return value.
*
* Biarch ones should also provide raw_copy_in_user() - similar to the above,
* but both source and destination are __user pointers (affected by set_fs()
* as usual) and both source and destination can trigger faults.
*/
static __always_inline __must_check unsigned long
__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
{
unsigned long res;
instrument_copy_from_user_before(to, from, n);
check_object_size(to, n, false);
res = raw_copy_from_user(to, from, n);
instrument_copy_from_user_after(to, from, n, res);
return res;
}
static __always_inline __must_check unsigned long
__copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned long res;
might_fault();
instrument_copy_from_user_before(to, from, n);
if (should_fail_usercopy())
return n;
check_object_size(to, n, false);
res = raw_copy_from_user(to, from, n);
instrument_copy_from_user_after(to, from, n, res);
return res;
}
/**
* __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
* @to: Destination address, in user space.
* @from: Source address, in kernel space.
* @n: Number of bytes to copy.
*
* Context: User context only.
*
* Copy data from kernel space to user space. Caller must check
* the specified block with access_ok() before calling this function.
* The caller should also make sure he pins the user space address
* so that we don't result in page fault and sleep.
*/
static __always_inline __must_check unsigned long
__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
{
if (should_fail_usercopy())
return n;
instrument_copy_to_user(to, from, n);
check_object_size(from, n, true);
return raw_copy_to_user(to, from, n);
}
static __always_inline __must_check unsigned long
__copy_to_user(void __user *to, const void *from, unsigned long n)
{
might_fault();
if (should_fail_usercopy())
return n;
instrument_copy_to_user(to, from, n);
check_object_size(from, n, true);
return raw_copy_to_user(to, from, n);
}
#ifdef INLINE_COPY_FROM_USER
static inline __must_check unsigned long
_copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned long res = n;
might_fault();
if (!should_fail_usercopy() && likely(access_ok(from, n))) {
instrument_copy_from_user_before(to, from, n);
res = raw_copy_from_user(to, from, n);
instrument_copy_from_user_after(to, from, n, res);
}
if (unlikely(res))
memset(to + (n - res), 0, res);
return res;
}
#else
extern __must_check unsigned long
_copy_from_user(void *, const void __user *, unsigned long);
#endif
#ifdef INLINE_COPY_TO_USER
static inline __must_check unsigned long
_copy_to_user(void __user *to, const void *from, unsigned long n)
{
might_fault();
if (should_fail_usercopy())
return n;
if (access_ok(to, n)) {
instrument_copy_to_user(to, from, n);
n = raw_copy_to_user(to, from, n);
}
return n;
}
#else
extern __must_check unsigned long
_copy_to_user(void __user *, const void *, unsigned long);
#endif
static __always_inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long n)
{
if (check_copy_size(to, n, false)) n = _copy_from_user(to, from, n);
return n;
}
static __always_inline unsigned long __must_check
copy_to_user(void __user *to, const void *from, unsigned long n)
{
if (check_copy_size(from, n, true)) n = _copy_to_user(to, from, n);
return n;
}
#ifndef copy_mc_to_kernel
/*
* Without arch opt-in this generic copy_mc_to_kernel() will not handle
* #MC (or arch equivalent) during source read.
*/
static inline unsigned long __must_check
copy_mc_to_kernel(void *dst, const void *src, size_t cnt)
{
memcpy(dst, src, cnt);
return 0;
}
#endif
static __always_inline void pagefault_disabled_inc(void)
{
current->pagefault_disabled++;
}
static __always_inline void pagefault_disabled_dec(void)
{
current->pagefault_disabled--;
}
/*
* These routines enable/disable the pagefault handler. If disabled, it will
* not take any locks and go straight to the fixup table.
*
* User access methods will not sleep when called from a pagefault_disabled()
* environment.
*/
static inline void pagefault_disable(void)
{
pagefault_disabled_inc();
/*
* make sure to have issued the store before a pagefault
* can hit.
*/
barrier();
}
static inline void pagefault_enable(void)
{
/*
* make sure to issue those last loads/stores before enabling
* the pagefault handler again.
*/
barrier();
pagefault_disabled_dec();
}
/*
* Is the pagefault handler disabled? If so, user access methods will not sleep.
*/
static inline bool pagefault_disabled(void)
{
return current->pagefault_disabled != 0;
}
/*
* The pagefault handler is in general disabled by pagefault_disable() or
* when in irq context (via in_atomic()).
*
* This function should only be used by the fault handlers. Other users should
* stick to pagefault_disabled().
* Please NEVER use preempt_disable() to disable the fault handler. With
* !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled.
* in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT.
*/
#define faulthandler_disabled() (pagefault_disabled() || in_atomic())
#ifndef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
/**
* probe_subpage_writeable: probe the user range for write faults at sub-page
* granularity (e.g. arm64 MTE)
* @uaddr: start of address range
* @size: size of address range
*
* Returns 0 on success, the number of bytes not probed on fault.
*
* It is expected that the caller checked for the write permission of each
* page in the range either by put_user() or GUP. The architecture port can
* implement a more efficient get_user() probing if the same sub-page faults
* are triggered by either a read or a write.
*/
static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size)
{
return 0;
}
#endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
#ifndef ARCH_HAS_NOCACHE_UACCESS
static inline __must_check unsigned long
__copy_from_user_inatomic_nocache(void *to, const void __user *from,
unsigned long n)
{
return __copy_from_user_inatomic(to, from, n);
}
#endif /* ARCH_HAS_NOCACHE_UACCESS */
extern __must_check int check_zeroed_user(const void __user *from, size_t size);
/**
* copy_struct_from_user: copy a struct from userspace
* @dst: Destination address, in kernel space. This buffer must be @ksize
* bytes long.
* @ksize: Size of @dst struct.
* @src: Source address, in userspace.
* @usize: (Alleged) size of @src struct.
*
* Copies a struct from userspace to kernel space, in a way that guarantees
* backwards-compatibility for struct syscall arguments (as long as future
* struct extensions are made such that all new fields are *appended* to the
* old struct, and zeroed-out new fields have the same meaning as the old
* struct).
*
* @ksize is just sizeof(*dst), and @usize should've been passed by userspace.
* The recommended usage is something like the following:
*
* SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize)
* {
* int err;
* struct foo karg = {};
*
* if (usize > PAGE_SIZE)
* return -E2BIG;
* if (usize < FOO_SIZE_VER0)
* return -EINVAL;
*
* err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize);
* if (err)
* return err;
*
* // ...
* }
*
* There are three cases to consider:
* * If @usize == @ksize, then it's copied verbatim.
* * If @usize < @ksize, then the userspace has passed an old struct to a
* newer kernel. The rest of the trailing bytes in @dst (@ksize - @usize)
* are to be zero-filled.
* * If @usize > @ksize, then the userspace has passed a new struct to an
* older kernel. The trailing bytes unknown to the kernel (@usize - @ksize)
* are checked to ensure they are zeroed, otherwise -E2BIG is returned.
*
* Returns (in all cases, some data may have been copied):
* * -E2BIG: (@usize > @ksize) and there are non-zero trailing bytes in @src.
* * -EFAULT: access to userspace failed.
*/
static __always_inline __must_check int
copy_struct_from_user(void *dst, size_t ksize, const void __user *src,
size_t usize)
{
size_t size = min(ksize, usize);
size_t rest = max(ksize, usize) - size;
/* Double check if ksize is larger than a known object size. */
if (WARN_ON_ONCE(ksize > __builtin_object_size(dst, 1)))
return -E2BIG;
/* Deal with trailing bytes. */
if (usize < ksize) {
memset(dst + size, 0, rest);
} else if (usize > ksize) {
int ret = check_zeroed_user(src + size, rest);
if (ret <= 0)
return ret ?: -E2BIG;
}
/* Copy the interoperable parts of the struct. */
if (copy_from_user(dst, src, size))
return -EFAULT;
return 0;
}
bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size);
long copy_from_kernel_nofault(void *dst, const void *src, size_t size);
long notrace copy_to_kernel_nofault(void *dst, const void *src, size_t size);
long copy_from_user_nofault(void *dst, const void __user *src, size_t size);
long notrace copy_to_user_nofault(void __user *dst, const void *src,
size_t size);
long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr,
long count);
long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
long count);
long strnlen_user_nofault(const void __user *unsafe_addr, long count);
#ifndef __get_kernel_nofault
#define __get_kernel_nofault(dst, src, type, label) \
do { \
type __user *p = (type __force __user *)(src); \
type data; \
if (__get_user(data, p)) \
goto label; \
*(type *)dst = data; \
} while (0)
#define __put_kernel_nofault(dst, src, type, label) \
do { \
type __user *p = (type __force __user *)(dst); \
type data = *(type *)src; \
if (__put_user(data, p)) \
goto label; \
} while (0)
#endif
/**
* get_kernel_nofault(): safely attempt to read from a location
* @val: read into this variable
* @ptr: address to read from
*
* Returns 0 on success, or -EFAULT.
*/
#define get_kernel_nofault(val, ptr) ({ \
const typeof(val) *__gk_ptr = (ptr); \
copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\
})
#ifndef user_access_begin
#define user_access_begin(ptr,len) access_ok(ptr, len)
#define user_access_end() do { } while (0)
#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)
#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)
#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)
#define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)
static inline unsigned long user_access_save(void) { return 0UL; }
static inline void user_access_restore(unsigned long flags) { }
#endif
#ifndef user_write_access_begin
#define user_write_access_begin user_access_begin
#define user_write_access_end user_access_end
#endif
#ifndef user_read_access_begin
#define user_read_access_begin user_access_begin
#define user_read_access_end user_access_end
#endif
#ifdef CONFIG_HARDENED_USERCOPY
void __noreturn usercopy_abort(const char *name, const char *detail,
bool to_user, unsigned long offset,
unsigned long len);
#endif
#endif /* __LINUX_UACCESS_H__ */
// SPDX-License-Identifier: GPL-2.0
/*
* SafeSetID Linux Security Module
*
* Author: Micah Morton <mortonm@chromium.org>
*
* Copyright (C) 2018 The Chromium OS Authors.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2, as
* published by the Free Software Foundation.
*
*/
#define pr_fmt(fmt) "SafeSetID: " fmt
#include <linux/lsm_hooks.h>
#include <linux/module.h>
#include <linux/ptrace.h>
#include <linux/sched/task_stack.h>
#include <linux/security.h>
#include <uapi/linux/lsm.h>
#include "lsm.h"
/* Flag indicating whether initialization completed */
int safesetid_initialized __initdata;
struct setid_ruleset __rcu *safesetid_setuid_rules;
struct setid_ruleset __rcu *safesetid_setgid_rules;
/* Compute a decision for a transition from @src to @dst under @policy. */
enum sid_policy_type _setid_policy_lookup(struct setid_ruleset *policy,
kid_t src, kid_t dst)
{
struct setid_rule *rule;
enum sid_policy_type result = SIDPOL_DEFAULT;
if (policy->type == UID) {
hash_for_each_possible(policy->rules, rule, next, __kuid_val(src.uid)) {
if (!uid_eq(rule->src_id.uid, src.uid))
continue;
if (uid_eq(rule->dst_id.uid, dst.uid))
return SIDPOL_ALLOWED;
result = SIDPOL_CONSTRAINED;
}
} else if (policy->type == GID) {
hash_for_each_possible(policy->rules, rule, next, __kgid_val(src.gid)) {
if (!gid_eq(rule->src_id.gid, src.gid))
continue;
if (gid_eq(rule->dst_id.gid, dst.gid)){
return SIDPOL_ALLOWED;
}
result = SIDPOL_CONSTRAINED;
}
} else {
/* Should not reach here, report the ID as contrainsted */
result = SIDPOL_CONSTRAINED;
}
return result;
}
/*
* Compute a decision for a transition from @src to @dst under the active
* policy.
*/
static enum sid_policy_type setid_policy_lookup(kid_t src, kid_t dst, enum setid_type new_type)
{
enum sid_policy_type result = SIDPOL_DEFAULT;
struct setid_ruleset *pol;
rcu_read_lock();
if (new_type == UID)
pol = rcu_dereference(safesetid_setuid_rules);
else if (new_type == GID)
pol = rcu_dereference(safesetid_setgid_rules);
else { /* Should not reach here */
result = SIDPOL_CONSTRAINED;
rcu_read_unlock();
return result;
}
if (pol) {
pol->type = new_type;
result = _setid_policy_lookup(pol, src, dst);
}
rcu_read_unlock();
return result;
}
static int safesetid_security_capable(const struct cred *cred,
struct user_namespace *ns,
int cap,
unsigned int opts)
{
/* We're only interested in CAP_SETUID and CAP_SETGID. */
if (cap != CAP_SETUID && cap != CAP_SETGID) return 0;
/*
* If CAP_SET{U/G}ID is currently used for a setid or setgroups syscall, we
* want to let it go through here; the real security check happens later, in
* the task_fix_set{u/g}id or task_fix_setgroups hooks.
*/
if ((opts & CAP_OPT_INSETID) != 0)
return 0;
switch (cap) {
case CAP_SETUID:
/*
* If no policy applies to this task, allow the use of CAP_SETUID for
* other purposes.
*/
if (setid_policy_lookup((kid_t){.uid = cred->uid}, INVALID_ID, UID) == SIDPOL_DEFAULT) return 0;
/*
* Reject use of CAP_SETUID for functionality other than calling
* set*uid() (e.g. setting up userns uid mappings).
*/
pr_warn("Operation requires CAP_SETUID, which is not available to UID %u for operations besides approved set*uid transitions\n",
__kuid_val(cred->uid));
return -EPERM;
case CAP_SETGID:
/*
* If no policy applies to this task, allow the use of CAP_SETGID for
* other purposes.
*/
if (setid_policy_lookup((kid_t){.gid = cred->gid}, INVALID_ID, GID) == SIDPOL_DEFAULT)
return 0;
/*
* Reject use of CAP_SETUID for functionality other than calling
* set*gid() (e.g. setting up userns gid mappings).
*/
pr_warn("Operation requires CAP_SETGID, which is not available to GID %u for operations besides approved set*gid transitions\n",
__kgid_val(cred->gid));
return -EPERM;
default:
/* Error, the only capabilities were checking for is CAP_SETUID/GID */
return 0;
}
return 0;
}
/*
* Check whether a caller with old credentials @old is allowed to switch to
* credentials that contain @new_id.
*/
static bool id_permitted_for_cred(const struct cred *old, kid_t new_id, enum setid_type new_type)
{
bool permitted;
/* If our old creds already had this ID in it, it's fine. */
if (new_type == UID) {
if (uid_eq(new_id.uid, old->uid) || uid_eq(new_id.uid, old->euid) ||
uid_eq(new_id.uid, old->suid))
return true;
} else if (new_type == GID){
if (gid_eq(new_id.gid, old->gid) || gid_eq(new_id.gid, old->egid) ||
gid_eq(new_id.gid, old->sgid))
return true;
} else /* Error, new_type is an invalid type */
return false;
/*
* Transitions to new UIDs require a check against the policy of the old
* RUID.
*/
permitted =
setid_policy_lookup((kid_t){.uid = old->uid}, new_id, new_type) != SIDPOL_CONSTRAINED;
if (!permitted) {
if (new_type == UID) {
pr_warn("UID transition ((%d,%d,%d) -> %d) blocked\n",
__kuid_val(old->uid), __kuid_val(old->euid),
__kuid_val(old->suid), __kuid_val(new_id.uid));
} else if (new_type == GID) {
pr_warn("GID transition ((%d,%d,%d) -> %d) blocked\n",
__kgid_val(old->gid), __kgid_val(old->egid),
__kgid_val(old->sgid), __kgid_val(new_id.gid));
} else /* Error, new_type is an invalid type */
return false;
}
return permitted;
}
/*
* Check whether there is either an exception for user under old cred struct to
* set*uid to user under new cred struct, or the UID transition is allowed (by
* Linux set*uid rules) even without CAP_SETUID.
*/
static int safesetid_task_fix_setuid(struct cred *new,
const struct cred *old,
int flags)
{
/* Do nothing if there are no setuid restrictions for our old RUID. */
if (setid_policy_lookup((kid_t){.uid = old->uid}, INVALID_ID, UID) == SIDPOL_DEFAULT)
return 0;
if (id_permitted_for_cred(old, (kid_t){.uid = new->uid}, UID) &&
id_permitted_for_cred(old, (kid_t){.uid = new->euid}, UID) &&
id_permitted_for_cred(old, (kid_t){.uid = new->suid}, UID) &&
id_permitted_for_cred(old, (kid_t){.uid = new->fsuid}, UID))
return 0;
/*
* Kill this process to avoid potential security vulnerabilities
* that could arise from a missing allowlist entry preventing a
* privileged process from dropping to a lesser-privileged one.
*/
force_sig(SIGKILL);
return -EACCES;
}
static int safesetid_task_fix_setgid(struct cred *new,
const struct cred *old,
int flags)
{
/* Do nothing if there are no setgid restrictions for our old RGID. */
if (setid_policy_lookup((kid_t){.gid = old->gid}, INVALID_ID, GID) == SIDPOL_DEFAULT)
return 0;
if (id_permitted_for_cred(old, (kid_t){.gid = new->gid}, GID) &&
id_permitted_for_cred(old, (kid_t){.gid = new->egid}, GID) &&
id_permitted_for_cred(old, (kid_t){.gid = new->sgid}, GID) &&
id_permitted_for_cred(old, (kid_t){.gid = new->fsgid}, GID))
return 0;
/*
* Kill this process to avoid potential security vulnerabilities
* that could arise from a missing allowlist entry preventing a
* privileged process from dropping to a lesser-privileged one.
*/
force_sig(SIGKILL);
return -EACCES;
}
static int safesetid_task_fix_setgroups(struct cred *new, const struct cred *old)
{
int i;
/* Do nothing if there are no setgid restrictions for our old RGID. */
if (setid_policy_lookup((kid_t){.gid = old->gid}, INVALID_ID, GID) == SIDPOL_DEFAULT)
return 0;
get_group_info(new->group_info);
for (i = 0; i < new->group_info->ngroups; i++) {
if (!id_permitted_for_cred(old, (kid_t){.gid = new->group_info->gid[i]}, GID)) {
put_group_info(new->group_info);
/*
* Kill this process to avoid potential security vulnerabilities
* that could arise from a missing allowlist entry preventing a
* privileged process from dropping to a lesser-privileged one.
*/
force_sig(SIGKILL);
return -EACCES;
}
}
put_group_info(new->group_info);
return 0;
}
static const struct lsm_id safesetid_lsmid = {
.name = "safesetid",
.id = LSM_ID_SAFESETID,
};
static struct security_hook_list safesetid_security_hooks[] = {
LSM_HOOK_INIT(task_fix_setuid, safesetid_task_fix_setuid),
LSM_HOOK_INIT(task_fix_setgid, safesetid_task_fix_setgid),
LSM_HOOK_INIT(task_fix_setgroups, safesetid_task_fix_setgroups),
LSM_HOOK_INIT(capable, safesetid_security_capable)
};
static int __init safesetid_security_init(void)
{
security_add_hooks(safesetid_security_hooks,
ARRAY_SIZE(safesetid_security_hooks),
&safesetid_lsmid);
/* Report that SafeSetID successfully initialized */
safesetid_initialized = 1;
return 0;
}
DEFINE_LSM(safesetid_security_init) = {
.init = safesetid_security_init,
.name = "safesetid",
};
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/usb/core/generic.c - generic driver for USB devices (not interfaces)
*
* (C) Copyright 2005 Greg Kroah-Hartman <gregkh@suse.de>
*
* based on drivers/usb/usb.c which had the following copyrights:
* (C) Copyright Linus Torvalds 1999
* (C) Copyright Johannes Erdfelt 1999-2001
* (C) Copyright Andreas Gal 1999
* (C) Copyright Gregory P. Smith 1999
* (C) Copyright Deti Fliegl 1999 (new USB architecture)
* (C) Copyright Randy Dunlap 2000
* (C) Copyright David Brownell 2000-2004
* (C) Copyright Yggdrasil Computing, Inc. 2000
* (usb_device_id matching changes by Adam J. Richter)
* (C) Copyright Greg Kroah-Hartman 2002-2003
*
* Released under the GPLv2 only.
*/
#include <linux/usb.h>
#include <linux/usb/hcd.h>
#include <uapi/linux/usb/audio.h>
#include "usb.h"
static inline const char *plural(int n)
{
return (n == 1 ? "" : "s");
}
static int is_rndis(struct usb_interface_descriptor *desc)
{
return desc->bInterfaceClass == USB_CLASS_COMM
&& desc->bInterfaceSubClass == 2 && desc->bInterfaceProtocol == 0xff;
}
static int is_activesync(struct usb_interface_descriptor *desc)
{
return desc->bInterfaceClass == USB_CLASS_MISC
&& desc->bInterfaceSubClass == 1 && desc->bInterfaceProtocol == 1;
}
static bool is_audio(struct usb_interface_descriptor *desc)
{
return desc->bInterfaceClass == USB_CLASS_AUDIO;
}
static bool is_uac3_config(struct usb_interface_descriptor *desc)
{
return desc->bInterfaceProtocol == UAC_VERSION_3;
}
int usb_choose_configuration(struct usb_device *udev)
{
int i;
int num_configs;
int insufficient_power = 0;
struct usb_host_config *c, *best;
struct usb_device_driver *udriver;
/*
* If a USB device (not an interface) doesn't have a driver then the
* kernel has no business trying to select or install a configuration
* for it.
*/
if (!udev->dev.driver)
return -1;
udriver = to_usb_device_driver(udev->dev.driver);
if (usb_device_is_owned(udev))
return 0;
if (udriver->choose_configuration) { i = udriver->choose_configuration(udev);
if (i >= 0)
return i;
}
best = NULL;
c = udev->config;
num_configs = udev->descriptor.bNumConfigurations;
for (i = 0; i < num_configs; (i++, c++)) {
struct usb_interface_descriptor *desc = NULL;
/* It's possible that a config has no interfaces! */
if (c->desc.bNumInterfaces > 0) desc = &c->intf_cache[0]->altsetting->desc;
/*
* HP's USB bus-powered keyboard has only one configuration
* and it claims to be self-powered; other devices may have
* similar errors in their descriptors. If the next test
* were allowed to execute, such configurations would always
* be rejected and the devices would not work as expected.
* In the meantime, we run the risk of selecting a config
* that requires external power at a time when that power
* isn't available. It seems to be the lesser of two evils.
*
* Bugzilla #6448 reports a device that appears to crash
* when it receives a GET_DEVICE_STATUS request! We don't
* have any other way to tell whether a device is self-powered,
* but since we don't use that information anywhere but here,
* the call has been removed.
*
* Maybe the GET_DEVICE_STATUS call and the test below can
* be reinstated when device firmwares become more reliable.
* Don't hold your breath.
*/
#if 0
/* Rule out self-powered configs for a bus-powered device */
if (bus_powered && (c->desc.bmAttributes &
USB_CONFIG_ATT_SELFPOWER))
continue;
#endif
/*
* The next test may not be as effective as it should be.
* Some hubs have errors in their descriptor, claiming
* to be self-powered when they are really bus-powered.
* We will overestimate the amount of current such hubs
* make available for each port.
*
* This is a fairly benign sort of failure. It won't
* cause us to reject configurations that we should have
* accepted.
*/
/* Rule out configs that draw too much bus current */
if (usb_get_max_power(udev, c) > udev->bus_mA) { insufficient_power++;
continue;
}
/*
* Select first configuration as default for audio so that
* devices that don't comply with UAC3 protocol are supported.
* But, still iterate through other configurations and
* select UAC3 compliant config if present.
*/
if (desc && is_audio(desc)) {
/* Always prefer the first found UAC3 config */
if (is_uac3_config(desc)) {
best = c;
break;
}
/* If there is no UAC3 config, prefer the first config */
else if (i == 0)
best = c;
/* Unconditional continue, because the rest of the code
* in the loop is irrelevant for audio devices, and
* because it can reassign best, which for audio devices
* we don't want.
*/
continue;
}
/* When the first config's first interface is one of Microsoft's
* pet nonstandard Ethernet-over-USB protocols, ignore it unless
* this kernel has enabled the necessary host side driver.
* But: Don't ignore it if it's the only config.
*/
if (i == 0 && num_configs > 1 && desc && (is_rndis(desc) || is_activesync(desc))) {
#if !defined(CONFIG_USB_NET_RNDIS_HOST) && !defined(CONFIG_USB_NET_RNDIS_HOST_MODULE)
continue;
#else
best = c;
#endif
}
/* From the remaining configs, choose the first one whose
* first interface is for a non-vendor-specific class.
* Reason: Linux is more likely to have a class driver
* than a vendor-specific driver. */
else if (udev->descriptor.bDeviceClass !=
USB_CLASS_VENDOR_SPEC &&
(desc && desc->bInterfaceClass !=
USB_CLASS_VENDOR_SPEC)) {
best = c;
break;
}
/* If all the remaining configs are vendor-specific,
* choose the first one. */
else if (!best)
best = c;
}
if (insufficient_power > 0) dev_info(&udev->dev, "rejected %d configuration%s "
"due to insufficient available bus power\n",
insufficient_power, plural(insufficient_power));
if (best) { i = best->desc.bConfigurationValue; dev_dbg(&udev->dev,
"configuration #%d chosen from %d choice%s\n",
i, num_configs, plural(num_configs));
} else {
i = -1;
dev_warn(&udev->dev,
"no configuration chosen from %d choice%s\n",
num_configs, plural(num_configs));
}
return i;
}
EXPORT_SYMBOL_GPL(usb_choose_configuration);
static int __check_for_non_generic_match(struct device_driver *drv, void *data)
{
struct usb_device *udev = data;
struct usb_device_driver *udrv;
if (!is_usb_device_driver(drv))
return 0;
udrv = to_usb_device_driver(drv);
if (udrv == &usb_generic_driver)
return 0;
return usb_driver_applicable(udev, udrv);}
static bool usb_generic_driver_match(struct usb_device *udev)
{
if (udev->use_generic_driver)
return true;
/*
* If any other driver wants the device, leave the device to this other
* driver.
*/
if (bus_for_each_drv(&usb_bus_type, NULL, udev, __check_for_non_generic_match))
return false;
return true;
}
int usb_generic_driver_probe(struct usb_device *udev)
{
int err, c;
/* Choose and set the configuration. This registers the interfaces
* with the driver core and lets interface drivers bind to them.
*/
if (udev->authorized == 0) dev_err(&udev->dev, "Device is not authorized for usage\n");
else {
c = usb_choose_configuration(udev);
if (c >= 0) {
err = usb_set_configuration(udev, c); if (err && err != -ENODEV) { dev_err(&udev->dev, "can't set config #%d, error %d\n",
c, err);
/* This need not be fatal. The user can try to
* set other configurations. */
}
}
}
/* USB device state == configured ... usable */
usb_notify_add_device(udev);
return 0;
}
void usb_generic_driver_disconnect(struct usb_device *udev)
{
usb_notify_remove_device(udev);
/* if this is only an unbind, not a physical disconnect, then
* unconfigure the device */
if (udev->actconfig)
usb_set_configuration(udev, -1);
}
#ifdef CONFIG_PM
int usb_generic_driver_suspend(struct usb_device *udev, pm_message_t msg)
{
int rc;
/* Normal USB devices suspend through their upstream port.
* Root hubs don't have upstream ports to suspend,
* so we have to shut down their downstream HC-to-USB
* interfaces manually by doing a bus (or "global") suspend.
*/
if (!udev->parent)
rc = hcd_bus_suspend(udev, msg);
/*
* Non-root USB2 devices don't need to do anything for FREEZE
* or PRETHAW. USB3 devices don't support global suspend and
* needs to be selectively suspended.
*/
else if ((msg.event == PM_EVENT_FREEZE || msg.event == PM_EVENT_PRETHAW)
&& (udev->speed < USB_SPEED_SUPER))
rc = 0;
else
rc = usb_port_suspend(udev, msg);
if (rc == 0)
usbfs_notify_suspend(udev);
return rc;
}
int usb_generic_driver_resume(struct usb_device *udev, pm_message_t msg)
{
int rc;
/* Normal USB devices resume/reset through their upstream port.
* Root hubs don't have upstream ports to resume or reset,
* so we have to start up their downstream HC-to-USB
* interfaces manually by doing a bus (or "global") resume.
*/
if (!udev->parent) rc = hcd_bus_resume(udev, msg);
else
rc = usb_port_resume(udev, msg); if (rc == 0) usbfs_notify_resume(udev); return rc;
}
#endif /* CONFIG_PM */
struct usb_device_driver usb_generic_driver = {
.name = "usb",
.match = usb_generic_driver_match,
.probe = usb_generic_driver_probe,
.disconnect = usb_generic_driver_disconnect,
#ifdef CONFIG_PM
.suspend = usb_generic_driver_suspend,
.resume = usb_generic_driver_resume,
#endif
.supports_autosuspend = 1,
};
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Variant of atomic_t specialized for reference counts.
*
* The interface matches the atomic_t interface (to aid in porting) but only
* provides the few functions one should use for reference counting.
*
* Saturation semantics
* ====================
*
* refcount_t differs from atomic_t in that the counter saturates at
* REFCOUNT_SATURATED and will not move once there. This avoids wrapping the
* counter and causing 'spurious' use-after-free issues. In order to avoid the
* cost associated with introducing cmpxchg() loops into all of the saturating
* operations, we temporarily allow the counter to take on an unchecked value
* and then explicitly set it to REFCOUNT_SATURATED on detecting that underflow
* or overflow has occurred. Although this is racy when multiple threads
* access the refcount concurrently, by placing REFCOUNT_SATURATED roughly
* equidistant from 0 and INT_MAX we minimise the scope for error:
*
* INT_MAX REFCOUNT_SATURATED UINT_MAX
* 0 (0x7fff_ffff) (0xc000_0000) (0xffff_ffff)
* +--------------------------------+----------------+----------------+
* <---------- bad value! ---------->
*
* (in a signed view of the world, the "bad value" range corresponds to
* a negative counter value).
*
* As an example, consider a refcount_inc() operation that causes the counter
* to overflow:
*
* int old = atomic_fetch_add_relaxed(r);
* // old is INT_MAX, refcount now INT_MIN (0x8000_0000)
* if (old < 0)
* atomic_set(r, REFCOUNT_SATURATED);
*
* If another thread also performs a refcount_inc() operation between the two
* atomic operations, then the count will continue to edge closer to 0. If it
* reaches a value of 1 before /any/ of the threads reset it to the saturated
* value, then a concurrent refcount_dec_and_test() may erroneously free the
* underlying object.
* Linux limits the maximum number of tasks to PID_MAX_LIMIT, which is currently
* 0x400000 (and can't easily be raised in the future beyond FUTEX_TID_MASK).
* With the current PID limit, if no batched refcounting operations are used and
* the attacker can't repeatedly trigger kernel oopses in the middle of refcount
* operations, this makes it impossible for a saturated refcount to leave the
* saturation range, even if it is possible for multiple uses of the same
* refcount to nest in the context of a single task:
*
* (UINT_MAX+1-REFCOUNT_SATURATED) / PID_MAX_LIMIT =
* 0x40000000 / 0x400000 = 0x100 = 256
*
* If hundreds of references are added/removed with a single refcounting
* operation, it may potentially be possible to leave the saturation range; but
* given the precise timing details involved with the round-robin scheduling of
* each thread manipulating the refcount and the need to hit the race multiple
* times in succession, there doesn't appear to be a practical avenue of attack
* even if using refcount_add() operations with larger increments.
*
* Memory ordering
* ===============
*
* Memory ordering rules are slightly relaxed wrt regular atomic_t functions
* and provide only what is strictly required for refcounts.
*
* The increments are fully relaxed; these will not provide ordering. The
* rationale is that whatever is used to obtain the object we're increasing the
* reference count on will provide the ordering. For locked data structures,
* its the lock acquire, for RCU/lockless data structures its the dependent
* load.
*
* Do note that inc_not_zero() provides a control dependency which will order
* future stores against the inc, this ensures we'll never modify the object
* if we did not in fact acquire a reference.
*
* The decrements will provide release order, such that all the prior loads and
* stores will be issued before, it also provides a control dependency, which
* will order us against the subsequent free().
*
* The control dependency is against the load of the cmpxchg (ll/sc) that
* succeeded. This means the stores aren't fully ordered, but this is fine
* because the 1->0 transition indicates no concurrency.
*
* Note that the allocator is responsible for ordering things between free()
* and alloc().
*
* The decrements dec_and_test() and sub_and_test() also provide acquire
* ordering on success.
*
*/
#ifndef _LINUX_REFCOUNT_H
#define _LINUX_REFCOUNT_H
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/compiler.h>
#include <linux/limits.h>
#include <linux/refcount_types.h>
#include <linux/spinlock_types.h>
struct mutex;
#define REFCOUNT_INIT(n) { .refs = ATOMIC_INIT(n), }
#define REFCOUNT_MAX INT_MAX
#define REFCOUNT_SATURATED (INT_MIN / 2)
enum refcount_saturation_type {
REFCOUNT_ADD_NOT_ZERO_OVF,
REFCOUNT_ADD_OVF,
REFCOUNT_ADD_UAF,
REFCOUNT_SUB_UAF,
REFCOUNT_DEC_LEAK,
};
void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t);
/**
* refcount_set - set a refcount's value
* @r: the refcount
* @n: value to which the refcount will be set
*/
static inline void refcount_set(refcount_t *r, int n)
{
atomic_set(&r->refs, n);
}
/**
* refcount_read - get a refcount's value
* @r: the refcount
*
* Return: the refcount's value
*/
static inline unsigned int refcount_read(const refcount_t *r)
{
return atomic_read(&r->refs);
}
static inline __must_check __signed_wrap
bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp)
{
int old = refcount_read(r);
do {
if (!old)
break;
} while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));
if (oldp)
*oldp = old;
if (unlikely(old < 0 || old + i < 0)) refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF); return old;
}
/**
* refcount_add_not_zero - add a value to a refcount unless it is 0
* @i: the value to add to the refcount
* @r: the refcount
*
* Will saturate at REFCOUNT_SATURATED and WARN.
*
* Provides no memory ordering, it is assumed the caller has guaranteed the
* object memory to be stable (RCU, etc.). It does provide a control dependency
* and thereby orders future stores. See the comment on top.
*
* Use of this function is not recommended for the normal reference counting
* use case in which references are taken and released one at a time. In these
* cases, refcount_inc(), or one of its variants, should instead be used to
* increment a reference count.
*
* Return: false if the passed refcount is 0, true otherwise
*/
static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
{
return __refcount_add_not_zero(i, r, NULL);
}
static inline __signed_wrap
void __refcount_add(int i, refcount_t *r, int *oldp)
{
int old = atomic_fetch_add_relaxed(i, &r->refs);
if (oldp)
*oldp = old;
if (unlikely(!old))
refcount_warn_saturate(r, REFCOUNT_ADD_UAF); else if (unlikely(old < 0 || old + i < 0)) refcount_warn_saturate(r, REFCOUNT_ADD_OVF);
}
/**
* refcount_add - add a value to a refcount
* @i: the value to add to the refcount
* @r: the refcount
*
* Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN.
*
* Provides no memory ordering, it is assumed the caller has guaranteed the
* object memory to be stable (RCU, etc.). It does provide a control dependency
* and thereby orders future stores. See the comment on top.
*
* Use of this function is not recommended for the normal reference counting
* use case in which references are taken and released one at a time. In these
* cases, refcount_inc(), or one of its variants, should instead be used to
* increment a reference count.
*/
static inline void refcount_add(int i, refcount_t *r)
{
__refcount_add(i, r, NULL);
}
static inline __must_check bool __refcount_inc_not_zero(refcount_t *r, int *oldp)
{
return __refcount_add_not_zero(1, r, oldp);
}
/**
* refcount_inc_not_zero - increment a refcount unless it is 0
* @r: the refcount to increment
*
* Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED
* and WARN.
*
* Provides no memory ordering, it is assumed the caller has guaranteed the
* object memory to be stable (RCU, etc.). It does provide a control dependency
* and thereby orders future stores. See the comment on top.
*
* Return: true if the increment was successful, false otherwise
*/
static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
{
return __refcount_inc_not_zero(r, NULL);
}
static inline void __refcount_inc(refcount_t *r, int *oldp)
{
__refcount_add(1, r, oldp);
}
/**
* refcount_inc - increment a refcount
* @r: the refcount to increment
*
* Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN.
*
* Provides no memory ordering, it is assumed the caller already has a
* reference on the object.
*
* Will WARN if the refcount is 0, as this represents a possible use-after-free
* condition.
*/
static inline void refcount_inc(refcount_t *r)
{
__refcount_inc(r, NULL);
}
static inline __must_check __signed_wrap
bool __refcount_sub_and_test(int i, refcount_t *r, int *oldp)
{
int old = atomic_fetch_sub_release(i, &r->refs);
if (oldp)
*oldp = old;
if (old == i) {
smp_acquire__after_ctrl_dep();
return true;
}
if (unlikely(old < 0 || old - i < 0)) refcount_warn_saturate(r, REFCOUNT_SUB_UAF);
return false;
}
/**
* refcount_sub_and_test - subtract from a refcount and test if it is 0
* @i: amount to subtract from the refcount
* @r: the refcount
*
* Similar to atomic_dec_and_test(), but it will WARN, return false and
* ultimately leak on underflow and will fail to decrement when saturated
* at REFCOUNT_SATURATED.
*
* Provides release memory ordering, such that prior loads and stores are done
* before, and provides an acquire ordering on success such that free()
* must come after.
*
* Use of this function is not recommended for the normal reference counting
* use case in which references are taken and released one at a time. In these
* cases, refcount_dec(), or one of its variants, should instead be used to
* decrement a reference count.
*
* Return: true if the resulting refcount is 0, false otherwise
*/
static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
{
return __refcount_sub_and_test(i, r, NULL);
}
static inline __must_check bool __refcount_dec_and_test(refcount_t *r, int *oldp)
{
return __refcount_sub_and_test(1, r, oldp);
}
/**
* refcount_dec_and_test - decrement a refcount and test if it is 0
* @r: the refcount
*
* Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
* decrement when saturated at REFCOUNT_SATURATED.
*
* Provides release memory ordering, such that prior loads and stores are done
* before, and provides an acquire ordering on success such that free()
* must come after.
*
* Return: true if the resulting refcount is 0, false otherwise
*/
static inline __must_check bool refcount_dec_and_test(refcount_t *r)
{
return __refcount_dec_and_test(r, NULL);
}
static inline void __refcount_dec(refcount_t *r, int *oldp)
{
int old = atomic_fetch_sub_release(1, &r->refs);
if (oldp)
*oldp = old;
if (unlikely(old <= 1))
refcount_warn_saturate(r, REFCOUNT_DEC_LEAK);
}
/**
* refcount_dec - decrement a refcount
* @r: the refcount
*
* Similar to atomic_dec(), it will WARN on underflow and fail to decrement
* when saturated at REFCOUNT_SATURATED.
*
* Provides release memory ordering, such that prior loads and stores are done
* before.
*/
static inline void refcount_dec(refcount_t *r)
{
__refcount_dec(r, NULL);
}
extern __must_check bool refcount_dec_if_one(refcount_t *r);
extern __must_check bool refcount_dec_not_one(refcount_t *r);
extern __must_check bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock) __cond_acquires(lock);
extern __must_check bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock) __cond_acquires(lock);
extern __must_check bool refcount_dec_and_lock_irqsave(refcount_t *r,
spinlock_t *lock,
unsigned long *flags) __cond_acquires(lock);
#endif /* _LINUX_REFCOUNT_H */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Directory notifications for Linux.
*
* Copyright (C) 2000,2001,2002 Stephen Rothwell
*
* Copyright (C) 2009 Eric Paris <Red Hat Inc>
* dnotify was largly rewritten to use the new fsnotify infrastructure
*/
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/sched/signal.h>
#include <linux/dnotify.h>
#include <linux/init.h>
#include <linux/security.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/fdtable.h>
#include <linux/fsnotify_backend.h>
static int dir_notify_enable __read_mostly = 1;
#ifdef CONFIG_SYSCTL
static struct ctl_table dnotify_sysctls[] = {
{
.procname = "dir-notify-enable",
.data = &dir_notify_enable,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
};
static void __init dnotify_sysctl_init(void)
{
register_sysctl_init("fs", dnotify_sysctls);
}
#else
#define dnotify_sysctl_init() do { } while (0)
#endif
static struct kmem_cache *dnotify_struct_cache __ro_after_init;
static struct kmem_cache *dnotify_mark_cache __ro_after_init;
static struct fsnotify_group *dnotify_group __ro_after_init;
/*
* dnotify will attach one of these to each inode (i_fsnotify_marks) which
* is being watched by dnotify. If multiple userspace applications are watching
* the same directory with dnotify their information is chained in dn
*/
struct dnotify_mark {
struct fsnotify_mark fsn_mark;
struct dnotify_struct *dn;
};
/*
* When a process starts or stops watching an inode the set of events which
* dnotify cares about for that inode may change. This function runs the
* list of everything receiving dnotify events about this directory and calculates
* the set of all those events. After it updates what dnotify is interested in
* it calls the fsnotify function so it can update the set of all events relevant
* to this inode.
*/
static void dnotify_recalc_inode_mask(struct fsnotify_mark *fsn_mark)
{
__u32 new_mask = 0;
struct dnotify_struct *dn;
struct dnotify_mark *dn_mark = container_of(fsn_mark,
struct dnotify_mark,
fsn_mark);
assert_spin_locked(&fsn_mark->lock);
for (dn = dn_mark->dn; dn != NULL; dn = dn->dn_next)
new_mask |= (dn->dn_mask & ~FS_DN_MULTISHOT);
if (fsn_mark->mask == new_mask)
return;
fsn_mark->mask = new_mask;
fsnotify_recalc_mask(fsn_mark->connector);
}
/*
* Mains fsnotify call where events are delivered to dnotify.
* Find the dnotify mark on the relevant inode, run the list of dnotify structs
* on that mark and determine which of them has expressed interest in receiving
* events of this type. When found send the correct process and signal and
* destroy the dnotify struct if it was not registered to receive multiple
* events.
*/
static int dnotify_handle_event(struct fsnotify_mark *inode_mark, u32 mask,
struct inode *inode, struct inode *dir,
const struct qstr *name, u32 cookie)
{
struct dnotify_mark *dn_mark;
struct dnotify_struct *dn;
struct dnotify_struct **prev;
struct fown_struct *fown;
__u32 test_mask = mask & ~FS_EVENT_ON_CHILD;
/* not a dir, dnotify doesn't care */
if (!dir && !(mask & FS_ISDIR))
return 0;
dn_mark = container_of(inode_mark, struct dnotify_mark, fsn_mark);
spin_lock(&inode_mark->lock);
prev = &dn_mark->dn;
while ((dn = *prev) != NULL) {
if ((dn->dn_mask & test_mask) == 0) {
prev = &dn->dn_next;
continue;
}
fown = &dn->dn_filp->f_owner;
send_sigio(fown, dn->dn_fd, POLL_MSG);
if (dn->dn_mask & FS_DN_MULTISHOT)
prev = &dn->dn_next;
else {
*prev = dn->dn_next;
kmem_cache_free(dnotify_struct_cache, dn);
dnotify_recalc_inode_mask(inode_mark);
}
}
spin_unlock(&inode_mark->lock);
return 0;
}
static void dnotify_free_mark(struct fsnotify_mark *fsn_mark)
{
struct dnotify_mark *dn_mark = container_of(fsn_mark,
struct dnotify_mark,
fsn_mark);
BUG_ON(dn_mark->dn);
kmem_cache_free(dnotify_mark_cache, dn_mark);
}
static const struct fsnotify_ops dnotify_fsnotify_ops = {
.handle_inode_event = dnotify_handle_event,
.free_mark = dnotify_free_mark,
};
/*
* Called every time a file is closed. Looks first for a dnotify mark on the
* inode. If one is found run all of the ->dn structures attached to that
* mark for one relevant to this process closing the file and remove that
* dnotify_struct. If that was the last dnotify_struct also remove the
* fsnotify_mark.
*/
void dnotify_flush(struct file *filp, fl_owner_t id)
{
struct fsnotify_mark *fsn_mark;
struct dnotify_mark *dn_mark;
struct dnotify_struct *dn;
struct dnotify_struct **prev;
struct inode *inode;
bool free = false;
inode = file_inode(filp);
if (!S_ISDIR(inode->i_mode))
return;
fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, dnotify_group);
if (!fsn_mark)
return;
dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark);
fsnotify_group_lock(dnotify_group); spin_lock(&fsn_mark->lock);
prev = &dn_mark->dn;
while ((dn = *prev) != NULL) {
if ((dn->dn_owner == id) && (dn->dn_filp == filp)) { *prev = dn->dn_next;
kmem_cache_free(dnotify_struct_cache, dn);
dnotify_recalc_inode_mask(fsn_mark);
break;
}
prev = &dn->dn_next;
}
spin_unlock(&fsn_mark->lock);
/* nothing else could have found us thanks to the dnotify_groups
mark_mutex */
if (dn_mark->dn == NULL) {
fsnotify_detach_mark(fsn_mark);
free = true;
}
fsnotify_group_unlock(dnotify_group);
if (free)
fsnotify_free_mark(fsn_mark); fsnotify_put_mark(fsn_mark);
}
/* this conversion is done only at watch creation */
static __u32 convert_arg(unsigned int arg)
{
__u32 new_mask = FS_EVENT_ON_CHILD;
if (arg & DN_MULTISHOT)
new_mask |= FS_DN_MULTISHOT;
if (arg & DN_DELETE)
new_mask |= (FS_DELETE | FS_MOVED_FROM);
if (arg & DN_MODIFY)
new_mask |= FS_MODIFY;
if (arg & DN_ACCESS)
new_mask |= FS_ACCESS;
if (arg & DN_ATTRIB)
new_mask |= FS_ATTRIB;
if (arg & DN_RENAME)
new_mask |= FS_RENAME;
if (arg & DN_CREATE)
new_mask |= (FS_CREATE | FS_MOVED_TO);
return new_mask;
}
/*
* If multiple processes watch the same inode with dnotify there is only one
* dnotify mark in inode->i_fsnotify_marks but we chain a dnotify_struct
* onto that mark. This function either attaches the new dnotify_struct onto
* that list, or it |= the mask onto an existing dnofiy_struct.
*/
static int attach_dn(struct dnotify_struct *dn, struct dnotify_mark *dn_mark,
fl_owner_t id, int fd, struct file *filp, __u32 mask)
{
struct dnotify_struct *odn;
odn = dn_mark->dn;
while (odn != NULL) {
/* adding more events to existing dnofiy_struct? */
if ((odn->dn_owner == id) && (odn->dn_filp == filp)) {
odn->dn_fd = fd;
odn->dn_mask |= mask;
return -EEXIST;
}
odn = odn->dn_next;
}
dn->dn_mask = mask;
dn->dn_fd = fd;
dn->dn_filp = filp;
dn->dn_owner = id;
dn->dn_next = dn_mark->dn;
dn_mark->dn = dn;
return 0;
}
/*
* When a process calls fcntl to attach a dnotify watch to a directory it ends
* up here. Allocate both a mark for fsnotify to add and a dnotify_struct to be
* attached to the fsnotify_mark.
*/
int fcntl_dirnotify(int fd, struct file *filp, unsigned int arg)
{
struct dnotify_mark *new_dn_mark, *dn_mark;
struct fsnotify_mark *new_fsn_mark, *fsn_mark;
struct dnotify_struct *dn;
struct inode *inode;
fl_owner_t id = current->files;
struct file *f = NULL;
int destroy = 0, error = 0;
__u32 mask;
/* we use these to tell if we need to kfree */
new_fsn_mark = NULL;
dn = NULL;
if (!dir_notify_enable) {
error = -EINVAL;
goto out_err;
}
/* a 0 mask means we are explicitly removing the watch */
if ((arg & ~DN_MULTISHOT) == 0) {
dnotify_flush(filp, id);
error = 0;
goto out_err;
}
/* dnotify only works on directories */
inode = file_inode(filp);
if (!S_ISDIR(inode->i_mode)) {
error = -ENOTDIR;
goto out_err;
}
/*
* convert the userspace DN_* "arg" to the internal FS_*
* defined in fsnotify
*/
mask = convert_arg(arg);
error = security_path_notify(&filp->f_path, mask,
FSNOTIFY_OBJ_TYPE_INODE);
if (error)
goto out_err;
/* expect most fcntl to add new rather than augment old */
dn = kmem_cache_alloc(dnotify_struct_cache, GFP_KERNEL);
if (!dn) {
error = -ENOMEM;
goto out_err;
}
/* new fsnotify mark, we expect most fcntl calls to add a new mark */
new_dn_mark = kmem_cache_alloc(dnotify_mark_cache, GFP_KERNEL);
if (!new_dn_mark) {
error = -ENOMEM;
goto out_err;
}
/* set up the new_fsn_mark and new_dn_mark */
new_fsn_mark = &new_dn_mark->fsn_mark;
fsnotify_init_mark(new_fsn_mark, dnotify_group);
new_fsn_mark->mask = mask;
new_dn_mark->dn = NULL;
/* this is needed to prevent the fcntl/close race described below */
fsnotify_group_lock(dnotify_group);
/* add the new_fsn_mark or find an old one. */
fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, dnotify_group);
if (fsn_mark) {
dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark);
spin_lock(&fsn_mark->lock);
} else {
error = fsnotify_add_inode_mark_locked(new_fsn_mark, inode, 0);
if (error) {
fsnotify_group_unlock(dnotify_group);
goto out_err;
}
spin_lock(&new_fsn_mark->lock);
fsn_mark = new_fsn_mark;
dn_mark = new_dn_mark;
/* we used new_fsn_mark, so don't free it */
new_fsn_mark = NULL;
}
rcu_read_lock();
f = lookup_fdget_rcu(fd);
rcu_read_unlock();
/* if (f != filp) means that we lost a race and another task/thread
* actually closed the fd we are still playing with before we grabbed
* the dnotify_groups mark_mutex and fsn_mark->lock. Since closing the
* fd is the only time we clean up the marks we need to get our mark
* off the list. */
if (f != filp) {
/* if we added ourselves, shoot ourselves, it's possible that
* the flush actually did shoot this fsn_mark. That's fine too
* since multiple calls to destroy_mark is perfectly safe, if
* we found a dn_mark already attached to the inode, just sod
* off silently as the flush at close time dealt with it.
*/
if (dn_mark == new_dn_mark)
destroy = 1;
error = 0;
goto out;
}
__f_setown(filp, task_pid(current), PIDTYPE_TGID, 0);
error = attach_dn(dn, dn_mark, id, fd, filp, mask);
/* !error means that we attached the dn to the dn_mark, so don't free it */
if (!error)
dn = NULL;
/* -EEXIST means that we didn't add this new dn and used an old one.
* that isn't an error (and the unused dn should be freed) */
else if (error == -EEXIST)
error = 0;
dnotify_recalc_inode_mask(fsn_mark);
out:
spin_unlock(&fsn_mark->lock);
if (destroy)
fsnotify_detach_mark(fsn_mark);
fsnotify_group_unlock(dnotify_group);
if (destroy)
fsnotify_free_mark(fsn_mark);
fsnotify_put_mark(fsn_mark);
out_err:
if (new_fsn_mark)
fsnotify_put_mark(new_fsn_mark);
if (dn)
kmem_cache_free(dnotify_struct_cache, dn);
if (f)
fput(f);
return error;
}
static int __init dnotify_init(void)
{
dnotify_struct_cache = KMEM_CACHE(dnotify_struct,
SLAB_PANIC|SLAB_ACCOUNT);
dnotify_mark_cache = KMEM_CACHE(dnotify_mark, SLAB_PANIC|SLAB_ACCOUNT);
dnotify_group = fsnotify_alloc_group(&dnotify_fsnotify_ops,
FSNOTIFY_GROUP_NOFS);
if (IS_ERR(dnotify_group))
panic("unable to allocate fsnotify group for dnotify\n");
dnotify_sysctl_init();
return 0;
}
module_init(dnotify_init)
// SPDX-License-Identifier: GPL-2.0
/*
* Kernel timekeeping code and accessor functions. Based on code from
* timer.c, moved in commit 8524070b7982.
*/
#include <linux/timekeeper_internal.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/nmi.h>
#include <linux/sched.h>
#include <linux/sched/loadavg.h>
#include <linux/sched/clock.h>
#include <linux/syscore_ops.h>
#include <linux/clocksource.h>
#include <linux/jiffies.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/tick.h>
#include <linux/stop_machine.h>
#include <linux/pvclock_gtod.h>
#include <linux/compiler.h>
#include <linux/audit.h>
#include <linux/random.h>
#include "tick-internal.h"
#include "ntp_internal.h"
#include "timekeeping_internal.h"
#define TK_CLEAR_NTP (1 << 0)
#define TK_MIRROR (1 << 1)
#define TK_CLOCK_WAS_SET (1 << 2)
enum timekeeping_adv_mode {
/* Update timekeeper when a tick has passed */
TK_ADV_TICK,
/* Update timekeeper on a direct frequency change */
TK_ADV_FREQ
};
DEFINE_RAW_SPINLOCK(timekeeper_lock);
/*
* The most important data for readout fits into a single 64 byte
* cache line.
*/
static struct {
seqcount_raw_spinlock_t seq;
struct timekeeper timekeeper;
} tk_core ____cacheline_aligned = {
.seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_core.seq, &timekeeper_lock),
};
static struct timekeeper shadow_timekeeper;
/* flag for if timekeeping is suspended */
int __read_mostly timekeeping_suspended;
/**
* struct tk_fast - NMI safe timekeeper
* @seq: Sequence counter for protecting updates. The lowest bit
* is the index for the tk_read_base array
* @base: tk_read_base array. Access is indexed by the lowest bit of
* @seq.
*
* See @update_fast_timekeeper() below.
*/
struct tk_fast {
seqcount_latch_t seq;
struct tk_read_base base[2];
};
/* Suspend-time cycles value for halted fast timekeeper. */
static u64 cycles_at_suspend;
static u64 dummy_clock_read(struct clocksource *cs)
{
if (timekeeping_suspended)
return cycles_at_suspend;
return local_clock();
}
static struct clocksource dummy_clock = {
.read = dummy_clock_read,
};
/*
* Boot time initialization which allows local_clock() to be utilized
* during early boot when clocksources are not available. local_clock()
* returns nanoseconds already so no conversion is required, hence mult=1
* and shift=0. When the first proper clocksource is installed then
* the fast time keepers are updated with the correct values.
*/
#define FAST_TK_INIT \
{ \
.clock = &dummy_clock, \
.mask = CLOCKSOURCE_MASK(64), \
.mult = 1, \
.shift = 0, \
}
static struct tk_fast tk_fast_mono ____cacheline_aligned = {
.seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
.base[0] = FAST_TK_INIT,
.base[1] = FAST_TK_INIT,
};
static struct tk_fast tk_fast_raw ____cacheline_aligned = {
.seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
.base[0] = FAST_TK_INIT,
.base[1] = FAST_TK_INIT,
};
static inline void tk_normalize_xtime(struct timekeeper *tk)
{
while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
tk->xtime_sec++;
}
while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
tk->raw_sec++;
}
}
static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
{
struct timespec64 ts;
ts.tv_sec = tk->xtime_sec;
ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
return ts;
}
static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
{
tk->xtime_sec = ts->tv_sec;
tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
}
static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
{
tk->xtime_sec += ts->tv_sec;
tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
tk_normalize_xtime(tk);
}
static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
{
struct timespec64 tmp;
/*
* Verify consistency of: offset_real = -wall_to_monotonic
* before modifying anything
*/
set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
-tk->wall_to_monotonic.tv_nsec);
WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
tk->wall_to_monotonic = wtm;
set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
tk->offs_real = timespec64_to_ktime(tmp);
tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
}
static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
{
tk->offs_boot = ktime_add(tk->offs_boot, delta);
/*
* Timespec representation for VDSO update to avoid 64bit division
* on every update.
*/
tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
}
/*
* tk_clock_read - atomic clocksource read() helper
*
* This helper is necessary to use in the read paths because, while the
* seqcount ensures we don't return a bad value while structures are updated,
* it doesn't protect from potential crashes. There is the possibility that
* the tkr's clocksource may change between the read reference, and the
* clock reference passed to the read function. This can cause crashes if
* the wrong clocksource is passed to the wrong read function.
* This isn't necessary to use when holding the timekeeper_lock or doing
* a read of the fast-timekeeper tkrs (which is protected by its own locking
* and update logic).
*/
static inline u64 tk_clock_read(const struct tk_read_base *tkr)
{
struct clocksource *clock = READ_ONCE(tkr->clock);
return clock->read(clock);
}
#ifdef CONFIG_DEBUG_TIMEKEEPING
#define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
{
u64 max_cycles = tk->tkr_mono.clock->max_cycles;
const char *name = tk->tkr_mono.clock->name;
if (offset > max_cycles) {
printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
offset, name, max_cycles);
printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
} else {
if (offset > (max_cycles >> 1)) { printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
offset, name, max_cycles >> 1);
printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
}
}
if (tk->underflow_seen) { if (jiffies - tk->last_warning > WARNING_FREQ) { printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
printk_deferred(" Your kernel is probably still fine.\n");
tk->last_warning = jiffies;
}
tk->underflow_seen = 0;
}
if (tk->overflow_seen) { if (jiffies - tk->last_warning > WARNING_FREQ) { printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
printk_deferred(" Your kernel is probably still fine.\n");
tk->last_warning = jiffies;
}
tk->overflow_seen = 0;
}
}
static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
{
struct timekeeper *tk = &tk_core.timekeeper;
u64 now, last, mask, max, delta;
unsigned int seq;
/*
* Since we're called holding a seqcount, the data may shift
* under us while we're doing the calculation. This can cause
* false positives, since we'd note a problem but throw the
* results away. So nest another seqcount here to atomically
* grab the points we are checking with.
*/
do {
seq = read_seqcount_begin(&tk_core.seq);
now = tk_clock_read(tkr);
last = tkr->cycle_last;
mask = tkr->mask;
max = tkr->clock->max_cycles;
} while (read_seqcount_retry(&tk_core.seq, seq));
delta = clocksource_delta(now, last, mask);
/*
* Try to catch underflows by checking if we are seeing small
* mask-relative negative values.
*/
if (unlikely((~delta & mask) < (mask >> 3))) { tk->underflow_seen = 1;
delta = 0;
}
/* Cap delta value to the max_cycles values to avoid mult overflows */
if (unlikely(delta > max)) { tk->overflow_seen = 1;
delta = tkr->clock->max_cycles;
}
return delta;
}
#else
static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
{
}
static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
{
u64 cycle_now, delta;
/* read clocksource */
cycle_now = tk_clock_read(tkr);
/* calculate the delta since the last update_wall_time */
delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
return delta;
}
#endif
/**
* tk_setup_internals - Set up internals to use clocksource clock.
*
* @tk: The target timekeeper to setup.
* @clock: Pointer to clocksource.
*
* Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
* pair and interval request.
*
* Unless you're the timekeeping code, you should not be using this!
*/
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
{
u64 interval;
u64 tmp, ntpinterval;
struct clocksource *old_clock;
++tk->cs_was_changed_seq;
old_clock = tk->tkr_mono.clock;
tk->tkr_mono.clock = clock;
tk->tkr_mono.mask = clock->mask;
tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
tk->tkr_raw.clock = clock;
tk->tkr_raw.mask = clock->mask;
tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
/* Do the ns -> cycle conversion first, using original mult */
tmp = NTP_INTERVAL_LENGTH;
tmp <<= clock->shift;
ntpinterval = tmp;
tmp += clock->mult/2;
do_div(tmp, clock->mult);
if (tmp == 0)
tmp = 1;
interval = (u64) tmp;
tk->cycle_interval = interval;
/* Go back from cycles -> shifted ns */
tk->xtime_interval = interval * clock->mult;
tk->xtime_remainder = ntpinterval - tk->xtime_interval;
tk->raw_interval = interval * clock->mult;
/* if changing clocks, convert xtime_nsec shift units */
if (old_clock) {
int shift_change = clock->shift - old_clock->shift;
if (shift_change < 0) {
tk->tkr_mono.xtime_nsec >>= -shift_change;
tk->tkr_raw.xtime_nsec >>= -shift_change;
} else {
tk->tkr_mono.xtime_nsec <<= shift_change;
tk->tkr_raw.xtime_nsec <<= shift_change;
}
}
tk->tkr_mono.shift = clock->shift;
tk->tkr_raw.shift = clock->shift;
tk->ntp_error = 0;
tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
/*
* The timekeeper keeps its own mult values for the currently
* active clocksource. These value will be adjusted via NTP
* to counteract clock drifting.
*/
tk->tkr_mono.mult = clock->mult;
tk->tkr_raw.mult = clock->mult;
tk->ntp_err_mult = 0;
tk->skip_second_overflow = 0;
}
/* Timekeeper helper functions. */
static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
{
u64 nsec;
nsec = delta * tkr->mult + tkr->xtime_nsec; nsec >>= tkr->shift;
return nsec;
}
static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
{
u64 delta;
delta = timekeeping_get_delta(tkr); return timekeeping_delta_to_ns(tkr, delta);
}
static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
{
u64 delta;
/* calculate the delta since the last update_wall_time */
delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
return timekeeping_delta_to_ns(tkr, delta);
}
/**
* update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
* @tkr: Timekeeping readout base from which we take the update
* @tkf: Pointer to NMI safe timekeeper
*
* We want to use this from any context including NMI and tracing /
* instrumenting the timekeeping code itself.
*
* Employ the latch technique; see @raw_write_seqcount_latch.
*
* So if a NMI hits the update of base[0] then it will use base[1]
* which is still consistent. In the worst case this can result is a
* slightly wrong timestamp (a few nanoseconds). See
* @ktime_get_mono_fast_ns.
*/
static void update_fast_timekeeper(const struct tk_read_base *tkr,
struct tk_fast *tkf)
{
struct tk_read_base *base = tkf->base;
/* Force readers off to base[1] */
raw_write_seqcount_latch(&tkf->seq);
/* Update base[0] */
memcpy(base, tkr, sizeof(*base));
/* Force readers back to base[0] */
raw_write_seqcount_latch(&tkf->seq);
/* Update base[1] */
memcpy(base + 1, base, sizeof(*base));
}
static __always_inline u64 fast_tk_get_delta_ns(struct tk_read_base *tkr)
{
u64 delta, cycles = tk_clock_read(tkr);
delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
return timekeeping_delta_to_ns(tkr, delta);
}
static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
{
struct tk_read_base *tkr;
unsigned int seq;
u64 now;
do {
seq = raw_read_seqcount_latch(&tkf->seq);
tkr = tkf->base + (seq & 0x01);
now = ktime_to_ns(tkr->base);
now += fast_tk_get_delta_ns(tkr); } while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
return now;
}
/**
* ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
*
* This timestamp is not guaranteed to be monotonic across an update.
* The timestamp is calculated by:
*
* now = base_mono + clock_delta * slope
*
* So if the update lowers the slope, readers who are forced to the
* not yet updated second array are still using the old steeper slope.
*
* tmono
* ^
* | o n
* | o n
* | u
* | o
* |o
* |12345678---> reader order
*
* o = old slope
* u = update
* n = new slope
*
* So reader 6 will observe time going backwards versus reader 5.
*
* While other CPUs are likely to be able to observe that, the only way
* for a CPU local observation is when an NMI hits in the middle of
* the update. Timestamps taken from that NMI context might be ahead
* of the following timestamps. Callers need to be aware of that and
* deal with it.
*/
u64 notrace ktime_get_mono_fast_ns(void)
{
return __ktime_get_fast_ns(&tk_fast_mono);
}
EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
/**
* ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
*
* Contrary to ktime_get_mono_fast_ns() this is always correct because the
* conversion factor is not affected by NTP/PTP correction.
*/
u64 notrace ktime_get_raw_fast_ns(void)
{
return __ktime_get_fast_ns(&tk_fast_raw);
}
EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
/**
* ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
*
* To keep it NMI safe since we're accessing from tracing, we're not using a
* separate timekeeper with updates to monotonic clock and boot offset
* protected with seqcounts. This has the following minor side effects:
*
* (1) Its possible that a timestamp be taken after the boot offset is updated
* but before the timekeeper is updated. If this happens, the new boot offset
* is added to the old timekeeping making the clock appear to update slightly
* earlier:
* CPU 0 CPU 1
* timekeeping_inject_sleeptime64()
* __timekeeping_inject_sleeptime(tk, delta);
* timestamp();
* timekeeping_update(tk, TK_CLEAR_NTP...);
*
* (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
* partially updated. Since the tk->offs_boot update is a rare event, this
* should be a rare occurrence which postprocessing should be able to handle.
*
* The caveats vs. timestamp ordering as documented for ktime_get_mono_fast_ns()
* apply as well.
*/
u64 notrace ktime_get_boot_fast_ns(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));
}
EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
/**
* ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
*
* The same limitations as described for ktime_get_boot_fast_ns() apply. The
* mono time and the TAI offset are not read atomically which may yield wrong
* readouts. However, an update of the TAI offset is an rare event e.g., caused
* by settime or adjtimex with an offset. The user of this function has to deal
* with the possibility of wrong timestamps in post processing.
*/
u64 notrace ktime_get_tai_fast_ns(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));
}
EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns);
static __always_inline u64 __ktime_get_real_fast(struct tk_fast *tkf, u64 *mono)
{
struct tk_read_base *tkr;
u64 basem, baser, delta;
unsigned int seq;
do {
seq = raw_read_seqcount_latch(&tkf->seq);
tkr = tkf->base + (seq & 0x01);
basem = ktime_to_ns(tkr->base);
baser = ktime_to_ns(tkr->base_real);
delta = fast_tk_get_delta_ns(tkr);
} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
if (mono)
*mono = basem + delta;
return baser + delta;
}
/**
* ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
*
* See ktime_get_mono_fast_ns() for documentation of the time stamp ordering.
*/
u64 ktime_get_real_fast_ns(void)
{
return __ktime_get_real_fast(&tk_fast_mono, NULL);
}
EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
/**
* ktime_get_fast_timestamps: - NMI safe timestamps
* @snapshot: Pointer to timestamp storage
*
* Stores clock monotonic, boottime and realtime timestamps.
*
* Boot time is a racy access on 32bit systems if the sleep time injection
* happens late during resume and not in timekeeping_resume(). That could
* be avoided by expanding struct tk_read_base with boot offset for 32bit
* and adding more overhead to the update. As this is a hard to observe
* once per resume event which can be filtered with reasonable effort using
* the accurate mono/real timestamps, it's probably not worth the trouble.
*
* Aside of that it might be possible on 32 and 64 bit to observe the
* following when the sleep time injection happens late:
*
* CPU 0 CPU 1
* timekeeping_resume()
* ktime_get_fast_timestamps()
* mono, real = __ktime_get_real_fast()
* inject_sleep_time()
* update boot offset
* boot = mono + bootoffset;
*
* That means that boot time already has the sleep time adjustment, but
* real time does not. On the next readout both are in sync again.
*
* Preventing this for 64bit is not really feasible without destroying the
* careful cache layout of the timekeeper because the sequence count and
* struct tk_read_base would then need two cache lines instead of one.
*
* Access to the time keeper clock source is disabled across the innermost
* steps of suspend/resume. The accessors still work, but the timestamps
* are frozen until time keeping is resumed which happens very early.
*
* For regular suspend/resume there is no observable difference vs. sched
* clock, but it might affect some of the nasty low level debug printks.
*
* OTOH, access to sched clock is not guaranteed across suspend/resume on
* all systems either so it depends on the hardware in use.
*
* If that turns out to be a real problem then this could be mitigated by
* using sched clock in a similar way as during early boot. But it's not as
* trivial as on early boot because it needs some careful protection
* against the clock monotonic timestamp jumping backwards on resume.
*/
void ktime_get_fast_timestamps(struct ktime_timestamps *snapshot)
{
struct timekeeper *tk = &tk_core.timekeeper;
snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono);
snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot));
}
/**
* halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
* @tk: Timekeeper to snapshot.
*
* It generally is unsafe to access the clocksource after timekeeping has been
* suspended, so take a snapshot of the readout base of @tk and use it as the
* fast timekeeper's readout base while suspended. It will return the same
* number of cycles every time until timekeeping is resumed at which time the
* proper readout base for the fast timekeeper will be restored automatically.
*/
static void halt_fast_timekeeper(const struct timekeeper *tk)
{
static struct tk_read_base tkr_dummy;
const struct tk_read_base *tkr = &tk->tkr_mono;
memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
cycles_at_suspend = tk_clock_read(tkr);
tkr_dummy.clock = &dummy_clock;
tkr_dummy.base_real = tkr->base + tk->offs_real;
update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
tkr = &tk->tkr_raw;
memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
tkr_dummy.clock = &dummy_clock;
update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
}
static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
{
raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
}
/**
* pvclock_gtod_register_notifier - register a pvclock timedata update listener
* @nb: Pointer to the notifier block to register
*/
int pvclock_gtod_register_notifier(struct notifier_block *nb)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long flags;
int ret;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
update_pvclock_gtod(tk, true);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
/**
* pvclock_gtod_unregister_notifier - unregister a pvclock
* timedata update listener
* @nb: Pointer to the notifier block to unregister
*/
int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
{
unsigned long flags;
int ret;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
/*
* tk_update_leap_state - helper to update the next_leap_ktime
*/
static inline void tk_update_leap_state(struct timekeeper *tk)
{
tk->next_leap_ktime = ntp_get_next_leap();
if (tk->next_leap_ktime != KTIME_MAX)
/* Convert to monotonic time */
tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
}
/*
* Update the ktime_t based scalar nsec members of the timekeeper
*/
static inline void tk_update_ktime_data(struct timekeeper *tk)
{
u64 seconds;
u32 nsec;
/*
* The xtime based monotonic readout is:
* nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
* The ktime based monotonic readout is:
* nsec = base_mono + now();
* ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
*/
seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
nsec = (u32) tk->wall_to_monotonic.tv_nsec;
tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
/*
* The sum of the nanoseconds portions of xtime and
* wall_to_monotonic can be greater/equal one second. Take
* this into account before updating tk->ktime_sec.
*/
nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
if (nsec >= NSEC_PER_SEC)
seconds++; tk->ktime_sec = seconds;
/* Update the monotonic raw base */
tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
}
/* must hold timekeeper_lock */
static void timekeeping_update(struct timekeeper *tk, unsigned int action)
{
if (action & TK_CLEAR_NTP) { tk->ntp_error = 0;
ntp_clear();
}
tk_update_leap_state(tk); tk_update_ktime_data(tk);
update_vsyscall(tk);
update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
if (action & TK_CLOCK_WAS_SET)
tk->clock_was_set_seq++;
/*
* The mirroring of the data to the shadow-timekeeper needs
* to happen last here to ensure we don't over-write the
* timekeeper structure on the next update with stale data
*/
if (action & TK_MIRROR) memcpy(&shadow_timekeeper, &tk_core.timekeeper,
sizeof(tk_core.timekeeper));
}
/**
* timekeeping_forward_now - update clock to the current time
* @tk: Pointer to the timekeeper to update
*
* Forward the current clock to update its state since the last call to
* update_wall_time(). This is useful before significant clock changes,
* as it avoids having to deal with this time offset explicitly.
*/
static void timekeeping_forward_now(struct timekeeper *tk)
{
u64 cycle_now, delta;
cycle_now = tk_clock_read(&tk->tkr_mono);
delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
tk->tkr_mono.cycle_last = cycle_now;
tk->tkr_raw.cycle_last = cycle_now;
tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
tk_normalize_xtime(tk);
}
/**
* ktime_get_real_ts64 - Returns the time of day in a timespec64.
* @ts: pointer to the timespec to be set
*
* Returns the time of day in a timespec64 (WARN if suspended).
*/
void ktime_get_real_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
u64 nsecs;
WARN_ON(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
ts->tv_sec = tk->xtime_sec;
nsecs = timekeeping_get_ns(&tk->tkr_mono);
} while (read_seqcount_retry(&tk_core.seq, seq));
ts->tv_nsec = 0;
timespec64_add_ns(ts, nsecs);
}
EXPORT_SYMBOL(ktime_get_real_ts64);
ktime_t ktime_get(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base;
u64 nsecs;
WARN_ON(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
base = tk->tkr_mono.base;
nsecs = timekeeping_get_ns(&tk->tkr_mono); } while (read_seqcount_retry(&tk_core.seq, seq)); return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get);
u32 ktime_get_resolution_ns(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
u32 nsecs;
WARN_ON(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
} while (read_seqcount_retry(&tk_core.seq, seq));
return nsecs;
}
EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
static ktime_t *offsets[TK_OFFS_MAX] = {
[TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
[TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
[TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
};
ktime_t ktime_get_with_offset(enum tk_offsets offs)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base, *offset = offsets[offs];
u64 nsecs;
WARN_ON(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq); base = ktime_add(tk->tkr_mono.base, *offset); nsecs = timekeeping_get_ns(&tk->tkr_mono); } while (read_seqcount_retry(&tk_core.seq, seq));
return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_with_offset);
ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base, *offset = offsets[offs];
u64 nsecs;
WARN_ON(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
base = ktime_add(tk->tkr_mono.base, *offset);
nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
} while (read_seqcount_retry(&tk_core.seq, seq));
return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
/**
* ktime_mono_to_any() - convert monotonic time to any other time
* @tmono: time to convert.
* @offs: which offset to use
*/
ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
{
ktime_t *offset = offsets[offs];
unsigned int seq;
ktime_t tconv;
do {
seq = read_seqcount_begin(&tk_core.seq); tconv = ktime_add(tmono, *offset);
} while (read_seqcount_retry(&tk_core.seq, seq));
return tconv;
}
EXPORT_SYMBOL_GPL(ktime_mono_to_any);
/**
* ktime_get_raw - Returns the raw monotonic time in ktime_t format
*/
ktime_t ktime_get_raw(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base;
u64 nsecs;
do {
seq = read_seqcount_begin(&tk_core.seq);
base = tk->tkr_raw.base;
nsecs = timekeeping_get_ns(&tk->tkr_raw);
} while (read_seqcount_retry(&tk_core.seq, seq));
return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_raw);
/**
* ktime_get_ts64 - get the monotonic clock in timespec64 format
* @ts: pointer to timespec variable
*
* The function calculates the monotonic clock from the realtime
* clock and the wall_to_monotonic offset and stores the result
* in normalized timespec64 format in the variable pointed to by @ts.
*/
void ktime_get_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct timespec64 tomono;
unsigned int seq;
u64 nsec;
WARN_ON(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
ts->tv_sec = tk->xtime_sec;
nsec = timekeeping_get_ns(&tk->tkr_mono);
tomono = tk->wall_to_monotonic;
} while (read_seqcount_retry(&tk_core.seq, seq));
ts->tv_sec += tomono.tv_sec;
ts->tv_nsec = 0;
timespec64_add_ns(ts, nsec + tomono.tv_nsec);
}
EXPORT_SYMBOL_GPL(ktime_get_ts64);
/**
* ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
*
* Returns the seconds portion of CLOCK_MONOTONIC with a single non
* serialized read. tk->ktime_sec is of type 'unsigned long' so this
* works on both 32 and 64 bit systems. On 32 bit systems the readout
* covers ~136 years of uptime which should be enough to prevent
* premature wrap arounds.
*/
time64_t ktime_get_seconds(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
WARN_ON(timekeeping_suspended);
return tk->ktime_sec;
}
EXPORT_SYMBOL_GPL(ktime_get_seconds);
/**
* ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
*
* Returns the wall clock seconds since 1970.
*
* For 64bit systems the fast access to tk->xtime_sec is preserved. On
* 32bit systems the access must be protected with the sequence
* counter to provide "atomic" access to the 64bit tk->xtime_sec
* value.
*/
time64_t ktime_get_real_seconds(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
time64_t seconds;
unsigned int seq;
if (IS_ENABLED(CONFIG_64BIT))
return tk->xtime_sec;
do {
seq = read_seqcount_begin(&tk_core.seq);
seconds = tk->xtime_sec;
} while (read_seqcount_retry(&tk_core.seq, seq));
return seconds;
}
EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
/**
* __ktime_get_real_seconds - The same as ktime_get_real_seconds
* but without the sequence counter protect. This internal function
* is called just when timekeeping lock is already held.
*/
noinstr time64_t __ktime_get_real_seconds(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
return tk->xtime_sec;
}
/**
* ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
* @systime_snapshot: pointer to struct receiving the system time snapshot
*/
void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base_raw;
ktime_t base_real;
u64 nsec_raw;
u64 nsec_real;
u64 now;
WARN_ON_ONCE(timekeeping_suspended);
do {
seq = read_seqcount_begin(&tk_core.seq);
now = tk_clock_read(&tk->tkr_mono);
systime_snapshot->cs_id = tk->tkr_mono.clock->id;
systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
base_real = ktime_add(tk->tkr_mono.base,
tk_core.timekeeper.offs_real);
base_raw = tk->tkr_raw.base;
nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
} while (read_seqcount_retry(&tk_core.seq, seq));
systime_snapshot->cycles = now;
systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
}
EXPORT_SYMBOL_GPL(ktime_get_snapshot);
/* Scale base by mult/div checking for overflow */
static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
{
u64 tmp, rem;
tmp = div64_u64_rem(*base, div, &rem);
if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
return -EOVERFLOW;
tmp *= mult;
rem = div64_u64(rem * mult, div);
*base = tmp + rem;
return 0;
}
/**
* adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
* @history: Snapshot representing start of history
* @partial_history_cycles: Cycle offset into history (fractional part)
* @total_history_cycles: Total history length in cycles
* @discontinuity: True indicates clock was set on history period
* @ts: Cross timestamp that should be adjusted using
* partial/total ratio
*
* Helper function used by get_device_system_crosststamp() to correct the
* crosstimestamp corresponding to the start of the current interval to the
* system counter value (timestamp point) provided by the driver. The
* total_history_* quantities are the total history starting at the provided
* reference point and ending at the start of the current interval. The cycle
* count between the driver timestamp point and the start of the current
* interval is partial_history_cycles.
*/
static int adjust_historical_crosststamp(struct system_time_snapshot *history,
u64 partial_history_cycles,
u64 total_history_cycles,
bool discontinuity,
struct system_device_crosststamp *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
u64 corr_raw, corr_real;
bool interp_forward;
int ret;
if (total_history_cycles == 0 || partial_history_cycles == 0)
return 0;
/* Interpolate shortest distance from beginning or end of history */
interp_forward = partial_history_cycles > total_history_cycles / 2;
partial_history_cycles = interp_forward ?
total_history_cycles - partial_history_cycles :
partial_history_cycles;
/*
* Scale the monotonic raw time delta by:
* partial_history_cycles / total_history_cycles
*/
corr_raw = (u64)ktime_to_ns(
ktime_sub(ts->sys_monoraw, history->raw));
ret = scale64_check_overflow(partial_history_cycles,
total_history_cycles, &corr_raw);
if (ret)
return ret;
/*
* If there is a discontinuity in the history, scale monotonic raw
* correction by:
* mult(real)/mult(raw) yielding the realtime correction
* Otherwise, calculate the realtime correction similar to monotonic
* raw calculation
*/
if (discontinuity) {
corr_real = mul_u64_u32_div
(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
} else {
corr_real = (u64)ktime_to_ns(
ktime_sub(ts->sys_realtime, history->real));
ret = scale64_check_overflow(partial_history_cycles,
total_history_cycles, &corr_real);
if (ret)
return ret;
}
/* Fixup monotonic raw and real time time values */
if (interp_forward) {
ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
ts->sys_realtime = ktime_add_ns(history->real, corr_real);
} else {
ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
}
return 0;
}
/*
* timestamp_in_interval - true if ts is chronologically in [start, end]
*
* True if ts occurs chronologically at or after start, and before or at end.
*/
static bool timestamp_in_interval(u64 start, u64 end, u64 ts)
{
if (ts >= start && ts <= end)
return true;
if (start > end && (ts >= start || ts <= end))
return true;
return false;
}
/**
* get_device_system_crosststamp - Synchronously capture system/device timestamp
* @get_time_fn: Callback to get simultaneous device time and
* system counter from the device driver
* @ctx: Context passed to get_time_fn()
* @history_begin: Historical reference point used to interpolate system
* time when counter provided by the driver is before the current interval
* @xtstamp: Receives simultaneously captured system and device time
*
* Reads a timestamp from a device and correlates it to system time
*/
int get_device_system_crosststamp(int (*get_time_fn)
(ktime_t *device_time,
struct system_counterval_t *sys_counterval,
void *ctx),
void *ctx,
struct system_time_snapshot *history_begin,
struct system_device_crosststamp *xtstamp)
{
struct system_counterval_t system_counterval;
struct timekeeper *tk = &tk_core.timekeeper;
u64 cycles, now, interval_start;
unsigned int clock_was_set_seq = 0;
ktime_t base_real, base_raw;
u64 nsec_real, nsec_raw;
u8 cs_was_changed_seq;
unsigned int seq;
bool do_interp;
int ret;
do {
seq = read_seqcount_begin(&tk_core.seq);
/*
* Try to synchronously capture device time and a system
* counter value calling back into the device driver
*/
ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
if (ret)
return ret;
/*
* Verify that the clocksource ID associated with the captured
* system counter value is the same as for the currently
* installed timekeeper clocksource
*/
if (system_counterval.cs_id == CSID_GENERIC ||
tk->tkr_mono.clock->id != system_counterval.cs_id)
return -ENODEV;
cycles = system_counterval.cycles;
/*
* Check whether the system counter value provided by the
* device driver is on the current timekeeping interval.
*/
now = tk_clock_read(&tk->tkr_mono);
interval_start = tk->tkr_mono.cycle_last;
if (!timestamp_in_interval(interval_start, now, cycles)) {
clock_was_set_seq = tk->clock_was_set_seq;
cs_was_changed_seq = tk->cs_was_changed_seq;
cycles = interval_start;
do_interp = true;
} else {
do_interp = false;
}
base_real = ktime_add(tk->tkr_mono.base,
tk_core.timekeeper.offs_real);
base_raw = tk->tkr_raw.base;
nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);
nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles);
} while (read_seqcount_retry(&tk_core.seq, seq));
xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
/*
* Interpolate if necessary, adjusting back from the start of the
* current interval
*/
if (do_interp) {
u64 partial_history_cycles, total_history_cycles;
bool discontinuity;
/*
* Check that the counter value is not before the provided
* history reference and that the history doesn't cross a
* clocksource change
*/
if (!history_begin ||
!timestamp_in_interval(history_begin->cycles,
cycles, system_counterval.cycles) ||
history_begin->cs_was_changed_seq != cs_was_changed_seq)
return -EINVAL;
partial_history_cycles = cycles - system_counterval.cycles;
total_history_cycles = cycles - history_begin->cycles;
discontinuity =
history_begin->clock_was_set_seq != clock_was_set_seq;
ret = adjust_historical_crosststamp(history_begin,
partial_history_cycles,
total_history_cycles,
discontinuity, xtstamp);
if (ret)
return ret;
}
return 0;
}
EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
/**
* do_settimeofday64 - Sets the time of day.
* @ts: pointer to the timespec64 variable containing the new time
*
* Sets the time of day to the new time and update NTP and notify hrtimers
*/
int do_settimeofday64(const struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct timespec64 ts_delta, xt;
unsigned long flags;
int ret = 0;
if (!timespec64_valid_settod(ts))
return -EINVAL;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
timekeeping_forward_now(tk);
xt = tk_xtime(tk);
ts_delta = timespec64_sub(*ts, xt);
if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
ret = -EINVAL;
goto out;
}
tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
tk_set_xtime(tk, ts);
out:
timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL);
if (!ret) {
audit_tk_injoffset(ts_delta);
add_device_randomness(ts, sizeof(*ts));
}
return ret;
}
EXPORT_SYMBOL(do_settimeofday64);
/**
* timekeeping_inject_offset - Adds or subtracts from the current time.
* @ts: Pointer to the timespec variable containing the offset
*
* Adds or subtracts an offset value from the current time.
*/
static int timekeeping_inject_offset(const struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long flags;
struct timespec64 tmp;
int ret = 0;
if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
return -EINVAL;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
timekeeping_forward_now(tk);
/* Make sure the proposed value is valid */
tmp = timespec64_add(tk_xtime(tk), *ts);
if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
!timespec64_valid_settod(&tmp)) {
ret = -EINVAL;
goto error;
}
tk_xtime_add(tk, ts);
tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
error: /* even if we error out, we forwarded the time, so call update */
timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL);
return ret;
}
/*
* Indicates if there is an offset between the system clock and the hardware
* clock/persistent clock/rtc.
*/
int persistent_clock_is_local;
/*
* Adjust the time obtained from the CMOS to be UTC time instead of
* local time.
*
* This is ugly, but preferable to the alternatives. Otherwise we
* would either need to write a program to do it in /etc/rc (and risk
* confusion if the program gets run more than once; it would also be
* hard to make the program warp the clock precisely n hours) or
* compile in the timezone information into the kernel. Bad, bad....
*
* - TYT, 1992-01-01
*
* The best thing to do is to keep the CMOS clock in universal time (UTC)
* as real UNIX machines always do it. This avoids all headaches about
* daylight saving times and warping kernel clocks.
*/
void timekeeping_warp_clock(void)
{
if (sys_tz.tz_minuteswest != 0) {
struct timespec64 adjust;
persistent_clock_is_local = 1;
adjust.tv_sec = sys_tz.tz_minuteswest * 60;
adjust.tv_nsec = 0;
timekeeping_inject_offset(&adjust);
}
}
/*
* __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
*/
static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
{
tk->tai_offset = tai_offset;
tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
}
/*
* change_clocksource - Swaps clocksources if a new one is available
*
* Accumulates current time interval and initializes new clocksource
*/
static int change_clocksource(void *data)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct clocksource *new, *old = NULL;
unsigned long flags;
bool change = false;
new = (struct clocksource *) data;
/*
* If the cs is in module, get a module reference. Succeeds
* for built-in code (owner == NULL) as well.
*/
if (try_module_get(new->owner)) {
if (!new->enable || new->enable(new) == 0)
change = true;
else
module_put(new->owner);
}
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
timekeeping_forward_now(tk);
if (change) {
old = tk->tkr_mono.clock;
tk_setup_internals(tk, new);
}
timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
if (old) {
if (old->disable)
old->disable(old);
module_put(old->owner);
}
return 0;
}
/**
* timekeeping_notify - Install a new clock source
* @clock: pointer to the clock source
*
* This function is called from clocksource.c after a new, better clock
* source has been registered. The caller holds the clocksource_mutex.
*/
int timekeeping_notify(struct clocksource *clock)
{
struct timekeeper *tk = &tk_core.timekeeper;
if (tk->tkr_mono.clock == clock)
return 0;
stop_machine(change_clocksource, clock, NULL);
tick_clock_notify();
return tk->tkr_mono.clock == clock ? 0 : -1;
}
/**
* ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
* @ts: pointer to the timespec64 to be set
*
* Returns the raw monotonic time (completely un-modified by ntp)
*/
void ktime_get_raw_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
u64 nsecs;
do {
seq = read_seqcount_begin(&tk_core.seq);
ts->tv_sec = tk->raw_sec;
nsecs = timekeeping_get_ns(&tk->tkr_raw);
} while (read_seqcount_retry(&tk_core.seq, seq));
ts->tv_nsec = 0;
timespec64_add_ns(ts, nsecs);
}
EXPORT_SYMBOL(ktime_get_raw_ts64);
/**
* timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
*/
int timekeeping_valid_for_hres(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
int ret;
do {
seq = read_seqcount_begin(&tk_core.seq);
ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
} while (read_seqcount_retry(&tk_core.seq, seq));
return ret;
}
/**
* timekeeping_max_deferment - Returns max time the clocksource can be deferred
*/
u64 timekeeping_max_deferment(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
u64 ret;
do {
seq = read_seqcount_begin(&tk_core.seq);
ret = tk->tkr_mono.clock->max_idle_ns;
} while (read_seqcount_retry(&tk_core.seq, seq));
return ret;
}
/**
* read_persistent_clock64 - Return time from the persistent clock.
* @ts: Pointer to the storage for the readout value
*
* Weak dummy function for arches that do not yet support it.
* Reads the time from the battery backed persistent clock.
* Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
*
* XXX - Do be sure to remove it once all arches implement it.
*/
void __weak read_persistent_clock64(struct timespec64 *ts)
{
ts->tv_sec = 0;
ts->tv_nsec = 0;
}
/**
* read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
* from the boot.
* @wall_time: current time as returned by persistent clock
* @boot_offset: offset that is defined as wall_time - boot_time
*
* Weak dummy function for arches that do not yet support it.
*
* The default function calculates offset based on the current value of
* local_clock(). This way architectures that support sched_clock() but don't
* support dedicated boot time clock will provide the best estimate of the
* boot time.
*/
void __weak __init
read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
struct timespec64 *boot_offset)
{
read_persistent_clock64(wall_time);
*boot_offset = ns_to_timespec64(local_clock());
}
/*
* Flag reflecting whether timekeeping_resume() has injected sleeptime.
*
* The flag starts of false and is only set when a suspend reaches
* timekeeping_suspend(), timekeeping_resume() sets it to false when the
* timekeeper clocksource is not stopping across suspend and has been
* used to update sleep time. If the timekeeper clocksource has stopped
* then the flag stays true and is used by the RTC resume code to decide
* whether sleeptime must be injected and if so the flag gets false then.
*
* If a suspend fails before reaching timekeeping_resume() then the flag
* stays false and prevents erroneous sleeptime injection.
*/
static bool suspend_timing_needed;
/* Flag for if there is a persistent clock on this platform */
static bool persistent_clock_exists;
/*
* timekeeping_init - Initializes the clocksource and common timekeeping values
*/
void __init timekeeping_init(void)
{
struct timespec64 wall_time, boot_offset, wall_to_mono;
struct timekeeper *tk = &tk_core.timekeeper;
struct clocksource *clock;
unsigned long flags;
read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
if (timespec64_valid_settod(&wall_time) &&
timespec64_to_ns(&wall_time) > 0) {
persistent_clock_exists = true;
} else if (timespec64_to_ns(&wall_time) != 0) {
pr_warn("Persistent clock returned invalid value");
wall_time = (struct timespec64){0};
}
if (timespec64_compare(&wall_time, &boot_offset) < 0)
boot_offset = (struct timespec64){0};
/*
* We want set wall_to_mono, so the following is true:
* wall time + wall_to_mono = boot time
*/
wall_to_mono = timespec64_sub(boot_offset, wall_time);
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
ntp_init();
clock = clocksource_default_clock();
if (clock->enable)
clock->enable(clock);
tk_setup_internals(tk, clock);
tk_set_xtime(tk, &wall_time);
tk->raw_sec = 0;
tk_set_wall_to_mono(tk, wall_to_mono);
timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
}
/* time in seconds when suspend began for persistent clock */
static struct timespec64 timekeeping_suspend_time;
/**
* __timekeeping_inject_sleeptime - Internal function to add sleep interval
* @tk: Pointer to the timekeeper to be updated
* @delta: Pointer to the delta value in timespec64 format
*
* Takes a timespec offset measuring a suspend interval and properly
* adds the sleep offset to the timekeeping variables.
*/
static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
const struct timespec64 *delta)
{
if (!timespec64_valid_strict(delta)) {
printk_deferred(KERN_WARNING
"__timekeeping_inject_sleeptime: Invalid "
"sleep delta value!\n");
return;
}
tk_xtime_add(tk, delta);
tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
tk_debug_account_sleep_time(delta);
}
#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
/*
* We have three kinds of time sources to use for sleep time
* injection, the preference order is:
* 1) non-stop clocksource
* 2) persistent clock (ie: RTC accessible when irqs are off)
* 3) RTC
*
* 1) and 2) are used by timekeeping, 3) by RTC subsystem.
* If system has neither 1) nor 2), 3) will be used finally.
*
*
* If timekeeping has injected sleeptime via either 1) or 2),
* 3) becomes needless, so in this case we don't need to call
* rtc_resume(), and this is what timekeeping_rtc_skipresume()
* means.
*/
bool timekeeping_rtc_skipresume(void)
{
return !suspend_timing_needed;
}
/*
* 1) can be determined whether to use or not only when doing
* timekeeping_resume() which is invoked after rtc_suspend(),
* so we can't skip rtc_suspend() surely if system has 1).
*
* But if system has 2), 2) will definitely be used, so in this
* case we don't need to call rtc_suspend(), and this is what
* timekeeping_rtc_skipsuspend() means.
*/
bool timekeeping_rtc_skipsuspend(void)
{
return persistent_clock_exists;
}
/**
* timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
* @delta: pointer to a timespec64 delta value
*
* This hook is for architectures that cannot support read_persistent_clock64
* because their RTC/persistent clock is only accessible when irqs are enabled.
* and also don't have an effective nonstop clocksource.
*
* This function should only be called by rtc_resume(), and allows
* a suspend offset to be injected into the timekeeping values.
*/
void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long flags;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
suspend_timing_needed = false;
timekeeping_forward_now(tk);
__timekeeping_inject_sleeptime(tk, delta);
timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT);
}
#endif
/**
* timekeeping_resume - Resumes the generic timekeeping subsystem.
*/
void timekeeping_resume(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct clocksource *clock = tk->tkr_mono.clock;
unsigned long flags;
struct timespec64 ts_new, ts_delta;
u64 cycle_now, nsec;
bool inject_sleeptime = false;
read_persistent_clock64(&ts_new);
clockevents_resume();
clocksource_resume();
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
/*
* After system resumes, we need to calculate the suspended time and
* compensate it for the OS time. There are 3 sources that could be
* used: Nonstop clocksource during suspend, persistent clock and rtc
* device.
*
* One specific platform may have 1 or 2 or all of them, and the
* preference will be:
* suspend-nonstop clocksource -> persistent clock -> rtc
* The less preferred source will only be tried if there is no better
* usable source. The rtc part is handled separately in rtc core code.
*/
cycle_now = tk_clock_read(&tk->tkr_mono);
nsec = clocksource_stop_suspend_timing(clock, cycle_now);
if (nsec > 0) {
ts_delta = ns_to_timespec64(nsec);
inject_sleeptime = true;
} else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
inject_sleeptime = true;
}
if (inject_sleeptime) {
suspend_timing_needed = false;
__timekeeping_inject_sleeptime(tk, &ts_delta);
}
/* Re-base the last cycle value */
tk->tkr_mono.cycle_last = cycle_now;
tk->tkr_raw.cycle_last = cycle_now;
tk->ntp_error = 0;
timekeeping_suspended = 0;
timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
touch_softlockup_watchdog();
/* Resume the clockevent device(s) and hrtimers */
tick_resume();
/* Notify timerfd as resume is equivalent to clock_was_set() */
timerfd_resume();
}
int timekeeping_suspend(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long flags;
struct timespec64 delta, delta_delta;
static struct timespec64 old_delta;
struct clocksource *curr_clock;
u64 cycle_now;
read_persistent_clock64(&timekeeping_suspend_time);
/*
* On some systems the persistent_clock can not be detected at
* timekeeping_init by its return value, so if we see a valid
* value returned, update the persistent_clock_exists flag.
*/
if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
persistent_clock_exists = true;
suspend_timing_needed = true;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
timekeeping_forward_now(tk);
timekeeping_suspended = 1;
/*
* Since we've called forward_now, cycle_last stores the value
* just read from the current clocksource. Save this to potentially
* use in suspend timing.
*/
curr_clock = tk->tkr_mono.clock;
cycle_now = tk->tkr_mono.cycle_last;
clocksource_start_suspend_timing(curr_clock, cycle_now);
if (persistent_clock_exists) {
/*
* To avoid drift caused by repeated suspend/resumes,
* which each can add ~1 second drift error,
* try to compensate so the difference in system time
* and persistent_clock time stays close to constant.
*/
delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
delta_delta = timespec64_sub(delta, old_delta);
if (abs(delta_delta.tv_sec) >= 2) {
/*
* if delta_delta is too large, assume time correction
* has occurred and set old_delta to the current delta.
*/
old_delta = delta;
} else {
/* Otherwise try to adjust old_system to compensate */
timekeeping_suspend_time =
timespec64_add(timekeeping_suspend_time, delta_delta);
}
}
timekeeping_update(tk, TK_MIRROR);
halt_fast_timekeeper(tk);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
tick_suspend();
clocksource_suspend();
clockevents_suspend();
return 0;
}
/* sysfs resume/suspend bits for timekeeping */
static struct syscore_ops timekeeping_syscore_ops = {
.resume = timekeeping_resume,
.suspend = timekeeping_suspend,
};
static int __init timekeeping_init_ops(void)
{
register_syscore_ops(&timekeeping_syscore_ops);
return 0;
}
device_initcall(timekeeping_init_ops);
/*
* Apply a multiplier adjustment to the timekeeper
*/
static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
s64 offset,
s32 mult_adj)
{
s64 interval = tk->cycle_interval;
if (mult_adj == 0) {
return;
} else if (mult_adj == -1) {
interval = -interval;
offset = -offset;
} else if (mult_adj != 1) { interval *= mult_adj;
offset *= mult_adj;
}
/*
* So the following can be confusing.
*
* To keep things simple, lets assume mult_adj == 1 for now.
*
* When mult_adj != 1, remember that the interval and offset values
* have been appropriately scaled so the math is the same.
*
* The basic idea here is that we're increasing the multiplier
* by one, this causes the xtime_interval to be incremented by
* one cycle_interval. This is because:
* xtime_interval = cycle_interval * mult
* So if mult is being incremented by one:
* xtime_interval = cycle_interval * (mult + 1)
* Its the same as:
* xtime_interval = (cycle_interval * mult) + cycle_interval
* Which can be shortened to:
* xtime_interval += cycle_interval
*
* So offset stores the non-accumulated cycles. Thus the current
* time (in shifted nanoseconds) is:
* now = (offset * adj) + xtime_nsec
* Now, even though we're adjusting the clock frequency, we have
* to keep time consistent. In other words, we can't jump back
* in time, and we also want to avoid jumping forward in time.
*
* So given the same offset value, we need the time to be the same
* both before and after the freq adjustment.
* now = (offset * adj_1) + xtime_nsec_1
* now = (offset * adj_2) + xtime_nsec_2
* So:
* (offset * adj_1) + xtime_nsec_1 =
* (offset * adj_2) + xtime_nsec_2
* And we know:
* adj_2 = adj_1 + 1
* So:
* (offset * adj_1) + xtime_nsec_1 =
* (offset * (adj_1+1)) + xtime_nsec_2
* (offset * adj_1) + xtime_nsec_1 =
* (offset * adj_1) + offset + xtime_nsec_2
* Canceling the sides:
* xtime_nsec_1 = offset + xtime_nsec_2
* Which gives us:
* xtime_nsec_2 = xtime_nsec_1 - offset
* Which simplifies to:
* xtime_nsec -= offset
*/
if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
/* NTP adjustment caused clocksource mult overflow */
WARN_ON_ONCE(1);
return;
}
tk->tkr_mono.mult += mult_adj;
tk->xtime_interval += interval;
tk->tkr_mono.xtime_nsec -= offset;
}
/*
* Adjust the timekeeper's multiplier to the correct frequency
* and also to reduce the accumulated error value.
*/
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
u32 mult;
/*
* Determine the multiplier from the current NTP tick length.
* Avoid expensive division when the tick length doesn't change.
*/
if (likely(tk->ntp_tick == ntp_tick_length())) { mult = tk->tkr_mono.mult - tk->ntp_err_mult;
} else {
tk->ntp_tick = ntp_tick_length(); mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
tk->xtime_remainder, tk->cycle_interval);
}
/*
* If the clock is behind the NTP time, increase the multiplier by 1
* to catch up with it. If it's ahead and there was a remainder in the
* tick division, the clock will slow down. Otherwise it will stay
* ahead until the tick length changes to a non-divisible value.
*/
tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
mult += tk->ntp_err_mult;
timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult); if (unlikely(tk->tkr_mono.clock->maxadj &&
(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
> tk->tkr_mono.clock->maxadj))) {
printk_once(KERN_WARNING
"Adjusting %s more than 11%% (%ld vs %ld)\n",
tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
}
/*
* It may be possible that when we entered this function, xtime_nsec
* was very small. Further, if we're slightly speeding the clocksource
* in the code above, its possible the required corrective factor to
* xtime_nsec could cause it to underflow.
*
* Now, since we have already accumulated the second and the NTP
* subsystem has been notified via second_overflow(), we need to skip
* the next update.
*/
if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
tk->tkr_mono.shift;
tk->xtime_sec--;
tk->skip_second_overflow = 1;
}
}
/*
* accumulate_nsecs_to_secs - Accumulates nsecs into secs
*
* Helper function that accumulates the nsecs greater than a second
* from the xtime_nsec field to the xtime_secs field.
* It also calls into the NTP code to handle leapsecond processing.
*/
static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
{
u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
unsigned int clock_set = 0;
while (tk->tkr_mono.xtime_nsec >= nsecps) {
int leap;
tk->tkr_mono.xtime_nsec -= nsecps;
tk->xtime_sec++;
/*
* Skip NTP update if this second was accumulated before,
* i.e. xtime_nsec underflowed in timekeeping_adjust()
*/
if (unlikely(tk->skip_second_overflow)) {
tk->skip_second_overflow = 0;
continue;
}
/* Figure out if its a leap sec and apply if needed */
leap = second_overflow(tk->xtime_sec);
if (unlikely(leap)) {
struct timespec64 ts;
tk->xtime_sec += leap;
ts.tv_sec = leap;
ts.tv_nsec = 0;
tk_set_wall_to_mono(tk,
timespec64_sub(tk->wall_to_monotonic, ts));
__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
clock_set = TK_CLOCK_WAS_SET;
}
}
return clock_set;
}
/*
* logarithmic_accumulation - shifted accumulation of cycles
*
* This functions accumulates a shifted interval of cycles into
* a shifted interval nanoseconds. Allows for O(log) accumulation
* loop.
*
* Returns the unconsumed cycles.
*/
static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
u32 shift, unsigned int *clock_set)
{
u64 interval = tk->cycle_interval << shift;
u64 snsec_per_sec;
/* If the offset is smaller than a shifted interval, do nothing */
if (offset < interval)
return offset;
/* Accumulate one shifted interval */
offset -= interval;
tk->tkr_mono.cycle_last += interval;
tk->tkr_raw.cycle_last += interval;
tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; *clock_set |= accumulate_nsecs_to_secs(tk);
/* Accumulate raw time */
tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { tk->tkr_raw.xtime_nsec -= snsec_per_sec;
tk->raw_sec++;
}
/* Accumulate error between NTP and clock interval */
tk->ntp_error += tk->ntp_tick << shift; tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
(tk->ntp_error_shift + shift);
return offset;
}
/*
* timekeeping_advance - Updates the timekeeper to the current time and
* current NTP tick length
*/
static bool timekeeping_advance(enum timekeeping_adv_mode mode)
{
struct timekeeper *real_tk = &tk_core.timekeeper;
struct timekeeper *tk = &shadow_timekeeper;
u64 offset;
int shift = 0, maxshift;
unsigned int clock_set = 0;
unsigned long flags;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
/* Make sure we're fully resumed: */
if (unlikely(timekeeping_suspended))
goto out;
offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
/* Check if there's really nothing to do */
if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
goto out;
/* Do some additional sanity checking */
timekeeping_check_update(tk, offset);
/*
* With NO_HZ we may have to accumulate many cycle_intervals
* (think "ticks") worth of time at once. To do this efficiently,
* we calculate the largest doubling multiple of cycle_intervals
* that is smaller than the offset. We then accumulate that
* chunk in one go, and then try to consume the next smaller
* doubled multiple.
*/
shift = ilog2(offset) - ilog2(tk->cycle_interval);
shift = max(0, shift);
/* Bound shift to one less than what overflows tick_length */
maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
shift = min(shift, maxshift);
while (offset >= tk->cycle_interval) { offset = logarithmic_accumulation(tk, offset, shift,
&clock_set);
if (offset < tk->cycle_interval<<shift) shift--;
}
/* Adjust the multiplier to correct NTP error */
timekeeping_adjust(tk, offset);
/*
* Finally, make sure that after the rounding
* xtime_nsec isn't larger than NSEC_PER_SEC
*/
clock_set |= accumulate_nsecs_to_secs(tk); write_seqcount_begin(&tk_core.seq);
/*
* Update the real timekeeper.
*
* We could avoid this memcpy by switching pointers, but that
* requires changes to all other timekeeper usage sites as
* well, i.e. move the timekeeper pointer getter into the
* spinlocked/seqcount protected sections. And we trade this
* memcpy under the tk_core.seq against one before we start
* updating.
*/
timekeeping_update(tk, clock_set);
memcpy(real_tk, tk, sizeof(*tk));
/* The memcpy must come last. Do not put anything here! */
write_seqcount_end(&tk_core.seq);
out:
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
return !!clock_set;
}
/**
* update_wall_time - Uses the current clocksource to increment the wall time
*
*/
void update_wall_time(void)
{
if (timekeeping_advance(TK_ADV_TICK)) clock_was_set_delayed();
}
/**
* getboottime64 - Return the real time of system boot.
* @ts: pointer to the timespec64 to be set
*
* Returns the wall-time of boot in a timespec64.
*
* This is based on the wall_to_monotonic offset and the total suspend
* time. Calls to settimeofday will affect the value returned (which
* basically means that however wrong your real time clock is at boot time,
* you get the right time here).
*/
void getboottime64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
*ts = ktime_to_timespec64(t);
}
EXPORT_SYMBOL_GPL(getboottime64);
void ktime_get_coarse_real_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
do {
seq = read_seqcount_begin(&tk_core.seq); *ts = tk_xtime(tk);
} while (read_seqcount_retry(&tk_core.seq, seq));
}
EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
void ktime_get_coarse_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct timespec64 now, mono;
unsigned int seq;
do {
seq = read_seqcount_begin(&tk_core.seq);
now = tk_xtime(tk);
mono = tk->wall_to_monotonic;
} while (read_seqcount_retry(&tk_core.seq, seq));
set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
now.tv_nsec + mono.tv_nsec);
}
EXPORT_SYMBOL(ktime_get_coarse_ts64);
/*
* Must hold jiffies_lock
*/
void do_timer(unsigned long ticks)
{
jiffies_64 += ticks;
calc_global_load();
}
/**
* ktime_get_update_offsets_now - hrtimer helper
* @cwsseq: pointer to check and store the clock was set sequence number
* @offs_real: pointer to storage for monotonic -> realtime offset
* @offs_boot: pointer to storage for monotonic -> boottime offset
* @offs_tai: pointer to storage for monotonic -> clock tai offset
*
* Returns current monotonic time and updates the offsets if the
* sequence number in @cwsseq and timekeeper.clock_was_set_seq are
* different.
*
* Called from hrtimer_interrupt() or retrigger_next_event()
*/
ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
ktime_t *offs_boot, ktime_t *offs_tai)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base;
u64 nsecs;
do {
seq = read_seqcount_begin(&tk_core.seq);
base = tk->tkr_mono.base;
nsecs = timekeeping_get_ns(&tk->tkr_mono);
base = ktime_add_ns(base, nsecs);
if (*cwsseq != tk->clock_was_set_seq) {
*cwsseq = tk->clock_was_set_seq;
*offs_real = tk->offs_real;
*offs_boot = tk->offs_boot;
*offs_tai = tk->offs_tai;
}
/* Handle leapsecond insertion adjustments */
if (unlikely(base >= tk->next_leap_ktime)) *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); } while (read_seqcount_retry(&tk_core.seq, seq)); return base;
}
/*
* timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
*/
static int timekeeping_validate_timex(const struct __kernel_timex *txc)
{
if (txc->modes & ADJ_ADJTIME) {
/* singleshot must not be used with any other mode bits */
if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
return -EINVAL;
if (!(txc->modes & ADJ_OFFSET_READONLY) &&
!capable(CAP_SYS_TIME))
return -EPERM;
} else {
/* In order to modify anything, you gotta be super-user! */
if (txc->modes && !capable(CAP_SYS_TIME))
return -EPERM;
/*
* if the quartz is off by more than 10% then
* something is VERY wrong!
*/
if (txc->modes & ADJ_TICK &&
(txc->tick < 900000/USER_HZ ||
txc->tick > 1100000/USER_HZ))
return -EINVAL;
}
if (txc->modes & ADJ_SETOFFSET) {
/* In order to inject time, you gotta be super-user! */
if (!capable(CAP_SYS_TIME))
return -EPERM;
/*
* Validate if a timespec/timeval used to inject a time
* offset is valid. Offsets can be positive or negative, so
* we don't check tv_sec. The value of the timeval/timespec
* is the sum of its fields,but *NOTE*:
* The field tv_usec/tv_nsec must always be non-negative and
* we can't have more nanoseconds/microseconds than a second.
*/
if (txc->time.tv_usec < 0)
return -EINVAL;
if (txc->modes & ADJ_NANO) {
if (txc->time.tv_usec >= NSEC_PER_SEC)
return -EINVAL;
} else {
if (txc->time.tv_usec >= USEC_PER_SEC)
return -EINVAL;
}
}
/*
* Check for potential multiplication overflows that can
* only happen on 64-bit systems:
*/
if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
if (LLONG_MIN / PPM_SCALE > txc->freq)
return -EINVAL;
if (LLONG_MAX / PPM_SCALE < txc->freq)
return -EINVAL;
}
return 0;
}
/**
* random_get_entropy_fallback - Returns the raw clock source value,
* used by random.c for platforms with no valid random_get_entropy().
*/
unsigned long random_get_entropy_fallback(void)
{
struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
struct clocksource *clock = READ_ONCE(tkr->clock);
if (unlikely(timekeeping_suspended || !clock))
return 0;
return clock->read(clock);
}
EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
/**
* do_adjtimex() - Accessor function to NTP __do_adjtimex function
*/
int do_adjtimex(struct __kernel_timex *txc)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct audit_ntp_data ad;
bool clock_set = false;
struct timespec64 ts;
unsigned long flags;
s32 orig_tai, tai;
int ret;
/* Validate the data before disabling interrupts */
ret = timekeeping_validate_timex(txc);
if (ret)
return ret;
add_device_randomness(txc, sizeof(*txc));
if (txc->modes & ADJ_SETOFFSET) {
struct timespec64 delta;
delta.tv_sec = txc->time.tv_sec;
delta.tv_nsec = txc->time.tv_usec;
if (!(txc->modes & ADJ_NANO))
delta.tv_nsec *= 1000;
ret = timekeeping_inject_offset(&delta);
if (ret)
return ret;
audit_tk_injoffset(delta);
}
audit_ntp_init(&ad);
ktime_get_real_ts64(&ts);
add_device_randomness(&ts, sizeof(ts));
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
orig_tai = tai = tk->tai_offset;
ret = __do_adjtimex(txc, &ts, &tai, &ad);
if (tai != orig_tai) {
__timekeeping_set_tai_offset(tk, tai);
timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
clock_set = true;
}
tk_update_leap_state(tk);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
audit_ntp_log(&ad);
/* Update the multiplier immediately if frequency was set directly */
if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
clock_set |= timekeeping_advance(TK_ADV_FREQ);
if (clock_set)
clock_was_set(CLOCK_REALTIME);
ntp_notify_cmos_timer();
return ret;
}
#ifdef CONFIG_NTP_PPS
/**
* hardpps() - Accessor function to NTP __hardpps function
*/
void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
{
unsigned long flags;
raw_spin_lock_irqsave(&timekeeper_lock, flags);
write_seqcount_begin(&tk_core.seq);
__hardpps(phase_ts, raw_ts);
write_seqcount_end(&tk_core.seq);
raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
}
EXPORT_SYMBOL(hardpps);
#endif /* CONFIG_NTP_PPS */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_GFP_H
#define __LINUX_GFP_H
#include <linux/gfp_types.h>
#include <linux/mmzone.h>
#include <linux/topology.h>
struct vm_area_struct;
struct mempolicy;
/* Convert GFP flags to their corresponding migrate type */
#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
#define GFP_MOVABLE_SHIFT 3
static inline int gfp_migratetype(const gfp_t gfp_flags)
{
VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE);
BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >>
GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC);
if (unlikely(page_group_by_mobility_disabled))
return MIGRATE_UNMOVABLE;
/* Group based on mobility */
return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
}
#undef GFP_MOVABLE_MASK
#undef GFP_MOVABLE_SHIFT
static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
{
return !!(gfp_flags & __GFP_DIRECT_RECLAIM);
}
#ifdef CONFIG_HIGHMEM
#define OPT_ZONE_HIGHMEM ZONE_HIGHMEM
#else
#define OPT_ZONE_HIGHMEM ZONE_NORMAL
#endif
#ifdef CONFIG_ZONE_DMA
#define OPT_ZONE_DMA ZONE_DMA
#else
#define OPT_ZONE_DMA ZONE_NORMAL
#endif
#ifdef CONFIG_ZONE_DMA32
#define OPT_ZONE_DMA32 ZONE_DMA32
#else
#define OPT_ZONE_DMA32 ZONE_NORMAL
#endif
/*
* GFP_ZONE_TABLE is a word size bitstring that is used for looking up the
* zone to use given the lowest 4 bits of gfp_t. Entries are GFP_ZONES_SHIFT
* bits long and there are 16 of them to cover all possible combinations of
* __GFP_DMA, __GFP_DMA32, __GFP_MOVABLE and __GFP_HIGHMEM.
*
* The zone fallback order is MOVABLE=>HIGHMEM=>NORMAL=>DMA32=>DMA.
* But GFP_MOVABLE is not only a zone specifier but also an allocation
* policy. Therefore __GFP_MOVABLE plus another zone selector is valid.
* Only 1 bit of the lowest 3 bits (DMA,DMA32,HIGHMEM) can be set to "1".
*
* bit result
* =================
* 0x0 => NORMAL
* 0x1 => DMA or NORMAL
* 0x2 => HIGHMEM or NORMAL
* 0x3 => BAD (DMA+HIGHMEM)
* 0x4 => DMA32 or NORMAL
* 0x5 => BAD (DMA+DMA32)
* 0x6 => BAD (HIGHMEM+DMA32)
* 0x7 => BAD (HIGHMEM+DMA32+DMA)
* 0x8 => NORMAL (MOVABLE+0)
* 0x9 => DMA or NORMAL (MOVABLE+DMA)
* 0xa => MOVABLE (Movable is valid only if HIGHMEM is set too)
* 0xb => BAD (MOVABLE+HIGHMEM+DMA)
* 0xc => DMA32 or NORMAL (MOVABLE+DMA32)
* 0xd => BAD (MOVABLE+DMA32+DMA)
* 0xe => BAD (MOVABLE+DMA32+HIGHMEM)
* 0xf => BAD (MOVABLE+DMA32+HIGHMEM+DMA)
*
* GFP_ZONES_SHIFT must be <= 2 on 32 bit platforms.
*/
#if defined(CONFIG_ZONE_DEVICE) && (MAX_NR_ZONES-1) <= 4
/* ZONE_DEVICE is not a valid GFP zone specifier */
#define GFP_ZONES_SHIFT 2
#else
#define GFP_ZONES_SHIFT ZONES_SHIFT
#endif
#if 16 * GFP_ZONES_SHIFT > BITS_PER_LONG
#error GFP_ZONES_SHIFT too large to create GFP_ZONE_TABLE integer
#endif
#define GFP_ZONE_TABLE ( \
(ZONE_NORMAL << 0 * GFP_ZONES_SHIFT) \
| (OPT_ZONE_DMA << ___GFP_DMA * GFP_ZONES_SHIFT) \
| (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * GFP_ZONES_SHIFT) \
| (OPT_ZONE_DMA32 << ___GFP_DMA32 * GFP_ZONES_SHIFT) \
| (ZONE_NORMAL << ___GFP_MOVABLE * GFP_ZONES_SHIFT) \
| (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * GFP_ZONES_SHIFT) \
| (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * GFP_ZONES_SHIFT)\
| (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * GFP_ZONES_SHIFT)\
)
/*
* GFP_ZONE_BAD is a bitmap for all combinations of __GFP_DMA, __GFP_DMA32
* __GFP_HIGHMEM and __GFP_MOVABLE that are not permitted. One flag per
* entry starting with bit 0. Bit is set if the combination is not
* allowed.
*/
#define GFP_ZONE_BAD ( \
1 << (___GFP_DMA | ___GFP_HIGHMEM) \
| 1 << (___GFP_DMA | ___GFP_DMA32) \
| 1 << (___GFP_DMA32 | ___GFP_HIGHMEM) \
| 1 << (___GFP_DMA | ___GFP_DMA32 | ___GFP_HIGHMEM) \
| 1 << (___GFP_MOVABLE | ___GFP_HIGHMEM | ___GFP_DMA) \
| 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA) \
| 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_HIGHMEM) \
| 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA | ___GFP_HIGHMEM) \
)
static inline enum zone_type gfp_zone(gfp_t flags)
{
enum zone_type z;
int bit = (__force int) (flags & GFP_ZONEMASK);
z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) &
((1 << GFP_ZONES_SHIFT) - 1);
VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1);
return z;
}
/*
* There is only one page-allocator function, and two main namespaces to
* it. The alloc_page*() variants return 'struct page *' and as such
* can allocate highmem pages, the *get*page*() variants return
* virtual kernel addresses to the allocated page(s).
*/
static inline int gfp_zonelist(gfp_t flags)
{
#ifdef CONFIG_NUMA
if (unlikely(flags & __GFP_THISNODE))
return ZONELIST_NOFALLBACK;
#endif
return ZONELIST_FALLBACK;
}
/*
* We get the zone list from the current node and the gfp_mask.
* This zone list contains a maximum of MAX_NUMNODES*MAX_NR_ZONES zones.
* There are two zonelists per node, one for all zones with memory and
* one containing just zones from the node the zonelist belongs to.
*
* For the case of non-NUMA systems the NODE_DATA() gets optimized to
* &contig_page_data at compile-time.
*/
static inline struct zonelist *node_zonelist(int nid, gfp_t flags)
{
return NODE_DATA(nid)->node_zonelists + gfp_zonelist(flags);
}
#ifndef HAVE_ARCH_FREE_PAGE
static inline void arch_free_page(struct page *page, int order) { }
#endif
#ifndef HAVE_ARCH_ALLOC_PAGE
static inline void arch_alloc_page(struct page *page, int order) { }
#endif
struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
nodemask_t *nodemask);
struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid,
nodemask_t *nodemask);
unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
nodemask_t *nodemask, int nr_pages,
struct list_head *page_list,
struct page **page_array);
unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,
unsigned long nr_pages,
struct page **page_array);
/* Bulk allocate order-0 pages */
static inline unsigned long
alloc_pages_bulk_list(gfp_t gfp, unsigned long nr_pages, struct list_head *list)
{
return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, list, NULL);
}
static inline unsigned long
alloc_pages_bulk_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array)
{
return __alloc_pages_bulk(gfp, numa_mem_id(), NULL, nr_pages, NULL, page_array);
}
static inline unsigned long
alloc_pages_bulk_array_node(gfp_t gfp, int nid, unsigned long nr_pages, struct page **page_array)
{
if (nid == NUMA_NO_NODE)
nid = numa_mem_id();
return __alloc_pages_bulk(gfp, nid, NULL, nr_pages, NULL, page_array);
}
static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask)
{
gfp_t warn_gfp = gfp_mask & (__GFP_THISNODE|__GFP_NOWARN);
if (warn_gfp != (__GFP_THISNODE|__GFP_NOWARN))
return;
if (node_online(this_node))
return;
pr_warn("%pGg allocation from offline node %d\n", &gfp_mask, this_node);
dump_stack();
}
/*
* Allocate pages, preferring the node given as nid. The node must be valid and
* online. For more general interface, see alloc_pages_node().
*/
static inline struct page *
__alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
{
VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp_mask);
return __alloc_pages(gfp_mask, order, nid, NULL);
}
static inline
struct folio *__folio_alloc_node(gfp_t gfp, unsigned int order, int nid)
{
VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
warn_if_node_offline(nid, gfp);
return __folio_alloc(gfp, order, nid, NULL);
}
/*
* Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE,
* prefer the current CPU's closest node. Otherwise node must be valid and
* online.
*/
static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask,
unsigned int order)
{
if (nid == NUMA_NO_NODE)
nid = numa_mem_id();
return __alloc_pages_node(nid, gfp_mask, order);
}
#ifdef CONFIG_NUMA
struct page *alloc_pages(gfp_t gfp, unsigned int order);
struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
struct mempolicy *mpol, pgoff_t ilx, int nid);
struct folio *folio_alloc(gfp_t gfp, unsigned int order);
struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma,
unsigned long addr, bool hugepage);
#else
static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order)
{
return alloc_pages_node(numa_node_id(), gfp_mask, order);
}
static inline struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
struct mempolicy *mpol, pgoff_t ilx, int nid)
{
return alloc_pages(gfp, order);
}
static inline struct folio *folio_alloc(gfp_t gfp, unsigned int order)
{
return __folio_alloc_node(gfp, order, numa_node_id());
}
#define vma_alloc_folio(gfp, order, vma, addr, hugepage) \
folio_alloc(gfp, order)
#endif
#define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0)
static inline struct page *alloc_page_vma(gfp_t gfp,
struct vm_area_struct *vma, unsigned long addr)
{
struct folio *folio = vma_alloc_folio(gfp, 0, vma, addr, false);
return &folio->page;
}
extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);
extern unsigned long get_zeroed_page(gfp_t gfp_mask);
void *alloc_pages_exact(size_t size, gfp_t gfp_mask) __alloc_size(1);
void free_pages_exact(void *virt, size_t size);
__meminit void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
#define __get_free_page(gfp_mask) \
__get_free_pages((gfp_mask), 0)
#define __get_dma_pages(gfp_mask, order) \
__get_free_pages((gfp_mask) | GFP_DMA, (order))
extern void __free_pages(struct page *page, unsigned int order);
extern void free_pages(unsigned long addr, unsigned int order);
struct page_frag_cache;
void page_frag_cache_drain(struct page_frag_cache *nc);
extern void __page_frag_cache_drain(struct page *page, unsigned int count);
void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz,
gfp_t gfp_mask, unsigned int align_mask);
static inline void *page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
unsigned int align)
{
WARN_ON_ONCE(!is_power_of_2(align));
return __page_frag_alloc_align(nc, fragsz, gfp_mask, -align);
}
static inline void *page_frag_alloc(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask)
{
return __page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u);
}
extern void page_frag_free(void *addr);
#define __free_page(page) __free_pages((page), 0)
#define free_page(addr) free_pages((addr), 0)
void page_alloc_init_cpuhp(void);
int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp);
void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp);
void drain_all_pages(struct zone *zone);
void drain_local_pages(struct zone *zone);
void page_alloc_init_late(void);
void setup_pcp_cacheinfo(unsigned int cpu);
/*
* gfp_allowed_mask is set to GFP_BOOT_MASK during early boot to restrict what
* GFP flags are used before interrupts are enabled. Once interrupts are
* enabled, it is set to __GFP_BITS_MASK while the system is running. During
* hibernation, it is used by PM to avoid I/O during memory allocation while
* devices are suspended.
*/
extern gfp_t gfp_allowed_mask;
/* Returns true if the gfp_mask allows use of ALLOC_NO_WATERMARK */
bool gfp_pfmemalloc_allowed(gfp_t gfp_mask);
static inline bool gfp_has_io_fs(gfp_t gfp)
{
return (gfp & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS);
}
/*
* Check if the gfp flags allow compaction - GFP_NOIO is a really
* tricky context because the migration might require IO.
*/
static inline bool gfp_compaction_allowed(gfp_t gfp_mask)
{
return IS_ENABLED(CONFIG_COMPACTION) && (gfp_mask & __GFP_IO);
}
extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma);
#ifdef CONFIG_CONTIG_ALLOC
/* The below functions must be run on a range from a single zone. */
extern int alloc_contig_range(unsigned long start, unsigned long end,
unsigned migratetype, gfp_t gfp_mask);
extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
int nid, nodemask_t *nodemask);
#endif
void free_contig_range(unsigned long pfn, unsigned long nr_pages);
#endif /* __LINUX_GFP_H */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
*/
#include <linux/gfp.h>
#include <linux/init.h>
#include <linux/ratelimit.h>
#include <linux/usb.h>
#include <linux/usb/audio.h>
#include <linux/slab.h>
#include <sound/core.h>
#include <sound/pcm.h>
#include <sound/pcm_params.h>
#include "usbaudio.h"
#include "helper.h"
#include "card.h"
#include "endpoint.h"
#include "pcm.h"
#include "clock.h"
#include "quirks.h"
enum {
EP_STATE_STOPPED,
EP_STATE_RUNNING,
EP_STATE_STOPPING,
};
/* interface refcounting */
struct snd_usb_iface_ref {
unsigned char iface;
bool need_setup;
int opened;
int altset;
struct list_head list;
};
/* clock refcounting */
struct snd_usb_clock_ref {
unsigned char clock;
atomic_t locked;
int opened;
int rate;
bool need_setup;
struct list_head list;
};
/*
* snd_usb_endpoint is a model that abstracts everything related to an
* USB endpoint and its streaming.
*
* There are functions to activate and deactivate the streaming URBs and
* optional callbacks to let the pcm logic handle the actual content of the
* packets for playback and record. Thus, the bus streaming and the audio
* handlers are fully decoupled.
*
* There are two different types of endpoints in audio applications.
*
* SND_USB_ENDPOINT_TYPE_DATA handles full audio data payload for both
* inbound and outbound traffic.
*
* SND_USB_ENDPOINT_TYPE_SYNC endpoints are for inbound traffic only and
* expect the payload to carry Q10.14 / Q16.16 formatted sync information
* (3 or 4 bytes).
*
* Each endpoint has to be configured prior to being used by calling
* snd_usb_endpoint_set_params().
*
* The model incorporates a reference counting, so that multiple users
* can call snd_usb_endpoint_start() and snd_usb_endpoint_stop(), and
* only the first user will effectively start the URBs, and only the last
* one to stop it will tear the URBs down again.
*/
/*
* convert a sampling rate into our full speed format (fs/1000 in Q16.16)
* this will overflow at approx 524 kHz
*/
static inline unsigned get_usb_full_speed_rate(unsigned int rate)
{
return ((rate << 13) + 62) / 125;
}
/*
* convert a sampling rate into USB high speed format (fs/8000 in Q16.16)
* this will overflow at approx 4 MHz
*/
static inline unsigned get_usb_high_speed_rate(unsigned int rate)
{
return ((rate << 10) + 62) / 125;
}
/*
* release a urb data
*/
static void release_urb_ctx(struct snd_urb_ctx *u)
{
if (u->urb && u->buffer_size) usb_free_coherent(u->ep->chip->dev, u->buffer_size,
u->urb->transfer_buffer,
u->urb->transfer_dma);
usb_free_urb(u->urb);
u->urb = NULL;
u->buffer_size = 0;
}
static const char *usb_error_string(int err)
{
switch (err) {
case -ENODEV:
return "no device";
case -ENOENT:
return "endpoint not enabled";
case -EPIPE:
return "endpoint stalled";
case -ENOSPC:
return "not enough bandwidth";
case -ESHUTDOWN:
return "device disabled";
case -EHOSTUNREACH:
return "device suspended";
case -EINVAL:
case -EAGAIN:
case -EFBIG:
case -EMSGSIZE:
return "internal error";
default:
return "unknown error";
}
}
static inline bool ep_state_running(struct snd_usb_endpoint *ep)
{
return atomic_read(&ep->state) == EP_STATE_RUNNING;
}
static inline bool ep_state_update(struct snd_usb_endpoint *ep, int old, int new)
{
return atomic_try_cmpxchg(&ep->state, &old, new);
}
/**
* snd_usb_endpoint_implicit_feedback_sink: Report endpoint usage type
*
* @ep: The snd_usb_endpoint
*
* Determine whether an endpoint is driven by an implicit feedback
* data endpoint source.
*/
int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep)
{
return ep->implicit_fb_sync && usb_pipeout(ep->pipe);
}
/*
* Return the number of samples to be sent in the next packet
* for streaming based on information derived from sync endpoints
*
* This won't be used for implicit feedback which takes the packet size
* returned from the sync source
*/
static int slave_next_packet_size(struct snd_usb_endpoint *ep,
unsigned int avail)
{
unsigned long flags;
unsigned int phase;
int ret;
if (ep->fill_max)
return ep->maxframesize;
spin_lock_irqsave(&ep->lock, flags);
phase = (ep->phase & 0xffff) + (ep->freqm << ep->datainterval);
ret = min(phase >> 16, ep->maxframesize);
if (avail && ret >= avail)
ret = -EAGAIN;
else
ep->phase = phase;
spin_unlock_irqrestore(&ep->lock, flags);
return ret;
}
/*
* Return the number of samples to be sent in the next packet
* for adaptive and synchronous endpoints
*/
static int next_packet_size(struct snd_usb_endpoint *ep, unsigned int avail)
{
unsigned int sample_accum;
int ret;
if (ep->fill_max)
return ep->maxframesize;
sample_accum = ep->sample_accum + ep->sample_rem;
if (sample_accum >= ep->pps) {
sample_accum -= ep->pps;
ret = ep->packsize[1];
} else {
ret = ep->packsize[0];
}
if (avail && ret >= avail)
ret = -EAGAIN;
else
ep->sample_accum = sample_accum;
return ret;
}
/*
* snd_usb_endpoint_next_packet_size: Return the number of samples to be sent
* in the next packet
*
* If the size is equal or exceeds @avail, don't proceed but return -EAGAIN
* Exception: @avail = 0 for skipping the check.
*/
int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *ctx, int idx,
unsigned int avail)
{
unsigned int packet;
packet = ctx->packet_size[idx];
if (packet) {
if (avail && packet >= avail)
return -EAGAIN;
return packet;
}
if (ep->sync_source)
return slave_next_packet_size(ep, avail);
else
return next_packet_size(ep, avail);
}
static void call_retire_callback(struct snd_usb_endpoint *ep,
struct urb *urb)
{
struct snd_usb_substream *data_subs;
data_subs = READ_ONCE(ep->data_subs);
if (data_subs && ep->retire_data_urb)
ep->retire_data_urb(data_subs, urb);
}
static void retire_outbound_urb(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *urb_ctx)
{
call_retire_callback(ep, urb_ctx->urb);
}
static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep,
struct snd_usb_endpoint *sender,
const struct urb *urb);
static void retire_inbound_urb(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *urb_ctx)
{
struct urb *urb = urb_ctx->urb;
struct snd_usb_endpoint *sync_sink;
if (unlikely(ep->skip_packets > 0)) {
ep->skip_packets--;
return;
}
sync_sink = READ_ONCE(ep->sync_sink);
if (sync_sink)
snd_usb_handle_sync_urb(sync_sink, ep, urb);
call_retire_callback(ep, urb);
}
static inline bool has_tx_length_quirk(struct snd_usb_audio *chip)
{
return chip->quirk_flags & QUIRK_FLAG_TX_LENGTH;
}
static void prepare_silent_urb(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *ctx)
{
struct urb *urb = ctx->urb;
unsigned int offs = 0;
unsigned int extra = 0;
__le32 packet_length;
int i;
/* For tx_length_quirk, put packet length at start of packet */
if (has_tx_length_quirk(ep->chip))
extra = sizeof(packet_length);
for (i = 0; i < ctx->packets; ++i) {
unsigned int offset;
unsigned int length;
int counts;
counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0);
length = counts * ep->stride; /* number of silent bytes */
offset = offs * ep->stride + extra * i;
urb->iso_frame_desc[i].offset = offset;
urb->iso_frame_desc[i].length = length + extra;
if (extra) {
packet_length = cpu_to_le32(length);
memcpy(urb->transfer_buffer + offset,
&packet_length, sizeof(packet_length));
}
memset(urb->transfer_buffer + offset + extra,
ep->silence_value, length);
offs += counts;
}
urb->number_of_packets = ctx->packets;
urb->transfer_buffer_length = offs * ep->stride + ctx->packets * extra;
ctx->queued = 0;
}
/*
* Prepare a PLAYBACK urb for submission to the bus.
*/
static int prepare_outbound_urb(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *ctx,
bool in_stream_lock)
{
struct urb *urb = ctx->urb;
unsigned char *cp = urb->transfer_buffer;
struct snd_usb_substream *data_subs;
urb->dev = ep->chip->dev; /* we need to set this at each time */
switch (ep->type) {
case SND_USB_ENDPOINT_TYPE_DATA:
data_subs = READ_ONCE(ep->data_subs);
if (data_subs && ep->prepare_data_urb)
return ep->prepare_data_urb(data_subs, urb, in_stream_lock);
/* no data provider, so send silence */
prepare_silent_urb(ep, ctx);
break;
case SND_USB_ENDPOINT_TYPE_SYNC:
if (snd_usb_get_speed(ep->chip->dev) >= USB_SPEED_HIGH) {
/*
* fill the length and offset of each urb descriptor.
* the fixed 12.13 frequency is passed as 16.16 through the pipe.
*/
urb->iso_frame_desc[0].length = 4;
urb->iso_frame_desc[0].offset = 0;
cp[0] = ep->freqn;
cp[1] = ep->freqn >> 8;
cp[2] = ep->freqn >> 16;
cp[3] = ep->freqn >> 24;
} else {
/*
* fill the length and offset of each urb descriptor.
* the fixed 10.14 frequency is passed through the pipe.
*/
urb->iso_frame_desc[0].length = 3;
urb->iso_frame_desc[0].offset = 0;
cp[0] = ep->freqn >> 2;
cp[1] = ep->freqn >> 10;
cp[2] = ep->freqn >> 18;
}
break;
}
return 0;
}
/*
* Prepare a CAPTURE or SYNC urb for submission to the bus.
*/
static int prepare_inbound_urb(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *urb_ctx)
{
int i, offs;
struct urb *urb = urb_ctx->urb;
urb->dev = ep->chip->dev; /* we need to set this at each time */
switch (ep->type) {
case SND_USB_ENDPOINT_TYPE_DATA:
offs = 0;
for (i = 0; i < urb_ctx->packets; i++) {
urb->iso_frame_desc[i].offset = offs;
urb->iso_frame_desc[i].length = ep->curpacksize;
offs += ep->curpacksize;
}
urb->transfer_buffer_length = offs;
urb->number_of_packets = urb_ctx->packets;
break;
case SND_USB_ENDPOINT_TYPE_SYNC:
urb->iso_frame_desc[0].length = min(4u, ep->syncmaxsize);
urb->iso_frame_desc[0].offset = 0;
break;
}
return 0;
}
/* notify an error as XRUN to the assigned PCM data substream */
static void notify_xrun(struct snd_usb_endpoint *ep)
{
struct snd_usb_substream *data_subs;
data_subs = READ_ONCE(ep->data_subs);
if (data_subs && data_subs->pcm_substream)
snd_pcm_stop_xrun(data_subs->pcm_substream);
}
static struct snd_usb_packet_info *
next_packet_fifo_enqueue(struct snd_usb_endpoint *ep)
{
struct snd_usb_packet_info *p;
p = ep->next_packet + (ep->next_packet_head + ep->next_packet_queued) %
ARRAY_SIZE(ep->next_packet);
ep->next_packet_queued++;
return p;
}
static struct snd_usb_packet_info *
next_packet_fifo_dequeue(struct snd_usb_endpoint *ep)
{
struct snd_usb_packet_info *p;
p = ep->next_packet + ep->next_packet_head;
ep->next_packet_head++;
ep->next_packet_head %= ARRAY_SIZE(ep->next_packet);
ep->next_packet_queued--;
return p;
}
static void push_back_to_ready_list(struct snd_usb_endpoint *ep,
struct snd_urb_ctx *ctx)
{
unsigned long flags;
spin_lock_irqsave(&ep->lock, flags);
list_add_tail(&ctx->ready_list, &ep->ready_playback_urbs);
spin_unlock_irqrestore(&ep->lock, flags);
}
/*
* Send output urbs that have been prepared previously. URBs are dequeued
* from ep->ready_playback_urbs and in case there aren't any available
* or there are no packets that have been prepared, this function does
* nothing.
*
* The reason why the functionality of sending and preparing URBs is separated
* is that host controllers don't guarantee the order in which they return
* inbound and outbound packets to their submitters.
*
* This function is used both for implicit feedback endpoints and in low-
* latency playback mode.
*/
int snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep,
bool in_stream_lock)
{
bool implicit_fb = snd_usb_endpoint_implicit_feedback_sink(ep);
while (ep_state_running(ep)) {
unsigned long flags;
struct snd_usb_packet_info *packet;
struct snd_urb_ctx *ctx = NULL;
int err, i;
spin_lock_irqsave(&ep->lock, flags);
if ((!implicit_fb || ep->next_packet_queued > 0) &&
!list_empty(&ep->ready_playback_urbs)) {
/* take URB out of FIFO */
ctx = list_first_entry(&ep->ready_playback_urbs,
struct snd_urb_ctx, ready_list);
list_del_init(&ctx->ready_list);
if (implicit_fb)
packet = next_packet_fifo_dequeue(ep);
}
spin_unlock_irqrestore(&ep->lock, flags);
if (ctx == NULL)
break;
/* copy over the length information */
if (implicit_fb) {
for (i = 0; i < packet->packets; i++)
ctx->packet_size[i] = packet->packet_size[i];
}
/* call the data handler to fill in playback data */
err = prepare_outbound_urb(ep, ctx, in_stream_lock);
/* can be stopped during prepare callback */
if (unlikely(!ep_state_running(ep)))
break;
if (err < 0) {
/* push back to ready list again for -EAGAIN */
if (err == -EAGAIN) {
push_back_to_ready_list(ep, ctx);
break;
}
if (!in_stream_lock)
notify_xrun(ep);
return -EPIPE;
}
if (!atomic_read(&ep->chip->shutdown))
err = usb_submit_urb(ctx->urb, GFP_ATOMIC);
else
err = -ENODEV;
if (err < 0) {
if (!atomic_read(&ep->chip->shutdown)) {
usb_audio_err(ep->chip,
"Unable to submit urb #%d: %d at %s\n",
ctx->index, err, __func__);
if (!in_stream_lock)
notify_xrun(ep);
}
return -EPIPE;
}
set_bit(ctx->index, &ep->active_mask);
atomic_inc(&ep->submitted_urbs);
}
return 0;
}
/*
* complete callback for urbs
*/
static void snd_complete_urb(struct urb *urb)
{
struct snd_urb_ctx *ctx = urb->context;
struct snd_usb_endpoint *ep = ctx->ep;
int err;
if (unlikely(urb->status == -ENOENT || /* unlinked */
urb->status == -ENODEV || /* device removed */
urb->status == -ECONNRESET || /* unlinked */
urb->status == -ESHUTDOWN)) /* device disabled */
goto exit_clear;
/* device disconnected */
if (unlikely(atomic_read(&ep->chip->shutdown)))
goto exit_clear;
if (unlikely(!ep_state_running(ep)))
goto exit_clear;
if (usb_pipeout(ep->pipe)) {
retire_outbound_urb(ep, ctx);
/* can be stopped during retire callback */
if (unlikely(!ep_state_running(ep)))
goto exit_clear;
/* in low-latency and implicit-feedback modes, push back the
* URB to ready list at first, then process as much as possible
*/
if (ep->lowlatency_playback ||
snd_usb_endpoint_implicit_feedback_sink(ep)) {
push_back_to_ready_list(ep, ctx);
clear_bit(ctx->index, &ep->active_mask);
snd_usb_queue_pending_output_urbs(ep, false);
atomic_dec(&ep->submitted_urbs); /* decrement at last */
return;
}
/* in non-lowlatency mode, no error handling for prepare */
prepare_outbound_urb(ep, ctx, false);
/* can be stopped during prepare callback */
if (unlikely(!ep_state_running(ep)))
goto exit_clear;
} else {
retire_inbound_urb(ep, ctx);
/* can be stopped during retire callback */
if (unlikely(!ep_state_running(ep)))
goto exit_clear;
prepare_inbound_urb(ep, ctx);
}
if (!atomic_read(&ep->chip->shutdown))
err = usb_submit_urb(urb, GFP_ATOMIC);
else
err = -ENODEV;
if (err == 0)
return;
if (!atomic_read(&ep->chip->shutdown)) {
usb_audio_err(ep->chip, "cannot submit urb (err = %d)\n", err);
notify_xrun(ep);
}
exit_clear:
clear_bit(ctx->index, &ep->active_mask);
atomic_dec(&ep->submitted_urbs);
}
/*
* Find or create a refcount object for the given interface
*
* The objects are released altogether in snd_usb_endpoint_free_all()
*/
static struct snd_usb_iface_ref *
iface_ref_find(struct snd_usb_audio *chip, int iface)
{
struct snd_usb_iface_ref *ip;
list_for_each_entry(ip, &chip->iface_ref_list, list)
if (ip->iface == iface)
return ip;
ip = kzalloc(sizeof(*ip), GFP_KERNEL);
if (!ip)
return NULL;
ip->iface = iface;
list_add_tail(&ip->list, &chip->iface_ref_list);
return ip;
}
/* Similarly, a refcount object for clock */
static struct snd_usb_clock_ref *
clock_ref_find(struct snd_usb_audio *chip, int clock)
{
struct snd_usb_clock_ref *ref;
list_for_each_entry(ref, &chip->clock_ref_list, list)
if (ref->clock == clock)
return ref;
ref = kzalloc(sizeof(*ref), GFP_KERNEL);
if (!ref)
return NULL;
ref->clock = clock;
atomic_set(&ref->locked, 0);
list_add_tail(&ref->list, &chip->clock_ref_list);
return ref;
}
/*
* Get the existing endpoint object corresponding EP
* Returns NULL if not present.
*/
struct snd_usb_endpoint *
snd_usb_get_endpoint(struct snd_usb_audio *chip, int ep_num)
{
struct snd_usb_endpoint *ep;
list_for_each_entry(ep, &chip->ep_list, list) {
if (ep->ep_num == ep_num)
return ep;
}
return NULL;
}
#define ep_type_name(type) \
(type == SND_USB_ENDPOINT_TYPE_DATA ? "data" : "sync")
/**
* snd_usb_add_endpoint: Add an endpoint to an USB audio chip
*
* @chip: The chip
* @ep_num: The number of the endpoint to use
* @type: SND_USB_ENDPOINT_TYPE_DATA or SND_USB_ENDPOINT_TYPE_SYNC
*
* If the requested endpoint has not been added to the given chip before,
* a new instance is created.
*
* Returns zero on success or a negative error code.
*
* New endpoints will be added to chip->ep_list and freed by
* calling snd_usb_endpoint_free_all().
*
* For SND_USB_ENDPOINT_TYPE_SYNC, the caller needs to guarantee that
* bNumEndpoints > 1 beforehand.
*/
int snd_usb_add_endpoint(struct snd_usb_audio *chip, int ep_num, int type)
{
struct snd_usb_endpoint *ep;
bool is_playback;
ep = snd_usb_get_endpoint(chip, ep_num);
if (ep)
return 0;
usb_audio_dbg(chip, "Creating new %s endpoint #%x\n",
ep_type_name(type),
ep_num);
ep = kzalloc(sizeof(*ep), GFP_KERNEL);
if (!ep)
return -ENOMEM;
ep->chip = chip;
spin_lock_init(&ep->lock);
ep->type = type;
ep->ep_num = ep_num;
INIT_LIST_HEAD(&ep->ready_playback_urbs);
atomic_set(&ep->submitted_urbs, 0);
is_playback = ((ep_num & USB_ENDPOINT_DIR_MASK) == USB_DIR_OUT);
ep_num &= USB_ENDPOINT_NUMBER_MASK;
if (is_playback)
ep->pipe = usb_sndisocpipe(chip->dev, ep_num);
else
ep->pipe = usb_rcvisocpipe(chip->dev, ep_num);
list_add_tail(&ep->list, &chip->ep_list);
return 0;
}
/* Set up syncinterval and maxsyncsize for a sync EP */
static void endpoint_set_syncinterval(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep)
{
struct usb_host_interface *alts;
struct usb_endpoint_descriptor *desc;
alts = snd_usb_get_host_interface(chip, ep->iface, ep->altsetting);
if (!alts)
return;
desc = get_endpoint(alts, ep->ep_idx);
if (desc->bLength >= USB_DT_ENDPOINT_AUDIO_SIZE &&
desc->bRefresh >= 1 && desc->bRefresh <= 9)
ep->syncinterval = desc->bRefresh;
else if (snd_usb_get_speed(chip->dev) == USB_SPEED_FULL)
ep->syncinterval = 1;
else if (desc->bInterval >= 1 && desc->bInterval <= 16)
ep->syncinterval = desc->bInterval - 1;
else
ep->syncinterval = 3;
ep->syncmaxsize = le16_to_cpu(desc->wMaxPacketSize);
}
static bool endpoint_compatible(struct snd_usb_endpoint *ep,
const struct audioformat *fp,
const struct snd_pcm_hw_params *params)
{
if (!ep->opened)
return false;
if (ep->cur_audiofmt != fp)
return false;
if (ep->cur_rate != params_rate(params) ||
ep->cur_format != params_format(params) ||
ep->cur_period_frames != params_period_size(params) ||
ep->cur_buffer_periods != params_periods(params))
return false;
return true;
}
/*
* Check whether the given fp and hw params are compatible with the current
* setup of the target EP for implicit feedback sync
*/
bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep,
const struct audioformat *fp,
const struct snd_pcm_hw_params *params)
{
bool ret;
mutex_lock(&chip->mutex);
ret = endpoint_compatible(ep, fp, params);
mutex_unlock(&chip->mutex);
return ret;
}
/*
* snd_usb_endpoint_open: Open the endpoint
*
* Called from hw_params to assign the endpoint to the substream.
* It's reference-counted, and only the first opener is allowed to set up
* arbitrary parameters. The later opener must be compatible with the
* former opened parameters.
* The endpoint needs to be closed via snd_usb_endpoint_close() later.
*
* Note that this function doesn't configure the endpoint. The substream
* needs to set it up later via snd_usb_endpoint_set_params() and
* snd_usb_endpoint_prepare().
*/
struct snd_usb_endpoint *
snd_usb_endpoint_open(struct snd_usb_audio *chip,
const struct audioformat *fp,
const struct snd_pcm_hw_params *params,
bool is_sync_ep,
bool fixed_rate)
{
struct snd_usb_endpoint *ep;
int ep_num = is_sync_ep ? fp->sync_ep : fp->endpoint;
mutex_lock(&chip->mutex);
ep = snd_usb_get_endpoint(chip, ep_num);
if (!ep) {
usb_audio_err(chip, "Cannot find EP 0x%x to open\n", ep_num);
goto unlock;
}
if (!ep->opened) {
if (is_sync_ep) {
ep->iface = fp->sync_iface;
ep->altsetting = fp->sync_altsetting;
ep->ep_idx = fp->sync_ep_idx;
} else {
ep->iface = fp->iface;
ep->altsetting = fp->altsetting;
ep->ep_idx = fp->ep_idx;
}
usb_audio_dbg(chip, "Open EP 0x%x, iface=%d:%d, idx=%d\n",
ep_num, ep->iface, ep->altsetting, ep->ep_idx);
ep->iface_ref = iface_ref_find(chip, ep->iface);
if (!ep->iface_ref) {
ep = NULL;
goto unlock;
}
if (fp->protocol != UAC_VERSION_1) {
ep->clock_ref = clock_ref_find(chip, fp->clock);
if (!ep->clock_ref) {
ep = NULL;
goto unlock;
}
ep->clock_ref->opened++;
}
ep->cur_audiofmt = fp;
ep->cur_channels = fp->channels;
ep->cur_rate = params_rate(params);
ep->cur_format = params_format(params);
ep->cur_frame_bytes = snd_pcm_format_physical_width(ep->cur_format) *
ep->cur_channels / 8;
ep->cur_period_frames = params_period_size(params);
ep->cur_period_bytes = ep->cur_period_frames * ep->cur_frame_bytes;
ep->cur_buffer_periods = params_periods(params);
if (ep->type == SND_USB_ENDPOINT_TYPE_SYNC)
endpoint_set_syncinterval(chip, ep);
ep->implicit_fb_sync = fp->implicit_fb;
ep->need_setup = true;
ep->need_prepare = true;
ep->fixed_rate = fixed_rate;
usb_audio_dbg(chip, " channels=%d, rate=%d, format=%s, period_bytes=%d, periods=%d, implicit_fb=%d\n",
ep->cur_channels, ep->cur_rate,
snd_pcm_format_name(ep->cur_format),
ep->cur_period_bytes, ep->cur_buffer_periods,
ep->implicit_fb_sync);
} else {
if (WARN_ON(!ep->iface_ref)) {
ep = NULL;
goto unlock;
}
if (!endpoint_compatible(ep, fp, params)) {
usb_audio_err(chip, "Incompatible EP setup for 0x%x\n",
ep_num);
ep = NULL;
goto unlock;
}
usb_audio_dbg(chip, "Reopened EP 0x%x (count %d)\n",
ep_num, ep->opened);
}
if (!ep->iface_ref->opened++)
ep->iface_ref->need_setup = true;
ep->opened++;
unlock:
mutex_unlock(&chip->mutex);
return ep;
}
/*
* snd_usb_endpoint_set_sync: Link data and sync endpoints
*
* Pass NULL to sync_ep to unlink again
*/
void snd_usb_endpoint_set_sync(struct snd_usb_audio *chip,
struct snd_usb_endpoint *data_ep,
struct snd_usb_endpoint *sync_ep)
{
data_ep->sync_source = sync_ep;
}
/*
* Set data endpoint callbacks and the assigned data stream
*
* Called at PCM trigger and cleanups.
* Pass NULL to deactivate each callback.
*/
void snd_usb_endpoint_set_callback(struct snd_usb_endpoint *ep,
int (*prepare)(struct snd_usb_substream *subs,
struct urb *urb,
bool in_stream_lock),
void (*retire)(struct snd_usb_substream *subs,
struct urb *urb),
struct snd_usb_substream *data_subs)
{
ep->prepare_data_urb = prepare;
ep->retire_data_urb = retire;
if (data_subs)
ep->lowlatency_playback = data_subs->lowlatency_playback;
else
ep->lowlatency_playback = false;
WRITE_ONCE(ep->data_subs, data_subs);
}
static int endpoint_set_interface(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep,
bool set)
{
int altset = set ? ep->altsetting : 0;
int err;
if (ep->iface_ref->altset == altset)
return 0;
usb_audio_dbg(chip, "Setting usb interface %d:%d for EP 0x%x\n",
ep->iface, altset, ep->ep_num);
err = usb_set_interface(chip->dev, ep->iface, altset);
if (err < 0) {
usb_audio_err_ratelimited(
chip, "%d:%d: usb_set_interface failed (%d)\n",
ep->iface, altset, err);
return err;
}
if (chip->quirk_flags & QUIRK_FLAG_IFACE_DELAY)
msleep(50);
ep->iface_ref->altset = altset;
return 0;
}
/*
* snd_usb_endpoint_close: Close the endpoint
*
* Unreference the already opened endpoint via snd_usb_endpoint_open().
*/
void snd_usb_endpoint_close(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep)
{
mutex_lock(&chip->mutex);
usb_audio_dbg(chip, "Closing EP 0x%x (count %d)\n",
ep->ep_num, ep->opened);
if (!--ep->iface_ref->opened &&
!(chip->quirk_flags & QUIRK_FLAG_IFACE_SKIP_CLOSE))
endpoint_set_interface(chip, ep, false);
if (!--ep->opened) {
if (ep->clock_ref) {
if (!--ep->clock_ref->opened)
ep->clock_ref->rate = 0;
}
ep->iface = 0;
ep->altsetting = 0;
ep->cur_audiofmt = NULL;
ep->cur_rate = 0;
ep->iface_ref = NULL;
ep->clock_ref = NULL;
usb_audio_dbg(chip, "EP 0x%x closed\n", ep->ep_num);
}
mutex_unlock(&chip->mutex);
}
/* Prepare for suspening EP, called from the main suspend handler */
void snd_usb_endpoint_suspend(struct snd_usb_endpoint *ep)
{
ep->need_prepare = true;
if (ep->iface_ref)
ep->iface_ref->need_setup = true;
if (ep->clock_ref)
ep->clock_ref->rate = 0;
}
/*
* wait until all urbs are processed.
*/
static int wait_clear_urbs(struct snd_usb_endpoint *ep)
{
unsigned long end_time = jiffies + msecs_to_jiffies(1000);
int alive;
if (atomic_read(&ep->state) != EP_STATE_STOPPING)
return 0;
do {
alive = atomic_read(&ep->submitted_urbs);
if (!alive)
break;
schedule_timeout_uninterruptible(1);
} while (time_before(jiffies, end_time));
if (alive) usb_audio_err(ep->chip,
"timeout: still %d active urbs on EP #%x\n",
alive, ep->ep_num);
if (ep_state_update(ep, EP_STATE_STOPPING, EP_STATE_STOPPED)) { ep->sync_sink = NULL; snd_usb_endpoint_set_callback(ep, NULL, NULL, NULL);
}
return 0;
}
/* sync the pending stop operation;
* this function itself doesn't trigger the stop operation
*/
void snd_usb_endpoint_sync_pending_stop(struct snd_usb_endpoint *ep)
{
if (ep)
wait_clear_urbs(ep);
}
/*
* Stop active urbs
*
* This function moves the EP to STOPPING state if it's being RUNNING.
*/
static int stop_urbs(struct snd_usb_endpoint *ep, bool force, bool keep_pending)
{
unsigned int i;
unsigned long flags;
if (!force && atomic_read(&ep->running))
return -EBUSY;
if (!ep_state_update(ep, EP_STATE_RUNNING, EP_STATE_STOPPING)) return 0; spin_lock_irqsave(&ep->lock, flags);
INIT_LIST_HEAD(&ep->ready_playback_urbs);
ep->next_packet_head = 0;
ep->next_packet_queued = 0;
spin_unlock_irqrestore(&ep->lock, flags);
if (keep_pending)
return 0;
for (i = 0; i < ep->nurbs; i++) { if (test_bit(i, &ep->active_mask)) { if (!test_and_set_bit(i, &ep->unlink_mask)) { struct urb *u = ep->urb[i].urb;
usb_unlink_urb(u);
}
}
}
return 0;
}
/*
* release an endpoint's urbs
*/
static int release_urbs(struct snd_usb_endpoint *ep, bool force)
{
int i, err;
/* route incoming urbs to nirvana */
snd_usb_endpoint_set_callback(ep, NULL, NULL, NULL);
/* stop and unlink urbs */
err = stop_urbs(ep, force, false);
if (err)
return err;
wait_clear_urbs(ep);
for (i = 0; i < ep->nurbs; i++)
release_urb_ctx(&ep->urb[i]); usb_free_coherent(ep->chip->dev, SYNC_URBS * 4,
ep->syncbuf, ep->sync_dma);
ep->syncbuf = NULL;
ep->nurbs = 0;
return 0;
}
/*
* configure a data endpoint
*/
static int data_ep_set_params(struct snd_usb_endpoint *ep)
{
struct snd_usb_audio *chip = ep->chip;
unsigned int maxsize, minsize, packs_per_ms, max_packs_per_urb;
unsigned int max_packs_per_period, urbs_per_period, urb_packs;
unsigned int max_urbs, i;
const struct audioformat *fmt = ep->cur_audiofmt;
int frame_bits = ep->cur_frame_bytes * 8;
int tx_length_quirk = (has_tx_length_quirk(chip) &&
usb_pipeout(ep->pipe));
usb_audio_dbg(chip, "Setting params for data EP 0x%x, pipe 0x%x\n",
ep->ep_num, ep->pipe);
if (ep->cur_format == SNDRV_PCM_FORMAT_DSD_U16_LE && fmt->dsd_dop) {
/*
* When operating in DSD DOP mode, the size of a sample frame
* in hardware differs from the actual physical format width
* because we need to make room for the DOP markers.
*/
frame_bits += ep->cur_channels << 3;
}
ep->datainterval = fmt->datainterval;
ep->stride = frame_bits >> 3;
switch (ep->cur_format) {
case SNDRV_PCM_FORMAT_U8:
ep->silence_value = 0x80;
break;
case SNDRV_PCM_FORMAT_DSD_U8:
case SNDRV_PCM_FORMAT_DSD_U16_LE:
case SNDRV_PCM_FORMAT_DSD_U32_LE:
case SNDRV_PCM_FORMAT_DSD_U16_BE:
case SNDRV_PCM_FORMAT_DSD_U32_BE:
ep->silence_value = 0x69;
break;
default:
ep->silence_value = 0;
}
/* assume max. frequency is 50% higher than nominal */
ep->freqmax = ep->freqn + (ep->freqn >> 1);
/* Round up freqmax to nearest integer in order to calculate maximum
* packet size, which must represent a whole number of frames.
* This is accomplished by adding 0x0.ffff before converting the
* Q16.16 format into integer.
* In order to accurately calculate the maximum packet size when
* the data interval is more than 1 (i.e. ep->datainterval > 0),
* multiply by the data interval prior to rounding. For instance,
* a freqmax of 41 kHz will result in a max packet size of 6 (5.125)
* frames with a data interval of 1, but 11 (10.25) frames with a
* data interval of 2.
* (ep->freqmax << ep->datainterval overflows at 8.192 MHz for the
* maximum datainterval value of 3, at USB full speed, higher for
* USB high speed, noting that ep->freqmax is in units of
* frames per packet in Q16.16 format.)
*/
maxsize = (((ep->freqmax << ep->datainterval) + 0xffff) >> 16) *
(frame_bits >> 3);
if (tx_length_quirk)
maxsize += sizeof(__le32); /* Space for length descriptor */
/* but wMaxPacketSize might reduce this */
if (ep->maxpacksize && ep->maxpacksize < maxsize) {
/* whatever fits into a max. size packet */
unsigned int data_maxsize = maxsize = ep->maxpacksize;
if (tx_length_quirk)
/* Need to remove the length descriptor to calc freq */
data_maxsize -= sizeof(__le32);
ep->freqmax = (data_maxsize / (frame_bits >> 3))
<< (16 - ep->datainterval);
}
if (ep->fill_max)
ep->curpacksize = ep->maxpacksize;
else
ep->curpacksize = maxsize;
if (snd_usb_get_speed(chip->dev) != USB_SPEED_FULL) {
packs_per_ms = 8 >> ep->datainterval;
max_packs_per_urb = MAX_PACKS_HS;
} else {
packs_per_ms = 1;
max_packs_per_urb = MAX_PACKS;
}
if (ep->sync_source && !ep->implicit_fb_sync)
max_packs_per_urb = min(max_packs_per_urb,
1U << ep->sync_source->syncinterval);
max_packs_per_urb = max(1u, max_packs_per_urb >> ep->datainterval);
/*
* Capture endpoints need to use small URBs because there's no way
* to tell in advance where the next period will end, and we don't
* want the next URB to complete much after the period ends.
*
* Playback endpoints with implicit sync much use the same parameters
* as their corresponding capture endpoint.
*/
if (usb_pipein(ep->pipe) || ep->implicit_fb_sync) {
/* make capture URBs <= 1 ms and smaller than a period */
urb_packs = min(max_packs_per_urb, packs_per_ms);
while (urb_packs > 1 && urb_packs * maxsize >= ep->cur_period_bytes)
urb_packs >>= 1;
ep->nurbs = MAX_URBS;
/*
* Playback endpoints without implicit sync are adjusted so that
* a period fits as evenly as possible in the smallest number of
* URBs. The total number of URBs is adjusted to the size of the
* ALSA buffer, subject to the MAX_URBS and MAX_QUEUE limits.
*/
} else {
/* determine how small a packet can be */
minsize = (ep->freqn >> (16 - ep->datainterval)) *
(frame_bits >> 3);
/* with sync from device, assume it can be 12% lower */
if (ep->sync_source)
minsize -= minsize >> 3;
minsize = max(minsize, 1u);
/* how many packets will contain an entire ALSA period? */
max_packs_per_period = DIV_ROUND_UP(ep->cur_period_bytes, minsize);
/* how many URBs will contain a period? */
urbs_per_period = DIV_ROUND_UP(max_packs_per_period,
max_packs_per_urb);
/* how many packets are needed in each URB? */
urb_packs = DIV_ROUND_UP(max_packs_per_period, urbs_per_period);
/* limit the number of frames in a single URB */
ep->max_urb_frames = DIV_ROUND_UP(ep->cur_period_frames,
urbs_per_period);
/* try to use enough URBs to contain an entire ALSA buffer */
max_urbs = min((unsigned) MAX_URBS,
MAX_QUEUE * packs_per_ms / urb_packs);
ep->nurbs = min(max_urbs, urbs_per_period * ep->cur_buffer_periods);
}
/* allocate and initialize data urbs */
for (i = 0; i < ep->nurbs; i++) {
struct snd_urb_ctx *u = &ep->urb[i];
u->index = i;
u->ep = ep;
u->packets = urb_packs;
u->buffer_size = maxsize * u->packets;
if (fmt->fmt_type == UAC_FORMAT_TYPE_II)
u->packets++; /* for transfer delimiter */
u->urb = usb_alloc_urb(u->packets, GFP_KERNEL);
if (!u->urb)
goto out_of_memory;
u->urb->transfer_buffer =
usb_alloc_coherent(chip->dev, u->buffer_size,
GFP_KERNEL, &u->urb->transfer_dma);
if (!u->urb->transfer_buffer)
goto out_of_memory;
u->urb->pipe = ep->pipe;
u->urb->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
u->urb->interval = 1 << ep->datainterval;
u->urb->context = u;
u->urb->complete = snd_complete_urb;
INIT_LIST_HEAD(&u->ready_list);
}
return 0;
out_of_memory:
release_urbs(ep, false);
return -ENOMEM;
}
/*
* configure a sync endpoint
*/
static int sync_ep_set_params(struct snd_usb_endpoint *ep)
{
struct snd_usb_audio *chip = ep->chip;
int i;
usb_audio_dbg(chip, "Setting params for sync EP 0x%x, pipe 0x%x\n",
ep->ep_num, ep->pipe);
ep->syncbuf = usb_alloc_coherent(chip->dev, SYNC_URBS * 4,
GFP_KERNEL, &ep->sync_dma);
if (!ep->syncbuf)
return -ENOMEM;
ep->nurbs = SYNC_URBS;
for (i = 0; i < SYNC_URBS; i++) {
struct snd_urb_ctx *u = &ep->urb[i];
u->index = i;
u->ep = ep;
u->packets = 1;
u->urb = usb_alloc_urb(1, GFP_KERNEL);
if (!u->urb)
goto out_of_memory;
u->urb->transfer_buffer = ep->syncbuf + i * 4;
u->urb->transfer_dma = ep->sync_dma + i * 4;
u->urb->transfer_buffer_length = 4;
u->urb->pipe = ep->pipe;
u->urb->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
u->urb->number_of_packets = 1;
u->urb->interval = 1 << ep->syncinterval;
u->urb->context = u;
u->urb->complete = snd_complete_urb;
}
return 0;
out_of_memory:
release_urbs(ep, false);
return -ENOMEM;
}
/* update the rate of the referred clock; return the actual rate */
static int update_clock_ref_rate(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep)
{
struct snd_usb_clock_ref *clock = ep->clock_ref;
int rate = ep->cur_rate;
if (!clock || clock->rate == rate)
return rate;
if (clock->rate) {
if (atomic_read(&clock->locked))
return clock->rate;
if (clock->rate != rate) {
usb_audio_err(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n",
clock->rate, rate, ep->ep_num);
return clock->rate;
}
}
clock->rate = rate;
clock->need_setup = true;
return rate;
}
/*
* snd_usb_endpoint_set_params: configure an snd_usb_endpoint
*
* It's called either from hw_params callback.
* Determine the number of URBs to be used on this endpoint.
* An endpoint must be configured before it can be started.
* An endpoint that is already running can not be reconfigured.
*/
int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep)
{
const struct audioformat *fmt = ep->cur_audiofmt;
int err = 0;
mutex_lock(&chip->mutex);
if (!ep->need_setup)
goto unlock;
/* release old buffers, if any */
err = release_urbs(ep, false);
if (err < 0)
goto unlock;
ep->datainterval = fmt->datainterval;
ep->maxpacksize = fmt->maxpacksize;
ep->fill_max = !!(fmt->attributes & UAC_EP_CS_ATTR_FILL_MAX);
if (snd_usb_get_speed(chip->dev) == USB_SPEED_FULL) {
ep->freqn = get_usb_full_speed_rate(ep->cur_rate);
ep->pps = 1000 >> ep->datainterval;
} else {
ep->freqn = get_usb_high_speed_rate(ep->cur_rate);
ep->pps = 8000 >> ep->datainterval;
}
ep->sample_rem = ep->cur_rate % ep->pps;
ep->packsize[0] = ep->cur_rate / ep->pps;
ep->packsize[1] = (ep->cur_rate + (ep->pps - 1)) / ep->pps;
/* calculate the frequency in 16.16 format */
ep->freqm = ep->freqn;
ep->freqshift = INT_MIN;
ep->phase = 0;
switch (ep->type) {
case SND_USB_ENDPOINT_TYPE_DATA:
err = data_ep_set_params(ep);
break;
case SND_USB_ENDPOINT_TYPE_SYNC:
err = sync_ep_set_params(ep);
break;
default:
err = -EINVAL;
}
usb_audio_dbg(chip, "Set up %d URBS, ret=%d\n", ep->nurbs, err);
if (err < 0)
goto unlock;
/* some unit conversions in runtime */
ep->maxframesize = ep->maxpacksize / ep->cur_frame_bytes;
ep->curframesize = ep->curpacksize / ep->cur_frame_bytes;
err = update_clock_ref_rate(chip, ep);
if (err >= 0) {
ep->need_setup = false;
err = 0;
}
unlock:
mutex_unlock(&chip->mutex);
return err;
}
static int init_sample_rate(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep)
{
struct snd_usb_clock_ref *clock = ep->clock_ref;
int rate, err;
rate = update_clock_ref_rate(chip, ep);
if (rate < 0)
return rate;
if (clock && !clock->need_setup)
return 0;
if (!ep->fixed_rate) {
err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, rate);
if (err < 0) {
if (clock)
clock->rate = 0; /* reset rate */
return err;
}
}
if (clock)
clock->need_setup = false;
return 0;
}
/*
* snd_usb_endpoint_prepare: Prepare the endpoint
*
* This function sets up the EP to be fully usable state.
* It's called either from prepare callback.
* The function checks need_setup flag, and performs nothing unless needed,
* so it's safe to call this multiple times.
*
* This returns zero if unchanged, 1 if the configuration has changed,
* or a negative error code.
*/
int snd_usb_endpoint_prepare(struct snd_usb_audio *chip,
struct snd_usb_endpoint *ep)
{
bool iface_first;
int err = 0;
mutex_lock(&chip->mutex);
if (WARN_ON(!ep->iface_ref))
goto unlock;
if (!ep->need_prepare)
goto unlock;
/* If the interface has been already set up, just set EP parameters */
if (!ep->iface_ref->need_setup) {
/* sample rate setup of UAC1 is per endpoint, and we need
* to update at each EP configuration
*/
if (ep->cur_audiofmt->protocol == UAC_VERSION_1) {
err = init_sample_rate(chip, ep);
if (err < 0)
goto unlock;
}
goto done;
}
/* Need to deselect altsetting at first */
endpoint_set_interface(chip, ep, false);
/* Some UAC1 devices (e.g. Yamaha THR10) need the host interface
* to be set up before parameter setups
*/
iface_first = ep->cur_audiofmt->protocol == UAC_VERSION_1;
/* Workaround for devices that require the interface setup at first like UAC1 */
if (chip->quirk_flags & QUIRK_FLAG_SET_IFACE_FIRST)
iface_first = true;
if (iface_first) {
err = endpoint_set_interface(chip, ep, true);
if (err < 0)
goto unlock;
}
err = snd_usb_init_pitch(chip, ep->cur_audiofmt);
if (err < 0)
goto unlock;
err = init_sample_rate(chip, ep);
if (err < 0)
goto unlock;
err = snd_usb_select_mode_quirk(chip, ep->cur_audiofmt);
if (err < 0)
goto unlock;
/* for UAC2/3, enable the interface altset here at last */
if (!iface_first) {
err = endpoint_set_interface(chip, ep, true);
if (err < 0)
goto unlock;
}
ep->iface_ref->need_setup = false;
done:
ep->need_prepare = false;
err = 1;
unlock:
mutex_unlock(&chip->mutex);
return err;
}
/* get the current rate set to the given clock by any endpoint */
int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock)
{
struct snd_usb_clock_ref *ref;
int rate = 0;
if (!clock)
return 0;
mutex_lock(&chip->mutex);
list_for_each_entry(ref, &chip->clock_ref_list, list) {
if (ref->clock == clock) {
rate = ref->rate;
break;
}
}
mutex_unlock(&chip->mutex);
return rate;
}
/**
* snd_usb_endpoint_start: start an snd_usb_endpoint
*
* @ep: the endpoint to start
*
* A call to this function will increment the running count of the endpoint.
* In case it is not already running, the URBs for this endpoint will be
* submitted. Otherwise, this function does nothing.
*
* Must be balanced to calls of snd_usb_endpoint_stop().
*
* Returns an error if the URB submission failed, 0 in all other cases.
*/
int snd_usb_endpoint_start(struct snd_usb_endpoint *ep)
{
bool is_playback = usb_pipeout(ep->pipe);
int err;
unsigned int i;
if (atomic_read(&ep->chip->shutdown))
return -EBADFD;
if (ep->sync_source)
WRITE_ONCE(ep->sync_source->sync_sink, ep);
usb_audio_dbg(ep->chip, "Starting %s EP 0x%x (running %d)\n",
ep_type_name(ep->type), ep->ep_num,
atomic_read(&ep->running));
/* already running? */
if (atomic_inc_return(&ep->running) != 1)
return 0;
if (ep->clock_ref)
atomic_inc(&ep->clock_ref->locked);
ep->active_mask = 0;
ep->unlink_mask = 0;
ep->phase = 0;
ep->sample_accum = 0;
snd_usb_endpoint_start_quirk(ep);
/*
* If this endpoint has a data endpoint as implicit feedback source,
* don't start the urbs here. Instead, mark them all as available,
* wait for the record urbs to return and queue the playback urbs
* from that context.
*/
if (!ep_state_update(ep, EP_STATE_STOPPED, EP_STATE_RUNNING))
goto __error;
if (snd_usb_endpoint_implicit_feedback_sink(ep) &&
!(ep->chip->quirk_flags & QUIRK_FLAG_PLAYBACK_FIRST)) {
usb_audio_dbg(ep->chip, "No URB submission due to implicit fb sync\n");
i = 0;
goto fill_rest;
}
for (i = 0; i < ep->nurbs; i++) {
struct urb *urb = ep->urb[i].urb;
if (snd_BUG_ON(!urb))
goto __error;
if (is_playback)
err = prepare_outbound_urb(ep, urb->context, true);
else
err = prepare_inbound_urb(ep, urb->context);
if (err < 0) {
/* stop filling at applptr */
if (err == -EAGAIN)
break;
usb_audio_dbg(ep->chip,
"EP 0x%x: failed to prepare urb: %d\n",
ep->ep_num, err);
goto __error;
}
if (!atomic_read(&ep->chip->shutdown))
err = usb_submit_urb(urb, GFP_ATOMIC);
else
err = -ENODEV;
if (err < 0) {
if (!atomic_read(&ep->chip->shutdown))
usb_audio_err(ep->chip,
"cannot submit urb %d, error %d: %s\n",
i, err, usb_error_string(err));
goto __error;
}
set_bit(i, &ep->active_mask);
atomic_inc(&ep->submitted_urbs);
}
if (!i) {
usb_audio_dbg(ep->chip, "XRUN at starting EP 0x%x\n",
ep->ep_num);
goto __error;
}
usb_audio_dbg(ep->chip, "%d URBs submitted for EP 0x%x\n",
i, ep->ep_num);
fill_rest:
/* put the remaining URBs to ready list */
if (is_playback) {
for (; i < ep->nurbs; i++)
push_back_to_ready_list(ep, ep->urb + i);
}
return 0;
__error:
snd_usb_endpoint_stop(ep, false);
return -EPIPE;
}
/**
* snd_usb_endpoint_stop: stop an snd_usb_endpoint
*
* @ep: the endpoint to stop (may be NULL)
* @keep_pending: keep in-flight URBs
*
* A call to this function will decrement the running count of the endpoint.
* In case the last user has requested the endpoint stop, the URBs will
* actually be deactivated.
*
* Must be balanced to calls of snd_usb_endpoint_start().
*
* The caller needs to synchronize the pending stop operation via
* snd_usb_endpoint_sync_pending_stop().
*/
void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending)
{
if (!ep)
return;
usb_audio_dbg(ep->chip, "Stopping %s EP 0x%x (running %d)\n",
ep_type_name(ep->type), ep->ep_num,
atomic_read(&ep->running));
if (snd_BUG_ON(!atomic_read(&ep->running)))
return;
if (!atomic_dec_return(&ep->running)) {
if (ep->sync_source)
WRITE_ONCE(ep->sync_source->sync_sink, NULL);
stop_urbs(ep, false, keep_pending);
if (ep->clock_ref)
atomic_dec(&ep->clock_ref->locked);
if (ep->chip->quirk_flags & QUIRK_FLAG_FORCE_IFACE_RESET &&
usb_pipeout(ep->pipe)) {
ep->need_prepare = true;
if (ep->iface_ref)
ep->iface_ref->need_setup = true;
}
}
}
/**
* snd_usb_endpoint_release: Tear down an snd_usb_endpoint
*
* @ep: the endpoint to release
*
* This function does not care for the endpoint's running count but will tear
* down all the streaming URBs immediately.
*/
void snd_usb_endpoint_release(struct snd_usb_endpoint *ep)
{
release_urbs(ep, true);
}
/**
* snd_usb_endpoint_free_all: Free the resources of an snd_usb_endpoint
* @chip: The chip
*
* This free all endpoints and those resources
*/
void snd_usb_endpoint_free_all(struct snd_usb_audio *chip)
{
struct snd_usb_endpoint *ep, *en;
struct snd_usb_iface_ref *ip, *in;
struct snd_usb_clock_ref *cp, *cn;
list_for_each_entry_safe(ep, en, &chip->ep_list, list) kfree(ep); list_for_each_entry_safe(ip, in, &chip->iface_ref_list, list) kfree(ip); list_for_each_entry_safe(cp, cn, &chip->clock_ref_list, list) kfree(cp);
}
/*
* snd_usb_handle_sync_urb: parse an USB sync packet
*
* @ep: the endpoint to handle the packet
* @sender: the sending endpoint
* @urb: the received packet
*
* This function is called from the context of an endpoint that received
* the packet and is used to let another endpoint object handle the payload.
*/
static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep,
struct snd_usb_endpoint *sender,
const struct urb *urb)
{
int shift;
unsigned int f;
unsigned long flags;
snd_BUG_ON(ep == sender);
/*
* In case the endpoint is operating in implicit feedback mode, prepare
* a new outbound URB that has the same layout as the received packet
* and add it to the list of pending urbs. queue_pending_output_urbs()
* will take care of them later.
*/
if (snd_usb_endpoint_implicit_feedback_sink(ep) &&
atomic_read(&ep->running)) {
/* implicit feedback case */
int i, bytes = 0;
struct snd_urb_ctx *in_ctx;
struct snd_usb_packet_info *out_packet;
in_ctx = urb->context;
/* Count overall packet size */
for (i = 0; i < in_ctx->packets; i++)
if (urb->iso_frame_desc[i].status == 0)
bytes += urb->iso_frame_desc[i].actual_length;
/*
* skip empty packets. At least M-Audio's Fast Track Ultra stops
* streaming once it received a 0-byte OUT URB
*/
if (bytes == 0)
return;
spin_lock_irqsave(&ep->lock, flags);
if (ep->next_packet_queued >= ARRAY_SIZE(ep->next_packet)) {
spin_unlock_irqrestore(&ep->lock, flags);
usb_audio_err(ep->chip,
"next package FIFO overflow EP 0x%x\n",
ep->ep_num);
notify_xrun(ep);
return;
}
out_packet = next_packet_fifo_enqueue(ep);
/*
* Iterate through the inbound packet and prepare the lengths
* for the output packet. The OUT packet we are about to send
* will have the same amount of payload bytes per stride as the
* IN packet we just received. Since the actual size is scaled
* by the stride, use the sender stride to calculate the length
* in case the number of channels differ between the implicitly
* fed-back endpoint and the synchronizing endpoint.
*/
out_packet->packets = in_ctx->packets;
for (i = 0; i < in_ctx->packets; i++) {
if (urb->iso_frame_desc[i].status == 0)
out_packet->packet_size[i] =
urb->iso_frame_desc[i].actual_length / sender->stride;
else
out_packet->packet_size[i] = 0;
}
spin_unlock_irqrestore(&ep->lock, flags);
snd_usb_queue_pending_output_urbs(ep, false);
return;
}
/*
* process after playback sync complete
*
* Full speed devices report feedback values in 10.14 format as samples
* per frame, high speed devices in 16.16 format as samples per
* microframe.
*
* Because the Audio Class 1 spec was written before USB 2.0, many high
* speed devices use a wrong interpretation, some others use an
* entirely different format.
*
* Therefore, we cannot predict what format any particular device uses
* and must detect it automatically.
*/
if (urb->iso_frame_desc[0].status != 0 ||
urb->iso_frame_desc[0].actual_length < 3)
return;
f = le32_to_cpup(urb->transfer_buffer);
if (urb->iso_frame_desc[0].actual_length == 3)
f &= 0x00ffffff;
else
f &= 0x0fffffff;
if (f == 0)
return;
if (unlikely(sender->tenor_fb_quirk)) {
/*
* Devices based on Tenor 8802 chipsets (TEAC UD-H01
* and others) sometimes change the feedback value
* by +/- 0x1.0000.
*/
if (f < ep->freqn - 0x8000)
f += 0xf000;
else if (f > ep->freqn + 0x8000)
f -= 0xf000;
} else if (unlikely(ep->freqshift == INT_MIN)) {
/*
* The first time we see a feedback value, determine its format
* by shifting it left or right until it matches the nominal
* frequency value. This assumes that the feedback does not
* differ from the nominal value more than +50% or -25%.
*/
shift = 0;
while (f < ep->freqn - ep->freqn / 4) {
f <<= 1;
shift++;
}
while (f > ep->freqn + ep->freqn / 2) {
f >>= 1;
shift--;
}
ep->freqshift = shift;
} else if (ep->freqshift >= 0)
f <<= ep->freqshift;
else
f >>= -ep->freqshift;
if (likely(f >= ep->freqn - ep->freqn / 8 && f <= ep->freqmax)) {
/*
* If the frequency looks valid, set it.
* This value is referred to in prepare_playback_urb().
*/
spin_lock_irqsave(&ep->lock, flags);
ep->freqm = f;
spin_unlock_irqrestore(&ep->lock, flags);
} else {
/*
* Out of range; maybe the shift value is wrong.
* Reset it so that we autodetect again the next time.
*/
ep->freqshift = INT_MIN;
}
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_FIND_H_
#define __LINUX_FIND_H_
#ifndef __LINUX_BITMAP_H
#error only <linux/bitmap.h> can be included directly
#endif
#include <linux/bitops.h>
unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits,
unsigned long start);
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
unsigned long start);
extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n);
unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n);
unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n);
unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
const unsigned long *addr3, unsigned long size,
unsigned long n);
extern unsigned long _find_first_and_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size);
extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size);
extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size);
#ifdef __BIG_ENDIAN
unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size);
unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned
long size, unsigned long offset);
unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
long size, unsigned long offset);
#endif
#ifndef find_next_bit
/**
* find_next_bit - find the next set bit in a memory region
* @addr: The address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val;
if (unlikely(offset >= size))
return size;
val = *addr & GENMASK(size - 1, offset); return val ? __ffs(val) : size;
}
return _find_next_bit(addr, size, offset);
}
#endif
#ifndef find_next_and_bit
/**
* find_next_and_bit - find the next set bit in both memory regions
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_and_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size,
unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val;
if (unlikely(offset >= size))
return size;
val = *addr1 & *addr2 & GENMASK(size - 1, offset); return val ? __ffs(val) : size;
}
return _find_next_and_bit(addr1, addr2, size, offset);
}
#endif
#ifndef find_next_andnot_bit
/**
* find_next_andnot_bit - find the next set bit in *addr1 excluding all the bits
* in *addr2
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_andnot_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size,
unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val;
if (unlikely(offset >= size))
return size;
val = *addr1 & ~*addr2 & GENMASK(size - 1, offset);
return val ? __ffs(val) : size;
}
return _find_next_andnot_bit(addr1, addr2, size, offset);
}
#endif
#ifndef find_next_or_bit
/**
* find_next_or_bit - find the next set bit in either memory regions
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_or_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size,
unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val;
if (unlikely(offset >= size))
return size;
val = (*addr1 | *addr2) & GENMASK(size - 1, offset); return val ? __ffs(val) : size;
}
return _find_next_or_bit(addr1, addr2, size, offset);
}
#endif
#ifndef find_next_zero_bit
/**
* find_next_zero_bit - find the next cleared bit in a memory region
* @addr: The address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number of the next zero bit
* If no bits are zero, returns @size.
*/
static inline
unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val;
if (unlikely(offset >= size))
return size;
val = *addr | ~GENMASK(size - 1, offset); return val == ~0UL ? size : ffz(val);
}
return _find_next_zero_bit(addr, size, offset);
}
#endif
#ifndef find_first_bit
/**
* find_first_bit - find the first set bit in a memory region
* @addr: The address to start the search at
* @size: The maximum number of bits to search
*
* Returns the bit number of the first set bit.
* If no bits are set, returns @size.
*/
static inline
unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = *addr & GENMASK(size - 1, 0); return val ? __ffs(val) : size;
}
return _find_first_bit(addr, size);
}
#endif
/**
* find_nth_bit - find N'th set bit in a memory region
* @addr: The address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* The following is semantically equivalent:
* idx = find_nth_bit(addr, size, 0);
* idx = find_first_bit(addr, size);
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static inline
unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_bit(addr, size, n);
}
/**
* find_nth_and_bit - find N'th set bit in 2 memory regions
* @addr1: The 1st address to start the search at
* @addr2: The 2nd address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static inline
unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_and_bit(addr1, addr2, size, n);
}
/**
* find_nth_andnot_bit - find N'th set bit in 2 memory regions,
* flipping bits in 2nd region
* @addr1: The 1st address to start the search at
* @addr2: The 2nd address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static inline
unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_andnot_bit(addr1, addr2, size, n);
}
/**
* find_nth_and_andnot_bit - find N'th set bit in 2 memory regions,
* excluding those set in 3rd region
* @addr1: The 1st address to start the search at
* @addr2: The 2nd address to start the search at
* @addr3: The 3rd address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static __always_inline
unsigned long find_nth_and_andnot_bit(const unsigned long *addr1,
const unsigned long *addr2,
const unsigned long *addr3,
unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr1 & *addr2 & (~*addr3) & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_and_andnot_bit(addr1, addr2, addr3, size, n);
}
#ifndef find_first_and_bit
/**
* find_first_and_bit - find the first set bit in both memory regions
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
*/
static inline
unsigned long find_first_and_bit(const unsigned long *addr1,
const unsigned long *addr2,
unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0); return val ? __ffs(val) : size;
}
return _find_first_and_bit(addr1, addr2, size);
}
#endif
#ifndef find_first_zero_bit
/**
* find_first_zero_bit - find the first cleared bit in a memory region
* @addr: The address to start the search at
* @size: The maximum number of bits to search
*
* Returns the bit number of the first cleared bit.
* If no bits are zero, returns @size.
*/
static inline
unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = *addr | ~GENMASK(size - 1, 0);
return val == ~0UL ? size : ffz(val);
}
return _find_first_zero_bit(addr, size);
}
#endif
#ifndef find_last_bit
/**
* find_last_bit - find the last set bit in a memory region
* @addr: The address to start the search at
* @size: The number of bits to search
*
* Returns the bit number of the last set bit, or size.
*/
static inline
unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = *addr & GENMASK(size - 1, 0);
return val ? __fls(val) : size;
}
return _find_last_bit(addr, size);
}
#endif
/**
* find_next_and_bit_wrap - find the next set bit in both memory regions
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit, or first set bit up to @offset
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
const unsigned long *addr2,
unsigned long size, unsigned long offset)
{
unsigned long bit = find_next_and_bit(addr1, addr2, size, offset);
if (bit < size || offset == 0)
return bit;
bit = find_first_and_bit(addr1, addr2, offset);
return bit < offset ? bit : size;
}
/**
* find_next_bit_wrap - find the next set bit in a memory region
* @addr: The address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit, or first set bit up to @offset
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_bit_wrap(const unsigned long *addr,
unsigned long size, unsigned long offset)
{
unsigned long bit = find_next_bit(addr, size, offset);
if (bit < size || offset == 0)
return bit;
bit = find_first_bit(addr, offset);
return bit < offset ? bit : size;
}
/*
* Helper for for_each_set_bit_wrap(). Make sure you're doing right thing
* before using it alone.
*/
static inline
unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,
unsigned long start, unsigned long n)
{
unsigned long bit;
/* If not wrapped around */
if (n > start) {
/* and have a bit, just return it. */
bit = find_next_bit(bitmap, size, n);
if (bit < size)
return bit;
/* Otherwise, wrap around and ... */
n = 0;
}
/* Search the other part. */
bit = find_next_bit(bitmap, start, n);
return bit < start ? bit : size;
}
/**
* find_next_clump8 - find next 8-bit clump with set bits in a memory region
* @clump: location to store copy of found clump
* @addr: address to base the search on
* @size: bitmap size in number of bits
* @offset: bit offset at which to start searching
*
* Returns the bit offset for the next set clump; the found clump value is
* copied to the location pointed by @clump. If no bits are set, returns @size.
*/
extern unsigned long find_next_clump8(unsigned long *clump,
const unsigned long *addr,
unsigned long size, unsigned long offset);
#define find_first_clump8(clump, bits, size) \
find_next_clump8((clump), (bits), (size), 0)
#if defined(__LITTLE_ENDIAN)
static inline unsigned long find_next_zero_bit_le(const void *addr,
unsigned long size, unsigned long offset)
{
return find_next_zero_bit(addr, size, offset);
}
static inline unsigned long find_next_bit_le(const void *addr,
unsigned long size, unsigned long offset)
{
return find_next_bit(addr, size, offset);
}
static inline unsigned long find_first_zero_bit_le(const void *addr,
unsigned long size)
{
return find_first_zero_bit(addr, size);
}
#elif defined(__BIG_ENDIAN)
#ifndef find_next_zero_bit_le
static inline
unsigned long find_next_zero_bit_le(const void *addr, unsigned
long size, unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val = *(const unsigned long *)addr;
if (unlikely(offset >= size))
return size;
val = swab(val) | ~GENMASK(size - 1, offset);
return val == ~0UL ? size : ffz(val);
}
return _find_next_zero_bit_le(addr, size, offset);
}
#endif
#ifndef find_first_zero_bit_le
static inline
unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0);
return val == ~0UL ? size : ffz(val);
}
return _find_first_zero_bit_le(addr, size);
}
#endif
#ifndef find_next_bit_le
static inline
unsigned long find_next_bit_le(const void *addr, unsigned
long size, unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val = *(const unsigned long *)addr;
if (unlikely(offset >= size))
return size;
val = swab(val) & GENMASK(size - 1, offset);
return val ? __ffs(val) : size;
}
return _find_next_bit_le(addr, size, offset);
}
#endif
#else
#error "Please fix <asm/byteorder.h>"
#endif
#define for_each_set_bit(bit, addr, size) \
for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
#define for_each_and_bit(bit, addr1, addr2, size) \
for ((bit) = 0; \
(bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
(bit)++)
#define for_each_andnot_bit(bit, addr1, addr2, size) \
for ((bit) = 0; \
(bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
(bit)++)
#define for_each_or_bit(bit, addr1, addr2, size) \
for ((bit) = 0; \
(bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
(bit)++)
/* same as for_each_set_bit() but use bit as value to start with */
#define for_each_set_bit_from(bit, addr, size) \
for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
#define for_each_clear_bit(bit, addr, size) \
for ((bit) = 0; \
(bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); \
(bit)++)
/* same as for_each_clear_bit() but use bit as value to start with */
#define for_each_clear_bit_from(bit, addr, size) \
for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
/**
* for_each_set_bitrange - iterate over all set bit ranges [b; e)
* @b: bit offset of start of current bitrange (first set bit)
* @e: bit offset of end of current bitrange (first unset bit)
* @addr: bitmap address to base the search on
* @size: bitmap size in number of bits
*/
#define for_each_set_bitrange(b, e, addr, size) \
for ((b) = 0; \
(b) = find_next_bit((addr), (size), b), \
(e) = find_next_zero_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = (e) + 1)
/**
* for_each_set_bitrange_from - iterate over all set bit ranges [b; e)
* @b: bit offset of start of current bitrange (first set bit); must be initialized
* @e: bit offset of end of current bitrange (first unset bit)
* @addr: bitmap address to base the search on
* @size: bitmap size in number of bits
*/
#define for_each_set_bitrange_from(b, e, addr, size) \
for (; \
(b) = find_next_bit((addr), (size), (b)), \
(e) = find_next_zero_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = (e) + 1)
/**
* for_each_clear_bitrange - iterate over all unset bit ranges [b; e)
* @b: bit offset of start of current bitrange (first unset bit)
* @e: bit offset of end of current bitrange (first set bit)
* @addr: bitmap address to base the search on
* @size: bitmap size in number of bits
*/
#define for_each_clear_bitrange(b, e, addr, size) \
for ((b) = 0; \
(b) = find_next_zero_bit((addr), (size), (b)), \
(e) = find_next_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = (e) + 1)
/**
* for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e)
* @b: bit offset of start of current bitrange (first set bit); must be initialized
* @e: bit offset of end of current bitrange (first unset bit)
* @addr: bitmap address to base the search on
* @size: bitmap size in number of bits
*/
#define for_each_clear_bitrange_from(b, e, addr, size) \
for (; \
(b) = find_next_zero_bit((addr), (size), (b)), \
(e) = find_next_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = (e) + 1)
/**
* for_each_set_bit_wrap - iterate over all set bits starting from @start, and
* wrapping around the end of bitmap.
* @bit: offset for current iteration
* @addr: bitmap address to base the search on
* @size: bitmap size in number of bits
* @start: Starting bit for bitmap traversing, wrapping around the bitmap end
*/
#define for_each_set_bit_wrap(bit, addr, size, start) \
for ((bit) = find_next_bit_wrap((addr), (size), (start)); \
(bit) < (size); \
(bit) = __for_each_wrap((addr), (size), (start), (bit) + 1))
/**
* for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits
* @start: bit offset to start search and to store the current iteration offset
* @clump: location to store copy of current 8-bit clump
* @bits: bitmap address to base the search on
* @size: bitmap size in number of bits
*/
#define for_each_set_clump8(start, clump, bits, size) \
for ((start) = find_first_clump8(&(clump), (bits), (size)); \
(start) < (size); \
(start) = find_next_clump8(&(clump), (bits), (size), (start) + 8))
#endif /*__LINUX_FIND_H_ */
// SPDX-License-Identifier: GPL-2.0
/*
* USB hub driver.
*
* (C) Copyright 1999 Linus Torvalds
* (C) Copyright 1999 Johannes Erdfelt
* (C) Copyright 1999 Gregory P. Smith
* (C) Copyright 2001 Brad Hards (bhards@bigpond.net.au)
*
* Released under the GPLv2 only.
*/
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/completion.h>
#include <linux/sched/mm.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/kcov.h>
#include <linux/ioctl.h>
#include <linux/usb.h>
#include <linux/usbdevice_fs.h>
#include <linux/usb/hcd.h>
#include <linux/usb/onboard_dev.h>
#include <linux/usb/otg.h>
#include <linux/usb/quirks.h>
#include <linux/workqueue.h>
#include <linux/mutex.h>
#include <linux/random.h>
#include <linux/pm_qos.h>
#include <linux/kobject.h>
#include <linux/bitfield.h>
#include <linux/uaccess.h>
#include <asm/byteorder.h>
#include "hub.h"
#include "phy.h"
#include "otg_productlist.h"
#define USB_VENDOR_GENESYS_LOGIC 0x05e3
#define USB_VENDOR_SMSC 0x0424
#define USB_PRODUCT_USB5534B 0x5534
#define USB_VENDOR_CYPRESS 0x04b4
#define USB_PRODUCT_CY7C65632 0x6570
#define USB_VENDOR_TEXAS_INSTRUMENTS 0x0451
#define USB_PRODUCT_TUSB8041_USB3 0x8140
#define USB_PRODUCT_TUSB8041_USB2 0x8142
#define USB_VENDOR_MICROCHIP 0x0424
#define USB_PRODUCT_USB4913 0x4913
#define USB_PRODUCT_USB4914 0x4914
#define USB_PRODUCT_USB4915 0x4915
#define HUB_QUIRK_CHECK_PORT_AUTOSUSPEND BIT(0)
#define HUB_QUIRK_DISABLE_AUTOSUSPEND BIT(1)
#define HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL BIT(2)
#define USB_TP_TRANSMISSION_DELAY 40 /* ns */
#define USB_TP_TRANSMISSION_DELAY_MAX 65535 /* ns */
#define USB_PING_RESPONSE_TIME 400 /* ns */
#define USB_REDUCE_FRAME_INTR_BINTERVAL 9
/*
* The SET_ADDRESS request timeout will be 500 ms when
* USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT quirk flag is set.
*/
#define USB_SHORT_SET_ADDRESS_REQ_TIMEOUT 500 /* ms */
/* Protect struct usb_device->state and ->children members
* Note: Both are also protected by ->dev.sem, except that ->state can
* change to USB_STATE_NOTATTACHED even when the semaphore isn't held. */
static DEFINE_SPINLOCK(device_state_lock);
/* workqueue to process hub events */
static struct workqueue_struct *hub_wq;
static void hub_event(struct work_struct *work);
/* synchronize hub-port add/remove and peering operations */
DEFINE_MUTEX(usb_port_peer_mutex);
/* cycle leds on hubs that aren't blinking for attention */
static bool blinkenlights;
module_param(blinkenlights, bool, S_IRUGO);
MODULE_PARM_DESC(blinkenlights, "true to cycle leds on hubs");
/*
* Device SATA8000 FW1.0 from DATAST0R Technology Corp requires about
* 10 seconds to send reply for the initial 64-byte descriptor request.
*/
/* define initial 64-byte descriptor request timeout in milliseconds */
static int initial_descriptor_timeout = USB_CTRL_GET_TIMEOUT;
module_param(initial_descriptor_timeout, int, S_IRUGO|S_IWUSR);
MODULE_PARM_DESC(initial_descriptor_timeout,
"initial 64-byte descriptor request timeout in milliseconds "
"(default 5000 - 5.0 seconds)");
/*
* As of 2.6.10 we introduce a new USB device initialization scheme which
* closely resembles the way Windows works. Hopefully it will be compatible
* with a wider range of devices than the old scheme. However some previously
* working devices may start giving rise to "device not accepting address"
* errors; if that happens the user can try the old scheme by adjusting the
* following module parameters.
*
* For maximum flexibility there are two boolean parameters to control the
* hub driver's behavior. On the first initialization attempt, if the
* "old_scheme_first" parameter is set then the old scheme will be used,
* otherwise the new scheme is used. If that fails and "use_both_schemes"
* is set, then the driver will make another attempt, using the other scheme.
*/
static bool old_scheme_first;
module_param(old_scheme_first, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(old_scheme_first,
"start with the old device initialization scheme");
static bool use_both_schemes = true;
module_param(use_both_schemes, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(use_both_schemes,
"try the other device initialization scheme if the "
"first one fails");
/* Mutual exclusion for EHCI CF initialization. This interferes with
* port reset on some companion controllers.
*/
DECLARE_RWSEM(ehci_cf_port_reset_rwsem);
EXPORT_SYMBOL_GPL(ehci_cf_port_reset_rwsem);
#define HUB_DEBOUNCE_TIMEOUT 2000
#define HUB_DEBOUNCE_STEP 25
#define HUB_DEBOUNCE_STABLE 100
static int usb_reset_and_verify_device(struct usb_device *udev);
static int hub_port_disable(struct usb_hub *hub, int port1, int set_state);
static bool hub_port_warm_reset_required(struct usb_hub *hub, int port1,
u16 portstatus);
static inline char *portspeed(struct usb_hub *hub, int portstatus)
{
if (hub_is_superspeedplus(hub->hdev))
return "10.0 Gb/s";
if (hub_is_superspeed(hub->hdev))
return "5.0 Gb/s";
if (portstatus & USB_PORT_STAT_HIGH_SPEED)
return "480 Mb/s";
else if (portstatus & USB_PORT_STAT_LOW_SPEED)
return "1.5 Mb/s";
else
return "12 Mb/s";
}
/* Note that hdev or one of its children must be locked! */
struct usb_hub *usb_hub_to_struct_hub(struct usb_device *hdev)
{
if (!hdev || !hdev->actconfig || !hdev->maxchild)
return NULL;
return usb_get_intfdata(hdev->actconfig->interface[0]);}
int usb_device_supports_lpm(struct usb_device *udev)
{
/* Some devices have trouble with LPM */
if (udev->quirks & USB_QUIRK_NO_LPM)
return 0;
/* Skip if the device BOS descriptor couldn't be read */
if (!udev->bos)
return 0;
/* USB 2.1 (and greater) devices indicate LPM support through
* their USB 2.0 Extended Capabilities BOS descriptor.
*/
if (udev->speed == USB_SPEED_HIGH || udev->speed == USB_SPEED_FULL) { if (udev->bos->ext_cap &&
(USB_LPM_SUPPORT &
le32_to_cpu(udev->bos->ext_cap->bmAttributes)))
return 1;
return 0;
}
/*
* According to the USB 3.0 spec, all USB 3.0 devices must support LPM.
* However, there are some that don't, and they set the U1/U2 exit
* latencies to zero.
*/
if (!udev->bos->ss_cap) { dev_info(&udev->dev, "No LPM exit latency info found, disabling LPM.\n");
return 0;
}
if (udev->bos->ss_cap->bU1devExitLat == 0 && udev->bos->ss_cap->bU2DevExitLat == 0) {
if (udev->parent)
dev_info(&udev->dev, "LPM exit latency is zeroed, disabling LPM.\n");
else
dev_info(&udev->dev, "We don't know the algorithms for LPM for this host, disabling LPM.\n");
return 0;
}
if (!udev->parent || udev->parent->lpm_capable)
return 1;
return 0;
}
/*
* Set the Maximum Exit Latency (MEL) for the host to wakup up the path from
* U1/U2, send a PING to the device and receive a PING_RESPONSE.
* See USB 3.1 section C.1.5.2
*/
static void usb_set_lpm_mel(struct usb_device *udev,
struct usb3_lpm_parameters *udev_lpm_params,
unsigned int udev_exit_latency,
struct usb_hub *hub,
struct usb3_lpm_parameters *hub_lpm_params,
unsigned int hub_exit_latency)
{
unsigned int total_mel;
/*
* tMEL1. time to transition path from host to device into U0.
* MEL for parent already contains the delay up to parent, so only add
* the exit latency for the last link (pick the slower exit latency),
* and the hub header decode latency. See USB 3.1 section C 2.2.1
* Store MEL in nanoseconds
*/
total_mel = hub_lpm_params->mel +
max(udev_exit_latency, hub_exit_latency) * 1000 +
hub->descriptor->u.ss.bHubHdrDecLat * 100;
/*
* tMEL2. Time to submit PING packet. Sum of tTPTransmissionDelay for
* each link + wHubDelay for each hub. Add only for last link.
* tMEL4, the time for PING_RESPONSE to traverse upstream is similar.
* Multiply by 2 to include it as well.
*/
total_mel += (__le16_to_cpu(hub->descriptor->u.ss.wHubDelay) +
USB_TP_TRANSMISSION_DELAY) * 2;
/*
* tMEL3, tPingResponse. Time taken by device to generate PING_RESPONSE
* after receiving PING. Also add 2100ns as stated in USB 3.1 C 1.5.2.4
* to cover the delay if the PING_RESPONSE is queued behind a Max Packet
* Size DP.
* Note these delays should be added only once for the entire path, so
* add them to the MEL of the device connected to the roothub.
*/
if (!hub->hdev->parent)
total_mel += USB_PING_RESPONSE_TIME + 2100; udev_lpm_params->mel = total_mel;
}
/*
* Set the maximum Device to Host Exit Latency (PEL) for the device to initiate
* a transition from either U1 or U2.
*/
static void usb_set_lpm_pel(struct usb_device *udev,
struct usb3_lpm_parameters *udev_lpm_params,
unsigned int udev_exit_latency,
struct usb_hub *hub,
struct usb3_lpm_parameters *hub_lpm_params,
unsigned int hub_exit_latency,
unsigned int port_to_port_exit_latency)
{
unsigned int first_link_pel;
unsigned int hub_pel;
/*
* First, the device sends an LFPS to transition the link between the
* device and the parent hub into U0. The exit latency is the bigger of
* the device exit latency or the hub exit latency.
*/
if (udev_exit_latency > hub_exit_latency) first_link_pel = udev_exit_latency * 1000;
else
first_link_pel = hub_exit_latency * 1000;
/*
* When the hub starts to receive the LFPS, there is a slight delay for
* it to figure out that one of the ports is sending an LFPS. Then it
* will forward the LFPS to its upstream link. The exit latency is the
* delay, plus the PEL that we calculated for this hub.
*/
hub_pel = port_to_port_exit_latency * 1000 + hub_lpm_params->pel;
/*
* According to figure C-7 in the USB 3.0 spec, the PEL for this device
* is the greater of the two exit latencies.
*/
if (first_link_pel > hub_pel)
udev_lpm_params->pel = first_link_pel;
else
udev_lpm_params->pel = hub_pel;
}
/*
* Set the System Exit Latency (SEL) to indicate the total worst-case time from
* when a device initiates a transition to U0, until when it will receive the
* first packet from the host controller.
*
* Section C.1.5.1 describes the four components to this:
* - t1: device PEL
* - t2: time for the ERDY to make it from the device to the host.
* - t3: a host-specific delay to process the ERDY.
* - t4: time for the packet to make it from the host to the device.
*
* t3 is specific to both the xHCI host and the platform the host is integrated
* into. The Intel HW folks have said it's negligible, FIXME if a different
* vendor says otherwise.
*/
static void usb_set_lpm_sel(struct usb_device *udev,
struct usb3_lpm_parameters *udev_lpm_params)
{
struct usb_device *parent;
unsigned int num_hubs;
unsigned int total_sel;
/* t1 = device PEL */
total_sel = udev_lpm_params->pel;
/* How many external hubs are in between the device & the root port. */
for (parent = udev->parent, num_hubs = 0; parent->parent;
parent = parent->parent)
num_hubs++;
/* t2 = 2.1us + 250ns * (num_hubs - 1) */
if (num_hubs > 0) total_sel += 2100 + 250 * (num_hubs - 1);
/* t4 = 250ns * num_hubs */
total_sel += 250 * num_hubs;
udev_lpm_params->sel = total_sel;
}
static void usb_set_lpm_parameters(struct usb_device *udev)
{
struct usb_hub *hub;
unsigned int port_to_port_delay;
unsigned int udev_u1_del;
unsigned int udev_u2_del;
unsigned int hub_u1_del;
unsigned int hub_u2_del;
if (!udev->lpm_capable || udev->speed < USB_SPEED_SUPER)
return;
/* Skip if the device BOS descriptor couldn't be read */
if (!udev->bos)
return;
hub = usb_hub_to_struct_hub(udev->parent);
/* It doesn't take time to transition the roothub into U0, since it
* doesn't have an upstream link.
*/
if (!hub)
return;
udev_u1_del = udev->bos->ss_cap->bU1devExitLat;
udev_u2_del = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat);
hub_u1_del = udev->parent->bos->ss_cap->bU1devExitLat;
hub_u2_del = le16_to_cpu(udev->parent->bos->ss_cap->bU2DevExitLat);
usb_set_lpm_mel(udev, &udev->u1_params, udev_u1_del,
hub, &udev->parent->u1_params, hub_u1_del);
usb_set_lpm_mel(udev, &udev->u2_params, udev_u2_del,
hub, &udev->parent->u2_params, hub_u2_del);
/*
* Appendix C, section C.2.2.2, says that there is a slight delay from
* when the parent hub notices the downstream port is trying to
* transition to U0 to when the hub initiates a U0 transition on its
* upstream port. The section says the delays are tPort2PortU1EL and
* tPort2PortU2EL, but it doesn't define what they are.
*
* The hub chapter, sections 10.4.2.4 and 10.4.2.5 seem to be talking
* about the same delays. Use the maximum delay calculations from those
* sections. For U1, it's tHubPort2PortExitLat, which is 1us max. For
* U2, it's tHubPort2PortExitLat + U2DevExitLat - U1DevExitLat. I
* assume the device exit latencies they are talking about are the hub
* exit latencies.
*
* What do we do if the U2 exit latency is less than the U1 exit
* latency? It's possible, although not likely...
*/
port_to_port_delay = 1;
usb_set_lpm_pel(udev, &udev->u1_params, udev_u1_del,
hub, &udev->parent->u1_params, hub_u1_del,
port_to_port_delay);
if (hub_u2_del > hub_u1_del)
port_to_port_delay = 1 + hub_u2_del - hub_u1_del;
else
port_to_port_delay = 1 + hub_u1_del; usb_set_lpm_pel(udev, &udev->u2_params, udev_u2_del,
hub, &udev->parent->u2_params, hub_u2_del,
port_to_port_delay);
/* Now that we've got PEL, calculate SEL. */
usb_set_lpm_sel(udev, &udev->u1_params); usb_set_lpm_sel(udev, &udev->u2_params);
}
/* USB 2.0 spec Section 11.24.4.5 */
static int get_hub_descriptor(struct usb_device *hdev,
struct usb_hub_descriptor *desc)
{
int i, ret, size;
unsigned dtype;
if (hub_is_superspeed(hdev)) {
dtype = USB_DT_SS_HUB;
size = USB_DT_SS_HUB_SIZE;
} else {
dtype = USB_DT_HUB;
size = sizeof(struct usb_hub_descriptor);
}
for (i = 0; i < 3; i++) {
ret = usb_control_msg(hdev, usb_rcvctrlpipe(hdev, 0),
USB_REQ_GET_DESCRIPTOR, USB_DIR_IN | USB_RT_HUB,
dtype << 8, 0, desc, size,
USB_CTRL_GET_TIMEOUT);
if (hub_is_superspeed(hdev)) {
if (ret == size)
return ret;
} else if (ret >= USB_DT_HUB_NONVAR_SIZE + 2) {
/* Make sure we have the DeviceRemovable field. */
size = USB_DT_HUB_NONVAR_SIZE + desc->bNbrPorts / 8 + 1;
if (ret < size)
return -EMSGSIZE;
return ret;
}
}
return -EINVAL;
}
/*
* USB 2.0 spec Section 11.24.2.1
*/
static int clear_hub_feature(struct usb_device *hdev, int feature)
{
return usb_control_msg(hdev, usb_sndctrlpipe(hdev, 0),
USB_REQ_CLEAR_FEATURE, USB_RT_HUB, feature, 0, NULL, 0, 1000);
}
/*
* USB 2.0 spec Section 11.24.2.2
*/
int usb_clear_port_feature(struct usb_device *hdev, int port1, int feature)
{
return usb_control_msg(hdev, usb_sndctrlpipe(hdev, 0),
USB_REQ_CLEAR_FEATURE, USB_RT_PORT, feature, port1,
NULL, 0, 1000);
}
/*
* USB 2.0 spec Section 11.24.2.13
*/
static int set_port_feature(struct usb_device *hdev, int port1, int feature)
{
return usb_control_msg(hdev, usb_sndctrlpipe(hdev, 0),
USB_REQ_SET_FEATURE, USB_RT_PORT, feature, port1,
NULL, 0, 1000);
}
static char *to_led_name(int selector)
{
switch (selector) {
case HUB_LED_AMBER:
return "amber";
case HUB_LED_GREEN:
return "green";
case HUB_LED_OFF:
return "off";
case HUB_LED_AUTO:
return "auto";
default:
return "??";
}
}
/*
* USB 2.0 spec Section 11.24.2.7.1.10 and table 11-7
* for info about using port indicators
*/
static void set_port_led(struct usb_hub *hub, int port1, int selector)
{
struct usb_port *port_dev = hub->ports[port1 - 1];
int status;
status = set_port_feature(hub->hdev, (selector << 8) | port1,
USB_PORT_FEAT_INDICATOR);
dev_dbg(&port_dev->dev, "indicator %s status %d\n",
to_led_name(selector), status);
}
#define LED_CYCLE_PERIOD ((2*HZ)/3)
static void led_work(struct work_struct *work)
{
struct usb_hub *hub =
container_of(work, struct usb_hub, leds.work);
struct usb_device *hdev = hub->hdev;
unsigned i;
unsigned changed = 0;
int cursor = -1;
if (hdev->state != USB_STATE_CONFIGURED || hub->quiescing)
return;
for (i = 0; i < hdev->maxchild; i++) {
unsigned selector, mode;
/* 30%-50% duty cycle */
switch (hub->indicator[i]) {
/* cycle marker */
case INDICATOR_CYCLE:
cursor = i;
selector = HUB_LED_AUTO;
mode = INDICATOR_AUTO;
break;
/* blinking green = sw attention */
case INDICATOR_GREEN_BLINK:
selector = HUB_LED_GREEN;
mode = INDICATOR_GREEN_BLINK_OFF;
break;
case INDICATOR_GREEN_BLINK_OFF:
selector = HUB_LED_OFF;
mode = INDICATOR_GREEN_BLINK;
break;
/* blinking amber = hw attention */
case INDICATOR_AMBER_BLINK:
selector = HUB_LED_AMBER;
mode = INDICATOR_AMBER_BLINK_OFF;
break;
case INDICATOR_AMBER_BLINK_OFF:
selector = HUB_LED_OFF;
mode = INDICATOR_AMBER_BLINK;
break;
/* blink green/amber = reserved */
case INDICATOR_ALT_BLINK:
selector = HUB_LED_GREEN;
mode = INDICATOR_ALT_BLINK_OFF;
break;
case INDICATOR_ALT_BLINK_OFF:
selector = HUB_LED_AMBER;
mode = INDICATOR_ALT_BLINK;
break;
default:
continue;
}
if (selector != HUB_LED_AUTO)
changed = 1;
set_port_led(hub, i + 1, selector);
hub->indicator[i] = mode;
}
if (!changed && blinkenlights) {
cursor++;
cursor %= hdev->maxchild;
set_port_led(hub, cursor + 1, HUB_LED_GREEN);
hub->indicator[cursor] = INDICATOR_CYCLE;
changed++;
}
if (changed)
queue_delayed_work(system_power_efficient_wq,
&hub->leds, LED_CYCLE_PERIOD);
}
/* use a short timeout for hub/port status fetches */
#define USB_STS_TIMEOUT 1000
#define USB_STS_RETRIES 5
/*
* USB 2.0 spec Section 11.24.2.6
*/
static int get_hub_status(struct usb_device *hdev,
struct usb_hub_status *data)
{
int i, status = -ETIMEDOUT;
for (i = 0; i < USB_STS_RETRIES &&
(status == -ETIMEDOUT || status == -EPIPE); i++) {
status = usb_control_msg(hdev, usb_rcvctrlpipe(hdev, 0),
USB_REQ_GET_STATUS, USB_DIR_IN | USB_RT_HUB, 0, 0,
data, sizeof(*data), USB_STS_TIMEOUT);
}
return status;
}
/*
* USB 2.0 spec Section 11.24.2.7
* USB 3.1 takes into use the wValue and wLength fields, spec Section 10.16.2.6
*/
static int get_port_status(struct usb_device *hdev, int port1,
void *data, u16 value, u16 length)
{
int i, status = -ETIMEDOUT;
for (i = 0; i < USB_STS_RETRIES && (status == -ETIMEDOUT || status == -EPIPE); i++) { status = usb_control_msg(hdev, usb_rcvctrlpipe(hdev, 0),
USB_REQ_GET_STATUS, USB_DIR_IN | USB_RT_PORT, value,
port1, data, length, USB_STS_TIMEOUT);
}
return status;
}
static int hub_ext_port_status(struct usb_hub *hub, int port1, int type,
u16 *status, u16 *change, u32 *ext_status)
{
int ret;
int len = 4;
if (type != HUB_PORT_STATUS)
len = 8;
mutex_lock(&hub->status_mutex); ret = get_port_status(hub->hdev, port1, &hub->status->port, type, len); if (ret < len) { if (ret != -ENODEV) dev_err(hub->intfdev,
"%s failed (err = %d)\n", __func__, ret);
if (ret >= 0)
ret = -EIO;
} else {
*status = le16_to_cpu(hub->status->port.wPortStatus);
*change = le16_to_cpu(hub->status->port.wPortChange);
if (type != HUB_PORT_STATUS && ext_status) *ext_status = le32_to_cpu(
hub->status->port.dwExtPortStatus);
ret = 0;
}
mutex_unlock(&hub->status_mutex);
/*
* There is no need to lock status_mutex here, because status_mutex
* protects hub->status, and the phy driver only checks the port
* status without changing the status.
*/
if (!ret) {
struct usb_device *hdev = hub->hdev;
/*
* Only roothub will be notified of connection changes,
* since the USB PHY only cares about changes at the next
* level.
*/
if (is_root_hub(hdev)) {
struct usb_hcd *hcd = bus_to_hcd(hdev->bus);
bool connect;
bool connect_change;
connect_change = *change & USB_PORT_STAT_C_CONNECTION;
connect = *status & USB_PORT_STAT_CONNECTION;
if (connect_change && connect)
usb_phy_roothub_notify_connect(hcd->phy_roothub, port1 - 1);
else if (connect_change)
usb_phy_roothub_notify_disconnect(hcd->phy_roothub, port1 - 1);
}
}
return ret;
}
int usb_hub_port_status(struct usb_hub *hub, int port1,
u16 *status, u16 *change)
{
return hub_ext_port_status(hub, port1, HUB_PORT_STATUS,
status, change, NULL);
}
static void hub_resubmit_irq_urb(struct usb_hub *hub)
{
unsigned long flags;
int status;
spin_lock_irqsave(&hub->irq_urb_lock, flags);
if (hub->quiescing) {
spin_unlock_irqrestore(&hub->irq_urb_lock, flags);
return;
}
status = usb_submit_urb(hub->urb, GFP_ATOMIC); if (status && status != -ENODEV && status != -EPERM &&
status != -ESHUTDOWN) {
dev_err(hub->intfdev, "resubmit --> %d\n", status);
mod_timer(&hub->irq_urb_retry, jiffies + HZ);
}
spin_unlock_irqrestore(&hub->irq_urb_lock, flags);
}
static void hub_retry_irq_urb(struct timer_list *t)
{
struct usb_hub *hub = from_timer(hub, t, irq_urb_retry);
hub_resubmit_irq_urb(hub);
}
static void kick_hub_wq(struct usb_hub *hub)
{
struct usb_interface *intf;
if (hub->disconnected || work_pending(&hub->events))
return;
/*
* Suppress autosuspend until the event is proceed.
*
* Be careful and make sure that the symmetric operation is
* always called. We are here only when there is no pending
* work for this hub. Therefore put the interface either when
* the new work is called or when it is canceled.
*/
intf = to_usb_interface(hub->intfdev);
usb_autopm_get_interface_no_resume(intf);
hub_get(hub); if (queue_work(hub_wq, &hub->events))
return;
/* the work has already been scheduled */
usb_autopm_put_interface_async(intf); hub_put(hub);
}
void usb_kick_hub_wq(struct usb_device *hdev)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
if (hub)
kick_hub_wq(hub);
}
/*
* Let the USB core know that a USB 3.0 device has sent a Function Wake Device
* Notification, which indicates it had initiated remote wakeup.
*
* USB 3.0 hubs do not report the port link state change from U3 to U0 when the
* device initiates resume, so the USB core will not receive notice of the
* resume through the normal hub interrupt URB.
*/
void usb_wakeup_notification(struct usb_device *hdev,
unsigned int portnum)
{
struct usb_hub *hub;
struct usb_port *port_dev;
if (!hdev)
return;
hub = usb_hub_to_struct_hub(hdev);
if (hub) {
port_dev = hub->ports[portnum - 1];
if (port_dev && port_dev->child)
pm_wakeup_event(&port_dev->child->dev, 0);
set_bit(portnum, hub->wakeup_bits);
kick_hub_wq(hub);
}
}
EXPORT_SYMBOL_GPL(usb_wakeup_notification);
/* completion function, fires on port status changes and various faults */
static void hub_irq(struct urb *urb)
{
struct usb_hub *hub = urb->context;
int status = urb->status;
unsigned i;
unsigned long bits;
switch (status) {
case -ENOENT: /* synchronous unlink */
case -ECONNRESET: /* async unlink */
case -ESHUTDOWN: /* hardware going away */
return;
default: /* presumably an error */
/* Cause a hub reset after 10 consecutive errors */
dev_dbg(hub->intfdev, "transfer --> %d\n", status); if ((++hub->nerrors < 10) || hub->error)
goto resubmit;
hub->error = status;
fallthrough;
/* let hub_wq handle things */
case 0: /* we got data: port status changed */
bits = 0;
for (i = 0; i < urb->actual_length; ++i)
bits |= ((unsigned long) ((*hub->buffer)[i]))
<< (i*8); hub->event_bits[0] = bits;
break;
}
hub->nerrors = 0;
/* Something happened, let hub_wq figure it out */
kick_hub_wq(hub);
resubmit:
hub_resubmit_irq_urb(hub);
}
/* USB 2.0 spec Section 11.24.2.3 */
static inline int
hub_clear_tt_buffer(struct usb_device *hdev, u16 devinfo, u16 tt)
{
/* Need to clear both directions for control ep */
if (((devinfo >> 11) & USB_ENDPOINT_XFERTYPE_MASK) ==
USB_ENDPOINT_XFER_CONTROL) {
int status = usb_control_msg(hdev, usb_sndctrlpipe(hdev, 0),
HUB_CLEAR_TT_BUFFER, USB_RT_PORT,
devinfo ^ 0x8000, tt, NULL, 0, 1000);
if (status)
return status;
}
return usb_control_msg(hdev, usb_sndctrlpipe(hdev, 0),
HUB_CLEAR_TT_BUFFER, USB_RT_PORT, devinfo,
tt, NULL, 0, 1000);
}
/*
* enumeration blocks hub_wq for a long time. we use keventd instead, since
* long blocking there is the exception, not the rule. accordingly, HCDs
* talking to TTs must queue control transfers (not just bulk and iso), so
* both can talk to the same hub concurrently.
*/
static void hub_tt_work(struct work_struct *work)
{
struct usb_hub *hub =
container_of(work, struct usb_hub, tt.clear_work);
unsigned long flags;
spin_lock_irqsave(&hub->tt.lock, flags);
while (!list_empty(&hub->tt.clear_list)) {
struct list_head *next;
struct usb_tt_clear *clear;
struct usb_device *hdev = hub->hdev;
const struct hc_driver *drv;
int status;
next = hub->tt.clear_list.next;
clear = list_entry(next, struct usb_tt_clear, clear_list);
list_del(&clear->clear_list);
/* drop lock so HCD can concurrently report other TT errors */
spin_unlock_irqrestore(&hub->tt.lock, flags);
status = hub_clear_tt_buffer(hdev, clear->devinfo, clear->tt);
if (status && status != -ENODEV)
dev_err(&hdev->dev,
"clear tt %d (%04x) error %d\n",
clear->tt, clear->devinfo, status);
/* Tell the HCD, even if the operation failed */
drv = clear->hcd->driver;
if (drv->clear_tt_buffer_complete)
(drv->clear_tt_buffer_complete)(clear->hcd, clear->ep);
kfree(clear);
spin_lock_irqsave(&hub->tt.lock, flags);
}
spin_unlock_irqrestore(&hub->tt.lock, flags);
}
/**
* usb_hub_set_port_power - control hub port's power state
* @hdev: USB device belonging to the usb hub
* @hub: target hub
* @port1: port index
* @set: expected status
*
* call this function to control port's power via setting or
* clearing the port's PORT_POWER feature.
*
* Return: 0 if successful. A negative error code otherwise.
*/
int usb_hub_set_port_power(struct usb_device *hdev, struct usb_hub *hub,
int port1, bool set)
{
int ret;
if (set)
ret = set_port_feature(hdev, port1, USB_PORT_FEAT_POWER);
else
ret = usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_POWER);
if (ret)
return ret;
if (set)
set_bit(port1, hub->power_bits);
else
clear_bit(port1, hub->power_bits);
return 0;
}
/**
* usb_hub_clear_tt_buffer - clear control/bulk TT state in high speed hub
* @urb: an URB associated with the failed or incomplete split transaction
*
* High speed HCDs use this to tell the hub driver that some split control or
* bulk transaction failed in a way that requires clearing internal state of
* a transaction translator. This is normally detected (and reported) from
* interrupt context.
*
* It may not be possible for that hub to handle additional full (or low)
* speed transactions until that state is fully cleared out.
*
* Return: 0 if successful. A negative error code otherwise.
*/
int usb_hub_clear_tt_buffer(struct urb *urb)
{
struct usb_device *udev = urb->dev;
int pipe = urb->pipe;
struct usb_tt *tt = udev->tt;
unsigned long flags;
struct usb_tt_clear *clear;
/* we've got to cope with an arbitrary number of pending TT clears,
* since each TT has "at least two" buffers that can need it (and
* there can be many TTs per hub). even if they're uncommon.
*/
clear = kmalloc(sizeof *clear, GFP_ATOMIC);
if (clear == NULL) {
dev_err(&udev->dev, "can't save CLEAR_TT_BUFFER state\n");
/* FIXME recover somehow ... RESET_TT? */
return -ENOMEM;
}
/* info that CLEAR_TT_BUFFER needs */
clear->tt = tt->multi ? udev->ttport : 1;
clear->devinfo = usb_pipeendpoint (pipe);
clear->devinfo |= ((u16)udev->devaddr) << 4;
clear->devinfo |= usb_pipecontrol(pipe)
? (USB_ENDPOINT_XFER_CONTROL << 11)
: (USB_ENDPOINT_XFER_BULK << 11);
if (usb_pipein(pipe))
clear->devinfo |= 1 << 15;
/* info for completion callback */
clear->hcd = bus_to_hcd(udev->bus);
clear->ep = urb->ep;
/* tell keventd to clear state for this TT */
spin_lock_irqsave(&tt->lock, flags);
list_add_tail(&clear->clear_list, &tt->clear_list);
schedule_work(&tt->clear_work);
spin_unlock_irqrestore(&tt->lock, flags);
return 0;
}
EXPORT_SYMBOL_GPL(usb_hub_clear_tt_buffer);
static void hub_power_on(struct usb_hub *hub, bool do_delay)
{
int port1;
/* Enable power on each port. Some hubs have reserved values
* of LPSM (> 2) in their descriptors, even though they are
* USB 2.0 hubs. Some hubs do not implement port-power switching
* but only emulate it. In all cases, the ports won't work
* unless we send these messages to the hub.
*/
if (hub_is_port_power_switchable(hub)) dev_dbg(hub->intfdev, "enabling power on all ports\n");
else
dev_dbg(hub->intfdev, "trying to enable port power on "
"non-switchable hub\n");
for (port1 = 1; port1 <= hub->hdev->maxchild; port1++) if (test_bit(port1, hub->power_bits)) set_port_feature(hub->hdev, port1, USB_PORT_FEAT_POWER);
else
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_POWER);
if (do_delay) msleep(hub_power_on_good_delay(hub));
}
static int hub_hub_status(struct usb_hub *hub,
u16 *status, u16 *change)
{
int ret;
mutex_lock(&hub->status_mutex);
ret = get_hub_status(hub->hdev, &hub->status->hub);
if (ret < 0) {
if (ret != -ENODEV)
dev_err(hub->intfdev,
"%s failed (err = %d)\n", __func__, ret);
} else {
*status = le16_to_cpu(hub->status->hub.wHubStatus);
*change = le16_to_cpu(hub->status->hub.wHubChange);
ret = 0;
}
mutex_unlock(&hub->status_mutex);
return ret;
}
static int hub_set_port_link_state(struct usb_hub *hub, int port1,
unsigned int link_status)
{
return set_port_feature(hub->hdev,
port1 | (link_status << 3),
USB_PORT_FEAT_LINK_STATE);
}
/*
* Disable a port and mark a logical connect-change event, so that some
* time later hub_wq will disconnect() any existing usb_device on the port
* and will re-enumerate if there actually is a device attached.
*/
static void hub_port_logical_disconnect(struct usb_hub *hub, int port1)
{
dev_dbg(&hub->ports[port1 - 1]->dev, "logical disconnect\n"); hub_port_disable(hub, port1, 1);
/* FIXME let caller ask to power down the port:
* - some devices won't enumerate without a VBUS power cycle
* - SRP saves power that way
* - ... new call, TBD ...
* That's easy if this hub can switch power per-port, and
* hub_wq reactivates the port later (timer, SRP, etc).
* Powerdown must be optional, because of reset/DFU.
*/
set_bit(port1, hub->change_bits);
kick_hub_wq(hub);
}
/**
* usb_remove_device - disable a device's port on its parent hub
* @udev: device to be disabled and removed
* Context: @udev locked, must be able to sleep.
*
* After @udev's port has been disabled, hub_wq is notified and it will
* see that the device has been disconnected. When the device is
* physically unplugged and something is plugged in, the events will
* be received and processed normally.
*
* Return: 0 if successful. A negative error code otherwise.
*/
int usb_remove_device(struct usb_device *udev)
{
struct usb_hub *hub;
struct usb_interface *intf;
int ret;
if (!udev->parent) /* Can't remove a root hub */
return -EINVAL;
hub = usb_hub_to_struct_hub(udev->parent);
intf = to_usb_interface(hub->intfdev);
ret = usb_autopm_get_interface(intf);
if (ret < 0)
return ret;
set_bit(udev->portnum, hub->removed_bits);
hub_port_logical_disconnect(hub, udev->portnum);
usb_autopm_put_interface(intf);
return 0;
}
enum hub_activation_type {
HUB_INIT, HUB_INIT2, HUB_INIT3, /* INITs must come first */
HUB_POST_RESET, HUB_RESUME, HUB_RESET_RESUME,
};
static void hub_init_func2(struct work_struct *ws);
static void hub_init_func3(struct work_struct *ws);
static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
{
struct usb_device *hdev = hub->hdev;
struct usb_hcd *hcd;
int ret;
int port1;
int status;
bool need_debounce_delay = false;
unsigned delay;
/* Continue a partial initialization */
if (type == HUB_INIT2 || type == HUB_INIT3) {
device_lock(&hdev->dev);
/* Was the hub disconnected while we were waiting? */
if (hub->disconnected)
goto disconnected;
if (type == HUB_INIT2)
goto init2;
goto init3;
}
hub_get(hub);
/* The superspeed hub except for root hub has to use Hub Depth
* value as an offset into the route string to locate the bits
* it uses to determine the downstream port number. So hub driver
* should send a set hub depth request to superspeed hub after
* the superspeed hub is set configuration in initialization or
* reset procedure.
*
* After a resume, port power should still be on.
* For any other type of activation, turn it on.
*/
if (type != HUB_RESUME) { if (hdev->parent && hub_is_superspeed(hdev)) {
ret = usb_control_msg(hdev, usb_sndctrlpipe(hdev, 0),
HUB_SET_DEPTH, USB_RT_HUB,
hdev->level - 1, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
if (ret < 0)
dev_err(hub->intfdev,
"set hub depth failed\n");
}
/* Speed up system boot by using a delayed_work for the
* hub's initial power-up delays. This is pretty awkward
* and the implementation looks like a home-brewed sort of
* setjmp/longjmp, but it saves at least 100 ms for each
* root hub (assuming usbcore is compiled into the kernel
* rather than as a module). It adds up.
*
* This can't be done for HUB_RESUME or HUB_RESET_RESUME
* because for those activation types the ports have to be
* operational when we return. In theory this could be done
* for HUB_POST_RESET, but it's easier not to.
*/
if (type == HUB_INIT) { delay = hub_power_on_good_delay(hub); hub_power_on(hub, false);
INIT_DELAYED_WORK(&hub->init_work, hub_init_func2);
queue_delayed_work(system_power_efficient_wq,
&hub->init_work,
msecs_to_jiffies(delay));
/* Suppress autosuspend until init is done */
usb_autopm_get_interface_no_resume(
to_usb_interface(hub->intfdev));
return; /* Continues at init2: below */
} else if (type == HUB_RESET_RESUME) {
/* The internal host controller state for the hub device
* may be gone after a host power loss on system resume.
* Update the device's info so the HW knows it's a hub.
*/
hcd = bus_to_hcd(hdev->bus);
if (hcd->driver->update_hub_device) {
ret = hcd->driver->update_hub_device(hcd, hdev,
&hub->tt, GFP_NOIO);
if (ret < 0) {
dev_err(hub->intfdev,
"Host not accepting hub info update\n");
dev_err(hub->intfdev,
"LS/FS devices and hubs may not work under this hub\n");
}
}
hub_power_on(hub, true);
} else {
hub_power_on(hub, true);
}
/* Give some time on remote wakeup to let links to transit to U0 */
} else if (hub_is_superspeed(hub->hdev)) msleep(20);
init2:
/*
* Check each port and set hub->change_bits to let hub_wq know
* which ports need attention.
*/
for (port1 = 1; port1 <= hdev->maxchild; ++port1) { struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *udev = port_dev->child;
u16 portstatus, portchange;
portstatus = portchange = 0;
status = usb_hub_port_status(hub, port1, &portstatus, &portchange);
if (status)
goto abort; if (udev || (portstatus & USB_PORT_STAT_CONNECTION)) dev_dbg(&port_dev->dev, "status %04x change %04x\n",
portstatus, portchange);
/*
* After anything other than HUB_RESUME (i.e., initialization
* or any sort of reset), every port should be disabled.
* Unconnected ports should likewise be disabled (paranoia),
* and so should ports for which we have no usb_device.
*/
if ((portstatus & USB_PORT_STAT_ENABLE) && (
type != HUB_RESUME ||
!(portstatus & USB_PORT_STAT_CONNECTION) ||
!udev ||
udev->state == USB_STATE_NOTATTACHED)) {
/*
* USB3 protocol ports will automatically transition
* to Enabled state when detect an USB3.0 device attach.
* Do not disable USB3 protocol ports, just pretend
* power was lost
*/
portstatus &= ~USB_PORT_STAT_ENABLE; if (!hub_is_superspeed(hdev)) usb_clear_port_feature(hdev, port1,
USB_PORT_FEAT_ENABLE);
}
/* Make sure a warm-reset request is handled by port_event */
if (type == HUB_RESUME && hub_port_warm_reset_required(hub, port1, portstatus)) set_bit(port1, hub->event_bits);
/*
* Add debounce if USB3 link is in polling/link training state.
* Link will automatically transition to Enabled state after
* link training completes.
*/
if (hub_is_superspeed(hdev) && ((portstatus & USB_PORT_STAT_LINK_STATE) ==
USB_SS_PORT_LS_POLLING))
need_debounce_delay = true;
/* Clear status-change flags; we'll debounce later */
if (portchange & USB_PORT_STAT_C_CONNECTION) {
need_debounce_delay = true;
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_CONNECTION);
}
if (portchange & USB_PORT_STAT_C_ENABLE) {
need_debounce_delay = true;
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_ENABLE);
}
if (portchange & USB_PORT_STAT_C_RESET) {
need_debounce_delay = true;
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_RESET);
}
if ((portchange & USB_PORT_STAT_C_BH_RESET) && hub_is_superspeed(hub->hdev)) {
need_debounce_delay = true;
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_BH_PORT_RESET);
}
/* We can forget about a "removed" device when there's a
* physical disconnect or the connect status changes.
*/
if (!(portstatus & USB_PORT_STAT_CONNECTION) || (portchange & USB_PORT_STAT_C_CONNECTION)) clear_bit(port1, hub->removed_bits); if (!udev || udev->state == USB_STATE_NOTATTACHED) {
/* Tell hub_wq to disconnect the device or
* check for a new connection or over current condition.
* Based on USB2.0 Spec Section 11.12.5,
* C_PORT_OVER_CURRENT could be set while
* PORT_OVER_CURRENT is not. So check for any of them.
*/
if (udev || (portstatus & USB_PORT_STAT_CONNECTION) || (portchange & USB_PORT_STAT_C_CONNECTION) ||
(portstatus & USB_PORT_STAT_OVERCURRENT) ||
(portchange & USB_PORT_STAT_C_OVERCURRENT))
set_bit(port1, hub->change_bits); } else if (portstatus & USB_PORT_STAT_ENABLE) {
bool port_resumed = (portstatus &
USB_PORT_STAT_LINK_STATE) ==
USB_SS_PORT_LS_U0;
/* The power session apparently survived the resume.
* If there was an overcurrent or suspend change
* (i.e., remote wakeup request), have hub_wq
* take care of it. Look at the port link state
* for USB 3.0 hubs, since they don't have a suspend
* change bit, and they don't set the port link change
* bit on device-initiated resume.
*/
if (portchange || (hub_is_superspeed(hub->hdev) &&
port_resumed))
set_bit(port1, hub->event_bits); } else if (udev->persist_enabled) {
#ifdef CONFIG_PM
udev->reset_resume = 1;
#endif
/* Don't set the change_bits when the device
* was powered off.
*/
if (test_bit(port1, hub->power_bits))
set_bit(port1, hub->change_bits);
} else {
/* The power session is gone; tell hub_wq */
usb_set_device_state(udev, USB_STATE_NOTATTACHED);
set_bit(port1, hub->change_bits);
}
}
/* If no port-status-change flags were set, we don't need any
* debouncing. If flags were set we can try to debounce the
* ports all at once right now, instead of letting hub_wq do them
* one at a time later on.
*
* If any port-status changes do occur during this delay, hub_wq
* will see them later and handle them normally.
*/
if (need_debounce_delay) {
delay = HUB_DEBOUNCE_STABLE;
/* Don't do a long sleep inside a workqueue routine */
if (type == HUB_INIT2) { INIT_DELAYED_WORK(&hub->init_work, hub_init_func3);
queue_delayed_work(system_power_efficient_wq,
&hub->init_work,
msecs_to_jiffies(delay));
device_unlock(&hdev->dev);
return; /* Continues at init3: below */
} else {
msleep(delay);
}
}
init3:
hub->quiescing = 0;
status = usb_submit_urb(hub->urb, GFP_NOIO);
if (status < 0)
dev_err(hub->intfdev, "activate --> %d\n", status); if (hub->has_indicators && blinkenlights) queue_delayed_work(system_power_efficient_wq,
&hub->leds, LED_CYCLE_PERIOD);
/* Scan all ports that need attention */
kick_hub_wq(hub);
abort:
if (type == HUB_INIT2 || type == HUB_INIT3) {
/* Allow autosuspend if it was suppressed */
disconnected:
usb_autopm_put_interface_async(to_usb_interface(hub->intfdev));
device_unlock(&hdev->dev);
}
hub_put(hub);
}
/* Implement the continuations for the delays above */
static void hub_init_func2(struct work_struct *ws)
{
struct usb_hub *hub = container_of(ws, struct usb_hub, init_work.work);
hub_activate(hub, HUB_INIT2);
}
static void hub_init_func3(struct work_struct *ws)
{
struct usb_hub *hub = container_of(ws, struct usb_hub, init_work.work);
hub_activate(hub, HUB_INIT3);
}
enum hub_quiescing_type {
HUB_DISCONNECT, HUB_PRE_RESET, HUB_SUSPEND
};
static void hub_quiesce(struct usb_hub *hub, enum hub_quiescing_type type)
{
struct usb_device *hdev = hub->hdev;
unsigned long flags;
int i;
/* hub_wq and related activity won't re-trigger */
spin_lock_irqsave(&hub->irq_urb_lock, flags);
hub->quiescing = 1;
spin_unlock_irqrestore(&hub->irq_urb_lock, flags);
if (type != HUB_SUSPEND) {
/* Disconnect all the children */
for (i = 0; i < hdev->maxchild; ++i) { if (hub->ports[i]->child) usb_disconnect(&hub->ports[i]->child);
}
}
/* Stop hub_wq and related activity */
del_timer_sync(&hub->irq_urb_retry);
usb_kill_urb(hub->urb);
if (hub->has_indicators)
cancel_delayed_work_sync(&hub->leds); if (hub->tt.hub) flush_work(&hub->tt.clear_work);
}
static void hub_pm_barrier_for_all_ports(struct usb_hub *hub)
{
int i;
for (i = 0; i < hub->hdev->maxchild; ++i)
pm_runtime_barrier(&hub->ports[i]->dev);
}
/* caller has locked the hub device */
static int hub_pre_reset(struct usb_interface *intf)
{
struct usb_hub *hub = usb_get_intfdata(intf);
hub_quiesce(hub, HUB_PRE_RESET);
hub->in_reset = 1;
hub_pm_barrier_for_all_ports(hub);
return 0;
}
/* caller has locked the hub device */
static int hub_post_reset(struct usb_interface *intf)
{
struct usb_hub *hub = usb_get_intfdata(intf);
hub->in_reset = 0;
hub_pm_barrier_for_all_ports(hub);
hub_activate(hub, HUB_POST_RESET);
return 0;
}
static int hub_configure(struct usb_hub *hub,
struct usb_endpoint_descriptor *endpoint)
{
struct usb_hcd *hcd;
struct usb_device *hdev = hub->hdev;
struct device *hub_dev = hub->intfdev;
u16 hubstatus, hubchange;
u16 wHubCharacteristics;
unsigned int pipe;
int maxp, ret, i;
char *message = "out of memory";
unsigned unit_load;
unsigned full_load;
unsigned maxchild;
hub->buffer = kmalloc(sizeof(*hub->buffer), GFP_KERNEL);
if (!hub->buffer) {
ret = -ENOMEM;
goto fail;
}
hub->status = kmalloc(sizeof(*hub->status), GFP_KERNEL);
if (!hub->status) {
ret = -ENOMEM;
goto fail;
}
mutex_init(&hub->status_mutex);
hub->descriptor = kzalloc(sizeof(*hub->descriptor), GFP_KERNEL);
if (!hub->descriptor) {
ret = -ENOMEM;
goto fail;
}
/* Request the entire hub descriptor.
* hub->descriptor can handle USB_MAXCHILDREN ports,
* but a (non-SS) hub can/will return fewer bytes here.
*/
ret = get_hub_descriptor(hdev, hub->descriptor);
if (ret < 0) {
message = "can't read hub descriptor";
goto fail;
}
maxchild = USB_MAXCHILDREN;
if (hub_is_superspeed(hdev))
maxchild = min_t(unsigned, maxchild, USB_SS_MAXPORTS);
if (hub->descriptor->bNbrPorts > maxchild) {
message = "hub has too many ports!";
ret = -ENODEV;
goto fail;
} else if (hub->descriptor->bNbrPorts == 0) {
message = "hub doesn't have any ports!";
ret = -ENODEV;
goto fail;
}
/*
* Accumulate wHubDelay + 40ns for every hub in the tree of devices.
* The resulting value will be used for SetIsochDelay() request.
*/
if (hub_is_superspeed(hdev) || hub_is_superspeedplus(hdev)) {
u32 delay = __le16_to_cpu(hub->descriptor->u.ss.wHubDelay);
if (hdev->parent)
delay += hdev->parent->hub_delay;
delay += USB_TP_TRANSMISSION_DELAY;
hdev->hub_delay = min_t(u32, delay, USB_TP_TRANSMISSION_DELAY_MAX);
}
maxchild = hub->descriptor->bNbrPorts;
dev_info(hub_dev, "%d port%s detected\n", maxchild,
(maxchild == 1) ? "" : "s");
hub->ports = kcalloc(maxchild, sizeof(struct usb_port *), GFP_KERNEL);
if (!hub->ports) {
ret = -ENOMEM;
goto fail;
}
wHubCharacteristics = le16_to_cpu(hub->descriptor->wHubCharacteristics);
if (hub_is_superspeed(hdev)) {
unit_load = 150;
full_load = 900;
} else {
unit_load = 100;
full_load = 500;
}
/* FIXME for USB 3.0, skip for now */
if ((wHubCharacteristics & HUB_CHAR_COMPOUND) &&
!(hub_is_superspeed(hdev))) {
char portstr[USB_MAXCHILDREN + 1];
for (i = 0; i < maxchild; i++)
portstr[i] = hub->descriptor->u.hs.DeviceRemovable
[((i + 1) / 8)] & (1 << ((i + 1) % 8))
? 'F' : 'R';
portstr[maxchild] = 0;
dev_dbg(hub_dev, "compound device; port removable status: %s\n", portstr);
} else
dev_dbg(hub_dev, "standalone hub\n");
switch (wHubCharacteristics & HUB_CHAR_LPSM) {
case HUB_CHAR_COMMON_LPSM:
dev_dbg(hub_dev, "ganged power switching\n");
break;
case HUB_CHAR_INDV_PORT_LPSM:
dev_dbg(hub_dev, "individual port power switching\n");
break;
case HUB_CHAR_NO_LPSM:
case HUB_CHAR_LPSM:
dev_dbg(hub_dev, "no power switching (usb 1.0)\n");
break;
}
switch (wHubCharacteristics & HUB_CHAR_OCPM) {
case HUB_CHAR_COMMON_OCPM:
dev_dbg(hub_dev, "global over-current protection\n");
break;
case HUB_CHAR_INDV_PORT_OCPM:
dev_dbg(hub_dev, "individual port over-current protection\n");
break;
case HUB_CHAR_NO_OCPM:
case HUB_CHAR_OCPM:
dev_dbg(hub_dev, "no over-current protection\n");
break;
}
spin_lock_init(&hub->tt.lock);
INIT_LIST_HEAD(&hub->tt.clear_list);
INIT_WORK(&hub->tt.clear_work, hub_tt_work);
switch (hdev->descriptor.bDeviceProtocol) {
case USB_HUB_PR_FS:
break;
case USB_HUB_PR_HS_SINGLE_TT:
dev_dbg(hub_dev, "Single TT\n");
hub->tt.hub = hdev;
break;
case USB_HUB_PR_HS_MULTI_TT:
ret = usb_set_interface(hdev, 0, 1);
if (ret == 0) {
dev_dbg(hub_dev, "TT per port\n");
hub->tt.multi = 1;
} else
dev_err(hub_dev, "Using single TT (err %d)\n",
ret);
hub->tt.hub = hdev;
break;
case USB_HUB_PR_SS:
/* USB 3.0 hubs don't have a TT */
break;
default:
dev_dbg(hub_dev, "Unrecognized hub protocol %d\n",
hdev->descriptor.bDeviceProtocol);
break;
}
/* Note 8 FS bit times == (8 bits / 12000000 bps) ~= 666ns */
switch (wHubCharacteristics & HUB_CHAR_TTTT) {
case HUB_TTTT_8_BITS:
if (hdev->descriptor.bDeviceProtocol != 0) {
hub->tt.think_time = 666;
dev_dbg(hub_dev, "TT requires at most %d "
"FS bit times (%d ns)\n",
8, hub->tt.think_time);
}
break;
case HUB_TTTT_16_BITS:
hub->tt.think_time = 666 * 2;
dev_dbg(hub_dev, "TT requires at most %d "
"FS bit times (%d ns)\n",
16, hub->tt.think_time);
break;
case HUB_TTTT_24_BITS:
hub->tt.think_time = 666 * 3;
dev_dbg(hub_dev, "TT requires at most %d "
"FS bit times (%d ns)\n",
24, hub->tt.think_time);
break;
case HUB_TTTT_32_BITS:
hub->tt.think_time = 666 * 4;
dev_dbg(hub_dev, "TT requires at most %d "
"FS bit times (%d ns)\n",
32, hub->tt.think_time);
break;
}
/* probe() zeroes hub->indicator[] */
if (wHubCharacteristics & HUB_CHAR_PORTIND) {
hub->has_indicators = 1;
dev_dbg(hub_dev, "Port indicators are supported\n");
}
dev_dbg(hub_dev, "power on to power good time: %dms\n",
hub->descriptor->bPwrOn2PwrGood * 2);
/* power budgeting mostly matters with bus-powered hubs,
* and battery-powered root hubs (may provide just 8 mA).
*/
ret = usb_get_std_status(hdev, USB_RECIP_DEVICE, 0, &hubstatus);
if (ret) {
message = "can't get hub status";
goto fail;
}
hcd = bus_to_hcd(hdev->bus);
if (hdev == hdev->bus->root_hub) {
if (hcd->power_budget > 0)
hdev->bus_mA = hcd->power_budget;
else
hdev->bus_mA = full_load * maxchild;
if (hdev->bus_mA >= full_load)
hub->mA_per_port = full_load;
else {
hub->mA_per_port = hdev->bus_mA;
hub->limited_power = 1;
}
} else if ((hubstatus & (1 << USB_DEVICE_SELF_POWERED)) == 0) {
int remaining = hdev->bus_mA -
hub->descriptor->bHubContrCurrent;
dev_dbg(hub_dev, "hub controller current requirement: %dmA\n",
hub->descriptor->bHubContrCurrent);
hub->limited_power = 1;
if (remaining < maxchild * unit_load)
dev_warn(hub_dev,
"insufficient power available "
"to use all downstream ports\n");
hub->mA_per_port = unit_load; /* 7.2.1 */
} else { /* Self-powered external hub */
/* FIXME: What about battery-powered external hubs that
* provide less current per port? */
hub->mA_per_port = full_load;
}
if (hub->mA_per_port < full_load)
dev_dbg(hub_dev, "%umA bus power budget for each child\n",
hub->mA_per_port);
ret = hub_hub_status(hub, &hubstatus, &hubchange);
if (ret < 0) {
message = "can't get hub status";
goto fail;
}
/* local power status reports aren't always correct */
if (hdev->actconfig->desc.bmAttributes & USB_CONFIG_ATT_SELFPOWER)
dev_dbg(hub_dev, "local power source is %s\n",
(hubstatus & HUB_STATUS_LOCAL_POWER)
? "lost (inactive)" : "good");
if ((wHubCharacteristics & HUB_CHAR_OCPM) == 0)
dev_dbg(hub_dev, "%sover-current condition exists\n",
(hubstatus & HUB_STATUS_OVERCURRENT) ? "" : "no ");
/* set up the interrupt endpoint
* We use the EP's maxpacket size instead of (PORTS+1+7)/8
* bytes as USB2.0[11.12.3] says because some hubs are known
* to send more data (and thus cause overflow). For root hubs,
* maxpktsize is defined in hcd.c's fake endpoint descriptors
* to be big enough for at least USB_MAXCHILDREN ports. */
pipe = usb_rcvintpipe(hdev, endpoint->bEndpointAddress);
maxp = usb_maxpacket(hdev, pipe);
if (maxp > sizeof(*hub->buffer))
maxp = sizeof(*hub->buffer);
hub->urb = usb_alloc_urb(0, GFP_KERNEL);
if (!hub->urb) {
ret = -ENOMEM;
goto fail;
}
usb_fill_int_urb(hub->urb, hdev, pipe, *hub->buffer, maxp, hub_irq,
hub, endpoint->bInterval);
/* maybe cycle the hub leds */
if (hub->has_indicators && blinkenlights)
hub->indicator[0] = INDICATOR_CYCLE;
mutex_lock(&usb_port_peer_mutex);
for (i = 0; i < maxchild; i++) {
ret = usb_hub_create_port_device(hub, i + 1);
if (ret < 0) {
dev_err(hub->intfdev,
"couldn't create port%d device.\n", i + 1);
break;
}
}
hdev->maxchild = i;
for (i = 0; i < hdev->maxchild; i++) {
struct usb_port *port_dev = hub->ports[i];
pm_runtime_put(&port_dev->dev);
}
mutex_unlock(&usb_port_peer_mutex);
if (ret < 0)
goto fail;
/* Update the HCD's internal representation of this hub before hub_wq
* starts getting port status changes for devices under the hub.
*/
if (hcd->driver->update_hub_device) {
ret = hcd->driver->update_hub_device(hcd, hdev,
&hub->tt, GFP_KERNEL);
if (ret < 0) {
message = "can't update HCD hub info";
goto fail;
}
}
usb_hub_adjust_deviceremovable(hdev, hub->descriptor);
hub_activate(hub, HUB_INIT);
return 0;
fail:
dev_err(hub_dev, "config failed, %s (err %d)\n",
message, ret);
/* hub_disconnect() frees urb and descriptor */
return ret;
}
static void hub_release(struct kref *kref)
{
struct usb_hub *hub = container_of(kref, struct usb_hub, kref);
usb_put_dev(hub->hdev);
usb_put_intf(to_usb_interface(hub->intfdev));
kfree(hub);
}
void hub_get(struct usb_hub *hub)
{
kref_get(&hub->kref);
}
void hub_put(struct usb_hub *hub)
{
kref_put(&hub->kref, hub_release);
}
static unsigned highspeed_hubs;
static void hub_disconnect(struct usb_interface *intf)
{
struct usb_hub *hub = usb_get_intfdata(intf);
struct usb_device *hdev = interface_to_usbdev(intf);
int port1;
/*
* Stop adding new hub events. We do not want to block here and thus
* will not try to remove any pending work item.
*/
hub->disconnected = 1;
/* Disconnect all children and quiesce the hub */
hub->error = 0;
hub_quiesce(hub, HUB_DISCONNECT);
mutex_lock(&usb_port_peer_mutex);
/* Avoid races with recursively_mark_NOTATTACHED() */
spin_lock_irq(&device_state_lock);
port1 = hdev->maxchild;
hdev->maxchild = 0;
usb_set_intfdata(intf, NULL);
spin_unlock_irq(&device_state_lock);
for (; port1 > 0; --port1)
usb_hub_remove_port_device(hub, port1); mutex_unlock(&usb_port_peer_mutex);
if (hub->hdev->speed == USB_SPEED_HIGH)
highspeed_hubs--; usb_free_urb(hub->urb);
kfree(hub->ports);
kfree(hub->descriptor);
kfree(hub->status);
kfree(hub->buffer);
pm_suspend_ignore_children(&intf->dev, false);
if (hub->quirk_disable_autosuspend)
usb_autopm_put_interface(intf);
onboard_dev_destroy_pdevs(&hub->onboard_devs);
hub_put(hub);
}
static bool hub_descriptor_is_sane(struct usb_host_interface *desc)
{
/* Some hubs have a subclass of 1, which AFAICT according to the */
/* specs is not defined, but it works */
if (desc->desc.bInterfaceSubClass != 0 &&
desc->desc.bInterfaceSubClass != 1)
return false;
/* Multiple endpoints? What kind of mutant ninja-hub is this? */
if (desc->desc.bNumEndpoints != 1)
return false;
/* If the first endpoint is not interrupt IN, we'd better punt! */
if (!usb_endpoint_is_int_in(&desc->endpoint[0].desc))
return false;
return true;
}
static int hub_probe(struct usb_interface *intf, const struct usb_device_id *id)
{
struct usb_host_interface *desc;
struct usb_device *hdev;
struct usb_hub *hub;
desc = intf->cur_altsetting;
hdev = interface_to_usbdev(intf);
/*
* Set default autosuspend delay as 0 to speedup bus suspend,
* based on the below considerations:
*
* - Unlike other drivers, the hub driver does not rely on the
* autosuspend delay to provide enough time to handle a wakeup
* event, and the submitted status URB is just to check future
* change on hub downstream ports, so it is safe to do it.
*
* - The patch might cause one or more auto supend/resume for
* below very rare devices when they are plugged into hub
* first time:
*
* devices having trouble initializing, and disconnect
* themselves from the bus and then reconnect a second
* or so later
*
* devices just for downloading firmware, and disconnects
* themselves after completing it
*
* For these quite rare devices, their drivers may change the
* autosuspend delay of their parent hub in the probe() to one
* appropriate value to avoid the subtle problem if someone
* does care it.
*
* - The patch may cause one or more auto suspend/resume on
* hub during running 'lsusb', but it is probably too
* infrequent to worry about.
*
* - Change autosuspend delay of hub can avoid unnecessary auto
* suspend timer for hub, also may decrease power consumption
* of USB bus.
*
* - If user has indicated to prevent autosuspend by passing
* usbcore.autosuspend = -1 then keep autosuspend disabled.
*/
#ifdef CONFIG_PM
if (hdev->dev.power.autosuspend_delay >= 0)
pm_runtime_set_autosuspend_delay(&hdev->dev, 0);
#endif
/*
* Hubs have proper suspend/resume support, except for root hubs
* where the controller driver doesn't have bus_suspend and
* bus_resume methods.
*/
if (hdev->parent) { /* normal device */
usb_enable_autosuspend(hdev);
} else { /* root hub */
const struct hc_driver *drv = bus_to_hcd(hdev->bus)->driver;
if (drv->bus_suspend && drv->bus_resume)
usb_enable_autosuspend(hdev);
}
if (hdev->level == MAX_TOPO_LEVEL) {
dev_err(&intf->dev,
"Unsupported bus topology: hub nested too deep\n");
return -E2BIG;
}
#ifdef CONFIG_USB_OTG_DISABLE_EXTERNAL_HUB
if (hdev->parent) {
dev_warn(&intf->dev, "ignoring external hub\n");
return -ENODEV;
}
#endif
if (!hub_descriptor_is_sane(desc)) {
dev_err(&intf->dev, "bad descriptor, ignoring hub\n");
return -EIO;
}
/* We found a hub */
dev_info(&intf->dev, "USB hub found\n");
hub = kzalloc(sizeof(*hub), GFP_KERNEL);
if (!hub)
return -ENOMEM;
kref_init(&hub->kref);
hub->intfdev = &intf->dev;
hub->hdev = hdev;
INIT_DELAYED_WORK(&hub->leds, led_work);
INIT_DELAYED_WORK(&hub->init_work, NULL);
INIT_WORK(&hub->events, hub_event);
INIT_LIST_HEAD(&hub->onboard_devs);
spin_lock_init(&hub->irq_urb_lock);
timer_setup(&hub->irq_urb_retry, hub_retry_irq_urb, 0);
usb_get_intf(intf);
usb_get_dev(hdev);
usb_set_intfdata(intf, hub);
intf->needs_remote_wakeup = 1;
pm_suspend_ignore_children(&intf->dev, true);
if (hdev->speed == USB_SPEED_HIGH)
highspeed_hubs++;
if (id->driver_info & HUB_QUIRK_CHECK_PORT_AUTOSUSPEND)
hub->quirk_check_port_auto_suspend = 1;
if (id->driver_info & HUB_QUIRK_DISABLE_AUTOSUSPEND) {
hub->quirk_disable_autosuspend = 1;
usb_autopm_get_interface_no_resume(intf);
}
if ((id->driver_info & HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL) &&
desc->endpoint[0].desc.bInterval > USB_REDUCE_FRAME_INTR_BINTERVAL) {
desc->endpoint[0].desc.bInterval =
USB_REDUCE_FRAME_INTR_BINTERVAL;
/* Tell the HCD about the interrupt ep's new bInterval */
usb_set_interface(hdev, 0, 0);
}
if (hub_configure(hub, &desc->endpoint[0].desc) >= 0) {
onboard_dev_create_pdevs(hdev, &hub->onboard_devs);
return 0;
}
hub_disconnect(intf);
return -ENODEV;
}
static int
hub_ioctl(struct usb_interface *intf, unsigned int code, void *user_data)
{
struct usb_device *hdev = interface_to_usbdev(intf);
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
/* assert ifno == 0 (part of hub spec) */
switch (code) {
case USBDEVFS_HUB_PORTINFO: {
struct usbdevfs_hub_portinfo *info = user_data;
int i;
spin_lock_irq(&device_state_lock);
if (hdev->devnum <= 0)
info->nports = 0;
else {
info->nports = hdev->maxchild;
for (i = 0; i < info->nports; i++) {
if (hub->ports[i]->child == NULL)
info->port[i] = 0;
else
info->port[i] =
hub->ports[i]->child->devnum;
}
}
spin_unlock_irq(&device_state_lock);
return info->nports + 1;
}
default:
return -ENOSYS;
}
}
/*
* Allow user programs to claim ports on a hub. When a device is attached
* to one of these "claimed" ports, the program will "own" the device.
*/
static int find_port_owner(struct usb_device *hdev, unsigned port1,
struct usb_dev_state ***ppowner)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
if (hdev->state == USB_STATE_NOTATTACHED)
return -ENODEV;
if (port1 == 0 || port1 > hdev->maxchild)
return -EINVAL;
/* Devices not managed by the hub driver
* will always have maxchild equal to 0.
*/
*ppowner = &(hub->ports[port1 - 1]->port_owner);
return 0;
}
/* In the following three functions, the caller must hold hdev's lock */
int usb_hub_claim_port(struct usb_device *hdev, unsigned port1,
struct usb_dev_state *owner)
{
int rc;
struct usb_dev_state **powner;
rc = find_port_owner(hdev, port1, &powner);
if (rc)
return rc;
if (*powner)
return -EBUSY;
*powner = owner;
return rc;
}
EXPORT_SYMBOL_GPL(usb_hub_claim_port);
int usb_hub_release_port(struct usb_device *hdev, unsigned port1,
struct usb_dev_state *owner)
{
int rc;
struct usb_dev_state **powner;
rc = find_port_owner(hdev, port1, &powner);
if (rc)
return rc;
if (*powner != owner)
return -ENOENT;
*powner = NULL;
return rc;
}
EXPORT_SYMBOL_GPL(usb_hub_release_port);
void usb_hub_release_all_ports(struct usb_device *hdev, struct usb_dev_state *owner)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
int n;
for (n = 0; n < hdev->maxchild; n++) {
if (hub->ports[n]->port_owner == owner)
hub->ports[n]->port_owner = NULL;
}
}
/* The caller must hold udev's lock */
bool usb_device_is_owned(struct usb_device *udev)
{
struct usb_hub *hub;
if (udev->state == USB_STATE_NOTATTACHED || !udev->parent)
return false;
hub = usb_hub_to_struct_hub(udev->parent); return !!hub->ports[udev->portnum - 1]->port_owner;}
static void update_port_device_state(struct usb_device *udev)
{
struct usb_hub *hub;
struct usb_port *port_dev;
if (udev->parent) { hub = usb_hub_to_struct_hub(udev->parent);
/*
* The Link Layer Validation System Driver (lvstest)
* has a test step to unbind the hub before running the
* rest of the procedure. This triggers hub_disconnect
* which will set the hub's maxchild to 0, further
* resulting in usb_hub_to_struct_hub returning NULL.
*/
if (hub) {
port_dev = hub->ports[udev->portnum - 1];
WRITE_ONCE(port_dev->state, udev->state);
sysfs_notify_dirent(port_dev->state_kn);
}
}
}
static void recursively_mark_NOTATTACHED(struct usb_device *udev)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev);
int i;
for (i = 0; i < udev->maxchild; ++i) { if (hub->ports[i]->child) recursively_mark_NOTATTACHED(hub->ports[i]->child);
}
if (udev->state == USB_STATE_SUSPENDED) udev->active_duration -= jiffies; udev->state = USB_STATE_NOTATTACHED;
update_port_device_state(udev);
}
/**
* usb_set_device_state - change a device's current state (usbcore, hcds)
* @udev: pointer to device whose state should be changed
* @new_state: new state value to be stored
*
* udev->state is _not_ fully protected by the device lock. Although
* most transitions are made only while holding the lock, the state can
* can change to USB_STATE_NOTATTACHED at almost any time. This
* is so that devices can be marked as disconnected as soon as possible,
* without having to wait for any semaphores to be released. As a result,
* all changes to any device's state must be protected by the
* device_state_lock spinlock.
*
* Once a device has been added to the device tree, all changes to its state
* should be made using this routine. The state should _not_ be set directly.
*
* If udev->state is already USB_STATE_NOTATTACHED then no change is made.
* Otherwise udev->state is set to new_state, and if new_state is
* USB_STATE_NOTATTACHED then all of udev's descendants' states are also set
* to USB_STATE_NOTATTACHED.
*/
void usb_set_device_state(struct usb_device *udev,
enum usb_device_state new_state)
{
unsigned long flags;
int wakeup = -1;
spin_lock_irqsave(&device_state_lock, flags);
if (udev->state == USB_STATE_NOTATTACHED)
; /* do nothing */
else if (new_state != USB_STATE_NOTATTACHED) {
/* root hub wakeup capabilities are managed out-of-band
* and may involve silicon errata ... ignore them here.
*/
if (udev->parent) { if (udev->state == USB_STATE_SUSPENDED || new_state == USB_STATE_SUSPENDED)
; /* No change to wakeup settings */
else if (new_state == USB_STATE_CONFIGURED) wakeup = (udev->quirks &
USB_QUIRK_IGNORE_REMOTE_WAKEUP) ? 0 :
udev->actconfig->desc.bmAttributes &
USB_CONFIG_ATT_WAKEUP;
else
wakeup = 0;
}
if (udev->state == USB_STATE_SUSPENDED &&
new_state != USB_STATE_SUSPENDED)
udev->active_duration -= jiffies; else if (new_state == USB_STATE_SUSPENDED &&
udev->state != USB_STATE_SUSPENDED)
udev->active_duration += jiffies; udev->state = new_state;
update_port_device_state(udev);
} else
recursively_mark_NOTATTACHED(udev); spin_unlock_irqrestore(&device_state_lock, flags);
if (wakeup >= 0)
device_set_wakeup_capable(&udev->dev, wakeup);}
EXPORT_SYMBOL_GPL(usb_set_device_state);
/*
* Choose a device number.
*
* Device numbers are used as filenames in usbfs. On USB-1.1 and
* USB-2.0 buses they are also used as device addresses, however on
* USB-3.0 buses the address is assigned by the controller hardware
* and it usually is not the same as the device number.
*
* Devices connected under xHCI are not as simple. The host controller
* supports virtualization, so the hardware assigns device addresses and
* the HCD must setup data structures before issuing a set address
* command to the hardware.
*/
static void choose_devnum(struct usb_device *udev)
{
int devnum;
struct usb_bus *bus = udev->bus;
/* be safe when more hub events are proceed in parallel */
mutex_lock(&bus->devnum_next_mutex);
/* Try to allocate the next devnum beginning at bus->devnum_next. */
devnum = find_next_zero_bit(bus->devmap, 128, bus->devnum_next);
if (devnum >= 128)
devnum = find_next_zero_bit(bus->devmap, 128, 1); bus->devnum_next = (devnum >= 127 ? 1 : devnum + 1);
if (devnum < 128) {
set_bit(devnum, bus->devmap);
udev->devnum = devnum;
}
mutex_unlock(&bus->devnum_next_mutex);
}
static void release_devnum(struct usb_device *udev)
{
if (udev->devnum > 0) { clear_bit(udev->devnum, udev->bus->devmap);
udev->devnum = -1;
}
}
static void update_devnum(struct usb_device *udev, int devnum)
{
udev->devnum = devnum;
if (!udev->devaddr)
udev->devaddr = (u8)devnum;
}
static void hub_free_dev(struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
/* Root hubs aren't real devices, so don't free HCD resources */
if (hcd->driver->free_dev && udev->parent) hcd->driver->free_dev(hcd, udev);
}
static void hub_disconnect_children(struct usb_device *udev)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev);
int i;
/* Free up all the children before we remove this device */
for (i = 0; i < udev->maxchild; i++) { if (hub->ports[i]->child) usb_disconnect(&hub->ports[i]->child);
}
}
/**
* usb_disconnect - disconnect a device (usbcore-internal)
* @pdev: pointer to device being disconnected
*
* Context: task context, might sleep
*
* Something got disconnected. Get rid of it and all of its children.
*
* If *pdev is a normal device then the parent hub must already be locked.
* If *pdev is a root hub then the caller must hold the usb_bus_idr_lock,
* which protects the set of root hubs as well as the list of buses.
*
* Only hub drivers (including virtual root hub drivers for host
* controllers) should ever call this.
*
* This call is synchronous, and may not be used in an interrupt context.
*/
void usb_disconnect(struct usb_device **pdev)
{
struct usb_port *port_dev = NULL;
struct usb_device *udev = *pdev;
struct usb_hub *hub = NULL;
int port1 = 1;
/* mark the device as inactive, so any further urb submissions for
* this device (and any of its children) will fail immediately.
* this quiesces everything except pending urbs.
*/
usb_set_device_state(udev, USB_STATE_NOTATTACHED);
dev_info(&udev->dev, "USB disconnect, device number %d\n",
udev->devnum);
/*
* Ensure that the pm runtime code knows that the USB device
* is in the process of being disconnected.
*/
pm_runtime_barrier(&udev->dev);
usb_lock_device(udev);
hub_disconnect_children(udev);
/* deallocate hcd/hardware state ... nuking all pending urbs and
* cleaning up all state associated with the current configuration
* so that the hardware is now fully quiesced.
*/
dev_dbg(&udev->dev, "unregistering device\n"); usb_disable_device(udev, 0);
usb_hcd_synchronize_unlinks(udev);
if (udev->parent) {
port1 = udev->portnum; hub = usb_hub_to_struct_hub(udev->parent); port_dev = hub->ports[port1 - 1];
sysfs_remove_link(&udev->dev.kobj, "port");
sysfs_remove_link(&port_dev->dev.kobj, "device");
/*
* As usb_port_runtime_resume() de-references udev, make
* sure no resumes occur during removal
*/
if (!test_and_set_bit(port1, hub->child_usage_bits))
pm_runtime_get_sync(&port_dev->dev); typec_deattach(port_dev->connector, &udev->dev);
}
usb_remove_ep_devs(&udev->ep0);
usb_unlock_device(udev);
/* Unregister the device. The device driver is responsible
* for de-configuring the device and invoking the remove-device
* notifier chain (used by usbfs and possibly others).
*/
device_del(&udev->dev);
/* Free the device number and delete the parent's children[]
* (or root_hub) pointer.
*/
release_devnum(udev);
/* Avoid races with recursively_mark_NOTATTACHED() */
spin_lock_irq(&device_state_lock);
*pdev = NULL;
spin_unlock_irq(&device_state_lock);
if (port_dev && test_and_clear_bit(port1, hub->child_usage_bits)) pm_runtime_put(&port_dev->dev); hub_free_dev(udev); put_device(&udev->dev);
}
#ifdef CONFIG_USB_ANNOUNCE_NEW_DEVICES
static void show_string(struct usb_device *udev, char *id, char *string)
{
if (!string)
return;
dev_info(&udev->dev, "%s: %s\n", id, string);
}
static void announce_device(struct usb_device *udev)
{
u16 bcdDevice = le16_to_cpu(udev->descriptor.bcdDevice);
dev_info(&udev->dev,
"New USB device found, idVendor=%04x, idProduct=%04x, bcdDevice=%2x.%02x\n",
le16_to_cpu(udev->descriptor.idVendor),
le16_to_cpu(udev->descriptor.idProduct),
bcdDevice >> 8, bcdDevice & 0xff);
dev_info(&udev->dev,
"New USB device strings: Mfr=%d, Product=%d, SerialNumber=%d\n",
udev->descriptor.iManufacturer,
udev->descriptor.iProduct,
udev->descriptor.iSerialNumber);
show_string(udev, "Product", udev->product); show_string(udev, "Manufacturer", udev->manufacturer); show_string(udev, "SerialNumber", udev->serial);
}
#else
static inline void announce_device(struct usb_device *udev) { }
#endif
/**
* usb_enumerate_device_otg - FIXME (usbcore-internal)
* @udev: newly addressed device (in ADDRESS state)
*
* Finish enumeration for On-The-Go devices
*
* Return: 0 if successful. A negative error code otherwise.
*/
static int usb_enumerate_device_otg(struct usb_device *udev)
{
int err = 0;
#ifdef CONFIG_USB_OTG
/*
* OTG-aware devices on OTG-capable root hubs may be able to use SRP,
* to wake us after we've powered off VBUS; and HNP, switching roles
* "host" to "peripheral". The OTG descriptor helps figure this out.
*/
if (!udev->bus->is_b_host
&& udev->config && udev->parent == udev->bus->root_hub) { struct usb_otg_descriptor *desc = NULL;
struct usb_bus *bus = udev->bus;
unsigned port1 = udev->portnum;
/* descriptor may appear anywhere in config */
err = __usb_get_extra_descriptor(udev->rawdescriptors[0],
le16_to_cpu(udev->config[0].desc.wTotalLength),
USB_DT_OTG, (void **) &desc, sizeof(*desc));
if (err || !(desc->bmAttributes & USB_OTG_HNP)) return 0; dev_info(&udev->dev, "Dual-Role OTG device on %sHNP port\n",
(port1 == bus->otg_port) ? "" : "non-");
/* enable HNP before suspend, it's simpler */
if (port1 == bus->otg_port) {
bus->b_hnp_enable = 1;
err = usb_control_msg(udev,
usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE, 0,
USB_DEVICE_B_HNP_ENABLE,
0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
if (err < 0) {
/*
* OTG MESSAGE: report errors here,
* customize to match your product.
*/
dev_err(&udev->dev, "can't set HNP mode: %d\n",
err);
bus->b_hnp_enable = 0;
}
} else if (desc->bLength == sizeof
(struct usb_otg_descriptor)) {
/*
* We are operating on a legacy OTP device
* These should be told that they are operating
* on the wrong port if we have another port that does
* support HNP
*/
if (bus->otg_port != 0) {
/* Set a_alt_hnp_support for legacy otg device */
err = usb_control_msg(udev,
usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE, 0,
USB_DEVICE_A_ALT_HNP_SUPPORT,
0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
if (err < 0)
dev_err(&udev->dev,
"set a_alt_hnp_support failed: %d\n",
err);
}
}
}
#endif
return err;
}
/**
* usb_enumerate_device - Read device configs/intfs/otg (usbcore-internal)
* @udev: newly addressed device (in ADDRESS state)
*
* This is only called by usb_new_device() -- all comments that apply there
* apply here wrt to environment.
*
* If the device is WUSB and not authorized, we don't attempt to read
* the string descriptors, as they will be errored out by the device
* until it has been authorized.
*
* Return: 0 if successful. A negative error code otherwise.
*/
static int usb_enumerate_device(struct usb_device *udev)
{
int err;
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
if (udev->config == NULL) {
err = usb_get_configuration(udev);
if (err < 0) {
if (err != -ENODEV) dev_err(&udev->dev, "can't read configurations, error %d\n",
err);
return err;
}
}
/* read the standard strings and cache them if present */
udev->product = usb_cache_string(udev, udev->descriptor.iProduct);
udev->manufacturer = usb_cache_string(udev,
udev->descriptor.iManufacturer);
udev->serial = usb_cache_string(udev, udev->descriptor.iSerialNumber);
err = usb_enumerate_device_otg(udev);
if (err < 0)
return err;
if (IS_ENABLED(CONFIG_USB_OTG_PRODUCTLIST) && hcd->tpl_support &&
!is_targeted(udev)) {
/* Maybe it can talk to us, though we can't talk to it.
* (Includes HNP test device.)
*/
if (IS_ENABLED(CONFIG_USB_OTG) && (udev->bus->b_hnp_enable
|| udev->bus->is_b_host)) {
err = usb_port_suspend(udev, PMSG_AUTO_SUSPEND);
if (err < 0)
dev_dbg(&udev->dev, "HNP fail, %d\n", err);
}
return -ENOTSUPP;
}
usb_detect_interface_quirks(udev);
return 0;
}
static void set_usb_port_removable(struct usb_device *udev)
{
struct usb_device *hdev = udev->parent;
struct usb_hub *hub;
u8 port = udev->portnum;
u16 wHubCharacteristics;
bool removable = true;
dev_set_removable(&udev->dev, DEVICE_REMOVABLE_UNKNOWN);
if (!hdev)
return;
hub = usb_hub_to_struct_hub(udev->parent);
/*
* If the platform firmware has provided information about a port,
* use that to determine whether it's removable.
*/
switch (hub->ports[udev->portnum - 1]->connect_type) {
case USB_PORT_CONNECT_TYPE_HOT_PLUG:
dev_set_removable(&udev->dev, DEVICE_REMOVABLE);
return;
case USB_PORT_CONNECT_TYPE_HARD_WIRED:
case USB_PORT_NOT_USED:
dev_set_removable(&udev->dev, DEVICE_FIXED);
return;
default:
break;
}
/*
* Otherwise, check whether the hub knows whether a port is removable
* or not
*/
wHubCharacteristics = le16_to_cpu(hub->descriptor->wHubCharacteristics);
if (!(wHubCharacteristics & HUB_CHAR_COMPOUND))
return;
if (hub_is_superspeed(hdev)) { if (le16_to_cpu(hub->descriptor->u.ss.DeviceRemovable)
& (1 << port))
removable = false;
} else {
if (hub->descriptor->u.hs.DeviceRemovable[port / 8] & (1 << (port % 8)))
removable = false;
}
if (removable)
dev_set_removable(&udev->dev, DEVICE_REMOVABLE);
else
dev_set_removable(&udev->dev, DEVICE_FIXED);
}
/**
* usb_new_device - perform initial device setup (usbcore-internal)
* @udev: newly addressed device (in ADDRESS state)
*
* This is called with devices which have been detected but not fully
* enumerated. The device descriptor is available, but not descriptors
* for any device configuration. The caller must have locked either
* the parent hub (if udev is a normal device) or else the
* usb_bus_idr_lock (if udev is a root hub). The parent's pointer to
* udev has already been installed, but udev is not yet visible through
* sysfs or other filesystem code.
*
* This call is synchronous, and may not be used in an interrupt context.
*
* Only the hub driver or root-hub registrar should ever call this.
*
* Return: Whether the device is configured properly or not. Zero if the
* interface was registered with the driver core; else a negative errno
* value.
*
*/
int usb_new_device(struct usb_device *udev)
{
int err;
if (udev->parent) {
/* Initialize non-root-hub device wakeup to disabled;
* device (un)configuration controls wakeup capable
* sysfs power/wakeup controls wakeup enabled/disabled
*/
device_init_wakeup(&udev->dev, 0);
}
/* Tell the runtime-PM framework the device is active */
pm_runtime_set_active(&udev->dev);
pm_runtime_get_noresume(&udev->dev);
pm_runtime_use_autosuspend(&udev->dev);
pm_runtime_enable(&udev->dev);
/* By default, forbid autosuspend for all devices. It will be
* allowed for hubs during binding.
*/
usb_disable_autosuspend(udev);
err = usb_enumerate_device(udev); /* Read descriptors */
if (err < 0)
goto fail;
dev_dbg(&udev->dev, "udev %d, busnum %d, minor = %d\n",
udev->devnum, udev->bus->busnum,
(((udev->bus->busnum-1) * 128) + (udev->devnum-1)));
/* export the usbdev device-node for libusb */
udev->dev.devt = MKDEV(USB_DEVICE_MAJOR,
(((udev->bus->busnum-1) * 128) + (udev->devnum-1)));
/* Tell the world! */
announce_device(udev);
if (udev->serial)
add_device_randomness(udev->serial, strlen(udev->serial)); if (udev->product) add_device_randomness(udev->product, strlen(udev->product)); if (udev->manufacturer) add_device_randomness(udev->manufacturer,
strlen(udev->manufacturer));
device_enable_async_suspend(&udev->dev);
/* check whether the hub or firmware marks this port as non-removable */
set_usb_port_removable(udev);
/* Register the device. The device driver is responsible
* for configuring the device and invoking the add-device
* notifier chain (used by usbfs and possibly others).
*/
err = device_add(&udev->dev);
if (err) {
dev_err(&udev->dev, "can't device_add, error %d\n", err);
goto fail;
}
/* Create link files between child device and usb port device. */
if (udev->parent) { struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent); int port1 = udev->portnum;
struct usb_port *port_dev = hub->ports[port1 - 1];
err = sysfs_create_link(&udev->dev.kobj,
&port_dev->dev.kobj, "port");
if (err)
goto fail;
err = sysfs_create_link(&port_dev->dev.kobj,
&udev->dev.kobj, "device");
if (err) {
sysfs_remove_link(&udev->dev.kobj, "port");
goto fail;
}
if (!test_and_set_bit(port1, hub->child_usage_bits)) pm_runtime_get_sync(&port_dev->dev); typec_attach(port_dev->connector, &udev->dev);
}
(void) usb_create_ep_devs(&udev->dev, &udev->ep0, udev);
usb_mark_last_busy(udev);
pm_runtime_put_sync_autosuspend(&udev->dev);
return err;
fail:
usb_set_device_state(udev, USB_STATE_NOTATTACHED);
pm_runtime_disable(&udev->dev);
pm_runtime_set_suspended(&udev->dev);
return err;
}
/**
* usb_deauthorize_device - deauthorize a device (usbcore-internal)
* @usb_dev: USB device
*
* Move the USB device to a very basic state where interfaces are disabled
* and the device is in fact unconfigured and unusable.
*
* We share a lock (that we have) with device_del(), so we need to
* defer its call.
*
* Return: 0.
*/
int usb_deauthorize_device(struct usb_device *usb_dev)
{
usb_lock_device(usb_dev);
if (usb_dev->authorized == 0)
goto out_unauthorized;
usb_dev->authorized = 0;
usb_set_configuration(usb_dev, -1);
out_unauthorized:
usb_unlock_device(usb_dev);
return 0;
}
int usb_authorize_device(struct usb_device *usb_dev)
{
int result = 0, c;
usb_lock_device(usb_dev);
if (usb_dev->authorized == 1)
goto out_authorized;
result = usb_autoresume_device(usb_dev);
if (result < 0) {
dev_err(&usb_dev->dev,
"can't autoresume for authorization: %d\n", result);
goto error_autoresume;
}
usb_dev->authorized = 1;
/* Choose and set the configuration. This registers the interfaces
* with the driver core and lets interface drivers bind to them.
*/
c = usb_choose_configuration(usb_dev);
if (c >= 0) {
result = usb_set_configuration(usb_dev, c);
if (result) {
dev_err(&usb_dev->dev,
"can't set config #%d, error %d\n", c, result);
/* This need not be fatal. The user can try to
* set other configurations. */
}
}
dev_info(&usb_dev->dev, "authorized to connect\n");
usb_autosuspend_device(usb_dev);
error_autoresume:
out_authorized:
usb_unlock_device(usb_dev); /* complements locktree */
return result;
}
/**
* get_port_ssp_rate - Match the extended port status to SSP rate
* @hdev: The hub device
* @ext_portstatus: extended port status
*
* Match the extended port status speed id to the SuperSpeed Plus sublink speed
* capability attributes. Base on the number of connected lanes and speed,
* return the corresponding enum usb_ssp_rate.
*/
static enum usb_ssp_rate get_port_ssp_rate(struct usb_device *hdev,
u32 ext_portstatus)
{
struct usb_ssp_cap_descriptor *ssp_cap;
u32 attr;
u8 speed_id;
u8 ssac;
u8 lanes;
int i;
if (!hdev->bos)
goto out;
ssp_cap = hdev->bos->ssp_cap;
if (!ssp_cap)
goto out;
speed_id = ext_portstatus & USB_EXT_PORT_STAT_RX_SPEED_ID; lanes = USB_EXT_PORT_RX_LANES(ext_portstatus) + 1;
ssac = le32_to_cpu(ssp_cap->bmAttributes) &
USB_SSP_SUBLINK_SPEED_ATTRIBS;
for (i = 0; i <= ssac; i++) {
u8 ssid;
attr = le32_to_cpu(ssp_cap->bmSublinkSpeedAttr[i]);
ssid = FIELD_GET(USB_SSP_SUBLINK_SPEED_SSID, attr);
if (speed_id == ssid) {
u16 mantissa;
u8 lse;
u8 type;
/*
* Note: currently asymmetric lane types are only
* applicable for SSIC operate in SuperSpeed protocol
*/
type = FIELD_GET(USB_SSP_SUBLINK_SPEED_ST, attr);
if (type == USB_SSP_SUBLINK_SPEED_ST_ASYM_RX ||
type == USB_SSP_SUBLINK_SPEED_ST_ASYM_TX)
goto out;
if (FIELD_GET(USB_SSP_SUBLINK_SPEED_LP, attr) !=
USB_SSP_SUBLINK_SPEED_LP_SSP)
goto out;
lse = FIELD_GET(USB_SSP_SUBLINK_SPEED_LSE, attr);
mantissa = FIELD_GET(USB_SSP_SUBLINK_SPEED_LSM, attr);
/* Convert to Gbps */
for (; lse < USB_SSP_SUBLINK_SPEED_LSE_GBPS; lse++)
mantissa /= 1000;
if (mantissa >= 10 && lanes == 1)
return USB_SSP_GEN_2x1;
if (mantissa >= 10 && lanes == 2)
return USB_SSP_GEN_2x2;
if (mantissa >= 5 && lanes == 2)
return USB_SSP_GEN_1x2;
goto out;
}
}
out:
return USB_SSP_GEN_UNKNOWN;
}
#ifdef CONFIG_USB_FEW_INIT_RETRIES
#define PORT_RESET_TRIES 2
#define SET_ADDRESS_TRIES 1
#define GET_DESCRIPTOR_TRIES 1
#define GET_MAXPACKET0_TRIES 1
#define PORT_INIT_TRIES 4
#else
#define PORT_RESET_TRIES 5
#define SET_ADDRESS_TRIES 2
#define GET_DESCRIPTOR_TRIES 2
#define GET_MAXPACKET0_TRIES 3
#define PORT_INIT_TRIES 4
#endif /* CONFIG_USB_FEW_INIT_RETRIES */
#define DETECT_DISCONNECT_TRIES 5
#define HUB_ROOT_RESET_TIME 60 /* times are in msec */
#define HUB_SHORT_RESET_TIME 10
#define HUB_BH_RESET_TIME 50
#define HUB_LONG_RESET_TIME 200
#define HUB_RESET_TIMEOUT 800
static bool use_new_scheme(struct usb_device *udev, int retry,
struct usb_port *port_dev)
{
int old_scheme_first_port =
(port_dev->quirks & USB_PORT_QUIRK_OLD_SCHEME) ||
old_scheme_first;
/*
* "New scheme" enumeration causes an extra state transition to be
* exposed to an xhci host and causes USB3 devices to receive control
* commands in the default state. This has been seen to cause
* enumeration failures, so disable this enumeration scheme for USB3
* devices.
*/
if (udev->speed >= USB_SPEED_SUPER)
return false;
/*
* If use_both_schemes is set, use the first scheme (whichever
* it is) for the larger half of the retries, then use the other
* scheme. Otherwise, use the first scheme for all the retries.
*/
if (use_both_schemes && retry >= (PORT_INIT_TRIES + 1) / 2) return old_scheme_first_port; /* Second half */ return !old_scheme_first_port; /* First half or all */
}
/* Is a USB 3.0 port in the Inactive or Compliance Mode state?
* Port warm reset is required to recover
*/
static bool hub_port_warm_reset_required(struct usb_hub *hub, int port1,
u16 portstatus)
{
u16 link_state;
if (!hub_is_superspeed(hub->hdev))
return false;
if (test_bit(port1, hub->warm_reset_bits))
return true;
link_state = portstatus & USB_PORT_STAT_LINK_STATE;
return link_state == USB_SS_PORT_LS_SS_INACTIVE
|| link_state == USB_SS_PORT_LS_COMP_MOD;
}
static int hub_port_wait_reset(struct usb_hub *hub, int port1,
struct usb_device *udev, unsigned int delay, bool warm)
{
int delay_time, ret;
u16 portstatus;
u16 portchange;
u32 ext_portstatus = 0;
for (delay_time = 0;
delay_time < HUB_RESET_TIMEOUT;
delay_time += delay) {
/* wait to give the device a chance to reset */
msleep(delay);
/* read and decode port status */
if (hub_is_superspeedplus(hub->hdev)) ret = hub_ext_port_status(hub, port1,
HUB_EXT_PORT_STATUS,
&portstatus, &portchange,
&ext_portstatus);
else
ret = usb_hub_port_status(hub, port1, &portstatus,
&portchange);
if (ret < 0)
return ret;
/*
* The port state is unknown until the reset completes.
*
* On top of that, some chips may require additional time
* to re-establish a connection after the reset is complete,
* so also wait for the connection to be re-established.
*/
if (!(portstatus & USB_PORT_STAT_RESET) &&
(portstatus & USB_PORT_STAT_CONNECTION))
break;
/* switch to the long delay after two short delay failures */
if (delay_time >= 2 * HUB_SHORT_RESET_TIME)
delay = HUB_LONG_RESET_TIME;
dev_dbg(&hub->ports[port1 - 1]->dev,
"not %sreset yet, waiting %dms\n",
warm ? "warm " : "", delay);
}
if ((portstatus & USB_PORT_STAT_RESET))
return -EBUSY;
if (hub_port_warm_reset_required(hub, port1, portstatus))
return -ENOTCONN;
/* Device went away? */
if (!(portstatus & USB_PORT_STAT_CONNECTION))
return -ENOTCONN;
/* Retry if connect change is set but status is still connected.
* A USB 3.0 connection may bounce if multiple warm resets were issued,
* but the device may have successfully re-connected. Ignore it.
*/
if (!hub_is_superspeed(hub->hdev) && (portchange & USB_PORT_STAT_C_CONNECTION)) { usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_CONNECTION);
return -EAGAIN;
}
if (!(portstatus & USB_PORT_STAT_ENABLE))
return -EBUSY;
if (!udev)
return 0;
if (hub_is_superspeedplus(hub->hdev)) {
/* extended portstatus Rx and Tx lane count are zero based */
udev->rx_lanes = USB_EXT_PORT_RX_LANES(ext_portstatus) + 1;
udev->tx_lanes = USB_EXT_PORT_TX_LANES(ext_portstatus) + 1;
udev->ssp_rate = get_port_ssp_rate(hub->hdev, ext_portstatus);
} else {
udev->rx_lanes = 1;
udev->tx_lanes = 1;
udev->ssp_rate = USB_SSP_GEN_UNKNOWN;
}
if (udev->ssp_rate != USB_SSP_GEN_UNKNOWN)
udev->speed = USB_SPEED_SUPER_PLUS; else if (hub_is_superspeed(hub->hdev)) udev->speed = USB_SPEED_SUPER; else if (portstatus & USB_PORT_STAT_HIGH_SPEED) udev->speed = USB_SPEED_HIGH; else if (portstatus & USB_PORT_STAT_LOW_SPEED) udev->speed = USB_SPEED_LOW;
else
udev->speed = USB_SPEED_FULL;
return 0;
}
/* Handle port reset and port warm(BH) reset (for USB3 protocol ports) */
static int hub_port_reset(struct usb_hub *hub, int port1,
struct usb_device *udev, unsigned int delay, bool warm)
{
int i, status;
u16 portchange, portstatus;
struct usb_port *port_dev = hub->ports[port1 - 1];
int reset_recovery_time;
if (!hub_is_superspeed(hub->hdev)) {
if (warm) { dev_err(hub->intfdev, "only USB3 hub support "
"warm reset\n");
return -EINVAL;
}
/* Block EHCI CF initialization during the port reset.
* Some companion controllers don't like it when they mix.
*/
down_read(&ehci_cf_port_reset_rwsem); } else if (!warm) {
/*
* If the caller hasn't explicitly requested a warm reset,
* double check and see if one is needed.
*/
if (usb_hub_port_status(hub, port1, &portstatus,
&portchange) == 0)
if (hub_port_warm_reset_required(hub, port1,
portstatus))
warm = true;
}
clear_bit(port1, hub->warm_reset_bits);
/* Reset the port */
for (i = 0; i < PORT_RESET_TRIES; i++) { status = set_port_feature(hub->hdev, port1, (warm ?
USB_PORT_FEAT_BH_PORT_RESET :
USB_PORT_FEAT_RESET));
if (status == -ENODEV) {
; /* The hub is gone */
} else if (status) { dev_err(&port_dev->dev,
"cannot %sreset (err = %d)\n",
warm ? "warm " : "", status);
} else {
status = hub_port_wait_reset(hub, port1, udev, delay,
warm);
if (status && status != -ENOTCONN && status != -ENODEV) dev_dbg(hub->intfdev,
"port_wait_reset: err = %d\n",
status);
}
/*
* Check for disconnect or reset, and bail out after several
* reset attempts to avoid warm reset loop.
*/
if (status == 0 || status == -ENOTCONN || status == -ENODEV || (status == -EBUSY && i == PORT_RESET_TRIES - 1)) { usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_RESET);
if (!hub_is_superspeed(hub->hdev))
goto done;
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_BH_PORT_RESET);
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_PORT_LINK_STATE);
if (udev)
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_CONNECTION);
/*
* If a USB 3.0 device migrates from reset to an error
* state, re-issue the warm reset.
*/
if (usb_hub_port_status(hub, port1,
&portstatus, &portchange) < 0)
goto done;
if (!hub_port_warm_reset_required(hub, port1,
portstatus))
goto done;
/*
* If the port is in SS.Inactive or Compliance Mode, the
* hot or warm reset failed. Try another warm reset.
*/
if (!warm) { dev_dbg(&port_dev->dev,
"hot reset failed, warm reset\n");
warm = true;
}
}
dev_dbg(&port_dev->dev,
"not enabled, trying %sreset again...\n",
warm ? "warm " : "");
delay = HUB_LONG_RESET_TIME;
}
dev_err(&port_dev->dev, "Cannot enable. Maybe the USB cable is bad?\n");
done:
if (status == 0) { if (port_dev->quirks & USB_PORT_QUIRK_FAST_ENUM) usleep_range(10000, 12000);
else {
/* TRSTRCY = 10 ms; plus some extra */
reset_recovery_time = 10 + 40;
/* Hub needs extra delay after resetting its port. */
if (hub->hdev->quirks & USB_QUIRK_HUB_SLOW_RESET)
reset_recovery_time += 100;
msleep(reset_recovery_time);
}
if (udev) { struct usb_hcd *hcd = bus_to_hcd(udev->bus);
update_devnum(udev, 0);
/* The xHC may think the device is already reset,
* so ignore the status.
*/
if (hcd->driver->reset_device)
hcd->driver->reset_device(hcd, udev); usb_set_device_state(udev, USB_STATE_DEFAULT);
}
} else {
if (udev) usb_set_device_state(udev, USB_STATE_NOTATTACHED);
}
if (!hub_is_superspeed(hub->hdev)) up_read(&ehci_cf_port_reset_rwsem);
return status;
}
/*
* hub_port_stop_enumerate - stop USB enumeration or ignore port events
* @hub: target hub
* @port1: port num of the port
* @retries: port retries number of hub_port_init()
*
* Return:
* true: ignore port actions/events or give up connection attempts.
* false: keep original behavior.
*
* This function will be based on retries to check whether the port which is
* marked with early_stop attribute would stop enumeration or ignore events.
*
* Note:
* This function didn't change anything if early_stop is not set, and it will
* prevent all connection attempts when early_stop is set and the attempts of
* the port are more than 1.
*/
static bool hub_port_stop_enumerate(struct usb_hub *hub, int port1, int retries)
{
struct usb_port *port_dev = hub->ports[port1 - 1];
if (port_dev->early_stop) {
if (port_dev->ignore_event)
return true;
/*
* We want unsuccessful attempts to fail quickly.
* Since some devices may need one failure during
* port initialization, we allow two tries but no
* more.
*/
if (retries < 2)
return false;
port_dev->ignore_event = 1;
} else
port_dev->ignore_event = 0;
return port_dev->ignore_event;
}
/* Check if a port is power on */
int usb_port_is_power_on(struct usb_hub *hub, unsigned int portstatus)
{
int ret = 0;
if (hub_is_superspeed(hub->hdev)) { if (portstatus & USB_SS_PORT_STAT_POWER)
ret = 1;
} else {
if (portstatus & USB_PORT_STAT_POWER)
ret = 1;
}
return ret;
}
static void usb_lock_port(struct usb_port *port_dev)
__acquires(&port_dev->status_lock)
{
mutex_lock(&port_dev->status_lock);
__acquire(&port_dev->status_lock);
}
static void usb_unlock_port(struct usb_port *port_dev)
__releases(&port_dev->status_lock)
{
mutex_unlock(&port_dev->status_lock);
__release(&port_dev->status_lock);
}
#ifdef CONFIG_PM
/* Check if a port is suspended(USB2.0 port) or in U3 state(USB3.0 port) */
static int port_is_suspended(struct usb_hub *hub, unsigned portstatus)
{
int ret = 0;
if (hub_is_superspeed(hub->hdev)) { if ((portstatus & USB_PORT_STAT_LINK_STATE)
== USB_SS_PORT_LS_U3)
ret = 1;
} else {
if (portstatus & USB_PORT_STAT_SUSPEND)
ret = 1;
}
return ret;
}
/* Determine whether the device on a port is ready for a normal resume,
* is ready for a reset-resume, or should be disconnected.
*/
static int check_port_resume_type(struct usb_device *udev,
struct usb_hub *hub, int port1,
int status, u16 portchange, u16 portstatus)
{
struct usb_port *port_dev = hub->ports[port1 - 1];
int retries = 3;
retry:
/* Is a warm reset needed to recover the connection? */
if (status == 0 && udev->reset_resume && hub_port_warm_reset_required(hub, port1, portstatus)) {
/* pass */;
}
/* Is the device still present? */
else if (status || port_is_suspended(hub, portstatus) || !usb_port_is_power_on(hub, portstatus)) { if (status >= 0)
status = -ENODEV;
} else if (!(portstatus & USB_PORT_STAT_CONNECTION)) { if (retries--) { usleep_range(200, 300);
status = usb_hub_port_status(hub, port1, &portstatus,
&portchange);
goto retry;
}
status = -ENODEV;
}
/* Can't do a normal resume if the port isn't enabled,
* so try a reset-resume instead.
*/
else if (!(portstatus & USB_PORT_STAT_ENABLE) && !udev->reset_resume) { if (udev->persist_enabled) udev->reset_resume = 1;
else
status = -ENODEV;
}
if (status) { dev_dbg(&port_dev->dev, "status %04x.%04x after resume, %d\n",
portchange, portstatus, status);
} else if (udev->reset_resume) {
/* Late port handoff can set status-change bits */
if (portchange & USB_PORT_STAT_C_CONNECTION) usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_CONNECTION);
if (portchange & USB_PORT_STAT_C_ENABLE) usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_ENABLE);
/*
* Whatever made this reset-resume necessary may have
* turned on the port1 bit in hub->change_bits. But after
* a successful reset-resume we want the bit to be clear;
* if it was on it would indicate that something happened
* following the reset-resume.
*/
clear_bit(port1, hub->change_bits);
}
return status;
}
int usb_disable_ltm(struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
/* Check if the roothub and device supports LTM. */
if (!usb_device_supports_ltm(hcd->self.root_hub) || !usb_device_supports_ltm(udev))
return 0;
/* Clear Feature LTM Enable can only be sent if the device is
* configured.
*/
if (!udev->actconfig)
return 0;
return usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_CLEAR_FEATURE, USB_RECIP_DEVICE,
USB_DEVICE_LTM_ENABLE, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
}
EXPORT_SYMBOL_GPL(usb_disable_ltm);
void usb_enable_ltm(struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
/* Check if the roothub and device supports LTM. */
if (!usb_device_supports_ltm(hcd->self.root_hub) || !usb_device_supports_ltm(udev))
return;
/* Set Feature LTM Enable can only be sent if the device is
* configured.
*/
if (!udev->actconfig)
return;
usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE, USB_RECIP_DEVICE,
USB_DEVICE_LTM_ENABLE, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
}
EXPORT_SYMBOL_GPL(usb_enable_ltm);
/*
* usb_enable_remote_wakeup - enable remote wakeup for a device
* @udev: target device
*
* For USB-2 devices: Set the device's remote wakeup feature.
*
* For USB-3 devices: Assume there's only one function on the device and
* enable remote wake for the first interface. FIXME if the interface
* association descriptor shows there's more than one function.
*/
static int usb_enable_remote_wakeup(struct usb_device *udev)
{
if (udev->speed < USB_SPEED_SUPER)
return usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE, USB_RECIP_DEVICE,
USB_DEVICE_REMOTE_WAKEUP, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
else
return usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE, USB_RECIP_INTERFACE,
USB_INTRF_FUNC_SUSPEND,
USB_INTRF_FUNC_SUSPEND_RW |
USB_INTRF_FUNC_SUSPEND_LP,
NULL, 0, USB_CTRL_SET_TIMEOUT);
}
/*
* usb_disable_remote_wakeup - disable remote wakeup for a device
* @udev: target device
*
* For USB-2 devices: Clear the device's remote wakeup feature.
*
* For USB-3 devices: Assume there's only one function on the device and
* disable remote wake for the first interface. FIXME if the interface
* association descriptor shows there's more than one function.
*/
static int usb_disable_remote_wakeup(struct usb_device *udev)
{
if (udev->speed < USB_SPEED_SUPER) return usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_CLEAR_FEATURE, USB_RECIP_DEVICE,
USB_DEVICE_REMOTE_WAKEUP, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
else
return usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE, USB_RECIP_INTERFACE,
USB_INTRF_FUNC_SUSPEND, 0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
}
/* Count of wakeup-enabled devices at or below udev */
unsigned usb_wakeup_enabled_descendants(struct usb_device *udev)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev);
return udev->do_remote_wakeup +
(hub ? hub->wakeup_enabled_descendants : 0);
}
EXPORT_SYMBOL_GPL(usb_wakeup_enabled_descendants);
/*
* usb_port_suspend - suspend a usb device's upstream port
* @udev: device that's no longer in active use, not a root hub
* Context: must be able to sleep; device not locked; pm locks held
*
* Suspends a USB device that isn't in active use, conserving power.
* Devices may wake out of a suspend, if anything important happens,
* using the remote wakeup mechanism. They may also be taken out of
* suspend by the host, using usb_port_resume(). It's also routine
* to disconnect devices while they are suspended.
*
* This only affects the USB hardware for a device; its interfaces
* (and, for hubs, child devices) must already have been suspended.
*
* Selective port suspend reduces power; most suspended devices draw
* less than 500 uA. It's also used in OTG, along with remote wakeup.
* All devices below the suspended port are also suspended.
*
* Devices leave suspend state when the host wakes them up. Some devices
* also support "remote wakeup", where the device can activate the USB
* tree above them to deliver data, such as a keypress or packet. In
* some cases, this wakes the USB host.
*
* Suspending OTG devices may trigger HNP, if that's been enabled
* between a pair of dual-role devices. That will change roles, such
* as from A-Host to A-Peripheral or from B-Host back to B-Peripheral.
*
* Devices on USB hub ports have only one "suspend" state, corresponding
* to ACPI D2, "may cause the device to lose some context".
* State transitions include:
*
* - suspend, resume ... when the VBUS power link stays live
* - suspend, disconnect ... VBUS lost
*
* Once VBUS drop breaks the circuit, the port it's using has to go through
* normal re-enumeration procedures, starting with enabling VBUS power.
* Other than re-initializing the hub (plug/unplug, except for root hubs),
* Linux (2.6) currently has NO mechanisms to initiate that: no hub_wq
* timer, no SRP, no requests through sysfs.
*
* If Runtime PM isn't enabled or used, non-SuperSpeed devices may not get
* suspended until their bus goes into global suspend (i.e., the root
* hub is suspended). Nevertheless, we change @udev->state to
* USB_STATE_SUSPENDED as this is the device's "logical" state. The actual
* upstream port setting is stored in @udev->port_is_suspended.
*
* Returns 0 on success, else negative errno.
*/
int usb_port_suspend(struct usb_device *udev, pm_message_t msg)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent);
struct usb_port *port_dev = hub->ports[udev->portnum - 1];
int port1 = udev->portnum;
int status;
bool really_suspend = true;
usb_lock_port(port_dev);
/* enable remote wakeup when appropriate; this lets the device
* wake up the upstream hub (including maybe the root hub).
*
* NOTE: OTG devices may issue remote wakeup (or SRP) even when
* we don't explicitly enable it here.
*/
if (udev->do_remote_wakeup) {
status = usb_enable_remote_wakeup(udev);
if (status) {
dev_dbg(&udev->dev, "won't remote wakeup, status %d\n",
status);
/* bail if autosuspend is requested */
if (PMSG_IS_AUTO(msg))
goto err_wakeup;
}
}
/* disable USB2 hardware LPM */
usb_disable_usb2_hardware_lpm(udev);
if (usb_disable_ltm(udev)) {
dev_err(&udev->dev, "Failed to disable LTM before suspend\n");
status = -ENOMEM;
if (PMSG_IS_AUTO(msg))
goto err_ltm;
}
/* see 7.1.7.6 */
if (hub_is_superspeed(hub->hdev))
status = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_U3);
/*
* For system suspend, we do not need to enable the suspend feature
* on individual USB-2 ports. The devices will automatically go
* into suspend a few ms after the root hub stops sending packets.
* The USB 2.0 spec calls this "global suspend".
*
* However, many USB hubs have a bug: They don't relay wakeup requests
* from a downstream port if the port's suspend feature isn't on.
* Therefore we will turn on the suspend feature if udev or any of its
* descendants is enabled for remote wakeup.
*/
else if (PMSG_IS_AUTO(msg) || usb_wakeup_enabled_descendants(udev) > 0)
status = set_port_feature(hub->hdev, port1,
USB_PORT_FEAT_SUSPEND);
else {
really_suspend = false;
status = 0;
}
if (status) {
/* Check if the port has been suspended for the timeout case
* to prevent the suspended port from incorrect handling.
*/
if (status == -ETIMEDOUT) {
int ret;
u16 portstatus, portchange;
portstatus = portchange = 0;
ret = usb_hub_port_status(hub, port1, &portstatus,
&portchange);
dev_dbg(&port_dev->dev,
"suspend timeout, status %04x\n", portstatus);
if (ret == 0 && port_is_suspended(hub, portstatus)) {
status = 0;
goto suspend_done;
}
}
dev_dbg(&port_dev->dev, "can't suspend, status %d\n", status);
/* Try to enable USB3 LTM again */
usb_enable_ltm(udev);
err_ltm:
/* Try to enable USB2 hardware LPM again */
usb_enable_usb2_hardware_lpm(udev);
if (udev->do_remote_wakeup)
(void) usb_disable_remote_wakeup(udev);
err_wakeup:
/* System sleep transitions should never fail */
if (!PMSG_IS_AUTO(msg))
status = 0;
} else {
suspend_done:
dev_dbg(&udev->dev, "usb %ssuspend, wakeup %d\n",
(PMSG_IS_AUTO(msg) ? "auto-" : ""),
udev->do_remote_wakeup);
if (really_suspend) {
udev->port_is_suspended = 1;
/* device has up to 10 msec to fully suspend */
msleep(10);
}
usb_set_device_state(udev, USB_STATE_SUSPENDED);
}
if (status == 0 && !udev->do_remote_wakeup && udev->persist_enabled
&& test_and_clear_bit(port1, hub->child_usage_bits))
pm_runtime_put_sync(&port_dev->dev);
usb_mark_last_busy(hub->hdev);
usb_unlock_port(port_dev);
return status;
}
/*
* If the USB "suspend" state is in use (rather than "global suspend"),
* many devices will be individually taken out of suspend state using
* special "resume" signaling. This routine kicks in shortly after
* hardware resume signaling is finished, either because of selective
* resume (by host) or remote wakeup (by device) ... now see what changed
* in the tree that's rooted at this device.
*
* If @udev->reset_resume is set then the device is reset before the
* status check is done.
*/
static int finish_port_resume(struct usb_device *udev)
{
int status = 0;
u16 devstatus = 0;
/* caller owns the udev device lock */
dev_dbg(&udev->dev, "%s\n",
udev->reset_resume ? "finish reset-resume" : "finish resume");
/* usb ch9 identifies four variants of SUSPENDED, based on what
* state the device resumes to. Linux currently won't see the
* first two on the host side; they'd be inside hub_port_init()
* during many timeouts, but hub_wq can't suspend until later.
*/
usb_set_device_state(udev, udev->actconfig
? USB_STATE_CONFIGURED
: USB_STATE_ADDRESS);
/* 10.5.4.5 says not to reset a suspended port if the attached
* device is enabled for remote wakeup. Hence the reset
* operation is carried out here, after the port has been
* resumed.
*/
if (udev->reset_resume) {
/*
* If the device morphs or switches modes when it is reset,
* we don't want to perform a reset-resume. We'll fail the
* resume, which will cause a logical disconnect, and then
* the device will be rediscovered.
*/
retry_reset_resume:
if (udev->quirks & USB_QUIRK_RESET)
status = -ENODEV;
else
status = usb_reset_and_verify_device(udev);
}
/* 10.5.4.5 says be sure devices in the tree are still there.
* For now let's assume the device didn't go crazy on resume,
* and device drivers will know about any resume quirks.
*/
if (status == 0) {
devstatus = 0;
status = usb_get_std_status(udev, USB_RECIP_DEVICE, 0, &devstatus);
/* If a normal resume failed, try doing a reset-resume */
if (status && !udev->reset_resume && udev->persist_enabled) { dev_dbg(&udev->dev, "retry with reset-resume\n"); udev->reset_resume = 1;
goto retry_reset_resume;
}
}
if (status) {
dev_dbg(&udev->dev, "gone after usb resume? status %d\n",
status);
/*
* There are a few quirky devices which violate the standard
* by claiming to have remote wakeup enabled after a reset,
* which crash if the feature is cleared, hence check for
* udev->reset_resume
*/
} else if (udev->actconfig && !udev->reset_resume) { if (udev->speed < USB_SPEED_SUPER) { if (devstatus & (1 << USB_DEVICE_REMOTE_WAKEUP))
status = usb_disable_remote_wakeup(udev);
} else {
status = usb_get_std_status(udev, USB_RECIP_INTERFACE, 0,
&devstatus);
if (!status && devstatus & (USB_INTRF_STAT_FUNC_RW_CAP
| USB_INTRF_STAT_FUNC_RW))
status = usb_disable_remote_wakeup(udev);
}
if (status) dev_dbg(&udev->dev,
"disable remote wakeup, status %d\n",
status);
status = 0;
}
return status;
}
/*
* There are some SS USB devices which take longer time for link training.
* XHCI specs 4.19.4 says that when Link training is successful, port
* sets CCS bit to 1. So if SW reads port status before successful link
* training, then it will not find device to be present.
* USB Analyzer log with such buggy devices show that in some cases
* device switch on the RX termination after long delay of host enabling
* the VBUS. In few other cases it has been seen that device fails to
* negotiate link training in first attempt. It has been
* reported till now that few devices take as long as 2000 ms to train
* the link after host enabling its VBUS and termination. Following
* routine implements a 2000 ms timeout for link training. If in a case
* link trains before timeout, loop will exit earlier.
*
* There are also some 2.0 hard drive based devices and 3.0 thumb
* drives that, when plugged into a 2.0 only port, take a long
* time to set CCS after VBUS enable.
*
* FIXME: If a device was connected before suspend, but was removed
* while system was asleep, then the loop in the following routine will
* only exit at timeout.
*
* This routine should only be called when persist is enabled.
*/
static int wait_for_connected(struct usb_device *udev,
struct usb_hub *hub, int port1,
u16 *portchange, u16 *portstatus)
{
int status = 0, delay_ms = 0;
while (delay_ms < 2000) {
if (status || *portstatus & USB_PORT_STAT_CONNECTION)
break;
if (!usb_port_is_power_on(hub, *portstatus)) {
status = -ENODEV;
break;
}
msleep(20);
delay_ms += 20;
status = usb_hub_port_status(hub, port1, portstatus, portchange);
}
dev_dbg(&udev->dev, "Waited %dms for CONNECT\n", delay_ms);
return status;
}
/*
* usb_port_resume - re-activate a suspended usb device's upstream port
* @udev: device to re-activate, not a root hub
* Context: must be able to sleep; device not locked; pm locks held
*
* This will re-activate the suspended device, increasing power usage
* while letting drivers communicate again with its endpoints.
* USB resume explicitly guarantees that the power session between
* the host and the device is the same as it was when the device
* suspended.
*
* If @udev->reset_resume is set then this routine won't check that the
* port is still enabled. Furthermore, finish_port_resume() above will
* reset @udev. The end result is that a broken power session can be
* recovered and @udev will appear to persist across a loss of VBUS power.
*
* For example, if a host controller doesn't maintain VBUS suspend current
* during a system sleep or is reset when the system wakes up, all the USB
* power sessions below it will be broken. This is especially troublesome
* for mass-storage devices containing mounted filesystems, since the
* device will appear to have disconnected and all the memory mappings
* to it will be lost. Using the USB_PERSIST facility, the device can be
* made to appear as if it had not disconnected.
*
* This facility can be dangerous. Although usb_reset_and_verify_device() makes
* every effort to insure that the same device is present after the
* reset as before, it cannot provide a 100% guarantee. Furthermore it's
* quite possible for a device to remain unaltered but its media to be
* changed. If the user replaces a flash memory card while the system is
* asleep, he will have only himself to blame when the filesystem on the
* new card is corrupted and the system crashes.
*
* Returns 0 on success, else negative errno.
*/
int usb_port_resume(struct usb_device *udev, pm_message_t msg)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent); struct usb_port *port_dev = hub->ports[udev->portnum - 1];
int port1 = udev->portnum;
int status;
u16 portchange, portstatus;
if (!test_and_set_bit(port1, hub->child_usage_bits)) {
status = pm_runtime_resume_and_get(&port_dev->dev);
if (status < 0) {
dev_dbg(&udev->dev, "can't resume usb port, status %d\n",
status);
return status;
}
}
usb_lock_port(port_dev);
/* Skip the initial Clear-Suspend step for a remote wakeup */
status = usb_hub_port_status(hub, port1, &portstatus, &portchange);
if (status == 0 && !port_is_suspended(hub, portstatus)) { if (portchange & USB_PORT_STAT_C_SUSPEND) pm_wakeup_event(&udev->dev, 0);
goto SuspendCleared;
}
/* see 7.1.7.7; affects power usage, but not budgeting */
if (hub_is_superspeed(hub->hdev)) status = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_U0);
else
status = usb_clear_port_feature(hub->hdev,
port1, USB_PORT_FEAT_SUSPEND);
if (status) { dev_dbg(&port_dev->dev, "can't resume, status %d\n", status);
} else {
/* drive resume for USB_RESUME_TIMEOUT msec */
dev_dbg(&udev->dev, "usb %sresume\n",
(PMSG_IS_AUTO(msg) ? "auto-" : ""));
msleep(USB_RESUME_TIMEOUT);
/* Virtual root hubs can trigger on GET_PORT_STATUS to
* stop resume signaling. Then finish the resume
* sequence.
*/
status = usb_hub_port_status(hub, port1, &portstatus, &portchange);
}
SuspendCleared:
if (status == 0) {
udev->port_is_suspended = 0;
if (hub_is_superspeed(hub->hdev)) {
if (portchange & USB_PORT_STAT_C_LINK_STATE) usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_PORT_LINK_STATE);
} else {
if (portchange & USB_PORT_STAT_C_SUSPEND) usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_SUSPEND);
}
/* TRSMRCY = 10 msec */
msleep(10);
}
if (udev->persist_enabled) status = wait_for_connected(udev, hub, port1, &portchange,
&portstatus);
status = check_port_resume_type(udev,
hub, port1, status, portchange, portstatus);
if (status == 0)
status = finish_port_resume(udev);
if (status < 0) {
dev_dbg(&udev->dev, "can't resume, status %d\n", status); hub_port_logical_disconnect(hub, port1);
} else {
/* Try to enable USB2 hardware LPM */
usb_enable_usb2_hardware_lpm(udev);
/* Try to enable USB3 LTM */
usb_enable_ltm(udev);
}
usb_unlock_port(port_dev); return status;
}
int usb_remote_wakeup(struct usb_device *udev)
{
int status = 0;
usb_lock_device(udev);
if (udev->state == USB_STATE_SUSPENDED) {
dev_dbg(&udev->dev, "usb %sresume\n", "wakeup-"); status = usb_autoresume_device(udev);
if (status == 0) {
/* Let the drivers do their thing, then... */
usb_autosuspend_device(udev);
}
}
usb_unlock_device(udev);
return status;
}
/* Returns 1 if there was a remote wakeup and a connect status change. */
static int hub_handle_remote_wakeup(struct usb_hub *hub, unsigned int port,
u16 portstatus, u16 portchange)
__must_hold(&port_dev->status_lock)
{
struct usb_port *port_dev = hub->ports[port - 1];
struct usb_device *hdev;
struct usb_device *udev;
int connect_change = 0;
u16 link_state;
int ret;
hdev = hub->hdev;
udev = port_dev->child;
if (!hub_is_superspeed(hdev)) {
if (!(portchange & USB_PORT_STAT_C_SUSPEND))
return 0;
usb_clear_port_feature(hdev, port, USB_PORT_FEAT_C_SUSPEND);
} else {
link_state = portstatus & USB_PORT_STAT_LINK_STATE; if (!udev || udev->state != USB_STATE_SUSPENDED ||
(link_state != USB_SS_PORT_LS_U0 &&
link_state != USB_SS_PORT_LS_U1 &&
link_state != USB_SS_PORT_LS_U2))
return 0;
}
if (udev) {
/* TRSMRCY = 10 msec */
msleep(10);
usb_unlock_port(port_dev);
ret = usb_remote_wakeup(udev);
usb_lock_port(port_dev);
if (ret < 0)
connect_change = 1;
} else {
ret = -ENODEV;
hub_port_disable(hub, port, 1);
}
dev_dbg(&port_dev->dev, "resume, status %d\n", ret);
return connect_change;
}
static int check_ports_changed(struct usb_hub *hub)
{
int port1;
for (port1 = 1; port1 <= hub->hdev->maxchild; ++port1) {
u16 portstatus, portchange;
int status;
status = usb_hub_port_status(hub, port1, &portstatus, &portchange);
if (!status && portchange)
return 1;
}
return 0;
}
static int hub_suspend(struct usb_interface *intf, pm_message_t msg)
{
struct usb_hub *hub = usb_get_intfdata(intf);
struct usb_device *hdev = hub->hdev;
unsigned port1;
/*
* Warn if children aren't already suspended.
* Also, add up the number of wakeup-enabled descendants.
*/
hub->wakeup_enabled_descendants = 0;
for (port1 = 1; port1 <= hdev->maxchild; port1++) {
struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *udev = port_dev->child;
if (udev && udev->can_submit) {
dev_warn(&port_dev->dev, "device %s not suspended yet\n",
dev_name(&udev->dev));
if (PMSG_IS_AUTO(msg))
return -EBUSY;
}
if (udev)
hub->wakeup_enabled_descendants +=
usb_wakeup_enabled_descendants(udev);
}
if (hdev->do_remote_wakeup && hub->quirk_check_port_auto_suspend) {
/* check if there are changes pending on hub ports */
if (check_ports_changed(hub)) {
if (PMSG_IS_AUTO(msg))
return -EBUSY;
pm_wakeup_event(&hdev->dev, 2000);
}
}
if (hub_is_superspeed(hdev) && hdev->do_remote_wakeup) {
/* Enable hub to send remote wakeup for all ports. */
for (port1 = 1; port1 <= hdev->maxchild; port1++) {
set_port_feature(hdev,
port1 |
USB_PORT_FEAT_REMOTE_WAKE_CONNECT |
USB_PORT_FEAT_REMOTE_WAKE_DISCONNECT |
USB_PORT_FEAT_REMOTE_WAKE_OVER_CURRENT,
USB_PORT_FEAT_REMOTE_WAKE_MASK);
}
}
dev_dbg(&intf->dev, "%s\n", __func__);
/* stop hub_wq and related activity */
hub_quiesce(hub, HUB_SUSPEND);
return 0;
}
/* Report wakeup requests from the ports of a resuming root hub */
static void report_wakeup_requests(struct usb_hub *hub)
{
struct usb_device *hdev = hub->hdev;
struct usb_device *udev;
struct usb_hcd *hcd;
unsigned long resuming_ports;
int i;
if (hdev->parent)
return; /* Not a root hub */ hcd = bus_to_hcd(hdev->bus);
if (hcd->driver->get_resuming_ports) {
/*
* The get_resuming_ports() method returns a bitmap (origin 0)
* of ports which have started wakeup signaling but have not
* yet finished resuming. During system resume we will
* resume all the enabled ports, regardless of any wakeup
* signals, which means the wakeup requests would be lost.
* To prevent this, report them to the PM core here.
*/
resuming_ports = hcd->driver->get_resuming_ports(hcd); for (i = 0; i < hdev->maxchild; ++i) { if (test_bit(i, &resuming_ports)) { udev = hub->ports[i]->child;
if (udev)
pm_wakeup_event(&udev->dev, 0);
}
}
}
}
static int hub_resume(struct usb_interface *intf)
{
struct usb_hub *hub = usb_get_intfdata(intf); dev_dbg(&intf->dev, "%s\n", __func__); hub_activate(hub, HUB_RESUME);
/*
* This should be called only for system resume, not runtime resume.
* We can't tell the difference here, so some wakeup requests will be
* reported at the wrong time or more than once. This shouldn't
* matter much, so long as they do get reported.
*/
report_wakeup_requests(hub); return 0;
}
static int hub_reset_resume(struct usb_interface *intf)
{
struct usb_hub *hub = usb_get_intfdata(intf);
dev_dbg(&intf->dev, "%s\n", __func__);
hub_activate(hub, HUB_RESET_RESUME);
return 0;
}
/**
* usb_root_hub_lost_power - called by HCD if the root hub lost Vbus power
* @rhdev: struct usb_device for the root hub
*
* The USB host controller driver calls this function when its root hub
* is resumed and Vbus power has been interrupted or the controller
* has been reset. The routine marks @rhdev as having lost power.
* When the hub driver is resumed it will take notice and carry out
* power-session recovery for all the "USB-PERSIST"-enabled child devices;
* the others will be disconnected.
*/
void usb_root_hub_lost_power(struct usb_device *rhdev)
{
dev_notice(&rhdev->dev, "root hub lost power or was reset\n");
rhdev->reset_resume = 1;
}
EXPORT_SYMBOL_GPL(usb_root_hub_lost_power);
static const char * const usb3_lpm_names[] = {
"U0",
"U1",
"U2",
"U3",
};
/*
* Send a Set SEL control transfer to the device, prior to enabling
* device-initiated U1 or U2. This lets the device know the exit latencies from
* the time the device initiates a U1 or U2 exit, to the time it will receive a
* packet from the host.
*
* This function will fail if the SEL or PEL values for udev are greater than
* the maximum allowed values for the link state to be enabled.
*/
static int usb_req_set_sel(struct usb_device *udev)
{
struct usb_set_sel_req *sel_values;
unsigned long long u1_sel;
unsigned long long u1_pel;
unsigned long long u2_sel;
unsigned long long u2_pel;
int ret;
if (!udev->parent || udev->speed < USB_SPEED_SUPER || !udev->lpm_capable)
return 0;
/* Convert SEL and PEL stored in ns to us */
u1_sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
u1_pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
u2_sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
u2_pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
/*
* Make sure that the calculated SEL and PEL values for the link
* state we're enabling aren't bigger than the max SEL/PEL
* value that will fit in the SET SEL control transfer.
* Otherwise the device would get an incorrect idea of the exit
* latency for the link state, and could start a device-initiated
* U1/U2 when the exit latencies are too high.
*/
if (u1_sel > USB3_LPM_MAX_U1_SEL_PEL || u1_pel > USB3_LPM_MAX_U1_SEL_PEL || u2_sel > USB3_LPM_MAX_U2_SEL_PEL ||
u2_pel > USB3_LPM_MAX_U2_SEL_PEL) {
dev_dbg(&udev->dev, "Device-initiated U1/U2 disabled due to long SEL or PEL\n");
return -EINVAL;
}
/*
* usb_enable_lpm() can be called as part of a failed device reset,
* which may be initiated by an error path of a mass storage driver.
* Therefore, use GFP_NOIO.
*/
sel_values = kmalloc(sizeof *(sel_values), GFP_NOIO);
if (!sel_values)
return -ENOMEM;
sel_values->u1_sel = u1_sel;
sel_values->u1_pel = u1_pel;
sel_values->u2_sel = cpu_to_le16(u2_sel);
sel_values->u2_pel = cpu_to_le16(u2_pel);
ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_SEL,
USB_RECIP_DEVICE,
0, 0,
sel_values, sizeof *(sel_values),
USB_CTRL_SET_TIMEOUT);
kfree(sel_values);
if (ret > 0)
udev->lpm_devinit_allow = 1;
return ret;
}
/*
* Enable or disable device-initiated U1 or U2 transitions.
*/
static int usb_set_device_initiated_lpm(struct usb_device *udev,
enum usb3_link_state state, bool enable)
{
int ret;
int feature;
switch (state) {
case USB3_LPM_U1:
feature = USB_DEVICE_U1_ENABLE;
break;
case USB3_LPM_U2:
feature = USB_DEVICE_U2_ENABLE;
break;
default:
dev_warn(&udev->dev, "%s: Can't %s non-U1 or U2 state.\n",
__func__, enable ? "enable" : "disable");
return -EINVAL;
}
if (udev->state != USB_STATE_CONFIGURED) {
dev_dbg(&udev->dev, "%s: Can't %s %s state "
"for unconfigured device.\n",
__func__, enable ? "enable" : "disable",
usb3_lpm_names[state]);
return 0;
}
if (enable) {
/*
* Now send the control transfer to enable device-initiated LPM
* for either U1 or U2.
*/
ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_FEATURE,
USB_RECIP_DEVICE,
feature,
0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
} else {
ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_CLEAR_FEATURE,
USB_RECIP_DEVICE,
feature,
0, NULL, 0,
USB_CTRL_SET_TIMEOUT);
}
if (ret < 0) {
dev_warn(&udev->dev, "%s of device-initiated %s failed.\n",
enable ? "Enable" : "Disable",
usb3_lpm_names[state]);
return -EBUSY;
}
return 0;
}
static int usb_set_lpm_timeout(struct usb_device *udev,
enum usb3_link_state state, int timeout)
{
int ret;
int feature;
switch (state) {
case USB3_LPM_U1:
feature = USB_PORT_FEAT_U1_TIMEOUT;
break;
case USB3_LPM_U2:
feature = USB_PORT_FEAT_U2_TIMEOUT;
break;
default:
dev_warn(&udev->dev, "%s: Can't set timeout for non-U1 or U2 state.\n",
__func__);
return -EINVAL;
}
if (state == USB3_LPM_U1 && timeout > USB3_LPM_U1_MAX_TIMEOUT &&
timeout != USB3_LPM_DEVICE_INITIATED) {
dev_warn(&udev->dev, "Failed to set %s timeout to 0x%x, "
"which is a reserved value.\n",
usb3_lpm_names[state], timeout);
return -EINVAL;
}
ret = set_port_feature(udev->parent,
USB_PORT_LPM_TIMEOUT(timeout) | udev->portnum,
feature);
if (ret < 0) {
dev_warn(&udev->dev, "Failed to set %s timeout to 0x%x,"
"error code %i\n", usb3_lpm_names[state],
timeout, ret);
return -EBUSY;
}
if (state == USB3_LPM_U1)
udev->u1_params.timeout = timeout;
else
udev->u2_params.timeout = timeout;
return 0;
}
/*
* Don't allow device intiated U1/U2 if the system exit latency + one bus
* interval is greater than the minimum service interval of any active
* periodic endpoint. See USB 3.2 section 9.4.9
*/
static bool usb_device_may_initiate_lpm(struct usb_device *udev,
enum usb3_link_state state)
{
unsigned int sel; /* us */
int i, j;
if (!udev->lpm_devinit_allow)
return false;
if (state == USB3_LPM_U1)
sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
else if (state == USB3_LPM_U2)
sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
else
return false;
for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) {
struct usb_interface *intf;
struct usb_endpoint_descriptor *desc;
unsigned int interval;
intf = udev->actconfig->interface[i];
if (!intf)
continue;
for (j = 0; j < intf->cur_altsetting->desc.bNumEndpoints; j++) {
desc = &intf->cur_altsetting->endpoint[j].desc;
if (usb_endpoint_xfer_int(desc) ||
usb_endpoint_xfer_isoc(desc)) {
interval = (1 << (desc->bInterval - 1)) * 125;
if (sel + 125 > interval)
return false;
}
}
}
return true;
}
/*
* Enable the hub-initiated U1/U2 idle timeouts, and enable device-initiated
* U1/U2 entry.
*
* We will attempt to enable U1 or U2, but there are no guarantees that the
* control transfers to set the hub timeout or enable device-initiated U1/U2
* will be successful.
*
* If the control transfer to enable device-initiated U1/U2 entry fails, then
* hub-initiated U1/U2 will be disabled.
*
* If we cannot set the parent hub U1/U2 timeout, we attempt to let the xHCI
* driver know about it. If that call fails, it should be harmless, and just
* take up more slightly more bus bandwidth for unnecessary U1/U2 exit latency.
*/
static void usb_enable_link_state(struct usb_hcd *hcd, struct usb_device *udev,
enum usb3_link_state state)
{
int timeout;
__u8 u1_mel;
__le16 u2_mel;
/* Skip if the device BOS descriptor couldn't be read */
if (!udev->bos)
return;
u1_mel = udev->bos->ss_cap->bU1devExitLat;
u2_mel = udev->bos->ss_cap->bU2DevExitLat;
/* If the device says it doesn't have *any* exit latency to come out of
* U1 or U2, it's probably lying. Assume it doesn't implement that link
* state.
*/
if ((state == USB3_LPM_U1 && u1_mel == 0) ||
(state == USB3_LPM_U2 && u2_mel == 0))
return;
/* We allow the host controller to set the U1/U2 timeout internally
* first, so that it can change its schedule to account for the
* additional latency to send data to a device in a lower power
* link state.
*/
timeout = hcd->driver->enable_usb3_lpm_timeout(hcd, udev, state);
/* xHCI host controller doesn't want to enable this LPM state. */
if (timeout == 0)
return;
if (timeout < 0) {
dev_warn(&udev->dev, "Could not enable %s link state, "
"xHCI error %i.\n", usb3_lpm_names[state],
timeout);
return;
}
if (usb_set_lpm_timeout(udev, state, timeout)) {
/* If we can't set the parent hub U1/U2 timeout,
* device-initiated LPM won't be allowed either, so let the xHCI
* host know that this link state won't be enabled.
*/
hcd->driver->disable_usb3_lpm_timeout(hcd, udev, state);
return;
}
/* Only a configured device will accept the Set Feature
* U1/U2_ENABLE
*/
if (udev->actconfig &&
usb_device_may_initiate_lpm(udev, state)) {
if (usb_set_device_initiated_lpm(udev, state, true)) {
/*
* Request to enable device initiated U1/U2 failed,
* better to turn off lpm in this case.
*/
usb_set_lpm_timeout(udev, state, 0);
hcd->driver->disable_usb3_lpm_timeout(hcd, udev, state);
return;
}
}
if (state == USB3_LPM_U1)
udev->usb3_lpm_u1_enabled = 1;
else if (state == USB3_LPM_U2)
udev->usb3_lpm_u2_enabled = 1;
}
/*
* Disable the hub-initiated U1/U2 idle timeouts, and disable device-initiated
* U1/U2 entry.
*
* If this function returns -EBUSY, the parent hub will still allow U1/U2 entry.
* If zero is returned, the parent will not allow the link to go into U1/U2.
*
* If zero is returned, device-initiated U1/U2 entry may still be enabled, but
* it won't have an effect on the bus link state because the parent hub will
* still disallow device-initiated U1/U2 entry.
*
* If zero is returned, the xHCI host controller may still think U1/U2 entry is
* possible. The result will be slightly more bus bandwidth will be taken up
* (to account for U1/U2 exit latency), but it should be harmless.
*/
static int usb_disable_link_state(struct usb_hcd *hcd, struct usb_device *udev,
enum usb3_link_state state)
{
switch (state) {
case USB3_LPM_U1:
case USB3_LPM_U2:
break;
default:
dev_warn(&udev->dev, "%s: Can't disable non-U1 or U2 state.\n",
__func__);
return -EINVAL;
}
if (usb_set_lpm_timeout(udev, state, 0))
return -EBUSY;
usb_set_device_initiated_lpm(udev, state, false);
if (hcd->driver->disable_usb3_lpm_timeout(hcd, udev, state))
dev_warn(&udev->dev, "Could not disable xHCI %s timeout, "
"bus schedule bandwidth may be impacted.\n",
usb3_lpm_names[state]);
/* As soon as usb_set_lpm_timeout(0) return 0, hub initiated LPM
* is disabled. Hub will disallows link to enter U1/U2 as well,
* even device is initiating LPM. Hence LPM is disabled if hub LPM
* timeout set to 0, no matter device-initiated LPM is disabled or
* not.
*/
if (state == USB3_LPM_U1)
udev->usb3_lpm_u1_enabled = 0;
else if (state == USB3_LPM_U2)
udev->usb3_lpm_u2_enabled = 0;
return 0;
}
/*
* Disable hub-initiated and device-initiated U1 and U2 entry.
* Caller must own the bandwidth_mutex.
*
* This will call usb_enable_lpm() on failure, which will decrement
* lpm_disable_count, and will re-enable LPM if lpm_disable_count reaches zero.
*/
int usb_disable_lpm(struct usb_device *udev)
{
struct usb_hcd *hcd;
if (!udev || !udev->parent || udev->speed < USB_SPEED_SUPER || !udev->lpm_capable || udev->state < USB_STATE_CONFIGURED) return 0; hcd = bus_to_hcd(udev->bus); if (!hcd || !hcd->driver->disable_usb3_lpm_timeout)
return 0;
udev->lpm_disable_count++; if ((udev->u1_params.timeout == 0 && udev->u2_params.timeout == 0))
return 0;
/* If LPM is enabled, attempt to disable it. */
if (usb_disable_link_state(hcd, udev, USB3_LPM_U1))
goto enable_lpm;
if (usb_disable_link_state(hcd, udev, USB3_LPM_U2))
goto enable_lpm;
return 0;
enable_lpm:
usb_enable_lpm(udev);
return -EBUSY;
}
EXPORT_SYMBOL_GPL(usb_disable_lpm);
/* Grab the bandwidth_mutex before calling usb_disable_lpm() */
int usb_unlocked_disable_lpm(struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
int ret;
if (!hcd)
return -EINVAL;
mutex_lock(hcd->bandwidth_mutex);
ret = usb_disable_lpm(udev);
mutex_unlock(hcd->bandwidth_mutex);
return ret;
}
EXPORT_SYMBOL_GPL(usb_unlocked_disable_lpm);
/*
* Attempt to enable device-initiated and hub-initiated U1 and U2 entry. The
* xHCI host policy may prevent U1 or U2 from being enabled.
*
* Other callers may have disabled link PM, so U1 and U2 entry will be disabled
* until the lpm_disable_count drops to zero. Caller must own the
* bandwidth_mutex.
*/
void usb_enable_lpm(struct usb_device *udev)
{
struct usb_hcd *hcd;
struct usb_hub *hub;
struct usb_port *port_dev;
if (!udev || !udev->parent || udev->speed < USB_SPEED_SUPER || !udev->lpm_capable || udev->state < USB_STATE_CONFIGURED)
return;
udev->lpm_disable_count--;
hcd = bus_to_hcd(udev->bus);
/* Double check that we can both enable and disable LPM.
* Device must be configured to accept set feature U1/U2 timeout.
*/
if (!hcd || !hcd->driver->enable_usb3_lpm_timeout || !hcd->driver->disable_usb3_lpm_timeout)
return;
if (udev->lpm_disable_count > 0)
return;
hub = usb_hub_to_struct_hub(udev->parent);
if (!hub)
return;
port_dev = hub->ports[udev->portnum - 1];
if (port_dev->usb3_lpm_u1_permit)
usb_enable_link_state(hcd, udev, USB3_LPM_U1); if (port_dev->usb3_lpm_u2_permit) usb_enable_link_state(hcd, udev, USB3_LPM_U2);
}
EXPORT_SYMBOL_GPL(usb_enable_lpm);
/* Grab the bandwidth_mutex before calling usb_enable_lpm() */
void usb_unlocked_enable_lpm(struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
if (!hcd)
return;
mutex_lock(hcd->bandwidth_mutex);
usb_enable_lpm(udev);
mutex_unlock(hcd->bandwidth_mutex);
}
EXPORT_SYMBOL_GPL(usb_unlocked_enable_lpm);
/* usb3 devices use U3 for disabled, make sure remote wakeup is disabled */
static void hub_usb3_port_prepare_disable(struct usb_hub *hub,
struct usb_port *port_dev)
{
struct usb_device *udev = port_dev->child;
int ret;
if (udev && udev->port_is_suspended && udev->do_remote_wakeup) { ret = hub_set_port_link_state(hub, port_dev->portnum,
USB_SS_PORT_LS_U0);
if (!ret) {
msleep(USB_RESUME_TIMEOUT);
ret = usb_disable_remote_wakeup(udev);
}
if (ret)
dev_warn(&udev->dev,
"Port disable: can't disable remote wake\n");
udev->do_remote_wakeup = 0;
}
}
#else /* CONFIG_PM */
#define hub_suspend NULL
#define hub_resume NULL
#define hub_reset_resume NULL
static inline void hub_usb3_port_prepare_disable(struct usb_hub *hub,
struct usb_port *port_dev) { }
int usb_disable_lpm(struct usb_device *udev)
{
return 0;
}
EXPORT_SYMBOL_GPL(usb_disable_lpm);
void usb_enable_lpm(struct usb_device *udev) { }
EXPORT_SYMBOL_GPL(usb_enable_lpm);
int usb_unlocked_disable_lpm(struct usb_device *udev)
{
return 0;
}
EXPORT_SYMBOL_GPL(usb_unlocked_disable_lpm);
void usb_unlocked_enable_lpm(struct usb_device *udev) { }
EXPORT_SYMBOL_GPL(usb_unlocked_enable_lpm);
int usb_disable_ltm(struct usb_device *udev)
{
return 0;
}
EXPORT_SYMBOL_GPL(usb_disable_ltm);
void usb_enable_ltm(struct usb_device *udev) { }
EXPORT_SYMBOL_GPL(usb_enable_ltm);
static int hub_handle_remote_wakeup(struct usb_hub *hub, unsigned int port,
u16 portstatus, u16 portchange)
{
return 0;
}
static int usb_req_set_sel(struct usb_device *udev)
{
return 0;
}
#endif /* CONFIG_PM */
/*
* USB-3 does not have a similar link state as USB-2 that will avoid negotiating
* a connection with a plugged-in cable but will signal the host when the cable
* is unplugged. Disable remote wake and set link state to U3 for USB-3 devices
*/
static int hub_port_disable(struct usb_hub *hub, int port1, int set_state)
{
struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *hdev = hub->hdev;
int ret = 0;
if (!hub->error) {
if (hub_is_superspeed(hub->hdev)) { hub_usb3_port_prepare_disable(hub, port_dev); ret = hub_set_port_link_state(hub, port_dev->portnum,
USB_SS_PORT_LS_U3);
} else {
ret = usb_clear_port_feature(hdev, port1,
USB_PORT_FEAT_ENABLE);
}
}
if (port_dev->child && set_state) usb_set_device_state(port_dev->child, USB_STATE_NOTATTACHED); if (ret && ret != -ENODEV) dev_err(&port_dev->dev, "cannot disable (err = %d)\n", ret); return ret;
}
/*
* usb_port_disable - disable a usb device's upstream port
* @udev: device to disable
* Context: @udev locked, must be able to sleep.
*
* Disables a USB device that isn't in active use.
*/
int usb_port_disable(struct usb_device *udev)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent);
return hub_port_disable(hub, udev->portnum, 0);
}
/* USB 2.0 spec, 7.1.7.3 / fig 7-29:
*
* Between connect detection and reset signaling there must be a delay
* of 100ms at least for debounce and power-settling. The corresponding
* timer shall restart whenever the downstream port detects a disconnect.
*
* Apparently there are some bluetooth and irda-dongles and a number of
* low-speed devices for which this debounce period may last over a second.
* Not covered by the spec - but easy to deal with.
*
* This implementation uses a 1500ms total debounce timeout; if the
* connection isn't stable by then it returns -ETIMEDOUT. It checks
* every 25ms for transient disconnects. When the port status has been
* unchanged for 100ms it returns the port status.
*/
int hub_port_debounce(struct usb_hub *hub, int port1, bool must_be_connected)
{
int ret;
u16 portchange, portstatus;
unsigned connection = 0xffff;
int total_time, stable_time = 0;
struct usb_port *port_dev = hub->ports[port1 - 1];
for (total_time = 0; ; total_time += HUB_DEBOUNCE_STEP) {
ret = usb_hub_port_status(hub, port1, &portstatus, &portchange);
if (ret < 0)
return ret;
if (!(portchange & USB_PORT_STAT_C_CONNECTION) &&
(portstatus & USB_PORT_STAT_CONNECTION) == connection) {
if (!must_be_connected ||
(connection == USB_PORT_STAT_CONNECTION))
stable_time += HUB_DEBOUNCE_STEP;
if (stable_time >= HUB_DEBOUNCE_STABLE)
break;
} else {
stable_time = 0;
connection = portstatus & USB_PORT_STAT_CONNECTION;
}
if (portchange & USB_PORT_STAT_C_CONNECTION) {
usb_clear_port_feature(hub->hdev, port1,
USB_PORT_FEAT_C_CONNECTION);
}
if (total_time >= HUB_DEBOUNCE_TIMEOUT)
break;
msleep(HUB_DEBOUNCE_STEP);
}
dev_dbg(&port_dev->dev, "debounce total %dms stable %dms status 0x%x\n",
total_time, stable_time, portstatus);
if (stable_time < HUB_DEBOUNCE_STABLE)
return -ETIMEDOUT;
return portstatus;
}
void usb_ep0_reinit(struct usb_device *udev)
{
usb_disable_endpoint(udev, 0 + USB_DIR_IN, true);
usb_disable_endpoint(udev, 0 + USB_DIR_OUT, true);
usb_enable_endpoint(udev, &udev->ep0, true);
}
EXPORT_SYMBOL_GPL(usb_ep0_reinit);
#define usb_sndaddr0pipe() (PIPE_CONTROL << 30)
#define usb_rcvaddr0pipe() ((PIPE_CONTROL << 30) | USB_DIR_IN)
static int hub_set_address(struct usb_device *udev, int devnum)
{
int retval;
unsigned int timeout_ms = USB_CTRL_SET_TIMEOUT;
struct usb_hcd *hcd = bus_to_hcd(udev->bus); struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent); if (hub->hdev->quirks & USB_QUIRK_SHORT_SET_ADDRESS_REQ_TIMEOUT)
timeout_ms = USB_SHORT_SET_ADDRESS_REQ_TIMEOUT;
/*
* The host controller will choose the device address,
* instead of the core having chosen it earlier
*/
if (!hcd->driver->address_device && devnum <= 1)
return -EINVAL;
if (udev->state == USB_STATE_ADDRESS)
return 0;
if (udev->state != USB_STATE_DEFAULT)
return -EINVAL;
if (hcd->driver->address_device)
retval = hcd->driver->address_device(hcd, udev, timeout_ms);
else
retval = usb_control_msg(udev, usb_sndaddr0pipe(),
USB_REQ_SET_ADDRESS, 0, devnum, 0,
NULL, 0, timeout_ms);
if (retval == 0) { update_devnum(udev, devnum);
/* Device now using proper address. */
usb_set_device_state(udev, USB_STATE_ADDRESS);
usb_ep0_reinit(udev);
}
return retval;
}
/*
* There are reports of USB 3.0 devices that say they support USB 2.0 Link PM
* when they're plugged into a USB 2.0 port, but they don't work when LPM is
* enabled.
*
* Only enable USB 2.0 Link PM if the port is internal (hardwired), or the
* device says it supports the new USB 2.0 Link PM errata by setting the BESL
* support bit in the BOS descriptor.
*/
static void hub_set_initial_usb2_lpm_policy(struct usb_device *udev)
{
struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent);
int connect_type = USB_PORT_CONNECT_TYPE_UNKNOWN;
if (!udev->usb2_hw_lpm_capable || !udev->bos)
return;
if (hub)
connect_type = hub->ports[udev->portnum - 1]->connect_type; if ((udev->bos->ext_cap->bmAttributes & cpu_to_le32(USB_BESL_SUPPORT)) ||
connect_type == USB_PORT_CONNECT_TYPE_HARD_WIRED) {
udev->usb2_hw_lpm_allowed = 1;
usb_enable_usb2_hardware_lpm(udev);
}
}
static int hub_enable_device(struct usb_device *udev)
{
struct usb_hcd *hcd = bus_to_hcd(udev->bus); if (!hcd->driver->enable_device)
return 0;
if (udev->state == USB_STATE_ADDRESS)
return 0;
if (udev->state != USB_STATE_DEFAULT)
return -EINVAL;
return hcd->driver->enable_device(hcd, udev);
}
/*
* Get the bMaxPacketSize0 value during initialization by reading the
* device's device descriptor. Since we don't already know this value,
* the transfer is unsafe and it ignores I/O errors, only testing for
* reasonable received values.
*
* For "old scheme" initialization, size will be 8 so we read just the
* start of the device descriptor, which should work okay regardless of
* the actual bMaxPacketSize0 value. For "new scheme" initialization,
* size will be 64 (and buf will point to a sufficiently large buffer),
* which might not be kosher according to the USB spec but it's what
* Windows does and what many devices expect.
*
* Returns: bMaxPacketSize0 or a negative error code.
*/
static int get_bMaxPacketSize0(struct usb_device *udev,
struct usb_device_descriptor *buf, int size, bool first_time)
{
int i, rc;
/*
* Retry on all errors; some devices are flakey.
* 255 is for WUSB devices, we actually need to use
* 512 (WUSB1.0[4.8.1]).
*/
for (i = 0; i < GET_MAXPACKET0_TRIES; ++i) {
/* Start with invalid values in case the transfer fails */
buf->bDescriptorType = buf->bMaxPacketSize0 = 0;
rc = usb_control_msg(udev, usb_rcvaddr0pipe(),
USB_REQ_GET_DESCRIPTOR, USB_DIR_IN,
USB_DT_DEVICE << 8, 0,
buf, size,
initial_descriptor_timeout);
switch (buf->bMaxPacketSize0) {
case 8: case 16: case 32: case 64: case 9:
if (buf->bDescriptorType == USB_DT_DEVICE) {
rc = buf->bMaxPacketSize0;
break;
}
fallthrough;
default:
if (rc >= 0)
rc = -EPROTO;
break;
}
/*
* Some devices time out if they are powered on
* when already connected. They need a second
* reset, so return early. But only on the first
* attempt, lest we get into a time-out/reset loop.
*/
if (rc > 0 || (rc == -ETIMEDOUT && first_time && udev->speed > USB_SPEED_FULL))
break;
}
return rc;
}
#define GET_DESCRIPTOR_BUFSIZE 64
/* Reset device, (re)assign address, get device descriptor.
* Device connection must be stable, no more debouncing needed.
* Returns device in USB_STATE_ADDRESS, except on error.
*
* If this is called for an already-existing device (as part of
* usb_reset_and_verify_device), the caller must own the device lock and
* the port lock. For a newly detected device that is not accessible
* through any global pointers, it's not necessary to lock the device,
* but it is still necessary to lock the port.
*
* For a newly detected device, @dev_descr must be NULL. The device
* descriptor retrieved from the device will then be stored in
* @udev->descriptor. For an already existing device, @dev_descr
* must be non-NULL. The device descriptor will be stored there,
* not in @udev->descriptor, because descriptors for registered
* devices are meant to be immutable.
*/
static int
hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1,
int retry_counter, struct usb_device_descriptor *dev_descr)
{
struct usb_device *hdev = hub->hdev;
struct usb_hcd *hcd = bus_to_hcd(hdev->bus);
struct usb_port *port_dev = hub->ports[port1 - 1];
int retries, operations, retval, i;
unsigned delay = HUB_SHORT_RESET_TIME;
enum usb_device_speed oldspeed = udev->speed;
const char *speed;
int devnum = udev->devnum;
const char *driver_name;
bool do_new_scheme;
const bool initial = !dev_descr;
int maxp0;
struct usb_device_descriptor *buf, *descr;
buf = kmalloc(GET_DESCRIPTOR_BUFSIZE, GFP_NOIO);
if (!buf)
return -ENOMEM;
/* root hub ports have a slightly longer reset period
* (from USB 2.0 spec, section 7.1.7.5)
*/
if (!hdev->parent) {
delay = HUB_ROOT_RESET_TIME;
if (port1 == hdev->bus->otg_port) hdev->bus->b_hnp_enable = 0;
}
/* Some low speed devices have problems with the quick delay, so */
/* be a bit pessimistic with those devices. RHbug #23670 */
if (oldspeed == USB_SPEED_LOW)
delay = HUB_LONG_RESET_TIME;
/* Reset the device; full speed may morph to high speed */
/* FIXME a USB 2.0 device may morph into SuperSpeed on reset. */
retval = hub_port_reset(hub, port1, udev, delay, false);
if (retval < 0) /* error or disconnect */
goto fail;
/* success, speed is known */
retval = -ENODEV;
/* Don't allow speed changes at reset, except usb 3.0 to faster */
if (oldspeed != USB_SPEED_UNKNOWN && oldspeed != udev->speed && !(oldspeed == USB_SPEED_SUPER && udev->speed > oldspeed)) { dev_dbg(&udev->dev, "device reset changed speed!\n");
goto fail;
}
oldspeed = udev->speed;
if (initial) {
/* USB 2.0 section 5.5.3 talks about ep0 maxpacket ...
* it's fixed size except for full speed devices.
*/
switch (udev->speed) {
case USB_SPEED_SUPER_PLUS:
case USB_SPEED_SUPER:
udev->ep0.desc.wMaxPacketSize = cpu_to_le16(512);
break;
case USB_SPEED_HIGH: /* fixed at 64 */
udev->ep0.desc.wMaxPacketSize = cpu_to_le16(64);
break;
case USB_SPEED_FULL: /* 8, 16, 32, or 64 */
/* to determine the ep0 maxpacket size, try to read
* the device descriptor to get bMaxPacketSize0 and
* then correct our initial guess.
*/
udev->ep0.desc.wMaxPacketSize = cpu_to_le16(64);
break;
case USB_SPEED_LOW: /* fixed at 8 */
udev->ep0.desc.wMaxPacketSize = cpu_to_le16(8);
break;
default:
goto fail;
}
}
speed = usb_speed_string(udev->speed);
/*
* The controller driver may be NULL if the controller device
* is the middle device between platform device and roothub.
* This middle device may not need a device driver due to
* all hardware control can be at platform device driver, this
* platform device is usually a dual-role USB controller device.
*/
if (udev->bus->controller->driver)
driver_name = udev->bus->controller->driver->name;
else
driver_name = udev->bus->sysdev->driver->name; if (udev->speed < USB_SPEED_SUPER) dev_info(&udev->dev,
"%s %s USB device number %d using %s\n",
(initial ? "new" : "reset"), speed,
devnum, driver_name);
if (initial) {
/* Set up TT records, if needed */
if (hdev->tt) { udev->tt = hdev->tt;
udev->ttport = hdev->ttport;
} else if (udev->speed != USB_SPEED_HIGH && hdev->speed == USB_SPEED_HIGH) { if (!hub->tt.hub) { dev_err(&udev->dev, "parent hub has no TT\n");
retval = -EINVAL;
goto fail;
}
udev->tt = &hub->tt;
udev->ttport = port1;
}
}
/* Why interleave GET_DESCRIPTOR and SET_ADDRESS this way?
* Because device hardware and firmware is sometimes buggy in
* this area, and this is how Linux has done it for ages.
* Change it cautiously.
*
* NOTE: If use_new_scheme() is true we will start by issuing
* a 64-byte GET_DESCRIPTOR request. This is what Windows does,
* so it may help with some non-standards-compliant devices.
* Otherwise we start with SET_ADDRESS and then try to read the
* first 8 bytes of the device descriptor to get the ep0 maxpacket
* value.
*/
do_new_scheme = use_new_scheme(udev, retry_counter, port_dev); for (retries = 0; retries < GET_DESCRIPTOR_TRIES; (++retries, msleep(100))) { if (hub_port_stop_enumerate(hub, port1, retries)) {
retval = -ENODEV;
break;
}
if (do_new_scheme) {
retval = hub_enable_device(udev);
if (retval < 0) {
dev_err(&udev->dev,
"hub failed to enable device, error %d\n",
retval);
goto fail;
}
maxp0 = get_bMaxPacketSize0(udev, buf,
GET_DESCRIPTOR_BUFSIZE, retries == 0);
if (maxp0 > 0 && !initial && maxp0 != udev->descriptor.bMaxPacketSize0) { dev_err(&udev->dev, "device reset changed ep0 maxpacket size!\n");
retval = -ENODEV;
goto fail;
}
retval = hub_port_reset(hub, port1, udev, delay, false);
if (retval < 0) /* error or disconnect */
goto fail;
if (oldspeed != udev->speed) { dev_dbg(&udev->dev,
"device reset changed speed!\n");
retval = -ENODEV;
goto fail;
}
if (maxp0 < 0) { if (maxp0 != -ENODEV) dev_err(&udev->dev, "device descriptor read/64, error %d\n",
maxp0);
retval = maxp0;
continue;
}
}
for (operations = 0; operations < SET_ADDRESS_TRIES; ++operations) {
retval = hub_set_address(udev, devnum); if (retval >= 0)
break;
msleep(200);
}
if (retval < 0) { if (retval != -ENODEV) dev_err(&udev->dev, "device not accepting address %d, error %d\n",
devnum, retval);
goto fail;
}
if (udev->speed >= USB_SPEED_SUPER) { devnum = udev->devnum; dev_info(&udev->dev,
"%s SuperSpeed%s%s USB device number %d using %s\n",
(udev->config) ? "reset" : "new",
(udev->speed == USB_SPEED_SUPER_PLUS) ?
" Plus" : "",
(udev->ssp_rate == USB_SSP_GEN_2x2) ?
" Gen 2x2" :
(udev->ssp_rate == USB_SSP_GEN_2x1) ?
" Gen 2x1" :
(udev->ssp_rate == USB_SSP_GEN_1x2) ?
" Gen 1x2" : "",
devnum, driver_name);
}
/*
* cope with hardware quirkiness:
* - let SET_ADDRESS settle, some device hardware wants it
* - read ep0 maxpacket even for high and low speed,
*/
msleep(10);
if (do_new_scheme)
break;
maxp0 = get_bMaxPacketSize0(udev, buf, 8, retries == 0);
if (maxp0 < 0) {
retval = maxp0;
if (retval != -ENODEV) dev_err(&udev->dev,
"device descriptor read/8, error %d\n",
retval);
} else {
u32 delay;
if (!initial && maxp0 != udev->descriptor.bMaxPacketSize0) {
dev_err(&udev->dev, "device reset changed ep0 maxpacket size!\n");
retval = -ENODEV;
goto fail;
}
delay = udev->parent->hub_delay;
udev->hub_delay = min_t(u32, delay,
USB_TP_TRANSMISSION_DELAY_MAX);
retval = usb_set_isoch_delay(udev);
if (retval) {
dev_dbg(&udev->dev,
"Failed set isoch delay, error %d\n",
retval);
retval = 0;
}
break;
}
}
if (retval)
goto fail;
/*
* Check the ep0 maxpacket guess and correct it if necessary.
* maxp0 is the value stored in the device descriptor;
* i is the value it encodes (logarithmic for SuperSpeed or greater).
*/
i = maxp0;
if (udev->speed >= USB_SPEED_SUPER) { if (maxp0 <= 16) i = 1 << maxp0;
else
i = 0; /* Invalid */
}
if (usb_endpoint_maxp(&udev->ep0.desc) == i) {
; /* Initial ep0 maxpacket guess is right */
} else if (((udev->speed == USB_SPEED_FULL ||
udev->speed == USB_SPEED_HIGH) &&
(i == 8 || i == 16 || i == 32 || i == 64)) || (udev->speed >= USB_SPEED_SUPER && i > 0)) {
/* Initial guess is wrong; use the descriptor's value */
if (udev->speed == USB_SPEED_FULL) dev_dbg(&udev->dev, "ep0 maxpacket = %d\n", i);
else
dev_warn(&udev->dev, "Using ep0 maxpacket: %d\n", i); udev->ep0.desc.wMaxPacketSize = cpu_to_le16(i);
usb_ep0_reinit(udev);
} else {
/* Initial guess is wrong and descriptor's value is invalid */
dev_err(&udev->dev, "Invalid ep0 maxpacket: %d\n", maxp0);
retval = -EMSGSIZE;
goto fail;
}
descr = usb_get_device_descriptor(udev);
if (IS_ERR(descr)) {
retval = PTR_ERR(descr);
if (retval != -ENODEV)
dev_err(&udev->dev, "device descriptor read/all, error %d\n",
retval);
goto fail;
}
if (initial) udev->descriptor = *descr;
else
*dev_descr = *descr; kfree(descr);
/*
* Some superspeed devices have finished the link training process
* and attached to a superspeed hub port, but the device descriptor
* got from those devices show they aren't superspeed devices. Warm
* reset the port attached by the devices can fix them.
*/
if ((udev->speed >= USB_SPEED_SUPER) &&
(le16_to_cpu(udev->descriptor.bcdUSB) < 0x0300)) { dev_err(&udev->dev, "got a wrong device descriptor, warm reset device\n");
hub_port_reset(hub, port1, udev, HUB_BH_RESET_TIME, true);
retval = -EINVAL;
goto fail;
}
usb_detect_quirks(udev);
if (le16_to_cpu(udev->descriptor.bcdUSB) >= 0x0201) {
retval = usb_get_bos_descriptor(udev);
if (!retval) {
udev->lpm_capable = usb_device_supports_lpm(udev);
udev->lpm_disable_count = 1;
usb_set_lpm_parameters(udev); usb_req_set_sel(udev);
}
}
retval = 0;
/* notify HCD that we have a device connected and addressed */
if (hcd->driver->update_device) hcd->driver->update_device(hcd, udev); hub_set_initial_usb2_lpm_policy(udev);
fail:
if (retval) { hub_port_disable(hub, port1, 0); update_devnum(udev, devnum); /* for disconnect processing */
}
kfree(buf); return retval;
}
static void
check_highspeed(struct usb_hub *hub, struct usb_device *udev, int port1)
{
struct usb_qualifier_descriptor *qual;
int status;
if (udev->quirks & USB_QUIRK_DEVICE_QUALIFIER)
return;
qual = kmalloc(sizeof *qual, GFP_KERNEL);
if (qual == NULL)
return;
status = usb_get_descriptor(udev, USB_DT_DEVICE_QUALIFIER, 0,
qual, sizeof *qual);
if (status == sizeof *qual) {
dev_info(&udev->dev, "not running at top speed; "
"connect to a high speed hub\n");
/* hub LEDs are probably harder to miss than syslog */
if (hub->has_indicators) {
hub->indicator[port1-1] = INDICATOR_GREEN_BLINK;
queue_delayed_work(system_power_efficient_wq,
&hub->leds, 0);
}
}
kfree(qual);
}
static unsigned
hub_power_remaining(struct usb_hub *hub)
{
struct usb_device *hdev = hub->hdev;
int remaining;
int port1;
if (!hub->limited_power)
return 0;
remaining = hdev->bus_mA - hub->descriptor->bHubContrCurrent; for (port1 = 1; port1 <= hdev->maxchild; ++port1) { struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *udev = port_dev->child;
unsigned unit_load;
int delta;
if (!udev)
continue;
if (hub_is_superspeed(udev))
unit_load = 150;
else
unit_load = 100;
/*
* Unconfigured devices may not use more than one unit load,
* or 8mA for OTG ports
*/
if (udev->actconfig) delta = usb_get_max_power(udev, udev->actconfig); else if (port1 != udev->bus->otg_port || hdev->parent) delta = unit_load;
else
delta = 8;
if (delta > hub->mA_per_port) dev_warn(&port_dev->dev, "%dmA is over %umA budget!\n",
delta, hub->mA_per_port);
remaining -= delta;
}
if (remaining < 0) { dev_warn(hub->intfdev, "%dmA over power budget!\n",
-remaining);
remaining = 0;
}
return remaining;
}
static int descriptors_changed(struct usb_device *udev,
struct usb_device_descriptor *new_device_descriptor,
struct usb_host_bos *old_bos)
{
int changed = 0;
unsigned index;
unsigned serial_len = 0;
unsigned len;
unsigned old_length;
int length;
char *buf;
if (memcmp(&udev->descriptor, new_device_descriptor,
sizeof(*new_device_descriptor)) != 0)
return 1; if ((old_bos && !udev->bos) || (!old_bos && udev->bos))
return 1;
if (udev->bos) {
len = le16_to_cpu(udev->bos->desc->wTotalLength);
if (len != le16_to_cpu(old_bos->desc->wTotalLength))
return 1;
if (memcmp(udev->bos->desc, old_bos->desc, len))
return 1;
}
/* Since the idVendor, idProduct, and bcdDevice values in the
* device descriptor haven't changed, we will assume the
* Manufacturer and Product strings haven't changed either.
* But the SerialNumber string could be different (e.g., a
* different flash card of the same brand).
*/
if (udev->serial) serial_len = strlen(udev->serial) + 1;
len = serial_len;
for (index = 0; index < udev->descriptor.bNumConfigurations; index++) { old_length = le16_to_cpu(udev->config[index].desc.wTotalLength);
len = max(len, old_length);
}
buf = kmalloc(len, GFP_NOIO);
if (!buf)
/* assume the worst */
return 1;
for (index = 0; index < udev->descriptor.bNumConfigurations; index++) { old_length = le16_to_cpu(udev->config[index].desc.wTotalLength);
length = usb_get_descriptor(udev, USB_DT_CONFIG, index, buf,
old_length);
if (length != old_length) {
dev_dbg(&udev->dev, "config index %d, error %d\n",
index, length);
changed = 1;
break;
}
if (memcmp(buf, udev->rawdescriptors[index], old_length)
!= 0) {
dev_dbg(&udev->dev, "config index %d changed (#%d)\n",
index,
((struct usb_config_descriptor *) buf)->
bConfigurationValue);
changed = 1;
break;
}
}
if (!changed && serial_len) { length = usb_string(udev, udev->descriptor.iSerialNumber,
buf, serial_len);
if (length + 1 != serial_len) {
dev_dbg(&udev->dev, "serial string error %d\n",
length);
changed = 1;
} else if (memcmp(buf, udev->serial, length) != 0) { dev_dbg(&udev->dev, "serial string changed\n");
changed = 1;
}
}
kfree(buf);
return changed;
}
static void hub_port_connect(struct usb_hub *hub, int port1, u16 portstatus,
u16 portchange)
{
int status = -ENODEV;
int i;
unsigned unit_load;
struct usb_device *hdev = hub->hdev;
struct usb_hcd *hcd = bus_to_hcd(hdev->bus);
struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *udev = port_dev->child;
static int unreliable_port = -1;
bool retry_locked;
/* Disconnect any existing devices under this port */
if (udev) {
if (hcd->usb_phy && !hdev->parent) usb_phy_notify_disconnect(hcd->usb_phy, udev->speed); usb_disconnect(&port_dev->child);
}
/* We can forget about a "removed" device when there's a physical
* disconnect or the connect status changes.
*/
if (!(portstatus & USB_PORT_STAT_CONNECTION) ||
(portchange & USB_PORT_STAT_C_CONNECTION))
clear_bit(port1, hub->removed_bits); if (portchange & (USB_PORT_STAT_C_CONNECTION |
USB_PORT_STAT_C_ENABLE)) {
status = hub_port_debounce_be_stable(hub, port1);
if (status < 0) {
if (status != -ENODEV && port1 != unreliable_port && printk_ratelimit()) dev_err(&port_dev->dev, "connect-debounce failed\n"); portstatus &= ~USB_PORT_STAT_CONNECTION;
unreliable_port = port1;
} else {
portstatus = status;
}
}
/* Return now if debouncing failed or nothing is connected or
* the device was "removed".
*/
if (!(portstatus & USB_PORT_STAT_CONNECTION) || test_bit(port1, hub->removed_bits)) {
/*
* maybe switch power back on (e.g. root hub was reset)
* but only if the port isn't owned by someone else.
*/
if (hub_is_port_power_switchable(hub) && !usb_port_is_power_on(hub, portstatus) && !port_dev->port_owner) set_port_feature(hdev, port1, USB_PORT_FEAT_POWER); if (portstatus & USB_PORT_STAT_ENABLE)
goto done;
return;
}
if (hub_is_superspeed(hub->hdev))
unit_load = 150;
else
unit_load = 100;
status = 0;
for (i = 0; i < PORT_INIT_TRIES; i++) { if (hub_port_stop_enumerate(hub, port1, i)) {
status = -ENODEV;
break;
}
usb_lock_port(port_dev);
mutex_lock(hcd->address0_mutex);
retry_locked = true;
/* reallocate for each attempt, since references
* to the previous one can escape in various ways
*/
udev = usb_alloc_dev(hdev, hdev->bus, port1);
if (!udev) {
dev_err(&port_dev->dev,
"couldn't allocate usb_device\n");
mutex_unlock(hcd->address0_mutex);
usb_unlock_port(port_dev);
goto done;
}
usb_set_device_state(udev, USB_STATE_POWERED);
udev->bus_mA = hub->mA_per_port;
udev->level = hdev->level + 1;
/* Devices connected to SuperSpeed hubs are USB 3.0 or later */
if (hub_is_superspeed(hub->hdev))
udev->speed = USB_SPEED_SUPER;
else
udev->speed = USB_SPEED_UNKNOWN;
choose_devnum(udev);
if (udev->devnum <= 0) {
status = -ENOTCONN; /* Don't retry */
goto loop;
}
/* reset (non-USB 3.0 devices) and get descriptor */
status = hub_port_init(hub, udev, port1, i, NULL);
if (status < 0)
goto loop;
mutex_unlock(hcd->address0_mutex);
usb_unlock_port(port_dev);
retry_locked = false;
if (udev->quirks & USB_QUIRK_DELAY_INIT)
msleep(2000);
/* consecutive bus-powered hubs aren't reliable; they can
* violate the voltage drop budget. if the new child has
* a "powered" LED, users should notice we didn't enable it
* (without reading syslog), even without per-port LEDs
* on the parent.
*/
if (udev->descriptor.bDeviceClass == USB_CLASS_HUB && udev->bus_mA <= unit_load) { u16 devstat;
status = usb_get_std_status(udev, USB_RECIP_DEVICE, 0,
&devstat);
if (status) {
dev_dbg(&udev->dev, "get status %d ?\n", status); goto loop_disable;
}
if ((devstat & (1 << USB_DEVICE_SELF_POWERED)) == 0) { dev_err(&udev->dev,
"can't connect bus-powered hub "
"to this port\n");
if (hub->has_indicators) {
hub->indicator[port1-1] =
INDICATOR_AMBER_BLINK;
queue_delayed_work(
system_power_efficient_wq,
&hub->leds, 0);
}
status = -ENOTCONN; /* Don't retry */
goto loop_disable;
}
}
/* check for devices running slower than they could */
if (le16_to_cpu(udev->descriptor.bcdUSB) >= 0x0200 && udev->speed == USB_SPEED_FULL && highspeed_hubs != 0) check_highspeed(hub, udev, port1);
/* Store the parent's children[] pointer. At this point
* udev becomes globally accessible, although presumably
* no one will look at it until hdev is unlocked.
*/
status = 0;
mutex_lock(&usb_port_peer_mutex);
/* We mustn't add new devices if the parent hub has
* been disconnected; we would race with the
* recursively_mark_NOTATTACHED() routine.
*/
spin_lock_irq(&device_state_lock);
if (hdev->state == USB_STATE_NOTATTACHED)
status = -ENOTCONN;
else
port_dev->child = udev; spin_unlock_irq(&device_state_lock);
mutex_unlock(&usb_port_peer_mutex);
/* Run it through the hoops (find a driver, etc) */
if (!status) {
status = usb_new_device(udev);
if (status) {
mutex_lock(&usb_port_peer_mutex);
spin_lock_irq(&device_state_lock);
port_dev->child = NULL;
spin_unlock_irq(&device_state_lock);
mutex_unlock(&usb_port_peer_mutex);
} else {
if (hcd->usb_phy && !hdev->parent) usb_phy_notify_connect(hcd->usb_phy,
udev->speed);
}
}
if (status)
goto loop_disable;
status = hub_power_remaining(hub); if (status) dev_dbg(hub->intfdev, "%dmA power budget left\n", status);
return;
loop_disable:
hub_port_disable(hub, port1, 1);
loop:
usb_ep0_reinit(udev);
release_devnum(udev);
hub_free_dev(udev); if (retry_locked) { mutex_unlock(hcd->address0_mutex);
usb_unlock_port(port_dev);
}
usb_put_dev(udev); if ((status == -ENOTCONN) || (status == -ENOTSUPP))
break;
/* When halfway through our retry count, power-cycle the port */
if (i == (PORT_INIT_TRIES - 1) / 2) { dev_info(&port_dev->dev, "attempt power cycle\n");
usb_hub_set_port_power(hdev, hub, port1, false);
msleep(2 * hub_power_on_good_delay(hub));
usb_hub_set_port_power(hdev, hub, port1, true);
msleep(hub_power_on_good_delay(hub));
}
}
if (hub->hdev->parent || !hcd->driver->port_handed_over || !(hcd->driver->port_handed_over)(hcd, port1)) { if (status != -ENOTCONN && status != -ENODEV) dev_err(&port_dev->dev,
"unable to enumerate USB device\n");
}
done:
hub_port_disable(hub, port1, 1); if (hcd->driver->relinquish_port && !hub->hdev->parent) { if (status != -ENOTCONN && status != -ENODEV) hcd->driver->relinquish_port(hcd, port1);
}
}
/* Handle physical or logical connection change events.
* This routine is called when:
* a port connection-change occurs;
* a port enable-change occurs (often caused by EMI);
* usb_reset_and_verify_device() encounters changed descriptors (as from
* a firmware download)
* caller already locked the hub
*/
static void hub_port_connect_change(struct usb_hub *hub, int port1,
u16 portstatus, u16 portchange)
__must_hold(&port_dev->status_lock)
{
struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *udev = port_dev->child;
struct usb_device_descriptor *descr;
int status = -ENODEV;
dev_dbg(&port_dev->dev, "status %04x, change %04x, %s\n", portstatus,
portchange, portspeed(hub, portstatus));
if (hub->has_indicators) { set_port_led(hub, port1, HUB_LED_AUTO);
hub->indicator[port1-1] = INDICATOR_AUTO;
}
#ifdef CONFIG_USB_OTG
/* during HNP, don't repeat the debounce */
if (hub->hdev->bus->is_b_host) portchange &= ~(USB_PORT_STAT_C_CONNECTION |
USB_PORT_STAT_C_ENABLE);
#endif
/* Try to resuscitate an existing device */
if ((portstatus & USB_PORT_STAT_CONNECTION) && udev && udev->state != USB_STATE_NOTATTACHED) { if (portstatus & USB_PORT_STAT_ENABLE) {
/*
* USB-3 connections are initialized automatically by
* the hostcontroller hardware. Therefore check for
* changed device descriptors before resuscitating the
* device.
*/
descr = usb_get_device_descriptor(udev);
if (IS_ERR(descr)) {
dev_dbg(&udev->dev,
"can't read device descriptor %ld\n",
PTR_ERR(descr));
} else {
if (descriptors_changed(udev, descr,
udev->bos)) {
dev_dbg(&udev->dev,
"device descriptor has changed\n");
} else {
status = 0; /* Nothing to do */
}
kfree(descr);
}
#ifdef CONFIG_PM
} else if (udev->state == USB_STATE_SUSPENDED &&
udev->persist_enabled) {
/* For a suspended device, treat this as a
* remote wakeup event.
*/
usb_unlock_port(port_dev);
status = usb_remote_wakeup(udev);
usb_lock_port(port_dev);
#endif
} else {
/* Don't resuscitate */;
}
}
clear_bit(port1, hub->change_bits);
/* successfully revalidated the connection */
if (status == 0)
return;
usb_unlock_port(port_dev); hub_port_connect(hub, port1, portstatus, portchange); usb_lock_port(port_dev);
}
/* Handle notifying userspace about hub over-current events */
static void port_over_current_notify(struct usb_port *port_dev)
{
char *envp[3] = { NULL, NULL, NULL };
struct device *hub_dev;
char *port_dev_path;
sysfs_notify(&port_dev->dev.kobj, NULL, "over_current_count");
hub_dev = port_dev->dev.parent;
if (!hub_dev)
return;
port_dev_path = kobject_get_path(&port_dev->dev.kobj, GFP_KERNEL);
if (!port_dev_path)
return;
envp[0] = kasprintf(GFP_KERNEL, "OVER_CURRENT_PORT=%s", port_dev_path);
if (!envp[0])
goto exit;
envp[1] = kasprintf(GFP_KERNEL, "OVER_CURRENT_COUNT=%u",
port_dev->over_current_count);
if (!envp[1])
goto exit;
kobject_uevent_env(&hub_dev->kobj, KOBJ_CHANGE, envp);
exit:
kfree(envp[1]);
kfree(envp[0]);
kfree(port_dev_path);
}
static void port_event(struct usb_hub *hub, int port1)
__must_hold(&port_dev->status_lock)
{
int connect_change;
struct usb_port *port_dev = hub->ports[port1 - 1];
struct usb_device *udev = port_dev->child;
struct usb_device *hdev = hub->hdev;
u16 portstatus, portchange;
int i = 0;
connect_change = test_bit(port1, hub->change_bits);
clear_bit(port1, hub->event_bits);
clear_bit(port1, hub->wakeup_bits);
if (usb_hub_port_status(hub, port1, &portstatus, &portchange) < 0)
return;
if (portchange & USB_PORT_STAT_C_CONNECTION) { usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION);
connect_change = 1;
}
if (portchange & USB_PORT_STAT_C_ENABLE) { if (!connect_change) dev_dbg(&port_dev->dev, "enable change, status %08x\n",
portstatus);
usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_ENABLE);
/*
* EM interference sometimes causes badly shielded USB devices
* to be shutdown by the hub, this hack enables them again.
* Works at least with mouse driver.
*/
if (!(portstatus & USB_PORT_STAT_ENABLE)
&& !connect_change && udev) { dev_err(&port_dev->dev, "disabled by hub (EMI?), re-enabling...\n");
connect_change = 1;
}
}
if (portchange & USB_PORT_STAT_C_OVERCURRENT) { u16 status = 0, unused;
port_dev->over_current_count++;
port_over_current_notify(port_dev); dev_dbg(&port_dev->dev, "over-current change #%u\n",
port_dev->over_current_count);
usb_clear_port_feature(hdev, port1,
USB_PORT_FEAT_C_OVER_CURRENT);
msleep(100); /* Cool down */
hub_power_on(hub, true);
usb_hub_port_status(hub, port1, &status, &unused);
if (status & USB_PORT_STAT_OVERCURRENT) dev_err(&port_dev->dev, "over-current condition\n");
}
if (portchange & USB_PORT_STAT_C_RESET) { dev_dbg(&port_dev->dev, "reset change\n"); usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_RESET);
}
if ((portchange & USB_PORT_STAT_C_BH_RESET) && hub_is_superspeed(hdev)) { dev_dbg(&port_dev->dev, "warm reset change\n"); usb_clear_port_feature(hdev, port1,
USB_PORT_FEAT_C_BH_PORT_RESET);
}
if (portchange & USB_PORT_STAT_C_LINK_STATE) { dev_dbg(&port_dev->dev, "link state change\n"); usb_clear_port_feature(hdev, port1,
USB_PORT_FEAT_C_PORT_LINK_STATE);
}
if (portchange & USB_PORT_STAT_C_CONFIG_ERROR) { dev_warn(&port_dev->dev, "config error\n");
usb_clear_port_feature(hdev, port1,
USB_PORT_FEAT_C_PORT_CONFIG_ERROR);
}
/* skip port actions that require the port to be powered on */
if (!pm_runtime_active(&port_dev->dev))
return;
/* skip port actions if ignore_event and early_stop are true */
if (port_dev->ignore_event && port_dev->early_stop)
return;
if (hub_handle_remote_wakeup(hub, port1, portstatus, portchange))
connect_change = 1;
/*
* Avoid trying to recover a USB3 SS.Inactive port with a warm reset if
* the device was disconnected. A 12ms disconnect detect timer in
* SS.Inactive state transitions the port to RxDetect automatically.
* SS.Inactive link error state is common during device disconnect.
*/
while (hub_port_warm_reset_required(hub, port1, portstatus)) { if ((i++ < DETECT_DISCONNECT_TRIES) && udev) { u16 unused;
msleep(20);
usb_hub_port_status(hub, port1, &portstatus, &unused);
dev_dbg(&port_dev->dev, "Wait for inactive link disconnect detect\n"); continue; } else if (!udev || !(portstatus & USB_PORT_STAT_CONNECTION) || udev->state == USB_STATE_NOTATTACHED) { dev_dbg(&port_dev->dev, "do warm reset, port only\n"); if (hub_port_reset(hub, port1, NULL,
HUB_BH_RESET_TIME, true) < 0)
hub_port_disable(hub, port1, 1);
} else {
dev_dbg(&port_dev->dev, "do warm reset, full device\n"); usb_unlock_port(port_dev);
usb_lock_device(udev);
usb_reset_device(udev);
usb_unlock_device(udev);
usb_lock_port(port_dev);
connect_change = 0;
}
break;
}
if (connect_change) hub_port_connect_change(hub, port1, portstatus, portchange);
}
static void hub_event(struct work_struct *work)
{
struct usb_device *hdev;
struct usb_interface *intf;
struct usb_hub *hub;
struct device *hub_dev;
u16 hubstatus;
u16 hubchange;
int i, ret;
hub = container_of(work, struct usb_hub, events);
hdev = hub->hdev;
hub_dev = hub->intfdev;
intf = to_usb_interface(hub_dev);
kcov_remote_start_usb((u64)hdev->bus->busnum); dev_dbg(hub_dev, "state %d ports %d chg %04x evt %04x\n",
hdev->state, hdev->maxchild,
/* NOTE: expects max 15 ports... */
(u16) hub->change_bits[0],
(u16) hub->event_bits[0]);
/* Lock the device, then check to see if we were
* disconnected while waiting for the lock to succeed. */
usb_lock_device(hdev);
if (unlikely(hub->disconnected))
goto out_hdev_lock;
/* If the hub has died, clean up after it */
if (hdev->state == USB_STATE_NOTATTACHED) { hub->error = -ENODEV;
hub_quiesce(hub, HUB_DISCONNECT);
goto out_hdev_lock;
}
/* Autoresume */
ret = usb_autopm_get_interface(intf);
if (ret) {
dev_dbg(hub_dev, "Can't autoresume: %d\n", ret);
goto out_hdev_lock;
}
/* If this is an inactive hub, do nothing */
if (hub->quiescing)
goto out_autopm;
if (hub->error) { dev_dbg(hub_dev, "resetting for error %d\n", hub->error); ret = usb_reset_device(hdev);
if (ret) {
dev_dbg(hub_dev, "error resetting hub: %d\n", ret);
goto out_autopm;
}
hub->nerrors = 0;
hub->error = 0;
}
/* deal with port status changes */
for (i = 1; i <= hdev->maxchild; i++) { struct usb_port *port_dev = hub->ports[i - 1];
if (test_bit(i, hub->event_bits)
|| test_bit(i, hub->change_bits) || test_bit(i, hub->wakeup_bits)) {
/*
* The get_noresume and barrier ensure that if
* the port was in the process of resuming, we
* flush that work and keep the port active for
* the duration of the port_event(). However,
* if the port is runtime pm suspended
* (powered-off), we leave it in that state, run
* an abbreviated port_event(), and move on.
*/
pm_runtime_get_noresume(&port_dev->dev);
pm_runtime_barrier(&port_dev->dev);
usb_lock_port(port_dev);
port_event(hub, i);
usb_unlock_port(port_dev);
pm_runtime_put_sync(&port_dev->dev);
}
}
/* deal with hub status changes */
if (test_and_clear_bit(0, hub->event_bits) == 0)
; /* do nothing */
else if (hub_hub_status(hub, &hubstatus, &hubchange) < 0) dev_err(hub_dev, "get_hub_status failed\n");
else {
if (hubchange & HUB_CHANGE_LOCAL_POWER) { dev_dbg(hub_dev, "power change\n"); clear_hub_feature(hdev, C_HUB_LOCAL_POWER);
if (hubstatus & HUB_STATUS_LOCAL_POWER)
/* FIXME: Is this always true? */
hub->limited_power = 1;
else
hub->limited_power = 0;
}
if (hubchange & HUB_CHANGE_OVERCURRENT) { u16 status = 0;
u16 unused;
dev_dbg(hub_dev, "over-current change\n"); clear_hub_feature(hdev, C_HUB_OVER_CURRENT);
msleep(500); /* Cool down */
hub_power_on(hub, true);
hub_hub_status(hub, &status, &unused);
if (status & HUB_STATUS_OVERCURRENT) dev_err(hub_dev, "over-current condition\n");
}
}
out_autopm:
/* Balance the usb_autopm_get_interface() above */
usb_autopm_put_interface_no_suspend(intf);
out_hdev_lock:
usb_unlock_device(hdev);
/* Balance the stuff in kick_hub_wq() and allow autosuspend */
usb_autopm_put_interface(intf);
hub_put(hub); kcov_remote_stop();
}
static const struct usb_device_id hub_id_table[] = {
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT
| USB_DEVICE_ID_MATCH_INT_CLASS,
.idVendor = USB_VENDOR_SMSC,
.idProduct = USB_PRODUCT_USB5534B,
.bInterfaceClass = USB_CLASS_HUB,
.driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT,
.idVendor = USB_VENDOR_CYPRESS,
.idProduct = USB_PRODUCT_CY7C65632,
.driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_INT_CLASS,
.idVendor = USB_VENDOR_GENESYS_LOGIC,
.bInterfaceClass = USB_CLASS_HUB,
.driver_info = HUB_QUIRK_CHECK_PORT_AUTOSUSPEND},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT,
.idVendor = USB_VENDOR_TEXAS_INSTRUMENTS,
.idProduct = USB_PRODUCT_TUSB8041_USB2,
.driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT,
.idVendor = USB_VENDOR_TEXAS_INSTRUMENTS,
.idProduct = USB_PRODUCT_TUSB8041_USB3,
.driver_info = HUB_QUIRK_DISABLE_AUTOSUSPEND},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT,
.idVendor = USB_VENDOR_MICROCHIP,
.idProduct = USB_PRODUCT_USB4913,
.driver_info = HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT,
.idVendor = USB_VENDOR_MICROCHIP,
.idProduct = USB_PRODUCT_USB4914,
.driver_info = HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL},
{ .match_flags = USB_DEVICE_ID_MATCH_VENDOR
| USB_DEVICE_ID_MATCH_PRODUCT,
.idVendor = USB_VENDOR_MICROCHIP,
.idProduct = USB_PRODUCT_USB4915,
.driver_info = HUB_QUIRK_REDUCE_FRAME_INTR_BINTERVAL},
{ .match_flags = USB_DEVICE_ID_MATCH_DEV_CLASS,
.bDeviceClass = USB_CLASS_HUB},
{ .match_flags = USB_DEVICE_ID_MATCH_INT_CLASS,
.bInterfaceClass = USB_CLASS_HUB},
{ } /* Terminating entry */
};
MODULE_DEVICE_TABLE(usb, hub_id_table);
static struct usb_driver hub_driver = {
.name = "hub",
.probe = hub_probe,
.disconnect = hub_disconnect,
.suspend = hub_suspend,
.resume = hub_resume,
.reset_resume = hub_reset_resume,
.pre_reset = hub_pre_reset,
.post_reset = hub_post_reset,
.unlocked_ioctl = hub_ioctl,
.id_table = hub_id_table,
.supports_autosuspend = 1,
};
int usb_hub_init(void)
{
if (usb_register(&hub_driver) < 0) {
printk(KERN_ERR "%s: can't register hub driver\n",
usbcore_name);
return -1;
}
/*
* The workqueue needs to be freezable to avoid interfering with
* USB-PERSIST port handover. Otherwise it might see that a full-speed
* device was gone before the EHCI controller had handed its port
* over to the companion full-speed controller.
*/
hub_wq = alloc_workqueue("usb_hub_wq", WQ_FREEZABLE, 0);
if (hub_wq)
return 0;
/* Fall through if kernel_thread failed */
usb_deregister(&hub_driver);
pr_err("%s: can't allocate workqueue for usb hub\n", usbcore_name);
return -1;
}
void usb_hub_cleanup(void)
{
destroy_workqueue(hub_wq);
/*
* Hub resources are freed for us by usb_deregister. It calls
* usb_driver_purge on every device which in turn calls that
* devices disconnect function if it is using this driver.
* The hub_disconnect function takes care of releasing the
* individual hub resources. -greg
*/
usb_deregister(&hub_driver);
} /* usb_hub_cleanup() */
/**
* usb_reset_and_verify_device - perform a USB port reset to reinitialize a device
* @udev: device to reset (not in SUSPENDED or NOTATTACHED state)
*
* WARNING - don't use this routine to reset a composite device
* (one with multiple interfaces owned by separate drivers)!
* Use usb_reset_device() instead.
*
* Do a port reset, reassign the device's address, and establish its
* former operating configuration. If the reset fails, or the device's
* descriptors change from their values before the reset, or the original
* configuration and altsettings cannot be restored, a flag will be set
* telling hub_wq to pretend the device has been disconnected and then
* re-connected. All drivers will be unbound, and the device will be
* re-enumerated and probed all over again.
*
* Return: 0 if the reset succeeded, -ENODEV if the device has been
* flagged for logical disconnection, or some other negative error code
* if the reset wasn't even attempted.
*
* Note:
* The caller must own the device lock and the port lock, the latter is
* taken by usb_reset_device(). For example, it's safe to use
* usb_reset_device() from a driver probe() routine after downloading
* new firmware. For calls that might not occur during probe(), drivers
* should lock the device using usb_lock_device_for_reset().
*
* Locking exception: This routine may also be called from within an
* autoresume handler. Such usage won't conflict with other tasks
* holding the device lock because these tasks should always call
* usb_autopm_resume_device(), thereby preventing any unwanted
* autoresume. The autoresume handler is expected to have already
* acquired the port lock before calling this routine.
*/
static int usb_reset_and_verify_device(struct usb_device *udev)
{
struct usb_device *parent_hdev = udev->parent;
struct usb_hub *parent_hub;
struct usb_hcd *hcd = bus_to_hcd(udev->bus);
struct usb_device_descriptor descriptor;
struct usb_host_bos *bos;
int i, j, ret = 0;
int port1 = udev->portnum;
if (udev->state == USB_STATE_NOTATTACHED ||
udev->state == USB_STATE_SUSPENDED) {
dev_dbg(&udev->dev, "device reset not allowed in state %d\n",
udev->state);
return -EINVAL;
}
if (!parent_hdev)
return -EISDIR;
parent_hub = usb_hub_to_struct_hub(parent_hdev);
/* Disable USB2 hardware LPM.
* It will be re-enabled by the enumeration process.
*/
usb_disable_usb2_hardware_lpm(udev);
bos = udev->bos;
udev->bos = NULL;
mutex_lock(hcd->address0_mutex);
for (i = 0; i < PORT_INIT_TRIES; ++i) { if (hub_port_stop_enumerate(parent_hub, port1, i)) {
ret = -ENODEV;
break;
}
/* ep0 maxpacket size may change; let the HCD know about it.
* Other endpoints will be handled by re-enumeration. */
usb_ep0_reinit(udev);
ret = hub_port_init(parent_hub, udev, port1, i, &descriptor);
if (ret >= 0 || ret == -ENOTCONN || ret == -ENODEV)
break;
}
mutex_unlock(hcd->address0_mutex);
if (ret < 0)
goto re_enumerate;
/* Device might have changed firmware (DFU or similar) */
if (descriptors_changed(udev, &descriptor, bos)) { dev_info(&udev->dev, "device firmware changed\n");
goto re_enumerate;
}
/* Restore the device's previous configuration */
if (!udev->actconfig)
goto done;
mutex_lock(hcd->bandwidth_mutex);
ret = usb_hcd_alloc_bandwidth(udev, udev->actconfig, NULL, NULL);
if (ret < 0) {
dev_warn(&udev->dev,
"Busted HC? Not enough HCD resources for "
"old configuration.\n");
mutex_unlock(hcd->bandwidth_mutex);
goto re_enumerate;
}
ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
USB_REQ_SET_CONFIGURATION, 0,
udev->actconfig->desc.bConfigurationValue, 0,
NULL, 0, USB_CTRL_SET_TIMEOUT);
if (ret < 0) {
dev_err(&udev->dev,
"can't restore configuration #%d (error=%d)\n",
udev->actconfig->desc.bConfigurationValue, ret);
mutex_unlock(hcd->bandwidth_mutex);
goto re_enumerate;
}
mutex_unlock(hcd->bandwidth_mutex);
usb_set_device_state(udev, USB_STATE_CONFIGURED);
/* Put interfaces back into the same altsettings as before.
* Don't bother to send the Set-Interface request for interfaces
* that were already in altsetting 0; besides being unnecessary,
* many devices can't handle it. Instead just reset the host-side
* endpoint state.
*/
for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) {
struct usb_host_config *config = udev->actconfig;
struct usb_interface *intf = config->interface[i];
struct usb_interface_descriptor *desc;
desc = &intf->cur_altsetting->desc;
if (desc->bAlternateSetting == 0) {
usb_disable_interface(udev, intf, true);
usb_enable_interface(udev, intf, true);
ret = 0;
} else {
/* Let the bandwidth allocation function know that this
* device has been reset, and it will have to use
* alternate setting 0 as the current alternate setting.
*/
intf->resetting_device = 1;
ret = usb_set_interface(udev, desc->bInterfaceNumber,
desc->bAlternateSetting);
intf->resetting_device = 0;
}
if (ret < 0) {
dev_err(&udev->dev, "failed to restore interface %d "
"altsetting %d (error=%d)\n",
desc->bInterfaceNumber,
desc->bAlternateSetting,
ret);
goto re_enumerate;
}
/* Resetting also frees any allocated streams */
for (j = 0; j < intf->cur_altsetting->desc.bNumEndpoints; j++) intf->cur_altsetting->endpoint[j].streams = 0;
}
done:
/* Now that the alt settings are re-installed, enable LTM and LPM. */
usb_enable_usb2_hardware_lpm(udev); usb_unlocked_enable_lpm(udev); usb_enable_ltm(udev);
usb_release_bos_descriptor(udev);
udev->bos = bos;
return 0;
re_enumerate:
usb_release_bos_descriptor(udev);
udev->bos = bos;
hub_port_logical_disconnect(parent_hub, port1);
return -ENODEV;
}
/**
* usb_reset_device - warn interface drivers and perform a USB port reset
* @udev: device to reset (not in NOTATTACHED state)
*
* Warns all drivers bound to registered interfaces (using their pre_reset
* method), performs the port reset, and then lets the drivers know that
* the reset is over (using their post_reset method).
*
* Return: The same as for usb_reset_and_verify_device().
* However, if a reset is already in progress (for instance, if a
* driver doesn't have pre_reset() or post_reset() callbacks, and while
* being unbound or re-bound during the ongoing reset its disconnect()
* or probe() routine tries to perform a second, nested reset), the
* routine returns -EINPROGRESS.
*
* Note:
* The caller must own the device lock. For example, it's safe to use
* this from a driver probe() routine after downloading new firmware.
* For calls that might not occur during probe(), drivers should lock
* the device using usb_lock_device_for_reset().
*
* If an interface is currently being probed or disconnected, we assume
* its driver knows how to handle resets. For all other interfaces,
* if the driver doesn't have pre_reset and post_reset methods then
* we attempt to unbind it and rebind afterward.
*/
int usb_reset_device(struct usb_device *udev)
{
int ret;
int i;
unsigned int noio_flag;
struct usb_port *port_dev;
struct usb_host_config *config = udev->actconfig;
struct usb_hub *hub = usb_hub_to_struct_hub(udev->parent);
if (udev->state == USB_STATE_NOTATTACHED) {
dev_dbg(&udev->dev, "device reset not allowed in state %d\n",
udev->state);
return -EINVAL;
}
if (!udev->parent) {
/* this requires hcd-specific logic; see ohci_restart() */
dev_dbg(&udev->dev, "%s for root hub!\n", __func__);
return -EISDIR;
}
if (udev->reset_in_progress)
return -EINPROGRESS;
udev->reset_in_progress = 1;
port_dev = hub->ports[udev->portnum - 1];
/*
* Don't allocate memory with GFP_KERNEL in current
* context to avoid possible deadlock if usb mass
* storage interface or usbnet interface(iSCSI case)
* is included in current configuration. The easist
* approach is to do it for every device reset,
* because the device 'memalloc_noio' flag may have
* not been set before reseting the usb device.
*/
noio_flag = memalloc_noio_save();
/* Prevent autosuspend during the reset */
usb_autoresume_device(udev);
if (config) {
for (i = 0; i < config->desc.bNumInterfaces; ++i) {
struct usb_interface *cintf = config->interface[i];
struct usb_driver *drv;
int unbind = 0;
if (cintf->dev.driver) {
drv = to_usb_driver(cintf->dev.driver);
if (drv->pre_reset && drv->post_reset)
unbind = (drv->pre_reset)(cintf);
else if (cintf->condition ==
USB_INTERFACE_BOUND)
unbind = 1;
if (unbind)
usb_forced_unbind_intf(cintf);
}
}
}
usb_lock_port(port_dev);
ret = usb_reset_and_verify_device(udev);
usb_unlock_port(port_dev);
if (config) {
for (i = config->desc.bNumInterfaces - 1; i >= 0; --i) {
struct usb_interface *cintf = config->interface[i];
struct usb_driver *drv;
int rebind = cintf->needs_binding;
if (!rebind && cintf->dev.driver) {
drv = to_usb_driver(cintf->dev.driver);
if (drv->post_reset)
rebind = (drv->post_reset)(cintf);
else if (cintf->condition ==
USB_INTERFACE_BOUND)
rebind = 1;
if (rebind)
cintf->needs_binding = 1;
}
}
/* If the reset failed, hub_wq will unbind drivers later */
if (ret == 0)
usb_unbind_and_rebind_marked_interfaces(udev);
}
usb_autosuspend_device(udev);
memalloc_noio_restore(noio_flag);
udev->reset_in_progress = 0;
return ret;
}
EXPORT_SYMBOL_GPL(usb_reset_device);
/**
* usb_queue_reset_device - Reset a USB device from an atomic context
* @iface: USB interface belonging to the device to reset
*
* This function can be used to reset a USB device from an atomic
* context, where usb_reset_device() won't work (as it blocks).
*
* Doing a reset via this method is functionally equivalent to calling
* usb_reset_device(), except for the fact that it is delayed to a
* workqueue. This means that any drivers bound to other interfaces
* might be unbound, as well as users from usbfs in user space.
*
* Corner cases:
*
* - Scheduling two resets at the same time from two different drivers
* attached to two different interfaces of the same device is
* possible; depending on how the driver attached to each interface
* handles ->pre_reset(), the second reset might happen or not.
*
* - If the reset is delayed so long that the interface is unbound from
* its driver, the reset will be skipped.
*
* - This function can be called during .probe(). It can also be called
* during .disconnect(), but doing so is pointless because the reset
* will not occur. If you really want to reset the device during
* .disconnect(), call usb_reset_device() directly -- but watch out
* for nested unbinding issues!
*/
void usb_queue_reset_device(struct usb_interface *iface)
{
if (schedule_work(&iface->reset_ws))
usb_get_intf(iface);
}
EXPORT_SYMBOL_GPL(usb_queue_reset_device);
/**
* usb_hub_find_child - Get the pointer of child device
* attached to the port which is specified by @port1.
* @hdev: USB device belonging to the usb hub
* @port1: port num to indicate which port the child device
* is attached to.
*
* USB drivers call this function to get hub's child device
* pointer.
*
* Return: %NULL if input param is invalid and
* child's usb_device pointer if non-NULL.
*/
struct usb_device *usb_hub_find_child(struct usb_device *hdev,
int port1)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev); if (port1 < 1 || port1 > hdev->maxchild)
return NULL;
return hub->ports[port1 - 1]->child;}
EXPORT_SYMBOL_GPL(usb_hub_find_child);
void usb_hub_adjust_deviceremovable(struct usb_device *hdev,
struct usb_hub_descriptor *desc)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
enum usb_port_connect_type connect_type;
int i;
if (!hub)
return;
if (!hub_is_superspeed(hdev)) {
for (i = 1; i <= hdev->maxchild; i++) {
struct usb_port *port_dev = hub->ports[i - 1];
connect_type = port_dev->connect_type;
if (connect_type == USB_PORT_CONNECT_TYPE_HARD_WIRED) {
u8 mask = 1 << (i%8);
if (!(desc->u.hs.DeviceRemovable[i/8] & mask)) {
dev_dbg(&port_dev->dev, "DeviceRemovable is changed to 1 according to platform information.\n");
desc->u.hs.DeviceRemovable[i/8] |= mask;
}
}
}
} else {
u16 port_removable = le16_to_cpu(desc->u.ss.DeviceRemovable);
for (i = 1; i <= hdev->maxchild; i++) {
struct usb_port *port_dev = hub->ports[i - 1];
connect_type = port_dev->connect_type;
if (connect_type == USB_PORT_CONNECT_TYPE_HARD_WIRED) {
u16 mask = 1 << i;
if (!(port_removable & mask)) {
dev_dbg(&port_dev->dev, "DeviceRemovable is changed to 1 according to platform information.\n");
port_removable |= mask;
}
}
}
desc->u.ss.DeviceRemovable = cpu_to_le16(port_removable);
}
}
#ifdef CONFIG_ACPI
/**
* usb_get_hub_port_acpi_handle - Get the usb port's acpi handle
* @hdev: USB device belonging to the usb hub
* @port1: port num of the port
*
* Return: Port's acpi handle if successful, %NULL if params are
* invalid.
*/
acpi_handle usb_get_hub_port_acpi_handle(struct usb_device *hdev,
int port1)
{
struct usb_hub *hub = usb_hub_to_struct_hub(hdev);
if (!hub)
return NULL;
return ACPI_HANDLE(&hub->ports[port1 - 1]->dev);
}
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
*
* NOHZ implementation for low and high resolution timers
*
* Started by: Thomas Gleixner and Ingo Molnar
*/
#include <linux/compiler.h>
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>
#include <linux/nmi.h>
#include <linux/profile.h>
#include <linux/sched/signal.h>
#include <linux/sched/clock.h>
#include <linux/sched/stat.h>
#include <linux/sched/nohz.h>
#include <linux/sched/loadavg.h>
#include <linux/module.h>
#include <linux/irq_work.h>
#include <linux/posix-timers.h>
#include <linux/context_tracking.h>
#include <linux/mm.h>
#include <asm/irq_regs.h>
#include "tick-internal.h"
#include <trace/events/timer.h>
/*
* Per-CPU nohz control structure
*/
static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
struct tick_sched *tick_get_tick_sched(int cpu)
{
return &per_cpu(tick_cpu_sched, cpu);
}
/*
* The time when the last jiffy update happened. Write access must hold
* jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a
* consistent view of jiffies and last_jiffies_update.
*/
static ktime_t last_jiffies_update;
/*
* Must be called with interrupts disabled !
*/
static void tick_do_update_jiffies64(ktime_t now)
{
unsigned long ticks = 1;
ktime_t delta, nextp;
/*
* 64-bit can do a quick check without holding the jiffies lock and
* without looking at the sequence count. The smp_load_acquire()
* pairs with the update done later in this function.
*
* 32-bit cannot do that because the store of 'tick_next_period'
* consists of two 32-bit stores, and the first store could be
* moved by the CPU to a random point in the future.
*/
if (IS_ENABLED(CONFIG_64BIT)) {
if (ktime_before(now, smp_load_acquire(&tick_next_period)))
return;
} else {
unsigned int seq;
/*
* Avoid contention on 'jiffies_lock' and protect the quick
* check with the sequence count.
*/
do {
seq = read_seqcount_begin(&jiffies_seq);
nextp = tick_next_period;
} while (read_seqcount_retry(&jiffies_seq, seq));
if (ktime_before(now, nextp))
return;
}
/* Quick check failed, i.e. update is required. */
raw_spin_lock(&jiffies_lock);
/*
* Re-evaluate with the lock held. Another CPU might have done the
* update already.
*/
if (ktime_before(now, tick_next_period)) {
raw_spin_unlock(&jiffies_lock);
return;
}
write_seqcount_begin(&jiffies_seq);
delta = ktime_sub(now, tick_next_period);
if (unlikely(delta >= TICK_NSEC)) {
/* Slow path for long idle sleep times */
s64 incr = TICK_NSEC;
ticks += ktime_divns(delta, incr);
last_jiffies_update = ktime_add_ns(last_jiffies_update,
incr * ticks);
} else {
last_jiffies_update = ktime_add_ns(last_jiffies_update,
TICK_NSEC);
}
/* Advance jiffies to complete the 'jiffies_seq' protected job */
jiffies_64 += ticks;
/* Keep the tick_next_period variable up to date */
nextp = ktime_add_ns(last_jiffies_update, TICK_NSEC);
if (IS_ENABLED(CONFIG_64BIT)) {
/*
* Pairs with smp_load_acquire() in the lockless quick
* check above, and ensures that the update to 'jiffies_64' is
* not reordered vs. the store to 'tick_next_period', neither
* by the compiler nor by the CPU.
*/
smp_store_release(&tick_next_period, nextp);
} else {
/*
* A plain store is good enough on 32-bit, as the quick check
* above is protected by the sequence count.
*/
tick_next_period = nextp;
}
/*
* Release the sequence count. calc_global_load() below is not
* protected by it, but 'jiffies_lock' needs to be held to prevent
* concurrent invocations.
*/
write_seqcount_end(&jiffies_seq);
calc_global_load();
raw_spin_unlock(&jiffies_lock);
update_wall_time();
}
/*
* Initialize and return retrieve the jiffies update.
*/
static ktime_t tick_init_jiffy_update(void)
{
ktime_t period;
raw_spin_lock(&jiffies_lock);
write_seqcount_begin(&jiffies_seq);
/* Have we started the jiffies update yet ? */
if (last_jiffies_update == 0) {
u32 rem;
/*
* Ensure that the tick is aligned to a multiple of
* TICK_NSEC.
*/
div_u64_rem(tick_next_period, TICK_NSEC, &rem);
if (rem)
tick_next_period += TICK_NSEC - rem;
last_jiffies_update = tick_next_period;
}
period = last_jiffies_update;
write_seqcount_end(&jiffies_seq);
raw_spin_unlock(&jiffies_lock);
return period;
}
static inline int tick_sched_flag_test(struct tick_sched *ts,
unsigned long flag)
{
return !!(ts->flags & flag);
}
static inline void tick_sched_flag_set(struct tick_sched *ts,
unsigned long flag)
{
lockdep_assert_irqs_disabled();
ts->flags |= flag;
}
static inline void tick_sched_flag_clear(struct tick_sched *ts,
unsigned long flag)
{
lockdep_assert_irqs_disabled();
ts->flags &= ~flag;
}
#define MAX_STALLED_JIFFIES 5
static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
{
int tick_cpu, cpu = smp_processor_id();
/*
* Check if the do_timer duty was dropped. We don't care about
* concurrency: This happens only when the CPU in charge went
* into a long sleep. If two CPUs happen to assign themselves to
* this duty, then the jiffies update is still serialized by
* 'jiffies_lock'.
*
* If nohz_full is enabled, this should not happen because the
* 'tick_do_timer_cpu' CPU never relinquishes.
*/
tick_cpu = READ_ONCE(tick_do_timer_cpu);
if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && unlikely(tick_cpu == TICK_DO_TIMER_NONE)) {
#ifdef CONFIG_NO_HZ_FULL
WARN_ON_ONCE(tick_nohz_full_running);
#endif
WRITE_ONCE(tick_do_timer_cpu, cpu);
tick_cpu = cpu;
}
/* Check if jiffies need an update */
if (tick_cpu == cpu) tick_do_update_jiffies64(now);
/*
* If the jiffies update stalled for too long (timekeeper in stop_machine()
* or VMEXIT'ed for several msecs), force an update.
*/
if (ts->last_tick_jiffies != jiffies) { ts->stalled_jiffies = 0;
ts->last_tick_jiffies = READ_ONCE(jiffies);
} else {
if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) { tick_do_update_jiffies64(now);
ts->stalled_jiffies = 0;
ts->last_tick_jiffies = READ_ONCE(jiffies);
}
}
if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) ts->got_idle_tick = 1;
}
static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
{
/*
* When we are idle and the tick is stopped, we have to touch
* the watchdog as we might not schedule for a really long
* time. This happens on completely idle SMP systems while
* waiting on the login prompt. We also increment the "start of
* idle" jiffy stamp so the idle accounting adjustment we do
* when we go busy again does not account too many ticks.
*/
if (IS_ENABLED(CONFIG_NO_HZ_COMMON) &&
tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { touch_softlockup_watchdog_sched();
if (is_idle_task(current))
ts->idle_jiffies++;
/*
* In case the current tick fired too early past its expected
* expiration, make sure we don't bypass the next clock reprogramming
* to the same deadline.
*/
ts->next_tick = 0;
}
update_process_times(user_mode(regs));
profile_tick(CPU_PROFILING);
}
/*
* We rearm the timer until we get disabled by the idle code.
* Called with interrupts disabled.
*/
static enum hrtimer_restart tick_nohz_handler(struct hrtimer *timer)
{
struct tick_sched *ts = container_of(timer, struct tick_sched, sched_timer);
struct pt_regs *regs = get_irq_regs();
ktime_t now = ktime_get();
tick_sched_do_timer(ts, now);
/*
* Do not call when we are not in IRQ context and have
* no valid 'regs' pointer
*/
if (regs) tick_sched_handle(ts, regs);
else
ts->next_tick = 0;
/*
* In dynticks mode, tick reprogram is deferred:
* - to the idle task if in dynticks-idle
* - to IRQ exit if in full-dynticks.
*/
if (unlikely(tick_sched_flag_test(ts, TS_FLAG_STOPPED)))
return HRTIMER_NORESTART;
hrtimer_forward(timer, now, TICK_NSEC); return HRTIMER_RESTART;
}
static void tick_sched_timer_cancel(struct tick_sched *ts)
{
if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
hrtimer_cancel(&ts->sched_timer);
else if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
tick_program_event(KTIME_MAX, 1);
}
#ifdef CONFIG_NO_HZ_FULL
cpumask_var_t tick_nohz_full_mask;
EXPORT_SYMBOL_GPL(tick_nohz_full_mask);
bool tick_nohz_full_running;
EXPORT_SYMBOL_GPL(tick_nohz_full_running);
static atomic_t tick_dep_mask;
static bool check_tick_dependency(atomic_t *dep)
{
int val = atomic_read(dep);
if (val & TICK_DEP_MASK_POSIX_TIMER) {
trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
return true;
}
if (val & TICK_DEP_MASK_PERF_EVENTS) {
trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
return true;
}
if (val & TICK_DEP_MASK_SCHED) {
trace_tick_stop(0, TICK_DEP_MASK_SCHED);
return true;
}
if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
return true;
}
if (val & TICK_DEP_MASK_RCU) {
trace_tick_stop(0, TICK_DEP_MASK_RCU);
return true;
}
if (val & TICK_DEP_MASK_RCU_EXP) {
trace_tick_stop(0, TICK_DEP_MASK_RCU_EXP);
return true;
}
return false;
}
static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
{
lockdep_assert_irqs_disabled();
if (unlikely(!cpu_online(cpu)))
return false;
if (check_tick_dependency(&tick_dep_mask))
return false;
if (check_tick_dependency(&ts->tick_dep_mask))
return false;
if (check_tick_dependency(¤t->tick_dep_mask))
return false;
if (check_tick_dependency(¤t->signal->tick_dep_mask))
return false;
return true;
}
static void nohz_full_kick_func(struct irq_work *work)
{
/* Empty, the tick restart happens on tick_nohz_irq_exit() */
}
static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) =
IRQ_WORK_INIT_HARD(nohz_full_kick_func);
/*
* Kick this CPU if it's full dynticks in order to force it to
* re-evaluate its dependency on the tick and restart it if necessary.
* This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
* is NMI safe.
*/
static void tick_nohz_full_kick(void)
{
if (!tick_nohz_full_cpu(smp_processor_id()))
return;
irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
}
/*
* Kick the CPU if it's full dynticks in order to force it to
* re-evaluate its dependency on the tick and restart it if necessary.
*/
void tick_nohz_full_kick_cpu(int cpu)
{
if (!tick_nohz_full_cpu(cpu))
return;
irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
}
static void tick_nohz_kick_task(struct task_struct *tsk)
{
int cpu;
/*
* If the task is not running, run_posix_cpu_timers()
* has nothing to elapse, and an IPI can then be optimized out.
*
* activate_task() STORE p->tick_dep_mask
* STORE p->on_rq
* __schedule() (switch to task 'p') smp_mb() (atomic_fetch_or())
* LOCK rq->lock LOAD p->on_rq
* smp_mb__after_spin_lock()
* tick_nohz_task_switch()
* LOAD p->tick_dep_mask
*/
if (!sched_task_on_rq(tsk))
return;
/*
* If the task concurrently migrates to another CPU,
* we guarantee it sees the new tick dependency upon
* schedule.
*
* set_task_cpu(p, cpu);
* STORE p->cpu = @cpu
* __schedule() (switch to task 'p')
* LOCK rq->lock
* smp_mb__after_spin_lock() STORE p->tick_dep_mask
* tick_nohz_task_switch() smp_mb() (atomic_fetch_or())
* LOAD p->tick_dep_mask LOAD p->cpu
*/
cpu = task_cpu(tsk);
preempt_disable();
if (cpu_online(cpu))
tick_nohz_full_kick_cpu(cpu);
preempt_enable();
}
/*
* Kick all full dynticks CPUs in order to force these to re-evaluate
* their dependency on the tick and restart it if necessary.
*/
static void tick_nohz_full_kick_all(void)
{
int cpu;
if (!tick_nohz_full_running)
return;
preempt_disable();
for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
tick_nohz_full_kick_cpu(cpu);
preempt_enable();
}
static void tick_nohz_dep_set_all(atomic_t *dep,
enum tick_dep_bits bit)
{
int prev;
prev = atomic_fetch_or(BIT(bit), dep);
if (!prev)
tick_nohz_full_kick_all();
}
/*
* Set a global tick dependency. Used by perf events that rely on freq and
* unstable clocks.
*/
void tick_nohz_dep_set(enum tick_dep_bits bit)
{
tick_nohz_dep_set_all(&tick_dep_mask, bit);
}
void tick_nohz_dep_clear(enum tick_dep_bits bit)
{
atomic_andnot(BIT(bit), &tick_dep_mask);
}
/*
* Set per-CPU tick dependency. Used by scheduler and perf events in order to
* manage event-throttling.
*/
void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
{
int prev;
struct tick_sched *ts;
ts = per_cpu_ptr(&tick_cpu_sched, cpu);
prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
if (!prev) {
preempt_disable();
/* Perf needs local kick that is NMI safe */
if (cpu == smp_processor_id()) {
tick_nohz_full_kick();
} else {
/* Remote IRQ work not NMI-safe */
if (!WARN_ON_ONCE(in_nmi()))
tick_nohz_full_kick_cpu(cpu);
}
preempt_enable();
}
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
{
struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
atomic_andnot(BIT(bit), &ts->tick_dep_mask);
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
/*
* Set a per-task tick dependency. RCU needs this. Also posix CPU timers
* in order to elapse per task timers.
*/
void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
{
if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask))
tick_nohz_kick_task(tsk);
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
{
atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
/*
* Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
* per process timers.
*/
void tick_nohz_dep_set_signal(struct task_struct *tsk,
enum tick_dep_bits bit)
{
int prev;
struct signal_struct *sig = tsk->signal;
prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask);
if (!prev) {
struct task_struct *t;
lockdep_assert_held(&tsk->sighand->siglock);
__for_each_thread(sig, t)
tick_nohz_kick_task(t);
}
}
void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
{
atomic_andnot(BIT(bit), &sig->tick_dep_mask);
}
/*
* Re-evaluate the need for the tick as we switch the current task.
* It might need the tick due to per task/process properties:
* perf events, posix CPU timers, ...
*/
void __tick_nohz_task_switch(void)
{
struct tick_sched *ts;
if (!tick_nohz_full_cpu(smp_processor_id()))
return;
ts = this_cpu_ptr(&tick_cpu_sched);
if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
if (atomic_read(¤t->tick_dep_mask) ||
atomic_read(¤t->signal->tick_dep_mask))
tick_nohz_full_kick();
}
}
/* Get the boot-time nohz CPU list from the kernel parameters. */
void __init tick_nohz_full_setup(cpumask_var_t cpumask)
{
alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
cpumask_copy(tick_nohz_full_mask, cpumask);
tick_nohz_full_running = true;
}
bool tick_nohz_cpu_hotpluggable(unsigned int cpu)
{
/*
* The 'tick_do_timer_cpu' CPU handles housekeeping duty (unbound
* timers, workqueues, timekeeping, ...) on behalf of full dynticks
* CPUs. It must remain online when nohz full is enabled.
*/
if (tick_nohz_full_running && READ_ONCE(tick_do_timer_cpu) == cpu)
return false;
return true;
}
static int tick_nohz_cpu_down(unsigned int cpu)
{
return tick_nohz_cpu_hotpluggable(cpu) ? 0 : -EBUSY;
}
void __init tick_nohz_init(void)
{
int cpu, ret;
if (!tick_nohz_full_running)
return;
/*
* Full dynticks uses IRQ work to drive the tick rescheduling on safe
* locking contexts. But then we need IRQ work to raise its own
* interrupts to avoid circular dependency on the tick.
*/
if (!arch_irq_work_has_interrupt()) {
pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support IRQ work self-IPIs\n");
cpumask_clear(tick_nohz_full_mask);
tick_nohz_full_running = false;
return;
}
if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
!IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
cpu = smp_processor_id();
if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
pr_warn("NO_HZ: Clearing %d from nohz_full range "
"for timekeeping\n", cpu);
cpumask_clear_cpu(cpu, tick_nohz_full_mask);
}
}
for_each_cpu(cpu, tick_nohz_full_mask)
ct_cpu_track_user(cpu);
ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
"kernel/nohz:predown", NULL,
tick_nohz_cpu_down);
WARN_ON(ret < 0);
pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
cpumask_pr_args(tick_nohz_full_mask));
}
#endif /* #ifdef CONFIG_NO_HZ_FULL */
/*
* NOHZ - aka dynamic tick functionality
*/
#ifdef CONFIG_NO_HZ_COMMON
/*
* NO HZ enabled ?
*/
bool tick_nohz_enabled __read_mostly = true;
unsigned long tick_nohz_active __read_mostly;
/*
* Enable / Disable tickless mode
*/
static int __init setup_tick_nohz(char *str)
{
return (kstrtobool(str, &tick_nohz_enabled) == 0);
}
__setup("nohz=", setup_tick_nohz);
bool tick_nohz_tick_stopped(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
return tick_sched_flag_test(ts, TS_FLAG_STOPPED);
}
bool tick_nohz_tick_stopped_cpu(int cpu)
{
struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
return tick_sched_flag_test(ts, TS_FLAG_STOPPED);
}
/**
* tick_nohz_update_jiffies - update jiffies when idle was interrupted
* @now: current ktime_t
*
* Called from interrupt entry when the CPU was idle
*
* In case the sched_tick was stopped on this CPU, we have to check if jiffies
* must be updated. Otherwise an interrupt handler could use a stale jiffy
* value. We do this unconditionally on any CPU, as we don't know whether the
* CPU, which has the update task assigned, is in a long sleep.
*/
static void tick_nohz_update_jiffies(ktime_t now)
{
unsigned long flags;
__this_cpu_write(tick_cpu_sched.idle_waketime, now);
local_irq_save(flags);
tick_do_update_jiffies64(now);
local_irq_restore(flags);
touch_softlockup_watchdog_sched();
}
static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
{
ktime_t delta;
if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)))
return;
delta = ktime_sub(now, ts->idle_entrytime);
write_seqcount_begin(&ts->idle_sleeptime_seq);
if (nr_iowait_cpu(smp_processor_id()) > 0)
ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
else
ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
ts->idle_entrytime = now;
tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE);
write_seqcount_end(&ts->idle_sleeptime_seq);
sched_clock_idle_wakeup_event();
}
static void tick_nohz_start_idle(struct tick_sched *ts)
{
write_seqcount_begin(&ts->idle_sleeptime_seq);
ts->idle_entrytime = ktime_get();
tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE);
write_seqcount_end(&ts->idle_sleeptime_seq);
sched_clock_idle_sleep_event();
}
static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,
bool compute_delta, u64 *last_update_time)
{
ktime_t now, idle;
unsigned int seq;
if (!tick_nohz_active)
return -1;
now = ktime_get();
if (last_update_time)
*last_update_time = ktime_to_us(now);
do {
seq = read_seqcount_begin(&ts->idle_sleeptime_seq);
if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) {
ktime_t delta = ktime_sub(now, ts->idle_entrytime);
idle = ktime_add(*sleeptime, delta);
} else {
idle = *sleeptime;
}
} while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));
return ktime_to_us(idle);
}
/**
* get_cpu_idle_time_us - get the total idle time of a CPU
* @cpu: CPU number to query
* @last_update_time: variable to store update time in. Do not update
* counters if NULL.
*
* Return the cumulative idle time (since boot) for a given
* CPU, in microseconds. Note that this is partially broken due to
* the counter of iowait tasks that can be remotely updated without
* any synchronization. Therefore it is possible to observe backward
* values within two consecutive reads.
*
* This time is measured via accounting rather than sampling,
* and is as accurate as ktime_get() is.
*
* Return: -1 if NOHZ is not enabled, else total idle time of the @cpu
*/
u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
{
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,
!nr_iowait_cpu(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
/**
* get_cpu_iowait_time_us - get the total iowait time of a CPU
* @cpu: CPU number to query
* @last_update_time: variable to store update time in. Do not update
* counters if NULL.
*
* Return the cumulative iowait time (since boot) for a given
* CPU, in microseconds. Note this is partially broken due to
* the counter of iowait tasks that can be remotely updated without
* any synchronization. Therefore it is possible to observe backward
* values within two consecutive reads.
*
* This time is measured via accounting rather than sampling,
* and is as accurate as ktime_get() is.
*
* Return: -1 if NOHZ is not enabled, else total iowait time of @cpu
*/
u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
{
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,
nr_iowait_cpu(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
{
hrtimer_cancel(&ts->sched_timer);
hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
/* Forward the time to expire in the future */
hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);
if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
hrtimer_start_expires(&ts->sched_timer,
HRTIMER_MODE_ABS_PINNED_HARD);
} else {
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
}
/*
* Reset to make sure the next tick stop doesn't get fooled by past
* cached clock deadline.
*/
ts->next_tick = 0;
}
static inline bool local_timer_softirq_pending(void)
{
return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
}
/*
* Read jiffies and the time when jiffies were updated last
*/
u64 get_jiffies_update(unsigned long *basej)
{
unsigned long basejiff;
unsigned int seq;
u64 basemono;
do {
seq = read_seqcount_begin(&jiffies_seq);
basemono = last_jiffies_update;
basejiff = jiffies;
} while (read_seqcount_retry(&jiffies_seq, seq));
*basej = basejiff;
return basemono;
}
/**
* tick_nohz_next_event() - return the clock monotonic based next event
* @ts: pointer to tick_sched struct
* @cpu: CPU number
*
* Return:
* *%0 - When the next event is a maximum of TICK_NSEC in the future
* and the tick is not stopped yet
* *%next_event - Next event based on clock monotonic
*/
static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
{
u64 basemono, next_tick, delta, expires;
unsigned long basejiff;
int tick_cpu;
basemono = get_jiffies_update(&basejiff);
ts->last_jiffies = basejiff;
ts->timer_expires_base = basemono;
/*
* Keep the periodic tick, when RCU, architecture or irq_work
* requests it.
* Aside of that, check whether the local timer softirq is
* pending. If so, its a bad idea to call get_next_timer_interrupt(),
* because there is an already expired timer, so it will request
* immediate expiry, which rearms the hardware timer with a
* minimal delta, which brings us back to this place
* immediately. Lather, rinse and repeat...
*/
if (rcu_needs_cpu() || arch_needs_cpu() ||
irq_work_needs_cpu() || local_timer_softirq_pending()) {
next_tick = basemono + TICK_NSEC;
} else {
/*
* Get the next pending timer. If high resolution
* timers are enabled this only takes the timer wheel
* timers into account. If high resolution timers are
* disabled this also looks at the next expiring
* hrtimer.
*/
next_tick = get_next_timer_interrupt(basejiff, basemono);
ts->next_timer = next_tick;
}
/* Make sure next_tick is never before basemono! */
if (WARN_ON_ONCE(basemono > next_tick))
next_tick = basemono;
/*
* If the tick is due in the next period, keep it ticking or
* force prod the timer.
*/
delta = next_tick - basemono;
if (delta <= (u64)TICK_NSEC) {
/*
* We've not stopped the tick yet, and there's a timer in the
* next period, so no point in stopping it either, bail.
*/
if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
ts->timer_expires = 0;
goto out;
}
}
/*
* If this CPU is the one which had the do_timer() duty last, we limit
* the sleep time to the timekeeping 'max_deferment' value.
* Otherwise we can sleep as long as we want.
*/
delta = timekeeping_max_deferment();
tick_cpu = READ_ONCE(tick_do_timer_cpu);
if (tick_cpu != cpu &&
(tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST)))
delta = KTIME_MAX;
/* Calculate the next expiry time */
if (delta < (KTIME_MAX - basemono))
expires = basemono + delta;
else
expires = KTIME_MAX;
ts->timer_expires = min_t(u64, expires, next_tick);
out:
return ts->timer_expires;
}
static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
{
struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
unsigned long basejiff = ts->last_jiffies;
u64 basemono = ts->timer_expires_base;
bool timer_idle = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
int tick_cpu;
u64 expires;
/* Make sure we won't be trying to stop it twice in a row. */
ts->timer_expires_base = 0;
/*
* Now the tick should be stopped definitely - so the timer base needs
* to be marked idle as well to not miss a newly queued timer.
*/
expires = timer_base_try_to_set_idle(basejiff, basemono, &timer_idle);
if (expires > ts->timer_expires) {
/*
* This path could only happen when the first timer was removed
* between calculating the possible sleep length and now (when
* high resolution mode is not active, timer could also be a
* hrtimer).
*
* We have to stick to the original calculated expiry value to
* not stop the tick for too long with a shallow C-state (which
* was programmed by cpuidle because of an early next expiration
* value).
*/
expires = ts->timer_expires;
}
/* If the timer base is not idle, retain the not yet stopped tick. */
if (!timer_idle)
return;
/*
* If this CPU is the one which updates jiffies, then give up
* the assignment and let it be taken by the CPU which runs
* the tick timer next, which might be this CPU as well. If we
* don't drop this here, the jiffies might be stale and
* do_timer() never gets invoked. Keep track of the fact that it
* was the one which had the do_timer() duty last.
*/
tick_cpu = READ_ONCE(tick_do_timer_cpu);
if (tick_cpu == cpu) {
WRITE_ONCE(tick_do_timer_cpu, TICK_DO_TIMER_NONE);
tick_sched_flag_set(ts, TS_FLAG_DO_TIMER_LAST);
} else if (tick_cpu != TICK_DO_TIMER_NONE) {
tick_sched_flag_clear(ts, TS_FLAG_DO_TIMER_LAST);
}
/* Skip reprogram of event if it's not changed */
if (tick_sched_flag_test(ts, TS_FLAG_STOPPED) && (expires == ts->next_tick)) {
/* Sanity check: make sure clockevent is actually programmed */
if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
return;
WARN_ON_ONCE(1);
printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
basemono, ts->next_tick, dev->next_event,
hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
}
/*
* tick_nohz_stop_tick() can be called several times before
* tick_nohz_restart_sched_tick() is called. This happens when
* interrupts arrive which do not cause a reschedule. In the first
* call we save the current tick time, so we can restart the
* scheduler tick in tick_nohz_restart_sched_tick().
*/
if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
calc_load_nohz_start();
quiet_vmstat();
ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
tick_sched_flag_set(ts, TS_FLAG_STOPPED);
trace_tick_stop(1, TICK_DEP_MASK_NONE);
}
ts->next_tick = expires;
/*
* If the expiration time == KTIME_MAX, then we simply stop
* the tick timer.
*/
if (unlikely(expires == KTIME_MAX)) {
tick_sched_timer_cancel(ts);
return;
}
if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
hrtimer_start(&ts->sched_timer, expires,
HRTIMER_MODE_ABS_PINNED_HARD);
} else {
hrtimer_set_expires(&ts->sched_timer, expires);
tick_program_event(expires, 1);
}
}
static void tick_nohz_retain_tick(struct tick_sched *ts)
{
ts->timer_expires_base = 0;
}
#ifdef CONFIG_NO_HZ_FULL
static void tick_nohz_full_stop_tick(struct tick_sched *ts, int cpu)
{
if (tick_nohz_next_event(ts, cpu))
tick_nohz_stop_tick(ts, cpu);
else
tick_nohz_retain_tick(ts);
}
#endif /* CONFIG_NO_HZ_FULL */
static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
{
/* Update jiffies first */
tick_do_update_jiffies64(now);
/*
* Clear the timer idle flag, so we avoid IPIs on remote queueing and
* the clock forward checks in the enqueue path:
*/
timer_clear_idle();
calc_load_nohz_stop();
touch_softlockup_watchdog_sched();
/* Cancel the scheduled timer and restore the tick: */
tick_sched_flag_clear(ts, TS_FLAG_STOPPED);
tick_nohz_restart(ts, now);
}
static void __tick_nohz_full_update_tick(struct tick_sched *ts,
ktime_t now)
{
#ifdef CONFIG_NO_HZ_FULL
int cpu = smp_processor_id();
if (can_stop_full_tick(cpu, ts))
tick_nohz_full_stop_tick(ts, cpu);
else if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
tick_nohz_restart_sched_tick(ts, now);
#endif
}
static void tick_nohz_full_update_tick(struct tick_sched *ts)
{
if (!tick_nohz_full_cpu(smp_processor_id()))
return;
if (!tick_sched_flag_test(ts, TS_FLAG_NOHZ))
return;
__tick_nohz_full_update_tick(ts, ktime_get());
}
/*
* A pending softirq outside an IRQ (or softirq disabled section) context
* should be waiting for ksoftirqd to handle it. Therefore we shouldn't
* reach this code due to the need_resched() early check in can_stop_idle_tick().
*
* However if we are between CPUHP_AP_SMPBOOT_THREADS and CPU_TEARDOWN_CPU on the
* cpu_down() process, softirqs can still be raised while ksoftirqd is parked,
* triggering the code below, since wakep_softirqd() is ignored.
*
*/
static bool report_idle_softirq(void)
{
static int ratelimit;
unsigned int pending = local_softirq_pending();
if (likely(!pending))
return false;
/* Some softirqs claim to be safe against hotplug and ksoftirqd parking */
if (!cpu_active(smp_processor_id())) {
pending &= ~SOFTIRQ_HOTPLUG_SAFE_MASK;
if (!pending)
return false;
}
if (ratelimit >= 10)
return false;
/* On RT, softirq handling may be waiting on some lock */
if (local_bh_blocked())
return false;
pr_warn("NOHZ tick-stop error: local softirq work is pending, handler #%02x!!!\n",
pending);
ratelimit++;
return true;
}
static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
{
WARN_ON_ONCE(cpu_is_offline(cpu));
if (unlikely(!tick_sched_flag_test(ts, TS_FLAG_NOHZ)))
return false;
if (need_resched())
return false;
if (unlikely(report_idle_softirq()))
return false;
if (tick_nohz_full_enabled()) {
int tick_cpu = READ_ONCE(tick_do_timer_cpu);
/*
* Keep the tick alive to guarantee timekeeping progression
* if there are full dynticks CPUs around
*/
if (tick_cpu == cpu)
return false;
/* Should not happen for nohz-full */
if (WARN_ON_ONCE(tick_cpu == TICK_DO_TIMER_NONE))
return false;
}
return true;
}
/**
* tick_nohz_idle_stop_tick - stop the idle tick from the idle task
*
* When the next event is more than a tick into the future, stop the idle tick
*/
void tick_nohz_idle_stop_tick(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
int cpu = smp_processor_id();
ktime_t expires;
/*
* If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
* tick timer expiration time is known already.
*/
if (ts->timer_expires_base)
expires = ts->timer_expires;
else if (can_stop_idle_tick(cpu, ts))
expires = tick_nohz_next_event(ts, cpu);
else
return;
ts->idle_calls++;
if (expires > 0LL) {
int was_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
tick_nohz_stop_tick(ts, cpu);
ts->idle_sleeps++;
ts->idle_expires = expires;
if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
ts->idle_jiffies = ts->last_jiffies;
nohz_balance_enter_idle(cpu);
}
} else {
tick_nohz_retain_tick(ts);
}
}
void tick_nohz_idle_retain_tick(void)
{
tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
}
/**
* tick_nohz_idle_enter - prepare for entering idle on the current CPU
*
* Called when we start the idle loop.
*/
void tick_nohz_idle_enter(void)
{
struct tick_sched *ts;
lockdep_assert_irqs_enabled();
local_irq_disable();
ts = this_cpu_ptr(&tick_cpu_sched);
WARN_ON_ONCE(ts->timer_expires_base);
tick_sched_flag_set(ts, TS_FLAG_INIDLE);
tick_nohz_start_idle(ts);
local_irq_enable();
}
/**
* tick_nohz_irq_exit - Notify the tick about IRQ exit
*
* A timer may have been added/modified/deleted either by the current IRQ,
* or by another place using this IRQ as a notification. This IRQ may have
* also updated the RCU callback list. These events may require a
* re-evaluation of the next tick. Depending on the context:
*
* 1) If the CPU is idle and no resched is pending, just proceed with idle
* time accounting. The next tick will be re-evaluated on the next idle
* loop iteration.
*
* 2) If the CPU is nohz_full:
*
* 2.1) If there is any tick dependency, restart the tick if stopped.
*
* 2.2) If there is no tick dependency, (re-)evaluate the next tick and
* stop/update it accordingly.
*/
void tick_nohz_irq_exit(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
tick_nohz_start_idle(ts);
else
tick_nohz_full_update_tick(ts);
}
/**
* tick_nohz_idle_got_tick - Check whether or not the tick handler has run
*
* Return: %true if the tick handler has run, otherwise %false
*/
bool tick_nohz_idle_got_tick(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
if (ts->got_idle_tick) {
ts->got_idle_tick = 0;
return true;
}
return false;
}
/**
* tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
* or the tick, whichever expires first. Note that, if the tick has been
* stopped, it returns the next hrtimer.
*
* Called from power state control code with interrupts disabled
*
* Return: the next expiration time
*/
ktime_t tick_nohz_get_next_hrtimer(void)
{
return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
}
/**
* tick_nohz_get_sleep_length - return the expected length of the current sleep
* @delta_next: duration until the next event if the tick cannot be stopped
*
* Called from power state control code with interrupts disabled.
*
* The return value of this function and/or the value returned by it through the
* @delta_next pointer can be negative which must be taken into account by its
* callers.
*
* Return: the expected length of the current sleep
*/
ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
{
struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
int cpu = smp_processor_id();
/*
* The idle entry time is expected to be a sufficient approximation of
* the current time at this point.
*/
ktime_t now = ts->idle_entrytime;
ktime_t next_event;
WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
*delta_next = ktime_sub(dev->next_event, now);
if (!can_stop_idle_tick(cpu, ts))
return *delta_next;
next_event = tick_nohz_next_event(ts, cpu);
if (!next_event)
return *delta_next;
/*
* If the next highres timer to expire is earlier than 'next_event', the
* idle governor needs to know that.
*/
next_event = min_t(u64, next_event,
hrtimer_next_event_without(&ts->sched_timer));
return ktime_sub(next_event, now);
}
/**
* tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
* for a particular CPU.
* @cpu: target CPU number
*
* Called from the schedutil frequency scaling governor in scheduler context.
*
* Return: the current idle calls counter value for @cpu
*/
unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
{
struct tick_sched *ts = tick_get_tick_sched(cpu);
return ts->idle_calls;
}
/**
* tick_nohz_get_idle_calls - return the current idle calls counter value
*
* Called from the schedutil frequency scaling governor in scheduler context.
*
* Return: the current idle calls counter value for the current CPU
*/
unsigned long tick_nohz_get_idle_calls(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
return ts->idle_calls;
}
static void tick_nohz_account_idle_time(struct tick_sched *ts,
ktime_t now)
{
unsigned long ticks;
ts->idle_exittime = now;
if (vtime_accounting_enabled_this_cpu())
return;
/*
* We stopped the tick in idle. update_process_times() would miss the
* time we slept, as it does only a 1 tick accounting.
* Enforce that this is accounted to idle !
*/
ticks = jiffies - ts->idle_jiffies;
/*
* We might be one off. Do not randomly account a huge number of ticks!
*/
if (ticks && ticks < LONG_MAX)
account_idle_ticks(ticks);
}
void tick_nohz_idle_restart_tick(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
ktime_t now = ktime_get();
tick_nohz_restart_sched_tick(ts, now);
tick_nohz_account_idle_time(ts, now);
}
}
static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
{
if (tick_nohz_full_cpu(smp_processor_id()))
__tick_nohz_full_update_tick(ts, now);
else
tick_nohz_restart_sched_tick(ts, now);
tick_nohz_account_idle_time(ts, now);
}
/**
* tick_nohz_idle_exit - Update the tick upon idle task exit
*
* When the idle task exits, update the tick depending on the
* following situations:
*
* 1) If the CPU is not in nohz_full mode (most cases), then
* restart the tick.
*
* 2) If the CPU is in nohz_full mode (corner case):
* 2.1) If the tick can be kept stopped (no tick dependencies)
* then re-evaluate the next tick and try to keep it stopped
* as long as possible.
* 2.2) If the tick has dependencies, restart the tick.
*
*/
void tick_nohz_idle_exit(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
bool idle_active, tick_stopped;
ktime_t now;
local_irq_disable();
WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
WARN_ON_ONCE(ts->timer_expires_base);
tick_sched_flag_clear(ts, TS_FLAG_INIDLE);
idle_active = tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE);
tick_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
if (idle_active || tick_stopped)
now = ktime_get();
if (idle_active)
tick_nohz_stop_idle(ts, now);
if (tick_stopped)
tick_nohz_idle_update_tick(ts, now);
local_irq_enable();
}
/*
* In low-resolution mode, the tick handler must be implemented directly
* at the clockevent level. hrtimer can't be used instead, because its
* infrastructure actually relies on the tick itself as a backend in
* low-resolution mode (see hrtimer_run_queues()).
*/
static void tick_nohz_lowres_handler(struct clock_event_device *dev)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
dev->next_event = KTIME_MAX;
if (likely(tick_nohz_handler(&ts->sched_timer) == HRTIMER_RESTART))
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
}
static inline void tick_nohz_activate(struct tick_sched *ts)
{
if (!tick_nohz_enabled)
return;
tick_sched_flag_set(ts, TS_FLAG_NOHZ);
/* One update is enough */
if (!test_and_set_bit(0, &tick_nohz_active))
timers_update_nohz();
}
/**
* tick_nohz_switch_to_nohz - switch to NOHZ mode
*/
static void tick_nohz_switch_to_nohz(void)
{
if (!tick_nohz_enabled)
return;
if (tick_switch_to_oneshot(tick_nohz_lowres_handler))
return;
/*
* Recycle the hrtimer in 'ts', so we can share the
* highres code.
*/
tick_setup_sched_timer(false);
}
static inline void tick_nohz_irq_enter(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
ktime_t now;
if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE))
return;
now = ktime_get();
if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))
tick_nohz_stop_idle(ts, now);
/*
* If all CPUs are idle we may need to update a stale jiffies value.
* Note nohz_full is a special case: a timekeeper is guaranteed to stay
* alive but it might be busy looping with interrupts disabled in some
* rare case (typically stop machine). So we must make sure we have a
* last resort.
*/
if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
tick_nohz_update_jiffies(now);
}
#else
static inline void tick_nohz_switch_to_nohz(void) { }
static inline void tick_nohz_irq_enter(void) { }
static inline void tick_nohz_activate(struct tick_sched *ts) { }
#endif /* CONFIG_NO_HZ_COMMON */
/*
* Called from irq_enter() to notify about the possible interruption of idle()
*/
void tick_irq_enter(void)
{
tick_check_oneshot_broadcast_this_cpu();
tick_nohz_irq_enter();
}
static int sched_skew_tick;
static int __init skew_tick(char *str)
{
get_option(&str, &sched_skew_tick);
return 0;
}
early_param("skew_tick", skew_tick);
/**
* tick_setup_sched_timer - setup the tick emulation timer
* @hrtimer: whether to use the hrtimer or not
*/
void tick_setup_sched_timer(bool hrtimer)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
/* Emulate tick processing via per-CPU hrtimers: */
hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && hrtimer) {
tick_sched_flag_set(ts, TS_FLAG_HIGHRES);
ts->sched_timer.function = tick_nohz_handler;
}
/* Get the next period (per-CPU) */
hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
/* Offset the tick to avert 'jiffies_lock' contention. */
if (sched_skew_tick) {
u64 offset = TICK_NSEC >> 1;
do_div(offset, num_possible_cpus());
offset *= smp_processor_id();
hrtimer_add_expires_ns(&ts->sched_timer, offset);
}
hrtimer_forward_now(&ts->sched_timer, TICK_NSEC);
if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && hrtimer)
hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
else
tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
tick_nohz_activate(ts);
}
/*
* Shut down the tick and make sure the CPU won't try to retake the timekeeping
* duty before disabling IRQs in idle for the last time.
*/
void tick_sched_timer_dying(int cpu)
{
struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
struct clock_event_device *dev = td->evtdev;
ktime_t idle_sleeptime, iowait_sleeptime;
unsigned long idle_calls, idle_sleeps;
/* This must happen before hrtimers are migrated! */
tick_sched_timer_cancel(ts);
/*
* If the clockevents doesn't support CLOCK_EVT_STATE_ONESHOT_STOPPED,
* make sure not to call low-res tick handler.
*/
if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
dev->event_handler = clockevents_handle_noop;
idle_sleeptime = ts->idle_sleeptime;
iowait_sleeptime = ts->iowait_sleeptime;
idle_calls = ts->idle_calls;
idle_sleeps = ts->idle_sleeps;
memset(ts, 0, sizeof(*ts));
ts->idle_sleeptime = idle_sleeptime;
ts->iowait_sleeptime = iowait_sleeptime;
ts->idle_calls = idle_calls;
ts->idle_sleeps = idle_sleeps;
}
/*
* Async notification about clocksource changes
*/
void tick_clock_notify(void)
{
int cpu;
for_each_possible_cpu(cpu)
set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
}
/*
* Async notification about clock event changes
*/
void tick_oneshot_notify(void)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
set_bit(0, &ts->check_clocks);
}
/*
* Check if a change happened, which makes oneshot possible.
*
* Called cyclically from the hrtimer softirq (driven by the timer
* softirq). 'allow_nohz' signals that we can switch into low-res NOHZ
* mode, because high resolution timers are disabled (either compile
* or runtime). Called with interrupts disabled.
*/
int tick_check_oneshot_change(int allow_nohz)
{
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
if (!test_and_clear_bit(0, &ts->check_clocks))
return 0;
if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
return 0;
if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
return 0;
if (!allow_nohz)
return 1;
tick_nohz_switch_to_nohz();
return 0;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_NOSPEC_BRANCH_H_
#define _ASM_X86_NOSPEC_BRANCH_H_
#include <linux/static_key.h>
#include <linux/objtool.h>
#include <linux/linkage.h>
#include <asm/alternative.h>
#include <asm/cpufeatures.h>
#include <asm/msr-index.h>
#include <asm/unwind_hints.h>
#include <asm/percpu.h>
#include <asm/current.h>
/*
* Call depth tracking for Intel SKL CPUs to address the RSB underflow
* issue in software.
*
* The tracking does not use a counter. It uses uses arithmetic shift
* right on call entry and logical shift left on return.
*
* The depth tracking variable is initialized to 0x8000.... when the call
* depth is zero. The arithmetic shift right sign extends the MSB and
* saturates after the 12th call. The shift count is 5 for both directions
* so the tracking covers 12 nested calls.
*
* Call
* 0: 0x8000000000000000 0x0000000000000000
* 1: 0xfc00000000000000 0xf000000000000000
* ...
* 11: 0xfffffffffffffff8 0xfffffffffffffc00
* 12: 0xffffffffffffffff 0xffffffffffffffe0
*
* After a return buffer fill the depth is credited 12 calls before the
* next stuffing has to take place.
*
* There is a inaccuracy for situations like this:
*
* 10 calls
* 5 returns
* 3 calls
* 4 returns
* 3 calls
* ....
*
* The shift count might cause this to be off by one in either direction,
* but there is still a cushion vs. the RSB depth. The algorithm does not
* claim to be perfect and it can be speculated around by the CPU, but it
* is considered that it obfuscates the problem enough to make exploitation
* extremely difficult.
*/
#define RET_DEPTH_SHIFT 5
#define RSB_RET_STUFF_LOOPS 16
#define RET_DEPTH_INIT 0x8000000000000000ULL
#define RET_DEPTH_INIT_FROM_CALL 0xfc00000000000000ULL
#define RET_DEPTH_CREDIT 0xffffffffffffffffULL
#ifdef CONFIG_CALL_THUNKS_DEBUG
# define CALL_THUNKS_DEBUG_INC_CALLS \
incq PER_CPU_VAR(__x86_call_count);
# define CALL_THUNKS_DEBUG_INC_RETS \
incq PER_CPU_VAR(__x86_ret_count);
# define CALL_THUNKS_DEBUG_INC_STUFFS \
incq PER_CPU_VAR(__x86_stuffs_count);
# define CALL_THUNKS_DEBUG_INC_CTXSW \
incq PER_CPU_VAR(__x86_ctxsw_count);
#else
# define CALL_THUNKS_DEBUG_INC_CALLS
# define CALL_THUNKS_DEBUG_INC_RETS
# define CALL_THUNKS_DEBUG_INC_STUFFS
# define CALL_THUNKS_DEBUG_INC_CTXSW
#endif
#if defined(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) && !defined(COMPILE_OFFSETS)
#include <asm/asm-offsets.h>
#define CREDIT_CALL_DEPTH \
movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
#define RESET_CALL_DEPTH \
xor %eax, %eax; \
bts $63, %rax; \
movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);
#define RESET_CALL_DEPTH_FROM_CALL \
movb $0xfc, %al; \
shl $56, %rax; \
movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
CALL_THUNKS_DEBUG_INC_CALLS
#define INCREMENT_CALL_DEPTH \
sarq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
CALL_THUNKS_DEBUG_INC_CALLS
#else
#define CREDIT_CALL_DEPTH
#define RESET_CALL_DEPTH
#define RESET_CALL_DEPTH_FROM_CALL
#define INCREMENT_CALL_DEPTH
#endif
/*
* Fill the CPU return stack buffer.
*
* Each entry in the RSB, if used for a speculative 'ret', contains an
* infinite 'pause; lfence; jmp' loop to capture speculative execution.
*
* This is required in various cases for retpoline and IBRS-based
* mitigations for the Spectre variant 2 vulnerability. Sometimes to
* eliminate potentially bogus entries from the RSB, and sometimes
* purely to ensure that it doesn't get empty, which on some CPUs would
* allow predictions from other (unwanted!) sources to be used.
*
* We define a CPP macro such that it can be used from both .S files and
* inline assembly. It's possible to do a .macro and then include that
* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
*/
#define RETPOLINE_THUNK_SIZE 32
#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
/*
* Common helper for __FILL_RETURN_BUFFER and __FILL_ONE_RETURN.
*/
#define __FILL_RETURN_SLOT \
ANNOTATE_INTRA_FUNCTION_CALL; \
call 772f; \
int3; \
772:
/*
* Stuff the entire RSB.
*
* Google experimented with loop-unrolling and this turned out to be
* the optimal version - two calls, each with their own speculation
* trap should their return address end up getting used, in a loop.
*/
#ifdef CONFIG_X86_64
#define __FILL_RETURN_BUFFER(reg, nr) \
mov $(nr/2), reg; \
771: \
__FILL_RETURN_SLOT \
__FILL_RETURN_SLOT \
add $(BITS_PER_LONG/8) * 2, %_ASM_SP; \
dec reg; \
jnz 771b; \
/* barrier for jnz misprediction */ \
lfence; \
CREDIT_CALL_DEPTH \
CALL_THUNKS_DEBUG_INC_CTXSW
#else
/*
* i386 doesn't unconditionally have LFENCE, as such it can't
* do a loop.
*/
#define __FILL_RETURN_BUFFER(reg, nr) \
.rept nr; \
__FILL_RETURN_SLOT; \
.endr; \
add $(BITS_PER_LONG/8) * nr, %_ASM_SP;
#endif
/*
* Stuff a single RSB slot.
*
* To mitigate Post-Barrier RSB speculation, one CALL instruction must be
* forced to retire before letting a RET instruction execute.
*
* On PBRSB-vulnerable CPUs, it is not safe for a RET to be executed
* before this point.
*/
#define __FILL_ONE_RETURN \
__FILL_RETURN_SLOT \
add $(BITS_PER_LONG/8), %_ASM_SP; \
lfence;
#ifdef __ASSEMBLY__
/*
* This should be used immediately before an indirect jump/call. It tells
* objtool the subsequent indirect jump/call is vouched safe for retpoline
* builds.
*/
.macro ANNOTATE_RETPOLINE_SAFE
.Lhere_\@:
.pushsection .discard.retpoline_safe
.long .Lhere_\@
.popsection
.endm
/*
* (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions
* vs RETBleed validation.
*/
#define ANNOTATE_UNRET_SAFE ANNOTATE_RETPOLINE_SAFE
/*
* Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should
* eventually turn into its own annotation.
*/
.macro VALIDATE_UNRET_END
#if defined(CONFIG_NOINSTR_VALIDATION) && \
(defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO))
ANNOTATE_RETPOLINE_SAFE
nop
#endif
.endm
/*
* Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call
* to the retpoline thunk with a CS prefix when the register requires
* a RAX prefix byte to encode. Also see apply_retpolines().
*/
.macro __CS_PREFIX reg:req
.irp rs,r8,r9,r10,r11,r12,r13,r14,r15
.ifc \reg,\rs
.byte 0x2e
.endif
.endr
.endm
/*
* JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
* indirect jmp/call which may be susceptible to the Spectre variant 2
* attack.
*
* NOTE: these do not take kCFI into account and are thus not comparable to C
* indirect calls, take care when using. The target of these should be an ENDBR
* instruction irrespective of kCFI.
*/
.macro JMP_NOSPEC reg:req
#ifdef CONFIG_MITIGATION_RETPOLINE
__CS_PREFIX \reg
jmp __x86_indirect_thunk_\reg
#else
jmp *%\reg
int3
#endif
.endm
.macro CALL_NOSPEC reg:req
#ifdef CONFIG_MITIGATION_RETPOLINE
__CS_PREFIX \reg
call __x86_indirect_thunk_\reg
#else
call *%\reg
#endif
.endm
/*
* A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
* monstrosity above, manually.
*/
.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2=ALT_NOT(X86_FEATURE_ALWAYS)
ALTERNATIVE_2 "jmp .Lskip_rsb_\@", \
__stringify(__FILL_RETURN_BUFFER(\reg,\nr)), \ftr, \
__stringify(nop;nop;__FILL_ONE_RETURN), \ftr2
.Lskip_rsb_\@:
.endm
/*
* The CALL to srso_alias_untrain_ret() must be patched in directly at
* the spot where untraining must be done, ie., srso_alias_untrain_ret()
* must be the target of a CALL instruction instead of indirectly
* jumping to a wrapper which then calls it. Therefore, this macro is
* called outside of __UNTRAIN_RET below, for the time being, before the
* kernel can support nested alternatives with arbitrary nesting.
*/
.macro CALL_UNTRAIN_RET
#if defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO)
ALTERNATIVE_2 "", "call entry_untrain_ret", X86_FEATURE_UNRET, \
"call srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS
#endif
.endm
/*
* Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
* return thunk isn't mapped into the userspace tables (then again, AMD
* typically has NO_MELTDOWN).
*
* While retbleed_untrain_ret() doesn't clobber anything but requires stack,
* entry_ibpb() will clobber AX, CX, DX.
*
* As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
* where we have a stack but before any RET instruction.
*/
.macro __UNTRAIN_RET ibpb_feature, call_depth_insns
#if defined(CONFIG_MITIGATION_RETHUNK) || defined(CONFIG_MITIGATION_IBPB_ENTRY)
VALIDATE_UNRET_END
CALL_UNTRAIN_RET
ALTERNATIVE_2 "", \
"call entry_ibpb", \ibpb_feature, \
__stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH
#endif
.endm
#define UNTRAIN_RET \
__UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH)
#define UNTRAIN_RET_VM \
__UNTRAIN_RET X86_FEATURE_IBPB_ON_VMEXIT, __stringify(RESET_CALL_DEPTH)
#define UNTRAIN_RET_FROM_CALL \
__UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL)
.macro CALL_DEPTH_ACCOUNT
#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
ALTERNATIVE "", \
__stringify(INCREMENT_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
#endif
.endm
/*
* Macro to execute VERW instruction that mitigate transient data sampling
* attacks such as MDS. On affected systems a microcode update overloaded VERW
* instruction to also clear the CPU buffers. VERW clobbers CFLAGS.ZF.
*
* Note: Only the memory operand variant of VERW clears the CPU buffers.
*/
.macro CLEAR_CPU_BUFFERS
ALTERNATIVE "", __stringify(verw _ASM_RIP(mds_verw_sel)), X86_FEATURE_CLEAR_CPU_BUF
.endm
#ifdef CONFIG_X86_64
.macro CLEAR_BRANCH_HISTORY
ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP
.endm
.macro CLEAR_BRANCH_HISTORY_VMEXIT
ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT
.endm
#else
#define CLEAR_BRANCH_HISTORY
#define CLEAR_BRANCH_HISTORY_VMEXIT
#endif
#else /* __ASSEMBLY__ */
#define ANNOTATE_RETPOLINE_SAFE \
"999:\n\t" \
".pushsection .discard.retpoline_safe\n\t" \
".long 999b\n\t" \
".popsection\n\t"
typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
extern retpoline_thunk_t __x86_indirect_thunk_array[];
extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
#ifdef CONFIG_MITIGATION_RETHUNK
extern void __x86_return_thunk(void);
#else
static inline void __x86_return_thunk(void) {}
#endif
#ifdef CONFIG_MITIGATION_UNRET_ENTRY
extern void retbleed_return_thunk(void);
#else
static inline void retbleed_return_thunk(void) {}
#endif
extern void srso_alias_untrain_ret(void);
#ifdef CONFIG_MITIGATION_SRSO
extern void srso_return_thunk(void);
extern void srso_alias_return_thunk(void);
#else
static inline void srso_return_thunk(void) {}
static inline void srso_alias_return_thunk(void) {}
#endif
extern void retbleed_return_thunk(void);
extern void srso_return_thunk(void);
extern void srso_alias_return_thunk(void);
extern void entry_untrain_ret(void);
extern void entry_ibpb(void);
#ifdef CONFIG_X86_64
extern void clear_bhb_loop(void);
#endif
extern void (*x86_return_thunk)(void);
extern void __warn_thunk(void);
#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
extern void call_depth_return_thunk(void);
#define CALL_DEPTH_ACCOUNT \
ALTERNATIVE("", \
__stringify(INCREMENT_CALL_DEPTH), \
X86_FEATURE_CALL_DEPTH)
#ifdef CONFIG_CALL_THUNKS_DEBUG
DECLARE_PER_CPU(u64, __x86_call_count);
DECLARE_PER_CPU(u64, __x86_ret_count);
DECLARE_PER_CPU(u64, __x86_stuffs_count);
DECLARE_PER_CPU(u64, __x86_ctxsw_count);
#endif
#else /* !CONFIG_MITIGATION_CALL_DEPTH_TRACKING */
static inline void call_depth_return_thunk(void) {}
#define CALL_DEPTH_ACCOUNT ""
#endif /* CONFIG_MITIGATION_CALL_DEPTH_TRACKING */
#ifdef CONFIG_MITIGATION_RETPOLINE
#define GEN(reg) \
extern retpoline_thunk_t __x86_indirect_thunk_ ## reg;
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) \
extern retpoline_thunk_t __x86_indirect_call_thunk_ ## reg;
#include <asm/GEN-for-each-reg.h>
#undef GEN
#define GEN(reg) \
extern retpoline_thunk_t __x86_indirect_jump_thunk_ ## reg;
#include <asm/GEN-for-each-reg.h>
#undef GEN
#ifdef CONFIG_X86_64
/*
* Inline asm uses the %V modifier which is only in newer GCC
* which is ensured when CONFIG_MITIGATION_RETPOLINE is defined.
*/
# define CALL_NOSPEC \
ALTERNATIVE_2( \
ANNOTATE_RETPOLINE_SAFE \
"call *%[thunk_target]\n", \
"call __x86_indirect_thunk_%V[thunk_target]\n", \
X86_FEATURE_RETPOLINE, \
"lfence;\n" \
ANNOTATE_RETPOLINE_SAFE \
"call *%[thunk_target]\n", \
X86_FEATURE_RETPOLINE_LFENCE)
# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
#else /* CONFIG_X86_32 */
/*
* For i386 we use the original ret-equivalent retpoline, because
* otherwise we'll run out of registers. We don't care about CET
* here, anyway.
*/
# define CALL_NOSPEC \
ALTERNATIVE_2( \
ANNOTATE_RETPOLINE_SAFE \
"call *%[thunk_target]\n", \
" jmp 904f;\n" \
" .align 16\n" \
"901: call 903f;\n" \
"902: pause;\n" \
" lfence;\n" \
" jmp 902b;\n" \
" .align 16\n" \
"903: lea 4(%%esp), %%esp;\n" \
" pushl %[thunk_target];\n" \
" ret;\n" \
" .align 16\n" \
"904: call 901b;\n", \
X86_FEATURE_RETPOLINE, \
"lfence;\n" \
ANNOTATE_RETPOLINE_SAFE \
"call *%[thunk_target]\n", \
X86_FEATURE_RETPOLINE_LFENCE)
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
#endif
#else /* No retpoline for C / inline asm */
# define CALL_NOSPEC "call *%[thunk_target]\n"
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
#endif
/* The Spectre V2 mitigation variants */
enum spectre_v2_mitigation {
SPECTRE_V2_NONE,
SPECTRE_V2_RETPOLINE,
SPECTRE_V2_LFENCE,
SPECTRE_V2_EIBRS,
SPECTRE_V2_EIBRS_RETPOLINE,
SPECTRE_V2_EIBRS_LFENCE,
SPECTRE_V2_IBRS,
};
/* The indirect branch speculation control variants */
enum spectre_v2_user_mitigation {
SPECTRE_V2_USER_NONE,
SPECTRE_V2_USER_STRICT,
SPECTRE_V2_USER_STRICT_PREFERRED,
SPECTRE_V2_USER_PRCTL,
SPECTRE_V2_USER_SECCOMP,
};
/* The Speculative Store Bypass disable variants */
enum ssb_mitigation {
SPEC_STORE_BYPASS_NONE,
SPEC_STORE_BYPASS_DISABLE,
SPEC_STORE_BYPASS_PRCTL,
SPEC_STORE_BYPASS_SECCOMP,
};
static __always_inline
void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
{
asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
: : "c" (msr),
"a" ((u32)val),
"d" ((u32)(val >> 32)),
[feature] "i" (feature)
: "memory");
}
extern u64 x86_pred_cmd;
static inline void indirect_branch_prediction_barrier(void)
{
alternative_msr_write(MSR_IA32_PRED_CMD, x86_pred_cmd, X86_FEATURE_USE_IBPB);
}
/* The Intel SPEC CTRL MSR base value cache */
extern u64 x86_spec_ctrl_base;
DECLARE_PER_CPU(u64, x86_spec_ctrl_current);
extern void update_spec_ctrl_cond(u64 val);
extern u64 spec_ctrl_current(void);
/*
* With retpoline, we must use IBRS to restrict branch prediction
* before calling into firmware.
*
* (Implemented as CPP macros due to header hell.)
*/
#define firmware_restrict_branch_speculation_start() \
do { \
preempt_disable(); \
alternative_msr_write(MSR_IA32_SPEC_CTRL, \
spec_ctrl_current() | SPEC_CTRL_IBRS, \
X86_FEATURE_USE_IBRS_FW); \
alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, \
X86_FEATURE_USE_IBPB_FW); \
} while (0)
#define firmware_restrict_branch_speculation_end() \
do { \
alternative_msr_write(MSR_IA32_SPEC_CTRL, \
spec_ctrl_current(), \
X86_FEATURE_USE_IBRS_FW); \
preempt_enable(); \
} while (0)
DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear);
extern u16 mds_verw_sel;
#include <asm/segment.h>
/**
* mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
*
* This uses the otherwise unused and obsolete VERW instruction in
* combination with microcode which triggers a CPU buffer flush when the
* instruction is executed.
*/
static __always_inline void mds_clear_cpu_buffers(void)
{
static const u16 ds = __KERNEL_DS;
/*
* Has to be the memory-operand variant because only that
* guarantees the CPU buffer flush functionality according to
* documentation. The register-operand variant does not.
* Works with any segment selector, but a valid writable
* data segment is the fastest variant.
*
* "cc" clobber is required because VERW modifies ZF.
*/
asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
}
/**
* mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
*
* Clear CPU buffers if the corresponding static key is enabled
*/
static __always_inline void mds_idle_clear_cpu_buffers(void)
{
if (static_branch_likely(&mds_idle_clear)) mds_clear_cpu_buffers();
}
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
/* SPDX-License-Identifier: GPL-2.0 */
/*
* generic net pointers
*/
#ifndef __NET_GENERIC_H__
#define __NET_GENERIC_H__
#include <linux/bug.h>
#include <linux/rcupdate.h>
#include <net/net_namespace.h>
/*
* Generic net pointers are to be used by modules to put some private
* stuff on the struct net without explicit struct net modification
*
* The rules are simple:
* 1. set pernet_operations->id. After register_pernet_device you
* will have the id of your private pointer.
* 2. set pernet_operations->size to have the code allocate and free
* a private structure pointed to from struct net.
* 3. do not change this pointer while the net is alive;
* 4. do not try to have any private reference on the net_generic object.
*
* After accomplishing all of the above, the private pointer can be
* accessed with the net_generic() call.
*/
struct net_generic {
union {
struct {
unsigned int len;
struct rcu_head rcu;
} s;
DECLARE_FLEX_ARRAY(void *, ptr);
};
};
static inline void *net_generic(const struct net *net, unsigned int id)
{
struct net_generic *ng;
void *ptr;
rcu_read_lock(); ng = rcu_dereference(net->gen); ptr = ng->ptr[id]; rcu_read_unlock();
return ptr;
}
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
*
* 1997-11-28 Modified for POSIX.1b signals by Richard Henderson
* 2000-06-20 Pentium III FXSR, SSE support by Gareth Hughes
* 2000-2002 x86-64 support by Andi Kleen
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/kstrtox.h>
#include <linux/errno.h>
#include <linux/wait.h>
#include <linux/unistd.h>
#include <linux/stddef.h>
#include <linux/personality.h>
#include <linux/uaccess.h>
#include <linux/user-return-notifier.h>
#include <linux/uprobes.h>
#include <linux/context_tracking.h>
#include <linux/entry-common.h>
#include <linux/syscalls.h>
#include <linux/rseq.h>
#include <asm/processor.h>
#include <asm/ucontext.h>
#include <asm/fpu/signal.h>
#include <asm/fpu/xstate.h>
#include <asm/vdso.h>
#include <asm/mce.h>
#include <asm/sighandling.h>
#include <asm/vm86.h>
#include <asm/syscall.h>
#include <asm/sigframe.h>
#include <asm/signal.h>
#include <asm/shstk.h>
static inline int is_ia32_compat_frame(struct ksignal *ksig)
{
return IS_ENABLED(CONFIG_IA32_EMULATION) &&
ksig->ka.sa.sa_flags & SA_IA32_ABI;
}
static inline int is_ia32_frame(struct ksignal *ksig)
{
return IS_ENABLED(CONFIG_X86_32) || is_ia32_compat_frame(ksig);
}
static inline int is_x32_frame(struct ksignal *ksig)
{
return IS_ENABLED(CONFIG_X86_X32_ABI) &&
ksig->ka.sa.sa_flags & SA_X32_ABI;
}
/*
* Set up a signal frame.
*/
/* x86 ABI requires 16-byte alignment */
#define FRAME_ALIGNMENT 16UL
#define MAX_FRAME_PADDING (FRAME_ALIGNMENT - 1)
/*
* Determine which stack to use..
*/
void __user *
get_sigframe(struct ksignal *ksig, struct pt_regs *regs, size_t frame_size,
void __user **fpstate)
{
struct k_sigaction *ka = &ksig->ka;
int ia32_frame = is_ia32_frame(ksig);
/* Default to using normal stack */
bool nested_altstack = on_sig_stack(regs->sp);
bool entering_altstack = false;
unsigned long math_size = 0;
unsigned long sp = regs->sp;
unsigned long buf_fx = 0;
/* redzone */
if (!ia32_frame)
sp -= 128;
/* This is the X/Open sanctioned signal stack switching. */
if (ka->sa.sa_flags & SA_ONSTACK) {
/*
* This checks nested_altstack via sas_ss_flags(). Sensible
* programs use SS_AUTODISARM, which disables that check, and
* programs that don't use SS_AUTODISARM get compatible.
*/
if (sas_ss_flags(sp) == 0) { sp = current->sas_ss_sp + current->sas_ss_size;
entering_altstack = true;
}
} else if (ia32_frame &&
!nested_altstack &&
regs->ss != __USER_DS && !(ka->sa.sa_flags & SA_RESTORER) && ka->sa.sa_restorer) {
/* This is the legacy signal stack switching. */
sp = (unsigned long) ka->sa.sa_restorer;
entering_altstack = true;
}
sp = fpu__alloc_mathframe(sp, ia32_frame, &buf_fx, &math_size);
*fpstate = (void __user *)sp;
sp -= frame_size;
if (ia32_frame)
/*
* Align the stack pointer according to the i386 ABI,
* i.e. so that on function entry ((sp + 4) & 15) == 0.
*/
sp = ((sp + 4) & -FRAME_ALIGNMENT) - 4;
else
sp = round_down(sp, FRAME_ALIGNMENT) - 8;
/*
* If we are on the alternate signal stack and would overflow it, don't.
* Return an always-bogus address instead so we will die with SIGSEGV.
*/
if (unlikely((nested_altstack || entering_altstack) &&
!__on_sig_stack(sp))) {
if (show_unhandled_signals && printk_ratelimit()) pr_info("%s[%d] overflowed sigaltstack\n",
current->comm, task_pid_nr(current));
return (void __user *)-1L;
}
/* save i387 and extended state */
if (!copy_fpstate_to_sigframe(*fpstate, (void __user *)buf_fx, math_size))
return (void __user *)-1L;
return (void __user *)sp;
}
/*
* There are four different struct types for signal frame: sigframe_ia32,
* rt_sigframe_ia32, rt_sigframe_x32, and rt_sigframe. Use the worst case
* -- the largest size. It means the size for 64-bit apps is a bit more
* than needed, but this keeps the code simple.
*/
#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
# define MAX_FRAME_SIGINFO_UCTXT_SIZE sizeof(struct sigframe_ia32)
#else
# define MAX_FRAME_SIGINFO_UCTXT_SIZE sizeof(struct rt_sigframe)
#endif
/*
* The FP state frame contains an XSAVE buffer which must be 64-byte aligned.
* If a signal frame starts at an unaligned address, extra space is required.
* This is the max alignment padding, conservatively.
*/
#define MAX_XSAVE_PADDING 63UL
/*
* The frame data is composed of the following areas and laid out as:
*
* -------------------------
* | alignment padding |
* -------------------------
* | (f)xsave frame |
* -------------------------
* | fsave header |
* -------------------------
* | alignment padding |
* -------------------------
* | siginfo + ucontext |
* -------------------------
*/
/* max_frame_size tells userspace the worst case signal stack size. */
static unsigned long __ro_after_init max_frame_size;
static unsigned int __ro_after_init fpu_default_state_size;
static int __init init_sigframe_size(void)
{
fpu_default_state_size = fpu__get_fpstate_size();
max_frame_size = MAX_FRAME_SIGINFO_UCTXT_SIZE + MAX_FRAME_PADDING;
max_frame_size += fpu_default_state_size + MAX_XSAVE_PADDING;
/* Userspace expects an aligned size. */
max_frame_size = round_up(max_frame_size, FRAME_ALIGNMENT);
pr_info("max sigframe size: %lu\n", max_frame_size);
return 0;
}
early_initcall(init_sigframe_size);
unsigned long get_sigframe_size(void)
{
return max_frame_size;
}
static int
setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
{
/* Perform fixup for the pre-signal frame. */
rseq_signal_deliver(ksig, regs);
/* Set up the stack frame */
if (is_ia32_frame(ksig)) { if (ksig->ka.sa.sa_flags & SA_SIGINFO) return ia32_setup_rt_frame(ksig, regs);
else
return ia32_setup_frame(ksig, regs); } else if (is_x32_frame(ksig)) { return x32_setup_rt_frame(ksig, regs);
} else {
return x64_setup_rt_frame(ksig, regs);
}
}
static void
handle_signal(struct ksignal *ksig, struct pt_regs *regs)
{
bool stepping, failed;
struct fpu *fpu = ¤t->thread.fpu;
if (v8086_mode(regs))
save_v86_state((struct kernel_vm86_regs *) regs, VM86_SIGNAL);
/* Are we from a system call? */
if (syscall_get_nr(current, regs) != -1) {
/* If so, check system call restarting.. */
switch (syscall_get_error(current, regs)) {
case -ERESTART_RESTARTBLOCK:
case -ERESTARTNOHAND:
regs->ax = -EINTR;
break;
case -ERESTARTSYS:
if (!(ksig->ka.sa.sa_flags & SA_RESTART)) { regs->ax = -EINTR;
break;
}
fallthrough;
case -ERESTARTNOINTR:
regs->ax = regs->orig_ax;
regs->ip -= 2;
break;
}
}
/*
* If TF is set due to a debugger (TIF_FORCED_TF), clear TF now
* so that register information in the sigcontext is correct and
* then notify the tracer before entering the signal handler.
*/
stepping = test_thread_flag(TIF_SINGLESTEP);
if (stepping)
user_disable_single_step(current); failed = (setup_rt_frame(ksig, regs) < 0);
if (!failed) {
/*
* Clear the direction flag as per the ABI for function entry.
*
* Clear RF when entering the signal handler, because
* it might disable possible debug exception from the
* signal handler.
*
* Clear TF for the case when it wasn't set by debugger to
* avoid the recursive send_sigtrap() in SIGTRAP handler.
*/
regs->flags &= ~(X86_EFLAGS_DF|X86_EFLAGS_RF|X86_EFLAGS_TF);
/*
* Ensure the signal handler starts with the new fpu state.
*/
fpu__clear_user_states(fpu);
}
signal_setup_done(failed, ksig, stepping);
}
static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
{
#ifdef CONFIG_IA32_EMULATION
if (current->restart_block.arch_data & TS_COMPAT)
return __NR_ia32_restart_syscall;
#endif
#ifdef CONFIG_X86_X32_ABI
return __NR_restart_syscall | (regs->orig_ax & __X32_SYSCALL_BIT);
#else
return __NR_restart_syscall;
#endif
}
/*
* Note that 'init' is a special process: it doesn't get signals it doesn't
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*/
void arch_do_signal_or_restart(struct pt_regs *regs)
{
struct ksignal ksig;
if (get_signal(&ksig)) {
/* Whee! Actually deliver the signal. */
handle_signal(&ksig, regs);
return;
}
/* Did we come from a system call? */
if (syscall_get_nr(current, regs) != -1) {
/* Restart the system call - no handlers present */
switch (syscall_get_error(current, regs)) {
case -ERESTARTNOHAND:
case -ERESTARTSYS:
case -ERESTARTNOINTR:
regs->ax = regs->orig_ax;
regs->ip -= 2;
break;
case -ERESTART_RESTARTBLOCK:
regs->ax = get_nr_restart_syscall(regs); regs->ip -= 2;
break;
}
}
/*
* If there's no signal to deliver, we just put the saved sigmask
* back.
*/
restore_saved_sigmask();
}
void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
{
struct task_struct *me = current;
if (show_unhandled_signals && printk_ratelimit()) {
printk("%s"
"%s[%d] bad frame in %s frame:%p ip:%lx sp:%lx orax:%lx",
task_pid_nr(current) > 1 ? KERN_INFO : KERN_EMERG,
me->comm, me->pid, where, frame,
regs->ip, regs->sp, regs->orig_ax);
print_vma_addr(KERN_CONT " in ", regs->ip);
pr_cont("\n");
}
force_sig(SIGSEGV);
}
#ifdef CONFIG_DYNAMIC_SIGFRAME
#ifdef CONFIG_STRICT_SIGALTSTACK_SIZE
static bool strict_sigaltstack_size __ro_after_init = true;
#else
static bool strict_sigaltstack_size __ro_after_init = false;
#endif
static int __init strict_sas_size(char *arg)
{
return kstrtobool(arg, &strict_sigaltstack_size) == 0;
}
__setup("strict_sas_size", strict_sas_size);
/*
* MINSIGSTKSZ is 2048 and can't be changed despite the fact that AVX512
* exceeds that size already. As such programs might never use the
* sigaltstack they just continued to work. While always checking against
* the real size would be correct, this might be considered a regression.
*
* Therefore avoid the sanity check, unless enforced by kernel
* configuration or command line option.
*
* When dynamic FPU features are supported, the check is also enforced when
* the task has permissions to use dynamic features. Tasks which have no
* permission are checked against the size of the non-dynamic feature set
* if strict checking is enabled. This avoids forcing all tasks on the
* system to allocate large sigaltstacks even if they are never going
* to use a dynamic feature. As this is serialized via sighand::siglock
* any permission request for a dynamic feature either happened already
* or will see the newly install sigaltstack size in the permission checks.
*/
bool sigaltstack_size_valid(size_t ss_size)
{
unsigned long fsize = max_frame_size - fpu_default_state_size;
u64 mask;
lockdep_assert_held(¤t->sighand->siglock);
if (!fpu_state_size_dynamic() && !strict_sigaltstack_size)
return true;
fsize += current->group_leader->thread.fpu.perm.__user_state_size;
if (likely(ss_size > fsize))
return true;
if (strict_sigaltstack_size)
return ss_size > fsize;
mask = current->group_leader->thread.fpu.perm.__state_perm;
if (mask & XFEATURE_MASK_USER_DYNAMIC)
return ss_size > fsize;
return true;
}
#endif /* CONFIG_DYNAMIC_SIGFRAME */
// SPDX-License-Identifier: GPL-2.0
/*
* Kernel internal timers
*
* Copyright (C) 1991, 1992 Linus Torvalds
*
* 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
*
* 1997-09-10 Updated NTP code according to technical memorandum Jan '96
* "A Kernel Model for Precision Timekeeping" by Dave Mills
* 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
* serialize accesses to xtime/lost_ticks).
* Copyright (C) 1998 Andrea Arcangeli
* 1999-03-10 Improved NTP compatibility by Ulrich Windl
* 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
* 2000-10-05 Implemented scalable SMP per-CPU timer handling.
* Copyright (C) 2000, 2001, 2002 Ingo Molnar
* Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
*/
#include <linux/kernel_stat.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/pid_namespace.h>
#include <linux/notifier.h>
#include <linux/thread_info.h>
#include <linux/time.h>
#include <linux/jiffies.h>
#include <linux/posix-timers.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
#include <linux/delay.h>
#include <linux/tick.h>
#include <linux/kallsyms.h>
#include <linux/irq_work.h>
#include <linux/sched/signal.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/nohz.h>
#include <linux/sched/debug.h>
#include <linux/slab.h>
#include <linux/compat.h>
#include <linux/random.h>
#include <linux/sysctl.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
#include <asm/div64.h>
#include <asm/timex.h>
#include <asm/io.h>
#include "tick-internal.h"
#include "timer_migration.h"
#define CREATE_TRACE_POINTS
#include <trace/events/timer.h>
__visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
EXPORT_SYMBOL(jiffies_64);
/*
* The timer wheel has LVL_DEPTH array levels. Each level provides an array of
* LVL_SIZE buckets. Each level is driven by its own clock and therefore each
* level has a different granularity.
*
* The level granularity is: LVL_CLK_DIV ^ level
* The level clock frequency is: HZ / (LVL_CLK_DIV ^ level)
*
* The array level of a newly armed timer depends on the relative expiry
* time. The farther the expiry time is away the higher the array level and
* therefore the granularity becomes.
*
* Contrary to the original timer wheel implementation, which aims for 'exact'
* expiry of the timers, this implementation removes the need for recascading
* the timers into the lower array levels. The previous 'classic' timer wheel
* implementation of the kernel already violated the 'exact' expiry by adding
* slack to the expiry time to provide batched expiration. The granularity
* levels provide implicit batching.
*
* This is an optimization of the original timer wheel implementation for the
* majority of the timer wheel use cases: timeouts. The vast majority of
* timeout timers (networking, disk I/O ...) are canceled before expiry. If
* the timeout expires it indicates that normal operation is disturbed, so it
* does not matter much whether the timeout comes with a slight delay.
*
* The only exception to this are networking timers with a small expiry
* time. They rely on the granularity. Those fit into the first wheel level,
* which has HZ granularity.
*
* We don't have cascading anymore. timers with a expiry time above the
* capacity of the last wheel level are force expired at the maximum timeout
* value of the last wheel level. From data sampling we know that the maximum
* value observed is 5 days (network connection tracking), so this should not
* be an issue.
*
* The currently chosen array constants values are a good compromise between
* array size and granularity.
*
* This results in the following granularity and range levels:
*
* HZ 1000 steps
* Level Offset Granularity Range
* 0 0 1 ms 0 ms - 63 ms
* 1 64 8 ms 64 ms - 511 ms
* 2 128 64 ms 512 ms - 4095 ms (512ms - ~4s)
* 3 192 512 ms 4096 ms - 32767 ms (~4s - ~32s)
* 4 256 4096 ms (~4s) 32768 ms - 262143 ms (~32s - ~4m)
* 5 320 32768 ms (~32s) 262144 ms - 2097151 ms (~4m - ~34m)
* 6 384 262144 ms (~4m) 2097152 ms - 16777215 ms (~34m - ~4h)
* 7 448 2097152 ms (~34m) 16777216 ms - 134217727 ms (~4h - ~1d)
* 8 512 16777216 ms (~4h) 134217728 ms - 1073741822 ms (~1d - ~12d)
*
* HZ 300
* Level Offset Granularity Range
* 0 0 3 ms 0 ms - 210 ms
* 1 64 26 ms 213 ms - 1703 ms (213ms - ~1s)
* 2 128 213 ms 1706 ms - 13650 ms (~1s - ~13s)
* 3 192 1706 ms (~1s) 13653 ms - 109223 ms (~13s - ~1m)
* 4 256 13653 ms (~13s) 109226 ms - 873810 ms (~1m - ~14m)
* 5 320 109226 ms (~1m) 873813 ms - 6990503 ms (~14m - ~1h)
* 6 384 873813 ms (~14m) 6990506 ms - 55924050 ms (~1h - ~15h)
* 7 448 6990506 ms (~1h) 55924053 ms - 447392423 ms (~15h - ~5d)
* 8 512 55924053 ms (~15h) 447392426 ms - 3579139406 ms (~5d - ~41d)
*
* HZ 250
* Level Offset Granularity Range
* 0 0 4 ms 0 ms - 255 ms
* 1 64 32 ms 256 ms - 2047 ms (256ms - ~2s)
* 2 128 256 ms 2048 ms - 16383 ms (~2s - ~16s)
* 3 192 2048 ms (~2s) 16384 ms - 131071 ms (~16s - ~2m)
* 4 256 16384 ms (~16s) 131072 ms - 1048575 ms (~2m - ~17m)
* 5 320 131072 ms (~2m) 1048576 ms - 8388607 ms (~17m - ~2h)
* 6 384 1048576 ms (~17m) 8388608 ms - 67108863 ms (~2h - ~18h)
* 7 448 8388608 ms (~2h) 67108864 ms - 536870911 ms (~18h - ~6d)
* 8 512 67108864 ms (~18h) 536870912 ms - 4294967288 ms (~6d - ~49d)
*
* HZ 100
* Level Offset Granularity Range
* 0 0 10 ms 0 ms - 630 ms
* 1 64 80 ms 640 ms - 5110 ms (640ms - ~5s)
* 2 128 640 ms 5120 ms - 40950 ms (~5s - ~40s)
* 3 192 5120 ms (~5s) 40960 ms - 327670 ms (~40s - ~5m)
* 4 256 40960 ms (~40s) 327680 ms - 2621430 ms (~5m - ~43m)
* 5 320 327680 ms (~5m) 2621440 ms - 20971510 ms (~43m - ~5h)
* 6 384 2621440 ms (~43m) 20971520 ms - 167772150 ms (~5h - ~1d)
* 7 448 20971520 ms (~5h) 167772160 ms - 1342177270 ms (~1d - ~15d)
*/
/* Clock divisor for the next level */
#define LVL_CLK_SHIFT 3
#define LVL_CLK_DIV (1UL << LVL_CLK_SHIFT)
#define LVL_CLK_MASK (LVL_CLK_DIV - 1)
#define LVL_SHIFT(n) ((n) * LVL_CLK_SHIFT)
#define LVL_GRAN(n) (1UL << LVL_SHIFT(n))
/*
* The time start value for each level to select the bucket at enqueue
* time. We start from the last possible delta of the previous level
* so that we can later add an extra LVL_GRAN(n) to n (see calc_index()).
*/
#define LVL_START(n) ((LVL_SIZE - 1) << (((n) - 1) * LVL_CLK_SHIFT))
/* Size of each clock level */
#define LVL_BITS 6
#define LVL_SIZE (1UL << LVL_BITS)
#define LVL_MASK (LVL_SIZE - 1)
#define LVL_OFFS(n) ((n) * LVL_SIZE)
/* Level depth */
#if HZ > 100
# define LVL_DEPTH 9
# else
# define LVL_DEPTH 8
#endif
/* The cutoff (max. capacity of the wheel) */
#define WHEEL_TIMEOUT_CUTOFF (LVL_START(LVL_DEPTH))
#define WHEEL_TIMEOUT_MAX (WHEEL_TIMEOUT_CUTOFF - LVL_GRAN(LVL_DEPTH - 1))
/*
* The resulting wheel size. If NOHZ is configured we allocate two
* wheels so we have a separate storage for the deferrable timers.
*/
#define WHEEL_SIZE (LVL_SIZE * LVL_DEPTH)
#ifdef CONFIG_NO_HZ_COMMON
/*
* If multiple bases need to be locked, use the base ordering for lock
* nesting, i.e. lowest number first.
*/
# define NR_BASES 3
# define BASE_LOCAL 0
# define BASE_GLOBAL 1
# define BASE_DEF 2
#else
# define NR_BASES 1
# define BASE_LOCAL 0
# define BASE_GLOBAL 0
# define BASE_DEF 0
#endif
/**
* struct timer_base - Per CPU timer base (number of base depends on config)
* @lock: Lock protecting the timer_base
* @running_timer: When expiring timers, the lock is dropped. To make
* sure not to race against deleting/modifying a
* currently running timer, the pointer is set to the
* timer, which expires at the moment. If no timer is
* running, the pointer is NULL.
* @expiry_lock: PREEMPT_RT only: Lock is taken in softirq around
* timer expiry callback execution and when trying to
* delete a running timer and it wasn't successful in
* the first glance. It prevents priority inversion
* when callback was preempted on a remote CPU and a
* caller tries to delete the running timer. It also
* prevents a life lock, when the task which tries to
* delete a timer preempted the softirq thread which
* is running the timer callback function.
* @timer_waiters: PREEMPT_RT only: Tells, if there is a waiter
* waiting for the end of the timer callback function
* execution.
* @clk: clock of the timer base; is updated before enqueue
* of a timer; during expiry, it is 1 offset ahead of
* jiffies to avoid endless requeuing to current
* jiffies
* @next_expiry: expiry value of the first timer; it is updated when
* finding the next timer and during enqueue; the
* value is not valid, when next_expiry_recalc is set
* @cpu: Number of CPU the timer base belongs to
* @next_expiry_recalc: States, whether a recalculation of next_expiry is
* required. Value is set true, when a timer was
* deleted.
* @is_idle: Is set, when timer_base is idle. It is triggered by NOHZ
* code. This state is only used in standard
* base. Deferrable timers, which are enqueued remotely
* never wake up an idle CPU. So no matter of supporting it
* for this base.
* @timers_pending: Is set, when a timer is pending in the base. It is only
* reliable when next_expiry_recalc is not set.
* @pending_map: bitmap of the timer wheel; each bit reflects a
* bucket of the wheel. When a bit is set, at least a
* single timer is enqueued in the related bucket.
* @vectors: Array of lists; Each array member reflects a bucket
* of the timer wheel. The list contains all timers
* which are enqueued into a specific bucket.
*/
struct timer_base {
raw_spinlock_t lock;
struct timer_list *running_timer;
#ifdef CONFIG_PREEMPT_RT
spinlock_t expiry_lock;
atomic_t timer_waiters;
#endif
unsigned long clk;
unsigned long next_expiry;
unsigned int cpu;
bool next_expiry_recalc;
bool is_idle;
bool timers_pending;
DECLARE_BITMAP(pending_map, WHEEL_SIZE);
struct hlist_head vectors[WHEEL_SIZE];
} ____cacheline_aligned;
static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
#ifdef CONFIG_NO_HZ_COMMON
static DEFINE_STATIC_KEY_FALSE(timers_nohz_active);
static DEFINE_MUTEX(timer_keys_mutex);
static void timer_update_keys(struct work_struct *work);
static DECLARE_WORK(timer_update_work, timer_update_keys);
#ifdef CONFIG_SMP
static unsigned int sysctl_timer_migration = 1;
DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
static void timers_update_migration(void)
{
if (sysctl_timer_migration && tick_nohz_active)
static_branch_enable(&timers_migration_enabled);
else
static_branch_disable(&timers_migration_enabled);
}
#ifdef CONFIG_SYSCTL
static int timer_migration_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int ret;
mutex_lock(&timer_keys_mutex);
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (!ret && write)
timers_update_migration();
mutex_unlock(&timer_keys_mutex);
return ret;
}
static struct ctl_table timer_sysctl[] = {
{
.procname = "timer_migration",
.data = &sysctl_timer_migration,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = timer_migration_handler,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{}
};
static int __init timer_sysctl_init(void)
{
register_sysctl("kernel", timer_sysctl);
return 0;
}
device_initcall(timer_sysctl_init);
#endif /* CONFIG_SYSCTL */
#else /* CONFIG_SMP */
static inline void timers_update_migration(void) { }
#endif /* !CONFIG_SMP */
static void timer_update_keys(struct work_struct *work)
{
mutex_lock(&timer_keys_mutex);
timers_update_migration();
static_branch_enable(&timers_nohz_active);
mutex_unlock(&timer_keys_mutex);
}
void timers_update_nohz(void)
{
schedule_work(&timer_update_work);
}
static inline bool is_timers_nohz_active(void)
{
return static_branch_unlikely(&timers_nohz_active);
}
#else
static inline bool is_timers_nohz_active(void) { return false; }
#endif /* NO_HZ_COMMON */
static unsigned long round_jiffies_common(unsigned long j, int cpu,
bool force_up)
{
int rem;
unsigned long original = j;
/*
* We don't want all cpus firing their timers at once hitting the
* same lock or cachelines, so we skew each extra cpu with an extra
* 3 jiffies. This 3 jiffies came originally from the mm/ code which
* already did this.
* The skew is done by adding 3*cpunr, then round, then subtract this
* extra offset again.
*/
j += cpu * 3;
rem = j % HZ;
/*
* If the target jiffie is just after a whole second (which can happen
* due to delays of the timer irq, long irq off times etc etc) then
* we should round down to the whole second, not up. Use 1/4th second
* as cutoff for this rounding as an extreme upper bound for this.
* But never round down if @force_up is set.
*/
if (rem < HZ/4 && !force_up) /* round down */
j = j - rem;
else /* round up */
j = j - rem + HZ;
/* now that we have rounded, subtract the extra skew again */
j -= cpu * 3;
/*
* Make sure j is still in the future. Otherwise return the
* unmodified value.
*/
return time_is_after_jiffies(j) ? j : original;
}
/**
* __round_jiffies - function to round jiffies to a full second
* @j: the time in (absolute) jiffies that should be rounded
* @cpu: the processor number on which the timeout will happen
*
* __round_jiffies() rounds an absolute time in the future (in jiffies)
* up or down to (approximately) full seconds. This is useful for timers
* for which the exact time they fire does not matter too much, as long as
* they fire approximately every X seconds.
*
* By rounding these timers to whole seconds, all such timers will fire
* at the same time, rather than at various times spread out. The goal
* of this is to have the CPU wake up less, which saves power.
*
* The exact rounding is skewed for each processor to avoid all
* processors firing at the exact same time, which could lead
* to lock contention or spurious cache line bouncing.
*
* The return value is the rounded version of the @j parameter.
*/
unsigned long __round_jiffies(unsigned long j, int cpu)
{
return round_jiffies_common(j, cpu, false);
}
EXPORT_SYMBOL_GPL(__round_jiffies);
/**
* __round_jiffies_relative - function to round jiffies to a full second
* @j: the time in (relative) jiffies that should be rounded
* @cpu: the processor number on which the timeout will happen
*
* __round_jiffies_relative() rounds a time delta in the future (in jiffies)
* up or down to (approximately) full seconds. This is useful for timers
* for which the exact time they fire does not matter too much, as long as
* they fire approximately every X seconds.
*
* By rounding these timers to whole seconds, all such timers will fire
* at the same time, rather than at various times spread out. The goal
* of this is to have the CPU wake up less, which saves power.
*
* The exact rounding is skewed for each processor to avoid all
* processors firing at the exact same time, which could lead
* to lock contention or spurious cache line bouncing.
*
* The return value is the rounded version of the @j parameter.
*/
unsigned long __round_jiffies_relative(unsigned long j, int cpu)
{
unsigned long j0 = jiffies;
/* Use j0 because jiffies might change while we run */
return round_jiffies_common(j + j0, cpu, false) - j0;
}
EXPORT_SYMBOL_GPL(__round_jiffies_relative);
/**
* round_jiffies - function to round jiffies to a full second
* @j: the time in (absolute) jiffies that should be rounded
*
* round_jiffies() rounds an absolute time in the future (in jiffies)
* up or down to (approximately) full seconds. This is useful for timers
* for which the exact time they fire does not matter too much, as long as
* they fire approximately every X seconds.
*
* By rounding these timers to whole seconds, all such timers will fire
* at the same time, rather than at various times spread out. The goal
* of this is to have the CPU wake up less, which saves power.
*
* The return value is the rounded version of the @j parameter.
*/
unsigned long round_jiffies(unsigned long j)
{
return round_jiffies_common(j, raw_smp_processor_id(), false);
}
EXPORT_SYMBOL_GPL(round_jiffies);
/**
* round_jiffies_relative - function to round jiffies to a full second
* @j: the time in (relative) jiffies that should be rounded
*
* round_jiffies_relative() rounds a time delta in the future (in jiffies)
* up or down to (approximately) full seconds. This is useful for timers
* for which the exact time they fire does not matter too much, as long as
* they fire approximately every X seconds.
*
* By rounding these timers to whole seconds, all such timers will fire
* at the same time, rather than at various times spread out. The goal
* of this is to have the CPU wake up less, which saves power.
*
* The return value is the rounded version of the @j parameter.
*/
unsigned long round_jiffies_relative(unsigned long j)
{
return __round_jiffies_relative(j, raw_smp_processor_id());
}
EXPORT_SYMBOL_GPL(round_jiffies_relative);
/**
* __round_jiffies_up - function to round jiffies up to a full second
* @j: the time in (absolute) jiffies that should be rounded
* @cpu: the processor number on which the timeout will happen
*
* This is the same as __round_jiffies() except that it will never
* round down. This is useful for timeouts for which the exact time
* of firing does not matter too much, as long as they don't fire too
* early.
*/
unsigned long __round_jiffies_up(unsigned long j, int cpu)
{
return round_jiffies_common(j, cpu, true);
}
EXPORT_SYMBOL_GPL(__round_jiffies_up);
/**
* __round_jiffies_up_relative - function to round jiffies up to a full second
* @j: the time in (relative) jiffies that should be rounded
* @cpu: the processor number on which the timeout will happen
*
* This is the same as __round_jiffies_relative() except that it will never
* round down. This is useful for timeouts for which the exact time
* of firing does not matter too much, as long as they don't fire too
* early.
*/
unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)
{
unsigned long j0 = jiffies;
/* Use j0 because jiffies might change while we run */
return round_jiffies_common(j + j0, cpu, true) - j0;
}
EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);
/**
* round_jiffies_up - function to round jiffies up to a full second
* @j: the time in (absolute) jiffies that should be rounded
*
* This is the same as round_jiffies() except that it will never
* round down. This is useful for timeouts for which the exact time
* of firing does not matter too much, as long as they don't fire too
* early.
*/
unsigned long round_jiffies_up(unsigned long j)
{
return round_jiffies_common(j, raw_smp_processor_id(), true);
}
EXPORT_SYMBOL_GPL(round_jiffies_up);
/**
* round_jiffies_up_relative - function to round jiffies up to a full second
* @j: the time in (relative) jiffies that should be rounded
*
* This is the same as round_jiffies_relative() except that it will never
* round down. This is useful for timeouts for which the exact time
* of firing does not matter too much, as long as they don't fire too
* early.
*/
unsigned long round_jiffies_up_relative(unsigned long j)
{
return __round_jiffies_up_relative(j, raw_smp_processor_id());
}
EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
static inline unsigned int timer_get_idx(struct timer_list *timer)
{
return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;
}
static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)
{
timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |
idx << TIMER_ARRAYSHIFT;
}
/*
* Helper function to calculate the array index for a given expiry
* time.
*/
static inline unsigned calc_index(unsigned long expires, unsigned lvl,
unsigned long *bucket_expiry)
{
/*
* The timer wheel has to guarantee that a timer does not fire
* early. Early expiry can happen due to:
* - Timer is armed at the edge of a tick
* - Truncation of the expiry time in the outer wheel levels
*
* Round up with level granularity to prevent this.
*/
expires = (expires >> LVL_SHIFT(lvl)) + 1; *bucket_expiry = expires << LVL_SHIFT(lvl);
return LVL_OFFS(lvl) + (expires & LVL_MASK);
}
static int calc_wheel_index(unsigned long expires, unsigned long clk,
unsigned long *bucket_expiry)
{
unsigned long delta = expires - clk;
unsigned int idx;
if (delta < LVL_START(1)) {
idx = calc_index(expires, 0, bucket_expiry); } else if (delta < LVL_START(2)) { idx = calc_index(expires, 1, bucket_expiry); } else if (delta < LVL_START(3)) { idx = calc_index(expires, 2, bucket_expiry); } else if (delta < LVL_START(4)) { idx = calc_index(expires, 3, bucket_expiry); } else if (delta < LVL_START(5)) { idx = calc_index(expires, 4, bucket_expiry); } else if (delta < LVL_START(6)) { idx = calc_index(expires, 5, bucket_expiry); } else if (delta < LVL_START(7)) { idx = calc_index(expires, 6, bucket_expiry);
} else if (LVL_DEPTH > 8 && delta < LVL_START(8)) {
idx = calc_index(expires, 7, bucket_expiry);
} else if ((long) delta < 0) { idx = clk & LVL_MASK;
*bucket_expiry = clk;
} else {
/*
* Force expire obscene large timeouts to expire at the
* capacity limit of the wheel.
*/
if (delta >= WHEEL_TIMEOUT_CUTOFF) expires = clk + WHEEL_TIMEOUT_MAX; idx = calc_index(expires, LVL_DEPTH - 1, bucket_expiry);
}
return idx;
}
static void
trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
{
/*
* Deferrable timers do not prevent the CPU from entering dynticks and
* are not taken into account on the idle/nohz_full path. An IPI when a
* new deferrable timer is enqueued will wake up the remote CPU but
* nothing will be done with the deferrable timer base. Therefore skip
* the remote IPI for deferrable timers completely.
*/
if (!is_timers_nohz_active() || timer->flags & TIMER_DEFERRABLE)
return;
/*
* We might have to IPI the remote CPU if the base is idle and the
* timer is pinned. If it is a non pinned timer, it is only queued
* on the remote CPU, when timer was running during queueing. Then
* everything is handled by remote CPU anyway. If the other CPU is
* on the way to idle then it can't set base->is_idle as we hold
* the base lock:
*/
if (base->is_idle) { WARN_ON_ONCE(!(timer->flags & TIMER_PINNED ||
tick_nohz_full_cpu(base->cpu)));
wake_up_nohz_cpu(base->cpu);
}
}
/*
* Enqueue the timer into the hash bucket, mark it pending in
* the bitmap, store the index in the timer flags then wake up
* the target CPU if needed.
*/
static void enqueue_timer(struct timer_base *base, struct timer_list *timer,
unsigned int idx, unsigned long bucket_expiry)
{
hlist_add_head(&timer->entry, base->vectors + idx);
__set_bit(idx, base->pending_map);
timer_set_idx(timer, idx);
trace_timer_start(timer, bucket_expiry);
/*
* Check whether this is the new first expiring timer. The
* effective expiry time of the timer is required here
* (bucket_expiry) instead of timer->expires.
*/
if (time_before(bucket_expiry, base->next_expiry)) {
/*
* Set the next expiry time and kick the CPU so it
* can reevaluate the wheel:
*/
base->next_expiry = bucket_expiry;
base->timers_pending = true;
base->next_expiry_recalc = false;
trigger_dyntick_cpu(base, timer);
}
}
static void internal_add_timer(struct timer_base *base, struct timer_list *timer)
{
unsigned long bucket_expiry;
unsigned int idx;
idx = calc_wheel_index(timer->expires, base->clk, &bucket_expiry);
enqueue_timer(base, timer, idx, bucket_expiry);
}
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
static const struct debug_obj_descr timer_debug_descr;
struct timer_hint {
void (*function)(struct timer_list *t);
long offset;
};
#define TIMER_HINT(fn, container, timr, hintfn) \
{ \
.function = fn, \
.offset = offsetof(container, hintfn) - \
offsetof(container, timr) \
}
static const struct timer_hint timer_hints[] = {
TIMER_HINT(delayed_work_timer_fn,
struct delayed_work, timer, work.func),
TIMER_HINT(kthread_delayed_work_timer_fn,
struct kthread_delayed_work, timer, work.func),
};
static void *timer_debug_hint(void *addr)
{
struct timer_list *timer = addr;
int i;
for (i = 0; i < ARRAY_SIZE(timer_hints); i++) {
if (timer_hints[i].function == timer->function) {
void (**fn)(void) = addr + timer_hints[i].offset;
return *fn;
}
}
return timer->function;
}
static bool timer_is_static_object(void *addr)
{
struct timer_list *timer = addr;
return (timer->entry.pprev == NULL &&
timer->entry.next == TIMER_ENTRY_STATIC);
}
/*
* timer_fixup_init is called when:
* - an active object is initialized
*/
static bool timer_fixup_init(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
del_timer_sync(timer);
debug_object_init(timer, &timer_debug_descr);
return true;
default:
return false;
}
}
/* Stub timer callback for improperly used timers. */
static void stub_timer(struct timer_list *unused)
{
WARN_ON(1);
}
/*
* timer_fixup_activate is called when:
* - an active object is activated
* - an unknown non-static object is activated
*/
static bool timer_fixup_activate(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
timer_setup(timer, stub_timer, 0);
return true;
case ODEBUG_STATE_ACTIVE:
WARN_ON(1);
fallthrough;
default:
return false;
}
}
/*
* timer_fixup_free is called when:
* - an active object is freed
*/
static bool timer_fixup_free(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
switch (state) {
case ODEBUG_STATE_ACTIVE:
del_timer_sync(timer);
debug_object_free(timer, &timer_debug_descr);
return true;
default:
return false;
}
}
/*
* timer_fixup_assert_init is called when:
* - an untracked/uninit-ed object is found
*/
static bool timer_fixup_assert_init(void *addr, enum debug_obj_state state)
{
struct timer_list *timer = addr;
switch (state) {
case ODEBUG_STATE_NOTAVAILABLE:
timer_setup(timer, stub_timer, 0);
return true;
default:
return false;
}
}
static const struct debug_obj_descr timer_debug_descr = {
.name = "timer_list",
.debug_hint = timer_debug_hint,
.is_static_object = timer_is_static_object,
.fixup_init = timer_fixup_init,
.fixup_activate = timer_fixup_activate,
.fixup_free = timer_fixup_free,
.fixup_assert_init = timer_fixup_assert_init,
};
static inline void debug_timer_init(struct timer_list *timer)
{
debug_object_init(timer, &timer_debug_descr);
}
static inline void debug_timer_activate(struct timer_list *timer)
{
debug_object_activate(timer, &timer_debug_descr);
}
static inline void debug_timer_deactivate(struct timer_list *timer)
{
debug_object_deactivate(timer, &timer_debug_descr);
}
static inline void debug_timer_assert_init(struct timer_list *timer)
{
debug_object_assert_init(timer, &timer_debug_descr);
}
static void do_init_timer(struct timer_list *timer,
void (*func)(struct timer_list *),
unsigned int flags,
const char *name, struct lock_class_key *key);
void init_timer_on_stack_key(struct timer_list *timer,
void (*func)(struct timer_list *),
unsigned int flags,
const char *name, struct lock_class_key *key)
{
debug_object_init_on_stack(timer, &timer_debug_descr);
do_init_timer(timer, func, flags, name, key);
}
EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
void destroy_timer_on_stack(struct timer_list *timer)
{
debug_object_free(timer, &timer_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_timer_on_stack);
#else
static inline void debug_timer_init(struct timer_list *timer) { }
static inline void debug_timer_activate(struct timer_list *timer) { }
static inline void debug_timer_deactivate(struct timer_list *timer) { }
static inline void debug_timer_assert_init(struct timer_list *timer) { }
#endif
static inline void debug_init(struct timer_list *timer)
{
debug_timer_init(timer); trace_timer_init(timer);
}
static inline void debug_deactivate(struct timer_list *timer)
{
debug_timer_deactivate(timer); trace_timer_cancel(timer);
}
static inline void debug_assert_init(struct timer_list *timer)
{
debug_timer_assert_init(timer);
}
static void do_init_timer(struct timer_list *timer,
void (*func)(struct timer_list *),
unsigned int flags,
const char *name, struct lock_class_key *key)
{
timer->entry.pprev = NULL;
timer->function = func;
if (WARN_ON_ONCE(flags & ~TIMER_INIT_FLAGS))
flags &= TIMER_INIT_FLAGS;
timer->flags = flags | raw_smp_processor_id();
lockdep_init_map(&timer->lockdep_map, name, key, 0);
}
/**
* init_timer_key - initialize a timer
* @timer: the timer to be initialized
* @func: timer callback function
* @flags: timer flags
* @name: name of the timer
* @key: lockdep class key of the fake lock used for tracking timer
* sync lock dependencies
*
* init_timer_key() must be done to a timer prior to calling *any* of the
* other timer functions.
*/
void init_timer_key(struct timer_list *timer,
void (*func)(struct timer_list *), unsigned int flags,
const char *name, struct lock_class_key *key)
{
debug_init(timer); do_init_timer(timer, func, flags, name, key);
}
EXPORT_SYMBOL(init_timer_key);
static inline void detach_timer(struct timer_list *timer, bool clear_pending)
{
struct hlist_node *entry = &timer->entry;
debug_deactivate(timer); __hlist_del(entry); if (clear_pending) entry->pprev = NULL; entry->next = LIST_POISON2;
}
static int detach_if_pending(struct timer_list *timer, struct timer_base *base,
bool clear_pending)
{
unsigned idx = timer_get_idx(timer); if (!timer_pending(timer))
return 0;
if (hlist_is_singular_node(&timer->entry, base->vectors + idx)) { __clear_bit(idx, base->pending_map);
base->next_expiry_recalc = true;
}
detach_timer(timer, clear_pending); return 1;
}
static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu)
{
int index = tflags & TIMER_PINNED ? BASE_LOCAL : BASE_GLOBAL;
struct timer_base *base;
base = per_cpu_ptr(&timer_bases[index], cpu);
/*
* If the timer is deferrable and NO_HZ_COMMON is set then we need
* to use the deferrable base.
*/
if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu);
return base;
}
static inline struct timer_base *get_timer_this_cpu_base(u32 tflags)
{
int index = tflags & TIMER_PINNED ? BASE_LOCAL : BASE_GLOBAL;
struct timer_base *base;
base = this_cpu_ptr(&timer_bases[index]);
/*
* If the timer is deferrable and NO_HZ_COMMON is set then we need
* to use the deferrable base.
*/
if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
base = this_cpu_ptr(&timer_bases[BASE_DEF]);
return base;
}
static inline struct timer_base *get_timer_base(u32 tflags)
{
return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
}
static inline void __forward_timer_base(struct timer_base *base,
unsigned long basej)
{
/*
* Check whether we can forward the base. We can only do that when
* @basej is past base->clk otherwise we might rewind base->clk.
*/
if (time_before_eq(basej, base->clk))
return;
/*
* If the next expiry value is > jiffies, then we fast forward to
* jiffies otherwise we forward to the next expiry value.
*/
if (time_after(base->next_expiry, basej)) { base->clk = basej;
} else {
if (WARN_ON_ONCE(time_before(base->next_expiry, base->clk)))
return;
base->clk = base->next_expiry;
}
}
static inline void forward_timer_base(struct timer_base *base)
{
__forward_timer_base(base, READ_ONCE(jiffies));
}
/*
* We are using hashed locking: Holding per_cpu(timer_bases[x]).lock means
* that all timers which are tied to this base are locked, and the base itself
* is locked too.
*
* So __run_timers/migrate_timers can safely modify all timers which could
* be found in the base->vectors array.
*
* When a timer is migrating then the TIMER_MIGRATING flag is set and we need
* to wait until the migration is done.
*/
static struct timer_base *lock_timer_base(struct timer_list *timer,
unsigned long *flags)
__acquires(timer->base->lock)
{
for (;;) {
struct timer_base *base;
u32 tf;
/*
* We need to use READ_ONCE() here, otherwise the compiler
* might re-read @tf between the check for TIMER_MIGRATING
* and spin_lock().
*/
tf = READ_ONCE(timer->flags);
if (!(tf & TIMER_MIGRATING)) {
base = get_timer_base(tf); raw_spin_lock_irqsave(&base->lock, *flags);
if (timer->flags == tf)
return base; raw_spin_unlock_irqrestore(&base->lock, *flags);
}
cpu_relax();
}
}
#define MOD_TIMER_PENDING_ONLY 0x01
#define MOD_TIMER_REDUCE 0x02
#define MOD_TIMER_NOTPENDING 0x04
static inline int
__mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
{
unsigned long clk = 0, flags, bucket_expiry;
struct timer_base *base, *new_base;
unsigned int idx = UINT_MAX;
int ret = 0;
debug_assert_init(timer);
/*
* This is a common optimization triggered by the networking code - if
* the timer is re-modified to have the same timeout or ends up in the
* same array bucket then just return:
*/
if (!(options & MOD_TIMER_NOTPENDING) && timer_pending(timer)) {
/*
* The downside of this optimization is that it can result in
* larger granularity than you would get from adding a new
* timer with this expiry.
*/
long diff = timer->expires - expires;
if (!diff)
return 1;
if (options & MOD_TIMER_REDUCE && diff <= 0)
return 1;
/*
* We lock timer base and calculate the bucket index right
* here. If the timer ends up in the same bucket, then we
* just update the expiry time and avoid the whole
* dequeue/enqueue dance.
*/
base = lock_timer_base(timer, &flags);
/*
* Has @timer been shutdown? This needs to be evaluated
* while holding base lock to prevent a race against the
* shutdown code.
*/
if (!timer->function)
goto out_unlock;
forward_timer_base(base); if (timer_pending(timer) && (options & MOD_TIMER_REDUCE) && time_before_eq(timer->expires, expires)) {
ret = 1;
goto out_unlock;
}
clk = base->clk;
idx = calc_wheel_index(expires, clk, &bucket_expiry);
/*
* Retrieve and compare the array index of the pending
* timer. If it matches set the expiry to the new value so a
* subsequent call will exit in the expires check above.
*/
if (idx == timer_get_idx(timer)) {
if (!(options & MOD_TIMER_REDUCE)) timer->expires = expires; else if (time_after(timer->expires, expires)) timer->expires = expires;
ret = 1;
goto out_unlock;
}
} else {
base = lock_timer_base(timer, &flags);
/*
* Has @timer been shutdown? This needs to be evaluated
* while holding base lock to prevent a race against the
* shutdown code.
*/
if (!timer->function)
goto out_unlock;
forward_timer_base(base);
}
ret = detach_if_pending(timer, base, false); if (!ret && (options & MOD_TIMER_PENDING_ONLY))
goto out_unlock;
new_base = get_timer_this_cpu_base(timer->flags); if (base != new_base) {
/*
* We are trying to schedule the timer on the new base.
* However we can't change timer's base while it is running,
* otherwise timer_delete_sync() can't detect that the timer's
* handler yet has not finished. This also guarantees that the
* timer is serialized wrt itself.
*/
if (likely(base->running_timer != timer)) {
/* See the comment in lock_timer_base() */
timer->flags |= TIMER_MIGRATING; raw_spin_unlock(&base->lock);
base = new_base;
raw_spin_lock(&base->lock);
WRITE_ONCE(timer->flags,
(timer->flags & ~TIMER_BASEMASK) | base->cpu);
forward_timer_base(base);
}
}
debug_timer_activate(timer);
timer->expires = expires;
/*
* If 'idx' was calculated above and the base time did not advance
* between calculating 'idx' and possibly switching the base, only
* enqueue_timer() is required. Otherwise we need to (re)calculate
* the wheel index via internal_add_timer().
*/
if (idx != UINT_MAX && clk == base->clk) enqueue_timer(base, timer, idx, bucket_expiry);
else
internal_add_timer(base, timer);
out_unlock:
raw_spin_unlock_irqrestore(&base->lock, flags); return ret;
}
/**
* mod_timer_pending - Modify a pending timer's timeout
* @timer: The pending timer to be modified
* @expires: New absolute timeout in jiffies
*
* mod_timer_pending() is the same for pending timers as mod_timer(), but
* will not activate inactive timers.
*
* If @timer->function == NULL then the start operation is silently
* discarded.
*
* Return:
* * %0 - The timer was inactive and not modified or was in
* shutdown state and the operation was discarded
* * %1 - The timer was active and requeued to expire at @expires
*/
int mod_timer_pending(struct timer_list *timer, unsigned long expires)
{
return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);
}
EXPORT_SYMBOL(mod_timer_pending);
/**
* mod_timer - Modify a timer's timeout
* @timer: The timer to be modified
* @expires: New absolute timeout in jiffies
*
* mod_timer(timer, expires) is equivalent to:
*
* del_timer(timer); timer->expires = expires; add_timer(timer);
*
* mod_timer() is more efficient than the above open coded sequence. In
* case that the timer is inactive, the del_timer() part is a NOP. The
* timer is in any case activated with the new expiry time @expires.
*
* Note that if there are multiple unserialized concurrent users of the
* same timer, then mod_timer() is the only safe way to modify the timeout,
* since add_timer() cannot modify an already running timer.
*
* If @timer->function == NULL then the start operation is silently
* discarded. In this case the return value is 0 and meaningless.
*
* Return:
* * %0 - The timer was inactive and started or was in shutdown
* state and the operation was discarded
* * %1 - The timer was active and requeued to expire at @expires or
* the timer was active and not modified because @expires did
* not change the effective expiry time
*/
int mod_timer(struct timer_list *timer, unsigned long expires)
{
return __mod_timer(timer, expires, 0);
}
EXPORT_SYMBOL(mod_timer);
/**
* timer_reduce - Modify a timer's timeout if it would reduce the timeout
* @timer: The timer to be modified
* @expires: New absolute timeout in jiffies
*
* timer_reduce() is very similar to mod_timer(), except that it will only
* modify an enqueued timer if that would reduce the expiration time. If
* @timer is not enqueued it starts the timer.
*
* If @timer->function == NULL then the start operation is silently
* discarded.
*
* Return:
* * %0 - The timer was inactive and started or was in shutdown
* state and the operation was discarded
* * %1 - The timer was active and requeued to expire at @expires or
* the timer was active and not modified because @expires
* did not change the effective expiry time such that the
* timer would expire earlier than already scheduled
*/
int timer_reduce(struct timer_list *timer, unsigned long expires)
{
return __mod_timer(timer, expires, MOD_TIMER_REDUCE);
}
EXPORT_SYMBOL(timer_reduce);
/**
* add_timer - Start a timer
* @timer: The timer to be started
*
* Start @timer to expire at @timer->expires in the future. @timer->expires
* is the absolute expiry time measured in 'jiffies'. When the timer expires
* timer->function(timer) will be invoked from soft interrupt context.
*
* The @timer->expires and @timer->function fields must be set prior
* to calling this function.
*
* If @timer->function == NULL then the start operation is silently
* discarded.
*
* If @timer->expires is already in the past @timer will be queued to
* expire at the next timer tick.
*
* This can only operate on an inactive timer. Attempts to invoke this on
* an active timer are rejected with a warning.
*/
void add_timer(struct timer_list *timer)
{
if (WARN_ON_ONCE(timer_pending(timer)))
return;
__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
}
EXPORT_SYMBOL(add_timer);
/**
* add_timer_local() - Start a timer on the local CPU
* @timer: The timer to be started
*
* Same as add_timer() except that the timer flag TIMER_PINNED is set.
*
* See add_timer() for further details.
*/
void add_timer_local(struct timer_list *timer)
{
if (WARN_ON_ONCE(timer_pending(timer)))
return;
timer->flags |= TIMER_PINNED;
__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
}
EXPORT_SYMBOL(add_timer_local);
/**
* add_timer_global() - Start a timer without TIMER_PINNED flag set
* @timer: The timer to be started
*
* Same as add_timer() except that the timer flag TIMER_PINNED is unset.
*
* See add_timer() for further details.
*/
void add_timer_global(struct timer_list *timer)
{
if (WARN_ON_ONCE(timer_pending(timer)))
return;
timer->flags &= ~TIMER_PINNED;
__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
}
EXPORT_SYMBOL(add_timer_global);
/**
* add_timer_on - Start a timer on a particular CPU
* @timer: The timer to be started
* @cpu: The CPU to start it on
*
* Same as add_timer() except that it starts the timer on the given CPU and
* the TIMER_PINNED flag is set. When timer shouldn't be a pinned timer in
* the next round, add_timer_global() should be used instead as it unsets
* the TIMER_PINNED flag.
*
* See add_timer() for further details.
*/
void add_timer_on(struct timer_list *timer, int cpu)
{
struct timer_base *new_base, *base;
unsigned long flags;
debug_assert_init(timer);
if (WARN_ON_ONCE(timer_pending(timer)))
return;
/* Make sure timer flags have TIMER_PINNED flag set */
timer->flags |= TIMER_PINNED; new_base = get_timer_cpu_base(timer->flags, cpu);
/*
* If @timer was on a different CPU, it should be migrated with the
* old base locked to prevent other operations proceeding with the
* wrong base locked. See lock_timer_base().
*/
base = lock_timer_base(timer, &flags);
/*
* Has @timer been shutdown? This needs to be evaluated while
* holding base lock to prevent a race against the shutdown code.
*/
if (!timer->function)
goto out_unlock;
if (base != new_base) {
timer->flags |= TIMER_MIGRATING; raw_spin_unlock(&base->lock);
base = new_base;
raw_spin_lock(&base->lock);
WRITE_ONCE(timer->flags,
(timer->flags & ~TIMER_BASEMASK) | cpu);
}
forward_timer_base(base); debug_timer_activate(timer);
internal_add_timer(base, timer);
out_unlock:
raw_spin_unlock_irqrestore(&base->lock, flags);
}
EXPORT_SYMBOL_GPL(add_timer_on);
/**
* __timer_delete - Internal function: Deactivate a timer
* @timer: The timer to be deactivated
* @shutdown: If true, this indicates that the timer is about to be
* shutdown permanently.
*
* If @shutdown is true then @timer->function is set to NULL under the
* timer base lock which prevents further rearming of the time. In that
* case any attempt to rearm @timer after this function returns will be
* silently ignored.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
*/
static int __timer_delete(struct timer_list *timer, bool shutdown)
{
struct timer_base *base;
unsigned long flags;
int ret = 0;
debug_assert_init(timer);
/*
* If @shutdown is set then the lock has to be taken whether the
* timer is pending or not to protect against a concurrent rearm
* which might hit between the lockless pending check and the lock
* acquisition. By taking the lock it is ensured that such a newly
* enqueued timer is dequeued and cannot end up with
* timer->function == NULL in the expiry code.
*
* If timer->function is currently executed, then this makes sure
* that the callback cannot requeue the timer.
*/
if (timer_pending(timer) || shutdown) { base = lock_timer_base(timer, &flags);
ret = detach_if_pending(timer, base, true);
if (shutdown)
timer->function = NULL; raw_spin_unlock_irqrestore(&base->lock, flags);
}
return ret;
}
/**
* timer_delete - Deactivate a timer
* @timer: The timer to be deactivated
*
* The function only deactivates a pending timer, but contrary to
* timer_delete_sync() it does not take into account whether the timer's
* callback function is concurrently executed on a different CPU or not.
* It neither prevents rearming of the timer. If @timer can be rearmed
* concurrently then the return value of this function is meaningless.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
*/
int timer_delete(struct timer_list *timer)
{
return __timer_delete(timer, false);
}
EXPORT_SYMBOL(timer_delete);
/**
* timer_shutdown - Deactivate a timer and prevent rearming
* @timer: The timer to be deactivated
*
* The function does not wait for an eventually running timer callback on a
* different CPU but it prevents rearming of the timer. Any attempt to arm
* @timer after this function returns will be silently ignored.
*
* This function is useful for teardown code and should only be used when
* timer_shutdown_sync() cannot be invoked due to locking or context constraints.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending
*/
int timer_shutdown(struct timer_list *timer)
{
return __timer_delete(timer, true);
}
EXPORT_SYMBOL_GPL(timer_shutdown);
/**
* __try_to_del_timer_sync - Internal function: Try to deactivate a timer
* @timer: Timer to deactivate
* @shutdown: If true, this indicates that the timer is about to be
* shutdown permanently.
*
* If @shutdown is true then @timer->function is set to NULL under the
* timer base lock which prevents further rearming of the timer. Any
* attempt to rearm @timer after this function returns will be silently
* ignored.
*
* This function cannot guarantee that the timer cannot be rearmed
* right after dropping the base lock if @shutdown is false. That
* needs to be prevented by the calling code if necessary.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
* * %-1 - The timer callback function is running on a different CPU
*/
static int __try_to_del_timer_sync(struct timer_list *timer, bool shutdown)
{
struct timer_base *base;
unsigned long flags;
int ret = -1;
debug_assert_init(timer);
base = lock_timer_base(timer, &flags);
if (base->running_timer != timer)
ret = detach_if_pending(timer, base, true); if (shutdown) timer->function = NULL; raw_spin_unlock_irqrestore(&base->lock, flags);
return ret;
}
/**
* try_to_del_timer_sync - Try to deactivate a timer
* @timer: Timer to deactivate
*
* This function tries to deactivate a timer. On success the timer is not
* queued and the timer callback function is not running on any CPU.
*
* This function does not guarantee that the timer cannot be rearmed right
* after dropping the base lock. That needs to be prevented by the calling
* code if necessary.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
* * %-1 - The timer callback function is running on a different CPU
*/
int try_to_del_timer_sync(struct timer_list *timer)
{
return __try_to_del_timer_sync(timer, false);
}
EXPORT_SYMBOL(try_to_del_timer_sync);
#ifdef CONFIG_PREEMPT_RT
static __init void timer_base_init_expiry_lock(struct timer_base *base)
{
spin_lock_init(&base->expiry_lock);
}
static inline void timer_base_lock_expiry(struct timer_base *base)
{
spin_lock(&base->expiry_lock);
}
static inline void timer_base_unlock_expiry(struct timer_base *base)
{
spin_unlock(&base->expiry_lock);
}
/*
* The counterpart to del_timer_wait_running().
*
* If there is a waiter for base->expiry_lock, then it was waiting for the
* timer callback to finish. Drop expiry_lock and reacquire it. That allows
* the waiter to acquire the lock and make progress.
*/
static void timer_sync_wait_running(struct timer_base *base)
{
if (atomic_read(&base->timer_waiters)) {
raw_spin_unlock_irq(&base->lock);
spin_unlock(&base->expiry_lock);
spin_lock(&base->expiry_lock);
raw_spin_lock_irq(&base->lock);
}
}
/*
* This function is called on PREEMPT_RT kernels when the fast path
* deletion of a timer failed because the timer callback function was
* running.
*
* This prevents priority inversion, if the softirq thread on a remote CPU
* got preempted, and it prevents a life lock when the task which tries to
* delete a timer preempted the softirq thread running the timer callback
* function.
*/
static void del_timer_wait_running(struct timer_list *timer)
{
u32 tf;
tf = READ_ONCE(timer->flags);
if (!(tf & (TIMER_MIGRATING | TIMER_IRQSAFE))) {
struct timer_base *base = get_timer_base(tf);
/*
* Mark the base as contended and grab the expiry lock,
* which is held by the softirq across the timer
* callback. Drop the lock immediately so the softirq can
* expire the next timer. In theory the timer could already
* be running again, but that's more than unlikely and just
* causes another wait loop.
*/
atomic_inc(&base->timer_waiters);
spin_lock_bh(&base->expiry_lock);
atomic_dec(&base->timer_waiters);
spin_unlock_bh(&base->expiry_lock);
}
}
#else
static inline void timer_base_init_expiry_lock(struct timer_base *base) { }
static inline void timer_base_lock_expiry(struct timer_base *base) { }
static inline void timer_base_unlock_expiry(struct timer_base *base) { }
static inline void timer_sync_wait_running(struct timer_base *base) { }
static inline void del_timer_wait_running(struct timer_list *timer) { }
#endif
/**
* __timer_delete_sync - Internal function: Deactivate a timer and wait
* for the handler to finish.
* @timer: The timer to be deactivated
* @shutdown: If true, @timer->function will be set to NULL under the
* timer base lock which prevents rearming of @timer
*
* If @shutdown is not set the timer can be rearmed later. If the timer can
* be rearmed concurrently, i.e. after dropping the base lock then the
* return value is meaningless.
*
* If @shutdown is set then @timer->function is set to NULL under timer
* base lock which prevents rearming of the timer. Any attempt to rearm
* a shutdown timer is silently ignored.
*
* If the timer should be reused after shutdown it has to be initialized
* again.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
*/
static int __timer_delete_sync(struct timer_list *timer, bool shutdown)
{
int ret;
#ifdef CONFIG_LOCKDEP
unsigned long flags;
/*
* If lockdep gives a backtrace here, please reference
* the synchronization rules above.
*/
local_irq_save(flags); lock_map_acquire(&timer->lockdep_map);
lock_map_release(&timer->lockdep_map);
local_irq_restore(flags);
#endif
/*
* don't use it in hardirq context, because it
* could lead to deadlock.
*/
WARN_ON(in_hardirq() && !(timer->flags & TIMER_IRQSAFE));
/*
* Must be able to sleep on PREEMPT_RT because of the slowpath in
* del_timer_wait_running().
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(timer->flags & TIMER_IRQSAFE))
lockdep_assert_preemption_enabled();
do {
ret = __try_to_del_timer_sync(timer, shutdown);
if (unlikely(ret < 0)) {
del_timer_wait_running(timer);
cpu_relax();
}
} while (ret < 0);
return ret;
}
/**
* timer_delete_sync - Deactivate a timer and wait for the handler to finish.
* @timer: The timer to be deactivated
*
* Synchronization rules: Callers must prevent restarting of the timer,
* otherwise this function is meaningless. It must not be called from
* interrupt contexts unless the timer is an irqsafe one. The caller must
* not hold locks which would prevent completion of the timer's callback
* function. The timer's handler must not call add_timer_on(). Upon exit
* the timer is not queued and the handler is not running on any CPU.
*
* For !irqsafe timers, the caller must not hold locks that are held in
* interrupt context. Even if the lock has nothing to do with the timer in
* question. Here's why::
*
* CPU0 CPU1
* ---- ----
* <SOFTIRQ>
* call_timer_fn();
* base->running_timer = mytimer;
* spin_lock_irq(somelock);
* <IRQ>
* spin_lock(somelock);
* timer_delete_sync(mytimer);
* while (base->running_timer == mytimer);
*
* Now timer_delete_sync() will never return and never release somelock.
* The interrupt on the other CPU is waiting to grab somelock but it has
* interrupted the softirq that CPU0 is waiting to finish.
*
* This function cannot guarantee that the timer is not rearmed again by
* some concurrent or preempting code, right after it dropped the base
* lock. If there is the possibility of a concurrent rearm then the return
* value of the function is meaningless.
*
* If such a guarantee is needed, e.g. for teardown situations then use
* timer_shutdown_sync() instead.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending and deactivated
*/
int timer_delete_sync(struct timer_list *timer)
{
return __timer_delete_sync(timer, false);
}
EXPORT_SYMBOL(timer_delete_sync);
/**
* timer_shutdown_sync - Shutdown a timer and prevent rearming
* @timer: The timer to be shutdown
*
* When the function returns it is guaranteed that:
* - @timer is not queued
* - The callback function of @timer is not running
* - @timer cannot be enqueued again. Any attempt to rearm
* @timer is silently ignored.
*
* See timer_delete_sync() for synchronization rules.
*
* This function is useful for final teardown of an infrastructure where
* the timer is subject to a circular dependency problem.
*
* A common pattern for this is a timer and a workqueue where the timer can
* schedule work and work can arm the timer. On shutdown the workqueue must
* be destroyed and the timer must be prevented from rearming. Unless the
* code has conditionals like 'if (mything->in_shutdown)' to prevent that
* there is no way to get this correct with timer_delete_sync().
*
* timer_shutdown_sync() is solving the problem. The correct ordering of
* calls in this case is:
*
* timer_shutdown_sync(&mything->timer);
* workqueue_destroy(&mything->workqueue);
*
* After this 'mything' can be safely freed.
*
* This obviously implies that the timer is not required to be functional
* for the rest of the shutdown operation.
*
* Return:
* * %0 - The timer was not pending
* * %1 - The timer was pending
*/
int timer_shutdown_sync(struct timer_list *timer)
{
return __timer_delete_sync(timer, true);
}
EXPORT_SYMBOL_GPL(timer_shutdown_sync);
static void call_timer_fn(struct timer_list *timer,
void (*fn)(struct timer_list *),
unsigned long baseclk)
{
int count = preempt_count();
#ifdef CONFIG_LOCKDEP
/*
* It is permissible to free the timer from inside the
* function that is called from it, this we need to take into
* account for lockdep too. To avoid bogus "held lock freed"
* warnings as well as problems when looking into
* timer->lockdep_map, make a copy and use that here.
*/
struct lockdep_map lockdep_map;
lockdep_copy_map(&lockdep_map, &timer->lockdep_map);
#endif
/*
* Couple the lock chain with the lock chain at
* timer_delete_sync() by acquiring the lock_map around the fn()
* call here and in timer_delete_sync().
*/
lock_map_acquire(&lockdep_map);
trace_timer_expire_entry(timer, baseclk);
fn(timer);
trace_timer_expire_exit(timer);
lock_map_release(&lockdep_map);
if (count != preempt_count()) {
WARN_ONCE(1, "timer: %pS preempt leak: %08x -> %08x\n",
fn, count, preempt_count());
/*
* Restore the preempt count. That gives us a decent
* chance to survive and extract information. If the
* callback kept a lock held, bad luck, but not worse
* than the BUG() we had.
*/
preempt_count_set(count);
}
}
static void expire_timers(struct timer_base *base, struct hlist_head *head)
{
/*
* This value is required only for tracing. base->clk was
* incremented directly before expire_timers was called. But expiry
* is related to the old base->clk value.
*/
unsigned long baseclk = base->clk - 1;
while (!hlist_empty(head)) {
struct timer_list *timer;
void (*fn)(struct timer_list *);
timer = hlist_entry(head->first, struct timer_list, entry);
base->running_timer = timer;
detach_timer(timer, true);
fn = timer->function;
if (WARN_ON_ONCE(!fn)) {
/* Should never happen. Emphasis on should! */
base->running_timer = NULL;
continue;
}
if (timer->flags & TIMER_IRQSAFE) {
raw_spin_unlock(&base->lock);
call_timer_fn(timer, fn, baseclk);
raw_spin_lock(&base->lock);
base->running_timer = NULL;
} else {
raw_spin_unlock_irq(&base->lock);
call_timer_fn(timer, fn, baseclk);
raw_spin_lock_irq(&base->lock);
base->running_timer = NULL;
timer_sync_wait_running(base);
}
}
}
static int collect_expired_timers(struct timer_base *base,
struct hlist_head *heads)
{
unsigned long clk = base->clk = base->next_expiry;
struct hlist_head *vec;
int i, levels = 0;
unsigned int idx;
for (i = 0; i < LVL_DEPTH; i++) {
idx = (clk & LVL_MASK) + i * LVL_SIZE;
if (__test_and_clear_bit(idx, base->pending_map)) {
vec = base->vectors + idx;
hlist_move_list(vec, heads++);
levels++;
}
/* Is it time to look at the next level? */
if (clk & LVL_CLK_MASK)
break;
/* Shift clock for the next level granularity */
clk >>= LVL_CLK_SHIFT;
}
return levels;
}
/*
* Find the next pending bucket of a level. Search from level start (@offset)
* + @clk upwards and if nothing there, search from start of the level
* (@offset) up to @offset + clk.
*/
static int next_pending_bucket(struct timer_base *base, unsigned offset,
unsigned clk)
{
unsigned pos, start = offset + clk;
unsigned end = offset + LVL_SIZE;
pos = find_next_bit(base->pending_map, end, start);
if (pos < end)
return pos - start;
pos = find_next_bit(base->pending_map, start, offset);
return pos < start ? pos + LVL_SIZE - start : -1;
}
/*
* Search the first expiring timer in the various clock levels. Caller must
* hold base->lock.
*
* Store next expiry time in base->next_expiry.
*/
static void next_expiry_recalc(struct timer_base *base)
{
unsigned long clk, next, adj;
unsigned lvl, offset = 0;
next = base->clk + NEXT_TIMER_MAX_DELTA;
clk = base->clk;
for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) {
int pos = next_pending_bucket(base, offset, clk & LVL_MASK);
unsigned long lvl_clk = clk & LVL_CLK_MASK;
if (pos >= 0) {
unsigned long tmp = clk + (unsigned long) pos;
tmp <<= LVL_SHIFT(lvl);
if (time_before(tmp, next))
next = tmp;
/*
* If the next expiration happens before we reach
* the next level, no need to check further.
*/
if (pos <= ((LVL_CLK_DIV - lvl_clk) & LVL_CLK_MASK))
break;
}
/*
* Clock for the next level. If the current level clock lower
* bits are zero, we look at the next level as is. If not we
* need to advance it by one because that's going to be the
* next expiring bucket in that level. base->clk is the next
* expiring jiffie. So in case of:
*
* LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
* 0 0 0 0 0 0
*
* we have to look at all levels @index 0. With
*
* LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
* 0 0 0 0 0 2
*
* LVL0 has the next expiring bucket @index 2. The upper
* levels have the next expiring bucket @index 1.
*
* In case that the propagation wraps the next level the same
* rules apply:
*
* LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
* 0 0 0 0 F 2
*
* So after looking at LVL0 we get:
*
* LVL5 LVL4 LVL3 LVL2 LVL1
* 0 0 0 1 0
*
* So no propagation from LVL1 to LVL2 because that happened
* with the add already, but then we need to propagate further
* from LVL2 to LVL3.
*
* So the simple check whether the lower bits of the current
* level are 0 or not is sufficient for all cases.
*/
adj = lvl_clk ? 1 : 0;
clk >>= LVL_CLK_SHIFT;
clk += adj;
}
base->next_expiry = next;
base->next_expiry_recalc = false;
base->timers_pending = !(next == base->clk + NEXT_TIMER_MAX_DELTA);
}
#ifdef CONFIG_NO_HZ_COMMON
/*
* Check, if the next hrtimer event is before the next timer wheel
* event:
*/
static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)
{
u64 nextevt = hrtimer_get_next_event();
/*
* If high resolution timers are enabled
* hrtimer_get_next_event() returns KTIME_MAX.
*/
if (expires <= nextevt)
return expires;
/*
* If the next timer is already expired, return the tick base
* time so the tick is fired immediately.
*/
if (nextevt <= basem)
return basem;
/*
* Round up to the next jiffie. High resolution timers are
* off, so the hrtimers are expired in the tick and we need to
* make sure that this tick really expires the timer to avoid
* a ping pong of the nohz stop code.
*
* Use DIV_ROUND_UP_ULL to prevent gcc calling __divdi3
*/
return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC;
}
static unsigned long next_timer_interrupt(struct timer_base *base,
unsigned long basej)
{
if (base->next_expiry_recalc)
next_expiry_recalc(base);
/*
* Move next_expiry for the empty base into the future to prevent an
* unnecessary raise of the timer softirq when the next_expiry value
* will be reached even if there is no timer pending.
*
* This update is also required to make timer_base::next_expiry values
* easy comparable to find out which base holds the first pending timer.
*/
if (!base->timers_pending)
base->next_expiry = basej + NEXT_TIMER_MAX_DELTA;
return base->next_expiry;
}
static unsigned long fetch_next_timer_interrupt(unsigned long basej, u64 basem,
struct timer_base *base_local,
struct timer_base *base_global,
struct timer_events *tevt)
{
unsigned long nextevt, nextevt_local, nextevt_global;
bool local_first;
nextevt_local = next_timer_interrupt(base_local, basej);
nextevt_global = next_timer_interrupt(base_global, basej);
local_first = time_before_eq(nextevt_local, nextevt_global);
nextevt = local_first ? nextevt_local : nextevt_global;
/*
* If the @nextevt is at max. one tick away, use @nextevt and store
* it in the local expiry value. The next global event is irrelevant in
* this case and can be left as KTIME_MAX.
*/
if (time_before_eq(nextevt, basej + 1)) {
/* If we missed a tick already, force 0 delta */
if (time_before(nextevt, basej))
nextevt = basej;
tevt->local = basem + (u64)(nextevt - basej) * TICK_NSEC;
/*
* This is required for the remote check only but it doesn't
* hurt, when it is done for both call sites:
*
* * The remote callers will only take care of the global timers
* as local timers will be handled by CPU itself. When not
* updating tevt->global with the already missed first global
* timer, it is possible that it will be missed completely.
*
* * The local callers will ignore the tevt->global anyway, when
* nextevt is max. one tick away.
*/
if (!local_first)
tevt->global = tevt->local;
return nextevt;
}
/*
* Update tevt.* values:
*
* If the local queue expires first, then the global event can be
* ignored. If the global queue is empty, nothing to do either.
*/
if (!local_first && base_global->timers_pending)
tevt->global = basem + (u64)(nextevt_global - basej) * TICK_NSEC;
if (base_local->timers_pending)
tevt->local = basem + (u64)(nextevt_local - basej) * TICK_NSEC;
return nextevt;
}
# ifdef CONFIG_SMP
/**
* fetch_next_timer_interrupt_remote() - Store next timers into @tevt
* @basej: base time jiffies
* @basem: base time clock monotonic
* @tevt: Pointer to the storage for the expiry values
* @cpu: Remote CPU
*
* Stores the next pending local and global timer expiry values in the
* struct pointed to by @tevt. If a queue is empty the corresponding
* field is set to KTIME_MAX. If local event expires before global
* event, global event is set to KTIME_MAX as well.
*
* Caller needs to make sure timer base locks are held (use
* timer_lock_remote_bases() for this purpose).
*/
void fetch_next_timer_interrupt_remote(unsigned long basej, u64 basem,
struct timer_events *tevt,
unsigned int cpu)
{
struct timer_base *base_local, *base_global;
/* Preset local / global events */
tevt->local = tevt->global = KTIME_MAX;
base_local = per_cpu_ptr(&timer_bases[BASE_LOCAL], cpu);
base_global = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu);
lockdep_assert_held(&base_local->lock);
lockdep_assert_held(&base_global->lock);
fetch_next_timer_interrupt(basej, basem, base_local, base_global, tevt);
}
/**
* timer_unlock_remote_bases - unlock timer bases of cpu
* @cpu: Remote CPU
*
* Unlocks the remote timer bases.
*/
void timer_unlock_remote_bases(unsigned int cpu)
__releases(timer_bases[BASE_LOCAL]->lock)
__releases(timer_bases[BASE_GLOBAL]->lock)
{
struct timer_base *base_local, *base_global;
base_local = per_cpu_ptr(&timer_bases[BASE_LOCAL], cpu);
base_global = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu);
raw_spin_unlock(&base_global->lock);
raw_spin_unlock(&base_local->lock);
}
/**
* timer_lock_remote_bases - lock timer bases of cpu
* @cpu: Remote CPU
*
* Locks the remote timer bases.
*/
void timer_lock_remote_bases(unsigned int cpu)
__acquires(timer_bases[BASE_LOCAL]->lock)
__acquires(timer_bases[BASE_GLOBAL]->lock)
{
struct timer_base *base_local, *base_global;
base_local = per_cpu_ptr(&timer_bases[BASE_LOCAL], cpu);
base_global = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu);
lockdep_assert_irqs_disabled();
raw_spin_lock(&base_local->lock);
raw_spin_lock_nested(&base_global->lock, SINGLE_DEPTH_NESTING);
}
/**
* timer_base_is_idle() - Return whether timer base is set idle
*
* Returns value of local timer base is_idle value.
*/
bool timer_base_is_idle(void)
{
return __this_cpu_read(timer_bases[BASE_LOCAL].is_idle);
}
static void __run_timer_base(struct timer_base *base);
/**
* timer_expire_remote() - expire global timers of cpu
* @cpu: Remote CPU
*
* Expire timers of global base of remote CPU.
*/
void timer_expire_remote(unsigned int cpu)
{
struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu);
__run_timer_base(base);
}
static void timer_use_tmigr(unsigned long basej, u64 basem,
unsigned long *nextevt, bool *tick_stop_path,
bool timer_base_idle, struct timer_events *tevt)
{
u64 next_tmigr;
if (timer_base_idle)
next_tmigr = tmigr_cpu_new_timer(tevt->global);
else if (tick_stop_path)
next_tmigr = tmigr_cpu_deactivate(tevt->global);
else
next_tmigr = tmigr_quick_check(tevt->global);
/*
* If the CPU is the last going idle in timer migration hierarchy, make
* sure the CPU will wake up in time to handle remote timers.
* next_tmigr == KTIME_MAX if other CPUs are still active.
*/
if (next_tmigr < tevt->local) {
u64 tmp;
/* If we missed a tick already, force 0 delta */
if (next_tmigr < basem)
next_tmigr = basem;
tmp = div_u64(next_tmigr - basem, TICK_NSEC);
*nextevt = basej + (unsigned long)tmp;
tevt->local = next_tmigr;
}
}
# else
static void timer_use_tmigr(unsigned long basej, u64 basem,
unsigned long *nextevt, bool *tick_stop_path,
bool timer_base_idle, struct timer_events *tevt)
{
/*
* Make sure first event is written into tevt->local to not miss a
* timer on !SMP systems.
*/
tevt->local = min_t(u64, tevt->local, tevt->global);
}
# endif /* CONFIG_SMP */
static inline u64 __get_next_timer_interrupt(unsigned long basej, u64 basem,
bool *idle)
{
struct timer_events tevt = { .local = KTIME_MAX, .global = KTIME_MAX };
struct timer_base *base_local, *base_global;
unsigned long nextevt;
bool idle_is_possible;
/*
* When the CPU is offline, the tick is cancelled and nothing is supposed
* to try to stop it.
*/
if (WARN_ON_ONCE(cpu_is_offline(smp_processor_id()))) {
if (idle)
*idle = true;
return tevt.local;
}
base_local = this_cpu_ptr(&timer_bases[BASE_LOCAL]);
base_global = this_cpu_ptr(&timer_bases[BASE_GLOBAL]);
raw_spin_lock(&base_local->lock);
raw_spin_lock_nested(&base_global->lock, SINGLE_DEPTH_NESTING);
nextevt = fetch_next_timer_interrupt(basej, basem, base_local,
base_global, &tevt);
/*
* If the next event is only one jiffie ahead there is no need to call
* timer migration hierarchy related functions. The value for the next
* global timer in @tevt struct equals then KTIME_MAX. This is also
* true, when the timer base is idle.
*
* The proper timer migration hierarchy function depends on the callsite
* and whether timer base is idle or not. @nextevt will be updated when
* this CPU needs to handle the first timer migration hierarchy
* event. See timer_use_tmigr() for detailed information.
*/
idle_is_possible = time_after(nextevt, basej + 1);
if (idle_is_possible)
timer_use_tmigr(basej, basem, &nextevt, idle,
base_local->is_idle, &tevt);
/*
* We have a fresh next event. Check whether we can forward the
* base.
*/
__forward_timer_base(base_local, basej);
__forward_timer_base(base_global, basej);
/*
* Set base->is_idle only when caller is timer_base_try_to_set_idle()
*/
if (idle) {
/*
* Bases are idle if the next event is more than a tick
* away. Caution: @nextevt could have changed by enqueueing a
* global timer into timer migration hierarchy. Therefore a new
* check is required here.
*
* If the base is marked idle then any timer add operation must
* forward the base clk itself to keep granularity small. This
* idle logic is only maintained for the BASE_LOCAL and
* BASE_GLOBAL base, deferrable timers may still see large
* granularity skew (by design).
*/
if (!base_local->is_idle && time_after(nextevt, basej + 1)) {
base_local->is_idle = true;
/*
* Global timers queued locally while running in a task
* in nohz_full mode need a self-IPI to kick reprogramming
* in IRQ tail.
*/
if (tick_nohz_full_cpu(base_local->cpu))
base_global->is_idle = true;
trace_timer_base_idle(true, base_local->cpu);
}
*idle = base_local->is_idle;
/*
* When timer base is not set idle, undo the effect of
* tmigr_cpu_deactivate() to prevent inconsistent states - active
* timer base but inactive timer migration hierarchy.
*
* When timer base was already marked idle, nothing will be
* changed here.
*/
if (!base_local->is_idle && idle_is_possible)
tmigr_cpu_activate();
}
raw_spin_unlock(&base_global->lock);
raw_spin_unlock(&base_local->lock);
return cmp_next_hrtimer_event(basem, tevt.local);
}
/**
* get_next_timer_interrupt() - return the time (clock mono) of the next timer
* @basej: base time jiffies
* @basem: base time clock monotonic
*
* Returns the tick aligned clock monotonic time of the next pending timer or
* KTIME_MAX if no timer is pending. If timer of global base was queued into
* timer migration hierarchy, first global timer is not taken into account. If
* it was the last CPU of timer migration hierarchy going idle, first global
* event is taken into account.
*/
u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
{
return __get_next_timer_interrupt(basej, basem, NULL);
}
/**
* timer_base_try_to_set_idle() - Try to set the idle state of the timer bases
* @basej: base time jiffies
* @basem: base time clock monotonic
* @idle: pointer to store the value of timer_base->is_idle on return;
* *idle contains the information whether tick was already stopped
*
* Returns the tick aligned clock monotonic time of the next pending timer or
* KTIME_MAX if no timer is pending. When tick was already stopped KTIME_MAX is
* returned as well.
*/
u64 timer_base_try_to_set_idle(unsigned long basej, u64 basem, bool *idle)
{
if (*idle)
return KTIME_MAX;
return __get_next_timer_interrupt(basej, basem, idle);
}
/**
* timer_clear_idle - Clear the idle state of the timer base
*
* Called with interrupts disabled
*/
void timer_clear_idle(void)
{
/*
* We do this unlocked. The worst outcome is a remote pinned timer
* enqueue sending a pointless IPI, but taking the lock would just
* make the window for sending the IPI a few instructions smaller
* for the cost of taking the lock in the exit from idle
* path. Required for BASE_LOCAL only.
*/
__this_cpu_write(timer_bases[BASE_LOCAL].is_idle, false);
if (tick_nohz_full_cpu(smp_processor_id()))
__this_cpu_write(timer_bases[BASE_GLOBAL].is_idle, false);
trace_timer_base_idle(false, smp_processor_id());
/* Activate without holding the timer_base->lock */
tmigr_cpu_activate();
}
#endif
/**
* __run_timers - run all expired timers (if any) on this CPU.
* @base: the timer vector to be processed.
*/
static inline void __run_timers(struct timer_base *base)
{
struct hlist_head heads[LVL_DEPTH];
int levels;
lockdep_assert_held(&base->lock);
if (base->running_timer)
return;
while (time_after_eq(jiffies, base->clk) &&
time_after_eq(jiffies, base->next_expiry)) {
levels = collect_expired_timers(base, heads);
/*
* The two possible reasons for not finding any expired
* timer at this clk are that all matching timers have been
* dequeued or no timer has been queued since
* base::next_expiry was set to base::clk +
* NEXT_TIMER_MAX_DELTA.
*/
WARN_ON_ONCE(!levels && !base->next_expiry_recalc
&& base->timers_pending);
/*
* While executing timers, base->clk is set 1 offset ahead of
* jiffies to avoid endless requeuing to current jiffies.
*/
base->clk++;
next_expiry_recalc(base);
while (levels--)
expire_timers(base, heads + levels);
}
}
static void __run_timer_base(struct timer_base *base)
{
if (time_before(jiffies, base->next_expiry))
return;
timer_base_lock_expiry(base);
raw_spin_lock_irq(&base->lock);
__run_timers(base);
raw_spin_unlock_irq(&base->lock);
timer_base_unlock_expiry(base);
}
static void run_timer_base(int index)
{
struct timer_base *base = this_cpu_ptr(&timer_bases[index]);
__run_timer_base(base);
}
/*
* This function runs timers and the timer-tq in bottom half context.
*/
static __latent_entropy void run_timer_softirq(struct softirq_action *h)
{
run_timer_base(BASE_LOCAL);
if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
run_timer_base(BASE_GLOBAL);
run_timer_base(BASE_DEF);
if (is_timers_nohz_active())
tmigr_handle_remote();
}
}
/*
* Called by the local, per-CPU timer interrupt on SMP.
*/
static void run_local_timers(void)
{
struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_LOCAL]);
hrtimer_run_queues();
for (int i = 0; i < NR_BASES; i++, base++) {
/* Raise the softirq only if required. */
if (time_after_eq(jiffies, base->next_expiry) || (i == BASE_DEF && tmigr_requires_handle_remote())) { raise_softirq(TIMER_SOFTIRQ);
return;
}
}
}
/*
* Called from the timer interrupt handler to charge one tick to the current
* process. user_tick is 1 if the tick is user time, 0 for system.
*/
void update_process_times(int user_tick)
{
struct task_struct *p = current;
/* Note: this timer irq context must be accounted for as well. */
account_process_tick(p, user_tick);
run_local_timers(); rcu_sched_clock_irq(user_tick);
#ifdef CONFIG_IRQ_WORK
if (in_irq())
irq_work_tick();
#endif
scheduler_tick();
if (IS_ENABLED(CONFIG_POSIX_TIMERS))
run_posix_cpu_timers();
}
/*
* Since schedule_timeout()'s timer is defined on the stack, it must store
* the target task on the stack as well.
*/
struct process_timer {
struct timer_list timer;
struct task_struct *task;
};
static void process_timeout(struct timer_list *t)
{
struct process_timer *timeout = from_timer(timeout, t, timer);
wake_up_process(timeout->task);
}
/**
* schedule_timeout - sleep until timeout
* @timeout: timeout value in jiffies
*
* Make the current task sleep until @timeout jiffies have elapsed.
* The function behavior depends on the current task state
* (see also set_current_state() description):
*
* %TASK_RUNNING - the scheduler is called, but the task does not sleep
* at all. That happens because sched_submit_work() does nothing for
* tasks in %TASK_RUNNING state.
*
* %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
* pass before the routine returns unless the current task is explicitly
* woken up, (e.g. by wake_up_process()).
*
* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
* delivered to the current task or the current task is explicitly woken
* up.
*
* The current task state is guaranteed to be %TASK_RUNNING when this
* routine returns.
*
* Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
* the CPU away without a bound on the timeout. In this case the return
* value will be %MAX_SCHEDULE_TIMEOUT.
*
* Returns 0 when the timer has expired otherwise the remaining time in
* jiffies will be returned. In all cases the return value is guaranteed
* to be non-negative.
*/
signed long __sched schedule_timeout(signed long timeout)
{
struct process_timer timer;
unsigned long expire;
switch (timeout)
{
case MAX_SCHEDULE_TIMEOUT:
/*
* These two special cases are useful to be comfortable
* in the caller. Nothing more. We could take
* MAX_SCHEDULE_TIMEOUT from one of the negative value
* but I' d like to return a valid offset (>=0) to allow
* the caller to do everything it want with the retval.
*/
schedule();
goto out;
default:
/*
* Another bit of PARANOID. Note that the retval will be
* 0 since no piece of kernel is supposed to do a check
* for a negative retval of schedule_timeout() (since it
* should never happens anyway). You just have the printk()
* that will tell you if something is gone wrong and where.
*/
if (timeout < 0) { printk(KERN_ERR "schedule_timeout: wrong timeout "
"value %lx\n", timeout);
dump_stack();
__set_current_state(TASK_RUNNING);
goto out;
}
}
expire = timeout + jiffies;
timer.task = current;
timer_setup_on_stack(&timer.timer, process_timeout, 0);
__mod_timer(&timer.timer, expire, MOD_TIMER_NOTPENDING);
schedule();
del_timer_sync(&timer.timer);
/* Remove the timer from the object tracker */
destroy_timer_on_stack(&timer.timer);
timeout = expire - jiffies;
out:
return timeout < 0 ? 0 : timeout;
}
EXPORT_SYMBOL(schedule_timeout);
/*
* We can use __set_current_state() here because schedule_timeout() calls
* schedule() unconditionally.
*/
signed long __sched schedule_timeout_interruptible(signed long timeout)
{
__set_current_state(TASK_INTERRUPTIBLE);
return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_interruptible);
signed long __sched schedule_timeout_killable(signed long timeout)
{
__set_current_state(TASK_KILLABLE);
return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_killable);
signed long __sched schedule_timeout_uninterruptible(signed long timeout)
{
__set_current_state(TASK_UNINTERRUPTIBLE);
return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_uninterruptible);
/*
* Like schedule_timeout_uninterruptible(), except this task will not contribute
* to load average.
*/
signed long __sched schedule_timeout_idle(signed long timeout)
{
__set_current_state(TASK_IDLE);
return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_idle);
#ifdef CONFIG_HOTPLUG_CPU
static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head)
{
struct timer_list *timer;
int cpu = new_base->cpu;
while (!hlist_empty(head)) {
timer = hlist_entry(head->first, struct timer_list, entry);
detach_timer(timer, false);
timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;
internal_add_timer(new_base, timer);
}
}
int timers_prepare_cpu(unsigned int cpu)
{
struct timer_base *base;
int b;
for (b = 0; b < NR_BASES; b++) {
base = per_cpu_ptr(&timer_bases[b], cpu);
base->clk = jiffies;
base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA;
base->next_expiry_recalc = false;
base->timers_pending = false;
base->is_idle = false;
}
return 0;
}
int timers_dead_cpu(unsigned int cpu)
{
struct timer_base *old_base;
struct timer_base *new_base;
int b, i;
for (b = 0; b < NR_BASES; b++) {
old_base = per_cpu_ptr(&timer_bases[b], cpu);
new_base = get_cpu_ptr(&timer_bases[b]);
/*
* The caller is globally serialized and nobody else
* takes two locks at once, deadlock is not possible.
*/
raw_spin_lock_irq(&new_base->lock);
raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
/*
* The current CPUs base clock might be stale. Update it
* before moving the timers over.
*/
forward_timer_base(new_base);
WARN_ON_ONCE(old_base->running_timer);
old_base->running_timer = NULL;
for (i = 0; i < WHEEL_SIZE; i++)
migrate_timer_list(new_base, old_base->vectors + i);
raw_spin_unlock(&old_base->lock);
raw_spin_unlock_irq(&new_base->lock);
put_cpu_ptr(&timer_bases);
}
return 0;
}
#endif /* CONFIG_HOTPLUG_CPU */
static void __init init_timer_cpu(int cpu)
{
struct timer_base *base;
int i;
for (i = 0; i < NR_BASES; i++) {
base = per_cpu_ptr(&timer_bases[i], cpu);
base->cpu = cpu;
raw_spin_lock_init(&base->lock);
base->clk = jiffies;
base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA;
timer_base_init_expiry_lock(base);
}
}
static void __init init_timer_cpus(void)
{
int cpu;
for_each_possible_cpu(cpu)
init_timer_cpu(cpu);
}
void __init init_timers(void)
{
init_timer_cpus();
posix_cputimers_init_work();
open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
}
/**
* msleep - sleep safely even with waitqueue interruptions
* @msecs: Time in milliseconds to sleep for
*/
void msleep(unsigned int msecs)
{
unsigned long timeout = msecs_to_jiffies(msecs) + 1;
while (timeout)
timeout = schedule_timeout_uninterruptible(timeout);
}
EXPORT_SYMBOL(msleep);
/**
* msleep_interruptible - sleep waiting for signals
* @msecs: Time in milliseconds to sleep for
*/
unsigned long msleep_interruptible(unsigned int msecs)
{
unsigned long timeout = msecs_to_jiffies(msecs) + 1;
while (timeout && !signal_pending(current))
timeout = schedule_timeout_interruptible(timeout);
return jiffies_to_msecs(timeout);
}
EXPORT_SYMBOL(msleep_interruptible);
/**
* usleep_range_state - Sleep for an approximate time in a given state
* @min: Minimum time in usecs to sleep
* @max: Maximum time in usecs to sleep
* @state: State of the current task that will be while sleeping
*
* In non-atomic context where the exact wakeup time is flexible, use
* usleep_range_state() instead of udelay(). The sleep improves responsiveness
* by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces
* power usage by allowing hrtimers to take advantage of an already-
* scheduled interrupt instead of scheduling a new one just for this sleep.
*/
void __sched usleep_range_state(unsigned long min, unsigned long max,
unsigned int state)
{
ktime_t exp = ktime_add_us(ktime_get(), min);
u64 delta = (u64)(max - min) * NSEC_PER_USEC;
for (;;) {
__set_current_state(state);
/* Do not return before the requested sleep time has elapsed */
if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
break;
}
}
EXPORT_SYMBOL(usleep_range_state);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* printk_safe.c - Safe printk for printk-deadlock-prone contexts
*/
#include <linux/preempt.h>
#include <linux/kdb.h>
#include <linux/smp.h>
#include <linux/cpumask.h>
#include <linux/printk.h>
#include <linux/kprobes.h>
#include "internal.h"
static DEFINE_PER_CPU(int, printk_context);
/* Can be preempted by NMI. */
void __printk_safe_enter(void)
{
this_cpu_inc(printk_context);
}
/* Can be preempted by NMI. */
void __printk_safe_exit(void)
{
this_cpu_dec(printk_context);
}
asmlinkage int vprintk(const char *fmt, va_list args)
{
#ifdef CONFIG_KGDB_KDB
/* Allow to pass printk() to kdb but avoid a recursion. */
if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0))
return vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args);
#endif
/*
* Use the main logbuf even in NMI. But avoid calling console
* drivers that might have their own locks.
*/
if (this_cpu_read(printk_context) || in_nmi()) return vprintk_deferred(fmt, args);
/* No obstacles. */
return vprintk_default(fmt, args);
}
EXPORT_SYMBOL(vprintk);
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Generic Timer-queue
*
* Manages a simple queue of timers, ordered by expiration time.
* Uses rbtrees for quick list adds and expiration.
*
* NOTE: All of the following functions need to be serialized
* to avoid races. No locking is done by this library code.
*/
#include <linux/bug.h>
#include <linux/timerqueue.h>
#include <linux/rbtree.h>
#include <linux/export.h>
#define __node_2_tq(_n) \
rb_entry((_n), struct timerqueue_node, node)
static inline bool __timerqueue_less(struct rb_node *a, const struct rb_node *b)
{
return __node_2_tq(a)->expires < __node_2_tq(b)->expires;
}
/**
* timerqueue_add - Adds timer to timerqueue.
*
* @head: head of timerqueue
* @node: timer node to be added
*
* Adds the timer node to the timerqueue, sorted by the node's expires
* value. Returns true if the newly added timer is the first expiring timer in
* the queue.
*/
bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
{
/* Make sure we don't add nodes that are already added */
WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node)); return rb_add_cached(&node->node, &head->rb_root, __timerqueue_less);
}
EXPORT_SYMBOL_GPL(timerqueue_add);
/**
* timerqueue_del - Removes a timer from the timerqueue.
*
* @head: head of timerqueue
* @node: timer node to be removed
*
* Removes the timer node from the timerqueue. Returns true if the queue is
* not empty after the remove.
*/
bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node)
{
WARN_ON_ONCE(RB_EMPTY_NODE(&node->node)); rb_erase_cached(&node->node, &head->rb_root);
RB_CLEAR_NODE(&node->node);
return !RB_EMPTY_ROOT(&head->rb_root.rb_root);
}
EXPORT_SYMBOL_GPL(timerqueue_del);
/**
* timerqueue_iterate_next - Returns the timer after the provided timer
*
* @node: Pointer to a timer.
*
* Provides the timer that is after the given node. This is used, when
* necessary, to iterate through the list of timers in a timer list
* without modifying the list.
*/
struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node)
{
struct rb_node *next;
if (!node)
return NULL;
next = rb_next(&node->node);
if (!next)
return NULL;
return container_of(next, struct timerqueue_node, node);
}
EXPORT_SYMBOL_GPL(timerqueue_iterate_next);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Routines to manage notifier chains for passing status changes to any
* interested routines. We need this instead of hard coded call lists so
* that modules can poke their nose into the innards. The network devices
* needed them so here they are for the rest of you.
*
* Alan Cox <Alan.Cox@linux.org>
*/
#ifndef _LINUX_NOTIFIER_H
#define _LINUX_NOTIFIER_H
#include <linux/errno.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/srcu.h>
/*
* Notifier chains are of four types:
*
* Atomic notifier chains: Chain callbacks run in interrupt/atomic
* context. Callouts are not allowed to block.
* Blocking notifier chains: Chain callbacks run in process context.
* Callouts are allowed to block.
* Raw notifier chains: There are no restrictions on callbacks,
* registration, or unregistration. All locking and protection
* must be provided by the caller.
* SRCU notifier chains: A variant of blocking notifier chains, with
* the same restrictions.
*
* atomic_notifier_chain_register() may be called from an atomic context,
* but blocking_notifier_chain_register() and srcu_notifier_chain_register()
* must be called from a process context. Ditto for the corresponding
* _unregister() routines.
*
* atomic_notifier_chain_unregister(), blocking_notifier_chain_unregister(),
* and srcu_notifier_chain_unregister() _must not_ be called from within
* the call chain.
*
* SRCU notifier chains are an alternative form of blocking notifier chains.
* They use SRCU (Sleepable Read-Copy Update) instead of rw-semaphores for
* protection of the chain links. This means there is _very_ low overhead
* in srcu_notifier_call_chain(): no cache bounces and no memory barriers.
* As compensation, srcu_notifier_chain_unregister() is rather expensive.
* SRCU notifier chains should be used when the chain will be called very
* often but notifier_blocks will seldom be removed.
*/
struct notifier_block;
typedef int (*notifier_fn_t)(struct notifier_block *nb,
unsigned long action, void *data);
struct notifier_block {
notifier_fn_t notifier_call;
struct notifier_block __rcu *next;
int priority;
};
struct atomic_notifier_head {
spinlock_t lock;
struct notifier_block __rcu *head;
};
struct blocking_notifier_head {
struct rw_semaphore rwsem;
struct notifier_block __rcu *head;
};
struct raw_notifier_head {
struct notifier_block __rcu *head;
};
struct srcu_notifier_head {
struct mutex mutex;
struct srcu_usage srcuu;
struct srcu_struct srcu;
struct notifier_block __rcu *head;
};
#define ATOMIC_INIT_NOTIFIER_HEAD(name) do { \
spin_lock_init(&(name)->lock); \
(name)->head = NULL; \
} while (0)
#define BLOCKING_INIT_NOTIFIER_HEAD(name) do { \
init_rwsem(&(name)->rwsem); \
(name)->head = NULL; \
} while (0)
#define RAW_INIT_NOTIFIER_HEAD(name) do { \
(name)->head = NULL; \
} while (0)
/* srcu_notifier_heads must be cleaned up dynamically */
extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
#define srcu_cleanup_notifier_head(name) \
cleanup_srcu_struct(&(name)->srcu);
#define ATOMIC_NOTIFIER_INIT(name) { \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.head = NULL }
#define BLOCKING_NOTIFIER_INIT(name) { \
.rwsem = __RWSEM_INITIALIZER((name).rwsem), \
.head = NULL }
#define RAW_NOTIFIER_INIT(name) { \
.head = NULL }
#define SRCU_NOTIFIER_INIT(name, pcpu) \
{ \
.mutex = __MUTEX_INITIALIZER(name.mutex), \
.head = NULL, \
.srcuu = __SRCU_USAGE_INIT(name.srcuu), \
.srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \
}
#define ATOMIC_NOTIFIER_HEAD(name) \
struct atomic_notifier_head name = \
ATOMIC_NOTIFIER_INIT(name)
#define BLOCKING_NOTIFIER_HEAD(name) \
struct blocking_notifier_head name = \
BLOCKING_NOTIFIER_INIT(name)
#define RAW_NOTIFIER_HEAD(name) \
struct raw_notifier_head name = \
RAW_NOTIFIER_INIT(name)
#ifdef CONFIG_TREE_SRCU
#define _SRCU_NOTIFIER_HEAD(name, mod) \
static DEFINE_PER_CPU(struct srcu_data, name##_head_srcu_data); \
mod struct srcu_notifier_head name = \
SRCU_NOTIFIER_INIT(name, name##_head_srcu_data)
#else
#define _SRCU_NOTIFIER_HEAD(name, mod) \
mod struct srcu_notifier_head name = \
SRCU_NOTIFIER_INIT(name, name)
#endif
#define SRCU_NOTIFIER_HEAD(name) \
_SRCU_NOTIFIER_HEAD(name, /* not static */)
#define SRCU_NOTIFIER_HEAD_STATIC(name) \
_SRCU_NOTIFIER_HEAD(name, static)
#ifdef __KERNEL__
extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
struct notifier_block *nb);
extern int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
struct notifier_block *nb);
extern int raw_notifier_chain_register(struct raw_notifier_head *nh,
struct notifier_block *nb);
extern int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
struct notifier_block *nb);
extern int atomic_notifier_chain_register_unique_prio(
struct atomic_notifier_head *nh, struct notifier_block *nb);
extern int blocking_notifier_chain_register_unique_prio(
struct blocking_notifier_head *nh, struct notifier_block *nb);
extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
struct notifier_block *nb);
extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
struct notifier_block *nb);
extern int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
struct notifier_block *nb);
extern int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
struct notifier_block *nb);
extern int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
unsigned long val, void *v);
extern int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
unsigned long val, void *v);
extern int raw_notifier_call_chain(struct raw_notifier_head *nh,
unsigned long val, void *v);
extern int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
unsigned long val, void *v);
extern int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
unsigned long val_up, unsigned long val_down, void *v);
extern int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
unsigned long val_up, unsigned long val_down, void *v);
extern bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh);
#define NOTIFY_DONE 0x0000 /* Don't care */
#define NOTIFY_OK 0x0001 /* Suits me */
#define NOTIFY_STOP_MASK 0x8000 /* Don't call further */
#define NOTIFY_BAD (NOTIFY_STOP_MASK|0x0002)
/* Bad/Veto action */
/*
* Clean way to return from the notifier and stop further calls.
*/
#define NOTIFY_STOP (NOTIFY_OK|NOTIFY_STOP_MASK)
/* Encapsulate (negative) errno value (in particular, NOTIFY_BAD <=> EPERM). */
static inline int notifier_from_errno(int err)
{
if (err)
return NOTIFY_STOP_MASK | (NOTIFY_OK - err);
return NOTIFY_OK;
}
/* Restore (negative) errno value from notify return value. */
static inline int notifier_to_errno(int ret)
{
ret &= ~NOTIFY_STOP_MASK;
return ret > NOTIFY_OK ? NOTIFY_OK - ret : 0;
}
/*
* Declared notifiers so far. I can imagine quite a few more chains
* over time (eg laptop power reset chains, reboot chain (to clean
* device units up), device [un]mount chain, module load/unload chain,
* low memory chain, screenblank chain (for plug in modular screenblankers)
* VC switch chains (for loadable kernel svgalib VC switch helpers) etc...
*/
/* CPU notfiers are defined in include/linux/cpu.h. */
/* netdevice notifiers are defined in include/linux/netdevice.h */
/* reboot notifiers are defined in include/linux/reboot.h. */
/* Hibernation and suspend events are defined in include/linux/suspend.h. */
/* Virtual Terminal events are defined in include/linux/vt.h. */
#define NETLINK_URELEASE 0x0001 /* Unicast netlink socket released */
/* Console keyboard events.
* Note: KBD_KEYCODE is always sent before KBD_UNBOUND_KEYCODE, KBD_UNICODE and
* KBD_KEYSYM. */
#define KBD_KEYCODE 0x0001 /* Keyboard keycode, called before any other */
#define KBD_UNBOUND_KEYCODE 0x0002 /* Keyboard keycode which is not bound to any other */
#define KBD_UNICODE 0x0003 /* Keyboard unicode */
#define KBD_KEYSYM 0x0004 /* Keyboard keysym */
#define KBD_POST_KEYSYM 0x0005 /* Called after keyboard keysym interpretation */
extern struct blocking_notifier_head reboot_notifier_list;
#endif /* __KERNEL__ */
#endif /* _LINUX_NOTIFIER_H */
// SPDX-License-Identifier: GPL-2.0-only
/* Kernel thread helper functions.
* Copyright (C) 2004 IBM Corporation, Rusty Russell.
* Copyright (C) 2009 Red Hat, Inc.
*
* Creation is done via kthreadd, so that we get a clean environment
* even if we're invoked from userspace (think modprobe, hotplug cpu,
* etc.).
*/
#include <uapi/linux/sched/types.h>
#include <linux/mm.h>
#include <linux/mmu_context.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/kthread.h>
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/cgroup.h>
#include <linux/cpuset.h>
#include <linux/unistd.h>
#include <linux/file.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/freezer.h>
#include <linux/ptrace.h>
#include <linux/uaccess.h>
#include <linux/numa.h>
#include <linux/sched/isolation.h>
#include <trace/events/sched.h>
static DEFINE_SPINLOCK(kthread_create_lock);
static LIST_HEAD(kthread_create_list);
struct task_struct *kthreadd_task;
struct kthread_create_info
{
/* Information passed to kthread() from kthreadd. */
char *full_name;
int (*threadfn)(void *data);
void *data;
int node;
/* Result passed back to kthread_create() from kthreadd. */
struct task_struct *result;
struct completion *done;
struct list_head list;
};
struct kthread {
unsigned long flags;
unsigned int cpu;
int result;
int (*threadfn)(void *);
void *data;
struct completion parked;
struct completion exited;
#ifdef CONFIG_BLK_CGROUP
struct cgroup_subsys_state *blkcg_css;
#endif
/* To store the full name if task comm is truncated. */
char *full_name;
};
enum KTHREAD_BITS {
KTHREAD_IS_PER_CPU = 0,
KTHREAD_SHOULD_STOP,
KTHREAD_SHOULD_PARK,
};
static inline struct kthread *to_kthread(struct task_struct *k)
{
WARN_ON(!(k->flags & PF_KTHREAD)); return k->worker_private;
}
/*
* Variant of to_kthread() that doesn't assume @p is a kthread.
*
* Per construction; when:
*
* (p->flags & PF_KTHREAD) && p->worker_private
*
* the task is both a kthread and struct kthread is persistent. However
* PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and
* begin_new_exec()).
*/
static inline struct kthread *__to_kthread(struct task_struct *p)
{
void *kthread = p->worker_private; if (kthread && !(p->flags & PF_KTHREAD))
kthread = NULL;
return kthread;
}
void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk)
{
struct kthread *kthread = to_kthread(tsk);
if (!kthread || !kthread->full_name) {
__get_task_comm(buf, buf_size, tsk);
return;
}
strscpy_pad(buf, kthread->full_name, buf_size);
}
bool set_kthread_struct(struct task_struct *p)
{
struct kthread *kthread;
if (WARN_ON_ONCE(to_kthread(p)))
return false;
kthread = kzalloc(sizeof(*kthread), GFP_KERNEL);
if (!kthread)
return false;
init_completion(&kthread->exited);
init_completion(&kthread->parked);
p->vfork_done = &kthread->exited;
p->worker_private = kthread;
return true;
}
void free_kthread_struct(struct task_struct *k)
{
struct kthread *kthread;
/*
* Can be NULL if kmalloc() in set_kthread_struct() failed.
*/
kthread = to_kthread(k);
if (!kthread)
return;
#ifdef CONFIG_BLK_CGROUP
WARN_ON_ONCE(kthread->blkcg_css);
#endif
k->worker_private = NULL;
kfree(kthread->full_name);
kfree(kthread);
}
/**
* kthread_should_stop - should this kthread return now?
*
* When someone calls kthread_stop() on your kthread, it will be woken
* and this will return true. You should then return, and your return
* value will be passed through to kthread_stop().
*/
bool kthread_should_stop(void)
{
return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
}
EXPORT_SYMBOL(kthread_should_stop);
static bool __kthread_should_park(struct task_struct *k)
{
return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
}
/**
* kthread_should_park - should this kthread park now?
*
* When someone calls kthread_park() on your kthread, it will be woken
* and this will return true. You should then do the necessary
* cleanup and call kthread_parkme()
*
* Similar to kthread_should_stop(), but this keeps the thread alive
* and in a park position. kthread_unpark() "restarts" the thread and
* calls the thread function again.
*/
bool kthread_should_park(void)
{
return __kthread_should_park(current);
}
EXPORT_SYMBOL_GPL(kthread_should_park);
bool kthread_should_stop_or_park(void)
{
struct kthread *kthread = __to_kthread(current);
if (!kthread)
return false;
return kthread->flags & (BIT(KTHREAD_SHOULD_STOP) | BIT(KTHREAD_SHOULD_PARK));
}
/**
* kthread_freezable_should_stop - should this freezable kthread return now?
* @was_frozen: optional out parameter, indicates whether %current was frozen
*
* kthread_should_stop() for freezable kthreads, which will enter
* refrigerator if necessary. This function is safe from kthread_stop() /
* freezer deadlock and freezable kthreads should use this function instead
* of calling try_to_freeze() directly.
*/
bool kthread_freezable_should_stop(bool *was_frozen)
{
bool frozen = false;
might_sleep();
if (unlikely(freezing(current)))
frozen = __refrigerator(true);
if (was_frozen)
*was_frozen = frozen;
return kthread_should_stop();
}
EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
/**
* kthread_func - return the function specified on kthread creation
* @task: kthread task in question
*
* Returns NULL if the task is not a kthread.
*/
void *kthread_func(struct task_struct *task)
{
struct kthread *kthread = __to_kthread(task);
if (kthread)
return kthread->threadfn;
return NULL;
}
EXPORT_SYMBOL_GPL(kthread_func);
/**
* kthread_data - return data value specified on kthread creation
* @task: kthread task in question
*
* Return the data value specified when kthread @task was created.
* The caller is responsible for ensuring the validity of @task when
* calling this function.
*/
void *kthread_data(struct task_struct *task)
{
return to_kthread(task)->data;
}
EXPORT_SYMBOL_GPL(kthread_data);
/**
* kthread_probe_data - speculative version of kthread_data()
* @task: possible kthread task in question
*
* @task could be a kthread task. Return the data value specified when it
* was created if accessible. If @task isn't a kthread task or its data is
* inaccessible for any reason, %NULL is returned. This function requires
* that @task itself is safe to dereference.
*/
void *kthread_probe_data(struct task_struct *task)
{
struct kthread *kthread = __to_kthread(task);
void *data = NULL;
if (kthread)
copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));
return data;
}
static void __kthread_parkme(struct kthread *self)
{
for (;;) {
/*
* TASK_PARKED is a special state; we must serialize against
* possible pending wakeups to avoid store-store collisions on
* task->state.
*
* Such a collision might possibly result in the task state
* changin from TASK_PARKED and us failing the
* wait_task_inactive() in kthread_park().
*/
set_special_state(TASK_PARKED);
if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
break;
/*
* Thread is going to call schedule(), do not preempt it,
* or the caller of kthread_park() may spend more time in
* wait_task_inactive().
*/
preempt_disable();
complete(&self->parked);
schedule_preempt_disabled();
preempt_enable();
}
__set_current_state(TASK_RUNNING);
}
void kthread_parkme(void)
{
__kthread_parkme(to_kthread(current));
}
EXPORT_SYMBOL_GPL(kthread_parkme);
/**
* kthread_exit - Cause the current kthread return @result to kthread_stop().
* @result: The integer value to return to kthread_stop().
*
* While kthread_exit can be called directly, it exists so that
* functions which do some additional work in non-modular code such as
* module_put_and_kthread_exit can be implemented.
*
* Does not return.
*/
void __noreturn kthread_exit(long result)
{
struct kthread *kthread = to_kthread(current);
kthread->result = result;
do_exit(0);
}
/**
* kthread_complete_and_exit - Exit the current kthread.
* @comp: Completion to complete
* @code: The integer value to return to kthread_stop().
*
* If present, complete @comp and then return code to kthread_stop().
*
* A kernel thread whose module may be removed after the completion of
* @comp can use this function to exit safely.
*
* Does not return.
*/
void __noreturn kthread_complete_and_exit(struct completion *comp, long code)
{
if (comp)
complete(comp);
kthread_exit(code);
}
EXPORT_SYMBOL(kthread_complete_and_exit);
static int kthread(void *_create)
{
static const struct sched_param param = { .sched_priority = 0 };
/* Copy data: it's on kthread's stack */
struct kthread_create_info *create = _create;
int (*threadfn)(void *data) = create->threadfn;
void *data = create->data;
struct completion *done;
struct kthread *self;
int ret;
self = to_kthread(current);
/* Release the structure when caller killed by a fatal signal. */
done = xchg(&create->done, NULL);
if (!done) {
kfree(create->full_name);
kfree(create);
kthread_exit(-EINTR);
}
self->full_name = create->full_name;
self->threadfn = threadfn;
self->data = data;
/*
* The new thread inherited kthreadd's priority and CPU mask. Reset
* back to default in case they have been changed.
*/
sched_setscheduler_nocheck(current, SCHED_NORMAL, ¶m);
set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_KTHREAD));
/* OK, tell user we're spawned, wait for stop or wakeup */
__set_current_state(TASK_UNINTERRUPTIBLE);
create->result = current;
/*
* Thread is going to call schedule(), do not preempt it,
* or the creator may spend more time in wait_task_inactive().
*/
preempt_disable();
complete(done);
schedule_preempt_disabled();
preempt_enable();
ret = -EINTR;
if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
cgroup_kthread_ready();
__kthread_parkme(self);
ret = threadfn(data);
}
kthread_exit(ret);
}
/* called from kernel_clone() to get node information for about to be created task */
int tsk_fork_get_node(struct task_struct *tsk)
{
#ifdef CONFIG_NUMA
if (tsk == kthreadd_task)
return tsk->pref_node_fork;
#endif
return NUMA_NO_NODE;
}
static void create_kthread(struct kthread_create_info *create)
{
int pid;
#ifdef CONFIG_NUMA
current->pref_node_fork = create->node;
#endif
/* We want our own signal handler (we take no signals by default). */
pid = kernel_thread(kthread, create, create->full_name,
CLONE_FS | CLONE_FILES | SIGCHLD);
if (pid < 0) {
/* Release the structure when caller killed by a fatal signal. */
struct completion *done = xchg(&create->done, NULL);
kfree(create->full_name);
if (!done) {
kfree(create);
return;
}
create->result = ERR_PTR(pid);
complete(done);
}
}
static __printf(4, 0)
struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
void *data, int node,
const char namefmt[],
va_list args)
{
DECLARE_COMPLETION_ONSTACK(done);
struct task_struct *task;
struct kthread_create_info *create = kmalloc(sizeof(*create),
GFP_KERNEL);
if (!create)
return ERR_PTR(-ENOMEM);
create->threadfn = threadfn;
create->data = data;
create->node = node;
create->done = &done;
create->full_name = kvasprintf(GFP_KERNEL, namefmt, args);
if (!create->full_name) {
task = ERR_PTR(-ENOMEM);
goto free_create;
}
spin_lock(&kthread_create_lock);
list_add_tail(&create->list, &kthread_create_list);
spin_unlock(&kthread_create_lock);
wake_up_process(kthreadd_task);
/*
* Wait for completion in killable state, for I might be chosen by
* the OOM killer while kthreadd is trying to allocate memory for
* new kernel thread.
*/
if (unlikely(wait_for_completion_killable(&done))) {
/*
* If I was killed by a fatal signal before kthreadd (or new
* kernel thread) calls complete(), leave the cleanup of this
* structure to that thread.
*/
if (xchg(&create->done, NULL))
return ERR_PTR(-EINTR);
/*
* kthreadd (or new kernel thread) will call complete()
* shortly.
*/
wait_for_completion(&done);
}
task = create->result;
free_create:
kfree(create);
return task;
}
/**
* kthread_create_on_node - create a kthread.
* @threadfn: the function to run until signal_pending(current).
* @data: data ptr for @threadfn.
* @node: task and thread structures for the thread are allocated on this node
* @namefmt: printf-style name for the thread.
*
* Description: This helper function creates and names a kernel
* thread. The thread will be stopped: use wake_up_process() to start
* it. See also kthread_run(). The new thread has SCHED_NORMAL policy and
* is affine to all CPUs.
*
* If thread is going to be bound on a particular cpu, give its node
* in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
* When woken, the thread will run @threadfn() with @data as its
* argument. @threadfn() can either return directly if it is a
* standalone thread for which no one will call kthread_stop(), or
* return when 'kthread_should_stop()' is true (which means
* kthread_stop() has been called). The return value should be zero
* or a negative error number; it will be passed to kthread_stop().
*
* Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
*/
struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
void *data, int node,
const char namefmt[],
...)
{
struct task_struct *task;
va_list args;
va_start(args, namefmt);
task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
va_end(args);
return task;
}
EXPORT_SYMBOL(kthread_create_on_node);
static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, unsigned int state)
{
unsigned long flags;
if (!wait_task_inactive(p, state)) {
WARN_ON(1);
return;
}
/* It's safe because the task is inactive. */
raw_spin_lock_irqsave(&p->pi_lock, flags);
do_set_cpus_allowed(p, mask);
p->flags |= PF_NO_SETAFFINITY;
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
}
static void __kthread_bind(struct task_struct *p, unsigned int cpu, unsigned int state)
{
__kthread_bind_mask(p, cpumask_of(cpu), state);
}
void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
{
__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
}
/**
* kthread_bind - bind a just-created kthread to a cpu.
* @p: thread created by kthread_create().
* @cpu: cpu (might not be online, must be possible) for @k to run on.
*
* Description: This function is equivalent to set_cpus_allowed(),
* except that @cpu doesn't need to be online, and the thread must be
* stopped (i.e., just returned from kthread_create()).
*/
void kthread_bind(struct task_struct *p, unsigned int cpu)
{
__kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(kthread_bind);
/**
* kthread_create_on_cpu - Create a cpu bound kthread
* @threadfn: the function to run until signal_pending(current).
* @data: data ptr for @threadfn.
* @cpu: The cpu on which the thread should be bound,
* @namefmt: printf-style name for the thread. Format is restricted
* to "name.*%u". Code fills in cpu number.
*
* Description: This helper function creates and names a kernel thread
*/
struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
void *data, unsigned int cpu,
const char *namefmt)
{
struct task_struct *p;
p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
cpu);
if (IS_ERR(p))
return p;
kthread_bind(p, cpu);
/* CPU hotplug need to bind once again when unparking the thread. */
to_kthread(p)->cpu = cpu;
return p;
}
EXPORT_SYMBOL(kthread_create_on_cpu);
void kthread_set_per_cpu(struct task_struct *k, int cpu)
{
struct kthread *kthread = to_kthread(k);
if (!kthread)
return;
WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY));
if (cpu < 0) {
clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
return;
}
kthread->cpu = cpu;
set_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
}
bool kthread_is_per_cpu(struct task_struct *p)
{
struct kthread *kthread = __to_kthread(p);
if (!kthread)
return false;
return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags);}
/**
* kthread_unpark - unpark a thread created by kthread_create().
* @k: thread created by kthread_create().
*
* Sets kthread_should_park() for @k to return false, wakes it, and
* waits for it to return. If the thread is marked percpu then its
* bound to the cpu again.
*/
void kthread_unpark(struct task_struct *k)
{
struct kthread *kthread = to_kthread(k);
/*
* Newly created kthread was parked when the CPU was offline.
* The binding was lost and we need to set it again.
*/
if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
__kthread_bind(k, kthread->cpu, TASK_PARKED); clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
/*
* __kthread_parkme() will either see !SHOULD_PARK or get the wakeup.
*/
wake_up_state(k, TASK_PARKED);
}
EXPORT_SYMBOL_GPL(kthread_unpark);
/**
* kthread_park - park a thread created by kthread_create().
* @k: thread created by kthread_create().
*
* Sets kthread_should_park() for @k to return true, wakes it, and
* waits for it to return. This can also be called after kthread_create()
* instead of calling wake_up_process(): the thread will park without
* calling threadfn().
*
* Returns 0 if the thread is parked, -ENOSYS if the thread exited.
* If called by the kthread itself just the park bit is set.
*/
int kthread_park(struct task_struct *k)
{
struct kthread *kthread = to_kthread(k);
if (WARN_ON(k->flags & PF_EXITING))
return -ENOSYS;
if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)))
return -EBUSY;
set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
if (k != current) {
wake_up_process(k);
/*
* Wait for __kthread_parkme() to complete(), this means we
* _will_ have TASK_PARKED and are about to call schedule().
*/
wait_for_completion(&kthread->parked);
/*
* Now wait for that schedule() to complete and the task to
* get scheduled out.
*/
WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED));
}
return 0;
}
EXPORT_SYMBOL_GPL(kthread_park);
/**
* kthread_stop - stop a thread created by kthread_create().
* @k: thread created by kthread_create().
*
* Sets kthread_should_stop() for @k to return true, wakes it, and
* waits for it to exit. This can also be called after kthread_create()
* instead of calling wake_up_process(): the thread will exit without
* calling threadfn().
*
* If threadfn() may call kthread_exit() itself, the caller must ensure
* task_struct can't go away.
*
* Returns the result of threadfn(), or %-EINTR if wake_up_process()
* was never called.
*/
int kthread_stop(struct task_struct *k)
{
struct kthread *kthread;
int ret;
trace_sched_kthread_stop(k); get_task_struct(k); kthread = to_kthread(k);
set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
kthread_unpark(k);
set_tsk_thread_flag(k, TIF_NOTIFY_SIGNAL);
wake_up_process(k);
wait_for_completion(&kthread->exited);
ret = kthread->result;
put_task_struct(k); trace_sched_kthread_stop_ret(ret); return ret;
}
EXPORT_SYMBOL(kthread_stop);
/**
* kthread_stop_put - stop a thread and put its task struct
* @k: thread created by kthread_create().
*
* Stops a thread created by kthread_create() and put its task_struct.
* Only use when holding an extra task struct reference obtained by
* calling get_task_struct().
*/
int kthread_stop_put(struct task_struct *k)
{
int ret;
ret = kthread_stop(k);
put_task_struct(k);
return ret;
}
EXPORT_SYMBOL(kthread_stop_put);
int kthreadd(void *unused)
{
struct task_struct *tsk = current;
/* Setup a clean context for our children to inherit. */
set_task_comm(tsk, "kthreadd");
ignore_signals(tsk);
set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD));
set_mems_allowed(node_states[N_MEMORY]);
current->flags |= PF_NOFREEZE;
cgroup_init_kthreadd();
for (;;) {
set_current_state(TASK_INTERRUPTIBLE);
if (list_empty(&kthread_create_list))
schedule();
__set_current_state(TASK_RUNNING);
spin_lock(&kthread_create_lock);
while (!list_empty(&kthread_create_list)) {
struct kthread_create_info *create;
create = list_entry(kthread_create_list.next,
struct kthread_create_info, list);
list_del_init(&create->list);
spin_unlock(&kthread_create_lock);
create_kthread(create);
spin_lock(&kthread_create_lock);
}
spin_unlock(&kthread_create_lock);
}
return 0;
}
void __kthread_init_worker(struct kthread_worker *worker,
const char *name,
struct lock_class_key *key)
{
memset(worker, 0, sizeof(struct kthread_worker));
raw_spin_lock_init(&worker->lock);
lockdep_set_class_and_name(&worker->lock, key, name);
INIT_LIST_HEAD(&worker->work_list);
INIT_LIST_HEAD(&worker->delayed_work_list);
}
EXPORT_SYMBOL_GPL(__kthread_init_worker);
/**
* kthread_worker_fn - kthread function to process kthread_worker
* @worker_ptr: pointer to initialized kthread_worker
*
* This function implements the main cycle of kthread worker. It processes
* work_list until it is stopped with kthread_stop(). It sleeps when the queue
* is empty.
*
* The works are not allowed to keep any locks, disable preemption or interrupts
* when they finish. There is defined a safe point for freezing when one work
* finishes and before a new one is started.
*
* Also the works must not be handled by more than one worker at the same time,
* see also kthread_queue_work().
*/
int kthread_worker_fn(void *worker_ptr)
{
struct kthread_worker *worker = worker_ptr;
struct kthread_work *work;
/*
* FIXME: Update the check and remove the assignment when all kthread
* worker users are created using kthread_create_worker*() functions.
*/
WARN_ON(worker->task && worker->task != current);
worker->task = current;
if (worker->flags & KTW_FREEZABLE)
set_freezable();
repeat:
set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */
if (kthread_should_stop()) {
__set_current_state(TASK_RUNNING);
raw_spin_lock_irq(&worker->lock);
worker->task = NULL;
raw_spin_unlock_irq(&worker->lock);
return 0;
}
work = NULL;
raw_spin_lock_irq(&worker->lock);
if (!list_empty(&worker->work_list)) {
work = list_first_entry(&worker->work_list,
struct kthread_work, node);
list_del_init(&work->node);
}
worker->current_work = work;
raw_spin_unlock_irq(&worker->lock);
if (work) {
kthread_work_func_t func = work->func;
__set_current_state(TASK_RUNNING);
trace_sched_kthread_work_execute_start(work);
work->func(work);
/*
* Avoid dereferencing work after this point. The trace
* event only cares about the address.
*/
trace_sched_kthread_work_execute_end(work, func);
} else if (!freezing(current))
schedule();
try_to_freeze();
cond_resched();
goto repeat;
}
EXPORT_SYMBOL_GPL(kthread_worker_fn);
static __printf(3, 0) struct kthread_worker *
__kthread_create_worker(int cpu, unsigned int flags,
const char namefmt[], va_list args)
{
struct kthread_worker *worker;
struct task_struct *task;
int node = NUMA_NO_NODE;
worker = kzalloc(sizeof(*worker), GFP_KERNEL);
if (!worker)
return ERR_PTR(-ENOMEM);
kthread_init_worker(worker);
if (cpu >= 0)
node = cpu_to_node(cpu);
task = __kthread_create_on_node(kthread_worker_fn, worker,
node, namefmt, args);
if (IS_ERR(task))
goto fail_task;
if (cpu >= 0)
kthread_bind(task, cpu);
worker->flags = flags;
worker->task = task;
wake_up_process(task);
return worker;
fail_task:
kfree(worker);
return ERR_CAST(task);
}
/**
* kthread_create_worker - create a kthread worker
* @flags: flags modifying the default behavior of the worker
* @namefmt: printf-style name for the kthread worker (task).
*
* Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
* when the needed structures could not get allocated, and ERR_PTR(-EINTR)
* when the caller was killed by a fatal signal.
*/
struct kthread_worker *
kthread_create_worker(unsigned int flags, const char namefmt[], ...)
{
struct kthread_worker *worker;
va_list args;
va_start(args, namefmt);
worker = __kthread_create_worker(-1, flags, namefmt, args);
va_end(args);
return worker;
}
EXPORT_SYMBOL(kthread_create_worker);
/**
* kthread_create_worker_on_cpu - create a kthread worker and bind it
* to a given CPU and the associated NUMA node.
* @cpu: CPU number
* @flags: flags modifying the default behavior of the worker
* @namefmt: printf-style name for the kthread worker (task).
*
* Use a valid CPU number if you want to bind the kthread worker
* to the given CPU and the associated NUMA node.
*
* A good practice is to add the cpu number also into the worker name.
* For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
*
* CPU hotplug:
* The kthread worker API is simple and generic. It just provides a way
* to create, use, and destroy workers.
*
* It is up to the API user how to handle CPU hotplug. They have to decide
* how to handle pending work items, prevent queuing new ones, and
* restore the functionality when the CPU goes off and on. There are a
* few catches:
*
* - CPU affinity gets lost when it is scheduled on an offline CPU.
*
* - The worker might not exist when the CPU was off when the user
* created the workers.
*
* Good practice is to implement two CPU hotplug callbacks and to
* destroy/create the worker when the CPU goes down/up.
*
* Return:
* The pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
* when the needed structures could not get allocated, and ERR_PTR(-EINTR)
* when the caller was killed by a fatal signal.
*/
struct kthread_worker *
kthread_create_worker_on_cpu(int cpu, unsigned int flags,
const char namefmt[], ...)
{
struct kthread_worker *worker;
va_list args;
va_start(args, namefmt);
worker = __kthread_create_worker(cpu, flags, namefmt, args);
va_end(args);
return worker;
}
EXPORT_SYMBOL(kthread_create_worker_on_cpu);
/*
* Returns true when the work could not be queued at the moment.
* It happens when it is already pending in a worker list
* or when it is being cancelled.
*/
static inline bool queuing_blocked(struct kthread_worker *worker,
struct kthread_work *work)
{
lockdep_assert_held(&worker->lock); return !list_empty(&work->node) || work->canceling;
}
static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
struct kthread_work *work)
{
lockdep_assert_held(&worker->lock); WARN_ON_ONCE(!list_empty(&work->node));
/* Do not use a work with >1 worker, see kthread_queue_work() */
WARN_ON_ONCE(work->worker && work->worker != worker);
}
/* insert @work before @pos in @worker */
static void kthread_insert_work(struct kthread_worker *worker,
struct kthread_work *work,
struct list_head *pos)
{
kthread_insert_work_sanity_check(worker, work); trace_sched_kthread_work_queue_work(worker, work); list_add_tail(&work->node, pos); work->worker = worker;
if (!worker->current_work && likely(worker->task))
wake_up_process(worker->task);
}
/**
* kthread_queue_work - queue a kthread_work
* @worker: target kthread_worker
* @work: kthread_work to queue
*
* Queue @work to work processor @task for async execution. @task
* must have been created with kthread_worker_create(). Returns %true
* if @work was successfully queued, %false if it was already pending.
*
* Reinitialize the work if it needs to be used by another worker.
* For example, when the worker was stopped and started again.
*/
bool kthread_queue_work(struct kthread_worker *worker,
struct kthread_work *work)
{
bool ret = false;
unsigned long flags;
raw_spin_lock_irqsave(&worker->lock, flags); if (!queuing_blocked(worker, work)) { kthread_insert_work(worker, work, &worker->work_list);
ret = true;
}
raw_spin_unlock_irqrestore(&worker->lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(kthread_queue_work);
/**
* kthread_delayed_work_timer_fn - callback that queues the associated kthread
* delayed work when the timer expires.
* @t: pointer to the expired timer
*
* The format of the function is defined by struct timer_list.
* It should have been called from irqsafe timer with irq already off.
*/
void kthread_delayed_work_timer_fn(struct timer_list *t)
{
struct kthread_delayed_work *dwork = from_timer(dwork, t, timer);
struct kthread_work *work = &dwork->work;
struct kthread_worker *worker = work->worker;
unsigned long flags;
/*
* This might happen when a pending work is reinitialized.
* It means that it is used a wrong way.
*/
if (WARN_ON_ONCE(!worker))
return;
raw_spin_lock_irqsave(&worker->lock, flags);
/* Work must not be used with >1 worker, see kthread_queue_work(). */
WARN_ON_ONCE(work->worker != worker);
/* Move the work from worker->delayed_work_list. */
WARN_ON_ONCE(list_empty(&work->node));
list_del_init(&work->node);
if (!work->canceling)
kthread_insert_work(worker, work, &worker->work_list);
raw_spin_unlock_irqrestore(&worker->lock, flags);
}
EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
static void __kthread_queue_delayed_work(struct kthread_worker *worker,
struct kthread_delayed_work *dwork,
unsigned long delay)
{
struct timer_list *timer = &dwork->timer;
struct kthread_work *work = &dwork->work;
WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
/*
* If @delay is 0, queue @dwork->work immediately. This is for
* both optimization and correctness. The earliest @timer can
* expire is on the closest next tick and delayed_work users depend
* on that there's no such delay when @delay is 0.
*/
if (!delay) {
kthread_insert_work(worker, work, &worker->work_list);
return;
}
/* Be paranoid and try to detect possible races already now. */
kthread_insert_work_sanity_check(worker, work);
list_add(&work->node, &worker->delayed_work_list);
work->worker = worker;
timer->expires = jiffies + delay;
add_timer(timer);
}
/**
* kthread_queue_delayed_work - queue the associated kthread work
* after a delay.
* @worker: target kthread_worker
* @dwork: kthread_delayed_work to queue
* @delay: number of jiffies to wait before queuing
*
* If the work has not been pending it starts a timer that will queue
* the work after the given @delay. If @delay is zero, it queues the
* work immediately.
*
* Return: %false if the @work has already been pending. It means that
* either the timer was running or the work was queued. It returns %true
* otherwise.
*/
bool kthread_queue_delayed_work(struct kthread_worker *worker,
struct kthread_delayed_work *dwork,
unsigned long delay)
{
struct kthread_work *work = &dwork->work;
unsigned long flags;
bool ret = false;
raw_spin_lock_irqsave(&worker->lock, flags);
if (!queuing_blocked(worker, work)) {
__kthread_queue_delayed_work(worker, dwork, delay);
ret = true;
}
raw_spin_unlock_irqrestore(&worker->lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
struct kthread_flush_work {
struct kthread_work work;
struct completion done;
};
static void kthread_flush_work_fn(struct kthread_work *work)
{
struct kthread_flush_work *fwork =
container_of(work, struct kthread_flush_work, work);
complete(&fwork->done);
}
/**
* kthread_flush_work - flush a kthread_work
* @work: work to flush
*
* If @work is queued or executing, wait for it to finish execution.
*/
void kthread_flush_work(struct kthread_work *work)
{
struct kthread_flush_work fwork = {
KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
COMPLETION_INITIALIZER_ONSTACK(fwork.done),
};
struct kthread_worker *worker;
bool noop = false;
worker = work->worker;
if (!worker)
return;
raw_spin_lock_irq(&worker->lock);
/* Work must not be used with >1 worker, see kthread_queue_work(). */
WARN_ON_ONCE(work->worker != worker);
if (!list_empty(&work->node))
kthread_insert_work(worker, &fwork.work, work->node.next);
else if (worker->current_work == work)
kthread_insert_work(worker, &fwork.work,
worker->work_list.next);
else
noop = true;
raw_spin_unlock_irq(&worker->lock);
if (!noop)
wait_for_completion(&fwork.done);
}
EXPORT_SYMBOL_GPL(kthread_flush_work);
/*
* Make sure that the timer is neither set nor running and could
* not manipulate the work list_head any longer.
*
* The function is called under worker->lock. The lock is temporary
* released but the timer can't be set again in the meantime.
*/
static void kthread_cancel_delayed_work_timer(struct kthread_work *work,
unsigned long *flags)
{
struct kthread_delayed_work *dwork =
container_of(work, struct kthread_delayed_work, work);
struct kthread_worker *worker = work->worker;
/*
* del_timer_sync() must be called to make sure that the timer
* callback is not running. The lock must be temporary released
* to avoid a deadlock with the callback. In the meantime,
* any queuing is blocked by setting the canceling counter.
*/
work->canceling++;
raw_spin_unlock_irqrestore(&worker->lock, *flags);
del_timer_sync(&dwork->timer);
raw_spin_lock_irqsave(&worker->lock, *flags);
work->canceling--;
}
/*
* This function removes the work from the worker queue.
*
* It is called under worker->lock. The caller must make sure that
* the timer used by delayed work is not running, e.g. by calling
* kthread_cancel_delayed_work_timer().
*
* The work might still be in use when this function finishes. See the
* current_work proceed by the worker.
*
* Return: %true if @work was pending and successfully canceled,
* %false if @work was not pending
*/
static bool __kthread_cancel_work(struct kthread_work *work)
{
/*
* Try to remove the work from a worker list. It might either
* be from worker->work_list or from worker->delayed_work_list.
*/
if (!list_empty(&work->node)) {
list_del_init(&work->node);
return true;
}
return false;
}
/**
* kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
* @worker: kthread worker to use
* @dwork: kthread delayed work to queue
* @delay: number of jiffies to wait before queuing
*
* If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
* modify @dwork's timer so that it expires after @delay. If @delay is zero,
* @work is guaranteed to be queued immediately.
*
* Return: %false if @dwork was idle and queued, %true otherwise.
*
* A special case is when the work is being canceled in parallel.
* It might be caused either by the real kthread_cancel_delayed_work_sync()
* or yet another kthread_mod_delayed_work() call. We let the other command
* win and return %true here. The return value can be used for reference
* counting and the number of queued works stays the same. Anyway, the caller
* is supposed to synchronize these operations a reasonable way.
*
* This function is safe to call from any context including IRQ handler.
* See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
* for details.
*/
bool kthread_mod_delayed_work(struct kthread_worker *worker,
struct kthread_delayed_work *dwork,
unsigned long delay)
{
struct kthread_work *work = &dwork->work;
unsigned long flags;
int ret;
raw_spin_lock_irqsave(&worker->lock, flags);
/* Do not bother with canceling when never queued. */
if (!work->worker) {
ret = false;
goto fast_queue;
}
/* Work must not be used with >1 worker, see kthread_queue_work() */
WARN_ON_ONCE(work->worker != worker);
/*
* Temporary cancel the work but do not fight with another command
* that is canceling the work as well.
*
* It is a bit tricky because of possible races with another
* mod_delayed_work() and cancel_delayed_work() callers.
*
* The timer must be canceled first because worker->lock is released
* when doing so. But the work can be removed from the queue (list)
* only when it can be queued again so that the return value can
* be used for reference counting.
*/
kthread_cancel_delayed_work_timer(work, &flags);
if (work->canceling) {
/* The number of works in the queue does not change. */
ret = true;
goto out;
}
ret = __kthread_cancel_work(work);
fast_queue:
__kthread_queue_delayed_work(worker, dwork, delay);
out:
raw_spin_unlock_irqrestore(&worker->lock, flags);
return ret;
}
EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
{
struct kthread_worker *worker = work->worker;
unsigned long flags;
int ret = false;
if (!worker)
goto out;
raw_spin_lock_irqsave(&worker->lock, flags);
/* Work must not be used with >1 worker, see kthread_queue_work(). */
WARN_ON_ONCE(work->worker != worker);
if (is_dwork)
kthread_cancel_delayed_work_timer(work, &flags);
ret = __kthread_cancel_work(work);
if (worker->current_work != work)
goto out_fast;
/*
* The work is in progress and we need to wait with the lock released.
* In the meantime, block any queuing by setting the canceling counter.
*/
work->canceling++;
raw_spin_unlock_irqrestore(&worker->lock, flags);
kthread_flush_work(work);
raw_spin_lock_irqsave(&worker->lock, flags);
work->canceling--;
out_fast:
raw_spin_unlock_irqrestore(&worker->lock, flags);
out:
return ret;
}
/**
* kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
* @work: the kthread work to cancel
*
* Cancel @work and wait for its execution to finish. This function
* can be used even if the work re-queues itself. On return from this
* function, @work is guaranteed to be not pending or executing on any CPU.
*
* kthread_cancel_work_sync(&delayed_work->work) must not be used for
* delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
*
* The caller must ensure that the worker on which @work was last
* queued can't be destroyed before this function returns.
*
* Return: %true if @work was pending, %false otherwise.
*/
bool kthread_cancel_work_sync(struct kthread_work *work)
{
return __kthread_cancel_work_sync(work, false);
}
EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);
/**
* kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
* wait for it to finish.
* @dwork: the kthread delayed work to cancel
*
* This is kthread_cancel_work_sync() for delayed works.
*
* Return: %true if @dwork was pending, %false otherwise.
*/
bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
{
return __kthread_cancel_work_sync(&dwork->work, true);
}
EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);
/**
* kthread_flush_worker - flush all current works on a kthread_worker
* @worker: worker to flush
*
* Wait until all currently executing or pending works on @worker are
* finished.
*/
void kthread_flush_worker(struct kthread_worker *worker)
{
struct kthread_flush_work fwork = {
KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
COMPLETION_INITIALIZER_ONSTACK(fwork.done),
};
kthread_queue_work(worker, &fwork.work);
wait_for_completion(&fwork.done);
}
EXPORT_SYMBOL_GPL(kthread_flush_worker);
/**
* kthread_destroy_worker - destroy a kthread worker
* @worker: worker to be destroyed
*
* Flush and destroy @worker. The simple flush is enough because the kthread
* worker API is used only in trivial scenarios. There are no multi-step state
* machines needed.
*
* Note that this function is not responsible for handling delayed work, so
* caller should be responsible for queuing or canceling all delayed work items
* before invoke this function.
*/
void kthread_destroy_worker(struct kthread_worker *worker)
{
struct task_struct *task;
task = worker->task; if (WARN_ON(!task))
return;
kthread_flush_worker(worker);
kthread_stop(task);
WARN_ON(!list_empty(&worker->delayed_work_list)); WARN_ON(!list_empty(&worker->work_list)); kfree(worker);
}
EXPORT_SYMBOL(kthread_destroy_worker);
/**
* kthread_use_mm - make the calling kthread operate on an address space
* @mm: address space to operate on
*/
void kthread_use_mm(struct mm_struct *mm)
{
struct mm_struct *active_mm;
struct task_struct *tsk = current;
WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
WARN_ON_ONCE(tsk->mm);
/*
* It is possible for mm to be the same as tsk->active_mm, but
* we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm),
* because these references are not equivalent.
*/
mmgrab(mm);
task_lock(tsk);
/* Hold off tlb flush IPIs while switching mm's */
local_irq_disable();
active_mm = tsk->active_mm;
tsk->active_mm = mm;
tsk->mm = mm;
membarrier_update_current_mm(mm);
switch_mm_irqs_off(active_mm, mm, tsk);
local_irq_enable();
task_unlock(tsk);
#ifdef finish_arch_post_lock_switch
finish_arch_post_lock_switch();
#endif
/*
* When a kthread starts operating on an address space, the loop
* in membarrier_{private,global}_expedited() may not observe
* that tsk->mm, and not issue an IPI. Membarrier requires a
* memory barrier after storing to tsk->mm, before accessing
* user-space memory. A full memory barrier for membarrier
* {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by
* mmdrop_lazy_tlb().
*/
mmdrop_lazy_tlb(active_mm);
}
EXPORT_SYMBOL_GPL(kthread_use_mm);
/**
* kthread_unuse_mm - reverse the effect of kthread_use_mm()
* @mm: address space to operate on
*/
void kthread_unuse_mm(struct mm_struct *mm)
{
struct task_struct *tsk = current;
WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
WARN_ON_ONCE(!tsk->mm);
task_lock(tsk);
/*
* When a kthread stops operating on an address space, the loop
* in membarrier_{private,global}_expedited() may not observe
* that tsk->mm, and not issue an IPI. Membarrier requires a
* memory barrier after accessing user-space memory, before
* clearing tsk->mm.
*/
smp_mb__after_spinlock();
local_irq_disable();
tsk->mm = NULL;
membarrier_update_current_mm(NULL);
mmgrab_lazy_tlb(mm);
/* active_mm is still 'mm' */
enter_lazy_tlb(mm, tsk);
local_irq_enable();
task_unlock(tsk);
mmdrop(mm);
}
EXPORT_SYMBOL_GPL(kthread_unuse_mm);
#ifdef CONFIG_BLK_CGROUP
/**
* kthread_associate_blkcg - associate blkcg to current kthread
* @css: the cgroup info
*
* Current thread must be a kthread. The thread is running jobs on behalf of
* other threads. In some cases, we expect the jobs attach cgroup info of
* original threads instead of that of current thread. This function stores
* original thread's cgroup info in current kthread context for later
* retrieval.
*/
void kthread_associate_blkcg(struct cgroup_subsys_state *css)
{
struct kthread *kthread;
if (!(current->flags & PF_KTHREAD))
return;
kthread = to_kthread(current);
if (!kthread)
return;
if (kthread->blkcg_css) {
css_put(kthread->blkcg_css);
kthread->blkcg_css = NULL;
}
if (css) {
css_get(css);
kthread->blkcg_css = css;
}
}
EXPORT_SYMBOL(kthread_associate_blkcg);
/**
* kthread_blkcg - get associated blkcg css of current kthread
*
* Current thread must be a kthread.
*/
struct cgroup_subsys_state *kthread_blkcg(void)
{
struct kthread *kthread;
if (current->flags & PF_KTHREAD) { kthread = to_kthread(current);
if (kthread)
return kthread->blkcg_css;
}
return NULL;
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PAGE_REF_H
#define _LINUX_PAGE_REF_H
#include <linux/atomic.h>
#include <linux/mm_types.h>
#include <linux/page-flags.h>
#include <linux/tracepoint-defs.h>
DECLARE_TRACEPOINT(page_ref_set);
DECLARE_TRACEPOINT(page_ref_mod);
DECLARE_TRACEPOINT(page_ref_mod_and_test);
DECLARE_TRACEPOINT(page_ref_mod_and_return);
DECLARE_TRACEPOINT(page_ref_mod_unless);
DECLARE_TRACEPOINT(page_ref_freeze);
DECLARE_TRACEPOINT(page_ref_unfreeze);
#ifdef CONFIG_DEBUG_PAGE_REF
/*
* Ideally we would want to use the trace_<tracepoint>_enabled() helper
* functions. But due to include header file issues, that is not
* feasible. Instead we have to open code the static key functions.
*
* See trace_##name##_enabled(void) in include/linux/tracepoint.h
*/
#define page_ref_tracepoint_active(t) tracepoint_enabled(t)
extern void __page_ref_set(struct page *page, int v);
extern void __page_ref_mod(struct page *page, int v);
extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
extern void __page_ref_mod_unless(struct page *page, int v, int u);
extern void __page_ref_freeze(struct page *page, int v, int ret);
extern void __page_ref_unfreeze(struct page *page, int v);
#else
#define page_ref_tracepoint_active(t) false
static inline void __page_ref_set(struct page *page, int v)
{
}
static inline void __page_ref_mod(struct page *page, int v)
{
}
static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)
{
}
static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)
{
}
static inline void __page_ref_mod_unless(struct page *page, int v, int u)
{
}
static inline void __page_ref_freeze(struct page *page, int v, int ret)
{
}
static inline void __page_ref_unfreeze(struct page *page, int v)
{
}
#endif
static inline int page_ref_count(const struct page *page)
{
return atomic_read(&page->_refcount);
}
/**
* folio_ref_count - The reference count on this folio.
* @folio: The folio.
*
* The refcount is usually incremented by calls to folio_get() and
* decremented by calls to folio_put(). Some typical users of the
* folio refcount:
*
* - Each reference from a page table
* - The page cache
* - Filesystem private data
* - The LRU list
* - Pipes
* - Direct IO which references this page in the process address space
*
* Return: The number of references to this folio.
*/
static inline int folio_ref_count(const struct folio *folio)
{
return page_ref_count(&folio->page);
}
static inline int page_count(const struct page *page)
{
return folio_ref_count(page_folio(page));
}
static inline void set_page_count(struct page *page, int v)
{
atomic_set(&page->_refcount, v);
if (page_ref_tracepoint_active(page_ref_set))
__page_ref_set(page, v);
}
static inline void folio_set_count(struct folio *folio, int v)
{
set_page_count(&folio->page, v);
}
/*
* Setup the page count before being freed into the page allocator for
* the first time (boot or memory hotplug)
*/
static inline void init_page_count(struct page *page)
{
set_page_count(page, 1);
}
static inline void page_ref_add(struct page *page, int nr)
{
atomic_add(nr, &page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod))
__page_ref_mod(page, nr);
}
static inline void folio_ref_add(struct folio *folio, int nr)
{
page_ref_add(&folio->page, nr);
}
static inline void page_ref_sub(struct page *page, int nr)
{
atomic_sub(nr, &page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod))
__page_ref_mod(page, -nr);
}
static inline void folio_ref_sub(struct folio *folio, int nr)
{
page_ref_sub(&folio->page, nr);
}
static inline int page_ref_sub_return(struct page *page, int nr)
{
int ret = atomic_sub_return(nr, &page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_return))
__page_ref_mod_and_return(page, -nr, ret);
return ret;
}
static inline int folio_ref_sub_return(struct folio *folio, int nr)
{
return page_ref_sub_return(&folio->page, nr);
}
static inline void page_ref_inc(struct page *page)
{
atomic_inc(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod))
__page_ref_mod(page, 1);
}
static inline void folio_ref_inc(struct folio *folio)
{
page_ref_inc(&folio->page);
}
static inline void page_ref_dec(struct page *page)
{
atomic_dec(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod))
__page_ref_mod(page, -1);
}
static inline void folio_ref_dec(struct folio *folio)
{
page_ref_dec(&folio->page);
}
static inline int page_ref_sub_and_test(struct page *page, int nr)
{
int ret = atomic_sub_and_test(nr, &page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_test))
__page_ref_mod_and_test(page, -nr, ret);
return ret;
}
static inline int folio_ref_sub_and_test(struct folio *folio, int nr)
{
return page_ref_sub_and_test(&folio->page, nr);
}
static inline int page_ref_inc_return(struct page *page)
{
int ret = atomic_inc_return(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_return))
__page_ref_mod_and_return(page, 1, ret);
return ret;
}
static inline int folio_ref_inc_return(struct folio *folio)
{
return page_ref_inc_return(&folio->page);
}
static inline int page_ref_dec_and_test(struct page *page)
{
int ret = atomic_dec_and_test(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_test))
__page_ref_mod_and_test(page, -1, ret);
return ret;
}
static inline int folio_ref_dec_and_test(struct folio *folio)
{
return page_ref_dec_and_test(&folio->page);
}
static inline int page_ref_dec_return(struct page *page)
{
int ret = atomic_dec_return(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_return))
__page_ref_mod_and_return(page, -1, ret);
return ret;
}
static inline int folio_ref_dec_return(struct folio *folio)
{
return page_ref_dec_return(&folio->page);
}
static inline bool page_ref_add_unless(struct page *page, int nr, int u)
{
bool ret = atomic_add_unless(&page->_refcount, nr, u);
if (page_ref_tracepoint_active(page_ref_mod_unless))
__page_ref_mod_unless(page, nr, ret);
return ret;
}
static inline bool folio_ref_add_unless(struct folio *folio, int nr, int u)
{
return page_ref_add_unless(&folio->page, nr, u);
}
/**
* folio_try_get - Attempt to increase the refcount on a folio.
* @folio: The folio.
*
* If you do not already have a reference to a folio, you can attempt to
* get one using this function. It may fail if, for example, the folio
* has been freed since you found a pointer to it, or it is frozen for
* the purposes of splitting or migration.
*
* Return: True if the reference count was successfully incremented.
*/
static inline bool folio_try_get(struct folio *folio)
{
return folio_ref_add_unless(folio, 1, 0);
}
static inline bool folio_ref_try_add_rcu(struct folio *folio, int count)
{
#ifdef CONFIG_TINY_RCU
/*
* The caller guarantees the folio will not be freed from interrupt
* context, so (on !SMP) we only need preemption to be disabled
* and TINY_RCU does that for us.
*/
# ifdef CONFIG_PREEMPT_COUNT
VM_BUG_ON(!in_atomic() && !irqs_disabled());
# endif
VM_BUG_ON_FOLIO(folio_ref_count(folio) == 0, folio);
folio_ref_add(folio, count);
#else
if (unlikely(!folio_ref_add_unless(folio, count, 0))) {
/* Either the folio has been freed, or will be freed. */
return false;
}
#endif
return true;
}
/**
* folio_try_get_rcu - Attempt to increase the refcount on a folio.
* @folio: The folio.
*
* This is a version of folio_try_get() optimised for non-SMP kernels.
* If you are still holding the rcu_read_lock() after looking up the
* page and know that the page cannot have its refcount decreased to
* zero in interrupt context, you can use this instead of folio_try_get().
*
* Example users include get_user_pages_fast() (as pages are not unmapped
* from interrupt context) and the page cache lookups (as pages are not
* truncated from interrupt context). We also know that pages are not
* frozen in interrupt context for the purposes of splitting or migration.
*
* You can also use this function if you're holding a lock that prevents
* pages being frozen & removed; eg the i_pages lock for the page cache
* or the mmap_lock or page table lock for page tables. In this case,
* it will always succeed, and you could have used a plain folio_get(),
* but it's sometimes more convenient to have a common function called
* from both locked and RCU-protected contexts.
*
* Return: True if the reference count was successfully incremented.
*/
static inline bool folio_try_get_rcu(struct folio *folio)
{
return folio_ref_try_add_rcu(folio, 1);
}
static inline int page_ref_freeze(struct page *page, int count)
{
int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);
if (page_ref_tracepoint_active(page_ref_freeze))
__page_ref_freeze(page, count, ret);
return ret;
}
static inline int folio_ref_freeze(struct folio *folio, int count)
{
return page_ref_freeze(&folio->page, count);
}
static inline void page_ref_unfreeze(struct page *page, int count)
{
VM_BUG_ON_PAGE(page_count(page) != 0, page);
VM_BUG_ON(count == 0);
atomic_set_release(&page->_refcount, count);
if (page_ref_tracepoint_active(page_ref_unfreeze))
__page_ref_unfreeze(page, count);
}
static inline void folio_ref_unfreeze(struct folio *folio, int count)
{
page_ref_unfreeze(&folio->page, count);
}
#endif
// SPDX-License-Identifier: GPL-2.0
/*
* drivers/usb/core/usb.c
*
* (C) Copyright Linus Torvalds 1999
* (C) Copyright Johannes Erdfelt 1999-2001
* (C) Copyright Andreas Gal 1999
* (C) Copyright Gregory P. Smith 1999
* (C) Copyright Deti Fliegl 1999 (new USB architecture)
* (C) Copyright Randy Dunlap 2000
* (C) Copyright David Brownell 2000-2004
* (C) Copyright Yggdrasil Computing, Inc. 2000
* (usb_device_id matching changes by Adam J. Richter)
* (C) Copyright Greg Kroah-Hartman 2002-2003
*
* Released under the GPLv2 only.
*
* NOTE! This is not actually a driver at all, rather this is
* just a collection of helper routines that implement the
* generic USB things that the real drivers can use..
*
* Think of this as a "USB library" rather than anything else,
* with no callbacks. Callbacks are evil.
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/of.h>
#include <linux/string.h>
#include <linux/bitops.h>
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/errno.h>
#include <linux/usb.h>
#include <linux/usb/hcd.h>
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/debugfs.h>
#include <linux/usb/of.h>
#include <asm/io.h>
#include <linux/scatterlist.h>
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include "hub.h"
const char *usbcore_name = "usbcore";
static bool nousb; /* Disable USB when built into kernel image */
module_param(nousb, bool, 0444);
/*
* for external read access to <nousb>
*/
int usb_disabled(void)
{
return nousb;
}
EXPORT_SYMBOL_GPL(usb_disabled);
#ifdef CONFIG_PM
/* Default delay value, in seconds */
static int usb_autosuspend_delay = CONFIG_USB_AUTOSUSPEND_DELAY;
module_param_named(autosuspend, usb_autosuspend_delay, int, 0644);
MODULE_PARM_DESC(autosuspend, "default autosuspend delay");
#else
#define usb_autosuspend_delay 0
#endif
static bool match_endpoint(struct usb_endpoint_descriptor *epd,
struct usb_endpoint_descriptor **bulk_in,
struct usb_endpoint_descriptor **bulk_out,
struct usb_endpoint_descriptor **int_in,
struct usb_endpoint_descriptor **int_out)
{
switch (usb_endpoint_type(epd)) {
case USB_ENDPOINT_XFER_BULK:
if (usb_endpoint_dir_in(epd)) {
if (bulk_in && !*bulk_in) {
*bulk_in = epd;
break;
}
} else {
if (bulk_out && !*bulk_out) {
*bulk_out = epd;
break;
}
}
return false;
case USB_ENDPOINT_XFER_INT:
if (usb_endpoint_dir_in(epd)) {
if (int_in && !*int_in) {
*int_in = epd;
break;
}
} else {
if (int_out && !*int_out) {
*int_out = epd;
break;
}
}
return false;
default:
return false;
}
return (!bulk_in || *bulk_in) && (!bulk_out || *bulk_out) &&
(!int_in || *int_in) && (!int_out || *int_out);
}
/**
* usb_find_common_endpoints() -- look up common endpoint descriptors
* @alt: alternate setting to search
* @bulk_in: pointer to descriptor pointer, or NULL
* @bulk_out: pointer to descriptor pointer, or NULL
* @int_in: pointer to descriptor pointer, or NULL
* @int_out: pointer to descriptor pointer, or NULL
*
* Search the alternate setting's endpoint descriptors for the first bulk-in,
* bulk-out, interrupt-in and interrupt-out endpoints and return them in the
* provided pointers (unless they are NULL).
*
* If a requested endpoint is not found, the corresponding pointer is set to
* NULL.
*
* Return: Zero if all requested descriptors were found, or -ENXIO otherwise.
*/
int usb_find_common_endpoints(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_in,
struct usb_endpoint_descriptor **bulk_out,
struct usb_endpoint_descriptor **int_in,
struct usb_endpoint_descriptor **int_out)
{
struct usb_endpoint_descriptor *epd;
int i;
if (bulk_in)
*bulk_in = NULL;
if (bulk_out)
*bulk_out = NULL;
if (int_in)
*int_in = NULL;
if (int_out)
*int_out = NULL;
for (i = 0; i < alt->desc.bNumEndpoints; ++i) {
epd = &alt->endpoint[i].desc;
if (match_endpoint(epd, bulk_in, bulk_out, int_in, int_out))
return 0;
}
return -ENXIO;
}
EXPORT_SYMBOL_GPL(usb_find_common_endpoints);
/**
* usb_find_common_endpoints_reverse() -- look up common endpoint descriptors
* @alt: alternate setting to search
* @bulk_in: pointer to descriptor pointer, or NULL
* @bulk_out: pointer to descriptor pointer, or NULL
* @int_in: pointer to descriptor pointer, or NULL
* @int_out: pointer to descriptor pointer, or NULL
*
* Search the alternate setting's endpoint descriptors for the last bulk-in,
* bulk-out, interrupt-in and interrupt-out endpoints and return them in the
* provided pointers (unless they are NULL).
*
* If a requested endpoint is not found, the corresponding pointer is set to
* NULL.
*
* Return: Zero if all requested descriptors were found, or -ENXIO otherwise.
*/
int usb_find_common_endpoints_reverse(struct usb_host_interface *alt,
struct usb_endpoint_descriptor **bulk_in,
struct usb_endpoint_descriptor **bulk_out,
struct usb_endpoint_descriptor **int_in,
struct usb_endpoint_descriptor **int_out)
{
struct usb_endpoint_descriptor *epd;
int i;
if (bulk_in)
*bulk_in = NULL;
if (bulk_out)
*bulk_out = NULL;
if (int_in)
*int_in = NULL;
if (int_out)
*int_out = NULL;
for (i = alt->desc.bNumEndpoints - 1; i >= 0; --i) {
epd = &alt->endpoint[i].desc;
if (match_endpoint(epd, bulk_in, bulk_out, int_in, int_out))
return 0;
}
return -ENXIO;
}
EXPORT_SYMBOL_GPL(usb_find_common_endpoints_reverse);
/**
* usb_find_endpoint() - Given an endpoint address, search for the endpoint's
* usb_host_endpoint structure in an interface's current altsetting.
* @intf: the interface whose current altsetting should be searched
* @ep_addr: the endpoint address (number and direction) to find
*
* Search the altsetting's list of endpoints for one with the specified address.
*
* Return: Pointer to the usb_host_endpoint if found, %NULL otherwise.
*/
static const struct usb_host_endpoint *usb_find_endpoint(
const struct usb_interface *intf, unsigned int ep_addr)
{
int n;
const struct usb_host_endpoint *ep;
n = intf->cur_altsetting->desc.bNumEndpoints;
ep = intf->cur_altsetting->endpoint;
for (; n > 0; (--n, ++ep)) {
if (ep->desc.bEndpointAddress == ep_addr)
return ep;
}
return NULL;
}
/**
* usb_check_bulk_endpoints - Check whether an interface's current altsetting
* contains a set of bulk endpoints with the given addresses.
* @intf: the interface whose current altsetting should be searched
* @ep_addrs: 0-terminated array of the endpoint addresses (number and
* direction) to look for
*
* Search for endpoints with the specified addresses and check their types.
*
* Return: %true if all the endpoints are found and are bulk, %false otherwise.
*/
bool usb_check_bulk_endpoints(
const struct usb_interface *intf, const u8 *ep_addrs)
{
const struct usb_host_endpoint *ep;
for (; *ep_addrs; ++ep_addrs) {
ep = usb_find_endpoint(intf, *ep_addrs);
if (!ep || !usb_endpoint_xfer_bulk(&ep->desc))
return false;
}
return true;
}
EXPORT_SYMBOL_GPL(usb_check_bulk_endpoints);
/**
* usb_check_int_endpoints - Check whether an interface's current altsetting
* contains a set of interrupt endpoints with the given addresses.
* @intf: the interface whose current altsetting should be searched
* @ep_addrs: 0-terminated array of the endpoint addresses (number and
* direction) to look for
*
* Search for endpoints with the specified addresses and check their types.
*
* Return: %true if all the endpoints are found and are interrupt,
* %false otherwise.
*/
bool usb_check_int_endpoints(
const struct usb_interface *intf, const u8 *ep_addrs)
{
const struct usb_host_endpoint *ep;
for (; *ep_addrs; ++ep_addrs) {
ep = usb_find_endpoint(intf, *ep_addrs);
if (!ep || !usb_endpoint_xfer_int(&ep->desc))
return false;
}
return true;
}
EXPORT_SYMBOL_GPL(usb_check_int_endpoints);
/**
* usb_find_alt_setting() - Given a configuration, find the alternate setting
* for the given interface.
* @config: the configuration to search (not necessarily the current config).
* @iface_num: interface number to search in
* @alt_num: alternate interface setting number to search for.
*
* Search the configuration's interface cache for the given alt setting.
*
* Return: The alternate setting, if found. %NULL otherwise.
*/
struct usb_host_interface *usb_find_alt_setting(
struct usb_host_config *config,
unsigned int iface_num,
unsigned int alt_num)
{
struct usb_interface_cache *intf_cache = NULL;
int i;
if (!config)
return NULL;
for (i = 0; i < config->desc.bNumInterfaces; i++) {
if (config->intf_cache[i]->altsetting[0].desc.bInterfaceNumber
== iface_num) {
intf_cache = config->intf_cache[i];
break;
}
}
if (!intf_cache)
return NULL;
for (i = 0; i < intf_cache->num_altsetting; i++)
if (intf_cache->altsetting[i].desc.bAlternateSetting == alt_num)
return &intf_cache->altsetting[i];
printk(KERN_DEBUG "Did not find alt setting %u for intf %u, "
"config %u\n", alt_num, iface_num,
config->desc.bConfigurationValue);
return NULL;
}
EXPORT_SYMBOL_GPL(usb_find_alt_setting);
/**
* usb_ifnum_to_if - get the interface object with a given interface number
* @dev: the device whose current configuration is considered
* @ifnum: the desired interface
*
* This walks the device descriptor for the currently active configuration
* to find the interface object with the particular interface number.
*
* Note that configuration descriptors are not required to assign interface
* numbers sequentially, so that it would be incorrect to assume that
* the first interface in that descriptor corresponds to interface zero.
* This routine helps device drivers avoid such mistakes.
* However, you should make sure that you do the right thing with any
* alternate settings available for this interfaces.
*
* Don't call this function unless you are bound to one of the interfaces
* on this device or you have locked the device!
*
* Return: A pointer to the interface that has @ifnum as interface number,
* if found. %NULL otherwise.
*/
struct usb_interface *usb_ifnum_to_if(const struct usb_device *dev,
unsigned ifnum)
{
struct usb_host_config *config = dev->actconfig;
int i;
if (!config)
return NULL; for (i = 0; i < config->desc.bNumInterfaces; i++) if (config->interface[i]->altsetting[0]
.desc.bInterfaceNumber == ifnum)
return config->interface[i];
return NULL;
}
EXPORT_SYMBOL_GPL(usb_ifnum_to_if);
/**
* usb_altnum_to_altsetting - get the altsetting structure with a given alternate setting number.
* @intf: the interface containing the altsetting in question
* @altnum: the desired alternate setting number
*
* This searches the altsetting array of the specified interface for
* an entry with the correct bAlternateSetting value.
*
* Note that altsettings need not be stored sequentially by number, so
* it would be incorrect to assume that the first altsetting entry in
* the array corresponds to altsetting zero. This routine helps device
* drivers avoid such mistakes.
*
* Don't call this function unless you are bound to the intf interface
* or you have locked the device!
*
* Return: A pointer to the entry of the altsetting array of @intf that
* has @altnum as the alternate setting number. %NULL if not found.
*/
struct usb_host_interface *usb_altnum_to_altsetting(
const struct usb_interface *intf,
unsigned int altnum)
{
int i;
for (i = 0; i < intf->num_altsetting; i++) {
if (intf->altsetting[i].desc.bAlternateSetting == altnum)
return &intf->altsetting[i];
}
return NULL;
}
EXPORT_SYMBOL_GPL(usb_altnum_to_altsetting);
struct find_interface_arg {
int minor;
struct device_driver *drv;
};
static int __find_interface(struct device *dev, const void *data)
{
const struct find_interface_arg *arg = data;
struct usb_interface *intf;
if (!is_usb_interface(dev))
return 0;
if (dev->driver != arg->drv)
return 0;
intf = to_usb_interface(dev);
return intf->minor == arg->minor;
}
/**
* usb_find_interface - find usb_interface pointer for driver and device
* @drv: the driver whose current configuration is considered
* @minor: the minor number of the desired device
*
* This walks the bus device list and returns a pointer to the interface
* with the matching minor and driver. Note, this only works for devices
* that share the USB major number.
*
* Return: A pointer to the interface with the matching major and @minor.
*/
struct usb_interface *usb_find_interface(struct usb_driver *drv, int minor)
{
struct find_interface_arg argb;
struct device *dev;
argb.minor = minor;
argb.drv = &drv->driver;
dev = bus_find_device(&usb_bus_type, NULL, &argb, __find_interface);
/* Drop reference count from bus_find_device */
put_device(dev);
return dev ? to_usb_interface(dev) : NULL;
}
EXPORT_SYMBOL_GPL(usb_find_interface);
struct each_dev_arg {
void *data;
int (*fn)(struct usb_device *, void *);
};
static int __each_dev(struct device *dev, void *data)
{
struct each_dev_arg *arg = (struct each_dev_arg *)data;
/* There are struct usb_interface on the same bus, filter them out */
if (!is_usb_device(dev))
return 0;
return arg->fn(to_usb_device(dev), arg->data);
}
/**
* usb_for_each_dev - iterate over all USB devices in the system
* @data: data pointer that will be handed to the callback function
* @fn: callback function to be called for each USB device
*
* Iterate over all USB devices and call @fn for each, passing it @data. If it
* returns anything other than 0, we break the iteration prematurely and return
* that value.
*/
int usb_for_each_dev(void *data, int (*fn)(struct usb_device *, void *))
{
struct each_dev_arg arg = {data, fn};
return bus_for_each_dev(&usb_bus_type, NULL, &arg, __each_dev);
}
EXPORT_SYMBOL_GPL(usb_for_each_dev);
/**
* usb_release_dev - free a usb device structure when all users of it are finished.
* @dev: device that's been disconnected
*
* Will be called only by the device core when all users of this usb device are
* done.
*/
static void usb_release_dev(struct device *dev)
{
struct usb_device *udev;
struct usb_hcd *hcd;
udev = to_usb_device(dev);
hcd = bus_to_hcd(udev->bus);
usb_destroy_configuration(udev);
usb_release_bos_descriptor(udev);
of_node_put(dev->of_node);
usb_put_hcd(hcd);
kfree(udev->product);
kfree(udev->manufacturer);
kfree(udev->serial);
kfree(udev);
}
static int usb_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct usb_device *usb_dev;
usb_dev = to_usb_device(dev);
if (add_uevent_var(env, "BUSNUM=%03d", usb_dev->bus->busnum))
return -ENOMEM;
if (add_uevent_var(env, "DEVNUM=%03d", usb_dev->devnum))
return -ENOMEM;
return 0;
}
#ifdef CONFIG_PM
/* USB device Power-Management thunks.
* There's no need to distinguish here between quiescing a USB device
* and powering it down; the generic_suspend() routine takes care of
* it by skipping the usb_port_suspend() call for a quiesce. And for
* USB interfaces there's no difference at all.
*/
static int usb_dev_prepare(struct device *dev)
{
return 0; /* Implement eventually? */
}
static void usb_dev_complete(struct device *dev)
{
/* Currently used only for rebinding interfaces */
usb_resume_complete(dev);
}
static int usb_dev_suspend(struct device *dev)
{
return usb_suspend(dev, PMSG_SUSPEND);
}
static int usb_dev_resume(struct device *dev)
{
return usb_resume(dev, PMSG_RESUME);
}
static int usb_dev_freeze(struct device *dev)
{
return usb_suspend(dev, PMSG_FREEZE);
}
static int usb_dev_thaw(struct device *dev)
{
return usb_resume(dev, PMSG_THAW);
}
static int usb_dev_poweroff(struct device *dev)
{
return usb_suspend(dev, PMSG_HIBERNATE);
}
static int usb_dev_restore(struct device *dev)
{
return usb_resume(dev, PMSG_RESTORE);
}
static const struct dev_pm_ops usb_device_pm_ops = {
.prepare = usb_dev_prepare,
.complete = usb_dev_complete,
.suspend = usb_dev_suspend,
.resume = usb_dev_resume,
.freeze = usb_dev_freeze,
.thaw = usb_dev_thaw,
.poweroff = usb_dev_poweroff,
.restore = usb_dev_restore,
.runtime_suspend = usb_runtime_suspend,
.runtime_resume = usb_runtime_resume,
.runtime_idle = usb_runtime_idle,
};
#endif /* CONFIG_PM */
static char *usb_devnode(const struct device *dev,
umode_t *mode, kuid_t *uid, kgid_t *gid)
{
const struct usb_device *usb_dev;
usb_dev = to_usb_device(dev);
return kasprintf(GFP_KERNEL, "bus/usb/%03d/%03d",
usb_dev->bus->busnum, usb_dev->devnum);
}
const struct device_type usb_device_type = {
.name = "usb_device",
.release = usb_release_dev,
.uevent = usb_dev_uevent,
.devnode = usb_devnode,
#ifdef CONFIG_PM
.pm = &usb_device_pm_ops,
#endif
};
static bool usb_dev_authorized(struct usb_device *dev, struct usb_hcd *hcd)
{
struct usb_hub *hub;
if (!dev->parent)
return true; /* Root hub always ok [and always wired] */
switch (hcd->dev_policy) {
case USB_DEVICE_AUTHORIZE_NONE:
default:
return false;
case USB_DEVICE_AUTHORIZE_ALL:
return true;
case USB_DEVICE_AUTHORIZE_INTERNAL:
hub = usb_hub_to_struct_hub(dev->parent);
return hub->ports[dev->portnum - 1]->connect_type ==
USB_PORT_CONNECT_TYPE_HARD_WIRED;
}
}
/**
* usb_alloc_dev - usb device constructor (usbcore-internal)
* @parent: hub to which device is connected; null to allocate a root hub
* @bus: bus used to access the device
* @port1: one-based index of port; ignored for root hubs
*
* Context: task context, might sleep.
*
* Only hub drivers (including virtual root hub drivers for host
* controllers) should ever call this.
*
* This call may not be used in a non-sleeping context.
*
* Return: On success, a pointer to the allocated usb device. %NULL on
* failure.
*/
struct usb_device *usb_alloc_dev(struct usb_device *parent,
struct usb_bus *bus, unsigned port1)
{
struct usb_device *dev;
struct usb_hcd *usb_hcd = bus_to_hcd(bus);
unsigned raw_port = port1;
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (!dev)
return NULL;
if (!usb_get_hcd(usb_hcd)) { kfree(dev);
return NULL;
}
/* Root hubs aren't true devices, so don't allocate HCD resources */
if (usb_hcd->driver->alloc_dev && parent && !usb_hcd->driver->alloc_dev(usb_hcd, dev)) { usb_put_hcd(bus_to_hcd(bus));
kfree(dev);
return NULL;
}
device_initialize(&dev->dev);
dev->dev.bus = &usb_bus_type;
dev->dev.type = &usb_device_type;
dev->dev.groups = usb_device_groups;
set_dev_node(&dev->dev, dev_to_node(bus->sysdev));
dev->state = USB_STATE_ATTACHED;
dev->lpm_disable_count = 1;
atomic_set(&dev->urbnum, 0);
INIT_LIST_HEAD(&dev->ep0.urb_list);
dev->ep0.desc.bLength = USB_DT_ENDPOINT_SIZE;
dev->ep0.desc.bDescriptorType = USB_DT_ENDPOINT;
/* ep0 maxpacket comes later, from device descriptor */
usb_enable_endpoint(dev, &dev->ep0, false);
dev->can_submit = 1;
/* Save readable and stable topology id, distinguishing devices
* by location for diagnostics, tools, driver model, etc. The
* string is a path along hub ports, from the root. Each device's
* dev->devpath will be stable until USB is re-cabled, and hubs
* are often labeled with these port numbers. The name isn't
* as stable: bus->busnum changes easily from modprobe order,
* cardbus or pci hotplugging, and so on.
*/
if (unlikely(!parent)) {
dev->devpath[0] = '0';
dev->route = 0;
dev->dev.parent = bus->controller;
device_set_of_node_from_dev(&dev->dev, bus->sysdev);
dev_set_name(&dev->dev, "usb%d", bus->busnum);
} else {
/* match any labeling on the hubs; it's one-based */
if (parent->devpath[0] == '0') { snprintf(dev->devpath, sizeof dev->devpath,
"%d", port1);
/* Root ports are not counted in route string */
dev->route = 0;
} else {
snprintf(dev->devpath, sizeof dev->devpath,
"%s.%d", parent->devpath, port1);
/* Route string assumes hubs have less than 16 ports */
if (port1 < 15)
dev->route = parent->route + (port1 << ((parent->level - 1)*4));
else
dev->route = parent->route + (15 << ((parent->level - 1)*4));
}
dev->dev.parent = &parent->dev;
dev_set_name(&dev->dev, "%d-%s", bus->busnum, dev->devpath);
if (!parent->parent) {
/* device under root hub's port */
raw_port = usb_hcd_find_raw_port_number(usb_hcd,
port1);
}
dev->dev.of_node = usb_of_get_device_node(parent, raw_port);
/* hub driver sets up TT records */
}
dev->portnum = port1;
dev->bus = bus;
dev->parent = parent;
INIT_LIST_HEAD(&dev->filelist);
#ifdef CONFIG_PM
pm_runtime_set_autosuspend_delay(&dev->dev,
usb_autosuspend_delay * 1000);
dev->connect_time = jiffies;
dev->active_duration = -jiffies;
#endif
dev->authorized = usb_dev_authorized(dev, usb_hcd); return dev;
}
EXPORT_SYMBOL_GPL(usb_alloc_dev);
/**
* usb_get_dev - increments the reference count of the usb device structure
* @dev: the device being referenced
*
* Each live reference to a device should be refcounted.
*
* Drivers for USB interfaces should normally record such references in
* their probe() methods, when they bind to an interface, and release
* them by calling usb_put_dev(), in their disconnect() methods.
* However, if a driver does not access the usb_device structure after
* its disconnect() method returns then refcounting is not necessary,
* because the USB core guarantees that a usb_device will not be
* deallocated until after all of its interface drivers have been unbound.
*
* Return: A pointer to the device with the incremented reference counter.
*/
struct usb_device *usb_get_dev(struct usb_device *dev)
{
if (dev) get_device(&dev->dev); return dev;
}
EXPORT_SYMBOL_GPL(usb_get_dev);
/**
* usb_put_dev - release a use of the usb device structure
* @dev: device that's been disconnected
*
* Must be called when a user of a device is finished with it. When the last
* user of the device calls this function, the memory of the device is freed.
*/
void usb_put_dev(struct usb_device *dev)
{
if (dev) put_device(&dev->dev);
}
EXPORT_SYMBOL_GPL(usb_put_dev);
/**
* usb_get_intf - increments the reference count of the usb interface structure
* @intf: the interface being referenced
*
* Each live reference to a interface must be refcounted.
*
* Drivers for USB interfaces should normally record such references in
* their probe() methods, when they bind to an interface, and release
* them by calling usb_put_intf(), in their disconnect() methods.
* However, if a driver does not access the usb_interface structure after
* its disconnect() method returns then refcounting is not necessary,
* because the USB core guarantees that a usb_interface will not be
* deallocated until after its driver has been unbound.
*
* Return: A pointer to the interface with the incremented reference counter.
*/
struct usb_interface *usb_get_intf(struct usb_interface *intf)
{
if (intf)
get_device(&intf->dev);
return intf;
}
EXPORT_SYMBOL_GPL(usb_get_intf);
/**
* usb_put_intf - release a use of the usb interface structure
* @intf: interface that's been decremented
*
* Must be called when a user of an interface is finished with it. When the
* last user of the interface calls this function, the memory of the interface
* is freed.
*/
void usb_put_intf(struct usb_interface *intf)
{
if (intf) put_device(&intf->dev);
}
EXPORT_SYMBOL_GPL(usb_put_intf);
/**
* usb_intf_get_dma_device - acquire a reference on the usb interface's DMA endpoint
* @intf: the usb interface
*
* While a USB device cannot perform DMA operations by itself, many USB
* controllers can. A call to usb_intf_get_dma_device() returns the DMA endpoint
* for the given USB interface, if any. The returned device structure must be
* released with put_device().
*
* See also usb_get_dma_device().
*
* Returns: A reference to the usb interface's DMA endpoint; or NULL if none
* exists.
*/
struct device *usb_intf_get_dma_device(struct usb_interface *intf)
{
struct usb_device *udev = interface_to_usbdev(intf);
struct device *dmadev;
if (!udev->bus)
return NULL;
dmadev = get_device(udev->bus->sysdev);
if (!dmadev || !dmadev->dma_mask) {
put_device(dmadev);
return NULL;
}
return dmadev;
}
EXPORT_SYMBOL_GPL(usb_intf_get_dma_device);
/* USB device locking
*
* USB devices and interfaces are locked using the semaphore in their
* embedded struct device. The hub driver guarantees that whenever a
* device is connected or disconnected, drivers are called with the
* USB device locked as well as their particular interface.
*
* Complications arise when several devices are to be locked at the same
* time. Only hub-aware drivers that are part of usbcore ever have to
* do this; nobody else needs to worry about it. The rule for locking
* is simple:
*
* When locking both a device and its parent, always lock the
* parent first.
*/
/**
* usb_lock_device_for_reset - cautiously acquire the lock for a usb device structure
* @udev: device that's being locked
* @iface: interface bound to the driver making the request (optional)
*
* Attempts to acquire the device lock, but fails if the device is
* NOTATTACHED or SUSPENDED, or if iface is specified and the interface
* is neither BINDING nor BOUND. Rather than sleeping to wait for the
* lock, the routine polls repeatedly. This is to prevent deadlock with
* disconnect; in some drivers (such as usb-storage) the disconnect()
* or suspend() method will block waiting for a device reset to complete.
*
* Return: A negative error code for failure, otherwise 0.
*/
int usb_lock_device_for_reset(struct usb_device *udev,
const struct usb_interface *iface)
{
unsigned long jiffies_expire = jiffies + HZ;
if (udev->state == USB_STATE_NOTATTACHED)
return -ENODEV;
if (udev->state == USB_STATE_SUSPENDED)
return -EHOSTUNREACH;
if (iface && (iface->condition == USB_INTERFACE_UNBINDING ||
iface->condition == USB_INTERFACE_UNBOUND))
return -EINTR;
while (!usb_trylock_device(udev)) {
/* If we can't acquire the lock after waiting one second,
* we're probably deadlocked */
if (time_after(jiffies, jiffies_expire))
return -EBUSY;
msleep(15);
if (udev->state == USB_STATE_NOTATTACHED)
return -ENODEV;
if (udev->state == USB_STATE_SUSPENDED)
return -EHOSTUNREACH;
if (iface && (iface->condition == USB_INTERFACE_UNBINDING ||
iface->condition == USB_INTERFACE_UNBOUND))
return -EINTR;
}
return 0;
}
EXPORT_SYMBOL_GPL(usb_lock_device_for_reset);
/**
* usb_get_current_frame_number - return current bus frame number
* @dev: the device whose bus is being queried
*
* Return: The current frame number for the USB host controller used
* with the given USB device. This can be used when scheduling
* isochronous requests.
*
* Note: Different kinds of host controller have different "scheduling
* horizons". While one type might support scheduling only 32 frames
* into the future, others could support scheduling up to 1024 frames
* into the future.
*
*/
int usb_get_current_frame_number(struct usb_device *dev)
{
return usb_hcd_get_frame_number(dev);
}
EXPORT_SYMBOL_GPL(usb_get_current_frame_number);
/*-------------------------------------------------------------------*/
/*
* __usb_get_extra_descriptor() finds a descriptor of specific type in the
* extra field of the interface and endpoint descriptor structs.
*/
int __usb_get_extra_descriptor(char *buffer, unsigned size,
unsigned char type, void **ptr, size_t minsize)
{
struct usb_descriptor_header *header;
while (size >= sizeof(struct usb_descriptor_header)) {
header = (struct usb_descriptor_header *)buffer;
if (header->bLength < 2 || header->bLength > size) { printk(KERN_ERR
"%s: bogus descriptor, type %d length %d\n",
usbcore_name,
header->bDescriptorType,
header->bLength);
return -1;
}
if (header->bDescriptorType == type && header->bLength >= minsize) { *ptr = header; return 0;
}
buffer += header->bLength;
size -= header->bLength;
}
return -1;
}
EXPORT_SYMBOL_GPL(__usb_get_extra_descriptor);
/**
* usb_alloc_coherent - allocate dma-consistent buffer for URB_NO_xxx_DMA_MAP
* @dev: device the buffer will be used with
* @size: requested buffer size
* @mem_flags: affect whether allocation may block
* @dma: used to return DMA address of buffer
*
* Return: Either null (indicating no buffer could be allocated), or the
* cpu-space pointer to a buffer that may be used to perform DMA to the
* specified device. Such cpu-space buffers are returned along with the DMA
* address (through the pointer provided).
*
* Note:
* These buffers are used with URB_NO_xxx_DMA_MAP set in urb->transfer_flags
* to avoid behaviors like using "DMA bounce buffers", or thrashing IOMMU
* hardware during URB completion/resubmit. The implementation varies between
* platforms, depending on details of how DMA will work to this device.
* Using these buffers also eliminates cacheline sharing problems on
* architectures where CPU caches are not DMA-coherent. On systems without
* bus-snooping caches, these buffers are uncached.
*
* When the buffer is no longer used, free it with usb_free_coherent().
*/
void *usb_alloc_coherent(struct usb_device *dev, size_t size, gfp_t mem_flags,
dma_addr_t *dma)
{
if (!dev || !dev->bus)
return NULL;
return hcd_buffer_alloc(dev->bus, size, mem_flags, dma);
}
EXPORT_SYMBOL_GPL(usb_alloc_coherent);
/**
* usb_free_coherent - free memory allocated with usb_alloc_coherent()
* @dev: device the buffer was used with
* @size: requested buffer size
* @addr: CPU address of buffer
* @dma: DMA address of buffer
*
* This reclaims an I/O buffer, letting it be reused. The memory must have
* been allocated using usb_alloc_coherent(), and the parameters must match
* those provided in that allocation request.
*/
void usb_free_coherent(struct usb_device *dev, size_t size, void *addr,
dma_addr_t dma)
{
if (!dev || !dev->bus)
return;
if (!addr)
return;
hcd_buffer_free(dev->bus, size, addr, dma);
}
EXPORT_SYMBOL_GPL(usb_free_coherent);
/*
* Notifications of device and interface registration
*/
static int usb_bus_notify(struct notifier_block *nb, unsigned long action,
void *data)
{
struct device *dev = data;
switch (action) {
case BUS_NOTIFY_ADD_DEVICE:
if (dev->type == &usb_device_type) (void) usb_create_sysfs_dev_files(to_usb_device(dev)); else if (dev->type == &usb_if_device_type) usb_create_sysfs_intf_files(to_usb_interface(dev));
break;
case BUS_NOTIFY_DEL_DEVICE:
if (dev->type == &usb_device_type) usb_remove_sysfs_dev_files(to_usb_device(dev)); else if (dev->type == &usb_if_device_type) usb_remove_sysfs_intf_files(to_usb_interface(dev));
break;
}
return 0;
}
static struct notifier_block usb_bus_nb = {
.notifier_call = usb_bus_notify,
};
static void usb_debugfs_init(void)
{
debugfs_create_file("devices", 0444, usb_debug_root, NULL,
&usbfs_devices_fops);
}
static void usb_debugfs_cleanup(void)
{
debugfs_lookup_and_remove("devices", usb_debug_root);
}
/*
* Init
*/
static int __init usb_init(void)
{
int retval;
if (usb_disabled()) {
pr_info("%s: USB support disabled\n", usbcore_name);
return 0;
}
usb_init_pool_max();
usb_debugfs_init();
usb_acpi_register();
retval = bus_register(&usb_bus_type);
if (retval)
goto bus_register_failed;
retval = bus_register_notifier(&usb_bus_type, &usb_bus_nb);
if (retval)
goto bus_notifier_failed;
retval = usb_major_init();
if (retval)
goto major_init_failed;
retval = class_register(&usbmisc_class);
if (retval)
goto class_register_failed;
retval = usb_register(&usbfs_driver);
if (retval)
goto driver_register_failed;
retval = usb_devio_init();
if (retval)
goto usb_devio_init_failed;
retval = usb_hub_init();
if (retval)
goto hub_init_failed;
retval = usb_register_device_driver(&usb_generic_driver, THIS_MODULE);
if (!retval)
goto out;
usb_hub_cleanup();
hub_init_failed:
usb_devio_cleanup();
usb_devio_init_failed:
usb_deregister(&usbfs_driver);
driver_register_failed:
class_unregister(&usbmisc_class);
class_register_failed:
usb_major_cleanup();
major_init_failed:
bus_unregister_notifier(&usb_bus_type, &usb_bus_nb);
bus_notifier_failed:
bus_unregister(&usb_bus_type);
bus_register_failed:
usb_acpi_unregister();
usb_debugfs_cleanup();
out:
return retval;
}
/*
* Cleanup
*/
static void __exit usb_exit(void)
{
/* This will matter if shutdown/reboot does exitcalls. */
if (usb_disabled())
return;
usb_release_quirk_list();
usb_deregister_device_driver(&usb_generic_driver);
usb_major_cleanup();
usb_deregister(&usbfs_driver);
usb_devio_cleanup();
usb_hub_cleanup();
class_unregister(&usbmisc_class);
bus_unregister_notifier(&usb_bus_type, &usb_bus_nb);
bus_unregister(&usb_bus_type);
usb_acpi_unregister();
usb_debugfs_cleanup();
idr_destroy(&usb_bus_idr);
}
subsys_initcall(usb_init);
module_exit(usb_exit);
MODULE_LICENSE("GPL");
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __FS_NOTIFY_FSNOTIFY_H_
#define __FS_NOTIFY_FSNOTIFY_H_
#include <linux/list.h>
#include <linux/fsnotify.h>
#include <linux/srcu.h>
#include <linux/types.h>
#include "../mount.h"
static inline struct inode *fsnotify_conn_inode(
struct fsnotify_mark_connector *conn)
{
return container_of(conn->obj, struct inode, i_fsnotify_marks);
}
static inline struct mount *fsnotify_conn_mount(
struct fsnotify_mark_connector *conn)
{
return container_of(conn->obj, struct mount, mnt_fsnotify_marks);
}
static inline struct super_block *fsnotify_conn_sb(
struct fsnotify_mark_connector *conn)
{
return container_of(conn->obj, struct super_block, s_fsnotify_marks);
}
static inline struct super_block *fsnotify_connector_sb(
struct fsnotify_mark_connector *conn)
{
switch (conn->type) {
case FSNOTIFY_OBJ_TYPE_INODE:
return fsnotify_conn_inode(conn)->i_sb;
case FSNOTIFY_OBJ_TYPE_VFSMOUNT:
return fsnotify_conn_mount(conn)->mnt.mnt_sb;
case FSNOTIFY_OBJ_TYPE_SB:
return fsnotify_conn_sb(conn);
default:
return NULL;
}
}
/* destroy all events sitting in this groups notification queue */
extern void fsnotify_flush_notify(struct fsnotify_group *group);
/* protects reads of inode and vfsmount marks list */
extern struct srcu_struct fsnotify_mark_srcu;
/* compare two groups for sorting of marks lists */
extern int fsnotify_compare_groups(struct fsnotify_group *a,
struct fsnotify_group *b);
/* Destroy all marks attached to an object via connector */
extern void fsnotify_destroy_marks(fsnotify_connp_t *connp);
/* run the list of all marks associated with inode and destroy them */
static inline void fsnotify_clear_marks_by_inode(struct inode *inode)
{
fsnotify_destroy_marks(&inode->i_fsnotify_marks);
}
/* run the list of all marks associated with vfsmount and destroy them */
static inline void fsnotify_clear_marks_by_mount(struct vfsmount *mnt)
{
fsnotify_destroy_marks(&real_mount(mnt)->mnt_fsnotify_marks);
}
/* run the list of all marks associated with sb and destroy them */
static inline void fsnotify_clear_marks_by_sb(struct super_block *sb)
{
fsnotify_destroy_marks(&sb->s_fsnotify_marks);
}
/*
* update the dentry->d_flags of all of inode's children to indicate if inode cares
* about events that happen to its children.
*/
extern void __fsnotify_update_child_dentry_flags(struct inode *inode);
extern struct kmem_cache *fsnotify_mark_connector_cachep;
#endif /* __FS_NOTIFY_FSNOTIFY_H_ */
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* acpi_bus.c - ACPI Bus Driver ($Revision: 80 $)
*
* Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
*/
#define pr_fmt(fmt) "ACPI: " fmt
#include <linux/module.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/sched.h>
#include <linux/pm.h>
#include <linux/device.h>
#include <linux/proc_fs.h>
#include <linux/acpi.h>
#include <linux/slab.h>
#include <linux/regulator/machine.h>
#include <linux/workqueue.h>
#include <linux/reboot.h>
#include <linux/delay.h>
#ifdef CONFIG_X86
#include <asm/mpspec.h>
#include <linux/dmi.h>
#endif
#include <linux/acpi_viot.h>
#include <linux/pci.h>
#include <acpi/apei.h>
#include <linux/suspend.h>
#include <linux/prmt.h>
#include "internal.h"
struct acpi_device *acpi_root;
struct proc_dir_entry *acpi_root_dir;
EXPORT_SYMBOL(acpi_root_dir);
#ifdef CONFIG_X86
#ifdef CONFIG_ACPI_CUSTOM_DSDT
static inline int set_copy_dsdt(const struct dmi_system_id *id)
{
return 0;
}
#else
static int set_copy_dsdt(const struct dmi_system_id *id)
{
pr_notice("%s detected - force copy of DSDT to local memory\n", id->ident);
acpi_gbl_copy_dsdt_locally = 1;
return 0;
}
#endif
static const struct dmi_system_id dsdt_dmi_table[] __initconst = {
/*
* Invoke DSDT corruption work-around on all Toshiba Satellite.
* https://bugzilla.kernel.org/show_bug.cgi?id=14679
*/
{
.callback = set_copy_dsdt,
.ident = "TOSHIBA Satellite",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
DMI_MATCH(DMI_PRODUCT_NAME, "Satellite"),
},
},
{}
};
#endif
/* --------------------------------------------------------------------------
Device Management
-------------------------------------------------------------------------- */
acpi_status acpi_bus_get_status_handle(acpi_handle handle,
unsigned long long *sta)
{
acpi_status status;
status = acpi_evaluate_integer(handle, "_STA", NULL, sta);
if (ACPI_SUCCESS(status))
return AE_OK;
if (status == AE_NOT_FOUND) {
*sta = ACPI_STA_DEVICE_PRESENT | ACPI_STA_DEVICE_ENABLED |
ACPI_STA_DEVICE_UI | ACPI_STA_DEVICE_FUNCTIONING;
return AE_OK;
}
return status;
}
EXPORT_SYMBOL_GPL(acpi_bus_get_status_handle);
int acpi_bus_get_status(struct acpi_device *device)
{
acpi_status status;
unsigned long long sta;
if (acpi_device_override_status(device, &sta)) {
acpi_set_device_status(device, sta);
return 0;
}
/* Battery devices must have their deps met before calling _STA */
if (acpi_device_is_battery(device) && device->dep_unmet) {
acpi_set_device_status(device, 0);
return 0;
}
status = acpi_bus_get_status_handle(device->handle, &sta);
if (ACPI_FAILURE(status))
return -ENODEV;
acpi_set_device_status(device, sta);
if (device->status.functional && !device->status.present) {
pr_debug("Device [%s] status [%08x]: functional but not present\n",
device->pnp.bus_id, (u32)sta);
}
pr_debug("Device [%s] status [%08x]\n", device->pnp.bus_id, (u32)sta);
return 0;
}
EXPORT_SYMBOL(acpi_bus_get_status);
void acpi_bus_private_data_handler(acpi_handle handle,
void *context)
{
return;
}
EXPORT_SYMBOL(acpi_bus_private_data_handler);
int acpi_bus_attach_private_data(acpi_handle handle, void *data)
{
acpi_status status;
status = acpi_attach_data(handle,
acpi_bus_private_data_handler, data);
if (ACPI_FAILURE(status)) {
acpi_handle_debug(handle, "Error attaching device data\n");
return -ENODEV;
}
return 0;
}
EXPORT_SYMBOL_GPL(acpi_bus_attach_private_data);
int acpi_bus_get_private_data(acpi_handle handle, void **data)
{
acpi_status status;
if (!data)
return -EINVAL;
status = acpi_get_data(handle, acpi_bus_private_data_handler, data);
if (ACPI_FAILURE(status)) {
acpi_handle_debug(handle, "No context for object\n");
return -ENODEV;
}
return 0;
}
EXPORT_SYMBOL_GPL(acpi_bus_get_private_data);
void acpi_bus_detach_private_data(acpi_handle handle)
{
acpi_detach_data(handle, acpi_bus_private_data_handler);
}
EXPORT_SYMBOL_GPL(acpi_bus_detach_private_data);
static void acpi_print_osc_error(acpi_handle handle,
struct acpi_osc_context *context, char *error)
{
int i;
acpi_handle_debug(handle, "(%s): %s\n", context->uuid_str, error);
pr_debug("_OSC request data:");
for (i = 0; i < context->cap.length; i += sizeof(u32))
pr_debug(" %x", *((u32 *)(context->cap.pointer + i)));
pr_debug("\n");
}
acpi_status acpi_run_osc(acpi_handle handle, struct acpi_osc_context *context)
{
acpi_status status;
struct acpi_object_list input;
union acpi_object in_params[4];
union acpi_object *out_obj;
guid_t guid;
u32 errors;
struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL};
if (!context)
return AE_ERROR;
if (guid_parse(context->uuid_str, &guid))
return AE_ERROR;
context->ret.length = ACPI_ALLOCATE_BUFFER;
context->ret.pointer = NULL;
/* Setting up input parameters */
input.count = 4;
input.pointer = in_params;
in_params[0].type = ACPI_TYPE_BUFFER;
in_params[0].buffer.length = 16;
in_params[0].buffer.pointer = (u8 *)&guid;
in_params[1].type = ACPI_TYPE_INTEGER;
in_params[1].integer.value = context->rev;
in_params[2].type = ACPI_TYPE_INTEGER;
in_params[2].integer.value = context->cap.length/sizeof(u32);
in_params[3].type = ACPI_TYPE_BUFFER;
in_params[3].buffer.length = context->cap.length;
in_params[3].buffer.pointer = context->cap.pointer;
status = acpi_evaluate_object(handle, "_OSC", &input, &output);
if (ACPI_FAILURE(status))
return status;
if (!output.length)
return AE_NULL_OBJECT;
out_obj = output.pointer;
if (out_obj->type != ACPI_TYPE_BUFFER
|| out_obj->buffer.length != context->cap.length) {
acpi_print_osc_error(handle, context,
"_OSC evaluation returned wrong type");
status = AE_TYPE;
goto out_kfree;
}
/* Need to ignore the bit0 in result code */
errors = *((u32 *)out_obj->buffer.pointer) & ~(1 << 0);
if (errors) {
if (errors & OSC_REQUEST_ERROR)
acpi_print_osc_error(handle, context,
"_OSC request failed");
if (errors & OSC_INVALID_UUID_ERROR)
acpi_print_osc_error(handle, context,
"_OSC invalid UUID");
if (errors & OSC_INVALID_REVISION_ERROR)
acpi_print_osc_error(handle, context,
"_OSC invalid revision");
if (errors & OSC_CAPABILITIES_MASK_ERROR) {
if (((u32 *)context->cap.pointer)[OSC_QUERY_DWORD]
& OSC_QUERY_ENABLE)
goto out_success;
status = AE_SUPPORT;
goto out_kfree;
}
status = AE_ERROR;
goto out_kfree;
}
out_success:
context->ret.length = out_obj->buffer.length;
context->ret.pointer = kmemdup(out_obj->buffer.pointer,
context->ret.length, GFP_KERNEL);
if (!context->ret.pointer) {
status = AE_NO_MEMORY;
goto out_kfree;
}
status = AE_OK;
out_kfree:
kfree(output.pointer);
return status;
}
EXPORT_SYMBOL(acpi_run_osc);
bool osc_sb_apei_support_acked;
/*
* ACPI 6.0 Section 8.4.4.2 Idle State Coordination
* OSPM supports platform coordinated low power idle(LPI) states
*/
bool osc_pc_lpi_support_confirmed;
EXPORT_SYMBOL_GPL(osc_pc_lpi_support_confirmed);
/*
* ACPI 6.2 Section 6.2.11.2 'Platform-Wide OSPM Capabilities':
* Starting with ACPI Specification 6.2, all _CPC registers can be in
* PCC, System Memory, System IO, or Functional Fixed Hardware address
* spaces. OSPM support for this more flexible register space scheme is
* indicated by the “Flexible Address Space for CPPC Registers” _OSC bit.
*
* Otherwise (cf ACPI 6.1, s8.4.7.1.1.X), _CPC registers must be in:
* - PCC or Functional Fixed Hardware address space if defined
* - SystemMemory address space (NULL register) if not defined
*/
bool osc_cpc_flexible_adr_space_confirmed;
EXPORT_SYMBOL_GPL(osc_cpc_flexible_adr_space_confirmed);
/*
* ACPI 6.4 Operating System Capabilities for USB.
*/
bool osc_sb_native_usb4_support_confirmed;
EXPORT_SYMBOL_GPL(osc_sb_native_usb4_support_confirmed);
bool osc_sb_cppc2_support_acked;
static u8 sb_uuid_str[] = "0811B06E-4A27-44F9-8D60-3CBBC22E7B48";
static void acpi_bus_osc_negotiate_platform_control(void)
{
u32 capbuf[2], *capbuf_ret;
struct acpi_osc_context context = {
.uuid_str = sb_uuid_str,
.rev = 1,
.cap.length = 8,
.cap.pointer = capbuf,
};
acpi_handle handle;
capbuf[OSC_QUERY_DWORD] = OSC_QUERY_ENABLE;
capbuf[OSC_SUPPORT_DWORD] = OSC_SB_PR3_SUPPORT; /* _PR3 is in use */
if (IS_ENABLED(CONFIG_ACPI_PROCESSOR_AGGREGATOR))
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PAD_SUPPORT;
if (IS_ENABLED(CONFIG_ACPI_PROCESSOR))
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PPC_OST_SUPPORT;
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_HOTPLUG_OST_SUPPORT;
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PCLPI_SUPPORT;
if (IS_ENABLED(CONFIG_ACPI_PRMT))
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PRM_SUPPORT;
if (IS_ENABLED(CONFIG_ACPI_FFH))
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_FFH_OPR_SUPPORT;
#ifdef CONFIG_ARM64
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_GENERIC_INITIATOR_SUPPORT;
#endif
#ifdef CONFIG_X86
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_GENERIC_INITIATOR_SUPPORT;
#endif
#ifdef CONFIG_ACPI_CPPC_LIB
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPC_SUPPORT;
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPCV2_SUPPORT;
#endif
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPC_FLEXIBLE_ADR_SPACE;
if (IS_ENABLED(CONFIG_SCHED_MC_PRIO))
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPC_DIVERSE_HIGH_SUPPORT;
if (IS_ENABLED(CONFIG_USB4))
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_NATIVE_USB4_SUPPORT;
if (!ghes_disable)
capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_APEI_SUPPORT;
if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &handle)))
return;
if (ACPI_FAILURE(acpi_run_osc(handle, &context)))
return;
capbuf_ret = context.ret.pointer;
if (context.ret.length <= OSC_SUPPORT_DWORD) {
kfree(context.ret.pointer);
return;
}
/*
* Now run _OSC again with query flag clear and with the caps
* supported by both the OS and the platform.
*/
capbuf[OSC_QUERY_DWORD] = 0;
capbuf[OSC_SUPPORT_DWORD] = capbuf_ret[OSC_SUPPORT_DWORD];
kfree(context.ret.pointer);
if (ACPI_FAILURE(acpi_run_osc(handle, &context)))
return;
capbuf_ret = context.ret.pointer;
if (context.ret.length > OSC_SUPPORT_DWORD) {
#ifdef CONFIG_ACPI_CPPC_LIB
osc_sb_cppc2_support_acked = capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_CPCV2_SUPPORT;
#endif
osc_sb_apei_support_acked =
capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_APEI_SUPPORT;
osc_pc_lpi_support_confirmed =
capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_PCLPI_SUPPORT;
osc_sb_native_usb4_support_confirmed =
capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_NATIVE_USB4_SUPPORT;
osc_cpc_flexible_adr_space_confirmed =
capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_CPC_FLEXIBLE_ADR_SPACE;
}
kfree(context.ret.pointer);
}
/*
* Native control of USB4 capabilities. If any of the tunneling bits is
* set it means OS is in control and we use software based connection
* manager.
*/
u32 osc_sb_native_usb4_control;
EXPORT_SYMBOL_GPL(osc_sb_native_usb4_control);
static void acpi_bus_decode_usb_osc(const char *msg, u32 bits)
{
pr_info("%s USB3%c DisplayPort%c PCIe%c XDomain%c\n", msg,
(bits & OSC_USB_USB3_TUNNELING) ? '+' : '-',
(bits & OSC_USB_DP_TUNNELING) ? '+' : '-',
(bits & OSC_USB_PCIE_TUNNELING) ? '+' : '-',
(bits & OSC_USB_XDOMAIN) ? '+' : '-');
}
static u8 sb_usb_uuid_str[] = "23A0D13A-26AB-486C-9C5F-0FFA525A575A";
static void acpi_bus_osc_negotiate_usb_control(void)
{
u32 capbuf[3], *capbuf_ret;
struct acpi_osc_context context = {
.uuid_str = sb_usb_uuid_str,
.rev = 1,
.cap.length = sizeof(capbuf),
.cap.pointer = capbuf,
};
acpi_handle handle;
acpi_status status;
u32 control;
if (!osc_sb_native_usb4_support_confirmed)
return;
if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &handle)))
return;
control = OSC_USB_USB3_TUNNELING | OSC_USB_DP_TUNNELING |
OSC_USB_PCIE_TUNNELING | OSC_USB_XDOMAIN;
/*
* Run _OSC first with query bit set, trying to get control over
* all tunneling. The platform can then clear out bits in the
* control dword that it does not want to grant to the OS.
*/
capbuf[OSC_QUERY_DWORD] = OSC_QUERY_ENABLE;
capbuf[OSC_SUPPORT_DWORD] = 0;
capbuf[OSC_CONTROL_DWORD] = control;
status = acpi_run_osc(handle, &context);
if (ACPI_FAILURE(status))
return;
if (context.ret.length != sizeof(capbuf)) {
pr_info("USB4 _OSC: returned invalid length buffer\n");
goto out_free;
}
/*
* Run _OSC again now with query bit clear and the control dword
* matching what the platform granted (which may not have all
* the control bits set).
*/
capbuf_ret = context.ret.pointer;
capbuf[OSC_QUERY_DWORD] = 0;
capbuf[OSC_CONTROL_DWORD] = capbuf_ret[OSC_CONTROL_DWORD];
kfree(context.ret.pointer);
status = acpi_run_osc(handle, &context);
if (ACPI_FAILURE(status))
return;
if (context.ret.length != sizeof(capbuf)) {
pr_info("USB4 _OSC: returned invalid length buffer\n");
goto out_free;
}
osc_sb_native_usb4_control =
control & acpi_osc_ctx_get_pci_control(&context);
acpi_bus_decode_usb_osc("USB4 _OSC: OS supports", control);
acpi_bus_decode_usb_osc("USB4 _OSC: OS controls",
osc_sb_native_usb4_control);
out_free:
kfree(context.ret.pointer);
}
/* --------------------------------------------------------------------------
Notification Handling
-------------------------------------------------------------------------- */
/**
* acpi_bus_notify - Global system-level (0x00-0x7F) notifications handler
* @handle: Target ACPI object.
* @type: Notification type.
* @data: Ignored.
*
* This only handles notifications related to device hotplug.
*/
static void acpi_bus_notify(acpi_handle handle, u32 type, void *data)
{
struct acpi_device *adev;
switch (type) {
case ACPI_NOTIFY_BUS_CHECK:
acpi_handle_debug(handle, "ACPI_NOTIFY_BUS_CHECK event\n");
break;
case ACPI_NOTIFY_DEVICE_CHECK:
acpi_handle_debug(handle, "ACPI_NOTIFY_DEVICE_CHECK event\n");
break;
case ACPI_NOTIFY_DEVICE_WAKE:
acpi_handle_debug(handle, "ACPI_NOTIFY_DEVICE_WAKE event\n");
return;
case ACPI_NOTIFY_EJECT_REQUEST:
acpi_handle_debug(handle, "ACPI_NOTIFY_EJECT_REQUEST event\n");
break;
case ACPI_NOTIFY_DEVICE_CHECK_LIGHT:
acpi_handle_debug(handle, "ACPI_NOTIFY_DEVICE_CHECK_LIGHT event\n");
/* TBD: Exactly what does 'light' mean? */
return;
case ACPI_NOTIFY_FREQUENCY_MISMATCH:
acpi_handle_err(handle, "Device cannot be configured due "
"to a frequency mismatch\n");
return;
case ACPI_NOTIFY_BUS_MODE_MISMATCH:
acpi_handle_err(handle, "Device cannot be configured due "
"to a bus mode mismatch\n");
return;
case ACPI_NOTIFY_POWER_FAULT:
acpi_handle_err(handle, "Device has suffered a power fault\n");
return;
default:
acpi_handle_debug(handle, "Unknown event type 0x%x\n", type);
return;
}
adev = acpi_get_acpi_dev(handle);
if (adev && ACPI_SUCCESS(acpi_hotplug_schedule(adev, type)))
return;
acpi_put_acpi_dev(adev);
acpi_evaluate_ost(handle, type, ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL);
}
static void acpi_notify_device(acpi_handle handle, u32 event, void *data)
{
struct acpi_device *device = data;
struct acpi_driver *acpi_drv = to_acpi_driver(device->dev.driver);
acpi_drv->ops.notify(device, event);
}
static int acpi_device_install_notify_handler(struct acpi_device *device,
struct acpi_driver *acpi_drv)
{
u32 type = acpi_drv->flags & ACPI_DRIVER_ALL_NOTIFY_EVENTS ?
ACPI_ALL_NOTIFY : ACPI_DEVICE_NOTIFY;
acpi_status status;
status = acpi_install_notify_handler(device->handle, type,
acpi_notify_device, device);
if (ACPI_FAILURE(status))
return -EINVAL;
return 0;
}
static void acpi_device_remove_notify_handler(struct acpi_device *device,
struct acpi_driver *acpi_drv)
{
u32 type = acpi_drv->flags & ACPI_DRIVER_ALL_NOTIFY_EVENTS ?
ACPI_ALL_NOTIFY : ACPI_DEVICE_NOTIFY;
acpi_remove_notify_handler(device->handle, type,
acpi_notify_device);
acpi_os_wait_events_complete();
}
int acpi_dev_install_notify_handler(struct acpi_device *adev,
u32 handler_type,
acpi_notify_handler handler, void *context)
{
acpi_status status;
status = acpi_install_notify_handler(adev->handle, handler_type,
handler, context);
if (ACPI_FAILURE(status))
return -ENODEV;
return 0;
}
EXPORT_SYMBOL_GPL(acpi_dev_install_notify_handler);
void acpi_dev_remove_notify_handler(struct acpi_device *adev,
u32 handler_type,
acpi_notify_handler handler)
{
acpi_remove_notify_handler(adev->handle, handler_type, handler);
acpi_os_wait_events_complete();
}
EXPORT_SYMBOL_GPL(acpi_dev_remove_notify_handler);
/* Handle events targeting \_SB device (at present only graceful shutdown) */
#define ACPI_SB_NOTIFY_SHUTDOWN_REQUEST 0x81
#define ACPI_SB_INDICATE_INTERVAL 10000
static void sb_notify_work(struct work_struct *dummy)
{
acpi_handle sb_handle;
orderly_poweroff(true);
/*
* After initiating graceful shutdown, the ACPI spec requires OSPM
* to evaluate _OST method once every 10seconds to indicate that
* the shutdown is in progress
*/
acpi_get_handle(NULL, "\\_SB", &sb_handle);
while (1) {
pr_info("Graceful shutdown in progress.\n");
acpi_evaluate_ost(sb_handle, ACPI_OST_EC_OSPM_SHUTDOWN,
ACPI_OST_SC_OS_SHUTDOWN_IN_PROGRESS, NULL);
msleep(ACPI_SB_INDICATE_INTERVAL);
}
}
static void acpi_sb_notify(acpi_handle handle, u32 event, void *data)
{
static DECLARE_WORK(acpi_sb_work, sb_notify_work);
if (event == ACPI_SB_NOTIFY_SHUTDOWN_REQUEST) {
if (!work_busy(&acpi_sb_work))
schedule_work(&acpi_sb_work);
} else {
pr_warn("event %x is not supported by \\_SB device\n", event);
}
}
static int __init acpi_setup_sb_notify_handler(void)
{
acpi_handle sb_handle;
if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &sb_handle)))
return -ENXIO;
if (ACPI_FAILURE(acpi_install_notify_handler(sb_handle, ACPI_DEVICE_NOTIFY,
acpi_sb_notify, NULL)))
return -EINVAL;
return 0;
}
/* --------------------------------------------------------------------------
Device Matching
-------------------------------------------------------------------------- */
/**
* acpi_get_first_physical_node - Get first physical node of an ACPI device
* @adev: ACPI device in question
*
* Return: First physical node of ACPI device @adev
*/
struct device *acpi_get_first_physical_node(struct acpi_device *adev)
{
struct mutex *physical_node_lock = &adev->physical_node_lock;
struct device *phys_dev;
mutex_lock(physical_node_lock);
if (list_empty(&adev->physical_node_list)) {
phys_dev = NULL;
} else {
const struct acpi_device_physical_node *node;
node = list_first_entry(&adev->physical_node_list,
struct acpi_device_physical_node, node);
phys_dev = node->dev;
}
mutex_unlock(physical_node_lock);
return phys_dev;
}
EXPORT_SYMBOL_GPL(acpi_get_first_physical_node);
static struct acpi_device *acpi_primary_dev_companion(struct acpi_device *adev,
const struct device *dev)
{
const struct device *phys_dev = acpi_get_first_physical_node(adev); return phys_dev && phys_dev == dev ? adev : NULL;
}
/**
* acpi_device_is_first_physical_node - Is given dev first physical node
* @adev: ACPI companion device
* @dev: Physical device to check
*
* Function checks if given @dev is the first physical devices attached to
* the ACPI companion device. This distinction is needed in some cases
* where the same companion device is shared between many physical devices.
*
* Note that the caller have to provide valid @adev pointer.
*/
bool acpi_device_is_first_physical_node(struct acpi_device *adev,
const struct device *dev)
{
return !!acpi_primary_dev_companion(adev, dev);
}
/*
* acpi_companion_match() - Can we match via ACPI companion device
* @dev: Device in question
*
* Check if the given device has an ACPI companion and if that companion has
* a valid list of PNP IDs, and if the device is the first (primary) physical
* device associated with it. Return the companion pointer if that's the case
* or NULL otherwise.
*
* If multiple physical devices are attached to a single ACPI companion, we need
* to be careful. The usage scenario for this kind of relationship is that all
* of the physical devices in question use resources provided by the ACPI
* companion. A typical case is an MFD device where all the sub-devices share
* the parent's ACPI companion. In such cases we can only allow the primary
* (first) physical device to be matched with the help of the companion's PNP
* IDs.
*
* Additional physical devices sharing the ACPI companion can still use
* resources available from it but they will be matched normally using functions
* provided by their bus types (and analogously for their modalias).
*/
const struct acpi_device *acpi_companion_match(const struct device *dev)
{
struct acpi_device *adev;
adev = ACPI_COMPANION(dev);
if (!adev)
return NULL;
if (list_empty(&adev->pnp.ids))
return NULL;
return acpi_primary_dev_companion(adev, dev);
}
/**
* acpi_of_match_device - Match device object using the "compatible" property.
* @adev: ACPI device object to match.
* @of_match_table: List of device IDs to match against.
* @of_id: OF ID if matched
*
* If @dev has an ACPI companion which has ACPI_DT_NAMESPACE_HID in its list of
* identifiers and a _DSD object with the "compatible" property, use that
* property to match against the given list of identifiers.
*/
static bool acpi_of_match_device(const struct acpi_device *adev,
const struct of_device_id *of_match_table,
const struct of_device_id **of_id)
{
const union acpi_object *of_compatible, *obj;
int i, nval;
if (!adev)
return false;
of_compatible = adev->data.of_compatible;
if (!of_match_table || !of_compatible)
return false;
if (of_compatible->type == ACPI_TYPE_PACKAGE) {
nval = of_compatible->package.count;
obj = of_compatible->package.elements;
} else { /* Must be ACPI_TYPE_STRING. */
nval = 1;
obj = of_compatible;
}
/* Now we can look for the driver DT compatible strings */
for (i = 0; i < nval; i++, obj++) {
const struct of_device_id *id;
for (id = of_match_table; id->compatible[0]; id++)
if (!strcasecmp(obj->string.pointer, id->compatible)) {
if (of_id)
*of_id = id;
return true;
}
}
return false;
}
static bool acpi_of_modalias(struct acpi_device *adev,
char *modalias, size_t len)
{
const union acpi_object *of_compatible;
const union acpi_object *obj;
const char *str, *chr;
of_compatible = adev->data.of_compatible;
if (!of_compatible)
return false;
if (of_compatible->type == ACPI_TYPE_PACKAGE)
obj = of_compatible->package.elements;
else /* Must be ACPI_TYPE_STRING. */
obj = of_compatible;
str = obj->string.pointer;
chr = strchr(str, ',');
strscpy(modalias, chr ? chr + 1 : str, len);
return true;
}
/**
* acpi_set_modalias - Set modalias using "compatible" property or supplied ID
* @adev: ACPI device object to match
* @default_id: ID string to use as default if no compatible string found
* @modalias: Pointer to buffer that modalias value will be copied into
* @len: Length of modalias buffer
*
* This is a counterpart of of_alias_from_compatible() for struct acpi_device
* objects. If there is a compatible string for @adev, it will be copied to
* @modalias with the vendor prefix stripped; otherwise, @default_id will be
* used.
*/
void acpi_set_modalias(struct acpi_device *adev, const char *default_id,
char *modalias, size_t len)
{
if (!acpi_of_modalias(adev, modalias, len))
strscpy(modalias, default_id, len);
}
EXPORT_SYMBOL_GPL(acpi_set_modalias);
static bool __acpi_match_device_cls(const struct acpi_device_id *id,
struct acpi_hardware_id *hwid)
{
int i, msk, byte_shift;
char buf[3];
if (!id->cls)
return false;
/* Apply class-code bitmask, before checking each class-code byte */
for (i = 1; i <= 3; i++) {
byte_shift = 8 * (3 - i);
msk = (id->cls_msk >> byte_shift) & 0xFF;
if (!msk)
continue;
sprintf(buf, "%02x", (id->cls >> byte_shift) & msk);
if (strncmp(buf, &hwid->id[(i - 1) * 2], 2))
return false;
}
return true;
}
static bool __acpi_match_device(const struct acpi_device *device,
const struct acpi_device_id *acpi_ids,
const struct of_device_id *of_ids,
const struct acpi_device_id **acpi_id,
const struct of_device_id **of_id)
{
const struct acpi_device_id *id;
struct acpi_hardware_id *hwid;
/*
* If the device is not present, it is unnecessary to load device
* driver for it.
*/
if (!device || !device->status.present)
return false;
list_for_each_entry(hwid, &device->pnp.ids, list) {
/* First, check the ACPI/PNP IDs provided by the caller. */
if (acpi_ids) {
for (id = acpi_ids; id->id[0] || id->cls; id++) {
if (id->id[0] && !strcmp((char *)id->id, hwid->id))
goto out_acpi_match;
if (id->cls && __acpi_match_device_cls(id, hwid))
goto out_acpi_match;
}
}
/*
* Next, check ACPI_DT_NAMESPACE_HID and try to match the
* "compatible" property if found.
*/
if (!strcmp(ACPI_DT_NAMESPACE_HID, hwid->id))
return acpi_of_match_device(device, of_ids, of_id);
}
return false;
out_acpi_match:
if (acpi_id)
*acpi_id = id;
return true;
}
/**
* acpi_match_acpi_device - Match an ACPI device against a given list of ACPI IDs
* @ids: Array of struct acpi_device_id objects to match against.
* @adev: The ACPI device pointer to match.
*
* Match the ACPI device @adev against a given list of ACPI IDs @ids.
*
* Return:
* a pointer to the first matching ACPI ID on success or %NULL on failure.
*/
const struct acpi_device_id *acpi_match_acpi_device(const struct acpi_device_id *ids,
const struct acpi_device *adev)
{
const struct acpi_device_id *id = NULL;
__acpi_match_device(adev, ids, NULL, &id, NULL);
return id;
}
EXPORT_SYMBOL_GPL(acpi_match_acpi_device);
/**
* acpi_match_device - Match a struct device against a given list of ACPI IDs
* @ids: Array of struct acpi_device_id object to match against.
* @dev: The device structure to match.
*
* Check if @dev has a valid ACPI handle and if there is a struct acpi_device
* object for that handle and use that object to match against a given list of
* device IDs.
*
* Return a pointer to the first matching ID on success or %NULL on failure.
*/
const struct acpi_device_id *acpi_match_device(const struct acpi_device_id *ids,
const struct device *dev)
{
return acpi_match_acpi_device(ids, acpi_companion_match(dev));
}
EXPORT_SYMBOL_GPL(acpi_match_device);
static const void *acpi_of_device_get_match_data(const struct device *dev)
{
struct acpi_device *adev = ACPI_COMPANION(dev);
const struct of_device_id *match = NULL;
if (!acpi_of_match_device(adev, dev->driver->of_match_table, &match))
return NULL;
return match->data;
}
const void *acpi_device_get_match_data(const struct device *dev)
{
const struct acpi_device_id *acpi_ids = dev->driver->acpi_match_table;
const struct acpi_device_id *match;
if (!acpi_ids)
return acpi_of_device_get_match_data(dev);
match = acpi_match_device(acpi_ids, dev);
if (!match)
return NULL;
return (const void *)match->driver_data;
}
EXPORT_SYMBOL_GPL(acpi_device_get_match_data);
int acpi_match_device_ids(struct acpi_device *device,
const struct acpi_device_id *ids)
{
return __acpi_match_device(device, ids, NULL, NULL, NULL) ? 0 : -ENOENT;
}
EXPORT_SYMBOL(acpi_match_device_ids);
bool acpi_driver_match_device(struct device *dev,
const struct device_driver *drv)
{
const struct acpi_device_id *acpi_ids = drv->acpi_match_table;
const struct of_device_id *of_ids = drv->of_match_table;
if (!acpi_ids)
return acpi_of_match_device(ACPI_COMPANION(dev), of_ids, NULL);
return __acpi_match_device(acpi_companion_match(dev), acpi_ids, of_ids, NULL, NULL);
}
EXPORT_SYMBOL_GPL(acpi_driver_match_device);
/* --------------------------------------------------------------------------
ACPI Driver Management
-------------------------------------------------------------------------- */
/**
* acpi_bus_register_driver - register a driver with the ACPI bus
* @driver: driver being registered
*
* Registers a driver with the ACPI bus. Searches the namespace for all
* devices that match the driver's criteria and binds. Returns zero for
* success or a negative error status for failure.
*/
int acpi_bus_register_driver(struct acpi_driver *driver)
{
if (acpi_disabled)
return -ENODEV;
driver->drv.name = driver->name;
driver->drv.bus = &acpi_bus_type;
driver->drv.owner = driver->owner;
return driver_register(&driver->drv);
}
EXPORT_SYMBOL(acpi_bus_register_driver);
/**
* acpi_bus_unregister_driver - unregisters a driver with the ACPI bus
* @driver: driver to unregister
*
* Unregisters a driver with the ACPI bus. Searches the namespace for all
* devices that match the driver's criteria and unbinds.
*/
void acpi_bus_unregister_driver(struct acpi_driver *driver)
{
driver_unregister(&driver->drv);
}
EXPORT_SYMBOL(acpi_bus_unregister_driver);
/* --------------------------------------------------------------------------
ACPI Bus operations
-------------------------------------------------------------------------- */
static int acpi_bus_match(struct device *dev, struct device_driver *drv)
{
struct acpi_device *acpi_dev = to_acpi_device(dev);
struct acpi_driver *acpi_drv = to_acpi_driver(drv);
return acpi_dev->flags.match_driver
&& !acpi_match_device_ids(acpi_dev, acpi_drv->ids);
}
static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
return __acpi_device_uevent_modalias(to_acpi_device(dev), env);
}
static int acpi_device_probe(struct device *dev)
{
struct acpi_device *acpi_dev = to_acpi_device(dev);
struct acpi_driver *acpi_drv = to_acpi_driver(dev->driver);
int ret;
if (acpi_dev->handler && !acpi_is_pnp_device(acpi_dev))
return -EINVAL;
if (!acpi_drv->ops.add)
return -ENOSYS;
ret = acpi_drv->ops.add(acpi_dev);
if (ret) {
acpi_dev->driver_data = NULL;
return ret;
}
pr_debug("Driver [%s] successfully bound to device [%s]\n",
acpi_drv->name, acpi_dev->pnp.bus_id);
if (acpi_drv->ops.notify) {
ret = acpi_device_install_notify_handler(acpi_dev, acpi_drv);
if (ret) {
if (acpi_drv->ops.remove)
acpi_drv->ops.remove(acpi_dev);
acpi_dev->driver_data = NULL;
return ret;
}
}
pr_debug("Found driver [%s] for device [%s]\n", acpi_drv->name,
acpi_dev->pnp.bus_id);
get_device(dev);
return 0;
}
static void acpi_device_remove(struct device *dev)
{
struct acpi_device *acpi_dev = to_acpi_device(dev);
struct acpi_driver *acpi_drv = to_acpi_driver(dev->driver);
if (acpi_drv->ops.notify)
acpi_device_remove_notify_handler(acpi_dev, acpi_drv);
if (acpi_drv->ops.remove)
acpi_drv->ops.remove(acpi_dev);
acpi_dev->driver_data = NULL;
put_device(dev);
}
const struct bus_type acpi_bus_type = {
.name = "acpi",
.match = acpi_bus_match,
.probe = acpi_device_probe,
.remove = acpi_device_remove,
.uevent = acpi_device_uevent,
};
int acpi_bus_for_each_dev(int (*fn)(struct device *, void *), void *data)
{
return bus_for_each_dev(&acpi_bus_type, NULL, data, fn);
}
EXPORT_SYMBOL_GPL(acpi_bus_for_each_dev);
struct acpi_dev_walk_context {
int (*fn)(struct acpi_device *, void *);
void *data;
};
static int acpi_dev_for_one_check(struct device *dev, void *context)
{
struct acpi_dev_walk_context *adwc = context;
if (dev->bus != &acpi_bus_type)
return 0;
return adwc->fn(to_acpi_device(dev), adwc->data);
}
EXPORT_SYMBOL_GPL(acpi_dev_for_each_child);
int acpi_dev_for_each_child(struct acpi_device *adev,
int (*fn)(struct acpi_device *, void *), void *data)
{
struct acpi_dev_walk_context adwc = {
.fn = fn,
.data = data,
};
return device_for_each_child(&adev->dev, &adwc, acpi_dev_for_one_check);
}
int acpi_dev_for_each_child_reverse(struct acpi_device *adev,
int (*fn)(struct acpi_device *, void *),
void *data)
{
struct acpi_dev_walk_context adwc = {
.fn = fn,
.data = data,
};
return device_for_each_child_reverse(&adev->dev, &adwc, acpi_dev_for_one_check);
}
/* --------------------------------------------------------------------------
Initialization/Cleanup
-------------------------------------------------------------------------- */
static int __init acpi_bus_init_irq(void)
{
acpi_status status;
char *message = NULL;
/*
* Let the system know what interrupt model we are using by
* evaluating the \_PIC object, if exists.
*/
switch (acpi_irq_model) {
case ACPI_IRQ_MODEL_PIC:
message = "PIC";
break;
case ACPI_IRQ_MODEL_IOAPIC:
message = "IOAPIC";
break;
case ACPI_IRQ_MODEL_IOSAPIC:
message = "IOSAPIC";
break;
case ACPI_IRQ_MODEL_GIC:
message = "GIC";
break;
case ACPI_IRQ_MODEL_PLATFORM:
message = "platform specific model";
break;
case ACPI_IRQ_MODEL_LPIC:
message = "LPIC";
break;
default:
pr_info("Unknown interrupt routing model\n");
return -ENODEV;
}
pr_info("Using %s for interrupt routing\n", message);
status = acpi_execute_simple_method(NULL, "\\_PIC", acpi_irq_model);
if (ACPI_FAILURE(status) && (status != AE_NOT_FOUND)) {
pr_info("_PIC evaluation failed: %s\n", acpi_format_exception(status));
return -ENODEV;
}
return 0;
}
/**
* acpi_early_init - Initialize ACPICA and populate the ACPI namespace.
*
* The ACPI tables are accessible after this, but the handling of events has not
* been initialized and the global lock is not available yet, so AML should not
* be executed at this point.
*
* Doing this before switching the EFI runtime services to virtual mode allows
* the EfiBootServices memory to be freed slightly earlier on boot.
*/
void __init acpi_early_init(void)
{
acpi_status status;
if (acpi_disabled)
return;
pr_info("Core revision %08x\n", ACPI_CA_VERSION);
/* enable workarounds, unless strict ACPI spec. compliance */
if (!acpi_strict)
acpi_gbl_enable_interpreter_slack = TRUE;
acpi_permanent_mmap = true;
#ifdef CONFIG_X86
/*
* If the machine falls into the DMI check table,
* DSDT will be copied to memory.
* Note that calling dmi_check_system() here on other architectures
* would not be OK because only x86 initializes dmi early enough.
* Thankfully only x86 systems need such quirks for now.
*/
dmi_check_system(dsdt_dmi_table);
#endif
status = acpi_reallocate_root_table();
if (ACPI_FAILURE(status)) {
pr_err("Unable to reallocate ACPI tables\n");
goto error0;
}
status = acpi_initialize_subsystem();
if (ACPI_FAILURE(status)) {
pr_err("Unable to initialize the ACPI Interpreter\n");
goto error0;
}
#ifdef CONFIG_X86
if (!acpi_ioapic) {
/* compatible (0) means level (3) */
if (!(acpi_sci_flags & ACPI_MADT_TRIGGER_MASK)) {
acpi_sci_flags &= ~ACPI_MADT_TRIGGER_MASK;
acpi_sci_flags |= ACPI_MADT_TRIGGER_LEVEL;
}
/* Set PIC-mode SCI trigger type */
acpi_pic_sci_set_trigger(acpi_gbl_FADT.sci_interrupt,
(acpi_sci_flags & ACPI_MADT_TRIGGER_MASK) >> 2);
} else {
/*
* now that acpi_gbl_FADT is initialized,
* update it with result from INT_SRC_OVR parsing
*/
acpi_gbl_FADT.sci_interrupt = acpi_sci_override_gsi;
}
#endif
return;
error0:
disable_acpi();
}
/**
* acpi_subsystem_init - Finalize the early initialization of ACPI.
*
* Switch over the platform to the ACPI mode (if possible).
*
* Doing this too early is generally unsafe, but at the same time it needs to be
* done before all things that really depend on ACPI. The right spot appears to
* be before finalizing the EFI initialization.
*/
void __init acpi_subsystem_init(void)
{
acpi_status status;
if (acpi_disabled)
return;
status = acpi_enable_subsystem(~ACPI_NO_ACPI_ENABLE);
if (ACPI_FAILURE(status)) {
pr_err("Unable to enable ACPI\n");
disable_acpi();
} else {
/*
* If the system is using ACPI then we can be reasonably
* confident that any regulators are managed by the firmware
* so tell the regulator core it has everything it needs to
* know.
*/
regulator_has_full_constraints();
}
}
static acpi_status acpi_bus_table_handler(u32 event, void *table, void *context)
{
if (event == ACPI_TABLE_EVENT_LOAD)
acpi_scan_table_notify();
return acpi_sysfs_table_handler(event, table, context);
}
static int __init acpi_bus_init(void)
{
int result;
acpi_status status;
acpi_os_initialize1();
status = acpi_load_tables();
if (ACPI_FAILURE(status)) {
pr_err("Unable to load the System Description Tables\n");
goto error1;
}
/*
* ACPI 2.0 requires the EC driver to be loaded and work before the EC
* device is found in the namespace.
*
* This is accomplished by looking for the ECDT table and getting the EC
* parameters out of that.
*
* Do that before calling acpi_initialize_objects() which may trigger EC
* address space accesses.
*/
acpi_ec_ecdt_probe();
status = acpi_enable_subsystem(ACPI_NO_ACPI_ENABLE);
if (ACPI_FAILURE(status)) {
pr_err("Unable to start the ACPI Interpreter\n");
goto error1;
}
status = acpi_initialize_objects(ACPI_FULL_INITIALIZATION);
if (ACPI_FAILURE(status)) {
pr_err("Unable to initialize ACPI objects\n");
goto error1;
}
/*
* _OSC method may exist in module level code,
* so it must be run after ACPI_FULL_INITIALIZATION
*/
acpi_bus_osc_negotiate_platform_control();
acpi_bus_osc_negotiate_usb_control();
/*
* _PDC control method may load dynamic SSDT tables,
* and we need to install the table handler before that.
*/
status = acpi_install_table_handler(acpi_bus_table_handler, NULL);
acpi_sysfs_init();
acpi_early_processor_control_setup();
/*
* Maybe EC region is required at bus_scan/acpi_get_devices. So it
* is necessary to enable it as early as possible.
*/
acpi_ec_dsdt_probe();
pr_info("Interpreter enabled\n");
/* Initialize sleep structures */
acpi_sleep_init();
/*
* Get the system interrupt model and evaluate \_PIC.
*/
result = acpi_bus_init_irq();
if (result)
goto error1;
/*
* Register the for all standard device notifications.
*/
status =
acpi_install_notify_handler(ACPI_ROOT_OBJECT, ACPI_SYSTEM_NOTIFY,
&acpi_bus_notify, NULL);
if (ACPI_FAILURE(status)) {
pr_err("Unable to register for system notifications\n");
goto error1;
}
/*
* Create the top ACPI proc directory
*/
acpi_root_dir = proc_mkdir(ACPI_BUS_FILE_ROOT, NULL);
result = bus_register(&acpi_bus_type);
if (!result)
return 0;
/* Mimic structured exception handling */
error1:
acpi_terminate();
return -ENODEV;
}
struct kobject *acpi_kobj;
EXPORT_SYMBOL_GPL(acpi_kobj);
static int __init acpi_init(void)
{
int result;
if (acpi_disabled) {
pr_info("Interpreter disabled.\n");
return -ENODEV;
}
acpi_kobj = kobject_create_and_add("acpi", firmware_kobj);
if (!acpi_kobj)
pr_debug("%s: kset create error\n", __func__);
init_prmt();
acpi_init_pcc();
result = acpi_bus_init();
if (result) {
kobject_put(acpi_kobj);
disable_acpi();
return result;
}
acpi_init_ffh();
pci_mmcfg_late_init();
acpi_viot_early_init();
acpi_hest_init();
acpi_ghes_init();
acpi_arm_init();
acpi_scan_init();
acpi_ec_init();
acpi_debugfs_init();
acpi_sleep_proc_init();
acpi_wakeup_device_init();
acpi_debugger_init();
acpi_setup_sb_notify_handler();
acpi_viot_init();
return 0;
}
subsys_initcall(acpi_init);
// SPDX-License-Identifier: GPL-2.0-only
/*
* mm/truncate.c - code for taking down pages from address_spaces
*
* Copyright (C) 2002, Linus Torvalds
*
* 10Sep2002 Andrew Morton
* Initial version.
*/
#include <linux/kernel.h>
#include <linux/backing-dev.h>
#include <linux/dax.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/export.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/pagevec.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/shmem_fs.h>
#include <linux/rmap.h>
#include "internal.h"
/*
* Regular page slots are stabilized by the page lock even without the tree
* itself locked. These unlocked entries need verification under the tree
* lock.
*/
static inline void __clear_shadow_entry(struct address_space *mapping,
pgoff_t index, void *entry)
{
XA_STATE(xas, &mapping->i_pages, index);
xas_set_update(&xas, workingset_update_node);
if (xas_load(&xas) != entry)
return;
xas_store(&xas, NULL);
}
static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
void *entry)
{
spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages);
__clear_shadow_entry(mapping, index, entry);
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping))
inode_add_lru(mapping->host);
spin_unlock(&mapping->host->i_lock);
}
/*
* Unconditionally remove exceptional entries. Usually called from truncate
* path. Note that the folio_batch may be altered by this function by removing
* exceptional entries similar to what folio_batch_remove_exceptionals() does.
*/
static void truncate_folio_batch_exceptionals(struct address_space *mapping,
struct folio_batch *fbatch, pgoff_t *indices)
{
int i, j;
bool dax;
/* Handled by shmem itself */
if (shmem_mapping(mapping))
return;
for (j = 0; j < folio_batch_count(fbatch); j++)
if (xa_is_value(fbatch->folios[j]))
break;
if (j == folio_batch_count(fbatch))
return;
dax = dax_mapping(mapping);
if (!dax) {
spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages);
}
for (i = j; i < folio_batch_count(fbatch); i++) {
struct folio *folio = fbatch->folios[i];
pgoff_t index = indices[i];
if (!xa_is_value(folio)) {
fbatch->folios[j++] = folio;
continue;
}
if (unlikely(dax)) {
dax_delete_mapping_entry(mapping, index);
continue;
}
__clear_shadow_entry(mapping, index, folio);
}
if (!dax) {
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping))
inode_add_lru(mapping->host);
spin_unlock(&mapping->host->i_lock);
}
fbatch->nr = j;
}
/*
* Invalidate exceptional entry if easily possible. This handles exceptional
* entries for invalidate_inode_pages().
*/
static int invalidate_exceptional_entry(struct address_space *mapping,
pgoff_t index, void *entry)
{
/* Handled by shmem itself, or for DAX we do nothing. */
if (shmem_mapping(mapping) || dax_mapping(mapping))
return 1;
clear_shadow_entry(mapping, index, entry);
return 1;
}
/*
* Invalidate exceptional entry if clean. This handles exceptional entries for
* invalidate_inode_pages2() so for DAX it evicts only clean entries.
*/
static int invalidate_exceptional_entry2(struct address_space *mapping,
pgoff_t index, void *entry)
{
/* Handled by shmem itself */
if (shmem_mapping(mapping))
return 1;
if (dax_mapping(mapping))
return dax_invalidate_mapping_entry_sync(mapping, index);
clear_shadow_entry(mapping, index, entry);
return 1;
}
/**
* folio_invalidate - Invalidate part or all of a folio.
* @folio: The folio which is affected.
* @offset: start of the range to invalidate
* @length: length of the range to invalidate
*
* folio_invalidate() is called when all or part of the folio has become
* invalidated by a truncate operation.
*
* folio_invalidate() does not have to release all buffers, but it must
* ensure that no dirty buffer is left outside @offset and that no I/O
* is underway against any of the blocks which are outside the truncation
* point. Because the caller is about to free (and possibly reuse) those
* blocks on-disk.
*/
void folio_invalidate(struct folio *folio, size_t offset, size_t length)
{
const struct address_space_operations *aops = folio->mapping->a_ops;
if (aops->invalidate_folio)
aops->invalidate_folio(folio, offset, length);
}
EXPORT_SYMBOL_GPL(folio_invalidate);
/*
* If truncate cannot remove the fs-private metadata from the page, the page
* becomes orphaned. It will be left on the LRU and may even be mapped into
* user pagetables if we're racing with filemap_fault().
*
* We need to bail out if page->mapping is no longer equal to the original
* mapping. This happens a) when the VM reclaimed the page while we waited on
* its lock, b) when a concurrent invalidate_mapping_pages got there first and
* c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
*/
static void truncate_cleanup_folio(struct folio *folio)
{
if (folio_mapped(folio)) unmap_mapping_folio(folio); if (folio_has_private(folio)) folio_invalidate(folio, 0, folio_size(folio));
/*
* Some filesystems seem to re-dirty the page even after
* the VM has canceled the dirty bit (eg ext3 journaling).
* Hence dirty accounting check is placed after invalidation.
*/
folio_cancel_dirty(folio); folio_clear_mappedtodisk(folio);
}
int truncate_inode_folio(struct address_space *mapping, struct folio *folio)
{
if (folio->mapping != mapping)
return -EIO;
truncate_cleanup_folio(folio);
filemap_remove_folio(folio);
return 0;
}
/*
* Handle partial folios. The folio may be entirely within the
* range if a split has raced with us. If not, we zero the part of the
* folio that's within the [start, end] range, and then split the folio if
* it's large. split_page_range() will discard pages which now lie beyond
* i_size, and we rely on the caller to discard pages which lie within a
* newly created hole.
*
* Returns false if splitting failed so the caller can avoid
* discarding the entire folio which is stubbornly unsplit.
*/
bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
{
loff_t pos = folio_pos(folio);
unsigned int offset, length;
if (pos < start)
offset = start - pos;
else
offset = 0;
length = folio_size(folio); if (pos + length <= (u64)end) length = length - offset;
else
length = end + 1 - pos - offset; folio_wait_writeback(folio); if (length == folio_size(folio)) { truncate_inode_folio(folio->mapping, folio);
return true;
}
/*
* We may be zeroing pages we're about to discard, but it avoids
* doing a complex calculation here, and then doing the zeroing
* anyway if the page split fails.
*/
folio_zero_range(folio, offset, length); if (folio_has_private(folio)) folio_invalidate(folio, offset, length); if (!folio_test_large(folio))
return true;
if (split_folio(folio) == 0)
return true;
if (folio_test_dirty(folio))
return false;
truncate_inode_folio(folio->mapping, folio);
return true;
}
/*
* Used to get rid of pages on hardware memory corruption.
*/
int generic_error_remove_folio(struct address_space *mapping,
struct folio *folio)
{
if (!mapping)
return -EINVAL;
/*
* Only punch for normal data pages for now.
* Handling other types like directories would need more auditing.
*/
if (!S_ISREG(mapping->host->i_mode))
return -EIO;
return truncate_inode_folio(mapping, folio);
}
EXPORT_SYMBOL(generic_error_remove_folio);
/**
* mapping_evict_folio() - Remove an unused folio from the page-cache.
* @mapping: The mapping this folio belongs to.
* @folio: The folio to remove.
*
* Safely remove one folio from the page cache.
* It only drops clean, unused folios.
*
* Context: Folio must be locked.
* Return: The number of pages successfully removed.
*/
long mapping_evict_folio(struct address_space *mapping, struct folio *folio)
{
/* The page may have been truncated before it was locked */
if (!mapping)
return 0;
if (folio_test_dirty(folio) || folio_test_writeback(folio))
return 0;
/* The refcount will be elevated if any page in the folio is mapped */
if (folio_ref_count(folio) >
folio_nr_pages(folio) + folio_has_private(folio) + 1)
return 0;
if (!filemap_release_folio(folio, 0))
return 0;
return remove_mapping(mapping, folio);
}
/**
* truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
* @mapping: mapping to truncate
* @lstart: offset from which to truncate
* @lend: offset to which to truncate (inclusive)
*
* Truncate the page cache, removing the pages that are between
* specified offsets (and zeroing out partial pages
* if lstart or lend + 1 is not page aligned).
*
* Truncate takes two passes - the first pass is nonblocking. It will not
* block on page locks and it will not block on writeback. The second pass
* will wait. This is to prevent as much IO as possible in the affected region.
* The first pass will remove most pages, so the search cost of the second pass
* is low.
*
* We pass down the cache-hot hint to the page freeing code. Even if the
* mapping is large, it is probably the case that the final pages are the most
* recently touched, and freeing happens in ascending file offset order.
*
* Note that since ->invalidate_folio() accepts range to invalidate
* truncate_inode_pages_range is able to handle cases where lend + 1 is not
* page aligned properly.
*/
void truncate_inode_pages_range(struct address_space *mapping,
loff_t lstart, loff_t lend)
{
pgoff_t start; /* inclusive */
pgoff_t end; /* exclusive */
struct folio_batch fbatch;
pgoff_t indices[PAGEVEC_SIZE];
pgoff_t index;
int i;
struct folio *folio;
bool same_folio;
if (mapping_empty(mapping))
return;
/*
* 'start' and 'end' always covers the range of pages to be fully
* truncated. Partial pages are covered with 'partial_start' at the
* start of the range and 'partial_end' at the end of the range.
* Note that 'end' is exclusive while 'lend' is inclusive.
*/
start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (lend == -1)
/*
* lend == -1 indicates end-of-file so we have to set 'end'
* to the highest possible pgoff_t and since the type is
* unsigned we're using -1.
*/
end = -1;
else
end = (lend + 1) >> PAGE_SHIFT; folio_batch_init(&fbatch);
index = start;
while (index < end && find_lock_entries(mapping, &index, end - 1,
&fbatch, indices)) {
truncate_folio_batch_exceptionals(mapping, &fbatch, indices); for (i = 0; i < folio_batch_count(&fbatch); i++) truncate_cleanup_folio(fbatch.folios[i]); delete_from_page_cache_batch(mapping, &fbatch);
for (i = 0; i < folio_batch_count(&fbatch); i++)
folio_unlock(fbatch.folios[i]); folio_batch_release(&fbatch); cond_resched();
}
same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
folio = __filemap_get_folio(mapping, lstart >> PAGE_SHIFT, FGP_LOCK, 0);
if (!IS_ERR(folio)) {
same_folio = lend < folio_pos(folio) + folio_size(folio);
if (!truncate_inode_partial_folio(folio, lstart, lend)) {
start = folio_next_index(folio);
if (same_folio)
end = folio->index;
}
folio_unlock(folio); folio_put(folio);
folio = NULL;
}
if (!same_folio) { folio = __filemap_get_folio(mapping, lend >> PAGE_SHIFT,
FGP_LOCK, 0);
if (!IS_ERR(folio)) {
if (!truncate_inode_partial_folio(folio, lstart, lend)) end = folio->index; folio_unlock(folio); folio_put(folio);
}
}
index = start; while (index < end) { cond_resched(); if (!find_get_entries(mapping, &index, end - 1, &fbatch,
indices)) {
/* If all gone from start onwards, we're done */
if (index == start)
break;
/* Otherwise restart to make sure all gone */
index = start;
continue;
}
for (i = 0; i < folio_batch_count(&fbatch); i++) { struct folio *folio = fbatch.folios[i];
/* We rely upon deletion not changing page->index */
if (xa_is_value(folio))
continue;
folio_lock(folio); VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio); folio_wait_writeback(folio); truncate_inode_folio(mapping, folio); folio_unlock(folio);
}
truncate_folio_batch_exceptionals(mapping, &fbatch, indices); folio_batch_release(&fbatch);
}
}
EXPORT_SYMBOL(truncate_inode_pages_range);
/**
* truncate_inode_pages - truncate *all* the pages from an offset
* @mapping: mapping to truncate
* @lstart: offset from which to truncate
*
* Called under (and serialised by) inode->i_rwsem and
* mapping->invalidate_lock.
*
* Note: When this function returns, there can be a page in the process of
* deletion (inside __filemap_remove_folio()) in the specified range. Thus
* mapping->nrpages can be non-zero when this function returns even after
* truncation of the whole mapping.
*/
void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
{
truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
}
EXPORT_SYMBOL(truncate_inode_pages);
/**
* truncate_inode_pages_final - truncate *all* pages before inode dies
* @mapping: mapping to truncate
*
* Called under (and serialized by) inode->i_rwsem.
*
* Filesystems have to use this in the .evict_inode path to inform the
* VM that this is the final truncate and the inode is going away.
*/
void truncate_inode_pages_final(struct address_space *mapping)
{
/*
* Page reclaim can not participate in regular inode lifetime
* management (can't call iput()) and thus can race with the
* inode teardown. Tell it when the address space is exiting,
* so that it does not install eviction information after the
* final truncate has begun.
*/
mapping_set_exiting(mapping);
if (!mapping_empty(mapping)) {
/*
* As truncation uses a lockless tree lookup, cycle
* the tree lock to make sure any ongoing tree
* modification that does not see AS_EXITING is
* completed before starting the final truncate.
*/
xa_lock_irq(&mapping->i_pages);
xa_unlock_irq(&mapping->i_pages);
}
truncate_inode_pages(mapping, 0);
}
EXPORT_SYMBOL(truncate_inode_pages_final);
/**
* mapping_try_invalidate - Invalidate all the evictable folios of one inode
* @mapping: the address_space which holds the folios to invalidate
* @start: the offset 'from' which to invalidate
* @end: the offset 'to' which to invalidate (inclusive)
* @nr_failed: How many folio invalidations failed
*
* This function is similar to invalidate_mapping_pages(), except that it
* returns the number of folios which could not be evicted in @nr_failed.
*/
unsigned long mapping_try_invalidate(struct address_space *mapping,
pgoff_t start, pgoff_t end, unsigned long *nr_failed)
{
pgoff_t indices[PAGEVEC_SIZE];
struct folio_batch fbatch;
pgoff_t index = start;
unsigned long ret;
unsigned long count = 0;
int i;
folio_batch_init(&fbatch);
while (find_lock_entries(mapping, &index, end, &fbatch, indices)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
/* We rely upon deletion not changing folio->index */
if (xa_is_value(folio)) {
count += invalidate_exceptional_entry(mapping,
indices[i], folio);
continue;
}
ret = mapping_evict_folio(mapping, folio);
folio_unlock(folio);
/*
* Invalidation is a hint that the folio is no longer
* of interest and try to speed up its reclaim.
*/
if (!ret) {
deactivate_file_folio(folio);
/* Likely in the lru cache of a remote CPU */
if (nr_failed)
(*nr_failed)++;
}
count += ret;
}
folio_batch_remove_exceptionals(&fbatch);
folio_batch_release(&fbatch);
cond_resched();
}
return count;
}
/**
* invalidate_mapping_pages - Invalidate all clean, unlocked cache of one inode
* @mapping: the address_space which holds the cache to invalidate
* @start: the offset 'from' which to invalidate
* @end: the offset 'to' which to invalidate (inclusive)
*
* This function removes pages that are clean, unmapped and unlocked,
* as well as shadow entries. It will not block on IO activity.
*
* If you want to remove all the pages of one inode, regardless of
* their use and writeback state, use truncate_inode_pages().
*
* Return: The number of indices that had their contents invalidated
*/
unsigned long invalidate_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
return mapping_try_invalidate(mapping, start, end, NULL);
}
EXPORT_SYMBOL(invalidate_mapping_pages);
/*
* This is like mapping_evict_folio(), except it ignores the folio's
* refcount. We do this because invalidate_inode_pages2() needs stronger
* invalidation guarantees, and cannot afford to leave folios behind because
* shrink_page_list() has a temp ref on them, or because they're transiently
* sitting in the folio_add_lru() caches.
*/
static int invalidate_complete_folio2(struct address_space *mapping,
struct folio *folio)
{
if (folio->mapping != mapping)
return 0;
if (!filemap_release_folio(folio, GFP_KERNEL))
return 0;
spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages);
if (folio_test_dirty(folio))
goto failed;
BUG_ON(folio_has_private(folio));
__filemap_remove_folio(folio, NULL);
xa_unlock_irq(&mapping->i_pages);
if (mapping_shrinkable(mapping))
inode_add_lru(mapping->host);
spin_unlock(&mapping->host->i_lock);
filemap_free_folio(mapping, folio);
return 1;
failed:
xa_unlock_irq(&mapping->i_pages);
spin_unlock(&mapping->host->i_lock);
return 0;
}
static int folio_launder(struct address_space *mapping, struct folio *folio)
{
if (!folio_test_dirty(folio))
return 0;
if (folio->mapping != mapping || mapping->a_ops->launder_folio == NULL)
return 0;
return mapping->a_ops->launder_folio(folio);
}
/**
* invalidate_inode_pages2_range - remove range of pages from an address_space
* @mapping: the address_space
* @start: the page offset 'from' which to invalidate
* @end: the page offset 'to' which to invalidate (inclusive)
*
* Any pages which are found to be mapped into pagetables are unmapped prior to
* invalidation.
*
* Return: -EBUSY if any pages could not be invalidated.
*/
int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
pgoff_t indices[PAGEVEC_SIZE];
struct folio_batch fbatch;
pgoff_t index;
int i;
int ret = 0;
int ret2 = 0;
int did_range_unmap = 0;
if (mapping_empty(mapping))
return 0;
folio_batch_init(&fbatch);
index = start;
while (find_get_entries(mapping, &index, end, &fbatch, indices)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
/* We rely upon deletion not changing folio->index */
if (xa_is_value(folio)) {
if (!invalidate_exceptional_entry2(mapping,
indices[i], folio))
ret = -EBUSY;
continue;
}
if (!did_range_unmap && folio_mapped(folio)) {
/*
* If folio is mapped, before taking its lock,
* zap the rest of the file in one hit.
*/
unmap_mapping_pages(mapping, indices[i],
(1 + end - indices[i]), false);
did_range_unmap = 1;
}
folio_lock(folio);
if (unlikely(folio->mapping != mapping)) {
folio_unlock(folio);
continue;
}
VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio);
folio_wait_writeback(folio);
if (folio_mapped(folio))
unmap_mapping_folio(folio);
BUG_ON(folio_mapped(folio));
ret2 = folio_launder(mapping, folio);
if (ret2 == 0) {
if (!invalidate_complete_folio2(mapping, folio))
ret2 = -EBUSY;
}
if (ret2 < 0)
ret = ret2;
folio_unlock(folio);
}
folio_batch_remove_exceptionals(&fbatch);
folio_batch_release(&fbatch);
cond_resched();
}
/*
* For DAX we invalidate page tables after invalidating page cache. We
* could invalidate page tables while invalidating each entry however
* that would be expensive. And doing range unmapping before doesn't
* work as we have no cheap way to find whether page cache entry didn't
* get remapped later.
*/
if (dax_mapping(mapping)) {
unmap_mapping_pages(mapping, start, end - start + 1, false);
}
return ret;
}
EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
/**
* invalidate_inode_pages2 - remove all pages from an address_space
* @mapping: the address_space
*
* Any pages which are found to be mapped into pagetables are unmapped prior to
* invalidation.
*
* Return: -EBUSY if any pages could not be invalidated.
*/
int invalidate_inode_pages2(struct address_space *mapping)
{
return invalidate_inode_pages2_range(mapping, 0, -1);
}
EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
/**
* truncate_pagecache - unmap and remove pagecache that has been truncated
* @inode: inode
* @newsize: new file size
*
* inode's new i_size must already be written before truncate_pagecache
* is called.
*
* This function should typically be called before the filesystem
* releases resources associated with the freed range (eg. deallocates
* blocks). This way, pagecache will always stay logically coherent
* with on-disk format, and the filesystem would not have to deal with
* situations such as writepage being called for a page that has already
* had its underlying blocks deallocated.
*/
void truncate_pagecache(struct inode *inode, loff_t newsize)
{
struct address_space *mapping = inode->i_mapping;
loff_t holebegin = round_up(newsize, PAGE_SIZE);
/*
* unmap_mapping_range is called twice, first simply for
* efficiency so that truncate_inode_pages does fewer
* single-page unmaps. However after this first call, and
* before truncate_inode_pages finishes, it is possible for
* private pages to be COWed, which remain after
* truncate_inode_pages finishes, hence the second
* unmap_mapping_range call must be made for correctness.
*/
unmap_mapping_range(mapping, holebegin, 0, 1);
truncate_inode_pages(mapping, newsize);
unmap_mapping_range(mapping, holebegin, 0, 1);
}
EXPORT_SYMBOL(truncate_pagecache);
/**
* truncate_setsize - update inode and pagecache for a new file size
* @inode: inode
* @newsize: new file size
*
* truncate_setsize updates i_size and performs pagecache truncation (if
* necessary) to @newsize. It will be typically be called from the filesystem's
* setattr function when ATTR_SIZE is passed in.
*
* Must be called with a lock serializing truncates and writes (generally
* i_rwsem but e.g. xfs uses a different lock) and before all filesystem
* specific block truncation has been performed.
*/
void truncate_setsize(struct inode *inode, loff_t newsize)
{
loff_t oldsize = inode->i_size;
i_size_write(inode, newsize);
if (newsize > oldsize)
pagecache_isize_extended(inode, oldsize, newsize);
truncate_pagecache(inode, newsize);
}
EXPORT_SYMBOL(truncate_setsize);
/**
* pagecache_isize_extended - update pagecache after extension of i_size
* @inode: inode for which i_size was extended
* @from: original inode size
* @to: new inode size
*
* Handle extension of inode size either caused by extending truncate or by
* write starting after current i_size. We mark the page straddling current
* i_size RO so that page_mkwrite() is called on the nearest write access to
* the page. This way filesystem can be sure that page_mkwrite() is called on
* the page before user writes to the page via mmap after the i_size has been
* changed.
*
* The function must be called after i_size is updated so that page fault
* coming after we unlock the page will already see the new i_size.
* The function must be called while we still hold i_rwsem - this not only
* makes sure i_size is stable but also that userspace cannot observe new
* i_size value before we are prepared to store mmap writes at new inode size.
*/
void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
{
int bsize = i_blocksize(inode);
loff_t rounded_from;
struct page *page;
pgoff_t index;
WARN_ON(to > inode->i_size);
if (from >= to || bsize == PAGE_SIZE)
return;
/* Page straddling @from will not have any hole block created? */
rounded_from = round_up(from, bsize);
if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
return;
index = from >> PAGE_SHIFT;
page = find_lock_page(inode->i_mapping, index);
/* Page not cached? Nothing to do */
if (!page)
return;
/*
* See clear_page_dirty_for_io() for details why set_page_dirty()
* is needed.
*/
if (page_mkclean(page))
set_page_dirty(page);
unlock_page(page);
put_page(page);
}
EXPORT_SYMBOL(pagecache_isize_extended);
/**
* truncate_pagecache_range - unmap and remove pagecache that is hole-punched
* @inode: inode
* @lstart: offset of beginning of hole
* @lend: offset of last byte of hole
*
* This function should typically be called before the filesystem
* releases resources associated with the freed range (eg. deallocates
* blocks). This way, pagecache will always stay logically coherent
* with on-disk format, and the filesystem would not have to deal with
* situations such as writepage being called for a page that has already
* had its underlying blocks deallocated.
*/
void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
{
struct address_space *mapping = inode->i_mapping;
loff_t unmap_start = round_up(lstart, PAGE_SIZE);
loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
/*
* This rounding is currently just for example: unmap_mapping_range
* expands its hole outwards, whereas we want it to contract the hole
* inwards. However, existing callers of truncate_pagecache_range are
* doing their own page rounding first. Note that unmap_mapping_range
* allows holelen 0 for all, and we allow lend -1 for end of file.
*/
/*
* Unlike in truncate_pagecache, unmap_mapping_range is called only
* once (before truncating pagecache), and without "even_cows" flag:
* hole-punching should not remove private COWed pages from the hole.
*/
if ((u64)unmap_end > (u64)unmap_start)
unmap_mapping_range(mapping, unmap_start,
1 + unmap_end - unmap_start, 0);
truncate_inode_pages_range(mapping, lstart, lend);
}
EXPORT_SYMBOL(truncate_pagecache_range);
/*
* linux/include/linux/console.h
*
* Copyright (C) 1993 Hamish Macdonald
*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file COPYING in the main directory of this archive
* for more details.
*
* Changed:
* 10-Mar-94: Arno Griffioen: Conversion for vt100 emulator port from PC LINUX
*/
#ifndef _LINUX_CONSOLE_H_
#define _LINUX_CONSOLE_H_ 1
#include <linux/atomic.h>
#include <linux/bits.h>
#include <linux/rculist.h>
#include <linux/types.h>
#include <linux/vesa.h>
struct vc_data;
struct console_font_op;
struct console_font;
struct module;
struct tty_struct;
struct notifier_block;
enum con_scroll {
SM_UP,
SM_DOWN,
};
enum vc_intensity;
/**
* struct consw - callbacks for consoles
*
* @owner: the module to get references of when this console is used
* @con_startup: set up the console and return its name (like VGA, EGA, ...)
* @con_init: initialize the console on @vc. @init is true for the very first
* call on this @vc.
* @con_deinit: deinitialize the console from @vc.
* @con_clear: erase @count characters at [@x, @y] on @vc. @count >= 1.
* @con_putc: emit one character with attributes @ca to [@x, @y] on @vc.
* (optional -- @con_putcs would be called instead)
* @con_putcs: emit @count characters with attributes @s to [@x, @y] on @vc.
* @con_cursor: enable/disable cursor depending on @enable
* @con_scroll: move lines from @top to @bottom in direction @dir by @lines.
* Return true if no generic handling should be done.
* Invoked by csi_M and printing to the console.
* @con_switch: notifier about the console switch; it is supposed to return
* true if a redraw is needed.
* @con_blank: blank/unblank the console. The target mode is passed in @blank.
* @mode_switch is set if changing from/to text/graphics. The hook
* is supposed to return true if a redraw is needed.
* @con_font_set: set console @vc font to @font with height @vpitch. @flags can
* be %KD_FONT_FLAG_DONT_RECALC. (optional)
* @con_font_get: fetch the current font on @vc of height @vpitch into @font.
* (optional)
* @con_font_default: set default font on @vc. @name can be %NULL or font name
* to search for. @font can be filled back. (optional)
* @con_resize: resize the @vc console to @width x @height. @from_user is true
* when this change comes from the user space.
* @con_set_palette: sets the palette of the console @vc to @table (optional)
* @con_scrolldelta: the contents of the console should be scrolled by @lines.
* Invoked by user. (optional)
* @con_set_origin: set origin (see &vc_data::vc_origin) of the @vc. If not
* provided or returns false, the origin is set to
* @vc->vc_screenbuf. (optional)
* @con_save_screen: save screen content into @vc->vc_screenbuf. Called e.g.
* upon entering graphics. (optional)
* @con_build_attr: build attributes based on @color, @intensity and other
* parameters. The result is used for both normal and erase
* characters. (optional)
* @con_invert_region: invert a region of length @count on @vc starting at @p.
* (optional)
* @con_debug_enter: prepare the console for the debugger. This includes, but
* is not limited to, unblanking the console, loading an
* appropriate palette, and allowing debugger generated output.
* (optional)
* @con_debug_leave: restore the console to its pre-debug state as closely as
* possible. (optional)
*/
struct consw {
struct module *owner;
const char *(*con_startup)(void);
void (*con_init)(struct vc_data *vc, bool init);
void (*con_deinit)(struct vc_data *vc);
void (*con_clear)(struct vc_data *vc, unsigned int y,
unsigned int x, unsigned int count);
void (*con_putc)(struct vc_data *vc, u16 ca, unsigned int y,
unsigned int x);
void (*con_putcs)(struct vc_data *vc, const u16 *s,
unsigned int count, unsigned int ypos,
unsigned int xpos);
void (*con_cursor)(struct vc_data *vc, bool enable);
bool (*con_scroll)(struct vc_data *vc, unsigned int top,
unsigned int bottom, enum con_scroll dir,
unsigned int lines);
bool (*con_switch)(struct vc_data *vc);
bool (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank,
bool mode_switch);
int (*con_font_set)(struct vc_data *vc,
const struct console_font *font,
unsigned int vpitch, unsigned int flags);
int (*con_font_get)(struct vc_data *vc, struct console_font *font,
unsigned int vpitch);
int (*con_font_default)(struct vc_data *vc,
struct console_font *font, const char *name);
int (*con_resize)(struct vc_data *vc, unsigned int width,
unsigned int height, bool from_user);
void (*con_set_palette)(struct vc_data *vc,
const unsigned char *table);
void (*con_scrolldelta)(struct vc_data *vc, int lines);
bool (*con_set_origin)(struct vc_data *vc);
void (*con_save_screen)(struct vc_data *vc);
u8 (*con_build_attr)(struct vc_data *vc, u8 color,
enum vc_intensity intensity,
bool blink, bool underline, bool reverse, bool italic);
void (*con_invert_region)(struct vc_data *vc, u16 *p, int count);
void (*con_debug_enter)(struct vc_data *vc);
void (*con_debug_leave)(struct vc_data *vc);
};
extern const struct consw *conswitchp;
extern const struct consw dummy_con; /* dummy console buffer */
extern const struct consw vga_con; /* VGA text console */
extern const struct consw newport_con; /* SGI Newport console */
struct screen_info;
#ifdef CONFIG_VGA_CONSOLE
void vgacon_register_screen(struct screen_info *si);
#else
static inline void vgacon_register_screen(struct screen_info *si) { }
#endif
int con_is_bound(const struct consw *csw);
int do_unregister_con_driver(const struct consw *csw);
int do_take_over_console(const struct consw *sw, int first, int last, int deflt);
void give_up_console(const struct consw *sw);
#ifdef CONFIG_VT
void con_debug_enter(struct vc_data *vc);
void con_debug_leave(void);
#else
static inline void con_debug_enter(struct vc_data *vc) { }
static inline void con_debug_leave(void) { }
#endif
/*
* The interface for a console, or any other device that wants to capture
* console messages (printer driver?)
*/
/**
* enum cons_flags - General console flags
* @CON_PRINTBUFFER: Used by newly registered consoles to avoid duplicate
* output of messages that were already shown by boot
* consoles or read by userspace via syslog() syscall.
* @CON_CONSDEV: Indicates that the console driver is backing
* /dev/console.
* @CON_ENABLED: Indicates if a console is allowed to print records. If
* false, the console also will not advance to later
* records.
* @CON_BOOT: Marks the console driver as early console driver which
* is used during boot before the real driver becomes
* available. It will be automatically unregistered
* when the real console driver is registered unless
* "keep_bootcon" parameter is used.
* @CON_ANYTIME: A misnomed historical flag which tells the core code
* that the legacy @console::write callback can be invoked
* on a CPU which is marked OFFLINE. That is misleading as
* it suggests that there is no contextual limit for
* invoking the callback. The original motivation was
* readiness of the per-CPU areas.
* @CON_BRL: Indicates a braille device which is exempt from
* receiving the printk spam for obvious reasons.
* @CON_EXTENDED: The console supports the extended output format of
* /dev/kmesg which requires a larger output buffer.
* @CON_SUSPENDED: Indicates if a console is suspended. If true, the
* printing callbacks must not be called.
* @CON_NBCON: Console can operate outside of the legacy style console_lock
* constraints.
*/
enum cons_flags {
CON_PRINTBUFFER = BIT(0),
CON_CONSDEV = BIT(1),
CON_ENABLED = BIT(2),
CON_BOOT = BIT(3),
CON_ANYTIME = BIT(4),
CON_BRL = BIT(5),
CON_EXTENDED = BIT(6),
CON_SUSPENDED = BIT(7),
CON_NBCON = BIT(8),
};
/**
* struct nbcon_state - console state for nbcon consoles
* @atom: Compound of the state fields for atomic operations
*
* @req_prio: The priority of a handover request
* @prio: The priority of the current owner
* @unsafe: Console is busy in a non takeover region
* @unsafe_takeover: A hostile takeover in an unsafe state happened in the
* past. The console cannot be safe until re-initialized.
* @cpu: The CPU on which the owner runs
*
* To be used for reading and preparing of the value stored in the nbcon
* state variable @console::nbcon_state.
*
* The @prio and @req_prio fields are particularly important to allow
* spin-waiting to timeout and give up without the risk of a waiter being
* assigned the lock after giving up.
*/
struct nbcon_state {
union {
unsigned int atom;
struct {
unsigned int prio : 2;
unsigned int req_prio : 2;
unsigned int unsafe : 1;
unsigned int unsafe_takeover : 1;
unsigned int cpu : 24;
};
};
};
/*
* The nbcon_state struct is used to easily create and interpret values that
* are stored in the @console::nbcon_state variable. Ensure this struct stays
* within the size boundaries of the atomic variable's underlying type in
* order to avoid any accidental truncation.
*/
static_assert(sizeof(struct nbcon_state) <= sizeof(int));
/**
* enum nbcon_prio - console owner priority for nbcon consoles
* @NBCON_PRIO_NONE: Unused
* @NBCON_PRIO_NORMAL: Normal (non-emergency) usage
* @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...)
* @NBCON_PRIO_PANIC: Panic output
* @NBCON_PRIO_MAX: The number of priority levels
*
* A higher priority context can takeover the console when it is
* in the safe state. The final attempt to flush consoles in panic()
* can be allowed to do so even in an unsafe state (Hope and pray).
*/
enum nbcon_prio {
NBCON_PRIO_NONE = 0,
NBCON_PRIO_NORMAL,
NBCON_PRIO_EMERGENCY,
NBCON_PRIO_PANIC,
NBCON_PRIO_MAX,
};
struct console;
struct printk_buffers;
/**
* struct nbcon_context - Context for console acquire/release
* @console: The associated console
* @spinwait_max_us: Limit for spin-wait acquire
* @prio: Priority of the context
* @allow_unsafe_takeover: Allow performing takeover even if unsafe. Can
* be used only with NBCON_PRIO_PANIC @prio. It
* might cause a system freeze when the console
* is used later.
* @backlog: Ringbuffer has pending records
* @pbufs: Pointer to the text buffer for this context
* @seq: The sequence number to print for this context
*/
struct nbcon_context {
/* members set by caller */
struct console *console;
unsigned int spinwait_max_us;
enum nbcon_prio prio;
unsigned int allow_unsafe_takeover : 1;
/* members set by emit */
unsigned int backlog : 1;
/* members set by acquire */
struct printk_buffers *pbufs;
u64 seq;
};
/**
* struct nbcon_write_context - Context handed to the nbcon write callbacks
* @ctxt: The core console context
* @outbuf: Pointer to the text buffer for output
* @len: Length to write
* @unsafe_takeover: If a hostile takeover in an unsafe state has occurred
*/
struct nbcon_write_context {
struct nbcon_context __private ctxt;
char *outbuf;
unsigned int len;
bool unsafe_takeover;
};
/**
* struct console - The console descriptor structure
* @name: The name of the console driver
* @write: Write callback to output messages (Optional)
* @read: Read callback for console input (Optional)
* @device: The underlying TTY device driver (Optional)
* @unblank: Callback to unblank the console (Optional)
* @setup: Callback for initializing the console (Optional)
* @exit: Callback for teardown of the console (Optional)
* @match: Callback for matching a console (Optional)
* @flags: Console flags. See enum cons_flags
* @index: Console index, e.g. port number
* @cflag: TTY control mode flags
* @ispeed: TTY input speed
* @ospeed: TTY output speed
* @seq: Sequence number of the next ringbuffer record to print
* @dropped: Number of unreported dropped ringbuffer records
* @data: Driver private data
* @node: hlist node for the console list
*
* @write_atomic: Write callback for atomic context
* @nbcon_state: State for nbcon consoles
* @nbcon_seq: Sequence number of the next record for nbcon to print
* @pbufs: Pointer to nbcon private buffer
*/
struct console {
char name[16];
void (*write)(struct console *co, const char *s, unsigned int count);
int (*read)(struct console *co, char *s, unsigned int count);
struct tty_driver *(*device)(struct console *co, int *index);
void (*unblank)(void);
int (*setup)(struct console *co, char *options);
int (*exit)(struct console *co);
int (*match)(struct console *co, char *name, int idx, char *options);
short flags;
short index;
int cflag;
uint ispeed;
uint ospeed;
u64 seq;
unsigned long dropped;
void *data;
struct hlist_node node;
/* nbcon console specific members */
bool (*write_atomic)(struct console *con,
struct nbcon_write_context *wctxt);
atomic_t __private nbcon_state;
atomic_long_t __private nbcon_seq;
struct printk_buffers *pbufs;
};
#ifdef CONFIG_LOCKDEP
extern void lockdep_assert_console_list_lock_held(void);
#else
static inline void lockdep_assert_console_list_lock_held(void)
{
}
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern bool console_srcu_read_lock_is_held(void);
#else
static inline bool console_srcu_read_lock_is_held(void)
{
return 1;
}
#endif
extern int console_srcu_read_lock(void);
extern void console_srcu_read_unlock(int cookie);
extern void console_list_lock(void) __acquires(console_mutex);
extern void console_list_unlock(void) __releases(console_mutex);
extern struct hlist_head console_list;
/**
* console_srcu_read_flags - Locklessly read the console flags
* @con: struct console pointer of console to read flags from
*
* This function provides the necessary READ_ONCE() and data_race()
* notation for locklessly reading the console flags. The READ_ONCE()
* in this function matches the WRITE_ONCE() when @flags are modified
* for registered consoles with console_srcu_write_flags().
*
* Only use this function to read console flags when locklessly
* iterating the console list via srcu.
*
* Context: Any context.
*/
static inline short console_srcu_read_flags(const struct console *con)
{
WARN_ON_ONCE(!console_srcu_read_lock_is_held());
/*
* Locklessly reading console->flags provides a consistent
* read value because there is at most one CPU modifying
* console->flags and that CPU is using only read-modify-write
* operations to do so.
*/
return data_race(READ_ONCE(con->flags));
}
/**
* console_srcu_write_flags - Write flags for a registered console
* @con: struct console pointer of console to write flags to
* @flags: new flags value to write
*
* Only use this function to write flags for registered consoles. It
* requires holding the console_list_lock.
*
* Context: Any context.
*/
static inline void console_srcu_write_flags(struct console *con, short flags)
{
lockdep_assert_console_list_lock_held();
/* This matches the READ_ONCE() in console_srcu_read_flags(). */
WRITE_ONCE(con->flags, flags);
}
/* Variant of console_is_registered() when the console_list_lock is held. */
static inline bool console_is_registered_locked(const struct console *con)
{
lockdep_assert_console_list_lock_held();
return !hlist_unhashed(&con->node);
}
/*
* console_is_registered - Check if the console is registered
* @con: struct console pointer of console to check
*
* Context: Process context. May sleep while acquiring console list lock.
* Return: true if the console is in the console list, otherwise false.
*
* If false is returned for a console that was previously registered, it
* can be assumed that the console's unregistration is fully completed,
* including the exit() callback after console list removal.
*/
static inline bool console_is_registered(const struct console *con)
{
bool ret;
console_list_lock();
ret = console_is_registered_locked(con);
console_list_unlock();
return ret;
}
/**
* for_each_console_srcu() - Iterator over registered consoles
* @con: struct console pointer used as loop cursor
*
* Although SRCU guarantees the console list will be consistent, the
* struct console fields may be updated by other CPUs while iterating.
*
* Requires console_srcu_read_lock to be held. Can be invoked from
* any context.
*/
#define for_each_console_srcu(con) \
hlist_for_each_entry_srcu(con, &console_list, node, \
console_srcu_read_lock_is_held())
/**
* for_each_console() - Iterator over registered consoles
* @con: struct console pointer used as loop cursor
*
* The console list and the &console.flags are immutable while iterating.
*
* Requires console_list_lock to be held.
*/
#define for_each_console(con) \
lockdep_assert_console_list_lock_held(); \
hlist_for_each_entry(con, &console_list, node)
#ifdef CONFIG_PRINTK
extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
#else
static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
#endif
extern int console_set_on_cmdline;
extern struct console *early_console;
enum con_flush_mode {
CONSOLE_FLUSH_PENDING,
CONSOLE_REPLAY_ALL,
};
extern int add_preferred_console(const char *name, const short idx, char *options);
extern void console_force_preferred_locked(struct console *con);
extern void register_console(struct console *);
extern int unregister_console(struct console *);
extern void console_lock(void);
extern int console_trylock(void);
extern void console_unlock(void);
extern void console_conditional_schedule(void);
extern void console_unblank(void);
extern void console_flush_on_panic(enum con_flush_mode mode);
extern struct tty_driver *console_device(int *);
extern void console_stop(struct console *);
extern void console_start(struct console *);
extern int is_console_locked(void);
extern int braille_register_console(struct console *, int index,
char *console_options, char *braille_options);
extern int braille_unregister_console(struct console *);
#ifdef CONFIG_TTY
extern void console_sysfs_notify(void);
#else
static inline void console_sysfs_notify(void)
{ }
#endif
extern bool console_suspend_enabled;
/* Suspend and resume console messages over PM events */
extern void suspend_console(void);
extern void resume_console(void);
int mda_console_init(void);
void vcs_make_sysfs(int index);
void vcs_remove_sysfs(int index);
/* Some debug stub to catch some of the obvious races in the VT code */
#define WARN_CONSOLE_UNLOCKED() \
WARN_ON(!atomic_read(&ignore_console_lock_warning) && \
!is_console_locked() && !oops_in_progress)
/*
* Increment ignore_console_lock_warning if you need to quiet
* WARN_CONSOLE_UNLOCKED() for debugging purposes.
*/
extern atomic_t ignore_console_lock_warning;
extern void console_init(void);
/* For deferred console takeover */
void dummycon_register_output_notifier(struct notifier_block *nb);
void dummycon_unregister_output_notifier(struct notifier_block *nb);
#endif /* _LINUX_CONSOLE_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BITOPS_H
#define _LINUX_BITOPS_H
#include <asm/types.h>
#include <linux/bits.h>
#include <linux/typecheck.h>
#include <uapi/linux/kernel.h>
/* Set bits in the first 'n' bytes when loaded from memory */
#ifdef __LITTLE_ENDIAN
# define aligned_byte_mask(n) ((1UL << 8*(n))-1)
#else
# define aligned_byte_mask(n) (~0xffUL << (BITS_PER_LONG - 8 - 8*(n)))
#endif
#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE)
#define BITS_TO_LONGS(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(long))
#define BITS_TO_U64(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(u64))
#define BITS_TO_U32(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(u32))
#define BITS_TO_BYTES(nr) __KERNEL_DIV_ROUND_UP(nr, BITS_PER_TYPE(char))
extern unsigned int __sw_hweight8(unsigned int w);
extern unsigned int __sw_hweight16(unsigned int w);
extern unsigned int __sw_hweight32(unsigned int w);
extern unsigned long __sw_hweight64(__u64 w);
/*
* Defined here because those may be needed by architecture-specific static
* inlines.
*/
#include <asm-generic/bitops/generic-non-atomic.h>
/*
* Many architecture-specific non-atomic bitops contain inline asm code and due
* to that the compiler can't optimize them to compile-time expressions or
* constants. In contrary, generic_*() helpers are defined in pure C and
* compilers optimize them just well.
* Therefore, to make `unsigned long foo = 0; __set_bit(BAR, &foo)` effectively
* equal to `unsigned long foo = BIT(BAR)`, pick the generic C alternative when
* the arguments can be resolved at compile time. That expression itself is a
* constant and doesn't bring any functional changes to the rest of cases.
* The casts to `uintptr_t` are needed to mitigate `-Waddress` warnings when
* passing a bitmap from .bss or .data (-> `!!addr` is always true).
*/
#define bitop(op, nr, addr) \
((__builtin_constant_p(nr) && \
__builtin_constant_p((uintptr_t)(addr) != (uintptr_t)NULL) && \
(uintptr_t)(addr) != (uintptr_t)NULL && \
__builtin_constant_p(*(const unsigned long *)(addr))) ? \
const##op(nr, addr) : op(nr, addr))
#define __set_bit(nr, addr) bitop(___set_bit, nr, addr)
#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr)
#define __change_bit(nr, addr) bitop(___change_bit, nr, addr)
#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr)
#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr)
#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr)
#define test_bit(nr, addr) bitop(_test_bit, nr, addr)
#define test_bit_acquire(nr, addr) bitop(_test_bit_acquire, nr, addr)
/*
* Include this here because some architectures need generic_ffs/fls in
* scope
*/
#include <asm/bitops.h>
/* Check that the bitops prototypes are sane */
#define __check_bitop_pr(name) \
static_assert(__same_type(arch_##name, generic_##name) && \
__same_type(const_##name, generic_##name) && \
__same_type(_##name, generic_##name))
__check_bitop_pr(__set_bit);
__check_bitop_pr(__clear_bit);
__check_bitop_pr(__change_bit);
__check_bitop_pr(__test_and_set_bit);
__check_bitop_pr(__test_and_clear_bit);
__check_bitop_pr(__test_and_change_bit);
__check_bitop_pr(test_bit);
#undef __check_bitop_pr
static inline int get_bitmask_order(unsigned int count)
{
int order;
order = fls(count);
return order; /* We could be slightly more clever with -1 here... */
}
static __always_inline unsigned long hweight_long(unsigned long w)
{
return sizeof(w) == 4 ? hweight32(w) : hweight64((__u64)w);
}
/**
* rol64 - rotate a 64-bit value left
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u64 rol64(__u64 word, unsigned int shift)
{
return (word << (shift & 63)) | (word >> ((-shift) & 63));
}
/**
* ror64 - rotate a 64-bit value right
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u64 ror64(__u64 word, unsigned int shift)
{
return (word >> (shift & 63)) | (word << ((-shift) & 63));
}
/**
* rol32 - rotate a 32-bit value left
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u32 rol32(__u32 word, unsigned int shift)
{
return (word << (shift & 31)) | (word >> ((-shift) & 31));
}
/**
* ror32 - rotate a 32-bit value right
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u32 ror32(__u32 word, unsigned int shift)
{
return (word >> (shift & 31)) | (word << ((-shift) & 31));
}
/**
* rol16 - rotate a 16-bit value left
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u16 rol16(__u16 word, unsigned int shift)
{
return (word << (shift & 15)) | (word >> ((-shift) & 15));
}
/**
* ror16 - rotate a 16-bit value right
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u16 ror16(__u16 word, unsigned int shift)
{
return (word >> (shift & 15)) | (word << ((-shift) & 15));
}
/**
* rol8 - rotate an 8-bit value left
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u8 rol8(__u8 word, unsigned int shift)
{
return (word << (shift & 7)) | (word >> ((-shift) & 7));
}
/**
* ror8 - rotate an 8-bit value right
* @word: value to rotate
* @shift: bits to roll
*/
static inline __u8 ror8(__u8 word, unsigned int shift)
{
return (word >> (shift & 7)) | (word << ((-shift) & 7));
}
/**
* sign_extend32 - sign extend a 32-bit value using specified bit as sign-bit
* @value: value to sign extend
* @index: 0 based bit index (0<=index<32) to sign bit
*
* This is safe to use for 16- and 8-bit types as well.
*/
static __always_inline __s32 sign_extend32(__u32 value, int index)
{
__u8 shift = 31 - index;
return (__s32)(value << shift) >> shift;
}
/**
* sign_extend64 - sign extend a 64-bit value using specified bit as sign-bit
* @value: value to sign extend
* @index: 0 based bit index (0<=index<64) to sign bit
*/
static __always_inline __s64 sign_extend64(__u64 value, int index)
{
__u8 shift = 63 - index;
return (__s64)(value << shift) >> shift;
}
static inline unsigned fls_long(unsigned long l)
{
if (sizeof(l) == 4)
return fls(l);
return fls64(l);
}
static inline int get_count_order(unsigned int count)
{
if (count == 0)
return -1;
return fls(--count);
}
/**
* get_count_order_long - get order after rounding @l up to power of 2
* @l: parameter
*
* it is same as get_count_order() but with long type parameter
*/
static inline int get_count_order_long(unsigned long l)
{
if (l == 0UL)
return -1;
return (int)fls_long(--l);
}
/**
* __ffs64 - find first set bit in a 64 bit word
* @word: The 64 bit word
*
* On 64 bit arches this is a synonym for __ffs
* The result is not defined if no bits are set, so check that @word
* is non-zero before calling this.
*/
static inline unsigned long __ffs64(u64 word)
{
#if BITS_PER_LONG == 32
if (((u32)word) == 0UL)
return __ffs((u32)(word >> 32)) + 32;
#elif BITS_PER_LONG != 64
#error BITS_PER_LONG not 32 or 64
#endif
return __ffs((unsigned long)word);
}
/**
* fns - find N'th set bit in a word
* @word: The word to search
* @n: Bit to find
*/
static inline unsigned long fns(unsigned long word, unsigned int n)
{
unsigned int bit;
while (word) {
bit = __ffs(word);
if (n-- == 0)
return bit;
__clear_bit(bit, &word);
}
return BITS_PER_LONG;
}
/**
* assign_bit - Assign value to a bit in memory
* @nr: the bit to set
* @addr: the address to start counting from
* @value: the value to assign
*/
static __always_inline void assign_bit(long nr, volatile unsigned long *addr,
bool value)
{
if (value)
set_bit(nr, addr);
else
clear_bit(nr, addr);
}
static __always_inline void __assign_bit(long nr, volatile unsigned long *addr,
bool value)
{
if (value)
__set_bit(nr, addr);
else
__clear_bit(nr, addr);
}
/**
* __ptr_set_bit - Set bit in a pointer's value
* @nr: the bit to set
* @addr: the address of the pointer variable
*
* Example:
* void *p = foo();
* __ptr_set_bit(bit, &p);
*/
#define __ptr_set_bit(nr, addr) \
({ \
typecheck_pointer(*(addr)); \
__set_bit(nr, (unsigned long *)(addr)); \
})
/**
* __ptr_clear_bit - Clear bit in a pointer's value
* @nr: the bit to clear
* @addr: the address of the pointer variable
*
* Example:
* void *p = foo();
* __ptr_clear_bit(bit, &p);
*/
#define __ptr_clear_bit(nr, addr) \
({ \
typecheck_pointer(*(addr)); \
__clear_bit(nr, (unsigned long *)(addr)); \
})
/**
* __ptr_test_bit - Test bit in a pointer's value
* @nr: the bit to test
* @addr: the address of the pointer variable
*
* Example:
* void *p = foo();
* if (__ptr_test_bit(bit, &p)) {
* ...
* } else {
* ...
* }
*/
#define __ptr_test_bit(nr, addr) \
({ \
typecheck_pointer(*(addr)); \
test_bit(nr, (unsigned long *)(addr)); \
})
#ifdef __KERNEL__
#ifndef set_mask_bits
#define set_mask_bits(ptr, mask, bits) \
({ \
const typeof(*(ptr)) mask__ = (mask), bits__ = (bits); \
typeof(*(ptr)) old__, new__; \
\
old__ = READ_ONCE(*(ptr)); \
do { \
new__ = (old__ & ~mask__) | bits__; \
} while (!try_cmpxchg(ptr, &old__, new__)); \
\
old__; \
})
#endif
#ifndef bit_clear_unless
#define bit_clear_unless(ptr, clear, test) \
({ \
const typeof(*(ptr)) clear__ = (clear), test__ = (test);\
typeof(*(ptr)) old__, new__; \
\
old__ = READ_ONCE(*(ptr)); \
do { \
if (old__ & test__) \
break; \
new__ = old__ & ~clear__; \
} while (!try_cmpxchg(ptr, &old__, new__)); \
\
!(old__ & test__); \
})
#endif
#endif /* __KERNEL__ */
#endif
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Credentials management - see Documentation/security/credentials.rst
*
* Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*/
#ifndef _LINUX_CRED_H
#define _LINUX_CRED_H
#include <linux/capability.h>
#include <linux/init.h>
#include <linux/key.h>
#include <linux/atomic.h>
#include <linux/refcount.h>
#include <linux/uidgid.h>
#include <linux/sched.h>
#include <linux/sched/user.h>
struct cred;
struct inode;
/*
* COW Supplementary groups list
*/
struct group_info {
refcount_t usage;
int ngroups;
kgid_t gid[];
} __randomize_layout;
/**
* get_group_info - Get a reference to a group info structure
* @group_info: The group info to reference
*
* This gets a reference to a set of supplementary groups.
*
* If the caller is accessing a task's credentials, they must hold the RCU read
* lock when reading.
*/
static inline struct group_info *get_group_info(struct group_info *gi)
{
refcount_inc(&gi->usage);
return gi;
}
/**
* put_group_info - Release a reference to a group info structure
* @group_info: The group info to release
*/
#define put_group_info(group_info) \
do { \
if (refcount_dec_and_test(&(group_info)->usage)) \
groups_free(group_info); \
} while (0)
#ifdef CONFIG_MULTIUSER
extern struct group_info *groups_alloc(int);
extern void groups_free(struct group_info *);
extern int in_group_p(kgid_t);
extern int in_egroup_p(kgid_t);
extern int groups_search(const struct group_info *, kgid_t);
extern int set_current_groups(struct group_info *);
extern void set_groups(struct cred *, struct group_info *);
extern bool may_setgroups(void);
extern void groups_sort(struct group_info *);
#else
static inline void groups_free(struct group_info *group_info)
{
}
static inline int in_group_p(kgid_t grp)
{
return 1;
}
static inline int in_egroup_p(kgid_t grp)
{
return 1;
}
static inline int groups_search(const struct group_info *group_info, kgid_t grp)
{
return 1;
}
#endif
/*
* The security context of a task
*
* The parts of the context break down into two categories:
*
* (1) The objective context of a task. These parts are used when some other
* task is attempting to affect this one.
*
* (2) The subjective context. These details are used when the task is acting
* upon another object, be that a file, a task, a key or whatever.
*
* Note that some members of this structure belong to both categories - the
* LSM security pointer for instance.
*
* A task has two security pointers. task->real_cred points to the objective
* context that defines that task's actual details. The objective part of this
* context is used whenever that task is acted upon.
*
* task->cred points to the subjective context that defines the details of how
* that task is going to act upon another object. This may be overridden
* temporarily to point to another security context, but normally points to the
* same context as task->real_cred.
*/
struct cred {
atomic_long_t usage;
kuid_t uid; /* real UID of the task */
kgid_t gid; /* real GID of the task */
kuid_t suid; /* saved UID of the task */
kgid_t sgid; /* saved GID of the task */
kuid_t euid; /* effective UID of the task */
kgid_t egid; /* effective GID of the task */
kuid_t fsuid; /* UID for VFS ops */
kgid_t fsgid; /* GID for VFS ops */
unsigned securebits; /* SUID-less security management */
kernel_cap_t cap_inheritable; /* caps our children can inherit */
kernel_cap_t cap_permitted; /* caps we're permitted */
kernel_cap_t cap_effective; /* caps we can actually use */
kernel_cap_t cap_bset; /* capability bounding set */
kernel_cap_t cap_ambient; /* Ambient capability set */
#ifdef CONFIG_KEYS
unsigned char jit_keyring; /* default keyring to attach requested
* keys to */
struct key *session_keyring; /* keyring inherited over fork */
struct key *process_keyring; /* keyring private to this process */
struct key *thread_keyring; /* keyring private to this thread */
struct key *request_key_auth; /* assumed request_key authority */
#endif
#ifdef CONFIG_SECURITY
void *security; /* LSM security */
#endif
struct user_struct *user; /* real user ID subscription */
struct user_namespace *user_ns; /* user_ns the caps and keyrings are relative to. */
struct ucounts *ucounts;
struct group_info *group_info; /* supplementary groups for euid/fsgid */
/* RCU deletion */
union {
int non_rcu; /* Can we skip RCU deletion? */
struct rcu_head rcu; /* RCU deletion hook */
};
} __randomize_layout;
extern void __put_cred(struct cred *);
extern void exit_creds(struct task_struct *);
extern int copy_creds(struct task_struct *, unsigned long);
extern const struct cred *get_task_cred(struct task_struct *);
extern struct cred *cred_alloc_blank(void);
extern struct cred *prepare_creds(void);
extern struct cred *prepare_exec_creds(void);
extern int commit_creds(struct cred *);
extern void abort_creds(struct cred *);
extern const struct cred *override_creds(const struct cred *);
extern void revert_creds(const struct cred *);
extern struct cred *prepare_kernel_cred(struct task_struct *);
extern int set_security_override(struct cred *, u32);
extern int set_security_override_from_ctx(struct cred *, const char *);
extern int set_create_files_as(struct cred *, struct inode *);
extern int cred_fscmp(const struct cred *, const struct cred *);
extern void __init cred_init(void);
extern int set_cred_ucounts(struct cred *);
static inline bool cap_ambient_invariant_ok(const struct cred *cred)
{
return cap_issubset(cred->cap_ambient,
cap_intersect(cred->cap_permitted,
cred->cap_inheritable));
}
/**
* get_new_cred_many - Get references on a new set of credentials
* @cred: The new credentials to reference
* @nr: Number of references to acquire
*
* Get references on the specified set of new credentials. The caller must
* release all acquired references.
*/
static inline struct cred *get_new_cred_many(struct cred *cred, int nr)
{
atomic_long_add(nr, &cred->usage);
return cred;
}
/**
* get_new_cred - Get a reference on a new set of credentials
* @cred: The new credentials to reference
*
* Get a reference on the specified set of new credentials. The caller must
* release the reference.
*/
static inline struct cred *get_new_cred(struct cred *cred)
{
return get_new_cred_many(cred, 1);
}
/**
* get_cred_many - Get references on a set of credentials
* @cred: The credentials to reference
* @nr: Number of references to acquire
*
* Get references on the specified set of credentials. The caller must release
* all acquired reference. If %NULL is passed, it is returned with no action.
*
* This is used to deal with a committed set of credentials. Although the
* pointer is const, this will temporarily discard the const and increment the
* usage count. The purpose of this is to attempt to catch at compile time the
* accidental alteration of a set of credentials that should be considered
* immutable.
*/
static inline const struct cred *get_cred_many(const struct cred *cred, int nr)
{
struct cred *nonconst_cred = (struct cred *) cred;
if (!cred)
return cred;
nonconst_cred->non_rcu = 0;
return get_new_cred_many(nonconst_cred, nr);
}
/*
* get_cred - Get a reference on a set of credentials
* @cred: The credentials to reference
*
* Get a reference on the specified set of credentials. The caller must
* release the reference. If %NULL is passed, it is returned with no action.
*
* This is used to deal with a committed set of credentials.
*/
static inline const struct cred *get_cred(const struct cred *cred)
{
return get_cred_many(cred, 1);
}
static inline const struct cred *get_cred_rcu(const struct cred *cred)
{
struct cred *nonconst_cred = (struct cred *) cred;
if (!cred)
return NULL;
if (!atomic_long_inc_not_zero(&nonconst_cred->usage))
return NULL;
nonconst_cred->non_rcu = 0;
return cred;
}
/**
* put_cred - Release a reference to a set of credentials
* @cred: The credentials to release
* @nr: Number of references to release
*
* Release a reference to a set of credentials, deleting them when the last ref
* is released. If %NULL is passed, nothing is done.
*
* This takes a const pointer to a set of credentials because the credentials
* on task_struct are attached by const pointers to prevent accidental
* alteration of otherwise immutable credential sets.
*/
static inline void put_cred_many(const struct cred *_cred, int nr)
{
struct cred *cred = (struct cred *) _cred;
if (cred) {
if (atomic_long_sub_and_test(nr, &cred->usage)) __put_cred(cred);
}
}
/*
* put_cred - Release a reference to a set of credentials
* @cred: The credentials to release
*
* Release a reference to a set of credentials, deleting them when the last ref
* is released. If %NULL is passed, nothing is done.
*/
static inline void put_cred(const struct cred *cred)
{
put_cred_many(cred, 1);
}
/**
* current_cred - Access the current task's subjective credentials
*
* Access the subjective credentials of the current task. RCU-safe,
* since nobody else can modify it.
*/
#define current_cred() \
rcu_dereference_protected(current->cred, 1)
/**
* current_real_cred - Access the current task's objective credentials
*
* Access the objective credentials of the current task. RCU-safe,
* since nobody else can modify it.
*/
#define current_real_cred() \
rcu_dereference_protected(current->real_cred, 1)
/**
* __task_cred - Access a task's objective credentials
* @task: The task to query
*
* Access the objective credentials of a task. The caller must hold the RCU
* readlock.
*
* The result of this function should not be passed directly to get_cred();
* rather get_task_cred() should be used instead.
*/
#define __task_cred(task) \
rcu_dereference((task)->real_cred)
/**
* get_current_cred - Get the current task's subjective credentials
*
* Get the subjective credentials of the current task, pinning them so that
* they can't go away. Accessing the current task's credentials directly is
* not permitted.
*/
#define get_current_cred() \
(get_cred(current_cred()))
/**
* get_current_user - Get the current task's user_struct
*
* Get the user record of the current task, pinning it so that it can't go
* away.
*/
#define get_current_user() \
({ \
struct user_struct *__u; \
const struct cred *__cred; \
__cred = current_cred(); \
__u = get_uid(__cred->user); \
__u; \
})
/**
* get_current_groups - Get the current task's supplementary group list
*
* Get the supplementary group list of the current task, pinning it so that it
* can't go away.
*/
#define get_current_groups() \
({ \
struct group_info *__groups; \
const struct cred *__cred; \
__cred = current_cred(); \
__groups = get_group_info(__cred->group_info); \
__groups; \
})
#define task_cred_xxx(task, xxx) \
({ \
__typeof__(((struct cred *)NULL)->xxx) ___val; \
rcu_read_lock(); \
___val = __task_cred((task))->xxx; \
rcu_read_unlock(); \
___val; \
})
#define task_uid(task) (task_cred_xxx((task), uid))
#define task_euid(task) (task_cred_xxx((task), euid))
#define task_ucounts(task) (task_cred_xxx((task), ucounts))
#define current_cred_xxx(xxx) \
({ \
current_cred()->xxx; \
})
#define current_uid() (current_cred_xxx(uid))
#define current_gid() (current_cred_xxx(gid))
#define current_euid() (current_cred_xxx(euid))
#define current_egid() (current_cred_xxx(egid))
#define current_suid() (current_cred_xxx(suid))
#define current_sgid() (current_cred_xxx(sgid))
#define current_fsuid() (current_cred_xxx(fsuid))
#define current_fsgid() (current_cred_xxx(fsgid))
#define current_cap() (current_cred_xxx(cap_effective))
#define current_user() (current_cred_xxx(user))
#define current_ucounts() (current_cred_xxx(ucounts))
extern struct user_namespace init_user_ns;
#ifdef CONFIG_USER_NS
#define current_user_ns() (current_cred_xxx(user_ns))
#else
static inline struct user_namespace *current_user_ns(void)
{
return &init_user_ns;
}
#endif
#define current_uid_gid(_uid, _gid) \
do { \
const struct cred *__cred; \
__cred = current_cred(); \
*(_uid) = __cred->uid; \
*(_gid) = __cred->gid; \
} while(0)
#define current_euid_egid(_euid, _egid) \
do { \
const struct cred *__cred; \
__cred = current_cred(); \
*(_euid) = __cred->euid; \
*(_egid) = __cred->egid; \
} while(0)
#define current_fsuid_fsgid(_fsuid, _fsgid) \
do { \
const struct cred *__cred; \
__cred = current_cred(); \
*(_fsuid) = __cred->fsuid; \
*(_fsgid) = __cred->fsgid; \
} while(0)
#endif /* _LINUX_CRED_H */
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/uaccess.h>
#include <linux/kernel.h>
#include <asm/vsyscall.h>
#ifdef CONFIG_X86_64
bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
{
unsigned long vaddr = (unsigned long)unsafe_src;
/*
* Do not allow userspace addresses. This disallows
* normal userspace and the userspace guard page:
*/
if (vaddr < TASK_SIZE_MAX + PAGE_SIZE)
return false;
/*
* Reading from the vsyscall page may cause an unhandled fault in
* certain cases. Though it is at an address above TASK_SIZE_MAX, it is
* usually considered as a user space address.
*/
if (is_vsyscall_vaddr(vaddr))
return false;
/*
* Allow everything during early boot before 'x86_virt_bits'
* is initialized. Needed for instruction decoding in early
* exception handlers.
*/
if (!boot_cpu_data.x86_virt_bits)
return true;
return __is_canonical_address(vaddr, boot_cpu_data.x86_virt_bits);
}
#else
bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
{
return (unsigned long)unsafe_src >= TASK_SIZE_MAX;
}
#endif
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef IOPRIO_H
#define IOPRIO_H
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/iocontext.h>
#include <uapi/linux/ioprio.h>
/*
* Default IO priority.
*/
#define IOPRIO_DEFAULT IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0)
/*
* Check that a priority value has a valid class.
*/
static inline bool ioprio_valid(unsigned short ioprio)
{
unsigned short class = IOPRIO_PRIO_CLASS(ioprio);
return class > IOPRIO_CLASS_NONE && class <= IOPRIO_CLASS_IDLE;
}
/*
* if process has set io priority explicitly, use that. if not, convert
* the cpu scheduler nice value to an io priority
*/
static inline int task_nice_ioprio(struct task_struct *task)
{
return (task_nice(task) + 20) / 5;
}
/*
* This is for the case where the task hasn't asked for a specific IO class.
* Check for idle and rt task process, and return appropriate IO class.
*/
static inline int task_nice_ioclass(struct task_struct *task)
{
if (task->policy == SCHED_IDLE)
return IOPRIO_CLASS_IDLE;
else if (task_is_realtime(task))
return IOPRIO_CLASS_RT;
else
return IOPRIO_CLASS_BE;
}
#ifdef CONFIG_BLOCK
/*
* If the task has set an I/O priority, use that. Otherwise, return
* the default I/O priority.
*
* Expected to be called for current task or with task_lock() held to keep
* io_context stable.
*/
static inline int __get_task_ioprio(struct task_struct *p)
{
struct io_context *ioc = p->io_context;
int prio;
if (!ioc)
return IOPRIO_DEFAULT;
if (p != current) lockdep_assert_held(&p->alloc_lock); prio = ioc->ioprio;
if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE)
prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(p),
task_nice_ioprio(p));
return prio;
}
#else
static inline int __get_task_ioprio(struct task_struct *p)
{
return IOPRIO_DEFAULT;
}
#endif /* CONFIG_BLOCK */
static inline int get_current_ioprio(void)
{
return __get_task_ioprio(current);
}
extern int set_task_ioprio(struct task_struct *task, int ioprio);
#ifdef CONFIG_BLOCK
extern int ioprio_check_cap(int ioprio);
#else
static inline int ioprio_check_cap(int ioprio)
{
return -ENOTBLK;
}
#endif /* CONFIG_BLOCK */
#endif
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
/*
* Copyright (C) 2017-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
* Copyright Matt Mackall <mpm@selenic.com>, 2003, 2004, 2005
* Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All rights reserved.
*
* This driver produces cryptographically secure pseudorandom data. It is divided
* into roughly six sections, each with a section header:
*
* - Initialization and readiness waiting.
* - Fast key erasure RNG, the "crng".
* - Entropy accumulation and extraction routines.
* - Entropy collection routines.
* - Userspace reader/writer interfaces.
* - Sysctl interface.
*
* The high level overview is that there is one input pool, into which
* various pieces of data are hashed. Prior to initialization, some of that
* data is then "credited" as having a certain number of bits of entropy.
* When enough bits of entropy are available, the hash is finalized and
* handed as a key to a stream cipher that expands it indefinitely for
* various consumers. This key is periodically refreshed as the various
* entropy collectors, described below, add data to the input pool.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/utsname.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/major.h>
#include <linux/string.h>
#include <linux/fcntl.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/poll.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/blkdev.h>
#include <linux/interrupt.h>
#include <linux/mm.h>
#include <linux/nodemask.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/percpu.h>
#include <linux/ptrace.h>
#include <linux/workqueue.h>
#include <linux/irq.h>
#include <linux/ratelimit.h>
#include <linux/syscalls.h>
#include <linux/completion.h>
#include <linux/uuid.h>
#include <linux/uaccess.h>
#include <linux/suspend.h>
#include <linux/siphash.h>
#include <linux/sched/isolation.h>
#include <crypto/chacha.h>
#include <crypto/blake2s.h>
#include <asm/archrandom.h>
#include <asm/processor.h>
#include <asm/irq.h>
#include <asm/irq_regs.h>
#include <asm/io.h>
/*********************************************************************
*
* Initialization and readiness waiting.
*
* Much of the RNG infrastructure is devoted to various dependencies
* being able to wait until the RNG has collected enough entropy and
* is ready for safe consumption.
*
*********************************************************************/
/*
* crng_init is protected by base_crng->lock, and only increases
* its value (from empty->early->ready).
*/
static enum {
CRNG_EMPTY = 0, /* Little to no entropy collected */
CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
} crng_init __read_mostly = CRNG_EMPTY;
static DEFINE_STATIC_KEY_FALSE(crng_is_ready);
#define crng_ready() (static_branch_likely(&crng_is_ready) || crng_init >= CRNG_READY)
/* Various types of waiters for crng_init->CRNG_READY transition. */
static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
static struct fasync_struct *fasync;
static ATOMIC_NOTIFIER_HEAD(random_ready_notifier);
/* Control how we warn userspace. */
static struct ratelimit_state urandom_warning =
RATELIMIT_STATE_INIT_FLAGS("urandom_warning", HZ, 3, RATELIMIT_MSG_ON_RELEASE);
static int ratelimit_disable __read_mostly =
IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM);
module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
/*
* Returns whether or not the input pool has been seeded and thus guaranteed
* to supply cryptographically secure random numbers. This applies to: the
* /dev/urandom device, the get_random_bytes function, and the get_random_{u8,
* u16,u32,u64,long} family of functions.
*
* Returns: true if the input pool has been seeded.
* false if the input pool has not been seeded.
*/
bool rng_is_initialized(void)
{
return crng_ready();
}
EXPORT_SYMBOL(rng_is_initialized);
static void __cold crng_set_ready(struct work_struct *work)
{
static_branch_enable(&crng_is_ready);
}
/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
static void try_to_generate_entropy(void);
/*
* Wait for the input pool to be seeded and thus guaranteed to supply
* cryptographically secure random numbers. This applies to: the /dev/urandom
* device, the get_random_bytes function, and the get_random_{u8,u16,u32,u64,
* long} family of functions. Using any of these functions without first
* calling this function forfeits the guarantee of security.
*
* Returns: 0 if the input pool has been seeded.
* -ERESTARTSYS if the function was interrupted by a signal.
*/
int wait_for_random_bytes(void)
{
while (!crng_ready()) {
int ret;
try_to_generate_entropy();
ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
if (ret)
return ret > 0 ? 0 : ret;
}
return 0;
}
EXPORT_SYMBOL(wait_for_random_bytes);
/*
* Add a callback function that will be invoked when the crng is initialised,
* or immediately if it already has been. Only use this is you are absolutely
* sure it is required. Most users should instead be able to test
* `rng_is_initialized()` on demand, or make use of `get_random_bytes_wait()`.
*/
int __cold execute_with_initialized_rng(struct notifier_block *nb)
{
unsigned long flags;
int ret = 0;
spin_lock_irqsave(&random_ready_notifier.lock, flags);
if (crng_ready())
nb->notifier_call(nb, 0, NULL);
else
ret = raw_notifier_chain_register((struct raw_notifier_head *)&random_ready_notifier.head, nb);
spin_unlock_irqrestore(&random_ready_notifier.lock, flags);
return ret;
}
#define warn_unseeded_randomness() \
if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \
printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \
__func__, (void *)_RET_IP_, crng_init)
/*********************************************************************
*
* Fast key erasure RNG, the "crng".
*
* These functions expand entropy from the entropy extractor into
* long streams for external consumption using the "fast key erasure"
* RNG described at <https://blog.cr.yp.to/20170723-random.html>.
*
* There are a few exported interfaces for use by other drivers:
*
* void get_random_bytes(void *buf, size_t len)
* u8 get_random_u8()
* u16 get_random_u16()
* u32 get_random_u32()
* u32 get_random_u32_below(u32 ceil)
* u32 get_random_u32_above(u32 floor)
* u32 get_random_u32_inclusive(u32 floor, u32 ceil)
* u64 get_random_u64()
* unsigned long get_random_long()
*
* These interfaces will return the requested number of random bytes
* into the given buffer or as a return value. This is equivalent to
* a read from /dev/urandom. The u8, u16, u32, u64, long family of
* functions may be higher performance for one-off random integers,
* because they do a bit of buffering and do not invoke reseeding
* until the buffer is emptied.
*
*********************************************************************/
enum {
CRNG_RESEED_START_INTERVAL = HZ,
CRNG_RESEED_INTERVAL = 60 * HZ
};
static struct {
u8 key[CHACHA_KEY_SIZE] __aligned(__alignof__(long));
unsigned long generation;
spinlock_t lock;
} base_crng = {
.lock = __SPIN_LOCK_UNLOCKED(base_crng.lock)
};
struct crng {
u8 key[CHACHA_KEY_SIZE];
unsigned long generation;
local_lock_t lock;
};
static DEFINE_PER_CPU(struct crng, crngs) = {
.generation = ULONG_MAX,
.lock = INIT_LOCAL_LOCK(crngs.lock),
};
/*
* Return the interval until the next reseeding, which is normally
* CRNG_RESEED_INTERVAL, but during early boot, it is at an interval
* proportional to the uptime.
*/
static unsigned int crng_reseed_interval(void)
{
static bool early_boot = true;
if (unlikely(READ_ONCE(early_boot))) {
time64_t uptime = ktime_get_seconds();
if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2)
WRITE_ONCE(early_boot, false);
else
return max_t(unsigned int, CRNG_RESEED_START_INTERVAL,
(unsigned int)uptime / 2 * HZ);
}
return CRNG_RESEED_INTERVAL;
}
/* Used by crng_reseed() and crng_make_state() to extract a new seed from the input pool. */
static void extract_entropy(void *buf, size_t len);
/* This extracts a new crng key from the input pool. */
static void crng_reseed(struct work_struct *work)
{
static DECLARE_DELAYED_WORK(next_reseed, crng_reseed);
unsigned long flags;
unsigned long next_gen;
u8 key[CHACHA_KEY_SIZE];
/* Immediately schedule the next reseeding, so that it fires sooner rather than later. */
if (likely(system_unbound_wq))
queue_delayed_work(system_unbound_wq, &next_reseed, crng_reseed_interval());
extract_entropy(key, sizeof(key));
/*
* We copy the new key into the base_crng, overwriting the old one,
* and update the generation counter. We avoid hitting ULONG_MAX,
* because the per-cpu crngs are initialized to ULONG_MAX, so this
* forces new CPUs that come online to always initialize.
*/
spin_lock_irqsave(&base_crng.lock, flags);
memcpy(base_crng.key, key, sizeof(base_crng.key));
next_gen = base_crng.generation + 1;
if (next_gen == ULONG_MAX)
++next_gen;
WRITE_ONCE(base_crng.generation, next_gen);
if (!static_branch_likely(&crng_is_ready))
crng_init = CRNG_READY;
spin_unlock_irqrestore(&base_crng.lock, flags);
memzero_explicit(key, sizeof(key));
}
/*
* This generates a ChaCha block using the provided key, and then
* immediately overwrites that key with half the block. It returns
* the resultant ChaCha state to the user, along with the second
* half of the block containing 32 bytes of random data that may
* be used; random_data_len may not be greater than 32.
*
* The returned ChaCha state contains within it a copy of the old
* key value, at index 4, so the state should always be zeroed out
* immediately after using in order to maintain forward secrecy.
* If the state cannot be erased in a timely manner, then it is
* safer to set the random_data parameter to &chacha_state[4] so
* that this function overwrites it before returning.
*/
static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE],
u32 chacha_state[CHACHA_STATE_WORDS],
u8 *random_data, size_t random_data_len)
{
u8 first_block[CHACHA_BLOCK_SIZE];
BUG_ON(random_data_len > 32);
chacha_init_consts(chacha_state);
memcpy(&chacha_state[4], key, CHACHA_KEY_SIZE);
memset(&chacha_state[12], 0, sizeof(u32) * 4);
chacha20_block(chacha_state, first_block);
memcpy(key, first_block, CHACHA_KEY_SIZE);
memcpy(random_data, first_block + CHACHA_KEY_SIZE, random_data_len);
memzero_explicit(first_block, sizeof(first_block));
}
/*
* This function returns a ChaCha state that you may use for generating
* random data. It also returns up to 32 bytes on its own of random data
* that may be used; random_data_len may not be greater than 32.
*/
static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS],
u8 *random_data, size_t random_data_len)
{
unsigned long flags;
struct crng *crng;
BUG_ON(random_data_len > 32);
/*
* For the fast path, we check whether we're ready, unlocked first, and
* then re-check once locked later. In the case where we're really not
* ready, we do fast key erasure with the base_crng directly, extracting
* when crng_init is CRNG_EMPTY.
*/
if (!crng_ready()) {
bool ready;
spin_lock_irqsave(&base_crng.lock, flags);
ready = crng_ready();
if (!ready) {
if (crng_init == CRNG_EMPTY)
extract_entropy(base_crng.key, sizeof(base_crng.key));
crng_fast_key_erasure(base_crng.key, chacha_state,
random_data, random_data_len);
}
spin_unlock_irqrestore(&base_crng.lock, flags);
if (!ready)
return;
}
local_lock_irqsave(&crngs.lock, flags);
crng = raw_cpu_ptr(&crngs);
/*
* If our per-cpu crng is older than the base_crng, then it means
* somebody reseeded the base_crng. In that case, we do fast key
* erasure on the base_crng, and use its output as the new key
* for our per-cpu crng. This brings us up to date with base_crng.
*/
if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) {
spin_lock(&base_crng.lock);
crng_fast_key_erasure(base_crng.key, chacha_state,
crng->key, sizeof(crng->key));
crng->generation = base_crng.generation;
spin_unlock(&base_crng.lock);
}
/*
* Finally, when we've made it this far, our per-cpu crng has an up
* to date key, and we can do fast key erasure with it to produce
* some random data and a ChaCha state for the caller. All other
* branches of this function are "unlikely", so most of the time we
* should wind up here immediately.
*/
crng_fast_key_erasure(crng->key, chacha_state, random_data, random_data_len);
local_unlock_irqrestore(&crngs.lock, flags);
}
static void _get_random_bytes(void *buf, size_t len)
{
u32 chacha_state[CHACHA_STATE_WORDS];
u8 tmp[CHACHA_BLOCK_SIZE];
size_t first_block_len;
if (!len)
return;
first_block_len = min_t(size_t, 32, len);
crng_make_state(chacha_state, buf, first_block_len);
len -= first_block_len;
buf += first_block_len;
while (len) {
if (len < CHACHA_BLOCK_SIZE) {
chacha20_block(chacha_state, tmp);
memcpy(buf, tmp, len);
memzero_explicit(tmp, sizeof(tmp));
break;
}
chacha20_block(chacha_state, buf);
if (unlikely(chacha_state[12] == 0))
++chacha_state[13];
len -= CHACHA_BLOCK_SIZE;
buf += CHACHA_BLOCK_SIZE;
}
memzero_explicit(chacha_state, sizeof(chacha_state));
}
/*
* This returns random bytes in arbitrary quantities. The quality of the
* random bytes is good as /dev/urandom. In order to ensure that the
* randomness provided by this function is okay, the function
* wait_for_random_bytes() should be called and return 0 at least once
* at any point prior.
*/
void get_random_bytes(void *buf, size_t len)
{
warn_unseeded_randomness();
_get_random_bytes(buf, len);
}
EXPORT_SYMBOL(get_random_bytes);
static ssize_t get_random_bytes_user(struct iov_iter *iter)
{
u32 chacha_state[CHACHA_STATE_WORDS];
u8 block[CHACHA_BLOCK_SIZE];
size_t ret = 0, copied;
if (unlikely(!iov_iter_count(iter)))
return 0;
/*
* Immediately overwrite the ChaCha key at index 4 with random
* bytes, in case userspace causes copy_to_iter() below to sleep
* forever, so that we still retain forward secrecy in that case.
*/
crng_make_state(chacha_state, (u8 *)&chacha_state[4], CHACHA_KEY_SIZE);
/*
* However, if we're doing a read of len <= 32, we don't need to
* use chacha_state after, so we can simply return those bytes to
* the user directly.
*/
if (iov_iter_count(iter) <= CHACHA_KEY_SIZE) {
ret = copy_to_iter(&chacha_state[4], CHACHA_KEY_SIZE, iter);
goto out_zero_chacha;
}
for (;;) {
chacha20_block(chacha_state, block);
if (unlikely(chacha_state[12] == 0))
++chacha_state[13];
copied = copy_to_iter(block, sizeof(block), iter);
ret += copied;
if (!iov_iter_count(iter) || copied != sizeof(block))
break;
BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
if (ret % PAGE_SIZE == 0) {
if (signal_pending(current))
break;
cond_resched();
}
}
memzero_explicit(block, sizeof(block));
out_zero_chacha:
memzero_explicit(chacha_state, sizeof(chacha_state));
return ret ? ret : -EFAULT;
}
/*
* Batched entropy returns random integers. The quality of the random
* number is good as /dev/urandom. In order to ensure that the randomness
* provided by this function is okay, the function wait_for_random_bytes()
* should be called and return 0 at least once at any point prior.
*/
#define DEFINE_BATCHED_ENTROPY(type) \
struct batch_ ##type { \
/* \
* We make this 1.5x a ChaCha block, so that we get the \
* remaining 32 bytes from fast key erasure, plus one full \
* block from the detached ChaCha state. We can increase \
* the size of this later if needed so long as we keep the \
* formula of (integer_blocks + 0.5) * CHACHA_BLOCK_SIZE. \
*/ \
type entropy[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(type))]; \
local_lock_t lock; \
unsigned long generation; \
unsigned int position; \
}; \
\
static DEFINE_PER_CPU(struct batch_ ##type, batched_entropy_ ##type) = { \
.lock = INIT_LOCAL_LOCK(batched_entropy_ ##type.lock), \
.position = UINT_MAX \
}; \
\
type get_random_ ##type(void) \
{ \
type ret; \
unsigned long flags; \
struct batch_ ##type *batch; \
unsigned long next_gen; \
\
warn_unseeded_randomness(); \
\
if (!crng_ready()) { \
_get_random_bytes(&ret, sizeof(ret)); \
return ret; \
} \
\
local_lock_irqsave(&batched_entropy_ ##type.lock, flags); \
batch = raw_cpu_ptr(&batched_entropy_##type); \
\
next_gen = READ_ONCE(base_crng.generation); \
if (batch->position >= ARRAY_SIZE(batch->entropy) || \
next_gen != batch->generation) { \
_get_random_bytes(batch->entropy, sizeof(batch->entropy)); \
batch->position = 0; \
batch->generation = next_gen; \
} \
\
ret = batch->entropy[batch->position]; \
batch->entropy[batch->position] = 0; \
++batch->position; \
local_unlock_irqrestore(&batched_entropy_ ##type.lock, flags); \
return ret; \
} \
EXPORT_SYMBOL(get_random_ ##type);
DEFINE_BATCHED_ENTROPY(u8)
DEFINE_BATCHED_ENTROPY(u16)
DEFINE_BATCHED_ENTROPY(u32)
DEFINE_BATCHED_ENTROPY(u64)
u32 __get_random_u32_below(u32 ceil)
{
/*
* This is the slow path for variable ceil. It is still fast, most of
* the time, by doing traditional reciprocal multiplication and
* opportunistically comparing the lower half to ceil itself, before
* falling back to computing a larger bound, and then rejecting samples
* whose lower half would indicate a range indivisible by ceil. The use
* of `-ceil % ceil` is analogous to `2^32 % ceil`, but is computable
* in 32-bits.
*/
u32 rand = get_random_u32();
u64 mult;
/*
* This function is technically undefined for ceil == 0, and in fact
* for the non-underscored constant version in the header, we build bug
* on that. But for the non-constant case, it's convenient to have that
* evaluate to being a straight call to get_random_u32(), so that
* get_random_u32_inclusive() can work over its whole range without
* undefined behavior.
*/
if (unlikely(!ceil))
return rand;
mult = (u64)ceil * rand;
if (unlikely((u32)mult < ceil)) {
u32 bound = -ceil % ceil;
while (unlikely((u32)mult < bound))
mult = (u64)ceil * get_random_u32();
}
return mult >> 32;
}
EXPORT_SYMBOL(__get_random_u32_below);
#ifdef CONFIG_SMP
/*
* This function is called when the CPU is coming up, with entry
* CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP.
*/
int __cold random_prepare_cpu(unsigned int cpu)
{
/*
* When the cpu comes back online, immediately invalidate both
* the per-cpu crng and all batches, so that we serve fresh
* randomness.
*/
per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX;
per_cpu_ptr(&batched_entropy_u8, cpu)->position = UINT_MAX;
per_cpu_ptr(&batched_entropy_u16, cpu)->position = UINT_MAX;
per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX;
per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX;
return 0;
}
#endif
/**********************************************************************
*
* Entropy accumulation and extraction routines.
*
* Callers may add entropy via:
*
* static void mix_pool_bytes(const void *buf, size_t len)
*
* After which, if added entropy should be credited:
*
* static void credit_init_bits(size_t bits)
*
* Finally, extract entropy via:
*
* static void extract_entropy(void *buf, size_t len)
*
**********************************************************************/
enum {
POOL_BITS = BLAKE2S_HASH_SIZE * 8,
POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
};
static struct {
struct blake2s_state hash;
spinlock_t lock;
unsigned int init_bits;
} input_pool = {
.hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE),
BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4,
BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 },
.hash.outlen = BLAKE2S_HASH_SIZE,
.lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
};
static void _mix_pool_bytes(const void *buf, size_t len)
{
blake2s_update(&input_pool.hash, buf, len);
}
/*
* This function adds bytes into the input pool. It does not
* update the initialization bit counter; the caller should call
* credit_init_bits if this is appropriate.
*/
static void mix_pool_bytes(const void *buf, size_t len)
{
unsigned long flags;
spin_lock_irqsave(&input_pool.lock, flags);
_mix_pool_bytes(buf, len);
spin_unlock_irqrestore(&input_pool.lock, flags);
}
/*
* This is an HKDF-like construction for using the hashed collected entropy
* as a PRF key, that's then expanded block-by-block.
*/
static void extract_entropy(void *buf, size_t len)
{
unsigned long flags;
u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE];
struct {
unsigned long rdseed[32 / sizeof(long)];
size_t counter;
} block;
size_t i, longs;
for (i = 0; i < ARRAY_SIZE(block.rdseed);) {
longs = arch_get_random_seed_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i);
if (longs) {
i += longs;
continue;
}
longs = arch_get_random_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i);
if (longs) {
i += longs;
continue;
}
block.rdseed[i++] = random_get_entropy();
}
spin_lock_irqsave(&input_pool.lock, flags);
/* seed = HASHPRF(last_key, entropy_input) */
blake2s_final(&input_pool.hash, seed);
/* next_key = HASHPRF(seed, RDSEED || 0) */
block.counter = 0;
blake2s(next_key, (u8 *)&block, seed, sizeof(next_key), sizeof(block), sizeof(seed));
blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key));
spin_unlock_irqrestore(&input_pool.lock, flags);
memzero_explicit(next_key, sizeof(next_key));
while (len) {
i = min_t(size_t, len, BLAKE2S_HASH_SIZE);
/* output = HASHPRF(seed, RDSEED || ++counter) */
++block.counter;
blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed));
len -= i;
buf += i;
}
memzero_explicit(seed, sizeof(seed));
memzero_explicit(&block, sizeof(block));
}
#define credit_init_bits(bits) if (!crng_ready()) _credit_init_bits(bits)
static void __cold _credit_init_bits(size_t bits)
{
static DECLARE_WORK(set_ready, crng_set_ready);
unsigned int new, orig, add;
unsigned long flags;
if (!bits)
return;
add = min_t(size_t, bits, POOL_BITS);
orig = READ_ONCE(input_pool.init_bits);
do {
new = min_t(unsigned int, POOL_BITS, orig + add);
} while (!try_cmpxchg(&input_pool.init_bits, &orig, new));
if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
crng_reseed(NULL); /* Sets crng_init to CRNG_READY under base_crng.lock. */
if (static_key_initialized && system_unbound_wq)
queue_work(system_unbound_wq, &set_ready);
atomic_notifier_call_chain(&random_ready_notifier, 0, NULL);
wake_up_interruptible(&crng_init_wait);
kill_fasync(&fasync, SIGIO, POLL_IN);
pr_notice("crng init done\n");
if (urandom_warning.missed)
pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
urandom_warning.missed);
} else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
spin_lock_irqsave(&base_crng.lock, flags);
/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
if (crng_init == CRNG_EMPTY) {
extract_entropy(base_crng.key, sizeof(base_crng.key));
crng_init = CRNG_EARLY;
}
spin_unlock_irqrestore(&base_crng.lock, flags);
}
}
/**********************************************************************
*
* Entropy collection routines.
*
* The following exported functions are used for pushing entropy into
* the above entropy accumulation routines:
*
* void add_device_randomness(const void *buf, size_t len);
* void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after);
* void add_bootloader_randomness(const void *buf, size_t len);
* void add_vmfork_randomness(const void *unique_vm_id, size_t len);
* void add_interrupt_randomness(int irq);
* void add_input_randomness(unsigned int type, unsigned int code, unsigned int value);
* void add_disk_randomness(struct gendisk *disk);
*
* add_device_randomness() adds data to the input pool that
* is likely to differ between two devices (or possibly even per boot).
* This would be things like MAC addresses or serial numbers, or the
* read-out of the RTC. This does *not* credit any actual entropy to
* the pool, but it initializes the pool to different values for devices
* that might otherwise be identical and have very little entropy
* available to them (particularly common in the embedded world).
*
* add_hwgenerator_randomness() is for true hardware RNGs, and will credit
* entropy as specified by the caller. If the entropy pool is full it will
* block until more entropy is needed.
*
* add_bootloader_randomness() is called by bootloader drivers, such as EFI
* and device tree, and credits its input depending on whether or not the
* command line option 'random.trust_bootloader'.
*
* add_vmfork_randomness() adds a unique (but not necessarily secret) ID
* representing the current instance of a VM to the pool, without crediting,
* and then force-reseeds the crng so that it takes effect immediately.
*
* add_interrupt_randomness() uses the interrupt timing as random
* inputs to the entropy pool. Using the cycle counters and the irq source
* as inputs, it feeds the input pool roughly once a second or after 64
* interrupts, crediting 1 bit of entropy for whichever comes first.
*
* add_input_randomness() uses the input layer interrupt timing, as well
* as the event type information from the hardware.
*
* add_disk_randomness() uses what amounts to the seek time of block
* layer request events, on a per-disk_devt basis, as input to the
* entropy pool. Note that high-speed solid state drives with very low
* seek times do not make for good sources of entropy, as their seek
* times are usually fairly consistent.
*
* The last two routines try to estimate how many bits of entropy
* to credit. They do this by keeping track of the first and second
* order deltas of the event timings.
*
**********************************************************************/
static bool trust_cpu __initdata = true;
static bool trust_bootloader __initdata = true;
static int __init parse_trust_cpu(char *arg)
{
return kstrtobool(arg, &trust_cpu);
}
static int __init parse_trust_bootloader(char *arg)
{
return kstrtobool(arg, &trust_bootloader);
}
early_param("random.trust_cpu", parse_trust_cpu);
early_param("random.trust_bootloader", parse_trust_bootloader);
static int random_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
unsigned long flags, entropy = random_get_entropy();
/*
* Encode a representation of how long the system has been suspended,
* in a way that is distinct from prior system suspends.
*/
ktime_t stamps[] = { ktime_get(), ktime_get_boottime(), ktime_get_real() };
spin_lock_irqsave(&input_pool.lock, flags);
_mix_pool_bytes(&action, sizeof(action));
_mix_pool_bytes(stamps, sizeof(stamps));
_mix_pool_bytes(&entropy, sizeof(entropy));
spin_unlock_irqrestore(&input_pool.lock, flags);
if (crng_ready() && (action == PM_RESTORE_PREPARE ||
(action == PM_POST_SUSPEND && !IS_ENABLED(CONFIG_PM_AUTOSLEEP) &&
!IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP)))) {
crng_reseed(NULL);
pr_notice("crng reseeded on system resumption\n");
}
return 0;
}
static struct notifier_block pm_notifier = { .notifier_call = random_pm_notification };
/*
* This is called extremely early, before time keeping functionality is
* available, but arch randomness is. Interrupts are not yet enabled.
*/
void __init random_init_early(const char *command_line)
{
unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)];
size_t i, longs, arch_bits;
#if defined(LATENT_ENTROPY_PLUGIN)
static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy;
_mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed));
#endif
for (i = 0, arch_bits = sizeof(entropy) * 8; i < ARRAY_SIZE(entropy);) {
longs = arch_get_random_seed_longs(entropy, ARRAY_SIZE(entropy) - i);
if (longs) {
_mix_pool_bytes(entropy, sizeof(*entropy) * longs);
i += longs;
continue;
}
longs = arch_get_random_longs(entropy, ARRAY_SIZE(entropy) - i);
if (longs) {
_mix_pool_bytes(entropy, sizeof(*entropy) * longs);
i += longs;
continue;
}
arch_bits -= sizeof(*entropy) * 8;
++i;
}
_mix_pool_bytes(init_utsname(), sizeof(*(init_utsname())));
_mix_pool_bytes(command_line, strlen(command_line));
/* Reseed if already seeded by earlier phases. */
if (crng_ready())
crng_reseed(NULL);
else if (trust_cpu)
_credit_init_bits(arch_bits);
}
/*
* This is called a little bit after the prior function, and now there is
* access to timestamps counters. Interrupts are not yet enabled.
*/
void __init random_init(void)
{
unsigned long entropy = random_get_entropy();
ktime_t now = ktime_get_real();
_mix_pool_bytes(&now, sizeof(now));
_mix_pool_bytes(&entropy, sizeof(entropy));
add_latent_entropy();
/*
* If we were initialized by the cpu or bootloader before jump labels
* or workqueues are initialized, then we should enable the static
* branch here, where it's guaranteed that these have been initialized.
*/
if (!static_branch_likely(&crng_is_ready) && crng_init >= CRNG_READY)
crng_set_ready(NULL);
/* Reseed if already seeded by earlier phases. */
if (crng_ready())
crng_reseed(NULL);
WARN_ON(register_pm_notifier(&pm_notifier));
WARN(!entropy, "Missing cycle counter and fallback timer; RNG "
"entropy collection will consequently suffer.");
}
/*
* Add device- or boot-specific data to the input pool to help
* initialize it.
*
* None of this adds any entropy; it is meant to avoid the problem of
* the entropy pool having similar initial state across largely
* identical devices.
*/
void add_device_randomness(const void *buf, size_t len)
{
unsigned long entropy = random_get_entropy();
unsigned long flags;
spin_lock_irqsave(&input_pool.lock, flags);
_mix_pool_bytes(&entropy, sizeof(entropy));
_mix_pool_bytes(buf, len);
spin_unlock_irqrestore(&input_pool.lock, flags);
}
EXPORT_SYMBOL(add_device_randomness);
/*
* Interface for in-kernel drivers of true hardware RNGs. Those devices
* may produce endless random bits, so this function will sleep for
* some amount of time after, if the sleep_after parameter is true.
*/
void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after)
{
mix_pool_bytes(buf, len);
credit_init_bits(entropy);
/*
* Throttle writing to once every reseed interval, unless we're not yet
* initialized or no entropy is credited.
*/
if (sleep_after && !kthread_should_stop() && (crng_ready() || !entropy))
schedule_timeout_interruptible(crng_reseed_interval());
}
EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
/*
* Handle random seed passed by bootloader, and credit it depending
* on the command line option 'random.trust_bootloader'.
*/
void __init add_bootloader_randomness(const void *buf, size_t len)
{
mix_pool_bytes(buf, len);
if (trust_bootloader)
credit_init_bits(len * 8);
}
#if IS_ENABLED(CONFIG_VMGENID)
static BLOCKING_NOTIFIER_HEAD(vmfork_chain);
/*
* Handle a new unique VM ID, which is unique, not secret, so we
* don't credit it, but we do immediately force a reseed after so
* that it's used by the crng posthaste.
*/
void __cold add_vmfork_randomness(const void *unique_vm_id, size_t len)
{
add_device_randomness(unique_vm_id, len);
if (crng_ready()) {
crng_reseed(NULL);
pr_notice("crng reseeded due to virtual machine fork\n");
}
blocking_notifier_call_chain(&vmfork_chain, 0, NULL);
}
#if IS_MODULE(CONFIG_VMGENID)
EXPORT_SYMBOL_GPL(add_vmfork_randomness);
#endif
int __cold register_random_vmfork_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_register(&vmfork_chain, nb);
}
EXPORT_SYMBOL_GPL(register_random_vmfork_notifier);
int __cold unregister_random_vmfork_notifier(struct notifier_block *nb)
{
return blocking_notifier_chain_unregister(&vmfork_chain, nb);
}
EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier);
#endif
struct fast_pool {
unsigned long pool[4];
unsigned long last;
unsigned int count;
struct timer_list mix;
};
static void mix_interrupt_randomness(struct timer_list *work);
static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
#ifdef CONFIG_64BIT
#define FASTMIX_PERM SIPHASH_PERMUTATION
.pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 },
#else
#define FASTMIX_PERM HSIPHASH_PERMUTATION
.pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 },
#endif
.mix = __TIMER_INITIALIZER(mix_interrupt_randomness, 0)
};
/*
* This is [Half]SipHash-1-x, starting from an empty key. Because
* the key is fixed, it assumes that its inputs are non-malicious,
* and therefore this has no security on its own. s represents the
* four-word SipHash state, while v represents a two-word input.
*/
static void fast_mix(unsigned long s[4], unsigned long v1, unsigned long v2)
{
s[3] ^= v1;
FASTMIX_PERM(s[0], s[1], s[2], s[3]);
s[0] ^= v1;
s[3] ^= v2;
FASTMIX_PERM(s[0], s[1], s[2], s[3]);
s[0] ^= v2;
}
#ifdef CONFIG_SMP
/*
* This function is called when the CPU has just come online, with
* entry CPUHP_AP_RANDOM_ONLINE, just after CPUHP_AP_WORKQUEUE_ONLINE.
*/
int __cold random_online_cpu(unsigned int cpu)
{
/*
* During CPU shutdown and before CPU onlining, add_interrupt_
* randomness() may schedule mix_interrupt_randomness(), and
* set the MIX_INFLIGHT flag. However, because the worker can
* be scheduled on a different CPU during this period, that
* flag will never be cleared. For that reason, we zero out
* the flag here, which runs just after workqueues are onlined
* for the CPU again. This also has the effect of setting the
* irq randomness count to zero so that new accumulated irqs
* are fresh.
*/
per_cpu_ptr(&irq_randomness, cpu)->count = 0;
return 0;
}
#endif
static void mix_interrupt_randomness(struct timer_list *work)
{
struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix);
/*
* The size of the copied stack pool is explicitly 2 longs so that we
* only ever ingest half of the siphash output each time, retaining
* the other half as the next "key" that carries over. The entropy is
* supposed to be sufficiently dispersed between bits so on average
* we don't wind up "losing" some.
*/
unsigned long pool[2];
unsigned int count;
/* Check to see if we're running on the wrong CPU due to hotplug. */
local_irq_disable();
if (fast_pool != this_cpu_ptr(&irq_randomness)) {
local_irq_enable();
return;
}
/*
* Copy the pool to the stack so that the mixer always has a
* consistent view, before we reenable irqs again.
*/
memcpy(pool, fast_pool->pool, sizeof(pool));
count = fast_pool->count;
fast_pool->count = 0;
fast_pool->last = jiffies;
local_irq_enable();
mix_pool_bytes(pool, sizeof(pool));
credit_init_bits(clamp_t(unsigned int, (count & U16_MAX) / 64, 1, sizeof(pool) * 8));
memzero_explicit(pool, sizeof(pool));
}
void add_interrupt_randomness(int irq)
{
enum { MIX_INFLIGHT = 1U << 31 };
unsigned long entropy = random_get_entropy();
struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
struct pt_regs *regs = get_irq_regs();
unsigned int new_count;
fast_mix(fast_pool->pool, entropy,
(regs ? instruction_pointer(regs) : _RET_IP_) ^ swab(irq));
new_count = ++fast_pool->count;
if (new_count & MIX_INFLIGHT)
return;
if (new_count < 1024 && !time_is_before_jiffies(fast_pool->last + HZ))
return;
fast_pool->count |= MIX_INFLIGHT; if (!timer_pending(&fast_pool->mix)) { fast_pool->mix.expires = jiffies;
add_timer_on(&fast_pool->mix, raw_smp_processor_id());
}
}
EXPORT_SYMBOL_GPL(add_interrupt_randomness);
/* There is one of these per entropy source */
struct timer_rand_state {
unsigned long last_time;
long last_delta, last_delta2;
};
/*
* This function adds entropy to the entropy "pool" by using timing
* delays. It uses the timer_rand_state structure to make an estimate
* of how many bits of entropy this call has added to the pool. The
* value "num" is also added to the pool; it should somehow describe
* the type of event that just happened.
*/
static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
{
unsigned long entropy = random_get_entropy(), now = jiffies, flags;
long delta, delta2, delta3;
unsigned int bits;
/*
* If we're in a hard IRQ, add_interrupt_randomness() will be called
* sometime after, so mix into the fast pool.
*/
if (in_hardirq()) {
fast_mix(this_cpu_ptr(&irq_randomness)->pool, entropy, num);
} else {
spin_lock_irqsave(&input_pool.lock, flags);
_mix_pool_bytes(&entropy, sizeof(entropy));
_mix_pool_bytes(&num, sizeof(num));
spin_unlock_irqrestore(&input_pool.lock, flags);
}
if (crng_ready()) return;
/*
* Calculate number of bits of randomness we probably added.
* We take into account the first, second and third-order deltas
* in order to make our estimate.
*/
delta = now - READ_ONCE(state->last_time);
WRITE_ONCE(state->last_time, now);
delta2 = delta - READ_ONCE(state->last_delta);
WRITE_ONCE(state->last_delta, delta);
delta3 = delta2 - READ_ONCE(state->last_delta2);
WRITE_ONCE(state->last_delta2, delta2);
if (delta < 0)
delta = -delta;
if (delta2 < 0)
delta2 = -delta2; if (delta3 < 0) delta3 = -delta3; if (delta > delta2)
delta = delta2;
if (delta > delta3)
delta = delta3;
/*
* delta is now minimum absolute delta. Round down by 1 bit
* on general principles, and limit entropy estimate to 11 bits.
*/
bits = min(fls(delta >> 1), 11);
/*
* As mentioned above, if we're in a hard IRQ, add_interrupt_randomness()
* will run after this, which uses a different crediting scheme of 1 bit
* per every 64 interrupts. In order to let that function do accounting
* close to the one in this function, we credit a full 64/64 bit per bit,
* and then subtract one to account for the extra one added.
*/
if (in_hardirq())
this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;
else
_credit_init_bits(bits);
}
void add_input_randomness(unsigned int type, unsigned int code, unsigned int value)
{
static unsigned char last_value;
static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
/* Ignore autorepeat and the like. */
if (value == last_value)
return;
last_value = value;
add_timer_randomness(&input_timer_state,
(type << 4) ^ code ^ (code >> 4) ^ value);
}
EXPORT_SYMBOL_GPL(add_input_randomness);
#ifdef CONFIG_BLOCK
void add_disk_randomness(struct gendisk *disk)
{
if (!disk || !disk->random)
return;
/* First major is 1, so we get >= 0x200 here. */
add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
}
EXPORT_SYMBOL_GPL(add_disk_randomness);
void __cold rand_initialize_disk(struct gendisk *disk)
{
struct timer_rand_state *state;
/*
* If kzalloc returns null, we just won't use that entropy
* source.
*/
state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
if (state) {
state->last_time = INITIAL_JIFFIES;
disk->random = state;
}
}
#endif
struct entropy_timer_state {
unsigned long entropy;
struct timer_list timer;
atomic_t samples;
unsigned int samples_per_bit;
};
/*
* Each time the timer fires, we expect that we got an unpredictable jump in
* the cycle counter. Even if the timer is running on another CPU, the timer
* activity will be touching the stack of the CPU that is generating entropy.
*
* Note that we don't re-arm the timer in the timer itself - we are happy to be
* scheduled away, since that just makes the load more complex, but we do not
* want the timer to keep ticking unless the entropy loop is running.
*
* So the re-arming always happens in the entropy loop itself.
*/
static void __cold entropy_timer(struct timer_list *timer)
{
struct entropy_timer_state *state = container_of(timer, struct entropy_timer_state, timer);
unsigned long entropy = random_get_entropy();
mix_pool_bytes(&entropy, sizeof(entropy));
if (atomic_inc_return(&state->samples) % state->samples_per_bit == 0)
credit_init_bits(1);
}
/*
* If we have an actual cycle counter, see if we can generate enough entropy
* with timing noise.
*/
static void __cold try_to_generate_entropy(void)
{
enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 15 };
u8 stack_bytes[sizeof(struct entropy_timer_state) + SMP_CACHE_BYTES - 1];
struct entropy_timer_state *stack = PTR_ALIGN((void *)stack_bytes, SMP_CACHE_BYTES);
unsigned int i, num_different = 0;
unsigned long last = random_get_entropy();
int cpu = -1;
for (i = 0; i < NUM_TRIAL_SAMPLES - 1; ++i) {
stack->entropy = random_get_entropy();
if (stack->entropy != last)
++num_different;
last = stack->entropy;
}
stack->samples_per_bit = DIV_ROUND_UP(NUM_TRIAL_SAMPLES, num_different + 1);
if (stack->samples_per_bit > MAX_SAMPLES_PER_BIT)
return;
atomic_set(&stack->samples, 0);
timer_setup_on_stack(&stack->timer, entropy_timer, 0);
while (!crng_ready() && !signal_pending(current)) {
/*
* Check !timer_pending() and then ensure that any previous callback has finished
* executing by checking try_to_del_timer_sync(), before queueing the next one.
*/
if (!timer_pending(&stack->timer) && try_to_del_timer_sync(&stack->timer) >= 0) {
struct cpumask timer_cpus;
unsigned int num_cpus;
/*
* Preemption must be disabled here, both to read the current CPU number
* and to avoid scheduling a timer on a dead CPU.
*/
preempt_disable();
/* Only schedule callbacks on timer CPUs that are online. */
cpumask_and(&timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask);
num_cpus = cpumask_weight(&timer_cpus);
/* In very bizarre case of misconfiguration, fallback to all online. */
if (unlikely(num_cpus == 0)) {
timer_cpus = *cpu_online_mask;
num_cpus = cpumask_weight(&timer_cpus);
}
/* Basic CPU round-robin, which avoids the current CPU. */
do {
cpu = cpumask_next(cpu, &timer_cpus);
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(&timer_cpus);
} while (cpu == smp_processor_id() && num_cpus > 1);
/* Expiring the timer at `jiffies` means it's the next tick. */
stack->timer.expires = jiffies;
add_timer_on(&stack->timer, cpu);
preempt_enable();
}
mix_pool_bytes(&stack->entropy, sizeof(stack->entropy));
schedule();
stack->entropy = random_get_entropy();
}
mix_pool_bytes(&stack->entropy, sizeof(stack->entropy));
del_timer_sync(&stack->timer);
destroy_timer_on_stack(&stack->timer);
}
/**********************************************************************
*
* Userspace reader/writer interfaces.
*
* getrandom(2) is the primary modern interface into the RNG and should
* be used in preference to anything else.
*
* Reading from /dev/random has the same functionality as calling
* getrandom(2) with flags=0. In earlier versions, however, it had
* vastly different semantics and should therefore be avoided, to
* prevent backwards compatibility issues.
*
* Reading from /dev/urandom has the same functionality as calling
* getrandom(2) with flags=GRND_INSECURE. Because it does not block
* waiting for the RNG to be ready, it should not be used.
*
* Writing to either /dev/random or /dev/urandom adds entropy to
* the input pool but does not credit it.
*
* Polling on /dev/random indicates when the RNG is initialized, on
* the read side, and when it wants new entropy, on the write side.
*
* Both /dev/random and /dev/urandom have the same set of ioctls for
* adding entropy, getting the entropy count, zeroing the count, and
* reseeding the crng.
*
**********************************************************************/
SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags)
{
struct iov_iter iter;
int ret;
if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE))
return -EINVAL;
/*
* Requesting insecure and blocking randomness at the same time makes
* no sense.
*/
if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM))
return -EINVAL;
if (!crng_ready() && !(flags & GRND_INSECURE)) {
if (flags & GRND_NONBLOCK)
return -EAGAIN;
ret = wait_for_random_bytes();
if (unlikely(ret))
return ret;
}
ret = import_ubuf(ITER_DEST, ubuf, len, &iter);
if (unlikely(ret))
return ret;
return get_random_bytes_user(&iter);
}
static __poll_t random_poll(struct file *file, poll_table *wait)
{
poll_wait(file, &crng_init_wait, wait);
return crng_ready() ? EPOLLIN | EPOLLRDNORM : EPOLLOUT | EPOLLWRNORM;
}
static ssize_t write_pool_user(struct iov_iter *iter)
{
u8 block[BLAKE2S_BLOCK_SIZE];
ssize_t ret = 0;
size_t copied;
if (unlikely(!iov_iter_count(iter)))
return 0;
for (;;) {
copied = copy_from_iter(block, sizeof(block), iter);
ret += copied;
mix_pool_bytes(block, copied);
if (!iov_iter_count(iter) || copied != sizeof(block))
break;
BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
if (ret % PAGE_SIZE == 0) {
if (signal_pending(current))
break;
cond_resched();
}
}
memzero_explicit(block, sizeof(block));
return ret ? ret : -EFAULT;
}
static ssize_t random_write_iter(struct kiocb *kiocb, struct iov_iter *iter)
{
return write_pool_user(iter);
}
static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
{
static int maxwarn = 10;
/*
* Opportunistically attempt to initialize the RNG on platforms that
* have fast cycle counters, but don't (for now) require it to succeed.
*/
if (!crng_ready())
try_to_generate_entropy();
if (!crng_ready()) {
if (!ratelimit_disable && maxwarn <= 0)
++urandom_warning.missed;
else if (ratelimit_disable || __ratelimit(&urandom_warning)) {
--maxwarn;
pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
current->comm, iov_iter_count(iter));
}
}
return get_random_bytes_user(iter);
}
static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
{
int ret;
if (!crng_ready() &&
((kiocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) ||
(kiocb->ki_filp->f_flags & O_NONBLOCK)))
return -EAGAIN;
ret = wait_for_random_bytes();
if (ret != 0)
return ret;
return get_random_bytes_user(iter);
}
static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
{
int __user *p = (int __user *)arg;
int ent_count;
switch (cmd) {
case RNDGETENTCNT:
/* Inherently racy, no point locking. */
if (put_user(input_pool.init_bits, p))
return -EFAULT;
return 0;
case RNDADDTOENTCNT:
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (get_user(ent_count, p))
return -EFAULT;
if (ent_count < 0)
return -EINVAL;
credit_init_bits(ent_count);
return 0;
case RNDADDENTROPY: {
struct iov_iter iter;
ssize_t ret;
int len;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (get_user(ent_count, p++))
return -EFAULT;
if (ent_count < 0)
return -EINVAL;
if (get_user(len, p++))
return -EFAULT;
ret = import_ubuf(ITER_SOURCE, p, len, &iter);
if (unlikely(ret))
return ret;
ret = write_pool_user(&iter);
if (unlikely(ret < 0))
return ret;
/* Since we're crediting, enforce that it was all written into the pool. */
if (unlikely(ret != len))
return -EFAULT;
credit_init_bits(ent_count);
return 0;
}
case RNDZAPENTCNT:
case RNDCLEARPOOL:
/* No longer has any effect. */
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
return 0;
case RNDRESEEDCRNG:
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (!crng_ready())
return -ENODATA;
crng_reseed(NULL);
return 0;
default:
return -EINVAL;
}
}
static int random_fasync(int fd, struct file *filp, int on)
{
return fasync_helper(fd, filp, on, &fasync);
}
const struct file_operations random_fops = {
.read_iter = random_read_iter,
.write_iter = random_write_iter,
.poll = random_poll,
.unlocked_ioctl = random_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.fasync = random_fasync,
.llseek = noop_llseek,
.splice_read = copy_splice_read,
.splice_write = iter_file_splice_write,
};
const struct file_operations urandom_fops = {
.read_iter = urandom_read_iter,
.write_iter = random_write_iter,
.unlocked_ioctl = random_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.fasync = random_fasync,
.llseek = noop_llseek,
.splice_read = copy_splice_read,
.splice_write = iter_file_splice_write,
};
/********************************************************************
*
* Sysctl interface.
*
* These are partly unused legacy knobs with dummy values to not break
* userspace and partly still useful things. They are usually accessible
* in /proc/sys/kernel/random/ and are as follows:
*
* - boot_id - a UUID representing the current boot.
*
* - uuid - a random UUID, different each time the file is read.
*
* - poolsize - the number of bits of entropy that the input pool can
* hold, tied to the POOL_BITS constant.
*
* - entropy_avail - the number of bits of entropy currently in the
* input pool. Always <= poolsize.
*
* - write_wakeup_threshold - the amount of entropy in the input pool
* below which write polls to /dev/random will unblock, requesting
* more entropy, tied to the POOL_READY_BITS constant. It is writable
* to avoid breaking old userspaces, but writing to it does not
* change any behavior of the RNG.
*
* - urandom_min_reseed_secs - fixed to the value CRNG_RESEED_INTERVAL.
* It is writable to avoid breaking old userspaces, but writing
* to it does not change any behavior of the RNG.
*
********************************************************************/
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
static int sysctl_random_min_urandom_seed = CRNG_RESEED_INTERVAL / HZ;
static int sysctl_random_write_wakeup_bits = POOL_READY_BITS;
static int sysctl_poolsize = POOL_BITS;
static u8 sysctl_bootid[UUID_SIZE];
/*
* This function is used to return both the bootid UUID, and random
* UUID. The difference is in whether table->data is NULL; if it is,
* then a new UUID is generated and returned to the user.
*/
static int proc_do_uuid(struct ctl_table *table, int write, void *buf,
size_t *lenp, loff_t *ppos)
{
u8 tmp_uuid[UUID_SIZE], *uuid;
char uuid_string[UUID_STRING_LEN + 1];
struct ctl_table fake_table = {
.data = uuid_string,
.maxlen = UUID_STRING_LEN
};
if (write)
return -EPERM;
uuid = table->data;
if (!uuid) {
uuid = tmp_uuid;
generate_random_uuid(uuid);
} else {
static DEFINE_SPINLOCK(bootid_spinlock);
spin_lock(&bootid_spinlock);
if (!uuid[8])
generate_random_uuid(uuid);
spin_unlock(&bootid_spinlock);
}
snprintf(uuid_string, sizeof(uuid_string), "%pU", uuid);
return proc_dostring(&fake_table, 0, buf, lenp, ppos);
}
/* The same as proc_dointvec, but writes don't change anything. */
static int proc_do_rointvec(struct ctl_table *table, int write, void *buf,
size_t *lenp, loff_t *ppos)
{
return write ? 0 : proc_dointvec(table, 0, buf, lenp, ppos);
}
static struct ctl_table random_table[] = {
{
.procname = "poolsize",
.data = &sysctl_poolsize,
.maxlen = sizeof(int),
.mode = 0444,
.proc_handler = proc_dointvec,
},
{
.procname = "entropy_avail",
.data = &input_pool.init_bits,
.maxlen = sizeof(int),
.mode = 0444,
.proc_handler = proc_dointvec,
},
{
.procname = "write_wakeup_threshold",
.data = &sysctl_random_write_wakeup_bits,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_do_rointvec,
},
{
.procname = "urandom_min_reseed_secs",
.data = &sysctl_random_min_urandom_seed,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_do_rointvec,
},
{
.procname = "boot_id",
.data = &sysctl_bootid,
.mode = 0444,
.proc_handler = proc_do_uuid,
},
{
.procname = "uuid",
.mode = 0444,
.proc_handler = proc_do_uuid,
},
};
/*
* random_init() is called before sysctl_init(),
* so we cannot call register_sysctl_init() in random_init()
*/
static int __init random_sysctls_init(void)
{
register_sysctl_init("kernel/random", random_table);
return 0;
}
device_initcall(random_sysctls_init);
#endif
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/slab.h>
#include <linux/stat.h>
#include <linux/sched/xacct.h>
#include <linux/fcntl.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/fsnotify.h>
#include <linux/security.h>
#include <linux/export.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/splice.h>
#include <linux/compat.h>
#include <linux/mount.h>
#include <linux/fs.h>
#include <linux/dax.h>
#include <linux/overflow.h>
#include "internal.h"
#include <linux/uaccess.h>
#include <asm/unistd.h>
/*
* Performs necessary checks before doing a clone.
*
* Can adjust amount of bytes to clone via @req_count argument.
* Returns appropriate error code that caller should return or
* zero in case the clone should be allowed.
*/
static int generic_remap_checks(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t *req_count, unsigned int remap_flags)
{
struct inode *inode_in = file_in->f_mapping->host;
struct inode *inode_out = file_out->f_mapping->host;
uint64_t count = *req_count;
uint64_t bcount;
loff_t size_in, size_out;
loff_t bs = inode_out->i_sb->s_blocksize;
int ret;
/* The start of both ranges must be aligned to an fs block. */
if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs))
return -EINVAL;
/* Ensure offsets don't wrap. */
if (pos_in + count < pos_in || pos_out + count < pos_out)
return -EINVAL;
size_in = i_size_read(inode_in);
size_out = i_size_read(inode_out);
/* Dedupe requires both ranges to be within EOF. */
if ((remap_flags & REMAP_FILE_DEDUP) &&
(pos_in >= size_in || pos_in + count > size_in ||
pos_out >= size_out || pos_out + count > size_out))
return -EINVAL;
/* Ensure the infile range is within the infile. */
if (pos_in >= size_in)
return -EINVAL;
count = min(count, size_in - (uint64_t)pos_in);
ret = generic_write_check_limits(file_out, pos_out, &count);
if (ret)
return ret;
/*
* If the user wanted us to link to the infile's EOF, round up to the
* next block boundary for this check.
*
* Otherwise, make sure the count is also block-aligned, having
* already confirmed the starting offsets' block alignment.
*/
if (pos_in + count == size_in &&
(!(remap_flags & REMAP_FILE_DEDUP) || pos_out + count == size_out)) {
bcount = ALIGN(size_in, bs) - pos_in;
} else {
if (!IS_ALIGNED(count, bs))
count = ALIGN_DOWN(count, bs);
bcount = count;
}
/* Don't allow overlapped cloning within the same file. */
if (inode_in == inode_out &&
pos_out + bcount > pos_in &&
pos_out < pos_in + bcount)
return -EINVAL;
/*
* We shortened the request but the caller can't deal with that, so
* bounce the request back to userspace.
*/
if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN))
return -EINVAL;
*req_count = count;
return 0;
}
static int remap_verify_area(struct file *file, loff_t pos, loff_t len,
bool write)
{
int mask = write ? MAY_WRITE : MAY_READ;
loff_t tmp;
int ret;
if (unlikely(pos < 0 || len < 0))
return -EINVAL;
if (unlikely(check_add_overflow(pos, len, &tmp)))
return -EINVAL;
ret = security_file_permission(file, mask);
if (ret)
return ret;
return fsnotify_file_area_perm(file, mask, &pos, len);
}
/*
* Ensure that we don't remap a partial EOF block in the middle of something
* else. Assume that the offsets have already been checked for block
* alignment.
*
* For clone we only link a partial EOF block above or at the destination file's
* EOF. For deduplication we accept a partial EOF block only if it ends at the
* destination file's EOF (can not link it into the middle of a file).
*
* Shorten the request if possible.
*/
static int generic_remap_check_len(struct inode *inode_in,
struct inode *inode_out,
loff_t pos_out,
loff_t *len,
unsigned int remap_flags)
{
u64 blkmask = i_blocksize(inode_in) - 1;
loff_t new_len = *len;
if ((*len & blkmask) == 0)
return 0;
if (pos_out + *len < i_size_read(inode_out))
new_len &= ~blkmask;
if (new_len == *len)
return 0;
if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
*len = new_len;
return 0;
}
return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
}
/* Read a page's worth of file data into the page cache. */
static struct folio *vfs_dedupe_get_folio(struct file *file, loff_t pos)
{
return read_mapping_folio(file->f_mapping, pos >> PAGE_SHIFT, file);
}
/*
* Lock two folios, ensuring that we lock in offset order if the folios
* are from the same file.
*/
static void vfs_lock_two_folios(struct folio *folio1, struct folio *folio2)
{
/* Always lock in order of increasing index. */
if (folio1->index > folio2->index)
swap(folio1, folio2);
folio_lock(folio1);
if (folio1 != folio2)
folio_lock(folio2);
}
/* Unlock two folios, being careful not to unlock the same folio twice. */
static void vfs_unlock_two_folios(struct folio *folio1, struct folio *folio2)
{
folio_unlock(folio1);
if (folio1 != folio2)
folio_unlock(folio2);
}
/*
* Compare extents of two files to see if they are the same.
* Caller must have locked both inodes to prevent write races.
*/
static int vfs_dedupe_file_range_compare(struct file *src, loff_t srcoff,
struct file *dest, loff_t dstoff,
loff_t len, bool *is_same)
{
bool same = true;
int error = -EINVAL;
while (len) {
struct folio *src_folio, *dst_folio;
void *src_addr, *dst_addr;
loff_t cmp_len = min(PAGE_SIZE - offset_in_page(srcoff),
PAGE_SIZE - offset_in_page(dstoff));
cmp_len = min(cmp_len, len);
if (cmp_len <= 0)
goto out_error;
src_folio = vfs_dedupe_get_folio(src, srcoff);
if (IS_ERR(src_folio)) {
error = PTR_ERR(src_folio);
goto out_error;
}
dst_folio = vfs_dedupe_get_folio(dest, dstoff);
if (IS_ERR(dst_folio)) {
error = PTR_ERR(dst_folio);
folio_put(src_folio);
goto out_error;
}
vfs_lock_two_folios(src_folio, dst_folio);
/*
* Now that we've locked both folios, make sure they're still
* mapped to the file data we're interested in. If not,
* someone is invalidating pages on us and we lose.
*/
if (!folio_test_uptodate(src_folio) || !folio_test_uptodate(dst_folio) ||
src_folio->mapping != src->f_mapping ||
dst_folio->mapping != dest->f_mapping) {
same = false;
goto unlock;
}
src_addr = kmap_local_folio(src_folio,
offset_in_folio(src_folio, srcoff));
dst_addr = kmap_local_folio(dst_folio,
offset_in_folio(dst_folio, dstoff));
flush_dcache_folio(src_folio);
flush_dcache_folio(dst_folio);
if (memcmp(src_addr, dst_addr, cmp_len))
same = false;
kunmap_local(dst_addr);
kunmap_local(src_addr);
unlock:
vfs_unlock_two_folios(src_folio, dst_folio);
folio_put(dst_folio);
folio_put(src_folio);
if (!same)
break;
srcoff += cmp_len;
dstoff += cmp_len;
len -= cmp_len;
}
*is_same = same;
return 0;
out_error:
return error;
}
/*
* Check that the two inodes are eligible for cloning, the ranges make
* sense, and then flush all dirty data. Caller must ensure that the
* inodes have been locked against any other modifications.
*
* If there's an error, then the usual negative error code is returned.
* Otherwise returns 0 with *len set to the request length.
*/
int
__generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t *len, unsigned int remap_flags,
const struct iomap_ops *dax_read_ops)
{
struct inode *inode_in = file_inode(file_in);
struct inode *inode_out = file_inode(file_out);
bool same_inode = (inode_in == inode_out);
int ret;
/* Don't touch certain kinds of inodes */
if (IS_IMMUTABLE(inode_out))
return -EPERM;
if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
return -ETXTBSY;
/* Don't reflink dirs, pipes, sockets... */
if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
return -EISDIR;
if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
return -EINVAL;
/* Zero length dedupe exits immediately; reflink goes to EOF. */
if (*len == 0) {
loff_t isize = i_size_read(inode_in);
if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
return 0;
if (pos_in > isize)
return -EINVAL;
*len = isize - pos_in;
if (*len == 0)
return 0;
}
/* Check that we don't violate system file offset limits. */
ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
remap_flags);
if (ret || *len == 0)
return ret;
/* Wait for the completion of any pending IOs on both files */
inode_dio_wait(inode_in);
if (!same_inode)
inode_dio_wait(inode_out);
ret = filemap_write_and_wait_range(inode_in->i_mapping,
pos_in, pos_in + *len - 1);
if (ret)
return ret;
ret = filemap_write_and_wait_range(inode_out->i_mapping,
pos_out, pos_out + *len - 1);
if (ret)
return ret;
/*
* Check that the extents are the same.
*/
if (remap_flags & REMAP_FILE_DEDUP) {
bool is_same = false;
if (!IS_DAX(inode_in))
ret = vfs_dedupe_file_range_compare(file_in, pos_in,
file_out, pos_out, *len, &is_same);
else if (dax_read_ops)
ret = dax_dedupe_file_range_compare(inode_in, pos_in,
inode_out, pos_out, *len, &is_same,
dax_read_ops);
else
return -EINVAL;
if (ret)
return ret;
if (!is_same)
return -EBADE;
}
ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
remap_flags);
if (ret || *len == 0)
return ret;
/* If can't alter the file contents, we're done. */
if (!(remap_flags & REMAP_FILE_DEDUP))
ret = file_modified(file_out);
return ret;
}
int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t *len, unsigned int remap_flags)
{
return __generic_remap_file_range_prep(file_in, pos_in, file_out,
pos_out, len, remap_flags, NULL);
}
EXPORT_SYMBOL(generic_remap_file_range_prep);
loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
struct file *file_out, loff_t pos_out,
loff_t len, unsigned int remap_flags)
{
loff_t ret;
WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP); if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
return -EXDEV;
ret = generic_file_rw_checks(file_in, file_out);
if (ret < 0)
return ret;
if (!file_in->f_op->remap_file_range)
return -EOPNOTSUPP;
ret = remap_verify_area(file_in, pos_in, len, false);
if (ret)
return ret;
ret = remap_verify_area(file_out, pos_out, len, true);
if (ret)
return ret;
file_start_write(file_out); ret = file_in->f_op->remap_file_range(file_in, pos_in,
file_out, pos_out, len, remap_flags);
file_end_write(file_out); if (ret < 0)
return ret;
fsnotify_access(file_in); fsnotify_modify(file_out);
return ret;
}
EXPORT_SYMBOL(vfs_clone_file_range);
/* Check whether we are allowed to dedupe the destination file */
static bool may_dedupe_file(struct file *file)
{
struct mnt_idmap *idmap = file_mnt_idmap(file);
struct inode *inode = file_inode(file);
if (capable(CAP_SYS_ADMIN))
return true;
if (file->f_mode & FMODE_WRITE)
return true;
if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid()))
return true;
if (!inode_permission(idmap, inode, MAY_WRITE))
return true;
return false;
}
loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
struct file *dst_file, loff_t dst_pos,
loff_t len, unsigned int remap_flags)
{
loff_t ret;
WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
REMAP_FILE_CAN_SHORTEN));
/*
* This is redundant if called from vfs_dedupe_file_range(), but other
* callers need it and it's not performance sesitive...
*/
ret = remap_verify_area(src_file, src_pos, len, false);
if (ret)
return ret;
ret = remap_verify_area(dst_file, dst_pos, len, true);
if (ret)
return ret;
/*
* This needs to be called after remap_verify_area() because of
* sb_start_write() and before may_dedupe_file() because the mount's
* MAY_WRITE need to be checked with mnt_get_write_access_file() held.
*/
ret = mnt_want_write_file(dst_file);
if (ret)
return ret;
ret = -EPERM;
if (!may_dedupe_file(dst_file))
goto out_drop_write;
ret = -EXDEV;
if (file_inode(src_file)->i_sb != file_inode(dst_file)->i_sb)
goto out_drop_write;
ret = -EISDIR;
if (S_ISDIR(file_inode(dst_file)->i_mode))
goto out_drop_write;
ret = -EINVAL;
if (!dst_file->f_op->remap_file_range)
goto out_drop_write;
if (len == 0) {
ret = 0;
goto out_drop_write;
}
ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
out_drop_write:
mnt_drop_write_file(dst_file);
return ret;
}
EXPORT_SYMBOL(vfs_dedupe_file_range_one);
int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
{
struct file_dedupe_range_info *info;
struct inode *src = file_inode(file);
u64 off;
u64 len;
int i;
int ret;
u16 count = same->dest_count;
loff_t deduped;
if (!(file->f_mode & FMODE_READ))
return -EINVAL;
if (same->reserved1 || same->reserved2)
return -EINVAL;
off = same->src_offset;
len = same->src_length;
if (S_ISDIR(src->i_mode))
return -EISDIR;
if (!S_ISREG(src->i_mode))
return -EINVAL;
if (!file->f_op->remap_file_range)
return -EOPNOTSUPP;
ret = remap_verify_area(file, off, len, false);
if (ret < 0)
return ret;
ret = 0;
if (off + len > i_size_read(src))
return -EINVAL;
/* Arbitrary 1G limit on a single dedupe request, can be raised. */
len = min_t(u64, len, 1 << 30);
/* pre-format output fields to sane values */
for (i = 0; i < count; i++) {
same->info[i].bytes_deduped = 0ULL;
same->info[i].status = FILE_DEDUPE_RANGE_SAME;
}
for (i = 0, info = same->info; i < count; i++, info++) { struct fd dst_fd = fdget(info->dest_fd);
struct file *dst_file = dst_fd.file;
if (!dst_file) {
info->status = -EBADF;
goto next_loop;
}
if (info->reserved) {
info->status = -EINVAL;
goto next_fdput;
}
deduped = vfs_dedupe_file_range_one(file, off, dst_file,
info->dest_offset, len,
REMAP_FILE_CAN_SHORTEN);
if (deduped == -EBADE)
info->status = FILE_DEDUPE_RANGE_DIFFERS; else if (deduped < 0) info->status = deduped;
else
info->bytes_deduped = len;
next_fdput:
fdput(dst_fd);
next_loop:
if (fatal_signal_pending(current))
break;
}
return ret;
}
EXPORT_SYMBOL(vfs_dedupe_file_range);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PID_H
#define _LINUX_PID_H
#include <linux/pid_types.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/refcount.h>
#include <linux/sched.h>
#include <linux/wait.h>
/*
* What is struct pid?
*
* A struct pid is the kernel's internal notion of a process identifier.
* It refers to individual tasks, process groups, and sessions. While
* there are processes attached to it the struct pid lives in a hash
* table, so it and then the processes that it refers to can be found
* quickly from the numeric pid value. The attached processes may be
* quickly accessed by following pointers from struct pid.
*
* Storing pid_t values in the kernel and referring to them later has a
* problem. The process originally with that pid may have exited and the
* pid allocator wrapped, and another process could have come along
* and been assigned that pid.
*
* Referring to user space processes by holding a reference to struct
* task_struct has a problem. When the user space process exits
* the now useless task_struct is still kept. A task_struct plus a
* stack consumes around 10K of low kernel memory. More precisely
* this is THREAD_SIZE + sizeof(struct task_struct). By comparison
* a struct pid is about 64 bytes.
*
* Holding a reference to struct pid solves both of these problems.
* It is small so holding a reference does not consume a lot of
* resources, and since a new struct pid is allocated when the numeric pid
* value is reused (when pids wrap around) we don't mistakenly refer to new
* processes.
*/
/*
* struct upid is used to get the id of the struct pid, as it is
* seen in particular namespace. Later the struct pid is found with
* find_pid_ns() using the int nr and struct pid_namespace *ns.
*/
#define RESERVED_PIDS 300
struct upid {
int nr;
struct pid_namespace *ns;
};
struct pid
{
refcount_t count;
unsigned int level;
spinlock_t lock;
struct dentry *stashed;
u64 ino;
/* lists of tasks that use this pid */
struct hlist_head tasks[PIDTYPE_MAX];
struct hlist_head inodes;
/* wait queue for pidfd notifications */
wait_queue_head_t wait_pidfd;
struct rcu_head rcu;
struct upid numbers[];
};
extern struct pid init_struct_pid;
struct file;
struct pid *pidfd_pid(const struct file *file);
struct pid *pidfd_get_pid(unsigned int fd, unsigned int *flags);
struct task_struct *pidfd_get_task(int pidfd, unsigned int *flags);
int pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret);
void do_notify_pidfd(struct task_struct *task);
static inline struct pid *get_pid(struct pid *pid)
{
if (pid)
refcount_inc(&pid->count);
return pid;
}
extern void put_pid(struct pid *pid);
extern struct task_struct *pid_task(struct pid *pid, enum pid_type);
static inline bool pid_has_task(struct pid *pid, enum pid_type type)
{
return !hlist_empty(&pid->tasks[type]);
}
extern struct task_struct *get_pid_task(struct pid *pid, enum pid_type);
extern struct pid *get_task_pid(struct task_struct *task, enum pid_type type);
/*
* these helpers must be called with the tasklist_lock write-held.
*/
extern void attach_pid(struct task_struct *task, enum pid_type);
extern void detach_pid(struct task_struct *task, enum pid_type);
extern void change_pid(struct task_struct *task, enum pid_type,
struct pid *pid);
extern void exchange_tids(struct task_struct *task, struct task_struct *old);
extern void transfer_pid(struct task_struct *old, struct task_struct *new,
enum pid_type);
extern int pid_max;
extern int pid_max_min, pid_max_max;
/*
* look up a PID in the hash table. Must be called with the tasklist_lock
* or rcu_read_lock() held.
*
* find_pid_ns() finds the pid in the namespace specified
* find_vpid() finds the pid by its virtual id, i.e. in the current namespace
*
* see also find_task_by_vpid() set in include/linux/sched.h
*/
extern struct pid *find_pid_ns(int nr, struct pid_namespace *ns);
extern struct pid *find_vpid(int nr);
/*
* Lookup a PID in the hash table, and return with it's count elevated.
*/
extern struct pid *find_get_pid(int nr);
extern struct pid *find_ge_pid(int nr, struct pid_namespace *);
extern struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
size_t set_tid_size);
extern void free_pid(struct pid *pid);
extern void disable_pid_allocation(struct pid_namespace *ns);
/*
* ns_of_pid() returns the pid namespace in which the specified pid was
* allocated.
*
* NOTE:
* ns_of_pid() is expected to be called for a process (task) that has
* an attached 'struct pid' (see attach_pid(), detach_pid()) i.e @pid
* is expected to be non-NULL. If @pid is NULL, caller should handle
* the resulting NULL pid-ns.
*/
static inline struct pid_namespace *ns_of_pid(struct pid *pid)
{
struct pid_namespace *ns = NULL;
if (pid)
ns = pid->numbers[pid->level].ns;
return ns;
}
/*
* is_child_reaper returns true if the pid is the init process
* of the current namespace. As this one could be checked before
* pid_ns->child_reaper is assigned in copy_process, we check
* with the pid number.
*/
static inline bool is_child_reaper(struct pid *pid)
{
return pid->numbers[pid->level].nr == 1;
}
/*
* the helpers to get the pid's id seen from different namespaces
*
* pid_nr() : global id, i.e. the id seen from the init namespace;
* pid_vnr() : virtual id, i.e. the id seen from the pid namespace of
* current.
* pid_nr_ns() : id seen from the ns specified.
*
* see also task_xid_nr() etc in include/linux/sched.h
*/
static inline pid_t pid_nr(struct pid *pid)
{
pid_t nr = 0;
if (pid)
nr = pid->numbers[0].nr;
return nr;
}
pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns);
pid_t pid_vnr(struct pid *pid);
#define do_each_pid_task(pid, type, task) \
do { \
if ((pid) != NULL) \
hlist_for_each_entry_rcu((task), \
&(pid)->tasks[type], pid_links[type]) {
/*
* Both old and new leaders may be attached to
* the same pid in the middle of de_thread().
*/
#define while_each_pid_task(pid, type, task) \
if (type == PIDTYPE_PID) \
break; \
} \
} while (0)
#define do_each_pid_thread(pid, type, task) \
do_each_pid_task(pid, type, task) { \
struct task_struct *tg___ = task; \
for_each_thread(tg___, task) {
#define while_each_pid_thread(pid, type, task) \
} \
task = tg___; \
} while_each_pid_task(pid, type, task)
static inline struct pid *task_pid(struct task_struct *task)
{
return task->thread_pid;
}
/*
* the helpers to get the task's different pids as they are seen
* from various namespaces
*
* task_xid_nr() : global id, i.e. the id seen from the init namespace;
* task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of
* current.
* task_xid_nr_ns() : id seen from the ns specified;
*
* see also pid_nr() etc in include/linux/pid.h
*/
pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
static inline pid_t task_pid_nr(struct task_struct *tsk)
{
return tsk->pid;
}
static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
}
static inline pid_t task_pid_vnr(struct task_struct *tsk)
{
return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
}
static inline pid_t task_tgid_nr(struct task_struct *tsk)
{
return tsk->tgid;
}
/**
* pid_alive - check that a task structure is not stale
* @p: Task structure to be checked.
*
* Test if a process is not yet dead (at most zombie state)
* If pid_alive fails, then pointers within the task structure
* can be stale and must not be dereferenced.
*
* Return: 1 if the process is alive. 0 otherwise.
*/
static inline int pid_alive(const struct task_struct *p)
{
return p->thread_pid != NULL;
}
static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
}
static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
{
return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
}
static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
}
static inline pid_t task_session_vnr(struct task_struct *tsk)
{
return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
}
static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
{
return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
}
static inline pid_t task_tgid_vnr(struct task_struct *tsk)
{
return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
}
static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
{
pid_t pid = 0;
rcu_read_lock();
if (pid_alive(tsk))
pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
rcu_read_unlock();
return pid;
}
static inline pid_t task_ppid_nr(const struct task_struct *tsk)
{
return task_ppid_nr_ns(tsk, &init_pid_ns);
}
/* Obsolete, do not use: */
static inline pid_t task_pgrp_nr(struct task_struct *tsk)
{
return task_pgrp_nr_ns(tsk, &init_pid_ns);
}
/**
* is_global_init - check if a task structure is init. Since init
* is free to have sub-threads we need to check tgid.
* @tsk: Task structure to be checked.
*
* Check if a task structure is the first user space task the kernel created.
*
* Return: 1 if the task structure is init. 0 otherwise.
*/
static inline int is_global_init(struct task_struct *tsk)
{
return task_tgid_nr(tsk) == 1;
}
#endif /* _LINUX_PID_H */
// SPDX-License-Identifier: GPL-2.0
/*
* platform.c - platform 'pseudo' bus for legacy devices
*
* Copyright (c) 2002-3 Patrick Mochel
* Copyright (c) 2002-3 Open Source Development Labs
*
* Please see Documentation/driver-api/driver-model/platform.rst for more
* information.
*/
#include <linux/string.h>
#include <linux/platform_device.h>
#include <linux/of_device.h>
#include <linux/of_irq.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/ioport.h>
#include <linux/dma-mapping.h>
#include <linux/memblock.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/pm_runtime.h>
#include <linux/pm_domain.h>
#include <linux/idr.h>
#include <linux/acpi.h>
#include <linux/clk/clk-conf.h>
#include <linux/limits.h>
#include <linux/property.h>
#include <linux/kmemleak.h>
#include <linux/types.h>
#include <linux/iommu.h>
#include <linux/dma-map-ops.h>
#include "base.h"
#include "power/power.h"
/* For automatically allocated device IDs */
static DEFINE_IDA(platform_devid_ida);
struct device platform_bus = {
.init_name = "platform",
};
EXPORT_SYMBOL_GPL(platform_bus);
/**
* platform_get_resource - get a resource for a device
* @dev: platform device
* @type: resource type
* @num: resource index
*
* Return: a pointer to the resource or NULL on failure.
*/
struct resource *platform_get_resource(struct platform_device *dev,
unsigned int type, unsigned int num)
{
u32 i;
for (i = 0; i < dev->num_resources; i++) {
struct resource *r = &dev->resource[i];
if (type == resource_type(r) && num-- == 0)
return r;
}
return NULL;
}
EXPORT_SYMBOL_GPL(platform_get_resource);
struct resource *platform_get_mem_or_io(struct platform_device *dev,
unsigned int num)
{
u32 i;
for (i = 0; i < dev->num_resources; i++) {
struct resource *r = &dev->resource[i];
if ((resource_type(r) & (IORESOURCE_MEM|IORESOURCE_IO)) && num-- == 0)
return r;
}
return NULL;
}
EXPORT_SYMBOL_GPL(platform_get_mem_or_io);
#ifdef CONFIG_HAS_IOMEM
/**
* devm_platform_get_and_ioremap_resource - call devm_ioremap_resource() for a
* platform device and get resource
*
* @pdev: platform device to use both for memory resource lookup as well as
* resource management
* @index: resource index
* @res: optional output parameter to store a pointer to the obtained resource.
*
* Return: a pointer to the remapped memory or an ERR_PTR() encoded error code
* on failure.
*/
void __iomem *
devm_platform_get_and_ioremap_resource(struct platform_device *pdev,
unsigned int index, struct resource **res)
{
struct resource *r;
r = platform_get_resource(pdev, IORESOURCE_MEM, index);
if (res)
*res = r;
return devm_ioremap_resource(&pdev->dev, r);
}
EXPORT_SYMBOL_GPL(devm_platform_get_and_ioremap_resource);
/**
* devm_platform_ioremap_resource - call devm_ioremap_resource() for a platform
* device
*
* @pdev: platform device to use both for memory resource lookup as well as
* resource management
* @index: resource index
*
* Return: a pointer to the remapped memory or an ERR_PTR() encoded error code
* on failure.
*/
void __iomem *devm_platform_ioremap_resource(struct platform_device *pdev,
unsigned int index)
{
return devm_platform_get_and_ioremap_resource(pdev, index, NULL);
}
EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource);
/**
* devm_platform_ioremap_resource_byname - call devm_ioremap_resource for
* a platform device, retrieve the
* resource by name
*
* @pdev: platform device to use both for memory resource lookup as well as
* resource management
* @name: name of the resource
*
* Return: a pointer to the remapped memory or an ERR_PTR() encoded error code
* on failure.
*/
void __iomem *
devm_platform_ioremap_resource_byname(struct platform_device *pdev,
const char *name)
{
struct resource *res;
res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
return devm_ioremap_resource(&pdev->dev, res);
}
EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource_byname);
#endif /* CONFIG_HAS_IOMEM */
/**
* platform_get_irq_optional - get an optional IRQ for a device
* @dev: platform device
* @num: IRQ number index
*
* Gets an IRQ for a platform device. Device drivers should check the return
* value for errors so as to not pass a negative integer value to the
* request_irq() APIs. This is the same as platform_get_irq(), except that it
* does not print an error message if an IRQ can not be obtained.
*
* For example::
*
* int irq = platform_get_irq_optional(pdev, 0);
* if (irq < 0)
* return irq;
*
* Return: non-zero IRQ number on success, negative error number on failure.
*/
int platform_get_irq_optional(struct platform_device *dev, unsigned int num)
{
int ret;
#ifdef CONFIG_SPARC
/* sparc does not have irqs represented as IORESOURCE_IRQ resources */
if (!dev || num >= dev->archdata.num_irqs)
goto out_not_found;
ret = dev->archdata.irqs[num];
goto out;
#else
struct fwnode_handle *fwnode = dev_fwnode(&dev->dev);
struct resource *r;
if (is_of_node(fwnode)) {
ret = of_irq_get(to_of_node(fwnode), num);
if (ret > 0 || ret == -EPROBE_DEFER)
goto out;
}
r = platform_get_resource(dev, IORESOURCE_IRQ, num);
if (is_acpi_device_node(fwnode)) {
if (r && r->flags & IORESOURCE_DISABLED) {
ret = acpi_irq_get(ACPI_HANDLE_FWNODE(fwnode), num, r);
if (ret)
goto out;
}
}
/*
* The resources may pass trigger flags to the irqs that need
* to be set up. It so happens that the trigger flags for
* IORESOURCE_BITS correspond 1-to-1 to the IRQF_TRIGGER*
* settings.
*/
if (r && r->flags & IORESOURCE_BITS) {
struct irq_data *irqd;
irqd = irq_get_irq_data(r->start);
if (!irqd)
goto out_not_found;
irqd_set_trigger_type(irqd, r->flags & IORESOURCE_BITS);
}
if (r) {
ret = r->start;
goto out;
}
/*
* For the index 0 interrupt, allow falling back to GpioInt
* resources. While a device could have both Interrupt and GpioInt
* resources, making this fallback ambiguous, in many common cases
* the device will only expose one IRQ, and this fallback
* allows a common code path across either kind of resource.
*/
if (num == 0 && is_acpi_device_node(fwnode)) {
ret = acpi_dev_gpio_irq_get(to_acpi_device_node(fwnode), num);
/* Our callers expect -ENXIO for missing IRQs. */
if (ret >= 0 || ret == -EPROBE_DEFER)
goto out;
}
#endif
out_not_found:
ret = -ENXIO;
out:
if (WARN(!ret, "0 is an invalid IRQ number\n"))
return -EINVAL;
return ret;
}
EXPORT_SYMBOL_GPL(platform_get_irq_optional);
/**
* platform_get_irq - get an IRQ for a device
* @dev: platform device
* @num: IRQ number index
*
* Gets an IRQ for a platform device and prints an error message if finding the
* IRQ fails. Device drivers should check the return value for errors so as to
* not pass a negative integer value to the request_irq() APIs.
*
* For example::
*
* int irq = platform_get_irq(pdev, 0);
* if (irq < 0)
* return irq;
*
* Return: non-zero IRQ number on success, negative error number on failure.
*/
int platform_get_irq(struct platform_device *dev, unsigned int num)
{
int ret;
ret = platform_get_irq_optional(dev, num);
if (ret < 0)
return dev_err_probe(&dev->dev, ret,
"IRQ index %u not found\n", num);
return ret;
}
EXPORT_SYMBOL_GPL(platform_get_irq);
/**
* platform_irq_count - Count the number of IRQs a platform device uses
* @dev: platform device
*
* Return: Number of IRQs a platform device uses or EPROBE_DEFER
*/
int platform_irq_count(struct platform_device *dev)
{
int ret, nr = 0;
while ((ret = platform_get_irq_optional(dev, nr)) >= 0)
nr++;
if (ret == -EPROBE_DEFER)
return ret;
return nr;
}
EXPORT_SYMBOL_GPL(platform_irq_count);
struct irq_affinity_devres {
unsigned int count;
unsigned int irq[] __counted_by(count);
};
static void platform_disable_acpi_irq(struct platform_device *pdev, int index)
{
struct resource *r;
r = platform_get_resource(pdev, IORESOURCE_IRQ, index);
if (r)
irqresource_disabled(r, 0);
}
static void devm_platform_get_irqs_affinity_release(struct device *dev,
void *res)
{
struct irq_affinity_devres *ptr = res;
int i;
for (i = 0; i < ptr->count; i++) {
irq_dispose_mapping(ptr->irq[i]);
if (is_acpi_device_node(dev_fwnode(dev)))
platform_disable_acpi_irq(to_platform_device(dev), i);
}
}
/**
* devm_platform_get_irqs_affinity - devm method to get a set of IRQs for a
* device using an interrupt affinity descriptor
* @dev: platform device pointer
* @affd: affinity descriptor
* @minvec: minimum count of interrupt vectors
* @maxvec: maximum count of interrupt vectors
* @irqs: pointer holder for IRQ numbers
*
* Gets a set of IRQs for a platform device, and updates IRQ afffinty according
* to the passed affinity descriptor
*
* Return: Number of vectors on success, negative error number on failure.
*/
int devm_platform_get_irqs_affinity(struct platform_device *dev,
struct irq_affinity *affd,
unsigned int minvec,
unsigned int maxvec,
int **irqs)
{
struct irq_affinity_devres *ptr;
struct irq_affinity_desc *desc;
size_t size;
int i, ret, nvec;
if (!affd)
return -EPERM;
if (maxvec < minvec)
return -ERANGE;
nvec = platform_irq_count(dev);
if (nvec < 0)
return nvec;
if (nvec < minvec)
return -ENOSPC;
nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
if (nvec < minvec)
return -ENOSPC;
if (nvec > maxvec)
nvec = maxvec;
size = sizeof(*ptr) + sizeof(unsigned int) * nvec;
ptr = devres_alloc(devm_platform_get_irqs_affinity_release, size,
GFP_KERNEL);
if (!ptr)
return -ENOMEM;
ptr->count = nvec;
for (i = 0; i < nvec; i++) {
int irq = platform_get_irq(dev, i);
if (irq < 0) {
ret = irq;
goto err_free_devres;
}
ptr->irq[i] = irq;
}
desc = irq_create_affinity_masks(nvec, affd);
if (!desc) {
ret = -ENOMEM;
goto err_free_devres;
}
for (i = 0; i < nvec; i++) {
ret = irq_update_affinity_desc(ptr->irq[i], &desc[i]);
if (ret) {
dev_err(&dev->dev, "failed to update irq%d affinity descriptor (%d)\n",
ptr->irq[i], ret);
goto err_free_desc;
}
}
devres_add(&dev->dev, ptr);
kfree(desc);
*irqs = ptr->irq;
return nvec;
err_free_desc:
kfree(desc);
err_free_devres:
devres_free(ptr);
return ret;
}
EXPORT_SYMBOL_GPL(devm_platform_get_irqs_affinity);
/**
* platform_get_resource_byname - get a resource for a device by name
* @dev: platform device
* @type: resource type
* @name: resource name
*/
struct resource *platform_get_resource_byname(struct platform_device *dev,
unsigned int type,
const char *name)
{
u32 i;
for (i = 0; i < dev->num_resources; i++) {
struct resource *r = &dev->resource[i];
if (unlikely(!r->name))
continue;
if (type == resource_type(r) && !strcmp(r->name, name))
return r;
}
return NULL;
}
EXPORT_SYMBOL_GPL(platform_get_resource_byname);
static int __platform_get_irq_byname(struct platform_device *dev,
const char *name)
{
struct resource *r;
int ret;
ret = fwnode_irq_get_byname(dev_fwnode(&dev->dev), name);
if (ret > 0 || ret == -EPROBE_DEFER)
return ret;
r = platform_get_resource_byname(dev, IORESOURCE_IRQ, name);
if (r) {
if (WARN(!r->start, "0 is an invalid IRQ number\n"))
return -EINVAL;
return r->start;
}
return -ENXIO;
}
/**
* platform_get_irq_byname - get an IRQ for a device by name
* @dev: platform device
* @name: IRQ name
*
* Get an IRQ like platform_get_irq(), but then by name rather then by index.
*
* Return: non-zero IRQ number on success, negative error number on failure.
*/
int platform_get_irq_byname(struct platform_device *dev, const char *name)
{
int ret;
ret = __platform_get_irq_byname(dev, name);
if (ret < 0)
return dev_err_probe(&dev->dev, ret, "IRQ %s not found\n",
name);
return ret;
}
EXPORT_SYMBOL_GPL(platform_get_irq_byname);
/**
* platform_get_irq_byname_optional - get an optional IRQ for a device by name
* @dev: platform device
* @name: IRQ name
*
* Get an optional IRQ by name like platform_get_irq_byname(). Except that it
* does not print an error message if an IRQ can not be obtained.
*
* Return: non-zero IRQ number on success, negative error number on failure.
*/
int platform_get_irq_byname_optional(struct platform_device *dev,
const char *name)
{
return __platform_get_irq_byname(dev, name);
}
EXPORT_SYMBOL_GPL(platform_get_irq_byname_optional);
/**
* platform_add_devices - add a numbers of platform devices
* @devs: array of platform devices to add
* @num: number of platform devices in array
*
* Return: 0 on success, negative error number on failure.
*/
int platform_add_devices(struct platform_device **devs, int num)
{
int i, ret = 0;
for (i = 0; i < num; i++) {
ret = platform_device_register(devs[i]);
if (ret) {
while (--i >= 0)
platform_device_unregister(devs[i]);
break;
}
}
return ret;
}
EXPORT_SYMBOL_GPL(platform_add_devices);
struct platform_object {
struct platform_device pdev;
char name[];
};
/*
* Set up default DMA mask for platform devices if the they weren't
* previously set by the architecture / DT.
*/
static void setup_pdev_dma_masks(struct platform_device *pdev)
{
pdev->dev.dma_parms = &pdev->dma_parms;
if (!pdev->dev.coherent_dma_mask)
pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
if (!pdev->dev.dma_mask) {
pdev->platform_dma_mask = DMA_BIT_MASK(32);
pdev->dev.dma_mask = &pdev->platform_dma_mask;
}
};
/**
* platform_device_put - destroy a platform device
* @pdev: platform device to free
*
* Free all memory associated with a platform device. This function must
* _only_ be externally called in error cases. All other usage is a bug.
*/
void platform_device_put(struct platform_device *pdev)
{
if (!IS_ERR_OR_NULL(pdev))
put_device(&pdev->dev);
}
EXPORT_SYMBOL_GPL(platform_device_put);
static void platform_device_release(struct device *dev)
{
struct platform_object *pa = container_of(dev, struct platform_object,
pdev.dev);
of_node_put(pa->pdev.dev.of_node);
kfree(pa->pdev.dev.platform_data);
kfree(pa->pdev.mfd_cell);
kfree(pa->pdev.resource);
kfree(pa->pdev.driver_override);
kfree(pa);
}
/**
* platform_device_alloc - create a platform device
* @name: base name of the device we're adding
* @id: instance id
*
* Create a platform device object which can have other objects attached
* to it, and which will have attached objects freed when it is released.
*/
struct platform_device *platform_device_alloc(const char *name, int id)
{
struct platform_object *pa;
pa = kzalloc(sizeof(*pa) + strlen(name) + 1, GFP_KERNEL);
if (pa) {
strcpy(pa->name, name);
pa->pdev.name = pa->name;
pa->pdev.id = id;
device_initialize(&pa->pdev.dev);
pa->pdev.dev.release = platform_device_release;
setup_pdev_dma_masks(&pa->pdev);
}
return pa ? &pa->pdev : NULL;
}
EXPORT_SYMBOL_GPL(platform_device_alloc);
/**
* platform_device_add_resources - add resources to a platform device
* @pdev: platform device allocated by platform_device_alloc to add resources to
* @res: set of resources that needs to be allocated for the device
* @num: number of resources
*
* Add a copy of the resources to the platform device. The memory
* associated with the resources will be freed when the platform device is
* released.
*/
int platform_device_add_resources(struct platform_device *pdev,
const struct resource *res, unsigned int num)
{
struct resource *r = NULL;
if (res) {
r = kmemdup(res, sizeof(struct resource) * num, GFP_KERNEL);
if (!r)
return -ENOMEM;
}
kfree(pdev->resource);
pdev->resource = r;
pdev->num_resources = num;
return 0;
}
EXPORT_SYMBOL_GPL(platform_device_add_resources);
/**
* platform_device_add_data - add platform-specific data to a platform device
* @pdev: platform device allocated by platform_device_alloc to add resources to
* @data: platform specific data for this platform device
* @size: size of platform specific data
*
* Add a copy of platform specific data to the platform device's
* platform_data pointer. The memory associated with the platform data
* will be freed when the platform device is released.
*/
int platform_device_add_data(struct platform_device *pdev, const void *data,
size_t size)
{
void *d = NULL;
if (data) {
d = kmemdup(data, size, GFP_KERNEL);
if (!d)
return -ENOMEM;
}
kfree(pdev->dev.platform_data);
pdev->dev.platform_data = d;
return 0;
}
EXPORT_SYMBOL_GPL(platform_device_add_data);
/**
* platform_device_add - add a platform device to device hierarchy
* @pdev: platform device we're adding
*
* This is part 2 of platform_device_register(), though may be called
* separately _iff_ pdev was allocated by platform_device_alloc().
*/
int platform_device_add(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
u32 i;
int ret;
if (!dev->parent)
dev->parent = &platform_bus;
dev->bus = &platform_bus_type;
switch (pdev->id) {
default:
dev_set_name(dev, "%s.%d", pdev->name, pdev->id);
break;
case PLATFORM_DEVID_NONE:
dev_set_name(dev, "%s", pdev->name);
break;
case PLATFORM_DEVID_AUTO:
/*
* Automatically allocated device ID. We mark it as such so
* that we remember it must be freed, and we append a suffix
* to avoid namespace collision with explicit IDs.
*/
ret = ida_alloc(&platform_devid_ida, GFP_KERNEL);
if (ret < 0)
return ret;
pdev->id = ret;
pdev->id_auto = true;
dev_set_name(dev, "%s.%d.auto", pdev->name, pdev->id);
break;
}
for (i = 0; i < pdev->num_resources; i++) {
struct resource *p, *r = &pdev->resource[i];
if (r->name == NULL)
r->name = dev_name(dev);
p = r->parent;
if (!p) {
if (resource_type(r) == IORESOURCE_MEM)
p = &iomem_resource;
else if (resource_type(r) == IORESOURCE_IO)
p = &ioport_resource;
}
if (p) {
ret = insert_resource(p, r);
if (ret) {
dev_err(dev, "failed to claim resource %d: %pR\n", i, r);
goto failed;
}
}
}
pr_debug("Registering platform device '%s'. Parent at %s\n", dev_name(dev),
dev_name(dev->parent));
ret = device_add(dev);
if (ret)
goto failed;
return 0;
failed:
if (pdev->id_auto) {
ida_free(&platform_devid_ida, pdev->id);
pdev->id = PLATFORM_DEVID_AUTO;
}
while (i--) {
struct resource *r = &pdev->resource[i];
if (r->parent)
release_resource(r);
}
return ret;
}
EXPORT_SYMBOL_GPL(platform_device_add);
/**
* platform_device_del - remove a platform-level device
* @pdev: platform device we're removing
*
* Note that this function will also release all memory- and port-based
* resources owned by the device (@dev->resource). This function must
* _only_ be externally called in error cases. All other usage is a bug.
*/
void platform_device_del(struct platform_device *pdev)
{
u32 i;
if (!IS_ERR_OR_NULL(pdev)) { device_del(&pdev->dev);
if (pdev->id_auto) {
ida_free(&platform_devid_ida, pdev->id);
pdev->id = PLATFORM_DEVID_AUTO;
}
for (i = 0; i < pdev->num_resources; i++) { struct resource *r = &pdev->resource[i];
if (r->parent)
release_resource(r);
}
}
}
EXPORT_SYMBOL_GPL(platform_device_del);
/**
* platform_device_register - add a platform-level device
* @pdev: platform device we're adding
*
* NOTE: _Never_ directly free @pdev after calling this function, even if it
* returned an error! Always use platform_device_put() to give up the
* reference initialised in this function instead.
*/
int platform_device_register(struct platform_device *pdev)
{
device_initialize(&pdev->dev);
setup_pdev_dma_masks(pdev);
return platform_device_add(pdev);
}
EXPORT_SYMBOL_GPL(platform_device_register);
/**
* platform_device_unregister - unregister a platform-level device
* @pdev: platform device we're unregistering
*
* Unregistration is done in 2 steps. First we release all resources
* and remove it from the subsystem, then we drop reference count by
* calling platform_device_put().
*/
void platform_device_unregister(struct platform_device *pdev)
{
platform_device_del(pdev);
platform_device_put(pdev);
}
EXPORT_SYMBOL_GPL(platform_device_unregister);
/**
* platform_device_register_full - add a platform-level device with
* resources and platform-specific data
*
* @pdevinfo: data used to create device
*
* Returns &struct platform_device pointer on success, or ERR_PTR() on error.
*/
struct platform_device *platform_device_register_full(
const struct platform_device_info *pdevinfo)
{
int ret;
struct platform_device *pdev;
pdev = platform_device_alloc(pdevinfo->name, pdevinfo->id);
if (!pdev)
return ERR_PTR(-ENOMEM);
pdev->dev.parent = pdevinfo->parent;
pdev->dev.fwnode = pdevinfo->fwnode;
pdev->dev.of_node = of_node_get(to_of_node(pdev->dev.fwnode));
pdev->dev.of_node_reused = pdevinfo->of_node_reused;
if (pdevinfo->dma_mask) {
pdev->platform_dma_mask = pdevinfo->dma_mask;
pdev->dev.dma_mask = &pdev->platform_dma_mask;
pdev->dev.coherent_dma_mask = pdevinfo->dma_mask;
}
ret = platform_device_add_resources(pdev,
pdevinfo->res, pdevinfo->num_res);
if (ret)
goto err;
ret = platform_device_add_data(pdev,
pdevinfo->data, pdevinfo->size_data);
if (ret)
goto err;
if (pdevinfo->properties) {
ret = device_create_managed_software_node(&pdev->dev,
pdevinfo->properties, NULL);
if (ret)
goto err;
}
ret = platform_device_add(pdev);
if (ret) {
err:
ACPI_COMPANION_SET(&pdev->dev, NULL);
platform_device_put(pdev);
return ERR_PTR(ret);
}
return pdev;
}
EXPORT_SYMBOL_GPL(platform_device_register_full);
/**
* __platform_driver_register - register a driver for platform-level devices
* @drv: platform driver structure
* @owner: owning module/driver
*/
int __platform_driver_register(struct platform_driver *drv,
struct module *owner)
{
drv->driver.owner = owner;
drv->driver.bus = &platform_bus_type;
return driver_register(&drv->driver);
}
EXPORT_SYMBOL_GPL(__platform_driver_register);
/**
* platform_driver_unregister - unregister a driver for platform-level devices
* @drv: platform driver structure
*/
void platform_driver_unregister(struct platform_driver *drv)
{
driver_unregister(&drv->driver);
}
EXPORT_SYMBOL_GPL(platform_driver_unregister);
static int platform_probe_fail(struct platform_device *pdev)
{
return -ENXIO;
}
static int is_bound_to_driver(struct device *dev, void *driver)
{
if (dev->driver == driver)
return 1;
return 0;
}
/**
* __platform_driver_probe - register driver for non-hotpluggable device
* @drv: platform driver structure
* @probe: the driver probe routine, probably from an __init section
* @module: module which will be the owner of the driver
*
* Use this instead of platform_driver_register() when you know the device
* is not hotpluggable and has already been registered, and you want to
* remove its run-once probe() infrastructure from memory after the driver
* has bound to the device.
*
* One typical use for this would be with drivers for controllers integrated
* into system-on-chip processors, where the controller devices have been
* configured as part of board setup.
*
* Note that this is incompatible with deferred probing.
*
* Returns zero if the driver registered and bound to a device, else returns
* a negative error code and with the driver not registered.
*/
int __init_or_module __platform_driver_probe(struct platform_driver *drv,
int (*probe)(struct platform_device *), struct module *module)
{
int retval;
if (drv->driver.probe_type == PROBE_PREFER_ASYNCHRONOUS) {
pr_err("%s: drivers registered with %s can not be probed asynchronously\n",
drv->driver.name, __func__);
return -EINVAL;
}
/*
* We have to run our probes synchronously because we check if
* we find any devices to bind to and exit with error if there
* are any.
*/
drv->driver.probe_type = PROBE_FORCE_SYNCHRONOUS;
/*
* Prevent driver from requesting probe deferral to avoid further
* futile probe attempts.
*/
drv->prevent_deferred_probe = true;
/* make sure driver won't have bind/unbind attributes */
drv->driver.suppress_bind_attrs = true;
/* temporary section violation during probe() */
drv->probe = probe;
retval = __platform_driver_register(drv, module);
if (retval)
return retval;
/* Force all new probes of this driver to fail */
drv->probe = platform_probe_fail;
/* Walk all platform devices and see if any actually bound to this driver.
* If not, return an error as the device should have done so by now.
*/
if (!bus_for_each_dev(&platform_bus_type, NULL, &drv->driver, is_bound_to_driver)) {
retval = -ENODEV;
platform_driver_unregister(drv);
}
return retval;
}
EXPORT_SYMBOL_GPL(__platform_driver_probe);
/**
* __platform_create_bundle - register driver and create corresponding device
* @driver: platform driver structure
* @probe: the driver probe routine, probably from an __init section
* @res: set of resources that needs to be allocated for the device
* @n_res: number of resources
* @data: platform specific data for this platform device
* @size: size of platform specific data
* @module: module which will be the owner of the driver
*
* Use this in legacy-style modules that probe hardware directly and
* register a single platform device and corresponding platform driver.
*
* Returns &struct platform_device pointer on success, or ERR_PTR() on error.
*/
struct platform_device * __init_or_module __platform_create_bundle(
struct platform_driver *driver,
int (*probe)(struct platform_device *),
struct resource *res, unsigned int n_res,
const void *data, size_t size, struct module *module)
{
struct platform_device *pdev;
int error;
pdev = platform_device_alloc(driver->driver.name, -1);
if (!pdev) {
error = -ENOMEM;
goto err_out;
}
error = platform_device_add_resources(pdev, res, n_res);
if (error)
goto err_pdev_put;
error = platform_device_add_data(pdev, data, size);
if (error)
goto err_pdev_put;
error = platform_device_add(pdev);
if (error)
goto err_pdev_put;
error = __platform_driver_probe(driver, probe, module);
if (error)
goto err_pdev_del;
return pdev;
err_pdev_del:
platform_device_del(pdev);
err_pdev_put:
platform_device_put(pdev);
err_out:
return ERR_PTR(error);
}
EXPORT_SYMBOL_GPL(__platform_create_bundle);
/**
* __platform_register_drivers - register an array of platform drivers
* @drivers: an array of drivers to register
* @count: the number of drivers to register
* @owner: module owning the drivers
*
* Registers platform drivers specified by an array. On failure to register a
* driver, all previously registered drivers will be unregistered. Callers of
* this API should use platform_unregister_drivers() to unregister drivers in
* the reverse order.
*
* Returns: 0 on success or a negative error code on failure.
*/
int __platform_register_drivers(struct platform_driver * const *drivers,
unsigned int count, struct module *owner)
{
unsigned int i;
int err;
for (i = 0; i < count; i++) {
pr_debug("registering platform driver %ps\n", drivers[i]);
err = __platform_driver_register(drivers[i], owner);
if (err < 0) {
pr_err("failed to register platform driver %ps: %d\n",
drivers[i], err);
goto error;
}
}
return 0;
error:
while (i--) {
pr_debug("unregistering platform driver %ps\n", drivers[i]);
platform_driver_unregister(drivers[i]);
}
return err;
}
EXPORT_SYMBOL_GPL(__platform_register_drivers);
/**
* platform_unregister_drivers - unregister an array of platform drivers
* @drivers: an array of drivers to unregister
* @count: the number of drivers to unregister
*
* Unregisters platform drivers specified by an array. This is typically used
* to complement an earlier call to platform_register_drivers(). Drivers are
* unregistered in the reverse order in which they were registered.
*/
void platform_unregister_drivers(struct platform_driver * const *drivers,
unsigned int count)
{
while (count--) {
pr_debug("unregistering platform driver %ps\n", drivers[count]);
platform_driver_unregister(drivers[count]);
}
}
EXPORT_SYMBOL_GPL(platform_unregister_drivers);
static const struct platform_device_id *platform_match_id(
const struct platform_device_id *id,
struct platform_device *pdev)
{
while (id->name[0]) {
if (strcmp(pdev->name, id->name) == 0) {
pdev->id_entry = id;
return id;
}
id++;
}
return NULL;
}
#ifdef CONFIG_PM_SLEEP
static int platform_legacy_suspend(struct device *dev, pm_message_t mesg)
{
struct platform_driver *pdrv = to_platform_driver(dev->driver);
struct platform_device *pdev = to_platform_device(dev);
int ret = 0;
if (dev->driver && pdrv->suspend)
ret = pdrv->suspend(pdev, mesg);
return ret;
}
static int platform_legacy_resume(struct device *dev)
{
struct platform_driver *pdrv = to_platform_driver(dev->driver);
struct platform_device *pdev = to_platform_device(dev);
int ret = 0;
if (dev->driver && pdrv->resume)
ret = pdrv->resume(pdev);
return ret;
}
#endif /* CONFIG_PM_SLEEP */
#ifdef CONFIG_SUSPEND
int platform_pm_suspend(struct device *dev)
{
struct device_driver *drv = dev->driver;
int ret = 0;
if (!drv)
return 0;
if (drv->pm) {
if (drv->pm->suspend)
ret = drv->pm->suspend(dev);
} else {
ret = platform_legacy_suspend(dev, PMSG_SUSPEND);
}
return ret;
}
int platform_pm_resume(struct device *dev)
{
struct device_driver *drv = dev->driver;
int ret = 0;
if (!drv)
return 0;
if (drv->pm) {
if (drv->pm->resume)
ret = drv->pm->resume(dev);
} else {
ret = platform_legacy_resume(dev);
}
return ret;
}
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
int platform_pm_freeze(struct device *dev)
{
struct device_driver *drv = dev->driver;
int ret = 0;
if (!drv)
return 0;
if (drv->pm) {
if (drv->pm->freeze)
ret = drv->pm->freeze(dev);
} else {
ret = platform_legacy_suspend(dev, PMSG_FREEZE);
}
return ret;
}
int platform_pm_thaw(struct device *dev)
{
struct device_driver *drv = dev->driver;
int ret = 0;
if (!drv)
return 0;
if (drv->pm) {
if (drv->pm->thaw)
ret = drv->pm->thaw(dev);
} else {
ret = platform_legacy_resume(dev);
}
return ret;
}
int platform_pm_poweroff(struct device *dev)
{
struct device_driver *drv = dev->driver;
int ret = 0;
if (!drv)
return 0;
if (drv->pm) {
if (drv->pm->poweroff)
ret = drv->pm->poweroff(dev);
} else {
ret = platform_legacy_suspend(dev, PMSG_HIBERNATE);
}
return ret;
}
int platform_pm_restore(struct device *dev)
{
struct device_driver *drv = dev->driver;
int ret = 0;
if (!drv)
return 0;
if (drv->pm) {
if (drv->pm->restore)
ret = drv->pm->restore(dev);
} else {
ret = platform_legacy_resume(dev);
}
return ret;
}
#endif /* CONFIG_HIBERNATE_CALLBACKS */
/* modalias support enables more hands-off userspace setup:
* (a) environment variable lets new-style hotplug events work once system is
* fully running: "modprobe $MODALIAS"
* (b) sysfs attribute lets new-style coldplug recover from hotplug events
* mishandled before system is fully running: "modprobe $(cat modalias)"
*/
static ssize_t modalias_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct platform_device *pdev = to_platform_device(dev);
int len;
len = of_device_modalias(dev, buf, PAGE_SIZE);
if (len != -ENODEV)
return len;
len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1);
if (len != -ENODEV)
return len;
return sysfs_emit(buf, "platform:%s\n", pdev->name);
}
static DEVICE_ATTR_RO(modalias);
static ssize_t numa_node_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
return sysfs_emit(buf, "%d\n", dev_to_node(dev));
}
static DEVICE_ATTR_RO(numa_node);
static ssize_t driver_override_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct platform_device *pdev = to_platform_device(dev);
ssize_t len;
device_lock(dev);
len = sysfs_emit(buf, "%s\n", pdev->driver_override);
device_unlock(dev);
return len;
}
static ssize_t driver_override_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct platform_device *pdev = to_platform_device(dev);
int ret;
ret = driver_set_override(dev, &pdev->driver_override, buf, count);
if (ret)
return ret;
return count;
}
static DEVICE_ATTR_RW(driver_override);
static struct attribute *platform_dev_attrs[] = {
&dev_attr_modalias.attr,
&dev_attr_numa_node.attr,
&dev_attr_driver_override.attr,
NULL,
};
static umode_t platform_dev_attrs_visible(struct kobject *kobj, struct attribute *a,
int n)
{
struct device *dev = container_of(kobj, typeof(*dev), kobj);
if (a == &dev_attr_numa_node.attr &&
dev_to_node(dev) == NUMA_NO_NODE)
return 0;
return a->mode;
}
static const struct attribute_group platform_dev_group = {
.attrs = platform_dev_attrs,
.is_visible = platform_dev_attrs_visible,
};
__ATTRIBUTE_GROUPS(platform_dev);
/**
* platform_match - bind platform device to platform driver.
* @dev: device.
* @drv: driver.
*
* Platform device IDs are assumed to be encoded like this:
* "<name><instance>", where <name> is a short description of the type of
* device, like "pci" or "floppy", and <instance> is the enumerated
* instance of the device, like '0' or '42'. Driver IDs are simply
* "<name>". So, extract the <name> from the platform_device structure,
* and compare it against the name of the driver. Return whether they match
* or not.
*/
static int platform_match(struct device *dev, struct device_driver *drv)
{
struct platform_device *pdev = to_platform_device(dev);
struct platform_driver *pdrv = to_platform_driver(drv);
/* When driver_override is set, only bind to the matching driver */
if (pdev->driver_override)
return !strcmp(pdev->driver_override, drv->name);
/* Attempt an OF style match first */
if (of_driver_match_device(dev, drv))
return 1;
/* Then try ACPI style match */
if (acpi_driver_match_device(dev, drv))
return 1;
/* Then try to match against the id table */
if (pdrv->id_table)
return platform_match_id(pdrv->id_table, pdev) != NULL;
/* fall-back to driver name match */
return (strcmp(pdev->name, drv->name) == 0);
}
static int platform_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const struct platform_device *pdev = to_platform_device(dev);
int rc;
/* Some devices have extra OF data and an OF-style MODALIAS */
rc = of_device_uevent_modalias(dev, env); if (rc != -ENODEV)
return rc;
rc = acpi_device_uevent_modalias(dev, env);
if (rc != -ENODEV)
return rc;
add_uevent_var(env, "MODALIAS=%s%s", PLATFORM_MODULE_PREFIX,
pdev->name); return 0;
}
static int platform_probe(struct device *_dev)
{
struct platform_driver *drv = to_platform_driver(_dev->driver);
struct platform_device *dev = to_platform_device(_dev);
int ret;
/*
* A driver registered using platform_driver_probe() cannot be bound
* again later because the probe function usually lives in __init code
* and so is gone. For these drivers .probe is set to
* platform_probe_fail in __platform_driver_probe(). Don't even prepare
* clocks and PM domains for these to match the traditional behaviour.
*/
if (unlikely(drv->probe == platform_probe_fail))
return -ENXIO;
ret = of_clk_set_defaults(_dev->of_node, false);
if (ret < 0)
return ret;
ret = dev_pm_domain_attach(_dev, true);
if (ret)
goto out;
if (drv->probe) {
ret = drv->probe(dev);
if (ret)
dev_pm_domain_detach(_dev, true);
}
out:
if (drv->prevent_deferred_probe && ret == -EPROBE_DEFER) {
dev_warn(_dev, "probe deferral not supported\n");
ret = -ENXIO;
}
return ret;
}
static void platform_remove(struct device *_dev)
{
struct platform_driver *drv = to_platform_driver(_dev->driver);
struct platform_device *dev = to_platform_device(_dev);
if (drv->remove_new) {
drv->remove_new(dev); } else if (drv->remove) { int ret = drv->remove(dev);
if (ret)
dev_warn(_dev, "remove callback returned a non-zero value. This will be ignored.\n");
}
dev_pm_domain_detach(_dev, true);
}
static void platform_shutdown(struct device *_dev)
{
struct platform_device *dev = to_platform_device(_dev);
struct platform_driver *drv;
if (!_dev->driver)
return;
drv = to_platform_driver(_dev->driver);
if (drv->shutdown)
drv->shutdown(dev);
}
static int platform_dma_configure(struct device *dev)
{
struct platform_driver *drv = to_platform_driver(dev->driver);
struct fwnode_handle *fwnode = dev_fwnode(dev);
enum dev_dma_attr attr;
int ret = 0;
if (is_of_node(fwnode)) {
ret = of_dma_configure(dev, to_of_node(fwnode), true);
} else if (is_acpi_device_node(fwnode)) {
attr = acpi_get_dma_attr(to_acpi_device_node(fwnode));
ret = acpi_dma_configure(dev, attr);
}
if (ret || drv->driver_managed_dma)
return ret;
ret = iommu_device_use_default_domain(dev);
if (ret)
arch_teardown_dma_ops(dev);
return ret;
}
static void platform_dma_cleanup(struct device *dev)
{
struct platform_driver *drv = to_platform_driver(dev->driver);
if (!drv->driver_managed_dma)
iommu_device_unuse_default_domain(dev);
}
static const struct dev_pm_ops platform_dev_pm_ops = {
SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend, pm_generic_runtime_resume, NULL)
USE_PLATFORM_PM_SLEEP_OPS
};
struct bus_type platform_bus_type = {
.name = "platform",
.dev_groups = platform_dev_groups,
.match = platform_match,
.uevent = platform_uevent,
.probe = platform_probe,
.remove = platform_remove,
.shutdown = platform_shutdown,
.dma_configure = platform_dma_configure,
.dma_cleanup = platform_dma_cleanup,
.pm = &platform_dev_pm_ops,
};
EXPORT_SYMBOL_GPL(platform_bus_type);
static inline int __platform_match(struct device *dev, const void *drv)
{
return platform_match(dev, (struct device_driver *)drv);
}
/**
* platform_find_device_by_driver - Find a platform device with a given
* driver.
* @start: The device to start the search from.
* @drv: The device driver to look for.
*/
struct device *platform_find_device_by_driver(struct device *start,
const struct device_driver *drv)
{
return bus_find_device(&platform_bus_type, start, drv,
__platform_match);
}
EXPORT_SYMBOL_GPL(platform_find_device_by_driver);
void __weak __init early_platform_cleanup(void) { }
int __init platform_bus_init(void)
{
int error;
early_platform_cleanup();
error = device_register(&platform_bus);
if (error) {
put_device(&platform_bus);
return error;
}
error = bus_register(&platform_bus_type);
if (error)
device_unregister(&platform_bus);
return error;
}
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_MM_H
#define _LINUX_SCHED_MM_H
#include <linux/kernel.h>
#include <linux/atomic.h>
#include <linux/sched.h>
#include <linux/mm_types.h>
#include <linux/gfp.h>
#include <linux/sync_core.h>
/*
* Routines for handling mm_structs
*/
extern struct mm_struct *mm_alloc(void);
/**
* mmgrab() - Pin a &struct mm_struct.
* @mm: The &struct mm_struct to pin.
*
* Make sure that @mm will not get freed even after the owning task
* exits. This doesn't guarantee that the associated address space
* will still exist later on and mmget_not_zero() has to be used before
* accessing it.
*
* This is a preferred way to pin @mm for a longer/unbounded amount
* of time.
*
* Use mmdrop() to release the reference acquired by mmgrab().
*
* See also <Documentation/mm/active_mm.rst> for an in-depth explanation
* of &mm_struct.mm_count vs &mm_struct.mm_users.
*/
static inline void mmgrab(struct mm_struct *mm)
{
atomic_inc(&mm->mm_count);
}
static inline void smp_mb__after_mmgrab(void)
{
smp_mb__after_atomic();
}
extern void __mmdrop(struct mm_struct *mm);
static inline void mmdrop(struct mm_struct *mm)
{
/*
* The implicit full barrier implied by atomic_dec_and_test() is
* required by the membarrier system call before returning to
* user-space, after storing to rq->curr.
*/
if (unlikely(atomic_dec_and_test(&mm->mm_count)))
__mmdrop(mm);
}
#ifdef CONFIG_PREEMPT_RT
/*
* RCU callback for delayed mm drop. Not strictly RCU, but call_rcu() is
* by far the least expensive way to do that.
*/
static inline void __mmdrop_delayed(struct rcu_head *rhp)
{
struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop);
__mmdrop(mm);
}
/*
* Invoked from finish_task_switch(). Delegates the heavy lifting on RT
* kernels via RCU.
*/
static inline void mmdrop_sched(struct mm_struct *mm)
{
/* Provides a full memory barrier. See mmdrop() */
if (atomic_dec_and_test(&mm->mm_count))
call_rcu(&mm->delayed_drop, __mmdrop_delayed);
}
#else
static inline void mmdrop_sched(struct mm_struct *mm)
{
mmdrop(mm);
}
#endif
/* Helpers for lazy TLB mm refcounting */
static inline void mmgrab_lazy_tlb(struct mm_struct *mm)
{
if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT))
mmgrab(mm);
}
static inline void mmdrop_lazy_tlb(struct mm_struct *mm)
{
if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) {
mmdrop(mm);
} else {
/*
* mmdrop_lazy_tlb must provide a full memory barrier, see the
* membarrier comment finish_task_switch which relies on this.
*/
smp_mb();
}
}
static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm)
{
if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT))
mmdrop_sched(mm);
else
smp_mb(); /* see mmdrop_lazy_tlb() above */
}
/**
* mmget() - Pin the address space associated with a &struct mm_struct.
* @mm: The address space to pin.
*
* Make sure that the address space of the given &struct mm_struct doesn't
* go away. This does not protect against parts of the address space being
* modified or freed, however.
*
* Never use this function to pin this address space for an
* unbounded/indefinite amount of time.
*
* Use mmput() to release the reference acquired by mmget().
*
* See also <Documentation/mm/active_mm.rst> for an in-depth explanation
* of &mm_struct.mm_count vs &mm_struct.mm_users.
*/
static inline void mmget(struct mm_struct *mm)
{
atomic_inc(&mm->mm_users);
}
static inline bool mmget_not_zero(struct mm_struct *mm)
{
return atomic_inc_not_zero(&mm->mm_users);
}
/* mmput gets rid of the mappings and all user-space */
extern void mmput(struct mm_struct *);
#ifdef CONFIG_MMU
/* same as above but performs the slow path from the async context. Can
* be called from the atomic context as well
*/
void mmput_async(struct mm_struct *);
#endif
/* Grab a reference to a task's mm, if it is not already going away */
extern struct mm_struct *get_task_mm(struct task_struct *task);
/*
* Grab a reference to a task's mm, if it is not already going away
* and ptrace_may_access with the mode parameter passed to it
* succeeds.
*/
extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
/* Remove the current tasks stale references to the old mm_struct on exit() */
extern void exit_mm_release(struct task_struct *, struct mm_struct *);
/* Remove the current tasks stale references to the old mm_struct on exec() */
extern void exec_mm_release(struct task_struct *, struct mm_struct *);
#ifdef CONFIG_MEMCG
extern void mm_update_next_owner(struct mm_struct *mm);
#else
static inline void mm_update_next_owner(struct mm_struct *mm)
{
}
#endif /* CONFIG_MEMCG */
#ifdef CONFIG_MMU
#ifndef arch_get_mmap_end
#define arch_get_mmap_end(addr, len, flags) (TASK_SIZE)
#endif
#ifndef arch_get_mmap_base
#define arch_get_mmap_base(addr, base) (base)
#endif
extern void arch_pick_mmap_layout(struct mm_struct *mm,
struct rlimit *rlim_stack);
extern unsigned long
arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
unsigned long, unsigned long);
extern unsigned long
arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags);
unsigned long
generic_get_unmapped_area(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags);
unsigned long
generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags);
#else
static inline void arch_pick_mmap_layout(struct mm_struct *mm,
struct rlimit *rlim_stack) {}
#endif
static inline bool in_vfork(struct task_struct *tsk)
{
bool ret;
/*
* need RCU to access ->real_parent if CLONE_VM was used along with
* CLONE_PARENT.
*
* We check real_parent->mm == tsk->mm because CLONE_VFORK does not
* imply CLONE_VM
*
* CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
* ->real_parent is not necessarily the task doing vfork(), so in
* theory we can't rely on task_lock() if we want to dereference it.
*
* And in this case we can't trust the real_parent->mm == tsk->mm
* check, it can be false negative. But we do not care, if init or
* another oom-unkillable task does this it should blame itself.
*/
rcu_read_lock();
ret = tsk->vfork_done &&
rcu_dereference(tsk->real_parent)->mm == tsk->mm;
rcu_read_unlock();
return ret;
}
/*
* Applies per-task gfp context to the given allocation flags.
* PF_MEMALLOC_NOIO implies GFP_NOIO
* PF_MEMALLOC_NOFS implies GFP_NOFS
* PF_MEMALLOC_PIN implies !GFP_MOVABLE
*/
static inline gfp_t current_gfp_context(gfp_t flags)
{
unsigned int pflags = READ_ONCE(current->flags);
if (unlikely(pflags & (PF_MEMALLOC_NOIO |
PF_MEMALLOC_NOFS |
PF_MEMALLOC_NORECLAIM |
PF_MEMALLOC_NOWARN |
PF_MEMALLOC_PIN))) {
/*
* Stronger flags before weaker flags:
* NORECLAIM implies NOIO, which in turn implies NOFS
*/
if (pflags & PF_MEMALLOC_NORECLAIM)
flags &= ~__GFP_DIRECT_RECLAIM;
else if (pflags & PF_MEMALLOC_NOIO)
flags &= ~(__GFP_IO | __GFP_FS);
else if (pflags & PF_MEMALLOC_NOFS)
flags &= ~__GFP_FS;
if (pflags & PF_MEMALLOC_NOWARN)
flags |= __GFP_NOWARN;
if (pflags & PF_MEMALLOC_PIN)
flags &= ~__GFP_MOVABLE;
}
return flags;
}
#ifdef CONFIG_LOCKDEP
extern void __fs_reclaim_acquire(unsigned long ip);
extern void __fs_reclaim_release(unsigned long ip);
extern void fs_reclaim_acquire(gfp_t gfp_mask);
extern void fs_reclaim_release(gfp_t gfp_mask);
#else
static inline void __fs_reclaim_acquire(unsigned long ip) { }
static inline void __fs_reclaim_release(unsigned long ip) { }
static inline void fs_reclaim_acquire(gfp_t gfp_mask) { }
static inline void fs_reclaim_release(gfp_t gfp_mask) { }
#endif
/* Any memory-allocation retry loop should use
* memalloc_retry_wait(), and pass the flags for the most
* constrained allocation attempt that might have failed.
* This provides useful documentation of where loops are,
* and a central place to fine tune the waiting as the MM
* implementation changes.
*/
static inline void memalloc_retry_wait(gfp_t gfp_flags)
{
/* We use io_schedule_timeout because waiting for memory
* typically included waiting for dirty pages to be
* written out, which requires IO.
*/
__set_current_state(TASK_UNINTERRUPTIBLE);
gfp_flags = current_gfp_context(gfp_flags);
if (gfpflags_allow_blocking(gfp_flags) &&
!(gfp_flags & __GFP_NORETRY))
/* Probably waited already, no need for much more */
io_schedule_timeout(1);
else
/* Probably didn't wait, and has now released a lock,
* so now is a good time to wait
*/
io_schedule_timeout(HZ/50);
}
/**
* might_alloc - Mark possible allocation sites
* @gfp_mask: gfp_t flags that would be used to allocate
*
* Similar to might_sleep() and other annotations, this can be used in functions
* that might allocate, but often don't. Compiles to nothing without
* CONFIG_LOCKDEP. Includes a conditional might_sleep() if @gfp allows blocking.
*/
static inline void might_alloc(gfp_t gfp_mask)
{
fs_reclaim_acquire(gfp_mask);
fs_reclaim_release(gfp_mask);
might_sleep_if(gfpflags_allow_blocking(gfp_mask));
}
/**
* memalloc_flags_save - Add a PF_* flag to current->flags, save old value
*
* This allows PF_* flags to be conveniently added, irrespective of current
* value, and then the old version restored with memalloc_flags_restore().
*/
static inline unsigned memalloc_flags_save(unsigned flags)
{
unsigned oldflags = ~current->flags & flags;
current->flags |= flags;
return oldflags;
}
static inline void memalloc_flags_restore(unsigned flags)
{
current->flags &= ~flags;
}
/**
* memalloc_noio_save - Marks implicit GFP_NOIO allocation scope.
*
* This functions marks the beginning of the GFP_NOIO allocation scope.
* All further allocations will implicitly drop __GFP_IO flag and so
* they are safe for the IO critical section from the allocation recursion
* point of view. Use memalloc_noio_restore to end the scope with flags
* returned by this function.
*
* Context: This function is safe to be used from any context.
* Return: The saved flags to be passed to memalloc_noio_restore.
*/
static inline unsigned int memalloc_noio_save(void)
{
return memalloc_flags_save(PF_MEMALLOC_NOIO);
}
/**
* memalloc_noio_restore - Ends the implicit GFP_NOIO scope.
* @flags: Flags to restore.
*
* Ends the implicit GFP_NOIO scope started by memalloc_noio_save function.
* Always make sure that the given flags is the return value from the
* pairing memalloc_noio_save call.
*/
static inline void memalloc_noio_restore(unsigned int flags)
{
memalloc_flags_restore(flags);
}
/**
* memalloc_nofs_save - Marks implicit GFP_NOFS allocation scope.
*
* This functions marks the beginning of the GFP_NOFS allocation scope.
* All further allocations will implicitly drop __GFP_FS flag and so
* they are safe for the FS critical section from the allocation recursion
* point of view. Use memalloc_nofs_restore to end the scope with flags
* returned by this function.
*
* Context: This function is safe to be used from any context.
* Return: The saved flags to be passed to memalloc_nofs_restore.
*/
static inline unsigned int memalloc_nofs_save(void)
{
return memalloc_flags_save(PF_MEMALLOC_NOFS);
}
/**
* memalloc_nofs_restore - Ends the implicit GFP_NOFS scope.
* @flags: Flags to restore.
*
* Ends the implicit GFP_NOFS scope started by memalloc_nofs_save function.
* Always make sure that the given flags is the return value from the
* pairing memalloc_nofs_save call.
*/
static inline void memalloc_nofs_restore(unsigned int flags)
{
memalloc_flags_restore(flags);
}
/**
* memalloc_noreclaim_save - Marks implicit __GFP_MEMALLOC scope.
*
* This function marks the beginning of the __GFP_MEMALLOC allocation scope.
* All further allocations will implicitly add the __GFP_MEMALLOC flag, which
* prevents entering reclaim and allows access to all memory reserves. This
* should only be used when the caller guarantees the allocation will allow more
* memory to be freed very shortly, i.e. it needs to allocate some memory in
* the process of freeing memory, and cannot reclaim due to potential recursion.
*
* Users of this scope have to be extremely careful to not deplete the reserves
* completely and implement a throttling mechanism which controls the
* consumption of the reserve based on the amount of freed memory. Usage of a
* pre-allocated pool (e.g. mempool) should be always considered before using
* this scope.
*
* Individual allocations under the scope can opt out using __GFP_NOMEMALLOC
*
* Context: This function should not be used in an interrupt context as that one
* does not give PF_MEMALLOC access to reserves.
* See __gfp_pfmemalloc_flags().
* Return: The saved flags to be passed to memalloc_noreclaim_restore.
*/
static inline unsigned int memalloc_noreclaim_save(void)
{
return memalloc_flags_save(PF_MEMALLOC);
}
/**
* memalloc_noreclaim_restore - Ends the implicit __GFP_MEMALLOC scope.
* @flags: Flags to restore.
*
* Ends the implicit __GFP_MEMALLOC scope started by memalloc_noreclaim_save
* function. Always make sure that the given flags is the return value from the
* pairing memalloc_noreclaim_save call.
*/
static inline void memalloc_noreclaim_restore(unsigned int flags)
{
memalloc_flags_restore(flags);
}
/**
* memalloc_pin_save - Marks implicit ~__GFP_MOVABLE scope.
*
* This function marks the beginning of the ~__GFP_MOVABLE allocation scope.
* All further allocations will implicitly remove the __GFP_MOVABLE flag, which
* will constraint the allocations to zones that allow long term pinning, i.e.
* not ZONE_MOVABLE zones.
*
* Return: The saved flags to be passed to memalloc_pin_restore.
*/
static inline unsigned int memalloc_pin_save(void)
{
return memalloc_flags_save(PF_MEMALLOC_PIN);
}
/**
* memalloc_pin_restore - Ends the implicit ~__GFP_MOVABLE scope.
* @flags: Flags to restore.
*
* Ends the implicit ~__GFP_MOVABLE scope started by memalloc_pin_save function.
* Always make sure that the given flags is the return value from the pairing
* memalloc_pin_save call.
*/
static inline void memalloc_pin_restore(unsigned int flags)
{
memalloc_flags_restore(flags);
}
#ifdef CONFIG_MEMCG
DECLARE_PER_CPU(struct mem_cgroup *, int_active_memcg);
/**
* set_active_memcg - Starts the remote memcg charging scope.
* @memcg: memcg to charge.
*
* This function marks the beginning of the remote memcg charging scope. All the
* __GFP_ACCOUNT allocations till the end of the scope will be charged to the
* given memcg.
*
* Please, make sure that caller has a reference to the passed memcg structure,
* so its lifetime is guaranteed to exceed the scope between two
* set_active_memcg() calls.
*
* NOTE: This function can nest. Users must save the return value and
* reset the previous value after their own charging scope is over.
*/
static inline struct mem_cgroup *
set_active_memcg(struct mem_cgroup *memcg)
{
struct mem_cgroup *old;
if (!in_task()) {
old = this_cpu_read(int_active_memcg);
this_cpu_write(int_active_memcg, memcg);
} else {
old = current->active_memcg;
current->active_memcg = memcg;
}
return old;
}
#else
static inline struct mem_cgroup *
set_active_memcg(struct mem_cgroup *memcg)
{
return NULL;
}
#endif
#ifdef CONFIG_MEMBARRIER
enum {
MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY = (1U << 0),
MEMBARRIER_STATE_PRIVATE_EXPEDITED = (1U << 1),
MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY = (1U << 2),
MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3),
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4),
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5),
MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY = (1U << 6),
MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ = (1U << 7),
};
enum {
MEMBARRIER_FLAG_SYNC_CORE = (1U << 0),
MEMBARRIER_FLAG_RSEQ = (1U << 1),
};
#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
#include <asm/membarrier.h>
#endif
static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)
{
if (current->mm != mm)
return;
if (likely(!(atomic_read(&mm->membarrier_state) &
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE)))
return;
sync_core_before_usermode();
}
extern void membarrier_exec_mmap(struct mm_struct *mm);
extern void membarrier_update_current_mm(struct mm_struct *next_mm);
#else
#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
struct mm_struct *next,
struct task_struct *tsk)
{
}
#endif
static inline void membarrier_exec_mmap(struct mm_struct *mm)
{
}
static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)
{
}
static inline void membarrier_update_current_mm(struct mm_struct *next_mm)
{
}
#endif
#endif /* _LINUX_SCHED_MM_H */
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_address.h>
#include <linux/of_iommu.h>
#include <linux/of_reserved_mem.h>
#include <linux/dma-direct.h> /* for bus_dma_region */
#include <linux/dma-map-ops.h>
#include <linux/init.h>
#include <linux/mod_devicetable.h>
#include <linux/slab.h>
#include <linux/platform_device.h>
#include <asm/errno.h>
#include "of_private.h"
/**
* of_match_device - Tell if a struct device matches an of_device_id list
* @matches: array of of device match structures to search in
* @dev: the of device structure to match against
*
* Used by a driver to check whether an platform_device present in the
* system is in its list of supported devices.
*/
const struct of_device_id *of_match_device(const struct of_device_id *matches,
const struct device *dev)
{
if (!matches || !dev->of_node || dev->of_node_reused)
return NULL;
return of_match_node(matches, dev->of_node);
}
EXPORT_SYMBOL(of_match_device);
static void
of_dma_set_restricted_buffer(struct device *dev, struct device_node *np)
{
struct device_node *node, *of_node = dev->of_node;
int count, i;
if (!IS_ENABLED(CONFIG_DMA_RESTRICTED_POOL))
return;
count = of_property_count_elems_of_size(of_node, "memory-region",
sizeof(u32));
/*
* If dev->of_node doesn't exist or doesn't contain memory-region, try
* the OF node having DMA configuration.
*/
if (count <= 0) {
of_node = np;
count = of_property_count_elems_of_size(
of_node, "memory-region", sizeof(u32));
}
for (i = 0; i < count; i++) {
node = of_parse_phandle(of_node, "memory-region", i);
/*
* There might be multiple memory regions, but only one
* restricted-dma-pool region is allowed.
*/
if (of_device_is_compatible(node, "restricted-dma-pool") &&
of_device_is_available(node)) {
of_node_put(node);
break;
}
of_node_put(node);
}
/*
* Attempt to initialize a restricted-dma-pool region if one was found.
* Note that count can hold a negative error code.
*/
if (i < count && of_reserved_mem_device_init_by_idx(dev, of_node, i))
dev_warn(dev, "failed to initialise \"restricted-dma-pool\" memory node\n");
}
/**
* of_dma_configure_id - Setup DMA configuration
* @dev: Device to apply DMA configuration
* @np: Pointer to OF node having DMA configuration
* @force_dma: Whether device is to be set up by of_dma_configure() even if
* DMA capability is not explicitly described by firmware.
* @id: Optional const pointer value input id
*
* Try to get devices's DMA configuration from DT and update it
* accordingly.
*
* If platform code needs to use its own special DMA configuration, it
* can use a platform bus notifier and handle BUS_NOTIFY_ADD_DEVICE events
* to fix up DMA configuration.
*/
int of_dma_configure_id(struct device *dev, struct device_node *np,
bool force_dma, const u32 *id)
{
const struct bus_dma_region *map = NULL;
struct device_node *bus_np;
u64 dma_start = 0;
u64 mask, end, size = 0;
bool coherent;
int iommu_ret;
int ret;
if (np == dev->of_node)
bus_np = __of_get_dma_parent(np);
else
bus_np = of_node_get(np);
ret = of_dma_get_range(bus_np, &map);
of_node_put(bus_np);
if (ret < 0) {
/*
* For legacy reasons, we have to assume some devices need
* DMA configuration regardless of whether "dma-ranges" is
* correctly specified or not.
*/
if (!force_dma)
return ret == -ENODEV ? 0 : ret;
} else {
const struct bus_dma_region *r = map;
u64 dma_end = 0;
/* Determine the overall bounds of all DMA regions */
for (dma_start = ~0; r->size; r++) {
/* Take lower and upper limits */
if (r->dma_start < dma_start)
dma_start = r->dma_start;
if (r->dma_start + r->size > dma_end)
dma_end = r->dma_start + r->size;
}
size = dma_end - dma_start;
/*
* Add a work around to treat the size as mask + 1 in case
* it is defined in DT as a mask.
*/
if (size & 1) {
dev_warn(dev, "Invalid size 0x%llx for dma-range(s)\n",
size);
size = size + 1;
}
if (!size) {
dev_err(dev, "Adjusted size 0x%llx invalid\n", size);
kfree(map);
return -EINVAL;
}
}
/*
* If @dev is expected to be DMA-capable then the bus code that created
* it should have initialised its dma_mask pointer by this point. For
* now, we'll continue the legacy behaviour of coercing it to the
* coherent mask if not, but we'll no longer do so quietly.
*/
if (!dev->dma_mask) {
dev_warn(dev, "DMA mask not set\n");
dev->dma_mask = &dev->coherent_dma_mask;
}
if (!size && dev->coherent_dma_mask)
size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1);
else if (!size)
size = 1ULL << 32;
/*
* Limit coherent and dma mask based on size and default mask
* set by the driver.
*/
end = dma_start + size - 1;
mask = DMA_BIT_MASK(ilog2(end) + 1);
dev->coherent_dma_mask &= mask;
*dev->dma_mask &= mask;
/* ...but only set bus limit and range map if we found valid dma-ranges earlier */
if (!ret) {
dev->bus_dma_limit = end;
dev->dma_range_map = map;
}
coherent = of_dma_is_coherent(np);
dev_dbg(dev, "device is%sdma coherent\n",
coherent ? " " : " not ");
iommu_ret = of_iommu_configure(dev, np, id);
if (iommu_ret == -EPROBE_DEFER) {
/* Don't touch range map if it wasn't set from a valid dma-ranges */
if (!ret)
dev->dma_range_map = NULL;
kfree(map);
return -EPROBE_DEFER;
} else if (iommu_ret == -ENODEV) {
dev_dbg(dev, "device is not behind an iommu\n");
} else if (iommu_ret) {
dev_err(dev, "iommu configuration for device failed with %pe\n",
ERR_PTR(iommu_ret));
/*
* Historically this routine doesn't fail driver probing
* due to errors in of_iommu_configure()
*/
} else
dev_dbg(dev, "device is behind an iommu\n");
arch_setup_dma_ops(dev, dma_start, size, coherent);
if (iommu_ret)
of_dma_set_restricted_buffer(dev, np);
return 0;
}
EXPORT_SYMBOL_GPL(of_dma_configure_id);
const void *of_device_get_match_data(const struct device *dev)
{
const struct of_device_id *match;
match = of_match_device(dev->driver->of_match_table, dev);
if (!match)
return NULL;
return match->data;
}
EXPORT_SYMBOL(of_device_get_match_data);
/**
* of_device_modalias - Fill buffer with newline terminated modalias string
* @dev: Calling device
* @str: Modalias string
* @len: Size of @str
*/
ssize_t of_device_modalias(struct device *dev, char *str, ssize_t len)
{
ssize_t sl;
if (!dev || !dev->of_node || dev->of_node_reused)
return -ENODEV;
sl = of_modalias(dev->of_node, str, len - 2);
if (sl < 0)
return sl;
if (sl > len - 2)
return -ENOMEM;
str[sl++] = '\n';
str[sl] = 0;
return sl;
}
EXPORT_SYMBOL_GPL(of_device_modalias);
/**
* of_device_uevent - Display OF related uevent information
* @dev: Device to display the uevent information for
* @env: Kernel object's userspace event reference to fill up
*/
void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
const char *compat, *type;
struct alias_prop *app;
struct property *p;
int seen = 0;
if ((!dev) || (!dev->of_node))
return;
add_uevent_var(env, "OF_NAME=%pOFn", dev->of_node);
add_uevent_var(env, "OF_FULLNAME=%pOF", dev->of_node);
type = of_node_get_device_type(dev->of_node);
if (type)
add_uevent_var(env, "OF_TYPE=%s", type);
/* Since the compatible field can contain pretty much anything
* it's not really legal to split it out with commas. We split it
* up using a number of environment variables instead. */
of_property_for_each_string(dev->of_node, "compatible", p, compat) { add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);
seen++;
}
add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);
seen = 0;
mutex_lock(&of_mutex);
list_for_each_entry(app, &aliases_lookup, link) { if (dev->of_node == app->np) { add_uevent_var(env, "OF_ALIAS_%d=%s", seen,
app->alias);
seen++;
}
}
mutex_unlock(&of_mutex);
}
EXPORT_SYMBOL_GPL(of_device_uevent);
int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env)
{
int sl;
if ((!dev) || (!dev->of_node) || dev->of_node_reused)
return -ENODEV;
/* Devicetree modalias is tricky, we add it in 2 steps */
if (add_uevent_var(env, "MODALIAS="))
return -ENOMEM;
sl = of_modalias(dev->of_node, &env->buf[env->buflen-1],
sizeof(env->buf) - env->buflen);
if (sl < 0)
return sl;
if (sl >= (sizeof(env->buf) - env->buflen))
return -ENOMEM;
env->buflen += sl; return 0;
}
EXPORT_SYMBOL_GPL(of_device_uevent_modalias);
/**
* of_device_make_bus_id - Use the device node data to assign a unique name
* @dev: pointer to device structure that is linked to a device tree node
*
* This routine will first try using the translated bus address to
* derive a unique name. If it cannot, then it will prepend names from
* parent nodes until a unique name can be derived.
*/
void of_device_make_bus_id(struct device *dev)
{
struct device_node *node = dev->of_node;
const __be32 *reg;
u64 addr;
u32 mask;
/* Construct the name, using parent nodes if necessary to ensure uniqueness */
while (node->parent) {
/*
* If the address can be translated, then that is as much
* uniqueness as we need. Make it the first component and return
*/
reg = of_get_property(node, "reg", NULL);
if (reg && (addr = of_translate_address(node, reg)) != OF_BAD_ADDR) {
if (!of_property_read_u32(node, "mask", &mask))
dev_set_name(dev, dev_name(dev) ? "%llx.%x.%pOFn:%s" : "%llx.%x.%pOFn",
addr, ffs(mask) - 1, node, dev_name(dev));
else
dev_set_name(dev, dev_name(dev) ? "%llx.%pOFn:%s" : "%llx.%pOFn",
addr, node, dev_name(dev));
return;
}
/* format arguments only used if dev_name() resolves to NULL */
dev_set_name(dev, dev_name(dev) ? "%s:%s" : "%s",
kbasename(node->full_name), dev_name(dev));
node = node->parent;
}
}
EXPORT_SYMBOL_GPL(of_device_make_bus_id);
// SPDX-License-Identifier: GPL-2.0
/*
* Tag allocation using scalable bitmaps. Uses active queue tracking to support
* fairer distribution of tags between multiple submitters when a shared tag map
* is used.
*
* Copyright (C) 2013-2014 Jens Axboe
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/delay.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-sched.h"
/*
* Recalculate wakeup batch when tag is shared by hctx.
*/
static void blk_mq_update_wake_batch(struct blk_mq_tags *tags,
unsigned int users)
{
if (!users)
return;
sbitmap_queue_recalculate_wake_batch(&tags->bitmap_tags,
users);
sbitmap_queue_recalculate_wake_batch(&tags->breserved_tags,
users);
}
/*
* If a previously inactive queue goes active, bump the active user count.
* We need to do this before try to allocate driver tag, then even if fail
* to get tag when first time, the other shared-tag users could reserve
* budget for it.
*/
void __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
{
unsigned int users;
struct blk_mq_tags *tags = hctx->tags;
/*
* calling test_bit() prior to test_and_set_bit() is intentional,
* it avoids dirtying the cacheline if the queue is already active.
*/
if (blk_mq_is_shared_tags(hctx->flags)) {
struct request_queue *q = hctx->queue;
if (test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags) ||
test_and_set_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags))
return;
} else {
if (test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) ||
test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
return;
}
spin_lock_irq(&tags->lock);
users = tags->active_queues + 1;
WRITE_ONCE(tags->active_queues, users);
blk_mq_update_wake_batch(tags, users);
spin_unlock_irq(&tags->lock);
}
/*
* Wakeup all potentially sleeping on tags
*/
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
{
sbitmap_queue_wake_all(&tags->bitmap_tags);
if (include_reserve)
sbitmap_queue_wake_all(&tags->breserved_tags);
}
/*
* If a previously busy queue goes inactive, potential waiters could now
* be allowed to queue. Wake them up and check.
*/
void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
{
struct blk_mq_tags *tags = hctx->tags;
unsigned int users;
if (blk_mq_is_shared_tags(hctx->flags)) {
struct request_queue *q = hctx->queue;
if (!test_and_clear_bit(QUEUE_FLAG_HCTX_ACTIVE,
&q->queue_flags))
return;
} else {
if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
return;
}
spin_lock_irq(&tags->lock);
users = tags->active_queues - 1;
WRITE_ONCE(tags->active_queues, users);
blk_mq_update_wake_batch(tags, users);
spin_unlock_irq(&tags->lock);
blk_mq_tag_wakeup_all(tags, false);
}
static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
struct sbitmap_queue *bt)
{
if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) &&
!hctx_may_queue(data->hctx, bt))
return BLK_MQ_NO_TAG;
if (data->shallow_depth)
return sbitmap_queue_get_shallow(bt, data->shallow_depth);
else
return __sbitmap_queue_get(bt);
}
unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags,
unsigned int *offset)
{
struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
struct sbitmap_queue *bt = &tags->bitmap_tags;
unsigned long ret;
if (data->shallow_depth ||data->flags & BLK_MQ_REQ_RESERVED ||
data->hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
return 0;
ret = __sbitmap_queue_get_batch(bt, nr_tags, offset);
*offset += tags->nr_reserved_tags;
return ret;
}
unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
struct sbitmap_queue *bt;
struct sbq_wait_state *ws;
DEFINE_SBQ_WAIT(wait);
unsigned int tag_offset;
int tag;
if (data->flags & BLK_MQ_REQ_RESERVED) {
if (unlikely(!tags->nr_reserved_tags)) {
WARN_ON_ONCE(1);
return BLK_MQ_NO_TAG;
}
bt = &tags->breserved_tags;
tag_offset = 0;
} else {
bt = &tags->bitmap_tags;
tag_offset = tags->nr_reserved_tags;
}
tag = __blk_mq_get_tag(data, bt);
if (tag != BLK_MQ_NO_TAG)
goto found_tag;
if (data->flags & BLK_MQ_REQ_NOWAIT)
return BLK_MQ_NO_TAG;
ws = bt_wait_ptr(bt, data->hctx);
do {
struct sbitmap_queue *bt_prev;
/*
* We're out of tags on this hardware queue, kick any
* pending IO submits before going to sleep waiting for
* some to complete.
*/
blk_mq_run_hw_queue(data->hctx, false);
/*
* Retry tag allocation after running the hardware queue,
* as running the queue may also have found completions.
*/
tag = __blk_mq_get_tag(data, bt);
if (tag != BLK_MQ_NO_TAG)
break;
sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE);
tag = __blk_mq_get_tag(data, bt);
if (tag != BLK_MQ_NO_TAG)
break;
bt_prev = bt;
io_schedule();
sbitmap_finish_wait(bt, ws, &wait);
data->ctx = blk_mq_get_ctx(data->q);
data->hctx = blk_mq_map_queue(data->q, data->cmd_flags,
data->ctx);
tags = blk_mq_tags_from_data(data);
if (data->flags & BLK_MQ_REQ_RESERVED)
bt = &tags->breserved_tags;
else
bt = &tags->bitmap_tags;
/*
* If destination hw queue is changed, fake wake up on
* previous queue for compensating the wake up miss, so
* other allocations on previous queue won't be starved.
*/
if (bt != bt_prev)
sbitmap_queue_wake_up(bt_prev, 1);
ws = bt_wait_ptr(bt, data->hctx);
} while (1);
sbitmap_finish_wait(bt, ws, &wait);
found_tag:
/*
* Give up this allocation if the hctx is inactive. The caller will
* retry on an active hctx.
*/
if (unlikely(test_bit(BLK_MQ_S_INACTIVE, &data->hctx->state))) {
blk_mq_put_tag(tags, data->ctx, tag + tag_offset);
return BLK_MQ_NO_TAG;
}
return tag + tag_offset;
}
void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
unsigned int tag)
{
if (!blk_mq_tag_is_reserved(tags, tag)) {
const int real_tag = tag - tags->nr_reserved_tags;
BUG_ON(real_tag >= tags->nr_tags);
sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
} else {
sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
}
}
void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags)
{
sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags,
tag_array, nr_tags);
}
struct bt_iter_data {
struct blk_mq_hw_ctx *hctx;
struct request_queue *q;
busy_tag_iter_fn *fn;
void *data;
bool reserved;
};
static struct request *blk_mq_find_and_get_req(struct blk_mq_tags *tags,
unsigned int bitnr)
{
struct request *rq;
unsigned long flags;
spin_lock_irqsave(&tags->lock, flags);
rq = tags->rqs[bitnr];
if (!rq || rq->tag != bitnr || !req_ref_inc_not_zero(rq))
rq = NULL;
spin_unlock_irqrestore(&tags->lock, flags);
return rq;
}
static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{
struct bt_iter_data *iter_data = data;
struct blk_mq_hw_ctx *hctx = iter_data->hctx;
struct request_queue *q = iter_data->q;
struct blk_mq_tag_set *set = q->tag_set;
struct blk_mq_tags *tags;
struct request *rq;
bool ret = true;
if (blk_mq_is_shared_tags(set->flags))
tags = set->shared_tags;
else
tags = hctx->tags;
if (!iter_data->reserved)
bitnr += tags->nr_reserved_tags;
/*
* We can hit rq == NULL here, because the tagging functions
* test and set the bit before assigning ->rqs[].
*/
rq = blk_mq_find_and_get_req(tags, bitnr);
if (!rq)
return true;
if (rq->q == q && (!hctx || rq->mq_hctx == hctx))
ret = iter_data->fn(rq, iter_data->data);
blk_mq_put_rq_ref(rq);
return ret;
}
/**
* bt_for_each - iterate over the requests associated with a hardware queue
* @hctx: Hardware queue to examine.
* @q: Request queue to examine.
* @bt: sbitmap to examine. This is either the breserved_tags member
* or the bitmap_tags member of struct blk_mq_tags.
* @fn: Pointer to the function that will be called for each request
* associated with @hctx that has been assigned a driver tag.
* @fn will be called as follows: @fn(@hctx, rq, @data, @reserved)
* where rq is a pointer to a request. Return true to continue
* iterating tags, false to stop.
* @data: Will be passed as third argument to @fn.
* @reserved: Indicates whether @bt is the breserved_tags member or the
* bitmap_tags member of struct blk_mq_tags.
*/
static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct request_queue *q,
struct sbitmap_queue *bt, busy_tag_iter_fn *fn,
void *data, bool reserved)
{
struct bt_iter_data iter_data = {
.hctx = hctx,
.fn = fn,
.data = data,
.reserved = reserved,
.q = q,
};
sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data);
}
struct bt_tags_iter_data {
struct blk_mq_tags *tags;
busy_tag_iter_fn *fn;
void *data;
unsigned int flags;
};
#define BT_TAG_ITER_RESERVED (1 << 0)
#define BT_TAG_ITER_STARTED (1 << 1)
#define BT_TAG_ITER_STATIC_RQS (1 << 2)
static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{
struct bt_tags_iter_data *iter_data = data;
struct blk_mq_tags *tags = iter_data->tags;
struct request *rq;
bool ret = true;
bool iter_static_rqs = !!(iter_data->flags & BT_TAG_ITER_STATIC_RQS);
if (!(iter_data->flags & BT_TAG_ITER_RESERVED))
bitnr += tags->nr_reserved_tags;
/*
* We can hit rq == NULL here, because the tagging functions
* test and set the bit before assigning ->rqs[].
*/
if (iter_static_rqs)
rq = tags->static_rqs[bitnr];
else
rq = blk_mq_find_and_get_req(tags, bitnr);
if (!rq)
return true;
if (!(iter_data->flags & BT_TAG_ITER_STARTED) ||
blk_mq_request_started(rq))
ret = iter_data->fn(rq, iter_data->data);
if (!iter_static_rqs)
blk_mq_put_rq_ref(rq);
return ret;
}
/**
* bt_tags_for_each - iterate over the requests in a tag map
* @tags: Tag map to iterate over.
* @bt: sbitmap to examine. This is either the breserved_tags member
* or the bitmap_tags member of struct blk_mq_tags.
* @fn: Pointer to the function that will be called for each started
* request. @fn will be called as follows: @fn(rq, @data,
* @reserved) where rq is a pointer to a request. Return true
* to continue iterating tags, false to stop.
* @data: Will be passed as second argument to @fn.
* @flags: BT_TAG_ITER_*
*/
static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
busy_tag_iter_fn *fn, void *data, unsigned int flags)
{
struct bt_tags_iter_data iter_data = {
.tags = tags,
.fn = fn,
.data = data,
.flags = flags,
};
if (tags->rqs)
sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
}
static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags,
busy_tag_iter_fn *fn, void *priv, unsigned int flags)
{
WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);
if (tags->nr_reserved_tags)
bt_tags_for_each(tags, &tags->breserved_tags, fn, priv,
flags | BT_TAG_ITER_RESERVED);
bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags);
}
/**
* blk_mq_all_tag_iter - iterate over all requests in a tag map
* @tags: Tag map to iterate over.
* @fn: Pointer to the function that will be called for each
* request. @fn will be called as follows: @fn(rq, @priv,
* reserved) where rq is a pointer to a request. 'reserved'
* indicates whether or not @rq is a reserved request. Return
* true to continue iterating tags, false to stop.
* @priv: Will be passed as second argument to @fn.
*
* Caller has to pass the tag map from which requests are allocated.
*/
void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
void *priv)
{
__blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS);
}
/**
* blk_mq_tagset_busy_iter - iterate over all started requests in a tag set
* @tagset: Tag set to iterate over.
* @fn: Pointer to the function that will be called for each started
* request. @fn will be called as follows: @fn(rq, @priv,
* reserved) where rq is a pointer to a request. 'reserved'
* indicates whether or not @rq is a reserved request. Return
* true to continue iterating tags, false to stop.
* @priv: Will be passed as second argument to @fn.
*
* We grab one request reference before calling @fn and release it after
* @fn returns.
*/
void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
busy_tag_iter_fn *fn, void *priv)
{
unsigned int flags = tagset->flags;
int i, nr_tags;
nr_tags = blk_mq_is_shared_tags(flags) ? 1 : tagset->nr_hw_queues;
for (i = 0; i < nr_tags; i++) {
if (tagset->tags && tagset->tags[i])
__blk_mq_all_tag_iter(tagset->tags[i], fn, priv,
BT_TAG_ITER_STARTED);
}
}
EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
static bool blk_mq_tagset_count_completed_rqs(struct request *rq, void *data)
{
unsigned *count = data;
if (blk_mq_request_completed(rq))
(*count)++;
return true;
}
/**
* blk_mq_tagset_wait_completed_request - Wait until all scheduled request
* completions have finished.
* @tagset: Tag set to drain completed request
*
* Note: This function has to be run after all IO queues are shutdown
*/
void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset)
{
while (true) {
unsigned count = 0;
blk_mq_tagset_busy_iter(tagset,
blk_mq_tagset_count_completed_rqs, &count);
if (!count)
break;
msleep(5);
}
}
EXPORT_SYMBOL(blk_mq_tagset_wait_completed_request);
/**
* blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag
* @q: Request queue to examine.
* @fn: Pointer to the function that will be called for each request
* on @q. @fn will be called as follows: @fn(hctx, rq, @priv,
* reserved) where rq is a pointer to a request and hctx points
* to the hardware queue associated with the request. 'reserved'
* indicates whether or not @rq is a reserved request.
* @priv: Will be passed as third argument to @fn.
*
* Note: if @q->tag_set is shared with other request queues then @fn will be
* called for all requests on all queues that share that tag set and not only
* for requests associated with @q.
*/
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn,
void *priv)
{
/*
* __blk_mq_update_nr_hw_queues() updates nr_hw_queues and hctx_table
* while the queue is frozen. So we can use q_usage_counter to avoid
* racing with it.
*/
if (!percpu_ref_tryget(&q->q_usage_counter))
return;
if (blk_mq_is_shared_tags(q->tag_set->flags)) {
struct blk_mq_tags *tags = q->tag_set->shared_tags;
struct sbitmap_queue *bresv = &tags->breserved_tags;
struct sbitmap_queue *btags = &tags->bitmap_tags;
if (tags->nr_reserved_tags)
bt_for_each(NULL, q, bresv, fn, priv, true);
bt_for_each(NULL, q, btags, fn, priv, false);
} else {
struct blk_mq_hw_ctx *hctx;
unsigned long i;
queue_for_each_hw_ctx(q, hctx, i) {
struct blk_mq_tags *tags = hctx->tags;
struct sbitmap_queue *bresv = &tags->breserved_tags;
struct sbitmap_queue *btags = &tags->bitmap_tags;
/*
* If no software queues are currently mapped to this
* hardware queue, there's nothing to check
*/
if (!blk_mq_hw_queue_mapped(hctx))
continue;
if (tags->nr_reserved_tags)
bt_for_each(hctx, q, bresv, fn, priv, true);
bt_for_each(hctx, q, btags, fn, priv, false);
}
}
blk_queue_exit(q);
}
static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
bool round_robin, int node)
{
return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL,
node);
}
int blk_mq_init_bitmaps(struct sbitmap_queue *bitmap_tags,
struct sbitmap_queue *breserved_tags,
unsigned int queue_depth, unsigned int reserved,
int node, int alloc_policy)
{
unsigned int depth = queue_depth - reserved;
bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
if (bt_alloc(bitmap_tags, depth, round_robin, node))
return -ENOMEM;
if (bt_alloc(breserved_tags, reserved, round_robin, node))
goto free_bitmap_tags;
return 0;
free_bitmap_tags:
sbitmap_queue_free(bitmap_tags);
return -ENOMEM;
}
struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
unsigned int reserved_tags,
int node, int alloc_policy)
{
struct blk_mq_tags *tags;
if (total_tags > BLK_MQ_TAG_MAX) {
pr_err("blk-mq: tag depth too large\n");
return NULL;
}
tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
if (!tags)
return NULL;
tags->nr_tags = total_tags;
tags->nr_reserved_tags = reserved_tags;
spin_lock_init(&tags->lock);
if (blk_mq_init_bitmaps(&tags->bitmap_tags, &tags->breserved_tags,
total_tags, reserved_tags, node,
alloc_policy) < 0) {
kfree(tags);
return NULL;
}
return tags;
}
void blk_mq_free_tags(struct blk_mq_tags *tags)
{
sbitmap_queue_free(&tags->bitmap_tags);
sbitmap_queue_free(&tags->breserved_tags);
kfree(tags);
}
int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
struct blk_mq_tags **tagsptr, unsigned int tdepth,
bool can_grow)
{
struct blk_mq_tags *tags = *tagsptr;
if (tdepth <= tags->nr_reserved_tags)
return -EINVAL;
/*
* If we are allowed to grow beyond the original size, allocate
* a new set of tags before freeing the old one.
*/
if (tdepth > tags->nr_tags) {
struct blk_mq_tag_set *set = hctx->queue->tag_set;
struct blk_mq_tags *new;
if (!can_grow)
return -EINVAL;
/*
* We need some sort of upper limit, set it high enough that
* no valid use cases should require more.
*/
if (tdepth > MAX_SCHED_RQ)
return -EINVAL;
/*
* Only the sbitmap needs resizing since we allocated the max
* initially.
*/
if (blk_mq_is_shared_tags(set->flags))
return 0;
new = blk_mq_alloc_map_and_rqs(set, hctx->queue_num, tdepth);
if (!new)
return -ENOMEM;
blk_mq_free_map_and_rqs(set, *tagsptr, hctx->queue_num);
*tagsptr = new;
} else {
/*
* Don't need (or can't) update reserved tags here, they
* remain static and should never need resizing.
*/
sbitmap_queue_resize(&tags->bitmap_tags,
tdepth - tags->nr_reserved_tags);
}
return 0;
}
void blk_mq_tag_resize_shared_tags(struct blk_mq_tag_set *set, unsigned int size)
{
struct blk_mq_tags *tags = set->shared_tags;
sbitmap_queue_resize(&tags->bitmap_tags, size - set->reserved_tags);
}
void blk_mq_tag_update_sched_shared_tags(struct request_queue *q)
{
sbitmap_queue_resize(&q->sched_shared_tags->bitmap_tags,
q->nr_requests - q->tag_set->reserved_tags);
}
/**
* blk_mq_unique_tag() - return a tag that is unique queue-wide
* @rq: request for which to compute a unique tag
*
* The tag field in struct request is unique per hardware queue but not over
* all hardware queues. Hence this function that returns a tag with the
* hardware context index in the upper bits and the per hardware queue tag in
* the lower bits.
*
* Note: When called for a request that is queued on a non-multiqueue request
* queue, the hardware context index is set to zero.
*/
u32 blk_mq_unique_tag(struct request *rq)
{
return (rq->mq_hctx->queue_num << BLK_MQ_UNIQUE_TAG_BITS) |
(rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
}
EXPORT_SYMBOL(blk_mq_unique_tag);
// SPDX-License-Identifier: GPL-2.0
// Generated by scripts/atomic/gen-atomic-instrumented.sh
// DO NOT MODIFY THIS FILE DIRECTLY
/*
* This file provoides atomic operations with explicit instrumentation (e.g.
* KASAN, KCSAN), which should be used unless it is necessary to avoid
* instrumentation. Where it is necessary to aovid instrumenation, the
* raw_atomic*() operations should be used.
*/
#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
#define _LINUX_ATOMIC_INSTRUMENTED_H
#include <linux/build_bug.h>
#include <linux/compiler.h>
#include <linux/instrumented.h>
/**
* atomic_read() - atomic load with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically loads the value of @v with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_read() there.
*
* Return: The value loaded from @v.
*/
static __always_inline int
atomic_read(const atomic_t *v)
{
instrument_atomic_read(v, sizeof(*v));
return raw_atomic_read(v);
}
/**
* atomic_read_acquire() - atomic load with acquire ordering
* @v: pointer to atomic_t
*
* Atomically loads the value of @v with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
*
* Return: The value loaded from @v.
*/
static __always_inline int
atomic_read_acquire(const atomic_t *v)
{
instrument_atomic_read(v, sizeof(*v));
return raw_atomic_read_acquire(v);
}
/**
* atomic_set() - atomic set with relaxed ordering
* @v: pointer to atomic_t
* @i: int value to assign
*
* Atomically sets @v to @i with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_set() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_set(atomic_t *v, int i)
{
instrument_atomic_write(v, sizeof(*v));
raw_atomic_set(v, i);
}
/**
* atomic_set_release() - atomic set with release ordering
* @v: pointer to atomic_t
* @i: int value to assign
*
* Atomically sets @v to @i with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_set_release() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_set_release(atomic_t *v, int i)
{
kcsan_release();
instrument_atomic_write(v, sizeof(*v));
raw_atomic_set_release(v, i);
}
/**
* atomic_add() - atomic add with relaxed ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_add(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_add(i, v);
}
/**
* atomic_add_return() - atomic add with full ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_add_return(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_return(i, v);
}
/**
* atomic_add_return_acquire() - atomic add with acquire ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_add_return_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_return_acquire(i, v);
}
/**
* atomic_add_return_release() - atomic add with release ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_add_return_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_return_release(i, v);
}
/**
* atomic_add_return_relaxed() - atomic add with relaxed ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_add_return_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_return_relaxed(i, v);
}
/**
* atomic_fetch_add() - atomic add with full ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_add(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_add(i, v);
}
/**
* atomic_fetch_add_acquire() - atomic add with acquire ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_add_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_add_acquire(i, v);
}
/**
* atomic_fetch_add_release() - atomic add with release ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_add_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_add_release(i, v);
}
/**
* atomic_fetch_add_relaxed() - atomic add with relaxed ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_add_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_add_relaxed(i, v);
}
/**
* atomic_sub() - atomic subtract with relaxed ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_sub() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_sub(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_sub(i, v);
}
/**
* atomic_sub_return() - atomic subtract with full ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_sub_return(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_sub_return(i, v);
}
/**
* atomic_sub_return_acquire() - atomic subtract with acquire ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_sub_return_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_sub_return_acquire(i, v);
}
/**
* atomic_sub_return_release() - atomic subtract with release ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_sub_return_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_sub_return_release(i, v);
}
/**
* atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_sub_return_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_sub_return_relaxed(i, v);
}
/**
* atomic_fetch_sub() - atomic subtract with full ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_sub(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_sub(i, v);
}
/**
* atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_sub_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_sub_acquire(i, v);
}
/**
* atomic_fetch_sub_release() - atomic subtract with release ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_sub_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_sub_release(i, v);
}
/**
* atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
* @i: int value to subtract
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_sub_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_sub_relaxed(i, v);
}
/**
* atomic_inc() - atomic increment with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_inc() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_inc(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_inc(v);
}
/**
* atomic_inc_return() - atomic increment with full ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_inc_return(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_return(v);
}
/**
* atomic_inc_return_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_inc_return_acquire(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_return_acquire(v);
}
/**
* atomic_inc_return_release() - atomic increment with release ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_inc_return_release(atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_return_release(v);
}
/**
* atomic_inc_return_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_inc_return_relaxed(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_return_relaxed(v);
}
/**
* atomic_fetch_inc() - atomic increment with full ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_inc(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_inc(v);
}
/**
* atomic_fetch_inc_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_inc_acquire(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_inc_acquire(v);
}
/**
* atomic_fetch_inc_release() - atomic increment with release ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_inc_release(atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_inc_release(v);
}
/**
* atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_inc_relaxed(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_inc_relaxed(v);
}
/**
* atomic_dec() - atomic decrement with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_dec() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_dec(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_dec(v);
}
/**
* atomic_dec_return() - atomic decrement with full ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_dec_return(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_return(v);
}
/**
* atomic_dec_return_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_dec_return_acquire(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_return_acquire(v);
}
/**
* atomic_dec_return_release() - atomic decrement with release ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_dec_return_release(atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_return_release(v);
}
/**
* atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline int
atomic_dec_return_relaxed(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_return_relaxed(v);
}
/**
* atomic_fetch_dec() - atomic decrement with full ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_dec(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_dec(v);
}
/**
* atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_dec_acquire(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_dec_acquire(v);
}
/**
* atomic_fetch_dec_release() - atomic decrement with release ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_dec_release(atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_dec_release(v);
}
/**
* atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_dec_relaxed(atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_dec_relaxed(v);
}
/**
* atomic_and() - atomic bitwise AND with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_and() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_and(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_and(i, v);
}
/**
* atomic_fetch_and() - atomic bitwise AND with full ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_and(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_and(i, v);
}
/**
* atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_and_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_and_acquire(i, v);
}
/**
* atomic_fetch_and_release() - atomic bitwise AND with release ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_and_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_and_release(i, v);
}
/**
* atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_and_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_and_relaxed(i, v);
}
/**
* atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_andnot() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_andnot(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_andnot(i, v);
}
/**
* atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & ~@i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_andnot(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_andnot(i, v);
}
/**
* atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & ~@i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_andnot_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_andnot_acquire(i, v);
}
/**
* atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & ~@i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_andnot_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_andnot_release(i, v);
}
/**
* atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_andnot_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_andnot_relaxed(i, v);
}
/**
* atomic_or() - atomic bitwise OR with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_or() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_or(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_or(i, v);
}
/**
* atomic_fetch_or() - atomic bitwise OR with full ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v | @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_or(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_or(i, v);
}
/**
* atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v | @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_or_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_or_acquire(i, v);
}
/**
* atomic_fetch_or_release() - atomic bitwise OR with release ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v | @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_or_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_or_release(i, v);
}
/**
* atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_or_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_or_relaxed(i, v);
}
/**
* atomic_xor() - atomic bitwise XOR with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_xor() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_xor(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_xor(i, v);
}
/**
* atomic_fetch_xor() - atomic bitwise XOR with full ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v ^ @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_xor(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_xor(i, v);
}
/**
* atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v ^ @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_xor_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_xor_acquire(i, v);
}
/**
* atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v ^ @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_xor_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_xor_release(i, v);
}
/**
* atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
* @i: int value
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_xor_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_xor_relaxed(i, v);
}
/**
* atomic_xchg() - atomic exchange with full ordering
* @v: pointer to atomic_t
* @new: int value to assign
*
* Atomically updates @v to @new with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_xchg() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_xchg(atomic_t *v, int new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_xchg(v, new);
}
/**
* atomic_xchg_acquire() - atomic exchange with acquire ordering
* @v: pointer to atomic_t
* @new: int value to assign
*
* Atomically updates @v to @new with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_xchg_acquire(atomic_t *v, int new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_xchg_acquire(v, new);
}
/**
* atomic_xchg_release() - atomic exchange with release ordering
* @v: pointer to atomic_t
* @new: int value to assign
*
* Atomically updates @v to @new with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_xchg_release(atomic_t *v, int new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_xchg_release(v, new);
}
/**
* atomic_xchg_relaxed() - atomic exchange with relaxed ordering
* @v: pointer to atomic_t
* @new: int value to assign
*
* Atomically updates @v to @new with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_xchg_relaxed(atomic_t *v, int new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_xchg_relaxed(v, new);
}
/**
* atomic_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic_t
* @old: int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_cmpxchg(atomic_t *v, int old, int new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_cmpxchg(v, old, new);
}
/**
* atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic_t
* @old: int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_cmpxchg_acquire(v, old, new);
}
/**
* atomic_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic_t
* @old: int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_cmpxchg_release(atomic_t *v, int old, int new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_cmpxchg_release(v, old, new);
}
/**
* atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic_t
* @old: int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_cmpxchg_relaxed(v, old, new);
}
/**
* atomic_try_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic_t
* @old: pointer to int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_try_cmpxchg(v, old, new);
}
/**
* atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic_t
* @old: pointer to int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
{
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_try_cmpxchg_acquire(v, old, new);
}
/**
* atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic_t
* @old: pointer to int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_try_cmpxchg_release(v, old, new);
}
/**
* atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic_t
* @old: pointer to int value to compare with
* @new: int value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
{
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_try_cmpxchg_relaxed(v, old, new);
}
/**
* atomic_sub_and_test() - atomic subtract and test if zero with full ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic_sub_and_test(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_sub_and_test(i, v);
}
/**
* atomic_dec_and_test() - atomic decrement and test if zero with full ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic_dec_and_test(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_and_test(v);
}
/**
* atomic_inc_and_test() - atomic increment and test if zero with full ordering
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic_inc_and_test(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_and_test(v);
}
/**
* atomic_add_negative() - atomic add and test if negative with full ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_add_negative(int i, atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_negative(i, v);
}
/**
* atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_add_negative_acquire(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_negative_acquire(i, v);
}
/**
* atomic_add_negative_release() - atomic add and test if negative with release ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_add_negative_release(int i, atomic_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_negative_release(i, v);
}
/**
* atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
* @i: int value to add
* @v: pointer to atomic_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_add_negative_relaxed(int i, atomic_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_add_negative_relaxed(i, v);
}
/**
* atomic_fetch_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic_t
* @a: int value to add
* @u: int value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
*
* Return: The original value of @v.
*/
static __always_inline int
atomic_fetch_add_unless(atomic_t *v, int a, int u)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_fetch_add_unless(v, a, u);
}
/**
* atomic_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic_t
* @a: int value to add
* @u: int value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_add_unless(atomic_t *v, int a, int u)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v)); return raw_atomic_add_unless(v, a, u);
}
/**
* atomic_inc_not_zero() - atomic increment unless zero with full ordering
* @v: pointer to atomic_t
*
* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_inc_not_zero(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_not_zero(v);
}
/**
* atomic_inc_unless_negative() - atomic increment unless negative with full ordering
* @v: pointer to atomic_t
*
* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_inc_unless_negative(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_inc_unless_negative(v);
}
/**
* atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
* @v: pointer to atomic_t
*
* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_dec_unless_positive(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_unless_positive(v);
}
/**
* atomic_dec_if_positive() - atomic decrement if positive with full ordering
* @v: pointer to atomic_t
*
* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
*
* Return: The old value of (@v - 1), regardless of whether @v was updated.
*/
static __always_inline int
atomic_dec_if_positive(atomic_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_dec_if_positive(v);
}
/**
* atomic64_read() - atomic load with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically loads the value of @v with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_read() there.
*
* Return: The value loaded from @v.
*/
static __always_inline s64
atomic64_read(const atomic64_t *v)
{
instrument_atomic_read(v, sizeof(*v));
return raw_atomic64_read(v);
}
/**
* atomic64_read_acquire() - atomic load with acquire ordering
* @v: pointer to atomic64_t
*
* Atomically loads the value of @v with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
*
* Return: The value loaded from @v.
*/
static __always_inline s64
atomic64_read_acquire(const atomic64_t *v)
{
instrument_atomic_read(v, sizeof(*v));
return raw_atomic64_read_acquire(v);
}
/**
* atomic64_set() - atomic set with relaxed ordering
* @v: pointer to atomic64_t
* @i: s64 value to assign
*
* Atomically sets @v to @i with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_set() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_set(atomic64_t *v, s64 i)
{
instrument_atomic_write(v, sizeof(*v));
raw_atomic64_set(v, i);
}
/**
* atomic64_set_release() - atomic set with release ordering
* @v: pointer to atomic64_t
* @i: s64 value to assign
*
* Atomically sets @v to @i with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_set_release(atomic64_t *v, s64 i)
{
kcsan_release();
instrument_atomic_write(v, sizeof(*v));
raw_atomic64_set_release(v, i);
}
/**
* atomic64_add() - atomic add with relaxed ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_add(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_add(i, v);
}
/**
* atomic64_add_return() - atomic add with full ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_add_return(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_return(i, v);
}
/**
* atomic64_add_return_acquire() - atomic add with acquire ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_add_return_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_return_acquire(i, v);
}
/**
* atomic64_add_return_release() - atomic add with release ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_add_return_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_return_release(i, v);
}
/**
* atomic64_add_return_relaxed() - atomic add with relaxed ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_add_return_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_return_relaxed(i, v);
}
/**
* atomic64_fetch_add() - atomic add with full ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_add(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_add(i, v);
}
/**
* atomic64_fetch_add_acquire() - atomic add with acquire ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_add_acquire(i, v);
}
/**
* atomic64_fetch_add_release() - atomic add with release ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_add_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_add_release(i, v);
}
/**
* atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_add_relaxed(i, v);
}
/**
* atomic64_sub() - atomic subtract with relaxed ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_sub() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_sub(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_sub(i, v);
}
/**
* atomic64_sub_return() - atomic subtract with full ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_sub_return(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_sub_return(i, v);
}
/**
* atomic64_sub_return_acquire() - atomic subtract with acquire ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_sub_return_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_sub_return_acquire(i, v);
}
/**
* atomic64_sub_return_release() - atomic subtract with release ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_sub_return_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_sub_return_release(i, v);
}
/**
* atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_sub_return_relaxed(i, v);
}
/**
* atomic64_fetch_sub() - atomic subtract with full ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_sub(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_sub(i, v);
}
/**
* atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_sub_acquire(i, v);
}
/**
* atomic64_fetch_sub_release() - atomic subtract with release ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_sub_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_sub_release(i, v);
}
/**
* atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
* @i: s64 value to subtract
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_sub_relaxed(i, v);
}
/**
* atomic64_inc() - atomic increment with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_inc(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_inc(v);
}
/**
* atomic64_inc_return() - atomic increment with full ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_inc_return(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_return(v);
}
/**
* atomic64_inc_return_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_inc_return_acquire(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_return_acquire(v);
}
/**
* atomic64_inc_return_release() - atomic increment with release ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_inc_return_release(atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_return_release(v);
}
/**
* atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_inc_return_relaxed(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_return_relaxed(v);
}
/**
* atomic64_fetch_inc() - atomic increment with full ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_inc(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_inc(v);
}
/**
* atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_inc_acquire(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_inc_acquire(v);
}
/**
* atomic64_fetch_inc_release() - atomic increment with release ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_inc_release(atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_inc_release(v);
}
/**
* atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_inc_relaxed(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_inc_relaxed(v);
}
/**
* atomic64_dec() - atomic decrement with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_dec(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_dec(v);
}
/**
* atomic64_dec_return() - atomic decrement with full ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_dec_return(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_return(v);
}
/**
* atomic64_dec_return_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_dec_return_acquire(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_return_acquire(v);
}
/**
* atomic64_dec_return_release() - atomic decrement with release ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_dec_return_release(atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_return_release(v);
}
/**
* atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline s64
atomic64_dec_return_relaxed(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_return_relaxed(v);
}
/**
* atomic64_fetch_dec() - atomic decrement with full ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_dec(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_dec(v);
}
/**
* atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_dec_acquire(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_dec_acquire(v);
}
/**
* atomic64_fetch_dec_release() - atomic decrement with release ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_dec_release(atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_dec_release(v);
}
/**
* atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_dec_relaxed(atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_dec_relaxed(v);
}
/**
* atomic64_and() - atomic bitwise AND with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_and() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_and(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_and(i, v);
}
/**
* atomic64_fetch_and() - atomic bitwise AND with full ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_and(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_and(i, v);
}
/**
* atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_and_acquire(i, v);
}
/**
* atomic64_fetch_and_release() - atomic bitwise AND with release ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_and_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_and_release(i, v);
}
/**
* atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_and_relaxed(i, v);
}
/**
* atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_andnot(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_andnot(i, v);
}
/**
* atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & ~@i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_andnot(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_andnot(i, v);
}
/**
* atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & ~@i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_andnot_acquire(i, v);
}
/**
* atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & ~@i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_andnot_release(i, v);
}
/**
* atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_andnot_relaxed(i, v);
}
/**
* atomic64_or() - atomic bitwise OR with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_or() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_or(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_or(i, v);
}
/**
* atomic64_fetch_or() - atomic bitwise OR with full ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v | @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_or(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_or(i, v);
}
/**
* atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v | @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_or_acquire(i, v);
}
/**
* atomic64_fetch_or_release() - atomic bitwise OR with release ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v | @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_or_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_or_release(i, v);
}
/**
* atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_or_relaxed(i, v);
}
/**
* atomic64_xor() - atomic bitwise XOR with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_xor() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic64_xor(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic64_xor(i, v);
}
/**
* atomic64_fetch_xor() - atomic bitwise XOR with full ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v ^ @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_xor(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_xor(i, v);
}
/**
* atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v ^ @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_xor_acquire(i, v);
}
/**
* atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v ^ @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_xor_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_xor_release(i, v);
}
/**
* atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
* @i: s64 value
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_xor_relaxed(i, v);
}
/**
* atomic64_xchg() - atomic exchange with full ordering
* @v: pointer to atomic64_t
* @new: s64 value to assign
*
* Atomically updates @v to @new with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_xchg(atomic64_t *v, s64 new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_xchg(v, new);
}
/**
* atomic64_xchg_acquire() - atomic exchange with acquire ordering
* @v: pointer to atomic64_t
* @new: s64 value to assign
*
* Atomically updates @v to @new with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_xchg_acquire(atomic64_t *v, s64 new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_xchg_acquire(v, new);
}
/**
* atomic64_xchg_release() - atomic exchange with release ordering
* @v: pointer to atomic64_t
* @new: s64 value to assign
*
* Atomically updates @v to @new with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_xchg_release(atomic64_t *v, s64 new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_xchg_release(v, new);
}
/**
* atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
* @v: pointer to atomic64_t
* @new: s64 value to assign
*
* Atomically updates @v to @new with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_xchg_relaxed(atomic64_t *v, s64 new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_xchg_relaxed(v, new);
}
/**
* atomic64_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic64_t
* @old: s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_cmpxchg(v, old, new);
}
/**
* atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic64_t
* @old: s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_cmpxchg_acquire(v, old, new);
}
/**
* atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic64_t
* @old: s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_cmpxchg_release(v, old, new);
}
/**
* atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic64_t
* @old: s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_cmpxchg_relaxed(v, old, new);
}
/**
* atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic64_t
* @old: pointer to s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic64_try_cmpxchg(v, old, new);
}
/**
* atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic64_t
* @old: pointer to s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
{
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic64_try_cmpxchg_acquire(v, old, new);
}
/**
* atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic64_t
* @old: pointer to s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic64_try_cmpxchg_release(v, old, new);
}
/**
* atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic64_t
* @old: pointer to s64 value to compare with
* @new: s64 value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
{
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
}
/**
* atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic64_sub_and_test(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_sub_and_test(i, v);
}
/**
* atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic64_dec_and_test(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_and_test(v);
}
/**
* atomic64_inc_and_test() - atomic increment and test if zero with full ordering
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic64_inc_and_test(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_and_test(v);
}
/**
* atomic64_add_negative() - atomic add and test if negative with full ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic64_add_negative(s64 i, atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_negative(i, v);
}
/**
* atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic64_add_negative_acquire(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_negative_acquire(i, v);
}
/**
* atomic64_add_negative_release() - atomic add and test if negative with release ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic64_add_negative_release(s64 i, atomic64_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_negative_release(i, v);
}
/**
* atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
* @i: s64 value to add
* @v: pointer to atomic64_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_negative_relaxed(i, v);
}
/**
* atomic64_fetch_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic64_t
* @a: s64 value to add
* @u: s64 value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
*
* Return: The original value of @v.
*/
static __always_inline s64
atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_fetch_add_unless(v, a, u);
}
/**
* atomic64_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic64_t
* @a: s64 value to add
* @u: s64 value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_add_unless(v, a, u);
}
/**
* atomic64_inc_not_zero() - atomic increment unless zero with full ordering
* @v: pointer to atomic64_t
*
* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic64_inc_not_zero(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_not_zero(v);
}
/**
* atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
* @v: pointer to atomic64_t
*
* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic64_inc_unless_negative(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_inc_unless_negative(v);
}
/**
* atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
* @v: pointer to atomic64_t
*
* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic64_dec_unless_positive(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_unless_positive(v);
}
/**
* atomic64_dec_if_positive() - atomic decrement if positive with full ordering
* @v: pointer to atomic64_t
*
* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
*
* Return: The old value of (@v - 1), regardless of whether @v was updated.
*/
static __always_inline s64
atomic64_dec_if_positive(atomic64_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic64_dec_if_positive(v);
}
/**
* atomic_long_read() - atomic load with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically loads the value of @v with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_read() there.
*
* Return: The value loaded from @v.
*/
static __always_inline long
atomic_long_read(const atomic_long_t *v)
{
instrument_atomic_read(v, sizeof(*v));
return raw_atomic_long_read(v);
}
/**
* atomic_long_read_acquire() - atomic load with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically loads the value of @v with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
*
* Return: The value loaded from @v.
*/
static __always_inline long
atomic_long_read_acquire(const atomic_long_t *v)
{
instrument_atomic_read(v, sizeof(*v));
return raw_atomic_long_read_acquire(v);
}
/**
* atomic_long_set() - atomic set with relaxed ordering
* @v: pointer to atomic_long_t
* @i: long value to assign
*
* Atomically sets @v to @i with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_set() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_set(atomic_long_t *v, long i)
{
instrument_atomic_write(v, sizeof(*v));
raw_atomic_long_set(v, i);
}
/**
* atomic_long_set_release() - atomic set with release ordering
* @v: pointer to atomic_long_t
* @i: long value to assign
*
* Atomically sets @v to @i with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_set_release(atomic_long_t *v, long i)
{
kcsan_release();
instrument_atomic_write(v, sizeof(*v));
raw_atomic_long_set_release(v, i);
}
/**
* atomic_long_add() - atomic add with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_add(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_add(i, v);
}
/**
* atomic_long_add_return() - atomic add with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_add_return(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_return(i, v);
}
/**
* atomic_long_add_return_acquire() - atomic add with acquire ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_add_return_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_return_acquire(i, v);
}
/**
* atomic_long_add_return_release() - atomic add with release ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_add_return_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_return_release(i, v);
}
/**
* atomic_long_add_return_relaxed() - atomic add with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_add_return_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_return_relaxed(i, v);
}
/**
* atomic_long_fetch_add() - atomic add with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_add(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_add(i, v);
}
/**
* atomic_long_fetch_add_acquire() - atomic add with acquire ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_add_acquire(i, v);
}
/**
* atomic_long_fetch_add_release() - atomic add with release ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_add_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_add_release(i, v);
}
/**
* atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_add_relaxed(i, v);
}
/**
* atomic_long_sub() - atomic subtract with relaxed ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_sub(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_sub(i, v);
}
/**
* atomic_long_sub_return() - atomic subtract with full ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_sub_return(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_sub_return(i, v);
}
/**
* atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_sub_return_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_sub_return_acquire(i, v);
}
/**
* atomic_long_sub_return_release() - atomic subtract with release ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_sub_return_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_sub_return_release(i, v);
}
/**
* atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_sub_return_relaxed(i, v);
}
/**
* atomic_long_fetch_sub() - atomic subtract with full ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_sub(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_sub(i, v);
}
/**
* atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_sub_acquire(i, v);
}
/**
* atomic_long_fetch_sub_release() - atomic subtract with release ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_sub_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_sub_release(i, v);
}
/**
* atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
* @i: long value to subtract
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_sub_relaxed(i, v);
}
/**
* atomic_long_inc() - atomic increment with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_inc(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_inc(v);
}
/**
* atomic_long_inc_return() - atomic increment with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_inc_return(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_return(v);
}
/**
* atomic_long_inc_return_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_inc_return_acquire(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_return_acquire(v);
}
/**
* atomic_long_inc_return_release() - atomic increment with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_inc_return_release(atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_return_release(v);
}
/**
* atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_inc_return_relaxed(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_return_relaxed(v);
}
/**
* atomic_long_fetch_inc() - atomic increment with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_inc(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_inc(v);
}
/**
* atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_inc_acquire(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_inc_acquire(v);
}
/**
* atomic_long_fetch_inc_release() - atomic increment with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_inc_release(atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_inc_release(v);
}
/**
* atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_inc_relaxed(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_inc_relaxed(v);
}
/**
* atomic_long_dec() - atomic decrement with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_dec(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_dec(v);
}
/**
* atomic_long_dec_return() - atomic decrement with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_dec_return(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_return(v);
}
/**
* atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_dec_return_acquire(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_return_acquire(v);
}
/**
* atomic_long_dec_return_release() - atomic decrement with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_dec_return_release(atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_return_release(v);
}
/**
* atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
*
* Return: The updated value of @v.
*/
static __always_inline long
atomic_long_dec_return_relaxed(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_return_relaxed(v);
}
/**
* atomic_long_fetch_dec() - atomic decrement with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_dec(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_dec(v);
}
/**
* atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_dec_acquire(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_dec_acquire(v);
}
/**
* atomic_long_fetch_dec_release() - atomic decrement with release ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_dec_release(atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_dec_release(v);
}
/**
* atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_dec_relaxed(atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_dec_relaxed(v);
}
/**
* atomic_long_and() - atomic bitwise AND with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_and() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_and(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_and(i, v);
}
/**
* atomic_long_fetch_and() - atomic bitwise AND with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_and(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_and(i, v);
}
/**
* atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_and_acquire(i, v);
}
/**
* atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_and_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_and_release(i, v);
}
/**
* atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_and_relaxed(i, v);
}
/**
* atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_andnot(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_andnot(i, v);
}
/**
* atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_andnot(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_andnot(i, v);
}
/**
* atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_andnot_acquire(i, v);
}
/**
* atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_andnot_release(i, v);
}
/**
* atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v & ~@i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_andnot_relaxed(i, v);
}
/**
* atomic_long_or() - atomic bitwise OR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_or() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_or(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_or(i, v);
}
/**
* atomic_long_fetch_or() - atomic bitwise OR with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_or(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_or(i, v);
}
/**
* atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_or_acquire(i, v);
}
/**
* atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_or_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_or_release(i, v);
}
/**
* atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v | @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_or_relaxed(i, v);
}
/**
* atomic_long_xor() - atomic bitwise XOR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
*
* Return: Nothing.
*/
static __always_inline void
atomic_long_xor(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
raw_atomic_long_xor(i, v);
}
/**
* atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_xor(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_xor(i, v);
}
/**
* atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_xor_acquire(i, v);
}
/**
* atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_xor_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_xor_release(i, v);
}
/**
* atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
* @i: long value
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v ^ @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_xor_relaxed(i, v);
}
/**
* atomic_long_xchg() - atomic exchange with full ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_xchg(atomic_long_t *v, long new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_xchg(v, new);
}
/**
* atomic_long_xchg_acquire() - atomic exchange with acquire ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_xchg_acquire(atomic_long_t *v, long new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_xchg_acquire(v, new);
}
/**
* atomic_long_xchg_release() - atomic exchange with release ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_xchg_release(atomic_long_t *v, long new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_xchg_release(v, new);
}
/**
* atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
* @v: pointer to atomic_long_t
* @new: long value to assign
*
* Atomically updates @v to @new with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_xchg_relaxed(atomic_long_t *v, long new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_xchg_relaxed(v, new);
}
/**
* atomic_long_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_cmpxchg(v, old, new);
}
/**
* atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_cmpxchg_acquire(v, old, new);
}
/**
* atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_cmpxchg_release(v, old, new);
}
/**
* atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic_long_t
* @old: long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_cmpxchg_relaxed(v, old, new);
}
/**
* atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with full ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_long_try_cmpxchg(v, old, new);
}
/**
* atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with acquire ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
{
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
}
/**
* atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with release ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_long_try_cmpxchg_release(v, old, new);
}
/**
* atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
* @v: pointer to atomic_long_t
* @old: pointer to long value to compare with
* @new: long value to assign
*
* If (@v == @old), atomically updates @v to @new with relaxed ordering.
* Otherwise, @v is not modified, @old is updated to the current value of @v,
* and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
*
* Return: @true if the exchange occured, @false otherwise.
*/
static __always_inline bool
atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
{
instrument_atomic_read_write(v, sizeof(*v));
instrument_atomic_read_write(old, sizeof(*old));
return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
}
/**
* atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic_long_sub_and_test(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_sub_and_test(i, v);
}
/**
* atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v - 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic_long_dec_and_test(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_and_test(v);
}
/**
* atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + 1) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
*
* Return: @true if the resulting value of @v is zero, @false otherwise.
*/
static __always_inline bool
atomic_long_inc_and_test(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_and_test(v);
}
/**
* atomic_long_add_negative() - atomic add and test if negative with full ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with full ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_long_add_negative(long i, atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_negative(i, v);
}
/**
* atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with acquire ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_long_add_negative_acquire(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_negative_acquire(i, v);
}
/**
* atomic_long_add_negative_release() - atomic add and test if negative with release ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with release ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_long_add_negative_release(long i, atomic_long_t *v)
{
kcsan_release();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_negative_release(i, v);
}
/**
* atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
* @i: long value to add
* @v: pointer to atomic_long_t
*
* Atomically updates @v to (@v + @i) with relaxed ordering.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
*
* Return: @true if the resulting value of @v is negative, @false otherwise.
*/
static __always_inline bool
atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
{
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_negative_relaxed(i, v);
}
/**
* atomic_long_fetch_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic_long_t
* @a: long value to add
* @u: long value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
*
* Return: The original value of @v.
*/
static __always_inline long
atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_fetch_add_unless(v, a, u);
}
/**
* atomic_long_add_unless() - atomic add unless value with full ordering
* @v: pointer to atomic_long_t
* @a: long value to add
* @u: long value to compare with
*
* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_long_add_unless(atomic_long_t *v, long a, long u)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_add_unless(v, a, u);
}
/**
* atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
* @v: pointer to atomic_long_t
*
* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_long_inc_not_zero(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_not_zero(v);
}
/**
* atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
* @v: pointer to atomic_long_t
*
* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_long_inc_unless_negative(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_inc_unless_negative(v);
}
/**
* atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
* @v: pointer to atomic_long_t
*
* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
*
* Return: @true if @v was updated, @false otherwise.
*/
static __always_inline bool
atomic_long_dec_unless_positive(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_unless_positive(v);
}
/**
* atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
* @v: pointer to atomic_long_t
*
* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
* Otherwise, @v is not modified and relaxed ordering is provided.
*
* Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
*
* Return: The old value of (@v - 1), regardless of whether @v was updated.
*/
static __always_inline long
atomic_long_dec_if_positive(atomic_long_t *v)
{
kcsan_mb();
instrument_atomic_read_write(v, sizeof(*v));
return raw_atomic_long_dec_if_positive(v);
}
#define xchg(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_xchg(__ai_ptr, __VA_ARGS__); \
})
#define xchg_acquire(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
})
#define xchg_release(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_xchg_release(__ai_ptr, __VA_ARGS__); \
})
#define xchg_relaxed(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg_acquire(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg_release(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg_relaxed(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg64(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg64_acquire(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg64_release(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg64_relaxed(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_acquire(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_release(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_relaxed(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
})
#define try_cmpxchg(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define cmpxchg_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg64_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
})
#define sync_cmpxchg(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
})
#define try_cmpxchg_local(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_local(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_local(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define sync_try_cmpxchg(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
})
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
// ce5b65e0f1f8a276268b667194581d24bed219d4
// SPDX-License-Identifier: GPL-2.0-only
/*
* mm/page-writeback.c
*
* Copyright (C) 2002, Linus Torvalds.
* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
*
* Contains functions related to writing back dirty pages at the
* address_space level.
*
* 10Apr2002 Andrew Morton
* Initial version
*/
#include <linux/kernel.h>
#include <linux/math64.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/writeback.h>
#include <linux/init.h>
#include <linux/backing-dev.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/blkdev.h>
#include <linux/mpage.h>
#include <linux/rmap.h>
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
#include <linux/pagevec.h>
#include <linux/timer.h>
#include <linux/sched/rt.h>
#include <linux/sched/signal.h>
#include <linux/mm_inline.h>
#include <trace/events/writeback.h>
#include "internal.h"
/*
* Sleep at most 200ms at a time in balance_dirty_pages().
*/
#define MAX_PAUSE max(HZ/5, 1)
/*
* Try to keep balance_dirty_pages() call intervals higher than this many pages
* by raising pause time to max_pause when falls below it.
*/
#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10))
/*
* Estimate write bandwidth at 200ms intervals.
*/
#define BANDWIDTH_INTERVAL max(HZ/5, 1)
#define RATELIMIT_CALC_SHIFT 10
/*
* After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
* will look to see if it needs to force writeback or throttling.
*/
static long ratelimit_pages = 32;
/* The following parameters are exported via /proc/sys/vm */
/*
* Start background writeback (via writeback threads) at this percentage
*/
static int dirty_background_ratio = 10;
/*
* dirty_background_bytes starts at 0 (disabled) so that it is a function of
* dirty_background_ratio * the amount of dirtyable memory
*/
static unsigned long dirty_background_bytes;
/*
* free highmem will not be subtracted from the total free memory
* for calculating free ratios if vm_highmem_is_dirtyable is true
*/
static int vm_highmem_is_dirtyable;
/*
* The generator of dirty data starts writeback at this percentage
*/
static int vm_dirty_ratio = 20;
/*
* vm_dirty_bytes starts at 0 (disabled) so that it is a function of
* vm_dirty_ratio * the amount of dirtyable memory
*/
static unsigned long vm_dirty_bytes;
/*
* The interval between `kupdate'-style writebacks
*/
unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
EXPORT_SYMBOL_GPL(dirty_writeback_interval);
/*
* The longest time for which data is allowed to remain dirty
*/
unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
/*
* Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
* a full sync is triggered after this time elapses without any disk activity.
*/
int laptop_mode;
EXPORT_SYMBOL(laptop_mode);
/* End of sysctl-exported parameters */
struct wb_domain global_wb_domain;
/* consolidated parameters for balance_dirty_pages() and its subroutines */
struct dirty_throttle_control {
#ifdef CONFIG_CGROUP_WRITEBACK
struct wb_domain *dom;
struct dirty_throttle_control *gdtc; /* only set in memcg dtc's */
#endif
struct bdi_writeback *wb;
struct fprop_local_percpu *wb_completions;
unsigned long avail; /* dirtyable */
unsigned long dirty; /* file_dirty + write + nfs */
unsigned long thresh; /* dirty threshold */
unsigned long bg_thresh; /* dirty background threshold */
unsigned long wb_dirty; /* per-wb counterparts */
unsigned long wb_thresh;
unsigned long wb_bg_thresh;
unsigned long pos_ratio;
};
/*
* Length of period for aging writeout fractions of bdis. This is an
* arbitrarily chosen number. The longer the period, the slower fractions will
* reflect changes in current writeout rate.
*/
#define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
#ifdef CONFIG_CGROUP_WRITEBACK
#define GDTC_INIT(__wb) .wb = (__wb), \
.dom = &global_wb_domain, \
.wb_completions = &(__wb)->completions
#define GDTC_INIT_NO_WB .dom = &global_wb_domain
#define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \
.dom = mem_cgroup_wb_domain(__wb), \
.wb_completions = &(__wb)->memcg_completions, \
.gdtc = __gdtc
static bool mdtc_valid(struct dirty_throttle_control *dtc)
{
return dtc->dom;
}
static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
{
return dtc->dom;
}
static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
{
return mdtc->gdtc;
}
static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
{
return &wb->memcg_completions;
}
static void wb_min_max_ratio(struct bdi_writeback *wb,
unsigned long *minp, unsigned long *maxp)
{
unsigned long this_bw = READ_ONCE(wb->avg_write_bandwidth);
unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
unsigned long long min = wb->bdi->min_ratio;
unsigned long long max = wb->bdi->max_ratio;
/*
* @wb may already be clean by the time control reaches here and
* the total may not include its bw.
*/
if (this_bw < tot_bw) {
if (min) {
min *= this_bw;
min = div64_ul(min, tot_bw);
}
if (max < 100 * BDI_RATIO_SCALE) {
max *= this_bw;
max = div64_ul(max, tot_bw);
}
}
*minp = min;
*maxp = max;
}
#else /* CONFIG_CGROUP_WRITEBACK */
#define GDTC_INIT(__wb) .wb = (__wb), \
.wb_completions = &(__wb)->completions
#define GDTC_INIT_NO_WB
#define MDTC_INIT(__wb, __gdtc)
static bool mdtc_valid(struct dirty_throttle_control *dtc)
{
return false;
}
static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
{
return &global_wb_domain;
}
static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
{
return NULL;
}
static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
{
return NULL;
}
static void wb_min_max_ratio(struct bdi_writeback *wb,
unsigned long *minp, unsigned long *maxp)
{
*minp = wb->bdi->min_ratio;
*maxp = wb->bdi->max_ratio;
}
#endif /* CONFIG_CGROUP_WRITEBACK */
/*
* In a memory zone, there is a certain amount of pages we consider
* available for the page cache, which is essentially the number of
* free and reclaimable pages, minus some zone reserves to protect
* lowmem and the ability to uphold the zone's watermarks without
* requiring writeback.
*
* This number of dirtyable pages is the base value of which the
* user-configurable dirty ratio is the effective number of pages that
* are allowed to be actually dirtied. Per individual zone, or
* globally by using the sum of dirtyable pages over all zones.
*
* Because the user is allowed to specify the dirty limit globally as
* absolute number of bytes, calculating the per-zone dirty limit can
* require translating the configured limit into a percentage of
* global dirtyable memory first.
*/
/**
* node_dirtyable_memory - number of dirtyable pages in a node
* @pgdat: the node
*
* Return: the node's number of pages potentially available for dirty
* page cache. This is the base value for the per-node dirty limits.
*/
static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)
{
unsigned long nr_pages = 0;
int z;
for (z = 0; z < MAX_NR_ZONES; z++) {
struct zone *zone = pgdat->node_zones + z;
if (!populated_zone(zone))
continue;
nr_pages += zone_page_state(zone, NR_FREE_PAGES);
}
/*
* Pages reserved for the kernel should not be considered
* dirtyable, to prevent a situation where reclaim has to
* clean pages in order to balance the zones.
*/
nr_pages -= min(nr_pages, pgdat->totalreserve_pages);
nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE);
nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE);
return nr_pages;
}
static unsigned long highmem_dirtyable_memory(unsigned long total)
{
#ifdef CONFIG_HIGHMEM
int node;
unsigned long x = 0;
int i;
for_each_node_state(node, N_HIGH_MEMORY) {
for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
struct zone *z;
unsigned long nr_pages;
if (!is_highmem_idx(i))
continue;
z = &NODE_DATA(node)->node_zones[i];
if (!populated_zone(z))
continue;
nr_pages = zone_page_state(z, NR_FREE_PAGES);
/* watch for underflows */
nr_pages -= min(nr_pages, high_wmark_pages(z));
nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
x += nr_pages;
}
}
/*
* Make sure that the number of highmem pages is never larger
* than the number of the total dirtyable memory. This can only
* occur in very strange VM situations but we want to make sure
* that this does not occur.
*/
return min(x, total);
#else
return 0;
#endif
}
/**
* global_dirtyable_memory - number of globally dirtyable pages
*
* Return: the global number of pages potentially available for dirty
* page cache. This is the base value for the global dirty limits.
*/
static unsigned long global_dirtyable_memory(void)
{
unsigned long x;
x = global_zone_page_state(NR_FREE_PAGES);
/*
* Pages reserved for the kernel should not be considered
* dirtyable, to prevent a situation where reclaim has to
* clean pages in order to balance the zones.
*/
x -= min(x, totalreserve_pages);
x += global_node_page_state(NR_INACTIVE_FILE);
x += global_node_page_state(NR_ACTIVE_FILE);
if (!vm_highmem_is_dirtyable)
x -= highmem_dirtyable_memory(x);
return x + 1; /* Ensure that we never return 0 */
}
/**
* domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
* @dtc: dirty_throttle_control of interest
*
* Calculate @dtc->thresh and ->bg_thresh considering
* vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}. The caller
* must ensure that @dtc->avail is set before calling this function. The
* dirty limits will be lifted by 1/4 for real-time tasks.
*/
static void domain_dirty_limits(struct dirty_throttle_control *dtc)
{
const unsigned long available_memory = dtc->avail;
struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);
unsigned long bytes = vm_dirty_bytes;
unsigned long bg_bytes = dirty_background_bytes;
/* convert ratios to per-PAGE_SIZE for higher precision */
unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;
unsigned long thresh;
unsigned long bg_thresh;
struct task_struct *tsk;
/* gdtc is !NULL iff @dtc is for memcg domain */
if (gdtc) {
unsigned long global_avail = gdtc->avail;
/*
* The byte settings can't be applied directly to memcg
* domains. Convert them to ratios by scaling against
* globally available memory. As the ratios are in
* per-PAGE_SIZE, they can be obtained by dividing bytes by
* number of pages.
*/
if (bytes)
ratio = min(DIV_ROUND_UP(bytes, global_avail),
PAGE_SIZE);
if (bg_bytes)
bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
PAGE_SIZE);
bytes = bg_bytes = 0;
}
if (bytes)
thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);
else
thresh = (ratio * available_memory) / PAGE_SIZE;
if (bg_bytes)
bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);
else
bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
if (bg_thresh >= thresh)
bg_thresh = thresh / 2;
tsk = current;
if (rt_task(tsk)) {
bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
}
dtc->thresh = thresh;
dtc->bg_thresh = bg_thresh;
/* we should eventually report the domain in the TP */
if (!gdtc)
trace_global_dirty_state(bg_thresh, thresh);
}
/**
* global_dirty_limits - background-writeback and dirty-throttling thresholds
* @pbackground: out parameter for bg_thresh
* @pdirty: out parameter for thresh
*
* Calculate bg_thresh and thresh for global_wb_domain. See
* domain_dirty_limits() for details.
*/
void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
{
struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
gdtc.avail = global_dirtyable_memory();
domain_dirty_limits(&gdtc);
*pbackground = gdtc.bg_thresh;
*pdirty = gdtc.thresh;
}
/**
* node_dirty_limit - maximum number of dirty pages allowed in a node
* @pgdat: the node
*
* Return: the maximum number of dirty pages allowed in a node, based
* on the node's dirtyable memory.
*/
static unsigned long node_dirty_limit(struct pglist_data *pgdat)
{
unsigned long node_memory = node_dirtyable_memory(pgdat);
struct task_struct *tsk = current;
unsigned long dirty;
if (vm_dirty_bytes)
dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
node_memory / global_dirtyable_memory();
else
dirty = vm_dirty_ratio * node_memory / 100;
if (rt_task(tsk))
dirty += dirty / 4;
return dirty;
}
/**
* node_dirty_ok - tells whether a node is within its dirty limits
* @pgdat: the node to check
*
* Return: %true when the dirty pages in @pgdat are within the node's
* dirty limit, %false if the limit is exceeded.
*/
bool node_dirty_ok(struct pglist_data *pgdat)
{
unsigned long limit = node_dirty_limit(pgdat);
unsigned long nr_pages = 0;
nr_pages += node_page_state(pgdat, NR_FILE_DIRTY);
nr_pages += node_page_state(pgdat, NR_WRITEBACK);
return nr_pages <= limit;
}
#ifdef CONFIG_SYSCTL
static int dirty_background_ratio_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int ret;
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (ret == 0 && write)
dirty_background_bytes = 0;
return ret;
}
static int dirty_background_bytes_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
int ret;
ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
if (ret == 0 && write)
dirty_background_ratio = 0;
return ret;
}
static int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
int old_ratio = vm_dirty_ratio;
int ret;
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
writeback_set_ratelimit();
vm_dirty_bytes = 0;
}
return ret;
}
static int dirty_bytes_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
unsigned long old_bytes = vm_dirty_bytes;
int ret;
ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
writeback_set_ratelimit();
vm_dirty_ratio = 0;
}
return ret;
}
#endif
static unsigned long wp_next_time(unsigned long cur_time)
{
cur_time += VM_COMPLETIONS_PERIOD_LEN;
/* 0 has a special meaning... */
if (!cur_time)
return 1;
return cur_time;
}
static void wb_domain_writeout_add(struct wb_domain *dom,
struct fprop_local_percpu *completions,
unsigned int max_prop_frac, long nr)
{
__fprop_add_percpu_max(&dom->completions, completions,
max_prop_frac, nr);
/* First event after period switching was turned off? */
if (unlikely(!dom->period_time)) {
/*
* We can race with other __bdi_writeout_inc calls here but
* it does not cause any harm since the resulting time when
* timer will fire and what is in writeout_period_time will be
* roughly the same.
*/
dom->period_time = wp_next_time(jiffies);
mod_timer(&dom->period_timer, dom->period_time);
}
}
/*
* Increment @wb's writeout completion count and the global writeout
* completion count. Called from __folio_end_writeback().
*/
static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr)
{
struct wb_domain *cgdom;
wb_stat_mod(wb, WB_WRITTEN, nr);
wb_domain_writeout_add(&global_wb_domain, &wb->completions,
wb->bdi->max_prop_frac, nr);
cgdom = mem_cgroup_wb_domain(wb);
if (cgdom)
wb_domain_writeout_add(cgdom, wb_memcg_completions(wb),
wb->bdi->max_prop_frac, nr);
}
void wb_writeout_inc(struct bdi_writeback *wb)
{
unsigned long flags;
local_irq_save(flags);
__wb_writeout_add(wb, 1);
local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(wb_writeout_inc);
/*
* On idle system, we can be called long after we scheduled because we use
* deferred timers so count with missed periods.
*/
static void writeout_period(struct timer_list *t)
{
struct wb_domain *dom = from_timer(dom, t, period_timer);
int miss_periods = (jiffies - dom->period_time) /
VM_COMPLETIONS_PERIOD_LEN;
if (fprop_new_period(&dom->completions, miss_periods + 1)) {
dom->period_time = wp_next_time(dom->period_time +
miss_periods * VM_COMPLETIONS_PERIOD_LEN);
mod_timer(&dom->period_timer, dom->period_time);
} else {
/*
* Aging has zeroed all fractions. Stop wasting CPU on period
* updates.
*/
dom->period_time = 0;
}
}
int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
{
memset(dom, 0, sizeof(*dom));
spin_lock_init(&dom->lock);
timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);
dom->dirty_limit_tstamp = jiffies;
return fprop_global_init(&dom->completions, gfp);
}
#ifdef CONFIG_CGROUP_WRITEBACK
void wb_domain_exit(struct wb_domain *dom)
{
del_timer_sync(&dom->period_timer);
fprop_global_destroy(&dom->completions);
}
#endif
/*
* bdi_min_ratio keeps the sum of the minimum dirty shares of all
* registered backing devices, which, for obvious reasons, can not
* exceed 100%.
*/
static unsigned int bdi_min_ratio;
static int bdi_check_pages_limit(unsigned long pages)
{
unsigned long max_dirty_pages = global_dirtyable_memory();
if (pages > max_dirty_pages)
return -EINVAL;
return 0;
}
static unsigned long bdi_ratio_from_pages(unsigned long pages)
{
unsigned long background_thresh;
unsigned long dirty_thresh;
unsigned long ratio;
global_dirty_limits(&background_thresh, &dirty_thresh);
ratio = div64_u64(pages * 100ULL * BDI_RATIO_SCALE, dirty_thresh);
return ratio;
}
static u64 bdi_get_bytes(unsigned int ratio)
{
unsigned long background_thresh;
unsigned long dirty_thresh;
u64 bytes;
global_dirty_limits(&background_thresh, &dirty_thresh);
bytes = (dirty_thresh * PAGE_SIZE * ratio) / BDI_RATIO_SCALE / 100;
return bytes;
}
static int __bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
{
unsigned int delta;
int ret = 0;
if (min_ratio > 100 * BDI_RATIO_SCALE)
return -EINVAL;
spin_lock_bh(&bdi_lock);
if (min_ratio > bdi->max_ratio) {
ret = -EINVAL;
} else {
if (min_ratio < bdi->min_ratio) {
delta = bdi->min_ratio - min_ratio;
bdi_min_ratio -= delta;
bdi->min_ratio = min_ratio;
} else {
delta = min_ratio - bdi->min_ratio;
if (bdi_min_ratio + delta < 100 * BDI_RATIO_SCALE) {
bdi_min_ratio += delta;
bdi->min_ratio = min_ratio;
} else {
ret = -EINVAL;
}
}
}
spin_unlock_bh(&bdi_lock);
return ret;
}
static int __bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio)
{
int ret = 0;
if (max_ratio > 100 * BDI_RATIO_SCALE)
return -EINVAL;
spin_lock_bh(&bdi_lock);
if (bdi->min_ratio > max_ratio) {
ret = -EINVAL;
} else {
bdi->max_ratio = max_ratio;
bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) /
(100 * BDI_RATIO_SCALE);
}
spin_unlock_bh(&bdi_lock);
return ret;
}
int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio)
{
return __bdi_set_min_ratio(bdi, min_ratio);
}
int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio)
{
return __bdi_set_max_ratio(bdi, max_ratio);
}
int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
{
return __bdi_set_min_ratio(bdi, min_ratio * BDI_RATIO_SCALE);
}
int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio)
{
return __bdi_set_max_ratio(bdi, max_ratio * BDI_RATIO_SCALE);
}
EXPORT_SYMBOL(bdi_set_max_ratio);
u64 bdi_get_min_bytes(struct backing_dev_info *bdi)
{
return bdi_get_bytes(bdi->min_ratio);
}
int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes)
{
int ret;
unsigned long pages = min_bytes >> PAGE_SHIFT;
unsigned long min_ratio;
ret = bdi_check_pages_limit(pages);
if (ret)
return ret;
min_ratio = bdi_ratio_from_pages(pages);
return __bdi_set_min_ratio(bdi, min_ratio);
}
u64 bdi_get_max_bytes(struct backing_dev_info *bdi)
{
return bdi_get_bytes(bdi->max_ratio);
}
int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes)
{
int ret;
unsigned long pages = max_bytes >> PAGE_SHIFT;
unsigned long max_ratio;
ret = bdi_check_pages_limit(pages);
if (ret)
return ret;
max_ratio = bdi_ratio_from_pages(pages);
return __bdi_set_max_ratio(bdi, max_ratio);
}
int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit)
{
if (strict_limit > 1)
return -EINVAL;
spin_lock_bh(&bdi_lock);
if (strict_limit)
bdi->capabilities |= BDI_CAP_STRICTLIMIT;
else
bdi->capabilities &= ~BDI_CAP_STRICTLIMIT;
spin_unlock_bh(&bdi_lock);
return 0;
}
static unsigned long dirty_freerun_ceiling(unsigned long thresh,
unsigned long bg_thresh)
{
return (thresh + bg_thresh) / 2;
}
static unsigned long hard_dirty_limit(struct wb_domain *dom,
unsigned long thresh)
{
return max(thresh, dom->dirty_limit);
}
/*
* Memory which can be further allocated to a memcg domain is capped by
* system-wide clean memory excluding the amount being used in the domain.
*/
static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
unsigned long filepages, unsigned long headroom)
{
struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);
unsigned long clean = filepages - min(filepages, mdtc->dirty);
unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
unsigned long other_clean = global_clean - min(global_clean, clean);
mdtc->avail = filepages + min(headroom, other_clean);
}
/**
* __wb_calc_thresh - @wb's share of dirty throttling threshold
* @dtc: dirty_throttle_context of interest
*
* Note that balance_dirty_pages() will only seriously take it as a hard limit
* when sleeping max_pause per page is not enough to keep the dirty pages under
* control. For example, when the device is completely stalled due to some error
* conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
* In the other normal situations, it acts more gently by throttling the tasks
* more (rather than completely block them) when the wb dirty pages go high.
*
* It allocates high/low dirty limits to fast/slow devices, in order to prevent
* - starving fast devices
* - piling up dirty pages (that will take long time to sync) on slow devices
*
* The wb's share of dirty limit will be adapting to its throughput and
* bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
*
* Return: @wb's dirty limit in pages. The term "dirty" in the context of
* dirty balancing includes all PG_dirty and PG_writeback pages.
*/
static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc)
{
struct wb_domain *dom = dtc_dom(dtc);
unsigned long thresh = dtc->thresh;
u64 wb_thresh;
unsigned long numerator, denominator;
unsigned long wb_min_ratio, wb_max_ratio;
/*
* Calculate this BDI's share of the thresh ratio.
*/
fprop_fraction_percpu(&dom->completions, dtc->wb_completions,
&numerator, &denominator);
wb_thresh = (thresh * (100 * BDI_RATIO_SCALE - bdi_min_ratio)) / (100 * BDI_RATIO_SCALE);
wb_thresh *= numerator;
wb_thresh = div64_ul(wb_thresh, denominator);
wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio);
wb_thresh += (thresh * wb_min_ratio) / (100 * BDI_RATIO_SCALE);
if (wb_thresh > (thresh * wb_max_ratio) / (100 * BDI_RATIO_SCALE))
wb_thresh = thresh * wb_max_ratio / (100 * BDI_RATIO_SCALE);
return wb_thresh;
}
unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
{
struct dirty_throttle_control gdtc = { GDTC_INIT(wb),
.thresh = thresh };
return __wb_calc_thresh(&gdtc);
}
/*
* setpoint - dirty 3
* f(dirty) := 1.0 + (----------------)
* limit - setpoint
*
* it's a 3rd order polynomial that subjects to
*
* (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast
* (2) f(setpoint) = 1.0 => the balance point
* (3) f(limit) = 0 => the hard limit
* (4) df/dx <= 0 => negative feedback control
* (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
* => fast response on large errors; small oscillation near setpoint
*/
static long long pos_ratio_polynom(unsigned long setpoint,
unsigned long dirty,
unsigned long limit)
{
long long pos_ratio;
long x;
x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,
(limit - setpoint) | 1);
pos_ratio = x;
pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
}
/*
* Dirty position control.
*
* (o) global/bdi setpoints
*
* We want the dirty pages be balanced around the global/wb setpoints.
* When the number of dirty pages is higher/lower than the setpoint, the
* dirty position control ratio (and hence task dirty ratelimit) will be
* decreased/increased to bring the dirty pages back to the setpoint.
*
* pos_ratio = 1 << RATELIMIT_CALC_SHIFT
*
* if (dirty < setpoint) scale up pos_ratio
* if (dirty > setpoint) scale down pos_ratio
*
* if (wb_dirty < wb_setpoint) scale up pos_ratio
* if (wb_dirty > wb_setpoint) scale down pos_ratio
*
* task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
*
* (o) global control line
*
* ^ pos_ratio
* |
* | |<===== global dirty control scope ======>|
* 2.0 * * * * * * *
* | .*
* | . *
* | . *
* | . *
* | . *
* | . *
* 1.0 ................................*
* | . . *
* | . . *
* | . . *
* | . . *
* | . . *
* 0 +------------.------------------.----------------------*------------->
* freerun^ setpoint^ limit^ dirty pages
*
* (o) wb control line
*
* ^ pos_ratio
* |
* | *
* | *
* | *
* | *
* | * |<=========== span ============>|
* 1.0 .......................*
* | . *
* | . *
* | . *
* | . *
* | . *
* | . *
* | . *
* | . *
* | . *
* | . *
* | . *
* 1/4 ...............................................* * * * * * * * * * * *
* | . .
* | . .
* | . .
* 0 +----------------------.-------------------------------.------------->
* wb_setpoint^ x_intercept^
*
* The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can
* be smoothly throttled down to normal if it starts high in situations like
* - start writing to a slow SD card and a fast disk at the same time. The SD
* card's wb_dirty may rush to many times higher than wb_setpoint.
* - the wb dirty thresh drops quickly due to change of JBOD workload
*/
static void wb_position_ratio(struct dirty_throttle_control *dtc)
{
struct bdi_writeback *wb = dtc->wb;
unsigned long write_bw = READ_ONCE(wb->avg_write_bandwidth);
unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
unsigned long wb_thresh = dtc->wb_thresh;
unsigned long x_intercept;
unsigned long setpoint; /* dirty pages' target balance point */
unsigned long wb_setpoint;
unsigned long span;
long long pos_ratio; /* for scaling up/down the rate limit */
long x;
dtc->pos_ratio = 0;
if (unlikely(dtc->dirty >= limit))
return;
/*
* global setpoint
*
* See comment for pos_ratio_polynom().
*/
setpoint = (freerun + limit) / 2;
pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);
/*
* The strictlimit feature is a tool preventing mistrusted filesystems
* from growing a large number of dirty pages before throttling. For
* such filesystems balance_dirty_pages always checks wb counters
* against wb limits. Even if global "nr_dirty" is under "freerun".
* This is especially important for fuse which sets bdi->max_ratio to
* 1% by default. Without strictlimit feature, fuse writeback may
* consume arbitrary amount of RAM because it is accounted in
* NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty".
*
* Here, in wb_position_ratio(), we calculate pos_ratio based on
* two values: wb_dirty and wb_thresh. Let's consider an example:
* total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global
* limits are set by default to 10% and 20% (background and throttle).
* Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages.
* wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is
* about ~6K pages (as the average of background and throttle wb
* limits). The 3rd order polynomial will provide positive feedback if
* wb_dirty is under wb_setpoint and vice versa.
*
* Note, that we cannot use global counters in these calculations
* because we want to throttle process writing to a strictlimit wb
* much earlier than global "freerun" is reached (~23MB vs. ~2.3GB
* in the example above).
*/
if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
long long wb_pos_ratio;
if (dtc->wb_dirty < 8) {
dtc->pos_ratio = min_t(long long, pos_ratio * 2,
2 << RATELIMIT_CALC_SHIFT);
return;
}
if (dtc->wb_dirty >= wb_thresh)
return;
wb_setpoint = dirty_freerun_ceiling(wb_thresh,
dtc->wb_bg_thresh);
if (wb_setpoint == 0 || wb_setpoint == wb_thresh)
return;
wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,
wb_thresh);
/*
* Typically, for strictlimit case, wb_setpoint << setpoint
* and pos_ratio >> wb_pos_ratio. In the other words global
* state ("dirty") is not limiting factor and we have to
* make decision based on wb counters. But there is an
* important case when global pos_ratio should get precedence:
* global limits are exceeded (e.g. due to activities on other
* wb's) while given strictlimit wb is below limit.
*
* "pos_ratio * wb_pos_ratio" would work for the case above,
* but it would look too non-natural for the case of all
* activity in the system coming from a single strictlimit wb
* with bdi->max_ratio == 100%.
*
* Note that min() below somewhat changes the dynamics of the
* control system. Normally, pos_ratio value can be well over 3
* (when globally we are at freerun and wb is well below wb
* setpoint). Now the maximum pos_ratio in the same situation
* is 2. We might want to tweak this if we observe the control
* system is too slow to adapt.
*/
dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
return;
}
/*
* We have computed basic pos_ratio above based on global situation. If
* the wb is over/under its share of dirty pages, we want to scale
* pos_ratio further down/up. That is done by the following mechanism.
*/
/*
* wb setpoint
*
* f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint)
*
* x_intercept - wb_dirty
* := --------------------------
* x_intercept - wb_setpoint
*
* The main wb control line is a linear function that subjects to
*
* (1) f(wb_setpoint) = 1.0
* (2) k = - 1 / (8 * write_bw) (in single wb case)
* or equally: x_intercept = wb_setpoint + 8 * write_bw
*
* For single wb case, the dirty pages are observed to fluctuate
* regularly within range
* [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2]
* for various filesystems, where (2) can yield in a reasonable 12.5%
* fluctuation range for pos_ratio.
*
* For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its
* own size, so move the slope over accordingly and choose a slope that
* yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh.
*/
if (unlikely(wb_thresh > dtc->thresh))
wb_thresh = dtc->thresh;
/*
* It's very possible that wb_thresh is close to 0 not because the
* device is slow, but that it has remained inactive for long time.
* Honour such devices a reasonable good (hopefully IO efficient)
* threshold, so that the occasional writes won't be blocked and active
* writes can rampup the threshold quickly.
*/
wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8);
/*
* scale global setpoint to wb's:
* wb_setpoint = setpoint * wb_thresh / thresh
*/
x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);
wb_setpoint = setpoint * (u64)x >> 16;
/*
* Use span=(8*write_bw) in single wb case as indicated by
* (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case.
*
* wb_thresh thresh - wb_thresh
* span = --------- * (8 * write_bw) + ------------------ * wb_thresh
* thresh thresh
*/
span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;
x_intercept = wb_setpoint + span;
if (dtc->wb_dirty < x_intercept - span / 4) {
pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),
(x_intercept - wb_setpoint) | 1);
} else
pos_ratio /= 4;
/*
* wb reserve area, safeguard against dirty pool underrun and disk idle
* It may push the desired control point of global dirty pages higher
* than setpoint.
*/
x_intercept = wb_thresh / 2;
if (dtc->wb_dirty < x_intercept) {
if (dtc->wb_dirty > x_intercept / 8)
pos_ratio = div_u64(pos_ratio * x_intercept,
dtc->wb_dirty);
else
pos_ratio *= 8;
}
dtc->pos_ratio = pos_ratio;
}
static void wb_update_write_bandwidth(struct bdi_writeback *wb,
unsigned long elapsed,
unsigned long written)
{
const unsigned long period = roundup_pow_of_two(3 * HZ);
unsigned long avg = wb->avg_write_bandwidth;
unsigned long old = wb->write_bandwidth;
u64 bw;
/*
* bw = written * HZ / elapsed
*
* bw * elapsed + write_bandwidth * (period - elapsed)
* write_bandwidth = ---------------------------------------------------
* period
*
* @written may have decreased due to folio_redirty_for_writepage().
* Avoid underflowing @bw calculation.
*/
bw = written - min(written, wb->written_stamp);
bw *= HZ;
if (unlikely(elapsed > period)) {
bw = div64_ul(bw, elapsed);
avg = bw;
goto out;
}
bw += (u64)wb->write_bandwidth * (period - elapsed);
bw >>= ilog2(period);
/*
* one more level of smoothing, for filtering out sudden spikes
*/
if (avg > old && old >= (unsigned long)bw)
avg -= (avg - old) >> 3;
if (avg < old && old <= (unsigned long)bw)
avg += (old - avg) >> 3;
out:
/* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */
avg = max(avg, 1LU);
if (wb_has_dirty_io(wb)) {
long delta = avg - wb->avg_write_bandwidth;
WARN_ON_ONCE(atomic_long_add_return(delta,
&wb->bdi->tot_write_bandwidth) <= 0);
}
wb->write_bandwidth = bw;
WRITE_ONCE(wb->avg_write_bandwidth, avg);
}
static void update_dirty_limit(struct dirty_throttle_control *dtc)
{
struct wb_domain *dom = dtc_dom(dtc);
unsigned long thresh = dtc->thresh;
unsigned long limit = dom->dirty_limit;
/*
* Follow up in one step.
*/
if (limit < thresh) {
limit = thresh;
goto update;
}
/*
* Follow down slowly. Use the higher one as the target, because thresh
* may drop below dirty. This is exactly the reason to introduce
* dom->dirty_limit which is guaranteed to lie above the dirty pages.
*/
thresh = max(thresh, dtc->dirty);
if (limit > thresh) {
limit -= (limit - thresh) >> 5;
goto update;
}
return;
update:
dom->dirty_limit = limit;
}
static void domain_update_dirty_limit(struct dirty_throttle_control *dtc,
unsigned long now)
{
struct wb_domain *dom = dtc_dom(dtc);
/*
* check locklessly first to optimize away locking for the most time
*/
if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL))
return;
spin_lock(&dom->lock);
if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) {
update_dirty_limit(dtc);
dom->dirty_limit_tstamp = now;
}
spin_unlock(&dom->lock);
}
/*
* Maintain wb->dirty_ratelimit, the base dirty throttle rate.
*
* Normal wb tasks will be curbed at or below it in long term.
* Obviously it should be around (write_bw / N) when there are N dd tasks.
*/
static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,
unsigned long dirtied,
unsigned long elapsed)
{
struct bdi_writeback *wb = dtc->wb;
unsigned long dirty = dtc->dirty;
unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
unsigned long setpoint = (freerun + limit) / 2;
unsigned long write_bw = wb->avg_write_bandwidth;
unsigned long dirty_ratelimit = wb->dirty_ratelimit;
unsigned long dirty_rate;
unsigned long task_ratelimit;
unsigned long balanced_dirty_ratelimit;
unsigned long step;
unsigned long x;
unsigned long shift;
/*
* The dirty rate will match the writeout rate in long term, except
* when dirty pages are truncated by userspace or re-dirtied by FS.
*/
dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;
/*
* task_ratelimit reflects each dd's dirty rate for the past 200ms.
*/
task_ratelimit = (u64)dirty_ratelimit *
dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;
task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */
/*
* A linear estimation of the "balanced" throttle rate. The theory is,
* if there are N dd tasks, each throttled at task_ratelimit, the wb's
* dirty_rate will be measured to be (N * task_ratelimit). So the below
* formula will yield the balanced rate limit (write_bw / N).
*
* Note that the expanded form is not a pure rate feedback:
* rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1)
* but also takes pos_ratio into account:
* rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2)
*
* (1) is not realistic because pos_ratio also takes part in balancing
* the dirty rate. Consider the state
* pos_ratio = 0.5 (3)
* rate = 2 * (write_bw / N) (4)
* If (1) is used, it will stuck in that state! Because each dd will
* be throttled at
* task_ratelimit = pos_ratio * rate = (write_bw / N) (5)
* yielding
* dirty_rate = N * task_ratelimit = write_bw (6)
* put (6) into (1) we get
* rate_(i+1) = rate_(i) (7)
*
* So we end up using (2) to always keep
* rate_(i+1) ~= (write_bw / N) (8)
* regardless of the value of pos_ratio. As long as (8) is satisfied,
* pos_ratio is able to drive itself to 1.0, which is not only where
* the dirty count meet the setpoint, but also where the slope of
* pos_ratio is most flat and hence task_ratelimit is least fluctuated.
*/
balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
dirty_rate | 1);
/*
* balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
*/
if (unlikely(balanced_dirty_ratelimit > write_bw))
balanced_dirty_ratelimit = write_bw;
/*
* We could safely do this and return immediately:
*
* wb->dirty_ratelimit = balanced_dirty_ratelimit;
*
* However to get a more stable dirty_ratelimit, the below elaborated
* code makes use of task_ratelimit to filter out singular points and
* limit the step size.
*
* The below code essentially only uses the relative value of
*
* task_ratelimit - dirty_ratelimit
* = (pos_ratio - 1) * dirty_ratelimit
*
* which reflects the direction and size of dirty position error.
*/
/*
* dirty_ratelimit will follow balanced_dirty_ratelimit iff
* task_ratelimit is on the same side of dirty_ratelimit, too.
* For example, when
* - dirty_ratelimit > balanced_dirty_ratelimit
* - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
* lowering dirty_ratelimit will help meet both the position and rate
* control targets. Otherwise, don't update dirty_ratelimit if it will
* only help meet the rate target. After all, what the users ultimately
* feel and care are stable dirty rate and small position error.
*
* |task_ratelimit - dirty_ratelimit| is used to limit the step size
* and filter out the singular points of balanced_dirty_ratelimit. Which
* keeps jumping around randomly and can even leap far away at times
* due to the small 200ms estimation period of dirty_rate (we want to
* keep that period small to reduce time lags).
*/
step = 0;
/*
* For strictlimit case, calculations above were based on wb counters
* and limits (starting from pos_ratio = wb_position_ratio() and up to
* balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate).
* Hence, to calculate "step" properly, we have to use wb_dirty as
* "dirty" and wb_setpoint as "setpoint".
*
* We rampup dirty_ratelimit forcibly if wb_dirty is low because
* it's possible that wb_thresh is close to zero due to inactivity
* of backing device.
*/
if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
dirty = dtc->wb_dirty;
if (dtc->wb_dirty < 8)
setpoint = dtc->wb_dirty + 1;
else
setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;
}
if (dirty < setpoint) {
x = min3(wb->balanced_dirty_ratelimit,
balanced_dirty_ratelimit, task_ratelimit);
if (dirty_ratelimit < x)
step = x - dirty_ratelimit;
} else {
x = max3(wb->balanced_dirty_ratelimit,
balanced_dirty_ratelimit, task_ratelimit);
if (dirty_ratelimit > x)
step = dirty_ratelimit - x;
}
/*
* Don't pursue 100% rate matching. It's impossible since the balanced
* rate itself is constantly fluctuating. So decrease the track speed
* when it gets close to the target. Helps eliminate pointless tremors.
*/
shift = dirty_ratelimit / (2 * step + 1);
if (shift < BITS_PER_LONG)
step = DIV_ROUND_UP(step >> shift, 8);
else
step = 0;
if (dirty_ratelimit < balanced_dirty_ratelimit)
dirty_ratelimit += step;
else
dirty_ratelimit -= step;
WRITE_ONCE(wb->dirty_ratelimit, max(dirty_ratelimit, 1UL));
wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;
trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);
}
static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc,
struct dirty_throttle_control *mdtc,
bool update_ratelimit)
{
struct bdi_writeback *wb = gdtc->wb;
unsigned long now = jiffies;
unsigned long elapsed;
unsigned long dirtied;
unsigned long written;
spin_lock(&wb->list_lock);
/*
* Lockless checks for elapsed time are racy and delayed update after
* IO completion doesn't do it at all (to make sure written pages are
* accounted reasonably quickly). Make sure elapsed >= 1 to avoid
* division errors.
*/
elapsed = max(now - wb->bw_time_stamp, 1UL);
dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]);
written = percpu_counter_read(&wb->stat[WB_WRITTEN]);
if (update_ratelimit) {
domain_update_dirty_limit(gdtc, now);
wb_update_dirty_ratelimit(gdtc, dirtied, elapsed);
/*
* @mdtc is always NULL if !CGROUP_WRITEBACK but the
* compiler has no way to figure that out. Help it.
*/
if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) {
domain_update_dirty_limit(mdtc, now);
wb_update_dirty_ratelimit(mdtc, dirtied, elapsed);
}
}
wb_update_write_bandwidth(wb, elapsed, written);
wb->dirtied_stamp = dirtied;
wb->written_stamp = written;
WRITE_ONCE(wb->bw_time_stamp, now);
spin_unlock(&wb->list_lock);
}
void wb_update_bandwidth(struct bdi_writeback *wb)
{
struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
__wb_update_bandwidth(&gdtc, NULL, false);
}
/* Interval after which we consider wb idle and don't estimate bandwidth */
#define WB_BANDWIDTH_IDLE_JIF (HZ)
static void wb_bandwidth_estimate_start(struct bdi_writeback *wb)
{
unsigned long now = jiffies;
unsigned long elapsed = now - READ_ONCE(wb->bw_time_stamp);
if (elapsed > WB_BANDWIDTH_IDLE_JIF &&
!atomic_read(&wb->writeback_inodes)) {
spin_lock(&wb->list_lock);
wb->dirtied_stamp = wb_stat(wb, WB_DIRTIED);
wb->written_stamp = wb_stat(wb, WB_WRITTEN);
WRITE_ONCE(wb->bw_time_stamp, now);
spin_unlock(&wb->list_lock);
}
}
/*
* After a task dirtied this many pages, balance_dirty_pages_ratelimited()
* will look to see if it needs to start dirty throttling.
*
* If dirty_poll_interval is too low, big NUMA machines will call the expensive
* global_zone_page_state() too often. So scale it near-sqrt to the safety margin
* (the number of pages we may dirty without exceeding the dirty limits).
*/
static unsigned long dirty_poll_interval(unsigned long dirty,
unsigned long thresh)
{
if (thresh > dirty)
return 1UL << (ilog2(thresh - dirty) >> 1);
return 1;
}
static unsigned long wb_max_pause(struct bdi_writeback *wb,
unsigned long wb_dirty)
{
unsigned long bw = READ_ONCE(wb->avg_write_bandwidth);
unsigned long t;
/*
* Limit pause time for small memory systems. If sleeping for too long
* time, a small pool of dirty/writeback pages may go empty and disk go
* idle.
*
* 8 serves as the safety ratio.
*/
t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
t++;
return min_t(unsigned long, t, MAX_PAUSE);
}
static long wb_min_pause(struct bdi_writeback *wb,
long max_pause,
unsigned long task_ratelimit,
unsigned long dirty_ratelimit,
int *nr_dirtied_pause)
{
long hi = ilog2(READ_ONCE(wb->avg_write_bandwidth));
long lo = ilog2(READ_ONCE(wb->dirty_ratelimit));
long t; /* target pause */
long pause; /* estimated next pause */
int pages; /* target nr_dirtied_pause */
/* target for 10ms pause on 1-dd case */
t = max(1, HZ / 100);
/*
* Scale up pause time for concurrent dirtiers in order to reduce CPU
* overheads.
*
* (N * 10ms) on 2^N concurrent tasks.
*/
if (hi > lo)
t += (hi - lo) * (10 * HZ) / 1024;
/*
* This is a bit convoluted. We try to base the next nr_dirtied_pause
* on the much more stable dirty_ratelimit. However the next pause time
* will be computed based on task_ratelimit and the two rate limits may
* depart considerably at some time. Especially if task_ratelimit goes
* below dirty_ratelimit/2 and the target pause is max_pause, the next
* pause time will be max_pause*2 _trimmed down_ to max_pause. As a
* result task_ratelimit won't be executed faithfully, which could
* eventually bring down dirty_ratelimit.
*
* We apply two rules to fix it up:
* 1) try to estimate the next pause time and if necessary, use a lower
* nr_dirtied_pause so as not to exceed max_pause. When this happens,
* nr_dirtied_pause will be "dancing" with task_ratelimit.
* 2) limit the target pause time to max_pause/2, so that the normal
* small fluctuations of task_ratelimit won't trigger rule (1) and
* nr_dirtied_pause will remain as stable as dirty_ratelimit.
*/
t = min(t, 1 + max_pause / 2);
pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
/*
* Tiny nr_dirtied_pause is found to hurt I/O performance in the test
* case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
* When the 16 consecutive reads are often interrupted by some dirty
* throttling pause during the async writes, cfq will go into idles
* (deadline is fine). So push nr_dirtied_pause as high as possible
* until reaches DIRTY_POLL_THRESH=32 pages.
*/
if (pages < DIRTY_POLL_THRESH) {
t = max_pause;
pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
if (pages > DIRTY_POLL_THRESH) {
pages = DIRTY_POLL_THRESH;
t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
}
}
pause = HZ * pages / (task_ratelimit + 1);
if (pause > max_pause) {
t = max_pause;
pages = task_ratelimit * t / roundup_pow_of_two(HZ);
}
*nr_dirtied_pause = pages;
/*
* The minimal pause time will normally be half the target pause time.
*/
return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
}
static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)
{
struct bdi_writeback *wb = dtc->wb;
unsigned long wb_reclaimable;
/*
* wb_thresh is not treated as some limiting factor as
* dirty_thresh, due to reasons
* - in JBOD setup, wb_thresh can fluctuate a lot
* - in a system with HDD and USB key, the USB key may somehow
* go into state (wb_dirty >> wb_thresh) either because
* wb_dirty starts high, or because wb_thresh drops low.
* In this case we don't want to hard throttle the USB key
* dirtiers for 100 seconds until wb_dirty drops under
* wb_thresh. Instead the auxiliary wb control line in
* wb_position_ratio() will let the dirtier task progress
* at some rate <= (write_bw / 2) for bringing down wb_dirty.
*/
dtc->wb_thresh = __wb_calc_thresh(dtc);
dtc->wb_bg_thresh = dtc->thresh ?
div64_u64(dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0;
/*
* In order to avoid the stacked BDI deadlock we need
* to ensure we accurately count the 'dirty' pages when
* the threshold is low.
*
* Otherwise it would be possible to get thresh+n pages
* reported dirty, even though there are thresh-m pages
* actually dirty; with m+n sitting in the percpu
* deltas.
*/
if (dtc->wb_thresh < 2 * wb_stat_error()) {
wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK);
} else {
wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE);
dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK);
}
}
/*
* balance_dirty_pages() must be called by processes which are generating dirty
* data. It looks at the number of dirty pages in the machine and will force
* the caller to wait once crossing the (background_thresh + dirty_thresh) / 2.
* If we're over `background_thresh' then the writeback threads are woken to
* perform some writeout.
*/
static int balance_dirty_pages(struct bdi_writeback *wb,
unsigned long pages_dirtied, unsigned int flags)
{
struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
struct dirty_throttle_control * const gdtc = &gdtc_stor;
struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
&mdtc_stor : NULL;
struct dirty_throttle_control *sdtc;
unsigned long nr_reclaimable; /* = file_dirty */
long period;
long pause;
long max_pause;
long min_pause;
int nr_dirtied_pause;
bool dirty_exceeded = false;
unsigned long task_ratelimit;
unsigned long dirty_ratelimit;
struct backing_dev_info *bdi = wb->bdi;
bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT;
unsigned long start_time = jiffies;
int ret = 0;
for (;;) {
unsigned long now = jiffies;
unsigned long dirty, thresh, bg_thresh;
unsigned long m_dirty = 0; /* stop bogus uninit warnings */
unsigned long m_thresh = 0;
unsigned long m_bg_thresh = 0;
nr_reclaimable = global_node_page_state(NR_FILE_DIRTY);
gdtc->avail = global_dirtyable_memory();
gdtc->dirty = nr_reclaimable + global_node_page_state(NR_WRITEBACK);
domain_dirty_limits(gdtc);
if (unlikely(strictlimit)) {
wb_dirty_limits(gdtc);
dirty = gdtc->wb_dirty;
thresh = gdtc->wb_thresh;
bg_thresh = gdtc->wb_bg_thresh;
} else {
dirty = gdtc->dirty;
thresh = gdtc->thresh;
bg_thresh = gdtc->bg_thresh;
}
if (mdtc) {
unsigned long filepages, headroom, writeback;
/*
* If @wb belongs to !root memcg, repeat the same
* basic calculations for the memcg domain.
*/
mem_cgroup_wb_stats(wb, &filepages, &headroom,
&mdtc->dirty, &writeback);
mdtc->dirty += writeback;
mdtc_calc_avail(mdtc, filepages, headroom);
domain_dirty_limits(mdtc);
if (unlikely(strictlimit)) {
wb_dirty_limits(mdtc);
m_dirty = mdtc->wb_dirty;
m_thresh = mdtc->wb_thresh;
m_bg_thresh = mdtc->wb_bg_thresh;
} else {
m_dirty = mdtc->dirty;
m_thresh = mdtc->thresh;
m_bg_thresh = mdtc->bg_thresh;
}
}
/*
* In laptop mode, we wait until hitting the higher threshold
* before starting background writeout, and then write out all
* the way down to the lower threshold. So slow writers cause
* minimal disk activity.
*
* In normal mode, we start background writeout at the lower
* background_thresh, to keep the amount of dirty memory low.
*/
if (!laptop_mode && nr_reclaimable > gdtc->bg_thresh &&
!writeback_in_progress(wb))
wb_start_background_writeback(wb);
/*
* Throttle it only when the background writeback cannot
* catch-up. This avoids (excessively) small writeouts
* when the wb limits are ramping up in case of !strictlimit.
*
* In strictlimit case make decision based on the wb counters
* and limits. Small writeouts when the wb limits are ramping
* up are the price we consciously pay for strictlimit-ing.
*
* If memcg domain is in effect, @dirty should be under
* both global and memcg freerun ceilings.
*/
if (dirty <= dirty_freerun_ceiling(thresh, bg_thresh) &&
(!mdtc ||
m_dirty <= dirty_freerun_ceiling(m_thresh, m_bg_thresh))) {
unsigned long intv;
unsigned long m_intv;
free_running:
intv = dirty_poll_interval(dirty, thresh);
m_intv = ULONG_MAX;
current->dirty_paused_when = now;
current->nr_dirtied = 0;
if (mdtc)
m_intv = dirty_poll_interval(m_dirty, m_thresh);
current->nr_dirtied_pause = min(intv, m_intv);
break;
}
/* Start writeback even when in laptop mode */
if (unlikely(!writeback_in_progress(wb)))
wb_start_background_writeback(wb);
mem_cgroup_flush_foreign(wb);
/*
* Calculate global domain's pos_ratio and select the
* global dtc by default.
*/
if (!strictlimit) {
wb_dirty_limits(gdtc);
if ((current->flags & PF_LOCAL_THROTTLE) &&
gdtc->wb_dirty <
dirty_freerun_ceiling(gdtc->wb_thresh,
gdtc->wb_bg_thresh))
/*
* LOCAL_THROTTLE tasks must not be throttled
* when below the per-wb freerun ceiling.
*/
goto free_running;
}
dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) &&
((gdtc->dirty > gdtc->thresh) || strictlimit);
wb_position_ratio(gdtc);
sdtc = gdtc;
if (mdtc) {
/*
* If memcg domain is in effect, calculate its
* pos_ratio. @wb should satisfy constraints from
* both global and memcg domains. Choose the one
* w/ lower pos_ratio.
*/
if (!strictlimit) {
wb_dirty_limits(mdtc);
if ((current->flags & PF_LOCAL_THROTTLE) &&
mdtc->wb_dirty <
dirty_freerun_ceiling(mdtc->wb_thresh,
mdtc->wb_bg_thresh))
/*
* LOCAL_THROTTLE tasks must not be
* throttled when below the per-wb
* freerun ceiling.
*/
goto free_running;
}
dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) &&
((mdtc->dirty > mdtc->thresh) || strictlimit);
wb_position_ratio(mdtc);
if (mdtc->pos_ratio < gdtc->pos_ratio)
sdtc = mdtc;
}
if (dirty_exceeded != wb->dirty_exceeded)
wb->dirty_exceeded = dirty_exceeded;
if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
BANDWIDTH_INTERVAL))
__wb_update_bandwidth(gdtc, mdtc, true);
/* throttle according to the chosen dtc */
dirty_ratelimit = READ_ONCE(wb->dirty_ratelimit);
task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >>
RATELIMIT_CALC_SHIFT;
max_pause = wb_max_pause(wb, sdtc->wb_dirty);
min_pause = wb_min_pause(wb, max_pause,
task_ratelimit, dirty_ratelimit,
&nr_dirtied_pause);
if (unlikely(task_ratelimit == 0)) {
period = max_pause;
pause = max_pause;
goto pause;
}
period = HZ * pages_dirtied / task_ratelimit;
pause = period;
if (current->dirty_paused_when)
pause -= now - current->dirty_paused_when;
/*
* For less than 1s think time (ext3/4 may block the dirtier
* for up to 800ms from time to time on 1-HDD; so does xfs,
* however at much less frequency), try to compensate it in
* future periods by updating the virtual time; otherwise just
* do a reset, as it may be a light dirtier.
*/
if (pause < min_pause) {
trace_balance_dirty_pages(wb,
sdtc->thresh,
sdtc->bg_thresh,
sdtc->dirty,
sdtc->wb_thresh,
sdtc->wb_dirty,
dirty_ratelimit,
task_ratelimit,
pages_dirtied,
period,
min(pause, 0L),
start_time);
if (pause < -HZ) {
current->dirty_paused_when = now;
current->nr_dirtied = 0;
} else if (period) {
current->dirty_paused_when += period;
current->nr_dirtied = 0;
} else if (current->nr_dirtied_pause <= pages_dirtied)
current->nr_dirtied_pause += pages_dirtied;
break;
}
if (unlikely(pause > max_pause)) {
/* for occasional dropped task_ratelimit */
now += min(pause - max_pause, max_pause);
pause = max_pause;
}
pause:
trace_balance_dirty_pages(wb,
sdtc->thresh,
sdtc->bg_thresh,
sdtc->dirty,
sdtc->wb_thresh,
sdtc->wb_dirty,
dirty_ratelimit,
task_ratelimit,
pages_dirtied,
period,
pause,
start_time);
if (flags & BDP_ASYNC) {
ret = -EAGAIN;
break;
}
__set_current_state(TASK_KILLABLE);
bdi->last_bdp_sleep = jiffies;
io_schedule_timeout(pause);
current->dirty_paused_when = now + pause;
current->nr_dirtied = 0;
current->nr_dirtied_pause = nr_dirtied_pause;
/*
* This is typically equal to (dirty < thresh) and can also
* keep "1000+ dd on a slow USB stick" under control.
*/
if (task_ratelimit)
break;
/*
* In the case of an unresponsive NFS server and the NFS dirty
* pages exceeds dirty_thresh, give the other good wb's a pipe
* to go through, so that tasks on them still remain responsive.
*
* In theory 1 page is enough to keep the consumer-producer
* pipe going: the flusher cleans 1 page => the task dirties 1
* more page. However wb_dirty has accounting errors. So use
* the larger and more IO friendly wb_stat_error.
*/
if (sdtc->wb_dirty <= wb_stat_error())
break;
if (fatal_signal_pending(current))
break;
}
return ret;
}
static DEFINE_PER_CPU(int, bdp_ratelimits);
/*
* Normal tasks are throttled by
* loop {
* dirty tsk->nr_dirtied_pause pages;
* take a snap in balance_dirty_pages();
* }
* However there is a worst case. If every task exit immediately when dirtied
* (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
* called to throttle the page dirties. The solution is to save the not yet
* throttled page dirties in dirty_throttle_leaks on task exit and charge them
* randomly into the running tasks. This works well for the above worst case,
* as the new task will pick up and accumulate the old task's leaked dirty
* count and eventually get throttled.
*/
DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
/**
* balance_dirty_pages_ratelimited_flags - Balance dirty memory state.
* @mapping: address_space which was dirtied.
* @flags: BDP flags.
*
* Processes which are dirtying memory should call in here once for each page
* which was newly dirtied. The function will periodically check the system's
* dirty state and will initiate writeback if needed.
*
* See balance_dirty_pages_ratelimited() for details.
*
* Return: If @flags contains BDP_ASYNC, it may return -EAGAIN to
* indicate that memory is out of balance and the caller must wait
* for I/O to complete. Otherwise, it will return 0 to indicate
* that either memory was already in balance, or it was able to sleep
* until the amount of dirty memory returned to balance.
*/
int balance_dirty_pages_ratelimited_flags(struct address_space *mapping,
unsigned int flags)
{
struct inode *inode = mapping->host;
struct backing_dev_info *bdi = inode_to_bdi(inode);
struct bdi_writeback *wb = NULL;
int ratelimit;
int ret = 0;
int *p;
if (!(bdi->capabilities & BDI_CAP_WRITEBACK))
return ret;
if (inode_cgwb_enabled(inode)) wb = wb_get_create_current(bdi, GFP_KERNEL); if (!wb) wb = &bdi->wb; ratelimit = current->nr_dirtied_pause;
if (wb->dirty_exceeded)
ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); preempt_disable();
/*
* This prevents one CPU to accumulate too many dirtied pages without
* calling into balance_dirty_pages(), which can happen when there are
* 1000+ tasks, all of them start dirtying pages at exactly the same
* time, hence all honoured too large initial task->nr_dirtied_pause.
*/
p = this_cpu_ptr(&bdp_ratelimits);
if (unlikely(current->nr_dirtied >= ratelimit))
*p = 0; else if (unlikely(*p >= ratelimit_pages)) { *p = 0;
ratelimit = 0;
}
/*
* Pick up the dirtied pages by the exited tasks. This avoids lots of
* short-lived tasks (eg. gcc invocations in a kernel build) escaping
* the dirty throttling and livelock other long-run dirtiers.
*/
p = this_cpu_ptr(&dirty_throttle_leaks); if (*p > 0 && current->nr_dirtied < ratelimit) {
unsigned long nr_pages_dirtied;
nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
*p -= nr_pages_dirtied;
current->nr_dirtied += nr_pages_dirtied;
}
preempt_enable(); if (unlikely(current->nr_dirtied >= ratelimit)) ret = balance_dirty_pages(wb, current->nr_dirtied, flags); wb_put(wb);
return ret;
}
EXPORT_SYMBOL_GPL(balance_dirty_pages_ratelimited_flags);
/**
* balance_dirty_pages_ratelimited - balance dirty memory state.
* @mapping: address_space which was dirtied.
*
* Processes which are dirtying memory should call in here once for each page
* which was newly dirtied. The function will periodically check the system's
* dirty state and will initiate writeback if needed.
*
* Once we're over the dirty memory limit we decrease the ratelimiting
* by a lot, to prevent individual processes from overshooting the limit
* by (ratelimit_pages) each.
*/
void balance_dirty_pages_ratelimited(struct address_space *mapping)
{
balance_dirty_pages_ratelimited_flags(mapping, 0);
}
EXPORT_SYMBOL(balance_dirty_pages_ratelimited);
/**
* wb_over_bg_thresh - does @wb need to be written back?
* @wb: bdi_writeback of interest
*
* Determines whether background writeback should keep writing @wb or it's
* clean enough.
*
* Return: %true if writeback should continue.
*/
bool wb_over_bg_thresh(struct bdi_writeback *wb)
{
struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
struct dirty_throttle_control * const gdtc = &gdtc_stor;
struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
&mdtc_stor : NULL;
unsigned long reclaimable;
unsigned long thresh;
/*
* Similar to balance_dirty_pages() but ignores pages being written
* as we're trying to decide whether to put more under writeback.
*/
gdtc->avail = global_dirtyable_memory();
gdtc->dirty = global_node_page_state(NR_FILE_DIRTY);
domain_dirty_limits(gdtc);
if (gdtc->dirty > gdtc->bg_thresh)
return true;
thresh = wb_calc_thresh(gdtc->wb, gdtc->bg_thresh);
if (thresh < 2 * wb_stat_error())
reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
else
reclaimable = wb_stat(wb, WB_RECLAIMABLE);
if (reclaimable > thresh)
return true;
if (mdtc) {
unsigned long filepages, headroom, writeback;
mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty,
&writeback);
mdtc_calc_avail(mdtc, filepages, headroom);
domain_dirty_limits(mdtc); /* ditto, ignore writeback */
if (mdtc->dirty > mdtc->bg_thresh)
return true;
thresh = wb_calc_thresh(mdtc->wb, mdtc->bg_thresh);
if (thresh < 2 * wb_stat_error())
reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
else
reclaimable = wb_stat(wb, WB_RECLAIMABLE);
if (reclaimable > thresh)
return true;
}
return false;
}
#ifdef CONFIG_SYSCTL
/*
* sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
*/
static int dirty_writeback_centisecs_handler(struct ctl_table *table, int write,
void *buffer, size_t *length, loff_t *ppos)
{
unsigned int old_interval = dirty_writeback_interval;
int ret;
ret = proc_dointvec(table, write, buffer, length, ppos);
/*
* Writing 0 to dirty_writeback_interval will disable periodic writeback
* and a different non-zero value will wakeup the writeback threads.
* wb_wakeup_delayed() would be more appropriate, but it's a pain to
* iterate over all bdis and wbs.
* The reason we do this is to make the change take effect immediately.
*/
if (!ret && write && dirty_writeback_interval &&
dirty_writeback_interval != old_interval)
wakeup_flusher_threads(WB_REASON_PERIODIC);
return ret;
}
#endif
void laptop_mode_timer_fn(struct timer_list *t)
{
struct backing_dev_info *backing_dev_info =
from_timer(backing_dev_info, t, laptop_mode_wb_timer);
wakeup_flusher_threads_bdi(backing_dev_info, WB_REASON_LAPTOP_TIMER);
}
/*
* We've spun up the disk and we're in laptop mode: schedule writeback
* of all dirty data a few seconds from now. If the flush is already scheduled
* then push it back - the user is still using the disk.
*/
void laptop_io_completion(struct backing_dev_info *info)
{
mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
}
/*
* We're in laptop mode and we've just synced. The sync's writes will have
* caused another writeback to be scheduled by laptop_io_completion.
* Nothing needs to be written back anymore, so we unschedule the writeback.
*/
void laptop_sync_completion(void)
{
struct backing_dev_info *bdi;
rcu_read_lock();
list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
del_timer(&bdi->laptop_mode_wb_timer);
rcu_read_unlock();
}
/*
* If ratelimit_pages is too high then we can get into dirty-data overload
* if a large number of processes all perform writes at the same time.
*
* Here we set ratelimit_pages to a level which ensures that when all CPUs are
* dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
* thresholds.
*/
void writeback_set_ratelimit(void)
{
struct wb_domain *dom = &global_wb_domain;
unsigned long background_thresh;
unsigned long dirty_thresh;
global_dirty_limits(&background_thresh, &dirty_thresh);
dom->dirty_limit = dirty_thresh;
ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
if (ratelimit_pages < 16)
ratelimit_pages = 16;
}
static int page_writeback_cpu_online(unsigned int cpu)
{
writeback_set_ratelimit();
return 0;
}
#ifdef CONFIG_SYSCTL
/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE;
static struct ctl_table vm_page_writeback_sysctls[] = {
{
.procname = "dirty_background_ratio",
.data = &dirty_background_ratio,
.maxlen = sizeof(dirty_background_ratio),
.mode = 0644,
.proc_handler = dirty_background_ratio_handler,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE_HUNDRED,
},
{
.procname = "dirty_background_bytes",
.data = &dirty_background_bytes,
.maxlen = sizeof(dirty_background_bytes),
.mode = 0644,
.proc_handler = dirty_background_bytes_handler,
.extra1 = SYSCTL_LONG_ONE,
},
{
.procname = "dirty_ratio",
.data = &vm_dirty_ratio,
.maxlen = sizeof(vm_dirty_ratio),
.mode = 0644,
.proc_handler = dirty_ratio_handler,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE_HUNDRED,
},
{
.procname = "dirty_bytes",
.data = &vm_dirty_bytes,
.maxlen = sizeof(vm_dirty_bytes),
.mode = 0644,
.proc_handler = dirty_bytes_handler,
.extra1 = (void *)&dirty_bytes_min,
},
{
.procname = "dirty_writeback_centisecs",
.data = &dirty_writeback_interval,
.maxlen = sizeof(dirty_writeback_interval),
.mode = 0644,
.proc_handler = dirty_writeback_centisecs_handler,
},
{
.procname = "dirty_expire_centisecs",
.data = &dirty_expire_interval,
.maxlen = sizeof(dirty_expire_interval),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
},
#ifdef CONFIG_HIGHMEM
{
.procname = "highmem_is_dirtyable",
.data = &vm_highmem_is_dirtyable,
.maxlen = sizeof(vm_highmem_is_dirtyable),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
#endif
{
.procname = "laptop_mode",
.data = &laptop_mode,
.maxlen = sizeof(laptop_mode),
.mode = 0644,
.proc_handler = proc_dointvec_jiffies,
},
{}
};
#endif
/*
* Called early on to tune the page writeback dirty limits.
*
* We used to scale dirty pages according to how total memory
* related to pages that could be allocated for buffers.
*
* However, that was when we used "dirty_ratio" to scale with
* all memory, and we don't do that any more. "dirty_ratio"
* is now applied to total non-HIGHPAGE memory, and as such we can't
* get into the old insane situation any more where we had
* large amounts of dirty pages compared to a small amount of
* non-HIGHMEM memory.
*
* But we might still want to scale the dirty_ratio by how
* much memory the box has..
*/
void __init page_writeback_init(void)
{
BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL));
cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/writeback:online",
page_writeback_cpu_online, NULL);
cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL,
page_writeback_cpu_online);
#ifdef CONFIG_SYSCTL
register_sysctl_init("vm", vm_page_writeback_sysctls);
#endif
}
/**
* tag_pages_for_writeback - tag pages to be written by writeback
* @mapping: address space structure to write
* @start: starting page index
* @end: ending page index (inclusive)
*
* This function scans the page range from @start to @end (inclusive) and tags
* all pages that have DIRTY tag set with a special TOWRITE tag. The caller
* can then use the TOWRITE tag to identify pages eligible for writeback.
* This mechanism is used to avoid livelocking of writeback by a process
* steadily creating new dirty pages in the file (thus it is important for this
* function to be quick so that it can tag pages faster than a dirtying process
* can create them).
*/
void tag_pages_for_writeback(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
XA_STATE(xas, &mapping->i_pages, start);
unsigned int tagged = 0;
void *page;
xas_lock_irq(&xas);
xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) {
xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
if (++tagged % XA_CHECK_SCHED)
continue;
xas_pause(&xas);
xas_unlock_irq(&xas);
cond_resched();
xas_lock_irq(&xas);
}
xas_unlock_irq(&xas);
}
EXPORT_SYMBOL(tag_pages_for_writeback);
static bool folio_prepare_writeback(struct address_space *mapping,
struct writeback_control *wbc, struct folio *folio)
{
/*
* Folio truncated or invalidated. We can freely skip it then,
* even for data integrity operations: the folio has disappeared
* concurrently, so there could be no real expectation of this
* data integrity operation even if there is now a new, dirty
* folio at the same pagecache index.
*/
if (unlikely(folio->mapping != mapping))
return false;
/*
* Did somebody else write it for us?
*/
if (!folio_test_dirty(folio))
return false;
if (folio_test_writeback(folio)) {
if (wbc->sync_mode == WB_SYNC_NONE)
return false;
folio_wait_writeback(folio);
}
BUG_ON(folio_test_writeback(folio));
if (!folio_clear_dirty_for_io(folio))
return false;
return true;
}
static xa_mark_t wbc_to_tag(struct writeback_control *wbc)
{
if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
return PAGECACHE_TAG_TOWRITE;
return PAGECACHE_TAG_DIRTY;
}
static pgoff_t wbc_end(struct writeback_control *wbc)
{
if (wbc->range_cyclic)
return -1;
return wbc->range_end >> PAGE_SHIFT;
}
static struct folio *writeback_get_folio(struct address_space *mapping,
struct writeback_control *wbc)
{
struct folio *folio;
retry:
folio = folio_batch_next(&wbc->fbatch);
if (!folio) {
folio_batch_release(&wbc->fbatch);
cond_resched();
filemap_get_folios_tag(mapping, &wbc->index, wbc_end(wbc),
wbc_to_tag(wbc), &wbc->fbatch);
folio = folio_batch_next(&wbc->fbatch);
if (!folio)
return NULL;
}
folio_lock(folio);
if (unlikely(!folio_prepare_writeback(mapping, wbc, folio))) {
folio_unlock(folio);
goto retry;
}
trace_wbc_writepage(wbc, inode_to_bdi(mapping->host));
return folio;
}
/**
* writeback_iter - iterate folio of a mapping for writeback
* @mapping: address space structure to write
* @wbc: writeback context
* @folio: previously iterated folio (%NULL to start)
* @error: in-out pointer for writeback errors (see below)
*
* This function returns the next folio for the writeback operation described by
* @wbc on @mapping and should be called in a while loop in the ->writepages
* implementation.
*
* To start the writeback operation, %NULL is passed in the @folio argument, and
* for every subsequent iteration the folio returned previously should be passed
* back in.
*
* If there was an error in the per-folio writeback inside the writeback_iter()
* loop, @error should be set to the error value.
*
* Once the writeback described in @wbc has finished, this function will return
* %NULL and if there was an error in any iteration restore it to @error.
*
* Note: callers should not manually break out of the loop using break or goto
* but must keep calling writeback_iter() until it returns %NULL.
*
* Return: the folio to write or %NULL if the loop is done.
*/
struct folio *writeback_iter(struct address_space *mapping,
struct writeback_control *wbc, struct folio *folio, int *error)
{
if (!folio) {
folio_batch_init(&wbc->fbatch);
wbc->saved_err = *error = 0;
/*
* For range cyclic writeback we remember where we stopped so
* that we can continue where we stopped.
*
* For non-cyclic writeback we always start at the beginning of
* the passed in range.
*/
if (wbc->range_cyclic)
wbc->index = mapping->writeback_index;
else
wbc->index = wbc->range_start >> PAGE_SHIFT;
/*
* To avoid livelocks when other processes dirty new pages, we
* first tag pages which should be written back and only then
* start writing them.
*
* For data-integrity writeback we have to be careful so that we
* do not miss some pages (e.g., because some other process has
* cleared the TOWRITE tag we set). The rule we follow is that
* TOWRITE tag can be cleared only by the process clearing the
* DIRTY tag (and submitting the page for I/O).
*/
if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
tag_pages_for_writeback(mapping, wbc->index,
wbc_end(wbc));
} else {
wbc->nr_to_write -= folio_nr_pages(folio);
WARN_ON_ONCE(*error > 0);
/*
* For integrity writeback we have to keep going until we have
* written all the folios we tagged for writeback above, even if
* we run past wbc->nr_to_write or encounter errors.
* We stash away the first error we encounter in wbc->saved_err
* so that it can be retrieved when we're done. This is because
* the file system may still have state to clear for each folio.
*
* For background writeback we exit as soon as we run past
* wbc->nr_to_write or encounter the first error.
*/
if (wbc->sync_mode == WB_SYNC_ALL) {
if (*error && !wbc->saved_err)
wbc->saved_err = *error;
} else {
if (*error || wbc->nr_to_write <= 0)
goto done;
}
}
folio = writeback_get_folio(mapping, wbc);
if (!folio) {
/*
* To avoid deadlocks between range_cyclic writeback and callers
* that hold pages in PageWriteback to aggregate I/O until
* the writeback iteration finishes, we do not loop back to the
* start of the file. Doing so causes a page lock/page
* writeback access order inversion - we should only ever lock
* multiple pages in ascending page->index order, and looping
* back to the start of the file violates that rule and causes
* deadlocks.
*/
if (wbc->range_cyclic)
mapping->writeback_index = 0;
/*
* Return the first error we encountered (if there was any) to
* the caller.
*/
*error = wbc->saved_err;
}
return folio;
done:
if (wbc->range_cyclic)
mapping->writeback_index = folio->index + folio_nr_pages(folio);
folio_batch_release(&wbc->fbatch);
return NULL;
}
/**
* write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
* @mapping: address space structure to write
* @wbc: subtract the number of written pages from *@wbc->nr_to_write
* @writepage: function called for each page
* @data: data passed to writepage function
*
* Return: %0 on success, negative error code otherwise
*
* Note: please use writeback_iter() instead.
*/
int write_cache_pages(struct address_space *mapping,
struct writeback_control *wbc, writepage_t writepage,
void *data)
{
struct folio *folio = NULL;
int error;
while ((folio = writeback_iter(mapping, wbc, folio, &error))) {
error = writepage(folio, wbc, data);
if (error == AOP_WRITEPAGE_ACTIVATE) {
folio_unlock(folio);
error = 0;
}
}
return error;
}
EXPORT_SYMBOL(write_cache_pages);
static int writeback_use_writepage(struct address_space *mapping,
struct writeback_control *wbc)
{
struct folio *folio = NULL;
struct blk_plug plug;
int err;
blk_start_plug(&plug);
while ((folio = writeback_iter(mapping, wbc, folio, &err))) {
err = mapping->a_ops->writepage(&folio->page, wbc);
if (err == AOP_WRITEPAGE_ACTIVATE) {
folio_unlock(folio);
err = 0;
}
mapping_set_error(mapping, err);
}
blk_finish_plug(&plug);
return err;
}
int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
int ret;
struct bdi_writeback *wb;
if (wbc->nr_to_write <= 0)
return 0;
wb = inode_to_wb_wbc(mapping->host, wbc);
wb_bandwidth_estimate_start(wb);
while (1) {
if (mapping->a_ops->writepages) {
ret = mapping->a_ops->writepages(mapping, wbc);
} else if (mapping->a_ops->writepage) {
ret = writeback_use_writepage(mapping, wbc);
} else {
/* deal with chardevs and other special files */
ret = 0;
}
if (ret != -ENOMEM || wbc->sync_mode != WB_SYNC_ALL)
break;
/*
* Lacking an allocation context or the locality or writeback
* state of any of the inode's pages, throttle based on
* writeback activity on the local node. It's as good a
* guess as any.
*/
reclaim_throttle(NODE_DATA(numa_node_id()),
VMSCAN_THROTTLE_WRITEBACK);
}
/*
* Usually few pages are written by now from those we've just submitted
* but if there's constant writeback being submitted, this makes sure
* writeback bandwidth is updated once in a while.
*/
if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
BANDWIDTH_INTERVAL))
wb_update_bandwidth(wb);
return ret;
}
/*
* For address_spaces which do not use buffers nor write back.
*/
bool noop_dirty_folio(struct address_space *mapping, struct folio *folio)
{
if (!folio_test_dirty(folio)) return !folio_test_set_dirty(folio);
return false;
}
EXPORT_SYMBOL(noop_dirty_folio);
/*
* Helper function for set_page_dirty family.
*
* Caller must hold folio_memcg_lock().
*
* NOTE: This relies on being atomic wrt interrupts.
*/
static void folio_account_dirtied(struct folio *folio,
struct address_space *mapping)
{
struct inode *inode = mapping->host;
trace_writeback_dirty_folio(folio, mapping);
if (mapping_can_writeback(mapping)) {
struct bdi_writeback *wb;
long nr = folio_nr_pages(folio);
inode_attach_wb(inode, folio);
wb = inode_to_wb(inode);
__lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, nr);
__zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr);
__node_stat_mod_folio(folio, NR_DIRTIED, nr);
wb_stat_mod(wb, WB_RECLAIMABLE, nr);
wb_stat_mod(wb, WB_DIRTIED, nr);
task_io_account_write(nr * PAGE_SIZE);
current->nr_dirtied += nr;
__this_cpu_add(bdp_ratelimits, nr);
mem_cgroup_track_foreign_dirty(folio, wb);
}
}
/*
* Helper function for deaccounting dirty page without writeback.
*
* Caller must hold folio_memcg_lock().
*/
void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb)
{
long nr = folio_nr_pages(folio);
lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr);
zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr);
wb_stat_mod(wb, WB_RECLAIMABLE, -nr);
task_io_account_cancelled_write(nr * PAGE_SIZE);
}
/*
* Mark the folio dirty, and set it dirty in the page cache, and mark
* the inode dirty.
*
* If warn is true, then emit a warning if the folio is not uptodate and has
* not been truncated.
*
* The caller must hold folio_memcg_lock(). Most callers have the folio
* locked. A few have the folio blocked from truncation through other
* means (eg zap_vma_pages() has it mapped and is holding the page table
* lock). This can also be called from mark_buffer_dirty(), which I
* cannot prove is always protected against truncate.
*/
void __folio_mark_dirty(struct folio *folio, struct address_space *mapping,
int warn)
{
unsigned long flags;
xa_lock_irqsave(&mapping->i_pages, flags);
if (folio->mapping) { /* Race with truncate? */
WARN_ON_ONCE(warn && !folio_test_uptodate(folio));
folio_account_dirtied(folio, mapping);
__xa_set_mark(&mapping->i_pages, folio_index(folio),
PAGECACHE_TAG_DIRTY);
}
xa_unlock_irqrestore(&mapping->i_pages, flags);
}
/**
* filemap_dirty_folio - Mark a folio dirty for filesystems which do not use buffer_heads.
* @mapping: Address space this folio belongs to.
* @folio: Folio to be marked as dirty.
*
* Filesystems which do not use buffer heads should call this function
* from their dirty_folio address space operation. It ignores the
* contents of folio_get_private(), so if the filesystem marks individual
* blocks as dirty, the filesystem should handle that itself.
*
* This is also sometimes used by filesystems which use buffer_heads when
* a single buffer is being dirtied: we want to set the folio dirty in
* that case, but not all the buffers. This is a "bottom-up" dirtying,
* whereas block_dirty_folio() is a "top-down" dirtying.
*
* The caller must ensure this doesn't race with truncation. Most will
* simply hold the folio lock, but e.g. zap_pte_range() calls with the
* folio mapped and the pte lock held, which also locks out truncation.
*/
bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio)
{
folio_memcg_lock(folio);
if (folio_test_set_dirty(folio)) {
folio_memcg_unlock(folio);
return false;
}
__folio_mark_dirty(folio, mapping, !folio_test_private(folio));
folio_memcg_unlock(folio);
if (mapping->host) {
/* !PageAnon && !swapper_space */
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
}
return true;
}
EXPORT_SYMBOL(filemap_dirty_folio);
/**
* folio_redirty_for_writepage - Decline to write a dirty folio.
* @wbc: The writeback control.
* @folio: The folio.
*
* When a writepage implementation decides that it doesn't want to write
* @folio for some reason, it should call this function, unlock @folio and
* return 0.
*
* Return: True if we redirtied the folio. False if someone else dirtied
* it first.
*/
bool folio_redirty_for_writepage(struct writeback_control *wbc,
struct folio *folio)
{
struct address_space *mapping = folio->mapping;
long nr = folio_nr_pages(folio);
bool ret;
wbc->pages_skipped += nr;
ret = filemap_dirty_folio(mapping, folio);
if (mapping && mapping_can_writeback(mapping)) {
struct inode *inode = mapping->host;
struct bdi_writeback *wb;
struct wb_lock_cookie cookie = {};
wb = unlocked_inode_to_wb_begin(inode, &cookie);
current->nr_dirtied -= nr;
node_stat_mod_folio(folio, NR_DIRTIED, -nr);
wb_stat_mod(wb, WB_DIRTIED, -nr);
unlocked_inode_to_wb_end(inode, &cookie);
}
return ret;
}
EXPORT_SYMBOL(folio_redirty_for_writepage);
/**
* folio_mark_dirty - Mark a folio as being modified.
* @folio: The folio.
*
* The folio may not be truncated while this function is running.
* Holding the folio lock is sufficient to prevent truncation, but some
* callers cannot acquire a sleeping lock. These callers instead hold
* the page table lock for a page table which contains at least one page
* in this folio. Truncation will block on the page table lock as it
* unmaps pages before removing the folio from its mapping.
*
* Return: True if the folio was newly dirtied, false if it was already dirty.
*/
bool folio_mark_dirty(struct folio *folio)
{
struct address_space *mapping = folio_mapping(folio);
if (likely(mapping)) {
/*
* readahead/folio_deactivate could remain
* PG_readahead/PG_reclaim due to race with folio_end_writeback
* About readahead, if the folio is written, the flags would be
* reset. So no problem.
* About folio_deactivate, if the folio is redirtied,
* the flag will be reset. So no problem. but if the
* folio is used by readahead it will confuse readahead
* and make it restart the size rampup process. But it's
* a trivial problem.
*/
if (folio_test_reclaim(folio)) folio_clear_reclaim(folio); return mapping->a_ops->dirty_folio(mapping, folio);
}
return noop_dirty_folio(mapping, folio);
}
EXPORT_SYMBOL(folio_mark_dirty);
/*
* set_page_dirty() is racy if the caller has no reference against
* page->mapping->host, and if the page is unlocked. This is because another
* CPU could truncate the page off the mapping and then free the mapping.
*
* Usually, the page _is_ locked, or the caller is a user-space process which
* holds a reference on the inode by having an open file.
*
* In other cases, the page should be locked before running set_page_dirty().
*/
int set_page_dirty_lock(struct page *page)
{
int ret;
lock_page(page);
ret = set_page_dirty(page);
unlock_page(page);
return ret;
}
EXPORT_SYMBOL(set_page_dirty_lock);
/*
* This cancels just the dirty bit on the kernel page itself, it does NOT
* actually remove dirty bits on any mmap's that may be around. It also
* leaves the page tagged dirty, so any sync activity will still find it on
* the dirty lists, and in particular, clear_page_dirty_for_io() will still
* look at the dirty bits in the VM.
*
* Doing this should *normally* only ever be done when a page is truncated,
* and is not actually mapped anywhere at all. However, fs/buffer.c does
* this when it notices that somebody has cleaned out all the buffers on a
* page without actually doing it through the VM. Can you say "ext3 is
* horribly ugly"? Thought you could.
*/
void __folio_cancel_dirty(struct folio *folio)
{
struct address_space *mapping = folio_mapping(folio);
if (mapping_can_writeback(mapping)) {
struct inode *inode = mapping->host;
struct bdi_writeback *wb;
struct wb_lock_cookie cookie = {};
folio_memcg_lock(folio);
wb = unlocked_inode_to_wb_begin(inode, &cookie);
if (folio_test_clear_dirty(folio))
folio_account_cleaned(folio, wb); unlocked_inode_to_wb_end(inode, &cookie);
folio_memcg_unlock(folio);
} else {
folio_clear_dirty(folio);
}
}
EXPORT_SYMBOL(__folio_cancel_dirty);
/*
* Clear a folio's dirty flag, while caring for dirty memory accounting.
* Returns true if the folio was previously dirty.
*
* This is for preparing to put the folio under writeout. We leave
* the folio tagged as dirty in the xarray so that a concurrent
* write-for-sync can discover it via a PAGECACHE_TAG_DIRTY walk.
* The ->writepage implementation will run either folio_start_writeback()
* or folio_mark_dirty(), at which stage we bring the folio's dirty flag
* and xarray dirty tag back into sync.
*
* This incoherency between the folio's dirty flag and xarray tag is
* unfortunate, but it only exists while the folio is locked.
*/
bool folio_clear_dirty_for_io(struct folio *folio)
{
struct address_space *mapping = folio_mapping(folio);
bool ret = false;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (mapping && mapping_can_writeback(mapping)) {
struct inode *inode = mapping->host;
struct bdi_writeback *wb;
struct wb_lock_cookie cookie = {};
/*
* Yes, Virginia, this is indeed insane.
*
* We use this sequence to make sure that
* (a) we account for dirty stats properly
* (b) we tell the low-level filesystem to
* mark the whole folio dirty if it was
* dirty in a pagetable. Only to then
* (c) clean the folio again and return 1 to
* cause the writeback.
*
* This way we avoid all nasty races with the
* dirty bit in multiple places and clearing
* them concurrently from different threads.
*
* Note! Normally the "folio_mark_dirty(folio)"
* has no effect on the actual dirty bit - since
* that will already usually be set. But we
* need the side effects, and it can help us
* avoid races.
*
* We basically use the folio "master dirty bit"
* as a serialization point for all the different
* threads doing their things.
*/
if (folio_mkclean(folio))
folio_mark_dirty(folio);
/*
* We carefully synchronise fault handlers against
* installing a dirty pte and marking the folio dirty
* at this point. We do this by having them hold the
* page lock while dirtying the folio, and folios are
* always locked coming in here, so we get the desired
* exclusion.
*/
wb = unlocked_inode_to_wb_begin(inode, &cookie);
if (folio_test_clear_dirty(folio)) {
long nr = folio_nr_pages(folio);
lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr);
zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr);
wb_stat_mod(wb, WB_RECLAIMABLE, -nr);
ret = true;
}
unlocked_inode_to_wb_end(inode, &cookie);
return ret;
}
return folio_test_clear_dirty(folio);
}
EXPORT_SYMBOL(folio_clear_dirty_for_io);
static void wb_inode_writeback_start(struct bdi_writeback *wb)
{
atomic_inc(&wb->writeback_inodes);
}
static void wb_inode_writeback_end(struct bdi_writeback *wb)
{
unsigned long flags;
atomic_dec(&wb->writeback_inodes);
/*
* Make sure estimate of writeback throughput gets updated after
* writeback completed. We delay the update by BANDWIDTH_INTERVAL
* (which is the interval other bandwidth updates use for batching) so
* that if multiple inodes end writeback at a similar time, they get
* batched into one bandwidth update.
*/
spin_lock_irqsave(&wb->work_lock, flags);
if (test_bit(WB_registered, &wb->state))
queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL);
spin_unlock_irqrestore(&wb->work_lock, flags);
}
bool __folio_end_writeback(struct folio *folio)
{
long nr = folio_nr_pages(folio);
struct address_space *mapping = folio_mapping(folio);
bool ret;
folio_memcg_lock(folio);
if (mapping && mapping_use_writeback_tags(mapping)) {
struct inode *inode = mapping->host;
struct backing_dev_info *bdi = inode_to_bdi(inode);
unsigned long flags;
xa_lock_irqsave(&mapping->i_pages, flags);
ret = folio_xor_flags_has_waiters(folio, 1 << PG_writeback);
__xa_clear_mark(&mapping->i_pages, folio_index(folio),
PAGECACHE_TAG_WRITEBACK);
if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
struct bdi_writeback *wb = inode_to_wb(inode);
wb_stat_mod(wb, WB_WRITEBACK, -nr);
__wb_writeout_add(wb, nr);
if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
wb_inode_writeback_end(wb);
}
if (mapping->host && !mapping_tagged(mapping,
PAGECACHE_TAG_WRITEBACK))
sb_clear_inode_writeback(mapping->host);
xa_unlock_irqrestore(&mapping->i_pages, flags);
} else {
ret = folio_xor_flags_has_waiters(folio, 1 << PG_writeback);
}
lruvec_stat_mod_folio(folio, NR_WRITEBACK, -nr);
zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr);
node_stat_mod_folio(folio, NR_WRITTEN, nr);
folio_memcg_unlock(folio);
return ret;
}
void __folio_start_writeback(struct folio *folio, bool keep_write)
{
long nr = folio_nr_pages(folio);
struct address_space *mapping = folio_mapping(folio);
int access_ret;
VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
folio_memcg_lock(folio);
if (mapping && mapping_use_writeback_tags(mapping)) {
XA_STATE(xas, &mapping->i_pages, folio_index(folio));
struct inode *inode = mapping->host;
struct backing_dev_info *bdi = inode_to_bdi(inode);
unsigned long flags;
bool on_wblist;
xas_lock_irqsave(&xas, flags);
xas_load(&xas);
folio_test_set_writeback(folio);
on_wblist = mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
struct bdi_writeback *wb = inode_to_wb(inode);
wb_stat_mod(wb, WB_WRITEBACK, nr);
if (!on_wblist)
wb_inode_writeback_start(wb);
}
/*
* We can come through here when swapping anonymous
* folios, so we don't necessarily have an inode to
* track for sync.
*/
if (mapping->host && !on_wblist)
sb_mark_inode_writeback(mapping->host);
if (!folio_test_dirty(folio))
xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
if (!keep_write)
xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
xas_unlock_irqrestore(&xas, flags);
} else {
folio_test_set_writeback(folio);
}
lruvec_stat_mod_folio(folio, NR_WRITEBACK, nr);
zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr);
folio_memcg_unlock(folio);
access_ret = arch_make_folio_accessible(folio);
/*
* If writeback has been triggered on a page that cannot be made
* accessible, it is too late to recover here.
*/
VM_BUG_ON_FOLIO(access_ret != 0, folio);
}
EXPORT_SYMBOL(__folio_start_writeback);
/**
* folio_wait_writeback - Wait for a folio to finish writeback.
* @folio: The folio to wait for.
*
* If the folio is currently being written back to storage, wait for the
* I/O to complete.
*
* Context: Sleeps. Must be called in process context and with
* no spinlocks held. Caller should hold a reference on the folio.
* If the folio is not locked, writeback may start again after writeback
* has finished.
*/
void folio_wait_writeback(struct folio *folio)
{
while (folio_test_writeback(folio)) { trace_folio_wait_writeback(folio, folio_mapping(folio)); folio_wait_bit(folio, PG_writeback);
}
}
EXPORT_SYMBOL_GPL(folio_wait_writeback);
/**
* folio_wait_writeback_killable - Wait for a folio to finish writeback.
* @folio: The folio to wait for.
*
* If the folio is currently being written back to storage, wait for the
* I/O to complete or a fatal signal to arrive.
*
* Context: Sleeps. Must be called in process context and with
* no spinlocks held. Caller should hold a reference on the folio.
* If the folio is not locked, writeback may start again after writeback
* has finished.
* Return: 0 on success, -EINTR if we get a fatal signal while waiting.
*/
int folio_wait_writeback_killable(struct folio *folio)
{
while (folio_test_writeback(folio)) {
trace_folio_wait_writeback(folio, folio_mapping(folio));
if (folio_wait_bit_killable(folio, PG_writeback))
return -EINTR;
}
return 0;
}
EXPORT_SYMBOL_GPL(folio_wait_writeback_killable);
/**
* folio_wait_stable() - wait for writeback to finish, if necessary.
* @folio: The folio to wait on.
*
* This function determines if the given folio is related to a backing
* device that requires folio contents to be held stable during writeback.
* If so, then it will wait for any pending writeback to complete.
*
* Context: Sleeps. Must be called in process context and with
* no spinlocks held. Caller should hold a reference on the folio.
* If the folio is not locked, writeback may start again after writeback
* has finished.
*/
void folio_wait_stable(struct folio *folio)
{
if (mapping_stable_writes(folio_mapping(folio)))
folio_wait_writeback(folio);
}
EXPORT_SYMBOL_GPL(folio_wait_stable);
/* SPDX-License-Identifier: GPL-2.0 */
/*
* include/linux/pagevec.h
*
* In many places it is efficient to batch an operation up against multiple
* folios. A folio_batch is a container which is used for that.
*/
#ifndef _LINUX_PAGEVEC_H
#define _LINUX_PAGEVEC_H
#include <linux/types.h>
/* 31 pointers + header align the folio_batch structure to a power of two */
#define PAGEVEC_SIZE 31
struct folio;
/**
* struct folio_batch - A collection of folios.
*
* The folio_batch is used to amortise the cost of retrieving and
* operating on a set of folios. The order of folios in the batch may be
* significant (eg delete_from_page_cache_batch()). Some users of the
* folio_batch store "exceptional" entries in it which can be removed
* by calling folio_batch_remove_exceptionals().
*/
struct folio_batch {
unsigned char nr;
unsigned char i;
bool percpu_pvec_drained;
struct folio *folios[PAGEVEC_SIZE];
};
/**
* folio_batch_init() - Initialise a batch of folios
* @fbatch: The folio batch.
*
* A freshly initialised folio_batch contains zero folios.
*/
static inline void folio_batch_init(struct folio_batch *fbatch)
{
fbatch->nr = 0;
fbatch->i = 0;
fbatch->percpu_pvec_drained = false;
}
static inline void folio_batch_reinit(struct folio_batch *fbatch)
{
fbatch->nr = 0;
fbatch->i = 0;
}
static inline unsigned int folio_batch_count(struct folio_batch *fbatch)
{
return fbatch->nr;
}
static inline unsigned int folio_batch_space(struct folio_batch *fbatch)
{
return PAGEVEC_SIZE - fbatch->nr;
}
/**
* folio_batch_add() - Add a folio to a batch.
* @fbatch: The folio batch.
* @folio: The folio to add.
*
* The folio is added to the end of the batch.
* The batch must have previously been initialised using folio_batch_init().
*
* Return: The number of slots still available.
*/
static inline unsigned folio_batch_add(struct folio_batch *fbatch,
struct folio *folio)
{
fbatch->folios[fbatch->nr++] = folio;
return folio_batch_space(fbatch);
}
/**
* folio_batch_next - Return the next folio to process.
* @fbatch: The folio batch being processed.
*
* Use this function to implement a queue of folios.
*
* Return: The next folio in the queue, or NULL if the queue is empty.
*/
static inline struct folio *folio_batch_next(struct folio_batch *fbatch)
{
if (fbatch->i == fbatch->nr)
return NULL;
return fbatch->folios[fbatch->i++];
}
void __folio_batch_release(struct folio_batch *pvec);
static inline void folio_batch_release(struct folio_batch *fbatch)
{
if (folio_batch_count(fbatch)) __folio_batch_release(fbatch);
}
void folio_batch_remove_exceptionals(struct folio_batch *fbatch);
#endif /* _LINUX_PAGEVEC_H */
// SPDX-License-Identifier: GPL-2.0+
/*
* Maple Tree implementation
* Copyright (c) 2018-2022 Oracle Corporation
* Authors: Liam R. Howlett <Liam.Howlett@oracle.com>
* Matthew Wilcox <willy@infradead.org>
* Copyright (c) 2023 ByteDance
* Author: Peng Zhang <zhangpeng.00@bytedance.com>
*/
/*
* DOC: Interesting implementation details of the Maple Tree
*
* Each node type has a number of slots for entries and a number of slots for
* pivots. In the case of dense nodes, the pivots are implied by the position
* and are simply the slot index + the minimum of the node.
*
* In regular B-Tree terms, pivots are called keys. The term pivot is used to
* indicate that the tree is specifying ranges. Pivots may appear in the
* subtree with an entry attached to the value whereas keys are unique to a
* specific position of a B-tree. Pivot values are inclusive of the slot with
* the same index.
*
*
* The following illustrates the layout of a range64 nodes slots and pivots.
*
*
* Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 |
* ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬
* │ │ │ │ │ │ │ │ └─ Implied maximum
* │ │ │ │ │ │ │ └─ Pivot 14
* │ │ │ │ │ │ └─ Pivot 13
* │ │ │ │ │ └─ Pivot 12
* │ │ │ │ └─ Pivot 11
* │ │ │ └─ Pivot 2
* │ │ └─ Pivot 1
* │ └─ Pivot 0
* └─ Implied minimum
*
* Slot contents:
* Internal (non-leaf) nodes contain pointers to other nodes.
* Leaf nodes contain entries.
*
* The location of interest is often referred to as an offset. All offsets have
* a slot, but the last offset has an implied pivot from the node above (or
* UINT_MAX for the root node.
*
* Ranges complicate certain write activities. When modifying any of
* the B-tree variants, it is known that one entry will either be added or
* deleted. When modifying the Maple Tree, one store operation may overwrite
* the entire data set, or one half of the tree, or the middle half of the tree.
*
*/
#include <linux/maple_tree.h>
#include <linux/xarray.h>
#include <linux/types.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/limits.h>
#include <asm/barrier.h>
#define CREATE_TRACE_POINTS
#include <trace/events/maple_tree.h>
#define MA_ROOT_PARENT 1
/*
* Maple state flags
* * MA_STATE_BULK - Bulk insert mode
* * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert
* * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation
*/
#define MA_STATE_BULK 1
#define MA_STATE_REBALANCE 2
#define MA_STATE_PREALLOC 4
#define ma_parent_ptr(x) ((struct maple_pnode *)(x))
#define mas_tree_parent(x) ((unsigned long)(x->tree) | MA_ROOT_PARENT)
#define ma_mnode_ptr(x) ((struct maple_node *)(x))
#define ma_enode_ptr(x) ((struct maple_enode *)(x))
static struct kmem_cache *maple_node_cache;
#ifdef CONFIG_DEBUG_MAPLE_TREE
static const unsigned long mt_max[] = {
[maple_dense] = MAPLE_NODE_SLOTS,
[maple_leaf_64] = ULONG_MAX,
[maple_range_64] = ULONG_MAX,
[maple_arange_64] = ULONG_MAX,
};
#define mt_node_max(x) mt_max[mte_node_type(x)]
#endif
static const unsigned char mt_slots[] = {
[maple_dense] = MAPLE_NODE_SLOTS,
[maple_leaf_64] = MAPLE_RANGE64_SLOTS,
[maple_range_64] = MAPLE_RANGE64_SLOTS,
[maple_arange_64] = MAPLE_ARANGE64_SLOTS,
};
#define mt_slot_count(x) mt_slots[mte_node_type(x)]
static const unsigned char mt_pivots[] = {
[maple_dense] = 0,
[maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1,
[maple_range_64] = MAPLE_RANGE64_SLOTS - 1,
[maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1,
};
#define mt_pivot_count(x) mt_pivots[mte_node_type(x)]
static const unsigned char mt_min_slots[] = {
[maple_dense] = MAPLE_NODE_SLOTS / 2,
[maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2,
[maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2,
[maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1,
};
#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)]
#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2)
#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1)
struct maple_big_node {
struct maple_pnode *parent;
unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1];
union {
struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS];
struct {
unsigned long padding[MAPLE_BIG_NODE_GAPS];
unsigned long gap[MAPLE_BIG_NODE_GAPS];
};
};
unsigned char b_end;
enum maple_type type;
};
/*
* The maple_subtree_state is used to build a tree to replace a segment of an
* existing tree in a more atomic way. Any walkers of the older tree will hit a
* dead node and restart on updates.
*/
struct maple_subtree_state {
struct ma_state *orig_l; /* Original left side of subtree */
struct ma_state *orig_r; /* Original right side of subtree */
struct ma_state *l; /* New left side of subtree */
struct ma_state *m; /* New middle of subtree (rare) */
struct ma_state *r; /* New right side of subtree */
struct ma_topiary *free; /* nodes to be freed */
struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */
struct maple_big_node *bn;
};
#ifdef CONFIG_KASAN_STACK
/* Prevent mas_wr_bnode() from exceeding the stack frame limit */
#define noinline_for_kasan noinline_for_stack
#else
#define noinline_for_kasan inline
#endif
/* Functions */
static inline struct maple_node *mt_alloc_one(gfp_t gfp)
{
return kmem_cache_alloc(maple_node_cache, gfp);
}
static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes)
{
return kmem_cache_alloc_bulk(maple_node_cache, gfp, size, nodes);
}
static inline void mt_free_one(struct maple_node *node)
{
kmem_cache_free(maple_node_cache, node);
}
static inline void mt_free_bulk(size_t size, void __rcu **nodes)
{
kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes);
}
static void mt_free_rcu(struct rcu_head *head)
{
struct maple_node *node = container_of(head, struct maple_node, rcu);
kmem_cache_free(maple_node_cache, node);
}
/*
* ma_free_rcu() - Use rcu callback to free a maple node
* @node: The node to free
*
* The maple tree uses the parent pointer to indicate this node is no longer in
* use and will be freed.
*/
static void ma_free_rcu(struct maple_node *node)
{
WARN_ON(node->parent != ma_parent_ptr(node));
call_rcu(&node->rcu, mt_free_rcu);
}
static void mas_set_height(struct ma_state *mas)
{
unsigned int new_flags = mas->tree->ma_flags;
new_flags &= ~MT_FLAGS_HEIGHT_MASK;
MAS_BUG_ON(mas, mas->depth > MAPLE_HEIGHT_MAX);
new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET;
mas->tree->ma_flags = new_flags;
}
static unsigned int mas_mt_height(struct ma_state *mas)
{
return mt_height(mas->tree);
}
static inline unsigned int mt_attr(struct maple_tree *mt)
{
return mt->ma_flags & ~MT_FLAGS_HEIGHT_MASK;
}
static __always_inline enum maple_type mte_node_type(
const struct maple_enode *entry)
{
return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) &
MAPLE_NODE_TYPE_MASK;
}
static __always_inline bool ma_is_dense(const enum maple_type type)
{
return type < maple_leaf_64;
}
static __always_inline bool ma_is_leaf(const enum maple_type type)
{
return type < maple_range_64;
}
static __always_inline bool mte_is_leaf(const struct maple_enode *entry)
{
return ma_is_leaf(mte_node_type(entry));
}
/*
* We also reserve values with the bottom two bits set to '10' which are
* below 4096
*/
static __always_inline bool mt_is_reserved(const void *entry)
{
return ((unsigned long)entry < MAPLE_RESERVED_RANGE) && xa_is_internal(entry);
}
static __always_inline void mas_set_err(struct ma_state *mas, long err)
{
mas->node = MA_ERROR(err);
mas->status = ma_error;
}
static __always_inline bool mas_is_ptr(const struct ma_state *mas)
{
return mas->status == ma_root;
}
static __always_inline bool mas_is_start(const struct ma_state *mas)
{
return mas->status == ma_start;
}
static __always_inline bool mas_is_none(const struct ma_state *mas)
{
return mas->status == ma_none;
}
static __always_inline bool mas_is_paused(const struct ma_state *mas)
{
return mas->status == ma_pause;
}
static __always_inline bool mas_is_overflow(struct ma_state *mas)
{
return mas->status == ma_overflow;
}
static inline bool mas_is_underflow(struct ma_state *mas)
{
return mas->status == ma_underflow;
}
static __always_inline struct maple_node *mte_to_node(
const struct maple_enode *entry)
{
return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK);
}
/*
* mte_to_mat() - Convert a maple encoded node to a maple topiary node.
* @entry: The maple encoded node
*
* Return: a maple topiary pointer
*/
static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry)
{
return (struct maple_topiary *)
((unsigned long)entry & ~MAPLE_NODE_MASK);
}
/*
* mas_mn() - Get the maple state node.
* @mas: The maple state
*
* Return: the maple node (not encoded - bare pointer).
*/
static inline struct maple_node *mas_mn(const struct ma_state *mas)
{
return mte_to_node(mas->node);
}
/*
* mte_set_node_dead() - Set a maple encoded node as dead.
* @mn: The maple encoded node.
*/
static inline void mte_set_node_dead(struct maple_enode *mn)
{
mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn));
smp_wmb(); /* Needed for RCU */
}
/* Bit 1 indicates the root is a node */
#define MAPLE_ROOT_NODE 0x02
/* maple_type stored bit 3-6 */
#define MAPLE_ENODE_TYPE_SHIFT 0x03
/* Bit 2 means a NULL somewhere below */
#define MAPLE_ENODE_NULL 0x04
static inline struct maple_enode *mt_mk_node(const struct maple_node *node,
enum maple_type type)
{
return (void *)((unsigned long)node | (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL);
}
static inline void *mte_mk_root(const struct maple_enode *node)
{
return (void *)((unsigned long)node | MAPLE_ROOT_NODE);
}
static inline void *mte_safe_root(const struct maple_enode *node)
{
return (void *)((unsigned long)node & ~MAPLE_ROOT_NODE);
}
static inline void *mte_set_full(const struct maple_enode *node)
{
return (void *)((unsigned long)node & ~MAPLE_ENODE_NULL);
}
static inline void *mte_clear_full(const struct maple_enode *node)
{
return (void *)((unsigned long)node | MAPLE_ENODE_NULL);
}
static inline bool mte_has_null(const struct maple_enode *node)
{
return (unsigned long)node & MAPLE_ENODE_NULL;
}
static __always_inline bool ma_is_root(struct maple_node *node)
{
return ((unsigned long)node->parent & MA_ROOT_PARENT);
}
static __always_inline bool mte_is_root(const struct maple_enode *node)
{
return ma_is_root(mte_to_node(node));
}
static inline bool mas_is_root_limits(const struct ma_state *mas)
{
return !mas->min && mas->max == ULONG_MAX;
}
static __always_inline bool mt_is_alloc(struct maple_tree *mt)
{
return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE);
}
/*
* The Parent Pointer
* Excluding root, the parent pointer is 256B aligned like all other tree nodes.
* When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16
* bit values need an extra bit to store the offset. This extra bit comes from
* a reuse of the last bit in the node type. This is possible by using bit 1 to
* indicate if bit 2 is part of the type or the slot.
*
* Note types:
* 0x??1 = Root
* 0x?00 = 16 bit nodes
* 0x010 = 32 bit nodes
* 0x110 = 64 bit nodes
*
* Slot size and alignment
* 0b??1 : Root
* 0b?00 : 16 bit values, type in 0-1, slot in 2-7
* 0b010 : 32 bit values, type in 0-2, slot in 3-7
* 0b110 : 64 bit values, type in 0-2, slot in 3-7
*/
#define MAPLE_PARENT_ROOT 0x01
#define MAPLE_PARENT_SLOT_SHIFT 0x03
#define MAPLE_PARENT_SLOT_MASK 0xF8
#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02
#define MAPLE_PARENT_16B_SLOT_MASK 0xFC
#define MAPLE_PARENT_RANGE64 0x06
#define MAPLE_PARENT_RANGE32 0x04
#define MAPLE_PARENT_NOT_RANGE16 0x02
/*
* mte_parent_shift() - Get the parent shift for the slot storage.
* @parent: The parent pointer cast as an unsigned long
* Return: The shift into that pointer to the star to of the slot
*/
static inline unsigned long mte_parent_shift(unsigned long parent)
{
/* Note bit 1 == 0 means 16B */
if (likely(parent & MAPLE_PARENT_NOT_RANGE16))
return MAPLE_PARENT_SLOT_SHIFT;
return MAPLE_PARENT_16B_SLOT_SHIFT;
}
/*
* mte_parent_slot_mask() - Get the slot mask for the parent.
* @parent: The parent pointer cast as an unsigned long.
* Return: The slot mask for that parent.
*/
static inline unsigned long mte_parent_slot_mask(unsigned long parent)
{
/* Note bit 1 == 0 means 16B */
if (likely(parent & MAPLE_PARENT_NOT_RANGE16))
return MAPLE_PARENT_SLOT_MASK;
return MAPLE_PARENT_16B_SLOT_MASK;
}
/*
* mas_parent_type() - Return the maple_type of the parent from the stored
* parent type.
* @mas: The maple state
* @enode: The maple_enode to extract the parent's enum
* Return: The node->parent maple_type
*/
static inline
enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode)
{
unsigned long p_type;
p_type = (unsigned long)mte_to_node(enode)->parent; if (WARN_ON(p_type & MAPLE_PARENT_ROOT))
return 0;
p_type &= MAPLE_NODE_MASK;
p_type &= ~mte_parent_slot_mask(p_type); switch (p_type) {
case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */
if (mt_is_alloc(mas->tree))
return maple_arange_64;
return maple_range_64;
}
return 0;
}
/*
* mas_set_parent() - Set the parent node and encode the slot
* @enode: The encoded maple node.
* @parent: The encoded maple node that is the parent of @enode.
* @slot: The slot that @enode resides in @parent.
*
* Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the
* parent type.
*/
static inline
void mas_set_parent(struct ma_state *mas, struct maple_enode *enode,
const struct maple_enode *parent, unsigned char slot)
{
unsigned long val = (unsigned long)parent;
unsigned long shift;
unsigned long type;
enum maple_type p_type = mte_node_type(parent);
MAS_BUG_ON(mas, p_type == maple_dense);
MAS_BUG_ON(mas, p_type == maple_leaf_64);
switch (p_type) {
case maple_range_64:
case maple_arange_64:
shift = MAPLE_PARENT_SLOT_SHIFT;
type = MAPLE_PARENT_RANGE64;
break;
default:
case maple_dense:
case maple_leaf_64:
shift = type = 0;
break;
}
val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */
val |= (slot << shift) | type;
mte_to_node(enode)->parent = ma_parent_ptr(val);
}
/*
* mte_parent_slot() - get the parent slot of @enode.
* @enode: The encoded maple node.
*
* Return: The slot in the parent node where @enode resides.
*/
static __always_inline
unsigned int mte_parent_slot(const struct maple_enode *enode)
{
unsigned long val = (unsigned long)mte_to_node(enode)->parent; if (unlikely(val & MA_ROOT_PARENT)) return 0;
/*
* Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost
* by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT
*/
return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val);
}
/*
* mte_parent() - Get the parent of @node.
* @node: The encoded maple node.
*
* Return: The parent maple node.
*/
static __always_inline
struct maple_node *mte_parent(const struct maple_enode *enode)
{
return (void *)((unsigned long)
(mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK);
}
/*
* ma_dead_node() - check if the @enode is dead.
* @enode: The encoded maple node
*
* Return: true if dead, false otherwise.
*/
static __always_inline bool ma_dead_node(const struct maple_node *node)
{
struct maple_node *parent;
/* Do not reorder reads from the node prior to the parent check */
smp_rmb();
parent = (void *)((unsigned long) node->parent & ~MAPLE_NODE_MASK);
return (parent == node);
}
/*
* mte_dead_node() - check if the @enode is dead.
* @enode: The encoded maple node
*
* Return: true if dead, false otherwise.
*/
static __always_inline bool mte_dead_node(const struct maple_enode *enode)
{
struct maple_node *parent, *node;
node = mte_to_node(enode);
/* Do not reorder reads from the node prior to the parent check */
smp_rmb();
parent = mte_parent(enode);
return (parent == node);
}
/*
* mas_allocated() - Get the number of nodes allocated in a maple state.
* @mas: The maple state
*
* The ma_state alloc member is overloaded to hold a pointer to the first
* allocated node or to the number of requested nodes to allocate. If bit 0 is
* set, then the alloc contains the number of requested nodes. If there is an
* allocated node, then the total allocated nodes is in that node.
*
* Return: The total number of nodes allocated
*/
static inline unsigned long mas_allocated(const struct ma_state *mas)
{
if (!mas->alloc || ((unsigned long)mas->alloc & 0x1))
return 0;
return mas->alloc->total;
}
/*
* mas_set_alloc_req() - Set the requested number of allocations.
* @mas: the maple state
* @count: the number of allocations.
*
* The requested number of allocations is either in the first allocated node,
* located in @mas->alloc->request_count, or directly in @mas->alloc if there is
* no allocated node. Set the request either in the node or do the necessary
* encoding to store in @mas->alloc directly.
*/
static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count)
{
if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) {
if (!count)
mas->alloc = NULL;
else
mas->alloc = (struct maple_alloc *)(((count) << 1U) | 1U);
return;
}
mas->alloc->request_count = count;
}
/*
* mas_alloc_req() - get the requested number of allocations.
* @mas: The maple state
*
* The alloc count is either stored directly in @mas, or in
* @mas->alloc->request_count if there is at least one node allocated. Decode
* the request count if it's stored directly in @mas->alloc.
*
* Return: The allocation request count.
*/
static inline unsigned int mas_alloc_req(const struct ma_state *mas)
{
if ((unsigned long)mas->alloc & 0x1)
return (unsigned long)(mas->alloc) >> 1;
else if (mas->alloc)
return mas->alloc->request_count;
return 0;
}
/*
* ma_pivots() - Get a pointer to the maple node pivots.
* @node - the maple node
* @type - the node type
*
* In the event of a dead node, this array may be %NULL
*
* Return: A pointer to the maple node pivots
*/
static inline unsigned long *ma_pivots(struct maple_node *node,
enum maple_type type)
{
switch (type) {
case maple_arange_64:
return node->ma64.pivot;
case maple_range_64:
case maple_leaf_64:
return node->mr64.pivot;
case maple_dense:
return NULL;
}
return NULL;
}
/*
* ma_gaps() - Get a pointer to the maple node gaps.
* @node - the maple node
* @type - the node type
*
* Return: A pointer to the maple node gaps
*/
static inline unsigned long *ma_gaps(struct maple_node *node,
enum maple_type type)
{
switch (type) {
case maple_arange_64:
return node->ma64.gap;
case maple_range_64:
case maple_leaf_64:
case maple_dense:
return NULL;
}
return NULL;
}
/*
* mas_safe_pivot() - get the pivot at @piv or mas->max.
* @mas: The maple state
* @pivots: The pointer to the maple node pivots
* @piv: The pivot to fetch
* @type: The maple node type
*
* Return: The pivot at @piv within the limit of the @pivots array, @mas->max
* otherwise.
*/
static __always_inline unsigned long
mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots,
unsigned char piv, enum maple_type type)
{
if (piv >= mt_pivots[type]) return mas->max; return pivots[piv];
}
/*
* mas_safe_min() - Return the minimum for a given offset.
* @mas: The maple state
* @pivots: The pointer to the maple node pivots
* @offset: The offset into the pivot array
*
* Return: The minimum range value that is contained in @offset.
*/
static inline unsigned long
mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset)
{
if (likely(offset))
return pivots[offset - 1] + 1; return mas->min;
}
/*
* mte_set_pivot() - Set a pivot to a value in an encoded maple node.
* @mn: The encoded maple node
* @piv: The pivot offset
* @val: The value of the pivot
*/
static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv,
unsigned long val)
{
struct maple_node *node = mte_to_node(mn);
enum maple_type type = mte_node_type(mn);
BUG_ON(piv >= mt_pivots[type]);
switch (type) {
case maple_range_64:
case maple_leaf_64:
node->mr64.pivot[piv] = val;
break;
case maple_arange_64:
node->ma64.pivot[piv] = val;
break;
case maple_dense:
break;
}
}
/*
* ma_slots() - Get a pointer to the maple node slots.
* @mn: The maple node
* @mt: The maple node type
*
* Return: A pointer to the maple node slots
*/
static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt)
{
switch (mt) {
case maple_arange_64:
return mn->ma64.slot;
case maple_range_64:
case maple_leaf_64:
return mn->mr64.slot;
case maple_dense:
return mn->slot;
}
return NULL;
}
static inline bool mt_write_locked(const struct maple_tree *mt)
{
return mt_external_lock(mt) ? mt_write_lock_is_held(mt) : lockdep_is_held(&mt->ma_lock);
}
static __always_inline bool mt_locked(const struct maple_tree *mt)
{
return mt_external_lock(mt) ? mt_lock_is_held(mt) : lockdep_is_held(&mt->ma_lock);
}
static __always_inline void *mt_slot(const struct maple_tree *mt,
void __rcu **slots, unsigned char offset)
{
return rcu_dereference_check(slots[offset], mt_locked(mt));
}
static __always_inline void *mt_slot_locked(struct maple_tree *mt,
void __rcu **slots, unsigned char offset)
{
return rcu_dereference_protected(slots[offset], mt_write_locked(mt));
}
/*
* mas_slot_locked() - Get the slot value when holding the maple tree lock.
* @mas: The maple state
* @slots: The pointer to the slots
* @offset: The offset into the slots array to fetch
*
* Return: The entry stored in @slots at the @offset.
*/
static __always_inline void *mas_slot_locked(struct ma_state *mas,
void __rcu **slots, unsigned char offset)
{
return mt_slot_locked(mas->tree, slots, offset);
}
/*
* mas_slot() - Get the slot value when not holding the maple tree lock.
* @mas: The maple state
* @slots: The pointer to the slots
* @offset: The offset into the slots array to fetch
*
* Return: The entry stored in @slots at the @offset
*/
static __always_inline void *mas_slot(struct ma_state *mas, void __rcu **slots,
unsigned char offset)
{
return mt_slot(mas->tree, slots, offset);
}
/*
* mas_root() - Get the maple tree root.
* @mas: The maple state.
*
* Return: The pointer to the root of the tree
*/
static __always_inline void *mas_root(struct ma_state *mas)
{
return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree));
}
static inline void *mt_root_locked(struct maple_tree *mt)
{
return rcu_dereference_protected(mt->ma_root, mt_write_locked(mt));
}
/*
* mas_root_locked() - Get the maple tree root when holding the maple tree lock.
* @mas: The maple state.
*
* Return: The pointer to the root of the tree
*/
static inline void *mas_root_locked(struct ma_state *mas)
{
return mt_root_locked(mas->tree);
}
static inline struct maple_metadata *ma_meta(struct maple_node *mn,
enum maple_type mt)
{
switch (mt) {
case maple_arange_64:
return &mn->ma64.meta;
default:
return &mn->mr64.meta;
}
}
/*
* ma_set_meta() - Set the metadata information of a node.
* @mn: The maple node
* @mt: The maple node type
* @offset: The offset of the highest sub-gap in this node.
* @end: The end of the data in this node.
*/
static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt,
unsigned char offset, unsigned char end)
{
struct maple_metadata *meta = ma_meta(mn, mt); meta->gap = offset;
meta->end = end;
}
/*
* mt_clear_meta() - clear the metadata information of a node, if it exists
* @mt: The maple tree
* @mn: The maple node
* @type: The maple node type
* @offset: The offset of the highest sub-gap in this node.
* @end: The end of the data in this node.
*/
static inline void mt_clear_meta(struct maple_tree *mt, struct maple_node *mn,
enum maple_type type)
{
struct maple_metadata *meta;
unsigned long *pivots;
void __rcu **slots;
void *next;
switch (type) {
case maple_range_64:
pivots = mn->mr64.pivot;
if (unlikely(pivots[MAPLE_RANGE64_SLOTS - 2])) {
slots = mn->mr64.slot;
next = mt_slot_locked(mt, slots,
MAPLE_RANGE64_SLOTS - 1);
if (unlikely((mte_to_node(next) &&
mte_node_type(next))))
return; /* no metadata, could be node */
}
fallthrough;
case maple_arange_64:
meta = ma_meta(mn, type);
break;
default:
return;
}
meta->gap = 0;
meta->end = 0;
}
/*
* ma_meta_end() - Get the data end of a node from the metadata
* @mn: The maple node
* @mt: The maple node type
*/
static inline unsigned char ma_meta_end(struct maple_node *mn,
enum maple_type mt)
{
struct maple_metadata *meta = ma_meta(mn, mt); return meta->end;
}
/*
* ma_meta_gap() - Get the largest gap location of a node from the metadata
* @mn: The maple node
*/
static inline unsigned char ma_meta_gap(struct maple_node *mn)
{
return mn->ma64.meta.gap;
}
/*
* ma_set_meta_gap() - Set the largest gap location in a nodes metadata
* @mn: The maple node
* @mn: The maple node type
* @offset: The location of the largest gap.
*/
static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt,
unsigned char offset)
{
struct maple_metadata *meta = ma_meta(mn, mt); meta->gap = offset;}
/*
* mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.
* @mat - the ma_topiary, a linked list of dead nodes.
* @dead_enode - the node to be marked as dead and added to the tail of the list
*
* Add the @dead_enode to the linked list in @mat.
*/
static inline void mat_add(struct ma_topiary *mat,
struct maple_enode *dead_enode)
{
mte_set_node_dead(dead_enode);
mte_to_mat(dead_enode)->next = NULL;
if (!mat->tail) {
mat->tail = mat->head = dead_enode;
return;
}
mte_to_mat(mat->tail)->next = dead_enode;
mat->tail = dead_enode;
}
static void mt_free_walk(struct rcu_head *head);
static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt,
bool free);
/*
* mas_mat_destroy() - Free all nodes and subtrees in a dead list.
* @mas - the maple state
* @mat - the ma_topiary linked list of dead nodes to free.
*
* Destroy walk a dead list.
*/
static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat)
{
struct maple_enode *next;
struct maple_node *node;
bool in_rcu = mt_in_rcu(mas->tree);
while (mat->head) {
next = mte_to_mat(mat->head)->next;
node = mte_to_node(mat->head);
mt_destroy_walk(mat->head, mas->tree, !in_rcu);
if (in_rcu)
call_rcu(&node->rcu, mt_free_walk);
mat->head = next;
}
}
/*
* mas_descend() - Descend into the slot stored in the ma_state.
* @mas - the maple state.
*
* Note: Not RCU safe, only use in write side or debug code.
*/
static inline void mas_descend(struct ma_state *mas)
{
enum maple_type type;
unsigned long *pivots;
struct maple_node *node;
void __rcu **slots;
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type); slots = ma_slots(node, type); if (mas->offset) mas->min = pivots[mas->offset - 1] + 1; mas->max = mas_safe_pivot(mas, pivots, mas->offset, type); mas->node = mas_slot(mas, slots, mas->offset);
}
/*
* mte_set_gap() - Set a maple node gap.
* @mn: The encoded maple node
* @gap: The offset of the gap to set
* @val: The gap value
*/
static inline void mte_set_gap(const struct maple_enode *mn,
unsigned char gap, unsigned long val)
{
switch (mte_node_type(mn)) {
default:
break;
case maple_arange_64:
mte_to_node(mn)->ma64.gap[gap] = val;
break;
}
}
/*
* mas_ascend() - Walk up a level of the tree.
* @mas: The maple state
*
* Sets the @mas->max and @mas->min to the correct values when walking up. This
* may cause several levels of walking up to find the correct min and max.
* May find a dead node which will cause a premature return.
* Return: 1 on dead node, 0 otherwise
*/
static int mas_ascend(struct ma_state *mas)
{
struct maple_enode *p_enode; /* parent enode. */
struct maple_enode *a_enode; /* ancestor enode. */
struct maple_node *a_node; /* ancestor node. */
struct maple_node *p_node; /* parent node. */
unsigned char a_slot;
enum maple_type a_type;
unsigned long min, max;
unsigned long *pivots;
bool set_max = false, set_min = false;
a_node = mas_mn(mas);
if (ma_is_root(a_node)) {
mas->offset = 0; return 0;
}
p_node = mte_parent(mas->node);
if (unlikely(a_node == p_node))
return 1; a_type = mas_parent_type(mas, mas->node); mas->offset = mte_parent_slot(mas->node); a_enode = mt_mk_node(p_node, a_type);
/* Check to make sure all parent information is still accurate */
if (p_node != mte_parent(mas->node))
return 1;
mas->node = a_enode;
if (mte_is_root(a_enode)) {
mas->max = ULONG_MAX;
mas->min = 0;
return 0;
}
min = 0;
max = ULONG_MAX;
if (!mas->offset) { min = mas->min;
set_min = true;
}
if (mas->max == ULONG_MAX)
set_max = true;
do {
p_enode = a_enode;
a_type = mas_parent_type(mas, p_enode);
a_node = mte_parent(p_enode);
a_slot = mte_parent_slot(p_enode); a_enode = mt_mk_node(a_node, a_type); pivots = ma_pivots(a_node, a_type); if (unlikely(ma_dead_node(a_node)))
return 1;
if (!set_min && a_slot) {
set_min = true;
min = pivots[a_slot - 1] + 1;
}
if (!set_max && a_slot < mt_pivots[a_type]) {
set_max = true;
max = pivots[a_slot];
}
if (unlikely(ma_dead_node(a_node)))
return 1;
if (unlikely(ma_is_root(a_node)))
break;
} while (!set_min || !set_max); mas->max = max;
mas->min = min;
return 0;
}
/*
* mas_pop_node() - Get a previously allocated maple node from the maple state.
* @mas: The maple state
*
* Return: A pointer to a maple node.
*/
static inline struct maple_node *mas_pop_node(struct ma_state *mas)
{
struct maple_alloc *ret, *node = mas->alloc;
unsigned long total = mas_allocated(mas);
unsigned int req = mas_alloc_req(mas);
/* nothing or a request pending. */
if (WARN_ON(!total))
return NULL;
if (total == 1) {
/* single allocation in this ma_state */
mas->alloc = NULL;
ret = node;
goto single_node;
}
if (node->node_count == 1) {
/* Single allocation in this node. */
mas->alloc = node->slot[0];
mas->alloc->total = node->total - 1;
ret = node;
goto new_head;
}
node->total--;
ret = node->slot[--node->node_count];
node->slot[node->node_count] = NULL;
single_node:
new_head:
if (req) {
req++;
mas_set_alloc_req(mas, req);
}
memset(ret, 0, sizeof(*ret));
return (struct maple_node *)ret;
}
/*
* mas_push_node() - Push a node back on the maple state allocation.
* @mas: The maple state
* @used: The used maple node
*
* Stores the maple node back into @mas->alloc for reuse. Updates allocated and
* requested node count as necessary.
*/
static inline void mas_push_node(struct ma_state *mas, struct maple_node *used)
{
struct maple_alloc *reuse = (struct maple_alloc *)used;
struct maple_alloc *head = mas->alloc;
unsigned long count;
unsigned int requested = mas_alloc_req(mas);
count = mas_allocated(mas);
reuse->request_count = 0;
reuse->node_count = 0;
if (count && (head->node_count < MAPLE_ALLOC_SLOTS)) {
head->slot[head->node_count++] = reuse;
head->total++;
goto done;
}
reuse->total = 1;
if ((head) && !((unsigned long)head & 0x1)) {
reuse->slot[0] = head;
reuse->node_count = 1;
reuse->total += head->total;
}
mas->alloc = reuse;
done:
if (requested > 1)
mas_set_alloc_req(mas, requested - 1);
}
/*
* mas_alloc_nodes() - Allocate nodes into a maple state
* @mas: The maple state
* @gfp: The GFP Flags
*/
static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp)
{
struct maple_alloc *node;
unsigned long allocated = mas_allocated(mas);
unsigned int requested = mas_alloc_req(mas);
unsigned int count;
void **slots = NULL;
unsigned int max_req = 0;
if (!requested)
return;
mas_set_alloc_req(mas, 0);
if (mas->mas_flags & MA_STATE_PREALLOC) {
if (allocated)
return;
BUG_ON(!allocated);
WARN_ON(!allocated);
}
if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS) {
node = (struct maple_alloc *)mt_alloc_one(gfp);
if (!node)
goto nomem_one;
if (allocated) {
node->slot[0] = mas->alloc;
node->node_count = 1;
} else {
node->node_count = 0;
}
mas->alloc = node;
node->total = ++allocated;
requested--;
}
node = mas->alloc;
node->request_count = 0;
while (requested) {
max_req = MAPLE_ALLOC_SLOTS - node->node_count;
slots = (void **)&node->slot[node->node_count];
max_req = min(requested, max_req);
count = mt_alloc_bulk(gfp, max_req, slots);
if (!count)
goto nomem_bulk;
if (node->node_count == 0) {
node->slot[0]->node_count = 0;
node->slot[0]->request_count = 0;
}
node->node_count += count;
allocated += count;
node = node->slot[0];
requested -= count;
}
mas->alloc->total = allocated;
return;
nomem_bulk:
/* Clean up potential freed allocations on bulk failure */
memset(slots, 0, max_req * sizeof(unsigned long));
nomem_one:
mas_set_alloc_req(mas, requested);
if (mas->alloc && !(((unsigned long)mas->alloc & 0x1)))
mas->alloc->total = allocated;
mas_set_err(mas, -ENOMEM);
}
/*
* mas_free() - Free an encoded maple node
* @mas: The maple state
* @used: The encoded maple node to free.
*
* Uses rcu free if necessary, pushes @used back on the maple state allocations
* otherwise.
*/
static inline void mas_free(struct ma_state *mas, struct maple_enode *used)
{
struct maple_node *tmp = mte_to_node(used);
if (mt_in_rcu(mas->tree))
ma_free_rcu(tmp);
else
mas_push_node(mas, tmp);
}
/*
* mas_node_count_gfp() - Check if enough nodes are allocated and request more
* if there is not enough nodes.
* @mas: The maple state
* @count: The number of nodes needed
* @gfp: the gfp flags
*/
static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp)
{
unsigned long allocated = mas_allocated(mas);
if (allocated < count) {
mas_set_alloc_req(mas, count - allocated);
mas_alloc_nodes(mas, gfp);
}
}
/*
* mas_node_count() - Check if enough nodes are allocated and request more if
* there is not enough nodes.
* @mas: The maple state
* @count: The number of nodes needed
*
* Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags.
*/
static void mas_node_count(struct ma_state *mas, int count)
{
return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN);
}
/*
* mas_start() - Sets up maple state for operations.
* @mas: The maple state.
*
* If mas->status == mas_start, then set the min, max and depth to
* defaults.
*
* Return:
* - If mas->node is an error or not mas_start, return NULL.
* - If it's an empty tree: NULL & mas->status == ma_none
* - If it's a single entry: The entry & mas->status == mas_root
* - If it's a tree: NULL & mas->status == safe root node.
*/
static inline struct maple_enode *mas_start(struct ma_state *mas)
{
if (likely(mas_is_start(mas))) {
struct maple_enode *root;
mas->min = 0;
mas->max = ULONG_MAX;
retry:
mas->depth = 0; root = mas_root(mas);
/* Tree with nodes */
if (likely(xa_is_node(root))) { mas->depth = 1;
mas->status = ma_active;
mas->node = mte_safe_root(root);
mas->offset = 0;
if (mte_dead_node(mas->node))
goto retry;
return NULL;
}
/* empty tree */
if (unlikely(!root)) { mas->node = NULL;
mas->status = ma_none;
mas->offset = MAPLE_NODE_SLOTS;
return NULL;
}
/* Single entry tree */
mas->status = ma_root;
mas->offset = MAPLE_NODE_SLOTS;
/* Single entry tree. */
if (mas->index > 0)
return NULL;
return root;
}
return NULL;
}
/*
* ma_data_end() - Find the end of the data in a node.
* @node: The maple node
* @type: The maple node type
* @pivots: The array of pivots in the node
* @max: The maximum value in the node
*
* Uses metadata to find the end of the data when possible.
* Return: The zero indexed last slot with data (may be null).
*/
static __always_inline unsigned char ma_data_end(struct maple_node *node,
enum maple_type type, unsigned long *pivots, unsigned long max)
{
unsigned char offset;
if (!pivots)
return 0;
if (type == maple_arange_64) return ma_meta_end(node, type); offset = mt_pivots[type] - 1;
if (likely(!pivots[offset]))
return ma_meta_end(node, type); if (likely(pivots[offset] == max))
return offset;
return mt_pivots[type];
}
/*
* mas_data_end() - Find the end of the data (slot).
* @mas: the maple state
*
* This method is optimized to check the metadata of a node if the node type
* supports data end metadata.
*
* Return: The zero indexed last slot with data (may be null).
*/
static inline unsigned char mas_data_end(struct ma_state *mas)
{
enum maple_type type;
struct maple_node *node;
unsigned char offset;
unsigned long *pivots;
type = mte_node_type(mas->node);
node = mas_mn(mas);
if (type == maple_arange_64) return ma_meta_end(node, type); pivots = ma_pivots(node, type); if (unlikely(ma_dead_node(node)))
return 0;
offset = mt_pivots[type] - 1;
if (likely(!pivots[offset]))
return ma_meta_end(node, type); if (likely(pivots[offset] == mas->max))
return offset;
return mt_pivots[type];
}
/*
* mas_leaf_max_gap() - Returns the largest gap in a leaf node
* @mas - the maple state
*
* Return: The maximum gap in the leaf.
*/
static unsigned long mas_leaf_max_gap(struct ma_state *mas)
{
enum maple_type mt;
unsigned long pstart, gap, max_gap;
struct maple_node *mn;
unsigned long *pivots;
void __rcu **slots;
unsigned char i;
unsigned char max_piv;
mt = mte_node_type(mas->node);
mn = mas_mn(mas);
slots = ma_slots(mn, mt);
max_gap = 0;
if (unlikely(ma_is_dense(mt))) {
gap = 0;
for (i = 0; i < mt_slots[mt]; i++) { if (slots[i]) { if (gap > max_gap)
max_gap = gap;
gap = 0;
} else {
gap++;
}
}
if (gap > max_gap)
max_gap = gap;
return max_gap;
}
/*
* Check the first implied pivot optimizes the loop below and slot 1 may
* be skipped if there is a gap in slot 0.
*/
pivots = ma_pivots(mn, mt);
if (likely(!slots[0])) { max_gap = pivots[0] - mas->min + 1;
i = 2;
} else {
i = 1;
}
/* reduce max_piv as the special case is checked before the loop */
max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1;
/*
* Check end implied pivot which can only be a gap on the right most
* node.
*/
if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) { gap = ULONG_MAX - pivots[max_piv];
if (gap > max_gap)
max_gap = gap;
if (max_gap > pivots[max_piv] - mas->min)
return max_gap;
}
for (; i <= max_piv; i++) {
/* data == no gap. */
if (likely(slots[i]))
continue;
pstart = pivots[i - 1];
gap = pivots[i] - pstart;
if (gap > max_gap)
max_gap = gap;
/* There cannot be two gaps in a row. */
i++;
}
return max_gap;
}
/*
* ma_max_gap() - Get the maximum gap in a maple node (non-leaf)
* @node: The maple node
* @gaps: The pointer to the gaps
* @mt: The maple node type
* @*off: Pointer to store the offset location of the gap.
*
* Uses the metadata data end to scan backwards across set gaps.
*
* Return: The maximum gap value
*/
static inline unsigned long
ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt,
unsigned char *off)
{
unsigned char offset, i;
unsigned long max_gap = 0;
i = offset = ma_meta_end(node, mt);
do {
if (gaps[i] > max_gap) {
max_gap = gaps[i];
offset = i;
}
} while (i--);
*off = offset;
return max_gap;
}
/*
* mas_max_gap() - find the largest gap in a non-leaf node and set the slot.
* @mas: The maple state.
*
* Return: The gap value.
*/
static inline unsigned long mas_max_gap(struct ma_state *mas)
{
unsigned long *gaps;
unsigned char offset;
enum maple_type mt;
struct maple_node *node;
mt = mte_node_type(mas->node);
if (ma_is_leaf(mt))
return mas_leaf_max_gap(mas);
node = mas_mn(mas);
MAS_BUG_ON(mas, mt != maple_arange_64);
offset = ma_meta_gap(node);
gaps = ma_gaps(node, mt);
return gaps[offset];
}
/*
* mas_parent_gap() - Set the parent gap and any gaps above, as needed
* @mas: The maple state
* @offset: The gap offset in the parent to set
* @new: The new gap value.
*
* Set the parent gap then continue to set the gap upwards, using the metadata
* of the parent to see if it is necessary to check the node above.
*/
static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset,
unsigned long new)
{
unsigned long meta_gap = 0;
struct maple_node *pnode;
struct maple_enode *penode;
unsigned long *pgaps;
unsigned char meta_offset;
enum maple_type pmt;
pnode = mte_parent(mas->node); pmt = mas_parent_type(mas, mas->node); penode = mt_mk_node(pnode, pmt);
pgaps = ma_gaps(pnode, pmt);
ascend:
MAS_BUG_ON(mas, pmt != maple_arange_64); meta_offset = ma_meta_gap(pnode);
meta_gap = pgaps[meta_offset];
pgaps[offset] = new;
if (meta_gap == new)
return;
if (offset != meta_offset) { if (meta_gap > new)
return;
ma_set_meta_gap(pnode, pmt, offset); } else if (new < meta_gap) { new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); ma_set_meta_gap(pnode, pmt, meta_offset);
}
if (ma_is_root(pnode))
return;
/* Go to the parent node. */
pnode = mte_parent(penode); pmt = mas_parent_type(mas, penode); pgaps = ma_gaps(pnode, pmt); offset = mte_parent_slot(penode); penode = mt_mk_node(pnode, pmt);
goto ascend;
}
/*
* mas_update_gap() - Update a nodes gaps and propagate up if necessary.
* @mas - the maple state.
*/
static inline void mas_update_gap(struct ma_state *mas)
{
unsigned char pslot;
unsigned long p_gap;
unsigned long max_gap;
if (!mt_is_alloc(mas->tree))
return;
if (mte_is_root(mas->node))
return;
max_gap = mas_max_gap(mas); pslot = mte_parent_slot(mas->node); p_gap = ma_gaps(mte_parent(mas->node),
mas_parent_type(mas, mas->node))[pslot];
if (p_gap != max_gap)
mas_parent_gap(mas, pslot, max_gap);
}
/*
* mas_adopt_children() - Set the parent pointer of all nodes in @parent to
* @parent with the slot encoded.
* @mas - the maple state (for the tree)
* @parent - the maple encoded node containing the children.
*/
static inline void mas_adopt_children(struct ma_state *mas,
struct maple_enode *parent)
{
enum maple_type type = mte_node_type(parent);
struct maple_node *node = mte_to_node(parent);
void __rcu **slots = ma_slots(node, type);
unsigned long *pivots = ma_pivots(node, type);
struct maple_enode *child;
unsigned char offset;
offset = ma_data_end(node, type, pivots, mas->max);
do {
child = mas_slot_locked(mas, slots, offset);
mas_set_parent(mas, child, parent, offset);
} while (offset--);
}
/*
* mas_put_in_tree() - Put a new node in the tree, smp_wmb(), and mark the old
* node as dead.
* @mas - the maple state with the new node
* @old_enode - The old maple encoded node to replace.
*/
static inline void mas_put_in_tree(struct ma_state *mas,
struct maple_enode *old_enode)
__must_hold(mas->tree->ma_lock)
{
unsigned char offset;
void __rcu **slots;
if (mte_is_root(mas->node)) {
mas_mn(mas)->parent = ma_parent_ptr(mas_tree_parent(mas));
rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
mas_set_height(mas);
} else {
offset = mte_parent_slot(mas->node);
slots = ma_slots(mte_parent(mas->node),
mas_parent_type(mas, mas->node));
rcu_assign_pointer(slots[offset], mas->node);
}
mte_set_node_dead(old_enode);
}
/*
* mas_replace_node() - Replace a node by putting it in the tree, marking it
* dead, and freeing it.
* the parent encoding to locate the maple node in the tree.
* @mas - the ma_state with @mas->node pointing to the new node.
* @old_enode - The old maple encoded node.
*/
static inline void mas_replace_node(struct ma_state *mas,
struct maple_enode *old_enode)
__must_hold(mas->tree->ma_lock)
{
mas_put_in_tree(mas, old_enode);
mas_free(mas, old_enode);
}
/*
* mas_find_child() - Find a child who has the parent @mas->node.
* @mas: the maple state with the parent.
* @child: the maple state to store the child.
*/
static inline bool mas_find_child(struct ma_state *mas, struct ma_state *child)
__must_hold(mas->tree->ma_lock)
{
enum maple_type mt;
unsigned char offset;
unsigned char end;
unsigned long *pivots;
struct maple_enode *entry;
struct maple_node *node;
void __rcu **slots;
mt = mte_node_type(mas->node);
node = mas_mn(mas);
slots = ma_slots(node, mt);
pivots = ma_pivots(node, mt);
end = ma_data_end(node, mt, pivots, mas->max);
for (offset = mas->offset; offset <= end; offset++) {
entry = mas_slot_locked(mas, slots, offset);
if (mte_parent(entry) == node) {
*child = *mas;
mas->offset = offset + 1;
child->offset = offset;
mas_descend(child);
child->offset = 0;
return true;
}
}
return false;
}
/*
* mab_shift_right() - Shift the data in mab right. Note, does not clean out the
* old data or set b_node->b_end.
* @b_node: the maple_big_node
* @shift: the shift count
*/
static inline void mab_shift_right(struct maple_big_node *b_node,
unsigned char shift)
{
unsigned long size = b_node->b_end * sizeof(unsigned long);
memmove(b_node->pivot + shift, b_node->pivot, size);
memmove(b_node->slot + shift, b_node->slot, size);
if (b_node->type == maple_arange_64)
memmove(b_node->gap + shift, b_node->gap, size);
}
/*
* mab_middle_node() - Check if a middle node is needed (unlikely)
* @b_node: the maple_big_node that contains the data.
* @size: the amount of data in the b_node
* @split: the potential split location
* @slot_count: the size that can be stored in a single node being considered.
*
* Return: true if a middle node is required.
*/
static inline bool mab_middle_node(struct maple_big_node *b_node, int split,
unsigned char slot_count)
{
unsigned char size = b_node->b_end;
if (size >= 2 * slot_count)
return true;
if (!b_node->slot[split] && (size >= 2 * slot_count - 1))
return true;
return false;
}
/*
* mab_no_null_split() - ensure the split doesn't fall on a NULL
* @b_node: the maple_big_node with the data
* @split: the suggested split location
* @slot_count: the number of slots in the node being considered.
*
* Return: the split location.
*/
static inline int mab_no_null_split(struct maple_big_node *b_node,
unsigned char split, unsigned char slot_count)
{
if (!b_node->slot[split]) {
/*
* If the split is less than the max slot && the right side will
* still be sufficient, then increment the split on NULL.
*/
if ((split < slot_count - 1) &&
(b_node->b_end - split) > (mt_min_slots[b_node->type]))
split++;
else
split--;
}
return split;
}
/*
* mab_calc_split() - Calculate the split location and if there needs to be two
* splits.
* @bn: The maple_big_node with the data
* @mid_split: The second split, if required. 0 otherwise.
*
* Return: The first split location. The middle split is set in @mid_split.
*/
static inline int mab_calc_split(struct ma_state *mas,
struct maple_big_node *bn, unsigned char *mid_split, unsigned long min)
{
unsigned char b_end = bn->b_end;
int split = b_end / 2; /* Assume equal split. */
unsigned char slot_min, slot_count = mt_slots[bn->type];
/*
* To support gap tracking, all NULL entries are kept together and a node cannot
* end on a NULL entry, with the exception of the left-most leaf. The
* limitation means that the split of a node must be checked for this condition
* and be able to put more data in one direction or the other.
*/
if (unlikely((mas->mas_flags & MA_STATE_BULK))) {
*mid_split = 0;
split = b_end - mt_min_slots[bn->type];
if (!ma_is_leaf(bn->type))
return split;
mas->mas_flags |= MA_STATE_REBALANCE;
if (!bn->slot[split])
split--;
return split;
}
/*
* Although extremely rare, it is possible to enter what is known as the 3-way
* split scenario. The 3-way split comes about by means of a store of a range
* that overwrites the end and beginning of two full nodes. The result is a set
* of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can
* also be located in different parent nodes which are also full. This can
* carry upwards all the way to the root in the worst case.
*/
if (unlikely(mab_middle_node(bn, split, slot_count))) {
split = b_end / 3;
*mid_split = split * 2;
} else {
slot_min = mt_min_slots[bn->type];
*mid_split = 0;
/*
* Avoid having a range less than the slot count unless it
* causes one node to be deficient.
* NOTE: mt_min_slots is 1 based, b_end and split are zero.
*/
while ((split < slot_count - 1) &&
((bn->pivot[split] - min) < slot_count - 1) &&
(b_end - split > slot_min))
split++;
}
/* Avoid ending a node on a NULL entry */
split = mab_no_null_split(bn, split, slot_count);
if (unlikely(*mid_split))
*mid_split = mab_no_null_split(bn, *mid_split, slot_count);
return split;
}
/*
* mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node
* and set @b_node->b_end to the next free slot.
* @mas: The maple state
* @mas_start: The starting slot to copy
* @mas_end: The end slot to copy (inclusively)
* @b_node: The maple_big_node to place the data
* @mab_start: The starting location in maple_big_node to store the data.
*/
static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start,
unsigned char mas_end, struct maple_big_node *b_node,
unsigned char mab_start)
{
enum maple_type mt;
struct maple_node *node;
void __rcu **slots;
unsigned long *pivots, *gaps;
int i = mas_start, j = mab_start;
unsigned char piv_end;
node = mas_mn(mas);
mt = mte_node_type(mas->node);
pivots = ma_pivots(node, mt);
if (!i) {
b_node->pivot[j] = pivots[i++];
if (unlikely(i > mas_end))
goto complete;
j++;
}
piv_end = min(mas_end, mt_pivots[mt]);
for (; i < piv_end; i++, j++) {
b_node->pivot[j] = pivots[i];
if (unlikely(!b_node->pivot[j]))
break;
if (unlikely(mas->max == b_node->pivot[j]))
goto complete;
}
if (likely(i <= mas_end))
b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt);
complete:
b_node->b_end = ++j;
j -= mab_start;
slots = ma_slots(node, mt);
memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j);
if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) {
gaps = ma_gaps(node, mt);
memcpy(b_node->gap + mab_start, gaps + mas_start,
sizeof(unsigned long) * j);
}
}
/*
* mas_leaf_set_meta() - Set the metadata of a leaf if possible.
* @node: The maple node
* @mt: The maple type
* @end: The node end
*/
static inline void mas_leaf_set_meta(struct maple_node *node,
enum maple_type mt, unsigned char end)
{
if (end < mt_slots[mt] - 1)
ma_set_meta(node, mt, 0, end);
}
/*
* mab_mas_cp() - Copy data from maple_big_node to a maple encoded node.
* @b_node: the maple_big_node that has the data
* @mab_start: the start location in @b_node.
* @mab_end: The end location in @b_node (inclusively)
* @mas: The maple state with the maple encoded node.
*/
static inline void mab_mas_cp(struct maple_big_node *b_node,
unsigned char mab_start, unsigned char mab_end,
struct ma_state *mas, bool new_max)
{
int i, j = 0;
enum maple_type mt = mte_node_type(mas->node);
struct maple_node *node = mte_to_node(mas->node);
void __rcu **slots = ma_slots(node, mt);
unsigned long *pivots = ma_pivots(node, mt);
unsigned long *gaps = NULL;
unsigned char end;
if (mab_end - mab_start > mt_pivots[mt])
mab_end--;
if (!pivots[mt_pivots[mt] - 1])
slots[mt_pivots[mt]] = NULL;
i = mab_start;
do {
pivots[j++] = b_node->pivot[i++];
} while (i <= mab_end && likely(b_node->pivot[i]));
memcpy(slots, b_node->slot + mab_start,
sizeof(void *) * (i - mab_start));
if (new_max)
mas->max = b_node->pivot[i - 1];
end = j - 1;
if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) {
unsigned long max_gap = 0;
unsigned char offset = 0;
gaps = ma_gaps(node, mt);
do {
gaps[--j] = b_node->gap[--i];
if (gaps[j] > max_gap) {
offset = j;
max_gap = gaps[j];
}
} while (j);
ma_set_meta(node, mt, offset, end);
} else {
mas_leaf_set_meta(node, mt, end);
}
}
/*
* mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert.
* @mas: The maple state
* @end: The maple node end
* @mt: The maple node type
*/
static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end,
enum maple_type mt)
{
if (!(mas->mas_flags & MA_STATE_BULK))
return;
if (mte_is_root(mas->node))
return;
if (end > mt_min_slots[mt]) {
mas->mas_flags &= ~MA_STATE_REBALANCE;
return;
}
}
/*
* mas_store_b_node() - Store an @entry into the b_node while also copying the
* data from a maple encoded node.
* @wr_mas: the maple write state
* @b_node: the maple_big_node to fill with data
* @offset_end: the offset to end copying
*
* Return: The actual end of the data stored in @b_node
*/
static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas,
struct maple_big_node *b_node, unsigned char offset_end)
{
unsigned char slot;
unsigned char b_end;
/* Possible underflow of piv will wrap back to 0 before use. */
unsigned long piv;
struct ma_state *mas = wr_mas->mas;
b_node->type = wr_mas->type;
b_end = 0;
slot = mas->offset;
if (slot) {
/* Copy start data up to insert. */
mas_mab_cp(mas, 0, slot - 1, b_node, 0);
b_end = b_node->b_end;
piv = b_node->pivot[b_end - 1];
} else
piv = mas->min - 1;
if (piv + 1 < mas->index) {
/* Handle range starting after old range */
b_node->slot[b_end] = wr_mas->content;
if (!wr_mas->content)
b_node->gap[b_end] = mas->index - 1 - piv;
b_node->pivot[b_end++] = mas->index - 1;
}
/* Store the new entry. */
mas->offset = b_end;
b_node->slot[b_end] = wr_mas->entry;
b_node->pivot[b_end] = mas->last;
/* Appended. */
if (mas->last >= mas->max)
goto b_end;
/* Handle new range ending before old range ends */
piv = mas_safe_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type);
if (piv > mas->last) {
if (piv == ULONG_MAX)
mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type);
if (offset_end != slot)
wr_mas->content = mas_slot_locked(mas, wr_mas->slots,
offset_end);
b_node->slot[++b_end] = wr_mas->content;
if (!wr_mas->content)
b_node->gap[b_end] = piv - mas->last + 1;
b_node->pivot[b_end] = piv;
}
slot = offset_end + 1;
if (slot > mas->end)
goto b_end;
/* Copy end data to the end of the node. */
mas_mab_cp(mas, slot, mas->end + 1, b_node, ++b_end);
b_node->b_end--;
return;
b_end:
b_node->b_end = b_end;
}
/*
* mas_prev_sibling() - Find the previous node with the same parent.
* @mas: the maple state
*
* Return: True if there is a previous sibling, false otherwise.
*/
static inline bool mas_prev_sibling(struct ma_state *mas)
{
unsigned int p_slot = mte_parent_slot(mas->node);
if (mte_is_root(mas->node))
return false;
if (!p_slot)
return false;
mas_ascend(mas);
mas->offset = p_slot - 1;
mas_descend(mas);
return true;
}
/*
* mas_next_sibling() - Find the next node with the same parent.
* @mas: the maple state
*
* Return: true if there is a next sibling, false otherwise.
*/
static inline bool mas_next_sibling(struct ma_state *mas)
{
MA_STATE(parent, mas->tree, mas->index, mas->last);
if (mte_is_root(mas->node))
return false;
parent = *mas;
mas_ascend(&parent);
parent.offset = mte_parent_slot(mas->node) + 1;
if (parent.offset > mas_data_end(&parent))
return false;
*mas = parent;
mas_descend(mas);
return true;
}
/*
* mte_node_or_none() - Set the enode and state.
* @enode: The encoded maple node.
*
* Set the node to the enode and the status.
*/
static inline void mas_node_or_none(struct ma_state *mas,
struct maple_enode *enode)
{
if (enode) {
mas->node = enode;
mas->status = ma_active;
} else {
mas->node = NULL;
mas->status = ma_none;
}
}
/*
* mas_wr_node_walk() - Find the correct offset for the index in the @mas.
* @wr_mas: The maple write state
*
* Uses mas_slot_locked() and does not need to worry about dead nodes.
*/
static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
unsigned char count, offset;
if (unlikely(ma_is_dense(wr_mas->type))) {
wr_mas->r_max = wr_mas->r_min = mas->index;
mas->offset = mas->index = mas->min;
return;
}
wr_mas->node = mas_mn(wr_mas->mas); wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); count = mas->end = ma_data_end(wr_mas->node, wr_mas->type,
wr_mas->pivots, mas->max);
offset = mas->offset;
while (offset < count && mas->index > wr_mas->pivots[offset]) offset++; wr_mas->r_max = offset < count ? wr_mas->pivots[offset] : mas->max; wr_mas->r_min = mas_safe_min(mas, wr_mas->pivots, offset);
wr_mas->offset_end = mas->offset = offset;
}
/*
* mast_rebalance_next() - Rebalance against the next node
* @mast: The maple subtree state
* @old_r: The encoded maple node to the right (next node).
*/
static inline void mast_rebalance_next(struct maple_subtree_state *mast)
{
unsigned char b_end = mast->bn->b_end;
mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node),
mast->bn, b_end);
mast->orig_r->last = mast->orig_r->max;
}
/*
* mast_rebalance_prev() - Rebalance against the previous node
* @mast: The maple subtree state
* @old_l: The encoded maple node to the left (previous node)
*/
static inline void mast_rebalance_prev(struct maple_subtree_state *mast)
{
unsigned char end = mas_data_end(mast->orig_l) + 1;
unsigned char b_end = mast->bn->b_end;
mab_shift_right(mast->bn, end);
mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0);
mast->l->min = mast->orig_l->min;
mast->orig_l->index = mast->orig_l->min;
mast->bn->b_end = end + b_end;
mast->l->offset += end;
}
/*
* mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring
* the node to the right. Checking the nodes to the right then the left at each
* level upwards until root is reached.
* Data is copied into the @mast->bn.
* @mast: The maple_subtree_state.
*/
static inline
bool mast_spanning_rebalance(struct maple_subtree_state *mast)
{
struct ma_state r_tmp = *mast->orig_r;
struct ma_state l_tmp = *mast->orig_l;
unsigned char depth = 0;
do {
mas_ascend(mast->orig_r);
mas_ascend(mast->orig_l);
depth++;
if (mast->orig_r->offset < mas_data_end(mast->orig_r)) {
mast->orig_r->offset++;
do {
mas_descend(mast->orig_r);
mast->orig_r->offset = 0;
} while (--depth);
mast_rebalance_next(mast);
*mast->orig_l = l_tmp;
return true;
} else if (mast->orig_l->offset != 0) {
mast->orig_l->offset--;
do {
mas_descend(mast->orig_l);
mast->orig_l->offset =
mas_data_end(mast->orig_l);
} while (--depth);
mast_rebalance_prev(mast);
*mast->orig_r = r_tmp;
return true;
}
} while (!mte_is_root(mast->orig_r->node));
*mast->orig_r = r_tmp;
*mast->orig_l = l_tmp;
return false;
}
/*
* mast_ascend() - Ascend the original left and right maple states.
* @mast: the maple subtree state.
*
* Ascend the original left and right sides. Set the offsets to point to the
* data already in the new tree (@mast->l and @mast->r).
*/
static inline void mast_ascend(struct maple_subtree_state *mast)
{
MA_WR_STATE(wr_mas, mast->orig_r, NULL);
mas_ascend(mast->orig_l);
mas_ascend(mast->orig_r);
mast->orig_r->offset = 0;
mast->orig_r->index = mast->r->max;
/* last should be larger than or equal to index */
if (mast->orig_r->last < mast->orig_r->index)
mast->orig_r->last = mast->orig_r->index;
wr_mas.type = mte_node_type(mast->orig_r->node);
mas_wr_node_walk(&wr_mas);
/* Set up the left side of things */
mast->orig_l->offset = 0;
mast->orig_l->index = mast->l->min;
wr_mas.mas = mast->orig_l;
wr_mas.type = mte_node_type(mast->orig_l->node);
mas_wr_node_walk(&wr_mas);
mast->bn->type = wr_mas.type;
}
/*
* mas_new_ma_node() - Create and return a new maple node. Helper function.
* @mas: the maple state with the allocations.
* @b_node: the maple_big_node with the type encoding.
*
* Use the node type from the maple_big_node to allocate a new node from the
* ma_state. This function exists mainly for code readability.
*
* Return: A new maple encoded node
*/
static inline struct maple_enode
*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node)
{
return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type);
}
/*
* mas_mab_to_node() - Set up right and middle nodes
*
* @mas: the maple state that contains the allocations.
* @b_node: the node which contains the data.
* @left: The pointer which will have the left node
* @right: The pointer which may have the right node
* @middle: the pointer which may have the middle node (rare)
* @mid_split: the split location for the middle node
*
* Return: the split of left.
*/
static inline unsigned char mas_mab_to_node(struct ma_state *mas,
struct maple_big_node *b_node, struct maple_enode **left,
struct maple_enode **right, struct maple_enode **middle,
unsigned char *mid_split, unsigned long min)
{
unsigned char split = 0;
unsigned char slot_count = mt_slots[b_node->type];
*left = mas_new_ma_node(mas, b_node);
*right = NULL;
*middle = NULL;
*mid_split = 0;
if (b_node->b_end < slot_count) {
split = b_node->b_end;
} else {
split = mab_calc_split(mas, b_node, mid_split, min);
*right = mas_new_ma_node(mas, b_node);
}
if (*mid_split)
*middle = mas_new_ma_node(mas, b_node);
return split;
}
/*
* mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end
* pointer.
* @b_node - the big node to add the entry
* @mas - the maple state to get the pivot (mas->max)
* @entry - the entry to add, if NULL nothing happens.
*/
static inline void mab_set_b_end(struct maple_big_node *b_node,
struct ma_state *mas,
void *entry)
{
if (!entry)
return;
b_node->slot[b_node->b_end] = entry;
if (mt_is_alloc(mas->tree))
b_node->gap[b_node->b_end] = mas_max_gap(mas);
b_node->pivot[b_node->b_end++] = mas->max;
}
/*
* mas_set_split_parent() - combine_then_separate helper function. Sets the parent
* of @mas->node to either @left or @right, depending on @slot and @split
*
* @mas - the maple state with the node that needs a parent
* @left - possible parent 1
* @right - possible parent 2
* @slot - the slot the mas->node was placed
* @split - the split location between @left and @right
*/
static inline void mas_set_split_parent(struct ma_state *mas,
struct maple_enode *left,
struct maple_enode *right,
unsigned char *slot, unsigned char split)
{
if (mas_is_none(mas))
return;
if ((*slot) <= split)
mas_set_parent(mas, mas->node, left, *slot);
else if (right)
mas_set_parent(mas, mas->node, right, (*slot) - split - 1);
(*slot)++;
}
/*
* mte_mid_split_check() - Check if the next node passes the mid-split
* @**l: Pointer to left encoded maple node.
* @**m: Pointer to middle encoded maple node.
* @**r: Pointer to right encoded maple node.
* @slot: The offset
* @*split: The split location.
* @mid_split: The middle split.
*/
static inline void mte_mid_split_check(struct maple_enode **l,
struct maple_enode **r,
struct maple_enode *right,
unsigned char slot,
unsigned char *split,
unsigned char mid_split)
{
if (*r == right)
return;
if (slot < mid_split)
return;
*l = *r;
*r = right;
*split = mid_split;
}
/*
* mast_set_split_parents() - Helper function to set three nodes parents. Slot
* is taken from @mast->l.
* @mast - the maple subtree state
* @left - the left node
* @right - the right node
* @split - the split location.
*/
static inline void mast_set_split_parents(struct maple_subtree_state *mast,
struct maple_enode *left,
struct maple_enode *middle,
struct maple_enode *right,
unsigned char split,
unsigned char mid_split)
{
unsigned char slot;
struct maple_enode *l = left;
struct maple_enode *r = right;
if (mas_is_none(mast->l))
return;
if (middle)
r = middle;
slot = mast->l->offset;
mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
mas_set_split_parent(mast->l, l, r, &slot, split);
mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
mas_set_split_parent(mast->m, l, r, &slot, split);
mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
mas_set_split_parent(mast->r, l, r, &slot, split);
}
/*
* mas_topiary_node() - Dispose of a single node
* @mas: The maple state for pushing nodes
* @enode: The encoded maple node
* @in_rcu: If the tree is in rcu mode
*
* The node will either be RCU freed or pushed back on the maple state.
*/
static inline void mas_topiary_node(struct ma_state *mas,
struct ma_state *tmp_mas, bool in_rcu)
{
struct maple_node *tmp;
struct maple_enode *enode;
if (mas_is_none(tmp_mas))
return;
enode = tmp_mas->node;
tmp = mte_to_node(enode);
mte_set_node_dead(enode);
if (in_rcu)
ma_free_rcu(tmp);
else
mas_push_node(mas, tmp);
}
/*
* mas_topiary_replace() - Replace the data with new data, then repair the
* parent links within the new tree. Iterate over the dead sub-tree and collect
* the dead subtrees and topiary the nodes that are no longer of use.
*
* The new tree will have up to three children with the correct parent. Keep
* track of the new entries as they need to be followed to find the next level
* of new entries.
*
* The old tree will have up to three children with the old parent. Keep track
* of the old entries as they may have more nodes below replaced. Nodes within
* [index, last] are dead subtrees, others need to be freed and followed.
*
* @mas: The maple state pointing at the new data
* @old_enode: The maple encoded node being replaced
*
*/
static inline void mas_topiary_replace(struct ma_state *mas,
struct maple_enode *old_enode)
{
struct ma_state tmp[3], tmp_next[3];
MA_TOPIARY(subtrees, mas->tree);
bool in_rcu;
int i, n;
/* Place data in tree & then mark node as old */
mas_put_in_tree(mas, old_enode);
/* Update the parent pointers in the tree */
tmp[0] = *mas;
tmp[0].offset = 0;
tmp[1].status = ma_none;
tmp[2].status = ma_none;
while (!mte_is_leaf(tmp[0].node)) {
n = 0;
for (i = 0; i < 3; i++) {
if (mas_is_none(&tmp[i]))
continue;
while (n < 3) {
if (!mas_find_child(&tmp[i], &tmp_next[n]))
break;
n++;
}
mas_adopt_children(&tmp[i], tmp[i].node);
}
if (MAS_WARN_ON(mas, n == 0))
break;
while (n < 3)
tmp_next[n++].status = ma_none;
for (i = 0; i < 3; i++)
tmp[i] = tmp_next[i];
}
/* Collect the old nodes that need to be discarded */
if (mte_is_leaf(old_enode))
return mas_free(mas, old_enode);
tmp[0] = *mas;
tmp[0].offset = 0;
tmp[0].node = old_enode;
tmp[1].status = ma_none;
tmp[2].status = ma_none;
in_rcu = mt_in_rcu(mas->tree);
do {
n = 0;
for (i = 0; i < 3; i++) {
if (mas_is_none(&tmp[i]))
continue;
while (n < 3) {
if (!mas_find_child(&tmp[i], &tmp_next[n]))
break;
if ((tmp_next[n].min >= tmp_next->index) &&
(tmp_next[n].max <= tmp_next->last)) {
mat_add(&subtrees, tmp_next[n].node);
tmp_next[n].status = ma_none;
} else {
n++;
}
}
}
if (MAS_WARN_ON(mas, n == 0))
break;
while (n < 3)
tmp_next[n++].status = ma_none;
for (i = 0; i < 3; i++) {
mas_topiary_node(mas, &tmp[i], in_rcu);
tmp[i] = tmp_next[i];
}
} while (!mte_is_leaf(tmp[0].node));
for (i = 0; i < 3; i++)
mas_topiary_node(mas, &tmp[i], in_rcu);
mas_mat_destroy(mas, &subtrees);
}
/*
* mas_wmb_replace() - Write memory barrier and replace
* @mas: The maple state
* @old: The old maple encoded node that is being replaced.
*
* Updates gap as necessary.
*/
static inline void mas_wmb_replace(struct ma_state *mas,
struct maple_enode *old_enode)
{
/* Insert the new data in the tree */
mas_topiary_replace(mas, old_enode);
if (mte_is_leaf(mas->node))
return;
mas_update_gap(mas);
}
/*
* mast_cp_to_nodes() - Copy data out to nodes.
* @mast: The maple subtree state
* @left: The left encoded maple node
* @middle: The middle encoded maple node
* @right: The right encoded maple node
* @split: The location to split between left and (middle ? middle : right)
* @mid_split: The location to split between middle and right.
*/
static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
struct maple_enode *left, struct maple_enode *middle,
struct maple_enode *right, unsigned char split, unsigned char mid_split)
{
bool new_lmax = true;
mas_node_or_none(mast->l, left);
mas_node_or_none(mast->m, middle);
mas_node_or_none(mast->r, right);
mast->l->min = mast->orig_l->min;
if (split == mast->bn->b_end) {
mast->l->max = mast->orig_r->max;
new_lmax = false;
}
mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax);
if (middle) {
mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true);
mast->m->min = mast->bn->pivot[split] + 1;
split = mid_split;
}
mast->r->max = mast->orig_r->max;
if (right) {
mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false);
mast->r->min = mast->bn->pivot[split] + 1;
}
}
/*
* mast_combine_cp_left - Copy in the original left side of the tree into the
* combined data set in the maple subtree state big node.
* @mast: The maple subtree state
*/
static inline void mast_combine_cp_left(struct maple_subtree_state *mast)
{
unsigned char l_slot = mast->orig_l->offset;
if (!l_slot)
return;
mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0);
}
/*
* mast_combine_cp_right: Copy in the original right side of the tree into the
* combined data set in the maple subtree state big node.
* @mast: The maple subtree state
*/
static inline void mast_combine_cp_right(struct maple_subtree_state *mast)
{
if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max)
return;
mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1,
mt_slot_count(mast->orig_r->node), mast->bn,
mast->bn->b_end);
mast->orig_r->last = mast->orig_r->max;
}
/*
* mast_sufficient: Check if the maple subtree state has enough data in the big
* node to create at least one sufficient node
* @mast: the maple subtree state
*/
static inline bool mast_sufficient(struct maple_subtree_state *mast)
{
if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node))
return true;
return false;
}
/*
* mast_overflow: Check if there is too much data in the subtree state for a
* single node.
* @mast: The maple subtree state
*/
static inline bool mast_overflow(struct maple_subtree_state *mast)
{
if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node))
return true;
return false;
}
static inline void *mtree_range_walk(struct ma_state *mas)
{
unsigned long *pivots;
unsigned char offset;
struct maple_node *node;
struct maple_enode *next, *last;
enum maple_type type;
void __rcu **slots;
unsigned char end;
unsigned long max, min;
unsigned long prev_max, prev_min;
next = mas->node;
min = mas->min;
max = mas->max;
do {
last = next;
node = mte_to_node(next);
type = mte_node_type(next);
pivots = ma_pivots(node, type); end = ma_data_end(node, type, pivots, max);
prev_min = min;
prev_max = max;
if (pivots[0] >= mas->index) {
offset = 0;
max = pivots[0];
goto next;
}
offset = 1;
while (offset < end) { if (pivots[offset] >= mas->index) {
max = pivots[offset];
break;
}
offset++;
}
min = pivots[offset - 1] + 1;
next:
slots = ma_slots(node, type); next = mt_slot(mas->tree, slots, offset); if (unlikely(ma_dead_node(node)))
goto dead_node;
} while (!ma_is_leaf(type)); mas->end = end;
mas->offset = offset;
mas->index = min;
mas->last = max;
mas->min = prev_min;
mas->max = prev_max;
mas->node = last;
return (void *)next;
dead_node:
mas_reset(mas);
return NULL;
}
/*
* mas_spanning_rebalance() - Rebalance across two nodes which may not be peers.
* @mas: The starting maple state
* @mast: The maple_subtree_state, keeps track of 4 maple states.
* @count: The estimated count of iterations needed.
*
* Follow the tree upwards from @l_mas and @r_mas for @count, or until the root
* is hit. First @b_node is split into two entries which are inserted into the
* next iteration of the loop. @b_node is returned populated with the final
* iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the
* nodes that will remain active by using orig_l_mas->index and orig_l_mas->last
* to account of what has been copied into the new sub-tree. The update of
* orig_l_mas->last is used in mas_consume to find the slots that will need to
* be either freed or destroyed. orig_l_mas->depth keeps track of the height of
* the new sub-tree in case the sub-tree becomes the full tree.
*
* Return: the number of elements in b_node during the last loop.
*/
static int mas_spanning_rebalance(struct ma_state *mas,
struct maple_subtree_state *mast, unsigned char count)
{
unsigned char split, mid_split;
unsigned char slot = 0;
struct maple_enode *left = NULL, *middle = NULL, *right = NULL;
struct maple_enode *old_enode;
MA_STATE(l_mas, mas->tree, mas->index, mas->index);
MA_STATE(r_mas, mas->tree, mas->index, mas->last);
MA_STATE(m_mas, mas->tree, mas->index, mas->index);
/*
* The tree needs to be rebalanced and leaves need to be kept at the same level.
* Rebalancing is done by use of the ``struct maple_topiary``.
*/
mast->l = &l_mas;
mast->m = &m_mas;
mast->r = &r_mas;
l_mas.status = r_mas.status = m_mas.status = ma_none;
/* Check if this is not root and has sufficient data. */
if (((mast->orig_l->min != 0) || (mast->orig_r->max != ULONG_MAX)) &&
unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type]))
mast_spanning_rebalance(mast);
l_mas.depth = 0;
/*
* Each level of the tree is examined and balanced, pushing data to the left or
* right, or rebalancing against left or right nodes is employed to avoid
* rippling up the tree to limit the amount of churn. Once a new sub-section of
* the tree is created, there may be a mix of new and old nodes. The old nodes
* will have the incorrect parent pointers and currently be in two trees: the
* original tree and the partially new tree. To remedy the parent pointers in
* the old tree, the new data is swapped into the active tree and a walk down
* the tree is performed and the parent pointers are updated.
* See mas_topiary_replace() for more information.
*/
while (count--) {
mast->bn->b_end--;
mast->bn->type = mte_node_type(mast->orig_l->node);
split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle,
&mid_split, mast->orig_l->min);
mast_set_split_parents(mast, left, middle, right, split,
mid_split);
mast_cp_to_nodes(mast, left, middle, right, split, mid_split);
/*
* Copy data from next level in the tree to mast->bn from next
* iteration
*/
memset(mast->bn, 0, sizeof(struct maple_big_node));
mast->bn->type = mte_node_type(left);
l_mas.depth++;
/* Root already stored in l->node. */
if (mas_is_root_limits(mast->l))
goto new_root;
mast_ascend(mast);
mast_combine_cp_left(mast);
l_mas.offset = mast->bn->b_end;
mab_set_b_end(mast->bn, &l_mas, left);
mab_set_b_end(mast->bn, &m_mas, middle);
mab_set_b_end(mast->bn, &r_mas, right);
/* Copy anything necessary out of the right node. */
mast_combine_cp_right(mast);
mast->orig_l->last = mast->orig_l->max;
if (mast_sufficient(mast))
continue;
if (mast_overflow(mast))
continue;
/* May be a new root stored in mast->bn */
if (mas_is_root_limits(mast->orig_l))
break;
mast_spanning_rebalance(mast);
/* rebalancing from other nodes may require another loop. */
if (!count)
count++;
}
l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)),
mte_node_type(mast->orig_l->node));
l_mas.depth++;
mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true);
mas_set_parent(mas, left, l_mas.node, slot);
if (middle)
mas_set_parent(mas, middle, l_mas.node, ++slot);
if (right)
mas_set_parent(mas, right, l_mas.node, ++slot);
if (mas_is_root_limits(mast->l)) {
new_root:
mas_mn(mast->l)->parent = ma_parent_ptr(mas_tree_parent(mas));
while (!mte_is_root(mast->orig_l->node))
mast_ascend(mast);
} else {
mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent;
}
old_enode = mast->orig_l->node;
mas->depth = l_mas.depth;
mas->node = l_mas.node;
mas->min = l_mas.min;
mas->max = l_mas.max;
mas->offset = l_mas.offset;
mas_wmb_replace(mas, old_enode);
mtree_range_walk(mas);
return mast->bn->b_end;
}
/*
* mas_rebalance() - Rebalance a given node.
* @mas: The maple state
* @b_node: The big maple node.
*
* Rebalance two nodes into a single node or two new nodes that are sufficient.
* Continue upwards until tree is sufficient.
*
* Return: the number of elements in b_node during the last loop.
*/
static inline int mas_rebalance(struct ma_state *mas,
struct maple_big_node *b_node)
{
char empty_count = mas_mt_height(mas);
struct maple_subtree_state mast;
unsigned char shift, b_end = ++b_node->b_end;
MA_STATE(l_mas, mas->tree, mas->index, mas->last);
MA_STATE(r_mas, mas->tree, mas->index, mas->last);
trace_ma_op(__func__, mas);
/*
* Rebalancing occurs if a node is insufficient. Data is rebalanced
* against the node to the right if it exists, otherwise the node to the
* left of this node is rebalanced against this node. If rebalancing
* causes just one node to be produced instead of two, then the parent
* is also examined and rebalanced if it is insufficient. Every level
* tries to combine the data in the same way. If one node contains the
* entire range of the tree, then that node is used as a new root node.
*/
mas_node_count(mas, empty_count * 2 - 1);
if (mas_is_err(mas))
return 0;
mast.orig_l = &l_mas;
mast.orig_r = &r_mas;
mast.bn = b_node;
mast.bn->type = mte_node_type(mas->node);
l_mas = r_mas = *mas;
if (mas_next_sibling(&r_mas)) {
mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end);
r_mas.last = r_mas.index = r_mas.max;
} else {
mas_prev_sibling(&l_mas);
shift = mas_data_end(&l_mas) + 1;
mab_shift_right(b_node, shift);
mas->offset += shift;
mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0);
b_node->b_end = shift + b_end;
l_mas.index = l_mas.last = l_mas.min;
}
return mas_spanning_rebalance(mas, &mast, empty_count);
}
/*
* mas_destroy_rebalance() - Rebalance left-most node while destroying the maple
* state.
* @mas: The maple state
* @end: The end of the left-most node.
*
* During a mass-insert event (such as forking), it may be necessary to
* rebalance the left-most node when it is not sufficient.
*/
static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end)
{
enum maple_type mt = mte_node_type(mas->node);
struct maple_node reuse, *newnode, *parent, *new_left, *left, *node;
struct maple_enode *eparent, *old_eparent;
unsigned char offset, tmp, split = mt_slots[mt] / 2;
void __rcu **l_slots, **slots;
unsigned long *l_pivs, *pivs, gap;
bool in_rcu = mt_in_rcu(mas->tree);
MA_STATE(l_mas, mas->tree, mas->index, mas->last);
l_mas = *mas;
mas_prev_sibling(&l_mas);
/* set up node. */
if (in_rcu) {
/* Allocate for both left and right as well as parent. */
mas_node_count(mas, 3);
if (mas_is_err(mas))
return;
newnode = mas_pop_node(mas);
} else {
newnode = &reuse;
}
node = mas_mn(mas);
newnode->parent = node->parent;
slots = ma_slots(newnode, mt);
pivs = ma_pivots(newnode, mt);
left = mas_mn(&l_mas);
l_slots = ma_slots(left, mt);
l_pivs = ma_pivots(left, mt);
if (!l_slots[split])
split++;
tmp = mas_data_end(&l_mas) - split;
memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp);
memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp);
pivs[tmp] = l_mas.max;
memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end);
memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end);
l_mas.max = l_pivs[split];
mas->min = l_mas.max + 1;
old_eparent = mt_mk_node(mte_parent(l_mas.node),
mas_parent_type(&l_mas, l_mas.node));
tmp += end;
if (!in_rcu) {
unsigned char max_p = mt_pivots[mt];
unsigned char max_s = mt_slots[mt];
if (tmp < max_p)
memset(pivs + tmp, 0,
sizeof(unsigned long) * (max_p - tmp));
if (tmp < mt_slots[mt])
memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp));
memcpy(node, newnode, sizeof(struct maple_node));
ma_set_meta(node, mt, 0, tmp - 1);
mte_set_pivot(old_eparent, mte_parent_slot(l_mas.node),
l_pivs[split]);
/* Remove data from l_pivs. */
tmp = split + 1;
memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp));
memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp));
ma_set_meta(left, mt, 0, split);
eparent = old_eparent;
goto done;
}
/* RCU requires replacing both l_mas, mas, and parent. */
mas->node = mt_mk_node(newnode, mt);
ma_set_meta(newnode, mt, 0, tmp);
new_left = mas_pop_node(mas);
new_left->parent = left->parent;
mt = mte_node_type(l_mas.node);
slots = ma_slots(new_left, mt);
pivs = ma_pivots(new_left, mt);
memcpy(slots, l_slots, sizeof(void *) * split);
memcpy(pivs, l_pivs, sizeof(unsigned long) * split);
ma_set_meta(new_left, mt, 0, split);
l_mas.node = mt_mk_node(new_left, mt);
/* replace parent. */
offset = mte_parent_slot(mas->node);
mt = mas_parent_type(&l_mas, l_mas.node);
parent = mas_pop_node(mas);
slots = ma_slots(parent, mt);
pivs = ma_pivots(parent, mt);
memcpy(parent, mte_to_node(old_eparent), sizeof(struct maple_node));
rcu_assign_pointer(slots[offset], mas->node);
rcu_assign_pointer(slots[offset - 1], l_mas.node);
pivs[offset - 1] = l_mas.max;
eparent = mt_mk_node(parent, mt);
done:
gap = mas_leaf_max_gap(mas);
mte_set_gap(eparent, mte_parent_slot(mas->node), gap);
gap = mas_leaf_max_gap(&l_mas);
mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap);
mas_ascend(mas);
if (in_rcu) {
mas_replace_node(mas, old_eparent);
mas_adopt_children(mas, mas->node);
}
mas_update_gap(mas);
}
/*
* mas_split_final_node() - Split the final node in a subtree operation.
* @mast: the maple subtree state
* @mas: The maple state
* @height: The height of the tree in case it's a new root.
*/
static inline void mas_split_final_node(struct maple_subtree_state *mast,
struct ma_state *mas, int height)
{
struct maple_enode *ancestor;
if (mte_is_root(mas->node)) {
if (mt_is_alloc(mas->tree))
mast->bn->type = maple_arange_64;
else
mast->bn->type = maple_range_64;
mas->depth = height;
}
/*
* Only a single node is used here, could be root.
* The Big_node data should just fit in a single node.
*/
ancestor = mas_new_ma_node(mas, mast->bn);
mas_set_parent(mas, mast->l->node, ancestor, mast->l->offset);
mas_set_parent(mas, mast->r->node, ancestor, mast->r->offset);
mte_to_node(ancestor)->parent = mas_mn(mas)->parent;
mast->l->node = ancestor;
mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true);
mas->offset = mast->bn->b_end - 1;
}
/*
* mast_fill_bnode() - Copy data into the big node in the subtree state
* @mast: The maple subtree state
* @mas: the maple state
* @skip: The number of entries to skip for new nodes insertion.
*/
static inline void mast_fill_bnode(struct maple_subtree_state *mast,
struct ma_state *mas,
unsigned char skip)
{
bool cp = true;
unsigned char split;
memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap));
memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot));
memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot));
mast->bn->b_end = 0;
if (mte_is_root(mas->node)) {
cp = false;
} else {
mas_ascend(mas);
mas->offset = mte_parent_slot(mas->node);
}
if (cp && mast->l->offset)
mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0);
split = mast->bn->b_end;
mab_set_b_end(mast->bn, mast->l, mast->l->node);
mast->r->offset = mast->bn->b_end;
mab_set_b_end(mast->bn, mast->r, mast->r->node);
if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max)
cp = false;
if (cp)
mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1,
mast->bn, mast->bn->b_end);
mast->bn->b_end--;
mast->bn->type = mte_node_type(mas->node);
}
/*
* mast_split_data() - Split the data in the subtree state big node into regular
* nodes.
* @mast: The maple subtree state
* @mas: The maple state
* @split: The location to split the big node
*/
static inline void mast_split_data(struct maple_subtree_state *mast,
struct ma_state *mas, unsigned char split)
{
unsigned char p_slot;
mab_mas_cp(mast->bn, 0, split, mast->l, true);
mte_set_pivot(mast->r->node, 0, mast->r->max);
mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false);
mast->l->offset = mte_parent_slot(mas->node);
mast->l->max = mast->bn->pivot[split];
mast->r->min = mast->l->max + 1;
if (mte_is_leaf(mas->node))
return;
p_slot = mast->orig_l->offset;
mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node,
&p_slot, split);
mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node,
&p_slot, split);
}
/*
* mas_push_data() - Instead of splitting a node, it is beneficial to push the
* data to the right or left node if there is room.
* @mas: The maple state
* @height: The current height of the maple state
* @mast: The maple subtree state
* @left: Push left or not.
*
* Keeping the height of the tree low means faster lookups.
*
* Return: True if pushed, false otherwise.
*/
static inline bool mas_push_data(struct ma_state *mas, int height,
struct maple_subtree_state *mast, bool left)
{
unsigned char slot_total = mast->bn->b_end;
unsigned char end, space, split;
MA_STATE(tmp_mas, mas->tree, mas->index, mas->last);
tmp_mas = *mas;
tmp_mas.depth = mast->l->depth;
if (left && !mas_prev_sibling(&tmp_mas))
return false;
else if (!left && !mas_next_sibling(&tmp_mas))
return false;
end = mas_data_end(&tmp_mas);
slot_total += end;
space = 2 * mt_slot_count(mas->node) - 2;
/* -2 instead of -1 to ensure there isn't a triple split */
if (ma_is_leaf(mast->bn->type))
space--;
if (mas->max == ULONG_MAX)
space--;
if (slot_total >= space)
return false;
/* Get the data; Fill mast->bn */
mast->bn->b_end++;
if (left) {
mab_shift_right(mast->bn, end + 1);
mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0);
mast->bn->b_end = slot_total + 1;
} else {
mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end);
}
/* Configure mast for splitting of mast->bn */
split = mt_slots[mast->bn->type] - 2;
if (left) {
/* Switch mas to prev node */
*mas = tmp_mas;
/* Start using mast->l for the left side. */
tmp_mas.node = mast->l->node;
*mast->l = tmp_mas;
} else {
tmp_mas.node = mast->r->node;
*mast->r = tmp_mas;
split = slot_total - split;
}
split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]);
/* Update parent slot for split calculation. */
if (left)
mast->orig_l->offset += end + 1;
mast_split_data(mast, mas, split);
mast_fill_bnode(mast, mas, 2);
mas_split_final_node(mast, mas, height + 1);
return true;
}
/*
* mas_split() - Split data that is too big for one node into two.
* @mas: The maple state
* @b_node: The maple big node
* Return: 1 on success, 0 on failure.
*/
static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
{
struct maple_subtree_state mast;
int height = 0;
unsigned char mid_split, split = 0;
struct maple_enode *old;
/*
* Splitting is handled differently from any other B-tree; the Maple
* Tree splits upwards. Splitting up means that the split operation
* occurs when the walk of the tree hits the leaves and not on the way
* down. The reason for splitting up is that it is impossible to know
* how much space will be needed until the leaf is (or leaves are)
* reached. Since overwriting data is allowed and a range could
* overwrite more than one range or result in changing one entry into 3
* entries, it is impossible to know if a split is required until the
* data is examined.
*
* Splitting is a balancing act between keeping allocations to a minimum
* and avoiding a 'jitter' event where a tree is expanded to make room
* for an entry followed by a contraction when the entry is removed. To
* accomplish the balance, there are empty slots remaining in both left
* and right nodes after a split.
*/
MA_STATE(l_mas, mas->tree, mas->index, mas->last);
MA_STATE(r_mas, mas->tree, mas->index, mas->last);
MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last);
MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last);
trace_ma_op(__func__, mas);
mas->depth = mas_mt_height(mas);
/* Allocation failures will happen early. */
mas_node_count(mas, 1 + mas->depth * 2);
if (mas_is_err(mas))
return 0;
mast.l = &l_mas;
mast.r = &r_mas;
mast.orig_l = &prev_l_mas;
mast.orig_r = &prev_r_mas;
mast.bn = b_node;
while (height++ <= mas->depth) {
if (mt_slots[b_node->type] > b_node->b_end) {
mas_split_final_node(&mast, mas, height);
break;
}
l_mas = r_mas = *mas;
l_mas.node = mas_new_ma_node(mas, b_node);
r_mas.node = mas_new_ma_node(mas, b_node);
/*
* Another way that 'jitter' is avoided is to terminate a split up early if the
* left or right node has space to spare. This is referred to as "pushing left"
* or "pushing right" and is similar to the B* tree, except the nodes left or
* right can rarely be reused due to RCU, but the ripple upwards is halted which
* is a significant savings.
*/
/* Try to push left. */
if (mas_push_data(mas, height, &mast, true))
break;
/* Try to push right. */
if (mas_push_data(mas, height, &mast, false))
break;
split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min);
mast_split_data(&mast, mas, split);
/*
* Usually correct, mab_mas_cp in the above call overwrites
* r->max.
*/
mast.r->max = mas->max;
mast_fill_bnode(&mast, mas, 1);
prev_l_mas = *mast.l;
prev_r_mas = *mast.r;
}
/* Set the original node as dead */
old = mas->node;
mas->node = l_mas.node;
mas_wmb_replace(mas, old);
mtree_range_walk(mas);
return 1;
}
/*
* mas_reuse_node() - Reuse the node to store the data.
* @wr_mas: The maple write state
* @bn: The maple big node
* @end: The end of the data.
*
* Will always return false in RCU mode.
*
* Return: True if node was reused, false otherwise.
*/
static inline bool mas_reuse_node(struct ma_wr_state *wr_mas,
struct maple_big_node *bn, unsigned char end)
{
/* Need to be rcu safe. */
if (mt_in_rcu(wr_mas->mas->tree))
return false;
if (end > bn->b_end) {
int clear = mt_slots[wr_mas->type] - bn->b_end;
memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--);
memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear);
}
mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false);
return true;
}
/*
* mas_commit_b_node() - Commit the big node into the tree.
* @wr_mas: The maple write state
* @b_node: The maple big node
* @end: The end of the data.
*/
static noinline_for_kasan int mas_commit_b_node(struct ma_wr_state *wr_mas,
struct maple_big_node *b_node, unsigned char end)
{
struct maple_node *node;
struct maple_enode *old_enode;
unsigned char b_end = b_node->b_end;
enum maple_type b_type = b_node->type;
old_enode = wr_mas->mas->node;
if ((b_end < mt_min_slots[b_type]) &&
(!mte_is_root(old_enode)) &&
(mas_mt_height(wr_mas->mas) > 1))
return mas_rebalance(wr_mas->mas, b_node);
if (b_end >= mt_slots[b_type])
return mas_split(wr_mas->mas, b_node);
if (mas_reuse_node(wr_mas, b_node, end))
goto reuse_node;
mas_node_count(wr_mas->mas, 1);
if (mas_is_err(wr_mas->mas))
return 0;
node = mas_pop_node(wr_mas->mas);
node->parent = mas_mn(wr_mas->mas)->parent;
wr_mas->mas->node = mt_mk_node(node, b_type);
mab_mas_cp(b_node, 0, b_end, wr_mas->mas, false);
mas_replace_node(wr_mas->mas, old_enode);
reuse_node:
mas_update_gap(wr_mas->mas);
wr_mas->mas->end = b_end;
return 1;
}
/*
* mas_root_expand() - Expand a root to a node
* @mas: The maple state
* @entry: The entry to store into the tree
*/
static inline int mas_root_expand(struct ma_state *mas, void *entry)
{
void *contents = mas_root_locked(mas);
enum maple_type type = maple_leaf_64;
struct maple_node *node;
void __rcu **slots;
unsigned long *pivots;
int slot = 0;
mas_node_count(mas, 1);
if (unlikely(mas_is_err(mas)))
return 0;
node = mas_pop_node(mas);
pivots = ma_pivots(node, type);
slots = ma_slots(node, type);
node->parent = ma_parent_ptr(mas_tree_parent(mas));
mas->node = mt_mk_node(node, type);
mas->status = ma_active;
if (mas->index) {
if (contents) { rcu_assign_pointer(slots[slot], contents);
if (likely(mas->index > 1))
slot++;
}
pivots[slot++] = mas->index - 1;
}
rcu_assign_pointer(slots[slot], entry);
mas->offset = slot;
pivots[slot] = mas->last;
if (mas->last != ULONG_MAX)
pivots[++slot] = ULONG_MAX; mas->depth = 1;
mas_set_height(mas);
ma_set_meta(node, maple_leaf_64, 0, slot);
/* swap the new root into the tree */
rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
return slot;
}
static inline void mas_store_root(struct ma_state *mas, void *entry)
{
if (likely((mas->last != 0) || (mas->index != 0))) mas_root_expand(mas, entry); else if (((unsigned long) (entry) & 3) == 2) mas_root_expand(mas, entry);
else {
rcu_assign_pointer(mas->tree->ma_root, entry);
mas->status = ma_start;
}
}
/*
* mas_is_span_wr() - Check if the write needs to be treated as a write that
* spans the node.
* @mas: The maple state
* @piv: The pivot value being written
* @type: The maple node type
* @entry: The data to write
*
* Spanning writes are writes that start in one node and end in another OR if
* the write of a %NULL will cause the node to end with a %NULL.
*
* Return: True if this is a spanning write, false otherwise.
*/
static bool mas_is_span_wr(struct ma_wr_state *wr_mas)
{
unsigned long max = wr_mas->r_max;
unsigned long last = wr_mas->mas->last;
enum maple_type type = wr_mas->type;
void *entry = wr_mas->entry;
/* Contained in this pivot, fast path */
if (last < max)
return false;
if (ma_is_leaf(type)) { max = wr_mas->mas->max;
if (last < max)
return false;
}
if (last == max) {
/*
* The last entry of leaf node cannot be NULL unless it is the
* rightmost node (writing ULONG_MAX), otherwise it spans slots.
*/
if (entry || last == ULONG_MAX)
return false;
}
trace_ma_write(__func__, wr_mas->mas, wr_mas->r_max, entry); return true;
}
static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas)
{
wr_mas->type = mte_node_type(wr_mas->mas->node); mas_wr_node_walk(wr_mas); wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type);
}
static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas)
{
wr_mas->mas->max = wr_mas->r_max;
wr_mas->mas->min = wr_mas->r_min;
wr_mas->mas->node = wr_mas->content;
wr_mas->mas->offset = 0;
wr_mas->mas->depth++;
}
/*
* mas_wr_walk() - Walk the tree for a write.
* @wr_mas: The maple write state
*
* Uses mas_slot_locked() and does not need to worry about dead nodes.
*
* Return: True if it's contained in a node, false on spanning write.
*/
static bool mas_wr_walk(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
while (true) {
mas_wr_walk_descend(wr_mas); if (unlikely(mas_is_span_wr(wr_mas)))
return false;
wr_mas->content = mas_slot_locked(mas, wr_mas->slots, mas->offset);
if (ma_is_leaf(wr_mas->type))
return true;
mas_wr_walk_traverse(wr_mas);
}
return true;
}
static bool mas_wr_walk_index(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
while (true) {
mas_wr_walk_descend(wr_mas);
wr_mas->content = mas_slot_locked(mas, wr_mas->slots,
mas->offset);
if (ma_is_leaf(wr_mas->type))
return true;
mas_wr_walk_traverse(wr_mas);
}
return true;
}
/*
* mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs.
* @l_wr_mas: The left maple write state
* @r_wr_mas: The right maple write state
*/
static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
struct ma_wr_state *r_wr_mas)
{
struct ma_state *r_mas = r_wr_mas->mas;
struct ma_state *l_mas = l_wr_mas->mas;
unsigned char l_slot;
l_slot = l_mas->offset;
if (!l_wr_mas->content)
l_mas->index = l_wr_mas->r_min;
if ((l_mas->index == l_wr_mas->r_min) &&
(l_slot &&
!mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) {
if (l_slot > 1)
l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1;
else
l_mas->index = l_mas->min;
l_mas->offset = l_slot - 1;
}
if (!r_wr_mas->content) {
if (r_mas->last < r_wr_mas->r_max)
r_mas->last = r_wr_mas->r_max;
r_mas->offset++;
} else if ((r_mas->last == r_wr_mas->r_max) &&
(r_mas->last < r_mas->max) &&
!mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots,
r_wr_mas->type, r_mas->offset + 1);
r_mas->offset++;
}
}
static inline void *mas_state_walk(struct ma_state *mas)
{
void *entry;
entry = mas_start(mas); if (mas_is_none(mas))
return NULL;
if (mas_is_ptr(mas))
return entry;
return mtree_range_walk(mas);
}
/*
* mtree_lookup_walk() - Internal quick lookup that does not keep maple state up
* to date.
*
* @mas: The maple state.
*
* Note: Leaves mas in undesirable state.
* Return: The entry for @mas->index or %NULL on dead node.
*/
static inline void *mtree_lookup_walk(struct ma_state *mas)
{
unsigned long *pivots;
unsigned char offset;
struct maple_node *node;
struct maple_enode *next;
enum maple_type type;
void __rcu **slots;
unsigned char end;
next = mas->node;
do {
node = mte_to_node(next);
type = mte_node_type(next);
pivots = ma_pivots(node, type); end = mt_pivots[type];
offset = 0;
do {
if (pivots[offset] >= mas->index)
break;
} while (++offset < end); slots = ma_slots(node, type); next = mt_slot(mas->tree, slots, offset); if (unlikely(ma_dead_node(node)))
goto dead_node;
} while (!ma_is_leaf(type));
return (void *)next;
dead_node:
mas_reset(mas);
return NULL;
}
static void mte_destroy_walk(struct maple_enode *, struct maple_tree *);
/*
* mas_new_root() - Create a new root node that only contains the entry passed
* in.
* @mas: The maple state
* @entry: The entry to store.
*
* Only valid when the index == 0 and the last == ULONG_MAX
*
* Return 0 on error, 1 on success.
*/
static inline int mas_new_root(struct ma_state *mas, void *entry)
{
struct maple_enode *root = mas_root_locked(mas);
enum maple_type type = maple_leaf_64;
struct maple_node *node;
void __rcu **slots;
unsigned long *pivots;
if (!entry && !mas->index && mas->last == ULONG_MAX) {
mas->depth = 0;
mas_set_height(mas);
rcu_assign_pointer(mas->tree->ma_root, entry);
mas->status = ma_start;
goto done;
}
mas_node_count(mas, 1);
if (mas_is_err(mas))
return 0;
node = mas_pop_node(mas);
pivots = ma_pivots(node, type);
slots = ma_slots(node, type);
node->parent = ma_parent_ptr(mas_tree_parent(mas));
mas->node = mt_mk_node(node, type);
mas->status = ma_active;
rcu_assign_pointer(slots[0], entry);
pivots[0] = mas->last;
mas->depth = 1;
mas_set_height(mas);
rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
done:
if (xa_is_node(root))
mte_destroy_walk(root, mas->tree);
return 1;
}
/*
* mas_wr_spanning_store() - Create a subtree with the store operation completed
* and new nodes where necessary, then place the sub-tree in the actual tree.
* Note that mas is expected to point to the node which caused the store to
* span.
* @wr_mas: The maple write state
*
* Return: 0 on error, positive on success.
*/
static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas)
{
struct maple_subtree_state mast;
struct maple_big_node b_node;
struct ma_state *mas;
unsigned char height;
/* Left and Right side of spanning store */
MA_STATE(l_mas, NULL, 0, 0);
MA_STATE(r_mas, NULL, 0, 0);
MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry);
MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry);
/*
* A store operation that spans multiple nodes is called a spanning
* store and is handled early in the store call stack by the function
* mas_is_span_wr(). When a spanning store is identified, the maple
* state is duplicated. The first maple state walks the left tree path
* to ``index``, the duplicate walks the right tree path to ``last``.
* The data in the two nodes are combined into a single node, two nodes,
* or possibly three nodes (see the 3-way split above). A ``NULL``
* written to the last entry of a node is considered a spanning store as
* a rebalance is required for the operation to complete and an overflow
* of data may happen.
*/
mas = wr_mas->mas;
trace_ma_op(__func__, mas);
if (unlikely(!mas->index && mas->last == ULONG_MAX))
return mas_new_root(mas, wr_mas->entry);
/*
* Node rebalancing may occur due to this store, so there may be three new
* entries per level plus a new root.
*/
height = mas_mt_height(mas);
mas_node_count(mas, 1 + height * 3);
if (mas_is_err(mas))
return 0;
/*
* Set up right side. Need to get to the next offset after the spanning
* store to ensure it's not NULL and to combine both the next node and
* the node with the start together.
*/
r_mas = *mas;
/* Avoid overflow, walk to next slot in the tree. */
if (r_mas.last + 1)
r_mas.last++;
r_mas.index = r_mas.last;
mas_wr_walk_index(&r_wr_mas);
r_mas.last = r_mas.index = mas->last;
/* Set up left side. */
l_mas = *mas;
mas_wr_walk_index(&l_wr_mas);
if (!wr_mas->entry) {
mas_extend_spanning_null(&l_wr_mas, &r_wr_mas);
mas->offset = l_mas.offset;
mas->index = l_mas.index;
mas->last = l_mas.last = r_mas.last;
}
/* expanding NULLs may make this cover the entire range */
if (!l_mas.index && r_mas.last == ULONG_MAX) {
mas_set_range(mas, 0, ULONG_MAX);
return mas_new_root(mas, wr_mas->entry);
}
memset(&b_node, 0, sizeof(struct maple_big_node));
/* Copy l_mas and store the value in b_node. */
mas_store_b_node(&l_wr_mas, &b_node, l_mas.end);
/* Copy r_mas into b_node. */
if (r_mas.offset <= r_mas.end)
mas_mab_cp(&r_mas, r_mas.offset, r_mas.end,
&b_node, b_node.b_end + 1);
else
b_node.b_end++;
/* Stop spanning searches by searching for just index. */
l_mas.index = l_mas.last = mas->index;
mast.bn = &b_node;
mast.orig_l = &l_mas;
mast.orig_r = &r_mas;
/* Combine l_mas and r_mas and split them up evenly again. */
return mas_spanning_rebalance(mas, &mast, height + 1);
}
/*
* mas_wr_node_store() - Attempt to store the value in a node
* @wr_mas: The maple write state
*
* Attempts to reuse the node, but may allocate.
*
* Return: True if stored, false otherwise
*/
static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas,
unsigned char new_end)
{
struct ma_state *mas = wr_mas->mas;
void __rcu **dst_slots;
unsigned long *dst_pivots;
unsigned char dst_offset, offset_end = wr_mas->offset_end;
struct maple_node reuse, *newnode;
unsigned char copy_size, node_pivots = mt_pivots[wr_mas->type];
bool in_rcu = mt_in_rcu(mas->tree);
/* Check if there is enough data. The room is enough. */
if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) &&
!(mas->mas_flags & MA_STATE_BULK))
return false;
if (mas->last == wr_mas->end_piv)
offset_end++; /* don't copy this offset */
else if (unlikely(wr_mas->r_max == ULONG_MAX))
mas_bulk_rebalance(mas, mas->end, wr_mas->type);
/* set up node. */
if (in_rcu) {
mas_node_count(mas, 1);
if (mas_is_err(mas))
return false;
newnode = mas_pop_node(mas);
} else {
memset(&reuse, 0, sizeof(struct maple_node));
newnode = &reuse;
}
newnode->parent = mas_mn(mas)->parent;
dst_pivots = ma_pivots(newnode, wr_mas->type);
dst_slots = ma_slots(newnode, wr_mas->type);
/* Copy from start to insert point */
memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * mas->offset);
memcpy(dst_slots, wr_mas->slots, sizeof(void *) * mas->offset);
/* Handle insert of new range starting after old range */
if (wr_mas->r_min < mas->index) {
rcu_assign_pointer(dst_slots[mas->offset], wr_mas->content);
dst_pivots[mas->offset++] = mas->index - 1;
}
/* Store the new entry and range end. */
if (mas->offset < node_pivots)
dst_pivots[mas->offset] = mas->last;
rcu_assign_pointer(dst_slots[mas->offset], wr_mas->entry);
/*
* this range wrote to the end of the node or it overwrote the rest of
* the data
*/
if (offset_end > mas->end)
goto done;
dst_offset = mas->offset + 1;
/* Copy to the end of node if necessary. */
copy_size = mas->end - offset_end + 1;
memcpy(dst_slots + dst_offset, wr_mas->slots + offset_end,
sizeof(void *) * copy_size);
memcpy(dst_pivots + dst_offset, wr_mas->pivots + offset_end,
sizeof(unsigned long) * (copy_size - 1));
if (new_end < node_pivots)
dst_pivots[new_end] = mas->max;
done:
mas_leaf_set_meta(newnode, maple_leaf_64, new_end);
if (in_rcu) {
struct maple_enode *old_enode = mas->node;
mas->node = mt_mk_node(newnode, wr_mas->type);
mas_replace_node(mas, old_enode);
} else {
memcpy(wr_mas->node, newnode, sizeof(struct maple_node));
}
trace_ma_write(__func__, mas, 0, wr_mas->entry);
mas_update_gap(mas);
mas->end = new_end;
return true;
}
/*
* mas_wr_slot_store: Attempt to store a value in a slot.
* @wr_mas: the maple write state
*
* Return: True if stored, false otherwise
*/
static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
unsigned char offset = mas->offset;
void __rcu **slots = wr_mas->slots;
bool gap = false;
gap |= !mt_slot_locked(mas->tree, slots, offset);
gap |= !mt_slot_locked(mas->tree, slots, offset + 1);
if (wr_mas->offset_end - offset == 1) {
if (mas->index == wr_mas->r_min) {
/* Overwriting the range and a part of the next one */
rcu_assign_pointer(slots[offset], wr_mas->entry);
wr_mas->pivots[offset] = mas->last;
} else {
/* Overwriting a part of the range and the next one */
rcu_assign_pointer(slots[offset + 1], wr_mas->entry);
wr_mas->pivots[offset] = mas->index - 1;
mas->offset++; /* Keep mas accurate. */
}
} else if (!mt_in_rcu(mas->tree)) {
/*
* Expand the range, only partially overwriting the previous and
* next ranges
*/
gap |= !mt_slot_locked(mas->tree, slots, offset + 2);
rcu_assign_pointer(slots[offset + 1], wr_mas->entry);
wr_mas->pivots[offset] = mas->index - 1;
wr_mas->pivots[offset + 1] = mas->last;
mas->offset++; /* Keep mas accurate. */
} else {
return false;
}
trace_ma_write(__func__, mas, 0, wr_mas->entry);
/*
* Only update gap when the new entry is empty or there is an empty
* entry in the original two ranges.
*/
if (!wr_mas->entry || gap)
mas_update_gap(mas);
return true;
}
static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
if (!wr_mas->slots[wr_mas->offset_end]) {
/* If this one is null, the next and prev are not */
mas->last = wr_mas->end_piv;
} else {
/* Check next slot(s) if we are overwriting the end */
if ((mas->last == wr_mas->end_piv) && (mas->end != wr_mas->offset_end) && !wr_mas->slots[wr_mas->offset_end + 1]) { wr_mas->offset_end++;
if (wr_mas->offset_end == mas->end)
mas->last = mas->max;
else
mas->last = wr_mas->pivots[wr_mas->offset_end];
wr_mas->end_piv = mas->last;
}
}
if (!wr_mas->content) {
/* If this one is null, the next and prev are not */
mas->index = wr_mas->r_min;
} else {
/* Check prev slot if we are overwriting the start */
if (mas->index == wr_mas->r_min && mas->offset && !wr_mas->slots[mas->offset - 1]) { mas->offset--; wr_mas->r_min = mas->index = mas_safe_min(mas, wr_mas->pivots, mas->offset);
wr_mas->r_max = wr_mas->pivots[mas->offset];
}
}
}
static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas)
{
while ((wr_mas->offset_end < wr_mas->mas->end) && (wr_mas->mas->last > wr_mas->pivots[wr_mas->offset_end])) wr_mas->offset_end++; if (wr_mas->offset_end < wr_mas->mas->end) wr_mas->end_piv = wr_mas->pivots[wr_mas->offset_end];
else
wr_mas->end_piv = wr_mas->mas->max;
if (!wr_mas->entry)
mas_wr_extend_null(wr_mas);
}
static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
unsigned char new_end = mas->end + 2;
new_end -= wr_mas->offset_end - mas->offset;
if (wr_mas->r_min == mas->index)
new_end--; if (wr_mas->end_piv == mas->last) new_end--;
return new_end;
}
/*
* mas_wr_append: Attempt to append
* @wr_mas: the maple write state
* @new_end: The end of the node after the modification
*
* This is currently unsafe in rcu mode since the end of the node may be cached
* by readers while the node contents may be updated which could result in
* inaccurate information.
*
* Return: True if appended, false otherwise
*/
static inline bool mas_wr_append(struct ma_wr_state *wr_mas,
unsigned char new_end)
{
struct ma_state *mas;
void __rcu **slots;
unsigned char end;
mas = wr_mas->mas;
if (mt_in_rcu(mas->tree))
return false; end = mas->end;
if (mas->offset != end)
return false;
if (new_end < mt_pivots[wr_mas->type]) { wr_mas->pivots[new_end] = wr_mas->pivots[end]; ma_set_meta(wr_mas->node, wr_mas->type, 0, new_end);
}
slots = wr_mas->slots;
if (new_end == end + 1) {
if (mas->last == wr_mas->r_max) {
/* Append to end of range */
rcu_assign_pointer(slots[new_end], wr_mas->entry);
wr_mas->pivots[end] = mas->index - 1;
mas->offset = new_end;
} else {
/* Append to start of range */
rcu_assign_pointer(slots[new_end], wr_mas->content);
wr_mas->pivots[end] = mas->last;
rcu_assign_pointer(slots[end], wr_mas->entry);
}
} else {
/* Append to the range without touching any boundaries. */
rcu_assign_pointer(slots[new_end], wr_mas->content);
wr_mas->pivots[end + 1] = mas->last;
rcu_assign_pointer(slots[end + 1], wr_mas->entry);
wr_mas->pivots[end] = mas->index - 1;
mas->offset = end + 1;
}
if (!wr_mas->content || !wr_mas->entry) mas_update_gap(mas); mas->end = new_end;
trace_ma_write(__func__, mas, new_end, wr_mas->entry);
return true;
}
/*
* mas_wr_bnode() - Slow path for a modification.
* @wr_mas: The write maple state
*
* This is where split, rebalance end up.
*/
static void mas_wr_bnode(struct ma_wr_state *wr_mas)
{
struct maple_big_node b_node;
trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry);
memset(&b_node, 0, sizeof(struct maple_big_node));
mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end);
mas_commit_b_node(wr_mas, &b_node, wr_mas->mas->end);
}
static inline void mas_wr_modify(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
unsigned char new_end;
/* Direct replacement */
if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) { rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry);
if (!!wr_mas->entry ^ !!wr_mas->content)
mas_update_gap(mas);
return;
}
/*
* new_end exceeds the size of the maple node and cannot enter the fast
* path.
*/
new_end = mas_wr_new_end(wr_mas); if (new_end >= mt_slots[wr_mas->type])
goto slow_path;
/* Attempt to append */
if (mas_wr_append(wr_mas, new_end))
return;
if (new_end == mas->end && mas_wr_slot_store(wr_mas))
return;
if (mas_wr_node_store(wr_mas, new_end))
return;
if (mas_is_err(mas))
return;
slow_path:
mas_wr_bnode(wr_mas);
}
/*
* mas_wr_store_entry() - Internal call to store a value
* @mas: The maple state
* @entry: The entry to store.
*
* Return: The contents that was stored at the index.
*/
static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas)
{
struct ma_state *mas = wr_mas->mas;
wr_mas->content = mas_start(mas);
if (mas_is_none(mas) || mas_is_ptr(mas)) {
mas_store_root(mas, wr_mas->entry);
return wr_mas->content;
}
if (unlikely(!mas_wr_walk(wr_mas))) {
mas_wr_spanning_store(wr_mas);
return wr_mas->content;
}
/* At this point, we are at the leaf node that needs to be altered. */
mas_wr_end_piv(wr_mas);
/* New root for a single pointer */
if (unlikely(!mas->index && mas->last == ULONG_MAX)) {
mas_new_root(mas, wr_mas->entry);
return wr_mas->content;
}
mas_wr_modify(wr_mas);
return wr_mas->content;
}
/**
* mas_insert() - Internal call to insert a value
* @mas: The maple state
* @entry: The entry to store
*
* Return: %NULL or the contents that already exists at the requested index
* otherwise. The maple state needs to be checked for error conditions.
*/
static inline void *mas_insert(struct ma_state *mas, void *entry)
{
MA_WR_STATE(wr_mas, mas, entry);
/*
* Inserting a new range inserts either 0, 1, or 2 pivots within the
* tree. If the insert fits exactly into an existing gap with a value
* of NULL, then the slot only needs to be written with the new value.
* If the range being inserted is adjacent to another range, then only a
* single pivot needs to be inserted (as well as writing the entry). If
* the new range is within a gap but does not touch any other ranges,
* then two pivots need to be inserted: the start - 1, and the end. As
* usual, the entry must be written. Most operations require a new node
* to be allocated and replace an existing node to ensure RCU safety,
* when in RCU mode. The exception to requiring a newly allocated node
* is when inserting at the end of a node (appending). When done
* carefully, appending can reuse the node in place.
*/
wr_mas.content = mas_start(mas);
if (wr_mas.content)
goto exists;
if (mas_is_none(mas) || mas_is_ptr(mas)) { mas_store_root(mas, entry);
return NULL;
}
/* spanning writes always overwrite something */
if (!mas_wr_walk(&wr_mas))
goto exists;
/* At this point, we are at the leaf node that needs to be altered. */
wr_mas.offset_end = mas->offset;
wr_mas.end_piv = wr_mas.r_max;
if (wr_mas.content || (mas->last > wr_mas.r_max))
goto exists;
if (!entry)
return NULL;
mas_wr_modify(&wr_mas);
return wr_mas.content;
exists:
mas_set_err(mas, -EEXIST); return wr_mas.content;
}
/**
* mas_alloc_cyclic() - Internal call to find somewhere to store an entry
* @mas: The maple state.
* @startp: Pointer to ID.
* @range_lo: Lower bound of range to search.
* @range_hi: Upper bound of range to search.
* @entry: The entry to store.
* @next: Pointer to next ID to allocate.
* @gfp: The GFP_FLAGS to use for allocations.
*
* Return: 0 if the allocation succeeded without wrapping, 1 if the
* allocation succeeded after wrapping, or -EBUSY if there are no
* free entries.
*/
int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp,
void *entry, unsigned long range_lo, unsigned long range_hi,
unsigned long *next, gfp_t gfp)
{
unsigned long min = range_lo;
int ret = 0;
range_lo = max(min, *next);
ret = mas_empty_area(mas, range_lo, range_hi, 1);
if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
ret = 1;
}
if (ret < 0 && range_lo > min) { ret = mas_empty_area(mas, min, range_hi, 1);
if (ret == 0)
ret = 1;
}
if (ret < 0)
return ret;
do {
mas_insert(mas, entry);
} while (mas_nomem(mas, gfp));
if (mas_is_err(mas)) return xa_err(mas->node); *startp = mas->index;
*next = *startp + 1;
if (*next == 0)
mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED;
return ret;
}
EXPORT_SYMBOL(mas_alloc_cyclic);
static __always_inline void mas_rewalk(struct ma_state *mas, unsigned long index)
{
retry:
mas_set(mas, index); mas_state_walk(mas);
if (mas_is_start(mas))
goto retry;
}
static __always_inline bool mas_rewalk_if_dead(struct ma_state *mas,
struct maple_node *node, const unsigned long index)
{
if (unlikely(ma_dead_node(node))) { mas_rewalk(mas, index);
return true;
}
return false;
}
/*
* mas_prev_node() - Find the prev non-null entry at the same level in the
* tree. The prev value will be mas->node[mas->offset] or the status will be
* ma_none.
* @mas: The maple state
* @min: The lower limit to search
*
* The prev node value will be mas->node[mas->offset] or the status will be
* ma_none.
* Return: 1 if the node is dead, 0 otherwise.
*/
static int mas_prev_node(struct ma_state *mas, unsigned long min)
{
enum maple_type mt;
int offset, level;
void __rcu **slots;
struct maple_node *node;
unsigned long *pivots;
unsigned long max;
node = mas_mn(mas);
if (!mas->min)
goto no_entry;
max = mas->min - 1;
if (max < min)
goto no_entry;
level = 0;
do {
if (ma_is_root(node))
goto no_entry;
/* Walk up. */
if (unlikely(mas_ascend(mas))) return 1; offset = mas->offset;
level++;
node = mas_mn(mas);
} while (!offset);
offset--;
mt = mte_node_type(mas->node);
while (level > 1) { level--; slots = ma_slots(node, mt); mas->node = mas_slot(mas, slots, offset);
if (unlikely(ma_dead_node(node)))
return 1;
mt = mte_node_type(mas->node);
node = mas_mn(mas);
pivots = ma_pivots(node, mt); offset = ma_data_end(node, mt, pivots, max); if (unlikely(ma_dead_node(node)))
return 1;
}
slots = ma_slots(node, mt); mas->node = mas_slot(mas, slots, offset); pivots = ma_pivots(node, mt); if (unlikely(ma_dead_node(node)))
return 1;
if (likely(offset)) mas->min = pivots[offset - 1] + 1; mas->max = max; mas->offset = mas_data_end(mas); if (unlikely(mte_dead_node(mas->node)))
return 1;
mas->end = mas->offset;
return 0;
no_entry:
if (unlikely(ma_dead_node(node)))
return 1;
mas->status = ma_underflow;
return 0;
}
/*
* mas_prev_slot() - Get the entry in the previous slot
*
* @mas: The maple state
* @max: The minimum starting range
* @empty: Can be empty
* @set_underflow: Set the @mas->node to underflow state on limit.
*
* Return: The entry in the previous slot which is possibly NULL
*/
static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty)
{
void *entry;
void __rcu **slots;
unsigned long pivot;
enum maple_type type;
unsigned long *pivots;
struct maple_node *node;
unsigned long save_point = mas->index;
retry:
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type); if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (mas->min <= min) { pivot = mas_safe_min(mas, pivots, mas->offset); if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (pivot <= min)
goto underflow;
}
again:
if (likely(mas->offset)) { mas->offset--; mas->last = mas->index - 1; mas->index = mas_safe_min(mas, pivots, mas->offset);
} else {
if (mas->index <= min)
goto underflow;
if (mas_prev_node(mas, min)) { mas_rewalk(mas, save_point);
goto retry;
}
if (WARN_ON_ONCE(mas_is_underflow(mas)))
return NULL;
mas->last = mas->max;
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type); mas->index = pivots[mas->offset - 1] + 1;
}
slots = ma_slots(node, type); entry = mas_slot(mas, slots, mas->offset); if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (likely(entry))
return entry;
if (!empty) { if (mas->index <= min) { mas->status = ma_underflow;
return NULL;
}
goto again;
}
return entry;
underflow:
mas->status = ma_underflow;
return NULL;
}
/*
* mas_next_node() - Get the next node at the same level in the tree.
* @mas: The maple state
* @max: The maximum pivot value to check.
*
* The next value will be mas->node[mas->offset] or the status will have
* overflowed.
* Return: 1 on dead node, 0 otherwise.
*/
static int mas_next_node(struct ma_state *mas, struct maple_node *node,
unsigned long max)
{
unsigned long min;
unsigned long *pivots;
struct maple_enode *enode;
struct maple_node *tmp;
int level = 0;
unsigned char node_end;
enum maple_type mt;
void __rcu **slots;
if (mas->max >= max)
goto overflow;
min = mas->max + 1;
level = 0;
do {
if (ma_is_root(node))
goto overflow;
/* Walk up. */
if (unlikely(mas_ascend(mas)))
return 1;
level++;
node = mas_mn(mas);
mt = mte_node_type(mas->node);
pivots = ma_pivots(node, mt); node_end = ma_data_end(node, mt, pivots, mas->max); if (unlikely(ma_dead_node(node)))
return 1;
} while (unlikely(mas->offset == node_end)); slots = ma_slots(node, mt); mas->offset++; enode = mas_slot(mas, slots, mas->offset); if (unlikely(ma_dead_node(node)))
return 1;
if (level > 1) mas->offset = 0;
while (unlikely(level > 1)) {
level--;
mas->node = enode;
node = mas_mn(mas);
mt = mte_node_type(mas->node);
slots = ma_slots(node, mt); enode = mas_slot(mas, slots, 0); if (unlikely(ma_dead_node(node)))
return 1;
}
if (!mas->offset) pivots = ma_pivots(node, mt); mas->max = mas_safe_pivot(mas, pivots, mas->offset, mt);
tmp = mte_to_node(enode);
mt = mte_node_type(enode);
pivots = ma_pivots(tmp, mt); mas->end = ma_data_end(tmp, mt, pivots, mas->max);
if (unlikely(ma_dead_node(node)))
return 1;
mas->node = enode;
mas->min = min;
return 0;
overflow:
if (unlikely(ma_dead_node(node)))
return 1;
mas->status = ma_overflow;
return 0;
}
/*
* mas_next_slot() - Get the entry in the next slot
*
* @mas: The maple state
* @max: The maximum starting range
* @empty: Can be empty
* @set_overflow: Should @mas->node be set to overflow when the limit is
* reached.
*
* Return: The entry in the next slot which is possibly NULL
*/
static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty)
{
void __rcu **slots;
unsigned long *pivots;
unsigned long pivot;
enum maple_type type;
struct maple_node *node;
unsigned long save_point = mas->last;
void *entry;
retry:
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type); if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (mas->max >= max) { if (likely(mas->offset < mas->end)) pivot = pivots[mas->offset];
else
pivot = mas->max;
if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (pivot >= max) { /* Was at the limit, next will extend beyond */ mas->status = ma_overflow; return NULL;
}
}
if (likely(mas->offset < mas->end)) { mas->index = pivots[mas->offset] + 1;
again:
mas->offset++;
if (likely(mas->offset < mas->end))
mas->last = pivots[mas->offset];
else
mas->last = mas->max;
} else {
if (mas->last >= max) {
mas->status = ma_overflow;
return NULL;
}
if (mas_next_node(mas, node, max)) { mas_rewalk(mas, save_point);
goto retry;
}
if (WARN_ON_ONCE(mas_is_overflow(mas)))
return NULL;
mas->offset = 0;
mas->index = mas->min;
node = mas_mn(mas);
type = mte_node_type(mas->node);
pivots = ma_pivots(node, type); mas->last = pivots[0];
}
slots = ma_slots(node, type); entry = mt_slot(mas->tree, slots, mas->offset); if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
goto retry;
if (entry)
return entry;
if (!empty) { if (mas->last >= max) {
mas->status = ma_overflow;
return NULL;
}
mas->index = mas->last + 1;
goto again;
}
return entry;
}
/*
* mas_next_entry() - Internal function to get the next entry.
* @mas: The maple state
* @limit: The maximum range start.
*
* Set the @mas->node to the next entry and the range_start to
* the beginning value for the entry. Does not check beyond @limit.
* Sets @mas->index and @mas->last to the range, Does not update @mas->index and
* @mas->last on overflow.
* Restarts on dead nodes.
*
* Return: the next entry or %NULL.
*/
static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit)
{
if (mas->last >= limit) {
mas->status = ma_overflow;
return NULL;
}
return mas_next_slot(mas, limit, false);
}
/*
* mas_rev_awalk() - Internal function. Reverse allocation walk. Find the
* highest gap address of a given size in a given node and descend.
* @mas: The maple state
* @size: The needed size.
*
* Return: True if found in a leaf, false otherwise.
*
*/
static bool mas_rev_awalk(struct ma_state *mas, unsigned long size,
unsigned long *gap_min, unsigned long *gap_max)
{
enum maple_type type = mte_node_type(mas->node);
struct maple_node *node = mas_mn(mas);
unsigned long *pivots, *gaps;
void __rcu **slots;
unsigned long gap = 0;
unsigned long max, min;
unsigned char offset;
if (unlikely(mas_is_err(mas)))
return true;
if (ma_is_dense(type)) {
/* dense nodes. */
mas->offset = (unsigned char)(mas->index - mas->min);
return true;
}
pivots = ma_pivots(node, type);
slots = ma_slots(node, type);
gaps = ma_gaps(node, type);
offset = mas->offset;
min = mas_safe_min(mas, pivots, offset);
/* Skip out of bounds. */
while (mas->last < min)
min = mas_safe_min(mas, pivots, --offset);
max = mas_safe_pivot(mas, pivots, offset, type);
while (mas->index <= max) {
gap = 0;
if (gaps)
gap = gaps[offset];
else if (!mas_slot(mas, slots, offset))
gap = max - min + 1;
if (gap) {
if ((size <= gap) && (size <= mas->last - min + 1))
break;
if (!gaps) {
/* Skip the next slot, it cannot be a gap. */
if (offset < 2)
goto ascend;
offset -= 2;
max = pivots[offset];
min = mas_safe_min(mas, pivots, offset);
continue;
}
}
if (!offset)
goto ascend;
offset--;
max = min - 1;
min = mas_safe_min(mas, pivots, offset);
}
if (unlikely((mas->index > max) || (size - 1 > max - mas->index)))
goto no_space;
if (unlikely(ma_is_leaf(type))) {
mas->offset = offset;
*gap_min = min;
*gap_max = min + gap - 1;
return true;
}
/* descend, only happens under lock. */
mas->node = mas_slot(mas, slots, offset);
mas->min = min;
mas->max = max;
mas->offset = mas_data_end(mas);
return false;
ascend:
if (!mte_is_root(mas->node))
return false;
no_space:
mas_set_err(mas, -EBUSY);
return false;
}
static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size)
{
enum maple_type type = mte_node_type(mas->node);
unsigned long pivot, min, gap = 0;
unsigned char offset, data_end;
unsigned long *gaps, *pivots;
void __rcu **slots;
struct maple_node *node;
bool found = false;
if (ma_is_dense(type)) {
mas->offset = (unsigned char)(mas->index - mas->min);
return true;
}
node = mas_mn(mas); pivots = ma_pivots(node, type);
slots = ma_slots(node, type);
gaps = ma_gaps(node, type);
offset = mas->offset; min = mas_safe_min(mas, pivots, offset); data_end = ma_data_end(node, type, pivots, mas->max); for (; offset <= data_end; offset++) { pivot = mas_safe_pivot(mas, pivots, offset, type);
/* Not within lower bounds */
if (mas->index > pivot)
goto next_slot;
if (gaps)
gap = gaps[offset]; else if (!mas_slot(mas, slots, offset)) gap = min(pivot, mas->last) - max(mas->index, min) + 1;
else
goto next_slot;
if (gap >= size) { if (ma_is_leaf(type)) {
found = true;
goto done;
}
if (mas->index <= pivot) { mas->node = mas_slot(mas, slots, offset);
mas->min = min;
mas->max = pivot;
offset = 0;
break;
}
}
next_slot:
min = pivot + 1;
if (mas->last <= pivot) {
mas_set_err(mas, -EBUSY);
return true;
}
}
if (mte_is_root(mas->node))
found = true;
done:
mas->offset = offset;
return found;
}
/**
* mas_walk() - Search for @mas->index in the tree.
* @mas: The maple state.
*
* mas->index and mas->last will be set to the range if there is a value. If
* mas->status is ma_none, reset to ma_start
*
* Return: the entry at the location or %NULL.
*/
void *mas_walk(struct ma_state *mas)
{
void *entry;
if (!mas_is_active(mas) || !mas_is_start(mas))
mas->status = ma_start;
retry:
entry = mas_state_walk(mas); if (mas_is_start(mas)) {
goto retry;
} else if (mas_is_none(mas)) { mas->index = 0; mas->last = ULONG_MAX; } else if (mas_is_ptr(mas)) { if (!mas->index) { mas->last = 0;
return entry;
}
mas->index = 1;
mas->last = ULONG_MAX;
mas->status = ma_none;
return NULL;
}
return entry;
}
EXPORT_SYMBOL_GPL(mas_walk);
static inline bool mas_rewind_node(struct ma_state *mas)
{
unsigned char slot;
do {
if (mte_is_root(mas->node)) {
slot = mas->offset;
if (!slot)
return false;
} else {
mas_ascend(mas);
slot = mas->offset;
}
} while (!slot);
mas->offset = --slot;
return true;
}
/*
* mas_skip_node() - Internal function. Skip over a node.
* @mas: The maple state.
*
* Return: true if there is another node, false otherwise.
*/
static inline bool mas_skip_node(struct ma_state *mas)
{
if (mas_is_err(mas))
return false;
do {
if (mte_is_root(mas->node)) { if (mas->offset >= mas_data_end(mas)) { mas_set_err(mas, -EBUSY);
return false;
}
} else {
mas_ascend(mas);
}
} while (mas->offset >= mas_data_end(mas)); mas->offset++;
return true;
}
/*
* mas_awalk() - Allocation walk. Search from low address to high, for a gap of
* @size
* @mas: The maple state
* @size: The size of the gap required
*
* Search between @mas->index and @mas->last for a gap of @size.
*/
static inline void mas_awalk(struct ma_state *mas, unsigned long size)
{
struct maple_enode *last = NULL;
/*
* There are 4 options:
* go to child (descend)
* go back to parent (ascend)
* no gap found. (return, slot == MAPLE_NODE_SLOTS)
* found the gap. (return, slot != MAPLE_NODE_SLOTS)
*/
while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) { if (last == mas->node) mas_skip_node(mas);
else
last = mas->node;
}
}
/*
* mas_sparse_area() - Internal function. Return upper or lower limit when
* searching for a gap in an empty tree.
* @mas: The maple state
* @min: the minimum range
* @max: The maximum range
* @size: The size of the gap
* @fwd: Searching forward or back
*/
static inline int mas_sparse_area(struct ma_state *mas, unsigned long min,
unsigned long max, unsigned long size, bool fwd)
{
if (!unlikely(mas_is_none(mas)) && min == 0) {
min++;
/*
* At this time, min is increased, we need to recheck whether
* the size is satisfied.
*/
if (min > max || max - min + 1 < size)
return -EBUSY;
}
/* mas_is_ptr */
if (fwd) {
mas->index = min; mas->last = min + size - 1;
} else {
mas->last = max;
mas->index = max - size + 1;
}
return 0;
}
/*
* mas_empty_area() - Get the lowest address within the range that is
* sufficient for the size requested.
* @mas: The maple state
* @min: The lowest value of the range
* @max: The highest value of the range
* @size: The size needed
*/
int mas_empty_area(struct ma_state *mas, unsigned long min,
unsigned long max, unsigned long size)
{
unsigned char offset;
unsigned long *pivots;
enum maple_type mt;
struct maple_node *node;
if (min > max)
return -EINVAL;
if (size == 0 || max - min < size - 1)
return -EINVAL;
if (mas_is_start(mas)) mas_start(mas); else if (mas->offset >= 2) mas->offset -= 2; else if (!mas_skip_node(mas))
return -EBUSY;
/* Empty set */
if (mas_is_none(mas) || mas_is_ptr(mas)) return mas_sparse_area(mas, min, max, size, true);
/* The start of the window can only be within these values */
mas->index = min;
mas->last = max;
mas_awalk(mas, size); if (unlikely(mas_is_err(mas))) return xa_err(mas->node); offset = mas->offset; if (unlikely(offset == MAPLE_NODE_SLOTS))
return -EBUSY;
node = mas_mn(mas);
mt = mte_node_type(mas->node);
pivots = ma_pivots(node, mt); min = mas_safe_min(mas, pivots, offset); if (mas->index < min) mas->index = min; mas->last = mas->index + size - 1; mas->end = ma_data_end(node, mt, pivots, mas->max); return 0;
}
EXPORT_SYMBOL_GPL(mas_empty_area);
/*
* mas_empty_area_rev() - Get the highest address within the range that is
* sufficient for the size requested.
* @mas: The maple state
* @min: The lowest value of the range
* @max: The highest value of the range
* @size: The size needed
*/
int mas_empty_area_rev(struct ma_state *mas, unsigned long min,
unsigned long max, unsigned long size)
{
struct maple_enode *last = mas->node;
if (min > max)
return -EINVAL;
if (size == 0 || max - min < size - 1)
return -EINVAL;
if (mas_is_start(mas)) {
mas_start(mas);
mas->offset = mas_data_end(mas);
} else if (mas->offset >= 2) {
mas->offset -= 2;
} else if (!mas_rewind_node(mas)) {
return -EBUSY;
}
/* Empty set. */
if (mas_is_none(mas) || mas_is_ptr(mas))
return mas_sparse_area(mas, min, max, size, false);
/* The start of the window can only be within these values. */
mas->index = min;
mas->last = max;
while (!mas_rev_awalk(mas, size, &min, &max)) {
if (last == mas->node) {
if (!mas_rewind_node(mas))
return -EBUSY;
} else {
last = mas->node;
}
}
if (mas_is_err(mas))
return xa_err(mas->node);
if (unlikely(mas->offset == MAPLE_NODE_SLOTS))
return -EBUSY;
/* Trim the upper limit to the max. */
if (max < mas->last)
mas->last = max;
mas->index = mas->last - size + 1;
mas->end = mas_data_end(mas);
return 0;
}
EXPORT_SYMBOL_GPL(mas_empty_area_rev);
/*
* mte_dead_leaves() - Mark all leaves of a node as dead.
* @mas: The maple state
* @slots: Pointer to the slot array
* @type: The maple node type
*
* Must hold the write lock.
*
* Return: The number of leaves marked as dead.
*/
static inline
unsigned char mte_dead_leaves(struct maple_enode *enode, struct maple_tree *mt,
void __rcu **slots)
{
struct maple_node *node;
enum maple_type type;
void *entry;
int offset;
for (offset = 0; offset < mt_slot_count(enode); offset++) {
entry = mt_slot(mt, slots, offset);
type = mte_node_type(entry);
node = mte_to_node(entry);
/* Use both node and type to catch LE & BE metadata */
if (!node || !type)
break;
mte_set_node_dead(entry);
node->type = type;
rcu_assign_pointer(slots[offset], node);
}
return offset;
}
/**
* mte_dead_walk() - Walk down a dead tree to just before the leaves
* @enode: The maple encoded node
* @offset: The starting offset
*
* Note: This can only be used from the RCU callback context.
*/
static void __rcu **mte_dead_walk(struct maple_enode **enode, unsigned char offset)
{
struct maple_node *node, *next;
void __rcu **slots = NULL;
next = mte_to_node(*enode);
do {
*enode = ma_enode_ptr(next);
node = mte_to_node(*enode);
slots = ma_slots(node, node->type);
next = rcu_dereference_protected(slots[offset],
lock_is_held(&rcu_callback_map));
offset = 0;
} while (!ma_is_leaf(next->type));
return slots;
}
/**
* mt_free_walk() - Walk & free a tree in the RCU callback context
* @head: The RCU head that's within the node.
*
* Note: This can only be used from the RCU callback context.
*/
static void mt_free_walk(struct rcu_head *head)
{
void __rcu **slots;
struct maple_node *node, *start;
struct maple_enode *enode;
unsigned char offset;
enum maple_type type;
node = container_of(head, struct maple_node, rcu);
if (ma_is_leaf(node->type))
goto free_leaf;
start = node;
enode = mt_mk_node(node, node->type);
slots = mte_dead_walk(&enode, 0);
node = mte_to_node(enode);
do {
mt_free_bulk(node->slot_len, slots);
offset = node->parent_slot + 1;
enode = node->piv_parent;
if (mte_to_node(enode) == node)
goto free_leaf;
type = mte_node_type(enode);
slots = ma_slots(mte_to_node(enode), type);
if ((offset < mt_slots[type]) &&
rcu_dereference_protected(slots[offset],
lock_is_held(&rcu_callback_map)))
slots = mte_dead_walk(&enode, offset);
node = mte_to_node(enode);
} while ((node != start) || (node->slot_len < offset));
slots = ma_slots(node, node->type);
mt_free_bulk(node->slot_len, slots);
free_leaf:
mt_free_rcu(&node->rcu);
}
static inline void __rcu **mte_destroy_descend(struct maple_enode **enode,
struct maple_tree *mt, struct maple_enode *prev, unsigned char offset)
{
struct maple_node *node;
struct maple_enode *next = *enode;
void __rcu **slots = NULL;
enum maple_type type;
unsigned char next_offset = 0;
do {
*enode = next;
node = mte_to_node(*enode);
type = mte_node_type(*enode);
slots = ma_slots(node, type);
next = mt_slot_locked(mt, slots, next_offset);
if ((mte_dead_node(next)))
next = mt_slot_locked(mt, slots, ++next_offset);
mte_set_node_dead(*enode);
node->type = type;
node->piv_parent = prev;
node->parent_slot = offset;
offset = next_offset;
next_offset = 0;
prev = *enode;
} while (!mte_is_leaf(next));
return slots;
}
static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt,
bool free)
{
void __rcu **slots;
struct maple_node *node = mte_to_node(enode);
struct maple_enode *start;
if (mte_is_leaf(enode)) {
node->type = mte_node_type(enode);
goto free_leaf;
}
start = enode;
slots = mte_destroy_descend(&enode, mt, start, 0);
node = mte_to_node(enode); // Updated in the above call.
do {
enum maple_type type;
unsigned char offset;
struct maple_enode *parent, *tmp;
node->slot_len = mte_dead_leaves(enode, mt, slots);
if (free)
mt_free_bulk(node->slot_len, slots);
offset = node->parent_slot + 1;
enode = node->piv_parent;
if (mte_to_node(enode) == node)
goto free_leaf;
type = mte_node_type(enode);
slots = ma_slots(mte_to_node(enode), type);
if (offset >= mt_slots[type])
goto next;
tmp = mt_slot_locked(mt, slots, offset);
if (mte_node_type(tmp) && mte_to_node(tmp)) {
parent = enode;
enode = tmp;
slots = mte_destroy_descend(&enode, mt, parent, offset);
}
next:
node = mte_to_node(enode);
} while (start != enode);
node = mte_to_node(enode);
node->slot_len = mte_dead_leaves(enode, mt, slots);
if (free)
mt_free_bulk(node->slot_len, slots);
free_leaf:
if (free)
mt_free_rcu(&node->rcu);
else
mt_clear_meta(mt, node, node->type);
}
/*
* mte_destroy_walk() - Free a tree or sub-tree.
* @enode: the encoded maple node (maple_enode) to start
* @mt: the tree to free - needed for node types.
*
* Must hold the write lock.
*/
static inline void mte_destroy_walk(struct maple_enode *enode,
struct maple_tree *mt)
{
struct maple_node *node = mte_to_node(enode);
if (mt_in_rcu(mt)) {
mt_destroy_walk(enode, mt, false);
call_rcu(&node->rcu, mt_free_walk);
} else {
mt_destroy_walk(enode, mt, true);
}
}
static void mas_wr_store_setup(struct ma_wr_state *wr_mas)
{
if (!mas_is_active(wr_mas->mas)) { if (mas_is_start(wr_mas->mas))
return;
if (unlikely(mas_is_paused(wr_mas->mas)))
goto reset;
if (unlikely(mas_is_none(wr_mas->mas)))
goto reset;
if (unlikely(mas_is_overflow(wr_mas->mas)))
goto reset;
if (unlikely(mas_is_underflow(wr_mas->mas)))
goto reset;
}
/*
* A less strict version of mas_is_span_wr() where we allow spanning
* writes within this node. This is to stop partial walks in
* mas_prealloc() from being reset.
*/
if (wr_mas->mas->last > wr_mas->mas->max)
goto reset;
if (wr_mas->entry)
return;
if (mte_is_leaf(wr_mas->mas->node) &&
wr_mas->mas->last == wr_mas->mas->max)
goto reset;
return;
reset:
mas_reset(wr_mas->mas);
}
/* Interface */
/**
* mas_store() - Store an @entry.
* @mas: The maple state.
* @entry: The entry to store.
*
* The @mas->index and @mas->last is used to set the range for the @entry.
* Note: The @mas should have pre-allocated entries to ensure there is memory to
* store the entry. Please see mas_expected_entries()/mas_destroy() for more details.
*
* Return: the first entry between mas->index and mas->last or %NULL.
*/
void *mas_store(struct ma_state *mas, void *entry)
{
MA_WR_STATE(wr_mas, mas, entry);
trace_ma_write(__func__, mas, 0, entry);
#ifdef CONFIG_DEBUG_MAPLE_TREE
if (MAS_WARN_ON(mas, mas->index > mas->last))
pr_err("Error %lX > %lX %p\n", mas->index, mas->last, entry);
if (mas->index > mas->last) {
mas_set_err(mas, -EINVAL);
return NULL;
}
#endif
/*
* Storing is the same operation as insert with the added caveat that it
* can overwrite entries. Although this seems simple enough, one may
* want to examine what happens if a single store operation was to
* overwrite multiple entries within a self-balancing B-Tree.
*/
mas_wr_store_setup(&wr_mas);
mas_wr_store_entry(&wr_mas);
return wr_mas.content;
}
EXPORT_SYMBOL_GPL(mas_store);
/**
* mas_store_gfp() - Store a value into the tree.
* @mas: The maple state
* @entry: The entry to store
* @gfp: The GFP_FLAGS to use for allocations if necessary.
*
* Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not
* be allocated.
*/
int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp)
{
MA_WR_STATE(wr_mas, mas, entry);
mas_wr_store_setup(&wr_mas);
trace_ma_write(__func__, mas, 0, entry);
retry:
mas_wr_store_entry(&wr_mas);
if (unlikely(mas_nomem(mas, gfp)))
goto retry;
if (unlikely(mas_is_err(mas)))
return xa_err(mas->node);
return 0;
}
EXPORT_SYMBOL_GPL(mas_store_gfp);
/**
* mas_store_prealloc() - Store a value into the tree using memory
* preallocated in the maple state.
* @mas: The maple state
* @entry: The entry to store.
*/
void mas_store_prealloc(struct ma_state *mas, void *entry)
{
MA_WR_STATE(wr_mas, mas, entry);
mas_wr_store_setup(&wr_mas);
trace_ma_write(__func__, mas, 0, entry);
mas_wr_store_entry(&wr_mas);
MAS_WR_BUG_ON(&wr_mas, mas_is_err(mas));
mas_destroy(mas);
}
EXPORT_SYMBOL_GPL(mas_store_prealloc);
/**
* mas_preallocate() - Preallocate enough nodes for a store operation
* @mas: The maple state
* @entry: The entry that will be stored
* @gfp: The GFP_FLAGS to use for allocations.
*
* Return: 0 on success, -ENOMEM if memory could not be allocated.
*/
int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp)
{
MA_WR_STATE(wr_mas, mas, entry);
unsigned char node_size;
int request = 1;
int ret;
if (unlikely(!mas->index && mas->last == ULONG_MAX))
goto ask_now;
mas_wr_store_setup(&wr_mas); wr_mas.content = mas_start(mas);
/* Root expand */
if (unlikely(mas_is_none(mas) || mas_is_ptr(mas)))
goto ask_now;
if (unlikely(!mas_wr_walk(&wr_mas))) {
/* Spanning store, use worst case for now */
request = 1 + mas_mt_height(mas) * 3;
goto ask_now;
}
/* At this point, we are at the leaf node that needs to be altered. */
/* Exact fit, no nodes needed. */
if (wr_mas.r_min == mas->index && wr_mas.r_max == mas->last) return 0; mas_wr_end_piv(&wr_mas); node_size = mas_wr_new_end(&wr_mas);
/* Slot store, does not require additional nodes */
if (node_size == mas->end) {
/* reuse node */
if (!mt_in_rcu(mas->tree))
return 0;
/* shifting boundary */
if (wr_mas.offset_end - mas->offset == 1)
return 0;
}
if (node_size >= mt_slots[wr_mas.type]) {
/* Split, worst case for now. */
request = 1 + mas_mt_height(mas) * 2;
goto ask_now;
}
/* New root needs a single node */
if (unlikely(mte_is_root(mas->node)))
goto ask_now;
/* Potential spanning rebalance collapsing a node, use worst-case */
if (node_size - 1 <= mt_min_slots[wr_mas.type]) request = mas_mt_height(mas) * 2 - 1;
/* node store, slot store needs one node */
ask_now:
mas_node_count_gfp(mas, request, gfp);
mas->mas_flags |= MA_STATE_PREALLOC;
if (likely(!mas_is_err(mas)))
return 0;
mas_set_alloc_req(mas, 0); ret = xa_err(mas->node); mas_reset(mas);
mas_destroy(mas);
mas_reset(mas);
return ret;
}
EXPORT_SYMBOL_GPL(mas_preallocate);
/*
* mas_destroy() - destroy a maple state.
* @mas: The maple state
*
* Upon completion, check the left-most node and rebalance against the node to
* the right if necessary. Frees any allocated nodes associated with this maple
* state.
*/
void mas_destroy(struct ma_state *mas)
{
struct maple_alloc *node;
unsigned long total;
/*
* When using mas_for_each() to insert an expected number of elements,
* it is possible that the number inserted is less than the expected
* number. To fix an invalid final node, a check is performed here to
* rebalance the previous node with the final node.
*/
if (mas->mas_flags & MA_STATE_REBALANCE) {
unsigned char end;
mas_start(mas); mtree_range_walk(mas);
end = mas->end + 1;
if (end < mt_min_slot_count(mas->node) - 1)
mas_destroy_rebalance(mas, end); mas->mas_flags &= ~MA_STATE_REBALANCE;
}
mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC); total = mas_allocated(mas);
while (total) {
node = mas->alloc; mas->alloc = node->slot[0];
if (node->node_count > 1) {
size_t count = node->node_count - 1;
mt_free_bulk(count, (void __rcu **)&node->slot[1]);
total -= count;
}
mt_free_one(ma_mnode_ptr(node));
total--;
}
mas->alloc = NULL;
}
EXPORT_SYMBOL_GPL(mas_destroy);
/*
* mas_expected_entries() - Set the expected number of entries that will be inserted.
* @mas: The maple state
* @nr_entries: The number of expected entries.
*
* This will attempt to pre-allocate enough nodes to store the expected number
* of entries. The allocations will occur using the bulk allocator interface
* for speed. Please call mas_destroy() on the @mas after inserting the entries
* to ensure any unused nodes are freed.
*
* Return: 0 on success, -ENOMEM if memory could not be allocated.
*/
int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries)
{
int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2;
struct maple_enode *enode = mas->node;
int nr_nodes;
int ret;
/*
* Sometimes it is necessary to duplicate a tree to a new tree, such as
* forking a process and duplicating the VMAs from one tree to a new
* tree. When such a situation arises, it is known that the new tree is
* not going to be used until the entire tree is populated. For
* performance reasons, it is best to use a bulk load with RCU disabled.
* This allows for optimistic splitting that favours the left and reuse
* of nodes during the operation.
*/
/* Optimize splitting for bulk insert in-order */
mas->mas_flags |= MA_STATE_BULK;
/*
* Avoid overflow, assume a gap between each entry and a trailing null.
* If this is wrong, it just means allocation can happen during
* insertion of entries.
*/
nr_nodes = max(nr_entries, nr_entries * 2 + 1);
if (!mt_is_alloc(mas->tree))
nonleaf_cap = MAPLE_RANGE64_SLOTS - 2;
/* Leaves; reduce slots to keep space for expansion */
nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2);
/* Internal nodes */
nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap);
/* Add working room for split (2 nodes) + new parents */
mas_node_count_gfp(mas, nr_nodes + 3, GFP_KERNEL);
/* Detect if allocations run out */
mas->mas_flags |= MA_STATE_PREALLOC;
if (!mas_is_err(mas))
return 0;
ret = xa_err(mas->node);
mas->node = enode;
mas_destroy(mas);
return ret;
}
EXPORT_SYMBOL_GPL(mas_expected_entries);
static bool mas_next_setup(struct ma_state *mas, unsigned long max,
void **entry)
{
bool was_none = mas_is_none(mas);
if (unlikely(mas->last >= max)) {
mas->status = ma_overflow;
return true;
}
switch (mas->status) {
case ma_active:
return false;
case ma_none:
fallthrough;
case ma_pause:
mas->status = ma_start;
fallthrough;
case ma_start:
mas_walk(mas); /* Retries on dead nodes handled by mas_walk */
break;
case ma_overflow:
/* Overflowed before, but the max changed */
mas->status = ma_active;
break;
case ma_underflow:
/* The user expects the mas to be one before where it is */
mas->status = ma_active;
*entry = mas_walk(mas);
if (*entry)
return true;
break;
case ma_root:
break;
case ma_error:
return true;
}
if (likely(mas_is_active(mas))) /* Fast path */
return false;
if (mas_is_ptr(mas)) { *entry = NULL; if (was_none && mas->index == 0) { mas->index = mas->last = 0;
return true;
}
mas->index = 1;
mas->last = ULONG_MAX;
mas->status = ma_none;
return true;
}
if (mas_is_none(mas))
return true;
return false;
}
/**
* mas_next() - Get the next entry.
* @mas: The maple state
* @max: The maximum index to check.
*
* Returns the next entry after @mas->index.
* Must hold rcu_read_lock or the write lock.
* Can return the zero entry.
*
* Return: The next entry or %NULL
*/
void *mas_next(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
if (mas_next_setup(mas, max, &entry))
return entry;
/* Retries on dead nodes handled by mas_next_slot */
return mas_next_slot(mas, max, false);
}
EXPORT_SYMBOL_GPL(mas_next);
/**
* mas_next_range() - Advance the maple state to the next range
* @mas: The maple state
* @max: The maximum index to check.
*
* Sets @mas->index and @mas->last to the range.
* Must hold rcu_read_lock or the write lock.
* Can return the zero entry.
*
* Return: The next entry or %NULL
*/
void *mas_next_range(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
if (mas_next_setup(mas, max, &entry))
return entry;
/* Retries on dead nodes handled by mas_next_slot */
return mas_next_slot(mas, max, true);
}
EXPORT_SYMBOL_GPL(mas_next_range);
/**
* mt_next() - get the next value in the maple tree
* @mt: The maple tree
* @index: The start index
* @max: The maximum index to check
*
* Takes RCU read lock internally to protect the search, which does not
* protect the returned pointer after dropping RCU read lock.
* See also: Documentation/core-api/maple_tree.rst
*
* Return: The entry higher than @index or %NULL if nothing is found.
*/
void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max)
{
void *entry = NULL;
MA_STATE(mas, mt, index, index);
rcu_read_lock();
entry = mas_next(&mas, max);
rcu_read_unlock();
return entry;
}
EXPORT_SYMBOL_GPL(mt_next);
static bool mas_prev_setup(struct ma_state *mas, unsigned long min, void **entry)
{
if (unlikely(mas->index <= min)) { mas->status = ma_underflow;
return true;
}
switch (mas->status) {
case ma_active:
return false;
case ma_start:
break;
case ma_none:
fallthrough;
case ma_pause:
mas->status = ma_start;
break;
case ma_underflow:
/* underflowed before but the min changed */
mas->status = ma_active;
break;
case ma_overflow:
/* User expects mas to be one after where it is */
mas->status = ma_active;
*entry = mas_walk(mas);
if (*entry)
return true;
break;
case ma_root:
break;
case ma_error:
return true;
}
if (mas_is_start(mas)) mas_walk(mas); if (unlikely(mas_is_ptr(mas))) { if (!mas->index) { mas->status = ma_none;
return true;
}
mas->index = mas->last = 0; *entry = mas_root(mas);
return true;
}
if (mas_is_none(mas)) { if (mas->index) {
/* Walked to out-of-range pointer? */
mas->index = mas->last = 0;
mas->status = ma_root;
*entry = mas_root(mas);
return true;
}
return true;
}
return false;
}
/**
* mas_prev() - Get the previous entry
* @mas: The maple state
* @min: The minimum value to check.
*
* Must hold rcu_read_lock or the write lock.
* Will reset mas to ma_start if the status is ma_none. Will stop on not
* searchable nodes.
*
* Return: the previous value or %NULL.
*/
void *mas_prev(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
if (mas_prev_setup(mas, min, &entry))
return entry; return mas_prev_slot(mas, min, false);
}
EXPORT_SYMBOL_GPL(mas_prev);
/**
* mas_prev_range() - Advance to the previous range
* @mas: The maple state
* @min: The minimum value to check.
*
* Sets @mas->index and @mas->last to the range.
* Must hold rcu_read_lock or the write lock.
* Will reset mas to ma_start if the node is ma_none. Will stop on not
* searchable nodes.
*
* Return: the previous value or %NULL.
*/
void *mas_prev_range(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
if (mas_prev_setup(mas, min, &entry))
return entry;
return mas_prev_slot(mas, min, true);
}
EXPORT_SYMBOL_GPL(mas_prev_range);
/**
* mt_prev() - get the previous value in the maple tree
* @mt: The maple tree
* @index: The start index
* @min: The minimum index to check
*
* Takes RCU read lock internally to protect the search, which does not
* protect the returned pointer after dropping RCU read lock.
* See also: Documentation/core-api/maple_tree.rst
*
* Return: The entry before @index or %NULL if nothing is found.
*/
void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min)
{
void *entry = NULL;
MA_STATE(mas, mt, index, index);
rcu_read_lock();
entry = mas_prev(&mas, min);
rcu_read_unlock();
return entry;
}
EXPORT_SYMBOL_GPL(mt_prev);
/**
* mas_pause() - Pause a mas_find/mas_for_each to drop the lock.
* @mas: The maple state to pause
*
* Some users need to pause a walk and drop the lock they're holding in
* order to yield to a higher priority thread or carry out an operation
* on an entry. Those users should call this function before they drop
* the lock. It resets the @mas to be suitable for the next iteration
* of the loop after the user has reacquired the lock. If most entries
* found during a walk require you to call mas_pause(), the mt_for_each()
* iterator may be more appropriate.
*
*/
void mas_pause(struct ma_state *mas)
{
mas->status = ma_pause;
mas->node = NULL;
}
EXPORT_SYMBOL_GPL(mas_pause);
/**
* mas_find_setup() - Internal function to set up mas_find*().
* @mas: The maple state
* @max: The maximum index
* @entry: Pointer to the entry
*
* Returns: True if entry is the answer, false otherwise.
*/
static __always_inline bool mas_find_setup(struct ma_state *mas, unsigned long max, void **entry)
{
switch (mas->status) {
case ma_active:
if (mas->last < max)
return false;
return true;
case ma_start:
break;
case ma_pause:
if (unlikely(mas->last >= max))
return true;
mas->index = ++mas->last;
mas->status = ma_start;
break;
case ma_none:
if (unlikely(mas->last >= max))
return true;
mas->index = mas->last;
mas->status = ma_start;
break;
case ma_underflow:
/* mas is pointing at entry before unable to go lower */
if (unlikely(mas->index >= max)) { mas->status = ma_overflow;
return true;
}
mas->status = ma_active;
*entry = mas_walk(mas);
if (*entry)
return true;
break;
case ma_overflow:
if (unlikely(mas->last >= max))
return true;
mas->status = ma_active;
*entry = mas_walk(mas);
if (*entry)
return true;
break;
case ma_root:
break;
case ma_error:
return true;
}
if (mas_is_start(mas)) {
/* First run or continue */
if (mas->index > max)
return true;
*entry = mas_walk(mas);
if (*entry)
return true;
}
if (unlikely(mas_is_ptr(mas)))
goto ptr_out_of_range;
if (unlikely(mas_is_none(mas)))
return true;
if (mas->index == max)
return true;
return false;
ptr_out_of_range:
mas->status = ma_none;
mas->index = 1;
mas->last = ULONG_MAX;
return true;
}
/**
* mas_find() - On the first call, find the entry at or after mas->index up to
* %max. Otherwise, find the entry after mas->index.
* @mas: The maple state
* @max: The maximum value to check.
*
* Must hold rcu_read_lock or the write lock.
* If an entry exists, last and index are updated accordingly.
* May set @mas->status to ma_overflow.
*
* Return: The entry or %NULL.
*/
void *mas_find(struct ma_state *mas, unsigned long max)
{
void *entry = NULL; if (mas_find_setup(mas, max, &entry))
return entry;
/* Retries on dead nodes handled by mas_next_slot */
entry = mas_next_slot(mas, max, false);
/* Ignore overflow */
mas->status = ma_active;
return entry;
}
EXPORT_SYMBOL_GPL(mas_find);
/**
* mas_find_range() - On the first call, find the entry at or after
* mas->index up to %max. Otherwise, advance to the next slot mas->index.
* @mas: The maple state
* @max: The maximum value to check.
*
* Must hold rcu_read_lock or the write lock.
* If an entry exists, last and index are updated accordingly.
* May set @mas->status to ma_overflow.
*
* Return: The entry or %NULL.
*/
void *mas_find_range(struct ma_state *mas, unsigned long max)
{
void *entry = NULL;
if (mas_find_setup(mas, max, &entry))
return entry;
/* Retries on dead nodes handled by mas_next_slot */
return mas_next_slot(mas, max, true);
}
EXPORT_SYMBOL_GPL(mas_find_range);
/**
* mas_find_rev_setup() - Internal function to set up mas_find_*_rev()
* @mas: The maple state
* @min: The minimum index
* @entry: Pointer to the entry
*
* Returns: True if entry is the answer, false otherwise.
*/
static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
void **entry)
{
switch (mas->status) {
case ma_active:
goto active;
case ma_start:
break;
case ma_pause:
if (unlikely(mas->index <= min)) {
mas->status = ma_underflow;
return true;
}
mas->last = --mas->index;
mas->status = ma_start;
break;
case ma_none:
if (mas->index <= min)
goto none;
mas->last = mas->index;
mas->status = ma_start;
break;
case ma_overflow: /* user expects the mas to be one after where it is */
if (unlikely(mas->index <= min)) {
mas->status = ma_underflow;
return true;
}
mas->status = ma_active;
break;
case ma_underflow: /* user expects the mas to be one before where it is */
if (unlikely(mas->index <= min))
return true;
mas->status = ma_active;
break;
case ma_root:
break;
case ma_error:
return true;
}
if (mas_is_start(mas)) {
/* First run or continue */
if (mas->index < min)
return true;
*entry = mas_walk(mas);
if (*entry)
return true;
}
if (unlikely(mas_is_ptr(mas)))
goto none;
if (unlikely(mas_is_none(mas))) {
/*
* Walked to the location, and there was nothing so the previous
* location is 0.
*/
mas->last = mas->index = 0;
mas->status = ma_root;
*entry = mas_root(mas);
return true;
}
active:
if (mas->index < min)
return true;
return false;
none:
mas->status = ma_none;
return true;
}
/**
* mas_find_rev: On the first call, find the first non-null entry at or below
* mas->index down to %min. Otherwise find the first non-null entry below
* mas->index down to %min.
* @mas: The maple state
* @min: The minimum value to check.
*
* Must hold rcu_read_lock or the write lock.
* If an entry exists, last and index are updated accordingly.
* May set @mas->status to ma_underflow.
*
* Return: The entry or %NULL.
*/
void *mas_find_rev(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
if (mas_find_rev_setup(mas, min, &entry))
return entry;
/* Retries on dead nodes handled by mas_prev_slot */
return mas_prev_slot(mas, min, false);
}
EXPORT_SYMBOL_GPL(mas_find_rev);
/**
* mas_find_range_rev: On the first call, find the first non-null entry at or
* below mas->index down to %min. Otherwise advance to the previous slot after
* mas->index down to %min.
* @mas: The maple state
* @min: The minimum value to check.
*
* Must hold rcu_read_lock or the write lock.
* If an entry exists, last and index are updated accordingly.
* May set @mas->status to ma_underflow.
*
* Return: The entry or %NULL.
*/
void *mas_find_range_rev(struct ma_state *mas, unsigned long min)
{
void *entry = NULL;
if (mas_find_rev_setup(mas, min, &entry))
return entry;
/* Retries on dead nodes handled by mas_prev_slot */
return mas_prev_slot(mas, min, true);
}
EXPORT_SYMBOL_GPL(mas_find_range_rev);
/**
* mas_erase() - Find the range in which index resides and erase the entire
* range.
* @mas: The maple state
*
* Must hold the write lock.
* Searches for @mas->index, sets @mas->index and @mas->last to the range and
* erases that range.
*
* Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated.
*/
void *mas_erase(struct ma_state *mas)
{
void *entry;
MA_WR_STATE(wr_mas, mas, NULL);
if (!mas_is_active(mas) || !mas_is_start(mas))
mas->status = ma_start;
/* Retry unnecessary when holding the write lock. */
entry = mas_state_walk(mas);
if (!entry)
return NULL;
write_retry:
/* Must reset to ensure spanning writes of last slot are detected */
mas_reset(mas);
mas_wr_store_setup(&wr_mas);
mas_wr_store_entry(&wr_mas);
if (mas_nomem(mas, GFP_KERNEL))
goto write_retry;
return entry;
}
EXPORT_SYMBOL_GPL(mas_erase);
/**
* mas_nomem() - Check if there was an error allocating and do the allocation
* if necessary If there are allocations, then free them.
* @mas: The maple state
* @gfp: The GFP_FLAGS to use for allocations
* Return: true on allocation, false otherwise.
*/
bool mas_nomem(struct ma_state *mas, gfp_t gfp)
__must_hold(mas->tree->ma_lock)
{
if (likely(mas->node != MA_ERROR(-ENOMEM))) { mas_destroy(mas); return false;
}
if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { mtree_unlock(mas->tree);
mas_alloc_nodes(mas, gfp);
mtree_lock(mas->tree);
} else {
mas_alloc_nodes(mas, gfp);
}
if (!mas_allocated(mas))
return false;
mas->status = ma_start;
return true;
}
void __init maple_tree_init(void)
{
maple_node_cache = kmem_cache_create("maple_node",
sizeof(struct maple_node), sizeof(struct maple_node),
SLAB_PANIC, NULL);
}
/**
* mtree_load() - Load a value stored in a maple tree
* @mt: The maple tree
* @index: The index to load
*
* Return: the entry or %NULL
*/
void *mtree_load(struct maple_tree *mt, unsigned long index)
{
MA_STATE(mas, mt, index, index);
void *entry;
trace_ma_read(__func__, &mas);
rcu_read_lock();
retry:
entry = mas_start(&mas); if (unlikely(mas_is_none(&mas)))
goto unlock;
if (unlikely(mas_is_ptr(&mas))) { if (index)
entry = NULL;
goto unlock;
}
entry = mtree_lookup_walk(&mas); if (!entry && unlikely(mas_is_start(&mas)))
goto retry;
unlock:
rcu_read_unlock(); if (xa_is_zero(entry))
return NULL;
return entry;
}
EXPORT_SYMBOL(mtree_load);
/**
* mtree_store_range() - Store an entry at a given range.
* @mt: The maple tree
* @index: The start of the range
* @last: The end of the range
* @entry: The entry to store
* @gfp: The GFP_FLAGS to use for allocations
*
* Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not
* be allocated.
*/
int mtree_store_range(struct maple_tree *mt, unsigned long index,
unsigned long last, void *entry, gfp_t gfp)
{
MA_STATE(mas, mt, index, last);
MA_WR_STATE(wr_mas, &mas, entry);
trace_ma_write(__func__, &mas, 0, entry);
if (WARN_ON_ONCE(xa_is_advanced(entry)))
return -EINVAL;
if (index > last)
return -EINVAL;
mtree_lock(mt);
retry:
mas_wr_store_entry(&wr_mas);
if (mas_nomem(&mas, gfp))
goto retry;
mtree_unlock(mt);
if (mas_is_err(&mas))
return xa_err(mas.node);
return 0;
}
EXPORT_SYMBOL(mtree_store_range);
/**
* mtree_store() - Store an entry at a given index.
* @mt: The maple tree
* @index: The index to store the value
* @entry: The entry to store
* @gfp: The GFP_FLAGS to use for allocations
*
* Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not
* be allocated.
*/
int mtree_store(struct maple_tree *mt, unsigned long index, void *entry,
gfp_t gfp)
{
return mtree_store_range(mt, index, index, entry, gfp);
}
EXPORT_SYMBOL(mtree_store);
/**
* mtree_insert_range() - Insert an entry at a given range if there is no value.
* @mt: The maple tree
* @first: The start of the range
* @last: The end of the range
* @entry: The entry to store
* @gfp: The GFP_FLAGS to use for allocations.
*
* Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid
* request, -ENOMEM if memory could not be allocated.
*/
int mtree_insert_range(struct maple_tree *mt, unsigned long first,
unsigned long last, void *entry, gfp_t gfp)
{
MA_STATE(ms, mt, first, last);
if (WARN_ON_ONCE(xa_is_advanced(entry)))
return -EINVAL;
if (first > last)
return -EINVAL;
mtree_lock(mt);
retry:
mas_insert(&ms, entry);
if (mas_nomem(&ms, gfp))
goto retry;
mtree_unlock(mt);
if (mas_is_err(&ms))
return xa_err(ms.node);
return 0;
}
EXPORT_SYMBOL(mtree_insert_range);
/**
* mtree_insert() - Insert an entry at a given index if there is no value.
* @mt: The maple tree
* @index : The index to store the value
* @entry: The entry to store
* @gfp: The GFP_FLAGS to use for allocations.
*
* Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid
* request, -ENOMEM if memory could not be allocated.
*/
int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry,
gfp_t gfp)
{
return mtree_insert_range(mt, index, index, entry, gfp);
}
EXPORT_SYMBOL(mtree_insert);
int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp,
void *entry, unsigned long size, unsigned long min,
unsigned long max, gfp_t gfp)
{
int ret = 0;
MA_STATE(mas, mt, 0, 0);
if (!mt_is_alloc(mt))
return -EINVAL;
if (WARN_ON_ONCE(mt_is_reserved(entry)))
return -EINVAL;
mtree_lock(mt);
retry:
ret = mas_empty_area(&mas, min, max, size);
if (ret)
goto unlock;
mas_insert(&mas, entry);
/*
* mas_nomem() may release the lock, causing the allocated area
* to be unavailable, so try to allocate a free area again.
*/
if (mas_nomem(&mas, gfp))
goto retry;
if (mas_is_err(&mas))
ret = xa_err(mas.node);
else
*startp = mas.index;
unlock:
mtree_unlock(mt);
return ret;
}
EXPORT_SYMBOL(mtree_alloc_range);
/**
* mtree_alloc_cyclic() - Find somewhere to store this entry in the tree.
* @mt: The maple tree.
* @startp: Pointer to ID.
* @range_lo: Lower bound of range to search.
* @range_hi: Upper bound of range to search.
* @entry: The entry to store.
* @next: Pointer to next ID to allocate.
* @gfp: The GFP_FLAGS to use for allocations.
*
* Finds an empty entry in @mt after @next, stores the new index into
* the @id pointer, stores the entry at that index, then updates @next.
*
* @mt must be initialized with the MT_FLAGS_ALLOC_RANGE flag.
*
* Context: Any context. Takes and releases the mt.lock. May sleep if
* the @gfp flags permit.
*
* Return: 0 if the allocation succeeded without wrapping, 1 if the
* allocation succeeded after wrapping, -ENOMEM if memory could not be
* allocated, -EINVAL if @mt cannot be used, or -EBUSY if there are no
* free entries.
*/
int mtree_alloc_cyclic(struct maple_tree *mt, unsigned long *startp,
void *entry, unsigned long range_lo, unsigned long range_hi,
unsigned long *next, gfp_t gfp)
{
int ret;
MA_STATE(mas, mt, 0, 0);
if (!mt_is_alloc(mt))
return -EINVAL;
if (WARN_ON_ONCE(mt_is_reserved(entry)))
return -EINVAL;
mtree_lock(mt);
ret = mas_alloc_cyclic(&mas, startp, entry, range_lo, range_hi,
next, gfp);
mtree_unlock(mt);
return ret;
}
EXPORT_SYMBOL(mtree_alloc_cyclic);
int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp,
void *entry, unsigned long size, unsigned long min,
unsigned long max, gfp_t gfp)
{
int ret = 0;
MA_STATE(mas, mt, 0, 0);
if (!mt_is_alloc(mt))
return -EINVAL;
if (WARN_ON_ONCE(mt_is_reserved(entry)))
return -EINVAL;
mtree_lock(mt);
retry:
ret = mas_empty_area_rev(&mas, min, max, size);
if (ret)
goto unlock;
mas_insert(&mas, entry);
/*
* mas_nomem() may release the lock, causing the allocated area
* to be unavailable, so try to allocate a free area again.
*/
if (mas_nomem(&mas, gfp))
goto retry;
if (mas_is_err(&mas))
ret = xa_err(mas.node);
else
*startp = mas.index;
unlock:
mtree_unlock(mt);
return ret;
}
EXPORT_SYMBOL(mtree_alloc_rrange);
/**
* mtree_erase() - Find an index and erase the entire range.
* @mt: The maple tree
* @index: The index to erase
*
* Erasing is the same as a walk to an entry then a store of a NULL to that
* ENTIRE range. In fact, it is implemented as such using the advanced API.
*
* Return: The entry stored at the @index or %NULL
*/
void *mtree_erase(struct maple_tree *mt, unsigned long index)
{
void *entry = NULL;
MA_STATE(mas, mt, index, index);
trace_ma_op(__func__, &mas);
mtree_lock(mt);
entry = mas_erase(&mas);
mtree_unlock(mt);
return entry;
}
EXPORT_SYMBOL(mtree_erase);
/*
* mas_dup_free() - Free an incomplete duplication of a tree.
* @mas: The maple state of a incomplete tree.
*
* The parameter @mas->node passed in indicates that the allocation failed on
* this node. This function frees all nodes starting from @mas->node in the
* reverse order of mas_dup_build(). There is no need to hold the source tree
* lock at this time.
*/
static void mas_dup_free(struct ma_state *mas)
{
struct maple_node *node;
enum maple_type type;
void __rcu **slots;
unsigned char count, i;
/* Maybe the first node allocation failed. */
if (mas_is_none(mas))
return;
while (!mte_is_root(mas->node)) {
mas_ascend(mas);
if (mas->offset) {
mas->offset--;
do {
mas_descend(mas);
mas->offset = mas_data_end(mas);
} while (!mte_is_leaf(mas->node));
mas_ascend(mas);
}
node = mte_to_node(mas->node);
type = mte_node_type(mas->node);
slots = ma_slots(node, type);
count = mas_data_end(mas) + 1;
for (i = 0; i < count; i++)
((unsigned long *)slots)[i] &= ~MAPLE_NODE_MASK;
mt_free_bulk(count, slots);
}
node = mte_to_node(mas->node);
mt_free_one(node);
}
/*
* mas_copy_node() - Copy a maple node and replace the parent.
* @mas: The maple state of source tree.
* @new_mas: The maple state of new tree.
* @parent: The parent of the new node.
*
* Copy @mas->node to @new_mas->node, set @parent to be the parent of
* @new_mas->node. If memory allocation fails, @mas is set to -ENOMEM.
*/
static inline void mas_copy_node(struct ma_state *mas, struct ma_state *new_mas,
struct maple_pnode *parent)
{
struct maple_node *node = mte_to_node(mas->node);
struct maple_node *new_node = mte_to_node(new_mas->node);
unsigned long val;
/* Copy the node completely. */
memcpy(new_node, node, sizeof(struct maple_node));
/* Update the parent node pointer. */
val = (unsigned long)node->parent & MAPLE_NODE_MASK;
new_node->parent = ma_parent_ptr(val | (unsigned long)parent);
}
/*
* mas_dup_alloc() - Allocate child nodes for a maple node.
* @mas: The maple state of source tree.
* @new_mas: The maple state of new tree.
* @gfp: The GFP_FLAGS to use for allocations.
*
* This function allocates child nodes for @new_mas->node during the duplication
* process. If memory allocation fails, @mas is set to -ENOMEM.
*/
static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas,
gfp_t gfp)
{
struct maple_node *node = mte_to_node(mas->node);
struct maple_node *new_node = mte_to_node(new_mas->node);
enum maple_type type;
unsigned char request, count, i;
void __rcu **slots;
void __rcu **new_slots;
unsigned long val;
/* Allocate memory for child nodes. */
type = mte_node_type(mas->node);
new_slots = ma_slots(new_node, type);
request = mas_data_end(mas) + 1;
count = mt_alloc_bulk(gfp, request, (void **)new_slots);
if (unlikely(count < request)) {
memset(new_slots, 0, request * sizeof(void *));
mas_set_err(mas, -ENOMEM);
return;
}
/* Restore node type information in slots. */
slots = ma_slots(node, type);
for (i = 0; i < count; i++) {
val = (unsigned long)mt_slot_locked(mas->tree, slots, i);
val &= MAPLE_NODE_MASK;
((unsigned long *)new_slots)[i] |= val;
}
}
/*
* mas_dup_build() - Build a new maple tree from a source tree
* @mas: The maple state of source tree, need to be in MAS_START state.
* @new_mas: The maple state of new tree, need to be in MAS_START state.
* @gfp: The GFP_FLAGS to use for allocations.
*
* This function builds a new tree in DFS preorder. If the memory allocation
* fails, the error code -ENOMEM will be set in @mas, and @new_mas points to the
* last node. mas_dup_free() will free the incomplete duplication of a tree.
*
* Note that the attributes of the two trees need to be exactly the same, and the
* new tree needs to be empty, otherwise -EINVAL will be set in @mas.
*/
static inline void mas_dup_build(struct ma_state *mas, struct ma_state *new_mas,
gfp_t gfp)
{
struct maple_node *node;
struct maple_pnode *parent = NULL;
struct maple_enode *root;
enum maple_type type;
if (unlikely(mt_attr(mas->tree) != mt_attr(new_mas->tree)) ||
unlikely(!mtree_empty(new_mas->tree))) {
mas_set_err(mas, -EINVAL);
return;
}
root = mas_start(mas);
if (mas_is_ptr(mas) || mas_is_none(mas))
goto set_new_tree;
node = mt_alloc_one(gfp);
if (!node) {
new_mas->status = ma_none;
mas_set_err(mas, -ENOMEM);
return;
}
type = mte_node_type(mas->node);
root = mt_mk_node(node, type);
new_mas->node = root;
new_mas->min = 0;
new_mas->max = ULONG_MAX;
root = mte_mk_root(root);
while (1) {
mas_copy_node(mas, new_mas, parent);
if (!mte_is_leaf(mas->node)) {
/* Only allocate child nodes for non-leaf nodes. */
mas_dup_alloc(mas, new_mas, gfp);
if (unlikely(mas_is_err(mas)))
return;
} else {
/*
* This is the last leaf node and duplication is
* completed.
*/
if (mas->max == ULONG_MAX)
goto done;
/* This is not the last leaf node and needs to go up. */
do {
mas_ascend(mas);
mas_ascend(new_mas);
} while (mas->offset == mas_data_end(mas));
/* Move to the next subtree. */
mas->offset++;
new_mas->offset++;
}
mas_descend(mas);
parent = ma_parent_ptr(mte_to_node(new_mas->node));
mas_descend(new_mas);
mas->offset = 0;
new_mas->offset = 0;
}
done:
/* Specially handle the parent of the root node. */
mte_to_node(root)->parent = ma_parent_ptr(mas_tree_parent(new_mas));
set_new_tree:
/* Make them the same height */
new_mas->tree->ma_flags = mas->tree->ma_flags;
rcu_assign_pointer(new_mas->tree->ma_root, root);
}
/**
* __mt_dup(): Duplicate an entire maple tree
* @mt: The source maple tree
* @new: The new maple tree
* @gfp: The GFP_FLAGS to use for allocations
*
* This function duplicates a maple tree in Depth-First Search (DFS) pre-order
* traversal. It uses memcpy() to copy nodes in the source tree and allocate
* new child nodes in non-leaf nodes. The new node is exactly the same as the
* source node except for all the addresses stored in it. It will be faster than
* traversing all elements in the source tree and inserting them one by one into
* the new tree.
* The user needs to ensure that the attributes of the source tree and the new
* tree are the same, and the new tree needs to be an empty tree, otherwise
* -EINVAL will be returned.
* Note that the user needs to manually lock the source tree and the new tree.
*
* Return: 0 on success, -ENOMEM if memory could not be allocated, -EINVAL If
* the attributes of the two trees are different or the new tree is not an empty
* tree.
*/
int __mt_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp)
{
int ret = 0;
MA_STATE(mas, mt, 0, 0);
MA_STATE(new_mas, new, 0, 0);
mas_dup_build(&mas, &new_mas, gfp);
if (unlikely(mas_is_err(&mas))) {
ret = xa_err(mas.node);
if (ret == -ENOMEM)
mas_dup_free(&new_mas);
}
return ret;
}
EXPORT_SYMBOL(__mt_dup);
/**
* mtree_dup(): Duplicate an entire maple tree
* @mt: The source maple tree
* @new: The new maple tree
* @gfp: The GFP_FLAGS to use for allocations
*
* This function duplicates a maple tree in Depth-First Search (DFS) pre-order
* traversal. It uses memcpy() to copy nodes in the source tree and allocate
* new child nodes in non-leaf nodes. The new node is exactly the same as the
* source node except for all the addresses stored in it. It will be faster than
* traversing all elements in the source tree and inserting them one by one into
* the new tree.
* The user needs to ensure that the attributes of the source tree and the new
* tree are the same, and the new tree needs to be an empty tree, otherwise
* -EINVAL will be returned.
*
* Return: 0 on success, -ENOMEM if memory could not be allocated, -EINVAL If
* the attributes of the two trees are different or the new tree is not an empty
* tree.
*/
int mtree_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp)
{
int ret = 0;
MA_STATE(mas, mt, 0, 0);
MA_STATE(new_mas, new, 0, 0);
mas_lock(&new_mas);
mas_lock_nested(&mas, SINGLE_DEPTH_NESTING);
mas_dup_build(&mas, &new_mas, gfp);
mas_unlock(&mas);
if (unlikely(mas_is_err(&mas))) {
ret = xa_err(mas.node);
if (ret == -ENOMEM)
mas_dup_free(&new_mas);
}
mas_unlock(&new_mas);
return ret;
}
EXPORT_SYMBOL(mtree_dup);
/**
* __mt_destroy() - Walk and free all nodes of a locked maple tree.
* @mt: The maple tree
*
* Note: Does not handle locking.
*/
void __mt_destroy(struct maple_tree *mt)
{
void *root = mt_root_locked(mt);
rcu_assign_pointer(mt->ma_root, NULL);
if (xa_is_node(root))
mte_destroy_walk(root, mt);
mt->ma_flags = mt_attr(mt);
}
EXPORT_SYMBOL_GPL(__mt_destroy);
/**
* mtree_destroy() - Destroy a maple tree
* @mt: The maple tree
*
* Frees all resources used by the tree. Handles locking.
*/
void mtree_destroy(struct maple_tree *mt)
{
mtree_lock(mt);
__mt_destroy(mt);
mtree_unlock(mt);
}
EXPORT_SYMBOL(mtree_destroy);
/**
* mt_find() - Search from the start up until an entry is found.
* @mt: The maple tree
* @index: Pointer which contains the start location of the search
* @max: The maximum value of the search range
*
* Takes RCU read lock internally to protect the search, which does not
* protect the returned pointer after dropping RCU read lock.
* See also: Documentation/core-api/maple_tree.rst
*
* In case that an entry is found @index is updated to point to the next
* possible entry independent whether the found entry is occupying a
* single index or a range if indices.
*
* Return: The entry at or after the @index or %NULL
*/
void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max)
{
MA_STATE(mas, mt, *index, *index);
void *entry;
#ifdef CONFIG_DEBUG_MAPLE_TREE
unsigned long copy = *index;
#endif
trace_ma_read(__func__, &mas);
if ((*index) > max)
return NULL;
rcu_read_lock();
retry:
entry = mas_state_walk(&mas); if (mas_is_start(&mas))
goto retry;
if (unlikely(xa_is_zero(entry)))
entry = NULL;
if (entry)
goto unlock;
while (mas_is_active(&mas) && (mas.last < max)) { entry = mas_next_entry(&mas, max); if (likely(entry && !xa_is_zero(entry)))
break;
}
if (unlikely(xa_is_zero(entry)))
entry = NULL;
unlock:
rcu_read_unlock();
if (likely(entry)) {
*index = mas.last + 1;
#ifdef CONFIG_DEBUG_MAPLE_TREE
if (MT_WARN_ON(mt, (*index) && ((*index) <= copy))) pr_err("index not increased! %lx <= %lx\n",
*index, copy);
#endif
}
return entry;
}
EXPORT_SYMBOL(mt_find);
/**
* mt_find_after() - Search from the start up until an entry is found.
* @mt: The maple tree
* @index: Pointer which contains the start location of the search
* @max: The maximum value to check
*
* Same as mt_find() except that it checks @index for 0 before
* searching. If @index == 0, the search is aborted. This covers a wrap
* around of @index to 0 in an iterator loop.
*
* Return: The entry at or after the @index or %NULL
*/
void *mt_find_after(struct maple_tree *mt, unsigned long *index,
unsigned long max)
{
if (!(*index))
return NULL;
return mt_find(mt, index, max);
}
EXPORT_SYMBOL(mt_find_after);
#ifdef CONFIG_DEBUG_MAPLE_TREE
atomic_t maple_tree_tests_run;
EXPORT_SYMBOL_GPL(maple_tree_tests_run);
atomic_t maple_tree_tests_passed;
EXPORT_SYMBOL_GPL(maple_tree_tests_passed);
#ifndef __KERNEL__
extern void kmem_cache_set_non_kernel(struct kmem_cache *, unsigned int);
void mt_set_non_kernel(unsigned int val)
{
kmem_cache_set_non_kernel(maple_node_cache, val);
}
extern unsigned long kmem_cache_get_alloc(struct kmem_cache *);
unsigned long mt_get_alloc_size(void)
{
return kmem_cache_get_alloc(maple_node_cache);
}
extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *);
void mt_zero_nr_tallocated(void)
{
kmem_cache_zero_nr_tallocated(maple_node_cache);
}
extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *);
unsigned int mt_nr_tallocated(void)
{
return kmem_cache_nr_tallocated(maple_node_cache);
}
extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *);
unsigned int mt_nr_allocated(void)
{
return kmem_cache_nr_allocated(maple_node_cache);
}
void mt_cache_shrink(void)
{
}
#else
/*
* mt_cache_shrink() - For testing, don't use this.
*
* Certain testcases can trigger an OOM when combined with other memory
* debugging configuration options. This function is used to reduce the
* possibility of an out of memory even due to kmem_cache objects remaining
* around for longer than usual.
*/
void mt_cache_shrink(void)
{
kmem_cache_shrink(maple_node_cache);
}
EXPORT_SYMBOL_GPL(mt_cache_shrink);
#endif /* not defined __KERNEL__ */
/*
* mas_get_slot() - Get the entry in the maple state node stored at @offset.
* @mas: The maple state
* @offset: The offset into the slot array to fetch.
*
* Return: The entry stored at @offset.
*/
static inline struct maple_enode *mas_get_slot(struct ma_state *mas,
unsigned char offset)
{
return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)),
offset);
}
/* Depth first search, post-order */
static void mas_dfs_postorder(struct ma_state *mas, unsigned long max)
{
struct maple_enode *p, *mn = mas->node;
unsigned long p_min, p_max;
mas_next_node(mas, mas_mn(mas), max);
if (!mas_is_overflow(mas))
return;
if (mte_is_root(mn))
return;
mas->node = mn;
mas_ascend(mas);
do {
p = mas->node;
p_min = mas->min;
p_max = mas->max;
mas_prev_node(mas, 0);
} while (!mas_is_underflow(mas));
mas->node = p;
mas->max = p_max;
mas->min = p_min;
}
/* Tree validations */
static void mt_dump_node(const struct maple_tree *mt, void *entry,
unsigned long min, unsigned long max, unsigned int depth,
enum mt_dump_format format);
static void mt_dump_range(unsigned long min, unsigned long max,
unsigned int depth, enum mt_dump_format format)
{
static const char spaces[] = " ";
switch(format) {
case mt_dump_hex:
if (min == max)
pr_info("%.*s%lx: ", depth * 2, spaces, min);
else
pr_info("%.*s%lx-%lx: ", depth * 2, spaces, min, max);
break;
case mt_dump_dec:
if (min == max)
pr_info("%.*s%lu: ", depth * 2, spaces, min);
else
pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max);
}
}
static void mt_dump_entry(void *entry, unsigned long min, unsigned long max,
unsigned int depth, enum mt_dump_format format)
{
mt_dump_range(min, max, depth, format);
if (xa_is_value(entry))
pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry),
xa_to_value(entry), entry);
else if (xa_is_zero(entry))
pr_cont("zero (%ld)\n", xa_to_internal(entry));
else if (mt_is_reserved(entry))
pr_cont("UNKNOWN ENTRY (%p)\n", entry);
else
pr_cont("%p\n", entry);
}
static void mt_dump_range64(const struct maple_tree *mt, void *entry,
unsigned long min, unsigned long max, unsigned int depth,
enum mt_dump_format format)
{
struct maple_range_64 *node = &mte_to_node(entry)->mr64;
bool leaf = mte_is_leaf(entry);
unsigned long first = min;
int i;
pr_cont(" contents: ");
for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) {
switch(format) {
case mt_dump_hex:
pr_cont("%p %lX ", node->slot[i], node->pivot[i]);
break;
case mt_dump_dec:
pr_cont("%p %lu ", node->slot[i], node->pivot[i]);
}
}
pr_cont("%p\n", node->slot[i]);
for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) {
unsigned long last = max;
if (i < (MAPLE_RANGE64_SLOTS - 1))
last = node->pivot[i];
else if (!node->slot[i] && max != mt_node_max(entry))
break;
if (last == 0 && i > 0)
break;
if (leaf)
mt_dump_entry(mt_slot(mt, node->slot, i),
first, last, depth + 1, format);
else if (node->slot[i])
mt_dump_node(mt, mt_slot(mt, node->slot, i),
first, last, depth + 1, format);
if (last == max)
break;
if (last > max) {
switch(format) {
case mt_dump_hex:
pr_err("node %p last (%lx) > max (%lx) at pivot %d!\n",
node, last, max, i);
break;
case mt_dump_dec:
pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n",
node, last, max, i);
}
}
first = last + 1;
}
}
static void mt_dump_arange64(const struct maple_tree *mt, void *entry,
unsigned long min, unsigned long max, unsigned int depth,
enum mt_dump_format format)
{
struct maple_arange_64 *node = &mte_to_node(entry)->ma64;
bool leaf = mte_is_leaf(entry);
unsigned long first = min;
int i;
pr_cont(" contents: ");
for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) {
switch (format) {
case mt_dump_hex:
pr_cont("%lx ", node->gap[i]);
break;
case mt_dump_dec:
pr_cont("%lu ", node->gap[i]);
}
}
pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap);
for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) {
switch (format) {
case mt_dump_hex:
pr_cont("%p %lX ", node->slot[i], node->pivot[i]);
break;
case mt_dump_dec:
pr_cont("%p %lu ", node->slot[i], node->pivot[i]);
}
}
pr_cont("%p\n", node->slot[i]);
for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) {
unsigned long last = max;
if (i < (MAPLE_ARANGE64_SLOTS - 1))
last = node->pivot[i];
else if (!node->slot[i])
break;
if (last == 0 && i > 0)
break;
if (leaf)
mt_dump_entry(mt_slot(mt, node->slot, i),
first, last, depth + 1, format);
else if (node->slot[i])
mt_dump_node(mt, mt_slot(mt, node->slot, i),
first, last, depth + 1, format);
if (last == max)
break;
if (last > max) {
pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n",
node, last, max, i);
break;
}
first = last + 1;
}
}
static void mt_dump_node(const struct maple_tree *mt, void *entry,
unsigned long min, unsigned long max, unsigned int depth,
enum mt_dump_format format)
{
struct maple_node *node = mte_to_node(entry);
unsigned int type = mte_node_type(entry);
unsigned int i;
mt_dump_range(min, max, depth, format);
pr_cont("node %p depth %d type %d parent %p", node, depth, type,
node ? node->parent : NULL);
switch (type) {
case maple_dense:
pr_cont("\n");
for (i = 0; i < MAPLE_NODE_SLOTS; i++) {
if (min + i > max)
pr_cont("OUT OF RANGE: ");
mt_dump_entry(mt_slot(mt, node->slot, i),
min + i, min + i, depth, format);
}
break;
case maple_leaf_64:
case maple_range_64:
mt_dump_range64(mt, entry, min, max, depth, format);
break;
case maple_arange_64:
mt_dump_arange64(mt, entry, min, max, depth, format);
break;
default:
pr_cont(" UNKNOWN TYPE\n");
}
}
void mt_dump(const struct maple_tree *mt, enum mt_dump_format format)
{
void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt));
pr_info("maple_tree(%p) flags %X, height %u root %p\n",
mt, mt->ma_flags, mt_height(mt), entry);
if (!xa_is_node(entry))
mt_dump_entry(entry, 0, 0, 0, format);
else if (entry)
mt_dump_node(mt, entry, 0, mt_node_max(entry), 0, format);
}
EXPORT_SYMBOL_GPL(mt_dump);
/*
* Calculate the maximum gap in a node and check if that's what is reported in
* the parent (unless root).
*/
static void mas_validate_gaps(struct ma_state *mas)
{
struct maple_enode *mte = mas->node;
struct maple_node *p_mn, *node = mte_to_node(mte);
enum maple_type mt = mte_node_type(mas->node);
unsigned long gap = 0, max_gap = 0;
unsigned long p_end, p_start = mas->min;
unsigned char p_slot, offset;
unsigned long *gaps = NULL;
unsigned long *pivots = ma_pivots(node, mt);
unsigned int i;
if (ma_is_dense(mt)) { for (i = 0; i < mt_slot_count(mte); i++) { if (mas_get_slot(mas, i)) { if (gap > max_gap)
max_gap = gap;
gap = 0;
continue;
}
gap++;
}
goto counted;
}
gaps = ma_gaps(node, mt);
for (i = 0; i < mt_slot_count(mte); i++) { p_end = mas_safe_pivot(mas, pivots, i, mt); if (!gaps) { if (!mas_get_slot(mas, i)) gap = p_end - p_start + 1;
} else {
void *entry = mas_get_slot(mas, i); gap = gaps[i]; MT_BUG_ON(mas->tree, !entry); if (gap > p_end - p_start + 1) { pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n",
mas_mn(mas), i, gap, p_end, p_start,
p_end - p_start + 1);
MT_BUG_ON(mas->tree, gap > p_end - p_start + 1);
}
}
if (gap > max_gap)
max_gap = gap;
p_start = p_end + 1;
if (p_end >= mas->max)
break;
}
counted:
if (mt == maple_arange_64) { MT_BUG_ON(mas->tree, !gaps); offset = ma_meta_gap(node);
if (offset > i) {
pr_err("gap offset %p[%u] is invalid\n", node, offset);
MT_BUG_ON(mas->tree, 1);
}
if (gaps[offset] != max_gap) { pr_err("gap %p[%u] is not the largest gap %lu\n",
node, offset, max_gap);
MT_BUG_ON(mas->tree, 1);
}
for (i++ ; i < mt_slot_count(mte); i++) { if (gaps[i] != 0) { pr_err("gap %p[%u] beyond node limit != 0\n",
node, i);
MT_BUG_ON(mas->tree, 1);
}
}
}
if (mte_is_root(mte))
return;
p_slot = mte_parent_slot(mas->node); p_mn = mte_parent(mte); MT_BUG_ON(mas->tree, max_gap > mas->max); if (ma_gaps(p_mn, mas_parent_type(mas, mte))[p_slot] != max_gap) { pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap);
mt_dump(mas->tree, mt_dump_hex);
MT_BUG_ON(mas->tree, 1);
}
}
static void mas_validate_parent_slot(struct ma_state *mas)
{
struct maple_node *parent;
struct maple_enode *node;
enum maple_type p_type;
unsigned char p_slot;
void __rcu **slots;
int i;
if (mte_is_root(mas->node))
return;
p_slot = mte_parent_slot(mas->node); p_type = mas_parent_type(mas, mas->node);
parent = mte_parent(mas->node);
slots = ma_slots(parent, p_type); MT_BUG_ON(mas->tree, mas_mn(mas) == parent);
/* Check prev/next parent slot for duplicate node entry */
for (i = 0; i < mt_slots[p_type]; i++) { node = mas_slot(mas, slots, i); if (i == p_slot) { if (node != mas->node) pr_err("parent %p[%u] does not have %p\n",
parent, i, mas_mn(mas));
MT_BUG_ON(mas->tree, node != mas->node); } else if (node == mas->node) { pr_err("Invalid child %p at parent %p[%u] p_slot %u\n",
mas_mn(mas), parent, i, p_slot);
MT_BUG_ON(mas->tree, node == mas->node);
}
}
}
static void mas_validate_child_slot(struct ma_state *mas)
{
enum maple_type type = mte_node_type(mas->node); void __rcu **slots = ma_slots(mte_to_node(mas->node), type);
unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type);
struct maple_enode *child;
unsigned char i;
if (mte_is_leaf(mas->node))
return;
for (i = 0; i < mt_slots[type]; i++) { child = mas_slot(mas, slots, i); if (!child) { pr_err("Non-leaf node lacks child at %p[%u]\n",
mas_mn(mas), i);
MT_BUG_ON(mas->tree, 1);
}
if (mte_parent_slot(child) != i) { pr_err("Slot error at %p[%u]: child %p has pslot %u\n",
mas_mn(mas), i, mte_to_node(child),
mte_parent_slot(child));
MT_BUG_ON(mas->tree, 1);
}
if (mte_parent(child) != mte_to_node(mas->node)) { pr_err("child %p has parent %p not %p\n",
mte_to_node(child), mte_parent(child),
mte_to_node(mas->node));
MT_BUG_ON(mas->tree, 1);
}
if (i < mt_pivots[type] && pivots[i] == mas->max)
break;
}
}
/*
* Validate all pivots are within mas->min and mas->max, check metadata ends
* where the maximum ends and ensure there is no slots or pivots set outside of
* the end of the data.
*/
static void mas_validate_limits(struct ma_state *mas)
{
int i;
unsigned long prev_piv = 0;
enum maple_type type = mte_node_type(mas->node); void __rcu **slots = ma_slots(mte_to_node(mas->node), type);
unsigned long *pivots = ma_pivots(mas_mn(mas), type);
for (i = 0; i < mt_slots[type]; i++) {
unsigned long piv;
piv = mas_safe_pivot(mas, pivots, i, type); if (!piv && (i != 0)) { pr_err("Missing node limit pivot at %p[%u]",
mas_mn(mas), i);
MAS_WARN_ON(mas, 1);
}
if (prev_piv > piv) { pr_err("%p[%u] piv %lu < prev_piv %lu\n",
mas_mn(mas), i, piv, prev_piv);
MAS_WARN_ON(mas, piv < prev_piv);
}
if (piv < mas->min) { pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i,
piv, mas->min);
MAS_WARN_ON(mas, piv < mas->min);
}
if (piv > mas->max) { pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i,
piv, mas->max);
MAS_WARN_ON(mas, piv > mas->max);
}
prev_piv = piv;
if (piv == mas->max)
break;
}
if (mas_data_end(mas) != i) { pr_err("node%p: data_end %u != the last slot offset %u\n",
mas_mn(mas), mas_data_end(mas), i);
MT_BUG_ON(mas->tree, 1);
}
for (i += 1; i < mt_slots[type]; i++) { void *entry = mas_slot(mas, slots, i); if (entry && (i != mt_slots[type] - 1)) { pr_err("%p[%u] should not have entry %p\n", mas_mn(mas),
i, entry);
MT_BUG_ON(mas->tree, entry != NULL);
}
if (i < mt_pivots[type]) { unsigned long piv = pivots[i];
if (!piv)
continue;
pr_err("%p[%u] should not have piv %lu\n",
mas_mn(mas), i, piv);
MAS_WARN_ON(mas, i < mt_pivots[type] - 1);
}
}
}
static void mt_validate_nulls(struct maple_tree *mt)
{
void *entry, *last = (void *)1;
unsigned char offset = 0;
void __rcu **slots;
MA_STATE(mas, mt, 0, 0); mas_start(&mas); if (mas_is_none(&mas) || (mas_is_ptr(&mas)))
return;
while (!mte_is_leaf(mas.node)) mas_descend(&mas); slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node));
do {
entry = mas_slot(&mas, slots, offset); if (!last && !entry) { pr_err("Sequential nulls end at %p[%u]\n",
mas_mn(&mas), offset);
}
MT_BUG_ON(mt, !last && !entry);
last = entry;
if (offset == mas_data_end(&mas)) { mas_next_node(&mas, mas_mn(&mas), ULONG_MAX);
if (mas_is_overflow(&mas))
return;
offset = 0;
slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node));
} else {
offset++;
}
} while (!mas_is_overflow(&mas));
}
/*
* validate a maple tree by checking:
* 1. The limits (pivots are within mas->min to mas->max)
* 2. The gap is correctly set in the parents
*/
void mt_validate(struct maple_tree *mt)
{
unsigned char end;
MA_STATE(mas, mt, 0, 0); rcu_read_lock(); mas_start(&mas); if (!mas_is_active(&mas))
goto done;
while (!mte_is_leaf(mas.node)) mas_descend(&mas); while (!mas_is_overflow(&mas)) { MAS_WARN_ON(&mas, mte_dead_node(mas.node)); end = mas_data_end(&mas); if (MAS_WARN_ON(&mas, (end < mt_min_slot_count(mas.node)) &&
(mas.max != ULONG_MAX))) {
pr_err("Invalid size %u of %p\n", end, mas_mn(&mas));
}
mas_validate_parent_slot(&mas); mas_validate_limits(&mas); mas_validate_child_slot(&mas); if (mt_is_alloc(mt)) mas_validate_gaps(&mas); mas_dfs_postorder(&mas, ULONG_MAX);
}
mt_validate_nulls(mt);
done:
rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(mt_validate);
void mas_dump(const struct ma_state *mas)
{
pr_err("MAS: tree=%p enode=%p ", mas->tree, mas->node);
switch (mas->status) {
case ma_active:
pr_err("(ma_active)");
break;
case ma_none:
pr_err("(ma_none)");
break;
case ma_root:
pr_err("(ma_root)");
break;
case ma_start:
pr_err("(ma_start) ");
break;
case ma_pause:
pr_err("(ma_pause) ");
break;
case ma_overflow:
pr_err("(ma_overflow) ");
break;
case ma_underflow:
pr_err("(ma_underflow) ");
break;
case ma_error:
pr_err("(ma_error) ");
break;
}
pr_err("[%u/%u] index=%lx last=%lx\n", mas->offset, mas->end,
mas->index, mas->last);
pr_err(" min=%lx max=%lx alloc=%p, depth=%u, flags=%x\n",
mas->min, mas->max, mas->alloc, mas->depth, mas->mas_flags);
if (mas->index > mas->last)
pr_err("Check index & last\n");
}
EXPORT_SYMBOL_GPL(mas_dump);
void mas_wr_dump(const struct ma_wr_state *wr_mas)
{
pr_err("WR_MAS: node=%p r_min=%lx r_max=%lx\n",
wr_mas->node, wr_mas->r_min, wr_mas->r_max);
pr_err(" type=%u off_end=%u, node_end=%u, end_piv=%lx\n",
wr_mas->type, wr_mas->offset_end, wr_mas->mas->end,
wr_mas->end_piv);
}
EXPORT_SYMBOL_GPL(mas_wr_dump);
#endif /* CONFIG_DEBUG_MAPLE_TREE */
// SPDX-License-Identifier: GPL-2.0-only
/*
* A generic implementation of binary search for the Linux kernel
*
* Copyright (C) 2008-2009 Ksplice, Inc.
* Author: Tim Abbott <tabbott@ksplice.com>
*/
#include <linux/export.h>
#include <linux/bsearch.h>
#include <linux/kprobes.h>
/*
* bsearch - binary search an array of elements
* @key: pointer to item being searched for
* @base: pointer to first element to search
* @num: number of elements
* @size: size of each element
* @cmp: pointer to comparison function
*
* This function does a binary search on the given array. The
* contents of the array should already be in ascending sorted order
* under the provided comparison function.
*
* Note that the key need not have the same type as the elements in
* the array, e.g. key could be a string and the comparison function
* could compare the string with the struct's name field. However, if
* the key and elements in the array are of the same type, you can use
* the same comparison function for both sort() and bsearch().
*/
void *bsearch(const void *key, const void *base, size_t num, size_t size, cmp_func_t cmp)
{
return __inline_bsearch(key, base, num, size, cmp);
}
EXPORT_SYMBOL(bsearch);
NOKPROBE_SYMBOL(bsearch);
// SPDX-License-Identifier: GPL-2.0-only
/*
* fs/dcache.c
*
* Complete reimplementation
* (C) 1997 Thomas Schoebel-Theuer,
* with heavy changes by Linus Torvalds
*/
/*
* Notes on the allocation strategy:
*
* The dcache is a master of the icache - whenever a dcache entry
* exists, the inode will always exist. "iput()" is done either when
* the dcache entry is deleted or garbage collected.
*/
#include <linux/ratelimit.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/fscrypt.h>
#include <linux/fsnotify.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/cache.h>
#include <linux/export.h>
#include <linux/security.h>
#include <linux/seqlock.h>
#include <linux/memblock.h>
#include <linux/bit_spinlock.h>
#include <linux/rculist_bl.h>
#include <linux/list_lru.h>
#include "internal.h"
#include "mount.h"
/*
* Usage:
* dcache->d_inode->i_lock protects:
* - i_dentry, d_u.d_alias, d_inode of aliases
* dcache_hash_bucket lock protects:
* - the dcache hash table
* s_roots bl list spinlock protects:
* - the s_roots list (see __d_drop)
* dentry->d_sb->s_dentry_lru_lock protects:
* - the dcache lru lists and counters
* d_lock protects:
* - d_flags
* - d_name
* - d_lru
* - d_count
* - d_unhashed()
* - d_parent and d_chilren
* - childrens' d_sib and d_parent
* - d_u.d_alias, d_inode
*
* Ordering:
* dentry->d_inode->i_lock
* dentry->d_lock
* dentry->d_sb->s_dentry_lru_lock
* dcache_hash_bucket lock
* s_roots lock
*
* If there is an ancestor relationship:
* dentry->d_parent->...->d_parent->d_lock
* ...
* dentry->d_parent->d_lock
* dentry->d_lock
*
* If no ancestor relationship:
* arbitrary, since it's serialized on rename_lock
*/
int sysctl_vfs_cache_pressure __read_mostly = 100;
EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
EXPORT_SYMBOL(rename_lock);
static struct kmem_cache *dentry_cache __ro_after_init;
const struct qstr empty_name = QSTR_INIT("", 0);
EXPORT_SYMBOL(empty_name);
const struct qstr slash_name = QSTR_INIT("/", 1);
EXPORT_SYMBOL(slash_name);
const struct qstr dotdot_name = QSTR_INIT("..", 2);
EXPORT_SYMBOL(dotdot_name);
/*
* This is the single most critical data structure when it comes
* to the dcache: the hashtable for lookups. Somebody should try
* to make this good - I've just made it work.
*
* This hash-function tries to avoid losing too many bits of hash
* information, yet avoid using a prime hash-size or similar.
*/
static unsigned int d_hash_shift __ro_after_init;
static struct hlist_bl_head *dentry_hashtable __ro_after_init;
static inline struct hlist_bl_head *d_hash(unsigned int hash)
{
return dentry_hashtable + (hash >> d_hash_shift);
}
#define IN_LOOKUP_SHIFT 10
static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
unsigned int hash)
{
hash += (unsigned long) parent / L1_CACHE_BYTES;
return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
}
struct dentry_stat_t {
long nr_dentry;
long nr_unused;
long age_limit; /* age in seconds */
long want_pages; /* pages requested by system */
long nr_negative; /* # of unused negative dentries */
long dummy; /* Reserved for future use */
};
static DEFINE_PER_CPU(long, nr_dentry);
static DEFINE_PER_CPU(long, nr_dentry_unused);
static DEFINE_PER_CPU(long, nr_dentry_negative);
#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
/* Statistics gathering. */
static struct dentry_stat_t dentry_stat = {
.age_limit = 45,
};
/*
* Here we resort to our own counters instead of using generic per-cpu counters
* for consistency with what the vfs inode code does. We are expected to harvest
* better code and performance by having our own specialized counters.
*
* Please note that the loop is done over all possible CPUs, not over all online
* CPUs. The reason for this is that we don't want to play games with CPUs going
* on and off. If one of them goes off, we will just keep their counters.
*
* glommer: See cffbc8a for details, and if you ever intend to change this,
* please update all vfs counters to match.
*/
static long get_nr_dentry(void)
{
int i;
long sum = 0;
for_each_possible_cpu(i)
sum += per_cpu(nr_dentry, i);
return sum < 0 ? 0 : sum;
}
static long get_nr_dentry_unused(void)
{
int i;
long sum = 0;
for_each_possible_cpu(i)
sum += per_cpu(nr_dentry_unused, i);
return sum < 0 ? 0 : sum;
}
static long get_nr_dentry_negative(void)
{
int i;
long sum = 0;
for_each_possible_cpu(i)
sum += per_cpu(nr_dentry_negative, i);
return sum < 0 ? 0 : sum;
}
static int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
dentry_stat.nr_dentry = get_nr_dentry();
dentry_stat.nr_unused = get_nr_dentry_unused();
dentry_stat.nr_negative = get_nr_dentry_negative();
return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
}
static struct ctl_table fs_dcache_sysctls[] = {
{
.procname = "dentry-state",
.data = &dentry_stat,
.maxlen = 6*sizeof(long),
.mode = 0444,
.proc_handler = proc_nr_dentry,
},
};
static int __init init_fs_dcache_sysctls(void)
{
register_sysctl_init("fs", fs_dcache_sysctls);
return 0;
}
fs_initcall(init_fs_dcache_sysctls);
#endif
/*
* Compare 2 name strings, return 0 if they match, otherwise non-zero.
* The strings are both count bytes long, and count is non-zero.
*/
#ifdef CONFIG_DCACHE_WORD_ACCESS
#include <asm/word-at-a-time.h>
/*
* NOTE! 'cs' and 'scount' come from a dentry, so it has a
* aligned allocation for this particular component. We don't
* strictly need the load_unaligned_zeropad() safety, but it
* doesn't hurt either.
*
* In contrast, 'ct' and 'tcount' can be from a pathname, and do
* need the careful unaligned handling.
*/
static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
{
unsigned long a,b,mask;
for (;;) {
a = read_word_at_a_time(cs);
b = load_unaligned_zeropad(ct);
if (tcount < sizeof(unsigned long))
break;
if (unlikely(a != b))
return 1;
cs += sizeof(unsigned long);
ct += sizeof(unsigned long);
tcount -= sizeof(unsigned long);
if (!tcount)
return 0;
}
mask = bytemask_from_count(tcount);
return unlikely(!!((a ^ b) & mask));
}
#else
static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
{
do {
if (*cs != *ct)
return 1;
cs++;
ct++;
tcount--;
} while (tcount);
return 0;
}
#endif
static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
{
/*
* Be careful about RCU walk racing with rename:
* use 'READ_ONCE' to fetch the name pointer.
*
* NOTE! Even if a rename will mean that the length
* was not loaded atomically, we don't care. The
* RCU walk will check the sequence count eventually,
* and catch it. And we won't overrun the buffer,
* because we're reading the name pointer atomically,
* and a dentry name is guaranteed to be properly
* terminated with a NUL byte.
*
* End result: even if 'len' is wrong, we'll exit
* early because the data cannot match (there can
* be no NUL in the ct/tcount data)
*/
const unsigned char *cs = READ_ONCE(dentry->d_name.name);
return dentry_string_cmp(cs, ct, tcount);
}
struct external_name {
union {
atomic_t count;
struct rcu_head head;
} u;
unsigned char name[];
};
static inline struct external_name *external_name(struct dentry *dentry)
{
return container_of(dentry->d_name.name, struct external_name, name[0]);
}
static void __d_free(struct rcu_head *head)
{
struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
kmem_cache_free(dentry_cache, dentry);
}
static void __d_free_external(struct rcu_head *head)
{
struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
kfree(external_name(dentry));
kmem_cache_free(dentry_cache, dentry);
}
static inline int dname_external(const struct dentry *dentry)
{
return dentry->d_name.name != dentry->d_iname;
}
void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
{
spin_lock(&dentry->d_lock);
name->name = dentry->d_name;
if (unlikely(dname_external(dentry))) {
atomic_inc(&external_name(dentry)->u.count);
} else {
memcpy(name->inline_name, dentry->d_iname,
dentry->d_name.len + 1);
name->name.name = name->inline_name;
}
spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(take_dentry_name_snapshot);
void release_dentry_name_snapshot(struct name_snapshot *name)
{
if (unlikely(name->name.name != name->inline_name)) {
struct external_name *p;
p = container_of(name->name.name, struct external_name, name[0]);
if (unlikely(atomic_dec_and_test(&p->u.count)))
kfree_rcu(p, u.head);
}
}
EXPORT_SYMBOL(release_dentry_name_snapshot);
static inline void __d_set_inode_and_type(struct dentry *dentry,
struct inode *inode,
unsigned type_flags)
{
unsigned flags;
dentry->d_inode = inode;
flags = READ_ONCE(dentry->d_flags);
flags &= ~DCACHE_ENTRY_TYPE;
flags |= type_flags;
smp_store_release(&dentry->d_flags, flags);
}
static inline void __d_clear_type_and_inode(struct dentry *dentry)
{
unsigned flags = READ_ONCE(dentry->d_flags);
flags &= ~DCACHE_ENTRY_TYPE;
WRITE_ONCE(dentry->d_flags, flags);
dentry->d_inode = NULL;
if (dentry->d_flags & DCACHE_LRU_LIST)
this_cpu_inc(nr_dentry_negative);
}
static void dentry_free(struct dentry *dentry)
{
WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); if (unlikely(dname_external(dentry))) {
struct external_name *p = external_name(dentry);
if (likely(atomic_dec_and_test(&p->u.count))) { call_rcu(&dentry->d_u.d_rcu, __d_free_external);
return;
}
}
/* if dentry was never visible to RCU, immediate free is OK */
if (dentry->d_flags & DCACHE_NORCU) __d_free(&dentry->d_u.d_rcu);
else
call_rcu(&dentry->d_u.d_rcu, __d_free);
}
/*
* Release the dentry's inode, using the filesystem
* d_iput() operation if defined.
*/
static void dentry_unlink_inode(struct dentry * dentry)
__releases(dentry->d_lock)
__releases(dentry->d_inode->i_lock)
{
struct inode *inode = dentry->d_inode;
raw_write_seqcount_begin(&dentry->d_seq);
__d_clear_type_and_inode(dentry); hlist_del_init(&dentry->d_u.d_alias); raw_write_seqcount_end(&dentry->d_seq);
spin_unlock(&dentry->d_lock);
spin_unlock(&inode->i_lock);
if (!inode->i_nlink)
fsnotify_inoderemove(inode); if (dentry->d_op && dentry->d_op->d_iput) dentry->d_op->d_iput(dentry, inode);
else
iput(inode);
}
/*
* The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
* is in use - which includes both the "real" per-superblock
* LRU list _and_ the DCACHE_SHRINK_LIST use.
*
* The DCACHE_SHRINK_LIST bit is set whenever the dentry is
* on the shrink list (ie not on the superblock LRU list).
*
* The per-cpu "nr_dentry_unused" counters are updated with
* the DCACHE_LRU_LIST bit.
*
* The per-cpu "nr_dentry_negative" counters are only updated
* when deleted from or added to the per-superblock LRU list, not
* from/to the shrink list. That is to avoid an unneeded dec/inc
* pair when moving from LRU to shrink list in select_collect().
*
* These helper functions make sure we always follow the
* rules. d_lock must be held by the caller.
*/
#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
static void d_lru_add(struct dentry *dentry)
{
D_FLAG_VERIFY(dentry, 0); dentry->d_flags |= DCACHE_LRU_LIST;
this_cpu_inc(nr_dentry_unused);
if (d_is_negative(dentry))
this_cpu_inc(nr_dentry_negative); WARN_ON_ONCE(!list_lru_add_obj(
&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
}
static void d_lru_del(struct dentry *dentry)
{
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); dentry->d_flags &= ~DCACHE_LRU_LIST;
this_cpu_dec(nr_dentry_unused);
if (d_is_negative(dentry))
this_cpu_dec(nr_dentry_negative); WARN_ON_ONCE(!list_lru_del_obj(
&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
}
static void d_shrink_del(struct dentry *dentry)
{
D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
list_del_init(&dentry->d_lru);
dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
this_cpu_dec(nr_dentry_unused);
}
static void d_shrink_add(struct dentry *dentry, struct list_head *list)
{
D_FLAG_VERIFY(dentry, 0);
list_add(&dentry->d_lru, list);
dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
this_cpu_inc(nr_dentry_unused);
}
/*
* These can only be called under the global LRU lock, ie during the
* callback for freeing the LRU list. "isolate" removes it from the
* LRU lists entirely, while shrink_move moves it to the indicated
* private list.
*/
static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
{
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
dentry->d_flags &= ~DCACHE_LRU_LIST;
this_cpu_dec(nr_dentry_unused);
if (d_is_negative(dentry))
this_cpu_dec(nr_dentry_negative);
list_lru_isolate(lru, &dentry->d_lru);
}
static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
struct list_head *list)
{
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
dentry->d_flags |= DCACHE_SHRINK_LIST;
if (d_is_negative(dentry))
this_cpu_dec(nr_dentry_negative);
list_lru_isolate_move(lru, &dentry->d_lru, list);
}
static void ___d_drop(struct dentry *dentry)
{
struct hlist_bl_head *b;
/*
* Hashed dentries are normally on the dentry hashtable,
* with the exception of those newly allocated by
* d_obtain_root, which are always IS_ROOT:
*/
if (unlikely(IS_ROOT(dentry))) b = &dentry->d_sb->s_roots;
else
b = d_hash(dentry->d_name.hash); hlist_bl_lock(b); __hlist_bl_del(&dentry->d_hash); hlist_bl_unlock(b);
}
void __d_drop(struct dentry *dentry)
{
if (!d_unhashed(dentry)) { ___d_drop(dentry);
dentry->d_hash.pprev = NULL;
write_seqcount_invalidate(&dentry->d_seq);
}
}
EXPORT_SYMBOL(__d_drop);
/**
* d_drop - drop a dentry
* @dentry: dentry to drop
*
* d_drop() unhashes the entry from the parent dentry hashes, so that it won't
* be found through a VFS lookup any more. Note that this is different from
* deleting the dentry - d_delete will try to mark the dentry negative if
* possible, giving a successful _negative_ lookup, while d_drop will
* just make the cache lookup fail.
*
* d_drop() is used mainly for stuff that wants to invalidate a dentry for some
* reason (NFS timeouts or autofs deletes).
*
* __d_drop requires dentry->d_lock
*
* ___d_drop doesn't mark dentry as "unhashed"
* (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
*/
void d_drop(struct dentry *dentry)
{
spin_lock(&dentry->d_lock);
__d_drop(dentry);
spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(d_drop);
static inline void dentry_unlist(struct dentry *dentry)
{
struct dentry *next;
/*
* Inform d_walk() and shrink_dentry_list() that we are no longer
* attached to the dentry tree
*/
dentry->d_flags |= DCACHE_DENTRY_KILLED;
if (unlikely(hlist_unhashed(&dentry->d_sib)))
return;
__hlist_del(&dentry->d_sib);
/*
* Cursors can move around the list of children. While we'd been
* a normal list member, it didn't matter - ->d_sib.next would've
* been updated. However, from now on it won't be and for the
* things like d_walk() it might end up with a nasty surprise.
* Normally d_walk() doesn't care about cursors moving around -
* ->d_lock on parent prevents that and since a cursor has no children
* of its own, we get through it without ever unlocking the parent.
* There is one exception, though - if we ascend from a child that
* gets killed as soon as we unlock it, the next sibling is found
* using the value left in its ->d_sib.next. And if _that_
* pointed to a cursor, and cursor got moved (e.g. by lseek())
* before d_walk() regains parent->d_lock, we'll end up skipping
* everything the cursor had been moved past.
*
* Solution: make sure that the pointer left behind in ->d_sib.next
* points to something that won't be moving around. I.e. skip the
* cursors.
*/
while (dentry->d_sib.next) {
next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib);
if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
break;
dentry->d_sib.next = next->d_sib.next;
}
}
static struct dentry *__dentry_kill(struct dentry *dentry)
{
struct dentry *parent = NULL;
bool can_free = true;
/*
* The dentry is now unrecoverably dead to the world.
*/
lockref_mark_dead(&dentry->d_lockref);
/*
* inform the fs via d_prune that this dentry is about to be
* unhashed and destroyed.
*/
if (dentry->d_flags & DCACHE_OP_PRUNE)
dentry->d_op->d_prune(dentry); if (dentry->d_flags & DCACHE_LRU_LIST) { if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) d_lru_del(dentry);
}
/* if it was on the hash then remove it */
__d_drop(dentry); if (dentry->d_inode) dentry_unlink_inode(dentry);
else
spin_unlock(&dentry->d_lock); this_cpu_dec(nr_dentry); if (dentry->d_op && dentry->d_op->d_release) dentry->d_op->d_release(dentry); cond_resched();
/* now that it's negative, ->d_parent is stable */
if (!IS_ROOT(dentry)) {
parent = dentry->d_parent;
spin_lock(&parent->d_lock);
}
spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); dentry_unlist(dentry); if (dentry->d_flags & DCACHE_SHRINK_LIST)
can_free = false;
spin_unlock(&dentry->d_lock);
if (likely(can_free))
dentry_free(dentry);
if (parent && --parent->d_lockref.count) { spin_unlock(&parent->d_lock);
return NULL;
}
return parent;
}
/*
* Lock a dentry for feeding it to __dentry_kill().
* Called under rcu_read_lock() and dentry->d_lock; the former
* guarantees that nothing we access will be freed under us.
* Note that dentry is *not* protected from concurrent dentry_kill(),
* d_delete(), etc.
*
* Return false if dentry is busy. Otherwise, return true and have
* that dentry's inode locked.
*/
static bool lock_for_kill(struct dentry *dentry)
{
struct inode *inode = dentry->d_inode;
if (unlikely(dentry->d_lockref.count))
return false;
if (!inode || likely(spin_trylock(&inode->i_lock))) return true;
do {
spin_unlock(&dentry->d_lock); spin_lock(&inode->i_lock);
spin_lock(&dentry->d_lock);
if (likely(inode == dentry->d_inode))
break;
spin_unlock(&inode->i_lock);
inode = dentry->d_inode;
} while (inode);
if (likely(!dentry->d_lockref.count))
return true;
if (inode) spin_unlock(&inode->i_lock);
return false;
}
/*
* Decide if dentry is worth retaining. Usually this is called with dentry
* locked; if not locked, we are more limited and might not be able to tell
* without a lock. False in this case means "punt to locked path and recheck".
*
* In case we aren't locked, these predicates are not "stable". However, it is
* sufficient that at some point after we dropped the reference the dentry was
* hashed and the flags had the proper value. Other dentry users may have
* re-gotten a reference to the dentry and change that, but our work is done -
* we can leave the dentry around with a zero refcount.
*/
static inline bool retain_dentry(struct dentry *dentry, bool locked)
{
unsigned int d_flags;
smp_rmb();
d_flags = READ_ONCE(dentry->d_flags);
// Unreachable? Nobody would be able to look it up, no point retaining
if (unlikely(d_unhashed(dentry)))
return false;
// Same if it's disconnected
if (unlikely(d_flags & DCACHE_DISCONNECTED))
return false;
// ->d_delete() might tell us not to bother, but that requires
// ->d_lock; can't decide without it
if (unlikely(d_flags & DCACHE_OP_DELETE)) { if (!locked || dentry->d_op->d_delete(dentry))
return false;
}
// Explicitly told not to bother
if (unlikely(d_flags & DCACHE_DONTCACHE))
return false;
// At this point it looks like we ought to keep it. We also might
// need to do something - put it on LRU if it wasn't there already
// and mark it referenced if it was on LRU, but not marked yet.
// Unfortunately, both actions require ->d_lock, so in lockless
// case we'd have to punt rather than doing those.
if (unlikely(!(d_flags & DCACHE_LRU_LIST))) {
if (!locked)
return false;
d_lru_add(dentry); } else if (unlikely(!(d_flags & DCACHE_REFERENCED))) {
if (!locked)
return false;
dentry->d_flags |= DCACHE_REFERENCED;
}
return true;
}
void d_mark_dontcache(struct inode *inode)
{
struct dentry *de;
spin_lock(&inode->i_lock);
hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
spin_lock(&de->d_lock);
de->d_flags |= DCACHE_DONTCACHE;
spin_unlock(&de->d_lock);
}
inode->i_state |= I_DONTCACHE;
spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL(d_mark_dontcache);
/*
* Try to do a lockless dput(), and return whether that was successful.
*
* If unsuccessful, we return false, having already taken the dentry lock.
* In that case refcount is guaranteed to be zero and we have already
* decided that it's not worth keeping around.
*
* The caller needs to hold the RCU read lock, so that the dentry is
* guaranteed to stay around even if the refcount goes down to zero!
*/
static inline bool fast_dput(struct dentry *dentry)
{
int ret;
/*
* try to decrement the lockref optimistically.
*/
ret = lockref_put_return(&dentry->d_lockref);
/*
* If the lockref_put_return() failed due to the lock being held
* by somebody else, the fast path has failed. We will need to
* get the lock, and then check the count again.
*/
if (unlikely(ret < 0)) {
spin_lock(&dentry->d_lock); if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) {
spin_unlock(&dentry->d_lock);
return true;
}
dentry->d_lockref.count--;
goto locked;
}
/*
* If we weren't the last ref, we're done.
*/
if (ret)
return true;
/*
* Can we decide that decrement of refcount is all we needed without
* taking the lock? There's a very common case when it's all we need -
* dentry looks like it ought to be retained and there's nothing else
* to do.
*/
if (retain_dentry(dentry, false))
return true;
/*
* Either not worth retaining or we can't tell without the lock.
* Get the lock, then. We've already decremented the refcount to 0,
* but we'll need to re-check the situation after getting the lock.
*/
spin_lock(&dentry->d_lock);
/*
* Did somebody else grab a reference to it in the meantime, and
* we're no longer the last user after all? Alternatively, somebody
* else could have killed it and marked it dead. Either way, we
* don't need to do anything else.
*/
locked:
if (dentry->d_lockref.count || retain_dentry(dentry, true)) { spin_unlock(&dentry->d_lock);
return true;
}
return false;
}
/*
* This is dput
*
* This is complicated by the fact that we do not want to put
* dentries that are no longer on any hash chain on the unused
* list: we'd much rather just get rid of them immediately.
*
* However, that implies that we have to traverse the dentry
* tree upwards to the parents which might _also_ now be
* scheduled for deletion (it may have been only waiting for
* its last child to go away).
*
* This tail recursion is done by hand as we don't want to depend
* on the compiler to always get this right (gcc generally doesn't).
* Real recursion would eat up our stack space.
*/
/*
* dput - release a dentry
* @dentry: dentry to release
*
* Release a dentry. This will drop the usage count and if appropriate
* call the dentry unlink method as well as removing it from the queues and
* releasing its resources. If the parent dentries were scheduled for release
* they too may now get deleted.
*/
void dput(struct dentry *dentry)
{
if (!dentry)
return;
might_sleep(); rcu_read_lock(); if (likely(fast_dput(dentry))) { rcu_read_unlock();
return;
}
while (lock_for_kill(dentry)) { rcu_read_unlock();
dentry = __dentry_kill(dentry);
if (!dentry)
return;
if (retain_dentry(dentry, true)) { spin_unlock(&dentry->d_lock);
return;
}
rcu_read_lock();
}
rcu_read_unlock();
spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(dput);
static void to_shrink_list(struct dentry *dentry, struct list_head *list)
__must_hold(&dentry->d_lock)
{
if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
if (dentry->d_flags & DCACHE_LRU_LIST)
d_lru_del(dentry);
d_shrink_add(dentry, list);
}
}
void dput_to_list(struct dentry *dentry, struct list_head *list)
{
rcu_read_lock();
if (likely(fast_dput(dentry))) {
rcu_read_unlock();
return;
}
rcu_read_unlock();
to_shrink_list(dentry, list);
spin_unlock(&dentry->d_lock);
}
struct dentry *dget_parent(struct dentry *dentry)
{
int gotref;
struct dentry *ret;
unsigned seq;
/*
* Do optimistic parent lookup without any
* locking.
*/
rcu_read_lock(); seq = raw_seqcount_begin(&dentry->d_seq);
ret = READ_ONCE(dentry->d_parent);
gotref = lockref_get_not_zero(&ret->d_lockref);
rcu_read_unlock();
if (likely(gotref)) {
if (!read_seqcount_retry(&dentry->d_seq, seq))
return ret;
dput(ret);
}
repeat:
/*
* Don't need rcu_dereference because we re-check it was correct under
* the lock.
*/
rcu_read_lock(); ret = dentry->d_parent;
spin_lock(&ret->d_lock);
if (unlikely(ret != dentry->d_parent)) {
spin_unlock(&ret->d_lock); rcu_read_unlock();
goto repeat;
}
rcu_read_unlock(); BUG_ON(!ret->d_lockref.count); ret->d_lockref.count++;
spin_unlock(&ret->d_lock);
return ret;
}
EXPORT_SYMBOL(dget_parent);
static struct dentry * __d_find_any_alias(struct inode *inode)
{
struct dentry *alias;
if (hlist_empty(&inode->i_dentry))
return NULL;
alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
lockref_get(&alias->d_lockref);
return alias;
}
/**
* d_find_any_alias - find any alias for a given inode
* @inode: inode to find an alias for
*
* If any aliases exist for the given inode, take and return a
* reference for one of them. If no aliases exist, return %NULL.
*/
struct dentry *d_find_any_alias(struct inode *inode)
{
struct dentry *de;
spin_lock(&inode->i_lock); de = __d_find_any_alias(inode); spin_unlock(&inode->i_lock);
return de;
}
EXPORT_SYMBOL(d_find_any_alias);
static struct dentry *__d_find_alias(struct inode *inode)
{
struct dentry *alias;
if (S_ISDIR(inode->i_mode))
return __d_find_any_alias(inode); hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) { spin_lock(&alias->d_lock);
if (!d_unhashed(alias)) {
dget_dlock(alias);
spin_unlock(&alias->d_lock);
return alias;
}
spin_unlock(&alias->d_lock);
}
return NULL;
}
/**
* d_find_alias - grab a hashed alias of inode
* @inode: inode in question
*
* If inode has a hashed alias, or is a directory and has any alias,
* acquire the reference to alias and return it. Otherwise return NULL.
* Notice that if inode is a directory there can be only one alias and
* it can be unhashed only if it has no children, or if it is the root
* of a filesystem, or if the directory was renamed and d_revalidate
* was the first vfs operation to notice.
*
* If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
* any other hashed alias over that one.
*/
struct dentry *d_find_alias(struct inode *inode)
{
struct dentry *de = NULL;
if (!hlist_empty(&inode->i_dentry)) { spin_lock(&inode->i_lock); de = __d_find_alias(inode); spin_unlock(&inode->i_lock);
}
return de;
}
EXPORT_SYMBOL(d_find_alias);
/*
* Caller MUST be holding rcu_read_lock() and be guaranteed
* that inode won't get freed until rcu_read_unlock().
*/
struct dentry *d_find_alias_rcu(struct inode *inode)
{
struct hlist_head *l = &inode->i_dentry;
struct dentry *de = NULL;
spin_lock(&inode->i_lock);
// ->i_dentry and ->i_rcu are colocated, but the latter won't be
// used without having I_FREEING set, which means no aliases left
if (likely(!(inode->i_state & I_FREEING) && !hlist_empty(l))) {
if (S_ISDIR(inode->i_mode)) {
de = hlist_entry(l->first, struct dentry, d_u.d_alias);
} else {
hlist_for_each_entry(de, l, d_u.d_alias)
if (!d_unhashed(de))
break;
}
}
spin_unlock(&inode->i_lock);
return de;
}
/*
* Try to kill dentries associated with this inode.
* WARNING: you must own a reference to inode.
*/
void d_prune_aliases(struct inode *inode)
{
LIST_HEAD(dispose);
struct dentry *dentry;
spin_lock(&inode->i_lock);
hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
spin_lock(&dentry->d_lock);
if (!dentry->d_lockref.count)
to_shrink_list(dentry, &dispose);
spin_unlock(&dentry->d_lock);
}
spin_unlock(&inode->i_lock);
shrink_dentry_list(&dispose);
}
EXPORT_SYMBOL(d_prune_aliases);
static inline void shrink_kill(struct dentry *victim)
{
do {
rcu_read_unlock();
victim = __dentry_kill(victim);
rcu_read_lock(); } while (victim && lock_for_kill(victim)); rcu_read_unlock();
if (victim)
spin_unlock(&victim->d_lock);
}
void shrink_dentry_list(struct list_head *list)
{
while (!list_empty(list)) {
struct dentry *dentry;
dentry = list_entry(list->prev, struct dentry, d_lru);
spin_lock(&dentry->d_lock);
rcu_read_lock();
if (!lock_for_kill(dentry)) {
bool can_free;
rcu_read_unlock();
d_shrink_del(dentry);
can_free = dentry->d_flags & DCACHE_DENTRY_KILLED;
spin_unlock(&dentry->d_lock);
if (can_free)
dentry_free(dentry);
continue;
}
d_shrink_del(dentry);
shrink_kill(dentry);
}
}
static enum lru_status dentry_lru_isolate(struct list_head *item,
struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
{
struct list_head *freeable = arg;
struct dentry *dentry = container_of(item, struct dentry, d_lru);
/*
* we are inverting the lru lock/dentry->d_lock here,
* so use a trylock. If we fail to get the lock, just skip
* it
*/
if (!spin_trylock(&dentry->d_lock))
return LRU_SKIP;
/*
* Referenced dentries are still in use. If they have active
* counts, just remove them from the LRU. Otherwise give them
* another pass through the LRU.
*/
if (dentry->d_lockref.count) {
d_lru_isolate(lru, dentry);
spin_unlock(&dentry->d_lock);
return LRU_REMOVED;
}
if (dentry->d_flags & DCACHE_REFERENCED) {
dentry->d_flags &= ~DCACHE_REFERENCED;
spin_unlock(&dentry->d_lock);
/*
* The list move itself will be made by the common LRU code. At
* this point, we've dropped the dentry->d_lock but keep the
* lru lock. This is safe to do, since every list movement is
* protected by the lru lock even if both locks are held.
*
* This is guaranteed by the fact that all LRU management
* functions are intermediated by the LRU API calls like
* list_lru_add_obj and list_lru_del_obj. List movement in this file
* only ever occur through this functions or through callbacks
* like this one, that are called from the LRU API.
*
* The only exceptions to this are functions like
* shrink_dentry_list, and code that first checks for the
* DCACHE_SHRINK_LIST flag. Those are guaranteed to be
* operating only with stack provided lists after they are
* properly isolated from the main list. It is thus, always a
* local access.
*/
return LRU_ROTATE;
}
d_lru_shrink_move(lru, dentry, freeable);
spin_unlock(&dentry->d_lock);
return LRU_REMOVED;
}
/**
* prune_dcache_sb - shrink the dcache
* @sb: superblock
* @sc: shrink control, passed to list_lru_shrink_walk()
*
* Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
* is done when we need more memory and called from the superblock shrinker
* function.
*
* This function may fail to free any resources if all the dentries are in
* use.
*/
long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
{
LIST_HEAD(dispose);
long freed;
freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
dentry_lru_isolate, &dispose);
shrink_dentry_list(&dispose);
return freed;
}
static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
{
struct list_head *freeable = arg;
struct dentry *dentry = container_of(item, struct dentry, d_lru);
/*
* we are inverting the lru lock/dentry->d_lock here,
* so use a trylock. If we fail to get the lock, just skip
* it
*/
if (!spin_trylock(&dentry->d_lock))
return LRU_SKIP;
d_lru_shrink_move(lru, dentry, freeable);
spin_unlock(&dentry->d_lock);
return LRU_REMOVED;
}
/**
* shrink_dcache_sb - shrink dcache for a superblock
* @sb: superblock
*
* Shrink the dcache for the specified super block. This is used to free
* the dcache before unmounting a file system.
*/
void shrink_dcache_sb(struct super_block *sb)
{
do {
LIST_HEAD(dispose);
list_lru_walk(&sb->s_dentry_lru,
dentry_lru_isolate_shrink, &dispose, 1024);
shrink_dentry_list(&dispose);
} while (list_lru_count(&sb->s_dentry_lru) > 0);
}
EXPORT_SYMBOL(shrink_dcache_sb);
/**
* enum d_walk_ret - action to talke during tree walk
* @D_WALK_CONTINUE: contrinue walk
* @D_WALK_QUIT: quit walk
* @D_WALK_NORETRY: quit when retry is needed
* @D_WALK_SKIP: skip this dentry and its children
*/
enum d_walk_ret {
D_WALK_CONTINUE,
D_WALK_QUIT,
D_WALK_NORETRY,
D_WALK_SKIP,
};
/**
* d_walk - walk the dentry tree
* @parent: start of walk
* @data: data passed to @enter() and @finish()
* @enter: callback when first entering the dentry
*
* The @enter() callbacks are called with d_lock held.
*/
static void d_walk(struct dentry *parent, void *data,
enum d_walk_ret (*enter)(void *, struct dentry *))
{
struct dentry *this_parent, *dentry;
unsigned seq = 0;
enum d_walk_ret ret;
bool retry = true;
again:
read_seqbegin_or_lock(&rename_lock, &seq);
this_parent = parent;
spin_lock(&this_parent->d_lock);
ret = enter(data, this_parent);
switch (ret) {
case D_WALK_CONTINUE:
break;
case D_WALK_QUIT:
case D_WALK_SKIP:
goto out_unlock;
case D_WALK_NORETRY:
retry = false;
break;
}
repeat:
dentry = d_first_child(this_parent);
resume:
hlist_for_each_entry_from(dentry, d_sib) { if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
continue;
spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
ret = enter(data, dentry);
switch (ret) {
case D_WALK_CONTINUE:
break;
case D_WALK_QUIT:
spin_unlock(&dentry->d_lock);
goto out_unlock;
case D_WALK_NORETRY:
retry = false;
break;
case D_WALK_SKIP:
spin_unlock(&dentry->d_lock);
continue;
}
if (!hlist_empty(&dentry->d_children)) { spin_unlock(&this_parent->d_lock);
spin_release(&dentry->d_lock.dep_map, _RET_IP_);
this_parent = dentry;
spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
goto repeat;
}
spin_unlock(&dentry->d_lock);
}
/*
* All done at this level ... ascend and resume the search.
*/
rcu_read_lock();
ascend:
if (this_parent != parent) {
dentry = this_parent;
this_parent = dentry->d_parent;
spin_unlock(&dentry->d_lock);
spin_lock(&this_parent->d_lock);
/* might go back up the wrong parent if we have had a rename. */
if (need_seqretry(&rename_lock, seq))
goto rename_retry;
/* go into the first sibling still alive */
hlist_for_each_entry_continue(dentry, d_sib) { if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) { rcu_read_unlock();
goto resume;
}
}
goto ascend;
}
if (need_seqretry(&rename_lock, seq))
goto rename_retry;
rcu_read_unlock();
out_unlock:
spin_unlock(&this_parent->d_lock); done_seqretry(&rename_lock, seq);
return;
rename_retry:
spin_unlock(&this_parent->d_lock); rcu_read_unlock(); BUG_ON(seq & 1); if (!retry)
return;
seq = 1;
goto again;
}
struct check_mount {
struct vfsmount *mnt;
unsigned int mounted;
};
static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
{
struct check_mount *info = data;
struct path path = { .mnt = info->mnt, .dentry = dentry };
if (likely(!d_mountpoint(dentry)))
return D_WALK_CONTINUE;
if (__path_is_mountpoint(&path)) {
info->mounted = 1;
return D_WALK_QUIT;
}
return D_WALK_CONTINUE;
}
/**
* path_has_submounts - check for mounts over a dentry in the
* current namespace.
* @parent: path to check.
*
* Return true if the parent or its subdirectories contain
* a mount point in the current namespace.
*/
int path_has_submounts(const struct path *parent)
{
struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
read_seqlock_excl(&mount_lock);
d_walk(parent->dentry, &data, path_check_mount);
read_sequnlock_excl(&mount_lock);
return data.mounted;
}
EXPORT_SYMBOL(path_has_submounts);
/*
* Called by mount code to set a mountpoint and check if the mountpoint is
* reachable (e.g. NFS can unhash a directory dentry and then the complete
* subtree can become unreachable).
*
* Only one of d_invalidate() and d_set_mounted() must succeed. For
* this reason take rename_lock and d_lock on dentry and ancestors.
*/
int d_set_mounted(struct dentry *dentry)
{
struct dentry *p;
int ret = -ENOENT;
write_seqlock(&rename_lock);
for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
/* Need exclusion wrt. d_invalidate() */
spin_lock(&p->d_lock);
if (unlikely(d_unhashed(p))) {
spin_unlock(&p->d_lock);
goto out;
}
spin_unlock(&p->d_lock);
}
spin_lock(&dentry->d_lock);
if (!d_unlinked(dentry)) {
ret = -EBUSY;
if (!d_mountpoint(dentry)) {
dentry->d_flags |= DCACHE_MOUNTED;
ret = 0;
}
}
spin_unlock(&dentry->d_lock);
out:
write_sequnlock(&rename_lock);
return ret;
}
/*
* Search the dentry child list of the specified parent,
* and move any unused dentries to the end of the unused
* list for prune_dcache(). We descend to the next level
* whenever the d_children list is non-empty and continue
* searching.
*
* It returns zero iff there are no unused children,
* otherwise it returns the number of children moved to
* the end of the unused list. This may not be the total
* number of unused children, because select_parent can
* drop the lock and return early due to latency
* constraints.
*/
struct select_data {
struct dentry *start;
union {
long found;
struct dentry *victim;
};
struct list_head dispose;
};
static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
{
struct select_data *data = _data;
enum d_walk_ret ret = D_WALK_CONTINUE;
if (data->start == dentry)
goto out;
if (dentry->d_flags & DCACHE_SHRINK_LIST) { data->found++; } else if (!dentry->d_lockref.count) { to_shrink_list(dentry, &data->dispose);
data->found++;
} else if (dentry->d_lockref.count < 0) {
data->found++;
}
/*
* We can return to the caller if we have found some (this
* ensures forward progress). We'll be coming back to find
* the rest.
*/
if (!list_empty(&data->dispose)) ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
out:
return ret;
}
static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
{
struct select_data *data = _data;
enum d_walk_ret ret = D_WALK_CONTINUE;
if (data->start == dentry)
goto out;
if (!dentry->d_lockref.count) {
if (dentry->d_flags & DCACHE_SHRINK_LIST) {
rcu_read_lock();
data->victim = dentry;
return D_WALK_QUIT;
}
to_shrink_list(dentry, &data->dispose);
}
/*
* We can return to the caller if we have found some (this
* ensures forward progress). We'll be coming back to find
* the rest.
*/
if (!list_empty(&data->dispose))
ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
out:
return ret;
}
/**
* shrink_dcache_parent - prune dcache
* @parent: parent of entries to prune
*
* Prune the dcache to remove unused children of the parent dentry.
*/
void shrink_dcache_parent(struct dentry *parent)
{
for (;;) {
struct select_data data = {.start = parent};
INIT_LIST_HEAD(&data.dispose);
d_walk(parent, &data, select_collect);
if (!list_empty(&data.dispose)) {
shrink_dentry_list(&data.dispose);
continue;
}
cond_resched();
if (!data.found)
break;
data.victim = NULL;
d_walk(parent, &data, select_collect2);
if (data.victim) {
spin_lock(&data.victim->d_lock);
if (!lock_for_kill(data.victim)) {
spin_unlock(&data.victim->d_lock); rcu_read_unlock();
} else {
shrink_kill(data.victim);
}
}
if (!list_empty(&data.dispose)) shrink_dentry_list(&data.dispose);
}
}
EXPORT_SYMBOL(shrink_dcache_parent);
static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
{
/* it has busy descendents; complain about those instead */
if (!hlist_empty(&dentry->d_children))
return D_WALK_CONTINUE;
/* root with refcount 1 is fine */
if (dentry == _data && dentry->d_lockref.count == 1)
return D_WALK_CONTINUE;
WARN(1, "BUG: Dentry %p{i=%lx,n=%pd} "
" still in use (%d) [unmount of %s %s]\n",
dentry,
dentry->d_inode ?
dentry->d_inode->i_ino : 0UL,
dentry,
dentry->d_lockref.count,
dentry->d_sb->s_type->name,
dentry->d_sb->s_id);
return D_WALK_CONTINUE;
}
static void do_one_tree(struct dentry *dentry)
{
shrink_dcache_parent(dentry);
d_walk(dentry, dentry, umount_check);
d_drop(dentry);
dput(dentry);
}
/*
* destroy the dentries attached to a superblock on unmounting
*/
void shrink_dcache_for_umount(struct super_block *sb)
{
struct dentry *dentry;
WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
dentry = sb->s_root;
sb->s_root = NULL;
do_one_tree(dentry);
while (!hlist_bl_empty(&sb->s_roots)) {
dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
do_one_tree(dentry);
}
}
static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
{
struct dentry **victim = _data;
if (d_mountpoint(dentry)) { *victim = dget_dlock(dentry); return D_WALK_QUIT;
}
return D_WALK_CONTINUE;
}
/**
* d_invalidate - detach submounts, prune dcache, and drop
* @dentry: dentry to invalidate (aka detach, prune and drop)
*/
void d_invalidate(struct dentry *dentry)
{
bool had_submounts = false;
spin_lock(&dentry->d_lock);
if (d_unhashed(dentry)) {
spin_unlock(&dentry->d_lock);
return;
}
__d_drop(dentry);
spin_unlock(&dentry->d_lock);
/* Negative dentries can be dropped without further checks */
if (!dentry->d_inode)
return;
shrink_dcache_parent(dentry);
for (;;) {
struct dentry *victim = NULL;
d_walk(dentry, &victim, find_submount);
if (!victim) {
if (had_submounts) shrink_dcache_parent(dentry); return;
}
had_submounts = true;
detach_mounts(victim); dput(victim);
}
}
EXPORT_SYMBOL(d_invalidate);
/**
* __d_alloc - allocate a dcache entry
* @sb: filesystem it will belong to
* @name: qstr of the name
*
* Allocates a dentry. It returns %NULL if there is insufficient memory
* available. On a success the dentry is returned. The name passed in is
* copied and the copy passed in may be reused after this call.
*/
static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
{
struct dentry *dentry;
char *dname;
int err;
dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru,
GFP_KERNEL);
if (!dentry)
return NULL;
/*
* We guarantee that the inline name is always NUL-terminated.
* This way the memcpy() done by the name switching in rename
* will still always have a NUL at the end, even if we might
* be overwriting an internal NUL character
*/
dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
if (unlikely(!name)) {
name = &slash_name;
dname = dentry->d_iname; } else if (name->len > DNAME_INLINE_LEN-1) {
size_t size = offsetof(struct external_name, name[1]);
struct external_name *p = kmalloc(size + name->len,
GFP_KERNEL_ACCOUNT |
__GFP_RECLAIMABLE);
if (!p) {
kmem_cache_free(dentry_cache, dentry);
return NULL;
}
atomic_set(&p->u.count, 1);
dname = p->name;
} else {
dname = dentry->d_iname;
}
dentry->d_name.len = name->len;
dentry->d_name.hash = name->hash;
memcpy(dname, name->name, name->len);
dname[name->len] = 0;
/* Make sure we always see the terminating NUL character */
smp_store_release(&dentry->d_name.name, dname); /* ^^^ */
dentry->d_lockref.count = 1;
dentry->d_flags = 0;
spin_lock_init(&dentry->d_lock);
seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
dentry->d_inode = NULL;
dentry->d_parent = dentry;
dentry->d_sb = sb;
dentry->d_op = NULL;
dentry->d_fsdata = NULL;
INIT_HLIST_BL_NODE(&dentry->d_hash);
INIT_LIST_HEAD(&dentry->d_lru);
INIT_HLIST_HEAD(&dentry->d_children);
INIT_HLIST_NODE(&dentry->d_u.d_alias);
INIT_HLIST_NODE(&dentry->d_sib);
d_set_d_op(dentry, dentry->d_sb->s_d_op);
if (dentry->d_op && dentry->d_op->d_init) { err = dentry->d_op->d_init(dentry);
if (err) {
if (dname_external(dentry)) kfree(external_name(dentry));
kmem_cache_free(dentry_cache, dentry);
return NULL;
}
}
this_cpu_inc(nr_dentry); return dentry;
}
/**
* d_alloc - allocate a dcache entry
* @parent: parent of entry to allocate
* @name: qstr of the name
*
* Allocates a dentry. It returns %NULL if there is insufficient memory
* available. On a success the dentry is returned. The name passed in is
* copied and the copy passed in may be reused after this call.
*/
struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
{
struct dentry *dentry = __d_alloc(parent->d_sb, name);
if (!dentry)
return NULL;
spin_lock(&parent->d_lock);
/*
* don't need child lock because it is not subject
* to concurrency here
*/
dentry->d_parent = dget_dlock(parent);
hlist_add_head(&dentry->d_sib, &parent->d_children);
spin_unlock(&parent->d_lock);
return dentry;
}
EXPORT_SYMBOL(d_alloc);
struct dentry *d_alloc_anon(struct super_block *sb)
{
return __d_alloc(sb, NULL);
}
EXPORT_SYMBOL(d_alloc_anon);
struct dentry *d_alloc_cursor(struct dentry * parent)
{
struct dentry *dentry = d_alloc_anon(parent->d_sb);
if (dentry) {
dentry->d_flags |= DCACHE_DENTRY_CURSOR;
dentry->d_parent = dget(parent);
}
return dentry;
}
/**
* d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
* @sb: the superblock
* @name: qstr of the name
*
* For a filesystem that just pins its dentries in memory and never
* performs lookups at all, return an unhashed IS_ROOT dentry.
* This is used for pipes, sockets et.al. - the stuff that should
* never be anyone's children or parents. Unlike all other
* dentries, these will not have RCU delay between dropping the
* last reference and freeing them.
*
* The only user is alloc_file_pseudo() and that's what should
* be considered a public interface. Don't use directly.
*/
struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
{
static const struct dentry_operations anon_ops = {
.d_dname = simple_dname
};
struct dentry *dentry = __d_alloc(sb, name);
if (likely(dentry)) {
dentry->d_flags |= DCACHE_NORCU;
if (!sb->s_d_op)
d_set_d_op(dentry, &anon_ops);
}
return dentry;
}
struct dentry *d_alloc_name(struct dentry *parent, const char *name)
{
struct qstr q;
q.name = name;
q.hash_len = hashlen_string(parent, name);
return d_alloc(parent, &q);
}
EXPORT_SYMBOL(d_alloc_name);
void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
{
WARN_ON_ONCE(dentry->d_op); WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
DCACHE_OP_COMPARE |
DCACHE_OP_REVALIDATE |
DCACHE_OP_WEAK_REVALIDATE |
DCACHE_OP_DELETE |
DCACHE_OP_REAL));
dentry->d_op = op;
if (!op)
return;
if (op->d_hash) dentry->d_flags |= DCACHE_OP_HASH; if (op->d_compare) dentry->d_flags |= DCACHE_OP_COMPARE; if (op->d_revalidate) dentry->d_flags |= DCACHE_OP_REVALIDATE; if (op->d_weak_revalidate) dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE; if (op->d_delete) dentry->d_flags |= DCACHE_OP_DELETE; if (op->d_prune) dentry->d_flags |= DCACHE_OP_PRUNE; if (op->d_real) dentry->d_flags |= DCACHE_OP_REAL;
}
EXPORT_SYMBOL(d_set_d_op);
static unsigned d_flags_for_inode(struct inode *inode)
{
unsigned add_flags = DCACHE_REGULAR_TYPE;
if (!inode)
return DCACHE_MISS_TYPE;
if (S_ISDIR(inode->i_mode)) {
add_flags = DCACHE_DIRECTORY_TYPE;
if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) { if (unlikely(!inode->i_op->lookup))
add_flags = DCACHE_AUTODIR_TYPE;
else
inode->i_opflags |= IOP_LOOKUP;
}
goto type_determined;
}
if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) { if (unlikely(inode->i_op->get_link)) {
add_flags = DCACHE_SYMLINK_TYPE;
goto type_determined;
}
inode->i_opflags |= IOP_NOFOLLOW;
}
if (unlikely(!S_ISREG(inode->i_mode)))
add_flags = DCACHE_SPECIAL_TYPE;
type_determined:
if (unlikely(IS_AUTOMOUNT(inode))) add_flags |= DCACHE_NEED_AUTOMOUNT;
return add_flags;
}
static void __d_instantiate(struct dentry *dentry, struct inode *inode)
{
unsigned add_flags = d_flags_for_inode(inode); WARN_ON(d_in_lookup(dentry)); spin_lock(&dentry->d_lock);
/*
* Decrement negative dentry count if it was in the LRU list.
*/
if (dentry->d_flags & DCACHE_LRU_LIST)
this_cpu_dec(nr_dentry_negative); hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
raw_write_seqcount_begin(&dentry->d_seq);
__d_set_inode_and_type(dentry, inode, add_flags);
raw_write_seqcount_end(&dentry->d_seq);
fsnotify_update_flags(dentry);
spin_unlock(&dentry->d_lock);
}
/**
* d_instantiate - fill in inode information for a dentry
* @entry: dentry to complete
* @inode: inode to attach to this dentry
*
* Fill in inode information in the entry.
*
* This turns negative dentries into productive full members
* of society.
*
* NOTE! This assumes that the inode count has been incremented
* (or otherwise set) by the caller to indicate that it is now
* in use by the dcache.
*/
void d_instantiate(struct dentry *entry, struct inode * inode)
{
BUG_ON(!hlist_unhashed(&entry->d_u.d_alias)); if (inode) { security_d_instantiate(entry, inode);
spin_lock(&inode->i_lock);
__d_instantiate(entry, inode);
spin_unlock(&inode->i_lock);
}
}
EXPORT_SYMBOL(d_instantiate);
/*
* This should be equivalent to d_instantiate() + unlock_new_inode(),
* with lockdep-related part of unlock_new_inode() done before
* anything else. Use that instead of open-coding d_instantiate()/
* unlock_new_inode() combinations.
*/
void d_instantiate_new(struct dentry *entry, struct inode *inode)
{
BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
BUG_ON(!inode);
lockdep_annotate_inode_mutex_key(inode);
security_d_instantiate(entry, inode);
spin_lock(&inode->i_lock);
__d_instantiate(entry, inode);
WARN_ON(!(inode->i_state & I_NEW));
inode->i_state &= ~I_NEW & ~I_CREATING;
smp_mb();
wake_up_bit(&inode->i_state, __I_NEW);
spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL(d_instantiate_new);
struct dentry *d_make_root(struct inode *root_inode)
{
struct dentry *res = NULL;
if (root_inode) {
res = d_alloc_anon(root_inode->i_sb);
if (res)
d_instantiate(res, root_inode);
else
iput(root_inode);
}
return res;
}
EXPORT_SYMBOL(d_make_root);
static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
{
struct super_block *sb;
struct dentry *new, *res;
if (!inode)
return ERR_PTR(-ESTALE);
if (IS_ERR(inode))
return ERR_CAST(inode);
sb = inode->i_sb;
res = d_find_any_alias(inode); /* existing alias? */
if (res)
goto out;
new = d_alloc_anon(sb);
if (!new) {
res = ERR_PTR(-ENOMEM);
goto out;
}
security_d_instantiate(new, inode);
spin_lock(&inode->i_lock);
res = __d_find_any_alias(inode); /* recheck under lock */
if (likely(!res)) { /* still no alias, attach a disconnected dentry */
unsigned add_flags = d_flags_for_inode(inode);
if (disconnected)
add_flags |= DCACHE_DISCONNECTED;
spin_lock(&new->d_lock);
__d_set_inode_and_type(new, inode, add_flags);
hlist_add_head(&new->d_u.d_alias, &inode->i_dentry);
if (!disconnected) {
hlist_bl_lock(&sb->s_roots);
hlist_bl_add_head(&new->d_hash, &sb->s_roots);
hlist_bl_unlock(&sb->s_roots);
}
spin_unlock(&new->d_lock);
spin_unlock(&inode->i_lock);
inode = NULL; /* consumed by new->d_inode */
res = new;
} else {
spin_unlock(&inode->i_lock);
dput(new);
}
out:
iput(inode);
return res;
}
/**
* d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
* @inode: inode to allocate the dentry for
*
* Obtain a dentry for an inode resulting from NFS filehandle conversion or
* similar open by handle operations. The returned dentry may be anonymous,
* or may have a full name (if the inode was already in the cache).
*
* When called on a directory inode, we must ensure that the inode only ever
* has one dentry. If a dentry is found, that is returned instead of
* allocating a new one.
*
* On successful return, the reference to the inode has been transferred
* to the dentry. In case of an error the reference on the inode is released.
* To make it easier to use in export operations a %NULL or IS_ERR inode may
* be passed in and the error will be propagated to the return value,
* with a %NULL @inode replaced by ERR_PTR(-ESTALE).
*/
struct dentry *d_obtain_alias(struct inode *inode)
{
return __d_obtain_alias(inode, true);
}
EXPORT_SYMBOL(d_obtain_alias);
/**
* d_obtain_root - find or allocate a dentry for a given inode
* @inode: inode to allocate the dentry for
*
* Obtain an IS_ROOT dentry for the root of a filesystem.
*
* We must ensure that directory inodes only ever have one dentry. If a
* dentry is found, that is returned instead of allocating a new one.
*
* On successful return, the reference to the inode has been transferred
* to the dentry. In case of an error the reference on the inode is
* released. A %NULL or IS_ERR inode may be passed in and will be the
* error will be propagate to the return value, with a %NULL @inode
* replaced by ERR_PTR(-ESTALE).
*/
struct dentry *d_obtain_root(struct inode *inode)
{
return __d_obtain_alias(inode, false);
}
EXPORT_SYMBOL(d_obtain_root);
/**
* d_add_ci - lookup or allocate new dentry with case-exact name
* @inode: the inode case-insensitive lookup has found
* @dentry: the negative dentry that was passed to the parent's lookup func
* @name: the case-exact name to be associated with the returned dentry
*
* This is to avoid filling the dcache with case-insensitive names to the
* same inode, only the actual correct case is stored in the dcache for
* case-insensitive filesystems.
*
* For a case-insensitive lookup match and if the case-exact dentry
* already exists in the dcache, use it and return it.
*
* If no entry exists with the exact case name, allocate new dentry with
* the exact case, and return the spliced entry.
*/
struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
struct qstr *name)
{
struct dentry *found, *res;
/*
* First check if a dentry matching the name already exists,
* if not go ahead and create it now.
*/
found = d_hash_and_lookup(dentry->d_parent, name);
if (found) {
iput(inode);
return found;
}
if (d_in_lookup(dentry)) {
found = d_alloc_parallel(dentry->d_parent, name,
dentry->d_wait);
if (IS_ERR(found) || !d_in_lookup(found)) {
iput(inode);
return found;
}
} else {
found = d_alloc(dentry->d_parent, name);
if (!found) {
iput(inode);
return ERR_PTR(-ENOMEM);
}
}
res = d_splice_alias(inode, found);
if (res) {
d_lookup_done(found);
dput(found);
return res;
}
return found;
}
EXPORT_SYMBOL(d_add_ci);
/**
* d_same_name - compare dentry name with case-exact name
* @parent: parent dentry
* @dentry: the negative dentry that was passed to the parent's lookup func
* @name: the case-exact name to be associated with the returned dentry
*
* Return: true if names are same, or false
*/
bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
const struct qstr *name)
{
if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) { if (dentry->d_name.len != name->len)
return false;
return dentry_cmp(dentry, name->name, name->len) == 0;
}
return parent->d_op->d_compare(dentry,
dentry->d_name.len, dentry->d_name.name,
name) == 0;
}
EXPORT_SYMBOL_GPL(d_same_name);
/*
* This is __d_lookup_rcu() when the parent dentry has
* DCACHE_OP_COMPARE, which makes things much nastier.
*/
static noinline struct dentry *__d_lookup_rcu_op_compare(
const struct dentry *parent,
const struct qstr *name,
unsigned *seqp)
{
u64 hashlen = name->hash_len;
struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
struct hlist_bl_node *node;
struct dentry *dentry;
hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
int tlen;
const char *tname;
unsigned seq;
seqretry:
seq = raw_seqcount_begin(&dentry->d_seq);
if (dentry->d_parent != parent)
continue;
if (d_unhashed(dentry))
continue;
if (dentry->d_name.hash != hashlen_hash(hashlen))
continue;
tlen = dentry->d_name.len;
tname = dentry->d_name.name;
/* we want a consistent (name,len) pair */
if (read_seqcount_retry(&dentry->d_seq, seq)) {
cpu_relax();
goto seqretry;
}
if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0)
continue;
*seqp = seq;
return dentry;
}
return NULL;
}
/**
* __d_lookup_rcu - search for a dentry (racy, store-free)
* @parent: parent dentry
* @name: qstr of name we wish to find
* @seqp: returns d_seq value at the point where the dentry was found
* Returns: dentry, or NULL
*
* __d_lookup_rcu is the dcache lookup function for rcu-walk name
* resolution (store-free path walking) design described in
* Documentation/filesystems/path-lookup.txt.
*
* This is not to be used outside core vfs.
*
* __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
* held, and rcu_read_lock held. The returned dentry must not be stored into
* without taking d_lock and checking d_seq sequence count against @seq
* returned here.
*
* A refcount may be taken on the found dentry with the d_rcu_to_refcount
* function.
*
* Alternatively, __d_lookup_rcu may be called again to look up the child of
* the returned dentry, so long as its parent's seqlock is checked after the
* child is looked up. Thus, an interlocking stepping of sequence lock checks
* is formed, giving integrity down the path walk.
*
* NOTE! The caller *has* to check the resulting dentry against the sequence
* number we've returned before using any of the resulting dentry state!
*/
struct dentry *__d_lookup_rcu(const struct dentry *parent,
const struct qstr *name,
unsigned *seqp)
{
u64 hashlen = name->hash_len;
const unsigned char *str = name->name;
struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
struct hlist_bl_node *node;
struct dentry *dentry;
/*
* Note: There is significant duplication with __d_lookup_rcu which is
* required to prevent single threaded performance regressions
* especially on architectures where smp_rmb (in seqcounts) are costly.
* Keep the two functions in sync.
*/
if (unlikely(parent->d_flags & DCACHE_OP_COMPARE))
return __d_lookup_rcu_op_compare(parent, name, seqp);
/*
* The hash list is protected using RCU.
*
* Carefully use d_seq when comparing a candidate dentry, to avoid
* races with d_move().
*
* It is possible that concurrent renames can mess up our list
* walk here and result in missing our dentry, resulting in the
* false-negative result. d_lookup() protects against concurrent
* renames using rename_lock seqlock.
*
* See Documentation/filesystems/path-lookup.txt for more details.
*/
hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
unsigned seq;
/*
* The dentry sequence count protects us from concurrent
* renames, and thus protects parent and name fields.
*
* The caller must perform a seqcount check in order
* to do anything useful with the returned dentry.
*
* NOTE! We do a "raw" seqcount_begin here. That means that
* we don't wait for the sequence count to stabilize if it
* is in the middle of a sequence change. If we do the slow
* dentry compare, we will do seqretries until it is stable,
* and if we end up with a successful lookup, we actually
* want to exit RCU lookup anyway.
*
* Note that raw_seqcount_begin still *does* smp_rmb(), so
* we are still guaranteed NUL-termination of ->d_name.name.
*/
seq = raw_seqcount_begin(&dentry->d_seq);
if (dentry->d_parent != parent)
continue;
if (d_unhashed(dentry))
continue;
if (dentry->d_name.hash_len != hashlen)
continue;
if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
continue;
*seqp = seq;
return dentry;
}
return NULL;
}
/**
* d_lookup - search for a dentry
* @parent: parent dentry
* @name: qstr of name we wish to find
* Returns: dentry, or NULL
*
* d_lookup searches the children of the parent dentry for the name in
* question. If the dentry is found its reference count is incremented and the
* dentry is returned. The caller must use dput to free the entry when it has
* finished using it. %NULL is returned if the dentry does not exist.
*/
struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
{
struct dentry *dentry;
unsigned seq;
do {
seq = read_seqbegin(&rename_lock);
dentry = __d_lookup(parent, name);
if (dentry)
break;
} while (read_seqretry(&rename_lock, seq)); return dentry;
}
EXPORT_SYMBOL(d_lookup);
/**
* __d_lookup - search for a dentry (racy)
* @parent: parent dentry
* @name: qstr of name we wish to find
* Returns: dentry, or NULL
*
* __d_lookup is like d_lookup, however it may (rarely) return a
* false-negative result due to unrelated rename activity.
*
* __d_lookup is slightly faster by avoiding rename_lock read seqlock,
* however it must be used carefully, eg. with a following d_lookup in
* the case of failure.
*
* __d_lookup callers must be commented.
*/
struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
{
unsigned int hash = name->hash; struct hlist_bl_head *b = d_hash(hash);
struct hlist_bl_node *node;
struct dentry *found = NULL;
struct dentry *dentry;
/*
* Note: There is significant duplication with __d_lookup_rcu which is
* required to prevent single threaded performance regressions
* especially on architectures where smp_rmb (in seqcounts) are costly.
* Keep the two functions in sync.
*/
/*
* The hash list is protected using RCU.
*
* Take d_lock when comparing a candidate dentry, to avoid races
* with d_move().
*
* It is possible that concurrent renames can mess up our list
* walk here and result in missing our dentry, resulting in the
* false-negative result. d_lookup() protects against concurrent
* renames using rename_lock seqlock.
*
* See Documentation/filesystems/path-lookup.txt for more details.
*/
rcu_read_lock(); hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { if (dentry->d_name.hash != hash)
continue;
spin_lock(&dentry->d_lock);
if (dentry->d_parent != parent)
goto next;
if (d_unhashed(dentry))
goto next;
if (!d_same_name(dentry, parent, name))
goto next;
dentry->d_lockref.count++;
found = dentry;
spin_unlock(&dentry->d_lock);
break;
next:
spin_unlock(&dentry->d_lock);
}
rcu_read_unlock();
return found;
}
/**
* d_hash_and_lookup - hash the qstr then search for a dentry
* @dir: Directory to search in
* @name: qstr of name we wish to find
*
* On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
*/
struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
{
/*
* Check for a fs-specific hash function. Note that we must
* calculate the standard hash first, as the d_op->d_hash()
* routine may choose to leave the hash value unchanged.
*/
name->hash = full_name_hash(dir, name->name, name->len);
if (dir->d_flags & DCACHE_OP_HASH) {
int err = dir->d_op->d_hash(dir, name);
if (unlikely(err < 0))
return ERR_PTR(err);
}
return d_lookup(dir, name);
}
EXPORT_SYMBOL(d_hash_and_lookup);
/*
* When a file is deleted, we have two options:
* - turn this dentry into a negative dentry
* - unhash this dentry and free it.
*
* Usually, we want to just turn this into
* a negative dentry, but if anybody else is
* currently using the dentry or the inode
* we can't do that and we fall back on removing
* it from the hash queues and waiting for
* it to be deleted later when it has no users
*/
/**
* d_delete - delete a dentry
* @dentry: The dentry to delete
*
* Turn the dentry into a negative dentry if possible, otherwise
* remove it from the hash queues so it can be deleted later
*/
void d_delete(struct dentry * dentry)
{
struct inode *inode = dentry->d_inode;
spin_lock(&inode->i_lock);
spin_lock(&dentry->d_lock);
/*
* Are we the only user?
*/
if (dentry->d_lockref.count == 1) {
dentry->d_flags &= ~DCACHE_CANT_MOUNT;
dentry_unlink_inode(dentry);
} else {
__d_drop(dentry);
spin_unlock(&dentry->d_lock);
spin_unlock(&inode->i_lock);
}
}
EXPORT_SYMBOL(d_delete);
static void __d_rehash(struct dentry *entry)
{
struct hlist_bl_head *b = d_hash(entry->d_name.hash); hlist_bl_lock(b); hlist_bl_add_head_rcu(&entry->d_hash, b); hlist_bl_unlock(b);
}
/**
* d_rehash - add an entry back to the hash
* @entry: dentry to add to the hash
*
* Adds a dentry to the hash according to its name.
*/
void d_rehash(struct dentry * entry)
{
spin_lock(&entry->d_lock);
__d_rehash(entry);
spin_unlock(&entry->d_lock);
}
EXPORT_SYMBOL(d_rehash);
static inline unsigned start_dir_add(struct inode *dir)
{
preempt_disable_nested();
for (;;) {
unsigned n = dir->i_dir_seq; if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
return n;
cpu_relax();
}
}
static inline void end_dir_add(struct inode *dir, unsigned int n,
wait_queue_head_t *d_wait)
{
smp_store_release(&dir->i_dir_seq, n + 2);
preempt_enable_nested();
wake_up_all(d_wait);
}
static void d_wait_lookup(struct dentry *dentry)
{
if (d_in_lookup(dentry)) {
DECLARE_WAITQUEUE(wait, current);
add_wait_queue(dentry->d_wait, &wait);
do {
set_current_state(TASK_UNINTERRUPTIBLE);
spin_unlock(&dentry->d_lock);
schedule();
spin_lock(&dentry->d_lock);
} while (d_in_lookup(dentry));
}
}
struct dentry *d_alloc_parallel(struct dentry *parent,
const struct qstr *name,
wait_queue_head_t *wq)
{
unsigned int hash = name->hash;
struct hlist_bl_head *b = in_lookup_hash(parent, hash);
struct hlist_bl_node *node;
struct dentry *new = d_alloc(parent, name);
struct dentry *dentry;
unsigned seq, r_seq, d_seq;
if (unlikely(!new))
return ERR_PTR(-ENOMEM);
retry:
rcu_read_lock(); seq = smp_load_acquire(&parent->d_inode->i_dir_seq); r_seq = read_seqbegin(&rename_lock);
dentry = __d_lookup_rcu(parent, name, &d_seq);
if (unlikely(dentry)) {
if (!lockref_get_not_dead(&dentry->d_lockref)) {
rcu_read_unlock();
goto retry;
}
if (read_seqcount_retry(&dentry->d_seq, d_seq)) { rcu_read_unlock();
dput(dentry);
goto retry;
}
rcu_read_unlock(); dput(new);
return dentry;
}
if (unlikely(read_seqretry(&rename_lock, r_seq))) {
rcu_read_unlock();
goto retry;
}
if (unlikely(seq & 1)) {
rcu_read_unlock();
goto retry;
}
hlist_bl_lock(b); if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) { hlist_bl_unlock(b);
rcu_read_unlock();
goto retry;
}
/*
* No changes for the parent since the beginning of d_lookup().
* Since all removals from the chain happen with hlist_bl_lock(),
* any potential in-lookup matches are going to stay here until
* we unlock the chain. All fields are stable in everything
* we encounter.
*/
hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) { if (dentry->d_name.hash != hash)
continue;
if (dentry->d_parent != parent)
continue;
if (!d_same_name(dentry, parent, name))
continue;
hlist_bl_unlock(b);
/* now we can try to grab a reference */
if (!lockref_get_not_dead(&dentry->d_lockref)) { rcu_read_unlock();
goto retry;
}
rcu_read_unlock();
/*
* somebody is likely to be still doing lookup for it;
* wait for them to finish
*/
spin_lock(&dentry->d_lock);
d_wait_lookup(dentry);
/*
* it's not in-lookup anymore; in principle we should repeat
* everything from dcache lookup, but it's likely to be what
* d_lookup() would've found anyway. If it is, just return it;
* otherwise we really have to repeat the whole thing.
*/
if (unlikely(dentry->d_name.hash != hash))
goto mismatch;
if (unlikely(dentry->d_parent != parent))
goto mismatch;
if (unlikely(d_unhashed(dentry)))
goto mismatch;
if (unlikely(!d_same_name(dentry, parent, name)))
goto mismatch;
/* OK, it *is* a hashed match; return it */
spin_unlock(&dentry->d_lock);
dput(new);
return dentry;
}
rcu_read_unlock();
/* we can't take ->d_lock here; it's OK, though. */
new->d_flags |= DCACHE_PAR_LOOKUP;
new->d_wait = wq;
hlist_bl_add_head(&new->d_u.d_in_lookup_hash, b); hlist_bl_unlock(b);
return new;
mismatch:
spin_unlock(&dentry->d_lock); dput(dentry);
goto retry;
}
EXPORT_SYMBOL(d_alloc_parallel);
/*
* - Unhash the dentry
* - Retrieve and clear the waitqueue head in dentry
* - Return the waitqueue head
*/
static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry)
{
wait_queue_head_t *d_wait;
struct hlist_bl_head *b;
lockdep_assert_held(&dentry->d_lock); b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash); hlist_bl_lock(b); dentry->d_flags &= ~DCACHE_PAR_LOOKUP; __hlist_bl_del(&dentry->d_u.d_in_lookup_hash); d_wait = dentry->d_wait;
dentry->d_wait = NULL;
hlist_bl_unlock(b); INIT_HLIST_NODE(&dentry->d_u.d_alias);
INIT_LIST_HEAD(&dentry->d_lru);
return d_wait;
}
void __d_lookup_unhash_wake(struct dentry *dentry)
{
spin_lock(&dentry->d_lock);
wake_up_all(__d_lookup_unhash(dentry));
spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(__d_lookup_unhash_wake);
/* inode->i_lock held if inode is non-NULL */
static inline void __d_add(struct dentry *dentry, struct inode *inode)
{
wait_queue_head_t *d_wait;
struct inode *dir = NULL;
unsigned n;
spin_lock(&dentry->d_lock);
if (unlikely(d_in_lookup(dentry))) {
dir = dentry->d_parent->d_inode; n = start_dir_add(dir); d_wait = __d_lookup_unhash(dentry);
}
if (inode) {
unsigned add_flags = d_flags_for_inode(inode); hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
raw_write_seqcount_begin(&dentry->d_seq);
__d_set_inode_and_type(dentry, inode, add_flags);
raw_write_seqcount_end(&dentry->d_seq);
fsnotify_update_flags(dentry);
}
__d_rehash(dentry);
if (dir)
end_dir_add(dir, n, d_wait); spin_unlock(&dentry->d_lock);
if (inode)
spin_unlock(&inode->i_lock);
}
/**
* d_add - add dentry to hash queues
* @entry: dentry to add
* @inode: The inode to attach to this dentry
*
* This adds the entry to the hash queues and initializes @inode.
* The entry was actually filled in earlier during d_alloc().
*/
void d_add(struct dentry *entry, struct inode *inode)
{
if (inode) {
security_d_instantiate(entry, inode);
spin_lock(&inode->i_lock);
}
__d_add(entry, inode);
}
EXPORT_SYMBOL(d_add);
/**
* d_exact_alias - find and hash an exact unhashed alias
* @entry: dentry to add
* @inode: The inode to go with this dentry
*
* If an unhashed dentry with the same name/parent and desired
* inode already exists, hash and return it. Otherwise, return
* NULL.
*
* Parent directory should be locked.
*/
struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
{
struct dentry *alias;
unsigned int hash = entry->d_name.hash;
spin_lock(&inode->i_lock);
hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
/*
* Don't need alias->d_lock here, because aliases with
* d_parent == entry->d_parent are not subject to name or
* parent changes, because the parent inode i_mutex is held.
*/
if (alias->d_name.hash != hash)
continue;
if (alias->d_parent != entry->d_parent)
continue;
if (!d_same_name(alias, entry->d_parent, &entry->d_name))
continue;
spin_lock(&alias->d_lock);
if (!d_unhashed(alias)) {
spin_unlock(&alias->d_lock);
alias = NULL;
} else {
dget_dlock(alias);
__d_rehash(alias);
spin_unlock(&alias->d_lock);
}
spin_unlock(&inode->i_lock);
return alias;
}
spin_unlock(&inode->i_lock);
return NULL;
}
EXPORT_SYMBOL(d_exact_alias);
static void swap_names(struct dentry *dentry, struct dentry *target)
{
if (unlikely(dname_external(target))) {
if (unlikely(dname_external(dentry))) {
/*
* Both external: swap the pointers
*/
swap(target->d_name.name, dentry->d_name.name);
} else {
/*
* dentry:internal, target:external. Steal target's
* storage and make target internal.
*/
memcpy(target->d_iname, dentry->d_name.name,
dentry->d_name.len + 1);
dentry->d_name.name = target->d_name.name;
target->d_name.name = target->d_iname;
}
} else {
if (unlikely(dname_external(dentry))) {
/*
* dentry:external, target:internal. Give dentry's
* storage to target and make dentry internal
*/
memcpy(dentry->d_iname, target->d_name.name,
target->d_name.len + 1);
target->d_name.name = dentry->d_name.name;
dentry->d_name.name = dentry->d_iname;
} else {
/*
* Both are internal.
*/
unsigned int i;
BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
swap(((long *) &dentry->d_iname)[i],
((long *) &target->d_iname)[i]);
}
}
}
swap(dentry->d_name.hash_len, target->d_name.hash_len);
}
static void copy_name(struct dentry *dentry, struct dentry *target)
{
struct external_name *old_name = NULL;
if (unlikely(dname_external(dentry)))
old_name = external_name(dentry);
if (unlikely(dname_external(target))) {
atomic_inc(&external_name(target)->u.count);
dentry->d_name = target->d_name;
} else {
memcpy(dentry->d_iname, target->d_name.name,
target->d_name.len + 1);
dentry->d_name.name = dentry->d_iname;
dentry->d_name.hash_len = target->d_name.hash_len;
}
if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
kfree_rcu(old_name, u.head);
}
/*
* __d_move - move a dentry
* @dentry: entry to move
* @target: new dentry
* @exchange: exchange the two dentries
*
* Update the dcache to reflect the move of a file name. Negative
* dcache entries should not be moved in this way. Caller must hold
* rename_lock, the i_mutex of the source and target directories,
* and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
*/
static void __d_move(struct dentry *dentry, struct dentry *target,
bool exchange)
{
struct dentry *old_parent, *p;
wait_queue_head_t *d_wait;
struct inode *dir = NULL;
unsigned n;
WARN_ON(!dentry->d_inode);
if (WARN_ON(dentry == target))
return;
BUG_ON(d_ancestor(target, dentry));
old_parent = dentry->d_parent;
p = d_ancestor(old_parent, target);
if (IS_ROOT(dentry)) {
BUG_ON(p);
spin_lock(&target->d_parent->d_lock);
} else if (!p) {
/* target is not a descendent of dentry->d_parent */
spin_lock(&target->d_parent->d_lock);
spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
} else {
BUG_ON(p == dentry);
spin_lock(&old_parent->d_lock);
if (p != target)
spin_lock_nested(&target->d_parent->d_lock,
DENTRY_D_LOCK_NESTED);
}
spin_lock_nested(&dentry->d_lock, 2);
spin_lock_nested(&target->d_lock, 3);
if (unlikely(d_in_lookup(target))) {
dir = target->d_parent->d_inode;
n = start_dir_add(dir);
d_wait = __d_lookup_unhash(target);
}
write_seqcount_begin(&dentry->d_seq);
write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
/* unhash both */
if (!d_unhashed(dentry))
___d_drop(dentry);
if (!d_unhashed(target))
___d_drop(target);
/* ... and switch them in the tree */
dentry->d_parent = target->d_parent;
if (!exchange) {
copy_name(dentry, target);
target->d_hash.pprev = NULL;
dentry->d_parent->d_lockref.count++;
if (dentry != old_parent) /* wasn't IS_ROOT */
WARN_ON(!--old_parent->d_lockref.count);
} else {
target->d_parent = old_parent;
swap_names(dentry, target);
if (!hlist_unhashed(&target->d_sib))
__hlist_del(&target->d_sib);
hlist_add_head(&target->d_sib, &target->d_parent->d_children);
__d_rehash(target);
fsnotify_update_flags(target);
}
if (!hlist_unhashed(&dentry->d_sib))
__hlist_del(&dentry->d_sib);
hlist_add_head(&dentry->d_sib, &dentry->d_parent->d_children);
__d_rehash(dentry);
fsnotify_update_flags(dentry);
fscrypt_handle_d_move(dentry);
write_seqcount_end(&target->d_seq);
write_seqcount_end(&dentry->d_seq);
if (dir)
end_dir_add(dir, n, d_wait);
if (dentry->d_parent != old_parent)
spin_unlock(&dentry->d_parent->d_lock);
if (dentry != old_parent)
spin_unlock(&old_parent->d_lock);
spin_unlock(&target->d_lock);
spin_unlock(&dentry->d_lock);
}
/*
* d_move - move a dentry
* @dentry: entry to move
* @target: new dentry
*
* Update the dcache to reflect the move of a file name. Negative
* dcache entries should not be moved in this way. See the locking
* requirements for __d_move.
*/
void d_move(struct dentry *dentry, struct dentry *target)
{
write_seqlock(&rename_lock);
__d_move(dentry, target, false);
write_sequnlock(&rename_lock);
}
EXPORT_SYMBOL(d_move);
/*
* d_exchange - exchange two dentries
* @dentry1: first dentry
* @dentry2: second dentry
*/
void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
{
write_seqlock(&rename_lock);
WARN_ON(!dentry1->d_inode);
WARN_ON(!dentry2->d_inode);
WARN_ON(IS_ROOT(dentry1));
WARN_ON(IS_ROOT(dentry2));
__d_move(dentry1, dentry2, true);
write_sequnlock(&rename_lock);
}
/**
* d_ancestor - search for an ancestor
* @p1: ancestor dentry
* @p2: child dentry
*
* Returns the ancestor dentry of p2 which is a child of p1, if p1 is
* an ancestor of p2, else NULL.
*/
struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
{
struct dentry *p;
for (p = p2; !IS_ROOT(p); p = p->d_parent) {
if (p->d_parent == p1)
return p;
}
return NULL;
}
/*
* This helper attempts to cope with remotely renamed directories
*
* It assumes that the caller is already holding
* dentry->d_parent->d_inode->i_mutex, and rename_lock
*
* Note: If ever the locking in lock_rename() changes, then please
* remember to update this too...
*/
static int __d_unalias(struct dentry *dentry, struct dentry *alias)
{
struct mutex *m1 = NULL;
struct rw_semaphore *m2 = NULL;
int ret = -ESTALE;
/* If alias and dentry share a parent, then no extra locks required */
if (alias->d_parent == dentry->d_parent)
goto out_unalias;
/* See lock_rename() */
if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
goto out_err;
m1 = &dentry->d_sb->s_vfs_rename_mutex;
if (!inode_trylock_shared(alias->d_parent->d_inode))
goto out_err;
m2 = &alias->d_parent->d_inode->i_rwsem;
out_unalias:
__d_move(alias, dentry, false);
ret = 0;
out_err:
if (m2)
up_read(m2);
if (m1)
mutex_unlock(m1);
return ret;
}
/**
* d_splice_alias - splice a disconnected dentry into the tree if one exists
* @inode: the inode which may have a disconnected dentry
* @dentry: a negative dentry which we want to point to the inode.
*
* If inode is a directory and has an IS_ROOT alias, then d_move that in
* place of the given dentry and return it, else simply d_add the inode
* to the dentry and return NULL.
*
* If a non-IS_ROOT directory is found, the filesystem is corrupt, and
* we should error out: directories can't have multiple aliases.
*
* This is needed in the lookup routine of any filesystem that is exportable
* (via knfsd) so that we can build dcache paths to directories effectively.
*
* If a dentry was found and moved, then it is returned. Otherwise NULL
* is returned. This matches the expected return value of ->lookup.
*
* Cluster filesystems may call this function with a negative, hashed dentry.
* In that case, we know that the inode will be a regular file, and also this
* will only occur during atomic_open. So we need to check for the dentry
* being already hashed only in the final case.
*/
struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
{
if (IS_ERR(inode))
return ERR_CAST(inode);
BUG_ON(!d_unhashed(dentry));
if (!inode)
goto out;
security_d_instantiate(dentry, inode);
spin_lock(&inode->i_lock);
if (S_ISDIR(inode->i_mode)) {
struct dentry *new = __d_find_any_alias(inode);
if (unlikely(new)) {
/* The reference to new ensures it remains an alias */
spin_unlock(&inode->i_lock);
write_seqlock(&rename_lock);
if (unlikely(d_ancestor(new, dentry))) {
write_sequnlock(&rename_lock);
dput(new);
new = ERR_PTR(-ELOOP);
pr_warn_ratelimited(
"VFS: Lookup of '%s' in %s %s"
" would have caused loop\n",
dentry->d_name.name,
inode->i_sb->s_type->name,
inode->i_sb->s_id);
} else if (!IS_ROOT(new)) {
struct dentry *old_parent = dget(new->d_parent);
int err = __d_unalias(dentry, new);
write_sequnlock(&rename_lock);
if (err) {
dput(new);
new = ERR_PTR(err);
}
dput(old_parent);
} else {
__d_move(new, dentry, false);
write_sequnlock(&rename_lock);
}
iput(inode);
return new;
}
}
out:
__d_add(dentry, inode);
return NULL;
}
EXPORT_SYMBOL(d_splice_alias);
/*
* Test whether new_dentry is a subdirectory of old_dentry.
*
* Trivially implemented using the dcache structure
*/
/**
* is_subdir - is new dentry a subdirectory of old_dentry
* @new_dentry: new dentry
* @old_dentry: old dentry
*
* Returns true if new_dentry is a subdirectory of the parent (at any depth).
* Returns false otherwise.
* Caller must ensure that "new_dentry" is pinned before calling is_subdir()
*/
bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
{
bool result;
unsigned seq;
if (new_dentry == old_dentry)
return true;
do {
/* for restarting inner loop in case of seq retry */
seq = read_seqbegin(&rename_lock);
/*
* Need rcu_readlock to protect against the d_parent trashing
* due to d_move
*/
rcu_read_lock();
if (d_ancestor(old_dentry, new_dentry))
result = true;
else
result = false;
rcu_read_unlock();
} while (read_seqretry(&rename_lock, seq));
return result;
}
EXPORT_SYMBOL(is_subdir);
static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
{
struct dentry *root = data;
if (dentry != root) {
if (d_unhashed(dentry) || !dentry->d_inode)
return D_WALK_SKIP;
if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
dentry->d_flags |= DCACHE_GENOCIDE;
dentry->d_lockref.count--;
}
}
return D_WALK_CONTINUE;
}
void d_genocide(struct dentry *parent)
{
d_walk(parent, parent, d_genocide_kill);
}
void d_mark_tmpfile(struct file *file, struct inode *inode)
{
struct dentry *dentry = file->f_path.dentry;
BUG_ON(dentry->d_name.name != dentry->d_iname ||
!hlist_unhashed(&dentry->d_u.d_alias) ||
!d_unlinked(dentry));
spin_lock(&dentry->d_parent->d_lock);
spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
(unsigned long long)inode->i_ino);
spin_unlock(&dentry->d_lock);
spin_unlock(&dentry->d_parent->d_lock);
}
EXPORT_SYMBOL(d_mark_tmpfile);
void d_tmpfile(struct file *file, struct inode *inode)
{
struct dentry *dentry = file->f_path.dentry;
inode_dec_link_count(inode);
d_mark_tmpfile(file, inode);
d_instantiate(dentry, inode);
}
EXPORT_SYMBOL(d_tmpfile);
static __initdata unsigned long dhash_entries;
static int __init set_dhash_entries(char *str)
{
if (!str)
return 0;
dhash_entries = simple_strtoul(str, &str, 0);
return 1;
}
__setup("dhash_entries=", set_dhash_entries);
static void __init dcache_init_early(void)
{
/* If hashes are distributed across NUMA nodes, defer
* hash allocation until vmalloc space is available.
*/
if (hashdist)
return;
dentry_hashtable =
alloc_large_system_hash("Dentry cache",
sizeof(struct hlist_bl_head),
dhash_entries,
13,
HASH_EARLY | HASH_ZERO,
&d_hash_shift,
NULL,
0,
0);
d_hash_shift = 32 - d_hash_shift;
}
static void __init dcache_init(void)
{
/*
* A constructor could be added for stable state like the lists,
* but it is probably not worth it because of the cache nature
* of the dcache.
*/
dentry_cache = KMEM_CACHE_USERCOPY(dentry,
SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT,
d_iname);
/* Hash may have been set up in dcache_init_early */
if (!hashdist)
return;
dentry_hashtable =
alloc_large_system_hash("Dentry cache",
sizeof(struct hlist_bl_head),
dhash_entries,
13,
HASH_ZERO,
&d_hash_shift,
NULL,
0,
0);
d_hash_shift = 32 - d_hash_shift;
}
/* SLAB cache for __getname() consumers */
struct kmem_cache *names_cachep __ro_after_init;
EXPORT_SYMBOL(names_cachep);
void __init vfs_caches_init_early(void)
{
int i;
for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
dcache_init_early();
inode_init_early();
}
void __init vfs_caches_init(void)
{
names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
dcache_init();
inode_init();
files_init();
files_maxfiles_init();
mnt_init();
bdev_cache_init();
chrdev_init();
}